Merge branch 'cassandra-3.11' into trunk
diff --git a/.circleci/config-2_1.yml b/.circleci/config-2_1.yml
index 2485c16..ab62124 100644
--- a/.circleci/config-2_1.yml
+++ b/.circleci/config-2_1.yml
@@ -1,7 +1,6 @@
version: 2.1
default_env_vars: &default_env_vars
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
ANT_HOME: /usr/share/ant
LANG: en_US.UTF-8
KEEP_TEST_DIR: true
@@ -41,81 +40,235 @@
#exec_resource_class: xlarge
parallelism: 1 # sequential, single container tests: no parallelism benefits
-with_dtests_jobs: &with_dtest_jobs
- jobs:
- - build
- # Java 8 unit tests will be run automatically
- - j8_unit_tests:
- requires:
- - build
- - j8_jvm_dtests:
- requires:
- - build
- # specialized unit tests (all run on request using Java 8)
- - start_utests_long:
- type: approval
- requires:
- - build
- - utests_long:
- requires:
- - start_utests_long
- - start_utests_compression:
- type: approval
- requires:
- - build
- - utests_compression:
- requires:
- - start_utests_compression
- - start_utests_stress:
- type: approval
- requires:
- - build
- - utests_stress:
- requires:
- - start_utests_stress
- - start_jvm_upgrade_dtest:
- type: approval
- - j8_dtest_jars_build:
- requires:
- - build
- - start_jvm_upgrade_dtest
- - j8_jvm_upgrade_dtests:
- requires:
- - j8_dtest_jars_build
- # Java 8 dtests (on request)
- - start_j8_dtests:
- type: approval
- requires:
- - build
- - j8_dtests-with-vnodes:
- requires:
- - start_j8_dtests
- - j8_dtests-no-vnodes:
- requires:
- - start_j8_dtests
- # Java 8 upgrade tests
- - start_upgrade_tests:
- type: approval
- requires:
- - build
- - j8_upgradetests-no-vnodes:
- requires:
- - start_upgrade_tests
+j11_par_executor: &j11_par_executor
+ executor:
+ name: java11-executor
+ #exec_resource_class: xlarge
+ parallelism: 4
-with_dtest_jobs_only: &with_dtest_jobs_only
+j11_small_par_executor: &j11_small_par_executor
+ executor:
+ name: java11-executor
+ #exec_resource_class: xlarge
+ parallelism: 1
+
+j8_with_dtests_jobs: &j8_with_dtests_jobs
+ jobs:
+ - j8_build
+ # Java 8 unit tests will be run automatically
+ - j8_unit_tests:
+ requires:
+ - j8_build
+ - j8_jvm_dtests:
+ requires:
+ - j8_build
+ # Java 11 unit tests (on request, currently not working)
+ - start_j11_unit_tests:
+ type: approval
+ requires:
+ - j8_build
+ - j11_unit_tests:
+ requires:
+ - start_j11_unit_tests
+ # specialized unit tests (all run on request using Java 8)
+ - start_utests_long:
+ type: approval
+ requires:
+ - j8_build
+ - utests_long:
+ requires:
+ - start_utests_long
+ - start_utests_compression:
+ type: approval
+ requires:
+ - j8_build
+ - utests_compression:
+ requires:
+ - start_utests_compression
+ - start_utests_stress:
+ type: approval
+ requires:
+ - j8_build
+ - utests_stress:
+ requires:
+ - start_utests_stress
+ - start_utests_fqltool:
+ type: approval
+ requires:
+ - j8_build
+ - utests_fqltool:
+ requires:
+ - start_utests_fqltool
+ - start_jvm_upgrade_dtest:
+ type: approval
+ - j8_dtest_jars_build:
+ requires:
+ - j8_build
+ - start_jvm_upgrade_dtest
+ - j8_jvm_upgrade_dtests:
+ requires:
+ - j8_dtest_jars_build
+ # Java 8 dtests (on request)
+ - start_j8_dtests:
+ type: approval
+ requires:
+ - j8_build
+ - j8_dtests-with-vnodes:
+ requires:
+ - start_j8_dtests
+ - j8_dtests-no-vnodes:
+ requires:
+ - start_j8_dtests
+ # Java 11 dtests (on request)
+ - start_j11_dtests:
+ type: approval
+ requires:
+ - j8_build
+ - j11_dtests-with-vnodes:
+ requires:
+ - start_j11_dtests
+ - j11_dtests-no-vnodes:
+ requires:
+ - start_j11_dtests
+ # Java 8 upgrade tests
+ - start_upgrade_tests:
+ type: approval
+ requires:
+ - j8_build
+ - j8_upgradetests-no-vnodes:
+ requires:
+ - start_upgrade_tests
+ - start_j8_cqlsh_tests-with-vnodes:
+ type: approval
+ requires:
+ - j8_build
+ - j8_cqlsh-dtests-py2-with-vnodes:
+ requires:
+ - start_j8_cqlsh_tests-with-vnodes
+ - j8_cqlsh-dtests-py3-with-vnodes:
+ requires:
+ - start_j8_cqlsh_tests-with-vnodes
+ - j8_cqlsh-dtests-py38-with-vnodes:
+ requires:
+ - start_j8_cqlsh_tests-with-vnodes
+ - start_j8_cqlsh_tests-no-vnodes:
+ type: approval
+ requires:
+ - j8_build
+ - j8_cqlsh-dtests-py2-no-vnodes:
+ requires:
+ - start_j8_cqlsh_tests-no-vnodes
+ - j8_cqlsh-dtests-py3-no-vnodes:
+ requires:
+ - start_j8_cqlsh_tests-no-vnodes
+ - j8_cqlsh-dtests-py38-no-vnodes:
+ requires:
+ - start_j8_cqlsh_tests-no-vnodes
+ - start_j11_cqlsh_tests-with-vnodes:
+ type: approval
+ requires:
+ - j8_build
+ - j11_cqlsh-dtests-py2-with-vnodes:
+ requires:
+ - start_j11_cqlsh_tests-with-vnodes
+ - j11_cqlsh-dtests-py3-with-vnodes:
+ requires:
+ - start_j11_cqlsh_tests-with-vnodes
+ - j11_cqlsh-dtests-py38-with-vnodes:
+ requires:
+ - start_j11_cqlsh_tests-with-vnodes
+ - start_j11_cqlsh_tests-no-vnodes:
+ type: approval
+ requires:
+ - j8_build
+ - j11_cqlsh-dtests-py2-no-vnodes:
+ requires:
+ - start_j11_cqlsh_tests-no-vnodes
+ - j11_cqlsh-dtests-py3-no-vnodes:
+ requires:
+ - start_j11_cqlsh_tests-no-vnodes
+ - j11_cqlsh-dtests-py38-no-vnodes:
+ requires:
+ - start_j11_cqlsh_tests-no-vnodes
+
+j11_with_dtests_jobs: &j11_with_dtests_jobs
+ jobs:
+ - j11_build
+ # Java 11 unit tests (on request, currently not working)
+ - start_j11_unit_tests:
+ type: approval
+ requires:
+ - j11_build
+ - j11_unit_tests:
+ requires:
+ - start_j11_unit_tests
+ - j11_jvm_dtests:
+ requires:
+ - j11_build
+ # Java 11 dtests (on request)
+ - start_j11_dtests:
+ type: approval
+ requires:
+ - j11_build
+ - j11_dtests-with-vnodes:
+ requires:
+ - start_j11_dtests
+ - j11_dtests-no-vnodes:
+ requires:
+ - start_j11_dtests
+ - start_j11_cqlsh_tests-with-vnodes:
+ type: approval
+ requires:
+ - j11_build
+ - j11_cqlsh-dtests-py2-with-vnodes:
+ requires:
+ - start_j11_cqlsh_tests-with-vnodes
+ - j11_cqlsh-dtests-py3-with-vnodes:
+ requires:
+ - start_j11_cqlsh_tests-with-vnodes
+ - j11_cqlsh-dtests-py38-with-vnodes:
+ requires:
+ - start_j11_cqlsh_tests-with-vnodes
+ - start_j11_cqlsh_tests-no-vnodes:
+ type: approval
+ requires:
+ - j11_build
+ - j11_cqlsh-dtests-py2-no-vnodes:
+ requires:
+ - start_j11_cqlsh_tests-no-vnodes
+ - j11_cqlsh-dtests-py3-no-vnodes:
+ requires:
+ - start_j11_cqlsh_tests-no-vnodes
+ - j11_cqlsh-dtests-py38-no-vnodes:
+ requires:
+ - start_j11_cqlsh_tests-no-vnodes
+
+j8_with_dtest_jobs_only: &j8_with_dtest_jobs_only
jobs:
- - build
+ - j8_build
- j8_dtests-with-vnodes:
requires:
- - build
+ - j8_build
- j8_dtests-no-vnodes:
requires:
- - build
+ - j8_build
+
+j11_with_dtest_jobs_only: &j11_with_dtest_jobs_only
+ jobs:
+ - build
+ - j11-with-vnodes:
+ requires:
+ - j11_build
+ - j11_dtests-no-vnodes:
+ requires:
+ - j11_build
workflows:
version: 2
- build_and_run_tests: *with_dtest_jobs
- #build_and_run_tests: *with_dtest_jobs_only
+ java8_build_and_run_tests: *j8_with_dtests_jobs
+# java8_build_and_run_tests: *j8_with_dtest_jobs_only
+ java11_build_and_run_tests: *j11_with_dtests_jobs
+# java11_build_and_run_tests: *j11_with_dtest_jobs_only
executors:
java8-executor:
@@ -124,7 +277,7 @@
type: string
default: medium
docker:
- - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
+ - image: nastra/cassandra-testing-ubuntu1910-java11-w-dependencies:20200603
resource_class: << parameters.exec_resource_class >>
working_directory: ~/
shell: /bin/bash -eo pipefail -l
@@ -133,20 +286,43 @@
JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ java11-executor:
+ parameters:
+ exec_resource_class:
+ type: string
+ default: medium
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11:20200603
+ resource_class: << parameters.exec_resource_class >>
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ environment:
+ <<: *default_env_vars
+ JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ CASSANDRA_USE_JDK11: true
+
+build_common: &build_common
+ parallelism: 1 # This job doesn't benefit from parallelism
+ steps:
+ - log_environment
+ - clone_cassandra
+ - build_cassandra
+ - run_eclipse_warnings
+ - persist_to_workspace:
+ root: /home/cassandra
+ paths:
+ - cassandra
+ - .m2
+
jobs:
- build:
+ j8_build:
executor: java8-executor
- parallelism: 1 # This job doesn't benefit from parallelism
- steps:
- - log_environment
- - clone_cassandra
- - build_cassandra
- - run_eclipse_warnings
- - persist_to_workspace:
- root: /home/cassandra
- paths:
- - cassandra
- - .m2
+ <<: *build_common
+
+ j11_build:
+ executor: java11-executor
+ <<: *build_common
j8_dtest_jars_build:
executor: java8-executor
@@ -181,6 +357,18 @@
- run_parallel_junit_tests:
classlistprefix: distributed
+ j11_jvm_dtests:
+ <<: *j11_small_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - create_junit_containers:
+ classlistprefix: distributed
+ extra_filters: "| grep -v upgrade"
+ - log_environment
+ - run_parallel_junit_tests:
+ classlistprefix: distributed
+
j8_jvm_upgrade_dtests:
<<: *j8_medium_par_executor
steps:
@@ -193,6 +381,15 @@
- run_parallel_junit_tests:
classlistprefix: distributed
+ j11_unit_tests:
+ <<: *j11_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - create_junit_containers
+ - log_environment
+ - run_parallel_junit_tests
+
utests_long:
<<: *j8_seq_executor
steps:
@@ -219,6 +416,14 @@
- run_junit_tests:
target: stress-test
+ utests_fqltool:
+ <<: *j8_seq_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run_junit_tests:
+ target: fqltool-test
+
j8_dtests-with-vnodes:
<<: *j8_par_executor
steps:
@@ -228,11 +433,26 @@
- create_venv
- create_dtest_containers:
file_tag: j8_with_vnodes
- run_dtests_extra_args: '--use-vnodes --skip-resource-intensive-tests'
+ run_dtests_extra_args: "--use-vnodes --skip-resource-intensive-tests --pytest-options '-k not cql'"
- run_dtests:
file_tag: j8_with_vnodes
pytest_extra_args: '--use-vnodes --num-tokens=32 --skip-resource-intensive-tests'
+ j11_dtests-with-vnodes:
+ <<: *j11_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - log_environment
+ - clone_dtest
+ - create_venv
+ - create_dtest_containers:
+ file_tag: j11_with_vnodes
+ run_dtests_extra_args: "--use-vnodes --skip-resource-intensive-tests --pytest-options '-k not cql'"
+ - run_dtests:
+ file_tag: j11_with_vnodes
+ pytest_extra_args: '--use-vnodes --num-tokens=32 --skip-resource-intensive-tests'
+
j8_dtests-no-vnodes:
<<: *j8_par_executor
steps:
@@ -242,11 +462,26 @@
- create_venv
- create_dtest_containers:
file_tag: j8_without_vnodes
- run_dtests_extra_args: '--skip-resource-intensive-tests'
+ run_dtests_extra_args: "--skip-resource-intensive-tests --pytest-options '-k not cql'"
- run_dtests:
file_tag: j8_without_vnodes
pytest_extra_args: '--skip-resource-intensive-tests'
+ j11_dtests-no-vnodes:
+ <<: *j11_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - log_environment
+ - clone_dtest
+ - create_venv
+ - create_dtest_containers:
+ file_tag: j11_without_vnodes
+ run_dtests_extra_args: "--skip-resource-intensive-tests --pytest-options '-k not cql'"
+ - run_dtests:
+ file_tag: j11_without_vnodes
+ pytest_extra_args: '--skip-resource-intensive-tests'
+
j8_upgradetests-no-vnodes:
<<: *j8_par_executor
steps:
@@ -264,6 +499,198 @@
extra_env_args: 'RUN_STATIC_UPGRADE_MATRIX=true'
pytest_extra_args: '--execute-upgrade-tests'
+ j8_cqlsh-dtests-py2-with-vnodes:
+ <<: *j8_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - clone_dtest
+ - create_venv
+ - create_dtest_containers:
+ file_tag: j8_with_vnodes
+ run_dtests_extra_args: "--use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql'"
+ - run_dtests:
+ file_tag: j8_with_vnodes
+ pytest_extra_args: '--use-vnodes --num-tokens=32 --skip-resource-intensive-tests'
+ extra_env_args: 'CQLSH_PYTHON=/usr/bin/python2.7'
+
+ j8_cqlsh-dtests-py3-with-vnodes:
+ <<: *j8_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - clone_dtest
+ - create_venv
+ - create_dtest_containers:
+ file_tag: j8_with_vnodes
+ run_dtests_extra_args: "--use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql'"
+ - run_dtests:
+ file_tag: j8_with_vnodes
+ pytest_extra_args: '--use-vnodes --num-tokens=32 --skip-resource-intensive-tests'
+ extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6'
+
+ j8_cqlsh-dtests-py38-with-vnodes:
+ <<: *j8_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - clone_dtest
+ - create_venv:
+ python_version: '3.8'
+ - create_dtest_containers:
+ file_tag: j8_with_vnodes
+ run_dtests_extra_args: "--use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql'"
+ python_version: '3.8'
+ - run_dtests:
+ file_tag: j8_with_vnodes
+ pytest_extra_args: '--use-vnodes --num-tokens=32 --skip-resource-intensive-tests'
+ extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.8'
+ python_version: '3.8'
+
+ j8_cqlsh-dtests-py2-no-vnodes:
+ <<: *j8_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - clone_dtest
+ - create_venv
+ - create_dtest_containers:
+ file_tag: j8_without_vnodes
+ run_dtests_extra_args: "--skip-resource-intensive-tests --pytest-options '-k cql'"
+ - run_dtests:
+ file_tag: j8_without_vnodes
+ pytest_extra_args: '--skip-resource-intensive-tests'
+ extra_env_args: 'CQLSH_PYTHON=/usr/bin/python2.7'
+
+ j8_cqlsh-dtests-py3-no-vnodes:
+ <<: *j8_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - clone_dtest
+ - create_venv
+ - create_dtest_containers:
+ file_tag: j8_without_vnodes
+ run_dtests_extra_args: "--skip-resource-intensive-tests --pytest-options '-k cql'"
+ - run_dtests:
+ file_tag: j8_without_vnodes
+ pytest_extra_args: '--skip-resource-intensive-tests'
+ extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6'
+
+ j8_cqlsh-dtests-py38-no-vnodes:
+ <<: *j8_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - clone_dtest
+ - create_venv:
+ python_version: '3.8'
+ - create_dtest_containers:
+ file_tag: j8_without_vnodes
+ run_dtests_extra_args: "--skip-resource-intensive-tests --pytest-options '-k cql'"
+ python_version: '3.8'
+ - run_dtests:
+ file_tag: j8_without_vnodes
+ pytest_extra_args: '--skip-resource-intensive-tests'
+ extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.8'
+ python_version: '3.8'
+
+ j11_cqlsh-dtests-py2-with-vnodes:
+ <<: *j11_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - clone_dtest
+ - create_venv
+ - create_dtest_containers:
+ file_tag: j11_with_vnodes
+ run_dtests_extra_args: "--use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql'"
+ - run_dtests:
+ file_tag: j11_with_vnodes
+ pytest_extra_args: '--use-vnodes --num-tokens=32 --skip-resource-intensive-tests'
+ extra_env_args: 'CQLSH_PYTHON=/usr/bin/python2.7'
+
+ j11_cqlsh-dtests-py3-with-vnodes:
+ <<: *j11_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - clone_dtest
+ - create_venv
+ - create_dtest_containers:
+ file_tag: j11_with_vnodes
+ run_dtests_extra_args: "--use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql'"
+ - run_dtests:
+ file_tag: j11_with_vnodes
+ pytest_extra_args: '--use-vnodes --num-tokens=32 --skip-resource-intensive-tests'
+ extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6'
+
+ j11_cqlsh-dtests-py38-with-vnodes:
+ <<: *j11_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - clone_dtest
+ - create_venv:
+ python_version: '3.8'
+ - create_dtest_containers:
+ file_tag: j11_with_vnodes
+ run_dtests_extra_args: "--use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql'"
+ python_version: '3.8'
+ - run_dtests:
+ file_tag: j11_with_vnodes
+ pytest_extra_args: '--use-vnodes --num-tokens=32 --skip-resource-intensive-tests'
+ extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.8'
+ python_version: '3.8'
+
+ j11_cqlsh-dtests-py2-no-vnodes:
+ <<: *j11_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - clone_dtest
+ - create_venv
+ - create_dtest_containers:
+ file_tag: j11_without_vnodes
+ run_dtests_extra_args: "--skip-resource-intensive-tests --pytest-options '-k cql'"
+ - run_dtests:
+ file_tag: j11_without_vnodes
+ pytest_extra_args: '--skip-resource-intensive-tests'
+ extra_env_args: 'CQLSH_PYTHON=/usr/bin/python2.7'
+
+ j11_cqlsh-dtests-py3-no-vnodes:
+ <<: *j11_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - clone_dtest
+ - create_venv
+ - create_dtest_containers:
+ file_tag: j11_without_vnodes
+ run_dtests_extra_args: "--skip-resource-intensive-tests --pytest-options '-k cql'"
+ - run_dtests:
+ file_tag: j11_without_vnodes
+ pytest_extra_args: '--skip-resource-intensive-tests'
+ extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6'
+
+ j11_cqlsh-dtests-py38-no-vnodes:
+ <<: *j11_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - clone_dtest
+ - create_venv:
+ python_version: '3.8'
+ - create_dtest_containers:
+ file_tag: j11_without_vnodes
+ run_dtests_extra_args: "--skip-resource-intensive-tests --pytest-options '-k cql'"
+ python_version: '3.8'
+ - run_dtests:
+ file_tag: j11_without_vnodes
+ pytest_extra_args: '--skip-resource-intensive-tests'
+ extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.8'
+ python_version: '3.8'
+
commands:
log_environment:
steps:
@@ -313,7 +740,7 @@
cd ~/cassandra
# Loop to prevent failure due to maven-ant-tasks not downloading a jar..
for x in $(seq 1 3); do
- ${ANT_HOME}/bin/ant clean jar
+ ${ANT_HOME}/bin/ant clean realclean jar
RETURN="$?"
if [ "${RETURN}" -eq "0" ]; then
break
@@ -413,14 +840,9 @@
no_output_timeout:
type: string
default: 15m
- classlistprefix:
- type: string
- default: unit
steps:
- run:
name: Run Unit Tests (<<parameters.target>>)
- # Please note that we run `clean` and therefore rebuild the project, as we can't run tests on Java 8 in case
- # based on Java 11 builds.
command: |
export PATH=$JAVA_HOME/bin:$PATH
time mv ~/cassandra /tmp
@@ -428,7 +850,7 @@
if [ -d ~/dtest_jars ]; then
cp ~/dtest_jars/dtest* /tmp/cassandra/build/
fi
- ant <<parameters.target>> -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=<<parameters.classlistprefix>>
+ ant <<parameters.target>>
no_output_timeout: <<parameters.no_output_timeout>>
- store_test_results:
path: /tmp/cassandra/build/test/output/
@@ -453,8 +875,6 @@
steps:
- run:
name: Run Unit Tests (<<parameters.target>>)
- # Please note that we run `clean` and therefore rebuild the project, as we can't run tests on Java 8 in case
- # based on Java 11 builds.
command: |
set -x
export PATH=$JAVA_HOME/bin:$PATH
@@ -479,6 +899,11 @@
destination: logs
create_venv:
+ parameters:
+ python_version:
+ type: enum
+ default: "3.6"
+ enum: ["3.6", "3.7", "3.8"]
steps:
- run:
name: Configure virtualenv and python Dependencies
@@ -487,9 +912,10 @@
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
- source ~/env/bin/activate
+ source ~/env<<parameters.python_version>>/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
- pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
pip3 freeze
create_dtest_containers:
@@ -505,6 +931,10 @@
tests_filter_pattern:
type: string
default: ''
+ python_version:
+ type: enum
+ default: "3.6"
+ enum: ["3.6", "3.7", "3.8"]
steps:
- run:
name: Determine Tests to Run (<<parameters.file_tag>>)
@@ -515,7 +945,7 @@
# which we do via the `circleci` cli tool.
cd cassandra-dtest
- source ~/env/bin/activate
+ source ~/env<<parameters.python_version>>/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
if [ -n '<<parameters.extra_env_args>>' ]; then
@@ -543,6 +973,10 @@
extra_env_args:
type: string
default: ''
+ python_version:
+ type: enum
+ default: "3.6"
+ enum: ["3.6", "3.7", "3.8"]
steps:
- run:
name: Run dtests (<<parameters.file_tag>>)
@@ -551,7 +985,7 @@
echo "cat /tmp/split_dtest_tests_<<parameters.file_tag>>_final.txt"
cat /tmp/split_dtest_tests_<<parameters.file_tag>>_final.txt
- source ~/env/bin/activate
+ source ~/env<<parameters.python_version>>/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
if [ -n '<<parameters.extra_env_args>>' ]; then
export <<parameters.extra_env_args>>
diff --git a/.circleci/config-2_1.yml.high_res.patch b/.circleci/config-2_1.yml.high_res.patch
index 3c85668..d0799e4 100644
--- a/.circleci/config-2_1.yml.high_res.patch
+++ b/.circleci/config-2_1.yml.high_res.patch
@@ -1,28 +1,34 @@
-17,18c17,18
-< CCM_MAX_HEAP_SIZE: 1024M
-< CCM_HEAP_NEWSIZE: 256M
----
-> CCM_MAX_HEAP_SIZE: 2048M
-> CCM_HEAP_NEWSIZE: 512M
-23,24c23,24
+22,23c22,23
< #exec_resource_class: xlarge
< parallelism: 4
---
> exec_resource_class: xlarge
> parallelism: 100
-29,30c29,30
+28,29c28,29
+< #exec_resource_class: xlarge
+< parallelism: 1
+---
+> exec_resource_class: xlarge
+> parallelism: 5
+34,35c34,35
< #exec_resource_class: xlarge
< parallelism: 1
---
> exec_resource_class: xlarge
> parallelism: 2
-35,36c35,36
+40c40
+< #exec_resource_class: xlarge
+---
+> exec_resource_class: xlarge
+46,47c46,47
+< #exec_resource_class: xlarge
+< parallelism: 4
+---
+> exec_resource_class: xlarge
+> parallelism: 100
+52,53c52,53
< #exec_resource_class: xlarge
< parallelism: 1
---
> exec_resource_class: xlarge
> parallelism: 2
-41c41
-< #exec_resource_class: xlarge
----
-> exec_resource_class: xlarge
diff --git a/.circleci/config-2_1.yml.mid_res.patch b/.circleci/config-2_1.yml.mid_res.patch
new file mode 100644
index 0000000..56b0887
--- /dev/null
+++ b/.circleci/config-2_1.yml.mid_res.patch
@@ -0,0 +1,224 @@
+diff --git .circleci/config-2_1.yml .circleci/config-2_1.yml
+index ab621241a4..9c11f60d5d 100644
+--- .circleci/config-2_1.yml
++++ .circleci/config-2_1.yml
+@@ -19,32 +19,44 @@ default_env_vars: &default_env_vars
+ j8_par_executor: &j8_par_executor
+ executor:
+ name: java8-executor
+- #exec_resource_class: xlarge
+- parallelism: 4
++ exec_resource_class: medium
++ parallelism: 25
+
+ j8_small_par_executor: &j8_small_par_executor
+ executor:
+ name: java8-executor
+- #exec_resource_class: xlarge
+- parallelism: 1
++ exec_resource_class: large
++ parallelism: 10
+
+ j8_medium_par_executor: &j8_medium_par_executor
+ executor:
+ name: java8-executor
+- #exec_resource_class: xlarge
+- parallelism: 1
++ exec_resource_class: large
++ parallelism: 10
++
++j8_large_par_executor: &j8_large_par_executor
++ executor:
++ name: java8-executor
++ exec_resource_class: large
++ parallelism: 50
++
++j8_very_large_par_executor: &j8_very_large_par_executor
++ executor:
++ name: java8-executor
++ exec_resource_class: xlarge
++ parallelism: 100
+
+ j8_seq_executor: &j8_seq_executor
+ executor:
+ name: java8-executor
+- #exec_resource_class: xlarge
++ exec_resource_class: medium
+ parallelism: 1 # sequential, single container tests: no parallelism benefits
+
+ j11_par_executor: &j11_par_executor
+ executor:
+ name: java11-executor
+- #exec_resource_class: xlarge
+- parallelism: 4
++ exec_resource_class: medium
++ parallelism: 25
+
+ j11_small_par_executor: &j11_small_par_executor
+ executor:
+@@ -52,6 +64,12 @@ j11_small_par_executor: &j11_small_par_executor
+ #exec_resource_class: xlarge
+ parallelism: 1
+
++j11_large_par_executor: &j11_large_par_executor
++ executor:
++ name: java11-executor
++ exec_resource_class: large
++ parallelism: 50
++
+ j8_with_dtests_jobs: &j8_with_dtests_jobs
+ jobs:
+ - j8_build
+@@ -425,7 +443,7 @@ jobs:
+ target: fqltool-test
+
+ j8_dtests-with-vnodes:
+- <<: *j8_par_executor
++ <<: *j8_large_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+@@ -439,7 +457,7 @@ jobs:
+ pytest_extra_args: '--use-vnodes --num-tokens=32 --skip-resource-intensive-tests'
+
+ j11_dtests-with-vnodes:
+- <<: *j11_par_executor
++ <<: *j11_large_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+@@ -454,7 +472,7 @@ jobs:
+ pytest_extra_args: '--use-vnodes --num-tokens=32 --skip-resource-intensive-tests'
+
+ j8_dtests-no-vnodes:
+- <<: *j8_par_executor
++ <<: *j8_large_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+@@ -468,7 +486,7 @@ jobs:
+ pytest_extra_args: '--skip-resource-intensive-tests'
+
+ j11_dtests-no-vnodes:
+- <<: *j11_par_executor
++ <<: *j11_large_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+@@ -483,7 +501,7 @@ jobs:
+ pytest_extra_args: '--skip-resource-intensive-tests'
+
+ j8_upgradetests-no-vnodes:
+- <<: *j8_par_executor
++ <<: *j8_very_large_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+@@ -500,7 +518,7 @@ jobs:
+ pytest_extra_args: '--execute-upgrade-tests'
+
+ j8_cqlsh-dtests-py2-with-vnodes:
+- <<: *j8_par_executor
++ <<: *j8_large_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+@@ -515,7 +533,7 @@ jobs:
+ extra_env_args: 'CQLSH_PYTHON=/usr/bin/python2.7'
+
+ j8_cqlsh-dtests-py3-with-vnodes:
+- <<: *j8_par_executor
++ <<: *j8_large_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+@@ -530,7 +548,7 @@ jobs:
+ extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6'
+
+ j8_cqlsh-dtests-py38-with-vnodes:
+- <<: *j8_par_executor
++ <<: *j8_large_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+@@ -548,7 +566,7 @@ jobs:
+ python_version: '3.8'
+
+ j8_cqlsh-dtests-py2-no-vnodes:
+- <<: *j8_par_executor
++ <<: *j8_large_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+@@ -563,7 +581,7 @@ jobs:
+ extra_env_args: 'CQLSH_PYTHON=/usr/bin/python2.7'
+
+ j8_cqlsh-dtests-py3-no-vnodes:
+- <<: *j8_par_executor
++ <<: *j8_large_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+@@ -578,7 +596,7 @@ jobs:
+ extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6'
+
+ j8_cqlsh-dtests-py38-no-vnodes:
+- <<: *j8_par_executor
++ <<: *j8_large_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+@@ -596,7 +614,7 @@ jobs:
+ python_version: '3.8'
+
+ j11_cqlsh-dtests-py2-with-vnodes:
+- <<: *j11_par_executor
++ <<: *j11_large_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+@@ -611,7 +629,7 @@ jobs:
+ extra_env_args: 'CQLSH_PYTHON=/usr/bin/python2.7'
+
+ j11_cqlsh-dtests-py3-with-vnodes:
+- <<: *j11_par_executor
++ <<: *j11_large_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+@@ -626,7 +644,7 @@ jobs:
+ extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6'
+
+ j11_cqlsh-dtests-py38-with-vnodes:
+- <<: *j11_par_executor
++ <<: *j11_large_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+@@ -644,7 +662,7 @@ jobs:
+ python_version: '3.8'
+
+ j11_cqlsh-dtests-py2-no-vnodes:
+- <<: *j11_par_executor
++ <<: *j11_large_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+@@ -659,7 +677,7 @@ jobs:
+ extra_env_args: 'CQLSH_PYTHON=/usr/bin/python2.7'
+
+ j11_cqlsh-dtests-py3-no-vnodes:
+- <<: *j11_par_executor
++ <<: *j11_large_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+@@ -674,7 +692,7 @@ jobs:
+ extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6'
+
+ j11_cqlsh-dtests-py38-no-vnodes:
+- <<: *j11_par_executor
++ <<: *j11_large_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
diff --git a/.circleci/config.yml b/.circleci/config.yml
index 8123ab9..ffcb107 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -2,7 +2,7 @@
jobs:
j8_jvm_upgrade_dtests:
docker:
- - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
+ - image: nastra/cassandra-testing-ubuntu1910-java11-w-dependencies:20200603
resource_class: medium
working_directory: ~/
shell: /bin/bash -eo pipefail -l
@@ -76,7 +76,6 @@
path: /tmp/cassandra/build/test/logs
destination: logs
environment:
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
@@ -92,90 +91,9 @@
- CCM_HEAP_NEWSIZE: 256M
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- build:
+ j8_cqlsh-dtests-py2-with-vnodes:
docker:
- - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
- resource_class: medium
- working_directory: ~/
- shell: /bin/bash -eo pipefail -l
- parallelism: 1
- steps:
- - run:
- name: Log Environment Information
- command: |
- echo '*** id ***'
- id
- echo '*** cat /proc/cpuinfo ***'
- cat /proc/cpuinfo
- echo '*** free -m ***'
- free -m
- echo '*** df -m ***'
- df -m
- echo '*** ifconfig -a ***'
- ifconfig -a
- echo '*** uname -a ***'
- uname -a
- echo '*** mount ***'
- mount
- echo '*** env ***'
- env
- echo '*** java ***'
- which java
- java -version
- - run:
- name: Clone Cassandra Repository (via git)
- command: |
- git clone --single-branch --depth 1 --branch $CIRCLE_BRANCH git://github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME.git ~/cassandra
- - run:
- name: Build Cassandra
- command: |
- export PATH=$JAVA_HOME/bin:$PATH
- cd ~/cassandra
- # Loop to prevent failure due to maven-ant-tasks not downloading a jar..
- for x in $(seq 1 3); do
- ${ANT_HOME}/bin/ant clean jar
- RETURN="$?"
- if [ "${RETURN}" -eq "0" ]; then
- break
- fi
- done
- # Exit, if we didn't build successfully
- if [ "${RETURN}" -ne "0" ]; then
- echo "Build failed with exit code: ${RETURN}"
- exit ${RETURN}
- fi
- no_output_timeout: 15m
- - run:
- name: Run eclipse-warnings
- command: |
- export PATH=$JAVA_HOME/bin:$PATH
- cd ~/cassandra
- ant eclipse-warnings
- - persist_to_workspace:
- root: /home/cassandra
- paths:
- - cassandra
- - .m2
- environment:
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - ANT_HOME: /usr/share/ant
- - LANG: en_US.UTF-8
- - KEEP_TEST_DIR: true
- - DEFAULT_DIR: /home/cassandra/cassandra-dtest
- - PYTHONIOENCODING: utf-8
- - PYTHONUNBUFFERED: true
- - CASS_DRIVER_NO_EXTENSIONS: true
- - CASS_DRIVER_NO_CYTHON: true
- - CASSANDRA_SKIP_SYNC: true
- - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
- - DTEST_BRANCH: master
- - CCM_MAX_HEAP_SIZE: 1024M
- - CCM_HEAP_NEWSIZE: 256M
- - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_dtests-no-vnodes:
- docker:
- - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
+ - image: nastra/cassandra-testing-ubuntu1910-java11-w-dependencies:20200603
resource_class: medium
working_directory: ~/
shell: /bin/bash -eo pipefail -l
@@ -194,105 +112,26 @@
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
- source ~/env/bin/activate
+ source ~/env3.6/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
- pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
pip3 freeze
- run:
- name: Determine Tests to Run (j8_without_vnodes)
+ name: Determine Tests to Run (j8_with_vnodes)
no_output_timeout: 5m
- command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n"
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n"
- run:
- name: Run dtests (j8_without_vnodes)
- no_output_timeout: 15m
- command: "echo \"cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n"
- - store_test_results:
- path: /tmp/results
- - store_artifacts:
- path: /tmp/dtest
- destination: dtest_j8_without_vnodes
- - store_artifacts:
- path: ~/cassandra-dtest/logs
- destination: dtest_j8_without_vnodes_logs
- environment:
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - ANT_HOME: /usr/share/ant
- - LANG: en_US.UTF-8
- - KEEP_TEST_DIR: true
- - DEFAULT_DIR: /home/cassandra/cassandra-dtest
- - PYTHONIOENCODING: utf-8
- - PYTHONUNBUFFERED: true
- - CASS_DRIVER_NO_EXTENSIONS: true
- - CASS_DRIVER_NO_CYTHON: true
- - CASSANDRA_SKIP_SYNC: true
- - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
- - DTEST_BRANCH: master
- - CCM_MAX_HEAP_SIZE: 1024M
- - CCM_HEAP_NEWSIZE: 256M
- - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_upgradetests-no-vnodes:
- docker:
- - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
- resource_class: medium
- working_directory: ~/
- shell: /bin/bash -eo pipefail -l
- parallelism: 4
- steps:
- - attach_workspace:
- at: /home/cassandra
- - run:
- name: Clone Cassandra dtest Repository (via git)
- command: |
- git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
- - run:
- name: Configure virtualenv and python Dependencies
- command: |
- # note, this should be super quick as all dependencies should be pre-installed in the docker image
- # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
- # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
- # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
- source ~/env/bin/activate
- export PATH=$JAVA_HOME/bin:$PATH
- pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
- pip3 freeze
- - run:
- name: Determine Tests to Run (j8_upgradetests_without_vnodes)
- no_output_timeout: 5m
- command: |
- # reminder: this code (along with all the steps) is independently executed on every circle container
- # so the goal here is to get the circleci script to return the tests *this* container will run
- # which we do via the `circleci` cli tool.
-
- cd cassandra-dtest
- source ~/env/bin/activate
- export PATH=$JAVA_HOME/bin:$PATH
-
- if [ -n 'RUN_STATIC_UPGRADE_MATRIX=true' ]; then
- export RUN_STATIC_UPGRADE_MATRIX=true
- fi
-
- echo "***Collected DTests (j8_upgradetests_without_vnodes)***"
- set -eo pipefail && ./run_dtests.py --execute-upgrade-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw --cassandra-dir=../cassandra
- if [ -z '^upgrade_tests' ]; then
- mv /tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw /tmp/all_dtest_tests_j8_upgradetests_without_vnodes
- else
- grep -e '^upgrade_tests' /tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw > /tmp/all_dtest_tests_j8_upgradetests_without_vnodes || { echo "Filter did not match any tests! Exiting build."; exit 0; }
- fi
- set -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_upgradetests_without_vnodes > /tmp/split_dtest_tests_j8_upgradetests_without_vnodes.txt
- cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes.txt | tr '\n' ' ' > /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt
- cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt
- - run:
- name: Run dtests (j8_upgradetests_without_vnodes)
+ name: Run dtests (j8_with_vnodes)
no_output_timeout: 15m
command: |
- echo "cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt"
- cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt
+ echo "cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt
- source ~/env/bin/activate
+ source ~/env3.6/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
- if [ -n 'RUN_STATIC_UPGRADE_MATRIX=true' ]; then
- export RUN_STATIC_UPGRADE_MATRIX=true
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python2.7' ]; then
+ export CQLSH_PYTHON=/usr/bin/python2.7
fi
java -version
@@ -303,18 +142,17 @@
echo "** done env"
mkdir -p /tmp/results/dtests
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
- export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt`
- set -o pipefail && cd ~/cassandra-dtest && pytest --execute-upgrade-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j8_upgradetests_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt`
+ set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
- store_test_results:
path: /tmp/results
- store_artifacts:
path: /tmp/dtest
- destination: dtest_j8_upgradetests_without_vnodes
+ destination: dtest_j8_with_vnodes
- store_artifacts:
path: ~/cassandra-dtest/logs
- destination: dtest_j8_upgradetests_without_vnodes_logs
+ destination: dtest_j8_with_vnodes_logs
environment:
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
@@ -330,55 +168,9 @@
- CCM_HEAP_NEWSIZE: 256M
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- utests_stress:
+ j11_unit_tests:
docker:
- - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
- resource_class: medium
- working_directory: ~/
- shell: /bin/bash -eo pipefail -l
- parallelism: 1
- steps:
- - attach_workspace:
- at: /home/cassandra
- - run:
- name: Run Unit Tests (stress-test)
- command: |
- export PATH=$JAVA_HOME/bin:$PATH
- time mv ~/cassandra /tmp
- cd /tmp/cassandra
- if [ -d ~/dtest_jars ]; then
- cp ~/dtest_jars/dtest* /tmp/cassandra/build/
- fi
- ant stress-test -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
- no_output_timeout: 15m
- - store_test_results:
- path: /tmp/cassandra/build/test/output/
- - store_artifacts:
- path: /tmp/cassandra/build/test/output
- destination: junitxml
- - store_artifacts:
- path: /tmp/cassandra/build/test/logs
- destination: logs
- environment:
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - ANT_HOME: /usr/share/ant
- - LANG: en_US.UTF-8
- - KEEP_TEST_DIR: true
- - DEFAULT_DIR: /home/cassandra/cassandra-dtest
- - PYTHONIOENCODING: utf-8
- - PYTHONUNBUFFERED: true
- - CASS_DRIVER_NO_EXTENSIONS: true
- - CASS_DRIVER_NO_CYTHON: true
- - CASSANDRA_SKIP_SYNC: true
- - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
- - DTEST_BRANCH: master
- - CCM_MAX_HEAP_SIZE: 1024M
- - CCM_HEAP_NEWSIZE: 256M
- - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_unit_tests:
- docker:
- - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
+ - image: nastra/cassandra-testing-ubuntu1910-java11:20200603
resource_class: medium
working_directory: ~/
shell: /bin/bash -eo pipefail -l
@@ -452,7 +244,6 @@
path: /tmp/cassandra/build/test/logs
destination: logs
environment:
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
@@ -466,11 +257,12 @@
- DTEST_BRANCH: master
- CCM_MAX_HEAP_SIZE: 1024M
- CCM_HEAP_NEWSIZE: 256M
- - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_dtests-with-vnodes:
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j8_cqlsh-dtests-py38-no-vnodes:
docker:
- - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
+ - image: nastra/cassandra-testing-ubuntu1910-java11-w-dependencies:20200603
resource_class: medium
working_directory: ~/
shell: /bin/bash -eo pipefail -l
@@ -489,28 +281,47 @@
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
- source ~/env/bin/activate
+ source ~/env3.8/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
- pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
pip3 freeze
- run:
- name: Determine Tests to Run (j8_with_vnodes)
+ name: Determine Tests to Run (j8_without_vnodes)
no_output_timeout: 5m
- command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n"
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n"
- run:
- name: Run dtests (j8_with_vnodes)
+ name: Run dtests (j8_without_vnodes)
no_output_timeout: 15m
- command: "echo \"cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n"
+ command: |
+ echo "cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt
+
+ source ~/env3.8/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.8' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.8
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt`
+ set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
- store_test_results:
path: /tmp/results
- store_artifacts:
path: /tmp/dtest
- destination: dtest_j8_with_vnodes
+ destination: dtest_j8_without_vnodes
- store_artifacts:
path: ~/cassandra-dtest/logs
- destination: dtest_j8_with_vnodes_logs
+ destination: dtest_j8_without_vnodes_logs
environment:
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
@@ -526,9 +337,849 @@
- CCM_HEAP_NEWSIZE: 256M
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_jvm_dtests:
+ j11_cqlsh-dtests-py3-with-vnodes:
docker:
- - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
+ - image: nastra/cassandra-testing-ubuntu1910-java11:20200603
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_with_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_with_vnodes_raw /tmp/all_dtest_tests_j11_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_with_vnodes_raw > /tmp/all_dtest_tests_j11_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_with_vnodes > /tmp/split_dtest_tests_j11_with_vnodes.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j11_with_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.6
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt`
+ set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j11_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_with_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_with_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j11_cqlsh-dtests-py3-no-vnodes:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11:20200603
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_without_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_without_vnodes_raw /tmp/all_dtest_tests_j11_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_without_vnodes_raw > /tmp/all_dtest_tests_j11_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_without_vnodes > /tmp/split_dtest_tests_j11_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j11_without_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.6
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt`
+ set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j11_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_without_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j11_cqlsh-dtests-py38-with-vnodes:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11:20200603
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.8/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_with_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_with_vnodes_raw /tmp/all_dtest_tests_j11_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_with_vnodes_raw > /tmp/all_dtest_tests_j11_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_with_vnodes > /tmp/split_dtest_tests_j11_with_vnodes.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j11_with_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt
+
+ source ~/env3.8/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.8' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.8
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt`
+ set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j11_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_with_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_with_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j8_cqlsh-dtests-py3-with-vnodes:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11-w-dependencies:20200603
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j8_with_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j8_with_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.6
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt`
+ set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j8_with_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j8_with_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j8_cqlsh-dtests-py2-no-vnodes:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11-w-dependencies:20200603
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j8_without_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j8_without_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python2.7' ]; then
+ export CQLSH_PYTHON=/usr/bin/python2.7
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt`
+ set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j8_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j8_without_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j11_cqlsh-dtests-py2-with-vnodes:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11:20200603
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_with_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_with_vnodes_raw /tmp/all_dtest_tests_j11_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_with_vnodes_raw > /tmp/all_dtest_tests_j11_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_with_vnodes > /tmp/split_dtest_tests_j11_with_vnodes.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j11_with_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python2.7' ]; then
+ export CQLSH_PYTHON=/usr/bin/python2.7
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt`
+ set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j11_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_with_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_with_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j11_dtests-with-vnodes:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11:20200603
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_with_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k not cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_with_vnodes_raw /tmp/all_dtest_tests_j11_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_with_vnodes_raw > /tmp/all_dtest_tests_j11_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_with_vnodes > /tmp/split_dtest_tests_j11_with_vnodes.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j11_with_vnodes)
+ no_output_timeout: 15m
+ command: "echo \"cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j11_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n"
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_with_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_with_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j8_dtests-no-vnodes:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11-w-dependencies:20200603
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j8_without_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k not cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j8_without_vnodes)
+ no_output_timeout: 15m
+ command: "echo \"cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n"
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j8_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j8_without_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j8_upgradetests-no-vnodes:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11-w-dependencies:20200603
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j8_upgradetests_without_vnodes)
+ no_output_timeout: 5m
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ cd cassandra-dtest
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+
+ if [ -n 'RUN_STATIC_UPGRADE_MATRIX=true' ]; then
+ export RUN_STATIC_UPGRADE_MATRIX=true
+ fi
+
+ echo "***Collected DTests (j8_upgradetests_without_vnodes)***"
+ set -eo pipefail && ./run_dtests.py --execute-upgrade-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw --cassandra-dir=../cassandra
+ if [ -z '^upgrade_tests' ]; then
+ mv /tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw /tmp/all_dtest_tests_j8_upgradetests_without_vnodes
+ else
+ grep -e '^upgrade_tests' /tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw > /tmp/all_dtest_tests_j8_upgradetests_without_vnodes || { echo "Filter did not match any tests! Exiting build."; exit 0; }
+ fi
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_upgradetests_without_vnodes > /tmp/split_dtest_tests_j8_upgradetests_without_vnodes.txt
+ cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes.txt | tr '\n' ' ' > /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt
+ cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt
+ - run:
+ name: Run dtests (j8_upgradetests_without_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'RUN_STATIC_UPGRADE_MATRIX=true' ]; then
+ export RUN_STATIC_UPGRADE_MATRIX=true
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt`
+ set -o pipefail && cd ~/cassandra-dtest && pytest --execute-upgrade-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j8_upgradetests_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j8_upgradetests_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j8_upgradetests_without_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ utests_stress:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11-w-dependencies:20200603
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Run Unit Tests (stress-test)
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ ant stress-test
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j8_unit_tests:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11-w-dependencies:20200603
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine unit Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
+ ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j11_jvm_dtests:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11:20200603
resource_class: medium
working_directory: ~/
shell: /bin/bash -eo pipefail -l
@@ -602,7 +1253,627 @@
path: /tmp/cassandra/build/test/logs
destination: logs
environment:
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j11_build:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11:20200603
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Clone Cassandra Repository (via git)
+ command: |
+ git clone --single-branch --depth 1 --branch $CIRCLE_BRANCH git://github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME.git ~/cassandra
+ - run:
+ name: Build Cassandra
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ cd ~/cassandra
+ # Loop to prevent failure due to maven-ant-tasks not downloading a jar..
+ for x in $(seq 1 3); do
+ ${ANT_HOME}/bin/ant clean realclean jar
+ RETURN="$?"
+ if [ "${RETURN}" -eq "0" ]; then
+ break
+ fi
+ done
+ # Exit, if we didn't build successfully
+ if [ "${RETURN}" -ne "0" ]; then
+ echo "Build failed with exit code: ${RETURN}"
+ exit ${RETURN}
+ fi
+ no_output_timeout: 15m
+ - run:
+ name: Run eclipse-warnings
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ cd ~/cassandra
+ ant eclipse-warnings
+ - persist_to_workspace:
+ root: /home/cassandra
+ paths:
+ - cassandra
+ - .m2
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j11_cqlsh-dtests-py2-no-vnodes:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11:20200603
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_without_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_without_vnodes_raw /tmp/all_dtest_tests_j11_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_without_vnodes_raw > /tmp/all_dtest_tests_j11_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_without_vnodes > /tmp/split_dtest_tests_j11_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j11_without_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python2.7' ]; then
+ export CQLSH_PYTHON=/usr/bin/python2.7
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt`
+ set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j11_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_without_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j8_dtests-with-vnodes:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11-w-dependencies:20200603
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j8_with_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k not cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j8_with_vnodes)
+ no_output_timeout: 15m
+ command: "echo \"cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n"
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j8_with_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j8_with_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j11_cqlsh-dtests-py38-no-vnodes:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11:20200603
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.8/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_without_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_without_vnodes_raw /tmp/all_dtest_tests_j11_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_without_vnodes_raw > /tmp/all_dtest_tests_j11_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_without_vnodes > /tmp/split_dtest_tests_j11_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j11_without_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt
+
+ source ~/env3.8/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.8' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.8
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt`
+ set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j11_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_without_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j8_jvm_dtests:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11-w-dependencies:20200603
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine distributed Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/distributed/;;g" | grep "Test\.java$" | grep -v upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.distributed.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
+ ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j8_build:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11-w-dependencies:20200603
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Clone Cassandra Repository (via git)
+ command: |
+ git clone --single-branch --depth 1 --branch $CIRCLE_BRANCH git://github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME.git ~/cassandra
+ - run:
+ name: Build Cassandra
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ cd ~/cassandra
+ # Loop to prevent failure due to maven-ant-tasks not downloading a jar..
+ for x in $(seq 1 3); do
+ ${ANT_HOME}/bin/ant clean realclean jar
+ RETURN="$?"
+ if [ "${RETURN}" -eq "0" ]; then
+ break
+ fi
+ done
+ # Exit, if we didn't build successfully
+ if [ "${RETURN}" -ne "0" ]; then
+ echo "Build failed with exit code: ${RETURN}"
+ exit ${RETURN}
+ fi
+ no_output_timeout: 15m
+ - run:
+ name: Run eclipse-warnings
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ cd ~/cassandra
+ ant eclipse-warnings
+ - persist_to_workspace:
+ root: /home/cassandra
+ paths:
+ - cassandra
+ - .m2
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j8_cqlsh-dtests-py3-no-vnodes:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11-w-dependencies:20200603
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j8_without_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j8_without_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.6
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt`
+ set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j8_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j8_without_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j8_cqlsh-dtests-py38-with-vnodes:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11-w-dependencies:20200603
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.8/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j8_with_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j8_with_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt
+
+ source ~/env3.8/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.8' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.8
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt`
+ set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j8_with_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j8_with_vnodes_logs
+ environment:
- ANT_HOME: /usr/share/ant
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
@@ -620,7 +1891,7 @@
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
utests_long:
docker:
- - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
+ - image: nastra/cassandra-testing-ubuntu1910-java11-w-dependencies:20200603
resource_class: medium
working_directory: ~/
shell: /bin/bash -eo pipefail -l
@@ -637,7 +1908,7 @@
if [ -d ~/dtest_jars ]; then
cp ~/dtest_jars/dtest* /tmp/cassandra/build/
fi
- ant long-test -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
+ ant long-test
no_output_timeout: 15m
- store_test_results:
path: /tmp/cassandra/build/test/output/
@@ -648,7 +1919,6 @@
path: /tmp/cassandra/build/test/logs
destination: logs
environment:
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
@@ -664,9 +1934,135 @@
- CCM_HEAP_NEWSIZE: 256M
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ utests_fqltool:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11-w-dependencies:20200603
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Run Unit Tests (fqltool-test)
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ ant fqltool-test
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j11_dtests-no-vnodes:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11:20200603
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_without_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k not cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_without_vnodes_raw /tmp/all_dtest_tests_j11_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_without_vnodes_raw > /tmp/all_dtest_tests_j11_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_without_vnodes > /tmp/split_dtest_tests_j11_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j11_without_vnodes)
+ no_output_timeout: 15m
+ command: "echo \"cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j11_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n"
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_without_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
utests_compression:
docker:
- - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
+ - image: nastra/cassandra-testing-ubuntu1910-java11-w-dependencies:20200603
resource_class: medium
working_directory: ~/
shell: /bin/bash -eo pipefail -l
@@ -740,7 +2136,6 @@
path: /tmp/cassandra/build/test/logs
destination: logs
environment:
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
@@ -758,7 +2153,7 @@
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_dtest_jars_build:
docker:
- - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
+ - image: nastra/cassandra-testing-ubuntu1910-java11-w-dependencies:20200603
resource_class: medium
working_directory: ~/
shell: /bin/bash -eo pipefail -l
@@ -809,7 +2204,6 @@
paths:
- dtest_jars
environment:
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
@@ -827,41 +2221,55 @@
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
workflows:
version: 2
- build_and_run_tests:
+ java8_build_and_run_tests:
jobs:
- - build
+ - j8_build
- j8_unit_tests:
requires:
- - build
+ - j8_build
- j8_jvm_dtests:
requires:
- - build
+ - j8_build
+ - start_j11_unit_tests:
+ type: approval
+ requires:
+ - j8_build
+ - j11_unit_tests:
+ requires:
+ - start_j11_unit_tests
- start_utests_long:
type: approval
requires:
- - build
+ - j8_build
- utests_long:
requires:
- start_utests_long
- start_utests_compression:
type: approval
requires:
- - build
+ - j8_build
- utests_compression:
requires:
- start_utests_compression
- start_utests_stress:
type: approval
requires:
- - build
+ - j8_build
- utests_stress:
requires:
- start_utests_stress
+ - start_utests_fqltool:
+ type: approval
+ requires:
+ - j8_build
+ - utests_fqltool:
+ requires:
+ - start_utests_fqltool
- start_jvm_upgrade_dtest:
type: approval
- j8_dtest_jars_build:
requires:
- - build
+ - j8_build
- start_jvm_upgrade_dtest
- j8_jvm_upgrade_dtests:
requires:
@@ -869,17 +2277,128 @@
- start_j8_dtests:
type: approval
requires:
- - build
+ - j8_build
- j8_dtests-with-vnodes:
requires:
- start_j8_dtests
- j8_dtests-no-vnodes:
requires:
- start_j8_dtests
+ - start_j11_dtests:
+ type: approval
+ requires:
+ - j8_build
+ - j11_dtests-with-vnodes:
+ requires:
+ - start_j11_dtests
+ - j11_dtests-no-vnodes:
+ requires:
+ - start_j11_dtests
- start_upgrade_tests:
type: approval
requires:
- - build
+ - j8_build
- j8_upgradetests-no-vnodes:
requires:
- start_upgrade_tests
+ - start_j8_cqlsh_tests-with-vnodes:
+ type: approval
+ requires:
+ - j8_build
+ - j8_cqlsh-dtests-py2-with-vnodes:
+ requires:
+ - start_j8_cqlsh_tests-with-vnodes
+ - j8_cqlsh-dtests-py3-with-vnodes:
+ requires:
+ - start_j8_cqlsh_tests-with-vnodes
+ - j8_cqlsh-dtests-py38-with-vnodes:
+ requires:
+ - start_j8_cqlsh_tests-with-vnodes
+ - start_j8_cqlsh_tests-no-vnodes:
+ type: approval
+ requires:
+ - j8_build
+ - j8_cqlsh-dtests-py2-no-vnodes:
+ requires:
+ - start_j8_cqlsh_tests-no-vnodes
+ - j8_cqlsh-dtests-py3-no-vnodes:
+ requires:
+ - start_j8_cqlsh_tests-no-vnodes
+ - j8_cqlsh-dtests-py38-no-vnodes:
+ requires:
+ - start_j8_cqlsh_tests-no-vnodes
+ - start_j11_cqlsh_tests-with-vnodes:
+ type: approval
+ requires:
+ - j8_build
+ - j11_cqlsh-dtests-py2-with-vnodes:
+ requires:
+ - start_j11_cqlsh_tests-with-vnodes
+ - j11_cqlsh-dtests-py3-with-vnodes:
+ requires:
+ - start_j11_cqlsh_tests-with-vnodes
+ - j11_cqlsh-dtests-py38-with-vnodes:
+ requires:
+ - start_j11_cqlsh_tests-with-vnodes
+ - start_j11_cqlsh_tests-no-vnodes:
+ type: approval
+ requires:
+ - j8_build
+ - j11_cqlsh-dtests-py2-no-vnodes:
+ requires:
+ - start_j11_cqlsh_tests-no-vnodes
+ - j11_cqlsh-dtests-py3-no-vnodes:
+ requires:
+ - start_j11_cqlsh_tests-no-vnodes
+ - j11_cqlsh-dtests-py38-no-vnodes:
+ requires:
+ - start_j11_cqlsh_tests-no-vnodes
+ java11_build_and_run_tests:
+ jobs:
+ - j11_build
+ - start_j11_unit_tests:
+ type: approval
+ requires:
+ - j11_build
+ - j11_unit_tests:
+ requires:
+ - start_j11_unit_tests
+ - j11_jvm_dtests:
+ requires:
+ - j11_build
+ - start_j11_dtests:
+ type: approval
+ requires:
+ - j11_build
+ - j11_dtests-with-vnodes:
+ requires:
+ - start_j11_dtests
+ - j11_dtests-no-vnodes:
+ requires:
+ - start_j11_dtests
+ - start_j11_cqlsh_tests-with-vnodes:
+ type: approval
+ requires:
+ - j11_build
+ - j11_cqlsh-dtests-py2-with-vnodes:
+ requires:
+ - start_j11_cqlsh_tests-with-vnodes
+ - j11_cqlsh-dtests-py3-with-vnodes:
+ requires:
+ - start_j11_cqlsh_tests-with-vnodes
+ - j11_cqlsh-dtests-py38-with-vnodes:
+ requires:
+ - start_j11_cqlsh_tests-with-vnodes
+ - start_j11_cqlsh_tests-no-vnodes:
+ type: approval
+ requires:
+ - j11_build
+ - j11_cqlsh-dtests-py2-no-vnodes:
+ requires:
+ - start_j11_cqlsh_tests-no-vnodes
+ - j11_cqlsh-dtests-py3-no-vnodes:
+ requires:
+ - start_j11_cqlsh_tests-no-vnodes
+ - j11_cqlsh-dtests-py38-no-vnodes:
+ requires:
+ - start_j11_cqlsh_tests-no-vnodes
diff --git a/.circleci/config.yml.HIGHRES b/.circleci/config.yml.HIGHRES
index f7ef17e..51b02cc 100644
--- a/.circleci/config.yml.HIGHRES
+++ b/.circleci/config.yml.HIGHRES
@@ -2,7 +2,7 @@
jobs:
j8_jvm_upgrade_dtests:
docker:
- - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
+ - image: nastra/cassandra-testing-ubuntu1910-java11-w-dependencies:20200603
resource_class: xlarge
working_directory: ~/
shell: /bin/bash -eo pipefail -l
@@ -76,7 +76,6 @@
path: /tmp/cassandra/build/test/logs
destination: logs
environment:
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
@@ -88,94 +87,13 @@
- CASSANDRA_SKIP_SYNC: true
- DTEST_REPO: git://github.com/apache/cassandra-dtest.git
- DTEST_BRANCH: master
- - CCM_MAX_HEAP_SIZE: 2048M
- - CCM_HEAP_NEWSIZE: 512M
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- build:
+ j8_cqlsh-dtests-py2-with-vnodes:
docker:
- - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
- resource_class: medium
- working_directory: ~/
- shell: /bin/bash -eo pipefail -l
- parallelism: 1
- steps:
- - run:
- name: Log Environment Information
- command: |
- echo '*** id ***'
- id
- echo '*** cat /proc/cpuinfo ***'
- cat /proc/cpuinfo
- echo '*** free -m ***'
- free -m
- echo '*** df -m ***'
- df -m
- echo '*** ifconfig -a ***'
- ifconfig -a
- echo '*** uname -a ***'
- uname -a
- echo '*** mount ***'
- mount
- echo '*** env ***'
- env
- echo '*** java ***'
- which java
- java -version
- - run:
- name: Clone Cassandra Repository (via git)
- command: |
- git clone --single-branch --depth 1 --branch $CIRCLE_BRANCH git://github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME.git ~/cassandra
- - run:
- name: Build Cassandra
- command: |
- export PATH=$JAVA_HOME/bin:$PATH
- cd ~/cassandra
- # Loop to prevent failure due to maven-ant-tasks not downloading a jar..
- for x in $(seq 1 3); do
- ${ANT_HOME}/bin/ant clean jar
- RETURN="$?"
- if [ "${RETURN}" -eq "0" ]; then
- break
- fi
- done
- # Exit, if we didn't build successfully
- if [ "${RETURN}" -ne "0" ]; then
- echo "Build failed with exit code: ${RETURN}"
- exit ${RETURN}
- fi
- no_output_timeout: 15m
- - run:
- name: Run eclipse-warnings
- command: |
- export PATH=$JAVA_HOME/bin:$PATH
- cd ~/cassandra
- ant eclipse-warnings
- - persist_to_workspace:
- root: /home/cassandra
- paths:
- - cassandra
- - .m2
- environment:
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - ANT_HOME: /usr/share/ant
- - LANG: en_US.UTF-8
- - KEEP_TEST_DIR: true
- - DEFAULT_DIR: /home/cassandra/cassandra-dtest
- - PYTHONIOENCODING: utf-8
- - PYTHONUNBUFFERED: true
- - CASS_DRIVER_NO_EXTENSIONS: true
- - CASS_DRIVER_NO_CYTHON: true
- - CASSANDRA_SKIP_SYNC: true
- - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
- - DTEST_BRANCH: master
- - CCM_MAX_HEAP_SIZE: 2048M
- - CCM_HEAP_NEWSIZE: 512M
- - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_dtests-no-vnodes:
- docker:
- - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
+ - image: nastra/cassandra-testing-ubuntu1910-java11-w-dependencies:20200603
resource_class: xlarge
working_directory: ~/
shell: /bin/bash -eo pipefail -l
@@ -194,105 +112,26 @@
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
- source ~/env/bin/activate
+ source ~/env3.6/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
- pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
pip3 freeze
- run:
- name: Determine Tests to Run (j8_without_vnodes)
+ name: Determine Tests to Run (j8_with_vnodes)
no_output_timeout: 5m
- command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n"
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n"
- run:
- name: Run dtests (j8_without_vnodes)
- no_output_timeout: 15m
- command: "echo \"cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n"
- - store_test_results:
- path: /tmp/results
- - store_artifacts:
- path: /tmp/dtest
- destination: dtest_j8_without_vnodes
- - store_artifacts:
- path: ~/cassandra-dtest/logs
- destination: dtest_j8_without_vnodes_logs
- environment:
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - ANT_HOME: /usr/share/ant
- - LANG: en_US.UTF-8
- - KEEP_TEST_DIR: true
- - DEFAULT_DIR: /home/cassandra/cassandra-dtest
- - PYTHONIOENCODING: utf-8
- - PYTHONUNBUFFERED: true
- - CASS_DRIVER_NO_EXTENSIONS: true
- - CASS_DRIVER_NO_CYTHON: true
- - CASSANDRA_SKIP_SYNC: true
- - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
- - DTEST_BRANCH: master
- - CCM_MAX_HEAP_SIZE: 2048M
- - CCM_HEAP_NEWSIZE: 512M
- - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_upgradetests-no-vnodes:
- docker:
- - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
- resource_class: xlarge
- working_directory: ~/
- shell: /bin/bash -eo pipefail -l
- parallelism: 100
- steps:
- - attach_workspace:
- at: /home/cassandra
- - run:
- name: Clone Cassandra dtest Repository (via git)
- command: |
- git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
- - run:
- name: Configure virtualenv and python Dependencies
- command: |
- # note, this should be super quick as all dependencies should be pre-installed in the docker image
- # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
- # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
- # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
- source ~/env/bin/activate
- export PATH=$JAVA_HOME/bin:$PATH
- pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
- pip3 freeze
- - run:
- name: Determine Tests to Run (j8_upgradetests_without_vnodes)
- no_output_timeout: 5m
- command: |
- # reminder: this code (along with all the steps) is independently executed on every circle container
- # so the goal here is to get the circleci script to return the tests *this* container will run
- # which we do via the `circleci` cli tool.
-
- cd cassandra-dtest
- source ~/env/bin/activate
- export PATH=$JAVA_HOME/bin:$PATH
-
- if [ -n 'RUN_STATIC_UPGRADE_MATRIX=true' ]; then
- export RUN_STATIC_UPGRADE_MATRIX=true
- fi
-
- echo "***Collected DTests (j8_upgradetests_without_vnodes)***"
- set -eo pipefail && ./run_dtests.py --execute-upgrade-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw --cassandra-dir=../cassandra
- if [ -z '^upgrade_tests' ]; then
- mv /tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw /tmp/all_dtest_tests_j8_upgradetests_without_vnodes
- else
- grep -e '^upgrade_tests' /tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw > /tmp/all_dtest_tests_j8_upgradetests_without_vnodes || { echo "Filter did not match any tests! Exiting build."; exit 0; }
- fi
- set -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_upgradetests_without_vnodes > /tmp/split_dtest_tests_j8_upgradetests_without_vnodes.txt
- cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes.txt | tr '\n' ' ' > /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt
- cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt
- - run:
- name: Run dtests (j8_upgradetests_without_vnodes)
+ name: Run dtests (j8_with_vnodes)
no_output_timeout: 15m
command: |
- echo "cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt"
- cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt
+ echo "cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt
- source ~/env/bin/activate
+ source ~/env3.6/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
- if [ -n 'RUN_STATIC_UPGRADE_MATRIX=true' ]; then
- export RUN_STATIC_UPGRADE_MATRIX=true
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python2.7' ]; then
+ export CQLSH_PYTHON=/usr/bin/python2.7
fi
java -version
@@ -303,18 +142,17 @@
echo "** done env"
mkdir -p /tmp/results/dtests
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
- export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt`
- set -o pipefail && cd ~/cassandra-dtest && pytest --execute-upgrade-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j8_upgradetests_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt`
+ set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
- store_test_results:
path: /tmp/results
- store_artifacts:
path: /tmp/dtest
- destination: dtest_j8_upgradetests_without_vnodes
+ destination: dtest_j8_with_vnodes
- store_artifacts:
path: ~/cassandra-dtest/logs
- destination: dtest_j8_upgradetests_without_vnodes_logs
+ destination: dtest_j8_with_vnodes_logs
environment:
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
@@ -326,59 +164,13 @@
- CASSANDRA_SKIP_SYNC: true
- DTEST_REPO: git://github.com/apache/cassandra-dtest.git
- DTEST_BRANCH: master
- - CCM_MAX_HEAP_SIZE: 2048M
- - CCM_HEAP_NEWSIZE: 512M
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- utests_stress:
+ j11_unit_tests:
docker:
- - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
- resource_class: xlarge
- working_directory: ~/
- shell: /bin/bash -eo pipefail -l
- parallelism: 1
- steps:
- - attach_workspace:
- at: /home/cassandra
- - run:
- name: Run Unit Tests (stress-test)
- command: |
- export PATH=$JAVA_HOME/bin:$PATH
- time mv ~/cassandra /tmp
- cd /tmp/cassandra
- if [ -d ~/dtest_jars ]; then
- cp ~/dtest_jars/dtest* /tmp/cassandra/build/
- fi
- ant stress-test -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
- no_output_timeout: 15m
- - store_test_results:
- path: /tmp/cassandra/build/test/output/
- - store_artifacts:
- path: /tmp/cassandra/build/test/output
- destination: junitxml
- - store_artifacts:
- path: /tmp/cassandra/build/test/logs
- destination: logs
- environment:
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - ANT_HOME: /usr/share/ant
- - LANG: en_US.UTF-8
- - KEEP_TEST_DIR: true
- - DEFAULT_DIR: /home/cassandra/cassandra-dtest
- - PYTHONIOENCODING: utf-8
- - PYTHONUNBUFFERED: true
- - CASS_DRIVER_NO_EXTENSIONS: true
- - CASS_DRIVER_NO_CYTHON: true
- - CASSANDRA_SKIP_SYNC: true
- - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
- - DTEST_BRANCH: master
- - CCM_MAX_HEAP_SIZE: 2048M
- - CCM_HEAP_NEWSIZE: 512M
- - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_unit_tests:
- docker:
- - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
+ - image: nastra/cassandra-testing-ubuntu1910-java11:20200603
resource_class: xlarge
working_directory: ~/
shell: /bin/bash -eo pipefail -l
@@ -452,7 +244,6 @@
path: /tmp/cassandra/build/test/logs
destination: logs
environment:
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
@@ -464,13 +255,14 @@
- CASSANDRA_SKIP_SYNC: true
- DTEST_REPO: git://github.com/apache/cassandra-dtest.git
- DTEST_BRANCH: master
- - CCM_MAX_HEAP_SIZE: 2048M
- - CCM_HEAP_NEWSIZE: 512M
- - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_dtests-with-vnodes:
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j8_cqlsh-dtests-py38-no-vnodes:
docker:
- - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
+ - image: nastra/cassandra-testing-ubuntu1910-java11-w-dependencies:20200603
resource_class: xlarge
working_directory: ~/
shell: /bin/bash -eo pipefail -l
@@ -489,28 +281,47 @@
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
- source ~/env/bin/activate
+ source ~/env3.8/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
- pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
pip3 freeze
- run:
- name: Determine Tests to Run (j8_with_vnodes)
+ name: Determine Tests to Run (j8_without_vnodes)
no_output_timeout: 5m
- command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n"
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n"
- run:
- name: Run dtests (j8_with_vnodes)
+ name: Run dtests (j8_without_vnodes)
no_output_timeout: 15m
- command: "echo \"cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n"
+ command: |
+ echo "cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt
+
+ source ~/env3.8/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.8' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.8
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt`
+ set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
- store_test_results:
path: /tmp/results
- store_artifacts:
path: /tmp/dtest
- destination: dtest_j8_with_vnodes
+ destination: dtest_j8_without_vnodes
- store_artifacts:
path: ~/cassandra-dtest/logs
- destination: dtest_j8_with_vnodes_logs
+ destination: dtest_j8_without_vnodes_logs
environment:
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
@@ -522,13 +333,853 @@
- CASSANDRA_SKIP_SYNC: true
- DTEST_REPO: git://github.com/apache/cassandra-dtest.git
- DTEST_BRANCH: master
- - CCM_MAX_HEAP_SIZE: 2048M
- - CCM_HEAP_NEWSIZE: 512M
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_jvm_dtests:
+ j11_cqlsh-dtests-py3-with-vnodes:
docker:
- - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
+ - image: nastra/cassandra-testing-ubuntu1910-java11:20200603
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 100
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_with_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_with_vnodes_raw /tmp/all_dtest_tests_j11_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_with_vnodes_raw > /tmp/all_dtest_tests_j11_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_with_vnodes > /tmp/split_dtest_tests_j11_with_vnodes.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j11_with_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.6
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt`
+ set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j11_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_with_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_with_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j11_cqlsh-dtests-py3-no-vnodes:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11:20200603
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 100
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_without_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_without_vnodes_raw /tmp/all_dtest_tests_j11_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_without_vnodes_raw > /tmp/all_dtest_tests_j11_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_without_vnodes > /tmp/split_dtest_tests_j11_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j11_without_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.6
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt`
+ set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j11_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_without_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j11_cqlsh-dtests-py38-with-vnodes:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11:20200603
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 100
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.8/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_with_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_with_vnodes_raw /tmp/all_dtest_tests_j11_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_with_vnodes_raw > /tmp/all_dtest_tests_j11_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_with_vnodes > /tmp/split_dtest_tests_j11_with_vnodes.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j11_with_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt
+
+ source ~/env3.8/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.8' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.8
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt`
+ set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j11_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_with_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_with_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j8_cqlsh-dtests-py3-with-vnodes:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11-w-dependencies:20200603
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 100
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j8_with_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j8_with_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.6
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt`
+ set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j8_with_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j8_with_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j8_cqlsh-dtests-py2-no-vnodes:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11-w-dependencies:20200603
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 100
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j8_without_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j8_without_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python2.7' ]; then
+ export CQLSH_PYTHON=/usr/bin/python2.7
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt`
+ set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j8_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j8_without_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j11_cqlsh-dtests-py2-with-vnodes:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11:20200603
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 100
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_with_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_with_vnodes_raw /tmp/all_dtest_tests_j11_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_with_vnodes_raw > /tmp/all_dtest_tests_j11_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_with_vnodes > /tmp/split_dtest_tests_j11_with_vnodes.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j11_with_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python2.7' ]; then
+ export CQLSH_PYTHON=/usr/bin/python2.7
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt`
+ set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j11_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_with_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_with_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j11_dtests-with-vnodes:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11:20200603
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 100
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_with_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k not cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_with_vnodes_raw /tmp/all_dtest_tests_j11_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_with_vnodes_raw > /tmp/all_dtest_tests_j11_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_with_vnodes > /tmp/split_dtest_tests_j11_with_vnodes.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j11_with_vnodes)
+ no_output_timeout: 15m
+ command: "echo \"cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j11_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n"
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_with_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_with_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j8_dtests-no-vnodes:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11-w-dependencies:20200603
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 100
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j8_without_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k not cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j8_without_vnodes)
+ no_output_timeout: 15m
+ command: "echo \"cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n"
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j8_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j8_without_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j8_upgradetests-no-vnodes:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11-w-dependencies:20200603
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 100
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j8_upgradetests_without_vnodes)
+ no_output_timeout: 5m
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ cd cassandra-dtest
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+
+ if [ -n 'RUN_STATIC_UPGRADE_MATRIX=true' ]; then
+ export RUN_STATIC_UPGRADE_MATRIX=true
+ fi
+
+ echo "***Collected DTests (j8_upgradetests_without_vnodes)***"
+ set -eo pipefail && ./run_dtests.py --execute-upgrade-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw --cassandra-dir=../cassandra
+ if [ -z '^upgrade_tests' ]; then
+ mv /tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw /tmp/all_dtest_tests_j8_upgradetests_without_vnodes
+ else
+ grep -e '^upgrade_tests' /tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw > /tmp/all_dtest_tests_j8_upgradetests_without_vnodes || { echo "Filter did not match any tests! Exiting build."; exit 0; }
+ fi
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_upgradetests_without_vnodes > /tmp/split_dtest_tests_j8_upgradetests_without_vnodes.txt
+ cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes.txt | tr '\n' ' ' > /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt
+ cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt
+ - run:
+ name: Run dtests (j8_upgradetests_without_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'RUN_STATIC_UPGRADE_MATRIX=true' ]; then
+ export RUN_STATIC_UPGRADE_MATRIX=true
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt`
+ set -o pipefail && cd ~/cassandra-dtest && pytest --execute-upgrade-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j8_upgradetests_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j8_upgradetests_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j8_upgradetests_without_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ utests_stress:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11-w-dependencies:20200603
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Run Unit Tests (stress-test)
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ ant stress-test
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j8_unit_tests:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11-w-dependencies:20200603
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 100
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine unit Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
+ ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j11_jvm_dtests:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11:20200603
resource_class: xlarge
working_directory: ~/
shell: /bin/bash -eo pipefail -l
@@ -602,7 +1253,6 @@
path: /tmp/cassandra/build/test/logs
destination: logs
environment:
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
@@ -614,13 +1264,634 @@
- CASSANDRA_SKIP_SYNC: true
- DTEST_REPO: git://github.com/apache/cassandra-dtest.git
- DTEST_BRANCH: master
- - CCM_MAX_HEAP_SIZE: 2048M
- - CCM_HEAP_NEWSIZE: 512M
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j11_build:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11:20200603
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Clone Cassandra Repository (via git)
+ command: |
+ git clone --single-branch --depth 1 --branch $CIRCLE_BRANCH git://github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME.git ~/cassandra
+ - run:
+ name: Build Cassandra
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ cd ~/cassandra
+ # Loop to prevent failure due to maven-ant-tasks not downloading a jar..
+ for x in $(seq 1 3); do
+ ${ANT_HOME}/bin/ant clean realclean jar
+ RETURN="$?"
+ if [ "${RETURN}" -eq "0" ]; then
+ break
+ fi
+ done
+ # Exit, if we didn't build successfully
+ if [ "${RETURN}" -ne "0" ]; then
+ echo "Build failed with exit code: ${RETURN}"
+ exit ${RETURN}
+ fi
+ no_output_timeout: 15m
+ - run:
+ name: Run eclipse-warnings
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ cd ~/cassandra
+ ant eclipse-warnings
+ - persist_to_workspace:
+ root: /home/cassandra
+ paths:
+ - cassandra
+ - .m2
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j11_cqlsh-dtests-py2-no-vnodes:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11:20200603
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 100
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_without_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_without_vnodes_raw /tmp/all_dtest_tests_j11_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_without_vnodes_raw > /tmp/all_dtest_tests_j11_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_without_vnodes > /tmp/split_dtest_tests_j11_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j11_without_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python2.7' ]; then
+ export CQLSH_PYTHON=/usr/bin/python2.7
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt`
+ set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j11_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_without_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j8_dtests-with-vnodes:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11-w-dependencies:20200603
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 100
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j8_with_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k not cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j8_with_vnodes)
+ no_output_timeout: 15m
+ command: "echo \"cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n"
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j8_with_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j8_with_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j11_cqlsh-dtests-py38-no-vnodes:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11:20200603
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 100
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.8/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_without_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_without_vnodes_raw /tmp/all_dtest_tests_j11_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_without_vnodes_raw > /tmp/all_dtest_tests_j11_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_without_vnodes > /tmp/split_dtest_tests_j11_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j11_without_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt
+
+ source ~/env3.8/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.8' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.8
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt`
+ set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j11_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_without_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j8_jvm_dtests:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11-w-dependencies:20200603
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 5
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine distributed Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/distributed/;;g" | grep "Test\.java$" | grep -v upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.distributed.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
+ ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j8_build:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11-w-dependencies:20200603
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Clone Cassandra Repository (via git)
+ command: |
+ git clone --single-branch --depth 1 --branch $CIRCLE_BRANCH git://github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME.git ~/cassandra
+ - run:
+ name: Build Cassandra
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ cd ~/cassandra
+ # Loop to prevent failure due to maven-ant-tasks not downloading a jar..
+ for x in $(seq 1 3); do
+ ${ANT_HOME}/bin/ant clean realclean jar
+ RETURN="$?"
+ if [ "${RETURN}" -eq "0" ]; then
+ break
+ fi
+ done
+ # Exit, if we didn't build successfully
+ if [ "${RETURN}" -ne "0" ]; then
+ echo "Build failed with exit code: ${RETURN}"
+ exit ${RETURN}
+ fi
+ no_output_timeout: 15m
+ - run:
+ name: Run eclipse-warnings
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ cd ~/cassandra
+ ant eclipse-warnings
+ - persist_to_workspace:
+ root: /home/cassandra
+ paths:
+ - cassandra
+ - .m2
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j8_cqlsh-dtests-py3-no-vnodes:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11-w-dependencies:20200603
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 100
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j8_without_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j8_without_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.6
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt`
+ set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j8_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j8_without_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j8_cqlsh-dtests-py38-with-vnodes:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11-w-dependencies:20200603
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 100
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.8/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j8_with_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j8_with_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt
+
+ source ~/env3.8/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.8' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.8
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt`
+ set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j8_with_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j8_with_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
utests_long:
docker:
- - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
+ - image: nastra/cassandra-testing-ubuntu1910-java11-w-dependencies:20200603
resource_class: xlarge
working_directory: ~/
shell: /bin/bash -eo pipefail -l
@@ -637,7 +1908,7 @@
if [ -d ~/dtest_jars ]; then
cp ~/dtest_jars/dtest* /tmp/cassandra/build/
fi
- ant long-test -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
+ ant long-test
no_output_timeout: 15m
- store_test_results:
path: /tmp/cassandra/build/test/output/
@@ -648,7 +1919,6 @@
path: /tmp/cassandra/build/test/logs
destination: logs
environment:
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
@@ -660,13 +1930,139 @@
- CASSANDRA_SKIP_SYNC: true
- DTEST_REPO: git://github.com/apache/cassandra-dtest.git
- DTEST_BRANCH: master
- - CCM_MAX_HEAP_SIZE: 2048M
- - CCM_HEAP_NEWSIZE: 512M
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ utests_fqltool:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11-w-dependencies:20200603
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Run Unit Tests (fqltool-test)
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ ant fqltool-test
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j11_dtests-no-vnodes:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11:20200603
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 100
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_without_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k not cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_without_vnodes_raw /tmp/all_dtest_tests_j11_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_without_vnodes_raw > /tmp/all_dtest_tests_j11_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_without_vnodes > /tmp/split_dtest_tests_j11_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j11_without_vnodes)
+ no_output_timeout: 15m
+ command: "echo \"cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j11_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n"
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_without_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
utests_compression:
docker:
- - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
+ - image: nastra/cassandra-testing-ubuntu1910-java11-w-dependencies:20200603
resource_class: xlarge
working_directory: ~/
shell: /bin/bash -eo pipefail -l
@@ -740,7 +2136,6 @@
path: /tmp/cassandra/build/test/logs
destination: logs
environment:
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
@@ -752,13 +2147,13 @@
- CASSANDRA_SKIP_SYNC: true
- DTEST_REPO: git://github.com/apache/cassandra-dtest.git
- DTEST_BRANCH: master
- - CCM_MAX_HEAP_SIZE: 2048M
- - CCM_HEAP_NEWSIZE: 512M
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_dtest_jars_build:
docker:
- - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
+ - image: nastra/cassandra-testing-ubuntu1910-java11-w-dependencies:20200603
resource_class: medium
working_directory: ~/
shell: /bin/bash -eo pipefail -l
@@ -809,7 +2204,6 @@
paths:
- dtest_jars
environment:
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
@@ -821,47 +2215,61 @@
- CASSANDRA_SKIP_SYNC: true
- DTEST_REPO: git://github.com/apache/cassandra-dtest.git
- DTEST_BRANCH: master
- - CCM_MAX_HEAP_SIZE: 2048M
- - CCM_HEAP_NEWSIZE: 512M
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
workflows:
version: 2
- build_and_run_tests:
+ java8_build_and_run_tests:
jobs:
- - build
+ - j8_build
- j8_unit_tests:
requires:
- - build
+ - j8_build
- j8_jvm_dtests:
requires:
- - build
+ - j8_build
+ - start_j11_unit_tests:
+ type: approval
+ requires:
+ - j8_build
+ - j11_unit_tests:
+ requires:
+ - start_j11_unit_tests
- start_utests_long:
type: approval
requires:
- - build
+ - j8_build
- utests_long:
requires:
- start_utests_long
- start_utests_compression:
type: approval
requires:
- - build
+ - j8_build
- utests_compression:
requires:
- start_utests_compression
- start_utests_stress:
type: approval
requires:
- - build
+ - j8_build
- utests_stress:
requires:
- start_utests_stress
+ - start_utests_fqltool:
+ type: approval
+ requires:
+ - j8_build
+ - utests_fqltool:
+ requires:
+ - start_utests_fqltool
- start_jvm_upgrade_dtest:
type: approval
- j8_dtest_jars_build:
requires:
- - build
+ - j8_build
- start_jvm_upgrade_dtest
- j8_jvm_upgrade_dtests:
requires:
@@ -869,17 +2277,128 @@
- start_j8_dtests:
type: approval
requires:
- - build
+ - j8_build
- j8_dtests-with-vnodes:
requires:
- start_j8_dtests
- j8_dtests-no-vnodes:
requires:
- start_j8_dtests
+ - start_j11_dtests:
+ type: approval
+ requires:
+ - j8_build
+ - j11_dtests-with-vnodes:
+ requires:
+ - start_j11_dtests
+ - j11_dtests-no-vnodes:
+ requires:
+ - start_j11_dtests
- start_upgrade_tests:
type: approval
requires:
- - build
+ - j8_build
- j8_upgradetests-no-vnodes:
requires:
- start_upgrade_tests
+ - start_j8_cqlsh_tests-with-vnodes:
+ type: approval
+ requires:
+ - j8_build
+ - j8_cqlsh-dtests-py2-with-vnodes:
+ requires:
+ - start_j8_cqlsh_tests-with-vnodes
+ - j8_cqlsh-dtests-py3-with-vnodes:
+ requires:
+ - start_j8_cqlsh_tests-with-vnodes
+ - j8_cqlsh-dtests-py38-with-vnodes:
+ requires:
+ - start_j8_cqlsh_tests-with-vnodes
+ - start_j8_cqlsh_tests-no-vnodes:
+ type: approval
+ requires:
+ - j8_build
+ - j8_cqlsh-dtests-py2-no-vnodes:
+ requires:
+ - start_j8_cqlsh_tests-no-vnodes
+ - j8_cqlsh-dtests-py3-no-vnodes:
+ requires:
+ - start_j8_cqlsh_tests-no-vnodes
+ - j8_cqlsh-dtests-py38-no-vnodes:
+ requires:
+ - start_j8_cqlsh_tests-no-vnodes
+ - start_j11_cqlsh_tests-with-vnodes:
+ type: approval
+ requires:
+ - j8_build
+ - j11_cqlsh-dtests-py2-with-vnodes:
+ requires:
+ - start_j11_cqlsh_tests-with-vnodes
+ - j11_cqlsh-dtests-py3-with-vnodes:
+ requires:
+ - start_j11_cqlsh_tests-with-vnodes
+ - j11_cqlsh-dtests-py38-with-vnodes:
+ requires:
+ - start_j11_cqlsh_tests-with-vnodes
+ - start_j11_cqlsh_tests-no-vnodes:
+ type: approval
+ requires:
+ - j8_build
+ - j11_cqlsh-dtests-py2-no-vnodes:
+ requires:
+ - start_j11_cqlsh_tests-no-vnodes
+ - j11_cqlsh-dtests-py3-no-vnodes:
+ requires:
+ - start_j11_cqlsh_tests-no-vnodes
+ - j11_cqlsh-dtests-py38-no-vnodes:
+ requires:
+ - start_j11_cqlsh_tests-no-vnodes
+ java11_build_and_run_tests:
+ jobs:
+ - j11_build
+ - start_j11_unit_tests:
+ type: approval
+ requires:
+ - j11_build
+ - j11_unit_tests:
+ requires:
+ - start_j11_unit_tests
+ - j11_jvm_dtests:
+ requires:
+ - j11_build
+ - start_j11_dtests:
+ type: approval
+ requires:
+ - j11_build
+ - j11_dtests-with-vnodes:
+ requires:
+ - start_j11_dtests
+ - j11_dtests-no-vnodes:
+ requires:
+ - start_j11_dtests
+ - start_j11_cqlsh_tests-with-vnodes:
+ type: approval
+ requires:
+ - j11_build
+ - j11_cqlsh-dtests-py2-with-vnodes:
+ requires:
+ - start_j11_cqlsh_tests-with-vnodes
+ - j11_cqlsh-dtests-py3-with-vnodes:
+ requires:
+ - start_j11_cqlsh_tests-with-vnodes
+ - j11_cqlsh-dtests-py38-with-vnodes:
+ requires:
+ - start_j11_cqlsh_tests-with-vnodes
+ - start_j11_cqlsh_tests-no-vnodes:
+ type: approval
+ requires:
+ - j11_build
+ - j11_cqlsh-dtests-py2-no-vnodes:
+ requires:
+ - start_j11_cqlsh_tests-no-vnodes
+ - j11_cqlsh-dtests-py3-no-vnodes:
+ requires:
+ - start_j11_cqlsh_tests-no-vnodes
+ - j11_cqlsh-dtests-py38-no-vnodes:
+ requires:
+ - start_j11_cqlsh_tests-no-vnodes
diff --git a/.circleci/config.yml.LOWRES b/.circleci/config.yml.LOWRES
index 8123ab9..ffcb107 100644
--- a/.circleci/config.yml.LOWRES
+++ b/.circleci/config.yml.LOWRES
@@ -2,7 +2,7 @@
jobs:
j8_jvm_upgrade_dtests:
docker:
- - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
+ - image: nastra/cassandra-testing-ubuntu1910-java11-w-dependencies:20200603
resource_class: medium
working_directory: ~/
shell: /bin/bash -eo pipefail -l
@@ -76,7 +76,6 @@
path: /tmp/cassandra/build/test/logs
destination: logs
environment:
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
@@ -92,90 +91,9 @@
- CCM_HEAP_NEWSIZE: 256M
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- build:
+ j8_cqlsh-dtests-py2-with-vnodes:
docker:
- - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
- resource_class: medium
- working_directory: ~/
- shell: /bin/bash -eo pipefail -l
- parallelism: 1
- steps:
- - run:
- name: Log Environment Information
- command: |
- echo '*** id ***'
- id
- echo '*** cat /proc/cpuinfo ***'
- cat /proc/cpuinfo
- echo '*** free -m ***'
- free -m
- echo '*** df -m ***'
- df -m
- echo '*** ifconfig -a ***'
- ifconfig -a
- echo '*** uname -a ***'
- uname -a
- echo '*** mount ***'
- mount
- echo '*** env ***'
- env
- echo '*** java ***'
- which java
- java -version
- - run:
- name: Clone Cassandra Repository (via git)
- command: |
- git clone --single-branch --depth 1 --branch $CIRCLE_BRANCH git://github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME.git ~/cassandra
- - run:
- name: Build Cassandra
- command: |
- export PATH=$JAVA_HOME/bin:$PATH
- cd ~/cassandra
- # Loop to prevent failure due to maven-ant-tasks not downloading a jar..
- for x in $(seq 1 3); do
- ${ANT_HOME}/bin/ant clean jar
- RETURN="$?"
- if [ "${RETURN}" -eq "0" ]; then
- break
- fi
- done
- # Exit, if we didn't build successfully
- if [ "${RETURN}" -ne "0" ]; then
- echo "Build failed with exit code: ${RETURN}"
- exit ${RETURN}
- fi
- no_output_timeout: 15m
- - run:
- name: Run eclipse-warnings
- command: |
- export PATH=$JAVA_HOME/bin:$PATH
- cd ~/cassandra
- ant eclipse-warnings
- - persist_to_workspace:
- root: /home/cassandra
- paths:
- - cassandra
- - .m2
- environment:
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - ANT_HOME: /usr/share/ant
- - LANG: en_US.UTF-8
- - KEEP_TEST_DIR: true
- - DEFAULT_DIR: /home/cassandra/cassandra-dtest
- - PYTHONIOENCODING: utf-8
- - PYTHONUNBUFFERED: true
- - CASS_DRIVER_NO_EXTENSIONS: true
- - CASS_DRIVER_NO_CYTHON: true
- - CASSANDRA_SKIP_SYNC: true
- - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
- - DTEST_BRANCH: master
- - CCM_MAX_HEAP_SIZE: 1024M
- - CCM_HEAP_NEWSIZE: 256M
- - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_dtests-no-vnodes:
- docker:
- - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
+ - image: nastra/cassandra-testing-ubuntu1910-java11-w-dependencies:20200603
resource_class: medium
working_directory: ~/
shell: /bin/bash -eo pipefail -l
@@ -194,105 +112,26 @@
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
- source ~/env/bin/activate
+ source ~/env3.6/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
- pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
pip3 freeze
- run:
- name: Determine Tests to Run (j8_without_vnodes)
+ name: Determine Tests to Run (j8_with_vnodes)
no_output_timeout: 5m
- command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n"
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n"
- run:
- name: Run dtests (j8_without_vnodes)
- no_output_timeout: 15m
- command: "echo \"cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n"
- - store_test_results:
- path: /tmp/results
- - store_artifacts:
- path: /tmp/dtest
- destination: dtest_j8_without_vnodes
- - store_artifacts:
- path: ~/cassandra-dtest/logs
- destination: dtest_j8_without_vnodes_logs
- environment:
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - ANT_HOME: /usr/share/ant
- - LANG: en_US.UTF-8
- - KEEP_TEST_DIR: true
- - DEFAULT_DIR: /home/cassandra/cassandra-dtest
- - PYTHONIOENCODING: utf-8
- - PYTHONUNBUFFERED: true
- - CASS_DRIVER_NO_EXTENSIONS: true
- - CASS_DRIVER_NO_CYTHON: true
- - CASSANDRA_SKIP_SYNC: true
- - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
- - DTEST_BRANCH: master
- - CCM_MAX_HEAP_SIZE: 1024M
- - CCM_HEAP_NEWSIZE: 256M
- - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_upgradetests-no-vnodes:
- docker:
- - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
- resource_class: medium
- working_directory: ~/
- shell: /bin/bash -eo pipefail -l
- parallelism: 4
- steps:
- - attach_workspace:
- at: /home/cassandra
- - run:
- name: Clone Cassandra dtest Repository (via git)
- command: |
- git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
- - run:
- name: Configure virtualenv and python Dependencies
- command: |
- # note, this should be super quick as all dependencies should be pre-installed in the docker image
- # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
- # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
- # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
- source ~/env/bin/activate
- export PATH=$JAVA_HOME/bin:$PATH
- pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
- pip3 freeze
- - run:
- name: Determine Tests to Run (j8_upgradetests_without_vnodes)
- no_output_timeout: 5m
- command: |
- # reminder: this code (along with all the steps) is independently executed on every circle container
- # so the goal here is to get the circleci script to return the tests *this* container will run
- # which we do via the `circleci` cli tool.
-
- cd cassandra-dtest
- source ~/env/bin/activate
- export PATH=$JAVA_HOME/bin:$PATH
-
- if [ -n 'RUN_STATIC_UPGRADE_MATRIX=true' ]; then
- export RUN_STATIC_UPGRADE_MATRIX=true
- fi
-
- echo "***Collected DTests (j8_upgradetests_without_vnodes)***"
- set -eo pipefail && ./run_dtests.py --execute-upgrade-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw --cassandra-dir=../cassandra
- if [ -z '^upgrade_tests' ]; then
- mv /tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw /tmp/all_dtest_tests_j8_upgradetests_without_vnodes
- else
- grep -e '^upgrade_tests' /tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw > /tmp/all_dtest_tests_j8_upgradetests_without_vnodes || { echo "Filter did not match any tests! Exiting build."; exit 0; }
- fi
- set -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_upgradetests_without_vnodes > /tmp/split_dtest_tests_j8_upgradetests_without_vnodes.txt
- cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes.txt | tr '\n' ' ' > /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt
- cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt
- - run:
- name: Run dtests (j8_upgradetests_without_vnodes)
+ name: Run dtests (j8_with_vnodes)
no_output_timeout: 15m
command: |
- echo "cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt"
- cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt
+ echo "cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt
- source ~/env/bin/activate
+ source ~/env3.6/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
- if [ -n 'RUN_STATIC_UPGRADE_MATRIX=true' ]; then
- export RUN_STATIC_UPGRADE_MATRIX=true
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python2.7' ]; then
+ export CQLSH_PYTHON=/usr/bin/python2.7
fi
java -version
@@ -303,18 +142,17 @@
echo "** done env"
mkdir -p /tmp/results/dtests
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
- export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt`
- set -o pipefail && cd ~/cassandra-dtest && pytest --execute-upgrade-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j8_upgradetests_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt`
+ set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
- store_test_results:
path: /tmp/results
- store_artifacts:
path: /tmp/dtest
- destination: dtest_j8_upgradetests_without_vnodes
+ destination: dtest_j8_with_vnodes
- store_artifacts:
path: ~/cassandra-dtest/logs
- destination: dtest_j8_upgradetests_without_vnodes_logs
+ destination: dtest_j8_with_vnodes_logs
environment:
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
@@ -330,55 +168,9 @@
- CCM_HEAP_NEWSIZE: 256M
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- utests_stress:
+ j11_unit_tests:
docker:
- - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
- resource_class: medium
- working_directory: ~/
- shell: /bin/bash -eo pipefail -l
- parallelism: 1
- steps:
- - attach_workspace:
- at: /home/cassandra
- - run:
- name: Run Unit Tests (stress-test)
- command: |
- export PATH=$JAVA_HOME/bin:$PATH
- time mv ~/cassandra /tmp
- cd /tmp/cassandra
- if [ -d ~/dtest_jars ]; then
- cp ~/dtest_jars/dtest* /tmp/cassandra/build/
- fi
- ant stress-test -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
- no_output_timeout: 15m
- - store_test_results:
- path: /tmp/cassandra/build/test/output/
- - store_artifacts:
- path: /tmp/cassandra/build/test/output
- destination: junitxml
- - store_artifacts:
- path: /tmp/cassandra/build/test/logs
- destination: logs
- environment:
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - ANT_HOME: /usr/share/ant
- - LANG: en_US.UTF-8
- - KEEP_TEST_DIR: true
- - DEFAULT_DIR: /home/cassandra/cassandra-dtest
- - PYTHONIOENCODING: utf-8
- - PYTHONUNBUFFERED: true
- - CASS_DRIVER_NO_EXTENSIONS: true
- - CASS_DRIVER_NO_CYTHON: true
- - CASSANDRA_SKIP_SYNC: true
- - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
- - DTEST_BRANCH: master
- - CCM_MAX_HEAP_SIZE: 1024M
- - CCM_HEAP_NEWSIZE: 256M
- - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_unit_tests:
- docker:
- - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
+ - image: nastra/cassandra-testing-ubuntu1910-java11:20200603
resource_class: medium
working_directory: ~/
shell: /bin/bash -eo pipefail -l
@@ -452,7 +244,6 @@
path: /tmp/cassandra/build/test/logs
destination: logs
environment:
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
@@ -466,11 +257,12 @@
- DTEST_BRANCH: master
- CCM_MAX_HEAP_SIZE: 1024M
- CCM_HEAP_NEWSIZE: 256M
- - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_dtests-with-vnodes:
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j8_cqlsh-dtests-py38-no-vnodes:
docker:
- - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
+ - image: nastra/cassandra-testing-ubuntu1910-java11-w-dependencies:20200603
resource_class: medium
working_directory: ~/
shell: /bin/bash -eo pipefail -l
@@ -489,28 +281,47 @@
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
- source ~/env/bin/activate
+ source ~/env3.8/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
- pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
pip3 freeze
- run:
- name: Determine Tests to Run (j8_with_vnodes)
+ name: Determine Tests to Run (j8_without_vnodes)
no_output_timeout: 5m
- command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n"
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n"
- run:
- name: Run dtests (j8_with_vnodes)
+ name: Run dtests (j8_without_vnodes)
no_output_timeout: 15m
- command: "echo \"cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n"
+ command: |
+ echo "cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt
+
+ source ~/env3.8/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.8' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.8
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt`
+ set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
- store_test_results:
path: /tmp/results
- store_artifacts:
path: /tmp/dtest
- destination: dtest_j8_with_vnodes
+ destination: dtest_j8_without_vnodes
- store_artifacts:
path: ~/cassandra-dtest/logs
- destination: dtest_j8_with_vnodes_logs
+ destination: dtest_j8_without_vnodes_logs
environment:
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
@@ -526,9 +337,849 @@
- CCM_HEAP_NEWSIZE: 256M
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_jvm_dtests:
+ j11_cqlsh-dtests-py3-with-vnodes:
docker:
- - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
+ - image: nastra/cassandra-testing-ubuntu1910-java11:20200603
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_with_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_with_vnodes_raw /tmp/all_dtest_tests_j11_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_with_vnodes_raw > /tmp/all_dtest_tests_j11_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_with_vnodes > /tmp/split_dtest_tests_j11_with_vnodes.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j11_with_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.6
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt`
+ set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j11_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_with_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_with_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j11_cqlsh-dtests-py3-no-vnodes:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11:20200603
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_without_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_without_vnodes_raw /tmp/all_dtest_tests_j11_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_without_vnodes_raw > /tmp/all_dtest_tests_j11_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_without_vnodes > /tmp/split_dtest_tests_j11_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j11_without_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.6
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt`
+ set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j11_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_without_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j11_cqlsh-dtests-py38-with-vnodes:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11:20200603
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.8/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_with_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_with_vnodes_raw /tmp/all_dtest_tests_j11_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_with_vnodes_raw > /tmp/all_dtest_tests_j11_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_with_vnodes > /tmp/split_dtest_tests_j11_with_vnodes.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j11_with_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt
+
+ source ~/env3.8/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.8' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.8
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt`
+ set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j11_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_with_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_with_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j8_cqlsh-dtests-py3-with-vnodes:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11-w-dependencies:20200603
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j8_with_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j8_with_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.6
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt`
+ set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j8_with_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j8_with_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j8_cqlsh-dtests-py2-no-vnodes:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11-w-dependencies:20200603
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j8_without_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j8_without_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python2.7' ]; then
+ export CQLSH_PYTHON=/usr/bin/python2.7
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt`
+ set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j8_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j8_without_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j11_cqlsh-dtests-py2-with-vnodes:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11:20200603
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_with_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_with_vnodes_raw /tmp/all_dtest_tests_j11_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_with_vnodes_raw > /tmp/all_dtest_tests_j11_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_with_vnodes > /tmp/split_dtest_tests_j11_with_vnodes.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j11_with_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python2.7' ]; then
+ export CQLSH_PYTHON=/usr/bin/python2.7
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt`
+ set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j11_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_with_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_with_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j11_dtests-with-vnodes:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11:20200603
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_with_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k not cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_with_vnodes_raw /tmp/all_dtest_tests_j11_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_with_vnodes_raw > /tmp/all_dtest_tests_j11_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_with_vnodes > /tmp/split_dtest_tests_j11_with_vnodes.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j11_with_vnodes)
+ no_output_timeout: 15m
+ command: "echo \"cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j11_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n"
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_with_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_with_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j8_dtests-no-vnodes:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11-w-dependencies:20200603
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j8_without_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k not cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j8_without_vnodes)
+ no_output_timeout: 15m
+ command: "echo \"cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n"
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j8_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j8_without_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j8_upgradetests-no-vnodes:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11-w-dependencies:20200603
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j8_upgradetests_without_vnodes)
+ no_output_timeout: 5m
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ cd cassandra-dtest
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+
+ if [ -n 'RUN_STATIC_UPGRADE_MATRIX=true' ]; then
+ export RUN_STATIC_UPGRADE_MATRIX=true
+ fi
+
+ echo "***Collected DTests (j8_upgradetests_without_vnodes)***"
+ set -eo pipefail && ./run_dtests.py --execute-upgrade-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw --cassandra-dir=../cassandra
+ if [ -z '^upgrade_tests' ]; then
+ mv /tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw /tmp/all_dtest_tests_j8_upgradetests_without_vnodes
+ else
+ grep -e '^upgrade_tests' /tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw > /tmp/all_dtest_tests_j8_upgradetests_without_vnodes || { echo "Filter did not match any tests! Exiting build."; exit 0; }
+ fi
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_upgradetests_without_vnodes > /tmp/split_dtest_tests_j8_upgradetests_without_vnodes.txt
+ cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes.txt | tr '\n' ' ' > /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt
+ cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt
+ - run:
+ name: Run dtests (j8_upgradetests_without_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'RUN_STATIC_UPGRADE_MATRIX=true' ]; then
+ export RUN_STATIC_UPGRADE_MATRIX=true
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt`
+ set -o pipefail && cd ~/cassandra-dtest && pytest --execute-upgrade-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j8_upgradetests_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j8_upgradetests_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j8_upgradetests_without_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ utests_stress:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11-w-dependencies:20200603
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Run Unit Tests (stress-test)
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ ant stress-test
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j8_unit_tests:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11-w-dependencies:20200603
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine unit Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
+ ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j11_jvm_dtests:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11:20200603
resource_class: medium
working_directory: ~/
shell: /bin/bash -eo pipefail -l
@@ -602,7 +1253,627 @@
path: /tmp/cassandra/build/test/logs
destination: logs
environment:
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j11_build:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11:20200603
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Clone Cassandra Repository (via git)
+ command: |
+ git clone --single-branch --depth 1 --branch $CIRCLE_BRANCH git://github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME.git ~/cassandra
+ - run:
+ name: Build Cassandra
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ cd ~/cassandra
+ # Loop to prevent failure due to maven-ant-tasks not downloading a jar..
+ for x in $(seq 1 3); do
+ ${ANT_HOME}/bin/ant clean realclean jar
+ RETURN="$?"
+ if [ "${RETURN}" -eq "0" ]; then
+ break
+ fi
+ done
+ # Exit, if we didn't build successfully
+ if [ "${RETURN}" -ne "0" ]; then
+ echo "Build failed with exit code: ${RETURN}"
+ exit ${RETURN}
+ fi
+ no_output_timeout: 15m
+ - run:
+ name: Run eclipse-warnings
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ cd ~/cassandra
+ ant eclipse-warnings
+ - persist_to_workspace:
+ root: /home/cassandra
+ paths:
+ - cassandra
+ - .m2
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j11_cqlsh-dtests-py2-no-vnodes:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11:20200603
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_without_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_without_vnodes_raw /tmp/all_dtest_tests_j11_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_without_vnodes_raw > /tmp/all_dtest_tests_j11_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_without_vnodes > /tmp/split_dtest_tests_j11_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j11_without_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python2.7' ]; then
+ export CQLSH_PYTHON=/usr/bin/python2.7
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt`
+ set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j11_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_without_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j8_dtests-with-vnodes:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11-w-dependencies:20200603
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j8_with_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k not cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j8_with_vnodes)
+ no_output_timeout: 15m
+ command: "echo \"cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n"
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j8_with_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j8_with_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j11_cqlsh-dtests-py38-no-vnodes:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11:20200603
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.8/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_without_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_without_vnodes_raw /tmp/all_dtest_tests_j11_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_without_vnodes_raw > /tmp/all_dtest_tests_j11_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_without_vnodes > /tmp/split_dtest_tests_j11_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j11_without_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt
+
+ source ~/env3.8/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.8' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.8
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt`
+ set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j11_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_without_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j8_jvm_dtests:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11-w-dependencies:20200603
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine distributed Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/distributed/;;g" | grep "Test\.java$" | grep -v upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.distributed.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
+ ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j8_build:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11-w-dependencies:20200603
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Clone Cassandra Repository (via git)
+ command: |
+ git clone --single-branch --depth 1 --branch $CIRCLE_BRANCH git://github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME.git ~/cassandra
+ - run:
+ name: Build Cassandra
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ cd ~/cassandra
+ # Loop to prevent failure due to maven-ant-tasks not downloading a jar..
+ for x in $(seq 1 3); do
+ ${ANT_HOME}/bin/ant clean realclean jar
+ RETURN="$?"
+ if [ "${RETURN}" -eq "0" ]; then
+ break
+ fi
+ done
+ # Exit, if we didn't build successfully
+ if [ "${RETURN}" -ne "0" ]; then
+ echo "Build failed with exit code: ${RETURN}"
+ exit ${RETURN}
+ fi
+ no_output_timeout: 15m
+ - run:
+ name: Run eclipse-warnings
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ cd ~/cassandra
+ ant eclipse-warnings
+ - persist_to_workspace:
+ root: /home/cassandra
+ paths:
+ - cassandra
+ - .m2
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j8_cqlsh-dtests-py3-no-vnodes:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11-w-dependencies:20200603
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j8_without_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j8_without_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.6
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt`
+ set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j8_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j8_without_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j8_cqlsh-dtests-py38-with-vnodes:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11-w-dependencies:20200603
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.8/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j8_with_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j8_with_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt
+
+ source ~/env3.8/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.8' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.8
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt`
+ set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j8_with_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j8_with_vnodes_logs
+ environment:
- ANT_HOME: /usr/share/ant
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
@@ -620,7 +1891,7 @@
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
utests_long:
docker:
- - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
+ - image: nastra/cassandra-testing-ubuntu1910-java11-w-dependencies:20200603
resource_class: medium
working_directory: ~/
shell: /bin/bash -eo pipefail -l
@@ -637,7 +1908,7 @@
if [ -d ~/dtest_jars ]; then
cp ~/dtest_jars/dtest* /tmp/cassandra/build/
fi
- ant long-test -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
+ ant long-test
no_output_timeout: 15m
- store_test_results:
path: /tmp/cassandra/build/test/output/
@@ -648,7 +1919,6 @@
path: /tmp/cassandra/build/test/logs
destination: logs
environment:
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
@@ -664,9 +1934,135 @@
- CCM_HEAP_NEWSIZE: 256M
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ utests_fqltool:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11-w-dependencies:20200603
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Run Unit Tests (fqltool-test)
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ ant fqltool-test
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j11_dtests-no-vnodes:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11:20200603
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_without_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k not cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_without_vnodes_raw /tmp/all_dtest_tests_j11_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_without_vnodes_raw > /tmp/all_dtest_tests_j11_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_without_vnodes > /tmp/split_dtest_tests_j11_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j11_without_vnodes)
+ no_output_timeout: 15m
+ command: "echo \"cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j11_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n"
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_without_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
utests_compression:
docker:
- - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
+ - image: nastra/cassandra-testing-ubuntu1910-java11-w-dependencies:20200603
resource_class: medium
working_directory: ~/
shell: /bin/bash -eo pipefail -l
@@ -740,7 +2136,6 @@
path: /tmp/cassandra/build/test/logs
destination: logs
environment:
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
@@ -758,7 +2153,7 @@
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_dtest_jars_build:
docker:
- - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
+ - image: nastra/cassandra-testing-ubuntu1910-java11-w-dependencies:20200603
resource_class: medium
working_directory: ~/
shell: /bin/bash -eo pipefail -l
@@ -809,7 +2204,6 @@
paths:
- dtest_jars
environment:
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
@@ -827,41 +2221,55 @@
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
workflows:
version: 2
- build_and_run_tests:
+ java8_build_and_run_tests:
jobs:
- - build
+ - j8_build
- j8_unit_tests:
requires:
- - build
+ - j8_build
- j8_jvm_dtests:
requires:
- - build
+ - j8_build
+ - start_j11_unit_tests:
+ type: approval
+ requires:
+ - j8_build
+ - j11_unit_tests:
+ requires:
+ - start_j11_unit_tests
- start_utests_long:
type: approval
requires:
- - build
+ - j8_build
- utests_long:
requires:
- start_utests_long
- start_utests_compression:
type: approval
requires:
- - build
+ - j8_build
- utests_compression:
requires:
- start_utests_compression
- start_utests_stress:
type: approval
requires:
- - build
+ - j8_build
- utests_stress:
requires:
- start_utests_stress
+ - start_utests_fqltool:
+ type: approval
+ requires:
+ - j8_build
+ - utests_fqltool:
+ requires:
+ - start_utests_fqltool
- start_jvm_upgrade_dtest:
type: approval
- j8_dtest_jars_build:
requires:
- - build
+ - j8_build
- start_jvm_upgrade_dtest
- j8_jvm_upgrade_dtests:
requires:
@@ -869,17 +2277,128 @@
- start_j8_dtests:
type: approval
requires:
- - build
+ - j8_build
- j8_dtests-with-vnodes:
requires:
- start_j8_dtests
- j8_dtests-no-vnodes:
requires:
- start_j8_dtests
+ - start_j11_dtests:
+ type: approval
+ requires:
+ - j8_build
+ - j11_dtests-with-vnodes:
+ requires:
+ - start_j11_dtests
+ - j11_dtests-no-vnodes:
+ requires:
+ - start_j11_dtests
- start_upgrade_tests:
type: approval
requires:
- - build
+ - j8_build
- j8_upgradetests-no-vnodes:
requires:
- start_upgrade_tests
+ - start_j8_cqlsh_tests-with-vnodes:
+ type: approval
+ requires:
+ - j8_build
+ - j8_cqlsh-dtests-py2-with-vnodes:
+ requires:
+ - start_j8_cqlsh_tests-with-vnodes
+ - j8_cqlsh-dtests-py3-with-vnodes:
+ requires:
+ - start_j8_cqlsh_tests-with-vnodes
+ - j8_cqlsh-dtests-py38-with-vnodes:
+ requires:
+ - start_j8_cqlsh_tests-with-vnodes
+ - start_j8_cqlsh_tests-no-vnodes:
+ type: approval
+ requires:
+ - j8_build
+ - j8_cqlsh-dtests-py2-no-vnodes:
+ requires:
+ - start_j8_cqlsh_tests-no-vnodes
+ - j8_cqlsh-dtests-py3-no-vnodes:
+ requires:
+ - start_j8_cqlsh_tests-no-vnodes
+ - j8_cqlsh-dtests-py38-no-vnodes:
+ requires:
+ - start_j8_cqlsh_tests-no-vnodes
+ - start_j11_cqlsh_tests-with-vnodes:
+ type: approval
+ requires:
+ - j8_build
+ - j11_cqlsh-dtests-py2-with-vnodes:
+ requires:
+ - start_j11_cqlsh_tests-with-vnodes
+ - j11_cqlsh-dtests-py3-with-vnodes:
+ requires:
+ - start_j11_cqlsh_tests-with-vnodes
+ - j11_cqlsh-dtests-py38-with-vnodes:
+ requires:
+ - start_j11_cqlsh_tests-with-vnodes
+ - start_j11_cqlsh_tests-no-vnodes:
+ type: approval
+ requires:
+ - j8_build
+ - j11_cqlsh-dtests-py2-no-vnodes:
+ requires:
+ - start_j11_cqlsh_tests-no-vnodes
+ - j11_cqlsh-dtests-py3-no-vnodes:
+ requires:
+ - start_j11_cqlsh_tests-no-vnodes
+ - j11_cqlsh-dtests-py38-no-vnodes:
+ requires:
+ - start_j11_cqlsh_tests-no-vnodes
+ java11_build_and_run_tests:
+ jobs:
+ - j11_build
+ - start_j11_unit_tests:
+ type: approval
+ requires:
+ - j11_build
+ - j11_unit_tests:
+ requires:
+ - start_j11_unit_tests
+ - j11_jvm_dtests:
+ requires:
+ - j11_build
+ - start_j11_dtests:
+ type: approval
+ requires:
+ - j11_build
+ - j11_dtests-with-vnodes:
+ requires:
+ - start_j11_dtests
+ - j11_dtests-no-vnodes:
+ requires:
+ - start_j11_dtests
+ - start_j11_cqlsh_tests-with-vnodes:
+ type: approval
+ requires:
+ - j11_build
+ - j11_cqlsh-dtests-py2-with-vnodes:
+ requires:
+ - start_j11_cqlsh_tests-with-vnodes
+ - j11_cqlsh-dtests-py3-with-vnodes:
+ requires:
+ - start_j11_cqlsh_tests-with-vnodes
+ - j11_cqlsh-dtests-py38-with-vnodes:
+ requires:
+ - start_j11_cqlsh_tests-with-vnodes
+ - start_j11_cqlsh_tests-no-vnodes:
+ type: approval
+ requires:
+ - j11_build
+ - j11_cqlsh-dtests-py2-no-vnodes:
+ requires:
+ - start_j11_cqlsh_tests-no-vnodes
+ - j11_cqlsh-dtests-py3-no-vnodes:
+ requires:
+ - start_j11_cqlsh_tests-no-vnodes
+ - j11_cqlsh-dtests-py38-no-vnodes:
+ requires:
+ - start_j11_cqlsh_tests-no-vnodes
diff --git a/.circleci/config.yml.MIDRES b/.circleci/config.yml.MIDRES
new file mode 100644
index 0000000..8eca9ef
--- /dev/null
+++ b/.circleci/config.yml.MIDRES
@@ -0,0 +1,2404 @@
+version: 2
+jobs:
+ j8_jvm_upgrade_dtests:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11-w-dependencies:20200603
+ resource_class: large
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 10
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine distributed Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/distributed/;;g" | grep "Test\.java$" | grep upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.distributed.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
+ ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j8_cqlsh-dtests-py2-with-vnodes:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11-w-dependencies:20200603
+ resource_class: large
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 50
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j8_with_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j8_with_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python2.7' ]; then
+ export CQLSH_PYTHON=/usr/bin/python2.7
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt`
+ set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j8_with_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j8_with_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j11_unit_tests:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11:20200603
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 25
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine unit Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
+ ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j8_cqlsh-dtests-py38-no-vnodes:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11-w-dependencies:20200603
+ resource_class: large
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 50
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.8/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j8_without_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j8_without_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt
+
+ source ~/env3.8/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.8' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.8
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt`
+ set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j8_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j8_without_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j11_cqlsh-dtests-py3-with-vnodes:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11:20200603
+ resource_class: large
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 50
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_with_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_with_vnodes_raw /tmp/all_dtest_tests_j11_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_with_vnodes_raw > /tmp/all_dtest_tests_j11_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_with_vnodes > /tmp/split_dtest_tests_j11_with_vnodes.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j11_with_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.6
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt`
+ set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j11_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_with_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_with_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j11_cqlsh-dtests-py3-no-vnodes:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11:20200603
+ resource_class: large
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 50
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_without_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_without_vnodes_raw /tmp/all_dtest_tests_j11_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_without_vnodes_raw > /tmp/all_dtest_tests_j11_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_without_vnodes > /tmp/split_dtest_tests_j11_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j11_without_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.6
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt`
+ set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j11_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_without_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j11_cqlsh-dtests-py38-with-vnodes:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11:20200603
+ resource_class: large
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 50
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.8/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_with_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_with_vnodes_raw /tmp/all_dtest_tests_j11_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_with_vnodes_raw > /tmp/all_dtest_tests_j11_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_with_vnodes > /tmp/split_dtest_tests_j11_with_vnodes.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j11_with_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt
+
+ source ~/env3.8/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.8' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.8
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt`
+ set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j11_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_with_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_with_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j8_cqlsh-dtests-py3-with-vnodes:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11-w-dependencies:20200603
+ resource_class: large
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 50
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j8_with_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j8_with_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.6
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt`
+ set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j8_with_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j8_with_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j8_cqlsh-dtests-py2-no-vnodes:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11-w-dependencies:20200603
+ resource_class: large
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 50
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j8_without_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j8_without_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python2.7' ]; then
+ export CQLSH_PYTHON=/usr/bin/python2.7
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt`
+ set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j8_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j8_without_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j11_cqlsh-dtests-py2-with-vnodes:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11:20200603
+ resource_class: large
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 50
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_with_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_with_vnodes_raw /tmp/all_dtest_tests_j11_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_with_vnodes_raw > /tmp/all_dtest_tests_j11_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_with_vnodes > /tmp/split_dtest_tests_j11_with_vnodes.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j11_with_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python2.7' ]; then
+ export CQLSH_PYTHON=/usr/bin/python2.7
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt`
+ set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j11_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_with_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_with_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j11_dtests-with-vnodes:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11:20200603
+ resource_class: large
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 50
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_with_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k not cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_with_vnodes_raw /tmp/all_dtest_tests_j11_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_with_vnodes_raw > /tmp/all_dtest_tests_j11_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_with_vnodes > /tmp/split_dtest_tests_j11_with_vnodes.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j11_with_vnodes)
+ no_output_timeout: 15m
+ command: "echo \"cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j11_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n"
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_with_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_with_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j8_dtests-no-vnodes:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11-w-dependencies:20200603
+ resource_class: large
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 50
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j8_without_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k not cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j8_without_vnodes)
+ no_output_timeout: 15m
+ command: "echo \"cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n"
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j8_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j8_without_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j8_upgradetests-no-vnodes:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11-w-dependencies:20200603
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 100
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j8_upgradetests_without_vnodes)
+ no_output_timeout: 5m
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ cd cassandra-dtest
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+
+ if [ -n 'RUN_STATIC_UPGRADE_MATRIX=true' ]; then
+ export RUN_STATIC_UPGRADE_MATRIX=true
+ fi
+
+ echo "***Collected DTests (j8_upgradetests_without_vnodes)***"
+ set -eo pipefail && ./run_dtests.py --execute-upgrade-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw --cassandra-dir=../cassandra
+ if [ -z '^upgrade_tests' ]; then
+ mv /tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw /tmp/all_dtest_tests_j8_upgradetests_without_vnodes
+ else
+ grep -e '^upgrade_tests' /tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw > /tmp/all_dtest_tests_j8_upgradetests_without_vnodes || { echo "Filter did not match any tests! Exiting build."; exit 0; }
+ fi
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_upgradetests_without_vnodes > /tmp/split_dtest_tests_j8_upgradetests_without_vnodes.txt
+ cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes.txt | tr '\n' ' ' > /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt
+ cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt
+ - run:
+ name: Run dtests (j8_upgradetests_without_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'RUN_STATIC_UPGRADE_MATRIX=true' ]; then
+ export RUN_STATIC_UPGRADE_MATRIX=true
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt`
+ set -o pipefail && cd ~/cassandra-dtest && pytest --execute-upgrade-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j8_upgradetests_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j8_upgradetests_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j8_upgradetests_without_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ utests_stress:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11-w-dependencies:20200603
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Run Unit Tests (stress-test)
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ ant stress-test
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j8_unit_tests:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11-w-dependencies:20200603
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 25
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine unit Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
+ ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j11_jvm_dtests:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11:20200603
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine distributed Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/distributed/;;g" | grep "Test\.java$" | grep -v upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.distributed.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
+ ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j11_build:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11:20200603
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Clone Cassandra Repository (via git)
+ command: |
+ git clone --single-branch --depth 1 --branch $CIRCLE_BRANCH git://github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME.git ~/cassandra
+ - run:
+ name: Build Cassandra
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ cd ~/cassandra
+ # Loop to prevent failure due to maven-ant-tasks not downloading a jar..
+ for x in $(seq 1 3); do
+ ${ANT_HOME}/bin/ant clean realclean jar
+ RETURN="$?"
+ if [ "${RETURN}" -eq "0" ]; then
+ break
+ fi
+ done
+ # Exit, if we didn't build successfully
+ if [ "${RETURN}" -ne "0" ]; then
+ echo "Build failed with exit code: ${RETURN}"
+ exit ${RETURN}
+ fi
+ no_output_timeout: 15m
+ - run:
+ name: Run eclipse-warnings
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ cd ~/cassandra
+ ant eclipse-warnings
+ - persist_to_workspace:
+ root: /home/cassandra
+ paths:
+ - cassandra
+ - .m2
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j11_cqlsh-dtests-py2-no-vnodes:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11:20200603
+ resource_class: large
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 50
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_without_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_without_vnodes_raw /tmp/all_dtest_tests_j11_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_without_vnodes_raw > /tmp/all_dtest_tests_j11_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_without_vnodes > /tmp/split_dtest_tests_j11_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j11_without_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python2.7' ]; then
+ export CQLSH_PYTHON=/usr/bin/python2.7
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt`
+ set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j11_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_without_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j8_dtests-with-vnodes:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11-w-dependencies:20200603
+ resource_class: large
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 50
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j8_with_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k not cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j8_with_vnodes)
+ no_output_timeout: 15m
+ command: "echo \"cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n"
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j8_with_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j8_with_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j11_cqlsh-dtests-py38-no-vnodes:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11:20200603
+ resource_class: large
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 50
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.8/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_without_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_without_vnodes_raw /tmp/all_dtest_tests_j11_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_without_vnodes_raw > /tmp/all_dtest_tests_j11_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_without_vnodes > /tmp/split_dtest_tests_j11_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j11_without_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt
+
+ source ~/env3.8/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.8' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.8
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt`
+ set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j11_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_without_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j8_jvm_dtests:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11-w-dependencies:20200603
+ resource_class: large
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 10
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine distributed Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/distributed/;;g" | grep "Test\.java$" | grep -v upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.distributed.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
+ ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j8_build:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11-w-dependencies:20200603
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Clone Cassandra Repository (via git)
+ command: |
+ git clone --single-branch --depth 1 --branch $CIRCLE_BRANCH git://github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME.git ~/cassandra
+ - run:
+ name: Build Cassandra
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ cd ~/cassandra
+ # Loop to prevent failure due to maven-ant-tasks not downloading a jar..
+ for x in $(seq 1 3); do
+ ${ANT_HOME}/bin/ant clean realclean jar
+ RETURN="$?"
+ if [ "${RETURN}" -eq "0" ]; then
+ break
+ fi
+ done
+ # Exit, if we didn't build successfully
+ if [ "${RETURN}" -ne "0" ]; then
+ echo "Build failed with exit code: ${RETURN}"
+ exit ${RETURN}
+ fi
+ no_output_timeout: 15m
+ - run:
+ name: Run eclipse-warnings
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ cd ~/cassandra
+ ant eclipse-warnings
+ - persist_to_workspace:
+ root: /home/cassandra
+ paths:
+ - cassandra
+ - .m2
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j8_cqlsh-dtests-py3-no-vnodes:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11-w-dependencies:20200603
+ resource_class: large
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 50
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j8_without_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j8_without_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.6
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt`
+ set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j8_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j8_without_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j8_cqlsh-dtests-py38-with-vnodes:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11-w-dependencies:20200603
+ resource_class: large
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 50
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.8/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j8_with_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j8_with_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt
+
+ source ~/env3.8/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.8' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.8
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt`
+ set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j8_with_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j8_with_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ utests_long:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11-w-dependencies:20200603
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Run Unit Tests (long-test)
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ ant long-test
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ utests_fqltool:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11-w-dependencies:20200603
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Run Unit Tests (fqltool-test)
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ ant fqltool-test
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j11_dtests-no-vnodes:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11:20200603
+ resource_class: large
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 50
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_without_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k not cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_without_vnodes_raw /tmp/all_dtest_tests_j11_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_without_vnodes_raw > /tmp/all_dtest_tests_j11_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_without_vnodes > /tmp/split_dtest_tests_j11_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j11_without_vnodes)
+ no_output_timeout: 15m
+ command: "echo \"cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j11_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n"
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_without_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ utests_compression:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11-w-dependencies:20200603
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 25
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine unit Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist-compression)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
+ ant testclasslist-compression -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j8_dtest_jars_build:
+ docker:
+ - image: nastra/cassandra-testing-ubuntu1910-java11-w-dependencies:20200603
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Build Cassandra DTest jars
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ cd ~/cassandra
+ git remote add apache git://github.com/apache/cassandra.git
+ for branch in cassandra-2.2 cassandra-3.0 cassandra-3.11 trunk; do
+ # check out the correct cassandra version:
+ git remote set-branches --add apache '$branch'
+ git fetch --depth 1 apache $branch
+ git checkout $branch
+ # Loop to prevent failure due to maven-ant-tasks not downloading a jar..
+ for x in $(seq 1 3); do
+ ${ANT_HOME}/bin/ant clean jar dtest-jar
+ RETURN="$?"
+ if [ "${RETURN}" -eq "0" ]; then
+ break
+ fi
+ done
+ # Exit, if we didn't build successfully
+ if [ "${RETURN}" -ne "0" ]; then
+ echo "Build failed with exit code: ${RETURN}"
+ exit ${RETURN}
+ fi
+ done
+ # and build the dtest-jar for the branch under test
+ git checkout origin/$CIRCLE_BRANCH
+ for x in $(seq 1 3); do
+ ${ANT_HOME}/bin/ant clean jar dtest-jar
+ RETURN="$?"
+ if [ "${RETURN}" -eq "0" ]; then
+ break
+ fi
+ done
+ mkdir ~/dtest_jars
+ cp build/dtest*.jar ~/dtest_jars
+ ls -l ~/dtest_jars
+ no_output_timeout: 15m
+ - persist_to_workspace:
+ root: /home/cassandra
+ paths:
+ - dtest_jars
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+workflows:
+ version: 2
+ java8_build_and_run_tests:
+ jobs:
+ - j8_build
+ - j8_unit_tests:
+ requires:
+ - j8_build
+ - j8_jvm_dtests:
+ requires:
+ - j8_build
+ - start_j11_unit_tests:
+ type: approval
+ requires:
+ - j8_build
+ - j11_unit_tests:
+ requires:
+ - start_j11_unit_tests
+ - start_utests_long:
+ type: approval
+ requires:
+ - j8_build
+ - utests_long:
+ requires:
+ - start_utests_long
+ - start_utests_compression:
+ type: approval
+ requires:
+ - j8_build
+ - utests_compression:
+ requires:
+ - start_utests_compression
+ - start_utests_stress:
+ type: approval
+ requires:
+ - j8_build
+ - utests_stress:
+ requires:
+ - start_utests_stress
+ - start_utests_fqltool:
+ type: approval
+ requires:
+ - j8_build
+ - utests_fqltool:
+ requires:
+ - start_utests_fqltool
+ - start_jvm_upgrade_dtest:
+ type: approval
+ - j8_dtest_jars_build:
+ requires:
+ - j8_build
+ - start_jvm_upgrade_dtest
+ - j8_jvm_upgrade_dtests:
+ requires:
+ - j8_dtest_jars_build
+ - start_j8_dtests:
+ type: approval
+ requires:
+ - j8_build
+ - j8_dtests-with-vnodes:
+ requires:
+ - start_j8_dtests
+ - j8_dtests-no-vnodes:
+ requires:
+ - start_j8_dtests
+ - start_j11_dtests:
+ type: approval
+ requires:
+ - j8_build
+ - j11_dtests-with-vnodes:
+ requires:
+ - start_j11_dtests
+ - j11_dtests-no-vnodes:
+ requires:
+ - start_j11_dtests
+ - start_upgrade_tests:
+ type: approval
+ requires:
+ - j8_build
+ - j8_upgradetests-no-vnodes:
+ requires:
+ - start_upgrade_tests
+ - start_j8_cqlsh_tests-with-vnodes:
+ type: approval
+ requires:
+ - j8_build
+ - j8_cqlsh-dtests-py2-with-vnodes:
+ requires:
+ - start_j8_cqlsh_tests-with-vnodes
+ - j8_cqlsh-dtests-py3-with-vnodes:
+ requires:
+ - start_j8_cqlsh_tests-with-vnodes
+ - j8_cqlsh-dtests-py38-with-vnodes:
+ requires:
+ - start_j8_cqlsh_tests-with-vnodes
+ - start_j8_cqlsh_tests-no-vnodes:
+ type: approval
+ requires:
+ - j8_build
+ - j8_cqlsh-dtests-py2-no-vnodes:
+ requires:
+ - start_j8_cqlsh_tests-no-vnodes
+ - j8_cqlsh-dtests-py3-no-vnodes:
+ requires:
+ - start_j8_cqlsh_tests-no-vnodes
+ - j8_cqlsh-dtests-py38-no-vnodes:
+ requires:
+ - start_j8_cqlsh_tests-no-vnodes
+ - start_j11_cqlsh_tests-with-vnodes:
+ type: approval
+ requires:
+ - j8_build
+ - j11_cqlsh-dtests-py2-with-vnodes:
+ requires:
+ - start_j11_cqlsh_tests-with-vnodes
+ - j11_cqlsh-dtests-py3-with-vnodes:
+ requires:
+ - start_j11_cqlsh_tests-with-vnodes
+ - j11_cqlsh-dtests-py38-with-vnodes:
+ requires:
+ - start_j11_cqlsh_tests-with-vnodes
+ - start_j11_cqlsh_tests-no-vnodes:
+ type: approval
+ requires:
+ - j8_build
+ - j11_cqlsh-dtests-py2-no-vnodes:
+ requires:
+ - start_j11_cqlsh_tests-no-vnodes
+ - j11_cqlsh-dtests-py3-no-vnodes:
+ requires:
+ - start_j11_cqlsh_tests-no-vnodes
+ - j11_cqlsh-dtests-py38-no-vnodes:
+ requires:
+ - start_j11_cqlsh_tests-no-vnodes
+ java11_build_and_run_tests:
+ jobs:
+ - j11_build
+ - start_j11_unit_tests:
+ type: approval
+ requires:
+ - j11_build
+ - j11_unit_tests:
+ requires:
+ - start_j11_unit_tests
+ - j11_jvm_dtests:
+ requires:
+ - j11_build
+ - start_j11_dtests:
+ type: approval
+ requires:
+ - j11_build
+ - j11_dtests-with-vnodes:
+ requires:
+ - start_j11_dtests
+ - j11_dtests-no-vnodes:
+ requires:
+ - start_j11_dtests
+ - start_j11_cqlsh_tests-with-vnodes:
+ type: approval
+ requires:
+ - j11_build
+ - j11_cqlsh-dtests-py2-with-vnodes:
+ requires:
+ - start_j11_cqlsh_tests-with-vnodes
+ - j11_cqlsh-dtests-py3-with-vnodes:
+ requires:
+ - start_j11_cqlsh_tests-with-vnodes
+ - j11_cqlsh-dtests-py38-with-vnodes:
+ requires:
+ - start_j11_cqlsh_tests-with-vnodes
+ - start_j11_cqlsh_tests-no-vnodes:
+ type: approval
+ requires:
+ - j11_build
+ - j11_cqlsh-dtests-py2-no-vnodes:
+ requires:
+ - start_j11_cqlsh_tests-no-vnodes
+ - j11_cqlsh-dtests-py3-no-vnodes:
+ requires:
+ - start_j11_cqlsh_tests-no-vnodes
+ - j11_cqlsh-dtests-py38-no-vnodes:
+ requires:
+ - start_j11_cqlsh_tests-no-vnodes
diff --git a/.circleci/generate.sh b/.circleci/generate.sh
index f3f4361..f0944a6 100755
--- a/.circleci/generate.sh
+++ b/.circleci/generate.sh
@@ -6,4 +6,3 @@
patch -o $BASEDIR/config-2_1.yml.HIGHRES $BASEDIR/config-2_1.yml $BASEDIR/config-2_1.yml.high_res.patch
circleci config process $BASEDIR/config-2_1.yml.HIGHRES > $BASEDIR/config.yml.HIGHRES
rm $BASEDIR/config-2_1.yml.HIGHRES
-
diff --git a/.circleci/generate_midres.sh b/.circleci/generate_midres.sh
new file mode 100755
index 0000000..d2e179e
--- /dev/null
+++ b/.circleci/generate_midres.sh
@@ -0,0 +1,8 @@
+#!/bin/sh
+
+BASEDIR=`dirname $0`
+
+circleci config process $BASEDIR/config-2_1.yml > $BASEDIR/config.yml.LOWRES
+patch -o $BASEDIR/config-2_1.yml.MIDRES $BASEDIR/config-2_1.yml $BASEDIR/config-2_1.yml.mid_res.patch
+circleci config process $BASEDIR/config-2_1.yml.MIDRES > $BASEDIR/config.yml.MIDRES
+rm $BASEDIR/config-2_1.yml.MIDRES
diff --git a/.circleci/readme.md b/.circleci/readme.md
index 965eaf7..6c23d93 100644
--- a/.circleci/readme.md
+++ b/.circleci/readme.md
@@ -6,6 +6,9 @@
`cp .circleci/config.yml.HIGHRES .circleci/config.yml`
+config.yml.LOWRES is the default config.
+MIDRES and HIGHRES are custom configs for those who have access to premium CircleCI resources.
+
Make sure you never edit the config.yml manually.
## Updating the config master
@@ -28,5 +31,5 @@
the patch file based on the diff (don't commit it though).
1. generate the HIGHRES file:
`circleci config process config-2_1.yml.HIGHRES > config.yml.HIGHRES`
-1. and remove the temporary patched highres `config-2_1.yml.HIGHRES`
+1. and remove the temporary patched HIGHRES file: `rm config-2_1.yml.HIGHRES`
diff --git a/.gitignore b/.gitignore
index b6fd009..d03cf42 100644
--- a/.gitignore
+++ b/.gitignore
@@ -43,7 +43,7 @@
# NetBeans
nbbuild/
nbdist/
-nbproject/
+ide/nbproject/private
nb-configuration.xml
nbactions.xml
@@ -76,3 +76,7 @@
# Generated files from the documentation
doc/source/configuration/cassandra_config_file.rst
+doc/source/tools/nodetool
+
+# Python virtual environment
+venv/
diff --git a/.jenkins/Jenkinsfile b/.jenkins/Jenkinsfile
index bb828e3..586c048 100644
--- a/.jenkins/Jenkinsfile
+++ b/.jenkins/Jenkinsfile
@@ -26,132 +26,153 @@
pipeline {
agent { label 'cassandra' }
stages {
- stage('Init') {
- steps {
- cleanWs()
- }
+ stage('Init') {
+ steps {
+ cleanWs()
}
- stage('Build') {
- steps {
- build job: "${env.JOB_NAME}-artifacts"
- }
+ }
+ stage('Build') {
+ steps {
+ build job: "${env.JOB_NAME}-artifacts"
}
- stage('Test') {
- parallel {
- stage('stress') {
- steps {
- build job: "${env.JOB_NAME}-stress-test"
- }
- post {
- success {
- warnError('missing test xml files') {
- script {
- copyTestResults('stress-test')
- }
- }
- }
- unstable {
- warnError('missing test xml files') {
- script {
- copyTestResults('stress-test')
- }
- }
- }
- }
+ }
+ stage('Test') {
+ parallel {
+ stage('stress') {
+ steps {
+ build job: "${env.JOB_NAME}-stress-test"
}
- stage('JVM DTests') {
- steps {
- warnError('Tests unstable') {
- build job: "${env.JOB_NAME}-jvm-dtest"
+ post {
+ success {
+ warnError('missing test xml files') {
+ script {
+ copyTestResults('stress-test')
+ }
}
}
- post {
- success {
- warnError('missing test xml files') {
- script {
- copyTestResults('jvm-dtest')
- }
- }
- }
- unstable {
- warnError('missing test xml files') {
- script {
- copyTestResults('jvm-dtest')
- }
- }
- }
- }
- }
- stage('units') {
- steps {
- warnError('Tests unstable') {
- build job: "${env.JOB_NAME}-test"
- }
- }
- post {
- success {
- warnError('missing test xml files') {
- script {
- copyTestResults('test')
- }
- }
- }
- unstable {
- warnError('missing test xml files') {
- script {
- copyTestResults('test')
- }
- }
- }
- }
- }
- stage('long units') {
- steps {
- warnError('Tests unstable') {
- build job: "${env.JOB_NAME}-long-test"
+ unstable {
+ warnError('missing test xml files') {
+ script {
+ copyTestResults('stress-test')
+ }
}
}
- post {
- success {
- warnError('missing test xml files') {
- script {
- copyTestResults('long-test')
- }
- }
- }
- unstable {
- warnError('missing test xml files') {
- script {
- copyTestResults('long-test')
- }
- }
- }
- }
}
- stage('burn') {
- steps {
- warnError('Tests unstable') {
- build job: "${env.JOB_NAME}-test-burn"
+ }
+ stage('fqltool') {
+ steps {
+ build job: "${env.JOB_NAME}-fqltool-test"
+ }
+ post {
+ success {
+ warnError('missing test xml files') {
+ script {
+ copyTestResults('fqltool-test')
+ }
}
}
- post {
- success {
- warnError('missing test xml files') {
- script {
- copyTestResults('test-burn')
- }
- }
- }
- unstable {
- warnError('missing test xml files') {
- script {
- copyTestResults('test-burn')
- }
- }
- }
+ unstable {
+ warnError('missing test xml files') {
+ script {
+ copyTestResults('fqltool-test')
+ }
+ }
}
}
- stage('cdc') {
+ }
+ stage('JVM DTests') {
+ steps {
+ warnError('Tests unstable') {
+ build job: "${env.JOB_NAME}-jvm-dtest"
+ }
+ }
+ post {
+ success {
+ warnError('missing test xml files') {
+ script {
+ copyTestResults('jvm-dtest')
+ }
+ }
+ }
+ unstable {
+ warnError('missing test xml files') {
+ script {
+ copyTestResults('jvm-dtest')
+ }
+ }
+ }
+ }
+ }
+ stage('units') {
+ steps {
+ warnError('Tests unstable') {
+ build job: "${env.JOB_NAME}-test"
+ }
+ }
+ post {
+ success {
+ warnError('missing test xml files') {
+ script {
+ copyTestResults('test')
+ }
+ }
+ }
+ unstable {
+ warnError('missing test xml files') {
+ script {
+ copyTestResults('test')
+ }
+ }
+ }
+ }
+ }
+ stage('long units') {
+ steps {
+ warnError('Tests unstable') {
+ build job: "${env.JOB_NAME}-long-test"
+ }
+ }
+ post {
+ success {
+ warnError('missing test xml files') {
+ script {
+ copyTestResults('long-test')
+ }
+ }
+ }
+ unstable {
+ warnError('missing test xml files') {
+ script {
+ copyTestResults('long-test')
+ }
+ }
+ }
+ }
+ }
+ stage('burn') {
+ steps {
+ warnError('Tests unstable') {
+ build job: "${env.JOB_NAME}-test-burn"
+ }
+ }
+ post {
+ success {
+ warnError('missing test xml files') {
+ script {
+ copyTestResults('test-burn')
+ }
+ }
+ }
+ unstable {
+ warnError('missing test xml files') {
+ script {
+ copyTestResults('test-burn')
+ }
+ }
+ }
+ }
+ }
+ stage('cdc') {
steps {
warnError('Tests unstable') {
build job: "${env.JOB_NAME}-test-cdc"
@@ -161,14 +182,14 @@
success {
warnError('missing test xml files') {
script {
- copyTestResults('test-cdc')
+ copyTestResults('test-cdc')
}
}
}
unstable {
warnError('missing test xml files') {
script {
- copyTestResults('test-cdc')
+ copyTestResults('test-cdc')
}
}
}
@@ -184,14 +205,14 @@
success {
warnError('missing test xml files') {
script {
- copyTestResults('test-compression')
+ copyTestResults('test-compression')
}
}
}
unstable {
warnError('missing test xml files') {
script {
- copyTestResults('test-compression')
+ copyTestResults('test-compression')
}
}
}
@@ -207,117 +228,117 @@
success {
warnError('missing test xml files') {
script {
- copyTestResults('cqlsh-test')
+ copyTestResults('cqlsh-tests')
}
}
}
unstable {
warnError('missing test xml files') {
script {
- copyTestResults('cqlsh-test')
+ copyTestResults('cqlsh-tests')
}
}
}
}
}
+ }
+ }
+ stage('Distributed Test') {
+ parallel {
+ stage('dtest') {
+ steps {
+ warnError('Tests unstable') {
+ build job: "${env.JOB_NAME}-dtest"
+ }
+ }
+ post {
+ success {
+ warnError('missing test xml files') {
+ script {
+ copyTestResults('dtest')
+ }
+ }
+ }
+ unstable {
+ warnError('missing test xml files') {
+ script {
+ copyTestResults('dtest')
+ }
+ }
+ }
+ }
}
- }
- stage('Distributed Test') {
- parallel {
- stage('dtest') {
- steps {
- warnError('Tests unstable') {
- build job: "${env.JOB_NAME}-dtest"
+ stage('dtest-large') {
+ steps {
+ warnError('Tests unstable') {
+ build job: "${env.JOB_NAME}-dtest-large"
+ }
+ }
+ post {
+ success {
+ warnError('missing test xml files') {
+ script {
+ copyTestResults('dtest-large')
+ }
}
}
- post {
- success {
- warnError('missing test xml files') {
- script {
- copyTestResults('dtest')
- }
- }
- }
- unstable {
- warnError('missing test xml files') {
- script {
- copyTestResults('dtest')
- }
- }
- }
+ unstable {
+ warnError('missing test xml files') {
+ script {
+ copyTestResults('dtest-large')
+ }
+ }
}
}
- stage('dtest-large') {
- steps {
- warnError('Tests unstable') {
- build job: "${env.JOB_NAME}-dtest-large"
+ }
+ stage('dtest-novnode') {
+ steps {
+ warnError('Tests unstable') {
+ build job: "${env.JOB_NAME}-dtest-novnode"
+ }
+ }
+ post {
+ success {
+ warnError('missing test xml files') {
+ script {
+ copyTestResults('dtest-novnode')
+ }
}
}
- post {
- success {
- warnError('missing test xml files') {
- script {
- copyTestResults('dtest-large')
- }
- }
- }
- unstable {
- warnError('missing test xml files') {
- script {
- copyTestResults('dtest-large')
- }
- }
- }
+ unstable {
+ warnError('missing test xml files') {
+ script {
+ copyTestResults('dtest-novnode')
+ }
+ }
}
}
- stage('dtest-novnode') {
- steps {
- warnError('Tests unstable') {
- build job: "${env.JOB_NAME}-dtest-novnode"
- }
- }
- post {
- success {
- warnError('missing test xml files') {
- script {
- copyTestResults('dtest-novnode')
- }
- }
+ }
+ stage('dtest-offheap') {
+ steps {
+ warnError('Tests unstable') {
+ build job: "${env.JOB_NAME}-dtest-offheap"
}
- unstable {
- warnError('missing test xml files') {
- script {
- copyTestResults('dtest-novnode')
- }
- }
- }
- }
}
- stage('dtest-offheap') {
- steps {
- warnError('Tests unstable') {
- build job: "${env.JOB_NAME}-dtest-offheap"
+ post {
+ success {
+ warnError('missing test xml files') {
+ script {
+ copyTestResults('dtest-offheap')
+ }
}
}
- post {
- success {
- warnError('missing test xml files') {
- script {
- copyTestResults('dtest-offheap')
- }
- }
- }
- unstable {
- warnError('missing test xml files') {
- script {
- copyTestResults('dtest-offheap')
- }
- }
- }
+ unstable {
+ warnError('missing test xml files') {
+ script {
+ copyTestResults('dtest-offheap')
+ }
+ }
}
}
}
}
+ }
stage('Summary') {
steps {
sh "rm -fR cassandra-builds"
diff --git a/CHANGES.txt b/CHANGES.txt
index 9463403..15dd123 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,8 +1,64 @@
-3.11.7
+4.0-alpha5
+ * Improve messaging on indexing frozen collections (CASSANDRA-15908)
+ * USING_G1 is incorrectly set in cassandra-env.sh if G1 is explicitly disabled with -UseG1GC (CASSANDRA-15931)
+ * Update compaction_throughput_mb_per_sec throttle default to 64 (CASSANDRA-14902)
+ * Add option to disable compaction at startup (CASSANDRA-15927)
+ * FBUtilities.getJustLocalAddress falls back to lo ip on misconfigured nodes (CASSANDRA-15901)
+ * Close channel and reduce buffer allocation during entire sstable streaming with SSL (CASSANDRA-15900)
+ * Prune expired messages less frequently in internode messaging (CASSANDRA-15700)
+ * Fix Ec2Snitch handling of legacy mode for dc names matching both formats, eg "us-west-2" (CASSANDRA-15878)
+ * Add support for server side DESCRIBE statements (CASSANDRA-14825)
+ * Fail startup if -Xmn is set when the G1 garbage collector is used (CASSANDRA-15839)
+ * generateSplits method replaced the generateRandomTokens for ReplicationAwareTokenAllocator. (CASSANDRA-15877)
+ * Several mbeans are not unregistered when dropping a keyspace and table (CASSANDRA-14888)
+ * Update defaults for server and client TLS settings (CASSANDRA-15262)
+ * Differentiate follower/initator in StreamMessageHeader (CASSANDRA-15665)
+ * Add a startup check to detect if LZ4 uses java rather than native implementation (CASSANDRA-15884)
+ * Fix missing topology events when running multiple nodes on the same network interface (CASSANDRA-15677)
+ * Create config.yml.MIDRES (CASSANDRA-15712)
+ * Fix handling of fully purged static rows in repaired data tracking (CASSANDRA-15848)
+ * Prevent validation request submission from blocking ANTI_ENTROPY stage (CASSANDRA-15812)
+ * Add fqltool and auditlogviewer to rpm and deb packages (CASSANDRA-14712)
+ * Include DROPPED_COLUMNS in schema digest computation (CASSANDRA-15843)
+ * Fix Cassandra restart from rpm install (CASSANDRA-15830)
+ * Improve handling of 2i initialization failures (CASSANDRA-13606)
+ * Add completion_ratio column to sstable_tasks virtual table (CASANDRA-15759)
+ * Add support for adding custom Verbs (CASSANDRA-15725)
+ * Speed up entire-file-streaming file containment check and allow entire-file-streaming for all compaction strategies (CASSANDRA-15657,CASSANDRA-15783)
+ * Provide ability to configure IAuditLogger (CASSANDRA-15748)
+ * Fix nodetool enablefullquerylog blocking param parsing (CASSANDRA-15819)
+ * Add isTransient to SSTableMetadataView (CASSANDRA-15806)
+ * Fix tools/bin/fqltool for all shells (CASSANDRA-15820)
+ * Fix clearing of legacy size_estimates (CASSANDRA-15776)
+ * Update port when reconnecting to pre-4.0 SSL storage (CASSANDRA-15727)
+ * Only calculate dynamicBadnessThreshold once per loop in DynamicEndpointSnitch (CASSANDRA-15798)
+ * Cleanup redundant nodetool commands added in 4.0 (CASSANDRA-15256)
+ * Update to Python driver 3.23 for cqlsh (CASSANDRA-15793)
+ * Add tunable initial size and growth factor to RangeTombstoneList (CASSANDRA-15763)
+ * Improve debug logging in SSTableReader for index summary (CASSANDRA-15755)
+ * bin/sstableverify should support user provided token ranges (CASSANDRA-15753)
+ * Improve logging when mutation passed to commit log is too large (CASSANDRA-14781)
+ * replace LZ4FastDecompressor with LZ4SafeDecompressor (CASSANDRA-15560)
+ * Fix buffer pool NPE with concurrent release due to in-progress tiny pool eviction (CASSANDRA-15726)
+ * Avoid race condition when completing stream sessions (CASSANDRA-15666)
+ * Flush with fast compressors by default (CASSANDRA-15379)
+ * Fix CqlInputFormat regression from the switch to system.size_estimates (CASSANDRA-15637)
+ * Allow sending Entire SSTables over SSL (CASSANDRA-15740)
+ * Fix CQLSH UTF-8 encoding issue for Python 2/3 compatibility (CASSANDRA-15739)
+ * Fix batch statement preparation when multiple tables and parameters are used (CASSANDRA-15730)
+ * Fix regression with traceOutgoingMessage printing message size (CASSANDRA-15687)
+ * Ensure repaired data tracking reads a consistent amount of data across replicas (CASSANDRA-15601)
+ * Fix CQLSH to avoid arguments being evaluated (CASSANDRA-15660)
+ * Correct Visibility and Improve Safety of Methods in LatencyMetrics (CASSANDRA-15597)
+ * Allow cqlsh to run with Python2.7/Python3.6+ (CASSANDRA-15659,CASSANDRA-15573)
+ * Improve logging around incremental repair (CASSANDRA-15599)
+ * Do not check cdc_raw_directory filesystem space if CDC disabled (CASSANDRA-15688)
+ * Replace array iterators with get by index (CASSANDRA-15394)
+ * Minimize BTree iterator allocations (CASSANDRA-15389)
+Merged from 3.11:
* Fix cqlsh output when fetching all rows in batch mode (CASSANDRA-15905)
* Upgrade Jackson to 2.9.10 (CASSANDRA-15867)
* Fix CQL formatting of read command restrictions for slow query log (CASSANDRA-15503)
- * Allow sstableloader to use SSL on the native port (CASSANDRA-14904)
Merged from 3.0:
* Avoid hinted handoff per-host throttle being arounded to 0 in large cluster (CASSANDRA-15859)
* Avoid emitting empty range tombstones from RangeTombstoneList (CASSANDRA-15924)
@@ -11,7 +67,6 @@
* Fixed range read concurrency factor computation and capped as 10 times tpc cores (CASSANDRA-15752)
* Catch exception on bootstrap resume and init native transport (CASSANDRA-15863)
* Fix replica-side filtering returning stale data with CL > ONE (CASSANDRA-8272, CASSANDRA-8273)
- * Fix duplicated row on 2.x upgrades when multi-rows range tombstones interact with collection ones (CASSANDRA-15805)
* Rely on snapshotted session infos on StreamResultFuture.maybeComplete to avoid race conditions (CASSANDRA-15667)
* EmptyType doesn't override writeValue so could attempt to write bytes when expected not to (CASSANDRA-15790)
* Fix index queries on partition key columns when some partitions contains only static data (CASSANDRA-13666)
@@ -23,21 +78,511 @@
* Allow selecting static column only when querying static index (CASSANDRA-14242)
* cqlsh return non-zero status when STDIN CQL fails (CASSANDRA-15623)
* Don't skip sstables in slice queries based only on local min/max/deletion timestamp (CASSANDRA-15690)
- * Memtable memory allocations may deadlock (CASSANDRA-15367)
- * Run evictFromMembership in GossipStage (CASSANDRA-15592)
Merged from 2.2:
* Fix nomenclature of allow and deny lists (CASSANDRA-15862)
* Remove generated files from source artifact (CASSANDRA-15849)
* Remove duplicated tools binaries from tarballs (CASSANDRA-15768)
* Duplicate results with DISTINCT queries in mixed mode (CASSANDRA-15501)
- * Disable JMX rebinding (CASSANDRA-15653)
Merged from 2.1:
* Fix writing of snapshot manifest when the table has table-backed secondary indexes (CASSANDRA-10968)
+
+4.0-alpha4
+ * Add client request size server metrics (CASSANDRA-15704)
+ * Add additional logging around FileUtils and compaction leftover cleanup (CASSANDRA-15705)
+ * Mark system_views/system_virtual_schema as non-alterable keyspaces in cqlsh (CASSANDRA-15711)
+ * Fail incremental repair if an old version sstable is involved (CASSANDRA-15612)
+ * Fix overflows on StreamingTombstoneHistogramBuilder produced by large deletion times (CASSANDRA-14773)
+ * Mark system_views/system_virtual_schema as system keyspaces in cqlsh (CASSANDRA-15706)
+ * Avoid unnecessary collection/iterator allocations during btree construction (CASSANDRA-15390)
+ * Repair history tables should have TTL and TWCS (CASSANDRA-12701)
+ * Fix cqlsh erroring out on Python 3.7 due to webbrowser module being absent (CASSANDRA-15572)
+ * Fix IMH#acquireCapacity() to return correct Outcome when endpoint reserve runs out (CASSANDRA-15607)
+ * Fix nodetool describering output (CASSANDRA-15682)
+ * Only track ideal CL failure when request CL met (CASSANDRA-15696)
+ * Fix flaky CoordinatorMessagingTest and docstring in OutboundSink and ConsistentSession (CASSANDRA-15672)
+ * Fix force compaction of wrapping ranges (CASSANDRA-15664)
+ * Expose repair streaming metrics (CASSANDRA-15656)
+ * Set now in seconds in the future for validation repairs (CASSANDRA-15655)
+ * Emit metric on preview repair failure (CASSANDRA-15654)
+ * Use more appropriate logging levels (CASSANDRA-15661)
+ * Fixed empty check in TrieMemIndex due to potential state inconsistency in ConcurrentSkipListMap (CASSANDRA-15526)
+ * Added UnleveledSSTables global and table level metric (CASSANDRA-15620)
+ * Added Virtual Table exposing Cassandra relevant system properties (CASSANDRA-15616, CASSANDRA-15643)
+ * Improve the algorithmic token allocation in case racks = RF (CASSANDRA-15600)
+ * Fix ConnectionTest.testAcquireReleaseOutbound (CASSANDRA-15308)
+ * Include finalized pending sstables in preview repair (CASSANDRA-15553)
+ * Reverted to the original behavior of CLUSTERING ORDER on CREATE TABLE (CASSANDRA-15271)
+ * Correct inaccurate logging message (CASSANDRA-15549)
+ * Unset GREP_OPTIONS (CASSANDRA-14487)
+ * Update to Python driver 3.21 for cqlsh (CASSANDRA-14872)
+ * Fix missing Keyspaces in cqlsh describe output (CASSANDRA-15576)
+ * Fix multi DC nodetool status output (CASSANDRA-15305)
+ * updateCoordinatorWriteLatencyTableMetric can produce misleading metrics (CASSANDRA-15569)
+ * Make cqlsh and cqlshlib Python 2 & 3 compatible (CASSANDRA-10190)
+ * Improve the description of nodetool listsnapshots command (CASSANDRA-14587)
+ * allow embedded cassandra launched from a one-jar or uno-jar (CASSANDRA-15494)
+ * Update hppc library to version 0.8.1 (CASSANDRA-12995)
+ * Limit the dependencies used by UDFs/UDAs (CASSANDRA-14737)
+ * Make native_transport_max_concurrent_requests_in_bytes updatable (CASSANDRA-15519)
+ * Cleanup and improvements to IndexInfo/ColumnIndex (CASSANDRA-15469)
+ * Potential Overflow in DatabaseDescriptor Functions That Convert Between KB/MB & Bytes (CASSANDRA-15470)
+Merged from 3.11:
+ * Allow sstableloader to use SSL on the native port (CASSANDRA-14904)
+Merged from 3.0:
+ * cqlsh return non-zero status when STDIN CQL fails (CASSANDRA-15623)
+ * Don't skip sstables in slice queries based only on local min/max/deletion timestamp (CASSANDRA-15690)
+ * Memtable memory allocations may deadlock (CASSANDRA-15367)
+ * Run evictFromMembership in GossipStage (CASSANDRA-15592)
+Merged from 2.2:
+ * Duplicate results with DISTINCT queries in mixed mode (CASSANDRA-15501)
+ * Disable JMX rebinding (CASSANDRA-15653)
+Merged from 2.1:
* Fix parse error in cqlsh COPY FROM and formatting for map of blobs (CASSANDRA-15679)
* Fix Commit log replays when static column clustering keys are collections (CASSANDRA-14365)
* Fix Red Hat init script on newer systemd versions (CASSANDRA-15273)
* Allow EXTRA_CLASSPATH to work on tar/source installations (CASSANDRA-15567)
+4.0-alpha3
+ * Restore monotonic read consistency guarantees for blocking read repair (CASSANDRA-14740)
+ * Separate exceptions for CAS write timeout exceptions caused by contention and unkown result (CASSANDRA-15350)
+ * Fix in-jvm dtest java 11 compatibility (CASSANDRA-15463)
+ * Remove joda time dependency (CASSANDRA-15257)
+ * Exclude purgeable tombstones from repaired data tracking (CASSANDRA-15462)
+ * Exclude legacy counter shards from repaired data tracking (CASSANDRA-15461)
+ * Make it easier to add trace headers to messages (CASSANDRA-15499)
+ * Fix and optimise partial compressed sstable streaming (CASSANDRA-13938)
+ * Improve error when JVM 11 can't access required modules (CASSANDRA-15468)
+ * Better handling of file deletion failures by DiskFailurePolicy (CASSANDRA-15143)
+ * Prevent read repair mutations from increasing read timeout (CASSANDRA-15442)
+ * Document 4.0 system keyspace changes, bump generations (CASSANDRA-15454)
+ * Make it possible to disable STCS-in-L0 during runtime (CASSANDRA-15445)
+ * Removed obsolete OldNetworkTopologyStrategy (CASSANDRA-13990)
+ * Align record header of FQL and audit binary log (CASSANDRA-15076)
+ * Shuffle forwarding replica for messages to non-local DC (CASSANDRA-15318)
+ * Optimise native protocol ASCII string encoding (CASSANDRA-15410)
+ * Make sure all exceptions are propagated in DebuggableThreadPoolExecutor (CASSANDRA-15332)
+ * Make it possible to resize concurrent read / write thread pools at runtime (CASSANDRA-15277)
+ * Close channels on error (CASSANDRA-15407)
+ * Integrate SJK into nodetool (CASSANDRA-12197)
+ * Ensure that empty clusterings with kind==CLUSTERING are Clustering.EMPTY (CASSANDRA-15498)
+ * The flag 'cross_node_timeout' has been set as true by default. This change
+ is done under the assumption that users have setup NTP on their clusters or
+ otherwise synchronize their clocks, and that clocks are mostly in sync, since
+ this is a requirement for general correctness of last write wins. (CASSANDRA-15216)
+Merged from 3.11:
+ * Fix bad UDT sstable metadata serialization headers written by C* 3.0 on upgrade and in sstablescrub (CASSANDRA-15035)
+ * Fix nodetool compactionstats showing extra pending task for TWCS - patch implemented (CASSANDRA-15409)
+ * Fix SELECT JSON formatting for the "duration" type (CASSANDRA-15075)
+ * Update nodetool help stop output (CASSANDRA-15401)
+Merged from 3.0:
+ * Fix race condition when setting bootstrap flags (CASSANDRA-14878)
+ * Run in-jvm upgrade dtests in circleci (CASSANDRA-15506)
+ * Include updates to static column in mutation size calculations (CASSANDRA-15293)
+ * Fix point-in-time recoevery ignoring timestamp of updates to static columns (CASSANDRA-15292)
+ * GC logs are also put under $CASSANDRA_LOG_DIR (CASSANDRA-14306)
+ * Fix sstabledump's position key value when partitions have multiple rows (CASSANDRA-14721)
+ * Avoid over-scanning data directories in LogFile.verify() (CASSANDRA-15364)
+ * Bump generations and document changes to system_distributed and system_traces in 3.0, 3.11
+ (CASSANDRA-15441)
+ * Fix system_traces creation timestamp; optimise system keyspace upgrades (CASSANDRA-15398)
+ * Make sure index summary redistribution does not start when compactions are paused (CASSANDRA-15265)
+ * Fix NativeLibrary.tryOpenDirectory callers for Windows (CASSANDRA-15426)
+Merged from 2.2:
+ * Fix SELECT JSON output for empty blobs (CASSANDRA-15435)
+ * In-JVM DTest: Set correct internode message version for upgrade test (CASSANDRA-15371)
+ * In-JVM DTest: Support NodeTool in dtest (CASSANDRA-15429)
+ * Added data modeling documentation (CASSANDRA-15443)
+
+4.0-alpha2
+ * Fix SASI non-literal string comparisons (range operators) (CASSANDRA-15169)
+ * Upgrade Guava to 27, and to java-driver 3.6.0 (from 3.4.0-SNAPSHOT) (CASSANDRA-14655)
+ * Extract an AbstractCompactionController to allow for custom implementations (CASSANDRA-15286)
+ * Move chronicle-core version from snapshot to stable, and include carrotsearch in generated pom.xml (CASSANDRA-15321)
+ * Untangle RepairMessage sub-hierarchy of messages, use new messaging (more) correctly (CASSANDRA-15163)
+ * Add `allocate_tokens_for_local_replication_factor` option for token allocation (CASSANDRA-15260)
+ * Add Alibaba Cloud Platform snitch (CASSANDRA-15092)
+Merged from 3.0:
+ * Fix various data directory prefix matching issues (CASSANDRA-13974)
+ * Minimize clustering values in metadata collector (CASSANDRA-15400)
+ * Make sure index summary redistribution does not start when compactions are paused (CASSANDRA-15265)
+ * Add ability to cap max negotiable protocol version (CASSANDRA-15193)
+ * Gossip tokens on startup if available (CASSANDRA-15335)
+ * Fix resource leak in CompressedSequentialWriter (CASSANDRA-15340)
+ * Fix bad merge that reverted CASSANDRA-14993 (CASSANDRA-15289)
+ * Add support for network topology and query tracing for inJVM dtest (CASSANDRA-15319)
+
+
+4.0-alpha1
+ * Inaccurate exception message with nodetool snapshot (CASSANDRA-15287)
+ * Fix InternodeOutboundMetrics overloaded bytes/count mixup (CASSANDRA-15186)
+ * Enhance & reenable RepairTest with compression=off and compression=on (CASSANDRA-15272)
+ * Improve readability of Table metrics Virtual tables units (CASSANDRA-15194)
+ * Fix error with non-existent table for nodetool tablehistograms (CASSANDRA-14410)
+ * Avoid result truncation in decimal operations (CASSANDRA-15232)
+ * Catch non-IOException in FileUtils.close to make sure that all resources are closed (CASSANDRA-15225)
+ * Align load column in nodetool status output (CASSANDRA-14787)
+ * CassandraNetworkAuthorizer uses cached roles info (CASSANDRA-15089)
+ * Introduce optional timeouts for idle client sessions (CASSANDRA-11097)
+ * Fix AlterTableStatement dropped type validation order (CASSANDRA-15203)
+ * Update Netty dependencies to latest, clean up SocketFactory (CASSANDRA-15195)
+ * Native Transport - Apply noSpamLogger to ConnectionLimitHandler (CASSANDRA-15167)
+ * Reduce heap pressure during compactions (CASSANDRA-14654)
+ * Support building Cassandra with JDK 11 (CASSANDRA-15108)
+ * Use quilt to patch cassandra.in.sh in Debian packaging (CASSANDRA-14710)
+ * Take sstable references before calculating approximate key count (CASSANDRA-14647)
+ * Restore snapshotting of system keyspaces on version change (CASSANDRA-14412)
+ * Fix AbstractBTreePartition locking in java 11 (CASSANDRA-14607)
+ * SimpleClient should pass connection properties as options (CASSANDRA-15056)
+ * Set repaired data tracking flag on range reads if enabled (CASSANDRA-15019)
+ * Calculate pending ranges for BOOTSTRAP_REPLACE correctly (CASSANDRA-14802)
+ * Make TableCQLHelper reuse the single quote pattern (CASSANDRA-15033)
+ * Add Zstd compressor (CASSANDRA-14482)
+ * Fix IR prepare anti-compaction race (CASSANDRA-15027)
+ * Fix SimpleStrategy option validation (CASSANDRA-15007)
+ * Don't try to cancel 2i compactions when starting anticompaction (CASSANDRA-15024)
+ * Avoid NPE in RepairRunnable.recordFailure (CASSANDRA-15025)
+ * SSL Cert Hot Reloading should check for sanity of the new keystore/truststore before loading it (CASSANDRA-14991)
+ * Avoid leaking threads when failing anticompactions and rate limit anticompactions (CASSANDRA-15002)
+ * Validate token() arguments early instead of throwing NPE at execution (CASSANDRA-14989)
+ * Add a new tool to dump audit logs (CASSANDRA-14885)
+ * Fix generating javadoc with Java11 (CASSANDRA-14988)
+ * Only cancel conflicting compactions when starting anticompactions and sub range compactions (CASSANDRA-14935)
+ * Use a stub IndexRegistry for non-daemon use cases (CASSANDRA-14938)
+ * Don't enable client transports when bootstrap is pending (CASSANDRA-14525)
+ * Make antiCompactGroup throw exception on error and anticompaction non cancellable
+ again (CASSANDRA-14936)
+ * Catch empty/invalid bounds in SelectStatement (CASSANDRA-14849)
+ * Auto-expand replication_factor for NetworkTopologyStrategy (CASSANDRA-14303)
+ * Transient Replication: support EACH_QUORUM (CASSANDRA-14727)
+ * BufferPool: allocating thread for new chunks should acquire directly (CASSANDRA-14832)
+ * Send correct messaging version in internode messaging handshake's third message (CASSANDRA-14896)
+ * Make Read and Write Latency columns consistent for proxyhistograms and tablehistograms (CASSANDRA-11939)
+ * Make protocol checksum type option case insensitive (CASSANDRA-14716)
+ * Forbid re-adding static columns as regular and vice versa (CASSANDRA-14913)
+ * Audit log allows system keyspaces to be audited via configuration options (CASSANDRA-14498)
+ * Lower default chunk_length_in_kb from 64kb to 16kb (CASSANDRA-13241)
+ * Startup checker should wait for count rather than percentage (CASSANDRA-14297)
+ * Fix incorrect sorting of replicas in SimpleStrategy.calculateNaturalReplicas (CASSANDRA-14862)
+ * Partitioned outbound internode TCP connections can occur when nodes restart (CASSANDRA-14358)
+ * Don't write to system_distributed.repair_history, system_traces.sessions, system_traces.events in mixed version 3.X/4.0 clusters (CASSANDRA-14841)
+ * Avoid running query to self through messaging service (CASSANDRA-14807)
+ * Allow using custom script for chronicle queue BinLog archival (CASSANDRA-14373)
+ * Transient->Full range movements mishandle consistency level upgrade (CASSANDRA-14759)
+ * ReplicaCollection follow-up (CASSANDRA-14726)
+ * Transient node receives full data requests (CASSANDRA-14762)
+ * Enable snapshot artifacts publish (CASSANDRA-12704)
+ * Introduce RangesAtEndpoint.unwrap to simplify StreamSession.addTransferRanges (CASSANDRA-14770)
+ * LOCAL_QUORUM may speculate to non-local nodes, resulting in Timeout instead of Unavailable (CASSANDRA-14735)
+ * Avoid creating empty compaction tasks after truncate (CASSANDRA-14780)
+ * Fail incremental repair prepare phase if it encounters sstables from un-finalized sessions (CASSANDRA-14763)
+ * Add a check for receiving digest response from transient node (CASSANDRA-14750)
+ * Fail query on transient replica if coordinator only expects full data (CASSANDRA-14704)
+ * Remove mentions of transient replication from repair path (CASSANDRA-14698)
+ * Fix handleRepairStatusChangedNotification to remove first then add (CASSANDRA-14720)
+ * Allow transient node to serve as a repair coordinator (CASSANDRA-14693)
+ * DecayingEstimatedHistogramReservoir.EstimatedHistogramReservoirSnapshot returns wrong value for size() and incorrectly calculates count (CASSANDRA-14696)
+ * AbstractReplicaCollection equals and hash code should throw due to conflict between order sensitive/insensitive uses (CASSANDRA-14700)
+ * Detect inconsistencies in repaired data on the read path (CASSANDRA-14145)
+ * Add checksumming to the native protocol (CASSANDRA-13304)
+ * Make AuthCache more easily extendable (CASSANDRA-14662)
+ * Extend RolesCache to include detailed role info (CASSANDRA-14497)
+ * Add fqltool compare (CASSANDRA-14619)
+ * Add fqltool replay (CASSANDRA-14618)
+ * Log keyspace in full query log (CASSANDRA-14656)
+ * Transient Replication and Cheap Quorums (CASSANDRA-14404)
+ * Log server-generated timestamp and nowInSeconds used by queries in FQL (CASSANDRA-14675)
+ * Add diagnostic events for read repairs (CASSANDRA-14668)
+ * Use consistent nowInSeconds and timestamps values within a request (CASSANDRA-14671)
+ * Add sampler for query time and expose with nodetool (CASSANDRA-14436)
+ * Clean up Message.Request implementations (CASSANDRA-14677)
+ * Disable old native protocol versions on demand (CASANDRA-14659)
+ * Allow specifying now-in-seconds in native protocol (CASSANDRA-14664)
+ * Improve BTree build performance by avoiding data copy (CASSANDRA-9989)
+ * Make monotonic read / read repair configurable (CASSANDRA-14635)
+ * Refactor CompactionStrategyManager (CASSANDRA-14621)
+ * Flush netty client messages immediately by default (CASSANDRA-13651)
+ * Improve read repair blocking behavior (CASSANDRA-10726)
+ * Add a virtual table to expose settings (CASSANDRA-14573)
+ * Fix up chunk cache handling of metrics (CASSANDRA-14628)
+ * Extend IAuthenticator to accept peer SSL certificates (CASSANDRA-14652)
+ * Incomplete handling of exceptions when decoding incoming messages (CASSANDRA-14574)
+ * Add diagnostic events for user audit logging (CASSANDRA-13668)
+ * Allow retrieving diagnostic events via JMX (CASSANDRA-14435)
+ * Add base classes for diagnostic events (CASSANDRA-13457)
+ * Clear view system metadata when dropping keyspace (CASSANDRA-14646)
+ * Allocate ReentrantLock on-demand in java11 AtomicBTreePartitionerBase (CASSANDRA-14637)
+ * Make all existing virtual tables use LocalPartitioner (CASSANDRA-14640)
+ * Revert 4.0 GC alg back to CMS (CASANDRA-14636)
+ * Remove hardcoded java11 jvm args in idea workspace files (CASSANDRA-14627)
+ * Update netty to 4.1.128 (CASSANDRA-14633)
+ * Add a virtual table to expose thread pools (CASSANDRA-14523)
+ * Add a virtual table to expose caches (CASSANDRA-14538, CASSANDRA-14626)
+ * Fix toDate function for timestamp arguments (CASSANDRA-14502)
+ * Stream entire SSTables when possible (CASSANDRA-14556)
+ * Cell reconciliation should not depend on nowInSec (CASSANDRA-14592)
+ * Add experimental support for Java 11 (CASSANDRA-9608)
+ * Make PeriodicCommitLogService.blockWhenSyncLagsNanos configurable (CASSANDRA-14580)
+ * Improve logging in MessageInHandler's constructor (CASSANDRA-14576)
+ * Set broadcast address in internode messaging handshake (CASSANDRA-14579)
+ * Wait for schema agreement prior to building MVs (CASSANDRA-14571)
+ * Make all DDL statements idempotent and not dependent on global state (CASSANDRA-13426)
+ * Bump the hints messaging version to match the current one (CASSANDRA-14536)
+ * OffsetAwareConfigurationLoader doesn't set ssl storage port causing bind errors in CircleCI (CASSANDRA-14546)
+ * Report why native_transport_port fails to bind (CASSANDRA-14544)
+ * Optimize internode messaging protocol (CASSANDRA-14485)
+ * Internode messaging handshake sends wrong messaging version number (CASSANDRA-14540)
+ * Add a virtual table to expose active client connections (CASSANDRA-14458)
+ * Clean up and refactor client metrics (CASSANDRA-14524)
+ * Nodetool import row cache invalidation races with adding sstables to tracker (CASSANDRA-14529)
+ * Fix assertions in LWTs after TableMetadata was made immutable (CASSANDRA-14356)
+ * Abort compactions quicker (CASSANDRA-14397)
+ * Support light-weight transactions in cassandra-stress (CASSANDRA-13529)
+ * Make AsyncOneResponse use the correct timeout (CASSANDRA-14509)
+ * Add option to sanity check tombstones on reads/compactions (CASSANDRA-14467)
+ * Add a virtual table to expose all running sstable tasks (CASSANDRA-14457)
+ * Let nodetool import take a list of directories (CASSANDRA-14442)
+ * Avoid unneeded memory allocations / cpu for disabled log levels (CASSANDRA-14488)
+ * Implement virtual keyspace interface (CASSANDRA-7622)
+ * nodetool import cleanup and improvements (CASSANDRA-14417)
+ * Bump jackson version to >= 2.9.5 (CASSANDRA-14427)
+ * Allow nodetool toppartitions without specifying table (CASSANDRA-14360)
+ * Audit logging for database activity (CASSANDRA-12151)
+ * Clean up build artifacts in docs container (CASSANDRA-14432)
+ * Minor network authz improvements (Cassandra-14413)
+ * Automatic sstable upgrades (CASSANDRA-14197)
+ * Replace deprecated junit.framework.Assert usages with org.junit.Assert (CASSANDRA-14431)
+ * Cassandra-stress throws NPE if insert section isn't specified in user profile (CASSSANDRA-14426)
+ * List clients by protocol versions `nodetool clientstats --by-protocol` (CASSANDRA-14335)
+ * Improve LatencyMetrics performance by reducing write path processing (CASSANDRA-14281)
+ * Add network authz (CASSANDRA-13985)
+ * Use the correct IP/Port for Streaming when localAddress is left unbound (CASSANDRA-14389)
+ * nodetool listsnapshots is missing local system keyspace snapshots (CASSANDRA-14381)
+ * Remove StreamCoordinator.streamExecutor thread pool (CASSANDRA-14402)
+ * Rename nodetool --with-port to --print-port to disambiguate from --port (CASSANDRA-14392)
+ * Client TOPOLOGY_CHANGE messages have wrong port. (CASSANDRA-14398)
+ * Add ability to load new SSTables from a separate directory (CASSANDRA-6719)
+ * Eliminate background repair and probablistic read_repair_chance table options
+ (CASSANDRA-13910)
+ * Bind to correct local address in 4.0 streaming (CASSANDRA-14362)
+ * Use standard Amazon naming for datacenter and rack in Ec2Snitch (CASSANDRA-7839)
+ * Abstract write path for pluggable storage (CASSANDRA-14118)
+ * nodetool describecluster should be more informative (CASSANDRA-13853)
+ * Compaction performance improvements (CASSANDRA-14261)
+ * Refactor Pair usage to avoid boxing ints/longs (CASSANDRA-14260)
+ * Add options to nodetool tablestats to sort and limit output (CASSANDRA-13889)
+ * Rename internals to reflect CQL vocabulary (CASSANDRA-14354)
+ * Add support for hybrid MIN(), MAX() speculative retry policies
+ (CASSANDRA-14293, CASSANDRA-14338, CASSANDRA-14352)
+ * Fix some regressions caused by 14058 (CASSANDRA-14353)
+ * Abstract repair for pluggable storage (CASSANDRA-14116)
+ * Add meaningful toString() impls (CASSANDRA-13653)
+ * Add sstableloader option to accept target keyspace name (CASSANDRA-13884)
+ * Move processing of EchoMessage response to gossip stage (CASSANDRA-13713)
+ * Add coordinator write metric per CF (CASSANDRA-14232)
+ * Correct and clarify SSLFactory.getSslContext method and call sites (CASSANDRA-14314)
+ * Handle static and partition deletion properly on ThrottledUnfilteredIterator (CASSANDRA-14315)
+ * NodeTool clientstats should show SSL Cipher (CASSANDRA-14322)
+ * Add ability to specify driver name and version (CASSANDRA-14275)
+ * Abstract streaming for pluggable storage (CASSANDRA-14115)
+ * Forced incremental repairs should promote sstables if they can (CASSANDRA-14294)
+ * Use Murmur3 for validation compactions (CASSANDRA-14002)
+ * Comma at the end of the seed list is interpretated as localhost (CASSANDRA-14285)
+ * Refactor read executor and response resolver, abstract read repair (CASSANDRA-14058)
+ * Add optional startup delay to wait until peers are ready (CASSANDRA-13993)
+ * Add a few options to nodetool verify (CASSANDRA-14201)
+ * CVE-2017-5929 Security vulnerability and redefine default log rotation policy (CASSANDRA-14183)
+ * Use JVM default SSL validation algorithm instead of custom default (CASSANDRA-13259)
+ * Better document in code InetAddressAndPort usage post 7544, incorporate port into UUIDGen node (CASSANDRA-14226)
+ * Fix sstablemetadata date string for minLocalDeletionTime (CASSANDRA-14132)
+ * Make it possible to change neverPurgeTombstones during runtime (CASSANDRA-14214)
+ * Remove GossipDigestSynVerbHandler#doSort() (CASSANDRA-14174)
+ * Add nodetool clientlist (CASSANDRA-13665)
+ * Revert ProtocolVersion changes from CASSANDRA-7544 (CASSANDRA-14211)
+ * Non-disruptive seed node list reload (CASSANDRA-14190)
+ * Nodetool tablehistograms to print statics for all the tables (CASSANDRA-14185)
+ * Migrate dtests to use pytest and python3 (CASSANDRA-14134)
+ * Allow storage port to be configurable per node (CASSANDRA-7544)
+ * Make sub-range selection for non-frozen collections return null instead of empty (CASSANDRA-14182)
+ * BloomFilter serialization format should not change byte ordering (CASSANDRA-9067)
+ * Remove unused on-heap BloomFilter implementation (CASSANDRA-14152)
+ * Delete temp test files on exit (CASSANDRA-14153)
+ * Make PartitionUpdate and Mutation immutable (CASSANDRA-13867)
+ * Fix CommitLogReplayer exception for CDC data (CASSANDRA-14066)
+ * Fix cassandra-stress startup failure (CASSANDRA-14106)
+ * Remove initialDirectories from CFS (CASSANDRA-13928)
+ * Fix trivial log format error (CASSANDRA-14015)
+ * Allow sstabledump to do a json object per partition (CASSANDRA-13848)
+ * Add option to optimise merkle tree comparison across replicas (CASSANDRA-3200)
+ * Remove unused and deprecated methods from AbstractCompactionStrategy (CASSANDRA-14081)
+ * Fix Distribution.average in cassandra-stress (CASSANDRA-14090)
+ * Support a means of logging all queries as they were invoked (CASSANDRA-13983)
+ * Presize collections (CASSANDRA-13760)
+ * Add GroupCommitLogService (CASSANDRA-13530)
+ * Parallelize initial materialized view build (CASSANDRA-12245)
+ * Make LWTs send resultset metadata on every request (CASSANDRA-13992)
+ * Fix flaky indexWithFailedInitializationIsNotQueryableAfterPartialRebuild (CASSANDRA-13963)
+ * Introduce leaf-only iterator (CASSANDRA-9988)
+ * Upgrade Guava to 23.3 and Airline to 0.8 (CASSANDRA-13997)
+ * Allow only one concurrent call to StatusLogger (CASSANDRA-12182)
+ * Refactoring to specialised functional interfaces (CASSANDRA-13982)
+ * Speculative retry should allow more friendly params (CASSANDRA-13876)
+ * Throw exception if we send/receive repair messages to incompatible nodes (CASSANDRA-13944)
+ * Replace usages of MessageDigest with Guava's Hasher (CASSANDRA-13291)
+ * Add nodetool cmd to print hinted handoff window (CASSANDRA-13728)
+ * Fix some alerts raised by static analysis (CASSANDRA-13799)
+ * Checksum sstable metadata (CASSANDRA-13321, CASSANDRA-13593)
+ * Add result set metadata to prepared statement MD5 hash calculation (CASSANDRA-10786)
+ * Refactor GcCompactionTest to avoid boxing (CASSANDRA-13941)
+ * Expose recent histograms in JmxHistograms (CASSANDRA-13642)
+ * Fix buffer length comparison when decompressing in netty-based streaming (CASSANDRA-13899)
+ * Properly close StreamCompressionInputStream to release any ByteBuf (CASSANDRA-13906)
+ * Add SERIAL and LOCAL_SERIAL support for cassandra-stress (CASSANDRA-13925)
+ * LCS needlessly checks for L0 STCS candidates multiple times (CASSANDRA-12961)
+ * Correctly close netty channels when a stream session ends (CASSANDRA-13905)
+ * Update lz4 to 1.4.0 (CASSANDRA-13741)
+ * Optimize Paxos prepare and propose stage for local requests (CASSANDRA-13862)
+ * Throttle base partitions during MV repair streaming to prevent OOM (CASSANDRA-13299)
+ * Use compaction threshold for STCS in L0 (CASSANDRA-13861)
+ * Fix problem with min_compress_ratio: 1 and disallow ratio < 1 (CASSANDRA-13703)
+ * Add extra information to SASI timeout exception (CASSANDRA-13677)
+ * Add incremental repair support for --hosts, --force, and subrange repair (CASSANDRA-13818)
+ * Rework CompactionStrategyManager.getScanners synchronization (CASSANDRA-13786)
+ * Add additional unit tests for batch behavior, TTLs, Timestamps (CASSANDRA-13846)
+ * Add keyspace and table name in schema validation exception (CASSANDRA-13845)
+ * Emit metrics whenever we hit tombstone failures and warn thresholds (CASSANDRA-13771)
+ * Make netty EventLoopGroups daemon threads (CASSANDRA-13837)
+ * Race condition when closing stream sessions (CASSANDRA-13852)
+ * NettyFactoryTest is failing in trunk on macOS (CASSANDRA-13831)
+ * Allow changing log levels via nodetool for related classes (CASSANDRA-12696)
+ * Add stress profile yaml with LWT (CASSANDRA-7960)
+ * Reduce memory copies and object creations when acting on ByteBufs (CASSANDRA-13789)
+ * Simplify mx4j configuration (Cassandra-13578)
+ * Fix trigger example on 4.0 (CASSANDRA-13796)
+ * Force minumum timeout value (CASSANDRA-9375)
+ * Use netty for streaming (CASSANDRA-12229)
+ * Use netty for internode messaging (CASSANDRA-8457)
+ * Add bytes repaired/unrepaired to nodetool tablestats (CASSANDRA-13774)
+ * Don't delete incremental repair sessions if they still have sstables (CASSANDRA-13758)
+ * Fix pending repair manager index out of bounds check (CASSANDRA-13769)
+ * Don't use RangeFetchMapCalculator when RF=1 (CASSANDRA-13576)
+ * Don't optimise trivial ranges in RangeFetchMapCalculator (CASSANDRA-13664)
+ * Use an ExecutorService for repair commands instead of new Thread(..).start() (CASSANDRA-13594)
+ * Fix race / ref leak in anticompaction (CASSANDRA-13688)
+ * Expose tasks queue length via JMX (CASSANDRA-12758)
+ * Fix race / ref leak in PendingRepairManager (CASSANDRA-13751)
+ * Enable ppc64le runtime as unsupported architecture (CASSANDRA-13615)
+ * Improve sstablemetadata output (CASSANDRA-11483)
+ * Support for migrating legacy users to roles has been dropped (CASSANDRA-13371)
+ * Introduce error metrics for repair (CASSANDRA-13387)
+ * Refactoring to primitive functional interfaces in AuthCache (CASSANDRA-13732)
+ * Update metrics to 3.1.5 (CASSANDRA-13648)
+ * batch_size_warn_threshold_in_kb can now be set at runtime (CASSANDRA-13699)
+ * Avoid always rebuilding secondary indexes at startup (CASSANDRA-13725)
+ * Upgrade JMH from 1.13 to 1.19 (CASSANDRA-13727)
+ * Upgrade SLF4J from 1.7.7 to 1.7.25 (CASSANDRA-12996)
+ * Default for start_native_transport now true if not set in config (CASSANDRA-13656)
+ * Don't add localhost to the graph when calculating where to stream from (CASSANDRA-13583)
+ * Make CDC availability more deterministic via hard-linking (CASSANDRA-12148)
+ * Allow skipping equality-restricted clustering columns in ORDER BY clause (CASSANDRA-10271)
+ * Use common nowInSec for validation compactions (CASSANDRA-13671)
+ * Improve handling of IR prepare failures (CASSANDRA-13672)
+ * Send IR coordinator messages synchronously (CASSANDRA-13673)
+ * Flush system.repair table before IR finalize promise (CASSANDRA-13660)
+ * Fix column filter creation for wildcard queries (CASSANDRA-13650)
+ * Add 'nodetool getbatchlogreplaythrottle' and 'nodetool setbatchlogreplaythrottle' (CASSANDRA-13614)
+ * fix race condition in PendingRepairManager (CASSANDRA-13659)
+ * Allow noop incremental repair state transitions (CASSANDRA-13658)
+ * Run repair with down replicas (CASSANDRA-10446)
+ * Added started & completed repair metrics (CASSANDRA-13598)
+ * Added started & completed repair metrics (CASSANDRA-13598)
+ * Improve secondary index (re)build failure and concurrency handling (CASSANDRA-10130)
+ * Improve calculation of available disk space for compaction (CASSANDRA-13068)
+ * Change the accessibility of RowCacheSerializer for third party row cache plugins (CASSANDRA-13579)
+ * Allow sub-range repairs for a preview of repaired data (CASSANDRA-13570)
+ * NPE in IR cleanup when columnfamily has no sstables (CASSANDRA-13585)
+ * Fix Randomness of stress values (CASSANDRA-12744)
+ * Allow selecting Map values and Set elements (CASSANDRA-7396)
+ * Fast and garbage-free Streaming Histogram (CASSANDRA-13444)
+ * Update repairTime for keyspaces on completion (CASSANDRA-13539)
+ * Add configurable upper bound for validation executor threads (CASSANDRA-13521)
+ * Bring back maxHintTTL propery (CASSANDRA-12982)
+ * Add testing guidelines (CASSANDRA-13497)
+ * Add more repair metrics (CASSANDRA-13531)
+ * RangeStreamer should be smarter when picking endpoints for streaming (CASSANDRA-4650)
+ * Avoid rewrapping an exception thrown for cache load functions (CASSANDRA-13367)
+ * Log time elapsed for each incremental repair phase (CASSANDRA-13498)
+ * Add multiple table operation support to cassandra-stress (CASSANDRA-8780)
+ * Fix incorrect cqlsh results when selecting same columns multiple times (CASSANDRA-13262)
+ * Fix WriteResponseHandlerTest is sensitive to test execution order (CASSANDRA-13421)
+ * Improve incremental repair logging (CASSANDRA-13468)
+ * Start compaction when incremental repair finishes (CASSANDRA-13454)
+ * Add repair streaming preview (CASSANDRA-13257)
+ * Cleanup isIncremental/repairedAt usage (CASSANDRA-13430)
+ * Change protocol to allow sending key space independent of query string (CASSANDRA-10145)
+ * Make gc_log and gc_warn settable at runtime (CASSANDRA-12661)
+ * Take number of files in L0 in account when estimating remaining compaction tasks (CASSANDRA-13354)
+ * Skip building views during base table streams on range movements (CASSANDRA-13065)
+ * Improve error messages for +/- operations on maps and tuples (CASSANDRA-13197)
+ * Remove deprecated repair JMX APIs (CASSANDRA-11530)
+ * Fix version check to enable streaming keep-alive (CASSANDRA-12929)
+ * Make it possible to monitor an ideal consistency level separate from actual consistency level (CASSANDRA-13289)
+ * Outbound TCP connections ignore internode authenticator (CASSANDRA-13324)
+ * Cleanup ParentRepairSession after repairs (CASSANDRA-13359)
+ * Upgrade snappy-java to 1.1.2.6 (CASSANDRA-13336)
+ * Incremental repair not streaming correct sstables (CASSANDRA-13328)
+ * Upgrade the jna version to 4.3.0 (CASSANDRA-13300)
+ * Add the currentTimestamp, currentDate, currentTime and currentTimeUUID functions (CASSANDRA-13132)
+ * Remove config option index_interval (CASSANDRA-10671)
+ * Reduce lock contention for collection types and serializers (CASSANDRA-13271)
+ * Make it possible to override MessagingService.Verb ids (CASSANDRA-13283)
+ * Avoid synchronized on prepareForRepair in ActiveRepairService (CASSANDRA-9292)
+ * Adds the ability to use uncompressed chunks in compressed files (CASSANDRA-10520)
+ * Don't flush sstables when streaming for incremental repair (CASSANDRA-13226)
+ * Remove unused method (CASSANDRA-13227)
+ * Fix minor bugs related to #9143 (CASSANDRA-13217)
+ * Output warning if user increases RF (CASSANDRA-13079)
+ * Remove pre-3.0 streaming compatibility code for 4.0 (CASSANDRA-13081)
+ * Add support for + and - operations on dates (CASSANDRA-11936)
+ * Fix consistency of incrementally repaired data (CASSANDRA-9143)
+ * Increase commitlog version (CASSANDRA-13161)
+ * Make TableMetadata immutable, optimize Schema (CASSANDRA-9425)
+ * Refactor ColumnCondition (CASSANDRA-12981)
+ * Parallelize streaming of different keyspaces (CASSANDRA-4663)
+ * Improved compactions metrics (CASSANDRA-13015)
+ * Speed-up start-up sequence by avoiding un-needed flushes (CASSANDRA-13031)
+ * Use Caffeine (W-TinyLFU) for on-heap caches (CASSANDRA-10855)
+ * Thrift removal (CASSANDRA-11115)
+ * Remove pre-3.0 compatibility code for 4.0 (CASSANDRA-12716)
+ * Add column definition kind to dropped columns in schema (CASSANDRA-12705)
+ * Add (automate) Nodetool Documentation (CASSANDRA-12672)
+ * Update bundled cqlsh python driver to 3.7.0 (CASSANDRA-12736)
+ * Reject invalid replication settings when creating or altering a keyspace (CASSANDRA-12681)
+ * Clean up the SSTableReader#getScanner API wrt removal of RateLimiter (CASSANDRA-12422)
+ * Use new token allocation for non bootstrap case as well (CASSANDRA-13080)
+ * Avoid byte-array copy when key cache is disabled (CASSANDRA-13084)
+ * Require forceful decommission if number of nodes is less than replication factor (CASSANDRA-12510)
+ * Allow IN restrictions on column families with collections (CASSANDRA-12654)
+ * Log message size in trace message in OutboundTcpConnection (CASSANDRA-13028)
+ * Add timeUnit Days for cassandra-stress (CASSANDRA-13029)
+ * Add mutation size and batch metrics (CASSANDRA-12649)
+ * Add method to get size of endpoints to TokenMetadata (CASSANDRA-12999)
+ * Expose time spent waiting in thread pool queue (CASSANDRA-8398)
+ * Conditionally update index built status to avoid unnecessary flushes (CASSANDRA-12969)
+ * cqlsh auto completion: refactor definition of compaction strategy options (CASSANDRA-12946)
+ * Add support for arithmetic operators (CASSANDRA-11935)
+ * Add histogram for delay to deliver hints (CASSANDRA-13234)
+ * Fix cqlsh automatic protocol downgrade regression (CASSANDRA-13307)
+ * Changing `max_hint_window_in_ms` at runtime (CASSANDRA-11720)
+ * Trivial format error in StorageProxy (CASSANDRA-13551)
+ * Nodetool repair can hang forever if we lose the notification for the repair completing/failing (CASSANDRA-13480)
+ * Anticompaction can cause noisy log messages (CASSANDRA-13684)
+ * Switch to client init for sstabledump (CASSANDRA-13683)
+ * CQLSH: Don't pause when capturing data (CASSANDRA-13743)
+ * nodetool clearsnapshot requires --all to clear all snapshots (CASSANDRA-13391)
+ * Correctly count range tombstones in traces and tombstone thresholds (CASSANDRA-8527)
+ * cqlshrc.sample uses incorrect option for time formatting (CASSANDRA-14243)
+ * Multi-version in-JVM dtests (CASSANDRA-14937)
+ * Allow instance class loaders to be garbage collected for inJVM dtest (CASSANDRA-15170)
3.11.6
* Fix bad UDT sstable metadata serialization headers written by C* 3.0 on upgrade and in sstablescrub (CASSANDRA-15035)
@@ -63,25 +608,17 @@
* Make sure index summary redistribution does not start when compactions are paused (CASSANDRA-15265)
* Ensure legacy rows have primary key livenessinfo when they contain illegal cells (CASSANDRA-15365)
* Fix race condition when setting bootstrap flags (CASSANDRA-14878)
- * Fix NativeLibrary.tryOpenDirectory callers for Windows (CASSANDRA-15426)
Merged from 2.2:
* Fix SELECT JSON output for empty blobs (CASSANDRA-15435)
* In-JVM DTest: Set correct internode message version for upgrade test (CASSANDRA-15371)
* In-JVM DTest: Support NodeTool in dtest (CASSANDRA-15429)
* Fix NativeLibrary.tryOpenDirectory callers for Windows (CASSANDRA-15426)
-
3.11.5
- * Fix SASI non-literal string comparisons (range operators) (CASSANDRA-15169)
- * Make sure user defined compaction transactions are always closed (CASSANDRA-15123)
* Fix cassandra-env.sh to use $CASSANDRA_CONF to find cassandra-jaas.config (CASSANDRA-14305)
* Fixed nodetool cfstats printing index name twice (CASSANDRA-14903)
* Add flag to disable SASI indexes, and warnings on creation (CASSANDRA-14866)
Merged from 3.0:
- * Add ability to cap max negotiable protocol version (CASSANDRA-15193)
- * Gossip tokens on startup if available (CASSANDRA-15335)
- * Fix resource leak in CompressedSequentialWriter (CASSANDRA-15340)
- * Fix bad merge that reverted CASSANDRA-14993 (CASSANDRA-15289)
* Fix LegacyLayout RangeTombstoneList IndexOutOfBoundsException when upgrading and RangeTombstone bounds are asymmetric (CASSANDRA-15172)
* Fix NPE when using allocate_tokens_for_keyspace on new DC/rack (CASSANDRA-14952)
* Filter sstables earlier when running cleanup (CASSANDRA-15100)
@@ -93,14 +630,11 @@
* LegacyLayout should handle paging states that cross a collection column (CASSANDRA-15201)
* Prevent RuntimeException when username or password is empty/null (CASSANDRA-15198)
* Multiget thrift query returns null records after digest mismatch (CASSANDRA-14812)
- * Skipping illegal legacy cells can break reverse iteration of indexed partitions (CASSANDRA-15178)
* Handle paging states serialized with a different version than the session's (CASSANDRA-15176)
* Throw IOE instead of asserting on unsupporter peer versions (CASSANDRA-15066)
* Update token metadata when handling MOVING/REMOVING_TOKEN events (CASSANDRA-15120)
* Add ability to customize cassandra log directory using $CASSANDRA_LOG_DIR (CASSANDRA-15090)
- * Skip cells with illegal column names when reading legacy sstables (CASSANDRA-15086)
* Fix assorted gossip races and add related runtime checks (CASSANDRA-15059)
- * Fix mixed mode partition range scans with limit (CASSANDRA-15072)
* cassandra-stress works with frozen collections: list and set (CASSANDRA-14907)
* Fix handling FS errors on writing and reading flat files - LogTransaction and hints (CASSANDRA-15053)
* Avoid double closing the iterator to avoid overcounting the number of requests (CASSANDRA-15058)
@@ -109,28 +643,23 @@
* Add missing commands to nodetool_completion (CASSANDRA-14916)
* Anti-compaction temporarily corrupts sstable state for readers (CASSANDRA-15004)
Merged from 2.2:
- * Catch non-IOException in FileUtils.close to make sure that all resources are closed (CASSANDRA-15225)
* Handle exceptions during authentication/authorization (CASSANDRA-15041)
* Support cross version messaging in in-jvm upgrade dtests (CASSANDRA-15078)
* Fix index summary redistribution cancellation (CASSANDRA-15045)
* Refactor Circle CI configuration (CASSANDRA-14806)
* Fixing invalid CQL in security documentation (CASSANDRA-15020)
- * Multi-version in-JVM dtests (CASSANDRA-14937)
- * Allow instance class loaders to be garbage collected for inJVM dtest (CASSANDRA-15170)
- * Add support for network topology and query tracing for inJVM dtest (CASSANDRA-15319)
3.11.4
* Make stop-server.bat wait for Cassandra to terminate (CASSANDRA-14829)
* Correct sstable sorting for garbagecollect and levelled compaction (CASSANDRA-14870)
Merged from 3.0:
+ * Improve merkle tree size and time on heap (CASSANDRA-14096)
* Severe concurrency issues in STCS,DTCS,TWCS,TMD.Topology,TypeParser
* Add a script to make running the cqlsh tests in cassandra repo easier (CASSANDRA-14951)
* If SizeEstimatesRecorder misses a 'onDropTable' notification, the size_estimates table will never be cleared for that table. (CASSANDRA-14905)
- * Counters fail to increment in 2.1/2.2 to 3.X mixed version clusters (CASSANDRA-14958)
* Streaming needs to synchronise access to LifecycleTransaction (CASSANDRA-14554)
* Fix cassandra-stress write hang with default options (CASSANDRA-14616)
- * Differentiate between slices and RTs when decoding legacy bounds (CASSANDRA-14919)
* Netty epoll IOExceptions caused by unclean client disconnects being logged at INFO (CASSANDRA-14909)
* Unfiltered.isEmpty conflicts with Row extends AbstractCollection.isEmpty (CASSANDRA-14588)
* RangeTombstoneList doesn't properly clean up mergeable or superseded rts in some cases (CASSANDRA-14894)
@@ -157,9 +686,7 @@
* Fix reading columns with non-UTF names from schema (CASSANDRA-14468)
Merged from 2.2:
* CircleCI docker image should bake in more dependencies (CASSANDRA-14985)
- * Don't enable client transports when bootstrap is pending (CASSANDRA-14525)
* MigrationManager attempts to pull schema from different major version nodes (CASSANDRA-14928)
- * Fix incorrect cqlsh results when selecting same columns multiple times (CASSANDRA-13262)
* Returns null instead of NaN or Infinity in JSON strings (CASSANDRA-14377)
Merged from 2.1:
* Paged Range Slice queries with DISTINCT can drop rows from results (CASSANDRA-14956)
@@ -172,7 +699,6 @@
* Reduce nodetool GC thread count (CASSANDRA-14475)
* Fix New SASI view creation during Index Redistribution (CASSANDRA-14055)
* Remove string formatting lines from BufferPool hot path (CASSANDRA-14416)
- * Update metrics to 3.1.5 (CASSANDRA-12924)
* Detect OpenJDK jvm type and architecture (CASSANDRA-12793)
* Don't use guava collections in the non-system keyspace jmx attributes (CASSANDRA-12271)
* Allow existing nodes to use all peers in shadow round (CASSANDRA-13851)
@@ -193,19 +719,17 @@
* Fix regression of lagging commitlog flush log message (CASSANDRA-14451)
* Add Missing dependencies in pom-all (CASSANDRA-14422)
* Cleanup StartupClusterConnectivityChecker and PING Verb (CASSANDRA-14447)
- * Fix deprecated repair error notifications from 3.x clusters to legacy JMX clients (CASSANDRA-13121)
* Cassandra not starting when using enhanced startup scripts in windows (CASSANDRA-14418)
* Fix progress stats and units in compactionstats (CASSANDRA-12244)
* Better handle missing partition columns in system_schema.columns (CASSANDRA-14379)
* Delay hints store excise by write timeout to avoid race with decommission (CASSANDRA-13740)
- * Deprecate background repair and probablistic read_repair_chance table options
- (CASSANDRA-13910)
* Add missed CQL keywords to documentation (CASSANDRA-14359)
* Fix unbounded validation compactions on repair / revert CASSANDRA-13797 (CASSANDRA-14332)
* Avoid deadlock when running nodetool refresh before node is fully up (CASSANDRA-14310)
* Handle all exceptions when opening sstables (CASSANDRA-14202)
* Handle incompletely written hint descriptors during startup (CASSANDRA-14080)
* Handle repeat open bound from SRP in read repair (CASSANDRA-14330)
+ * Use zero as default score in DynamicEndpointSnitch (CASSANDRA-14252)
* Respect max hint window when hinting for LWT (CASSANDRA-14215)
* Adding missing WriteType enum values to v3, v4, and v5 spec (CASSANDRA-13697)
* Don't regenerate bloomfilter and summaries on startup (CASSANDRA-11163)
@@ -217,32 +741,24 @@
* Fully utilise specified compaction threads (CASSANDRA-14210)
* Pre-create deletion log records to finish compactions quicker (CASSANDRA-12763)
Merged from 2.2:
- * Fix bug that prevented compaction of SSTables after full repairs (CASSANDRA-14423)
- * Incorrect counting of pending messages in OutboundTcpConnection (CASSANDRA-11551)
* Fix compaction failure caused by reading un-flushed data (CASSANDRA-12743)
* Use Bounds instead of Range for sstables in anticompaction (CASSANDRA-14411)
* Fix JSON queries with IN restrictions and ORDER BY clause (CASSANDRA-14286)
- * Backport circleci yaml (CASSANDRA-14240)
+ * CQL fromJson(null) throws NullPointerException (CASSANDRA-13891)
Merged from 2.1:
* Check checksum before decompressing data (CASSANDRA-14284)
- * CVE-2017-5929 Security vulnerability in Logback warning in NEWS.txt (CASSANDRA-14183)
3.11.2
* Fix ReadCommandTest (CASSANDRA-14234)
* Remove trailing period from latency reports at keyspace level (CASSANDRA-14233)
- * Backport CASSANDRA-13080: Use new token allocation for non bootstrap case as well (CASSANDRA-14212)
* Remove dependencies on JVM internal classes from JMXServerUtils (CASSANDRA-14173)
* Add DEFAULT, UNSET, MBEAN and MBEANS to `ReservedKeywords` (CASSANDRA-14205)
- * Add Unittest for schema migration fix (CASSANDRA-14140)
* Print correct snitch info from nodetool describecluster (CASSANDRA-13528)
- * Close socket on error during connect on OutboundTcpConnection (CASSANDRA-9630)
* Enable CDC unittest (CASSANDRA-14141)
* Acquire read lock before accessing CompactionStrategyManager fields (CASSANDRA-14139)
- * Split CommitLogStressTest to avoid timeout (CASSANDRA-14143)
* Avoid invalidating disk boundaries unnecessarily (CASSANDRA-14083)
* Avoid exposing compaction strategy index externally (CASSANDRA-14082)
- * Prevent continuous schema exchange between 3.0 and 3.11 nodes (CASSANDRA-14109)
* Fix imbalanced disks when replacing node with same address with JBOD (CASSANDRA-14084)
* Reload compaction strategies when disk boundaries are invalidated (CASSANDRA-13948)
* Remove OpenJDK log warning (CASSANDRA-13916)
@@ -252,8 +768,8 @@
* Round buffer size to powers of 2 for the chunk cache (CASSANDRA-13897)
* Update jackson JSON jars (CASSANDRA-13949)
* Avoid locks when checking LCS fanout and if we should defrag (CASSANDRA-13930)
- * Correctly count range tombstones in traces and tombstone thresholds (CASSANDRA-8527)
Merged from 3.0:
+ * Fix unit test failures in ViewComplexTest (CASSANDRA-14219)
* Add MinGW uname check to start scripts (CASSANDRA-12840)
* Use the correct digest file and reload sstable metadata in nodetool verify (CASSANDRA-14217)
* Handle failure when mutating repaired status in Verifier (CASSANDRA-13933)
@@ -265,7 +781,7 @@
* Accept role names containing forward-slash (CASSANDRA-14088)
* Optimize CRC check chance probability calculations (CASSANDRA-14094)
* Fix cleanup on keyspace with no replicas (CASSANDRA-13526)
- * Fix updating base table rows with TTL not removing view entries (CASSANDRA-14071)
+ * Fix updating base table rows with TTL not removing materialized view entries (CASSANDRA-14071)
* Reduce garbage created by DynamicSnitch (CASSANDRA-14091)
* More frequent commitlog chained markers (CASSANDRA-13987)
* Fix serialized size of DataLimits (CASSANDRA-14057)
@@ -286,7 +802,7 @@
Merged from 2.1:
* Protect against overflow of local expiration time (CASSANDRA-14092)
* RPM package spec: fix permissions for installed jars and config files (CASSANDRA-14181)
- * More PEP8 compiance for cqlsh (CASSANDRA-14021)
+ * More PEP8 compliance for cqlsh
3.11.1
@@ -400,7 +916,6 @@
* Support unaligned memory access for AArch64 (CASSANDRA-13326)
* Improve SASI range iterator efficiency on intersection with an empty range (CASSANDRA-12915).
* Fix equality comparisons of columns using the duration type (CASSANDRA-13174)
- * Obfuscate password in stress-graphs (CASSANDRA-12233)
* Move to FastThreadLocalThread and FastThreadLocal (CASSANDRA-13034)
* nodetool stopdaemon errors out (CASSANDRA-13030)
* Tables in system_distributed should not use gcgs of 0 (CASSANDRA-12954)
@@ -412,6 +927,7 @@
* Fix cqlsh automatic protocol downgrade regression (CASSANDRA-13307)
* Tracing payload not passed from QueryMessage to tracing session (CASSANDRA-12835)
Merged from 3.0:
+ * Filter header only commit logs before recovery (CASSANDRA-13918)
* Ensure int overflow doesn't occur when calculating large partition warning size (CASSANDRA-13172)
* Ensure consistent view of partition columns between coordinator and replica in ColumnFilter (CASSANDRA-13004)
* Failed unregistering mbean during drop keyspace (CASSANDRA-13346)
@@ -420,11 +936,9 @@
* Fix schema digest mismatch during rolling upgrades from versions before 3.0.12 (CASSANDRA-13559)
* Upgrade JNA version to 4.4.0 (CASSANDRA-13072)
* Interned ColumnIdentifiers should use minimal ByteBuffers (CASSANDRA-13533)
- * ReverseIndexedReader may drop rows during 2.1 to 3.0 upgrade (CASSANDRA-13525)
* Fix repair process violating start/end token limits for small ranges (CASSANDRA-13052)
* Add storage port options to sstableloader (CASSANDRA-13518)
* Properly handle quoted index names in cqlsh DESCRIBE output (CASSANDRA-12847)
- * Avoid reading static row twice from old format sstables (CASSANDRA-13236)
* Fix NPE in StorageService.excise() (CASSANDRA-13163)
* Expire OutboundTcpConnection messages by a single Thread (CASSANDRA-13265)
* Fail repair if insufficient responses received (CASSANDRA-13397)
@@ -440,7 +954,6 @@
* Fix 2i page size calculation when there are no regular columns (CASSANDRA-13400)
* Fix the conversion of 2.X expired rows without regular column data (CASSANDRA-13395)
* Fix hint delivery when using ext+internal IPs with prefer_local enabled (CASSANDRA-13020)
- * Fix possible NPE on upgrade to 3.0/3.X in case of IO errors (CASSANDRA-13389)
* Legacy deserializer can create empty range tombstones (CASSANDRA-13341)
* Legacy caching options can prevent 3.0 upgrade (CASSANDRA-13384)
* Use the Kernel32 library to retrieve the PID on Windows and fix startup checks (CASSANDRA-13333)
@@ -463,6 +976,7 @@
* Fix cqlsh COPY for dates before 1900 (CASSANDRA-13185)
* Use keyspace replication settings on system.size_estimates table (CASSANDRA-9639)
* Add vm.max_map_count StartupCheck (CASSANDRA-13008)
+ * Obfuscate password in stress-graphs (CASSANDRA-12233)
* Hint related logging should include the IP address of the destination in addition to
host ID (CASSANDRA-13205)
* Reloading logback.xml does not work (CASSANDRA-13173)
@@ -501,6 +1015,7 @@
* Log stacktrace of uncaught exceptions (CASSANDRA-13108)
* Use portable stderr for java error in startup (CASSANDRA-13211)
* Fix Thread Leak in OutboundTcpConnection (CASSANDRA-13204)
+ * Upgrade netty version to fix memory leak with client encryption (CASSANDRA-13114)
* Coalescing strategy can enter infinite loop (CASSANDRA-13159)
@@ -535,8 +1050,8 @@
* Avoid potential AttributeError in cqlsh due to no table metadata (CASSANDRA-12815)
* Fix RandomReplicationAwareTokenAllocatorTest.testExistingCluster (CASSANDRA-12812)
* Upgrade commons-codec to 1.9 (CASSANDRA-12790)
- * Make the fanout size for LeveledCompactionStrategy to be configurable (CASSANDRA-11550)
* Add duration data type (CASSANDRA-11873)
+ * Make the fanout size for LeveledCompactionStrategy to be configurable (CASSANDRA-11550)
* Fix timeout in ReplicationAwareTokenAllocatorTest (CASSANDRA-12784)
* Improve sum aggregate functions (CASSANDRA-12417)
* Make cassandra.yaml docs for batch_size_*_threshold_in_kb reflect changes in CASSANDRA-10876 (CASSANDRA-12761)
@@ -619,16 +1134,15 @@
* Remove pre-startup check for open JMX port (CASSANDRA-12074)
* Remove compaction Severity from DynamicEndpointSnitch (CASSANDRA-11738)
* Restore resumable hints delivery (CASSANDRA-11960)
- * Properly report LWT contention (CASSANDRA-12626)
+ * Properly record CAS contention (CASSANDRA-12626)
Merged from 3.0:
* Dump threads when unit tests time out (CASSANDRA-13117)
* Better error when modifying function permissions without explicit keyspace (CASSANDRA-12925)
* Indexer is not correctly invoked when building indexes over sstables (CASSANDRA-13075)
+ * Stress daemon help is incorrect (CASSANDRA-12563)
* Read repair is not blocking repair to finish in foreground repair (CASSANDRA-13115)
- * Stress daemon help is incorrect(CASSANDRA-12563)
- * Remove ALTER TYPE support (CASSANDRA-12443)
- * Fix assertion for certain legacy range tombstone pattern (CASSANDRA-12203)
* Replace empty strings with null values if they cannot be converted (CASSANDRA-12794)
+ * Remove support for non-JavaScript UDFs (CASSANDRA-12883)
* Fix deserialization of 2.x DeletedCells (CASSANDRA-12620)
* Add parent repair session id to anticompaction log message (CASSANDRA-12186)
* Improve contention handling on failure to acquire MV lock for streaming and hints (CASSANDRA-12905)
@@ -698,7 +1212,6 @@
* Fix handling of nulls and unsets in IN conditions (CASSANDRA-12981)
* Fix race causing infinite loop if Thrift server is stopped before it starts listening (CASSANDRA-12856)
* CompactionTasks now correctly drops sstables out of compaction when not enough disk space is available (CASSANDRA-12979)
- * Remove support for non-JavaScript UDFs (CASSANDRA-12883)
* Fix DynamicEndpointSnitch noop in multi-datacenter situations (CASSANDRA-13074)
* cqlsh copy-from: encode column names to avoid primary key parsing errors (CASSANDRA-12909)
* Temporarily fix bug that creates commit log when running offline tools (CASSANDRA-8616)
@@ -720,7 +1233,6 @@
* Fix leak errors and execution rejected exceptions when draining (CASSANDRA-12457)
* Fix merkle tree depth calculation (CASSANDRA-12580)
* Make Collections deserialization more robust (CASSANDRA-12618)
- * Better handle invalid system roles table (CASSANDRA-12700)
* Fix exceptions when enabling gossip on nodes that haven't joined the ring (CASSANDRA-12253)
* Fix authentication problem when invoking cqlsh copy from a SOURCE command (CASSANDRA-12642)
* Decrement pending range calculator jobs counter in finally block
@@ -728,9 +1240,9 @@
* Forward writes to replacement node when replace_address != broadcast_address (CASSANDRA-8523)
* Fail repair on non-existing table (CASSANDRA-12279)
* Enable repair -pr and -local together (fix regression of CASSANDRA-7450) (CASSANDRA-12522)
+ * Better handle invalid system roles table (CASSANDRA-12700)
* Split consistent range movement flag correction (CASSANDRA-12786)
Merged from 2.1:
- * Upgrade netty version to fix memory leak with client encryption (CASSANDRA-13114)
* cqlsh copy-from: sort user type fields in csv (CASSANDRA-12959)
* Don't skip sstables based on maxLocalDeletionTime (CASSANDRA-12765)
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 8366579..0a93816 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -1,7 +1,7 @@
# Apache Cassandra and Pull Requests
Apache Cassandra doesn't use GitHub pull requests as part of the development process.
-In fact, this repository is a GitHub mirror of [the official repo](https://git-wip-us.apache.org/repos/asf/cassandra.git). The development team has no control over it. We cannot merge or close any pull requests opened for the apache/cassandra repository, so please don't open them.
+In fact, this repository is a GitHub mirror of [the official repo](https://gitbox.apache.org/repos/asf/cassandra.git).
# How to Contribute
@@ -14,4 +14,5 @@
- Running Cassandra in IDEA [guide](https://wiki.apache.org/cassandra/RunningCassandraInIDEA)
- Running Cassandra in Eclipse [guide](https://wiki.apache.org/cassandra/RunningCassandraInEclipse)
- Cassandra Cluster Manager - [CCM](https://github.com/pcmanus/ccm) and a guide [blog post](http://www.datastax.com/dev/blog/ccm-a-development-tool-for-creating-local-cassandra-clusters)
-- Cassandra Distributed Tests aka [dtests](https://github.com/riptano/cassandra-dtest)
+- Cassandra Distributed Tests aka [dtests](https://github.com/apache/cassandra-dtest)
+- Cassandra Testing Guidelines - see TESTING.md
diff --git a/NEWS.txt b/NEWS.txt
index 077bd8b..7b9676b 100644
--- a/NEWS.txt
+++ b/NEWS.txt
@@ -18,15 +18,6 @@
If you use or plan to use very large TTLS (10 to 20 years), read CASSANDRA-14092.txt
for more information.
-PLEASE READ: CVE-2017-5929 LOGBACK BEFORE 1.2.0 SERIALIZATION VULNERABILITY
-------------------------------------------------------------------
-QOS.ch Logback before 1.2.0 has a serialization vulnerability affecting the
-SocketServer and ServerSocketReceiver components.
-
-Logback has not been upgraded to avoid breaking deployments and customizations
-based on older versions. If you are using vulnerable components you will need
-to upgrade to a newer version of Logback or stop using the vulnerable components.
-
GENERAL UPGRADING ADVICE FOR ANY VERSION
========================================
@@ -42,13 +33,246 @@
'sstableloader' tool. You can upgrade the file format of your snapshots
using the provided 'sstableupgrade' tool.
-3.11.7
-======
+4.0
+===
+
+New features
+------------
+ - Nodes will now bootstrap all intra-cluster connections at startup by default and wait
+ 10 seconds for the all but one node in the local data center to be connected and marked
+ UP in gossip. This prevents nodes from coordinating requests and failing because they
+ aren't able to connect to the cluster fast enough. block_for_peers_timeout_in_secs in
+ cassandra.yaml can be used to configure how long to wait (or whether to wait at all)
+ and block_for_peers_in_remote_dcs can be used to also block on all but one node in
+ each remote DC as well. See CASSANDRA-14297 and CASSANDRA-13993 for more information.
+ - *Experimental* support for Transient Replication and Cheap Quorums introduced by CASSANDRA-14404
+ The intended audience for this functionality is expert users of Cassandra who are prepared
+ to validate every aspect of the database for their application and deployment practices. Future
+ releases of Cassandra will make this feature suitable for a wider audience.
+ - *Experimental* support for Java 11 has been added. JVM options that differ between or are
+ specific for Java 8 and 11 have been moved from jvm.options into jvm8.options and jvm11.options.
+ IMPORTANT: Running C* on Java 11 is *experimental* and do it at your own risk.
+ Compilation recommendations: configure Java 11 SDK via JAVA_HOME and Java 8 SDK via JAVA8_HOME.
+ Release builds require Java 11 + Java 8. Development builds can use Java 8 without 11.
+ - LCS now respects the max_threshold parameter when compacting - this was hard coded to 32
+ before, but now it is possible to do bigger compactions when compacting from L0 to L1.
+ This also applies to STCS-compactions in L0 - if there are more than 32 sstables in L0
+ we will compact at most max_threshold sstables in an L0 STCS compaction. See CASSANDRA-14388
+ for more information.
+ - There is now an option to automatically upgrade sstables after Cassandra upgrade, enable
+ either in `cassandra.yaml:automatic_sstable_upgrade` or via JMX during runtime. See
+ CASSANDRA-14197.
+ - `nodetool refresh` has been deprecated in favour of `nodetool import` - see CASSANDRA-6719
+ for details
+ - An experimental option to compare all merkle trees together has been added - for example, in
+ a 3 node cluster with 2 replicas identical and 1 out-of-date, with this option enabled, the
+ out-of-date replica will only stream a single copy from up-to-date replica. Enable it by adding
+ "-os" to nodetool repair. See CASSANDRA-3200.
+ - The currentTimestamp, currentDate, currentTime and currentTimeUUID functions have been added.
+ See CASSANDRA-13132
+ - Support for arithmetic operations between `timestamp`/`date` and `duration` has been added.
+ See CASSANDRA-11936
+ - Support for arithmetic operations on number has been added. See CASSANDRA-11935
+ - Preview expected streaming required for a repair (nodetool repair --preview), and validate the
+ consistency of repaired data between nodes (nodetool repair --validate). See CASSANDRA-13257
+ - Support for selecting Map values and Set elements has been added for SELECT queries. See CASSANDRA-7396
+ - Change-Data-Capture has been modified to make CommitLogSegments available
+ immediately upon creation via hard-linking the files. This means that incomplete
+ segments will be available in cdc_raw rather than fully flushed. See documentation
+ and CASSANDRA-12148 for more detail.
+ - The initial build of materialized views can be parallelized. The number of concurrent builder
+ threads is specified by the property `cassandra.yaml:concurrent_materialized_view_builders`.
+ This property can be modified at runtime through both JMX and the new `setconcurrentviewbuilders`
+ and `getconcurrentviewbuilders` nodetool commands. See CASSANDRA-12245 for more details.
+ - There is now a binary full query log based on Chronicle Queue that can be controlled using
+ nodetool enablefullquerylog, disablefullquerylog, and resetfullquerylog. The log
+ contains all queries invoked, approximate time they were invoked, any parameters necessary
+ to bind wildcard values, and all query options. A human readable version of the log can be
+ dumped or tailed using the new bin/fqltool utility. The full query log is designed to be safe
+ to use in production and limits utilization of heap memory and disk space with limits
+ you can specify when enabling the log.
+ See nodetool and fqltool help text for more information.
+ - SSTableDump now supports the -l option to output each partition as it's own json object
+ See CASSANDRA-13848 for more detail
+ - Metric for coordinator writes per table has been added. See CASSANDRA-14232
+ - Nodetool cfstats now has options to sort by various metrics as well as limit results.
+ - Operators can restrict login user activity to one or more datacenters. See `network_authorizer`
+ in cassandra.yaml, and the docs for create and alter role statements. CASSANDRA-13985
+ - Roles altered from login=true to login=false will prevent existing connections from executing any
+ statements after the cache has been refreshed. CASSANDRA-13985
+ - Support for audit logging of database activity. If enabled, logs every incoming
+ CQL command request, Authentication (successful as well as unsuccessful login) to a node.
+ - Faster streaming of entire SSTables using ZeroCopy APIs. If enabled, Cassandra will use stream
+ entire SSTables, significantly speeding up transfers. Any streaming related operations will see
+ corresponding improvement. See CASSANDRA-14556.
+ - NetworkTopologyStrategy now supports auto-expanding the replication_factor
+ option into all available datacenters at CREATE or ALTER time. For example,
+ specifying replication_factor: 3 translates to three replicas in every
+ datacenter. This auto-expansion will _only add_ datacenters for safety.
+ See CASSANDRA-14303 for more details.
+ - Added Python 3 support so cqlsh and cqlshlib is now compatible with Python 2.7 and Python 3.6.
+ Added --python option to cqlsh so users can specify the path to their chosen Python interpreter.
+ See CASSANDRA-10190 for details.
+ - Support for server side DESCRIBE statements has been added. See CASSANDRA-14825
Upgrading
---------
- - Nothing specific to this release, but please see previous upgrading sections,
- especially if you are upgrading from 3.0.
+ - Sstables for tables using with a frozen UDT written by C* 3.0 appear as corrupted.
+
+ Background: The serialization-header in the -Statistics.db sstable component contains the type information
+ of the table columns. C* 3.0 write incorrect type information for frozen UDTs by omitting the
+ "frozen" information. Non-frozen UDTs were introduced by CASSANDRA-7423 in C* 3.6. Since then, the missing
+ "frozen" information leads to deserialization issues that result in CorruptSSTableExceptions, potentially other
+ exceptions as well.
+
+ As a mitigation, the sstable serialization-headers are rewritten to contain the missing "frozen" information for
+ UDTs once, when an upgrade from C* 3.0 is detected. This migration does not touch snapshots or backups.
+
+ The sstablescrub tool now performs a check of the sstable serialization-header against the schema. A mismatch of
+ the types in the serialization-header and the schema will cause sstablescrub to error out and stop by default.
+ See the new `-e` option. `-e off` disables the new validation code. `-e fix` or `-e fix-only`, e.g.
+ `sstablescrub -e fix keyspace table`, will validate the serialization-header, rewrite the non-frozen UDTs
+ in the serialzation-header to frozen UDTs, if that matches the schema, and continue with scrub.
+ See `sstablescrub -h`.
+ (CASSANDRA-15035)
+ - CASSANDRA-13241 lowered the default chunk_lengh_in_kb for compresesd tables from
+ 64kb to 16kb. For highly compressible data this can have a noticeable impact
+ on space utilization. You may want to consider manually specifying this value.
+ - Additional columns have been added to system_distributed.repair_history,
+ system_traces.sessions and system_traces.events. As a result select queries
+ against these tables - including queries against tracing tables performed
+ automatically by the drivers and cqlsh - will fail and generate an error in the log
+ during upgrade when the cluster is mixed version. On 3.x side this will also lead
+ to broken internode connections and lost messages.
+ Cassandra versions 3.0.20 and 3.11.6 pre-add these columns (see CASSANDRA-15385),
+ so please make sure to upgrade to those versions or higher before upgrading to
+ 4.0 for query tracing to not cause any issues during the upgrade to 4.0.
+ - Timestamp ties between values resolve differently: if either value has a TTL,
+ this value always wins. This is to provide consistent reconciliation before
+ and after the value expires into a tombstone.
+ - Cassandra 4.0 removed support for COMPACT STORAGE tables. All Compact Tables
+ have to be migrated using `ALTER ... DROP COMPACT STORAGE` statement in 3.0/3.11.
+ Cassandra starting 4.0 will not start if flags indicate that the table is non-CQL.
+ Syntax for creating compact tables is also deprecated.
+ - Support for legacy auth tables in the system_auth keyspace (users,
+ permissions, credentials) and the migration code has been removed. Migration
+ of these legacy auth tables must have been completed before the upgrade to
+ 4.0 and the legacy tables must have been removed. See the 'Upgrading' section
+ for version 2.2 for migration instructions.
+ - Cassandra 4.0 removed support for the deprecated Thrift interface. Amongst
+ other things, this implies the removal of all yaml options related to thrift
+ ('start_rpc', rpc_port, ...).
+ - Cassandra 4.0 removed support for any pre-3.0 format. This means you
+ cannot upgrade from a 2.x version to 4.0 directly, you have to upgrade to
+ a 3.0.x/3.x version first (and run upgradesstable). In particular, this
+ mean Cassandra 4.0 cannot load or read pre-3.0 sstables in any way: you
+ will need to upgrade those sstable in 3.0.x/3.x first.
+ - Upgrades from 3.0.x or 3.x are supported since 3.0.13 or 3.11.0, previous
+ versions will causes issues during rolling upgrades (CASSANDRA-13274).
+ - Cassandra will no longer allow invalid keyspace replication options, such
+ as invalid datacenter names for NetworkTopologyStrategy. Operators MUST
+ add new nodes to a datacenter before they can set set ALTER or CREATE
+ keyspace replication policies using that datacenter. Existing keyspaces
+ will continue to operate, but CREATE and ALTER will validate that all
+ datacenters specified exist in the cluster.
+ - Cassandra 4.0 fixes a problem with incremental repair which caused repaired
+ data to be inconsistent between nodes. The fix changes the behavior of both
+ full and incremental repairs. For full repairs, data is no longer marked
+ repaired. For incremental repairs, anticompaction is run at the beginning
+ of the repair, instead of at the end. If incremental repair was being used
+ prior to upgrading, a full repair should be run after upgrading to resolve
+ any inconsistencies.
+ - Config option index_interval has been removed (it was deprecated since 2.0)
+ - Deprecated repair JMX APIs are removed.
+ - The version of snappy-java has been upgraded to 1.1.2.6
+ - the miniumum value for internode message timeouts is 10ms. Previously, any
+ positive value was allowed. See cassandra.yaml entries like
+ read_request_timeout_in_ms for more details.
+ - Cassandra 4.0 allows a single port to be used for both secure and insecure
+ connections between cassandra nodes (CASSANDRA-10404). See the yaml for
+ specific property changes, and see the security doc for full details.
+ - Due to the parallelization of the initial build of materialized views,
+ the per token range view building status is stored in the new table
+ `system.view_builds_in_progress`. The old table `system.views_builds_in_progress`
+ is no longer used and can be removed. See CASSANDRA-12245 for more details.
+ - Config option commitlog_sync_batch_window_in_ms has been deprecated as it's
+ documentation has been incorrect and the setting itself near useless.
+ Batch mode remains a valid commit log mode, however.
+ - There is a new commit log mode, group, which is similar to batch mode
+ but blocks for up to a configurable number of milliseconds between disk flushes.
+ - nodetool clearsnapshot now required the --all flag to remove all snapshots.
+ Previous behavior would delete all snapshots by default.
+ - Nodes are now identified by a combination of IP, and storage port.
+ Existing JMX APIs, nodetool, and system tables continue to work
+ and accept/return just an IP, but there is a new
+ version of each that works with the full unambiguous identifier.
+ You should prefer these over the deprecated ambiguous versions that only
+ work with an IP. This was done to support multiple instances per IP.
+ Additionally we are moving to only using a single port for encrypted and
+ unencrypted traffic and if you want multiple instances per IP you must
+ first switch encrypted traffic to the storage port and not a separate
+ encrypted port. If you want to use multiple instances per IP
+ with SSL you will need to use StartTLS on storage_port and set
+ outgoing_encrypted_port_source to gossip outbound connections
+ know what port to connect to for each instance. Before changing
+ storage port or native port at nodes you must first upgrade the entire cluster
+ and clients to 4.0 so they can handle the port not being consistent across
+ the cluster.
+ - Names of AWS regions/availability zones have been cleaned up to more correctly
+ match the Amazon names. There is now a new option in conf/cassandra-rackdc.properties
+ that lets users enable the correct names for new clusters, or use the legacy
+ names for existing clusters. See conf/cassandra-rackdc.properties for details.
+ - Background repair has been removed. dclocal_read_repair_chance and
+ read_repair_chance table options have been removed and are now rejected.
+ See CASSANDRA-13910 for details.
+ - Internode TCP connections that do not ack segments for 30s will now
+ be automatically detected and closed via the Linux TCP_USER_TIMEOUT
+ socket option. This should be exceedingly rare, but AWS networks (and
+ other stateful firewalls) apparently suffer from this issue. You can
+ tune the timeouts on TCP connection and segment ack via the
+ `cassandra.yaml:internode_tcp_connect_timeout_in_ms` and
+ `cassandra.yaml:internode_tcp_user_timeout_in_ms` options respectively.
+ See CASSANDRA-14358 for details.
+ - repair_session_space_in_mb setting has been added to cassandra.yaml to allow operators to reduce
+ merkle tree size if repair is creating too much heap pressure. The repair_session_max_tree_depth
+ setting added in 3.0.19 and 3.11.5 is deprecated in favor of this setting. See CASSANDRA-14096
+ - The flags 'enable_materialized_views' and 'enable_sasi_indexes' in cassandra.yaml
+ have been set as false by default. Operators should modify them to allow the
+ creation of new views and SASI indexes, the existing ones will continue working.
+ See CASSANDRA-14866 for details.
+ - CASSANDRA-15216 - The flag 'cross_node_timeout' has been set as true by default.
+ This change is done under the assumption that users have setup NTP on
+ their clusters or otherwise synchronize their clocks, and that clocks are
+ mostly in sync, since this is a requirement for general correctness of
+ last write wins.
+ - CASSANDRA-15257 removed the joda time dependency. Any time formats
+ passed will now need to conform to java.time.format.DateTimeFormatter.
+ Most notably, days and months must be two digits, and years exceeding
+ four digits need to be prefixed with a plus or minus sign.
+ - cqlsh now returns a non-zero code in case of errors. This is a backward incompatible change so it may
+ break existing scripts that rely on the current behavior. See CASSANDRA-15623 for more details.
+ - Updated the default compaction_throughput_mb_per_sec to to 64. The original
+ default (16) was meant for spinning disk volumes. See CASSANDRA-14902 for details.
+
+
+Deprecation
+-----------
+
+ - The JMX MBean org.apache.cassandra.db:type=BlacklistedDirectories has been
+ deprecated in favor of org.apache.cassandra.db:type=DisallowedDirectories
+ and will be removed in a subsequent major version.
+
+
+Materialized Views
+-------------------
+ - Following a discussion regarding concerns about the design and safety of Materialized Views, the C* development
+ community no longer recommends them for production use, and considers them experimental. Warnings messages will
+ now be logged when they are created. (See https://www.mail-archive.com/dev@cassandra.apache.org/msg11511.html)
+ - An 'enable_materialized_views' flag has been added to cassandra.yaml to allow operators to prevent creation of
+ views
+ - CREATE MATERIALIZED VIEW syntax has become stricter. Partition key columns are no longer implicitly considered
+ to be NOT NULL, and no base primary key columns get automatically included in view definition. You have to
+ specify them explicitly now.
3.11.6
======
@@ -73,21 +297,12 @@
in the serialzation-header to frozen UDTs, if that matches the schema, and continue with scrub.
See `sstablescrub -h`.
(CASSANDRA-15035)
- - repair_session_max_tree_depth setting has been added to cassandra.yaml to allow operators to reduce
- merkle tree size if repair is creating too much heap pressure. See CASSANDRA-14096 for details.
+ - repair_session_max_tree_depth setting has been added to cassandra.yaml to allow operators to reduce
+ merkle tree size if repair is creating too much heap pressure. See CASSANDRA-14096 for details.
3.11.5
======
-Upgrading
----------
- - repair_session_max_tree_depth setting has been added to cassandra.yaml to allow operators to reduce
- merkle tree size if repair is creating too much heap pressure. See CASSANDRA-14096 for details.
- - native_transport_max_negotiable_protocol_version has been added to cassandra.yaml to allow operators to
- enforce an upper limit on the version of the native protocol that servers will negotiate with clients.
- This can be used during upgrades from 2.1 to 3.0 to prevent errors due to incompatible paging state formats
- between the two versions. See CASSANDRA-15193 for details.
-
Experimental features
---------------------
- An 'enable_sasi_indexes' flag, true by default, has been added to cassandra.yaml to allow operators to prevent
@@ -111,26 +326,17 @@
Upgrading
---------
- - Materialized view users upgrading from 3.0.15 (3.0.X series) or 3.11.1 (3.11.X series) and
- later that have performed range movements (join, decommission, move, etc), should run repair
- on the base tables, and subsequently on the views to ensure data affected by CASSANDRA-14251
- is correctly propagated to all replicas.
- - Changes to bloom_filter_fp_chance will no longer take effect on existing sstables when the
- node is restarted. Only compactions/upgradesstables regenerates bloom filters and Summaries
- sstable components. See CASSANDRA-11163
-
-Deprecation
------------
- - Background read repair has been deprecated. dclocal_read_repair_chance and read_repair_chance
- table options have been deprecated, and will be removed entirely in 4.0. See CASSANDRA-13910
- for details.
+ - Materialized view users upgrading from 3.0.15 (3.0.X series) or 3.11.1 (3.11.X series) and later that have performed range movements (join, decommission, move, etc),
+ should run repair on the base tables, and subsequently on the views to ensure data affected by CASSANDRA-14251 is correctly propagated to all replicas.
+ - Changes to bloom_filter_fp_chance will no longer take effect on existing sstables when the node is restarted. Only
+ compactions/upgradesstables regenerates bloom filters and Summaries sstable components. See CASSANDRA-11163
3.11.2
======
Upgrading
---------
- - See MAXIMUM TTL EXPIRATION DATE NOTICE above.
+ - See MAXIMUM TTL EXPIRATION DATE NOTICE above.
- Cassandra is now relying on the JVM options to properly shutdown on OutOfMemoryError. By default it will
rely on the OnOutOfMemoryError option as the ExitOnOutOfMemoryError and CrashOnOutOfMemoryError options
are not supported by the older 1.7 and 1.8 JVMs. A warning will be logged at startup if none of those JVM
@@ -138,23 +344,6 @@
- Cassandra is not logging anymore by default an Heap histogram on OutOfMemoryError. To enable that behavior
set the 'cassandra.printHeapHistogramOnOutOfMemoryError' System property to 'true'. See CASSANDRA-13006
for more details.
- - Upgrades from 3.0 might have produced unnecessary schema migrations while
- there was at least one 3.0 node in the cluster. It is therefore highly
- recommended to upgrade from 3.0 to at least 3.11.2. The root cause of
- this schema mismatch was a difference in the way how schema digests were computed
- in 3.0 and 3.11.2. To mitigate this issue, 3.11.2 and newer announce
- 3.0 compatible digests as long as there is at least one 3.0 node in the
- cluster. Once all nodes have been upgraded, the "real" schema version will be
- announced. Note: this fix is only necessary in 3.11.2 and therefore only applies
- to 3.11. (CASSANDRA-14109)
-
-Materialized Views
--------------------
- - Following a discussion regarding concerns about the design and safety of Materialized Views, the C* development
- community no longer recommends them for production use, and considers them experimental. Warnings messages will
- now be logged when they are created. (See https://www.mail-archive.com/dev@cassandra.apache.org/msg11511.html)
- - An 'enable_materialized_views' flag has been added to cassandra.yaml to allow operators to prevent creation of
- views
3.11.1
======
@@ -189,7 +378,7 @@
Materialized Views
-------------------
-Materialized Views (only when upgrading from 3.X or any version lower than 3.0.15)
+Materialized Views (only when upgrading from any version lower than 3.0.15 (3.0 series) or 3.11.1 (3.X series))
---------------------------------------------------------------------------------------
- Cassandra will no longer allow dropping columns on tables with Materialized Views.
- A change was made in the way the Materialized View timestamp is computed, which
@@ -212,31 +401,12 @@
Upgrading
---------
- - ALTER TABLE (ADD/DROP COLUMN) operations concurrent with a read might
- result into data corruption (see CASSANDRA-13004 for more details).
- Fixing this bug required a messaging protocol version bump. By default,
- Cassandra 3.11 will use 3014 version for messaging.
-
- Since Schema Migrations rely the on exact messaging protocol version
- match between nodes, if you need schema changes during the upgrade
- process, you have to start your nodes with `-Dcassandra.force_3_0_protocol_version=true`
- first, in order to temporarily force a backwards compatible protocol.
- After the whole cluster is upgraded to 3.11, do a rolling
- restart of the cluster without setting that flag.
-
- 3.11 nodes with and withouot the flag set will be able to do schema
- migrations with other 3.x and 3.0.x releases.
-
- While running the cluster with the flag set to true on 3.11 (in
- compatibility mode), avoid adding or removing any columns to/from
- existing tables.
-
- If your cluster can do without schema migrations during the upgrade
- time, just start the cluster normally without setting aforementioned
- flag.
-
- If you are upgrading from 3.0.14+ (of 3.0.x branch), you do not have
- to set an flag while upgrading to ensure schema migrations.
+ - Creating Materialized View with filtering on non-primary-key base column
+ (added in CASSANDRA-10368) is disabled, because the liveness of view row
+ is depending on multiple filtered base non-key columns and base non-key
+ column used in view primary-key. This semantic cannot be supported without
+ storage format change, see CASSANDRA-13826. For append-only use case, you
+ may still use this feature with a startup flag: "-Dcassandra.mv.allow_filtering_nonkey_columns_unsafe=true"
- The NativeAccessMBean isAvailable method will only return true if the
native library has been successfully linked. Previously it was returning
true if JNA could be found but was not taking into account link failures.
diff --git a/NOTICE.txt b/NOTICE.txt
index 8162b32..d135e1e 100644
--- a/NOTICE.txt
+++ b/NOTICE.txt
@@ -1,9 +1,13 @@
Apache Cassandra
-Copyright 2009-2019 The Apache Software Foundation
+Copyright 2009-2020 The Apache Software Foundation
This product includes software developed by The Apache Software
Foundation (http://www.apache.org/).
+Parts of the DataStax Java Driver included in source form
+(https://github.com/datastax/java-driver)
+Copyright DataStax, Inc.
+
Some alternate data structures provided by high-scale-lib from
http://sourceforge.net/projects/high-scale-lib/.
Written by Cliff Click and released as Public Domain.
@@ -46,13 +50,6 @@
Contains bindings to the C LZ4 implementation (http://code.google.com/p/lz4/)
Copyright (C) 2011-2012, Yann Collet.
-Alternative Disruptor backed thrift server from https://github.com/xedin/disruptor_thrift_server
-Written by Pavel Yaskevich.
-
-LMAX Disruptor
-(http://lmax-exchange.github.io/disruptor/)
-Copyright 2011 LMAX Ltd.
-
Airline
(https://github.com/airlift/airline)
Copyright 2011, Dain Sundstrom dain@iq80.com
diff --git a/README.asc b/README.asc
index 910fb64..20fee32 100644
--- a/README.asc
+++ b/README.asc
@@ -84,7 +84,8 @@
Wondering where to go from here?
- * Join us in #cassandra on irc.freenode.net and ask questions
+ * Join us in #cassandra on the https://s.apache.org/slack-invite[ASF Slack] and ask questions
* Subscribe to the Users mailing list by sending a mail to
user-subscribe@cassandra.apache.org
* Visit the http://cassandra.apache.org/community/[community section] of the Cassandra website for more information on getting involved.
+ * Visit the http://cassandra.apache.org/doc/latest/development/index.html[development section] of the Cassandra website for more information on how to contribute.
diff --git a/TESTING.md b/TESTING.md
new file mode 100644
index 0000000..b6e27aa
--- /dev/null
+++ b/TESTING.md
@@ -0,0 +1,448 @@
+The goal of this document is to establish guidelines on writing tests that drive incremental improvement of the test coverage and testability of
+Cassandra, without overly burdening day to day work. While not every point here will be immediately practical to implement or relevant for every
+contribution, it errs on the side of not making rules out of potential exceptions. It provides guidelines on test scope, style, and goals, as
+well as guidelines for dealing with global state and refactoring untested code.
+
+## What to Test
+
+There are 3 main types of tests in Cassandra, unit tests, integration tests, and dtests. Below, each type of test is described, and a high level description of
+what should and shouldn't be tested in each is given.
+
+### Unit Tests
+JUnit tests of smaller components that are fairly narrow in scope (ie: data structures, verb handlers, helper classes)
+
+#### What should be tested
+ * All state transitions should be tested
+ * Illegal state transitions should be tested (that they throw exceptions)
+ * all conditional branches should be tested.
+ * Code that deals with ranges of values should have tests around expected ranges, unexpected ranges, different functional ranges and their boundaries.
+ * Exception handling should be tested.
+
+#### What shouldn't be tested
+* implementation details (test that the system under test works a certain way, not that it's implemented a certain way)
+
+### Integration Tests
+JUnit tests of larger components with a lot of moving parts, usually involved in some internode communication (ie: Gossip, MessagingService).
+The smaller components that make up the system under test in an integration test should be individually unit tested.
+
+#### What should be tested
+* messages are sent when expected
+* received messages have the intended side effects
+* internal interfaces work as expected
+* external interfaces are interacted with as expected
+* multiple instances of components interact as expected (with a mocked messaging service, and other dependencies, where appropriate)
+* dry start - test that a system starts up properly the first time a node start
+* restart - test that a system starts up properly on node restart (from both clean and unclean shutdown)
+* shutdown - test that a system can shutdown properly
+* upgrade - test that a system is able to restart with data from a previous version
+
+#### What shouldn't be tested
+* The rest of the application. It should be possible to test large systems without starting the entire database through the use of mocks.
+
+**Note:** it's generally not a good idea to mock out the storage layer if the component under test needs to interact with it. If you're testing
+how multiple instances interact with each other, AND they need to use the storage layer, parameterizing the keyspace/table they store data is
+the way to do it.
+
+### dtests
+python/ccm tests that start local clusters and interact with them via the python client.
+
+dtests are effectively black box tests, and should be used for checking that clusters and client side interfaces work as expected. They are not
+a replacement for proper functional java tests. They take much longer to run, and are much less flexible in what they can test.
+
+Systems under test in dtest should have more granular integration tests as well.
+
+#### What should be tested
+* end to end cluster functionality
+* client contracts
+* trivial (to create) failure cases
+
+#### What shouldn't be tested
+* internal implementation details
+
+## Expanded Guidelines
+
+This section has more in depth descriptions and reasoning about some of the points raised in the previous section.
+
+### Test structure
+
+Tests cases should have a clear progression of setup, precondition check, performing some action under test, then a postcondition check.
+
+**Example**
+
+```java
+@Test
+public void increment() throws Exception
+{
+ // setup code
+ int x = 1;
+
+ // check preconditions
+ assertEquals(1, x);
+
+ // perform the state changing action under test
+ x++;
+
+ // check post conditions
+ assertEquals(2, x);
+}
+```
+
+#### Reason
+
+Test cases should be optimized for readability
+
+#### Exceptions
+
+Cases where simple cases can be tested in one line. Such as validation logic checks:
+property-based state testing (ie: ScalaCheck/QuickCheck)
+
+*Example:*
+```java
+@Test
+public void validation()
+{
+ assertValidationFailure(b -> b.withState(null));
+ assertValidationFailure(b -> b.withSessionID(null));
+ assertValidationFailure(b -> b.withCoordinator(null));
+ assertValidationFailure(b -> b.withTableIds(null));
+ assertValidationFailure(b -> b.withTableIds(new HashSet<>()));
+ assertValidationFailure(b -> b.withRepairedAt(0));
+ assertValidationFailure(b -> b.withRepairedAt(-1));
+ assertValidationFailure(b -> b.withRanges(null));
+ assertValidationFailure(b -> b.withRanges(new HashSet<>()));
+ assertValidationFailure(b -> b.withParticipants(null));
+ assertValidationFailure(b -> b.withParticipants(new HashSet<>()));
+ assertValidationFailure(b -> b.withStartedAt(0));
+ assertValidationFailure(b -> b.withLastUpdate(0));
+}
+```
+
+### Test distributed components in junit
+
+Components that rely on nodes communicating with each other should be testable in java.
+
+#### Reason
+
+One of the more difficult aspects of distributed systems is ensuring that they continue to behave correctly during various failure modes. This includes weird
+edge cases involving specific ordering of events between nodes that rarely occur in the wild. Testing these sorts of scenarios is much easier in junit because
+mock 'clusters' can be placed in specific states, and deterministically stepped through a sequence of events, ensuring that they behave as expected, and are in
+the expected state after each step.
+
+#### Exceptions
+
+This rule mainly applies to new or significantly overhauled systems. Older systems *should* be refactored to be thoroughly tested, but it's not necessarily a
+prerequisite for working on them.
+
+### Test all branches and inputs.
+
+All branches and inputs of a method should be exercised. For branches that require multiple criteria to be met, (ie `x > 10 && y < 100`), there
+should be tests demonstrating that the branch is taken when all critera are met (ie `x=11,y=99`), and that it is not taken when only one is met
+(ie: `x=11, y=200 or x=5,y=99`). If a method deals with ranges of values, (ie `x >= 10`), the boundaries of the ranges should be tested (ie: `x=9, x=10`)
+
+In the following example
+
+**Example**
+```java
+class SomeClass
+{
+ public static int someFunction(bool aFlag, int aValue)
+ {
+ if (aFlag && aValue > 10)
+ {
+ return 20;
+ }
+ else if (aValue > 5)
+ {
+ return 10;
+ else
+ {
+ return 0;
+ }
+ }
+}
+
+class SomeTest
+{
+ public void someFunction() throws Exception
+ {
+ assertEquals(10, somefunction(true, 11));
+ assertEquals(5, somefunction(false, 11));
+ assertEquals(5, somefunction(true, 8));
+ assertEquals(5, somefunction(false, 8));
+ assertEquals(0, somefunction(false, 4));
+ }
+}
+```
+
+### Test any state transitions
+
+As an extension of testing all branches and inputs. For stateful systems, there should be tests demonstrating that states change under the intended
+circumstances, and that state changes have the intended side effects.
+
+### Test unsupported arguments and states throw exceptions
+
+If a system is not intended to perform an action in a given state (ie: a node performing reads during bootstrap), or a method is not intended
+to encounter some type of argument (ie: a method that is only designed to work with numeric values > 0), then there should be tests demonstrating
+that an appropriate exception is raised (IllegalStateException or IllegalArgumentException, respectively) in that case.
+
+The guava preconditions module makes this straightforward.
+
+#### Reason
+
+Inadvertent misuse of methods and systems cause bugs that are often silent and subtle. Raising exceptions on unintended usage helps
+protect against future bugs and reduces developer surprise.
+
+## Dealing with global state
+
+Unfortunately, the project has extensive amounts of global state which makes actually writing robust tests difficult, but not impossible.
+
+Having dependencies on global state is not an excuse to not test something, or to throw a dtest or assertion at it and call it a day.
+
+Structuring code in a way that interacts with global state that can still be deterministically tested just takes a few tweaks
+
+**Example, bad**
+```java
+class SomeVerbHandler implements IVerbHandler<SomeMessage>
+{
+ public void doVerb(MessageIn<SomeMessage> msg)
+ {
+ if (FailureDetector.instance.isAlive(msg.payload.otherNode))
+ {
+ new StreamPlan(msg.payload.otherNode).requestRanges(someRanges).execute();
+ }
+ else
+ {
+ CompactionManager.instance.submitBackground(msg.payload.cfs);
+ }
+ }
+}
+```
+
+In this made up example, we're checking global state, and then taking some action against other global state. None of the global state
+we're working with is easy to manipulate for tests, so comprehensive tests for this aren't very likely to be written. Even worse, whether
+the FailureDetector, streaming, or compaction work properly are out of scope for our purposes. We're concerned with whether SomeVerbHandler
+works as expected.
+
+Ideally though, classes won't have dependencies on global state at all, and have everything they need to work passed in as constructor arguments.
+This also enables comprehensive testing, and stops the spread of global state.
+
+This prevents the spread of global state, and also begins to identify and define the internal interfaces that will replace global state.
+
+**Example, better**
+```java
+class SomeVerbHandler implements IVerbHandler<SomeMessage>
+{
+ private final IFailureDetector failureDetector;
+ private final ICompactionManager compactionManager;
+ private final IStreamManager streamManager;
+
+ public SomeVerbHandler(IFailureDetector failureDetector, ICompactionManager compactionManager, IStreamManager streamManager)
+ {
+ this.failureDetector = failureDetector;
+ this.compactionManager = compactionManager;
+ this.streamManager = streamManager;
+ }
+
+ public void doVerb(MessageIn<SomeMessage> msg)
+ {
+ if (failureDetector.isAlive(msg.payload.otherNode))
+ {
+ streamExecutor.submitPlan(new StreamPlan(msg.payload.otherNode).requestRanges(someRanges));
+ }
+ else
+ {
+ compactionManager.submitBackground(msg.payload.cfs);
+ }
+ }
+}
+```
+
+**Example test**
+```java
+class SomeVerbTest
+{
+ class InstrumentedFailureDetector implements IFailureDetector
+ {
+ boolean alive = false;
+ @Override
+ public boolean isAlive(InetAddress address)
+ {
+ return alive;
+ }
+ }
+
+ class InstrumentedCompactionManager implements ICompactionManager
+ {
+ boolean submitted = false;
+ @Override
+ public void submitBackground(ColumnFamilyStore cfs)
+ {
+ submitted = true;
+ }
+ }
+
+ class InstrumentedStreamManager implements IStreamManager
+ {
+ boolean submitted = false;
+ @Override
+ public void submitPlan(StreamPlan plan)
+ {
+ submitted = true;
+ }
+ }
+
+ @Test
+ public void liveNode() throws Exception
+ {
+ InstrumentedFailureDetector failureDetector = new InstrumentedFailureDetector();
+ failureDetector.alive = true;
+ InstrumentedCompactionManager compactionManager = new InstrumentedCompactionManager();
+ InstrumentedStreamManager streamManager = new InstrumentedStreamManager();
+ SomeVerbHandler handler = new SomeVerbHandler(failureDetector, compactionManager, streamManager);
+
+ MessageIn<SomeMessage> msg = new MessageIn<>(...);
+
+ assertFalse(streamManager.submitted);
+ assertFalse(compactionManager.submitted);
+
+ handler.doVerb(msg);
+
+ assertTrue(streamManager.submitted);
+ assertFalse(compactionManager.submitted);
+ }
+
+ @Test
+ public void deadNode() throws Exception
+ {
+ InstrumentedFailureDetector failureDetector = new InstrumentedFailureDetector();
+ failureDetector.alive = false;
+ InstrumentedCompactionManager compactionManager = new InstrumentedCompactionManager();
+ InstrumentedStreamManager streamManager = new InstrumentedStreamManager();
+ SomeVerbHandler handler = new SomeVerbHandler(failureDetector, compactionManager, streamManager);
+
+ MessageIn<SomeMessage> msg = new MessageIn<>(...);
+
+ assertFalse(streamManager.submitted);
+ assertFalse(compactionManager.submitted);
+
+ handler.doVerb(msg);
+
+ assertFalse(streamManager.submitted);
+ assertTrue(compactionManager.submitted);
+ }
+}
+```
+
+By abstracting away accesses to global state we can exhaustively test the paths this verb handler can take, and directly confirm that it's taking the correct
+actions. Obviously, this is a simple example, but for classes or functions with more complex logic, this makes comprehensive testing much easier.
+
+Note that the interfaces used here probably shouldn't be the same ones we use for MBeans.
+
+However, in some cases, passing interfaces into the constructor may not be practical. Classes that are instantiated on startup need to be handled with care, since accessing
+a singleton may change the initialization order of the database. It may also be a larger change than is warranted for something like a bug fix. In any case, if passing
+dependencies into the constructor is not practical, wrapping accesses to global state in protected methods that are overridden for tests will achieve the same thing.
+
+
+**Example, alternative**
+```javayy
+class SomeVerbHandler implements IVerbHandler<SomeMessage>
+{
+ @VisibleForTesting
+ protected boolean isAlive(InetAddress addr) { return FailureDetector.instance.isAlive(msg.payload.otherNode); }
+
+ @VisibleForTesting
+ protected void streamSomethind(InetAddress to) { new StreamPlan(to).requestRanges(someRanges).execute(); }
+
+ @VisibleForTesting
+ protected void compactSomething(ColumnFamilyStore cfs ) { CompactionManager.instance.submitBackground(); }
+
+ public void doVerb(MessageIn<SomeMessage> msg)
+ {
+ if (isAlive(msg.payload.otherNode))
+ {
+ streamSomething(msg.payload.otherNode);
+ }
+ else
+ {
+ compactSomething();
+ }
+ }
+}
+```
+
+**Example test**
+```java
+class SomeVerbTest
+{
+ static class InstrumentedSomeVerbHandler extends SomeVerbHandler
+ {
+ public boolean alive = false;
+ public boolean streamCalled = false;
+ public boolean compactCalled = false;
+
+ @Override
+ protected boolean isAlive(InetAddress addr) { return alive; }
+
+ @Override
+ protected void streamSomethind(InetAddress to) { streamCalled = true; }
+
+ @Override
+ protected void compactSomething(ColumnFamilyStore cfs ) { compactCalled = true; }
+ }
+
+ @Test
+ public void liveNode() throws Exception
+ {
+ InstrumentedSomeVerbHandler handler = new InstrumentedSomeVerbHandler();
+ handler.alive = true;
+ MessageIn<SomeMessage> msg = new MessageIn<>(...);
+
+ assertFalse(handler.streamCalled);
+ assertFalse(handler.compactCalled);
+
+ handler.doVerb(msg);
+
+ assertTrue(handler.streamCalled);
+ assertFalse(handler.compactCalled);
+ }
+
+ @Test
+ public void deadNode() throws Exception
+ {
+ InstrumentedSomeVerbHandler handler = new InstrumentedSomeVerbHandler();
+ handler.alive = false;
+ MessageIn<SomeMessage> msg = new MessageIn<>(...);
+
+ assertFalse(handler.streamCalled);
+ assertFalse(handler.compactCalled);
+
+ handler.doVerb(msg);
+
+ assertFalse(handler.streamCalled);
+ assertTrue(handler.compactCalled);
+ }
+}
+```
+
+## Refactoring Existing Code
+
+If you're working on a section of the project that historically hasn't been well tested, it will likely be more difficult for you to write tests around
+whatever you're doing, since the code may not have been written with testing in mind. You do need to add tests around the subject of you're
+jira, and this will probably involve some refactoring, but you're also not expected to completely refactor a huge class to submit a bugfix.
+
+Basically, you need to be able to verify the behavior you intend to modify before and after your patch. The amount of testing debt you pay back should be
+roughly proportional to the scope of your change. If you're doing a small bugfix, you can get away with refactoring just enough to make the subject of your
+fix testable, even if you start to get into 'testing implementation details' territory'. The goal is incremental improvement, not making things perfect on
+the first iteration. If you're doing something more ambitious though, you may have to do some extra work to sufficiently test your changes.
+
+## Refactoring Untested Code
+
+There are several components that have very little, if any, direct test coverage. We really should try to improve the test coverage of these components.
+For people interested in getting involved with the project, adding tests for these is a great way to get familiar with the codebase.
+
+First, get feedback on the subject and scope of your proposed refactor, especially larger ones. The smaller and more narrowly focused your proposed
+refactor is, the easier it will be for you to get it reviewed and committed.
+
+Start with smaller pieces, refactor and test them, and work outwards, iteratively. Preferably in several jiras. Ideally, each patch should add some value
+to the project on it's own in terms of test coverage. Patches that are heavy on refactoring, and light on tests are not likely to get committed. People come and go
+from projects, and having a many small improvements is better for the project than several unfinished or ongoing refactors that don't add much test coverage.
diff --git a/bin/cassandra b/bin/cassandra
index 1df927f..031196d 100755
--- a/bin/cassandra
+++ b/bin/cassandra
@@ -67,7 +67,16 @@
# NB: Developers should be aware that this script should remain compatible with
# POSIX sh and Solaris sh. This means, in particular, no $(( )) and no $( ).
+# Unset any grep options that may include `--color=always` per say.
+# Using `unset GREP_OPTIONS` will also work on the non-deprecated use case
+# of setting a new grep alias.
+# See CASSANDRA-14487 for more details.
+unset GREP_OPTIONS
+
# If an include wasn't specified in the environment, then search for one...
+
+jvmoptions_variant="-server"
+
if [ "x$CASSANDRA_INCLUDE" = "x" ]; then
# Locations (in order) to use when searching for an include file.
for include in "`dirname "$0"`/cassandra.in.sh" \
@@ -85,27 +94,6 @@
. "$CASSANDRA_INCLUDE"
fi
-# Use JAVA_HOME if set, otherwise look for java in PATH
-if [ -n "$JAVA_HOME" ]; then
- # Why we can't have nice things: Solaris combines x86 and x86_64
- # installations in the same tree, using an unconventional path for the
- # 64bit JVM. Since we prefer 64bit, search the alternate path first,
- # (see https://issues.apache.org/jira/browse/CASSANDRA-4638).
- for java in "$JAVA_HOME"/bin/amd64/java "$JAVA_HOME"/bin/java; do
- if [ -x "$java" ]; then
- JAVA="$java"
- break
- fi
- done
-else
- JAVA=java
-fi
-
-if [ -z $JAVA ] ; then
- echo Unable to find java executable. Check JAVA_HOME and PATH environment variables. >&2
- exit 1;
-fi
-
# If numactl is available, use it. For Cassandra, the priority is to
# avoid disk I/O. Even for the purpose of CPU efficiency, we don't
# really have CPU<->data affinity anyway. Also, empirically test that numactl
diff --git a/bin/cassandra.bat b/bin/cassandra.bat
index 0d207cd..24889d8 100644
--- a/bin/cassandra.bat
+++ b/bin/cassandra.bat
@@ -54,7 +54,7 @@
REM -----------------------------------------------------------------------------
REM JVM Opts we'll use in legacy run or installation
set JAVA_OPTS=-ea^
- -javaagent:"%CASSANDRA_HOME%\lib\jamm-0.3.0.jar"^
+ -javaagent:"%CASSANDRA_HOME%\lib\jamm-0.3.2"^
-Xms2G^
-Xmx2G^
-XX:+HeapDumpOnOutOfMemoryError^
@@ -112,7 +112,7 @@
)
REM Include the build\classes\main directory so it works in development
-set CASSANDRA_CLASSPATH=%CLASSPATH%;"%CASSANDRA_HOME%\build\classes\main";"%CASSANDRA_HOME%\build\classes\thrift"
+set CASSANDRA_CLASSPATH=%CLASSPATH%;"%CASSANDRA_HOME%\build\classes\main"
set CASSANDRA_PARAMS=-Dcassandra -Dcassandra-foreground=yes
set CASSANDRA_PARAMS=%CASSANDRA_PARAMS% -Dcassandra.logdir="%CASSANDRA_HOME%\logs"
set CASSANDRA_PARAMS=%CASSANDRA_PARAMS% -Dcassandra.storagedir="%CASSANDRA_HOME%\data"
diff --git a/bin/cassandra.in.bat b/bin/cassandra.in.bat
index 5682f9d..0e760a0 100644
--- a/bin/cassandra.in.bat
+++ b/bin/cassandra.in.bat
@@ -44,7 +44,7 @@
:okClasspath
REM Include the build\classes\main directory so it works in development
-set CASSANDRA_CLASSPATH=%CLASSPATH%;"%CASSANDRA_HOME%\build\classes\main";%CASSANDRA_CONF%;"%CASSANDRA_HOME%\build\classes\thrift"
+set CASSANDRA_CLASSPATH=%CLASSPATH%;"%CASSANDRA_HOME%\build\classes\main";%CASSANDRA_CONF%
REM Add the default storage location. Can be overridden in conf\cassandra.yaml
set CASSANDRA_PARAMS=%CASSANDRA_PARAMS% "-Dcassandra.storagedir=%CASSANDRA_HOME%\data"
diff --git a/bin/cassandra.in.sh b/bin/cassandra.in.sh
index 13b1291..58b4dd2 100644
--- a/bin/cassandra.in.sh
+++ b/bin/cassandra.in.sh
@@ -23,12 +23,18 @@
CASSANDRA_CONF="$CASSANDRA_HOME/conf"
fi
+# The java classpath (required)
+CLASSPATH="$CASSANDRA_CONF"
+
# This can be the path to a jar file, or a directory containing the
# compiled classes. NOTE: This isn't needed by the startup script,
# it's just used here in constructing the classpath.
-cassandra_bin="$CASSANDRA_HOME/build/classes/main"
-cassandra_bin="$cassandra_bin:$CASSANDRA_HOME/build/classes/thrift"
-#cassandra_bin="$CASSANDRA_HOME/build/cassandra.jar"
+if [ -d $CASSANDRA_HOME/build ] ; then
+ #cassandra_bin="$CASSANDRA_HOME/build/classes/main"
+ cassandra_bin=`ls -1 $CASSANDRA_HOME/build/apache-cassandra*.jar`
+
+ CLASSPATH="$CLASSPATH:$cassandra_bin"
+fi
# the default location for commitlogs, sstables, and saved caches
# if not set in cassandra.yaml
@@ -37,9 +43,6 @@
# JAVA_HOME can optionally be set here
#JAVA_HOME=/usr/local/jdk6
-# The java classpath (required)
-CLASSPATH="$CASSANDRA_CONF:$cassandra_bin"
-
for jar in "$CASSANDRA_HOME"/lib/*.jar; do
CLASSPATH="$CLASSPATH:$jar"
done
@@ -69,11 +72,87 @@
fi
# set JVM javaagent opts to avoid warnings/errors
-if [ "$JVM_VENDOR" != "OpenJDK" -o "$JVM_VERSION" \> "1.6.0" ] \
- || [ "$JVM_VERSION" = "1.6.0" -a "$JVM_PATCH_VERSION" -ge 23 ]
-then
- JAVA_AGENT="$JAVA_AGENT -javaagent:$CASSANDRA_HOME/lib/jamm-0.3.0.jar"
-fi
+JAVA_AGENT="$JAVA_AGENT -javaagent:$CASSANDRA_HOME/lib/jamm-0.3.2.jar"
# Added sigar-bin to the java.library.path CASSANDRA-7838
JAVA_OPTS="$JAVA_OPTS:-Djava.library.path=$CASSANDRA_HOME/lib/sigar-bin"
+
+
+#
+# Java executable and per-Java version JVM settings
+#
+
+# Use JAVA_HOME if set, otherwise look for java in PATH
+if [ -n "$JAVA_HOME" ]; then
+ # Why we can't have nice things: Solaris combines x86 and x86_64
+ # installations in the same tree, using an unconventional path for the
+ # 64bit JVM. Since we prefer 64bit, search the alternate path first,
+ # (see https://issues.apache.org/jira/browse/CASSANDRA-4638).
+ for java in "$JAVA_HOME"/bin/amd64/java "$JAVA_HOME"/bin/java; do
+ if [ -x "$java" ]; then
+ JAVA="$java"
+ break
+ fi
+ done
+else
+ JAVA=java
+fi
+
+if [ -z $JAVA ] ; then
+ echo Unable to find java executable. Check JAVA_HOME and PATH environment variables. >&2
+ exit 1;
+fi
+
+# Determine the sort of JVM we'll be running on.
+java_ver_output=`"${JAVA:-java}" -version 2>&1`
+jvmver=`echo "$java_ver_output" | grep '[openjdk|java] version' | awk -F'"' 'NR==1 {print $2}' | cut -d\- -f1`
+JVM_VERSION=${jvmver%_*}
+
+JAVA_VERSION=11
+if [ "$JVM_VERSION" = "1.8.0" ] ; then
+ JVM_PATCH_VERSION=${jvmver#*_}
+ if [ "$JVM_VERSION" \< "1.8" ] || [ "$JVM_VERSION" \> "1.8.2" ] ; then
+ echo "Cassandra 4.0 requires either Java 8 (update 151 or newer) or Java 11 (or newer). Java $JVM_VERSION is not supported."
+ exit 1;
+ fi
+ if [ "$JVM_PATCH_VERSION" -lt 151 ] ; then
+ echo "Cassandra 4.0 requires either Java 8 (update 151 or newer) or Java 11 (or newer). Java 8 update $JVM_PATCH_VERSION is not supported."
+ exit 1;
+ fi
+ JAVA_VERSION=8
+elif [ "$JVM_VERSION" \< "11" ] ; then
+ echo "Cassandra 4.0 requires either Java 8 (update 151 or newer) or Java 11 (or newer)."
+ exit 1;
+fi
+
+jvm=`echo "$java_ver_output" | grep -A 1 '[openjdk|java] version' | awk 'NR==2 {print $1}'`
+case "$jvm" in
+ OpenJDK)
+ JVM_VENDOR=OpenJDK
+ # this will be "64-Bit" or "32-Bit"
+ JVM_ARCH=`echo "$java_ver_output" | awk 'NR==3 {print $2}'`
+ ;;
+ "Java(TM)")
+ JVM_VENDOR=Oracle
+ # this will be "64-Bit" or "32-Bit"
+ JVM_ARCH=`echo "$java_ver_output" | awk 'NR==3 {print $3}'`
+ ;;
+ *)
+ # Help fill in other JVM values
+ JVM_VENDOR=other
+ JVM_ARCH=unknown
+ ;;
+esac
+
+# Read user-defined JVM options from jvm-server.options file
+JVM_OPTS_FILE=$CASSANDRA_CONF/jvm${jvmoptions_variant:--clients}.options
+if [ $JAVA_VERSION -ge 11 ] ; then
+ JVM_DEP_OPTS_FILE=$CASSANDRA_CONF/jvm11${jvmoptions_variant:--clients}.options
+else
+ JVM_DEP_OPTS_FILE=$CASSANDRA_CONF/jvm8${jvmoptions_variant:--clients}.options
+fi
+
+for opt in `grep "^-" $JVM_OPTS_FILE` `grep "^-" $JVM_DEP_OPTS_FILE`
+do
+ JVM_OPTS="$JVM_OPTS $opt"
+done
diff --git a/bin/cassandra.ps1 b/bin/cassandra.ps1
index ee3b566..6e6a3cf 100644
--- a/bin/cassandra.ps1
+++ b/bin/cassandra.ps1
@@ -298,9 +298,8 @@
# storage_port
# ssl_storage_port
# native_transport_port
- # rpc_port, which we'll match to rpc_address
# and from env: JMX_PORT which we cache in our environment during SetCassandraEnvironment for this check
- $yamlRegex = "storage_port:|ssl_storage_port:|native_transport_port:|rpc_port"
+ $yamlRegex = "storage_port:|ssl_storage_port:|native_transport_port:"
$yaml = Get-Content "$env:CASSANDRA_CONF\cassandra.yaml"
$portRegex = ":$env:JMX_PORT |"
diff --git a/bin/cqlsh b/bin/cqlsh
index 82a4a53..0774d52 100755
--- a/bin/cqlsh
+++ b/bin/cqlsh
@@ -15,12 +15,84 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-# bash code here; finds a suitable python interpreter and execs this file.
-# prefer unqualified "python" if suitable:
-python -c 'import sys; sys.exit(not (0x020700b0 < sys.hexversion < 0x03000000))' 2>/dev/null \
- && exec python "`python -c "import os;print(os.path.dirname(os.path.realpath('$0')))"`/cqlsh.py" "$@"
-for pyver in 2.7; do
- which python$pyver > /dev/null 2>&1 && exec python$pyver "`python$pyver -c "import os;print(os.path.dirname(os.path.realpath('$0')))"`/cqlsh.py" "$@"
+# shell script to find a suitable Python interpreter and run cqlsh.py
+
+# Use the Python that is specified in the env
+if [ -n "$CQLSH_PYTHON" ]; then
+ USER_SPECIFIED_PYTHON="$CQLSH_PYTHON"
+fi
+
+
+# filter "--python" option and its value, and keep remaining arguments as it is
+USER_SPECIFIED_PYTHON_OPTION=false
+for arg do
+ shift
+ case "$arg" in
+ --python)
+ USER_SPECIFIED_PYTHON_OPTION=true
+ ;;
+ --)
+ break
+ ;;
+ *)
+ if [ "$USER_SPECIFIED_PYTHON_OPTION" = true ] ; then
+ USER_SPECIFIED_PYTHON_OPTION=false
+ USER_SPECIFIED_PYTHON="$arg"
+ else
+ set -- "$@" "$arg"
+ fi
+ ;;
+ esac
done
-echo "No appropriate python interpreter found." >&2
+
+if [ "$USER_SPECIFIED_PYTHON_OPTION" = true ] ; then
+ echo "You must specify a python interpreter path with the --python option"
+ exit 1
+fi
+
+# get a version string for a Python interpreter
+get_python_version() {
+ interpreter=$1
+ version=$(command -v "$interpreter" > /dev/null 2>&1 && $interpreter -c "import os; print('{}.{}'.format(os.sys.version_info.major, os.sys.version_info.minor))")
+ echo "$version"
+}
+
+# test whether a version string matches one of the supported versions for cqlsh
+is_supported_version() {
+ version=$1
+ # shellcheck disable=SC2039
+ # shellcheck disable=SC2072
+ # python2.7 or python3.6+ is supported
+ if [ "$version" = "3.6" ] || [ "$version" \> "3.6" ] || [ "$version" = "2.7" ]; then
+ echo "supported"
+ else
+ echo "unsupported"
+ fi
+}
+
+run_if_supported_version() {
+ # get the interpreter and remove it from argument
+ interpreter="$1" shift
+
+ version=$(get_python_version "$interpreter")
+ if [ -n "$version" ]; then
+ if [ "$(is_supported_version "$version")" = "supported" ]; then
+ exec "$interpreter" "$($interpreter -c "import os; print(os.path.dirname(os.path.realpath('$0')))")/cqlsh.py" "$@"
+ exit
+ fi
+ fi
+}
+
+
+if [ "$USER_SPECIFIED_PYTHON" != "" ]; then
+ # run a user specified Python interpreter
+ run_if_supported_version "$USER_SPECIFIED_PYTHON" "$@"
+else
+ # try unqualified python first, then python3, then python2.7
+ for interpreter in python python3 python2.7; do
+ run_if_supported_version "$interpreter" "$@"
+ done
+fi
+
+echo "No appropriate Python interpreter found." >&2
exit 1
diff --git a/bin/cqlsh.py b/bin/cqlsh.py
index 44d4d50..db13bb8 100644
--- a/bin/cqlsh.py
+++ b/bin/cqlsh.py
@@ -19,21 +19,21 @@
""":"
# bash code here; finds a suitable python interpreter and execs this file.
+# this implementation of cqlsh is compatible with both Python 3 and Python 2.7.
# prefer unqualified "python" if suitable:
-python -c 'import sys; sys.exit(not (0x020700b0 < sys.hexversion < 0x03000000))' 2>/dev/null \
+python -c 'import sys; sys.exit(not (0x020700b0 < sys.hexversion))' 2>/dev/null \
&& exec python "$0" "$@"
-for pyver in 2.7; do
+for pyver in 3 2.7; do
which python$pyver > /dev/null 2>&1 && exec python$pyver "$0" "$@"
done
echo "No appropriate python interpreter found." >&2
exit 1
":"""
-from __future__ import with_statement
+from __future__ import division, unicode_literals
import cmd
import codecs
-import ConfigParser
import csv
import getpass
import optparse
@@ -43,13 +43,12 @@
import traceback
import warnings
import webbrowser
-from StringIO import StringIO
from contextlib import contextmanager
from glob import glob
from uuid import UUID
-if sys.version_info[0] != 2 or sys.version_info[1] != 7:
- sys.exit("\nCQL Shell supports only Python 2.7\n")
+if sys.version_info.major != 3 and (sys.version_info.major == 2 and sys.version_info.minor != 7):
+ sys.exit("\nCQL Shell supports only Python 3 or Python 2.7\n")
# see CASSANDRA-10428
if platform.python_implementation().startswith('Jython'):
@@ -97,14 +96,15 @@
# >>> webbrowser._tryorder
# >>> webbrowser._browser
#
-if len(webbrowser._tryorder) == 0:
+# webbrowser._tryorder is None in python3.7+
+if webbrowser._tryorder is None or len(webbrowser._tryorder) == 0:
CASSANDRA_CQL_HTML = CASSANDRA_CQL_HTML_FALLBACK
elif webbrowser._tryorder[0] == 'xdg-open' and os.environ.get('XDG_DATA_DIRS', '') == '':
# only on Linux (some OS with xdg-open)
webbrowser._tryorder.remove('xdg-open')
webbrowser._tryorder.append('xdg-open')
-# use bundled libs for python-cql and thrift, if available. if there
+# use bundled lib for python-cql if available. if there
# is a ../lib dir, use bundled libs there preferentially.
ZIPLIB_DIRS = [os.path.join(CASSANDRA_PATH, 'lib')]
myplatform = platform.system()
@@ -133,17 +133,24 @@
ver = os.path.splitext(os.path.basename(cql_zip))[0][len(CQL_LIB_PREFIX):]
sys.path.insert(0, os.path.join(cql_zip, 'cassandra-driver-' + ver))
-third_parties = ('futures-', 'six-')
+third_parties = ('futures-', 'six-', 'geomet-')
for lib in third_parties:
lib_zip = find_zip(lib)
if lib_zip:
sys.path.insert(0, lib_zip)
+# We cannot import six until we add its location to sys.path so the Python
+# interpreter can find it. Do not move this to the top.
+import six
+
+from six.moves import configparser, input
+from six import StringIO, ensure_text, ensure_str
+
warnings.filterwarnings("ignore", r".*blist.*")
try:
import cassandra
-except ImportError, e:
+except ImportError as e:
sys.exit("\nPython Cassandra driver not installed, or not on PYTHONPATH.\n"
'You might try "pip install cassandra-driver".\n\n'
'Python: %s\n'
@@ -210,15 +217,16 @@
- one of the supported browsers in https://docs.python.org/2/library/webbrowser.html.
- browser path followed by %s, example: /usr/bin/google-chrome-stable %s""")
parser.add_option('--ssl', action='store_true', help='Use SSL', default=False)
-parser.add_option('--no_compact', action='store_true', help='No Compact', default=False)
parser.add_option("-u", "--username", help="Authenticate as user.")
parser.add_option("-p", "--password", help="Authenticate using password.")
parser.add_option('-k', '--keyspace', help='Authenticate to the given keyspace.')
parser.add_option("-f", "--file", help="Execute commands from FILE, then exit")
parser.add_option('--debug', action='store_true',
help='Show additional debugging information')
-parser.add_option("--encoding", help="Specify a non-default encoding for output." +
- " (Default: %s)" % (UTF8,))
+parser.add_option('--coverage', action='store_true',
+ help='Collect coverage data')
+parser.add_option("--encoding", help="Specify a non-default encoding for output."
+ + " (Default: %s)" % (UTF8,))
parser.add_option("--cqlshrc", help="Specify an alternative cqlshrc file location.")
parser.add_option('--cqlversion', default=None,
help='Specify a particular CQL version, '
@@ -244,7 +252,7 @@
if hasattr(options, 'cqlshrc'):
CONFIG_FILE = options.cqlshrc
if not os.path.exists(CONFIG_FILE):
- print '\nWarning: Specified cqlshrc location `%s` does not exist. Using `%s` instead.\n' % (CONFIG_FILE, HISTORY_DIR)
+ print('\nWarning: Specified cqlshrc location `%s` does not exist. Using `%s` instead.\n' % (CONFIG_FILE, HISTORY_DIR))
CONFIG_FILE = os.path.join(HISTORY_DIR, 'cqlshrc')
else:
CONFIG_FILE = os.path.join(HISTORY_DIR, 'cqlshrc')
@@ -254,16 +262,16 @@
try:
os.mkdir(HISTORY_DIR)
except OSError:
- print '\nWarning: Cannot create directory at `%s`. Command history will not be saved.\n' % HISTORY_DIR
+ print('\nWarning: Cannot create directory at `%s`. Command history will not be saved.\n' % HISTORY_DIR)
OLD_CONFIG_FILE = os.path.expanduser(os.path.join('~', '.cqlshrc'))
if os.path.exists(OLD_CONFIG_FILE):
if os.path.exists(CONFIG_FILE):
- print '\nWarning: cqlshrc config files were found at both the old location (%s) and \
- the new location (%s), the old config file will not be migrated to the new \
- location, and the new location will be used for now. You should manually \
- consolidate the config files at the new location and remove the old file.' \
- % (OLD_CONFIG_FILE, CONFIG_FILE)
+ print('\nWarning: cqlshrc config files were found at both the old location ({0})'
+ + ' and the new location ({1}), the old config file will not be migrated to the new'
+ + ' location, and the new location will be used for now. You should manually'
+ + ' consolidate the config files at the new location and remove the old file.'
+ .format(OLD_CONFIG_FILE, CONFIG_FILE))
else:
os.rename(OLD_CONFIG_FILE, CONFIG_FILE)
OLD_HISTORY = os.path.expanduser(os.path.join('~', '.cqlsh_history'))
@@ -352,7 +360,7 @@
while ver.count('.') < 2:
ver += '.0'
ver_parts = ver.split('-', 1) + ['']
- vertuple = tuple(map(int, ver_parts[0].split('.')) + [ver_parts[1]])
+ vertuple = tuple(list(map(int, ver_parts[0].split('.'))) + [ver_parts[1]])
return ver, vertuple
@@ -405,34 +413,18 @@
cassandra.cqltypes.CassandraType.support_empty_values = True
-class FrozenType(cassandra.cqltypes._ParameterizedType):
- """
- Needed until the bundled python driver adds FrozenType.
- """
- typename = "frozen"
- num_subtypes = 1
-
- @classmethod
- def deserialize_safe(cls, byts, protocol_version):
- subtype, = cls.subtypes
- return subtype.from_binary(byts)
-
- @classmethod
- def serialize_safe(cls, val, protocol_version):
- subtype, = cls.subtypes
- return subtype.to_binary(val, protocol_version)
-
-
class Shell(cmd.Cmd):
custom_prompt = os.getenv('CQLSH_PROMPT', '')
- if custom_prompt is not '':
+ if custom_prompt != '':
custom_prompt += "\n"
default_prompt = custom_prompt + "cqlsh> "
continue_prompt = " ... "
- keyspace_prompt = custom_prompt + "cqlsh:%s> "
- keyspace_continue_prompt = "%s ... "
+ keyspace_prompt = custom_prompt + "cqlsh:{}> "
+ keyspace_continue_prompt = "{} ... "
show_line_nums = False
debug = False
+ coverage = False
+ coveragerc_path = None
stop = False
last_hist = None
shunted_query_out = None
@@ -445,7 +437,6 @@
completekey=DEFAULT_COMPLETEKEY, browser=None, use_conn=None,
cqlver=None, keyspace=None,
tracing_enabled=False, expand_enabled=False,
- no_compact=False,
display_nanotime_format=DEFAULT_NANOTIME_FORMAT,
display_timestamp_format=DEFAULT_TIMESTAMP_FORMAT,
display_date_format=DEFAULT_DATE_FORMAT,
@@ -457,7 +448,8 @@
single_statement=None,
request_timeout=DEFAULT_REQUEST_TIMEOUT_SECONDS,
protocol_version=None,
- connect_timeout=DEFAULT_CONNECT_TIMEOUT_SECONDS):
+ connect_timeout=DEFAULT_CONNECT_TIMEOUT_SECONDS,
+ is_subshell=False):
cmd.Cmd.__init__(self, completekey=completekey)
self.hostname = hostname
self.port = port
@@ -480,7 +472,6 @@
kwargs['protocol_version'] = protocol_version
self.conn = Cluster(contact_points=(self.hostname,), port=self.port, cql_version=cqlver,
auth_provider=self.auth_provider,
- no_compact=no_compact,
ssl_options=sslhandling.ssl_settings(hostname, CONFIG_FILE) if ssl else None,
load_balancing_policy=WhiteListRoundRobinPolicy([self.hostname]),
control_connection_timeout=connect_timeout,
@@ -535,7 +526,7 @@
if tty:
self.reset_prompt()
self.report_connection()
- print 'Use HELP for help.'
+ print('Use HELP for help.')
else:
self.show_line_nums = True
self.stdin = stdin
@@ -546,6 +537,7 @@
self.empty_lines = 0
self.statement_error = False
self.single_statement = single_statement
+ self.is_subshell = is_subshell
@property
def batch_mode(self):
@@ -584,7 +576,7 @@
return format_value(val, cqltype=cqltype, encoding=self.output_codec.name,
addcolor=self.color, date_time_format=dtformats,
float_precision=precision, **kwargs)
- except Exception, e:
+ except Exception as e:
err = FormatError(val, e)
self.decoding_errors.append(err)
return format_value(err, cqltype=cqltype, encoding=self.output_codec.name, addcolor=self.color)
@@ -606,10 +598,10 @@
self.show_version()
def show_host(self):
- print "Connected to %s at %s:%d." % \
- (self.applycolor(self.get_cluster_name(), BLUE),
- self.hostname,
- self.port)
+ print("Connected to {0} at {1}:{2}."
+ .format(self.applycolor(self.get_cluster_name(), BLUE),
+ self.hostname,
+ self.port))
def show_version(self):
vers = self.connection_versions.copy()
@@ -617,7 +609,7 @@
# system.Versions['cql'] apparently does not reflect changes with
# set_cql_version.
vers['cql'] = self.cql_version
- print "[cqlsh %(shver)s | Cassandra %(build)s | CQL spec %(cql)s | Native protocol v%(protocol)s]" % vers
+ print("[cqlsh %(shver)s | Cassandra %(build)s | CQL spec %(cql)s | Native protocol v%(protocol)s]" % vers)
def show_session(self, sessionid, partial_session=False):
print_trace_session(self, self.session, sessionid, partial_session)
@@ -626,43 +618,43 @@
result, = self.session.execute("select * from system.local where key = 'local'")
vers = {
'build': result['release_version'],
+ 'protocol': self.conn.protocol_version,
'cql': result['cql_version'],
}
- vers['protocol'] = self.conn.protocol_version
self.connection_versions = vers
def get_keyspace_names(self):
- return map(str, self.conn.metadata.keyspaces.keys())
+ return list(map(str, list(self.conn.metadata.keyspaces.keys())))
def get_columnfamily_names(self, ksname=None):
if ksname is None:
ksname = self.current_keyspace
- return map(str, self.get_keyspace_meta(ksname).tables.keys())
+ return list(map(str, list(self.get_keyspace_meta(ksname).tables.keys())))
def get_materialized_view_names(self, ksname=None):
if ksname is None:
ksname = self.current_keyspace
- return map(str, self.get_keyspace_meta(ksname).views.keys())
+ return list(map(str, list(self.get_keyspace_meta(ksname).views.keys())))
def get_index_names(self, ksname=None):
if ksname is None:
ksname = self.current_keyspace
- return map(str, self.get_keyspace_meta(ksname).indexes.keys())
+ return list(map(str, list(self.get_keyspace_meta(ksname).indexes.keys())))
def get_column_names(self, ksname, cfname):
if ksname is None:
ksname = self.current_keyspace
layout = self.get_table_meta(ksname, cfname)
- return [unicode(col) for col in layout.columns]
+ return [str(col) for col in layout.columns]
def get_usertype_names(self, ksname=None):
if ksname is None:
ksname = self.current_keyspace
- return self.get_keyspace_meta(ksname).user_types.keys()
+ return list(self.get_keyspace_meta(ksname).user_types.keys())
def get_usertype_layout(self, ksname, typename):
if ksname is None:
@@ -673,21 +665,21 @@
try:
user_type = ks_meta.user_types[typename]
except KeyError:
- raise UserTypeNotFound("User type %r not found" % typename)
+ raise UserTypeNotFound("User type {!r} not found".format(typename))
- return zip(user_type.field_names, user_type.field_types)
+ return list(zip(user_type.field_names, user_type.field_types))
def get_userfunction_names(self, ksname=None):
if ksname is None:
ksname = self.current_keyspace
- return map(lambda f: f.name, self.get_keyspace_meta(ksname).functions.values())
+ return [f.name for f in list(self.get_keyspace_meta(ksname).functions.values())]
def get_useraggregate_names(self, ksname=None):
if ksname is None:
ksname = self.current_keyspace
- return map(lambda f: f.name, self.get_keyspace_meta(ksname).aggregates.values())
+ return [f.name for f in list(self.get_keyspace_meta(ksname).aggregates.values())]
def get_cluster_name(self):
return self.conn.metadata.cluster_name
@@ -696,12 +688,13 @@
return self.conn.metadata.partitioner
def get_keyspace_meta(self, ksname):
- if ksname not in self.conn.metadata.keyspaces:
- raise KeyspaceNotFound('Keyspace %r not found.' % ksname)
- return self.conn.metadata.keyspaces[ksname]
+ if ksname in self.conn.metadata.keyspaces:
+ return self.conn.metadata.keyspaces[ksname]
+
+ raise KeyspaceNotFound('Keyspace %r not found.' % ksname)
def get_keyspaces(self):
- return self.conn.metadata.keyspaces.values()
+ return list(self.conn.metadata.keyspaces.values())
def get_ring(self, ks):
self.conn.metadata.token_map.rebuild_keyspace(ks, build_if_absent=True)
@@ -711,12 +704,11 @@
if ksname is None:
ksname = self.current_keyspace
ksmeta = self.get_keyspace_meta(ksname)
-
if tablename not in ksmeta.tables:
if ksname == 'system_auth' and tablename in ['roles', 'role_permissions']:
self.get_fake_auth_table_meta(ksname, tablename)
else:
- raise ColumnFamilyNotFound("Column family %r not found" % tablename)
+ raise ColumnFamilyNotFound("Column family {} not found".format(tablename))
else:
return ksmeta.tables[tablename]
@@ -737,7 +729,7 @@
table_meta.columns['resource'] = ColumnMetadata(table_meta, 'resource', cassandra.cqltypes.UTF8Type)
table_meta.columns['permission'] = ColumnMetadata(table_meta, 'permission', cassandra.cqltypes.UTF8Type)
else:
- raise ColumnFamilyNotFound("Column family %r not found" % tablename)
+ raise ColumnFamilyNotFound("Column family {} not found".format(tablename))
def get_index_meta(self, ksname, idxname):
if ksname is None:
@@ -745,7 +737,7 @@
ksmeta = self.get_keyspace_meta(ksname)
if idxname not in ksmeta.indexes:
- raise IndexNotFound("Index %r not found" % idxname)
+ raise IndexNotFound("Index {} not found".format(idxname))
return ksmeta.indexes[idxname]
@@ -755,7 +747,7 @@
ksmeta = self.get_keyspace_meta(ksname)
if viewname not in ksmeta.views:
- raise MaterializedViewNotFound("Materialized view %r not found" % viewname)
+ raise MaterializedViewNotFound("Materialized view '{}' not found".format(viewname))
return ksmeta.views[viewname]
def get_object_meta(self, ks, name):
@@ -763,7 +755,7 @@
if ks and ks in self.conn.metadata.keyspaces:
return self.conn.metadata.keyspaces[ks]
elif self.current_keyspace is None:
- raise ObjectNotFound("%r not found in keyspaces" % (ks))
+ raise ObjectNotFound("'{}' not found in keyspaces".format(ks))
else:
name = ks
ks = self.current_keyspace
@@ -780,7 +772,7 @@
elif name in ksmeta.views:
return ksmeta.views[name]
- raise ObjectNotFound("%r not found in keyspace %r" % (name, ks))
+ raise ObjectNotFound("'{}' not found in keyspace '{}'".format(name, ks))
def get_usertypes_meta(self):
data = self.session.execute("select * from system.schema_usertypes")
@@ -794,19 +786,20 @@
ksname = self.current_keyspace
return [trigger.name
- for table in self.get_keyspace_meta(ksname).tables.values()
- for trigger in table.triggers.values()]
+ for table in list(self.get_keyspace_meta(ksname).tables.values())
+ for trigger in list(table.triggers.values())]
def reset_statement(self):
self.reset_prompt()
self.statement.truncate(0)
+ self.statement.seek(0)
self.empty_lines = 0
def reset_prompt(self):
if self.current_keyspace is None:
self.set_prompt(self.default_prompt, True)
else:
- self.set_prompt(self.keyspace_prompt % self.current_keyspace, True)
+ self.set_prompt(self.keyspace_prompt.format(self.current_keyspace), True)
def set_continue_prompt(self):
if self.empty_lines >= 3:
@@ -817,7 +810,7 @@
self.set_prompt(self.continue_prompt)
else:
spaces = ' ' * len(str(self.current_keyspace))
- self.set_prompt(self.keyspace_continue_prompt % spaces)
+ self.set_prompt(self.keyspace_continue_prompt.format(spaces))
self.empty_lines = self.empty_lines + 1 if not self.lastcmd else 0
@contextmanager
@@ -828,7 +821,7 @@
import readline
except ImportError:
if is_win:
- print "WARNING: pyreadline dependency missing. Install to enable tab completion."
+ print("WARNING: pyreadline dependency missing. Install to enable tab completion.")
pass
else:
old_completer = readline.get_completer()
@@ -839,27 +832,42 @@
readline.parse_and_bind("bind ^R em-inc-search-prev")
else:
readline.parse_and_bind(self.completekey + ": complete")
+ # start coverage collection if requested, unless in subshell
+ if self.coverage and not self.is_subshell:
+ # check for coveragerc file, write it if missing
+ if os.path.exists(HISTORY_DIR):
+ self.coveragerc_path = os.path.join(HISTORY_DIR, '.coveragerc')
+ covdata_path = os.path.join(HISTORY_DIR, '.coverage')
+ if not os.path.isfile(self.coveragerc_path):
+ with open(self.coveragerc_path, 'w') as f:
+ f.writelines(["[run]\n",
+ "concurrency = multiprocessing\n",
+ "data_file = {}\n".format(covdata_path),
+ "parallel = true\n"]
+ )
+ # start coverage
+ import coverage
+ self.cov = coverage.Coverage(config_file=self.coveragerc_path)
+ self.cov.start()
try:
yield
finally:
if readline is not None:
readline.set_completer(old_completer)
+ if self.coverage and not self.is_subshell:
+ self.stop_coverage()
def get_input_line(self, prompt=''):
if self.tty:
- try:
- self.lastcmd = raw_input(prompt).decode(self.encoding)
- except UnicodeDecodeError:
- self.lastcmd = ''
- traceback.print_exc()
- self.check_windows_encoding()
- line = self.lastcmd + '\n'
+ self.lastcmd = input(prompt)
+ line = ensure_text(self.lastcmd) + '\n'
else:
- self.lastcmd = self.stdin.readline()
+ self.lastcmd = ensure_text(self.stdin.readline())
line = self.lastcmd
if not len(line):
raise EOFError
self.lineno += 1
+ line = ensure_text(line)
return line
def use_stdin_reader(self, until='', prompt=''):
@@ -892,29 +900,29 @@
self.reset_statement()
except EOFError:
self.handle_eof()
- except CQL_ERRORS, cqlerr:
- self.printerr(cqlerr.message.decode(encoding='utf-8'))
+ except CQL_ERRORS as cqlerr:
+ self.printerr(cqlerr.message)
except KeyboardInterrupt:
self.reset_statement()
- print
+ print('')
def onecmd(self, statementtext):
"""
Returns true if the statement is complete and was handled (meaning it
can be reset).
"""
-
+ statementtext = ensure_text(statementtext)
try:
statements, endtoken_escaped = cqlruleset.cql_split_statements(statementtext)
- except pylexotron.LexingError, e:
+ except pylexotron.LexingError as e:
if self.show_line_nums:
- self.printerr('Invalid syntax at char %d' % (e.charnum,))
+ self.printerr('Invalid syntax at line {0}, char {1}'
+ .format(e.linenum, e.charnum))
else:
- self.printerr('Invalid syntax at line %d, char %d'
- % (e.linenum, e.charnum))
+ self.printerr('Invalid syntax at char {0}'.format(e.charnum))
statementline = statementtext.split('\n')[e.linenum - 1]
- self.printerr(' %s' % statementline)
- self.printerr(' %s^' % (' ' * e.charnum))
+ self.printerr(' {0}'.format(statementline))
+ self.printerr(' {0}^'.format(' ' * e.charnum))
return True
while statements and not statements[-1]:
@@ -927,7 +935,7 @@
for st in statements:
try:
self.handle_statement(st, statementtext)
- except Exception, e:
+ except Exception as e:
if self.debug:
traceback.print_exc()
else:
@@ -936,7 +944,7 @@
def handle_eof(self):
if self.tty:
- print
+ print('')
statement = self.statement.getvalue()
if statement.strip():
if not self.onecmd(statement):
@@ -951,7 +959,7 @@
new_hist = srcstr.replace("\n", " ").rstrip()
if nl_count > 1 and self.last_hist != new_hist:
- readline.add_history(new_hist.encode(self.encoding))
+ readline.add_history(new_hist)
self.last_hist = new_hist
cmdword = tokens[0][1]
@@ -999,6 +1007,8 @@
self.tracing_enabled = tracing_was_enabled
def perform_statement(self, statement):
+ statement = ensure_text(statement)
+
stmt = SimpleStatement(statement, consistency_level=self.consistency_level, serial_consistency_level=self.serial_consistency_level, fetch_size=self.page_size if self.use_paging else None)
success, future = self.perform_simple_statement(stmt)
@@ -1015,7 +1025,7 @@
self.writeresult(msg, color=RED)
for trace_id in future.get_query_trace_ids():
self.show_session(trace_id, partial_session=True)
- except Exception, err:
+ except Exception as err:
self.printerr("Unable to fetch query trace: %s" % (str(err),))
return success
@@ -1033,7 +1043,7 @@
try:
return self.get_view_meta(ks, name)
except MaterializedViewNotFound:
- raise ObjectNotFound("%r not found in keyspace %r" % (name, ks))
+ raise ObjectNotFound("'{}' not found in keyspace '{}'".format(name, ks))
def parse_for_update_meta(self, query_string):
try:
@@ -1052,8 +1062,9 @@
result = None
try:
result = future.result()
- except CQL_ERRORS, err:
- self.printerr(unicode(err.__class__.__name__) + u": " + err.message.decode(encoding='utf-8'))
+ except CQL_ERRORS as err:
+ err_msg = ensure_text(err.message if hasattr(err, 'message') else str(err))
+ self.printerr(str(err.__class__.__name__) + ": " + err_msg)
except Exception:
import traceback
self.printerr(traceback.format_exc())
@@ -1101,7 +1112,7 @@
if result.has_more_pages:
if self.shunted_query_out is None and tty:
# Only pause when not capturing.
- raw_input("---MORE---")
+ input("---MORE---")
result.fetch_next_page()
else:
if not tty:
@@ -1124,7 +1135,7 @@
if not result.column_names and not table_meta:
return
- column_names = result.column_names or table_meta.columns.keys()
+ column_names = result.column_names or list(table_meta.columns.keys())
formatted_names = [self.myformat_colname(name, table_meta) for name in column_names]
if not result.current_rows:
# print header only
@@ -1137,7 +1148,7 @@
ks_meta = self.conn.metadata.keyspaces.get(ks_name, None)
cql_types = [CqlType(cql_typename(t), ks_meta) for t in result.column_types]
- formatted_values = [map(self.myformat_value, [row[column] for column in column_names], cql_types) for row in result.current_rows]
+ formatted_values = [list(map(self.myformat_value, [row[c] for c in column_names], cql_types)) for row in result.current_rows]
if self.expand_enabled:
self.print_formatted_result_vertically(formatted_names, formatted_values)
@@ -1230,7 +1241,7 @@
def set_prompt(self, prompt, prepend_user=False):
if prepend_user and self.username:
- self.prompt = "%s@%s" % (self.username, prompt)
+ self.prompt = "{0}@{1}".format(self.username, prompt)
return
self.prompt = prompt
@@ -1243,208 +1254,16 @@
if valstr is not None:
return cqlruleset.dequote_value(valstr)
- def print_recreate_keyspace(self, ksdef, out):
- out.write(ksdef.export_as_string())
- out.write("\n")
-
- def print_recreate_columnfamily(self, ksname, cfname, out):
- """
- Output CQL commands which should be pasteable back into a CQL session
- to recreate the given table.
-
- Writes output to the given out stream.
- """
- out.write(self.get_table_meta(ksname, cfname).export_as_string())
- out.write("\n")
-
- def print_recreate_index(self, ksname, idxname, out):
- """
- Output CQL commands which should be pasteable back into a CQL session
- to recreate the given index.
-
- Writes output to the given out stream.
- """
- out.write(self.get_index_meta(ksname, idxname).export_as_string())
- out.write("\n")
-
- def print_recreate_materialized_view(self, ksname, viewname, out):
- """
- Output CQL commands which should be pasteable back into a CQL session
- to recreate the given materialized view.
-
- Writes output to the given out stream.
- """
- out.write(self.get_view_meta(ksname, viewname).export_as_string())
- out.write("\n")
-
- def print_recreate_object(self, ks, name, out):
- """
- Output CQL commands which should be pasteable back into a CQL session
- to recreate the given object (ks, table or index).
-
- Writes output to the given out stream.
- """
- out.write(self.get_object_meta(ks, name).export_as_string())
- out.write("\n")
-
- def describe_keyspaces(self):
- print
- cmd.Cmd.columnize(self, protect_names(self.get_keyspace_names()))
- print
-
- def describe_keyspace(self, ksname):
- print
- self.print_recreate_keyspace(self.get_keyspace_meta(ksname), sys.stdout)
- print
-
- def describe_columnfamily(self, ksname, cfname):
- if ksname is None:
- ksname = self.current_keyspace
- if ksname is None:
- raise NoKeyspaceError("No keyspace specified and no current keyspace")
- print
- self.print_recreate_columnfamily(ksname, cfname, sys.stdout)
- print
-
- def describe_index(self, ksname, idxname):
- print
- self.print_recreate_index(ksname, idxname, sys.stdout)
- print
-
- def describe_materialized_view(self, ksname, viewname):
- if ksname is None:
- ksname = self.current_keyspace
- if ksname is None:
- raise NoKeyspaceError("No keyspace specified and no current keyspace")
- print
- self.print_recreate_materialized_view(ksname, viewname, sys.stdout)
- print
-
- def describe_object(self, ks, name):
- print
- self.print_recreate_object(ks, name, sys.stdout)
- print
-
- def describe_columnfamilies(self, ksname):
- print
- if ksname is None:
- for k in self.get_keyspaces():
- name = protect_name(k.name)
- print 'Keyspace %s' % (name,)
- print '---------%s' % ('-' * len(name))
- cmd.Cmd.columnize(self, protect_names(self.get_columnfamily_names(k.name)))
- print
- else:
- cmd.Cmd.columnize(self, protect_names(self.get_columnfamily_names(ksname)))
- print
-
- def describe_functions(self, ksname):
- print
- if ksname is None:
- for ksmeta in self.get_keyspaces():
- name = protect_name(ksmeta.name)
- print 'Keyspace %s' % (name,)
- print '---------%s' % ('-' * len(name))
- self._columnize_unicode(ksmeta.functions.keys())
- else:
- ksmeta = self.get_keyspace_meta(ksname)
- self._columnize_unicode(ksmeta.functions.keys())
-
- def describe_function(self, ksname, functionname):
- if ksname is None:
- ksname = self.current_keyspace
- if ksname is None:
- raise NoKeyspaceError("No keyspace specified and no current keyspace")
- print
- ksmeta = self.get_keyspace_meta(ksname)
- functions = filter(lambda f: f.name == functionname, ksmeta.functions.values())
- if len(functions) == 0:
- raise FunctionNotFound("User defined function %r not found" % functionname)
- print "\n\n".join(func.export_as_string() for func in functions)
- print
-
- def describe_aggregates(self, ksname):
- print
- if ksname is None:
- for ksmeta in self.get_keyspaces():
- name = protect_name(ksmeta.name)
- print 'Keyspace %s' % (name,)
- print '---------%s' % ('-' * len(name))
- self._columnize_unicode(ksmeta.aggregates.keys())
- else:
- ksmeta = self.get_keyspace_meta(ksname)
- self._columnize_unicode(ksmeta.aggregates.keys())
-
- def describe_aggregate(self, ksname, aggregatename):
- if ksname is None:
- ksname = self.current_keyspace
- if ksname is None:
- raise NoKeyspaceError("No keyspace specified and no current keyspace")
- print
- ksmeta = self.get_keyspace_meta(ksname)
- aggregates = filter(lambda f: f.name == aggregatename, ksmeta.aggregates.values())
- if len(aggregates) == 0:
- raise FunctionNotFound("User defined aggregate %r not found" % aggregatename)
- print "\n\n".join(aggr.export_as_string() for aggr in aggregates)
- print
-
- def describe_usertypes(self, ksname):
- print
- if ksname is None:
- for ksmeta in self.get_keyspaces():
- name = protect_name(ksmeta.name)
- print 'Keyspace %s' % (name,)
- print '---------%s' % ('-' * len(name))
- self._columnize_unicode(ksmeta.user_types.keys(), quote=True)
- else:
- ksmeta = self.get_keyspace_meta(ksname)
- self._columnize_unicode(ksmeta.user_types.keys(), quote=True)
-
- def describe_usertype(self, ksname, typename):
- if ksname is None:
- ksname = self.current_keyspace
- if ksname is None:
- raise NoKeyspaceError("No keyspace specified and no current keyspace")
- print
- ksmeta = self.get_keyspace_meta(ksname)
- try:
- usertype = ksmeta.user_types[typename]
- except KeyError:
- raise UserTypeNotFound("User type %r not found" % typename)
- print usertype.export_as_string()
-
- def _columnize_unicode(self, name_list, quote=False):
+ def _columnize_unicode(self, name_list):
"""
Used when columnizing identifiers that may contain unicode
"""
- names = [n.encode('utf-8') for n in name_list]
- if quote:
- names = protect_names(names)
+ names = [n for n in name_list]
cmd.Cmd.columnize(self, names)
- print
-
- def describe_cluster(self):
- print '\nCluster: %s' % self.get_cluster_name()
- p = trim_if_present(self.get_partitioner(), 'org.apache.cassandra.dht.')
- print 'Partitioner: %s\n' % p
- # TODO: snitch?
- # snitch = trim_if_present(self.get_snitch(), 'org.apache.cassandra.locator.')
- # print 'Snitch: %s\n' % snitch
- if self.current_keyspace is not None and self.current_keyspace != 'system':
- print "Range ownership:"
- ring = self.get_ring(self.current_keyspace)
- for entry in ring.items():
- print ' %39s [%s]' % (str(entry[0].value), ', '.join([host.address for host in entry[1]]))
- print
-
- def describe_schema(self, include_system=False):
- print
- for k in self.get_keyspaces():
- if include_system or k.name not in cql3handling.SYSTEM_KEYSPACES:
- self.print_recreate_keyspace(k, sys.stdout)
- print
+ print('')
def do_describe(self, parsed):
+
"""
DESCRIBE [cqlsh only]
@@ -1463,7 +1282,6 @@
and the objects in it (such as tables, types, functions, etc.).
In some cases, as the CQL interface matures, there will be some metadata
about a keyspace that is not representable with CQL. That metadata will not be shown.
-
The '<keyspacename>' argument may be omitted, in which case the current
keyspace will be described.
@@ -1535,66 +1353,102 @@
Output CQL commands that could be used to recreate the entire object schema,
where object can be either a keyspace or a table or an index or a materialized
view (in this order).
- """
- what = parsed.matched[1][1].lower()
- if what == 'functions':
- self.describe_functions(self.current_keyspace)
- elif what == 'function':
- ksname = self.cql_unprotect_name(parsed.get_binding('ksname', None))
- functionname = self.cql_unprotect_name(parsed.get_binding('udfname'))
- self.describe_function(ksname, functionname)
- elif what == 'aggregates':
- self.describe_aggregates(self.current_keyspace)
- elif what == 'aggregate':
- ksname = self.cql_unprotect_name(parsed.get_binding('ksname', None))
- aggregatename = self.cql_unprotect_name(parsed.get_binding('udaname'))
- self.describe_aggregate(ksname, aggregatename)
- elif what == 'keyspaces':
- self.describe_keyspaces()
- elif what == 'keyspace':
- ksname = self.cql_unprotect_name(parsed.get_binding('ksname', ''))
- if not ksname:
- ksname = self.current_keyspace
- if ksname is None:
- self.printerr('Not in any keyspace.')
- return
- self.describe_keyspace(ksname)
- elif what in ('columnfamily', 'table'):
- ks = self.cql_unprotect_name(parsed.get_binding('ksname', None))
- cf = self.cql_unprotect_name(parsed.get_binding('cfname'))
- self.describe_columnfamily(ks, cf)
- elif what == 'index':
- ks = self.cql_unprotect_name(parsed.get_binding('ksname', None))
- idx = self.cql_unprotect_name(parsed.get_binding('idxname', None))
- self.describe_index(ks, idx)
- elif what == 'materialized' and parsed.matched[2][1].lower() == 'view':
- ks = self.cql_unprotect_name(parsed.get_binding('ksname', None))
- mv = self.cql_unprotect_name(parsed.get_binding('mvname'))
- self.describe_materialized_view(ks, mv)
- elif what in ('columnfamilies', 'tables'):
- self.describe_columnfamilies(self.current_keyspace)
- elif what == 'types':
- self.describe_usertypes(self.current_keyspace)
- elif what == 'type':
- ks = self.cql_unprotect_name(parsed.get_binding('ksname', None))
- ut = self.cql_unprotect_name(parsed.get_binding('utname'))
- self.describe_usertype(ks, ut)
- elif what == 'cluster':
- self.describe_cluster()
- elif what == 'schema':
- self.describe_schema(False)
- elif what == 'full' and parsed.matched[2][1].lower() == 'schema':
- self.describe_schema(True)
- elif what:
- ks = self.cql_unprotect_name(parsed.get_binding('ksname', None))
- name = self.cql_unprotect_name(parsed.get_binding('cfname'))
- if not name:
- name = self.cql_unprotect_name(parsed.get_binding('idxname', None))
- if not name:
- name = self.cql_unprotect_name(parsed.get_binding('mvname', None))
- self.describe_object(ks, name)
+ """
+ stmt = SimpleStatement(parsed.extract_orig(), consistency_level=cassandra.ConsistencyLevel.LOCAL_ONE, fetch_size=self.page_size if self.use_paging else None)
+ future = self.session.execute_async(stmt)
+
+ try:
+ result = future.result()
+
+ what = parsed.matched[1][1].lower()
+
+ if what in ('columnfamilies', 'tables', 'types', 'functions', 'aggregates'):
+ self.describe_list(result)
+ elif what == 'keyspaces':
+ self.describe_keyspaces(result)
+ elif what == 'cluster':
+ self.describe_cluster(result)
+ elif what:
+ self.describe_element(result)
+
+ except CQL_ERRORS as err:
+ err_msg = ensure_text(err.message if hasattr(err, 'message') else str(err))
+ self.printerr(err_msg.partition("message=")[2].strip('"'))
+ except Exception:
+ import traceback
+ self.printerr(traceback.format_exc())
+
+ if future:
+ if future.warnings:
+ self.print_warnings(future.warnings)
+
do_desc = do_describe
+ def describe_keyspaces(self, rows):
+ """
+ Print the output for a DESCRIBE KEYSPACES query
+ """
+ names = list()
+ for row in rows:
+ names.append(str(row['name']))
+
+ print('')
+ cmd.Cmd.columnize(self, names)
+ print('')
+
+ def describe_list(self, rows):
+ """
+ Print the output for all the DESCRIBE queries for element names (e.g DESCRIBE TABLES, DESCRIBE FUNCTIONS ...)
+ """
+ keyspace = None
+ names = list()
+ for row in rows:
+ if row['keyspace_name'] != keyspace:
+ if keyspace is not None:
+ self.print_keyspace_element_names(keyspace, names)
+
+ keyspace = row['keyspace_name']
+ names = list()
+
+ names.append(str(row['name']))
+
+ if keyspace is not None:
+ self.print_keyspace_element_names(keyspace, names)
+ print('')
+
+ def print_keyspace_element_names(self, keyspace, names):
+ print('')
+ if self.current_keyspace is None:
+ print('Keyspace %s' % (keyspace))
+ print('---------%s' % ('-' * len(keyspace)))
+ cmd.Cmd.columnize(self, names)
+
+ def describe_element(self, rows):
+ """
+ Print the output for all the DESCRIBE queries where an element name as been specified (e.g DESCRIBE TABLE, DESCRIBE INDEX ...)
+ """
+ for row in rows:
+ print('')
+ self.query_out.write(row['create_statement'])
+ print('')
+
+ def describe_cluster(self, rows):
+ """
+ Print the output for a DESCRIBE CLUSTER query.
+
+ If a specified keyspace was in use the returned ResultSet will contains a 'range_ownership' column,
+ otherwise not.
+ """
+ for row in rows:
+ print('\nCluster: %s' % row['cluster'])
+ print('Partitioner: %s' % row['partitioner'])
+ print('Snitch: %s\n' % row['snitch'])
+ if 'range_ownership' in row:
+ print("Range ownership:")
+ for entry in list(row['range_ownership'].items()):
+ print(' %39s [%s]' % (entry[0], ', '.join([host for host in entry[1]])))
+ print('')
+
def do_copy(self, parsed):
r"""
COPY [cqlsh only]
@@ -1685,7 +1539,7 @@
table = self.cql_unprotect_name(parsed.get_binding('cfname'))
columns = parsed.get_binding('colnames', None)
if columns is not None:
- columns = map(self.cql_unprotect_name, columns)
+ columns = list(map(self.cql_unprotect_name, columns))
else:
# default to all known columns
columns = self.get_column_names(ks, table)
@@ -1694,9 +1548,9 @@
if fname is not None:
fname = self.cql_unprotect_value(fname)
- copyoptnames = map(str.lower, parsed.get_binding('optnames', ()))
- copyoptvals = map(self.cql_unprotect_value, parsed.get_binding('optvals', ()))
- opts = dict(zip(copyoptnames, copyoptvals))
+ copyoptnames = list(map(six.text_type.lower, parsed.get_binding('optnames', ())))
+ copyoptvals = list(map(self.cql_unprotect_value, parsed.get_binding('optvals', ())))
+ opts = dict(list(zip(copyoptnames, copyoptvals)))
direction = parsed.get_binding('dir').upper()
if direction == 'FROM':
@@ -1718,8 +1572,8 @@
SHOW VERSION
Shows the version and build of the connected Cassandra instance, as
- well as the versions of the CQL spec and the Thrift protocol that
- the connected Cassandra instance understands.
+ well as the version of the CQL spec that the connected Cassandra
+ instance understands.
SHOW HOST
@@ -1767,7 +1621,7 @@
encoding, bom_size = get_file_encoding_bomsize(fname)
f = codecs.open(fname, 'r', encoding)
f.seek(bom_size)
- except IOError, e:
+ except IOError as e:
self.printerr('Could not open %r: %s' % (fname, e))
return
username = self.auth_provider.username if self.auth_provider else None
@@ -1785,7 +1639,12 @@
display_timezone=self.display_timezone,
max_trace_wait=self.max_trace_wait, ssl=self.ssl,
request_timeout=self.session.default_timeout,
- connect_timeout=self.conn.connect_timeout)
+ connect_timeout=self.conn.connect_timeout,
+ is_subshell=True)
+ # duplicate coverage related settings in subshell
+ if self.coverage:
+ subshell.coverage = True
+ subshell.coveragerc_path = self.coveragerc_path
subshell.cmdloop()
f.close()
@@ -1819,9 +1678,9 @@
fname = parsed.get_binding('fname')
if fname is None:
if self.shunted_query_out is not None:
- print "Currently capturing query output to %r." % (self.query_out.name,)
+ print("Currently capturing query output to %r." % (self.query_out.name,))
else:
- print "Currently not capturing query output."
+ print("Currently not capturing query output.")
return
if fname.upper() == 'OFF':
@@ -1843,14 +1702,14 @@
fname = os.path.expanduser(self.cql_unprotect_value(fname))
try:
f = open(fname, 'a')
- except IOError, e:
+ except IOError as e:
self.printerr('Could not open %r for append: %s' % (fname, e))
return
self.shunted_query_out = self.query_out
self.shunted_color = self.color
self.query_out = f
self.color = False
- print 'Now capturing query output to %r.' % (fname,)
+ print('Now capturing query output to %r.' % (fname,))
def do_tracing(self, parsed):
"""
@@ -1914,11 +1773,11 @@
"""
level = parsed.get_binding('level')
if level is None:
- print 'Current consistency level is %s.' % (cassandra.ConsistencyLevel.value_to_name[self.consistency_level])
+ print('Current consistency level is %s.' % (cassandra.ConsistencyLevel.value_to_name[self.consistency_level]))
return
self.consistency_level = cassandra.ConsistencyLevel.name_to_value[level.upper()]
- print 'Consistency level set to %s.' % (level.upper(),)
+ print('Consistency level set to %s.' % (level.upper(),))
def do_serial(self, parsed):
"""
@@ -1940,11 +1799,11 @@
"""
level = parsed.get_binding('level')
if level is None:
- print 'Current serial consistency level is %s.' % (cassandra.ConsistencyLevel.value_to_name[self.serial_consistency_level])
+ print('Current serial consistency level is %s.' % (cassandra.ConsistencyLevel.value_to_name[self.serial_consistency_level]))
return
self.serial_consistency_level = cassandra.ConsistencyLevel.name_to_value[level.upper()]
- print 'Serial consistency level set to %s.' % (level.upper(),)
+ print('Serial consistency level set to %s.' % (level.upper(),))
def do_login(self, parsed):
"""
@@ -2097,7 +1956,7 @@
if self.use_paging and requested_page_size is not None:
self.page_size = requested_page_size
if self.use_paging:
- print("Page size: {}".format(self.page_size))
+ print(("Page size: {}".format(self.page_size)))
else:
self.page_size = self.default_page_size
@@ -2111,13 +1970,11 @@
out = self.query_out
# convert Exceptions, etc to text
- if not isinstance(text, (unicode, str)):
- text = unicode(text)
-
- if isinstance(text, unicode):
- text = text.encode(self.encoding)
+ if not isinstance(text, six.text_type):
+ text = "{}".format(text)
to_write = self.applycolor(text, color) + ('\n' if newline else '')
+ to_write = ensure_str(to_write)
out.write(to_write)
def flush_output(self):
@@ -2131,6 +1988,12 @@
text = '%s:%d:%s' % (self.stdin.name, self.lineno, text)
self.writeresult(text, color, newline=newline, out=sys.stderr)
+ def stop_coverage(self):
+ if self.coverage and self.cov is not None:
+ self.cov.stop()
+ self.cov.save()
+ self.cov = None
+
class SwitchCommand(object):
command = None
@@ -2144,11 +2007,11 @@
switch = parsed.get_binding('switch')
if switch is None:
if state:
- print "%s is currently enabled. Use %s OFF to disable" \
- % (self.description, self.command)
+ print("%s is currently enabled. Use %s OFF to disable"
+ % (self.description, self.command))
else:
- print "%s is currently disabled. Use %s ON to enable." \
- % (self.description, self.command)
+ print("%s is currently disabled. Use %s ON to enable."
+ % (self.description, self.command))
return state
if switch.upper() == 'ON':
@@ -2156,14 +2019,14 @@
printerr('%s is already enabled. Use %s OFF to disable.'
% (self.description, self.command))
return state
- print 'Now %s is enabled' % (self.description,)
+ print('Now %s is enabled' % (self.description,))
return True
if switch.upper() == 'OFF':
if not state:
printerr('%s is not enabled.' % (self.description,))
return state
- print 'Disabled %s.' % (self.description,)
+ print('Disabled %s.' % (self.description,))
return False
@@ -2195,7 +2058,7 @@
def option_with_default(cparser_getter, section, option, default=None):
try:
return cparser_getter(section, option)
- except ConfigParser.Error:
+ except configparser.Error:
return default
@@ -2206,7 +2069,7 @@
"""
try:
return configs.get(section, option, raw=True)
- except ConfigParser.Error:
+ except configparser.Error:
return default
@@ -2229,10 +2092,10 @@
def read_options(cmdlineargs, environment):
- configs = ConfigParser.SafeConfigParser()
+ configs = configparser.SafeConfigParser() if sys.version_info < (3, 2) else configparser.ConfigParser()
configs.read(CONFIG_FILE)
- rawconfigs = ConfigParser.RawConfigParser()
+ rawconfigs = configparser.RawConfigParser()
rawconfigs.read(CONFIG_FILE)
optvalues = optparse.Values()
@@ -2259,9 +2122,13 @@
optvalues.timezone = option_with_default(configs.get, 'ui', 'timezone', None)
optvalues.debug = False
+
+ optvalues.coverage = False
+ if 'CQLSH_COVERAGE' in environment.keys():
+ optvalues.coverage = True
+
optvalues.file = None
optvalues.ssl = option_with_default(configs.getboolean, 'connection', 'ssl', DEFAULT_SSL)
- optvalues.no_compact = False
optvalues.encoding = option_with_default(configs.get, 'ui', 'encoding', UTF8)
optvalues.tty = option_with_default(configs.getboolean, 'ui', 'tty', sys.stdin.isatty())
@@ -2370,7 +2237,7 @@
encoding, bom_size = get_file_encoding_bomsize(options.file)
stdin = codecs.open(options.file, 'r', encoding)
stdin.seek(bom_size)
- except IOError, e:
+ except IOError as e:
sys.exit("Can't open %r: %s" % (options.file, e))
if options.debug:
@@ -2406,8 +2273,10 @@
# we silently ignore and fallback to UTC unless a custom timestamp format (which likely
# does contain a TZ part) was specified
if options.time_format != DEFAULT_TIMESTAMP_FORMAT:
- sys.stderr.write("Warning: custom timestamp format specified in cqlshrc, but local timezone could not be detected.\n" +
- "Either install Python 'tzlocal' module for auto-detection or specify client timezone in your cqlshrc.\n\n")
+ sys.stderr.write("Warning: custom timestamp format specified in cqlshrc, "
+ + "but local timezone could not be detected.\n"
+ + "Either install Python 'tzlocal' module for auto-detection "
+ + "or specify client timezone in your cqlshrc.\n\n")
try:
shell = Shell(hostname,
@@ -2422,7 +2291,6 @@
protocol_version=options.protocol_version,
cqlver=options.cqlversion,
keyspace=options.keyspace,
- no_compact=options.no_compact,
display_timestamp_format=options.time_format,
display_nanotime_format=options.nanotime_format,
display_date_format=options.date_format,
@@ -2437,12 +2305,21 @@
encoding=options.encoding)
except KeyboardInterrupt:
sys.exit('Connection aborted.')
- except CQL_ERRORS, e:
+ except CQL_ERRORS as e:
sys.exit('Connection error: %s' % (e,))
- except VersionNotSupported, e:
+ except VersionNotSupported as e:
sys.exit('Unsupported CQL version: %s' % (e,))
if options.debug:
shell.debug = True
+ if options.coverage:
+ shell.coverage = True
+ import signal
+
+ def handle_sighup():
+ shell.stop_coverage()
+ shell.do_exit()
+
+ signal.signal(signal.SIGHUP, handle_sighup)
shell.cmdloop()
save_history()
diff --git a/bin/debug-cql b/bin/debug-cql
index 00d4093..9550ddf 100755
--- a/bin/debug-cql
+++ b/bin/debug-cql
@@ -32,18 +32,6 @@
. "$CASSANDRA_INCLUDE"
fi
-# Use JAVA_HOME if set, otherwise look for java in PATH
-if [ -n "$JAVA_HOME" ]; then
- JAVA="$JAVA_HOME/bin/java"
-else
- JAVA=java
-fi
-
-if [ -z "$CASSANDRA_CONF" -o -z "$CLASSPATH" ]; then
- echo "You must set the CASSANDRA_CONF and CLASSPATH vars" >&2
- exit 1
-fi
-
if [ -f "$CASSANDRA_CONF/cassandra-env.sh" ]; then
. "$CASSANDRA_CONF/cassandra-env.sh"
fi
@@ -58,7 +46,7 @@
class="org.apache.cassandra.transport.Client"
cassandra_parms="-Dlogback.configurationFile=logback-tools.xml"
-"$JAVA" $JVM_OPTS $cassandra_parms -cp "$CLASSPATH" "$class" $1 $2
+"$JAVA" $JVM_OPTS $cassandra_parms -cp "$CLASSPATH" "$class" $@
exit $?
diff --git a/bin/nodetool b/bin/nodetool
index 6456b19..7ba57b7 100755
--- a/bin/nodetool
+++ b/bin/nodetool
@@ -38,18 +38,6 @@
. "$CASSANDRA_INCLUDE"
fi
-# Use JAVA_HOME if set, otherwise look for java in PATH
-if [ -x "$JAVA_HOME/bin/java" ]; then
- JAVA="$JAVA_HOME/bin/java"
-else
- JAVA="`which java`"
-fi
-
-if [ "x$JAVA" = "x" ]; then
- echo "Java executable not found (hint: set JAVA_HOME)" >&2
- exit 1
-fi
-
if [ -z "$CASSANDRA_CONF" -o -z "$CLASSPATH" ]; then
echo "You must set the CASSANDRA_CONF and CLASSPATH vars" >&2
exit 1
@@ -61,7 +49,7 @@
MAX_HEAP_SIZE_SAVE=$MAX_HEAP_SIZE
. "$CASSANDRA_CONF/cassandra-env.sh"
MAX_HEAP_SIZE=$MAX_HEAP_SIZE_SAVE
- JVM_OPTS=$JVM_OPTS_SAVE
+ JVM_OPTS="$JVM_OPTS_SAVE"
fi
# JMX Port passed via cmd line args (-p 9999 / --port 9999 / --port=9999)
@@ -91,6 +79,11 @@
fi
JVM_ARGS="$JVM_ARGS -Dssl.enable=true $SSL_ARGS"
;;
+ --archive-command)
+ # archive-command can be multi-word, we need to special handle that in POSIX shell
+ ARCHIVE_COMMAND="$2"
+ shift
+ ;;
-D*)
JVM_ARGS="$JVM_ARGS $1"
;;
@@ -105,11 +98,19 @@
MAX_HEAP_SIZE="128m"
fi
-"$JAVA" $JAVA_AGENT -ea -cp "$CLASSPATH" $JVM_OPTS -Xmx$MAX_HEAP_SIZE \
- -XX:ParallelGCThreads=1 \
- -Dcassandra.storagedir="$cassandra_storagedir" \
- -Dlogback.configurationFile=logback-tools.xml \
- $JVM_ARGS \
- org.apache.cassandra.tools.NodeTool -p $JMX_PORT $ARGS
+CMD=$(echo "$JAVA" $JAVA_AGENT -ea -cp "$CLASSPATH" $JVM_OPTS -Xmx$MAX_HEAP_SIZE \
+ -XX:ParallelGCThreads=1 \
+ -Dcassandra.storagedir="$cassandra_storagedir" \
+ -Dlogback.configurationFile=logback-tools.xml \
+ $JVM_ARGS \
+ org.apache.cassandra.tools.NodeTool -p $JMX_PORT $ARGS)
+
+if [ "x$ARCHIVE_COMMAND" != "x" ]
+then
+ exec $CMD "--archive-command" "${ARCHIVE_COMMAND}"
+else
+ exec $CMD
+fi
+
# vi:ai sw=4 ts=4 tw=0 et
diff --git a/bin/sstableloader b/bin/sstableloader
index 03ab4f9..9045adf 100755
--- a/bin/sstableloader
+++ b/bin/sstableloader
@@ -32,18 +32,6 @@
. "$CASSANDRA_INCLUDE"
fi
-# Use JAVA_HOME if set, otherwise look for java in PATH
-if [ -x "$JAVA_HOME/bin/java" ]; then
- JAVA="$JAVA_HOME/bin/java"
-else
- JAVA="`which java`"
-fi
-
-if [ "x$JAVA" = "x" ]; then
- echo "Java executable not found (hint: set JAVA_HOME)" >&2
- exit 1
-fi
-
if [ -z "$CLASSPATH" ]; then
echo "You must set the CLASSPATH var" >&2
exit 1
diff --git a/bin/sstablescrub b/bin/sstablescrub
index 366a2b7..9adda77 100755
--- a/bin/sstablescrub
+++ b/bin/sstablescrub
@@ -32,18 +32,6 @@
. "$CASSANDRA_INCLUDE"
fi
-# Use JAVA_HOME if set, otherwise look for java in PATH
-if [ -x "$JAVA_HOME/bin/java" ]; then
- JAVA="$JAVA_HOME/bin/java"
-else
- JAVA="`which java`"
-fi
-
-if [ "x$JAVA" = "x" ]; then
- echo "Java executable not found (hint: set JAVA_HOME)" >&2
- exit 1
-fi
-
if [ -z "$CLASSPATH" ]; then
echo "You must set the CLASSPATH var" >&2
exit 1
diff --git a/bin/sstableupgrade b/bin/sstableupgrade
index 7b307eb..c48faf0 100755
--- a/bin/sstableupgrade
+++ b/bin/sstableupgrade
@@ -32,18 +32,6 @@
. "$CASSANDRA_INCLUDE"
fi
-# Use JAVA_HOME if set, otherwise look for java in PATH
-if [ -x "$JAVA_HOME/bin/java" ]; then
- JAVA="$JAVA_HOME/bin/java"
-else
- JAVA="`which java`"
-fi
-
-if [ "x$JAVA" = "x" ]; then
- echo "Java executable not found (hint: set JAVA_HOME)" >&2
- exit 1
-fi
-
if [ -z "$CLASSPATH" ]; then
echo "You must set the CLASSPATH var" >&2
exit 1
diff --git a/bin/sstableutil b/bin/sstableutil
index 7457834..5d0bf1f 100755
--- a/bin/sstableutil
+++ b/bin/sstableutil
@@ -32,18 +32,6 @@
. "$CASSANDRA_INCLUDE"
fi
-# Use JAVA_HOME if set, otherwise look for java in PATH
-if [ -x "$JAVA_HOME/bin/java" ]; then
- JAVA="$JAVA_HOME/bin/java"
-else
- JAVA="`which java`"
-fi
-
-if [ "x$JAVA" = "x" ]; then
- echo "Java executable not found (hint: set JAVA_HOME)" >&2
- exit 1
-fi
-
if [ -z "$CLASSPATH" ]; then
echo "You must set the CLASSPATH var" >&2
exit 1
diff --git a/bin/sstableverify b/bin/sstableverify
index 6b296cf..ff812b9 100755
--- a/bin/sstableverify
+++ b/bin/sstableverify
@@ -32,18 +32,6 @@
. "$CASSANDRA_INCLUDE"
fi
-# Use JAVA_HOME if set, otherwise look for java in PATH
-if [ -x "$JAVA_HOME/bin/java" ]; then
- JAVA="$JAVA_HOME/bin/java"
-else
- JAVA="`which java`"
-fi
-
-if [ "x$JAVA" = "x" ]; then
- echo "Java executable not found (hint: set JAVA_HOME)" >&2
- exit 1
-fi
-
if [ -z "$CLASSPATH" ]; then
echo "You must set the CLASSPATH var" >&2
exit 1
diff --git a/build.xml b/build.xml
index 9932604..9b71cb0 100644
--- a/build.xml
+++ b/build.xml
@@ -25,7 +25,7 @@
<property name="debuglevel" value="source,lines,vars"/>
<!-- default version and SCM information -->
- <property name="base.version" value="3.11.7"/>
+ <property name="base.version" value="4.0-alpha5"/>
<property name="scm.connection" value="scm:https://gitbox.apache.org/repos/asf/cassandra.git"/>
<property name="scm.developerConnection" value="scm:https://gitbox.apache.org/repos/asf/cassandra.git"/>
<property name="scm.url" value="https://gitbox.apache.org/repos/asf?p=cassandra.git;a=tree"/>
@@ -35,7 +35,6 @@
<property name="build.src" value="${basedir}/src"/>
<property name="build.src.java" value="${basedir}/src/java"/>
<property name="build.src.antlr" value="${basedir}/src/antlr"/>
- <property name="build.src.jdkoverride" value="${basedir}/src/jdkoverride" />
<property name="build.src.resources" value="${basedir}/src/resources"/>
<property name="build.src.gen-java" value="${basedir}/src/gen-java"/>
<property name="build.lib" value="${basedir}/lib"/>
@@ -44,12 +43,8 @@
<property name="build.test.dir" value="${build.dir}/test"/>
<property name="build.classes" value="${build.dir}/classes"/>
<property name="build.classes.main" value="${build.classes}/main" />
- <property name="build.classes.thrift" value="${build.classes}/thrift" />
<property name="javadoc.dir" value="${build.dir}/javadoc"/>
- <property name="javadoc.jars.dir" value="${build.dir}/javadocs"/>
<property name="interface.dir" value="${basedir}/interface"/>
- <property name="interface.thrift.dir" value="${interface.dir}/thrift"/>
- <property name="interface.thrift.gen-java" value="${interface.thrift.dir}/gen-java"/>
<property name="test.dir" value="${basedir}/test"/>
<property name="test.resources" value="${test.dir}/resources"/>
<property name="test.lib" value="${build.dir}/test/lib"/>
@@ -64,8 +59,10 @@
<property name="test.unit.src" value="${test.dir}/unit"/>
<property name="test.long.src" value="${test.dir}/long"/>
<property name="test.burn.src" value="${test.dir}/burn"/>
+ <property name="test.memory.src" value="${test.dir}/memory"/>
<property name="test.microbench.src" value="${test.dir}/microbench"/>
<property name="test.distributed.src" value="${test.dir}/distributed"/>
+ <property name="test.compression_algo" value="LZ4"/>
<property name="test.distributed.listfile" value="ant-jvm-dtest-list"/>
<property name="test.distributed.upgrade.listfile" value="ant-jvm-dtest-upgrade-list"/>
<property name="test.distributed.upgrade.package" value="org.apache.cassandra.distributed.upgrade"/>
@@ -74,9 +71,6 @@
<property name="doc.dir" value="${basedir}/doc"/>
- <property name="source.version" value="1.8"/>
- <property name="target.version" value="1.8"/>
-
<condition property="version" value="${base.version}">
<isset property="release"/>
</condition>
@@ -102,6 +96,7 @@
<property name="maven-repository-id" value="apache.snapshots.https"/>
<property name="test.timeout" value="240000" />
+ <property name="test.memory.timeout" value="480000" />
<property name="test.long.timeout" value="600000" />
<property name="test.burn.timeout" value="60000000" />
<property name="test.distributed.timeout" value="360000" />
@@ -119,10 +114,20 @@
<property name="jacoco.finalexecfile" value="${jacoco.export.dir}/jacoco.exec" />
<property name="jacoco.version" value="0.7.5.201505241946"/>
- <property name="byteman.version" value="3.0.3"/>
+ <property name="byteman.version" value="4.0.6"/>
+ <property name="jamm.version" value="0.3.2"/>
+ <property name="ecj.version" value="4.6.1"/>
+ <property name="ohc.version" value="0.5.1"/>
+ <property name="asm.version" value="7.1"/>
+ <property name="allocation-instrumenter.version" value="3.1.0"/>
<property name="bytebuddy.version" value="1.10.10"/>
- <property name="ecj.version" value="4.4.2"/>
+ <!-- https://mvnrepository.com/artifact/net.openhft/chronicle-bom/1.16.23 -->
+ <property name="chronicle-queue.version" value="4.16.3" />
+ <property name="chronicle-core.version" value="1.16.4" />
+ <property name="chronicle-bytes.version" value="1.16.3" />
+ <property name="chronicle-wire.version" value="1.16.1" />
+ <property name="chronicle-threads.version" value="1.16.0" />
<condition property="maven-ant-tasks.jar.exists">
<available file="${build.dir}/maven-ant-tasks-${maven-ant-tasks.version}.jar" />
@@ -150,13 +155,97 @@
}
</script>
+ <condition property="java.version.8">
+ <equals arg1="${ant.java.version}" arg2="1.8"/>
+ </condition>
+ <condition property="java.version.11">
+ <not><isset property="java.version.8"/></not>
+ </condition>
+ <fail><condition><not><or>
+ <isset property="java.version.8"/>
+ <isset property="java.version.11"/>
+ </or></not></condition></fail>
+
+ <resources id="_jvm11_arg_items">
+ <string>-Djdk.attach.allowAttachSelf=true</string>
+
+ <string>-XX:+UseConcMarkSweepGC</string>
+ <string>-XX:+CMSParallelRemarkEnabled</string>
+ <string>-XX:SurvivorRatio=8</string>
+ <string>-XX:MaxTenuringThreshold=1</string>
+ <string>-XX:CMSInitiatingOccupancyFraction=75</string>
+ <string>-XX:+UseCMSInitiatingOccupancyOnly</string>
+ <string>-XX:CMSWaitDuration=10000</string>
+ <string>-XX:+CMSParallelInitialMarkEnabled</string>
+ <string>-XX:+CMSEdenChunksRecordAlways</string>
+
+ <string>--add-exports java.base/jdk.internal.misc=ALL-UNNAMED</string>
+ <string>--add-exports java.base/jdk.internal.ref=ALL-UNNAMED</string>
+ <string>--add-exports java.base/sun.nio.ch=ALL-UNNAMED</string>
+ <string>--add-exports java.management.rmi/com.sun.jmx.remote.internal.rmi=ALL-UNNAMED</string>
+ <string>--add-exports java.rmi/sun.rmi.registry=ALL-UNNAMED</string>
+ <string>--add-exports java.rmi/sun.rmi.server=ALL-UNNAMED</string>
+ <string>--add-exports java.sql/java.sql=ALL-UNNAMED</string>
+
+ <string>--add-opens java.base/java.lang.module=ALL-UNNAMED</string>
+ <string>--add-opens java.base/java.net=ALL-UNNAMED</string>
+ <string>--add-opens java.base/jdk.internal.loader=ALL-UNNAMED</string>
+ <string>--add-opens java.base/jdk.internal.ref=ALL-UNNAMED</string>
+ <string>--add-opens java.base/jdk.internal.reflect=ALL-UNNAMED</string>
+ <string>--add-opens java.base/jdk.internal.math=ALL-UNNAMED</string>
+ <string>--add-opens java.base/jdk.internal.module=ALL-UNNAMED</string>
+ <string>--add-opens java.base/jdk.internal.util.jar=ALL-UNNAMED</string>
+ <string>--add-opens jdk.management/com.sun.management.internal=ALL-UNNAMED</string>
+ </resources>
+ <pathconvert property="_jvm_args_concat" refid="_jvm11_arg_items" pathsep=" "/>
+ <condition property="java11-jvmargs" value="${_jvm_args_concat}" else="">
+ <not>
+ <equals arg1="${ant.java.version}" arg2="1.8"/>
+ </not>
+ </condition>
+
+ <!-- needed to compile org.apache.cassandra.utils.JMXServerUtils -->
+ <condition property="jdk11-javac-exports" value="--add-exports java.rmi/sun.rmi.registry=ALL-UNNAMED" else="">
+ <not>
+ <equals arg1="${ant.java.version}" arg2="1.8"/>
+ </not>
+ </condition>
+ <condition property="jdk11-javadoc-exports" value="${jdk11-javac-exports} --frames" else="">
+ <not>
+ <equals arg1="${ant.java.version}" arg2="1.8"/>
+ </not>
+ </condition>
+
+ <condition property="build.java.11">
+ <istrue value="${use.jdk11}"/>
+ </condition>
+
+ <condition property="source.version" value="8" else="11">
+ <equals arg1="${java.version.8}" arg2="true"/>
+ </condition>
+ <condition property="target.version" value="8" else="11">
+ <equals arg1="${java.version.8}" arg2="true"/>
+ </condition>
+
<!--
Add all the dependencies.
-->
<path id="maven-ant-tasks.classpath" path="${build.dir}/maven-ant-tasks-${maven-ant-tasks.version}.jar" />
<path id="cassandra.classpath">
<pathelement location="${build.classes.main}" />
- <pathelement location="${build.classes.thrift}" />
+ <fileset dir="${build.lib}">
+ <include name="**/*.jar" />
+ <exclude name="**/*-sources.jar"/>
+ <exclude name="**/ant-*.jar"/>
+ </fileset>
+ <fileset dir="${build.dir.lib}">
+ <include name="**/*.jar" />
+ <exclude name="**/*-sources.jar"/>
+ <exclude name="**/ant-*.jar"/>
+ </fileset>
+ </path>
+ <path id="cassandra.classpath.test">
+ <file file="${build.dir}/${final.name}.jar"/> <!-- we need the jar for tests and benchmarks (multi-version jar) -->
<fileset dir="${build.lib}">
<include name="**/*.jar" />
<exclude name="**/*-sources.jar"/>
@@ -175,25 +264,79 @@
<sequential>
<javadoc destdir="@{destdir}" author="true" version="true" use="true"
windowtitle="${ant.project.name} API" classpathref="cassandra.classpath"
- bottom="Copyright &copy; 2009-2019 The Apache Software Foundation"
- useexternalfile="yes" encoding="UTF-8"
- maxmemory="256m">
+ bottom="Copyright &copy; 2009-2020 The Apache Software Foundation"
+ useexternalfile="yes" encoding="UTF-8" failonerror="false"
+ maxmemory="256m" additionalparam="${jdk11-javadoc-exports}">
<filesets/>
</javadoc>
+ <fail message="javadoc failed">
+ <condition>
+ <not>
+ <available file="@{destdir}/index-all.html" />
+ </not>
+ </condition>
+ </fail>
</sequential>
</macrodef>
+ <target name="validate-build-conf">
+ <condition property="use-jdk11">
+ <or>
+ <isset property="build.java.11"/>
+ <istrue value="${env.CASSANDRA_USE_JDK11}"/>
+ </or>
+ </condition>
+ <fail message="Inconsistent JDK11 options set">
+ <condition>
+ <and>
+ <istrue value="${env.CASSANDRA_USE_JDK11}"/>
+ <isset property="use.jdk11"/>
+ <not>
+ <istrue value="${use.jdk11}"/>
+ </not>
+ </and>
+ </condition>
+ </fail>
+ <fail message="Inconsistent JDK11 options set">
+ <condition>
+ <and>
+ <isset property="env.CASSANDRA_USE_JDK11"/>
+ <not>
+ <istrue value="${env.CASSANDRA_USE_JDK11}"/>
+ </not>
+ <istrue value="${use.jdk11}"/>
+ </and>
+ </condition>
+ </fail>
+ <fail message="-Duse.jdk11=true or $CASSANDRA_USE_JDK11=true cannot be set when building from java 8">
+ <condition>
+ <not><or>
+ <not><isset property="java.version.8"/></not>
+ <not><isset property="use-jdk11"/></not>
+ </or></not>
+ </condition>
+ </fail>
+ <fail message="-Duse.jdk11=true or $CASSANDRA_USE_JDK11=true must be set when building from java 11">
+ <condition>
+ <not><or>
+ <isset property="java.version.8"/>
+ <isset property="use-jdk11"/>
+ </or></not>
+ </condition>
+ </fail>
+ </target>
+
<!--
Setup the output directories.
-->
- <target name="init">
+ <target name="init" depends="validate-build-conf">
<fail unless="is.source.artifact"
message="Not a source artifact, stopping here." />
<mkdir dir="${build.classes.main}"/>
- <mkdir dir="${build.classes.thrift}"/>
<mkdir dir="${test.lib}"/>
<mkdir dir="${test.classes}"/>
<mkdir dir="${stress.test.classes}"/>
+ <mkdir dir="${fqltool.test.classes}"/>
<mkdir dir="${build.src.gen-java}"/>
<mkdir dir="${build.dir.lib}"/>
<mkdir dir="${jacoco.export.dir}"/>
@@ -212,6 +355,7 @@
<target name="realclean" depends="clean" description="Remove the entire build directory and all downloaded artifacts">
<delete dir="${build.dir}" />
+ <delete dir="${doc.dir}/build" />
</target>
<!--
@@ -258,7 +402,7 @@
</wikitext-to-html>
</target>
- <target name="gen-doc" depends="maven-ant-tasks-init" description="Generate documentation">
+ <target name="gen-doc" depends="maven-ant-tasks-init" description="Generate documentation" unless="ant.gen-doc.skip">
<exec executable="make" osfamily="unix" dir="${doc.dir}">
<arg value="html"/>
</exec>
@@ -386,57 +530,55 @@
<license name="The Apache Software License, Version 2.0" url="https://www.apache.org/licenses/LICENSE-2.0.txt"/>
<scm connection="${scm.connection}" developerConnection="${scm.developerConnection}" url="${scm.url}"/>
<dependencyManagement>
- <dependency groupId="org.xerial.snappy" artifactId="snappy-java" version="1.1.1.7"/>
- <dependency groupId="net.jpountz.lz4" artifactId="lz4" version="1.3.0"/>
+ <dependency groupId="org.xerial.snappy" artifactId="snappy-java" version="1.1.2.6"/>
+ <dependency groupId="org.lz4" artifactId="lz4-java" version="1.7.1"/>
<dependency groupId="com.ning" artifactId="compress-lzf" version="0.8.4"/>
- <dependency groupId="com.google.guava" artifactId="guava" version="18.0"/>
+ <dependency groupId="com.github.luben" artifactId="zstd-jni" version="1.3.8-5"/>
+ <dependency groupId="com.google.guava" artifactId="guava" version="27.0-jre"/>
<dependency groupId="org.hdrhistogram" artifactId="HdrHistogram" version="2.1.9"/>
<dependency groupId="commons-cli" artifactId="commons-cli" version="1.1"/>
<dependency groupId="commons-codec" artifactId="commons-codec" version="1.9"/>
<dependency groupId="org.apache.commons" artifactId="commons-lang3" version="3.1"/>
<dependency groupId="org.apache.commons" artifactId="commons-math3" version="3.2"/>
- <dependency groupId="com.googlecode.concurrentlinkedhashmap" artifactId="concurrentlinkedhashmap-lru" version="1.4"/>
<dependency groupId="org.antlr" artifactId="antlr" version="3.5.2">
<exclusion groupId="org.antlr" artifactId="stringtemplate"/>
</dependency>
<dependency groupId="org.antlr" artifactId="antlr-runtime" version="3.5.2">
<exclusion groupId="org.antlr" artifactId="stringtemplate"/>
</dependency>
- <dependency groupId="org.slf4j" artifactId="slf4j-api" version="1.7.7"/>
- <dependency groupId="org.slf4j" artifactId="log4j-over-slf4j" version="1.7.7"/>
- <dependency groupId="org.slf4j" artifactId="jcl-over-slf4j" version="1.7.7" />
- <dependency groupId="ch.qos.logback" artifactId="logback-core" version="1.1.3"/>
- <dependency groupId="ch.qos.logback" artifactId="logback-classic" version="1.1.3"/>
+ <dependency groupId="org.slf4j" artifactId="slf4j-api" version="1.7.25"/>
+ <dependency groupId="org.slf4j" artifactId="log4j-over-slf4j" version="1.7.25"/>
+ <dependency groupId="org.slf4j" artifactId="jcl-over-slf4j" version="1.7.25" />
+ <dependency groupId="ch.qos.logback" artifactId="logback-core" version="1.2.3"/>
+ <dependency groupId="ch.qos.logback" artifactId="logback-classic" version="1.2.3"/>
<dependency groupId="com.fasterxml.jackson.core" artifactId="jackson-core" version="2.9.10"/>
<dependency groupId="com.fasterxml.jackson.core" artifactId="jackson-databind" version="2.9.10.4"/>
<dependency groupId="com.fasterxml.jackson.core" artifactId="jackson-annotations" version="2.9.10"/>
<dependency groupId="com.googlecode.json-simple" artifactId="json-simple" version="1.1"/>
<dependency groupId="com.boundary" artifactId="high-scale-lib" version="1.0.6"/>
- <dependency groupId="com.github.jbellis" artifactId="jamm" version="0.3.0"/>
+ <dependency groupId="com.github.jbellis" artifactId="jamm" version="${jamm.version}"/>
- <dependency groupId="com.thinkaurelius.thrift" artifactId="thrift-server" version="0.3.7">
- <exclusion groupId="org.slf4j" artifactId="slf4j-log4j12"/>
- <exclusion groupId="junit" artifactId="junit"/>
- </dependency>
<dependency groupId="org.yaml" artifactId="snakeyaml" version="1.11"/>
- <dependency groupId="org.apache.thrift" artifactId="libthrift" version="0.9.2">
- <exclusion groupId="commons-logging" artifactId="commons-logging"/>
- </dependency>
- <dependency groupId="junit" artifactId="junit" version="4.6" />
+ <dependency groupId="junit" artifactId="junit" version="4.12" />
<dependency groupId="org.mockito" artifactId="mockito-core" version="3.2.4" />
+ <dependency groupId="org.quicktheories" artifactId="quicktheories" version="0.25" />
+ <dependency groupId="com.google.code.java-allocation-instrumenter" artifactId="java-allocation-instrumenter" version="${allocation-instrumenter.version}" />
<dependency groupId="org.apache.cassandra" artifactId="dtest-api" version="0.0.3" />
+
<dependency groupId="org.apache.rat" artifactId="apache-rat" version="0.10">
<exclusion groupId="commons-lang" artifactId="commons-lang"/>
</dependency>
<dependency groupId="org.apache.hadoop" artifactId="hadoop-core" version="1.0.3">
- <exclusion groupId="org.mortbay.jetty" artifactId="servlet-api"/>
- <exclusion groupId="commons-logging" artifactId="commons-logging"/>
- <exclusion groupId="org.eclipse.jdt" artifactId="core"/>
- <exclusion groupId="ant" artifactId="ant"/>
- <exclusion groupId="junit" artifactId="junit"/>
+ <exclusion groupId="org.mortbay.jetty" artifactId="servlet-api"/>
+ <exclusion groupId="commons-logging" artifactId="commons-logging"/>
+ <exclusion groupId="org.eclipse.jdt" artifactId="core"/>
+ <exclusion groupId="ant" artifactId="ant"/>
+ <exclusion groupId="junit" artifactId="junit"/>
+ <exclusion groupId="org.slf4j" artifactId="slf4j-api"/>
</dependency>
<dependency groupId="org.apache.hadoop" artifactId="hadoop-minicluster" version="1.0.3">
- <exclusion groupId="asm" artifactId="asm"/> <!-- this is the outdated version 3.1 -->
+ <exclusion groupId="asm" artifactId="asm"/> <!-- this is the outdated version 3.1 -->
+ <exclusion groupId="org.slf4j" artifactId="slf4j-api"/>
</dependency>
<dependency groupId="net.java.dev.jna" artifactId="jna" version="4.2.2"/>
@@ -455,36 +597,60 @@
<dependency groupId="org.openjdk.jmh" artifactId="jmh-generator-annprocess" version="1.21"/>
<dependency groupId="org.apache.cassandra" artifactId="cassandra-all" version="${version}" />
- <dependency groupId="org.apache.cassandra" artifactId="cassandra-thrift" version="${version}" />
<dependency groupId="io.dropwizard.metrics" artifactId="metrics-core" version="3.1.5" />
<dependency groupId="io.dropwizard.metrics" artifactId="metrics-jvm" version="3.1.5" />
<dependency groupId="com.addthis.metrics" artifactId="reporter-config3" version="3.0.3" />
<dependency groupId="org.mindrot" artifactId="jbcrypt" version="0.3m" />
- <dependency groupId="io.airlift" artifactId="airline" version="0.6" />
- <dependency groupId="io.netty" artifactId="netty-all" version="4.0.44.Final" />
+ <dependency groupId="io.airlift" artifactId="airline" version="0.8" />
+ <dependency groupId="io.netty" artifactId="netty-all" version="4.1.50.Final" />
+ <dependency groupId="io.netty" artifactId="netty-tcnative-boringssl-static" version="2.0.31.Final" />
+ <dependency groupId="net.openhft" artifactId="chronicle-queue" version="${chronicle-queue.version}"/>
+ <dependency groupId="net.openhft" artifactId="chronicle-core" version="${chronicle-core.version}"/>
+ <dependency groupId="net.openhft" artifactId="chronicle-bytes" version="${chronicle-bytes.version}"/>
+ <dependency groupId="net.openhft" artifactId="chronicle-wire" version="${chronicle-wire.version}"/>
+ <dependency groupId="net.openhft" artifactId="chronicle-threads" version="${chronicle-threads.version}">
+ <!-- Exclude JNA here, as we want to avoid breaking consumers of the cassandra-all jar -->
+ <exclusion groupId="net.java.dev.jna" artifactId="jna" />
+ <exclusion groupId="net.java.dev.jna" artifactId="jna-platform" />
+ </dependency>
<dependency groupId="com.google.code.findbugs" artifactId="jsr305" version="2.0.2" />
<dependency groupId="com.clearspring.analytics" artifactId="stream" version="2.5.2" />
- <dependency groupId="com.datastax.cassandra" artifactId="cassandra-driver-core" version="3.0.1" classifier="shaded">
+ <dependency groupId="com.datastax.cassandra" artifactId="cassandra-driver-core" version="3.9.0" classifier="shaded">
<exclusion groupId="io.netty" artifactId="netty-buffer"/>
<exclusion groupId="io.netty" artifactId="netty-codec"/>
<exclusion groupId="io.netty" artifactId="netty-handler"/>
<exclusion groupId="io.netty" artifactId="netty-transport"/>
+ <exclusion groupId="org.slf4j" artifactId="slf4j-api"/>
</dependency>
- <dependency groupId="org.eclipse.jdt.core.compiler" artifactId="ecj" version="4.4.2" />
- <dependency groupId="org.caffinitas.ohc" artifactId="ohc-core" version="0.4.4" />
- <dependency groupId="org.caffinitas.ohc" artifactId="ohc-core-j8" version="0.4.4" />
+ <dependency groupId="org.eclipse.jdt.core.compiler" artifactId="ecj" version="${ecj.version}" />
+ <dependency groupId="org.caffinitas.ohc" artifactId="ohc-core" version="${ohc.version}">
+ <exclusion groupId="org.slf4j" artifactId="slf4j-api"/>
+ </dependency>
+ <dependency groupId="org.caffinitas.ohc" artifactId="ohc-core-j8" version="${ohc.version}" />
<dependency groupId="net.ju-n.compile-command-annotations" artifactId="compile-command-annotations" version="1.2.0" />
<dependency groupId="org.fusesource" artifactId="sigar" version="1.6.4">
- <exclusion groupId="log4j" artifactId="log4j"/>
+ <exclusion groupId="log4j" artifactId="log4j"/>
</dependency>
- <dependency groupId="joda-time" artifactId="joda-time" version="2.4" />
- <dependency groupId="com.carrotsearch" artifactId="hppc" version="0.5.4" />
+ <dependency groupId="com.carrotsearch" artifactId="hppc" version="0.8.1" />
<dependency groupId="de.jflex" artifactId="jflex" version="1.6.0" />
<dependency groupId="com.github.rholder" artifactId="snowball-stemmer" version="1.3.0.581.1" />
<dependency groupId="com.googlecode.concurrent-trees" artifactId="concurrent-trees" version="2.4.0" />
- <dependency groupId="com.github.ben-manes.caffeine" artifactId="caffeine" version="2.2.6" />
+ <dependency groupId="com.github.ben-manes.caffeine" artifactId="caffeine" version="2.3.5" />
<dependency groupId="org.jctools" artifactId="jctools-core" version="1.2.1"/>
- <dependency groupId="org.ow2.asm" artifactId="asm" version="5.0.4" />
+ <dependency groupId="org.ow2.asm" artifactId="asm" version="${asm.version}" />
+ <dependency groupId="org.ow2.asm" artifactId="asm-tree" version="${asm.version}" />
+ <dependency groupId="org.ow2.asm" artifactId="asm-commons" version="${asm.version}" />
+ <dependency groupId="org.gridkit.jvmtool" artifactId="sjk-cli" version="0.14"/>
+ <dependency groupId="org.gridkit.jvmtool" artifactId="sjk-core" version="0.14"/>
+ <dependency groupId="org.gridkit.jvmtool" artifactId="sjk-stacktrace" version="0.14"/>
+ <dependency groupId="org.gridkit.jvmtool" artifactId="mxdump" version="0.14"/>
+ <dependency groupId="org.gridkit.lab" artifactId="jvm-attach-api" version="1.5"/>
+ <dependency groupId="org.gridkit.jvmtool" artifactId="sjk-json" version="0.14"/>
+ <dependency groupId="com.beust" artifactId="jcommander" version="1.30"/>
+ <!-- when updating assertj, make sure to also update the corresponding junit-bom dependency -->
+ <dependency groupId="org.assertj" artifactId="assertj-core" version="3.15.0"/>
+ <dependency groupId="org.awaitility" artifactId="awaitility" version="4.0.3" />
+
</dependencyManagement>
<developer id="adelapena" name="Andres de la Peña"/>
<developer id="alakshman" name="Avinash Lakshman"/>
@@ -541,25 +707,35 @@
version="${version}"/>
<dependency groupId="junit" artifactId="junit"/>
<dependency groupId="org.mockito" artifactId="mockito-core" />
+ <dependency groupId="org.quicktheories" artifactId="quicktheories" />
+ <dependency groupId="com.google.code.java-allocation-instrumenter" artifactId="java-allocation-instrumenter" version="${allocation-instrumenter.version}" />
<dependency groupId="org.apache.cassandra" artifactId="dtest-api" />
+ <dependency groupId="org.psjava" artifactId="psjava" version="0.1.19" />
<dependency groupId="org.apache.rat" artifactId="apache-rat"/>
<dependency groupId="org.apache.hadoop" artifactId="hadoop-core"/>
- <dependency groupId="org.apache.hadoop" artifactId="hadoop-minicluster"/>
- <dependency groupId="com.google.code.findbugs" artifactId="jsr305"/>
+ <dependency groupId="org.apache.hadoop" artifactId="hadoop-minicluster"/>
+ <dependency groupId="com.google.code.findbugs" artifactId="jsr305"/>
<dependency groupId="org.antlr" artifactId="antlr"/>
- <dependency groupId="com.datastax.cassandra" artifactId="cassandra-driver-core" classifier="shaded">
- <exclusion groupId="io.netty" artifactId="netty-buffer"/>
- <exclusion groupId="io.netty" artifactId="netty-codec"/>
- <exclusion groupId="io.netty" artifactId="netty-handler"/>
- <exclusion groupId="io.netty" artifactId="netty-transport"/>
- </dependency>
+ <dependency groupId="com.datastax.cassandra" artifactId="cassandra-driver-core" classifier="shaded"/>
<dependency groupId="org.eclipse.jdt.core.compiler" artifactId="ecj"/>
- <dependency groupId="org.caffinitas.ohc" artifactId="ohc-core" version="0.4.4" />
- <dependency groupId="org.caffinitas.ohc" artifactId="ohc-core-j8" version="0.4.4" />
+ <dependency groupId="org.caffinitas.ohc" artifactId="ohc-core"/>
+ <dependency groupId="org.caffinitas.ohc" artifactId="ohc-core-j8"/>
<dependency groupId="org.openjdk.jmh" artifactId="jmh-core"/>
<dependency groupId="org.openjdk.jmh" artifactId="jmh-generator-annprocess"/>
<dependency groupId="net.ju-n.compile-command-annotations" artifactId="compile-command-annotations"/>
- <dependency groupId="org.apache.ant" artifactId="ant-junit" version="1.9.4" />
+ <dependency groupId="org.apache.ant" artifactId="ant-junit" version="1.9.7" />
+ <dependency groupId="org.gridkit.jvmtool" artifactId="sjk-cli" />
+ <dependency groupId="org.gridkit.jvmtool" artifactId="sjk-core" />
+ <dependency groupId="org.gridkit.jvmtool" artifactId="sjk-stacktrace" />
+ <dependency groupId="org.gridkit.jvmtool" artifactId="mxdump" />
+ <dependency groupId="org.gridkit.lab" artifactId="jvm-attach-api" />
+ <dependency groupId="com.beust" artifactId="jcommander" />
+ <dependency groupId="org.gridkit.jvmtool" artifactId="sjk-json"/>
+ <!-- adding this dependency is necessary for assertj. When updating assertj, need to also update the version of
+ this that the new assertj's `assertj-parent-pom` depends on. -->
+ <dependency groupId="org.junit" artifactId="junit-bom" version="5.6.0" type="pom"/>
+ <dependency groupId="org.assertj" artifactId="assertj-core"/>
+ <dependency groupId="org.awaitility" artifactId="awaitility"/>
</artifact:pom>
<!-- this build-deps-pom-sources "artifact" is the same as build-deps-pom but only with those
artifacts that have "-source.jar" files -->
@@ -570,20 +746,16 @@
version="${version}"/>
<dependency groupId="junit" artifactId="junit"/>
<dependency groupId="org.mockito" artifactId="mockito-core" />
- <dependency groupId="org.apache.cassandra" artifactId="dtest-api" />
- <dependency groupId="com.datastax.cassandra" artifactId="cassandra-driver-core" classifier="shaded">
- <exclusion groupId="io.netty" artifactId="netty-buffer"/>
- <exclusion groupId="io.netty" artifactId="netty-codec"/>
- <exclusion groupId="io.netty" artifactId="netty-handler"/>
- <exclusion groupId="io.netty" artifactId="netty-transport"/>
- </dependency>
+ <dependency groupId="com.datastax.cassandra" artifactId="cassandra-driver-core" classifier="shaded"/>
<dependency groupId="io.netty" artifactId="netty-all"/>
<dependency groupId="org.eclipse.jdt.core.compiler" artifactId="ecj"/>
<dependency groupId="org.caffinitas.ohc" artifactId="ohc-core"/>
<dependency groupId="org.openjdk.jmh" artifactId="jmh-core"/>
<dependency groupId="org.openjdk.jmh" artifactId="jmh-generator-annprocess"/>
<dependency groupId="net.ju-n.compile-command-annotations" artifactId="compile-command-annotations"/>
- <dependency groupId="org.apache.ant" artifactId="ant-junit" version="1.9.4" />
+ <dependency groupId="org.apache.ant" artifactId="ant-junit" version="1.9.7" />
+ <dependency groupId="org.assertj" artifactId="assertj-core"/>
+ <dependency groupId="org.awaitility" artifactId="awaitility"/>
</artifact:pom>
<artifact:pom id="coverage-deps-pom"
@@ -604,7 +776,6 @@
<parent groupId="org.apache.cassandra"
artifactId="cassandra-parent"
version="${version}"/>
- <dependency groupId="joda-time" artifactId="joda-time"/>
</artifact:pom>
<!-- now the pom's for artifacts being deployed to Maven Central -->
@@ -618,14 +789,13 @@
version="${version}"/>
<scm connection="${scm.connection}" developerConnection="${scm.developerConnection}" url="${scm.url}"/>
<dependency groupId="org.xerial.snappy" artifactId="snappy-java"/>
- <dependency groupId="net.jpountz.lz4" artifactId="lz4"/>
+ <dependency groupId="org.lz4" artifactId="lz4-java"/>
<dependency groupId="com.ning" artifactId="compress-lzf"/>
<dependency groupId="com.google.guava" artifactId="guava"/>
<dependency groupId="commons-cli" artifactId="commons-cli"/>
<dependency groupId="commons-codec" artifactId="commons-codec"/>
<dependency groupId="org.apache.commons" artifactId="commons-lang3"/>
<dependency groupId="org.apache.commons" artifactId="commons-math3"/>
- <dependency groupId="com.googlecode.concurrentlinkedhashmap" artifactId="concurrentlinkedhashmap-lru"/>
<dependency groupId="org.antlr" artifactId="antlr"/>
<dependency groupId="org.antlr" artifactId="antlr-runtime"/>
<dependency groupId="org.slf4j" artifactId="slf4j-api"/>
@@ -642,15 +812,11 @@
<dependency groupId="io.dropwizard.metrics" artifactId="metrics-core"/>
<dependency groupId="io.dropwizard.metrics" artifactId="metrics-jvm"/>
<dependency groupId="com.addthis.metrics" artifactId="reporter-config3"/>
- <dependency groupId="com.thinkaurelius.thrift" artifactId="thrift-server"/>
<dependency groupId="com.clearspring.analytics" artifactId="stream"/>
<dependency groupId="ch.qos.logback" artifactId="logback-core"/>
<dependency groupId="ch.qos.logback" artifactId="logback-classic"/>
- <dependency groupId="org.apache.thrift" artifactId="libthrift"/>
- <dependency groupId="org.apache.cassandra" artifactId="cassandra-thrift"/>
-
<!-- don't need hadoop classes to run, but if you use the hadoop stuff -->
<dependency groupId="org.apache.hadoop" artifactId="hadoop-core" optional="true"/>
<dependency groupId="org.apache.hadoop" artifactId="hadoop-minicluster" optional="true"/>
@@ -670,7 +836,11 @@
<dependency groupId="com.github.jbellis" artifactId="jamm"/>
<dependency groupId="io.netty" artifactId="netty-all"/>
- <dependency groupId="joda-time" artifactId="joda-time"/>
+ <dependency groupId="net.openhft" artifactId="chronicle-queue" version="${chronicle-queue.version}"/>
+ <dependency groupId="net.openhft" artifactId="chronicle-core" version="${chronicle-core.version}"/>
+ <dependency groupId="net.openhft" artifactId="chronicle-bytes" version="${chronicle-bytes.version}"/>
+ <dependency groupId="net.openhft" artifactId="chronicle-wire" version="${chronicle-wire.version}"/>
+ <dependency groupId="net.openhft" artifactId="chronicle-threads" version="${chronicle-threads.version}"/>
<dependency groupId="org.fusesource" artifactId="sigar"/>
<dependency groupId="org.eclipse.jdt.core.compiler" artifactId="ecj"/>
<dependency groupId="org.caffinitas.ohc" artifactId="ohc-core"/>
@@ -678,25 +848,14 @@
<dependency groupId="com.github.ben-manes.caffeine" artifactId="caffeine" />
<dependency groupId="org.jctools" artifactId="jctools-core"/>
<dependency groupId="org.ow2.asm" artifactId="asm" />
- </artifact:pom>
- <artifact:pom id="thrift-pom"
- artifactId="cassandra-thrift"
- url="https://cassandra.apache.org"
- name="Apache Cassandra">
- <parent groupId="org.apache.cassandra"
- artifactId="cassandra-parent"
- version="${version}"/>
- <scm connection="${scm.connection}" developerConnection="${scm.developerConnection}" url="${scm.url}"/>
- <dependency groupId="org.apache.commons" artifactId="commons-lang3"/>
- <dependency groupId="org.slf4j" artifactId="slf4j-api"/>
- <dependency groupId="org.slf4j" artifactId="log4j-over-slf4j"/>
- <dependency groupId="org.slf4j" artifactId="jcl-over-slf4j"/>
- <dependency groupId="org.apache.thrift" artifactId="libthrift"/>
- <dependency groupId="com.carrotsearch" artifactId="hppc" version="0.5.4" />
- <dependency groupId="de.jflex" artifactId="jflex" version="1.6.0" />
- <dependency groupId="com.github.rholder" artifactId="snowball-stemmer" version="1.3.0.581.1" />
- <dependency groupId="com.googlecode.concurrent-trees" artifactId="concurrent-trees" version="2.4.0" />
-
+ <dependency groupId="com.carrotsearch" artifactId="hppc" />
+ <dependency groupId="org.gridkit.jvmtool" artifactId="sjk-cli" />
+ <dependency groupId="org.gridkit.jvmtool" artifactId="sjk-core" />
+ <dependency groupId="org.gridkit.jvmtool" artifactId="sjk-stacktrace" />
+ <dependency groupId="org.gridkit.jvmtool" artifactId="mxdump" />
+ <dependency groupId="org.gridkit.lab" artifactId="jvm-attach-api" />
+ <dependency groupId="com.beust" artifactId="jcommander" />
+ <dependency groupId="org.gridkit.jvmtool" artifactId="sjk-json"/>
</artifact:pom>
</target>
@@ -741,6 +900,14 @@
</patternset>
<mapper type="flatten"/>
</unzip>
+
+ <!-- Need to delete some dependencies from build/lib/jars as non-matching versions may get in there
+ due to transitive dependencies -->
+ <delete>
+ <fileset dir="${build.dir.lib}/jars">
+ <include name="asm-*" />
+ </fileset>
+ </delete>
</target>
<target name="maven-ant-tasks-retrieve-test" depends="maven-ant-tasks-init">
@@ -766,54 +933,6 @@
<echo message="${base.version}" />
</target>
- <!--
- Generate thrift code. We have targets to build java because
- Cassandra depends on it, and python because that is what the system
- tests run.
- -->
- <target name="check-gen-thrift-java">
- <uptodate property="thriftUpToDate" srcfile="${interface.dir}/cassandra.thrift"
- targetfile="${interface.thrift.gen-java}/org/apache/cassandra/thrift/Cassandra.java" />
- </target>
- <target name="gen-thrift-java" unless="thriftUpToDate" depends="check-gen-thrift-java"
- description="Generate Thrift Java artifacts">
- <echo>Generating Thrift Java code from ${basedir}/interface/cassandra.thrift...</echo>
- <exec executable="thrift" dir="${basedir}/interface" failonerror="true">
- <arg line="--gen java:hashcode" />
- <arg line="-o ${interface.thrift.dir}" />
- <arg line="cassandra.thrift" />
- </exec>
- <antcall target="write-java-license-headers" />
- </target>
-
- <target name="_write-java-license-headers" depends="rat-init">
- <java classname="org.apache.rat.Report" fork="true"
- output="${build.dir}/rat-report.log">
- <classpath refid="rat.classpath" />
- <arg value="-a" />
- <arg value="--force" />
- <arg value="interface/thrift" />
- </java>
- </target>
-
- <target name="write-java-license-headers" unless="without.rat" description="Add missing java license headers">
- <antcall target="_write-java-license-headers" />
- </target>
-
- <target name="gen-thrift-py" description="Generate Thrift Python artifacts">
- <echo>Generating Thrift Python code from ${basedir}/interface/cassandra.thrift...</echo>
- <exec executable="thrift" dir="${basedir}/interface" failonerror="true">
- <arg line="--gen py" />
- <arg line="-o ${interface.thrift.dir}" />
- <arg line="cassandra.thrift" />
- </exec>
- <exec executable="thrift" dir="${basedir}/interface" failonerror="true">
- <arg line="--gen py:twisted" />
- <arg line="-o ${interface.thrift.dir}" />
- <arg line="cassandra.thrift" />
- </exec>
- </target>
-
<!-- create properties file with C version -->
<target name="createVersionPropFile">
<taskdef name="propertyfile" classname="org.apache.tools.ant.taskdefs.optional.PropertyFile"/>
@@ -823,16 +942,17 @@
</propertyfile>
</target>
- <target name="test-run" depends="build"
+ <target name="test-run" depends="jar"
description="Run in test mode. Not for production use!">
<java classname="org.apache.cassandra.service.CassandraDaemon" fork="true">
<classpath>
- <path refid="cassandra.classpath"/>
+ <path refid="cassandra.classpath.test"/>
<pathelement location="${test.conf}"/>
</classpath>
<jvmarg value="-Dstorage-config=${test.conf}"/>
- <jvmarg value="-javaagent:${basedir}/lib/jamm-0.3.0.jar" />
+ <jvmarg value="-javaagent:${basedir}/lib/jamm-${jamm.version}.jar" />
<jvmarg value="-ea"/>
+ <jvmarg line="${java11-jvmargs}"/>
</java>
</target>
@@ -843,17 +963,10 @@
depends="maven-ant-tasks-retrieve-build,build-project" description="Compile Cassandra classes"/>
<target name="codecoverage" depends="jacoco-run,jacoco-report" description="Create code coverage report"/>
- <target depends="init,gen-cql3-grammar,generate-cql-html,generate-jflex-java"
- name="build-project">
- <echo message="${ant.project.name}: ${ant.file}"/>
- <!-- Order matters! -->
- <javac fork="true"
- debug="true" debuglevel="${debuglevel}" encoding="utf-8"
- destdir="${build.classes.thrift}" includeantruntime="false" source="${source.version}" target="${target.version}"
- memorymaximumsize="512M">
- <src path="${interface.thrift.dir}/gen-java"/>
- <classpath refid="cassandra.classpath"/>
- </javac>
+ <target name="_build_java">
+ <!-- Note: we cannot use javac's 'release' option, as that does not allow accessing sun.misc.Unsafe nor
+ Nashorn's ClassFilter class as any javac modules option is invalid for relase 8. -->
+ <echo message="Compiling for Java ${target.version}..."/>
<javac fork="true"
debug="true" debuglevel="${debuglevel}" encoding="utf-8"
destdir="${build.classes.main}" includeantruntime="false" source="${source.version}" target="${target.version}"
@@ -861,14 +974,23 @@
<src path="${build.src.java}"/>
<src path="${build.src.gen-java}"/>
<compilerarg value="-XDignore.symbol.file"/>
- <compilerarg value="-Xbootclasspath/p:${build.src.jdkoverride}"/>
- <classpath refid="cassandra.classpath"/>
+ <compilerarg line="${jdk11-javac-exports}"/>
+ <classpath>
+ <path refid="cassandra.classpath"/>
+ </classpath>
</javac>
+ </target>
+
+ <target depends="init,gen-cql3-grammar,generate-cql-html,generate-jflex-java"
+ name="build-project">
+ <echo message="${ant.project.name}: ${ant.file}"/>
+ <!-- Order matters! -->
+ <antcall target="_build_java"/>
<antcall target="createVersionPropFile"/>
<copy todir="${build.classes.main}">
<fileset dir="${build.src.resources}" />
</copy>
- <copy todir="${basedir}/conf" file="${build.classes.main}/META-INF/hotspot_compiler"/>
+ <copy todir="${basedir}/conf" file="${build.classes.main}/META-INF/hotspot_compiler"/>
</target>
<!-- Stress build file -->
@@ -879,15 +1001,12 @@
<property name="stress.manifest" value="${stress.build.classes}/MANIFEST.MF" />
<path id="cassandra.classes">
<pathelement location="${basedir}/build/classes/main" />
- <pathelement location="${basedir}/build/classes/thrift" />
</path>
<target name="stress-build-test" depends="stress-build" description="Compile stress tests">
<javac debug="true" debuglevel="${debuglevel}" destdir="${stress.test.classes}"
- includeantruntime="false"
- source="${source.version}"
- target="${target.version}"
- encoding="utf-8">
+ source="${source.version}" target="${target.version}"
+ includeantruntime="false" encoding="utf-8">
<classpath>
<path refid="cassandra.classpath"/>
<pathelement location="${stress.build.classes}" />
@@ -898,7 +1017,9 @@
<target name="stress-build" depends="build" description="build stress tool">
<mkdir dir="${stress.build.classes}" />
- <javac compiler="modern" debug="true" debuglevel="${debuglevel}" encoding="utf-8" destdir="${stress.build.classes}" includeantruntime="true" source="${source.version}" target="${target.version}">
+ <javac compiler="modern" debug="true" debuglevel="${debuglevel}"
+ source="${source.version}" target="${target.version}"
+ encoding="utf-8" destdir="${stress.build.classes}" includeantruntime="true">
<src path="${stress.build.src}" />
<classpath>
<path refid="cassandra.classes" />
@@ -920,10 +1041,52 @@
</testmacro>
</target>
+ <!--
+ fqltool build file
+ -->
+ <property name="fqltool.build.src" value="${basedir}/tools/fqltool/src" />
+ <property name="fqltool.test.src" value="${basedir}/tools/fqltool/test/unit" />
+ <property name="fqltool.build.classes" value="${build.classes}/fqltool" />
+ <property name="fqltool.test.classes" value="${build.dir}/test/fqltool-classes" />
+ <property name="fqltool.manifest" value="${fqltool.build.classes}/MANIFEST.MF" />
+
+ <target name="fqltool-build-test" depends="fqltool-build" description="Compile fqltool tests">
+ <javac debug="true" debuglevel="${debuglevel}" destdir="${fqltool.test.classes}"
+ source="${source.version}" target="${target.version}"
+ includeantruntime="false" encoding="utf-8">
+ <classpath>
+ <path refid="cassandra.classpath"/>
+ <pathelement location="${fqltool.build.classes}" />
+ </classpath>
+ <src path="${fqltool.test.src}"/>
+ </javac>
+ </target>
+
+ <target name="fqltool-build" depends="build" description="build fqltool">
+ <mkdir dir="${fqltool.build.classes}" />
+ <javac compiler="modern" debug="true" debuglevel="${debuglevel}"
+ source="${source.version}" target="${target.version}"
+ encoding="utf-8" destdir="${fqltool.build.classes}" includeantruntime="true">
+ <src path="${fqltool.build.src}" />
+ <classpath>
+ <path refid="cassandra.classes" />
+ <path>
+ <fileset dir="${build.lib}">
+ <include name="**/*.jar" />
+ </fileset>
+ </path>
+ </classpath>
+ </javac>
+ </target>
+
+ <target name="fqltool-test" depends="fqltool-build-test, build-test" description="Runs fqltool tests">
+ <testmacro inputdir="${fqltool.test.src}"
+ timeout="${test.timeout}">
+ </testmacro>
+ </target>
+
<target name="_write-poms" depends="maven-declare-dependencies">
<artifact:writepom pomRefId="parent-pom" file="${build.dir}/${final.name}-parent.pom"/>
- <artifact:writepom pomRefId="thrift-pom"
- file="${build.dir}/${ant.project.name}-thrift-${version}.pom"/>
<artifact:writepom pomRefId="all-pom" file="${build.dir}/${final.name}.pom"/>
</target>
@@ -934,53 +1097,32 @@
<!--
The jar target makes cassandra.jar output.
-->
- <target name="jar"
- depends="build, build-test, stress-build, write-poms"
+ <target name="_main-jar"
+ depends="build"
description="Assemble Cassandra JAR files">
<mkdir dir="${build.classes.main}/META-INF" />
- <mkdir dir="${build.classes.thrift}/META-INF" />
<copy file="LICENSE.txt"
tofile="${build.classes.main}/META-INF/LICENSE.txt"/>
- <copy file="LICENSE.txt"
- tofile="${build.classes.thrift}/META-INF/LICENSE.txt"/>
<copy file="NOTICE.txt"
tofile="${build.classes.main}/META-INF/NOTICE.txt"/>
- <copy file="NOTICE.txt"
- tofile="${build.classes.thrift}/META-INF/NOTICE.txt"/>
-
- <!-- Thrift Jar -->
- <jar jarfile="${build.dir}/${ant.project.name}-thrift-${version}.jar"
- basedir="${build.classes.thrift}">
- <fileset dir="${build.classes.main}">
- <include name="org/apache/cassandra/thrift/ITransportFactory*.class" />
- <include name="org/apache/cassandra/thrift/TFramedTransportFactory*.class" />
- </fileset>
- <manifest>
- <attribute name="Implementation-Title" value="Cassandra"/>
- <attribute name="Implementation-Version" value="${version}"/>
- <attribute name="Implementation-Vendor" value="Apache"/>
- </manifest>
- </jar>
<!-- Main Jar -->
<jar jarfile="${build.dir}/${final.name}.jar">
<fileset dir="${build.classes.main}">
- <exclude name="org/apache/cassandra/thrift/ITransportFactory*.class" />
- <exclude name="org/apache/cassandra/thrift/TFramedTransportFactory*.class" />
</fileset>
<manifest>
<!-- <section name="org/apache/cassandra/infrastructure"> -->
+ <attribute name="Multi-Release" value="true"/>
<attribute name="Implementation-Title" value="Cassandra"/>
<attribute name="Implementation-Version" value="${version}"/>
<attribute name="Implementation-Vendor" value="Apache"/>
- <attribute name="Premain-Class"
- value="org.apache.cassandra.infrastructure.continuations.CAgent"/>
- <attribute name="Class-Path"
- value="${ant.project.name}-thrift-${version}.jar" />
<!-- </section> -->
</manifest>
</jar>
-
+ </target>
+ <target name="jar"
+ depends="_main-jar, build-test, stress-build, fqltool-build, write-poms"
+ description="Assemble Cassandra JAR files">
<!-- Stress jar -->
<manifest file="${stress.manifest}">
<attribute name="Built-By" value="Pavel Yaskevich"/>
@@ -991,50 +1133,32 @@
<jar destfile="${build.dir}/tools/lib/stress.jar" manifest="${stress.manifest}">
<fileset dir="${stress.build.classes}"/>
</jar>
+ <!-- fqltool jar -->
+ <manifest file="${fqltool.manifest}">
+ <attribute name="Built-By" value="Marcus Eriksson"/>
+ <attribute name="Main-Class" value="org.apache.cassandra.fqltool.FullQueryLogTool"/>
+ </manifest>
+ <mkdir dir="${fqltool.build.classes}/META-INF" />
+ <mkdir dir="${build.dir}/tools/lib/" />
+ <jar destfile="${build.dir}/tools/lib/fqltool.jar" manifest="${stress.manifest}">
+ <fileset dir="${fqltool.build.classes}"/>
+ </jar>
</target>
<!--
The javadoc-jar target makes cassandra-javadoc.jar output required for publishing to Maven central repository.
-->
- <target name="javadoc-jar" description="Assemble Cassandra JavaDoc JAR file">
- <mkdir dir="${javadoc.jars.dir}"/>
- <create-javadoc destdir="${javadoc.jars.dir}/thrift">
- <filesets>
- <fileset dir="${interface.thrift.dir}/gen-java" defaultexcludes="yes">
- <include name="org/apache/**/*.java"/>
- </fileset>
- </filesets>
- </create-javadoc>
- <jar jarfile="${build.dir}/${ant.project.name}-thrift-${version}-javadoc.jar"
- basedir="${javadoc.jars.dir}/thrift"/>
-
- <create-javadoc destdir="${javadoc.jars.dir}/main">
- <filesets>
- <fileset dir="${build.src.java}" defaultexcludes="yes">
- <include name="org/apache/**/*.java"/>
- </fileset>
- <fileset dir="${build.src.gen-java}" defaultexcludes="yes">
- <include name="org/apache/**/*.java"/>
- </fileset>
- </filesets>
- </create-javadoc>
- <jar jarfile="${build.dir}/${final.name}-javadoc.jar"
- basedir="${javadoc.jars.dir}/main"/>
-
+ <target name="javadoc-jar" depends="javadoc" description="Assemble Cassandra JavaDoc JAR file">
+ <jar jarfile="${build.dir}/${final.name}-javadoc.jar" basedir="${javadoc.dir}"/>
<!-- javadoc task always rebuilds so might as well remove the generated docs to prevent
being pulled into the distribution by accident -->
- <delete quiet="true" dir="${javadoc.jars.dir}"/>
+ <delete quiet="true" dir="${javadoc.dir}"/>
</target>
<!--
The sources-jar target makes cassandra-sources.jar output required for publishing to Maven central repository.
-->
<target name="sources-jar" depends="init" description="Assemble Cassandra Sources JAR file">
- <jar jarfile="${build.dir}/${ant.project.name}-thrift-${version}-sources.jar">
- <fileset dir="${interface.thrift.dir}/gen-java" defaultexcludes="yes">
- <include name="org/apache/**/*.java"/>
- </fileset>
- </jar>
<jar jarfile="${build.dir}/${final.name}-sources.jar">
<fileset dir="${build.src.java}" defaultexcludes="yes">
<include name="org/apache/**/*.java"/>
@@ -1045,9 +1169,7 @@
</jar>
</target>
- <!-- creates release tarballs -->
- <target name="artifacts" depends="jar,javadoc,gen-doc"
- description="Create Cassandra release artifacts">
+ <target name="_artifacts-init" depends="jar,javadoc,gen-doc">
<mkdir dir="${dist.dir}"/>
<!-- fix the control linefeed so that builds on windows works on linux -->
<fixcrlf srcdir="bin" includes="**/*" excludes="**/*.bat, **/*.ps1" eol="lf" eof="remove" />
@@ -1057,20 +1179,19 @@
<fileset dir="${build.lib}"/>
<fileset dir="${build.dir}">
<include name="${final.name}.jar" />
- <include name="${ant.project.name}-thrift-${version}.jar" />
</fileset>
</copy>
- <copy todir="${dist.dir}/javadoc">
+ <copy todir="${dist.dir}/javadoc" failonerror="false">
<fileset dir="${javadoc.dir}"/>
</copy>
- <copy todir="${dist.dir}/doc">
+ <copy todir="${dist.dir}/doc" failonerror="false">
<fileset dir="doc">
<include name="cql3/CQL.html" />
<include name="cql3/CQL.css" />
<include name="SASI.md" />
</fileset>
</copy>
- <copy todir="${dist.dir}/doc/html">
+ <copy todir="${dist.dir}/doc/html" failonerror="false">
<fileset dir="doc" />
<globmapper from="build/html/*" to="*"/>
</copy>
@@ -1080,11 +1201,6 @@
<copy todir="${dist.dir}/conf">
<fileset dir="conf"/>
</copy>
- <copy todir="${dist.dir}/interface">
- <fileset dir="interface">
- <include name="**/*.thrift" />
- </fileset>
- </copy>
<copy todir="${dist.dir}/pylib">
<fileset dir="pylib">
<include name="**" />
@@ -1109,6 +1225,11 @@
<include name="*.jar" />
</fileset>
</copy>
+ </target>
+
+ <!-- creates release tarballs -->
+ <target name="artifacts" depends="_artifacts-init"
+ description="Create Cassandra release artifacts">
<tar compression="gzip" longfile="gnu"
destfile="${build.dir}/${final.name}-bin.tar.gz">
@@ -1199,7 +1320,7 @@
</rat:report>
</target>
- <target name="build-jmh" depends="build-test" description="Create JMH uber jar">
+ <target name="build-jmh" depends="build-test, jar" description="Create JMH uber jar">
<jar jarfile="${build.test.dir}/deps.jar">
<zipgroupfileset dir="${build.dir.lib}/jars">
<include name="*jmh*.jar"/>
@@ -1215,11 +1336,17 @@
<zipfileset src="${build.test.dir}/deps.jar" excludes="META-INF/*.SF" />
<fileset dir="${build.classes.main}"/>
<fileset dir="${test.classes}"/>
+ <fileset dir="${test.conf}" />
</jar>
</target>
- <target name="build-test" depends="build" description="Compile test classes">
+ <target name="build-test" depends="_main-jar, stress-build, fqltool-build, write-poms" description="Compile test classes">
+ <antcall target="_build-test"/>
+ </target>
+
+ <target name="_build-test">
<javac
+ fork="true"
compiler="modern"
debug="true"
debuglevel="${debuglevel}"
@@ -1230,11 +1357,13 @@
encoding="utf-8">
<classpath>
<path refid="cassandra.classpath"/>
+ <pathelement location="${fqltool.build.classes}"/>
</classpath>
<compilerarg value="-XDignore.symbol.file"/>
<src path="${test.unit.src}"/>
<src path="${test.long.src}"/>
<src path="${test.burn.src}"/>
+ <src path="${test.memory.src}"/>
<src path="${test.microbench.src}"/>
<src path="${test.distributed.src}"/>
</javac>
@@ -1280,7 +1409,7 @@
<jvmarg value="-Dstorage-config=${test.conf}"/>
<jvmarg value="-Djava.awt.headless=true"/>
<!-- Cassandra 3.0+ needs <jvmarg line="... ${additionalagent}" /> here! (not value=) -->
- <jvmarg line="-javaagent:${basedir}/lib/jamm-0.3.0.jar ${additionalagent}" />
+ <jvmarg line="-javaagent:${basedir}/lib/jamm-${jamm.version}.jar ${additionalagent}" />
<jvmarg value="-ea"/>
<jvmarg value="-Djava.io.tmpdir=${tmp.dir}"/>
<jvmarg value="-Dcassandra.debugrefcount=true"/>
@@ -1291,6 +1420,8 @@
more aggressively rather than waiting. See CASSANDRA-14922 for more details.
-->
<jvmarg value="-XX:MaxMetaspaceSize=384M" />
+ <jvmarg value="-XX:MetaspaceSize=128M" />
+ <jvmarg value="-XX:MaxMetaspaceExpansion=64M" />
<jvmarg value="-XX:SoftRefLRUPolicyMSPerMB=0" />
<jvmarg value="-Dcassandra.memtable_row_overhead_computation_step=100"/>
<jvmarg value="-Dcassandra.test.use_prepared=${cassandra.test.use_prepared}"/>
@@ -1300,14 +1431,23 @@
<jvmarg value="-Djava.security.egd=file:/dev/urandom" />
<jvmarg value="-Dcassandra.testtag=@{testtag}"/>
<jvmarg value="-Dcassandra.keepBriefBrief=${cassandra.keepBriefBrief}" />
- <jvmarg value="-Dcassandra.strict.runtime.checks=true" />
- <optjvmargs/>
+ <jvmarg value="-Dcassandra.strict.runtime.checks=true" />
+ <jvmarg line="${java11-jvmargs}"/>
+ <!-- disable shrinks in quicktheories CASSANDRA-15554 -->
+ <jvmarg value="-DQT_SHRINKS=0"/>
+ <optjvmargs/>
+ <!-- Uncomment to debug unittest, attach debugger to port 1416 -->
+ <!--
+ <jvmarg line="-agentlib:jdwp=transport=dt_socket,address=localhost:1416,server=y,suspend=y" />
+ -->
<classpath>
<pathelement path="${java.class.path}"/>
<pathelement location="${stress.build.classes}"/>
- <path refid="cassandra.classpath" />
+ <pathelement location="${fqltool.build.classes}"/>
+ <path refid="cassandra.classpath.test" />
<pathelement location="${test.classes}"/>
<pathelement location="${stress.test.classes}"/>
+ <pathelement location="${fqltool.test.classes}"/>
<pathelement location="${test.conf}"/>
<fileset dir="${test.lib}">
<include name="**/*.jar" />
@@ -1335,11 +1475,10 @@
</sequential>
</macrodef>
- <target name="testold" depends="build-test" description="Execute unit tests">
+ <target name="testold" depends="build-test" description="Execute unit tests">
<testmacro inputdir="${test.unit.src}" timeout="${test.timeout}">
<jvmarg value="-Dlegacy-sstable-root=${test.data}/legacy-sstables"/>
<jvmarg value="-Dinvalid-legacy-sstable-root=${test.data}/invalid-legacy-sstables"/>
- <jvmarg value="-Dmigration-sstable-root=${test.data}/migration-sstables"/>
<jvmarg value="-Dcassandra.ring_delay_ms=1000"/>
<jvmarg value="-Dcassandra.tolerate_sstable_size=true"/>
<jvmarg value="-Dcassandra.skip_sync=true" />
@@ -1356,7 +1495,6 @@
<testmacrohelper inputdir="${test.dir}/${test.classlistprefix}" filelist="@{test.file.list}" poffset="@{testlist.offset}" exclude="**/*.java" timeout="${test.timeout}">
<jvmarg value="-Dlegacy-sstable-root=${test.data}/legacy-sstables"/>
<jvmarg value="-Dinvalid-legacy-sstable-root=${test.data}/invalid-legacy-sstables"/>
- <jvmarg value="-Dmigration-sstable-root=${test.data}/migration-sstables"/>
<jvmarg value="-Dcassandra.ring_delay_ms=1000"/>
<jvmarg value="-Dcassandra.tolerate_sstable_size=true"/>
<jvmarg value="-Dcassandra.config.loader=org.apache.cassandra.OffsetAwareConfigurationLoader"/>
@@ -1380,7 +1518,6 @@
exclude="**/*.java" timeout="${test.timeout}" testtag="compression">
<jvmarg value="-Dlegacy-sstable-root=${test.data}/legacy-sstables"/>
<jvmarg value="-Dinvalid-legacy-sstable-root=${test.data}/invalid-legacy-sstables"/>
- <jvmarg value="-Dmigration-sstable-root=${test.data}/migration-sstables"/>
<jvmarg value="-Dcassandra.test.compression=true"/>
<jvmarg value="-Dcassandra.ring_delay_ms=1000"/>
<jvmarg value="-Dcassandra.tolerate_sstable_size=true"/>
@@ -1404,7 +1541,6 @@
exclude="**/*.java" timeout="${test.timeout}" testtag="cdc">
<jvmarg value="-Dlegacy-sstable-root=${test.data}/legacy-sstables"/>
<jvmarg value="-Dinvalid-legacy-sstable-root=${test.data}/invalid-legacy-sstables"/>
- <jvmarg value="-Dmigration-sstable-root=${test.data}/migration-sstables"/>
<jvmarg value="-Dcassandra.ring_delay_ms=1000"/>
<jvmarg value="-Dcassandra.tolerate_sstable_size=true"/>
<jvmarg value="-Dcassandra.config=file:///${cdc_yaml}"/>
@@ -1434,7 +1570,6 @@
<test name="${test.name}" methods="${test.methods}" outfile="build/test/output/TEST-${test.name}-${test.methods}"/>
<jvmarg value="-Dlegacy-sstable-root=${test.data}/legacy-sstables"/>
<jvmarg value="-Dinvalid-legacy-sstable-root=${test.data}/invalid-legacy-sstables"/>
- <jvmarg value="-Dmigration-sstable-root=${test.data}/migration-sstables"/>
<jvmarg value="-Dcassandra.ring_delay_ms=1000"/>
<jvmarg value="-Dcassandra.tolerate_sstable_size=true"/>
<jvmarg value="-Dcassandra.skip_sync=true" />
@@ -1442,13 +1577,26 @@
</target>
<!-- Use this with an FQDN for test class, and a csv list of methods like this:
+ ant long-testsome -Dtest.name=org.apache.cassandra.cql3.ViewLongTest -Dtest.methods=testConflictResolution
+ -->
+ <target name="long-testsome" depends="build-test" description="Execute specific long unit tests" >
+ <testmacro inputdir="${test.long.src}" timeout="${test.long.timeout}">
+ <test name="${test.name}" methods="${test.methods}"/>
+ <jvmarg value="-Dcassandra.ring_delay_ms=1000"/>
+ <jvmarg value="-Dcassandra.tolerate_sstable_size=true"/>
+ </testmacro>
+ </target>
+
+ <!-- Use this with an FQDN for test class, and a csv list of methods like this:
ant burn-testsome -Dtest.name=org.apache.cassandra.utils.memory.LongBufferPoolTest -Dtest.methods=testAllocate
-->
<target name="burn-testsome" depends="build-test" description="Execute specific burn unit tests" >
<testmacro inputdir="${test.burn.src}" timeout="${test.burn.timeout}">
<test name="${test.name}" methods="${test.methods}"/>
+ <jvmarg value="-Dlogback.configurationFile=test/conf/logback-burntest.xml"/>
</testmacro>
</target>
+
<target name="test-compression" depends="build-test,stress-build" description="Execute unit tests with sstable compression enabled">
<path id="all-test-classes-path">
<fileset dir="${test.unit.src}" includes="**/${test.name}.java" />
@@ -1506,6 +1654,13 @@
</testmacro>
</target>
+ <target name="test-memory" depends="build-test" description="Execute functional tests">
+ <testmacro inputdir="${test.memory.src}"
+ timeout="${test.memory.timeout}">
+ <jvmarg value="-javaagent:${build.dir}/lib/jars/java-allocation-instrumenter-${allocation-instrumenter.version}.jar"/>
+ </testmacro>
+ </target>
+
<target name="cql-test" depends="build-test" description="Execute CQL tests">
<sequential>
<echo message="running CQL tests"/>
@@ -1515,14 +1670,14 @@
<formatter type="brief" usefile="false"/>
<jvmarg value="-Dstorage-config=${test.conf}"/>
<jvmarg value="-Djava.awt.headless=true"/>
- <jvmarg value="-javaagent:${basedir}/lib/jamm-0.3.0.jar" />
+ <jvmarg value="-javaagent:${basedir}/lib/jamm-${jamm.version}.jar" />
<jvmarg value="-ea"/>
<jvmarg value="-Xss256k"/>
<jvmarg value="-Dcassandra.memtable_row_overhead_computation_step=100"/>
<jvmarg value="-Dcassandra.test.use_prepared=${cassandra.test.use_prepared}"/>
<jvmarg value="-Dcassandra.skip_sync=true" />
<classpath>
- <path refid="cassandra.classpath" />
+ <path refid="cassandra.classpath.test" />
<pathelement location="${test.classes}"/>
<pathelement location="${test.conf}"/>
<fileset dir="${test.lib}">
@@ -1557,14 +1712,14 @@
<formatter type="brief" usefile="false"/>
<jvmarg value="-Dstorage-config=${test.conf}"/>
<jvmarg value="-Djava.awt.headless=true"/>
- <jvmarg value="-javaagent:${basedir}/lib/jamm-0.3.0.jar" />
+ <jvmarg value="-javaagent:${basedir}/lib/jamm-${jamm.version}.jar" />
<jvmarg value="-ea"/>
<jvmarg value="-Xss256k"/>
<jvmarg value="-Dcassandra.test.use_prepared=${cassandra.test.use_prepared}"/>
<jvmarg value="-Dcassandra.memtable_row_overhead_computation_step=100"/>
<jvmarg value="-Dcassandra.skip_sync=true" />
<classpath>
- <path refid="cassandra.classpath" />
+ <path refid="cassandra.classpath.test" />
<pathelement location="${test.classes}"/>
<pathelement location="${test.conf}"/>
<fileset dir="${test.lib}">
@@ -1654,15 +1809,12 @@
</java>
</target>
- <target name="javadoc" depends="init" description="Create javadoc" unless="no-javadoc">
+ <target name="javadoc" depends="build" description="Create javadoc" unless="no-javadoc">
<create-javadoc destdir="${javadoc.dir}">
<filesets>
- <fileset dir="${build.src.java}" defaultexcludes="yes">
- <include name="org/apache/**/*.java"/>
- </fileset>
- <fileset dir="${interface.thrift.gen-java}" defaultexcludes="yes">
- <include name="org/apache/**/*.java"/>
- </fileset>
+ <fileset dir="${build.src.java}" defaultexcludes="yes">
+ <include name="org/apache/**/*.java"/>
+ </fileset>
</filesets>
</create-javadoc>
</target>
@@ -1848,7 +2000,7 @@
<target name="list-jvm-dtests" depends="build-test">
<java classname="org.apache.cassandra.distributed.test.TestLocator" fork="no">
<classpath>
- <path refid="cassandra.classpath" />
+ <path refid="cassandra.classpath.test" />
<pathelement location="${test.classes}"/>
<pathelement location="${test.conf}"/>
<fileset dir="${test.lib}">
@@ -1865,6 +2017,20 @@
<delete file="${test.distributed.listfile}"/>
</target>
+ <!-- Build a self-contained jar for e.g. remote execution; not currently used for running burn tests with this build script -->
+ <target name="burn-test-jar" depends="build-test, build" description="Create dtest-compatible jar, including all dependencies">
+ <jar jarfile="${build.dir}/burntest.jar">
+ <zipgroupfileset dir="${build.lib}" includes="*.jar" excludes="META-INF/*.SF"/>
+ <fileset dir="${build.classes.main}"/>
+ <fileset dir="${test.classes}"/>
+ <fileset dir="${test.conf}" excludes="logback*.xml"/>
+ <fileset dir="${basedir}/conf" includes="logback*.xml"/>
+ <zipgroupfileset dir="${build.dir.lib}/jars">
+ <include name="junit*.jar"/>
+ </zipgroupfileset>
+ </jar>
+ </target>
+
<target name="dtest-jar" depends="build-test, build" description="Create dtest-compatible jar, including all dependencies">
<jar jarfile="${build.dir}/dtest-${base.version}.jar">
<zipgroupfileset dir="${build.lib}" includes="*.jar" excludes="META-INF/*.SF"/>
@@ -1934,7 +2100,7 @@
fork="true"
failonerror="true">
<classpath>
- <path refid="cassandra.classpath" />
+ <path refid="cassandra.classpath.test" />
<pathelement location="${test.classes}"/>
<pathelement location="${test.conf}"/>
<fileset dir="${test.lib}">
@@ -1970,13 +2136,31 @@
</java>
</target>
+ <target name="_maybe_update_idea_to_java11" if="java.version.11">
+ <replace file="${eclipse.project.name}.iml" token="JDK_1_8" value="JDK_11"/>
+ <replace file=".idea/misc.xml" token="JDK_1_8" value="JDK_11"/>
+ <replace file=".idea/misc.xml" token="1.8" value="11"/>
+ <replaceregexp file=".idea/workspace.xml"
+ match="name="VM_PARAMETERS" value="(.*)""
+ replace="name="VM_PARAMETERS" value="\1 ${java11-jvmargs}""
+ byline="true"/>
+
+ <echo file=".idea/compiler.xml"><![CDATA[<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+ <component name="JavacSettings">
+ <option name="ADDITIONAL_OPTIONS_STRING" value="--add-exports java.rmi/sun.rmi.registry=ALL-UNNAMED" />
+ </component>
+</project>]]></echo>
+ </target>
+
<!-- Generate IDEA project description files -->
<target name="generate-idea-files" depends="build-test" description="Generate IDEA files">
<mkdir dir=".idea"/>
<mkdir dir=".idea/libraries"/>
- <copy todir=".idea">
+ <copy todir=".idea" overwrite="true">
<fileset dir="ide/idea"/>
</copy>
+ <replace file=".idea/workspace.xml" token="trunk" value="${eclipse.project.name}"/>
<copy tofile="${eclipse.project.name}.iml" file="ide/idea-iml-file.xml"/>
<echo file=".idea/.name">Apache Cassandra ${eclipse.project.name}</echo>
<echo file=".idea/modules.xml"><![CDATA[<?xml version="1.0" encoding="UTF-8"?>
@@ -1987,6 +2171,7 @@
</modules>
</component>
</project>]]></echo>
+ <antcall target="_maybe_update_idea_to_java11"/>
</target>
<!-- Generate Eclipse project description files -->
@@ -2013,16 +2198,16 @@
<classpathentry kind="src" path="src/resources"/>
<classpathentry kind="src" path="src/gen-java"/>
<classpathentry kind="src" path="conf" including="hotspot_compiler"/>
- <classpathentry kind="src" path="interface/thrift/gen-java"/>
<classpathentry kind="src" output="build/test/classes" path="test/unit"/>
<classpathentry kind="src" output="build/test/classes" path="test/long"/>
<classpathentry kind="src" output="build/test/classes" path="test/distributed"/>
<classpathentry kind="src" output="build/test/classes" path="test/resources" />
<classpathentry kind="src" path="tools/stress/src"/>
+ <classpathentry kind="src" path="tools/fqltool/src"/>
<classpathentry kind="src" output="build/test/stress-classes" path="tools/stress/test/unit" />
+ <classpathentry kind="src" output="build/test/fqltool-classes" path="tools/fqltool/test/unit" />
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
<classpathentry kind="output" path="build/classes/eclipse"/>
- <classpathentry kind="lib" path="build/classes/thrift" sourcepath="interface/thrift/gen-java/"/>
<classpathentry kind="lib" path="test/conf"/>
<classpathentry kind="lib" path="${java.home}/../lib/tools.jar"/>
]]>
@@ -2080,7 +2265,8 @@
</target>
- <target name="eclipse-warnings" depends="build" description="Run eclipse compiler code analysis">
+ <!-- ECJ 4.6.1 in standalone mode does not work with JPMS, so we skip this target for Java 11 -->
+ <target name="eclipse-warnings" depends="build" description="Run eclipse compiler code analysis" if="java.version.8">
<property name="ecj.log.dir" value="${build.dir}/ecj" />
<property name="ecj.warnings.file" value="${ecj.log.dir}/eclipse_compiler_checks.txt"/>
<mkdir dir="${ecj.log.dir}" />
@@ -2122,16 +2308,6 @@
file="${build.dir}/${final.name}-parent.pom"
packaging="pom"/>
- <!-- the cassandra-thrift jar -->
- <install pomFile="${build.dir}/${ant.project.name}-thrift-${version}.pom"
- file="${build.dir}/${ant.project.name}-thrift-${version}.jar"/>
- <install pomFile="${build.dir}/${ant.project.name}-thrift-${version}.pom"
- file="${build.dir}/${ant.project.name}-thrift-${version}-sources.jar"
- classifier="sources"/>
- <install pomFile="${build.dir}/${ant.project.name}-thrift-${version}.pom"
- file="${build.dir}/${ant.project.name}-thrift-${version}-javadoc.jar"
- classifier="javadoc"/>
-
<!-- the cassandra-all jar -->
<install pomFile="${build.dir}/${final.name}.pom"
file="${build.dir}/${final.name}.jar"/>
@@ -2146,7 +2322,6 @@
<!-- Publish artifacts to remote Maven repository -->
<target name="publish"
depends="mvn-install,artifacts"
- if="release"
description="Publishes the artifacts to the Maven repository">
<!-- the parent -->
@@ -2154,16 +2329,6 @@
file="${build.dir}/${final.name}-parent.pom"
packaging="pom"/>
- <!-- the cassandra-thrift jar -->
- <deploy pomFile="${build.dir}/${ant.project.name}-thrift-${version}.pom"
- file="${build.dir}/${ant.project.name}-thrift-${version}.jar"/>
- <deploy pomFile="${build.dir}/${ant.project.name}-thrift-${version}.pom"
- file="${build.dir}/${ant.project.name}-thrift-${version}-sources.jar"
- classifier="sources"/>
- <deploy pomFile="${build.dir}/${ant.project.name}-thrift-${version}.pom"
- file="${build.dir}/${ant.project.name}-thrift-${version}-javadoc.jar"
- classifier="javadoc"/>
-
<!-- the cassandra-all jar -->
<deploy pomFile="${build.dir}/${final.name}.pom"
file="${build.dir}/${final.name}.jar"/>
diff --git a/conf/cassandra-env.ps1 b/conf/cassandra-env.ps1
index c78a3fc..8ba8a5a 100644
--- a/conf/cassandra-env.ps1
+++ b/conf/cassandra-env.ps1
@@ -47,7 +47,7 @@
}
# Add build/classes/main so it works in development
- $cp = $cp + ";" + """$env:CASSANDRA_HOME\build\classes\main"";""$env:CASSANDRA_HOME\build\classes\thrift"""
+ $cp = $cp + ";" + """$env:CASSANDRA_HOME\build\classes\main"""
$env:CLASSPATH=$cp
}
@@ -380,11 +380,7 @@
$env:JVM_OPTS = "$env:JVM_OPTS -XX:CompileCommandFile=""$env:CASSANDRA_CONF\hotspot_compiler"""
# add the jamm javaagent
- if (($env:JVM_VENDOR -ne "OpenJDK") -or ($env:JVM_VERSION.CompareTo("1.6.0") -eq 1) -or
- (($env:JVM_VERSION -eq "1.6.0") -and ($env:JVM_PATCH_VERSION.CompareTo("22") -eq 1)))
- {
- $env:JVM_OPTS = "$env:JVM_OPTS -javaagent:""$env:CASSANDRA_HOME\lib\jamm-0.3.0.jar"""
- }
+ $env:JVM_OPTS = "$env:JVM_OPTS -javaagent:""$env:CASSANDRA_HOME\lib\jamm-0.3.2.jar"""
# set jvm HeapDumpPath with CASSANDRA_HEAPDUMP_DIR
if ($env:CASSANDRA_HEAPDUMP_DIR)
@@ -403,9 +399,10 @@
# print an heap histogram on OutOfMemoryError
# $env:JVM_OPTS="$env:JVM_OPTS -Dcassandra.printHeapHistogramOnOutOfMemoryError=true"
- if ($env:JVM_VERSION.CompareTo("1.8.0") -eq -1 -or [convert]::ToInt32($env:JVM_PATCH_VERSION) -lt 40)
+ $env:JAVA_VERSION=11
+ if ($env:JVM_VERSION.CompareTo("1.8.0") -eq -1 -or [convert]::ToInt32($env:JVM_PATCH_VERSION) -lt 151)
{
- echo "Cassandra 3.0 and later require Java 8u40 or later."
+ echo "Cassandra 4.0 requires either Java 8 (update 151 or newer) or Java 11 (or newer). Java $env:JVM_VERSION is not supported."
exit
}
diff --git a/conf/cassandra-env.sh b/conf/cassandra-env.sh
index a640847..a3e51e6 100644
--- a/conf/cassandra-env.sh
+++ b/conf/cassandra-env.sh
@@ -86,60 +86,32 @@
fi
}
-# Determine the sort of JVM we'll be running on.
-java_ver_output=`"${JAVA:-java}" -version 2>&1`
-jvmver=`echo "$java_ver_output" | grep '[openjdk|java] version' | awk -F'"' 'NR==1 {print $2}' | cut -d\- -f1`
-JVM_VERSION=${jvmver%_*}
-JVM_PATCH_VERSION=${jvmver#*_}
-
-if [ "$JVM_VERSION" \< "1.8" ] ; then
- echo "Cassandra 3.0 and later require Java 8u40 or later."
- exit 1;
-fi
-
-if [ "$JVM_VERSION" \< "1.8" ] && [ "$JVM_PATCH_VERSION" -lt 40 ] ; then
- echo "Cassandra 3.0 and later require Java 8u40 or later."
- exit 1;
-fi
-
-jvm=`echo "$java_ver_output" | grep -A 1 '[openjdk|java] version' | awk 'NR==2 {print $1}'`
-case "$jvm" in
- OpenJDK)
- JVM_VENDOR=OpenJDK
- # this will be "64-Bit" or "32-Bit"
- JVM_ARCH=`echo "$java_ver_output" | awk 'NR==3 {print $2}'`
- ;;
- "Java(TM)")
- JVM_VENDOR=Oracle
- # this will be "64-Bit" or "32-Bit"
- JVM_ARCH=`echo "$java_ver_output" | awk 'NR==3 {print $3}'`
- ;;
- *)
- # Help fill in other JVM values
- JVM_VENDOR=other
- JVM_ARCH=unknown
- ;;
-esac
-
# Sets the path where logback and GC logs are written.
if [ "x$CASSANDRA_LOG_DIR" = "x" ] ; then
CASSANDRA_LOG_DIR="$CASSANDRA_HOME/logs"
fi
#GC log path has to be defined here because it needs to access CASSANDRA_HOME
-JVM_OPTS="$JVM_OPTS -Xloggc:${CASSANDRA_LOG_DIR}/gc.log"
+if [ $JAVA_VERSION -ge 11 ] ; then
+ # See description of https://bugs.openjdk.java.net/browse/JDK-8046148 for details about the syntax
+ # The following is the equivalent to -XX:+PrintGCDetails -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=10M
+ echo "$JVM_OPTS" | grep -q "^-[X]log:gc"
+ if [ "$?" = "1" ] ; then # [X] to prevent ccm from replacing this line
+ # only add -Xlog:gc if it's not mentioned in jvm-server.options file
+ mkdir -p ${CASSANDRA_LOG_DIR}
+ JVM_OPTS="$JVM_OPTS -Xlog:gc=info,heap*=trace,age*=debug,safepoint=info,promotion*=trace:file=${CASSANDRA_LOG_DIR}/gc.log:time,uptime,pid,tid,level:filecount=10,filesize=10485760"
+ fi
+else
+ # Java 8
+ echo "$JVM_OPTS" | grep -q "^-[X]loggc"
+ if [ "$?" = "1" ] ; then # [X] to prevent ccm from replacing this line
+ # only add -Xlog:gc if it's not mentioned in jvm-server.options file
+ mkdir -p ${CASSANDRA_LOG_DIR}
+ JVM_OPTS="$JVM_OPTS -Xloggc:${CASSANDRA_LOG_DIR}/gc.log"
+ fi
+fi
-# Here we create the arguments that will get passed to the jvm when
-# starting cassandra.
-
-# Read user-defined JVM options from jvm.options file
-JVM_OPTS_FILE=$CASSANDRA_CONF/jvm.options
-for opt in `grep "^-" $JVM_OPTS_FILE`
-do
- JVM_OPTS="$JVM_OPTS $opt"
-done
-
-# Check what parameters were defined on jvm.options file to avoid conflicts
+# Check what parameters were defined on jvm-server.options file to avoid conflicts
echo $JVM_OPTS | grep -q Xmn
DEFINED_XMN=$?
echo $JVM_OPTS | grep -q Xmx
@@ -148,7 +120,7 @@
DEFINED_XMS=$?
echo $JVM_OPTS | grep -q UseConcMarkSweepGC
USING_CMS=$?
-echo $JVM_OPTS | grep -q UseG1GC
+echo $JVM_OPTS | grep -q +UseG1GC
USING_G1=$?
# Override these to set the amount of memory to allocate to the JVM at
@@ -184,26 +156,33 @@
export MALLOC_ARENA_MAX=4
fi
-# We only set -Xms and -Xmx if they were not defined on jvm.options file
+# We only set -Xms and -Xmx if they were not defined on jvm-server.options file
# If defined, both Xmx and Xms should be defined together.
if [ $DEFINED_XMX -ne 0 ] && [ $DEFINED_XMS -ne 0 ]; then
JVM_OPTS="$JVM_OPTS -Xms${MAX_HEAP_SIZE}"
JVM_OPTS="$JVM_OPTS -Xmx${MAX_HEAP_SIZE}"
elif [ $DEFINED_XMX -ne 0 ] || [ $DEFINED_XMS -ne 0 ]; then
- echo "Please set or unset -Xmx and -Xms flags in pairs on jvm.options file."
+ echo "Please set or unset -Xmx and -Xms flags in pairs on jvm-server.options file."
exit 1
fi
-# We only set -Xmn flag if it was not defined in jvm.options file
+# We only set -Xmn flag if it was not defined in jvm-server.options file
# and if the CMS GC is being used
# If defined, both Xmn and Xmx should be defined together.
if [ $DEFINED_XMN -eq 0 ] && [ $DEFINED_XMX -ne 0 ]; then
- echo "Please set or unset -Xmx and -Xmn flags in pairs on jvm.options file."
+ echo "Please set or unset -Xmx and -Xmn flags in pairs on jvm-server.options file."
exit 1
elif [ $DEFINED_XMN -ne 0 ] && [ $USING_CMS -eq 0 ]; then
JVM_OPTS="$JVM_OPTS -Xmn${HEAP_NEWSIZE}"
fi
+# We fail to start if -Xmn is used with G1 GC is being used
+# See comments for -Xmn in jvm-server.options
+if [ $DEFINED_XMN -eq 0 ] && [ $USING_G1 -eq 0 ]; then
+ echo "It is not recommended to set -Xmn with the G1 garbage collector. See comments for -Xmn in jvm-server.options for details."
+ exit 1
+fi
+
if [ "$JVM_ARCH" = "64-Bit" ] && [ $USING_CMS -eq 0 ]; then
JVM_OPTS="$JVM_OPTS -XX:+UseCondCardMark"
fi
@@ -212,7 +191,7 @@
JVM_OPTS="$JVM_OPTS -XX:CompileCommandFile=$CASSANDRA_CONF/hotspot_compiler"
# add the jamm javaagent
-JVM_OPTS="$JVM_OPTS -javaagent:$CASSANDRA_HOME/lib/jamm-0.3.0.jar"
+JVM_OPTS="$JVM_OPTS -javaagent:$CASSANDRA_HOME/lib/jamm-0.3.2.jar"
# set jvm HeapDumpPath with CASSANDRA_HEAPDUMP_DIR
if [ "x$CASSANDRA_HEAPDUMP_DIR" != "x" ]; then
@@ -296,17 +275,33 @@
# To use mx4j, an HTML interface for JMX, add mx4j-tools.jar to the lib/
# directory.
-# See http://cassandra.apache.org/doc/3.11/operating/metrics.html#jmx
+# See http://cassandra.apache.org/doc/latest/operating/metrics.html#jmx
# By default mx4j listens on 0.0.0.0:8081. Uncomment the following lines
# to control its listen address and port.
-#MX4J_ADDRESS="-Dmx4jaddress=127.0.0.1"
-#MX4J_PORT="-Dmx4jport=8081"
+#MX4J_ADDRESS="127.0.0.1"
+#MX4J_PORT="8081"
# Cassandra uses SIGAR to capture OS metrics CASSANDRA-7838
# for SIGAR we have to set the java.library.path
# to the location of the native libraries.
JVM_OPTS="$JVM_OPTS -Djava.library.path=$CASSANDRA_HOME/lib/sigar-bin"
-JVM_OPTS="$JVM_OPTS $MX4J_ADDRESS"
-JVM_OPTS="$JVM_OPTS $MX4J_PORT"
+if [ "x$MX4J_ADDRESS" != "x" ]; then
+ if [[ "$MX4J_ADDRESS" == \-Dmx4jaddress* ]]; then
+ # Backward compatible with the older style #13578
+ JVM_OPTS="$JVM_OPTS $MX4J_ADDRESS"
+ else
+ JVM_OPTS="$JVM_OPTS -Dmx4jaddress=$MX4J_ADDRESS"
+ fi
+fi
+if [ "x$MX4J_PORT" != "x" ]; then
+ if [[ "$MX4J_PORT" == \-Dmx4jport* ]]; then
+ # Backward compatible with the older style #13578
+ JVM_OPTS="$JVM_OPTS $MX4J_PORT"
+ else
+ JVM_OPTS="$JVM_OPTS -Dmx4jport=$MX4J_PORT"
+ fi
+fi
+
JVM_OPTS="$JVM_OPTS $JVM_EXTRA_OPTS"
+
diff --git a/conf/cassandra-rackdc.properties b/conf/cassandra-rackdc.properties
index 2ea6043..cc472b4 100644
--- a/conf/cassandra-rackdc.properties
+++ b/conf/cassandra-rackdc.properties
@@ -25,3 +25,15 @@
# Uncomment the following line to make this snitch prefer the internal ip when possible, as the Ec2MultiRegionSnitch does.
# prefer_local=true
+
+# Datacenter and rack naming convention used by the Ec2Snitch and Ec2MultiRegionSnitch.
+# Options are:
+# legacy : datacenter name is the part of the availability zone name preceding the last "-"
+# when the zone ends in -1 and includes the number if not -1. Rack is the portion of
+# the availability zone name following the last "-".
+# Examples: us-west-1a => dc: us-west, rack: 1a; us-west-2b => dc: us-west-2, rack: 2b;
+# YOU MUST USE THIS VALUE IF YOU ARE UPGRADING A PRE-4.0 CLUSTER
+# standard : Default value. datacenter name is the standard AWS region name, including the number.
+# rack name is the region plus the availability zone letter.
+# Examples: us-west-1a => dc: us-west-1, rack: us-west-1a; us-west-2b => dc: us-west-2, rack: us-west-2b;
+# ec2_naming_scheme=standard
diff --git a/conf/cassandra.yaml b/conf/cassandra.yaml
index 9182008..4bd72cf 100644
--- a/conf/cassandra.yaml
+++ b/conf/cassandra.yaml
@@ -1,7 +1,7 @@
# Cassandra storage config YAML
# NOTE:
-# See http://wiki.apache.org/cassandra/StorageConfiguration for
+# See https://cassandra.apache.org/doc/latest/configuration/ for
# full explanations of configuration directives
# /NOTE
@@ -20,28 +20,31 @@
# Specifying initial_token will override this setting on the node's initial start,
# on subsequent starts, this setting will apply even if initial token is set.
#
-# If you already have a cluster with 1 token per node, and wish to migrate to
-# multiple tokens per node, see http://wiki.apache.org/cassandra/Operations
num_tokens: 256
# Triggers automatic allocation of num_tokens tokens for this node. The allocation
# algorithm attempts to choose tokens in a way that optimizes replicated load over
-# the nodes in the datacenter for the replication strategy used by the specified
-# keyspace.
+# the nodes in the datacenter for the replica factor.
#
# The load assigned to each node will be close to proportional to its number of
# vnodes.
#
# Only supported with the Murmur3Partitioner.
+
+# Replica factor is determined via the replication strategy used by the specified
+# keyspace.
# allocate_tokens_for_keyspace: KEYSPACE
+# Replica factor is explicitly set, regardless of keyspace or datacenter.
+# This is the replica factor within the datacenter, like NTS.
+# allocate_tokens_for_local_replication_factor: 3
+
# initial_token allows you to specify tokens manually. While you can use it with
# vnodes (num_tokens > 1, above) -- in which case you should provide a
# comma-separated list -- it's primarily used when adding nodes to legacy clusters
# that do not have vnodes enabled.
# initial_token:
-# See http://wiki.apache.org/cassandra/HintedHandoff
# May either be "true" or "false" to enable globally
hinted_handoff_enabled: true
@@ -122,6 +125,16 @@
# increase system_auth keyspace replication factor if you use this role manager.
role_manager: CassandraRoleManager
+# Network authorization backend, implementing INetworkAuthorizer; used to restrict user
+# access to certain DCs
+# Out of the box, Cassandra provides org.apache.cassandra.auth.{AllowAllNetworkAuthorizer,
+# CassandraNetworkAuthorizer}.
+#
+# - AllowAllNetworkAuthorizer allows access to any DC to any user - set it to disable authorization.
+# - CassandraNetworkAuthorizer stores permissions in system_auth.network_permissions table. Please
+# increase system_auth keyspace replication factor if you use this authorizer.
+network_authorizer: AllowAllNetworkAuthorizer
+
# Validity period for roles cache (fetching granted roles can be an expensive
# operation depending on the role manager, CassandraRoleManager is one example)
# Granted roles are cached for authenticated sessions in AuthenticatedUser and
@@ -172,20 +185,20 @@
# credentials_update_interval_in_ms: 2000
# The partitioner is responsible for distributing groups of rows (by
-# partition key) across nodes in the cluster. You should leave this
-# alone for new clusters. The partitioner can NOT be changed without
-# reloading all data, so when upgrading you should set this to the
-# same partitioner you were already using.
+# partition key) across nodes in the cluster. The partitioner can NOT be
+# changed without reloading all data. If you are adding nodes or upgrading,
+# you should set this to the same partitioner that you are currently using.
#
-# Besides Murmur3Partitioner, partitioners included for backwards
-# compatibility include RandomPartitioner, ByteOrderedPartitioner, and
-# OrderPreservingPartitioner.
+# The default partitioner is the Murmur3Partitioner. Older partitioners
+# such as the RandomPartitioner, ByteOrderedPartitioner, and
+# OrderPreservingPartitioner have been included for backward compatibility only.
+# For new clusters, you should NOT change this value.
#
partitioner: org.apache.cassandra.dht.Murmur3Partitioner
-# Directories where Cassandra should store data on disk. Cassandra
-# will spread data evenly across them, subject to the granularity of
-# the configured compaction strategy.
+# Directories where Cassandra should store data on disk. If multiple
+# directories are specified, Cassandra will spread data evenly across
+# them by partitioning the token ranges.
# If not set, the default directory is $CASSANDRA_HOME/data/data.
# data_file_directories:
# - /var/lib/cassandra/data
@@ -232,10 +245,10 @@
# Policy for commit disk failures:
#
# die
-# shut down gossip and Thrift and kill the JVM, so the node can be replaced.
+# shut down the node and kill the JVM, so the node can be replaced.
#
# stop
-# shut down gossip and Thrift, leaving the node effectively dead, but
+# shut down the node, leaving the node effectively dead, but
# can still be inspected via JMX.
#
# stop_commit
@@ -265,15 +278,6 @@
# Default value ("auto") is 1/256th of the heap or 10MB, whichever is greater
prepared_statements_cache_size_mb:
-# Maximum size of the Thrift prepared statement cache
-#
-# If you do not use Thrift at all, it is safe to leave this value at "auto".
-#
-# See description of 'prepared_statements_cache_size_mb' above for more information.
-#
-# Default value ("auto") is 1/256th of the heap or 10MB, whichever is greater
-thrift_prepared_statements_cache_size_mb:
-
# Maximum size of the key cache in memory.
#
# Each key cache hit saves 1 seek and each row cache hit saves 2 seeks at the
@@ -367,24 +371,31 @@
# If not set, the default directory is $CASSANDRA_HOME/data/saved_caches.
# saved_caches_directory: /var/lib/cassandra/saved_caches
-# commitlog_sync may be either "periodic" or "batch."
+# commitlog_sync may be either "periodic", "group", or "batch."
#
# When in batch mode, Cassandra won't ack writes until the commit log
-# has been fsynced to disk. It will wait
-# commitlog_sync_batch_window_in_ms milliseconds between fsyncs.
-# This window should be kept short because the writer threads will
-# be unable to do extra work while waiting. (You may need to increase
-# concurrent_writes for the same reason.)
+# has been flushed to disk. Each incoming write will trigger the flush task.
+# commitlog_sync_batch_window_in_ms is a deprecated value. Previously it had
+# almost no value, and is being removed.
#
-# commitlog_sync: batch
# commitlog_sync_batch_window_in_ms: 2
#
-# the other option is "periodic" where writes may be acked immediately
+# group mode is similar to batch mode, where Cassandra will not ack writes
+# until the commit log has been flushed to disk. The difference is group
+# mode will wait up to commitlog_sync_group_window_in_ms between flushes.
+#
+# commitlog_sync_group_window_in_ms: 1000
+#
+# the default option is "periodic" where writes may be acked immediately
# and the CommitLog is simply synced every commitlog_sync_period_in_ms
# milliseconds.
commitlog_sync: periodic
commitlog_sync_period_in_ms: 10000
+# When in periodic commitlog mode, the number of milliseconds to block writes
+# while waiting for a slow disk flush to complete.
+# periodic_commitlog_sync_lag_block_in_ms:
+
# The size of the individual commitlog file segments. A commitlog
# segment may be archived, deleted, or recycled once all the data
# in it (potentially from each columnfamily in the system) has been
@@ -411,6 +422,21 @@
# parameters:
# -
+# Compression to apply to SSTables as they flush for compressed tables.
+# Note that tables without compression enabled do not respect this flag.
+#
+# As high ratio compressors like LZ4HC, Zstd, and Deflate can potentially
+# block flushes for too long, the default is to flush with a known fast
+# compressor in those cases. Options are:
+#
+# none : Flush without compressing blocks but while still doing checksums.
+# fast : Flush with a fast compressor. If the table is already using a
+# fast compressor that compressor is used.
+# table: Always flush with the same compressor that the table uses. This
+# was the pre 4.0 behavior.
+#
+# flush_compression: fast
+
# any class that implements the SeedProvider interface and has a
# constructor that takes a Map<String, String> of parameters will do.
seed_provider:
@@ -422,7 +448,7 @@
parameters:
# seeds is actually a comma-delimited list of addresses.
# Ex: "<ip1>,<ip2>,<ip3>"
- - seeds: "127.0.0.1"
+ - seeds: "127.0.0.1:7000"
# For workloads with more data than can fit in memory, Cassandra's
# bottleneck will be reads that need to fetch data from
@@ -498,18 +524,16 @@
# off heap objects
memtable_allocation_type: heap_buffers
-# Limits the maximum Merkle tree depth to avoid consuming too much
-# memory during repairs.
-#
-# The default setting of 18 generates trees of maximum size around
-# 50 MiB / tree. If you are running out of memory during repairs consider
-# lowering this to 15 (~6 MiB / tree) or lower, but try not to lower it
-# too much past that or you will lose too much resolution and stream
-# too much redundant data during repair. Cannot be set lower than 10.
+# Limit memory usage for Merkle tree calculations during repairs. The default
+# is 1/16th of the available heap. The main tradeoff is that smaller trees
+# have less resolution, which can lead to over-streaming data. If you see heap
+# pressure during repairs, consider lowering this, but you cannot go below
+# one megabyte. If you see lots of over-streaming, consider raising
+# this or using subrange repair.
#
# For more details see https://issues.apache.org/jira/browse/CASSANDRA-14096.
#
-# repair_session_max_tree_depth: 18
+# repair_session_space_in_mb:
# Total space to use for commit logs on disk.
#
@@ -592,9 +616,10 @@
# For security reasons, you should not expose this port to the internet. Firewall it if needed.
storage_port: 7000
-# SSL port, for encrypted communication. Unused unless enabled in
-# encryption_options
-# For security reasons, you should not expose this port to the internet. Firewall it if needed.
+# SSL port, for legacy encrypted communication. This property is unused unless enabled in
+# server_encryption_options (see below). As of cassandra 4.0, this property is deprecated
+# as a single port can be used for either/both secure and insecure connections.
+# For security reasons, you should not expose this port to the internet. Firewall it if needed.
ssl_storage_port: 7001
# Address or interface to bind to and tell other Cassandra nodes to connect to.
@@ -605,7 +630,8 @@
# Leaving it blank leaves it up to InetAddress.getLocalHost(). This
# will always do the Right Thing _if_ the node is properly configured
# (hostname, name resolution, etc), and the Right Thing is to use the
-# address associated with the hostname (it might not be).
+# address associated with the hostname (it might not be). If unresolvable
+# it will fall back to InetAddress.getLoopbackAddress(), which is wrong for production systems.
#
# Setting listen_address to 0.0.0.0 is always wrong.
#
@@ -638,8 +664,7 @@
# internode_authenticator: org.apache.cassandra.auth.AllowAllInternodeAuthenticator
# Whether to start the native transport server.
-# Please note that the address on which the native transport is bound is the
-# same as the rpc_address. The port however is different and specified below.
+# The address on which the native transport is bound is defined by rpc_address.
start_native_transport: true
# port for the CQL native transport to listen for clients on
# For security reasons, you should not expose this port to the internet. Firewall it if needed.
@@ -652,10 +677,8 @@
# from native_transport_port will use encryption for native_transport_port_ssl while
# keeping native_transport_port unencrypted.
# native_transport_port_ssl: 9142
-# The maximum threads for handling requests when the native transport is used.
-# This is similar to rpc_max_threads though the default differs slightly (and
-# there is no native_transport_min_threads, idle threads will always be stopped
-# after 30 seconds).
+# The maximum threads for handling requests (note that idle threads are stopped
+# after 30 seconds so there is not corresponding minimum setting).
# native_transport_max_threads: 128
#
# The maximum size of allowed frame. Frame (requests) larger than this will
@@ -663,6 +686,10 @@
# you may want to adjust max_value_size_in_mb accordingly. This should be positive and less than 2048.
# native_transport_max_frame_size_in_mb: 256
+# If checksumming is enabled as a protocol option, denotes the size of the chunks into which frame
+# are bodies will be broken and checksummed.
+# native_transport_frame_block_size_in_kb: 32
+
# The maximum number of concurrent client connections.
# The default is -1, which means unlimited.
# native_transport_max_concurrent_connections: -1
@@ -671,11 +698,21 @@
# The default is -1, which means unlimited.
# native_transport_max_concurrent_connections_per_ip: -1
-# Whether to start the thrift rpc server.
-start_rpc: false
+# Controls whether Cassandra honors older, yet currently supported, protocol versions.
+# The default is true, which means all supported protocols will be honored.
+native_transport_allow_older_protocols: true
-# The address or interface to bind the Thrift RPC service and native transport
-# server to.
+# Controls when idle client connections are closed. Idle connections are ones that had neither reads
+# nor writes for a time period.
+#
+# Clients may implement heartbeats by sending OPTIONS native protocol message after a timeout, which
+# will reset idle timeout timer on the server side. To close idle client connections, corresponding
+# values for heartbeat intervals have to be set on the client side.
+#
+# Idle connection timeouts are disabled by default.
+# native_transport_idle_timeout_in_ms: 60000
+
+# The address or interface to bind the native transport server to.
#
# Set rpc_address OR rpc_interface, not both.
#
@@ -698,9 +735,6 @@
# ipv4. If there is only one address it will be selected regardless of ipv4/ipv6.
# rpc_interface_prefer_ipv6: false
-# port for Thrift to listen for clients on
-rpc_port: 9160
-
# RPC address to broadcast to drivers and other Cassandra nodes. This cannot
# be set to 0.0.0.0. If left blank, this will be set to the value of
# rpc_address. If rpc_address is set to 0.0.0.0, broadcast_rpc_address must
@@ -710,45 +744,6 @@
# enable or disable keepalive on rpc/native connections
rpc_keepalive: true
-# Cassandra provides two out-of-the-box options for the RPC Server:
-#
-# sync
-# One thread per thrift connection. For a very large number of clients, memory
-# will be your limiting factor. On a 64 bit JVM, 180KB is the minimum stack size
-# per thread, and that will correspond to your use of virtual memory (but physical memory
-# may be limited depending on use of stack space).
-#
-# hsha
-# Stands for "half synchronous, half asynchronous." All thrift clients are handled
-# asynchronously using a small number of threads that does not vary with the amount
-# of thrift clients (and thus scales well to many clients). The rpc requests are still
-# synchronous (one thread per active request). If hsha is selected then it is essential
-# that rpc_max_threads is changed from the default value of unlimited.
-#
-# The default is sync because on Windows hsha is about 30% slower. On Linux,
-# sync/hsha performance is about the same, with hsha of course using less memory.
-#
-# Alternatively, can provide your own RPC server by providing the fully-qualified class name
-# of an o.a.c.t.TServerFactory that can create an instance of it.
-rpc_server_type: sync
-
-# Uncomment rpc_min|max_thread to set request pool size limits.
-#
-# Regardless of your choice of RPC server (see above), the number of maximum requests in the
-# RPC thread pool dictates how many concurrent requests are possible (but if you are using the sync
-# RPC server, it also dictates the number of clients that can be connected at all).
-#
-# The default is unlimited and thus provides no protection against clients overwhelming the server. You are
-# encouraged to set a maximum that makes sense for you in production, but do keep in mind that
-# rpc_max_threads represents the maximum number of client requests this server may execute concurrently.
-#
-# rpc_min_threads: 16
-# rpc_max_threads: 2048
-
-# uncomment to set socket buffer sizes on rpc connections
-# rpc_send_buff_size_in_bytes:
-# rpc_recv_buff_size_in_bytes:
-
# Uncomment to set socket buffer size for internode communication
# Note that when setting this, the buffer size is limited by net.core.wmem_max
# and when not setting it it is defined by net.ipv4.tcp_wmem
@@ -765,9 +760,6 @@
# and when not setting it it is defined by net.ipv4.tcp_wmem
# internode_recv_buff_size_in_bytes:
-# Frame size for thrift (maximum message length).
-thrift_framed_transport_size_in_mb: 15
-
# Set to true to have Cassandra create a hard link to each sstable
# flushed or streamed locally in a backups/ subdirectory of the
# keyspace data. Removing these links is the operator's
@@ -823,13 +815,26 @@
# to the number of cores.
#concurrent_compactors: 1
+# Number of simultaneous repair validations to allow. If not set or set to
+# a value less than 1, it defaults to the value of concurrent_compactors.
+# To set a value greeater than concurrent_compactors at startup, the system
+# property cassandra.allow_unlimited_concurrent_validations must be set to
+# true. To dynamically resize to a value > concurrent_compactors on a running
+# node, first call the bypassConcurrentValidatorsLimit method on the
+# org.apache.cassandra.db:type=StorageService mbean
+# concurrent_validations: 0
+
+# Number of simultaneous materialized view builder tasks to allow.
+concurrent_materialized_view_builders: 1
+
# Throttles compaction to the given total throughput across the entire
# system. The faster you insert data, the faster you need to compact in
# order to keep the sstable count down, but in general, setting this to
# 16 to 32 times the rate you are inserting data is more than sufficient.
-# Setting this to 0 disables throttling. Note that this account for all types
-# of compaction, including validation compaction.
-compaction_throughput_mb_per_sec: 16
+# Setting this to 0 disables throttling. Note that this accounts for all types
+# of compaction, including validation compaction (building Merkle trees
+# for repairs).
+compaction_throughput_mb_per_sec: 64
# When compacting, the replacement sstable(s) can be opened before they
# are completely written, and used in place of the prior sstables for
@@ -837,6 +842,18 @@
# between the sstables, reducing page cache churn and keeping hot rows hot
sstable_preemptive_open_interval_in_mb: 50
+# When enabled, permits Cassandra to zero-copy stream entire eligible
+# SSTables between nodes, including every component.
+# This speeds up the network transfer significantly subject to
+# throttling specified by stream_throughput_outbound_megabits_per_sec.
+# Enabling this will reduce the GC pressure on sending and receiving node.
+# When unset, the default is enabled. While this feature tries to keep the
+# disks balanced, it cannot guarantee it. This feature will be automatically
+# disabled if internode encryption is enabled. Currently this can be used with
+# Leveled Compaction. Once CASSANDRA-14586 is fixed other compaction strategies
+# will benefit as well when used in combination with CASSANDRA-6696.
+# stream_entire_sstables: true
+
# Throttles all outbound streaming file transfers on this node to the
# given total throughput in Mbps. This is necessary because Cassandra does
# mostly sequential IO when streaming data during bootstrap or repair, which
@@ -851,24 +868,69 @@
# When unset, the default is 200 Mbps or 25 MB/s
# inter_dc_stream_throughput_outbound_megabits_per_sec: 200
-# How long the coordinator should wait for read operations to complete
+# How long the coordinator should wait for read operations to complete.
+# Lowest acceptable value is 10 ms.
read_request_timeout_in_ms: 5000
-# How long the coordinator should wait for seq or index scans to complete
+# How long the coordinator should wait for seq or index scans to complete.
+# Lowest acceptable value is 10 ms.
range_request_timeout_in_ms: 10000
-# How long the coordinator should wait for writes to complete
+# How long the coordinator should wait for writes to complete.
+# Lowest acceptable value is 10 ms.
write_request_timeout_in_ms: 2000
-# How long the coordinator should wait for counter writes to complete
+# How long the coordinator should wait for counter writes to complete.
+# Lowest acceptable value is 10 ms.
counter_write_request_timeout_in_ms: 5000
# How long a coordinator should continue to retry a CAS operation
-# that contends with other proposals for the same row
+# that contends with other proposals for the same row.
+# Lowest acceptable value is 10 ms.
cas_contention_timeout_in_ms: 1000
# How long the coordinator should wait for truncates to complete
# (This can be much longer, because unless auto_snapshot is disabled
# we need to flush first so we can snapshot before removing the data.)
+# Lowest acceptable value is 10 ms.
truncate_request_timeout_in_ms: 60000
-# The default timeout for other, miscellaneous operations
+# The default timeout for other, miscellaneous operations.
+# Lowest acceptable value is 10 ms.
request_timeout_in_ms: 10000
+# Defensive settings for protecting Cassandra from true network partitions.
+# See (CASSANDRA-14358) for details.
+#
+# The amount of time to wait for internode tcp connections to establish.
+# internode_tcp_connect_timeout_in_ms = 2000
+#
+# The amount of time unacknowledged data is allowed on a connection before we throw out the connection
+# Note this is only supported on Linux + epoll, and it appears to behave oddly above a setting of 30000
+# (it takes much longer than 30s) as of Linux 4.12. If you want something that high set this to 0
+# which picks up the OS default and configure the net.ipv4.tcp_retries2 sysctl to be ~8.
+# internode_tcp_user_timeout_in_ms = 30000
+
+# The maximum continuous period a connection may be unwritable in application space
+# internode_application_timeout_in_ms = 30000
+
+# Global, per-endpoint and per-connection limits imposed on messages queued for delivery to other nodes
+# and waiting to be processed on arrival from other nodes in the cluster. These limits are applied to the on-wire
+# size of the message being sent or received.
+#
+# The basic per-link limit is consumed in isolation before any endpoint or global limit is imposed.
+# Each node-pair has three links: urgent, small and large. So any given node may have a maximum of
+# N*3*(internode_application_send_queue_capacity_in_bytes+internode_application_receive_queue_capacity_in_bytes)
+# messages queued without any coordination between them although in practice, with token-aware routing, only RF*tokens
+# nodes should need to communicate with significant bandwidth.
+#
+# The per-endpoint limit is imposed on all messages exceeding the per-link limit, simultaneously with the global limit,
+# on all links to or from a single node in the cluster.
+# The global limit is imposed on all messages exceeding the per-link limit, simultaneously with the per-endpoint limit,
+# on all links to or from any node in the cluster.
+#
+# internode_application_send_queue_capacity_in_bytes: 4194304 #4MiB
+# internode_application_send_queue_reserve_endpoint_capacity_in_bytes: 134217728 #128MiB
+# internode_application_send_queue_reserve_global_capacity_in_bytes: 536870912 #512MiB
+# internode_application_receive_queue_capacity_in_bytes: 4194304 #4MiB
+# internode_application_receive_queue_reserve_endpoint_capacity_in_bytes: 134217728 #128MiB
+# internode_application_receive_queue_reserve_global_capacity_in_bytes: 536870912 #512MiB
+
+
# How long before a node logs slow queries. Select queries that take longer than
# this timeout to execute, will generate an aggregated log message, so that slow queries
# can be identified. Set this value to zero to disable slow query logging.
@@ -880,9 +942,9 @@
# under overload conditions we will waste that much extra time processing
# already-timed-out requests.
#
-# Warning: before enabling this property make sure to ntp is installed
-# and the times are synchronized between the nodes.
-cross_node_timeout: false
+# Warning: It is generally assumed that users have setup NTP on their clusters, and that clocks are modestly in sync,
+# since this is a requirement for general correctness of last write wins.
+#cross_node_timeout: true
# Set keep-alive period for streaming
# This node will send a keep-alive message periodically with this period.
@@ -892,6 +954,12 @@
# times out in 10 minutes by default
# streaming_keep_alive_period_in_secs: 300
+# Limit number of connections per host for streaming
+# Increase this when you notice that joins are CPU-bound rather that network
+# bound (for example a few nodes with big files).
+# streaming_connections_per_host: 1
+
+
# phi value that must be reached for a host to be marked down.
# most users should never need to adjust this.
# phi_convict_threshold: 8
@@ -967,7 +1035,7 @@
# controls how often to reset all host scores, allowing a bad host to
# possibly recover
dynamic_snitch_reset_interval_in_ms: 600000
-# if set greater than zero and read_repair_chance is < 1.0, this will allow
+# if set greater than zero, this will allow
# 'pinning' of replicas to hosts in order to increase cache capacity.
# The badness threshold will control how much worse the pinned host has to be
# before the dynamic snitch will prefer other replicas over it. This is
@@ -976,101 +1044,99 @@
# until the pinned host was 20% worse than the fastest.
dynamic_snitch_badness_threshold: 0.1
-# request_scheduler -- Set this to a class that implements
-# RequestScheduler, which will schedule incoming client requests
-# according to the specific policy. This is useful for multi-tenancy
-# with a single Cassandra cluster.
-# NOTE: This is specifically for requests from the client and does
-# not affect inter node communication.
-# org.apache.cassandra.scheduler.NoScheduler - No scheduling takes place
-# org.apache.cassandra.scheduler.RoundRobinScheduler - Round robin of
-# client requests to a node with a separate queue for each
-# request_scheduler_id. The scheduler is further customized by
-# request_scheduler_options as described below.
-request_scheduler: org.apache.cassandra.scheduler.NoScheduler
-
-# Scheduler Options vary based on the type of scheduler
+# Configure server-to-server internode encryption
#
-# NoScheduler
-# Has no options
-#
-# RoundRobin
-# throttle_limit
-# The throttle_limit is the number of in-flight
-# requests per client. Requests beyond
-# that limit are queued up until
-# running requests can complete.
-# The value of 80 here is twice the number of
-# concurrent_reads + concurrent_writes.
-# default_weight
-# default_weight is optional and allows for
-# overriding the default which is 1.
-# weights
-# Weights are optional and will default to 1 or the
-# overridden default_weight. The weight translates into how
-# many requests are handled during each turn of the
-# RoundRobin, based on the scheduler id.
-#
-# request_scheduler_options:
-# throttle_limit: 80
-# default_weight: 5
-# weights:
-# Keyspace1: 1
-# Keyspace2: 5
-
-# request_scheduler_id -- An identifier based on which to perform
-# the request scheduling. Currently the only valid option is keyspace.
-# request_scheduler_id: keyspace
-
-# Enable or disable inter-node encryption
-# JVM defaults for supported SSL socket protocols and cipher suites can
+# JVM and netty defaults for supported SSL socket protocols and cipher suites can
# be replaced using custom encryption options. This is not recommended
# unless you have policies in place that dictate certain settings, or
# need to disable vulnerable ciphers or protocols in case the JVM cannot
# be updated.
+#
# FIPS compliant settings can be configured at JVM level and should not
# involve changing encryption settings here:
# https://docs.oracle.com/javase/8/docs/technotes/guides/security/jsse/FIPS.html
-# *NOTE* No custom encryption options are enabled at the moment
-# The available internode options are : all, none, dc, rack
#
-# If set to dc cassandra will encrypt the traffic between the DCs
-# If set to rack cassandra will encrypt the traffic between the racks
+# **NOTE** this default configuration is an insecure configuration. If you need to
+# enable server-to-server encryption generate server keystores (and truststores for mutual
+# authentication) per:
+# http://download.oracle.com/javase/8/docs/technotes/guides/security/jsse/JSSERefGuide.html#CreateKeystore
+# Then perform the following configuration changes:
#
-# The passwords used in these options must match the passwords used when generating
-# the keystore and truststore. For instructions on generating these files, see:
-# http://download.oracle.com/javase/6/docs/technotes/guides/security/jsse/JSSERefGuide.html#CreateKeystore
+# Step 1: Set internode_encryption=<dc|rack|all> and explicitly set optional=true. Restart all nodes
#
+# Step 2: Set optional=false (or remove it) and if you generated truststores and want to use mutual
+# auth set require_client_auth=true. Restart all nodes
server_encryption_options:
+ # On outbound connections, determine which type of peers to securely connect to.
+ # The available options are :
+ # none : Do not encrypt outgoing connections
+ # dc : Encrypt connections to peers in other datacenters but not within datacenters
+ # rack : Encrypt connections to peers in other racks but not within racks
+ # all : Always use encrypted connections
internode_encryption: none
+ # When set to true, encrypted and unencrypted connections are allowed on the storage_port
+ # This should _only be true_ while in unencrypted or transitional operation
+ # optional defaults to true if internode_encryption is none
+ # optional: true
+ # If enabled, will open up an encrypted listening socket on ssl_storage_port. Should only be used
+ # during upgrade to 4.0; otherwise, set to false.
+ enable_legacy_ssl_storage_port: false
+ # Set to a valid keystore if internode_encryption is dc, rack or all
keystore: conf/.keystore
keystore_password: cassandra
+ # Verify peer server certificates
+ require_client_auth: false
+ # Set to a valid trustore if require_client_auth is true
truststore: conf/.truststore
truststore_password: cassandra
- # More advanced defaults below:
+ # Verify that the host name in the certificate matches the connected host
+ require_endpoint_verification: false
+ # More advanced defaults:
# protocol: TLS
- # algorithm: SunX509
# store_type: JKS
- # cipher_suites: [TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_DHE_RSA_WITH_AES_128_CBC_SHA,TLS_DHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA]
- # require_client_auth: false
- # require_endpoint_verification: false
+ # cipher_suites: [
+ # TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
+ # TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,
+ # TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA, TLS_RSA_WITH_AES_128_GCM_SHA256, TLS_RSA_WITH_AES_128_CBC_SHA,
+ # TLS_RSA_WITH_AES_256_CBC_SHA
+ # ]
-# enable or disable client/server encryption.
+# Configure client-to-server encryption.
+#
+# **NOTE** this default configuration is an insecure configuration. If you need to
+# enable client-to-server encryption generate server keystores (and truststores for mutual
+# authentication) per:
+# http://download.oracle.com/javase/8/docs/technotes/guides/security/jsse/JSSERefGuide.html#CreateKeystore
+# Then perform the following configuration changes:
+#
+# Step 1: Set enabled=true and explicitly set optional=true. Restart all nodes
+#
+# Step 2: Set optional=false (or remove it) and if you generated truststores and want to use mutual
+# auth set require_client_auth=true. Restart all nodes
client_encryption_options:
+ # Enable client-to-server encryption
enabled: false
- # If enabled and optional is set to true encrypted and unencrypted connections are handled.
- optional: false
+ # When set to true, encrypted and unencrypted connections are allowed on the native_transport_port
+ # This should _only be true_ while in unencrypted or transitional operation
+ # optional defaults to true when enabled is false, and false when enabled is true.
+ # optional: true
+ # Set keystore and keystore_password to valid keystores if enabled is true
keystore: conf/.keystore
keystore_password: cassandra
- # require_client_auth: false
+ # Verify client certificates
+ require_client_auth: false
# Set trustore and truststore_password if require_client_auth is true
# truststore: conf/.truststore
# truststore_password: cassandra
- # More advanced defaults below:
+ # More advanced defaults:
# protocol: TLS
- # algorithm: SunX509
# store_type: JKS
- # cipher_suites: [TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_DHE_RSA_WITH_AES_128_CBC_SHA,TLS_DHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA]
+ # cipher_suites: [
+ # TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
+ # TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,
+ # TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA, TLS_RSA_WITH_AES_128_GCM_SHA256, TLS_RSA_WITH_AES_128_CBC_SHA,
+ # TLS_RSA_WITH_AES_256_CBC_SHA
+ # ]
# internode_compression controls whether traffic between nodes is
# compressed.
@@ -1096,10 +1162,6 @@
tracetype_query_ttl: 86400
tracetype_repair_ttl: 604800
-# By default, Cassandra logs GC Pauses greater than 200 ms at INFO level
-# This threshold can be adjusted to minimize logging if necessary
-# gc_log_threshold_in_ms: 200
-
# If unset, all GC Pauses greater than gc_log_threshold_in_ms will log at
# INFO level
# UDFs (user defined functions) are disabled by default.
@@ -1139,9 +1201,9 @@
key_alias: testing:1
# CBC IV length for AES needs to be 16 bytes (which is also the default size)
# iv_length: 16
- key_provider:
+ key_provider:
- class_name: org.apache.cassandra.security.JKSKeyProvider
- parameters:
+ parameters:
- keystore: conf/.keystore
keystore_password: cassandra
store_type: JCEKS
@@ -1177,10 +1239,14 @@
# Log a warning when compacting partitions larger than this value
compaction_large_partition_warning_threshold_mb: 100
+# GC Pauses greater than 200 ms will be logged at INFO level
+# This threshold can be adjusted to minimize logging if necessary
+# gc_log_threshold_in_ms: 200
+
# GC Pauses greater than gc_warn_threshold_in_ms will be logged at WARN level
-# Adjust the threshold based on your application throughput requirement
-# By default, Cassandra logs GC Pauses greater than 200 ms at INFO level
-gc_warn_threshold_in_ms: 1000
+# Adjust the threshold based on your application throughput requirement. Setting to 0
+# will deactivate the feature.
+# gc_warn_threshold_in_ms: 1000
# Maximum size of any value in SSTables. Safety measure to detect SSTable corruption
# early. Any value size larger than this threshold will result into marking an SSTable
@@ -1245,6 +1311,80 @@
#
# otc_backlog_expiration_interval_ms: 200
+# Track a metric per keyspace indicating whether replication achieved the ideal consistency
+# level for writes without timing out. This is different from the consistency level requested by
+# each write which may be lower in order to facilitate availability.
+# ideal_consistency_level: EACH_QUORUM
+
+# Automatically upgrade sstables after upgrade - if there is no ordinary compaction to do, the
+# oldest non-upgraded sstable will get upgraded to the latest version
+# automatic_sstable_upgrade: false
+# Limit the number of concurrent sstable upgrades
+# max_concurrent_automatic_sstable_upgrades: 1
+
+# Audit logging - Logs every incoming CQL command request, authentication to a node. See the docs
+# on audit_logging for full details about the various configuration options.
+audit_logging_options:
+ enabled: false
+ logger:
+ - class_name: BinAuditLogger
+ # audit_logs_dir:
+ # included_keyspaces:
+ # excluded_keyspaces: system, system_schema, system_virtual_schema
+ # included_categories:
+ # excluded_categories:
+ # included_users:
+ # excluded_users:
+ # roll_cycle: HOURLY
+ # block: true
+ # max_queue_weight: 268435456 # 256 MiB
+ # max_log_size: 17179869184 # 16 GiB
+ ## archive command is "/path/to/script.sh %path" where %path is replaced with the file being rolled:
+ # archive_command:
+ # max_archive_retries: 10
+
+
+# default options for full query logging - these can be overridden from command line when executing
+# nodetool enablefullquerylog
+#full_query_logging_options:
+ # log_dir:
+ # roll_cycle: HOURLY
+ # block: true
+ # max_queue_weight: 268435456 # 256 MiB
+ # max_log_size: 17179869184 # 16 GiB
+ ## archive command is "/path/to/script.sh %path" where %path is replaced with the file being rolled:
+ # archive_command:
+ # max_archive_retries: 10
+
+# validate tombstones on reads and compaction
+# can be either "disabled", "warn" or "exception"
+# corrupted_tombstone_strategy: disabled
+
+# Diagnostic Events #
+# If enabled, diagnostic events can be helpful for troubleshooting operational issues. Emitted events contain details
+# on internal state and temporal relationships across events, accessible by clients via JMX.
+diagnostic_events_enabled: false
+
+# Use native transport TCP message coalescing. If on upgrade to 4.0 you found your throughput decreasing, and in
+# particular you run an old kernel or have very fewer client connections, this option might be worth evaluating.
+#native_transport_flush_in_batches_legacy: false
+
+# Enable tracking of repaired state of data during reads and comparison between replicas
+# Mismatches between the repaired sets of replicas can be characterized as either confirmed
+# or unconfirmed. In this context, unconfirmed indicates that the presence of pending repair
+# sessions, unrepaired partition tombstones, or some other condition means that the disparity
+# cannot be considered conclusive. Confirmed mismatches should be a trigger for investigation
+# as they may be indicative of corruption or data loss.
+# There are separate flags for range vs partition reads as single partition reads are only tracked
+# when CL > 1 and a digest mismatch occurs. Currently, range queries don't use digests so if
+# enabled for range reads, all range reads will include repaired data tracking. As this adds
+# some overhead, operators may wish to disable it whilst still enabling it for partition reads
+repaired_data_tracking_for_range_reads_enabled: false
+repaired_data_tracking_for_partition_reads_enabled: false
+# If false, only confirmed mismatches will be reported. If true, a separate metric for unconfirmed
+# mismatches will also be recorded. This is to avoid potential signal:noise issues are unconfirmed
+# mismatches are less actionable than confirmed ones.
+report_unconfirmed_repaired_data_mismatches: false
#########################
# EXPERIMENTAL FEATURES #
@@ -1252,8 +1392,12 @@
# Enables materialized view creation on this node.
# Materialized views are considered experimental and are not recommended for production use.
-enable_materialized_views: true
+enable_materialized_views: false
# Enables SASI index creation on this node.
# SASI indexes are considered experimental and are not recommended for production use.
-enable_sasi_indexes: true
\ No newline at end of file
+enable_sasi_indexes: false
+
+# Enables creation of transiently replicated keyspaces on this node.
+# Transient replication is experimental and is not recommended for production use.
+enable_transient_replication: false
diff --git a/conf/cqlshrc.sample b/conf/cqlshrc.sample
index 0bf926f..0aa802d 100644
--- a/conf/cqlshrc.sample
+++ b/conf/cqlshrc.sample
@@ -30,7 +30,7 @@
; color = on
;; Used for displaying timestamps (and reading them with COPY)
-; datetimeformat = %Y-%m-%d %H:%M:%S%z
+; time_format = %Y-%m-%d %H:%M:%S%z
;; Display timezone
;timezone = Etc/UTC
diff --git a/conf/jvm-clients.options b/conf/jvm-clients.options
new file mode 100644
index 0000000..6181ed0
--- /dev/null
+++ b/conf/jvm-clients.options
@@ -0,0 +1,10 @@
+###########################################################################
+# jvm-clients.options #
+# #
+# See jvm8-clients.options and jvm11-clients.options for Java version #
+# specific options. #
+###########################################################################
+
+# intentionally left empty
+
+# The newline in the end of file is intentional
diff --git a/conf/jvm-server.options b/conf/jvm-server.options
new file mode 100644
index 0000000..c52e192
--- /dev/null
+++ b/conf/jvm-server.options
@@ -0,0 +1,191 @@
+###########################################################################
+# jvm-server.options #
+# #
+# - all flags defined here will be used by cassandra to startup the JVM #
+# - one flag should be specified per line #
+# - lines that do not start with '-' will be ignored #
+# - only static flags are accepted (no variables or parameters) #
+# - dynamic flags will be appended to these on cassandra-env #
+# #
+# See jvm8-server.options and jvm11-server.options for Java version #
+# specific options. #
+###########################################################################
+
+######################
+# STARTUP PARAMETERS #
+######################
+
+# Uncomment any of the following properties to enable specific startup parameters
+
+# In a multi-instance deployment, multiple Cassandra instances will independently assume that all
+# CPU processors are available to it. This setting allows you to specify a smaller set of processors
+# and perhaps have affinity.
+#-Dcassandra.available_processors=number_of_processors
+
+# The directory location of the cassandra.yaml file.
+#-Dcassandra.config=directory
+
+# Sets the initial partitioner token for a node the first time the node is started.
+#-Dcassandra.initial_token=token
+
+# Set to false to start Cassandra on a node but not have the node join the cluster.
+#-Dcassandra.join_ring=true|false
+
+# Set to false to clear all gossip state for the node on restart. Use when you have changed node
+# information in cassandra.yaml (such as listen_address).
+#-Dcassandra.load_ring_state=true|false
+
+# Enable pluggable metrics reporter. See Pluggable metrics reporting in Cassandra 2.0.2.
+#-Dcassandra.metricsReporterConfigFile=file
+
+# Set the port on which the CQL native transport listens for clients. (Default: 9042)
+#-Dcassandra.native_transport_port=port
+
+# Overrides the partitioner. (Default: org.apache.cassandra.dht.Murmur3Partitioner)
+#-Dcassandra.partitioner=partitioner
+
+# To replace a node that has died, restart a new node in its place specifying the address of the
+# dead node. The new node must not have any data in its data directory, that is, it must be in the
+# same state as before bootstrapping.
+#-Dcassandra.replace_address=listen_address or broadcast_address of dead node
+
+# Allow restoring specific tables from an archived commit log.
+#-Dcassandra.replayList=table
+
+# Allows overriding of the default RING_DELAY (30000ms), which is the amount of time a node waits
+# before joining the ring.
+#-Dcassandra.ring_delay_ms=ms
+
+# Set the SSL port for encrypted communication. (Default: 7001)
+#-Dcassandra.ssl_storage_port=port
+
+# Set the port for inter-node communication. (Default: 7000)
+#-Dcassandra.storage_port=port
+
+# Set the default location for the trigger JARs. (Default: conf/triggers)
+#-Dcassandra.triggers_dir=directory
+
+# For testing new compaction and compression strategies. It allows you to experiment with different
+# strategies and benchmark write performance differences without affecting the production workload.
+#-Dcassandra.write_survey=true
+
+# To disable configuration via JMX of auth caches (such as those for credentials, permissions and
+# roles). This will mean those config options can only be set (persistently) in cassandra.yaml
+# and will require a restart for new values to take effect.
+#-Dcassandra.disable_auth_caches_remote_configuration=true
+
+# To disable dynamic calculation of the page size used when indexing an entire partition (during
+# initial index build/rebuild). If set to true, the page size will be fixed to the default of
+# 10000 rows per page.
+#-Dcassandra.force_default_indexing_page_size=true
+
+# Imposes an upper bound on hint lifetime below the normal min gc_grace_seconds
+#-Dcassandra.maxHintTTL=max_hint_ttl_in_seconds
+
+########################
+# GENERAL JVM SETTINGS #
+########################
+
+# enable assertions. highly suggested for correct application functionality.
+-ea
+
+# disable assertions for net.openhft.** because it runs out of memory by design
+# if enabled and run for more than just brief testing
+-da:net.openhft...
+
+# enable thread priorities, primarily so we can give periodic tasks
+# a lower priority to avoid interfering with client workload
+-XX:+UseThreadPriorities
+
+# Enable heap-dump if there's an OOM
+-XX:+HeapDumpOnOutOfMemoryError
+
+# Per-thread stack size.
+-Xss256k
+
+# Larger interned string table, for gossip's benefit (CASSANDRA-6410)
+-XX:StringTableSize=1000003
+
+# Make sure all memory is faulted and zeroed on startup.
+# This helps prevent soft faults in containers and makes
+# transparent hugepage allocation more effective.
+-XX:+AlwaysPreTouch
+
+# Disable biased locking as it does not benefit Cassandra.
+-XX:-UseBiasedLocking
+
+# Enable thread-local allocation blocks and allow the JVM to automatically
+# resize them at runtime.
+-XX:+UseTLAB
+-XX:+ResizeTLAB
+-XX:+UseNUMA
+
+# http://www.evanjones.ca/jvm-mmap-pause.html
+-XX:+PerfDisableSharedMem
+
+# Prefer binding to IPv4 network intefaces (when net.ipv6.bindv6only=1). See
+# http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=6342561 (short version:
+# comment out this entry to enable IPv6 support).
+-Djava.net.preferIPv4Stack=true
+
+### Debug options
+
+# uncomment to enable flight recorder
+#-XX:+UnlockCommercialFeatures
+#-XX:+FlightRecorder
+
+# uncomment to have Cassandra JVM listen for remote debuggers/profilers on port 1414
+#-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=1414
+
+# uncomment to have Cassandra JVM log internal method compilation (developers only)
+#-XX:+UnlockDiagnosticVMOptions
+#-XX:+LogCompilation
+
+#################
+# HEAP SETTINGS #
+#################
+
+# Heap size is automatically calculated by cassandra-env based on this
+# formula: max(min(1/2 ram, 1024MB), min(1/4 ram, 8GB))
+# That is:
+# - calculate 1/2 ram and cap to 1024MB
+# - calculate 1/4 ram and cap to 8192MB
+# - pick the max
+#
+# For production use you may wish to adjust this for your environment.
+# If that's the case, uncomment the -Xmx and Xms options below to override the
+# automatic calculation of JVM heap memory.
+#
+# It is recommended to set min (-Xms) and max (-Xmx) heap sizes to
+# the same value to avoid stop-the-world GC pauses during resize, and
+# so that we can lock the heap in memory on startup to prevent any
+# of it from being swapped out.
+#-Xms4G
+#-Xmx4G
+
+# Young generation size is automatically calculated by cassandra-env
+# based on this formula: min(100 * num_cores, 1/4 * heap size)
+#
+# The main trade-off for the young generation is that the larger it
+# is, the longer GC pause times will be. The shorter it is, the more
+# expensive GC will be (usually).
+#
+# It is not recommended to set the young generation size if using the
+# G1 GC, since that will override the target pause-time goal.
+# More info: http://www.oracle.com/technetwork/articles/java/g1gc-1984535.html
+#
+# The example below assumes a modern 8-core+ machine for decent
+# times. If in doubt, and if you do not particularly want to tweak, go
+# 100 MB per physical CPU core.
+#-Xmn800M
+
+###################################
+# EXPIRATION DATE OVERFLOW POLICY #
+###################################
+
+# Defines how to handle INSERT requests with TTL exceeding the maximum supported expiration date:
+# * REJECT: this is the default policy and will reject any requests with expiration date timestamp after 2038-01-19T03:14:06+00:00.
+# * CAP: any insert with TTL expiring after 2038-01-19T03:14:06+00:00 will expire on 2038-01-19T03:14:06+00:00 and the client will receive a warning.
+# * CAP_NOWARN: same as previous, except that the client warning will not be emitted.
+#
+#-Dcassandra.expiration_date_overflow_policy=REJECT
diff --git a/conf/jvm.options b/conf/jvm.options
deleted file mode 100644
index 01bb168..0000000
--- a/conf/jvm.options
+++ /dev/null
@@ -1,256 +0,0 @@
-###########################################################################
-# jvm.options #
-# #
-# - all flags defined here will be used by cassandra to startup the JVM #
-# - one flag should be specified per line #
-# - lines that do not start with '-' will be ignored #
-# - only static flags are accepted (no variables or parameters) #
-# - dynamic flags will be appended to these on cassandra-env #
-###########################################################################
-
-######################
-# STARTUP PARAMETERS #
-######################
-
-# Uncomment any of the following properties to enable specific startup parameters
-
-# In a multi-instance deployment, multiple Cassandra instances will independently assume that all
-# CPU processors are available to it. This setting allows you to specify a smaller set of processors
-# and perhaps have affinity.
-#-Dcassandra.available_processors=number_of_processors
-
-# The directory location of the cassandra.yaml file.
-#-Dcassandra.config=directory
-
-# Sets the initial partitioner token for a node the first time the node is started.
-#-Dcassandra.initial_token=token
-
-# Set to false to start Cassandra on a node but not have the node join the cluster.
-#-Dcassandra.join_ring=true|false
-
-# Set to false to clear all gossip state for the node on restart. Use when you have changed node
-# information in cassandra.yaml (such as listen_address).
-#-Dcassandra.load_ring_state=true|false
-
-# Enable pluggable metrics reporter. See Pluggable metrics reporting in Cassandra 2.0.2.
-#-Dcassandra.metricsReporterConfigFile=file
-
-# Set the port on which the CQL native transport listens for clients. (Default: 9042)
-#-Dcassandra.native_transport_port=port
-
-# Overrides the partitioner. (Default: org.apache.cassandra.dht.Murmur3Partitioner)
-#-Dcassandra.partitioner=partitioner
-
-# To replace a node that has died, restart a new node in its place specifying the address of the
-# dead node. The new node must not have any data in its data directory, that is, it must be in the
-# same state as before bootstrapping.
-#-Dcassandra.replace_address=listen_address or broadcast_address of dead node
-
-# Allow restoring specific tables from an archived commit log.
-#-Dcassandra.replayList=table
-
-# Allows overriding of the default RING_DELAY (30000ms), which is the amount of time a node waits
-# before joining the ring.
-#-Dcassandra.ring_delay_ms=ms
-
-# Set the port for the Thrift RPC service, which is used for client connections. (Default: 9160)
-#-Dcassandra.rpc_port=port
-
-# Set the SSL port for encrypted communication. (Default: 7001)
-#-Dcassandra.ssl_storage_port=port
-
-# Enable or disable the native transport server. See start_native_transport in cassandra.yaml.
-# cassandra.start_native_transport=true|false
-
-# Enable or disable the Thrift RPC server. (Default: true)
-#-Dcassandra.start_rpc=true/false
-
-# Set the port for inter-node communication. (Default: 7000)
-#-Dcassandra.storage_port=port
-
-# Set the default location for the trigger JARs. (Default: conf/triggers)
-#-Dcassandra.triggers_dir=directory
-
-# For testing new compaction and compression strategies. It allows you to experiment with different
-# strategies and benchmark write performance differences without affecting the production workload.
-#-Dcassandra.write_survey=true
-
-# To disable configuration via JMX of auth caches (such as those for credentials, permissions and
-# roles). This will mean those config options can only be set (persistently) in cassandra.yaml
-# and will require a restart for new values to take effect.
-#-Dcassandra.disable_auth_caches_remote_configuration=true
-
-# To disable dynamic calculation of the page size used when indexing an entire partition (during
-# initial index build/rebuild). If set to true, the page size will be fixed to the default of
-# 10000 rows per page.
-#-Dcassandra.force_default_indexing_page_size=true
-
-########################
-# GENERAL JVM SETTINGS #
-########################
-
-# enable assertions. highly suggested for correct application functionality.
--ea
-
-# enable thread priorities, primarily so we can give periodic tasks
-# a lower priority to avoid interfering with client workload
--XX:+UseThreadPriorities
-
-# allows lowering thread priority without being root on linux - probably
-# not necessary on Windows but doesn't harm anything.
-# see http://tech.stolsvik.com/2010/01/linux-java-thread-priorities-workar
--XX:ThreadPriorityPolicy=42
-
-# Enable heap-dump if there's an OOM
--XX:+HeapDumpOnOutOfMemoryError
-
-# Per-thread stack size.
--Xss256k
-
-# Larger interned string table, for gossip's benefit (CASSANDRA-6410)
--XX:StringTableSize=1000003
-
-# Make sure all memory is faulted and zeroed on startup.
-# This helps prevent soft faults in containers and makes
-# transparent hugepage allocation more effective.
--XX:+AlwaysPreTouch
-
-# Disable biased locking as it does not benefit Cassandra.
--XX:-UseBiasedLocking
-
-# Enable thread-local allocation blocks and allow the JVM to automatically
-# resize them at runtime.
--XX:+UseTLAB
--XX:+ResizeTLAB
--XX:+UseNUMA
-
-# http://www.evanjones.ca/jvm-mmap-pause.html
--XX:+PerfDisableSharedMem
-
-# Prefer binding to IPv4 network intefaces (when net.ipv6.bindv6only=1). See
-# http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=6342561 (short version:
-# comment out this entry to enable IPv6 support).
--Djava.net.preferIPv4Stack=true
-
-### Debug options
-
-# uncomment to enable flight recorder
-#-XX:+UnlockCommercialFeatures
-#-XX:+FlightRecorder
-
-# uncomment to have Cassandra JVM listen for remote debuggers/profilers on port 1414
-#-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=1414
-
-# uncomment to have Cassandra JVM log internal method compilation (developers only)
-#-XX:+UnlockDiagnosticVMOptions
-#-XX:+LogCompilation
-
-#################
-# HEAP SETTINGS #
-#################
-
-# Heap size is automatically calculated by cassandra-env based on this
-# formula: max(min(1/2 ram, 1024MB), min(1/4 ram, 8GB))
-# That is:
-# - calculate 1/2 ram and cap to 1024MB
-# - calculate 1/4 ram and cap to 8192MB
-# - pick the max
-#
-# For production use you may wish to adjust this for your environment.
-# If that's the case, uncomment the -Xmx and Xms options below to override the
-# automatic calculation of JVM heap memory.
-#
-# It is recommended to set min (-Xms) and max (-Xmx) heap sizes to
-# the same value to avoid stop-the-world GC pauses during resize, and
-# so that we can lock the heap in memory on startup to prevent any
-# of it from being swapped out.
-#-Xms4G
-#-Xmx4G
-
-# Young generation size is automatically calculated by cassandra-env
-# based on this formula: min(100 * num_cores, 1/4 * heap size)
-#
-# The main trade-off for the young generation is that the larger it
-# is, the longer GC pause times will be. The shorter it is, the more
-# expensive GC will be (usually).
-#
-# It is not recommended to set the young generation size if using the
-# G1 GC, since that will override the target pause-time goal.
-# More info: http://www.oracle.com/technetwork/articles/java/g1gc-1984535.html
-#
-# The example below assumes a modern 8-core+ machine for decent
-# times. If in doubt, and if you do not particularly want to tweak, go
-# 100 MB per physical CPU core.
-#-Xmn800M
-
-###################################
-# EXPIRATION DATE OVERFLOW POLICY #
-###################################
-
-# Defines how to handle INSERT requests with TTL exceeding the maximum supported expiration date:
-# * REJECT: this is the default policy and will reject any requests with expiration date timestamp after 2038-01-19T03:14:06+00:00.
-# * CAP: any insert with TTL expiring after 2038-01-19T03:14:06+00:00 will expire on 2038-01-19T03:14:06+00:00 and the client will receive a warning.
-# * CAP_NOWARN: same as previous, except that the client warning will not be emitted.
-#
-#-Dcassandra.expiration_date_overflow_policy=REJECT
-
-#################
-# GC SETTINGS #
-#################
-
-### CMS Settings
-
--XX:+UseParNewGC
--XX:+UseConcMarkSweepGC
--XX:+CMSParallelRemarkEnabled
--XX:SurvivorRatio=8
--XX:MaxTenuringThreshold=1
--XX:CMSInitiatingOccupancyFraction=75
--XX:+UseCMSInitiatingOccupancyOnly
--XX:CMSWaitDuration=10000
--XX:+CMSParallelInitialMarkEnabled
--XX:+CMSEdenChunksRecordAlways
-# some JVMs will fill up their heap when accessed via JMX, see CASSANDRA-6541
--XX:+CMSClassUnloadingEnabled
-
-### G1 Settings (experimental, comment previous section and uncomment section below to enable)
-
-## Use the Hotspot garbage-first collector.
-#-XX:+UseG1GC
-#
-## Have the JVM do less remembered set work during STW, instead
-## preferring concurrent GC. Reduces p99.9 latency.
-#-XX:G1RSetUpdatingPauseTimePercent=5
-#
-## Main G1GC tunable: lowering the pause target will lower throughput and vise versa.
-## 200ms is the JVM default and lowest viable setting
-## 1000ms increases throughput. Keep it smaller than the timeouts in cassandra.yaml.
-#-XX:MaxGCPauseMillis=500
-
-## Optional G1 Settings
-
-# Save CPU time on large (>= 16GB) heaps by delaying region scanning
-# until the heap is 70% full. The default in Hotspot 8u40 is 40%.
-#-XX:InitiatingHeapOccupancyPercent=70
-
-# For systems with > 8 cores, the default ParallelGCThreads is 5/8 the number of logical cores.
-# Otherwise equal to the number of cores when 8 or less.
-# Machines with > 10 cores should try setting these to <= full cores.
-#-XX:ParallelGCThreads=16
-# By default, ConcGCThreads is 1/4 of ParallelGCThreads.
-# Setting both to the same value can reduce STW durations.
-#-XX:ConcGCThreads=16
-
-### GC logging options -- uncomment to enable
-
--XX:+PrintGCDetails
--XX:+PrintGCDateStamps
--XX:+PrintHeapAtGC
--XX:+PrintTenuringDistribution
--XX:+PrintGCApplicationStoppedTime
--XX:+PrintPromotionFailure
-#-XX:PrintFLSStatistics=1
-#-Xloggc:/var/log/cassandra/gc.log
--XX:+UseGCLogFileRotation
--XX:NumberOfGCLogFiles=10
--XX:GCLogFileSize=10M
diff --git a/conf/jvm11-clients.options b/conf/jvm11-clients.options
new file mode 100644
index 0000000..c88b7ab
--- /dev/null
+++ b/conf/jvm11-clients.options
@@ -0,0 +1,29 @@
+###########################################################################
+# jvm11-clients.options #
+# #
+# See jvm-clients.options. This file is specific for Java 11 and newer. #
+###########################################################################
+
+###################
+# JPMS SETTINGS #
+###################
+
+-Djdk.attach.allowAttachSelf=true
+--add-exports java.base/jdk.internal.misc=ALL-UNNAMED
+--add-exports java.base/jdk.internal.ref=ALL-UNNAMED
+--add-exports java.base/sun.nio.ch=ALL-UNNAMED
+--add-exports java.management.rmi/com.sun.jmx.remote.internal.rmi=ALL-UNNAMED
+--add-exports java.rmi/sun.rmi.registry=ALL-UNNAMED
+--add-exports java.rmi/sun.rmi.server=ALL-UNNAMED
+--add-exports java.sql/java.sql=ALL-UNNAMED
+
+--add-opens java.base/java.lang.module=ALL-UNNAMED
+--add-opens java.base/jdk.internal.loader=ALL-UNNAMED
+--add-opens java.base/jdk.internal.ref=ALL-UNNAMED
+--add-opens java.base/jdk.internal.reflect=ALL-UNNAMED
+--add-opens java.base/jdk.internal.math=ALL-UNNAMED
+--add-opens java.base/jdk.internal.module=ALL-UNNAMED
+--add-opens java.base/jdk.internal.util.jar=ALL-UNNAMED
+--add-opens jdk.management/com.sun.management.internal=ALL-UNNAMED
+
+# The newline in the end of file is intentional
diff --git a/conf/jvm11-server.options b/conf/jvm11-server.options
new file mode 100644
index 0000000..5fb829f
--- /dev/null
+++ b/conf/jvm11-server.options
@@ -0,0 +1,96 @@
+###########################################################################
+# jvm11-server.options #
+# #
+# See jvm-server.options. This file is specific for Java 11 and newer. #
+###########################################################################
+
+#################
+# GC SETTINGS #
+#################
+
+
+
+### CMS Settings
+-XX:+UseConcMarkSweepGC
+-XX:+CMSParallelRemarkEnabled
+-XX:SurvivorRatio=8
+-XX:MaxTenuringThreshold=1
+-XX:CMSInitiatingOccupancyFraction=75
+-XX:+UseCMSInitiatingOccupancyOnly
+-XX:CMSWaitDuration=10000
+-XX:+CMSParallelInitialMarkEnabled
+-XX:+CMSEdenChunksRecordAlways
+## some JVMs will fill up their heap when accessed via JMX, see CASSANDRA-6541
+-XX:+CMSClassUnloadingEnabled
+
+
+
+### G1 Settings
+## Use the Hotspot garbage-first collector.
+#-XX:+UseG1GC
+#-XX:+ParallelRefProcEnabled
+
+#
+## Have the JVM do less remembered set work during STW, instead
+## preferring concurrent GC. Reduces p99.9 latency.
+#-XX:G1RSetUpdatingPauseTimePercent=5
+#
+## Main G1GC tunable: lowering the pause target will lower throughput and vise versa.
+## 200ms is the JVM default and lowest viable setting
+## 1000ms increases throughput. Keep it smaller than the timeouts in cassandra.yaml.
+#-XX:MaxGCPauseMillis=500
+
+## Optional G1 Settings
+# Save CPU time on large (>= 16GB) heaps by delaying region scanning
+# until the heap is 70% full. The default in Hotspot 8u40 is 40%.
+#-XX:InitiatingHeapOccupancyPercent=70
+
+# For systems with > 8 cores, the default ParallelGCThreads is 5/8 the number of logical cores.
+# Otherwise equal to the number of cores when 8 or less.
+# Machines with > 10 cores should try setting these to <= full cores.
+#-XX:ParallelGCThreads=16
+# By default, ConcGCThreads is 1/4 of ParallelGCThreads.
+# Setting both to the same value can reduce STW durations.
+#-XX:ConcGCThreads=16
+
+
+### JPMS
+
+-Djdk.attach.allowAttachSelf=true
+--add-exports java.base/jdk.internal.misc=ALL-UNNAMED
+--add-exports java.base/jdk.internal.ref=ALL-UNNAMED
+--add-exports java.base/sun.nio.ch=ALL-UNNAMED
+--add-exports java.management.rmi/com.sun.jmx.remote.internal.rmi=ALL-UNNAMED
+--add-exports java.rmi/sun.rmi.registry=ALL-UNNAMED
+--add-exports java.rmi/sun.rmi.server=ALL-UNNAMED
+--add-exports java.sql/java.sql=ALL-UNNAMED
+
+--add-opens java.base/java.lang.module=ALL-UNNAMED
+--add-opens java.base/jdk.internal.loader=ALL-UNNAMED
+--add-opens java.base/jdk.internal.ref=ALL-UNNAMED
+--add-opens java.base/jdk.internal.reflect=ALL-UNNAMED
+--add-opens java.base/jdk.internal.math=ALL-UNNAMED
+--add-opens java.base/jdk.internal.module=ALL-UNNAMED
+--add-opens java.base/jdk.internal.util.jar=ALL-UNNAMED
+--add-opens jdk.management/com.sun.management.internal=ALL-UNNAMED
+
+
+### GC logging options -- uncomment to enable
+
+# Java 11 (and newer) GC logging options:
+# See description of https://bugs.openjdk.java.net/browse/JDK-8046148 for details about the syntax
+# The following is the equivalent to -XX:+PrintGCDetails -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=10M
+#-Xlog:gc=info,heap*=trace,age*=debug,safepoint=info,promotion*=trace:file=/var/log/cassandra/gc.log:time,uptime,pid,tid,level:filecount=10,filesize=10485760
+
+# Notes for Java 8 migration:
+#
+# -XX:+PrintGCDetails maps to -Xlog:gc*:... - i.e. add a '*' after "gc"
+# -XX:+PrintGCDateStamps maps to decorator 'time'
+#
+# -XX:+PrintHeapAtGC maps to 'heap' with level 'trace'
+# -XX:+PrintTenuringDistribution maps to 'age' with level 'debug'
+# -XX:+PrintGCApplicationStoppedTime maps to 'safepoint' with level 'info'
+# -XX:+PrintPromotionFailure maps to 'promotion' with level 'trace'
+# -XX:PrintFLSStatistics=1 maps to 'freelist' with level 'trace'
+
+# The newline in the end of file is intentional
diff --git a/conf/jvm8-clients.options b/conf/jvm8-clients.options
new file mode 100644
index 0000000..7d1b2ef
--- /dev/null
+++ b/conf/jvm8-clients.options
@@ -0,0 +1,9 @@
+###########################################################################
+# jvm8-clients.options #
+# #
+# See jvm-clients.options. This file is specific for Java 8 and newer. #
+###########################################################################
+
+# intentionally left empty
+
+# The newline in the end of file is intentional
diff --git a/conf/jvm8-server.options b/conf/jvm8-server.options
new file mode 100644
index 0000000..6214669
--- /dev/null
+++ b/conf/jvm8-server.options
@@ -0,0 +1,76 @@
+###########################################################################
+# jvm8-server.options #
+# #
+# See jvm-server.options. This file is specific for Java 8 and newer. #
+###########################################################################
+
+########################
+# GENERAL JVM SETTINGS #
+########################
+
+# allows lowering thread priority without being root on linux - probably
+# not necessary on Windows but doesn't harm anything.
+# see http://tech.stolsvik.com/2010/01/linux-java-thread-priorities-workaround.html
+-XX:ThreadPriorityPolicy=42
+
+#################
+# GC SETTINGS #
+#################
+
+### CMS Settings
+-XX:+UseParNewGC
+-XX:+UseConcMarkSweepGC
+-XX:+CMSParallelRemarkEnabled
+-XX:SurvivorRatio=8
+-XX:MaxTenuringThreshold=1
+-XX:CMSInitiatingOccupancyFraction=75
+-XX:+UseCMSInitiatingOccupancyOnly
+-XX:CMSWaitDuration=10000
+-XX:+CMSParallelInitialMarkEnabled
+-XX:+CMSEdenChunksRecordAlways
+## some JVMs will fill up their heap when accessed via JMX, see CASSANDRA-6541
+-XX:+CMSClassUnloadingEnabled
+
+### G1 Settings
+## Use the Hotspot garbage-first collector.
+#-XX:+UseG1GC
+#-XX:+ParallelRefProcEnabled
+
+#
+## Have the JVM do less remembered set work during STW, instead
+## preferring concurrent GC. Reduces p99.9 latency.
+#-XX:G1RSetUpdatingPauseTimePercent=5
+#
+## Main G1GC tunable: lowering the pause target will lower throughput and vise versa.
+## 200ms is the JVM default and lowest viable setting
+## 1000ms increases throughput. Keep it smaller than the timeouts in cassandra.yaml.
+#-XX:MaxGCPauseMillis=500
+
+## Optional G1 Settings
+# Save CPU time on large (>= 16GB) heaps by delaying region scanning
+# until the heap is 70% full. The default in Hotspot 8u40 is 40%.
+#-XX:InitiatingHeapOccupancyPercent=70
+
+# For systems with > 8 cores, the default ParallelGCThreads is 5/8 the number of logical cores.
+# Otherwise equal to the number of cores when 8 or less.
+# Machines with > 10 cores should try setting these to <= full cores.
+#-XX:ParallelGCThreads=16
+# By default, ConcGCThreads is 1/4 of ParallelGCThreads.
+# Setting both to the same value can reduce STW durations.
+#-XX:ConcGCThreads=16
+
+### GC logging options -- uncomment to enable
+
+-XX:+PrintGCDetails
+-XX:+PrintGCDateStamps
+-XX:+PrintHeapAtGC
+-XX:+PrintTenuringDistribution
+-XX:+PrintGCApplicationStoppedTime
+-XX:+PrintPromotionFailure
+#-XX:PrintFLSStatistics=1
+#-Xloggc:/var/log/cassandra/gc.log
+-XX:+UseGCLogFileRotation
+-XX:NumberOfGCLogFiles=10
+-XX:GCLogFileSize=10M
+
+# The newline in the end of file is intentional
diff --git a/conf/logback.xml b/conf/logback.xml
index 7bd1c6d..b2c5b10 100644
--- a/conf/logback.xml
+++ b/conf/logback.xml
@@ -22,7 +22,7 @@
appender reference in the root level section below.
-->
-<configuration scan="true">
+<configuration scan="true" scanPeriod="60 seconds">
<jmxConfigurator />
<!-- No shutdown hook; we run it ourselves in StorageService after shutdown -->
@@ -34,14 +34,14 @@
<level>INFO</level>
</filter>
<file>${cassandra.logdir}/system.log</file>
- <rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
- <fileNamePattern>${cassandra.logdir}/system.log.%i.zip</fileNamePattern>
- <minIndex>1</minIndex>
- <maxIndex>20</maxIndex>
+ <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
+ <!-- rollover daily -->
+ <fileNamePattern>${cassandra.logdir}/system.log.%d{yyyy-MM-dd}.%i.zip</fileNamePattern>
+ <!-- each file should be at most 50MB, keep 7 days worth of history, but at most 5GB -->
+ <maxFileSize>50MB</maxFileSize>
+ <maxHistory>7</maxHistory>
+ <totalSizeCap>5GB</totalSizeCap>
</rollingPolicy>
- <triggeringPolicy class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
- <maxFileSize>20MB</maxFileSize>
- </triggeringPolicy>
<encoder>
<pattern>%-5level [%thread] %date{ISO8601} %F:%L - %msg%n</pattern>
</encoder>
@@ -51,14 +51,14 @@
<appender name="DEBUGLOG" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${cassandra.logdir}/debug.log</file>
- <rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
- <fileNamePattern>${cassandra.logdir}/debug.log.%i.zip</fileNamePattern>
- <minIndex>1</minIndex>
- <maxIndex>20</maxIndex>
+ <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
+ <!-- rollover daily -->
+ <fileNamePattern>${cassandra.logdir}/debug.log.%d{yyyy-MM-dd}.%i.zip</fileNamePattern>
+ <!-- each file should be at most 50MB, keep 7 days worth of history, but at most 5GB -->
+ <maxFileSize>50MB</maxFileSize>
+ <maxHistory>7</maxHistory>
+ <totalSizeCap>5GB</totalSizeCap>
</rollingPolicy>
- <triggeringPolicy class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
- <maxFileSize>20MB</maxFileSize>
- </triggeringPolicy>
<encoder>
<pattern>%-5level [%thread] %date{ISO8601} %F:%L - %msg%n</pattern>
</encoder>
@@ -98,5 +98,4 @@
</root>
<logger name="org.apache.cassandra" level="DEBUG"/>
- <logger name="com.thinkaurelius.thrift" level="ERROR"/>
</configuration>
diff --git a/debian/cassandra.in.sh b/debian/cassandra.in.sh
deleted file mode 100644
index 8fcaf9c..0000000
--- a/debian/cassandra.in.sh
+++ /dev/null
@@ -1,34 +0,0 @@
-
-# The directory where Cassandra's configs live (required)
-CASSANDRA_CONF=/etc/cassandra
-
-CASSANDRA_HOME=/usr/share/cassandra
-
-# the default location for commitlogs, sstables, and saved caches
-# if not set in cassandra.yaml
-cassandra_storagedir=/var/lib/cassandra
-
-# The java classpath (required)
-if [ -n "$CLASSPATH" ]; then
- CLASSPATH=$CLASSPATH:$CASSANDRA_CONF
-else
- CLASSPATH=$CASSANDRA_CONF
-fi
-
-for jar in /usr/share/cassandra/lib/*.jar; do
- CLASSPATH=$CLASSPATH:$jar
-done
-
-for jar in /usr/share/cassandra/*.jar; do
- CLASSPATH=$CLASSPATH:$jar
-done
-
-CLASSPATH="$CLASSPATH:$EXTRA_CLASSPATH"
-
-
-# set JVM javaagent opts to avoid warnings/errors
-if [ "$JVM_VENDOR" != "OpenJDK" -o "$JVM_VERSION" \> "1.6.0" ] \
- || [ "$JVM_VERSION" = "1.6.0" -a "$JVM_PATCH_VERSION" -ge 23 ]
-then
- JAVA_AGENT="$JAVA_AGENT -javaagent:$CASSANDRA_HOME/lib/jamm-0.3.0.jar"
-fi
diff --git a/debian/cassandra.install b/debian/cassandra.install
index 50db32d..be34838 100644
--- a/debian/cassandra.install
+++ b/debian/cassandra.install
@@ -5,12 +5,12 @@
conf/cassandra-topology.properties etc/cassandra
conf/logback.xml etc/cassandra
conf/logback-tools.xml etc/cassandra
-conf/jvm.options etc/cassandra
+conf/jvm*.options etc/cassandra
conf/hotspot_compiler etc/cassandra
conf/triggers/* etc/cassandra/triggers
-debian/cassandra.in.sh usr/share/cassandra
debian/cassandra.conf etc/security/limits.d
debian/cassandra-sysctl.conf etc/sysctl.d
+bin/cassandra.in.sh usr/share/cassandra
bin/cassandra usr/sbin
bin/nodetool usr/bin
bin/sstableutil usr/bin
@@ -21,6 +21,8 @@
bin/sstableupgrade usr/bin
bin/sstableverify usr/bin
tools/bin/cassandra-stress usr/bin
+tools/bin/fqltool usr/bin
+tools/bin/auditlogviewer usr/bin
lib/*.jar usr/share/cassandra/lib
lib/*.zip usr/share/cassandra/lib
lib/sigar-bin/* usr/share/cassandra/lib/sigar-bin
diff --git a/debian/changelog b/debian/changelog
index d7414c6..fea0317 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,56 +1,32 @@
-cassandra (3.11.7) UNRELEASED; urgency=medium
+cassandra (4.0~alpha4) unstable; urgency=medium
* New release
- -- Michael Shuler <mshuler@apache.org> Fri, 14 Feb 2020 18:48:42 -0600
+ -- Mick Semb Wever <mck@apache.org> Sat, 11 Apr 2020 00:27:13 +0200
-cassandra (3.11.6) unstable; urgency=medium
+cassandra (4.0~alpha3) unstable; urgency=medium
* New release
- -- Mick Semb Wever <mck@apache.org> Mon, 10 Feb 2020 23:54:00 +0100
+ -- Mick Semb Wever <mck@apache.org> Mon, 30 Jan 2020 19:28:04 +0100
-cassandra (3.11.5) unstable; urgency=medium
+cassandra (4.0~alpha2) unstable; urgency=medium
* New release
- -- Michael Shuler <mshuler@apache.org> Thu, 24 Oct 2019 08:59:29 -0500
+ -- Michael Shuler <mshuler@apache.org> Thu, 24 Oct 2019 09:12:05 -0500
-cassandra (3.11.4) unstable; urgency=medium
+cassandra (4.0~alpha1) unstable; urgency=medium
* New release
- -- Michael Shuler <mshuler@apache.org> Fri, 01 Feb 2019 12:39:21 -0600
-
-cassandra (3.11.3) unstable; urgency=medium
-
- * New release
-
- -- Michael Shuler <michael@pbandjelly.org> Mon, 02 Jul 2018 13:36:16 -0500
-
-cassandra (3.11.2) unstable; urgency=medium
-
- * New release
-
- -- Michael Shuler <michael@pbandjelly.org> Tue, 10 Oct 2017 17:18:26 -0500
-
-cassandra (3.11.1) unstable; urgency=medium
-
- * New release
-
- -- Michael Shuler <michael@pbandjelly.org> Mon, 26 Jun 2017 18:51:37 -0500
-
-cassandra (3.11.0) unstable; urgency=medium
-
- * New release
-
- -- Michael Shuler <michael@pbandjelly.org> Mon, 19 Jun 2017 17:42:32 -0500
+ -- Michael Shuler <mshuler@apache.org> Tue, 03 Sep 2019 11:51:18 -0500
cassandra (3.10) unstable; urgency=medium
* New release
- -- Michael Shuler <michael@pbandjelly.org> Mon, 31 Oct 2016 08:54:45 -0500
+ -- Michael Shuler <mshuler@apache.org> Mon, 26 Sep 2016 09:07:34 -0500
cassandra (3.8) unstable; urgency=medium
diff --git a/debian/control b/debian/control
index 6a0128e..13f2a45 100644
--- a/debian/control
+++ b/debian/control
@@ -3,10 +3,10 @@
Priority: extra
Maintainer: Eric Evans <eevans@apache.org>
Uploaders: Sylvain Lebresne <slebresne@apache.org>
-Build-Depends: debhelper (>= 5), openjdk-8-jdk | java8-jdk, ant (>= 1.9), ant-optional (>= 1.9), dh-python, python-dev (>= 2.7), dpatch, bash-completion
+Build-Depends: debhelper (>= 5), openjdk-8-jdk | java8-jdk, ant (>= 1.9), ant-optional (>= 1.9), dh-python, python-dev (>= 2.7), quilt, bash-completion
Homepage: http://cassandra.apache.org
-Vcs-Git: http://git-wip-us.apache.org/repos/asf/cassandra.git
-Vcs-Browser: https://git-wip-us.apache.org/repos/asf?p=cassandra.git
+Vcs-Git: https://gitbox.apache.org/repos/asf/cassandra.git
+Vcs-Browser: https://gitbox.apache.org/repos/asf?p=cassandra.git
Standards-Version: 3.8.3
Package: cassandra
diff --git a/debian/nodetool-completion b/debian/nodetool-completion
index 732f4f4..614f7ea 100644
--- a/debian/nodetool-completion
+++ b/debian/nodetool-completion
@@ -54,33 +54,43 @@
local shopt='
bootstrap
+ clientstats
compactionhistory
compactionstats
decommission
describecluster
+ disableauditlog
disablebackup
disablebinary
+ disablefullquerylog
disablegossip
disablehandoff
disablehintsfordc
- disablethrift
+ disableoldprotocolversions
drain
+ enableauditlog
enablebackup
enablebinary
+ enablefullquerylog
enablegossip
enablehandoff
- enablethrift
enablehintsfordc
+ enableoldprotocolversions
failuredetector
gcstats
+ getbatchlogreplaythrottle
getcompactionthroughput
getconcurrentcompactors
+ getconcurrentviewbuilders
getinterdcstreamthroughput
getlogginglevels
+ getmaxhintwindow
+ getseeds
getstreamthroughput
gettimeout
gettraceprobability
gossipinfo
+ handoffwindow
help
invalidatecountercache
invalidatekeycache
@@ -92,20 +102,26 @@
rangekeysample
refreshsizeestimates
reloadlocalschema
+ reloadseeds
+ reloadssl
reloadtriggers
+ repair_admin
replaybatchlog
+ resetfullquerylog
resetlocalschema
resumehandoff
ring
+ setbatchlogreplaythrottle
setconcurrentcompactors
+ setconcurrentviewbuilders
sethintedhandoffthrottlekb
setinterdcstreamthroughput
setlogginglevel
+ setmaxhintwindow
settimeout
status
statusbackup
statusbinary
- statusthrift
statusgossip
statushandoff
stopdaemon
@@ -126,10 +142,13 @@
garbagecollect
getcompactionthreshold
getendpoints
+ getreplicas
getsstables
+ import
info
move
netstats
+ profileload
rebuild
rebuild_index
refresh
@@ -144,6 +163,7 @@
setstreamthroughput
settraceprobability
snapshot
+ statusautocompaction
stop
tablehistograms
toppartitions
@@ -162,7 +182,10 @@
garbagecollect
getcompactionthreshold
getendpoints
+ getreplicas
getsstables
+ import
+ profileload
rebuild_index
refresh
relocatesstables
@@ -170,6 +193,7 @@
scrub
setcompactionthreshold
snapshot
+ statusautocompaction
tablehistograms
toppartitions
verify
@@ -183,9 +207,13 @@
enableautocompaction
flush
garbagecollect
+ getreplicas
+ import
+ profileload
relocatesstables
repair
scrub
+ statusautocompaction
toppartitions
upgradesstables
verify
@@ -230,7 +258,7 @@
fi
elif [[ $COMP_CWORD -eq 3 ]] ; then
case "${COMP_WORDS[1]}" in
- cleanup|compact|flush|garbagecollect|getcompactionthreshold|getendpoints|getsstables|rebuild_index|refresh|relocatesstables|repair|scrub|setcompactionthreshold|tablehistograms|toppartitions|verify)
+ cleanup|compact|flush|garbagecollect|getcompactionthreshold|getendpoints|getreplicas|getsstables|import|profileload|rebuild_index|refresh|relocatesstables|repair|scrub|setcompactionthreshold|statusautocompaction|tablehistograms|toppartitions|verify)
show_cfs ${prev} ${cur}
return 0
;;
diff --git a/debian/patches/001cassandra_yaml_dirs.dpatch b/debian/patches/001cassandra_yaml_dirs.dpatch
deleted file mode 100644
index 3d545e5..0000000
--- a/debian/patches/001cassandra_yaml_dirs.dpatch
+++ /dev/null
@@ -1,36 +0,0 @@
-#! /bin/sh /usr/share/dpatch/dpatch-run
-## 001cassandra_yaml_dirs.dpatch by Tyler Hobbs <tyler@datastax.com>
-##
-## All lines beginning with `## DP:' are a description of the patch.
-## DP: No description.
-
-@DPATCH@
-diff -urNad '--exclude=CVS' '--exclude=.svn' '--exclude=.git' '--exclude=.arch' '--exclude=.hg' '--exclude=_darcs' '--exclude=.bzr' cassandra~/conf/cassandra.yaml cassandra/conf/cassandra.yaml
---- cassandra~/conf/cassandra.yaml 2014-06-05 13:36:22.000000000 -0500
-+++ cassandra/conf/cassandra.yaml 2014-06-05 13:39:20.569034040 -0500
-@@ -94,13 +94,13 @@
- # will spread data evenly across them, subject to the granularity of
- # the configured compaction strategy.
- # If not set, the default directory is $CASSANDRA_HOME/data/data.
--# data_file_directories:
--# - /var/lib/cassandra/data
-+data_file_directories:
-+ - /var/lib/cassandra/data
-
- # commit log. when running on magnetic HDD, this should be a
- # separate spindle than the data directories.
- # If not set, the default directory is $CASSANDRA_HOME/data/commitlog.
--# commitlog_directory: /var/lib/cassandra/commitlog
-+commitlog_directory: /var/lib/cassandra/commitlog
-
- # policy for data disk failures:
- # stop_paranoid: shut down gossip and Thrift even for single-sstable errors.
-@@ -203,7 +203,7 @@
-
- # saved caches
- # If not set, the default directory is $CASSANDRA_HOME/data/saved_caches.
--# saved_caches_directory: /var/lib/cassandra/saved_caches
-+saved_caches_directory: /var/lib/cassandra/saved_caches
-
- # commitlog_sync may be either "periodic" or "batch."
- # When in batch mode, Cassandra won't ack writes until the commit log
diff --git a/debian/patches/002cassandra_logdir_fix.dpatch b/debian/patches/002cassandra_logdir_fix.dpatch
deleted file mode 100755
index a0ff45f..0000000
--- a/debian/patches/002cassandra_logdir_fix.dpatch
+++ /dev/null
@@ -1,31 +0,0 @@
-#! /bin/sh /usr/share/dpatch/dpatch-run
-## cassandra_logdir_fix.dpatch by Michael Shuler <michael@pbandjelly.org>
-##
-## All lines beginning with `## DP:' are a description of the patch.
-## DP: No description.
-
-@DPATCH@
-diff -urNad '--exclude=CVS' '--exclude=.svn' '--exclude=.git' '--exclude=.arch' '--exclude=.hg' '--exclude=_darcs' '--exclude=.bzr' cassandra~/bin/cassandra cassandra/bin/cassandra
---- cassandra~/bin/cassandra 2019-06-27 09:35:32.000000000 -0500
-+++ cassandra/bin/cassandra 2019-06-27 09:43:28.756343141 -0500
-@@ -127,7 +127,7 @@
- fi
-
- if [ -z "$CASSANDRA_LOG_DIR" ]; then
-- CASSANDRA_LOG_DIR=$CASSANDRA_HOME/logs
-+ CASSANDRA_LOG_DIR=/var/log/cassandra
- fi
-
- # Special-case path variables.
-diff -urNad '--exclude=CVS' '--exclude=.svn' '--exclude=.git' '--exclude=.arch' '--exclude=.hg' '--exclude=_darcs' '--exclude=.bzr' cassandra~/conf/cassandra-env.sh cassandra/conf/cassandra-env.sh
---- cassandra~/conf/cassandra-env.sh 2019-06-27 09:35:32.000000000 -0500
-+++ cassandra/conf/cassandra-env.sh 2019-06-27 09:42:25.747715490 -0500
-@@ -123,7 +123,7 @@ esac
-
- # Sets the path where logback and GC logs are written.
- if [ "x$CASSANDRA_LOG_DIR" = "x" ] ; then
-- CASSANDRA_LOG_DIR="$CASSANDRA_HOME/logs"
-+ CASSANDRA_LOG_DIR="/var/log/cassandra"
- fi
-
- #GC log path has to be defined here because it needs to access CASSANDRA_HOME
\ No newline at end of file
diff --git a/debian/patches/00list b/debian/patches/00list
deleted file mode 100644
index 59b0d8b..0000000
--- a/debian/patches/00list
+++ /dev/null
@@ -1,2 +0,0 @@
-001cassandra_yaml_dirs.dpatch
-002cassandra_logdir_fix.dpatch
diff --git a/debian/patches/cassandra_in.sh_dirs.diff b/debian/patches/cassandra_in.sh_dirs.diff
new file mode 100644
index 0000000..6642165
--- /dev/null
+++ b/debian/patches/cassandra_in.sh_dirs.diff
@@ -0,0 +1,47 @@
+--- a/bin/cassandra.in.sh
++++ b/bin/cassandra.in.sh
+@@ -14,17 +14,17 @@
+ # See the License for the specific language governing permissions and
+ # limitations under the License.
+
+-if [ "x$CASSANDRA_HOME" = "x" ]; then
+- CASSANDRA_HOME="`dirname "$0"`/.."
+-fi
++CASSANDRA_HOME=/usr/share/cassandra
+
+ # The directory where Cassandra's configs live (required)
+-if [ "x$CASSANDRA_CONF" = "x" ]; then
+- CASSANDRA_CONF="$CASSANDRA_HOME/conf"
+-fi
++CASSANDRA_CONF=/etc/cassandra
+
+ # The java classpath (required)
+-CLASSPATH="$CASSANDRA_CONF"
++if [ -n "$CLASSPATH" ]; then
++ CLASSPATH=$CLASSPATH:$CASSANDRA_CONF
++else
++ CLASSPATH="$CASSANDRA_CONF"
++fi
+
+ # This can be the path to a jar file, or a directory containing the
+ # compiled classes. NOTE: This isn't needed by the startup script,
+@@ -38,7 +38,7 @@ fi
+
+ # the default location for commitlogs, sstables, and saved caches
+ # if not set in cassandra.yaml
+-cassandra_storagedir="$CASSANDRA_HOME/data"
++cassandra_storagedir=/var/lib/cassandra
+
+ # JAVA_HOME can optionally be set here
+ #JAVA_HOME=/usr/local/jdk6
+@@ -47,6 +47,10 @@ for jar in "$CASSANDRA_HOME"/lib/*.jar; do
+ CLASSPATH="$CLASSPATH:$jar"
+ done
+
++for jar in "$CASSANDRA_HOME"/*.jar; do
++ CLASSPATH="$CLASSPATH:$jar"
++done
++
+ CLASSPATH="$CLASSPATH:$EXTRA_CLASSPATH"
+
+ # JSR223 - collect all JSR223 engines' jars
diff --git a/debian/patches/cassandra_logdir_fix.diff b/debian/patches/cassandra_logdir_fix.diff
new file mode 100644
index 0000000..a13dec0
--- /dev/null
+++ b/debian/patches/cassandra_logdir_fix.diff
@@ -0,0 +1,22 @@
+--- a/bin/cassandra
++++ b/bin/cassandra
+@@ -109,7 +109,7 @@
+ fi
+
+ if [ -z "$CASSANDRA_LOG_DIR" ]; then
+- CASSANDRA_LOG_DIR=$CASSANDRA_HOME/logs
++ CASSANDRA_LOG_DIR=/var/log/cassandra
+ fi
+
+ # Special-case path variables.
+--- a/conf/cassandra-env.sh
++++ b/conf/cassandra-env.sh
+@@ -88,7 +88,7 @@ calculate_heap_sizes()
+
+ # Sets the path where logback and GC logs are written.
+ if [ "x$CASSANDRA_LOG_DIR" = "x" ] ; then
+- CASSANDRA_LOG_DIR="$CASSANDRA_HOME/logs"
++ CASSANDRA_LOG_DIR=/var/log/cassandra
+ fi
+
+ #GC log path has to be defined here because it needs to access CASSANDRA_HOME
diff --git a/debian/patches/cassandra_yaml_dirs.diff b/debian/patches/cassandra_yaml_dirs.diff
new file mode 100644
index 0000000..84219d0
--- /dev/null
+++ b/debian/patches/cassandra_yaml_dirs.diff
@@ -0,0 +1,28 @@
+--- a/conf/cassandra.yaml
++++ b/conf/cassandra.yaml
+@@ -197,13 +197,13 @@
+ # directories are specified, Cassandra will spread data evenly across
+ # them by partitioning the token ranges.
+ # If not set, the default directory is $CASSANDRA_HOME/data/data.
+-# data_file_directories:
+-# - /var/lib/cassandra/data
++data_file_directories:
++ - /var/lib/cassandra/data
+
+ # commit log. when running on magnetic HDD, this should be a
+ # separate spindle than the data directories.
+ # If not set, the default directory is $CASSANDRA_HOME/data/commitlog.
+-# commitlog_directory: /var/lib/cassandra/commitlog
++commitlog_directory: /var/lib/cassandra/commitlog
+
+ # Enable / disable CDC functionality on a per-node basis. This modifies the logic used
+ # for write path allocation rejection (standard: never reject. cdc: reject Mutation
+@@ -366,7 +366,7 @@
+
+ # saved caches
+ # If not set, the default directory is $CASSANDRA_HOME/data/saved_caches.
+-# saved_caches_directory: /var/lib/cassandra/saved_caches
++saved_caches_directory: /var/lib/cassandra/saved_caches
+
+ # commitlog_sync may be either "periodic", "group", or "batch."
+ #
diff --git a/debian/patches/series b/debian/patches/series
new file mode 100644
index 0000000..5e52a9c
--- /dev/null
+++ b/debian/patches/series
@@ -0,0 +1,3 @@
+cassandra_logdir_fix.diff
+cassandra_yaml_dirs.diff
+cassandra_in.sh_dirs.diff
diff --git a/debian/rules b/debian/rules
index ff1d64d..e7fa32f 100755
--- a/debian/rules
+++ b/debian/rules
@@ -3,7 +3,7 @@
# Uncomment to enable verbose mode.
#export DH_VERBOSE=1
-include /usr/share/dpatch/dpatch.make
+include /usr/share/quilt/quilt.make
ANT = /usr/bin/ant
VERSION = $(shell dpkg-parsechangelog | sed -ne 's/^Version: \([^-|~|+]*\).*/\1/p')
@@ -13,6 +13,7 @@
$(ANT) test
clean: unpatch
+ dh_clean build-stamp
dh_testdir
dh_testroot
$(ANT) realclean
@@ -24,7 +25,7 @@
dh_clean
build: build-stamp
-build-stamp: patch-stamp
+build-stamp: $(QUILT_STAMPFN)
dh_testdir
printf "version=%s" $(VERSION) > build.properties
@@ -45,13 +46,15 @@
# Copy in the jar and symlink to something stable
dh_install build/apache-cassandra-$(VERSION).jar \
usr/share/cassandra
- dh_install build/apache-cassandra-thrift-$(VERSION).jar \
- usr/share/cassandra
# Copy stress jars
dh_install build/tools/lib/stress.jar \
usr/share/cassandra
+ # Copy fqltool jars
+ dh_install build/tools/lib/fqltool.jar \
+ usr/share/cassandra
+
dh_link usr/share/cassandra/apache-cassandra-$(VERSION).jar \
usr/share/cassandra/apache-cassandra.jar
diff --git a/doc/Dockerfile b/doc/Dockerfile
new file mode 100644
index 0000000..fcb4c41
--- /dev/null
+++ b/doc/Dockerfile
@@ -0,0 +1,22 @@
+# Dockerfile for building the Cassandra documentation.
+# If wanting to regenerate the documentation from scratch,
+# run `ant realclean` from the root directory of this project.
+
+FROM python:2.7
+
+WORKDIR /usr/src/code
+
+RUN pip install --no-cache-dir sphinx sphinx_rtd_theme
+
+RUN apt-get update && apt-get install -y software-properties-common
+
+RUN wget -qO - https://adoptopenjdk.jfrog.io/adoptopenjdk/api/gpg/key/public | apt-key add - \
+ && add-apt-repository --yes https://adoptopenjdk.jfrog.io/adoptopenjdk/deb/ \
+ && apt-get update \
+ && apt-get install -y adoptopenjdk-11-hotspot ant
+
+
+RUN apt-get clean
+
+CMD CASSANDRA_USE_JDK11=true ant gen-doc \
+ && echo "The locally built documentation can be found here:\n\n build/html/index.html\n\n"
diff --git a/doc/Makefile b/doc/Makefile
index c6632a5..17ef395 100644
--- a/doc/Makefile
+++ b/doc/Makefile
@@ -19,6 +19,8 @@
MAKE_CASSANDRA_YAML = python convert_yaml_to_rst.py $(YAML_DOC_INPUT) $(YAML_DOC_OUTPUT)
+GENERATE_NODETOOL_DOCS = python gen-nodetool-docs.py
+
WEB_SITE_PRESENCE_FILE='source/.build_for_website'
.PHONY: help
@@ -60,6 +62,7 @@
.PHONY: html
html:
$(MAKE_CASSANDRA_YAML)
+ $(GENERATE_NODETOOL_DOCS)
$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
@echo
@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
@@ -68,6 +71,7 @@
website: clean
@touch $(WEB_SITE_PRESENCE_FILE)
$(MAKE_CASSANDRA_YAML)
+ $(GENERATE_NODETOOL_DOCS)
$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
@rm $(WEB_SITE_PRESENCE_FILE)
@echo
@@ -76,6 +80,7 @@
.PHONY: dirhtml
dirhtml:
$(MAKE_CASSANDRA_YAML)
+ $(GENERATE_NODETOOL_DOCS)
$(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml
@echo
@echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml."
@@ -83,6 +88,7 @@
.PHONY: singlehtml
singlehtml:
$(MAKE_CASSANDRA_YAML)
+ $(GENERATE_NODETOOL_DOCS)
$(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml
@echo
@echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml."
@@ -90,6 +96,7 @@
.PHONY: pickle
pickle:
$(MAKE_CASSANDRA_YAML)
+ $(GENERATE_NODETOOL_DOCS)
$(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle
@echo
@echo "Build finished; now you can process the pickle files."
@@ -97,6 +104,7 @@
.PHONY: json
json:
$(MAKE_CASSANDRA_YAML)
+ $(GENERATE_NODETOOL_DOCS)
$(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json
@echo
@echo "Build finished; now you can process the JSON files."
@@ -104,6 +112,7 @@
.PHONY: htmlhelp
htmlhelp:
$(MAKE_CASSANDRA_YAML)
+ $(GENERATE_NODETOOL_DOCS)
$(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp
@echo
@echo "Build finished; now you can run HTML Help Workshop with the" \
@@ -112,6 +121,7 @@
.PHONY: qthelp
qthelp:
$(MAKE_CASSANDRA_YAML)
+ $(GENERATE_NODETOOL_DOCS)
$(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp
@echo
@echo "Build finished; now you can run "qcollectiongenerator" with the" \
@@ -123,6 +133,7 @@
.PHONY: applehelp
applehelp:
$(MAKE_CASSANDRA_YAML)
+ $(GENERATE_NODETOOL_DOCS)
$(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp
@echo
@echo "Build finished. The help book is in $(BUILDDIR)/applehelp."
@@ -133,6 +144,7 @@
.PHONY: devhelp
devhelp:
$(MAKE_CASSANDRA_YAML)
+ $(GENERATE_NODETOOL_DOCS)
$(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp
@echo
@echo "Build finished."
@@ -144,6 +156,7 @@
.PHONY: epub
epub:
$(MAKE_CASSANDRA_YAML)
+ $(GENERATE_NODETOOL_DOCS)
$(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub
@echo
@echo "Build finished. The epub file is in $(BUILDDIR)/epub."
@@ -151,6 +164,7 @@
.PHONY: epub3
epub3:
$(MAKE_CASSANDRA_YAML)
+ $(GENERATE_NODETOOL_DOCS)
$(SPHINXBUILD) -b epub3 $(ALLSPHINXOPTS) $(BUILDDIR)/epub3
@echo
@echo "Build finished. The epub3 file is in $(BUILDDIR)/epub3."
@@ -158,6 +172,7 @@
.PHONY: latex
latex:
$(MAKE_CASSANDRA_YAML)
+ $(GENERATE_NODETOOL_DOCS)
$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
@echo
@echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex."
@@ -167,6 +182,7 @@
.PHONY: latexpdf
latexpdf:
$(MAKE_CASSANDRA_YAML)
+ $(GENERATE_NODETOOL_DOCS)
$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
@echo "Running LaTeX files through pdflatex..."
$(MAKE) -C $(BUILDDIR)/latex all-pdf
@@ -175,6 +191,7 @@
.PHONY: latexpdfja
latexpdfja:
$(MAKE_CASSANDRA_YAML)
+ $(GENERATE_NODETOOL_DOCS)
$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
@echo "Running LaTeX files through platex and dvipdfmx..."
$(MAKE) -C $(BUILDDIR)/latex all-pdf-ja
@@ -189,6 +206,7 @@
.PHONY: man
man:
$(MAKE_CASSANDRA_YAML)
+ $(GENERATE_NODETOOL_DOCS)
$(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man
@echo
@echo "Build finished. The manual pages are in $(BUILDDIR)/man."
@@ -196,6 +214,7 @@
.PHONY: texinfo
texinfo:
$(MAKE_CASSANDRA_YAML)
+ $(GENERATE_NODETOOL_DOCS)
$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
@echo
@echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo."
@@ -205,6 +224,7 @@
.PHONY: info
info:
$(MAKE_CASSANDRA_YAML)
+ $(GENERATE_NODETOOL_DOCS)
$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
@echo "Running Texinfo files through makeinfo..."
make -C $(BUILDDIR)/texinfo info
@@ -213,6 +233,7 @@
.PHONY: gettext
gettext:
$(MAKE_CASSANDRA_YAML)
+ $(GENERATE_NODETOOL_DOCS)
$(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale
@echo
@echo "Build finished. The message catalogs are in $(BUILDDIR)/locale."
@@ -220,6 +241,7 @@
.PHONY: changes
changes:
$(MAKE_CASSANDRA_YAML)
+ $(GENERATE_NODETOOL_DOCS)
$(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes
@echo
@echo "The overview file is in $(BUILDDIR)/changes."
@@ -227,6 +249,7 @@
.PHONY: linkcheck
linkcheck:
$(MAKE_CASSANDRA_YAML)
+ $(GENERATE_NODETOOL_DOCS)
$(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck
@echo
@echo "Link check complete; look for any errors in the above output " \
@@ -235,6 +258,7 @@
.PHONY: doctest
doctest:
$(MAKE_CASSANDRA_YAML)
+ $(GENERATE_NODETOOL_DOCS)
$(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest
@echo "Testing of doctests in the sources finished, look at the " \
"results in $(BUILDDIR)/doctest/output.txt."
@@ -242,6 +266,7 @@
.PHONY: coverage
coverage:
$(MAKE_CASSANDRA_YAML)
+ $(GENERATE_NODETOOL_DOCS)
$(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage
@echo "Testing of coverage in the sources finished, look at the " \
"results in $(BUILDDIR)/coverage/python.txt."
@@ -249,6 +274,7 @@
.PHONY: xml
xml:
$(MAKE_CASSANDRA_YAML)
+ $(GENERATE_NODETOOL_DOCS)
$(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml
@echo
@echo "Build finished. The XML files are in $(BUILDDIR)/xml."
@@ -256,6 +282,7 @@
.PHONY: pseudoxml
pseudoxml:
$(MAKE_CASSANDRA_YAML)
+ $(GENERATE_NODETOOL_DOCS)
$(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml
@echo
@echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml."
@@ -263,6 +290,7 @@
.PHONY: dummy
dummy:
$(MAKE_CASSANDRA_YAML)
+ $(GENERATE_NODETOOL_DOCS)
$(SPHINXBUILD) -b dummy $(ALLSPHINXOPTS) $(BUILDDIR)/dummy
@echo
@echo "Build finished. Dummy builder generates no files."
diff --git a/doc/README.md b/doc/README.md
index 9ba47a1..eeb5a1c 100644
--- a/doc/README.md
+++ b/doc/README.md
@@ -20,12 +20,35 @@
and is thus written in [reStructuredText](http://docutils.sourceforge.net/rst.html).
To build the HTML documentation, you will need to first install sphinx and the
-[sphinx ReadTheDocs theme](the https://pypi.python.org/pypi/sphinx_rtd_theme), which
-on unix you can do with:
+[sphinx ReadTheDocs theme](https://pypi.org/project/sphinx_rtd_theme/).
+When using Python 3.6 on Windows, use `py -m pip install sphinx sphinx_rtd_theme`, on unix
+use:
```
pip install sphinx sphinx_rtd_theme
```
The documentation can then be built from this directory by calling `make html`
(or `make.bat html` on windows). Alternatively, the top-level `ant gen-doc`
-target can be used.
+target can be used. When using Python 3.6 on Windows, use `sphinx_build -b html source build`.
+
+To build the documentation with Docker Compose, run:
+
+```bash
+cd ./doc
+
+# build the Docker image
+docker-compose build build-docs
+
+# build the documentation
+docker-compose run build-docs
+```
+
+To regenerate the documentation from scratch, run:
+
+```bash
+# return to the root directory of the Cassandra project
+cd ..
+
+# remove all generated documentation files based on the source code
+ant realclean
+```
diff --git a/doc/SASI.md b/doc/SASI.md
index a4762c9..30346ab 100644
--- a/doc/SASI.md
+++ b/doc/SASI.md
@@ -19,8 +19,7 @@
## Using SASI
The examples below walk through creating a table and indexes on its
-columns, and performing queries on some inserted data. The patchset in
-this repository includes support for the Thrift and CQL3 interfaces.
+columns, and performing queries on some inserted data.
The examples below assume the `demo` keyspace has been created and is
in use.
@@ -203,7 +202,7 @@
#### Suffix Queries
The next example demonstrates `CONTAINS` mode on the `last_name`
-column. By using this mode predicates can search for any strings
+column. By using this mode, predicates can search for any strings
containing the search string as a sub-string. In this case the strings
containing "a" or "an".
@@ -248,6 +247,27 @@
(4 rows)
```
+#### Delimiter based Tokenization Analysis
+
+A simple text analysis provided is delimiter based tokenization. This provides an alternative to indexing collections,
+as delimiter separated text can be indexed without the overhead of `CONTAINS` mode nor using `PREFIX` or `SUFFIX` queries.
+
+```
+cqlsh:demo> ALTER TABLE sasi ADD aliases text;
+cqlsh:demo> CREATE CUSTOM INDEX on sasi (aliases) USING 'org.apache.cassandra.index.sasi.SASIIndex'
+ ... WITH OPTIONS = {
+ ... 'analyzer_class': 'org.apache.cassandra.index.sasi.analyzer.DelimiterAnalyzer',
+ ... 'delimiter': ',',
+ ... 'mode': 'prefix',
+ ... 'analyzed': 'true'};
+cqlsh:demo> UPDATE sasi SET aliases = 'Mike,Mick,Mikey,Mickey' WHERE id = f5dfcabe-de96-4148-9b80-a1c41ed276b4;
+cqlsh:demo> SELECT * FROM sasi WHERE aliases LIKE 'Mikey' ALLOW FILTERING;
+
+ id | age | aliases | created_at | first_name | height | last_name
+--------------------------------------+-----+------------------------+---------------+------------+--------+-----------
+ f5dfcabe-de96-4148-9b80-a1c41ed276b4 | 26 | Mike,Mick,Mikey,Mickey | 1442959315021 | Michael | 180 | Kjellman
+```
+
#### Text Analysis (Tokenization and Stemming)
Lastly, to demonstrate text analysis an additional column is needed on
@@ -331,7 +351,7 @@
While SASI, at the surface, is simply an implementation of the
`Index` interface, at its core there are several data
structures and algorithms used to satisfy it. These are described
-here. Additionally, the changes internal to Cassandra to support SASIs
+here. Additionally, the changes internal to Cassandra to support SASI's
integration are described.
The `Index` interface divides responsibility of the
@@ -350,7 +370,7 @@
usage. These data structures are optimized for this use case.
Taking advantage of Cassandra's ordered data model, at query time,
-candidate indexes are narrowed down for searching minimize the amount
+candidate indexes are narrowed down for searching, minimizing the amount
of work done. Searching is then performed using an efficient method
that streams data off disk as needed.
@@ -369,7 +389,7 @@
for writing, and
[`TokenTree`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/disk/TokenTree.java)s
for querying. These index files are memory mapped after being written
-to disk, for quicker access. For indexing data in the memtable SASI
+to disk, for quicker access. For indexing data in the memtable, SASI
uses its
[`IndexMemtable`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/memory/IndexMemtable.java)
class.
@@ -396,8 +416,8 @@
The terms written to the
[`OnDiskIndex`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/disk/OnDiskIndex.java)
vary depending on its "mode": either `PREFIX`, `CONTAINS`, or
-`SPARSE`. In the `PREFIX` and `SPARSE` cases terms exact values are
-written exactly once per `OnDiskIndex`. For example, a `PREFIX` index
+`SPARSE`. In the `PREFIX` and `SPARSE` cases, terms' exact values are
+written exactly once per `OnDiskIndex`. For example, when using a `PREFIX` index
with terms `Jason`, `Jordan`, `Pavel`, all three will be included in
the index. A `CONTAINS` index writes additional terms for each suffix of
each term recursively. Continuing with the example, a `CONTAINS` index
@@ -431,7 +451,7 @@
the well-known algorithm optimized for bulk-loading the data
structure.
-[`TokenTree`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/disk/TokenTree.java)s provide the means to iterate a tokens, and file
+[`TokenTree`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/disk/TokenTree.java)s provide the means to iterate over tokens, and file
positions, that match a given term, and to skip forward in that
iteration, an operation used heavily at query time.
@@ -449,7 +469,7 @@
dependent. The
[`TrieMemIndex`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/memory/TrieMemIndex.java)
is used for literal types. `AsciiType` and `UTF8Type` are literal
-types by defualt but any column can be configured as a literal type
+types by default but any column can be configured as a literal type
using the `is_literal` option at index creation time. For non-literal
types the
[`SkipListMemIndex`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/memory/SkipListMemIndex.java)
@@ -458,7 +478,7 @@
is an implementation that can efficiently support prefix queries on
character-like data. The
[`SkipListMemIndex`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/memory/SkipListMemIndex.java),
-conversely, is better suited for Cassandra other data types like
+conversely, is better suited for other Cassandra data types like
numbers.
The
@@ -479,25 +499,25 @@
[`Operation`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/Operation.java)
and
[`Expression`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/Expression.java)
-tree, optimizing the tree to reduce the amount of work done, and
-driving the query itself the
+trees, optimizing the trees to reduce the amount of work done, and
+driving the query itself, the
[`QueryPlan`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/QueryPlan.java)
is the work horse of SASI's querying implementation. To efficiently
-perform union and intersection operations SASI provides several
-iterators similar to Cassandra's `MergeIterator` but tailored
-specifically for SASIs use, and with more features. The
+perform union and intersection operations, SASI provides several
+iterators similar to Cassandra's `MergeIterator`, but tailored
+specifically for SASI's use while including more features. The
[`RangeUnionIterator`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/utils/RangeUnionIterator.java),
-like its name suggests, performs set union over sets of tokens/keys
+like its name suggests, performs set unions over sets of tokens/keys
matching the query, only reading as much data as it needs from each
set to satisfy the query. The
[`RangeIntersectionIterator`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/utils/RangeIntersectionIterator.java),
-similar to its counterpart, performs set intersection over its data.
+similar to its counterpart, performs set intersections over its data.
#### QueryPlan
The
[`QueryPlan`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/QueryPlan.java)
-instantiated per search query is at the core of SASIs querying
+instantiated per search query is at the core of SASI's querying
implementation. Its work can be divided in two stages: analysis and
execution.
@@ -512,7 +532,7 @@
[`Operation`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/Operation.java)s, which in turn may contain [`Expression`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/Expression.java)s, all of which
provide an alternative, more efficient, representation of the query.
-During execution the
+During execution, the
[`QueryPlan`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/QueryPlan.java)
uses the `DecoratedKey`-generating iterator created from the
[`Operation`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/Operation.java) tree. These keys are read from disk and a final check to
@@ -525,7 +545,7 @@
are maintined per-table/column family.
SASI also supports concurrently iterating terms for the same index
-accross SSTables. The concurrency factor is controlled by the
+across SSTables. The concurrency factor is controlled by the
`cassandra.search_concurrency_factor` system property. The default is
`1`.
@@ -538,7 +558,7 @@
used throughout the execution phase. The
[`QueryController`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/QueryController.java)
has two responsibilities: to manage and ensure the proper cleanup of
-resources (indexes), and to strictly enforce the time bound for query,
+resources (indexes), and to strictly enforce the time bound per query,
specified by the user via the range slice timeout. All indexes are
accessed via the
[`QueryController`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/QueryController.java)
@@ -556,7 +576,7 @@
the execution phase.
The simplest optimization performed is compacting multiple expressions
-joined by logical intersection (`AND`) into a single [`Operation`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/Operation.java) with
+joined by logical intersections (`AND`) into a single [`Operation`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/Operation.java) with
three or more [`Expression`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/Expression.java)s. For example, the query `WHERE age < 100 AND
fname = 'p*' AND first_name != 'pa*' AND age > 21` would,
without modification, have the following tree:
@@ -660,11 +680,11 @@
[`QueryPlan`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/QueryPlan.java),
[`Operation`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/Operation.java)
is also responsible for taking a row that has been returned by the
-query and making a final validation that it in fact does match. This
+query and performing a final validation that it in fact does match. This
`satisfiesBy` operation is performed recursively from the root of the
[`Operation`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/Operation.java)
tree for a given query. These checks are performed directly on the
-data in a given row. For more details on how `satisfiesBy` works see
+data in a given row. For more details on how `satisfiesBy` works, see
the documentation
[in the code](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/Operation.java#L87-L123).
@@ -736,7 +756,7 @@
[`PerSSTableIndexWriter`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/disk/PerSSTableIndexWriter.java), and initiates searches with
`Searcher`. These classes glue the previously
mentioned indexing components together with Cassandra's SSTable
-life-cycle ensuring indexes are not only written when Memtable's flush
+life-cycle ensuring indexes are not only written when Memtable's flush,
but also as SSTable's are compacted. For querying, the
`Searcher` does little but defer to
[`QueryPlan`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/QueryPlan.java)
diff --git a/doc/convert_yaml_to_rst.py b/doc/convert_yaml_to_rst.py
index c17bbbb..4bdcf5b 100644
--- a/doc/convert_yaml_to_rst.py
+++ b/doc/convert_yaml_to_rst.py
@@ -41,7 +41,6 @@
# that these can be commented out (making it useless to use a yaml parser).
COMPLEX_OPTIONS = (
'seed_provider',
- 'request_scheduler_options',
'data_file_directories',
'commitlog_compression',
'hints_compression',
diff --git a/doc/cql3/CQL.textile b/doc/cql3/CQL.textile
index 735cfe1..fafca14 100644
--- a/doc/cql3/CQL.textile
+++ b/doc/cql3/CQL.textile
@@ -175,7 +175,6 @@
The @replication@ @<property>@ is mandatory. It must at least contains the @'class'@ sub-option which defines the replication strategy class to use. The rest of the sub-options depends on that replication strategy class. By default, Cassandra support the following @'class'@:
* @'SimpleStrategy'@: A simple strategy that defines a simple replication factor for the whole cluster. The only sub-options supported is @'replication_factor'@ to define that replication factor and is mandatory.
* @'NetworkTopologyStrategy'@: A replication strategy that allows to set the replication factor independently for each data-center. The rest of the sub-options are key-value pairs where each time the key is the name of a datacenter and the value the replication factor for that data-center.
-* @'OldNetworkTopologyStrategy'@: A legacy replication strategy. You should avoid this strategy for new keyspaces and prefer @'NetworkTopologyStrategy'@.
Attempting to create an already existing keyspace will return an error unless the @IF NOT EXISTS@ option is used. If it is used, the statement will be a no-op if the keyspace already exists.
@@ -250,8 +249,7 @@
common_name text,
population varint,
average_size int
-) WITH comment='Important biological records'
- AND read_repair_chance = 1.0;
+) WITH comment='Important biological records';
CREATE TABLE timeline (
userid uuid,
@@ -334,8 +332,6 @@
|_. option |_. kind |_. default |_. description|
|@comment@ | _simple_ | none | A free-form, human-readable comment.|
-|@read_repair_chance@ | _simple_ | 0.1 | The probability with which to query extra nodes (e.g. more nodes than required by the consistency level) for the purpose of read repairs.|
-|@dclocal_read_repair_chance@ | _simple_ | 0 | The probability with which to query extra nodes (e.g. more nodes than required by the consistency level) belonging to the same data center than the read coordinator for the purpose of read repairs.|
|@gc_grace_seconds@ | _simple_ | 864000 | Time to wait before garbage collecting tombstones (deletion markers).|
|@bloom_filter_fp_chance@ | _simple_ | 0.00075 | The target probability of false positive of the sstable bloom filters. Said bloom filters will be sized to provide the provided probability (thus lowering this value impact the size of bloom filters in-memory and on-disk)|
|@default_time_to_live@ | _simple_ | 0 | The default expiration time ("TTL") in seconds for a table.|
@@ -347,24 +343,24 @@
The @compaction@ property must at least define the @'class'@ sub-option, that defines the compaction strategy class to use. The default supported class are @'SizeTieredCompactionStrategy'@, @'LeveledCompactionStrategy'@, @'DateTieredCompactionStrategy'@ and @'TimeWindowCompactionStrategy'@. Custom strategy can be provided by specifying the full class name as a "string constant":#constants. The rest of the sub-options depends on the chosen class. The sub-options supported by the default classes are:
-|_. option |_. supported compaction strategy |_. default |_. description |
-| @enabled@ | _all_ | true | A boolean denoting whether compaction should be enabled or not.|
-| @tombstone_threshold@ | _all_ | 0.2 | A ratio such that if a sstable has more than this ratio of gcable tombstones over all contained columns, the sstable will be compacted (with no other sstables) for the purpose of purging those tombstones. |
-| @tombstone_compaction_interval@ | _all_ | 1 day | The minimum time to wait after an sstable creation time before considering it for "tombstone compaction", where "tombstone compaction" is the compaction triggered if the sstable has more gcable tombstones than @tombstone_threshold@. |
-| @unchecked_tombstone_compaction@ | _all_ | false | Setting this to true enables more aggressive tombstone compactions - single sstable tombstone compactions will run without checking how likely it is that they will be successful. |
-| @min_sstable_size@ | SizeTieredCompactionStrategy | 50MB | The size tiered strategy groups SSTables to compact in buckets. A bucket groups SSTables that differs from less than 50% in size. However, for small sizes, this would result in a bucketing that is too fine grained. @min_sstable_size@ defines a size threshold (in bytes) below which all SSTables belong to one unique bucket|
-| @min_threshold@ | SizeTieredCompactionStrategy | 4 | Minimum number of SSTables needed to start a minor compaction.|
-| @max_threshold@ | SizeTieredCompactionStrategy | 32 | Maximum number of SSTables processed by one minor compaction.|
-| @bucket_low@ | SizeTieredCompactionStrategy | 0.5 | Size tiered consider sstables to be within the same bucket if their size is within [average_size * @bucket_low@, average_size * @bucket_high@ ] (i.e the default groups sstable whose sizes diverges by at most 50%)|
-| @bucket_high@ | SizeTieredCompactionStrategy | 1.5 | Size tiered consider sstables to be within the same bucket if their size is within [average_size * @bucket_low@, average_size * @bucket_high@ ] (i.e the default groups sstable whose sizes diverges by at most 50%).|
-| @sstable_size_in_mb@ | LeveledCompactionStrategy | 5MB | The target size (in MB) for sstables in the leveled strategy. Note that while sstable sizes should stay less or equal to @sstable_size_in_mb@, it is possible to exceptionally have a larger sstable as during compaction, data for a given partition key are never split into 2 sstables|
-| @timestamp_resolution@ | DateTieredCompactionStrategy | MICROSECONDS | The timestamp resolution used when inserting data, could be MILLISECONDS, MICROSECONDS etc (should be understandable by Java TimeUnit) - don't change this unless you do mutations with USING TIMESTAMP <non_microsecond_timestamps> (or equivalent directly in the client)|
-| @base_time_seconds@ | DateTieredCompactionStrategy | 60 | The base size of the time windows. |
-| @max_sstable_age_days@ | DateTieredCompactionStrategy | 365 | SSTables only containing data that is older than this will never be compacted. |
-| @timestamp_resolution@ | TimeWindowCompactionStrategy | MICROSECONDS | The timestamp resolution used when inserting data, could be MILLISECONDS, MICROSECONDS etc (should be understandable by Java TimeUnit) - don't change this unless you do mutations with USING TIMESTAMP <non_microsecond_timestamps> (or equivalent directly in the client)|
-| @compaction_window_unit@ | TimeWindowCompactionStrategy | DAYS | The Java TimeUnit used for the window size, set in conjunction with @compaction_window_size@. Must be one of DAYS, HOURS, MINUTES |
-| @compaction_window_size@ | TimeWindowCompactionStrategy | 1 | The number of @compaction_window_unit@ units that make up a time window. |
-
+|_. option |_. supported compaction strategy |_. default |_. description |
+| @enabled@ | _all_ | true | A boolean denoting whether compaction should be enabled or not.|
+| @tombstone_threshold@ | _all_ | 0.2 | A ratio such that if a sstable has more than this ratio of gcable tombstones over all contained columns, the sstable will be compacted (with no other sstables) for the purpose of purging those tombstones. |
+| @tombstone_compaction_interval@ | _all_ | 1 day | The minimum time to wait after an sstable creation time before considering it for "tombstone compaction", where "tombstone compaction" is the compaction triggered if the sstable has more gcable tombstones than @tombstone_threshold@. |
+| @unchecked_tombstone_compaction@ | _all_ | false | Setting this to true enables more aggressive tombstone compactions - single sstable tombstone compactions will run without checking how likely it is that they will be successful. |
+| @min_sstable_size@ | SizeTieredCompactionStrategy | 50MB | The size tiered strategy groups SSTables to compact in buckets. A bucket groups SSTables that differs from less than 50% in size. However, for small sizes, this would result in a bucketing that is too fine grained. @min_sstable_size@ defines a size threshold (in bytes) below which all SSTables belong to one unique bucket|
+| @min_threshold@ | SizeTieredCompactionStrategy | 4 | Minimum number of SSTables needed to start a minor compaction.|
+| @max_threshold@ | SizeTieredCompactionStrategy | 32 | Maximum number of SSTables processed by one minor compaction.|
+| @bucket_low@ | SizeTieredCompactionStrategy | 0.5 | Size tiered consider sstables to be within the same bucket if their size is within [average_size * @bucket_low@, average_size * @bucket_high@ ] (i.e the default groups sstable whose sizes diverges by at most 50%)|
+| @bucket_high@ | SizeTieredCompactionStrategy | 1.5 | Size tiered consider sstables to be within the same bucket if their size is within [average_size * @bucket_low@, average_size * @bucket_high@ ] (i.e the default groups sstable whose sizes diverges by at most 50%).|
+| @sstable_size_in_mb@ | LeveledCompactionStrategy | 5MB | The target size (in MB) for sstables in the leveled strategy. Note that while sstable sizes should stay less or equal to @sstable_size_in_mb@, it is possible to exceptionally have a larger sstable as during compaction, data for a given partition key are never split into 2 sstables|
+| @timestamp_resolution@ | DateTieredCompactionStrategy | MICROSECONDS | The timestamp resolution used when inserting data, could be MILLISECONDS, MICROSECONDS etc (should be understandable by Java TimeUnit) - don't change this unless you do mutations with USING TIMESTAMP <non_microsecond_timestamps> (or equivalent directly in the client)|
+| @base_time_seconds@ | DateTieredCompactionStrategy | 60 | The base size of the time windows. |
+| @max_sstable_age_days@ | DateTieredCompactionStrategy | 365 | SSTables only containing data that is older than this will never be compacted. |
+| @timestamp_resolution@ | TimeWindowCompactionStrategy | MICROSECONDS | The timestamp resolution used when inserting data, could be MILLISECONDS, MICROSECONDS etc (should be understandable by Java TimeUnit) - don't change this unless you do mutations with USING TIMESTAMP <non_microsecond_timestamps> (or equivalent directly in the client)|
+| @compaction_window_unit@ | TimeWindowCompactionStrategy | DAYS | The Java TimeUnit used for the window size, set in conjunction with @compaction_window_size@. Must be one of DAYS, HOURS, MINUTES |
+| @compaction_window_size@ | TimeWindowCompactionStrategy | 1 | The number of @compaction_window_unit@ units that make up a time window. |
+| @unsafe_aggressive_sstable_expiration@ | TimeWindowCompactionStrategy | false | Expired sstables will be dropped without checking its data is shadowing other sstables. This is a potentially risky option that can lead to data loss or deleted data re-appearing, going beyond what `unchecked_tombstone_compaction` does for single sstable compaction. Due to the risk the jvm must also be started with `-Dcassandra.unsafe_aggressive_sstable_expiration=true`. |
h4(#compressionOptions). Compression options
@@ -411,8 +407,7 @@
ADD gravesite varchar;
ALTER TABLE addamsFamily
-WITH comment = 'A most excellent and useful column family'
- AND read_repair_chance = 0.2;
+WITH comment = 'A most excellent and useful column family';
p.
The @ALTER@ statement is used to manipulate table definitions. It allows for adding new columns, dropping existing ones, or updating the table options. As with table creation, @ALTER COLUMNFAMILY@ is allowed as an alias for @ALTER TABLE@.
@@ -547,12 +542,6 @@
The @<where-clause>@ is similar to the "where clause of a @SELECT@ statement":#selectWhere, with a few differences. First, the where clause must contain an expression that disallows @NULL@ values in columns in the view's primary key. If no other restriction is desired, this can be accomplished with an @IS NOT NULL@ expression. Second, only columns which are in the base table's primary key may be restricted with expressions other than @IS NOT NULL@. (Note that this second restriction may be lifted in the future.)
-h4. MV Limitations
-
-__Note:__
-Removal of columns not selected in the Materialized View (via `UPDATE base SET unselected_column = null` or `DELETE unselected_column FROM base`) may shadow missed updates to other columns received by hints or repair.
-For this reason, we advise against doing deletions on base columns not selected in views until this is fixed on CASSANDRA-13826.
-
h3(#alterMVStmt). ALTER MATERIALIZED VIEW
__Syntax:__
@@ -1076,6 +1065,9 @@
| TTL '(' <identifier> ')'
| CAST '(' <selector> AS <type> ')'
| <function> '(' (<selector> (',' <selector>)*)? ')'
+ | <selector> '.' <identifier>
+ | <selector> '[' <term> ']'
+ | <selector> '[' <term>? .. <term>? ']'
<where-clause> ::= <relation> ( AND <relation> )*
@@ -1119,6 +1111,8 @@
A @<selector>@ is either a column name to retrieve or a @<function>@ of one or more @<term>@s. The function allowed are the same as for @<term>@ and are described in the "function section":#functions. In addition to these generic functions, the @WRITETIME@ (resp. @TTL@) function allows to select the timestamp of when the column was inserted (resp. the time to live (in seconds) for the column (or null if the column has no expiration set)) and the "@CAST@":#castFun function can be used to convert one data type to another.
+Additionally, individual values of maps and sets can be selected using @[ <term> ]@. For maps, this will return the value corresponding to the key, if such entry exists. For sets, this will return the key that is selected if it exists and is thus mainly a way to check element existence. It is also possible to select a slice of a set or map with @[ <term> ... <term> @], where both bound can be omitted.
+
Any @<selector>@ can be aliased using @AS@ keyword (see examples). Please note that @<where-clause>@ and @<order-by>@ clause should refer to the columns by their original names and not by their aliases.
The @COUNT@ keyword can be used with parenthesis enclosing @*@. If so, the query will return a single result: the number of rows matching the query. Note that @COUNT(1)@ is supported as an alias.
@@ -2441,6 +2435,10 @@
h3. 3.4.2
+* Support for selecting elements and slices of a collection ("CASSANDRA-7396":https://issues.apache.org/jira/browse/CASSANDRA-7396).
+
+h3. 3.4.2
+
* "@INSERT/UPDATE options@":#updateOptions for tables having a default_time_to_live specifying a TTL of 0 will remove the TTL from the inserted or updated values
* "@ALTER TABLE@":#alterTableStmt @ADD@ and @DROP@ now allow mutiple columns to be added/removed
* New "@PER PARTITION LIMIT@":#selectLimit option (see "CASSANDRA-7017":https://issues.apache.org/jira/browse/CASSANDRA-7017).
diff --git a/doc/docker-compose.yml b/doc/docker-compose.yml
new file mode 100644
index 0000000..392f8d8
--- /dev/null
+++ b/doc/docker-compose.yml
@@ -0,0 +1,11 @@
+# docker-compose.yml for building the Cassandra documentation.
+
+version: '2.0'
+
+services:
+ build-docs:
+ build: .
+ volumes:
+ - ..:/usr/src/code
+ environment:
+ - SKIP_NODETOOL # set this to skip nodetool build, saves a lot of time when debugging html
diff --git a/doc/gen-nodetool-docs.py b/doc/gen-nodetool-docs.py
new file mode 100644
index 0000000..cc784c2
--- /dev/null
+++ b/doc/gen-nodetool-docs.py
@@ -0,0 +1,82 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+A script to use nodetool to generate documentation for nodetool
+"""
+from __future__ import print_function
+
+import os
+import re
+import sys
+import subprocess
+from subprocess import PIPE
+from subprocess import Popen
+
+if(os.environ.get("SKIP_NODETOOL") == "1"):
+ sys.exit(0)
+
+
+nodetool = "../bin/nodetool"
+outdir = "source/tools/nodetool"
+helpfilename = outdir + "/nodetool.txt"
+command_re = re.compile("( )([_a-z]+)")
+commandRSTContent = ".. _nodetool_{0}:\n\n{0}\n{1}\n\nUsage\n---------\n\n.. include:: {0}.txt\n :literal:\n\n"
+
+# create the documentation directory
+if not os.path.exists(outdir):
+ os.makedirs(outdir)
+
+# create the base help file to use for discovering the commands
+def create_help_file():
+ with open(helpfilename, "w+") as output_file:
+ try:
+ subprocess.check_call([nodetool, "help"], stdout=output_file)
+ except subprocess.CalledProcessError as cpe:
+ print(
+ 'ERROR: Nodetool failed to run, you likely need to build '
+ 'cassandra using ant jar from the top level directory'
+ )
+ raise cpe
+
+# for a given command, create the help file and an RST file to contain it
+def create_rst(command):
+ if command:
+ cmdName = command.group(0).strip()
+ cmdFilename = outdir + "/" + cmdName + ".txt"
+ rstFilename = outdir + "/" + cmdName + ".rst"
+ with open(cmdFilename, "w+") as cmdFile:
+ proc = Popen([nodetool, "help", cmdName], stdin=PIPE, stdout=PIPE)
+ (out, err) = proc.communicate()
+ cmdFile.write(out)
+ with open(rstFilename, "w+") as rstFile:
+ rstFile.write(commandRSTContent.format(cmdName, '-' * len(cmdName)))
+
+# create base file
+create_help_file()
+
+# create the main usage page
+with open(outdir + "/nodetool.rst", "w+") as output:
+ with open(helpfilename, "r+") as helpfile:
+ output.write(".. _nodetool\n\nNodetool\n--------\n\nUsage\n---------\n\n")
+ for commandLine in helpfile:
+ command = command_re.sub(r'\n\1:doc:`\2` - ',commandLine)
+ output.write(command)
+
+# create the command usage pages
+with open(helpfilename, "rw+") as helpfile:
+ for commandLine in helpfile:
+ command = command_re.match(commandLine)
+ create_rst(command)
diff --git a/doc/native_protocol_v5.spec b/doc/native_protocol_v5.spec
index ada0e9f..d279453 100644
--- a/doc/native_protocol_v5.spec
+++ b/doc/native_protocol_v5.spec
@@ -93,8 +93,8 @@
it is moving. The rest of that byte is the protocol version (5 for the protocol
defined in this document). In other words, for this version of the protocol,
version will be one of:
- 0x04 Request frame for this protocol version
- 0x84 Response frame for this protocol version
+ 0x05 Request frame for this protocol version
+ 0x85 Response frame for this protocol version
Please note that while every message ships with the version, only one version
of messages is accepted on a given connection. In other words, the first message
@@ -332,49 +332,57 @@
<query><query_parameters>
where <query> is a [long string] representing the query and
<query_parameters> must be
- <consistency><flags>[<n>[name_1]<value_1>...[name_n]<value_n>][<result_page_size>][<paging_state>][<serial_consistency>][<timestamp>]
+ <consistency><flags>[<n>[name_1]<value_1>...[name_n]<value_n>][<result_page_size>][<paging_state>][<serial_consistency>][<timestamp>][<keyspace>][<now_in_seconds>]
where:
- <consistency> is the [consistency] level for the operation.
- <flags> is a [int] whose bits define the options for this query and
in particular influence what the remainder of the message contains.
A flag is set if the bit corresponding to its `mask` is set. Supported
flags are, given their mask:
- 0x01: Values. If set, a [short] <n> followed by <n> [value]
- values are provided. Those values are used for bound variables in
- the query. Optionally, if the 0x40 flag is present, each value
- will be preceded by a [string] name, representing the name of
- the marker the value must be bound to.
- 0x02: Skip_metadata. If set, the Result Set returned as a response
- to the query (if any) will have the NO_METADATA flag (see
- Section 4.2.5.2).
- 0x04: Page_size. If set, <result_page_size> is an [int]
- controlling the desired page size of the result (in CQL3 rows).
- See the section on paging (Section 8) for more details.
- 0x08: With_paging_state. If set, <paging_state> should be present.
- <paging_state> is a [bytes] value that should have been returned
- in a result set (Section 4.2.5.2). The query will be
- executed but starting from a given paging state. This is also to
- continue paging on a different node than the one where it
- started (See Section 8 for more details).
- 0x10: With serial consistency. If set, <serial_consistency> should be
- present. <serial_consistency> is the [consistency] level for the
- serial phase of conditional updates. That consitency can only be
- either SERIAL or LOCAL_SERIAL and if not present, it defaults to
- SERIAL. This option will be ignored for anything else other than a
- conditional update/insert.
- 0x20: With default timestamp. If set, <timestamp> should be present.
- <timestamp> is a [long] representing the default timestamp for the query
- in microseconds (negative values are forbidden). This will
- replace the server side assigned timestamp as default timestamp.
- Note that a timestamp in the query itself will still override
- this timestamp. This is entirely optional.
- 0x40: With names for values. This only makes sense if the 0x01 flag is set and
- is ignored otherwise. If present, the values from the 0x01 flag will
- be preceded by a name (see above). Note that this is only useful for
- QUERY requests where named bind markers are used; for EXECUTE statements,
- since the names for the expected values was returned during preparation,
- a client can always provide values in the right order without any names
- and using this flag, while supported, is almost surely inefficient.
+ 0x0001: Values. If set, a [short] <n> followed by <n> [value]
+ values are provided. Those values are used for bound variables in
+ the query. Optionally, if the 0x40 flag is present, each value
+ will be preceded by a [string] name, representing the name of
+ the marker the value must be bound to.
+ 0x0002: Skip_metadata. If set, the Result Set returned as a response
+ to the query (if any) will have the NO_METADATA flag (see
+ Section 4.2.5.2).
+ 0x0004: Page_size. If set, <result_page_size> is an [int]
+ controlling the desired page size of the result (in CQL3 rows).
+ See the section on paging (Section 8) for more details.
+ 0x0008: With_paging_state. If set, <paging_state> should be present.
+ <paging_state> is a [bytes] value that should have been returned
+ in a result set (Section 4.2.5.2). The query will be
+ executed but starting from a given paging state. This is also to
+ continue paging on a different node than the one where it
+ started (See Section 8 for more details).
+ 0x0010: With serial consistency. If set, <serial_consistency> should be
+ present. <serial_consistency> is the [consistency] level for the
+ serial phase of conditional updates. That consitency can only be
+ either SERIAL or LOCAL_SERIAL and if not present, it defaults to
+ SERIAL. This option will be ignored for anything else other than a
+ conditional update/insert.
+ 0x0020: With default timestamp. If set, <timestamp> must be present.
+ <timestamp> is a [long] representing the default timestamp for the query
+ in microseconds (negative values are forbidden). This will
+ replace the server side assigned timestamp as default timestamp.
+ Note that a timestamp in the query itself will still override
+ this timestamp. This is entirely optional.
+ 0x0040: With names for values. This only makes sense if the 0x01 flag is set and
+ is ignored otherwise. If present, the values from the 0x01 flag will
+ be preceded by a name (see above). Note that this is only useful for
+ QUERY requests where named bind markers are used; for EXECUTE statements,
+ since the names for the expected values was returned during preparation,
+ a client can always provide values in the right order without any names
+ and using this flag, while supported, is almost surely inefficient.
+ 0x0080: With keyspace. If set, <keyspace> must be present. <keyspace> is a
+ [string] indicating the keyspace that the query should be executed in.
+ It supercedes the keyspace that the connection is bound to, if any.
+ 0x0100: With now in seconds. If set, <now_in_seconds> must be present.
+ <now_in_seconds> is an [int] representing the current time (now) for
+ the query. Affects TTL cell liveness in read queries and local deletion
+ time for tombstones and TTL cells in update requests. It's intended
+ for testing purposes and is optional.
Note that the consistency is ignored by some queries (USE, CREATE, ALTER,
TRUNCATE, ...).
@@ -385,8 +393,17 @@
4.1.5. PREPARE
- Prepare a query for later execution (through EXECUTE). The body consists of
- the CQL query to prepare as a [long string].
+ Prepare a query for later execution (through EXECUTE). The body of the message must be:
+ <query><flags>[<keyspace>]
+ where:
+ - <query> is a [long string] representing the CQL query.
+ - <flags> is a [int] whose bits define the options for this statement and in particular
+ influence what the remainder of the message contains.
+ A flag is set if the bit corresponding to its `mask` is set. Supported
+ flags are, given their mask:
+ 0x01: With keyspace. If set, <keyspace> must be present. <keyspace> is a
+ [string] indicating the keyspace that the query should be executed in.
+ It supercedes the keyspace that the connection is bound to, if any.
The server will respond with a RESULT message with a `prepared` kind (0x0004,
see Section 4.2.5).
@@ -395,12 +412,15 @@
4.1.6. EXECUTE
Executes a prepared query. The body of the message must be:
- <id><query_parameters>
- where <id> is the prepared query ID. It's the [short bytes] returned as a
- response to a PREPARE message. As for <query_parameters>, it has the exact
- same definition as in QUERY (see Section 4.1.4).
-
- The response from the server will be a RESULT message.
+ <id><result_metadata_id><query_parameters>
+ where
+ - <id> is the prepared query ID. It's the [short bytes] returned as a
+ response to a PREPARE message.
+ - <result_metadata_id> is the ID of the resultset metadata that was sent
+ along with response to PREPARE message. If a RESULT/Rows message reports
+ changed resultset metadata with the Metadata_changed flag, the reported new
+ resultset metadata must be used in subsequent executions.
+ - <query_parameters> has the exact same definition as in QUERY (see Section 4.1.4).
4.1.7. BATCH
@@ -408,7 +428,7 @@
Allows executing a list of queries (prepared or not) as a batch (note that
only DML statements are accepted in a batch). The body of the message must
be:
- <type><n><query_1>...<query_n><consistency><flags>[<serial_consistency>][<timestamp>]
+ <type><n><query_1>...<query_n><consistency><flags>[<serial_consistency>][<timestamp>][<keyspace>][<now_in_seconds>]
where:
- <type> is a [byte] indicating the type of batch to use:
- If <type> == 0, the batch will be "logged". This is equivalent to a
@@ -422,24 +442,32 @@
bits must always be 0 as their corresponding options do not make sense for
Batch. A flag is set if the bit corresponding to its `mask` is set. Supported
flags are, given their mask:
- 0x10: With serial consistency. If set, <serial_consistency> should be
- present. <serial_consistency> is the [consistency] level for the
- serial phase of conditional updates. That consistency can only be
- either SERIAL or LOCAL_SERIAL and if not present, it defaults to
- SERIAL. This option will be ignored for anything else other than a
- conditional update/insert.
- 0x20: With default timestamp. If set, <timestamp> should be present.
- <timestamp> is a [long] representing the default timestamp for the query
- in microseconds. This will replace the server side assigned
- timestamp as default timestamp. Note that a timestamp in the query itself
- will still override this timestamp. This is entirely optional.
- 0x40: With names for values. If set, then all values for all <query_i> must be
- preceded by a [string] <name_i> that have the same meaning as in QUERY
- requests [IMPORTANT NOTE: this feature does not work and should not be
- used. It is specified in a way that makes it impossible for the server
- to implement. This will be fixed in a future version of the native
- protocol. See https://issues.apache.org/jira/browse/CASSANDRA-10246 for
- more details].
+ 0x0010: With serial consistency. If set, <serial_consistency> should be
+ present. <serial_consistency> is the [consistency] level for the
+ serial phase of conditional updates. That consistency can only be
+ either SERIAL or LOCAL_SERIAL and if not present, it defaults to
+ SERIAL. This option will be ignored for anything else other than a
+ conditional update/insert.
+ 0x0020: With default timestamp. If set, <timestamp> should be present.
+ <timestamp> is a [long] representing the default timestamp for the query
+ in microseconds. This will replace the server side assigned
+ timestamp as default timestamp. Note that a timestamp in the query itself
+ will still override this timestamp. This is entirely optional.
+ 0x0040: With names for values. If set, then all values for all <query_i> must be
+ preceded by a [string] <name_i> that have the same meaning as in QUERY
+ requests [IMPORTANT NOTE: this feature does not work and should not be
+ used. It is specified in a way that makes it impossible for the server
+ to implement. This will be fixed in a future version of the native
+ protocol. See https://issues.apache.org/jira/browse/CASSANDRA-10246 for
+ more details].
+ 0x0080: With keyspace. If set, <keyspace> must be present. <keyspace> is a
+ [string] indicating the keyspace that the query should be executed in.
+ It supercedes the keyspace that the connection is bound to, if any.
+ 0x0100: With now in seconds. If set, <now_in_seconds> must be present.
+ <now_in_seconds> is an [int] representing the current time (now) for
+ the query. Affects TTL cell liveness in read queries and local deletion
+ time for tombstones and TTL cells in update requests. It's intended
+ for testing purposes and is optional.
- <n> is a [short] indicating the number of following queries.
- <query_1>...<query_n> are the queries to execute. A <query_i> must be of the
form:
@@ -568,7 +596,7 @@
<metadata><rows_count><rows_content>
where:
- <metadata> is composed of:
- <flags><columns_count>[<paging_state>][<global_table_spec>?<col_spec_1>...<col_spec_n>]
+ <flags><columns_count>[<paging_state>][<new_metadata_id>][<global_table_spec>?<col_spec_1>...<col_spec_n>]
where:
- <flags> is an [int]. The bits of <flags> provides information on the
formatting of the remaining information. A flag is set if the bit
@@ -589,9 +617,16 @@
no other information (so no <global_table_spec> nor <col_spec_i>).
This will only ever be the case if this was requested
during the query (see QUERY and RESULT messages).
+ 0x0008 Metadata_changed: if set, the No_metadata flag has to be unset
+ and <new_metadata_id> has to be supplied. This flag is to be
+ used to avoid a roundtrip in case of metadata changes for queries
+ that requested metadata to be skipped.
- <columns_count> is an [int] representing the number of columns selected
by the query that produced this result. It defines the number of <col_spec_i>
elements in and the number of elements for each row in <rows_content>.
+ - <new_metadata_id> is [short bytes] representing the new, changed resultset
+ metadata. The new metadata ID must also be used in subsequent executions of
+ the corresponding prepared statement, if any.
- <global_table_spec> is present if the Global_tables_spec is set in
<flags>. It is composed of two [string] representing the
(unique) keyspace name and table name the columns belong to.
@@ -673,9 +708,10 @@
4.2.5.4. Prepared
The result to a PREPARE message. The body of a Prepared result is:
- <id><metadata><result_metadata>
+ <id><result_metadata_id><metadata><result_metadata>
where:
- <id> is [short bytes] representing the prepared query ID.
+ - <result_metadata_id> is [short bytes] representing the resultset metadata ID.
- <metadata> is composed of:
<flags><columns_count><pk_count>[<pk_index_1>...<pk_index_n>][<global_table_spec>?<col_spec_1>...<col_spec_n>]
where:
@@ -1084,7 +1120,7 @@
0x1003 Truncate_error: error during a truncation error.
0x1100 Write_timeout: Timeout exception during a write request. The rest
of the ERROR message body will be
- <cl><received><blockfor><writeType>
+ <cl><received><blockfor><writeType><contentions>
where:
<cl> is the [consistency] level of the query having triggered
the exception.
@@ -1108,12 +1144,14 @@
- "BATCH_LOG": the timeout occurred during the
write to the batch log when a (logged) batch
write was requested.
- - "CAS": the timeout occured during the Compare And Set write/update.
- - "VIEW": the timeout occured when a write involves
- VIEW update and failure to acqiure local view(MV)
- lock for key within timeout
- - "CDC": the timeout occured when cdc_total_space_in_mb is
- exceeded when doing a write to data tracked by cdc.
+ - "CAS": the timeout occured during the Compare And Set write/update.
+ - "VIEW": the timeout occured when a write involves
+ VIEW update and failure to acqiure local view(MV)
+ lock for key within timeout
+ - "CDC": the timeout occured when cdc_total_space_in_mb is
+ exceeded when doing a write to data tracked by cdc.
+ <contentions> is a [short] that describes the number of contentions occured during the CAS operation.
+ The field only presents when the <writeType> is "CAS".
0x1200 Read_timeout: Timeout exception during a read request. The rest
of the ERROR message body will be
<cl><received><blockfor><data_present>
@@ -1189,12 +1227,24 @@
- "BATCH_LOG": the failure occured during the
write to the batch log when a (logged) batch
write was requested.
- - "CAS": the failure occured during the Compare And Set write/update.
- - "VIEW": the failure occured when a write involves
- VIEW update and failure to acqiure local view(MV)
- lock for key within timeout
- - "CDC": the failure occured when cdc_total_space_in_mb is
- exceeded when doing a write to data tracked by cdc.
+ - "CAS": the failure occured during the Compare And Set write/update.
+ - "VIEW": the failure occured when a write involves
+ VIEW update and failure to acqiure local view(MV)
+ lock for key within timeout
+ - "CDC": the failure occured when cdc_total_space_in_mb is
+ exceeded when doing a write to data tracked by cdc.
+ 0x1600 CDC_WRITE_FAILURE: // todo
+ 0x1700 CAS_WRITE_UNKNOWN: An exception occured due to contended Compare And Set write/update.
+ The CAS operation was only partially completed and the operation may or may not get completed by
+ the contending CAS write or SERIAL/LOCAL_SERIAL read. The rest of the ERROR message body will be
+ <cl><received><blockfor>
+ where:
+ <cl> is the [consistency] level of the query having triggered
+ the exception.
+ <received> is an [int] representing the number of nodes having
+ acknowledged the request.
+ <blockfor> is an [int] representing the number of replicas whose
+ acknowledgement is required to achieve <cl>.
0x2000 Syntax_error: The submitted query has a syntax error.
0x2100 Unauthorized: The logged user doesn't have the right to perform
@@ -1225,3 +1275,7 @@
* Enlarged flag's bitmaps for QUERY, EXECUTE and BATCH messages from [byte] to [int]
(Sections 4.1.4, 4.1.6 and 4.1.7).
* Add the duration data type
+ * Added keyspace field in QUERY, PREPARE, and BATCH messages (Sections 4.1.4, 4.1.5, and 4.1.7).
+ * Added now_in_seconds field in QUERY, EXECUTE, and BATCH messages (Sections 4.1.4, 4.1.6, and 4.1.7).
+ * Added [int] flags field in PREPARE message (Section 4.1.5).
+ * Removed NO_COMPACT startup option (Section 4.1.1.)
diff --git a/doc/source/_static/extra.css b/doc/source/_static/extra.css
index 715e2a8..5e40dd7 100644
--- a/doc/source/_static/extra.css
+++ b/doc/source/_static/extra.css
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
div:not(.highlight) > pre {
background: #fff;
border: 1px solid #e1e4e5;
diff --git a/doc/source/_templates/indexcontent.html b/doc/source/_templates/indexcontent.html
index 5d4b485..5851258 100644
--- a/doc/source/_templates/indexcontent.html
+++ b/doc/source/_templates/indexcontent.html
@@ -1,10 +1,21 @@
{% extends "defindex.html" %}
{% block tables %}
-<div id="wipwarning">This documentation is currently a work-in-progress and contains a number of TODO sections.
+<div id="wipwarning">This documentation is a work-in-progress.
<a href="{{ pathto("bugs") }}">Contributions</a> are welcome.</div>
<h3>Main documentation</h3>
+<div>
+<form id="doc-search-form" action="{{ pathto('search') }}" method="get" role="search">
+ <div class="form-group">
+ Search the documentation:
+ <input type="text" size="30" class="form-control input-sm" name="q" placeholder="Search docs">
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </div>
+</form><br />
+</div>
+
<table class="contentstable doc-landing-table" align="center">
<tr>
<td class="left-column">
@@ -29,7 +40,7 @@
<tr>
<td class="left-column">
<p class="biglink"><a class="biglink" href="{{ pathto("data_modeling/index") }}">{% trans %}Data Modeling{% endtrans %}</a><br/>
- <span class="linkdescr">{% trans %}Or how to make square pegs fit round holes{% endtrans %}</span></p>
+ <span class="linkdescr">{% trans %}Hint: it's not relational{% endtrans %}</span></p>
</td>
<td class="right-column">
<p class="biglink"><a class="biglink" href="{{ pathto("troubleshooting/index") }}">{% trans %}Troubleshooting{% endtrans %}</a><br/>
diff --git a/doc/source/architecture/Figure_1_guarantees.jpg b/doc/source/architecture/Figure_1_guarantees.jpg
new file mode 100644
index 0000000..859342d
--- /dev/null
+++ b/doc/source/architecture/Figure_1_guarantees.jpg
Binary files differ
diff --git a/doc/source/architecture/dynamo.rst b/doc/source/architecture/dynamo.rst
index a7dbb87..5b17d9a 100644
--- a/doc/source/architecture/dynamo.rst
+++ b/doc/source/architecture/dynamo.rst
@@ -15,71 +15,322 @@
.. limitations under the License.
Dynamo
-------
+======
-.. _gossip:
+Apache Cassandra relies on a number of techniques from Amazon's `Dynamo
+<http://courses.cse.tamu.edu/caverlee/csce438/readings/dynamo-paper.pdf>`_
+distributed storage key-value system. Each node in the Dynamo system has three
+main components:
-Gossip
-^^^^^^
+- Request coordination over a partitioned dataset
+- Ring membership and failure detection
+- A local persistence (storage) engine
-.. todo:: todo
+Cassandra primarily draws from the first two clustering components,
+while using a storage engine based on a Log Structured Merge Tree
+(`LSM <http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.44.2782&rep=rep1&type=pdf>`_).
+In particular, Cassandra relies on Dynamo style:
-Failure Detection
-^^^^^^^^^^^^^^^^^
+- Dataset partitioning using consistent hashing
+- Multi-master replication using versioned data and tunable consistency
+- Distributed cluster membership and failure detection via a gossip protocol
+- Incremental scale-out on commodity hardware
-.. todo:: todo
+Cassandra was designed this way to meet large-scale (PiB+) business-critical
+storage requirements. In particular, as applications demanded full global
+replication of petabyte scale datasets along with always available low-latency
+reads and writes, it became imperative to design a new kind of database model
+as the relational database systems of the time struggled to meet the new
+requirements of global scale applications.
-Token Ring/Ranges
-^^^^^^^^^^^^^^^^^
+Dataset Partitioning: Consistent Hashing
+----------------------------------------
-.. todo:: todo
+Cassandra achieves horizontal scalability by
+`partitioning <https://en.wikipedia.org/wiki/Partition_(database)>`_
+all data stored in the system using a hash function. Each partition is replicated
+to multiple physical nodes, often across failure domains such as racks and even
+datacenters. As every replica can independently accept mutations to every key
+that it owns, every key must be versioned. Unlike in the original Dynamo paper
+where deterministic versions and vector clocks were used to reconcile concurrent
+updates to a key, Cassandra uses a simpler last write wins model where every
+mutation is timestamped (including deletes) and then the latest version of data
+is the "winning" value. Formally speaking, Cassandra uses a Last-Write-Wins Element-Set
+conflict-free replicated data type for each CQL row (a.k.a `LWW-Element-Set CRDT
+<https://en.wikipedia.org/wiki/Conflict-free_replicated_data_type#LWW-Element-Set_(Last-Write-Wins-Element-Set)>`_)
+to resolve conflicting mutations on replica sets.
+
+ .. _consistent-hashing-token-ring:
+
+Consistent Hashing using a Token Ring
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Cassandra partitions data over storage nodes using a special form of hashing
+called `consistent hashing <https://en.wikipedia.org/wiki/Consistent_hashing>`_.
+In naive data hashing, you typically allocate keys to buckets by taking a hash
+of the key modulo the number of buckets. For example, if you want to distribute
+data to 100 nodes using naive hashing you might assign every node to a bucket
+between 0 and 100, hash the input key modulo 100, and store the data on the
+associated bucket. In this naive scheme, however, adding a single node might
+invalidate almost all of the mappings.
+
+Cassandra instead maps every node to one or more tokens on a continuous hash
+ring, and defines ownership by hashing a key onto the ring and then "walking"
+the ring in one direction, similar to the `Chord
+<https://pdos.csail.mit.edu/papers/chord:sigcomm01/chord_sigcomm.pdf>`_
+algorithm. The main difference of consistent hashing to naive data hashing is
+that when the number of nodes (buckets) to hash into changes, consistent
+hashing only has to move a small fraction of the keys.
+
+For example, if we have an eight node cluster with evenly spaced tokens, and
+a replication factor (RF) of 3, then to find the owning nodes for a key we
+first hash that key to generate a token (which is just the hash of the key),
+and then we "walk" the ring in a clockwise fashion until we encounter three
+distinct nodes, at which point we have found all the replicas of that key.
+This example of an eight node cluster with `RF=3` can be visualized as follows:
+
+.. figure:: images/ring.svg
+ :scale: 75 %
+ :alt: Dynamo Ring
+
+You can see that in a Dynamo like system, ranges of keys, also known as **token
+ranges**, map to the same physical set of nodes. In this example, all keys that
+fall in the token range excluding token 1 and including token 2 (`range(t1, t2]`)
+are stored on nodes 2, 3 and 4.
+
+Multiple Tokens per Physical Node (a.k.a. `vnodes`)
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Simple single token consistent hashing works well if you have many physical
+nodes to spread data over, but with evenly spaced tokens and a small number of
+physical nodes, incremental scaling (adding just a few nodes of capacity) is
+difficult because there are no token selections for new nodes that can leave
+the ring balanced. Cassandra seeks to avoid token imbalance because uneven
+token ranges lead to uneven request load. For example, in the previous example
+there is no way to add a ninth token without causing imbalance; instead we
+would have to insert ``8`` tokens in the midpoints of the existing ranges.
+
+The Dynamo paper advocates for the use of "virtual nodes" to solve this
+imbalance problem. Virtual nodes solve the problem by assigning multiple
+tokens in the token ring to each physical node. By allowing a single physical
+node to take multiple positions in the ring, we can make small clusters look
+larger and therefore even with a single physical node addition we can make it
+look like we added many more nodes, effectively taking many smaller pieces of
+data from more ring neighbors when we add even a single node.
+
+Cassandra introduces some nomenclature to handle these concepts:
+
+- **Token**: A single position on the `dynamo` style hash ring.
+- **Endpoint**: A single physical IP and port on the network.
+- **Host ID**: A unique identifier for a single "physical" node, usually
+ present at one `Endpoint` and containing one or more `Tokens`.
+- **Virtual Node** (or **vnode**): A `Token` on the hash ring owned by the same
+ physical node, one with the same `Host ID`.
+
+The mapping of **Tokens** to **Endpoints** gives rise to the **Token Map**
+where Cassandra keeps track of what ring positions map to which physical
+endpoints. For example, in the following figure we can represent an eight node
+cluster using only four physical nodes by assigning two tokens to every node:
+
+.. figure:: images/vnodes.svg
+ :scale: 75 %
+ :alt: Virtual Tokens Ring
+
+
+Multiple tokens per physical node provide the following benefits:
+
+1. When a new node is added it accepts approximately equal amounts of data from
+ other nodes in the ring, resulting in equal distribution of data across the
+ cluster.
+2. When a node is decommissioned, it loses data roughly equally to other members
+ of the ring, again keeping equal distribution of data across the cluster.
+3. If a node becomes unavailable, query load (especially token aware query load),
+ is evenly distributed across many other nodes.
+
+Multiple tokens, however, can also have disadvantages:
+
+1. Every token introduces up to ``2 * (RF - 1)`` additional neighbors on the
+ token ring, which means that there are more combinations of node failures
+ where we lose availability for a portion of the token ring. The more tokens
+ you have, `the higher the probability of an outage
+ <https://jolynch.github.io/pdf/cassandra-availability-virtual.pdf>`_.
+2. Cluster-wide maintenance operations are often slowed. For example, as the
+ number of tokens per node is increased, the number of discrete repair
+ operations the cluster must do also increases.
+3. Performance of operations that span token ranges could be affected.
+
+Note that in Cassandra ``2.x``, the only token allocation algorithm available
+was picking random tokens, which meant that to keep balance the default number
+of tokens per node had to be quite high, at ``256``. This had the effect of
+coupling many physical endpoints together, increasing the risk of
+unavailability. That is why in ``3.x +`` the new deterministic token allocator
+was added which intelligently picks tokens such that the ring is optimally
+balanced while requiring a much lower number of tokens per physical node.
+
+
+Multi-master Replication: Versioned Data and Tunable Consistency
+----------------------------------------------------------------
+
+Cassandra replicates every partition of data to many nodes across the cluster
+to maintain high availability and durability. When a mutation occurs, the
+coordinator hashes the partition key to determine the token range the data
+belongs to and then replicates the mutation to the replicas of that data
+according to the :ref:`Replication Strategy <replication-strategy>`.
+
+All replication strategies have the notion of a **replication factor** (``RF``),
+which indicates to Cassandra how many copies of the partition should exist.
+For example with a ``RF=3`` keyspace, the data will be written to three
+distinct **replicas**. Replicas are always chosen such that they are distinct
+physical nodes which is achieved by skipping virtual nodes if needed.
+Replication strategies may also choose to skip nodes present in the same failure
+domain such as racks or datacenters so that Cassandra clusters can tolerate
+failures of whole racks and even datacenters of nodes.
.. _replication-strategy:
-Replication
-^^^^^^^^^^^
+Replication Strategy
+^^^^^^^^^^^^^^^^^^^^
-The replication strategy of a keyspace determines which nodes are replicas for a given token range. The two main
-replication strategies are :ref:`simple-strategy` and :ref:`network-topology-strategy`.
+Cassandra supports pluggable **replication strategies**, which determine which
+physical nodes act as replicas for a given token range. Every keyspace of
+data has its own replication strategy. All production deployments should use
+the :ref:`network-topology-strategy` while the :ref:`simple-strategy` replication
+strategy is useful only for testing clusters where you do not yet know the
+datacenter layout of the cluster.
+
+.. _network-topology-strategy:
+
+``NetworkTopologyStrategy``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+``NetworkTopologyStrategy`` allows a replication factor to be specified for each
+datacenter in the cluster. Even if your cluster only uses a single datacenter,
+``NetworkTopologyStrategy`` should be preferred over ``SimpleStrategy`` to make it
+easier to add new physical or virtual datacenters to the cluster later.
+
+In addition to allowing the replication factor to be specified individually by
+datacenter, ``NetworkTopologyStrategy`` also attempts to choose replicas within a
+datacenter from different racks as specified by the :ref:`Snitch <snitch>`. If
+the number of racks is greater than or equal to the replication factor for the
+datacenter, each replica is guaranteed to be chosen from a different rack.
+Otherwise, each rack will hold at least one replica, but some racks may hold
+more than one. Note that this rack-aware behavior has some potentially
+`surprising implications
+<https://issues.apache.org/jira/browse/CASSANDRA-3810>`_. For example, if
+there are not an even number of nodes in each rack, the data load on the
+smallest rack may be much higher. Similarly, if a single node is bootstrapped
+into a brand new rack, it will be considered a replica for the entire ring.
+For this reason, many operators choose to configure all nodes in a single
+availability zone or similar failure domain as a single "rack".
.. _simple-strategy:
-SimpleStrategy
-~~~~~~~~~~~~~~
+``SimpleStrategy``
+~~~~~~~~~~~~~~~~~~
-SimpleStrategy allows a single integer ``replication_factor`` to be defined. This determines the number of nodes that
+``SimpleStrategy`` allows a single integer ``replication_factor`` to be defined. This determines the number of nodes that
should contain a copy of each row. For example, if ``replication_factor`` is 3, then three different nodes should store
a copy of each row.
-SimpleStrategy treats all nodes identically, ignoring any configured datacenters or racks. To determine the replicas
+``SimpleStrategy`` treats all nodes identically, ignoring any configured datacenters or racks. To determine the replicas
for a token range, Cassandra iterates through the tokens in the ring, starting with the token range of interest. For
each token, it checks whether the owning node has been added to the set of replicas, and if it has not, it is added to
the set. This process continues until ``replication_factor`` distinct nodes have been added to the set of replicas.
-.. _network-topology-strategy:
+.. _transient-replication:
-NetworkTopologyStrategy
+Transient Replication
+~~~~~~~~~~~~~~~~~~~~~
+
+Transient replication is an experimental feature in Cassandra 4.0 not present
+in the original Dynamo paper. It allows you to configure a subset of replicas
+to only replicate data that hasn't been incrementally repaired. This allows you
+to decouple data redundancy from availability. For instance, if you have a
+keyspace replicated at rf 3, and alter it to rf 5 with 2 transient replicas,
+you go from being able to tolerate one failed replica to being able to tolerate
+two, without corresponding increase in storage usage. This is because 3 nodes
+will replicate all the data for a given token range, and the other 2 will only
+replicate data that hasn't been incrementally repaired.
+
+To use transient replication, you first need to enable it in
+``cassandra.yaml``. Once enabled, both ``SimpleStrategy`` and
+``NetworkTopologyStrategy`` can be configured to transiently replicate data.
+You configure it by specifying replication factor as
+``<total_replicas>/<transient_replicas`` Both ``SimpleStrategy`` and
+``NetworkTopologyStrategy`` support configuring transient replication.
+
+Transiently replicated keyspaces only support tables created with read_repair
+set to ``NONE`` and monotonic reads are not currently supported. You also
+can't use ``LWT``, logged batches, or counters in 4.0. You will possibly never be
+able to use materialized views with transiently replicated keyspaces and
+probably never be able to use secondary indices with them.
+
+Transient replication is an experimental feature that may not be ready for
+production use. The expected audience is experienced users of Cassandra
+capable of fully validating a deployment of their particular application. That
+means being able check that operations like reads, writes, decommission,
+remove, rebuild, repair, and replace all work with your queries, data,
+configuration, operational practices, and availability requirements.
+
+It is anticipated that ``4.next`` will support monotonic reads with transient
+replication as well as LWT, logged batches, and counters.
+
+Data Versioning
+^^^^^^^^^^^^^^^
+
+Cassandra uses mutation timestamp versioning to guarantee eventual consistency of
+data. Specifically all mutations that enter the system do so with a timestamp
+provided either from a client clock or, absent a client provided timestamp,
+from the coordinator node's clock. Updates resolve according to the conflict
+resolution rule of last write wins. Cassandra's correctness does depend on
+these clocks, so make sure a proper time synchronization process is running
+such as NTP.
+
+Cassandra applies separate mutation timestamps to every column of every row
+within a CQL partition. Rows are guaranteed to be unique by primary key, and
+each column in a row resolve concurrent mutations according to last-write-wins
+conflict resolution. This means that updates to different primary keys within a
+partition can actually resolve without conflict! Furthermore the CQL collection
+types such as maps and sets use this same conflict free mechanism, meaning
+that concurrent updates to maps and sets are guaranteed to resolve as well.
+
+Replica Synchronization
~~~~~~~~~~~~~~~~~~~~~~~
-NetworkTopologyStrategy allows a replication factor to be specified for each datacenter in the cluster. Even if your
-cluster only uses a single datacenter, NetworkTopologyStrategy should be prefered over SimpleStrategy to make it easier
-to add new physical or virtual datacenters to the cluster later.
+As replicas in Cassandra can accept mutations independently, it is possible
+for some replicas to have newer data than others. Cassandra has many best-effort
+techniques to drive convergence of replicas including
+`Replica read repair <read-repair>` in the read path and
+`Hinted handoff <hints>` in the write path.
-In addition to allowing the replication factor to be specified per-DC, NetworkTopologyStrategy also attempts to choose
-replicas within a datacenter from different racks. If the number of racks is greater than or equal to the replication
-factor for the DC, each replica will be chosen from a different rack. Otherwise, each rack will hold at least one
-replica, but some racks may hold more than one. Note that this rack-aware behavior has some potentially `surprising
-implications <https://issues.apache.org/jira/browse/CASSANDRA-3810>`_. For example, if there are not an even number of
-nodes in each rack, the data load on the smallest rack may be much higher. Similarly, if a single node is bootstrapped
-into a new rack, it will be considered a replica for the entire ring. For this reason, many operators choose to
-configure all nodes on a single "rack".
+These techniques are only best-effort, however, and to guarantee eventual
+consistency Cassandra implements `anti-entropy repair <repair>` where replicas
+calculate hierarchical hash-trees over their datasets called `Merkle Trees
+<https://en.wikipedia.org/wiki/Merkle_tree>`_ that can then be compared across
+replicas to identify mismatched data. Like the original Dynamo paper Cassandra
+supports "full" repairs where replicas hash their entire dataset, create Merkle
+trees, send them to each other and sync any ranges that don't match.
+
+Unlike the original Dynamo paper, Cassandra also implements sub-range repair
+and incremental repair. Sub-range repair allows Cassandra to increase the
+resolution of the hash trees (potentially down to the single partition level)
+by creating a larger number of trees that span only a portion of the data
+range. Incremental repair allows Cassandra to only repair the partitions that
+have changed since the last repair.
Tunable Consistency
^^^^^^^^^^^^^^^^^^^
-Cassandra supports a per-operation tradeoff between consistency and availability through *Consistency Levels*.
-Essentially, an operation's consistency level specifies how many of the replicas need to respond to the coordinator in
-order to consider the operation a success.
+Cassandra supports a per-operation tradeoff between consistency and
+availability through **Consistency Levels**. Cassandra's consistency levels
+are a version of Dynamo's ``R + W > N`` consistency mechanism where operators
+could configure the number of nodes that must participate in reads (``R``)
+and writes (``W``) to be larger than the replication factor (``N``). In
+Cassandra, you instead choose from a menu of common consistency levels which
+allow the operator to pick ``R`` and ``W`` behavior without knowing the
+replication factor. Generally writes will be visible to subsequent reads when
+the read consistency level contains enough nodes to guarantee a quorum intersection
+with the write consistency level.
The following consistency levels are available:
@@ -113,27 +364,174 @@
attempt to replay the hint and deliver the mutation to the replicas. This consistency level is only accepted for
write operations.
-Write operations are always sent to all replicas, regardless of consistency level. The consistency level simply
-controls how many responses the coordinator waits for before responding to the client.
+Write operations **are always sent to all replicas**, regardless of consistency
+level. The consistency level simply controls how many responses the coordinator
+waits for before responding to the client.
-For read operations, the coordinator generally only issues read commands to enough replicas to satisfy the consistency
-level. There are a couple of exceptions to this:
-
-- Speculative retry may issue a redundant read request to an extra replica if the other replicas have not responded
- within a specified time window.
-- Based on ``read_repair_chance`` and ``dclocal_read_repair_chance`` (part of a table's schema), read requests may be
- randomly sent to all replicas in order to repair potentially inconsistent data.
+For read operations, the coordinator generally only issues read commands to
+enough replicas to satisfy the consistency level. The one exception to this is
+when speculative retry may issue a redundant read request to an extra replica
+if the original replicas have not responded within a specified time window.
Picking Consistency Levels
~~~~~~~~~~~~~~~~~~~~~~~~~~
-It is common to pick read and write consistency levels that are high enough to overlap, resulting in "strong"
-consistency. This is typically expressed as ``W + R > RF``, where ``W`` is the write consistency level, ``R`` is the
-read consistency level, and ``RF`` is the replication factor. For example, if ``RF = 3``, a ``QUORUM`` request will
-require responses from at least two of the three replicas. If ``QUORUM`` is used for both writes and reads, at least
-one of the replicas is guaranteed to participate in *both* the write and the read request, which in turn guarantees that
-the latest write will be read. In a multi-datacenter environment, ``LOCAL_QUORUM`` can be used to provide a weaker but
-still useful guarantee: reads are guaranteed to see the latest write from within the same datacenter.
+It is common to pick read and write consistency levels such that the replica
+sets overlap, resulting in all acknowledged writes being visible to subsequent
+reads. This is typically expressed in the same terms Dynamo does, in that ``W +
+R > RF``, where ``W`` is the write consistency level, ``R`` is the read
+consistency level, and ``RF`` is the replication factor. For example, if ``RF
+= 3``, a ``QUORUM`` request will require responses from at least ``2/3``
+replicas. If ``QUORUM`` is used for both writes and reads, at least one of the
+replicas is guaranteed to participate in *both* the write and the read request,
+which in turn guarantees that the quorums will overlap and the write will be
+visible to the read.
-If this type of strong consistency isn't required, lower consistency levels like ``ONE`` may be used to improve
-throughput, latency, and availability.
+In a multi-datacenter environment, ``LOCAL_QUORUM`` can be used to provide a
+weaker but still useful guarantee: reads are guaranteed to see the latest write
+from within the same datacenter. This is often sufficient as clients homed to
+a single datacenter will read their own writes.
+
+If this type of strong consistency isn't required, lower consistency levels
+like ``LOCAL_ONE`` or ``ONE`` may be used to improve throughput, latency, and
+availability. With replication spanning multiple datacenters, ``LOCAL_ONE`` is
+typically less available than ``ONE`` but is faster as a rule. Indeed ``ONE``
+will succeed if a single replica is available in any datacenter.
+
+Distributed Cluster Membership and Failure Detection
+----------------------------------------------------
+
+The replication protocols and dataset partitioning rely on knowing which nodes
+are alive and dead in the cluster so that write and read operations can be
+optimally routed. In Cassandra liveness information is shared in a distributed
+fashion through a failure detection mechanism based on a gossip protocol.
+
+.. _gossip:
+
+Gossip
+^^^^^^
+
+Gossip is how Cassandra propagates basic cluster bootstrapping information such
+as endpoint membership and internode network protocol versions. In Cassandra's
+gossip system, nodes exchange state information not only about themselves but
+also about other nodes they know about. This information is versioned with a
+vector clock of ``(generation, version)`` tuples, where the generation is a
+monotonic timestamp and version is a logical clock the increments roughly every
+second. These logical clocks allow Cassandra gossip to ignore old versions of
+cluster state just by inspecting the logical clocks presented with gossip
+messages.
+
+Every node in the Cassandra cluster runs the gossip task independently and
+periodically. Every second, every node in the cluster:
+
+1. Updates the local node's heartbeat state (the version) and constructs the
+ node's local view of the cluster gossip endpoint state.
+2. Picks a random other node in the cluster to exchange gossip endpoint state
+ with.
+3. Probabilistically attempts to gossip with any unreachable nodes (if one exists)
+4. Gossips with a seed node if that didn't happen in step 2.
+
+When an operator first bootstraps a Cassandra cluster they designate certain
+nodes as "seed" nodes. Any node can be a seed node and the only difference
+between seed and non-seed nodes is seed nodes are allowed to bootstrap into the
+ring without seeing any other seed nodes. Furthermore, once a cluster is
+bootstrapped, seed nodes become "hotspots" for gossip due to step 4 above.
+
+As non-seed nodes must be able to contact at least one seed node in order to
+bootstrap into the cluster, it is common to include multiple seed nodes, often
+one for each rack or datacenter. Seed nodes are often chosen using existing
+off-the-shelf service discovery mechanisms.
+
+.. note::
+ Nodes do not have to agree on the seed nodes, and indeed once a cluster is
+ bootstrapped, newly launched nodes can be configured to use any existing
+ nodes as "seeds". The only advantage to picking the same nodes as seeds
+ is it increases their usefullness as gossip hotspots.
+
+Currently, gossip also propagates token metadata and schema *version*
+information. This information forms the control plane for scheduling data
+movements and schema pulls. For example, if a node sees a mismatch in schema
+version in gossip state, it will schedule a schema sync task with the other
+nodes. As token information propagates via gossip it is also the control plane
+for teaching nodes which endpoints own what data.
+
+Ring Membership and Failure Detection
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Gossip forms the basis of ring membership, but the **failure detector**
+ultimately makes decisions about if nodes are ``UP`` or ``DOWN``. Every node in
+Cassandra runs a variant of the `Phi Accrual Failure Detector
+<https://www.computer.org/csdl/proceedings-article/srds/2004/22390066/12OmNvT2phv>`_,
+in which every node is constantly making an independent decision of if their
+peer nodes are available or not. This decision is primarily based on received
+heartbeat state. For example, if a node does not see an increasing heartbeat
+from a node for a certain amount of time, the failure detector "convicts" that
+node, at which point Cassandra will stop routing reads to it (writes will
+typically be written to hints). If/when the node starts heartbeating again,
+Cassandra will try to reach out and connect, and if it can open communication
+channels it will mark that node as available.
+
+.. note::
+ UP and DOWN state are local node decisions and are not propagated with
+ gossip. Heartbeat state is propagated with gossip, but nodes will not
+ consider each other as "UP" until they can successfully message each other
+ over an actual network channel.
+
+Cassandra will never remove a node from gossip state without explicit
+instruction from an operator via a decommission operation or a new node
+bootstrapping with a ``replace_address_first_boot`` option. This choice is
+intentional to allow Cassandra nodes to temporarily fail without causing data
+to needlessly re-balance. This also helps to prevent simultaneous range
+movements, where multiple replicas of a token range are moving at the same
+time, which can violate monotonic consistency and can even cause data loss.
+
+Incremental Scale-out on Commodity Hardware
+--------------------------------------------
+
+Cassandra scales-out to meet the requirements of growth in data size and
+request rates. Scaling-out means adding additional nodes to the ring, and
+every additional node brings linear improvements in compute and storage. In
+contrast, scaling-up implies adding more capacity to the existing database
+nodes. Cassandra is also capable of scale-up, and in certain environments it
+may be preferable depending on the deployment. Cassandra gives operators the
+flexibility to chose either scale-out or scale-up.
+
+One key aspect of Dynamo that Cassandra follows is to attempt to run on
+commodity hardware, and many engineering choices are made under this
+assumption. For example, Cassandra assumes nodes can fail at any time,
+auto-tunes to make the best use of CPU and memory resources available and makes
+heavy use of advanced compression and caching techniques to get the most
+storage out of limited memory and storage capabilities.
+
+Simple Query Model
+^^^^^^^^^^^^^^^^^^
+
+Cassandra, like Dynamo, chooses not to provide cross-partition transactions
+that are common in SQL Relational Database Management Systems (RDBMS). This
+both gives the programmer a simpler read and write API, and allows Cassandra to
+more easily scale horizontally since multi-partition transactions spanning
+multiple nodes are notoriously difficult to implement and typically very
+latent.
+
+Instead, Cassanda chooses to offer fast, consistent, latency at any scale for
+single partition operations, allowing retrieval of entire partitions or only
+subsets of partitions based on primary key filters. Furthermore, Cassandra does
+support single partition compare and swap functionality via the lightweight
+transaction CQL API.
+
+Simple Interface for Storing Records
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Cassandra, in a slight departure from Dynamo, chooses a storage interface that
+is more sophisticated then "simple key value" stores but significantly less
+complex than SQL relational data models. Cassandra presents a wide-column
+store interface, where partitions of data contain multiple rows, each of which
+contains a flexible set of individually typed columns. Every row is uniquely
+identified by the partition key and one or more clustering keys, and every row
+can have as many columns as needed.
+
+This allows users to flexibly add new columns to existing datasets as new
+requirements surface. Schema changes involve only metadata changes and run
+fully concurrently with live workloads. Therefore, users can safely add columns
+to existing Cassandra databases while remaining confident that query
+performance will not degrade.
diff --git a/doc/source/architecture/guarantees.rst b/doc/source/architecture/guarantees.rst
index c0b58d8..3cff808 100644
--- a/doc/source/architecture/guarantees.rst
+++ b/doc/source/architecture/guarantees.rst
@@ -14,7 +14,63 @@
.. See the License for the specific language governing permissions and
.. limitations under the License.
-Guarantees
-----------
+.. _guarantees:
-.. todo:: todo
+Guarantees
+==============
+Apache Cassandra is a highly scalable and reliable database. Cassandra is used in web based applications that serve large number of clients and the quantity of data processed is web-scale (Petabyte) large. Cassandra makes some guarantees about its scalability, availability and reliability. To fully understand the inherent limitations of a storage system in an environment in which a certain level of network partition failure is to be expected and taken into account when designing the system it is important to first briefly introduce the CAP theorem.
+
+What is CAP?
+^^^^^^^^^^^^^
+According to the CAP theorem it is not possible for a distributed data store to provide more than two of the following guarantees simultaneously.
+
+- Consistency: Consistency implies that every read receives the most recent write or errors out
+- Availability: Availability implies that every request receives a response. It is not guaranteed that the response contains the most recent write or data.
+- Partition tolerance: Partition tolerance refers to the tolerance of a storage system to failure of a network partition. Even if some of the messages are dropped or delayed the system continues to operate.
+
+CAP theorem implies that when using a network partition, with the inherent risk of partition failure, one has to choose between consistency and availability and both cannot be guaranteed at the same time. CAP theorem is illustrated in Figure 1.
+
+.. figure:: Figure_1_guarantees.jpg
+
+Figure 1. CAP Theorem
+
+High availability is a priority in web based applications and to this objective Cassandra chooses Availability and Partition Tolerance from the CAP guarantees, compromising on data Consistency to some extent.
+
+Cassandra makes the following guarantees.
+
+- High Scalability
+- High Availability
+- Durability
+- Eventual Consistency of writes to a single table
+- Lightweight transactions with linearizable consistency
+- Batched writes across multiple tables are guaranteed to succeed completely or not at all
+- Secondary indexes are guaranteed to be consistent with their local replicas data
+
+High Scalability
+^^^^^^^^^^^^^^^^^
+Cassandra is a highly scalable storage system in which nodes may be added/removed as needed. Using gossip-based protocol a unified and consistent membership list is kept at each node.
+
+High Availability
+^^^^^^^^^^^^^^^^^^^
+Cassandra guarantees high availability of data by implementing a fault-tolerant storage system. Failure detection in a node is detected using a gossip-based protocol.
+
+Durability
+^^^^^^^^^^^^
+Cassandra guarantees data durability by using replicas. Replicas are multiple copies of a data stored on different nodes in a cluster. In a multi-datacenter environment the replicas may be stored on different datacenters. If one replica is lost due to unrecoverable node/datacenter failure the data is not completely lost as replicas are still available.
+
+Eventual Consistency
+^^^^^^^^^^^^^^^^^^^^^^
+Meeting the requirements of performance, reliability, scalability and high availability in production Cassandra is an eventually consistent storage system. Eventually consistent implies that all updates reach all replicas eventually. Divergent versions of the same data may exist temporarily but they are eventually reconciled to a consistent state. Eventual consistency is a tradeoff to achieve high availability and it involves some read and write latencies.
+
+Lightweight transactions with linearizable consistency
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Data must be read and written in a sequential order. Paxos consensus protocol is used to implement lightweight transactions. Paxos protocol implements lightweight transactions that are able to handle concurrent operations using linearizable consistency. Linearizable consistency is sequential consistency with real-time constraints and it ensures transaction isolation with compare and set (CAS) transaction. With CAS replica data is compared and data that is found to be out of date is set to the most consistent value. Reads with linearizable consistency allow reading the current state of the data, which may possibly be uncommitted, without making a new addition or update.
+
+Batched Writes
+^^^^^^^^^^^^^^^
+
+The guarantee for batched writes across multiple tables is that they will eventually succeed, or none will. Batch data is first written to batchlog system data, and when the batch data has been successfully stored in the cluster the batchlog data is removed. The batch is replicated to another node to ensure the full batch completes in the event the coordinator node fails.
+
+Secondary Indexes
+^^^^^^^^^^^^^^^^^^
+A secondary index is an index on a column and is used to query a table that is normally not queryable. Secondary indexes when built are guaranteed to be consistent with their local replicas.
diff --git a/doc/source/architecture/images/ring.svg b/doc/source/architecture/images/ring.svg
new file mode 100644
index 0000000..d0db8c5
--- /dev/null
+++ b/doc/source/architecture/images/ring.svg
@@ -0,0 +1,11 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="651" height="709.4583740234375" style="
+ width:651px;
+ height:709.4583740234375px;
+ background: transparent;
+ fill: none;
+">
+
+
+ <svg xmlns="http://www.w3.org/2000/svg" class="role-diagram-draw-area"><g class="shapes-region" style="stroke: black; fill: none;"><g class="composite-shape"><path class="real" d=" M223.5,655 C223.5,634.84 239.84,618.5 260,618.5 C280.16,618.5 296.5,634.84 296.5,655 C296.5,675.16 280.16,691.5 260,691.5 C239.84,691.5 223.5,675.16 223.5,655 Z" style="stroke-width: 1; stroke: rgb(103, 148, 135); fill: rgb(103, 148, 135);"/></g><g class="composite-shape"><path class="real" d=" M229.26,655 C229.26,638.02 243.02,624.26 260,624.26 C276.98,624.26 290.74,638.02 290.74,655 C290.74,671.98 276.98,685.74 260,685.74 C243.02,685.74 229.26,671.98 229.26,655 Z" style="stroke-width: 1; stroke: rgb(202, 194, 126); fill: rgb(202, 194, 126);"/></g><g class="composite-shape"><path class="real" d=" M377.5,595 C377.5,571.53 396.53,552.5 420,552.5 C443.47,552.5 462.5,571.53 462.5,595 C462.5,618.47 443.47,637.5 420,637.5 C396.53,637.5 377.5,618.47 377.5,595 Z" style="stroke-width: 1; stroke: rgb(103, 148, 135); fill: rgb(103, 148, 135);"/></g><g class="composite-shape"><path class="real" d=" M384.06,595 C384.06,575.15 400.15,559.06 420,559.06 C439.85,559.06 455.94,575.15 455.94,595 C455.94,614.85 439.85,630.94 420,630.94 C400.15,630.94 384.06,614.85 384.06,595 Z" style="stroke-width: 1; stroke: rgb(202, 194, 126); fill: rgb(202, 194, 126);"/></g><g class="composite-shape"><path class="real" d=" M390,595 C390,578.43 403.43,565 420,565 C436.57,565 450,578.43 450,595 C450,611.57 436.57,625 420,625 C403.43,625 390,611.57 390,595 Z" style="stroke-width: 1; stroke: rgb(130, 192, 233); fill: rgb(130, 192, 233);"/></g><g class="composite-shape"><path class="real" d=" M444.06,435 C444.06,415.15 460.15,399.06 480,399.06 C499.85,399.06 515.94,415.15 515.94,435 C515.94,454.85 499.85,470.94 480,470.94 C460.15,470.94 444.06,454.85 444.06,435 Z" style="stroke-width: 1; stroke: rgb(202, 194, 126); fill: rgb(202, 194, 126);"/></g><g class="composite-shape"><path class="real" d=" M450,435 C450,418.43 463.43,405 480,405 C496.57,405 510,418.43 510,435 C510,451.57 496.57,465 480,465 C463.43,465 450,451.57 450,435 Z" style="stroke-width: 1; stroke: rgb(130, 192, 233); fill: rgb(130, 192, 233);"/></g><g class="composite-shape"><path class="real" d=" M390,275 C390,258.43 403.43,245 420,245 C436.57,245 450,258.43 450,275 C450,291.57 436.57,305 420,305 C403.43,305 390,291.57 390,275 Z" style="stroke-width: 1; stroke: rgb(130, 192, 233); fill: rgb(130, 192, 233);"/></g><g class="composite-shape"><path class="real" d=" M40,435 C40,313.5 138.5,215 260,215 C381.5,215 480,313.5 480,435 C480,556.5 381.5,655 260,655 C138.5,655 40,556.5 40,435 Z" style="stroke-width: 1; stroke: rgba(0, 0, 0, 0.52); fill: none; stroke-dasharray: 1.125, 3.35;"/></g><g class="grouped-shape"><g class="composite-shape"><path class="real" d=" M90,435 C90,341.11 166.11,265 260,265 C353.89,265 430,341.11 430,435 C430,528.89 353.89,605 260,605 C166.11,605 90,528.89 90,435 Z" style="stroke-width: 1; stroke: rgb(0, 0, 0); fill: none;"/></g><g class="composite-shape"><path class="real" d=" M111.25,435 C111.25,352.85 177.85,286.25 260,286.25 C342.15,286.25 408.75,352.85 408.75,435 C408.75,517.15 342.15,583.75 260,583.75 C177.85,583.75 111.25,517.15 111.25,435 Z" style="stroke-width: 1; stroke: rgb(0, 0, 0); fill: none;"/></g><g class="composite-shape"><path class="real" d=" M132.5,435 C132.5,364.58 189.58,307.5 260,307.5 C330.42,307.5 387.5,364.58 387.5,435 C387.5,505.42 330.42,562.5 260,562.5 C189.58,562.5 132.5,505.42 132.5,435 Z" style="stroke-width: 1; stroke: rgb(0, 0, 0); fill: none;"/></g></g><g class="composite-shape"><path class="real" d=" M235,655 C235,641.19 246.19,630 260,630 C273.81,630 285,641.19 285,655 C285,668.81 273.81,680 260,680 C246.19,680 235,668.81 235,655 Z" style="stroke-width: 1; stroke: rgb(0, 0, 0); fill: rgb(187, 187, 187);"/></g><g class="grouped-shape"><g class="composite-shape"><path class="real" d=" M235,215 C235,201.19 246.19,190 260,190 C273.81,190 285,201.19 285,215 C285,228.81 273.81,240 260,240 C246.19,240 235,228.81 235,215 Z" style="stroke-width: 1; stroke: rgb(0, 0, 0); fill: rgb(187, 187, 187);"/></g></g><g class="composite-shape"><path class="real" d=" M455,435 C455,421.19 466.19,410 480,410 C493.81,410 505,421.19 505,435 C505,448.81 493.81,460 480,460 C466.19,460 455,448.81 455,435 Z" style="stroke-width: 1; stroke: rgb(0, 0, 0); fill: rgb(187, 187, 187);"/></g><g class="composite-shape"><path class="real" d=" M15,435 C15,421.19 26.19,410 40,410 C53.81,410 65,421.19 65,435 C65,448.81 53.81,460 40,460 C26.19,460 15,448.81 15,435 Z" style="stroke-width: 1; stroke: rgb(0, 0, 0); fill: rgb(187, 187, 187);"/></g><g class="composite-shape"><path class="real" d=" M395,275 C395,261.19 406.19,250 420,250 C433.81,250 445,261.19 445,275 C445,288.81 433.81,300 420,300 C406.19,300 395,288.81 395,275 Z" style="stroke-width: 1; stroke: rgb(0, 0, 0); fill: rgb(187, 187, 187);"/></g><g class="composite-shape"><path class="real" d=" M395,595 C395,581.19 406.19,570 420,570 C433.81,570 445,581.19 445,595 C445,608.81 433.81,620 420,620 C406.19,620 395,608.81 395,595 Z" style="stroke-width: 1; stroke: rgb(0, 0, 0); fill: rgb(187, 187, 187);"/></g><g class="composite-shape"><path class="real" d=" M75,595 C75,581.19 86.19,570 100,570 C113.81,570 125,581.19 125,595 C125,608.81 113.81,620 100,620 C86.19,620 75,608.81 75,595 Z" style="stroke-width: 1; stroke: rgb(0, 0, 0); fill: rgb(187, 187, 187);"/></g><g class="grouped-shape"><g class="composite-shape"><path class="real" d=" M75,275 C75,261.19 86.19,250 100,250 C113.81,250 125,261.19 125,275 C125,288.81 113.81,300 100,300 C86.19,300 75,288.81 75,275 Z" style="stroke-width: 1; stroke: rgb(0, 0, 0); fill: rgb(187, 187, 187);"/></g></g><g class="arrow-line"><path class="connection real" stroke-dasharray="" d=" M268.22,265.22 C401.65,271.85 490.67,424.28 380,555" style="stroke: rgb(130, 192, 233); stroke-width: 6; fill: none;"/><g stroke="rgba(130,192,233,1)" transform="matrix(0.9999897039488793,0.00453784048117526,-0.00453784048117526,0.9999897039488793,260,265)" style="stroke: rgb(130, 192, 233); stroke-width: 6;"><circle cx="0" cy="0" r="9.045000000000002"/></g><g stroke="rgba(130,192,233,1)" fill="rgba(130,192,233,1)" transform="matrix(-0.6461239796429636,0.763232469782529,-0.763232469782529,-0.6461239796429636,380,555)" style="stroke: rgb(130, 192, 233); fill: rgb(130, 192, 233); stroke-width: 6;"><circle cx="0" cy="0" r="9.045000000000002"/></g></g><g class="arrow-line"><path class="connection real" stroke-dasharray="" d=" M366.47,330.7 C447.68,407.15 414.54,574.62 260,583.75" style="stroke: rgb(202, 194, 126); stroke-width: 6; fill: none;"/><g stroke="rgba(202,194,126,1)" transform="matrix(0.7722902597301384,0.6352698282824042,-0.6352698282824042,0.7722902597301384,360,325)" style="stroke: rgb(202, 194, 126); stroke-width: 6;"><circle cx="0" cy="0" r="9.045000000000002"/></g><g stroke="rgba(202,194,126,1)" fill="rgba(202,194,126,1)" transform="matrix(-0.9982604689368237,0.05895791853545039,-0.05895791853545039,-0.9982604689368237,260,583.7499999999999)" style="stroke: rgb(202, 194, 126); fill: rgb(202, 194, 126); stroke-width: 6;"><circle cx="0" cy="0" r="9.045000000000002"/></g></g><g class="arrow-line"><path class="connection real" stroke-dasharray="" d=" M387.19,443.58 C380.1,535.72 254.02,615.51 160,515" style="stroke: rgb(103, 148, 135); stroke-width: 6; fill: none;"/><g stroke="rgba(103,148,135,1)" transform="matrix(0.004537840481175261,0.9999897039488793,-0.9999897039488793,0.004537840481175261,387.5,435)" style="stroke: rgb(103, 148, 135); stroke-width: 6;"><circle cx="0" cy="0" r="9.045000000000002"/></g><g stroke="rgba(103,148,135,1)" fill="rgba(103,148,135,1)" transform="matrix(-0.6831463259165826,-0.7302815192695721,0.7302815192695721,-0.6831463259165826,160,515)" style="stroke: rgb(103, 148, 135); fill: rgb(103, 148, 135); stroke-width: 6;"><circle cx="0" cy="0" r="9.045000000000002"/></g></g><g class="arrow-line"><path class="connection real" stroke-dasharray="" d=" M345,195 L316.4,271.25" style="stroke: rgb(130, 192, 233); stroke-width: 3; fill: none;"/><g stroke="rgba(130,192,233,1)" transform="matrix(0.3511880698803796,-0.9363049394154095,0.9363049394154095,0.3511880698803796,315,275)" style="stroke: rgb(130, 192, 233); stroke-width: 3;"><path d=" M17.49,-5.26 Q7.94,-0.72 0,0 Q7.94,0.72 17.49,5.26"/></g></g><g class="arrow-line"><path class="connection real" stroke-dasharray="" d=" M485,330 L403.62,368.3" style="stroke: rgb(202, 194, 126); stroke-width: 3; fill: none;"/><g stroke="rgba(202,194,126,1)" transform="matrix(0.9048270524660194,-0.425779291565073,0.425779291565073,0.9048270524660194,400,370)" style="stroke: rgb(202, 194, 126); stroke-width: 3;"><path d=" M17.49,-5.26 Q7.94,-0.72 0,0 Q7.94,0.72 17.49,5.26"/></g></g><g/></g><g/><g/><g/></svg>
+ <svg xmlns="http://www.w3.org/2000/svg" width="649" height="707.4583740234375" style="width:649px;height:707.4583740234375px;font-family:Asana-Math, Asana;background:transparent;"><g><g><g><g><g><g style="transform:matrix(1,0,0,1,12.171875,40.31333587646485);"><path d="M175 386L316 386L316 444L175 444L175 571L106 571L106 444L19 444L19 386L103 386L103 119C103 59 117 -11 186 -11C256 -11 307 14 332 27L316 86C290 65 258 53 226 53C189 53 175 83 175 136ZM829 220C829 354 729 461 610 461C487 461 390 351 390 220C390 88 492 -11 609 -11C729 -11 829 90 829 220ZM609 53C540 53 468 109 468 230C468 351 544 400 609 400C679 400 751 348 751 230C751 112 683 53 609 53ZM1140 272L1308 444L1218 444L1015 236L1015 694L943 694L943 0L1012 0L1012 141L1092 224L1248 0L1330 0ZM1760 219C1760 421 1653 461 1582 461C1471 461 1381 355 1381 226C1381 94 1477 -11 1597 -11C1660 -11 1717 13 1756 41L1750 106C1687 54 1621 50 1598 50C1518 50 1454 121 1451 219ZM1456 274C1472 350 1525 400 1582 400C1634 400 1690 366 1703 274ZM2224 298C2224 364 2209 455 2087 455C1997 455 1948 387 1942 379L1942 450L1870 450L1870 0L1948 0L1948 245C1948 311 1973 394 2049 394C2145 394 2146 323 2146 291L2146 0L2224 0Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g style="transform:matrix(1,0,0,1,68.578125,40.31333587646485);"><path d="M146 266C146 526 243 632 301 700L282 726C225 675 60 542 60 266C60 159 85 58 133 -32C168 -99 200 -138 282 -215L301 -194C255 -137 146 -15 146 266Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g style="transform:matrix(1,0,0,1,76.71356201171875,40.31333587646485);"><path d="M435 298C435 364 420 455 298 455C208 455 159 387 153 379L153 450L81 450L81 0L159 0L159 245C159 311 184 394 260 394C356 394 357 323 357 291L357 0L435 0Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g><g><g><g style="transform:matrix(1,0,0,1,90.05731201171875,44.635999999999996);"><path d="M299 689L279 689C220 627 137 624 89 622L89 563C122 564 170 566 220 587L220 59L95 59L95 0L424 0L424 59L299 59Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017136,0,0,-0.017136,0,0);"></path></g></g></g></g><g style="transform:matrix(1,0,0,1,100.078125,40.31333587646485);"><path d="M51 726L32 700C87 636 187 526 187 266C187 -10 83 -131 32 -194L51 -215C104 -165 273 -23 273 265C273 542 108 675 51 726Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g style="transform:matrix(1,0,0,1,115.55731201171875,40.31333587646485);"><path d="M604 347L604 406L65 406L65 347ZM604 134L604 193L65 193L65 134Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g style="transform:matrix(1,0,0,1,139.2552490234375,40.31333587646485);"><path d="M175 386L316 386L316 444L175 444L175 571L106 571L106 444L19 444L19 386L103 386L103 119C103 59 117 -11 186 -11C256 -11 307 14 332 27L316 86C290 65 258 53 226 53C189 53 175 83 175 136Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g><g><g><g style="transform:matrix(1,0,0,1,148.80731201171875,44.635999999999996);"><path d="M299 689L279 689C220 627 137 624 89 622L89 563C122 564 170 566 220 587L220 59L95 59L95 0L424 0L424 59L299 59Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017136,0,0,-0.017136,0,0);"></path></g></g></g></g></g><g><g style="transform:matrix(1,0,0,1,12.171875,71.98);"><path d="M175 386L316 386L316 444L175 444L175 571L106 571L106 444L19 444L19 386L103 386L103 119C103 59 117 -11 186 -11C256 -11 307 14 332 27L316 86C290 65 258 53 226 53C189 53 175 83 175 136ZM829 220C829 354 729 461 610 461C487 461 390 351 390 220C390 88 492 -11 609 -11C729 -11 829 90 829 220ZM609 53C540 53 468 109 468 230C468 351 544 400 609 400C679 400 751 348 751 230C751 112 683 53 609 53ZM1140 272L1308 444L1218 444L1015 236L1015 694L943 694L943 0L1012 0L1012 141L1092 224L1248 0L1330 0ZM1760 219C1760 421 1653 461 1582 461C1471 461 1381 355 1381 226C1381 94 1477 -11 1597 -11C1660 -11 1717 13 1756 41L1750 106C1687 54 1621 50 1598 50C1518 50 1454 121 1451 219ZM1456 274C1472 350 1525 400 1582 400C1634 400 1690 366 1703 274ZM2224 298C2224 364 2209 455 2087 455C1997 455 1948 387 1942 379L1942 450L1870 450L1870 0L1948 0L1948 245C1948 311 1973 394 2049 394C2145 394 2146 323 2146 291L2146 0L2224 0Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g style="transform:matrix(1,0,0,1,68.578125,71.98);"><path d="M146 266C146 526 243 632 301 700L282 726C225 675 60 542 60 266C60 159 85 58 133 -32C168 -99 200 -138 282 -215L301 -194C255 -137 146 -15 146 266Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g style="transform:matrix(1,0,0,1,76.71356201171875,71.98);"><path d="M435 298C435 364 420 455 298 455C208 455 159 387 153 379L153 450L81 450L81 0L159 0L159 245C159 311 184 394 260 394C356 394 357 323 357 291L357 0L435 0Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g><g><g><g style="transform:matrix(1,0,0,1,90.05731201171875,76.30266412353515);"><path d="M83 466C103 545 131 624 222 624C316 624 367 548 367 468C367 382 310 324 251 263L174 191L50 65L50 0L449 0L449 72L267 72C255 72 243 71 231 71L122 71C154 100 230 176 261 205C333 274 449 347 449 471C449 587 368 689 236 689C122 689 66 610 42 522C66 487 59 501 83 466Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017136,0,0,-0.017136,0,0);"></path></g></g></g></g><g style="transform:matrix(1,0,0,1,100.078125,71.98);"><path d="M51 726L32 700C87 636 187 526 187 266C187 -10 83 -131 32 -194L51 -215C104 -165 273 -23 273 265C273 542 108 675 51 726Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g style="transform:matrix(1,0,0,1,115.55731201171875,71.98);"><path d="M604 347L604 406L65 406L65 347ZM604 134L604 193L65 193L65 134Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g style="transform:matrix(1,0,0,1,139.2552490234375,71.98);"><path d="M175 386L316 386L316 444L175 444L175 571L106 571L106 444L19 444L19 386L103 386L103 119C103 59 117 -11 186 -11C256 -11 307 14 332 27L316 86C290 65 258 53 226 53C189 53 175 83 175 136Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g><g><g><g style="transform:matrix(1,0,0,1,148.80731201171875,76.30266412353515);"><path d="M83 466C103 545 131 624 222 624C316 624 367 548 367 468C367 382 310 324 251 263L174 191L50 65L50 0L449 0L449 72L267 72C255 72 243 71 231 71L122 71C154 100 230 176 261 205C333 274 449 347 449 471C449 587 368 689 236 689C122 689 66 610 42 522C66 487 59 501 83 466Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017136,0,0,-0.017136,0,0);"></path></g></g></g></g></g><g><g style="transform:matrix(1,0,0,1,12.171875,103.64666412353516);"><path d="M175 386L316 386L316 444L175 444L175 571L106 571L106 444L19 444L19 386L103 386L103 119C103 59 117 -11 186 -11C256 -11 307 14 332 27L316 86C290 65 258 53 226 53C189 53 175 83 175 136ZM829 220C829 354 729 461 610 461C487 461 390 351 390 220C390 88 492 -11 609 -11C729 -11 829 90 829 220ZM609 53C540 53 468 109 468 230C468 351 544 400 609 400C679 400 751 348 751 230C751 112 683 53 609 53ZM1140 272L1308 444L1218 444L1015 236L1015 694L943 694L943 0L1012 0L1012 141L1092 224L1248 0L1330 0ZM1760 219C1760 421 1653 461 1582 461C1471 461 1381 355 1381 226C1381 94 1477 -11 1597 -11C1660 -11 1717 13 1756 41L1750 106C1687 54 1621 50 1598 50C1518 50 1454 121 1451 219ZM1456 274C1472 350 1525 400 1582 400C1634 400 1690 366 1703 274ZM2224 298C2224 364 2209 455 2087 455C1997 455 1948 387 1942 379L1942 450L1870 450L1870 0L1948 0L1948 245C1948 311 1973 394 2049 394C2145 394 2146 323 2146 291L2146 0L2224 0Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g style="transform:matrix(1,0,0,1,68.578125,103.64666412353516);"><path d="M146 266C146 526 243 632 301 700L282 726C225 675 60 542 60 266C60 159 85 58 133 -32C168 -99 200 -138 282 -215L301 -194C255 -137 146 -15 146 266Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g style="transform:matrix(1,0,0,1,76.71356201171875,103.64666412353516);"><path d="M435 298C435 364 420 455 298 455C208 455 159 387 153 379L153 450L81 450L81 0L159 0L159 245C159 311 184 394 260 394C356 394 357 323 357 291L357 0L435 0Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g><g><g><g style="transform:matrix(1,0,0,1,90.05731201171875,107.96933587646484);"><path d="M92 522C121 593 186 630 247 630C299 630 348 600 348 535C348 473 307 413 246 398C240 397 238 397 167 391L167 329L238 329C346 329 368 235 368 184C368 105 322 40 245 40C176 40 97 75 53 144L42 83C115 -12 207 -22 247 -22C369 -22 457 76 457 183C457 275 387 338 319 360C395 401 430 471 430 535C430 622 347 689 248 689C171 689 98 648 56 577Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017136,0,0,-0.017136,0,0);"></path></g></g></g></g><g style="transform:matrix(1,0,0,1,100.078125,103.64666412353516);"><path d="M51 726L32 700C87 636 187 526 187 266C187 -10 83 -131 32 -194L51 -215C104 -165 273 -23 273 265C273 542 108 675 51 726Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g style="transform:matrix(1,0,0,1,115.55731201171875,103.64666412353516);"><path d="M604 347L604 406L65 406L65 347ZM604 134L604 193L65 193L65 134Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g style="transform:matrix(1,0,0,1,139.2552490234375,103.64666412353516);"><path d="M175 386L316 386L316 444L175 444L175 571L106 571L106 444L19 444L19 386L103 386L103 119C103 59 117 -11 186 -11C256 -11 307 14 332 27L316 86C290 65 258 53 226 53C189 53 175 83 175 136Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g><g><g><g style="transform:matrix(1,0,0,1,148.80731201171875,107.96933587646484);"><path d="M92 522C121 593 186 630 247 630C299 630 348 600 348 535C348 473 307 413 246 398C240 397 238 397 167 391L167 329L238 329C346 329 368 235 368 184C368 105 322 40 245 40C176 40 97 75 53 144L42 83C115 -12 207 -22 247 -22C369 -22 457 76 457 183C457 275 387 338 319 360C395 401 430 471 430 535C430 622 347 689 248 689C171 689 98 648 56 577Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017136,0,0,-0.017136,0,0);"></path></g></g></g></g></g><g><g style="transform:matrix(1,0,0,1,12.171875,135.31333587646483);"><path d="M124 111C94 111 67 83 67 53C67 23 94 -5 123 -5C155 -5 183 22 183 53C183 83 155 111 124 111ZM373 111C343 111 316 83 316 53C316 23 343 -5 372 -5C404 -5 432 22 432 53C432 83 404 111 373 111ZM622 111C592 111 565 83 565 53C565 23 592 -5 621 -5C653 -5 681 22 681 53C681 83 653 111 622 111Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g></g></g></g></g></g><g><g><g><g><g><g style="transform:matrix(1,0,0,1,245.3802490234375,659.4047891235351);"><path d="M24 388L31 368L63 389C100 412 103 414 110 414C121 414 128 404 128 389C128 338 87 145 46 2L53 -9C78 -2 101 4 123 8C142 134 163 199 209 268C263 352 338 414 383 414C394 414 400 405 400 390C400 372 397 351 389 319L337 107C328 70 324 47 324 31C324 6 335 -9 354 -9C380 -9 416 12 514 85L504 103L478 86C449 67 427 56 417 56C410 56 404 65 404 76C404 81 405 92 406 96L472 372C479 401 483 429 483 446C483 469 472 482 452 482C410 482 341 444 282 389C244 354 216 320 164 247L202 408C206 426 208 438 208 449C208 470 200 482 185 482C164 482 125 460 52 408Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.02941,0,0,-0.02941,0,0);"></path></g><g><g><g><g style="transform:matrix(1,0,0,1,262.578125,665.2589131469726);"><path d="M459 253C459 366 378 446 264 446C216 446 180 443 127 396L127 605L432 604L432 689L75 690L75 322L95 316C142 363 169 377 218 377C314 377 374 309 374 201C374 90 310 25 201 25C147 25 97 43 83 69L37 151L13 137C36 80 48 48 62 4C90 -11 130 -20 173 -20C301 -20 459 89 459 253Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.020587,0,0,-0.020587,0,0);"></path></g></g></g></g></g></g></g></g></g><g><g><g><g><g><g style="transform:matrix(1,0,0,1,465.3802490234375,444.4047891235352);"><path d="M24 388L31 368L63 389C100 412 103 414 110 414C121 414 128 404 128 389C128 338 87 145 46 2L53 -9C78 -2 101 4 123 8C142 134 163 199 209 268C263 352 338 414 383 414C394 414 400 405 400 390C400 372 397 351 389 319L337 107C328 70 324 47 324 31C324 6 335 -9 354 -9C380 -9 416 12 514 85L504 103L478 86C449 67 427 56 417 56C410 56 404 65 404 76C404 81 405 92 406 96L472 372C479 401 483 429 483 446C483 469 472 482 452 482C410 482 341 444 282 389C244 354 216 320 164 247L202 408C206 426 208 438 208 449C208 470 200 482 185 482C164 482 125 460 52 408Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.02941,0,0,-0.02941,0,0);"></path></g><g><g><g><g style="transform:matrix(1,0,0,1,482.578125,450.2588826293945);"><path d="M462 224C462 345 355 366 308 374C388 436 418 482 418 541C418 630 344 689 233 689C165 689 120 670 72 622L43 498L74 498L92 554C103 588 166 622 218 622C283 622 336 569 336 506C336 431 277 368 206 368C198 368 187 369 174 370L159 371L147 318L154 312C192 329 211 334 238 334C321 334 369 281 369 190C369 88 308 21 215 21C169 21 128 36 98 64C74 86 61 109 42 163L15 153C36 92 44 56 50 6C103 -12 147 -20 184 -20C307 -20 462 87 462 224Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.020587,0,0,-0.020587,0,0);"></path></g></g></g></g></g></g></g></g></g><g><g><g><g><g><g style="transform:matrix(1,0,0,1,25.380218505859375,444.4047891235352);"><path d="M24 388L31 368L63 389C100 412 103 414 110 414C121 414 128 404 128 389C128 338 87 145 46 2L53 -9C78 -2 101 4 123 8C142 134 163 199 209 268C263 352 338 414 383 414C394 414 400 405 400 390C400 372 397 351 389 319L337 107C328 70 324 47 324 31C324 6 335 -9 354 -9C380 -9 416 12 514 85L504 103L478 86C449 67 427 56 417 56C410 56 404 65 404 76C404 81 405 92 406 96L472 372C479 401 483 429 483 446C483 469 472 482 452 482C410 482 341 444 282 389C244 354 216 320 164 247L202 408C206 426 208 438 208 449C208 470 200 482 185 482C164 482 125 460 52 408Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.02941,0,0,-0.02941,0,0);"></path></g><g><g><g><g style="transform:matrix(1,0,0,1,42.578125,450.2588826293945);"><path d="M409 603L47 -1L157 -1L497 659L497 689L44 689L44 477L74 477L81 533C89 595 96 603 142 603Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.020587,0,0,-0.020587,0,0);"></path></g></g></g></g></g></g></g></g></g><g><g><g><g><g><g style="transform:matrix(1,0,0,1,405.3802490234375,284.4047891235352);"><path d="M24 388L31 368L63 389C100 412 103 414 110 414C121 414 128 404 128 389C128 338 87 145 46 2L53 -9C78 -2 101 4 123 8C142 134 163 199 209 268C263 352 338 414 383 414C394 414 400 405 400 390C400 372 397 351 389 319L337 107C328 70 324 47 324 31C324 6 335 -9 354 -9C380 -9 416 12 514 85L504 103L478 86C449 67 427 56 417 56C410 56 404 65 404 76C404 81 405 92 406 96L472 372C479 401 483 429 483 446C483 469 472 482 452 482C410 482 341 444 282 389C244 354 216 320 164 247L202 408C206 426 208 438 208 449C208 470 200 482 185 482C164 482 125 460 52 408Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.02941,0,0,-0.02941,0,0);"></path></g><g><g><g><g style="transform:matrix(1,0,0,1,422.578125,290.2588826293945);"><path d="M16 23L16 -3C203 -3 203 0 239 0C275 0 275 -3 468 -3L468 82C353 77 307 81 122 77L304 270C401 373 431 428 431 503C431 618 353 689 226 689C154 689 105 669 56 619L39 483L68 483L81 529C97 587 133 612 200 612C286 612 341 558 341 473C341 398 299 324 186 204Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.020587,0,0,-0.020587,0,0);"></path></g></g></g></g></g></g></g></g></g><g><g><g><g><g><g style="transform:matrix(1,0,0,1,405.3802490234375,604.4047891235351);"><path d="M24 388L31 368L63 389C100 412 103 414 110 414C121 414 128 404 128 389C128 338 87 145 46 2L53 -9C78 -2 101 4 123 8C142 134 163 199 209 268C263 352 338 414 383 414C394 414 400 405 400 390C400 372 397 351 389 319L337 107C328 70 324 47 324 31C324 6 335 -9 354 -9C380 -9 416 12 514 85L504 103L478 86C449 67 427 56 417 56C410 56 404 65 404 76C404 81 405 92 406 96L472 372C479 401 483 429 483 446C483 469 472 482 452 482C410 482 341 444 282 389C244 354 216 320 164 247L202 408C206 426 208 438 208 449C208 470 200 482 185 482C164 482 125 460 52 408Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.02941,0,0,-0.02941,0,0);"></path></g><g><g><g><g style="transform:matrix(1,0,0,1,422.578125,610.2589131469726);"><path d="M280 181L280 106C280 46 269 32 220 30L158 27L158 -3C291 0 291 0 315 0C339 0 339 0 472 -3L472 27L424 30C375 33 364 46 364 106L364 181C423 181 444 180 472 177L472 248L364 245L365 697L285 667L2 204L2 181ZM280 245L65 245L280 597Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.020587,0,0,-0.020587,0,0);"></path></g></g></g></g></g></g></g></g></g><g><g><g><g><g><g style="transform:matrix(1,0,0,1,85.38021850585938,604.4047891235351);"><path d="M24 388L31 368L63 389C100 412 103 414 110 414C121 414 128 404 128 389C128 338 87 145 46 2L53 -9C78 -2 101 4 123 8C142 134 163 199 209 268C263 352 338 414 383 414C394 414 400 405 400 390C400 372 397 351 389 319L337 107C328 70 324 47 324 31C324 6 335 -9 354 -9C380 -9 416 12 514 85L504 103L478 86C449 67 427 56 417 56C410 56 404 65 404 76C404 81 405 92 406 96L472 372C479 401 483 429 483 446C483 469 472 482 452 482C410 482 341 444 282 389C244 354 216 320 164 247L202 408C206 426 208 438 208 449C208 470 200 482 185 482C164 482 125 460 52 408Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.02941,0,0,-0.02941,0,0);"></path></g><g><g><g><g style="transform:matrix(1,0,0,1,102.578125,610.2589131469726);"><path d="M131 331C152 512 241 611 421 665L379 689C283 657 242 637 191 593C88 506 32 384 32 247C32 82 112 -20 241 -20C371 -20 468 83 468 219C468 334 399 409 293 409C216 409 184 370 131 331ZM255 349C331 349 382 283 382 184C382 80 331 13 254 13C169 13 123 86 123 220C123 255 127 274 138 291C160 325 207 349 255 349Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.020587,0,0,-0.020587,0,0);"></path></g></g></g></g></g></g></g></g></g><g><g><g><g><g><g style="transform:matrix(1,0,0,1,332.04168701171875,37.480000000000004);"><path d="M157 214C157 314 229 386 327 388L327 455C238 454 183 405 152 359L152 450L82 450L82 0L157 0ZM739 289C739 391 666 461 574 461C509 461 464 445 417 418L423 352C475 389 525 402 574 402C621 402 661 362 661 288L661 245C511 243 384 201 384 113C384 70 411 -11 498 -11C512 -11 606 -9 664 36L664 0L739 0ZM661 132C661 113 661 88 627 69C598 51 560 50 549 50C501 50 456 73 456 115C456 185 618 192 661 194ZM1254 298C1254 364 1239 455 1117 455C1027 455 978 387 972 379L972 450L900 450L900 0L978 0L978 245C978 311 1003 394 1079 394C1175 394 1176 323 1176 291L1176 0L1254 0ZM1686 391C1708 391 1736 395 1760 395C1778 395 1817 392 1819 392L1808 455C1738 455 1680 436 1650 423C1629 440 1595 455 1555 455C1469 455 1396 383 1396 292C1396 255 1409 219 1429 193C1400 152 1400 113 1400 108C1400 82 1409 53 1426 32C1374 1 1362 -45 1362 -71C1362 -146 1461 -206 1583 -206C1706 -206 1805 -147 1805 -70C1805 69 1638 69 1599 69L1511 69C1498 69 1453 69 1453 122C1453 133 1457 149 1464 158C1485 143 1518 129 1555 129C1645 129 1715 203 1715 292C1715 340 1693 377 1682 392ZM1555 186C1518 186 1466 209 1466 292C1466 375 1518 398 1555 398C1598 398 1645 370 1645 292C1645 214 1598 186 1555 186ZM1600 -3C1622 -3 1735 -3 1735 -72C1735 -116 1666 -149 1584 -149C1503 -149 1432 -118 1432 -71C1432 -68 1432 -3 1510 -3ZM2247 219C2247 421 2140 461 2069 461C1958 461 1868 355 1868 226C1868 94 1964 -11 2084 -11C2147 -11 2204 13 2243 41L2237 106C2174 54 2108 50 2085 50C2005 50 1941 121 1938 219ZM1943 274C1959 350 2012 400 2069 400C2121 400 2177 366 2190 274Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g style="transform:matrix(1,0,0,1,387.76043701171875,37.480000000000004);"><path d="M146 266C146 526 243 632 301 700L282 726C225 675 60 542 60 266C60 159 85 58 133 -32C168 -99 200 -138 282 -215L301 -194C255 -137 146 -15 146 266Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g style="transform:matrix(1,0,0,1,395.8958740234375,37.480000000000004);"><path d="M175 386L316 386L316 444L175 444L175 571L106 571L106 444L19 444L19 386L103 386L103 119C103 59 117 -11 186 -11C256 -11 307 14 332 27L316 86C290 65 258 53 226 53C189 53 175 83 175 136Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g><g><g><g style="transform:matrix(1,0,0,1,405.44793701171875,41.80266412353515);"><path d="M299 689L279 689C220 627 137 624 89 622L89 563C122 564 170 566 220 587L220 59L95 59L95 0L424 0L424 59L299 59Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017136,0,0,-0.017136,0,0);"></path></g></g></g></g><g style="transform:matrix(1,0,0,1,415.46875,37.480000000000004);"><path d="M204 123C177 114 159 108 106 93C99 17 74 -48 16 -144L30 -155L71 -136C152 -31 190 32 218 109Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g style="transform:matrix(1,0,0,1,426.46875,37.480000000000004);"><path d="M424 386L565 386L565 444L424 444L424 571L355 571L355 444L268 444L268 386L352 386L352 119C352 59 366 -11 435 -11C505 -11 556 14 581 27L565 86C539 65 507 53 475 53C438 53 424 83 424 136Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g><g><g><g style="transform:matrix(1,0,0,1,442.1146240234375,41.80266412353515);"><path d="M83 466C103 545 131 624 222 624C316 624 367 548 367 468C367 382 310 324 251 263L174 191L50 65L50 0L449 0L449 72L267 72C255 72 243 71 231 71L122 71C154 100 230 176 261 205C333 274 449 347 449 471C449 587 368 689 236 689C122 689 66 610 42 522C66 487 59 501 83 466Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017136,0,0,-0.017136,0,0);"></path></g></g></g></g><g style="transform:matrix(1,0,0,1,452.13543701171875,37.480000000000004);"><path d="M245 -184L254 -171C248 -140 246 -112 246 -44L246 578C246 628 250 665 250 691C250 706 249 717 245 726L51 726L45 720L45 694L49 689L87 689C114 689 136 683 148 673C157 664 160 649 160 616L160 -75C160 -108 157 -122 148 -131C136 -141 114 -147 87 -147L49 -147L45 -152L45 -178L51 -184Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g style="transform:matrix(1,0,0,1,460.2708740234375,37.480000000000004);"><path d="" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g><g style="transform:matrix(1,0,0,1,471.2708740234375,37.480000000000004);"><path d="M949 272L743 486L711 452L836 300L65 300L65 241L836 241L711 89L743 55Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g></g><g style="transform:matrix(1,0,0,1,500.96875,37.480000000000004);"><path d="M289 -175C226 -161 206 -117 206 -45L206 128C206 207 197 253 125 272L125 274C194 292 206 335 206 409L206 595C206 667 224 707 289 726C189 726 134 703 134 578L134 392C134 327 120 292 58 273C124 254 134 223 134 151L134 -17C134 -149 176 -175 289 -175Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g style="transform:matrix(1,0,0,1,510.32293701171875,37.480000000000004);"><path d="M435 298C435 364 420 455 298 455C208 455 159 387 153 379L153 450L81 450L81 0L159 0L159 245C159 311 184 394 260 394C356 394 357 323 357 291L357 0L435 0Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g><g><g><g style="transform:matrix(1,0,0,1,523.6666870117188,41.80266412353515);"><path d="M83 466C103 545 131 624 222 624C316 624 367 548 367 468C367 382 310 324 251 263L174 191L50 65L50 0L449 0L449 72L267 72C255 72 243 71 231 71L122 71C154 100 230 176 261 205C333 274 449 347 449 471C449 587 368 689 236 689C122 689 66 610 42 522C66 487 59 501 83 466Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017136,0,0,-0.017136,0,0);"></path></g></g></g></g><g style="transform:matrix(1,0,0,1,533.6875,37.480000000000004);"><path d="M204 123C177 114 159 108 106 93C99 17 74 -48 16 -144L30 -155L71 -136C152 -31 190 32 218 109Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g style="transform:matrix(1,0,0,1,544.6875,37.480000000000004);"><path d="M684 298C684 364 669 455 547 455C457 455 408 387 402 379L402 450L330 450L330 0L408 0L408 245C408 311 433 394 509 394C605 394 606 323 606 291L606 0L684 0Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g><g><g><g style="transform:matrix(1,0,0,1,564.125,41.80266412353515);"><path d="M92 522C121 593 186 630 247 630C299 630 348 600 348 535C348 473 307 413 246 398C240 397 238 397 167 391L167 329L238 329C346 329 368 235 368 184C368 105 322 40 245 40C176 40 97 75 53 144L42 83C115 -12 207 -22 247 -22C369 -22 457 76 457 183C457 275 387 338 319 360C395 401 430 471 430 535C430 622 347 689 248 689C171 689 98 648 56 577Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017136,0,0,-0.017136,0,0);"></path></g></g></g></g><g style="transform:matrix(1,0,0,1,574.1458740234375,37.480000000000004);"><path d="M204 123C177 114 159 108 106 93C99 17 74 -48 16 -144L30 -155L71 -136C152 -31 190 32 218 109Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g style="transform:matrix(1,0,0,1,585.1458740234375,37.480000000000004);"><path d="M684 298C684 364 669 455 547 455C457 455 408 387 402 379L402 450L330 450L330 0L408 0L408 245C408 311 433 394 509 394C605 394 606 323 606 291L606 0L684 0Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g><g><g><g style="transform:matrix(1,0,0,1,604.5833740234375,41.80266412353515);"><path d="M372 174L471 174L471 236L372 236L372 667L281 667L28 236L28 174L293 174L293 0L372 0ZM106 236C158 323 299 564 299 622L299 236Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017136,0,0,-0.017136,0,0);"></path></g></g></g></g><g style="transform:matrix(1,0,0,1,615.822998046875,37.480000000000004);"><path d="M275 273C213 292 199 327 199 392L199 578C199 703 144 726 44 726C109 707 127 667 127 595L127 409C127 335 139 292 208 274L208 272C136 253 127 207 127 128L127 -45C127 -117 107 -161 44 -175C157 -175 199 -149 199 -17L199 151C199 223 209 254 275 273Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g></g></g></g></g><g><g><g><g><g style="transform:matrix(1,0,0,1,332.04168701171875,70.14666412353516);"><path d="M157 214C157 314 229 386 327 388L327 455C238 454 183 405 152 359L152 450L82 450L82 0L157 0ZM739 289C739 391 666 461 574 461C509 461 464 445 417 418L423 352C475 389 525 402 574 402C621 402 661 362 661 288L661 245C511 243 384 201 384 113C384 70 411 -11 498 -11C512 -11 606 -9 664 36L664 0L739 0ZM661 132C661 113 661 88 627 69C598 51 560 50 549 50C501 50 456 73 456 115C456 185 618 192 661 194ZM1254 298C1254 364 1239 455 1117 455C1027 455 978 387 972 379L972 450L900 450L900 0L978 0L978 245C978 311 1003 394 1079 394C1175 394 1176 323 1176 291L1176 0L1254 0ZM1686 391C1708 391 1736 395 1760 395C1778 395 1817 392 1819 392L1808 455C1738 455 1680 436 1650 423C1629 440 1595 455 1555 455C1469 455 1396 383 1396 292C1396 255 1409 219 1429 193C1400 152 1400 113 1400 108C1400 82 1409 53 1426 32C1374 1 1362 -45 1362 -71C1362 -146 1461 -206 1583 -206C1706 -206 1805 -147 1805 -70C1805 69 1638 69 1599 69L1511 69C1498 69 1453 69 1453 122C1453 133 1457 149 1464 158C1485 143 1518 129 1555 129C1645 129 1715 203 1715 292C1715 340 1693 377 1682 392ZM1555 186C1518 186 1466 209 1466 292C1466 375 1518 398 1555 398C1598 398 1645 370 1645 292C1645 214 1598 186 1555 186ZM1600 -3C1622 -3 1735 -3 1735 -72C1735 -116 1666 -149 1584 -149C1503 -149 1432 -118 1432 -71C1432 -68 1432 -3 1510 -3ZM2247 219C2247 421 2140 461 2069 461C1958 461 1868 355 1868 226C1868 94 1964 -11 2084 -11C2147 -11 2204 13 2243 41L2237 106C2174 54 2108 50 2085 50C2005 50 1941 121 1938 219ZM1943 274C1959 350 2012 400 2069 400C2121 400 2177 366 2190 274Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g style="transform:matrix(1,0,0,1,387.76043701171875,70.14666412353516);"><path d="M146 266C146 526 243 632 301 700L282 726C225 675 60 542 60 266C60 159 85 58 133 -32C168 -99 200 -138 282 -215L301 -194C255 -137 146 -15 146 266Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g style="transform:matrix(1,0,0,1,395.8958740234375,70.14666412353516);"><path d="M175 386L316 386L316 444L175 444L175 571L106 571L106 444L19 444L19 386L103 386L103 119C103 59 117 -11 186 -11C256 -11 307 14 332 27L316 86C290 65 258 53 226 53C189 53 175 83 175 136Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g><g><g><g style="transform:matrix(1,0,0,1,405.44793701171875,74.46933587646484);"><path d="M83 466C103 545 131 624 222 624C316 624 367 548 367 468C367 382 310 324 251 263L174 191L50 65L50 0L449 0L449 72L267 72C255 72 243 71 231 71L122 71C154 100 230 176 261 205C333 274 449 347 449 471C449 587 368 689 236 689C122 689 66 610 42 522C66 487 59 501 83 466Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017136,0,0,-0.017136,0,0);"></path></g></g></g></g><g style="transform:matrix(1,0,0,1,415.46875,70.14666412353516);"><path d="M204 123C177 114 159 108 106 93C99 17 74 -48 16 -144L30 -155L71 -136C152 -31 190 32 218 109Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g style="transform:matrix(1,0,0,1,426.46875,70.14666412353516);"><path d="M424 386L565 386L565 444L424 444L424 571L355 571L355 444L268 444L268 386L352 386L352 119C352 59 366 -11 435 -11C505 -11 556 14 581 27L565 86C539 65 507 53 475 53C438 53 424 83 424 136Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g><g><g><g style="transform:matrix(1,0,0,1,442.1146240234375,74.46933587646484);"><path d="M92 522C121 593 186 630 247 630C299 630 348 600 348 535C348 473 307 413 246 398C240 397 238 397 167 391L167 329L238 329C346 329 368 235 368 184C368 105 322 40 245 40C176 40 97 75 53 144L42 83C115 -12 207 -22 247 -22C369 -22 457 76 457 183C457 275 387 338 319 360C395 401 430 471 430 535C430 622 347 689 248 689C171 689 98 648 56 577Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017136,0,0,-0.017136,0,0);"></path></g></g></g></g><g style="transform:matrix(1,0,0,1,452.13543701171875,70.14666412353516);"><path d="M245 -184L254 -171C248 -140 246 -112 246 -44L246 578C246 628 250 665 250 691C250 706 249 717 245 726L51 726L45 720L45 694L49 689L87 689C114 689 136 683 148 673C157 664 160 649 160 616L160 -75C160 -108 157 -122 148 -131C136 -141 114 -147 87 -147L49 -147L45 -152L45 -178L51 -184Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g style="transform:matrix(1,0,0,1,460.2708740234375,70.14666412353516);"><path d="" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g><g style="transform:matrix(1,0,0,1,471.2708740234375,70.14666412353516);"><path d="M949 272L743 486L711 452L836 300L65 300L65 241L836 241L711 89L743 55Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g></g><g style="transform:matrix(1,0,0,1,500.96875,70.14666412353516);"><path d="M289 -175C226 -161 206 -117 206 -45L206 128C206 207 197 253 125 272L125 274C194 292 206 335 206 409L206 595C206 667 224 707 289 726C189 726 134 703 134 578L134 392C134 327 120 292 58 273C124 254 134 223 134 151L134 -17C134 -149 176 -175 289 -175Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g style="transform:matrix(1,0,0,1,510.32293701171875,70.14666412353516);"><path d="M435 298C435 364 420 455 298 455C208 455 159 387 153 379L153 450L81 450L81 0L159 0L159 245C159 311 184 394 260 394C356 394 357 323 357 291L357 0L435 0Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g><g><g><g style="transform:matrix(1,0,0,1,523.6666870117188,74.46933587646484);"><path d="M92 522C121 593 186 630 247 630C299 630 348 600 348 535C348 473 307 413 246 398C240 397 238 397 167 391L167 329L238 329C346 329 368 235 368 184C368 105 322 40 245 40C176 40 97 75 53 144L42 83C115 -12 207 -22 247 -22C369 -22 457 76 457 183C457 275 387 338 319 360C395 401 430 471 430 535C430 622 347 689 248 689C171 689 98 648 56 577Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017136,0,0,-0.017136,0,0);"></path></g></g></g></g><g style="transform:matrix(1,0,0,1,533.6875,70.14666412353516);"><path d="M204 123C177 114 159 108 106 93C99 17 74 -48 16 -144L30 -155L71 -136C152 -31 190 32 218 109Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g style="transform:matrix(1,0,0,1,544.6875,70.14666412353516);"><path d="M684 298C684 364 669 455 547 455C457 455 408 387 402 379L402 450L330 450L330 0L408 0L408 245C408 311 433 394 509 394C605 394 606 323 606 291L606 0L684 0Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g><g><g><g style="transform:matrix(1,0,0,1,564.125,74.46933587646484);"><path d="M372 174L471 174L471 236L372 236L372 667L281 667L28 236L28 174L293 174L293 0L372 0ZM106 236C158 323 299 564 299 622L299 236Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017136,0,0,-0.017136,0,0);"></path></g></g></g></g><g style="transform:matrix(1,0,0,1,574.1458740234375,70.14666412353516);"><path d="M204 123C177 114 159 108 106 93C99 17 74 -48 16 -144L30 -155L71 -136C152 -31 190 32 218 109Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g style="transform:matrix(1,0,0,1,585.1458740234375,70.14666412353516);"><path d="M684 298C684 364 669 455 547 455C457 455 408 387 402 379L402 450L330 450L330 0L408 0L408 245C408 311 433 394 509 394C605 394 606 323 606 291L606 0L684 0Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g><g><g><g style="transform:matrix(1,0,0,1,604.5833740234375,74.46933587646484);"><path d="M153 602L416 602L416 667L81 667L81 292L147 292C164 332 201 372 259 372C306 372 360 330 360 208C360 40 238 40 229 40C162 40 101 79 72 134L39 77C80 19 149 -22 230 -22C349 -22 449 78 449 206C449 333 363 434 260 434C220 434 182 419 153 392Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017136,0,0,-0.017136,0,0);"></path></g></g></g></g><g style="transform:matrix(1,0,0,1,615.822998046875,70.14666412353516);"><path d="M275 273C213 292 199 327 199 392L199 578C199 703 144 726 44 726C109 707 127 667 127 595L127 409C127 335 139 292 208 274L208 272C136 253 127 207 127 128L127 -45C127 -117 107 -161 44 -175C157 -175 199 -149 199 -17L199 151C199 223 209 254 275 273Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g></g></g></g></g><g><g><g><g><g style="transform:matrix(1,0,0,1,332.04168701171875,102.81333587646485);"><path d="M157 214C157 314 229 386 327 388L327 455C238 454 183 405 152 359L152 450L82 450L82 0L157 0ZM739 289C739 391 666 461 574 461C509 461 464 445 417 418L423 352C475 389 525 402 574 402C621 402 661 362 661 288L661 245C511 243 384 201 384 113C384 70 411 -11 498 -11C512 -11 606 -9 664 36L664 0L739 0ZM661 132C661 113 661 88 627 69C598 51 560 50 549 50C501 50 456 73 456 115C456 185 618 192 661 194ZM1254 298C1254 364 1239 455 1117 455C1027 455 978 387 972 379L972 450L900 450L900 0L978 0L978 245C978 311 1003 394 1079 394C1175 394 1176 323 1176 291L1176 0L1254 0ZM1686 391C1708 391 1736 395 1760 395C1778 395 1817 392 1819 392L1808 455C1738 455 1680 436 1650 423C1629 440 1595 455 1555 455C1469 455 1396 383 1396 292C1396 255 1409 219 1429 193C1400 152 1400 113 1400 108C1400 82 1409 53 1426 32C1374 1 1362 -45 1362 -71C1362 -146 1461 -206 1583 -206C1706 -206 1805 -147 1805 -70C1805 69 1638 69 1599 69L1511 69C1498 69 1453 69 1453 122C1453 133 1457 149 1464 158C1485 143 1518 129 1555 129C1645 129 1715 203 1715 292C1715 340 1693 377 1682 392ZM1555 186C1518 186 1466 209 1466 292C1466 375 1518 398 1555 398C1598 398 1645 370 1645 292C1645 214 1598 186 1555 186ZM1600 -3C1622 -3 1735 -3 1735 -72C1735 -116 1666 -149 1584 -149C1503 -149 1432 -118 1432 -71C1432 -68 1432 -3 1510 -3ZM2247 219C2247 421 2140 461 2069 461C1958 461 1868 355 1868 226C1868 94 1964 -11 2084 -11C2147 -11 2204 13 2243 41L2237 106C2174 54 2108 50 2085 50C2005 50 1941 121 1938 219ZM1943 274C1959 350 2012 400 2069 400C2121 400 2177 366 2190 274Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g style="transform:matrix(1,0,0,1,387.76043701171875,102.81333587646485);"><path d="M146 266C146 526 243 632 301 700L282 726C225 675 60 542 60 266C60 159 85 58 133 -32C168 -99 200 -138 282 -215L301 -194C255 -137 146 -15 146 266Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g style="transform:matrix(1,0,0,1,395.8958740234375,102.81333587646485);"><path d="M175 386L316 386L316 444L175 444L175 571L106 571L106 444L19 444L19 386L103 386L103 119C103 59 117 -11 186 -11C256 -11 307 14 332 27L316 86C290 65 258 53 226 53C189 53 175 83 175 136Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g><g><g><g style="transform:matrix(1,0,0,1,405.44793701171875,107.13600762939453);"><path d="M92 522C121 593 186 630 247 630C299 630 348 600 348 535C348 473 307 413 246 398C240 397 238 397 167 391L167 329L238 329C346 329 368 235 368 184C368 105 322 40 245 40C176 40 97 75 53 144L42 83C115 -12 207 -22 247 -22C369 -22 457 76 457 183C457 275 387 338 319 360C395 401 430 471 430 535C430 622 347 689 248 689C171 689 98 648 56 577Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017136,0,0,-0.017136,0,0);"></path></g></g></g></g><g style="transform:matrix(1,0,0,1,415.46875,102.81333587646485);"><path d="M204 123C177 114 159 108 106 93C99 17 74 -48 16 -144L30 -155L71 -136C152 -31 190 32 218 109Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g style="transform:matrix(1,0,0,1,426.46875,102.81333587646485);"><path d="M424 386L565 386L565 444L424 444L424 571L355 571L355 444L268 444L268 386L352 386L352 119C352 59 366 -11 435 -11C505 -11 556 14 581 27L565 86C539 65 507 53 475 53C438 53 424 83 424 136Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g><g><g><g style="transform:matrix(1,0,0,1,442.1146240234375,107.13600762939453);"><path d="M372 174L471 174L471 236L372 236L372 667L281 667L28 236L28 174L293 174L293 0L372 0ZM106 236C158 323 299 564 299 622L299 236Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017136,0,0,-0.017136,0,0);"></path></g></g></g></g><g style="transform:matrix(1,0,0,1,452.13543701171875,102.81333587646485);"><path d="M245 -184L254 -171C248 -140 246 -112 246 -44L246 578C246 628 250 665 250 691C250 706 249 717 245 726L51 726L45 720L45 694L49 689L87 689C114 689 136 683 148 673C157 664 160 649 160 616L160 -75C160 -108 157 -122 148 -131C136 -141 114 -147 87 -147L49 -147L45 -152L45 -178L51 -184Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g style="transform:matrix(1,0,0,1,460.2708740234375,102.81333587646485);"><path d="" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g><g style="transform:matrix(1,0,0,1,471.2708740234375,102.81333587646485);"><path d="M949 272L743 486L711 452L836 300L65 300L65 241L836 241L711 89L743 55Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g></g><g style="transform:matrix(1,0,0,1,500.96875,102.81333587646485);"><path d="M289 -175C226 -161 206 -117 206 -45L206 128C206 207 197 253 125 272L125 274C194 292 206 335 206 409L206 595C206 667 224 707 289 726C189 726 134 703 134 578L134 392C134 327 120 292 58 273C124 254 134 223 134 151L134 -17C134 -149 176 -175 289 -175Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g style="transform:matrix(1,0,0,1,510.32293701171875,102.81333587646485);"><path d="M435 298C435 364 420 455 298 455C208 455 159 387 153 379L153 450L81 450L81 0L159 0L159 245C159 311 184 394 260 394C356 394 357 323 357 291L357 0L435 0Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g><g><g><g style="transform:matrix(1,0,0,1,523.6666870117188,107.13600762939453);"><path d="M372 174L471 174L471 236L372 236L372 667L281 667L28 236L28 174L293 174L293 0L372 0ZM106 236C158 323 299 564 299 622L299 236Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017136,0,0,-0.017136,0,0);"></path></g></g></g></g><g style="transform:matrix(1,0,0,1,533.6875,102.81333587646485);"><path d="M204 123C177 114 159 108 106 93C99 17 74 -48 16 -144L30 -155L71 -136C152 -31 190 32 218 109Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g style="transform:matrix(1,0,0,1,544.6875,102.81333587646485);"><path d="M684 298C684 364 669 455 547 455C457 455 408 387 402 379L402 450L330 450L330 0L408 0L408 245C408 311 433 394 509 394C605 394 606 323 606 291L606 0L684 0Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g><g><g><g style="transform:matrix(1,0,0,1,564.125,107.13600762939453);"><path d="M153 602L416 602L416 667L81 667L81 292L147 292C164 332 201 372 259 372C306 372 360 330 360 208C360 40 238 40 229 40C162 40 101 79 72 134L39 77C80 19 149 -22 230 -22C349 -22 449 78 449 206C449 333 363 434 260 434C220 434 182 419 153 392Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017136,0,0,-0.017136,0,0);"></path></g></g></g></g><g style="transform:matrix(1,0,0,1,574.1458740234375,102.81333587646485);"><path d="M204 123C177 114 159 108 106 93C99 17 74 -48 16 -144L30 -155L71 -136C152 -31 190 32 218 109Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g style="transform:matrix(1,0,0,1,585.1458740234375,102.81333587646485);"><path d="M684 298C684 364 669 455 547 455C457 455 408 387 402 379L402 450L330 450L330 0L408 0L408 245C408 311 433 394 509 394C605 394 606 323 606 291L606 0L684 0Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g><g><g><g style="transform:matrix(1,0,0,1,604.5833740234375,107.13600762939453);"><path d="M415 669C364 689 323 689 309 689C168 689 42 546 42 327C42 40 166 -22 251 -22C311 -22 354 1 393 47C438 99 457 145 457 226C457 356 391 469 295 469C263 469 187 461 126 385C139 549 218 630 310 630C348 630 380 623 415 609ZM127 223C127 237 127 239 128 251C128 326 173 407 256 407C304 407 332 382 352 344C373 307 375 271 375 226C375 191 375 144 351 103C334 74 308 40 251 40C145 40 130 189 127 223Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017136,0,0,-0.017136,0,0);"></path></g></g></g></g><g style="transform:matrix(1,0,0,1,615.822998046875,102.81333587646485);"><path d="M275 273C213 292 199 327 199 392L199 578C199 703 144 726 44 726C109 707 127 667 127 595L127 409C127 335 139 292 208 274L208 272C136 253 127 207 127 128L127 -45C127 -117 107 -161 44 -175C157 -175 199 -149 199 -17L199 151C199 223 209 254 275 273Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g></g></g></g></g><g><text x="332.04168701171875" y="113.66666412353516" style="white-space:pre;stroke:none;fill:rgb(0, 0, 0);font-size:21.6px;font-family:Arial, Helvetica, sans-serif;font-weight:400;font-style:normal;dominant-baseline:text-before-edge;text-decoration:none solid rgb(0, 0, 0);"> ...</text></g></g><g><g><g><g><g></g></g></g></g></g><g><g><g><g><g><g style="transform:matrix(1,0,0,1,284.9114990234375,163.04063262939454);"><path d="M435 298C435 364 420 455 298 455C236 455 188 424 156 383L156 694L81 694L81 0L159 0L159 245C159 311 184 394 260 394C356 394 357 323 357 291L357 0L435 0ZM914 289C914 391 841 461 749 461C684 461 639 445 592 418L598 352C650 389 700 402 749 402C796 402 836 362 836 288L836 245C686 243 559 201 559 113C559 70 586 -11 673 -11C687 -11 781 -9 839 36L839 0L914 0ZM836 132C836 113 836 88 802 69C773 51 735 50 724 50C676 50 631 73 631 115C631 185 793 192 836 194ZM1354 128C1354 183 1317 217 1315 220C1276 255 1249 261 1199 270C1144 281 1098 291 1098 340C1098 402 1170 402 1183 402C1215 402 1268 398 1325 364L1337 429C1285 453 1244 461 1193 461C1168 461 1027 461 1027 330C1027 281 1056 249 1081 230C1112 208 1134 204 1189 193C1225 186 1283 174 1283 121C1283 52 1204 52 1189 52C1108 52 1052 89 1034 101L1022 33C1054 17 1109 -11 1190 -11C1328 -11 1354 75 1354 128ZM1811 298C1811 364 1796 455 1674 455C1612 455 1564 424 1532 383L1532 694L1457 694L1457 0L1535 0L1535 245C1535 311 1560 394 1636 394C1732 394 1733 323 1733 291L1733 0L1811 0Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.020399999999999998,0,0,-0.020399999999999998,0,0);"></path></g><g style="transform:matrix(1,0,0,1,323.46356201171875,163.70728912353516);"><path d="M146 266C146 526 243 632 301 700L282 726C225 675 60 542 60 266C60 159 85 58 133 -32C168 -99 200 -138 282 -215L301 -194C255 -137 146 -15 146 266Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.020399999999999998,0,0,-0.020399999999999998,0,0);"></path></g><g style="transform:matrix(1,0,0,1,330.234375,163.04063262939454);"><path d="M105 469L137 665C138 667 138 669 138 671C138 689 116 709 95 709C74 709 52 689 52 670L52 665L85 469ZM286 469L318 665C319 667 319 669 319 670C319 689 297 709 276 709C254 709 233 690 233 670L233 665L266 469ZM546 386L656 386L656 444L543 444L543 563C543 637 610 644 636 644C656 644 683 642 717 627L717 694C705 697 674 705 637 705C543 705 471 634 471 534L471 444L397 444L397 386L471 386L471 0L546 0ZM1143 220C1143 354 1043 461 924 461C801 461 704 351 704 220C704 88 806 -11 923 -11C1043 -11 1143 90 1143 220ZM923 53C854 53 782 109 782 230C782 351 858 400 923 400C993 400 1065 348 1065 230C1065 112 997 53 923 53ZM1642 220C1642 354 1542 461 1423 461C1300 461 1203 351 1203 220C1203 88 1305 -11 1422 -11C1542 -11 1642 90 1642 220ZM1422 53C1353 53 1281 109 1281 230C1281 351 1357 400 1422 400C1492 400 1564 348 1564 230C1564 112 1496 53 1422 53ZM1777 469L1809 665C1810 667 1810 669 1810 671C1810 689 1788 709 1767 709C1746 709 1724 689 1724 670L1724 665L1757 469ZM1958 469L1990 665C1991 667 1991 669 1991 670C1991 689 1969 709 1948 709C1926 709 1905 690 1905 670L1905 665L1938 469Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.020399999999999998,0,0,-0.020399999999999998,0,0);"></path></g><g style="transform:matrix(1,0,0,1,371.86981201171875,163.70728912353516);"><path d="M51 726L32 700C87 636 187 526 187 266C187 -10 83 -131 32 -194L51 -215C104 -165 273 -23 273 265C273 542 108 675 51 726Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.020399999999999998,0,0,-0.020399999999999998,0,0);"></path></g></g><g><g style="transform:matrix(1,0,0,1,284.9114990234375,187.04063262939454);"><path d="" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.020399999999999998,0,0,-0.020399999999999998,0,0);"></path></g><g><g style="transform:matrix(1,0,0,1,324.52606201171875,187.04063262939454);"><path d="M949 272L743 486L711 452L836 300L65 300L65 241L836 241L711 89L743 55Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.020399999999999998,0,0,-0.020399999999999998,0,0);"></path></g></g><g style="transform:matrix(1,0,0,1,349.2552490234375,187.70728912353516);"><path d="M289 -175C226 -161 206 -117 206 -45L206 128C206 207 197 253 125 272L125 274C194 292 206 335 206 409L206 595C206 667 224 707 289 726C189 726 134 703 134 578L134 392C134 327 120 292 58 273C124 254 134 223 134 151L134 -17C134 -149 176 -175 289 -175Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.020399999999999998,0,0,-0.020399999999999998,0,0);"></path></g><g style="transform:matrix(1,0,0,1,357.0364990234375,187.04063262939454);"><path d="M24 388L31 368L63 389C100 412 103 414 110 414C121 414 128 404 128 389C128 338 87 145 46 2L53 -9C78 -2 101 4 123 8C142 134 163 199 209 268C263 352 338 414 383 414C394 414 400 405 400 390C400 372 397 351 389 319L337 107C328 70 324 47 324 31C324 6 335 -9 354 -9C380 -9 416 12 514 85L504 103L478 86C449 67 427 56 417 56C410 56 404 65 404 76C404 81 405 92 406 96L472 372C479 401 483 429 483 446C483 469 472 482 452 482C410 482 341 444 282 389C244 354 216 320 164 247L202 408C206 426 208 438 208 449C208 470 200 482 185 482C164 482 125 460 52 408Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.020399999999999998,0,0,-0.020399999999999998,0,0);"></path></g><g><g><g><g style="transform:matrix(1,0,0,1,368.96356201171875,191.30603912353516);"><path d="M16 23L16 -3C203 -3 203 0 239 0C275 0 275 -3 468 -3L468 82C353 77 307 81 122 77L304 270C401 373 431 428 431 503C431 618 353 689 226 689C154 689 105 669 56 619L39 483L68 483L81 529C97 587 133 612 200 612C286 612 341 558 341 473C341 398 299 324 186 204Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.01428,0,0,-0.01428,0,0);"></path></g></g></g></g><g style="transform:matrix(1,0,0,1,377.30731201171875,187.04063262939454);"><path d="M204 123C177 114 159 108 106 93C99 17 74 -48 16 -144L30 -155L71 -136C152 -31 190 32 218 109Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.020399999999999998,0,0,-0.020399999999999998,0,0);"></path></g><g style="transform:matrix(1,0,0,1,386.46356201171875,187.04063262939454);"><path d="M273 388L280 368L312 389C349 412 352 414 359 414C370 414 377 404 377 389C377 338 336 145 295 2L302 -9C327 -2 350 4 372 8C391 134 412 199 458 268C512 352 587 414 632 414C643 414 649 405 649 390C649 372 646 351 638 319L586 107C577 70 573 47 573 31C573 6 584 -9 603 -9C629 -9 665 12 763 85L753 103L727 86C698 67 676 56 666 56C659 56 653 65 653 76C653 81 654 92 655 96L721 372C728 401 732 429 732 446C732 469 721 482 701 482C659 482 590 444 531 389C493 354 465 320 413 247L451 408C455 426 457 438 457 449C457 470 449 482 434 482C413 482 374 460 301 408Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.020399999999999998,0,0,-0.020399999999999998,0,0);"></path></g><g><g><g><g style="transform:matrix(1,0,0,1,403.46356201171875,191.30603912353516);"><path d="M462 224C462 345 355 366 308 374C388 436 418 482 418 541C418 630 344 689 233 689C165 689 120 670 72 622L43 498L74 498L92 554C103 588 166 622 218 622C283 622 336 569 336 506C336 431 277 368 206 368C198 368 187 369 174 370L159 371L147 318L154 312C192 329 211 334 238 334C321 334 369 281 369 190C369 88 308 21 215 21C169 21 128 36 98 64C74 86 61 109 42 163L15 153C36 92 44 56 50 6C103 -12 147 -20 184 -20C307 -20 462 87 462 224Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.01428,0,0,-0.01428,0,0);"></path></g></g></g></g><g style="transform:matrix(1,0,0,1,411.80731201171875,187.04063262939454);"><path d="M204 123C177 114 159 108 106 93C99 17 74 -48 16 -144L30 -155L71 -136C152 -31 190 32 218 109Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.020399999999999998,0,0,-0.020399999999999998,0,0);"></path></g><g style="transform:matrix(1,0,0,1,420.96356201171875,187.04063262939454);"><path d="M273 388L280 368L312 389C349 412 352 414 359 414C370 414 377 404 377 389C377 338 336 145 295 2L302 -9C327 -2 350 4 372 8C391 134 412 199 458 268C512 352 587 414 632 414C643 414 649 405 649 390C649 372 646 351 638 319L586 107C577 70 573 47 573 31C573 6 584 -9 603 -9C629 -9 665 12 763 85L753 103L727 86C698 67 676 56 666 56C659 56 653 65 653 76C653 81 654 92 655 96L721 372C728 401 732 429 732 446C732 469 721 482 701 482C659 482 590 444 531 389C493 354 465 320 413 247L451 408C455 426 457 438 457 449C457 470 449 482 434 482C413 482 374 460 301 408Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.020399999999999998,0,0,-0.020399999999999998,0,0);"></path></g><g><g><g><g style="transform:matrix(1,0,0,1,437.96356201171875,191.30603912353516);"><path d="M280 181L280 106C280 46 269 32 220 30L158 27L158 -3C291 0 291 0 315 0C339 0 339 0 472 -3L472 27L424 30C375 33 364 46 364 106L364 181C423 181 444 180 472 177L472 248L364 245L365 697L285 667L2 204L2 181ZM280 245L65 245L280 597Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.01428,0,0,-0.01428,0,0);"></path></g></g></g></g><g style="transform:matrix(1,0,0,1,447.3177490234375,187.70728912353516);"><path d="M275 273C213 292 199 327 199 392L199 578C199 703 144 726 44 726C109 707 127 667 127 595L127 409C127 335 139 292 208 274L208 272C136 253 127 207 127 128L127 -45C127 -117 107 -161 44 -175C157 -175 199 -149 199 -17L199 151C199 223 209 254 275 273Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.020399999999999998,0,0,-0.020399999999999998,0,0);"></path></g></g></g></g></g></g><g><g><g><g><g><g style="transform:matrix(1,0,0,1,245.3802490234375,224.40478912353515);"><path d="M24 388L31 368L63 389C100 412 103 414 110 414C121 414 128 404 128 389C128 338 87 145 46 2L53 -9C78 -2 101 4 123 8C142 134 163 199 209 268C263 352 338 414 383 414C394 414 400 405 400 390C400 372 397 351 389 319L337 107C328 70 324 47 324 31C324 6 335 -9 354 -9C380 -9 416 12 514 85L504 103L478 86C449 67 427 56 417 56C410 56 404 65 404 76C404 81 405 92 406 96L472 372C479 401 483 429 483 446C483 469 472 482 452 482C410 482 341 444 282 389C244 354 216 320 164 247L202 408C206 426 208 438 208 449C208 470 200 482 185 482C164 482 125 460 52 408Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.02941,0,0,-0.02941,0,0);"></path></g><g><g><g><g style="transform:matrix(1,0,0,1,262.578125,230.25888262939452);"><path d="M418 -3L418 27L366 30C311 33 301 44 301 96L301 700L60 598L67 548L217 614L217 96C217 44 206 33 152 30L96 27L96 -3C250 0 250 0 261 0C292 0 402 -3 418 -3Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.020587,0,0,-0.020587,0,0);"></path></g></g></g></g></g></g></g></g></g><g><g><g><g><g><g style="transform:matrix(1,0,0,1,85.38021850585938,284.4047891235352);"><path d="M24 388L31 368L63 389C100 412 103 414 110 414C121 414 128 404 128 389C128 338 87 145 46 2L53 -9C78 -2 101 4 123 8C142 134 163 199 209 268C263 352 338 414 383 414C394 414 400 405 400 390C400 372 397 351 389 319L337 107C328 70 324 47 324 31C324 6 335 -9 354 -9C380 -9 416 12 514 85L504 103L478 86C449 67 427 56 417 56C410 56 404 65 404 76C404 81 405 92 406 96L472 372C479 401 483 429 483 446C483 469 472 482 452 482C410 482 341 444 282 389C244 354 216 320 164 247L202 408C206 426 208 438 208 449C208 470 200 482 185 482C164 482 125 460 52 408Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.02941,0,0,-0.02941,0,0);"></path></g><g><g><g><g style="transform:matrix(1,0,0,1,102.578125,290.2588826293945);"><path d="M168 345C127 326 110 315 88 294C50 256 30 211 30 159C30 56 113 -20 226 -20C356 -20 464 82 464 206C464 286 427 329 313 381C404 443 436 485 436 545C436 631 365 689 259 689C140 689 53 613 53 508C53 440 80 402 168 345ZM284 295C347 267 385 218 385 164C385 80 322 14 241 14C156 14 101 74 101 167C101 240 130 286 204 331ZM223 423C160 454 128 494 128 544C128 610 176 655 247 655C320 655 368 607 368 534C368 477 343 438 278 396Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.020587,0,0,-0.020587,0,0);"></path></g></g></g></g></g></g></g></g></g><g><g><g><g><g><g style="transform:matrix(1,0,0,1,457.4114990234375,316.56666412353513);"><path d="M435 298C435 364 420 455 298 455C236 455 188 424 156 383L156 694L81 694L81 0L159 0L159 245C159 311 184 394 260 394C356 394 357 323 357 291L357 0L435 0ZM914 289C914 391 841 461 749 461C684 461 639 445 592 418L598 352C650 389 700 402 749 402C796 402 836 362 836 288L836 245C686 243 559 201 559 113C559 70 586 -11 673 -11C687 -11 781 -9 839 36L839 0L914 0ZM836 132C836 113 836 88 802 69C773 51 735 50 724 50C676 50 631 73 631 115C631 185 793 192 836 194ZM1354 128C1354 183 1317 217 1315 220C1276 255 1249 261 1199 270C1144 281 1098 291 1098 340C1098 402 1170 402 1183 402C1215 402 1268 398 1325 364L1337 429C1285 453 1244 461 1193 461C1168 461 1027 461 1027 330C1027 281 1056 249 1081 230C1112 208 1134 204 1189 193C1225 186 1283 174 1283 121C1283 52 1204 52 1189 52C1108 52 1052 89 1034 101L1022 33C1054 17 1109 -11 1190 -11C1328 -11 1354 75 1354 128ZM1811 298C1811 364 1796 455 1674 455C1612 455 1564 424 1532 383L1532 694L1457 694L1457 0L1535 0L1535 245C1535 311 1560 394 1636 394C1732 394 1733 323 1733 291L1733 0L1811 0Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.020399999999999998,0,0,-0.020399999999999998,0,0);"></path></g><g style="transform:matrix(1,0,0,1,495.96356201171875,317.2333511352539);"><path d="M146 266C146 526 243 632 301 700L282 726C225 675 60 542 60 266C60 159 85 58 133 -32C168 -99 200 -138 282 -215L301 -194C255 -137 146 -15 146 266Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.020399999999999998,0,0,-0.020399999999999998,0,0);"></path></g><g style="transform:matrix(1,0,0,1,502.734375,316.56666412353513);"><path d="M105 469L137 665C138 667 138 669 138 671C138 689 116 709 95 709C74 709 52 689 52 670L52 665L85 469ZM286 469L318 665C319 667 319 669 319 670C319 689 297 709 276 709C254 709 233 690 233 670L233 665L266 469ZM527 694L452 694L452 0L530 0L530 46C554 24 597 -11 664 -11C764 -11 850 89 850 223C850 347 782 455 688 455C649 455 587 445 527 396ZM530 335C546 359 582 394 637 394C696 394 772 351 772 223C772 93 688 50 627 50C588 50 555 68 530 114ZM1284 289C1284 391 1211 461 1119 461C1054 461 1009 445 962 418L968 352C1020 389 1070 402 1119 402C1166 402 1206 362 1206 288L1206 245C1056 243 929 201 929 113C929 70 956 -11 1043 -11C1057 -11 1151 -9 1209 36L1209 0L1284 0ZM1206 132C1206 113 1206 88 1172 69C1143 51 1105 50 1094 50C1046 50 1001 73 1001 115C1001 185 1163 192 1206 194ZM1521 214C1521 314 1593 386 1691 388L1691 455C1602 454 1547 405 1516 359L1516 450L1446 450L1446 0L1521 0ZM1809 469L1841 665C1842 667 1842 669 1842 671C1842 689 1820 709 1799 709C1778 709 1756 689 1756 670L1756 665L1789 469ZM1990 469L2022 665C2023 667 2023 669 2023 670C2023 689 2001 709 1980 709C1958 709 1937 690 1937 670L1937 665L1970 469Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.020399999999999998,0,0,-0.020399999999999998,0,0);"></path></g><g style="transform:matrix(1,0,0,1,545.015625,317.2333511352539);"><path d="M51 726L32 700C87 636 187 526 187 266C187 -10 83 -131 32 -194L51 -215C104 -165 273 -23 273 265C273 542 108 675 51 726Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.020399999999999998,0,0,-0.020399999999999998,0,0);"></path></g></g></g></g></g><g><g><g><g><g style="transform:matrix(1,0,0,1,457.4114990234375,343.2333511352539);"><path d="" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.020399999999999998,0,0,-0.020399999999999998,0,0);"></path></g><g><g style="transform:matrix(1,0,0,1,497.02606201171875,343.2333511352539);"><path d="M949 272L743 486L711 452L836 300L65 300L65 241L836 241L711 89L743 55Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.020399999999999998,0,0,-0.020399999999999998,0,0);"></path></g></g><g style="transform:matrix(1,0,0,1,521.7552490234375,343.9000076293945);"><path d="M289 -175C226 -161 206 -117 206 -45L206 128C206 207 197 253 125 272L125 274C194 292 206 335 206 409L206 595C206 667 224 707 289 726C189 726 134 703 134 578L134 392C134 327 120 292 58 273C124 254 134 223 134 151L134 -17C134 -149 176 -175 289 -175Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.020399999999999998,0,0,-0.020399999999999998,0,0);"></path></g><g style="transform:matrix(1,0,0,1,529.5364990234375,343.2333511352539);"><path d="M24 388L31 368L63 389C100 412 103 414 110 414C121 414 128 404 128 389C128 338 87 145 46 2L53 -9C78 -2 101 4 123 8C142 134 163 199 209 268C263 352 338 414 383 414C394 414 400 405 400 390C400 372 397 351 389 319L337 107C328 70 324 47 324 31C324 6 335 -9 354 -9C380 -9 416 12 514 85L504 103L478 86C449 67 427 56 417 56C410 56 404 65 404 76C404 81 405 92 406 96L472 372C479 401 483 429 483 446C483 469 472 482 452 482C410 482 341 444 282 389C244 354 216 320 164 247L202 408C206 426 208 438 208 449C208 470 200 482 185 482C164 482 125 460 52 408Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.020399999999999998,0,0,-0.020399999999999998,0,0);"></path></g><g><g><g><g style="transform:matrix(1,0,0,1,541.4635620117188,347.4987576293945);"><path d="M462 224C462 345 355 366 308 374C388 436 418 482 418 541C418 630 344 689 233 689C165 689 120 670 72 622L43 498L74 498L92 554C103 588 166 622 218 622C283 622 336 569 336 506C336 431 277 368 206 368C198 368 187 369 174 370L159 371L147 318L154 312C192 329 211 334 238 334C321 334 369 281 369 190C369 88 308 21 215 21C169 21 128 36 98 64C74 86 61 109 42 163L15 153C36 92 44 56 50 6C103 -12 147 -20 184 -20C307 -20 462 87 462 224Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.01428,0,0,-0.01428,0,0);"></path></g></g></g></g><g style="transform:matrix(1,0,0,1,549.8073120117188,343.2333511352539);"><path d="M204 123C177 114 159 108 106 93C99 17 74 -48 16 -144L30 -155L71 -136C152 -31 190 32 218 109Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.020399999999999998,0,0,-0.020399999999999998,0,0);"></path></g><g style="transform:matrix(1,0,0,1,558.9635620117188,343.2333511352539);"><path d="M273 388L280 368L312 389C349 412 352 414 359 414C370 414 377 404 377 389C377 338 336 145 295 2L302 -9C327 -2 350 4 372 8C391 134 412 199 458 268C512 352 587 414 632 414C643 414 649 405 649 390C649 372 646 351 638 319L586 107C577 70 573 47 573 31C573 6 584 -9 603 -9C629 -9 665 12 763 85L753 103L727 86C698 67 676 56 666 56C659 56 653 65 653 76C653 81 654 92 655 96L721 372C728 401 732 429 732 446C732 469 721 482 701 482C659 482 590 444 531 389C493 354 465 320 413 247L451 408C455 426 457 438 457 449C457 470 449 482 434 482C413 482 374 460 301 408Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.020399999999999998,0,0,-0.020399999999999998,0,0);"></path></g><g><g><g><g style="transform:matrix(1,0,0,1,575.9635620117188,347.4987576293945);"><path d="M280 181L280 106C280 46 269 32 220 30L158 27L158 -3C291 0 291 0 315 0C339 0 339 0 472 -3L472 27L424 30C375 33 364 46 364 106L364 181C423 181 444 180 472 177L472 248L364 245L365 697L285 667L2 204L2 181ZM280 245L65 245L280 597Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.01428,0,0,-0.01428,0,0);"></path></g></g></g></g><g style="transform:matrix(1,0,0,1,584.3073120117188,343.2333511352539);"><path d="M204 123C177 114 159 108 106 93C99 17 74 -48 16 -144L30 -155L71 -136C152 -31 190 32 218 109Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.020399999999999998,0,0,-0.020399999999999998,0,0);"></path></g><g style="transform:matrix(1,0,0,1,593.4635620117188,343.2333511352539);"><path d="M273 388L280 368L312 389C349 412 352 414 359 414C370 414 377 404 377 389C377 338 336 145 295 2L302 -9C327 -2 350 4 372 8C391 134 412 199 458 268C512 352 587 414 632 414C643 414 649 405 649 390C649 372 646 351 638 319L586 107C577 70 573 47 573 31C573 6 584 -9 603 -9C629 -9 665 12 763 85L753 103L727 86C698 67 676 56 666 56C659 56 653 65 653 76C653 81 654 92 655 96L721 372C728 401 732 429 732 446C732 469 721 482 701 482C659 482 590 444 531 389C493 354 465 320 413 247L451 408C455 426 457 438 457 449C457 470 449 482 434 482C413 482 374 460 301 408Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.020399999999999998,0,0,-0.020399999999999998,0,0);"></path></g><g><g><g><g style="transform:matrix(1,0,0,1,610.463623046875,347.4987576293945);"><path d="M459 253C459 366 378 446 264 446C216 446 180 443 127 396L127 605L432 604L432 689L75 690L75 322L95 316C142 363 169 377 218 377C314 377 374 309 374 201C374 90 310 25 201 25C147 25 97 43 83 69L37 151L13 137C36 80 48 48 62 4C90 -11 130 -20 173 -20C301 -20 459 89 459 253Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.01428,0,0,-0.01428,0,0);"></path></g></g></g></g><g style="transform:matrix(1,0,0,1,619.8177490234375,343.9000076293945);"><path d="M275 273C213 292 199 327 199 392L199 578C199 703 144 726 44 726C109 707 127 667 127 595L127 409C127 335 139 292 208 274L208 272C136 253 127 207 127 128L127 -45C127 -117 107 -161 44 -175C157 -175 199 -149 199 -17L199 151C199 223 209 254 275 273Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.020399999999999998,0,0,-0.020399999999999998,0,0);"></path></g></g></g></g></g></g><g><g><g><g><g><g style="transform:matrix(1,0,0,1,254.71356201171875,324.78125762939453);"><path d="M175 386L316 386L316 444L175 444L175 571L106 571L106 444L19 444L19 386L103 386L103 119C103 59 117 -11 186 -11C256 -11 307 14 332 27L316 86C290 65 258 53 226 53C189 53 175 83 175 136Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017,0,0,-0.017,0,0);"></path></g><g><g><g><g style="transform:matrix(1,0,0,1,261.33856201171875,328.4521011352539);"><path d="M299 689L279 689C220 627 137 624 89 622L89 563C122 564 170 566 220 587L220 59L95 59L95 0L424 0L424 59L299 59Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.0119,0,0,-0.0119,0,0);"></path></g></g></g></g></g></g></g></g></g><g><g><g><g><g><g style="transform:matrix(1,0,0,1,331.71356201171875,354.78125762939453);"><path d="M175 386L316 386L316 444L175 444L175 571L106 571L106 444L19 444L19 386L103 386L103 119C103 59 117 -11 186 -11C256 -11 307 14 332 27L316 86C290 65 258 53 226 53C189 53 175 83 175 136Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017,0,0,-0.017,0,0);"></path></g><g><g><g><g style="transform:matrix(1,0,0,1,338.33856201171875,358.4521011352539);"><path d="M83 466C103 545 131 624 222 624C316 624 367 548 367 468C367 382 310 324 251 263L174 191L50 65L50 0L449 0L449 72L267 72C255 72 243 71 231 71L122 71C154 100 230 176 261 205C333 274 449 347 449 471C449 587 368 689 236 689C122 689 66 610 42 522C66 487 59 501 83 466Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.0119,0,0,-0.0119,0,0);"></path></g></g></g></g></g></g></g></g></g><g><g><g><g><g><g style="transform:matrix(1,0,0,1,359.71356201171875,439.78125762939453);"><path d="M175 386L316 386L316 444L175 444L175 571L106 571L106 444L19 444L19 386L103 386L103 119C103 59 117 -11 186 -11C256 -11 307 14 332 27L316 86C290 65 258 53 226 53C189 53 175 83 175 136Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017,0,0,-0.017,0,0);"></path></g><g><g><g><g style="transform:matrix(1,0,0,1,366.33856201171875,443.4521011352539);"><path d="M92 522C121 593 186 630 247 630C299 630 348 600 348 535C348 473 307 413 246 398C240 397 238 397 167 391L167 329L238 329C346 329 368 235 368 184C368 105 322 40 245 40C176 40 97 75 53 144L42 83C115 -12 207 -22 247 -22C369 -22 457 76 457 183C457 275 387 338 319 360C395 401 430 471 430 535C430 622 347 689 248 689C171 689 98 648 56 577Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.0119,0,0,-0.0119,0,0);"></path></g></g></g></g></g></g></g></g></g></svg>
+</svg>
diff --git a/doc/source/architecture/images/vnodes.svg b/doc/source/architecture/images/vnodes.svg
new file mode 100644
index 0000000..71b4fa2
--- /dev/null
+++ b/doc/source/architecture/images/vnodes.svg
@@ -0,0 +1,11 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="651" height="384.66668701171875" style="
+ width:651px;
+ height:384.66668701171875px;
+ background: transparent;
+ fill: none;
+">
+
+
+ <svg xmlns="http://www.w3.org/2000/svg" class="role-diagram-draw-area"><g class="shapes-region" style="stroke: black; fill: none;"><g class="composite-shape"><path class="real" d=" M40.4,190 C40.4,107.38 107.38,40.4 190,40.4 C272.62,40.4 339.6,107.38 339.6,190 C339.6,272.62 272.62,339.6 190,339.6 C107.38,339.6 40.4,272.62 40.4,190 Z" style="stroke-width: 1; stroke: rgba(0, 0, 0, 0.52); fill: none; stroke-dasharray: 1.125, 3.35;"/></g><g class="composite-shape"><path class="real" d=" M160,340 C160,323.43 173.43,310 190,310 C206.57,310 220,323.43 220,340 C220,356.57 206.57,370 190,370 C173.43,370 160,356.57 160,340 Z" style="stroke-width: 1; stroke: rgb(0, 0, 0); fill: rgb(130, 192, 233); stroke-dasharray: 6, 6;"/></g><g class="composite-shape"><path class="real" d=" M160,40 C160,23.43 173.43,10 190,10 C206.57,10 220,23.43 220,40 C220,56.57 206.57,70 190,70 C173.43,70 160,56.57 160,40 Z" style="stroke-width: 1; stroke: rgb(0, 0, 0); fill: rgb(130, 192, 233); stroke-dasharray: 6, 6;"/></g><g class="composite-shape"><path class="real" d=" M310,190 C310,173.43 323.43,160 340,160 C356.57,160 370,173.43 370,190 C370,206.57 356.57,220 340,220 C323.43,220 310,206.57 310,190 Z" style="stroke-width: 1; stroke: rgb(0, 0, 0); fill: rgb(103, 148, 135); stroke-dasharray: 1.125, 3.35;"/></g><g class="composite-shape"><path class="real" d=" M10,190 C10,173.43 23.43,160 40,160 C56.57,160 70,173.43 70,190 C70,206.57 56.57,220 40,220 C23.43,220 10,206.57 10,190 Z" style="stroke-width: 1; stroke: rgb(0, 0, 0); fill: rgb(103, 148, 135); stroke-dasharray: 1.125, 3.35;"/></g><g class="composite-shape"><path class="real" d=" M270,80 C270,63.43 283.43,50 300,50 C316.57,50 330,63.43 330,80 C330,96.57 316.57,110 300,110 C283.43,110 270,96.57 270,80 Z" style="stroke-width: 1; stroke: rgb(0, 0, 0); fill: rgb(202, 194, 126);"/></g><g class="composite-shape"><path class="real" d=" M270,300 C270,283.43 283.43,270 300,270 C316.57,270 330,283.43 330,300 C330,316.57 316.57,330 300,330 C283.43,330 270,316.57 270,300 Z" style="stroke-width: 1; stroke: rgb(0, 0, 0); fill: rgb(187, 187, 187);"/></g><g class="composite-shape"><path class="real" d=" M50,300 C50,283.43 63.43,270 80,270 C96.57,270 110,283.43 110,300 C110,316.57 96.57,330 80,330 C63.43,330 50,316.57 50,300 Z" style="stroke-width: 1; stroke: rgb(0, 0, 0); fill: rgb(202, 194, 126);"/></g><g class="composite-shape"><path class="real" d=" M50,80 C50,63.43 63.43,50 80,50 C96.57,50 110,63.43 110,80 C110,96.57 96.57,110 80,110 C63.43,110 50,96.57 50,80 Z" style="stroke-width: 1; stroke: rgb(0, 0, 0); fill: rgb(187, 187, 187);"/></g><g class="composite-shape"><path class="real" d=" M380,158.4 C380,146.47 389.67,136.8 401.6,136.8 C413.53,136.8 423.2,146.47 423.2,158.4 C423.2,170.33 413.53,180 401.6,180 C389.67,180 380,170.33 380,158.4 Z" style="stroke-width: 1; stroke: rgb(0, 0, 0); fill: rgb(103, 148, 135); stroke-dasharray: 1.125, 3.35;"/></g><g class="composite-shape"><path class="real" d=" M380,101.6 C380,89.67 389.67,80 401.6,80 C413.53,80 423.2,89.67 423.2,101.6 C423.2,113.53 413.53,123.2 401.6,123.2 C389.67,123.2 380,113.53 380,101.6 Z" style="stroke-width: 1; stroke: rgb(0, 0, 0); fill: rgb(130, 192, 233); stroke-dasharray: 6, 6;"/></g><g class="composite-shape"><path class="real" d=" M380,218.4 C380,206.47 389.67,196.8 401.6,196.8 C413.53,196.8 423.2,206.47 423.2,218.4 C423.2,230.33 413.53,240 401.6,240 C389.67,240 380,230.33 380,218.4 Z" style="stroke-width: 1; stroke: rgb(0, 0, 0); fill: rgb(202, 194, 126);"/></g><g class="composite-shape"><path class="real" d=" M380,278.4 C380,266.47 389.67,256.8 401.6,256.8 C413.53,256.8 423.2,266.47 423.2,278.4 C423.2,290.33 413.53,300 401.6,300 C389.67,300 380,290.33 380,278.4 Z" style="stroke-width: 1; stroke: rgb(0, 0, 0); fill: rgb(187, 187, 187);"/></g><g class="composite-shape"><path class="real" d=" M430,80 L640,80 L640,300 L430,300 Z" style="stroke-width: 1; stroke: rgb(0, 0, 0); fill: none;"/></g><g/></g><g/><g/><g/></svg>
+ <svg xmlns="http://www.w3.org/2000/svg" width="649" height="382.66668701171875" style="width:649px;height:382.66668701171875px;font-family:Asana-Math, Asana;background:transparent;"><g><g><g><g><g><g style="transform:matrix(1,0,0,1,178.65625,348.9985620117188);"><path d="M125 390L69 107C68 99 56 61 56 31C56 6 67 -9 86 -9C121 -9 156 11 234 74L265 99L255 117L210 86C181 66 161 56 150 56C141 56 136 64 136 76C136 102 150 183 179 328L192 390L299 390L310 440C272 436 238 434 200 434C216 528 227 577 245 631L234 646C214 634 187 622 156 610L131 440C87 419 61 408 43 403L41 390Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.02941,0,0,-0.02941,0,0);"></path></g><g><g><g><g style="transform:matrix(1,0,0,1,189.3021240234375,354.852625);"><path d="M459 253C459 366 378 446 264 446C216 446 180 443 127 396L127 605L432 604L432 689L75 690L75 322L95 316C142 363 169 377 218 377C314 377 374 309 374 201C374 90 310 25 201 25C147 25 97 43 83 69L37 151L13 137C36 80 48 48 62 4C90 -11 130 -20 173 -20C301 -20 459 89 459 253Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.020587,0,0,-0.020587,0,0);"></path></g></g></g></g></g></g></g></g></g><g><g><g><g><g><g style="transform:matrix(1,0,0,1,178.65625,49.800625);"><path d="M125 390L69 107C68 99 56 61 56 31C56 6 67 -9 86 -9C121 -9 156 11 234 74L265 99L255 117L210 86C181 66 161 56 150 56C141 56 136 64 136 76C136 102 150 183 179 328L192 390L299 390L310 440C272 436 238 434 200 434C216 528 227 577 245 631L234 646C214 634 187 622 156 610L131 440C87 419 61 408 43 403L41 390Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.02941,0,0,-0.02941,0,0);"></path></g><g><g><g><g style="transform:matrix(1,0,0,1,189.3021240234375,55.65471850585938);"><path d="M418 -3L418 27L366 30C311 33 301 44 301 96L301 700L60 598L67 548L217 614L217 96C217 44 206 33 152 30L96 27L96 -3C250 0 250 0 261 0C292 0 402 -3 418 -3Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.020587,0,0,-0.020587,0,0);"></path></g></g></g></g></g></g></g></g></g><g><g><g><g><g><g style="transform:matrix(1,0,0,1,328.25,199.40481201171875);"><path d="M125 390L69 107C68 99 56 61 56 31C56 6 67 -9 86 -9C121 -9 156 11 234 74L265 99L255 117L210 86C181 66 161 56 150 56C141 56 136 64 136 76C136 102 150 183 179 328L192 390L299 390L310 440C272 436 238 434 200 434C216 528 227 577 245 631L234 646C214 634 187 622 156 610L131 440C87 419 61 408 43 403L41 390Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.02941,0,0,-0.02941,0,0);"></path></g><g><g><g><g style="transform:matrix(1,0,0,1,338.8958740234375,205.258875);"><path d="M462 224C462 345 355 366 308 374C388 436 418 482 418 541C418 630 344 689 233 689C165 689 120 670 72 622L43 498L74 498L92 554C103 588 166 622 218 622C283 622 336 569 336 506C336 431 277 368 206 368C198 368 187 369 174 370L159 371L147 318L154 312C192 329 211 334 238 334C321 334 369 281 369 190C369 88 308 21 215 21C169 21 128 36 98 64C74 86 61 109 42 163L15 153C36 92 44 56 50 6C103 -12 147 -20 184 -20C307 -20 462 87 462 224Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.020587,0,0,-0.020587,0,0);"></path></g></g></g></g></g></g></g></g></g><g><g><g><g><g><g style="transform:matrix(1,0,0,1,29.052093505859375,199.40481201171875);"><path d="M125 390L69 107C68 99 56 61 56 31C56 6 67 -9 86 -9C121 -9 156 11 234 74L265 99L255 117L210 86C181 66 161 56 150 56C141 56 136 64 136 76C136 102 150 183 179 328L192 390L299 390L310 440C272 436 238 434 200 434C216 528 227 577 245 631L234 646C214 634 187 622 156 610L131 440C87 419 61 408 43 403L41 390Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.02941,0,0,-0.02941,0,0);"></path></g><g><g><g><g style="transform:matrix(1,0,0,1,39.69793701171875,205.258875);"><path d="M409 603L47 -1L157 -1L497 659L497 689L44 689L44 477L74 477L81 533C89 595 96 603 142 603Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.020587,0,0,-0.020587,0,0);"></path></g></g></g></g></g></g></g></g></g><g><g><g><g><g><g style="transform:matrix(1,0,0,1,287.44793701171875,90.60271850585937);"><path d="M125 390L69 107C68 99 56 61 56 31C56 6 67 -9 86 -9C121 -9 156 11 234 74L265 99L255 117L210 86C181 66 161 56 150 56C141 56 136 64 136 76C136 102 150 183 179 328L192 390L299 390L310 440C272 436 238 434 200 434C216 528 227 577 245 631L234 646C214 634 187 622 156 610L131 440C87 419 61 408 43 403L41 390Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.02941,0,0,-0.02941,0,0);"></path></g><g><g><g><g style="transform:matrix(1,0,0,1,298.09375,96.45679675292969);"><path d="M16 23L16 -3C203 -3 203 0 239 0C275 0 275 -3 468 -3L468 82C353 77 307 81 122 77L304 270C401 373 431 428 431 503C431 618 353 689 226 689C154 689 105 669 56 619L39 483L68 483L81 529C97 587 133 612 200 612C286 612 341 558 341 473C341 398 299 324 186 204Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.020587,0,0,-0.020587,0,0);"></path></g></g></g></g></g></g></g></g></g><g><g><g><g><g><g style="transform:matrix(1,0,0,1,287.44793701171875,308.1964685058594);"><path d="M125 390L69 107C68 99 56 61 56 31C56 6 67 -9 86 -9C121 -9 156 11 234 74L265 99L255 117L210 86C181 66 161 56 150 56C141 56 136 64 136 76C136 102 150 183 179 328L192 390L299 390L310 440C272 436 238 434 200 434C216 528 227 577 245 631L234 646C214 634 187 622 156 610L131 440C87 419 61 408 43 403L41 390Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.02941,0,0,-0.02941,0,0);"></path></g><g><g><g><g style="transform:matrix(1,0,0,1,298.09375,314.05056201171874);"><path d="M280 181L280 106C280 46 269 32 220 30L158 27L158 -3C291 0 291 0 315 0C339 0 339 0 472 -3L472 27L424 30C375 33 364 46 364 106L364 181C423 181 444 180 472 177L472 248L364 245L365 697L285 667L2 204L2 181ZM280 245L65 245L280 597Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.020587,0,0,-0.020587,0,0);"></path></g></g></g></g></g></g></g></g></g><g><g><g><g><g><g style="transform:matrix(1,0,0,1,69.85418701171875,308.1964685058594);"><path d="M125 390L69 107C68 99 56 61 56 31C56 6 67 -9 86 -9C121 -9 156 11 234 74L265 99L255 117L210 86C181 66 161 56 150 56C141 56 136 64 136 76C136 102 150 183 179 328L192 390L299 390L310 440C272 436 238 434 200 434C216 528 227 577 245 631L234 646C214 634 187 622 156 610L131 440C87 419 61 408 43 403L41 390Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.02941,0,0,-0.02941,0,0);"></path></g><g><g><g><g style="transform:matrix(1,0,0,1,80.5,314.05056201171874);"><path d="M131 331C152 512 241 611 421 665L379 689C283 657 242 637 191 593C88 506 32 384 32 247C32 82 112 -20 241 -20C371 -20 468 83 468 219C468 334 399 409 293 409C216 409 184 370 131 331ZM255 349C331 349 382 283 382 184C382 80 331 13 254 13C169 13 123 86 123 220C123 255 127 274 138 291C160 325 207 349 255 349Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.020587,0,0,-0.020587,0,0);"></path></g></g></g></g></g></g></g></g></g><g><g><g><g><g><g style="transform:matrix(1,0,0,1,69.85418701171875,90.60271850585937);"><path d="M125 390L69 107C68 99 56 61 56 31C56 6 67 -9 86 -9C121 -9 156 11 234 74L265 99L255 117L210 86C181 66 161 56 150 56C141 56 136 64 136 76C136 102 150 183 179 328L192 390L299 390L310 440C272 436 238 434 200 434C216 528 227 577 245 631L234 646C214 634 187 622 156 610L131 440C87 419 61 408 43 403L41 390Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.02941,0,0,-0.02941,0,0);"></path></g><g><g><g><g style="transform:matrix(1,0,0,1,80.5,96.45679675292969);"><path d="M168 345C127 326 110 315 88 294C50 256 30 211 30 159C30 56 113 -20 226 -20C356 -20 464 82 464 206C464 286 427 329 313 381C404 443 436 485 436 545C436 631 365 689 259 689C140 689 53 613 53 508C53 440 80 402 168 345ZM284 295C347 267 385 218 385 164C385 80 322 14 241 14C156 14 101 74 101 167C101 240 130 286 204 331ZM223 423C160 454 128 494 128 544C128 610 176 655 247 655C320 655 368 607 368 534C368 477 343 438 278 396Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.020587,0,0,-0.020587,0,0);"></path></g></g></g></g></g></g></g></g></g><g><g><g><g><g><g style="transform:matrix(1,0,0,1,386.9739990234375,167.800625);"><path d="M24 388L31 368L63 389C100 412 103 414 110 414C121 414 128 404 128 389C128 338 87 145 46 2L53 -9C78 -2 101 4 123 8C142 134 163 199 209 268C263 352 338 414 383 414C394 414 400 405 400 390C400 372 397 351 389 319L337 107C328 70 324 47 324 31C324 6 335 -9 354 -9C380 -9 416 12 514 85L504 103L478 86C449 67 427 56 417 56C410 56 404 65 404 76C404 81 405 92 406 96L472 372C479 401 483 429 483 446C483 469 472 482 452 482C410 482 341 444 282 389C244 354 216 320 164 247L202 408C206 426 208 438 208 449C208 470 200 482 185 482C164 482 125 460 52 408Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.02941,0,0,-0.02941,0,0);"></path></g><g><g><g><g style="transform:matrix(1,0,0,1,404.171875,173.65471850585936);"><path d="M16 23L16 -3C203 -3 203 0 239 0C275 0 275 -3 468 -3L468 82C353 77 307 81 122 77L304 270C401 373 431 428 431 503C431 618 353 689 226 689C154 689 105 669 56 619L39 483L68 483L81 529C97 587 133 612 200 612C286 612 341 558 341 473C341 398 299 324 186 204Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.020587,0,0,-0.020587,0,0);"></path></g></g></g></g></g></g></g></g></g><g><g><g><g><g><g style="transform:matrix(1,0,0,1,386.9739990234375,110.99856201171875);"><path d="M24 388L31 368L63 389C100 412 103 414 110 414C121 414 128 404 128 389C128 338 87 145 46 2L53 -9C78 -2 101 4 123 8C142 134 163 199 209 268C263 352 338 414 383 414C394 414 400 405 400 390C400 372 397 351 389 319L337 107C328 70 324 47 324 31C324 6 335 -9 354 -9C380 -9 416 12 514 85L504 103L478 86C449 67 427 56 417 56C410 56 404 65 404 76C404 81 405 92 406 96L472 372C479 401 483 429 483 446C483 469 472 482 452 482C410 482 341 444 282 389C244 354 216 320 164 247L202 408C206 426 208 438 208 449C208 470 200 482 185 482C164 482 125 460 52 408Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.02941,0,0,-0.02941,0,0);"></path></g><g><g><g><g style="transform:matrix(1,0,0,1,404.171875,116.852625);"><path d="M418 -3L418 27L366 30C311 33 301 44 301 96L301 700L60 598L67 548L217 614L217 96C217 44 206 33 152 30L96 27L96 -3C250 0 250 0 261 0C292 0 402 -3 418 -3Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.020587,0,0,-0.020587,0,0);"></path></g></g></g></g></g></g></g></g></g><g><g><g><g><g><g style="transform:matrix(1,0,0,1,386.9739990234375,227.800625);"><path d="M24 388L31 368L63 389C100 412 103 414 110 414C121 414 128 404 128 389C128 338 87 145 46 2L53 -9C78 -2 101 4 123 8C142 134 163 199 209 268C263 352 338 414 383 414C394 414 400 405 400 390C400 372 397 351 389 319L337 107C328 70 324 47 324 31C324 6 335 -9 354 -9C380 -9 416 12 514 85L504 103L478 86C449 67 427 56 417 56C410 56 404 65 404 76C404 81 405 92 406 96L472 372C479 401 483 429 483 446C483 469 472 482 452 482C410 482 341 444 282 389C244 354 216 320 164 247L202 408C206 426 208 438 208 449C208 470 200 482 185 482C164 482 125 460 52 408Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.02941,0,0,-0.02941,0,0);"></path></g><g><g><g><g style="transform:matrix(1,0,0,1,404.171875,233.65471850585936);"><path d="M462 224C462 345 355 366 308 374C388 436 418 482 418 541C418 630 344 689 233 689C165 689 120 670 72 622L43 498L74 498L92 554C103 588 166 622 218 622C283 622 336 569 336 506C336 431 277 368 206 368C198 368 187 369 174 370L159 371L147 318L154 312C192 329 211 334 238 334C321 334 369 281 369 190C369 88 308 21 215 21C169 21 128 36 98 64C74 86 61 109 42 163L15 153C36 92 44 56 50 6C103 -12 147 -20 184 -20C307 -20 462 87 462 224Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.020587,0,0,-0.020587,0,0);"></path></g></g></g></g></g></g></g></g></g><g><g><g><g><g><g style="transform:matrix(1,0,0,1,386.9739990234375,287.800625);"><path d="M24 388L31 368L63 389C100 412 103 414 110 414C121 414 128 404 128 389C128 338 87 145 46 2L53 -9C78 -2 101 4 123 8C142 134 163 199 209 268C263 352 338 414 383 414C394 414 400 405 400 390C400 372 397 351 389 319L337 107C328 70 324 47 324 31C324 6 335 -9 354 -9C380 -9 416 12 514 85L504 103L478 86C449 67 427 56 417 56C410 56 404 65 404 76C404 81 405 92 406 96L472 372C479 401 483 429 483 446C483 469 472 482 452 482C410 482 341 444 282 389C244 354 216 320 164 247L202 408C206 426 208 438 208 449C208 470 200 482 185 482C164 482 125 460 52 408Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.02941,0,0,-0.02941,0,0);"></path></g><g><g><g><g style="transform:matrix(1,0,0,1,404.171875,293.65471850585936);"><path d="M280 181L280 106C280 46 269 32 220 30L158 27L158 -3C291 0 291 0 315 0C339 0 339 0 472 -3L472 27L424 30C375 33 364 46 364 106L364 181C423 181 444 180 472 177L472 248L364 245L365 697L285 667L2 204L2 181ZM280 245L65 245L280 597Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.020587,0,0,-0.020587,0,0);"></path></g></g></g></g></g></g></g></g></g><g><g><g><g><g><g style="transform:matrix(1,0,0,1,438.125,111.64668701171875);"><path d="M125 390L69 107C68 99 56 61 56 31C56 6 67 -9 86 -9C121 -9 156 11 234 74L265 99L255 117L210 86C181 66 161 56 150 56C141 56 136 64 136 76C136 102 150 183 179 328L192 390L299 390L310 440C272 436 238 434 200 434C216 528 227 577 245 631L234 646C214 634 187 622 156 610L131 440C87 419 61 408 43 403L41 390ZM349 152C349 46 394 -11 477 -11C532 -11 592 15 637 57C699 116 743 230 743 331C743 425 693 482 610 482C506 482 349 382 349 152ZM573 444C634 444 667 399 667 315C667 219 636 113 592 60C574 39 548 27 517 27C459 27 425 72 425 151C425 264 464 387 513 427C526 438 549 444 573 444ZM1015 722L1003 733C951 707 915 698 843 691L839 670L887 670C911 670 921 663 921 646C921 638 920 629 919 622L871 354C857 279 841 210 789 1L798 -9L865 8L888 132C897 180 921 225 955 259C1024 59 1056 -9 1081 -9C1094 -9 1118 4 1162 35L1208 67L1200 86L1157 62C1143 54 1136 52 1128 52C1118 52 1111 58 1102 75C1067 139 1045 193 1006 316L1020 330C1081 391 1127 416 1177 416C1185 416 1196 414 1213 410L1230 473C1212 479 1194 482 1182 482C1116 482 1033 405 908 230ZM1571 111L1547 94C1494 56 1446 36 1410 36C1363 36 1334 73 1334 133C1334 158 1337 185 1342 214C1359 218 1468 248 1493 259C1578 296 1617 342 1617 404C1617 451 1583 482 1533 482C1465 496 1355 423 1318 349C1288 299 1258 180 1258 113C1258 35 1302 -11 1374 -11C1431 -11 1487 17 1579 92ZM1356 274C1373 343 1393 386 1422 412C1440 428 1471 440 1495 440C1524 440 1543 420 1543 388C1543 344 1508 297 1456 272C1428 258 1392 247 1347 237ZM1655 388L1662 368L1694 389C1731 412 1734 414 1741 414C1752 414 1759 404 1759 389C1759 338 1718 145 1677 2L1684 -9C1709 -2 1732 4 1754 8C1773 134 1794 199 1840 268C1894 352 1969 414 2014 414C2025 414 2031 405 2031 390C2031 372 2028 351 2020 319L1968 107C1959 70 1955 47 1955 31C1955 6 1966 -9 1985 -9C2011 -9 2047 12 2145 85L2135 103L2109 86C2080 67 2058 56 2048 56C2041 56 2035 65 2035 76C2035 81 2036 92 2037 96L2103 372C2110 401 2114 429 2114 446C2114 469 2103 482 2083 482C2041 482 1972 444 1913 389C1875 354 1847 320 1795 247L1833 408C1837 426 1839 438 1839 449C1839 470 1831 482 1816 482C1795 482 1756 460 1683 408Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g style="transform:matrix(1,0,0,1,491.6458740234375,111.64668701171875);"><path d="M146 266C146 526 243 632 301 700L282 726C225 675 60 542 60 266C60 159 85 58 133 -32C168 -99 200 -138 282 -215L301 -194C255 -137 146 -15 146 266Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g style="transform:matrix(1,0,0,1,499.78125,111.64668701171875);"><path d="M24 388L31 368L63 389C100 412 103 414 110 414C121 414 128 404 128 389C128 338 87 145 46 2L53 -9C78 -2 101 4 123 8C142 134 163 199 209 268C263 352 338 414 383 414C394 414 400 405 400 390C400 372 397 351 389 319L337 107C328 70 324 47 324 31C324 6 335 -9 354 -9C380 -9 416 12 514 85L504 103L478 86C449 67 427 56 417 56C410 56 404 65 404 76C404 81 405 92 406 96L472 372C479 401 483 429 483 446C483 469 472 482 452 482C410 482 341 444 282 389C244 354 216 320 164 247L202 408C206 426 208 438 208 449C208 470 200 482 185 482C164 482 125 460 52 408Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g><g><g><g style="transform:matrix(1,0,0,1,514.1041870117188,115.96934350585937);"><path d="M418 -3L418 27L366 30C311 33 301 44 301 96L301 700L60 598L67 548L217 614L217 96C217 44 206 33 152 30L96 27L96 -3C250 0 250 0 261 0C292 0 402 -3 418 -3Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017136,0,0,-0.017136,0,0);"></path></g></g></g></g><g style="transform:matrix(1,0,0,1,524.125,111.64668701171875);"><path d="M51 726L32 700C87 636 187 526 187 266C187 -10 83 -131 32 -194L51 -215C104 -165 273 -23 273 265C273 542 108 675 51 726Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g style="transform:matrix(1,0,0,1,539.6041870117188,111.64668701171875);"><path d="M604 347L604 406L65 406L65 347ZM604 134L604 193L65 193L65 134Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g style="transform:matrix(1,0,0,1,563.3021240234375,111.64668701171875);"><path d="M289 -175C226 -161 206 -117 206 -45L206 128C206 207 197 253 125 272L125 274C194 292 206 335 206 409L206 595C206 667 224 707 289 726C189 726 134 703 134 578L134 392C134 327 120 292 58 273C124 254 134 223 134 151L134 -17C134 -149 176 -175 289 -175Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g style="transform:matrix(1,0,0,1,572.65625,111.64668701171875);"><path d="M125 390L69 107C68 99 56 61 56 31C56 6 67 -9 86 -9C121 -9 156 11 234 74L265 99L255 117L210 86C181 66 161 56 150 56C141 56 136 64 136 76C136 102 150 183 179 328L192 390L299 390L310 440C272 436 238 434 200 434C216 528 227 577 245 631L234 646C214 634 187 622 156 610L131 440C87 419 61 408 43 403L41 390Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g><g><g><g style="transform:matrix(1,0,0,1,581.5208740234375,115.96934350585937);"><path d="M418 -3L418 27L366 30C311 33 301 44 301 96L301 700L60 598L67 548L217 614L217 96C217 44 206 33 152 30L96 27L96 -3C250 0 250 0 261 0C292 0 402 -3 418 -3Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017136,0,0,-0.017136,0,0);"></path></g></g></g></g><g style="transform:matrix(1,0,0,1,591.5416870117188,111.64668701171875);"><path d="M204 123C177 114 159 108 106 93C99 17 74 -48 16 -144L30 -155L71 -136C152 -31 190 32 218 109Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g style="transform:matrix(1,0,0,1,602.541748046875,111.64668701171875);"><path d="M374 390L318 107C317 99 305 61 305 31C305 6 316 -9 335 -9C370 -9 405 11 483 74L514 99L504 117L459 86C430 66 410 56 399 56C390 56 385 64 385 76C385 102 399 183 428 328L441 390L548 390L559 440C521 436 487 434 449 434C465 528 476 577 494 631L483 646C463 634 436 622 405 610L380 440C336 419 310 408 292 403L290 390Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g><g><g><g style="transform:matrix(1,0,0,1,617.5,115.96934350585937);"><path d="M459 253C459 366 378 446 264 446C216 446 180 443 127 396L127 605L432 604L432 689L75 690L75 322L95 316C142 363 169 377 218 377C314 377 374 309 374 201C374 90 310 25 201 25C147 25 97 43 83 69L37 151L13 137C36 80 48 48 62 4C90 -11 130 -20 173 -20C301 -20 459 89 459 253Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017136,0,0,-0.017136,0,0);"></path></g></g></g></g><g style="transform:matrix(1,0,0,1,628.7396240234375,111.64668701171875);"><path d="M275 273C213 292 199 327 199 392L199 578C199 703 144 726 44 726C109 707 127 667 127 595L127 409C127 335 139 292 208 274L208 272C136 253 127 207 127 128L127 -45C127 -117 107 -161 44 -175C157 -175 199 -149 199 -17L199 151C199 223 209 254 275 273Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g></g></g></g></g></g><g><g><g><g><g><g style="transform:matrix(1,0,0,1,439.125,164.64668701171874);"><path d="M125 390L69 107C68 99 56 61 56 31C56 6 67 -9 86 -9C121 -9 156 11 234 74L265 99L255 117L210 86C181 66 161 56 150 56C141 56 136 64 136 76C136 102 150 183 179 328L192 390L299 390L310 440C272 436 238 434 200 434C216 528 227 577 245 631L234 646C214 634 187 622 156 610L131 440C87 419 61 408 43 403L41 390ZM349 152C349 46 394 -11 477 -11C532 -11 592 15 637 57C699 116 743 230 743 331C743 425 693 482 610 482C506 482 349 382 349 152ZM573 444C634 444 667 399 667 315C667 219 636 113 592 60C574 39 548 27 517 27C459 27 425 72 425 151C425 264 464 387 513 427C526 438 549 444 573 444ZM1015 722L1003 733C951 707 915 698 843 691L839 670L887 670C911 670 921 663 921 646C921 638 920 629 919 622L871 354C857 279 841 210 789 1L798 -9L865 8L888 132C897 180 921 225 955 259C1024 59 1056 -9 1081 -9C1094 -9 1118 4 1162 35L1208 67L1200 86L1157 62C1143 54 1136 52 1128 52C1118 52 1111 58 1102 75C1067 139 1045 193 1006 316L1020 330C1081 391 1127 416 1177 416C1185 416 1196 414 1213 410L1230 473C1212 479 1194 482 1182 482C1116 482 1033 405 908 230ZM1571 111L1547 94C1494 56 1446 36 1410 36C1363 36 1334 73 1334 133C1334 158 1337 185 1342 214C1359 218 1468 248 1493 259C1578 296 1617 342 1617 404C1617 451 1583 482 1533 482C1465 496 1355 423 1318 349C1288 299 1258 180 1258 113C1258 35 1302 -11 1374 -11C1431 -11 1487 17 1579 92ZM1356 274C1373 343 1393 386 1422 412C1440 428 1471 440 1495 440C1524 440 1543 420 1543 388C1543 344 1508 297 1456 272C1428 258 1392 247 1347 237ZM1655 388L1662 368L1694 389C1731 412 1734 414 1741 414C1752 414 1759 404 1759 389C1759 338 1718 145 1677 2L1684 -9C1709 -2 1732 4 1754 8C1773 134 1794 199 1840 268C1894 352 1969 414 2014 414C2025 414 2031 405 2031 390C2031 372 2028 351 2020 319L1968 107C1959 70 1955 47 1955 31C1955 6 1966 -9 1985 -9C2011 -9 2047 12 2145 85L2135 103L2109 86C2080 67 2058 56 2048 56C2041 56 2035 65 2035 76C2035 81 2036 92 2037 96L2103 372C2110 401 2114 429 2114 446C2114 469 2103 482 2083 482C2041 482 1972 444 1913 389C1875 354 1847 320 1795 247L1833 408C1837 426 1839 438 1839 449C1839 470 1831 482 1816 482C1795 482 1756 460 1683 408Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g style="transform:matrix(1,0,0,1,492.6458740234375,164.64668701171874);"><path d="M146 266C146 526 243 632 301 700L282 726C225 675 60 542 60 266C60 159 85 58 133 -32C168 -99 200 -138 282 -215L301 -194C255 -137 146 -15 146 266Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g style="transform:matrix(1,0,0,1,500.78125,164.64668701171874);"><path d="M24 388L31 368L63 389C100 412 103 414 110 414C121 414 128 404 128 389C128 338 87 145 46 2L53 -9C78 -2 101 4 123 8C142 134 163 199 209 268C263 352 338 414 383 414C394 414 400 405 400 390C400 372 397 351 389 319L337 107C328 70 324 47 324 31C324 6 335 -9 354 -9C380 -9 416 12 514 85L504 103L478 86C449 67 427 56 417 56C410 56 404 65 404 76C404 81 405 92 406 96L472 372C479 401 483 429 483 446C483 469 472 482 452 482C410 482 341 444 282 389C244 354 216 320 164 247L202 408C206 426 208 438 208 449C208 470 200 482 185 482C164 482 125 460 52 408Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g><g><g><g style="transform:matrix(1,0,0,1,515.1041870117188,168.96934350585937);"><path d="M16 23L16 -3C203 -3 203 0 239 0C275 0 275 -3 468 -3L468 82C353 77 307 81 122 77L304 270C401 373 431 428 431 503C431 618 353 689 226 689C154 689 105 669 56 619L39 483L68 483L81 529C97 587 133 612 200 612C286 612 341 558 341 473C341 398 299 324 186 204Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017136,0,0,-0.017136,0,0);"></path></g></g></g></g><g style="transform:matrix(1,0,0,1,525.125,164.64668701171874);"><path d="M51 726L32 700C87 636 187 526 187 266C187 -10 83 -131 32 -194L51 -215C104 -165 273 -23 273 265C273 542 108 675 51 726Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g style="transform:matrix(1,0,0,1,540.6041870117188,164.64668701171874);"><path d="M604 347L604 406L65 406L65 347ZM604 134L604 193L65 193L65 134Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g style="transform:matrix(1,0,0,1,564.3021240234375,164.64668701171874);"><path d="M289 -175C226 -161 206 -117 206 -45L206 128C206 207 197 253 125 272L125 274C194 292 206 335 206 409L206 595C206 667 224 707 289 726C189 726 134 703 134 578L134 392C134 327 120 292 58 273C124 254 134 223 134 151L134 -17C134 -149 176 -175 289 -175Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g style="transform:matrix(1,0,0,1,573.65625,164.64668701171874);"><path d="M125 390L69 107C68 99 56 61 56 31C56 6 67 -9 86 -9C121 -9 156 11 234 74L265 99L255 117L210 86C181 66 161 56 150 56C141 56 136 64 136 76C136 102 150 183 179 328L192 390L299 390L310 440C272 436 238 434 200 434C216 528 227 577 245 631L234 646C214 634 187 622 156 610L131 440C87 419 61 408 43 403L41 390Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g><g><g><g style="transform:matrix(1,0,0,1,582.5208740234375,168.96934350585937);"><path d="M462 224C462 345 355 366 308 374C388 436 418 482 418 541C418 630 344 689 233 689C165 689 120 670 72 622L43 498L74 498L92 554C103 588 166 622 218 622C283 622 336 569 336 506C336 431 277 368 206 368C198 368 187 369 174 370L159 371L147 318L154 312C192 329 211 334 238 334C321 334 369 281 369 190C369 88 308 21 215 21C169 21 128 36 98 64C74 86 61 109 42 163L15 153C36 92 44 56 50 6C103 -12 147 -20 184 -20C307 -20 462 87 462 224Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017136,0,0,-0.017136,0,0);"></path></g></g></g></g><g style="transform:matrix(1,0,0,1,592.5416870117188,164.64668701171874);"><path d="M204 123C177 114 159 108 106 93C99 17 74 -48 16 -144L30 -155L71 -136C152 -31 190 32 218 109Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g style="transform:matrix(1,0,0,1,603.541748046875,164.64668701171874);"><path d="M374 390L318 107C317 99 305 61 305 31C305 6 316 -9 335 -9C370 -9 405 11 483 74L514 99L504 117L459 86C430 66 410 56 399 56C390 56 385 64 385 76C385 102 399 183 428 328L441 390L548 390L559 440C521 436 487 434 449 434C465 528 476 577 494 631L483 646C463 634 436 622 405 610L380 440C336 419 310 408 292 403L290 390Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g><g><g><g style="transform:matrix(1,0,0,1,618.5,168.96934350585937);"><path d="M409 603L47 -1L157 -1L497 659L497 689L44 689L44 477L74 477L81 533C89 595 96 603 142 603Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017136,0,0,-0.017136,0,0);"></path></g></g></g></g><g style="transform:matrix(1,0,0,1,629.7396240234375,164.64668701171874);"><path d="M275 273C213 292 199 327 199 392L199 578C199 703 144 726 44 726C109 707 127 667 127 595L127 409C127 335 139 292 208 274L208 272C136 253 127 207 127 128L127 -45C127 -117 107 -161 44 -175C157 -175 199 -149 199 -17L199 151C199 223 209 254 275 273Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g></g></g></g></g></g><g><g><g><g><g><g style="transform:matrix(1,0,0,1,439.125,221.64668701171874);"><path d="M125 390L69 107C68 99 56 61 56 31C56 6 67 -9 86 -9C121 -9 156 11 234 74L265 99L255 117L210 86C181 66 161 56 150 56C141 56 136 64 136 76C136 102 150 183 179 328L192 390L299 390L310 440C272 436 238 434 200 434C216 528 227 577 245 631L234 646C214 634 187 622 156 610L131 440C87 419 61 408 43 403L41 390ZM349 152C349 46 394 -11 477 -11C532 -11 592 15 637 57C699 116 743 230 743 331C743 425 693 482 610 482C506 482 349 382 349 152ZM573 444C634 444 667 399 667 315C667 219 636 113 592 60C574 39 548 27 517 27C459 27 425 72 425 151C425 264 464 387 513 427C526 438 549 444 573 444ZM1015 722L1003 733C951 707 915 698 843 691L839 670L887 670C911 670 921 663 921 646C921 638 920 629 919 622L871 354C857 279 841 210 789 1L798 -9L865 8L888 132C897 180 921 225 955 259C1024 59 1056 -9 1081 -9C1094 -9 1118 4 1162 35L1208 67L1200 86L1157 62C1143 54 1136 52 1128 52C1118 52 1111 58 1102 75C1067 139 1045 193 1006 316L1020 330C1081 391 1127 416 1177 416C1185 416 1196 414 1213 410L1230 473C1212 479 1194 482 1182 482C1116 482 1033 405 908 230ZM1571 111L1547 94C1494 56 1446 36 1410 36C1363 36 1334 73 1334 133C1334 158 1337 185 1342 214C1359 218 1468 248 1493 259C1578 296 1617 342 1617 404C1617 451 1583 482 1533 482C1465 496 1355 423 1318 349C1288 299 1258 180 1258 113C1258 35 1302 -11 1374 -11C1431 -11 1487 17 1579 92ZM1356 274C1373 343 1393 386 1422 412C1440 428 1471 440 1495 440C1524 440 1543 420 1543 388C1543 344 1508 297 1456 272C1428 258 1392 247 1347 237ZM1655 388L1662 368L1694 389C1731 412 1734 414 1741 414C1752 414 1759 404 1759 389C1759 338 1718 145 1677 2L1684 -9C1709 -2 1732 4 1754 8C1773 134 1794 199 1840 268C1894 352 1969 414 2014 414C2025 414 2031 405 2031 390C2031 372 2028 351 2020 319L1968 107C1959 70 1955 47 1955 31C1955 6 1966 -9 1985 -9C2011 -9 2047 12 2145 85L2135 103L2109 86C2080 67 2058 56 2048 56C2041 56 2035 65 2035 76C2035 81 2036 92 2037 96L2103 372C2110 401 2114 429 2114 446C2114 469 2103 482 2083 482C2041 482 1972 444 1913 389C1875 354 1847 320 1795 247L1833 408C1837 426 1839 438 1839 449C1839 470 1831 482 1816 482C1795 482 1756 460 1683 408Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g style="transform:matrix(1,0,0,1,492.6458740234375,221.64668701171874);"><path d="M146 266C146 526 243 632 301 700L282 726C225 675 60 542 60 266C60 159 85 58 133 -32C168 -99 200 -138 282 -215L301 -194C255 -137 146 -15 146 266Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g style="transform:matrix(1,0,0,1,500.78125,221.64668701171874);"><path d="M24 388L31 368L63 389C100 412 103 414 110 414C121 414 128 404 128 389C128 338 87 145 46 2L53 -9C78 -2 101 4 123 8C142 134 163 199 209 268C263 352 338 414 383 414C394 414 400 405 400 390C400 372 397 351 389 319L337 107C328 70 324 47 324 31C324 6 335 -9 354 -9C380 -9 416 12 514 85L504 103L478 86C449 67 427 56 417 56C410 56 404 65 404 76C404 81 405 92 406 96L472 372C479 401 483 429 483 446C483 469 472 482 452 482C410 482 341 444 282 389C244 354 216 320 164 247L202 408C206 426 208 438 208 449C208 470 200 482 185 482C164 482 125 460 52 408Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g><g><g><g style="transform:matrix(1,0,0,1,515.1041870117188,225.96934350585937);"><path d="M462 224C462 345 355 366 308 374C388 436 418 482 418 541C418 630 344 689 233 689C165 689 120 670 72 622L43 498L74 498L92 554C103 588 166 622 218 622C283 622 336 569 336 506C336 431 277 368 206 368C198 368 187 369 174 370L159 371L147 318L154 312C192 329 211 334 238 334C321 334 369 281 369 190C369 88 308 21 215 21C169 21 128 36 98 64C74 86 61 109 42 163L15 153C36 92 44 56 50 6C103 -12 147 -20 184 -20C307 -20 462 87 462 224Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017136,0,0,-0.017136,0,0);"></path></g></g></g></g><g style="transform:matrix(1,0,0,1,525.125,221.64668701171874);"><path d="M51 726L32 700C87 636 187 526 187 266C187 -10 83 -131 32 -194L51 -215C104 -165 273 -23 273 265C273 542 108 675 51 726Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g style="transform:matrix(1,0,0,1,540.6041870117188,221.64668701171874);"><path d="M604 347L604 406L65 406L65 347ZM604 134L604 193L65 193L65 134Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g style="transform:matrix(1,0,0,1,564.3021240234375,221.64668701171874);"><path d="M289 -175C226 -161 206 -117 206 -45L206 128C206 207 197 253 125 272L125 274C194 292 206 335 206 409L206 595C206 667 224 707 289 726C189 726 134 703 134 578L134 392C134 327 120 292 58 273C124 254 134 223 134 151L134 -17C134 -149 176 -175 289 -175Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g style="transform:matrix(1,0,0,1,573.65625,221.64668701171874);"><path d="M125 390L69 107C68 99 56 61 56 31C56 6 67 -9 86 -9C121 -9 156 11 234 74L265 99L255 117L210 86C181 66 161 56 150 56C141 56 136 64 136 76C136 102 150 183 179 328L192 390L299 390L310 440C272 436 238 434 200 434C216 528 227 577 245 631L234 646C214 634 187 622 156 610L131 440C87 419 61 408 43 403L41 390Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g><g><g><g style="transform:matrix(1,0,0,1,582.5208740234375,225.96934350585937);"><path d="M16 23L16 -3C203 -3 203 0 239 0C275 0 275 -3 468 -3L468 82C353 77 307 81 122 77L304 270C401 373 431 428 431 503C431 618 353 689 226 689C154 689 105 669 56 619L39 483L68 483L81 529C97 587 133 612 200 612C286 612 341 558 341 473C341 398 299 324 186 204Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017136,0,0,-0.017136,0,0);"></path></g></g></g></g><g style="transform:matrix(1,0,0,1,592.5416870117188,221.64668701171874);"><path d="M204 123C177 114 159 108 106 93C99 17 74 -48 16 -144L30 -155L71 -136C152 -31 190 32 218 109Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g style="transform:matrix(1,0,0,1,603.541748046875,221.64668701171874);"><path d="M374 390L318 107C317 99 305 61 305 31C305 6 316 -9 335 -9C370 -9 405 11 483 74L514 99L504 117L459 86C430 66 410 56 399 56C390 56 385 64 385 76C385 102 399 183 428 328L441 390L548 390L559 440C521 436 487 434 449 434C465 528 476 577 494 631L483 646C463 634 436 622 405 610L380 440C336 419 310 408 292 403L290 390Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g><g><g><g style="transform:matrix(1,0,0,1,618.5,225.96934350585937);"><path d="M131 331C152 512 241 611 421 665L379 689C283 657 242 637 191 593C88 506 32 384 32 247C32 82 112 -20 241 -20C371 -20 468 83 468 219C468 334 399 409 293 409C216 409 184 370 131 331ZM255 349C331 349 382 283 382 184C382 80 331 13 254 13C169 13 123 86 123 220C123 255 127 274 138 291C160 325 207 349 255 349Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017136,0,0,-0.017136,0,0);"></path></g></g></g></g><g style="transform:matrix(1,0,0,1,629.7396240234375,221.64668701171874);"><path d="M275 273C213 292 199 327 199 392L199 578C199 703 144 726 44 726C109 707 127 667 127 595L127 409C127 335 139 292 208 274L208 272C136 253 127 207 127 128L127 -45C127 -117 107 -161 44 -175C157 -175 199 -149 199 -17L199 151C199 223 209 254 275 273Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g></g></g></g></g></g><g><g><g><g><g><g style="transform:matrix(1,0,0,1,439.125,281.64668701171877);"><path d="M125 390L69 107C68 99 56 61 56 31C56 6 67 -9 86 -9C121 -9 156 11 234 74L265 99L255 117L210 86C181 66 161 56 150 56C141 56 136 64 136 76C136 102 150 183 179 328L192 390L299 390L310 440C272 436 238 434 200 434C216 528 227 577 245 631L234 646C214 634 187 622 156 610L131 440C87 419 61 408 43 403L41 390ZM349 152C349 46 394 -11 477 -11C532 -11 592 15 637 57C699 116 743 230 743 331C743 425 693 482 610 482C506 482 349 382 349 152ZM573 444C634 444 667 399 667 315C667 219 636 113 592 60C574 39 548 27 517 27C459 27 425 72 425 151C425 264 464 387 513 427C526 438 549 444 573 444ZM1015 722L1003 733C951 707 915 698 843 691L839 670L887 670C911 670 921 663 921 646C921 638 920 629 919 622L871 354C857 279 841 210 789 1L798 -9L865 8L888 132C897 180 921 225 955 259C1024 59 1056 -9 1081 -9C1094 -9 1118 4 1162 35L1208 67L1200 86L1157 62C1143 54 1136 52 1128 52C1118 52 1111 58 1102 75C1067 139 1045 193 1006 316L1020 330C1081 391 1127 416 1177 416C1185 416 1196 414 1213 410L1230 473C1212 479 1194 482 1182 482C1116 482 1033 405 908 230ZM1571 111L1547 94C1494 56 1446 36 1410 36C1363 36 1334 73 1334 133C1334 158 1337 185 1342 214C1359 218 1468 248 1493 259C1578 296 1617 342 1617 404C1617 451 1583 482 1533 482C1465 496 1355 423 1318 349C1288 299 1258 180 1258 113C1258 35 1302 -11 1374 -11C1431 -11 1487 17 1579 92ZM1356 274C1373 343 1393 386 1422 412C1440 428 1471 440 1495 440C1524 440 1543 420 1543 388C1543 344 1508 297 1456 272C1428 258 1392 247 1347 237ZM1655 388L1662 368L1694 389C1731 412 1734 414 1741 414C1752 414 1759 404 1759 389C1759 338 1718 145 1677 2L1684 -9C1709 -2 1732 4 1754 8C1773 134 1794 199 1840 268C1894 352 1969 414 2014 414C2025 414 2031 405 2031 390C2031 372 2028 351 2020 319L1968 107C1959 70 1955 47 1955 31C1955 6 1966 -9 1985 -9C2011 -9 2047 12 2145 85L2135 103L2109 86C2080 67 2058 56 2048 56C2041 56 2035 65 2035 76C2035 81 2036 92 2037 96L2103 372C2110 401 2114 429 2114 446C2114 469 2103 482 2083 482C2041 482 1972 444 1913 389C1875 354 1847 320 1795 247L1833 408C1837 426 1839 438 1839 449C1839 470 1831 482 1816 482C1795 482 1756 460 1683 408Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g style="transform:matrix(1,0,0,1,492.6458740234375,281.64668701171877);"><path d="M146 266C146 526 243 632 301 700L282 726C225 675 60 542 60 266C60 159 85 58 133 -32C168 -99 200 -138 282 -215L301 -194C255 -137 146 -15 146 266Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g style="transform:matrix(1,0,0,1,500.78125,281.64668701171877);"><path d="M24 388L31 368L63 389C100 412 103 414 110 414C121 414 128 404 128 389C128 338 87 145 46 2L53 -9C78 -2 101 4 123 8C142 134 163 199 209 268C263 352 338 414 383 414C394 414 400 405 400 390C400 372 397 351 389 319L337 107C328 70 324 47 324 31C324 6 335 -9 354 -9C380 -9 416 12 514 85L504 103L478 86C449 67 427 56 417 56C410 56 404 65 404 76C404 81 405 92 406 96L472 372C479 401 483 429 483 446C483 469 472 482 452 482C410 482 341 444 282 389C244 354 216 320 164 247L202 408C206 426 208 438 208 449C208 470 200 482 185 482C164 482 125 460 52 408Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g><g><g><g style="transform:matrix(1,0,0,1,515.1041870117188,285.9693435058594);"><path d="M280 181L280 106C280 46 269 32 220 30L158 27L158 -3C291 0 291 0 315 0C339 0 339 0 472 -3L472 27L424 30C375 33 364 46 364 106L364 181C423 181 444 180 472 177L472 248L364 245L365 697L285 667L2 204L2 181ZM280 245L65 245L280 597Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017136,0,0,-0.017136,0,0);"></path></g></g></g></g><g style="transform:matrix(1,0,0,1,525.125,281.64668701171877);"><path d="M51 726L32 700C87 636 187 526 187 266C187 -10 83 -131 32 -194L51 -215C104 -165 273 -23 273 265C273 542 108 675 51 726Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g style="transform:matrix(1,0,0,1,540.6041870117188,281.64668701171877);"><path d="M604 347L604 406L65 406L65 347ZM604 134L604 193L65 193L65 134Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g style="transform:matrix(1,0,0,1,564.3021240234375,281.64668701171877);"><path d="M289 -175C226 -161 206 -117 206 -45L206 128C206 207 197 253 125 272L125 274C194 292 206 335 206 409L206 595C206 667 224 707 289 726C189 726 134 703 134 578L134 392C134 327 120 292 58 273C124 254 134 223 134 151L134 -17C134 -149 176 -175 289 -175Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g style="transform:matrix(1,0,0,1,573.65625,281.64668701171877);"><path d="M125 390L69 107C68 99 56 61 56 31C56 6 67 -9 86 -9C121 -9 156 11 234 74L265 99L255 117L210 86C181 66 161 56 150 56C141 56 136 64 136 76C136 102 150 183 179 328L192 390L299 390L310 440C272 436 238 434 200 434C216 528 227 577 245 631L234 646C214 634 187 622 156 610L131 440C87 419 61 408 43 403L41 390Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g><g><g><g style="transform:matrix(1,0,0,1,582.5208740234375,285.9693435058594);"><path d="M280 181L280 106C280 46 269 32 220 30L158 27L158 -3C291 0 291 0 315 0C339 0 339 0 472 -3L472 27L424 30C375 33 364 46 364 106L364 181C423 181 444 180 472 177L472 248L364 245L365 697L285 667L2 204L2 181ZM280 245L65 245L280 597Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017136,0,0,-0.017136,0,0);"></path></g></g></g></g><g style="transform:matrix(1,0,0,1,592.5416870117188,281.64668701171877);"><path d="M204 123C177 114 159 108 106 93C99 17 74 -48 16 -144L30 -155L71 -136C152 -31 190 32 218 109Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g style="transform:matrix(1,0,0,1,603.541748046875,281.64668701171877);"><path d="M374 390L318 107C317 99 305 61 305 31C305 6 316 -9 335 -9C370 -9 405 11 483 74L514 99L504 117L459 86C430 66 410 56 399 56C390 56 385 64 385 76C385 102 399 183 428 328L441 390L548 390L559 440C521 436 487 434 449 434C465 528 476 577 494 631L483 646C463 634 436 622 405 610L380 440C336 419 310 408 292 403L290 390Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g><g><g><g><g style="transform:matrix(1,0,0,1,618.5,285.9693435058594);"><path d="M168 345C127 326 110 315 88 294C50 256 30 211 30 159C30 56 113 -20 226 -20C356 -20 464 82 464 206C464 286 427 329 313 381C404 443 436 485 436 545C436 631 365 689 259 689C140 689 53 613 53 508C53 440 80 402 168 345ZM284 295C347 267 385 218 385 164C385 80 322 14 241 14C156 14 101 74 101 167C101 240 130 286 204 331ZM223 423C160 454 128 494 128 544C128 610 176 655 247 655C320 655 368 607 368 534C368 477 343 438 278 396Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017136,0,0,-0.017136,0,0);"></path></g></g></g></g><g style="transform:matrix(1,0,0,1,629.7396240234375,281.64668701171877);"><path d="M275 273C213 292 199 327 199 392L199 578C199 703 144 726 44 726C109 707 127 667 127 595L127 409C127 335 139 292 208 274L208 272C136 253 127 207 127 128L127 -45C127 -117 107 -161 44 -175C157 -175 199 -149 199 -17L199 151C199 223 209 254 275 273Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.024480000000000002,0,0,-0.024480000000000002,0,0);"></path></g></g></g></g></g></g><g><g><g style="transform:matrix(1,0,0,1,471.734375,61.95);"><path d="M35 623L177 623L295 624L295 0L384 0L384 624L501 623L643 623L643 689L35 689ZM627 221C627 85 732 -11 846 -11C960 -11 1066 84 1066 220C1066 345 974 461 846 461C728 461 627 357 627 221ZM705 231C705 352 782 399 846 399C910 399 987 353 987 231C987 106 913 53 846 53C780 53 705 105 705 231ZM1180 0L1250 0L1250 142L1330 224L1486 0L1568 0L1378 273L1546 445L1456 445L1252 236L1252 722L1180 722ZM1592 226C1592 102 1679 -11 1810 -11C1873 -11 1925 11 1968 40L1962 106C1918 74 1873 51 1810 51C1734 51 1666 117 1662 220L1972 220L1972 249C1963 420 1865 461 1794 461C1688 461 1592 360 1592 226ZM1667 275C1678 324 1718 399 1794 399C1823 399 1898 386 1915 275ZM2082 0L2160 0L2160 240C2160 314 2185 394 2262 394C2360 394 2360 319 2360 281L2360 0L2438 0L2438 288C2438 358 2429 455 2301 455C2213 455 2169 398 2155 380L2155 450L2082 450ZM2952 0L3030 0L3030 624C3039 580 3063 517 3102 411L3252 22L3324 22L3478 423L3538 587L3548 624L3549 592L3549 0L3627 0L3627 694L3512 694L3358 292C3309 164 3293 114 3290 92L3289 92C3285 115 3261 182 3220 293L3066 694L2952 694ZM3771 113C3771 67 3801 -11 3885 -11C3995 -11 4050 34 4051 35L4051 0L4127 0L4127 305C4122 389 4056 461 3961 461C3898 461 3852 445 3804 417L3810 350C3843 373 3885 401 3960 401C3973 401 3999 400 4023 371C4048 341 4048 307 4049 279L4049 245C3946 244 3771 219 3771 113ZM3844 114C3844 177 3977 192 4049 193L4049 130C4049 65 3980 51 3937 51C3883 51 3844 78 3844 114ZM4288 -195L4366 -195L4366 46C4399 15 4442 -11 4501 -11C4598 -11 4687 83 4687 224C4687 343 4624 455 4529 455C4495 455 4427 448 4363 396L4363 445L4288 445ZM4366 126L4366 334C4372 341 4406 391 4472 391C4554 391 4608 310 4608 222C4608 116 4533 50 4463 50C4404 50 4366 102 4366 126Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.02595,0,0,-0.02595,0,0);"></path></g></g></g></svg>
+</svg>
diff --git a/doc/source/architecture/overview.rst b/doc/source/architecture/overview.rst
index 005b15b..e5fcbe3 100644
--- a/doc/source/architecture/overview.rst
+++ b/doc/source/architecture/overview.rst
@@ -14,7 +14,101 @@
.. See the License for the specific language governing permissions and
.. limitations under the License.
+.. _overview:
+
Overview
+========
+
+Apache Cassandra is an open source, distributed, NoSQL database. It presents
+a partitioned wide column storage model with eventually consistent semantics.
+
+Apache Cassandra was initially designed at `Facebook
+<https://www.cs.cornell.edu/projects/ladis2009/papers/lakshman-ladis2009.pdf>`_
+using a staged event-driven architecture (`SEDA
+<http://www.sosp.org/2001/papers/welsh.pdf>`_) to implement a combination of
+Amazon’s `Dynamo
+<http://courses.cse.tamu.edu/caverlee/csce438/readings/dynamo-paper.pdf>`_
+distributed storage and replication techniques combined with Google's `Bigtable
+<https://static.googleusercontent.com/media/research.google.com/en//archive/bigtable-osdi06.pdf>`_
+data and storage engine model. Dynamo and Bigtable were both developed to meet
+emerging requirements for scalable, reliable and highly available storage
+systems, but each had areas that could be improved.
+
+Cassandra was designed as a best in class combination of both systems to meet
+emerging large scale, both in data footprint and query volume, storage
+requirements. As applications began to require full global replication and
+always available low-latency reads and writes, it became imperative to design a
+new kind of database model as the relational database systems of the time
+struggled to meet the new requirements of global scale applications.
+
+Systems like Cassandra are designed for these challenges and seek the
+following design objectives:
+
+- Full multi-master database replication
+- Global availability at low latency
+- Scaling out on commodity hardware
+- Linear throughput increase with each additional processor
+- Online load balancing and cluster growth
+- Partitioned key-oriented queries
+- Flexible schema
+
+Features
--------
-.. todo:: todo
+Cassandra provides the Cassandra Query Language (CQL), an SQL-like language,
+to create and update database schema and access data. CQL allows users to
+organize data within a cluster of Cassandra nodes using:
+
+- **Keyspace**: defines how a dataset is replicated, for example in which
+ datacenters and how many copies. Keyspaces contain tables.
+- **Table**: defines the typed schema for a collection of partitions. Cassandra
+ tables have flexible addition of new columns to tables with zero downtime.
+ Tables contain partitions, which contain partitions, which contain columns.
+- **Partition**: defines the mandatory part of the primary key all rows in
+ Cassandra must have. All performant queries supply the partition key in
+ the query.
+- **Row**: contains a collection of columns identified by a unique primary key
+ made up of the partition key and optionally additional clustering keys.
+- **Column**: A single datum with a type which belong to a row.
+
+CQL supports numerous advanced features over a partitioned dataset such as:
+
+- Single partition lightweight transactions with atomic compare and set
+ semantics.
+- User-defined types, functions and aggregates
+- Collection types including sets, maps, and lists.
+- Local secondary indices
+- (Experimental) materialized views
+
+Cassandra explicitly chooses not to implement operations that require cross
+partition coordination as they are typically slow and hard to provide highly
+available global semantics. For example Cassandra does not support:
+
+- Cross partition transactions
+- Distributed joins
+- Foreign keys or referential integrity.
+
+Operating
+---------
+
+Apache Cassandra configuration settings are configured in the ``cassandra.yaml``
+file that can be edited by hand or with the aid of configuration management tools.
+Some settings can be manipulated live using an online interface, but others
+require a restart of the database to take effect.
+
+Cassandra provides tools for managing a cluster. The ``nodetool`` command
+interacts with Cassandra's live control interface, allowing runtime manipulation
+of many settings from ``cassandra.yaml``. The ``auditlogviewer`` is used
+to view the audit logs. The ``fqltool`` is used to view, replay and compare
+full query logs. The ``auditlogviewer`` and ``fqltool`` are new tools in
+Apache Cassandra 4.0.
+
+In addition, Cassandra supports out of the box atomic snapshot functionality,
+which presents a point in time snapshot of Cassandra's data for easy
+integration with many backup tools. Cassandra also supports incremental backups
+where data can be backed up as it is written.
+
+Apache Cassandra 4.0 has added several new features including virtual tables.
+transient replication, audit logging, full query logging, and support for Java
+11. Two of these features are experimental: transient replication and Java 11
+support.
diff --git a/doc/source/architecture/storage_engine.rst b/doc/source/architecture/storage_engine.rst
index e4114e5..23b738d 100644
--- a/doc/source/architecture/storage_engine.rst
+++ b/doc/source/architecture/storage_engine.rst
@@ -22,7 +22,54 @@
CommitLog
^^^^^^^^^
-.. todo:: todo
+Commitlogs are an append only log of all mutations local to a Cassandra node. Any data written to Cassandra will first be written to a commit log before being written to a memtable. This provides durability in the case of unexpected shutdown. On startup, any mutations in the commit log will be applied to memtables.
+
+All mutations write optimized by storing in commitlog segments, reducing the number of seeks needed to write to disk. Commitlog Segments are limited by the "commitlog_segment_size_in_mb" option, once the size is reached, a new commitlog segment is created. Commitlog segments can be archived, deleted, or recycled once all its data has been flushed to SSTables. Commitlog segments are truncated when Cassandra has written data older than a certain point to the SSTables. Running "nodetool drain" before stopping Cassandra will write everything in the memtables to SSTables and remove the need to sync with the commitlogs on startup.
+
+- ``commitlog_segment_size_in_mb``: The default size is 32, which is almost always fine, but if you are archiving commitlog segments (see commitlog_archiving.properties), then you probably want a finer granularity of archiving; 8 or 16 MB is reasonable. Max mutation size is also configurable via max_mutation_size_in_kb setting in cassandra.yaml. The default is half the size commitlog_segment_size_in_mb * 1024.
+
+***NOTE: If max_mutation_size_in_kb is set explicitly then commitlog_segment_size_in_mb must be set to at least twice the size of max_mutation_size_in_kb / 1024***
+
+*Default Value:* 32
+
+Commitlogs are an append only log of all mutations local to a Cassandra node. Any data written to Cassandra will first be written to a commit log before being written to a memtable. This provides durability in the case of unexpected shutdown. On startup, any mutations in the commit log will be applied.
+
+- ``commitlog_sync``: may be either “periodic” or “batch.”
+
+ - ``batch``: In batch mode, Cassandra won’t ack writes until the commit log has been fsynced to disk. It will wait "commitlog_sync_batch_window_in_ms" milliseconds between fsyncs. This window should be kept short because the writer threads will be unable to do extra work while waiting. You may need to increase concurrent_writes for the same reason.
+
+ - ``commitlog_sync_batch_window_in_ms``: Time to wait between "batch" fsyncs
+ *Default Value:* 2
+
+ - ``periodic``: In periodic mode, writes are immediately ack'ed, and the CommitLog is simply synced every "commitlog_sync_period_in_ms" milliseconds.
+
+ - ``commitlog_sync_period_in_ms``: Time to wait between "periodic" fsyncs
+ *Default Value:* 10000
+
+*Default Value:* batch
+
+*** NOTE: In the event of an unexpected shutdown, Cassandra can lose up to the sync period or more if the sync is delayed. If using "batch" mode, it is recommended to store commitlogs in a separate, dedicated device.**
+
+
+- ``commitlog_directory``: This option is commented out by default When running on magnetic HDD, this should be a separate spindle than the data directories. If not set, the default directory is $CASSANDRA_HOME/data/commitlog.
+
+*Default Value:* /var/lib/cassandra/commitlog
+
+- ``commitlog_compression``: Compression to apply to the commitlog. If omitted, the commit log will be written uncompressed. LZ4, Snappy, Deflate and Zstd compressors are supported.
+
+(Default Value: (complex option)::
+
+ # - class_name: LZ4Compressor
+ # parameters:
+ # -
+
+- ``commitlog_total_space_in_mb``: Total space to use for commit logs on disk.
+
+If space gets above this value, Cassandra will flush every dirty CF in the oldest segment and remove it. So a small total commitlog space will tend to cause more flush activity on less-active columnfamilies.
+
+The default value is the smaller of 8192, and 1/4 of the total space of the commitlog volume.
+
+*Default Value:* 8192
.. _memtables:
@@ -80,3 +127,82 @@
stored in the order of their clustering keys.
SSTables can be optionally compressed using block-based compression.
+
+SSTable Versions
+^^^^^^^^^^^^^^^^
+
+This section was created using the following
+`gist <https://gist.github.com/shyamsalimkumar/49a61e5bc6f403d20c55>`_
+which utilized this original
+`source <http://www.bajb.net/2013/03/cassandra-sstable-format-version-numbers/>`_.
+
+The version numbers, to date are:
+
+Version 0
+~~~~~~~~~
+
+* b (0.7.0): added version to sstable filenames
+* c (0.7.0): bloom filter component computes hashes over raw key bytes instead of strings
+* d (0.7.0): row size in data component becomes a long instead of int
+* e (0.7.0): stores undecorated keys in data and index components
+* f (0.7.0): switched bloom filter implementations in data component
+* g (0.8): tracks flushed-at context in metadata component
+
+Version 1
+~~~~~~~~~
+
+* h (1.0): tracks max client timestamp in metadata component
+* hb (1.0.3): records compression ration in metadata component
+* hc (1.0.4): records partitioner in metadata component
+* hd (1.0.10): includes row tombstones in maxtimestamp
+* he (1.1.3): includes ancestors generation in metadata component
+* hf (1.1.6): marker that replay position corresponds to 1.1.5+ millis-based id (see CASSANDRA-4782)
+* ia (1.2.0):
+
+ * column indexes are promoted to the index file
+ * records estimated histogram of deletion times in tombstones
+ * bloom filter (keys and columns) upgraded to Murmur3
+* ib (1.2.1): tracks min client timestamp in metadata component
+* ic (1.2.5): omits per-row bloom filter of column names
+
+Version 2
+~~~~~~~~~
+
+* ja (2.0.0):
+
+ * super columns are serialized as composites (note that there is no real format change, this is mostly a marker to know if we should expect super columns or not. We do need a major version bump however, because we should not allow streaming of super columns into this new format)
+ * tracks max local deletiontime in sstable metadata
+ * records bloom_filter_fp_chance in metadata component
+ * remove data size and column count from data file (CASSANDRA-4180)
+ * tracks max/min column values (according to comparator)
+* jb (2.0.1):
+
+ * switch from crc32 to adler32 for compression checksums
+ * checksum the compressed data
+* ka (2.1.0):
+
+ * new Statistics.db file format
+ * index summaries can be downsampled and the sampling level is persisted
+ * switch uncompressed checksums to adler32
+ * tracks presense of legacy (local and remote) counter shards
+* la (2.2.0): new file name format
+* lb (2.2.7): commit log lower bound included
+
+Version 3
+~~~~~~~~~
+
+* ma (3.0.0):
+
+ * swap bf hash order
+ * store rows natively
+* mb (3.0.7, 3.7): commit log lower bound included
+* mc (3.0.8, 3.9): commit log intervals included
+
+Example Code
+~~~~~~~~~~~~
+
+The following example is useful for finding all sstables that do not match the "ib" SSTable version
+
+.. code-block:: bash
+
+ find /var/lib/cassandra/data/ -type f | grep -v -- -ib- | grep -v "/snapshots"
diff --git a/doc/source/bugs.rst b/doc/source/bugs.rst
index 240cfd4..32d676f 100644
--- a/doc/source/bugs.rst
+++ b/doc/source/bugs.rst
@@ -14,11 +14,11 @@
.. See the License for the specific language governing permissions and
.. limitations under the License.
-Reporting Bugs and Contributing
-===============================
+Reporting Bugs
+==============
If you encounter a problem with Cassandra, the first places to ask for help are the :ref:`user mailing list
-<mailing-lists>` and the ``#cassandra`` :ref:`IRC channel <irc-channels>`.
+<mailing-lists>` and the ``cassandra`` :ref:`Slack room <slack>`.
If, after having asked for help, you suspect that you have found a bug in Cassandra, you should report it by opening a
ticket through the `Apache Cassandra JIRA <https://issues.apache.org/jira/browse/CASSANDRA>`__. Please provide as much
diff --git a/doc/source/conf.py b/doc/source/conf.py
index 7143b23..48f87a8 100644
--- a/doc/source/conf.py
+++ b/doc/source/conf.py
@@ -75,7 +75,7 @@
# General information about the project.
project = u'Apache Cassandra'
-copyright = u'2016, The Apache Cassandra team'
+copyright = u'2020, The Apache Cassandra team'
author = u'The Apache Cassandra team'
# The version info for the project you're documenting, acts as replacement for
diff --git a/doc/source/configuration/cass_cl_archive_file.rst b/doc/source/configuration/cass_cl_archive_file.rst
new file mode 100644
index 0000000..fc14440
--- /dev/null
+++ b/doc/source/configuration/cass_cl_archive_file.rst
@@ -0,0 +1,46 @@
+.. _cassandra-cl-archive:
+
+commitlog-archiving.properties file
+================================
+
+The ``commitlog-archiving.properties`` configuration file can optionally set commands that are executed when archiving or restoring a commitlog segment.
+
+===========================
+Options
+===========================
+
+``archive_command=<command>``
+------
+One command can be inserted with %path and %name arguments. %path is the fully qualified path of the commitlog segment to archive. %name is the filename of the commitlog. STDOUT, STDIN, or multiple commands cannot be executed. If multiple commands are required, add a pointer to a script in this option.
+
+**Example:** archive_command=/bin/ln %path /backup/%name
+
+**Default value:** blank
+
+``restore_command=<command>``
+------
+One command can be inserted with %from and %to arguments. %from is the fully qualified path to an archived commitlog segment using the specified restore directories. %to defines the directory to the live commitlog location.
+
+**Example:** restore_command=/bin/cp -f %from %to
+
+**Default value:** blank
+
+``restore_directories=<directory>``
+------
+Defines the directory to scan the recovery files into.
+
+**Default value:** blank
+
+``restore_point_in_time=<timestamp>``
+------
+Restore mutations created up to and including this timestamp in GMT in the format ``yyyy:MM:dd HH:mm:ss``. Recovery will continue through the segment when the first client-supplied timestamp greater than this time is encountered, but only mutations less than or equal to this timestamp will be applied.
+
+**Example:** 2020:04:31 20:43:12
+
+**Default value:** blank
+
+``precision=<timestamp_precision>``
+------
+Precision of the timestamp used in the inserts. Choice is generally MILLISECONDS or MICROSECONDS
+
+**Default value:** MICROSECONDS
diff --git a/doc/source/configuration/cass_env_sh_file.rst b/doc/source/configuration/cass_env_sh_file.rst
new file mode 100644
index 0000000..457f39f
--- /dev/null
+++ b/doc/source/configuration/cass_env_sh_file.rst
@@ -0,0 +1,132 @@
+.. _cassandra-envsh:
+
+cassandra-env.sh file
+=====================
+
+The ``cassandra-env.sh`` bash script file can be used to pass additional options to the Java virtual machine (JVM), such as maximum and minimum heap size, rather than setting them in the environment. If the JVM settings are static and do not need to be computed from the node characteristics, the :ref:`cassandra-jvm-options` files should be used instead. For example, commonly computed values are the heap sizes, using the system values.
+
+For example, add ``JVM_OPTS="$JVM_OPTS -Dcassandra.load_ring_state=false"`` to the ``cassandra_env.sh`` file
+and run the command-line ``cassandra`` to start. The option is set from the ``cassandra-env.sh`` file, and is equivalent to starting Cassandra with the command-line option ``cassandra -Dcassandra.load_ring_state=false``.
+
+The ``-D`` option specifies the start-up parameters in both the command line and ``cassandra-env.sh`` file. The following options are available:
+
+``cassandra.auto_bootstrap=false``
+----------------------------------
+Facilitates setting auto_bootstrap to false on initial set-up of the cluster. The next time you start the cluster, you do not need to change the ``cassandra.yaml`` file on each node to revert to true, the default value.
+
+``cassandra.available_processors=<number_of_processors>``
+---------------------------------------------------------
+In a multi-instance deployment, multiple Cassandra instances will independently assume that all CPU processors are available to it. This setting allows you to specify a smaller set of processors.
+
+``cassandra.boot_without_jna=true``
+-----------------------------------
+If JNA fails to initialize, Cassandra fails to boot. Use this command to boot Cassandra without JNA.
+
+``cassandra.config=<directory>``
+--------------------------------
+The directory location of the ``cassandra.yaml file``. The default location depends on the type of installation.
+
+``cassandra.ignore_dynamic_snitch_severity=true|false``
+-------------------------------------------------------
+Setting this property to true causes the dynamic snitch to ignore the severity indicator from gossip when scoring nodes. Explore failure detection and recovery and dynamic snitching for more information.
+
+**Default:** false
+
+``cassandra.initial_token=<token>``
+-----------------------------------
+Use when virtual nodes (vnodes) are not used. Sets the initial partitioner token for a node the first time the node is started.
+Note: Vnodes are highly recommended as they automatically select tokens.
+
+**Default:** disabled
+
+``cassandra.join_ring=true|false``
+----------------------------------
+Set to false to start Cassandra on a node but not have the node join the cluster.
+You can use ``nodetool join`` and a JMX call to join the ring afterwards.
+
+**Default:** true
+
+``cassandra.load_ring_state=true|false``
+----------------------------------------
+Set to false to clear all gossip state for the node on restart.
+
+**Default:** true
+
+``cassandra.metricsReporterConfigFile=<filename>``
+--------------------------------------------------
+Enable pluggable metrics reporter. Explore pluggable metrics reporting for more information.
+
+``cassandra.partitioner=<partitioner>``
+---------------------------------------
+Set the partitioner.
+
+**Default:** org.apache.cassandra.dht.Murmur3Partitioner
+
+``cassandra.prepared_statements_cache_size_in_bytes=<cache_size>``
+------------------------------------------------------------------
+Set the cache size for prepared statements.
+
+``cassandra.replace_address=<listen_address of dead node>|<broadcast_address of dead node>``
+--------------------------------------------------------------------------------------------
+To replace a node that has died, restart a new node in its place specifying the ``listen_address`` or ``broadcast_address`` that the new node is assuming. The new node must not have any data in its data directory, the same state as before bootstrapping.
+Note: The ``broadcast_address`` defaults to the ``listen_address`` except when using the ``Ec2MultiRegionSnitch``.
+
+``cassandra.replayList=<table>``
+--------------------------------
+Allow restoring specific tables from an archived commit log.
+
+``cassandra.ring_delay_ms=<number_of_ms>``
+------------------------------------------
+Defines the amount of time a node waits to hear from other nodes before formally joining the ring.
+
+**Default:** 1000ms
+
+``cassandra.native_transport_port=<port>``
+------------------------------------------
+Set the port on which the CQL native transport listens for clients.
+
+**Default:** 9042
+
+``cassandra.rpc_port=<port>``
+-----------------------------
+Set the port for the Thrift RPC service, which is used for client connections.
+
+**Default:** 9160
+
+``cassandra.storage_port=<port>``
+---------------------------------
+Set the port for inter-node communication.
+
+**Default:** 7000
+
+``cassandra.ssl_storage_port=<port>``
+-------------------------------------
+Set the SSL port for encrypted communication.
+
+**Default:** 7001
+
+``cassandra.start_native_transport=true|false``
+-----------------------------------------------
+Enable or disable the native transport server. See ``start_native_transport`` in ``cassandra.yaml``.
+
+**Default:** true
+
+``cassandra.start_rpc=true|false``
+----------------------------------
+Enable or disable the Thrift RPC server.
+
+**Default:** true
+
+``cassandra.triggers_dir=<directory>``
+--------------------------------------
+Set the default location for the trigger JARs.
+
+**Default:** conf/triggers
+
+``cassandra.write_survey=true``
+-------------------------------
+For testing new compaction and compression strategies. It allows you to experiment with different strategies and benchmark write performance differences without affecting the production workload.
+
+``consistent.rangemovement=true|false``
+---------------------------------------
+Set to true makes Cassandra perform bootstrap safely without violating consistency. False disables this.
diff --git a/doc/source/configuration/cass_jvm_options_file.rst b/doc/source/configuration/cass_jvm_options_file.rst
new file mode 100644
index 0000000..f5a6326
--- /dev/null
+++ b/doc/source/configuration/cass_jvm_options_file.rst
@@ -0,0 +1,10 @@
+.. _cassandra-jvm-options:
+
+jvm-* files
+===========
+
+Several files for JVM configuration are included in Cassandra. The ``jvm-server.options`` file, and corresponding files ``jvm8-server.options`` and ``jvm11-server.options`` are the main file for settings that affect the operation of the Cassandra JVM on cluster nodes. The file includes startup parameters, general JVM settings such as garbage collection, and heap settings. The ``jvm-clients.options`` and corresponding ``jvm8-clients.options`` and ``jvm11-clients.options`` files can be used to configure JVM settings for clients like ``nodetool`` and the ``sstable`` tools.
+
+See each file for examples of settings.
+
+.. note:: The ``jvm-*`` files replace the :ref:`cassandra-envsh` file used in Cassandra versions prior to Cassandra 3.0. The ``cassandra-env.sh`` bash script file is still useful if JVM settings must be dynamically calculated based on system settings. The ``jvm-*`` files only store static JVM settings.
diff --git a/doc/source/configuration/cass_logback_xml_file.rst b/doc/source/configuration/cass_logback_xml_file.rst
new file mode 100644
index 0000000..3de1c77
--- /dev/null
+++ b/doc/source/configuration/cass_logback_xml_file.rst
@@ -0,0 +1,157 @@
+.. _cassandra-logback-xml:
+
+logback.xml file
+================================
+
+The ``logback.xml`` configuration file can optionally set logging levels for the logs written to ``system.log`` and ``debug.log``. The logging levels can also be set using ``nodetool setlogginglevels``.
+
+===========================
+Options
+===========================
+
+``appender name="<appender_choice>"...</appender>``
+------
+
+Specify log type and settings. Possible appender names are: ``SYSTEMLOG``, ``DEBUGLOG``, ``ASYNCDEBUGLOG``, and ``STDOUT``. ``SYSTEMLOG`` ensures that WARN and ERROR message are written synchronously to the specified file. ``DEBUGLOG`` and ``ASYNCDEBUGLOG`` ensure that DEBUG messages are written either synchronously or asynchronously, respectively, to the specified file. ``STDOUT`` writes all messages to the console in a human-readable format.
+
+**Example:** <appender name="SYSTEMLOG" class="ch.qos.logback.core.rolling.RollingFileAppender">
+
+``<file> <filename> </file>``
+------
+
+Specify the filename for a log.
+
+**Example:** <file>${cassandra.logdir}/system.log</file>
+
+``<level> <log_level> </level>``
+------
+
+Specify the level for a log. Part of the filter. Levels are: ``ALL``, ``TRACE``, ``DEBUG``, ``INFO``, ``WARN``, ``ERROR``, ``OFF``. ``TRACE`` creates the most verbose log, ``ERROR`` the least.
+
+.. note::
+Note: Increasing logging levels can generate heavy logging output on a moderately trafficked cluster.
+You can use the ``nodetool getlogginglevels`` command to see the current logging configuration.
+
+**Default:** INFO
+
+**Example:** <level>INFO</level>
+
+``<rollingPolicy class="<rolling_policy_choice>" <fileNamePattern><pattern_info></fileNamePattern> ... </rollingPolicy>``
+------
+
+Specify the policy for rolling logs over to an archive.
+
+**Example:** <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
+
+``<fileNamePattern> <pattern_info> </fileNamePattern>``
+------
+
+Specify the pattern information for rolling over the log to archive. Part of the rolling policy.
+
+**Example:** <fileNamePattern>${cassandra.logdir}/system.log.%d{yyyy-MM-dd}.%i.zip</fileNamePattern>
+
+``<maxFileSize> <size> </maxFileSize>``
+------
+
+Specify the maximum file size to trigger rolling a log. Part of the rolling policy.
+
+**Example:** <maxFileSize>50MB</maxFileSize>
+
+``<maxHistory> <number_of_days> </maxHistory>``
+------
+
+Specify the maximum history in days to trigger rolling a log. Part of the rolling policy.
+
+**Example:** <maxHistory>7</maxHistory>
+
+``<encoder> <pattern>...</pattern> </encoder>``
+------
+
+Specify the format of the message. Part of the rolling policy.
+
+**Example:** <maxHistory>7</maxHistory>
+**Example:** <encoder> <pattern>%-5level [%thread] %date{ISO8601} %F:%L - %msg%n</pattern> </encoder>
+
+Contents of default ``logback.xml``
+-----------------------
+
+.. code-block:: XML
+
+ <configuration scan="true" scanPeriod="60 seconds">
+ <jmxConfigurator />
+
+ <!-- No shutdown hook; we run it ourselves in StorageService after shutdown -->
+
+ <!-- SYSTEMLOG rolling file appender to system.log (INFO level) -->
+
+ <appender name="SYSTEMLOG" class="ch.qos.logback.core.rolling.RollingFileAppender">
+ <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
+ <level>INFO</level>
+ </filter>
+ <file>${cassandra.logdir}/system.log</file>
+ <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
+ <!-- rollover daily -->
+ <fileNamePattern>${cassandra.logdir}/system.log.%d{yyyy-MM-dd}.%i.zip</fileNamePattern>
+ <!-- each file should be at most 50MB, keep 7 days worth of history, but at most 5GB -->
+ <maxFileSize>50MB</maxFileSize>
+ <maxHistory>7</maxHistory>
+ <totalSizeCap>5GB</totalSizeCap>
+ </rollingPolicy>
+ <encoder>
+ <pattern>%-5level [%thread] %date{ISO8601} %F:%L - %msg%n</pattern>
+ </encoder>
+ </appender>
+
+ <!-- DEBUGLOG rolling file appender to debug.log (all levels) -->
+
+ <appender name="DEBUGLOG" class="ch.qos.logback.core.rolling.RollingFileAppender">
+ <file>${cassandra.logdir}/debug.log</file>
+ <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
+ <!-- rollover daily -->
+ <fileNamePattern>${cassandra.logdir}/debug.log.%d{yyyy-MM-dd}.%i.zip</fileNamePattern>
+ <!-- each file should be at most 50MB, keep 7 days worth of history, but at most 5GB -->
+ <maxFileSize>50MB</maxFileSize>
+ <maxHistory>7</maxHistory>
+ <totalSizeCap>5GB</totalSizeCap>
+ </rollingPolicy>
+ <encoder>
+ <pattern>%-5level [%thread] %date{ISO8601} %F:%L - %msg%n</pattern>
+ </encoder>
+ </appender>
+
+ <!-- ASYNCLOG assynchronous appender to debug.log (all levels) -->
+
+ <appender name="ASYNCDEBUGLOG" class="ch.qos.logback.classic.AsyncAppender">
+ <queueSize>1024</queueSize>
+ <discardingThreshold>0</discardingThreshold>
+ <includeCallerData>true</includeCallerData>
+ <appender-ref ref="DEBUGLOG" />
+ </appender>
+
+ <!-- STDOUT console appender to stdout (INFO level) -->
+
+ <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+ <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
+ <level>INFO</level>
+ </filter>
+ <encoder>
+ <pattern>%-5level [%thread] %date{ISO8601} %F:%L - %msg%n</pattern>
+ </encoder>
+ </appender>
+
+ <!-- Uncomment bellow and corresponding appender-ref to activate logback metrics
+ <appender name="LogbackMetrics" class="com.codahale.metrics.logback.InstrumentedAppender" />
+ -->
+
+ <root level="INFO">
+ <appender-ref ref="SYSTEMLOG" />
+ <appender-ref ref="STDOUT" />
+ <appender-ref ref="ASYNCDEBUGLOG" /> <!-- Comment this line to disable debug.log -->
+ <!--
+ <appender-ref ref="LogbackMetrics" />
+ -->
+ </root>
+
+ <logger name="org.apache.cassandra" level="DEBUG"/>
+ <logger name="com.thinkaurelius.thrift" level="ERROR"/>
+ </configuration>
diff --git a/doc/source/configuration/cass_rackdc_file.rst b/doc/source/configuration/cass_rackdc_file.rst
new file mode 100644
index 0000000..9921092
--- /dev/null
+++ b/doc/source/configuration/cass_rackdc_file.rst
@@ -0,0 +1,67 @@
+.. _cassandra-rackdc:
+
+cassandra-rackdc.properties file
+================================
+
+Several :term:`snitch` options use the ``cassandra-rackdc.properties`` configuration file to determine which :term:`datacenters` and racks cluster nodes belong to. Information about the
+network topology allows requests to be routed efficiently and to distribute replicas evenly. The following snitches can be configured here:
+
+- GossipingPropertyFileSnitch
+- AWS EC2 single-region snitch
+- AWS EC2 multi-region snitch
+
+The GossipingPropertyFileSnitch is recommended for production. This snitch uses the datacenter and rack information configured in a local node's ``cassandra-rackdc.properties``
+file and propagates the information to other nodes using :term:`gossip`. It is the default snitch and the settings in this properties file are enabled.
+
+The AWS EC2 snitches are configured for clusters in AWS. This snitch uses the ``cassandra-rackdc.properties`` options to designate one of two AWS EC2 datacenter and rack naming conventions:
+
+- legacy: Datacenter name is the part of the availability zone name preceding the last "-" when the zone ends in -1 and includes the number if not -1. Rack name is the portion of the availability zone name following the last "-".
+
+ Examples: us-west-1a => dc: us-west, rack: 1a; us-west-2b => dc: us-west-2, rack: 2b;
+
+- standard: Datacenter name is the standard AWS region name, including the number. Rack name is the region plus the availability zone letter.
+
+ Examples: us-west-1a => dc: us-west-1, rack: us-west-1a; us-west-2b => dc: us-west-2, rack: us-west-2b;
+
+Either snitch can set to use the local or internal IP address when multiple datacenters are not communicating.
+
+===========================
+GossipingPropertyFileSnitch
+===========================
+
+``dc``
+------
+Name of the datacenter. The value is case-sensitive.
+
+**Default value:** DC1
+
+``rack``
+--------
+Rack designation. The value is case-sensitive.
+
+**Default value:** RAC1
+
+===========================
+AWS EC2 snitch
+===========================
+
+``ec2_naming_scheme``
+---------------------
+Datacenter and rack naming convention. Options are ``legacy`` or ``standard`` (default). **This option is commented out by default.**
+
+**Default value:** standard
+
+
+.. NOTE::
+ YOU MUST USE THE ``legacy`` VALUE IF YOU ARE UPGRADING A PRE-4.0 CLUSTER.
+
+===========================
+Either snitch
+===========================
+
+``prefer_local``
+----------------
+Option to use the local or internal IP address when communication is not across different datacenters. **This option is commented out by default.**
+
+**Default value:** true
+
diff --git a/doc/source/configuration/cass_topo_file.rst b/doc/source/configuration/cass_topo_file.rst
new file mode 100644
index 0000000..264addc
--- /dev/null
+++ b/doc/source/configuration/cass_topo_file.rst
@@ -0,0 +1,48 @@
+.. _cassandra-topology:
+
+cassandra-topologies.properties file
+================================
+
+The ``PropertyFileSnitch`` :term:`snitch` option uses the ``cassandra-topologies.properties`` configuration file to determine which :term:`datacenters` and racks cluster nodes belong to. If other snitches are used, the
+:ref:cassandra_rackdc must be used. The snitch determines network topology (proximity by rack and datacenter) so that requests are routed efficiently and allows the database to distribute replicas evenly.
+
+Include every node in the cluster in the properties file, defining your datacenter names as in the keyspace definition. The datacenter and rack names are case-sensitive.
+
+The ``cassandra-topologies.properties`` file must be copied identically to every node in the cluster.
+
+
+===========================
+Example
+===========================
+This example uses three datacenters:
+
+.. code-block:: bash
+
+ # datacenter One
+
+ 175.56.12.105=DC1:RAC1
+ 175.50.13.200=DC1:RAC1
+ 175.54.35.197=DC1:RAC1
+
+ 120.53.24.101=DC1:RAC2
+ 120.55.16.200=DC1:RAC2
+ 120.57.102.103=DC1:RAC2
+
+ # datacenter Two
+
+ 110.56.12.120=DC2:RAC1
+ 110.50.13.201=DC2:RAC1
+ 110.54.35.184=DC2:RAC1
+
+ 50.33.23.120=DC2:RAC2
+ 50.45.14.220=DC2:RAC2
+ 50.17.10.203=DC2:RAC2
+
+ # datacenter Three
+
+ 172.106.12.120=DC3:RAC1
+ 172.106.12.121=DC3:RAC1
+ 172.106.12.122=DC3:RAC1
+
+ # default for unknown nodes
+ default =DC3:RAC1
diff --git a/doc/source/configuration/cass_yaml_file.rst b/doc/source/configuration/cass_yaml_file.rst
new file mode 100644
index 0000000..24e3be0
--- /dev/null
+++ b/doc/source/configuration/cass_yaml_file.rst
@@ -0,0 +1,2074 @@
+.. _cassandra-yaml:
+
+cassandra.yaml file configuration
+=================================
+
+``cluster_name``
+----------------
+The name of the cluster. This is mainly used to prevent machines in
+one logical cluster from joining another.
+
+*Default Value:* 'Test Cluster'
+
+``num_tokens``
+--------------
+
+This defines the number of tokens randomly assigned to this node on the ring
+The more tokens, relative to other nodes, the larger the proportion of data
+that this node will store. We recommend all nodes to have the same number
+of tokens assuming they have equal hardware capability.
+
+If you leave this unspecified, Cassandra will use the default of 1 token for legacy compatibility,
+and will use the initial_token as described below.
+
+Specifying initial_token will override this setting on the node's initial start,
+on subsequent starts, this setting will apply even if initial token is set.
+
+We recommend setting ``allocate_tokens_for_local_replication_factor`` in conjunction with this setting to ensure even allocation.
+
+*Default Value:* 256
+
+``allocate_tokens_for_keyspace``
+--------------------------------
+*This option is commented out by default.*
+
+Triggers automatic allocation of num_tokens tokens for this node. The allocation
+algorithm attempts to choose tokens in a way that optimizes replicated load over
+the nodes in the datacenter for the replica factor.
+
+The load assigned to each node will be close to proportional to its number of
+vnodes.
+
+Only supported with the Murmur3Partitioner.
+
+Replica factor is determined via the replication strategy used by the specified
+keyspace.
+
+We recommend using the ``allocate_tokens_for_local_replication_factor`` setting instead for operational simplicity.
+
+*Default Value:* KEYSPACE
+
+``allocate_tokens_for_local_replication_factor``
+------------------------------------------------
+*This option is commented out by default.*
+
+Tokens will be allocated based on this replication factor, regardless of keyspace or datacenter.
+
+*Default Value:* 3
+
+``initial_token``
+-----------------
+*This option is commented out by default.*
+
+initial_token allows you to specify tokens manually. While you can use it with
+vnodes (num_tokens > 1, above) -- in which case you should provide a
+comma-separated list -- it's primarily used when adding nodes to legacy clusters
+that do not have vnodes enabled.
+
+``hinted_handoff_enabled``
+--------------------------
+
+May either be "true" or "false" to enable globally
+
+*Default Value:* true
+
+``hinted_handoff_disabled_datacenters``
+---------------------------------------
+*This option is commented out by default.*
+
+When hinted_handoff_enabled is true, a black list of data centers that will not
+perform hinted handoff
+
+*Default Value (complex option)*::
+
+ # - DC1
+ # - DC2
+
+``max_hint_window_in_ms``
+-------------------------
+This defines the maximum amount of time a dead host will have hints
+generated. After it has been dead this long, new hints for it will not be
+created until it has been seen alive and gone down again.
+
+*Default Value:* 10800000 # 3 hours
+
+``hinted_handoff_throttle_in_kb``
+---------------------------------
+
+Maximum throttle in KBs per second, per delivery thread. This will be
+reduced proportionally to the number of nodes in the cluster. (If there
+are two nodes in the cluster, each delivery thread will use the maximum
+rate; if there are three, each will throttle to half of the maximum,
+since we expect two nodes to be delivering hints simultaneously.)
+
+*Default Value:* 1024
+
+``max_hints_delivery_threads``
+------------------------------
+
+Number of threads with which to deliver hints;
+Consider increasing this number when you have multi-dc deployments, since
+cross-dc handoff tends to be slower
+
+*Default Value:* 2
+
+``hints_directory``
+-------------------
+*This option is commented out by default.*
+
+Directory where Cassandra should store hints.
+If not set, the default directory is $CASSANDRA_HOME/data/hints.
+
+*Default Value:* /var/lib/cassandra/hints
+
+``hints_flush_period_in_ms``
+----------------------------
+
+How often hints should be flushed from the internal buffers to disk.
+Will *not* trigger fsync.
+
+*Default Value:* 10000
+
+``max_hints_file_size_in_mb``
+-----------------------------
+
+Maximum size for a single hints file, in megabytes.
+
+*Default Value:* 128
+
+``hints_compression``
+---------------------
+*This option is commented out by default.*
+
+Compression to apply to the hint files. If omitted, hints files
+will be written uncompressed. LZ4, Snappy, and Deflate compressors
+are supported.
+
+*Default Value (complex option)*::
+
+ # - class_name: LZ4Compressor
+ # parameters:
+ # -
+
+``batchlog_replay_throttle_in_kb``
+----------------------------------
+Maximum throttle in KBs per second, total. This will be
+reduced proportionally to the number of nodes in the cluster.
+
+*Default Value:* 1024
+
+``authenticator``
+-----------------
+
+Authentication backend, implementing IAuthenticator; used to identify users
+Out of the box, Cassandra provides org.apache.cassandra.auth.{AllowAllAuthenticator,
+PasswordAuthenticator}.
+
+- AllowAllAuthenticator performs no checks - set it to disable authentication.
+- PasswordAuthenticator relies on username/password pairs to authenticate
+ users. It keeps usernames and hashed passwords in system_auth.roles table.
+ Please increase system_auth keyspace replication factor if you use this authenticator.
+ If using PasswordAuthenticator, CassandraRoleManager must also be used (see below)
+
+*Default Value:* AllowAllAuthenticator
+
+``authorizer``
+--------------
+
+Authorization backend, implementing IAuthorizer; used to limit access/provide permissions
+Out of the box, Cassandra provides org.apache.cassandra.auth.{AllowAllAuthorizer,
+CassandraAuthorizer}.
+
+- AllowAllAuthorizer allows any action to any user - set it to disable authorization.
+- CassandraAuthorizer stores permissions in system_auth.role_permissions table. Please
+ increase system_auth keyspace replication factor if you use this authorizer.
+
+*Default Value:* AllowAllAuthorizer
+
+``role_manager``
+----------------
+
+Part of the Authentication & Authorization backend, implementing IRoleManager; used
+to maintain grants and memberships between roles.
+Out of the box, Cassandra provides org.apache.cassandra.auth.CassandraRoleManager,
+which stores role information in the system_auth keyspace. Most functions of the
+IRoleManager require an authenticated login, so unless the configured IAuthenticator
+actually implements authentication, most of this functionality will be unavailable.
+
+- CassandraRoleManager stores role data in the system_auth keyspace. Please
+ increase system_auth keyspace replication factor if you use this role manager.
+
+*Default Value:* CassandraRoleManager
+
+``network_authorizer``
+----------------------
+
+Network authorization backend, implementing INetworkAuthorizer; used to restrict user
+access to certain DCs
+Out of the box, Cassandra provides org.apache.cassandra.auth.{AllowAllNetworkAuthorizer,
+CassandraNetworkAuthorizer}.
+
+- AllowAllNetworkAuthorizer allows access to any DC to any user - set it to disable authorization.
+- CassandraNetworkAuthorizer stores permissions in system_auth.network_permissions table. Please
+ increase system_auth keyspace replication factor if you use this authorizer.
+
+*Default Value:* AllowAllNetworkAuthorizer
+
+``roles_validity_in_ms``
+------------------------
+
+Validity period for roles cache (fetching granted roles can be an expensive
+operation depending on the role manager, CassandraRoleManager is one example)
+Granted roles are cached for authenticated sessions in AuthenticatedUser and
+after the period specified here, become eligible for (async) reload.
+Defaults to 2000, set to 0 to disable caching entirely.
+Will be disabled automatically for AllowAllAuthenticator.
+
+*Default Value:* 2000
+
+``roles_update_interval_in_ms``
+-------------------------------
+*This option is commented out by default.*
+
+Refresh interval for roles cache (if enabled).
+After this interval, cache entries become eligible for refresh. Upon next
+access, an async reload is scheduled and the old value returned until it
+completes. If roles_validity_in_ms is non-zero, then this must be
+also.
+Defaults to the same value as roles_validity_in_ms.
+
+*Default Value:* 2000
+
+``permissions_validity_in_ms``
+------------------------------
+
+Validity period for permissions cache (fetching permissions can be an
+expensive operation depending on the authorizer, CassandraAuthorizer is
+one example). Defaults to 2000, set to 0 to disable.
+Will be disabled automatically for AllowAllAuthorizer.
+
+*Default Value:* 2000
+
+``permissions_update_interval_in_ms``
+-------------------------------------
+*This option is commented out by default.*
+
+Refresh interval for permissions cache (if enabled).
+After this interval, cache entries become eligible for refresh. Upon next
+access, an async reload is scheduled and the old value returned until it
+completes. If permissions_validity_in_ms is non-zero, then this must be
+also.
+Defaults to the same value as permissions_validity_in_ms.
+
+*Default Value:* 2000
+
+``credentials_validity_in_ms``
+------------------------------
+
+Validity period for credentials cache. This cache is tightly coupled to
+the provided PasswordAuthenticator implementation of IAuthenticator. If
+another IAuthenticator implementation is configured, this cache will not
+be automatically used and so the following settings will have no effect.
+Please note, credentials are cached in their encrypted form, so while
+activating this cache may reduce the number of queries made to the
+underlying table, it may not bring a significant reduction in the
+latency of individual authentication attempts.
+Defaults to 2000, set to 0 to disable credentials caching.
+
+*Default Value:* 2000
+
+``credentials_update_interval_in_ms``
+-------------------------------------
+*This option is commented out by default.*
+
+Refresh interval for credentials cache (if enabled).
+After this interval, cache entries become eligible for refresh. Upon next
+access, an async reload is scheduled and the old value returned until it
+completes. If credentials_validity_in_ms is non-zero, then this must be
+also.
+Defaults to the same value as credentials_validity_in_ms.
+
+*Default Value:* 2000
+
+``partitioner``
+---------------
+
+The partitioner is responsible for distributing groups of rows (by
+partition key) across nodes in the cluster. The partitioner can NOT be
+changed without reloading all data. If you are adding nodes or upgrading,
+you should set this to the same partitioner that you are currently using.
+
+The default partitioner is the Murmur3Partitioner. Older partitioners
+such as the RandomPartitioner, ByteOrderedPartitioner, and
+OrderPreservingPartitioner have been included for backward compatibility only.
+For new clusters, you should NOT change this value.
+
+
+*Default Value:* org.apache.cassandra.dht.Murmur3Partitioner
+
+``data_file_directories``
+-------------------------
+*This option is commented out by default.*
+
+Directories where Cassandra should store data on disk. If multiple
+directories are specified, Cassandra will spread data evenly across
+them by partitioning the token ranges.
+If not set, the default directory is $CASSANDRA_HOME/data/data.
+
+*Default Value (complex option)*::
+
+ # - /var/lib/cassandra/data
+
+``commitlog_directory``
+-----------------------
+*This option is commented out by default.*
+commit log. when running on magnetic HDD, this should be a
+separate spindle than the data directories.
+If not set, the default directory is $CASSANDRA_HOME/data/commitlog.
+
+*Default Value:* /var/lib/cassandra/commitlog
+
+``cdc_enabled``
+---------------
+
+Enable / disable CDC functionality on a per-node basis. This modifies the logic used
+for write path allocation rejection (standard: never reject. cdc: reject Mutation
+containing a CDC-enabled table if at space limit in cdc_raw_directory).
+
+*Default Value:* false
+
+``cdc_raw_directory``
+---------------------
+*This option is commented out by default.*
+
+CommitLogSegments are moved to this directory on flush if cdc_enabled: true and the
+segment contains mutations for a CDC-enabled table. This should be placed on a
+separate spindle than the data directories. If not set, the default directory is
+$CASSANDRA_HOME/data/cdc_raw.
+
+*Default Value:* /var/lib/cassandra/cdc_raw
+
+``disk_failure_policy``
+-----------------------
+
+Policy for data disk failures:
+
+die
+ shut down gossip and client transports and kill the JVM for any fs errors or
+ single-sstable errors, so the node can be replaced.
+
+stop_paranoid
+ shut down gossip and client transports even for single-sstable errors,
+ kill the JVM for errors during startup.
+
+stop
+ shut down gossip and client transports, leaving the node effectively dead, but
+ can still be inspected via JMX, kill the JVM for errors during startup.
+
+best_effort
+ stop using the failed disk and respond to requests based on
+ remaining available sstables. This means you WILL see obsolete
+ data at CL.ONE!
+
+ignore
+ ignore fatal errors and let requests fail, as in pre-1.2 Cassandra
+
+*Default Value:* stop
+
+``commit_failure_policy``
+-------------------------
+
+Policy for commit disk failures:
+
+die
+ shut down the node and kill the JVM, so the node can be replaced.
+
+stop
+ shut down the node, leaving the node effectively dead, but
+ can still be inspected via JMX.
+
+stop_commit
+ shutdown the commit log, letting writes collect but
+ continuing to service reads, as in pre-2.0.5 Cassandra
+
+ignore
+ ignore fatal errors and let the batches fail
+
+*Default Value:* stop
+
+``prepared_statements_cache_size_mb``
+-------------------------------------
+
+Maximum size of the native protocol prepared statement cache
+
+Valid values are either "auto" (omitting the value) or a value greater 0.
+
+Note that specifying a too large value will result in long running GCs and possbily
+out-of-memory errors. Keep the value at a small fraction of the heap.
+
+If you constantly see "prepared statements discarded in the last minute because
+cache limit reached" messages, the first step is to investigate the root cause
+of these messages and check whether prepared statements are used correctly -
+i.e. use bind markers for variable parts.
+
+Do only change the default value, if you really have more prepared statements than
+fit in the cache. In most cases it is not neccessary to change this value.
+Constantly re-preparing statements is a performance penalty.
+
+Default value ("auto") is 1/256th of the heap or 10MB, whichever is greater
+
+``key_cache_size_in_mb``
+------------------------
+
+Maximum size of the key cache in memory.
+
+Each key cache hit saves 1 seek and each row cache hit saves 2 seeks at the
+minimum, sometimes more. The key cache is fairly tiny for the amount of
+time it saves, so it's worthwhile to use it at large numbers.
+The row cache saves even more time, but must contain the entire row,
+so it is extremely space-intensive. It's best to only use the
+row cache if you have hot rows or static rows.
+
+NOTE: if you reduce the size, you may not get you hottest keys loaded on startup.
+
+Default value is empty to make it "auto" (min(5% of Heap (in MB), 100MB)). Set to 0 to disable key cache.
+
+``key_cache_save_period``
+-------------------------
+
+Duration in seconds after which Cassandra should
+save the key cache. Caches are saved to saved_caches_directory as
+specified in this configuration file.
+
+Saved caches greatly improve cold-start speeds, and is relatively cheap in
+terms of I/O for the key cache. Row cache saving is much more expensive and
+has limited use.
+
+Default is 14400 or 4 hours.
+
+*Default Value:* 14400
+
+``key_cache_keys_to_save``
+--------------------------
+*This option is commented out by default.*
+
+Number of keys from the key cache to save
+Disabled by default, meaning all keys are going to be saved
+
+*Default Value:* 100
+
+``row_cache_class_name``
+------------------------
+*This option is commented out by default.*
+
+Row cache implementation class name. Available implementations:
+
+org.apache.cassandra.cache.OHCProvider
+ Fully off-heap row cache implementation (default).
+
+org.apache.cassandra.cache.SerializingCacheProvider
+ This is the row cache implementation availabile
+ in previous releases of Cassandra.
+
+*Default Value:* org.apache.cassandra.cache.OHCProvider
+
+``row_cache_size_in_mb``
+------------------------
+
+Maximum size of the row cache in memory.
+Please note that OHC cache implementation requires some additional off-heap memory to manage
+the map structures and some in-flight memory during operations before/after cache entries can be
+accounted against the cache capacity. This overhead is usually small compared to the whole capacity.
+Do not specify more memory that the system can afford in the worst usual situation and leave some
+headroom for OS block level cache. Do never allow your system to swap.
+
+Default value is 0, to disable row caching.
+
+*Default Value:* 0
+
+``row_cache_save_period``
+-------------------------
+
+Duration in seconds after which Cassandra should save the row cache.
+Caches are saved to saved_caches_directory as specified in this configuration file.
+
+Saved caches greatly improve cold-start speeds, and is relatively cheap in
+terms of I/O for the key cache. Row cache saving is much more expensive and
+has limited use.
+
+Default is 0 to disable saving the row cache.
+
+*Default Value:* 0
+
+``row_cache_keys_to_save``
+--------------------------
+*This option is commented out by default.*
+
+Number of keys from the row cache to save.
+Specify 0 (which is the default), meaning all keys are going to be saved
+
+*Default Value:* 100
+
+``counter_cache_size_in_mb``
+----------------------------
+
+Maximum size of the counter cache in memory.
+
+Counter cache helps to reduce counter locks' contention for hot counter cells.
+In case of RF = 1 a counter cache hit will cause Cassandra to skip the read before
+write entirely. With RF > 1 a counter cache hit will still help to reduce the duration
+of the lock hold, helping with hot counter cell updates, but will not allow skipping
+the read entirely. Only the local (clock, count) tuple of a counter cell is kept
+in memory, not the whole counter, so it's relatively cheap.
+
+NOTE: if you reduce the size, you may not get you hottest keys loaded on startup.
+
+Default value is empty to make it "auto" (min(2.5% of Heap (in MB), 50MB)). Set to 0 to disable counter cache.
+NOTE: if you perform counter deletes and rely on low gcgs, you should disable the counter cache.
+
+``counter_cache_save_period``
+-----------------------------
+
+Duration in seconds after which Cassandra should
+save the counter cache (keys only). Caches are saved to saved_caches_directory as
+specified in this configuration file.
+
+Default is 7200 or 2 hours.
+
+*Default Value:* 7200
+
+``counter_cache_keys_to_save``
+------------------------------
+*This option is commented out by default.*
+
+Number of keys from the counter cache to save
+Disabled by default, meaning all keys are going to be saved
+
+*Default Value:* 100
+
+``saved_caches_directory``
+--------------------------
+*This option is commented out by default.*
+
+saved caches
+If not set, the default directory is $CASSANDRA_HOME/data/saved_caches.
+
+*Default Value:* /var/lib/cassandra/saved_caches
+
+``commitlog_sync_batch_window_in_ms``
+-------------------------------------
+*This option is commented out by default.*
+
+commitlog_sync may be either "periodic", "group", or "batch."
+
+When in batch mode, Cassandra won't ack writes until the commit log
+has been flushed to disk. Each incoming write will trigger the flush task.
+commitlog_sync_batch_window_in_ms is a deprecated value. Previously it had
+almost no value, and is being removed.
+
+
+*Default Value:* 2
+
+``commitlog_sync_group_window_in_ms``
+-------------------------------------
+*This option is commented out by default.*
+
+group mode is similar to batch mode, where Cassandra will not ack writes
+until the commit log has been flushed to disk. The difference is group
+mode will wait up to commitlog_sync_group_window_in_ms between flushes.
+
+
+*Default Value:* 1000
+
+``commitlog_sync``
+------------------
+
+the default option is "periodic" where writes may be acked immediately
+and the CommitLog is simply synced every commitlog_sync_period_in_ms
+milliseconds.
+
+*Default Value:* periodic
+
+``commitlog_sync_period_in_ms``
+-------------------------------
+
+*Default Value:* 10000
+
+``periodic_commitlog_sync_lag_block_in_ms``
+-------------------------------------------
+*This option is commented out by default.*
+
+When in periodic commitlog mode, the number of milliseconds to block writes
+while waiting for a slow disk flush to complete.
+
+``commitlog_segment_size_in_mb``
+--------------------------------
+
+The size of the individual commitlog file segments. A commitlog
+segment may be archived, deleted, or recycled once all the data
+in it (potentially from each columnfamily in the system) has been
+flushed to sstables.
+
+The default size is 32, which is almost always fine, but if you are
+archiving commitlog segments (see commitlog_archiving.properties),
+then you probably want a finer granularity of archiving; 8 or 16 MB
+is reasonable.
+Max mutation size is also configurable via max_mutation_size_in_kb setting in
+cassandra.yaml. The default is half the size commitlog_segment_size_in_mb * 1024.
+This should be positive and less than 2048.
+
+NOTE: If max_mutation_size_in_kb is set explicitly then commitlog_segment_size_in_mb must
+be set to at least twice the size of max_mutation_size_in_kb / 1024
+
+
+*Default Value:* 32
+
+``commitlog_compression``
+-------------------------
+*This option is commented out by default.*
+
+Compression to apply to the commit log. If omitted, the commit log
+will be written uncompressed. LZ4, Snappy, and Deflate compressors
+are supported.
+
+*Default Value (complex option)*::
+
+ # - class_name: LZ4Compressor
+ # parameters:
+ # -
+
+``table``
+---------
+*This option is commented out by default.*
+Compression to apply to SSTables as they flush for compressed tables.
+Note that tables without compression enabled do not respect this flag.
+
+As high ratio compressors like LZ4HC, Zstd, and Deflate can potentially
+block flushes for too long, the default is to flush with a known fast
+compressor in those cases. Options are:
+
+none : Flush without compressing blocks but while still doing checksums.
+fast : Flush with a fast compressor. If the table is already using a
+ fast compressor that compressor is used.
+
+*Default Value:* Always flush with the same compressor that the table uses. This
+
+``flush_compression``
+---------------------
+*This option is commented out by default.*
+ was the pre 4.0 behavior.
+
+
+*Default Value:* fast
+
+``seed_provider``
+-----------------
+
+any class that implements the SeedProvider interface and has a
+constructor that takes a Map<String, String> of parameters will do.
+
+*Default Value (complex option)*::
+
+ # Addresses of hosts that are deemed contact points.
+ # Cassandra nodes use this list of hosts to find each other and learn
+ # the topology of the ring. You must change this if you are running
+ # multiple nodes!
+ - class_name: org.apache.cassandra.locator.SimpleSeedProvider
+ parameters:
+ # seeds is actually a comma-delimited list of addresses.
+ # Ex: "<ip1>,<ip2>,<ip3>"
+ - seeds: "127.0.0.1:7000"
+
+``concurrent_reads``
+--------------------
+For workloads with more data than can fit in memory, Cassandra's
+bottleneck will be reads that need to fetch data from
+disk. "concurrent_reads" should be set to (16 * number_of_drives) in
+order to allow the operations to enqueue low enough in the stack
+that the OS and drives can reorder them. Same applies to
+"concurrent_counter_writes", since counter writes read the current
+values before incrementing and writing them back.
+
+On the other hand, since writes are almost never IO bound, the ideal
+number of "concurrent_writes" is dependent on the number of cores in
+your system; (8 * number_of_cores) is a good rule of thumb.
+
+*Default Value:* 32
+
+``concurrent_writes``
+---------------------
+
+*Default Value:* 32
+
+``concurrent_counter_writes``
+-----------------------------
+
+*Default Value:* 32
+
+``concurrent_materialized_view_writes``
+---------------------------------------
+
+For materialized view writes, as there is a read involved, so this should
+be limited by the less of concurrent reads or concurrent writes.
+
+*Default Value:* 32
+
+``file_cache_size_in_mb``
+-------------------------
+*This option is commented out by default.*
+
+Maximum memory to use for sstable chunk cache and buffer pooling.
+32MB of this are reserved for pooling buffers, the rest is used as an
+cache that holds uncompressed sstable chunks.
+Defaults to the smaller of 1/4 of heap or 512MB. This pool is allocated off-heap,
+so is in addition to the memory allocated for heap. The cache also has on-heap
+overhead which is roughly 128 bytes per chunk (i.e. 0.2% of the reserved size
+if the default 64k chunk size is used).
+Memory is only allocated when needed.
+
+*Default Value:* 512
+
+``buffer_pool_use_heap_if_exhausted``
+-------------------------------------
+*This option is commented out by default.*
+
+Flag indicating whether to allocate on or off heap when the sstable buffer
+pool is exhausted, that is when it has exceeded the maximum memory
+file_cache_size_in_mb, beyond which it will not cache buffers but allocate on request.
+
+
+*Default Value:* true
+
+``disk_optimization_strategy``
+------------------------------
+*This option is commented out by default.*
+
+The strategy for optimizing disk read
+Possible values are:
+ssd (for solid state disks, the default)
+spinning (for spinning disks)
+
+*Default Value:* ssd
+
+``memtable_heap_space_in_mb``
+-----------------------------
+*This option is commented out by default.*
+
+Total permitted memory to use for memtables. Cassandra will stop
+accepting writes when the limit is exceeded until a flush completes,
+and will trigger a flush based on memtable_cleanup_threshold
+If omitted, Cassandra will set both to 1/4 the size of the heap.
+
+*Default Value:* 2048
+
+``memtable_offheap_space_in_mb``
+--------------------------------
+*This option is commented out by default.*
+
+*Default Value:* 2048
+
+``memtable_cleanup_threshold``
+------------------------------
+*This option is commented out by default.*
+
+memtable_cleanup_threshold is deprecated. The default calculation
+is the only reasonable choice. See the comments on memtable_flush_writers
+for more information.
+
+Ratio of occupied non-flushing memtable size to total permitted size
+that will trigger a flush of the largest memtable. Larger mct will
+mean larger flushes and hence less compaction, but also less concurrent
+flush activity which can make it difficult to keep your disks fed
+under heavy write load.
+
+memtable_cleanup_threshold defaults to 1 / (memtable_flush_writers + 1)
+
+*Default Value:* 0.11
+
+``memtable_allocation_type``
+----------------------------
+
+Specify the way Cassandra allocates and manages memtable memory.
+Options are:
+
+heap_buffers
+ on heap nio buffers
+
+offheap_buffers
+ off heap (direct) nio buffers
+
+offheap_objects
+ off heap objects
+
+*Default Value:* heap_buffers
+
+``repair_session_space_in_mb``
+------------------------------
+*This option is commented out by default.*
+
+Limit memory usage for Merkle tree calculations during repairs. The default
+is 1/16th of the available heap. The main tradeoff is that smaller trees
+have less resolution, which can lead to over-streaming data. If you see heap
+pressure during repairs, consider lowering this, but you cannot go below
+one megabyte. If you see lots of over-streaming, consider raising
+this or using subrange repair.
+
+For more details see https://issues.apache.org/jira/browse/CASSANDRA-14096.
+
+
+``commitlog_total_space_in_mb``
+-------------------------------
+*This option is commented out by default.*
+
+Total space to use for commit logs on disk.
+
+If space gets above this value, Cassandra will flush every dirty CF
+in the oldest segment and remove it. So a small total commitlog space
+will tend to cause more flush activity on less-active columnfamilies.
+
+The default value is the smaller of 8192, and 1/4 of the total space
+of the commitlog volume.
+
+
+*Default Value:* 8192
+
+``memtable_flush_writers``
+--------------------------
+*This option is commented out by default.*
+
+This sets the number of memtable flush writer threads per disk
+as well as the total number of memtables that can be flushed concurrently.
+These are generally a combination of compute and IO bound.
+
+Memtable flushing is more CPU efficient than memtable ingest and a single thread
+can keep up with the ingest rate of a whole server on a single fast disk
+until it temporarily becomes IO bound under contention typically with compaction.
+At that point you need multiple flush threads. At some point in the future
+it may become CPU bound all the time.
+
+You can tell if flushing is falling behind using the MemtablePool.BlockedOnAllocation
+metric which should be 0, but will be non-zero if threads are blocked waiting on flushing
+to free memory.
+
+memtable_flush_writers defaults to two for a single data directory.
+This means that two memtables can be flushed concurrently to the single data directory.
+If you have multiple data directories the default is one memtable flushing at a time
+but the flush will use a thread per data directory so you will get two or more writers.
+
+Two is generally enough to flush on a fast disk [array] mounted as a single data directory.
+Adding more flush writers will result in smaller more frequent flushes that introduce more
+compaction overhead.
+
+There is a direct tradeoff between number of memtables that can be flushed concurrently
+and flush size and frequency. More is not better you just need enough flush writers
+to never stall waiting for flushing to free memory.
+
+
+*Default Value:* 2
+
+``cdc_total_space_in_mb``
+-------------------------
+*This option is commented out by default.*
+
+Total space to use for change-data-capture logs on disk.
+
+If space gets above this value, Cassandra will throw WriteTimeoutException
+on Mutations including tables with CDC enabled. A CDCCompactor is responsible
+for parsing the raw CDC logs and deleting them when parsing is completed.
+
+The default value is the min of 4096 mb and 1/8th of the total space
+of the drive where cdc_raw_directory resides.
+
+*Default Value:* 4096
+
+``cdc_free_space_check_interval_ms``
+------------------------------------
+*This option is commented out by default.*
+
+When we hit our cdc_raw limit and the CDCCompactor is either running behind
+or experiencing backpressure, we check at the following interval to see if any
+new space for cdc-tracked tables has been made available. Default to 250ms
+
+*Default Value:* 250
+
+``index_summary_capacity_in_mb``
+--------------------------------
+
+A fixed memory pool size in MB for for SSTable index summaries. If left
+empty, this will default to 5% of the heap size. If the memory usage of
+all index summaries exceeds this limit, SSTables with low read rates will
+shrink their index summaries in order to meet this limit. However, this
+is a best-effort process. In extreme conditions Cassandra may need to use
+more than this amount of memory.
+
+``index_summary_resize_interval_in_minutes``
+--------------------------------------------
+
+How frequently index summaries should be resampled. This is done
+periodically to redistribute memory from the fixed-size pool to sstables
+proportional their recent read rates. Setting to -1 will disable this
+process, leaving existing index summaries at their current sampling level.
+
+*Default Value:* 60
+
+``trickle_fsync``
+-----------------
+
+Whether to, when doing sequential writing, fsync() at intervals in
+order to force the operating system to flush the dirty
+buffers. Enable this to avoid sudden dirty buffer flushing from
+impacting read latencies. Almost always a good idea on SSDs; not
+necessarily on platters.
+
+*Default Value:* false
+
+``trickle_fsync_interval_in_kb``
+--------------------------------
+
+*Default Value:* 10240
+
+``storage_port``
+----------------
+
+TCP port, for commands and data
+For security reasons, you should not expose this port to the internet. Firewall it if needed.
+
+*Default Value:* 7000
+
+``ssl_storage_port``
+--------------------
+
+SSL port, for legacy encrypted communication. This property is unused unless enabled in
+server_encryption_options (see below). As of cassandra 4.0, this property is deprecated
+as a single port can be used for either/both secure and insecure connections.
+For security reasons, you should not expose this port to the internet. Firewall it if needed.
+
+*Default Value:* 7001
+
+``listen_address``
+------------------
+
+Address or interface to bind to and tell other Cassandra nodes to connect to.
+You _must_ change this if you want multiple nodes to be able to communicate!
+
+Set listen_address OR listen_interface, not both.
+
+Leaving it blank leaves it up to InetAddress.getLocalHost(). This
+will always do the Right Thing _if_ the node is properly configured
+(hostname, name resolution, etc), and the Right Thing is to use the
+address associated with the hostname (it might not be).
+
+Setting listen_address to 0.0.0.0 is always wrong.
+
+
+*Default Value:* localhost
+
+``listen_interface``
+--------------------
+*This option is commented out by default.*
+
+Set listen_address OR listen_interface, not both. Interfaces must correspond
+to a single address, IP aliasing is not supported.
+
+*Default Value:* eth0
+
+``listen_interface_prefer_ipv6``
+--------------------------------
+*This option is commented out by default.*
+
+If you choose to specify the interface by name and the interface has an ipv4 and an ipv6 address
+you can specify which should be chosen using listen_interface_prefer_ipv6. If false the first ipv4
+address will be used. If true the first ipv6 address will be used. Defaults to false preferring
+ipv4. If there is only one address it will be selected regardless of ipv4/ipv6.
+
+*Default Value:* false
+
+``broadcast_address``
+---------------------
+*This option is commented out by default.*
+
+Address to broadcast to other Cassandra nodes
+Leaving this blank will set it to the same value as listen_address
+
+*Default Value:* 1.2.3.4
+
+``listen_on_broadcast_address``
+-------------------------------
+*This option is commented out by default.*
+
+When using multiple physical network interfaces, set this
+to true to listen on broadcast_address in addition to
+the listen_address, allowing nodes to communicate in both
+interfaces.
+Ignore this property if the network configuration automatically
+routes between the public and private networks such as EC2.
+
+*Default Value:* false
+
+``internode_authenticator``
+---------------------------
+*This option is commented out by default.*
+
+Internode authentication backend, implementing IInternodeAuthenticator;
+used to allow/disallow connections from peer nodes.
+
+*Default Value:* org.apache.cassandra.auth.AllowAllInternodeAuthenticator
+
+``start_native_transport``
+--------------------------
+
+Whether to start the native transport server.
+The address on which the native transport is bound is defined by rpc_address.
+
+*Default Value:* true
+
+``native_transport_port``
+-------------------------
+port for the CQL native transport to listen for clients on
+For security reasons, you should not expose this port to the internet. Firewall it if needed.
+
+*Default Value:* 9042
+
+``native_transport_port_ssl``
+-----------------------------
+*This option is commented out by default.*
+Enabling native transport encryption in client_encryption_options allows you to either use
+encryption for the standard port or to use a dedicated, additional port along with the unencrypted
+standard native_transport_port.
+Enabling client encryption and keeping native_transport_port_ssl disabled will use encryption
+for native_transport_port. Setting native_transport_port_ssl to a different value
+from native_transport_port will use encryption for native_transport_port_ssl while
+keeping native_transport_port unencrypted.
+
+*Default Value:* 9142
+
+``native_transport_max_threads``
+--------------------------------
+*This option is commented out by default.*
+The maximum threads for handling requests (note that idle threads are stopped
+after 30 seconds so there is not corresponding minimum setting).
+
+*Default Value:* 128
+
+``native_transport_max_frame_size_in_mb``
+-----------------------------------------
+*This option is commented out by default.*
+
+The maximum size of allowed frame. Frame (requests) larger than this will
+be rejected as invalid. The default is 256MB. If you're changing this parameter,
+you may want to adjust max_value_size_in_mb accordingly. This should be positive and less than 2048.
+
+*Default Value:* 256
+
+``native_transport_frame_block_size_in_kb``
+-------------------------------------------
+*This option is commented out by default.*
+
+If checksumming is enabled as a protocol option, denotes the size of the chunks into which frame
+are bodies will be broken and checksummed.
+
+*Default Value:* 32
+
+``native_transport_max_concurrent_connections``
+-----------------------------------------------
+*This option is commented out by default.*
+
+The maximum number of concurrent client connections.
+The default is -1, which means unlimited.
+
+*Default Value:* -1
+
+``native_transport_max_concurrent_connections_per_ip``
+------------------------------------------------------
+*This option is commented out by default.*
+
+The maximum number of concurrent client connections per source ip.
+The default is -1, which means unlimited.
+
+*Default Value:* -1
+
+``native_transport_allow_older_protocols``
+------------------------------------------
+
+Controls whether Cassandra honors older, yet currently supported, protocol versions.
+The default is true, which means all supported protocols will be honored.
+
+*Default Value:* true
+
+``native_transport_idle_timeout_in_ms``
+---------------------------------------
+*This option is commented out by default.*
+
+Controls when idle client connections are closed. Idle connections are ones that had neither reads
+nor writes for a time period.
+
+Clients may implement heartbeats by sending OPTIONS native protocol message after a timeout, which
+will reset idle timeout timer on the server side. To close idle client connections, corresponding
+values for heartbeat intervals have to be set on the client side.
+
+Idle connection timeouts are disabled by default.
+
+*Default Value:* 60000
+
+``rpc_address``
+---------------
+
+The address or interface to bind the native transport server to.
+
+Set rpc_address OR rpc_interface, not both.
+
+Leaving rpc_address blank has the same effect as on listen_address
+(i.e. it will be based on the configured hostname of the node).
+
+Note that unlike listen_address, you can specify 0.0.0.0, but you must also
+set broadcast_rpc_address to a value other than 0.0.0.0.
+
+For security reasons, you should not expose this port to the internet. Firewall it if needed.
+
+*Default Value:* localhost
+
+``rpc_interface``
+-----------------
+*This option is commented out by default.*
+
+Set rpc_address OR rpc_interface, not both. Interfaces must correspond
+to a single address, IP aliasing is not supported.
+
+*Default Value:* eth1
+
+``rpc_interface_prefer_ipv6``
+-----------------------------
+*This option is commented out by default.*
+
+If you choose to specify the interface by name and the interface has an ipv4 and an ipv6 address
+you can specify which should be chosen using rpc_interface_prefer_ipv6. If false the first ipv4
+address will be used. If true the first ipv6 address will be used. Defaults to false preferring
+ipv4. If there is only one address it will be selected regardless of ipv4/ipv6.
+
+*Default Value:* false
+
+``broadcast_rpc_address``
+-------------------------
+*This option is commented out by default.*
+
+RPC address to broadcast to drivers and other Cassandra nodes. This cannot
+be set to 0.0.0.0. If left blank, this will be set to the value of
+rpc_address. If rpc_address is set to 0.0.0.0, broadcast_rpc_address must
+be set.
+
+*Default Value:* 1.2.3.4
+
+``rpc_keepalive``
+-----------------
+
+enable or disable keepalive on rpc/native connections
+
+*Default Value:* true
+
+``internode_send_buff_size_in_bytes``
+-------------------------------------
+*This option is commented out by default.*
+
+Uncomment to set socket buffer size for internode communication
+Note that when setting this, the buffer size is limited by net.core.wmem_max
+and when not setting it it is defined by net.ipv4.tcp_wmem
+See also:
+/proc/sys/net/core/wmem_max
+/proc/sys/net/core/rmem_max
+/proc/sys/net/ipv4/tcp_wmem
+/proc/sys/net/ipv4/tcp_wmem
+and 'man tcp'
+
+``internode_recv_buff_size_in_bytes``
+-------------------------------------
+*This option is commented out by default.*
+
+Uncomment to set socket buffer size for internode communication
+Note that when setting this, the buffer size is limited by net.core.wmem_max
+and when not setting it it is defined by net.ipv4.tcp_wmem
+
+``incremental_backups``
+-----------------------
+
+Set to true to have Cassandra create a hard link to each sstable
+flushed or streamed locally in a backups/ subdirectory of the
+keyspace data. Removing these links is the operator's
+responsibility.
+
+*Default Value:* false
+
+``snapshot_before_compaction``
+------------------------------
+
+Whether or not to take a snapshot before each compaction. Be
+careful using this option, since Cassandra won't clean up the
+snapshots for you. Mostly useful if you're paranoid when there
+is a data format change.
+
+*Default Value:* false
+
+``auto_snapshot``
+-----------------
+
+Whether or not a snapshot is taken of the data before keyspace truncation
+or dropping of column families. The STRONGLY advised default of true
+should be used to provide data safety. If you set this flag to false, you will
+lose data on truncation or drop.
+
+*Default Value:* true
+
+``column_index_size_in_kb``
+---------------------------
+
+Granularity of the collation index of rows within a partition.
+Increase if your rows are large, or if you have a very large
+number of rows per partition. The competing goals are these:
+
+- a smaller granularity means more index entries are generated
+ and looking up rows withing the partition by collation column
+ is faster
+- but, Cassandra will keep the collation index in memory for hot
+ rows (as part of the key cache), so a larger granularity means
+ you can cache more hot rows
+
+*Default Value:* 64
+
+``column_index_cache_size_in_kb``
+---------------------------------
+
+Per sstable indexed key cache entries (the collation index in memory
+mentioned above) exceeding this size will not be held on heap.
+This means that only partition information is held on heap and the
+index entries are read from disk.
+
+Note that this size refers to the size of the
+serialized index information and not the size of the partition.
+
+*Default Value:* 2
+
+``concurrent_compactors``
+-------------------------
+*This option is commented out by default.*
+
+Number of simultaneous compactions to allow, NOT including
+validation "compactions" for anti-entropy repair. Simultaneous
+compactions can help preserve read performance in a mixed read/write
+workload, by mitigating the tendency of small sstables to accumulate
+during a single long running compactions. The default is usually
+fine and if you experience problems with compaction running too
+slowly or too fast, you should look at
+compaction_throughput_mb_per_sec first.
+
+concurrent_compactors defaults to the smaller of (number of disks,
+number of cores), with a minimum of 2 and a maximum of 8.
+
+If your data directories are backed by SSD, you should increase this
+to the number of cores.
+
+*Default Value:* 1
+
+``concurrent_validations``
+--------------------------
+*This option is commented out by default.*
+
+Number of simultaneous repair validations to allow. Default is unbounded
+Values less than one are interpreted as unbounded (the default)
+
+*Default Value:* 0
+
+``concurrent_materialized_view_builders``
+-----------------------------------------
+
+Number of simultaneous materialized view builder tasks to allow.
+
+*Default Value:* 1
+
+``compaction_throughput_mb_per_sec``
+------------------------------------
+
+Throttles compaction to the given total throughput across the entire
+system. The faster you insert data, the faster you need to compact in
+order to keep the sstable count down, but in general, setting this to
+16 to 32 times the rate you are inserting data is more than sufficient.
+Setting this to 0 disables throttling. Note that this account for all types
+of compaction, including validation compaction.
+
+*Default Value:* 16
+
+``sstable_preemptive_open_interval_in_mb``
+------------------------------------------
+
+When compacting, the replacement sstable(s) can be opened before they
+are completely written, and used in place of the prior sstables for
+any range that has been written. This helps to smoothly transfer reads
+between the sstables, reducing page cache churn and keeping hot rows hot
+
+*Default Value:* 50
+
+``stream_entire_sstables``
+--------------------------
+*This option is commented out by default.*
+
+When enabled, permits Cassandra to zero-copy stream entire eligible
+SSTables between nodes, including every component.
+This speeds up the network transfer significantly subject to
+throttling specified by stream_throughput_outbound_megabits_per_sec.
+Enabling this will reduce the GC pressure on sending and receiving node.
+When unset, the default is enabled. While this feature tries to keep the
+disks balanced, it cannot guarantee it. This feature will be automatically
+disabled if internode encryption is enabled. Currently this can be used with
+Leveled Compaction. Once CASSANDRA-14586 is fixed other compaction strategies
+will benefit as well when used in combination with CASSANDRA-6696.
+
+*Default Value:* true
+
+``stream_throughput_outbound_megabits_per_sec``
+-----------------------------------------------
+*This option is commented out by default.*
+
+Throttles all outbound streaming file transfers on this node to the
+given total throughput in Mbps. This is necessary because Cassandra does
+mostly sequential IO when streaming data during bootstrap or repair, which
+can lead to saturating the network connection and degrading rpc performance.
+When unset, the default is 200 Mbps or 25 MB/s.
+
+*Default Value:* 200
+
+``inter_dc_stream_throughput_outbound_megabits_per_sec``
+--------------------------------------------------------
+*This option is commented out by default.*
+
+Throttles all streaming file transfer between the datacenters,
+this setting allows users to throttle inter dc stream throughput in addition
+to throttling all network stream traffic as configured with
+stream_throughput_outbound_megabits_per_sec
+When unset, the default is 200 Mbps or 25 MB/s
+
+*Default Value:* 200
+
+``read_request_timeout_in_ms``
+------------------------------
+
+How long the coordinator should wait for read operations to complete.
+Lowest acceptable value is 10 ms.
+
+*Default Value:* 5000
+
+``range_request_timeout_in_ms``
+-------------------------------
+How long the coordinator should wait for seq or index scans to complete.
+Lowest acceptable value is 10 ms.
+
+*Default Value:* 10000
+
+``write_request_timeout_in_ms``
+-------------------------------
+How long the coordinator should wait for writes to complete.
+Lowest acceptable value is 10 ms.
+
+*Default Value:* 2000
+
+``counter_write_request_timeout_in_ms``
+---------------------------------------
+How long the coordinator should wait for counter writes to complete.
+Lowest acceptable value is 10 ms.
+
+*Default Value:* 5000
+
+``cas_contention_timeout_in_ms``
+--------------------------------
+How long a coordinator should continue to retry a CAS operation
+that contends with other proposals for the same row.
+Lowest acceptable value is 10 ms.
+
+*Default Value:* 1000
+
+``truncate_request_timeout_in_ms``
+----------------------------------
+How long the coordinator should wait for truncates to complete
+(This can be much longer, because unless auto_snapshot is disabled
+we need to flush first so we can snapshot before removing the data.)
+Lowest acceptable value is 10 ms.
+
+*Default Value:* 60000
+
+``request_timeout_in_ms``
+-------------------------
+The default timeout for other, miscellaneous operations.
+Lowest acceptable value is 10 ms.
+
+*Default Value:* 10000
+
+``internode_application_send_queue_capacity_in_bytes``
+------------------------------------------------------
+*This option is commented out by default.*
+
+Defensive settings for protecting Cassandra from true network partitions.
+See (CASSANDRA-14358) for details.
+
+The amount of time to wait for internode tcp connections to establish.
+internode_tcp_connect_timeout_in_ms = 2000
+
+The amount of time unacknowledged data is allowed on a connection before we throw out the connection
+Note this is only supported on Linux + epoll, and it appears to behave oddly above a setting of 30000
+(it takes much longer than 30s) as of Linux 4.12. If you want something that high set this to 0
+which picks up the OS default and configure the net.ipv4.tcp_retries2 sysctl to be ~8.
+internode_tcp_user_timeout_in_ms = 30000
+
+The maximum continuous period a connection may be unwritable in application space
+internode_application_timeout_in_ms = 30000
+
+Global, per-endpoint and per-connection limits imposed on messages queued for delivery to other nodes
+and waiting to be processed on arrival from other nodes in the cluster. These limits are applied to the on-wire
+size of the message being sent or received.
+
+The basic per-link limit is consumed in isolation before any endpoint or global limit is imposed.
+Each node-pair has three links: urgent, small and large. So any given node may have a maximum of
+N*3*(internode_application_send_queue_capacity_in_bytes+internode_application_receive_queue_capacity_in_bytes)
+messages queued without any coordination between them although in practice, with token-aware routing, only RF*tokens
+nodes should need to communicate with significant bandwidth.
+
+The per-endpoint limit is imposed on all messages exceeding the per-link limit, simultaneously with the global limit,
+on all links to or from a single node in the cluster.
+The global limit is imposed on all messages exceeding the per-link limit, simultaneously with the per-endpoint limit,
+on all links to or from any node in the cluster.
+
+
+*Default Value:* 4194304 #4MiB
+
+``internode_application_send_queue_reserve_endpoint_capacity_in_bytes``
+-----------------------------------------------------------------------
+*This option is commented out by default.*
+
+*Default Value:* 134217728 #128MiB
+
+``internode_application_send_queue_reserve_global_capacity_in_bytes``
+---------------------------------------------------------------------
+*This option is commented out by default.*
+
+*Default Value:* 536870912 #512MiB
+
+``internode_application_receive_queue_capacity_in_bytes``
+---------------------------------------------------------
+*This option is commented out by default.*
+
+*Default Value:* 4194304 #4MiB
+
+``internode_application_receive_queue_reserve_endpoint_capacity_in_bytes``
+--------------------------------------------------------------------------
+*This option is commented out by default.*
+
+*Default Value:* 134217728 #128MiB
+
+``internode_application_receive_queue_reserve_global_capacity_in_bytes``
+------------------------------------------------------------------------
+*This option is commented out by default.*
+
+*Default Value:* 536870912 #512MiB
+
+``slow_query_log_timeout_in_ms``
+--------------------------------
+
+
+How long before a node logs slow queries. Select queries that take longer than
+this timeout to execute, will generate an aggregated log message, so that slow queries
+can be identified. Set this value to zero to disable slow query logging.
+
+*Default Value:* 500
+
+``cross_node_timeout``
+----------------------
+*This option is commented out by default.*
+
+Enable operation timeout information exchange between nodes to accurately
+measure request timeouts. If disabled, replicas will assume that requests
+were forwarded to them instantly by the coordinator, which means that
+under overload conditions we will waste that much extra time processing
+already-timed-out requests.
+
+Warning: It is generally assumed that users have setup NTP on their clusters, and that clocks are modestly in sync,
+since this is a requirement for general correctness of last write wins.
+
+*Default Value:* true
+
+``streaming_keep_alive_period_in_secs``
+---------------------------------------
+*This option is commented out by default.*
+
+Set keep-alive period for streaming
+This node will send a keep-alive message periodically with this period.
+If the node does not receive a keep-alive message from the peer for
+2 keep-alive cycles the stream session times out and fail
+Default value is 300s (5 minutes), which means stalled stream
+times out in 10 minutes by default
+
+*Default Value:* 300
+
+``streaming_connections_per_host``
+----------------------------------
+*This option is commented out by default.*
+
+Limit number of connections per host for streaming
+Increase this when you notice that joins are CPU-bound rather that network
+bound (for example a few nodes with big files).
+
+*Default Value:* 1
+
+``phi_convict_threshold``
+-------------------------
+*This option is commented out by default.*
+
+
+phi value that must be reached for a host to be marked down.
+most users should never need to adjust this.
+
+*Default Value:* 8
+
+``endpoint_snitch``
+-------------------
+
+endpoint_snitch -- Set this to a class that implements
+IEndpointSnitch. The snitch has two functions:
+
+- it teaches Cassandra enough about your network topology to route
+ requests efficiently
+- it allows Cassandra to spread replicas around your cluster to avoid
+ correlated failures. It does this by grouping machines into
+ "datacenters" and "racks." Cassandra will do its best not to have
+ more than one replica on the same "rack" (which may not actually
+ be a physical location)
+
+CASSANDRA WILL NOT ALLOW YOU TO SWITCH TO AN INCOMPATIBLE SNITCH
+ONCE DATA IS INSERTED INTO THE CLUSTER. This would cause data loss.
+This means that if you start with the default SimpleSnitch, which
+locates every node on "rack1" in "datacenter1", your only options
+if you need to add another datacenter are GossipingPropertyFileSnitch
+(and the older PFS). From there, if you want to migrate to an
+incompatible snitch like Ec2Snitch you can do it by adding new nodes
+under Ec2Snitch (which will locate them in a new "datacenter") and
+decommissioning the old ones.
+
+Out of the box, Cassandra provides:
+
+SimpleSnitch:
+ Treats Strategy order as proximity. This can improve cache
+ locality when disabling read repair. Only appropriate for
+ single-datacenter deployments.
+
+GossipingPropertyFileSnitch
+ This should be your go-to snitch for production use. The rack
+ and datacenter for the local node are defined in
+ cassandra-rackdc.properties and propagated to other nodes via
+ gossip. If cassandra-topology.properties exists, it is used as a
+ fallback, allowing migration from the PropertyFileSnitch.
+
+PropertyFileSnitch:
+ Proximity is determined by rack and data center, which are
+ explicitly configured in cassandra-topology.properties.
+
+Ec2Snitch:
+ Appropriate for EC2 deployments in a single Region. Loads Region
+ and Availability Zone information from the EC2 API. The Region is
+ treated as the datacenter, and the Availability Zone as the rack.
+ Only private IPs are used, so this will not work across multiple
+ Regions.
+
+Ec2MultiRegionSnitch:
+ Uses public IPs as broadcast_address to allow cross-region
+ connectivity. (Thus, you should set seed addresses to the public
+ IP as well.) You will need to open the storage_port or
+ ssl_storage_port on the public IP firewall. (For intra-Region
+ traffic, Cassandra will switch to the private IP after
+ establishing a connection.)
+
+RackInferringSnitch:
+ Proximity is determined by rack and data center, which are
+ assumed to correspond to the 3rd and 2nd octet of each node's IP
+ address, respectively. Unless this happens to match your
+ deployment conventions, this is best used as an example of
+ writing a custom Snitch class and is provided in that spirit.
+
+You can use a custom Snitch by setting this to the full class name
+of the snitch, which will be assumed to be on your classpath.
+
+*Default Value:* SimpleSnitch
+
+``dynamic_snitch_update_interval_in_ms``
+----------------------------------------
+
+controls how often to perform the more expensive part of host score
+calculation
+
+*Default Value:* 100
+
+``dynamic_snitch_reset_interval_in_ms``
+---------------------------------------
+controls how often to reset all host scores, allowing a bad host to
+possibly recover
+
+*Default Value:* 600000
+
+``dynamic_snitch_badness_threshold``
+------------------------------------
+if set greater than zero, this will allow
+'pinning' of replicas to hosts in order to increase cache capacity.
+The badness threshold will control how much worse the pinned host has to be
+before the dynamic snitch will prefer other replicas over it. This is
+expressed as a double which represents a percentage. Thus, a value of
+0.2 means Cassandra would continue to prefer the static snitch values
+until the pinned host was 20% worse than the fastest.
+
+*Default Value:* 0.1
+
+``server_encryption_options``
+-----------------------------
+
+Enable or disable inter-node encryption
+JVM and netty defaults for supported SSL socket protocols and cipher suites can
+be replaced using custom encryption options. This is not recommended
+unless you have policies in place that dictate certain settings, or
+need to disable vulnerable ciphers or protocols in case the JVM cannot
+be updated.
+FIPS compliant settings can be configured at JVM level and should not
+involve changing encryption settings here:
+https://docs.oracle.com/javase/8/docs/technotes/guides/security/jsse/FIPS.html
+
+*NOTE* No custom encryption options are enabled at the moment
+The available internode options are : all, none, dc, rack
+If set to dc cassandra will encrypt the traffic between the DCs
+If set to rack cassandra will encrypt the traffic between the racks
+
+The passwords used in these options must match the passwords used when generating
+the keystore and truststore. For instructions on generating these files, see:
+http://download.oracle.com/javase/8/docs/technotes/guides/security/jsse/JSSERefGuide.html#CreateKeystore
+
+
+*Default Value (complex option)*::
+
+ # set to true for allowing secure incoming connections
+ enabled: false
+ # If enabled and optional are both set to true, encrypted and unencrypted connections are handled on the storage_port
+ optional: false
+ # if enabled, will open up an encrypted listening socket on ssl_storage_port. Should be used
+ # during upgrade to 4.0; otherwise, set to false.
+ enable_legacy_ssl_storage_port: false
+ # on outbound connections, determine which type of peers to securely connect to. 'enabled' must be set to true.
+ internode_encryption: none
+ keystore: conf/.keystore
+ keystore_password: cassandra
+ truststore: conf/.truststore
+ truststore_password: cassandra
+ # More advanced defaults below:
+ # protocol: TLS
+ # store_type: JKS
+ # cipher_suites: [TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_DHE_RSA_WITH_AES_128_CBC_SHA,TLS_DHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA]
+ # require_client_auth: false
+ # require_endpoint_verification: false
+
+``client_encryption_options``
+-----------------------------
+enable or disable client-to-server encryption.
+
+*Default Value (complex option)*::
+
+ enabled: false
+ # If enabled and optional is set to true encrypted and unencrypted connections are handled.
+ optional: false
+ keystore: conf/.keystore
+ keystore_password: cassandra
+ # require_client_auth: false
+ # Set trustore and truststore_password if require_client_auth is true
+ # truststore: conf/.truststore
+ # truststore_password: cassandra
+ # More advanced defaults below:
+ # protocol: TLS
+ # store_type: JKS
+ # cipher_suites: [TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_DHE_RSA_WITH_AES_128_CBC_SHA,TLS_DHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA]
+
+``internode_compression``
+-------------------------
+internode_compression controls whether traffic between nodes is
+compressed.
+Can be:
+
+all
+ all traffic is compressed
+
+dc
+ traffic between different datacenters is compressed
+
+none
+ nothing is compressed.
+
+*Default Value:* dc
+
+``inter_dc_tcp_nodelay``
+------------------------
+
+Enable or disable tcp_nodelay for inter-dc communication.
+Disabling it will result in larger (but fewer) network packets being sent,
+reducing overhead from the TCP protocol itself, at the cost of increasing
+latency if you block for cross-datacenter responses.
+
+*Default Value:* false
+
+``tracetype_query_ttl``
+-----------------------
+
+TTL for different trace types used during logging of the repair process.
+
+*Default Value:* 86400
+
+``tracetype_repair_ttl``
+------------------------
+
+*Default Value:* 604800
+
+``enable_user_defined_functions``
+---------------------------------
+
+If unset, all GC Pauses greater than gc_log_threshold_in_ms will log at
+INFO level
+UDFs (user defined functions) are disabled by default.
+As of Cassandra 3.0 there is a sandbox in place that should prevent execution of evil code.
+
+*Default Value:* false
+
+``enable_scripted_user_defined_functions``
+------------------------------------------
+
+Enables scripted UDFs (JavaScript UDFs).
+Java UDFs are always enabled, if enable_user_defined_functions is true.
+Enable this option to be able to use UDFs with "language javascript" or any custom JSR-223 provider.
+This option has no effect, if enable_user_defined_functions is false.
+
+*Default Value:* false
+
+``windows_timer_interval``
+--------------------------
+
+The default Windows kernel timer and scheduling resolution is 15.6ms for power conservation.
+Lowering this value on Windows can provide much tighter latency and better throughput, however
+some virtualized environments may see a negative performance impact from changing this setting
+below their system default. The sysinternals 'clockres' tool can confirm your system's default
+setting.
+
+*Default Value:* 1
+
+``transparent_data_encryption_options``
+---------------------------------------
+
+
+Enables encrypting data at-rest (on disk). Different key providers can be plugged in, but the default reads from
+a JCE-style keystore. A single keystore can hold multiple keys, but the one referenced by
+the "key_alias" is the only key that will be used for encrypt opertaions; previously used keys
+can still (and should!) be in the keystore and will be used on decrypt operations
+(to handle the case of key rotation).
+
+It is strongly recommended to download and install Java Cryptography Extension (JCE)
+Unlimited Strength Jurisdiction Policy Files for your version of the JDK.
+(current link: http://www.oracle.com/technetwork/java/javase/downloads/jce8-download-2133166.html)
+
+Currently, only the following file types are supported for transparent data encryption, although
+more are coming in future cassandra releases: commitlog, hints
+
+*Default Value (complex option)*::
+
+ enabled: false
+ chunk_length_kb: 64
+ cipher: AES/CBC/PKCS5Padding
+ key_alias: testing:1
+ # CBC IV length for AES needs to be 16 bytes (which is also the default size)
+ # iv_length: 16
+ key_provider:
+ - class_name: org.apache.cassandra.security.JKSKeyProvider
+ parameters:
+ - keystore: conf/.keystore
+ keystore_password: cassandra
+ store_type: JCEKS
+ key_password: cassandra
+
+``tombstone_warn_threshold``
+----------------------------
+
+####################
+SAFETY THRESHOLDS #
+####################
+
+When executing a scan, within or across a partition, we need to keep the
+tombstones seen in memory so we can return them to the coordinator, which
+will use them to make sure other replicas also know about the deleted rows.
+With workloads that generate a lot of tombstones, this can cause performance
+problems and even exaust the server heap.
+(http://www.datastax.com/dev/blog/cassandra-anti-patterns-queues-and-queue-like-datasets)
+Adjust the thresholds here if you understand the dangers and want to
+scan more tombstones anyway. These thresholds may also be adjusted at runtime
+using the StorageService mbean.
+
+*Default Value:* 1000
+
+``tombstone_failure_threshold``
+-------------------------------
+
+*Default Value:* 100000
+
+``batch_size_warn_threshold_in_kb``
+-----------------------------------
+
+Log WARN on any multiple-partition batch size exceeding this value. 5kb per batch by default.
+Caution should be taken on increasing the size of this threshold as it can lead to node instability.
+
+*Default Value:* 5
+
+``batch_size_fail_threshold_in_kb``
+-----------------------------------
+
+Fail any multiple-partition batch exceeding this value. 50kb (10x warn threshold) by default.
+
+*Default Value:* 50
+
+``unlogged_batch_across_partitions_warn_threshold``
+---------------------------------------------------
+
+Log WARN on any batches not of type LOGGED than span across more partitions than this limit
+
+*Default Value:* 10
+
+``compaction_large_partition_warning_threshold_mb``
+---------------------------------------------------
+
+Log a warning when compacting partitions larger than this value
+
+*Default Value:* 100
+
+``gc_log_threshold_in_ms``
+--------------------------
+*This option is commented out by default.*
+
+GC Pauses greater than 200 ms will be logged at INFO level
+This threshold can be adjusted to minimize logging if necessary
+
+*Default Value:* 200
+
+``gc_warn_threshold_in_ms``
+---------------------------
+*This option is commented out by default.*
+
+GC Pauses greater than gc_warn_threshold_in_ms will be logged at WARN level
+Adjust the threshold based on your application throughput requirement. Setting to 0
+will deactivate the feature.
+
+*Default Value:* 1000
+
+``max_value_size_in_mb``
+------------------------
+*This option is commented out by default.*
+
+Maximum size of any value in SSTables. Safety measure to detect SSTable corruption
+early. Any value size larger than this threshold will result into marking an SSTable
+as corrupted. This should be positive and less than 2048.
+
+*Default Value:* 256
+
+``back_pressure_enabled``
+-------------------------
+
+Back-pressure settings #
+If enabled, the coordinator will apply the back-pressure strategy specified below to each mutation
+sent to replicas, with the aim of reducing pressure on overloaded replicas.
+
+*Default Value:* false
+
+``back_pressure_strategy``
+--------------------------
+The back-pressure strategy applied.
+The default implementation, RateBasedBackPressure, takes three arguments:
+high ratio, factor, and flow type, and uses the ratio between incoming mutation responses and outgoing mutation requests.
+If below high ratio, outgoing mutations are rate limited according to the incoming rate decreased by the given factor;
+if above high ratio, the rate limiting is increased by the given factor;
+such factor is usually best configured between 1 and 10, use larger values for a faster recovery
+at the expense of potentially more dropped mutations;
+the rate limiting is applied according to the flow type: if FAST, it's rate limited at the speed of the fastest replica,
+if SLOW at the speed of the slowest one.
+New strategies can be added. Implementors need to implement org.apache.cassandra.net.BackpressureStrategy and
+provide a public constructor accepting a Map<String, Object>.
+
+``otc_coalescing_strategy``
+---------------------------
+*This option is commented out by default.*
+
+Coalescing Strategies #
+Coalescing multiples messages turns out to significantly boost message processing throughput (think doubling or more).
+On bare metal, the floor for packet processing throughput is high enough that many applications won't notice, but in
+virtualized environments, the point at which an application can be bound by network packet processing can be
+surprisingly low compared to the throughput of task processing that is possible inside a VM. It's not that bare metal
+doesn't benefit from coalescing messages, it's that the number of packets a bare metal network interface can process
+is sufficient for many applications such that no load starvation is experienced even without coalescing.
+There are other benefits to coalescing network messages that are harder to isolate with a simple metric like messages
+per second. By coalescing multiple tasks together, a network thread can process multiple messages for the cost of one
+trip to read from a socket, and all the task submission work can be done at the same time reducing context switching
+and increasing cache friendliness of network message processing.
+See CASSANDRA-8692 for details.
+
+Strategy to use for coalescing messages in OutboundTcpConnection.
+Can be fixed, movingaverage, timehorizon, disabled (default).
+You can also specify a subclass of CoalescingStrategies.CoalescingStrategy by name.
+
+*Default Value:* DISABLED
+
+``otc_coalescing_window_us``
+----------------------------
+*This option is commented out by default.*
+
+How many microseconds to wait for coalescing. For fixed strategy this is the amount of time after the first
+message is received before it will be sent with any accompanying messages. For moving average this is the
+maximum amount of time that will be waited as well as the interval at which messages must arrive on average
+for coalescing to be enabled.
+
+*Default Value:* 200
+
+``otc_coalescing_enough_coalesced_messages``
+--------------------------------------------
+*This option is commented out by default.*
+
+Do not try to coalesce messages if we already got that many messages. This should be more than 2 and less than 128.
+
+*Default Value:* 8
+
+``otc_backlog_expiration_interval_ms``
+--------------------------------------
+*This option is commented out by default.*
+
+How many milliseconds to wait between two expiration runs on the backlog (queue) of the OutboundTcpConnection.
+Expiration is done if messages are piling up in the backlog. Droppable messages are expired to free the memory
+taken by expired messages. The interval should be between 0 and 1000, and in most installations the default value
+will be appropriate. A smaller value could potentially expire messages slightly sooner at the expense of more CPU
+time and queue contention while iterating the backlog of messages.
+An interval of 0 disables any wait time, which is the behavior of former Cassandra versions.
+
+
+*Default Value:* 200
+
+``ideal_consistency_level``
+---------------------------
+*This option is commented out by default.*
+
+Track a metric per keyspace indicating whether replication achieved the ideal consistency
+level for writes without timing out. This is different from the consistency level requested by
+each write which may be lower in order to facilitate availability.
+
+*Default Value:* EACH_QUORUM
+
+``automatic_sstable_upgrade``
+-----------------------------
+*This option is commented out by default.*
+
+Automatically upgrade sstables after upgrade - if there is no ordinary compaction to do, the
+oldest non-upgraded sstable will get upgraded to the latest version
+
+*Default Value:* false
+
+``max_concurrent_automatic_sstable_upgrades``
+---------------------------------------------
+*This option is commented out by default.*
+Limit the number of concurrent sstable upgrades
+
+*Default Value:* 1
+
+``audit_logging_options``
+-------------------------
+
+Audit logging - Logs every incoming CQL command request, authentication to a node. See the docs
+on audit_logging for full details about the various configuration options.
+
+``full_query_logging_options``
+------------------------------
+*This option is commented out by default.*
+
+
+default options for full query logging - these can be overridden from command line when executing
+nodetool enablefullquerylog
+
+``corrupted_tombstone_strategy``
+--------------------------------
+*This option is commented out by default.*
+
+validate tombstones on reads and compaction
+can be either "disabled", "warn" or "exception"
+
+*Default Value:* disabled
+
+``diagnostic_events_enabled``
+-----------------------------
+
+Diagnostic Events #
+If enabled, diagnostic events can be helpful for troubleshooting operational issues. Emitted events contain details
+on internal state and temporal relationships across events, accessible by clients via JMX.
+
+*Default Value:* false
+
+``native_transport_flush_in_batches_legacy``
+--------------------------------------------
+*This option is commented out by default.*
+
+Use native transport TCP message coalescing. If on upgrade to 4.0 you found your throughput decreasing, and in
+particular you run an old kernel or have very fewer client connections, this option might be worth evaluating.
+
+*Default Value:* false
+
+``repaired_data_tracking_for_range_reads_enabled``
+--------------------------------------------------
+
+Enable tracking of repaired state of data during reads and comparison between replicas
+Mismatches between the repaired sets of replicas can be characterized as either confirmed
+or unconfirmed. In this context, unconfirmed indicates that the presence of pending repair
+sessions, unrepaired partition tombstones, or some other condition means that the disparity
+cannot be considered conclusive. Confirmed mismatches should be a trigger for investigation
+as they may be indicative of corruption or data loss.
+There are separate flags for range vs partition reads as single partition reads are only tracked
+when CL > 1 and a digest mismatch occurs. Currently, range queries don't use digests so if
+enabled for range reads, all range reads will include repaired data tracking. As this adds
+some overhead, operators may wish to disable it whilst still enabling it for partition reads
+
+*Default Value:* false
+
+``repaired_data_tracking_for_partition_reads_enabled``
+------------------------------------------------------
+
+*Default Value:* false
+
+``report_unconfirmed_repaired_data_mismatches``
+-----------------------------------------------
+If false, only confirmed mismatches will be reported. If true, a separate metric for unconfirmed
+mismatches will also be recorded. This is to avoid potential signal:noise issues are unconfirmed
+mismatches are less actionable than confirmed ones.
+
+*Default Value:* false
+
+``enable_materialized_views``
+-----------------------------
+
+########################
+EXPERIMENTAL FEATURES #
+########################
+
+Enables materialized view creation on this node.
+Materialized views are considered experimental and are not recommended for production use.
+
+*Default Value:* false
+
+``enable_sasi_indexes``
+-----------------------
+
+Enables SASI index creation on this node.
+SASI indexes are considered experimental and are not recommended for production use.
+
+*Default Value:* false
+
+``enable_transient_replication``
+--------------------------------
+
+Enables creation of transiently replicated keyspaces on this node.
+Transient replication is experimental and is not recommended for production use.
+
+*Default Value:* false
diff --git a/doc/source/configuration/index.rst b/doc/source/configuration/index.rst
index f774fda..ea34af3 100644
--- a/doc/source/configuration/index.rst
+++ b/doc/source/configuration/index.rst
@@ -22,4 +22,10 @@
.. toctree::
:maxdepth: 1
- cassandra_config_file
+ cass_yaml_file
+ cass_rackdc_file
+ cass_env_sh_file
+ cass_topo_file
+ cass_cl_archive_file
+ cass_logback_xml_file
+ cass_jvm_options_file
diff --git a/doc/source/contactus.rst b/doc/source/contactus.rst
index 8d0f5dd..3ed9004 100644
--- a/doc/source/contactus.rst
+++ b/doc/source/contactus.rst
@@ -17,7 +17,7 @@
Contact us
==========
-You can get in touch with the Cassandra community either via the mailing lists or the freenode IRC channels.
+You can get in touch with the Cassandra community either via the mailing lists or :ref:`Slack rooms <slack>`.
.. _mailing-lists:
@@ -39,15 +39,12 @@
email to confirm your subscription. Make sure to keep the welcome email as it contains instructions on how to
unsubscribe.
-.. _irc-channels:
+.. _slack:
-IRC
----
+Slack
+-----
+To chat with developers or users in real-time, join our rooms on `ASF Slack <https://s.apache.org/slack-invite>`__:
-To chat with developers or users in real-time, join our channels on `IRC freenode <http://webchat.freenode.net/>`__. The
-following channels are available:
-
-- ``#cassandra`` - for user questions and general discussions.
-- ``#cassandra-dev`` - strictly for questions or discussions related to Cassandra development.
-- ``#cassandra-builds`` - results of automated test builds.
+- ``cassandra`` - for user questions and general discussions.
+- ``cassandra-dev`` - strictly for questions or discussions related to Cassandra development.
diff --git a/doc/source/cql/changes.rst b/doc/source/cql/changes.rst
index 1eee536..6691f15 100644
--- a/doc/source/cql/changes.rst
+++ b/doc/source/cql/changes.rst
@@ -21,6 +21,14 @@
The following describes the changes in each version of CQL.
+3.4.5
+^^^^^
+
+- Adds support for arithmetic operators (:jira:`11935`)
+- Adds support for ``+`` and ``-`` operations on dates (:jira:`11936`)
+- Adds ``currentTimestamp``, ``currentDate``, ``currentTime`` and ``currentTimeUUID`` functions (:jira:`13132`)
+
+
3.4.4
^^^^^
@@ -33,8 +41,7 @@
- Adds a new ``duration `` :ref:`data types <data-types>` (:jira:`11873`).
- Support for ``GROUP BY`` (:jira:`10707`).
- Adds a ``DEFAULT UNSET`` option for ``INSERT JSON`` to ignore omitted columns (:jira:`11424`).
-- Allows ``null`` as a legal value for TTL on insert and update. It will be treated as equivalent to
-inserting a 0 (:jira:`12216`).
+- Allows ``null`` as a legal value for TTL on insert and update. It will be treated as equivalent to inserting a 0 (:jira:`12216`).
3.4.2
^^^^^
diff --git a/doc/source/cql/ddl.rst b/doc/source/cql/ddl.rst
index 3027775..bd578ca 100644
--- a/doc/source/cql/ddl.rst
+++ b/doc/source/cql/ddl.rst
@@ -73,13 +73,15 @@
For instance::
- CREATE KEYSPACE Excelsior
- WITH replication = {'class': 'SimpleStrategy', 'replication_factor' : 3};
+ CREATE KEYSPACE excelsior
+ WITH replication = {'class': 'SimpleStrategy', 'replication_factor' : 3};
- CREATE KEYSPACE Excalibur
- WITH replication = {'class': 'NetworkTopologyStrategy', 'DC1' : 1, 'DC2' : 3}
- AND durable_writes = false;
+ CREATE KEYSPACE excalibur
+ WITH replication = {'class': 'NetworkTopologyStrategy', 'DC1' : 1, 'DC2' : 3}
+ AND durable_writes = false;
+Attempting to create a keyspace that already exists will return an error unless the ``IF NOT EXISTS`` option is used. If
+it is used, the statement will be a no-op if the keyspace already exists.
The supported ``options`` are:
@@ -96,14 +98,82 @@
:ref:`replication strategy <replication-strategy>` class to use. The rest of the sub-options depends on what replication
strategy is used. By default, Cassandra support the following ``'class'``:
-- ``'SimpleStrategy'``: A simple strategy that defines a replication factor for the whole cluster. The only sub-options
- supported is ``'replication_factor'`` to define that replication factor and is mandatory.
-- ``'NetworkTopologyStrategy'``: A replication strategy that allows to set the replication factor independently for
- each data-center. The rest of the sub-options are key-value pairs where a key is a data-center name and its value is
- the associated replication factor.
+.. _replication-strategy:
-Attempting to create a keyspace that already exists will return an error unless the ``IF NOT EXISTS`` option is used. If
-it is used, the statement will be a no-op if the keyspace already exists.
+``SimpleStrategy``
+""""""""""""""""""
+
+A simple strategy that defines a replication factor for data to be spread
+across the entire cluster. This is generally not a wise choice for production
+because it does not respect datacenter layouts and can lead to wildly varying
+query latency. For a production ready strategy, see
+``NetworkTopologyStrategy``. ``SimpleStrategy`` supports a single mandatory argument:
+
+========================= ====== ======= =============================================
+sub-option type since description
+========================= ====== ======= =============================================
+``'replication_factor'`` int all The number of replicas to store per range
+========================= ====== ======= =============================================
+
+``NetworkTopologyStrategy``
+"""""""""""""""""""""""""""
+
+A production ready replication strategy that allows to set the replication
+factor independently for each data-center. The rest of the sub-options are
+key-value pairs where a key is a data-center name and its value is the
+associated replication factor. Options:
+
+===================================== ====== ====== =============================================
+sub-option type since description
+===================================== ====== ====== =============================================
+``'<datacenter>'`` int all The number of replicas to store per range in
+ the provided datacenter.
+``'replication_factor'`` int 4.0 The number of replicas to use as a default
+ per datacenter if not specifically provided.
+ Note that this always defers to existing
+ definitions or explicit datacenter settings.
+ For example, to have three replicas per
+ datacenter, supply this with a value of 3.
+===================================== ====== ====== =============================================
+
+Note that when ``ALTER`` ing keyspaces and supplying ``replication_factor``,
+auto-expansion will only *add* new datacenters for safety, it will not alter
+existing datacenters or remove any even if they are no longer in the cluster.
+If you want to remove datacenters while still supplying ``replication_factor``,
+explicitly zero out the datacenter you want to have zero replicas.
+
+An example of auto-expanding datacenters with two datacenters: ``DC1`` and ``DC2``::
+
+ CREATE KEYSPACE excalibur
+ WITH replication = {'class': 'NetworkTopologyStrategy', 'replication_factor' : 3}
+
+ DESCRIBE KEYSPACE excalibur
+ CREATE KEYSPACE excalibur WITH replication = {'class': 'NetworkTopologyStrategy', 'DC1': '3', 'DC2': '3'} AND durable_writes = true;
+
+
+An example of auto-expanding and overriding a datacenter::
+
+ CREATE KEYSPACE excalibur
+ WITH replication = {'class': 'NetworkTopologyStrategy', 'replication_factor' : 3, 'DC2': 2}
+
+ DESCRIBE KEYSPACE excalibur
+ CREATE KEYSPACE excalibur WITH replication = {'class': 'NetworkTopologyStrategy', 'DC1': '3', 'DC2': '2'} AND durable_writes = true;
+
+An example that excludes a datacenter while using ``replication_factor``::
+
+ CREATE KEYSPACE excalibur
+ WITH replication = {'class': 'NetworkTopologyStrategy', 'replication_factor' : 3, 'DC2': 0} ;
+
+ DESCRIBE KEYSPACE excalibur
+ CREATE KEYSPACE excalibur WITH replication = {'class': 'NetworkTopologyStrategy', 'DC1': '3'} AND durable_writes = true;
+
+If transient replication has been enabled, transient replicas can be configured for both
+``SimpleStrategy`` and ``NetworkTopologyStrategy`` by defining replication factors in the format ``'<total_replicas>/<transient_replicas>'``
+
+For instance, this keyspace will have 3 replicas in DC1, 1 of which is transient, and 5 replicas in DC2, 2 of which are transient::
+
+ CREATE KEYSPACE some_keysopace
+ WITH replication = {'class': 'NetworkTopologyStrategy', 'DC1' : '3/1'', 'DC2' : '5/2'};
.. _use-statement:
@@ -131,7 +201,7 @@
For instance::
ALTER KEYSPACE Excelsior
- WITH replication = {'class': 'SimpleStrategy', 'replication_factor' : 4};
+ WITH replication = {'class': 'SimpleStrategy', 'replication_factor' : 4};
The supported options are the same than for :ref:`creating a keyspace <create-keyspace-statement>`.
@@ -186,8 +256,7 @@
common_name text,
population varint,
average_size int
- ) WITH comment='Important biological records'
- AND read_repair_chance = 1.0;
+ ) WITH comment='Important biological records';
CREATE TABLE timeline (
userid uuid,
@@ -359,7 +428,7 @@
a int,
b int,
c int,
- PRIMARY KEY (a, c, d)
+ PRIMARY KEY (a, b, c)
);
SELECT * FROM t;
@@ -396,15 +465,14 @@
.. warning:: Since Cassandra 3.0, compact tables have the exact same layout internally than non compact ones (for the
same schema obviously), and declaring a table compact **only** creates artificial limitations on the table definition
- and usage that are necessary to ensure backward compatibility with the deprecated Thrift API. And as ``COMPACT
+ and usage. It only exists for historical reason and is preserved for backward compatibility And as ``COMPACT
STORAGE`` cannot, as of Cassandra |version|, be removed, it is strongly discouraged to create new table with the
``COMPACT STORAGE`` option.
-A *compact* table is one defined with the ``COMPACT STORAGE`` option. This option is mainly targeted towards backward
-compatibility for definitions created before CQL version 3 (see `www.datastax.com/dev/blog/thrift-to-cql3
-<http://www.datastax.com/dev/blog/thrift-to-cql3>`__ for more details) and shouldn't be used for new tables. Declaring a
-table with this option creates limitations for the table which are largely arbitrary but necessary for backward
-compatibility with the (deprecated) Thrift API. Amongst those limitation:
+A *compact* table is one defined with the ``COMPACT STORAGE`` option. This option is only maintained for backward
+compatibility for definitions created before CQL version 3 and shouldn't be used for new tables. Declaring a
+table with this option creates limitations for the table which are largely arbitrary (and exists for historical
+reasons). Amongst those limitation:
- a compact table cannot use collections nor static columns.
- if a compact table has at least one clustering column, then it must have *exactly* one column outside of the primary
@@ -453,15 +521,13 @@
| option | kind | default | description |
+================================+==========+=============+===========================================================+
| ``comment`` | *simple* | none | A free-form, human-readable comment. |
+| ``speculative_retry`` | *simple* | 99PERCENTILE| :ref:`Speculative retry options |
+| | | | <speculative-retry-options>`. |
+--------------------------------+----------+-------------+-----------------------------------------------------------+
-| ``read_repair_chance`` | *simple* | 0.1 | The probability with which to query extra nodes (e.g. |
-| | | | more nodes than required by the consistency level) for |
-| | | | the purpose of read repairs. |
+| ``cdc`` | *boolean*| false | Create a Change Data Capture (CDC) log on the table. |
+--------------------------------+----------+-------------+-----------------------------------------------------------+
-| ``dclocal_read_repair_chance`` | *simple* | 0 | The probability with which to query extra nodes (e.g. |
-| | | | more nodes than required by the consistency level) |
-| | | | belonging to the same data center than the read |
-| | | | coordinator for the purpose of read repairs. |
+| ``additional_write_policy`` | *simple* | 99PERCENTILE| :ref:`Speculative retry options |
+| | | | <speculative-retry-options>`. |
+--------------------------------+----------+-------------+-----------------------------------------------------------+
| ``gc_grace_seconds`` | *simple* | 864000 | Time to wait before garbage collecting tombstones |
| | | | (deletion markers). |
@@ -480,6 +546,104 @@
+--------------------------------+----------+-------------+-----------------------------------------------------------+
| ``caching`` | *map* | *see below* | :ref:`Caching options <cql-caching-options>`. |
+--------------------------------+----------+-------------+-----------------------------------------------------------+
+| ``memtable_flush_period_in_ms``| *simple* | 0 | Time (in ms) before Cassandra flushes memtables to disk. |
++--------------------------------+----------+-------------+-----------------------------------------------------------+
+| ``read_repair`` | *simple* | BLOCKING | Sets read repair behavior (see below) |
++--------------------------------+----------+-------------+-----------------------------------------------------------+
+
+.. _speculative-retry-options:
+
+Speculative retry options
+#########################
+
+By default, Cassandra read coordinators only query as many replicas as necessary to satisfy
+consistency levels: one for consistency level ``ONE``, a quorum for ``QUORUM``, and so on.
+``speculative_retry`` determines when coordinators may query additional replicas, which is useful
+when replicas are slow or unresponsive. Speculative retries are used to reduce the latency. The speculative_retry option may be
+used to configure rapid read protection with which a coordinator sends more requests than needed to satisfy the Consistency level.
+
+Pre-4.0 speculative Retry Policy takes a single string as a parameter, this can be ``NONE``, ``ALWAYS``, ``99PERCENTILE`` (PERCENTILE), ``50MS`` (CUSTOM).
+
+Examples of setting speculative retry are:
+
+::
+
+ ALTER TABLE users WITH speculative_retry = '10ms';
+
+
+Or,
+
+::
+
+ ALTER TABLE users WITH speculative_retry = '99PERCENTILE';
+
+The problem with these settings is when a single host goes into an unavailable state this drags up the percentiles. This means if we
+are set to use ``p99`` alone, we might not speculate when we intended to to because the value at the specified percentile has gone so high.
+As a fix 4.0 adds support for hybrid ``MIN()``, ``MAX()`` speculative retry policies (`CASSANDRA-14293
+<https://issues.apache.org/jira/browse/CASSANDRA-14293>`_). This means if the normal ``p99`` for the
+table is <50ms, we will still speculate at this value and not drag the tail latencies up... but if the ``p99th`` goes above what we know we
+should never exceed we use that instead.
+
+In 4.0 the values (case-insensitive) discussed in the following table are supported:
+
+============================ ======================== =============================================================================
+ Format Example Description
+============================ ======================== =============================================================================
+ ``XPERCENTILE`` 90.5PERCENTILE Coordinators record average per-table response times for all replicas.
+ If a replica takes longer than ``X`` percent of this table's average
+ response time, the coordinator queries an additional replica.
+ ``X`` must be between 0 and 100.
+ ``XP`` 90.5P Synonym for ``XPERCENTILE``
+ ``Yms`` 25ms If a replica takes more than ``Y`` milliseconds to respond,
+ the coordinator queries an additional replica.
+ ``MIN(XPERCENTILE,YMS)`` MIN(99PERCENTILE,35MS) A hybrid policy that will use either the specified percentile or fixed
+ milliseconds depending on which value is lower at the time of calculation.
+ Parameters are ``XPERCENTILE``, ``XP``, or ``Yms``.
+ This is helpful to help protect against a single slow instance; in the
+ happy case the 99th percentile is normally lower than the specified
+ fixed value however, a slow host may skew the percentile very high
+ meaning the slower the cluster gets, the higher the value of the percentile,
+ and the higher the calculated time used to determine if we should
+ speculate or not. This allows us to set an upper limit that we want to
+ speculate at, but avoid skewing the tail latencies by speculating at the
+ lower value when the percentile is less than the specified fixed upper bound.
+ ``MAX(XPERCENTILE,YMS)`` MAX(90.5P,25ms) A hybrid policy that will use either the specified percentile or fixed
+ milliseconds depending on which value is higher at the time of calculation.
+ ``ALWAYS`` Coordinators always query all replicas.
+ ``NEVER`` Coordinators never query additional replicas.
+============================ =================== =============================================================================
+
+As of version 4.0 speculative retry allows more friendly params (`CASSANDRA-13876
+<https://issues.apache.org/jira/browse/CASSANDRA-13876>`_). The ``speculative_retry`` is more flexible with case. As an example a
+value does not have to be ``NONE``, and the following are supported alternatives.
+
+::
+
+ alter table users WITH speculative_retry = 'none';
+ alter table users WITH speculative_retry = 'None';
+
+The text component is case insensitive and for ``nPERCENTILE`` version 4.0 allows ``nP``, for instance ``99p``.
+In a hybrid value for speculative retry, one of the two values must be a fixed millisecond value and the other a percentile value.
+
+Some examples:
+
+::
+
+ min(99percentile,50ms)
+ max(99p,50MS)
+ MAX(99P,50ms)
+ MIN(99.9PERCENTILE,50ms)
+ max(90percentile,100MS)
+ MAX(100.0PERCENTILE,60ms)
+
+Two values of the same kind cannot be specified such as ``min(90percentile,99percentile)`` as it wouldn’t be a hybrid value.
+This setting does not affect reads with consistency level ``ALL`` because they already query all replicas.
+
+Note that frequently reading from additional replicas can hurt cluster performance.
+When in doubt, keep the default ``99PERCENTILE``.
+
+
+``additional_write_policy`` specifies the threshold at which a cheap quorum write will be upgraded to include transient replicas.
.. _cql-compaction-options:
@@ -487,10 +651,10 @@
##################
The ``compaction`` options must at least define the ``'class'`` sub-option, that defines the compaction strategy class
-to use. The default supported class are ``'SizeTieredCompactionStrategy'`` (:ref:`STCS <STCS>`),
+to use. The supported class are ``'SizeTieredCompactionStrategy'`` (:ref:`STCS <STCS>`),
``'LeveledCompactionStrategy'`` (:ref:`LCS <LCS>`) and ``'TimeWindowCompactionStrategy'`` (:ref:`TWCS <TWCS>`) (the
``'DateTieredCompactionStrategy'`` is also supported but is deprecated and ``'TimeWindowCompactionStrategy'`` should be
-preferred instead). Custom strategy can be provided by specifying the full class name as a :ref:`string constant
+preferred instead). The default is ``'SizeTieredCompactionStrategy'``. Custom strategy can be provided by specifying the full class name as a :ref:`string constant
<constants>`.
All default strategies support a number of :ref:`common options <compaction-options>`, as well as options specific to
@@ -502,27 +666,36 @@
Compression options
###################
-The ``compression`` options define if and how the sstables of the table are compressed. The following sub-options are
+The ``compression`` options define if and how the sstables of the table are compressed. Compression is configured on a per-table
+basis as an optional argument to ``CREATE TABLE`` or ``ALTER TABLE``. The following sub-options are
available:
========================= =============== =============================================================================
Option Default Description
========================= =============== =============================================================================
``class`` LZ4Compressor The compression algorithm to use. Default compressor are: LZ4Compressor,
- SnappyCompressor and DeflateCompressor. Use ``'enabled' : false`` to disable
+ SnappyCompressor, DeflateCompressor and ZstdCompressor. Use ``'enabled' : false`` to disable
compression. Custom compressor can be provided by specifying the full class
name as a “string constant”:#constants.
- ``enabled`` true Enable/disable sstable compression.
+
+ ``enabled`` true Enable/disable sstable compression. If the ``enabled`` option is set to ``false`` no other
+ options must be specified.
+
``chunk_length_in_kb`` 64 On disk SSTables are compressed by block (to allow random reads). This
defines the size (in KB) of said block. Bigger values may improve the
compression rate, but increases the minimum size of data to be read from disk
- for a read
- ``crc_check_chance`` 1.0 When compression is enabled, each compressed block includes a checksum of
- that block for the purpose of detecting disk bitrot and avoiding the
- propagation of corruption to other replica. This option defines the
- probability with which those checksums are checked during read. By default
- they are always checked. Set to 0 to disable checksum checking and to 0.5 for
- instance to check them every other read |
+ for a read. The default value is an optimal value for compressing tables. Chunk length must
+ be a power of 2 because so is assumed so when computing the chunk number from an uncompressed
+ file offset. Block size may be adjusted based on read/write access patterns such as:
+
+ - How much data is typically requested at once
+ - Average size of rows in the table
+
+ ``crc_check_chance`` 1.0 Determines how likely Cassandra is to verify the checksum on each compression chunk during
+ reads.
+
+ ``compression_level`` 3 Compression level. It is only applicable for ``ZstdCompressor`` and accepts values between
+ ``-131072`` and ``22``.
========================= =============== =============================================================================
@@ -541,7 +714,8 @@
Caching options
###############
-The ``caching`` options allows to configure both the *key cache* and the *row cache* for the table. The following
+Caching optimizes the use of cache memory of a table. The cached data is weighed by size and access frequency. The ``caching``
+options allows to configure both the *key cache* and the *row cache* for the table. The following
sub-options are available:
======================== ========= ====================================================================================
@@ -566,6 +740,36 @@
) WITH caching = {'keys': 'ALL', 'rows_per_partition': 10};
+Read Repair options
+###################
+
+The ``read_repair`` options configures the read repair behavior to allow tuning for various performance and
+consistency behaviors. Two consistency properties are affected by read repair behavior.
+
+- Monotonic Quorum Reads: Provided by ``BLOCKING``. Monotonic quorum reads prevents reads from appearing to go back
+ in time in some circumstances. When monotonic quorum reads are not provided and a write fails to reach a quorum of
+ replicas, it may be visible in one read, and then disappear in a subsequent read.
+- Write Atomicity: Provided by ``NONE``. Write atomicity prevents reads from returning partially applied writes.
+ Cassandra attempts to provide partition level write atomicity, but since only the data covered by a SELECT statement
+ is repaired by a read repair, read repair can break write atomicity when data is read at a more granular level than it
+ is written. For example read repair can break write atomicity if you write multiple rows to a clustered partition in a
+ batch, but then select a single row by specifying the clustering column in a SELECT statement.
+
+The available read repair settings are:
+
+Blocking
+````````
+The default setting. When ``read_repair`` is set to ``BLOCKING``, and a read repair is triggered, the read will block
+on writes sent to other replicas until the CL is reached by the writes. Provides monotonic quorum reads, but not partition
+level write atomicity
+
+None
+````
+
+When ``read_repair`` is set to ``NONE``, the coordinator will reconcile any differences between replicas, but will not
+attempt to repair them. Provides partition level write atomicity, but not monotonic quorum reads.
+
+
Other considerations:
#####################
@@ -590,8 +794,7 @@
ALTER TABLE addamsFamily ADD gravesite varchar;
ALTER TABLE addamsFamily
- WITH comment = 'A most excellent and useful table'
- AND read_repair_chance = 0.2;
+ WITH comment = 'A most excellent and useful table';
The ``ALTER TABLE`` statement can:
@@ -647,3 +850,325 @@
that can be truncated currently and so the ``TABLE`` keyword can be omitted.
Truncating a table permanently removes all existing data from the table, but without removing the table itself.
+
+.. _describe-statements:
+
+DESCRIBE
+^^^^^^^^
+
+Statements used to outputs information about the connected Cassandra cluster,
+or about the data objects stored in the cluster.
+
+.. warning:: Describe statement resultset that exceed the page size are paged. If a schema changes is detected between
+ two pages the query will fail with an ``InvalidRequestException``. It is safe to retry the whole ``DESCRIBE``
+ statement after such an error.
+
+DESCRIBE KEYSPACES
+""""""""""""""""""
+
+Output the names of all keyspaces.
+
+.. productionlist::
+ describe_keyspaces_statement: DESCRIBE KEYSPACES
+
+Returned columns:
+
+======================== ========= ====================================================================================
+ Columns Type Description
+======================== ========= ====================================================================================
+ keyspace_name text The keyspace name
+ type text The schema element type
+ name text The schema element name
+======================== ========= ====================================================================================
+
+DESCRIBE KEYSPACE
+"""""""""""""""""
+
+Output CQL commands that could be used to recreate the given keyspace, and the objects in it
+(such as tables, types, functions, etc.).
+
+.. productionlist::
+ describe_keyspace_statement: DESCRIBE [ONLY] KEYSPACE [`keyspace_name`] [WITH INTERNALS]
+
+The ``keyspace_name`` argument may be omitted, in which case the current keyspace will be described.
+
+If ``WITH INTERNALS`` is specified, the output contains the table IDs and is adopted to represent the DDL necessary
+to "re-create" dropped columns.
+
+If ``ONLY`` is specified, only the DDL to recreate the keyspace will be created. All keyspace elements, like tables,
+types, functions, etc will be omitted.
+
+Returned columns:
+
+======================== ========= ====================================================================================
+ Columns Type Description
+======================== ========= ====================================================================================
+ keyspace_name text The keyspace name
+ type text The schema element type
+ name text The schema element name
+ create_statement text The CQL statement to use to recreate the schema element
+======================== ========= ====================================================================================
+
+DESCRIBE TABLES
+"""""""""""""""
+
+Output the names of all tables in the current keyspace, or in all keyspaces if there is no current keyspace.
+
+.. productionlist::
+ describe_tables_statement: DESCRIBE TABLES
+
+Returned columns:
+
+======================== ========= ====================================================================================
+ Columns Type Description
+======================== ========= ====================================================================================
+ keyspace_name text The keyspace name
+ type text The schema element type
+ name text The schema element name
+======================== ========= ====================================================================================
+
+DESCRIBE TABLE
+""""""""""""""
+
+Output CQL commands that could be used to recreate the given table.
+
+.. productionlist::
+ describe_table_statement: DESCRIBE TABLE [`keyspace_name`.]`table_name` [WITH INTERNALS]
+
+If `WITH INTERNALS` is specified, the output contains the table ID and is adopted to represent the DDL necessary
+to "re-create" dropped columns.
+
+Returned columns:
+
+======================== ========= ====================================================================================
+ Columns Type Description
+======================== ========= ====================================================================================
+ keyspace_name text The keyspace name
+ type text The schema element type
+ name text The schema element name
+ create_statement text The CQL statement to use to recreate the schema element
+======================== ========= ====================================================================================
+
+DESCRIBE INDEX
+""""""""""""""
+
+Output the CQL command that could be used to recreate the given index.
+
+.. productionlist::
+ describe_index_statement: DESCRIBE INDEX [`keyspace_name`.]`index_name`
+
+Returned columns:
+
+======================== ========= ====================================================================================
+ Columns Type Description
+======================== ========= ====================================================================================
+ keyspace_name text The keyspace name
+ type text The schema element type
+ name text The schema element name
+ create_statement text The CQL statement to use to recreate the schema element
+======================== ========= ====================================================================================
+
+
+DESCRIBE MATERIALIZED VIEW
+""""""""""""""""""""""""""
+
+Output the CQL command that could be used to recreate the given materialized view.
+
+.. productionlist::
+ describe_materialized_view_statement: DESCRIBE MATERIALIZED VIEW [`keyspace_name`.]`view_name`
+
+Returned columns:
+
+======================== ========= ====================================================================================
+ Columns Type Description
+======================== ========= ====================================================================================
+ keyspace_name text The keyspace name
+ type text The schema element type
+ name text The schema element name
+ create_statement text The CQL statement to use to recreate the schema element
+======================== ========= ====================================================================================
+
+DESCRIBE CLUSTER
+""""""""""""""""
+
+Output information about the connected Cassandra cluster, such as the cluster name, and the partitioner and snitch
+in use. When you are connected to a non-system keyspace, also shows endpoint-range ownership information for
+the Cassandra ring.
+
+.. productionlist::
+ describe_cluster_statement: DESCRIBE CLUSTER
+
+Returned columns:
+
+======================== ====================== ========================================================================
+ Columns Type Description
+======================== ====================== ========================================================================
+ cluster text The cluster name
+ partitioner text The partitioner being used by the cluster
+ snitch text The snitch being used by the cluster
+ range_ownership map<text, list<text>> The CQL statement to use to recreate the schema element
+======================== ====================== ========================================================================
+
+DESCRIBE SCHEMA
+"""""""""""""""
+
+Output CQL commands that could be used to recreate the entire (non-system) schema.
+Works as though "DESCRIBE KEYSPACE k" was invoked for each non-system keyspace
+
+.. productionlist::
+ describe_schema_statement: DESCRIBE [FULL] SCHEMA [WITH INTERNALS]
+
+Use ``DESCRIBE FULL SCHEMA`` to include the system keyspaces.
+
+If ``WITH INTERNALS`` is specified, the output contains the table IDs and is adopted to represent the DDL necessary
+to "re-create" dropped columns.
+
+Returned columns:
+
+======================== ========= ====================================================================================
+ Columns Type Description
+======================== ========= ====================================================================================
+ keyspace_name text The keyspace name
+ type text The schema element type
+ name text The schema element name
+ create_statement text The CQL statement to use to recreate the schema element
+======================== ========= ====================================================================================
+
+DESCRIBE TYPES
+""""""""""""""
+
+Output the names of all user-defined-types in the current keyspace, or in all keyspaces if there is no current keyspace.
+
+.. productionlist::
+ describe_types_statement: DESCRIBE TYPES
+
+Returned columns:
+
+======================== ========= ====================================================================================
+ Columns Type Description
+======================== ========= ====================================================================================
+ keyspace_name text The keyspace name
+ type text The schema element type
+ name text The schema element name
+======================== ========= ====================================================================================
+
+DESCRIBE TYPE
+"""""""""""""
+
+Output the CQL command that could be used to recreate the given user-defined-type.
+
+.. productionlist::
+ describe_type_statement: DESCRIBE TYPE [`keyspace_name`.]`type_name`
+
+Returned columns:
+
+======================== ========= ====================================================================================
+ Columns Type Description
+======================== ========= ====================================================================================
+ keyspace_name text The keyspace name
+ type text The schema element type
+ name text The schema element name
+ create_statement text The CQL statement to use to recreate the schema element
+======================== ========= ====================================================================================
+
+DESCRIBE FUNCTIONS
+""""""""""""""""""
+
+Output the names of all user-defined-functions in the current keyspace, or in all keyspaces if there is no current
+keyspace.
+
+.. productionlist::
+ describe_functions_statement: DESCRIBE FUNCTIONS
+
+Returned columns:
+
+======================== ========= ====================================================================================
+ Columns Type Description
+======================== ========= ====================================================================================
+ keyspace_name text The keyspace name
+ type text The schema element type
+ name text The schema element name
+======================== ========= ====================================================================================
+
+DESCRIBE FUNCTION
+"""""""""""""""""
+
+Output the CQL command that could be used to recreate the given user-defined-function.
+
+.. productionlist::
+ describe_function_statement: DESCRIBE FUNCTION [`keyspace_name`.]`function_name`
+
+Returned columns:
+
+======================== ========= ====================================================================================
+ Columns Type Description
+======================== ========= ====================================================================================
+ keyspace_name text The keyspace name
+ type text The schema element type
+ name text The schema element name
+ create_statement text The CQL statement to use to recreate the schema element
+======================== ========= ====================================================================================
+
+DESCRIBE AGGREGATES
+"""""""""""""""""""
+
+Output the names of all user-defined-aggregates in the current keyspace, or in all keyspaces if there is no current
+keyspace.
+
+.. productionlist::
+ describe_aggregates_statement: DESCRIBE AGGREGATES
+
+Returned columns:
+
+======================== ========= ====================================================================================
+ Columns Type Description
+======================== ========= ====================================================================================
+ keyspace_name text The keyspace name
+ type text The schema element type
+ name text The schema element name
+======================== ========= ====================================================================================
+
+DESCRIBE AGGREGATE
+""""""""""""""""""
+
+Output the CQL command that could be used to recreate the given user-defined-aggregate.
+
+.. productionlist::
+ describe_aggregate_statement: DESCRIBE AGGREGATE [`keyspace_name`.]`aggregate_name`
+
+Returned columns:
+
+======================== ========= ====================================================================================
+ Columns Type Description
+======================== ========= ====================================================================================
+ keyspace_name text The keyspace name
+ type text The schema element type
+ name text The schema element name
+ create_statement text The CQL statement to use to recreate the schema element
+======================== ========= ====================================================================================
+
+DESCRIBE object
+"""""""""""""""
+
+Output CQL commands that could be used to recreate the entire object schema, where object can be either a keyspace
+or a table or an index or a materialized view (in this order).
+
+.. productionlist::
+ describe_object_statement: DESCRIBE `object_name` [WITH INTERNALS]
+
+If ``WITH INTERNALS`` is specified and ``object_name`` represents a keyspace or table the output contains the table IDs
+and is adopted to represent the DDL necessary to "re-create" dropped columns.
+
+``object_name`` cannot be any of the "describe what" qualifiers like "cluster", "table", etc.
+
+Returned columns:
+
+======================== ========= ====================================================================================
+ Columns Type Description
+======================== ========= ====================================================================================
+ keyspace_name text The keyspace name
+ type text The schema element type
+ name text The schema element name
+ create_statement text The CQL statement to use to recreate the schema element
+======================== ========= ====================================================================================
+
diff --git a/doc/source/cql/definitions.rst b/doc/source/cql/definitions.rst
index d4a5b59..3df6f20 100644
--- a/doc/source/cql/definitions.rst
+++ b/doc/source/cql/definitions.rst
@@ -119,9 +119,10 @@
CQL has the notion of a *term*, which denotes the kind of values that CQL support. Terms are defined by:
.. productionlist::
- term: `constant` | `literal` | `function_call` | `type_hint` | `bind_marker`
+ term: `constant` | `literal` | `function_call` | `arithmetic_operation` | `type_hint` | `bind_marker`
literal: `collection_literal` | `udt_literal` | `tuple_literal`
function_call: `identifier` '(' [ `term` (',' `term`)* ] ')'
+ arithmetic_operation: '-' `term` | `term` ('+' | '-' | '*' | '/' | '%') `term`
type_hint: '(' `cql_type` `)` term
bind_marker: '?' | ':' `identifier`
@@ -132,6 +133,7 @@
(see the linked sections for details).
- A function call: see :ref:`the section on functions <cql-functions>` for details on which :ref:`native function
<native-functions>` exists and how to define your own :ref:`user-defined ones <udfs>`.
+- An arithmetic operation between terms. see :ref:`the section on arithmetic operations <arithmetic_operators>`
- A *type hint*: see the :ref:`related section <type-hints>` for details.
- A bind marker, which denotes a variable to be bound at execution time. See the section on :ref:`prepared-statements`
for details. A bind marker can be either anonymous (``?``) or named (``:some_name``). The latter form provides a more
diff --git a/doc/source/cql/functions.rst b/doc/source/cql/functions.rst
index 47026cd..965125a 100644
--- a/doc/source/cql/functions.rst
+++ b/doc/source/cql/functions.rst
@@ -135,14 +135,16 @@
``now``
#######
-The ``now`` function takes no arguments and generates, on the coordinator node, a new unique timeuuid (at the time where
-the statement using it is executed). Note that this method is useful for insertion but is largely non-sensical in
+The ``now`` function takes no arguments and generates, on the coordinator node, a new unique timeuuid at the
+time the function is invoked. Note that this method is useful for insertion but is largely non-sensical in
``WHERE`` clauses. For instance, a query of the form::
SELECT * FROM myTable WHERE t = now()
will never return any result by design, since the value returned by ``now()`` is guaranteed to be unique.
+``currentTimeUUID`` is an alias of ``now``.
+
``minTimeuuid`` and ``maxTimeuuid``
###################################
@@ -164,8 +166,29 @@
particular, the value returned by these 2 methods will not be unique. This means you should only use those methods
for querying (as in the example above). Inserting the result of those methods is almost certainly *a bad idea*.
+Datetime functions
+``````````````````
+
+Retrieving the current date/time
+################################
+
+The following functions can be used to retrieve the date/time at the time where the function is invoked:
+
+===================== ===============
+ Function name Output type
+===================== ===============
+ ``currentTimestamp`` ``timestamp``
+ ``currentDate`` ``date``
+ ``currentTime`` ``time``
+ ``currentTimeUUID`` ``timeUUID``
+===================== ===============
+
+For example the last 2 days of data can be retrieved using::
+
+ SELECT * FROM myTable WHERE date >= currentDate() - 2d
+
Time conversion functions
-`````````````````````````
+#########################
A number of functions are provided to “convert” a ``timeuuid``, a ``timestamp`` or a ``date`` into another ``native``
type.
diff --git a/doc/source/cql/index.rst b/doc/source/cql/index.rst
index 00d90e4..b4c21cf 100644
--- a/doc/source/cql/index.rst
+++ b/doc/source/cql/index.rst
@@ -24,8 +24,7 @@
CQL offers a model close to SQL in the sense that data is put in *tables* containing *rows* of *columns*. For
that reason, when used in this document, these terms (tables, rows and columns) have the same definition than they have
-in SQL. But please note that as such, they do **not** refer to the concept of rows and columns found in the deprecated
-thrift API (and earlier version 1 and 2 of CQL).
+in SQL.
.. toctree::
:maxdepth: 2
@@ -38,6 +37,7 @@
mvs
security
functions
+ operators
json
triggers
appendices
diff --git a/doc/source/cql/mvs.rst b/doc/source/cql/mvs.rst
index aabea10..200090a 100644
--- a/doc/source/cql/mvs.rst
+++ b/doc/source/cql/mvs.rst
@@ -62,6 +62,11 @@
Attempting to create an already existing materialized view will return an error unless the ``IF NOT EXISTS`` option is
used. If it is used, the statement will be a no-op if the materialized view already exists.
+.. note:: By default, materialized views are built in a single thread. The initial build can be parallelized by
+ increasing the number of threads specified by the property ``concurrent_materialized_view_builders`` in
+ ``cassandra.yaml``. This property can also be manipulated at runtime through both JMX and the
+ ``setconcurrentviewbuilders`` and ``getconcurrentviewbuilders`` nodetool commands.
+
.. _mv-select:
MV select statement
@@ -164,3 +169,11 @@
If the materialized view does not exists, the statement will return an error, unless ``IF EXISTS`` is used in which case
the operation is a no-op.
+
+MV Limitations
+```````````````
+
+.. Note:: Removal of columns not selected in the Materialized View (via ``UPDATE base SET unselected_column = null`` or
+ ``DELETE unselected_column FROM base``) may shadow missed updates to other columns received by hints or repair.
+ For this reason, we advise against doing deletions on base columns not selected in views until this is
+ fixed on CASSANDRA-13826.
diff --git a/doc/source/cql/operators.rst b/doc/source/cql/operators.rst
new file mode 100644
index 0000000..1faf0d0
--- /dev/null
+++ b/doc/source/cql/operators.rst
@@ -0,0 +1,74 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements. See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership. The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License. You may obtain a copy of the License at
+..
+.. http://www.apache.org/licenses/LICENSE-2.0
+..
+.. Unless required by applicable law or agreed to in writing, software
+.. distributed under the License is distributed on an "AS IS" BASIS,
+.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+.. See the License for the specific language governing permissions and
+.. limitations under the License.
+
+.. highlight:: cql
+
+.. _arithmetic_operators:
+
+Arithmetic Operators
+--------------------
+
+CQL supports the following operators:
+
+=============== =======================================================================================================
+ Operator Description
+=============== =======================================================================================================
+ \- (unary) Negates operand
+ \+ Addition
+ \- Substraction
+ \* Multiplication
+ / Division
+ % Returns the remainder of a division
+=============== =======================================================================================================
+
+.. _number-arithmetic:
+
+Number Arithmetic
+^^^^^^^^^^^^^^^^^
+
+All arithmetic operations are supported on numeric types or counters.
+
+The return type of the operation will be based on the operand types:
+
+============= =========== ========== ========== ========== ========== ========== ========== ========== ==========
+ left/right tinyint smallint int bigint counter float double varint decimal
+============= =========== ========== ========== ========== ========== ========== ========== ========== ==========
+ **tinyint** tinyint smallint int bigint bigint float double varint decimal
+ **smallint** smallint smallint int bigint bigint float double varint decimal
+ **int** int int int bigint bigint float double varint decimal
+ **bigint** bigint bigint bigint bigint bigint double double varint decimal
+ **counter** bigint bigint bigint bigint bigint double double varint decimal
+ **float** float float float double double float double decimal decimal
+ **double** double double double double double double double decimal decimal
+ **varint** varint varint varint decimal decimal decimal decimal decimal decimal
+ **decimal** decimal decimal decimal decimal decimal decimal decimal decimal decimal
+============= =========== ========== ========== ========== ========== ========== ========== ========== ==========
+
+``*``, ``/`` and ``%`` operators have a higher precedence level than ``+`` and ``-`` operator. By consequence,
+they will be evaluated before. If two operator in an expression have the same precedence level, they will be evaluated
+left to right based on their position in the expression.
+
+.. _datetime--arithmetic:
+
+Datetime Arithmetic
+^^^^^^^^^^^^^^^^^^^
+
+A ``duration`` can be added (+) or substracted (-) from a ``timestamp`` or a ``date`` to create a new
+``timestamp`` or ``date``. So for instance::
+
+ SELECT * FROM myTable WHERE t = '2017-01-01' - 2d
+
+will select all the records with a value of ``t`` which is in the last 2 days of 2016.
diff --git a/doc/source/cql/security.rst b/doc/source/cql/security.rst
index 099fcc4..429a1ef 100644
--- a/doc/source/cql/security.rst
+++ b/doc/source/cql/security.rst
@@ -46,6 +46,8 @@
:| LOGIN '=' `boolean`
:| SUPERUSER '=' `boolean`
:| OPTIONS '=' `map_literal`
+ :| ACCESS TO DATACENTERS `set_literal`
+ :| ACCESS TO ALL DATACENTERS
For instance::
@@ -53,6 +55,8 @@
CREATE ROLE alice WITH PASSWORD = 'password_a' AND LOGIN = true;
CREATE ROLE bob WITH PASSWORD = 'password_b' AND LOGIN = true AND SUPERUSER = true;
CREATE ROLE carlos WITH OPTIONS = { 'custom_option1' : 'option1_value', 'custom_option2' : 99 };
+ CREATE ROLE alice WITH PASSWORD = 'password_a' AND LOGIN = true AND ACCESS TO DATACENTERS {'DC1', 'DC3'};
+ CREATE ROLE alice WITH PASSWORD = 'password_a' AND LOGIN = true AND ACCESS TO ALL DATACENTERS;
By default roles do not possess ``LOGIN`` privileges or ``SUPERUSER`` status.
@@ -81,6 +85,14 @@
If internal authentication has not been set up or the role does not have ``LOGIN`` privileges, the ``WITH PASSWORD``
clause is not necessary.
+Restricting connections to specific datacenters
+```````````````````````````````````````````````
+
+If a ``network_authorizer`` has been configured, you can restrict login roles to specific datacenters with the
+``ACCESS TO DATACENTERS`` clause followed by a set literal of datacenters the user can access. Not specifiying
+datacenters implicitly grants access to all datacenters. The clause ``ACCESS TO ALL DATACENTERS`` can be used for
+explicitness, but there's no functional difference.
+
Creating a role conditionally
`````````````````````````````
@@ -105,6 +117,13 @@
ALTER ROLE bob WITH PASSWORD = 'PASSWORD_B' AND SUPERUSER = false;
+Restricting connections to specific datacenters
+```````````````````````````````````````````````
+
+If a ``network_authorizer`` has been configured, you can restrict login roles to specific datacenters with the
+``ACCESS TO DATACENTERS`` clause followed by a set literal of datacenters the user can access. To remove any
+data center restrictions, use the ``ACCESS TO ALL DATACENTERS`` clause.
+
Conditions on executing ``ALTER ROLE`` statements:
- A client must have ``SUPERUSER`` status to alter the ``SUPERUSER`` status of another role
@@ -129,6 +148,14 @@
Attempting to drop a role which does not exist results in an invalid query condition unless the ``IF EXISTS`` option is
used. If the option is used and the role does not exist the statement is a no-op.
+.. note:: DROP ROLE intentionally does not terminate any open user sessions. Currently connected sessions will remain
+ connected and will retain the ability to perform any database actions which do not require :ref:`authorization<authorization>`.
+ However, if authorization is enabled, :ref:`permissions<cql-permissions>` of the dropped role are also revoked,
+ subject to the :ref:`caching options<auth-caching>` configured in :ref:`cassandra.yaml<cassandra-yaml>`.
+ Should a dropped role be subsequently recreated and have new :ref:`permissions<grant-permission-statement>` or
+ :ref:`roles<grant-role-statement>` granted to it, any client sessions still connected will acquire the newly granted
+ permissions and roles.
+
.. _grant-role-statement:
GRANT ROLE
@@ -472,6 +499,15 @@
REVOKE EXECUTE ON FUNCTION keyspace1.user_function( int ) FROM report_writer;
REVOKE DESCRIBE ON ALL ROLES FROM role_admin;
+Because of their function in normal driver operations, certain tables cannot have their `SELECT` permissions
+revoked. The following tables will be available to all authorized users regardless of their assigned role::
+
+* `system_schema.keyspaces`
+* `system_schema.columns`
+* `system_schema.tables`
+* `system.local`
+* `system.peers`
+
.. _list-permissions-statement:
LIST PERMISSIONS
diff --git a/doc/source/data_modeling/data_modeling_conceptual.rst b/doc/source/data_modeling/data_modeling_conceptual.rst
new file mode 100644
index 0000000..8749b79
--- /dev/null
+++ b/doc/source/data_modeling/data_modeling_conceptual.rst
@@ -0,0 +1,63 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements. See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership. The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License. You may obtain a copy of the License at
+..
+.. http://www.apache.org/licenses/LICENSE-2.0
+..
+.. Unless required by applicable law or agreed to in writing, software
+.. distributed under the License is distributed on an "AS IS" BASIS,
+.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+.. See the License for the specific language governing permissions and
+.. limitations under the License.
+
+.. conceptual_data_modeling
+
+Conceptual Data Modeling
+^^^^^^^^^^^^^^^^^^^^^^^^
+
+First, let’s create a simple domain model that is easy to understand in
+the relational world, and then see how you might map it from a relational
+to a distributed hashtable model in Cassandra.
+
+Let's use an example that is complex enough
+to show the various data structures and design patterns, but not
+something that will bog you down with details. Also, a domain that’s
+familiar to everyone will allow you to concentrate on how to work with
+Cassandra, not on what the application domain is all about.
+
+For example, let's use a domain that is easily understood and that
+everyone can relate to: making hotel reservations.
+
+The conceptual domain includes hotels, guests that stay in the hotels, a
+collection of rooms for each hotel, the rates and availability of those
+rooms, and a record of reservations booked for guests. Hotels typically
+also maintain a collection of “points of interest,” which are parks,
+museums, shopping galleries, monuments, or other places near the hotel
+that guests might want to visit during their stay. Both hotels and
+points of interest need to maintain geolocation data so that they can be
+found on maps for mashups, and to calculate distances.
+
+The conceptual domain is depicted below using the entity–relationship
+model popularized by Peter Chen. This simple diagram represents the
+entities in the domain with rectangles, and attributes of those entities
+with ovals. Attributes that represent unique identifiers for items are
+underlined. Relationships between entities are represented as diamonds,
+and the connectors between the relationship and each entity show the
+multiplicity of the connection.
+
+.. image:: images/data_modeling_hotel_erd.png
+
+Obviously, in the real world, there would be many more considerations
+and much more complexity. For example, hotel rates are notoriously
+dynamic, and calculating them involves a wide array of factors. Here
+you’re defining something complex enough to be interesting and touch on
+the important points, but simple enough to maintain the focus on
+learning Cassandra.
+
+*Material adapted from Cassandra, The Definitive Guide. Published by
+O'Reilly Media, Inc. Copyright © 2020 Jeff Carpenter, Eben Hewitt.
+All rights reserved. Used with permission.*
\ No newline at end of file
diff --git a/doc/source/data_modeling/data_modeling_logical.rst b/doc/source/data_modeling/data_modeling_logical.rst
new file mode 100644
index 0000000..27fa4be
--- /dev/null
+++ b/doc/source/data_modeling/data_modeling_logical.rst
@@ -0,0 +1,219 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements. See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership. The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License. You may obtain a copy of the License at
+..
+.. http://www.apache.org/licenses/LICENSE-2.0
+..
+.. Unless required by applicable law or agreed to in writing, software
+.. distributed under the License is distributed on an "AS IS" BASIS,
+.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+.. See the License for the specific language governing permissions and
+.. limitations under the License.
+
+Logical Data Modeling
+=====================
+
+Now that you have defined your queries, you’re ready to begin designing
+Cassandra tables. First, create a logical model containing a table
+for each query, capturing entities and relationships from the conceptual
+model.
+
+To name each table, you’ll identify the primary entity type for which you
+are querying and use that to start the entity name. If you are querying
+by attributes of other related entities, append those to the table
+name, separated with ``_by_``. For example, ``hotels_by_poi``.
+
+Next, you identify the primary key for the table, adding partition key
+columns based on the required query attributes, and clustering columns
+in order to guarantee uniqueness and support desired sort ordering.
+
+The design of the primary key is extremely important, as it will
+determine how much data will be stored in each partition and how that
+data is organized on disk, which in turn will affect how quickly
+Cassandra processes reads.
+
+Complete each table by adding any additional attributes identified by
+the query. If any of these additional attributes are the same for every
+instance of the partition key, mark the column as static.
+
+Now that was a pretty quick description of a fairly involved process, so
+it will be worthwhile to work through a detailed example. First,
+let’s introduce a notation that you can use to represent logical
+models.
+
+Several individuals within the Cassandra community have proposed
+notations for capturing data models in diagrammatic form. This document
+uses a notation popularized by Artem Chebotko which provides a simple,
+informative way to visualize the relationships between queries and
+tables in your designs. This figure shows the Chebotko notation for a
+logical data model.
+
+.. image:: images/data_modeling_chebotko_logical.png
+
+Each table is shown with its title and a list of columns. Primary key
+columns are identified via symbols such as **K** for partition key
+columns and **C**\ ↑ or **C**\ ↓ to represent clustering columns. Lines
+are shown entering tables or between tables to indicate the queries that
+each table is designed to support.
+
+Hotel Logical Data Model
+------------------------
+
+The figure below shows a Chebotko logical data model for the queries
+involving hotels, points of interest, rooms, and amenities. One thing you'll
+notice immediately is that the Cassandra design doesn’t include dedicated
+tables for rooms or amenities, as you had in the relational design. This
+is because the workflow didn’t identify any queries requiring this
+direct access.
+
+.. image:: images/data_modeling_hotel_logical.png
+
+Let’s explore the details of each of these tables.
+
+The first query Q1 is to find hotels near a point of interest, so you’ll
+call this table ``hotels_by_poi``. Searching by a named point of
+interest is a clue that the point of interest should be a part
+of the primary key. Let’s reference the point of interest by name,
+because according to the workflow that is how users will start their
+search.
+
+You’ll note that you certainly could have more than one hotel near a
+given point of interest, so you’ll need another component in the primary
+key in order to make sure you have a unique partition for each hotel. So
+you add the hotel key as a clustering column.
+
+An important consideration in designing your table’s primary key is
+making sure that it defines a unique data element. Otherwise you run the
+risk of accidentally overwriting data.
+
+Now for the second query (Q2), you’ll need a table to get information
+about a specific hotel. One approach would have been to put all of the
+attributes of a hotel in the ``hotels_by_poi`` table, but you added
+only those attributes that were required by the application workflow.
+
+From the workflow diagram, you know that the ``hotels_by_poi`` table is
+used to display a list of hotels with basic information on each hotel,
+and the application knows the unique identifiers of the hotels returned.
+When the user selects a hotel to view details, you can then use Q2, which
+is used to obtain details about the hotel. Because you already have the
+``hotel_id`` from Q1, you use that as a reference to the hotel you’re
+looking for. Therefore the second table is just called ``hotels``.
+
+Another option would have been to store a set of ``poi_names`` in the
+hotels table. This is an equally valid approach. You’ll learn through
+experience which approach is best for your application.
+
+Q3 is just a reverse of Q1—looking for points of interest near a hotel,
+rather than hotels near a point of interest. This time, however, you need
+to access the details of each point of interest, as represented by the
+``pois_by_hotel`` table. As previously, you add the point of
+interest name as a clustering key to guarantee uniqueness.
+
+At this point, let’s now consider how to support query Q4 to help the
+user find available rooms at a selected hotel for the nights they are
+interested in staying. Note that this query involves both a start date
+and an end date. Because you’re querying over a range instead of a single
+date, you know that you’ll need to use the date as a clustering key.
+Use the ``hotel_id`` as a primary key to group room data for each hotel
+on a single partition, which should help searches be super fast. Let’s
+call this the ``available_rooms_by_hotel_date`` table.
+
+To support searching over a range, use :ref:`clustering columns
+<clustering-columns>` to store
+attributes that you need to access in a range query. Remember that the
+order of the clustering columns is important.
+
+The design of the ``available_rooms_by_hotel_date`` table is an instance
+of the **wide partition** pattern. This
+pattern is sometimes called the **wide row** pattern when discussing
+databases that support similar models, but wide partition is a more
+accurate description from a Cassandra perspective. The essence of the
+pattern is to group multiple related rows in a partition in order to
+support fast access to multiple rows within the partition in a single
+query.
+
+In order to round out the shopping portion of the data model, add the
+``amenities_by_room`` table to support Q5. This will allow users to
+view the amenities of one of the rooms that is available for the desired
+stay dates.
+
+Reservation Logical Data Model
+------------------------------
+
+Now let's switch gears to look at the reservation queries. The figure
+shows a logical data model for reservations. You’ll notice that these
+tables represent a denormalized design; the same data appears in
+multiple tables, with differing keys.
+
+.. image:: images/data_modeling_reservation_logical.png
+
+In order to satisfy Q6, the ``reservations_by_guest`` table can be used
+to look up the reservation by guest name. You could envision query Q7
+being used on behalf of a guest on a self-serve website or a call center
+agent trying to assist the guest. Because the guest name might not be
+unique, you include the guest ID here as a clustering column as well.
+
+Q8 and Q9 in particular help to remind you to create queries
+that support various stakeholders of the application, not just customers
+but staff as well, and perhaps even the analytics team, suppliers, and so
+on.
+
+The hotel staff might wish to see a record of upcoming reservations by
+date in order to get insight into how the hotel is performing, such as
+what dates the hotel is sold out or undersold. Q8 supports the retrieval
+of reservations for a given hotel by date.
+
+Finally, you create a ``guests`` table. This provides a single
+location that used to store guest information. In this case, you specify a
+separate unique identifier for guest records, as it is not uncommon
+for guests to have the same name. In many organizations, a customer
+database such as the ``guests`` table would be part of a separate
+customer management application, which is why other guest
+access patterns were omitted from the example.
+
+
+Patterns and Anti-Patterns
+--------------------------
+
+As with other types of software design, there are some well-known
+patterns and anti-patterns for data modeling in Cassandra. You’ve already
+used one of the most common patterns in this hotel model—the wide
+partition pattern.
+
+The **time series** pattern is an extension of the wide partition
+pattern. In this pattern, a series of measurements at specific time
+intervals are stored in a wide partition, where the measurement time is
+used as part of the partition key. This pattern is frequently used in
+domains including business analysis, sensor data management, and
+scientific experiments.
+
+The time series pattern is also useful for data other than measurements.
+Consider the example of a banking application. You could store each
+customer’s balance in a row, but that might lead to a lot of read and
+write contention as various customers check their balance or make
+transactions. You’d probably be tempted to wrap a transaction around
+writes just to protect the balance from being updated in error. In
+contrast, a time series–style design would store each transaction as a
+timestamped row and leave the work of calculating the current balance to
+the application.
+
+One design trap that many new users fall into is attempting to use
+Cassandra as a queue. Each item in the queue is stored with a timestamp
+in a wide partition. Items are appended to the end of the queue and read
+from the front, being deleted after they are read. This is a design that
+seems attractive, especially given its apparent similarity to the time
+series pattern. The problem with this approach is that the deleted items
+are now :ref:`tombstones <asynch-deletes>` that Cassandra must scan past
+in order to read from the front of the queue. Over time, a growing number
+of tombstones begins to degrade read performance.
+
+The queue anti-pattern serves as a reminder that any design that relies
+on the deletion of data is potentially a poorly performing design.
+
+*Material adapted from Cassandra, The Definitive Guide. Published by
+O'Reilly Media, Inc. Copyright © 2020 Jeff Carpenter, Eben Hewitt.
+All rights reserved. Used with permission.*
\ No newline at end of file
diff --git a/doc/source/data_modeling/data_modeling_physical.rst b/doc/source/data_modeling/data_modeling_physical.rst
new file mode 100644
index 0000000..7584004
--- /dev/null
+++ b/doc/source/data_modeling/data_modeling_physical.rst
@@ -0,0 +1,117 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements. See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership. The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License. You may obtain a copy of the License at
+..
+.. http://www.apache.org/licenses/LICENSE-2.0
+..
+.. Unless required by applicable law or agreed to in writing, software
+.. distributed under the License is distributed on an "AS IS" BASIS,
+.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+.. See the License for the specific language governing permissions and
+.. limitations under the License.
+
+Physical Data Modeling
+======================
+
+Once you have a logical data model defined, creating the physical model
+is a relatively simple process.
+
+You walk through each of the logical model tables, assigning types to
+each item. You can use any valid :ref:`CQL data type <data-types>`,
+including the basic types, collections, and user-defined types. You may
+identify additional user-defined types that can be created to simplify
+your design.
+
+After you’ve assigned data types, you analyze the model by performing
+size calculations and testing out how the model works. You may make some
+adjustments based on your findings. Once again let's cover the data
+modeling process in more detail by working through an example.
+
+Before getting started, let’s look at a few additions to the Chebotko
+notation for physical data models. To draw physical models, you need to
+be able to add the typing information for each column. This figure
+shows the addition of a type for each column in a sample table.
+
+.. image:: images/data_modeling_chebotko_physical.png
+
+The figure includes a designation of the keyspace containing each table
+and visual cues for columns represented using collections and
+user-defined types. Note the designation of static columns and
+secondary index columns. There is no restriction on assigning these as
+part of a logical model, but they are typically more of a physical data
+modeling concern.
+
+Hotel Physical Data Model
+-------------------------
+
+Now let’s get to work on the physical model. First, you need keyspaces
+to contain the tables. To keep the design relatively simple, create a
+``hotel`` keyspace to contain tables for hotel and availability
+data, and a ``reservation`` keyspace to contain tables for reservation
+and guest data. In a real system, you might divide the tables across even
+more keyspaces in order to separate concerns.
+
+For the ``hotels`` table, use Cassandra’s ``text`` type to
+represent the hotel’s ``id``. For the address, create an
+``address`` user defined type. Use the ``text`` type to represent the
+phone number, as there is considerable variance in the formatting of
+numbers between countries.
+
+While it would make sense to use the ``uuid`` type for attributes such
+as the ``hotel_id``, this document uses mostly ``text`` attributes as
+identifiers, to keep the samples simple and readable. For example, a
+common convention in the hospitality industry is to reference properties
+by short codes like "AZ123" or "NY229". This example uses these values
+for ``hotel_ids``, while acknowledging they are not necessarily globally
+unique.
+
+You’ll find that it’s often helpful to use unique IDs to uniquely
+reference elements, and to use these ``uuids`` as references in tables
+representing other entities. This helps to minimize coupling between
+different entity types. This may prove especially effective if you are
+using a microservice architectural style for your application, in which
+there are separate services responsible for each entity type.
+
+As you work to create physical representations of various tables in the
+logical hotel data model, you use the same approach. The resulting design
+is shown in this figure:
+
+.. image:: images/data_modeling_hotel_physical.png
+
+Note that the ``address`` type is also included in the design. It
+is designated with an asterisk to denote that it is a user-defined type,
+and has no primary key columns identified. This type is used in
+the ``hotels`` and ``hotels_by_poi`` tables.
+
+User-defined types are frequently used to help reduce duplication of
+non-primary key columns, as was done with the ``address``
+user-defined type. This can reduce complexity in the design.
+
+Remember that the scope of a UDT is the keyspace in which it is defined.
+To use ``address`` in the ``reservation`` keyspace defined below
+design, you’ll have to declare it again. This is just one of the many
+trade-offs you have to make in data model design.
+
+Reservation Physical Data Model
+-------------------------------
+
+Now, let’s examine reservation tables in the design.
+Remember that the logical model contained three denormalized tables to
+support queries for reservations by confirmation number, guest, and
+hotel and date. For the first iteration of your physical data model
+design, assume you're going to manage this denormalization
+manually. Note that this design could be revised to use Cassandra’s
+(experimental) materialized view feature.
+
+.. image:: images/data_modeling_reservation_physical.png
+
+Note that the ``address`` type is reproduced in this keyspace and
+``guest_id`` is modeled as a ``uuid`` type in all of the tables.
+
+*Material adapted from Cassandra, The Definitive Guide. Published by
+O'Reilly Media, Inc. Copyright © 2020 Jeff Carpenter, Eben Hewitt.
+All rights reserved. Used with permission.*
\ No newline at end of file
diff --git a/doc/source/data_modeling/data_modeling_queries.rst b/doc/source/data_modeling/data_modeling_queries.rst
new file mode 100644
index 0000000..d011994
--- /dev/null
+++ b/doc/source/data_modeling/data_modeling_queries.rst
@@ -0,0 +1,85 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements. See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership. The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License. You may obtain a copy of the License at
+..
+.. http://www.apache.org/licenses/LICENSE-2.0
+..
+.. Unless required by applicable law or agreed to in writing, software
+.. distributed under the License is distributed on an "AS IS" BASIS,
+.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+.. See the License for the specific language governing permissions and
+.. limitations under the License.
+
+Defining Application Queries
+============================
+
+Let’s try the query-first approach to start designing the data model for
+a hotel application. The user interface design for the application is
+often a great artifact to use to begin identifying queries. Let’s assume
+that you’ve talked with the project stakeholders and your UX designers
+have produced user interface designs or wireframes for the key use
+cases. You’ll likely have a list of shopping queries like the following:
+
+- Q1. Find hotels near a given point of interest.
+
+- Q2. Find information about a given hotel, such as its name and
+ location.
+
+- Q3. Find points of interest near a given hotel.
+
+- Q4. Find an available room in a given date range.
+
+- Q5. Find the rate and amenities for a room.
+
+It is often helpful to be able to refer
+to queries by a shorthand number rather that explaining them in full.
+The queries listed here are numbered Q1, Q2, and so on, which is how they
+are referenced in diagrams throughout the example.
+
+Now if the application is to be a success, you’ll certainly want
+customers to be able to book reservations at hotels. This includes
+steps such as selecting an available room and entering their guest
+information. So clearly you will also need some queries that address the
+reservation and guest entities from the conceptual data model. Even
+here, however, you’ll want to think not only from the customer
+perspective in terms of how the data is written, but also in terms of
+how the data will be queried by downstream use cases.
+
+You natural tendency as might be to focus first on
+designing the tables to store reservation and guest records, and only
+then start thinking about the queries that would access them. You may
+have felt a similar tension already when discussing the
+shopping queries before, thinking “but where did the hotel and point of
+interest data come from?” Don’t worry, you will see soon enough.
+Here are some queries that describe how users will access
+reservations:
+
+- Q6. Lookup a reservation by confirmation number.
+
+- Q7. Lookup a reservation by hotel, date, and guest name.
+
+- Q8. Lookup all reservations by guest name.
+
+- Q9. View guest details.
+
+All of the queries are shown in the context of the workflow of the
+application in the figure below. Each box on the diagram represents a
+step in the application workflow, with arrows indicating the flows
+between steps and the associated query. If you’ve modeled the application
+well, each step of the workflow accomplishes a task that “unlocks”
+subsequent steps. For example, the “View hotels near POI” task helps
+the application learn about several hotels, including their unique keys.
+The key for a selected hotel may be used as part of Q2, in order to
+obtain detailed description of the hotel. The act of booking a room
+creates a reservation record that may be accessed by the guest and
+hotel staff at a later time through various additional queries.
+
+.. image:: images/data_modeling_hotel_queries.png
+
+*Material adapted from Cassandra, The Definitive Guide. Published by
+O'Reilly Media, Inc. Copyright © 2020 Jeff Carpenter, Eben Hewitt.
+All rights reserved. Used with permission.*
\ No newline at end of file
diff --git a/doc/source/data_modeling/data_modeling_rdbms.rst b/doc/source/data_modeling/data_modeling_rdbms.rst
new file mode 100644
index 0000000..7d67d69
--- /dev/null
+++ b/doc/source/data_modeling/data_modeling_rdbms.rst
@@ -0,0 +1,171 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements. See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership. The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License. You may obtain a copy of the License at
+..
+.. http://www.apache.org/licenses/LICENSE-2.0
+..
+.. Unless required by applicable law or agreed to in writing, software
+.. distributed under the License is distributed on an "AS IS" BASIS,
+.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+.. See the License for the specific language governing permissions and
+.. limitations under the License.
+
+RDBMS Design
+============
+
+When you set out to build a new data-driven application that will use a
+relational database, you might start by modeling the domain as a set of
+properly normalized tables and use foreign keys to reference related
+data in other tables.
+
+The figure below shows how you might represent the data storage for your application
+using a relational database model. The relational model includes a
+couple of “join” tables in order to realize the many-to-many
+relationships from the conceptual model of hotels-to-points of interest,
+rooms-to-amenities, rooms-to-availability, and guests-to-rooms (via a
+reservation).
+
+.. image:: images/data_modeling_hotel_relational.png
+
+.. design_differences_between_rdbms_and_cassandra
+
+Design Differences Between RDBMS and Cassandra
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Let’s take a minute to highlight some of the key differences in doing
+ata modeling for Cassandra versus a relational database.
+
+No joins
+~~~~~~~~
+
+You cannot perform joins in Cassandra. If you have designed a data model
+and find that you need something like a join, you’ll have to either do
+the work on the client side, or create a denormalized second table that
+represents the join results for you. This latter option is preferred in
+Cassandra data modeling. Performing joins on the client should be a very
+rare case; you really want to duplicate (denormalize) the data instead.
+
+No referential integrity
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+Although Cassandra supports features such as lightweight transactions
+and batches, Cassandra itself has no concept of referential integrity
+across tables. In a relational database, you could specify foreign keys
+in a table to reference the primary key of a record in another table.
+But Cassandra does not enforce this. It is still a common design
+requirement to store IDs related to other entities in your tables, but
+operations such as cascading deletes are not available.
+
+Denormalization
+~~~~~~~~~~~~~~~
+
+In relational database design, you are often taught the importance of
+normalization. This is not an advantage when working with Cassandra
+because it performs best when the data model is denormalized. It is
+often the case that companies end up denormalizing data in relational
+databases as well. There are two common reasons for this. One is
+performance. Companies simply can’t get the performance they need when
+they have to do so many joins on years’ worth of data, so they
+denormalize along the lines of known queries. This ends up working, but
+goes against the grain of how relational databases are intended to be
+designed, and ultimately makes one question whether using a relational
+database is the best approach in these circumstances.
+
+A second reason that relational databases get denormalized on purpose is
+a business document structure that requires retention. That is, you have
+an enclosing table that refers to a lot of external tables whose data
+could change over time, but you need to preserve the enclosing document
+as a snapshot in history. The common example here is with invoices. You
+already have customer and product tables, and you’d think that you could
+just make an invoice that refers to those tables. But this should never
+be done in practice. Customer or price information could change, and
+then you would lose the integrity of the invoice document as it was on
+the invoice date, which could violate audits, reports, or laws, and
+cause other problems.
+
+In the relational world, denormalization violates Codd’s normal forms,
+and you try to avoid it. But in Cassandra, denormalization is, well,
+perfectly normal. It’s not required if your data model is simple. But
+don’t be afraid of it.
+
+Historically, denormalization in Cassandra has required designing and
+managing multiple tables using techniques described in this documentation.
+Beginning with the 3.0 release, Cassandra provides a feature known
+as :ref:`materialized views <materialized-views>`
+which allows you to create multiple denormalized
+views of data based on a base table design. Cassandra manages
+materialized views on the server, including the work of keeping the
+views in sync with the table.
+
+Query-first design
+~~~~~~~~~~~~~~~~~~
+
+Relational modeling, in simple terms, means that you start from the
+conceptual domain and then represent the nouns in the domain in tables.
+You then assign primary keys and foreign keys to model relationships.
+When you have a many-to-many relationship, you create the join tables
+that represent just those keys. The join tables don’t exist in the real
+world, and are a necessary side effect of the way relational models
+work. After you have all your tables laid out, you can start writing
+queries that pull together disparate data using the relationships
+defined by the keys. The queries in the relational world are very much
+secondary. It is assumed that you can always get the data you want as
+long as you have your tables modeled properly. Even if you have to use
+several complex subqueries or join statements, this is usually true.
+
+By contrast, in Cassandra you don’t start with the data model; you start
+with the query model. Instead of modeling the data first and then
+writing queries, with Cassandra you model the queries and let the data
+be organized around them. Think of the most common query paths your
+application will use, and then create the tables that you need to
+support them.
+
+Detractors have suggested that designing the queries first is overly
+constraining on application design, not to mention database modeling.
+But it is perfectly reasonable to expect that you should think hard
+about the queries in your application, just as you would, presumably,
+think hard about your relational domain. You may get it wrong, and then
+you’ll have problems in either world. Or your query needs might change
+over time, and then you’ll have to work to update your data set. But
+this is no different from defining the wrong tables, or needing
+additional tables, in an RDBMS.
+
+Designing for optimal storage
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+In a relational database, it is frequently transparent to the user how
+tables are stored on disk, and it is rare to hear of recommendations
+about data modeling based on how the RDBMS might store tables on disk.
+However, that is an important consideration in Cassandra. Because
+Cassandra tables are each stored in separate files on disk, it’s
+important to keep related columns defined together in the same table.
+
+A key goal that you will see as you begin creating data models in
+Cassandra is to minimize the number of partitions that must be searched
+in order to satisfy a given query. Because the partition is a unit of
+storage that does not get divided across nodes, a query that searches a
+single partition will typically yield the best performance.
+
+Sorting is a design decision
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+In an RDBMS, you can easily change the order in which records are
+returned to you by using ``ORDER BY`` in your query. The default sort
+order is not configurable; by default, records are returned in the order
+in which they are written. If you want to change the order, you just
+modify your query, and you can sort by any list of columns.
+
+In Cassandra, however, sorting is treated differently; it is a design
+decision. The sort order available on queries is fixed, and is
+determined entirely by the selection of clustering columns you supply in
+the ``CREATE TABLE`` command. The CQL ``SELECT`` statement does support
+``ORDER BY`` semantics, but only in the order specified by the
+clustering columns.
+
+*Material adapted from Cassandra, The Definitive Guide. Published by
+O'Reilly Media, Inc. Copyright © 2020 Jeff Carpenter, Eben Hewitt.
+All rights reserved. Used with permission.*
\ No newline at end of file
diff --git a/doc/source/data_modeling/data_modeling_refining.rst b/doc/source/data_modeling/data_modeling_refining.rst
new file mode 100644
index 0000000..13a276e
--- /dev/null
+++ b/doc/source/data_modeling/data_modeling_refining.rst
@@ -0,0 +1,218 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements. See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership. The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License. You may obtain a copy of the License at
+..
+.. http://www.apache.org/licenses/LICENSE-2.0
+..
+.. Unless required by applicable law or agreed to in writing, software
+.. distributed under the License is distributed on an "AS IS" BASIS,
+.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+.. See the License for the specific language governing permissions and
+.. limitations under the License.
+
+.. role:: raw-latex(raw)
+ :format: latex
+..
+
+Evaluating and Refining Data Models
+===================================
+
+Once you’ve created a physical model, there are some steps you’ll want
+to take to evaluate and refine table designs to help ensure optimal
+performance.
+
+Calculating Partition Size
+--------------------------
+
+The first thing that you want to look for is whether your tables will have
+partitions that will be overly large, or to put it another way, too
+wide. Partition size is measured by the number of cells (values) that
+are stored in the partition. Cassandra’s hard limit is 2 billion cells
+per partition, but you’ll likely run into performance issues before
+reaching that limit.
+
+In order to calculate the size of partitions, use the following
+formula:
+
+.. math:: N_v = N_r (N_c - N_{pk} - N_s) + N_s
+
+The number of values (or cells) in the partition (N\ :sub:`v`) is equal to
+the number of static columns (N\ :sub:`s`) plus the product of the number
+of rows (N\ :sub:`r`) and the number of of values per row. The number of
+values per row is defined as the number of columns (N\ :sub:`c`) minus the
+number of primary key columns (N\ :sub:`pk`) and static columns
+(N\ :sub:`s`).
+
+The number of columns tends to be relatively static, although it
+is possible to alter tables at runtime. For this reason, a
+primary driver of partition size is the number of rows in the partition.
+This is a key factor that you must consider in determining whether a
+partition has the potential to get too large. Two billion values sounds
+like a lot, but in a sensor system where tens or hundreds of values are
+measured every millisecond, the number of values starts to add up pretty
+fast.
+
+Let’s take a look at one of the tables to analyze the partition size.
+Because it has a wide partition design with one partition per hotel,
+look at the ``available_rooms_by_hotel_date`` table. The table has
+four columns total (N\ :sub:`c` = 4), including three primary key columns
+(N\ :sub:`pk` = 3) and no static columns (N\ :sub:`s` = 0). Plugging these
+values into the formula, the result is:
+
+.. math:: N_v = N_r (4 - 3 - 0) + 0 = 1N_r
+
+Therefore the number of values for this table is equal to the number of
+rows. You still need to determine a number of rows. To do this, make
+estimates based on the application design. The table is
+storing a record for each room, in each of hotel, for every night.
+Let's assume the system will be used to store two years of
+inventory at a time, and there are 5,000 hotels in the system, with an
+average of 100 rooms in each hotel.
+
+Since there is a partition for each hotel, the estimated number of rows
+per partition is as follows:
+
+.. math:: N_r = 100 rooms/hotel \times 730 days = 73,000 rows
+
+This relatively small number of rows per partition is not going to get
+you in too much trouble, but if you start storing more dates of inventory,
+or don’t manage the size of the inventory well using TTL, you could start
+having issues. You still might want to look at breaking up this large
+partition, which you'll see how to do shortly.
+
+When performing sizing calculations, it is tempting to assume the
+nominal or average case for variables such as the number of rows.
+Consider calculating the worst case as well, as these sorts of
+predictions have a way of coming true in successful systems.
+
+Calculating Size on Disk
+------------------------
+
+In addition to calculating the size of a partition, it is also an
+excellent idea to estimate the amount of disk space that will be
+required for each table you plan to store in the cluster. In order to
+determine the size, use the following formula to determine the size
+S\ :sub:`t` of a partition:
+
+.. math:: S_t = \displaystyle\sum_i sizeOf\big (c_{k_i}\big) + \displaystyle\sum_j sizeOf\big(c_{s_j}\big) + N_r\times \bigg(\displaystyle\sum_k sizeOf\big(c_{r_k}\big) + \displaystyle\sum_l sizeOf\big(c_{c_l}\big)\bigg) +
+
+.. math:: N_v\times sizeOf\big(t_{avg}\big)
+
+This is a bit more complex than the previous formula, but let's break it
+down a bit at a time. Let’s take a look at the notation first:
+
+- In this formula, c\ :sub:`k` refers to partition key columns,
+ c\ :sub:`s` to static columns, c\ :sub:`r` to regular columns, and
+ c\ :sub:`c` to clustering columns.
+
+- The term t\ :sub:`avg` refers to the average number of bytes of
+ metadata stored per cell, such as timestamps. It is typical to use an
+ estimate of 8 bytes for this value.
+
+- You'll recognize the number of rows N\ :sub:`r` and number of values
+ N\ :sub:`v` from previous calculations.
+
+- The **sizeOf()** function refers to the size in bytes of the CQL data
+ type of each referenced column.
+
+The first term asks you to sum the size of the partition key columns. For
+this example, the ``available_rooms_by_hotel_date`` table has a single
+partition key column, the ``hotel_id``, which is of type
+``text``. Assuming that hotel identifiers are simple 5-character codes,
+you have a 5-byte value, so the sum of the partition key column sizes is
+5 bytes.
+
+The second term asks you to sum the size of the static columns. This table
+has no static columns, so the size is 0 bytes.
+
+The third term is the most involved, and for good reason—it is
+calculating the size of the cells in the partition. Sum the size of
+the clustering columns and regular columns. The two clustering columns
+are the ``date``, which is 4 bytes, and the ``room_number``,
+which is a 2-byte short integer, giving a sum of 6 bytes.
+There is only a single regular column, the boolean ``is_available``,
+which is 1 byte in size. Summing the regular column size
+(1 byte) plus the clustering column size (6 bytes) gives a total of 7
+bytes. To finish up the term, multiply this value by the number of
+rows (73,000), giving a result of 511,000 bytes (0.51 MB).
+
+The fourth term is simply counting the metadata that that Cassandra
+stores for each cell. In the storage format used by Cassandra 3.0 and
+later, the amount of metadata for a given cell varies based on the type
+of data being stored, and whether or not custom timestamp or TTL values
+are specified for individual cells. For this table, reuse the number
+of values from the previous calculation (73,000) and multiply by 8,
+which gives 0.58 MB.
+
+Adding these terms together, you get a final estimate:
+
+.. math:: Partition size = 16 bytes + 0 bytes + 0.51 MB + 0.58 MB = 1.1 MB
+
+This formula is an approximation of the actual size of a partition on
+disk, but is accurate enough to be quite useful. Remembering that the
+partition must be able to fit on a single node, it looks like the table
+design will not put a lot of strain on disk storage.
+
+Cassandra’s storage engine was re-implemented for the 3.0 release,
+including a new format for SSTable files. The previous format stored a
+separate copy of the clustering columns as part of the record for each
+cell. The newer format eliminates this duplication, which reduces the
+size of stored data and simplifies the formula for computing that size.
+
+Keep in mind also that this estimate only counts a single replica of
+data. You will need to multiply the value obtained here by the number of
+partitions and the number of replicas specified by the keyspace’s
+replication strategy in order to determine the total required total
+capacity for each table. This will come in handy when you
+plan your cluster.
+
+Breaking Up Large Partitions
+----------------------------
+
+As discussed previously, the goal is to design tables that can provide
+the data you need with queries that touch a single partition, or failing
+that, the minimum possible number of partitions. However, as shown in
+the examples, it is quite possible to design wide
+partition-style tables that approach Cassandra’s built-in limits.
+Performing sizing analysis on tables may reveal partitions that are
+potentially too large, either in number of values, size on disk, or
+both.
+
+The technique for splitting a large partition is straightforward: add an
+additional column to the partition key. In most cases, moving one of the
+existing columns into the partition key will be sufficient. Another
+option is to introduce an additional column to the table to act as a
+sharding key, but this requires additional application logic.
+
+Continuing to examine the available rooms example, if you add the ``date``
+column to the partition key for the ``available_rooms_by_hotel_date``
+table, each partition would then represent the availability of rooms
+at a specific hotel on a specific date. This will certainly yield
+partitions that are significantly smaller, perhaps too small, as the
+data for consecutive days will likely be on separate nodes.
+
+Another technique known as **bucketing** is often used to break the data
+into moderate-size partitions. For example, you could bucketize the
+``available_rooms_by_hotel_date`` table by adding a ``month`` column to
+the partition key, perhaps represented as an integer. The comparision
+with the original design is shown in the figure below. While the
+``month`` column is partially duplicative of the ``date``, it provides
+a nice way of grouping related data in a partition that will not get
+too large.
+
+.. image:: images/data_modeling_hotel_bucketing.png
+
+If you really felt strongly about preserving a wide partition design, you
+could instead add the ``room_id`` to the partition key, so that each
+partition would represent the availability of the room across all
+dates. Because there was no query identified that involves searching
+availability of a specific room, the first or second design approach
+is most suitable to the application needs.
+
+*Material adapted from Cassandra, The Definitive Guide. Published by
+O'Reilly Media, Inc. Copyright © 2020 Jeff Carpenter, Eben Hewitt.
+All rights reserved. Used with permission.*
\ No newline at end of file
diff --git a/doc/source/data_modeling/data_modeling_schema.rst b/doc/source/data_modeling/data_modeling_schema.rst
new file mode 100644
index 0000000..1876ec3
--- /dev/null
+++ b/doc/source/data_modeling/data_modeling_schema.rst
@@ -0,0 +1,144 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements. See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership. The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License. You may obtain a copy of the License at
+..
+.. http://www.apache.org/licenses/LICENSE-2.0
+..
+.. Unless required by applicable law or agreed to in writing, software
+.. distributed under the License is distributed on an "AS IS" BASIS,
+.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+.. See the License for the specific language governing permissions and
+.. limitations under the License.
+
+.. highlight:: cql
+
+Defining Database Schema
+========================
+
+Once you have finished evaluating and refining the physical model, you’re
+ready to implement the schema in CQL. Here is the schema for the
+``hotel`` keyspace, using CQL’s comment feature to document the query
+pattern supported by each table::
+
+ CREATE KEYSPACE hotel WITH replication =
+ {‘class’: ‘SimpleStrategy’, ‘replication_factor’ : 3};
+
+ CREATE TYPE hotel.address (
+ street text,
+ city text,
+ state_or_province text,
+ postal_code text,
+ country text );
+
+ CREATE TABLE hotel.hotels_by_poi (
+ poi_name text,
+ hotel_id text,
+ name text,
+ phone text,
+ address frozen<address>,
+ PRIMARY KEY ((poi_name), hotel_id) )
+ WITH comment = ‘Q1. Find hotels near given poi’
+ AND CLUSTERING ORDER BY (hotel_id ASC) ;
+
+ CREATE TABLE hotel.hotels (
+ id text PRIMARY KEY,
+ name text,
+ phone text,
+ address frozen<address>,
+ pois set )
+ WITH comment = ‘Q2. Find information about a hotel’;
+
+ CREATE TABLE hotel.pois_by_hotel (
+ poi_name text,
+ hotel_id text,
+ description text,
+ PRIMARY KEY ((hotel_id), poi_name) )
+ WITH comment = Q3. Find pois near a hotel’;
+
+ CREATE TABLE hotel.available_rooms_by_hotel_date (
+ hotel_id text,
+ date date,
+ room_number smallint,
+ is_available boolean,
+ PRIMARY KEY ((hotel_id), date, room_number) )
+ WITH comment = ‘Q4. Find available rooms by hotel date’;
+
+ CREATE TABLE hotel.amenities_by_room (
+ hotel_id text,
+ room_number smallint,
+ amenity_name text,
+ description text,
+ PRIMARY KEY ((hotel_id, room_number), amenity_name) )
+ WITH comment = ‘Q5. Find amenities for a room’;
+
+
+Notice that the elements of the partition key are surrounded
+with parentheses, even though the partition key consists
+of the single column ``poi_name``. This is a best practice that makes
+the selection of partition key more explicit to others reading your CQL.
+
+Similarly, here is the schema for the ``reservation`` keyspace::
+
+ CREATE KEYSPACE reservation WITH replication = {‘class’:
+ ‘SimpleStrategy’, ‘replication_factor’ : 3};
+
+ CREATE TYPE reservation.address (
+ street text,
+ city text,
+ state_or_province text,
+ postal_code text,
+ country text );
+
+ CREATE TABLE reservation.reservations_by_confirmation (
+ confirm_number text,
+ hotel_id text,
+ start_date date,
+ end_date date,
+ room_number smallint,
+ guest_id uuid,
+ PRIMARY KEY (confirm_number) )
+ WITH comment = ‘Q6. Find reservations by confirmation number’;
+
+ CREATE TABLE reservation.reservations_by_hotel_date (
+ hotel_id text,
+ start_date date,
+ end_date date,
+ room_number smallint,
+ confirm_number text,
+ guest_id uuid,
+ PRIMARY KEY ((hotel_id, start_date), room_number) )
+ WITH comment = ‘Q7. Find reservations by hotel and date’;
+
+ CREATE TABLE reservation.reservations_by_guest (
+ guest_last_name text,
+ hotel_id text,
+ start_date date,
+ end_date date,
+ room_number smallint,
+ confirm_number text,
+ guest_id uuid,
+ PRIMARY KEY ((guest_last_name), hotel_id) )
+ WITH comment = ‘Q8. Find reservations by guest name’;
+
+ CREATE TABLE reservation.guests (
+ guest_id uuid PRIMARY KEY,
+ first_name text,
+ last_name text,
+ title text,
+ emails set,
+ phone_numbers list,
+ addresses map<text,
+ frozen<address>,
+ confirm_number text )
+ WITH comment = ‘Q9. Find guest by ID’;
+
+You now have a complete Cassandra schema for storing data for a hotel
+application.
+
+*Material adapted from Cassandra, The Definitive Guide. Published by
+O'Reilly Media, Inc. Copyright © 2020 Jeff Carpenter, Eben Hewitt.
+All rights reserved. Used with permission.*
\ No newline at end of file
diff --git a/doc/source/data_modeling/data_modeling_tools.rst b/doc/source/data_modeling/data_modeling_tools.rst
new file mode 100644
index 0000000..46fad33
--- /dev/null
+++ b/doc/source/data_modeling/data_modeling_tools.rst
@@ -0,0 +1,64 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements. See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership. The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License. You may obtain a copy of the License at
+..
+.. http://www.apache.org/licenses/LICENSE-2.0
+..
+.. Unless required by applicable law or agreed to in writing, software
+.. distributed under the License is distributed on an "AS IS" BASIS,
+.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+.. See the License for the specific language governing permissions and
+.. limitations under the License.
+
+Cassandra Data Modeling Tools
+=============================
+
+There are several tools available to help you design and
+manage your Cassandra schema and build queries.
+
+* `Hackolade <https://hackolade.com/nosqldb.html#cassandra>`_
+ is a data modeling tool that supports schema design for Cassandra and
+ many other NoSQL databases. Hackolade supports the unique concepts of
+ CQL such as partition keys and clustering columns, as well as data types
+ including collections and UDTs. It also provides the ability to create
+ Chebotko diagrams.
+
+* `Kashlev Data Modeler <http://kdm.dataview.org/>`_ is a Cassandra
+ data modeling tool that automates the data modeling methodology
+ described in this documentation, including identifying
+ access patterns, conceptual, logical, and physical data modeling, and
+ schema generation. It also includes model patterns that you can
+ optionally leverage as a starting point for your designs.
+
+* DataStax DevCenter is a tool for managing
+ schema, executing queries and viewing results. While the tool is no
+ longer actively supported, it is still popular with many developers and
+ is available as a `free download <https://academy.datastax.com/downloads>`_.
+ DevCenter features syntax highlighting for CQL commands, types, and name
+ literals. DevCenter provides command completion as you type out CQL
+ commands and interprets the commands you type, highlighting any errors
+ you make. The tool provides panes for managing multiple CQL scripts and
+ connections to multiple clusters. The connections are used to run CQL
+ commands against live clusters and view the results. The tool also has a
+ query trace feature that is useful for gaining insight into the
+ performance of your queries.
+
+* IDE Plugins - There are CQL plugins available for several Integrated
+ Development Environments (IDEs), such as IntelliJ IDEA and Apache
+ NetBeans. These plugins typically provide features such as schema
+ management and query execution.
+
+Some IDEs and tools that claim to support Cassandra do not actually support
+CQL natively, but instead access Cassandra using a JDBC/ODBC driver and
+interact with Cassandra as if it were a relational database with SQL
+support. Wnen selecting tools for working with Cassandra you’ll want to
+make sure they support CQL and reinforce Cassandra best practices for
+data modeling as presented in this documentation.
+
+*Material adapted from Cassandra, The Definitive Guide. Published by
+O'Reilly Media, Inc. Copyright © 2020 Jeff Carpenter, Eben Hewitt.
+All rights reserved. Used with permission.*
\ No newline at end of file
diff --git a/doc/source/data_modeling/images/Figure_1_data_model.jpg b/doc/source/data_modeling/images/Figure_1_data_model.jpg
new file mode 100644
index 0000000..a3b330e
--- /dev/null
+++ b/doc/source/data_modeling/images/Figure_1_data_model.jpg
Binary files differ
diff --git a/doc/source/data_modeling/images/Figure_2_data_model.jpg b/doc/source/data_modeling/images/Figure_2_data_model.jpg
new file mode 100644
index 0000000..7acdeac
--- /dev/null
+++ b/doc/source/data_modeling/images/Figure_2_data_model.jpg
Binary files differ
diff --git a/doc/source/data_modeling/images/data_modeling_chebotko_logical.png b/doc/source/data_modeling/images/data_modeling_chebotko_logical.png
new file mode 100755
index 0000000..e54b5f2
--- /dev/null
+++ b/doc/source/data_modeling/images/data_modeling_chebotko_logical.png
Binary files differ
diff --git a/doc/source/data_modeling/images/data_modeling_chebotko_physical.png b/doc/source/data_modeling/images/data_modeling_chebotko_physical.png
new file mode 100644
index 0000000..bfdaec5
--- /dev/null
+++ b/doc/source/data_modeling/images/data_modeling_chebotko_physical.png
Binary files differ
diff --git a/doc/source/data_modeling/images/data_modeling_hotel_bucketing.png b/doc/source/data_modeling/images/data_modeling_hotel_bucketing.png
new file mode 100644
index 0000000..8b53e38
--- /dev/null
+++ b/doc/source/data_modeling/images/data_modeling_hotel_bucketing.png
Binary files differ
diff --git a/doc/source/data_modeling/images/data_modeling_hotel_erd.png b/doc/source/data_modeling/images/data_modeling_hotel_erd.png
new file mode 100755
index 0000000..e86fe68
--- /dev/null
+++ b/doc/source/data_modeling/images/data_modeling_hotel_erd.png
Binary files differ
diff --git a/doc/source/data_modeling/images/data_modeling_hotel_logical.png b/doc/source/data_modeling/images/data_modeling_hotel_logical.png
new file mode 100755
index 0000000..e920f12
--- /dev/null
+++ b/doc/source/data_modeling/images/data_modeling_hotel_logical.png
Binary files differ
diff --git a/doc/source/data_modeling/images/data_modeling_hotel_physical.png b/doc/source/data_modeling/images/data_modeling_hotel_physical.png
new file mode 100644
index 0000000..2d20a6d
--- /dev/null
+++ b/doc/source/data_modeling/images/data_modeling_hotel_physical.png
Binary files differ
diff --git a/doc/source/data_modeling/images/data_modeling_hotel_queries.png b/doc/source/data_modeling/images/data_modeling_hotel_queries.png
new file mode 100755
index 0000000..2434db3
--- /dev/null
+++ b/doc/source/data_modeling/images/data_modeling_hotel_queries.png
Binary files differ
diff --git a/doc/source/data_modeling/images/data_modeling_hotel_relational.png b/doc/source/data_modeling/images/data_modeling_hotel_relational.png
new file mode 100755
index 0000000..43e784e
--- /dev/null
+++ b/doc/source/data_modeling/images/data_modeling_hotel_relational.png
Binary files differ
diff --git a/doc/source/data_modeling/images/data_modeling_reservation_logical.png b/doc/source/data_modeling/images/data_modeling_reservation_logical.png
new file mode 100755
index 0000000..0460633
--- /dev/null
+++ b/doc/source/data_modeling/images/data_modeling_reservation_logical.png
Binary files differ
diff --git a/doc/source/data_modeling/images/data_modeling_reservation_physical.png b/doc/source/data_modeling/images/data_modeling_reservation_physical.png
new file mode 100755
index 0000000..1e6e76c
--- /dev/null
+++ b/doc/source/data_modeling/images/data_modeling_reservation_physical.png
Binary files differ
diff --git a/doc/source/data_modeling/index.rst b/doc/source/data_modeling/index.rst
index dde031a..2f799dc 100644
--- a/doc/source/data_modeling/index.rst
+++ b/doc/source/data_modeling/index.rst
@@ -15,6 +15,22 @@
.. limitations under the License.
Data Modeling
-=============
+*************
-.. todo:: TODO
+.. toctree::
+ :maxdepth: 2
+
+ intro
+ data_modeling_conceptual
+ data_modeling_rdbms
+ data_modeling_queries
+ data_modeling_logical
+ data_modeling_physical
+ data_modeling_refining
+ data_modeling_schema
+ data_modeling_tools
+
+
+
+
+
diff --git a/doc/source/data_modeling/intro.rst b/doc/source/data_modeling/intro.rst
new file mode 100644
index 0000000..630a7d1
--- /dev/null
+++ b/doc/source/data_modeling/intro.rst
@@ -0,0 +1,146 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements. See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership. The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License. You may obtain a copy of the License at
+..
+.. http://www.apache.org/licenses/LICENSE-2.0
+..
+.. Unless required by applicable law or agreed to in writing, software
+.. distributed under the License is distributed on an "AS IS" BASIS,
+.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+.. See the License for the specific language governing permissions and
+.. limitations under the License.
+
+Introduction
+============
+
+Apache Cassandra stores data in tables, with each table consisting of rows and columns. CQL (Cassandra Query Language) is used to query the data stored in tables. Apache Cassandra data model is based around and optimized for querying. Cassandra does not support relational data modeling intended for relational databases.
+
+What is Data Modeling?
+^^^^^^^^^^^^^^^^^^^^^^
+
+Data modeling is the process of identifying entities and their relationships. In relational databases, data is placed in normalized tables with foreign keys used to reference related data in other tables. Queries that the application will make are driven by the structure of the tables and related data are queried as table joins.
+
+In Cassandra, data modeling is query-driven. The data access patterns and application queries determine the structure and organization of data which then used to design the database tables.
+
+Data is modeled around specific queries. Queries are best designed to access a single table, which implies that all entities involved in a query must be in the same table to make data access (reads) very fast. Data is modeled to best suit a query or a set of queries. A table could have one or more entities as best suits a query. As entities do typically have relationships among them and queries could involve entities with relationships among them, a single entity may be included in multiple tables.
+
+Query-driven modeling
+^^^^^^^^^^^^^^^^^^^^^
+
+Unlike a relational database model in which queries make use of table joins to get data from multiple tables, joins are not supported in Cassandra so all required fields (columns) must be grouped together in a single table. Since each query is backed by a table, data is duplicated across multiple tables in a process known as denormalization. Data duplication and a high write throughput are used to achieve a high read performance.
+
+Goals
+^^^^^
+
+The choice of the primary key and partition key is important to distribute data evenly across the cluster. Keeping the number of partitions read for a query to a minimum is also important because different partitions could be located on different nodes and the coordinator would need to send a request to each node adding to the request overhead and latency. Even if the different partitions involved in a query are on the same node, fewer partitions make for a more efficient query.
+
+Partitions
+^^^^^^^^^^
+
+Apache Cassandra is a distributed database that stores data across a cluster of nodes. A partition key is used to partition data among the nodes. Cassandra partitions data over the storage nodes using a variant of consistent hashing for data distribution. Hashing is a technique used to map data with which given a key, a hash function generates a hash value (or simply a hash) that is stored in a hash table. A partition key is generated from the first field of a primary key. Data partitioned into hash tables using partition keys provides for rapid lookup. Fewer the partitions used for a query faster is the response time for the query.
+
+As an example of partitioning, consider table ``t`` in which ``id`` is the only field in the primary key.
+
+::
+
+ CREATE TABLE t (
+ id int,
+ k int,
+ v text,
+ PRIMARY KEY (id)
+ );
+
+The partition key is generated from the primary key ``id`` for data distribution across the nodes in a cluster.
+
+Consider a variation of table ``t`` that has two fields constituting the primary key to make a composite or compound primary key.
+
+::
+
+ CREATE TABLE t (
+ id int,
+ c text,
+ k int,
+ v text,
+ PRIMARY KEY (id,c)
+ );
+
+For the table ``t`` with a composite primary key the first field ``id`` is used to generate the partition key and the second field ``c`` is the clustering key used for sorting within a partition. Using clustering keys to sort data makes retrieval of adjacent data more efficient.
+
+In general, the first field or component of a primary key is hashed to generate the partition key and the remaining fields or components are the clustering keys that are used to sort data within a partition. Partitioning data improves the efficiency of reads and writes. The other fields that are not primary key fields may be indexed separately to further improve query performance.
+
+The partition key could be generated from multiple fields if they are grouped as the first component of a primary key. As another variation of the table ``t``, consider a table with the first component of the primary key made of two fields grouped using parentheses.
+
+::
+
+ CREATE TABLE t (
+ id1 int,
+ id2 int,
+ c1 text,
+ c2 text
+ k int,
+ v text,
+ PRIMARY KEY ((id1,id2),c1,c2)
+ );
+
+For the preceding table ``t`` the first component of the primary key constituting fields ``id1`` and ``id2`` is used to generate the partition key and the rest of the fields ``c1`` and ``c2`` are the clustering keys used for sorting within a partition.
+
+Comparing with Relational Data Model
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Relational databases store data in tables that have relations with other tables using foreign keys. A relational database’s approach to data modeling is table-centric. Queries must use table joins to get data from multiple tables that have a relation between them. Apache Cassandra does not have the concept of foreign keys or relational integrity. Apache Cassandra’s data model is based around designing efficient queries; queries that don’t involve multiple tables. Relational databases normalize data to avoid duplication. Apache Cassandra in contrast de-normalizes data by duplicating data in multiple tables for a query-centric data model. If a Cassandra data model cannot fully integrate the complexity of relationships between the different entities for a particular query, client-side joins in application code may be used.
+
+Examples of Data Modeling
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+As an example, a ``magazine`` data set consists of data for magazines with attributes such as magazine id, magazine name, publication frequency, publication date, and publisher. A basic query (Q1) for magazine data is to list all the magazine names including their publication frequency. As not all data attributes are needed for Q1 the data model would only consist of ``id`` ( for partition key), magazine name and publication frequency as shown in Figure 1.
+
+.. figure:: images/Figure_1_data_model.jpg
+
+Figure 1. Data Model for Q1
+
+Another query (Q2) is to list all the magazine names by publisher. For Q2 the data model would consist of an additional attribute ``publisher`` for the partition key. The ``id`` would become the clustering key for sorting within a partition. Data model for Q2 is illustrated in Figure 2.
+
+.. figure:: images/Figure_2_data_model.jpg
+
+Figure 2. Data Model for Q2
+
+Designing Schema
+^^^^^^^^^^^^^^^^
+
+After the conceptual data model has been created a schema may be designed for a query. For Q1 the following schema may be used.
+
+::
+
+ CREATE TABLE magazine_name (id int PRIMARY KEY, name text, publicationFrequency text)
+
+For Q2 the schema definition would include a clustering key for sorting.
+
+::
+
+ CREATE TABLE magazine_publisher (publisher text,id int,name text, publicationFrequency text,
+ PRIMARY KEY (publisher, id)) WITH CLUSTERING ORDER BY (id DESC)
+
+Data Model Analysis
+^^^^^^^^^^^^^^^^^^^
+
+The data model is a conceptual model that must be analyzed and optimized based on storage, capacity, redundancy and consistency. A data model may need to be modified as a result of the analysis. Considerations or limitations that are used in data model analysis include:
+
+- Partition Size
+- Data Redundancy
+- Disk space
+- Lightweight Transactions (LWT)
+
+The two measures of partition size are the number of values in a partition and partition size on disk. Though requirements for these measures may vary based on the application a general guideline is to keep number of values per partition to below 100,000 and disk space per partition to below 100MB.
+
+Data redundancies as duplicate data in tables and multiple partition replicates are to be expected in the design of a data model , but nevertheless should be kept in consideration as a parameter to keep to the minimum. LWT transactions (compare-and-set, conditional update) could affect performance and queries using LWT should be kept to the minimum.
+
+Using Materialized Views
+^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. warning:: Materialized views (MVs) are experimental in the latest (4.0) release.
+
+Materialized views (MVs) could be used to implement multiple queries for a single table. A materialized view is a table built from data from another table, the base table, with new primary key and new properties. Changes to the base table data automatically add and update data in a MV. Different queries may be implemented using a materialized view as an MV's primary key differs from the base table. Queries are optimized by the primary key definition.
diff --git a/doc/source/development/ci.rst b/doc/source/development/ci.rst
new file mode 100644
index 0000000..d7a1bb6
--- /dev/null
+++ b/doc/source/development/ci.rst
@@ -0,0 +1,81 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements. See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership. The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License. You may obtain a copy of the License at
+..
+.. http://www.apache.org/licenses/LICENSE-2.0
+..
+.. Unless required by applicable law or agreed to in writing, software
+.. distributed under the License is distributed on an "AS IS" BASIS,
+.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+.. See the License for the specific language governing permissions and
+.. limitations under the License.
+
+Jenkins CI Environment
+**********************
+
+About CI testing and Apache Cassandra
+=====================================
+
+Cassandra can be automatically tested using various test suites, that are either implemented based on JUnit or the `dtest <https://github.com/riptano/cassandra-dtest>`_ scripts written in Python. As outlined in :doc:`testing`, each kind of test suite addresses a different way how to test Cassandra. But in the end, all of them will be executed together on our CI platform at `builds.apache.org <https://builds.apache.org>`_, running `Jenkins <http://jenkins-ci.org>`_.
+
+
+
+Setting up your own Jenkins server
+==================================
+
+Jenkins is an open source solution that can be installed on a large number of platforms. Setting up a custom Jenkins instance for Cassandra may be desirable for users who have hardware to spare, or organizations that want to run Cassandra tests for custom patches before contribution.
+
+Please refer to the Jenkins download and documentation pages for details on how to get Jenkins running, possibly also including slave build executor instances. The rest of the document will focus on how to setup Cassandra jobs in your Jenkins environment.
+
+Required plugins
+----------------
+
+The following plugins need to be installed additionally to the standard plugins (git, ant, ..).
+
+You can install any missing plugins through the install manager.
+
+Go to ``Manage Jenkins -> Manage Plugins -> Available`` and install the following plugins and respective dependencies:
+
+* Job DSL
+* Javadoc Plugin
+* description setter plugin
+* Throttle Concurrent Builds Plug-in
+* Test stability history
+* Hudson Post build task
+* Slack Notification
+* Copy artifact
+
+
+Configure Throttle Category
+---------------------------
+
+Builds that are not containerized (e.g. cqlshlib tests and in-jvm dtests) use local resources for Cassandra (ccm). To prevent these builds running concurrently the ``Cassandra`` throttle category needs to be created.
+
+This is done under ``Manage Jenkins -> System Configuration -> Throttle Concurrent Builds``. Enter "Cassandra" for the ``Category Name`` and "1" for ``Maximum Concurrent Builds Per Node``.
+
+Setup seed job
+--------------
+
+Config ``New Item``
+
+* Name it ``Cassandra-Job-DSL``
+* Select ``Freestyle project``
+
+Under ``Source Code Management`` select Git using the repository: ``https://github.com/apache/cassandra-builds``
+
+Under ``Build``, confirm ``Add build step`` -> ``Process Job DSLs`` and enter at ``Look on Filesystem``: ``jenkins-dsl/cassandra_job_dsl_seed.groovy``
+
+Generated jobs will be created based on the Groovy script's default settings. You may want to override settings by checking ``This project is parameterized`` and add ``String Parameter`` for on the variables that can be found in the top of the script. This will allow you to setup jobs for your own repository and branches (e.g. working branches).
+
+**When done, confirm "Save"**
+
+You should now find a new entry with the given name in your project list. However, building the project will still fail and abort with an error message `"Processing DSL script cassandra_job_dsl_seed.groovy ERROR: script not yet approved for use"`. Goto ``Manage Jenkins`` -> ``In-process Script Approval`` to fix this issue. Afterwards you should be able to run the script and have it generate numerous new jobs based on the found branches and configured templates.
+
+Jobs are triggered by either changes in Git or are scheduled to execute periodically, e.g. on daily basis. Jenkins will use any available executor with the label "cassandra", once the job is to be run. Please make sure to make any executors available by selecting ``Build Executor Status`` -> ``Configure`` -> Add "``cassandra``" as label and save.
+
+Executors need to have "JDK 1.8 (latest)" installed. This is done under ``Manage Jenkins -> Global Tool Configuration -> JDK Installations…``. Executors also need to have the virtualenv package installed on their system.
+
diff --git a/doc/source/development/dependencies.rst b/doc/source/development/dependencies.rst
new file mode 100644
index 0000000..6dd1cc4
--- /dev/null
+++ b/doc/source/development/dependencies.rst
@@ -0,0 +1,53 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements. See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership. The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License. You may obtain a copy of the License at
+..
+.. http://www.apache.org/licenses/LICENSE-2.0
+..
+.. Unless required by applicable law or agreed to in writing, software
+.. distributed under the License is distributed on an "AS IS" BASIS,
+.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+.. See the License for the specific language governing permissions and
+.. limitations under the License.
+
+Dependency Management
+*********************
+
+Managing libraries for Cassandra is a bit less straight forward compared to other projects, as the build process is based on ant, maven and manually managed jars. Make sure to follow the steps below carefully and pay attention to any emerging issues in the :doc:`ci` and reported related issues on Jira/ML, in case of any project dependency changes.
+
+As Cassandra is an Apache product, all included libraries must follow Apache's `software license requirements <https://www.apache.org/legal/resolved.html>`_.
+
+Required steps to add or update libraries
+=========================================
+
+* Add or replace jar file in ``lib`` directory
+* Add or update ``lib/license`` files
+* Update dependencies in ``build.xml``
+
+ * Add to ``parent-pom`` with correct version
+ * Add to ``all-pom`` if simple Cassandra dependency (see below)
+
+
+POM file types
+==============
+
+* **parent-pom** - contains all dependencies with the respective version. All other poms will refer to the artifacts with specified versions listed here.
+* **build-deps-pom(-sources)** + **coverage-deps-pom** - used by ``ant build`` compile target. Listed dependenices will be resolved and copied to ``build/lib/{jar,sources}`` by executing the ``maven-ant-tasks-retrieve-build`` target. This should contain libraries that are required for build tools (grammar, docs, instrumentation), but are not shipped as part of the Cassandra distribution.
+* **test-deps-pom** - refered by ``maven-ant-tasks-retrieve-test`` to retrieve and save dependencies to ``build/test/lib``. Exclusively used during JUnit test execution.
+* **all-pom** - pom for `cassandra-all.jar <https://mvnrepository.com/artifact/org.apache.cassandra/cassandra-all>`_ that can be installed or deployed to public maven repos via ``ant publish``
+
+
+Troubleshooting and conflict resolution
+=======================================
+
+Here are some useful commands that may help you out resolving conflicts.
+
+* ``ant realclean`` - gets rid of the build directory, including build artifacts.
+* ``mvn dependency:tree -f build/apache-cassandra-*-SNAPSHOT.pom -Dverbose -Dincludes=org.slf4j`` - shows transitive dependency tree for artifacts, e.g. org.slf4j. In case the command above fails due to a missing parent pom file, try running ``ant mvn-install``.
+* ``rm ~/.m2/repository/org/apache/cassandra/apache-cassandra/`` - removes cached local Cassandra maven artifacts
+
+
diff --git a/doc/source/development/documentation.rst b/doc/source/development/documentation.rst
new file mode 100644
index 0000000..c623d54
--- /dev/null
+++ b/doc/source/development/documentation.rst
@@ -0,0 +1,104 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements. See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership. The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License. You may obtain a copy of the License at
+..
+.. http://www.apache.org/licenses/LICENSE-2.0
+..
+.. Unless required by applicable law or agreed to in writing, software
+.. distributed under the License is distributed on an "AS IS" BASIS,
+.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+.. See the License for the specific language governing permissions and
+.. limitations under the License.
+
+
+Working on Documentation
+*************************
+
+How Cassandra is documented
+===========================
+
+The official Cassandra documentation lives in the project's git repository. We use a static site generator, `Sphinx <http://www.sphinx-doc.org/>`_, to create pages hosted at `cassandra.apache.org <https://cassandra.apache.org/doc/latest/>`_. You'll also find developer centric content about Cassandra internals in our retired `wiki <https://wiki.apache.org/cassandra>`_ (not covered by this guide).
+
+Using a static site generator often requires to use a markup language instead of visual editors (which some people would call good news). Sphinx, the tool-set we use to generate our documentation, uses `reStructuredText <http://www.sphinx-doc.org/en/stable/rest.html>`_ for that. Markup languages allow you to format text by making use of certain syntax elements. Your document structure will also have to follow specific conventions. Feel free to take a look at `existing documents <..>`_ to get a better idea how we use reStructuredText to write our documents.
+
+So how do you actually start making contributions?
+
+GitHub based work flow
+======================
+
+*Recommended for shorter documents and minor changes on existing content (e.g. fixing typos or updating descriptions)*
+
+Follow these steps to contribute using GitHub. It's assumed that you're logged in with an existing account.
+
+1. Fork the GitHub mirror of the `Cassandra repository <https://github.com/apache/cassandra>`_
+
+.. image:: images/docs_fork.png
+
+2. Create a new branch that you can use to make your edits. It's recommended to have a separate branch for each of your working projects. It will also make it easier to create a pull request later to when you decide you’re ready to contribute your work.
+
+.. image:: images/docs_create_branch.png
+
+3. Navigate to document sources ``doc/source`` to find the ``.rst`` file to edit. The URL of the document should correspond to the directory structure. New files can be created using the "Create new file" button:
+
+.. image:: images/docs_create_file.png
+
+4. At this point you should be able to edit the file using the GitHub web editor. Start by naming your file and add some content. Have a look at other existing ``.rst`` files to get a better idea what format elements to use.
+
+.. image:: images/docs_editor.png
+
+Make sure to preview added content before committing any changes.
+
+.. image:: images/docs_preview.png
+
+5. Commit your work when you're done. Make sure to add a short description of all your edits since the last time you committed before.
+
+.. image:: images/docs_commit.png
+
+6. Finally if you decide that you're done working on your branch, it's time to create a pull request!
+
+.. image:: images/docs_pr.png
+
+Afterwards the GitHub Cassandra mirror will list your pull request and you're done. Congratulations! Please give us some time to look at your suggested changes before we get back to you.
+
+
+Jira based work flow
+====================
+
+*Recommended for major changes*
+
+Significant changes to the documentation are best managed through our Jira issue tracker. Please follow the same `contribution guides <https://cassandra.apache.org/doc/latest/development/patches.html>`_ as for regular code contributions. Creating high quality content takes a lot of effort. It’s therefor always a good idea to create a ticket before you start and explain what you’re planing to do. This will create the opportunity for other contributors and committers to comment on your ideas and work so far. Eventually your patch gets a formal review before it is committed.
+
+Working on documents locally using Sphinx
+=========================================
+
+*Recommended for advanced editing*
+
+Using the GitHub web interface should allow you to use most common layout elements including images. More advanced formatting options and navigation elements depend on Sphinx to render correctly. Therefor it’s a good idea to setup Sphinx locally for any serious editing. Please follow the instructions in the Cassandra source directory at ``doc/README.md``. Setup is very easy (at least on OSX and Linux).
+
+Notes for committers
+====================
+
+Please feel free to get involved and merge pull requests created on the GitHub mirror if you're a committer. As this is a read-only repository, you won't be able to merge a PR directly on GitHub. You'll have to commit the changes against the Apache repository with a comment that will close the PR when the committ syncs with GitHub.
+
+You may use a git work flow like this::
+
+ git remote add github https://github.com/apache/cassandra.git
+ git fetch github pull/<PR-ID>/head:<PR-ID>
+ git checkout <PR-ID>
+
+Now either rebase or squash the commit, e.g. for squashing::
+
+ git reset --soft origin/trunk
+ git commit --author <PR Author>
+
+Make sure to add a proper commit message including a "Closes #<PR-ID>" text to automatically close the PR.
+
+Publishing
+----------
+
+Details for building and publishing of the site at cassandra.apache.org can be found `here <https://github.com/apache/cassandra-website/blob/master/README.md>`_.
+
diff --git a/doc/source/development/gettingstarted.rst b/doc/source/development/gettingstarted.rst
new file mode 100644
index 0000000..c2f5ef3
--- /dev/null
+++ b/doc/source/development/gettingstarted.rst
@@ -0,0 +1,60 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements. See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership. The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License. You may obtain a copy of the License at
+..
+.. http://www.apache.org/licenses/LICENSE-2.0
+..
+.. Unless required by applicable law or agreed to in writing, software
+.. distributed under the License is distributed on an "AS IS" BASIS,
+.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+.. See the License for the specific language governing permissions and
+.. limitations under the License.
+
+.. highlight:: none
+.. _gettingstarted:
+
+Getting Started
+*************************
+
+Initial Contributions
+========================
+
+Writing a new feature is just one way to contribute to the Cassandra project. In fact, making sure that supporting tasks, such as QA, documentation and helping users, keep up with the development of new features is an ongoing challenge for the project (and most open source projects). So, before firing up your IDE to create that new feature, we'd suggest you consider some of the following activities as a way of introducing yourself to the project and getting to know how things work.
+ * Add to or update the documentation
+ * Answer questions on the user list
+ * Review and test a submitted patch
+ * Investigate and fix a reported bug
+ * Create unit tests and d-tests
+
+Updating documentation
+========================
+
+The Cassandra documentation is maintained in the Cassandra source repository along with the Cassandra code base. To submit changes to the documentation, follow the standard process for submitting a patch (:ref:`patches`).
+
+Answering questions on the user list
+====================================
+
+Subscribe to the user list, look out for some questions you know the answer to and reply with an answer. Simple as that!
+See the `community <http://cassandra.apache.org/community/>`_ page for details on how to subscribe to the mailing list.
+
+Reviewing and testing a submitted patch
+=======================================
+
+Reviewing patches is not the sole domain of committers, if others have reviewed a patch it can reduce the load on the committers allowing them to write more great features or review more patches. Follow the instructions in :ref:`_development_how_to_review` or create a build with the patch and test it with your own workload. Add a comment to the JIRA ticket to let others know what you have done and the results of your work. (For example, "I tested this performance enhacement on our application's standard production load test and found a 3% improvement.")
+
+Investigate and/or fix a reported bug
+=====================================
+
+Often, the hardest work in fixing a bug is reproducing it. Even if you don't have the knowledge to produce a fix, figuring out a way to reliable reproduce an issues can be a massive contribution to getting a bug fixed. Document your method of reproduction in a JIRA comment or, better yet, produce an automated test that reproduces the issue and attach it to the ticket. If you go as far as producing a fix, follow the process for submitting a patch (:ref:`patches`).
+
+Create unit tests and Dtests
+============================
+
+Test coverage in Cassandra is improving but, as with most code bases, it could benefit from more automated test coverage. Before starting work in an area, consider reviewing and enhancing the existing test coverage. This will both improve your knowledge of the code before you start on an enhancement and reduce the chances of your change in introducing new issues. See :ref:`testing` and :ref:`patches` for more detail.
+
+
+
diff --git a/doc/source/development/how_to_commit.rst b/doc/source/development/how_to_commit.rst
index d956c72..dff3983 100644
--- a/doc/source/development/how_to_commit.rst
+++ b/doc/source/development/how_to_commit.rst
@@ -31,15 +31,15 @@
On cassandra-3.3:
#. ``git merge cassandra-3.0 -s ours``
#. ``git apply -3 12345-3.3.patch`` (likely to have an issue with CHANGES.txt here: fix it ourselves, then git add CHANGES.txt)
- #. ``git commit —amend``
+ #. ``git commit -amend``
On trunk:
#. ``git merge cassandra-3.3 -s ours``
#. ``git apply -3 12345-trunk.patch`` (likely to have an issue with CHANGES.txt here: fix it ourselves, then git add CHANGES.txt)
- #. ``git commit —amend``
+ #. ``git commit -amend``
On any branch:
- #. ``git push origin cassandra-3.0 cassandra-3.3 trunk —atomic``
+ #. ``git push origin cassandra-3.0 cassandra-3.3 trunk -atomic``
Same scenario, but a branch-based contribution:
@@ -50,23 +50,23 @@
#. ``git merge cassandra-3.0 -s ours``
#. ``git format-patch -1 <sha-of-3.3-commit>``
#. ``git apply -3 <sha-of-3.3-commit>.patch`` (likely to have an issue with CHANGES.txt here: fix it ourselves, then git add CHANGES.txt)
- #. ``git commit —amend``
+ #. ``git commit -amend``
On trunk:
#. ``git merge cassandra-3.3 -s ours``
#. ``git format-patch -1 <sha-of-trunk-commit>``
#. ``git apply -3 <sha-of-trunk-commit>.patch`` (likely to have an issue with CHANGES.txt here: fix it ourselves, then git add CHANGES.txt)
- #. ``git commit —amend``
+ #. ``git commit -amend``
On any branch:
- #. ``git push origin cassandra-3.0 cassandra-3.3 trunk —atomic``
+ #. ``git push origin cassandra-3.0 cassandra-3.3 trunk -atomic``
.. tip::
Notes on git flags:
``-3`` flag to am and apply will instruct git to perform a 3-way merge for you. If a conflict is detected, you can either resolve it manually or invoke git mergetool - for both am and apply.
- ``—atomic`` flag to git push does the obvious thing: pushes all or nothing. Without the flag, the command is equivalent to running git push once per each branch. This is nifty in case a race condition happens - you won’t push half the branches, blocking other committers’ progress while you are resolving the issue.
+ ``-atomic`` flag to git push does the obvious thing: pushes all or nothing. Without the flag, the command is equivalent to running git push once per each branch. This is nifty in case a race condition happens - you won’t push half the branches, blocking other committers’ progress while you are resolving the issue.
.. tip::
diff --git a/doc/source/development/how_to_review.rst b/doc/source/development/how_to_review.rst
index dc97743..4778b69 100644
--- a/doc/source/development/how_to_review.rst
+++ b/doc/source/development/how_to_review.rst
@@ -14,6 +14,8 @@
.. See the License for the specific language governing permissions and
.. limitations under the License.
+.. _how_to_review:
+
Review Checklist
****************
diff --git a/doc/source/development/ide.rst b/doc/source/development/ide.rst
index 2986495..97c73ae 100644
--- a/doc/source/development/ide.rst
+++ b/doc/source/development/ide.rst
@@ -24,7 +24,7 @@
The source code for Cassandra is shared through the central Apache Git repository and organized by different branches. You can access the code for the current development branch through git as follows::
- git clone http://git-wip-us.apache.org/repos/asf/cassandra.git cassandra-trunk
+ git clone https://gitbox.apache.org/repos/asf/cassandra.git cassandra-trunk
Other branches will point to different versions of Cassandra. Switching to a different branch requires checking out the branch by its name::
@@ -42,9 +42,7 @@
You can setup multiple working trees for different Cassandra versions from the same repository using `git-worktree <https://git-scm.com/docs/git-worktree>`_.
-.. note::
-
- `Bleeding edge development snapshots <http://cassci.datastax.com/job/trunk/lastSuccessfulBuild/>`_ of Cassandra are available from Jenkins continuous integration.
+|
Setting up Cassandra in IntelliJ IDEA
=====================================
@@ -76,6 +74,32 @@
* Cassandra code style
* Inspections
+|
+
+Opening Cassandra in Apache NetBeans
+=======================================
+
+`Apache NetBeans <https://netbeans.apache.org/>`_ is the elder of the open sourced java IDEs, and can be used for Cassandra development. There is no project setup or generation required to open Cassandra in NetBeans.
+
+Open Cassandra as a Project (C* 4.0 and newer)
+-----------------------------------------------
+
+Please clone and build Cassandra as described above and execute the following steps:
+
+1. Start Apache NetBeans
+
+2. Open the NetBeans project from the `ide/` folder of the checked out Cassandra directory using the menu item "Open Project…" in NetBeans' File menu
+
+The project opened supports building, running, debugging, and profiling Cassandra from within the IDE. These actions delegate to the ant `build.xml` script.
+
+ * Build/Run/Debug Project is available via the Run/Debug menus, or the project context menu.
+ * Profile Project is available via the Profile menu. In the opened Profiler tab, click the green "Profile" button.
+ * Cassandra's code style is honored in `ide/nbproject/project.properties`
+
+The `JAVA8_HOME` system variable must be set in the environment that NetBeans starts in for the Run/Debug/Profile ant targets to execute.
+
+|
+
Setting up Cassandra in Eclipse
===============================
diff --git a/doc/source/development/images/docs_commit.png b/doc/source/development/images/docs_commit.png
new file mode 100644
index 0000000..d90d96a
--- /dev/null
+++ b/doc/source/development/images/docs_commit.png
Binary files differ
diff --git a/doc/source/development/images/docs_create_branch.png b/doc/source/development/images/docs_create_branch.png
new file mode 100644
index 0000000..a04cb54
--- /dev/null
+++ b/doc/source/development/images/docs_create_branch.png
Binary files differ
diff --git a/doc/source/development/images/docs_create_file.png b/doc/source/development/images/docs_create_file.png
new file mode 100644
index 0000000..b51e370
--- /dev/null
+++ b/doc/source/development/images/docs_create_file.png
Binary files differ
diff --git a/doc/source/development/images/docs_editor.png b/doc/source/development/images/docs_editor.png
new file mode 100644
index 0000000..5b9997b
--- /dev/null
+++ b/doc/source/development/images/docs_editor.png
Binary files differ
diff --git a/doc/source/development/images/docs_fork.png b/doc/source/development/images/docs_fork.png
new file mode 100644
index 0000000..20a592a
--- /dev/null
+++ b/doc/source/development/images/docs_fork.png
Binary files differ
diff --git a/doc/source/development/images/docs_pr.png b/doc/source/development/images/docs_pr.png
new file mode 100644
index 0000000..211eb25
--- /dev/null
+++ b/doc/source/development/images/docs_pr.png
Binary files differ
diff --git a/doc/source/development/images/docs_preview.png b/doc/source/development/images/docs_preview.png
new file mode 100644
index 0000000..207f0ac
--- /dev/null
+++ b/doc/source/development/images/docs_preview.png
Binary files differ
diff --git a/doc/source/development/index.rst b/doc/source/development/index.rst
index aefc599..ffa7134 100644
--- a/doc/source/development/index.rst
+++ b/doc/source/development/index.rst
@@ -14,15 +14,20 @@
.. See the License for the specific language governing permissions and
.. limitations under the License.
-Cassandra Development
-*********************
+Contributing to Cassandra
+*************************
.. toctree::
:maxdepth: 2
+ gettingstarted
ide
testing
patches
code_style
how_to_review
how_to_commit
+ documentation
+ ci
+ dependencies
+ release_process
diff --git a/doc/source/development/patches.rst b/doc/source/development/patches.rst
index e3d968f..92c0553 100644
--- a/doc/source/development/patches.rst
+++ b/doc/source/development/patches.rst
@@ -15,6 +15,7 @@
.. limitations under the License.
.. highlight:: none
+.. _patches:
Contributing Code Changes
*************************
@@ -32,12 +33,12 @@
.. hint::
- Not sure what to work? Just pick an issue tagged with the `low hanging fruit label <https://issues.apache.org/jira/secure/IssueNavigator.jspa?reset=true&jqlQuery=project+=+12310865+AND+labels+=+lhf+AND+status+!=+resolved>`_ in JIRA, which we use to flag issues that could turn out to be good starter tasks for beginners.
+ Not sure what to work? Just pick an issue marked as `Low Hanging Fruit <https://issues.apache.org/jira/issues/?jql=project%20%3D%20CASSANDRA%20AND%20Complexity%20%3D%20%22Low%20Hanging%20Fruit%22%20and%20status%20!%3D%20resolved>`_ Complexity in JIRA, which we use to flag issues that could turn out to be good starter tasks for beginners.
Before You Start Coding
=======================
-Although contributions are highly appreciated, we do not guarantee that each contribution will become a part of Cassandra. Therefor it's generally a good idea to first get some feedback on the things you plan to work on, especially about any new features or major changes to the code base. You can reach out to other developers on the mailing list or IRC channel listed on our `community page <http://cassandra.apache.org/community/>`_.
+Although contributions are highly appreciated, we do not guarantee that each contribution will become a part of Cassandra. Therefore it's generally a good idea to first get some feedback on the things you plan to work on, especially about any new features or major changes to the code base. You can reach out to other developers on the mailing list or :ref:`Slack <slack>`.
You should also
* Avoid redundant work by searching for already reported issues in `JIRA <https://issues.apache.org/jira/browse/CASSANDRA>`_
@@ -61,18 +62,26 @@
======= ======
Version Policy
======= ======
-3.x Tick-tock (see below)
-3.0 Bug fixes only
-2.2 Bug fixes only
+4.0 Code freeze (see below)
+3.11 Critical bug fixes only
+3.0 Critical bug fixes only
+2.2 Critical bug fixes only
2.1 Critical bug fixes only
======= ======
Corresponding branches in git are easy to recognize as they are named ``cassandra-<release>`` (e.g. ``cassandra-3.0``). The ``trunk`` branch is an exception, as it contains the most recent commits from all other branches and is used for creating new branches for future tick-tock releases.
-Tick-Tock Releases
-""""""""""""""""""
+4.0 Code Freeze
+"""""""""""""""
-New releases created as part of the `tick-tock release process <http://www.planetcassandra.org/blog/cassandra-2-2-3-0-and-beyond/>`_ will either focus on stability (odd version numbers) or introduce new features (even version numbers). Any code for new Cassandra features you should be based on the latest, unreleased 3.x branch with even version number or based on trunk.
+Patches for new features are currently not accepted for 4.0 or any earlier versions. Starting with the code freeze in September, all efforts should focus on stabilizing the 4.0 branch before the first official release. During that time, only the following patches will be considered for acceptance:
+
+ * Bug fixes
+ * Measurable performance improvements
+ * Changes not distributed as part of the release such as:
+ * Testing related improvements and fixes
+ * Build and infrastructure related changes
+ * Documentation
Bug Fixes
"""""""""
@@ -92,14 +101,21 @@
1. Create a branch for your changes if you haven't done already. Many contributors name their branches based on ticket number and Cassandra version, e.g. ``git checkout -b 12345-3.0``
2. Verify that you follow Cassandra's :doc:`code_style`
- 3. Make sure all tests (including yours) pass using ant as described in :doc:`testing`. If you suspect a test failure is unrelated to your change, it may be useful to check the test's status by searching the issue tracker or looking at `CI <https://cassci.datastax.com/>`_ results for the relevant upstream version. Note that the full test suites take many hours to complete, so it is common to only run specific relevant tests locally before uploading a patch. Once a patch has been uploaded, the reviewer or committer can help setup CI jobs to run the full test suites.
+ 3. Make sure all tests (including yours) pass using ant as described in :doc:`testing`. If you suspect a test failure is unrelated to your change, it may be useful to check the test's status by searching the issue tracker or looking at `CI <https://builds.apache.org/>`_ results for the relevant upstream version. Note that the full test suites take many hours to complete, so it is common to only run specific relevant tests locally before uploading a patch. Once a patch has been uploaded, the reviewer or committer can help setup CI jobs to run the full test suites.
4. Consider going through the :doc:`how_to_review` for your code. This will help you to understand how others will consider your change for inclusion.
5. Don’t make the committer squash commits for you in the root branch either. Multiple commits are fine - and often preferable - during review stage, especially for incremental review, but once +1d, do either:
a. Attach a patch to JIRA with a single squashed commit in it (per branch), or
b. Squash the commits in-place in your branches into one
- 6. Include a CHANGES.txt entry (put it at the top of the list), and format the commit message appropriately in your patch ending with the following statement on the last line: ``patch by X; reviewed by Y for CASSANDRA-ZZZZZ``
+ 6. Include a CHANGES.txt entry (put it at the top of the list), and format the commit message appropriately in your patch as below. Please note that only user-impacting items `should <https://lists.apache.org/thread.html/rde1128131a621e43b0a9c88778398c053a234da0f4c654b82dcbbe0e%40%3Cdev.cassandra.apache.org%3E>`_ be listed in CHANGES.txt. If you fix a test that does not affect users and does not require changes in runtime code, then no CHANGES.txt entry is necessary.
+
+ ::
+
+ <One sentence description, usually Jira title and CHANGES.txt summary>
+ <Optional lengthier description>
+ patch by <Authors>; reviewed by <Reviewers> for CASSANDRA-#####
+
7. When you're happy with the result, create a patch:
::
diff --git a/doc/source/development/release_process.rst b/doc/source/development/release_process.rst
new file mode 100644
index 0000000..fd86238
--- /dev/null
+++ b/doc/source/development/release_process.rst
@@ -0,0 +1,251 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements. See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership. The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License. You may obtain a copy of the License at
+..
+.. http://www.apache.org/licenses/LICENSE-2.0
+..
+.. Unless required by applicable law or agreed to in writing, software
+.. distributed under the License is distributed on an "AS IS" BASIS,
+.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+.. See the License for the specific language governing permissions and
+.. limitations under the License.
+
+.. highlight:: none
+.. release_process:
+
+Release Process
+***************
+
+.. contents:: :depth: 3
+
+|
+|
+
+
+
+The steps for Release Managers to create, vote and publish releases for Apache Cassandra.
+
+While a committer can perform the initial steps of creating and calling a vote on a proposed release, only a PMC member can complete the process of publishing and announcing the release.
+
+
+Prerequisites
+=============
+
+Background docs
+ * `ASF Release Policy <http://www.apache.org/legal/release-policy.html>`_
+ * `ASF Release Distribution Policy <http://www.apache.org/dev/release-distribution>`_
+ * `ASF Release Best Practices <http://www.eu.apache.org/dev/release-publishing.html>`_
+
+
+A debian based linux OS is required to run the release steps from. Debian-based distros provide the required RPM, dpkg and repository management tools.
+
+
+Create and publish your GPG key
+-------------------------------
+
+To create a GPG key, follow the `guidelines <http://www.apache.org/dev/openpgp.html>`_.
+The key must be 4096 bit RSA.
+Include your public key in::
+
+ https://dist.apache.org/repos/dist/release/cassandra/KEYS
+
+
+Publish your GPG key in a PGP key server, such as `MIT Keyserver <http://pgp.mit.edu/>`_.
+
+Bintray account with access to Apache organisation
+--------------------------------------------------
+
+Publishing a successfully voted upon release requires bintray access to the Apache organisation. Please verify that you have a bintray account and the Apache organisation is listed `here <https://bintray.com/profile/edit/organizations>`_.
+
+
+Create Release Artifacts
+========================
+
+Any committer can perform the following steps to create and call a vote on a proposed release.
+
+Check that there are no open urgent jira tickets currently being worked on. Also check with the PMC that there's security vulnerabilities currently being worked on in private.'
+Current project habit is to check the timing for a new release on the dev mailing lists.
+
+Perform the Release
+-------------------
+
+Run the following commands to generate and upload release artifacts, to the ASF nexus staging repository and dev distribution location::
+
+
+ cd ~/git
+ git clone https://github.com/apache/cassandra-builds.git
+ git clone https://github.com/apache/cassandra.git
+
+ # Edit the variables at the top of the `prepare_release.sh` file
+ edit cassandra-builds/cassandra-release/prepare_release.sh
+
+ # Ensure your 4096 RSA key is the default secret key
+ edit ~/.gnupg/gpg.conf # update the `default-key` line
+ edit ~/.rpmmacros # update the `%gpg_name <key_id>` line
+
+ # Ensure DEBFULLNAME and DEBEMAIL is defined and exported, in the debian scripts configuration
+ edit ~/.devscripts
+
+ # The prepare_release.sh is run from the actual cassandra git checkout,
+ # on the branch/commit that we wish to tag for the tentative release along with version number to tag.
+ cd cassandra
+ git switch cassandra-<version-branch>
+
+ # The following cuts the release artifacts (including deb and rpm packages) and deploy to staging environments
+ ../cassandra-builds/cassandra-release/prepare_release.sh -v <version>
+
+Follow the prompts.
+
+If building the deb or rpm packages fail, those steps can be repeated individually using the `-d` and `-r` flags, respectively.
+
+Call for a Vote
+===============
+
+Fill out the following email template and send to the dev mailing list::
+
+ I propose the following artifacts for release as <version>.
+
+ sha1: <git-sha>
+
+ Git: https://gitbox.apache.org/repos/asf?p=cassandra.git;a=shortlog;h=refs/tags/<version>-tentative
+
+ Artifacts: https://repository.apache.org/content/repositories/orgapachecassandra-<nexus-id>/org/apache/cassandra/apache-cassandra/<version>/
+
+ Staging repository: https://repository.apache.org/content/repositories/orgapachecassandra-<nexus-id>/
+
+ The distribution packages are available here: https://dist.apache.org/repos/dist/dev/cassandra/${version}/
+
+ The vote will be open for 72 hours (longer if needed).
+
+ [1]: (CHANGES.txt) https://git1-us-west.apache.org/repos/asf?p=cassandra.git;a=blob_plain;f=CHANGES.txt;hb=<version>-tentative
+ [2]: (NEWS.txt) https://git1-us-west.apache.org/repos/asf?p=cassandra.git;a=blob_plain;f=NEWS.txt;hb=<version>-tentative
+
+
+
+Post-vote operations
+====================
+
+Any PMC member can perform the following steps to formalize and publish a successfully voted release.
+
+Publish Artifacts
+-----------------
+
+Run the following commands to publish the voted release artifacts::
+
+ cd ~/git
+ # edit the variables at the top of the `finish_release.sh` file
+ edit cassandra-builds/cassandra-release/finish_release.sh
+
+ # After cloning cassandra-builds repo, `finish_release.sh` is run from the actual cassandra git checkout,
+ # on the tentative release tag that we wish to tag for the final release version number tag.
+ cd ~/git/cassandra/
+ git checkout <version>-tentative
+ ../cassandra-builds/cassandra-release/finish_release.sh -v <version>
+
+If successful, take note of the email text output which can be used in the next section "Send Release Announcement".
+The output will also list the next steps that are required.
+
+
+Promote Nexus Repository
+------------------------
+
+* Login to `Nexus repository <https://repository.apache.org>`_ again.
+* Click on "Staging" and then on the repository with id "cassandra-staging".
+* Find your closed staging repository, right click on it and choose "Promote".
+* Select the "Releases" repository and click "Promote".
+* Next click on "Repositories", select the "Releases" repository and validate that your artifacts exist as you expect them.
+
+Publish the Bintray Uploaded Distribution Packages
+---------------------------------------
+
+Log into bintray and publish the uploaded artifacts.
+
+Update and Publish Website
+--------------------------
+
+See `docs <https://svn.apache.org/repos/asf/cassandra/site/src/README>`_ for building and publishing the website.
+
+Also update the CQL doc if appropriate.
+
+Release version in JIRA
+-----------------------
+
+Release the JIRA version.
+
+* In JIRA go to the version that you want to release and release it.
+* Create a new version, if it has not been done before.
+
+Update to Next Development Version
+----------------------------------
+
+Update the codebase to point to the next development version::
+
+ cd ~/git/cassandra/
+ git checkout cassandra-<version-branch>
+ edit build.xml # update `<property name="base.version" value="…"/> `
+ edit debian/changelog # add entry for new version
+ edit CHANGES.txt # add entry for new version
+ git commit -m "Increment version to <next-version>" build.xml debian/changelog CHANGES.txt
+
+ # …and forward merge and push per normal procedure
+
+
+Wait for Artifacts to Sync
+--------------------------
+
+Wait for the artifacts to sync at https://downloads.apache.org/cassandra/
+
+Send Release Announcement
+-------------------------
+
+Fill out the following email template and send to both user and dev mailing lists::
+
+ The Cassandra team is pleased to announce the release of Apache Cassandra version <version>.
+
+ Apache Cassandra is a fully distributed database. It is the right choice
+ when you need scalability and high availability without compromising
+ performance.
+
+ http://cassandra.apache.org/
+
+ Downloads of source and binary distributions are listed in our download
+ section:
+
+ http://cassandra.apache.org/download/
+
+ This version is <the first|a bug fix> release[1] on the <version-base> series. As always,
+ please pay attention to the release notes[2] and let us know[3] if you
+ were to encounter any problem.
+
+ Enjoy!
+
+ [1]: (CHANGES.txt) https://git1-us-west.apache.org/repos/asf?p=cassandra.git;a=blob_plain;f=CHANGES.txt;hb=<version>
+ [2]: (NEWS.txt) https://git1-us-west.apache.org/repos/asf?p=cassandra.git;a=blob_plain;f=NEWS.txt;hb=<version>
+ [3]: https://issues.apache.org/jira/browse/CASSANDRA
+
+Update Slack Cassandra topic
+---------------------------
+
+Update topic in ``cassandra`` :ref:`Slack room <slack>`
+ /topic cassandra.apache.org | Latest releases: 3.11.4, 3.0.18, 2.2.14, 2.1.21 | ask, don't ask to ask
+
+Tweet from @Cassandra
+---------------------
+
+Tweet the new release, from the @Cassandra account
+
+Delete Old Releases
+-------------------
+
+As described in `When to Archive <http://www.apache.org/dev/release.html#when-to-archive>`_.
+
+An example of removing old releases::
+
+ svn co https://dist.apache.org/repos/dist/release/cassandra/ cassandra-dist
+ svn rm <previous_version> debian/pool/main/c/cassandra/<previous_version>*
+ svn st
+ # check and commit
\ No newline at end of file
diff --git a/doc/source/development/testing.rst b/doc/source/development/testing.rst
index b8eea6b..7f38fe5 100644
--- a/doc/source/development/testing.rst
+++ b/doc/source/development/testing.rst
@@ -15,6 +15,7 @@
.. limitations under the License.
.. highlight:: none
+.. _testing:
Testing
*******
@@ -51,6 +52,14 @@
ant testsome -Dtest.name=org.apache.cassandra.cql3.SimpleQueryTest -Dtest.methods=testStaticCompactTables
+If you see an error like this::
+
+ Throws: cassandra-trunk/build.xml:1134: taskdef A class needed by class org.krummas.junit.JStackJUnitTask cannot be found:
+ org/apache/tools/ant/taskdefs/optional/junit/JUnitTask using the classloader
+ AntClassLoader[/.../cassandra-trunk/lib/jstackjunit-0.0.1.jar]
+
+You will need to install the ant-optional package since it contains the ``JUnitTask`` class.
+
Long running tests
------------------
@@ -59,11 +68,11 @@
DTests
======
-One way of doing integration or system testing at larger scale is by using `dtest <https://github.com/riptano/cassandra-dtest>`_, which stands for “Cassandra Distributed Tests”. The idea is to automatically setup Cassandra clusters using various configurations and simulate certain use cases you want to test. This is done using Python scripts and ``ccmlib`` from the `ccm <https://github.com/pcmanus/ccm>`_ project. Dtests will setup clusters using this library just as you do running ad-hoc ``ccm`` commands on your local machine. Afterwards dtests will use the `Python driver <http://datastax.github.io/python-driver/installation.html>`_ to interact with the nodes, manipulate the file system, analyze logs or mess with individual nodes.
+One way of doing integration or system testing at larger scale is by using `dtest <https://github.com/apache/cassandra-dtest>`_, which stands for “Cassandra Distributed Tests”. The idea is to automatically setup Cassandra clusters using various configurations and simulate certain use cases you want to test. This is done using Python scripts and ``ccmlib`` from the `ccm <https://github.com/pcmanus/ccm>`_ project. Dtests will setup clusters using this library just as you do running ad-hoc ``ccm`` commands on your local machine. Afterwards dtests will use the `Python driver <http://datastax.github.io/python-driver/installation.html>`_ to interact with the nodes, manipulate the file system, analyze logs or mess with individual nodes.
-Using dtests helps us to prevent regression bugs by continually executing tests on the `CI server <http://cassci.datastax.com/>`_ against new patches. For frequent contributors, this Jenkins is set up to build branches from their GitHub repositories. It is likely that your reviewer will use this Jenkins instance to run tests for your patch. Read more on the motivation behind the CI server `here <http://www.datastax.com/dev/blog/cassandra-testing-improvements-for-developer-convenience-and-confidence>`_.
+Using dtests helps us to prevent regression bugs by continually executing tests on the `CI server <https://builds.apache.org/>`_ against new patches. Committers will be able to set up build branches there and your reviewer may use the CI environment to run tests for your patch. Read more on the motivation behind continuous integration `here <http://www.datastax.com/dev/blog/cassandra-testing-improvements-for-developer-convenience-and-confidence>`_.
-The best way to learn how to write dtests is probably by reading the introduction "`How to Write a Dtest <http://www.datastax.com/dev/blog/how-to-write-a-dtest>`_" and by looking at existing, recently updated tests in the project. New tests must follow certain `style conventions <https://github.com/riptano/cassandra-dtest/blob/master/CONTRIBUTING.md>`_ that are being checked before accepting contributions. In contrast to Cassandra, dtest issues and pull-requests are managed on github, therefor you should make sure to link any created dtests in your Cassandra ticket and also refer to the ticket number in your dtest PR.
+The best way to learn how to write dtests is probably by reading the introduction "`How to Write a Dtest <http://www.datastax.com/dev/blog/how-to-write-a-dtest>`_" and by looking at existing, recently updated tests in the project. New tests must follow certain `style conventions <https://github.com/apache/cassandra-dtest/blob/master/CONTRIBUTING.md>`_ that are being checked before accepting contributions. In contrast to Cassandra, dtest issues and pull-requests are managed on github, therefor you should make sure to link any created dtests in your Cassandra ticket and also refer to the ticket number in your dtest PR.
Creating a good dtest can be tough, but it should not prevent you from submitting patches! Please ask in the corresponding JIRA ticket how to write a good dtest for the patch. In most cases a reviewer or committer will able to support you, and in some cases they may offer to write a dtest for you.
@@ -75,7 +84,7 @@
Cassandra Stress Tool
---------------------
-TODO: `CASSANDRA-12365 <https://issues.apache.org/jira/browse/CASSANDRA-12365>`_
+See :ref:`cassandra_stress`
cstar_perf
----------
diff --git a/doc/source/faq/index.rst b/doc/source/faq/index.rst
index d985e37..acb7538 100644
--- a/doc/source/faq/index.rst
+++ b/doc/source/faq/index.rst
@@ -58,7 +58,7 @@
------------------------------
By default, Cassandra uses 7000 for cluster communication (7001 if SSL is enabled), 9042 for native protocol clients,
-and 7199 for JMX (and 9160 for the deprecated Thrift interface). The internode communication and native protocol ports
+and 7199 for JMX. The internode communication and native protocol ports
are configurable in the :ref:`cassandra-yaml`. The JMX port is configurable in ``cassandra-env.sh`` (through JVM
options). All ports are TCP.
@@ -98,15 +98,16 @@
Can I change the replication factor (a a keyspace) on a live cluster?
---------------------------------------------------------------------
-Yes, but it will require running repair (or cleanup) to change the replica count of existing data:
+Yes, but it will require running a full repair (or cleanup) to change the replica count of existing data:
- :ref:`Alter <alter-keyspace-statement>` the replication factor for desired keyspace (using cqlsh for instance).
- If you're reducing the replication factor, run ``nodetool cleanup`` on the cluster to remove surplus replicated data.
Cleanup runs on a per-node basis.
-- If you're increasing the replication factor, run ``nodetool repair`` to ensure data is replicated according to the new
+- If you're increasing the replication factor, run ``nodetool repair -full`` to ensure data is replicated according to the new
configuration. Repair runs on a per-replica set basis. This is an intensive process that may result in adverse cluster
performance. It's highly recommended to do rolling repairs, as an attempt to repair the entire cluster at once will
- most likely swamp it.
+ most likely swamp it. Note that you will need to run a full repair (``-full``) to make sure that already repaired
+ sstables are not skipped.
.. _can-large-blob:
diff --git a/doc/source/getting_started/configuring.rst b/doc/source/getting_started/configuring.rst
index 27fac78..adb86fa 100644
--- a/doc/source/getting_started/configuring.rst
+++ b/doc/source/getting_started/configuring.rst
@@ -17,38 +17,48 @@
Configuring Cassandra
---------------------
-For running Cassandra on a single node, the steps above are enough, you don't really need to change any configuration.
-However, when you deploy a cluster of nodes, or use clients that are not on the same host, then there are some
-parameters that must be changed.
+The :term:`Cassandra` configuration files location varies, depending on the type of installation:
-The Cassandra configuration files can be found in the ``conf`` directory of tarballs. For packages, the configuration
-files will be located in ``/etc/cassandra``.
+- tarball: ``conf`` directory within the tarball install location
+- package: ``/etc/cassandra`` directory
+
+Cassandra's default configuration file, ``cassandra.yaml``, is sufficient to explore a simple single-node :term:`cluster`.
+However, anything beyond running a single-node cluster locally requires additional configuration to various Cassandra configuration files.
+Some examples that require non-default configuration are deploying a multi-node cluster or using clients that are not running on a cluster node.
+
+- ``cassandra.yaml``: the main configuration file for Cassandra
+- ``cassandra-env.sh``: environment variables can be set
+- ``cassandra-rackdc.properties`` OR ``cassandra-topology.properties``: set rack and datacenter information for a cluster
+- ``logback.xml``: logging configuration including logging levels
+- ``jvm-*``: a number of JVM configuration files for both the server and clients
+- ``commitlog_archiving.properties``: set archiving parameters for the :term:`commitlog`
+
+Two sample configuration files can also be found in ``./conf``:
+
+- ``metrics-reporter-config-sample.yaml``: configuring what the metrics-report will collect
+- ``cqlshrc.sample``: how the CQL shell, cqlsh, can be configured
Main runtime properties
^^^^^^^^^^^^^^^^^^^^^^^
-Most of configuration in Cassandra is done via yaml properties that can be set in ``cassandra.yaml``. At a minimum you
+Configuring Cassandra is done by setting yaml properties in the ``cassandra.yaml`` file. At a minimum you
should consider setting the following properties:
-- ``cluster_name``: the name of your cluster.
-- ``seeds``: a comma separated list of the IP addresses of your cluster seeds.
-- ``storage_port``: you don't necessarily need to change this but make sure that there are no firewalls blocking this
- port.
-- ``listen_address``: the IP address of your node, this is what allows other nodes to communicate with this node so it
- is important that you change it. Alternatively, you can set ``listen_interface`` to tell Cassandra which interface to
- use, and consecutively which address to use. Set only one, not both.
-- ``native_transport_port``: as for storage\_port, make sure this port is not blocked by firewalls as clients will
- communicate with Cassandra on this port.
+- ``cluster_name``: Set the name of your cluster.
+- ``seeds``: A comma separated list of the IP addresses of your cluster :term:`seed nodes`.
+- ``storage_port``: Check that you don't have the default port of 7000 blocked by a firewall.
+- ``listen_address``: The :term:`listen address` is the IP address of a node that allows it to communicate with other nodes in the cluster. Set to `localhost` by default. Alternatively, you can set ``listen_interface`` to tell Cassandra which interface to use, and consecutively which address to use. Set one property, not both.
+- ``native_transport_port``: Check that you don't have the default port of 9042 blocked by a firewall, so that clients like cqlsh can communicate with Cassandra on this port.
Changing the location of directories
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
The following yaml properties control the location of directories:
-- ``data_file_directories``: one or more directories where data files are located.
-- ``commitlog_directory``: the directory where commitlog files are located.
-- ``saved_caches_directory``: the directory where saved caches are located.
-- ``hints_directory``: the directory where hints are located.
+- ``data_file_directories``: One or more directories where data files, like :term:`SSTables` are located.
+- ``commitlog_directory``: The directory where commitlog files are located.
+- ``saved_caches_directory``: The directory where saved caches are located.
+- ``hints_directory``: The directory where :term:`hints` are located.
For performance reasons, if you have multiple disks, consider putting commitlog and data files on different disks.
@@ -56,12 +66,15 @@
^^^^^^^^^^^^^^^^^^^^^
JVM-level settings such as heap size can be set in ``cassandra-env.sh``. You can add any additional JVM command line
-argument to the ``JVM_OPTS`` environment variable; when Cassandra starts these arguments will be passed to the JVM.
+argument to the ``JVM_OPTS`` environment variable; when Cassandra starts, these arguments will be passed to the JVM.
Logging
^^^^^^^
-The logger in use is logback. You can change logging properties by editing ``logback.xml``. By default it will log at
-INFO level into a file called ``system.log`` and at debug level into a file called ``debug.log``. When running in the
-foreground, it will also log at INFO level to the console.
+The default logger is `logback`. By default it will log:
+
+- **INFO** level in ``system.log``
+- **DEBUG** level in ``debug.log``
+
+When running in the foreground, it will also log at INFO level to the console. You can change logging properties by editing ``logback.xml`` or by running the `nodetool setlogginglevel` command.
diff --git a/doc/source/getting_started/drivers.rst b/doc/source/getting_started/drivers.rst
index baec823..9a2c156 100644
--- a/doc/source/getting_started/drivers.rst
+++ b/doc/source/getting_started/drivers.rst
@@ -105,3 +105,19 @@
^^^^
- `Rust CQL <https://github.com/neich/rust-cql>`__
+
+Perl
+^^^^
+
+- `Cassandra::Client and DBD::Cassandra <https://github.com/tvdw/perl-dbd-cassandra>`__
+
+Elixir
+^^^^^^
+
+- `Xandra <https://github.com/lexhide/xandra>`__
+- `CQEx <https://github.com/matehat/cqex>`__
+
+Dart
+^^^^
+
+- `dart_cassandra_cql <https://github.com/achilleasa/dart_cassandra_cql>`__
diff --git a/doc/source/getting_started/index.rst b/doc/source/getting_started/index.rst
index 4ca9c4d..a699aee 100644
--- a/doc/source/getting_started/index.rst
+++ b/doc/source/getting_started/index.rst
@@ -29,5 +29,6 @@
configuring
querying
drivers
+ production
diff --git a/doc/source/getting_started/installing.rst b/doc/source/getting_started/installing.rst
index 9be85e5..1d59b8b 100644
--- a/doc/source/getting_started/installing.rst
+++ b/doc/source/getting_started/installing.rst
@@ -19,88 +19,306 @@
Installing Cassandra
--------------------
+These are the instructions for deploying the supported releases of Apache Cassandra on Linux servers.
+
+Cassandra runs on a wide array of Linux distributions including (but not limited to):
+
+- Ubuntu, most notably LTS releases 16.04 to 18.04
+- CentOS & RedHat Enterprise Linux (RHEL) including 6.6 to 7.7
+- Amazon Linux AMIs including 2016.09 through to Linux 2
+- Debian versions 8 & 9
+- SUSE Enterprise Linux 12
+
+This is not an exhaustive list of operating system platforms, nor is it prescriptive. However users will be
+well-advised to conduct exhaustive tests of their own particularly for less-popular distributions of Linux.
+Deploying on older versions is not recommended unless you have previous experience with the older distribution
+in a production environment.
+
Prerequisites
^^^^^^^^^^^^^
-- The latest version of Java 8, either the `Oracle Java Standard Edition 8
+- Install the latest version of Java 8, either the `Oracle Java Standard Edition 8
<http://www.oracle.com/technetwork/java/javase/downloads/index.html>`__ or `OpenJDK 8 <http://openjdk.java.net/>`__. To
verify that you have the correct version of java installed, type ``java -version``.
-
-- For using cqlsh, the latest version of `Python 2.7 <https://www.python.org/downloads/>`__. To verify that you have
+- **NOTE**: *Experimental* support for Java 11 was added in Cassandra 4.0 (`CASSANDRA-9608 <https://issues.apache.org/jira/browse/CASSANDRA-9608>`__).
+ Running Cassandra on Java 11 is *experimental*. Do so at your own risk. For more information, see
+ `NEWS.txt <https://github.com/apache/cassandra/blob/trunk/NEWS.txt>`__.
+- For using cqlsh, the latest version of `Python 2.7 <https://www.python.org/downloads/>`__ or Python 3.6+. To verify that you have
the correct version of Python installed, type ``python --version``.
-Installation from binary tarball files
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Choosing an installation method
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-- Download the latest stable release from the `Apache Cassandra downloads website <http://cassandra.apache.org/download/>`__.
+For most users, installing the binary tarball is the simplest choice. The tarball unpacks all its contents
+into a single location with binaries and configuration files located in their own subdirectories. The most
+obvious attribute of the tarball installation is it does not require ``root`` permissions and can be
+installed on any Linux distribution.
-- Untar the file somewhere, for example:
+Packaged installations require ``root`` permissions. Install the RPM build on CentOS and RHEL-based
+distributions if you want to install Cassandra using YUM. Install the Debian build on Ubuntu and other
+Debian-based distributions if you want to install Cassandra using APT. Note that both the YUM and APT
+methods required ``root`` permissions and will install the binaries and configuration files as the
+``cassandra`` OS user.
+
+Installing the binary tarball
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+1. Verify the version of Java installed. For example:
::
- tar -xvf apache-cassandra-3.6-bin.tar.gz cassandra
+ $ java -version
+ openjdk version "1.8.0_222"
+ OpenJDK Runtime Environment (build 1.8.0_222-8u222-b10-1ubuntu1~16.04.1-b10)
+ OpenJDK 64-Bit Server VM (build 25.222-b10, mixed mode)
-The files will be extracted into ``apache-cassandra-3.6``, you need to substitute 3.6 with the release number that you
-have downloaded.
-
-- Optionally add ``apache-cassandra-3.6\bin`` to your path.
-- Start Cassandra in the foreground by invoking ``bin/cassandra -f`` from the command line. Press "Control-C" to stop
- Cassandra. Start Cassandra in the background by invoking ``bin/cassandra`` from the command line. Invoke ``kill pid``
- or ``pkill -f CassandraDaemon`` to stop Cassandra, where pid is the Cassandra process id, which you can find for
- example by invoking ``pgrep -f CassandraDaemon``.
-- Verify that Cassandra is running by invoking ``bin/nodetool status`` from the command line.
-- Configuration files are located in the ``conf`` sub-directory.
-- Since Cassandra 2.1, log and data directories are located in the ``logs`` and ``data`` sub-directories respectively.
- Older versions defaulted to ``/var/log/cassandra`` and ``/var/lib/cassandra``. Due to this, it is necessary to either
- start Cassandra with root privileges or change ``conf/cassandra.yaml`` to use directories owned by the current user,
- as explained below in the section on changing the location of directories.
-
-Installation from Debian packages
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-- Add the Apache repository of Cassandra to ``/etc/apt/sources.list.d/cassandra.sources.list``, for example for version
- 3.6:
+2. Download the binary tarball from one of the mirrors on the `Apache Cassandra Download <http://cassandra.apache.org/download/>`__
+ site. For example, to download 4.0:
::
- echo "deb https://downloads.apache.org/cassandra/debian 36x main" | sudo tee -a /etc/apt/sources.list.d/cassandra.sources.list
+ $ curl -OL http://apache.mirror.digitalpacific.com.au/cassandra/4.0.0/apache-cassandra-4.0.0-bin.tar.gz
-- Add the Apache Cassandra repository keys:
+NOTE: The mirrors only host the latest versions of each major supported release. To download an earlier
+version of Cassandra, visit the `Apache Archives <http://archive.apache.org/dist/cassandra/>`__.
+
+3. OPTIONAL: Verify the integrity of the downloaded tarball using one of the methods `here <https://www.apache.org/dyn/closer.cgi#verify>`__.
+ For example, to verify the hash of the downloaded file using GPG:
::
- curl https://downloads.apache.org/cassandra/KEYS | sudo apt-key add -
+ $ gpg --print-md SHA256 apache-cassandra-4.0.0-bin.tar.gz
+ apache-cassandra-4.0.0-bin.tar.gz: 28757DDE 589F7041 0F9A6A95 C39EE7E6
+ CDE63440 E2B06B91 AE6B2006 14FA364D
-- Update the repositories:
+Compare the signature with the SHA256 file from the Downloads site:
::
- sudo apt-get update
+ $ curl -L https://downloads.apache.org/cassandra/4.0.0/apache-cassandra-4.0.0-bin.tar.gz.sha256
+ 28757dde589f70410f9a6a95c39ee7e6cde63440e2b06b91ae6b200614fa364d
-- If you encounter this error:
+4. Unpack the tarball:
::
- GPG error: http://www.apache.org 36x InRelease: The following signatures couldn't be verified because the public key is not available: NO_PUBKEY A278B781FE4B2BDA
+ $ tar xzvf apache-cassandra-4.0.0-bin.tar.gz
-Then add the public key A278B781FE4B2BDA as follows:
+The files will be extracted to the ``apache-cassandra-4.0.0/`` directory. This is the tarball installation
+location.
+
+5. Located in the tarball installation location are the directories for the scripts, binaries, utilities, configuration, data and log files:
::
- sudo apt-key adv --keyserver pool.sks-keyservers.net --recv-key A278B781FE4B2BDA
+ <tarball_installation>/
+ bin/
+ conf/
+ data/
+ doc/
+ interface/
+ javadoc/
+ lib/
+ logs/
+ pylib/
+ tools/
+
+For information on how to configure your installation, see
+`Configuring Cassandra <http://cassandra.apache.org/doc/latest/getting_started/configuring.html>`__.
-and repeat ``sudo apt-get update``. The actual key may be different, you get it from the error message itself. For a
-full list of Apache contributors public keys, you can refer to `this link <https://downloads.apache.org/cassandra/KEYS>`__.
-
-- Install Cassandra:
+6. Start Cassandra:
::
- sudo apt-get install cassandra
+ $ cd apache-cassandra-4.0.0/
+ $ bin/cassandra
-- You can start Cassandra with ``sudo service cassandra start`` and stop it with ``sudo service cassandra stop``.
- However, normally the service will start automatically. For this reason be sure to stop it if you need to make any
- configuration changes.
-- Verify that Cassandra is running by invoking ``nodetool status`` from the command line.
-- The default location of configuration files is ``/etc/cassandra``.
-- The default location of log and data directories is ``/var/log/cassandra/`` and ``/var/lib/cassandra``.
+NOTE: This will run Cassandra as the authenticated Linux user.
+
+You can monitor the progress of the startup with:
+
+::
+
+ $ tail -f logs/system.log
+
+Cassandra is ready when you see an entry like this in the ``system.log``:
+
+::
+
+ INFO [main] 2019-12-17 03:03:37,526 Server.java:156 - Starting listening for CQL clients on localhost/127.0.0.1:9042 (unencrypted)...
+
+7. Check the status of Cassandra:
+
+::
+
+ $ bin/nodetool status
+
+The status column in the output should report UN which stands for "Up/Normal".
+
+Alternatively, connect to the database with:
+
+::
+
+ $ bin/cqlsh
+
+Installing the Debian packages
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+1. Verify the version of Java installed. For example:
+
+::
+
+ $ java -version
+ openjdk version "1.8.0_222"
+ OpenJDK Runtime Environment (build 1.8.0_222-8u222-b10-1ubuntu1~16.04.1-b10)
+ OpenJDK 64-Bit Server VM (build 25.222-b10, mixed mode)
+
+2. Add the Apache repository of Cassandra to the file ``cassandra.sources.list``. The latest major version
+ is 4.0 and the corresponding distribution name is ``40x`` (with an "x" as the suffix).
+ For older releases use ``311x`` for C* 3.11 series, ``30x`` for 3.0, ``22x`` for 2.2 and ``21x`` for 2.1.
+ For example, to add the repository for version 4.0 (``40x``):
+
+::
+
+ $ echo "deb http://downloads.apache.org/cassandra/debian 40x main" | sudo tee -a /etc/apt/sources.list.d/cassandra.sources.list
+ deb http://downloads.apache.org/cassandra/debian 40x main
+
+3. Add the Apache Cassandra repository keys to the list of trusted keys on the server:
+
+::
+
+ $ curl https://downloads.apache.org/cassandra/KEYS | sudo apt-key add -
+ % Total % Received % Xferd Average Speed Time Time Time Current
+ Dload Upload Total Spent Left Speed
+ 100 266k 100 266k 0 0 320k 0 --:--:-- --:--:-- --:--:-- 320k
+ OK
+
+4. Update the package index from sources:
+
+::
+
+ $ sudo apt-get update
+
+5. Install Cassandra with APT:
+
+::
+
+ $ sudo apt-get install cassandra
+
+
+NOTE: A new Linux user ``cassandra`` will get created as part of the installation. The Cassandra service
+will also be run as this user.
+
+6. The Cassandra service gets started automatically after installation. Monitor the progress of
+ the startup with:
+
+::
+
+ $ tail -f /var/log/cassandra/system.log
+
+Cassandra is ready when you see an entry like this in the ``system.log``:
+
+::
+
+ INFO [main] 2019-12-17 03:03:37,526 Server.java:156 - Starting listening for CQL clients on localhost/127.0.0.1:9042 (unencrypted)...
+
+NOTE: For information on how to configure your installation, see
+`Configuring Cassandra <http://cassandra.apache.org/doc/latest/getting_started/configuring.html>`__.
+
+7. Check the status of Cassandra:
+
+::
+
+ $ nodetool status
+
+The status column in the output should report ``UN`` which stands for "Up/Normal".
+
+Alternatively, connect to the database with:
+
+::
+
+ $ cqlsh
+
+Installing the RPM packages
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+1. Verify the version of Java installed. For example:
+
+::
+
+ $ java -version
+ openjdk version "1.8.0_222"
+ OpenJDK Runtime Environment (build 1.8.0_232-b09)
+ OpenJDK 64-Bit Server VM (build 25.232-b09, mixed mode)
+
+2. Add the Apache repository of Cassandra to the file ``/etc/yum.repos.d/cassandra.repo`` (as the ``root``
+ user). The latest major version is 4.0 and the corresponding distribution name is ``40x`` (with an "x" as the suffix).
+ For older releases use ``311x`` for C* 3.11 series, ``30x`` for 3.0, ``22x`` for 2.2 and ``21x`` for 2.1.
+ For example, to add the repository for version 4.0 (``40x``):
+
+::
+
+ [cassandra]
+ name=Apache Cassandra
+ baseurl=https://downloads.apache.org/cassandra/redhat/40x/
+ gpgcheck=1
+ repo_gpgcheck=1
+ gpgkey=https://downloads.apache.org/cassandra/KEYS
+
+3. Update the package index from sources:
+
+::
+
+ $ sudo yum update
+
+4. Install Cassandra with YUM:
+
+::
+
+ $ sudo yum install cassandra
+
+
+NOTE: A new Linux user ``cassandra`` will get created as part of the installation. The Cassandra service
+will also be run as this user.
+
+5. Start the Cassandra service:
+
+::
+
+ $ sudo service cassandra start
+
+6. Monitor the progress of the startup with:
+
+::
+
+ $ tail -f /var/log/cassandra/system.log
+
+Cassandra is ready when you see an entry like this in the ``system.log``:
+
+::
+
+ INFO [main] 2019-12-17 03:03:37,526 Server.java:156 - Starting listening for CQL clients on localhost/127.0.0.1:9042 (unencrypted)...
+
+NOTE: For information on how to configure your installation, see
+`Configuring Cassandra <http://cassandra.apache.org/doc/latest/getting_started/configuring.html>`__.
+
+7. Check the status of Cassandra:
+
+::
+
+ $ nodetool status
+
+The status column in the output should report ``UN`` which stands for "Up/Normal".
+
+Alternatively, connect to the database with:
+
+::
+
+ $ cqlsh
+
+Further installation info
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+For help with installation issues, see the `Troubleshooting <http://cassandra.apache.org/doc/latest/troubleshooting/index.html>`__ section.
+
+
diff --git a/doc/source/getting_started/production.rst b/doc/source/getting_started/production.rst
new file mode 100644
index 0000000..fe0c4a5
--- /dev/null
+++ b/doc/source/getting_started/production.rst
@@ -0,0 +1,156 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements. See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership. The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License. You may obtain a copy of the License at
+..
+.. http://www.apache.org/licenses/LICENSE-2.0
+..
+.. Unless required by applicable law or agreed to in writing, software
+.. distributed under the License is distributed on an "AS IS" BASIS,
+.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+.. See the License for the specific language governing permissions and
+.. limitations under the License.
+
+Production Recommendations
+----------------------------
+
+The ``cassandra.yaml`` and ``jvm.options`` files have a number of notes and recommendations for production usage. This page
+expands on some of the notes in these files with additional information.
+
+Tokens
+^^^^^^^
+
+Using more than 1 token (referred to as vnodes) allows for more flexible expansion and more streaming peers when
+bootstrapping new nodes into the cluster. This can limit the negative impact of streaming (I/O and CPU overhead)
+as well as allow for incremental cluster expansion.
+
+As a tradeoff, more tokens will lead to sharing data with more peers, which can result in decreased availability. To learn more about this we
+recommend reading `this paper <https://github.com/jolynch/python_performance_toolkit/raw/master/notebooks/cassandra_availability/whitepaper/cassandra-availability-virtual.pdf>`_.
+
+The number of tokens can be changed using the following setting:
+
+``num_tokens: 16``
+
+
+Here are the most common token counts with a brief explanation of when and why you would use each one.
+
++-------------+---------------------------------------------------------------------------------------------------+
+| Token Count | Description |
++=============+===================================================================================================+
+| 1 | Maximum availablility, maximum cluster size, fewest peers, |
+| | but inflexible expansion. Must always |
+| | double size of cluster to expand and remain balanced. |
++-------------+---------------------------------------------------------------------------------------------------+
+| 4 | A healthy mix of elasticity and availability. Recommended for clusters which will eventually |
+| | reach over 30 nodes. Requires adding approximately 20% more nodes to remain balanced. |
+| | Shrinking a cluster may result in cluster imbalance. |
++-------------+---------------------------------------------------------------------------------------------------+
+| 16 | Best for heavily elastic clusters which expand and shrink regularly, but may have issues |
+| | availability with larger clusters. Not recommended for clusters over 50 nodes. |
++-------------+---------------------------------------------------------------------------------------------------+
+
+
+In addition to setting the token count, it's extremely important that ``allocate_tokens_for_local_replication_factor`` be
+set as well, to ensure even token allocation.
+
+.. _read-ahead:
+
+Read Ahead
+^^^^^^^^^^^
+
+Read ahead is an operating system feature that attempts to keep as much data loaded in the page cache as possible. The
+goal is to decrease latency by using additional throughput on reads where the latency penalty is high due to seek times
+on spinning disks. By leveraging read ahead, the OS can pull additional data into memory without the cost of additional
+seeks. This works well when available RAM is greater than the size of the hot dataset, but can be problematic when the
+hot dataset is much larger than available RAM. The benefit of read ahead decreases as the size of your hot dataset gets
+bigger in proportion to available memory.
+
+With small partitions (usually tables with no partition key, but not limited to this case) and solid state drives, read
+ahead can increase disk usage without any of the latency benefits, and in some cases can result in up to
+a 5x latency and throughput performance penalty. Read heavy, key/value tables with small (under 1KB) rows are especially
+prone to this problem.
+
+We recommend the following read ahead settings:
+
++----------------+-------------------------+
+| Hardware | Initial Recommendation |
++================+=========================+
+|Spinning Disks | 64KB |
++----------------+-------------------------+
+|SSD | 4KB |
++----------------+-------------------------+
+
+Read ahead can be adjusted on Linux systems by using the `blockdev` tool.
+
+For example, we can set read ahead of ``/dev/sda1` to 4KB by doing the following::
+
+ blockdev --setra 8 /dev/sda1
+
+**Note**: blockdev accepts the number of 512 byte sectors to read ahead. The argument of 8 above is equivilent to 4KB.
+
+Since each system is different, use the above recommendations as a starting point and tuning based on your SLA and
+throughput requirements. To understand how read ahead impacts disk resource usage we recommend carefully reading through the
+:ref:`troubleshooting <use-os-tools>` portion of the documentation.
+
+
+Compression
+^^^^^^^^^^^^
+
+Compressed data is stored by compressing fixed size byte buffers and writing the data to disk. The buffer size is
+determined by the ``chunk_length_in_kb`` element in the compression map of the schema settings.
+
+The default setting is 16KB starting with Cassandra 4.0.
+
+Since the entire compressed buffer must be read off disk, using too high of a compression chunk length can lead to
+significant overhead when reading small records. Combined with the default read ahead setting this can result in massive
+read amplification for certain workloads.
+
+LZ4Compressor is the default and recommended compression algorithm.
+
+There is additional information on this topic on `The Last Pickle Blog <https://thelastpickle.com/blog/2018/08/08/compression_performance.html>`_.
+
+Compaction
+^^^^^^^^^^^^
+
+There are different :ref:`compaction <compaction>` strategies available for different workloads.
+We recommend reading up on the different strategies to understand which is the best for your environment. Different tables
+may (and frequently do) use different compaction strategies on the same cluster.
+
+Encryption
+^^^^^^^^^^^
+
+It is significantly easier to set up peer to peer encryption and client server encryption when setting up your production
+cluster as opposed to setting it up once the cluster is already serving production traffic. If you are planning on using network encryption
+eventually (in any form), we recommend setting it up now. Changing these configurations down the line is not impossible,
+but mistakes can result in downtime or data loss.
+
+Ensure Keyspaces are Created with NetworkTopologyStrategy
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Production clusters should never use SimpleStrategy. Production keyspaces should use the NetworkTopologyStrategy (NTS).
+
+For example::
+
+ create KEYSPACE mykeyspace WITH replication =
+ {'class': 'NetworkTopologyStrategy', 'datacenter1': 3};
+
+NetworkTopologyStrategy allows Cassandra to take advantage of multiple racks and data centers.
+
+Configure Racks and Snitch
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+**Correctly configuring or changing racks after a cluster has been provisioned is an unsupported process**. Migrating from
+a single rack to multiple racks is also unsupported and can result in data loss.
+
+Using ``GossipingPropertyFileSnitch`` is the most flexible solution for on premise or mixed cloud environments. ``Ec2Snitch``
+is reliable for AWS EC2 only environments.
+
+
+
+
+
+
+
diff --git a/doc/source/glossary.rst b/doc/source/glossary.rst
new file mode 100644
index 0000000..7f1d92f
--- /dev/null
+++ b/doc/source/glossary.rst
@@ -0,0 +1,35 @@
+.. glossary::
+
+Glossary
+========
+ Cassandra
+ Apache Cassandra is a distributed, high-available, eventually consistent NoSQL open-source database.
+
+ cluster
+ Two or more database instances that exchange messages using the gossip protocol.
+
+ commitlog
+ A file to which the database appends changed data for recovery in the event of a hardware failure.
+
+ datacenter
+ A group of related nodes that are configured together within a cluster for replication and workload segregation purposes.
+ Not necessarily a separate location or physical data center. Datacenter names are case-sensitive and cannot be changed.
+
+ gossip
+ A peer-to-peer communication protocol for exchanging location and state information between nodes.
+
+ hint
+ One of the three ways, in addition to read-repair and full/incremental anti-entropy repair, that Cassandra implements the eventual consistency guarantee that all updates are eventually received by all replicas.
+
+ listen address
+ Address or interface to bind to and tell other Cassandra nodes to connect to
+
+ seed node
+ A seed node is used to bootstrap the gossip process for new nodes joining a cluster. To learn the topology of the ring, a joining node contacts one of the nodes in the -seeds list in cassandra. yaml. The first time you bring up a node in a new cluster, only one node is the seed node.
+
+ snitch
+ The mapping from the IP addresses of nodes to physical and virtual locations, such as racks and data centers. There are several types of snitches.
+ The type of snitch affects the request routing mechanism.
+
+ SSTable
+ An SSTable provides a persistent,ordered immutable map from keys to values, where both keys and values are arbitrary byte strings.
diff --git a/doc/source/index.rst b/doc/source/index.rst
index 562603d..302f8e7 100644
--- a/doc/source/index.rst
+++ b/doc/source/index.rst
@@ -27,15 +27,17 @@
:maxdepth: 2
getting_started/index
+ new/index
architecture/index
- data_modeling/index
cql/index
+ data_modeling/index
configuration/index
operating/index
tools/index
troubleshooting/index
development/index
faq/index
+ plugins/index
bugs
contactus
diff --git a/doc/source/new/Figure_1.jpg b/doc/source/new/Figure_1.jpg
new file mode 100644
index 0000000..ccaec67
--- /dev/null
+++ b/doc/source/new/Figure_1.jpg
Binary files differ
diff --git a/doc/source/new/Figure_2.jpg b/doc/source/new/Figure_2.jpg
new file mode 100644
index 0000000..099e15f
--- /dev/null
+++ b/doc/source/new/Figure_2.jpg
Binary files differ
diff --git a/doc/source/new/auditlogging.rst b/doc/source/new/auditlogging.rst
new file mode 100644
index 0000000..0842810
--- /dev/null
+++ b/doc/source/new/auditlogging.rst
@@ -0,0 +1,461 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements. See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership. The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License. You may obtain a copy of the License at
+..
+.. http://www.apache.org/licenses/LICENSE-2.0
+..
+.. Unless required by applicable law or agreed to in writing, software
+.. distributed under the License is distributed on an "AS IS" BASIS,
+.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+.. See the License for the specific language governing permissions and
+.. limitations under the License.
+
+Audit Logging
+-------------
+
+Audit Logging is a new feature in Apache Cassandra 4.0 (`CASSANDRA-12151
+<https://issues.apache.org/jira/browse/CASSANDRA-12151>`_). All database activity is logged to a directory in the local filesystem and the audit log files are rolled periodically. All database operations are monitored and recorded. Audit logs are stored in local directory files instead of the database itself as it provides several benefits, some of which are:
+
+- No additional database capacity is needed to store audit logs
+- No query tool is required while storing the audit logs in the database would require a query tool
+- Latency of database operations is not affected; no performance impact
+- It is easier to implement file based logging than database based logging
+
+What does Audit Logging Log?
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Audit logging logs:
+
+1. All authentication which includes successful and failed login attempts
+2. All database command requests to CQL. Both failed and successful CQL is logged
+
+More specifically an audit log entry could be one of two types:
+
+a) CQL Audit Log Entry Type or
+b) Common Audit Log Entry Type
+
+Each of these types comprises of several database operations. The CQL Audit Log Entry Type could be one of the following; the category of the CQL audit log entry type is listed in parentheses.
+
+1. SELECT(QUERY),
+2. UPDATE(DML),
+3. DELETE(DML),
+4. TRUNCATE(DDL),
+5. CREATE_KEYSPACE(DDL),
+6. ALTER_KEYSPACE(DDL),
+7. DROP_KEYSPACE(DDL),
+8. CREATE_TABLE(DDL),
+9. DROP_TABLE(DDL),
+10. PREPARE_STATEMENT(PREPARE),
+11. DROP_TRIGGER(DDL),
+12. LIST_USERS(DCL),
+13. CREATE_INDEX(DDL),
+14. DROP_INDEX(DDL),
+15. GRANT(DCL),
+16. REVOKE(DCL),
+17. CREATE_TYPE(DDL),
+18. DROP_AGGREGATE(DDL),
+19. ALTER_VIEW(DDL),
+20. CREATE_VIEW(DDL),
+21. DROP_ROLE(DCL),
+22. CREATE_FUNCTION(DDL),
+23. ALTER_TABLE(DDL),
+24. BATCH(DML),
+25. CREATE_AGGREGATE(DDL),
+26. DROP_VIEW(DDL),
+27. DROP_TYPE(DDL),
+28. DROP_FUNCTION(DDL),
+29. ALTER_ROLE(DCL),
+30. CREATE_TRIGGER(DDL),
+31. LIST_ROLES(DCL),
+32. LIST_PERMISSIONS(DCL),
+33. ALTER_TYPE(DDL),
+34. CREATE_ROLE(DCL),
+35. USE_KEYSPACE (OTHER).
+
+The Common Audit Log Entry Type could be one of the following; the category of the Common audit log entry type is listed in parentheses.
+
+1. REQUEST_FAILURE(ERROR),
+2. LOGIN_ERROR(AUTH),
+3. UNAUTHORIZED_ATTEMPT(AUTH),
+4. LOGIN_SUCCESS (AUTH).
+
+What Audit Logging does not Log?
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Audit logging does not log:
+
+1. Configuration changes made in ``cassandra.yaml``
+2. Nodetool Commands
+
+Audit Logging is Flexible and Configurable
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Audit logging is flexible and configurable in ``cassandra.yaml`` as follows:
+
+- Keyspaces and tables to be monitored and audited may be specified.
+- Users to be included/excluded may be specified. By default all users are audit logged.
+- Categories of operations to audit or exclude may be specified.
+- The frequency at which to roll the log files may be specified. Default frequency is hourly.
+
+Configuring Audit Logging
+^^^^^^^^^^^^^^^^^^^^^^^^^
+Audit Logging is configured on each node separately. Audit Logging is configured in ``cassandra.yaml`` in the ``audit_logging_options`` setting.
+The settings may be same/different on each node.
+
+Enabling Audit Logging
+**********************
+Audit logging is enabled by setting the ``enabled`` option to ``true`` in the ``audit_logging_options`` setting.
+
+::
+
+ audit_logging_options:
+ enabled: true
+
+Setting the Logger
+******************
+The audit logger is set with the ``logger`` option.
+
+::
+
+ logger:
+ - class_name: BinAuditLogger
+
+Two types of audit loggers are supported: ``FileAuditLogger`` and ``BinAuditLogger``.
+``BinAuditLogger`` is the default setting. The ``BinAuditLogger`` is an efficient way to log events to file in a binary format.
+
+``FileAuditLogger`` is synchronous, file-based audit logger; just uses the standard logging mechanism. ``FileAuditLogger`` logs events to ``audit/audit.log`` file using ``slf4j`` logger.
+
+The ``NoOpAuditLogger`` is a No-Op implementation of the audit logger to be used as a default audit logger when audit logging is disabled.
+
+It is possible to configure your custom logger implementation by injecting a map of property keys and their respective values. Default `IAuditLogger`
+implementations shipped with Cassandra do not react on these properties but your custom logger might. They would be present as
+a parameter of logger constructor (as `Map<String, String>`). In ``cassandra.yaml`` file, you may configure it like this:
+
+::
+
+ logger:
+ - class_name: MyCustomAuditLogger
+ parameters:
+ - key1: value1
+ key2: value2
+
+When it comes to configuring these parameters, you can use respective ``enableAuditLog`` method in ``StorageServiceMBean``.
+There are two methods of same name with different signatures. The first one does not accept a map where your parameters would be. This method
+is used primarily e.g. from JConsole or similar tooling. JConsole can not accept a map to be sent over JMX so in order to be able to enable it
+from there, even without any parameters, use this method. ``BinAuditLogger`` does not need any parameters to run with so invoking this method is fine.
+The second one does accept a map with your custom parameters so you can pass them programmatically. ``enableauditlog`` command of ``nodetool`` uses
+the first ``enableAuditLog`` method mentioned. Hence, currently, there is not a way how to pass parameters to your custom audit logger from ``nodetool``.
+
+Setting the Audit Logs Directory
+********************************
+The audit logs directory is set with the ``audit_logs_dir`` option. A new directory is not created automatically and an existing directory must be set. Audit Logs directory can be configured using ``cassandra.logdir.audit`` system property or default is set to ``cassandra.logdir + /audit/``. A user created directory may be set. As an example, create a directory for the audit logs and set its permissions.
+
+::
+
+ sudo mkdir –p /cassandra/audit/logs/hourly
+ sudo chmod -R 777 /cassandra/audit/logs/hourly
+
+Set the directory for the audit logs directory using the ``audit_logs_dir`` option.
+
+::
+
+ audit_logs_dir: "/cassandra/audit/logs/hourly"
+
+
+Setting Keyspaces to Audit
+**************************
+Set the keyspaces to include with the ``included_keyspaces`` option and the keyspaces to exclude with the ``excluded_keyspaces`` option. By default all keyspaces are included. By default, ``system``, ``system_schema`` and ``system_virtual_schema`` are excluded.
+
+::
+
+ # included_keyspaces:
+ # excluded_keyspaces: system, system_schema, system_virtual_schema
+
+Setting Categories to Audit
+***************************
+
+The categories of database operations to be included are specified with the ``included_categories`` option as a comma separated list. By default all supported categories are included. The categories of database operations to be excluded are specified with ``excluded_categories`` option as a comma separated list. By default no category is excluded.
+
+::
+
+ # included_categories:
+ # excluded_categories:
+
+The supported categories for audit log are:
+
+1. QUERY
+2. DML
+3. DDL
+4. DCL
+5. OTHER
+6. AUTH
+7. ERROR
+8. PREPARE
+
+Setting Users to Audit
+**********************
+
+Users to audit log are set with the ``included_users`` and ``excluded_users`` options. The ``included_users`` option specifies a comma separated list of users to include explicitly and by default all users are included. The ``excluded_users`` option specifies a comma separated list of users to exclude explicitly and by default no user is excluded.
+
+::
+
+ # included_users:
+ # excluded_users:
+
+Setting the Roll Frequency
+***************************
+The ``roll_cycle`` option sets the frequency at which the audit log file is rolled. Supported values are ``MINUTELY``, ``HOURLY``, and ``DAILY``. Default value is ``HOURLY``, which implies that after every hour a new audit log file is created.
+
+::
+
+ roll_cycle: HOURLY
+
+An audit log file could get rolled for other reasons as well such as a log file reaches the configured size threshold.
+
+Setting Archiving Options
+*************************
+
+The archiving options are for archiving the rolled audit logs. The ``archive`` command to use is set with the ``archive_command`` option and the ``max_archive_retries`` sets the maximum # of tries of failed archive commands.
+
+::
+
+ # archive_command:
+ # max_archive_retries: 10
+
+Default archive command is ``"/path/to/script.sh %path"`` where ``%path`` is replaced with the file being rolled:
+
+Other Settings
+***************
+
+The other audit logs settings are as follows.
+
+::
+
+ # block: true
+ # max_queue_weight: 268435456 # 256 MiB
+ # max_log_size: 17179869184 # 16 GiB
+
+The ``block`` option specifies whether the audit logging should block if the logging falls behind or should drop log records.
+
+The ``max_queue_weight`` option sets the maximum weight of in memory queue for records waiting to be written to the file before blocking or dropping.
+
+The ``max_log_size`` option sets the maximum size of the rolled files to retain on disk before deleting the oldest.
+
+Using Nodetool to Enable Audit Logging
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+The ``nodetool enableauditlog`` command may be used to enable audit logs and it overrides the settings in ``cassandra.yaml``. The ``nodetool enableauditlog`` command syntax is as follows.
+
+::
+
+ nodetool [(-h <host> | --host <host>)] [(-p <port> | --port <port>)]
+ [(-pp | --print-port)] [(-pw <password> | --password <password>)]
+ [(-pwf <passwordFilePath> | --password-file <passwordFilePath>)]
+ [(-u <username> | --username <username>)] enableauditlog
+ [--excluded-categories <excluded_categories>]
+ [--excluded-keyspaces <excluded_keyspaces>]
+ [--excluded-users <excluded_users>]
+ [--included-categories <included_categories>]
+ [--included-keyspaces <included_keyspaces>]
+ [--included-users <included_users>] [--logger <logger>]
+
+OPTIONS
+ --excluded-categories <excluded_categories>
+ Comma separated list of Audit Log Categories to be excluded for
+ audit log. If not set the value from cassandra.yaml will be used
+
+ --excluded-keyspaces <excluded_keyspaces>
+ Comma separated list of keyspaces to be excluded for audit log. If
+ not set the value from cassandra.yaml will be used
+
+ --excluded-users <excluded_users>
+ Comma separated list of users to be excluded for audit log. If not
+ set the value from cassandra.yaml will be used
+
+ -h <host>, --host <host>
+ Node hostname or ip address
+
+ --included-categories <included_categories>
+ Comma separated list of Audit Log Categories to be included for
+ audit log. If not set the value from cassandra.yaml will be used
+
+ --included-keyspaces <included_keyspaces>
+ Comma separated list of keyspaces to be included for audit log. If
+ not set the value from cassandra.yaml will be used
+
+ --included-users <included_users>
+ Comma separated list of users to be included for audit log. If not
+ set the value from cassandra.yaml will be used
+
+ --logger <logger>
+ Logger name to be used for AuditLogging. Default BinAuditLogger. If
+ not set the value from cassandra.yaml will be used
+
+ -p <port>, --port <port>
+ Remote jmx agent port number
+
+ -pp, --print-port
+ Operate in 4.0 mode with hosts disambiguated by port number
+
+ -pw <password>, --password <password>
+ Remote jmx agent password
+
+ -pwf <passwordFilePath>, --password-file <passwordFilePath>
+ Path to the JMX password file
+
+ -u <username>, --username <username>
+ Remote jmx agent username
+
+
+The ``nodetool disableauditlog`` command disables audit log. The command syntax is as follows.
+
+::
+
+ nodetool [(-h <host> | --host <host>)] [(-p <port> | --port <port>)]
+ [(-pp | --print-port)] [(-pw <password> | --password <password>)]
+ [(-pwf <passwordFilePath> | --password-file <passwordFilePath>)]
+ [(-u <username> | --username <username>)] disableauditlog
+
+OPTIONS
+ -h <host>, --host <host>
+ Node hostname or ip address
+
+ -p <port>, --port <port>
+ Remote jmx agent port number
+
+ -pp, --print-port
+ Operate in 4.0 mode with hosts disambiguated by port number
+
+ -pw <password>, --password <password>
+ Remote jmx agent password
+
+ -pwf <passwordFilePath>, --password-file <passwordFilePath>
+ Path to the JMX password file
+
+ -u <username>, --username <username>
+ Remote jmx agent username
+
+Viewing the Audit Logs
+^^^^^^^^^^^^^^^^^^^^^^
+An audit log event comprises of a keyspace that is being audited, the operation that is being logged, the scope and the user. An audit log entry comprises of the following attributes concatenated with a "|".
+
+::
+
+ type (AuditLogEntryType): Type of request
+ source (InetAddressAndPort): Source IP Address from which request originated
+ user (String): User name
+ timestamp (long ): Timestamp of the request
+ batch (UUID): Batch of request
+ keyspace (String): Keyspace on which request is made
+ scope (String): Scope of request such as Table/Function/Aggregate name
+ operation (String): Database operation such as CQL command
+ options (QueryOptions): CQL Query options
+ state (QueryState): State related to a given query
+
+Some of these attributes may not be applicable to a given request and not all of these options must be set.
+
+An Audit Logging Demo
+^^^^^^^^^^^^^^^^^^^^^^
+To demonstrate audit logging enable and configure audit logs with following settings.
+
+::
+
+ audit_logging_options:
+ enabled: true
+ logger:
+ - class_name: BinAuditLogger
+ audit_logs_dir: "/cassandra/audit/logs/hourly"
+ # included_keyspaces:
+ # excluded_keyspaces: system, system_schema, system_virtual_schema
+ # included_categories:
+ # excluded_categories:
+ # included_users:
+ # excluded_users:
+ roll_cycle: HOURLY
+ # block: true
+ # max_queue_weight: 268435456 # 256 MiB
+ # max_log_size: 17179869184 # 16 GiB
+ ## archive command is "/path/to/script.sh %path" where %path is replaced with the file being rolled:
+ # archive_command:
+ # max_archive_retries: 10
+
+Create the audit log directory ``/cassandra/audit/logs/hourly`` and set its permissions as discussed earlier. Run some CQL commands such as create a keyspace, create a table and query a table. Any supported CQL commands may be run as discussed in section **What does Audit Logging Log?**. Change directory (with ``cd`` command) to the audit logs directory.
+
+::
+
+ cd /cassandra/audit/logs/hourly
+
+List the files/directories and some ``.cq4`` files should get listed. These are the audit logs files.
+
+::
+
+ [ec2-user@ip-10-0-2-238 hourly]$ ls -l
+ total 28
+ -rw-rw-r--. 1 ec2-user ec2-user 83886080 Aug 2 03:01 20190802-02.cq4
+ -rw-rw-r--. 1 ec2-user ec2-user 83886080 Aug 2 03:01 20190802-03.cq4
+ -rw-rw-r--. 1 ec2-user ec2-user 65536 Aug 2 03:01 directory-listing.cq4t
+
+The ``auditlogviewer`` tool is used to dump audit logs. Run the ``auditlogviewer`` tool. Audit log files directory path is a required argument. The output should be similar to the following output.
+
+::
+
+ [ec2-user@ip-10-0-2-238 hourly]$ auditlogviewer /cassandra/audit/logs/hourly
+ WARN 03:12:11,124 Using Pauser.sleepy() as not enough processors, have 2, needs 8+
+ Type: AuditLog
+ LogMessage:
+ user:anonymous|host:10.0.2.238:7000|source:/127.0.0.1|port:46264|timestamp:1564711427328|type :USE_KEYSPACE|category:OTHER|ks:auditlogkeyspace|operation:USE AuditLogKeyspace;
+ Type: AuditLog
+ LogMessage:
+ user:anonymous|host:10.0.2.238:7000|source:/127.0.0.1|port:46264|timestamp:1564711427329|type :USE_KEYSPACE|category:OTHER|ks:auditlogkeyspace|operation:USE "auditlogkeyspace"
+ Type: AuditLog
+ LogMessage:
+ user:anonymous|host:10.0.2.238:7000|source:/127.0.0.1|port:46264|timestamp:1564711446279|type :SELECT|category:QUERY|ks:auditlogkeyspace|scope:t|operation:SELECT * FROM t;
+ Type: AuditLog
+ LogMessage:
+ user:anonymous|host:10.0.2.238:7000|source:/127.0.0.1|port:46264|timestamp:1564713878834|type :DROP_TABLE|category:DDL|ks:auditlogkeyspace|scope:t|operation:DROP TABLE IF EXISTS
+ AuditLogKeyspace.t;
+ Type: AuditLog
+ LogMessage:
+ user:anonymous|host:10.0.2.238:7000|source:/3.91.56.164|port:42382|timestamp:1564714618360|ty
+ pe:REQUEST_FAILURE|category:ERROR|operation:CREATE KEYSPACE AuditLogKeyspace
+ WITH replication = {'class': 'SimpleStrategy', 'replication_factor' : 1};; Cannot add
+ existing keyspace "auditlogkeyspace"
+ Type: AuditLog
+ LogMessage:
+ user:anonymous|host:10.0.2.238:7000|source:/127.0.0.1|port:46264|timestamp:1564714690968|type :DROP_KEYSPACE|category:DDL|ks:auditlogkeyspace|operation:DROP KEYSPACE AuditLogKeyspace;
+ Type: AuditLog
+ LogMessage:
+ user:anonymous|host:10.0.2.238:7000|source:/3.91.56.164|port:42406|timestamp:1564714708329|ty pe:CREATE_KEYSPACE|category:DDL|ks:auditlogkeyspace|operation:CREATE KEYSPACE
+ AuditLogKeyspace
+ WITH replication = {'class': 'SimpleStrategy', 'replication_factor' : 1};
+ Type: AuditLog
+ LogMessage:
+ user:anonymous|host:10.0.2.238:7000|source:/127.0.0.1|port:46264|timestamp:1564714870678|type :USE_KEYSPACE|category:OTHER|ks:auditlogkeyspace|operation:USE auditlogkeyspace;
+ [ec2-user@ip-10-0-2-238 hourly]$
+
+
+The ``auditlogviewer`` tool usage syntax is as follows.
+
+::
+
+ ./auditlogviewer
+ Audit log files directory path is a required argument.
+ usage: auditlogviewer <path1> [<path2>...<pathN>] [options]
+ --
+ View the audit log contents in human readable format
+ --
+ Options are:
+ -f,--follow Upon reaching the end of the log continue indefinitely
+ waiting for more records
+ -h,--help display this help message
+ -r,--roll_cycle How often to roll the log file was rolled. May be
+ necessary for Chronicle to correctly parse file names. (MINUTELY, HOURLY,
+ DAILY). Default HOURLY.
+
+Diagnostic events for user audit logging
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Any native transport enabled client is able to subscribe to diagnostic events that are raised around authentication and CQL operations. These events can then be consumed and used by external tools to implement a Cassandra user auditing solution.
+
diff --git a/doc/source/new/fqllogging.rst b/doc/source/new/fqllogging.rst
new file mode 100644
index 0000000..881f39f
--- /dev/null
+++ b/doc/source/new/fqllogging.rst
@@ -0,0 +1,689 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements. See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership. The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License. You may obtain a copy of the License at
+..
+.. http://www.apache.org/licenses/LICENSE-2.0
+..
+.. Unless required by applicable law or agreed to in writing, software
+.. distributed under the License is distributed on an "AS IS" BASIS,
+.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+.. See the License for the specific language governing permissions and
+.. limitations under the License.
+
+Full Query Logging
+------------------
+
+Apache Cassandra 4.0 adds a new feature to support a means of logging all queries as they were invoked (`CASSANDRA-13983
+<https://issues.apache.org/jira/browse/CASSANDRA-13983>`_). For correctness testing it's useful to be able to capture production traffic so that it can be replayed against both the old and new versions of Cassandra while comparing the results.
+
+Cassandra 4.0 includes an implementation of a full query logging (FQL) that uses chronicle-queue to implement a rotating log of queries. Some of the features of FQL are:
+
+- Single thread asynchronously writes log entries to disk to reduce impact on query latency
+- Heap memory usage bounded by a weighted queue with configurable maximum weight sitting in front of logging thread
+- If the weighted queue is full producers can be blocked or samples can be dropped
+- Disk utilization is bounded by deleting old log segments once a configurable size is reached
+- The on disk serialization uses a flexible schema binary format (chronicle-wire) making it easy to skip unrecognized fields, add new ones, and omit old ones.
+- Can be enabled and configured via JMX, disabled, and reset (delete on disk data), logging path is configurable via both JMX and YAML
+- Introduce new ``fqltool`` in ``/bin`` that currently implements ``Dump`` which can dump in a readable format full query logs as well as follow active full query logs. FQL ``Replay`` and ``Compare`` are also available.
+
+Cassandra 4.0 has a binary full query log based on Chronicle Queue that can be controlled using ``nodetool enablefullquerylog``, ``disablefullquerylog``, and ``resetfullquerylog``. The log contains all queries invoked, approximate time they were invoked, any parameters necessary to bind wildcard values, and all query options. A readable version of the log can be dumped or tailed using the new ``bin/fqltool`` utility. The full query log is designed to be safe to use in production and limits utilization of heap memory and disk space with limits you can specify when enabling the log.
+
+Objective
+^^^^^^^^^^
+Full Query Logging logs all requests to the CQL interface. The full query logs could be used for debugging, performance benchmarking, testing and auditing CQL queries. The audit logs also include CQL requests but full query logging is dedicated to CQL requests only with features such as FQL Replay and FQL Compare that are not available in audit logging.
+
+Full Query Logger
+^^^^^^^^^^^^^^^^^^
+The Full Query Logger is a logger that logs entire query contents after the query finishes. FQL only logs the queries that successfully complete. The other queries (e.g. timed out, failed) are not to be logged. Queries are logged in one of two modes: single query or batch of queries. The log for an invocation of a batch of queries includes the following attributes:
+
+::
+
+ type - The type of the batch
+ queries - CQL text of the queries
+ values - Values to bind to as parameters for the queries
+ queryOptions - Options associated with the query invocation
+ queryState - Timestamp state associated with the query invocation
+ batchTimeMillis - Approximate time in milliseconds since the epoch since the batch was invoked
+
+The log for single CQL query includes the following attributes:
+
+::
+
+ query - CQL query text
+ queryOptions - Options associated with the query invocation
+ queryState - Timestamp state associated with the query invocation
+ queryTimeMillis - Approximate time in milliseconds since the epoch since the batch was invoked
+
+Full query logging is backed up by ``BinLog``. BinLog is a quick and dirty binary log. Its goal is good enough performance, predictable footprint, simplicity in terms of implementation and configuration and most importantly minimal impact on producers of log records. Performance safety is accomplished by feeding items to the binary log using a weighted queue and dropping records if the binary log falls sufficiently far behind. Simplicity and good enough performance is achieved by using a single log writing thread as well as Chronicle Queue to handle writing the log, making it available for readers, as well as log rolling.
+
+Weighted queue is a wrapper around any blocking queue that turns it into a blocking weighted queue. The queue will weigh each element being added and removed. Adding to the queue is blocked if adding would violate the weight bound. If an element weighs in at larger than the capacity of the queue then exactly one such element will be allowed into the queue at a time. If the weight of an object changes after it is added it could create issues. Checking weight should be cheap so memorize expensive to compute weights. If weight throws that can also result in leaked permits so it's always a good idea to memorize weight so it doesn't throw. In the interests of not writing unit tests for methods no one uses there is a lot of ``UnsupportedOperationException``. If you need them then add them and add proper unit tests to ``WeightedQueueTest``. "Good" tests. 100% coverage including exception paths and resource leaks.
+
+
+The FQL tracks information about store files:
+
+- Store files as they are added and their storage impact. Delete them if over storage limit.
+- The files in the chronicle queue that have already rolled
+- The number of bytes in store files that have already rolled
+
+FQL logger sequence is as follows:
+
+1. Start the consumer thread that writes log records. Can only be done once.
+2. Offer a record to the log. If the in memory queue is full the record will be dropped and offer will return false.
+3. Put a record into the log. If the in memory queue is full the putting thread will be blocked until there is space or it is interrupted.
+4. Clean up the buffers on thread exit, finalization will check again once this is no longer reachable ensuring there are no stragglers in the queue.
+5. Stop the consumer thread that writes log records. Can be called multiple times.
+
+Next, we shall demonstrate full query logging with an example.
+
+
+Configuring Full Query Logging
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Full Query Logger default options are configured on a per node basis in ``cassandra.yaml`` with following configuration property.
+
+::
+
+ full_query_logging_options:
+
+As an example setup create a three node Cassandra 4.0 cluster. The ``nodetool status`` command lists the nodes in the cluster.
+
+::
+
+ [ec2-user@ip-10-0-2-238 ~]$ nodetool status
+ Datacenter: us-east-1
+ =====================
+ Status=Up/Down
+ |/ State=Normal/Leaving/Joining/Moving
+ -- AddressLoad Tokens Owns (effective) Host ID Rack
+ UN 10.0.1.115 442.42 KiB 25632.6% b64cb32a-b32a-46b4-9eeb-e123fa8fc287 us-east-1b
+ UN 10.0.3.206 559.52 KiB 25631.9% 74863177-684b-45f4-99f7-d1006625dc9e us-east-1d
+ UN 10.0.2.238 587.87 KiB 25635.5% 4dcdadd2-41f9-4f34-9892-1f20868b27c7 us-east-1c
+
+
+In subsequent sub-sections we shall discuss enabling and configuring full query logging.
+
+Setting the FQL Directory
+*************************
+
+A dedicated directory path must be provided to write full query log data to when the full query log is enabled. The directory for FQL must exist, and have permissions set. The full query log will recursively delete the contents of this path at times. It is recommended not to place links in this directory to other sections of the filesystem. The ``full_query_log_dir`` property in ``cassandra.yaml`` is pre-configured.
+
+::
+
+ full_query_log_dir: /tmp/cassandrafullquerylog
+
+The ``log_dir`` option may be used to configure the FQL directory if the ``full_query_log_dir`` is not set.
+
+::
+
+ full_query_logging_options:
+ # log_dir:
+
+Create the FQL directory if it does not exist and set its permissions.
+
+::
+
+ sudo mkdir -p /tmp/cassandrafullquerylog
+ sudo chmod -R 777 /tmp/cassandrafullquerylog
+
+Setting the Roll Cycle
+**********************
+
+The ``roll_cycle`` option sets how often to roll FQL log segments so they can potentially be reclaimed. Supported values are ``MINUTELY``, ``HOURLY`` and ``DAILY``. Default setting is ``HOURLY``.
+
+::
+
+ roll_cycle: HOURLY
+
+Setting Other Options
+*********************
+
+The ``block`` option specifies whether the FQL should block if the FQL falls behind or should drop log records. Default value of ``block`` is ``true``. The ``max_queue_weight`` option sets the maximum weight of in memory queue for records waiting to be written to the file before blocking or dropping. The ``max_log_size`` option sets the maximum size of the rolled files to retain on disk before deleting the oldest file. The ``archive_command`` option sets the archive command to execute on rolled log files. The ``max_archive_retries`` option sets the max number of retries of failed archive commands.
+
+::
+
+ # block: true
+ # max_queue_weight: 268435456 # 256 MiB
+ # max_log_size: 17179869184 # 16 GiB
+ ## archive command is "/path/to/script.sh %path" where %path is replaced with the file
+ being rolled:
+ # archive_command:
+ # max_archive_retries: 10
+
+The ``max_queue_weight`` must be > 0. Similarly ``max_log_size`` must be > 0. An example full query logging options is as follows.
+
+::
+
+ full_query_log_dir: /tmp/cassandrafullquerylog
+
+ # default options for full query logging - these can be overridden from command line when
+ executing
+ # nodetool enablefullquerylog
+ # nodetool enablefullquerylog
+ #full_query_logging_options:
+ # log_dir:
+ roll_cycle: HOURLY
+ # block: true
+ # max_queue_weight: 268435456 # 256 MiB
+ # max_log_size: 17179869184 # 16 GiB
+ ## archive command is "/path/to/script.sh %path" where %path is replaced with the file
+ being rolled:
+ # archive_command:
+ # max_archive_retries: 10
+
+The ``full_query_log_dir`` setting is not within the ``full_query_logging_options`` but still is for full query logging.
+
+Enabling Full Query Logging
+***************************
+
+Full Query Logging is enabled on a per-node basis. . The ``nodetool enablefullquerylog`` command is used to enable full query logging. Defaults for the options are configured in ``cassandra.yaml`` and these can be overridden from command line.
+
+The syntax of the nodetool enablefullquerylog command is as follows:
+
+::
+
+ nodetool [(-h <host> | --host <host>)] [(-p <port> | --port <port>)]
+ [(-pp | --print-port)] [(-pw <password> | --password <password>)]
+ [(-pwf <passwordFilePath> | --password-file <passwordFilePath>)]
+ [(-u <username> | --username <username>)] enablefullquerylog
+ [--archive-command <archive_command>] [--blocking]
+ [--max-archive-retries <archive_retries>]
+ [--max-log-size <max_log_size>] [--max-queue-weight <max_queue_weight>]
+ [--path <path>] [--roll-cycle <roll_cycle>]
+
+ OPTIONS
+ --archive-command <archive_command>
+ Command that will handle archiving rolled full query log files.
+ Format is "/path/to/script.sh %path" where %path will be replaced
+ with the file to archive
+
+ --blocking
+ If the queue is full whether to block producers or drop samples.
+
+ -h <host>, --host <host>
+ Node hostname or ip address
+
+ --max-archive-retries <archive_retries>
+ Max number of archive retries.
+
+ --max-log-size <max_log_size>
+ How many bytes of log data to store before dropping segments. Might
+ not be respected if a log file hasn't rolled so it can be deleted.
+
+ --max-queue-weight <max_queue_weight>
+ Maximum number of bytes of query data to queue to disk before
+ blocking or dropping samples.
+
+ -p <port>, --port <port>
+ Remote jmx agent port number
+
+ --path <path>
+ Path to store the full query log at. Will have it's contents
+ recursively deleted.
+
+ -pp, --print-port
+ Operate in 4.0 mode with hosts disambiguated by port number
+
+ -pw <password>, --password <password>
+ Remote jmx agent password
+
+ -pwf <passwordFilePath>, --password-file <passwordFilePath>
+ Path to the JMX password file
+
+ --roll-cycle <roll_cycle>
+ How often to roll the log file (MINUTELY, HOURLY, DAILY).
+
+ -u <username>, --username <username>
+ Remote jmx agent username
+
+Run the following command on each node in the cluster.
+
+::
+
+ nodetool enablefullquerylog --path /tmp/cassandrafullquerylog
+
+After the full query logging has been enabled run some CQL statements to generate full query logs.
+
+Running CQL Statements
+^^^^^^^^^^^^^^^^^^^^^^^
+
+Start CQL interface with ``cqlsh`` command.
+
+::
+
+ [ec2-user@ip-10-0-2-238 ~]$ cqlsh
+ Connected to Cassandra Cluster at 127.0.0.1:9042.
+ [cqlsh 5.0.1 | Cassandra 4.0-SNAPSHOT | CQL spec 3.4.5 | Native protocol v4]
+ Use HELP for help.
+ cqlsh>
+
+Run some CQL statements. Create a keyspace. Create a table and add some data. Query the table.
+
+::
+
+ cqlsh> CREATE KEYSPACE AuditLogKeyspace
+ ... WITH replication = {'class': 'SimpleStrategy', 'replication_factor' : 1};
+ cqlsh> USE AuditLogKeyspace;
+ cqlsh:auditlogkeyspace> CREATE TABLE t (
+ ...id int,
+ ...k int,
+ ...v text,
+ ...PRIMARY KEY (id)
+ ... );
+ cqlsh:auditlogkeyspace> INSERT INTO t (id, k, v) VALUES (0, 0, 'val0');
+ cqlsh:auditlogkeyspace> INSERT INTO t (id, k, v) VALUES (0, 1, 'val1');
+ cqlsh:auditlogkeyspace> SELECT * FROM t;
+
+ id | k | v
+ ----+---+------
+ 0 | 1 | val1
+
+ (1 rows)
+ cqlsh:auditlogkeyspace>
+
+Viewing the Full Query Logs
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
+The ``fqltool`` is used to view the full query logs. The ``fqltool`` has the following usage syntax.
+
+::
+
+ fqltool <command> [<args>]
+
+ The most commonly used fqltool commands are:
+ compare Compare result files generated by fqltool replay
+ dump Dump the contents of a full query log
+ help Display help information
+ replay Replay full query logs
+
+ See 'fqltool help <command>' for more information on a specific command.
+
+The ``fqltool dump`` command is used to dump (list) the contents of a full query log. Run the ``fqltool dump`` command after some CQL statements have been run.
+
+The full query logs get listed. Truncated output is as follows:
+
+::
+
+ [ec2-user@ip-10-0-2-238 cassandrafullquerylog]$ fqltool dump ./
+ WARN [main] 2019-08-02 03:07:53,635 Slf4jExceptionHandler.java:42 - Using Pauser.sleepy() as not enough processors, have 2, needs 8+
+ Type: single-query
+ Query start time: 1564708322030
+ Protocol version: 4
+ Generated timestamp:-9223372036854775808
+ Generated nowInSeconds:1564708322
+ Query: SELECT * FROM system.peers
+ Values:
+
+ Type: single-query
+ Query start time: 1564708322054
+ Protocol version: 4
+ Generated timestamp:-9223372036854775808
+ Generated nowInSeconds:1564708322
+ Query: SELECT * FROM system.local WHERE key='local'
+ Values:
+
+ Type: single-query
+ Query start time: 1564708322109
+ Protocol version: 4
+ Generated timestamp:-9223372036854775808
+ Generated nowInSeconds:1564708322
+ Query: SELECT * FROM system_schema.keyspaces
+ Values:
+
+ Type: single-query
+ Query start time: 1564708322116
+ Protocol version: 4
+ Generated timestamp:-9223372036854775808
+ Generated nowInSeconds:1564708322
+ Query: SELECT * FROM system_schema.tables
+ Values:
+
+ Type: single-query
+ Query start time: 1564708322139
+ Protocol version: 4
+ Generated timestamp:-9223372036854775808
+ Generated nowInSeconds:1564708322
+ Query: SELECT * FROM system_schema.columns
+ Values:
+
+ Type: single-query
+ Query start time: 1564708322142
+ Protocol version: 4
+ Generated timestamp:-9223372036854775808
+ Generated nowInSeconds:1564708322
+ Query: SELECT * FROM system_schema.functions
+ Values:
+
+ Type: single-query
+ Query start time: 1564708322141
+ Protocol version: 4
+ Generated timestamp:-9223372036854775808
+ Generated nowInSeconds:1564708322
+ Query: SELECT * FROM system_schema.aggregates
+ Values:
+
+ Type: single-query
+ Query start time: 1564708322143
+ Protocol version: 4
+ Generated timestamp:-9223372036854775808
+ Generated nowInSeconds:1564708322
+ Query: SELECT * FROM system_schema.types
+ Values:
+
+ Type: single-query
+ Query start time: 1564708322144
+ Protocol version: 4
+ Generated timestamp:-9223372036854775808
+ Generated nowInSeconds:1564708322
+ Query: SELECT * FROM system_schema.indexes
+ Values:
+
+ Type: single-query
+ Query start time: 1564708322142
+ Protocol version: 4
+ Generated timestamp:-9223372036854775808
+ Generated nowInSeconds:1564708322
+ Query: SELECT * FROM system_schema.triggers
+ Values:
+
+ Type: single-query
+ Query start time: 1564708322145
+ Protocol version: 4
+ Generated timestamp:-9223372036854775808
+ Generated nowInSeconds:1564708322
+ Query: SELECT * FROM system_schema.views
+ Values:
+
+ Type: single-query
+ Query start time: 1564708345408
+ Protocol version: 4
+ Generated timestamp:-9223372036854775808
+ Generated nowInSeconds:-2147483648
+ Query: CREATE KEYSPACE AuditLogKeyspace
+ WITH replication = {'class': 'SimpleStrategy', 'replication_factor' : 1};
+ Values:
+
+ Type: single-query
+ Query start time: 1564708345675
+ Protocol version: 4
+ Generated timestamp:-9223372036854775808
+ Generated nowInSeconds:1564708345
+ Query: SELECT peer, rpc_address, schema_version FROM system.peers
+ Values:
+
+ Type: single-query
+ Query start time: 1564708345676
+ Protocol version: 4
+ Generated timestamp:-9223372036854775808
+ Generated nowInSeconds:1564708345
+ Query: SELECT schema_version FROM system.local WHERE key='local'
+ Values:
+
+ Type: single-query
+ Query start time: 1564708346323
+ Protocol version: 4
+ Generated timestamp:-9223372036854775808
+ Generated nowInSeconds:1564708346
+ Query: SELECT * FROM system_schema.keyspaces WHERE keyspace_name = 'auditlogkeyspace'
+ Values:
+
+ Type: single-query
+ Query start time: 1564708360873
+ Protocol version: 4
+ Generated timestamp:-9223372036854775808
+ Generated nowInSeconds:-2147483648
+ Query: USE AuditLogKeyspace;
+ Values:
+
+ Type: single-query
+ Query start time: 1564708360874
+ Protocol version: 4
+ Generated timestamp:-9223372036854775808
+ Generated nowInSeconds:-2147483648
+ Query: USE "auditlogkeyspace"
+ Values:
+
+ Type: single-query
+ Query start time: 1564708378837
+ Protocol version: 4
+ Generated timestamp:-9223372036854775808
+ Generated nowInSeconds:-2147483648
+ Query: CREATE TABLE t (
+ id int,
+ k int,
+ v text,
+ PRIMARY KEY (id)
+ );
+ Values:
+
+ Type: single-query
+ Query start time: 1564708379247
+ Protocol version: 4
+ Generated timestamp:-9223372036854775808
+ Generated nowInSeconds:1564708379
+ Query: SELECT * FROM system_schema.tables WHERE keyspace_name = 'auditlogkeyspace' AND table_name = 't'
+ Values:
+
+ Type: single-query
+ Query start time: 1564708379255
+ Protocol version: 4
+ Generated timestamp:-9223372036854775808
+ Generated nowInSeconds:1564708379
+ Query: SELECT * FROM system_schema.views WHERE keyspace_name = 'auditlogkeyspace' AND view_name = 't'
+ Values:
+
+ Type: single-query
+ Query start time: 1564708397144
+ Protocol version: 4
+ Generated timestamp:-9223372036854775808
+ Generated nowInSeconds:1564708397
+ Query: INSERT INTO t (id, k, v) VALUES (0, 0, 'val0');
+ Values:
+
+ Type: single-query
+ Query start time: 1564708397167
+ Protocol version: 4
+ Generated timestamp:-9223372036854775808
+ Generated nowInSeconds:1564708397
+ Query: INSERT INTO t (id, k, v) VALUES (0, 1, 'val1');
+ Values:
+
+ Type: single-query
+ Query start time: 1564708434782
+ Protocol version: 4
+ Generated timestamp:-9223372036854775808
+ Generated nowInSeconds:1564708434
+ Query: SELECT * FROM t;
+ Values:
+
+ [ec2-user@ip-10-0-2-238 cassandrafullquerylog]$
+
+
+
+Full query logs are generated on each node. Enabling of full query logging on one node and the log files generated on the node are as follows:
+
+::
+
+ [root@localhost ~]# ssh -i cassandra.pem ec2-user@52.1.243.83
+ Last login: Fri Aug 2 00:14:53 2019 from 75.155.255.51
+ [ec2-user@ip-10-0-3-206 ~]$ sudo mkdir /tmp/cassandrafullquerylog
+ [ec2-user@ip-10-0-3-206 ~]$ sudo chmod -R 777 /tmp/cassandrafullquerylog
+ [ec2-user@ip-10-0-3-206 ~]$ nodetool enablefullquerylog --path /tmp/cassandrafullquerylog
+ [ec2-user@ip-10-0-3-206 ~]$ cd /tmp/cassandrafullquerylog
+ [ec2-user@ip-10-0-3-206 cassandrafullquerylog]$ ls -l
+ total 44
+ -rw-rw-r--. 1 ec2-user ec2-user 83886080 Aug 2 01:24 20190802-01.cq4
+ -rw-rw-r--. 1 ec2-user ec2-user 65536 Aug 2 01:23 directory-listing.cq4t
+ [ec2-user@ip-10-0-3-206 cassandrafullquerylog]$
+
+Enabling of full query logging on another node and the log files generated on the node are as follows:
+
+::
+
+ [root@localhost ~]# ssh -i cassandra.pem ec2-user@3.86.103.229
+ Last login: Fri Aug 2 00:13:04 2019 from 75.155.255.51
+ [ec2-user@ip-10-0-1-115 ~]$ sudo mkdir /tmp/cassandrafullquerylog
+ [ec2-user@ip-10-0-1-115 ~]$ sudo chmod -R 777 /tmp/cassandrafullquerylog
+ [ec2-user@ip-10-0-1-115 ~]$ nodetool enablefullquerylog --path /tmp/cassandrafullquerylog
+ [ec2-user@ip-10-0-1-115 ~]$ cd /tmp/cassandrafullquerylog
+ [ec2-user@ip-10-0-1-115 cassandrafullquerylog]$ ls -l
+ total 44
+ -rw-rw-r--. 1 ec2-user ec2-user 83886080 Aug 2 01:24 20190802-01.cq4
+ -rw-rw-r--. 1 ec2-user ec2-user 65536 Aug 2 01:23 directory-listing.cq4t
+ [ec2-user@ip-10-0-1-115 cassandrafullquerylog]$
+
+The ``nodetool resetfullquerylog`` resets the full query logger if it is enabled. Also deletes any generated files in the last used full query log path as well as the one configured in ``cassandra.yaml``. It stops the full query log and cleans files in the configured full query log directory from ``cassandra.yaml`` as well as JMX.
+
+Full Query Replay
+^^^^^^^^^^^^^^^^^
+The ``fqltool`` provides the ``replay`` command (`CASSANDRA-14618
+<https://issues.apache.org/jira/browse/CASSANDRA-14618>`_) to replay the full query logs. The FQL replay could be run on a different machine or even a different cluster for testing, debugging and performance benchmarking.
+
+The main objectives of ``fqltool replay`` are:
+
+- To be able to compare different runs of production traffic against different versions/configurations of Cassandra.
+- Take FQL logs from several machines and replay them in "order" by the timestamps recorded.
+- Record the results from each run to be able to compare different runs (against different clusters/versions/etc).
+- If fqltool replay is run against 2 or more clusters, the results could be compared.
+
+The FQL replay could also be used on the same node on which the full query log are generated to recreate a dropped database object.
+
+ The syntax of ``fqltool replay`` is as follows:
+
+::
+
+ fqltool replay [--keyspace <keyspace>] [--results <results>]
+ [--store-queries <store_queries>] --target <target>... [--] <path1>
+ [<path2>...<pathN>]
+
+ OPTIONS
+ --keyspace <keyspace>
+ Only replay queries against this keyspace and queries without
+ keyspace set.
+
+ --results <results>
+ Where to store the results of the queries, this should be a
+ directory. Leave this option out to avoid storing results.
+
+ --store-queries <store_queries>
+ Path to store the queries executed. Stores queries in the same order
+ as the result sets are in the result files. Requires --results
+
+ --target <target>
+ Hosts to replay the logs to, can be repeated to replay to more
+ hosts.
+
+ --
+ This option can be used to separate command-line options from the
+ list of argument, (useful when arguments might be mistaken for
+ command-line options
+
+ <path1> [<path2>...<pathN>]
+ Paths containing the full query logs to replay.
+
+As an example of using ``fqltool replay``, drop a keyspace.
+
+::
+
+ cqlsh:auditlogkeyspace> DROP KEYSPACE AuditLogKeyspace;
+
+Subsequently run ``fqltool replay``. The directory to store results of queries and the directory to store the queries run are specified and these directories must be created and permissions set before running ``fqltool replay``. The ``--results`` and ``--store-queries`` directories are optional but if ``--store-queries`` is to be set the ``--results`` must also be set.
+
+::
+
+ [ec2-user@ip-10-0-2-238 cassandra]$ fqltool replay --keyspace AuditLogKeyspace --results
+ /cassandra/fql/logs/results/replay --store-queries /cassandra/fql/logs/queries/replay --
+ target 3.91.56.164 -- /tmp/cassandrafullquerylog
+
+Describe the keyspaces after running ``fqltool replay`` and the keyspace that was dropped gets listed again.
+
+::
+
+ cqlsh:auditlogkeyspace> DESC KEYSPACES;
+
+ system_schema system system_distributed system_virtual_schema
+ system_auth auditlogkeyspace system_traces system_views
+
+ cqlsh:auditlogkeyspace>
+
+Full Query Compare
+^^^^^^^^^^^^^^^^^^
+The ``fqltool compare`` command (`CASSANDRA-14619
+<https://issues.apache.org/jira/browse/CASSANDRA-14619>`_) is used to compare result files generated by ``fqltool replay``. The ``fqltool compare`` command that can take the recorded runs from ``fqltool replay`` and compares them, it should output any differences and potentially all queries against the mismatching partition up until the mismatch.
+
+The ``fqltool compare`` could be used for comparing result files generated by different versions of Cassandra or different Cassandra configurations as an example. The command usage is as follows:
+
+::
+
+ [ec2-user@ip-10-0-2-238 ~]$ fqltool help compare
+ NAME
+ fqltool compare - Compare result files generated by fqltool replay
+
+ SYNOPSIS
+ fqltool compare --queries <queries> [--] <path1> [<path2>...<pathN>]
+
+ OPTIONS
+ --queries <queries>
+ Directory to read the queries from. It is produced by the fqltool
+ replay --store-queries option.
+
+ --
+ This option can be used to separate command-line options from the
+ list of argument, (useful when arguments might be mistaken for
+ command-line options
+
+ <path1> [<path2>...<pathN>]
+ Directories containing result files to compare.
+
+The ``fqltool compare`` stores each row as a separate chronicle document to be able to avoid reading up the entire result set in memory when comparing document formats:
+
+To mark the start of a new result set:
+
+::
+
+ -------------------
+ version: int16
+ type: column_definitions
+ column_count: int32;
+ column_definition: text, text
+ column_definition: text, text
+ ....
+ --------------------
+
+
+To mark a failed query set:
+
+::
+
+ ---------------------
+ version: int16
+ type: query_failed
+ message: text
+ ---------------------
+
+To mark a row set:
+
+::
+
+ --------------------
+ version: int16
+ type: row
+ row_column_count: int32
+ column: bytes
+ ---------------------
+
+To mark the end of a result set:
+
+::
+
+ -------------------
+ version: int16
+ type: end_resultset
+ -------------------
+
+
+Performance Overhead of FQL
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
+In performance testing FQL appears to have little or no overhead in ``WRITE`` only workloads, and a minor overhead in ``MIXED`` workload.
diff --git a/doc/source/new/index.rst b/doc/source/new/index.rst
new file mode 100644
index 0000000..5ef867b
--- /dev/null
+++ b/doc/source/new/index.rst
@@ -0,0 +1,32 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements. See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership. The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License. You may obtain a copy of the License at
+..
+.. http://www.apache.org/licenses/LICENSE-2.0
+..
+.. Unless required by applicable law or agreed to in writing, software
+.. distributed under the License is distributed on an "AS IS" BASIS,
+.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+.. See the License for the specific language governing permissions and
+.. limitations under the License.
+
+New Features in Apache Cassandra 4.0
+====================================
+
+This section covers the new features in Apache Cassandra 4.0.
+
+.. toctree::
+ :maxdepth: 2
+
+ java11
+ virtualtables
+ auditlogging
+ fqllogging
+ messaging
+ streaming
+ transientreplication
+
diff --git a/doc/source/new/java11.rst b/doc/source/new/java11.rst
new file mode 100644
index 0000000..df906d4
--- /dev/null
+++ b/doc/source/new/java11.rst
@@ -0,0 +1,274 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements. See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership. The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License. You may obtain a copy of the License at
+..
+.. http://www.apache.org/licenses/LICENSE-2.0
+..
+.. Unless required by applicable law or agreed to in writing, software
+.. distributed under the License is distributed on an "AS IS" BASIS,
+.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+.. See the License for the specific language governing permissions and
+.. limitations under the License.
+
+Support for Java 11
+-------------------
+
+In the new Java release cadence a new Java version is made available every six months. The more frequent release cycle
+is favored as it brings new Java features to the developers as and when they are developed without the wait that the
+earlier 3 year release model incurred. Not every Java version is a Long Term Support (LTS) version. After Java 8 the
+next LTS version is Java 11. Java 9, 10, 12 and 13 are all non-LTS versions.
+
+One of the objectives of the Apache Cassandra 4.0 version is to support the recent LTS Java versions 8 and 11 (`CASSANDRA-9608
+<https://issues.apache.org/jira/browse/CASSANDRA-9608>`_). Java 8 and
+Java 11 may be used to build and run Apache Cassandra 4.0.
+
+**Note**: Support for JDK 11 in Apache Cassandra 4.0 is an experimental feature, and not recommended for production use.
+
+Support Matrix
+^^^^^^^^^^^^^^
+
+The support matrix for the Java versions for compiling and running Apache Cassandra 4.0 is detailed in Table 1. The
+build version is along the vertical axis and the run version is along the horizontal axis.
+
+Table 1 : Support Matrix for Java
+
++---------------+--------------+-----------------+
+| | Java 8 (Run) | Java 11 (Run) |
++---------------+--------------+-----------------+
+| Java 8 (Build)|Supported |Supported |
++---------------+--------------+-----------------+
+| Java 11(Build)| Not Supported|Supported |
++---------------+--------------+-----------------+
+
+Essentially Apache 4.0 source code built with Java 11 cannot be run with Java 8. Next, we shall discuss using each of Java 8 and 11 to build and run Apache Cassandra 4.0.
+
+Using Java 8 to Build
+^^^^^^^^^^^^^^^^^^^^^
+
+To start with, install Java 8. As an example, for installing Java 8 on RedHat Linux the command is as follows:
+
+::
+
+$ sudo yum install java-1.8.0-openjdk-devel
+
+Set ``JAVA_HOME`` and ``JRE_HOME`` environment variables in the shell bash script. First, open the bash script:
+
+::
+
+$ sudo vi ~/.bashrc
+
+Set the environment variables including the ``PATH``.
+
+::
+
+ $ export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk
+ $ export JRE_HOME=/usr/lib/jvm/java-1.8.0-openjdk/jre
+ $ export PATH=$PATH:$JAVA_HOME/bin:$JRE_HOME/bin
+
+Download and install Apache Cassandra 4.0 source code from the Git along with the dependencies.
+
+::
+
+ $ git clone https://github.com/apache/cassandra.git
+
+If Cassandra is already running stop Cassandra with the following command.
+
+::
+
+ [ec2-user@ip-172-30-3-146 bin]$ ./nodetool stopdaemon
+
+Build the source code from the ``cassandra`` directory, which has the ``build.xml`` build script. The Apache Ant uses the Java version set in the ``JAVA_HOME`` environment variable.
+
+::
+
+ $ cd ~/cassandra
+ $ ant
+
+Apache Cassandra 4.0 gets built with Java 8. Set the environment variable for ``CASSANDRA_HOME`` in the bash script. Also add the ``CASSANDRA_HOME/bin`` to the ``PATH`` variable.
+
+::
+
+ $ export CASSANDRA_HOME=~/cassandra
+ $ export PATH=$PATH:$JAVA_HOME/bin:$JRE_HOME/bin:$CASSANDRA_HOME/bin
+
+To run Apache Cassandra 4.0 with either of Java 8 or Java 11 run the Cassandra application in the ``CASSANDRA_HOME/bin`` directory, which is in the ``PATH`` env variable.
+
+::
+
+ $ cassandra
+
+The Java version used to run Cassandra gets output as Cassandra is getting started. As an example if Java 11 is used, the run output should include similar to the following output snippet:
+
+::
+
+ INFO [main] 2019-07-31 21:18:16,862 CassandraDaemon.java:480 - Hostname: ip-172-30-3-
+ 146.ec2.internal:7000:7001
+ INFO [main] 2019-07-31 21:18:16,862 CassandraDaemon.java:487 - JVM vendor/version: OpenJDK
+ 64-Bit Server VM/11.0.3
+ INFO [main] 2019-07-31 21:18:16,863 CassandraDaemon.java:488 - Heap size:
+ 1004.000MiB/1004.000MiB
+
+The following output indicates a single node Cassandra 4.0 cluster has started.
+
+::
+
+ INFO [main] 2019-07-31 21:18:19,687 InboundConnectionInitiator.java:130 - Listening on
+ address: (127.0.0.1:7000), nic: lo, encryption: enabled (openssl)
+ ...
+ ...
+ INFO [main] 2019-07-31 21:18:19,850 StorageService.java:512 - Unable to gossip with any
+ peers but continuing anyway since node is in its own seed list
+ INFO [main] 2019-07-31 21:18:19,864 StorageService.java:695 - Loading persisted ring state
+ INFO [main] 2019-07-31 21:18:19,865 StorageService.java:814 - Starting up server gossip
+ INFO [main] 2019-07-31 21:18:20,088 BufferPool.java:216 - Global buffer pool is enabled,
+ when pool is exhausted (max is 251.000MiB) it will allocate on heap
+ INFO [main] 2019-07-31 21:18:20,110 StorageService.java:875 - This node will not auto
+ bootstrap because it is configured to be a seed node.
+ ...
+ ...
+ INFO [main] 2019-07-31 21:18:20,809 StorageService.java:1507 - JOINING: Finish joining ring
+ INFO [main] 2019-07-31 21:18:20,921 StorageService.java:2508 - Node 127.0.0.1:7000 state
+ jump to NORMAL
+
+Using Java 11 to Build
+^^^^^^^^^^^^^^^^^^^^^^
+If Java 11 is used to build Apache Cassandra 4.0, first Java 11 must be installed and the environment variables set. As an example, to download and install Java 11 on RedHat Linux run the following command.
+
+::
+
+ $ yum install java-11-openjdk-devel
+
+Set the environment variables in the bash script for Java 11. The first command is to open the bash script.
+
+::
+
+ $ sudo vi ~/.bashrc
+ $ export JAVA_HOME=/usr/lib/jvm/java-11-openjdk
+ $ export JRE_HOME=/usr/lib/jvm/java-11-openjdk/jre
+ $ export PATH=$PATH:$JAVA_HOME/bin:$JRE_HOME/bin
+
+To build source code with Java 11 one of the following two options must be used.
+
+ 1. Include Apache Ant command-line option ``-Duse.jdk=11`` as follows:
+ ::
+
+ $ ant -Duse.jdk=11
+
+ 2. Set environment variable ``CASSANDRA_USE_JDK11`` to ``true``:
+ ::
+
+ $ export CASSANDRA_USE_JDK11=true
+
+As an example, set the environment variable ``CASSANDRA_USE_JDK11`` to ``true``.
+
+::
+
+ [ec2-user@ip-172-30-3-146 cassandra]$ export CASSANDRA_USE_JDK11=true
+ [ec2-user@ip-172-30-3-146 cassandra]$ ant
+ Buildfile: /home/ec2-user/cassandra/build.xml
+
+Or, set the command-line option.
+
+::
+
+ [ec2-user@ip-172-30-3-146 cassandra]$ ant -Duse.jdk11=true
+
+The build output should include the following.
+
+::
+
+ _build_java:
+ [echo] Compiling for Java 11
+ ...
+ ...
+ build:
+
+ _main-jar:
+ [copy] Copying 1 file to /home/ec2-user/cassandra/build/classes/main/META-INF
+ [jar] Building jar: /home/ec2-user/cassandra/build/apache-cassandra-4.0-SNAPSHOT.jar
+ ...
+ ...
+ _build-test:
+ [javac] Compiling 739 source files to /home/ec2-user/cassandra/build/test/classes
+ [copy] Copying 25 files to /home/ec2-user/cassandra/build/test/classes
+ ...
+ ...
+ jar:
+ [mkdir] Created dir: /home/ec2-user/cassandra/build/classes/stress/META-INF
+ [mkdir] Created dir: /home/ec2-user/cassandra/build/tools/lib
+ [jar] Building jar: /home/ec2-user/cassandra/build/tools/lib/stress.jar
+ [mkdir] Created dir: /home/ec2-user/cassandra/build/classes/fqltool/META-INF
+ [jar] Building jar: /home/ec2-user/cassandra/build/tools/lib/fqltool.jar
+
+ BUILD SUCCESSFUL
+ Total time: 1 minute 3 seconds
+ [ec2-user@ip-172-30-3-146 cassandra]$
+
+Common Issues
+^^^^^^^^^^^^^^
+One of the two options mentioned must be used to compile with JDK 11 or the build fails and the following error message is output.
+
+::
+
+ [ec2-user@ip-172-30-3-146 cassandra]$ ant
+ Buildfile: /home/ec2-user/cassandra/build.xml
+ validate-build-conf:
+
+ BUILD FAILED
+ /home/ec2-user/cassandra/build.xml:293: -Duse.jdk11=true or $CASSANDRA_USE_JDK11=true must
+ be set when building from java 11
+ Total time: 1 second
+ [ec2-user@ip-172-30-3-146 cassandra]$
+
+The Java 11 built Apache Cassandra 4.0 source code may be run with Java 11 only. If a Java 11 built code is run with Java 8 the following error message gets output.
+
+::
+
+ [root@localhost ~]# ssh -i cassandra.pem ec2-user@ec2-3-85-85-75.compute-1.amazonaws.com
+ Last login: Wed Jul 31 20:47:26 2019 from 75.155.255.51
+ [ec2-user@ip-172-30-3-146 ~]$ echo $JAVA_HOME
+ /usr/lib/jvm/java-1.8.0-openjdk
+ [ec2-user@ip-172-30-3-146 ~]$ cassandra
+ ...
+ ...
+ Error: A JNI error has occurred, please check your installation and try again
+ Exception in thread "main" java.lang.UnsupportedClassVersionError:
+ org/apache/cassandra/service/CassandraDaemon has been compiled by a more recent version of
+ the Java Runtime (class file version 55.0), this version of the Java Runtime only recognizes
+ class file versions up to 52.0
+ at java.lang.ClassLoader.defineClass1(Native Method)
+ at java.lang.ClassLoader.defineClass(ClassLoader.java:763)
+ at ...
+ ...
+
+The ``CASSANDRA_USE_JDK11`` variable or the command-line option ``-Duse.jdk11`` cannot be used to build with Java 8. To demonstrate set ``JAVA_HOME`` to version 8.
+
+::
+
+ [root@localhost ~]# ssh -i cassandra.pem ec2-user@ec2-3-85-85-75.compute-1.amazonaws.com
+ Last login: Wed Jul 31 21:41:50 2019 from 75.155.255.51
+ [ec2-user@ip-172-30-3-146 ~]$ echo $JAVA_HOME
+ /usr/lib/jvm/java-1.8.0-openjdk
+
+Set the ``CASSANDRA_USE_JDK11=true`` or command-line option ``-Duse.jdk11=true``. Subsequently, run Apache Ant to start the build. The build fails with error message listed.
+
+::
+
+ [ec2-user@ip-172-30-3-146 ~]$ cd
+ cassandra
+ [ec2-user@ip-172-30-3-146 cassandra]$ export CASSANDRA_USE_JDK11=true
+ [ec2-user@ip-172-30-3-146 cassandra]$ ant
+ Buildfile: /home/ec2-user/cassandra/build.xml
+
+ validate-build-conf:
+
+ BUILD FAILED
+ /home/ec2-user/cassandra/build.xml:285: -Duse.jdk11=true or $CASSANDRA_USE_JDK11=true cannot
+ be set when building from java 8
+
+ Total time: 0 seconds
+
diff --git a/doc/source/new/messaging.rst b/doc/source/new/messaging.rst
new file mode 100644
index 0000000..755c9d1
--- /dev/null
+++ b/doc/source/new/messaging.rst
@@ -0,0 +1,257 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements. See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership. The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License. You may obtain a copy of the License at
+..
+.. http://www.apache.org/licenses/LICENSE-2.0
+..
+.. Unless required by applicable law or agreed to in writing, software
+.. distributed under the License is distributed on an "AS IS" BASIS,
+.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+.. See the License for the specific language governing permissions and
+.. limitations under the License.
+
+Improved Internode Messaging
+------------------------------
+
+
+Apache Cassandra 4.0 has added several new improvements to internode messaging.
+
+Optimized Internode Messaging Protocol
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+The internode messaging protocol has been optimized (`CASSANDRA-14485
+<https://issues.apache.org/jira/browse/CASSANDRA-14485>`_). Previously the ``IPAddressAndPort`` of the sender was included with each message that was sent even though the ``IPAddressAndPort`` had already been sent once when the initial connection/session was established. In Cassandra 4.0 ``IPAddressAndPort`` has been removed from every separate message sent and only sent when connection/session is initiated.
+
+Another improvement is that at several instances (listed) a fixed 4-byte integer value has been replaced with ``vint`` as a ``vint`` is almost always less than 1 byte:
+
+- The ``paramSize`` (the number of parameters in the header)
+- Each individual parameter value
+- The ``payloadSize``
+
+
+NIO Messaging
+^^^^^^^^^^^^^^^
+In Cassandra 4.0 peer-to-peer (internode) messaging has been switched to non-blocking I/O (NIO) via Netty (`CASSANDRA-8457
+<https://issues.apache.org/jira/browse/CASSANDRA-8457>`_).
+
+As serialization format, each message contains a header with several fixed fields, an optional key-value parameters section, and then the message payload itself. Note: the IP address in the header may be either IPv4 (4 bytes) or IPv6 (16 bytes).
+
+ The diagram below shows the IPv4 address for brevity.
+
+::
+
+ 1 1 1 1 1 2 2 2 2 2 3 3 3 3 3 4 4 4 4 4 5 5 5 5 5 6 6
+ 0 2 4 6 8 0 2 4 6 8 0 2 4 6 8 0 2 4 6 8 0 2 4 6 8 0 2 4 6 8 0 2
+ +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ | PROTOCOL MAGIC |
+ +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ | Message ID |
+ +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ | Timestamp |
+ +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ | Addr len | IP Address (IPv4) /
+ +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ / | Verb /
+ +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ / | Parameters size /
+ +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ / | Parameter data /
+ +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ / |
+ +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ | Payload size |
+ +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ | /
+ / Payload /
+ / |
+ +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+
+An individual parameter has a String key and a byte array value. The key is serialized with its length, encoded as two bytes, followed by the UTF-8 byte encoding of the string. The body is serialized with its length, encoded as four bytes, followed by the bytes of the value.
+
+Resource limits on Queued Messages
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+System stability is improved by enforcing strict resource limits (`CASSANDRA-15066
+<https://issues.apache.org/jira/browse/CASSANDRA-15066>`_) on the number of outbound messages that are queued, measured by the ``serializedSize`` of the message. There are three separate limits imposed simultaneously to ensure that progress is always made without any reasonable combination of failures impacting a node’s stability.
+
+1. Global, per-endpoint and per-connection limits are imposed on messages queued for delivery to other nodes and waiting to be processed on arrival from other nodes in the cluster. These limits are applied to the on-wire size of the message being sent or received.
+2. The basic per-link limit is consumed in isolation before any endpoint or global limit is imposed. Each node-pair has three links: urgent, small and large. So any given node may have a maximum of ``N*3 * (internode_application_send_queue_capacity_in_bytes + internode_application_receive_queue_capacity_in_bytes)`` messages queued without any coordination between them although in practice, with token-aware routing, only RF*tokens nodes should need to communicate with significant bandwidth.
+3. The per-endpoint limit is imposed on all messages exceeding the per-link limit, simultaneously with the global limit, on all links to or from a single node in the cluster. The global limit is imposed on all messages exceeding the per-link limit, simultaneously with the per-endpoint limit, on all links to or from any node in the cluster. The following configuration settings have been added to ``cassandra.yaml`` for resource limits on queued messages.
+
+::
+
+ internode_application_send_queue_capacity_in_bytes: 4194304 #4MiB
+ internode_application_send_queue_reserve_endpoint_capacity_in_bytes: 134217728 #128MiB
+ internode_application_send_queue_reserve_global_capacity_in_bytes: 536870912 #512MiB
+ internode_application_receive_queue_capacity_in_bytes: 4194304 #4MiB
+ internode_application_receive_queue_reserve_endpoint_capacity_in_bytes: 134217728 #128MiB
+ internode_application_receive_queue_reserve_global_capacity_in_bytes: 536870912 #512MiB
+
+Virtual Tables for Messaging Metrics
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Metrics is improved by keeping metrics using virtual tables for inter-node inbound and outbound messaging (`CASSANDRA-15066
+<https://issues.apache.org/jira/browse/CASSANDRA-15066>`_). For inbound messaging a virtual table (``internode_inbound``) has been added to keep metrics for:
+
+- Bytes and count of messages that could not be serialized or flushed due to an error
+- Bytes and count of messages scheduled
+- Bytes and count of messages successfully processed
+- Bytes and count of messages successfully received
+- Nanos and count of messages throttled
+- Bytes and count of messages expired
+- Corrupt frames recovered and unrecovered
+
+A separate virtual table (``internode_outbound``) has been added for outbound inter-node messaging. The outbound virtual table keeps metrics for:
+
+- Bytes and count of messages pending
+- Bytes and count of messages sent
+- Bytes and count of messages expired
+- Bytes and count of messages that could not be sent due to an error
+- Bytes and count of messages overloaded
+- Active Connection Count
+- Connection Attempts
+- Successful Connection Attempts
+
+Hint Messaging
+^^^^^^^^^^^^^^
+
+A specialized version of hint message that takes an already encoded in a ``ByteBuffer`` hint and sends it verbatim has been added. It is an optimization for when dispatching a hint file of the current messaging version to a node of the same messaging version, which is the most common case. It saves on extra ``ByteBuffer`` allocations one redundant hint deserialization-serialization cycle.
+
+Internode Application Timeout
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+A configuration setting has been added to ``cassandra.yaml`` for the maximum continuous period a connection may be unwritable in application space.
+
+::
+
+# internode_application_timeout_in_ms = 30000
+
+Some other new features include logging of message size to trace message for tracing a query.
+
+Paxos prepare and propose stage for local requests optimized
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+In pre-4.0 Paxos prepare and propose messages always go through entire ``MessagingService`` stack in Cassandra even if request is to be served locally, we can enhance and make local requests severed w/o involving ``MessagingService``. Similar things are done elsewhere in Cassandra which skips ``MessagingService`` stage for local requests.
+
+This is what it looks like in pre 4.0 if we have tracing on and run a light-weight transaction:
+
+::
+
+ Sending PAXOS_PREPARE message to /A.B.C.D [MessagingService-Outgoing-/A.B.C.D] | 2017-09-11
+ 21:55:18.971000 | A.B.C.D | 15045
+ … REQUEST_RESPONSE message received from /A.B.C.D [MessagingService-Incoming-/A.B.C.D] |
+ 2017-09-11 21:55:18.976000 | A.B.C.D | 20270
+ … Processing response from /A.B.C.D [SharedPool-Worker-4] | 2017-09-11 21:55:18.976000 |
+ A.B.C.D | 20372
+
+Same thing applies for Propose stage as well.
+
+In version 4.0 Paxos prepare and propose stage for local requests are optimized (`CASSANDRA-13862
+<https://issues.apache.org/jira/browse/CASSANDRA-13862>`_).
+
+Quality Assurance
+^^^^^^^^^^^^^^^^^
+
+Several other quality assurance improvements have been made in version 4.0 (`CASSANDRA-15066
+<https://issues.apache.org/jira/browse/CASSANDRA-15066>`_).
+
+Framing
+*******
+Version 4.0 introduces framing to all internode messages, i.e. the grouping of messages into a single logical payload with headers and trailers; these frames are guaranteed to either contain at most one message, that is split into its own unique sequence of frames (for large messages), or that a frame contains only complete messages.
+
+Corruption prevention
+*********************
+Previously, intra-datacenter internode messages would be unprotected from corruption by default, as only LZ4 provided any integrity checks. All messages to post 4.0 nodes are written to explicit frames, which may be:
+
+- LZ4 encoded
+- CRC protected
+
+The Unprotected option is still available.
+
+Resilience
+**********
+For resilience, all frames are written with a separate CRC protected header, of 8 and 6 bytes respectively. If corruption occurs in this header, the connection must be reset, as before. If corruption occurs anywhere outside of the header, the corrupt frame will be skipped, leaving the connection intact and avoiding the loss of any messages unnecessarily.
+
+Previously, any issue at any point in the stream would result in the connection being reset, with the loss of any in-flight messages.
+
+Efficiency
+**********
+The overall memory usage, and number of byte shuffles, on both inbound and outbound messages is reduced.
+
+Outbound the Netty LZ4 encoder maintains a chunk size buffer (64KiB), that is filled before any compressed frame can be produced. Our frame encoders avoid this redundant copy, as well as freeing 192KiB per endpoint.
+
+Inbound, frame decoders guarantee only to copy the number of bytes necessary to parse a frame, and to never store more bytes than necessary. This improvement applies twice to LZ4 connections, improving both the message decode and the LZ4 frame decode.
+
+Inbound Path
+************
+Version 4.0 introduces several improvements to the inbound path.
+
+An appropriate message handler is used based on whether large or small messages are expected on a particular connection as set in a flag. ``NonblockingBufferHandler``, running on event loop, is used for small messages, and ``BlockingBufferHandler``, running off event loop, for large messages. The single implementation of ``InboundMessageHandler`` handles messages of any size effectively by deriving size of the incoming message from the byte stream. In addition to deriving size of the message from the stream, incoming message expiration time is proactively read, before attempting to deserialize the entire message. If it’s expired at the time when a message is encountered the message is just skipped in the byte stream altogether.
+And if a message fails to be deserialized while still on the receiving side - say, because of table id or column being unknown - bytes are skipped, without dropping the entire connection and losing all the buffered messages. An immediately reply back is sent to the coordinator node with the failure reason, rather than waiting for the coordinator callback to expire. This logic is extended to a corrupted frame; a corrupted frame is safely skipped over without dropping the connection.
+
+Inbound path imposes strict limits on memory utilization. Specifically, the memory occupied by all parsed, but unprocessed messages is bound - on per-connection, per-endpoint, and global basis. Once a connection exceeds its local unprocessed capacity and cannot borrow any permits from per-endpoint and global reserve, it simply stops processing further messages, providing natural backpressure - until sufficient capacity is regained.
+
+Outbound Connections
+********************
+
+Opening a connection
+++++++++++++++++++++
+A consistent approach is adopted for all kinds of failure to connect, including: refused by endpoint, incompatible versions, or unexpected exceptions;
+
+- Retry forever, until either success or no messages waiting to deliver.
+- Wait incrementally longer periods before reconnecting, up to a maximum of 1s.
+- While failing to connect, no reserve queue limits are acquired.
+
+Closing a connection
+++++++++++++++++++++
+- Correctly drains outbound messages that are waiting to be delivered (unless disconnected and fail to reconnect).
+- Messages written to a closing connection are either delivered or rejected, with a new connection being opened if the old is irrevocably closed.
+- Unused connections are pruned eventually.
+
+Reconnecting
+++++++++++++
+
+We sometimes need to reconnect a perfectly valid connection, e.g. if the preferred IP address changes. We ensure that the underlying connection has no in-progress operations before closing it and reconnecting.
+
+Message Failure
+++++++++++++++++
+Propagates to callbacks instantly, better preventing overload by reclaiming committed memory.
+
+Expiry
+~~~~~~~~
+- No longer experiences head-of-line blocking (e.g. undroppable message preventing all droppable messages from being expired).
+- While overloaded, expiry is attempted eagerly on enqueuing threads.
+- While disconnected we schedule regular pruning, to handle the case where messages are no longer being sent, but we have a large backlog to expire.
+
+Overload
+~~~~~~~~~
+- Tracked by bytes queued, as opposed to number of messages.
+
+Serialization Errors
+~~~~~~~~~~~~~~~~~~~~~
+- Do not result in the connection being invalidated; the message is simply completed with failure, and then erased from the frame.
+- Includes detected mismatch between calculated serialization size to actual.
+
+Failures to flush to network, perhaps because the connection has been reset are not currently notified to callback handlers, as the necessary information has been discarded, though it would be possible to do so in future if we decide it is worth our while.
+
+QoS
++++++
+"Gossip" connection has been replaced with a general purpose "Urgent" connection, for any small messages impacting system stability.
+
+Metrics
++++++++
+We track, and expose via Virtual Table and JMX, the number of messages and bytes that: we could not serialize or flush due to an error, we dropped due to overload or timeout, are pending, and have successfully sent.
+
+Added a Message size limit
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Cassandra pre-4.0 doesn't protect the server from allocating huge buffers for the inter-node Message objects. Adding a message size limit would be good to deal with issues such as a malfunctioning cluster participant. Version 4.0 introduced max message size config param, akin to max mutation size - set to endpoint reserve capacity by default.
+
+Recover from unknown table when deserializing internode messages
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+As discussed in (`CASSANDRA-9289
+<https://issues.apache.org/jira/browse/CASSANDRA-9289>`_) it would be nice to gracefully recover from seeing an unknown table in a message from another node. Pre-4.0, we close the connection and reconnect, which can cause other concurrent queries to fail.
+Version 4.0 fixes the issue by wrapping message in-stream with
+``TrackedDataInputPlus``, catching
+``UnknownCFException``, and skipping the remaining bytes in this message. TCP won't be closed and it will remain connected for other messages.
diff --git a/doc/source/new/streaming.rst b/doc/source/new/streaming.rst
new file mode 100644
index 0000000..1807eb4
--- /dev/null
+++ b/doc/source/new/streaming.rst
@@ -0,0 +1,162 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements. See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership. The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License. You may obtain a copy of the License at
+..
+.. http://www.apache.org/licenses/LICENSE-2.0
+..
+.. Unless required by applicable law or agreed to in writing, software
+.. distributed under the License is distributed on an "AS IS" BASIS,
+.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+.. See the License for the specific language governing permissions and
+.. limitations under the License.
+
+Improved Streaming
+---------------------
+
+Apache Cassandra 4.0 has made several improvements to streaming. Streaming is the process used by nodes of a cluster to exchange data in the form of SSTables. Streaming of SSTables is performed for several operations, such as:
+
+- SSTable Repair
+- Host Replacement
+- Range movements
+- Bootstrapping
+- Rebuild
+- Cluster expansion
+
+Streaming based on Netty
+^^^^^^^^^^^^^^^^^^^^^^^^
+
+Streaming in Cassandra 4.0 is based on Non-blocking Input/Output (NIO) with Netty (`CASSANDRA-12229
+<https://issues.apache.org/jira/browse/CASSANDRA-12229>`_). It replaces the single-threaded (or sequential), synchronous, blocking model of streaming messages and transfer of files. Netty supports non-blocking, asynchronous, multi-threaded streaming with which multiple connections are opened simultaneously. Non-blocking implies that threads are not blocked as they don’t wait for a response for a sent request. A response could be returned in a different thread. With asynchronous, connections and threads are decoupled and do not have a 1:1 relation. Several more connections than threads may be opened.
+
+Zero Copy Streaming
+^^^^^^^^^^^^^^^^^^^^
+
+Pre-4.0, during streaming Cassandra reifies the SSTables into objects. This creates unnecessary garbage and slows down the whole streaming process as some SSTables can be transferred as a whole file rather than individual partitions. Cassandra 4.0 has added support for streaming entire SSTables when possible (`CASSANDRA-14556
+<https://issues.apache.org/jira/browse/CASSANDRA-14556>`_) for faster Streaming using ZeroCopy APIs. If enabled, Cassandra will use ZeroCopy for eligible SSTables significantly speeding up transfers and increasing throughput. A zero-copy path avoids bringing data into user-space on both sending and receiving side. Any streaming related operations will notice corresponding improvement. Zero copy streaming is hardware bound; only limited by the hardware limitations (Network and Disk IO ).
+
+High Availability
+*****************
+In benchmark tests Zero Copy Streaming is 5x faster than partitions based streaming. Faster streaming provides the benefit of improved availability. A cluster’s recovery mainly depends on the streaming speed, Cassandra clusters with failed nodes will be able to recover much more quickly (5x faster). If a node fails, SSTables need to be streamed to a replacement node. During the replacement operation, the new Cassandra node streams SSTables from the neighboring nodes that hold copies of the data belonging to this new node’s token range. Depending on the amount of data stored, this process can require substantial network bandwidth, taking some time to complete. The longer these range movement operations take, the more the cluster availability is lost. Failure of multiple nodes would reduce high availability greatly. The faster the new node completes streaming its data, the faster it can serve traffic, increasing the availability of the cluster.
+
+Enabling Zero Copy Streaming
+*****************************
+Zero copy streaming is enabled by setting the following setting in ``cassandra.yaml``.
+
+::
+
+ stream_entire_sstables: true
+
+By default zero copy streaming is enabled.
+
+SSTables Eligible for Zero Copy Streaming
+*****************************************
+Zero copy streaming is used if all partitions within the SSTable need to be transmitted. This is common when using ``LeveledCompactionStrategy`` or when partitioning SSTables by token range has been enabled. All partition keys in the SSTables are iterated over to determine the eligibility for Zero Copy streaming.
+
+Benefits of Zero Copy Streaming
+********************************
+When enabled, it permits Cassandra to zero-copy stream entire eligible SSTables between nodes, including every component. This speeds up the network transfer significantly subject to throttling specified by ``stream_throughput_outbound_megabits_per_sec``.
+
+Enabling this will reduce the GC pressure on sending and receiving node. While this feature tries to keep the disks balanced, it cannot guarantee it. This feature will be automatically disabled if internode encryption is enabled. Currently this can be used with Leveled Compaction.
+
+Configuring for Zero Copy Streaming
+************************************
+Throttling would reduce the streaming speed. The ``stream_throughput_outbound_megabits_per_sec`` throttles all outbound streaming file transfers on a node to the given total throughput in Mbps. When unset, the default is 200 Mbps or 25 MB/s.
+
+::
+
+ stream_throughput_outbound_megabits_per_sec: 200
+
+To run any Zero Copy streaming benchmark the ``stream_throughput_outbound_megabits_per_sec`` must be set to a really high value otherwise, throttling will be significant and the benchmark results will not be meaningful.
+
+The ``inter_dc_stream_throughput_outbound_megabits_per_sec`` throttles all streaming file transfer between the datacenters, this setting allows users to throttle inter dc stream throughput in addition to throttling all network stream traffic as configured with ``stream_throughput_outbound_megabits_per_sec``. When unset, the default is 200 Mbps or 25 MB/s.
+
+::
+
+ inter_dc_stream_throughput_outbound_megabits_per_sec: 200
+
+SSTable Components Streamed with Zero Copy Streaming
+*****************************************************
+Zero Copy Streaming streams entire SSTables. SSTables are made up of multiple components in separate files. SSTable components streamed are listed in Table 1.
+
+Table 1. SSTable Components
+
++------------------+---------------------------------------------------+
+|SSTable Component | Description |
++------------------+---------------------------------------------------+
+| Data.db |The base data for an SSTable: the remaining |
+| |components can be regenerated based on the data |
+| |component. |
++------------------+---------------------------------------------------+
+| Index.db |Index of the row keys with pointers to their |
+| |positions in the data file. |
++------------------+---------------------------------------------------+
+| Filter.db |Serialized bloom filter for the row keys in the |
+| |SSTable. |
++------------------+---------------------------------------------------+
+|CompressionInfo.db|File to hold information about uncompressed |
+| |data length, chunk offsets etc. |
++------------------+---------------------------------------------------+
+| Statistics.db |Statistical metadata about the content of the |
+| |SSTable. |
++------------------+---------------------------------------------------+
+| Digest.crc32 |Holds CRC32 checksum of the data file |
+| |size_bytes. |
++------------------+---------------------------------------------------+
+| CRC.db |Holds the CRC32 for chunks in an uncompressed file.|
++------------------+---------------------------------------------------+
+| Summary.db |Holds SSTable Index Summary |
+| |(sampling of Index component) |
++------------------+---------------------------------------------------+
+| TOC.txt |Table of contents, stores the list of all |
+| |components for the SSTable. |
++------------------+---------------------------------------------------+
+
+Custom component, used by e.g. custom compaction strategy may also be included.
+
+Repair Streaming Preview
+^^^^^^^^^^^^^^^^^^^^^^^^
+
+Repair with ``nodetool repair`` involves streaming of repaired SSTables and a repair preview has been added to provide an estimate of the amount of repair streaming that would need to be performed. Repair preview (`CASSANDRA-13257
+<https://issues.apache.org/jira/browse/CASSANDRA-13257>`_) is invoke with ``nodetool repair --preview`` using option:
+
+::
+
+-prv, --preview
+
+It determines ranges and amount of data to be streamed, but doesn't actually perform repair.
+
+Parallelizing of Streaming of Keyspaces
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+The streaming of the different keyspaces for bootstrap and rebuild has been parallelized in Cassandra 4.0 (`CASSANDRA-4663
+<https://issues.apache.org/jira/browse/CASSANDRA-4663>`_).
+
+Unique nodes for Streaming in Multi-DC deployment
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Range Streamer picks unique nodes to stream data from when number of replicas in each DC is three or more (`CASSANDRA-4650
+<https://issues.apache.org/jira/browse/CASSANDRA-4650>`_). What the optimization does is to even out the streaming load across the cluster. Without the optimization, some node can be picked up to stream more data than others. This patch allows to select dedicated node to stream only one range.
+
+This will increase the performance of bootstrapping a node and will also put less pressure on nodes serving the data. This does not affect if N < 3 in each DC as then it streams data from only 2 nodes.
+
+Stream Operation Types
+^^^^^^^^^^^^^
+
+It is important to know the type or purpose of a certain stream. Version 4.0 (`CASSANDRA-13064
+<https://issues.apache.org/jira/browse/CASSANDRA-13064>`_) adds an ``enum`` to distinguish between the different types of streams. Stream types are available both in a stream request and a stream task. The different stream types are:
+
+- Restore replica count
+- Unbootstrap
+- Relocation
+- Bootstrap
+- Rebuild
+- Bulk Load
+- Repair
+
+Disallow Decommission when number of Replicas will drop below configured RF
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+`CASSANDRA-12510
+<https://issues.apache.org/jira/browse/CASSANDRA-12510>`_ guards against decommission that will drop # of replicas below configured replication factor (RF), and adds the ``--force`` option that allows decommission to continue if intentional; force decommission of this node even when it reduces the number of replicas to below configured RF.
diff --git a/doc/source/new/transientreplication.rst b/doc/source/new/transientreplication.rst
new file mode 100644
index 0000000..aa39a11
--- /dev/null
+++ b/doc/source/new/transientreplication.rst
@@ -0,0 +1,155 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements. See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership. The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License. You may obtain a copy of the License at
+..
+.. http://www.apache.org/licenses/LICENSE-2.0
+..
+.. Unless required by applicable law or agreed to in writing, software
+.. distributed under the License is distributed on an "AS IS" BASIS,
+.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+.. See the License for the specific language governing permissions and
+.. limitations under the License.
+
+Transient Replication
+---------------------
+
+**Note**:
+
+Transient Replication (`CASSANDRA-14404
+<https://issues.apache.org/jira/browse/CASSANDRA-14404>`_) is an experimental feature designed for expert Apache Cassandra users who are able to validate every aspect of the database for their application and deployment.
+That means being able to check that operations like reads, writes, decommission, remove, rebuild, repair, and replace all work with your queries, data, configuration, operational practices, and availability requirements.
+Apache Cassandra 4.0 has the initial implementation of transient replication. Future releases of Cassandra will make this feature suitable for a wider audience.
+It is anticipated that a future version will support monotonic reads with transient replication as well as LWT, logged batches, and counters. Being experimental, Transient replication is **not** recommended for production use.
+
+Objective
+^^^^^^^^^
+
+The objective of transient replication is to decouple storage requirements from data redundancy (or consensus group size) using incremental repair, in order to reduce storage overhead.
+Certain nodes act as full replicas (storing all the data for a given token range), and some nodes act as transient replicas, storing only unrepaired data for the same token ranges.
+
+The optimization that is made possible with transient replication is called "Cheap quorums", which implies that data redundancy is increased without corresponding increase in storage usage.
+
+Transient replication is useful when sufficient full replicas are available to receive and store all the data.
+Transient replication allows you to configure a subset of replicas to only replicate data that hasn't been incrementally repaired.
+As an optimization, we can avoid writing data to a transient replica if we have successfully written data to the full replicas.
+
+After incremental repair, transient data stored on transient replicas can be discarded.
+
+Enabling Transient Replication
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Transient replication is not enabled by default. Transient replication must be enabled on each node in a cluster separately by setting the following configuration property in ``cassandra.yaml``.
+
+::
+
+ enable_transient_replication: true
+
+Transient replication may be configured with both ``SimpleStrategy`` and ``NetworkTopologyStrategy``. Transient replication is configured by setting replication factor as ``<total_replicas>/<transient_replicas>``.
+
+As an example, create a keyspace with replication factor (RF) 3.
+
+::
+
+ CREATE KEYSPACE CassandraKeyspaceSimple WITH replication = {'class': 'SimpleStrategy',
+ 'replication_factor' : 3/1};
+
+
+As another example, ``some_keysopace keyspace`` will have 3 replicas in DC1, 1 of which is transient, and 5 replicas in DC2, 2 of which are transient:
+
+::
+
+ CREATE KEYSPACE some_keysopace WITH replication = {'class': 'NetworkTopologyStrategy',
+ 'DC1' : '3/1'', 'DC2' : '5/2'};
+
+Transiently replicated keyspaces only support tables with ``read_repair`` set to ``NONE``.
+
+Important Restrictions:
+
+- RF cannot be altered while some endpoints are not in a normal state (no range movements).
+- You can't add full replicas if there are any transient replicas. You must first remove all transient replicas, then change the # of full replicas, then add back the transient replicas.
+- You can only safely increase number of transients one at a time with incremental repair run in between each time.
+
+
+Additionally, transient replication cannot be used for:
+
+- Monotonic Reads
+- Lightweight Transactions (LWTs)
+- Logged Batches
+- Counters
+- Keyspaces using materialized views
+- Secondary indexes (2i)
+
+Cheap Quorums
+^^^^^^^^^^^^^
+
+Cheap quorums are a set of optimizations on the write path to avoid writing to transient replicas unless sufficient full replicas are not available to satisfy the requested consistency level.
+Hints are never written for transient replicas. Optimizations on the read path prefer reading from transient replicas.
+When writing at quorum to a table configured to use transient replication the quorum will always prefer available full
+replicas over transient replicas so that transient replicas don't have to process writes. Tail latency is reduced by
+rapid write protection (similar to rapid read protection) when full replicas are slow or unavailable by sending writes
+to transient replicas. Transient replicas can serve reads faster as they don't have to do anything beyond bloom filter
+checks if they have no data. With vnodes and large cluster sizes they will not have a large quantity of data
+even for failure of one or more full replicas where transient replicas start to serve a steady amount of write traffic
+for some of their transiently replicated ranges.
+
+Speculative Write Option
+^^^^^^^^^^^^^^^^^^^^^^^^
+The ``CREATE TABLE`` adds an option ``speculative_write_threshold`` for use with transient replicas. The option is of type ``simple`` with default value as ``99PERCENTILE``. When replicas are slow or unresponsive ``speculative_write_threshold`` specifies the threshold at which a cheap quorum write will be upgraded to include transient replicas.
+
+
+Pending Ranges and Transient Replicas
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Pending ranges refers to the movement of token ranges between transient replicas. When a transient range is moved, there
+will be a period of time where both transient replicas would need to receive any write intended for the logical
+transient replica so that after the movement takes effect a read quorum is able to return a response. Nodes are *not*
+temporarily transient replicas during expansion. They stream data like a full replica for the transient range before they
+can serve reads. A pending state is incurred similar to how there is a pending state for full replicas. Transient replicas
+also always receive writes when they are pending. Pending transient ranges are sent a bit more data and reading from
+them is avoided.
+
+
+Read Repair and Transient Replicas
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Read repair never attempts to repair a transient replica. Reads will always include at least one full replica.
+They should also prefer transient replicas where possible. Range scans ensure the entire scanned range performs
+replica selection that satisfies the requirement that every range scanned includes one full replica. During incremental
+& validation repair handling, at transient replicas anti-compaction does not output any data for transient ranges as the
+data will be dropped after repair, and transient replicas never have data streamed to them.
+
+
+Transitioning between Full Replicas and Transient Replicas
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The additional state transitions that transient replication introduces requires streaming and ``nodetool cleanup`` to
+behave differently. When data is streamed it is ensured that it is streamed from a full replica and not a transient replica.
+
+Transitioning from not replicated to transiently replicated means that a node must stay pending until the next incremental
+repair completes at which point the data for that range is known to be available at full replicas.
+
+Transitioning from transiently replicated to fully replicated requires streaming from a full replica and is identical
+to how data is streamed when transitioning from not replicated to replicated. The transition is managed so the transient
+replica is not read from as a full replica until streaming completes. It can be used immediately for a write quorum.
+
+Transitioning from fully replicated to transiently replicated requires cleanup to remove repaired data from the transiently
+replicated range to reclaim space. It can be used immediately for a write quorum.
+
+Transitioning from transiently replicated to not replicated requires cleanup to be run to remove the formerly transiently replicated data.
+
+When transient replication is in use ring changes are supported including add/remove node, change RF, add/remove DC.
+
+
+Transient Replication supports EACH_QUORUM
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+(`CASSANDRA-14727
+<https://issues.apache.org/jira/browse/CASSANDRA-14727>`_) adds support for Transient Replication support for ``EACH_QUORUM``. Per (`CASSANDRA-14768
+<https://issues.apache.org/jira/browse/CASSANDRA-14768>`_), we ensure we write to at least a ``QUORUM`` of nodes in every DC,
+regardless of how many responses we need to wait for and our requested consistency level. This is to minimally surprise
+users with transient replication; with normal writes, we soft-ensure that we reach ``QUORUM`` in all DCs we are able to,
+by writing to every node; even if we don't wait for ACK, we have in both cases sent sufficient messages.
diff --git a/doc/source/new/virtualtables.rst b/doc/source/new/virtualtables.rst
new file mode 100644
index 0000000..1a39dc6
--- /dev/null
+++ b/doc/source/new/virtualtables.rst
@@ -0,0 +1,342 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements. See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership. The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License. You may obtain a copy of the License at
+..
+.. http://www.apache.org/licenses/LICENSE-2.0
+..
+.. Unless required by applicable law or agreed to in writing, software
+.. distributed under the License is distributed on an "AS IS" BASIS,
+.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+.. See the License for the specific language governing permissions and
+.. limitations under the License.
+
+Virtual Tables
+--------------
+
+Apache Cassandra 4.0 implements virtual tables (`CASSANDRA-7622
+<https://issues.apache.org/jira/browse/CASSANDRA-7622>`_).
+
+Definition
+^^^^^^^^^^
+
+A virtual table is a table that is backed by an API instead of data explicitly managed and stored as SSTables. Apache Cassandra 4.0 implements a virtual keyspace interface for virtual tables. Virtual tables are specific to each node.
+
+Objective
+^^^^^^^^^
+
+A virtual table could have several uses including:
+
+- Expose metrics through CQL
+- Expose YAML configuration information
+
+How are Virtual Tables different from regular tables?
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Virtual tables and virtual keyspaces are quite different from regular tables and keyspaces respectively such as:
+
+- Virtual tables are read-only, but it is likely to change
+- Virtual tables are not replicated
+- Virtual tables are local only and non distributed
+- Virtual tables have no associated SSTables
+- Consistency level of the queries sent virtual tables are ignored
+- Virtual tables are managed by Cassandra and a user cannot run DDL to create new virtual tables or DML to modify existing virtual tables
+- Virtual tables are created in special keyspaces and not just any keyspace
+- All existing virtual tables use ``LocalPartitioner``. Since a virtual table is not replicated the partitioner sorts in order of partition keys instead of by their hash.
+- Making advanced queries with ``ALLOW FILTERING`` and aggregation functions may be used with virtual tables even though in normal tables we dont recommend it
+
+Virtual Keyspaces
+^^^^^^^^^^^^^^^^^
+
+Apache Cassandra 4.0 has added two new keyspaces for virtual tables: ``system_virtual_schema`` and ``system_views``. Run the following command to list the keyspaces:
+
+::
+
+ cqlsh> DESC KEYSPACES;
+ system_schema system system_distributed system_virtual_schema
+ system_auth system_traces system_views
+
+The ``system_virtual_schema keyspace`` contains schema information on virtual tables. The ``system_views`` keyspace contains the actual virtual tables.
+
+Virtual Table Limitations
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Virtual tables and virtual keyspaces have some limitations initially though some of these could change such as:
+
+- Cannot alter or drop virtual keyspaces or tables
+- Cannot truncate virtual tables
+- Expiring columns are not supported by virtual tables
+- Conditional updates are not supported by virtual tables
+- Cannot create tables in virtual keyspaces
+- Cannot perform any operations against virtual keyspace
+- Secondary indexes are not supported on virtual tables
+- Cannot create functions in virtual keyspaces
+- Cannot create types in virtual keyspaces
+- Materialized views are not supported on virtual tables
+- Virtual tables don't support ``DELETE`` statements
+- Cannot ``CREATE TRIGGER`` against a virtual table
+- Conditional ``BATCH`` statements cannot include mutations for virtual tables
+- Cannot include a virtual table statement in a logged batch
+- Mutations for virtual and regular tables cannot exist in the same batch
+- Conditional ``BATCH`` statements cannot include mutations for virtual tables
+- Cannot create aggregates in virtual keyspaces; but may run aggregate functions on select
+
+Listing and Describing Virtual Tables
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Virtual tables in a virtual keyspace may be listed with ``DESC TABLES``. The ``system_views`` virtual keyspace tables include the following:
+
+::
+
+ cqlsh> USE system_views;
+ cqlsh:system_views> DESC TABLES;
+ coordinator_scans clients tombstones_scanned internode_inbound
+ disk_usage sstable_tasks live_scanned caches
+ local_writes max_partition_size local_reads
+ coordinator_writes internode_outbound thread_pools
+ local_scans coordinator_reads settings
+
+Some of the salient virtual tables in ``system_views`` virtual keyspace are described in Table 1.
+
+Table 1 : Virtual Tables in system_views
+
++------------------+---------------------------------------------------+
+|Virtual Table | Description |
++------------------+---------------------------------------------------+
+| clients |Lists information about all connected clients. |
++------------------+---------------------------------------------------+
+| disk_usage |Disk usage including disk_space, keyspace_name, |
+| |and table_name by system keyspaces. |
++------------------+---------------------------------------------------+
+| local_writes |A table metric for local writes |
+| |including count, keyspace_name, |
+| |max, median, per_second, and |
+| |table_name. |
++------------------+---------------------------------------------------+
+| caches |Displays the general cache information including |
+| |cache name, capacity_bytes, entry_count, hit_count,|
+| |hit_ratio double, recent_hit_rate_per_second, |
+| |recent_request_rate_per_second, request_count, and |
+| |size_bytes. |
++------------------+---------------------------------------------------+
+| local_reads |A table metric for local reads information. |
++------------------+---------------------------------------------------+
+| sstable_tasks |Lists currently running tasks such as compactions |
+| |and upgrades on SSTables. |
++------------------+---------------------------------------------------+
+|internode_inbound |Lists information about the inbound |
+| |internode messaging. |
++------------------+---------------------------------------------------+
+| thread_pools |Lists metrics for each thread pool. |
++------------------+---------------------------------------------------+
+| settings |Displays configuration settings in cassandra.yaml. |
++------------------+---------------------------------------------------+
+|max_partition_size|A table metric for maximum partition size. |
++------------------+---------------------------------------------------+
+|internode_outbound|Information about the outbound internode messaging.|
+| | |
++------------------+---------------------------------------------------+
+
+We shall discuss some of the virtual tables in more detail next.
+
+Clients Virtual Table
+*********************
+
+The ``clients`` virtual table lists all active connections (connected clients) including their ip address, port, connection stage, driver name, driver version, hostname, protocol version, request count, ssl enabled, ssl protocol and user name:
+
+::
+
+ cqlsh:system_views> select * from system_views.clients;
+ address | port | connection_stage | driver_name | driver_version | hostname | protocol_version | request_count | ssl_cipher_suite | ssl_enabled | ssl_protocol | username
+ -----------+-------+------------------+-------------+----------------+-----------+------------------+---------------+------------------+-------------+--------------+-----------
+ 127.0.0.1 | 50628 | ready | null | null | localhost | 4 | 55 | null | False | null | anonymous
+ 127.0.0.1 | 50630 | ready | null | null | localhost | 4 | 70 | null | False | null | anonymous
+
+ (2 rows)
+
+Some examples of how ``clients`` can be used are:
+
+- To find applications using old incompatible versions of drivers before upgrading and with ``nodetool enableoldprotocolversions`` and ``nodetool disableoldprotocolversions`` during upgrades.
+- To identify clients sending too many requests.
+- To find if SSL is enabled during the migration to and from ssl.
+
+
+The virtual tables may be described with ``DESCRIBE`` statement. The DDL listed however cannot be run to create a virtual table. As an example describe the ``system_views.clients`` virtual table:
+
+::
+
+ cqlsh:system_views> DESC TABLE system_views.clients;
+ CREATE TABLE system_views.clients (
+ address inet,
+ connection_stage text,
+ driver_name text,
+ driver_version text,
+ hostname text,
+ port int,
+ protocol_version int,
+ request_count bigint,
+ ssl_cipher_suite text,
+ ssl_enabled boolean,
+ ssl_protocol text,
+ username text,
+ PRIMARY KEY (address, port)) WITH CLUSTERING ORDER BY (port ASC)
+ AND compaction = {'class': 'None'}
+ AND compression = {};
+
+Caches Virtual Table
+********************
+The ``caches`` virtual table lists information about the caches. The four caches presently created are chunks, counters, keys and rows. A query on the ``caches`` virtual table returns the following details:
+
+::
+
+ cqlsh:system_views> SELECT * FROM system_views.caches;
+ name | capacity_bytes | entry_count | hit_count | hit_ratio | recent_hit_rate_per_second | recent_request_rate_per_second | request_count | size_bytes
+ ---------+----------------+-------------+-----------+-----------+----------------------------+--------------------------------+---------------+------------
+ chunks | 229638144 | 29 | 166 | 0.83 | 5 | 6 | 200 | 475136
+ counters | 26214400 | 0 | 0 | NaN | 0 | 0 | 0 | 0
+ keys | 52428800 | 14 | 124 | 0.873239 | 4 | 4 | 142 | 1248
+ rows | 0 | 0 | 0 | NaN | 0 | 0 | 0 | 0
+
+ (4 rows)
+
+Settings Virtual Table
+**********************
+The ``settings`` table is rather useful and lists all the current configuration settings from the ``cassandra.yaml``. The encryption options are overridden to hide the sensitive truststore information or passwords. The configuration settings however cannot be set using DML on the virtual table presently:
+::
+
+ cqlsh:system_views> SELECT * FROM system_views.settings;
+
+ name | value
+ -------------------------------------+--------------------
+ allocate_tokens_for_keyspace | null
+ audit_logging_options_enabled | false
+ auto_snapshot | true
+ automatic_sstable_upgrade | false
+ cluster_name | Test Cluster
+ enable_transient_replication | false
+ hinted_handoff_enabled | true
+ hints_directory | /home/ec2-user/cassandra/data/hints
+ incremental_backups | false
+ initial_token | null
+ ...
+ ...
+ ...
+ rpc_address | localhost
+ ssl_storage_port | 7001
+ start_native_transport | true
+ storage_port | 7000
+ stream_entire_sstables | true
+ (224 rows)
+
+
+The ``settings`` table can be really useful if yaml file has been changed since startup and dont know running configuration, or to find if they have been modified via jmx/nodetool or virtual tables.
+
+
+Thread Pools Virtual Table
+**************************
+
+The ``thread_pools`` table lists information about all thread pools. Thread pool information includes active tasks, active tasks limit, blocked tasks, blocked tasks all time, completed tasks, and pending tasks. A query on the ``thread_pools`` returns following details:
+
+::
+
+ cqlsh:system_views> select * from system_views.thread_pools;
+
+ name | active_tasks | active_tasks_limit | blocked_tasks | blocked_tasks_all_time | completed_tasks | pending_tasks
+ ------------------------------+--------------+--------------------+---------------+------------------------+-----------------+---------------
+ AntiEntropyStage | 0 | 1 | 0 | 0 | 0 | 0
+ CacheCleanupExecutor | 0 | 1 | 0 | 0 | 0 | 0
+ CompactionExecutor | 0 | 2 | 0 | 0 | 881 | 0
+ CounterMutationStage | 0 | 32 | 0 | 0 | 0 | 0
+ GossipStage | 0 | 1 | 0 | 0 | 0 | 0
+ HintsDispatcher | 0 | 2 | 0 | 0 | 0 | 0
+ InternalResponseStage | 0 | 2 | 0 | 0 | 0 | 0
+ MemtableFlushWriter | 0 | 2 | 0 | 0 | 1 | 0
+ MemtablePostFlush | 0 | 1 | 0 | 0 | 2 | 0
+ MemtableReclaimMemory | 0 | 1 | 0 | 0 | 1 | 0
+ MigrationStage | 0 | 1 | 0 | 0 | 0 | 0
+ MiscStage | 0 | 1 | 0 | 0 | 0 | 0
+ MutationStage | 0 | 32 | 0 | 0 | 0 | 0
+ Native-Transport-Requests | 1 | 128 | 0 | 0 | 130 | 0
+ PendingRangeCalculator | 0 | 1 | 0 | 0 | 1 | 0
+ PerDiskMemtableFlushWriter_0 | 0 | 2 | 0 | 0 | 1 | 0
+ ReadStage | 0 | 32 | 0 | 0 | 13 | 0
+ Repair-Task | 0 | 2147483647 | 0 | 0 | 0 | 0
+ RequestResponseStage | 0 | 2 | 0 | 0 | 0 | 0
+ Sampler | 0 | 1 | 0 | 0 | 0 | 0
+ SecondaryIndexManagement | 0 | 1 | 0 | 0 | 0 | 0
+ ValidationExecutor | 0 | 2147483647 | 0 | 0 | 0 | 0
+ ViewBuildExecutor | 0 | 1 | 0 | 0 | 0 | 0
+ ViewMutationStage | 0 | 32 | 0 | 0 | 0 | 0
+
+(24 rows)
+
+Internode Inbound Messaging Virtual Table
+*****************************************
+
+The ``internode_inbound`` virtual table is for the internode inbound messaging. Initially no internode inbound messaging may get listed. In addition to the address, port, datacenter and rack information includes corrupt frames recovered, corrupt frames unrecovered, error bytes, error count, expired bytes, expired count, processed bytes, processed count, received bytes, received count, scheduled bytes, scheduled count, throttled count, throttled nanos, using bytes, using reserve bytes. A query on the ``internode_inbound`` returns following details:
+
+::
+
+ cqlsh:system_views> SELECT * FROM system_views.internode_inbound;
+ address | port | dc | rack | corrupt_frames_recovered | corrupt_frames_unrecovered |
+ error_bytes | error_count | expired_bytes | expired_count | processed_bytes |
+ processed_count | received_bytes | received_count | scheduled_bytes | scheduled_count | throttled_count | throttled_nanos | using_bytes | using_reserve_bytes
+ ---------+------+----+------+--------------------------+----------------------------+-
+ ----------
+ (0 rows)
+
+SSTables Tasks Virtual Table
+****************************
+
+The ``sstable_tasks`` could be used to get information about running tasks. It lists following columns:
+
+::
+
+ cqlsh:system_views> SELECT * FROM sstable_tasks;
+ keyspace_name | table_name | task_id | kind | progress | total | unit
+ ---------------+------------+--------------------------------------+------------+----------+----------+-------
+ basic | wide2 | c3909740-cdf7-11e9-a8ed-0f03de2d9ae1 | compaction | 60418761 | 70882110 | bytes
+ basic | wide2 | c7556770-cdf7-11e9-a8ed-0f03de2d9ae1 | compaction | 2995623 | 40314679 | bytes
+
+
+As another example, to find how much time is remaining for SSTable tasks, use the following query:
+
+::
+
+ SELECT total - progress AS remaining
+ FROM system_views.sstable_tasks;
+
+Other Virtual Tables
+********************
+
+Some examples of using other virtual tables are as follows.
+
+Find tables with most disk usage:
+
+::
+
+ cqlsh> SELECT * FROM disk_usage WHERE mebibytes > 1 ALLOW FILTERING;
+
+ keyspace_name | table_name | mebibytes
+ ---------------+------------+-----------
+ keyspace1 | standard1 | 288
+ tlp_stress | keyvalue | 3211
+
+Find queries on table/s with greatest read latency:
+
+::
+
+ cqlsh> SELECT * FROM local_read_latency WHERE per_second > 1 ALLOW FILTERING;
+
+ keyspace_name | table_name | p50th_ms | p99th_ms | count | max_ms | per_second
+ ---------------+------------+----------+----------+----------+---------+------------
+ tlp_stress | keyvalue | 0.043 | 0.152 | 49785158 | 186.563 | 11418.356
+
+
+The system_virtual_schema keyspace
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The ``system_virtual_schema`` keyspace has three tables: ``keyspaces``, ``columns`` and ``tables`` for the virtual keyspace definitions, virtual table definitions, and virtual column definitions respectively. It is used by Cassandra internally and a user would not need to access it directly.
diff --git a/doc/source/operating/Figure_1_backups.jpg b/doc/source/operating/Figure_1_backups.jpg
new file mode 100644
index 0000000..160013d
--- /dev/null
+++ b/doc/source/operating/Figure_1_backups.jpg
Binary files differ
diff --git a/doc/source/operating/Figure_1_read_repair.jpg b/doc/source/operating/Figure_1_read_repair.jpg
new file mode 100644
index 0000000..d771550
--- /dev/null
+++ b/doc/source/operating/Figure_1_read_repair.jpg
Binary files differ
diff --git a/doc/source/operating/Figure_2_read_repair.jpg b/doc/source/operating/Figure_2_read_repair.jpg
new file mode 100644
index 0000000..29a912b
--- /dev/null
+++ b/doc/source/operating/Figure_2_read_repair.jpg
Binary files differ
diff --git a/doc/source/operating/Figure_3_read_repair.jpg b/doc/source/operating/Figure_3_read_repair.jpg
new file mode 100644
index 0000000..f5cc189
--- /dev/null
+++ b/doc/source/operating/Figure_3_read_repair.jpg
Binary files differ
diff --git a/doc/source/operating/Figure_4_read_repair.jpg b/doc/source/operating/Figure_4_read_repair.jpg
new file mode 100644
index 0000000..25bdb34
--- /dev/null
+++ b/doc/source/operating/Figure_4_read_repair.jpg
Binary files differ
diff --git a/doc/source/operating/Figure_5_read_repair.jpg b/doc/source/operating/Figure_5_read_repair.jpg
new file mode 100644
index 0000000..d9c0485
--- /dev/null
+++ b/doc/source/operating/Figure_5_read_repair.jpg
Binary files differ
diff --git a/doc/source/operating/Figure_6_read_repair.jpg b/doc/source/operating/Figure_6_read_repair.jpg
new file mode 100644
index 0000000..6bb4d1e
--- /dev/null
+++ b/doc/source/operating/Figure_6_read_repair.jpg
Binary files differ
diff --git a/doc/source/operating/audit_logging.rst b/doc/source/operating/audit_logging.rst
new file mode 100644
index 0000000..068209e
--- /dev/null
+++ b/doc/source/operating/audit_logging.rst
@@ -0,0 +1,236 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements. See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership. The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License. You may obtain a copy of the License at
+..
+.. http://www.apache.org/licenses/LICENSE-2.0
+..
+.. Unless required by applicable law or agreed to in writing, software
+.. distributed under the License is distributed on an "AS IS" BASIS,
+.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+.. See the License for the specific language governing permissions and
+.. limitations under the License.
+
+.. highlight:: none
+
+
+
+Audit Logging
+------------------
+
+Audit logging in Cassandra logs every incoming CQL command request, Authentication (successful as well as unsuccessful login)
+to C* node. Currently, there are two implementations provided, the custom logger can be implemented and injected with the
+class name as a parameter in cassandra.yaml.
+
+- ``BinAuditLogger`` An efficient way to log events to file in a binary format.
+- ``FileAuditLogger`` Logs events to ``audit/audit.log`` file using slf4j logger.
+
+*Recommendation* ``BinAuditLogger`` is a community recommended logger considering the performance
+
+What does it capture
+^^^^^^^^^^^^^^^^^^^^^^^
+
+Audit logging captures following events
+
+- Successful as well as unsuccessful login attempts.
+
+- All database commands executed via Native protocol (CQL) attempted or successfully executed.
+
+Limitations
+^^^^^^^^^^^
+
+Executing prepared statements will log the query as provided by the client in the prepare call, along with the execution time stamp and all other attributes (see below). Actual values bound for prepared statement execution will not show up in the audit log.
+
+What does it log
+^^^^^^^^^^^^^^^^^^^
+Each audit log implementation has access to the following attributes, and for the default text based logger these fields are concatenated with `|` s to yield the final message.
+
+ - ``user``: User name(if available)
+ - ``host``: Host IP, where the command is being executed
+ - ``source ip address``: Source IP address from where the request initiated
+ - ``source port``: Source port number from where the request initiated
+ - ``timestamp``: unix time stamp
+ - ``type``: Type of the request (SELECT, INSERT, etc.,)
+ - ``category`` - Category of the request (DDL, DML, etc.,)
+ - ``keyspace`` - Keyspace(If applicable) on which request is targeted to be executed
+ - ``scope`` - Table/Aggregate name/ function name/ trigger name etc., as applicable
+ - ``operation`` - CQL command being executed
+
+How to configure
+^^^^^^^^^^^^^^^^^^
+Auditlog can be configured using cassandra.yaml. If you want to try Auditlog on one node, it can also be enabled and configured using ``nodetool``.
+
+cassandra.yaml configurations for AuditLog
+"""""""""""""""""""""""""""""""""""""""""""""
+ - ``enabled``: This option enables/ disables audit log
+ - ``logger``: Class name of the logger/ custom logger.
+ - ``audit_logs_dir``: Auditlogs directory location, if not set, default to `cassandra.logdir.audit` or `cassandra.logdir` + /audit/
+ - ``included_keyspaces``: Comma separated list of keyspaces to be included in audit log, default - includes all keyspaces
+ - ``excluded_keyspaces``: Comma separated list of keyspaces to be excluded from audit log, default - excludes no keyspace except `system`, `system_schema` and `system_virtual_schema`
+ - ``included_categories``: Comma separated list of Audit Log Categories to be included in audit log, default - includes all categories
+ - ``excluded_categories``: Comma separated list of Audit Log Categories to be excluded from audit log, default - excludes no category
+ - ``included_users``: Comma separated list of users to be included in audit log, default - includes all users
+ - ``excluded_users``: Comma separated list of users to be excluded from audit log, default - excludes no user
+
+
+List of available categories are: QUERY, DML, DDL, DCL, OTHER, AUTH, ERROR, PREPARE
+
+NodeTool command to enable AuditLog
+"""""""""""""""""""""""""""""""""""""
+``enableauditlog``: Enables AuditLog with yaml defaults. yaml configurations can be overridden using options via nodetool command.
+
+::
+
+ nodetool enableauditlog
+
+Options
+**********
+
+
+``--excluded-categories``
+ Comma separated list of Audit Log Categories to be excluded for
+ audit log. If not set the value from cassandra.yaml will be used
+
+``--excluded-keyspaces``
+ Comma separated list of keyspaces to be excluded for audit log. If
+ not set the value from cassandra.yaml will be used.
+ Please remeber that `system`, `system_schema` and `system_virtual_schema` are excluded by default,
+ if you are overwriting this option via nodetool,
+ remember to add these keyspaces back if you dont want them in audit logs
+
+``--excluded-users``
+ Comma separated list of users to be excluded for audit log. If not
+ set the value from cassandra.yaml will be used
+
+``--included-categories``
+ Comma separated list of Audit Log Categories to be included for
+ audit log. If not set the value from cassandra.yaml will be used
+
+``--included-keyspaces``
+ Comma separated list of keyspaces to be included for audit log. If
+ not set the value from cassandra.yaml will be used
+
+``--included-users``
+ Comma separated list of users to be included for audit log. If not
+ set the value from cassandra.yaml will be used
+
+``--logger``
+ Logger name to be used for AuditLogging. Default BinAuditLogger. If
+ not set the value from cassandra.yaml will be used
+
+
+NodeTool command to disable AuditLog
+"""""""""""""""""""""""""""""""""""""""
+
+``disableauditlog``: Disables AuditLog.
+
+::
+
+ nodetool disableuditlog
+
+
+
+
+
+
+
+NodeTool command to reload AuditLog filters
+"""""""""""""""""""""""""""""""""""""""""""""
+
+``enableauditlog``: NodeTool enableauditlog command can be used to reload auditlog filters when called with default or previous ``loggername`` and updated filters
+
+E.g.,
+
+::
+
+ nodetool enableauditlog --loggername <Default/ existing loggerName> --included-keyspaces <New Filter values>
+
+
+
+View the contents of AuditLog Files
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+``auditlogviewer`` is the new tool introduced to help view the contents of binlog file in human readable text format.
+
+::
+
+ auditlogviewer <path1> [<path2>...<pathN>] [options]
+
+Options
+""""""""
+
+``-f,--follow``
+ Upon reacahing the end of the log continue indefinitely
+ waiting for more records
+``-r,--roll_cycle``
+ How often to roll the log file was rolled. May be
+ necessary for Chronicle to correctly parse file names. (MINUTELY, HOURLY,
+ DAILY). Default HOURLY.
+
+``-h,--help``
+ display this help message
+
+For example, to dump the contents of audit log files on the console
+
+::
+
+ auditlogviewer /logs/cassandra/audit
+
+Sample output
+"""""""""""""
+
+::
+
+ LogMessage: user:anonymous|host:localhost/X.X.X.X|source:/X.X.X.X|port:60878|timestamp:1521158923615|type:USE_KS|category:DDL|ks:dev1|operation:USE "dev1"
+
+
+
+Configuring BinAuditLogger
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+To use ``BinAuditLogger`` as a logger in AuditLogging, set the logger to ``BinAuditLogger`` in cassandra.yaml under ``audit_logging_options`` section. ``BinAuditLogger`` can be futher configued using its advanced options in cassandra.yaml.
+
+
+Adcanced Options for BinAuditLogger
+""""""""""""""""""""""""""""""""""""""
+
+``block``
+ Indicates if the AuditLog should block if the it falls behind or should drop audit log records. Default is set to ``true`` so that AuditLog records wont be lost
+
+``max_queue_weight``
+ Maximum weight of in memory queue for records waiting to be written to the audit log file before blocking or dropping the log records. Default is set to ``256 * 1024 * 1024``
+
+``max_log_size``
+ Maximum size of the rolled files to retain on disk before deleting the oldest file. Default is set to ``16L * 1024L * 1024L * 1024L``
+
+``roll_cycle``
+ How often to roll Audit log segments so they can potentially be reclaimed. Available options are: MINUTELY, HOURLY, DAILY, LARGE_DAILY, XLARGE_DAILY, HUGE_DAILY.For more options, refer: net.openhft.chronicle.queue.RollCycles. Default is set to ``"HOURLY"``
+
+Configuring FileAuditLogger
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+To use ``FileAuditLogger`` as a logger in AuditLogging, apart from setting the class name in cassandra.yaml, following configuration is needed to have the audit log events to flow through separate log file instead of system.log
+
+
+.. code-block:: xml
+
+ <!-- Audit Logging (FileAuditLogger) rolling file appender to audit.log -->
+ <appender name="AUDIT" class="ch.qos.logback.core.rolling.RollingFileAppender">
+ <file>${cassandra.logdir}/audit/audit.log</file>
+ <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
+ <!-- rollover daily -->
+ <fileNamePattern>${cassandra.logdir}/audit/audit.log.%d{yyyy-MM-dd}.%i.zip</fileNamePattern>
+ <!-- each file should be at most 50MB, keep 30 days worth of history, but at most 5GB -->
+ <maxFileSize>50MB</maxFileSize>
+ <maxHistory>30</maxHistory>
+ <totalSizeCap>5GB</totalSizeCap>
+ </rollingPolicy>
+ <encoder>
+ <pattern>%-5level [%thread] %date{ISO8601} %F:%L - %msg%n</pattern>
+ </encoder>
+ </appender>
+
+ <!-- Audit Logging additivity to redirect audt logging events to audit/audit.log -->
+ <logger name="org.apache.cassandra.audit" additivity="false" level="INFO">
+ <appender-ref ref="AUDIT"/>
+ </logger>
diff --git a/doc/source/operating/backups.rst b/doc/source/operating/backups.rst
index c071e83..01cb6c5 100644
--- a/doc/source/operating/backups.rst
+++ b/doc/source/operating/backups.rst
@@ -13,10 +13,648 @@
.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
.. See the License for the specific language governing permissions and
.. limitations under the License.
-
.. highlight:: none
-Backups
-=======
+Backups
+-------
-.. todo:: TODO
+Apache Cassandra stores data in immutable SSTable files. Backups in Apache Cassandra database are backup copies of the database data that is stored as SSTable files. Backups are used for several purposes including the following:
+
+- To store a data copy for durability
+- To be able to restore a table if table data is lost due to node/partition/network failure
+- To be able to transfer the SSTable files to a different machine; for portability
+
+Types of Backups
+^^^^^^^^^^^^^^^^
+Apache Cassandra supports two kinds of backup strategies.
+
+- Snapshots
+- Incremental Backups
+
+A *snapshot* is a copy of a table’s SSTable files at a given time, created via hard links. The DDL to create the table is stored as well. Snapshots may be created by a user or created automatically.
+The setting (``snapshot_before_compaction``) in ``cassandra.yaml`` determines if snapshots are created before each compaction.
+By default ``snapshot_before_compaction`` is set to false.
+Snapshots may be created automatically before keyspace truncation or dropping of a table by setting ``auto_snapshot`` to true (default) in ``cassandra.yaml``.
+Truncates could be delayed due to the auto snapshots and another setting in ``cassandra.yaml`` determines how long the coordinator should wait for truncates to complete.
+By default Cassandra waits 60 seconds for auto snapshots to complete.
+
+An *incremental backup* is a copy of a table’s SSTable files created by a hard link when memtables are flushed to disk as SSTables.
+Typically incremental backups are paired with snapshots to reduce the backup time as well as reduce disk space.
+Incremental backups are not enabled by default and must be enabled explicitly in ``cassandra.yaml`` (with ``incremental_backups`` setting) or with the Nodetool.
+Once enabled, Cassandra creates a hard link to each SSTable flushed or streamed locally in a ``backups/`` subdirectory of the keyspace data. Incremental backups of system tables are also created.
+
+Data Directory Structure
+^^^^^^^^^^^^^^^^^^^^^^^^
+The directory structure of Cassandra data consists of different directories for keyspaces, and tables with the data files within the table directories. Directories backups and snapshots to store backups and snapshots respectively for a particular table are also stored within the table directory. The directory structure for Cassandra is illustrated in Figure 1.
+
+.. figure:: Figure_1_backups.jpg
+
+Figure 1. Directory Structure for Cassandra Data
+
+
+Setting Up Example Tables for Backups and Snapshots
+****************************************************
+In this section we shall create some example data that could be used to demonstrate incremental backups and snapshots. We have used a three node Cassandra cluster.
+First, the keyspaces are created. Subsequently tables are created within a keyspace and table data is added. We have used two keyspaces ``CQLKeyspace`` and ``CatalogKeyspace`` with two tables within each.
+Create ``CQLKeyspace``:
+
+::
+
+ cqlsh> CREATE KEYSPACE CQLKeyspace
+ ... WITH replication = {'class': 'SimpleStrategy', 'replication_factor' : 3};
+
+Create table ``t`` in the ``CQLKeyspace`` keyspace.
+
+::
+
+ cqlsh> USE CQLKeyspace;
+ cqlsh:cqlkeyspace> CREATE TABLE t (
+ ... id int,
+ ... k int,
+ ... v text,
+ ... PRIMARY KEY (id)
+ ... );
+
+
+Add data to table ``t``:
+
+::
+
+ cqlsh:cqlkeyspace>
+ cqlsh:cqlkeyspace> INSERT INTO t (id, k, v) VALUES (0, 0, 'val0');
+ cqlsh:cqlkeyspace> INSERT INTO t (id, k, v) VALUES (1, 1, 'val1');
+
+
+A table query lists the data:
+
+::
+
+ cqlsh:cqlkeyspace> SELECT * FROM t;
+
+ id | k | v
+ ----+---+------
+ 1 | 1 | val1
+ 0 | 0 | val0
+
+ (2 rows)
+
+Create another table ``t2``:
+
+::
+
+ cqlsh:cqlkeyspace> CREATE TABLE t2 (
+ ... id int,
+ ... k int,
+ ... v text,
+ ... PRIMARY KEY (id)
+ ... );
+
+Add data to table ``t2``:
+
+::
+
+ cqlsh:cqlkeyspace> INSERT INTO t2 (id, k, v) VALUES (0, 0, 'val0');
+ cqlsh:cqlkeyspace> INSERT INTO t2 (id, k, v) VALUES (1, 1, 'val1');
+ cqlsh:cqlkeyspace> INSERT INTO t2 (id, k, v) VALUES (2, 2, 'val2');
+
+
+A table query lists table data:
+
+::
+
+ cqlsh:cqlkeyspace> SELECT * FROM t2;
+
+ id | k | v
+ ----+---+------
+ 1 | 1 | val1
+ 0 | 0 | val0
+ 2 | 2 | val2
+
+ (3 rows)
+
+Create a second keyspace ``CatalogKeyspace``:
+
+::
+
+ cqlsh:cqlkeyspace> CREATE KEYSPACE CatalogKeyspace
+ ... WITH replication = {'class': 'SimpleStrategy', 'replication_factor' : 3};
+
+Create a table called ``journal`` in ``CatalogKeyspace``:
+
+::
+
+ cqlsh:cqlkeyspace> USE CatalogKeyspace;
+ cqlsh:catalogkeyspace> CREATE TABLE journal (
+ ... id int,
+ ... name text,
+ ... publisher text,
+ ... PRIMARY KEY (id)
+ ... );
+
+
+Add data to table ``journal``:
+
+::
+
+ cqlsh:catalogkeyspace> INSERT INTO journal (id, name, publisher) VALUES (0, 'Apache
+ Cassandra Magazine', 'Apache Cassandra');
+ cqlsh:catalogkeyspace> INSERT INTO journal (id, name, publisher) VALUES (1, 'Couchbase
+ Magazine', 'Couchbase');
+
+Query table ``journal`` to list its data:
+
+::
+
+ cqlsh:catalogkeyspace> SELECT * FROM journal;
+
+ id | name | publisher
+ ----+---------------------------+------------------
+ 1 | Couchbase Magazine | Couchbase
+ 0 | Apache Cassandra Magazine | Apache Cassandra
+
+ (2 rows)
+
+Add another table called ``magazine``:
+
+::
+
+ cqlsh:catalogkeyspace> CREATE TABLE magazine (
+ ... id int,
+ ... name text,
+ ... publisher text,
+ ... PRIMARY KEY (id)
+ ... );
+
+Add table data to ``magazine``:
+
+::
+
+ cqlsh:catalogkeyspace> INSERT INTO magazine (id, name, publisher) VALUES (0, 'Apache
+ Cassandra Magazine', 'Apache Cassandra');
+ cqlsh:catalogkeyspace> INSERT INTO magazine (id, name, publisher) VALUES (1, 'Couchbase
+ Magazine', 'Couchbase');
+
+List table ``magazine``’s data:
+
+::
+
+ cqlsh:catalogkeyspace> SELECT * from magazine;
+
+ id | name | publisher
+ ----+---------------------------+------------------
+ 1 | Couchbase Magazine | Couchbase
+ 0 | Apache Cassandra Magazine | Apache Cassandra
+
+ (2 rows)
+
+Snapshots
+^^^^^^^^^
+In this section including sub-sections we shall demonstrate creating snapshots. The command used to create a snapshot is ``nodetool snapshot`` and its usage is as follows:
+
+::
+
+ [ec2-user@ip-10-0-2-238 ~]$ nodetool help snapshot
+ NAME
+ nodetool snapshot - Take a snapshot of specified keyspaces or a snapshot
+ of the specified table
+
+ SYNOPSIS
+ nodetool [(-h <host> | --host <host>)] [(-p <port> | --port <port>)]
+ [(-pp | --print-port)] [(-pw <password> | --password <password>)]
+ [(-pwf <passwordFilePath> | --password-file <passwordFilePath>)]
+ [(-u <username> | --username <username>)] snapshot
+ [(-cf <table> | --column-family <table> | --table <table>)]
+ [(-kt <ktlist> | --kt-list <ktlist> | -kc <ktlist> | --kc.list <ktlist>)]
+ [(-sf | --skip-flush)] [(-t <tag> | --tag <tag>)] [--] [<keyspaces...>]
+
+ OPTIONS
+ -cf <table>, --column-family <table>, --table <table>
+ The table name (you must specify one and only one keyspace for using
+ this option)
+
+ -h <host>, --host <host>
+ Node hostname or ip address
+
+ -kt <ktlist>, --kt-list <ktlist>, -kc <ktlist>, --kc.list <ktlist>
+ The list of Keyspace.table to take snapshot.(you must not specify
+ only keyspace)
+
+ -p <port>, --port <port>
+ Remote jmx agent port number
+
+ -pp, --print-port
+ Operate in 4.0 mode with hosts disambiguated by port number
+
+ -pw <password>, --password <password>
+ Remote jmx agent password
+
+ -pwf <passwordFilePath>, --password-file <passwordFilePath>
+ Path to the JMX password file
+
+ -sf, --skip-flush
+ Do not flush memtables before snapshotting (snapshot will not
+ contain unflushed data)
+
+ -t <tag>, --tag <tag>
+ The name of the snapshot
+
+ -u <username>, --username <username>
+ Remote jmx agent username
+
+ --
+ This option can be used to separate command-line options from the
+ list of argument, (useful when arguments might be mistaken for
+ command-line options
+
+ [<keyspaces...>]
+ List of keyspaces. By default, all keyspaces
+
+Configuring for Snapshots
+***************************
+To demonstrate creating snapshots with Nodetool on the commandline we have set
+``auto_snapshots`` setting to ``false`` in ``cassandra.yaml``:
+
+::
+
+ auto_snapshot: false
+
+Also set ``snapshot_before_compaction`` to ``false`` to disable creating snapshots automatically before compaction:
+
+::
+
+ snapshot_before_compaction: false
+
+Creating Snapshots
+*******************
+To demonstrate creating snapshots start with no snapshots. Search for snapshots and none get listed:
+
+::
+
+ [ec2-user@ip-10-0-2-238 ~]$ find -name snapshots
+
+We shall be using the example keyspaces and tables to create snapshots.
+
+Taking Snapshots of all Tables in a Keyspace
++++++++++++++++++++++++++++++++++++++++++++++
+
+To take snapshots of all tables in a keyspace and also optionally tag the snapshot the syntax becomes:
+
+::
+
+ nodetool snapshot --tag <tag> --<keyspace>
+
+As an example create a snapshot called ``catalog-ks`` for all the tables in the ``catalogkeyspace`` keyspace:
+
+::
+
+ [ec2-user@ip-10-0-2-238 ~]$ nodetool snapshot --tag catalog-ks -- catalogkeyspace
+ Requested creating snapshot(s) for [catalogkeyspace] with snapshot name [catalog-ks] and
+ options {skipFlush=false}
+ Snapshot directory: catalog-ks
+
+Search for snapshots and ``snapshots`` directories for the tables ``journal`` and ``magazine``, which are in the ``catalogkeyspace`` keyspace should get listed:
+
+::
+
+ [ec2-user@ip-10-0-2-238 ~]$ find -name snapshots
+ ./cassandra/data/data/catalogkeyspace/journal-296a2d30c22a11e9b1350d927649052c/snapshots
+ ./cassandra/data/data/catalogkeyspace/magazine-446eae30c22a11e9b1350d927649052c/snapshots
+
+Snapshots of all tables in multiple keyspaces may be created similarly, as an example:
+
+::
+
+ nodetool snapshot --tag catalog-cql-ks --catalogkeyspace,cqlkeyspace
+
+Taking Snapshots of Single Table in a Keyspace
+++++++++++++++++++++++++++++++++++++++++++++++
+To take a snapshot of a single table the ``nodetool snapshot`` command syntax becomes as follows:
+
+::
+
+ nodetool snapshot --tag <tag> --table <table> --<keyspace>
+
+As an example create a snapshot for table ``magazine`` in keyspace ``catalokeyspace``:
+
+::
+
+ [ec2-user@ip-10-0-2-238 ~]$ nodetool snapshot --tag magazine --table magazine --
+ catalogkeyspace
+ Requested creating snapshot(s) for [catalogkeyspace] with snapshot name [magazine] and
+ options {skipFlush=false}
+ Snapshot directory: magazine
+
+Taking Snapshot of Multiple Tables from same Keyspace
+++++++++++++++++++++++++++++++++++++++++++++++++++++++
+To take snapshots of multiple tables in a keyspace the list of *Keyspace.table* must be specified with option ``--kt-list``. As an example create snapshots for tables ``t`` and ``t2`` in the ``cqlkeyspace`` keyspace:
+
+::
+
+ nodetool snapshot --kt-list cqlkeyspace.t,cqlkeyspace.t2 --tag multi-table
+ [ec2-user@ip-10-0-2-238 ~]$ nodetool snapshot --kt-list cqlkeyspace.t,cqlkeyspace.t2 --tag
+ multi-table
+ Requested creating snapshot(s) for [cqlkeyspace.t,cqlkeyspace.t2] with snapshot name [multi-
+ table] and options {skipFlush=false}
+ Snapshot directory: multi-table
+
+Multiple snapshots of the same set of tables may be created and tagged with a different name. As an example, create another snapshot for the same set of tables ``t`` and ``t2`` in the ``cqlkeyspace`` keyspace and tag the snapshots differently:
+
+::
+
+ [ec2-user@ip-10-0-2-238 ~]$ nodetool snapshot --kt-list cqlkeyspace.t,cqlkeyspace.t2 --tag
+ multi-table-2
+ Requested creating snapshot(s) for [cqlkeyspace.t,cqlkeyspace.t2] with snapshot name [multi-
+ table-2] and options {skipFlush=false}
+ Snapshot directory: multi-table-2
+
+Taking Snapshot of Multiple Tables from Different Keyspaces
+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+To take snapshots of multiple tables that are in different keyspaces the command syntax is the same as when multiple tables are in the same keyspace. Each *keyspace.table* must be specified separately in the ``--kt-list`` option. As an example, create a snapshot for table ``t`` in the ``cqlkeyspace`` and table ``journal`` in the catalogkeyspace and tag the snapshot ``multi-ks``.
+
+::
+
+ [ec2-user@ip-10-0-2-238 ~]$ nodetool snapshot --kt-list
+ catalogkeyspace.journal,cqlkeyspace.t --tag multi-ks
+ Requested creating snapshot(s) for [catalogkeyspace.journal,cqlkeyspace.t] with snapshot
+ name [multi-ks] and options {skipFlush=false}
+ Snapshot directory: multi-ks
+
+Listing Snapshots
+***************************
+To list snapshots use the ``nodetool listsnapshots`` command. All the snapshots that we created in the preceding examples get listed:
+
+::
+
+ [ec2-user@ip-10-0-2-238 ~]$ nodetool listsnapshots
+ Snapshot Details:
+ Snapshot name Keyspace name Column family name True size Size on disk
+ multi-table cqlkeyspace t2 4.86 KiB 5.67 KiB
+ multi-table cqlkeyspace t 4.89 KiB 5.7 KiB
+ multi-ks cqlkeyspace t 4.89 KiB 5.7 KiB
+ multi-ks catalogkeyspace journal 4.9 KiB 5.73 KiB
+ magazine catalogkeyspace magazine 4.9 KiB 5.73 KiB
+ multi-table-2 cqlkeyspace t2 4.86 KiB 5.67 KiB
+ multi-table-2 cqlkeyspace t 4.89 KiB 5.7 KiB
+ catalog-ks catalogkeyspace journal 4.9 KiB 5.73 KiB
+ catalog-ks catalogkeyspace magazine 4.9 KiB 5.73 KiB
+
+ Total TrueDiskSpaceUsed: 44.02 KiB
+
+Finding Snapshots Directories
+******************************
+The ``snapshots`` directories may be listed with ``find –name snapshots`` command:
+
+::
+
+ [ec2-user@ip-10-0-2-238 ~]$ find -name snapshots
+ ./cassandra/data/data/cqlkeyspace/t-d132e240c21711e9bbee19821dcea330/snapshots
+ ./cassandra/data/data/cqlkeyspace/t2-d993a390c22911e9b1350d927649052c/snapshots
+ ./cassandra/data/data/catalogkeyspace/journal-296a2d30c22a11e9b1350d927649052c/snapshots
+ ./cassandra/data/data/catalogkeyspace/magazine-446eae30c22a11e9b1350d927649052c/snapshots
+ [ec2-user@ip-10-0-2-238 ~]$
+
+To list the snapshots for a particular table first change directory ( with ``cd``) to the ``snapshots`` directory for the table. As an example, list the snapshots for the ``catalogkeyspace/journal`` table. Two snapshots get listed:
+
+::
+
+ [ec2-user@ip-10-0-2-238 ~]$ cd ./cassandra/data/data/catalogkeyspace/journal-
+ 296a2d30c22a11e9b1350d927649052c/snapshots
+ [ec2-user@ip-10-0-2-238 snapshots]$ ls -l
+ total 0
+ drwxrwxr-x. 2 ec2-user ec2-user 265 Aug 19 02:44 catalog-ks
+ drwxrwxr-x. 2 ec2-user ec2-user 265 Aug 19 02:52 multi-ks
+
+A ``snapshots`` directory lists the SSTable files in the snapshot. ``Schema.cql`` file is also created in each snapshot for the schema definition DDL that may be run in CQL to create the table when restoring from a snapshot:
+
+::
+
+ [ec2-user@ip-10-0-2-238 snapshots]$ cd catalog-ks
+ [ec2-user@ip-10-0-2-238 catalog-ks]$ ls -l
+ total 44
+ -rw-rw-r--. 1 ec2-user ec2-user 31 Aug 19 02:44 manifest.jsonZ
+
+ -rw-rw-r--. 4 ec2-user ec2-user 47 Aug 19 02:38 na-1-big-CompressionInfo.db
+ -rw-rw-r--. 4 ec2-user ec2-user 97 Aug 19 02:38 na-1-big-Data.db
+ -rw-rw-r--. 4 ec2-user ec2-user 10 Aug 19 02:38 na-1-big-Digest.crc32
+ -rw-rw-r--. 4 ec2-user ec2-user 16 Aug 19 02:38 na-1-big-Filter.db
+ -rw-rw-r--. 4 ec2-user ec2-user 16 Aug 19 02:38 na-1-big-Index.db
+ -rw-rw-r--. 4 ec2-user ec2-user 4687 Aug 19 02:38 na-1-big-Statistics.db
+ -rw-rw-r--. 4 ec2-user ec2-user 56 Aug 19 02:38 na-1-big-Summary.db
+ -rw-rw-r--. 4 ec2-user ec2-user 92 Aug 19 02:38 na-1-big-TOC.txt
+ -rw-rw-r--. 1 ec2-user ec2-user 814 Aug 19 02:44 schema.cql
+
+Clearing Snapshots
+******************
+Snapshots may be cleared or deleted with the ``nodetool clearsnapshot`` command. Either a specific snapshot name must be specified or the ``–all`` option must be specified.
+As an example delete a snapshot called ``magazine`` from keyspace ``cqlkeyspace``:
+
+::
+
+ nodetool clearsnapshot -t magazine – cqlkeyspace
+ Delete all snapshots from cqlkeyspace with the –all option.
+ nodetool clearsnapshot –all -- cqlkeyspace
+
+
+
+Incremental Backups
+^^^^^^^^^^^^^^^^^^^
+In the following sub-sections we shall discuss configuring and creating incremental backups.
+
+Configuring for Incremental Backups
+***********************************
+
+To create incremental backups set ``incremental_backups`` to ``true`` in ``cassandra.yaml``.
+
+::
+
+ incremental_backups: true
+
+This is the only setting needed to create incremental backups. By default ``incremental_backups`` setting is set to ``false`` because a new set of SSTable files is created for each data flush and if several CQL statements are to be run the ``backups`` directory could fill up quickly and use up storage that is needed to store table data.
+Incremental backups may also be enabled on the command line with the Nodetool command ``nodetool enablebackup``. Incremental backups may be disabled with ``nodetool disablebackup`` command. Status of incremental backups, whether they are enabled may be found with ``nodetool statusbackup``.
+
+
+
+Creating Incremental Backups
+******************************
+After each table is created flush the table data with ``nodetool flush`` command. Incremental backups get created.
+
+::
+
+ [ec2-user@ip-10-0-2-238 ~]$ nodetool flush cqlkeyspace t
+ [ec2-user@ip-10-0-2-238 ~]$ nodetool flush cqlkeyspace t2
+ [ec2-user@ip-10-0-2-238 ~]$ nodetool flush catalogkeyspace journal magazine
+
+Finding Incremental Backups
+***************************
+
+Incremental backups are created within the Cassandra’s ``data`` directory within a table directory. Backups may be found with following command.
+
+::
+
+ [ec2-user@ip-10-0-2-238 ~]$ find -name backups
+
+ ./cassandra/data/data/cqlkeyspace/t-d132e240c21711e9bbee19821dcea330/backups
+ ./cassandra/data/data/cqlkeyspace/t2-d993a390c22911e9b1350d927649052c/backups
+ ./cassandra/data/data/catalogkeyspace/journal-296a2d30c22a11e9b1350d927649052c/backups
+ ./cassandra/data/data/catalogkeyspace/magazine-446eae30c22a11e9b1350d927649052c/backups
+
+Creating an Incremental Backup
+******************************
+This section discusses how incremental backups are created in more detail starting with when a new keyspace is created and a table is added. Create a keyspace called ``CQLKeyspace`` (arbitrary name).
+
+::
+
+ cqlsh> CREATE KEYSPACE CQLKeyspace
+ ... WITH replication = {'class': 'SimpleStrategy', 'replication_factor' : 3}
+
+Create a table called ``t`` within the ``CQLKeyspace`` keyspace:
+
+::
+
+ cqlsh> USE CQLKeyspace;
+ cqlsh:cqlkeyspace> CREATE TABLE t (
+ ... id int,
+ ... k int,
+ ... v text,
+ ... PRIMARY KEY (id)
+ ... );
+
+Flush the keyspace and table:
+
+::
+
+ [ec2-user@ip-10-0-2-238 ~]$ nodetool flush cqlkeyspace t
+
+Search for backups and a ``backups`` directory should get listed even though we have added no table data yet.
+
+::
+
+ [ec2-user@ip-10-0-2-238 ~]$ find -name backups
+
+ ./cassandra/data/data/cqlkeyspace/t-d132e240c21711e9bbee19821dcea330/backups
+
+Change directory to the ``backups`` directory and list files and no files get listed as no table data has been added yet:
+
+::
+
+ [ec2-user@ip-10-0-2-238 ~]$ cd ./cassandra/data/data/cqlkeyspace/t-
+ d132e240c21711e9bbee19821dcea330/backups
+ [ec2-user@ip-10-0-2-238 backups]$ ls -l
+ total 0
+
+Next, add a row of data to table ``t`` that we created:
+
+::
+
+ cqlsh:cqlkeyspace> INSERT INTO t (id, k, v) VALUES (0, 0, 'val0');
+
+Run the ``nodetool flush`` command to flush table data:
+
+::
+
+ [ec2-user@ip-10-0-2-238 ~]$ nodetool flush cqlkeyspace t
+
+List the files and directories in the ``backups`` directory and SSTable files for an incremental backup get listed:
+
+::
+
+ [ec2-user@ip-10-0-2-238 ~]$ cd ./cassandra/data/data/cqlkeyspace/t-
+ d132e240c21711e9bbee19821dcea330/backups
+ [ec2-user@ip-10-0-2-238 backups]$ ls -l
+ total 36
+ -rw-rw-r--. 2 ec2-user ec2-user 47 Aug 19 00:32 na-1-big-CompressionInfo.db
+ -rw-rw-r--. 2 ec2-user ec2-user 43 Aug 19 00:32 na-1-big-Data.db
+ -rw-rw-r--. 2 ec2-user ec2-user 10 Aug 19 00:32 na-1-big-Digest.crc32
+ -rw-rw-r--. 2 ec2-user ec2-user 16 Aug 19 00:32 na-1-big-Filter.db
+ -rw-rw-r--. 2 ec2-user ec2-user 8 Aug 19 00:32 na-1-big-Index.db
+ -rw-rw-r--. 2 ec2-user ec2-user 4673 Aug 19 00:32 na-1-big-Statistics.db
+ -rw-rw-r--. 2 ec2-user ec2-user 56 Aug 19 00:32 na-1-big-Summary.db
+ -rw-rw-r--. 2 ec2-user ec2-user 92 Aug 19 00:32 na-1-big-TOC.txt
+
+Add another row of data:
+
+::
+
+ cqlsh:cqlkeyspace> INSERT INTO t (id, k, v) VALUES (1, 1, 'val1');
+
+Again, run the ``nodetool flush`` command:
+
+::
+
+ [ec2-user@ip-10-0-2-238 backups]$ nodetool flush cqlkeyspace t
+
+A new incremental backup gets created for the new data added. List the files in the ``backups`` directory for table ``t`` and two sets of SSTable files get listed, one for each incremental backup. The SSTable files are timestamped, which distinguishes the first incremental backup from the second:
+
+::
+
+ [ec2-user@ip-10-0-2-238 backups]$ ls -l
+ total 72
+ -rw-rw-r--. 2 ec2-user ec2-user 47 Aug 19 00:32 na-1-big-CompressionInfo.db
+ -rw-rw-r--. 2 ec2-user ec2-user 43 Aug 19 00:32 na-1-big-Data.db
+ -rw-rw-r--. 2 ec2-user ec2-user 10 Aug 19 00:32 na-1-big-Digest.crc32
+ -rw-rw-r--. 2 ec2-user ec2-user 16 Aug 19 00:32 na-1-big-Filter.db
+ -rw-rw-r--. 2 ec2-user ec2-user 8 Aug 19 00:32 na-1-big-Index.db
+ -rw-rw-r--. 2 ec2-user ec2-user 4673 Aug 19 00:32 na-1-big-Statistics.db
+ -rw-rw-r--. 2 ec2-user ec2-user 56 Aug 19 00:32 na-1-big-Summary.db
+ -rw-rw-r--. 2 ec2-user ec2-user 92 Aug 19 00:32 na-1-big-TOC.txt
+ -rw-rw-r--. 2 ec2-user ec2-user 47 Aug 19 00:35 na-2-big-CompressionInfo.db
+ -rw-rw-r--. 2 ec2-user ec2-user 41 Aug 19 00:35 na-2-big-Data.db
+ -rw-rw-r--. 2 ec2-user ec2-user 10 Aug 19 00:35 na-2-big-Digest.crc32
+ -rw-rw-r--. 2 ec2-user ec2-user 16 Aug 19 00:35 na-2-big-Filter.db
+ -rw-rw-r--. 2 ec2-user ec2-user 8 Aug 19 00:35 na-2-big-Index.db
+ -rw-rw-r--. 2 ec2-user ec2-user 4673 Aug 19 00:35 na-2-big-Statistics.db
+ -rw-rw-r--. 2 ec2-user ec2-user 56 Aug 19 00:35 na-2-big-Summary.db
+ -rw-rw-r--. 2 ec2-user ec2-user 92 Aug 19 00:35 na-2-big-TOC.txt
+ [ec2-user@ip-10-0-2-238 backups]$
+
+The ``backups`` directory for table ``cqlkeyspace/t`` is created within the ``data`` directory for the table:
+
+::
+
+ [ec2-user@ip-10-0-2-238 ~]$ cd ./cassandra/data/data/cqlkeyspace/t-
+ d132e240c21711e9bbee19821dcea330
+ [ec2-user@ip-10-0-2-238 t-d132e240c21711e9bbee19821dcea330]$ ls -l
+ total 36
+ drwxrwxr-x. 2 ec2-user ec2-user 226 Aug 19 02:30 backups
+ -rw-rw-r--. 2 ec2-user ec2-user 47 Aug 19 02:30 na-1-big-CompressionInfo.db
+ -rw-rw-r--. 2 ec2-user ec2-user 79 Aug 19 02:30 na-1-big-Data.db
+ -rw-rw-r--. 2 ec2-user ec2-user 10 Aug 19 02:30 na-1-big-Digest.crc32
+ -rw-rw-r--. 2 ec2-user ec2-user 16 Aug 19 02:30 na-1-big-Filter.db
+ -rw-rw-r--. 2 ec2-user ec2-user 16 Aug 19 02:30 na-1-big-Index.db
+ -rw-rw-r--. 2 ec2-user ec2-user 4696 Aug 19 02:30 na-1-big-Statistics.db
+ -rw-rw-r--. 2 ec2-user ec2-user 56 Aug 19 02:30 na-1-big-Summary.db
+ -rw-rw-r--. 2 ec2-user ec2-user 92 Aug 19 02:30 na-1-big-TOC.txt
+
+The incremental backups for the other keyspaces/tables get created similarly. As an example the ``backups`` directory for table ``catalogkeyspace/magazine`` is created within the data directory:
+
+::
+
+ [ec2-user@ip-10-0-2-238 ~]$ cd ./cassandra/data/data/catalogkeyspace/magazine-
+ 446eae30c22a11e9b1350d927649052c
+ [ec2-user@ip-10-0-2-238 magazine-446eae30c22a11e9b1350d927649052c]$ ls -l
+ total 36
+ drwxrwxr-x. 2 ec2-user ec2-user 226 Aug 19 02:38 backups
+ -rw-rw-r--. 2 ec2-user ec2-user 47 Aug 19 02:38 na-1-big-CompressionInfo.db
+ -rw-rw-r--. 2 ec2-user ec2-user 97 Aug 19 02:38 na-1-big-Data.db
+ -rw-rw-r--. 2 ec2-user ec2-user 10 Aug 19 02:38 na-1-big-Digest.crc32
+ -rw-rw-r--. 2 ec2-user ec2-user 16 Aug 19 02:38 na-1-big-Filter.db
+ -rw-rw-r--. 2 ec2-user ec2-user 16 Aug 19 02:38 na-1-big-Index.db
+ -rw-rw-r--. 2 ec2-user ec2-user 4687 Aug 19 02:38 na-1-big-Statistics.db
+ -rw-rw-r--. 2 ec2-user ec2-user 56 Aug 19 02:38 na-1-big-Summary.db
+ -rw-rw-r--. 2 ec2-user ec2-user 92 Aug 19 02:38 na-1-big-TOC.txt
+
+
+
+
+
+Restoring from Incremental Backups and Snapshots
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The two main tools/commands for restoring a table after it has been dropped are:
+
+- sstableloader
+- nodetool import
+
+A snapshot contains essentially the same set of SSTable files as an incremental backup does with a few additional files. A snapshot includes a ``schema.cql`` file for the schema DDL to create a table in CQL. A table backup does not include DDL which must be obtained from a snapshot when restoring from an incremental backup.
+
+
diff --git a/doc/source/operating/bulk_loading.rst b/doc/source/operating/bulk_loading.rst
index c8224d5..850260a 100644
--- a/doc/source/operating/bulk_loading.rst
+++ b/doc/source/operating/bulk_loading.rst
@@ -13,12 +13,648 @@
.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
.. See the License for the specific language governing permissions and
.. limitations under the License.
-
.. highlight:: none
.. _bulk-loading:
Bulk Loading
-------------
+==============
-.. todo:: TODO
+Bulk loading of data in Apache Cassandra is supported by different tools. The data to be bulk loaded must be in the form of SSTables. Cassandra does not support loading data in any other format such as CSV, JSON, and XML directly. Bulk loading could be used to:
+
+- Restore incremental backups and snapshots. Backups and snapshots are already in the form of SSTables.
+- Load existing SSTables into another cluster, which could have a different number of nodes or replication strategy.
+- Load external data into a cluster
+
+**Note*: CSV Data can be loaded via the cqlsh COPY command but we do not recommend this for bulk loading, which typically requires many GB or TB of data.
+
+Tools for Bulk Loading
+^^^^^^^^^^^^^^^^^^^^^^
+
+Cassandra provides two commands or tools for bulk loading data. These are:
+
+- Cassandra Bulk loader, also called ``sstableloader``
+- The ``nodetool import`` command
+
+The ``sstableloader`` and ``nodetool import`` are accessible if the Cassandra installation ``bin`` directory is in the ``PATH`` environment variable. Or these may be accessed directly from the ``bin`` directory. We shall discuss each of these next. We shall use the example or sample keyspaces and tables created in the Backups section.
+
+Using sstableloader
+^^^^^^^^^^^^^^^^^^^
+
+The ``sstableloader`` is the main tool for bulk uploading data. The ``sstableloader`` streams SSTable data files to a running cluster. The ``sstableloader`` loads data conforming to the replication strategy and replication factor. The table to upload data to does need not to be empty.
+
+The only requirements to run ``sstableloader`` are:
+
+1. One or more comma separated initial hosts to connect to and get ring information.
+2. A directory path for the SSTables to load.
+
+Its usage is as follows.
+
+::
+
+ sstableloader [options] <dir_path>
+
+Sstableloader bulk loads the SSTables found in the directory ``<dir_path>`` to the configured cluster. The ``<dir_path>`` is used as the target *keyspace/table* name. As an example, to load an SSTable named
+``Standard1-g-1-Data.db`` into ``Keyspace1/Standard1``, you will need to have the
+files ``Standard1-g-1-Data.db`` and ``Standard1-g-1-Index.db`` in a directory ``/path/to/Keyspace1/Standard1/``.
+
+Sstableloader Option to accept Target keyspace name
+****************************************************
+Often as part of a backup strategy some Cassandra DBAs store an entire data directory. When corruption in data is found then they would like to restore data in the same cluster (for large clusters 200 nodes) but with different keyspace name.
+
+Currently ``sstableloader`` derives keyspace name from the folder structure. As an option to specify target keyspace name as part of ``sstableloader``, version 4.0 adds support for the ``--target-keyspace`` option (`CASSANDRA-13884
+<https://issues.apache.org/jira/browse/CASSANDRA-13884>`_).
+
+The supported options are as follows from which only ``-d,--nodes <initial hosts>`` is required.
+
+::
+
+ -alg,--ssl-alg <ALGORITHM> Client SSL: algorithm
+
+ -ap,--auth-provider <auth provider> Custom
+ AuthProvider class name for
+ cassandra authentication
+ -ciphers,--ssl-ciphers <CIPHER-SUITES> Client SSL:
+ comma-separated list of
+ encryption suites to use
+ -cph,--connections-per-host <connectionsPerHost> Number of
+ concurrent connections-per-host.
+ -d,--nodes <initial hosts> Required.
+ Try to connect to these hosts (comma separated) initially for ring information
+
+ -f,--conf-path <path to config file> cassandra.yaml file path for streaming throughput and client/server SSL.
+
+ -h,--help Display this help message
+
+ -i,--ignore <NODES> Don't stream to this (comma separated) list of nodes
+
+ -idct,--inter-dc-throttle <inter-dc-throttle> Inter-datacenter throttle speed in Mbits (default unlimited)
+
+ -k,--target-keyspace <target keyspace name> Target
+ keyspace name
+ -ks,--keystore <KEYSTORE> Client SSL:
+ full path to keystore
+ -kspw,--keystore-password <KEYSTORE-PASSWORD> Client SSL:
+ password of the keystore
+ --no-progress Don't
+ display progress
+ -p,--port <native transport port> Port used
+ for native connection (default 9042)
+ -prtcl,--ssl-protocol <PROTOCOL> Client SSL:
+ connections protocol to use (default: TLS)
+ -pw,--password <password> Password for
+ cassandra authentication
+ -sp,--storage-port <storage port> Port used
+ for internode communication (default 7000)
+ -spd,--server-port-discovery <allow server port discovery> Use ports
+ published by server to decide how to connect. With SSL requires StartTLS
+ to be used.
+ -ssp,--ssl-storage-port <ssl storage port> Port used
+ for TLS internode communication (default 7001)
+ -st,--store-type <STORE-TYPE> Client SSL:
+ type of store
+ -t,--throttle <throttle> Throttle
+ speed in Mbits (default unlimited)
+ -ts,--truststore <TRUSTSTORE> Client SSL:
+ full path to truststore
+ -tspw,--truststore-password <TRUSTSTORE-PASSWORD> Client SSL:
+ Password of the truststore
+ -u,--username <username> Username for
+ cassandra authentication
+ -v,--verbose verbose
+ output
+
+The ``cassandra.yaml`` file could be provided on the command-line with ``-f`` option to set up streaming throughput, client and server encryption options. Only ``stream_throughput_outbound_megabits_per_sec``, ``server_encryption_options`` and ``client_encryption_options`` are read from yaml. You can override options read from ``cassandra.yaml`` with corresponding command line options.
+
+A sstableloader Demo
+********************
+We shall demonstrate using ``sstableloader`` by uploading incremental backup data for table ``catalogkeyspace.magazine``. We shall also use a snapshot of the same table to bulk upload in a different run of ``sstableloader``. The backups and snapshots for the ``catalogkeyspace.magazine`` table are listed as follows.
+
+::
+
+ [ec2-user@ip-10-0-2-238 ~]$ cd ./cassandra/data/data/catalogkeyspace/magazine-
+ 446eae30c22a11e9b1350d927649052c
+ [ec2-user@ip-10-0-2-238 magazine-446eae30c22a11e9b1350d927649052c]$ ls -l
+ total 0
+ drwxrwxr-x. 2 ec2-user ec2-user 226 Aug 19 02:38 backups
+ drwxrwxr-x. 4 ec2-user ec2-user 40 Aug 19 02:45 snapshots
+
+The directory path structure of SSTables to be uploaded using ``sstableloader`` is used as the target keyspace/table.
+
+We could have directly uploaded from the ``backups`` and ``snapshots`` directories respectively if the directory structure were in the format used by ``sstableloader``. But the directory path of backups and snapshots for SSTables is ``/catalogkeyspace/magazine-446eae30c22a11e9b1350d927649052c/backups`` and ``/catalogkeyspace/magazine-446eae30c22a11e9b1350d927649052c/snapshots`` respectively, which cannot be used to upload SSTables to ``catalogkeyspace.magazine`` table. The directory path structure must be ``/catalogkeyspace/magazine/`` to use ``sstableloader``. We need to create a new directory structure to upload SSTables with ``sstableloader`` which is typical when using ``sstableloader``. Create a directory structure ``/catalogkeyspace/magazine`` and set its permissions.
+
+::
+
+ [ec2-user@ip-10-0-2-238 ~]$ sudo mkdir -p /catalogkeyspace/magazine
+ [ec2-user@ip-10-0-2-238 ~]$ sudo chmod -R 777 /catalogkeyspace/magazine
+
+Bulk Loading from an Incremental Backup
++++++++++++++++++++++++++++++++++++++++
+An incremental backup does not include the DDL for a table. The table must already exist. If the table was dropped it may be created using the ``schema.cql`` generated with every snapshot of a table. As we shall be using ``sstableloader`` to load SSTables to the ``magazine`` table, the table must exist prior to running ``sstableloader``. The table does not need to be empty but we have used an empty table as indicated by a CQL query:
+
+::
+
+ cqlsh:catalogkeyspace> SELECT * FROM magazine;
+
+ id | name | publisher
+ ----+------+-----------
+
+ (0 rows)
+
+After the table to upload has been created copy the SSTable files from the ``backups`` directory to the ``/catalogkeyspace/magazine/`` directory that we created.
+
+::
+
+ [ec2-user@ip-10-0-2-238 ~]$ sudo cp ./cassandra/data/data/catalogkeyspace/magazine-
+ 446eae30c22a11e9b1350d927649052c/backups/* /catalogkeyspace/magazine/
+
+Run the ``sstableloader`` to upload SSTables from the ``/catalogkeyspace/magazine/`` directory.
+
+::
+
+ sstableloader --nodes 10.0.2.238 /catalogkeyspace/magazine/
+
+The output from the ``sstableloader`` command should be similar to the listed:
+
+::
+
+ [ec2-user@ip-10-0-2-238 ~]$ sstableloader --nodes 10.0.2.238 /catalogkeyspace/magazine/
+ Opening SSTables and calculating sections to stream
+ Streaming relevant part of /catalogkeyspace/magazine/na-1-big-Data.db
+ /catalogkeyspace/magazine/na-2-big-Data.db to [35.173.233.153:7000, 10.0.2.238:7000,
+ 54.158.45.75:7000]
+ progress: [35.173.233.153:7000]0:1/2 88 % total: 88% 0.018KiB/s (avg: 0.018KiB/s)
+ progress: [35.173.233.153:7000]0:2/2 176% total: 176% 33.807KiB/s (avg: 0.036KiB/s)
+ progress: [35.173.233.153:7000]0:2/2 176% total: 176% 0.000KiB/s (avg: 0.029KiB/s)
+ progress: [35.173.233.153:7000]0:2/2 176% [10.0.2.238:7000]0:1/2 39 % total: 81% 0.115KiB/s
+ (avg: 0.024KiB/s)
+ progress: [35.173.233.153:7000]0:2/2 176% [10.0.2.238:7000]0:2/2 78 % total: 108%
+ 97.683KiB/s (avg: 0.033KiB/s)
+ progress: [35.173.233.153:7000]0:2/2 176% [10.0.2.238:7000]0:2/2 78 %
+ [54.158.45.75:7000]0:1/2 39 % total: 80% 0.233KiB/s (avg: 0.040KiB/s)
+ progress: [35.173.233.153:7000]0:2/2 176% [10.0.2.238:7000]0:2/2 78 %
+ [54.158.45.75:7000]0:2/2 78 % total: 96% 88.522KiB/s (avg: 0.049KiB/s)
+ progress: [35.173.233.153:7000]0:2/2 176% [10.0.2.238:7000]0:2/2 78 %
+ [54.158.45.75:7000]0:2/2 78 % total: 96% 0.000KiB/s (avg: 0.045KiB/s)
+ progress: [35.173.233.153:7000]0:2/2 176% [10.0.2.238:7000]0:2/2 78 %
+ [54.158.45.75:7000]0:2/2 78 % total: 96% 0.000KiB/s (avg: 0.044KiB/s)
+
+After the ``sstableloader`` has run query the ``magazine`` table and the loaded table should get listed when a query is run.
+
+::
+
+ cqlsh:catalogkeyspace> SELECT * FROM magazine;
+
+ id | name | publisher
+ ----+---------------------------+------------------
+ 1 | Couchbase Magazine | Couchbase
+ 0 | Apache Cassandra Magazine | Apache Cassandra
+
+ (2 rows)
+ cqlsh:catalogkeyspace>
+
+Bulk Loading from a Snapshot
++++++++++++++++++++++++++++++
+In this section we shall demonstrate restoring a snapshot of the ``magazine`` table to the ``magazine`` table. As we used the same table to restore data from a backup the directory structure required by ``sstableloader`` should already exist. If the directory structure needed to load SSTables to ``catalogkeyspace.magazine`` does not exist create the directories and set their permissions.
+
+::
+
+ [ec2-user@ip-10-0-2-238 ~]$ sudo mkdir -p /catalogkeyspace/magazine
+ [ec2-user@ip-10-0-2-238 ~]$ sudo chmod -R 777 /catalogkeyspace/magazine
+
+As we shall be copying the snapshot files to the directory remove any files that may be in the directory.
+
+::
+
+ [ec2-user@ip-10-0-2-238 ~]$ sudo rm /catalogkeyspace/magazine/*
+ [ec2-user@ip-10-0-2-238 ~]$ cd /catalogkeyspace/magazine/
+ [ec2-user@ip-10-0-2-238 magazine]$ ls -l
+ total 0
+
+
+Copy the snapshot files to the ``/catalogkeyspace/magazine`` directory.
+
+::
+
+ [ec2-user@ip-10-0-2-238 ~]$ sudo cp ./cassandra/data/data/catalogkeyspace/magazine-
+ 446eae30c22a11e9b1350d927649052c/snapshots/magazine/* /catalogkeyspace/magazine
+
+List the files in the ``/catalogkeyspace/magazine`` directory and a ``schema.cql`` should also get listed.
+
+::
+
+ [ec2-user@ip-10-0-2-238 ~]$ cd /catalogkeyspace/magazine
+ [ec2-user@ip-10-0-2-238 magazine]$ ls -l
+ total 44
+ -rw-r--r--. 1 root root 31 Aug 19 04:13 manifest.json
+ -rw-r--r--. 1 root root 47 Aug 19 04:13 na-1-big-CompressionInfo.db
+ -rw-r--r--. 1 root root 97 Aug 19 04:13 na-1-big-Data.db
+ -rw-r--r--. 1 root root 10 Aug 19 04:13 na-1-big-Digest.crc32
+ -rw-r--r--. 1 root root 16 Aug 19 04:13 na-1-big-Filter.db
+ -rw-r--r--. 1 root root 16 Aug 19 04:13 na-1-big-Index.db
+ -rw-r--r--. 1 root root 4687 Aug 19 04:13 na-1-big-Statistics.db
+ -rw-r--r--. 1 root root 56 Aug 19 04:13 na-1-big-Summary.db
+ -rw-r--r--. 1 root root 92 Aug 19 04:13 na-1-big-TOC.txt
+ -rw-r--r--. 1 root root 815 Aug 19 04:13 schema.cql
+
+Alternatively create symlinks to the snapshot folder instead of copying the data, something like:
+
+::
+
+ mkdir keyspace_name
+ ln -s _path_to_snapshot_folder keyspace_name/table_name
+
+If the ``magazine`` table was dropped run the DDL in the ``schema.cql`` to create the table. Run the ``sstableloader`` with the following command.
+
+::
+
+ sstableloader --nodes 10.0.2.238 /catalogkeyspace/magazine/
+
+As the output from the command indicates SSTables get streamed to the cluster.
+
+::
+
+ [ec2-user@ip-10-0-2-238 ~]$ sstableloader --nodes 10.0.2.238 /catalogkeyspace/magazine/
+
+ Established connection to initial hosts
+ Opening SSTables and calculating sections to stream
+ Streaming relevant part of /catalogkeyspace/magazine/na-1-big-Data.db to
+ [35.173.233.153:7000, 10.0.2.238:7000, 54.158.45.75:7000]
+ progress: [35.173.233.153:7000]0:1/1 176% total: 176% 0.017KiB/s (avg: 0.017KiB/s)
+ progress: [35.173.233.153:7000]0:1/1 176% total: 176% 0.000KiB/s (avg: 0.014KiB/s)
+ progress: [35.173.233.153:7000]0:1/1 176% [10.0.2.238:7000]0:1/1 78 % total: 108% 0.115KiB/s
+ (avg: 0.017KiB/s)
+ progress: [35.173.233.153:7000]0:1/1 176% [10.0.2.238:7000]0:1/1 78 %
+ [54.158.45.75:7000]0:1/1 78 % total: 96% 0.232KiB/s (avg: 0.024KiB/s)
+ progress: [35.173.233.153:7000]0:1/1 176% [10.0.2.238:7000]0:1/1 78 %
+ [54.158.45.75:7000]0:1/1 78 % total: 96% 0.000KiB/s (avg: 0.022KiB/s)
+ progress: [35.173.233.153:7000]0:1/1 176% [10.0.2.238:7000]0:1/1 78 %
+ [54.158.45.75:7000]0:1/1 78 % total: 96% 0.000KiB/s (avg: 0.021KiB/s)
+
+Some other requirements of ``sstableloader`` that should be kept into consideration are:
+
+- The SSTables to be loaded must be compatible with the Cassandra version being loaded into.
+- Repairing tables that have been loaded into a different cluster does not repair the source tables.
+- Sstableloader makes use of port 7000 for internode communication.
+- Before restoring incremental backups run ``nodetool flush`` to backup any data in memtables
+
+Using nodetool import
+^^^^^^^^^^^^^^^^^^^^^
+In this section we shall import SSTables into a table using the ``nodetool import`` command. The ``nodetool refresh`` command is deprecated, and it is recommended to use ``nodetool import`` instead. The ``nodetool refresh`` does not have an option to load new SSTables from a separate directory which the ``nodetool import`` does.
+
+The command usage is as follows.
+
+::
+
+ nodetool [(-h <host> | --host <host>)] [(-p <port> | --port <port>)]
+ [(-pp | --print-port)] [(-pw <password> | --password <password>)]
+ [(-pwf <passwordFilePath> | --password-file <passwordFilePath>)]
+ [(-u <username> | --username <username>)] import
+ [(-c | --no-invalidate-caches)] [(-e | --extended-verify)]
+ [(-l | --keep-level)] [(-q | --quick)] [(-r | --keep-repaired)]
+ [(-t | --no-tokens)] [(-v | --no-verify)] [--] <keyspace> <table>
+ <directory> ...
+
+The arguments ``keyspace``, ``table`` name and ``directory`` to import SSTables from are required.
+
+The supported options are as follows.
+
+::
+
+ -c, --no-invalidate-caches
+ Don't invalidate the row cache when importing
+
+ -e, --extended-verify
+ Run an extended verify, verifying all values in the new SSTables
+
+ -h <host>, --host <host>
+ Node hostname or ip address
+
+ -l, --keep-level
+ Keep the level on the new SSTables
+
+ -p <port>, --port <port>
+ Remote jmx agent port number
+
+ -pp, --print-port
+ Operate in 4.0 mode with hosts disambiguated by port number
+
+ -pw <password>, --password <password>
+ Remote jmx agent password
+
+ -pwf <passwordFilePath>, --password-file <passwordFilePath>
+ Path to the JMX password file
+
+ -q, --quick
+ Do a quick import without verifying SSTables, clearing row cache or
+ checking in which data directory to put the file
+
+ -r, --keep-repaired
+ Keep any repaired information from the SSTables
+
+ -t, --no-tokens
+ Don't verify that all tokens in the new SSTable are owned by the
+ current node
+
+ -u <username>, --username <username>
+ Remote jmx agent username
+
+ -v, --no-verify
+ Don't verify new SSTables
+
+ --
+ This option can be used to separate command-line options from the
+ list of argument, (useful when arguments might be mistaken for
+ command-line options
+
+As the keyspace and table are specified on the command line ``nodetool import`` does not have the same requirement that ``sstableloader`` does, which is to have the SSTables in a specific directory path. When importing snapshots or incremental backups with ``nodetool import`` the SSTables don’t need to be copied to another directory.
+
+Importing Data from an Incremental Backup
+*****************************************
+
+In this section we shall demonstrate using ``nodetool import`` to import SSTables from an incremental backup. We shall use the example table ``cqlkeyspace.t``. Drop table ``t`` as we are demonstrating to restore the table.
+
+::
+
+ cqlsh:cqlkeyspace> DROP table t;
+
+An incremental backup for a table does not include the schema definition for the table. If the schema definition is not kept as a separate backup, the ``schema.cql`` from a backup of the table may be used to create the table as follows.
+
+::
+
+ cqlsh:cqlkeyspace> CREATE TABLE IF NOT EXISTS cqlkeyspace.t (
+ ... id int PRIMARY KEY,
+ ... k int,
+ ... v text)
+ ... WITH ID = d132e240-c217-11e9-bbee-19821dcea330
+ ... AND bloom_filter_fp_chance = 0.01
+ ... AND crc_check_chance = 1.0
+ ... AND default_time_to_live = 0
+ ... AND gc_grace_seconds = 864000
+ ... AND min_index_interval = 128
+ ... AND max_index_interval = 2048
+ ... AND memtable_flush_period_in_ms = 0
+ ... AND speculative_retry = '99p'
+ ... AND additional_write_policy = '99p'
+ ... AND comment = ''
+ ... AND caching = { 'keys': 'ALL', 'rows_per_partition': 'NONE' }
+ ... AND compaction = { 'max_threshold': '32', 'min_threshold': '4',
+ 'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy' }
+ ... AND compression = { 'chunk_length_in_kb': '16', 'class':
+ 'org.apache.cassandra.io.compress.LZ4Compressor' }
+ ... AND cdc = false
+ ... AND extensions = { };
+
+Initially the table could be empty, but does not have to be.
+
+::
+
+ cqlsh:cqlkeyspace> SELECT * FROM t;
+
+ id | k | v
+ ----+---+---
+
+ (0 rows)
+
+Run the ``nodetool import`` command by providing the keyspace, table and the backups directory. We don’t need to copy the table backups to another directory to run ``nodetool import`` as we had to when using ``sstableloader``.
+
+::
+
+ [ec2-user@ip-10-0-2-238 ~]$ nodetool import -- cqlkeyspace t
+ ./cassandra/data/data/cqlkeyspace/t-d132e240c21711e9bbee19821dcea330/backups
+ [ec2-user@ip-10-0-2-238 ~]$
+
+The SSTables get imported into the table. Run a query in cqlsh to list the data imported.
+
+::
+
+ cqlsh:cqlkeyspace> SELECT * FROM t;
+
+ id | k | v
+ ----+---+------
+ 1 | 1 | val1
+ 0 | 0 | val0
+
+
+Importing Data from a Snapshot
+********************************
+Importing SSTables from a snapshot with the ``nodetool import`` command is similar to importing SSTables from an incremental backup. To demonstrate we shall import a snapshot for table ``catalogkeyspace.journal``. Drop the table as we are demonstrating to restore the table from a snapshot.
+
+::
+
+ cqlsh:cqlkeyspace> use CATALOGKEYSPACE;
+ cqlsh:catalogkeyspace> DROP TABLE journal;
+
+We shall use the ``catalog-ks`` snapshot for the ``journal`` table. List the files in the snapshot. The snapshot includes a ``schema.cql``, which is the schema definition for the ``journal`` table.
+
+::
+
+ [ec2-user@ip-10-0-2-238 catalog-ks]$ ls -l
+ total 44
+ -rw-rw-r--. 1 ec2-user ec2-user 31 Aug 19 02:44 manifest.json
+ -rw-rw-r--. 3 ec2-user ec2-user 47 Aug 19 02:38 na-1-big-CompressionInfo.db
+ -rw-rw-r--. 3 ec2-user ec2-user 97 Aug 19 02:38 na-1-big-Data.db
+ -rw-rw-r--. 3 ec2-user ec2-user 10 Aug 19 02:38 na-1-big-Digest.crc32
+ -rw-rw-r--. 3 ec2-user ec2-user 16 Aug 19 02:38 na-1-big-Filter.db
+ -rw-rw-r--. 3 ec2-user ec2-user 16 Aug 19 02:38 na-1-big-Index.db
+ -rw-rw-r--. 3 ec2-user ec2-user 4687 Aug 19 02:38 na-1-big-Statistics.db
+ -rw-rw-r--. 3 ec2-user ec2-user 56 Aug 19 02:38 na-1-big-Summary.db
+ -rw-rw-r--. 3 ec2-user ec2-user 92 Aug 19 02:38 na-1-big-TOC.txt
+ -rw-rw-r--. 1 ec2-user ec2-user 814 Aug 19 02:44 schema.cql
+
+Copy the DDL from the ``schema.cql`` and run in cqlsh to create the ``catalogkeyspace.journal`` table.
+
+::
+
+ cqlsh:catalogkeyspace> CREATE TABLE IF NOT EXISTS catalogkeyspace.journal (
+ ... id int PRIMARY KEY,
+ ... name text,
+ ... publisher text)
+ ... WITH ID = 296a2d30-c22a-11e9-b135-0d927649052c
+ ... AND bloom_filter_fp_chance = 0.01
+ ... AND crc_check_chance = 1.0
+ ... AND default_time_to_live = 0
+ ... AND gc_grace_seconds = 864000
+ ... AND min_index_interval = 128
+ ... AND max_index_interval = 2048
+ ... AND memtable_flush_period_in_ms = 0
+ ... AND speculative_retry = '99p'
+ ... AND additional_write_policy = '99p'
+ ... AND comment = ''
+ ... AND caching = { 'keys': 'ALL', 'rows_per_partition': 'NONE' }
+ ... AND compaction = { 'min_threshold': '4', 'max_threshold':
+ '32', 'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy' }
+ ... AND compression = { 'chunk_length_in_kb': '16', 'class':
+ 'org.apache.cassandra.io.compress.LZ4Compressor' }
+ ... AND cdc = false
+ ... AND extensions = { };
+
+
+Run the ``nodetool import`` command to import the SSTables for the snapshot.
+
+::
+
+ [ec2-user@ip-10-0-2-238 ~]$ nodetool import -- catalogkeyspace journal
+ ./cassandra/data/data/catalogkeyspace/journal-
+ 296a2d30c22a11e9b1350d927649052c/snapshots/catalog-ks/
+ [ec2-user@ip-10-0-2-238 ~]$
+
+Subsequently run a CQL query on the ``journal`` table and the data imported gets listed.
+
+::
+
+ cqlsh:catalogkeyspace>
+ cqlsh:catalogkeyspace> SELECT * FROM journal;
+
+ id | name | publisher
+ ----+---------------------------+------------------
+ 1 | Couchbase Magazine | Couchbase
+ 0 | Apache Cassandra Magazine | Apache Cassandra
+
+ (2 rows)
+ cqlsh:catalogkeyspace>
+
+
+Bulk Loading External Data
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Bulk loading external data directly is not supported by any of the tools we have discussed which include ``sstableloader`` and ``nodetool import``. The ``sstableloader`` and ``nodetool import`` require data to be in the form of SSTables. Apache Cassandra supports a Java API for generating SSTables from input data. Subsequently the ``sstableloader`` or ``nodetool import`` could be used to bulk load the SSTables. Next, we shall discuss the ``org.apache.cassandra.io.sstable.CQLSSTableWriter`` Java class for generating SSTables.
+
+Generating SSTables with CQLSSTableWriter Java API
+***************************************************
+To generate SSTables using the ``CQLSSTableWriter`` class the following need to be supplied at the least.
+
+- An output directory to generate the SSTable in
+- The schema for the SSTable
+- A prepared insert statement
+- A partitioner
+
+The output directory must already have been created. Create a directory (``/sstables`` as an example) and set its permissions.
+
+::
+
+ sudo mkdir /sstables
+ sudo chmod 777 -R /sstables
+
+Next, we shall discuss To use ``CQLSSTableWriter`` could be used in a Java application. Create a Java constant for the output directory.
+
+::
+
+ public static final String OUTPUT_DIR = "./sstables";
+
+``CQLSSTableWriter`` Java API has the provision to create a user defined type. Create a new type to store ``int`` data:
+
+::
+
+ String type = "CREATE TYPE CQLKeyspace.intType (a int, b int)";
+ // Define a String variable for the SSTable schema.
+ String schema = "CREATE TABLE CQLKeyspace.t ("
+ + " id int PRIMARY KEY,"
+ + " k int,"
+ + " v1 text,"
+ + " v2 intType,"
+ + ")";
+
+Define a ``String`` variable for the prepared insert statement to use:
+
+::
+
+ String insertStmt = "INSERT INTO CQLKeyspace.t (id, k, v1, v2) VALUES (?, ?, ?, ?)";
+
+The partitioner to use does not need to be set as the default partitioner ``Murmur3Partitioner`` is used.
+
+All these variables or settings are used by the builder class ``CQLSSTableWriter.Builder`` to create a ``CQLSSTableWriter`` object.
+
+Create a File object for the output directory.
+
+::
+
+ File outputDir = new File(OUTPUT_DIR + File.separator + "CQLKeyspace" + File.separator + "t");
+
+Next, obtain a ``CQLSSTableWriter.Builder`` object using ``static`` method ``CQLSSTableWriter.builder()``. Set the output
+directory ``File`` object, user defined type, SSTable schema, buffer size, prepared insert statement, and optionally any of the other builder options, and invoke the ``build()`` method to create a ``CQLSSTableWriter`` object:
+
+::
+
+ CQLSSTableWriter writer = CQLSSTableWriter.builder()
+ .inDirectory(outputDir)
+ .withType(type)
+ .forTable(schema)
+ .withBufferSizeInMB(256)
+ .using(insertStmt).build();
+
+Next, set the SSTable data. If any user define types are used obtain a ``UserType`` object for these:
+
+::
+
+ UserType userType = writer.getUDType("intType");
+
+Add data rows for the resulting SSTable.
+
+::
+
+ writer.addRow(0, 0, "val0", userType.newValue().setInt("a", 0).setInt("b", 0));
+ writer.addRow(1, 1, "val1", userType.newValue().setInt("a", 1).setInt("b", 1));
+ writer.addRow(2, 2, "val2", userType.newValue().setInt("a", 2).setInt("b", 2));
+
+Close the writer, finalizing the SSTable.
+
+::
+
+ writer.close();
+
+All the public methods the ``CQLSSTableWriter`` class provides including some other methods that are not discussed in the preceding example are as follows.
+
+===================================================================== ============
+Method Description
+===================================================================== ============
+addRow(java.util.List<java.lang.Object> values) Adds a new row to the writer. Returns a CQLSSTableWriter object. Each provided value type should correspond to the types of the CQL column the value is for. The correspondence between java type and CQL type is the same one than the one documented at www.datastax.com/drivers/java/2.0/apidocs/com/datastax/driver/core/DataType.Name.html#asJavaC lass().
+addRow(java.util.Map<java.lang.String,java.lang.Object> values) Adds a new row to the writer. Returns a CQLSSTableWriter object. This is equivalent to the other addRow methods, but takes a map whose keys are the names of the columns to add instead of taking a list of the values in the order of the insert statement used during construction of this SSTable writer. The column names in the map keys must be in lowercase unless the declared column name is a case-sensitive quoted identifier in which case the map key must use the exact case of the column. The values parameter is a map of column name to column values representing the new row to add. If a column is not included in the map, it's value will be null. If the map contains keys that do not correspond to one of the columns of the insert statement used when creating this SSTable writer, the corresponding value is ignored.
+addRow(java.lang.Object... values) Adds a new row to the writer. Returns a CQLSSTableWriter object.
+CQLSSTableWriter.builder() Returns a new builder for a CQLSSTableWriter.
+close() Closes the writer.
+rawAddRow(java.nio.ByteBuffer... values) Adds a new row to the writer given already serialized binary values. Returns a CQLSSTableWriter object. The row values must correspond to the bind variables of the insertion statement used when creating by this SSTable writer.
+rawAddRow(java.util.List<java.nio.ByteBuffer> values) Adds a new row to the writer given already serialized binary values. Returns a CQLSSTableWriter object. The row values must correspond to the bind variables of the insertion statement used when creating by this SSTable writer. |
+rawAddRow(java.util.Map<java.lang.String, java.nio.ByteBuffer> values) Adds a new row to the writer given already serialized binary values. Returns a CQLSSTableWriter object. The row values must correspond to the bind variables of the insertion statement used when creating by this SSTable writer. |
+getUDType(String dataType) Returns the User Defined type used in this SSTable Writer that can be used to create UDTValue instances.
+===================================================================== ============
+
+
+All the public methods the ``CQLSSTableWriter.Builder`` class provides including some other methods that are not discussed in the preceding example are as follows.
+
+============================================ ============
+Method Description
+============================================ ============
+inDirectory(String directory) The directory where to write the SSTables. This is a mandatory option. The directory to use should already exist and be writable.
+inDirectory(File directory) The directory where to write the SSTables. This is a mandatory option. The directory to use should already exist and be writable.
+forTable(String schema) The schema (CREATE TABLE statement) for the table for which SSTable is to be created. The
+ provided CREATE TABLE statement must use a fully-qualified table name, one that includes the
+ keyspace name. This is a mandatory option.
+
+withPartitioner(IPartitioner partitioner) The partitioner to use. By default, Murmur3Partitioner will be used. If this is not the
+ partitioner used by the cluster for which the SSTables are created, the correct partitioner
+ needs to be provided.
+
+using(String insert) The INSERT or UPDATE statement defining the order of the values to add for a given CQL row.
+ The provided INSERT statement must use a fully-qualified table name, one that includes the
+ keyspace name. Moreover, said statement must use bind variables since these variables will
+ be bound to values by the resulting SSTable writer. This is a mandatory option.
+
+withBufferSizeInMB(int size) The size of the buffer to use. This defines how much data will be buffered before being
+ written as a new SSTable. This corresponds roughly to the data size that will have the
+ created SSTable. The default is 128MB, which should be reasonable for a 1GB heap. If
+ OutOfMemory exception gets generated while using the SSTable writer, should lower this
+ value.
+
+sorted() Creates a CQLSSTableWriter that expects sorted inputs. If this option is used, the resulting
+ SSTable writer will expect rows to be added in SSTable sorted order (and an exception will
+ be thrown if that is not the case during row insertion). The SSTable sorted order means that
+ rows are added such that their partition keys respect the partitioner order. This option
+ should only be used if the rows can be provided in order, which is rarely the case. If the
+ rows can be provided in order however, using this sorted might be more efficient. If this
+ option is used, some option like withBufferSizeInMB will be ignored.
+
+build() Builds a CQLSSTableWriter object.
+
+============================================ ============
+
diff --git a/doc/source/operating/cdc.rst b/doc/source/operating/cdc.rst
index 192f62a..a7177b5 100644
--- a/doc/source/operating/cdc.rst
+++ b/doc/source/operating/cdc.rst
@@ -23,18 +23,26 @@
^^^^^^^^
Change data capture (CDC) provides a mechanism to flag specific tables for archival as well as rejecting writes to those
-tables once a configurable size-on-disk for the combined flushed and unflushed CDC-log is reached. An operator can
-enable CDC on a table by setting the table property ``cdc=true`` (either when :ref:`creating the table
-<create-table-statement>` or :ref:`altering it <alter-table-statement>`), after which any CommitLogSegments containing
-data for a CDC-enabled table are moved to the directory specified in ``cassandra.yaml`` on segment discard. A threshold
-of total disk space allowed is specified in the yaml at which time newly allocated CommitLogSegments will not allow CDC
-data until a consumer parses and removes data from the destination archival directory.
+tables once a configurable size-on-disk for the CDC log is reached. An operator can enable CDC on a table by setting the
+table property ``cdc=true`` (either when :ref:`creating the table <create-table-statement>` or
+:ref:`altering it <alter-table-statement>`). Upon CommitLogSegment creation, a hard-link to the segment is created in the
+directory specified in ``cassandra.yaml``. On segment fsync to disk, if CDC data is present anywhere in the segment a
+<segment_name>_cdc.idx file is also created with the integer offset of how much data in the original segment is persisted
+to disk. Upon final segment flush, a second line with the human-readable word "COMPLETED" will be added to the _cdc.idx
+file indicating that Cassandra has completed all processing on the file.
+
+We we use an index file rather than just encouraging clients to parse the log realtime off a memory mapped handle as data
+can be reflected in a kernel buffer that is not yet persisted to disk. Parsing only up to the listed offset in the _cdc.idx
+file will ensure that you only parse CDC data for data that is durable.
+
+A threshold of total disk space allowed is specified in the yaml at which time newly allocated CommitLogSegments will
+not allow CDC data until a consumer parses and removes files from the specified cdc_raw directory.
Configuration
^^^^^^^^^^^^^
-Enabling or disable CDC on a table
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Enabling or disabling CDC on a table
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
CDC is enable or disable through the `cdc` table property, for instance::
@@ -64,7 +72,7 @@
Reading CommitLogSegments
^^^^^^^^^^^^^^^^^^^^^^^^^
-This implementation included a refactor of CommitLogReplayer into `CommitLogReader.java
+Use a `CommitLogReader.java
<https://github.com/apache/cassandra/blob/e31e216234c6b57a531cae607e0355666007deb2/src/java/org/apache/cassandra/db/commitlog/CommitLogReader.java>`__.
Usage is `fairly straightforward
<https://github.com/apache/cassandra/blob/e31e216234c6b57a531cae607e0355666007deb2/src/java/org/apache/cassandra/db/commitlog/CommitLogReplayer.java#L132-L140>`__
@@ -78,12 +86,11 @@
**Do not enable CDC without some kind of consumption process in-place.**
-The initial implementation of Change Data Capture does not include a parser (see :ref:`reading-commitlogsegments` above)
-so, if CDC is enabled on a node and then on a table, the ``cdc_free_space_in_mb`` will fill up and then writes to
+If CDC is enabled on a node and then on a table, the ``cdc_free_space_in_mb`` will fill up and then writes to
CDC-enabled tables will be rejected unless some consumption process is in place.
Further Reading
^^^^^^^^^^^^^^^
-- `Design doc <https://docs.google.com/document/d/1ZxCWYkeZTquxsvf5hdPc0fiUnUHna8POvgt6TIzML4Y/edit>`__
- `JIRA ticket <https://issues.apache.org/jira/browse/CASSANDRA-8844>`__
+- `JIRA ticket <https://issues.apache.org/jira/browse/CASSANDRA-12148>`__
diff --git a/doc/source/operating/compaction.rst b/doc/source/operating/compaction.rst
deleted file mode 100644
index 0f39000..0000000
--- a/doc/source/operating/compaction.rst
+++ /dev/null
@@ -1,442 +0,0 @@
-.. Licensed to the Apache Software Foundation (ASF) under one
-.. or more contributor license agreements. See the NOTICE file
-.. distributed with this work for additional information
-.. regarding copyright ownership. The ASF licenses this file
-.. to you under the Apache License, Version 2.0 (the
-.. "License"); you may not use this file except in compliance
-.. with the License. You may obtain a copy of the License at
-..
-.. http://www.apache.org/licenses/LICENSE-2.0
-..
-.. Unless required by applicable law or agreed to in writing, software
-.. distributed under the License is distributed on an "AS IS" BASIS,
-.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-.. See the License for the specific language governing permissions and
-.. limitations under the License.
-
-.. highlight:: none
-
-.. _compaction:
-
-Compaction
-----------
-
-Types of compaction
-^^^^^^^^^^^^^^^^^^^
-
-The concept of compaction is used for different kinds of operations in Cassandra, the common thing about these
-operations is that it takes one or more sstables and output new sstables. The types of compactions are;
-
-Minor compaction
- triggered automatically in Cassandra.
-Major compaction
- a user executes a compaction over all sstables on the node.
-User defined compaction
- a user triggers a compaction on a given set of sstables.
-Scrub
- try to fix any broken sstables. This can actually remove valid data if that data is corrupted, if that happens you
- will need to run a full repair on the node.
-Upgradesstables
- upgrade sstables to the latest version. Run this after upgrading to a new major version.
-Cleanup
- remove any ranges this node does not own anymore, typically triggered on neighbouring nodes after a node has been
- bootstrapped since that node will take ownership of some ranges from those nodes.
-Secondary index rebuild
- rebuild the secondary indexes on the node.
-Anticompaction
- after repair the ranges that were actually repaired are split out of the sstables that existed when repair started.
-Sub range compaction
- It is possible to only compact a given sub range - this could be useful if you know a token that has been
- misbehaving - either gathering many updates or many deletes. (``nodetool compact -st x -et y``) will pick
- all sstables containing the range between x and y and issue a compaction for those sstables. For STCS this will
- most likely include all sstables but with LCS it can issue the compaction for a subset of the sstables. With LCS
- the resulting sstable will end up in L0.
-
-When is a minor compaction triggered?
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-# When an sstable is added to the node through flushing/streaming etc.
-# When autocompaction is enabled after being disabled (``nodetool enableautocompaction``)
-# When compaction adds new sstables.
-# A check for new minor compactions every 5 minutes.
-
-Merging sstables
-^^^^^^^^^^^^^^^^
-
-Compaction is about merging sstables, since partitions in sstables are sorted based on the hash of the partition key it
-is possible to efficiently merge separate sstables. Content of each partition is also sorted so each partition can be
-merged efficiently.
-
-Tombstones and Garbage Collection (GC) Grace
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-Why Tombstones
-~~~~~~~~~~~~~~
-
-When a delete request is received by Cassandra it does not actually remove the data from the underlying store. Instead
-it writes a special piece of data known as a tombstone. The Tombstone represents the delete and causes all values which
-occurred before the tombstone to not appear in queries to the database. This approach is used instead of removing values
-because of the distributed nature of Cassandra.
-
-Deletes without tombstones
-~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Imagine a three node cluster which has the value [A] replicated to every node.::
-
- [A], [A], [A]
-
-If one of the nodes fails and and our delete operation only removes existing values we can end up with a cluster that
-looks like::
-
- [], [], [A]
-
-Then a repair operation would replace the value of [A] back onto the two
-nodes which are missing the value.::
-
- [A], [A], [A]
-
-This would cause our data to be resurrected even though it had been
-deleted.
-
-Deletes with Tombstones
-~~~~~~~~~~~~~~~~~~~~~~~
-
-Starting again with a three node cluster which has the value [A] replicated to every node.::
-
- [A], [A], [A]
-
-If instead of removing data we add a tombstone record, our single node failure situation will look like this.::
-
- [A, Tombstone[A]], [A, Tombstone[A]], [A]
-
-Now when we issue a repair the Tombstone will be copied to the replica, rather than the deleted data being
-resurrected.::
-
- [A, Tombstone[A]], [A, Tombstone[A]], [A, Tombstone[A]]
-
-Our repair operation will correctly put the state of the system to what we expect with the record [A] marked as deleted
-on all nodes. This does mean we will end up accruing Tombstones which will permanently accumulate disk space. To avoid
-keeping tombstones forever we have a parameter known as ``gc_grace_seconds`` for every table in Cassandra.
-
-The gc_grace_seconds parameter and Tombstone Removal
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-The table level ``gc_grace_seconds`` parameter controls how long Cassandra will retain tombstones through compaction
-events before finally removing them. This duration should directly reflect the amount of time a user expects to allow
-before recovering a failed node. After ``gc_grace_seconds`` has expired the tombstone may be removed (meaning there will
-no longer be any record that a certain piece of data was deleted), but as a tombstone can live in one sstable and the
-data it covers in another, a compaction must also include both sstable for a tombstone to be removed. More precisely, to
-be able to drop an actual tombstone the following needs to be true;
-
-- The tombstone must be older than ``gc_grace_seconds``
-- If partition X contains the tombstone, the sstable containing the partition plus all sstables containing data older
- than the tombstone containing X must be included in the same compaction. We don't need to care if the partition is in
- an sstable if we can guarantee that all data in that sstable is newer than the tombstone. If the tombstone is older
- than the data it cannot shadow that data.
-- If the option ``only_purge_repaired_tombstones`` is enabled, tombstones are only removed if the data has also been
- repaired.
-
-If a node remains down or disconnected for longer than ``gc_grace_seconds`` it's deleted data will be repaired back to
-the other nodes and re-appear in the cluster. This is basically the same as in the "Deletes without Tombstones" section.
-Note that tombstones will not be removed until a compaction event even if ``gc_grace_seconds`` has elapsed.
-
-The default value for ``gc_grace_seconds`` is 864000 which is equivalent to 10 days. This can be set when creating or
-altering a table using ``WITH gc_grace_seconds``.
-
-TTL
-^^^
-
-Data in Cassandra can have an additional property called time to live - this is used to automatically drop data that has
-expired once the time is reached. Once the TTL has expired the data is converted to a tombstone which stays around for
-at least ``gc_grace_seconds``. Note that if you mix data with TTL and data without TTL (or just different length of the
-TTL) Cassandra will have a hard time dropping the tombstones created since the partition might span many sstables and
-not all are compacted at once.
-
-Fully expired sstables
-^^^^^^^^^^^^^^^^^^^^^^
-
-If an sstable contains only tombstones and it is guaranteed that that sstable is not shadowing data in any other sstable
-compaction can drop that sstable. If you see sstables with only tombstones (note that TTL:ed data is considered
-tombstones once the time to live has expired) but it is not being dropped by compaction, it is likely that other
-sstables contain older data. There is a tool called ``sstableexpiredblockers`` that will list which sstables are
-droppable and which are blocking them from being dropped. This is especially useful for time series compaction with
-``TimeWindowCompactionStrategy`` (and the deprecated ``DateTieredCompactionStrategy``).
-
-Repaired/unrepaired data
-^^^^^^^^^^^^^^^^^^^^^^^^
-
-With incremental repairs Cassandra must keep track of what data is repaired and what data is unrepaired. With
-anticompaction repaired data is split out into repaired and unrepaired sstables. To avoid mixing up the data again
-separate compaction strategy instances are run on the two sets of data, each instance only knowing about either the
-repaired or the unrepaired sstables. This means that if you only run incremental repair once and then never again, you
-might have very old data in the repaired sstables that block compaction from dropping tombstones in the unrepaired
-(probably newer) sstables.
-
-Data directories
-^^^^^^^^^^^^^^^^
-
-Since tombstones and data can live in different sstables it is important to realize that losing an sstable might lead to
-data becoming live again - the most common way of losing sstables is to have a hard drive break down. To avoid making
-data live tombstones and actual data are always in the same data directory. This way, if a disk is lost, all versions of
-a partition are lost and no data can get undeleted. To achieve this a compaction strategy instance per data directory is
-run in addition to the compaction strategy instances containing repaired/unrepaired data, this means that if you have 4
-data directories there will be 8 compaction strategy instances running. This has a few more benefits than just avoiding
-data getting undeleted:
-
-- It is possible to run more compactions in parallel - leveled compaction will have several totally separate levelings
- and each one can run compactions independently from the others.
-- Users can backup and restore a single data directory.
-- Note though that currently all data directories are considered equal, so if you have a tiny disk and a big disk
- backing two data directories, the big one will be limited the by the small one. One work around to this is to create
- more data directories backed by the big disk.
-
-Single sstable tombstone compaction
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-When an sstable is written a histogram with the tombstone expiry times is created and this is used to try to find
-sstables with very many tombstones and run single sstable compaction on that sstable in hope of being able to drop
-tombstones in that sstable. Before starting this it is also checked how likely it is that any tombstones will actually
-will be able to be dropped how much this sstable overlaps with other sstables. To avoid most of these checks the
-compaction option ``unchecked_tombstone_compaction`` can be enabled.
-
-.. _compaction-options:
-
-Common options
-^^^^^^^^^^^^^^
-
-There is a number of common options for all the compaction strategies;
-
-``enabled`` (default: true)
- Whether minor compactions should run. Note that you can have 'enabled': true as a compaction option and then do
- 'nodetool enableautocompaction' to start running compactions.
-``tombstone_threshold`` (default: 0.2)
- How much of the sstable should be tombstones for us to consider doing a single sstable compaction of that sstable.
-``tombstone_compaction_interval`` (default: 86400s (1 day))
- Since it might not be possible to drop any tombstones when doing a single sstable compaction we need to make sure
- that one sstable is not constantly getting recompacted - this option states how often we should try for a given
- sstable.
-``log_all`` (default: false)
- New detailed compaction logging, see :ref:`below <detailed-compaction-logging>`.
-``unchecked_tombstone_compaction`` (default: false)
- The single sstable compaction has quite strict checks for whether it should be started, this option disables those
- checks and for some usecases this might be needed. Note that this does not change anything for the actual
- compaction, tombstones are only dropped if it is safe to do so - it might just rewrite an sstable without being able
- to drop any tombstones.
-``only_purge_repaired_tombstone`` (default: false)
- Option to enable the extra safety of making sure that tombstones are only dropped if the data has been repaired.
-``min_threshold`` (default: 4)
- Lower limit of number of sstables before a compaction is triggered. Not used for ``LeveledCompactionStrategy``.
-``max_threshold`` (default: 32)
- Upper limit of number of sstables before a compaction is triggered. Not used for ``LeveledCompactionStrategy``.
-
-Further, see the section on each strategy for specific additional options.
-
-Compaction nodetool commands
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-The :ref:`nodetool <nodetool>` utility provides a number of commands related to compaction:
-
-``enableautocompaction``
- Enable compaction.
-``disableautocompaction``
- Disable compaction.
-``setcompactionthroughput``
- How fast compaction should run at most - defaults to 16MB/s, but note that it is likely not possible to reach this
- throughput.
-``compactionstats``
- Statistics about current and pending compactions.
-``compactionhistory``
- List details about the last compactions.
-``setcompactionthreshold``
- Set the min/max sstable count for when to trigger compaction, defaults to 4/32.
-
-Switching the compaction strategy and options using JMX
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-It is possible to switch compaction strategies and its options on just a single node using JMX, this is a great way to
-experiment with settings without affecting the whole cluster. The mbean is::
-
- org.apache.cassandra.db:type=ColumnFamilies,keyspace=<keyspace_name>,columnfamily=<table_name>
-
-and the attribute to change is ``CompactionParameters`` or ``CompactionParametersJson`` if you use jconsole or jmc. The
-syntax for the json version is the same as you would use in an :ref:`ALTER TABLE <alter-table-statement>` statement -
-for example::
-
- { 'class': 'LeveledCompactionStrategy', 'sstable_size_in_mb': 123, 'fanout_size': 10}
-
-The setting is kept until someone executes an :ref:`ALTER TABLE <alter-table-statement>` that touches the compaction
-settings or restarts the node.
-
-.. _detailed-compaction-logging:
-
-More detailed compaction logging
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-Enable with the compaction option ``log_all`` and a more detailed compaction log file will be produced in your log
-directory.
-
-.. _STCS:
-
-Size Tiered Compaction Strategy
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-The basic idea of ``SizeTieredCompactionStrategy`` (STCS) is to merge sstables of approximately the same size. All
-sstables are put in different buckets depending on their size. An sstable is added to the bucket if size of the sstable
-is within ``bucket_low`` and ``bucket_high`` of the current average size of the sstables already in the bucket. This
-will create several buckets and the most interesting of those buckets will be compacted. The most interesting one is
-decided by figuring out which bucket's sstables takes the most reads.
-
-Major compaction
-~~~~~~~~~~~~~~~~
-
-When running a major compaction with STCS you will end up with two sstables per data directory (one for repaired data
-and one for unrepaired data). There is also an option (-s) to do a major compaction that splits the output into several
-sstables. The sizes of the sstables are approximately 50%, 25%, 12.5%... of the total size.
-
-.. _stcs-options:
-
-STCS options
-~~~~~~~~~~~~
-
-``min_sstable_size`` (default: 50MB)
- Sstables smaller than this are put in the same bucket.
-``bucket_low`` (default: 0.5)
- How much smaller than the average size of a bucket a sstable should be before not being included in the bucket. That
- is, if ``bucket_low * avg_bucket_size < sstable_size`` (and the ``bucket_high`` condition holds, see below), then
- the sstable is added to the bucket.
-``bucket_high`` (default: 1.5)
- How much bigger than the average size of a bucket a sstable should be before not being included in the bucket. That
- is, if ``sstable_size < bucket_high * avg_bucket_size`` (and the ``bucket_low`` condition holds, see above), then
- the sstable is added to the bucket.
-
-Defragmentation
-~~~~~~~~~~~~~~~
-
-Defragmentation is done when many sstables are touched during a read. The result of the read is put in to the memtable
-so that the next read will not have to touch as many sstables. This can cause writes on a read-only-cluster.
-
-.. _LCS:
-
-Leveled Compaction Strategy
-^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-The idea of ``LeveledCompactionStrategy`` (LCS) is that all sstables are put into different levels where we guarantee
-that no overlapping sstables are in the same level. By overlapping we mean that the first/last token of a single sstable
-are never overlapping with other sstables. This means that for a SELECT we will only have to look for the partition key
-in a single sstable per level. Each level is 10x the size of the previous one and each sstable is 160MB by default. L0
-is where sstables are streamed/flushed - no overlap guarantees are given here.
-
-When picking compaction candidates we have to make sure that the compaction does not create overlap in the target level.
-This is done by always including all overlapping sstables in the next level. For example if we select an sstable in L3,
-we need to guarantee that we pick all overlapping sstables in L4 and make sure that no currently ongoing compactions
-will create overlap if we start that compaction. We can start many parallel compactions in a level if we guarantee that
-we wont create overlap. For L0 -> L1 compactions we almost always need to include all L1 sstables since most L0 sstables
-cover the full range. We also can't compact all L0 sstables with all L1 sstables in a single compaction since that can
-use too much memory.
-
-When deciding which level to compact LCS checks the higher levels first (with LCS, a "higher" level is one with a higher
-number, L0 being the lowest one) and if the level is behind a compaction will be started in that level.
-
-Major compaction
-~~~~~~~~~~~~~~~~
-
-It is possible to do a major compaction with LCS - it will currently start by filling out L1 and then once L1 is full,
-it continues with L2 etc. This is sub optimal and will change to create all the sstables in a high level instead,
-CASSANDRA-11817.
-
-Bootstrapping
-~~~~~~~~~~~~~
-
-During bootstrap sstables are streamed from other nodes. The level of the remote sstable is kept to avoid many
-compactions after the bootstrap is done. During bootstrap the new node also takes writes while it is streaming the data
-from a remote node - these writes are flushed to L0 like all other writes and to avoid those sstables blocking the
-remote sstables from going to the correct level, we only do STCS in L0 until the bootstrap is done.
-
-STCS in L0
-~~~~~~~~~~
-
-If LCS gets very many L0 sstables reads are going to hit all (or most) of the L0 sstables since they are likely to be
-overlapping. To more quickly remedy this LCS does STCS compactions in L0 if there are more than 32 sstables there. This
-should improve read performance more quickly compared to letting LCS do its L0 -> L1 compactions. If you keep getting
-too many sstables in L0 it is likely that LCS is not the best fit for your workload and STCS could work out better.
-
-Starved sstables
-~~~~~~~~~~~~~~~~
-
-If a node ends up with a leveling where there are a few very high level sstables that are not getting compacted they
-might make it impossible for lower levels to drop tombstones etc. For example, if there are sstables in L6 but there is
-only enough data to actually get a L4 on the node the left over sstables in L6 will get starved and not compacted. This
-can happen if a user changes sstable\_size\_in\_mb from 5MB to 160MB for example. To avoid this LCS tries to include
-those starved high level sstables in other compactions if there has been 25 compaction rounds where the highest level
-has not been involved.
-
-.. _lcs-options:
-
-LCS options
-~~~~~~~~~~~
-
-``sstable_size_in_mb`` (default: 160MB)
- The target compressed (if using compression) sstable size - the sstables can end up being larger if there are very
- large partitions on the node.
-
-``fanout_size`` (default: 10)
- The target size of levels increases by this fanout_size multiplier. You can reduce the space amplification by tuning
- this option.
-
-LCS also support the ``cassandra.disable_stcs_in_l0`` startup option (``-Dcassandra.disable_stcs_in_l0=true``) to avoid
-doing STCS in L0.
-
-.. _TWCS:
-
-Time Window CompactionStrategy
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-``TimeWindowCompactionStrategy`` (TWCS) is designed specifically for workloads where it's beneficial to have data on
-disk grouped by the timestamp of the data, a common goal when the workload is time-series in nature or when all data is
-written with a TTL. In an expiring/TTL workload, the contents of an entire SSTable likely expire at approximately the
-same time, allowing them to be dropped completely, and space reclaimed much more reliably than when using
-``SizeTieredCompactionStrategy`` or ``LeveledCompactionStrategy``. The basic concept is that
-``TimeWindowCompactionStrategy`` will create 1 sstable per file for a given window, where a window is simply calculated
-as the combination of two primary options:
-
-``compaction_window_unit`` (default: DAYS)
- A Java TimeUnit (MINUTES, HOURS, or DAYS).
-``compaction_window_size`` (default: 1)
- The number of units that make up a window.
-
-Taken together, the operator can specify windows of virtually any size, and `TimeWindowCompactionStrategy` will work to
-create a single sstable for writes within that window. For efficiency during writing, the newest window will be
-compacted using `SizeTieredCompactionStrategy`.
-
-Ideally, operators should select a ``compaction_window_unit`` and ``compaction_window_size`` pair that produces
-approximately 20-30 windows - if writing with a 90 day TTL, for example, a 3 Day window would be a reasonable choice
-(``'compaction_window_unit':'DAYS','compaction_window_size':3``).
-
-TimeWindowCompactionStrategy Operational Concerns
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-The primary motivation for TWCS is to separate data on disk by timestamp and to allow fully expired SSTables to drop
-more efficiently. One potential way this optimal behavior can be subverted is if data is written to SSTables out of
-order, with new data and old data in the same SSTable. Out of order data can appear in two ways:
-
-- If the user mixes old data and new data in the traditional write path, the data will be comingled in the memtables
- and flushed into the same SSTable, where it will remain comingled.
-- If the user's read requests for old data cause read repairs that pull old data into the current memtable, that data
- will be comingled and flushed into the same SSTable.
-
-While TWCS tries to minimize the impact of comingled data, users should attempt to avoid this behavior. Specifically,
-users should avoid queries that explicitly set the timestamp via CQL ``USING TIMESTAMP``. Additionally, users should run
-frequent repairs (which streams data in such a way that it does not become comingled), and disable background read
-repair by setting the table's ``read_repair_chance`` and ``dclocal_read_repair_chance`` to 0.
-
-Changing TimeWindowCompactionStrategy Options
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Operators wishing to enable ``TimeWindowCompactionStrategy`` on existing data should consider running a major compaction
-first, placing all existing data into a single (old) window. Subsequent newer writes will then create typical SSTables
-as expected.
-
-Operators wishing to change ``compaction_window_unit`` or ``compaction_window_size`` can do so, but may trigger
-additional compactions as adjacent windows are joined together. If the window size is decrease d (for example, from 24
-hours to 12 hours), then the existing SSTables will not be modified - TWCS can not split existing SSTables into multiple
-windows.
diff --git a/doc/source/operating/compaction/index.rst b/doc/source/operating/compaction/index.rst
new file mode 100644
index 0000000..ea505dd
--- /dev/null
+++ b/doc/source/operating/compaction/index.rst
@@ -0,0 +1,301 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements. See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership. The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License. You may obtain a copy of the License at
+..
+.. http://www.apache.org/licenses/LICENSE-2.0
+..
+.. Unless required by applicable law or agreed to in writing, software
+.. distributed under the License is distributed on an "AS IS" BASIS,
+.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+.. See the License for the specific language governing permissions and
+.. limitations under the License.
+
+.. highlight:: none
+
+.. _compaction:
+
+Compaction
+----------
+
+Strategies
+^^^^^^^^^^
+
+Picking the right compaction strategy for your workload will ensure the best performance for both querying and for compaction itself.
+
+:ref:`Size Tiered Compaction Strategy <stcs>`
+ The default compaction strategy. Useful as a fallback when other strategies don't fit the workload. Most useful for
+ non pure time series workloads with spinning disks, or when the I/O from :ref:`LCS <lcs>` is too high.
+
+
+:ref:`Leveled Compaction Strategy <lcs>`
+ Leveled Compaction Strategy (LCS) is optimized for read heavy workloads, or workloads with lots of updates and deletes. It is not a good choice for immutable time series data.
+
+
+:ref:`Time Window Compaction Strategy <twcs>`
+ Time Window Compaction Strategy is designed for TTL'ed, mostly immutable time series data.
+
+
+
+Types of compaction
+^^^^^^^^^^^^^^^^^^^
+
+The concept of compaction is used for different kinds of operations in Cassandra, the common thing about these
+operations is that it takes one or more sstables and output new sstables. The types of compactions are;
+
+Minor compaction
+ triggered automatically in Cassandra.
+Major compaction
+ a user executes a compaction over all sstables on the node.
+User defined compaction
+ a user triggers a compaction on a given set of sstables.
+Scrub
+ try to fix any broken sstables. This can actually remove valid data if that data is corrupted, if that happens you
+ will need to run a full repair on the node.
+Upgradesstables
+ upgrade sstables to the latest version. Run this after upgrading to a new major version.
+Cleanup
+ remove any ranges this node does not own anymore, typically triggered on neighbouring nodes after a node has been
+ bootstrapped since that node will take ownership of some ranges from those nodes.
+Secondary index rebuild
+ rebuild the secondary indexes on the node.
+Anticompaction
+ after repair the ranges that were actually repaired are split out of the sstables that existed when repair started.
+Sub range compaction
+ It is possible to only compact a given sub range - this could be useful if you know a token that has been
+ misbehaving - either gathering many updates or many deletes. (``nodetool compact -st x -et y``) will pick
+ all sstables containing the range between x and y and issue a compaction for those sstables. For STCS this will
+ most likely include all sstables but with LCS it can issue the compaction for a subset of the sstables. With LCS
+ the resulting sstable will end up in L0.
+
+When is a minor compaction triggered?
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+# When an sstable is added to the node through flushing/streaming etc.
+# When autocompaction is enabled after being disabled (``nodetool enableautocompaction``)
+# When compaction adds new sstables.
+# A check for new minor compactions every 5 minutes.
+
+Merging sstables
+^^^^^^^^^^^^^^^^
+
+Compaction is about merging sstables, since partitions in sstables are sorted based on the hash of the partition key it
+is possible to efficiently merge separate sstables. Content of each partition is also sorted so each partition can be
+merged efficiently.
+
+Tombstones and Garbage Collection (GC) Grace
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Why Tombstones
+~~~~~~~~~~~~~~
+
+When a delete request is received by Cassandra it does not actually remove the data from the underlying store. Instead
+it writes a special piece of data known as a tombstone. The Tombstone represents the delete and causes all values which
+occurred before the tombstone to not appear in queries to the database. This approach is used instead of removing values
+because of the distributed nature of Cassandra.
+
+Deletes without tombstones
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Imagine a three node cluster which has the value [A] replicated to every node.::
+
+ [A], [A], [A]
+
+If one of the nodes fails and and our delete operation only removes existing values we can end up with a cluster that
+looks like::
+
+ [], [], [A]
+
+Then a repair operation would replace the value of [A] back onto the two
+nodes which are missing the value.::
+
+ [A], [A], [A]
+
+This would cause our data to be resurrected even though it had been
+deleted.
+
+Deletes with Tombstones
+~~~~~~~~~~~~~~~~~~~~~~~
+
+Starting again with a three node cluster which has the value [A] replicated to every node.::
+
+ [A], [A], [A]
+
+If instead of removing data we add a tombstone record, our single node failure situation will look like this.::
+
+ [A, Tombstone[A]], [A, Tombstone[A]], [A]
+
+Now when we issue a repair the Tombstone will be copied to the replica, rather than the deleted data being
+resurrected.::
+
+ [A, Tombstone[A]], [A, Tombstone[A]], [A, Tombstone[A]]
+
+Our repair operation will correctly put the state of the system to what we expect with the record [A] marked as deleted
+on all nodes. This does mean we will end up accruing Tombstones which will permanently accumulate disk space. To avoid
+keeping tombstones forever we have a parameter known as ``gc_grace_seconds`` for every table in Cassandra.
+
+The gc_grace_seconds parameter and Tombstone Removal
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The table level ``gc_grace_seconds`` parameter controls how long Cassandra will retain tombstones through compaction
+events before finally removing them. This duration should directly reflect the amount of time a user expects to allow
+before recovering a failed node. After ``gc_grace_seconds`` has expired the tombstone may be removed (meaning there will
+no longer be any record that a certain piece of data was deleted), but as a tombstone can live in one sstable and the
+data it covers in another, a compaction must also include both sstable for a tombstone to be removed. More precisely, to
+be able to drop an actual tombstone the following needs to be true;
+
+- The tombstone must be older than ``gc_grace_seconds``
+- If partition X contains the tombstone, the sstable containing the partition plus all sstables containing data older
+ than the tombstone containing X must be included in the same compaction. We don't need to care if the partition is in
+ an sstable if we can guarantee that all data in that sstable is newer than the tombstone. If the tombstone is older
+ than the data it cannot shadow that data.
+- If the option ``only_purge_repaired_tombstones`` is enabled, tombstones are only removed if the data has also been
+ repaired.
+
+If a node remains down or disconnected for longer than ``gc_grace_seconds`` it's deleted data will be repaired back to
+the other nodes and re-appear in the cluster. This is basically the same as in the "Deletes without Tombstones" section.
+Note that tombstones will not be removed until a compaction event even if ``gc_grace_seconds`` has elapsed.
+
+The default value for ``gc_grace_seconds`` is 864000 which is equivalent to 10 days. This can be set when creating or
+altering a table using ``WITH gc_grace_seconds``.
+
+TTL
+^^^
+
+Data in Cassandra can have an additional property called time to live - this is used to automatically drop data that has
+expired once the time is reached. Once the TTL has expired the data is converted to a tombstone which stays around for
+at least ``gc_grace_seconds``. Note that if you mix data with TTL and data without TTL (or just different length of the
+TTL) Cassandra will have a hard time dropping the tombstones created since the partition might span many sstables and
+not all are compacted at once.
+
+Fully expired sstables
+^^^^^^^^^^^^^^^^^^^^^^
+
+If an sstable contains only tombstones and it is guaranteed that that sstable is not shadowing data in any other sstable
+compaction can drop that sstable. If you see sstables with only tombstones (note that TTL:ed data is considered
+tombstones once the time to live has expired) but it is not being dropped by compaction, it is likely that other
+sstables contain older data. There is a tool called ``sstableexpiredblockers`` that will list which sstables are
+droppable and which are blocking them from being dropped. This is especially useful for time series compaction with
+``TimeWindowCompactionStrategy`` (and the deprecated ``DateTieredCompactionStrategy``). With ``TimeWindowCompactionStrategy``
+it is possible to remove the guarantee (not check for shadowing data) by enabling ``unsafe_aggressive_sstable_expiration``.
+
+Repaired/unrepaired data
+^^^^^^^^^^^^^^^^^^^^^^^^
+
+With incremental repairs Cassandra must keep track of what data is repaired and what data is unrepaired. With
+anticompaction repaired data is split out into repaired and unrepaired sstables. To avoid mixing up the data again
+separate compaction strategy instances are run on the two sets of data, each instance only knowing about either the
+repaired or the unrepaired sstables. This means that if you only run incremental repair once and then never again, you
+might have very old data in the repaired sstables that block compaction from dropping tombstones in the unrepaired
+(probably newer) sstables.
+
+Data directories
+^^^^^^^^^^^^^^^^
+
+Since tombstones and data can live in different sstables it is important to realize that losing an sstable might lead to
+data becoming live again - the most common way of losing sstables is to have a hard drive break down. To avoid making
+data live tombstones and actual data are always in the same data directory. This way, if a disk is lost, all versions of
+a partition are lost and no data can get undeleted. To achieve this a compaction strategy instance per data directory is
+run in addition to the compaction strategy instances containing repaired/unrepaired data, this means that if you have 4
+data directories there will be 8 compaction strategy instances running. This has a few more benefits than just avoiding
+data getting undeleted:
+
+- It is possible to run more compactions in parallel - leveled compaction will have several totally separate levelings
+ and each one can run compactions independently from the others.
+- Users can backup and restore a single data directory.
+- Note though that currently all data directories are considered equal, so if you have a tiny disk and a big disk
+ backing two data directories, the big one will be limited the by the small one. One work around to this is to create
+ more data directories backed by the big disk.
+
+Single sstable tombstone compaction
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+When an sstable is written a histogram with the tombstone expiry times is created and this is used to try to find
+sstables with very many tombstones and run single sstable compaction on that sstable in hope of being able to drop
+tombstones in that sstable. Before starting this it is also checked how likely it is that any tombstones will actually
+will be able to be dropped how much this sstable overlaps with other sstables. To avoid most of these checks the
+compaction option ``unchecked_tombstone_compaction`` can be enabled.
+
+.. _compaction-options:
+
+Common options
+^^^^^^^^^^^^^^
+
+There is a number of common options for all the compaction strategies;
+
+``enabled`` (default: true)
+ Whether minor compactions should run. Note that you can have 'enabled': true as a compaction option and then do
+ 'nodetool enableautocompaction' to start running compactions.
+``tombstone_threshold`` (default: 0.2)
+ How much of the sstable should be tombstones for us to consider doing a single sstable compaction of that sstable.
+``tombstone_compaction_interval`` (default: 86400s (1 day))
+ Since it might not be possible to drop any tombstones when doing a single sstable compaction we need to make sure
+ that one sstable is not constantly getting recompacted - this option states how often we should try for a given
+ sstable.
+``log_all`` (default: false)
+ New detailed compaction logging, see :ref:`below <detailed-compaction-logging>`.
+``unchecked_tombstone_compaction`` (default: false)
+ The single sstable compaction has quite strict checks for whether it should be started, this option disables those
+ checks and for some usecases this might be needed. Note that this does not change anything for the actual
+ compaction, tombstones are only dropped if it is safe to do so - it might just rewrite an sstable without being able
+ to drop any tombstones.
+``only_purge_repaired_tombstone`` (default: false)
+ Option to enable the extra safety of making sure that tombstones are only dropped if the data has been repaired.
+``min_threshold`` (default: 4)
+ Lower limit of number of sstables before a compaction is triggered. Not used for ``LeveledCompactionStrategy``.
+``max_threshold`` (default: 32)
+ Upper limit of number of sstables before a compaction is triggered. Not used for ``LeveledCompactionStrategy``.
+
+Further, see the section on each strategy for specific additional options.
+
+Compaction nodetool commands
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The :ref:`nodetool <nodetool>` utility provides a number of commands related to compaction:
+
+``enableautocompaction``
+ Enable compaction.
+``disableautocompaction``
+ Disable compaction.
+``setcompactionthroughput``
+ How fast compaction should run at most - defaults to 16MB/s, but note that it is likely not possible to reach this
+ throughput.
+``compactionstats``
+ Statistics about current and pending compactions.
+``compactionhistory``
+ List details about the last compactions.
+``setcompactionthreshold``
+ Set the min/max sstable count for when to trigger compaction, defaults to 4/32.
+
+Switching the compaction strategy and options using JMX
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+It is possible to switch compaction strategies and its options on just a single node using JMX, this is a great way to
+experiment with settings without affecting the whole cluster. The mbean is::
+
+ org.apache.cassandra.db:type=ColumnFamilies,keyspace=<keyspace_name>,columnfamily=<table_name>
+
+and the attribute to change is ``CompactionParameters`` or ``CompactionParametersJson`` if you use jconsole or jmc. The
+syntax for the json version is the same as you would use in an :ref:`ALTER TABLE <alter-table-statement>` statement -
+for example::
+
+ { 'class': 'LeveledCompactionStrategy', 'sstable_size_in_mb': 123, 'fanout_size': 10}
+
+The setting is kept until someone executes an :ref:`ALTER TABLE <alter-table-statement>` that touches the compaction
+settings or restarts the node.
+
+.. _detailed-compaction-logging:
+
+More detailed compaction logging
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Enable with the compaction option ``log_all`` and a more detailed compaction log file will be produced in your log
+directory.
+
+
+
+
+
diff --git a/doc/source/operating/compaction/lcs.rst b/doc/source/operating/compaction/lcs.rst
new file mode 100644
index 0000000..48c282e
--- /dev/null
+++ b/doc/source/operating/compaction/lcs.rst
@@ -0,0 +1,90 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements. See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership. The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License. You may obtain a copy of the License at
+..
+.. http://www.apache.org/licenses/LICENSE-2.0
+..
+.. Unless required by applicable law or agreed to in writing, software
+.. distributed under the License is distributed on an "AS IS" BASIS,
+.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+.. See the License for the specific language governing permissions and
+.. limitations under the License.
+
+
+
+.. _LCS:
+
+Leveled Compaction Strategy
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The idea of ``LeveledCompactionStrategy`` (LCS) is that all sstables are put into different levels where we guarantee
+that no overlapping sstables are in the same level. By overlapping we mean that the first/last token of a single sstable
+are never overlapping with other sstables. This means that for a SELECT we will only have to look for the partition key
+in a single sstable per level. Each level is 10x the size of the previous one and each sstable is 160MB by default. L0
+is where sstables are streamed/flushed - no overlap guarantees are given here.
+
+When picking compaction candidates we have to make sure that the compaction does not create overlap in the target level.
+This is done by always including all overlapping sstables in the next level. For example if we select an sstable in L3,
+we need to guarantee that we pick all overlapping sstables in L4 and make sure that no currently ongoing compactions
+will create overlap if we start that compaction. We can start many parallel compactions in a level if we guarantee that
+we wont create overlap. For L0 -> L1 compactions we almost always need to include all L1 sstables since most L0 sstables
+cover the full range. We also can't compact all L0 sstables with all L1 sstables in a single compaction since that can
+use too much memory.
+
+When deciding which level to compact LCS checks the higher levels first (with LCS, a "higher" level is one with a higher
+number, L0 being the lowest one) and if the level is behind a compaction will be started in that level.
+
+Major compaction
+~~~~~~~~~~~~~~~~
+
+It is possible to do a major compaction with LCS - it will currently start by filling out L1 and then once L1 is full,
+it continues with L2 etc. This is sub optimal and will change to create all the sstables in a high level instead,
+CASSANDRA-11817.
+
+Bootstrapping
+~~~~~~~~~~~~~
+
+During bootstrap sstables are streamed from other nodes. The level of the remote sstable is kept to avoid many
+compactions after the bootstrap is done. During bootstrap the new node also takes writes while it is streaming the data
+from a remote node - these writes are flushed to L0 like all other writes and to avoid those sstables blocking the
+remote sstables from going to the correct level, we only do STCS in L0 until the bootstrap is done.
+
+STCS in L0
+~~~~~~~~~~
+
+If LCS gets very many L0 sstables reads are going to hit all (or most) of the L0 sstables since they are likely to be
+overlapping. To more quickly remedy this LCS does STCS compactions in L0 if there are more than 32 sstables there. This
+should improve read performance more quickly compared to letting LCS do its L0 -> L1 compactions. If you keep getting
+too many sstables in L0 it is likely that LCS is not the best fit for your workload and STCS could work out better.
+
+Starved sstables
+~~~~~~~~~~~~~~~~
+
+If a node ends up with a leveling where there are a few very high level sstables that are not getting compacted they
+might make it impossible for lower levels to drop tombstones etc. For example, if there are sstables in L6 but there is
+only enough data to actually get a L4 on the node the left over sstables in L6 will get starved and not compacted. This
+can happen if a user changes sstable\_size\_in\_mb from 5MB to 160MB for example. To avoid this LCS tries to include
+those starved high level sstables in other compactions if there has been 25 compaction rounds where the highest level
+has not been involved.
+
+.. _lcs-options:
+
+LCS options
+~~~~~~~~~~~
+
+``sstable_size_in_mb`` (default: 160MB)
+ The target compressed (if using compression) sstable size - the sstables can end up being larger if there are very
+ large partitions on the node.
+
+``fanout_size`` (default: 10)
+ The target size of levels increases by this fanout_size multiplier. You can reduce the space amplification by tuning
+ this option.
+
+LCS also support the ``cassandra.disable_stcs_in_l0`` startup option (``-Dcassandra.disable_stcs_in_l0=true``) to avoid
+doing STCS in L0.
+
+
diff --git a/doc/source/operating/compaction/stcs.rst b/doc/source/operating/compaction/stcs.rst
new file mode 100644
index 0000000..6589337
--- /dev/null
+++ b/doc/source/operating/compaction/stcs.rst
@@ -0,0 +1,58 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements. See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership. The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License. You may obtain a copy of the License at
+..
+.. http://www.apache.org/licenses/LICENSE-2.0
+..
+.. Unless required by applicable law or agreed to in writing, software
+.. distributed under the License is distributed on an "AS IS" BASIS,
+.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+.. See the License for the specific language governing permissions and
+.. limitations under the License.
+
+
+.. _STCS:
+
+Leveled Compaction Strategy
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The basic idea of ``SizeTieredCompactionStrategy`` (STCS) is to merge sstables of approximately the same size. All
+sstables are put in different buckets depending on their size. An sstable is added to the bucket if size of the sstable
+is within ``bucket_low`` and ``bucket_high`` of the current average size of the sstables already in the bucket. This
+will create several buckets and the most interesting of those buckets will be compacted. The most interesting one is
+decided by figuring out which bucket's sstables takes the most reads.
+
+Major compaction
+~~~~~~~~~~~~~~~~
+
+When running a major compaction with STCS you will end up with two sstables per data directory (one for repaired data
+and one for unrepaired data). There is also an option (-s) to do a major compaction that splits the output into several
+sstables. The sizes of the sstables are approximately 50%, 25%, 12.5%... of the total size.
+
+.. _stcs-options:
+
+STCS options
+~~~~~~~~~~~~
+
+``min_sstable_size`` (default: 50MB)
+ Sstables smaller than this are put in the same bucket.
+``bucket_low`` (default: 0.5)
+ How much smaller than the average size of a bucket a sstable should be before not being included in the bucket. That
+ is, if ``bucket_low * avg_bucket_size < sstable_size`` (and the ``bucket_high`` condition holds, see below), then
+ the sstable is added to the bucket.
+``bucket_high`` (default: 1.5)
+ How much bigger than the average size of a bucket a sstable should be before not being included in the bucket. That
+ is, if ``sstable_size < bucket_high * avg_bucket_size`` (and the ``bucket_low`` condition holds, see above), then
+ the sstable is added to the bucket.
+
+Defragmentation
+~~~~~~~~~~~~~~~
+
+Defragmentation is done when many sstables are touched during a read. The result of the read is put in to the memtable
+so that the next read will not have to touch as many sstables. This can cause writes on a read-only-cluster.
+
+
diff --git a/doc/source/operating/compaction/twcs.rst b/doc/source/operating/compaction/twcs.rst
new file mode 100644
index 0000000..3641a5a
--- /dev/null
+++ b/doc/source/operating/compaction/twcs.rst
@@ -0,0 +1,76 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements. See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership. The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License. You may obtain a copy of the License at
+..
+.. http://www.apache.org/licenses/LICENSE-2.0
+..
+.. Unless required by applicable law or agreed to in writing, software
+.. distributed under the License is distributed on an "AS IS" BASIS,
+.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+.. See the License for the specific language governing permissions and
+.. limitations under the License.
+
+
+.. _TWCS:
+
+Time Window CompactionStrategy
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+``TimeWindowCompactionStrategy`` (TWCS) is designed specifically for workloads where it's beneficial to have data on
+disk grouped by the timestamp of the data, a common goal when the workload is time-series in nature or when all data is
+written with a TTL. In an expiring/TTL workload, the contents of an entire SSTable likely expire at approximately the
+same time, allowing them to be dropped completely, and space reclaimed much more reliably than when using
+``SizeTieredCompactionStrategy`` or ``LeveledCompactionStrategy``. The basic concept is that
+``TimeWindowCompactionStrategy`` will create 1 sstable per file for a given window, where a window is simply calculated
+as the combination of two primary options:
+
+``compaction_window_unit`` (default: DAYS)
+ A Java TimeUnit (MINUTES, HOURS, or DAYS).
+``compaction_window_size`` (default: 1)
+ The number of units that make up a window.
+``unsafe_aggressive_sstable_expiration`` (default: false)
+ Expired sstables will be dropped without checking its data is shadowing other sstables. This is a potentially
+ risky option that can lead to data loss or deleted data re-appearing, going beyond what
+ `unchecked_tombstone_compaction` does for single sstable compaction. Due to the risk the jvm must also be
+ started with `-Dcassandra.unsafe_aggressive_sstable_expiration=true`.
+
+Taken together, the operator can specify windows of virtually any size, and `TimeWindowCompactionStrategy` will work to
+create a single sstable for writes within that window. For efficiency during writing, the newest window will be
+compacted using `SizeTieredCompactionStrategy`.
+
+Ideally, operators should select a ``compaction_window_unit`` and ``compaction_window_size`` pair that produces
+approximately 20-30 windows - if writing with a 90 day TTL, for example, a 3 Day window would be a reasonable choice
+(``'compaction_window_unit':'DAYS','compaction_window_size':3``).
+
+TimeWindowCompactionStrategy Operational Concerns
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The primary motivation for TWCS is to separate data on disk by timestamp and to allow fully expired SSTables to drop
+more efficiently. One potential way this optimal behavior can be subverted is if data is written to SSTables out of
+order, with new data and old data in the same SSTable. Out of order data can appear in two ways:
+
+- If the user mixes old data and new data in the traditional write path, the data will be comingled in the memtables
+ and flushed into the same SSTable, where it will remain comingled.
+- If the user's read requests for old data cause read repairs that pull old data into the current memtable, that data
+ will be comingled and flushed into the same SSTable.
+
+While TWCS tries to minimize the impact of comingled data, users should attempt to avoid this behavior. Specifically,
+users should avoid queries that explicitly set the timestamp via CQL ``USING TIMESTAMP``. Additionally, users should run
+frequent repairs (which streams data in such a way that it does not become comingled).
+
+Changing TimeWindowCompactionStrategy Options
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Operators wishing to enable ``TimeWindowCompactionStrategy`` on existing data should consider running a major compaction
+first, placing all existing data into a single (old) window. Subsequent newer writes will then create typical SSTables
+as expected.
+
+Operators wishing to change ``compaction_window_unit`` or ``compaction_window_size`` can do so, but may trigger
+additional compactions as adjacent windows are joined together. If the window size is decrease d (for example, from 24
+hours to 12 hours), then the existing SSTables will not be modified - TWCS can not split existing SSTables into multiple
+windows.
+
diff --git a/doc/source/operating/compression.rst b/doc/source/operating/compression.rst
index 01da34b..74c992f 100644
--- a/doc/source/operating/compression.rst
+++ b/doc/source/operating/compression.rst
@@ -20,24 +20,88 @@
-----------
Cassandra offers operators the ability to configure compression on a per-table basis. Compression reduces the size of
-data on disk by compressing the SSTable in user-configurable compression ``chunk_length_in_kb``. Because Cassandra
-SSTables are immutable, the CPU cost of compressing is only necessary when the SSTable is written - subsequent updates
+data on disk by compressing the SSTable in user-configurable compression ``chunk_length_in_kb``. As Cassandra SSTables
+are immutable, the CPU cost of compressing is only necessary when the SSTable is written - subsequent updates
to data will land in different SSTables, so Cassandra will not need to decompress, overwrite, and recompress data when
UPDATE commands are issued. On reads, Cassandra will locate the relevant compressed chunks on disk, decompress the full
chunk, and then proceed with the remainder of the read path (merging data from disks and memtables, read repair, and so
on).
+Compression algorithms typically trade off between the following three areas:
+
+- **Compression speed**: How fast does the compression algorithm compress data. This is critical in the flush and
+ compaction paths because data must be compressed before it is written to disk.
+- **Decompression speed**: How fast does the compression algorithm de-compress data. This is critical in the read
+ and compaction paths as data must be read off disk in a full chunk and decompressed before it can be returned.
+- **Ratio**: By what ratio is the uncompressed data reduced by. Cassandra typically measures this as the size of data
+ on disk relative to the uncompressed size. For example a ratio of ``0.5`` means that the data on disk is 50% the size
+ of the uncompressed data. Cassandra exposes this ratio per table as the ``SSTable Compression Ratio`` field of
+ ``nodetool tablestats``.
+
+Cassandra offers five compression algorithms by default that make different tradeoffs in these areas. While
+benchmarking compression algorithms depends on many factors (algorithm parameters such as compression level,
+the compressibility of the input data, underlying processor class, etc ...), the following table should help you pick
+a starting point based on your application's requirements with an extremely rough grading of the different choices
+by their performance in these areas (A is relatively good, F is relatively bad):
+
++---------------------------------------------+-----------------------+-------------+---------------+-------+-------------+
+| Compression Algorithm | Cassandra Class | Compression | Decompression | Ratio | C* Version |
++=============================================+=======================+=============+===============+=======+=============+
+| `LZ4 <https://lz4.github.io/lz4/>`_ | ``LZ4Compressor`` | A+ | A+ | C+ | ``>=1.2.2`` |
++---------------------------------------------+-----------------------+-------------+---------------+-------+-------------+
+| `LZ4HC <https://lz4.github.io/lz4/>`_ | ``LZ4Compressor`` | C+ | A+ | B+ | ``>= 3.6`` |
++---------------------------------------------+-----------------------+-------------+---------------+-------+-------------+
+| `Zstd <https://facebook.github.io/zstd/>`_ | ``ZstdCompressor`` | A- | A- | A+ | ``>= 4.0`` |
++---------------------------------------------+-----------------------+-------------+---------------+-------+-------------+
+| `Snappy <http://google.github.io/snappy/>`_ | ``SnappyCompressor`` | A- | A | C | ``>= 1.0`` |
++---------------------------------------------+-----------------------+-------------+---------------+-------+-------------+
+| `Deflate (zlib) <https://zlib.net>`_ | ``DeflateCompressor`` | C | C | A | ``>= 1.0`` |
++---------------------------------------------+-----------------------+-------------+---------------+-------+-------------+
+
+Generally speaking for a performance critical (latency or throughput) application ``LZ4`` is the right choice as it
+gets excellent ratio per CPU cycle spent. This is why it is the default choice in Cassandra.
+
+For storage critical applications (disk footprint), however, ``Zstd`` may be a better choice as it can get significant
+additional ratio to ``LZ4``.
+
+``Snappy`` is kept for backwards compatibility and ``LZ4`` will typically be preferable.
+
+``Deflate`` is kept for backwards compatibility and ``Zstd`` will typically be preferable.
+
Configuring Compression
^^^^^^^^^^^^^^^^^^^^^^^
-Compression is configured on a per-table basis as an optional argument to ``CREATE TABLE`` or ``ALTER TABLE``. By
-default, three options are relevant:
+Compression is configured on a per-table basis as an optional argument to ``CREATE TABLE`` or ``ALTER TABLE``. Three
+options are available for all compressors:
-- ``class`` specifies the compression class - Cassandra provides three classes (``LZ4Compressor``,
- ``SnappyCompressor``, and ``DeflateCompressor`` ). The default is ``LZ4Compressor``.
-- ``chunk_length_in_kb`` specifies the number of kilobytes of data per compression chunk. The default is 64KB.
-- ``crc_check_chance`` determines how likely Cassandra is to verify the checksum on each compression chunk during
- reads. The default is 1.0.
+- ``class`` (default: ``LZ4Compressor``): specifies the compression class to use. The two "fast"
+ compressors are ``LZ4Compressor`` and ``SnappyCompressor`` and the two "good" ratio compressors are ``ZstdCompressor``
+ and ``DeflateCompressor``.
+- ``chunk_length_in_kb`` (default: ``16KiB``): specifies the number of kilobytes of data per compression chunk. The main
+ tradeoff here is that larger chunk sizes give compression algorithms more context and improve their ratio, but
+ require reads to deserialize and read more off disk.
+- ``crc_check_chance`` (default: ``1.0``): determines how likely Cassandra is to verify the checksum on each compression
+ chunk during reads to protect against data corruption. Unless you have profiles indicating this is a performance
+ problem it is highly encouraged not to turn this off as it is Cassandra's only protection against bitrot.
+
+The ``LZ4Compressor`` supports the following additional options:
+
+- ``lz4_compressor_type`` (default ``fast``): specifies if we should use the ``high`` (a.k.a ``LZ4HC``) ratio version
+ or the ``fast`` (a.k.a ``LZ4``) version of ``LZ4``. The ``high`` mode supports a configurable level, which can allow
+ operators to tune the performance <-> ratio tradeoff via the ``lz4_high_compressor_level`` option. Note that in
+ ``4.0`` and above it may be preferable to use the ``Zstd`` compressor.
+- ``lz4_high_compressor_level`` (default ``9``): A number between ``1`` and ``17`` inclusive that represents how much
+ CPU time to spend trying to get more compression ratio. Generally lower levels are "faster" but they get less ratio
+ and higher levels are slower but get more compression ratio.
+
+The ``ZstdCompressor`` supports the following options in addition:
+
+- ``compression_level`` (default ``3``): A number between ``-131072`` and ``22`` inclusive that represents how much CPU
+ time to spend trying to get more compression ratio. The lower the level, the faster the speed (at the cost of ratio).
+ Values from 20 to 22 are called "ultra levels" and should be used with caution, as they require more memory.
+ The default of ``3`` is a good choice for competing with ``Deflate`` ratios and ``1`` is a good choice for competing
+ with ``LZ4``.
+
Users can set compression using the following syntax:
@@ -49,7 +113,7 @@
::
- ALTER TABLE keyspace.table WITH compression = {'class': 'SnappyCompressor', 'chunk_length_in_kb': 128, 'crc_check_chance': 0.5};
+ ALTER TABLE keyspace.table WITH compression = {'class': 'LZ4Compressor', 'chunk_length_in_kb': 64, 'crc_check_chance': 0.5};
Once enabled, compression can be disabled with ``ALTER TABLE`` setting ``enabled`` to ``false``:
@@ -72,7 +136,8 @@
than the time it would take to read or write the larger volume of uncompressed data from disk.
Compression is most useful in tables comprised of many rows, where the rows are similar in nature. Tables containing
-similar text columns (such as repeated JSON blobs) often compress very well.
+similar text columns (such as repeated JSON blobs) often compress very well. Tables containing data that has already
+been compressed or random data (e.g. benchmark datasets) do not typically compress well.
Operational Impact
^^^^^^^^^^^^^^^^^^
@@ -83,6 +148,11 @@
- Streaming operations involve compressing and decompressing data on compressed tables - in some code paths (such as
non-vnode bootstrap), the CPU overhead of compression can be a limiting factor.
+- To prevent slow compressors (``Zstd``, ``Deflate``, ``LZ4HC``) from blocking flushes for too long, all three
+ flush with the default fast ``LZ4`` compressor and then rely on normal compaction to re-compress the data into the
+ desired compression strategy. See `CASSANDRA-15379 <https://issues.apache.org/jira/browse/CASSANDRA-15379>` for more
+ details.
+
- The compression path checksums data to ensure correctness - while the traditional Cassandra read path does not have a
way to ensure correctness of data on disk, compressed tables allow the user to set ``crc_check_chance`` (a float from
0.0 to 1.0) to allow Cassandra to probabilistically validate chunks on read to verify bits on disk are not corrupt.
diff --git a/doc/source/operating/hardware.rst b/doc/source/operating/hardware.rst
index ad3aa8d..d90550c 100644
--- a/doc/source/operating/hardware.rst
+++ b/doc/source/operating/hardware.rst
@@ -77,8 +77,6 @@
of these environments. Users should choose similar hardware to what would be needed in physical space. In EC2, popular
options include:
-- m1.xlarge instances, which provide 1.6TB of local ephemeral spinning storage and sufficient RAM to run moderate
- workloads
- i2 instances, which provide both a high RAM:CPU ratio and local ephemeral SSDs
- m4.2xlarge / c4.4xlarge instances, which provide modern CPUs, enhanced networking and work well with EBS GP2 (SSD)
storage
diff --git a/doc/source/operating/hints.rst b/doc/source/operating/hints.rst
index f79f18a..55c42a4 100644
--- a/doc/source/operating/hints.rst
+++ b/doc/source/operating/hints.rst
@@ -16,7 +16,264 @@
.. highlight:: none
-Hints
------
+.. _hints:
-.. todo:: todo
+Hints
+=====
+
+Hinting is a data repair technique applied during write operations. When
+replica nodes are unavailable to accept a mutation, either due to failure or
+more commonly routine maintenance, coordinators attempting to write to those
+replicas store temporary hints on their local filesystem for later application
+to the unavailable replica. Hints are an important way to help reduce the
+duration of data inconsistency. Coordinators replay hints quickly after
+unavailable replica nodes return to the ring. Hints are best effort, however,
+and do not guarantee eventual consistency like :ref:`anti-entropy repair
+<repair>` does.
+
+Hints are useful because of how Apache Cassandra replicates data to provide
+fault tolerance, high availability and durability. Cassandra :ref:`partitions
+data across the cluster <consistent-hashing-token-ring>` using consistent
+hashing, and then replicates keys to multiple nodes along the hash ring. To
+guarantee availability, all replicas of a key can accept mutations without
+consensus, but this means it is possible for some replicas to accept a mutation
+while others do not. When this happens an inconsistency is introduced.
+
+Hints are one of the three ways, in addition to read-repair and
+full/incremental anti-entropy repair, that Cassandra implements the eventual
+consistency guarantee that all updates are eventually received by all replicas.
+Hints, like read-repair, are best effort and not an alternative to performing
+full repair, but they do help reduce the duration of inconsistency between
+replicas in practice.
+
+Hinted Handoff
+--------------
+
+Hinted handoff is the process by which Cassandra applies hints to unavailable
+nodes.
+
+For example, consider a mutation is to be made at ``Consistency Level``
+``LOCAL_QUORUM`` against a keyspace with ``Replication Factor`` of ``3``.
+Normally the client sends the mutation to a single coordinator, who then sends
+the mutation to all three replicas, and when two of the three replicas
+acknowledge the mutation the coordinator responds successfully to the client.
+If a replica node is unavailable, however, the coordinator stores a hint
+locally to the filesystem for later application. New hints will be retained for
+up to ``max_hint_window_in_ms`` of downtime (defaults to ``3 hours``). If the
+unavailable replica does return to the cluster before the window expires, the
+coordinator applies any pending hinted mutations against the replica to ensure
+that eventual consistency is maintained.
+
+.. figure:: images/hints.svg
+ :alt: Hinted Handoff Example
+
+ Hinted Handoff in Action
+
+* (``t0``): The write is sent by the client, and the coordinator sends it
+ to the three replicas. Unfortunately ``replica_2`` is restarting and cannot
+ receive the mutation.
+* (``t1``): The client receives a quorum acknowledgement from the coordinator.
+ At this point the client believe the write to be durable and visible to reads
+ (which it is).
+* (``t2``): After the write timeout (default ``2s``), the coordinator decides
+ that ``replica_2`` is unavailable and stores a hint to its local disk.
+* (``t3``): Later, when ``replica_2`` starts back up it sends a gossip message
+ to all nodes, including the coordinator.
+* (``t4``): The coordinator replays hints including the missed mutation
+ against ``replica_2``.
+
+If the node does not return in time, the destination replica will be
+permanently out of sync until either read-repair or full/incremental
+anti-entropy repair propagates the mutation.
+
+Application of Hints
+^^^^^^^^^^^^^^^^^^^^
+
+Hints are streamed in bulk, a segment at a time, to the target replica node and
+the target node replays them locally. After the target node has replayed a
+segment it deletes the segment and receives the next segment. This continues
+until all hints are drained.
+
+Storage of Hints on Disk
+^^^^^^^^^^^^^^^^^^^^^^^^
+
+Hints are stored in flat files in the coordinator node’s
+``$CASSANDRA_HOME/data/hints`` directory. A hint includes a hint id, the target
+replica node on which the mutation is meant to be stored, the serialized
+mutation (stored as a blob) that couldn't be delivered to the replica node, the
+mutation timestamp, and the Cassandra version used to serialize the mutation.
+By default hints are compressed using ``LZ4Compressor``. Multiple hints are
+appended to the same hints file.
+
+Since hints contain the original unmodified mutation timestamp, hint application
+is idempotent and cannot overwrite a future mutation.
+
+Hints for Timed Out Write Requests
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Hints are also stored for write requests that time out. The
+``write_request_timeout_in_ms`` setting in ``cassandra.yaml`` configures the
+timeout for write requests.
+
+::
+
+ write_request_timeout_in_ms: 2000
+
+The coordinator waits for the configured amount of time for write requests to
+complete, at which point it will time out and generate a hint for the timed out
+request. The lowest acceptable value for ``write_request_timeout_in_ms`` is 10 ms.
+
+
+Configuring Hints
+-----------------
+
+Hints are enabled by default as they are critical for data consistency. The
+``cassandra.yaml`` configuration file provides several settings for configuring
+hints:
+
+Table 1. Settings for Hints
+
++--------------------------------------------+-------------------------------------------+-------------------------------+
+|Setting | Description |Default Value |
++--------------------------------------------+-------------------------------------------+-------------------------------+
+|``hinted_handoff_enabled`` |Enables/Disables hinted handoffs | ``true`` |
+| | | |
+| | | |
+| | | |
+| | | |
++--------------------------------------------+-------------------------------------------+-------------------------------+
+|``hinted_handoff_disabled_datacenters`` |A list of data centers that do not perform | ``unset`` |
+| |hinted handoffs even when handoff is | |
+| |otherwise enabled. | |
+| |Example: | |
+| | | |
+| | .. code-block:: yaml | |
+| | | |
+| | hinted_handoff_disabled_datacenters: | |
+| | - DC1 | |
+| | - DC2 | |
++--------------------------------------------+-------------------------------------------+-------------------------------+
+|``max_hint_window_in_ms`` |Defines the maximum amount of time (ms) | ``10800000`` # 3 hours |
+| |a node shall have hints generated after it | |
+| |has failed. | |
++--------------------------------------------+-------------------------------------------+-------------------------------+
+|``hinted_handoff_throttle_in_kb`` |Maximum throttle in KBs per second, per | |
+| |delivery thread. This will be reduced | ``1024`` |
+| |proportionally to the number of nodes in | |
+| |the cluster. | |
+| |(If there are two nodes in the cluster, | |
+| |each delivery thread will use the maximum | |
+| |rate; if there are 3, each will throttle | |
+| |to half of the maximum,since it is expected| |
+| |for two nodes to be delivering hints | |
+| |simultaneously.) | |
++--------------------------------------------+-------------------------------------------+-------------------------------+
+|``max_hints_delivery_threads`` |Number of threads with which to deliver | ``2`` |
+| |hints; Consider increasing this number when| |
+| |you have multi-dc deployments, since | |
+| |cross-dc handoff tends to be slower | |
++--------------------------------------------+-------------------------------------------+-------------------------------+
+|``hints_directory`` |Directory where Cassandra stores hints. |``$CASSANDRA_HOME/data/hints`` |
+| | | |
++--------------------------------------------+-------------------------------------------+-------------------------------+
+|``hints_flush_period_in_ms`` |How often hints should be flushed from the | ``10000`` |
+| |internal buffers to disk. Will *not* | |
+| |trigger fsync. | |
++--------------------------------------------+-------------------------------------------+-------------------------------+
+|``max_hints_file_size_in_mb`` |Maximum size for a single hints file, in | ``128`` |
+| |megabytes. | |
++--------------------------------------------+-------------------------------------------+-------------------------------+
+|``hints_compression`` |Compression to apply to the hint files. | ``LZ4Compressor`` |
+| |If omitted, hints files will be written | |
+| |uncompressed. LZ4, Snappy, and Deflate | |
+| |compressors are supported. | |
++--------------------------------------------+-------------------------------------------+-------------------------------+
+
+Configuring Hints at Runtime with ``nodetool``
+----------------------------------------------
+
+``nodetool`` provides several commands for configuring hints or getting hints
+related information. The nodetool commands override the corresponding
+settings if any in ``cassandra.yaml`` for the node running the command.
+
+Table 2. Nodetool Commands for Hints
+
++--------------------------------+-------------------------------------------+
+|Command | Description |
++--------------------------------+-------------------------------------------+
+|``nodetool disablehandoff`` |Disables storing and delivering hints |
++--------------------------------+-------------------------------------------+
+|``nodetool disablehintsfordc`` |Disables storing and delivering hints to a |
+| |data center |
++--------------------------------+-------------------------------------------+
+|``nodetool enablehandoff`` |Re-enables future hints storing and |
+| |delivery on the current node |
++--------------------------------+-------------------------------------------+
+|``nodetool enablehintsfordc`` |Enables hints for a data center that was |
+| |previously disabled |
++--------------------------------+-------------------------------------------+
+|``nodetool getmaxhintwindow`` |Prints the max hint window in ms. New in |
+| |Cassandra 4.0. |
++--------------------------------+-------------------------------------------+
+|``nodetool handoffwindow`` |Prints current hinted handoff window |
++--------------------------------+-------------------------------------------+
+|``nodetool pausehandoff`` |Pauses hints delivery process |
++--------------------------------+-------------------------------------------+
+|``nodetool resumehandoff`` |Resumes hints delivery process |
++--------------------------------+-------------------------------------------+
+|``nodetool |Sets hinted handoff throttle in kb |
+|sethintedhandoffthrottlekb`` |per second, per delivery thread |
++--------------------------------+-------------------------------------------+
+|``nodetool setmaxhintwindow`` |Sets the specified max hint window in ms |
++--------------------------------+-------------------------------------------+
+|``nodetool statushandoff`` |Status of storing future hints on the |
+| |current node |
++--------------------------------+-------------------------------------------+
+|``nodetool truncatehints`` |Truncates all hints on the local node, or |
+| |truncates hints for the endpoint(s) |
+| |specified. |
++--------------------------------+-------------------------------------------+
+
+Make Hints Play Faster at Runtime
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The default of ``1024 kbps`` handoff throttle is conservative for most modern
+networks, and it is entirely possible that in a simple node restart you may
+accumulate many gigabytes hints that may take hours to play back. For example if
+you are ingesting ``100 Mbps`` of data per node, a single 10 minute long
+restart will create ``10 minutes * (100 megabit / second) ~= 7 GiB`` of data
+which at ``(1024 KiB / second)`` would take ``7.5 GiB / (1024 KiB / second) =
+2.03 hours`` to play back. The exact math depends on the load balancing strategy
+(round robin is better than token aware), number of tokens per node (more
+tokens is better than fewer), and naturally the cluster's write rate, but
+regardless you may find yourself wanting to increase this throttle at runtime.
+
+If you find yourself in such a situation, you may consider raising
+the ``hinted_handoff_throttle`` dynamically via the
+``nodetool sethintedhandoffthrottlekb`` command.
+
+Allow a Node to be Down Longer at Runtime
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Sometimes a node may be down for more than the normal ``max_hint_window_in_ms``,
+(default of three hours), but the hardware and data itself will still be
+accessible. In such a case you may consider raising the
+``max_hint_window_in_ms`` dynamically via the ``nodetool setmaxhintwindow``
+command added in Cassandra 4.0 (`CASSANDRA-11720 <https://issues.apache.org/jira/browse/CASSANDRA-11720>`_).
+This will instruct Cassandra to continue holding hints for the down
+endpoint for a longer amount of time.
+
+This command should be applied on all nodes in the cluster that may be holding
+hints. If needed, the setting can be applied permanently by setting the
+``max_hint_window_in_ms`` setting in ``cassandra.yaml`` followed by a rolling
+restart.
+
+Monitoring Hint Delivery
+------------------------
+
+Cassandra 4.0 adds histograms available to understand how long it takes to deliver
+hints which is useful for operators to better identify problems (`CASSANDRA-13234
+<https://issues.apache.org/jira/browse/CASSANDRA-13234>`_).
+
+There are also metrics available for tracking :ref:`Hinted Handoff <handoff-metrics>`
+and :ref:`Hints Service <hintsservice-metrics>` metrics.
diff --git a/doc/source/operating/images/hints.svg b/doc/source/operating/images/hints.svg
new file mode 100644
index 0000000..5e952e7
--- /dev/null
+++ b/doc/source/operating/images/hints.svg
@@ -0,0 +1,9 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="661.2000122070312" height="422.26666259765625" style="
+ width:661.2000122070312px;
+ height:422.26666259765625px;
+ background: transparent;
+ fill: none;
+">
+ <svg xmlns="http://www.w3.org/2000/svg" class="role-diagram-draw-area"><g class="shapes-region" style="stroke: black; fill: none;"><g class="composite-shape"><path class="real" d=" M40,60 C40,43.43 53.43,30 70,30 C86.57,30 100,43.43 100,60 C100,76.57 86.57,90 70,90 C53.43,90 40,76.57 40,60 Z" style="stroke-width: 1px; stroke: rgb(0, 0, 0); fill: none;"/></g><g class="arrow-line"><path class="connection real" stroke-dasharray="" d=" M70,300 L70,387" style="stroke: rgb(0, 0, 0); stroke-width: 1px; fill: none;"/><g stroke="none" fill="#000" transform="matrix(-1.8369701987210297e-16,-1,1,-1.8369701987210297e-16,70,390)" style="stroke: none; fill: rgb(0, 0, 0); stroke-width: 1px;"><path d=" M10.72,-5.15 L0,0 L10.72,5.15 L7.12,0 Z"/></g></g><g class="composite-shape"><path class="real" d=" M300,58.5 C300,41.93 313.43,28.5 330,28.5 C346.57,28.5 360,41.93 360,58.5 C360,75.07 346.57,88.5 330,88.5 C313.43,88.5 300,75.07 300,58.5 Z" style="stroke-width: 1px; stroke: rgb(0, 0, 0); fill: none;"/></g><g class="arrow-line"><path class="connection real" stroke-dasharray="" d=" M80,120 L197,118.54" style="stroke: rgb(0, 0, 0); stroke-width: 1px; fill: none;"/><g stroke="none" fill="rgba(0,0,0,1)" transform="matrix(-0.9999210442038161,0.01256603988335397,-0.01256603988335397,-0.9999210442038161,200,118.5)" style="stroke: none; fill: rgb(0, 0, 0); stroke-width: 1px;"><path d=" M8.93,-4.29 L0,0 L8.93,4.29 Z"/></g></g><g class="arrow-line"><path class="connection real" stroke-dasharray="" d=" M330,300 L330,385.5" style="stroke: rgb(0, 0, 0); stroke-width: 1px; fill: none;"/><g stroke="none" fill="#000" transform="matrix(-1.8369701987210297e-16,-1,1,-1.8369701987210297e-16,330,388.49999999999994)" style="stroke: none; fill: rgb(0, 0, 0); stroke-width: 1px;"><path d=" M10.72,-5.15 L0,0 L10.72,5.15 L7.12,0 Z"/></g></g><g class="composite-shape"><path class="real" d=" M420,60 C420,43.43 433.43,30 450,30 C466.57,30 480,43.43 480,60 C480,76.57 466.57,90 450,90 C433.43,90 420,76.57 420,60 Z" style="stroke-width: 1px; stroke: rgb(0, 0, 0); fill: none;"/></g><g class="arrow-line"><path class="connection real" stroke-dasharray="" d=" M570,300 L570,385.5" style="stroke: rgb(0, 0, 0); stroke-width: 1px; fill: none;"/><g stroke="none" fill="#000" transform="matrix(-1.8369701987210297e-16,-1,1,-1.8369701987210297e-16,570,388.5)" style="stroke: none; fill: rgb(0, 0, 0); stroke-width: 1px;"><path d=" M10.72,-5.15 L0,0 L10.72,5.15 L7.12,0 Z"/></g></g><g class="composite-shape"><path class="real" d=" M540,60 C540,43.43 553.43,30 570,30 C586.57,30 600,43.43 600,60 C600,76.57 586.57,90 570,90 C553.43,90 540,76.57 540,60 Z" style="stroke-width: 1px; stroke: rgb(0, 0, 0); fill: none;"/></g><g class="arrow-line"><path class="connection real" stroke-dasharray="" d=" M450,100 L450,128.5" style="stroke: rgb(0, 0, 0); stroke-width: 1px; fill: none;"/></g><g class="arrow-line"><path class="connection real" stroke-dasharray="" d=" M450,320 L450,385.5" style="stroke: rgb(0, 0, 0); stroke-width: 1px; fill: none;"/><g stroke="none" fill="#000" transform="matrix(-1.8369701987210297e-16,-1,1,-1.8369701987210297e-16,450,388.49999999999994)" style="stroke: none; fill: rgb(0, 0, 0); stroke-width: 1px;"><path d=" M10.72,-5.15 L0,0 L10.72,5.15 L7.12,0 Z"/></g></g><g class="arrow-line"><path class="connection real" stroke-dasharray="1.125 3.35" d=" M450,135.1 L450,220" style="stroke: rgb(0, 0, 0); stroke-width: 1px; fill: none;"/><g stroke="#000" transform="matrix(6.123233995736766e-17,1,-1,6.123233995736766e-17,449.99999999999994,135.10000000000002)" style="stroke: rgb(0, 0, 0); stroke-width: 1px;"><path d=" M0,5.59 L0,-5.59 M-5.03,5.59 L-5.03,-5.59"/></g></g><g class="composite-shape"><path class="real" d=" M180,56.5 C180,39.93 193.43,26.5 210,26.5 C226.57,26.5 240,39.93 240,56.5 C240,73.07 226.57,86.5 210,86.5 C193.43,86.5 180,73.07 180,56.5 Z" style="stroke-width: 1px; stroke: rgb(0, 0, 0); fill: none;"/></g><g class="arrow-line"><path class="connection real" stroke-dasharray="" d=" M210,300 L210,383.5" style="stroke: rgb(0, 0, 0); stroke-width: 1px; fill: none;"/><g stroke="none" fill="#000" transform="matrix(-1.8369701987210297e-16,-1,1,-1.8369701987210297e-16,210,386.49999999999994)" style="stroke: none; fill: rgb(0, 0, 0); stroke-width: 1px;"><path d=" M10.72,-5.15 L0,0 L10.72,5.15 L7.12,0 Z"/></g></g><g class="arrow-line"><path class="connection real" stroke-dasharray="" d=" M220,119.5 L317,119.5" style="stroke: rgb(0, 0, 0); stroke-width: 1px; fill: none;"/><g stroke="none" fill="rgba(0,0,0,1)" transform="matrix(-1,1.2246467991473532e-16,-1.2246467991473532e-16,-1,320,119.49999999999999)" style="stroke: none; fill: rgb(0, 0, 0); stroke-width: 1px;"><path d=" M8.93,-4.29 L0,0 L8.93,4.29 Z"/></g></g><g class="arrow-line"><path class="connection real" stroke-dasharray="" d=" M220,141 L437,140.01" style="stroke: rgb(0, 0, 0); stroke-width: 1px; fill: none;"/><g stroke="none" fill="rgba(0,0,0,1)" transform="matrix(-0.9999897039488793,0.004537840481175345,-0.004537840481175345,-0.9999897039488793,440,140)" style="stroke: none; fill: rgb(0, 0, 0); stroke-width: 1px;"><path d=" M8.93,-4.29 L0,0 L8.93,4.29 Z"/></g></g><g class="arrow-line"><path class="connection real" stroke-dasharray="" d=" M220,160 L557,160" style="stroke: rgb(0, 0, 0); stroke-width: 1px; fill: none;"/><g stroke="none" fill="rgba(0,0,0,1)" transform="matrix(-1,1.2246467991473532e-16,-1.2246467991473532e-16,-1,560,160)" style="stroke: none; fill: rgb(0, 0, 0); stroke-width: 1px;"><path d=" M8.93,-4.29 L0,0 L8.93,4.29 Z"/></g></g><g class="arrow-line"><path class="connection real" stroke-dasharray="" d=" M330,190 L223,190" style="stroke: rgb(0, 0, 0); stroke-width: 1px; fill: none;"/><g stroke="none" fill="rgba(0,0,0,1)" transform="matrix(1,-2.4492935982947064e-16,2.4492935982947064e-16,1,220,190)" style="stroke: none; fill: rgb(0, 0, 0); stroke-width: 1px;"><path d=" M8.93,-4.29 L0,0 L8.93,4.29 Z"/></g></g><g class="arrow-line"><path class="connection real" stroke-dasharray="" d=" M570,200 L223,200" style="stroke: rgb(0, 0, 0); stroke-width: 1px; fill: none;"/><g stroke="none" fill="rgba(0,0,0,1)" transform="matrix(1,-2.4492935982947064e-16,2.4492935982947064e-16,1,220,200)" style="stroke: none; fill: rgb(0, 0, 0); stroke-width: 1px;"><path d=" M8.93,-4.29 L0,0 L8.93,4.29 Z"/></g></g><g class="arrow-line"><path class="connection real" stroke-dasharray="" d=" M200,200 L83,200" style="stroke: rgb(0, 0, 0); stroke-width: 1px; fill: none;"/><g stroke="none" fill="rgba(0,0,0,1)" transform="matrix(1,-2.4492935982947064e-16,2.4492935982947064e-16,1,80,200)" style="stroke: none; fill: rgb(0, 0, 0); stroke-width: 1px;"><path d=" M8.93,-4.29 L0,0 L8.93,4.29 Z"/></g></g><g class="arrow-line"><path class="connection real" stroke-dasharray="" d=" M220,260 C248.94,258.95 251.69,269.27 222.77,269.96" style="stroke: rgb(0, 0, 0); stroke-width: 1px; fill: none;"/><g stroke="none" transform="matrix(0.9999965730559848,-0.0026179908874171876,0.0026179908874171876,0.9999965730559848,220,270)" style="stroke: none; stroke-width: 1px; fill: rgb(0, 0, 0);" fill="#000"><path d=" M8.93,-4.29 L0,0 L8.93,4.29 Z"/></g></g><g class="arrow-line"><path class="connection real" stroke-dasharray="" d=" M220,360 L437,360" style="stroke: rgb(0, 0, 0); stroke-width: 1px; fill: none;"/><g stroke="none" fill="rgba(0,0,0,1)" transform="matrix(-1,1.2246467991473532e-16,-1.2246467991473532e-16,-1,439.99999999999994,360.00000000000006)" style="stroke: none; fill: rgb(0, 0, 0); stroke-width: 1px;"><path d=" M8.93,-4.29 L0,0 L8.93,4.29 Z"/></g></g><g class="grouped-shape"><g class="arrow-line"><path class="connection real" stroke-dasharray="" d=" M60,220 Q62.5,217.5 65,220 Q67.5,222.5 70,220 Q72.5,217.5 75,220 Q77.5,222.5 80,220 L80,220" style="stroke: rgb(0, 0, 0); stroke-width: 3px; fill: none;"/></g><g class="arrow-line"><path class="connection real" stroke-dasharray="" d=" M60,230 Q62.5,227.5 65,230 Q67.5,232.5 70,230 Q72.5,227.5 75,230 Q77.5,232.5 80,230 L80,230" style="stroke: rgb(0, 0, 0); stroke-width: 3px; fill: none;"/></g></g><g class="arrow-line"><path class="connection real" stroke-dasharray="" d=" M70,100 L70,220" style="stroke: rgb(0, 0, 0); stroke-width: 1px; fill: none;"/></g><g class="arrow-line"><path class="connection real" stroke-dasharray="" d=" M200,220 Q202.5,217.5 205,220 Q207.5,222.5 210,220 Q212.5,217.5 215,220 Q217.5,222.5 220,220 L220,220" style="stroke: rgb(0, 0, 0); stroke-width: 3px; fill: none;"/></g><g class="arrow-line"><path class="connection real" stroke-dasharray="" d=" M200,230 Q202.5,227.5 205,230 Q207.5,232.5 210,230 Q212.5,227.5 215,230 Q217.5,232.5 220,230 L220,230" style="stroke: rgb(0, 0, 0); stroke-width: 3px; fill: none;"/></g><g class="grouped-shape"><g class="arrow-line"><path class="connection real" stroke-dasharray="" d=" M320,220 Q322.5,217.5 325,220 Q327.5,222.5 330,220 Q332.5,217.5 335,220 Q337.5,222.5 340,220 L340,220" style="stroke: rgb(0, 0, 0); stroke-width: 3px; fill: none;"/></g><g class="arrow-line"><path class="connection real" stroke-dasharray="" d=" M320,230 Q322.5,227.5 325,230 Q327.5,232.5 330,230 Q332.5,227.5 335,230 Q337.5,232.5 340,230 L340,230" style="stroke: rgb(0, 0, 0); stroke-width: 3px; fill: none;"/></g></g><g class="arrow-line"><path class="connection real" stroke-dasharray="" d=" M440,220 Q442.5,217.5 445,220 Q447.5,222.5 450,220 Q452.5,217.5 455,220 Q457.5,222.5 460,220 L460,220" style="stroke: rgb(0, 0, 0); stroke-width: 3px; fill: none;"/></g><g class="arrow-line"><path class="connection real" stroke-dasharray="" d=" M440,230 Q442.5,227.5 445,230 Q447.5,232.5 450,230 Q452.5,227.5 455,230 Q457.5,232.5 460,230 L460,230" style="stroke: rgb(0, 0, 0); stroke-width: 3px; fill: none;"/></g><g class="arrow-line"><path class="connection real" stroke-dasharray="" d=" M560,220 Q562.5,217.5 565,220 Q567.5,222.5 570,220 Q572.5,217.5 575,220 Q577.5,222.5 580,220 L580,220" style="stroke: rgb(0, 0, 0); stroke-width: 3px; fill: none;"/></g><g class="arrow-line"><path class="connection real" stroke-dasharray="" d=" M560,230 Q562.5,227.5 565,230 Q567.5,232.5 570,230 Q572.5,227.5 575,230 Q577.5,232.5 580,230 L580,230" style="stroke: rgb(0, 0, 0); stroke-width: 3px; fill: none;"/></g><g class="arrow-line"><path class="connection real" stroke-dasharray="" d=" M210,100 L210,220" style="stroke: rgb(0, 0, 0); stroke-width: 1px; fill: none;"/></g><g class="arrow-line"><path class="connection real" stroke-dasharray="" d=" M330,100 L330,220" style="stroke: rgb(0, 0, 0); stroke-width: 1px; fill: none;"/></g><g class="arrow-line"><path class="connection real" stroke-dasharray="" d=" M570,100 L570,220" style="stroke: rgb(0, 0, 0); stroke-width: 1px; fill: none;"/></g><g class="arrow-line"><path class="connection real" stroke-dasharray="1.125 3.35" d=" M450,315 L450,300" style="stroke: rgb(0, 0, 0); stroke-width: 1px; fill: none;"/><g stroke="#000" transform="matrix(-1.8369701987210297e-16,-1,1,-1.8369701987210297e-16,450,314.99999999999994)" style="stroke: rgb(0, 0, 0); stroke-width: 1px;"><path d=" M0,5.59 L0,-5.59 M-5.03,5.59 L-5.03,-5.59"/></g></g><g class="arrow-line"><path class="connection real" stroke-dasharray="" d=" M440,330 L223,330" style="stroke: rgb(0, 0, 0); stroke-width: 1px; fill: none;"/><g stroke="none" fill="rgba(0,0,0,1)" transform="matrix(1,-2.4492935982947064e-16,2.4492935982947064e-16,1,220,330)" style="stroke: none; fill: rgb(0, 0, 0); stroke-width: 1px;"><path d=" M8.93,-4.29 L0,0 L8.93,4.29 Z"/></g></g><g class="grouped-shape"><g class="arrow-line"><path class="connection real" stroke-dasharray="" d=" M60,290 Q62.5,287.5 65,290 Q67.5,292.5 70,290 Q72.5,287.5 75,290 Q77.5,292.5 80,290 L80,290" style="stroke: rgb(0, 0, 0); stroke-width: 3px; fill: none;"/></g><g class="arrow-line"><path class="connection real" stroke-dasharray="" d=" M60,300 Q62.5,297.5 65,300 Q67.5,302.5 70,300 Q72.5,297.5 75,300 Q77.5,302.5 80,300 L80,300" style="stroke: rgb(0, 0, 0); stroke-width: 3px; fill: none;"/></g></g><g class="arrow-line"><path class="connection real" stroke-dasharray="" d=" M70,230 L70,290" style="stroke: rgb(0, 0, 0); stroke-width: 1px; fill: none;"/></g><g class="grouped-shape"><g class="arrow-line"><path class="connection real" stroke-dasharray="" d=" M200,290 Q202.5,287.5 205,290 Q207.5,292.5 210,290 Q212.5,287.5 215,290 Q217.5,292.5 220,290 L220,290" style="stroke: rgb(0, 0, 0); stroke-width: 3px; fill: none;"/></g><g class="arrow-line"><path class="connection real" stroke-dasharray="" d=" M200,300 Q202.5,297.5 205,300 Q207.5,302.5 210,300 Q212.5,297.5 215,300 Q217.5,302.5 220,300 L220,300" style="stroke: rgb(0, 0, 0); stroke-width: 3px; fill: none;"/></g></g><g class="grouped-shape"><g class="arrow-line"><path class="connection real" stroke-dasharray="" d=" M320,290 Q322.5,287.5 325,290 Q327.5,292.5 330,290 Q332.5,287.5 335,290 Q337.5,292.5 340,290 L340,290" style="stroke: rgb(0, 0, 0); stroke-width: 3px; fill: none;"/></g><g class="arrow-line"><path class="connection real" stroke-dasharray="" d=" M320,300 Q322.5,297.5 325,300 Q327.5,302.5 330,300 Q332.5,297.5 335,300 Q337.5,302.5 340,300 L340,300" style="stroke: rgb(0, 0, 0); stroke-width: 3px; fill: none;"/></g></g><g class="arrow-line"><path class="connection real" stroke-dasharray="" d=" M210,230 L210,290" style="stroke: rgb(0, 0, 0); stroke-width: 1px; fill: none;"/></g><g class="arrow-line"><path class="connection real" stroke-dasharray="" d=" M330,230 L330,290" style="stroke: rgb(0, 0, 0); stroke-width: 1px; fill: none;"/></g><g class="grouped-shape"><g class="arrow-line"><path class="connection real" stroke-dasharray="" d=" M440,290 Q442.5,287.5 445,290 Q447.5,292.5 450,290 Q452.5,287.5 455,290 Q457.5,292.5 460,290 L460,290" style="stroke: rgb(0, 0, 0); stroke-width: 3px; fill: none;"/></g><g class="arrow-line"><path class="connection real" stroke-dasharray="" d=" M440,300 Q442.5,297.5 445,300 Q447.5,302.5 450,300 Q452.5,297.5 455,300 Q457.5,302.5 460,300 L460,300" style="stroke: rgb(0, 0, 0); stroke-width: 3px; fill: none;"/></g></g><g class="arrow-line"><path class="connection real" stroke-dasharray="1.125 3.35" d=" M450,230 L450,290" style="stroke: rgb(0, 0, 0); stroke-width: 1px; fill: none;"/></g><g class="grouped-shape"><g class="arrow-line"><path class="connection real" stroke-dasharray="" d=" M560,290 Q562.5,287.5 565,290 Q567.5,292.5 570,290 Q572.5,287.5 575,290 Q577.5,292.5 580,290 L580,290" style="stroke: rgb(0, 0, 0); stroke-width: 3px; fill: none;"/></g><g class="arrow-line"><path class="connection real" stroke-dasharray="" d=" M560,300 Q562.5,297.5 565,300 Q567.5,302.5 570,300 Q572.5,297.5 575,300 Q577.5,302.5 580,300 L580,300" style="stroke: rgb(0, 0, 0); stroke-width: 3px; fill: none;"/></g></g><g class="arrow-line"><path class="connection real" stroke-dasharray="" d=" M570,230 L570,290" style="stroke: rgb(0, 0, 0); stroke-width: 1px; fill: none;"/></g><g/></g><g/><g/><g/></svg>
+ <svg xmlns="http://www.w3.org/2000/svg" width="660" height="421.066650390625" style="width:660px;height:421.066650390625px;font-family:Asana-Math, Asana;background:transparent;"><g><g><g style="transform:matrix(1,0,0,1,47.266693115234375,65.81666564941406);"><path d="M342 330L365 330C373 395 380 432 389 458C365 473 330 482 293 482C248 483 175 463 118 400C64 352 25 241 25 136C25 40 67 -11 147 -11C201 -11 249 9 304 54L354 95L346 115L331 105C259 57 221 40 186 40C130 40 101 80 101 159C101 267 136 371 185 409C206 425 230 433 261 433C306 433 342 414 342 390ZM657 722L645 733C593 707 557 698 485 691L481 670L529 670C553 670 563 663 563 648C563 645 563 640 560 622C549 567 476 182 461 132C448 82 442 52 442 31C442 6 453 -9 472 -9C498 -9 534 12 632 85L622 103L596 86C567 67 545 56 535 56C528 56 522 66 522 76C522 82 523 89 526 104ZM717 388L724 368L756 389C793 412 796 414 803 414C813 414 821 404 821 391C821 384 817 361 813 347L747 107C739 76 734 49 734 30C734 6 745 -9 764 -9C790 -9 826 12 924 85L914 103L888 86C859 67 836 56 827 56C820 56 814 66 814 76C814 86 816 95 821 116L898 420C902 437 904 448 904 456C904 473 895 482 879 482C857 482 820 461 745 408ZM911 712C882 712 853 679 853 645C853 620 868 604 892 604C923 604 947 633 947 671C947 695 932 712 911 712ZM1288 111L1264 94C1211 56 1163 36 1127 36C1080 36 1051 73 1051 133C1051 158 1054 185 1059 214C1076 218 1185 248 1210 259C1295 296 1334 342 1334 404C1334 451 1300 482 1250 482C1182 496 1072 423 1035 349C1005 299 975 180 975 113C975 35 1019 -11 1091 -11C1148 -11 1204 17 1296 92ZM1073 274C1090 343 1110 386 1139 412C1157 428 1188 440 1212 440C1241 440 1260 420 1260 388C1260 344 1225 297 1173 272C1145 258 1109 247 1064 237ZM1372 388L1379 368L1411 389C1448 412 1451 414 1458 414C1469 414 1476 404 1476 389C1476 338 1435 145 1394 2L1401 -9C1426 -2 1449 4 1471 8C1490 134 1511 199 1557 268C1611 352 1686 414 1731 414C1742 414 1748 405 1748 390C1748 372 1745 351 1737 319L1685 107C1676 70 1672 47 1672 31C1672 6 1683 -9 1702 -9C1728 -9 1764 12 1862 85L1852 103L1826 86C1797 67 1775 56 1765 56C1758 56 1752 65 1752 76C1752 81 1753 92 1754 96L1820 372C1827 401 1831 429 1831 446C1831 469 1820 482 1800 482C1758 482 1689 444 1630 389C1592 354 1564 320 1512 247L1550 408C1554 426 1556 438 1556 449C1556 470 1548 482 1533 482C1512 482 1473 460 1400 408ZM2028 390L1972 107C1971 99 1959 61 1959 31C1959 6 1970 -9 1989 -9C2024 -9 2059 11 2137 74L2168 99L2158 117L2113 86C2084 66 2064 56 2053 56C2044 56 2039 64 2039 76C2039 102 2053 183 2082 328L2095 390L2202 390L2213 440C2175 436 2141 434 2103 434C2119 528 2130 577 2148 631L2137 646C2117 634 2090 622 2059 610L2034 440C1990 419 1964 408 1946 403L1944 390Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017,0,0,-0.017,0,0);"></path></g><g><g><g><g style="transform:matrix(1,0,0,1,85.80001831054688,68.81665649414063);"><path d="M418 -3L418 27L366 30C311 33 301 44 301 96L301 700L60 598L67 548L217 614L217 96C217 44 206 33 152 30L96 27L96 -3C250 0 250 0 261 0C292 0 402 -3 418 -3Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.0119,0,0,-0.0119,0,0);"></path></g></g></g></g></g></g><g><g><g style="transform:matrix(1,0,0,1,303.5000305175781,64.31666564941406);"><path d="M368 365C371 403 376 435 384 476C373 481 369 482 364 482C333 482 302 458 266 407C227 351 188 291 172 256L204 408C208 425 210 438 210 450C210 470 202 482 187 482C166 482 128 461 54 408L26 388L33 368L65 389C93 407 104 412 113 412C123 412 130 403 130 390C130 332 87 126 47 -2L57 -9C72 -4 88 -1 111 4L124 6L150 126C168 209 191 262 235 319C269 363 296 384 318 384C333 384 343 379 354 365ZM716 111L692 94C639 56 591 36 555 36C508 36 479 73 479 133C479 158 482 185 487 214C504 218 613 248 638 259C723 296 762 342 762 404C762 451 728 482 678 482C610 496 500 423 463 349C433 299 403 180 403 113C403 35 447 -11 519 -11C576 -11 632 17 724 92ZM501 274C518 343 538 386 567 412C585 428 616 440 640 440C669 440 688 420 688 388C688 344 653 297 601 272C573 258 537 247 492 237ZM952 -11C1008 -11 1123 62 1166 125C1207 186 1241 296 1241 371C1241 438 1218 482 1182 482C1140 482 1087 456 1035 409C994 374 974 348 938 289L962 408C965 425 967 440 967 452C967 471 959 482 945 482C924 482 886 461 812 408L784 388L791 368L823 389C851 407 862 412 871 412C881 412 888 403 888 389C888 381 886 361 884 351L826 8C816 -52 797 -143 777 -233L769 -270L776 -276C797 -269 817 -264 849 -259L891 3C912 -4 934 -11 952 -11ZM919 165C941 293 1048 424 1131 424C1157 424 1169 402 1169 356C1169 275 1129 156 1076 80C1056 51 1024 36 983 36C952 36 927 43 901 59ZM1526 722L1514 733C1462 707 1426 698 1354 691L1350 670L1398 670C1422 670 1432 663 1432 648C1432 645 1432 640 1429 622C1418 567 1345 182 1330 132C1317 82 1311 52 1311 31C1311 6 1322 -9 1341 -9C1367 -9 1403 12 1501 85L1491 103L1465 86C1436 67 1414 56 1404 56C1397 56 1391 66 1391 76C1391 82 1392 89 1395 104ZM1586 388L1593 368L1625 389C1662 412 1665 414 1672 414C1682 414 1690 404 1690 391C1690 384 1686 361 1682 347L1616 107C1608 76 1603 49 1603 30C1603 6 1614 -9 1633 -9C1659 -9 1695 12 1793 85L1783 103L1757 86C1728 67 1705 56 1696 56C1689 56 1683 66 1683 76C1683 86 1685 95 1690 116L1767 420C1771 437 1773 448 1773 456C1773 473 1764 482 1748 482C1726 482 1689 461 1614 408ZM1780 712C1751 712 1722 679 1722 645C1722 620 1737 604 1761 604C1792 604 1816 633 1816 671C1816 695 1801 712 1780 712ZM2171 330L2194 330C2202 395 2209 432 2218 458C2194 473 2159 482 2122 482C2077 483 2004 463 1947 400C1893 352 1854 241 1854 136C1854 40 1896 -11 1976 -11C2030 -11 2078 9 2133 54L2183 95L2175 115L2160 105C2088 57 2050 40 2015 40C1959 40 1930 80 1930 159C1930 267 1965 371 2014 409C2035 425 2059 433 2090 433C2135 433 2171 414 2171 390ZM2506 204L2477 77C2473 60 2471 42 2471 26C2471 4 2480 -9 2495 -9C2518 -9 2559 17 2641 85L2634 106C2610 86 2581 59 2559 59C2550 59 2544 68 2544 82C2544 87 2544 90 2545 93L2637 472L2627 481L2594 463C2553 478 2536 482 2509 482C2481 482 2461 477 2434 464C2372 433 2339 403 2314 354C2270 265 2239 145 2239 67C2239 23 2254 -11 2273 -11C2310 -11 2390 41 2506 204ZM2554 414C2532 305 2513 253 2479 201C2422 117 2361 59 2329 59C2317 59 2311 72 2311 99C2311 163 2339 280 2374 360C2398 415 2421 433 2469 433C2492 433 2510 429 2554 414Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017,0,0,-0.017,0,0);"></path></g><g><g><g><g style="transform:matrix(1,0,0,1,349.5666809082031,67.31665649414063);"><path d="M418 -3L418 27L366 30C311 33 301 44 301 96L301 700L60 598L67 548L217 614L217 96C217 44 206 33 152 30L96 27L96 -3C250 0 250 0 261 0C292 0 402 -3 418 -3Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.0119,0,0,-0.0119,0,0);"></path></g></g></g></g></g></g><g><g><g style="transform:matrix(1,0,0,1,423.5000305175781,65.81666564941406);"><path d="M368 365C371 403 376 435 384 476C373 481 369 482 364 482C333 482 302 458 266 407C227 351 188 291 172 256L204 408C208 425 210 438 210 450C210 470 202 482 187 482C166 482 128 461 54 408L26 388L33 368L65 389C93 407 104 412 113 412C123 412 130 403 130 390C130 332 87 126 47 -2L57 -9C72 -4 88 -1 111 4L124 6L150 126C168 209 191 262 235 319C269 363 296 384 318 384C333 384 343 379 354 365ZM716 111L692 94C639 56 591 36 555 36C508 36 479 73 479 133C479 158 482 185 487 214C504 218 613 248 638 259C723 296 762 342 762 404C762 451 728 482 678 482C610 496 500 423 463 349C433 299 403 180 403 113C403 35 447 -11 519 -11C576 -11 632 17 724 92ZM501 274C518 343 538 386 567 412C585 428 616 440 640 440C669 440 688 420 688 388C688 344 653 297 601 272C573 258 537 247 492 237ZM952 -11C1008 -11 1123 62 1166 125C1207 186 1241 296 1241 371C1241 438 1218 482 1182 482C1140 482 1087 456 1035 409C994 374 974 348 938 289L962 408C965 425 967 440 967 452C967 471 959 482 945 482C924 482 886 461 812 408L784 388L791 368L823 389C851 407 862 412 871 412C881 412 888 403 888 389C888 381 886 361 884 351L826 8C816 -52 797 -143 777 -233L769 -270L776 -276C797 -269 817 -264 849 -259L891 3C912 -4 934 -11 952 -11ZM919 165C941 293 1048 424 1131 424C1157 424 1169 402 1169 356C1169 275 1129 156 1076 80C1056 51 1024 36 983 36C952 36 927 43 901 59ZM1526 722L1514 733C1462 707 1426 698 1354 691L1350 670L1398 670C1422 670 1432 663 1432 648C1432 645 1432 640 1429 622C1418 567 1345 182 1330 132C1317 82 1311 52 1311 31C1311 6 1322 -9 1341 -9C1367 -9 1403 12 1501 85L1491 103L1465 86C1436 67 1414 56 1404 56C1397 56 1391 66 1391 76C1391 82 1392 89 1395 104ZM1586 388L1593 368L1625 389C1662 412 1665 414 1672 414C1682 414 1690 404 1690 391C1690 384 1686 361 1682 347L1616 107C1608 76 1603 49 1603 30C1603 6 1614 -9 1633 -9C1659 -9 1695 12 1793 85L1783 103L1757 86C1728 67 1705 56 1696 56C1689 56 1683 66 1683 76C1683 86 1685 95 1690 116L1767 420C1771 437 1773 448 1773 456C1773 473 1764 482 1748 482C1726 482 1689 461 1614 408ZM1780 712C1751 712 1722 679 1722 645C1722 620 1737 604 1761 604C1792 604 1816 633 1816 671C1816 695 1801 712 1780 712ZM2171 330L2194 330C2202 395 2209 432 2218 458C2194 473 2159 482 2122 482C2077 483 2004 463 1947 400C1893 352 1854 241 1854 136C1854 40 1896 -11 1976 -11C2030 -11 2078 9 2133 54L2183 95L2175 115L2160 105C2088 57 2050 40 2015 40C1959 40 1930 80 1930 159C1930 267 1965 371 2014 409C2035 425 2059 433 2090 433C2135 433 2171 414 2171 390ZM2506 204L2477 77C2473 60 2471 42 2471 26C2471 4 2480 -9 2495 -9C2518 -9 2559 17 2641 85L2634 106C2610 86 2581 59 2559 59C2550 59 2544 68 2544 82C2544 87 2544 90 2545 93L2637 472L2627 481L2594 463C2553 478 2536 482 2509 482C2481 482 2461 477 2434 464C2372 433 2339 403 2314 354C2270 265 2239 145 2239 67C2239 23 2254 -11 2273 -11C2310 -11 2390 41 2506 204ZM2554 414C2532 305 2513 253 2479 201C2422 117 2361 59 2329 59C2317 59 2311 72 2311 99C2311 163 2339 280 2374 360C2398 415 2421 433 2469 433C2492 433 2510 429 2554 414Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017,0,0,-0.017,0,0);"></path></g><g><g><g><g style="transform:matrix(1,0,0,1,469.5666809082031,68.81665649414063);"><path d="M16 23L16 -3C203 -3 203 0 239 0C275 0 275 -3 468 -3L468 82C353 77 307 81 122 77L304 270C401 373 431 428 431 503C431 618 353 689 226 689C154 689 105 669 56 619L39 483L68 483L81 529C97 587 133 612 200 612C286 612 341 558 341 473C341 398 299 324 186 204Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.0119,0,0,-0.0119,0,0);"></path></g></g></g></g></g></g><g><g><g style="transform:matrix(1,0,0,1,543.5000305175781,65.81666564941406);"><path d="M368 365C371 403 376 435 384 476C373 481 369 482 364 482C333 482 302 458 266 407C227 351 188 291 172 256L204 408C208 425 210 438 210 450C210 470 202 482 187 482C166 482 128 461 54 408L26 388L33 368L65 389C93 407 104 412 113 412C123 412 130 403 130 390C130 332 87 126 47 -2L57 -9C72 -4 88 -1 111 4L124 6L150 126C168 209 191 262 235 319C269 363 296 384 318 384C333 384 343 379 354 365ZM716 111L692 94C639 56 591 36 555 36C508 36 479 73 479 133C479 158 482 185 487 214C504 218 613 248 638 259C723 296 762 342 762 404C762 451 728 482 678 482C610 496 500 423 463 349C433 299 403 180 403 113C403 35 447 -11 519 -11C576 -11 632 17 724 92ZM501 274C518 343 538 386 567 412C585 428 616 440 640 440C669 440 688 420 688 388C688 344 653 297 601 272C573 258 537 247 492 237ZM952 -11C1008 -11 1123 62 1166 125C1207 186 1241 296 1241 371C1241 438 1218 482 1182 482C1140 482 1087 456 1035 409C994 374 974 348 938 289L962 408C965 425 967 440 967 452C967 471 959 482 945 482C924 482 886 461 812 408L784 388L791 368L823 389C851 407 862 412 871 412C881 412 888 403 888 389C888 381 886 361 884 351L826 8C816 -52 797 -143 777 -233L769 -270L776 -276C797 -269 817 -264 849 -259L891 3C912 -4 934 -11 952 -11ZM919 165C941 293 1048 424 1131 424C1157 424 1169 402 1169 356C1169 275 1129 156 1076 80C1056 51 1024 36 983 36C952 36 927 43 901 59ZM1526 722L1514 733C1462 707 1426 698 1354 691L1350 670L1398 670C1422 670 1432 663 1432 648C1432 645 1432 640 1429 622C1418 567 1345 182 1330 132C1317 82 1311 52 1311 31C1311 6 1322 -9 1341 -9C1367 -9 1403 12 1501 85L1491 103L1465 86C1436 67 1414 56 1404 56C1397 56 1391 66 1391 76C1391 82 1392 89 1395 104ZM1586 388L1593 368L1625 389C1662 412 1665 414 1672 414C1682 414 1690 404 1690 391C1690 384 1686 361 1682 347L1616 107C1608 76 1603 49 1603 30C1603 6 1614 -9 1633 -9C1659 -9 1695 12 1793 85L1783 103L1757 86C1728 67 1705 56 1696 56C1689 56 1683 66 1683 76C1683 86 1685 95 1690 116L1767 420C1771 437 1773 448 1773 456C1773 473 1764 482 1748 482C1726 482 1689 461 1614 408ZM1780 712C1751 712 1722 679 1722 645C1722 620 1737 604 1761 604C1792 604 1816 633 1816 671C1816 695 1801 712 1780 712ZM2171 330L2194 330C2202 395 2209 432 2218 458C2194 473 2159 482 2122 482C2077 483 2004 463 1947 400C1893 352 1854 241 1854 136C1854 40 1896 -11 1976 -11C2030 -11 2078 9 2133 54L2183 95L2175 115L2160 105C2088 57 2050 40 2015 40C1959 40 1930 80 1930 159C1930 267 1965 371 2014 409C2035 425 2059 433 2090 433C2135 433 2171 414 2171 390ZM2506 204L2477 77C2473 60 2471 42 2471 26C2471 4 2480 -9 2495 -9C2518 -9 2559 17 2641 85L2634 106C2610 86 2581 59 2559 59C2550 59 2544 68 2544 82C2544 87 2544 90 2545 93L2637 472L2627 481L2594 463C2553 478 2536 482 2509 482C2481 482 2461 477 2434 464C2372 433 2339 403 2314 354C2270 265 2239 145 2239 67C2239 23 2254 -11 2273 -11C2310 -11 2390 41 2506 204ZM2554 414C2532 305 2513 253 2479 201C2422 117 2361 59 2329 59C2317 59 2311 72 2311 99C2311 163 2339 280 2374 360C2398 415 2421 433 2469 433C2492 433 2510 429 2554 414Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017,0,0,-0.017,0,0);"></path></g><g><g><g><g style="transform:matrix(1,0,0,1,589.5666809082031,68.81665649414063);"><path d="M462 224C462 345 355 366 308 374C388 436 418 482 418 541C418 630 344 689 233 689C165 689 120 670 72 622L43 498L74 498L92 554C103 588 166 622 218 622C283 622 336 569 336 506C336 431 277 368 206 368C198 368 187 369 174 370L159 371L147 318L154 312C192 329 211 334 238 334C321 334 369 281 369 190C369 88 308 21 215 21C169 21 128 36 98 64C74 86 61 109 42 163L15 153C36 92 44 56 50 6C103 -12 147 -20 184 -20C307 -20 462 87 462 224Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.0119,0,0,-0.0119,0,0);"></path></g></g></g></g></g></g><g><g><g><g><g><g style="transform:matrix(1,0,0,1,461.8333435058594,134.81666564941406);"><path d="M157 214C157 314 229 386 327 388L327 455C238 454 183 405 152 359L152 450L82 450L82 0L157 0ZM754 219C754 421 647 461 576 461C465 461 375 355 375 226C375 94 471 -11 591 -11C654 -11 711 13 750 41L744 106C681 54 615 50 592 50C512 50 448 121 445 219ZM450 274C466 350 519 400 576 400C628 400 684 366 697 274ZM1143 128C1143 183 1106 217 1104 220C1065 255 1038 261 988 270C933 281 887 291 887 340C887 402 959 402 972 402C1004 402 1057 398 1114 364L1126 429C1074 453 1033 461 982 461C957 461 816 461 816 330C816 281 845 249 870 230C901 208 923 204 978 193C1014 186 1072 174 1072 121C1072 52 993 52 978 52C897 52 841 89 823 101L811 33C843 17 898 -11 979 -11C1117 -11 1143 75 1143 128ZM1340 386L1481 386L1481 444L1340 444L1340 571L1271 571L1271 444L1184 444L1184 386L1268 386L1268 119C1268 59 1282 -11 1351 -11C1421 -11 1472 14 1497 27L1481 86C1455 65 1423 53 1391 53C1354 53 1340 83 1340 136ZM1924 289C1924 391 1851 461 1759 461C1694 461 1649 445 1602 418L1608 352C1660 389 1710 402 1759 402C1806 402 1846 362 1846 288L1846 245C1696 243 1569 201 1569 113C1569 70 1596 -11 1683 -11C1697 -11 1791 -9 1849 36L1849 0L1924 0ZM1846 132C1846 113 1846 88 1812 69C1783 51 1745 50 1734 50C1686 50 1641 73 1641 115C1641 185 1803 192 1846 194ZM2161 214C2161 314 2233 386 2331 388L2331 455C2242 454 2187 405 2156 359L2156 450L2086 450L2086 0L2161 0ZM2519 386L2660 386L2660 444L2519 444L2519 571L2450 571L2450 444L2363 444L2363 386L2447 386L2447 119C2447 59 2461 -11 2530 -11C2600 -11 2651 14 2676 27L2660 86C2634 65 2602 53 2570 53C2533 53 2519 83 2519 136Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017,0,0,-0.017,0,0);"></path></g><g style="transform:matrix(1,0,0,1,507.8166809082031,134.81666564941406);"><path d="M146 266C146 526 243 632 301 700L282 726C225 675 60 542 60 266C60 159 85 58 133 -32C168 -99 200 -138 282 -215L301 -194C255 -137 146 -15 146 266Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017,0,0,-0.017,0,0);"></path></g><g style="transform:matrix(1,0,0,1,513.4667053222656,134.81666564941406);"><path d="M368 365C371 403 376 435 384 476C373 481 369 482 364 482C333 482 302 458 266 407C227 351 188 291 172 256L204 408C208 425 210 438 210 450C210 470 202 482 187 482C166 482 128 461 54 408L26 388L33 368L65 389C93 407 104 412 113 412C123 412 130 403 130 390C130 332 87 126 47 -2L57 -9C72 -4 88 -1 111 4L124 6L150 126C168 209 191 262 235 319C269 363 296 384 318 384C333 384 343 379 354 365Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017,0,0,-0.017,0,0);"></path></g><g><g><g><g style="transform:matrix(1,0,0,1,520.5833435058594,137.81665649414063);"><path d="M16 23L16 -3C203 -3 203 0 239 0C275 0 275 -3 468 -3L468 82C353 77 307 81 122 77L304 270C401 373 431 428 431 503C431 618 353 689 226 689C154 689 105 669 56 619L39 483L68 483L81 529C97 587 133 612 200 612C286 612 341 558 341 473C341 398 299 324 186 204Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.0119,0,0,-0.0119,0,0);"></path></g></g></g></g><g style="transform:matrix(1,0,0,1,527.5333557128906,134.81666564941406);"><path d="M51 726L32 700C87 636 187 526 187 266C187 -10 83 -131 32 -194L51 -215C104 -165 273 -23 273 265C273 542 108 675 51 726Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017,0,0,-0.017,0,0);"></path></g></g></g></g></g></g><g><g><g style="transform:matrix(1,0,0,1,191.48330688476562,63.29998779296875);"><path d="M342 330L365 330C373 395 380 432 389 458C365 473 330 482 293 482C248 483 175 463 118 400C64 352 25 241 25 136C25 40 67 -11 147 -11C201 -11 249 9 304 54L354 95L346 115L331 105C259 57 221 40 186 40C130 40 101 80 101 159C101 267 136 371 185 409C206 425 230 433 261 433C306 433 342 414 342 390ZM423 152C423 46 468 -11 551 -11C606 -11 666 15 711 57C773 116 817 230 817 331C817 425 767 482 684 482C580 482 423 382 423 152ZM647 444C708 444 741 399 741 315C741 219 710 113 666 60C648 39 622 27 591 27C533 27 499 72 499 151C499 264 538 387 587 427C600 438 623 444 647 444ZM866 152C866 46 911 -11 994 -11C1049 -11 1109 15 1154 57C1216 116 1260 230 1260 331C1260 425 1210 482 1127 482C1023 482 866 382 866 152ZM1090 444C1151 444 1184 399 1184 315C1184 219 1153 113 1109 60C1091 39 1065 27 1034 27C976 27 942 72 942 151C942 264 981 387 1030 427C1043 438 1066 444 1090 444ZM1660 365C1663 403 1668 435 1676 476C1665 481 1661 482 1656 482C1625 482 1594 458 1558 407C1519 351 1480 291 1464 256L1496 408C1500 425 1502 438 1502 450C1502 470 1494 482 1479 482C1458 482 1420 461 1346 408L1318 388L1325 368L1357 389C1385 407 1396 412 1405 412C1415 412 1422 403 1422 390C1422 332 1379 126 1339 -2L1349 -9C1364 -4 1380 -1 1403 4L1416 6L1442 126C1460 209 1483 262 1527 319C1561 363 1588 384 1610 384C1625 384 1635 379 1646 365ZM2163 722L2151 733C2099 707 2063 698 1991 691L1987 670L2035 670C2059 670 2069 663 2069 646C2069 638 2068 629 2067 622L2039 468C2009 477 1982 482 1957 482C1888 482 1794 410 1748 323C1717 265 1697 170 1697 86C1697 21 1712 -11 1741 -11C1768 -11 1805 6 1839 33C1893 77 1926 116 1993 217L1970 126C1959 82 1954 50 1954 24C1954 3 1963 -9 1980 -9C1997 -9 2021 3 2055 28L2136 88L2126 107L2082 76C2068 66 2052 59 2043 59C2035 59 2029 68 2029 82C2029 90 2030 99 2037 128ZM1794 59C1778 59 1769 73 1769 98C1769 224 1815 380 1864 418C1877 428 1896 433 1923 433C1967 433 1996 427 2029 410L2017 351C1980 171 1839 59 1794 59Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017,0,0,-0.017,0,0);"></path></g></g></g><g><g><g><g><g><g style="transform:matrix(1,0,0,1,79.63333129882812,386.40000915527344);"><path d="M125 390L69 107C68 99 56 61 56 31C56 6 67 -9 86 -9C121 -9 156 11 234 74L265 99L255 117L210 86C181 66 161 56 150 56C141 56 136 64 136 76C136 102 150 183 179 328L192 390L299 390L310 440C272 436 238 434 200 434C216 528 227 577 245 631L234 646C214 634 187 622 156 610L131 440C87 419 61 408 43 403L41 390Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017,0,0,-0.017,0,0);"></path></g></g></g></g></g></g><g><g><g><g><g><g style="transform:matrix(1,0,0,1,231.00003051757812,115.31666564941406);"><path d="M260 229L443 444L361 444L227 279L89 444L6 444L194 229L0 0L82 0L227 188L377 0L460 0Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017,0,0,-0.017,0,0);"></path></g><g><g style="transform:matrix(1,0,0,1,246.44998168945312,115.31666564941406);"><path d="M949 241L949 300L179 300L304 452L272 486L65 269L272 55L304 89L179 241Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017,0,0,-0.017,0,0);"></path></g></g><g style="transform:matrix(1,0,0,1,267.0666809082031,115.31666564941406);"><path d="M299 689L279 689C220 627 137 624 89 622L89 563C122 564 170 566 220 587L220 59L95 59L95 0L424 0L424 59L299 59Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017,0,0,-0.017,0,0);"></path></g><g style="transform:matrix(1,0,0,1,275.5500183105469,115.31666564941406);"><path d="M204 123C177 114 159 108 106 93C99 17 74 -48 16 -144L30 -155L71 -136C152 -31 190 32 218 109Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017,0,0,-0.017,0,0);"></path></g><g style="transform:matrix(1,0,0,1,283.1833190917969,115.31666564941406);"><path d="M424 386L565 386L565 444L424 444L424 571L355 571L355 444L268 444L268 386L352 386L352 119C352 59 366 -11 435 -11C505 -11 556 14 581 27L565 86C539 65 507 53 475 53C438 53 424 83 424 136Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017,0,0,-0.017,0,0);"></path></g><g><g><g><g style="transform:matrix(1,0,0,1,294.0500183105469,118.31665649414063);"><path d="M457 331C457 412 453 506 411 588C370 665 301 689 250 689C191 689 121 662 80 571C47 497 42 413 42 331C42 251 46 177 76 103C116 5 192 -22 249 -22C322 -22 385 19 417 89C447 155 457 223 457 331ZM250 40C198 40 157 78 137 151C121 209 120 264 120 343C120 407 120 468 137 524C143 544 168 627 249 627C327 627 353 550 360 531C379 475 379 406 379 343C379 276 379 212 361 148C335 56 282 40 250 40Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.0119,0,0,-0.0119,0,0);"></path></g></g></g></g></g></g></g></g></g><g><g><g><g><g><g style="transform:matrix(1,0,0,1,250.25003051757812,184.81666564941406);"><path d="M399 289C399 391 326 461 234 461C169 461 124 445 77 418L83 352C135 389 185 402 234 402C281 402 321 362 321 288L321 245C171 243 44 201 44 113C44 70 71 -11 158 -11C172 -11 266 -9 324 36L324 0L399 0ZM321 132C321 113 321 88 287 69C258 51 220 50 209 50C161 50 116 73 116 115C116 185 278 192 321 194ZM889 418C830 452 796 461 735 461C596 461 515 340 515 222C515 98 606 -11 731 -11C785 -11 840 3 894 40L888 107C837 67 783 53 732 53C649 53 593 125 593 223C593 301 630 397 736 397C788 397 822 389 877 353ZM1203 272L1371 444L1281 444L1078 236L1078 694L1006 694L1006 0L1075 0L1075 141L1155 224L1311 0L1393 0Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017,0,0,-0.017,0,0);"></path></g><g style="transform:matrix(1,0,0,1,274.2167053222656,184.81666564941406);"><path d="M146 266C146 526 243 632 301 700L282 726C225 675 60 542 60 266C60 159 85 58 133 -32C168 -99 200 -138 282 -215L301 -194C255 -137 146 -15 146 266Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017,0,0,-0.017,0,0);"></path></g><g style="transform:matrix(1,0,0,1,279.8666687011719,184.81666564941406);"><path d="M157 214C157 314 229 386 327 388L327 455C238 454 183 405 152 359L152 450L82 450L82 0L157 0Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017,0,0,-0.017,0,0);"></path></g><g><g><g><g style="transform:matrix(1,0,0,1,286.1666564941406,187.81665649414063);"><path d="M299 689L279 689C220 627 137 624 89 622L89 563C122 564 170 566 220 587L220 59L95 59L95 0L424 0L424 59L299 59Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.0119,0,0,-0.0119,0,0);"></path></g></g></g></g><g style="transform:matrix(1,0,0,1,293.1166687011719,184.81666564941406);"><path d="M51 726L32 700C87 636 187 526 187 266C187 -10 83 -131 32 -194L51 -215C104 -165 273 -23 273 265C273 542 108 675 51 726Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017,0,0,-0.017,0,0);"></path></g></g></g></g></g></g><g><g><g><g><g><g style="transform:matrix(1,0,0,1,479.7500305175781,194.81666564941406);"><path d="M399 289C399 391 326 461 234 461C169 461 124 445 77 418L83 352C135 389 185 402 234 402C281 402 321 362 321 288L321 245C171 243 44 201 44 113C44 70 71 -11 158 -11C172 -11 266 -9 324 36L324 0L399 0ZM321 132C321 113 321 88 287 69C258 51 220 50 209 50C161 50 116 73 116 115C116 185 278 192 321 194ZM889 418C830 452 796 461 735 461C596 461 515 340 515 222C515 98 606 -11 731 -11C785 -11 840 3 894 40L888 107C837 67 783 53 732 53C649 53 593 125 593 223C593 301 630 397 736 397C788 397 822 389 877 353ZM1203 272L1371 444L1281 444L1078 236L1078 694L1006 694L1006 0L1075 0L1075 141L1155 224L1311 0L1393 0Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017,0,0,-0.017,0,0);"></path></g><g style="transform:matrix(1,0,0,1,503.7167053222656,194.81666564941406);"><path d="M146 266C146 526 243 632 301 700L282 726C225 675 60 542 60 266C60 159 85 58 133 -32C168 -99 200 -138 282 -215L301 -194C255 -137 146 -15 146 266Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017,0,0,-0.017,0,0);"></path></g><g style="transform:matrix(1,0,0,1,509.3666687011719,194.81666564941406);"><path d="M157 214C157 314 229 386 327 388L327 455C238 454 183 405 152 359L152 450L82 450L82 0L157 0Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017,0,0,-0.017,0,0);"></path></g><g><g><g><g style="transform:matrix(1,0,0,1,515.6666564941406,197.8166717529297);"><path d="M92 522C121 593 186 630 247 630C299 630 348 600 348 535C348 473 307 413 246 398C240 397 238 397 167 391L167 329L238 329C346 329 368 235 368 184C368 105 322 40 245 40C176 40 97 75 53 144L42 83C115 -12 207 -22 247 -22C369 -22 457 76 457 183C457 275 387 338 319 360C395 401 430 471 430 535C430 622 347 689 248 689C171 689 98 648 56 577Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.0119,0,0,-0.0119,0,0);"></path></g></g></g></g><g style="transform:matrix(1,0,0,1,522.6166687011719,194.81666564941406);"><path d="M51 726L32 700C87 636 187 526 187 266C187 -10 83 -131 32 -194L51 -215C104 -165 273 -23 273 265C273 542 108 675 51 726Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017,0,0,-0.017,0,0);"></path></g></g></g></g></g></g><g><g><g><g><g><g style="transform:matrix(1,0,0,1,81.03335571289062,182.5);"><path d="M399 289C399 391 326 461 234 461C169 461 124 445 77 418L83 352C135 389 185 402 234 402C281 402 321 362 321 288L321 245C171 243 44 201 44 113C44 70 71 -11 158 -11C172 -11 266 -9 324 36L324 0L399 0ZM321 132C321 113 321 88 287 69C258 51 220 50 209 50C161 50 116 73 116 115C116 185 278 192 321 194ZM889 418C830 452 796 461 735 461C596 461 515 340 515 222C515 98 606 -11 731 -11C785 -11 840 3 894 40L888 107C837 67 783 53 732 53C649 53 593 125 593 223C593 301 630 397 736 397C788 397 822 389 877 353ZM1203 272L1371 444L1281 444L1078 236L1078 694L1006 694L1006 0L1075 0L1075 141L1155 224L1311 0L1393 0Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017,0,0,-0.017,0,0);"></path></g><g><g><g><g style="transform:matrix(1,0,0,1,105.51669311523438,185.44998779296876);"><path d="M434 455L359 455L359 387C349 400 300 455 222 455C122 455 36 356 36 221C36 94 109 -11 205 -11C261 -11 314 12 356 50L356 -194L434 -194ZM359 140C359 122 359 120 349 107C323 67 286 50 250 50C174 50 114 128 114 221C114 323 186 391 259 391C328 391 359 320 359 280ZM950 444L872 444L872 154C872 79 816 44 752 44C681 44 674 70 674 113L674 444L596 444L596 109C596 37 619 -11 702 -11C755 -11 826 5 875 48L875 0L950 0ZM1499 220C1499 354 1399 461 1280 461C1157 461 1060 351 1060 220C1060 88 1162 -11 1279 -11C1399 -11 1499 90 1499 220ZM1279 53C1210 53 1138 109 1138 230C1138 351 1214 400 1279 400C1349 400 1421 348 1421 230C1421 112 1353 53 1279 53ZM1686 214C1686 314 1758 386 1856 388L1856 455C1767 454 1712 405 1681 359L1681 450L1611 450L1611 0L1686 0ZM2304 444L2226 444L2226 154C2226 79 2170 44 2106 44C2035 44 2028 70 2028 113L2028 444L1950 444L1950 109C1950 37 1973 -11 2056 -11C2109 -11 2180 5 2229 48L2229 0L2304 0ZM3097 298C3097 365 3081 455 2960 455C2900 455 2848 427 2811 373C2785 449 2715 455 2683 455C2611 455 2564 414 2537 378L2537 450L2465 450L2465 0L2543 0L2543 245C2543 313 2570 394 2644 394C2737 394 2742 329 2742 291L2742 0L2820 0L2820 245C2820 313 2847 394 2921 394C3014 394 3019 329 3019 291L3019 0L3097 0Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.0119,0,0,-0.0119,0,0);"></path></g></g></g></g><g><svg x="145.88333129882812" style="overflow:visible;" y="164.5" height="24" width="8.5"><path d=" M 6.61 1.70 q 0.00 -0.08 -0.08 -0.08 q -0.03 0.00 -0.08 0.02 q -1.01 0.55 -1.75 1.30 t -1.37 1.87 t -0.95 2.78 t -0.33 3.79 v 0.62 h 1.68 v -0.62 q 0.00 -1.20 0.04 -2.08 t 0.22 -2.04 t 0.50 -2.03 t 0.91 -1.74 t 1.43 -1.49 q 0.12 -0.09 0.12 -0.29 z M 0.90 12.00 v 0.00 h 1.68 v 0.00 z" style="fill:rgb(0, 0, 0);stroke-width:1px;stroke:none;"></path><path d=" M 6.61 22.30 q 0.00 0.08 -0.08 0.08 q -0.03 0.00 -0.08 -0.02 q -1.01 -0.55 -1.75 -1.30 t -1.37 -1.87 t -0.95 -2.78 t -0.33 -3.79 v -0.62 h 1.68 v 0.62 q 0.00 1.20 0.04 2.08 t 0.22 2.04 t 0.50 2.03 t 0.91 1.74 t 1.43 1.49 q 0.12 0.09 0.12 0.29 z" style="fill:rgb(0, 0, 0);stroke-width:1px;stroke:none;"></path></svg></g><g style="transform:matrix(1,0,0,1,152.68331909179688,182.5);"><path d="M260 229L443 444L361 444L227 279L89 444L6 444L194 229L0 0L82 0L227 188L377 0L460 0Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017,0,0,-0.017,0,0);"></path></g><g><g style="transform:matrix(1,0,0,1,163.89999389648438,182.5);"><path d="M949 241L949 300L179 300L304 452L272 486L65 269L272 55L304 89L179 241Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017,0,0,-0.017,0,0);"></path></g></g><g style="transform:matrix(1,0,0,1,184.51669311523438,182.5);"><path d="M299 689L279 689C220 627 137 624 89 622L89 563C122 564 170 566 220 587L220 59L95 59L95 0L424 0L424 59L299 59Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017,0,0,-0.017,0,0);"></path></g><g><svg x="191.30001831054688" style="overflow:visible;" y="164.5" height="24" width="8.5"><path d=" M 1.69 1.70 q 0.00 -0.08 0.08 -0.08 q 0.03 0.00 0.08 0.02 q 1.01 0.55 1.75 1.30 t 1.37 1.87 t 0.95 2.78 t 0.33 3.79 v 0.62 h -1.68 v -0.62 q 0.00 -1.20 -0.04 -2.08 t -0.22 -2.04 t -0.50 -2.03 t -0.91 -1.74 t -1.43 -1.49 q -0.12 -0.09 -0.12 -0.29 z M 7.40 12.00 v 0.00 h -1.68 v 0.00 z" style="fill:rgb(0, 0, 0);stroke-width:1px;stroke:none;"></path><path d=" M 1.69 22.30 q 0.00 0.08 0.08 0.08 q 0.03 0.00 0.08 -0.02 q 1.01 -0.55 1.75 -1.30 t 1.37 -1.87 t 0.95 -2.78 t 0.33 -3.79 v -0.62 h -1.68 v 0.62 q 0.00 1.20 -0.04 2.08 t -0.22 2.04 t -0.50 2.03 t -0.91 1.74 t -1.43 1.49 q -0.12 0.09 -0.12 0.29 z" style="fill:rgb(0, 0, 0);stroke-width:1px;stroke:none;"></path></svg></g></g></g></g></g></g><g><g><g><g><g><g style="transform:matrix(1,0,0,1,35.45001220703125,116.81666564941406);"><path d="M125 390L69 107C68 99 56 61 56 31C56 6 67 -9 86 -9C121 -9 156 11 234 74L265 99L255 117L210 86C181 66 161 56 150 56C141 56 136 64 136 76C136 102 150 183 179 328L192 390L299 390L310 440C272 436 238 434 200 434C216 528 227 577 245 631L234 646C214 634 187 622 156 610L131 440C87 419 61 408 43 403L41 390Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017,0,0,-0.017,0,0);"></path></g><g><g><g><g style="transform:matrix(1,0,0,1,41.616668701171875,119.81665649414063);"><path d="M263 689C108 689 29 566 29 324C29 207 50 106 85 57C120 8 176 -20 238 -20C389 -20 465 110 465 366C465 585 400 689 263 689ZM245 654C342 654 381 556 381 316C381 103 343 15 251 15C154 15 113 116 113 360C113 571 150 654 245 654Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.0119,0,0,-0.0119,0,0);"></path></g></g></g></g></g></g></g></g></g><g><g><g><g><g><g style="transform:matrix(1,0,0,1,35.45001220703125,206.81666564941406);"><path d="M125 390L69 107C68 99 56 61 56 31C56 6 67 -9 86 -9C121 -9 156 11 234 74L265 99L255 117L210 86C181 66 161 56 150 56C141 56 136 64 136 76C136 102 150 183 179 328L192 390L299 390L310 440C272 436 238 434 200 434C216 528 227 577 245 631L234 646C214 634 187 622 156 610L131 440C87 419 61 408 43 403L41 390Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017,0,0,-0.017,0,0);"></path></g><g><g><g><g style="transform:matrix(1,0,0,1,41.616668701171875,209.8166717529297);"><path d="M418 -3L418 27L366 30C311 33 301 44 301 96L301 700L60 598L67 548L217 614L217 96C217 44 206 33 152 30L96 27L96 -3C250 0 250 0 261 0C292 0 402 -3 418 -3Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.0119,0,0,-0.0119,0,0);"></path></g></g></g></g></g></g></g></g></g><g><g><g><g><g><g style="transform:matrix(1,0,0,1,35.45001220703125,256.81666564941406);"><path d="M125 390L69 107C68 99 56 61 56 31C56 6 67 -9 86 -9C121 -9 156 11 234 74L265 99L255 117L210 86C181 66 161 56 150 56C141 56 136 64 136 76C136 102 150 183 179 328L192 390L299 390L310 440C272 436 238 434 200 434C216 528 227 577 245 631L234 646C214 634 187 622 156 610L131 440C87 419 61 408 43 403L41 390Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017,0,0,-0.017,0,0);"></path></g><g><g><g><g style="transform:matrix(1,0,0,1,41.616668701171875,259.81667175292966);"><path d="M16 23L16 -3C203 -3 203 0 239 0C275 0 275 -3 468 -3L468 82C353 77 307 81 122 77L304 270C401 373 431 428 431 503C431 618 353 689 226 689C154 689 105 669 56 619L39 483L68 483L81 529C97 587 133 612 200 612C286 612 341 558 341 473C341 398 299 324 186 204Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.0119,0,0,-0.0119,0,0);"></path></g></g></g></g></g></g></g></g></g><g><g><g><g><g><g style="transform:matrix(1,0,0,1,35.45001220703125,336.81666564941406);"><path d="M125 390L69 107C68 99 56 61 56 31C56 6 67 -9 86 -9C121 -9 156 11 234 74L265 99L255 117L210 86C181 66 161 56 150 56C141 56 136 64 136 76C136 102 150 183 179 328L192 390L299 390L310 440C272 436 238 434 200 434C216 528 227 577 245 631L234 646C214 634 187 622 156 610L131 440C87 419 61 408 43 403L41 390Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017,0,0,-0.017,0,0);"></path></g><g><g><g><g style="transform:matrix(1,0,0,1,41.616668701171875,339.81667175292966);"><path d="M462 224C462 345 355 366 308 374C388 436 418 482 418 541C418 630 344 689 233 689C165 689 120 670 72 622L43 498L74 498L92 554C103 588 166 622 218 622C283 622 336 569 336 506C336 431 277 368 206 368C198 368 187 369 174 370L159 371L147 318L154 312C192 329 211 334 238 334C321 334 369 281 369 190C369 88 308 21 215 21C169 21 128 36 98 64C74 86 61 109 42 163L15 153C36 92 44 56 50 6C103 -12 147 -20 184 -20C307 -20 462 87 462 224Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.0119,0,0,-0.0119,0,0);"></path></g></g></g></g></g></g></g></g></g><g><g><g><g><g><g style="transform:matrix(1,0,0,1,217.80001831054688,249.49998474121094);"><path d="M435 298C435 364 420 455 298 455C236 455 188 424 156 383L156 694L81 694L81 0L159 0L159 245C159 311 184 394 260 394C356 394 357 323 357 291L357 0L435 0ZM678 680L589 680L589 591L678 591ZM671 444L596 444L596 0L671 0ZM1187 298C1187 364 1172 455 1050 455C960 455 911 387 905 379L905 450L833 450L833 0L911 0L911 245C911 311 936 394 1012 394C1108 394 1109 323 1109 291L1109 0L1187 0ZM1442 386L1583 386L1583 444L1442 444L1442 571L1373 571L1373 444L1286 444L1286 386L1370 386L1370 119C1370 59 1384 -11 1453 -11C1523 -11 1574 14 1599 27L1583 86C1557 65 1525 53 1493 53C1456 53 1442 83 1442 136Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017,0,0,-0.017,0,0);"></path></g><g><svg x="246.98330688476562" style="overflow:visible;" y="231.49998474121094" height="26" width="8.5"><path d=" M 6.61 1.70 q 0.00 -0.09 -0.08 -0.09 q -0.03 0.00 -0.08 0.03 q -1.01 0.61 -1.75 1.42 t -1.37 2.06 t -0.95 3.05 t -0.33 4.16 v 0.68 h 1.68 v -0.68 q 0.00 -1.31 0.04 -2.28 t 0.22 -2.24 t 0.50 -2.23 t 0.91 -1.91 t 1.43 -1.63 q 0.12 -0.10 0.12 -0.32 z M 0.90 13.00 v 0.00 h 1.68 v 0.00 z" style="fill:rgb(0, 0, 0);stroke-width:1px;stroke:none;"></path><path d=" M 6.61 24.30 q 0.00 0.09 -0.08 0.09 q -0.03 0.00 -0.08 -0.03 q -1.01 -0.61 -1.75 -1.42 t -1.37 -2.06 t -0.95 -3.05 t -0.33 -4.16 v -0.68 h 1.68 v 0.68 q 0.00 1.31 0.04 2.28 t 0.22 2.24 t 0.50 2.23 t 0.91 1.91 t 1.43 1.63 q 0.12 0.10 0.12 0.32 z" style="fill:rgb(0, 0, 0);stroke-width:1px;stroke:none;"></path></svg></g><g style="transform:matrix(1,0,0,1,253.78335571289062,249.49998474121094);"><path d="M157 214C157 314 229 386 327 388L327 455C238 454 183 405 152 359L152 450L82 450L82 0L157 0Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017,0,0,-0.017,0,0);"></path></g><g><g><g><g style="transform:matrix(1,0,0,1,260.0833435058594,252.49999084472657);"><path d="M83 466C103 545 131 624 222 624C316 624 367 548 367 468C367 382 310 324 251 263L174 191L50 65L50 0L449 0L449 72L267 72C255 72 243 71 231 71L122 71C154 100 230 176 261 205C333 274 449 347 449 471C449 587 368 689 236 689C122 689 66 610 42 522C66 487 59 501 83 466Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.0119,0,0,-0.0119,0,0);"></path></g></g></g></g><g style="transform:matrix(1,0,0,1,267.0333557128906,249.49998474121094);"><path d="M204 123C177 114 159 108 106 93C99 17 74 -48 16 -144L30 -155L71 -136C152 -31 190 32 218 109Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017,0,0,-0.017,0,0);"></path></g><g style="transform:matrix(1,0,0,1,274.6666564941406,249.49998474121094);"><path d="M509 229L692 444L610 444L476 279L338 444L255 444L443 229L249 0L331 0L476 188L626 0L709 0Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017,0,0,-0.017,0,0);"></path></g><g><g style="transform:matrix(1,0,0,1,290.1166687011719,249.49998474121094);"><path d="M949 241L949 300L179 300L304 452L272 486L65 269L272 55L304 89L179 241Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017,0,0,-0.017,0,0);"></path></g></g><g style="transform:matrix(1,0,0,1,310.7333068847656,249.49998474121094);"><path d="M299 689L279 689C220 627 137 624 89 622L89 563C122 564 170 566 220 587L220 59L95 59L95 0L424 0L424 59L299 59Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017,0,0,-0.017,0,0);"></path></g><g><svg x="317.5166931152344" style="overflow:visible;" y="231.49998474121094" height="26" width="8.5"><path d=" M 1.69 1.70 q 0.00 -0.09 0.08 -0.09 q 0.03 0.00 0.08 0.03 q 1.01 0.61 1.75 1.42 t 1.37 2.06 t 0.95 3.05 t 0.33 4.16 v 0.68 h -1.68 v -0.68 q 0.00 -1.31 -0.04 -2.28 t -0.22 -2.24 t -0.50 -2.23 t -0.91 -1.91 t -1.43 -1.63 q -0.12 -0.10 -0.12 -0.32 z M 7.40 13.00 v 0.00 h -1.68 v 0.00 z" style="fill:rgb(0, 0, 0);stroke-width:1px;stroke:none;"></path><path d=" M 1.69 24.30 q 0.00 0.09 0.08 0.09 q 0.03 0.00 0.08 -0.03 q 1.01 -0.61 1.75 -1.42 t 1.37 -2.06 t 0.95 -3.05 t 0.33 -4.16 v -0.68 h -1.68 v 0.68 q 0.00 1.31 -0.04 2.28 t -0.22 2.24 t -0.50 2.23 t -0.91 1.91 t -1.43 1.63 q -0.12 0.10 -0.12 0.32 z" style="fill:rgb(0, 0, 0);stroke-width:1px;stroke:none;"></path></svg></g></g></g></g></g></g><g><g><g><g><g><g style="transform:matrix(1,0,0,1,351.9499816894531,316.81666564941406);"><path d="M399 289C399 391 326 461 234 461C169 461 124 445 77 418L83 352C135 389 185 402 234 402C281 402 321 362 321 288L321 245C171 243 44 201 44 113C44 70 71 -11 158 -11C172 -11 266 -9 324 36L324 0L399 0ZM321 132C321 113 321 88 287 69C258 51 220 50 209 50C161 50 116 73 116 115C116 185 278 192 321 194ZM635 694L560 694L560 0L635 0ZM879 680L790 680L790 591L879 591ZM872 444L797 444L797 0L872 0ZM1399 444L1324 444C1272 299 1181 47 1185 53L1184 53L1045 444L967 444L1139 0L1227 0ZM1827 219C1827 421 1720 461 1649 461C1538 461 1448 355 1448 226C1448 94 1544 -11 1664 -11C1727 -11 1784 13 1823 41L1817 106C1754 54 1688 50 1665 50C1585 50 1521 121 1518 219ZM1523 274C1539 350 1592 400 1649 400C1701 400 1757 366 1770 274Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017,0,0,-0.017,0,0);"></path></g><g style="transform:matrix(1,0,0,1,383.5166931152344,316.81666564941406);"><path d="M146 266C146 526 243 632 301 700L282 726C225 675 60 542 60 266C60 159 85 58 133 -32C168 -99 200 -138 282 -215L301 -194C255 -137 146 -15 146 266Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017,0,0,-0.017,0,0);"></path></g><g style="transform:matrix(1,0,0,1,389.1666564941406,316.81666564941406);"><path d="M157 214C157 314 229 386 327 388L327 455C238 454 183 405 152 359L152 450L82 450L82 0L157 0Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017,0,0,-0.017,0,0);"></path></g><g><g><g><g style="transform:matrix(1,0,0,1,395.4667053222656,319.81667175292966);"><path d="M83 466C103 545 131 624 222 624C316 624 367 548 367 468C367 382 310 324 251 263L174 191L50 65L50 0L449 0L449 72L267 72C255 72 243 71 231 71L122 71C154 100 230 176 261 205C333 274 449 347 449 471C449 587 368 689 236 689C122 689 66 610 42 522C66 487 59 501 83 466Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.0119,0,0,-0.0119,0,0);"></path></g></g></g></g><g style="transform:matrix(1,0,0,1,402.4166564941406,316.81666564941406);"><path d="M51 726L32 700C87 636 187 526 187 266C187 -10 83 -131 32 -194L51 -215C104 -165 273 -23 273 265C273 542 108 675 51 726Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017,0,0,-0.017,0,0);"></path></g></g></g></g></g></g><g><g><g><g><g><g style="transform:matrix(1,0,0,1,35.45001220703125,366.81666564941406);"><path d="M125 390L69 107C68 99 56 61 56 31C56 6 67 -9 86 -9C121 -9 156 11 234 74L265 99L255 117L210 86C181 66 161 56 150 56C141 56 136 64 136 76C136 102 150 183 179 328L192 390L299 390L310 440C272 436 238 434 200 434C216 528 227 577 245 631L234 646C214 634 187 622 156 610L131 440C87 419 61 408 43 403L41 390Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017,0,0,-0.017,0,0);"></path></g><g><g><g><g style="transform:matrix(1,0,0,1,41.616668701171875,369.81667175292966);"><path d="M280 181L280 106C280 46 269 32 220 30L158 27L158 -3C291 0 291 0 315 0C339 0 339 0 472 -3L472 27L424 30C375 33 364 46 364 106L364 181C423 181 444 180 472 177L472 248L364 245L365 697L285 667L2 204L2 181ZM280 245L65 245L280 597Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.0119,0,0,-0.0119,0,0);"></path></g></g></g></g></g></g></g></g></g><g><g><g><g><g><g style="transform:matrix(1,0,0,1,231.50003051757812,134.81666564941406);"><path d="M260 229L443 444L361 444L227 279L89 444L6 444L194 229L0 0L82 0L227 188L377 0L460 0Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017,0,0,-0.017,0,0);"></path></g><g><g style="transform:matrix(1,0,0,1,246.94998168945312,134.81666564941406);"><path d="M949 241L949 300L179 300L304 452L272 486L65 269L272 55L304 89L179 241Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017,0,0,-0.017,0,0);"></path></g></g><g style="transform:matrix(1,0,0,1,267.5666809082031,134.81666564941406);"><path d="M299 689L279 689C220 627 137 624 89 622L89 563C122 564 170 566 220 587L220 59L95 59L95 0L424 0L424 59L299 59Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017,0,0,-0.017,0,0);"></path></g><g style="transform:matrix(1,0,0,1,276.0500183105469,134.81666564941406);"><path d="M204 123C177 114 159 108 106 93C99 17 74 -48 16 -144L30 -155L71 -136C152 -31 190 32 218 109Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017,0,0,-0.017,0,0);"></path></g><g style="transform:matrix(1,0,0,1,283.6833190917969,134.81666564941406);"><path d="M424 386L565 386L565 444L424 444L424 571L355 571L355 444L268 444L268 386L352 386L352 119C352 59 366 -11 435 -11C505 -11 556 14 581 27L565 86C539 65 507 53 475 53C438 53 424 83 424 136Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017,0,0,-0.017,0,0);"></path></g><g><g><g><g style="transform:matrix(1,0,0,1,294.5500183105469,137.81665649414063);"><path d="M457 331C457 412 453 506 411 588C370 665 301 689 250 689C191 689 121 662 80 571C47 497 42 413 42 331C42 251 46 177 76 103C116 5 192 -22 249 -22C322 -22 385 19 417 89C447 155 457 223 457 331ZM250 40C198 40 157 78 137 151C121 209 120 264 120 343C120 407 120 468 137 524C143 544 168 627 249 627C327 627 353 550 360 531C379 475 379 406 379 343C379 276 379 212 361 148C335 56 282 40 250 40Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.0119,0,0,-0.0119,0,0);"></path></g></g></g></g></g></g></g></g></g><g><g><g><g><g><g style="transform:matrix(1,0,0,1,231.50003051757812,156.81666564941406);"><path d="M260 229L443 444L361 444L227 279L89 444L6 444L194 229L0 0L82 0L227 188L377 0L460 0Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017,0,0,-0.017,0,0);"></path></g><g><g style="transform:matrix(1,0,0,1,246.94998168945312,156.81666564941406);"><path d="M949 241L949 300L179 300L304 452L272 486L65 269L272 55L304 89L179 241Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017,0,0,-0.017,0,0);"></path></g></g><g style="transform:matrix(1,0,0,1,267.5666809082031,156.81666564941406);"><path d="M299 689L279 689C220 627 137 624 89 622L89 563C122 564 170 566 220 587L220 59L95 59L95 0L424 0L424 59L299 59Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017,0,0,-0.017,0,0);"></path></g><g style="transform:matrix(1,0,0,1,276.0500183105469,156.81666564941406);"><path d="M204 123C177 114 159 108 106 93C99 17 74 -48 16 -144L30 -155L71 -136C152 -31 190 32 218 109Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017,0,0,-0.017,0,0);"></path></g><g style="transform:matrix(1,0,0,1,283.6833190917969,156.81666564941406);"><path d="M424 386L565 386L565 444L424 444L424 571L355 571L355 444L268 444L268 386L352 386L352 119C352 59 366 -11 435 -11C505 -11 556 14 581 27L565 86C539 65 507 53 475 53C438 53 424 83 424 136Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017,0,0,-0.017,0,0);"></path></g><g><g><g><g style="transform:matrix(1,0,0,1,294.5500183105469,159.81665649414063);"><path d="M457 331C457 412 453 506 411 588C370 665 301 689 250 689C191 689 121 662 80 571C47 497 42 413 42 331C42 251 46 177 76 103C116 5 192 -22 249 -22C322 -22 385 19 417 89C447 155 457 223 457 331ZM250 40C198 40 157 78 137 151C121 209 120 264 120 343C120 407 120 468 137 524C143 544 168 627 249 627C327 627 353 550 360 531C379 475 379 406 379 343C379 276 379 212 361 148C335 56 282 40 250 40Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.0119,0,0,-0.0119,0,0);"></path></g></g></g></g></g></g></g></g></g><g><g><g><g><g><g style="transform:matrix(1,0,0,1,101.5,114.81666564941406);"><path d="M260 229L443 444L361 444L227 279L89 444L6 444L194 229L0 0L82 0L227 188L377 0L460 0Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017,0,0,-0.017,0,0);"></path></g><g><g style="transform:matrix(1,0,0,1,116.95001220703125,114.81666564941406);"><path d="M949 241L949 300L179 300L304 452L272 486L65 269L272 55L304 89L179 241Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017,0,0,-0.017,0,0);"></path></g></g><g style="transform:matrix(1,0,0,1,137.56668090820312,114.81666564941406);"><path d="M299 689L279 689C220 627 137 624 89 622L89 563C122 564 170 566 220 587L220 59L95 59L95 0L424 0L424 59L299 59Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017,0,0,-0.017,0,0);"></path></g><g style="transform:matrix(1,0,0,1,146.05001831054688,114.81666564941406);"><path d="M204 123C177 114 159 108 106 93C99 17 74 -48 16 -144L30 -155L71 -136C152 -31 190 32 218 109Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017,0,0,-0.017,0,0);"></path></g><g style="transform:matrix(1,0,0,1,153.68331909179688,114.81666564941406);"><path d="M424 386L565 386L565 444L424 444L424 571L355 571L355 444L268 444L268 386L352 386L352 119C352 59 366 -11 435 -11C505 -11 556 14 581 27L565 86C539 65 507 53 475 53C438 53 424 83 424 136Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017,0,0,-0.017,0,0);"></path></g><g><g><g><g style="transform:matrix(1,0,0,1,164.55001831054688,117.81665649414063);"><path d="M457 331C457 412 453 506 411 588C370 665 301 689 250 689C191 689 121 662 80 571C47 497 42 413 42 331C42 251 46 177 76 103C116 5 192 -22 249 -22C322 -22 385 19 417 89C447 155 457 223 457 331ZM250 40C198 40 157 78 137 151C121 209 120 264 120 343C120 407 120 468 137 524C143 544 168 627 249 627C327 627 353 550 360 531C379 475 379 406 379 343C379 276 379 212 361 148C335 56 282 40 250 40Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.0119,0,0,-0.0119,0,0);"></path></g></g></g></g></g></g></g></g></g><g><g><g><g><g><g style="transform:matrix(1,0,0,1,231.50003051757812,354.81666564941406);"><path d="M260 229L443 444L361 444L227 279L89 444L6 444L194 229L0 0L82 0L227 188L377 0L460 0Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017,0,0,-0.017,0,0);"></path></g><g><g style="transform:matrix(1,0,0,1,246.94998168945312,354.81666564941406);"><path d="M949 241L949 300L179 300L304 452L272 486L65 269L272 55L304 89L179 241Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017,0,0,-0.017,0,0);"></path></g></g><g style="transform:matrix(1,0,0,1,267.5666809082031,354.81666564941406);"><path d="M299 689L279 689C220 627 137 624 89 622L89 563C122 564 170 566 220 587L220 59L95 59L95 0L424 0L424 59L299 59Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017,0,0,-0.017,0,0);"></path></g><g style="transform:matrix(1,0,0,1,276.0500183105469,354.81666564941406);"><path d="M204 123C177 114 159 108 106 93C99 17 74 -48 16 -144L30 -155L71 -136C152 -31 190 32 218 109Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017,0,0,-0.017,0,0);"></path></g><g style="transform:matrix(1,0,0,1,283.6833190917969,354.81666564941406);"><path d="M424 386L565 386L565 444L424 444L424 571L355 571L355 444L268 444L268 386L352 386L352 119C352 59 366 -11 435 -11C505 -11 556 14 581 27L565 86C539 65 507 53 475 53C438 53 424 83 424 136Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017,0,0,-0.017,0,0);"></path></g><g><g><g><g style="transform:matrix(1,0,0,1,294.5500183105469,357.81667175292966);"><path d="M457 331C457 412 453 506 411 588C370 665 301 689 250 689C191 689 121 662 80 571C47 497 42 413 42 331C42 251 46 177 76 103C116 5 192 -22 249 -22C322 -22 385 19 417 89C447 155 457 223 457 331ZM250 40C198 40 157 78 137 151C121 209 120 264 120 343C120 407 120 468 137 524C143 544 168 627 249 627C327 627 353 550 360 531C379 475 379 406 379 343C379 276 379 212 361 148C335 56 282 40 250 40Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.0119,0,0,-0.0119,0,0);"></path></g></g></g></g></g></g></g></g></g><g><g><g><g><g><g style="transform:matrix(1,0,0,1,219.63333129882812,386.40000915527344);"><path d="M125 390L69 107C68 99 56 61 56 31C56 6 67 -9 86 -9C121 -9 156 11 234 74L265 99L255 117L210 86C181 66 161 56 150 56C141 56 136 64 136 76C136 102 150 183 179 328L192 390L299 390L310 440C272 436 238 434 200 434C216 528 227 577 245 631L234 646C214 634 187 622 156 610L131 440C87 419 61 408 43 403L41 390Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017,0,0,-0.017,0,0);"></path></g></g></g></g></g></g><g><g><g><g><g><g style="transform:matrix(1,0,0,1,339.6166687011719,386.40000915527344);"><path d="M125 390L69 107C68 99 56 61 56 31C56 6 67 -9 86 -9C121 -9 156 11 234 74L265 99L255 117L210 86C181 66 161 56 150 56C141 56 136 64 136 76C136 102 150 183 179 328L192 390L299 390L310 440C272 436 238 434 200 434C216 528 227 577 245 631L234 646C214 634 187 622 156 610L131 440C87 419 61 408 43 403L41 390Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017,0,0,-0.017,0,0);"></path></g></g></g></g></g></g><g><g><g><g><g><g style="transform:matrix(1,0,0,1,459.6166687011719,386.40000915527344);"><path d="M125 390L69 107C68 99 56 61 56 31C56 6 67 -9 86 -9C121 -9 156 11 234 74L265 99L255 117L210 86C181 66 161 56 150 56C141 56 136 64 136 76C136 102 150 183 179 328L192 390L299 390L310 440C272 436 238 434 200 434C216 528 227 577 245 631L234 646C214 634 187 622 156 610L131 440C87 419 61 408 43 403L41 390Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017,0,0,-0.017,0,0);"></path></g></g></g></g></g></g><g><g><g><g><g><g style="transform:matrix(1,0,0,1,579.6166687011719,386.40000915527344);"><path d="M125 390L69 107C68 99 56 61 56 31C56 6 67 -9 86 -9C121 -9 156 11 234 74L265 99L255 117L210 86C181 66 161 56 150 56C141 56 136 64 136 76C136 102 150 183 179 328L192 390L299 390L310 440C272 436 238 434 200 434C216 528 227 577 245 631L234 646C214 634 187 622 156 610L131 440C87 419 61 408 43 403L41 390Z" stroke="rgb(0, 0, 0)" stroke-width="8" fill="rgb(0, 0, 0)" style="transform:matrix(0.017,0,0,-0.017,0,0);"></path></g></g></g></g></g></g></svg>
+</svg>
diff --git a/doc/source/operating/index.rst b/doc/source/operating/index.rst
index e2cead2..78c7eb6 100644
--- a/doc/source/operating/index.rst
+++ b/doc/source/operating/index.rst
@@ -27,7 +27,7 @@
repair
read_repair
hints
- compaction
+ compaction/index
bloom_filters
compression
cdc
diff --git a/doc/source/operating/metrics.rst b/doc/source/operating/metrics.rst
index 04abb48..fc37440 100644
--- a/doc/source/operating/metrics.rst
+++ b/doc/source/operating/metrics.rst
@@ -16,6 +16,8 @@
.. highlight:: none
+.. _monitoring-metrics:
+
Monitoring
----------
@@ -55,6 +57,8 @@
A meter metric which measures mean throughput and one-, five-, and fifteen-minute exponentially-weighted moving
average throughputs.
+.. _table-metrics:
+
Table Metrics
^^^^^^^^^^^^^
@@ -95,6 +99,7 @@
RangeLatency Latency Local range scan latency for this table.
WriteLatency Latency Local write latency for this table.
CoordinatorReadLatency Timer Coordinator read latency for this table.
+CoordinatorWriteLatency Timer Coordinator write latency for this table.
CoordinatorScanLatency Timer Coordinator range scan latency for this table.
PendingFlushes Counter Estimated number of flush tasks pending for this table.
BytesFlushed Counter Total number of bytes flushed since server [re]start.
@@ -126,16 +131,30 @@
CasPropose Latency Latency of paxos propose round.
CasCommit Latency Latency of paxos commit round.
PercentRepaired Gauge<Double> Percent of table data that is repaired on disk.
+BytesRepaired Gauge<Long> Size of table data repaired on disk
+BytesUnrepaired Gauge<Long> Size of table data unrepaired on disk
+BytesPendingRepair Gauge<Long> Size of table data isolated for an ongoing incremental repair
SpeculativeRetries Counter Number of times speculative retries were sent for this table.
+SpeculativeFailedRetries Counter Number of speculative retries that failed to prevent a timeout
+SpeculativeInsufficientReplicas Counter Number of speculative retries that couldn't be attempted due to lack of replicas
+SpeculativeSampleLatencyNanos Gauge<Long> Number of nanoseconds to wait before speculation is attempted. Value may be statically configured or updated periodically based on coordinator latency.
WaitingOnFreeMemtableSpace Histogram Histogram of time spent waiting for free memtable space, either on- or off-heap.
DroppedMutations Counter Number of dropped mutations on this table.
+AnticompactionTime Timer Time spent anticompacting before a consistent repair.
+ValidationTime Timer Time spent doing validation compaction during repair.
+SyncTime Timer Time spent doing streaming during repair.
+BytesValidated Histogram Histogram over the amount of bytes read during validation.
+PartitionsValidated Histogram Histogram over the number of partitions read during validation.
+BytesAnticompacted Counter How many bytes we anticompacted.
+BytesMutatedAnticompaction Counter How many bytes we avoided anticompacting because the sstable was fully contained in the repaired range.
+MutatedAnticompactionGauge Gauge<Double> Ratio of bytes mutated vs total bytes repaired.
======================================= ============== ===========
Keyspace Metrics
^^^^^^^^^^^^^^^^
Each keyspace in Cassandra has metrics responsible for tracking its state and performance.
-These metrics are the same as the ``Table Metrics`` above, only they are aggregated at the Keyspace level.
+Most of these metrics are the same as the ``Table Metrics`` above, only they are aggregated at the Keyspace level. The keyspace specific metrics are specified in the table below.
Reported name format:
@@ -145,6 +164,16 @@
**JMX MBean**
``org.apache.cassandra.metrics:type=Keyspace scope=<Keyspace> name=<MetricName>``
+
+======================================= ============== ===========
+Name Type Description
+======================================= ============== ===========
+WriteFailedIdeaCL Counter Number of writes that failed to achieve the configured ideal consistency level or 0 if none is configured
+IdealCLWriteLatency Latency Coordinator latency of writes at the configured ideal consistency level. No values are recorded if ideal consistency level is not configured
+RepairTime Timer Total time spent as repair coordinator.
+RepairPrepareTime Timer Total time spent preparing for repair.
+======================================= ============== ===========
+
ThreadPool Metrics
^^^^^^^^^^^^^^^^^^
@@ -161,7 +190,7 @@
``org.apache.cassandra.metrics.ThreadPools.<MetricName>.<Path>.<ThreadPoolName>``
**JMX MBean**
- ``org.apache.cassandra.metrics:type=ThreadPools scope=<ThreadPoolName> type=<Type> name=<MetricName>``
+ ``org.apache.cassandra.metrics:type=ThreadPools path=<Path> scope=<ThreadPoolName> name=<MetricName>``
===================== ============== ===========
Name Type Description
@@ -172,6 +201,7 @@
TotalBlockedTasks Counter Number of tasks that were blocked due to queue saturation.
CurrentlyBlockedTask Counter Number of tasks that are currently blocked due to queue saturation but on retry will become unblocked.
MaxPoolSize Gauge<Integer> The maximum number of threads in this pool.
+MaxTasksQueued Gauge<Integer> The maximum number of tasks queued before a task get blocked.
===================== ============== ===========
The following thread pools can be monitored.
@@ -202,6 +232,7 @@
Sampler internal Responsible for re-sampling the index summaries of SStables
SecondaryIndexManagement internal Performs updates to secondary indexes
ValidationExecutor internal Performs validation compaction or scrubbing
+ViewBuildExecutor internal Performs materialized views initial build
============================ ============== ===========
.. |nbsp| unicode:: 0xA0 .. nonbreaking space
@@ -249,6 +280,7 @@
UnfinishedCommit Counter Number of transactions that were committed on write.
ConditionNotMet Counter Number of transaction preconditions did not match current values.
ContentionHistogram Histogram How many contended writes were encountered
+ MutationSizeHistogram Histogram Total size in bytes of the requests mutations.
===================== ============== =============================================================
@@ -286,6 +318,7 @@
Failures Counter Number of write failures encountered.
|nbsp| Latency Write latency.
Unavailables Counter Number of unavailable exceptions encountered.
+ MutationSizeHistogram Histogram Total size in bytes of the requests mutations.
===================== ============== =============================================================
@@ -368,6 +401,7 @@
PreparedStatementsRatio Gauge<Double> Percentage of statements that are prepared vs unprepared.
========================== ============== ===========
+.. _dropped-metrics:
DroppedMessage Metrics
^^^^^^^^^^^^^^^^^^^^^^
@@ -378,10 +412,10 @@
Reported name format:
**Metric Name**
- ``org.apache.cassandra.metrics.DroppedMessages.<MetricName>.<Type>``
+ ``org.apache.cassandra.metrics.DroppedMessage.<MetricName>.<Type>``
**JMX MBean**
- ``org.apache.cassandra.metrics:type=DroppedMetrics scope=<Type> name=<MetricName>``
+ ``org.apache.cassandra.metrics:type=DroppedMessage scope=<Type> name=<MetricName>``
========================== ============== ===========
Name Type Description
@@ -500,6 +534,8 @@
TotalHintsInProgress Counter Number of hints attemping to be sent currently.
========================== ============== ===========
+.. _handoff-metrics:
+
HintedHandoff Metrics
^^^^^^^^^^^^^^^^^^^^^
@@ -522,6 +558,33 @@
Hints_not_stored-<PeerIP> Counter Number of hints not stored for this peer, due to being down past the configured hint window.
=========================== ============== ===========
+.. _hintsservice-metrics:
+
+HintsService Metrics
+^^^^^^^^^^^^^^^^^^^^^
+
+Metrics specific to the Hints delivery service. There are also some metrics related to hints tracked in ``Storage Metrics``
+
+These metrics include the peer endpoint **in the metric name**
+
+Reported name format:
+
+**Metric Name**
+ ``org.apache.cassandra.metrics.HintsService.<MetricName>``
+
+**JMX MBean**
+ ``org.apache.cassandra.metrics:type=HintsService name=<MetricName>``
+
+=========================== ============== ===========
+Name Type Description
+=========================== ============== ===========
+HintsSucceeded Meter A meter of the hints successfully delivered
+HintsFailed Meter A meter of the hints that failed deliver
+HintsTimedOut Meter A meter of the hints that timed out
+Hint_delays Histogram Histogram of hint delivery delays (in milliseconds)
+Hint_delays-<PeerIP> Histogram Histogram of hint delivery delays (in milliseconds) per peer
+=========================== ============== ===========
+
SSTable Index Metrics
^^^^^^^^^^^^^^^^^^^^^
@@ -578,13 +641,37 @@
**JMX MBean**
``org.apache.cassandra.metrics:type=Client name=<MetricName>``
+============================== =============================== ===========
+Name Type Description
+============================== =============================== ===========
+connectedNativeClients Gauge<Integer> Number of clients connected to this nodes native protocol server
+connections Gauge<List<Map<String, String>> List of all connections and their state information
+connectedNativeClientsByUser Gauge<Map<String, Int> Number of connnective native clients by username
+============================== =============================== ===========
+
+
+Batch Metrics
+^^^^^^^^^^^^^
+
+Metrics specifc to batch statements.
+
+Reported name format:
+
+**Metric Name**
+ ``org.apache.cassandra.metrics.Batch.<MetricName>``
+
+**JMX MBean**
+ ``org.apache.cassandra.metrics:type=Batch name=<MetricName>``
+
=========================== ============== ===========
Name Type Description
=========================== ============== ===========
-connectedNativeClients Counter Number of clients connected to this nodes native protocol server
-connectedThriftClients Counter Number of clients connected to this nodes thrift protocol server
+PartitionsPerCounterBatch Histogram Distribution of the number of partitions processed per counter batch
+PartitionsPerLoggedBatch Histogram Distribution of the number of partitions processed per logged batch
+PartitionsPerUnloggedBatch Histogram Distribution of the number of partitions processed per unlogged batch
=========================== ============== ===========
+
JVM Metrics
^^^^^^^^^^^
diff --git a/doc/source/operating/read_repair.rst b/doc/source/operating/read_repair.rst
index 0e52bf5..d280162 100644
--- a/doc/source/operating/read_repair.rst
+++ b/doc/source/operating/read_repair.rst
@@ -16,7 +16,154 @@
.. highlight:: none
-Read repair
------------
+.. _read-repair:
-.. todo:: todo
+Read repair
+==============
+Read Repair is the process of repairing data replicas during a read request. If all replicas involved in a read request at the given read consistency level are consistent the data is returned to the client and no read repair is needed. But if the replicas involved in a read request at the given consistency level are not consistent a read repair is performed to make replicas involved in the read request consistent. The most up-to-date data is returned to the client. The read repair runs in the foreground and is blocking in that a response is not returned to the client until the read repair has completed and up-to-date data is constructed.
+
+Expectation of Monotonic Quorum Reads
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Cassandra uses a blocking read repair to ensure the expectation of "monotonic quorum reads" i.e. that in 2 successive quorum reads, it’s guaranteed the 2nd one won't get something older than the 1st one, and this even if a failed quorum write made a write of the most up to date value only to a minority of replicas. "Quorum" means majority of nodes among replicas.
+
+Table level configuration of monotonic reads
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Cassandra 4.0 adds support for table level configuration of monotonic reads (`CASSANDRA-14635
+<https://issues.apache.org/jira/browse/CASSANDRA-14635>`_). The ``read_repair`` table option has been added to table schema, with the options ``blocking`` (default), and ``none``.
+
+The ``read_repair`` option configures the read repair behavior to allow tuning for various performance and consistency behaviors. Two consistency properties are affected by read repair behavior.
+
+- Monotonic Quorum Reads: Provided by ``BLOCKING``. Monotonic quorum reads prevents reads from appearing to go back in time in some circumstances. When monotonic quorum reads are not provided and a write fails to reach a quorum of replicas, it may be visible in one read, and then disappear in a subsequent read.
+- Write Atomicity: Provided by ``NONE``. Write atomicity prevents reads from returning partially applied writes. Cassandra attempts to provide partition level write atomicity, but since only the data covered by a ``SELECT`` statement is repaired by a read repair, read repair can break write atomicity when data is read at a more granular level than it is written. For example read repair can break write atomicity if you write multiple rows to a clustered partition in a batch, but then select a single row by specifying the clustering column in a ``SELECT`` statement.
+
+The available read repair settings are:
+
+Blocking
+*********
+The default setting. When ``read_repair`` is set to ``BLOCKING``, and a read repair is started, the read will block on writes sent to other replicas until the CL is reached by the writes. Provides monotonic quorum reads, but not partition level write atomicity.
+
+None
+*********
+When ``read_repair`` is set to ``NONE``, the coordinator will reconcile any differences between replicas, but will not attempt to repair them. Provides partition level write atomicity, but not monotonic quorum reads.
+
+An example of using the ``NONE`` setting for the ``read_repair`` option is as follows:
+
+::
+
+ CREATE TABLE ks.tbl (k INT, c INT, v INT, PRIMARY KEY (k,c)) with read_repair='NONE'");
+
+Read Repair Example
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+To illustrate read repair with an example, consider that a client sends a read request with read consistency level ``TWO`` to a 5-node cluster as illustrated in Figure 1. Read consistency level determines how many replica nodes must return a response before the read request is considered successful.
+
+
+.. figure:: Figure_1_read_repair.jpg
+
+
+Figure 1. Client sends read request to a 5-node Cluster
+
+Three nodes host replicas for the requested data as illustrated in Figure 2. With a read consistency level of ``TWO`` two replica nodes must return a response for the read request to be considered successful. If the node the client sends request to hosts a replica of the data requested only one other replica node needs to be sent a read request to. But if the receiving node does not host a replica for the requested data the node becomes a coordinator node and forwards the read request to a node that hosts a replica. A direct read request is forwarded to the fastest node (as determined by dynamic snitch) as shown in Figure 2. A direct read request is a full read and returns the requested data.
+
+.. figure:: Figure_2_read_repair.jpg
+
+Figure 2. Direct Read Request sent to Fastest Replica Node
+
+Next, the coordinator node sends the requisite number of additional requests to satisfy the consistency level, which is ``TWO``. The coordinator node needs to send one more read request for a total of two. All read requests additional to the first direct read request are digest read requests. A digest read request is not a full read and only returns the hash value of the data. Only a hash value is returned to reduce the network data traffic. In the example being discussed the coordinator node sends one digest read request to a node hosting a replica as illustrated in Figure 3.
+
+.. figure:: Figure_3_read_repair.jpg
+
+Figure 3. Coordinator Sends a Digest Read Request
+
+The coordinator node has received a full copy of data from one node and a hash value for the data from another node. To compare the data returned a hash value is calculated for the full copy of data. The two hash values are compared. If the hash values are the same no read repair is needed and the full copy of requested data is returned to the client. The coordinator node only performed a total of two replica read request because the read consistency level is ``TWO`` in the example. If the consistency level were higher such as ``THREE``, three replica nodes would need to respond to a read request and only if all digest or hash values were to match with the hash value of the full copy of data would the read request be considered successful and the data returned to the client.
+
+But, if the hash value/s from the digest read request/s are not the same as the hash value of the data from the full read request of the first replica node it implies that an inconsistency in the replicas exists. To fix the inconsistency a read repair is performed.
+
+For example, consider that that digest request returns a hash value that is not the same as the hash value of the data from the direct full read request. We would need to make the replicas consistent for which the coordinator node sends a direct (full) read request to the replica node that it sent a digest read request to earlier as illustrated in Figure 4.
+
+.. figure:: Figure_4_read_repair.jpg
+
+Figure 4. Coordinator sends Direct Read Request to Replica Node it had sent Digest Read Request to
+
+After receiving the data from the second replica node the coordinator has data from two of the replica nodes. It only needs two replicas as the read consistency level is ``TWO`` in the example. Data from the two replicas is compared and based on the timestamps the most recent replica is selected. Data may need to be merged to construct an up-to-date copy of data if one replica has data for only some of the columns. In the example, if the data from the first direct read request is found to be outdated and the data from the second full read request to be the latest read, repair needs to be performed on Replica 2. If a new up-to-date data is constructed by merging the two replicas a read repair would be needed on both the replicas involved. For example, a read repair is performed on Replica 2 as illustrated in Figure 5.
+
+.. figure:: Figure_5_read_repair.jpg
+
+Figure 5. Coordinator performs Read Repair
+
+
+The most up-to-date data is returned to the client as illustrated in Figure 6. From the three replicas Replica 1 is not even read and thus not repaired. Replica 2 is repaired. Replica 3 is the most up-to-date and returned to client.
+
+.. figure:: Figure_6_read_repair.jpg
+
+Figure 6. Most up-to-date Data returned to Client
+
+Read Consistency Level and Read Repair
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The read consistency is most significant in determining if a read repair needs to be performed. As discussed in Table 1 a read repair is not needed for all of the consistency levels.
+
+Table 1. Read Repair based on Read Consistency Level
+
++----------------------+-------------------------------------------+
+|Read Consistency Level| Description |
++----------------------+-------------------------------------------+
+| ONE |Read repair is not performed as the |
+| |data from the first direct read request |
+| |satisfies the consistency level ONE. |
+| |No digest read requests are involved |
+| |for finding mismatches in data. |
++----------------------+-------------------------------------------+
+| TWO |Read repair is performed if inconsistencies|
+| |in data are found as determined by the |
+| |direct and digest read requests. |
++----------------------+-------------------------------------------+
+| THREE |Read repair is performed if inconsistencies|
+| |in data are found as determined by the |
+| |direct and digest read requests. |
++----------------------+-------------------------------------------+
+|LOCAL_ONE |Read repair is not performed as the data |
+| |from the direct read request from the |
+| |closest replica satisfies the consistency |
+| |level LOCAL_ONE.No digest read requests are|
+| |involved for finding mismatches in data. |
++----------------------+-------------------------------------------+
+|LOCAL_QUORUM |Read repair is performed if inconsistencies|
+| |in data are found as determined by the |
+| |direct and digest read requests. |
++----------------------+-------------------------------------------+
+|QUORUM |Read repair is performed if inconsistencies|
+| |in data are found as determined by the |
+| |direct and digest read requests. |
++----------------------+-------------------------------------------+
+
+If read repair is performed it is made only on the replicas that are not up-to-date and that are involved in the read request. The number of replicas involved in a read request would be based on the read consistency level; in the example it is two.
+
+Improved Read Repair Blocking Behavior in Cassandra 4.0
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Cassandra 4.0 makes two improvements to read repair blocking behavior (`CASSANDRA-10726
+<https://issues.apache.org/jira/browse/CASSANDRA-10726>`_).
+
+1. Speculative Retry of Full Data Read Requests. Cassandra 4.0 makes use of speculative retry in sending read requests (full, not digest) to replicas if a full data response is not received, whether in the initial full read request or a full data read request during read repair. With speculative retry if it looks like a response may not be received from the initial set of replicas Cassandra sent messages to, to satisfy the consistency level, it speculatively sends additional read request to un-contacted replica/s. Cassandra 4.0 will also speculatively send a repair mutation to a minority of nodes not involved in the read repair data read / write cycle with the combined contents of all un-acknowledged mutations if it looks like one may not respond. Cassandra accepts acks from them in lieu of acks from the initial mutations sent out, so long as it receives the same number of acks as repair mutations transmitted.
+
+2. Only blocks on Full Data Responses to satisfy the Consistency Level. Cassandra 4.0 only blocks for what is needed for resolving the digest mismatch and wait for enough full data responses to meet the consistency level, no matter whether it’s speculative retry or read repair chance. As an example, if it looks like Cassandra might not receive full data requests from everyone in time, it sends additional requests to additional replicas not contacted in the initial full data read. If the collection of nodes that end up responding in time end up agreeing on the data, the response from the disagreeing replica that started the read repair is not considered, and won't be included in the response to the client, preserving the expectation of monotonic quorum reads.
+
+Diagnostic Events for Read Repairs
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Cassandra 4.0 adds diagnostic events for read repair (`CASSANDRA-14668
+<https://issues.apache.org/jira/browse/CASSANDRA-14668>`_) that can be used for exposing information such as:
+
+- Contacted endpoints
+- Digest responses by endpoint
+- Affected partition keys
+- Speculated reads / writes
+- Update oversized
+
+Background Read Repair
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Background read repair, which was configured using ``read_repair_chance`` and ``dclocal_read_repair_chance`` settings in ``cassandra.yaml`` is removed Cassandra 4.0 (`CASSANDRA-13910
+<https://issues.apache.org/jira/browse/CASSANDRA-13910>`_).
+
+Read repair is not an alternative for other kind of repairs such as full repairs or replacing a node that keeps failing. The data returned even after a read repair has been performed may not be the most up-to-date data if consistency level is other than one requiring response from all replicas.
diff --git a/doc/source/operating/repair.rst b/doc/source/operating/repair.rst
index 97d8ce8..94fdc11 100644
--- a/doc/source/operating/repair.rst
+++ b/doc/source/operating/repair.rst
@@ -16,7 +16,193 @@
.. highlight:: none
+.. _repair:
+
Repair
------
-.. todo:: todo
+Cassandra is designed to remain available if one of it's nodes is down or unreachable. However, when a node is down or
+unreachable, it needs to eventually discover the writes it missed. Hints attempt to inform a node of missed writes, but
+are a best effort, and aren't guaranteed to inform a node of 100% of the writes it missed. These inconsistencies can
+eventually result in data loss as nodes are replaced or tombstones expire.
+
+These inconsistencies are fixed with the repair process. Repair synchronizes the data between nodes by comparing their
+respective datasets for their common token ranges, and streaming the differences for any out of sync sections between
+the nodes. It compares the data with merkle trees, which are a hierarchy of hashes.
+
+Incremental and Full Repairs
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+There are 2 types of repairs: full repairs, and incremental repairs. Full repairs operate over all of the data in the
+token range being repaired. Incremental repairs only repair data that's been written since the previous incremental repair.
+
+Incremental repairs are the default repair type, and if run regularly, can significantly reduce the time and io cost of
+performing a repair. However, it's important to understand that once an incremental repair marks data as repaired, it won't
+try to repair it again. This is fine for syncing up missed writes, but it doesn't protect against things like disk corruption,
+data loss by operator error, or bugs in Cassandra. For this reason, full repairs should still be run occasionally.
+
+Usage and Best Practices
+^^^^^^^^^^^^^^^^^^^^^^^^
+
+Since repair can result in a lot of disk and network io, it's not run automatically by Cassandra. It is run by the operator
+via nodetool.
+
+Incremental repair is the default and is run with the following command:
+
+::
+
+ nodetool repair
+
+A full repair can be run with the following command:
+
+::
+
+ nodetool repair --full
+
+Additionally, repair can be run on a single keyspace:
+
+::
+
+ nodetool repair [options] <keyspace_name>
+
+Or even on specific tables:
+
+::
+
+ nodetool repair [options] <keyspace_name> <table1> <table2>
+
+
+The repair command only repairs token ranges on the node being repaired, it doesn't repair the whole cluster. By default, repair
+will operate on all token ranges replicated by the node you're running repair on, which will cause duplicate work if you run it
+on every node. The ``-pr`` flag will only repair the "primary" ranges on a node, so you can repair your entire cluster by running
+``nodetool repair -pr`` on each node in a single datacenter.
+
+The specific frequency of repair that's right for your cluster, of course, depends on several factors. However, if you're
+just starting out and looking for somewhere to start, running an incremental repair every 1-3 days, and a full repair every
+1-3 weeks is probably reasonable. If you don't want to run incremental repairs, a full repair every 5 days is a good place
+to start.
+
+At a minimum, repair should be run often enough that the gc grace period never expires on unrepaired data. Otherwise, deleted
+data could reappear. With a default gc grace period of 10 days, repairing every node in your cluster at least once every 7 days
+will prevent this, while providing enough slack to allow for delays.
+
+Other Options
+^^^^^^^^^^^^^
+
+``-pr, --partitioner-range``
+ Restricts repair to the 'primary' token ranges of the node being repaired. A primary range is just a token range for
+ which a node is the first replica in the ring.
+
+``-prv, --preview``
+ Estimates the amount of streaming that would occur for the given repair command. This builds the merkle trees, and prints
+ the expected streaming activity, but does not actually do any streaming. By default, incremental repairs are estimated,
+ add the ``--full`` flag to estimate a full repair.
+
+``-vd, --validate``
+ Verifies that the repaired data is the same across all nodes. Similiar to ``--preview``, this builds and compares merkle
+ trees of repaired data, but doesn't do any streaming. This is useful for troubleshooting. If this shows that the repaired
+ data is out of sync, a full repair should be run.
+
+.. seealso::
+ :ref:`nodetool repair docs <nodetool_repair>`
+
+Full Repair Example
+^^^^^^^^^^^^^^^^^^^^
+Full repair is typically needed to redistribute data after increasing the replication factor of a keyspace or after adding a node to the cluster. Full repair involves streaming SSTables. To demonstrate full repair start with a three node cluster.
+
+::
+
+ [ec2-user@ip-10-0-2-238 ~]$ nodetool status
+ Datacenter: us-east-1
+ =====================
+ Status=Up/Down
+ |/ State=Normal/Leaving/Joining/Moving
+ -- Address Load Tokens Owns Host ID Rack
+ UN 10.0.1.115 547 KiB 256 ? b64cb32a-b32a-46b4-9eeb-e123fa8fc287 us-east-1b
+ UN 10.0.3.206 617.91 KiB 256 ? 74863177-684b-45f4-99f7-d1006625dc9e us-east-1d
+ UN 10.0.2.238 670.26 KiB 256 ? 4dcdadd2-41f9-4f34-9892-1f20868b27c7 us-east-1c
+
+Create a keyspace with replication factor 3:
+
+::
+
+ cqlsh> DROP KEYSPACE cqlkeyspace;
+ cqlsh> CREATE KEYSPACE CQLKeyspace
+ ... WITH replication = {'class': 'SimpleStrategy', 'replication_factor' : 3};
+
+Add a table to the keyspace:
+
+::
+
+ cqlsh> use cqlkeyspace;
+ cqlsh:cqlkeyspace> CREATE TABLE t (
+ ... id int,
+ ... k int,
+ ... v text,
+ ... PRIMARY KEY (id)
+ ... );
+
+Add table data:
+
+::
+
+ cqlsh:cqlkeyspace> INSERT INTO t (id, k, v) VALUES (0, 0, 'val0');
+ cqlsh:cqlkeyspace> INSERT INTO t (id, k, v) VALUES (1, 1, 'val1');
+ cqlsh:cqlkeyspace> INSERT INTO t (id, k, v) VALUES (2, 2, 'val2');
+
+A query lists the data added:
+
+::
+
+ cqlsh:cqlkeyspace> SELECT * FROM t;
+
+ id | k | v
+ ----+---+------
+ 1 | 1 | val1
+ 0 | 0 | val0
+ 2 | 2 | val2
+ (3 rows)
+
+Make the following changes to a three node cluster:
+
+1. Increase the replication factor from 3 to 4.
+2. Add a 4th node to the cluster
+
+When the replication factor is increased the following message gets output indicating that a full repair is needed as per (`CASSANDRA-13079
+<https://issues.apache.org/jira/browse/CASSANDRA-13079>`_):
+
+::
+
+ cqlsh:cqlkeyspace> ALTER KEYSPACE CQLKeyspace
+ ... WITH replication = {'class': 'SimpleStrategy', 'replication_factor' : 4};
+ Warnings :
+ When increasing replication factor you need to run a full (-full) repair to distribute the
+ data.
+
+Perform a full repair on the keyspace ``cqlkeyspace`` table ``t`` with following command:
+
+::
+
+ nodetool repair -full cqlkeyspace t
+
+Full repair completes in about a second as indicated by the output:
+
+::
+
+[ec2-user@ip-10-0-2-238 ~]$ nodetool repair -full cqlkeyspace t
+[2019-08-17 03:06:21,445] Starting repair command #1 (fd576da0-c09b-11e9-b00c-1520e8c38f00), repairing keyspace cqlkeyspace with repair options (parallelism: parallel, primary range: false, incremental: false, job threads: 1, ColumnFamilies: [t], dataCenters: [], hosts: [], previewKind: NONE, # of ranges: 1024, pull repair: false, force repair: false, optimise streams: false)
+[2019-08-17 03:06:23,059] Repair session fd8e5c20-c09b-11e9-b00c-1520e8c38f00 for range [(-8792657144775336505,-8786320730900698730], (-5454146041421260303,-5439402053041523135], (4288357893651763201,4324309707046452322], ... , (4350676211955643098,4351706629422088296]] finished (progress: 0%)
+[2019-08-17 03:06:23,077] Repair completed successfully
+[2019-08-17 03:06:23,077] Repair command #1 finished in 1 second
+[ec2-user@ip-10-0-2-238 ~]$
+
+The ``nodetool tpstats`` command should list a repair having been completed as ``Repair-Task`` > ``Completed`` column value of 1:
+
+::
+
+ [ec2-user@ip-10-0-2-238 ~]$ nodetool tpstats
+ Pool Name Active Pending Completed Blocked All time blocked
+ ReadStage 0 0 99 0 0
+ …
+ Repair-Task 0 0 1 0 0
+ RequestResponseStage 0 0 2078 0 0
diff --git a/doc/source/operating/security.rst b/doc/source/operating/security.rst
index dfcd9e6..c945949 100644
--- a/doc/source/operating/security.rst
+++ b/doc/source/operating/security.rst
@@ -18,13 +18,26 @@
Security
--------
-
There are three main components to the security features provided by Cassandra:
- TLS/SSL encryption for client and inter-node communication
- Client authentication
- Authorization
+By default, these features are disabled as Cassandra is configured to easily find and be found by other members of a
+cluster. In other words, an out-of-the-box Cassandra installation presents a large attack surface for a bad actor.
+Enabling authentication for clients using the binary protocol is not sufficient to protect a cluster. Malicious users
+able to access internode communication and JMX ports can still:
+
+- Craft internode messages to insert users into authentication schema
+- Craft internode messages to truncate or drop schema
+- Use tools such as ``sstableloader`` to overwrite ``system_auth`` tables
+- Attach to the cluster directly to capture write traffic
+
+Correct configuration of all three security components should negate theses vectors. Therefore, understanding Cassandra's
+security features is crucial to configuring your cluster to meet your security needs.
+
+
TLS/SSL Encryption
^^^^^^^^^^^^^^^^^^
Cassandra provides secure communication between a client machine and a database cluster and between nodes within a
@@ -41,7 +54,17 @@
for more details.
For information on generating the keystore and truststore files used in SSL communications, see the
-`java documentation on creating keystores <http://download.oracle.com/javase/6/docs/technotes/guides/security/jsse/JSSERefGuide.html#CreateKeystore>`__
+`java documentation on creating keystores <https://docs.oracle.com/javase/8/docs/technotes/guides/security/jsse/JSSERefGuide.html#CreateKeystore>`__
+
+SSL Certificate Hot Reloading
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Beginning with Cassandra 4, Cassandra supports hot reloading of SSL Certificates. If SSL/TLS support is enabled in Cassandra,
+the node periodically polls the Trust and Key Stores specified in cassandra.yaml. When the files are updated, Cassandra will
+reload them and use them for subsequent connections. Please note that the Trust & Key Store passwords are part of the yaml so
+the updated files should also use the same passwords. The default polling interval is 10 minutes.
+
+Certificate Hot reloading may also be triggered using the ``nodetool reloadssl`` command. Use this if you want to Cassandra to
+immediately notice the changed certificates.
Inter-node Encryption
~~~~~~~~~~~~~~~~~~~~~
@@ -159,6 +182,8 @@
:ref:`ALTER ROLE <alter-role-statement>`, :ref:`ALTER KEYSPACE <alter-keyspace-statement>` and :ref:`GRANT PERMISSION
<grant-permission-statement>`,
+.. _authorization:
+
Authorization
^^^^^^^^^^^^^
@@ -210,6 +235,8 @@
See also: :ref:`GRANT PERMISSION <grant-permission-statement>`, `GRANT ALL <grant-all>` and :ref:`REVOKE PERMISSION
<revoke-permission-statement>`
+.. _auth-caching:
+
Caching
^^^^^^^
@@ -249,7 +276,7 @@
Cassandra's own auth subsystem.
The default settings for Cassandra make JMX accessible only from localhost. To enable remote JMX connections, edit
-``cassandra-env.sh`` (or ``cassandra-env.ps1`` on Windows) to change the ``LOCAL_JMX`` setting to ``yes``. Under the
+``cassandra-env.sh`` (or ``cassandra-env.ps1`` on Windows) to change the ``LOCAL_JMX`` setting to ``no``. Under the
standard configuration, when remote JMX connections are enabled, :ref:`standard JMX authentication <standard-jmx-auth>`
is also switched on.
@@ -356,6 +383,10 @@
GRANT EXECUTE ON MBEAN 'java.lang:type=Threading' TO jmx;
GRANT EXECUTE ON MBEAN 'com.sun.management:type=HotSpotDiagnostic' TO jmx;
+ # Grant the role with necessary permissions to use nodetool commands (including nodetool status) in read-only mode
+ GRANT EXECUTE ON MBEAN 'org.apache.cassandra.db:type=EndpointSnitchInfo' TO jmx;
+ GRANT EXECUTE ON MBEAN 'org.apache.cassandra.db:type=StorageService' TO jmx;
+
# Grant the jmx role to one with login permissions so that it can access the JMX tooling
CREATE ROLE ks_user WITH PASSWORD = 'password' AND LOGIN = true AND SUPERUSER = false;
GRANT jmx TO ks_user;
diff --git a/doc/source/operating/snitch.rst b/doc/source/operating/snitch.rst
index faea0b3..b716e82 100644
--- a/doc/source/operating/snitch.rst
+++ b/doc/source/operating/snitch.rst
@@ -16,6 +16,8 @@
.. highlight:: none
+.. _snitch:
+
Snitch
------
@@ -35,8 +37,8 @@
- ``dynamic_snitch``: whether the dynamic snitch should be enabled or disabled.
- ``dynamic_snitch_update_interval_in_ms``: controls how often to perform the more expensive part of host score
calculation.
-- ``dynamic_snitch_reset_interval_in_ms``: if set greater than zero and read_repair_chance is < 1.0, this will allow
- 'pinning' of replicas to hosts in order to increase cache capacity.
+- ``dynamic_snitch_reset_interval_in_ms``: if set greater than zero, this will allow 'pinning' of replicas to hosts
+ in order to increase cache capacity.
- ``dynamic_snitch_badness_threshold:``: The badness threshold will control how much worse the pinned host has to be
before the dynamic snitch will prefer other replicas over it. This is expressed as a double which represents a
percentage. Thus, a value of 0.2 means Cassandra would continue to prefer the static snitch values until the pinned
@@ -45,7 +47,7 @@
Snitch classes
^^^^^^^^^^^^^^
-The ``endpoint_snitch`` parameter in ``cassandra.yaml`` should be set to the class the class that implements
+The ``endpoint_snitch`` parameter in ``cassandra.yaml`` should be set to the class that implements
``IEndPointSnitch`` which will be wrapped by the dynamic snitch and decide if two endpoints are in the same data center
or on the same rack. Out of the box, Cassandra provides the snitch implementations:
@@ -63,9 +65,11 @@
``cassandra-topology.properties``.
Ec2Snitch
- Appropriate for EC2 deployments in a single Region. Loads Region and Availability Zone information from the EC2 API.
- The Region is treated as the datacenter, and the Availability Zone as the rack. Only private IPs are used, so this
- will not work across multiple regions.
+ Appropriate for EC2 deployments in a single Region, or in multiple regions with inter-region VPC enabled (available
+ since the end of 2017, see `AWS announcement <https://aws.amazon.com/about-aws/whats-new/2017/11/announcing-support-for-inter-region-vpc-peering/>`_).
+ Loads Region and Availability Zone information from the EC2 API. The Region is treated as the datacenter, and the
+ Availability Zone as the rack. Only private IPs are used, so this will work across multiple regions only if
+ inter-region VPC is enabled.
Ec2MultiRegionSnitch
Uses public IPs as broadcast_address to allow cross-region connectivity (thus, you should set seed addresses to the
diff --git a/doc/source/operating/topo_changes.rst b/doc/source/operating/topo_changes.rst
index c42708e..6c8f8ec 100644
--- a/doc/source/operating/topo_changes.rst
+++ b/doc/source/operating/topo_changes.rst
@@ -98,16 +98,21 @@
In order to replace a dead node, start cassandra with the JVM startup flag
``-Dcassandra.replace_address_first_boot=<dead_node_ip>``. Once this property is enabled the node starts in a hibernate
-state, during which all the other nodes will see this node to be down.
+state, during which all the other nodes will see this node to be DOWN (DN), however this node will see itself as UP
+(UN). Accurate replacement state can be found in ``nodetool netstats``.
-The replacing node will now start to bootstrap the data from the rest of the nodes in the cluster. The main difference
-between normal bootstrapping of a new node is that this new node will not accept any writes during this phase.
+The replacing node will now start to bootstrap the data from the rest of the nodes in the cluster. A replacing node will
+only receive writes during the bootstrapping phase if it has a different ip address to the node that is being replaced.
+(See CASSANDRA-8523 and CASSANDRA-12344)
-Once the bootstrapping is complete the node will be marked "UP", we rely on the hinted handoff's for making this node
-consistent (since we don't accept writes since the start of the bootstrap).
+Once the bootstrapping is complete the node will be marked "UP".
-.. Note:: If the replacement process takes longer than ``max_hint_window_in_ms`` you **MUST** run repair to make the
- replaced node consistent again, since it missed ongoing writes during bootstrapping.
+.. Note:: If any of the following cases apply, you **MUST** run repair to make the replaced node consistent again, since
+ it missed ongoing writes during/prior to bootstrapping. The *replacement* timeframe refers to the period from when the
+ node initially dies to when a new node completes the replacement process.
+
+ 1. The node is down for longer than ``max_hint_window_in_ms`` before being replaced.
+ 2. You are replacing using the same IP address as the dead node **and** replacement takes longer than ``max_hint_window_in_ms``.
Monitoring progress
^^^^^^^^^^^^^^^^^^^
diff --git a/doc/source/plugins/index.rst b/doc/source/plugins/index.rst
new file mode 100644
index 0000000..4073a92
--- /dev/null
+++ b/doc/source/plugins/index.rst
@@ -0,0 +1,35 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements. See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership. The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License. You may obtain a copy of the License at
+..
+.. http://www.apache.org/licenses/LICENSE-2.0
+..
+.. Unless required by applicable law or agreed to in writing, software
+.. distributed under the License is distributed on an "AS IS" BASIS,
+.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+.. See the License for the specific language governing permissions and
+.. limitations under the License.
+
+Third-Party Plugins
+===================
+
+Available third-party plugins for Apache Cassandra
+
+CAPI-Rowcache
+-------------
+
+The Coherent Accelerator Process Interface (CAPI) is a general term for the infrastructure of attaching a Coherent accelerator to an IBM POWER system. A key innovation in IBM POWER8’s open architecture is the CAPI. It provides a high bandwidth, low latency path between external devices, the POWER8 core, and the system’s open memory architecture. IBM Data Engine for NoSQL is an integrated platform for large and fast growing NoSQL data stores. It builds on the CAPI capability of POWER8 systems and provides super-fast access to large flash storage capacity and addresses the challenges associated with typical x86 server based scale-out deployments.
+
+The official page for the `CAPI-Rowcache plugin <https://github.com/ppc64le/capi-rowcache>`__ contains further details how to build/run/download the plugin.
+
+
+Stratio’s Cassandra Lucene Index
+--------------------------------
+
+Stratio’s Lucene index is a Cassandra secondary index implementation based on `Apache Lucene <http://lucene.apache.org/>`__. It extends Cassandra’s functionality to provide near real-time distributed search engine capabilities such as with ElasticSearch or `Apache Solr <http://lucene.apache.org/solr/>`__, including full text search capabilities, free multivariable, geospatial and bitemporal search, relevance queries and sorting based on column value, relevance or distance. Each node indexes its own data, so high availability and scalability is guaranteed.
+
+The official Github repository `Cassandra Lucene Index <http://www.github.com/stratio/cassandra-lucene-index>`__ contains everything you need to build/run/configure the plugin.
\ No newline at end of file
diff --git a/doc/source/tools/cassandra_stress.rst b/doc/source/tools/cassandra_stress.rst
new file mode 100644
index 0000000..c59d058
--- /dev/null
+++ b/doc/source/tools/cassandra_stress.rst
@@ -0,0 +1,273 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements. See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership. The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License. You may obtain a copy of the License at
+..
+.. http://www.apache.org/licenses/LICENSE-2.0
+..
+.. Unless required by applicable law or agreed to in writing, software
+.. distributed under the License is distributed on an "AS IS" BASIS,
+.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+.. See the License for the specific language governing permissions and
+.. limitations under the License.
+
+.. highlight:: yaml
+
+.. _cassandra_stress:
+
+Cassandra Stress
+----------------
+
+cassandra-stress is a tool for benchmarking and load testing a Cassandra
+cluster. cassandra-stress supports testing arbitrary CQL tables and queries
+to allow users to benchmark their data model.
+
+This documentation focuses on user mode as this allows the testing of your
+actual schema.
+
+Usage
+^^^^^
+There are several operation types:
+
+ * write-only, read-only, and mixed workloads of standard data
+ * write-only and read-only workloads for counter columns
+ * user configured workloads, running custom queries on custom schemas
+
+The syntax is `cassandra-stress <command> [options]`. If you want more information on a given command
+or options, just run `cassandra-stress help <command|option>`.
+
+Commands:
+ read:
+ Multiple concurrent reads - the cluster must first be populated by a write test
+ write:
+ Multiple concurrent writes against the cluster
+ mixed:
+ Interleaving of any basic commands, with configurable ratio and distribution - the cluster must first be populated by a write test
+ counter_write:
+ Multiple concurrent updates of counters.
+ counter_read:
+ Multiple concurrent reads of counters. The cluster must first be populated by a counterwrite test.
+ user:
+ Interleaving of user provided queries, with configurable ratio and distribution.
+ help:
+ Print help for a command or option
+ print:
+ Inspect the output of a distribution definition
+ legacy:
+ Legacy support mode
+
+Primary Options:
+ -pop:
+ Population distribution and intra-partition visit order
+ -insert:
+ Insert specific options relating to various methods for batching and splitting partition updates
+ -col:
+ Column details such as size and count distribution, data generator, names, comparator and if super columns should be used
+ -rate:
+ Thread count, rate limit or automatic mode (default is auto)
+ -mode:
+ Thrift or CQL with options
+ -errors:
+ How to handle errors when encountered during stress
+ -sample:
+ Specify the number of samples to collect for measuring latency
+ -schema:
+ Replication settings, compression, compaction, etc.
+ -node:
+ Nodes to connect to
+ -log:
+ Where to log progress to, and the interval at which to do it
+ -transport:
+ Custom transport factories
+ -port:
+ The port to connect to cassandra nodes on
+ -sendto:
+ Specify a stress server to send this command to
+ -graph:
+ Graph recorded metrics
+ -tokenrange:
+ Token range settings
+
+
+Suboptions:
+ Every command and primary option has its own collection of suboptions. These are too numerous to list here.
+ For information on the suboptions for each command or option, please use the help command,
+ `cassandra-stress help <command|option>`.
+
+User mode
+^^^^^^^^^
+
+User mode allows you to use your stress your own schemas. This can save time in
+the long run rather than building an application and then realising your schema
+doesn't scale.
+
+Profile
++++++++
+
+User mode requires a profile defined in YAML.
+Multiple YAML files may be specified in which case operations in the ops argument are referenced as specname.opname.
+
+An identifier for the profile::
+
+ specname: staff_activities
+
+The keyspace for the test::
+
+ keyspace: staff
+
+CQL for the keyspace. Optional if the keyspace already exists::
+
+ keyspace_definition: |
+ CREATE KEYSPACE stresscql WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 3};
+
+The table to be stressed::
+
+ table: staff_activities
+
+CQL for the table. Optional if the table already exists::
+
+ table_definition: |
+ CREATE TABLE staff_activities (
+ name text,
+ when timeuuid,
+ what text,
+ PRIMARY KEY(name, when, what)
+ )
+
+
+Optional meta information on the generated columns in the above table.
+The min and max only apply to text and blob types.
+The distribution field represents the total unique population
+distribution of that column across rows::
+
+ columnspec:
+ - name: name
+ size: uniform(5..10) # The names of the staff members are between 5-10 characters
+ population: uniform(1..10) # 10 possible staff members to pick from
+ - name: when
+ cluster: uniform(20..500) # Staff members do between 20 and 500 events
+ - name: what
+ size: normal(10..100,50)
+
+Supported types are:
+
+An exponential distribution over the range [min..max]::
+
+ EXP(min..max)
+
+An extreme value (Weibull) distribution over the range [min..max]::
+
+ EXTREME(min..max,shape)
+
+A gaussian/normal distribution, where mean=(min+max)/2, and stdev is (mean-min)/stdvrng::
+
+ GAUSSIAN(min..max,stdvrng)
+
+A gaussian/normal distribution, with explicitly defined mean and stdev::
+
+ GAUSSIAN(min..max,mean,stdev)
+
+A uniform distribution over the range [min, max]::
+
+ UNIFORM(min..max)
+
+A fixed distribution, always returning the same value::
+
+ FIXED(val)
+
+If preceded by ~, the distribution is inverted
+
+Defaults for all columns are size: uniform(4..8), population: uniform(1..100B), cluster: fixed(1)
+
+Insert distributions::
+
+ insert:
+ # How many partition to insert per batch
+ partitions: fixed(1)
+ # How many rows to update per partition
+ select: fixed(1)/500
+ # UNLOGGED or LOGGED batch for insert
+ batchtype: UNLOGGED
+
+
+Currently all inserts are done inside batches.
+
+Read statements to use during the test::
+
+ queries:
+ events:
+ cql: select * from staff_activities where name = ?
+ fields: samerow
+ latest_event:
+ cql: select * from staff_activities where name = ? LIMIT 1
+ fields: samerow
+
+Running a user mode test::
+
+ cassandra-stress user profile=./example.yaml duration=1m "ops(insert=1,latest_event=1,events=1)" truncate=once
+
+This will create the schema then run tests for 1 minute with an equal number of inserts, latest_event queries and events
+queries. Additionally the table will be truncated once before the test.
+
+The full example can be found here :download:`yaml <./stress-example.yaml>`
+
+Running a user mode test with multiple yaml files::
+ cassandra-stress user profile=./example.yaml,./example2.yaml duration=1m "ops(ex1.insert=1,ex1.latest_event=1,ex2.insert=2)" truncate=once
+
+This will run operations as specified in both the example.yaml and example2.yaml files. example.yaml and example2.yaml can reference the same table
+ although care must be taken that the table definition is identical (data generation specs can be different).
+
+Lightweight transaction support
++++++++++++++++++++++++++++++++
+
+cassandra-stress supports lightweight transactions. In this it will first read current data from Cassandra and then uses read value(s)
+to fulfill lightweight transaction condition(s).
+
+Lightweight transaction update query::
+
+ queries:
+ regularupdate:
+ cql: update blogposts set author = ? where domain = ? and published_date = ?
+ fields: samerow
+ updatewithlwt:
+ cql: update blogposts set author = ? where domain = ? and published_date = ? IF body = ? AND url = ?
+ fields: samerow
+
+The full example can be found here :download:`yaml <./stress-lwt-example.yaml>`
+
+Graphing
+^^^^^^^^
+
+Graphs can be generated for each run of stress.
+
+.. image:: example-stress-graph.png
+
+To create a new graph::
+
+ cassandra-stress user profile=./stress-example.yaml "ops(insert=1,latest_event=1,events=1)" -graph file=graph.html title="Awesome graph"
+
+To add a new run to an existing graph point to an existing file and add a revision name::
+
+ cassandra-stress user profile=./stress-example.yaml duration=1m "ops(insert=1,latest_event=1,events=1)" -graph file=graph.html title="Awesome graph" revision="Second run"
+
+FAQ
+^^^^
+
+**How do you use NetworkTopologyStrategy for the keyspace?**
+
+Use the schema option making sure to either escape the parenthesis or enclose in quotes::
+
+ cassandra-stress write -schema "replication(strategy=NetworkTopologyStrategy,datacenter1=3)"
+
+**How do you use SSL?**
+
+Use the transport option::
+
+ cassandra-stress "write n=100k cl=ONE no-warmup" -transport "truststore=$HOME/jks/truststore.jks truststore-password=cassandra"
+
+**Is Cassandra Stress a secured tool?**
+
+Cassandra stress is not a secured tool. Serialization and other aspects of the tool offer no security guarantees.
diff --git a/doc/source/tools/cqlsh.rst b/doc/source/tools/cqlsh.rst
index 45e2db8..b800b88 100644
--- a/doc/source/tools/cqlsh.rst
+++ b/doc/source/tools/cqlsh.rst
@@ -100,6 +100,9 @@
``--connect-timeout``
Specify the connection timeout in seconds (defaults to 2s)
+``--python /path/to/python``
+ Specify the full path to Python interpreter to override default on systems with multiple interpreters installed
+
``--request-timeout``
Specify the request timeout in seconds (defaults to 10s)
diff --git a/doc/source/tools/example-stress-graph.png b/doc/source/tools/example-stress-graph.png
new file mode 100644
index 0000000..a65b08b
--- /dev/null
+++ b/doc/source/tools/example-stress-graph.png
Binary files differ
diff --git a/doc/source/tools/index.rst b/doc/source/tools/index.rst
index 5a5e4d5..d28929c 100644
--- a/doc/source/tools/index.rst
+++ b/doc/source/tools/index.rst
@@ -20,7 +20,9 @@
This section describes the command line tools provided with Apache Cassandra.
.. toctree::
- :maxdepth: 1
+ :maxdepth: 3
cqlsh
- nodetool
+ nodetool/nodetool
+ sstable/index
+ cassandra_stress
diff --git a/doc/source/tools/nodetool.rst b/doc/source/tools/nodetool.rst
deleted file mode 100644
index e373031..0000000
--- a/doc/source/tools/nodetool.rst
+++ /dev/null
@@ -1,22 +0,0 @@
-.. Licensed to the Apache Software Foundation (ASF) under one
-.. or more contributor license agreements. See the NOTICE file
-.. distributed with this work for additional information
-.. regarding copyright ownership. The ASF licenses this file
-.. to you under the Apache License, Version 2.0 (the
-.. "License"); you may not use this file except in compliance
-.. with the License. You may obtain a copy of the License at
-..
-.. http://www.apache.org/licenses/LICENSE-2.0
-..
-.. Unless required by applicable law or agreed to in writing, software
-.. distributed under the License is distributed on an "AS IS" BASIS,
-.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-.. See the License for the specific language governing permissions and
-.. limitations under the License.
-
-.. _nodetool:
-
-Nodetool
---------
-
-.. todo:: Try to autogenerate this from Nodetool’s help.
diff --git a/doc/source/tools/sstable/index.rst b/doc/source/tools/sstable/index.rst
new file mode 100644
index 0000000..b9e483f
--- /dev/null
+++ b/doc/source/tools/sstable/index.rst
@@ -0,0 +1,39 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements. See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership. The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License. You may obtain a copy of the License at
+..
+.. http://www.apache.org/licenses/LICENSE-2.0
+..
+.. Unless required by applicable law or agreed to in writing, software
+.. distributed under the License is distributed on an "AS IS" BASIS,
+.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+.. See the License for the specific language governing permissions and
+.. limitations under the License.
+
+SSTable Tools
+=============
+
+This section describes the functionality of the various sstable tools.
+
+Cassandra must be stopped before these tools are executed, or unexpected results will occur. Note: the scripts do not verify that Cassandra is stopped.
+
+.. toctree::
+ :maxdepth: 2
+
+ sstabledump
+ sstableexpiredblockers
+ sstablelevelreset
+ sstableloader
+ sstablemetadata
+ sstableofflinerelevel
+ sstablerepairedset
+ sstablescrub
+ sstablesplit
+ sstableupgrade
+ sstableutil
+ sstableverify
+
diff --git a/doc/source/tools/sstable/sstabledump.rst b/doc/source/tools/sstable/sstabledump.rst
new file mode 100644
index 0000000..8f38afa
--- /dev/null
+++ b/doc/source/tools/sstable/sstabledump.rst
@@ -0,0 +1,294 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements. See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership. The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License. You may obtain a copy of the License at
+..
+.. http://www.apache.org/licenses/LICENSE-2.0
+..
+.. Unless required by applicable law or agreed to in writing, software
+.. distributed under the License is distributed on an "AS IS" BASIS,
+.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+.. See the License for the specific language governing permissions and
+.. limitations under the License.
+
+sstabledump
+-----------
+
+Dump contents of a given SSTable to standard output in JSON format.
+
+You must supply exactly one sstable.
+
+Cassandra must be stopped before this tool is executed, or unexpected results will occur. Note: the script does not verify that Cassandra is stopped.
+
+Usage
+^^^^^
+sstabledump <options> <sstable file path>
+
+=================================== ================================================================================
+-d CQL row per line internal representation
+-e Enumerate partition keys only
+-k <arg> Partition key
+-x <arg> Excluded partition key(s)
+-t Print raw timestamps instead of iso8601 date strings
+-l Output each row as a separate JSON object
+=================================== ================================================================================
+
+If necessary, use sstableutil first to find out the sstables used by a table.
+
+Dump entire table
+^^^^^^^^^^^^^^^^^
+
+Dump the entire table without any options.
+
+Example::
+
+ sstabledump /var/lib/cassandra/data/keyspace/eventlog-65c429e08c5a11e8939edf4f403979ef/mc-1-big-Data.db > eventlog_dump_2018Jul26
+
+ cat eventlog_dump_2018Jul26
+ [
+ {
+ "partition" : {
+ "key" : [ "3578d7de-c60d-4599-aefb-3f22a07b2bc6" ],
+ "position" : 0
+ },
+ "rows" : [
+ {
+ "type" : "row",
+ "position" : 61,
+ "liveness_info" : { "tstamp" : "2018-07-20T20:23:08.378711Z" },
+ "cells" : [
+ { "name" : "event", "value" : "party" },
+ { "name" : "insertedtimestamp", "value" : "2018-07-20 20:23:08.384Z" },
+ { "name" : "source", "value" : "asdf" }
+ ]
+ }
+ ]
+ },
+ {
+ "partition" : {
+ "key" : [ "d18250c0-84fc-4d40-b957-4248dc9d790e" ],
+ "position" : 62
+ },
+ "rows" : [
+ {
+ "type" : "row",
+ "position" : 123,
+ "liveness_info" : { "tstamp" : "2018-07-20T20:23:07.783522Z" },
+ "cells" : [
+ { "name" : "event", "value" : "party" },
+ { "name" : "insertedtimestamp", "value" : "2018-07-20 20:23:07.789Z" },
+ { "name" : "source", "value" : "asdf" }
+ ]
+ }
+ ]
+ },
+ {
+ "partition" : {
+ "key" : [ "cf188983-d85b-48d6-9365-25005289beb2" ],
+ "position" : 124
+ },
+ "rows" : [
+ {
+ "type" : "row",
+ "position" : 182,
+ "liveness_info" : { "tstamp" : "2018-07-20T20:22:27.028809Z" },
+ "cells" : [
+ { "name" : "event", "value" : "party" },
+ { "name" : "insertedtimestamp", "value" : "2018-07-20 20:22:27.055Z" },
+ { "name" : "source", "value" : "asdf" }
+ ]
+ }
+ ]
+ }
+ ]
+
+Dump table in a more manageable format
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Use the -l option to dump each row as a separate JSON object. This will make the output easier to manipulate for large data sets. ref: https://issues.apache.org/jira/browse/CASSANDRA-13848
+
+Example::
+
+ sstabledump /var/lib/cassandra/data/keyspace/eventlog-65c429e08c5a11e8939edf4f403979ef/mc-1-big-Data.db -l > eventlog_dump_2018Jul26_justlines
+
+ cat eventlog_dump_2018Jul26_justlines
+ [
+ {
+ "partition" : {
+ "key" : [ "3578d7de-c60d-4599-aefb-3f22a07b2bc6" ],
+ "position" : 0
+ },
+ "rows" : [
+ {
+ "type" : "row",
+ "position" : 61,
+ "liveness_info" : { "tstamp" : "2018-07-20T20:23:08.378711Z" },
+ "cells" : [
+ { "name" : "event", "value" : "party" },
+ { "name" : "insertedtimestamp", "value" : "2018-07-20 20:23:08.384Z" },
+ { "name" : "source", "value" : "asdf" }
+ ]
+ }
+ ]
+ },
+ {
+ "partition" : {
+ "key" : [ "d18250c0-84fc-4d40-b957-4248dc9d790e" ],
+ "position" : 62
+ },
+ "rows" : [
+ {
+ "type" : "row",
+ "position" : 123,
+ "liveness_info" : { "tstamp" : "2018-07-20T20:23:07.783522Z" },
+ "cells" : [
+ { "name" : "event", "value" : "party" },
+ { "name" : "insertedtimestamp", "value" : "2018-07-20 20:23:07.789Z" },
+ { "name" : "source", "value" : "asdf" }
+ ]
+ }
+ ]
+ },
+ {
+ "partition" : {
+ "key" : [ "cf188983-d85b-48d6-9365-25005289beb2" ],
+ "position" : 124
+ },
+ "rows" : [
+ {
+ "type" : "row",
+ "position" : 182,
+ "liveness_info" : { "tstamp" : "2018-07-20T20:22:27.028809Z" },
+ "cells" : [
+ { "name" : "event", "value" : "party" },
+ { "name" : "insertedtimestamp", "value" : "2018-07-20 20:22:27.055Z" },
+ { "name" : "source", "value" : "asdf" }
+ ]
+ }
+ ]
+ }
+
+Dump only keys
+^^^^^^^^^^^^^^
+
+Dump only the keys by using the -e option.
+
+Example::
+
+ sstabledump /var/lib/cassandra/data/keyspace/eventlog-65c429e08c5a11e8939edf4f403979ef/mc-1-big-Data.db -e > eventlog_dump_2018Jul26_justkeys
+
+ cat eventlog_dump_2018Jul26b
+ [ [ "3578d7de-c60d-4599-aefb-3f22a07b2bc6" ], [ "d18250c0-84fc-4d40-b957-4248dc9d790e" ], [ "cf188983-d85b-48d6-9365-25005289beb2" ]
+
+Dump row for a single key
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Dump a single key using the -k option.
+
+Example::
+
+ sstabledump /var/lib/cassandra/data/keyspace/eventlog-65c429e08c5a11e8939edf4f403979ef/mc-1-big-Data.db -k 3578d7de-c60d-4599-aefb-3f22a07b2bc6 > eventlog_dump_2018Jul26_singlekey
+
+ cat eventlog_dump_2018Jul26_singlekey
+ [
+ {
+ "partition" : {
+ "key" : [ "3578d7de-c60d-4599-aefb-3f22a07b2bc6" ],
+ "position" : 0
+ },
+ "rows" : [
+ {
+ "type" : "row",
+ "position" : 61,
+ "liveness_info" : { "tstamp" : "2018-07-20T20:23:08.378711Z" },
+ "cells" : [
+ { "name" : "event", "value" : "party" },
+ { "name" : "insertedtimestamp", "value" : "2018-07-20 20:23:08.384Z" },
+ { "name" : "source", "value" : "asdf" }
+ ]
+ }
+ ]
+ }
+
+Exclude a key or keys in dump of rows
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Dump a table except for the rows excluded with the -x option. Multiple keys can be used.
+
+Example::
+
+ sstabledump /var/lib/cassandra/data/keyspace/eventlog-65c429e08c5a11e8939edf4f403979ef/mc-1-big-Data.db -x 3578d7de-c60d-4599-aefb-3f22a07b2bc6 d18250c0-84fc-4d40-b957-4248dc9d790e > eventlog_dump_2018Jul26_excludekeys
+
+ cat eventlog_dump_2018Jul26_excludekeys
+ [
+ {
+ "partition" : {
+ "key" : [ "cf188983-d85b-48d6-9365-25005289beb2" ],
+ "position" : 0
+ },
+ "rows" : [
+ {
+ "type" : "row",
+ "position" : 182,
+ "liveness_info" : { "tstamp" : "2018-07-20T20:22:27.028809Z" },
+ "cells" : [
+ { "name" : "event", "value" : "party" },
+ { "name" : "insertedtimestamp", "value" : "2018-07-20 20:22:27.055Z" },
+ { "name" : "source", "value" : "asdf" }
+ ]
+ }
+ ]
+ }
+
+Display raw timestamps
+^^^^^^^^^^^^^^^^^^^^^^
+
+By default, dates are displayed in iso8601 date format. Using the -t option will dump the data with the raw timestamp.
+
+Example::
+
+ sstabledump /var/lib/cassandra/data/keyspace/eventlog-65c429e08c5a11e8939edf4f403979ef/mc-1-big-Data.db -t -k cf188983-d85b-48d6-9365-25005289beb2 > eventlog_dump_2018Jul26_times
+
+ cat eventlog_dump_2018Jul26_times
+ [
+ {
+ "partition" : {
+ "key" : [ "cf188983-d85b-48d6-9365-25005289beb2" ],
+ "position" : 124
+ },
+ "rows" : [
+ {
+ "type" : "row",
+ "position" : 182,
+ "liveness_info" : { "tstamp" : "1532118147028809" },
+ "cells" : [
+ { "name" : "event", "value" : "party" },
+ { "name" : "insertedtimestamp", "value" : "2018-07-20 20:22:27.055Z" },
+ { "name" : "source", "value" : "asdf" }
+ ]
+ }
+ ]
+ }
+
+
+Display internal structure in output
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Dump the table in a format that reflects the internal structure.
+
+Example::
+
+ sstabledump /var/lib/cassandra/data/keyspace/eventlog-65c429e08c5a11e8939edf4f403979ef/mc-1-big-Data.db -d > eventlog_dump_2018Jul26_d
+
+ cat eventlog_dump_2018Jul26_d
+ [3578d7de-c60d-4599-aefb-3f22a07b2bc6]@0 Row[info=[ts=1532118188378711] ]: | [event=party ts=1532118188378711], [insertedtimestamp=2018-07-20 20:23Z ts=1532118188378711], [source=asdf ts=1532118188378711]
+ [d18250c0-84fc-4d40-b957-4248dc9d790e]@62 Row[info=[ts=1532118187783522] ]: | [event=party ts=1532118187783522], [insertedtimestamp=2018-07-20 20:23Z ts=1532118187783522], [source=asdf ts=1532118187783522]
+ [cf188983-d85b-48d6-9365-25005289beb2]@124 Row[info=[ts=1532118147028809] ]: | [event=party ts=1532118147028809], [insertedtimestamp=2018-07-20 20:22Z ts=1532118147028809], [source=asdf ts=1532118147028809]
+
+
+
+
+
diff --git a/doc/source/tools/sstable/sstableexpiredblockers.rst b/doc/source/tools/sstable/sstableexpiredblockers.rst
new file mode 100644
index 0000000..ec83794
--- /dev/null
+++ b/doc/source/tools/sstable/sstableexpiredblockers.rst
@@ -0,0 +1,48 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements. See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership. The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License. You may obtain a copy of the License at
+..
+.. http://www.apache.org/licenses/LICENSE-2.0
+..
+.. Unless required by applicable law or agreed to in writing, software
+.. distributed under the License is distributed on an "AS IS" BASIS,
+.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+.. See the License for the specific language governing permissions and
+.. limitations under the License.
+
+sstableexpiredblockers
+----------------------
+
+During compaction, entire sstables can be dropped if they contain only expired tombstones, and if it is guaranteed that the data is not newer than the data in other sstables. An expired sstable can be blocked from getting dropped if its newest timestamp is newer than the oldest data in another sstable.
+
+This tool is used to list all sstables that are blocking other sstables from getting dropped (by having older data than the newest tombstone in an expired sstable) so a user can figure out why certain sstables are still on disk.
+
+ref: https://issues.apache.org/jira/browse/CASSANDRA-10015
+
+Cassandra must be stopped before this tool is executed, or unexpected results will occur. Note: the script does not verify that Cassandra is stopped.
+
+Usage
+^^^^^
+
+sstableexpiredblockers <keyspace> <table>
+
+Output blocked sstables
+^^^^^^^^^^^^^^^^^^^^^^^
+
+If the sstables exist for the table, but no tables have older data than the newest tombstone in an expired sstable, the script will return nothing.
+
+Otherwise, the script will return `<sstable> blocks <#> expired sstables from getting dropped` followed by a list of the blocked sstables.
+
+Example::
+
+ sstableexpiredblockers keyspace1 standard1
+
+ [BigTableReader(path='/var/lib/cassandra/data/keyspace1/standard1-0665ae80b2d711e886c66d2c86545d91/mc-2-big-Data.db') (minTS = 5, maxTS = 5, maxLDT = 2147483647)], blocks 1 expired sstables from getting dropped: [BigTableReader(path='/var/lib/cassandra/data/keyspace1/standard1-0665ae80b2d711e886c66d2c86545d91/mc-3-big-Data.db') (minTS = 1536349775157606, maxTS = 1536349780311159, maxLDT = 1536349780)],
+
+ [BigTableReader(path='/var/lib/cassandra/data/keyspace1/standard1-0665ae80b2d711e886c66d2c86545d91/mc-1-big-Data.db') (minTS = 1, maxTS = 10, maxLDT = 2147483647)], blocks 1 expired sstables from getting dropped: [BigTableReader(path='/var/lib/cassandra/data/keyspace1/standard1-0665ae80b2d711e886c66d2c86545d91/mc-3-big-Data.db') (minTS = 1536349775157606, maxTS = 1536349780311159, maxLDT = 1536349780)],
+
+
diff --git a/doc/source/tools/sstable/sstablelevelreset.rst b/doc/source/tools/sstable/sstablelevelreset.rst
new file mode 100644
index 0000000..7069094
--- /dev/null
+++ b/doc/source/tools/sstable/sstablelevelreset.rst
@@ -0,0 +1,82 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements. See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership. The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License. You may obtain a copy of the License at
+..
+.. http://www.apache.org/licenses/LICENSE-2.0
+..
+.. Unless required by applicable law or agreed to in writing, software
+.. distributed under the License is distributed on an "AS IS" BASIS,
+.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+.. See the License for the specific language governing permissions and
+.. limitations under the License.
+
+sstablelevelreset
+-----------------
+
+If LeveledCompactionStrategy is set, this script can be used to reset level to 0 on a given set of sstables. This is useful if you want to, for example, change the minimum sstable size, and therefore restart the compaction process using this new configuration.
+
+See http://cassandra.apache.org/doc/latest/operating/compaction.html#leveled-compaction-strategy for information on how levels are used in this compaction strategy.
+
+Cassandra must be stopped before this tool is executed, or unexpected results will occur. Note: the script does not verify that Cassandra is stopped.
+
+ref: https://issues.apache.org/jira/browse/CASSANDRA-5271
+
+Usage
+^^^^^
+
+sstablelevelreset --really-reset <keyspace> <table>
+
+The really-reset flag is required, to ensure this intrusive command is not run accidentally.
+
+Table not found
+^^^^^^^^^^^^^^^
+
+If the keyspace and/or table is not in the schema (e.g., if you misspelled the table name), the script will return an error.
+
+Example::
+
+ ColumnFamily not found: keyspace/evenlog.
+
+Table has no sstables
+^^^^^^^^^^^^^^^^^^^^^
+
+Example::
+
+ Found no sstables, did you give the correct keyspace/table?
+
+
+Table already at level 0
+^^^^^^^^^^^^^^^^^^^^^^^^
+
+The script will not set the level if it is already set to 0.
+
+Example::
+
+ Skipped /var/lib/cassandra/data/keyspace/eventlog-65c429e08c5a11e8939edf4f403979ef/mc-1-big-Data.db since it is already on level 0
+
+Table levels reduced to 0
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+If the level is not already 0, then this will reset it to 0.
+
+Example::
+
+ sstablemetadata /var/lib/cassandra/data/keyspace/eventlog-6365332094dd11e88f324f9c503e4753/mc-8-big-Data.db | grep -i level
+ SSTable Level: 1
+
+ sstablelevelreset --really-reset keyspace eventlog
+ Changing level from 1 to 0 on /var/lib/cassandra/data/keyspace/eventlog-6365332094dd11e88f324f9c503e4753/mc-8-big-Data.db
+
+ sstablemetadata /var/lib/cassandra/data/keyspace/eventlog-6365332094dd11e88f324f9c503e4753/mc-8-big-Data.db | grep -i level
+ SSTable Level: 0
+
+
+
+
+
+
+
diff --git a/doc/source/tools/sstable/sstableloader.rst b/doc/source/tools/sstable/sstableloader.rst
new file mode 100644
index 0000000..a9b3734
--- /dev/null
+++ b/doc/source/tools/sstable/sstableloader.rst
@@ -0,0 +1,273 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements. See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership. The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License. You may obtain a copy of the License at
+..
+.. http://www.apache.org/licenses/LICENSE-2.0
+..
+.. Unless required by applicable law or agreed to in writing, software
+.. distributed under the License is distributed on an "AS IS" BASIS,
+.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+.. See the License for the specific language governing permissions and
+.. limitations under the License.
+
+sstableloader
+---------------
+
+Bulk-load the sstables found in the directory <dir_path> to the configured cluster. The parent directories of <dir_path> are used as the target keyspace/table name. For example, to load an sstable named ma-1-big-Data.db into keyspace1/standard1, you will need to have the files ma-1-big-Data.db and ma-1-big-Index.db in a directory /path/to/keyspace1/standard1/. The tool will create new sstables, and does not clean up your copied files.
+
+Several of the options listed below don't work quite as intended, and in those cases, workarounds are mentioned for specific use cases.
+
+To avoid having the sstable files to be loaded compacted while reading them, place the files in an alternate keyspace/table path than the data directory.
+
+ref: https://issues.apache.org/jira/browse/CASSANDRA-1278
+
+Cassandra must be stopped before this tool is executed, or unexpected results will occur. Note: the script does not verify that Cassandra is stopped.
+
+Usage
+^^^^^
+
+sstableloader <options> <dir_path>
+
+=================================================== ================================================================================
+-d, --nodes <initial hosts> Required. Try to connect to these hosts (comma-separated)
+ initially for ring information
+-u, --username <username> username for Cassandra authentication
+-pw, --password <password> password for Cassandra authentication
+-p, --port <native transport port> port used for native connection (default 9042)
+-sp, --storage-port <storage port> port used for internode communication (default 7000)
+-ssp, --ssl-storage-port <ssl storage port> port used for TLS internode communication (default 7001)
+--no-progress don't display progress
+-t, --throttle <throttle> throttle speed in Mbits (default unlimited)
+-idct, --inter-dc-throttle <inter-dc-throttle> inter-datacenter throttle speed in Mbits (default unlimited)
+-cph, --connections-per-host <connectionsPerHost> number of concurrent connections-per-host
+-i, --ignore <NODES> don't stream to this (comma separated) list of nodes
+-alg, --ssl-alg <ALGORITHM> Client SSL: algorithm (default: SunX509)
+-ciphers, --ssl-ciphers <CIPHER-SUITES> Client SSL: comma-separated list of encryption suites to use
+-ks, --keystore <KEYSTORE> Client SSL: full path to keystore
+-kspw, --keystore-password <KEYSTORE-PASSWORD> Client SSL: password of the keystore
+-st, --store-type <STORE-TYPE> Client SSL: type of store
+-ts, --truststore <TRUSTSTORE> Client SSL: full path to truststore
+-tspw, --truststore-password <TRUSTSTORE-PASSWORD> Client SSL: password of the truststore
+-prtcl, --ssl-protocol <PROTOCOL> Client SSL: connections protocol to use (default: TLS)
+-ap, --auth-provider <auth provider> custom AuthProvider class name for cassandra authentication
+-f, --conf-path <path to config file> cassandra.yaml file path for streaming throughput and client/server SSL
+-v, --verbose verbose output
+-h, --help display this help message
+=================================================== ================================================================================
+
+You can provide a cassandra.yaml file with the -f command line option to set up streaming throughput, and client and server encryption options. Only stream_throughput_outbound_megabits_per_sec, server_encryption_options, and client_encryption_options are read from yaml. You can override options read from cassandra.yaml with corresponding command line options.
+
+Load sstables from a Snapshot
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Copy the snapshot sstables into an accessible directory and use sstableloader to restore them.
+
+Example::
+
+ cp snapshots/1535397029191/* /path/to/keyspace1/standard1/
+
+ sstableloader --nodes 172.17.0.2 /var/lib/cassandra/loadme/keyspace1/standard1-f8a4fa30aa2a11e8af27091830ac5256/
+ Established connection to initial hosts
+ Opening sstables and calculating sections to stream
+ Streaming relevant part of /var/lib/cassandra/loadme/keyspace1/standard1-f8a4fa30aa2a11e8af27091830ac5256/ma-3-big-Data.db to [/172.17.0.2]
+ progress: [/172.17.0.2]0:1/1 100% total: 100% 0 MB/s(avg: 1 MB/s)
+ Summary statistics:
+ Connections per host: : 1
+ Total files transferred: : 1
+ Total bytes transferred: : 4700000
+ Total duration (ms): : 4390
+ Average transfer rate (MB/s): : 1
+ Peak transfer rate (MB/s): : 1
+
+The -d or --nodes option is required, or the script will not run.
+
+Example::
+
+ sstableloader /var/lib/cassandra/loadme/keyspace1/standard1-f8a4fa30aa2a11e8af27091830ac5256/
+ Initial hosts must be specified (-d)
+
+Use a Config File for SSL Clusters
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+If SSL encryption is enabled in the cluster, use the --conf-path option with sstableloader to point the tool to the cassandra.yaml with the relevant server_encryption_options (e.g., truststore location, algorithm). This will work better than passing individual ssl options shown above to sstableloader on the command line.
+
+Example::
+
+ sstableloader --nodes 172.17.0.2 --conf-path /etc/cassandra/cassandra.yaml /var/lib/cassandra/loadme/keyspace1/standard1-0974e5a0aa5811e8a0a06d2c86545d91/snapshots/
+ Established connection to initial hosts
+ Opening sstables and calculating sections to stream
+ Streaming relevant part of /var/lib/cassandra/loadme/keyspace1/standard1-0974e5a0aa5811e8a0a06d2c86545d91/mc-1-big-Data.db to [/172.17.0.2]
+ progress: [/172.17.0.2]0:0/1 1 % total: 1% 9.165KiB/s (avg: 9.165KiB/s)
+ progress: [/172.17.0.2]0:0/1 2 % total: 2% 5.147MiB/s (avg: 18.299KiB/s)
+ progress: [/172.17.0.2]0:0/1 4 % total: 4% 9.751MiB/s (avg: 27.423KiB/s)
+ progress: [/172.17.0.2]0:0/1 5 % total: 5% 8.203MiB/s (avg: 36.524KiB/s)
+ ...
+ progress: [/172.17.0.2]0:1/1 100% total: 100% 0.000KiB/s (avg: 480.513KiB/s)
+
+ Summary statistics:
+ Connections per host : 1
+ Total files transferred : 1
+ Total bytes transferred : 4.387MiB
+ Total duration : 9356 ms
+ Average transfer rate : 480.105KiB/s
+ Peak transfer rate : 586.410KiB/s
+
+Hide Progress Output
+^^^^^^^^^^^^^^^^^^^^
+
+To hide the output of progress and the summary statistics (e.g., if you wanted to use this tool in a script), use the --no-progress option.
+
+Example::
+
+ sstableloader --nodes 172.17.0.2 --no-progress /var/lib/cassandra/loadme/keyspace1/standard1-f8a4fa30aa2a11e8af27091830ac5256/
+ Established connection to initial hosts
+ Opening sstables and calculating sections to stream
+ Streaming relevant part of /var/lib/cassandra/loadme/keyspace1/standard1-f8a4fa30aa2a11e8af27091830ac5256/ma-4-big-Data.db to [/172.17.0.2]
+
+Get More Detail
+^^^^^^^^^^^^^^^
+
+Using the --verbose option will provide much more progress output.
+
+Example::
+
+ sstableloader --nodes 172.17.0.2 --verbose /var/lib/cassandra/loadme/keyspace1/standard1-0974e5a0aa5811e8a0a06d2c86545d91/
+ Established connection to initial hosts
+ Opening sstables and calculating sections to stream
+ Streaming relevant part of /var/lib/cassandra/loadme/keyspace1/standard1-0974e5a0aa5811e8a0a06d2c86545d91/mc-1-big-Data.db to [/172.17.0.2]
+ progress: [/172.17.0.2]0:0/1 1 % total: 1% 12.056KiB/s (avg: 12.056KiB/s)
+ progress: [/172.17.0.2]0:0/1 2 % total: 2% 9.092MiB/s (avg: 24.081KiB/s)
+ progress: [/172.17.0.2]0:0/1 4 % total: 4% 18.832MiB/s (avg: 36.099KiB/s)
+ progress: [/172.17.0.2]0:0/1 5 % total: 5% 2.253MiB/s (avg: 47.882KiB/s)
+ progress: [/172.17.0.2]0:0/1 7 % total: 7% 6.388MiB/s (avg: 59.743KiB/s)
+ progress: [/172.17.0.2]0:0/1 8 % total: 8% 14.606MiB/s (avg: 71.635KiB/s)
+ progress: [/172.17.0.2]0:0/1 9 % total: 9% 8.880MiB/s (avg: 83.465KiB/s)
+ progress: [/172.17.0.2]0:0/1 11 % total: 11% 5.217MiB/s (avg: 95.176KiB/s)
+ progress: [/172.17.0.2]0:0/1 12 % total: 12% 12.563MiB/s (avg: 106.975KiB/s)
+ progress: [/172.17.0.2]0:0/1 14 % total: 14% 2.550MiB/s (avg: 118.322KiB/s)
+ progress: [/172.17.0.2]0:0/1 15 % total: 15% 16.638MiB/s (avg: 130.063KiB/s)
+ progress: [/172.17.0.2]0:0/1 17 % total: 17% 17.270MiB/s (avg: 141.793KiB/s)
+ progress: [/172.17.0.2]0:0/1 18 % total: 18% 11.280MiB/s (avg: 153.452KiB/s)
+ progress: [/172.17.0.2]0:0/1 19 % total: 19% 2.903MiB/s (avg: 164.603KiB/s)
+ progress: [/172.17.0.2]0:0/1 21 % total: 21% 6.744MiB/s (avg: 176.061KiB/s)
+ progress: [/172.17.0.2]0:0/1 22 % total: 22% 6.011MiB/s (avg: 187.440KiB/s)
+ progress: [/172.17.0.2]0:0/1 24 % total: 24% 9.690MiB/s (avg: 198.920KiB/s)
+ progress: [/172.17.0.2]0:0/1 25 % total: 25% 11.481MiB/s (avg: 210.412KiB/s)
+ progress: [/172.17.0.2]0:0/1 27 % total: 27% 9.957MiB/s (avg: 221.848KiB/s)
+ progress: [/172.17.0.2]0:0/1 28 % total: 28% 10.270MiB/s (avg: 233.265KiB/s)
+ progress: [/172.17.0.2]0:0/1 29 % total: 29% 7.812MiB/s (avg: 244.571KiB/s)
+ progress: [/172.17.0.2]0:0/1 31 % total: 31% 14.843MiB/s (avg: 256.021KiB/s)
+ progress: [/172.17.0.2]0:0/1 32 % total: 32% 11.457MiB/s (avg: 267.394KiB/s)
+ progress: [/172.17.0.2]0:0/1 34 % total: 34% 6.550MiB/s (avg: 278.536KiB/s)
+ progress: [/172.17.0.2]0:0/1 35 % total: 35% 9.115MiB/s (avg: 289.782KiB/s)
+ progress: [/172.17.0.2]0:0/1 37 % total: 37% 11.054MiB/s (avg: 301.064KiB/s)
+ progress: [/172.17.0.2]0:0/1 38 % total: 38% 10.449MiB/s (avg: 312.307KiB/s)
+ progress: [/172.17.0.2]0:0/1 39 % total: 39% 1.646MiB/s (avg: 321.665KiB/s)
+ progress: [/172.17.0.2]0:0/1 41 % total: 41% 13.300MiB/s (avg: 332.872KiB/s)
+ progress: [/172.17.0.2]0:0/1 42 % total: 42% 14.370MiB/s (avg: 344.082KiB/s)
+ progress: [/172.17.0.2]0:0/1 44 % total: 44% 16.734MiB/s (avg: 355.314KiB/s)
+ progress: [/172.17.0.2]0:0/1 45 % total: 45% 22.245MiB/s (avg: 366.592KiB/s)
+ progress: [/172.17.0.2]0:0/1 47 % total: 47% 25.561MiB/s (avg: 377.882KiB/s)
+ progress: [/172.17.0.2]0:0/1 48 % total: 48% 24.543MiB/s (avg: 389.155KiB/s)
+ progress: [/172.17.0.2]0:0/1 49 % total: 49% 4.894MiB/s (avg: 399.688KiB/s)
+ progress: [/172.17.0.2]0:0/1 51 % total: 51% 8.331MiB/s (avg: 410.559KiB/s)
+ progress: [/172.17.0.2]0:0/1 52 % total: 52% 5.771MiB/s (avg: 421.150KiB/s)
+ progress: [/172.17.0.2]0:0/1 54 % total: 54% 8.738MiB/s (avg: 431.983KiB/s)
+ progress: [/172.17.0.2]0:0/1 55 % total: 55% 3.406MiB/s (avg: 441.911KiB/s)
+ progress: [/172.17.0.2]0:0/1 56 % total: 56% 9.791MiB/s (avg: 452.730KiB/s)
+ progress: [/172.17.0.2]0:0/1 58 % total: 58% 3.401MiB/s (avg: 462.545KiB/s)
+ progress: [/172.17.0.2]0:0/1 59 % total: 59% 5.280MiB/s (avg: 472.840KiB/s)
+ progress: [/172.17.0.2]0:0/1 61 % total: 61% 12.232MiB/s (avg: 483.663KiB/s)
+ progress: [/172.17.0.2]0:0/1 62 % total: 62% 9.258MiB/s (avg: 494.325KiB/s)
+ progress: [/172.17.0.2]0:0/1 64 % total: 64% 2.877MiB/s (avg: 503.640KiB/s)
+ progress: [/172.17.0.2]0:0/1 65 % total: 65% 7.461MiB/s (avg: 514.078KiB/s)
+ progress: [/172.17.0.2]0:0/1 66 % total: 66% 24.247MiB/s (avg: 525.018KiB/s)
+ progress: [/172.17.0.2]0:0/1 68 % total: 68% 9.348MiB/s (avg: 535.563KiB/s)
+ progress: [/172.17.0.2]0:0/1 69 % total: 69% 5.130MiB/s (avg: 545.563KiB/s)
+ progress: [/172.17.0.2]0:0/1 71 % total: 71% 19.861MiB/s (avg: 556.392KiB/s)
+ progress: [/172.17.0.2]0:0/1 72 % total: 72% 15.501MiB/s (avg: 567.122KiB/s)
+ progress: [/172.17.0.2]0:0/1 74 % total: 74% 5.031MiB/s (avg: 576.996KiB/s)
+ progress: [/172.17.0.2]0:0/1 75 % total: 75% 22.771MiB/s (avg: 587.813KiB/s)
+ progress: [/172.17.0.2]0:0/1 76 % total: 76% 22.780MiB/s (avg: 598.619KiB/s)
+ progress: [/172.17.0.2]0:0/1 78 % total: 78% 20.684MiB/s (avg: 609.386KiB/s)
+ progress: [/172.17.0.2]0:0/1 79 % total: 79% 22.920MiB/s (avg: 620.173KiB/s)
+ progress: [/172.17.0.2]0:0/1 81 % total: 81% 7.458MiB/s (avg: 630.333KiB/s)
+ progress: [/172.17.0.2]0:0/1 82 % total: 82% 22.993MiB/s (avg: 641.090KiB/s)
+ progress: [/172.17.0.2]0:0/1 84 % total: 84% 21.392MiB/s (avg: 651.814KiB/s)
+ progress: [/172.17.0.2]0:0/1 85 % total: 85% 7.732MiB/s (avg: 661.938KiB/s)
+ progress: [/172.17.0.2]0:0/1 86 % total: 86% 3.476MiB/s (avg: 670.892KiB/s)
+ progress: [/172.17.0.2]0:0/1 88 % total: 88% 19.889MiB/s (avg: 681.521KiB/s)
+ progress: [/172.17.0.2]0:0/1 89 % total: 89% 21.077MiB/s (avg: 692.162KiB/s)
+ progress: [/172.17.0.2]0:0/1 91 % total: 91% 24.062MiB/s (avg: 702.835KiB/s)
+ progress: [/172.17.0.2]0:0/1 92 % total: 92% 19.798MiB/s (avg: 713.431KiB/s)
+ progress: [/172.17.0.2]0:0/1 94 % total: 94% 17.591MiB/s (avg: 723.965KiB/s)
+ progress: [/172.17.0.2]0:0/1 95 % total: 95% 13.725MiB/s (avg: 734.361KiB/s)
+ progress: [/172.17.0.2]0:0/1 96 % total: 96% 16.737MiB/s (avg: 744.846KiB/s)
+ progress: [/172.17.0.2]0:0/1 98 % total: 98% 22.701MiB/s (avg: 755.443KiB/s)
+ progress: [/172.17.0.2]0:0/1 99 % total: 99% 18.718MiB/s (avg: 765.954KiB/s)
+ progress: [/172.17.0.2]0:1/1 100% total: 100% 6.613MiB/s (avg: 767.802KiB/s)
+ progress: [/172.17.0.2]0:1/1 100% total: 100% 0.000KiB/s (avg: 670.295KiB/s)
+
+ Summary statistics:
+ Connections per host : 1
+ Total files transferred : 1
+ Total bytes transferred : 4.387MiB
+ Total duration : 6706 ms
+ Average transfer rate : 669.835KiB/s
+ Peak transfer rate : 767.802KiB/s
+
+
+Throttling Load
+^^^^^^^^^^^^^^^
+
+To prevent the table loader from overloading the system resources, you can throttle the process with the --throttle option. The default is unlimited (no throttling). Throttle units are in megabits. Note that the total duration is increased in the example below.
+
+Example::
+
+ sstableloader --nodes 172.17.0.2 --throttle 1 /var/lib/cassandra/loadme/keyspace1/standard1-f8a4fa30aa2a11e8af27091830ac5256/
+ Established connection to initial hosts
+ Opening sstables and calculating sections to stream
+ Streaming relevant part of /var/lib/cassandra/loadme/keyspace1/standard1-f8a4fa30aa2a11e8af27091830ac5256/ma-6-big-Data.db to [/172.17.0.2]
+ progress: [/172.17.0.2]0:1/1 100% total: 100% 0 MB/s(avg: 0 MB/s)
+ Summary statistics:
+ Connections per host: : 1
+ Total files transferred: : 1
+ Total bytes transferred: : 4595705
+ Total duration (ms): : 37634
+ Average transfer rate (MB/s): : 0
+ Peak transfer rate (MB/s): : 0
+
+Speeding up Load
+^^^^^^^^^^^^^^^^
+
+To speed up the load process, the number of connections per host can be increased.
+
+Example::
+
+ sstableloader --nodes 172.17.0.2 --connections-per-host 100 /var/lib/cassandra/loadme/keyspace1/standard1-f8a4fa30aa2a11e8af27091830ac5256/
+ Established connection to initial hosts
+ Opening sstables and calculating sections to stream
+ Streaming relevant part of /var/lib/cassandra/loadme/keyspace1/standard1-f8a4fa30aa2a11e8af27091830ac5256/ma-9-big-Data.db to [/172.17.0.2]
+ progress: [/172.17.0.2]0:1/1 100% total: 100% 0 MB/s(avg: 1 MB/s)
+ Summary statistics:
+ Connections per host: : 100
+ Total files transferred: : 1
+ Total bytes transferred: : 4595705
+ Total duration (ms): : 3486
+ Average transfer rate (MB/s): : 1
+ Peak transfer rate (MB/s): : 1
+
+This small data set doesn't benefit much from the increase in connections per host, but note that the total duration has decreased in this example.
+
+
+
+
+
+
+
+
+
diff --git a/doc/source/tools/sstable/sstablemetadata.rst b/doc/source/tools/sstable/sstablemetadata.rst
new file mode 100644
index 0000000..0a7a422
--- /dev/null
+++ b/doc/source/tools/sstable/sstablemetadata.rst
@@ -0,0 +1,300 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements. See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership. The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License. You may obtain a copy of the License at
+..
+.. http://www.apache.org/licenses/LICENSE-2.0
+..
+.. Unless required by applicable law or agreed to in writing, software
+.. distributed under the License is distributed on an "AS IS" BASIS,
+.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+.. See the License for the specific language governing permissions and
+.. limitations under the License.
+
+sstablemetadata
+---------------
+
+Print information about an sstable from the related Statistics.db and Summary.db files to standard output.
+
+ref: https://issues.apache.org/jira/browse/CASSANDRA-7159 and https://issues.apache.org/jira/browse/CASSANDRA-10838
+
+Cassandra must be stopped before this tool is executed, or unexpected results will occur. Note: the script does not verify that Cassandra is stopped.
+
+Usage
+^^^^^
+
+sstablemetadata <options> <sstable filename(s)>
+
+========================= ================================================================================
+--gc_grace_seconds <arg> The gc_grace_seconds to use when calculating droppable tombstones
+========================= ================================================================================
+
+Print all the metadata
+^^^^^^^^^^^^^^^^^^^^^^
+
+Run sstablemetadata against the *Data.db file(s) related to a table. If necessary, find the *Data.db file(s) using sstableutil.
+
+Example::
+
+ sstableutil keyspace1 standard1 | grep Data
+ /var/lib/cassandra/data/keyspace1/standard1-f6845640a6cb11e8b6836d2c86545d91/mc-1-big-Data.db
+
+ sstablemetadata /var/lib/cassandra/data/keyspace1/standard1-f6845640a6cb11e8b6836d2c86545d91/mc-1-big-Data.db
+
+ SSTable: /var/lib/cassandra/data/keyspace1/standard1-f6845640a6cb11e8b6836d2c86545d91/mc-1-big
+ Partitioner: org.apache.cassandra.dht.Murmur3Partitioner
+ Bloom Filter FP chance: 0.010000
+ Minimum timestamp: 1535025576141000
+ Maximum timestamp: 1535025604309000
+ SSTable min local deletion time: 2147483647
+ SSTable max local deletion time: 2147483647
+ Compressor: org.apache.cassandra.io.compress.LZ4Compressor
+ TTL min: 86400
+ TTL max: 86400
+ First token: -9223004712949498654 (key=39373333373831303130)
+ Last token: 9222554117157811897 (key=4f3438394e39374d3730)
+ Estimated droppable tombstones: 0.9188263888888889
+ SSTable Level: 0
+ Repaired at: 0
+ Replay positions covered: {CommitLogPosition(segmentId=1535025390651, position=226400)=CommitLogPosition(segmentId=1535025390651, position=6849139)}
+ totalColumnsSet: 100000
+ totalRows: 20000
+ Estimated tombstone drop times:
+ 1535039100: 80390
+ 1535039160: 5645
+ 1535039220: 13965
+ Count Row Size Cell Count
+ 1 0 0
+ 2 0 0
+ 3 0 0
+ 4 0 0
+ 5 0 20000
+ 6 0 0
+ 7 0 0
+ 8 0 0
+ 10 0 0
+ 12 0 0
+ 14 0 0
+ 17 0 0
+ 20 0 0
+ 24 0 0
+ 29 0 0
+ 35 0 0
+ 42 0 0
+ 50 0 0
+ 60 0 0
+ 72 0 0
+ 86 0 0
+ 103 0 0
+ 124 0 0
+ 149 0 0
+ 179 0 0
+ 215 0 0
+ 258 20000 0
+ 310 0 0
+ 372 0 0
+ 446 0 0
+ 535 0 0
+ 642 0 0
+ 770 0 0
+ 924 0 0
+ 1109 0 0
+ 1331 0 0
+ 1597 0 0
+ 1916 0 0
+ 2299 0 0
+ 2759 0 0
+ 3311 0 0
+ 3973 0 0
+ 4768 0 0
+ 5722 0 0
+ 6866 0 0
+ 8239 0 0
+ 9887 0 0
+ 11864 0 0
+ 14237 0 0
+ 17084 0 0
+ 20501 0 0
+ 24601 0 0
+ 29521 0 0
+ 35425 0 0
+ 42510 0 0
+ 51012 0 0
+ 61214 0 0
+ 73457 0 0
+ 88148 0 0
+ 105778 0 0
+ 126934 0 0
+ 152321 0 0
+ 182785 0 0
+ 219342 0 0
+ 263210 0 0
+ 315852 0 0
+ 379022 0 0
+ 454826 0 0
+ 545791 0 0
+ 654949 0 0
+ 785939 0 0
+ 943127 0 0
+ 1131752 0 0
+ 1358102 0 0
+ 1629722 0 0
+ 1955666 0 0
+ 2346799 0 0
+ 2816159 0 0
+ 3379391 0 0
+ 4055269 0 0
+ 4866323 0 0
+ 5839588 0 0
+ 7007506 0 0
+ 8409007 0 0
+ 10090808 0 0
+ 12108970 0 0
+ 14530764 0 0
+ 17436917 0 0
+ 20924300 0 0
+ 25109160 0 0
+ 30130992 0 0
+ 36157190 0 0
+ 43388628 0 0
+ 52066354 0 0
+ 62479625 0 0
+ 74975550 0 0
+ 89970660 0 0
+ 107964792 0 0
+ 129557750 0 0
+ 155469300 0 0
+ 186563160 0 0
+ 223875792 0 0
+ 268650950 0 0
+ 322381140 0 0
+ 386857368 0 0
+ 464228842 0 0
+ 557074610 0 0
+ 668489532 0 0
+ 802187438 0 0
+ 962624926 0 0
+ 1155149911 0 0
+ 1386179893 0 0
+ 1663415872 0 0
+ 1996099046 0 0
+ 2395318855 0 0
+ 2874382626 0
+ 3449259151 0
+ 4139110981 0
+ 4966933177 0
+ 5960319812 0
+ 7152383774 0
+ 8582860529 0
+ 10299432635 0
+ 12359319162 0
+ 14831182994 0
+ 17797419593 0
+ 21356903512 0
+ 25628284214 0
+ 30753941057 0
+ 36904729268 0
+ 44285675122 0
+ 53142810146 0
+ 63771372175 0
+ 76525646610 0
+ 91830775932 0
+ 110196931118 0
+ 132236317342 0
+ 158683580810 0
+ 190420296972 0
+ 228504356366 0
+ 274205227639 0
+ 329046273167 0
+ 394855527800 0
+ 473826633360 0
+ 568591960032 0
+ 682310352038 0
+ 818772422446 0
+ 982526906935 0
+ 1179032288322 0
+ 1414838745986 0
+ Estimated cardinality: 20196
+ EncodingStats minTTL: 0
+ EncodingStats minLocalDeletionTime: 1442880000
+ EncodingStats minTimestamp: 1535025565275000
+ KeyType: org.apache.cassandra.db.marshal.BytesType
+ ClusteringTypes: [org.apache.cassandra.db.marshal.UTF8Type]
+ StaticColumns: {C3:org.apache.cassandra.db.marshal.BytesType, C4:org.apache.cassandra.db.marshal.BytesType, C0:org.apache.cassandra.db.marshal.BytesType, C1:org.apache.cassandra.db.marshal.BytesType, C2:org.apache.cassandra.db.marshal.BytesType}
+ RegularColumns: {}
+
+Specify gc grace seconds
+^^^^^^^^^^^^^^^^^^^^^^^^
+
+To see the ratio of droppable tombstones given a configured gc grace seconds, use the gc_grace_seconds option. Because the sstablemetadata tool doesn't access the schema directly, this is a way to more accurately estimate droppable tombstones -- for example, if you pass in gc_grace_seconds matching what is configured in the schema. The gc_grace_seconds value provided is subtracted from the curent machine time (in seconds).
+
+ref: https://issues.apache.org/jira/browse/CASSANDRA-12208
+
+Example::
+
+ sstablemetadata /var/lib/cassandra/data/keyspace1/standard1-41b52700b4ed11e896476d2c86545d91/mc-12-big-Data.db | grep "Estimated tombstone drop times" -A4
+ Estimated tombstone drop times:
+ 1536599100: 1
+ 1536599640: 1
+ 1536599700: 2
+
+ echo $(date +%s)
+ 1536602005
+
+ # if gc_grace_seconds was configured at 100, all of the tombstones would be currently droppable
+ sstablemetadata --gc_grace_seconds 100 /var/lib/cassandra/data/keyspace1/standard1-41b52700b4ed11e896476d2c86545d91/mc-12-big-Data.db | grep "Estimated droppable tombstones"
+ Estimated droppable tombstones: 4.0E-5
+
+ # if gc_grace_seconds was configured at 4700, some of the tombstones would be currently droppable
+ sstablemetadata --gc_grace_seconds 4700 /var/lib/cassandra/data/keyspace1/standard1-41b52700b4ed11e896476d2c86545d91/mc-12-big-Data.db | grep "Estimated droppable tombstones"
+ Estimated droppable tombstones: 9.61111111111111E-6
+
+ # if gc_grace_seconds was configured at 100, none of the tombstones would be currently droppable
+ sstablemetadata --gc_grace_seconds 5000 /var/lib/cassandra/data/keyspace1/standard1-41b52700b4ed11e896476d2c86545d91/mc-12-big-Data.db | grep "Estimated droppable tombstones"
+ Estimated droppable tombstones: 0.0
+
+Explanation of each value printed above
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+=================================== ================================================================================
+ Value Explanation
+=================================== ================================================================================
+SSTable prefix of the sstable filenames related to this sstable
+Partitioner partitioner type used to distribute data across nodes; defined in cassandra.yaml
+Bloom Filter FP precision of Bloom filter used in reads; defined in the table definition
+Minimum timestamp minimum timestamp of any entry in this sstable, in epoch microseconds
+Maximum timestamp maximum timestamp of any entry in this sstable, in epoch microseconds
+SSTable min local deletion time minimum timestamp of deletion date, based on TTL, in epoch seconds
+SSTable max local deletion time maximum timestamp of deletion date, based on TTL, in epoch seconds
+Compressor blank (-) by default; if not blank, indicates type of compression enabled on the table
+TTL min time-to-live in seconds; default 0 unless defined in the table definition
+TTL max time-to-live in seconds; default 0 unless defined in the table definition
+First token lowest token and related key found in the sstable summary
+Last token highest token and related key found in the sstable summary
+Estimated droppable tombstones ratio of tombstones to columns, using configured gc grace seconds if relevant
+SSTable level compaction level of this sstable, if leveled compaction (LCS) is used
+Repaired at the timestamp this sstable was marked as repaired via sstablerepairedset, in epoch milliseconds
+Replay positions covered the interval of time and commitlog positions related to this sstable
+totalColumnsSet number of cells in the table
+totalRows number of rows in the table
+Estimated tombstone drop times approximate number of rows that will expire, ordered by epoch seconds
+Count Row Size Cell Count two histograms in two columns; one represents distribution of Row Size
+ and the other represents distribution of Cell Count
+Estimated cardinality an estimate of unique values, used for compaction
+EncodingStats* minTTL in epoch milliseconds
+EncodingStats* minLocalDeletionTime in epoch seconds
+EncodingStats* minTimestamp in epoch microseconds
+KeyType the type of partition key, useful in reading and writing data
+ from/to storage; defined in the table definition
+ClusteringTypes the type of clustering key, useful in reading and writing data
+ from/to storage; defined in the table definition
+StaticColumns a list of the shared columns in the table
+RegularColumns a list of non-static, non-key columns in the table
+=================================== ================================================================================
+* For the encoding stats values, the delta of this and the current epoch time is used when encoding and storing data in the most optimal way.
+
+
+
diff --git a/doc/source/tools/sstable/sstableofflinerelevel.rst b/doc/source/tools/sstable/sstableofflinerelevel.rst
new file mode 100644
index 0000000..c031d29
--- /dev/null
+++ b/doc/source/tools/sstable/sstableofflinerelevel.rst
@@ -0,0 +1,95 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements. See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership. The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License. You may obtain a copy of the License at
+..
+.. http://www.apache.org/licenses/LICENSE-2.0
+..
+.. Unless required by applicable law or agreed to in writing, software
+.. distributed under the License is distributed on an "AS IS" BASIS,
+.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+.. See the License for the specific language governing permissions and
+.. limitations under the License.
+
+sstableofflinerelevel
+---------------------
+
+When using LeveledCompactionStrategy, sstables can get stuck at L0 on a recently bootstrapped node, and compactions may never catch up. This tool is used to bump sstables into the highest level possible.
+
+ref: https://issues.apache.org/jira/browse/CASSANDRA-8301
+
+The way this is done is: sstables are storted by their last token. Given an original leveling like this (note that [ ] indicates token boundaries, not sstable size on disk; all sstables are the same size)::
+
+ L3 [][][][][][][][][][][]
+ L2 [ ][ ][ ][ ]
+ L1 [ ][ ]
+ L0 [ ]
+
+Will look like this after being dropped to L0 and sorted by last token (and, to illustrate overlap, the overlapping ones are put on a new line)::
+
+ [][][]
+ [ ][][][]
+ [ ]
+ [ ]
+ ...
+
+Then, we start iterating from the smallest last-token and adding all sstables that do not cause an overlap to a level. We will reconstruct the original leveling top-down. Whenever we add an sstable to the level, we remove it from the sorted list. Once we reach the end of the sorted list, we have a full level, and can start over with the level below.
+
+If we end up with more levels than expected, we put all levels exceeding the expected in L0, for example, original L0 files will most likely be put in a level of its own since they most often overlap many other sstables.
+
+Cassandra must be stopped before this tool is executed, or unexpected results will occur. Note: the script does not verify that Cassandra is stopped.
+
+Usage
+^^^^^
+
+sstableofflinerelevel [--dry-run] <keyspace> <table>
+
+Doing a dry run
+^^^^^^^^^^^^^^^
+
+Use the --dry-run option to see the current level distribution and predicted level after the change.
+
+Example::
+
+ sstableofflinerelevel --dry-run keyspace eventlog
+ For sstables in /var/lib/cassandra/data/keyspace/eventlog-6365332094dd11e88f324f9c503e4753:
+ Current leveling:
+ L0=2
+ Potential leveling:
+ L0=1
+ L1=1
+
+Running a relevel
+^^^^^^^^^^^^^^^^^
+
+Example::
+
+ sstableofflinerelevel keyspace eventlog
+ For sstables in /var/lib/cassandra/data/keyspace/eventlog-6365332094dd11e88f324f9c503e4753:
+ Current leveling:
+ L0=2
+ New leveling:
+ L0=1
+ L1=1
+
+Keyspace or table not found
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+If an invalid keyspace and/or table is provided, an exception will be thrown.
+
+Example::
+
+ sstableofflinerelevel --dry-run keyspace evenlog
+
+ Exception in thread "main" java.lang.IllegalArgumentException: Unknown keyspace/columnFamily keyspace1.evenlog
+ at org.apache.cassandra.tools.SSTableOfflineRelevel.main(SSTableOfflineRelevel.java:96)
+
+
+
+
+
+
+
diff --git a/doc/source/tools/sstable/sstablerepairedset.rst b/doc/source/tools/sstable/sstablerepairedset.rst
new file mode 100644
index 0000000..ebacef3
--- /dev/null
+++ b/doc/source/tools/sstable/sstablerepairedset.rst
@@ -0,0 +1,79 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements. See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership. The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License. You may obtain a copy of the License at
+..
+.. http://www.apache.org/licenses/LICENSE-2.0
+..
+.. Unless required by applicable law or agreed to in writing, software
+.. distributed under the License is distributed on an "AS IS" BASIS,
+.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+.. See the License for the specific language governing permissions and
+.. limitations under the License.
+
+sstablerepairedset
+------------------
+
+Repairs can take a very long time in some environments, for large sizes of data. Use this tool to set the repairedAt status on a given set of sstables, so that repairs can be run on only un-repaired sstables if desired.
+
+Note that running a repair (e.g., via nodetool repair) doesn't set the status of this metadata. Only setting the status of this metadata via this tool does.
+
+ref: https://issues.apache.org/jira/browse/CASSANDRA-5351
+
+Cassandra must be stopped before this tool is executed, or unexpected results will occur. Note: the script does not verify that Cassandra is stopped.
+
+Usage
+^^^^^
+sstablerepairedset --really-set <options> [-f <sstable-list> | <sstables>]
+
+=================================== ================================================================================
+--really-set required if you want to really set the status
+--is-repaired set the repairedAt status to the last modified time
+--is-unrepaired set the repairedAt status to 0
+-f use a file containing a list of sstables as the input
+=================================== ================================================================================
+
+Set a lot of sstables to unrepaired status
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+There are many ways to do this programmatically. This way would likely include variables for the keyspace and table.
+
+Example::
+
+ find /var/lib/cassandra/data/keyspace1/standard1-d936bd20a17c11e8bc92a55ed562cd82/* -name "*Data.db" -print0 | xargs -0 -I % sstablerepairedset --really-set --is-unrepaired %
+
+Set one to many sstables to repaired status
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Set the repairedAt status after a repair to mark the sstables as repaired. Again, using variables for the keyspace and table names is a good choice.
+
+Example::
+
+ nodetool repair keyspace1 standard1
+ find /var/lib/cassandra/data/keyspace1/standard1-d936bd20a17c11e8bc92a55ed562cd82/* -name "*Data.db" -print0 | xargs -0 -I % sstablerepairedset --really-set --is-repaired %
+
+Print metadata showing repaired status
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+sstablemetadata can be used to view the status set or unset using this command.
+
+Example:
+
+ sstablerepairedset --really-set --is-repaired /var/lib/cassandra/data/keyspace1/standard1-d936bd20a17c11e8bc92a55ed562cd82/mc-1-big-Data.db
+ sstablemetadata /var/lib/cassandra/data/keyspace1/standard1-d936bd20a17c11e8bc92a55ed562cd82/mc-1-big-Data.db | grep "Repaired at"
+ Repaired at: 1534443974000
+
+ sstablerepairedset --really-set --is-unrepaired /var/lib/cassandra/data/keyspace1/standard1-d936bd20a17c11e8bc92a55ed562cd82/mc-1-big-Data.db
+ sstablemetadata /var/lib/cassandra/data/keyspace1/standard1-d936bd20a17c11e8bc92a55ed562cd82/mc-1-big-Data.db | grep "Repaired at"
+ Repaired at: 0
+
+Using command in a script
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+If you know you ran repair 2 weeks ago, you can do something like the following::
+
+ sstablerepairset --is-repaired -f <(find /var/lib/cassandra/data/.../ -iname "*Data.db*" -mtime +14)
+
diff --git a/doc/source/tools/sstable/sstablescrub.rst b/doc/source/tools/sstable/sstablescrub.rst
new file mode 100644
index 0000000..0bbda9f
--- /dev/null
+++ b/doc/source/tools/sstable/sstablescrub.rst
@@ -0,0 +1,93 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements. See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership. The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License. You may obtain a copy of the License at
+..
+.. http://www.apache.org/licenses/LICENSE-2.0
+..
+.. Unless required by applicable law or agreed to in writing, software
+.. distributed under the License is distributed on an "AS IS" BASIS,
+.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+.. See the License for the specific language governing permissions and
+.. limitations under the License.
+
+sstablescrub
+------------
+
+Fix a broken sstable. The scrub process rewrites the sstable, skipping any corrupted rows. Because these rows are lost, follow this process with a repair.
+
+ref: https://issues.apache.org/jira/browse/CASSANDRA-4321
+
+Cassandra must be stopped before this tool is executed, or unexpected results will occur. Note: the script does not verify that Cassandra is stopped.
+
+Usage
+^^^^^
+sstablescrub <options> <keyspace> <table>
+
+=================================== ================================================================================
+--debug display stack traces
+-h,--help display this help message
+-m,--manifest-check only check and repair the leveled manifest, without actually scrubbing the sstables
+-n,--no-validate do not validate columns using column validator
+-r,--reinsert-overflowed-ttl Rewrites rows with overflowed expiration date affected by CASSANDRA-14092
+ with the maximum supported expiration date of 2038-01-19T03:14:06+00:00. The rows are rewritten with the original timestamp incremented by one millisecond to override/supersede any potential tombstone that may have been generated during compaction of the affected rows.
+-s,--skip-corrupted skip corrupt rows in counter tables
+-v,--verbose verbose output
+=================================== ================================================================================
+
+Basic Scrub
+^^^^^^^^^^^
+
+The scrub without options will do a snapshot first, then write all non-corrupted files to a new sstable.
+
+Example::
+
+ sstablescrub keyspace1 standard1
+ Pre-scrub sstables snapshotted into snapshot pre-scrub-1534424070883
+ Scrubbing BigTableReader(path='/var/lib/cassandra/data/keyspace1/standard1-6365332094dd11e88f324f9c503e4753/mc-5-big-Data.db') (17.142MiB)
+ Scrub of BigTableReader(path='/var/lib/cassandra/data/keyspace1/standard1-6365332094dd11e88f324f9c503e4753/mc-5-big-Data.db') complete: 73367 rows in new sstable and 0 empty (tombstoned) rows dropped
+ Checking leveled manifest
+
+Scrub without Validation
+^^^^^^^^^^^^^^^^^^^^^^^^
+ref: https://issues.apache.org/jira/browse/CASSANDRA-9406
+
+Use the --no-validate option to retain data that may be misrepresented (e.g., an integer stored in a long field) but not corrupt. This data usually doesn not present any errors to the client.
+
+Example::
+
+ sstablescrub --no-validate keyspace1 standard1
+ Pre-scrub sstables snapshotted into snapshot pre-scrub-1536243158517
+ Scrubbing BigTableReader(path='/var/lib/cassandra/data/keyspace1/standard1-bc9cf530b1da11e886c66d2c86545d91/mc-2-big-Data.db') (4.482MiB)
+ Scrub of BigTableReader(path='/var/lib/cassandra/data/keyspace1/standard1-bc9cf530b1da11e886c66d2c86545d91/mc-2-big-Data.db') complete; looks like all 0 rows were tombstoned
+
+Skip Corrupted Counter Tables
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+ref: https://issues.apache.org/jira/browse/CASSANDRA-5930
+
+If counter tables are corrupted in a way that prevents sstablescrub from completing, you can use the --skip-corrupted option to skip scrubbing those counter tables. This workaround is not necessary in versions 2.0+.
+
+Example::
+
+ sstablescrub --skip-corrupted keyspace1 counter1
+
+Dealing with Overflow Dates
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+ref: https://issues.apache.org/jira/browse/CASSANDRA-14092
+
+Using the option --reinsert-overflowed-ttl allows a rewriting of rows that had a max TTL going over the maximum (causing an overflow).
+
+Example::
+
+ sstablescrub --reinsert-overflowed-ttl keyspace1 counter1
+
+Manifest Check
+^^^^^^^^^^^^^^
+
+As of Cassandra version 2.0, this option is no longer relevant, since level data was moved from a separate manifest into the sstable metadata.
+
diff --git a/doc/source/tools/sstable/sstablesplit.rst b/doc/source/tools/sstable/sstablesplit.rst
new file mode 100644
index 0000000..5386fa4
--- /dev/null
+++ b/doc/source/tools/sstable/sstablesplit.rst
@@ -0,0 +1,93 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements. See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership. The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License. You may obtain a copy of the License at
+..
+.. http://www.apache.org/licenses/LICENSE-2.0
+..
+.. Unless required by applicable law or agreed to in writing, software
+.. distributed under the License is distributed on an "AS IS" BASIS,
+.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+.. See the License for the specific language governing permissions and
+.. limitations under the License.
+
+sstablesplit
+------------
+
+Big sstable files can take up a lot of disk space. The sstablesplit tool can be used to split those large files into smaller files. It can be thought of as a type of anticompaction.
+
+ref: https://issues.apache.org/jira/browse/CASSANDRA-4766
+
+Cassandra must be stopped before this tool is executed, or unexpected results will occur. Note: the script does not verify that Cassandra is stopped.
+
+Usage
+^^^^^
+sstablesplit <options> <filename>
+
+=================================== ================================================================================
+--debug display stack traces
+-h, --help display this help message
+--no-snapshot don't snapshot the sstables before splitting
+-s, --size <size> maximum size in MB for the output sstables (default: 50)
+=================================== ================================================================================
+
+This command should be run with Cassandra stopped. Note: the script does not verify that Cassandra is stopped.
+
+Split a File
+^^^^^^^^^^^^
+
+Split a large sstable into smaller sstables. By default, unless the option --no-snapshot is added, a snapshot will be done of the original sstable and placed in the snapshots folder.
+
+Example::
+
+ sstablesplit /var/lib/cassandra/data/keyspace/eventlog-6365332094dd11e88f324f9c503e4753/mc-8-big-Data.db
+
+ Pre-split sstables snapshotted into snapshot pre-split-1533144514795
+
+Split Multiple Files
+^^^^^^^^^^^^^^^^^^^^
+
+Wildcards can be used in the filename portion of the command to split multiple files.
+
+Example::
+
+ sstablesplit --size 1 /var/lib/cassandra/data/keyspace/eventlog-6365332094dd11e88f324f9c503e4753/mc-1*
+
+Attempt to Split a Small File
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+If the file is already smaller than the split size provided, the sstable will not be split.
+
+Example::
+
+ sstablesplit /var/lib/cassandra/data/keyspace/eventlog-6365332094dd11e88f324f9c503e4753/mc-8-big-Data.db
+ Skipping /var/lib/cassandra/data/keyspace/eventlog-6365332094dd11e88f324f9c503e4753/mc-8-big-Data.db: it's size (1.442 MB) is less than the split size (50 MB)
+ No sstables needed splitting.
+
+Split a File into Specified Size
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The default size used for splitting is 50MB. Specify another size with the --size option. The size is in megabytes (MB). Specify only the number, not the units. For example --size 50 is correct, but --size 50MB is not.
+
+Example::
+
+ sstablesplit --size 1 /var/lib/cassandra/data/keyspace/eventlog-6365332094dd11e88f324f9c503e4753/mc-9-big-Data.db
+ Pre-split sstables snapshotted into snapshot pre-split-1533144996008
+
+
+Split Without Snapshot
+^^^^^^^^^^^^^^^^^^^^^^
+
+By default, sstablesplit will create a snapshot before splitting. If a snapshot is not needed, use the --no-snapshot option to skip it.
+
+Example::
+
+ sstablesplit --size 1 --no-snapshot /var/lib/cassandra/data/keyspace/eventlog-6365332094dd11e88f324f9c503e4753/mc-11-big-Data.db
+
+Note: There is no output, but you can see the results in your file system.
+
+
+
diff --git a/doc/source/tools/sstable/sstableupgrade.rst b/doc/source/tools/sstable/sstableupgrade.rst
new file mode 100644
index 0000000..66386ac
--- /dev/null
+++ b/doc/source/tools/sstable/sstableupgrade.rst
@@ -0,0 +1,137 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements. See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership. The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License. You may obtain a copy of the License at
+..
+.. http://www.apache.org/licenses/LICENSE-2.0
+..
+.. Unless required by applicable law or agreed to in writing, software
+.. distributed under the License is distributed on an "AS IS" BASIS,
+.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+.. See the License for the specific language governing permissions and
+.. limitations under the License.
+
+sstableupgrade
+--------------
+
+Upgrade the sstables in the given table (or snapshot) to the current version of Cassandra. This process is typically done after a Cassandra version upgrade. This operation will rewrite the sstables in the specified table to match the currently installed version of Cassandra. The sstableupgrade command can also be used to downgrade sstables to a previous version.
+
+The snapshot option will only upgrade the specified snapshot. Upgrading snapshots is required before attempting to restore a snapshot taken in a major version older than the major version Cassandra is currently running. This will replace the files in the given snapshot as well as break any hard links to live sstables.
+
+Cassandra must be stopped before this tool is executed, or unexpected results will occur. Note: the script does not verify that Cassandra is stopped.
+
+Usage
+^^^^^
+sstableupgrade <options> <keyspace> <table> [snapshot_name]
+
+=================================== ================================================================================
+--debug display stack traces
+-h,--help display this help message
+-k,--keep-source do not delete the source sstables
+=================================== ================================================================================
+
+Rewrite tables to the current Cassandra version
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Start with a set of sstables in one version of Cassandra::
+
+ ls -al /tmp/cassandra/data/keyspace1/standard1-9695b790a63211e8a6fb091830ac5256/
+ ...
+ -rw-r--r-- 1 user wheel 348 Aug 22 13:45 keyspace1-standard1-ka-1-CRC.db
+ -rw-r--r-- 1 user wheel 5620000 Aug 22 13:45 keyspace1-standard1-ka-1-Data.db
+ -rw-r--r-- 1 user wheel 10 Aug 22 13:45 keyspace1-standard1-ka-1-Digest.sha1
+ -rw-r--r-- 1 user wheel 25016 Aug 22 13:45 keyspace1-standard1-ka-1-Filter.db
+ -rw-r--r-- 1 user wheel 480000 Aug 22 13:45 keyspace1-standard1-ka-1-Index.db
+ -rw-r--r-- 1 user wheel 9895 Aug 22 13:45 keyspace1-standard1-ka-1-Statistics.db
+ -rw-r--r-- 1 user wheel 3562 Aug 22 13:45 keyspace1-standard1-ka-1-Summary.db
+ -rw-r--r-- 1 user wheel 79 Aug 22 13:45 keyspace1-standard1-ka-1-TOC.txt
+
+After upgrading the Cassandra version, upgrade the sstables::
+
+ sstableupgrade keyspace1 standard1
+ Found 1 sstables that need upgrading.
+ Upgrading BigTableReader(path='/var/lib/cassandra/data/keyspace1/standard1-9695b790a63211e8a6fb091830ac5256/keyspace1-standard1-ka-1-Data.db')
+ Upgrade of BigTableReader(path='/var/lib/cassandra/data/keyspace1/standard1-9695b790a63211e8a6fb091830ac5256/keyspace1-standard1-ka-1-Data.db') complete.
+
+ ls -al /tmp/cassandra/data/keyspace1/standard1-9695b790a63211e8a6fb091830ac5256/
+ ...
+ drwxr-xr-x 2 user wheel 64 Aug 22 13:48 backups
+ -rw-r--r-- 1 user wheel 292 Aug 22 13:48 mc-2-big-CRC.db
+ -rw-r--r-- 1 user wheel 4599475 Aug 22 13:48 mc-2-big-Data.db
+ -rw-r--r-- 1 user wheel 10 Aug 22 13:48 mc-2-big-Digest.crc32
+ -rw-r--r-- 1 user wheel 25256 Aug 22 13:48 mc-2-big-Filter.db
+ -rw-r--r-- 1 user wheel 330807 Aug 22 13:48 mc-2-big-Index.db
+ -rw-r--r-- 1 user wheel 10312 Aug 22 13:48 mc-2-big-Statistics.db
+ -rw-r--r-- 1 user wheel 3506 Aug 22 13:48 mc-2-big-Summary.db
+ -rw-r--r-- 1 user wheel 80 Aug 22 13:48 mc-2-big-TOC.txt
+
+Rewrite tables to the current Cassandra version, and keep tables in old version
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Again, starting with a set of sstables in one version::
+
+ ls -al /tmp/cassandra/data/keyspace1/standard1-db532690a63411e8b4ae091830ac5256/
+ ...
+ -rw-r--r-- 1 user wheel 348 Aug 22 13:58 keyspace1-standard1-ka-1-CRC.db
+ -rw-r--r-- 1 user wheel 5620000 Aug 22 13:58 keyspace1-standard1-ka-1-Data.db
+ -rw-r--r-- 1 user wheel 10 Aug 22 13:58 keyspace1-standard1-ka-1-Digest.sha1
+ -rw-r--r-- 1 user wheel 25016 Aug 22 13:58 keyspace1-standard1-ka-1-Filter.db
+ -rw-r--r-- 1 user wheel 480000 Aug 22 13:58 keyspace1-standard1-ka-1-Index.db
+ -rw-r--r-- 1 user wheel 9895 Aug 22 13:58 keyspace1-standard1-ka-1-Statistics.db
+ -rw-r--r-- 1 user wheel 3562 Aug 22 13:58 keyspace1-standard1-ka-1-Summary.db
+ -rw-r--r-- 1 user wheel 79 Aug 22 13:58 keyspace1-standard1-ka-1-TOC.txt
+
+After upgrading the Cassandra version, upgrade the sstables, retaining the original sstables::
+
+ sstableupgrade keyspace1 standard1 -k
+ Found 1 sstables that need upgrading.
+ Upgrading BigTableReader(path='/var/lib/cassandra/data/keyspace1/standard1-db532690a63411e8b4ae091830ac5256/keyspace1-standard1-ka-1-Data.db')
+ Upgrade of BigTableReader(path='/var/lib/cassandra/data/keyspace1/standard1-db532690a63411e8b4ae091830ac5256/keyspace1-standard1-ka-1-Data.db') complete.
+
+ ls -al /tmp/cassandra/data/keyspace1/standard1-db532690a63411e8b4ae091830ac5256/
+ ...
+ drwxr-xr-x 2 user wheel 64 Aug 22 14:00 backups
+ -rw-r--r--@ 1 user wheel 348 Aug 22 13:58 keyspace1-standard1-ka-1-CRC.db
+ -rw-r--r--@ 1 user wheel 5620000 Aug 22 13:58 keyspace1-standard1-ka-1-Data.db
+ -rw-r--r--@ 1 user wheel 10 Aug 22 13:58 keyspace1-standard1-ka-1-Digest.sha1
+ -rw-r--r--@ 1 user wheel 25016 Aug 22 13:58 keyspace1-standard1-ka-1-Filter.db
+ -rw-r--r--@ 1 user wheel 480000 Aug 22 13:58 keyspace1-standard1-ka-1-Index.db
+ -rw-r--r--@ 1 user wheel 9895 Aug 22 13:58 keyspace1-standard1-ka-1-Statistics.db
+ -rw-r--r--@ 1 user wheel 3562 Aug 22 13:58 keyspace1-standard1-ka-1-Summary.db
+ -rw-r--r--@ 1 user wheel 79 Aug 22 13:58 keyspace1-standard1-ka-1-TOC.txt
+ -rw-r--r-- 1 user wheel 292 Aug 22 14:01 mc-2-big-CRC.db
+ -rw-r--r-- 1 user wheel 4596370 Aug 22 14:01 mc-2-big-Data.db
+ -rw-r--r-- 1 user wheel 10 Aug 22 14:01 mc-2-big-Digest.crc32
+ -rw-r--r-- 1 user wheel 25256 Aug 22 14:01 mc-2-big-Filter.db
+ -rw-r--r-- 1 user wheel 330801 Aug 22 14:01 mc-2-big-Index.db
+ -rw-r--r-- 1 user wheel 10312 Aug 22 14:01 mc-2-big-Statistics.db
+ -rw-r--r-- 1 user wheel 3506 Aug 22 14:01 mc-2-big-Summary.db
+ -rw-r--r-- 1 user wheel 80 Aug 22 14:01 mc-2-big-TOC.txt
+
+
+Rewrite a snapshot to the current Cassandra version
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Find the snapshot name::
+
+ nodetool listsnapshots
+
+ Snapshot Details:
+ Snapshot name Keyspace name Column family name True size Size on disk
+ ...
+ 1534962986979 keyspace1 standard1 5.85 MB 5.85 MB
+
+Then rewrite the snapshot::
+
+ sstableupgrade keyspace1 standard1 1534962986979
+ Found 1 sstables that need upgrading.
+ Upgrading BigTableReader(path='/var/lib/cassandra/data/keyspace1/standard1-5850e9f0a63711e8a5c5091830ac5256/snapshots/1534962986979/keyspace1-standard1-ka-1-Data.db')
+ Upgrade of BigTableReader(path='/var/lib/cassandra/data/keyspace1/standard1-5850e9f0a63711e8a5c5091830ac5256/snapshots/1534962986979/keyspace1-standard1-ka-1-Data.db') complete.
+
+
+
+
+
diff --git a/doc/source/tools/sstable/sstableutil.rst b/doc/source/tools/sstable/sstableutil.rst
new file mode 100644
index 0000000..30becd0
--- /dev/null
+++ b/doc/source/tools/sstable/sstableutil.rst
@@ -0,0 +1,91 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements. See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership. The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License. You may obtain a copy of the License at
+..
+.. http://www.apache.org/licenses/LICENSE-2.0
+..
+.. Unless required by applicable law or agreed to in writing, software
+.. distributed under the License is distributed on an "AS IS" BASIS,
+.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+.. See the License for the specific language governing permissions and
+.. limitations under the License.
+
+sstableutil
+-----------
+
+List sstable files for the provided table.
+
+ref: https://issues.apache.org/jira/browse/CASSANDRA-7066
+
+Cassandra must be stopped before this tool is executed, or unexpected results will occur. Note: the script does not verify that Cassandra is stopped.
+
+Usage
+^^^^^
+sstableutil <options> <keyspace> <table>
+
+=================================== ================================================================================
+-c, --cleanup clean up any outstanding transactions
+-d, --debug display stack traces
+-h, --help display this help message
+-o, --oplog include operation logs
+-t, --type <arg> all (list all files, final or temporary), tmp (list temporary files only),
+ final (list final files only),
+-v, --verbose verbose output
+=================================== ================================================================================
+
+List all sstables
+^^^^^^^^^^^^^^^^^
+
+The basic command lists the sstables associated with a given keyspace/table.
+
+Example::
+
+ sstableutil keyspace eventlog
+ Listing files...
+ /var/lib/cassandra/data/keyspace/eventlog-6365332094dd11e88f324f9c503e4753/mc-32-big-CRC.db
+ /var/lib/cassandra/data/keyspace/eventlog-6365332094dd11e88f324f9c503e4753/mc-32-big-Data.db
+ /var/lib/cassandra/data/keyspace/eventlog-6365332094dd11e88f324f9c503e4753/mc-32-big-Digest.crc32
+ /var/lib/cassandra/data/keyspace/eventlog-6365332094dd11e88f324f9c503e4753/mc-32-big-Filter.db
+ /var/lib/cassandra/data/keyspace/eventlog-6365332094dd11e88f324f9c503e4753/mc-32-big-Index.db
+ /var/lib/cassandra/data/keyspace/eventlog-6365332094dd11e88f324f9c503e4753/mc-32-big-Statistics.db
+ /var/lib/cassandra/data/keyspace/eventlog-6365332094dd11e88f324f9c503e4753/mc-32-big-Summary.db
+ /var/lib/cassandra/data/keyspace/eventlog-6365332094dd11e88f324f9c503e4753/mc-32-big-TOC.txt
+ /var/lib/cassandra/data/keyspace/eventlog-6365332094dd11e88f324f9c503e4753/mc-37-big-CRC.db
+ /var/lib/cassandra/data/keyspace/eventlog-6365332094dd11e88f324f9c503e4753/mc-37-big-Data.db
+ /var/lib/cassandra/data/keyspace/eventlog-6365332094dd11e88f324f9c503e4753/mc-37-big-Digest.crc32
+ /var/lib/cassandra/data/keyspace/eventlog-6365332094dd11e88f324f9c503e4753/mc-37-big-Filter.db
+ /var/lib/cassandra/data/keyspace/eventlog-6365332094dd11e88f324f9c503e4753/mc-37-big-Index.db
+ /var/lib/cassandra/data/keyspace/eventlog-6365332094dd11e88f324f9c503e4753/mc-37-big-Statistics.db
+ /var/lib/cassandra/data/keyspace/eventlog-6365332094dd11e88f324f9c503e4753/mc-37-big-Summary.db
+ /var/lib/cassandra/data/keyspace/eventlog-6365332094dd11e88f324f9c503e4753/mc-37-big-TOC.txt
+
+List only temporary sstables
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Using the -t option followed by `tmp` will list all temporary sstables, in the format above. Temporary sstables were used in pre-3.0 versions of Cassandra.
+
+List only final sstables
+^^^^^^^^^^^^^^^^^^^^^^^^
+
+Using the -t option followed by `final` will list all final sstables, in the format above. In recent versions of Cassandra, this is the same output as not using the -t option.
+
+Include transaction logs
+^^^^^^^^^^^^^^^^^^^^^^^^
+
+Using the -o option will include transaction logs in the listing, in the format above.
+
+Clean up sstables
+^^^^^^^^^^^^^^^^^
+
+Using the -c option removes any transactions left over from incomplete writes or compactions.
+
+From the 3.0 upgrade notes:
+
+New transaction log files have been introduced to replace the compactions_in_progress system table, temporary file markers (tmp and tmplink) and sstable ancestors. Therefore, compaction metadata no longer contains ancestors. Transaction log files list sstable descriptors involved in compactions and other operations such as flushing and streaming. Use the sstableutil tool to list any sstable files currently involved in operations not yet completed, which previously would have been marked as temporary. A transaction log file contains one sstable per line, with the prefix "add:" or "remove:". They also contain a special line "commit", only inserted at the end when the transaction is committed. On startup we use these files to cleanup any partial transactions that were in progress when the process exited. If the commit line is found, we keep new sstables (those with the "add" prefix) and delete the old sstables (those with the "remove" prefix), vice-versa if the commit line is missing. Should you lose or delete these log files, both old and new sstable files will be kept as live files, which will result in duplicated sstables. These files are protected by incremental checksums so you should not manually edit them. When restoring a full backup or moving sstable files, you should clean-up any left over transactions and their temporary files first.
+
+
+
diff --git a/doc/source/tools/sstable/sstableverify.rst b/doc/source/tools/sstable/sstableverify.rst
new file mode 100644
index 0000000..dad3f44
--- /dev/null
+++ b/doc/source/tools/sstable/sstableverify.rst
@@ -0,0 +1,91 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements. See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership. The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License. You may obtain a copy of the License at
+..
+.. http://www.apache.org/licenses/LICENSE-2.0
+..
+.. Unless required by applicable law or agreed to in writing, software
+.. distributed under the License is distributed on an "AS IS" BASIS,
+.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+.. See the License for the specific language governing permissions and
+.. limitations under the License.
+
+sstableverify
+-------------
+
+Check sstable(s) for errors or corruption, for the provided table.
+
+ref: https://issues.apache.org/jira/browse/CASSANDRA-5791
+
+Cassandra must be stopped before this tool is executed, or unexpected results will occur. Note: the script does not verify that Cassandra is stopped.
+
+Usage
+^^^^^
+sstableverify <options> <keyspace> <table>
+
+=================================== ================================================================================
+--debug display stack traces
+-e, --extended extended verification
+-h, --help display this help message
+-v, --verbose verbose output
+=================================== ================================================================================
+
+Basic Verification
+^^^^^^^^^^^^^^^^^^
+
+This is the basic verification. It is not a very quick process, and uses memory. You might need to increase your memory settings if you have many sstables.
+
+Example::
+
+ sstableverify keyspace eventlog
+ Verifying BigTableReader(path='/var/lib/cassandra/data/keyspace/eventlog-6365332094dd11e88f324f9c503e4753/mc-32-big-Data.db') (7.353MiB)
+ Deserializing sstable metadata for BigTableReader(path='/var/lib/cassandra/data/keyspace/eventlog-6365332094dd11e88f324f9c503e4753/mc-32-big-Data.db')
+ Checking computed hash of BigTableReader(path='/var/lib/cassandra/data/keyspace/eventlog-6365332094dd11e88f324f9c503e4753/mc-32-big-Data.db')
+ Verifying BigTableReader(path='/var/lib/cassandra/data/keyspace/eventlog-6365332094dd11e88f324f9c503e4753/mc-37-big-Data.db') (3.775MiB)
+ Deserializing sstable metadata for BigTableReader(path='/var/lib/cassandra/data/keyspace/eventlog-6365332094dd11e88f324f9c503e4753/mc-37-big-Data.db')
+ Checking computed hash of BigTableReader(path='/var/lib/cassandra/data/keyspace/eventlog-6365332094dd11e88f324f9c503e4753/mc-37-big-Data.db')
+
+Extended Verification
+^^^^^^^^^^^^^^^^^^^^^
+
+During an extended verification, the individual values will be validated for errors or corruption. This of course takes more time.
+
+Example::
+
+ root@DC1C1:/# sstableverify -e keyspace eventlog
+ WARN 14:08:06,255 Only 33.096GiB free across all data volumes. Consider adding more capacity to your cluster or removing obsolete snapshots
+ Verifying BigTableReader(path='/var/lib/cassandra/data/keyspace/eventlog-6365332094dd11e88f324f9c503e4753/mc-32-big-Data.db') (7.353MiB)
+ Deserializing sstable metadata for BigTableReader(path='/var/lib/cassandra/data/keyspace/eventlog-6365332094dd11e88f324f9c503e4753/mc-32-big-Data.db')
+ Checking computed hash of BigTableReader(path='/var/lib/cassandra/data/keyspace/eventlog-6365332094dd11e88f324f9c503e4753/mc-32-big-Data.db')
+ Extended Verify requested, proceeding to inspect values
+ Verify of BigTableReader(path='/var/lib/cassandra/data/keyspace/eventlog-6365332094dd11e88f324f9c503e4753/mc-32-big-Data.db') succeeded. All 33211 rows read successfully
+ Verifying BigTableReader(path='/var/lib/cassandra/data/keyspace/eventlog-6365332094dd11e88f324f9c503e4753/mc-37-big-Data.db') (3.775MiB)
+ Deserializing sstable metadata for BigTableReader(path='/var/lib/cassandra/data/keyspace/eventlog-6365332094dd11e88f324f9c503e4753/mc-37-big-Data.db')
+ Checking computed hash of BigTableReader(path='/var/lib/cassandra/data/keyspace/eventlog-6365332094dd11e88f324f9c503e4753/mc-37-big-Data.db')
+ Extended Verify requested, proceeding to inspect values
+ Verify of BigTableReader(path='/var/lib/cassandra/data/keyspace/eventlog-6365332094dd11e88f324f9c503e4753/mc-37-big-Data.db') succeeded. All 17068 rows read successfully
+
+Corrupted File
+^^^^^^^^^^^^^^
+
+Corrupted files are listed if they are detected by the script.
+
+Example::
+
+ sstableverify keyspace eventlog
+ Verifying BigTableReader(path='/var/lib/cassandra/data/keyspace/eventlog-6365332094dd11e88f324f9c503e4753/mc-40-big-Data.db') (7.416MiB)
+ Deserializing sstable metadata for BigTableReader(path='/var/lib/cassandra/data/keyspace/eventlog-6365332094dd11e88f324f9c503e4753/mc-40-big-Data.db')
+ Checking computed hash of BigTableReader(path='/var/lib/cassandra/data/keyspace/eventlog-6365332094dd11e88f324f9c503e4753/mc-40-big-Data.db')
+ Error verifying BigTableReader(path='/var/lib/cassandra/data/keyspace/eventlog-6365332094dd11e88f324f9c503e4753/mc-40-big-Data.db'): Corrupted: /var/lib/cassandra/data/keyspace/eventlog-6365332094dd11e88f324f9c503e4753/mc-40-big-Data.db
+
+A similar (but less verbose) tool will show the suggested actions::
+
+ nodetool verify keyspace eventlog
+ error: Invalid SSTable /var/lib/cassandra/data/keyspace/eventlog-6365332094dd11e88f324f9c503e4753/mc-40-big-Data.db, please force repair
+
+
+
diff --git a/doc/source/tools/stress-example.yaml b/doc/source/tools/stress-example.yaml
new file mode 100644
index 0000000..17161af
--- /dev/null
+++ b/doc/source/tools/stress-example.yaml
@@ -0,0 +1,44 @@
+spacenam: example # idenitifier for this spec if running with multiple yaml files
+keyspace: example
+
+# Would almost always be network topology unless running something locally
+keyspace_definition: |
+ CREATE KEYSPACE example WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 3};
+
+table: staff_activities
+
+# The table under test. Start with a partition per staff member
+# Is this a good idea?
+table_definition: |
+ CREATE TABLE staff_activities (
+ name text,
+ when timeuuid,
+ what text,
+ PRIMARY KEY(name, when)
+ )
+
+columnspec:
+ - name: name
+ size: uniform(5..10) # The names of the staff members are between 5-10 characters
+ population: uniform(1..10) # 10 possible staff members to pick from
+ - name: when
+ cluster: uniform(20..500) # Staff members do between 20 and 500 events
+ - name: what
+ size: normal(10..100,50)
+
+insert:
+ # we only update a single partition in any given insert
+ partitions: fixed(1)
+ # we want to insert a single row per partition and we have between 20 and 500
+ # rows per partition
+ select: fixed(1)/500
+ batchtype: UNLOGGED # Single partition unlogged batches are essentially noops
+
+queries:
+ events:
+ cql: select * from staff_activities where name = ?
+ fields: samerow
+ latest_event:
+ cql: select * from staff_activities where name = ? LIMIT 1
+ fields: samerow
+
diff --git a/doc/source/tools/stress-lwt-example.yaml b/doc/source/tools/stress-lwt-example.yaml
new file mode 100644
index 0000000..fc5db08
--- /dev/null
+++ b/doc/source/tools/stress-lwt-example.yaml
@@ -0,0 +1,70 @@
+# Keyspace Name
+keyspace: stresscql
+
+# The CQL for creating a keyspace (optional if it already exists)
+# Would almost always be network topology unless running something locall
+keyspace_definition: |
+ CREATE KEYSPACE stresscql WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1};
+
+# Table name
+table: blogposts
+
+# The CQL for creating a table you wish to stress (optional if it already exists)
+table_definition: |
+ CREATE TABLE blogposts (
+ domain text,
+ published_date timeuuid,
+ url text,
+ author text,
+ title text,
+ body text,
+ PRIMARY KEY(domain, published_date)
+ ) WITH CLUSTERING ORDER BY (published_date DESC)
+ AND compaction = { 'class':'LeveledCompactionStrategy' }
+ AND comment='A table to hold blog posts'
+
+### Column Distribution Specifications ###
+
+columnspec:
+ - name: domain
+ size: gaussian(5..100) #domain names are relatively short
+ population: uniform(1..10M) #10M possible domains to pick from
+
+ - name: published_date
+ cluster: fixed(1000) #under each domain we will have max 1000 posts
+
+ - name: url
+ size: uniform(30..300)
+
+ - name: title #titles shouldn't go beyond 200 chars
+ size: gaussian(10..200)
+
+ - name: author
+ size: uniform(5..20) #author names should be short
+
+ - name: body
+ size: gaussian(100..5000) #the body of the blog post can be long
+
+### Batch Ratio Distribution Specifications ###
+
+insert:
+ partitions: fixed(1) # Our partition key is the domain so only insert one per batch
+
+ select: fixed(1)/1000 # We have 1000 posts per domain so 1/1000 will allow 1 post per batch
+
+ batchtype: UNLOGGED # Unlogged batches
+
+
+#
+# A list of queries you wish to run against the schema
+#
+queries:
+ singlepost:
+ cql: select * from blogposts where domain = ? LIMIT 1
+ fields: samerow
+ regularupdate:
+ cql: update blogposts set author = ? where domain = ? and published_date = ?
+ fields: samerow
+ updatewithlwt:
+ cql: update blogposts set author = ? where domain = ? and published_date = ? IF body = ? AND url = ?
+ fields: samerow
diff --git a/doc/source/troubleshooting/finding_nodes.rst b/doc/source/troubleshooting/finding_nodes.rst
new file mode 100644
index 0000000..df5e16c
--- /dev/null
+++ b/doc/source/troubleshooting/finding_nodes.rst
@@ -0,0 +1,149 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements. See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership. The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License. You may obtain a copy of the License at
+..
+.. http://www.apache.org/licenses/LICENSE-2.0
+..
+.. Unless required by applicable law or agreed to in writing, software
+.. distributed under the License is distributed on an "AS IS" BASIS,
+.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+.. See the License for the specific language governing permissions and
+.. limitations under the License.
+
+Find The Misbehaving Nodes
+==========================
+
+The first step to troubleshooting a Cassandra issue is to use error messages,
+metrics and monitoring information to identify if the issue lies with the
+clients or the server and if it does lie with the server find the problematic
+nodes in the Cassandra cluster. The goal is to determine if this is a systemic
+issue (e.g. a query pattern that affects the entire cluster) or isolated to a
+subset of nodes (e.g. neighbors holding a shared token range or even a single
+node with bad hardware).
+
+There are many sources of information that help determine where the problem
+lies. Some of the most common are mentioned below.
+
+Client Logs and Errors
+----------------------
+Clients of the cluster often leave the best breadcrumbs to follow. Perhaps
+client latencies or error rates have increased in a particular datacenter
+(likely eliminating other datacenter's nodes), or clients are receiving a
+particular kind of error code indicating a particular kind of problem.
+Troubleshooters can often rule out many failure modes just by reading the error
+messages. In fact, many Cassandra error messages include the last coordinator
+contacted to help operators find nodes to start with.
+
+Some common errors (likely culprit in parenthesis) assuming the client has
+similar error names as the Datastax :ref:`drivers <client-drivers>`:
+
+* ``SyntaxError`` (**client**). This and other ``QueryValidationException``
+ indicate that the client sent a malformed request. These are rarely server
+ issues and usually indicate bad queries.
+* ``UnavailableException`` (**server**): This means that the Cassandra
+ coordinator node has rejected the query as it believes that insufficent
+ replica nodes are available. If many coordinators are throwing this error it
+ likely means that there really are (typically) multiple nodes down in the
+ cluster and you can identify them using :ref:`nodetool status
+ <nodetool-status>` If only a single coordinator is throwing this error it may
+ mean that node has been partitioned from the rest.
+* ``OperationTimedOutException`` (**server**): This is the most frequent
+ timeout message raised when clients set timeouts and means that the query
+ took longer than the supplied timeout. This is a *client side* timeout
+ meaning that it took longer than the client specified timeout. The error
+ message will include the coordinator node that was last tried which is
+ usually a good starting point. This error usually indicates either
+ aggressive client timeout values or latent server coordinators/replicas.
+* ``ReadTimeoutException`` or ``WriteTimeoutException`` (**server**): These
+ are raised when clients do not specify lower timeouts and there is a
+ *coordinator* timeouts based on the values supplied in the ``cassandra.yaml``
+ configuration file. They usually indicate a serious server side problem as
+ the default values are usually multiple seconds.
+
+Metrics
+-------
+
+If you have Cassandra :ref:`metrics <monitoring-metrics>` reporting to a
+centralized location such as `Graphite <https://graphiteapp.org/>`_ or
+`Grafana <https://grafana.com/>`_ you can typically use those to narrow down
+the problem. At this stage narrowing down the issue to a particular
+datacenter, rack, or even group of nodes is the main goal. Some helpful metrics
+to look at are:
+
+Errors
+^^^^^^
+Cassandra refers to internode messaging errors as "drops", and provided a
+number of :ref:`Dropped Message Metrics <dropped-metrics>` to help narrow
+down errors. If particular nodes are dropping messages actively, they are
+likely related to the issue.
+
+Latency
+^^^^^^^
+For timeouts or latency related issues you can start with :ref:`Table
+Metrics <table-metrics>` by comparing Coordinator level metrics e.g.
+``CoordinatorReadLatency`` or ``CoordinatorWriteLatency`` with their associated
+replica metrics e.g. ``ReadLatency`` or ``WriteLatency``. Issues usually show
+up on the ``99th`` percentile before they show up on the ``50th`` percentile or
+the ``mean``. While ``maximum`` coordinator latencies are not typically very
+helpful due to the exponentially decaying reservoir used internally to produce
+metrics, ``maximum`` replica latencies that correlate with increased ``99th``
+percentiles on coordinators can help narrow down the problem.
+
+There are usually three main possibilities:
+
+1. Coordinator latencies are high on all nodes, but only a few node's local
+ read latencies are high. This points to slow replica nodes and the
+ coordinator's are just side-effects. This usually happens when clients are
+ not token aware.
+2. Coordinator latencies and replica latencies increase at the
+ same time on the a few nodes. If clients are token aware this is almost
+ always what happens and points to slow replicas of a subset of token
+ ranges (only part of the ring).
+3. Coordinator and local latencies are high on many nodes. This usually
+ indicates either a tipping point in the cluster capacity (too many writes or
+ reads per second), or a new query pattern.
+
+It's important to remember that depending on the client's load balancing
+behavior and consistency levels coordinator and replica metrics may or may
+not correlate. In particular if you use ``TokenAware`` policies the same
+node's coordinator and replica latencies will often increase together, but if
+you just use normal ``DCAwareRoundRobin`` coordinator latencies can increase
+with unrelated replica node's latencies. For example:
+
+* ``TokenAware`` + ``LOCAL_ONE``: should always have coordinator and replica
+ latencies on the same node rise together
+* ``TokenAware`` + ``LOCAL_QUORUM``: should always have coordinator and
+ multiple replica latencies rise together in the same datacenter.
+* ``TokenAware`` + ``QUORUM``: replica latencies in other datacenters can
+ affect coordinator latencies.
+* ``DCAwareRoundRobin`` + ``LOCAL_ONE``: coordinator latencies and unrelated
+ replica node's latencies will rise together.
+* ``DCAwareRoundRobin`` + ``LOCAL_QUORUM``: different coordinator and replica
+ latencies will rise together with little correlation.
+
+Query Rates
+^^^^^^^^^^^
+Sometimes the :ref:`Table <table-metrics>` query rate metrics can help
+narrow down load issues as "small" increase in coordinator queries per second
+(QPS) may correlate with a very large increase in replica level QPS. This most
+often happens with ``BATCH`` writes, where a client may send a single ``BATCH``
+query that might contain 50 statements in it, which if you have 9 copies (RF=3,
+three datacenters) means that every coordinator ``BATCH`` write turns into 450
+replica writes! This is why keeping ``BATCH``'s to the same partition is so
+critical, otherwise you can exhaust significant CPU capacitity with a "single"
+query.
+
+
+Next Step: Investigate the Node(s)
+----------------------------------
+
+Once you have narrowed down the problem as much as possible (datacenter, rack
+, node), login to one of the nodes using SSH and proceed to debug using
+:ref:`logs <reading-logs>`, :ref:`nodetool <use-nodetool>`, and
+:ref:`os tools <use-os-tools>`. If you are not able to login you may still
+have access to :ref:`logs <reading-logs>` and :ref:`nodetool <use-nodetool>`
+remotely.
diff --git a/doc/source/troubleshooting/index.rst b/doc/source/troubleshooting/index.rst
index 2e5cf10..79b46d6 100644
--- a/doc/source/troubleshooting/index.rst
+++ b/doc/source/troubleshooting/index.rst
@@ -17,4 +17,23 @@
Troubleshooting
===============
-.. TODO: todo
+As any distributed database does, sometimes Cassandra breaks and you will have
+to troubleshoot what is going on. Generally speaking you can debug Cassandra
+like any other distributed Java program, meaning that you have to find which
+machines in your cluster are misbehaving and then isolate the problem using
+logs and tools. Luckily Cassandra had a great set of instrospection tools to
+help you.
+
+These pages include a number of command examples demonstrating various
+debugging and analysis techniques, mostly for Linux/Unix systems. If you don't
+have access to the machines running Cassandra, or are running on Windows or
+another operating system you may not be able to use the exact commands but
+there are likely equivalent tools you can use.
+
+.. toctree::
+ :maxdepth: 2
+
+ finding_nodes
+ reading_logs
+ use_nodetool
+ use_tools
diff --git a/doc/source/troubleshooting/reading_logs.rst b/doc/source/troubleshooting/reading_logs.rst
new file mode 100644
index 0000000..08f7d4d
--- /dev/null
+++ b/doc/source/troubleshooting/reading_logs.rst
@@ -0,0 +1,267 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements. See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership. The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License. You may obtain a copy of the License at
+..
+.. http://www.apache.org/licenses/LICENSE-2.0
+..
+.. Unless required by applicable law or agreed to in writing, software
+.. distributed under the License is distributed on an "AS IS" BASIS,
+.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+.. See the License for the specific language governing permissions and
+.. limitations under the License.
+
+.. _reading-logs:
+
+Cassandra Logs
+==============
+Cassandra has rich support for logging and attempts to give operators maximum
+insight into the database while at the same time limiting noise to the logs.
+
+Common Log Files
+----------------
+Cassandra has three main logs, the ``system.log``, ``debug.log`` and
+``gc.log`` which hold general logging messages, debugging logging messages, and
+java garbage collection logs respectively.
+
+These logs by default live in ``${CASSANDRA_HOME}/logs``, but most Linux
+distributions relocate logs to ``/var/log/cassandra``. Operators can tune
+this location as well as what levels are logged using the provided
+``logback.xml`` file.
+
+``system.log``
+^^^^^^^^^^^^^^
+This log is the default Cassandra log and is a good place to start any
+investigation. Some examples of activities logged to this log:
+
+* Uncaught exceptions. These can be very useful for debugging errors.
+* ``GCInspector`` messages indicating long garbage collector pauses. When long
+ pauses happen Cassandra will print how long and also what was the state of
+ the system (thread state) at the time of that pause. This can help narrow
+ down a capacity issue (either not enough heap or not enough spare CPU).
+* Information about nodes joining and leaving the cluster as well as token
+ metadata (data ownersip) changes. This is useful for debugging network
+ partitions, data movements, and more.
+* Keyspace/Table creation, modification, deletion.
+* ``StartupChecks`` that ensure optimal configuration of the operating system
+ to run Cassandra
+* Information about some background operational tasks (e.g. Index
+ Redistribution).
+
+As with any application, looking for ``ERROR`` or ``WARN`` lines can be a
+great first step::
+
+ $ # Search for warnings or errors in the latest system.log
+ $ grep 'WARN\|ERROR' system.log | tail
+ ...
+
+ $ # Search for warnings or errors in all rotated system.log
+ $ zgrep 'WARN\|ERROR' system.log.* | less
+ ...
+
+``debug.log``
+^^^^^^^^^^^^^^
+This log contains additional debugging information that may be useful when
+troubleshooting but may be much noiser than the normal ``system.log``. Some
+examples of activities logged to this log:
+
+* Information about compactions, including when they start, which sstables
+ they contain, and when they finish.
+* Information about memtable flushes to disk, including when they happened,
+ how large the flushes were, and which commitlog segments the flush impacted.
+
+This log can be *very* noisy, so it is highly recommended to use ``grep`` and
+other log analysis tools to dive deep. For example::
+
+ $ # Search for messages involving a CompactionTask with 5 lines of context
+ $ grep CompactionTask debug.log -C 5
+ ...
+
+ $ # Look at the distribution of flush tasks per keyspace
+ $ grep "Enqueuing flush" debug.log | cut -f 10 -d ' ' | sort | uniq -c
+ 6 compaction_history:
+ 1 test_keyspace:
+ 2 local:
+ 17 size_estimates:
+ 17 sstable_activity:
+
+
+``gc.log``
+^^^^^^^^^^^^^^
+The gc log is a standard Java GC log. With the default ``jvm.options``
+settings you get a lot of valuable information in this log such as
+application pause times, and why pauses happened. This may help narrow
+down throughput or latency issues to a mistuned JVM. For example you can
+view the last few pauses::
+
+ $ grep stopped gc.log.0.current | tail
+ 2018-08-29T00:19:39.522+0000: 3022663.591: Total time for which application threads were stopped: 0.0332813 seconds, Stopping threads took: 0.0008189 seconds
+ 2018-08-29T00:19:44.369+0000: 3022668.438: Total time for which application threads were stopped: 0.0312507 seconds, Stopping threads took: 0.0007025 seconds
+ 2018-08-29T00:19:49.796+0000: 3022673.865: Total time for which application threads were stopped: 0.0307071 seconds, Stopping threads took: 0.0006662 seconds
+ 2018-08-29T00:19:55.452+0000: 3022679.521: Total time for which application threads were stopped: 0.0309578 seconds, Stopping threads took: 0.0006832 seconds
+ 2018-08-29T00:20:00.127+0000: 3022684.197: Total time for which application threads were stopped: 0.0310082 seconds, Stopping threads took: 0.0007090 seconds
+ 2018-08-29T00:20:06.583+0000: 3022690.653: Total time for which application threads were stopped: 0.0317346 seconds, Stopping threads took: 0.0007106 seconds
+ 2018-08-29T00:20:10.079+0000: 3022694.148: Total time for which application threads were stopped: 0.0299036 seconds, Stopping threads took: 0.0006889 seconds
+ 2018-08-29T00:20:15.739+0000: 3022699.809: Total time for which application threads were stopped: 0.0078283 seconds, Stopping threads took: 0.0006012 seconds
+ 2018-08-29T00:20:15.770+0000: 3022699.839: Total time for which application threads were stopped: 0.0301285 seconds, Stopping threads took: 0.0003789 seconds
+ 2018-08-29T00:20:15.798+0000: 3022699.867: Total time for which application threads were stopped: 0.0279407 seconds, Stopping threads took: 0.0003627 seconds
+
+
+This shows a lot of valuable information including how long the application
+was paused (meaning zero user queries were being serviced during the e.g. 33ms
+JVM pause) as well as how long it took to enter the safepoint. You can use this
+raw data to e.g. get the longest pauses::
+
+ $ grep stopped gc.log.0.current | cut -f 11 -d ' ' | sort -n | tail | xargs -IX grep X gc.log.0.current | sort -k 1
+ 2018-08-28T17:13:40.520-0700: 1.193: Total time for which application threads were stopped: 0.0157914 seconds, Stopping threads took: 0.0000355 seconds
+ 2018-08-28T17:13:41.206-0700: 1.879: Total time for which application threads were stopped: 0.0249811 seconds, Stopping threads took: 0.0000318 seconds
+ 2018-08-28T17:13:41.638-0700: 2.311: Total time for which application threads were stopped: 0.0561130 seconds, Stopping threads took: 0.0000328 seconds
+ 2018-08-28T17:13:41.677-0700: 2.350: Total time for which application threads were stopped: 0.0362129 seconds, Stopping threads took: 0.0000597 seconds
+ 2018-08-28T17:13:41.781-0700: 2.454: Total time for which application threads were stopped: 0.0442846 seconds, Stopping threads took: 0.0000238 seconds
+ 2018-08-28T17:13:41.976-0700: 2.649: Total time for which application threads were stopped: 0.0377115 seconds, Stopping threads took: 0.0000250 seconds
+ 2018-08-28T17:13:42.172-0700: 2.845: Total time for which application threads were stopped: 0.0475415 seconds, Stopping threads took: 0.0001018 seconds
+ 2018-08-28T17:13:42.825-0700: 3.498: Total time for which application threads were stopped: 0.0379155 seconds, Stopping threads took: 0.0000571 seconds
+ 2018-08-28T17:13:43.574-0700: 4.247: Total time for which application threads were stopped: 0.0323812 seconds, Stopping threads took: 0.0000574 seconds
+ 2018-08-28T17:13:44.602-0700: 5.275: Total time for which application threads were stopped: 0.0238975 seconds, Stopping threads took: 0.0000788 seconds
+
+In this case any client waiting on a query would have experienced a `56ms`
+latency at 17:13:41.
+
+Note that GC pauses are not _only_ garbage collection, although
+generally speaking high pauses with fast safepoints indicate a lack of JVM heap
+or mistuned JVM GC algorithm. High pauses with slow safepoints typically
+indicate that the JVM is having trouble entering a safepoint which usually
+indicates slow disk drives (Cassandra makes heavy use of memory mapped reads
+which the JVM doesn't know could have disk latency, so the JVM safepoint logic
+doesn't handle a blocking memory mapped read particularly well).
+
+Using these logs you can even get a pause distribution with something like
+`histogram.py <https://github.com/bitly/data_hacks/blob/master/data_hacks/histogram.py>`_::
+
+ $ grep stopped gc.log.0.current | cut -f 11 -d ' ' | sort -n | histogram.py
+ # NumSamples = 410293; Min = 0.00; Max = 11.49
+ # Mean = 0.035346; Variance = 0.002216; SD = 0.047078; Median 0.036498
+ # each ∎ represents a count of 5470
+ 0.0001 - 1.1496 [410255]: ∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎
+ 1.1496 - 2.2991 [ 15]:
+ 2.2991 - 3.4486 [ 5]:
+ 3.4486 - 4.5981 [ 1]:
+ 4.5981 - 5.7475 [ 5]:
+ 5.7475 - 6.8970 [ 9]:
+ 6.8970 - 8.0465 [ 1]:
+ 8.0465 - 9.1960 [ 0]:
+ 9.1960 - 10.3455 [ 0]:
+ 10.3455 - 11.4949 [ 2]:
+
+We can see in this case while we have very good average performance something
+is causing multi second JVM pauses ... In this case it was mostly safepoint
+pauses caused by slow disks::
+
+ $ grep stopped gc.log.0.current | cut -f 11 -d ' ' | sort -n | tail | xargs -IX grep X gc.log.0.current| sort -k 1
+ 2018-07-27T04:52:27.413+0000: 187831.482: Total time for which application threads were stopped: 6.5037022 seconds, Stopping threads took: 0.0005212 seconds
+ 2018-07-30T23:38:18.354+0000: 514582.423: Total time for which application threads were stopped: 6.3262938 seconds, Stopping threads took: 0.0004882 seconds
+ 2018-08-01T02:37:48.380+0000: 611752.450: Total time for which application threads were stopped: 10.3879659 seconds, Stopping threads took: 0.0004475 seconds
+ 2018-08-06T22:04:14.990+0000: 1113739.059: Total time for which application threads were stopped: 6.0917409 seconds, Stopping threads took: 0.0005553 seconds
+ 2018-08-14T00:04:06.091+0000: 1725730.160: Total time for which application threads were stopped: 6.0141054 seconds, Stopping threads took: 0.0004976 seconds
+ 2018-08-17T06:23:06.755+0000: 2007670.824: Total time for which application threads were stopped: 6.0133694 seconds, Stopping threads took: 0.0006011 seconds
+ 2018-08-23T06:35:46.068+0000: 2526830.137: Total time for which application threads were stopped: 6.4767751 seconds, Stopping threads took: 6.4426849 seconds
+ 2018-08-23T06:36:29.018+0000: 2526873.087: Total time for which application threads were stopped: 11.4949489 seconds, Stopping threads took: 11.4638297 seconds
+ 2018-08-23T06:37:12.671+0000: 2526916.741: Total time for which application threads were stopped: 6.3867003 seconds, Stopping threads took: 6.3507166 seconds
+ 2018-08-23T06:37:47.156+0000: 2526951.225: Total time for which application threads were stopped: 7.9528200 seconds, Stopping threads took: 7.9197756 seconds
+
+Sometimes reading and understanding java GC logs is hard, but you can take the
+raw GC files and visualize them using tools such as `GCViewer
+<https://github.com/chewiebug/GCViewer>`_ which take the Cassandra GC log as
+input and show you detailed visual information on your garbage collection
+performance. This includes pause analysis as well as throughput information.
+For a stable Cassandra JVM you probably want to aim for pauses less than
+`200ms` and GC throughput greater than `99%` (ymmv).
+
+Java GC pauses are one of the leading causes of tail latency in Cassandra
+(along with drive latency) so sometimes this information can be crucial
+while debugging tail latency issues.
+
+
+Getting More Information
+------------------------
+
+If the default logging levels are insuficient, ``nodetool`` can set higher
+or lower logging levels for various packages and classes using the
+``nodetool setlogginglevel`` command. Start by viewing the current levels::
+
+ $ nodetool getlogginglevels
+
+ Logger Name Log Level
+ ROOT INFO
+ org.apache.cassandra DEBUG
+
+Perhaps the ``Gossiper`` is acting up and we wish to enable it at ``TRACE``
+level for even more insight::
+
+
+ $ nodetool setlogginglevel org.apache.cassandra.gms.Gossiper TRACE
+
+ $ nodetool getlogginglevels
+
+ Logger Name Log Level
+ ROOT INFO
+ org.apache.cassandra DEBUG
+ org.apache.cassandra.gms.Gossiper TRACE
+
+ $ grep TRACE debug.log | tail -2
+ TRACE [GossipStage:1] 2018-07-04 17:07:47,879 Gossiper.java:1234 - Updating
+ heartbeat state version to 2344 from 2343 for 127.0.0.2:7000 ...
+ TRACE [GossipStage:1] 2018-07-04 17:07:47,879 Gossiper.java:923 - local
+ heartbeat version 2341 greater than 2340 for 127.0.0.1:7000
+
+
+Note that any changes made this way are reverted on next Cassandra process
+restart. To make the changes permanent add the appropriate rule to
+``logback.xml``.
+
+.. code-block:: diff
+
+ diff --git a/conf/logback.xml b/conf/logback.xml
+ index b2c5b10..71b0a49 100644
+ --- a/conf/logback.xml
+ +++ b/conf/logback.xml
+ @@ -98,4 +98,5 @@ appender reference in the root level section below.
+ </root>
+
+ <logger name="org.apache.cassandra" level="DEBUG"/>
+ + <logger name="org.apache.cassandra.gms.Gossiper" level="TRACE"/>
+ </configuration>
+
+Full Query Logger
+^^^^^^^^^^^^^^^^^
+
+Cassandra 4.0 additionally ships with support for full query logging. This
+is a highly performant binary logging tool which captures Cassandra queries
+in real time, writes them (if possible) to a log file, and ensures the total
+size of the capture does not exceed a particular limit. FQL is enabled with
+``nodetool`` and the logs are read with the provided ``bin/fqltool`` utility::
+
+ $ mkdir /var/tmp/fql_logs
+ $ nodetool enablefullquerylog --path /var/tmp/fql_logs
+
+ # ... do some querying
+
+ $ bin/fqltool dump /var/tmp/fql_logs/20180705-00.cq4 | tail
+ Query time: 1530750927224
+ Query: SELECT * FROM system_virtual_schema.columns WHERE keyspace_name =
+ 'system_views' AND table_name = 'sstable_tasks';
+ Values:
+
+ Type: single
+ Protocol version: 4
+ Query time: 1530750934072
+ Query: select * from keyspace1.standard1 ;
+ Values:
+
+ $ nodetool disablefullquerylog
+
+Note that if you want more information than this tool provides, there are other
+live capture options available such as :ref:`packet capture <packet-capture>`.
diff --git a/doc/source/troubleshooting/use_nodetool.rst b/doc/source/troubleshooting/use_nodetool.rst
new file mode 100644
index 0000000..5072f85
--- /dev/null
+++ b/doc/source/troubleshooting/use_nodetool.rst
@@ -0,0 +1,245 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements. See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership. The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License. You may obtain a copy of the License at
+..
+.. http://www.apache.org/licenses/LICENSE-2.0
+..
+.. Unless required by applicable law or agreed to in writing, software
+.. distributed under the License is distributed on an "AS IS" BASIS,
+.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+.. See the License for the specific language governing permissions and
+.. limitations under the License.
+
+.. _use-nodetool:
+
+Use Nodetool
+============
+
+Cassandra's ``nodetool`` allows you to narrow problems from the cluster down
+to a particular node and gives a lot of insight into the state of the Cassandra
+process itself. There are dozens of useful commands (see ``nodetool help``
+for all the commands), but briefly some of the most useful for troubleshooting:
+
+.. _nodetool-status:
+
+Cluster Status
+--------------
+
+You can use ``nodetool status`` to assess status of the cluster::
+
+ $ nodetool status <optional keyspace>
+
+ Datacenter: dc1
+ =======================
+ Status=Up/Down
+ |/ State=Normal/Leaving/Joining/Moving
+ -- Address Load Tokens Owns (effective) Host ID Rack
+ UN 127.0.1.1 4.69 GiB 1 100.0% 35ea8c9f-b7a2-40a7-b9c5-0ee8b91fdd0e r1
+ UN 127.0.1.2 4.71 GiB 1 100.0% 752e278f-b7c5-4f58-974b-9328455af73f r2
+ UN 127.0.1.3 4.69 GiB 1 100.0% 9dc1a293-2cc0-40fa-a6fd-9e6054da04a7 r3
+
+In this case we can see that we have three nodes in one datacenter with about
+4.6GB of data each and they are all "up". The up/down status of a node is
+independently determined by every node in the cluster, so you may have to run
+``nodetool status`` on multiple nodes in a cluster to see the full view.
+
+You can use ``nodetool status`` plus a little grep to see which nodes are
+down::
+
+ $ nodetool status | grep -v '^UN'
+ Datacenter: dc1
+ ===============
+ Status=Up/Down
+ |/ State=Normal/Leaving/Joining/Moving
+ -- Address Load Tokens Owns (effective) Host ID Rack
+ Datacenter: dc2
+ ===============
+ Status=Up/Down
+ |/ State=Normal/Leaving/Joining/Moving
+ -- Address Load Tokens Owns (effective) Host ID Rack
+ DN 127.0.0.5 105.73 KiB 1 33.3% df303ac7-61de-46e9-ac79-6e630115fd75 r1
+
+In this case there are two datacenters and there is one node down in datacenter
+``dc2`` and rack ``r1``. This may indicate an issue on ``127.0.0.5``
+warranting investigation.
+
+.. _nodetool-proxyhistograms:
+
+Coordinator Query Latency
+-------------------------
+You can view latency distributions of coordinator read and write latency
+to help narrow down latency issues using ``nodetool proxyhistograms``::
+
+ $ nodetool proxyhistograms
+ Percentile Read Latency Write Latency Range Latency CAS Read Latency CAS Write Latency View Write Latency
+ (micros) (micros) (micros) (micros) (micros) (micros)
+ 50% 454.83 219.34 0.00 0.00 0.00 0.00
+ 75% 545.79 263.21 0.00 0.00 0.00 0.00
+ 95% 654.95 315.85 0.00 0.00 0.00 0.00
+ 98% 785.94 379.02 0.00 0.00 0.00 0.00
+ 99% 3379.39 2346.80 0.00 0.00 0.00 0.00
+ Min 42.51 105.78 0.00 0.00 0.00 0.00
+ Max 25109.16 43388.63 0.00 0.00 0.00 0.00
+
+Here you can see the full latency distribution of reads, writes, range requests
+(e.g. ``select * from keyspace.table``), CAS read (compare phase of CAS) and
+CAS write (set phase of compare and set). These can be useful for narrowing
+down high level latency problems, for example in this case if a client had a
+20 millisecond timeout on their reads they might experience the occasional
+timeout from this node but less than 1% (since the 99% read latency is 3.3
+milliseconds < 20 milliseconds).
+
+.. _nodetool-tablehistograms:
+
+Local Query Latency
+-------------------
+
+If you know which table is having latency/error issues, you can use
+``nodetool tablehistograms`` to get a better idea of what is happening
+locally on a node::
+
+ $ nodetool tablehistograms keyspace table
+ Percentile SSTables Write Latency Read Latency Partition Size Cell Count
+ (micros) (micros) (bytes)
+ 50% 0.00 73.46 182.79 17084 103
+ 75% 1.00 88.15 315.85 17084 103
+ 95% 2.00 126.93 545.79 17084 103
+ 98% 2.00 152.32 654.95 17084 103
+ 99% 2.00 182.79 785.94 17084 103
+ Min 0.00 42.51 24.60 14238 87
+ Max 2.00 12108.97 17436.92 17084 103
+
+This shows you percentile breakdowns particularly critical metrics.
+
+The first column contains how many sstables were read per logical read. A very
+high number here indicates that you may have chosen the wrong compaction
+strategy, e.g. ``SizeTieredCompactionStrategy`` typically has many more reads
+per read than ``LeveledCompactionStrategy`` does for update heavy workloads.
+
+The second column shows you a latency breakdown of *local* write latency. In
+this case we see that while the p50 is quite good at 73 microseconds, the
+maximum latency is quite slow at 12 milliseconds. High write max latencies
+often indicate a slow commitlog volume (slow to fsync) or large writes
+that quickly saturate commitlog segments.
+
+The third column shows you a latency breakdown of *local* read latency. We can
+see that local Cassandra reads are (as expected) slower than local writes, and
+the read speed correlates highly with the number of sstables read per read.
+
+The fourth and fifth columns show distributions of partition size and column
+count per partition. These are useful for determining if the table has on
+average skinny or wide partitions and can help you isolate bad data patterns.
+For example if you have a single cell that is 2 megabytes, that is probably
+going to cause some heap pressure when it's read.
+
+.. _nodetool-tpstats:
+
+Threadpool State
+----------------
+
+You can use ``nodetool tpstats`` to view the current outstanding requests on
+a particular node. This is useful for trying to find out which resource
+(read threads, write threads, compaction, request response threads) the
+Cassandra process lacks. For example::
+
+ $ nodetool tpstats
+ Pool Name Active Pending Completed Blocked All time blocked
+ ReadStage 2 0 12 0 0
+ MiscStage 0 0 0 0 0
+ CompactionExecutor 0 0 1940 0 0
+ MutationStage 0 0 0 0 0
+ GossipStage 0 0 10293 0 0
+ Repair-Task 0 0 0 0 0
+ RequestResponseStage 0 0 16 0 0
+ ReadRepairStage 0 0 0 0 0
+ CounterMutationStage 0 0 0 0 0
+ MemtablePostFlush 0 0 83 0 0
+ ValidationExecutor 0 0 0 0 0
+ MemtableFlushWriter 0 0 30 0 0
+ ViewMutationStage 0 0 0 0 0
+ CacheCleanupExecutor 0 0 0 0 0
+ MemtableReclaimMemory 0 0 30 0 0
+ PendingRangeCalculator 0 0 11 0 0
+ SecondaryIndexManagement 0 0 0 0 0
+ HintsDispatcher 0 0 0 0 0
+ Native-Transport-Requests 0 0 192 0 0
+ MigrationStage 0 0 14 0 0
+ PerDiskMemtableFlushWriter_0 0 0 30 0 0
+ Sampler 0 0 0 0 0
+ ViewBuildExecutor 0 0 0 0 0
+ InternalResponseStage 0 0 0 0 0
+ AntiEntropyStage 0 0 0 0 0
+
+ Message type Dropped Latency waiting in queue (micros)
+ 50% 95% 99% Max
+ READ 0 N/A N/A N/A N/A
+ RANGE_SLICE 0 0.00 0.00 0.00 0.00
+ _TRACE 0 N/A N/A N/A N/A
+ HINT 0 N/A N/A N/A N/A
+ MUTATION 0 N/A N/A N/A N/A
+ COUNTER_MUTATION 0 N/A N/A N/A N/A
+ BATCH_STORE 0 N/A N/A N/A N/A
+ BATCH_REMOVE 0 N/A N/A N/A N/A
+ REQUEST_RESPONSE 0 0.00 0.00 0.00 0.00
+ PAGED_RANGE 0 N/A N/A N/A N/A
+ READ_REPAIR 0 N/A N/A N/A N/A
+
+This command shows you all kinds of interesting statistics. The first section
+shows a detailed breakdown of threadpools for each Cassandra stage, including
+how many threads are current executing (Active) and how many are waiting to
+run (Pending). Typically if you see pending executions in a particular
+threadpool that indicates a problem localized to that type of operation. For
+example if the ``RequestResponseState`` queue is backing up, that means
+that the coordinators are waiting on a lot of downstream replica requests and
+may indicate a lack of token awareness, or very high consistency levels being
+used on read requests (for example reading at ``ALL`` ties up RF
+``RequestResponseState`` threads whereas ``LOCAL_ONE`` only uses a single
+thread in the ``ReadStage`` threadpool). On the other hand if you see a lot of
+pending compactions that may indicate that your compaction threads cannot keep
+up with the volume of writes and you may need to tune either the compaction
+strategy or the ``concurrent_compactors`` or ``compaction_throughput`` options.
+
+The second section shows drops (errors) and latency distributions for all the
+major request types. Drops are cumulative since process start, but if you
+have any that indicate a serious problem as the default timeouts to qualify as
+a drop are quite high (~5-10 seconds). Dropped messages often warrants further
+investigation.
+
+.. _nodetool-compactionstats:
+
+Compaction State
+----------------
+
+As Cassandra is a LSM datastore, Cassandra sometimes has to compact sstables
+together, which can have adverse effects on performance. In particular,
+compaction uses a reasonable quantity of CPU resources, invalidates large
+quantities of the OS `page cache <https://en.wikipedia.org/wiki/Page_cache>`_,
+and can put a lot of load on your disk drives. There are great
+:ref:`os tools <os-iostat>` to determine if this is the case, but often it's a
+good idea to check if compactions are even running using
+``nodetool compactionstats``::
+
+ $ nodetool compactionstats
+ pending tasks: 2
+ - keyspace.table: 2
+
+ id compaction type keyspace table completed total unit progress
+ 2062b290-7f3a-11e8-9358-cd941b956e60 Compaction keyspace table 21848273 97867583 bytes 22.32%
+ Active compaction remaining time : 0h00m04s
+
+In this case there is a single compaction running on the ``keyspace.table``
+table, has completed 21.8 megabytes of 97 and Cassandra estimates (based on
+the configured compaction throughput) that this will take 4 seconds. You can
+also pass ``-H`` to get the units in a human readable format.
+
+Generally each running compaction can consume a single core, but the more
+you do in parallel the faster data compacts. Compaction is crucial to ensuring
+good read performance so having the right balance of concurrent compactions
+such that compactions complete quickly but don't take too many resources
+away from query threads is very important for performance. If you notice
+compaction unable to keep up, try tuning Cassandra's ``concurrent_compactors``
+or ``compaction_throughput`` options.
diff --git a/doc/source/troubleshooting/use_tools.rst b/doc/source/troubleshooting/use_tools.rst
new file mode 100644
index 0000000..b1347cc
--- /dev/null
+++ b/doc/source/troubleshooting/use_tools.rst
@@ -0,0 +1,542 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements. See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership. The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License. You may obtain a copy of the License at
+..
+.. http://www.apache.org/licenses/LICENSE-2.0
+..
+.. Unless required by applicable law or agreed to in writing, software
+.. distributed under the License is distributed on an "AS IS" BASIS,
+.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+.. See the License for the specific language governing permissions and
+.. limitations under the License.
+
+.. _use-os-tools:
+
+Diving Deep, Use External Tools
+===============================
+
+Machine access allows operators to dive even deeper than logs and ``nodetool``
+allow. While every Cassandra operator may have their personal favorite
+toolsets for troubleshooting issues, this page contains some of the most common
+operator techniques and examples of those tools. Many of these commands work
+only on Linux, but if you are deploying on a different operating system you may
+have access to other substantially similar tools that assess similar OS level
+metrics and processes.
+
+JVM Tooling
+-----------
+The JVM ships with a number of useful tools. Some of them are useful for
+debugging Cassandra issues, especially related to heap and execution stacks.
+
+**NOTE**: There are two common gotchas with JVM tooling and Cassandra:
+
+1. By default Cassandra ships with ``-XX:+PerfDisableSharedMem`` set to prevent
+ long pauses (see ``CASSANDRA-9242`` and ``CASSANDRA-9483`` for details). If
+ you want to use JVM tooling you can instead have ``/tmp`` mounted on an in
+ memory ``tmpfs`` which also effectively works around ``CASSANDRA-9242``.
+2. Make sure you run the tools as the same user as Cassandra is running as,
+ e.g. if the database is running as ``cassandra`` the tool also has to be
+ run as ``cassandra``, e.g. via ``sudo -u cassandra <cmd>``.
+
+Garbage Collection State (jstat)
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+If you suspect heap pressure you can use ``jstat`` to dive deep into the
+garbage collection state of a Cassandra process. This command is always
+safe to run and yields detailed heap information including eden heap usage (E),
+old generation heap usage (O), count of eden collections (YGC), time spend in
+eden collections (YGCT), old/mixed generation collections (FGC) and time spent
+in old/mixed generation collections (FGCT)::
+
+
+ jstat -gcutil <cassandra pid> 500ms
+ S0 S1 E O M CCS YGC YGCT FGC FGCT GCT
+ 0.00 0.00 81.53 31.16 93.07 88.20 12 0.151 3 0.257 0.408
+ 0.00 0.00 82.36 31.16 93.07 88.20 12 0.151 3 0.257 0.408
+ 0.00 0.00 82.36 31.16 93.07 88.20 12 0.151 3 0.257 0.408
+ 0.00 0.00 83.19 31.16 93.07 88.20 12 0.151 3 0.257 0.408
+ 0.00 0.00 83.19 31.16 93.07 88.20 12 0.151 3 0.257 0.408
+ 0.00 0.00 84.19 31.16 93.07 88.20 12 0.151 3 0.257 0.408
+ 0.00 0.00 84.19 31.16 93.07 88.20 12 0.151 3 0.257 0.408
+ 0.00 0.00 85.03 31.16 93.07 88.20 12 0.151 3 0.257 0.408
+ 0.00 0.00 85.03 31.16 93.07 88.20 12 0.151 3 0.257 0.408
+ 0.00 0.00 85.94 31.16 93.07 88.20 12 0.151 3 0.257 0.408
+
+In this case we see we have a relatively healthy heap profile, with 31.16%
+old generation heap usage and 83% eden. If the old generation routinely is
+above 75% then you probably need more heap (assuming CMS with a 75% occupancy
+threshold). If you do have such persistently high old gen that often means you
+either have under-provisioned the old generation heap, or that there is too
+much live data on heap for Cassandra to collect (e.g. because of memtables).
+Another thing to watch for is time between young garbage collections (YGC),
+which indicate how frequently the eden heap is collected. Each young gc pause
+is about 20-50ms, so if you have a lot of them your clients will notice in
+their high percentile latencies.
+
+Thread Information (jstack)
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+To get a point in time snapshot of exactly what Cassandra is doing, run
+``jstack`` against the Cassandra PID. **Note** that this does pause the JVM for
+a very brief period (<20ms).::
+
+ $ jstack <cassandra pid> > threaddump
+
+ # display the threaddump
+ $ cat threaddump
+ ...
+
+ # look at runnable threads
+ $grep RUNNABLE threaddump -B 1
+ "Attach Listener" #15 daemon prio=9 os_prio=0 tid=0x00007f829c001000 nid=0x3a74 waiting on condition [0x0000000000000000]
+ java.lang.Thread.State: RUNNABLE
+ --
+ "DestroyJavaVM" #13 prio=5 os_prio=0 tid=0x00007f82e800e000 nid=0x2a19 waiting on condition [0x0000000000000000]
+ java.lang.Thread.State: RUNNABLE
+ --
+ "JPS thread pool" #10 prio=5 os_prio=0 tid=0x00007f82e84d0800 nid=0x2a2c runnable [0x00007f82d0856000]
+ java.lang.Thread.State: RUNNABLE
+ --
+ "Service Thread" #9 daemon prio=9 os_prio=0 tid=0x00007f82e80d7000 nid=0x2a2a runnable [0x0000000000000000]
+ java.lang.Thread.State: RUNNABLE
+ --
+ "C1 CompilerThread3" #8 daemon prio=9 os_prio=0 tid=0x00007f82e80cc000 nid=0x2a29 waiting on condition [0x0000000000000000]
+ java.lang.Thread.State: RUNNABLE
+ --
+ ...
+
+ # Note that the nid is the Linux thread id
+
+Some of the most important information in the threaddumps are waiting/blocking
+threads, including what locks or monitors the thread is blocking/waiting on.
+
+Basic OS Tooling
+----------------
+A great place to start when debugging a Cassandra issue is understanding how
+Cassandra is interacting with system resources. The following are all
+resources that Cassandra makes heavy uses of:
+
+* CPU cores. For executing concurrent user queries
+* CPU processing time. For query activity (data decompression, row merging,
+ etc...)
+* CPU processing time (low priority). For background tasks (compaction,
+ streaming, etc ...)
+* RAM for Java Heap. Used to hold internal data-structures and by default the
+ Cassandra memtables. Heap space is a crucial component of write performance
+ as well as generally.
+* RAM for OS disk cache. Used to cache frequently accessed SSTable blocks. OS
+ disk cache is a crucial component of read performance.
+* Disks. Cassandra cares a lot about disk read latency, disk write throughput,
+ and of course disk space.
+* Network latency. Cassandra makes many internode requests, so network latency
+ between nodes can directly impact performance.
+* Network throughput. Cassandra (as other databases) frequently have the
+ so called "incast" problem where a small request (e.g. ``SELECT * from
+ foo.bar``) returns a massively large result set (e.g. the entire dataset).
+ In such situations outgoing bandwidth is crucial.
+
+Often troubleshooting Cassandra comes down to troubleshooting what resource
+the machine or cluster is running out of. Then you create more of that resource
+or change the query pattern to make less use of that resource.
+
+High Level Resource Usage (top/htop)
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Cassandra makes signifiant use of system resources, and often the very first
+useful action is to run ``top`` or ``htop`` (`website
+<https://hisham.hm/htop/>`_)to see the state of the machine.
+
+Useful things to look at:
+
+* System load levels. While these numbers can be confusing, generally speaking
+ if the load average is greater than the number of CPU cores, Cassandra
+ probably won't have very good (sub 100 millisecond) latencies. See
+ `Linux Load Averages <http://www.brendangregg.com/blog/2017-08-08/linux-load-averages.html>`_
+ for more information.
+* CPU utilization. ``htop`` in particular can help break down CPU utilization
+ into ``user`` (low and normal priority), ``system`` (kernel), and ``io-wait``
+ . Cassandra query threads execute as normal priority ``user`` threads, while
+ compaction threads execute as low priority ``user`` threads. High ``system``
+ time could indicate problems like thread contention, and high ``io-wait``
+ may indicate slow disk drives. This can help you understand what Cassandra
+ is spending processing resources doing.
+* Memory usage. Look for which programs have the most resident memory, it is
+ probably Cassandra. The number for Cassandra is likely inaccurately high due
+ to how Linux (as of 2018) accounts for memory mapped file memory.
+
+.. _os-iostat:
+
+IO Usage (iostat)
+^^^^^^^^^^^^^^^^^
+Use iostat to determine how data drives are faring, including latency
+distributions, throughput, and utilization::
+
+ $ sudo iostat -xdm 2
+ Linux 4.13.0-13-generic (hostname) 07/03/2018 _x86_64_ (8 CPU)
+
+ Device: rrqm/s wrqm/s r/s w/s rMB/s wMB/s avgrq-sz avgqu-sz await r_await w_await svctm %util
+ sda 0.00 0.28 0.32 5.42 0.01 0.13 48.55 0.01 2.21 0.26 2.32 0.64 0.37
+ sdb 0.00 0.00 0.00 0.00 0.00 0.00 79.34 0.00 0.20 0.20 0.00 0.16 0.00
+ sdc 0.34 0.27 0.76 0.36 0.01 0.02 47.56 0.03 26.90 2.98 77.73 9.21 1.03
+
+ Device: rrqm/s wrqm/s r/s w/s rMB/s wMB/s avgrq-sz avgqu-sz await r_await w_await svctm %util
+ sda 0.00 0.00 2.00 32.00 0.01 4.04 244.24 0.54 16.00 0.00 17.00 1.06 3.60
+ sdb 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
+ sdc 0.00 24.50 0.00 114.00 0.00 11.62 208.70 5.56 48.79 0.00 48.79 1.12 12.80
+
+
+In this case we can see that ``/dev/sdc1`` is a very slow drive, having an
+``await`` close to 50 milliseconds and an ``avgqu-sz`` close to 5 ios. The
+drive is not particularly saturated (utilization is only 12.8%), but we should
+still be concerned about how this would affect our p99 latency since 50ms is
+quite long for typical Cassandra operations. That being said, in this case
+most of the latency is present in writes (typically writes are more latent
+than reads), which due to the LSM nature of Cassandra is often hidden from
+the user.
+
+Important metrics to assess using iostat:
+
+* Reads and writes per second. These numbers will change with the workload,
+ but generally speaking the more reads Cassandra has to do from disk the
+ slower Cassandra read latencies are. Large numbers of reads per second
+ can be a dead giveaway that the cluster has insufficient memory for OS
+ page caching.
+* Write throughput. Cassandra's LSM model defers user writes and batches them
+ together, which means that throughput to the underlying medium is the most
+ important write metric for Cassandra.
+* Read latency (``r_await``). When Cassandra missed the OS page cache and reads
+ from SSTables, the read latency directly determines how fast Cassandra can
+ respond with the data.
+* Write latency. Cassandra is less sensitive to write latency except when it
+ syncs the commit log. This typically enters into the very high percentiles of
+ write latency.
+
+Note that to get detailed latency breakdowns you will need a more advanced
+tool such as :ref:`bcc-tools <use-bcc-tools>`.
+
+OS page Cache Usage
+^^^^^^^^^^^^^^^^^^^
+As Cassandra makes heavy use of memory mapped files, the health of the
+operating system's `Page Cache <https://en.wikipedia.org/wiki/Page_cache>`_ is
+crucial to performance. Start by finding how much available cache is in the
+system::
+
+ $ free -g
+ total used free shared buff/cache available
+ Mem: 15 9 2 0 3 5
+ Swap: 0 0 0
+
+In this case 9GB of memory is used by user processes (Cassandra heap) and 8GB
+is available for OS page cache. Of that, 3GB is actually used to cache files.
+If most memory is used and unavailable to the page cache, Cassandra performance
+can suffer significantly. This is why Cassandra starts with a reasonably small
+amount of memory reserved for the heap.
+
+If you suspect that you are missing the OS page cache frequently you can use
+advanced tools like :ref:`cachestat <use-bcc-tools>` or
+:ref:`vmtouch <use-vmtouch>` to dive deeper.
+
+Network Latency and Reliability
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Whenever Cassandra does writes or reads that involve other replicas,
+``LOCAL_QUORUM`` reads for example, one of the dominant effects on latency is
+network latency. When trying to debug issues with multi machine operations,
+the network can be an important resource to investigate. You can determine
+internode latency using tools like ``ping`` and ``traceroute`` or most
+effectively ``mtr``::
+
+ $ mtr -nr www.google.com
+ Start: Sun Jul 22 13:10:28 2018
+ HOST: hostname Loss% Snt Last Avg Best Wrst StDev
+ 1.|-- 192.168.1.1 0.0% 10 2.0 1.9 1.1 3.7 0.7
+ 2.|-- 96.123.29.15 0.0% 10 11.4 11.0 9.0 16.4 1.9
+ 3.|-- 68.86.249.21 0.0% 10 10.6 10.7 9.0 13.7 1.1
+ 4.|-- 162.141.78.129 0.0% 10 11.5 10.6 9.6 12.4 0.7
+ 5.|-- 162.151.78.253 0.0% 10 10.9 12.1 10.4 20.2 2.8
+ 6.|-- 68.86.143.93 0.0% 10 12.4 12.6 9.9 23.1 3.8
+ 7.|-- 96.112.146.18 0.0% 10 11.9 12.4 10.6 15.5 1.6
+ 9.|-- 209.85.252.250 0.0% 10 13.7 13.2 12.5 13.9 0.0
+ 10.|-- 108.170.242.238 0.0% 10 12.7 12.4 11.1 13.0 0.5
+ 11.|-- 74.125.253.149 0.0% 10 13.4 13.7 11.8 19.2 2.1
+ 12.|-- 216.239.62.40 0.0% 10 13.4 14.7 11.5 26.9 4.6
+ 13.|-- 108.170.242.81 0.0% 10 14.4 13.2 10.9 16.0 1.7
+ 14.|-- 72.14.239.43 0.0% 10 12.2 16.1 11.0 32.8 7.1
+ 15.|-- 216.58.195.68 0.0% 10 25.1 15.3 11.1 25.1 4.8
+
+In this example of ``mtr``, we can rapidly assess the path that your packets
+are taking, as well as what their typical loss and latency are. Packet loss
+typically leads to between ``200ms`` and ``3s`` of additional latency, so that
+can be a common cause of latency issues.
+
+Network Throughput
+^^^^^^^^^^^^^^^^^^
+As Cassandra is sensitive to outgoing bandwidth limitations, sometimes it is
+useful to determine if network throughput is limited. One handy tool to do
+this is `iftop <https://www.systutorials.com/docs/linux/man/8-iftop/>`_ which
+shows both bandwidth usage as well as connection information at a glance. An
+example showing traffic during a stress run against a local ``ccm`` cluster::
+
+ $ # remove the -t for ncurses instead of pure text
+ $ sudo iftop -nNtP -i lo
+ interface: lo
+ IP address is: 127.0.0.1
+ MAC address is: 00:00:00:00:00:00
+ Listening on lo
+ # Host name (port/service if enabled) last 2s last 10s last 40s cumulative
+ --------------------------------------------------------------------------------------------
+ 1 127.0.0.1:58946 => 869Kb 869Kb 869Kb 217KB
+ 127.0.0.3:9042 <= 0b 0b 0b 0B
+ 2 127.0.0.1:54654 => 736Kb 736Kb 736Kb 184KB
+ 127.0.0.1:9042 <= 0b 0b 0b 0B
+ 3 127.0.0.1:51186 => 669Kb 669Kb 669Kb 167KB
+ 127.0.0.2:9042 <= 0b 0b 0b 0B
+ 4 127.0.0.3:9042 => 3.30Kb 3.30Kb 3.30Kb 845B
+ 127.0.0.1:58946 <= 0b 0b 0b 0B
+ 5 127.0.0.1:9042 => 2.79Kb 2.79Kb 2.79Kb 715B
+ 127.0.0.1:54654 <= 0b 0b 0b 0B
+ 6 127.0.0.2:9042 => 2.54Kb 2.54Kb 2.54Kb 650B
+ 127.0.0.1:51186 <= 0b 0b 0b 0B
+ 7 127.0.0.1:36894 => 1.65Kb 1.65Kb 1.65Kb 423B
+ 127.0.0.5:7000 <= 0b 0b 0b 0B
+ 8 127.0.0.1:38034 => 1.50Kb 1.50Kb 1.50Kb 385B
+ 127.0.0.2:7000 <= 0b 0b 0b 0B
+ 9 127.0.0.1:56324 => 1.50Kb 1.50Kb 1.50Kb 383B
+ 127.0.0.1:7000 <= 0b 0b 0b 0B
+ 10 127.0.0.1:53044 => 1.43Kb 1.43Kb 1.43Kb 366B
+ 127.0.0.4:7000 <= 0b 0b 0b 0B
+ --------------------------------------------------------------------------------------------
+ Total send rate: 2.25Mb 2.25Mb 2.25Mb
+ Total receive rate: 0b 0b 0b
+ Total send and receive rate: 2.25Mb 2.25Mb 2.25Mb
+ --------------------------------------------------------------------------------------------
+ Peak rate (sent/received/total): 2.25Mb 0b 2.25Mb
+ Cumulative (sent/received/total): 576KB 0B 576KB
+ ============================================================================================
+
+In this case we can see that bandwidth is fairly shared between many peers,
+but if the total was getting close to the rated capacity of the NIC or was focussed
+on a single client, that may indicate a clue as to what issue is occurring.
+
+Advanced tools
+--------------
+Sometimes as an operator you may need to really dive deep. This is where
+advanced OS tooling can come in handy.
+
+.. _use-bcc-tools:
+
+bcc-tools
+^^^^^^^^^
+Most modern Linux distributions (kernels newer than ``4.1``) support `bcc-tools
+<https://github.com/iovisor/bcc>`_ for diving deep into performance problems.
+First install ``bcc-tools``, e.g. via ``apt`` on Debian::
+
+ $ apt install bcc-tools
+
+Then you can use all the tools that ``bcc-tools`` contains. One of the most
+useful tools is ``cachestat``
+(`cachestat examples <https://github.com/iovisor/bcc/blob/master/tools/cachestat_example.txt>`_)
+which allows you to determine exactly how many OS page cache hits and misses
+are happening::
+
+ $ sudo /usr/share/bcc/tools/cachestat -T 1
+ TIME TOTAL MISSES HITS DIRTIES BUFFERS_MB CACHED_MB
+ 18:44:08 66 66 0 64 88 4427
+ 18:44:09 40 40 0 75 88 4427
+ 18:44:10 4353 45 4308 203 88 4427
+ 18:44:11 84 77 7 13 88 4428
+ 18:44:12 2511 14 2497 14 88 4428
+ 18:44:13 101 98 3 18 88 4428
+ 18:44:14 16741 0 16741 58 88 4428
+ 18:44:15 1935 36 1899 18 88 4428
+ 18:44:16 89 34 55 18 88 4428
+
+In this case there are not too many page cache ``MISSES`` which indicates a
+reasonably sized cache. These metrics are the most direct measurement of your
+Cassandra node's "hot" dataset. If you don't have enough cache, ``MISSES`` will
+be high and performance will be slow. If you have enough cache, ``MISSES`` will
+be low and performance will be fast (as almost all reads are being served out
+of memory).
+
+You can also measure disk latency distributions using ``biolatency``
+(`biolatency examples <https://github.com/iovisor/bcc/blob/master/tools/biolatency_example.txt>`_)
+to get an idea of how slow Cassandra will be when reads miss the OS page Cache
+and have to hit disks::
+
+ $ sudo /usr/share/bcc/tools/biolatency -D 10
+ Tracing block device I/O... Hit Ctrl-C to end.
+
+
+ disk = 'sda'
+ usecs : count distribution
+ 0 -> 1 : 0 | |
+ 2 -> 3 : 0 | |
+ 4 -> 7 : 0 | |
+ 8 -> 15 : 0 | |
+ 16 -> 31 : 12 |****************************************|
+ 32 -> 63 : 9 |****************************** |
+ 64 -> 127 : 1 |*** |
+ 128 -> 255 : 3 |********** |
+ 256 -> 511 : 7 |*********************** |
+ 512 -> 1023 : 2 |****** |
+
+ disk = 'sdc'
+ usecs : count distribution
+ 0 -> 1 : 0 | |
+ 2 -> 3 : 0 | |
+ 4 -> 7 : 0 | |
+ 8 -> 15 : 0 | |
+ 16 -> 31 : 0 | |
+ 32 -> 63 : 0 | |
+ 64 -> 127 : 41 |************ |
+ 128 -> 255 : 17 |***** |
+ 256 -> 511 : 13 |*** |
+ 512 -> 1023 : 2 | |
+ 1024 -> 2047 : 0 | |
+ 2048 -> 4095 : 0 | |
+ 4096 -> 8191 : 56 |***************** |
+ 8192 -> 16383 : 131 |****************************************|
+ 16384 -> 32767 : 9 |** |
+
+In this case most ios on the data drive (``sdc``) are fast, but many take
+between 8 and 16 milliseconds.
+
+Finally ``biosnoop`` (`examples <https://github.com/iovisor/bcc/blob/master/tools/biosnoop_example.txt>`_)
+can be used to dive even deeper and see per IO latencies::
+
+ $ sudo /usr/share/bcc/tools/biosnoop | grep java | head
+ 0.000000000 java 17427 sdc R 3972458600 4096 13.58
+ 0.000818000 java 17427 sdc R 3972459408 4096 0.35
+ 0.007098000 java 17416 sdc R 3972401824 4096 5.81
+ 0.007896000 java 17416 sdc R 3972489960 4096 0.34
+ 0.008920000 java 17416 sdc R 3972489896 4096 0.34
+ 0.009487000 java 17427 sdc R 3972401880 4096 0.32
+ 0.010238000 java 17416 sdc R 3972488368 4096 0.37
+ 0.010596000 java 17427 sdc R 3972488376 4096 0.34
+ 0.011236000 java 17410 sdc R 3972488424 4096 0.32
+ 0.011825000 java 17427 sdc R 3972488576 16384 0.65
+ ... time passes
+ 8.032687000 java 18279 sdc R 10899712 122880 3.01
+ 8.033175000 java 18279 sdc R 10899952 8192 0.46
+ 8.073295000 java 18279 sdc R 23384320 122880 3.01
+ 8.073768000 java 18279 sdc R 23384560 8192 0.46
+
+
+With ``biosnoop`` you see every single IO and how long they take. This data
+can be used to construct the latency distributions in ``biolatency`` but can
+also be used to better understand how disk latency affects performance. For
+example this particular drive takes ~3ms to service a memory mapped read due to
+the large default value (``128kb``) of ``read_ahead_kb``. To improve point read
+performance you may may want to decrease ``read_ahead_kb`` on fast data volumes
+such as SSDs while keeping the a higher value like ``128kb`` value is probably
+right for HDs. There are tradeoffs involved, see `queue-sysfs
+<https://www.kernel.org/doc/Documentation/block/queue-sysfs.txt>`_ docs for more
+information, but regardless ``biosnoop`` is useful for understanding *how*
+Cassandra uses drives.
+
+.. _use-vmtouch:
+
+vmtouch
+^^^^^^^
+Sometimes it's useful to know how much of the Cassandra data files are being
+cached by the OS. A great tool for answering this question is
+`vmtouch <https://github.com/hoytech/vmtouch>`_.
+
+First install it::
+
+ $ git clone https://github.com/hoytech/vmtouch.git
+ $ cd vmtouch
+ $ make
+
+Then run it on the Cassandra data directory::
+
+ $ ./vmtouch /var/lib/cassandra/data/
+ Files: 312
+ Directories: 92
+ Resident Pages: 62503/64308 244M/251M 97.2%
+ Elapsed: 0.005657 seconds
+
+In this case almost the entire dataset is hot in OS page Cache. Generally
+speaking the percentage doesn't really matter unless reads are missing the
+cache (per e.g. :ref:`cachestat <use-bcc-tools>`), in which case having
+additional memory may help read performance.
+
+CPU Flamegraphs
+^^^^^^^^^^^^^^^
+Cassandra often uses a lot of CPU, but telling *what* it is doing can prove
+difficult. One of the best ways to analyze Cassandra on CPU time is to use
+`CPU Flamegraphs <http://www.brendangregg.com/FlameGraphs/cpuflamegraphs.html>`_
+which display in a useful way which areas of Cassandra code are using CPU. This
+may help narrow down a compaction problem to a "compaction problem dropping
+tombstones" or just generally help you narrow down what Cassandra is doing
+while it is having an issue. To get CPU flamegraphs follow the instructions for
+`Java Flamegraphs
+<http://www.brendangregg.com/FlameGraphs/cpuflamegraphs.html#Java>`_.
+
+Generally:
+
+1. Enable the ``-XX:+PreserveFramePointer`` option in Cassandra's
+ ``jvm.options`` configuation file. This has a negligible performance impact
+ but allows you actually see what Cassandra is doing.
+2. Run ``perf`` to get some data.
+3. Send that data through the relevant scripts in the FlameGraph toolset and
+ convert the data into a pretty flamegraph. View the resulting SVG image in
+ a browser or other image browser.
+
+For example just cloning straight off github we first install the
+``perf-map-agent`` to the location of our JVMs (assumed to be
+``/usr/lib/jvm``)::
+
+ $ sudo bash
+ $ export JAVA_HOME=/usr/lib/jvm/java-8-oracle/
+ $ cd /usr/lib/jvm
+ $ git clone --depth=1 https://github.com/jvm-profiling-tools/perf-map-agent
+ $ cd perf-map-agent
+ $ cmake .
+ $ make
+
+Now to get a flamegraph::
+
+ $ git clone --depth=1 https://github.com/brendangregg/FlameGraph
+ $ sudo bash
+ $ cd FlameGraph
+ $ # Record traces of Cassandra and map symbols for all java processes
+ $ perf record -F 49 -a -g -p <CASSANDRA PID> -- sleep 30; ./jmaps
+ $ # Translate the data
+ $ perf script > cassandra_stacks
+ $ cat cassandra_stacks | ./stackcollapse-perf.pl | grep -v cpu_idle | \
+ ./flamegraph.pl --color=java --hash > cassandra_flames.svg
+
+
+The resulting SVG is searchable, zoomable, and generally easy to introspect
+using a browser.
+
+.. _packet-capture:
+
+Packet Capture
+^^^^^^^^^^^^^^
+Sometimes you have to understand what queries a Cassandra node is performing
+*right now* to troubleshoot an issue. For these times trusty packet capture
+tools like ``tcpdump`` and `Wireshark
+<https://www.wireshark.org/>`_ can be very helpful to dissect packet captures.
+Wireshark even has native `CQL support
+<https://www.wireshark.org/docs/dfref/c/cql.html>`_ although it sometimes has
+compatibility issues with newer Cassandra protocol releases.
+
+To get a packet capture first capture some packets::
+
+ $ sudo tcpdump -U -s0 -i <INTERFACE> -w cassandra.pcap -n "tcp port 9042"
+
+Now open it up with wireshark::
+
+ $ wireshark cassandra.pcap
+
+If you don't see CQL like statements try telling to decode as CQL by right
+clicking on a packet going to 9042 -> ``Decode as`` -> select CQL from the
+dropdown for port 9042.
+
+If you don't want to do this manually or use a GUI, you can also use something
+like `cqltrace <https://github.com/jolynch/cqltrace>`_ to ease obtaining and
+parsing CQL packet captures.
diff --git a/examples/hadoop_cql3_word_count/README.txt b/examples/hadoop_cql3_word_count/README.txt
deleted file mode 100644
index b6ee33f..0000000
--- a/examples/hadoop_cql3_word_count/README.txt
+++ /dev/null
@@ -1,51 +0,0 @@
-Introduction
-============
-
-WordCount hadoop example: Inserts a bunch of words across multiple rows,
-and counts them, with RandomPartitioner. The word_count_counters example sums
-the value of counter columns for a key.
-
-The scripts in bin/ assume you are running with cwd of examples/word_count.
-
-
-Running
-=======
-
-First build and start a Cassandra server with the default configuration*. Ensure that the Thrift
-interface is enabled, either by setting start_rpc:true in cassandra.yaml or by running
-`nodetool enablethrift` after startup.
-Once Cassandra has started and the Thrift interface is available, run
-
-contrib/word_count$ ant
-contrib/word_count$ bin/word_count_setup
-contrib/word_count$ bin/word_count
-contrib/word_count$ bin/word_count_counters
-
-In order to view the results in Cassandra, one can use bin/cqlsh and
-perform the following operations:
-$ bin/cqlsh localhost
-> use cql3_wordcount;
-> select * from output_words;
-
-The output of the word count can now be configured. In the bin/word_count
-file, you can specify the OUTPUT_REDUCER. The two options are 'filesystem'
-and 'cassandra'. The filesystem option outputs to the /tmp/word_count*
-directories. The cassandra option outputs to the 'output_words' column family
-in the 'cql3_wordcount' keyspace. 'cassandra' is the default.
-
-Read the code in src/ for more details.
-
-The word_count_counters example sums the counter columns for a row. The output
-is written to a text file in /tmp/word_count_counters.
-
-*It is recommended to turn off vnodes when running Cassandra with hadoop.
-This is done by setting "num_tokens: 1" in cassandra.yaml. If you want to
-point wordcount at a real cluster, modify the seed and listenaddress
-settings accordingly.
-
-
-Troubleshooting
-===============
-
-word_count uses conf/logback.xml to log to wc.out.
-
diff --git a/examples/hadoop_cql3_word_count/bin/word_count b/examples/hadoop_cql3_word_count/bin/word_count
deleted file mode 100755
index 76cca7d..0000000
--- a/examples/hadoop_cql3_word_count/bin/word_count
+++ /dev/null
@@ -1,62 +0,0 @@
-#!/bin/sh
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-cwd=`dirname $0`
-
-# Cassandra class files.
-if [ ! -d $cwd/../../../build/classes/main ]; then
- echo "Unable to locate cassandra class files" >&2
- exit 1
-fi
-
-# word_count Jar.
-if [ ! -e $cwd/../build/word_count.jar ]; then
- echo "Unable to locate word_count jar" >&2
- exit 1
-fi
-
-CLASSPATH=$CLASSPATH:$cwd/../conf
-CLASSPATH=$CLASSPATH:$cwd/../build/word_count.jar
-CLASSPATH=$CLASSPATH:$cwd/../../../build/classes/main
-CLASSPATH=$CLASSPATH:$cwd/../../../build/classes/thrift
-for jar in $cwd/../build/lib/jars/*.jar; do
- CLASSPATH=$CLASSPATH:$jar
-done
-for jar in $cwd/../../../lib/*.jar; do
- CLASSPATH=$CLASSPATH:$jar
-done
-for jar in $cwd/../../../build/lib/jars/*.jar; do
- CLASSPATH=$CLASSPATH:$jar
-done
-
-if [ -x $JAVA_HOME/bin/java ]; then
- JAVA=$JAVA_HOME/bin/java
-else
- JAVA=`which java`
-fi
-
-if [ "x$JAVA" = "x" ]; then
- echo "Java executable not found (hint: set JAVA_HOME)" >&2
- exit 1
-fi
-
-OUTPUT_REDUCER=cassandra
-INPUT_MAPPER=native
-
-#echo $CLASSPATH
-"$JAVA" -Xmx1G -ea -cp "$CLASSPATH" WordCount output_reducer=$OUTPUT_REDUCER input_mapper=$INPUT_MAPPER
diff --git a/examples/hadoop_cql3_word_count/bin/word_count_counters b/examples/hadoop_cql3_word_count/bin/word_count_counters
deleted file mode 100755
index cc1243f..0000000
--- a/examples/hadoop_cql3_word_count/bin/word_count_counters
+++ /dev/null
@@ -1,61 +0,0 @@
-#!/bin/sh
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-cwd=`dirname $0`
-
-# Cassandra class files.
-if [ ! -d $cwd/../../../build/classes/main ]; then
- echo "Unable to locate cassandra class files" >&2
- exit 1
-fi
-
-# word_count Jar.
-if [ ! -e $cwd/../build/word_count.jar ]; then
- echo "Unable to locate word_count jar" >&2
- exit 1
-fi
-
-CLASSPATH=$CLASSPATH:$cwd/../conf
-CLASSPATH=$CLASSPATH:$cwd/../build/word_count.jar
-CLASSPATH=$CLASSPATH:$cwd/../../../build/classes/main
-CLASSPATH=$CLASSPATH:$cwd/../../../build/classes/thrift
-for jar in $cwd/../build/lib/jars/*.jar; do
- CLASSPATH=$CLASSPATH:$jar
-done
-for jar in $cwd/../../../lib/*.jar; do
- CLASSPATH=$CLASSPATH:$jar
-done
-for jar in $cwd/../../../build/lib/jars/*.jar; do
- CLASSPATH=$CLASSPATH:$jar
-done
-
-if [ -x $JAVA_HOME/bin/java ]; then
- JAVA=$JAVA_HOME/bin/java
-else
- JAVA=`which java`
-fi
-
-if [ "x$JAVA" = "x" ]; then
- echo "Java executable not found (hint: set JAVA_HOME)" >&2
- exit 1
-fi
-
-INPUT_MAPPER=native
-
-#echo $CLASSPATH
-"$JAVA" -Xmx1G -ea -cp "$CLASSPATH" WordCountCounters input_mapper=$INPUT_MAPPER
diff --git a/examples/hadoop_cql3_word_count/bin/word_count_setup b/examples/hadoop_cql3_word_count/bin/word_count_setup
deleted file mode 100755
index 6e5650f..0000000
--- a/examples/hadoop_cql3_word_count/bin/word_count_setup
+++ /dev/null
@@ -1,61 +0,0 @@
-#!/bin/sh
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-cwd=`dirname $0`
-
-# Cassandra class files.
-if [ ! -d $cwd/../../../build/classes/main ]; then
- echo "Unable to locate cassandra class files" >&2
- exit 1
-fi
-
-# word_count Jar.
-if [ ! -e $cwd/../build/word_count.jar ]; then
- echo "Unable to locate word_count jar" >&2
- exit 1
-fi
-
-CLASSPATH=$CLASSPATH:$cwd/../build/word_count.jar
-CLASSPATH=$CLASSPATH:.:$cwd/../../../build/classes/main
-CLASSPATH=$CLASSPATH:.:$cwd/../../../build/classes/thrift
-for jar in $cwd/../build/lib/jars/*.jar; do
- CLASSPATH=$CLASSPATH:$jar
-done
-for jar in $cwd/../../../lib/*.jar; do
- CLASSPATH=$CLASSPATH:$jar
-done
-for jar in $cwd/../../../build/lib/jars/*.jar; do
- CLASSPATH=$CLASSPATH:$jar
-done
-
-if [ -x $JAVA_HOME/bin/java ]; then
- JAVA=$JAVA_HOME/bin/java
-else
- JAVA=`which java`
-fi
-
-if [ "x$JAVA" = "x" ]; then
- echo "Java executable not found (hint: set JAVA_HOME)" >&2
- exit 1
-fi
-
-HOST=localhost
-PORT=9160
-FRAMED=true
-
-"$JAVA" -Xmx1G -ea -Dcassandra.host=$HOST -Dcassandra.port=$PORT -Dcassandra.framed=$FRAMED -cp "$CLASSPATH" WordCountSetup
diff --git a/examples/hadoop_cql3_word_count/build.xml b/examples/hadoop_cql3_word_count/build.xml
deleted file mode 100644
index 939e1b3..0000000
--- a/examples/hadoop_cql3_word_count/build.xml
+++ /dev/null
@@ -1,113 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~ with the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-<project default="jar" name="word_count" xmlns:ivy="antlib:org.apache.ivy.ant">
- <property name="cassandra.dir" value="../.." />
- <property name="cassandra.dir.lib" value="${cassandra.dir}/lib" />
- <property name="cassandra.classes" value="${cassandra.dir}/build/classes" />
- <property name="build.src" value="${basedir}/src" />
- <property name="build.dir" value="${basedir}/build" />
- <property name="ivy.lib.dir" value="${build.dir}/lib" />
- <property name="build.classes" value="${build.dir}/classes" />
- <property name="final.name" value="word_count" />
- <property name="ivy.version" value="2.1.0" />
- <property name="ivy.url"
- value="http://repo2.maven.org/maven2/org/apache/ivy/ivy" />
-
- <condition property="ivy.jar.exists">
- <available file="${build.dir}/ivy-${ivy.version}.jar" />
- </condition>
-
- <path id="autoivy.classpath">
- <fileset dir="${ivy.lib.dir}">
- <include name="**/*.jar" />
- </fileset>
- <pathelement location="${build.dir}/ivy-${ivy.version}.jar"/>
- </path>
-
- <path id="wordcount.build.classpath">
- <fileset dir="${ivy.lib.dir}">
- <include name="**/*.jar" />
- </fileset>
- <!-- cassandra dependencies -->
- <fileset dir="${cassandra.dir.lib}">
- <include name="**/*.jar" />
- </fileset>
- <fileset dir="${cassandra.dir}/build/lib/jars">
- <include name="**/*.jar" />
- </fileset>
- <pathelement location="${cassandra.classes}/main" />
- <pathelement location="${cassandra.classes}/thrift" />
- </path>
-
- <target name="init">
- <mkdir dir="${build.classes}" />
- </target>
-
- <target depends="init,ivy-retrieve-build" name="build">
- <javac destdir="${build.classes}">
- <src path="${build.src}" />
- <classpath refid="wordcount.build.classpath" />
- </javac>
- </target>
-
- <target name="jar" depends="build">
- <mkdir dir="${build.classes}/META-INF" />
- <jar jarfile="${build.dir}/${final.name}.jar">
- <fileset dir="${build.classes}" />
- <fileset dir="${cassandra.classes}/main" />
- <fileset dir="${cassandra.classes}/thrift" />
- <fileset dir="${cassandra.dir}">
- <include name="lib/**/*.jar" />
- </fileset>
- <zipfileset dir="${cassandra.dir}/build/lib/jars/" prefix="lib">
- <include name="**/*.jar" />
- </zipfileset>
- <fileset file="${basedir}/cassandra.yaml" />
- </jar>
- </target>
-
- <target name="clean">
- <delete dir="${build.dir}" />
- </target>
-
- <!--
- Ivy Specific targets
- to fetch Ivy and this project's dependencies
- -->
- <target name="ivy-download" unless="ivy.jar.exists">
- <echo>Downloading Ivy...</echo>
- <mkdir dir="${build.dir}" />
- <get src="${ivy.url}/${ivy.version}/ivy-${ivy.version}.jar"
- dest="${build.dir}/ivy-${ivy.version}.jar" usetimestamp="true" />
- </target>
-
- <target name="ivy-init" depends="ivy-download" unless="ivy.initialized">
- <mkdir dir="${ivy.lib.dir}"/>
- <taskdef resource="org/apache/ivy/ant/antlib.xml"
- uri="antlib:org.apache.ivy.ant"
- classpathref="autoivy.classpath"/>
- <property name="ivy.initialized" value="true"/>
- </target>
-
- <target name="ivy-retrieve-build" depends="ivy-init">
- <ivy:retrieve type="jar,source" sync="true"
- pattern="${ivy.lib.dir}/[type]s/[artifact]-[revision].[ext]" />
- </target>
-</project>
diff --git a/examples/hadoop_cql3_word_count/conf/logback.xml b/examples/hadoop_cql3_word_count/conf/logback.xml
deleted file mode 100644
index 443bd1c..0000000
--- a/examples/hadoop_cql3_word_count/conf/logback.xml
+++ /dev/null
@@ -1,42 +0,0 @@
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-
-<configuration scan="true">
-
- <jmxConfigurator />
-
- <appender name="FILE" class="ch.qos.logback.core.FileAppender">
- <file>wc.out</file>
- <encoder>
- <pattern>%-5level [%thread] %date{ISO8601} %F:%L - %msg%n</pattern>
- </encoder>
- </appender>
-
- <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
- <encoder>
- <pattern>%-5level %date{HH:mm:ss,SSS} %msg%n</pattern>
- </encoder>
- </appender>
-
- <root level="INFO">
- <appender-ref ref="FILE" />
- <appender-ref ref="STDOUT" />
- </root>
-
-</configuration>
diff --git a/examples/hadoop_cql3_word_count/ivy.xml b/examples/hadoop_cql3_word_count/ivy.xml
deleted file mode 100644
index 2016eb8..0000000
--- a/examples/hadoop_cql3_word_count/ivy.xml
+++ /dev/null
@@ -1,24 +0,0 @@
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~ with the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-<ivy-module version="2.0">
- <info organisation="apache-cassandra" module="word-count"/>
- <dependencies>
- <dependency org="org.apache.hadoop" name="hadoop-core" rev="1.0.3"/>
- </dependencies>
-</ivy-module>
diff --git a/examples/hadoop_cql3_word_count/src/WordCount.java b/examples/hadoop_cql3_word_count/src/WordCount.java
deleted file mode 100644
index bc95736..0000000
--- a/examples/hadoop_cql3_word_count/src/WordCount.java
+++ /dev/null
@@ -1,259 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.*;
-import java.util.Map.Entry;
-
-import org.apache.cassandra.hadoop.cql3.CqlConfigHelper;
-import org.apache.cassandra.hadoop.cql3.CqlOutputFormat;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.cassandra.hadoop.cql3.CqlInputFormat;
-import org.apache.cassandra.hadoop.ConfigHelper;
-import org.apache.cassandra.utils.ByteBufferUtil;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.Mapper;
-import org.apache.hadoop.mapreduce.Reducer;
-import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
-import org.apache.hadoop.util.Tool;
-import org.apache.hadoop.util.ToolRunner;
-import com.datastax.driver.core.Row;
-
-/**
- * This counts the occurrences of words in ColumnFamily
- * cql3_wordcount ( id uuid,
- * line text,
- * PRIMARY KEY (id))
- *
- * For each word, we output the total number of occurrences across all body texts.
- *
- * When outputting to Cassandra, we write the word counts to column family
- * output_words ( word text,
- * count_num text,
- * PRIMARY KEY (word))
- * as a {word, count} to columns: word, count_num with a row key of "word sum"
- */
-public class WordCount extends Configured implements Tool
-{
- private static final Logger logger = LoggerFactory.getLogger(WordCount.class);
- static final String INPUT_MAPPER_VAR = "input_mapper";
- static final String KEYSPACE = "cql3_wordcount";
- static final String COLUMN_FAMILY = "inputs";
-
- static final String OUTPUT_REDUCER_VAR = "output_reducer";
- static final String OUTPUT_COLUMN_FAMILY = "output_words";
-
- private static final String OUTPUT_PATH_PREFIX = "/tmp/word_count";
- private static final String PRIMARY_KEY = "row_key";
-
- public static void main(String[] args) throws Exception
- {
- // Let ToolRunner handle generic command-line options
- ToolRunner.run(new Configuration(), new WordCount(), args);
- System.exit(0);
- }
-
- public static class TokenizerMapper extends Mapper<Map<String, ByteBuffer>, Map<String, ByteBuffer>, Text, IntWritable>
- {
- private final static IntWritable one = new IntWritable(1);
- private Text word = new Text();
- private ByteBuffer sourceColumn;
-
- protected void setup(org.apache.hadoop.mapreduce.Mapper.Context context)
- throws IOException, InterruptedException
- {
- }
-
- public void map(Map<String, ByteBuffer> keys, Map<String, ByteBuffer> columns, Context context) throws IOException, InterruptedException
- {
- for (Entry<String, ByteBuffer> column : columns.entrySet())
- {
- if (!"line".equalsIgnoreCase(column.getKey()))
- continue;
-
- String value = ByteBufferUtil.string(column.getValue());
-
- StringTokenizer itr = new StringTokenizer(value);
- while (itr.hasMoreTokens())
- {
- word.set(itr.nextToken());
- context.write(word, one);
- }
- }
- }
- }
-
- public static class NativeTokenizerMapper extends Mapper<Long, Row, Text, IntWritable>
- {
- private final static IntWritable one = new IntWritable(1);
- private Text word = new Text();
- private ByteBuffer sourceColumn;
-
- protected void setup(org.apache.hadoop.mapreduce.Mapper.Context context)
- throws IOException, InterruptedException
- {
- }
-
- public void map(Long key, Row row, Context context) throws IOException, InterruptedException
- {
- String value = row.getString("line");
- logger.debug("read {}:{}={} from {}", key, "line", value, context.getInputSplit());
- StringTokenizer itr = new StringTokenizer(value);
- while (itr.hasMoreTokens())
- {
- word.set(itr.nextToken());
- context.write(word, one);
- }
- }
- }
-
- public static class ReducerToFilesystem extends Reducer<Text, IntWritable, Text, IntWritable>
- {
- public void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException
- {
- int sum = 0;
- for (IntWritable val : values)
- sum += val.get();
- context.write(key, new IntWritable(sum));
- }
- }
-
- public static class ReducerToCassandra extends Reducer<Text, IntWritable, Map<String, ByteBuffer>, List<ByteBuffer>>
- {
- private Map<String, ByteBuffer> keys;
- private ByteBuffer key;
- protected void setup(org.apache.hadoop.mapreduce.Reducer.Context context)
- throws IOException, InterruptedException
- {
- keys = new LinkedHashMap<String, ByteBuffer>();
- }
-
- public void reduce(Text word, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException
- {
- int sum = 0;
- for (IntWritable val : values)
- sum += val.get();
- keys.put("word", ByteBufferUtil.bytes(word.toString()));
- context.write(keys, getBindVariables(word, sum));
- }
-
- private List<ByteBuffer> getBindVariables(Text word, int sum)
- {
- List<ByteBuffer> variables = new ArrayList<ByteBuffer>();
- variables.add(ByteBufferUtil.bytes(String.valueOf(sum)));
- return variables;
- }
- }
-
- public int run(String[] args) throws Exception
- {
- String outputReducerType = "filesystem";
- String inputMapperType = "native";
- String outputReducer = null;
- String inputMapper = null;
-
- if (args != null)
- {
- if(args[0].startsWith(OUTPUT_REDUCER_VAR))
- outputReducer = args[0];
- if(args[0].startsWith(INPUT_MAPPER_VAR))
- inputMapper = args[0];
-
- if (args.length == 2)
- {
- if(args[1].startsWith(OUTPUT_REDUCER_VAR))
- outputReducer = args[1];
- if(args[1].startsWith(INPUT_MAPPER_VAR))
- inputMapper = args[1];
- }
- }
-
- if (outputReducer != null)
- {
- String[] s = outputReducer.split("=");
- if (s != null && s.length == 2)
- outputReducerType = s[1];
- }
- logger.info("output reducer type: " + outputReducerType);
- if (inputMapper != null)
- {
- String[] s = inputMapper.split("=");
- if (s != null && s.length == 2)
- inputMapperType = s[1];
- }
- Job job = new Job(getConf(), "wordcount");
- job.setJarByClass(WordCount.class);
-
- if (outputReducerType.equalsIgnoreCase("filesystem"))
- {
- job.setCombinerClass(ReducerToFilesystem.class);
- job.setReducerClass(ReducerToFilesystem.class);
- job.setOutputKeyClass(Text.class);
- job.setOutputValueClass(IntWritable.class);
- FileOutputFormat.setOutputPath(job, new Path(OUTPUT_PATH_PREFIX));
- }
- else
- {
- job.setReducerClass(ReducerToCassandra.class);
-
- job.setMapOutputKeyClass(Text.class);
- job.setMapOutputValueClass(IntWritable.class);
- job.setOutputKeyClass(Map.class);
- job.setOutputValueClass(List.class);
-
- job.setOutputFormatClass(CqlOutputFormat.class);
-
- ConfigHelper.setOutputColumnFamily(job.getConfiguration(), KEYSPACE, OUTPUT_COLUMN_FAMILY);
- job.getConfiguration().set(PRIMARY_KEY, "word,sum");
- String query = "UPDATE " + KEYSPACE + "." + OUTPUT_COLUMN_FAMILY +
- " SET count_num = ? ";
- CqlConfigHelper.setOutputCql(job.getConfiguration(), query);
- ConfigHelper.setOutputInitialAddress(job.getConfiguration(), "localhost");
- ConfigHelper.setOutputPartitioner(job.getConfiguration(), "Murmur3Partitioner");
- }
-
- if (inputMapperType.equalsIgnoreCase("native"))
- {
- job.setMapperClass(NativeTokenizerMapper.class);
- job.setInputFormatClass(CqlInputFormat.class);
- CqlConfigHelper.setInputCql(job.getConfiguration(), "select * from " + COLUMN_FAMILY + " where token(id) > ? and token(id) <= ? allow filtering");
- }
- else
- {
- job.setMapperClass(TokenizerMapper.class);
- job.setInputFormatClass(CqlInputFormat.class);
- ConfigHelper.setInputRpcPort(job.getConfiguration(), "9160");
- }
-
- ConfigHelper.setInputInitialAddress(job.getConfiguration(), "localhost");
- ConfigHelper.setInputColumnFamily(job.getConfiguration(), KEYSPACE, COLUMN_FAMILY);
- ConfigHelper.setInputPartitioner(job.getConfiguration(), "Murmur3Partitioner");
-
- CqlConfigHelper.setInputCQLPageRowSize(job.getConfiguration(), "3");
- job.waitForCompletion(true);
- return 0;
- }
-}
diff --git a/examples/hadoop_cql3_word_count/src/WordCountCounters.java b/examples/hadoop_cql3_word_count/src/WordCountCounters.java
deleted file mode 100644
index 150d18d..0000000
--- a/examples/hadoop_cql3_word_count/src/WordCountCounters.java
+++ /dev/null
@@ -1,168 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.nio.charset.CharacterCodingException;
-import java.util.*;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.cassandra.hadoop.cql3.CqlConfigHelper;
-import org.apache.cassandra.hadoop.cql3.CqlInputFormat;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.Mapper;
-import org.apache.hadoop.mapreduce.Reducer;
-import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
-import org.apache.hadoop.util.Tool;
-import org.apache.hadoop.util.ToolRunner;
-import com.datastax.driver.core.Row;
-import org.apache.cassandra.hadoop.ConfigHelper;
-import org.apache.cassandra.utils.ByteBufferUtil;
-
-
-/**
- * This sums the word count stored in the input_words_count ColumnFamily for the key "sum".
- *
- * Output is written to a text file.
- */
-public class WordCountCounters extends Configured implements Tool
-{
- private static final Logger logger = LoggerFactory.getLogger(WordCountCounters.class);
-
- static final String INPUT_MAPPER_VAR = "input_mapper";
- static final String COUNTER_COLUMN_FAMILY = "input_words_count";
- private static final String OUTPUT_PATH_PREFIX = "/tmp/word_count_counters";
-
- public static void main(String[] args) throws Exception
- {
- // Let ToolRunner handle generic command-line options
- ToolRunner.run(new Configuration(), new WordCountCounters(), args);
- System.exit(0);
- }
-
- public static class SumNativeMapper extends Mapper<Long, Row, Text, LongWritable>
- {
- long sum = -1;
- public void map(Long key, Row row, Context context) throws IOException, InterruptedException
- {
- if (sum < 0)
- sum = 0;
-
- logger.debug("read " + key + ":count_num from " + context.getInputSplit());
- sum += Long.valueOf(row.getString("count_num"));
- }
-
- protected void cleanup(Context context) throws IOException, InterruptedException {
- if (sum > 0)
- context.write(new Text("total_count"), new LongWritable(sum));
- }
- }
-
- public static class SumMapper extends Mapper<Map<String, ByteBuffer>, Map<String, ByteBuffer>, Text, LongWritable>
- {
- long sum = -1;
-
- public void map(Map<String, ByteBuffer> key, Map<String, ByteBuffer> columns, Context context) throws IOException, InterruptedException
- {
- if (sum < 0)
- sum = 0;
-
- logger.debug("read " + toString(key) + ":count_num from " + context.getInputSplit());
- sum += Long.valueOf(ByteBufferUtil.string(columns.get("count_num")));
- }
-
- protected void cleanup(Context context) throws IOException, InterruptedException {
- if (sum > 0)
- context.write(new Text("total_count"), new LongWritable(sum));
- }
-
- private String toString(Map<String, ByteBuffer> keys)
- {
- String result = "";
- try
- {
- for (ByteBuffer key : keys.values())
- result = result + ByteBufferUtil.string(key) + ":";
- }
- catch (CharacterCodingException e)
- {
- logger.error("Failed to print keys", e);
- }
- return result;
- }
- }
-
- public static class ReducerToFilesystem extends Reducer<Text, LongWritable, Text, LongWritable>
- {
- long sum = 0;
-
- public void reduce(Text key, Iterable<LongWritable> values, Context context) throws IOException, InterruptedException
- {
- for (LongWritable val : values)
- sum += val.get();
- context.write(key, new LongWritable(sum));
- }
- }
-
- public int run(String[] args) throws Exception
- {
- String inputMapperType = "native";
- if (args != null && args[0].startsWith(INPUT_MAPPER_VAR))
- {
- String[] arg0 = args[0].split("=");
- if (arg0 != null && arg0.length == 2)
- inputMapperType = arg0[1];
- }
- Job job = new Job(getConf(), "wordcountcounters");
-
- job.setCombinerClass(ReducerToFilesystem.class);
- job.setReducerClass(ReducerToFilesystem.class);
- job.setJarByClass(WordCountCounters.class);
-
- ConfigHelper.setInputInitialAddress(job.getConfiguration(), "localhost");
- ConfigHelper.setInputPartitioner(job.getConfiguration(), "Murmur3Partitioner");
- ConfigHelper.setInputColumnFamily(job.getConfiguration(), WordCount.KEYSPACE, WordCount.OUTPUT_COLUMN_FAMILY);
-
- CqlConfigHelper.setInputCQLPageRowSize(job.getConfiguration(), "3");
- if ("native".equals(inputMapperType))
- {
- job.setMapperClass(SumNativeMapper.class);
- job.setInputFormatClass(CqlInputFormat.class);
- CqlConfigHelper.setInputCql(job.getConfiguration(), "select * from " + WordCount.OUTPUT_COLUMN_FAMILY + " where token(word) > ? and token(word) <= ? allow filtering");
- }
- else
- {
- job.setMapperClass(SumMapper.class);
- job.setInputFormatClass(CqlInputFormat.class);
- ConfigHelper.setInputRpcPort(job.getConfiguration(), "9160");
- }
-
- job.setOutputKeyClass(Text.class);
- job.setOutputValueClass(LongWritable.class);
- FileOutputFormat.setOutputPath(job, new Path(OUTPUT_PATH_PREFIX));
- job.waitForCompletion(true);
- return 0;
- }
-}
diff --git a/examples/hadoop_cql3_word_count/src/WordCountSetup.java b/examples/hadoop_cql3_word_count/src/WordCountSetup.java
deleted file mode 100644
index e514d63..0000000
--- a/examples/hadoop_cql3_word_count/src/WordCountSetup.java
+++ /dev/null
@@ -1,181 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import java.nio.ByteBuffer;
-import java.util.*;
-
-import org.apache.cassandra.thrift.*;
-import org.apache.cassandra.utils.ByteBufferUtil;
-import org.apache.thrift.TException;
-import org.apache.thrift.protocol.TBinaryProtocol;
-import org.apache.thrift.protocol.TProtocol;
-import org.apache.thrift.transport.TFramedTransport;
-import org.apache.thrift.transport.TSocket;
-import org.apache.thrift.transport.TTransport;
-import org.apache.thrift.transport.TTransportException;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class WordCountSetup
-{
- private static final Logger logger = LoggerFactory.getLogger(WordCountSetup.class);
-
- public static final int TEST_COUNT = 6;
-
- public static void main(String[] args) throws Exception
- {
- Cassandra.Iface client = createConnection();
-
- setupKeyspace(client);
- client.set_keyspace(WordCount.KEYSPACE);
- setupTable(client);
- insertData(client);
-
- System.exit(0);
- }
-
- private static void setupKeyspace(Cassandra.Iface client)
- throws InvalidRequestException,
- UnavailableException,
- TimedOutException,
- SchemaDisagreementException,
- TException
- {
- KsDef ks;
- try
- {
- ks = client.describe_keyspace(WordCount.KEYSPACE);
- }
- catch(NotFoundException e)
- {
- logger.info("set up keyspace " + WordCount.KEYSPACE);
- String query = "CREATE KEYSPACE " + WordCount.KEYSPACE +
- " WITH replication = {'class': 'SimpleStrategy', 'replication_factor' : 1}";
-
- client.execute_cql3_query(ByteBufferUtil.bytes(query), Compression.NONE, ConsistencyLevel.ONE);
-
- String verifyQuery = "select count(*) from system.peers";
- CqlResult result = client.execute_cql3_query(ByteBufferUtil.bytes(verifyQuery), Compression.NONE, ConsistencyLevel.ONE);
-
- long magnitude = ByteBufferUtil.toLong(result.rows.get(0).columns.get(0).value);
- try
- {
- Thread.sleep(1000 * magnitude);
- }
- catch (InterruptedException ie)
- {
- throw new RuntimeException(ie);
- }
- }
- }
-
- private static void setupTable(Cassandra.Iface client)
- throws InvalidRequestException,
- UnavailableException,
- TimedOutException,
- SchemaDisagreementException,
- TException
- {
- String query = "CREATE TABLE " + WordCount.KEYSPACE + "." + WordCount.COLUMN_FAMILY +
- " ( id uuid," +
- " line text, " +
- " PRIMARY KEY (id) ) ";
-
- try
- {
- logger.info("set up table " + WordCount.COLUMN_FAMILY);
- client.execute_cql3_query(ByteBufferUtil.bytes(query), Compression.NONE, ConsistencyLevel.ONE);
- }
- catch (InvalidRequestException e)
- {
- logger.error("failed to create table " + WordCount.KEYSPACE + "." + WordCount.COLUMN_FAMILY, e);
- }
-
- query = "CREATE TABLE " + WordCount.KEYSPACE + "." + WordCount.OUTPUT_COLUMN_FAMILY +
- " ( word text," +
- " count_num text," +
- " PRIMARY KEY (word) ) ";
-
- try
- {
- logger.info("set up table " + WordCount.OUTPUT_COLUMN_FAMILY);
- client.execute_cql3_query(ByteBufferUtil.bytes(query), Compression.NONE, ConsistencyLevel.ONE);
- }
- catch (InvalidRequestException e)
- {
- logger.error("failed to create table " + WordCount.KEYSPACE + "." + WordCount.OUTPUT_COLUMN_FAMILY, e);
- }
- }
-
- private static Cassandra.Iface createConnection() throws TTransportException
- {
- if (System.getProperty("cassandra.host") == null || System.getProperty("cassandra.port") == null)
- {
- logger.warn("cassandra.host or cassandra.port is not defined, using default");
- }
- return createConnection(System.getProperty("cassandra.host", "localhost"),
- Integer.valueOf(System.getProperty("cassandra.port", "9160")));
- }
-
- private static Cassandra.Client createConnection(String host, Integer port) throws TTransportException
- {
- TSocket socket = new TSocket(host, port);
- TTransport trans = new TFramedTransport(socket);
- trans.open();
- TProtocol protocol = new TBinaryProtocol(trans);
-
- return new Cassandra.Client(protocol);
- }
-
- private static void insertData(Cassandra.Iface client)
- throws InvalidRequestException,
- UnavailableException,
- TimedOutException,
- SchemaDisagreementException,
- TException
- {
- String query = "INSERT INTO " + WordCount.COLUMN_FAMILY +
- "(id, line) " +
- " values (?, ?) ";
- CqlPreparedResult result = client.prepare_cql3_query(ByteBufferUtil.bytes(query), Compression.NONE);
-
- String [] body = bodyData();
- for (int i = 0; i < 5; i++)
- {
- for (int j = 1; j <= 200; j++)
- {
- List<ByteBuffer> values = new ArrayList<ByteBuffer>();
- values.add(ByteBufferUtil.bytes(UUID.randomUUID()));
- values.add(ByteBufferUtil.bytes(body[i]));
- client.execute_prepared_cql3_query(result.itemId, values, ConsistencyLevel.ONE);
- }
- }
- }
-
- private static String[] bodyData()
- { // Public domain context, source http://en.wikisource.org/wiki/If%E2%80%94
- return new String[]{
- "If you can keep your head when all about you",
- "Are losing theirs and blaming it on you",
- "If you can trust yourself when all men doubt you,",
- "But make allowance for their doubting too:",
- "If you can wait and not be tired by waiting,"
- };
- }
-}
diff --git a/examples/triggers/src/org/apache/cassandra/triggers/AuditTrigger.java b/examples/triggers/src/org/apache/cassandra/triggers/AuditTrigger.java
index 1efbf13..b0172b0 100644
--- a/examples/triggers/src/org/apache/cassandra/triggers/AuditTrigger.java
+++ b/examples/triggers/src/org/apache/cassandra/triggers/AuditTrigger.java
@@ -22,8 +22,8 @@
import java.util.Collections;
import java.util.Properties;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.Schema;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.db.Mutation;
import org.apache.cassandra.db.partitions.Partition;
import org.apache.cassandra.db.partitions.PartitionUpdate;
@@ -40,13 +40,13 @@
String auditKeyspace = properties.getProperty("keyspace");
String auditTable = properties.getProperty("table");
- CFMetaData metadata = Schema.instance.getCFMetaData(auditKeyspace, auditTable);
+ TableMetadata metadata = Schema.instance.getTableMetadata(auditKeyspace, auditTable);
PartitionUpdate.SimpleBuilder audit = PartitionUpdate.simpleBuilder(metadata, UUIDGen.getTimeUUID());
audit.row()
- .add("keyspace_name", update.metadata().ksName)
- .add("table_name", update.metadata().cfName)
- .add("primary_key", update.metadata().getKeyValidator().getString(update.partitionKey().getKey()));
+ .add("keyspace_name", update.metadata().keyspace)
+ .add("table_name", update.metadata().name)
+ .add("primary_key", update.metadata().partitionKeyType.getString(update.partitionKey().getKey()));
return Collections.singletonList(audit.buildAsMutation());
}
diff --git a/ide/idea-iml-file.xml b/ide/idea-iml-file.xml
index 72b16f2..86827c3 100644
--- a/ide/idea-iml-file.xml
+++ b/ide/idea-iml-file.xml
@@ -24,13 +24,15 @@
<exclude-output />
<content url="file://$MODULE_DIR$">
<sourceFolder url="file://$MODULE_DIR$/src/java" isTestSource="false" />
- <sourceFolder url="file://$MODULE_DIR$/src/gen-java" isTestSource="false" />
+ <sourceFolder url="file://$MODULE_DIR$/src/gen-java" isTestSource="false" generated="true" />
<sourceFolder url="file://$MODULE_DIR$/src/resources" type="java-resource" />
- <sourceFolder url="file://$MODULE_DIR$/interface/thrift/gen-java" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/tools/stress/src" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/tools/stress/test/unit" isTestSource="true" />
+ <sourceFolder url="file://$MODULE_DIR$/tools/fqltool/src" isTestSource="false" />
+ <sourceFolder url="file://$MODULE_DIR$/tools/fqltool/test/unit" isTestSource="true" />
<sourceFolder url="file://$MODULE_DIR$/test/unit" isTestSource="true" />
<sourceFolder url="file://$MODULE_DIR$/test/long" isTestSource="true" />
+ <sourceFolder url="file://$MODULE_DIR$/test/memory" isTestSource="true" />
<sourceFolder url="file://$MODULE_DIR$/test/microbench" isTestSource="true" />
<sourceFolder url="file://$MODULE_DIR$/test/burn" isTestSource="true" />
<sourceFolder url="file://$MODULE_DIR$/test/distributed" isTestSource="true" />
diff --git a/ide/idea/misc.xml b/ide/idea/misc.xml
index 7aa7466..ac1f706 100644
--- a/ide/idea/misc.xml
+++ b/ide/idea/misc.xml
@@ -1,4 +1,4 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
-<component name="ProjectRootManager" version="2" languageLevel="JDK_1_8" default="false" assert-keyword="true" jdk-15="true" project-jdk-name="1.8" project-jdk-type="JavaSDK" />
-</project>
\ No newline at end of file
+ <component name="ProjectRootManager" version="2" languageLevel="JDK_1_8" default="false" project-jdk-name="1.8" project-jdk-type="JavaSDK" />
+</project>
diff --git a/ide/idea/workspace.xml b/ide/idea/workspace.xml
index 680ad1d..d1d8f87 100644
--- a/ide/idea/workspace.xml
+++ b/ide/idea/workspace.xml
@@ -143,7 +143,7 @@
<configuration default="true" type="Application" factoryName="Application">
<extension name="coverage" enabled="false" merge="false" sample_coverage="true" runner="idea" />
<option name="MAIN_CLASS_NAME" value="" />
- <option name="VM_PARAMETERS" value="-Dcassandra.config=file://$PROJECT_DIR$/conf/cassandra.yaml -Dcassandra.storagedir=$PROJECT_DIR$/data -Dlogback.configurationFile=file://$PROJECT_DIR$/conf/logback.xml -Dcassandra.logdir=$PROJECT_DIR$/data/logs -Djava.library.path=$PROJECT_DIR$/lib/sigar-bin -ea" />
+ <option name="VM_PARAMETERS" value="-Dcassandra.config=file://$PROJECT_DIR$/conf/cassandra.yaml -Dcassandra.storagedir=$PROJECT_DIR$/data -Dlogback.configurationFile=file://$PROJECT_DIR$/conf/logback.xml -Dcassandra.logdir=$PROJECT_DIR$/data/logs -Djava.library.path=$PROJECT_DIR$/lib/sigar-bin -DQT_SHRINKS=0 -ea" />
<option name="PROGRAM_PARAMETERS" value="" />
<option name="WORKING_DIRECTORY" value="" />
<option name="ALTERNATIVE_JRE_PATH_ENABLED" value="false" />
@@ -167,7 +167,7 @@
<option name="MAIN_CLASS_NAME" value="" />
<option name="METHOD_NAME" value="" />
<option name="TEST_OBJECT" value="class" />
- <option name="VM_PARAMETERS" value="-Dcassandra.config=file://$PROJECT_DIR$/test/conf/cassandra.yaml -Dlogback.configurationFile=file://$PROJECT_DIR$/test/conf/logback-test.xml -Dcassandra.logdir=$PROJECT_DIR$/build/test/logs -Djava.library.path=$PROJECT_DIR$/lib/sigar-bin -ea -XX:MaxMetaspaceSize=384M -XX:SoftRefLRUPolicyMSPerMB=0 -Dcassandra.strict.runtime.checks=true" />
+ <option name="VM_PARAMETERS" value="-Dcassandra.config=file://$PROJECT_DIR$/test/conf/cassandra.yaml -Dlogback.configurationFile=file://$PROJECT_DIR$/test/conf/logback-test.xml -Dcassandra.logdir=$PROJECT_DIR$/build/test/logs -Djava.library.path=$PROJECT_DIR$/lib/sigar-bin -Dlegacy-sstable-root=$PROJECT_DIR$/test/data/legacy-sstables -Dinvalid-legacy-sstable-root=$PROJECT_DIR$/test/data/invalid-legacy-sstables -Dcassandra.ring_delay_ms=1000 -Dcassandra.skip_sync=true -ea -XX:MaxMetaspaceSize=384M -XX:SoftRefLRUPolicyMSPerMB=0 -Dcassandra.strict.runtime.checks=true" />
<option name="PARAMETERS" value="" />
<option name="WORKING_DIRECTORY" value="" />
<option name="ENV_VARIABLES" />
@@ -289,17 +289,15 @@
<filter targetName="maven-ant-tasks-retrieve-build" isVisible="false" />
<filter targetName="maven-ant-tasks-retrieve-test" isVisible="false" />
<filter targetName="maven-ant-tasks-retrieve-pig-test" isVisible="false" />
- <filter targetName="check-gen-thrift-java" isVisible="false" />
- <filter targetName="gen-thrift-java" isVisible="true" />
<filter targetName="_write-java-license-headers" isVisible="false" />
<filter targetName="write-java-license-headers" isVisible="true" />
- <filter targetName="gen-thrift-py" isVisible="true" />
<filter targetName="createVersionPropFile" isVisible="false" />
<filter targetName="test-run" isVisible="true" />
<filter targetName="build" isVisible="true" />
<filter targetName="codecoverage" isVisible="true" />
<filter targetName="build-project" isVisible="false" />
<filter targetName="stress-build" isVisible="true" />
+ <filter targetName="fqltool-build" isVisible="true" />
<filter targetName="_write-poms" isVisible="false" />
<filter targetName="write-poms" isVisible="false" />
<filter targetName="jar" isVisible="true" />
diff --git a/ide/nbproject/ide-actions.xml b/ide/nbproject/ide-actions.xml
new file mode 100644
index 0000000..7a02dcc
--- /dev/null
+++ b/ide/nbproject/ide-actions.xml
@@ -0,0 +1,60 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project basedir=".." name="apache-cassandra">
+
+ <condition property="java.version.8">
+ <equals arg1="${ant.java.version}" arg2="1.8"/>
+ </condition>
+
+ <target name="_check_java8_home_defined" unless="java.version.8">
+ <property environment="env"/>
+ <!-- Copied from build.xml and used here for a fail-fast error check -->
+ <fail message="JAVA8_HOME env variable must be set when building with java >= 11">
+ <condition><not><isset property="env.JAVA8_HOME"/></not></condition>
+ </fail>
+ </target>
+
+ <target name="build">
+ <ant antfile="../build.xml" inheritall="false" target="build"/>
+ <ant antfile="../build.xml" inheritall="false" target="build-test"/>
+ <ant antfile="../build.xml" inheritall="false" target="fqltool-build-test"/>
+ <ant antfile="../build.xml" inheritall="false" target="stress-build-test"/>
+ </target>
+ <target name="clean">
+ <ant antfile="../build.xml" inheritall="false" target="clean"/>
+ </target>
+ <target name="run" depends="_check_java8_home_defined">
+ <ant antfile="../build.xml" inheritall="false" target="_artifacts-init">
+ <property name="no-javadoc" value="true"/>
+ <property name="ant.gen-doc.skip" value="true"/>
+ </ant>
+ <property environment="env"/>
+ <exec executable="sh">
+ <arg value="../build/dist/bin/cassandra"/>
+ <arg value="-f"/>
+ </exec>
+ </target>
+ <target name="debug" depends="_check_java8_home_defined">
+ <ant antfile="../build.xml" inheritall="false" target="_artifacts-init">
+ <property name="no-javadoc" value="true"/>
+ <property name="ant.gen-doc.skip" value="true"/>
+ </ant>
+ <nbjpdastart addressproperty="nbjpdastart.address" name="Cassandra" transport="dt_socket"/>
+ <exec executable="sh">
+ <env key="JVM_OPTS" value="-Xdebug -Xnoagent -Xrunjdwp:transport=dt_socket,server=n,suspend=y,address=${nbjpdastart.address}"/>
+ <arg value="../build/dist/bin/cassandra"/>
+ <arg value="-f"/>
+ </exec>
+ </target>
+ <target name="profile" depends="_check_java8_home_defined">
+ <ant antfile="../build.xml" inheritall="false" target="_artifacts-init">
+ <property name="no-javadoc" value="true"/>
+ <property name="ant.gen-doc.skip" value="true"/>
+ </ant>
+ <startprofiler freeform="true"/>
+ <exec executable="sh">
+ <env key="JVM_OPTS" value="-agentpath:${netbeans.home}/../profiler/lib/deployed/jdk16/mac/libprofilerinterface.jnilib=${netbeans.home}/../profiler/lib,5140"/>
+ <arg value="../build/dist/bin/cassandra"/>
+ <arg value="-f"/>
+ </exec>
+ </target>
+</project>
diff --git a/ide/nbproject/jdk.xml b/ide/nbproject/jdk.xml
new file mode 100644
index 0000000..237e529
--- /dev/null
+++ b/ide/nbproject/jdk.xml
@@ -0,0 +1,157 @@
+<?xml version="1.0" encoding="UTF-8"?><project name="jdk" basedir=".">
+
+
+ <description>
+ Permits selection of a JDK to use when building and running project.
+ See: http://www.netbeans.org/issues/show_bug.cgi?id=64160
+ </description>
+
+ <target name="-jdk-pre-preinit">
+ <condition property="nbjdk.active-or-nbjdk.home">
+ <or>
+ <and>
+ <isset property="nbjdk.active"/>
+ <not>
+ <equals arg1="${nbjdk.active}" arg2="default_platform"/>
+ </not>
+ </and>
+ <and>
+ <isset property="nbjdk.home"/>
+ <not>
+ <isset property="nbjdk.home.defaulted"/>
+ </not>
+ </and>
+ </or>
+ </condition>
+ </target>
+
+ <target xmlns:common="http://java.netbeans.org/freeform/jdk.xml" name="-jdk-preinit" depends="-jdk-pre-preinit" if="nbjdk.active-or-nbjdk.home">
+ <macrodef name="property" uri="http://java.netbeans.org/freeform/jdk.xml">
+ <attribute name="name"/>
+ <attribute name="value"/>
+ <sequential>
+ <property name="@{name}" value="${@{value}}"/>
+ </sequential>
+ </macrodef>
+ <common:property name="nbjdk.home" value="platforms.${nbjdk.active}.home"/>
+ <common:property name="nbjdk.javac.tmp" value="platforms.${nbjdk.active}.javac"/>
+ <condition property=".exe" value=".exe">
+ <os family="windows"/>
+ </condition>
+ <property name=".exe" value=""/>
+ <condition property="nbjdk.javac" value="${nbjdk.home}/bin/javac${.exe}">
+ <equals arg1="${nbjdk.javac.tmp}" arg2="$${platforms.${nbjdk.active}.javac}"/>
+ </condition>
+ <property name="nbjdk.javac" value="${nbjdk.javac.tmp}"/>
+ <common:property name="nbjdk.java.tmp" value="platforms.${nbjdk.active}.java"/>
+ <condition property="nbjdk.java" value="${nbjdk.home}/bin/java${.exe}">
+ <equals arg1="${nbjdk.java.tmp}" arg2="$${platforms.${nbjdk.active}.java}"/>
+ </condition>
+ <property name="nbjdk.java" value="${nbjdk.java.tmp}"/>
+ <common:property name="nbjdk.javadoc.tmp" value="platforms.${nbjdk.active}.javadoc"/>
+ <condition property="nbjdk.javadoc" value="${nbjdk.home}/bin/javadoc${.exe}">
+ <equals arg1="${nbjdk.javadoc.tmp}" arg2="$${platforms.${nbjdk.active}.javadoc}"/>
+ </condition>
+ <property name="nbjdk.javadoc" value="${nbjdk.javadoc.tmp}"/>
+ <common:property name="nbjdk.bootclasspath.tmp" value="platforms.${nbjdk.active}.bootclasspath"/>
+ <condition property="nbjdk.bootclasspath" value="${nbjdk.home}/jre/lib/rt.jar">
+ <equals arg1="${nbjdk.bootclasspath.tmp}" arg2="$${platforms.${nbjdk.active}.bootclasspath}"/>
+ </condition>
+ <property name="nbjdk.bootclasspath" value="${nbjdk.bootclasspath.tmp}"/>
+ <condition property="nbjdk.valid">
+ <and>
+ <available file="${nbjdk.home}" type="dir"/>
+ <available file="${nbjdk.javac}" type="file"/>
+ <available file="${nbjdk.java}" type="file"/>
+ <available file="${nbjdk.javadoc}" type="file"/>
+
+ </and>
+ </condition>
+ <echo level="verbose">nbjdk.active=${nbjdk.active} nbjdk.home=${nbjdk.home} nbjdk.java=${nbjdk.java} nbjdk.javac=${nbjdk.javac} nbjdk.javadoc=${nbjdk.javadoc} nbjdk.bootclasspath=${nbjdk.bootclasspath} nbjdk.valid=${nbjdk.valid} have-jdk-1.4=${have-jdk-1.4} have-jdk-1.5=${have-jdk-1.5}</echo>
+ </target>
+
+ <target name="-jdk-warn" depends="-jdk-preinit" if="nbjdk.active-or-nbjdk.home" unless="nbjdk.valid">
+ <property name="jdkhome.presumed" location="${java.home}/.."/>
+ <echo level="warning">Warning: nbjdk.active=${nbjdk.active} or nbjdk.home=${nbjdk.home} is an invalid Java platform; ignoring and using ${jdkhome.presumed}</echo>
+ </target>
+
+ <target name="-jdk-presetdef-basic" depends="-jdk-preinit" if="nbjdk.valid" unless="nbjdk.presetdef.basic.done">
+
+
+ <macrodef name="javac-presetdef">
+ <attribute name="javacval"/>
+ <sequential>
+ <presetdef name="javac">
+ <javac fork="yes" executable="@{javacval}"/>
+ </presetdef>
+ </sequential>
+ </macrodef>
+ <javac-presetdef javacval="${nbjdk.javac}"/>
+ <macrodef name="java-presetdef">
+ <attribute name="javaval"/>
+ <sequential>
+ <presetdef name="java">
+ <java fork="yes" jvm="@{javaval}"/>
+ </presetdef>
+ </sequential>
+ </macrodef>
+ <java-presetdef javaval="${nbjdk.java}"/>
+ <macrodef name="javadoc-presetdef">
+ <attribute name="javadocval"/>
+ <sequential>
+ <presetdef name="javadoc">
+ <javadoc executable="@{javadocval}"/>
+ </presetdef>
+ </sequential>
+ </macrodef>
+ <javadoc-presetdef javadocval="${nbjdk.javadoc}"/>
+ <macrodef name="junit-presetdef">
+ <attribute name="javaval"/>
+ <sequential>
+ <presetdef name="junit">
+ <junit fork="yes" jvm="@{javaval}"/>
+ </presetdef>
+ </sequential>
+ </macrodef>
+ <junit-presetdef javaval="${nbjdk.java}"/>
+ <property name="nbjdk.presetdef.basic.done" value="true"/>
+ </target>
+
+ <target name="-jdk-presetdef-nbjpdastart" depends="-jdk-preinit" if="nbjdk.valid" unless="nbjdk.presetdef.nbjpdastart.done">
+ <macrodef name="nbjpdastart-presetdef">
+ <attribute name="bootcpval"/>
+ <sequential>
+ <presetdef name="nbjpdastart">
+ <nbjpdastart>
+ <bootclasspath>
+ <path path="@{bootcpval}"/>
+ </bootclasspath>
+ </nbjpdastart>
+ </presetdef>
+ </sequential>
+ </macrodef>
+ <nbjpdastart-presetdef bootcpval="${nbjdk.bootclasspath}"/>
+ <property name="nbjdk.presetdef.nbjpdastart.done" value="true"/>
+ </target>
+
+ <target name="-jdk-default" unless="nbjdk.active-or-nbjdk.home">
+
+ <property name="java.home.parent" location="${java.home}/.."/>
+ <condition property="nbjdk.home" value="${java.home.parent}">
+ <available file="${java.home.parent}/lib/tools.jar" type="file"/>
+ </condition>
+ <condition property="nbjdk.home" value="${java.home}">
+ <available file="${java.home}/lib/tools.jar" type="file"/>
+ </condition>
+
+ <condition property="nbjdk.home" value="/Library/Java/Home">
+ <available file="/Library/Java/Home" type="dir"/>
+ </condition>
+
+ <property name="nbjdk.home" location="${java.home.parent}"/>
+ <property name="nbjdk.home.defaulted" value="true"/>
+ </target>
+
+ <target name="-jdk-init" depends="-jdk-preinit,-jdk-warn,-jdk-presetdef-basic,-jdk-default"/>
+
+</project>
\ No newline at end of file
diff --git a/ide/nbproject/nbjdk.properties b/ide/nbproject/nbjdk.properties
new file mode 100644
index 0000000..2ac925d
--- /dev/null
+++ b/ide/nbproject/nbjdk.properties
@@ -0,0 +1 @@
+nbjdk.active=JDK_1.8
diff --git a/ide/nbproject/project.properties b/ide/nbproject/project.properties
new file mode 100644
index 0000000..b06ebb7
--- /dev/null
+++ b/ide/nbproject/project.properties
@@ -0,0 +1,23 @@
+auxiliary.show.customizer=false
+auxiliary.show.customizer.message=ide/nbproject/project.xml is to be edited by hand, ref: CASSANDRA-15073
+auxiliary.org-netbeans-modules-editor-indent.CodeStyle.project.expand-tabs=true
+auxiliary.org-netbeans-modules-editor-indent.CodeStyle.project.indent-shift-width=4
+auxiliary.org-netbeans-modules-editor-indent.CodeStyle.project.spaces-per-tab=4
+auxiliary.org-netbeans-modules-editor-indent.CodeStyle.project.tab-size=4
+auxiliary.org-netbeans-modules-editor-indent.CodeStyle.project.text-limit-width=110
+auxiliary.org-netbeans-modules-editor-indent.CodeStyle.project.text-line-wrap=none
+auxiliary.org-netbeans-modules-editor-indent.CodeStyle.usedProfile=project
+auxiliary.org-netbeans-modules-editor-indent.text.x-java.CodeStyle.project.alignMultilineMethodParams=true
+auxiliary.org-netbeans-modules-editor-indent.text.x-java.CodeStyle.project.classDeclBracePlacement=NEW_LINE
+auxiliary.org-netbeans-modules-editor-indent.text.x-java.CodeStyle.project.enable-indent=true
+auxiliary.org-netbeans-modules-editor-indent.text.x-java.CodeStyle.project.importGroupsOrder=java;com.google.common;org.apache.commons;org.junit;org.slf4j;*
+auxiliary.org-netbeans-modules-editor-indent.text.x-java.CodeStyle.project.methodDeclBracePlacement=NEW_LINE
+auxiliary.org-netbeans-modules-editor-indent.text.x-java.CodeStyle.project.otherBracePlacement=NEW_LINE
+auxiliary.org-netbeans-modules-editor-indent.text.x-java.CodeStyle.project.placeCatchOnNewLine=true
+auxiliary.org-netbeans-modules-editor-indent.text.x-java.CodeStyle.project.placeElseOnNewLine=true
+auxiliary.org-netbeans-modules-editor-indent.text.x-java.CodeStyle.project.placeFinallyOnNewLine=true
+auxiliary.org-netbeans-modules-editor-indent.text.x-java.CodeStyle.project.placeWhileOnNewLine=true
+auxiliary.org-netbeans-modules-editor-indent.text.x-java.CodeStyle.project.redundantDoWhileBraces=LEAVE_ALONE
+auxiliary.org-netbeans-modules-editor-indent.text.x-java.CodeStyle.project.redundantForBraces=LEAVE_ALONE
+auxiliary.org-netbeans-modules-editor-indent.text.x-java.CodeStyle.project.redundantIfBraces=LEAVE_ALONE
+auxiliary.org-netbeans-modules-editor-indent.text.x-java.CodeStyle.project.redundantWhileBraces=LEAVE_ALONE
diff --git a/ide/nbproject/project.xml b/ide/nbproject/project.xml
new file mode 100644
index 0000000..6b8a31e
--- /dev/null
+++ b/ide/nbproject/project.xml
@@ -0,0 +1,254 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://www.netbeans.org/ns/project/1">
+ <type>org.netbeans.modules.ant.freeform</type>
+ <configuration>
+ <general-data xmlns="http://www.netbeans.org/ns/freeform-project/2">
+ <name>apache-cassandra</name>
+ <properties>
+ <property name="project.dir">..</property>
+ <!-- the compile classpaths should be distinct per compilation unit… but it is kept simple and the build will catch errors -->
+ <property name="cassandra.classpath.jars">${project.dir}/lib/HdrHistogram-2.1.9.jar:${project.dir}/lib/ST4-4.0.8.jar:${project.dir}/lib/airline-0.8.jar:${project.dir}/lib/antlr-runtime-3.5.2.jar:${project.dir}/lib/asm-7.1.jar:${project.dir}/lib/caffeine-2.3.5.jar:${project.dir}/lib/cassandra-driver-core-3.6.0-shaded.jar:${project.dir}/lib/chronicle-bytes-1.16.3.jar:${project.dir}/lib/chronicle-core-1.16.4.jar:${project.dir}/lib/chronicle-queue-4.16.3.jar:${project.dir}/lib/chronicle-threads-1.16.0.jar:${project.dir}/lib/chronicle-wire-1.16.1.jar:${project.dir}/lib/commons-cli-1.1.jar:${project.dir}/lib/commons-codec-1.9.jar:${project.dir}/lib/commons-lang3-3.1.jar:${project.dir}/lib/commons-math3-3.2.jar:${project.dir}/lib/concurrent-trees-2.4.0.jar:${project.dir}/lib/ecj-4.6.1.jar:${project.dir}/lib/guava-27.0-jre.jar:${project.dir}/lib/high-scale-lib-1.0.6.jar:${project.dir}/lib/hppc-0.5.4.jar:${project.dir}/lib/j2objc-annotations-1.3.jar:${project.dir}/lib/jackson-annotations-2.9.5.jar:${project.dir}/lib/jackson-core-2.9.5.jar:${project.dir}/lib/jackson-databind-2.9.5.jar:${project.dir}/lib/jamm-0.3.2.jar:${project.dir}/lib/javax.inject.jar:${project.dir}/lib/jbcrypt-0.3m.jar:${project.dir}/lib/jcl-over-slf4j-1.7.25.jar:${project.dir}/lib/jctools-core-1.2.1.jar:${project.dir}/lib/jflex-1.6.0.jar:${project.dir}/lib/jna-4.2.2.jar:${project.dir}/lib/json-simple-1.1.jar:${project.dir}/lib/jstackjunit-0.0.1.jar:${project.dir}/lib/log4j-over-slf4j-1.7.25.jar:${project.dir}/lib/logback-classic-1.2.3.jar:${project.dir}/lib/logback-core-1.2.3.jar:${project.dir}/lib/lz4-java-1.4.0.jar:${project.dir}/lib/metrics-core-3.1.5.jar:${project.dir}/lib/metrics-jvm-3.1.5.jar:${project.dir}/lib/metrics-logback-3.1.5.jar:${project.dir}/lib/netty-all-4.1.37.Final.jar:${project.dir}/lib/netty-tcnative-boringssl-static-2.0.25.Final.jar:${project.dir}/lib/ohc-core-0.5.1.jar:${project.dir}/lib/ohc-core-j8-0.5.1.jar:${project.dir}/lib/psjava-0.1.19.jar:${project.dir}/lib/reporter-config-base-3.0.3.jar:${project.dir}/lib/reporter-config3-3.0.3.jar:${project.dir}/lib/sigar-1.6.4.jar:${project.dir}/lib/slf4j-api-1.7.25.jar:${project.dir}/lib/snakeyaml-1.11.jar:${project.dir}/lib/snappy-java-1.1.2.6.jar:${project.dir}/lib/snowball-stemmer-1.3.0.581.1.jar:${project.dir}/lib/stream-2.5.2.jar:${project.dir}/lib/zstd-jni-1.3.8-5.jar:${project.dir}/build/lib/jars/ST4-4.0.8.jar:${project.dir}/build/lib/jars/ant-1.9.7.jar:${project.dir}/build/lib/jars/ant-junit-1.9.7.jar:${project.dir}/build/lib/jars/ant-launcher-1.9.7.jar:${project.dir}/build/lib/jars/antlr-3.5.2.jar:${project.dir}/build/lib/jars/antlr-runtime-3.5.2.jar:${project.dir}/build/lib/jars/apache-rat-0.10.jar:${project.dir}/build/lib/jars/apache-rat-core-0.10.jar:${project.dir}/build/lib/jars/apache-rat-tasks-0.10.jar:${project.dir}/build/lib/jars/byteman-4.0.6.jar:${project.dir}/build/lib/jars/byteman-bmunit-4.0.6.jar:${project.dir}/build/lib/jars/byteman-install-4.0.6.jar:${project.dir}/build/lib/jars/byteman-submit-4.0.6.jar:${project.dir}/build/lib/jars/cassandra-driver-core-3.6.0-shaded.jar:${project.dir}/build/lib/jars/commons-beanutils-1.7.0.jar:${project.dir}/build/lib/jars/commons-beanutils-core-1.8.0.jar:${project.dir}/build/lib/jars/commons-cli-1.2.jar:${project.dir}/build/lib/jars/commons-codec-1.4.jar:${project.dir}/build/lib/jars/commons-collections-3.2.1.jar:${project.dir}/build/lib/jars/commons-compress-1.5.jar:${project.dir}/build/lib/jars/commons-configuration-1.6.jar:${project.dir}/build/lib/jars/commons-digester-1.8.jar:${project.dir}/build/lib/jars/commons-el-1.0.jar:${project.dir}/build/lib/jars/commons-httpclient-3.0.1.jar:${project.dir}/build/lib/jars/commons-io-2.2.jar:${project.dir}/build/lib/jars/commons-lang-2.4.jar:${project.dir}/build/lib/jars/commons-math-2.1.jar:${project.dir}/build/lib/jars/commons-math3-3.2.jar:${project.dir}/build/lib/jars/commons-net-1.4.1.jar:${project.dir}/build/lib/jars/compile-command-annotations-1.2.0.jar:${project.dir}/build/lib/jars/ecj-4.6.1.jar:${project.dir}/build/lib/jars/ftplet-api-1.0.0.jar:${project.dir}/build/lib/jars/ftpserver-core-1.0.0.jar:${project.dir}/build/lib/jars/ftpserver-deprecated-1.0.0-M2.jar:${project.dir}/build/lib/jars/guava-19.0.jar:${project.dir}/build/lib/jars/hadoop-core-1.0.3.jar:${project.dir}/build/lib/jars/hadoop-minicluster-1.0.3.jar:${project.dir}/build/lib/jars/hadoop-test-1.0.3.jar:${project.dir}/build/lib/jars/hamcrest-core-1.3.jar:${project.dir}/build/lib/jars/hsqldb-1.8.0.10.jar:${project.dir}/build/lib/jars/jackson-core-asl-1.0.1.jar:${project.dir}/build/lib/jars/jackson-mapper-asl-1.0.1.jar:${project.dir}/build/lib/jars/jacocoagent.jar:${project.dir}/build/lib/jars/jasper-compiler-5.5.12.jar:${project.dir}/build/lib/jars/jasper-runtime-5.5.12.jar:${project.dir}/build/lib/jars/jersey-core-1.0.jar:${project.dir}/build/lib/jars/jersey-server-1.0.jar:${project.dir}/build/lib/jars/jets3t-0.7.1.jar:${project.dir}/build/lib/jars/jetty-6.1.26.jar:${project.dir}/build/lib/jars/jetty-util-6.1.26.jar:${project.dir}/build/lib/jars/jffi-1.2.16-native.jar:${project.dir}/build/lib/jars/jffi-1.2.16.jar:${project.dir}/build/lib/jars/jmh-core-1.21.jar:${project.dir}/build/lib/jars/jmh-generator-annprocess-1.21.jar:${project.dir}/build/lib/jars/jna-4.1.0.jar:${project.dir}/build/lib/jars/jnr-constants-0.9.9.jar:${project.dir}/build/lib/jars/jnr-ffi-2.1.7.jar:${project.dir}/build/lib/jars/jnr-posix-3.0.44.jar:${project.dir}/build/lib/jars/jnr-x86asm-1.0.2.jar:${project.dir}/build/lib/jars/jopt-simple-4.6.jar:${project.dir}/build/lib/jars/jsp-2.1-6.1.14.jar:${project.dir}/build/lib/jars/jsp-api-2.1-6.1.14.jar:${project.dir}/build/lib/jars/jsr305-2.0.2.jar:${project.dir}/build/lib/jars/jsr311-api-1.0.jar:${project.dir}/build/lib/jars/junit-4.12.jar:${project.dir}/build/lib/jars/kfs-0.3.jar:${project.dir}/build/lib/jars/metrics-core-3.2.2.jar:${project.dir}/build/lib/jars/mina-core-2.0.0-M5.jar:${project.dir}/build/lib/jars/ohc-core-0.5.1.jar:${project.dir}/build/lib/jars/ohc-core-j8-0.5.1.jar:${project.dir}/build/lib/jars/org.jacoco.agent-0.7.5.201505241946.jar:${project.dir}/build/lib/jars/org.jacoco.ant-0.7.5.201505241946.jar:${project.dir}/build/lib/jars/org.jacoco.core-0.7.5.201505241946.jar:${project.dir}/build/lib/jars/org.jacoco.report-0.7.5.201505241946.jar:${project.dir}/build/lib/jars/oro-2.0.8.jar:${project.dir}/build/lib/jars/psjava-0.1.19.jar:${project.dir}/build/lib/jars/quicktheories-0.25.jar:${project.dir}/build/lib/jars/servlet-api-2.5-6.1.14.jar:${project.dir}/build/lib/jars/xmlenc-0.52.jar:</property>
+ </properties>
+ <folders>
+ <source-folder>
+ <label>src/gen-java</label>
+ <type>java</type>
+ <location>${project.dir}/src/gen-java</location>
+ <encoding>UTF-8</encoding>
+ </source-folder>
+ <source-folder>
+ <label>src/java</label>
+ <type>java</type>
+ <location>${project.dir}/src/java</location>
+ <encoding>UTF-8</encoding>
+ </source-folder>
+ <source-folder>
+ <label>src/fqltool</label>
+ <type>java</type>
+ <location>${project.dir}/tools/fqltool/src</location>
+ <encoding>UTF-8</encoding>
+ </source-folder>
+ <source-folder>
+ <label>src/stress</label>
+ <type>java</type>
+ <location>${project.dir}/tools/stress/src</location>
+ <encoding>UTF-8</encoding>
+ </source-folder>
+ <source-folder>
+ <label>test/unit</label>
+ <type>java</type>
+ <location>${project.dir}/test/unit</location>
+ <encoding>UTF-8</encoding>
+ </source-folder>
+ <source-folder>
+ <label>test/fqltool/</label>
+ <type>java</type>
+ <location>${project.dir}/tools/fqltool/test/unit</location>
+ <encoding>UTF-8</encoding>
+ </source-folder>
+ <source-folder>
+ <label>test/stress</label>
+ <type>java</type>
+ <location>${project.dir}/tools/stress/test/unit</location>
+ <encoding>UTF-8</encoding>
+ </source-folder>
+ <source-folder>
+ <label>test/distributed</label>
+ <type>java</type>
+ <location>${project.dir}/test/distributed</location>
+ <encoding>UTF-8</encoding>
+ </source-folder>
+ <source-folder>
+ <label>test/burn</label>
+ <type>java</type>
+ <location>${project.dir}/test/burn</location>
+ <encoding>UTF-8</encoding>
+ </source-folder>
+ <source-folder>
+ <label>test/long</label>
+ <type>java</type>
+ <location>${project.dir}/test/long</location>
+ <encoding>UTF-8</encoding>
+ </source-folder>
+ <source-folder>
+ <label>test/microbench</label>
+ <type>java</type>
+ <location>${project.dir}/test/microbench</location>
+ <encoding>UTF-8</encoding>
+ </source-folder>
+ <source-folder>
+ <label>apache-cassandra</label>
+ <location>${project.dir}</location>
+ <encoding>UTF-8</encoding>
+ </source-folder>
+ </folders>
+ <ide-actions>
+ <action name="build">
+ <script>nbproject/ide-actions.xml</script>
+ <target>build</target>
+ </action>
+ <action name="clean">
+ <script>nbproject/ide-actions.xml</script>
+ <target>clean</target>
+ </action>
+ <action name="rebuild">
+ <script>nbproject/ide-actions.xml</script>
+ <target>clean</target>
+ <target>build</target>
+ </action>
+ <action name="run">
+ <script>nbproject/ide-actions.xml</script>
+ <target>run</target>
+ </action>
+ <action name="debug">
+ <script>nbproject/ide-actions.xml</script>
+ <target>debug</target>
+ </action>
+ <action name="profile">
+ <script>nbproject/ide-actions.xml</script>
+ <target>profile</target>
+ </action>
+ </ide-actions>
+ <export>
+ <type>folder</type>
+ <location>${project.dir}/build/classes/main</location>
+ <script>../build.xml</script>
+ <build-target>build</build-target>
+ </export>
+ <export>
+ <type>folder</type>
+ <location>${project.dir}/build/classes/fqltool</location>
+ <script>../build.xml</script>
+ <build-target>fqltool-build</build-target>
+ </export>
+ <export>
+ <type>folder</type>
+ <location>${project.dir}/build/classes/stress</location>
+ <script>../build.xml</script>
+ <build-target>stress-build</build-target>
+ </export>
+ <export>
+ <type>folder</type>
+ <location>${project.dir}/build/test/classes</location>
+ <script>../build.xml</script>
+ <build-target>build-test</build-target>
+ </export>
+ <export>
+ <type>folder</type>
+ <location>${project.dir}/build/test/fqltool-classes</location>
+ <script>../build.xml</script>
+ <build-target>fqltool-build-test</build-target>
+ </export>
+ <export>
+ <type>folder</type>
+ <location>${project.dir}/build/test/stress-classes</location>
+ <script>../build.xml</script>
+ <build-target>stress-build-test</build-target>
+ </export>
+ <view>
+ <items>
+ <source-folder style="packages">
+ <label>src/gen-java</label>
+ <location>${project.dir}/src/gen-java</location>
+ </source-folder>
+ <source-folder style="packages">
+ <label>src/java</label>
+ <location>${project.dir}/src/java</location>
+ </source-folder>
+ <source-folder style="packages">
+ <label>src/fqltool</label>
+ <location>${project.dir}/tools/fqltool/src</location>
+ </source-folder>
+ <source-folder style="packages">
+ <label>src/stress</label>
+ <location>${project.dir}/tools/stress/src</location>
+ </source-folder>
+ <source-folder style="packages">
+ <label>test/unit</label>
+ <location>${project.dir}/test/unit</location>
+ </source-folder>
+ <source-folder style="packages">
+ <label>test/fqltool/</label>
+ <location>${project.dir}/tools/fqltool/test/unit</location>
+ </source-folder>
+ <source-folder style="packages">
+ <label>test/stress</label>
+ <location>${project.dir}/tools/stress/test/unit</location>
+ </source-folder>
+ <source-folder style="packages">
+ <label>test/distributed</label>
+ <location>${project.dir}/test/distributed</location>
+ </source-folder>
+ <source-folder style="packages">
+ <label>test/burn</label>
+ <location>${project.dir}/test/burn</location>
+ </source-folder>
+ <source-folder style="packages">
+ <label>test/long</label>
+ <location>${project.dir}/test/long</location>
+ </source-folder>
+ <source-folder style="packages">
+ <label>test/microbench</label>
+ <location>${project.dir}/test/microbench</location>
+ </source-folder>
+ <source-file>
+ <location>${project.dir}/build.xml</location>
+ </source-file>
+ </items>
+ <context-menu>
+ <ide-action name="build"/>
+ <ide-action name="rebuild"/>
+ <ide-action name="clean"/>
+ <ide-action name="run"/>
+ <ide-action name="debug"/>
+ </context-menu>
+ </view>
+ </general-data>
+ <java-data xmlns="http://www.netbeans.org/ns/freeform-project-java/4">
+ <compilation-unit>
+ <package-root>${project.dir}/src/gen-java</package-root>
+ <package-root>${project.dir}/src/java</package-root>
+ <classpath mode="compile">${cassandra.classpath.jars}</classpath>
+ <built-to>${project.dir}/build/classes/main</built-to>
+ <source-level>1.8</source-level>
+ </compilation-unit>
+ <compilation-unit>
+ <package-root>${project.dir}/tools/fqltool/src</package-root>
+ <classpath mode="compile">${cassandra.classpath.jars}:${project.dir}/build/classes/main</classpath>
+ <built-to>${project.dir}/build/classes/fqltool</built-to>
+ <source-level>1.8</source-level>
+ </compilation-unit>
+ <compilation-unit>
+ <package-root>${project.dir}/tools/stress/src</package-root>
+ <classpath mode="compile">${cassandra.classpath.jars}:${project.dir}/build/classes/main</classpath>
+ <built-to>${project.dir}/build/classes/stress</built-to>
+ <source-level>1.8</source-level>
+ </compilation-unit>
+ <compilation-unit>
+ <package-root>${project.dir}/test/unit</package-root>
+ <package-root>${project.dir}/test/distributed</package-root>
+ <package-root>${project.dir}/test/long</package-root>
+ <package-root>${project.dir}/test/microbench</package-root>
+ <package-root>${project.dir}/test/burn</package-root>
+ <unit-tests/>
+ <classpath mode="compile">${cassandra.classpath.jars}:${project.dir}/build/classes/main:${project.dir}/build/classes/fqltool/:${project.dir}/build/classes/stress/</classpath>
+ <built-to>${project.dir}/build/test/classes</built-to>
+ <source-level>1.8</source-level>
+ </compilation-unit>
+ <compilation-unit>
+ <package-root>${project.dir}/tools/fqltool/test/unit</package-root>
+ <unit-tests/>
+ <classpath mode="compile">${cassandra.classpath.jars}:${project.dir}/build/classes/main:${project.dir}/build/classes/fqltool/</classpath>
+ <built-to>${project.dir}/build/test/fqltool-classes</built-to>
+ <source-level>1.8</source-level>
+ </compilation-unit>
+ <compilation-unit>
+ <package-root>${project.dir}/tools/stress/test/unit</package-root>
+ <unit-tests/>
+ <classpath mode="compile">${cassandra.classpath.jars}:${project.dir}/build/classes/main:${project.dir}/build/classes/stress/</classpath>
+ <built-to>${project.dir}/build/test/stress-classes</built-to>
+ <source-level>1.8</source-level>
+ </compilation-unit>
+ </java-data>
+ </configuration>
+</project>
diff --git a/ide/nbproject/update-netbeans-classpaths.sh b/ide/nbproject/update-netbeans-classpaths.sh
new file mode 100755
index 0000000..afef864
--- /dev/null
+++ b/ide/nbproject/update-netbeans-classpaths.sh
@@ -0,0 +1,14 @@
+#!/bin/bash
+#
+# Update the classpaths elements in the project.xml found in the same directory
+# Works around the lack of wildcarded classpaths in netbeans freeform projects
+# ref: https://netbeans.org/bugzilla/show_bug.cgi?id=116185
+#
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
+cd $DIR/../..
+CLASSPATH=`for f in lib/*.jar ; do echo -n '${project.dir}/'$f: ; done ; for f in build/lib/jars/*.jar ; do echo -n '${project.dir}/'$f: ; done ;`
+
+sed -i '' 's/cassandra\.classpath\.jars\">.*<\/property>/cassandra\.classpath\.jars\">NEW_CLASSPATH<\/property>/' $DIR/project.xml
+sed -i '' "s@NEW_CLASSPATH@"$CLASSPATH"@" $DIR/project.xml
diff --git a/interface/cassandra.thrift b/interface/cassandra.thrift
deleted file mode 100644
index 2970cff..0000000
--- a/interface/cassandra.thrift
+++ /dev/null
@@ -1,945 +0,0 @@
-#!/usr/local/bin/thrift --java --php --py
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-# *** PLEASE REMEMBER TO EDIT THE VERSION CONSTANT WHEN MAKING CHANGES ***
-# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-#
-# Interface definition for Cassandra Service
-#
-
-namespace java org.apache.cassandra.thrift
-namespace cpp org.apache.cassandra
-namespace csharp Apache.Cassandra
-namespace py cassandra
-namespace php cassandra
-namespace perl Cassandra
-
-# Thrift.rb has a bug where top-level modules that include modules
-# with the same name are not properly referenced, so we can't do
-# Cassandra::Cassandra::Client.
-namespace rb CassandraThrift
-
-# The API version (NOT the product version), composed as a dot delimited
-# string with major, minor, and patch level components.
-#
-# - Major: Incremented for backward incompatible changes. An example would
-# be changes to the number or disposition of method arguments.
-# - Minor: Incremented for backward compatible changes. An example would
-# be the addition of a new (optional) method.
-# - Patch: Incremented for bug fixes. The patch level should be increased
-# for every edit that doesn't result in a change to major/minor.
-#
-# See the Semantic Versioning Specification (SemVer) http://semver.org.
-#
-# Note that this backwards compatibility is from the perspective of the server,
-# not the client. Cassandra should always be able to talk to older client
-# software, but client software may not be able to talk to older Cassandra
-# instances.
-#
-# An effort should be made not to break forward-client-compatibility either
-# (e.g. one should avoid removing obsolete fields from the IDL), but no
-# guarantees in this respect are made by the Cassandra project.
-const string VERSION = "20.1.0"
-
-
-#
-# data structures
-#
-
-/** Basic unit of data within a ColumnFamily.
- * @param name, the name by which this column is set and retrieved. Maximum 64KB long.
- * @param value. The data associated with the name. Maximum 2GB long, but in practice you should limit it to small numbers of MB (since Thrift must read the full value into memory to operate on it).
- * @param timestamp. The timestamp is used for conflict detection/resolution when two columns with same name need to be compared.
- * @param ttl. An optional, positive delay (in seconds) after which the column will be automatically deleted.
- */
-struct Column {
- 1: required binary name,
- 2: optional binary value,
- 3: optional i64 timestamp,
- 4: optional i32 ttl,
-}
-
-/** A named list of columns.
- * @param name. see Column.name.
- * @param columns. A collection of standard Columns. The columns within a super column are defined in an adhoc manner.
- * Columns within a super column do not have to have matching structures (similarly named child columns).
- */
-struct SuperColumn {
- 1: required binary name,
- 2: required list<Column> columns,
-}
-
-struct CounterColumn {
- 1: required binary name,
- 2: required i64 value
-}
-
-struct CounterSuperColumn {
- 1: required binary name,
- 2: required list<CounterColumn> columns
-}
-
-/**
- Methods for fetching rows/records from Cassandra will return either a single instance of ColumnOrSuperColumn or a list
- of ColumnOrSuperColumns (get_slice()). If you're looking up a SuperColumn (or list of SuperColumns) then the resulting
- instances of ColumnOrSuperColumn will have the requested SuperColumn in the attribute super_column. For queries resulting
- in Columns, those values will be in the attribute column. This change was made between 0.3 and 0.4 to standardize on
- single query methods that may return either a SuperColumn or Column.
-
- If the query was on a counter column family, you will either get a counter_column (instead of a column) or a
- counter_super_column (instead of a super_column)
-
- @param column. The Column returned by get() or get_slice().
- @param super_column. The SuperColumn returned by get() or get_slice().
- @param counter_column. The Counterolumn returned by get() or get_slice().
- @param counter_super_column. The CounterSuperColumn returned by get() or get_slice().
- */
-struct ColumnOrSuperColumn {
- 1: optional Column column,
- 2: optional SuperColumn super_column,
- 3: optional CounterColumn counter_column,
- 4: optional CounterSuperColumn counter_super_column
-}
-
-
-#
-# Exceptions
-# (note that internal server errors will raise a TApplicationException, courtesy of Thrift)
-#
-
-/** A specific column was requested that does not exist. */
-exception NotFoundException {
-}
-
-/** Invalid request could mean keyspace or column family does not exist, required parameters are missing, or a parameter is malformed.
- why contains an associated error message.
-*/
-exception InvalidRequestException {
- 1: required string why
-}
-
-/** Not all the replicas required could be created and/or read. */
-exception UnavailableException {
-}
-
-/** RPC timeout was exceeded. either a node failed mid-operation, or load was too high, or the requested op was too large. */
-exception TimedOutException {
- /**
- * if a write operation was acknowledged by some replicas but not by enough to
- * satisfy the required ConsistencyLevel, the number of successful
- * replies will be given here. In case of atomic_batch_mutate method this field
- * will be set to -1 if the batch was written to the batchlog and to 0 if it wasn't.
- */
- 1: optional i32 acknowledged_by
-
- /**
- * in case of atomic_batch_mutate method this field tells if the batch
- * was written to the batchlog.
- */
- 2: optional bool acknowledged_by_batchlog
-
- /**
- * for the CAS method, this field tells if we timed out during the paxos
- * protocol, as opposed to during the commit of our update
- */
- 3: optional bool paxos_in_progress
-}
-
-/** invalid authentication request (invalid keyspace, user does not exist, or credentials invalid) */
-exception AuthenticationException {
- 1: required string why
-}
-
-/** invalid authorization request (user does not have access to keyspace) */
-exception AuthorizationException {
- 1: required string why
-}
-
-/**
- * NOTE: This up outdated exception left for backward compatibility reasons,
- * no actual schema agreement validation is done starting from Cassandra 1.2
- *
- * schemas are not in agreement across all nodes
- */
-exception SchemaDisagreementException {
-}
-
-
-#
-# service api
-#
-/**
- * The ConsistencyLevel is an enum that controls both read and write
- * behavior based on the ReplicationFactor of the keyspace. The
- * different consistency levels have different meanings, depending on
- * if you're doing a write or read operation.
- *
- * If W + R > ReplicationFactor, where W is the number of nodes to
- * block for on write, and R the number to block for on reads, you
- * will have strongly consistent behavior; that is, readers will
- * always see the most recent write. Of these, the most interesting is
- * to do QUORUM reads and writes, which gives you consistency while
- * still allowing availability in the face of node failures up to half
- * of <ReplicationFactor>. Of course if latency is more important than
- * consistency then you can use lower values for either or both.
- *
- * Some ConsistencyLevels (ONE, TWO, THREE) refer to a specific number
- * of replicas rather than a logical concept that adjusts
- * automatically with the replication factor. Of these, only ONE is
- * commonly used; TWO and (even more rarely) THREE are only useful
- * when you care more about guaranteeing a certain level of
- * durability, than consistency.
- *
- * Write consistency levels make the following guarantees before reporting success to the client:
- * ANY Ensure that the write has been written once somewhere, including possibly being hinted in a non-target node.
- * ONE Ensure that the write has been written to at least 1 node's commit log and memory table
- * TWO Ensure that the write has been written to at least 2 node's commit log and memory table
- * THREE Ensure that the write has been written to at least 3 node's commit log and memory table
- * QUORUM Ensure that the write has been written to <ReplicationFactor> / 2 + 1 nodes
- * LOCAL_ONE Ensure that the write has been written to 1 node within the local datacenter (requires NetworkTopologyStrategy)
- * LOCAL_QUORUM Ensure that the write has been written to <ReplicationFactor> / 2 + 1 nodes, within the local datacenter (requires NetworkTopologyStrategy)
- * EACH_QUORUM Ensure that the write has been written to <ReplicationFactor> / 2 + 1 nodes in each datacenter (requires NetworkTopologyStrategy)
- * ALL Ensure that the write is written to <code><ReplicationFactor></code> nodes before responding to the client.
- *
- * Read consistency levels make the following guarantees before returning successful results to the client:
- * ANY Not supported. You probably want ONE instead.
- * ONE Returns the record obtained from a single replica.
- * TWO Returns the record with the most recent timestamp once two replicas have replied.
- * THREE Returns the record with the most recent timestamp once three replicas have replied.
- * QUORUM Returns the record with the most recent timestamp once a majority of replicas have replied.
- * LOCAL_ONE Returns the record with the most recent timestamp once a single replica within the local datacenter have replied.
- * LOCAL_QUORUM Returns the record with the most recent timestamp once a majority of replicas within the local datacenter have replied.
- * EACH_QUORUM Returns the record with the most recent timestamp once a majority of replicas within each datacenter have replied.
- * ALL Returns the record with the most recent timestamp once all replicas have replied (implies no replica may be down)..
-*/
-enum ConsistencyLevel {
- ONE = 1,
- QUORUM = 2,
- LOCAL_QUORUM = 3,
- EACH_QUORUM = 4,
- ALL = 5,
- ANY = 6,
- TWO = 7,
- THREE = 8,
- SERIAL = 9,
- LOCAL_SERIAL = 10,
- LOCAL_ONE = 11,
-}
-
-/**
- ColumnParent is used when selecting groups of columns from the same ColumnFamily. In directory structure terms, imagine
- ColumnParent as ColumnPath + '/../'.
-
- See also <a href="cassandra.html#Struct_ColumnPath">ColumnPath</a>
- */
-struct ColumnParent {
- 3: required string column_family,
- 4: optional binary super_column,
-}
-
-/** The ColumnPath is the path to a single column in Cassandra. It might make sense to think of ColumnPath and
- * ColumnParent in terms of a directory structure.
- *
- * ColumnPath is used to looking up a single column.
- *
- * @param column_family. The name of the CF of the column being looked up.
- * @param super_column. The super column name.
- * @param column. The column name.
- */
-struct ColumnPath {
- 3: required string column_family,
- 4: optional binary super_column,
- 5: optional binary column,
-}
-
-/**
- A slice range is a structure that stores basic range, ordering and limit information for a query that will return
- multiple columns. It could be thought of as Cassandra's version of LIMIT and ORDER BY
-
- @param start. The column name to start the slice with. This attribute is not required, though there is no default value,
- and can be safely set to '', i.e., an empty byte array, to start with the first column name. Otherwise, it
- must a valid value under the rules of the Comparator defined for the given ColumnFamily.
- @param finish. The column name to stop the slice at. This attribute is not required, though there is no default value,
- and can be safely set to an empty byte array to not stop until 'count' results are seen. Otherwise, it
- must also be a valid value to the ColumnFamily Comparator.
- @param reversed. Whether the results should be ordered in reversed order. Similar to ORDER BY blah DESC in SQL.
- @param count. How many columns to return. Similar to LIMIT in SQL. May be arbitrarily large, but Thrift will
- materialize the whole result into memory before returning it to the client, so be aware that you may
- be better served by iterating through slices by passing the last value of one call in as the 'start'
- of the next instead of increasing 'count' arbitrarily large.
- */
-struct SliceRange {
- 1: required binary start,
- 2: required binary finish,
- 3: required bool reversed=0,
- 4: required i32 count=100,
-}
-
-/**
- A SlicePredicate is similar to a mathematic predicate (see http://en.wikipedia.org/wiki/Predicate_(mathematical_logic)),
- which is described as "a property that the elements of a set have in common."
-
- SlicePredicate's in Cassandra are described with either a list of column_names or a SliceRange. If column_names is
- specified, slice_range is ignored.
-
- @param column_name. A list of column names to retrieve. This can be used similar to Memcached's "multi-get" feature
- to fetch N known column names. For instance, if you know you wish to fetch columns 'Joe', 'Jack',
- and 'Jim' you can pass those column names as a list to fetch all three at once.
- @param slice_range. A SliceRange describing how to range, order, and/or limit the slice.
- */
-struct SlicePredicate {
- 1: optional list<binary> column_names,
- 2: optional SliceRange slice_range,
-}
-
-enum IndexOperator {
- EQ,
- GTE,
- GT,
- LTE,
- LT
-}
-
-struct IndexExpression {
- 1: required binary column_name,
- 2: required IndexOperator op,
- 3: required binary value,
-}
-
-/**
- * @deprecated use a KeyRange with row_filter in get_range_slices instead
- */
-struct IndexClause {
- 1: required list<IndexExpression> expressions,
- 2: required binary start_key,
- 3: required i32 count=100,
-}
-
-
-/**
-The semantics of start keys and tokens are slightly different.
-Keys are start-inclusive; tokens are start-exclusive. Token
-ranges may also wrap -- that is, the end token may be less
-than the start one. Thus, a range from keyX to keyX is a
-one-element range, but a range from tokenY to tokenY is the
-full ring.
-*/
-struct KeyRange {
- 1: optional binary start_key,
- 2: optional binary end_key,
- 3: optional string start_token,
- 4: optional string end_token,
- 6: optional list<IndexExpression> row_filter,
- 5: required i32 count=100
-}
-
-/**
- A KeySlice is key followed by the data it maps to. A collection of KeySlice is returned by the get_range_slice operation.
-
- @param key. a row key
- @param columns. List of data represented by the key. Typically, the list is pared down to only the columns specified by
- a SlicePredicate.
- */
-struct KeySlice {
- 1: required binary key,
- 2: required list<ColumnOrSuperColumn> columns,
-}
-
-struct KeyCount {
- 1: required binary key,
- 2: required i32 count
-}
-
-/**
- * Note that the timestamp is only optional in case of counter deletion.
- */
-struct Deletion {
- 1: optional i64 timestamp,
- 2: optional binary super_column,
- 3: optional SlicePredicate predicate,
-}
-
-/**
- A Mutation is either an insert (represented by filling column_or_supercolumn) or a deletion (represented by filling the deletion attribute).
- @param column_or_supercolumn. An insert to a column or supercolumn (possibly counter column or supercolumn)
- @param deletion. A deletion of a column or supercolumn
-*/
-struct Mutation {
- 1: optional ColumnOrSuperColumn column_or_supercolumn,
- 2: optional Deletion deletion,
-}
-
-struct EndpointDetails {
- 1: string host,
- 2: string datacenter,
- 3: optional string rack
-}
-
-struct CASResult {
- 1: required bool success,
- 2: optional list<Column> current_values,
-}
-
-/**
- A TokenRange describes part of the Cassandra ring, it is a mapping from a range to
- endpoints responsible for that range.
- @param start_token The first token in the range
- @param end_token The last token in the range
- @param endpoints The endpoints responsible for the range (listed by their configured listen_address)
- @param rpc_endpoints The endpoints responsible for the range (listed by their configured rpc_address)
-*/
-struct TokenRange {
- 1: required string start_token,
- 2: required string end_token,
- 3: required list<string> endpoints,
- 4: optional list<string> rpc_endpoints
- 5: optional list<EndpointDetails> endpoint_details,
-}
-
-/**
- Authentication requests can contain any data, dependent on the IAuthenticator used
-*/
-struct AuthenticationRequest {
- 1: required map<string, string> credentials
-}
-
-enum IndexType {
- KEYS,
- CUSTOM,
- COMPOSITES
-}
-
-/* describes a column in a column family. */
-struct ColumnDef {
- 1: required binary name,
- 2: required string validation_class,
- 3: optional IndexType index_type,
- 4: optional string index_name,
- 5: optional map<string,string> index_options
-}
-
-/**
- Describes a trigger.
- `options` should include at least 'class' param.
- Other options are not supported yet.
-*/
-struct TriggerDef {
- 1: required string name,
- 2: required map<string,string> options
-}
-
-/* describes a column family. */
-struct CfDef {
- 1: required string keyspace,
- 2: required string name,
- 3: optional string column_type="Standard",
- 5: optional string comparator_type="BytesType",
- 6: optional string subcomparator_type,
- 8: optional string comment,
- 12: optional double read_repair_chance,
- 13: optional list<ColumnDef> column_metadata,
- 14: optional i32 gc_grace_seconds,
- 15: optional string default_validation_class,
- 16: optional i32 id,
- 17: optional i32 min_compaction_threshold,
- 18: optional i32 max_compaction_threshold,
- 26: optional string key_validation_class,
- 28: optional binary key_alias,
- 29: optional string compaction_strategy,
- 30: optional map<string,string> compaction_strategy_options,
- 32: optional map<string,string> compression_options,
- 33: optional double bloom_filter_fp_chance,
- 34: optional string caching="keys_only",
- 37: optional double dclocal_read_repair_chance = 0.0,
- 39: optional i32 memtable_flush_period_in_ms,
- 40: optional i32 default_time_to_live,
- 42: optional string speculative_retry="NONE",
- 43: optional list<TriggerDef> triggers,
- 44: optional string cells_per_row_to_cache = "100",
- 45: optional i32 min_index_interval,
- 46: optional i32 max_index_interval,
-
- /* All of the following are now ignored and unsupplied. */
-
- /** @deprecated */
- 9: optional double row_cache_size,
- /** @deprecated */
- 11: optional double key_cache_size,
- /** @deprecated */
- 19: optional i32 row_cache_save_period_in_seconds,
- /** @deprecated */
- 20: optional i32 key_cache_save_period_in_seconds,
- /** @deprecated */
- 21: optional i32 memtable_flush_after_mins,
- /** @deprecated */
- 22: optional i32 memtable_throughput_in_mb,
- /** @deprecated */
- 23: optional double memtable_operations_in_millions,
- /** @deprecated */
- 24: optional bool replicate_on_write,
- /** @deprecated */
- 25: optional double merge_shards_chance,
- /** @deprecated */
- 27: optional string row_cache_provider,
- /** @deprecated */
- 31: optional i32 row_cache_keys_to_save,
- /** @deprecated */
- 38: optional bool populate_io_cache_on_flush,
- /** @deprecated */
- 41: optional i32 index_interval,
-}
-
-/* describes a keyspace. */
-struct KsDef {
- 1: required string name,
- 2: required string strategy_class,
- 3: optional map<string,string> strategy_options,
-
- /** @deprecated ignored */
- 4: optional i32 replication_factor,
-
- 5: required list<CfDef> cf_defs,
- 6: optional bool durable_writes=1,
-}
-
-/** CQL query compression */
-enum Compression {
- GZIP = 1,
- NONE = 2
-}
-
-enum CqlResultType {
- ROWS = 1,
- VOID = 2,
- INT = 3
-}
-
-/**
- Row returned from a CQL query.
-
- This struct is used for both CQL2 and CQL3 queries. For CQL2, the partition key
- is special-cased and is always returned. For CQL3, it is not special cased;
- it will be included in the columns list if it was included in the SELECT and
- the key field is always null.
-*/
-struct CqlRow {
- 1: required binary key,
- 2: required list<Column> columns
-}
-
-struct CqlMetadata {
- 1: required map<binary,string> name_types,
- 2: required map<binary,string> value_types,
- 3: required string default_name_type,
- 4: required string default_value_type
-}
-
-struct CqlResult {
- 1: required CqlResultType type,
- 2: optional list<CqlRow> rows,
- 3: optional i32 num,
- 4: optional CqlMetadata schema
-}
-
-struct CqlPreparedResult {
- 1: required i32 itemId,
- 2: required i32 count,
- 3: optional list<string> variable_types,
- 4: optional list<string> variable_names
-}
-
-/** Represents input splits used by hadoop ColumnFamilyRecordReaders */
-struct CfSplit {
- 1: required string start_token,
- 2: required string end_token,
- 3: required i64 row_count
-}
-
-/** The ColumnSlice is used to select a set of columns from inside a row.
- * If start or finish are unspecified they will default to the start-of
- * end-of value.
- * @param start. The start of the ColumnSlice inclusive
- * @param finish. The end of the ColumnSlice inclusive
- */
-struct ColumnSlice {
- 1: optional binary start,
- 2: optional binary finish
-}
-
-/**
- * Used to perform multiple slices on a single row key in one rpc operation
- * @param key. The row key to be multi sliced
- * @param column_parent. The column family (super columns are unsupported)
- * @param column_slices. 0 to many ColumnSlice objects each will be used to select columns
- * @param reversed. Direction of slice
- * @param count. Maximum number of columns
- * @param consistency_level. Level to perform the operation at
- */
-struct MultiSliceRequest {
- 1: optional binary key,
- 2: optional ColumnParent column_parent,
- 3: optional list<ColumnSlice> column_slices,
- 4: optional bool reversed=false,
- 5: optional i32 count=1000,
- 6: optional ConsistencyLevel consistency_level=ConsistencyLevel.ONE
-}
-
-service Cassandra {
- # auth methods
- void login(1: required AuthenticationRequest auth_request) throws (1:AuthenticationException authnx, 2:AuthorizationException authzx),
-
- # set keyspace
- void set_keyspace(1: required string keyspace) throws (1:InvalidRequestException ire),
-
- # retrieval methods
-
- /**
- Get the Column or SuperColumn at the given column_path. If no value is present, NotFoundException is thrown. (This is
- the only method that can throw an exception under non-failure conditions.)
- */
- ColumnOrSuperColumn get(1:required binary key,
- 2:required ColumnPath column_path,
- 3:required ConsistencyLevel consistency_level=ConsistencyLevel.ONE)
- throws (1:InvalidRequestException ire, 2:NotFoundException nfe, 3:UnavailableException ue, 4:TimedOutException te),
-
- /**
- Get the group of columns contained by column_parent (either a ColumnFamily name or a ColumnFamily/SuperColumn name
- pair) specified by the given SlicePredicate. If no matching values are found, an empty list is returned.
- */
- list<ColumnOrSuperColumn> get_slice(1:required binary key,
- 2:required ColumnParent column_parent,
- 3:required SlicePredicate predicate,
- 4:required ConsistencyLevel consistency_level=ConsistencyLevel.ONE)
- throws (1:InvalidRequestException ire, 2:UnavailableException ue, 3:TimedOutException te),
-
- /**
- returns the number of columns matching <code>predicate</code> for a particular <code>key</code>,
- <code>ColumnFamily</code> and optionally <code>SuperColumn</code>.
- */
- i32 get_count(1:required binary key,
- 2:required ColumnParent column_parent,
- 3:required SlicePredicate predicate,
- 4:required ConsistencyLevel consistency_level=ConsistencyLevel.ONE)
- throws (1:InvalidRequestException ire, 2:UnavailableException ue, 3:TimedOutException te),
-
- /**
- Performs a get_slice for column_parent and predicate for the given keys in parallel.
- */
- map<binary,list<ColumnOrSuperColumn>> multiget_slice(1:required list<binary> keys,
- 2:required ColumnParent column_parent,
- 3:required SlicePredicate predicate,
- 4:required ConsistencyLevel consistency_level=ConsistencyLevel.ONE)
- throws (1:InvalidRequestException ire, 2:UnavailableException ue, 3:TimedOutException te),
-
- /**
- Perform a get_count in parallel on the given {@code List<binary>} keys. The return value maps keys to the count found.
- */
- map<binary, i32> multiget_count(1:required list<binary> keys,
- 2:required ColumnParent column_parent,
- 3:required SlicePredicate predicate,
- 4:required ConsistencyLevel consistency_level=ConsistencyLevel.ONE)
- throws (1:InvalidRequestException ire, 2:UnavailableException ue, 3:TimedOutException te),
-
- /**
- returns a subset of columns for a contiguous range of keys.
- */
- list<KeySlice> get_range_slices(1:required ColumnParent column_parent,
- 2:required SlicePredicate predicate,
- 3:required KeyRange range,
- 4:required ConsistencyLevel consistency_level=ConsistencyLevel.ONE)
- throws (1:InvalidRequestException ire, 2:UnavailableException ue, 3:TimedOutException te),
-
- /**
- returns a range of columns, wrapping to the next rows if necessary to collect max_results.
- */
- list<KeySlice> get_paged_slice(1:required string column_family,
- 2:required KeyRange range,
- 3:required binary start_column,
- 4:required ConsistencyLevel consistency_level=ConsistencyLevel.ONE)
- throws (1:InvalidRequestException ire, 2:UnavailableException ue, 3:TimedOutException te),
-
- /**
- Returns the subset of columns specified in SlicePredicate for the rows matching the IndexClause
- @deprecated use get_range_slices instead with range.row_filter specified
- */
- list<KeySlice> get_indexed_slices(1:required ColumnParent column_parent,
- 2:required IndexClause index_clause,
- 3:required SlicePredicate column_predicate,
- 4:required ConsistencyLevel consistency_level=ConsistencyLevel.ONE)
- throws (1:InvalidRequestException ire, 2:UnavailableException ue, 3:TimedOutException te),
-
- # modification methods
-
- /**
- * Insert a Column at the given column_parent.column_family and optional column_parent.super_column.
- */
- void insert(1:required binary key,
- 2:required ColumnParent column_parent,
- 3:required Column column,
- 4:required ConsistencyLevel consistency_level=ConsistencyLevel.ONE)
- throws (1:InvalidRequestException ire, 2:UnavailableException ue, 3:TimedOutException te),
-
- /**
- * Increment or decrement a counter.
- */
- void add(1:required binary key,
- 2:required ColumnParent column_parent,
- 3:required CounterColumn column,
- 4:required ConsistencyLevel consistency_level=ConsistencyLevel.ONE)
- throws (1:InvalidRequestException ire, 2:UnavailableException ue, 3:TimedOutException te),
-
- /**
- * Atomic compare and set.
- *
- * If the cas is successfull, the success boolean in CASResult will be true and there will be no current_values.
- * Otherwise, success will be false and current_values will contain the current values for the columns in
- * expected (that, by definition of compare-and-set, will differ from the values in expected).
- *
- * A cas operation takes 2 consistency level. The first one, serial_consistency_level, simply indicates the
- * level of serialization required. This can be either ConsistencyLevel.SERIAL or ConsistencyLevel.LOCAL_SERIAL.
- * The second one, commit_consistency_level, defines the consistency level for the commit phase of the cas. This
- * is a more traditional consistency level (the same CL than for traditional writes are accepted) that impact
- * the visibility for reads of the operation. For instance, if commit_consistency_level is QUORUM, then it is
- * guaranteed that a followup QUORUM read will see the cas write (if that one was successful obviously). If
- * commit_consistency_level is ANY, you will need to use a SERIAL/LOCAL_SERIAL read to be guaranteed to see
- * the write.
- */
- CASResult cas(1:required binary key,
- 2:required string column_family,
- 3:list<Column> expected,
- 4:list<Column> updates,
- 5:required ConsistencyLevel serial_consistency_level=ConsistencyLevel.SERIAL,
- 6:required ConsistencyLevel commit_consistency_level=ConsistencyLevel.QUORUM)
- throws (1:InvalidRequestException ire, 2:UnavailableException ue, 3:TimedOutException te),
-
- /**
- Remove data from the row specified by key at the granularity specified by column_path, and the given timestamp. Note
- that all the values in column_path besides column_path.column_family are truly optional: you can remove the entire
- row by just specifying the ColumnFamily, or you can remove a SuperColumn or a single Column by specifying those levels too.
- */
- void remove(1:required binary key,
- 2:required ColumnPath column_path,
- 3:required i64 timestamp,
- 4:ConsistencyLevel consistency_level=ConsistencyLevel.ONE)
- throws (1:InvalidRequestException ire, 2:UnavailableException ue, 3:TimedOutException te),
-
- /**
- * Remove a counter at the specified location.
- * Note that counters have limited support for deletes: if you remove a counter, you must wait to issue any following update
- * until the delete has reached all the nodes and all of them have been fully compacted.
- */
- void remove_counter(1:required binary key,
- 2:required ColumnPath path,
- 3:required ConsistencyLevel consistency_level=ConsistencyLevel.ONE)
- throws (1:InvalidRequestException ire, 2:UnavailableException ue, 3:TimedOutException te),
-
- /**
- Mutate many columns or super columns for many row keys. See also: Mutation.
-
- mutation_map maps key to column family to a list of Mutation objects to take place at that scope.
- **/
- void batch_mutate(1:required map<binary, map<string, list<Mutation>>> mutation_map,
- 2:required ConsistencyLevel consistency_level=ConsistencyLevel.ONE)
- throws (1:InvalidRequestException ire, 2:UnavailableException ue, 3:TimedOutException te),
-
- /**
- Atomically mutate many columns or super columns for many row keys. See also: Mutation.
-
- mutation_map maps key to column family to a list of Mutation objects to take place at that scope.
- **/
- void atomic_batch_mutate(1:required map<binary, map<string, list<Mutation>>> mutation_map,
- 2:required ConsistencyLevel consistency_level=ConsistencyLevel.ONE)
- throws (1:InvalidRequestException ire, 2:UnavailableException ue, 3:TimedOutException te),
-
- /**
- Truncate will mark and entire column family as deleted.
- From the user's perspective a successful call to truncate will result complete data deletion from cfname.
- Internally, however, disk space will not be immediatily released, as with all deletes in cassandra, this one
- only marks the data as deleted.
- The operation succeeds only if all hosts in the cluster at available and will throw an UnavailableException if
- some hosts are down.
- */
- void truncate(1:required string cfname)
- throws (1: InvalidRequestException ire, 2: UnavailableException ue, 3: TimedOutException te),
-
- /**
- * Select multiple slices of a key in a single RPC operation
- */
- list<ColumnOrSuperColumn> get_multi_slice(1:required MultiSliceRequest request)
- throws (1:InvalidRequestException ire, 2:UnavailableException ue, 3:TimedOutException te),
-
- // Meta-APIs -- APIs to get information about the node or cluster,
- // rather than user data. The nodeprobe program provides usage examples.
-
- /**
- * for each schema version present in the cluster, returns a list of nodes at that version.
- * hosts that do not respond will be under the key DatabaseDescriptor.INITIAL_VERSION.
- * the cluster is all on the same version if the size of the map is 1.
- */
- map<string, list<string>> describe_schema_versions()
- throws (1: InvalidRequestException ire),
-
- /** list the defined keyspaces in this cluster */
- list<KsDef> describe_keyspaces()
- throws (1:InvalidRequestException ire),
-
- /** get the cluster name */
- string describe_cluster_name(),
-
- /** get the thrift api version */
- string describe_version(),
-
- /** get the token ring: a map of ranges to host addresses,
- represented as a set of TokenRange instead of a map from range
- to list of endpoints, because you can't use Thrift structs as
- map keys:
- https://issues.apache.org/jira/browse/THRIFT-162
-
- for the same reason, we can't return a set here, even though
- order is neither important nor predictable. */
- list<TokenRange> describe_ring(1:required string keyspace)
- throws (1:InvalidRequestException ire),
-
-
- /** same as describe_ring, but considers only nodes in the local DC */
- list<TokenRange> describe_local_ring(1:required string keyspace)
- throws (1:InvalidRequestException ire),
-
- /** get the mapping between token->node ip
- without taking replication into consideration
- https://issues.apache.org/jira/browse/CASSANDRA-4092 */
- map<string, string> describe_token_map()
- throws (1:InvalidRequestException ire),
-
- /** returns the partitioner used by this cluster */
- string describe_partitioner(),
-
- /** returns the snitch used by this cluster */
- string describe_snitch(),
-
- /** describe specified keyspace */
- KsDef describe_keyspace(1:required string keyspace)
- throws (1:NotFoundException nfe, 2:InvalidRequestException ire),
-
- /** experimental API for hadoop/parallel query support.
- may change violently and without warning.
-
- returns list of token strings such that first subrange is (list[0], list[1]],
- next is (list[1], list[2]], etc. */
- list<string> describe_splits(1:required string cfName,
- 2:required string start_token,
- 3:required string end_token,
- 4:required i32 keys_per_split)
- throws (1:InvalidRequestException ire),
-
- /** Enables tracing for the next query in this connection and returns the UUID for that trace session
- The next query will be traced idependently of trace probability and the returned UUID can be used to query the trace keyspace */
- binary trace_next_query(),
-
- list<CfSplit> describe_splits_ex(1:required string cfName,
- 2:required string start_token,
- 3:required string end_token,
- 4:required i32 keys_per_split)
- throws (1:InvalidRequestException ire),
-
- /** adds a column family. returns the new schema id. */
- string system_add_column_family(1:required CfDef cf_def)
- throws (1:InvalidRequestException ire, 2:SchemaDisagreementException sde),
-
- /** drops a column family. returns the new schema id. */
- string system_drop_column_family(1:required string column_family)
- throws (1:InvalidRequestException ire, 2:SchemaDisagreementException sde),
-
- /** adds a keyspace and any column families that are part of it. returns the new schema id. */
- string system_add_keyspace(1:required KsDef ks_def)
- throws (1:InvalidRequestException ire, 2:SchemaDisagreementException sde),
-
- /** drops a keyspace and any column families that are part of it. returns the new schema id. */
- string system_drop_keyspace(1:required string keyspace)
- throws (1:InvalidRequestException ire, 2:SchemaDisagreementException sde),
-
- /** updates properties of a keyspace. returns the new schema id. */
- string system_update_keyspace(1:required KsDef ks_def)
- throws (1:InvalidRequestException ire, 2:SchemaDisagreementException sde),
-
- /** updates properties of a column family. returns the new schema id. */
- string system_update_column_family(1:required CfDef cf_def)
- throws (1:InvalidRequestException ire, 2:SchemaDisagreementException sde),
-
-
- /**
- * @deprecated Throws InvalidRequestException since 2.2. Please use the CQL3 version instead.
- */
- CqlResult execute_cql_query(1:required binary query, 2:required Compression compression)
- throws (1:InvalidRequestException ire,
- 2:UnavailableException ue,
- 3:TimedOutException te,
- 4:SchemaDisagreementException sde)
-
- /**
- * Executes a CQL3 (Cassandra Query Language) statement and returns a
- * CqlResult containing the results.
- */
- CqlResult execute_cql3_query(1:required binary query, 2:required Compression compression, 3:required ConsistencyLevel consistency)
- throws (1:InvalidRequestException ire,
- 2:UnavailableException ue,
- 3:TimedOutException te,
- 4:SchemaDisagreementException sde)
-
-
- /**
- * @deprecated Throws InvalidRequestException since 2.2. Please use the CQL3 version instead.
- */
- CqlPreparedResult prepare_cql_query(1:required binary query, 2:required Compression compression)
- throws (1:InvalidRequestException ire)
-
- /**
- * Prepare a CQL3 (Cassandra Query Language) statement by compiling and returning
- * - the type of CQL statement
- * - an id token of the compiled CQL stored on the server side.
- * - a count of the discovered bound markers in the statement
- */
- CqlPreparedResult prepare_cql3_query(1:required binary query, 2:required Compression compression)
- throws (1:InvalidRequestException ire)
-
-
- /**
- * @deprecated Throws InvalidRequestException since 2.2. Please use the CQL3 version instead.
- */
- CqlResult execute_prepared_cql_query(1:required i32 itemId, 2:required list<binary> values)
- throws (1:InvalidRequestException ire,
- 2:UnavailableException ue,
- 3:TimedOutException te,
- 4:SchemaDisagreementException sde)
-
- /**
- * Executes a prepared CQL3 (Cassandra Query Language) statement by passing an id token, a list of variables
- * to bind, and the consistency level, and returns a CqlResult containing the results.
- */
- CqlResult execute_prepared_cql3_query(1:required i32 itemId, 2:required list<binary> values, 3:required ConsistencyLevel consistency)
- throws (1:InvalidRequestException ire,
- 2:UnavailableException ue,
- 3:TimedOutException te,
- 4:SchemaDisagreementException sde)
-
- /**
- * @deprecated This is now a no-op. Please use the CQL3 specific methods instead.
- */
- void set_cql_version(1: required string version) throws (1:InvalidRequestException ire)
-}
diff --git a/interface/thrift/gen-java/org/apache/cassandra/thrift/AuthenticationException.java b/interface/thrift/gen-java/org/apache/cassandra/thrift/AuthenticationException.java
deleted file mode 100644
index 381b052..0000000
--- a/interface/thrift/gen-java/org/apache/cassandra/thrift/AuthenticationException.java
+++ /dev/null
@@ -1,400 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.1)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- * @generated
- */
-package org.apache.cassandra.thrift;
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
-
-import org.apache.commons.lang3.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.TException;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.EnumSet;
-import java.util.Collections;
-
-/**
- * invalid authentication request (invalid keyspace, user does not exist, or credentials invalid)
- */
-public class AuthenticationException extends TException implements org.apache.thrift.TBase<AuthenticationException, AuthenticationException._Fields>, java.io.Serializable, Cloneable, Comparable<AuthenticationException> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("AuthenticationException");
-
- private static final org.apache.thrift.protocol.TField WHY_FIELD_DESC = new org.apache.thrift.protocol.TField("why", org.apache.thrift.protocol.TType.STRING, (short)1);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new AuthenticationExceptionStandardSchemeFactory());
- schemes.put(TupleScheme.class, new AuthenticationExceptionTupleSchemeFactory());
- }
-
- public String why; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- WHY((short)1, "why");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // WHY
- return WHY;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.WHY, new org.apache.thrift.meta_data.FieldMetaData("why", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(AuthenticationException.class, metaDataMap);
- }
-
- public AuthenticationException() {
- }
-
- public AuthenticationException(
- String why)
- {
- this();
- this.why = why;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public AuthenticationException(AuthenticationException other) {
- if (other.isSetWhy()) {
- this.why = other.why;
- }
- }
-
- public AuthenticationException deepCopy() {
- return new AuthenticationException(this);
- }
-
- @Override
- public void clear() {
- this.why = null;
- }
-
- public String getWhy() {
- return this.why;
- }
-
- public AuthenticationException setWhy(String why) {
- this.why = why;
- return this;
- }
-
- public void unsetWhy() {
- this.why = null;
- }
-
- /** Returns true if field why is set (has been assigned a value) and false otherwise */
- public boolean isSetWhy() {
- return this.why != null;
- }
-
- public void setWhyIsSet(boolean value) {
- if (!value) {
- this.why = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case WHY:
- if (value == null) {
- unsetWhy();
- } else {
- setWhy((String)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case WHY:
- return getWhy();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case WHY:
- return isSetWhy();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof AuthenticationException)
- return this.equals((AuthenticationException)that);
- return false;
- }
-
- public boolean equals(AuthenticationException that) {
- if (that == null)
- return false;
-
- boolean this_present_why = true && this.isSetWhy();
- boolean that_present_why = true && that.isSetWhy();
- if (this_present_why || that_present_why) {
- if (!(this_present_why && that_present_why))
- return false;
- if (!this.why.equals(that.why))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_why = true && (isSetWhy());
- builder.append(present_why);
- if (present_why)
- builder.append(why);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(AuthenticationException other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetWhy()).compareTo(other.isSetWhy());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetWhy()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.why, other.why);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("AuthenticationException(");
- boolean first = true;
-
- sb.append("why:");
- if (this.why == null) {
- sb.append("null");
- } else {
- sb.append(this.why);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- if (why == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'why' was not present! Struct: " + toString());
- }
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class AuthenticationExceptionStandardSchemeFactory implements SchemeFactory {
- public AuthenticationExceptionStandardScheme getScheme() {
- return new AuthenticationExceptionStandardScheme();
- }
- }
-
- private static class AuthenticationExceptionStandardScheme extends StandardScheme<AuthenticationException> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, AuthenticationException struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // WHY
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.why = iprot.readString();
- struct.setWhyIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, AuthenticationException struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.why != null) {
- oprot.writeFieldBegin(WHY_FIELD_DESC);
- oprot.writeString(struct.why);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class AuthenticationExceptionTupleSchemeFactory implements SchemeFactory {
- public AuthenticationExceptionTupleScheme getScheme() {
- return new AuthenticationExceptionTupleScheme();
- }
- }
-
- private static class AuthenticationExceptionTupleScheme extends TupleScheme<AuthenticationException> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, AuthenticationException struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- oprot.writeString(struct.why);
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, AuthenticationException struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- struct.why = iprot.readString();
- struct.setWhyIsSet(true);
- }
- }
-
-}
-
diff --git a/interface/thrift/gen-java/org/apache/cassandra/thrift/AuthenticationRequest.java b/interface/thrift/gen-java/org/apache/cassandra/thrift/AuthenticationRequest.java
deleted file mode 100644
index 5778fa5..0000000
--- a/interface/thrift/gen-java/org/apache/cassandra/thrift/AuthenticationRequest.java
+++ /dev/null
@@ -1,465 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.1)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- * @generated
- */
-package org.apache.cassandra.thrift;
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
-
-import org.apache.commons.lang3.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Authentication requests can contain any data, dependent on the IAuthenticator used
- */
-public class AuthenticationRequest implements org.apache.thrift.TBase<AuthenticationRequest, AuthenticationRequest._Fields>, java.io.Serializable, Cloneable, Comparable<AuthenticationRequest> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("AuthenticationRequest");
-
- private static final org.apache.thrift.protocol.TField CREDENTIALS_FIELD_DESC = new org.apache.thrift.protocol.TField("credentials", org.apache.thrift.protocol.TType.MAP, (short)1);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new AuthenticationRequestStandardSchemeFactory());
- schemes.put(TupleScheme.class, new AuthenticationRequestTupleSchemeFactory());
- }
-
- public Map<String,String> credentials; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- CREDENTIALS((short)1, "credentials");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // CREDENTIALS
- return CREDENTIALS;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.CREDENTIALS, new org.apache.thrift.meta_data.FieldMetaData("credentials", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING),
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(AuthenticationRequest.class, metaDataMap);
- }
-
- public AuthenticationRequest() {
- }
-
- public AuthenticationRequest(
- Map<String,String> credentials)
- {
- this();
- this.credentials = credentials;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public AuthenticationRequest(AuthenticationRequest other) {
- if (other.isSetCredentials()) {
- Map<String,String> __this__credentials = new HashMap<String,String>(other.credentials);
- this.credentials = __this__credentials;
- }
- }
-
- public AuthenticationRequest deepCopy() {
- return new AuthenticationRequest(this);
- }
-
- @Override
- public void clear() {
- this.credentials = null;
- }
-
- public int getCredentialsSize() {
- return (this.credentials == null) ? 0 : this.credentials.size();
- }
-
- public void putToCredentials(String key, String val) {
- if (this.credentials == null) {
- this.credentials = new HashMap<String,String>();
- }
- this.credentials.put(key, val);
- }
-
- public Map<String,String> getCredentials() {
- return this.credentials;
- }
-
- public AuthenticationRequest setCredentials(Map<String,String> credentials) {
- this.credentials = credentials;
- return this;
- }
-
- public void unsetCredentials() {
- this.credentials = null;
- }
-
- /** Returns true if field credentials is set (has been assigned a value) and false otherwise */
- public boolean isSetCredentials() {
- return this.credentials != null;
- }
-
- public void setCredentialsIsSet(boolean value) {
- if (!value) {
- this.credentials = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case CREDENTIALS:
- if (value == null) {
- unsetCredentials();
- } else {
- setCredentials((Map<String,String>)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case CREDENTIALS:
- return getCredentials();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case CREDENTIALS:
- return isSetCredentials();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof AuthenticationRequest)
- return this.equals((AuthenticationRequest)that);
- return false;
- }
-
- public boolean equals(AuthenticationRequest that) {
- if (that == null)
- return false;
-
- boolean this_present_credentials = true && this.isSetCredentials();
- boolean that_present_credentials = true && that.isSetCredentials();
- if (this_present_credentials || that_present_credentials) {
- if (!(this_present_credentials && that_present_credentials))
- return false;
- if (!this.credentials.equals(that.credentials))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_credentials = true && (isSetCredentials());
- builder.append(present_credentials);
- if (present_credentials)
- builder.append(credentials);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(AuthenticationRequest other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetCredentials()).compareTo(other.isSetCredentials());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetCredentials()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.credentials, other.credentials);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("AuthenticationRequest(");
- boolean first = true;
-
- sb.append("credentials:");
- if (this.credentials == null) {
- sb.append("null");
- } else {
- sb.append(this.credentials);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- if (credentials == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'credentials' was not present! Struct: " + toString());
- }
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class AuthenticationRequestStandardSchemeFactory implements SchemeFactory {
- public AuthenticationRequestStandardScheme getScheme() {
- return new AuthenticationRequestStandardScheme();
- }
- }
-
- private static class AuthenticationRequestStandardScheme extends StandardScheme<AuthenticationRequest> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, AuthenticationRequest struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // CREDENTIALS
- if (schemeField.type == org.apache.thrift.protocol.TType.MAP) {
- {
- org.apache.thrift.protocol.TMap _map80 = iprot.readMapBegin();
- struct.credentials = new HashMap<String,String>(2*_map80.size);
- for (int _i81 = 0; _i81 < _map80.size; ++_i81)
- {
- String _key82;
- String _val83;
- _key82 = iprot.readString();
- _val83 = iprot.readString();
- struct.credentials.put(_key82, _val83);
- }
- iprot.readMapEnd();
- }
- struct.setCredentialsIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, AuthenticationRequest struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.credentials != null) {
- oprot.writeFieldBegin(CREDENTIALS_FIELD_DESC);
- {
- oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, struct.credentials.size()));
- for (Map.Entry<String, String> _iter84 : struct.credentials.entrySet())
- {
- oprot.writeString(_iter84.getKey());
- oprot.writeString(_iter84.getValue());
- }
- oprot.writeMapEnd();
- }
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class AuthenticationRequestTupleSchemeFactory implements SchemeFactory {
- public AuthenticationRequestTupleScheme getScheme() {
- return new AuthenticationRequestTupleScheme();
- }
- }
-
- private static class AuthenticationRequestTupleScheme extends TupleScheme<AuthenticationRequest> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, AuthenticationRequest struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- {
- oprot.writeI32(struct.credentials.size());
- for (Map.Entry<String, String> _iter85 : struct.credentials.entrySet())
- {
- oprot.writeString(_iter85.getKey());
- oprot.writeString(_iter85.getValue());
- }
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, AuthenticationRequest struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- {
- org.apache.thrift.protocol.TMap _map86 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32());
- struct.credentials = new HashMap<String,String>(2*_map86.size);
- for (int _i87 = 0; _i87 < _map86.size; ++_i87)
- {
- String _key88;
- String _val89;
- _key88 = iprot.readString();
- _val89 = iprot.readString();
- struct.credentials.put(_key88, _val89);
- }
- }
- struct.setCredentialsIsSet(true);
- }
- }
-
-}
-
diff --git a/interface/thrift/gen-java/org/apache/cassandra/thrift/AuthorizationException.java b/interface/thrift/gen-java/org/apache/cassandra/thrift/AuthorizationException.java
deleted file mode 100644
index cd1bdf7..0000000
--- a/interface/thrift/gen-java/org/apache/cassandra/thrift/AuthorizationException.java
+++ /dev/null
@@ -1,413 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.1)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- * @generated
- */
-package org.apache.cassandra.thrift;
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
-
-import org.apache.commons.lang3.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * invalid authorization request (user does not have access to keyspace)
- */
-public class AuthorizationException extends TException implements org.apache.thrift.TBase<AuthorizationException, AuthorizationException._Fields>, java.io.Serializable, Cloneable, Comparable<AuthorizationException> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("AuthorizationException");
-
- private static final org.apache.thrift.protocol.TField WHY_FIELD_DESC = new org.apache.thrift.protocol.TField("why", org.apache.thrift.protocol.TType.STRING, (short)1);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new AuthorizationExceptionStandardSchemeFactory());
- schemes.put(TupleScheme.class, new AuthorizationExceptionTupleSchemeFactory());
- }
-
- public String why; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- WHY((short)1, "why");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // WHY
- return WHY;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.WHY, new org.apache.thrift.meta_data.FieldMetaData("why", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(AuthorizationException.class, metaDataMap);
- }
-
- public AuthorizationException() {
- }
-
- public AuthorizationException(
- String why)
- {
- this();
- this.why = why;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public AuthorizationException(AuthorizationException other) {
- if (other.isSetWhy()) {
- this.why = other.why;
- }
- }
-
- public AuthorizationException deepCopy() {
- return new AuthorizationException(this);
- }
-
- @Override
- public void clear() {
- this.why = null;
- }
-
- public String getWhy() {
- return this.why;
- }
-
- public AuthorizationException setWhy(String why) {
- this.why = why;
- return this;
- }
-
- public void unsetWhy() {
- this.why = null;
- }
-
- /** Returns true if field why is set (has been assigned a value) and false otherwise */
- public boolean isSetWhy() {
- return this.why != null;
- }
-
- public void setWhyIsSet(boolean value) {
- if (!value) {
- this.why = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case WHY:
- if (value == null) {
- unsetWhy();
- } else {
- setWhy((String)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case WHY:
- return getWhy();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case WHY:
- return isSetWhy();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof AuthorizationException)
- return this.equals((AuthorizationException)that);
- return false;
- }
-
- public boolean equals(AuthorizationException that) {
- if (that == null)
- return false;
-
- boolean this_present_why = true && this.isSetWhy();
- boolean that_present_why = true && that.isSetWhy();
- if (this_present_why || that_present_why) {
- if (!(this_present_why && that_present_why))
- return false;
- if (!this.why.equals(that.why))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_why = true && (isSetWhy());
- builder.append(present_why);
- if (present_why)
- builder.append(why);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(AuthorizationException other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetWhy()).compareTo(other.isSetWhy());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetWhy()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.why, other.why);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("AuthorizationException(");
- boolean first = true;
-
- sb.append("why:");
- if (this.why == null) {
- sb.append("null");
- } else {
- sb.append(this.why);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- if (why == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'why' was not present! Struct: " + toString());
- }
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class AuthorizationExceptionStandardSchemeFactory implements SchemeFactory {
- public AuthorizationExceptionStandardScheme getScheme() {
- return new AuthorizationExceptionStandardScheme();
- }
- }
-
- private static class AuthorizationExceptionStandardScheme extends StandardScheme<AuthorizationException> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, AuthorizationException struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // WHY
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.why = iprot.readString();
- struct.setWhyIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, AuthorizationException struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.why != null) {
- oprot.writeFieldBegin(WHY_FIELD_DESC);
- oprot.writeString(struct.why);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class AuthorizationExceptionTupleSchemeFactory implements SchemeFactory {
- public AuthorizationExceptionTupleScheme getScheme() {
- return new AuthorizationExceptionTupleScheme();
- }
- }
-
- private static class AuthorizationExceptionTupleScheme extends TupleScheme<AuthorizationException> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, AuthorizationException struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- oprot.writeString(struct.why);
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, AuthorizationException struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- struct.why = iprot.readString();
- struct.setWhyIsSet(true);
- }
- }
-
-}
-
diff --git a/interface/thrift/gen-java/org/apache/cassandra/thrift/CASResult.java b/interface/thrift/gen-java/org/apache/cassandra/thrift/CASResult.java
deleted file mode 100644
index 4d21bfe..0000000
--- a/interface/thrift/gen-java/org/apache/cassandra/thrift/CASResult.java
+++ /dev/null
@@ -1,574 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.1)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- * @generated
- */
-package org.apache.cassandra.thrift;
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
-
-import org.apache.commons.lang3.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class CASResult implements org.apache.thrift.TBase<CASResult, CASResult._Fields>, java.io.Serializable, Cloneable, Comparable<CASResult> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("CASResult");
-
- private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.BOOL, (short)1);
- private static final org.apache.thrift.protocol.TField CURRENT_VALUES_FIELD_DESC = new org.apache.thrift.protocol.TField("current_values", org.apache.thrift.protocol.TType.LIST, (short)2);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new CASResultStandardSchemeFactory());
- schemes.put(TupleScheme.class, new CASResultTupleSchemeFactory());
- }
-
- public boolean success; // required
- public List<Column> current_values; // optional
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- SUCCESS((short)1, "success"),
- CURRENT_VALUES((short)2, "current_values");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // SUCCESS
- return SUCCESS;
- case 2: // CURRENT_VALUES
- return CURRENT_VALUES;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- private static final int __SUCCESS_ISSET_ID = 0;
- private byte __isset_bitfield = 0;
- private _Fields optionals[] = {_Fields.CURRENT_VALUES};
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.BOOL)));
- tmpMap.put(_Fields.CURRENT_VALUES, new org.apache.thrift.meta_data.FieldMetaData("current_values", org.apache.thrift.TFieldRequirementType.OPTIONAL,
- new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST,
- new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, Column.class))));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(CASResult.class, metaDataMap);
- }
-
- public CASResult() {
- }
-
- public CASResult(
- boolean success)
- {
- this();
- this.success = success;
- setSuccessIsSet(true);
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public CASResult(CASResult other) {
- __isset_bitfield = other.__isset_bitfield;
- this.success = other.success;
- if (other.isSetCurrent_values()) {
- List<Column> __this__current_values = new ArrayList<Column>(other.current_values.size());
- for (Column other_element : other.current_values) {
- __this__current_values.add(new Column(other_element));
- }
- this.current_values = __this__current_values;
- }
- }
-
- public CASResult deepCopy() {
- return new CASResult(this);
- }
-
- @Override
- public void clear() {
- setSuccessIsSet(false);
- this.success = false;
- this.current_values = null;
- }
-
- public boolean isSuccess() {
- return this.success;
- }
-
- public CASResult setSuccess(boolean success) {
- this.success = success;
- setSuccessIsSet(true);
- return this;
- }
-
- public void unsetSuccess() {
- __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __SUCCESS_ISSET_ID);
- }
-
- /** Returns true if field success is set (has been assigned a value) and false otherwise */
- public boolean isSetSuccess() {
- return EncodingUtils.testBit(__isset_bitfield, __SUCCESS_ISSET_ID);
- }
-
- public void setSuccessIsSet(boolean value) {
- __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __SUCCESS_ISSET_ID, value);
- }
-
- public int getCurrent_valuesSize() {
- return (this.current_values == null) ? 0 : this.current_values.size();
- }
-
- public java.util.Iterator<Column> getCurrent_valuesIterator() {
- return (this.current_values == null) ? null : this.current_values.iterator();
- }
-
- public void addToCurrent_values(Column elem) {
- if (this.current_values == null) {
- this.current_values = new ArrayList<Column>();
- }
- this.current_values.add(elem);
- }
-
- public List<Column> getCurrent_values() {
- return this.current_values;
- }
-
- public CASResult setCurrent_values(List<Column> current_values) {
- this.current_values = current_values;
- return this;
- }
-
- public void unsetCurrent_values() {
- this.current_values = null;
- }
-
- /** Returns true if field current_values is set (has been assigned a value) and false otherwise */
- public boolean isSetCurrent_values() {
- return this.current_values != null;
- }
-
- public void setCurrent_valuesIsSet(boolean value) {
- if (!value) {
- this.current_values = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case SUCCESS:
- if (value == null) {
- unsetSuccess();
- } else {
- setSuccess((Boolean)value);
- }
- break;
-
- case CURRENT_VALUES:
- if (value == null) {
- unsetCurrent_values();
- } else {
- setCurrent_values((List<Column>)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case SUCCESS:
- return Boolean.valueOf(isSuccess());
-
- case CURRENT_VALUES:
- return getCurrent_values();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case SUCCESS:
- return isSetSuccess();
- case CURRENT_VALUES:
- return isSetCurrent_values();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof CASResult)
- return this.equals((CASResult)that);
- return false;
- }
-
- public boolean equals(CASResult that) {
- if (that == null)
- return false;
-
- boolean this_present_success = true;
- boolean that_present_success = true;
- if (this_present_success || that_present_success) {
- if (!(this_present_success && that_present_success))
- return false;
- if (this.success != that.success)
- return false;
- }
-
- boolean this_present_current_values = true && this.isSetCurrent_values();
- boolean that_present_current_values = true && that.isSetCurrent_values();
- if (this_present_current_values || that_present_current_values) {
- if (!(this_present_current_values && that_present_current_values))
- return false;
- if (!this.current_values.equals(that.current_values))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_success = true;
- builder.append(present_success);
- if (present_success)
- builder.append(success);
-
- boolean present_current_values = true && (isSetCurrent_values());
- builder.append(present_current_values);
- if (present_current_values)
- builder.append(current_values);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(CASResult other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(other.isSetSuccess());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetSuccess()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, other.success);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetCurrent_values()).compareTo(other.isSetCurrent_values());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetCurrent_values()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.current_values, other.current_values);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("CASResult(");
- boolean first = true;
-
- sb.append("success:");
- sb.append(this.success);
- first = false;
- if (isSetCurrent_values()) {
- if (!first) sb.append(", ");
- sb.append("current_values:");
- if (this.current_values == null) {
- sb.append("null");
- } else {
- sb.append(this.current_values);
- }
- first = false;
- }
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // alas, we cannot check 'success' because it's a primitive and you chose the non-beans generator.
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
- __isset_bitfield = 0;
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class CASResultStandardSchemeFactory implements SchemeFactory {
- public CASResultStandardScheme getScheme() {
- return new CASResultStandardScheme();
- }
- }
-
- private static class CASResultStandardScheme extends StandardScheme<CASResult> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, CASResult struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // SUCCESS
- if (schemeField.type == org.apache.thrift.protocol.TType.BOOL) {
- struct.success = iprot.readBool();
- struct.setSuccessIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 2: // CURRENT_VALUES
- if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
- {
- org.apache.thrift.protocol.TList _list48 = iprot.readListBegin();
- struct.current_values = new ArrayList<Column>(_list48.size);
- for (int _i49 = 0; _i49 < _list48.size; ++_i49)
- {
- Column _elem50;
- _elem50 = new Column();
- _elem50.read(iprot);
- struct.current_values.add(_elem50);
- }
- iprot.readListEnd();
- }
- struct.setCurrent_valuesIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- if (!struct.isSetSuccess()) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'success' was not found in serialized data! Struct: " + toString());
- }
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, CASResult struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
- oprot.writeBool(struct.success);
- oprot.writeFieldEnd();
- if (struct.current_values != null) {
- if (struct.isSetCurrent_values()) {
- oprot.writeFieldBegin(CURRENT_VALUES_FIELD_DESC);
- {
- oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.current_values.size()));
- for (Column _iter51 : struct.current_values)
- {
- _iter51.write(oprot);
- }
- oprot.writeListEnd();
- }
- oprot.writeFieldEnd();
- }
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class CASResultTupleSchemeFactory implements SchemeFactory {
- public CASResultTupleScheme getScheme() {
- return new CASResultTupleScheme();
- }
- }
-
- private static class CASResultTupleScheme extends TupleScheme<CASResult> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, CASResult struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- oprot.writeBool(struct.success);
- BitSet optionals = new BitSet();
- if (struct.isSetCurrent_values()) {
- optionals.set(0);
- }
- oprot.writeBitSet(optionals, 1);
- if (struct.isSetCurrent_values()) {
- {
- oprot.writeI32(struct.current_values.size());
- for (Column _iter52 : struct.current_values)
- {
- _iter52.write(oprot);
- }
- }
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, CASResult struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- struct.success = iprot.readBool();
- struct.setSuccessIsSet(true);
- BitSet incoming = iprot.readBitSet(1);
- if (incoming.get(0)) {
- {
- org.apache.thrift.protocol.TList _list53 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
- struct.current_values = new ArrayList<Column>(_list53.size);
- for (int _i54 = 0; _i54 < _list53.size; ++_i54)
- {
- Column _elem55;
- _elem55 = new Column();
- _elem55.read(iprot);
- struct.current_values.add(_elem55);
- }
- }
- struct.setCurrent_valuesIsSet(true);
- }
- }
- }
-
-}
-
diff --git a/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java b/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java
deleted file mode 100644
index cd4314b..0000000
--- a/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java
+++ /dev/null
@@ -1,55794 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.1)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- * @generated
- */
-package org.apache.cassandra.thrift;
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
-
-import org.apache.commons.lang3.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class Cassandra {
-
- public interface Iface {
-
- public void login(AuthenticationRequest auth_request) throws AuthenticationException, AuthorizationException, org.apache.thrift.TException;
-
- public void set_keyspace(String keyspace) throws InvalidRequestException, org.apache.thrift.TException;
-
- /**
- * Get the Column or SuperColumn at the given column_path. If no value is present, NotFoundException is thrown. (This is
- * the only method that can throw an exception under non-failure conditions.)
- *
- * @param key
- * @param column_path
- * @param consistency_level
- */
- public ColumnOrSuperColumn get(ByteBuffer key, ColumnPath column_path, ConsistencyLevel consistency_level) throws InvalidRequestException, NotFoundException, UnavailableException, TimedOutException, org.apache.thrift.TException;
-
- /**
- * Get the group of columns contained by column_parent (either a ColumnFamily name or a ColumnFamily/SuperColumn name
- * pair) specified by the given SlicePredicate. If no matching values are found, an empty list is returned.
- *
- * @param key
- * @param column_parent
- * @param predicate
- * @param consistency_level
- */
- public List<ColumnOrSuperColumn> get_slice(ByteBuffer key, ColumnParent column_parent, SlicePredicate predicate, ConsistencyLevel consistency_level) throws InvalidRequestException, UnavailableException, TimedOutException, org.apache.thrift.TException;
-
- /**
- * returns the number of columns matching <code>predicate</code> for a particular <code>key</code>,
- * <code>ColumnFamily</code> and optionally <code>SuperColumn</code>.
- *
- * @param key
- * @param column_parent
- * @param predicate
- * @param consistency_level
- */
- public int get_count(ByteBuffer key, ColumnParent column_parent, SlicePredicate predicate, ConsistencyLevel consistency_level) throws InvalidRequestException, UnavailableException, TimedOutException, org.apache.thrift.TException;
-
- /**
- * Performs a get_slice for column_parent and predicate for the given keys in parallel.
- *
- * @param keys
- * @param column_parent
- * @param predicate
- * @param consistency_level
- */
- public Map<ByteBuffer,List<ColumnOrSuperColumn>> multiget_slice(List<ByteBuffer> keys, ColumnParent column_parent, SlicePredicate predicate, ConsistencyLevel consistency_level) throws InvalidRequestException, UnavailableException, TimedOutException, org.apache.thrift.TException;
-
- /**
- * Perform a get_count in parallel on the given list<binary> keys. The return value maps keys to the count found.
- *
- * @param keys
- * @param column_parent
- * @param predicate
- * @param consistency_level
- */
- public Map<ByteBuffer,Integer> multiget_count(List<ByteBuffer> keys, ColumnParent column_parent, SlicePredicate predicate, ConsistencyLevel consistency_level) throws InvalidRequestException, UnavailableException, TimedOutException, org.apache.thrift.TException;
-
- /**
- * returns a subset of columns for a contiguous range of keys.
- *
- * @param column_parent
- * @param predicate
- * @param range
- * @param consistency_level
- */
- public List<KeySlice> get_range_slices(ColumnParent column_parent, SlicePredicate predicate, KeyRange range, ConsistencyLevel consistency_level) throws InvalidRequestException, UnavailableException, TimedOutException, org.apache.thrift.TException;
-
- /**
- * returns a range of columns, wrapping to the next rows if necessary to collect max_results.
- *
- * @param column_family
- * @param range
- * @param start_column
- * @param consistency_level
- */
- public List<KeySlice> get_paged_slice(String column_family, KeyRange range, ByteBuffer start_column, ConsistencyLevel consistency_level) throws InvalidRequestException, UnavailableException, TimedOutException, org.apache.thrift.TException;
-
- /**
- * Returns the subset of columns specified in SlicePredicate for the rows matching the IndexClause
- * @deprecated use get_range_slices instead with range.row_filter specified
- *
- * @param column_parent
- * @param index_clause
- * @param column_predicate
- * @param consistency_level
- */
- public List<KeySlice> get_indexed_slices(ColumnParent column_parent, IndexClause index_clause, SlicePredicate column_predicate, ConsistencyLevel consistency_level) throws InvalidRequestException, UnavailableException, TimedOutException, org.apache.thrift.TException;
-
- /**
- * Insert a Column at the given column_parent.column_family and optional column_parent.super_column.
- *
- * @param key
- * @param column_parent
- * @param column
- * @param consistency_level
- */
- public void insert(ByteBuffer key, ColumnParent column_parent, Column column, ConsistencyLevel consistency_level) throws InvalidRequestException, UnavailableException, TimedOutException, org.apache.thrift.TException;
-
- /**
- * Increment or decrement a counter.
- *
- * @param key
- * @param column_parent
- * @param column
- * @param consistency_level
- */
- public void add(ByteBuffer key, ColumnParent column_parent, CounterColumn column, ConsistencyLevel consistency_level) throws InvalidRequestException, UnavailableException, TimedOutException, org.apache.thrift.TException;
-
- /**
- * Atomic compare and set.
- *
- * If the cas is successfull, the success boolean in CASResult will be true and there will be no current_values.
- * Otherwise, success will be false and current_values will contain the current values for the columns in
- * expected (that, by definition of compare-and-set, will differ from the values in expected).
- *
- * A cas operation takes 2 consistency level. The first one, serial_consistency_level, simply indicates the
- * level of serialization required. This can be either ConsistencyLevel.SERIAL or ConsistencyLevel.LOCAL_SERIAL.
- * The second one, commit_consistency_level, defines the consistency level for the commit phase of the cas. This
- * is a more traditional consistency level (the same CL than for traditional writes are accepted) that impact
- * the visibility for reads of the operation. For instance, if commit_consistency_level is QUORUM, then it is
- * guaranteed that a followup QUORUM read will see the cas write (if that one was successful obviously). If
- * commit_consistency_level is ANY, you will need to use a SERIAL/LOCAL_SERIAL read to be guaranteed to see
- * the write.
- *
- * @param key
- * @param column_family
- * @param expected
- * @param updates
- * @param serial_consistency_level
- * @param commit_consistency_level
- */
- public CASResult cas(ByteBuffer key, String column_family, List<Column> expected, List<Column> updates, ConsistencyLevel serial_consistency_level, ConsistencyLevel commit_consistency_level) throws InvalidRequestException, UnavailableException, TimedOutException, org.apache.thrift.TException;
-
- /**
- * Remove data from the row specified by key at the granularity specified by column_path, and the given timestamp. Note
- * that all the values in column_path besides column_path.column_family are truly optional: you can remove the entire
- * row by just specifying the ColumnFamily, or you can remove a SuperColumn or a single Column by specifying those levels too.
- *
- * @param key
- * @param column_path
- * @param timestamp
- * @param consistency_level
- */
- public void remove(ByteBuffer key, ColumnPath column_path, long timestamp, ConsistencyLevel consistency_level) throws InvalidRequestException, UnavailableException, TimedOutException, org.apache.thrift.TException;
-
- /**
- * Remove a counter at the specified location.
- * Note that counters have limited support for deletes: if you remove a counter, you must wait to issue any following update
- * until the delete has reached all the nodes and all of them have been fully compacted.
- *
- * @param key
- * @param path
- * @param consistency_level
- */
- public void remove_counter(ByteBuffer key, ColumnPath path, ConsistencyLevel consistency_level) throws InvalidRequestException, UnavailableException, TimedOutException, org.apache.thrift.TException;
-
- /**
- * Mutate many columns or super columns for many row keys. See also: Mutation.
- *
- * mutation_map maps key to column family to a list of Mutation objects to take place at that scope.
- * *
- *
- * @param mutation_map
- * @param consistency_level
- */
- public void batch_mutate(Map<ByteBuffer,Map<String,List<Mutation>>> mutation_map, ConsistencyLevel consistency_level) throws InvalidRequestException, UnavailableException, TimedOutException, org.apache.thrift.TException;
-
- /**
- * Atomically mutate many columns or super columns for many row keys. See also: Mutation.
- *
- * mutation_map maps key to column family to a list of Mutation objects to take place at that scope.
- * *
- *
- * @param mutation_map
- * @param consistency_level
- */
- public void atomic_batch_mutate(Map<ByteBuffer,Map<String,List<Mutation>>> mutation_map, ConsistencyLevel consistency_level) throws InvalidRequestException, UnavailableException, TimedOutException, org.apache.thrift.TException;
-
- /**
- * Truncate will mark and entire column family as deleted.
- * From the user's perspective a successful call to truncate will result complete data deletion from cfname.
- * Internally, however, disk space will not be immediatily released, as with all deletes in cassandra, this one
- * only marks the data as deleted.
- * The operation succeeds only if all hosts in the cluster at available and will throw an UnavailableException if
- * some hosts are down.
- *
- * @param cfname
- */
- public void truncate(String cfname) throws InvalidRequestException, UnavailableException, TimedOutException, org.apache.thrift.TException;
-
- /**
- * Select multiple slices of a key in a single RPC operation
- *
- * @param request
- */
- public List<ColumnOrSuperColumn> get_multi_slice(MultiSliceRequest request) throws InvalidRequestException, UnavailableException, TimedOutException, org.apache.thrift.TException;
-
- /**
- * for each schema version present in the cluster, returns a list of nodes at that version.
- * hosts that do not respond will be under the key DatabaseDescriptor.INITIAL_VERSION.
- * the cluster is all on the same version if the size of the map is 1.
- */
- public Map<String,List<String>> describe_schema_versions() throws InvalidRequestException, org.apache.thrift.TException;
-
- /**
- * list the defined keyspaces in this cluster
- */
- public List<KsDef> describe_keyspaces() throws InvalidRequestException, org.apache.thrift.TException;
-
- /**
- * get the cluster name
- */
- public String describe_cluster_name() throws org.apache.thrift.TException;
-
- /**
- * get the thrift api version
- */
- public String describe_version() throws org.apache.thrift.TException;
-
- /**
- * get the token ring: a map of ranges to host addresses,
- * represented as a set of TokenRange instead of a map from range
- * to list of endpoints, because you can't use Thrift structs as
- * map keys:
- * https://issues.apache.org/jira/browse/THRIFT-162
- *
- * for the same reason, we can't return a set here, even though
- * order is neither important nor predictable.
- *
- * @param keyspace
- */
- public List<TokenRange> describe_ring(String keyspace) throws InvalidRequestException, org.apache.thrift.TException;
-
- /**
- * same as describe_ring, but considers only nodes in the local DC
- *
- * @param keyspace
- */
- public List<TokenRange> describe_local_ring(String keyspace) throws InvalidRequestException, org.apache.thrift.TException;
-
- /**
- * get the mapping between token->node ip
- * without taking replication into consideration
- * https://issues.apache.org/jira/browse/CASSANDRA-4092
- */
- public Map<String,String> describe_token_map() throws InvalidRequestException, org.apache.thrift.TException;
-
- /**
- * returns the partitioner used by this cluster
- */
- public String describe_partitioner() throws org.apache.thrift.TException;
-
- /**
- * returns the snitch used by this cluster
- */
- public String describe_snitch() throws org.apache.thrift.TException;
-
- /**
- * describe specified keyspace
- *
- * @param keyspace
- */
- public KsDef describe_keyspace(String keyspace) throws NotFoundException, InvalidRequestException, org.apache.thrift.TException;
-
- /**
- * experimental API for hadoop/parallel query support.
- * may change violently and without warning.
- *
- * returns list of token strings such that first subrange is (list[0], list[1]],
- * next is (list[1], list[2]], etc.
- *
- * @param cfName
- * @param start_token
- * @param end_token
- * @param keys_per_split
- */
- public List<String> describe_splits(String cfName, String start_token, String end_token, int keys_per_split) throws InvalidRequestException, org.apache.thrift.TException;
-
- /**
- * Enables tracing for the next query in this connection and returns the UUID for that trace session
- * The next query will be traced idependently of trace probability and the returned UUID can be used to query the trace keyspace
- */
- public ByteBuffer trace_next_query() throws org.apache.thrift.TException;
-
- public List<CfSplit> describe_splits_ex(String cfName, String start_token, String end_token, int keys_per_split) throws InvalidRequestException, org.apache.thrift.TException;
-
- /**
- * adds a column family. returns the new schema id.
- *
- * @param cf_def
- */
- public String system_add_column_family(CfDef cf_def) throws InvalidRequestException, SchemaDisagreementException, org.apache.thrift.TException;
-
- /**
- * drops a column family. returns the new schema id.
- *
- * @param column_family
- */
- public String system_drop_column_family(String column_family) throws InvalidRequestException, SchemaDisagreementException, org.apache.thrift.TException;
-
- /**
- * adds a keyspace and any column families that are part of it. returns the new schema id.
- *
- * @param ks_def
- */
- public String system_add_keyspace(KsDef ks_def) throws InvalidRequestException, SchemaDisagreementException, org.apache.thrift.TException;
-
- /**
- * drops a keyspace and any column families that are part of it. returns the new schema id.
- *
- * @param keyspace
- */
- public String system_drop_keyspace(String keyspace) throws InvalidRequestException, SchemaDisagreementException, org.apache.thrift.TException;
-
- /**
- * updates properties of a keyspace. returns the new schema id.
- *
- * @param ks_def
- */
- public String system_update_keyspace(KsDef ks_def) throws InvalidRequestException, SchemaDisagreementException, org.apache.thrift.TException;
-
- /**
- * updates properties of a column family. returns the new schema id.
- *
- * @param cf_def
- */
- public String system_update_column_family(CfDef cf_def) throws InvalidRequestException, SchemaDisagreementException, org.apache.thrift.TException;
-
- /**
- * @deprecated Throws InvalidRequestException since 2.2. Please use the CQL3 version instead.
- *
- * @param query
- * @param compression
- */
- public CqlResult execute_cql_query(ByteBuffer query, Compression compression) throws InvalidRequestException, UnavailableException, TimedOutException, SchemaDisagreementException, org.apache.thrift.TException;
-
- /**
- * Executes a CQL3 (Cassandra Query Language) statement and returns a
- * CqlResult containing the results.
- *
- * @param query
- * @param compression
- * @param consistency
- */
- public CqlResult execute_cql3_query(ByteBuffer query, Compression compression, ConsistencyLevel consistency) throws InvalidRequestException, UnavailableException, TimedOutException, SchemaDisagreementException, org.apache.thrift.TException;
-
- /**
- * @deprecated Throws InvalidRequestException since 2.2. Please use the CQL3 version instead.
- *
- * @param query
- * @param compression
- */
- public CqlPreparedResult prepare_cql_query(ByteBuffer query, Compression compression) throws InvalidRequestException, org.apache.thrift.TException;
-
- /**
- * Prepare a CQL3 (Cassandra Query Language) statement by compiling and returning
- * - the type of CQL statement
- * - an id token of the compiled CQL stored on the server side.
- * - a count of the discovered bound markers in the statement
- *
- * @param query
- * @param compression
- */
- public CqlPreparedResult prepare_cql3_query(ByteBuffer query, Compression compression) throws InvalidRequestException, org.apache.thrift.TException;
-
- /**
- * @deprecated Throws InvalidRequestException since 2.2. Please use the CQL3 version instead.
- *
- * @param itemId
- * @param values
- */
- public CqlResult execute_prepared_cql_query(int itemId, List<ByteBuffer> values) throws InvalidRequestException, UnavailableException, TimedOutException, SchemaDisagreementException, org.apache.thrift.TException;
-
- /**
- * Executes a prepared CQL3 (Cassandra Query Language) statement by passing an id token, a list of variables
- * to bind, and the consistency level, and returns a CqlResult containing the results.
- *
- * @param itemId
- * @param values
- * @param consistency
- */
- public CqlResult execute_prepared_cql3_query(int itemId, List<ByteBuffer> values, ConsistencyLevel consistency) throws InvalidRequestException, UnavailableException, TimedOutException, SchemaDisagreementException, org.apache.thrift.TException;
-
- /**
- * @deprecated This is now a no-op. Please use the CQL3 specific methods instead.
- *
- * @param version
- */
- public void set_cql_version(String version) throws InvalidRequestException, org.apache.thrift.TException;
-
- }
-
- public interface AsyncIface {
-
- public void login(AuthenticationRequest auth_request, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException;
-
- public void set_keyspace(String keyspace, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException;
-
- public void get(ByteBuffer key, ColumnPath column_path, ConsistencyLevel consistency_level, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException;
-
- public void get_slice(ByteBuffer key, ColumnParent column_parent, SlicePredicate predicate, ConsistencyLevel consistency_level, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException;
-
- public void get_count(ByteBuffer key, ColumnParent column_parent, SlicePredicate predicate, ConsistencyLevel consistency_level, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException;
-
- public void multiget_slice(List<ByteBuffer> keys, ColumnParent column_parent, SlicePredicate predicate, ConsistencyLevel consistency_level, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException;
-
- public void multiget_count(List<ByteBuffer> keys, ColumnParent column_parent, SlicePredicate predicate, ConsistencyLevel consistency_level, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException;
-
- public void get_range_slices(ColumnParent column_parent, SlicePredicate predicate, KeyRange range, ConsistencyLevel consistency_level, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException;
-
- public void get_paged_slice(String column_family, KeyRange range, ByteBuffer start_column, ConsistencyLevel consistency_level, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException;
-
- public void get_indexed_slices(ColumnParent column_parent, IndexClause index_clause, SlicePredicate column_predicate, ConsistencyLevel consistency_level, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException;
-
- public void insert(ByteBuffer key, ColumnParent column_parent, Column column, ConsistencyLevel consistency_level, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException;
-
- public void add(ByteBuffer key, ColumnParent column_parent, CounterColumn column, ConsistencyLevel consistency_level, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException;
-
- public void cas(ByteBuffer key, String column_family, List<Column> expected, List<Column> updates, ConsistencyLevel serial_consistency_level, ConsistencyLevel commit_consistency_level, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException;
-
- public void remove(ByteBuffer key, ColumnPath column_path, long timestamp, ConsistencyLevel consistency_level, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException;
-
- public void remove_counter(ByteBuffer key, ColumnPath path, ConsistencyLevel consistency_level, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException;
-
- public void batch_mutate(Map<ByteBuffer,Map<String,List<Mutation>>> mutation_map, ConsistencyLevel consistency_level, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException;
-
- public void atomic_batch_mutate(Map<ByteBuffer,Map<String,List<Mutation>>> mutation_map, ConsistencyLevel consistency_level, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException;
-
- public void truncate(String cfname, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException;
-
- public void get_multi_slice(MultiSliceRequest request, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException;
-
- public void describe_schema_versions(org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException;
-
- public void describe_keyspaces(org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException;
-
- public void describe_cluster_name(org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException;
-
- public void describe_version(org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException;
-
- public void describe_ring(String keyspace, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException;
-
- public void describe_local_ring(String keyspace, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException;
-
- public void describe_token_map(org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException;
-
- public void describe_partitioner(org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException;
-
- public void describe_snitch(org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException;
-
- public void describe_keyspace(String keyspace, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException;
-
- public void describe_splits(String cfName, String start_token, String end_token, int keys_per_split, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException;
-
- public void trace_next_query(org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException;
-
- public void describe_splits_ex(String cfName, String start_token, String end_token, int keys_per_split, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException;
-
- public void system_add_column_family(CfDef cf_def, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException;
-
- public void system_drop_column_family(String column_family, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException;
-
- public void system_add_keyspace(KsDef ks_def, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException;
-
- public void system_drop_keyspace(String keyspace, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException;
-
- public void system_update_keyspace(KsDef ks_def, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException;
-
- public void system_update_column_family(CfDef cf_def, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException;
-
- public void execute_cql_query(ByteBuffer query, Compression compression, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException;
-
- public void execute_cql3_query(ByteBuffer query, Compression compression, ConsistencyLevel consistency, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException;
-
- public void prepare_cql_query(ByteBuffer query, Compression compression, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException;
-
- public void prepare_cql3_query(ByteBuffer query, Compression compression, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException;
-
- public void execute_prepared_cql_query(int itemId, List<ByteBuffer> values, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException;
-
- public void execute_prepared_cql3_query(int itemId, List<ByteBuffer> values, ConsistencyLevel consistency, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException;
-
- public void set_cql_version(String version, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException;
-
- }
-
- public static class Client extends org.apache.thrift.TServiceClient implements Iface {
- public static class Factory implements org.apache.thrift.TServiceClientFactory<Client> {
- public Factory() {}
- public Client getClient(org.apache.thrift.protocol.TProtocol prot) {
- return new Client(prot);
- }
- public Client getClient(org.apache.thrift.protocol.TProtocol iprot, org.apache.thrift.protocol.TProtocol oprot) {
- return new Client(iprot, oprot);
- }
- }
-
- public Client(org.apache.thrift.protocol.TProtocol prot)
- {
- super(prot, prot);
- }
-
- public Client(org.apache.thrift.protocol.TProtocol iprot, org.apache.thrift.protocol.TProtocol oprot) {
- super(iprot, oprot);
- }
-
- public void login(AuthenticationRequest auth_request) throws AuthenticationException, AuthorizationException, org.apache.thrift.TException
- {
- send_login(auth_request);
- recv_login();
- }
-
- public void send_login(AuthenticationRequest auth_request) throws org.apache.thrift.TException
- {
- login_args args = new login_args();
- args.setAuth_request(auth_request);
- sendBase("login", args);
- }
-
- public void recv_login() throws AuthenticationException, AuthorizationException, org.apache.thrift.TException
- {
- login_result result = new login_result();
- receiveBase(result, "login");
- if (result.authnx != null) {
- throw result.authnx;
- }
- if (result.authzx != null) {
- throw result.authzx;
- }
- return;
- }
-
- public void set_keyspace(String keyspace) throws InvalidRequestException, org.apache.thrift.TException
- {
- send_set_keyspace(keyspace);
- recv_set_keyspace();
- }
-
- public void send_set_keyspace(String keyspace) throws org.apache.thrift.TException
- {
- set_keyspace_args args = new set_keyspace_args();
- args.setKeyspace(keyspace);
- sendBase("set_keyspace", args);
- }
-
- public void recv_set_keyspace() throws InvalidRequestException, org.apache.thrift.TException
- {
- set_keyspace_result result = new set_keyspace_result();
- receiveBase(result, "set_keyspace");
- if (result.ire != null) {
- throw result.ire;
- }
- return;
- }
-
- public ColumnOrSuperColumn get(ByteBuffer key, ColumnPath column_path, ConsistencyLevel consistency_level) throws InvalidRequestException, NotFoundException, UnavailableException, TimedOutException, org.apache.thrift.TException
- {
- send_get(key, column_path, consistency_level);
- return recv_get();
- }
-
- public void send_get(ByteBuffer key, ColumnPath column_path, ConsistencyLevel consistency_level) throws org.apache.thrift.TException
- {
- get_args args = new get_args();
- args.setKey(key);
- args.setColumn_path(column_path);
- args.setConsistency_level(consistency_level);
- sendBase("get", args);
- }
-
- public ColumnOrSuperColumn recv_get() throws InvalidRequestException, NotFoundException, UnavailableException, TimedOutException, org.apache.thrift.TException
- {
- get_result result = new get_result();
- receiveBase(result, "get");
- if (result.isSetSuccess()) {
- return result.success;
- }
- if (result.ire != null) {
- throw result.ire;
- }
- if (result.nfe != null) {
- throw result.nfe;
- }
- if (result.ue != null) {
- throw result.ue;
- }
- if (result.te != null) {
- throw result.te;
- }
- throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "get failed: unknown result");
- }
-
- public List<ColumnOrSuperColumn> get_slice(ByteBuffer key, ColumnParent column_parent, SlicePredicate predicate, ConsistencyLevel consistency_level) throws InvalidRequestException, UnavailableException, TimedOutException, org.apache.thrift.TException
- {
- send_get_slice(key, column_parent, predicate, consistency_level);
- return recv_get_slice();
- }
-
- public void send_get_slice(ByteBuffer key, ColumnParent column_parent, SlicePredicate predicate, ConsistencyLevel consistency_level) throws org.apache.thrift.TException
- {
- get_slice_args args = new get_slice_args();
- args.setKey(key);
- args.setColumn_parent(column_parent);
- args.setPredicate(predicate);
- args.setConsistency_level(consistency_level);
- sendBase("get_slice", args);
- }
-
- public List<ColumnOrSuperColumn> recv_get_slice() throws InvalidRequestException, UnavailableException, TimedOutException, org.apache.thrift.TException
- {
- get_slice_result result = new get_slice_result();
- receiveBase(result, "get_slice");
- if (result.isSetSuccess()) {
- return result.success;
- }
- if (result.ire != null) {
- throw result.ire;
- }
- if (result.ue != null) {
- throw result.ue;
- }
- if (result.te != null) {
- throw result.te;
- }
- throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "get_slice failed: unknown result");
- }
-
- public int get_count(ByteBuffer key, ColumnParent column_parent, SlicePredicate predicate, ConsistencyLevel consistency_level) throws InvalidRequestException, UnavailableException, TimedOutException, org.apache.thrift.TException
- {
- send_get_count(key, column_parent, predicate, consistency_level);
- return recv_get_count();
- }
-
- public void send_get_count(ByteBuffer key, ColumnParent column_parent, SlicePredicate predicate, ConsistencyLevel consistency_level) throws org.apache.thrift.TException
- {
- get_count_args args = new get_count_args();
- args.setKey(key);
- args.setColumn_parent(column_parent);
- args.setPredicate(predicate);
- args.setConsistency_level(consistency_level);
- sendBase("get_count", args);
- }
-
- public int recv_get_count() throws InvalidRequestException, UnavailableException, TimedOutException, org.apache.thrift.TException
- {
- get_count_result result = new get_count_result();
- receiveBase(result, "get_count");
- if (result.isSetSuccess()) {
- return result.success;
- }
- if (result.ire != null) {
- throw result.ire;
- }
- if (result.ue != null) {
- throw result.ue;
- }
- if (result.te != null) {
- throw result.te;
- }
- throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "get_count failed: unknown result");
- }
-
- public Map<ByteBuffer,List<ColumnOrSuperColumn>> multiget_slice(List<ByteBuffer> keys, ColumnParent column_parent, SlicePredicate predicate, ConsistencyLevel consistency_level) throws InvalidRequestException, UnavailableException, TimedOutException, org.apache.thrift.TException
- {
- send_multiget_slice(keys, column_parent, predicate, consistency_level);
- return recv_multiget_slice();
- }
-
- public void send_multiget_slice(List<ByteBuffer> keys, ColumnParent column_parent, SlicePredicate predicate, ConsistencyLevel consistency_level) throws org.apache.thrift.TException
- {
- multiget_slice_args args = new multiget_slice_args();
- args.setKeys(keys);
- args.setColumn_parent(column_parent);
- args.setPredicate(predicate);
- args.setConsistency_level(consistency_level);
- sendBase("multiget_slice", args);
- }
-
- public Map<ByteBuffer,List<ColumnOrSuperColumn>> recv_multiget_slice() throws InvalidRequestException, UnavailableException, TimedOutException, org.apache.thrift.TException
- {
- multiget_slice_result result = new multiget_slice_result();
- receiveBase(result, "multiget_slice");
- if (result.isSetSuccess()) {
- return result.success;
- }
- if (result.ire != null) {
- throw result.ire;
- }
- if (result.ue != null) {
- throw result.ue;
- }
- if (result.te != null) {
- throw result.te;
- }
- throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "multiget_slice failed: unknown result");
- }
-
- public Map<ByteBuffer,Integer> multiget_count(List<ByteBuffer> keys, ColumnParent column_parent, SlicePredicate predicate, ConsistencyLevel consistency_level) throws InvalidRequestException, UnavailableException, TimedOutException, org.apache.thrift.TException
- {
- send_multiget_count(keys, column_parent, predicate, consistency_level);
- return recv_multiget_count();
- }
-
- public void send_multiget_count(List<ByteBuffer> keys, ColumnParent column_parent, SlicePredicate predicate, ConsistencyLevel consistency_level) throws org.apache.thrift.TException
- {
- multiget_count_args args = new multiget_count_args();
- args.setKeys(keys);
- args.setColumn_parent(column_parent);
- args.setPredicate(predicate);
- args.setConsistency_level(consistency_level);
- sendBase("multiget_count", args);
- }
-
- public Map<ByteBuffer,Integer> recv_multiget_count() throws InvalidRequestException, UnavailableException, TimedOutException, org.apache.thrift.TException
- {
- multiget_count_result result = new multiget_count_result();
- receiveBase(result, "multiget_count");
- if (result.isSetSuccess()) {
- return result.success;
- }
- if (result.ire != null) {
- throw result.ire;
- }
- if (result.ue != null) {
- throw result.ue;
- }
- if (result.te != null) {
- throw result.te;
- }
- throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "multiget_count failed: unknown result");
- }
-
- public List<KeySlice> get_range_slices(ColumnParent column_parent, SlicePredicate predicate, KeyRange range, ConsistencyLevel consistency_level) throws InvalidRequestException, UnavailableException, TimedOutException, org.apache.thrift.TException
- {
- send_get_range_slices(column_parent, predicate, range, consistency_level);
- return recv_get_range_slices();
- }
-
- public void send_get_range_slices(ColumnParent column_parent, SlicePredicate predicate, KeyRange range, ConsistencyLevel consistency_level) throws org.apache.thrift.TException
- {
- get_range_slices_args args = new get_range_slices_args();
- args.setColumn_parent(column_parent);
- args.setPredicate(predicate);
- args.setRange(range);
- args.setConsistency_level(consistency_level);
- sendBase("get_range_slices", args);
- }
-
- public List<KeySlice> recv_get_range_slices() throws InvalidRequestException, UnavailableException, TimedOutException, org.apache.thrift.TException
- {
- get_range_slices_result result = new get_range_slices_result();
- receiveBase(result, "get_range_slices");
- if (result.isSetSuccess()) {
- return result.success;
- }
- if (result.ire != null) {
- throw result.ire;
- }
- if (result.ue != null) {
- throw result.ue;
- }
- if (result.te != null) {
- throw result.te;
- }
- throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "get_range_slices failed: unknown result");
- }
-
- public List<KeySlice> get_paged_slice(String column_family, KeyRange range, ByteBuffer start_column, ConsistencyLevel consistency_level) throws InvalidRequestException, UnavailableException, TimedOutException, org.apache.thrift.TException
- {
- send_get_paged_slice(column_family, range, start_column, consistency_level);
- return recv_get_paged_slice();
- }
-
- public void send_get_paged_slice(String column_family, KeyRange range, ByteBuffer start_column, ConsistencyLevel consistency_level) throws org.apache.thrift.TException
- {
- get_paged_slice_args args = new get_paged_slice_args();
- args.setColumn_family(column_family);
- args.setRange(range);
- args.setStart_column(start_column);
- args.setConsistency_level(consistency_level);
- sendBase("get_paged_slice", args);
- }
-
- public List<KeySlice> recv_get_paged_slice() throws InvalidRequestException, UnavailableException, TimedOutException, org.apache.thrift.TException
- {
- get_paged_slice_result result = new get_paged_slice_result();
- receiveBase(result, "get_paged_slice");
- if (result.isSetSuccess()) {
- return result.success;
- }
- if (result.ire != null) {
- throw result.ire;
- }
- if (result.ue != null) {
- throw result.ue;
- }
- if (result.te != null) {
- throw result.te;
- }
- throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "get_paged_slice failed: unknown result");
- }
-
- public List<KeySlice> get_indexed_slices(ColumnParent column_parent, IndexClause index_clause, SlicePredicate column_predicate, ConsistencyLevel consistency_level) throws InvalidRequestException, UnavailableException, TimedOutException, org.apache.thrift.TException
- {
- send_get_indexed_slices(column_parent, index_clause, column_predicate, consistency_level);
- return recv_get_indexed_slices();
- }
-
- public void send_get_indexed_slices(ColumnParent column_parent, IndexClause index_clause, SlicePredicate column_predicate, ConsistencyLevel consistency_level) throws org.apache.thrift.TException
- {
- get_indexed_slices_args args = new get_indexed_slices_args();
- args.setColumn_parent(column_parent);
- args.setIndex_clause(index_clause);
- args.setColumn_predicate(column_predicate);
- args.setConsistency_level(consistency_level);
- sendBase("get_indexed_slices", args);
- }
-
- public List<KeySlice> recv_get_indexed_slices() throws InvalidRequestException, UnavailableException, TimedOutException, org.apache.thrift.TException
- {
- get_indexed_slices_result result = new get_indexed_slices_result();
- receiveBase(result, "get_indexed_slices");
- if (result.isSetSuccess()) {
- return result.success;
- }
- if (result.ire != null) {
- throw result.ire;
- }
- if (result.ue != null) {
- throw result.ue;
- }
- if (result.te != null) {
- throw result.te;
- }
- throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "get_indexed_slices failed: unknown result");
- }
-
- public void insert(ByteBuffer key, ColumnParent column_parent, Column column, ConsistencyLevel consistency_level) throws InvalidRequestException, UnavailableException, TimedOutException, org.apache.thrift.TException
- {
- send_insert(key, column_parent, column, consistency_level);
- recv_insert();
- }
-
- public void send_insert(ByteBuffer key, ColumnParent column_parent, Column column, ConsistencyLevel consistency_level) throws org.apache.thrift.TException
- {
- insert_args args = new insert_args();
- args.setKey(key);
- args.setColumn_parent(column_parent);
- args.setColumn(column);
- args.setConsistency_level(consistency_level);
- sendBase("insert", args);
- }
-
- public void recv_insert() throws InvalidRequestException, UnavailableException, TimedOutException, org.apache.thrift.TException
- {
- insert_result result = new insert_result();
- receiveBase(result, "insert");
- if (result.ire != null) {
- throw result.ire;
- }
- if (result.ue != null) {
- throw result.ue;
- }
- if (result.te != null) {
- throw result.te;
- }
- return;
- }
-
- public void add(ByteBuffer key, ColumnParent column_parent, CounterColumn column, ConsistencyLevel consistency_level) throws InvalidRequestException, UnavailableException, TimedOutException, org.apache.thrift.TException
- {
- send_add(key, column_parent, column, consistency_level);
- recv_add();
- }
-
- public void send_add(ByteBuffer key, ColumnParent column_parent, CounterColumn column, ConsistencyLevel consistency_level) throws org.apache.thrift.TException
- {
- add_args args = new add_args();
- args.setKey(key);
- args.setColumn_parent(column_parent);
- args.setColumn(column);
- args.setConsistency_level(consistency_level);
- sendBase("add", args);
- }
-
- public void recv_add() throws InvalidRequestException, UnavailableException, TimedOutException, org.apache.thrift.TException
- {
- add_result result = new add_result();
- receiveBase(result, "add");
- if (result.ire != null) {
- throw result.ire;
- }
- if (result.ue != null) {
- throw result.ue;
- }
- if (result.te != null) {
- throw result.te;
- }
- return;
- }
-
- public CASResult cas(ByteBuffer key, String column_family, List<Column> expected, List<Column> updates, ConsistencyLevel serial_consistency_level, ConsistencyLevel commit_consistency_level) throws InvalidRequestException, UnavailableException, TimedOutException, org.apache.thrift.TException
- {
- send_cas(key, column_family, expected, updates, serial_consistency_level, commit_consistency_level);
- return recv_cas();
- }
-
- public void send_cas(ByteBuffer key, String column_family, List<Column> expected, List<Column> updates, ConsistencyLevel serial_consistency_level, ConsistencyLevel commit_consistency_level) throws org.apache.thrift.TException
- {
- cas_args args = new cas_args();
- args.setKey(key);
- args.setColumn_family(column_family);
- args.setExpected(expected);
- args.setUpdates(updates);
- args.setSerial_consistency_level(serial_consistency_level);
- args.setCommit_consistency_level(commit_consistency_level);
- sendBase("cas", args);
- }
-
- public CASResult recv_cas() throws InvalidRequestException, UnavailableException, TimedOutException, org.apache.thrift.TException
- {
- cas_result result = new cas_result();
- receiveBase(result, "cas");
- if (result.isSetSuccess()) {
- return result.success;
- }
- if (result.ire != null) {
- throw result.ire;
- }
- if (result.ue != null) {
- throw result.ue;
- }
- if (result.te != null) {
- throw result.te;
- }
- throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "cas failed: unknown result");
- }
-
- public void remove(ByteBuffer key, ColumnPath column_path, long timestamp, ConsistencyLevel consistency_level) throws InvalidRequestException, UnavailableException, TimedOutException, org.apache.thrift.TException
- {
- send_remove(key, column_path, timestamp, consistency_level);
- recv_remove();
- }
-
- public void send_remove(ByteBuffer key, ColumnPath column_path, long timestamp, ConsistencyLevel consistency_level) throws org.apache.thrift.TException
- {
- remove_args args = new remove_args();
- args.setKey(key);
- args.setColumn_path(column_path);
- args.setTimestamp(timestamp);
- args.setConsistency_level(consistency_level);
- sendBase("remove", args);
- }
-
- public void recv_remove() throws InvalidRequestException, UnavailableException, TimedOutException, org.apache.thrift.TException
- {
- remove_result result = new remove_result();
- receiveBase(result, "remove");
- if (result.ire != null) {
- throw result.ire;
- }
- if (result.ue != null) {
- throw result.ue;
- }
- if (result.te != null) {
- throw result.te;
- }
- return;
- }
-
- public void remove_counter(ByteBuffer key, ColumnPath path, ConsistencyLevel consistency_level) throws InvalidRequestException, UnavailableException, TimedOutException, org.apache.thrift.TException
- {
- send_remove_counter(key, path, consistency_level);
- recv_remove_counter();
- }
-
- public void send_remove_counter(ByteBuffer key, ColumnPath path, ConsistencyLevel consistency_level) throws org.apache.thrift.TException
- {
- remove_counter_args args = new remove_counter_args();
- args.setKey(key);
- args.setPath(path);
- args.setConsistency_level(consistency_level);
- sendBase("remove_counter", args);
- }
-
- public void recv_remove_counter() throws InvalidRequestException, UnavailableException, TimedOutException, org.apache.thrift.TException
- {
- remove_counter_result result = new remove_counter_result();
- receiveBase(result, "remove_counter");
- if (result.ire != null) {
- throw result.ire;
- }
- if (result.ue != null) {
- throw result.ue;
- }
- if (result.te != null) {
- throw result.te;
- }
- return;
- }
-
- public void batch_mutate(Map<ByteBuffer,Map<String,List<Mutation>>> mutation_map, ConsistencyLevel consistency_level) throws InvalidRequestException, UnavailableException, TimedOutException, org.apache.thrift.TException
- {
- send_batch_mutate(mutation_map, consistency_level);
- recv_batch_mutate();
- }
-
- public void send_batch_mutate(Map<ByteBuffer,Map<String,List<Mutation>>> mutation_map, ConsistencyLevel consistency_level) throws org.apache.thrift.TException
- {
- batch_mutate_args args = new batch_mutate_args();
- args.setMutation_map(mutation_map);
- args.setConsistency_level(consistency_level);
- sendBase("batch_mutate", args);
- }
-
- public void recv_batch_mutate() throws InvalidRequestException, UnavailableException, TimedOutException, org.apache.thrift.TException
- {
- batch_mutate_result result = new batch_mutate_result();
- receiveBase(result, "batch_mutate");
- if (result.ire != null) {
- throw result.ire;
- }
- if (result.ue != null) {
- throw result.ue;
- }
- if (result.te != null) {
- throw result.te;
- }
- return;
- }
-
- public void atomic_batch_mutate(Map<ByteBuffer,Map<String,List<Mutation>>> mutation_map, ConsistencyLevel consistency_level) throws InvalidRequestException, UnavailableException, TimedOutException, org.apache.thrift.TException
- {
- send_atomic_batch_mutate(mutation_map, consistency_level);
- recv_atomic_batch_mutate();
- }
-
- public void send_atomic_batch_mutate(Map<ByteBuffer,Map<String,List<Mutation>>> mutation_map, ConsistencyLevel consistency_level) throws org.apache.thrift.TException
- {
- atomic_batch_mutate_args args = new atomic_batch_mutate_args();
- args.setMutation_map(mutation_map);
- args.setConsistency_level(consistency_level);
- sendBase("atomic_batch_mutate", args);
- }
-
- public void recv_atomic_batch_mutate() throws InvalidRequestException, UnavailableException, TimedOutException, org.apache.thrift.TException
- {
- atomic_batch_mutate_result result = new atomic_batch_mutate_result();
- receiveBase(result, "atomic_batch_mutate");
- if (result.ire != null) {
- throw result.ire;
- }
- if (result.ue != null) {
- throw result.ue;
- }
- if (result.te != null) {
- throw result.te;
- }
- return;
- }
-
- public void truncate(String cfname) throws InvalidRequestException, UnavailableException, TimedOutException, org.apache.thrift.TException
- {
- send_truncate(cfname);
- recv_truncate();
- }
-
- public void send_truncate(String cfname) throws org.apache.thrift.TException
- {
- truncate_args args = new truncate_args();
- args.setCfname(cfname);
- sendBase("truncate", args);
- }
-
- public void recv_truncate() throws InvalidRequestException, UnavailableException, TimedOutException, org.apache.thrift.TException
- {
- truncate_result result = new truncate_result();
- receiveBase(result, "truncate");
- if (result.ire != null) {
- throw result.ire;
- }
- if (result.ue != null) {
- throw result.ue;
- }
- if (result.te != null) {
- throw result.te;
- }
- return;
- }
-
- public List<ColumnOrSuperColumn> get_multi_slice(MultiSliceRequest request) throws InvalidRequestException, UnavailableException, TimedOutException, org.apache.thrift.TException
- {
- send_get_multi_slice(request);
- return recv_get_multi_slice();
- }
-
- public void send_get_multi_slice(MultiSliceRequest request) throws org.apache.thrift.TException
- {
- get_multi_slice_args args = new get_multi_slice_args();
- args.setRequest(request);
- sendBase("get_multi_slice", args);
- }
-
- public List<ColumnOrSuperColumn> recv_get_multi_slice() throws InvalidRequestException, UnavailableException, TimedOutException, org.apache.thrift.TException
- {
- get_multi_slice_result result = new get_multi_slice_result();
- receiveBase(result, "get_multi_slice");
- if (result.isSetSuccess()) {
- return result.success;
- }
- if (result.ire != null) {
- throw result.ire;
- }
- if (result.ue != null) {
- throw result.ue;
- }
- if (result.te != null) {
- throw result.te;
- }
- throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "get_multi_slice failed: unknown result");
- }
-
- public Map<String,List<String>> describe_schema_versions() throws InvalidRequestException, org.apache.thrift.TException
- {
- send_describe_schema_versions();
- return recv_describe_schema_versions();
- }
-
- public void send_describe_schema_versions() throws org.apache.thrift.TException
- {
- describe_schema_versions_args args = new describe_schema_versions_args();
- sendBase("describe_schema_versions", args);
- }
-
- public Map<String,List<String>> recv_describe_schema_versions() throws InvalidRequestException, org.apache.thrift.TException
- {
- describe_schema_versions_result result = new describe_schema_versions_result();
- receiveBase(result, "describe_schema_versions");
- if (result.isSetSuccess()) {
- return result.success;
- }
- if (result.ire != null) {
- throw result.ire;
- }
- throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "describe_schema_versions failed: unknown result");
- }
-
- public List<KsDef> describe_keyspaces() throws InvalidRequestException, org.apache.thrift.TException
- {
- send_describe_keyspaces();
- return recv_describe_keyspaces();
- }
-
- public void send_describe_keyspaces() throws org.apache.thrift.TException
- {
- describe_keyspaces_args args = new describe_keyspaces_args();
- sendBase("describe_keyspaces", args);
- }
-
- public List<KsDef> recv_describe_keyspaces() throws InvalidRequestException, org.apache.thrift.TException
- {
- describe_keyspaces_result result = new describe_keyspaces_result();
- receiveBase(result, "describe_keyspaces");
- if (result.isSetSuccess()) {
- return result.success;
- }
- if (result.ire != null) {
- throw result.ire;
- }
- throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "describe_keyspaces failed: unknown result");
- }
-
- public String describe_cluster_name() throws org.apache.thrift.TException
- {
- send_describe_cluster_name();
- return recv_describe_cluster_name();
- }
-
- public void send_describe_cluster_name() throws org.apache.thrift.TException
- {
- describe_cluster_name_args args = new describe_cluster_name_args();
- sendBase("describe_cluster_name", args);
- }
-
- public String recv_describe_cluster_name() throws org.apache.thrift.TException
- {
- describe_cluster_name_result result = new describe_cluster_name_result();
- receiveBase(result, "describe_cluster_name");
- if (result.isSetSuccess()) {
- return result.success;
- }
- throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "describe_cluster_name failed: unknown result");
- }
-
- public String describe_version() throws org.apache.thrift.TException
- {
- send_describe_version();
- return recv_describe_version();
- }
-
- public void send_describe_version() throws org.apache.thrift.TException
- {
- describe_version_args args = new describe_version_args();
- sendBase("describe_version", args);
- }
-
- public String recv_describe_version() throws org.apache.thrift.TException
- {
- describe_version_result result = new describe_version_result();
- receiveBase(result, "describe_version");
- if (result.isSetSuccess()) {
- return result.success;
- }
- throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "describe_version failed: unknown result");
- }
-
- public List<TokenRange> describe_ring(String keyspace) throws InvalidRequestException, org.apache.thrift.TException
- {
- send_describe_ring(keyspace);
- return recv_describe_ring();
- }
-
- public void send_describe_ring(String keyspace) throws org.apache.thrift.TException
- {
- describe_ring_args args = new describe_ring_args();
- args.setKeyspace(keyspace);
- sendBase("describe_ring", args);
- }
-
- public List<TokenRange> recv_describe_ring() throws InvalidRequestException, org.apache.thrift.TException
- {
- describe_ring_result result = new describe_ring_result();
- receiveBase(result, "describe_ring");
- if (result.isSetSuccess()) {
- return result.success;
- }
- if (result.ire != null) {
- throw result.ire;
- }
- throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "describe_ring failed: unknown result");
- }
-
- public List<TokenRange> describe_local_ring(String keyspace) throws InvalidRequestException, org.apache.thrift.TException
- {
- send_describe_local_ring(keyspace);
- return recv_describe_local_ring();
- }
-
- public void send_describe_local_ring(String keyspace) throws org.apache.thrift.TException
- {
- describe_local_ring_args args = new describe_local_ring_args();
- args.setKeyspace(keyspace);
- sendBase("describe_local_ring", args);
- }
-
- public List<TokenRange> recv_describe_local_ring() throws InvalidRequestException, org.apache.thrift.TException
- {
- describe_local_ring_result result = new describe_local_ring_result();
- receiveBase(result, "describe_local_ring");
- if (result.isSetSuccess()) {
- return result.success;
- }
- if (result.ire != null) {
- throw result.ire;
- }
- throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "describe_local_ring failed: unknown result");
- }
-
- public Map<String,String> describe_token_map() throws InvalidRequestException, org.apache.thrift.TException
- {
- send_describe_token_map();
- return recv_describe_token_map();
- }
-
- public void send_describe_token_map() throws org.apache.thrift.TException
- {
- describe_token_map_args args = new describe_token_map_args();
- sendBase("describe_token_map", args);
- }
-
- public Map<String,String> recv_describe_token_map() throws InvalidRequestException, org.apache.thrift.TException
- {
- describe_token_map_result result = new describe_token_map_result();
- receiveBase(result, "describe_token_map");
- if (result.isSetSuccess()) {
- return result.success;
- }
- if (result.ire != null) {
- throw result.ire;
- }
- throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "describe_token_map failed: unknown result");
- }
-
- public String describe_partitioner() throws org.apache.thrift.TException
- {
- send_describe_partitioner();
- return recv_describe_partitioner();
- }
-
- public void send_describe_partitioner() throws org.apache.thrift.TException
- {
- describe_partitioner_args args = new describe_partitioner_args();
- sendBase("describe_partitioner", args);
- }
-
- public String recv_describe_partitioner() throws org.apache.thrift.TException
- {
- describe_partitioner_result result = new describe_partitioner_result();
- receiveBase(result, "describe_partitioner");
- if (result.isSetSuccess()) {
- return result.success;
- }
- throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "describe_partitioner failed: unknown result");
- }
-
- public String describe_snitch() throws org.apache.thrift.TException
- {
- send_describe_snitch();
- return recv_describe_snitch();
- }
-
- public void send_describe_snitch() throws org.apache.thrift.TException
- {
- describe_snitch_args args = new describe_snitch_args();
- sendBase("describe_snitch", args);
- }
-
- public String recv_describe_snitch() throws org.apache.thrift.TException
- {
- describe_snitch_result result = new describe_snitch_result();
- receiveBase(result, "describe_snitch");
- if (result.isSetSuccess()) {
- return result.success;
- }
- throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "describe_snitch failed: unknown result");
- }
-
- public KsDef describe_keyspace(String keyspace) throws NotFoundException, InvalidRequestException, org.apache.thrift.TException
- {
- send_describe_keyspace(keyspace);
- return recv_describe_keyspace();
- }
-
- public void send_describe_keyspace(String keyspace) throws org.apache.thrift.TException
- {
- describe_keyspace_args args = new describe_keyspace_args();
- args.setKeyspace(keyspace);
- sendBase("describe_keyspace", args);
- }
-
- public KsDef recv_describe_keyspace() throws NotFoundException, InvalidRequestException, org.apache.thrift.TException
- {
- describe_keyspace_result result = new describe_keyspace_result();
- receiveBase(result, "describe_keyspace");
- if (result.isSetSuccess()) {
- return result.success;
- }
- if (result.nfe != null) {
- throw result.nfe;
- }
- if (result.ire != null) {
- throw result.ire;
- }
- throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "describe_keyspace failed: unknown result");
- }
-
- public List<String> describe_splits(String cfName, String start_token, String end_token, int keys_per_split) throws InvalidRequestException, org.apache.thrift.TException
- {
- send_describe_splits(cfName, start_token, end_token, keys_per_split);
- return recv_describe_splits();
- }
-
- public void send_describe_splits(String cfName, String start_token, String end_token, int keys_per_split) throws org.apache.thrift.TException
- {
- describe_splits_args args = new describe_splits_args();
- args.setCfName(cfName);
- args.setStart_token(start_token);
- args.setEnd_token(end_token);
- args.setKeys_per_split(keys_per_split);
- sendBase("describe_splits", args);
- }
-
- public List<String> recv_describe_splits() throws InvalidRequestException, org.apache.thrift.TException
- {
- describe_splits_result result = new describe_splits_result();
- receiveBase(result, "describe_splits");
- if (result.isSetSuccess()) {
- return result.success;
- }
- if (result.ire != null) {
- throw result.ire;
- }
- throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "describe_splits failed: unknown result");
- }
-
- public ByteBuffer trace_next_query() throws org.apache.thrift.TException
- {
- send_trace_next_query();
- return recv_trace_next_query();
- }
-
- public void send_trace_next_query() throws org.apache.thrift.TException
- {
- trace_next_query_args args = new trace_next_query_args();
- sendBase("trace_next_query", args);
- }
-
- public ByteBuffer recv_trace_next_query() throws org.apache.thrift.TException
- {
- trace_next_query_result result = new trace_next_query_result();
- receiveBase(result, "trace_next_query");
- if (result.isSetSuccess()) {
- return result.success;
- }
- throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "trace_next_query failed: unknown result");
- }
-
- public List<CfSplit> describe_splits_ex(String cfName, String start_token, String end_token, int keys_per_split) throws InvalidRequestException, org.apache.thrift.TException
- {
- send_describe_splits_ex(cfName, start_token, end_token, keys_per_split);
- return recv_describe_splits_ex();
- }
-
- public void send_describe_splits_ex(String cfName, String start_token, String end_token, int keys_per_split) throws org.apache.thrift.TException
- {
- describe_splits_ex_args args = new describe_splits_ex_args();
- args.setCfName(cfName);
- args.setStart_token(start_token);
- args.setEnd_token(end_token);
- args.setKeys_per_split(keys_per_split);
- sendBase("describe_splits_ex", args);
- }
-
- public List<CfSplit> recv_describe_splits_ex() throws InvalidRequestException, org.apache.thrift.TException
- {
- describe_splits_ex_result result = new describe_splits_ex_result();
- receiveBase(result, "describe_splits_ex");
- if (result.isSetSuccess()) {
- return result.success;
- }
- if (result.ire != null) {
- throw result.ire;
- }
- throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "describe_splits_ex failed: unknown result");
- }
-
- public String system_add_column_family(CfDef cf_def) throws InvalidRequestException, SchemaDisagreementException, org.apache.thrift.TException
- {
- send_system_add_column_family(cf_def);
- return recv_system_add_column_family();
- }
-
- public void send_system_add_column_family(CfDef cf_def) throws org.apache.thrift.TException
- {
- system_add_column_family_args args = new system_add_column_family_args();
- args.setCf_def(cf_def);
- sendBase("system_add_column_family", args);
- }
-
- public String recv_system_add_column_family() throws InvalidRequestException, SchemaDisagreementException, org.apache.thrift.TException
- {
- system_add_column_family_result result = new system_add_column_family_result();
- receiveBase(result, "system_add_column_family");
- if (result.isSetSuccess()) {
- return result.success;
- }
- if (result.ire != null) {
- throw result.ire;
- }
- if (result.sde != null) {
- throw result.sde;
- }
- throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "system_add_column_family failed: unknown result");
- }
-
- public String system_drop_column_family(String column_family) throws InvalidRequestException, SchemaDisagreementException, org.apache.thrift.TException
- {
- send_system_drop_column_family(column_family);
- return recv_system_drop_column_family();
- }
-
- public void send_system_drop_column_family(String column_family) throws org.apache.thrift.TException
- {
- system_drop_column_family_args args = new system_drop_column_family_args();
- args.setColumn_family(column_family);
- sendBase("system_drop_column_family", args);
- }
-
- public String recv_system_drop_column_family() throws InvalidRequestException, SchemaDisagreementException, org.apache.thrift.TException
- {
- system_drop_column_family_result result = new system_drop_column_family_result();
- receiveBase(result, "system_drop_column_family");
- if (result.isSetSuccess()) {
- return result.success;
- }
- if (result.ire != null) {
- throw result.ire;
- }
- if (result.sde != null) {
- throw result.sde;
- }
- throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "system_drop_column_family failed: unknown result");
- }
-
- public String system_add_keyspace(KsDef ks_def) throws InvalidRequestException, SchemaDisagreementException, org.apache.thrift.TException
- {
- send_system_add_keyspace(ks_def);
- return recv_system_add_keyspace();
- }
-
- public void send_system_add_keyspace(KsDef ks_def) throws org.apache.thrift.TException
- {
- system_add_keyspace_args args = new system_add_keyspace_args();
- args.setKs_def(ks_def);
- sendBase("system_add_keyspace", args);
- }
-
- public String recv_system_add_keyspace() throws InvalidRequestException, SchemaDisagreementException, org.apache.thrift.TException
- {
- system_add_keyspace_result result = new system_add_keyspace_result();
- receiveBase(result, "system_add_keyspace");
- if (result.isSetSuccess()) {
- return result.success;
- }
- if (result.ire != null) {
- throw result.ire;
- }
- if (result.sde != null) {
- throw result.sde;
- }
- throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "system_add_keyspace failed: unknown result");
- }
-
- public String system_drop_keyspace(String keyspace) throws InvalidRequestException, SchemaDisagreementException, org.apache.thrift.TException
- {
- send_system_drop_keyspace(keyspace);
- return recv_system_drop_keyspace();
- }
-
- public void send_system_drop_keyspace(String keyspace) throws org.apache.thrift.TException
- {
- system_drop_keyspace_args args = new system_drop_keyspace_args();
- args.setKeyspace(keyspace);
- sendBase("system_drop_keyspace", args);
- }
-
- public String recv_system_drop_keyspace() throws InvalidRequestException, SchemaDisagreementException, org.apache.thrift.TException
- {
- system_drop_keyspace_result result = new system_drop_keyspace_result();
- receiveBase(result, "system_drop_keyspace");
- if (result.isSetSuccess()) {
- return result.success;
- }
- if (result.ire != null) {
- throw result.ire;
- }
- if (result.sde != null) {
- throw result.sde;
- }
- throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "system_drop_keyspace failed: unknown result");
- }
-
- public String system_update_keyspace(KsDef ks_def) throws InvalidRequestException, SchemaDisagreementException, org.apache.thrift.TException
- {
- send_system_update_keyspace(ks_def);
- return recv_system_update_keyspace();
- }
-
- public void send_system_update_keyspace(KsDef ks_def) throws org.apache.thrift.TException
- {
- system_update_keyspace_args args = new system_update_keyspace_args();
- args.setKs_def(ks_def);
- sendBase("system_update_keyspace", args);
- }
-
- public String recv_system_update_keyspace() throws InvalidRequestException, SchemaDisagreementException, org.apache.thrift.TException
- {
- system_update_keyspace_result result = new system_update_keyspace_result();
- receiveBase(result, "system_update_keyspace");
- if (result.isSetSuccess()) {
- return result.success;
- }
- if (result.ire != null) {
- throw result.ire;
- }
- if (result.sde != null) {
- throw result.sde;
- }
- throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "system_update_keyspace failed: unknown result");
- }
-
- public String system_update_column_family(CfDef cf_def) throws InvalidRequestException, SchemaDisagreementException, org.apache.thrift.TException
- {
- send_system_update_column_family(cf_def);
- return recv_system_update_column_family();
- }
-
- public void send_system_update_column_family(CfDef cf_def) throws org.apache.thrift.TException
- {
- system_update_column_family_args args = new system_update_column_family_args();
- args.setCf_def(cf_def);
- sendBase("system_update_column_family", args);
- }
-
- public String recv_system_update_column_family() throws InvalidRequestException, SchemaDisagreementException, org.apache.thrift.TException
- {
- system_update_column_family_result result = new system_update_column_family_result();
- receiveBase(result, "system_update_column_family");
- if (result.isSetSuccess()) {
- return result.success;
- }
- if (result.ire != null) {
- throw result.ire;
- }
- if (result.sde != null) {
- throw result.sde;
- }
- throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "system_update_column_family failed: unknown result");
- }
-
- public CqlResult execute_cql_query(ByteBuffer query, Compression compression) throws InvalidRequestException, UnavailableException, TimedOutException, SchemaDisagreementException, org.apache.thrift.TException
- {
- send_execute_cql_query(query, compression);
- return recv_execute_cql_query();
- }
-
- public void send_execute_cql_query(ByteBuffer query, Compression compression) throws org.apache.thrift.TException
- {
- execute_cql_query_args args = new execute_cql_query_args();
- args.setQuery(query);
- args.setCompression(compression);
- sendBase("execute_cql_query", args);
- }
-
- public CqlResult recv_execute_cql_query() throws InvalidRequestException, UnavailableException, TimedOutException, SchemaDisagreementException, org.apache.thrift.TException
- {
- execute_cql_query_result result = new execute_cql_query_result();
- receiveBase(result, "execute_cql_query");
- if (result.isSetSuccess()) {
- return result.success;
- }
- if (result.ire != null) {
- throw result.ire;
- }
- if (result.ue != null) {
- throw result.ue;
- }
- if (result.te != null) {
- throw result.te;
- }
- if (result.sde != null) {
- throw result.sde;
- }
- throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "execute_cql_query failed: unknown result");
- }
-
- public CqlResult execute_cql3_query(ByteBuffer query, Compression compression, ConsistencyLevel consistency) throws InvalidRequestException, UnavailableException, TimedOutException, SchemaDisagreementException, org.apache.thrift.TException
- {
- send_execute_cql3_query(query, compression, consistency);
- return recv_execute_cql3_query();
- }
-
- public void send_execute_cql3_query(ByteBuffer query, Compression compression, ConsistencyLevel consistency) throws org.apache.thrift.TException
- {
- execute_cql3_query_args args = new execute_cql3_query_args();
- args.setQuery(query);
- args.setCompression(compression);
- args.setConsistency(consistency);
- sendBase("execute_cql3_query", args);
- }
-
- public CqlResult recv_execute_cql3_query() throws InvalidRequestException, UnavailableException, TimedOutException, SchemaDisagreementException, org.apache.thrift.TException
- {
- execute_cql3_query_result result = new execute_cql3_query_result();
- receiveBase(result, "execute_cql3_query");
- if (result.isSetSuccess()) {
- return result.success;
- }
- if (result.ire != null) {
- throw result.ire;
- }
- if (result.ue != null) {
- throw result.ue;
- }
- if (result.te != null) {
- throw result.te;
- }
- if (result.sde != null) {
- throw result.sde;
- }
- throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "execute_cql3_query failed: unknown result");
- }
-
- public CqlPreparedResult prepare_cql_query(ByteBuffer query, Compression compression) throws InvalidRequestException, org.apache.thrift.TException
- {
- send_prepare_cql_query(query, compression);
- return recv_prepare_cql_query();
- }
-
- public void send_prepare_cql_query(ByteBuffer query, Compression compression) throws org.apache.thrift.TException
- {
- prepare_cql_query_args args = new prepare_cql_query_args();
- args.setQuery(query);
- args.setCompression(compression);
- sendBase("prepare_cql_query", args);
- }
-
- public CqlPreparedResult recv_prepare_cql_query() throws InvalidRequestException, org.apache.thrift.TException
- {
- prepare_cql_query_result result = new prepare_cql_query_result();
- receiveBase(result, "prepare_cql_query");
- if (result.isSetSuccess()) {
- return result.success;
- }
- if (result.ire != null) {
- throw result.ire;
- }
- throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "prepare_cql_query failed: unknown result");
- }
-
- public CqlPreparedResult prepare_cql3_query(ByteBuffer query, Compression compression) throws InvalidRequestException, org.apache.thrift.TException
- {
- send_prepare_cql3_query(query, compression);
- return recv_prepare_cql3_query();
- }
-
- public void send_prepare_cql3_query(ByteBuffer query, Compression compression) throws org.apache.thrift.TException
- {
- prepare_cql3_query_args args = new prepare_cql3_query_args();
- args.setQuery(query);
- args.setCompression(compression);
- sendBase("prepare_cql3_query", args);
- }
-
- public CqlPreparedResult recv_prepare_cql3_query() throws InvalidRequestException, org.apache.thrift.TException
- {
- prepare_cql3_query_result result = new prepare_cql3_query_result();
- receiveBase(result, "prepare_cql3_query");
- if (result.isSetSuccess()) {
- return result.success;
- }
- if (result.ire != null) {
- throw result.ire;
- }
- throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "prepare_cql3_query failed: unknown result");
- }
-
- public CqlResult execute_prepared_cql_query(int itemId, List<ByteBuffer> values) throws InvalidRequestException, UnavailableException, TimedOutException, SchemaDisagreementException, org.apache.thrift.TException
- {
- send_execute_prepared_cql_query(itemId, values);
- return recv_execute_prepared_cql_query();
- }
-
- public void send_execute_prepared_cql_query(int itemId, List<ByteBuffer> values) throws org.apache.thrift.TException
- {
- execute_prepared_cql_query_args args = new execute_prepared_cql_query_args();
- args.setItemId(itemId);
- args.setValues(values);
- sendBase("execute_prepared_cql_query", args);
- }
-
- public CqlResult recv_execute_prepared_cql_query() throws InvalidRequestException, UnavailableException, TimedOutException, SchemaDisagreementException, org.apache.thrift.TException
- {
- execute_prepared_cql_query_result result = new execute_prepared_cql_query_result();
- receiveBase(result, "execute_prepared_cql_query");
- if (result.isSetSuccess()) {
- return result.success;
- }
- if (result.ire != null) {
- throw result.ire;
- }
- if (result.ue != null) {
- throw result.ue;
- }
- if (result.te != null) {
- throw result.te;
- }
- if (result.sde != null) {
- throw result.sde;
- }
- throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "execute_prepared_cql_query failed: unknown result");
- }
-
- public CqlResult execute_prepared_cql3_query(int itemId, List<ByteBuffer> values, ConsistencyLevel consistency) throws InvalidRequestException, UnavailableException, TimedOutException, SchemaDisagreementException, org.apache.thrift.TException
- {
- send_execute_prepared_cql3_query(itemId, values, consistency);
- return recv_execute_prepared_cql3_query();
- }
-
- public void send_execute_prepared_cql3_query(int itemId, List<ByteBuffer> values, ConsistencyLevel consistency) throws org.apache.thrift.TException
- {
- execute_prepared_cql3_query_args args = new execute_prepared_cql3_query_args();
- args.setItemId(itemId);
- args.setValues(values);
- args.setConsistency(consistency);
- sendBase("execute_prepared_cql3_query", args);
- }
-
- public CqlResult recv_execute_prepared_cql3_query() throws InvalidRequestException, UnavailableException, TimedOutException, SchemaDisagreementException, org.apache.thrift.TException
- {
- execute_prepared_cql3_query_result result = new execute_prepared_cql3_query_result();
- receiveBase(result, "execute_prepared_cql3_query");
- if (result.isSetSuccess()) {
- return result.success;
- }
- if (result.ire != null) {
- throw result.ire;
- }
- if (result.ue != null) {
- throw result.ue;
- }
- if (result.te != null) {
- throw result.te;
- }
- if (result.sde != null) {
- throw result.sde;
- }
- throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "execute_prepared_cql3_query failed: unknown result");
- }
-
- public void set_cql_version(String version) throws InvalidRequestException, org.apache.thrift.TException
- {
- send_set_cql_version(version);
- recv_set_cql_version();
- }
-
- public void send_set_cql_version(String version) throws org.apache.thrift.TException
- {
- set_cql_version_args args = new set_cql_version_args();
- args.setVersion(version);
- sendBase("set_cql_version", args);
- }
-
- public void recv_set_cql_version() throws InvalidRequestException, org.apache.thrift.TException
- {
- set_cql_version_result result = new set_cql_version_result();
- receiveBase(result, "set_cql_version");
- if (result.ire != null) {
- throw result.ire;
- }
- return;
- }
-
- }
- public static class AsyncClient extends org.apache.thrift.async.TAsyncClient implements AsyncIface {
- public static class Factory implements org.apache.thrift.async.TAsyncClientFactory<AsyncClient> {
- private org.apache.thrift.async.TAsyncClientManager clientManager;
- private org.apache.thrift.protocol.TProtocolFactory protocolFactory;
- public Factory(org.apache.thrift.async.TAsyncClientManager clientManager, org.apache.thrift.protocol.TProtocolFactory protocolFactory) {
- this.clientManager = clientManager;
- this.protocolFactory = protocolFactory;
- }
- public AsyncClient getAsyncClient(org.apache.thrift.transport.TNonblockingTransport transport) {
- return new AsyncClient(protocolFactory, clientManager, transport);
- }
- }
-
- public AsyncClient(org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.async.TAsyncClientManager clientManager, org.apache.thrift.transport.TNonblockingTransport transport) {
- super(protocolFactory, clientManager, transport);
- }
-
- public void login(AuthenticationRequest auth_request, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException {
- checkReady();
- login_call method_call = new login_call(auth_request, resultHandler, this, ___protocolFactory, ___transport);
- this.___currentMethod = method_call;
- ___manager.call(method_call);
- }
-
- public static class login_call extends org.apache.thrift.async.TAsyncMethodCall {
- private AuthenticationRequest auth_request;
- public login_call(AuthenticationRequest auth_request, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
- super(client, protocolFactory, transport, resultHandler, false);
- this.auth_request = auth_request;
- }
-
- public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
- prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("login", org.apache.thrift.protocol.TMessageType.CALL, 0));
- login_args args = new login_args();
- args.setAuth_request(auth_request);
- args.write(prot);
- prot.writeMessageEnd();
- }
-
- public void getResult() throws AuthenticationException, AuthorizationException, org.apache.thrift.TException {
- if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
- throw new IllegalStateException("Method call not finished!");
- }
- org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
- org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
- (new Client(prot)).recv_login();
- }
- }
-
- public void set_keyspace(String keyspace, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException {
- checkReady();
- set_keyspace_call method_call = new set_keyspace_call(keyspace, resultHandler, this, ___protocolFactory, ___transport);
- this.___currentMethod = method_call;
- ___manager.call(method_call);
- }
-
- public static class set_keyspace_call extends org.apache.thrift.async.TAsyncMethodCall {
- private String keyspace;
- public set_keyspace_call(String keyspace, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
- super(client, protocolFactory, transport, resultHandler, false);
- this.keyspace = keyspace;
- }
-
- public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
- prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("set_keyspace", org.apache.thrift.protocol.TMessageType.CALL, 0));
- set_keyspace_args args = new set_keyspace_args();
- args.setKeyspace(keyspace);
- args.write(prot);
- prot.writeMessageEnd();
- }
-
- public void getResult() throws InvalidRequestException, org.apache.thrift.TException {
- if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
- throw new IllegalStateException("Method call not finished!");
- }
- org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
- org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
- (new Client(prot)).recv_set_keyspace();
- }
- }
-
- public void get(ByteBuffer key, ColumnPath column_path, ConsistencyLevel consistency_level, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException {
- checkReady();
- get_call method_call = new get_call(key, column_path, consistency_level, resultHandler, this, ___protocolFactory, ___transport);
- this.___currentMethod = method_call;
- ___manager.call(method_call);
- }
-
- public static class get_call extends org.apache.thrift.async.TAsyncMethodCall {
- private ByteBuffer key;
- private ColumnPath column_path;
- private ConsistencyLevel consistency_level;
- public get_call(ByteBuffer key, ColumnPath column_path, ConsistencyLevel consistency_level, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
- super(client, protocolFactory, transport, resultHandler, false);
- this.key = key;
- this.column_path = column_path;
- this.consistency_level = consistency_level;
- }
-
- public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
- prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("get", org.apache.thrift.protocol.TMessageType.CALL, 0));
- get_args args = new get_args();
- args.setKey(key);
- args.setColumn_path(column_path);
- args.setConsistency_level(consistency_level);
- args.write(prot);
- prot.writeMessageEnd();
- }
-
- public ColumnOrSuperColumn getResult() throws InvalidRequestException, NotFoundException, UnavailableException, TimedOutException, org.apache.thrift.TException {
- if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
- throw new IllegalStateException("Method call not finished!");
- }
- org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
- org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
- return (new Client(prot)).recv_get();
- }
- }
-
- public void get_slice(ByteBuffer key, ColumnParent column_parent, SlicePredicate predicate, ConsistencyLevel consistency_level, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException {
- checkReady();
- get_slice_call method_call = new get_slice_call(key, column_parent, predicate, consistency_level, resultHandler, this, ___protocolFactory, ___transport);
- this.___currentMethod = method_call;
- ___manager.call(method_call);
- }
-
- public static class get_slice_call extends org.apache.thrift.async.TAsyncMethodCall {
- private ByteBuffer key;
- private ColumnParent column_parent;
- private SlicePredicate predicate;
- private ConsistencyLevel consistency_level;
- public get_slice_call(ByteBuffer key, ColumnParent column_parent, SlicePredicate predicate, ConsistencyLevel consistency_level, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
- super(client, protocolFactory, transport, resultHandler, false);
- this.key = key;
- this.column_parent = column_parent;
- this.predicate = predicate;
- this.consistency_level = consistency_level;
- }
-
- public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
- prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("get_slice", org.apache.thrift.protocol.TMessageType.CALL, 0));
- get_slice_args args = new get_slice_args();
- args.setKey(key);
- args.setColumn_parent(column_parent);
- args.setPredicate(predicate);
- args.setConsistency_level(consistency_level);
- args.write(prot);
- prot.writeMessageEnd();
- }
-
- public List<ColumnOrSuperColumn> getResult() throws InvalidRequestException, UnavailableException, TimedOutException, org.apache.thrift.TException {
- if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
- throw new IllegalStateException("Method call not finished!");
- }
- org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
- org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
- return (new Client(prot)).recv_get_slice();
- }
- }
-
- public void get_count(ByteBuffer key, ColumnParent column_parent, SlicePredicate predicate, ConsistencyLevel consistency_level, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException {
- checkReady();
- get_count_call method_call = new get_count_call(key, column_parent, predicate, consistency_level, resultHandler, this, ___protocolFactory, ___transport);
- this.___currentMethod = method_call;
- ___manager.call(method_call);
- }
-
- public static class get_count_call extends org.apache.thrift.async.TAsyncMethodCall {
- private ByteBuffer key;
- private ColumnParent column_parent;
- private SlicePredicate predicate;
- private ConsistencyLevel consistency_level;
- public get_count_call(ByteBuffer key, ColumnParent column_parent, SlicePredicate predicate, ConsistencyLevel consistency_level, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
- super(client, protocolFactory, transport, resultHandler, false);
- this.key = key;
- this.column_parent = column_parent;
- this.predicate = predicate;
- this.consistency_level = consistency_level;
- }
-
- public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
- prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("get_count", org.apache.thrift.protocol.TMessageType.CALL, 0));
- get_count_args args = new get_count_args();
- args.setKey(key);
- args.setColumn_parent(column_parent);
- args.setPredicate(predicate);
- args.setConsistency_level(consistency_level);
- args.write(prot);
- prot.writeMessageEnd();
- }
-
- public int getResult() throws InvalidRequestException, UnavailableException, TimedOutException, org.apache.thrift.TException {
- if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
- throw new IllegalStateException("Method call not finished!");
- }
- org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
- org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
- return (new Client(prot)).recv_get_count();
- }
- }
-
- public void multiget_slice(List<ByteBuffer> keys, ColumnParent column_parent, SlicePredicate predicate, ConsistencyLevel consistency_level, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException {
- checkReady();
- multiget_slice_call method_call = new multiget_slice_call(keys, column_parent, predicate, consistency_level, resultHandler, this, ___protocolFactory, ___transport);
- this.___currentMethod = method_call;
- ___manager.call(method_call);
- }
-
- public static class multiget_slice_call extends org.apache.thrift.async.TAsyncMethodCall {
- private List<ByteBuffer> keys;
- private ColumnParent column_parent;
- private SlicePredicate predicate;
- private ConsistencyLevel consistency_level;
- public multiget_slice_call(List<ByteBuffer> keys, ColumnParent column_parent, SlicePredicate predicate, ConsistencyLevel consistency_level, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
- super(client, protocolFactory, transport, resultHandler, false);
- this.keys = keys;
- this.column_parent = column_parent;
- this.predicate = predicate;
- this.consistency_level = consistency_level;
- }
-
- public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
- prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("multiget_slice", org.apache.thrift.protocol.TMessageType.CALL, 0));
- multiget_slice_args args = new multiget_slice_args();
- args.setKeys(keys);
- args.setColumn_parent(column_parent);
- args.setPredicate(predicate);
- args.setConsistency_level(consistency_level);
- args.write(prot);
- prot.writeMessageEnd();
- }
-
- public Map<ByteBuffer,List<ColumnOrSuperColumn>> getResult() throws InvalidRequestException, UnavailableException, TimedOutException, org.apache.thrift.TException {
- if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
- throw new IllegalStateException("Method call not finished!");
- }
- org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
- org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
- return (new Client(prot)).recv_multiget_slice();
- }
- }
-
- public void multiget_count(List<ByteBuffer> keys, ColumnParent column_parent, SlicePredicate predicate, ConsistencyLevel consistency_level, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException {
- checkReady();
- multiget_count_call method_call = new multiget_count_call(keys, column_parent, predicate, consistency_level, resultHandler, this, ___protocolFactory, ___transport);
- this.___currentMethod = method_call;
- ___manager.call(method_call);
- }
-
- public static class multiget_count_call extends org.apache.thrift.async.TAsyncMethodCall {
- private List<ByteBuffer> keys;
- private ColumnParent column_parent;
- private SlicePredicate predicate;
- private ConsistencyLevel consistency_level;
- public multiget_count_call(List<ByteBuffer> keys, ColumnParent column_parent, SlicePredicate predicate, ConsistencyLevel consistency_level, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
- super(client, protocolFactory, transport, resultHandler, false);
- this.keys = keys;
- this.column_parent = column_parent;
- this.predicate = predicate;
- this.consistency_level = consistency_level;
- }
-
- public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
- prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("multiget_count", org.apache.thrift.protocol.TMessageType.CALL, 0));
- multiget_count_args args = new multiget_count_args();
- args.setKeys(keys);
- args.setColumn_parent(column_parent);
- args.setPredicate(predicate);
- args.setConsistency_level(consistency_level);
- args.write(prot);
- prot.writeMessageEnd();
- }
-
- public Map<ByteBuffer,Integer> getResult() throws InvalidRequestException, UnavailableException, TimedOutException, org.apache.thrift.TException {
- if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
- throw new IllegalStateException("Method call not finished!");
- }
- org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
- org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
- return (new Client(prot)).recv_multiget_count();
- }
- }
-
- public void get_range_slices(ColumnParent column_parent, SlicePredicate predicate, KeyRange range, ConsistencyLevel consistency_level, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException {
- checkReady();
- get_range_slices_call method_call = new get_range_slices_call(column_parent, predicate, range, consistency_level, resultHandler, this, ___protocolFactory, ___transport);
- this.___currentMethod = method_call;
- ___manager.call(method_call);
- }
-
- public static class get_range_slices_call extends org.apache.thrift.async.TAsyncMethodCall {
- private ColumnParent column_parent;
- private SlicePredicate predicate;
- private KeyRange range;
- private ConsistencyLevel consistency_level;
- public get_range_slices_call(ColumnParent column_parent, SlicePredicate predicate, KeyRange range, ConsistencyLevel consistency_level, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
- super(client, protocolFactory, transport, resultHandler, false);
- this.column_parent = column_parent;
- this.predicate = predicate;
- this.range = range;
- this.consistency_level = consistency_level;
- }
-
- public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
- prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("get_range_slices", org.apache.thrift.protocol.TMessageType.CALL, 0));
- get_range_slices_args args = new get_range_slices_args();
- args.setColumn_parent(column_parent);
- args.setPredicate(predicate);
- args.setRange(range);
- args.setConsistency_level(consistency_level);
- args.write(prot);
- prot.writeMessageEnd();
- }
-
- public List<KeySlice> getResult() throws InvalidRequestException, UnavailableException, TimedOutException, org.apache.thrift.TException {
- if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
- throw new IllegalStateException("Method call not finished!");
- }
- org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
- org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
- return (new Client(prot)).recv_get_range_slices();
- }
- }
-
- public void get_paged_slice(String column_family, KeyRange range, ByteBuffer start_column, ConsistencyLevel consistency_level, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException {
- checkReady();
- get_paged_slice_call method_call = new get_paged_slice_call(column_family, range, start_column, consistency_level, resultHandler, this, ___protocolFactory, ___transport);
- this.___currentMethod = method_call;
- ___manager.call(method_call);
- }
-
- public static class get_paged_slice_call extends org.apache.thrift.async.TAsyncMethodCall {
- private String column_family;
- private KeyRange range;
- private ByteBuffer start_column;
- private ConsistencyLevel consistency_level;
- public get_paged_slice_call(String column_family, KeyRange range, ByteBuffer start_column, ConsistencyLevel consistency_level, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
- super(client, protocolFactory, transport, resultHandler, false);
- this.column_family = column_family;
- this.range = range;
- this.start_column = start_column;
- this.consistency_level = consistency_level;
- }
-
- public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
- prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("get_paged_slice", org.apache.thrift.protocol.TMessageType.CALL, 0));
- get_paged_slice_args args = new get_paged_slice_args();
- args.setColumn_family(column_family);
- args.setRange(range);
- args.setStart_column(start_column);
- args.setConsistency_level(consistency_level);
- args.write(prot);
- prot.writeMessageEnd();
- }
-
- public List<KeySlice> getResult() throws InvalidRequestException, UnavailableException, TimedOutException, org.apache.thrift.TException {
- if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
- throw new IllegalStateException("Method call not finished!");
- }
- org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
- org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
- return (new Client(prot)).recv_get_paged_slice();
- }
- }
-
- public void get_indexed_slices(ColumnParent column_parent, IndexClause index_clause, SlicePredicate column_predicate, ConsistencyLevel consistency_level, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException {
- checkReady();
- get_indexed_slices_call method_call = new get_indexed_slices_call(column_parent, index_clause, column_predicate, consistency_level, resultHandler, this, ___protocolFactory, ___transport);
- this.___currentMethod = method_call;
- ___manager.call(method_call);
- }
-
- public static class get_indexed_slices_call extends org.apache.thrift.async.TAsyncMethodCall {
- private ColumnParent column_parent;
- private IndexClause index_clause;
- private SlicePredicate column_predicate;
- private ConsistencyLevel consistency_level;
- public get_indexed_slices_call(ColumnParent column_parent, IndexClause index_clause, SlicePredicate column_predicate, ConsistencyLevel consistency_level, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
- super(client, protocolFactory, transport, resultHandler, false);
- this.column_parent = column_parent;
- this.index_clause = index_clause;
- this.column_predicate = column_predicate;
- this.consistency_level = consistency_level;
- }
-
- public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
- prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("get_indexed_slices", org.apache.thrift.protocol.TMessageType.CALL, 0));
- get_indexed_slices_args args = new get_indexed_slices_args();
- args.setColumn_parent(column_parent);
- args.setIndex_clause(index_clause);
- args.setColumn_predicate(column_predicate);
- args.setConsistency_level(consistency_level);
- args.write(prot);
- prot.writeMessageEnd();
- }
-
- public List<KeySlice> getResult() throws InvalidRequestException, UnavailableException, TimedOutException, org.apache.thrift.TException {
- if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
- throw new IllegalStateException("Method call not finished!");
- }
- org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
- org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
- return (new Client(prot)).recv_get_indexed_slices();
- }
- }
-
- public void insert(ByteBuffer key, ColumnParent column_parent, Column column, ConsistencyLevel consistency_level, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException {
- checkReady();
- insert_call method_call = new insert_call(key, column_parent, column, consistency_level, resultHandler, this, ___protocolFactory, ___transport);
- this.___currentMethod = method_call;
- ___manager.call(method_call);
- }
-
- public static class insert_call extends org.apache.thrift.async.TAsyncMethodCall {
- private ByteBuffer key;
- private ColumnParent column_parent;
- private Column column;
- private ConsistencyLevel consistency_level;
- public insert_call(ByteBuffer key, ColumnParent column_parent, Column column, ConsistencyLevel consistency_level, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
- super(client, protocolFactory, transport, resultHandler, false);
- this.key = key;
- this.column_parent = column_parent;
- this.column = column;
- this.consistency_level = consistency_level;
- }
-
- public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
- prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("insert", org.apache.thrift.protocol.TMessageType.CALL, 0));
- insert_args args = new insert_args();
- args.setKey(key);
- args.setColumn_parent(column_parent);
- args.setColumn(column);
- args.setConsistency_level(consistency_level);
- args.write(prot);
- prot.writeMessageEnd();
- }
-
- public void getResult() throws InvalidRequestException, UnavailableException, TimedOutException, org.apache.thrift.TException {
- if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
- throw new IllegalStateException("Method call not finished!");
- }
- org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
- org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
- (new Client(prot)).recv_insert();
- }
- }
-
- public void add(ByteBuffer key, ColumnParent column_parent, CounterColumn column, ConsistencyLevel consistency_level, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException {
- checkReady();
- add_call method_call = new add_call(key, column_parent, column, consistency_level, resultHandler, this, ___protocolFactory, ___transport);
- this.___currentMethod = method_call;
- ___manager.call(method_call);
- }
-
- public static class add_call extends org.apache.thrift.async.TAsyncMethodCall {
- private ByteBuffer key;
- private ColumnParent column_parent;
- private CounterColumn column;
- private ConsistencyLevel consistency_level;
- public add_call(ByteBuffer key, ColumnParent column_parent, CounterColumn column, ConsistencyLevel consistency_level, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
- super(client, protocolFactory, transport, resultHandler, false);
- this.key = key;
- this.column_parent = column_parent;
- this.column = column;
- this.consistency_level = consistency_level;
- }
-
- public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
- prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("add", org.apache.thrift.protocol.TMessageType.CALL, 0));
- add_args args = new add_args();
- args.setKey(key);
- args.setColumn_parent(column_parent);
- args.setColumn(column);
- args.setConsistency_level(consistency_level);
- args.write(prot);
- prot.writeMessageEnd();
- }
-
- public void getResult() throws InvalidRequestException, UnavailableException, TimedOutException, org.apache.thrift.TException {
- if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
- throw new IllegalStateException("Method call not finished!");
- }
- org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
- org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
- (new Client(prot)).recv_add();
- }
- }
-
- public void cas(ByteBuffer key, String column_family, List<Column> expected, List<Column> updates, ConsistencyLevel serial_consistency_level, ConsistencyLevel commit_consistency_level, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException {
- checkReady();
- cas_call method_call = new cas_call(key, column_family, expected, updates, serial_consistency_level, commit_consistency_level, resultHandler, this, ___protocolFactory, ___transport);
- this.___currentMethod = method_call;
- ___manager.call(method_call);
- }
-
- public static class cas_call extends org.apache.thrift.async.TAsyncMethodCall {
- private ByteBuffer key;
- private String column_family;
- private List<Column> expected;
- private List<Column> updates;
- private ConsistencyLevel serial_consistency_level;
- private ConsistencyLevel commit_consistency_level;
- public cas_call(ByteBuffer key, String column_family, List<Column> expected, List<Column> updates, ConsistencyLevel serial_consistency_level, ConsistencyLevel commit_consistency_level, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
- super(client, protocolFactory, transport, resultHandler, false);
- this.key = key;
- this.column_family = column_family;
- this.expected = expected;
- this.updates = updates;
- this.serial_consistency_level = serial_consistency_level;
- this.commit_consistency_level = commit_consistency_level;
- }
-
- public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
- prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("cas", org.apache.thrift.protocol.TMessageType.CALL, 0));
- cas_args args = new cas_args();
- args.setKey(key);
- args.setColumn_family(column_family);
- args.setExpected(expected);
- args.setUpdates(updates);
- args.setSerial_consistency_level(serial_consistency_level);
- args.setCommit_consistency_level(commit_consistency_level);
- args.write(prot);
- prot.writeMessageEnd();
- }
-
- public CASResult getResult() throws InvalidRequestException, UnavailableException, TimedOutException, org.apache.thrift.TException {
- if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
- throw new IllegalStateException("Method call not finished!");
- }
- org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
- org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
- return (new Client(prot)).recv_cas();
- }
- }
-
- public void remove(ByteBuffer key, ColumnPath column_path, long timestamp, ConsistencyLevel consistency_level, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException {
- checkReady();
- remove_call method_call = new remove_call(key, column_path, timestamp, consistency_level, resultHandler, this, ___protocolFactory, ___transport);
- this.___currentMethod = method_call;
- ___manager.call(method_call);
- }
-
- public static class remove_call extends org.apache.thrift.async.TAsyncMethodCall {
- private ByteBuffer key;
- private ColumnPath column_path;
- private long timestamp;
- private ConsistencyLevel consistency_level;
- public remove_call(ByteBuffer key, ColumnPath column_path, long timestamp, ConsistencyLevel consistency_level, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
- super(client, protocolFactory, transport, resultHandler, false);
- this.key = key;
- this.column_path = column_path;
- this.timestamp = timestamp;
- this.consistency_level = consistency_level;
- }
-
- public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
- prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("remove", org.apache.thrift.protocol.TMessageType.CALL, 0));
- remove_args args = new remove_args();
- args.setKey(key);
- args.setColumn_path(column_path);
- args.setTimestamp(timestamp);
- args.setConsistency_level(consistency_level);
- args.write(prot);
- prot.writeMessageEnd();
- }
-
- public void getResult() throws InvalidRequestException, UnavailableException, TimedOutException, org.apache.thrift.TException {
- if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
- throw new IllegalStateException("Method call not finished!");
- }
- org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
- org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
- (new Client(prot)).recv_remove();
- }
- }
-
- public void remove_counter(ByteBuffer key, ColumnPath path, ConsistencyLevel consistency_level, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException {
- checkReady();
- remove_counter_call method_call = new remove_counter_call(key, path, consistency_level, resultHandler, this, ___protocolFactory, ___transport);
- this.___currentMethod = method_call;
- ___manager.call(method_call);
- }
-
- public static class remove_counter_call extends org.apache.thrift.async.TAsyncMethodCall {
- private ByteBuffer key;
- private ColumnPath path;
- private ConsistencyLevel consistency_level;
- public remove_counter_call(ByteBuffer key, ColumnPath path, ConsistencyLevel consistency_level, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
- super(client, protocolFactory, transport, resultHandler, false);
- this.key = key;
- this.path = path;
- this.consistency_level = consistency_level;
- }
-
- public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
- prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("remove_counter", org.apache.thrift.protocol.TMessageType.CALL, 0));
- remove_counter_args args = new remove_counter_args();
- args.setKey(key);
- args.setPath(path);
- args.setConsistency_level(consistency_level);
- args.write(prot);
- prot.writeMessageEnd();
- }
-
- public void getResult() throws InvalidRequestException, UnavailableException, TimedOutException, org.apache.thrift.TException {
- if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
- throw new IllegalStateException("Method call not finished!");
- }
- org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
- org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
- (new Client(prot)).recv_remove_counter();
- }
- }
-
- public void batch_mutate(Map<ByteBuffer,Map<String,List<Mutation>>> mutation_map, ConsistencyLevel consistency_level, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException {
- checkReady();
- batch_mutate_call method_call = new batch_mutate_call(mutation_map, consistency_level, resultHandler, this, ___protocolFactory, ___transport);
- this.___currentMethod = method_call;
- ___manager.call(method_call);
- }
-
- public static class batch_mutate_call extends org.apache.thrift.async.TAsyncMethodCall {
- private Map<ByteBuffer,Map<String,List<Mutation>>> mutation_map;
- private ConsistencyLevel consistency_level;
- public batch_mutate_call(Map<ByteBuffer,Map<String,List<Mutation>>> mutation_map, ConsistencyLevel consistency_level, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
- super(client, protocolFactory, transport, resultHandler, false);
- this.mutation_map = mutation_map;
- this.consistency_level = consistency_level;
- }
-
- public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
- prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("batch_mutate", org.apache.thrift.protocol.TMessageType.CALL, 0));
- batch_mutate_args args = new batch_mutate_args();
- args.setMutation_map(mutation_map);
- args.setConsistency_level(consistency_level);
- args.write(prot);
- prot.writeMessageEnd();
- }
-
- public void getResult() throws InvalidRequestException, UnavailableException, TimedOutException, org.apache.thrift.TException {
- if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
- throw new IllegalStateException("Method call not finished!");
- }
- org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
- org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
- (new Client(prot)).recv_batch_mutate();
- }
- }
-
- public void atomic_batch_mutate(Map<ByteBuffer,Map<String,List<Mutation>>> mutation_map, ConsistencyLevel consistency_level, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException {
- checkReady();
- atomic_batch_mutate_call method_call = new atomic_batch_mutate_call(mutation_map, consistency_level, resultHandler, this, ___protocolFactory, ___transport);
- this.___currentMethod = method_call;
- ___manager.call(method_call);
- }
-
- public static class atomic_batch_mutate_call extends org.apache.thrift.async.TAsyncMethodCall {
- private Map<ByteBuffer,Map<String,List<Mutation>>> mutation_map;
- private ConsistencyLevel consistency_level;
- public atomic_batch_mutate_call(Map<ByteBuffer,Map<String,List<Mutation>>> mutation_map, ConsistencyLevel consistency_level, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
- super(client, protocolFactory, transport, resultHandler, false);
- this.mutation_map = mutation_map;
- this.consistency_level = consistency_level;
- }
-
- public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
- prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("atomic_batch_mutate", org.apache.thrift.protocol.TMessageType.CALL, 0));
- atomic_batch_mutate_args args = new atomic_batch_mutate_args();
- args.setMutation_map(mutation_map);
- args.setConsistency_level(consistency_level);
- args.write(prot);
- prot.writeMessageEnd();
- }
-
- public void getResult() throws InvalidRequestException, UnavailableException, TimedOutException, org.apache.thrift.TException {
- if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
- throw new IllegalStateException("Method call not finished!");
- }
- org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
- org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
- (new Client(prot)).recv_atomic_batch_mutate();
- }
- }
-
- public void truncate(String cfname, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException {
- checkReady();
- truncate_call method_call = new truncate_call(cfname, resultHandler, this, ___protocolFactory, ___transport);
- this.___currentMethod = method_call;
- ___manager.call(method_call);
- }
-
- public static class truncate_call extends org.apache.thrift.async.TAsyncMethodCall {
- private String cfname;
- public truncate_call(String cfname, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
- super(client, protocolFactory, transport, resultHandler, false);
- this.cfname = cfname;
- }
-
- public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
- prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("truncate", org.apache.thrift.protocol.TMessageType.CALL, 0));
- truncate_args args = new truncate_args();
- args.setCfname(cfname);
- args.write(prot);
- prot.writeMessageEnd();
- }
-
- public void getResult() throws InvalidRequestException, UnavailableException, TimedOutException, org.apache.thrift.TException {
- if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
- throw new IllegalStateException("Method call not finished!");
- }
- org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
- org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
- (new Client(prot)).recv_truncate();
- }
- }
-
- public void get_multi_slice(MultiSliceRequest request, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException {
- checkReady();
- get_multi_slice_call method_call = new get_multi_slice_call(request, resultHandler, this, ___protocolFactory, ___transport);
- this.___currentMethod = method_call;
- ___manager.call(method_call);
- }
-
- public static class get_multi_slice_call extends org.apache.thrift.async.TAsyncMethodCall {
- private MultiSliceRequest request;
- public get_multi_slice_call(MultiSliceRequest request, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
- super(client, protocolFactory, transport, resultHandler, false);
- this.request = request;
- }
-
- public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
- prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("get_multi_slice", org.apache.thrift.protocol.TMessageType.CALL, 0));
- get_multi_slice_args args = new get_multi_slice_args();
- args.setRequest(request);
- args.write(prot);
- prot.writeMessageEnd();
- }
-
- public List<ColumnOrSuperColumn> getResult() throws InvalidRequestException, UnavailableException, TimedOutException, org.apache.thrift.TException {
- if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
- throw new IllegalStateException("Method call not finished!");
- }
- org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
- org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
- return (new Client(prot)).recv_get_multi_slice();
- }
- }
-
- public void describe_schema_versions(org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException {
- checkReady();
- describe_schema_versions_call method_call = new describe_schema_versions_call(resultHandler, this, ___protocolFactory, ___transport);
- this.___currentMethod = method_call;
- ___manager.call(method_call);
- }
-
- public static class describe_schema_versions_call extends org.apache.thrift.async.TAsyncMethodCall {
- public describe_schema_versions_call(org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
- super(client, protocolFactory, transport, resultHandler, false);
- }
-
- public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
- prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("describe_schema_versions", org.apache.thrift.protocol.TMessageType.CALL, 0));
- describe_schema_versions_args args = new describe_schema_versions_args();
- args.write(prot);
- prot.writeMessageEnd();
- }
-
- public Map<String,List<String>> getResult() throws InvalidRequestException, org.apache.thrift.TException {
- if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
- throw new IllegalStateException("Method call not finished!");
- }
- org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
- org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
- return (new Client(prot)).recv_describe_schema_versions();
- }
- }
-
- public void describe_keyspaces(org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException {
- checkReady();
- describe_keyspaces_call method_call = new describe_keyspaces_call(resultHandler, this, ___protocolFactory, ___transport);
- this.___currentMethod = method_call;
- ___manager.call(method_call);
- }
-
- public static class describe_keyspaces_call extends org.apache.thrift.async.TAsyncMethodCall {
- public describe_keyspaces_call(org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
- super(client, protocolFactory, transport, resultHandler, false);
- }
-
- public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
- prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("describe_keyspaces", org.apache.thrift.protocol.TMessageType.CALL, 0));
- describe_keyspaces_args args = new describe_keyspaces_args();
- args.write(prot);
- prot.writeMessageEnd();
- }
-
- public List<KsDef> getResult() throws InvalidRequestException, org.apache.thrift.TException {
- if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
- throw new IllegalStateException("Method call not finished!");
- }
- org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
- org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
- return (new Client(prot)).recv_describe_keyspaces();
- }
- }
-
- public void describe_cluster_name(org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException {
- checkReady();
- describe_cluster_name_call method_call = new describe_cluster_name_call(resultHandler, this, ___protocolFactory, ___transport);
- this.___currentMethod = method_call;
- ___manager.call(method_call);
- }
-
- public static class describe_cluster_name_call extends org.apache.thrift.async.TAsyncMethodCall {
- public describe_cluster_name_call(org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
- super(client, protocolFactory, transport, resultHandler, false);
- }
-
- public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
- prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("describe_cluster_name", org.apache.thrift.protocol.TMessageType.CALL, 0));
- describe_cluster_name_args args = new describe_cluster_name_args();
- args.write(prot);
- prot.writeMessageEnd();
- }
-
- public String getResult() throws org.apache.thrift.TException {
- if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
- throw new IllegalStateException("Method call not finished!");
- }
- org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
- org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
- return (new Client(prot)).recv_describe_cluster_name();
- }
- }
-
- public void describe_version(org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException {
- checkReady();
- describe_version_call method_call = new describe_version_call(resultHandler, this, ___protocolFactory, ___transport);
- this.___currentMethod = method_call;
- ___manager.call(method_call);
- }
-
- public static class describe_version_call extends org.apache.thrift.async.TAsyncMethodCall {
- public describe_version_call(org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
- super(client, protocolFactory, transport, resultHandler, false);
- }
-
- public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
- prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("describe_version", org.apache.thrift.protocol.TMessageType.CALL, 0));
- describe_version_args args = new describe_version_args();
- args.write(prot);
- prot.writeMessageEnd();
- }
-
- public String getResult() throws org.apache.thrift.TException {
- if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
- throw new IllegalStateException("Method call not finished!");
- }
- org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
- org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
- return (new Client(prot)).recv_describe_version();
- }
- }
-
- public void describe_ring(String keyspace, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException {
- checkReady();
- describe_ring_call method_call = new describe_ring_call(keyspace, resultHandler, this, ___protocolFactory, ___transport);
- this.___currentMethod = method_call;
- ___manager.call(method_call);
- }
-
- public static class describe_ring_call extends org.apache.thrift.async.TAsyncMethodCall {
- private String keyspace;
- public describe_ring_call(String keyspace, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
- super(client, protocolFactory, transport, resultHandler, false);
- this.keyspace = keyspace;
- }
-
- public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
- prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("describe_ring", org.apache.thrift.protocol.TMessageType.CALL, 0));
- describe_ring_args args = new describe_ring_args();
- args.setKeyspace(keyspace);
- args.write(prot);
- prot.writeMessageEnd();
- }
-
- public List<TokenRange> getResult() throws InvalidRequestException, org.apache.thrift.TException {
- if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
- throw new IllegalStateException("Method call not finished!");
- }
- org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
- org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
- return (new Client(prot)).recv_describe_ring();
- }
- }
-
- public void describe_local_ring(String keyspace, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException {
- checkReady();
- describe_local_ring_call method_call = new describe_local_ring_call(keyspace, resultHandler, this, ___protocolFactory, ___transport);
- this.___currentMethod = method_call;
- ___manager.call(method_call);
- }
-
- public static class describe_local_ring_call extends org.apache.thrift.async.TAsyncMethodCall {
- private String keyspace;
- public describe_local_ring_call(String keyspace, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
- super(client, protocolFactory, transport, resultHandler, false);
- this.keyspace = keyspace;
- }
-
- public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
- prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("describe_local_ring", org.apache.thrift.protocol.TMessageType.CALL, 0));
- describe_local_ring_args args = new describe_local_ring_args();
- args.setKeyspace(keyspace);
- args.write(prot);
- prot.writeMessageEnd();
- }
-
- public List<TokenRange> getResult() throws InvalidRequestException, org.apache.thrift.TException {
- if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
- throw new IllegalStateException("Method call not finished!");
- }
- org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
- org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
- return (new Client(prot)).recv_describe_local_ring();
- }
- }
-
- public void describe_token_map(org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException {
- checkReady();
- describe_token_map_call method_call = new describe_token_map_call(resultHandler, this, ___protocolFactory, ___transport);
- this.___currentMethod = method_call;
- ___manager.call(method_call);
- }
-
- public static class describe_token_map_call extends org.apache.thrift.async.TAsyncMethodCall {
- public describe_token_map_call(org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
- super(client, protocolFactory, transport, resultHandler, false);
- }
-
- public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
- prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("describe_token_map", org.apache.thrift.protocol.TMessageType.CALL, 0));
- describe_token_map_args args = new describe_token_map_args();
- args.write(prot);
- prot.writeMessageEnd();
- }
-
- public Map<String,String> getResult() throws InvalidRequestException, org.apache.thrift.TException {
- if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
- throw new IllegalStateException("Method call not finished!");
- }
- org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
- org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
- return (new Client(prot)).recv_describe_token_map();
- }
- }
-
- public void describe_partitioner(org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException {
- checkReady();
- describe_partitioner_call method_call = new describe_partitioner_call(resultHandler, this, ___protocolFactory, ___transport);
- this.___currentMethod = method_call;
- ___manager.call(method_call);
- }
-
- public static class describe_partitioner_call extends org.apache.thrift.async.TAsyncMethodCall {
- public describe_partitioner_call(org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
- super(client, protocolFactory, transport, resultHandler, false);
- }
-
- public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
- prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("describe_partitioner", org.apache.thrift.protocol.TMessageType.CALL, 0));
- describe_partitioner_args args = new describe_partitioner_args();
- args.write(prot);
- prot.writeMessageEnd();
- }
-
- public String getResult() throws org.apache.thrift.TException {
- if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
- throw new IllegalStateException("Method call not finished!");
- }
- org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
- org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
- return (new Client(prot)).recv_describe_partitioner();
- }
- }
-
- public void describe_snitch(org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException {
- checkReady();
- describe_snitch_call method_call = new describe_snitch_call(resultHandler, this, ___protocolFactory, ___transport);
- this.___currentMethod = method_call;
- ___manager.call(method_call);
- }
-
- public static class describe_snitch_call extends org.apache.thrift.async.TAsyncMethodCall {
- public describe_snitch_call(org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
- super(client, protocolFactory, transport, resultHandler, false);
- }
-
- public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
- prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("describe_snitch", org.apache.thrift.protocol.TMessageType.CALL, 0));
- describe_snitch_args args = new describe_snitch_args();
- args.write(prot);
- prot.writeMessageEnd();
- }
-
- public String getResult() throws org.apache.thrift.TException {
- if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
- throw new IllegalStateException("Method call not finished!");
- }
- org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
- org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
- return (new Client(prot)).recv_describe_snitch();
- }
- }
-
- public void describe_keyspace(String keyspace, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException {
- checkReady();
- describe_keyspace_call method_call = new describe_keyspace_call(keyspace, resultHandler, this, ___protocolFactory, ___transport);
- this.___currentMethod = method_call;
- ___manager.call(method_call);
- }
-
- public static class describe_keyspace_call extends org.apache.thrift.async.TAsyncMethodCall {
- private String keyspace;
- public describe_keyspace_call(String keyspace, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
- super(client, protocolFactory, transport, resultHandler, false);
- this.keyspace = keyspace;
- }
-
- public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
- prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("describe_keyspace", org.apache.thrift.protocol.TMessageType.CALL, 0));
- describe_keyspace_args args = new describe_keyspace_args();
- args.setKeyspace(keyspace);
- args.write(prot);
- prot.writeMessageEnd();
- }
-
- public KsDef getResult() throws NotFoundException, InvalidRequestException, org.apache.thrift.TException {
- if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
- throw new IllegalStateException("Method call not finished!");
- }
- org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
- org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
- return (new Client(prot)).recv_describe_keyspace();
- }
- }
-
- public void describe_splits(String cfName, String start_token, String end_token, int keys_per_split, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException {
- checkReady();
- describe_splits_call method_call = new describe_splits_call(cfName, start_token, end_token, keys_per_split, resultHandler, this, ___protocolFactory, ___transport);
- this.___currentMethod = method_call;
- ___manager.call(method_call);
- }
-
- public static class describe_splits_call extends org.apache.thrift.async.TAsyncMethodCall {
- private String cfName;
- private String start_token;
- private String end_token;
- private int keys_per_split;
- public describe_splits_call(String cfName, String start_token, String end_token, int keys_per_split, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
- super(client, protocolFactory, transport, resultHandler, false);
- this.cfName = cfName;
- this.start_token = start_token;
- this.end_token = end_token;
- this.keys_per_split = keys_per_split;
- }
-
- public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
- prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("describe_splits", org.apache.thrift.protocol.TMessageType.CALL, 0));
- describe_splits_args args = new describe_splits_args();
- args.setCfName(cfName);
- args.setStart_token(start_token);
- args.setEnd_token(end_token);
- args.setKeys_per_split(keys_per_split);
- args.write(prot);
- prot.writeMessageEnd();
- }
-
- public List<String> getResult() throws InvalidRequestException, org.apache.thrift.TException {
- if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
- throw new IllegalStateException("Method call not finished!");
- }
- org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
- org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
- return (new Client(prot)).recv_describe_splits();
- }
- }
-
- public void trace_next_query(org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException {
- checkReady();
- trace_next_query_call method_call = new trace_next_query_call(resultHandler, this, ___protocolFactory, ___transport);
- this.___currentMethod = method_call;
- ___manager.call(method_call);
- }
-
- public static class trace_next_query_call extends org.apache.thrift.async.TAsyncMethodCall {
- public trace_next_query_call(org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
- super(client, protocolFactory, transport, resultHandler, false);
- }
-
- public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
- prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("trace_next_query", org.apache.thrift.protocol.TMessageType.CALL, 0));
- trace_next_query_args args = new trace_next_query_args();
- args.write(prot);
- prot.writeMessageEnd();
- }
-
- public ByteBuffer getResult() throws org.apache.thrift.TException {
- if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
- throw new IllegalStateException("Method call not finished!");
- }
- org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
- org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
- return (new Client(prot)).recv_trace_next_query();
- }
- }
-
- public void describe_splits_ex(String cfName, String start_token, String end_token, int keys_per_split, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException {
- checkReady();
- describe_splits_ex_call method_call = new describe_splits_ex_call(cfName, start_token, end_token, keys_per_split, resultHandler, this, ___protocolFactory, ___transport);
- this.___currentMethod = method_call;
- ___manager.call(method_call);
- }
-
- public static class describe_splits_ex_call extends org.apache.thrift.async.TAsyncMethodCall {
- private String cfName;
- private String start_token;
- private String end_token;
- private int keys_per_split;
- public describe_splits_ex_call(String cfName, String start_token, String end_token, int keys_per_split, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
- super(client, protocolFactory, transport, resultHandler, false);
- this.cfName = cfName;
- this.start_token = start_token;
- this.end_token = end_token;
- this.keys_per_split = keys_per_split;
- }
-
- public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
- prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("describe_splits_ex", org.apache.thrift.protocol.TMessageType.CALL, 0));
- describe_splits_ex_args args = new describe_splits_ex_args();
- args.setCfName(cfName);
- args.setStart_token(start_token);
- args.setEnd_token(end_token);
- args.setKeys_per_split(keys_per_split);
- args.write(prot);
- prot.writeMessageEnd();
- }
-
- public List<CfSplit> getResult() throws InvalidRequestException, org.apache.thrift.TException {
- if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
- throw new IllegalStateException("Method call not finished!");
- }
- org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
- org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
- return (new Client(prot)).recv_describe_splits_ex();
- }
- }
-
- public void system_add_column_family(CfDef cf_def, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException {
- checkReady();
- system_add_column_family_call method_call = new system_add_column_family_call(cf_def, resultHandler, this, ___protocolFactory, ___transport);
- this.___currentMethod = method_call;
- ___manager.call(method_call);
- }
-
- public static class system_add_column_family_call extends org.apache.thrift.async.TAsyncMethodCall {
- private CfDef cf_def;
- public system_add_column_family_call(CfDef cf_def, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
- super(client, protocolFactory, transport, resultHandler, false);
- this.cf_def = cf_def;
- }
-
- public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
- prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("system_add_column_family", org.apache.thrift.protocol.TMessageType.CALL, 0));
- system_add_column_family_args args = new system_add_column_family_args();
- args.setCf_def(cf_def);
- args.write(prot);
- prot.writeMessageEnd();
- }
-
- public String getResult() throws InvalidRequestException, SchemaDisagreementException, org.apache.thrift.TException {
- if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
- throw new IllegalStateException("Method call not finished!");
- }
- org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
- org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
- return (new Client(prot)).recv_system_add_column_family();
- }
- }
-
- public void system_drop_column_family(String column_family, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException {
- checkReady();
- system_drop_column_family_call method_call = new system_drop_column_family_call(column_family, resultHandler, this, ___protocolFactory, ___transport);
- this.___currentMethod = method_call;
- ___manager.call(method_call);
- }
-
- public static class system_drop_column_family_call extends org.apache.thrift.async.TAsyncMethodCall {
- private String column_family;
- public system_drop_column_family_call(String column_family, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
- super(client, protocolFactory, transport, resultHandler, false);
- this.column_family = column_family;
- }
-
- public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
- prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("system_drop_column_family", org.apache.thrift.protocol.TMessageType.CALL, 0));
- system_drop_column_family_args args = new system_drop_column_family_args();
- args.setColumn_family(column_family);
- args.write(prot);
- prot.writeMessageEnd();
- }
-
- public String getResult() throws InvalidRequestException, SchemaDisagreementException, org.apache.thrift.TException {
- if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
- throw new IllegalStateException("Method call not finished!");
- }
- org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
- org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
- return (new Client(prot)).recv_system_drop_column_family();
- }
- }
-
- public void system_add_keyspace(KsDef ks_def, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException {
- checkReady();
- system_add_keyspace_call method_call = new system_add_keyspace_call(ks_def, resultHandler, this, ___protocolFactory, ___transport);
- this.___currentMethod = method_call;
- ___manager.call(method_call);
- }
-
- public static class system_add_keyspace_call extends org.apache.thrift.async.TAsyncMethodCall {
- private KsDef ks_def;
- public system_add_keyspace_call(KsDef ks_def, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
- super(client, protocolFactory, transport, resultHandler, false);
- this.ks_def = ks_def;
- }
-
- public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
- prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("system_add_keyspace", org.apache.thrift.protocol.TMessageType.CALL, 0));
- system_add_keyspace_args args = new system_add_keyspace_args();
- args.setKs_def(ks_def);
- args.write(prot);
- prot.writeMessageEnd();
- }
-
- public String getResult() throws InvalidRequestException, SchemaDisagreementException, org.apache.thrift.TException {
- if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
- throw new IllegalStateException("Method call not finished!");
- }
- org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
- org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
- return (new Client(prot)).recv_system_add_keyspace();
- }
- }
-
- public void system_drop_keyspace(String keyspace, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException {
- checkReady();
- system_drop_keyspace_call method_call = new system_drop_keyspace_call(keyspace, resultHandler, this, ___protocolFactory, ___transport);
- this.___currentMethod = method_call;
- ___manager.call(method_call);
- }
-
- public static class system_drop_keyspace_call extends org.apache.thrift.async.TAsyncMethodCall {
- private String keyspace;
- public system_drop_keyspace_call(String keyspace, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
- super(client, protocolFactory, transport, resultHandler, false);
- this.keyspace = keyspace;
- }
-
- public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
- prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("system_drop_keyspace", org.apache.thrift.protocol.TMessageType.CALL, 0));
- system_drop_keyspace_args args = new system_drop_keyspace_args();
- args.setKeyspace(keyspace);
- args.write(prot);
- prot.writeMessageEnd();
- }
-
- public String getResult() throws InvalidRequestException, SchemaDisagreementException, org.apache.thrift.TException {
- if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
- throw new IllegalStateException("Method call not finished!");
- }
- org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
- org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
- return (new Client(prot)).recv_system_drop_keyspace();
- }
- }
-
- public void system_update_keyspace(KsDef ks_def, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException {
- checkReady();
- system_update_keyspace_call method_call = new system_update_keyspace_call(ks_def, resultHandler, this, ___protocolFactory, ___transport);
- this.___currentMethod = method_call;
- ___manager.call(method_call);
- }
-
- public static class system_update_keyspace_call extends org.apache.thrift.async.TAsyncMethodCall {
- private KsDef ks_def;
- public system_update_keyspace_call(KsDef ks_def, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
- super(client, protocolFactory, transport, resultHandler, false);
- this.ks_def = ks_def;
- }
-
- public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
- prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("system_update_keyspace", org.apache.thrift.protocol.TMessageType.CALL, 0));
- system_update_keyspace_args args = new system_update_keyspace_args();
- args.setKs_def(ks_def);
- args.write(prot);
- prot.writeMessageEnd();
- }
-
- public String getResult() throws InvalidRequestException, SchemaDisagreementException, org.apache.thrift.TException {
- if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
- throw new IllegalStateException("Method call not finished!");
- }
- org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
- org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
- return (new Client(prot)).recv_system_update_keyspace();
- }
- }
-
- public void system_update_column_family(CfDef cf_def, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException {
- checkReady();
- system_update_column_family_call method_call = new system_update_column_family_call(cf_def, resultHandler, this, ___protocolFactory, ___transport);
- this.___currentMethod = method_call;
- ___manager.call(method_call);
- }
-
- public static class system_update_column_family_call extends org.apache.thrift.async.TAsyncMethodCall {
- private CfDef cf_def;
- public system_update_column_family_call(CfDef cf_def, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
- super(client, protocolFactory, transport, resultHandler, false);
- this.cf_def = cf_def;
- }
-
- public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
- prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("system_update_column_family", org.apache.thrift.protocol.TMessageType.CALL, 0));
- system_update_column_family_args args = new system_update_column_family_args();
- args.setCf_def(cf_def);
- args.write(prot);
- prot.writeMessageEnd();
- }
-
- public String getResult() throws InvalidRequestException, SchemaDisagreementException, org.apache.thrift.TException {
- if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
- throw new IllegalStateException("Method call not finished!");
- }
- org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
- org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
- return (new Client(prot)).recv_system_update_column_family();
- }
- }
-
- public void execute_cql_query(ByteBuffer query, Compression compression, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException {
- checkReady();
- execute_cql_query_call method_call = new execute_cql_query_call(query, compression, resultHandler, this, ___protocolFactory, ___transport);
- this.___currentMethod = method_call;
- ___manager.call(method_call);
- }
-
- public static class execute_cql_query_call extends org.apache.thrift.async.TAsyncMethodCall {
- private ByteBuffer query;
- private Compression compression;
- public execute_cql_query_call(ByteBuffer query, Compression compression, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
- super(client, protocolFactory, transport, resultHandler, false);
- this.query = query;
- this.compression = compression;
- }
-
- public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
- prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("execute_cql_query", org.apache.thrift.protocol.TMessageType.CALL, 0));
- execute_cql_query_args args = new execute_cql_query_args();
- args.setQuery(query);
- args.setCompression(compression);
- args.write(prot);
- prot.writeMessageEnd();
- }
-
- public CqlResult getResult() throws InvalidRequestException, UnavailableException, TimedOutException, SchemaDisagreementException, org.apache.thrift.TException {
- if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
- throw new IllegalStateException("Method call not finished!");
- }
- org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
- org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
- return (new Client(prot)).recv_execute_cql_query();
- }
- }
-
- public void execute_cql3_query(ByteBuffer query, Compression compression, ConsistencyLevel consistency, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException {
- checkReady();
- execute_cql3_query_call method_call = new execute_cql3_query_call(query, compression, consistency, resultHandler, this, ___protocolFactory, ___transport);
- this.___currentMethod = method_call;
- ___manager.call(method_call);
- }
-
- public static class execute_cql3_query_call extends org.apache.thrift.async.TAsyncMethodCall {
- private ByteBuffer query;
- private Compression compression;
- private ConsistencyLevel consistency;
- public execute_cql3_query_call(ByteBuffer query, Compression compression, ConsistencyLevel consistency, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
- super(client, protocolFactory, transport, resultHandler, false);
- this.query = query;
- this.compression = compression;
- this.consistency = consistency;
- }
-
- public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
- prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("execute_cql3_query", org.apache.thrift.protocol.TMessageType.CALL, 0));
- execute_cql3_query_args args = new execute_cql3_query_args();
- args.setQuery(query);
- args.setCompression(compression);
- args.setConsistency(consistency);
- args.write(prot);
- prot.writeMessageEnd();
- }
-
- public CqlResult getResult() throws InvalidRequestException, UnavailableException, TimedOutException, SchemaDisagreementException, org.apache.thrift.TException {
- if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
- throw new IllegalStateException("Method call not finished!");
- }
- org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
- org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
- return (new Client(prot)).recv_execute_cql3_query();
- }
- }
-
- public void prepare_cql_query(ByteBuffer query, Compression compression, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException {
- checkReady();
- prepare_cql_query_call method_call = new prepare_cql_query_call(query, compression, resultHandler, this, ___protocolFactory, ___transport);
- this.___currentMethod = method_call;
- ___manager.call(method_call);
- }
-
- public static class prepare_cql_query_call extends org.apache.thrift.async.TAsyncMethodCall {
- private ByteBuffer query;
- private Compression compression;
- public prepare_cql_query_call(ByteBuffer query, Compression compression, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
- super(client, protocolFactory, transport, resultHandler, false);
- this.query = query;
- this.compression = compression;
- }
-
- public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
- prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("prepare_cql_query", org.apache.thrift.protocol.TMessageType.CALL, 0));
- prepare_cql_query_args args = new prepare_cql_query_args();
- args.setQuery(query);
- args.setCompression(compression);
- args.write(prot);
- prot.writeMessageEnd();
- }
-
- public CqlPreparedResult getResult() throws InvalidRequestException, org.apache.thrift.TException {
- if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
- throw new IllegalStateException("Method call not finished!");
- }
- org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
- org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
- return (new Client(prot)).recv_prepare_cql_query();
- }
- }
-
- public void prepare_cql3_query(ByteBuffer query, Compression compression, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException {
- checkReady();
- prepare_cql3_query_call method_call = new prepare_cql3_query_call(query, compression, resultHandler, this, ___protocolFactory, ___transport);
- this.___currentMethod = method_call;
- ___manager.call(method_call);
- }
-
- public static class prepare_cql3_query_call extends org.apache.thrift.async.TAsyncMethodCall {
- private ByteBuffer query;
- private Compression compression;
- public prepare_cql3_query_call(ByteBuffer query, Compression compression, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
- super(client, protocolFactory, transport, resultHandler, false);
- this.query = query;
- this.compression = compression;
- }
-
- public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
- prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("prepare_cql3_query", org.apache.thrift.protocol.TMessageType.CALL, 0));
- prepare_cql3_query_args args = new prepare_cql3_query_args();
- args.setQuery(query);
- args.setCompression(compression);
- args.write(prot);
- prot.writeMessageEnd();
- }
-
- public CqlPreparedResult getResult() throws InvalidRequestException, org.apache.thrift.TException {
- if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
- throw new IllegalStateException("Method call not finished!");
- }
- org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
- org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
- return (new Client(prot)).recv_prepare_cql3_query();
- }
- }
-
- public void execute_prepared_cql_query(int itemId, List<ByteBuffer> values, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException {
- checkReady();
- execute_prepared_cql_query_call method_call = new execute_prepared_cql_query_call(itemId, values, resultHandler, this, ___protocolFactory, ___transport);
- this.___currentMethod = method_call;
- ___manager.call(method_call);
- }
-
- public static class execute_prepared_cql_query_call extends org.apache.thrift.async.TAsyncMethodCall {
- private int itemId;
- private List<ByteBuffer> values;
- public execute_prepared_cql_query_call(int itemId, List<ByteBuffer> values, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
- super(client, protocolFactory, transport, resultHandler, false);
- this.itemId = itemId;
- this.values = values;
- }
-
- public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
- prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("execute_prepared_cql_query", org.apache.thrift.protocol.TMessageType.CALL, 0));
- execute_prepared_cql_query_args args = new execute_prepared_cql_query_args();
- args.setItemId(itemId);
- args.setValues(values);
- args.write(prot);
- prot.writeMessageEnd();
- }
-
- public CqlResult getResult() throws InvalidRequestException, UnavailableException, TimedOutException, SchemaDisagreementException, org.apache.thrift.TException {
- if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
- throw new IllegalStateException("Method call not finished!");
- }
- org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
- org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
- return (new Client(prot)).recv_execute_prepared_cql_query();
- }
- }
-
- public void execute_prepared_cql3_query(int itemId, List<ByteBuffer> values, ConsistencyLevel consistency, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException {
- checkReady();
- execute_prepared_cql3_query_call method_call = new execute_prepared_cql3_query_call(itemId, values, consistency, resultHandler, this, ___protocolFactory, ___transport);
- this.___currentMethod = method_call;
- ___manager.call(method_call);
- }
-
- public static class execute_prepared_cql3_query_call extends org.apache.thrift.async.TAsyncMethodCall {
- private int itemId;
- private List<ByteBuffer> values;
- private ConsistencyLevel consistency;
- public execute_prepared_cql3_query_call(int itemId, List<ByteBuffer> values, ConsistencyLevel consistency, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
- super(client, protocolFactory, transport, resultHandler, false);
- this.itemId = itemId;
- this.values = values;
- this.consistency = consistency;
- }
-
- public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
- prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("execute_prepared_cql3_query", org.apache.thrift.protocol.TMessageType.CALL, 0));
- execute_prepared_cql3_query_args args = new execute_prepared_cql3_query_args();
- args.setItemId(itemId);
- args.setValues(values);
- args.setConsistency(consistency);
- args.write(prot);
- prot.writeMessageEnd();
- }
-
- public CqlResult getResult() throws InvalidRequestException, UnavailableException, TimedOutException, SchemaDisagreementException, org.apache.thrift.TException {
- if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
- throw new IllegalStateException("Method call not finished!");
- }
- org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
- org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
- return (new Client(prot)).recv_execute_prepared_cql3_query();
- }
- }
-
- public void set_cql_version(String version, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException {
- checkReady();
- set_cql_version_call method_call = new set_cql_version_call(version, resultHandler, this, ___protocolFactory, ___transport);
- this.___currentMethod = method_call;
- ___manager.call(method_call);
- }
-
- public static class set_cql_version_call extends org.apache.thrift.async.TAsyncMethodCall {
- private String version;
- public set_cql_version_call(String version, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
- super(client, protocolFactory, transport, resultHandler, false);
- this.version = version;
- }
-
- public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
- prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("set_cql_version", org.apache.thrift.protocol.TMessageType.CALL, 0));
- set_cql_version_args args = new set_cql_version_args();
- args.setVersion(version);
- args.write(prot);
- prot.writeMessageEnd();
- }
-
- public void getResult() throws InvalidRequestException, org.apache.thrift.TException {
- if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
- throw new IllegalStateException("Method call not finished!");
- }
- org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
- org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
- (new Client(prot)).recv_set_cql_version();
- }
- }
-
- }
-
- public static class Processor<I extends Iface> extends org.apache.thrift.TBaseProcessor<I> implements org.apache.thrift.TProcessor {
- private static final Logger LOGGER = LoggerFactory.getLogger(Processor.class.getName());
- public Processor(I iface) {
- super(iface, getProcessMap(new HashMap<String, org.apache.thrift.ProcessFunction<I, ? extends org.apache.thrift.TBase>>()));
- }
-
- protected Processor(I iface, Map<String, org.apache.thrift.ProcessFunction<I, ? extends org.apache.thrift.TBase>> processMap) {
- super(iface, getProcessMap(processMap));
- }
-
- private static <I extends Iface> Map<String, org.apache.thrift.ProcessFunction<I, ? extends org.apache.thrift.TBase>> getProcessMap(Map<String, org.apache.thrift.ProcessFunction<I, ? extends org.apache.thrift.TBase>> processMap) {
- processMap.put("login", new login());
- processMap.put("set_keyspace", new set_keyspace());
- processMap.put("get", new get());
- processMap.put("get_slice", new get_slice());
- processMap.put("get_count", new get_count());
- processMap.put("multiget_slice", new multiget_slice());
- processMap.put("multiget_count", new multiget_count());
- processMap.put("get_range_slices", new get_range_slices());
- processMap.put("get_paged_slice", new get_paged_slice());
- processMap.put("get_indexed_slices", new get_indexed_slices());
- processMap.put("insert", new insert());
- processMap.put("add", new add());
- processMap.put("cas", new cas());
- processMap.put("remove", new remove());
- processMap.put("remove_counter", new remove_counter());
- processMap.put("batch_mutate", new batch_mutate());
- processMap.put("atomic_batch_mutate", new atomic_batch_mutate());
- processMap.put("truncate", new truncate());
- processMap.put("get_multi_slice", new get_multi_slice());
- processMap.put("describe_schema_versions", new describe_schema_versions());
- processMap.put("describe_keyspaces", new describe_keyspaces());
- processMap.put("describe_cluster_name", new describe_cluster_name());
- processMap.put("describe_version", new describe_version());
- processMap.put("describe_ring", new describe_ring());
- processMap.put("describe_local_ring", new describe_local_ring());
- processMap.put("describe_token_map", new describe_token_map());
- processMap.put("describe_partitioner", new describe_partitioner());
- processMap.put("describe_snitch", new describe_snitch());
- processMap.put("describe_keyspace", new describe_keyspace());
- processMap.put("describe_splits", new describe_splits());
- processMap.put("trace_next_query", new trace_next_query());
- processMap.put("describe_splits_ex", new describe_splits_ex());
- processMap.put("system_add_column_family", new system_add_column_family());
- processMap.put("system_drop_column_family", new system_drop_column_family());
- processMap.put("system_add_keyspace", new system_add_keyspace());
- processMap.put("system_drop_keyspace", new system_drop_keyspace());
- processMap.put("system_update_keyspace", new system_update_keyspace());
- processMap.put("system_update_column_family", new system_update_column_family());
- processMap.put("execute_cql_query", new execute_cql_query());
- processMap.put("execute_cql3_query", new execute_cql3_query());
- processMap.put("prepare_cql_query", new prepare_cql_query());
- processMap.put("prepare_cql3_query", new prepare_cql3_query());
- processMap.put("execute_prepared_cql_query", new execute_prepared_cql_query());
- processMap.put("execute_prepared_cql3_query", new execute_prepared_cql3_query());
- processMap.put("set_cql_version", new set_cql_version());
- return processMap;
- }
-
- public static class login<I extends Iface> extends org.apache.thrift.ProcessFunction<I, login_args> {
- public login() {
- super("login");
- }
-
- public login_args getEmptyArgsInstance() {
- return new login_args();
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public login_result getResult(I iface, login_args args) throws org.apache.thrift.TException {
- login_result result = new login_result();
- try {
- iface.login(args.auth_request);
- } catch (AuthenticationException authnx) {
- result.authnx = authnx;
- } catch (AuthorizationException authzx) {
- result.authzx = authzx;
- }
- return result;
- }
- }
-
- public static class set_keyspace<I extends Iface> extends org.apache.thrift.ProcessFunction<I, set_keyspace_args> {
- public set_keyspace() {
- super("set_keyspace");
- }
-
- public set_keyspace_args getEmptyArgsInstance() {
- return new set_keyspace_args();
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public set_keyspace_result getResult(I iface, set_keyspace_args args) throws org.apache.thrift.TException {
- set_keyspace_result result = new set_keyspace_result();
- try {
- iface.set_keyspace(args.keyspace);
- } catch (InvalidRequestException ire) {
- result.ire = ire;
- }
- return result;
- }
- }
-
- public static class get<I extends Iface> extends org.apache.thrift.ProcessFunction<I, get_args> {
- public get() {
- super("get");
- }
-
- public get_args getEmptyArgsInstance() {
- return new get_args();
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public get_result getResult(I iface, get_args args) throws org.apache.thrift.TException {
- get_result result = new get_result();
- try {
- result.success = iface.get(args.key, args.column_path, args.consistency_level);
- } catch (InvalidRequestException ire) {
- result.ire = ire;
- } catch (NotFoundException nfe) {
- result.nfe = nfe;
- } catch (UnavailableException ue) {
- result.ue = ue;
- } catch (TimedOutException te) {
- result.te = te;
- }
- return result;
- }
- }
-
- public static class get_slice<I extends Iface> extends org.apache.thrift.ProcessFunction<I, get_slice_args> {
- public get_slice() {
- super("get_slice");
- }
-
- public get_slice_args getEmptyArgsInstance() {
- return new get_slice_args();
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public get_slice_result getResult(I iface, get_slice_args args) throws org.apache.thrift.TException {
- get_slice_result result = new get_slice_result();
- try {
- result.success = iface.get_slice(args.key, args.column_parent, args.predicate, args.consistency_level);
- } catch (InvalidRequestException ire) {
- result.ire = ire;
- } catch (UnavailableException ue) {
- result.ue = ue;
- } catch (TimedOutException te) {
- result.te = te;
- }
- return result;
- }
- }
-
- public static class get_count<I extends Iface> extends org.apache.thrift.ProcessFunction<I, get_count_args> {
- public get_count() {
- super("get_count");
- }
-
- public get_count_args getEmptyArgsInstance() {
- return new get_count_args();
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public get_count_result getResult(I iface, get_count_args args) throws org.apache.thrift.TException {
- get_count_result result = new get_count_result();
- try {
- result.success = iface.get_count(args.key, args.column_parent, args.predicate, args.consistency_level);
- result.setSuccessIsSet(true);
- } catch (InvalidRequestException ire) {
- result.ire = ire;
- } catch (UnavailableException ue) {
- result.ue = ue;
- } catch (TimedOutException te) {
- result.te = te;
- }
- return result;
- }
- }
-
- public static class multiget_slice<I extends Iface> extends org.apache.thrift.ProcessFunction<I, multiget_slice_args> {
- public multiget_slice() {
- super("multiget_slice");
- }
-
- public multiget_slice_args getEmptyArgsInstance() {
- return new multiget_slice_args();
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public multiget_slice_result getResult(I iface, multiget_slice_args args) throws org.apache.thrift.TException {
- multiget_slice_result result = new multiget_slice_result();
- try {
- result.success = iface.multiget_slice(args.keys, args.column_parent, args.predicate, args.consistency_level);
- } catch (InvalidRequestException ire) {
- result.ire = ire;
- } catch (UnavailableException ue) {
- result.ue = ue;
- } catch (TimedOutException te) {
- result.te = te;
- }
- return result;
- }
- }
-
- public static class multiget_count<I extends Iface> extends org.apache.thrift.ProcessFunction<I, multiget_count_args> {
- public multiget_count() {
- super("multiget_count");
- }
-
- public multiget_count_args getEmptyArgsInstance() {
- return new multiget_count_args();
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public multiget_count_result getResult(I iface, multiget_count_args args) throws org.apache.thrift.TException {
- multiget_count_result result = new multiget_count_result();
- try {
- result.success = iface.multiget_count(args.keys, args.column_parent, args.predicate, args.consistency_level);
- } catch (InvalidRequestException ire) {
- result.ire = ire;
- } catch (UnavailableException ue) {
- result.ue = ue;
- } catch (TimedOutException te) {
- result.te = te;
- }
- return result;
- }
- }
-
- public static class get_range_slices<I extends Iface> extends org.apache.thrift.ProcessFunction<I, get_range_slices_args> {
- public get_range_slices() {
- super("get_range_slices");
- }
-
- public get_range_slices_args getEmptyArgsInstance() {
- return new get_range_slices_args();
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public get_range_slices_result getResult(I iface, get_range_slices_args args) throws org.apache.thrift.TException {
- get_range_slices_result result = new get_range_slices_result();
- try {
- result.success = iface.get_range_slices(args.column_parent, args.predicate, args.range, args.consistency_level);
- } catch (InvalidRequestException ire) {
- result.ire = ire;
- } catch (UnavailableException ue) {
- result.ue = ue;
- } catch (TimedOutException te) {
- result.te = te;
- }
- return result;
- }
- }
-
- public static class get_paged_slice<I extends Iface> extends org.apache.thrift.ProcessFunction<I, get_paged_slice_args> {
- public get_paged_slice() {
- super("get_paged_slice");
- }
-
- public get_paged_slice_args getEmptyArgsInstance() {
- return new get_paged_slice_args();
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public get_paged_slice_result getResult(I iface, get_paged_slice_args args) throws org.apache.thrift.TException {
- get_paged_slice_result result = new get_paged_slice_result();
- try {
- result.success = iface.get_paged_slice(args.column_family, args.range, args.start_column, args.consistency_level);
- } catch (InvalidRequestException ire) {
- result.ire = ire;
- } catch (UnavailableException ue) {
- result.ue = ue;
- } catch (TimedOutException te) {
- result.te = te;
- }
- return result;
- }
- }
-
- public static class get_indexed_slices<I extends Iface> extends org.apache.thrift.ProcessFunction<I, get_indexed_slices_args> {
- public get_indexed_slices() {
- super("get_indexed_slices");
- }
-
- public get_indexed_slices_args getEmptyArgsInstance() {
- return new get_indexed_slices_args();
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public get_indexed_slices_result getResult(I iface, get_indexed_slices_args args) throws org.apache.thrift.TException {
- get_indexed_slices_result result = new get_indexed_slices_result();
- try {
- result.success = iface.get_indexed_slices(args.column_parent, args.index_clause, args.column_predicate, args.consistency_level);
- } catch (InvalidRequestException ire) {
- result.ire = ire;
- } catch (UnavailableException ue) {
- result.ue = ue;
- } catch (TimedOutException te) {
- result.te = te;
- }
- return result;
- }
- }
-
- public static class insert<I extends Iface> extends org.apache.thrift.ProcessFunction<I, insert_args> {
- public insert() {
- super("insert");
- }
-
- public insert_args getEmptyArgsInstance() {
- return new insert_args();
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public insert_result getResult(I iface, insert_args args) throws org.apache.thrift.TException {
- insert_result result = new insert_result();
- try {
- iface.insert(args.key, args.column_parent, args.column, args.consistency_level);
- } catch (InvalidRequestException ire) {
- result.ire = ire;
- } catch (UnavailableException ue) {
- result.ue = ue;
- } catch (TimedOutException te) {
- result.te = te;
- }
- return result;
- }
- }
-
- public static class add<I extends Iface> extends org.apache.thrift.ProcessFunction<I, add_args> {
- public add() {
- super("add");
- }
-
- public add_args getEmptyArgsInstance() {
- return new add_args();
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public add_result getResult(I iface, add_args args) throws org.apache.thrift.TException {
- add_result result = new add_result();
- try {
- iface.add(args.key, args.column_parent, args.column, args.consistency_level);
- } catch (InvalidRequestException ire) {
- result.ire = ire;
- } catch (UnavailableException ue) {
- result.ue = ue;
- } catch (TimedOutException te) {
- result.te = te;
- }
- return result;
- }
- }
-
- public static class cas<I extends Iface> extends org.apache.thrift.ProcessFunction<I, cas_args> {
- public cas() {
- super("cas");
- }
-
- public cas_args getEmptyArgsInstance() {
- return new cas_args();
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public cas_result getResult(I iface, cas_args args) throws org.apache.thrift.TException {
- cas_result result = new cas_result();
- try {
- result.success = iface.cas(args.key, args.column_family, args.expected, args.updates, args.serial_consistency_level, args.commit_consistency_level);
- } catch (InvalidRequestException ire) {
- result.ire = ire;
- } catch (UnavailableException ue) {
- result.ue = ue;
- } catch (TimedOutException te) {
- result.te = te;
- }
- return result;
- }
- }
-
- public static class remove<I extends Iface> extends org.apache.thrift.ProcessFunction<I, remove_args> {
- public remove() {
- super("remove");
- }
-
- public remove_args getEmptyArgsInstance() {
- return new remove_args();
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public remove_result getResult(I iface, remove_args args) throws org.apache.thrift.TException {
- remove_result result = new remove_result();
- try {
- iface.remove(args.key, args.column_path, args.timestamp, args.consistency_level);
- } catch (InvalidRequestException ire) {
- result.ire = ire;
- } catch (UnavailableException ue) {
- result.ue = ue;
- } catch (TimedOutException te) {
- result.te = te;
- }
- return result;
- }
- }
-
- public static class remove_counter<I extends Iface> extends org.apache.thrift.ProcessFunction<I, remove_counter_args> {
- public remove_counter() {
- super("remove_counter");
- }
-
- public remove_counter_args getEmptyArgsInstance() {
- return new remove_counter_args();
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public remove_counter_result getResult(I iface, remove_counter_args args) throws org.apache.thrift.TException {
- remove_counter_result result = new remove_counter_result();
- try {
- iface.remove_counter(args.key, args.path, args.consistency_level);
- } catch (InvalidRequestException ire) {
- result.ire = ire;
- } catch (UnavailableException ue) {
- result.ue = ue;
- } catch (TimedOutException te) {
- result.te = te;
- }
- return result;
- }
- }
-
- public static class batch_mutate<I extends Iface> extends org.apache.thrift.ProcessFunction<I, batch_mutate_args> {
- public batch_mutate() {
- super("batch_mutate");
- }
-
- public batch_mutate_args getEmptyArgsInstance() {
- return new batch_mutate_args();
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public batch_mutate_result getResult(I iface, batch_mutate_args args) throws org.apache.thrift.TException {
- batch_mutate_result result = new batch_mutate_result();
- try {
- iface.batch_mutate(args.mutation_map, args.consistency_level);
- } catch (InvalidRequestException ire) {
- result.ire = ire;
- } catch (UnavailableException ue) {
- result.ue = ue;
- } catch (TimedOutException te) {
- result.te = te;
- }
- return result;
- }
- }
-
- public static class atomic_batch_mutate<I extends Iface> extends org.apache.thrift.ProcessFunction<I, atomic_batch_mutate_args> {
- public atomic_batch_mutate() {
- super("atomic_batch_mutate");
- }
-
- public atomic_batch_mutate_args getEmptyArgsInstance() {
- return new atomic_batch_mutate_args();
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public atomic_batch_mutate_result getResult(I iface, atomic_batch_mutate_args args) throws org.apache.thrift.TException {
- atomic_batch_mutate_result result = new atomic_batch_mutate_result();
- try {
- iface.atomic_batch_mutate(args.mutation_map, args.consistency_level);
- } catch (InvalidRequestException ire) {
- result.ire = ire;
- } catch (UnavailableException ue) {
- result.ue = ue;
- } catch (TimedOutException te) {
- result.te = te;
- }
- return result;
- }
- }
-
- public static class truncate<I extends Iface> extends org.apache.thrift.ProcessFunction<I, truncate_args> {
- public truncate() {
- super("truncate");
- }
-
- public truncate_args getEmptyArgsInstance() {
- return new truncate_args();
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public truncate_result getResult(I iface, truncate_args args) throws org.apache.thrift.TException {
- truncate_result result = new truncate_result();
- try {
- iface.truncate(args.cfname);
- } catch (InvalidRequestException ire) {
- result.ire = ire;
- } catch (UnavailableException ue) {
- result.ue = ue;
- } catch (TimedOutException te) {
- result.te = te;
- }
- return result;
- }
- }
-
- public static class get_multi_slice<I extends Iface> extends org.apache.thrift.ProcessFunction<I, get_multi_slice_args> {
- public get_multi_slice() {
- super("get_multi_slice");
- }
-
- public get_multi_slice_args getEmptyArgsInstance() {
- return new get_multi_slice_args();
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public get_multi_slice_result getResult(I iface, get_multi_slice_args args) throws org.apache.thrift.TException {
- get_multi_slice_result result = new get_multi_slice_result();
- try {
- result.success = iface.get_multi_slice(args.request);
- } catch (InvalidRequestException ire) {
- result.ire = ire;
- } catch (UnavailableException ue) {
- result.ue = ue;
- } catch (TimedOutException te) {
- result.te = te;
- }
- return result;
- }
- }
-
- public static class describe_schema_versions<I extends Iface> extends org.apache.thrift.ProcessFunction<I, describe_schema_versions_args> {
- public describe_schema_versions() {
- super("describe_schema_versions");
- }
-
- public describe_schema_versions_args getEmptyArgsInstance() {
- return new describe_schema_versions_args();
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public describe_schema_versions_result getResult(I iface, describe_schema_versions_args args) throws org.apache.thrift.TException {
- describe_schema_versions_result result = new describe_schema_versions_result();
- try {
- result.success = iface.describe_schema_versions();
- } catch (InvalidRequestException ire) {
- result.ire = ire;
- }
- return result;
- }
- }
-
- public static class describe_keyspaces<I extends Iface> extends org.apache.thrift.ProcessFunction<I, describe_keyspaces_args> {
- public describe_keyspaces() {
- super("describe_keyspaces");
- }
-
- public describe_keyspaces_args getEmptyArgsInstance() {
- return new describe_keyspaces_args();
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public describe_keyspaces_result getResult(I iface, describe_keyspaces_args args) throws org.apache.thrift.TException {
- describe_keyspaces_result result = new describe_keyspaces_result();
- try {
- result.success = iface.describe_keyspaces();
- } catch (InvalidRequestException ire) {
- result.ire = ire;
- }
- return result;
- }
- }
-
- public static class describe_cluster_name<I extends Iface> extends org.apache.thrift.ProcessFunction<I, describe_cluster_name_args> {
- public describe_cluster_name() {
- super("describe_cluster_name");
- }
-
- public describe_cluster_name_args getEmptyArgsInstance() {
- return new describe_cluster_name_args();
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public describe_cluster_name_result getResult(I iface, describe_cluster_name_args args) throws org.apache.thrift.TException {
- describe_cluster_name_result result = new describe_cluster_name_result();
- result.success = iface.describe_cluster_name();
- return result;
- }
- }
-
- public static class describe_version<I extends Iface> extends org.apache.thrift.ProcessFunction<I, describe_version_args> {
- public describe_version() {
- super("describe_version");
- }
-
- public describe_version_args getEmptyArgsInstance() {
- return new describe_version_args();
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public describe_version_result getResult(I iface, describe_version_args args) throws org.apache.thrift.TException {
- describe_version_result result = new describe_version_result();
- result.success = iface.describe_version();
- return result;
- }
- }
-
- public static class describe_ring<I extends Iface> extends org.apache.thrift.ProcessFunction<I, describe_ring_args> {
- public describe_ring() {
- super("describe_ring");
- }
-
- public describe_ring_args getEmptyArgsInstance() {
- return new describe_ring_args();
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public describe_ring_result getResult(I iface, describe_ring_args args) throws org.apache.thrift.TException {
- describe_ring_result result = new describe_ring_result();
- try {
- result.success = iface.describe_ring(args.keyspace);
- } catch (InvalidRequestException ire) {
- result.ire = ire;
- }
- return result;
- }
- }
-
- public static class describe_local_ring<I extends Iface> extends org.apache.thrift.ProcessFunction<I, describe_local_ring_args> {
- public describe_local_ring() {
- super("describe_local_ring");
- }
-
- public describe_local_ring_args getEmptyArgsInstance() {
- return new describe_local_ring_args();
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public describe_local_ring_result getResult(I iface, describe_local_ring_args args) throws org.apache.thrift.TException {
- describe_local_ring_result result = new describe_local_ring_result();
- try {
- result.success = iface.describe_local_ring(args.keyspace);
- } catch (InvalidRequestException ire) {
- result.ire = ire;
- }
- return result;
- }
- }
-
- public static class describe_token_map<I extends Iface> extends org.apache.thrift.ProcessFunction<I, describe_token_map_args> {
- public describe_token_map() {
- super("describe_token_map");
- }
-
- public describe_token_map_args getEmptyArgsInstance() {
- return new describe_token_map_args();
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public describe_token_map_result getResult(I iface, describe_token_map_args args) throws org.apache.thrift.TException {
- describe_token_map_result result = new describe_token_map_result();
- try {
- result.success = iface.describe_token_map();
- } catch (InvalidRequestException ire) {
- result.ire = ire;
- }
- return result;
- }
- }
-
- public static class describe_partitioner<I extends Iface> extends org.apache.thrift.ProcessFunction<I, describe_partitioner_args> {
- public describe_partitioner() {
- super("describe_partitioner");
- }
-
- public describe_partitioner_args getEmptyArgsInstance() {
- return new describe_partitioner_args();
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public describe_partitioner_result getResult(I iface, describe_partitioner_args args) throws org.apache.thrift.TException {
- describe_partitioner_result result = new describe_partitioner_result();
- result.success = iface.describe_partitioner();
- return result;
- }
- }
-
- public static class describe_snitch<I extends Iface> extends org.apache.thrift.ProcessFunction<I, describe_snitch_args> {
- public describe_snitch() {
- super("describe_snitch");
- }
-
- public describe_snitch_args getEmptyArgsInstance() {
- return new describe_snitch_args();
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public describe_snitch_result getResult(I iface, describe_snitch_args args) throws org.apache.thrift.TException {
- describe_snitch_result result = new describe_snitch_result();
- result.success = iface.describe_snitch();
- return result;
- }
- }
-
- public static class describe_keyspace<I extends Iface> extends org.apache.thrift.ProcessFunction<I, describe_keyspace_args> {
- public describe_keyspace() {
- super("describe_keyspace");
- }
-
- public describe_keyspace_args getEmptyArgsInstance() {
- return new describe_keyspace_args();
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public describe_keyspace_result getResult(I iface, describe_keyspace_args args) throws org.apache.thrift.TException {
- describe_keyspace_result result = new describe_keyspace_result();
- try {
- result.success = iface.describe_keyspace(args.keyspace);
- } catch (NotFoundException nfe) {
- result.nfe = nfe;
- } catch (InvalidRequestException ire) {
- result.ire = ire;
- }
- return result;
- }
- }
-
- public static class describe_splits<I extends Iface> extends org.apache.thrift.ProcessFunction<I, describe_splits_args> {
- public describe_splits() {
- super("describe_splits");
- }
-
- public describe_splits_args getEmptyArgsInstance() {
- return new describe_splits_args();
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public describe_splits_result getResult(I iface, describe_splits_args args) throws org.apache.thrift.TException {
- describe_splits_result result = new describe_splits_result();
- try {
- result.success = iface.describe_splits(args.cfName, args.start_token, args.end_token, args.keys_per_split);
- } catch (InvalidRequestException ire) {
- result.ire = ire;
- }
- return result;
- }
- }
-
- public static class trace_next_query<I extends Iface> extends org.apache.thrift.ProcessFunction<I, trace_next_query_args> {
- public trace_next_query() {
- super("trace_next_query");
- }
-
- public trace_next_query_args getEmptyArgsInstance() {
- return new trace_next_query_args();
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public trace_next_query_result getResult(I iface, trace_next_query_args args) throws org.apache.thrift.TException {
- trace_next_query_result result = new trace_next_query_result();
- result.success = iface.trace_next_query();
- return result;
- }
- }
-
- public static class describe_splits_ex<I extends Iface> extends org.apache.thrift.ProcessFunction<I, describe_splits_ex_args> {
- public describe_splits_ex() {
- super("describe_splits_ex");
- }
-
- public describe_splits_ex_args getEmptyArgsInstance() {
- return new describe_splits_ex_args();
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public describe_splits_ex_result getResult(I iface, describe_splits_ex_args args) throws org.apache.thrift.TException {
- describe_splits_ex_result result = new describe_splits_ex_result();
- try {
- result.success = iface.describe_splits_ex(args.cfName, args.start_token, args.end_token, args.keys_per_split);
- } catch (InvalidRequestException ire) {
- result.ire = ire;
- }
- return result;
- }
- }
-
- public static class system_add_column_family<I extends Iface> extends org.apache.thrift.ProcessFunction<I, system_add_column_family_args> {
- public system_add_column_family() {
- super("system_add_column_family");
- }
-
- public system_add_column_family_args getEmptyArgsInstance() {
- return new system_add_column_family_args();
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public system_add_column_family_result getResult(I iface, system_add_column_family_args args) throws org.apache.thrift.TException {
- system_add_column_family_result result = new system_add_column_family_result();
- try {
- result.success = iface.system_add_column_family(args.cf_def);
- } catch (InvalidRequestException ire) {
- result.ire = ire;
- } catch (SchemaDisagreementException sde) {
- result.sde = sde;
- }
- return result;
- }
- }
-
- public static class system_drop_column_family<I extends Iface> extends org.apache.thrift.ProcessFunction<I, system_drop_column_family_args> {
- public system_drop_column_family() {
- super("system_drop_column_family");
- }
-
- public system_drop_column_family_args getEmptyArgsInstance() {
- return new system_drop_column_family_args();
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public system_drop_column_family_result getResult(I iface, system_drop_column_family_args args) throws org.apache.thrift.TException {
- system_drop_column_family_result result = new system_drop_column_family_result();
- try {
- result.success = iface.system_drop_column_family(args.column_family);
- } catch (InvalidRequestException ire) {
- result.ire = ire;
- } catch (SchemaDisagreementException sde) {
- result.sde = sde;
- }
- return result;
- }
- }
-
- public static class system_add_keyspace<I extends Iface> extends org.apache.thrift.ProcessFunction<I, system_add_keyspace_args> {
- public system_add_keyspace() {
- super("system_add_keyspace");
- }
-
- public system_add_keyspace_args getEmptyArgsInstance() {
- return new system_add_keyspace_args();
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public system_add_keyspace_result getResult(I iface, system_add_keyspace_args args) throws org.apache.thrift.TException {
- system_add_keyspace_result result = new system_add_keyspace_result();
- try {
- result.success = iface.system_add_keyspace(args.ks_def);
- } catch (InvalidRequestException ire) {
- result.ire = ire;
- } catch (SchemaDisagreementException sde) {
- result.sde = sde;
- }
- return result;
- }
- }
-
- public static class system_drop_keyspace<I extends Iface> extends org.apache.thrift.ProcessFunction<I, system_drop_keyspace_args> {
- public system_drop_keyspace() {
- super("system_drop_keyspace");
- }
-
- public system_drop_keyspace_args getEmptyArgsInstance() {
- return new system_drop_keyspace_args();
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public system_drop_keyspace_result getResult(I iface, system_drop_keyspace_args args) throws org.apache.thrift.TException {
- system_drop_keyspace_result result = new system_drop_keyspace_result();
- try {
- result.success = iface.system_drop_keyspace(args.keyspace);
- } catch (InvalidRequestException ire) {
- result.ire = ire;
- } catch (SchemaDisagreementException sde) {
- result.sde = sde;
- }
- return result;
- }
- }
-
- public static class system_update_keyspace<I extends Iface> extends org.apache.thrift.ProcessFunction<I, system_update_keyspace_args> {
- public system_update_keyspace() {
- super("system_update_keyspace");
- }
-
- public system_update_keyspace_args getEmptyArgsInstance() {
- return new system_update_keyspace_args();
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public system_update_keyspace_result getResult(I iface, system_update_keyspace_args args) throws org.apache.thrift.TException {
- system_update_keyspace_result result = new system_update_keyspace_result();
- try {
- result.success = iface.system_update_keyspace(args.ks_def);
- } catch (InvalidRequestException ire) {
- result.ire = ire;
- } catch (SchemaDisagreementException sde) {
- result.sde = sde;
- }
- return result;
- }
- }
-
- public static class system_update_column_family<I extends Iface> extends org.apache.thrift.ProcessFunction<I, system_update_column_family_args> {
- public system_update_column_family() {
- super("system_update_column_family");
- }
-
- public system_update_column_family_args getEmptyArgsInstance() {
- return new system_update_column_family_args();
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public system_update_column_family_result getResult(I iface, system_update_column_family_args args) throws org.apache.thrift.TException {
- system_update_column_family_result result = new system_update_column_family_result();
- try {
- result.success = iface.system_update_column_family(args.cf_def);
- } catch (InvalidRequestException ire) {
- result.ire = ire;
- } catch (SchemaDisagreementException sde) {
- result.sde = sde;
- }
- return result;
- }
- }
-
- public static class execute_cql_query<I extends Iface> extends org.apache.thrift.ProcessFunction<I, execute_cql_query_args> {
- public execute_cql_query() {
- super("execute_cql_query");
- }
-
- public execute_cql_query_args getEmptyArgsInstance() {
- return new execute_cql_query_args();
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public execute_cql_query_result getResult(I iface, execute_cql_query_args args) throws org.apache.thrift.TException {
- execute_cql_query_result result = new execute_cql_query_result();
- try {
- result.success = iface.execute_cql_query(args.query, args.compression);
- } catch (InvalidRequestException ire) {
- result.ire = ire;
- } catch (UnavailableException ue) {
- result.ue = ue;
- } catch (TimedOutException te) {
- result.te = te;
- } catch (SchemaDisagreementException sde) {
- result.sde = sde;
- }
- return result;
- }
- }
-
- public static class execute_cql3_query<I extends Iface> extends org.apache.thrift.ProcessFunction<I, execute_cql3_query_args> {
- public execute_cql3_query() {
- super("execute_cql3_query");
- }
-
- public execute_cql3_query_args getEmptyArgsInstance() {
- return new execute_cql3_query_args();
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public execute_cql3_query_result getResult(I iface, execute_cql3_query_args args) throws org.apache.thrift.TException {
- execute_cql3_query_result result = new execute_cql3_query_result();
- try {
- result.success = iface.execute_cql3_query(args.query, args.compression, args.consistency);
- } catch (InvalidRequestException ire) {
- result.ire = ire;
- } catch (UnavailableException ue) {
- result.ue = ue;
- } catch (TimedOutException te) {
- result.te = te;
- } catch (SchemaDisagreementException sde) {
- result.sde = sde;
- }
- return result;
- }
- }
-
- public static class prepare_cql_query<I extends Iface> extends org.apache.thrift.ProcessFunction<I, prepare_cql_query_args> {
- public prepare_cql_query() {
- super("prepare_cql_query");
- }
-
- public prepare_cql_query_args getEmptyArgsInstance() {
- return new prepare_cql_query_args();
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public prepare_cql_query_result getResult(I iface, prepare_cql_query_args args) throws org.apache.thrift.TException {
- prepare_cql_query_result result = new prepare_cql_query_result();
- try {
- result.success = iface.prepare_cql_query(args.query, args.compression);
- } catch (InvalidRequestException ire) {
- result.ire = ire;
- }
- return result;
- }
- }
-
- public static class prepare_cql3_query<I extends Iface> extends org.apache.thrift.ProcessFunction<I, prepare_cql3_query_args> {
- public prepare_cql3_query() {
- super("prepare_cql3_query");
- }
-
- public prepare_cql3_query_args getEmptyArgsInstance() {
- return new prepare_cql3_query_args();
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public prepare_cql3_query_result getResult(I iface, prepare_cql3_query_args args) throws org.apache.thrift.TException {
- prepare_cql3_query_result result = new prepare_cql3_query_result();
- try {
- result.success = iface.prepare_cql3_query(args.query, args.compression);
- } catch (InvalidRequestException ire) {
- result.ire = ire;
- }
- return result;
- }
- }
-
- public static class execute_prepared_cql_query<I extends Iface> extends org.apache.thrift.ProcessFunction<I, execute_prepared_cql_query_args> {
- public execute_prepared_cql_query() {
- super("execute_prepared_cql_query");
- }
-
- public execute_prepared_cql_query_args getEmptyArgsInstance() {
- return new execute_prepared_cql_query_args();
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public execute_prepared_cql_query_result getResult(I iface, execute_prepared_cql_query_args args) throws org.apache.thrift.TException {
- execute_prepared_cql_query_result result = new execute_prepared_cql_query_result();
- try {
- result.success = iface.execute_prepared_cql_query(args.itemId, args.values);
- } catch (InvalidRequestException ire) {
- result.ire = ire;
- } catch (UnavailableException ue) {
- result.ue = ue;
- } catch (TimedOutException te) {
- result.te = te;
- } catch (SchemaDisagreementException sde) {
- result.sde = sde;
- }
- return result;
- }
- }
-
- public static class execute_prepared_cql3_query<I extends Iface> extends org.apache.thrift.ProcessFunction<I, execute_prepared_cql3_query_args> {
- public execute_prepared_cql3_query() {
- super("execute_prepared_cql3_query");
- }
-
- public execute_prepared_cql3_query_args getEmptyArgsInstance() {
- return new execute_prepared_cql3_query_args();
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public execute_prepared_cql3_query_result getResult(I iface, execute_prepared_cql3_query_args args) throws org.apache.thrift.TException {
- execute_prepared_cql3_query_result result = new execute_prepared_cql3_query_result();
- try {
- result.success = iface.execute_prepared_cql3_query(args.itemId, args.values, args.consistency);
- } catch (InvalidRequestException ire) {
- result.ire = ire;
- } catch (UnavailableException ue) {
- result.ue = ue;
- } catch (TimedOutException te) {
- result.te = te;
- } catch (SchemaDisagreementException sde) {
- result.sde = sde;
- }
- return result;
- }
- }
-
- public static class set_cql_version<I extends Iface> extends org.apache.thrift.ProcessFunction<I, set_cql_version_args> {
- public set_cql_version() {
- super("set_cql_version");
- }
-
- public set_cql_version_args getEmptyArgsInstance() {
- return new set_cql_version_args();
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public set_cql_version_result getResult(I iface, set_cql_version_args args) throws org.apache.thrift.TException {
- set_cql_version_result result = new set_cql_version_result();
- try {
- iface.set_cql_version(args.version);
- } catch (InvalidRequestException ire) {
- result.ire = ire;
- }
- return result;
- }
- }
-
- }
-
- public static class AsyncProcessor<I extends AsyncIface> extends org.apache.thrift.TBaseAsyncProcessor<I> {
- private static final Logger LOGGER = LoggerFactory.getLogger(AsyncProcessor.class.getName());
- public AsyncProcessor(I iface) {
- super(iface, getProcessMap(new HashMap<String, org.apache.thrift.AsyncProcessFunction<I, ? extends org.apache.thrift.TBase, ?>>()));
- }
-
- protected AsyncProcessor(I iface, Map<String, org.apache.thrift.AsyncProcessFunction<I, ? extends org.apache.thrift.TBase, ?>> processMap) {
- super(iface, getProcessMap(processMap));
- }
-
- private static <I extends AsyncIface> Map<String, org.apache.thrift.AsyncProcessFunction<I, ? extends org.apache.thrift.TBase,?>> getProcessMap(Map<String, org.apache.thrift.AsyncProcessFunction<I, ? extends org.apache.thrift.TBase, ?>> processMap) {
- processMap.put("login", new login());
- processMap.put("set_keyspace", new set_keyspace());
- processMap.put("get", new get());
- processMap.put("get_slice", new get_slice());
- processMap.put("get_count", new get_count());
- processMap.put("multiget_slice", new multiget_slice());
- processMap.put("multiget_count", new multiget_count());
- processMap.put("get_range_slices", new get_range_slices());
- processMap.put("get_paged_slice", new get_paged_slice());
- processMap.put("get_indexed_slices", new get_indexed_slices());
- processMap.put("insert", new insert());
- processMap.put("add", new add());
- processMap.put("cas", new cas());
- processMap.put("remove", new remove());
- processMap.put("remove_counter", new remove_counter());
- processMap.put("batch_mutate", new batch_mutate());
- processMap.put("atomic_batch_mutate", new atomic_batch_mutate());
- processMap.put("truncate", new truncate());
- processMap.put("get_multi_slice", new get_multi_slice());
- processMap.put("describe_schema_versions", new describe_schema_versions());
- processMap.put("describe_keyspaces", new describe_keyspaces());
- processMap.put("describe_cluster_name", new describe_cluster_name());
- processMap.put("describe_version", new describe_version());
- processMap.put("describe_ring", new describe_ring());
- processMap.put("describe_local_ring", new describe_local_ring());
- processMap.put("describe_token_map", new describe_token_map());
- processMap.put("describe_partitioner", new describe_partitioner());
- processMap.put("describe_snitch", new describe_snitch());
- processMap.put("describe_keyspace", new describe_keyspace());
- processMap.put("describe_splits", new describe_splits());
- processMap.put("trace_next_query", new trace_next_query());
- processMap.put("describe_splits_ex", new describe_splits_ex());
- processMap.put("system_add_column_family", new system_add_column_family());
- processMap.put("system_drop_column_family", new system_drop_column_family());
- processMap.put("system_add_keyspace", new system_add_keyspace());
- processMap.put("system_drop_keyspace", new system_drop_keyspace());
- processMap.put("system_update_keyspace", new system_update_keyspace());
- processMap.put("system_update_column_family", new system_update_column_family());
- processMap.put("execute_cql_query", new execute_cql_query());
- processMap.put("execute_cql3_query", new execute_cql3_query());
- processMap.put("prepare_cql_query", new prepare_cql_query());
- processMap.put("prepare_cql3_query", new prepare_cql3_query());
- processMap.put("execute_prepared_cql_query", new execute_prepared_cql_query());
- processMap.put("execute_prepared_cql3_query", new execute_prepared_cql3_query());
- processMap.put("set_cql_version", new set_cql_version());
- return processMap;
- }
-
- public static class login<I extends AsyncIface> extends org.apache.thrift.AsyncProcessFunction<I, login_args, Void> {
- public login() {
- super("login");
- }
-
- public login_args getEmptyArgsInstance() {
- return new login_args();
- }
-
- public AsyncMethodCallback<Void> getResultHandler(final AsyncFrameBuffer fb, final int seqid) {
- final org.apache.thrift.AsyncProcessFunction fcall = this;
- return new AsyncMethodCallback<Void>() {
- public void onComplete(Void o) {
- login_result result = new login_result();
- try {
- fcall.sendResponse(fb,result, org.apache.thrift.protocol.TMessageType.REPLY,seqid);
- return;
- } catch (Exception e) {
- LOGGER.error("Exception writing to internal frame buffer", e);
- }
- fb.close();
- }
- public void onError(Exception e) {
- byte msgType = org.apache.thrift.protocol.TMessageType.REPLY;
- org.apache.thrift.TBase msg;
- login_result result = new login_result();
- if (e instanceof AuthenticationException) {
- result.authnx = (AuthenticationException) e;
- result.setAuthnxIsSet(true);
- msg = result;
- }
- else if (e instanceof AuthorizationException) {
- result.authzx = (AuthorizationException) e;
- result.setAuthzxIsSet(true);
- msg = result;
- }
- else
- {
- msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION;
- msg = (org.apache.thrift.TBase)new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, e.getMessage());
- }
- try {
- fcall.sendResponse(fb,msg,msgType,seqid);
- return;
- } catch (Exception ex) {
- LOGGER.error("Exception writing to internal frame buffer", ex);
- }
- fb.close();
- }
- };
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public void start(I iface, login_args args, org.apache.thrift.async.AsyncMethodCallback<Void> resultHandler) throws TException {
- iface.login(args.auth_request,resultHandler);
- }
- }
-
- public static class set_keyspace<I extends AsyncIface> extends org.apache.thrift.AsyncProcessFunction<I, set_keyspace_args, Void> {
- public set_keyspace() {
- super("set_keyspace");
- }
-
- public set_keyspace_args getEmptyArgsInstance() {
- return new set_keyspace_args();
- }
-
- public AsyncMethodCallback<Void> getResultHandler(final AsyncFrameBuffer fb, final int seqid) {
- final org.apache.thrift.AsyncProcessFunction fcall = this;
- return new AsyncMethodCallback<Void>() {
- public void onComplete(Void o) {
- set_keyspace_result result = new set_keyspace_result();
- try {
- fcall.sendResponse(fb,result, org.apache.thrift.protocol.TMessageType.REPLY,seqid);
- return;
- } catch (Exception e) {
- LOGGER.error("Exception writing to internal frame buffer", e);
- }
- fb.close();
- }
- public void onError(Exception e) {
- byte msgType = org.apache.thrift.protocol.TMessageType.REPLY;
- org.apache.thrift.TBase msg;
- set_keyspace_result result = new set_keyspace_result();
- if (e instanceof InvalidRequestException) {
- result.ire = (InvalidRequestException) e;
- result.setIreIsSet(true);
- msg = result;
- }
- else
- {
- msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION;
- msg = (org.apache.thrift.TBase)new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, e.getMessage());
- }
- try {
- fcall.sendResponse(fb,msg,msgType,seqid);
- return;
- } catch (Exception ex) {
- LOGGER.error("Exception writing to internal frame buffer", ex);
- }
- fb.close();
- }
- };
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public void start(I iface, set_keyspace_args args, org.apache.thrift.async.AsyncMethodCallback<Void> resultHandler) throws TException {
- iface.set_keyspace(args.keyspace,resultHandler);
- }
- }
-
- public static class get<I extends AsyncIface> extends org.apache.thrift.AsyncProcessFunction<I, get_args, ColumnOrSuperColumn> {
- public get() {
- super("get");
- }
-
- public get_args getEmptyArgsInstance() {
- return new get_args();
- }
-
- public AsyncMethodCallback<ColumnOrSuperColumn> getResultHandler(final AsyncFrameBuffer fb, final int seqid) {
- final org.apache.thrift.AsyncProcessFunction fcall = this;
- return new AsyncMethodCallback<ColumnOrSuperColumn>() {
- public void onComplete(ColumnOrSuperColumn o) {
- get_result result = new get_result();
- result.success = o;
- try {
- fcall.sendResponse(fb,result, org.apache.thrift.protocol.TMessageType.REPLY,seqid);
- return;
- } catch (Exception e) {
- LOGGER.error("Exception writing to internal frame buffer", e);
- }
- fb.close();
- }
- public void onError(Exception e) {
- byte msgType = org.apache.thrift.protocol.TMessageType.REPLY;
- org.apache.thrift.TBase msg;
- get_result result = new get_result();
- if (e instanceof InvalidRequestException) {
- result.ire = (InvalidRequestException) e;
- result.setIreIsSet(true);
- msg = result;
- }
- else if (e instanceof NotFoundException) {
- result.nfe = (NotFoundException) e;
- result.setNfeIsSet(true);
- msg = result;
- }
- else if (e instanceof UnavailableException) {
- result.ue = (UnavailableException) e;
- result.setUeIsSet(true);
- msg = result;
- }
- else if (e instanceof TimedOutException) {
- result.te = (TimedOutException) e;
- result.setTeIsSet(true);
- msg = result;
- }
- else
- {
- msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION;
- msg = (org.apache.thrift.TBase)new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, e.getMessage());
- }
- try {
- fcall.sendResponse(fb,msg,msgType,seqid);
- return;
- } catch (Exception ex) {
- LOGGER.error("Exception writing to internal frame buffer", ex);
- }
- fb.close();
- }
- };
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public void start(I iface, get_args args, org.apache.thrift.async.AsyncMethodCallback<ColumnOrSuperColumn> resultHandler) throws TException {
- iface.get(args.key, args.column_path, args.consistency_level,resultHandler);
- }
- }
-
- public static class get_slice<I extends AsyncIface> extends org.apache.thrift.AsyncProcessFunction<I, get_slice_args, List<ColumnOrSuperColumn>> {
- public get_slice() {
- super("get_slice");
- }
-
- public get_slice_args getEmptyArgsInstance() {
- return new get_slice_args();
- }
-
- public AsyncMethodCallback<List<ColumnOrSuperColumn>> getResultHandler(final AsyncFrameBuffer fb, final int seqid) {
- final org.apache.thrift.AsyncProcessFunction fcall = this;
- return new AsyncMethodCallback<List<ColumnOrSuperColumn>>() {
- public void onComplete(List<ColumnOrSuperColumn> o) {
- get_slice_result result = new get_slice_result();
- result.success = o;
- try {
- fcall.sendResponse(fb,result, org.apache.thrift.protocol.TMessageType.REPLY,seqid);
- return;
- } catch (Exception e) {
- LOGGER.error("Exception writing to internal frame buffer", e);
- }
- fb.close();
- }
- public void onError(Exception e) {
- byte msgType = org.apache.thrift.protocol.TMessageType.REPLY;
- org.apache.thrift.TBase msg;
- get_slice_result result = new get_slice_result();
- if (e instanceof InvalidRequestException) {
- result.ire = (InvalidRequestException) e;
- result.setIreIsSet(true);
- msg = result;
- }
- else if (e instanceof UnavailableException) {
- result.ue = (UnavailableException) e;
- result.setUeIsSet(true);
- msg = result;
- }
- else if (e instanceof TimedOutException) {
- result.te = (TimedOutException) e;
- result.setTeIsSet(true);
- msg = result;
- }
- else
- {
- msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION;
- msg = (org.apache.thrift.TBase)new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, e.getMessage());
- }
- try {
- fcall.sendResponse(fb,msg,msgType,seqid);
- return;
- } catch (Exception ex) {
- LOGGER.error("Exception writing to internal frame buffer", ex);
- }
- fb.close();
- }
- };
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public void start(I iface, get_slice_args args, org.apache.thrift.async.AsyncMethodCallback<List<ColumnOrSuperColumn>> resultHandler) throws TException {
- iface.get_slice(args.key, args.column_parent, args.predicate, args.consistency_level,resultHandler);
- }
- }
-
- public static class get_count<I extends AsyncIface> extends org.apache.thrift.AsyncProcessFunction<I, get_count_args, Integer> {
- public get_count() {
- super("get_count");
- }
-
- public get_count_args getEmptyArgsInstance() {
- return new get_count_args();
- }
-
- public AsyncMethodCallback<Integer> getResultHandler(final AsyncFrameBuffer fb, final int seqid) {
- final org.apache.thrift.AsyncProcessFunction fcall = this;
- return new AsyncMethodCallback<Integer>() {
- public void onComplete(Integer o) {
- get_count_result result = new get_count_result();
- result.success = o;
- result.setSuccessIsSet(true);
- try {
- fcall.sendResponse(fb,result, org.apache.thrift.protocol.TMessageType.REPLY,seqid);
- return;
- } catch (Exception e) {
- LOGGER.error("Exception writing to internal frame buffer", e);
- }
- fb.close();
- }
- public void onError(Exception e) {
- byte msgType = org.apache.thrift.protocol.TMessageType.REPLY;
- org.apache.thrift.TBase msg;
- get_count_result result = new get_count_result();
- if (e instanceof InvalidRequestException) {
- result.ire = (InvalidRequestException) e;
- result.setIreIsSet(true);
- msg = result;
- }
- else if (e instanceof UnavailableException) {
- result.ue = (UnavailableException) e;
- result.setUeIsSet(true);
- msg = result;
- }
- else if (e instanceof TimedOutException) {
- result.te = (TimedOutException) e;
- result.setTeIsSet(true);
- msg = result;
- }
- else
- {
- msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION;
- msg = (org.apache.thrift.TBase)new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, e.getMessage());
- }
- try {
- fcall.sendResponse(fb,msg,msgType,seqid);
- return;
- } catch (Exception ex) {
- LOGGER.error("Exception writing to internal frame buffer", ex);
- }
- fb.close();
- }
- };
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public void start(I iface, get_count_args args, org.apache.thrift.async.AsyncMethodCallback<Integer> resultHandler) throws TException {
- iface.get_count(args.key, args.column_parent, args.predicate, args.consistency_level,resultHandler);
- }
- }
-
- public static class multiget_slice<I extends AsyncIface> extends org.apache.thrift.AsyncProcessFunction<I, multiget_slice_args, Map<ByteBuffer,List<ColumnOrSuperColumn>>> {
- public multiget_slice() {
- super("multiget_slice");
- }
-
- public multiget_slice_args getEmptyArgsInstance() {
- return new multiget_slice_args();
- }
-
- public AsyncMethodCallback<Map<ByteBuffer,List<ColumnOrSuperColumn>>> getResultHandler(final AsyncFrameBuffer fb, final int seqid) {
- final org.apache.thrift.AsyncProcessFunction fcall = this;
- return new AsyncMethodCallback<Map<ByteBuffer,List<ColumnOrSuperColumn>>>() {
- public void onComplete(Map<ByteBuffer,List<ColumnOrSuperColumn>> o) {
- multiget_slice_result result = new multiget_slice_result();
- result.success = o;
- try {
- fcall.sendResponse(fb,result, org.apache.thrift.protocol.TMessageType.REPLY,seqid);
- return;
- } catch (Exception e) {
- LOGGER.error("Exception writing to internal frame buffer", e);
- }
- fb.close();
- }
- public void onError(Exception e) {
- byte msgType = org.apache.thrift.protocol.TMessageType.REPLY;
- org.apache.thrift.TBase msg;
- multiget_slice_result result = new multiget_slice_result();
- if (e instanceof InvalidRequestException) {
- result.ire = (InvalidRequestException) e;
- result.setIreIsSet(true);
- msg = result;
- }
- else if (e instanceof UnavailableException) {
- result.ue = (UnavailableException) e;
- result.setUeIsSet(true);
- msg = result;
- }
- else if (e instanceof TimedOutException) {
- result.te = (TimedOutException) e;
- result.setTeIsSet(true);
- msg = result;
- }
- else
- {
- msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION;
- msg = (org.apache.thrift.TBase)new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, e.getMessage());
- }
- try {
- fcall.sendResponse(fb,msg,msgType,seqid);
- return;
- } catch (Exception ex) {
- LOGGER.error("Exception writing to internal frame buffer", ex);
- }
- fb.close();
- }
- };
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public void start(I iface, multiget_slice_args args, org.apache.thrift.async.AsyncMethodCallback<Map<ByteBuffer,List<ColumnOrSuperColumn>>> resultHandler) throws TException {
- iface.multiget_slice(args.keys, args.column_parent, args.predicate, args.consistency_level,resultHandler);
- }
- }
-
- public static class multiget_count<I extends AsyncIface> extends org.apache.thrift.AsyncProcessFunction<I, multiget_count_args, Map<ByteBuffer,Integer>> {
- public multiget_count() {
- super("multiget_count");
- }
-
- public multiget_count_args getEmptyArgsInstance() {
- return new multiget_count_args();
- }
-
- public AsyncMethodCallback<Map<ByteBuffer,Integer>> getResultHandler(final AsyncFrameBuffer fb, final int seqid) {
- final org.apache.thrift.AsyncProcessFunction fcall = this;
- return new AsyncMethodCallback<Map<ByteBuffer,Integer>>() {
- public void onComplete(Map<ByteBuffer,Integer> o) {
- multiget_count_result result = new multiget_count_result();
- result.success = o;
- try {
- fcall.sendResponse(fb,result, org.apache.thrift.protocol.TMessageType.REPLY,seqid);
- return;
- } catch (Exception e) {
- LOGGER.error("Exception writing to internal frame buffer", e);
- }
- fb.close();
- }
- public void onError(Exception e) {
- byte msgType = org.apache.thrift.protocol.TMessageType.REPLY;
- org.apache.thrift.TBase msg;
- multiget_count_result result = new multiget_count_result();
- if (e instanceof InvalidRequestException) {
- result.ire = (InvalidRequestException) e;
- result.setIreIsSet(true);
- msg = result;
- }
- else if (e instanceof UnavailableException) {
- result.ue = (UnavailableException) e;
- result.setUeIsSet(true);
- msg = result;
- }
- else if (e instanceof TimedOutException) {
- result.te = (TimedOutException) e;
- result.setTeIsSet(true);
- msg = result;
- }
- else
- {
- msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION;
- msg = (org.apache.thrift.TBase)new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, e.getMessage());
- }
- try {
- fcall.sendResponse(fb,msg,msgType,seqid);
- return;
- } catch (Exception ex) {
- LOGGER.error("Exception writing to internal frame buffer", ex);
- }
- fb.close();
- }
- };
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public void start(I iface, multiget_count_args args, org.apache.thrift.async.AsyncMethodCallback<Map<ByteBuffer,Integer>> resultHandler) throws TException {
- iface.multiget_count(args.keys, args.column_parent, args.predicate, args.consistency_level,resultHandler);
- }
- }
-
- public static class get_range_slices<I extends AsyncIface> extends org.apache.thrift.AsyncProcessFunction<I, get_range_slices_args, List<KeySlice>> {
- public get_range_slices() {
- super("get_range_slices");
- }
-
- public get_range_slices_args getEmptyArgsInstance() {
- return new get_range_slices_args();
- }
-
- public AsyncMethodCallback<List<KeySlice>> getResultHandler(final AsyncFrameBuffer fb, final int seqid) {
- final org.apache.thrift.AsyncProcessFunction fcall = this;
- return new AsyncMethodCallback<List<KeySlice>>() {
- public void onComplete(List<KeySlice> o) {
- get_range_slices_result result = new get_range_slices_result();
- result.success = o;
- try {
- fcall.sendResponse(fb,result, org.apache.thrift.protocol.TMessageType.REPLY,seqid);
- return;
- } catch (Exception e) {
- LOGGER.error("Exception writing to internal frame buffer", e);
- }
- fb.close();
- }
- public void onError(Exception e) {
- byte msgType = org.apache.thrift.protocol.TMessageType.REPLY;
- org.apache.thrift.TBase msg;
- get_range_slices_result result = new get_range_slices_result();
- if (e instanceof InvalidRequestException) {
- result.ire = (InvalidRequestException) e;
- result.setIreIsSet(true);
- msg = result;
- }
- else if (e instanceof UnavailableException) {
- result.ue = (UnavailableException) e;
- result.setUeIsSet(true);
- msg = result;
- }
- else if (e instanceof TimedOutException) {
- result.te = (TimedOutException) e;
- result.setTeIsSet(true);
- msg = result;
- }
- else
- {
- msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION;
- msg = (org.apache.thrift.TBase)new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, e.getMessage());
- }
- try {
- fcall.sendResponse(fb,msg,msgType,seqid);
- return;
- } catch (Exception ex) {
- LOGGER.error("Exception writing to internal frame buffer", ex);
- }
- fb.close();
- }
- };
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public void start(I iface, get_range_slices_args args, org.apache.thrift.async.AsyncMethodCallback<List<KeySlice>> resultHandler) throws TException {
- iface.get_range_slices(args.column_parent, args.predicate, args.range, args.consistency_level,resultHandler);
- }
- }
-
- public static class get_paged_slice<I extends AsyncIface> extends org.apache.thrift.AsyncProcessFunction<I, get_paged_slice_args, List<KeySlice>> {
- public get_paged_slice() {
- super("get_paged_slice");
- }
-
- public get_paged_slice_args getEmptyArgsInstance() {
- return new get_paged_slice_args();
- }
-
- public AsyncMethodCallback<List<KeySlice>> getResultHandler(final AsyncFrameBuffer fb, final int seqid) {
- final org.apache.thrift.AsyncProcessFunction fcall = this;
- return new AsyncMethodCallback<List<KeySlice>>() {
- public void onComplete(List<KeySlice> o) {
- get_paged_slice_result result = new get_paged_slice_result();
- result.success = o;
- try {
- fcall.sendResponse(fb,result, org.apache.thrift.protocol.TMessageType.REPLY,seqid);
- return;
- } catch (Exception e) {
- LOGGER.error("Exception writing to internal frame buffer", e);
- }
- fb.close();
- }
- public void onError(Exception e) {
- byte msgType = org.apache.thrift.protocol.TMessageType.REPLY;
- org.apache.thrift.TBase msg;
- get_paged_slice_result result = new get_paged_slice_result();
- if (e instanceof InvalidRequestException) {
- result.ire = (InvalidRequestException) e;
- result.setIreIsSet(true);
- msg = result;
- }
- else if (e instanceof UnavailableException) {
- result.ue = (UnavailableException) e;
- result.setUeIsSet(true);
- msg = result;
- }
- else if (e instanceof TimedOutException) {
- result.te = (TimedOutException) e;
- result.setTeIsSet(true);
- msg = result;
- }
- else
- {
- msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION;
- msg = (org.apache.thrift.TBase)new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, e.getMessage());
- }
- try {
- fcall.sendResponse(fb,msg,msgType,seqid);
- return;
- } catch (Exception ex) {
- LOGGER.error("Exception writing to internal frame buffer", ex);
- }
- fb.close();
- }
- };
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public void start(I iface, get_paged_slice_args args, org.apache.thrift.async.AsyncMethodCallback<List<KeySlice>> resultHandler) throws TException {
- iface.get_paged_slice(args.column_family, args.range, args.start_column, args.consistency_level,resultHandler);
- }
- }
-
- public static class get_indexed_slices<I extends AsyncIface> extends org.apache.thrift.AsyncProcessFunction<I, get_indexed_slices_args, List<KeySlice>> {
- public get_indexed_slices() {
- super("get_indexed_slices");
- }
-
- public get_indexed_slices_args getEmptyArgsInstance() {
- return new get_indexed_slices_args();
- }
-
- public AsyncMethodCallback<List<KeySlice>> getResultHandler(final AsyncFrameBuffer fb, final int seqid) {
- final org.apache.thrift.AsyncProcessFunction fcall = this;
- return new AsyncMethodCallback<List<KeySlice>>() {
- public void onComplete(List<KeySlice> o) {
- get_indexed_slices_result result = new get_indexed_slices_result();
- result.success = o;
- try {
- fcall.sendResponse(fb,result, org.apache.thrift.protocol.TMessageType.REPLY,seqid);
- return;
- } catch (Exception e) {
- LOGGER.error("Exception writing to internal frame buffer", e);
- }
- fb.close();
- }
- public void onError(Exception e) {
- byte msgType = org.apache.thrift.protocol.TMessageType.REPLY;
- org.apache.thrift.TBase msg;
- get_indexed_slices_result result = new get_indexed_slices_result();
- if (e instanceof InvalidRequestException) {
- result.ire = (InvalidRequestException) e;
- result.setIreIsSet(true);
- msg = result;
- }
- else if (e instanceof UnavailableException) {
- result.ue = (UnavailableException) e;
- result.setUeIsSet(true);
- msg = result;
- }
- else if (e instanceof TimedOutException) {
- result.te = (TimedOutException) e;
- result.setTeIsSet(true);
- msg = result;
- }
- else
- {
- msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION;
- msg = (org.apache.thrift.TBase)new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, e.getMessage());
- }
- try {
- fcall.sendResponse(fb,msg,msgType,seqid);
- return;
- } catch (Exception ex) {
- LOGGER.error("Exception writing to internal frame buffer", ex);
- }
- fb.close();
- }
- };
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public void start(I iface, get_indexed_slices_args args, org.apache.thrift.async.AsyncMethodCallback<List<KeySlice>> resultHandler) throws TException {
- iface.get_indexed_slices(args.column_parent, args.index_clause, args.column_predicate, args.consistency_level,resultHandler);
- }
- }
-
- public static class insert<I extends AsyncIface> extends org.apache.thrift.AsyncProcessFunction<I, insert_args, Void> {
- public insert() {
- super("insert");
- }
-
- public insert_args getEmptyArgsInstance() {
- return new insert_args();
- }
-
- public AsyncMethodCallback<Void> getResultHandler(final AsyncFrameBuffer fb, final int seqid) {
- final org.apache.thrift.AsyncProcessFunction fcall = this;
- return new AsyncMethodCallback<Void>() {
- public void onComplete(Void o) {
- insert_result result = new insert_result();
- try {
- fcall.sendResponse(fb,result, org.apache.thrift.protocol.TMessageType.REPLY,seqid);
- return;
- } catch (Exception e) {
- LOGGER.error("Exception writing to internal frame buffer", e);
- }
- fb.close();
- }
- public void onError(Exception e) {
- byte msgType = org.apache.thrift.protocol.TMessageType.REPLY;
- org.apache.thrift.TBase msg;
- insert_result result = new insert_result();
- if (e instanceof InvalidRequestException) {
- result.ire = (InvalidRequestException) e;
- result.setIreIsSet(true);
- msg = result;
- }
- else if (e instanceof UnavailableException) {
- result.ue = (UnavailableException) e;
- result.setUeIsSet(true);
- msg = result;
- }
- else if (e instanceof TimedOutException) {
- result.te = (TimedOutException) e;
- result.setTeIsSet(true);
- msg = result;
- }
- else
- {
- msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION;
- msg = (org.apache.thrift.TBase)new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, e.getMessage());
- }
- try {
- fcall.sendResponse(fb,msg,msgType,seqid);
- return;
- } catch (Exception ex) {
- LOGGER.error("Exception writing to internal frame buffer", ex);
- }
- fb.close();
- }
- };
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public void start(I iface, insert_args args, org.apache.thrift.async.AsyncMethodCallback<Void> resultHandler) throws TException {
- iface.insert(args.key, args.column_parent, args.column, args.consistency_level,resultHandler);
- }
- }
-
- public static class add<I extends AsyncIface> extends org.apache.thrift.AsyncProcessFunction<I, add_args, Void> {
- public add() {
- super("add");
- }
-
- public add_args getEmptyArgsInstance() {
- return new add_args();
- }
-
- public AsyncMethodCallback<Void> getResultHandler(final AsyncFrameBuffer fb, final int seqid) {
- final org.apache.thrift.AsyncProcessFunction fcall = this;
- return new AsyncMethodCallback<Void>() {
- public void onComplete(Void o) {
- add_result result = new add_result();
- try {
- fcall.sendResponse(fb,result, org.apache.thrift.protocol.TMessageType.REPLY,seqid);
- return;
- } catch (Exception e) {
- LOGGER.error("Exception writing to internal frame buffer", e);
- }
- fb.close();
- }
- public void onError(Exception e) {
- byte msgType = org.apache.thrift.protocol.TMessageType.REPLY;
- org.apache.thrift.TBase msg;
- add_result result = new add_result();
- if (e instanceof InvalidRequestException) {
- result.ire = (InvalidRequestException) e;
- result.setIreIsSet(true);
- msg = result;
- }
- else if (e instanceof UnavailableException) {
- result.ue = (UnavailableException) e;
- result.setUeIsSet(true);
- msg = result;
- }
- else if (e instanceof TimedOutException) {
- result.te = (TimedOutException) e;
- result.setTeIsSet(true);
- msg = result;
- }
- else
- {
- msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION;
- msg = (org.apache.thrift.TBase)new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, e.getMessage());
- }
- try {
- fcall.sendResponse(fb,msg,msgType,seqid);
- return;
- } catch (Exception ex) {
- LOGGER.error("Exception writing to internal frame buffer", ex);
- }
- fb.close();
- }
- };
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public void start(I iface, add_args args, org.apache.thrift.async.AsyncMethodCallback<Void> resultHandler) throws TException {
- iface.add(args.key, args.column_parent, args.column, args.consistency_level,resultHandler);
- }
- }
-
- public static class cas<I extends AsyncIface> extends org.apache.thrift.AsyncProcessFunction<I, cas_args, CASResult> {
- public cas() {
- super("cas");
- }
-
- public cas_args getEmptyArgsInstance() {
- return new cas_args();
- }
-
- public AsyncMethodCallback<CASResult> getResultHandler(final AsyncFrameBuffer fb, final int seqid) {
- final org.apache.thrift.AsyncProcessFunction fcall = this;
- return new AsyncMethodCallback<CASResult>() {
- public void onComplete(CASResult o) {
- cas_result result = new cas_result();
- result.success = o;
- try {
- fcall.sendResponse(fb,result, org.apache.thrift.protocol.TMessageType.REPLY,seqid);
- return;
- } catch (Exception e) {
- LOGGER.error("Exception writing to internal frame buffer", e);
- }
- fb.close();
- }
- public void onError(Exception e) {
- byte msgType = org.apache.thrift.protocol.TMessageType.REPLY;
- org.apache.thrift.TBase msg;
- cas_result result = new cas_result();
- if (e instanceof InvalidRequestException) {
- result.ire = (InvalidRequestException) e;
- result.setIreIsSet(true);
- msg = result;
- }
- else if (e instanceof UnavailableException) {
- result.ue = (UnavailableException) e;
- result.setUeIsSet(true);
- msg = result;
- }
- else if (e instanceof TimedOutException) {
- result.te = (TimedOutException) e;
- result.setTeIsSet(true);
- msg = result;
- }
- else
- {
- msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION;
- msg = (org.apache.thrift.TBase)new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, e.getMessage());
- }
- try {
- fcall.sendResponse(fb,msg,msgType,seqid);
- return;
- } catch (Exception ex) {
- LOGGER.error("Exception writing to internal frame buffer", ex);
- }
- fb.close();
- }
- };
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public void start(I iface, cas_args args, org.apache.thrift.async.AsyncMethodCallback<CASResult> resultHandler) throws TException {
- iface.cas(args.key, args.column_family, args.expected, args.updates, args.serial_consistency_level, args.commit_consistency_level,resultHandler);
- }
- }
-
- public static class remove<I extends AsyncIface> extends org.apache.thrift.AsyncProcessFunction<I, remove_args, Void> {
- public remove() {
- super("remove");
- }
-
- public remove_args getEmptyArgsInstance() {
- return new remove_args();
- }
-
- public AsyncMethodCallback<Void> getResultHandler(final AsyncFrameBuffer fb, final int seqid) {
- final org.apache.thrift.AsyncProcessFunction fcall = this;
- return new AsyncMethodCallback<Void>() {
- public void onComplete(Void o) {
- remove_result result = new remove_result();
- try {
- fcall.sendResponse(fb,result, org.apache.thrift.protocol.TMessageType.REPLY,seqid);
- return;
- } catch (Exception e) {
- LOGGER.error("Exception writing to internal frame buffer", e);
- }
- fb.close();
- }
- public void onError(Exception e) {
- byte msgType = org.apache.thrift.protocol.TMessageType.REPLY;
- org.apache.thrift.TBase msg;
- remove_result result = new remove_result();
- if (e instanceof InvalidRequestException) {
- result.ire = (InvalidRequestException) e;
- result.setIreIsSet(true);
- msg = result;
- }
- else if (e instanceof UnavailableException) {
- result.ue = (UnavailableException) e;
- result.setUeIsSet(true);
- msg = result;
- }
- else if (e instanceof TimedOutException) {
- result.te = (TimedOutException) e;
- result.setTeIsSet(true);
- msg = result;
- }
- else
- {
- msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION;
- msg = (org.apache.thrift.TBase)new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, e.getMessage());
- }
- try {
- fcall.sendResponse(fb,msg,msgType,seqid);
- return;
- } catch (Exception ex) {
- LOGGER.error("Exception writing to internal frame buffer", ex);
- }
- fb.close();
- }
- };
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public void start(I iface, remove_args args, org.apache.thrift.async.AsyncMethodCallback<Void> resultHandler) throws TException {
- iface.remove(args.key, args.column_path, args.timestamp, args.consistency_level,resultHandler);
- }
- }
-
- public static class remove_counter<I extends AsyncIface> extends org.apache.thrift.AsyncProcessFunction<I, remove_counter_args, Void> {
- public remove_counter() {
- super("remove_counter");
- }
-
- public remove_counter_args getEmptyArgsInstance() {
- return new remove_counter_args();
- }
-
- public AsyncMethodCallback<Void> getResultHandler(final AsyncFrameBuffer fb, final int seqid) {
- final org.apache.thrift.AsyncProcessFunction fcall = this;
- return new AsyncMethodCallback<Void>() {
- public void onComplete(Void o) {
- remove_counter_result result = new remove_counter_result();
- try {
- fcall.sendResponse(fb,result, org.apache.thrift.protocol.TMessageType.REPLY,seqid);
- return;
- } catch (Exception e) {
- LOGGER.error("Exception writing to internal frame buffer", e);
- }
- fb.close();
- }
- public void onError(Exception e) {
- byte msgType = org.apache.thrift.protocol.TMessageType.REPLY;
- org.apache.thrift.TBase msg;
- remove_counter_result result = new remove_counter_result();
- if (e instanceof InvalidRequestException) {
- result.ire = (InvalidRequestException) e;
- result.setIreIsSet(true);
- msg = result;
- }
- else if (e instanceof UnavailableException) {
- result.ue = (UnavailableException) e;
- result.setUeIsSet(true);
- msg = result;
- }
- else if (e instanceof TimedOutException) {
- result.te = (TimedOutException) e;
- result.setTeIsSet(true);
- msg = result;
- }
- else
- {
- msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION;
- msg = (org.apache.thrift.TBase)new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, e.getMessage());
- }
- try {
- fcall.sendResponse(fb,msg,msgType,seqid);
- return;
- } catch (Exception ex) {
- LOGGER.error("Exception writing to internal frame buffer", ex);
- }
- fb.close();
- }
- };
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public void start(I iface, remove_counter_args args, org.apache.thrift.async.AsyncMethodCallback<Void> resultHandler) throws TException {
- iface.remove_counter(args.key, args.path, args.consistency_level,resultHandler);
- }
- }
-
- public static class batch_mutate<I extends AsyncIface> extends org.apache.thrift.AsyncProcessFunction<I, batch_mutate_args, Void> {
- public batch_mutate() {
- super("batch_mutate");
- }
-
- public batch_mutate_args getEmptyArgsInstance() {
- return new batch_mutate_args();
- }
-
- public AsyncMethodCallback<Void> getResultHandler(final AsyncFrameBuffer fb, final int seqid) {
- final org.apache.thrift.AsyncProcessFunction fcall = this;
- return new AsyncMethodCallback<Void>() {
- public void onComplete(Void o) {
- batch_mutate_result result = new batch_mutate_result();
- try {
- fcall.sendResponse(fb,result, org.apache.thrift.protocol.TMessageType.REPLY,seqid);
- return;
- } catch (Exception e) {
- LOGGER.error("Exception writing to internal frame buffer", e);
- }
- fb.close();
- }
- public void onError(Exception e) {
- byte msgType = org.apache.thrift.protocol.TMessageType.REPLY;
- org.apache.thrift.TBase msg;
- batch_mutate_result result = new batch_mutate_result();
- if (e instanceof InvalidRequestException) {
- result.ire = (InvalidRequestException) e;
- result.setIreIsSet(true);
- msg = result;
- }
- else if (e instanceof UnavailableException) {
- result.ue = (UnavailableException) e;
- result.setUeIsSet(true);
- msg = result;
- }
- else if (e instanceof TimedOutException) {
- result.te = (TimedOutException) e;
- result.setTeIsSet(true);
- msg = result;
- }
- else
- {
- msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION;
- msg = (org.apache.thrift.TBase)new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, e.getMessage());
- }
- try {
- fcall.sendResponse(fb,msg,msgType,seqid);
- return;
- } catch (Exception ex) {
- LOGGER.error("Exception writing to internal frame buffer", ex);
- }
- fb.close();
- }
- };
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public void start(I iface, batch_mutate_args args, org.apache.thrift.async.AsyncMethodCallback<Void> resultHandler) throws TException {
- iface.batch_mutate(args.mutation_map, args.consistency_level,resultHandler);
- }
- }
-
- public static class atomic_batch_mutate<I extends AsyncIface> extends org.apache.thrift.AsyncProcessFunction<I, atomic_batch_mutate_args, Void> {
- public atomic_batch_mutate() {
- super("atomic_batch_mutate");
- }
-
- public atomic_batch_mutate_args getEmptyArgsInstance() {
- return new atomic_batch_mutate_args();
- }
-
- public AsyncMethodCallback<Void> getResultHandler(final AsyncFrameBuffer fb, final int seqid) {
- final org.apache.thrift.AsyncProcessFunction fcall = this;
- return new AsyncMethodCallback<Void>() {
- public void onComplete(Void o) {
- atomic_batch_mutate_result result = new atomic_batch_mutate_result();
- try {
- fcall.sendResponse(fb,result, org.apache.thrift.protocol.TMessageType.REPLY,seqid);
- return;
- } catch (Exception e) {
- LOGGER.error("Exception writing to internal frame buffer", e);
- }
- fb.close();
- }
- public void onError(Exception e) {
- byte msgType = org.apache.thrift.protocol.TMessageType.REPLY;
- org.apache.thrift.TBase msg;
- atomic_batch_mutate_result result = new atomic_batch_mutate_result();
- if (e instanceof InvalidRequestException) {
- result.ire = (InvalidRequestException) e;
- result.setIreIsSet(true);
- msg = result;
- }
- else if (e instanceof UnavailableException) {
- result.ue = (UnavailableException) e;
- result.setUeIsSet(true);
- msg = result;
- }
- else if (e instanceof TimedOutException) {
- result.te = (TimedOutException) e;
- result.setTeIsSet(true);
- msg = result;
- }
- else
- {
- msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION;
- msg = (org.apache.thrift.TBase)new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, e.getMessage());
- }
- try {
- fcall.sendResponse(fb,msg,msgType,seqid);
- return;
- } catch (Exception ex) {
- LOGGER.error("Exception writing to internal frame buffer", ex);
- }
- fb.close();
- }
- };
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public void start(I iface, atomic_batch_mutate_args args, org.apache.thrift.async.AsyncMethodCallback<Void> resultHandler) throws TException {
- iface.atomic_batch_mutate(args.mutation_map, args.consistency_level,resultHandler);
- }
- }
-
- public static class truncate<I extends AsyncIface> extends org.apache.thrift.AsyncProcessFunction<I, truncate_args, Void> {
- public truncate() {
- super("truncate");
- }
-
- public truncate_args getEmptyArgsInstance() {
- return new truncate_args();
- }
-
- public AsyncMethodCallback<Void> getResultHandler(final AsyncFrameBuffer fb, final int seqid) {
- final org.apache.thrift.AsyncProcessFunction fcall = this;
- return new AsyncMethodCallback<Void>() {
- public void onComplete(Void o) {
- truncate_result result = new truncate_result();
- try {
- fcall.sendResponse(fb,result, org.apache.thrift.protocol.TMessageType.REPLY,seqid);
- return;
- } catch (Exception e) {
- LOGGER.error("Exception writing to internal frame buffer", e);
- }
- fb.close();
- }
- public void onError(Exception e) {
- byte msgType = org.apache.thrift.protocol.TMessageType.REPLY;
- org.apache.thrift.TBase msg;
- truncate_result result = new truncate_result();
- if (e instanceof InvalidRequestException) {
- result.ire = (InvalidRequestException) e;
- result.setIreIsSet(true);
- msg = result;
- }
- else if (e instanceof UnavailableException) {
- result.ue = (UnavailableException) e;
- result.setUeIsSet(true);
- msg = result;
- }
- else if (e instanceof TimedOutException) {
- result.te = (TimedOutException) e;
- result.setTeIsSet(true);
- msg = result;
- }
- else
- {
- msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION;
- msg = (org.apache.thrift.TBase)new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, e.getMessage());
- }
- try {
- fcall.sendResponse(fb,msg,msgType,seqid);
- return;
- } catch (Exception ex) {
- LOGGER.error("Exception writing to internal frame buffer", ex);
- }
- fb.close();
- }
- };
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public void start(I iface, truncate_args args, org.apache.thrift.async.AsyncMethodCallback<Void> resultHandler) throws TException {
- iface.truncate(args.cfname,resultHandler);
- }
- }
-
- public static class get_multi_slice<I extends AsyncIface> extends org.apache.thrift.AsyncProcessFunction<I, get_multi_slice_args, List<ColumnOrSuperColumn>> {
- public get_multi_slice() {
- super("get_multi_slice");
- }
-
- public get_multi_slice_args getEmptyArgsInstance() {
- return new get_multi_slice_args();
- }
-
- public AsyncMethodCallback<List<ColumnOrSuperColumn>> getResultHandler(final AsyncFrameBuffer fb, final int seqid) {
- final org.apache.thrift.AsyncProcessFunction fcall = this;
- return new AsyncMethodCallback<List<ColumnOrSuperColumn>>() {
- public void onComplete(List<ColumnOrSuperColumn> o) {
- get_multi_slice_result result = new get_multi_slice_result();
- result.success = o;
- try {
- fcall.sendResponse(fb,result, org.apache.thrift.protocol.TMessageType.REPLY,seqid);
- return;
- } catch (Exception e) {
- LOGGER.error("Exception writing to internal frame buffer", e);
- }
- fb.close();
- }
- public void onError(Exception e) {
- byte msgType = org.apache.thrift.protocol.TMessageType.REPLY;
- org.apache.thrift.TBase msg;
- get_multi_slice_result result = new get_multi_slice_result();
- if (e instanceof InvalidRequestException) {
- result.ire = (InvalidRequestException) e;
- result.setIreIsSet(true);
- msg = result;
- }
- else if (e instanceof UnavailableException) {
- result.ue = (UnavailableException) e;
- result.setUeIsSet(true);
- msg = result;
- }
- else if (e instanceof TimedOutException) {
- result.te = (TimedOutException) e;
- result.setTeIsSet(true);
- msg = result;
- }
- else
- {
- msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION;
- msg = (org.apache.thrift.TBase)new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, e.getMessage());
- }
- try {
- fcall.sendResponse(fb,msg,msgType,seqid);
- return;
- } catch (Exception ex) {
- LOGGER.error("Exception writing to internal frame buffer", ex);
- }
- fb.close();
- }
- };
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public void start(I iface, get_multi_slice_args args, org.apache.thrift.async.AsyncMethodCallback<List<ColumnOrSuperColumn>> resultHandler) throws TException {
- iface.get_multi_slice(args.request,resultHandler);
- }
- }
-
- public static class describe_schema_versions<I extends AsyncIface> extends org.apache.thrift.AsyncProcessFunction<I, describe_schema_versions_args, Map<String,List<String>>> {
- public describe_schema_versions() {
- super("describe_schema_versions");
- }
-
- public describe_schema_versions_args getEmptyArgsInstance() {
- return new describe_schema_versions_args();
- }
-
- public AsyncMethodCallback<Map<String,List<String>>> getResultHandler(final AsyncFrameBuffer fb, final int seqid) {
- final org.apache.thrift.AsyncProcessFunction fcall = this;
- return new AsyncMethodCallback<Map<String,List<String>>>() {
- public void onComplete(Map<String,List<String>> o) {
- describe_schema_versions_result result = new describe_schema_versions_result();
- result.success = o;
- try {
- fcall.sendResponse(fb,result, org.apache.thrift.protocol.TMessageType.REPLY,seqid);
- return;
- } catch (Exception e) {
- LOGGER.error("Exception writing to internal frame buffer", e);
- }
- fb.close();
- }
- public void onError(Exception e) {
- byte msgType = org.apache.thrift.protocol.TMessageType.REPLY;
- org.apache.thrift.TBase msg;
- describe_schema_versions_result result = new describe_schema_versions_result();
- if (e instanceof InvalidRequestException) {
- result.ire = (InvalidRequestException) e;
- result.setIreIsSet(true);
- msg = result;
- }
- else
- {
- msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION;
- msg = (org.apache.thrift.TBase)new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, e.getMessage());
- }
- try {
- fcall.sendResponse(fb,msg,msgType,seqid);
- return;
- } catch (Exception ex) {
- LOGGER.error("Exception writing to internal frame buffer", ex);
- }
- fb.close();
- }
- };
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public void start(I iface, describe_schema_versions_args args, org.apache.thrift.async.AsyncMethodCallback<Map<String,List<String>>> resultHandler) throws TException {
- iface.describe_schema_versions(resultHandler);
- }
- }
-
- public static class describe_keyspaces<I extends AsyncIface> extends org.apache.thrift.AsyncProcessFunction<I, describe_keyspaces_args, List<KsDef>> {
- public describe_keyspaces() {
- super("describe_keyspaces");
- }
-
- public describe_keyspaces_args getEmptyArgsInstance() {
- return new describe_keyspaces_args();
- }
-
- public AsyncMethodCallback<List<KsDef>> getResultHandler(final AsyncFrameBuffer fb, final int seqid) {
- final org.apache.thrift.AsyncProcessFunction fcall = this;
- return new AsyncMethodCallback<List<KsDef>>() {
- public void onComplete(List<KsDef> o) {
- describe_keyspaces_result result = new describe_keyspaces_result();
- result.success = o;
- try {
- fcall.sendResponse(fb,result, org.apache.thrift.protocol.TMessageType.REPLY,seqid);
- return;
- } catch (Exception e) {
- LOGGER.error("Exception writing to internal frame buffer", e);
- }
- fb.close();
- }
- public void onError(Exception e) {
- byte msgType = org.apache.thrift.protocol.TMessageType.REPLY;
- org.apache.thrift.TBase msg;
- describe_keyspaces_result result = new describe_keyspaces_result();
- if (e instanceof InvalidRequestException) {
- result.ire = (InvalidRequestException) e;
- result.setIreIsSet(true);
- msg = result;
- }
- else
- {
- msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION;
- msg = (org.apache.thrift.TBase)new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, e.getMessage());
- }
- try {
- fcall.sendResponse(fb,msg,msgType,seqid);
- return;
- } catch (Exception ex) {
- LOGGER.error("Exception writing to internal frame buffer", ex);
- }
- fb.close();
- }
- };
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public void start(I iface, describe_keyspaces_args args, org.apache.thrift.async.AsyncMethodCallback<List<KsDef>> resultHandler) throws TException {
- iface.describe_keyspaces(resultHandler);
- }
- }
-
- public static class describe_cluster_name<I extends AsyncIface> extends org.apache.thrift.AsyncProcessFunction<I, describe_cluster_name_args, String> {
- public describe_cluster_name() {
- super("describe_cluster_name");
- }
-
- public describe_cluster_name_args getEmptyArgsInstance() {
- return new describe_cluster_name_args();
- }
-
- public AsyncMethodCallback<String> getResultHandler(final AsyncFrameBuffer fb, final int seqid) {
- final org.apache.thrift.AsyncProcessFunction fcall = this;
- return new AsyncMethodCallback<String>() {
- public void onComplete(String o) {
- describe_cluster_name_result result = new describe_cluster_name_result();
- result.success = o;
- try {
- fcall.sendResponse(fb,result, org.apache.thrift.protocol.TMessageType.REPLY,seqid);
- return;
- } catch (Exception e) {
- LOGGER.error("Exception writing to internal frame buffer", e);
- }
- fb.close();
- }
- public void onError(Exception e) {
- byte msgType = org.apache.thrift.protocol.TMessageType.REPLY;
- org.apache.thrift.TBase msg;
- describe_cluster_name_result result = new describe_cluster_name_result();
- {
- msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION;
- msg = (org.apache.thrift.TBase)new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, e.getMessage());
- }
- try {
- fcall.sendResponse(fb,msg,msgType,seqid);
- return;
- } catch (Exception ex) {
- LOGGER.error("Exception writing to internal frame buffer", ex);
- }
- fb.close();
- }
- };
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public void start(I iface, describe_cluster_name_args args, org.apache.thrift.async.AsyncMethodCallback<String> resultHandler) throws TException {
- iface.describe_cluster_name(resultHandler);
- }
- }
-
- public static class describe_version<I extends AsyncIface> extends org.apache.thrift.AsyncProcessFunction<I, describe_version_args, String> {
- public describe_version() {
- super("describe_version");
- }
-
- public describe_version_args getEmptyArgsInstance() {
- return new describe_version_args();
- }
-
- public AsyncMethodCallback<String> getResultHandler(final AsyncFrameBuffer fb, final int seqid) {
- final org.apache.thrift.AsyncProcessFunction fcall = this;
- return new AsyncMethodCallback<String>() {
- public void onComplete(String o) {
- describe_version_result result = new describe_version_result();
- result.success = o;
- try {
- fcall.sendResponse(fb,result, org.apache.thrift.protocol.TMessageType.REPLY,seqid);
- return;
- } catch (Exception e) {
- LOGGER.error("Exception writing to internal frame buffer", e);
- }
- fb.close();
- }
- public void onError(Exception e) {
- byte msgType = org.apache.thrift.protocol.TMessageType.REPLY;
- org.apache.thrift.TBase msg;
- describe_version_result result = new describe_version_result();
- {
- msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION;
- msg = (org.apache.thrift.TBase)new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, e.getMessage());
- }
- try {
- fcall.sendResponse(fb,msg,msgType,seqid);
- return;
- } catch (Exception ex) {
- LOGGER.error("Exception writing to internal frame buffer", ex);
- }
- fb.close();
- }
- };
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public void start(I iface, describe_version_args args, org.apache.thrift.async.AsyncMethodCallback<String> resultHandler) throws TException {
- iface.describe_version(resultHandler);
- }
- }
-
- public static class describe_ring<I extends AsyncIface> extends org.apache.thrift.AsyncProcessFunction<I, describe_ring_args, List<TokenRange>> {
- public describe_ring() {
- super("describe_ring");
- }
-
- public describe_ring_args getEmptyArgsInstance() {
- return new describe_ring_args();
- }
-
- public AsyncMethodCallback<List<TokenRange>> getResultHandler(final AsyncFrameBuffer fb, final int seqid) {
- final org.apache.thrift.AsyncProcessFunction fcall = this;
- return new AsyncMethodCallback<List<TokenRange>>() {
- public void onComplete(List<TokenRange> o) {
- describe_ring_result result = new describe_ring_result();
- result.success = o;
- try {
- fcall.sendResponse(fb,result, org.apache.thrift.protocol.TMessageType.REPLY,seqid);
- return;
- } catch (Exception e) {
- LOGGER.error("Exception writing to internal frame buffer", e);
- }
- fb.close();
- }
- public void onError(Exception e) {
- byte msgType = org.apache.thrift.protocol.TMessageType.REPLY;
- org.apache.thrift.TBase msg;
- describe_ring_result result = new describe_ring_result();
- if (e instanceof InvalidRequestException) {
- result.ire = (InvalidRequestException) e;
- result.setIreIsSet(true);
- msg = result;
- }
- else
- {
- msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION;
- msg = (org.apache.thrift.TBase)new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, e.getMessage());
- }
- try {
- fcall.sendResponse(fb,msg,msgType,seqid);
- return;
- } catch (Exception ex) {
- LOGGER.error("Exception writing to internal frame buffer", ex);
- }
- fb.close();
- }
- };
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public void start(I iface, describe_ring_args args, org.apache.thrift.async.AsyncMethodCallback<List<TokenRange>> resultHandler) throws TException {
- iface.describe_ring(args.keyspace,resultHandler);
- }
- }
-
- public static class describe_local_ring<I extends AsyncIface> extends org.apache.thrift.AsyncProcessFunction<I, describe_local_ring_args, List<TokenRange>> {
- public describe_local_ring() {
- super("describe_local_ring");
- }
-
- public describe_local_ring_args getEmptyArgsInstance() {
- return new describe_local_ring_args();
- }
-
- public AsyncMethodCallback<List<TokenRange>> getResultHandler(final AsyncFrameBuffer fb, final int seqid) {
- final org.apache.thrift.AsyncProcessFunction fcall = this;
- return new AsyncMethodCallback<List<TokenRange>>() {
- public void onComplete(List<TokenRange> o) {
- describe_local_ring_result result = new describe_local_ring_result();
- result.success = o;
- try {
- fcall.sendResponse(fb,result, org.apache.thrift.protocol.TMessageType.REPLY,seqid);
- return;
- } catch (Exception e) {
- LOGGER.error("Exception writing to internal frame buffer", e);
- }
- fb.close();
- }
- public void onError(Exception e) {
- byte msgType = org.apache.thrift.protocol.TMessageType.REPLY;
- org.apache.thrift.TBase msg;
- describe_local_ring_result result = new describe_local_ring_result();
- if (e instanceof InvalidRequestException) {
- result.ire = (InvalidRequestException) e;
- result.setIreIsSet(true);
- msg = result;
- }
- else
- {
- msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION;
- msg = (org.apache.thrift.TBase)new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, e.getMessage());
- }
- try {
- fcall.sendResponse(fb,msg,msgType,seqid);
- return;
- } catch (Exception ex) {
- LOGGER.error("Exception writing to internal frame buffer", ex);
- }
- fb.close();
- }
- };
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public void start(I iface, describe_local_ring_args args, org.apache.thrift.async.AsyncMethodCallback<List<TokenRange>> resultHandler) throws TException {
- iface.describe_local_ring(args.keyspace,resultHandler);
- }
- }
-
- public static class describe_token_map<I extends AsyncIface> extends org.apache.thrift.AsyncProcessFunction<I, describe_token_map_args, Map<String,String>> {
- public describe_token_map() {
- super("describe_token_map");
- }
-
- public describe_token_map_args getEmptyArgsInstance() {
- return new describe_token_map_args();
- }
-
- public AsyncMethodCallback<Map<String,String>> getResultHandler(final AsyncFrameBuffer fb, final int seqid) {
- final org.apache.thrift.AsyncProcessFunction fcall = this;
- return new AsyncMethodCallback<Map<String,String>>() {
- public void onComplete(Map<String,String> o) {
- describe_token_map_result result = new describe_token_map_result();
- result.success = o;
- try {
- fcall.sendResponse(fb,result, org.apache.thrift.protocol.TMessageType.REPLY,seqid);
- return;
- } catch (Exception e) {
- LOGGER.error("Exception writing to internal frame buffer", e);
- }
- fb.close();
- }
- public void onError(Exception e) {
- byte msgType = org.apache.thrift.protocol.TMessageType.REPLY;
- org.apache.thrift.TBase msg;
- describe_token_map_result result = new describe_token_map_result();
- if (e instanceof InvalidRequestException) {
- result.ire = (InvalidRequestException) e;
- result.setIreIsSet(true);
- msg = result;
- }
- else
- {
- msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION;
- msg = (org.apache.thrift.TBase)new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, e.getMessage());
- }
- try {
- fcall.sendResponse(fb,msg,msgType,seqid);
- return;
- } catch (Exception ex) {
- LOGGER.error("Exception writing to internal frame buffer", ex);
- }
- fb.close();
- }
- };
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public void start(I iface, describe_token_map_args args, org.apache.thrift.async.AsyncMethodCallback<Map<String,String>> resultHandler) throws TException {
- iface.describe_token_map(resultHandler);
- }
- }
-
- public static class describe_partitioner<I extends AsyncIface> extends org.apache.thrift.AsyncProcessFunction<I, describe_partitioner_args, String> {
- public describe_partitioner() {
- super("describe_partitioner");
- }
-
- public describe_partitioner_args getEmptyArgsInstance() {
- return new describe_partitioner_args();
- }
-
- public AsyncMethodCallback<String> getResultHandler(final AsyncFrameBuffer fb, final int seqid) {
- final org.apache.thrift.AsyncProcessFunction fcall = this;
- return new AsyncMethodCallback<String>() {
- public void onComplete(String o) {
- describe_partitioner_result result = new describe_partitioner_result();
- result.success = o;
- try {
- fcall.sendResponse(fb,result, org.apache.thrift.protocol.TMessageType.REPLY,seqid);
- return;
- } catch (Exception e) {
- LOGGER.error("Exception writing to internal frame buffer", e);
- }
- fb.close();
- }
- public void onError(Exception e) {
- byte msgType = org.apache.thrift.protocol.TMessageType.REPLY;
- org.apache.thrift.TBase msg;
- describe_partitioner_result result = new describe_partitioner_result();
- {
- msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION;
- msg = (org.apache.thrift.TBase)new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, e.getMessage());
- }
- try {
- fcall.sendResponse(fb,msg,msgType,seqid);
- return;
- } catch (Exception ex) {
- LOGGER.error("Exception writing to internal frame buffer", ex);
- }
- fb.close();
- }
- };
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public void start(I iface, describe_partitioner_args args, org.apache.thrift.async.AsyncMethodCallback<String> resultHandler) throws TException {
- iface.describe_partitioner(resultHandler);
- }
- }
-
- public static class describe_snitch<I extends AsyncIface> extends org.apache.thrift.AsyncProcessFunction<I, describe_snitch_args, String> {
- public describe_snitch() {
- super("describe_snitch");
- }
-
- public describe_snitch_args getEmptyArgsInstance() {
- return new describe_snitch_args();
- }
-
- public AsyncMethodCallback<String> getResultHandler(final AsyncFrameBuffer fb, final int seqid) {
- final org.apache.thrift.AsyncProcessFunction fcall = this;
- return new AsyncMethodCallback<String>() {
- public void onComplete(String o) {
- describe_snitch_result result = new describe_snitch_result();
- result.success = o;
- try {
- fcall.sendResponse(fb,result, org.apache.thrift.protocol.TMessageType.REPLY,seqid);
- return;
- } catch (Exception e) {
- LOGGER.error("Exception writing to internal frame buffer", e);
- }
- fb.close();
- }
- public void onError(Exception e) {
- byte msgType = org.apache.thrift.protocol.TMessageType.REPLY;
- org.apache.thrift.TBase msg;
- describe_snitch_result result = new describe_snitch_result();
- {
- msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION;
- msg = (org.apache.thrift.TBase)new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, e.getMessage());
- }
- try {
- fcall.sendResponse(fb,msg,msgType,seqid);
- return;
- } catch (Exception ex) {
- LOGGER.error("Exception writing to internal frame buffer", ex);
- }
- fb.close();
- }
- };
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public void start(I iface, describe_snitch_args args, org.apache.thrift.async.AsyncMethodCallback<String> resultHandler) throws TException {
- iface.describe_snitch(resultHandler);
- }
- }
-
- public static class describe_keyspace<I extends AsyncIface> extends org.apache.thrift.AsyncProcessFunction<I, describe_keyspace_args, KsDef> {
- public describe_keyspace() {
- super("describe_keyspace");
- }
-
- public describe_keyspace_args getEmptyArgsInstance() {
- return new describe_keyspace_args();
- }
-
- public AsyncMethodCallback<KsDef> getResultHandler(final AsyncFrameBuffer fb, final int seqid) {
- final org.apache.thrift.AsyncProcessFunction fcall = this;
- return new AsyncMethodCallback<KsDef>() {
- public void onComplete(KsDef o) {
- describe_keyspace_result result = new describe_keyspace_result();
- result.success = o;
- try {
- fcall.sendResponse(fb,result, org.apache.thrift.protocol.TMessageType.REPLY,seqid);
- return;
- } catch (Exception e) {
- LOGGER.error("Exception writing to internal frame buffer", e);
- }
- fb.close();
- }
- public void onError(Exception e) {
- byte msgType = org.apache.thrift.protocol.TMessageType.REPLY;
- org.apache.thrift.TBase msg;
- describe_keyspace_result result = new describe_keyspace_result();
- if (e instanceof NotFoundException) {
- result.nfe = (NotFoundException) e;
- result.setNfeIsSet(true);
- msg = result;
- }
- else if (e instanceof InvalidRequestException) {
- result.ire = (InvalidRequestException) e;
- result.setIreIsSet(true);
- msg = result;
- }
- else
- {
- msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION;
- msg = (org.apache.thrift.TBase)new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, e.getMessage());
- }
- try {
- fcall.sendResponse(fb,msg,msgType,seqid);
- return;
- } catch (Exception ex) {
- LOGGER.error("Exception writing to internal frame buffer", ex);
- }
- fb.close();
- }
- };
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public void start(I iface, describe_keyspace_args args, org.apache.thrift.async.AsyncMethodCallback<KsDef> resultHandler) throws TException {
- iface.describe_keyspace(args.keyspace,resultHandler);
- }
- }
-
- public static class describe_splits<I extends AsyncIface> extends org.apache.thrift.AsyncProcessFunction<I, describe_splits_args, List<String>> {
- public describe_splits() {
- super("describe_splits");
- }
-
- public describe_splits_args getEmptyArgsInstance() {
- return new describe_splits_args();
- }
-
- public AsyncMethodCallback<List<String>> getResultHandler(final AsyncFrameBuffer fb, final int seqid) {
- final org.apache.thrift.AsyncProcessFunction fcall = this;
- return new AsyncMethodCallback<List<String>>() {
- public void onComplete(List<String> o) {
- describe_splits_result result = new describe_splits_result();
- result.success = o;
- try {
- fcall.sendResponse(fb,result, org.apache.thrift.protocol.TMessageType.REPLY,seqid);
- return;
- } catch (Exception e) {
- LOGGER.error("Exception writing to internal frame buffer", e);
- }
- fb.close();
- }
- public void onError(Exception e) {
- byte msgType = org.apache.thrift.protocol.TMessageType.REPLY;
- org.apache.thrift.TBase msg;
- describe_splits_result result = new describe_splits_result();
- if (e instanceof InvalidRequestException) {
- result.ire = (InvalidRequestException) e;
- result.setIreIsSet(true);
- msg = result;
- }
- else
- {
- msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION;
- msg = (org.apache.thrift.TBase)new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, e.getMessage());
- }
- try {
- fcall.sendResponse(fb,msg,msgType,seqid);
- return;
- } catch (Exception ex) {
- LOGGER.error("Exception writing to internal frame buffer", ex);
- }
- fb.close();
- }
- };
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public void start(I iface, describe_splits_args args, org.apache.thrift.async.AsyncMethodCallback<List<String>> resultHandler) throws TException {
- iface.describe_splits(args.cfName, args.start_token, args.end_token, args.keys_per_split,resultHandler);
- }
- }
-
- public static class trace_next_query<I extends AsyncIface> extends org.apache.thrift.AsyncProcessFunction<I, trace_next_query_args, ByteBuffer> {
- public trace_next_query() {
- super("trace_next_query");
- }
-
- public trace_next_query_args getEmptyArgsInstance() {
- return new trace_next_query_args();
- }
-
- public AsyncMethodCallback<ByteBuffer> getResultHandler(final AsyncFrameBuffer fb, final int seqid) {
- final org.apache.thrift.AsyncProcessFunction fcall = this;
- return new AsyncMethodCallback<ByteBuffer>() {
- public void onComplete(ByteBuffer o) {
- trace_next_query_result result = new trace_next_query_result();
- result.success = o;
- try {
- fcall.sendResponse(fb,result, org.apache.thrift.protocol.TMessageType.REPLY,seqid);
- return;
- } catch (Exception e) {
- LOGGER.error("Exception writing to internal frame buffer", e);
- }
- fb.close();
- }
- public void onError(Exception e) {
- byte msgType = org.apache.thrift.protocol.TMessageType.REPLY;
- org.apache.thrift.TBase msg;
- trace_next_query_result result = new trace_next_query_result();
- {
- msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION;
- msg = (org.apache.thrift.TBase)new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, e.getMessage());
- }
- try {
- fcall.sendResponse(fb,msg,msgType,seqid);
- return;
- } catch (Exception ex) {
- LOGGER.error("Exception writing to internal frame buffer", ex);
- }
- fb.close();
- }
- };
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public void start(I iface, trace_next_query_args args, org.apache.thrift.async.AsyncMethodCallback<ByteBuffer> resultHandler) throws TException {
- iface.trace_next_query(resultHandler);
- }
- }
-
- public static class describe_splits_ex<I extends AsyncIface> extends org.apache.thrift.AsyncProcessFunction<I, describe_splits_ex_args, List<CfSplit>> {
- public describe_splits_ex() {
- super("describe_splits_ex");
- }
-
- public describe_splits_ex_args getEmptyArgsInstance() {
- return new describe_splits_ex_args();
- }
-
- public AsyncMethodCallback<List<CfSplit>> getResultHandler(final AsyncFrameBuffer fb, final int seqid) {
- final org.apache.thrift.AsyncProcessFunction fcall = this;
- return new AsyncMethodCallback<List<CfSplit>>() {
- public void onComplete(List<CfSplit> o) {
- describe_splits_ex_result result = new describe_splits_ex_result();
- result.success = o;
- try {
- fcall.sendResponse(fb,result, org.apache.thrift.protocol.TMessageType.REPLY,seqid);
- return;
- } catch (Exception e) {
- LOGGER.error("Exception writing to internal frame buffer", e);
- }
- fb.close();
- }
- public void onError(Exception e) {
- byte msgType = org.apache.thrift.protocol.TMessageType.REPLY;
- org.apache.thrift.TBase msg;
- describe_splits_ex_result result = new describe_splits_ex_result();
- if (e instanceof InvalidRequestException) {
- result.ire = (InvalidRequestException) e;
- result.setIreIsSet(true);
- msg = result;
- }
- else
- {
- msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION;
- msg = (org.apache.thrift.TBase)new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, e.getMessage());
- }
- try {
- fcall.sendResponse(fb,msg,msgType,seqid);
- return;
- } catch (Exception ex) {
- LOGGER.error("Exception writing to internal frame buffer", ex);
- }
- fb.close();
- }
- };
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public void start(I iface, describe_splits_ex_args args, org.apache.thrift.async.AsyncMethodCallback<List<CfSplit>> resultHandler) throws TException {
- iface.describe_splits_ex(args.cfName, args.start_token, args.end_token, args.keys_per_split,resultHandler);
- }
- }
-
- public static class system_add_column_family<I extends AsyncIface> extends org.apache.thrift.AsyncProcessFunction<I, system_add_column_family_args, String> {
- public system_add_column_family() {
- super("system_add_column_family");
- }
-
- public system_add_column_family_args getEmptyArgsInstance() {
- return new system_add_column_family_args();
- }
-
- public AsyncMethodCallback<String> getResultHandler(final AsyncFrameBuffer fb, final int seqid) {
- final org.apache.thrift.AsyncProcessFunction fcall = this;
- return new AsyncMethodCallback<String>() {
- public void onComplete(String o) {
- system_add_column_family_result result = new system_add_column_family_result();
- result.success = o;
- try {
- fcall.sendResponse(fb,result, org.apache.thrift.protocol.TMessageType.REPLY,seqid);
- return;
- } catch (Exception e) {
- LOGGER.error("Exception writing to internal frame buffer", e);
- }
- fb.close();
- }
- public void onError(Exception e) {
- byte msgType = org.apache.thrift.protocol.TMessageType.REPLY;
- org.apache.thrift.TBase msg;
- system_add_column_family_result result = new system_add_column_family_result();
- if (e instanceof InvalidRequestException) {
- result.ire = (InvalidRequestException) e;
- result.setIreIsSet(true);
- msg = result;
- }
- else if (e instanceof SchemaDisagreementException) {
- result.sde = (SchemaDisagreementException) e;
- result.setSdeIsSet(true);
- msg = result;
- }
- else
- {
- msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION;
- msg = (org.apache.thrift.TBase)new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, e.getMessage());
- }
- try {
- fcall.sendResponse(fb,msg,msgType,seqid);
- return;
- } catch (Exception ex) {
- LOGGER.error("Exception writing to internal frame buffer", ex);
- }
- fb.close();
- }
- };
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public void start(I iface, system_add_column_family_args args, org.apache.thrift.async.AsyncMethodCallback<String> resultHandler) throws TException {
- iface.system_add_column_family(args.cf_def,resultHandler);
- }
- }
-
- public static class system_drop_column_family<I extends AsyncIface> extends org.apache.thrift.AsyncProcessFunction<I, system_drop_column_family_args, String> {
- public system_drop_column_family() {
- super("system_drop_column_family");
- }
-
- public system_drop_column_family_args getEmptyArgsInstance() {
- return new system_drop_column_family_args();
- }
-
- public AsyncMethodCallback<String> getResultHandler(final AsyncFrameBuffer fb, final int seqid) {
- final org.apache.thrift.AsyncProcessFunction fcall = this;
- return new AsyncMethodCallback<String>() {
- public void onComplete(String o) {
- system_drop_column_family_result result = new system_drop_column_family_result();
- result.success = o;
- try {
- fcall.sendResponse(fb,result, org.apache.thrift.protocol.TMessageType.REPLY,seqid);
- return;
- } catch (Exception e) {
- LOGGER.error("Exception writing to internal frame buffer", e);
- }
- fb.close();
- }
- public void onError(Exception e) {
- byte msgType = org.apache.thrift.protocol.TMessageType.REPLY;
- org.apache.thrift.TBase msg;
- system_drop_column_family_result result = new system_drop_column_family_result();
- if (e instanceof InvalidRequestException) {
- result.ire = (InvalidRequestException) e;
- result.setIreIsSet(true);
- msg = result;
- }
- else if (e instanceof SchemaDisagreementException) {
- result.sde = (SchemaDisagreementException) e;
- result.setSdeIsSet(true);
- msg = result;
- }
- else
- {
- msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION;
- msg = (org.apache.thrift.TBase)new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, e.getMessage());
- }
- try {
- fcall.sendResponse(fb,msg,msgType,seqid);
- return;
- } catch (Exception ex) {
- LOGGER.error("Exception writing to internal frame buffer", ex);
- }
- fb.close();
- }
- };
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public void start(I iface, system_drop_column_family_args args, org.apache.thrift.async.AsyncMethodCallback<String> resultHandler) throws TException {
- iface.system_drop_column_family(args.column_family,resultHandler);
- }
- }
-
- public static class system_add_keyspace<I extends AsyncIface> extends org.apache.thrift.AsyncProcessFunction<I, system_add_keyspace_args, String> {
- public system_add_keyspace() {
- super("system_add_keyspace");
- }
-
- public system_add_keyspace_args getEmptyArgsInstance() {
- return new system_add_keyspace_args();
- }
-
- public AsyncMethodCallback<String> getResultHandler(final AsyncFrameBuffer fb, final int seqid) {
- final org.apache.thrift.AsyncProcessFunction fcall = this;
- return new AsyncMethodCallback<String>() {
- public void onComplete(String o) {
- system_add_keyspace_result result = new system_add_keyspace_result();
- result.success = o;
- try {
- fcall.sendResponse(fb,result, org.apache.thrift.protocol.TMessageType.REPLY,seqid);
- return;
- } catch (Exception e) {
- LOGGER.error("Exception writing to internal frame buffer", e);
- }
- fb.close();
- }
- public void onError(Exception e) {
- byte msgType = org.apache.thrift.protocol.TMessageType.REPLY;
- org.apache.thrift.TBase msg;
- system_add_keyspace_result result = new system_add_keyspace_result();
- if (e instanceof InvalidRequestException) {
- result.ire = (InvalidRequestException) e;
- result.setIreIsSet(true);
- msg = result;
- }
- else if (e instanceof SchemaDisagreementException) {
- result.sde = (SchemaDisagreementException) e;
- result.setSdeIsSet(true);
- msg = result;
- }
- else
- {
- msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION;
- msg = (org.apache.thrift.TBase)new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, e.getMessage());
- }
- try {
- fcall.sendResponse(fb,msg,msgType,seqid);
- return;
- } catch (Exception ex) {
- LOGGER.error("Exception writing to internal frame buffer", ex);
- }
- fb.close();
- }
- };
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public void start(I iface, system_add_keyspace_args args, org.apache.thrift.async.AsyncMethodCallback<String> resultHandler) throws TException {
- iface.system_add_keyspace(args.ks_def,resultHandler);
- }
- }
-
- public static class system_drop_keyspace<I extends AsyncIface> extends org.apache.thrift.AsyncProcessFunction<I, system_drop_keyspace_args, String> {
- public system_drop_keyspace() {
- super("system_drop_keyspace");
- }
-
- public system_drop_keyspace_args getEmptyArgsInstance() {
- return new system_drop_keyspace_args();
- }
-
- public AsyncMethodCallback<String> getResultHandler(final AsyncFrameBuffer fb, final int seqid) {
- final org.apache.thrift.AsyncProcessFunction fcall = this;
- return new AsyncMethodCallback<String>() {
- public void onComplete(String o) {
- system_drop_keyspace_result result = new system_drop_keyspace_result();
- result.success = o;
- try {
- fcall.sendResponse(fb,result, org.apache.thrift.protocol.TMessageType.REPLY,seqid);
- return;
- } catch (Exception e) {
- LOGGER.error("Exception writing to internal frame buffer", e);
- }
- fb.close();
- }
- public void onError(Exception e) {
- byte msgType = org.apache.thrift.protocol.TMessageType.REPLY;
- org.apache.thrift.TBase msg;
- system_drop_keyspace_result result = new system_drop_keyspace_result();
- if (e instanceof InvalidRequestException) {
- result.ire = (InvalidRequestException) e;
- result.setIreIsSet(true);
- msg = result;
- }
- else if (e instanceof SchemaDisagreementException) {
- result.sde = (SchemaDisagreementException) e;
- result.setSdeIsSet(true);
- msg = result;
- }
- else
- {
- msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION;
- msg = (org.apache.thrift.TBase)new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, e.getMessage());
- }
- try {
- fcall.sendResponse(fb,msg,msgType,seqid);
- return;
- } catch (Exception ex) {
- LOGGER.error("Exception writing to internal frame buffer", ex);
- }
- fb.close();
- }
- };
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public void start(I iface, system_drop_keyspace_args args, org.apache.thrift.async.AsyncMethodCallback<String> resultHandler) throws TException {
- iface.system_drop_keyspace(args.keyspace,resultHandler);
- }
- }
-
- public static class system_update_keyspace<I extends AsyncIface> extends org.apache.thrift.AsyncProcessFunction<I, system_update_keyspace_args, String> {
- public system_update_keyspace() {
- super("system_update_keyspace");
- }
-
- public system_update_keyspace_args getEmptyArgsInstance() {
- return new system_update_keyspace_args();
- }
-
- public AsyncMethodCallback<String> getResultHandler(final AsyncFrameBuffer fb, final int seqid) {
- final org.apache.thrift.AsyncProcessFunction fcall = this;
- return new AsyncMethodCallback<String>() {
- public void onComplete(String o) {
- system_update_keyspace_result result = new system_update_keyspace_result();
- result.success = o;
- try {
- fcall.sendResponse(fb,result, org.apache.thrift.protocol.TMessageType.REPLY,seqid);
- return;
- } catch (Exception e) {
- LOGGER.error("Exception writing to internal frame buffer", e);
- }
- fb.close();
- }
- public void onError(Exception e) {
- byte msgType = org.apache.thrift.protocol.TMessageType.REPLY;
- org.apache.thrift.TBase msg;
- system_update_keyspace_result result = new system_update_keyspace_result();
- if (e instanceof InvalidRequestException) {
- result.ire = (InvalidRequestException) e;
- result.setIreIsSet(true);
- msg = result;
- }
- else if (e instanceof SchemaDisagreementException) {
- result.sde = (SchemaDisagreementException) e;
- result.setSdeIsSet(true);
- msg = result;
- }
- else
- {
- msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION;
- msg = (org.apache.thrift.TBase)new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, e.getMessage());
- }
- try {
- fcall.sendResponse(fb,msg,msgType,seqid);
- return;
- } catch (Exception ex) {
- LOGGER.error("Exception writing to internal frame buffer", ex);
- }
- fb.close();
- }
- };
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public void start(I iface, system_update_keyspace_args args, org.apache.thrift.async.AsyncMethodCallback<String> resultHandler) throws TException {
- iface.system_update_keyspace(args.ks_def,resultHandler);
- }
- }
-
- public static class system_update_column_family<I extends AsyncIface> extends org.apache.thrift.AsyncProcessFunction<I, system_update_column_family_args, String> {
- public system_update_column_family() {
- super("system_update_column_family");
- }
-
- public system_update_column_family_args getEmptyArgsInstance() {
- return new system_update_column_family_args();
- }
-
- public AsyncMethodCallback<String> getResultHandler(final AsyncFrameBuffer fb, final int seqid) {
- final org.apache.thrift.AsyncProcessFunction fcall = this;
- return new AsyncMethodCallback<String>() {
- public void onComplete(String o) {
- system_update_column_family_result result = new system_update_column_family_result();
- result.success = o;
- try {
- fcall.sendResponse(fb,result, org.apache.thrift.protocol.TMessageType.REPLY,seqid);
- return;
- } catch (Exception e) {
- LOGGER.error("Exception writing to internal frame buffer", e);
- }
- fb.close();
- }
- public void onError(Exception e) {
- byte msgType = org.apache.thrift.protocol.TMessageType.REPLY;
- org.apache.thrift.TBase msg;
- system_update_column_family_result result = new system_update_column_family_result();
- if (e instanceof InvalidRequestException) {
- result.ire = (InvalidRequestException) e;
- result.setIreIsSet(true);
- msg = result;
- }
- else if (e instanceof SchemaDisagreementException) {
- result.sde = (SchemaDisagreementException) e;
- result.setSdeIsSet(true);
- msg = result;
- }
- else
- {
- msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION;
- msg = (org.apache.thrift.TBase)new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, e.getMessage());
- }
- try {
- fcall.sendResponse(fb,msg,msgType,seqid);
- return;
- } catch (Exception ex) {
- LOGGER.error("Exception writing to internal frame buffer", ex);
- }
- fb.close();
- }
- };
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public void start(I iface, system_update_column_family_args args, org.apache.thrift.async.AsyncMethodCallback<String> resultHandler) throws TException {
- iface.system_update_column_family(args.cf_def,resultHandler);
- }
- }
-
- public static class execute_cql_query<I extends AsyncIface> extends org.apache.thrift.AsyncProcessFunction<I, execute_cql_query_args, CqlResult> {
- public execute_cql_query() {
- super("execute_cql_query");
- }
-
- public execute_cql_query_args getEmptyArgsInstance() {
- return new execute_cql_query_args();
- }
-
- public AsyncMethodCallback<CqlResult> getResultHandler(final AsyncFrameBuffer fb, final int seqid) {
- final org.apache.thrift.AsyncProcessFunction fcall = this;
- return new AsyncMethodCallback<CqlResult>() {
- public void onComplete(CqlResult o) {
- execute_cql_query_result result = new execute_cql_query_result();
- result.success = o;
- try {
- fcall.sendResponse(fb,result, org.apache.thrift.protocol.TMessageType.REPLY,seqid);
- return;
- } catch (Exception e) {
- LOGGER.error("Exception writing to internal frame buffer", e);
- }
- fb.close();
- }
- public void onError(Exception e) {
- byte msgType = org.apache.thrift.protocol.TMessageType.REPLY;
- org.apache.thrift.TBase msg;
- execute_cql_query_result result = new execute_cql_query_result();
- if (e instanceof InvalidRequestException) {
- result.ire = (InvalidRequestException) e;
- result.setIreIsSet(true);
- msg = result;
- }
- else if (e instanceof UnavailableException) {
- result.ue = (UnavailableException) e;
- result.setUeIsSet(true);
- msg = result;
- }
- else if (e instanceof TimedOutException) {
- result.te = (TimedOutException) e;
- result.setTeIsSet(true);
- msg = result;
- }
- else if (e instanceof SchemaDisagreementException) {
- result.sde = (SchemaDisagreementException) e;
- result.setSdeIsSet(true);
- msg = result;
- }
- else
- {
- msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION;
- msg = (org.apache.thrift.TBase)new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, e.getMessage());
- }
- try {
- fcall.sendResponse(fb,msg,msgType,seqid);
- return;
- } catch (Exception ex) {
- LOGGER.error("Exception writing to internal frame buffer", ex);
- }
- fb.close();
- }
- };
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public void start(I iface, execute_cql_query_args args, org.apache.thrift.async.AsyncMethodCallback<CqlResult> resultHandler) throws TException {
- iface.execute_cql_query(args.query, args.compression,resultHandler);
- }
- }
-
- public static class execute_cql3_query<I extends AsyncIface> extends org.apache.thrift.AsyncProcessFunction<I, execute_cql3_query_args, CqlResult> {
- public execute_cql3_query() {
- super("execute_cql3_query");
- }
-
- public execute_cql3_query_args getEmptyArgsInstance() {
- return new execute_cql3_query_args();
- }
-
- public AsyncMethodCallback<CqlResult> getResultHandler(final AsyncFrameBuffer fb, final int seqid) {
- final org.apache.thrift.AsyncProcessFunction fcall = this;
- return new AsyncMethodCallback<CqlResult>() {
- public void onComplete(CqlResult o) {
- execute_cql3_query_result result = new execute_cql3_query_result();
- result.success = o;
- try {
- fcall.sendResponse(fb,result, org.apache.thrift.protocol.TMessageType.REPLY,seqid);
- return;
- } catch (Exception e) {
- LOGGER.error("Exception writing to internal frame buffer", e);
- }
- fb.close();
- }
- public void onError(Exception e) {
- byte msgType = org.apache.thrift.protocol.TMessageType.REPLY;
- org.apache.thrift.TBase msg;
- execute_cql3_query_result result = new execute_cql3_query_result();
- if (e instanceof InvalidRequestException) {
- result.ire = (InvalidRequestException) e;
- result.setIreIsSet(true);
- msg = result;
- }
- else if (e instanceof UnavailableException) {
- result.ue = (UnavailableException) e;
- result.setUeIsSet(true);
- msg = result;
- }
- else if (e instanceof TimedOutException) {
- result.te = (TimedOutException) e;
- result.setTeIsSet(true);
- msg = result;
- }
- else if (e instanceof SchemaDisagreementException) {
- result.sde = (SchemaDisagreementException) e;
- result.setSdeIsSet(true);
- msg = result;
- }
- else
- {
- msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION;
- msg = (org.apache.thrift.TBase)new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, e.getMessage());
- }
- try {
- fcall.sendResponse(fb,msg,msgType,seqid);
- return;
- } catch (Exception ex) {
- LOGGER.error("Exception writing to internal frame buffer", ex);
- }
- fb.close();
- }
- };
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public void start(I iface, execute_cql3_query_args args, org.apache.thrift.async.AsyncMethodCallback<CqlResult> resultHandler) throws TException {
- iface.execute_cql3_query(args.query, args.compression, args.consistency,resultHandler);
- }
- }
-
- public static class prepare_cql_query<I extends AsyncIface> extends org.apache.thrift.AsyncProcessFunction<I, prepare_cql_query_args, CqlPreparedResult> {
- public prepare_cql_query() {
- super("prepare_cql_query");
- }
-
- public prepare_cql_query_args getEmptyArgsInstance() {
- return new prepare_cql_query_args();
- }
-
- public AsyncMethodCallback<CqlPreparedResult> getResultHandler(final AsyncFrameBuffer fb, final int seqid) {
- final org.apache.thrift.AsyncProcessFunction fcall = this;
- return new AsyncMethodCallback<CqlPreparedResult>() {
- public void onComplete(CqlPreparedResult o) {
- prepare_cql_query_result result = new prepare_cql_query_result();
- result.success = o;
- try {
- fcall.sendResponse(fb,result, org.apache.thrift.protocol.TMessageType.REPLY,seqid);
- return;
- } catch (Exception e) {
- LOGGER.error("Exception writing to internal frame buffer", e);
- }
- fb.close();
- }
- public void onError(Exception e) {
- byte msgType = org.apache.thrift.protocol.TMessageType.REPLY;
- org.apache.thrift.TBase msg;
- prepare_cql_query_result result = new prepare_cql_query_result();
- if (e instanceof InvalidRequestException) {
- result.ire = (InvalidRequestException) e;
- result.setIreIsSet(true);
- msg = result;
- }
- else
- {
- msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION;
- msg = (org.apache.thrift.TBase)new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, e.getMessage());
- }
- try {
- fcall.sendResponse(fb,msg,msgType,seqid);
- return;
- } catch (Exception ex) {
- LOGGER.error("Exception writing to internal frame buffer", ex);
- }
- fb.close();
- }
- };
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public void start(I iface, prepare_cql_query_args args, org.apache.thrift.async.AsyncMethodCallback<CqlPreparedResult> resultHandler) throws TException {
- iface.prepare_cql_query(args.query, args.compression,resultHandler);
- }
- }
-
- public static class prepare_cql3_query<I extends AsyncIface> extends org.apache.thrift.AsyncProcessFunction<I, prepare_cql3_query_args, CqlPreparedResult> {
- public prepare_cql3_query() {
- super("prepare_cql3_query");
- }
-
- public prepare_cql3_query_args getEmptyArgsInstance() {
- return new prepare_cql3_query_args();
- }
-
- public AsyncMethodCallback<CqlPreparedResult> getResultHandler(final AsyncFrameBuffer fb, final int seqid) {
- final org.apache.thrift.AsyncProcessFunction fcall = this;
- return new AsyncMethodCallback<CqlPreparedResult>() {
- public void onComplete(CqlPreparedResult o) {
- prepare_cql3_query_result result = new prepare_cql3_query_result();
- result.success = o;
- try {
- fcall.sendResponse(fb,result, org.apache.thrift.protocol.TMessageType.REPLY,seqid);
- return;
- } catch (Exception e) {
- LOGGER.error("Exception writing to internal frame buffer", e);
- }
- fb.close();
- }
- public void onError(Exception e) {
- byte msgType = org.apache.thrift.protocol.TMessageType.REPLY;
- org.apache.thrift.TBase msg;
- prepare_cql3_query_result result = new prepare_cql3_query_result();
- if (e instanceof InvalidRequestException) {
- result.ire = (InvalidRequestException) e;
- result.setIreIsSet(true);
- msg = result;
- }
- else
- {
- msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION;
- msg = (org.apache.thrift.TBase)new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, e.getMessage());
- }
- try {
- fcall.sendResponse(fb,msg,msgType,seqid);
- return;
- } catch (Exception ex) {
- LOGGER.error("Exception writing to internal frame buffer", ex);
- }
- fb.close();
- }
- };
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public void start(I iface, prepare_cql3_query_args args, org.apache.thrift.async.AsyncMethodCallback<CqlPreparedResult> resultHandler) throws TException {
- iface.prepare_cql3_query(args.query, args.compression,resultHandler);
- }
- }
-
- public static class execute_prepared_cql_query<I extends AsyncIface> extends org.apache.thrift.AsyncProcessFunction<I, execute_prepared_cql_query_args, CqlResult> {
- public execute_prepared_cql_query() {
- super("execute_prepared_cql_query");
- }
-
- public execute_prepared_cql_query_args getEmptyArgsInstance() {
- return new execute_prepared_cql_query_args();
- }
-
- public AsyncMethodCallback<CqlResult> getResultHandler(final AsyncFrameBuffer fb, final int seqid) {
- final org.apache.thrift.AsyncProcessFunction fcall = this;
- return new AsyncMethodCallback<CqlResult>() {
- public void onComplete(CqlResult o) {
- execute_prepared_cql_query_result result = new execute_prepared_cql_query_result();
- result.success = o;
- try {
- fcall.sendResponse(fb,result, org.apache.thrift.protocol.TMessageType.REPLY,seqid);
- return;
- } catch (Exception e) {
- LOGGER.error("Exception writing to internal frame buffer", e);
- }
- fb.close();
- }
- public void onError(Exception e) {
- byte msgType = org.apache.thrift.protocol.TMessageType.REPLY;
- org.apache.thrift.TBase msg;
- execute_prepared_cql_query_result result = new execute_prepared_cql_query_result();
- if (e instanceof InvalidRequestException) {
- result.ire = (InvalidRequestException) e;
- result.setIreIsSet(true);
- msg = result;
- }
- else if (e instanceof UnavailableException) {
- result.ue = (UnavailableException) e;
- result.setUeIsSet(true);
- msg = result;
- }
- else if (e instanceof TimedOutException) {
- result.te = (TimedOutException) e;
- result.setTeIsSet(true);
- msg = result;
- }
- else if (e instanceof SchemaDisagreementException) {
- result.sde = (SchemaDisagreementException) e;
- result.setSdeIsSet(true);
- msg = result;
- }
- else
- {
- msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION;
- msg = (org.apache.thrift.TBase)new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, e.getMessage());
- }
- try {
- fcall.sendResponse(fb,msg,msgType,seqid);
- return;
- } catch (Exception ex) {
- LOGGER.error("Exception writing to internal frame buffer", ex);
- }
- fb.close();
- }
- };
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public void start(I iface, execute_prepared_cql_query_args args, org.apache.thrift.async.AsyncMethodCallback<CqlResult> resultHandler) throws TException {
- iface.execute_prepared_cql_query(args.itemId, args.values,resultHandler);
- }
- }
-
- public static class execute_prepared_cql3_query<I extends AsyncIface> extends org.apache.thrift.AsyncProcessFunction<I, execute_prepared_cql3_query_args, CqlResult> {
- public execute_prepared_cql3_query() {
- super("execute_prepared_cql3_query");
- }
-
- public execute_prepared_cql3_query_args getEmptyArgsInstance() {
- return new execute_prepared_cql3_query_args();
- }
-
- public AsyncMethodCallback<CqlResult> getResultHandler(final AsyncFrameBuffer fb, final int seqid) {
- final org.apache.thrift.AsyncProcessFunction fcall = this;
- return new AsyncMethodCallback<CqlResult>() {
- public void onComplete(CqlResult o) {
- execute_prepared_cql3_query_result result = new execute_prepared_cql3_query_result();
- result.success = o;
- try {
- fcall.sendResponse(fb,result, org.apache.thrift.protocol.TMessageType.REPLY,seqid);
- return;
- } catch (Exception e) {
- LOGGER.error("Exception writing to internal frame buffer", e);
- }
- fb.close();
- }
- public void onError(Exception e) {
- byte msgType = org.apache.thrift.protocol.TMessageType.REPLY;
- org.apache.thrift.TBase msg;
- execute_prepared_cql3_query_result result = new execute_prepared_cql3_query_result();
- if (e instanceof InvalidRequestException) {
- result.ire = (InvalidRequestException) e;
- result.setIreIsSet(true);
- msg = result;
- }
- else if (e instanceof UnavailableException) {
- result.ue = (UnavailableException) e;
- result.setUeIsSet(true);
- msg = result;
- }
- else if (e instanceof TimedOutException) {
- result.te = (TimedOutException) e;
- result.setTeIsSet(true);
- msg = result;
- }
- else if (e instanceof SchemaDisagreementException) {
- result.sde = (SchemaDisagreementException) e;
- result.setSdeIsSet(true);
- msg = result;
- }
- else
- {
- msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION;
- msg = (org.apache.thrift.TBase)new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, e.getMessage());
- }
- try {
- fcall.sendResponse(fb,msg,msgType,seqid);
- return;
- } catch (Exception ex) {
- LOGGER.error("Exception writing to internal frame buffer", ex);
- }
- fb.close();
- }
- };
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public void start(I iface, execute_prepared_cql3_query_args args, org.apache.thrift.async.AsyncMethodCallback<CqlResult> resultHandler) throws TException {
- iface.execute_prepared_cql3_query(args.itemId, args.values, args.consistency,resultHandler);
- }
- }
-
- public static class set_cql_version<I extends AsyncIface> extends org.apache.thrift.AsyncProcessFunction<I, set_cql_version_args, Void> {
- public set_cql_version() {
- super("set_cql_version");
- }
-
- public set_cql_version_args getEmptyArgsInstance() {
- return new set_cql_version_args();
- }
-
- public AsyncMethodCallback<Void> getResultHandler(final AsyncFrameBuffer fb, final int seqid) {
- final org.apache.thrift.AsyncProcessFunction fcall = this;
- return new AsyncMethodCallback<Void>() {
- public void onComplete(Void o) {
- set_cql_version_result result = new set_cql_version_result();
- try {
- fcall.sendResponse(fb,result, org.apache.thrift.protocol.TMessageType.REPLY,seqid);
- return;
- } catch (Exception e) {
- LOGGER.error("Exception writing to internal frame buffer", e);
- }
- fb.close();
- }
- public void onError(Exception e) {
- byte msgType = org.apache.thrift.protocol.TMessageType.REPLY;
- org.apache.thrift.TBase msg;
- set_cql_version_result result = new set_cql_version_result();
- if (e instanceof InvalidRequestException) {
- result.ire = (InvalidRequestException) e;
- result.setIreIsSet(true);
- msg = result;
- }
- else
- {
- msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION;
- msg = (org.apache.thrift.TBase)new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, e.getMessage());
- }
- try {
- fcall.sendResponse(fb,msg,msgType,seqid);
- return;
- } catch (Exception ex) {
- LOGGER.error("Exception writing to internal frame buffer", ex);
- }
- fb.close();
- }
- };
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public void start(I iface, set_cql_version_args args, org.apache.thrift.async.AsyncMethodCallback<Void> resultHandler) throws TException {
- iface.set_cql_version(args.version,resultHandler);
- }
- }
-
- }
-
- public static class login_args implements org.apache.thrift.TBase<login_args, login_args._Fields>, java.io.Serializable, Cloneable, Comparable<login_args> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("login_args");
-
- private static final org.apache.thrift.protocol.TField AUTH_REQUEST_FIELD_DESC = new org.apache.thrift.protocol.TField("auth_request", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new login_argsStandardSchemeFactory());
- schemes.put(TupleScheme.class, new login_argsTupleSchemeFactory());
- }
-
- public AuthenticationRequest auth_request; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- AUTH_REQUEST((short)1, "auth_request");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // AUTH_REQUEST
- return AUTH_REQUEST;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.AUTH_REQUEST, new org.apache.thrift.meta_data.FieldMetaData("auth_request", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, AuthenticationRequest.class)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(login_args.class, metaDataMap);
- }
-
- public login_args() {
- }
-
- public login_args(
- AuthenticationRequest auth_request)
- {
- this();
- this.auth_request = auth_request;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public login_args(login_args other) {
- if (other.isSetAuth_request()) {
- this.auth_request = new AuthenticationRequest(other.auth_request);
- }
- }
-
- public login_args deepCopy() {
- return new login_args(this);
- }
-
- @Override
- public void clear() {
- this.auth_request = null;
- }
-
- public AuthenticationRequest getAuth_request() {
- return this.auth_request;
- }
-
- public login_args setAuth_request(AuthenticationRequest auth_request) {
- this.auth_request = auth_request;
- return this;
- }
-
- public void unsetAuth_request() {
- this.auth_request = null;
- }
-
- /** Returns true if field auth_request is set (has been assigned a value) and false otherwise */
- public boolean isSetAuth_request() {
- return this.auth_request != null;
- }
-
- public void setAuth_requestIsSet(boolean value) {
- if (!value) {
- this.auth_request = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case AUTH_REQUEST:
- if (value == null) {
- unsetAuth_request();
- } else {
- setAuth_request((AuthenticationRequest)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case AUTH_REQUEST:
- return getAuth_request();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case AUTH_REQUEST:
- return isSetAuth_request();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof login_args)
- return this.equals((login_args)that);
- return false;
- }
-
- public boolean equals(login_args that) {
- if (that == null)
- return false;
-
- boolean this_present_auth_request = true && this.isSetAuth_request();
- boolean that_present_auth_request = true && that.isSetAuth_request();
- if (this_present_auth_request || that_present_auth_request) {
- if (!(this_present_auth_request && that_present_auth_request))
- return false;
- if (!this.auth_request.equals(that.auth_request))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_auth_request = true && (isSetAuth_request());
- builder.append(present_auth_request);
- if (present_auth_request)
- builder.append(auth_request);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(login_args other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetAuth_request()).compareTo(other.isSetAuth_request());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetAuth_request()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.auth_request, other.auth_request);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("login_args(");
- boolean first = true;
-
- sb.append("auth_request:");
- if (this.auth_request == null) {
- sb.append("null");
- } else {
- sb.append(this.auth_request);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- if (auth_request == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'auth_request' was not present! Struct: " + toString());
- }
- // check for sub-struct validity
- if (auth_request != null) {
- auth_request.validate();
- }
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class login_argsStandardSchemeFactory implements SchemeFactory {
- public login_argsStandardScheme getScheme() {
- return new login_argsStandardScheme();
- }
- }
-
- private static class login_argsStandardScheme extends StandardScheme<login_args> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, login_args struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // AUTH_REQUEST
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.auth_request = new AuthenticationRequest();
- struct.auth_request.read(iprot);
- struct.setAuth_requestIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, login_args struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.auth_request != null) {
- oprot.writeFieldBegin(AUTH_REQUEST_FIELD_DESC);
- struct.auth_request.write(oprot);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class login_argsTupleSchemeFactory implements SchemeFactory {
- public login_argsTupleScheme getScheme() {
- return new login_argsTupleScheme();
- }
- }
-
- private static class login_argsTupleScheme extends TupleScheme<login_args> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, login_args struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- struct.auth_request.write(oprot);
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, login_args struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- struct.auth_request = new AuthenticationRequest();
- struct.auth_request.read(iprot);
- struct.setAuth_requestIsSet(true);
- }
- }
-
- }
-
- public static class login_result implements org.apache.thrift.TBase<login_result, login_result._Fields>, java.io.Serializable, Cloneable, Comparable<login_result> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("login_result");
-
- private static final org.apache.thrift.protocol.TField AUTHNX_FIELD_DESC = new org.apache.thrift.protocol.TField("authnx", org.apache.thrift.protocol.TType.STRUCT, (short)1);
- private static final org.apache.thrift.protocol.TField AUTHZX_FIELD_DESC = new org.apache.thrift.protocol.TField("authzx", org.apache.thrift.protocol.TType.STRUCT, (short)2);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new login_resultStandardSchemeFactory());
- schemes.put(TupleScheme.class, new login_resultTupleSchemeFactory());
- }
-
- public AuthenticationException authnx; // required
- public AuthorizationException authzx; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- AUTHNX((short)1, "authnx"),
- AUTHZX((short)2, "authzx");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // AUTHNX
- return AUTHNX;
- case 2: // AUTHZX
- return AUTHZX;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.AUTHNX, new org.apache.thrift.meta_data.FieldMetaData("authnx", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- tmpMap.put(_Fields.AUTHZX, new org.apache.thrift.meta_data.FieldMetaData("authzx", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(login_result.class, metaDataMap);
- }
-
- public login_result() {
- }
-
- public login_result(
- AuthenticationException authnx,
- AuthorizationException authzx)
- {
- this();
- this.authnx = authnx;
- this.authzx = authzx;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public login_result(login_result other) {
- if (other.isSetAuthnx()) {
- this.authnx = new AuthenticationException(other.authnx);
- }
- if (other.isSetAuthzx()) {
- this.authzx = new AuthorizationException(other.authzx);
- }
- }
-
- public login_result deepCopy() {
- return new login_result(this);
- }
-
- @Override
- public void clear() {
- this.authnx = null;
- this.authzx = null;
- }
-
- public AuthenticationException getAuthnx() {
- return this.authnx;
- }
-
- public login_result setAuthnx(AuthenticationException authnx) {
- this.authnx = authnx;
- return this;
- }
-
- public void unsetAuthnx() {
- this.authnx = null;
- }
-
- /** Returns true if field authnx is set (has been assigned a value) and false otherwise */
- public boolean isSetAuthnx() {
- return this.authnx != null;
- }
-
- public void setAuthnxIsSet(boolean value) {
- if (!value) {
- this.authnx = null;
- }
- }
-
- public AuthorizationException getAuthzx() {
- return this.authzx;
- }
-
- public login_result setAuthzx(AuthorizationException authzx) {
- this.authzx = authzx;
- return this;
- }
-
- public void unsetAuthzx() {
- this.authzx = null;
- }
-
- /** Returns true if field authzx is set (has been assigned a value) and false otherwise */
- public boolean isSetAuthzx() {
- return this.authzx != null;
- }
-
- public void setAuthzxIsSet(boolean value) {
- if (!value) {
- this.authzx = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case AUTHNX:
- if (value == null) {
- unsetAuthnx();
- } else {
- setAuthnx((AuthenticationException)value);
- }
- break;
-
- case AUTHZX:
- if (value == null) {
- unsetAuthzx();
- } else {
- setAuthzx((AuthorizationException)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case AUTHNX:
- return getAuthnx();
-
- case AUTHZX:
- return getAuthzx();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case AUTHNX:
- return isSetAuthnx();
- case AUTHZX:
- return isSetAuthzx();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof login_result)
- return this.equals((login_result)that);
- return false;
- }
-
- public boolean equals(login_result that) {
- if (that == null)
- return false;
-
- boolean this_present_authnx = true && this.isSetAuthnx();
- boolean that_present_authnx = true && that.isSetAuthnx();
- if (this_present_authnx || that_present_authnx) {
- if (!(this_present_authnx && that_present_authnx))
- return false;
- if (!this.authnx.equals(that.authnx))
- return false;
- }
-
- boolean this_present_authzx = true && this.isSetAuthzx();
- boolean that_present_authzx = true && that.isSetAuthzx();
- if (this_present_authzx || that_present_authzx) {
- if (!(this_present_authzx && that_present_authzx))
- return false;
- if (!this.authzx.equals(that.authzx))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_authnx = true && (isSetAuthnx());
- builder.append(present_authnx);
- if (present_authnx)
- builder.append(authnx);
-
- boolean present_authzx = true && (isSetAuthzx());
- builder.append(present_authzx);
- if (present_authzx)
- builder.append(authzx);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(login_result other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetAuthnx()).compareTo(other.isSetAuthnx());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetAuthnx()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.authnx, other.authnx);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetAuthzx()).compareTo(other.isSetAuthzx());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetAuthzx()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.authzx, other.authzx);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("login_result(");
- boolean first = true;
-
- sb.append("authnx:");
- if (this.authnx == null) {
- sb.append("null");
- } else {
- sb.append(this.authnx);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("authzx:");
- if (this.authzx == null) {
- sb.append("null");
- } else {
- sb.append(this.authzx);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class login_resultStandardSchemeFactory implements SchemeFactory {
- public login_resultStandardScheme getScheme() {
- return new login_resultStandardScheme();
- }
- }
-
- private static class login_resultStandardScheme extends StandardScheme<login_result> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, login_result struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // AUTHNX
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.authnx = new AuthenticationException();
- struct.authnx.read(iprot);
- struct.setAuthnxIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 2: // AUTHZX
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.authzx = new AuthorizationException();
- struct.authzx.read(iprot);
- struct.setAuthzxIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, login_result struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.authnx != null) {
- oprot.writeFieldBegin(AUTHNX_FIELD_DESC);
- struct.authnx.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.authzx != null) {
- oprot.writeFieldBegin(AUTHZX_FIELD_DESC);
- struct.authzx.write(oprot);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class login_resultTupleSchemeFactory implements SchemeFactory {
- public login_resultTupleScheme getScheme() {
- return new login_resultTupleScheme();
- }
- }
-
- private static class login_resultTupleScheme extends TupleScheme<login_result> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, login_result struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- BitSet optionals = new BitSet();
- if (struct.isSetAuthnx()) {
- optionals.set(0);
- }
- if (struct.isSetAuthzx()) {
- optionals.set(1);
- }
- oprot.writeBitSet(optionals, 2);
- if (struct.isSetAuthnx()) {
- struct.authnx.write(oprot);
- }
- if (struct.isSetAuthzx()) {
- struct.authzx.write(oprot);
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, login_result struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- BitSet incoming = iprot.readBitSet(2);
- if (incoming.get(0)) {
- struct.authnx = new AuthenticationException();
- struct.authnx.read(iprot);
- struct.setAuthnxIsSet(true);
- }
- if (incoming.get(1)) {
- struct.authzx = new AuthorizationException();
- struct.authzx.read(iprot);
- struct.setAuthzxIsSet(true);
- }
- }
- }
-
- }
-
- public static class set_keyspace_args implements org.apache.thrift.TBase<set_keyspace_args, set_keyspace_args._Fields>, java.io.Serializable, Cloneable, Comparable<set_keyspace_args> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("set_keyspace_args");
-
- private static final org.apache.thrift.protocol.TField KEYSPACE_FIELD_DESC = new org.apache.thrift.protocol.TField("keyspace", org.apache.thrift.protocol.TType.STRING, (short)1);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new set_keyspace_argsStandardSchemeFactory());
- schemes.put(TupleScheme.class, new set_keyspace_argsTupleSchemeFactory());
- }
-
- public String keyspace; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- KEYSPACE((short)1, "keyspace");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // KEYSPACE
- return KEYSPACE;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.KEYSPACE, new org.apache.thrift.meta_data.FieldMetaData("keyspace", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(set_keyspace_args.class, metaDataMap);
- }
-
- public set_keyspace_args() {
- }
-
- public set_keyspace_args(
- String keyspace)
- {
- this();
- this.keyspace = keyspace;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public set_keyspace_args(set_keyspace_args other) {
- if (other.isSetKeyspace()) {
- this.keyspace = other.keyspace;
- }
- }
-
- public set_keyspace_args deepCopy() {
- return new set_keyspace_args(this);
- }
-
- @Override
- public void clear() {
- this.keyspace = null;
- }
-
- public String getKeyspace() {
- return this.keyspace;
- }
-
- public set_keyspace_args setKeyspace(String keyspace) {
- this.keyspace = keyspace;
- return this;
- }
-
- public void unsetKeyspace() {
- this.keyspace = null;
- }
-
- /** Returns true if field keyspace is set (has been assigned a value) and false otherwise */
- public boolean isSetKeyspace() {
- return this.keyspace != null;
- }
-
- public void setKeyspaceIsSet(boolean value) {
- if (!value) {
- this.keyspace = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case KEYSPACE:
- if (value == null) {
- unsetKeyspace();
- } else {
- setKeyspace((String)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case KEYSPACE:
- return getKeyspace();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case KEYSPACE:
- return isSetKeyspace();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof set_keyspace_args)
- return this.equals((set_keyspace_args)that);
- return false;
- }
-
- public boolean equals(set_keyspace_args that) {
- if (that == null)
- return false;
-
- boolean this_present_keyspace = true && this.isSetKeyspace();
- boolean that_present_keyspace = true && that.isSetKeyspace();
- if (this_present_keyspace || that_present_keyspace) {
- if (!(this_present_keyspace && that_present_keyspace))
- return false;
- if (!this.keyspace.equals(that.keyspace))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_keyspace = true && (isSetKeyspace());
- builder.append(present_keyspace);
- if (present_keyspace)
- builder.append(keyspace);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(set_keyspace_args other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetKeyspace()).compareTo(other.isSetKeyspace());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetKeyspace()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.keyspace, other.keyspace);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("set_keyspace_args(");
- boolean first = true;
-
- sb.append("keyspace:");
- if (this.keyspace == null) {
- sb.append("null");
- } else {
- sb.append(this.keyspace);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- if (keyspace == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'keyspace' was not present! Struct: " + toString());
- }
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class set_keyspace_argsStandardSchemeFactory implements SchemeFactory {
- public set_keyspace_argsStandardScheme getScheme() {
- return new set_keyspace_argsStandardScheme();
- }
- }
-
- private static class set_keyspace_argsStandardScheme extends StandardScheme<set_keyspace_args> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, set_keyspace_args struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // KEYSPACE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.keyspace = iprot.readString();
- struct.setKeyspaceIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, set_keyspace_args struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.keyspace != null) {
- oprot.writeFieldBegin(KEYSPACE_FIELD_DESC);
- oprot.writeString(struct.keyspace);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class set_keyspace_argsTupleSchemeFactory implements SchemeFactory {
- public set_keyspace_argsTupleScheme getScheme() {
- return new set_keyspace_argsTupleScheme();
- }
- }
-
- private static class set_keyspace_argsTupleScheme extends TupleScheme<set_keyspace_args> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, set_keyspace_args struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- oprot.writeString(struct.keyspace);
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, set_keyspace_args struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- struct.keyspace = iprot.readString();
- struct.setKeyspaceIsSet(true);
- }
- }
-
- }
-
- public static class set_keyspace_result implements org.apache.thrift.TBase<set_keyspace_result, set_keyspace_result._Fields>, java.io.Serializable, Cloneable, Comparable<set_keyspace_result> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("set_keyspace_result");
-
- private static final org.apache.thrift.protocol.TField IRE_FIELD_DESC = new org.apache.thrift.protocol.TField("ire", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new set_keyspace_resultStandardSchemeFactory());
- schemes.put(TupleScheme.class, new set_keyspace_resultTupleSchemeFactory());
- }
-
- public InvalidRequestException ire; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- IRE((short)1, "ire");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // IRE
- return IRE;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.IRE, new org.apache.thrift.meta_data.FieldMetaData("ire", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(set_keyspace_result.class, metaDataMap);
- }
-
- public set_keyspace_result() {
- }
-
- public set_keyspace_result(
- InvalidRequestException ire)
- {
- this();
- this.ire = ire;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public set_keyspace_result(set_keyspace_result other) {
- if (other.isSetIre()) {
- this.ire = new InvalidRequestException(other.ire);
- }
- }
-
- public set_keyspace_result deepCopy() {
- return new set_keyspace_result(this);
- }
-
- @Override
- public void clear() {
- this.ire = null;
- }
-
- public InvalidRequestException getIre() {
- return this.ire;
- }
-
- public set_keyspace_result setIre(InvalidRequestException ire) {
- this.ire = ire;
- return this;
- }
-
- public void unsetIre() {
- this.ire = null;
- }
-
- /** Returns true if field ire is set (has been assigned a value) and false otherwise */
- public boolean isSetIre() {
- return this.ire != null;
- }
-
- public void setIreIsSet(boolean value) {
- if (!value) {
- this.ire = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case IRE:
- if (value == null) {
- unsetIre();
- } else {
- setIre((InvalidRequestException)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case IRE:
- return getIre();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case IRE:
- return isSetIre();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof set_keyspace_result)
- return this.equals((set_keyspace_result)that);
- return false;
- }
-
- public boolean equals(set_keyspace_result that) {
- if (that == null)
- return false;
-
- boolean this_present_ire = true && this.isSetIre();
- boolean that_present_ire = true && that.isSetIre();
- if (this_present_ire || that_present_ire) {
- if (!(this_present_ire && that_present_ire))
- return false;
- if (!this.ire.equals(that.ire))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_ire = true && (isSetIre());
- builder.append(present_ire);
- if (present_ire)
- builder.append(ire);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(set_keyspace_result other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetIre()).compareTo(other.isSetIre());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetIre()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.ire, other.ire);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("set_keyspace_result(");
- boolean first = true;
-
- sb.append("ire:");
- if (this.ire == null) {
- sb.append("null");
- } else {
- sb.append(this.ire);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class set_keyspace_resultStandardSchemeFactory implements SchemeFactory {
- public set_keyspace_resultStandardScheme getScheme() {
- return new set_keyspace_resultStandardScheme();
- }
- }
-
- private static class set_keyspace_resultStandardScheme extends StandardScheme<set_keyspace_result> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, set_keyspace_result struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // IRE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.ire = new InvalidRequestException();
- struct.ire.read(iprot);
- struct.setIreIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, set_keyspace_result struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.ire != null) {
- oprot.writeFieldBegin(IRE_FIELD_DESC);
- struct.ire.write(oprot);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class set_keyspace_resultTupleSchemeFactory implements SchemeFactory {
- public set_keyspace_resultTupleScheme getScheme() {
- return new set_keyspace_resultTupleScheme();
- }
- }
-
- private static class set_keyspace_resultTupleScheme extends TupleScheme<set_keyspace_result> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, set_keyspace_result struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- BitSet optionals = new BitSet();
- if (struct.isSetIre()) {
- optionals.set(0);
- }
- oprot.writeBitSet(optionals, 1);
- if (struct.isSetIre()) {
- struct.ire.write(oprot);
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, set_keyspace_result struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- BitSet incoming = iprot.readBitSet(1);
- if (incoming.get(0)) {
- struct.ire = new InvalidRequestException();
- struct.ire.read(iprot);
- struct.setIreIsSet(true);
- }
- }
- }
-
- }
-
- public static class get_args implements org.apache.thrift.TBase<get_args, get_args._Fields>, java.io.Serializable, Cloneable, Comparable<get_args> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("get_args");
-
- private static final org.apache.thrift.protocol.TField KEY_FIELD_DESC = new org.apache.thrift.protocol.TField("key", org.apache.thrift.protocol.TType.STRING, (short)1);
- private static final org.apache.thrift.protocol.TField COLUMN_PATH_FIELD_DESC = new org.apache.thrift.protocol.TField("column_path", org.apache.thrift.protocol.TType.STRUCT, (short)2);
- private static final org.apache.thrift.protocol.TField CONSISTENCY_LEVEL_FIELD_DESC = new org.apache.thrift.protocol.TField("consistency_level", org.apache.thrift.protocol.TType.I32, (short)3);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new get_argsStandardSchemeFactory());
- schemes.put(TupleScheme.class, new get_argsTupleSchemeFactory());
- }
-
- public ByteBuffer key; // required
- public ColumnPath column_path; // required
- /**
- *
- * @see ConsistencyLevel
- */
- public ConsistencyLevel consistency_level; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- KEY((short)1, "key"),
- COLUMN_PATH((short)2, "column_path"),
- /**
- *
- * @see ConsistencyLevel
- */
- CONSISTENCY_LEVEL((short)3, "consistency_level");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // KEY
- return KEY;
- case 2: // COLUMN_PATH
- return COLUMN_PATH;
- case 3: // CONSISTENCY_LEVEL
- return CONSISTENCY_LEVEL;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.KEY, new org.apache.thrift.meta_data.FieldMetaData("key", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true)));
- tmpMap.put(_Fields.COLUMN_PATH, new org.apache.thrift.meta_data.FieldMetaData("column_path", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, ColumnPath.class)));
- tmpMap.put(_Fields.CONSISTENCY_LEVEL, new org.apache.thrift.meta_data.FieldMetaData("consistency_level", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.EnumMetaData(org.apache.thrift.protocol.TType.ENUM, ConsistencyLevel.class)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(get_args.class, metaDataMap);
- }
-
- public get_args() {
- this.consistency_level = org.apache.cassandra.thrift.ConsistencyLevel.ONE;
-
- }
-
- public get_args(
- ByteBuffer key,
- ColumnPath column_path,
- ConsistencyLevel consistency_level)
- {
- this();
- this.key = key;
- this.column_path = column_path;
- this.consistency_level = consistency_level;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public get_args(get_args other) {
- if (other.isSetKey()) {
- this.key = org.apache.thrift.TBaseHelper.copyBinary(other.key);
-;
- }
- if (other.isSetColumn_path()) {
- this.column_path = new ColumnPath(other.column_path);
- }
- if (other.isSetConsistency_level()) {
- this.consistency_level = other.consistency_level;
- }
- }
-
- public get_args deepCopy() {
- return new get_args(this);
- }
-
- @Override
- public void clear() {
- this.key = null;
- this.column_path = null;
- this.consistency_level = org.apache.cassandra.thrift.ConsistencyLevel.ONE;
-
- }
-
- public byte[] getKey() {
- setKey(org.apache.thrift.TBaseHelper.rightSize(key));
- return key == null ? null : key.array();
- }
-
- public ByteBuffer bufferForKey() {
- return key;
- }
-
- public get_args setKey(byte[] key) {
- setKey(key == null ? (ByteBuffer)null : ByteBuffer.wrap(key));
- return this;
- }
-
- public get_args setKey(ByteBuffer key) {
- this.key = key;
- return this;
- }
-
- public void unsetKey() {
- this.key = null;
- }
-
- /** Returns true if field key is set (has been assigned a value) and false otherwise */
- public boolean isSetKey() {
- return this.key != null;
- }
-
- public void setKeyIsSet(boolean value) {
- if (!value) {
- this.key = null;
- }
- }
-
- public ColumnPath getColumn_path() {
- return this.column_path;
- }
-
- public get_args setColumn_path(ColumnPath column_path) {
- this.column_path = column_path;
- return this;
- }
-
- public void unsetColumn_path() {
- this.column_path = null;
- }
-
- /** Returns true if field column_path is set (has been assigned a value) and false otherwise */
- public boolean isSetColumn_path() {
- return this.column_path != null;
- }
-
- public void setColumn_pathIsSet(boolean value) {
- if (!value) {
- this.column_path = null;
- }
- }
-
- /**
- *
- * @see ConsistencyLevel
- */
- public ConsistencyLevel getConsistency_level() {
- return this.consistency_level;
- }
-
- /**
- *
- * @see ConsistencyLevel
- */
- public get_args setConsistency_level(ConsistencyLevel consistency_level) {
- this.consistency_level = consistency_level;
- return this;
- }
-
- public void unsetConsistency_level() {
- this.consistency_level = null;
- }
-
- /** Returns true if field consistency_level is set (has been assigned a value) and false otherwise */
- public boolean isSetConsistency_level() {
- return this.consistency_level != null;
- }
-
- public void setConsistency_levelIsSet(boolean value) {
- if (!value) {
- this.consistency_level = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case KEY:
- if (value == null) {
- unsetKey();
- } else {
- setKey((ByteBuffer)value);
- }
- break;
-
- case COLUMN_PATH:
- if (value == null) {
- unsetColumn_path();
- } else {
- setColumn_path((ColumnPath)value);
- }
- break;
-
- case CONSISTENCY_LEVEL:
- if (value == null) {
- unsetConsistency_level();
- } else {
- setConsistency_level((ConsistencyLevel)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case KEY:
- return getKey();
-
- case COLUMN_PATH:
- return getColumn_path();
-
- case CONSISTENCY_LEVEL:
- return getConsistency_level();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case KEY:
- return isSetKey();
- case COLUMN_PATH:
- return isSetColumn_path();
- case CONSISTENCY_LEVEL:
- return isSetConsistency_level();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof get_args)
- return this.equals((get_args)that);
- return false;
- }
-
- public boolean equals(get_args that) {
- if (that == null)
- return false;
-
- boolean this_present_key = true && this.isSetKey();
- boolean that_present_key = true && that.isSetKey();
- if (this_present_key || that_present_key) {
- if (!(this_present_key && that_present_key))
- return false;
- if (!this.key.equals(that.key))
- return false;
- }
-
- boolean this_present_column_path = true && this.isSetColumn_path();
- boolean that_present_column_path = true && that.isSetColumn_path();
- if (this_present_column_path || that_present_column_path) {
- if (!(this_present_column_path && that_present_column_path))
- return false;
- if (!this.column_path.equals(that.column_path))
- return false;
- }
-
- boolean this_present_consistency_level = true && this.isSetConsistency_level();
- boolean that_present_consistency_level = true && that.isSetConsistency_level();
- if (this_present_consistency_level || that_present_consistency_level) {
- if (!(this_present_consistency_level && that_present_consistency_level))
- return false;
- if (!this.consistency_level.equals(that.consistency_level))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_key = true && (isSetKey());
- builder.append(present_key);
- if (present_key)
- builder.append(key);
-
- boolean present_column_path = true && (isSetColumn_path());
- builder.append(present_column_path);
- if (present_column_path)
- builder.append(column_path);
-
- boolean present_consistency_level = true && (isSetConsistency_level());
- builder.append(present_consistency_level);
- if (present_consistency_level)
- builder.append(consistency_level.getValue());
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(get_args other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetKey()).compareTo(other.isSetKey());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetKey()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.key, other.key);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetColumn_path()).compareTo(other.isSetColumn_path());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetColumn_path()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.column_path, other.column_path);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetConsistency_level()).compareTo(other.isSetConsistency_level());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetConsistency_level()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.consistency_level, other.consistency_level);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("get_args(");
- boolean first = true;
-
- sb.append("key:");
- if (this.key == null) {
- sb.append("null");
- } else {
- org.apache.thrift.TBaseHelper.toString(this.key, sb);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("column_path:");
- if (this.column_path == null) {
- sb.append("null");
- } else {
- sb.append(this.column_path);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("consistency_level:");
- if (this.consistency_level == null) {
- sb.append("null");
- } else {
- sb.append(this.consistency_level);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- if (key == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'key' was not present! Struct: " + toString());
- }
- if (column_path == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'column_path' was not present! Struct: " + toString());
- }
- if (consistency_level == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'consistency_level' was not present! Struct: " + toString());
- }
- // check for sub-struct validity
- if (column_path != null) {
- column_path.validate();
- }
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class get_argsStandardSchemeFactory implements SchemeFactory {
- public get_argsStandardScheme getScheme() {
- return new get_argsStandardScheme();
- }
- }
-
- private static class get_argsStandardScheme extends StandardScheme<get_args> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, get_args struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // KEY
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.key = iprot.readBinary();
- struct.setKeyIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 2: // COLUMN_PATH
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.column_path = new ColumnPath();
- struct.column_path.read(iprot);
- struct.setColumn_pathIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 3: // CONSISTENCY_LEVEL
- if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
- struct.consistency_level = ConsistencyLevel.findByValue(iprot.readI32());
- struct.setConsistency_levelIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, get_args struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.key != null) {
- oprot.writeFieldBegin(KEY_FIELD_DESC);
- oprot.writeBinary(struct.key);
- oprot.writeFieldEnd();
- }
- if (struct.column_path != null) {
- oprot.writeFieldBegin(COLUMN_PATH_FIELD_DESC);
- struct.column_path.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.consistency_level != null) {
- oprot.writeFieldBegin(CONSISTENCY_LEVEL_FIELD_DESC);
- oprot.writeI32(struct.consistency_level.getValue());
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class get_argsTupleSchemeFactory implements SchemeFactory {
- public get_argsTupleScheme getScheme() {
- return new get_argsTupleScheme();
- }
- }
-
- private static class get_argsTupleScheme extends TupleScheme<get_args> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, get_args struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- oprot.writeBinary(struct.key);
- struct.column_path.write(oprot);
- oprot.writeI32(struct.consistency_level.getValue());
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, get_args struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- struct.key = iprot.readBinary();
- struct.setKeyIsSet(true);
- struct.column_path = new ColumnPath();
- struct.column_path.read(iprot);
- struct.setColumn_pathIsSet(true);
- struct.consistency_level = ConsistencyLevel.findByValue(iprot.readI32());
- struct.setConsistency_levelIsSet(true);
- }
- }
-
- }
-
- public static class get_result implements org.apache.thrift.TBase<get_result, get_result._Fields>, java.io.Serializable, Cloneable, Comparable<get_result> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("get_result");
-
- private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRUCT, (short)0);
- private static final org.apache.thrift.protocol.TField IRE_FIELD_DESC = new org.apache.thrift.protocol.TField("ire", org.apache.thrift.protocol.TType.STRUCT, (short)1);
- private static final org.apache.thrift.protocol.TField NFE_FIELD_DESC = new org.apache.thrift.protocol.TField("nfe", org.apache.thrift.protocol.TType.STRUCT, (short)2);
- private static final org.apache.thrift.protocol.TField UE_FIELD_DESC = new org.apache.thrift.protocol.TField("ue", org.apache.thrift.protocol.TType.STRUCT, (short)3);
- private static final org.apache.thrift.protocol.TField TE_FIELD_DESC = new org.apache.thrift.protocol.TField("te", org.apache.thrift.protocol.TType.STRUCT, (short)4);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new get_resultStandardSchemeFactory());
- schemes.put(TupleScheme.class, new get_resultTupleSchemeFactory());
- }
-
- public ColumnOrSuperColumn success; // required
- public InvalidRequestException ire; // required
- public NotFoundException nfe; // required
- public UnavailableException ue; // required
- public TimedOutException te; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- SUCCESS((short)0, "success"),
- IRE((short)1, "ire"),
- NFE((short)2, "nfe"),
- UE((short)3, "ue"),
- TE((short)4, "te");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 0: // SUCCESS
- return SUCCESS;
- case 1: // IRE
- return IRE;
- case 2: // NFE
- return NFE;
- case 3: // UE
- return UE;
- case 4: // TE
- return TE;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, ColumnOrSuperColumn.class)));
- tmpMap.put(_Fields.IRE, new org.apache.thrift.meta_data.FieldMetaData("ire", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- tmpMap.put(_Fields.NFE, new org.apache.thrift.meta_data.FieldMetaData("nfe", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- tmpMap.put(_Fields.UE, new org.apache.thrift.meta_data.FieldMetaData("ue", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- tmpMap.put(_Fields.TE, new org.apache.thrift.meta_data.FieldMetaData("te", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(get_result.class, metaDataMap);
- }
-
- public get_result() {
- }
-
- public get_result(
- ColumnOrSuperColumn success,
- InvalidRequestException ire,
- NotFoundException nfe,
- UnavailableException ue,
- TimedOutException te)
- {
- this();
- this.success = success;
- this.ire = ire;
- this.nfe = nfe;
- this.ue = ue;
- this.te = te;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public get_result(get_result other) {
- if (other.isSetSuccess()) {
- this.success = new ColumnOrSuperColumn(other.success);
- }
- if (other.isSetIre()) {
- this.ire = new InvalidRequestException(other.ire);
- }
- if (other.isSetNfe()) {
- this.nfe = new NotFoundException(other.nfe);
- }
- if (other.isSetUe()) {
- this.ue = new UnavailableException(other.ue);
- }
- if (other.isSetTe()) {
- this.te = new TimedOutException(other.te);
- }
- }
-
- public get_result deepCopy() {
- return new get_result(this);
- }
-
- @Override
- public void clear() {
- this.success = null;
- this.ire = null;
- this.nfe = null;
- this.ue = null;
- this.te = null;
- }
-
- public ColumnOrSuperColumn getSuccess() {
- return this.success;
- }
-
- public get_result setSuccess(ColumnOrSuperColumn success) {
- this.success = success;
- return this;
- }
-
- public void unsetSuccess() {
- this.success = null;
- }
-
- /** Returns true if field success is set (has been assigned a value) and false otherwise */
- public boolean isSetSuccess() {
- return this.success != null;
- }
-
- public void setSuccessIsSet(boolean value) {
- if (!value) {
- this.success = null;
- }
- }
-
- public InvalidRequestException getIre() {
- return this.ire;
- }
-
- public get_result setIre(InvalidRequestException ire) {
- this.ire = ire;
- return this;
- }
-
- public void unsetIre() {
- this.ire = null;
- }
-
- /** Returns true if field ire is set (has been assigned a value) and false otherwise */
- public boolean isSetIre() {
- return this.ire != null;
- }
-
- public void setIreIsSet(boolean value) {
- if (!value) {
- this.ire = null;
- }
- }
-
- public NotFoundException getNfe() {
- return this.nfe;
- }
-
- public get_result setNfe(NotFoundException nfe) {
- this.nfe = nfe;
- return this;
- }
-
- public void unsetNfe() {
- this.nfe = null;
- }
-
- /** Returns true if field nfe is set (has been assigned a value) and false otherwise */
- public boolean isSetNfe() {
- return this.nfe != null;
- }
-
- public void setNfeIsSet(boolean value) {
- if (!value) {
- this.nfe = null;
- }
- }
-
- public UnavailableException getUe() {
- return this.ue;
- }
-
- public get_result setUe(UnavailableException ue) {
- this.ue = ue;
- return this;
- }
-
- public void unsetUe() {
- this.ue = null;
- }
-
- /** Returns true if field ue is set (has been assigned a value) and false otherwise */
- public boolean isSetUe() {
- return this.ue != null;
- }
-
- public void setUeIsSet(boolean value) {
- if (!value) {
- this.ue = null;
- }
- }
-
- public TimedOutException getTe() {
- return this.te;
- }
-
- public get_result setTe(TimedOutException te) {
- this.te = te;
- return this;
- }
-
- public void unsetTe() {
- this.te = null;
- }
-
- /** Returns true if field te is set (has been assigned a value) and false otherwise */
- public boolean isSetTe() {
- return this.te != null;
- }
-
- public void setTeIsSet(boolean value) {
- if (!value) {
- this.te = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case SUCCESS:
- if (value == null) {
- unsetSuccess();
- } else {
- setSuccess((ColumnOrSuperColumn)value);
- }
- break;
-
- case IRE:
- if (value == null) {
- unsetIre();
- } else {
- setIre((InvalidRequestException)value);
- }
- break;
-
- case NFE:
- if (value == null) {
- unsetNfe();
- } else {
- setNfe((NotFoundException)value);
- }
- break;
-
- case UE:
- if (value == null) {
- unsetUe();
- } else {
- setUe((UnavailableException)value);
- }
- break;
-
- case TE:
- if (value == null) {
- unsetTe();
- } else {
- setTe((TimedOutException)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case SUCCESS:
- return getSuccess();
-
- case IRE:
- return getIre();
-
- case NFE:
- return getNfe();
-
- case UE:
- return getUe();
-
- case TE:
- return getTe();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case SUCCESS:
- return isSetSuccess();
- case IRE:
- return isSetIre();
- case NFE:
- return isSetNfe();
- case UE:
- return isSetUe();
- case TE:
- return isSetTe();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof get_result)
- return this.equals((get_result)that);
- return false;
- }
-
- public boolean equals(get_result that) {
- if (that == null)
- return false;
-
- boolean this_present_success = true && this.isSetSuccess();
- boolean that_present_success = true && that.isSetSuccess();
- if (this_present_success || that_present_success) {
- if (!(this_present_success && that_present_success))
- return false;
- if (!this.success.equals(that.success))
- return false;
- }
-
- boolean this_present_ire = true && this.isSetIre();
- boolean that_present_ire = true && that.isSetIre();
- if (this_present_ire || that_present_ire) {
- if (!(this_present_ire && that_present_ire))
- return false;
- if (!this.ire.equals(that.ire))
- return false;
- }
-
- boolean this_present_nfe = true && this.isSetNfe();
- boolean that_present_nfe = true && that.isSetNfe();
- if (this_present_nfe || that_present_nfe) {
- if (!(this_present_nfe && that_present_nfe))
- return false;
- if (!this.nfe.equals(that.nfe))
- return false;
- }
-
- boolean this_present_ue = true && this.isSetUe();
- boolean that_present_ue = true && that.isSetUe();
- if (this_present_ue || that_present_ue) {
- if (!(this_present_ue && that_present_ue))
- return false;
- if (!this.ue.equals(that.ue))
- return false;
- }
-
- boolean this_present_te = true && this.isSetTe();
- boolean that_present_te = true && that.isSetTe();
- if (this_present_te || that_present_te) {
- if (!(this_present_te && that_present_te))
- return false;
- if (!this.te.equals(that.te))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_success = true && (isSetSuccess());
- builder.append(present_success);
- if (present_success)
- builder.append(success);
-
- boolean present_ire = true && (isSetIre());
- builder.append(present_ire);
- if (present_ire)
- builder.append(ire);
-
- boolean present_nfe = true && (isSetNfe());
- builder.append(present_nfe);
- if (present_nfe)
- builder.append(nfe);
-
- boolean present_ue = true && (isSetUe());
- builder.append(present_ue);
- if (present_ue)
- builder.append(ue);
-
- boolean present_te = true && (isSetTe());
- builder.append(present_te);
- if (present_te)
- builder.append(te);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(get_result other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(other.isSetSuccess());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetSuccess()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, other.success);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetIre()).compareTo(other.isSetIre());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetIre()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.ire, other.ire);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetNfe()).compareTo(other.isSetNfe());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetNfe()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.nfe, other.nfe);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetUe()).compareTo(other.isSetUe());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetUe()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.ue, other.ue);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetTe()).compareTo(other.isSetTe());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetTe()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.te, other.te);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("get_result(");
- boolean first = true;
-
- sb.append("success:");
- if (this.success == null) {
- sb.append("null");
- } else {
- sb.append(this.success);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("ire:");
- if (this.ire == null) {
- sb.append("null");
- } else {
- sb.append(this.ire);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("nfe:");
- if (this.nfe == null) {
- sb.append("null");
- } else {
- sb.append(this.nfe);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("ue:");
- if (this.ue == null) {
- sb.append("null");
- } else {
- sb.append(this.ue);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("te:");
- if (this.te == null) {
- sb.append("null");
- } else {
- sb.append(this.te);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // check for sub-struct validity
- if (success != null) {
- success.validate();
- }
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class get_resultStandardSchemeFactory implements SchemeFactory {
- public get_resultStandardScheme getScheme() {
- return new get_resultStandardScheme();
- }
- }
-
- private static class get_resultStandardScheme extends StandardScheme<get_result> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, get_result struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 0: // SUCCESS
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.success = new ColumnOrSuperColumn();
- struct.success.read(iprot);
- struct.setSuccessIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 1: // IRE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.ire = new InvalidRequestException();
- struct.ire.read(iprot);
- struct.setIreIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 2: // NFE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.nfe = new NotFoundException();
- struct.nfe.read(iprot);
- struct.setNfeIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 3: // UE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.ue = new UnavailableException();
- struct.ue.read(iprot);
- struct.setUeIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 4: // TE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.te = new TimedOutException();
- struct.te.read(iprot);
- struct.setTeIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, get_result struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.success != null) {
- oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
- struct.success.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.ire != null) {
- oprot.writeFieldBegin(IRE_FIELD_DESC);
- struct.ire.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.nfe != null) {
- oprot.writeFieldBegin(NFE_FIELD_DESC);
- struct.nfe.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.ue != null) {
- oprot.writeFieldBegin(UE_FIELD_DESC);
- struct.ue.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.te != null) {
- oprot.writeFieldBegin(TE_FIELD_DESC);
- struct.te.write(oprot);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class get_resultTupleSchemeFactory implements SchemeFactory {
- public get_resultTupleScheme getScheme() {
- return new get_resultTupleScheme();
- }
- }
-
- private static class get_resultTupleScheme extends TupleScheme<get_result> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, get_result struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- BitSet optionals = new BitSet();
- if (struct.isSetSuccess()) {
- optionals.set(0);
- }
- if (struct.isSetIre()) {
- optionals.set(1);
- }
- if (struct.isSetNfe()) {
- optionals.set(2);
- }
- if (struct.isSetUe()) {
- optionals.set(3);
- }
- if (struct.isSetTe()) {
- optionals.set(4);
- }
- oprot.writeBitSet(optionals, 5);
- if (struct.isSetSuccess()) {
- struct.success.write(oprot);
- }
- if (struct.isSetIre()) {
- struct.ire.write(oprot);
- }
- if (struct.isSetNfe()) {
- struct.nfe.write(oprot);
- }
- if (struct.isSetUe()) {
- struct.ue.write(oprot);
- }
- if (struct.isSetTe()) {
- struct.te.write(oprot);
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, get_result struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- BitSet incoming = iprot.readBitSet(5);
- if (incoming.get(0)) {
- struct.success = new ColumnOrSuperColumn();
- struct.success.read(iprot);
- struct.setSuccessIsSet(true);
- }
- if (incoming.get(1)) {
- struct.ire = new InvalidRequestException();
- struct.ire.read(iprot);
- struct.setIreIsSet(true);
- }
- if (incoming.get(2)) {
- struct.nfe = new NotFoundException();
- struct.nfe.read(iprot);
- struct.setNfeIsSet(true);
- }
- if (incoming.get(3)) {
- struct.ue = new UnavailableException();
- struct.ue.read(iprot);
- struct.setUeIsSet(true);
- }
- if (incoming.get(4)) {
- struct.te = new TimedOutException();
- struct.te.read(iprot);
- struct.setTeIsSet(true);
- }
- }
- }
-
- }
-
- public static class get_slice_args implements org.apache.thrift.TBase<get_slice_args, get_slice_args._Fields>, java.io.Serializable, Cloneable, Comparable<get_slice_args> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("get_slice_args");
-
- private static final org.apache.thrift.protocol.TField KEY_FIELD_DESC = new org.apache.thrift.protocol.TField("key", org.apache.thrift.protocol.TType.STRING, (short)1);
- private static final org.apache.thrift.protocol.TField COLUMN_PARENT_FIELD_DESC = new org.apache.thrift.protocol.TField("column_parent", org.apache.thrift.protocol.TType.STRUCT, (short)2);
- private static final org.apache.thrift.protocol.TField PREDICATE_FIELD_DESC = new org.apache.thrift.protocol.TField("predicate", org.apache.thrift.protocol.TType.STRUCT, (short)3);
- private static final org.apache.thrift.protocol.TField CONSISTENCY_LEVEL_FIELD_DESC = new org.apache.thrift.protocol.TField("consistency_level", org.apache.thrift.protocol.TType.I32, (short)4);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new get_slice_argsStandardSchemeFactory());
- schemes.put(TupleScheme.class, new get_slice_argsTupleSchemeFactory());
- }
-
- public ByteBuffer key; // required
- public ColumnParent column_parent; // required
- public SlicePredicate predicate; // required
- /**
- *
- * @see ConsistencyLevel
- */
- public ConsistencyLevel consistency_level; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- KEY((short)1, "key"),
- COLUMN_PARENT((short)2, "column_parent"),
- PREDICATE((short)3, "predicate"),
- /**
- *
- * @see ConsistencyLevel
- */
- CONSISTENCY_LEVEL((short)4, "consistency_level");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // KEY
- return KEY;
- case 2: // COLUMN_PARENT
- return COLUMN_PARENT;
- case 3: // PREDICATE
- return PREDICATE;
- case 4: // CONSISTENCY_LEVEL
- return CONSISTENCY_LEVEL;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.KEY, new org.apache.thrift.meta_data.FieldMetaData("key", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true)));
- tmpMap.put(_Fields.COLUMN_PARENT, new org.apache.thrift.meta_data.FieldMetaData("column_parent", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, ColumnParent.class)));
- tmpMap.put(_Fields.PREDICATE, new org.apache.thrift.meta_data.FieldMetaData("predicate", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, SlicePredicate.class)));
- tmpMap.put(_Fields.CONSISTENCY_LEVEL, new org.apache.thrift.meta_data.FieldMetaData("consistency_level", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.EnumMetaData(org.apache.thrift.protocol.TType.ENUM, ConsistencyLevel.class)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(get_slice_args.class, metaDataMap);
- }
-
- public get_slice_args() {
- this.consistency_level = org.apache.cassandra.thrift.ConsistencyLevel.ONE;
-
- }
-
- public get_slice_args(
- ByteBuffer key,
- ColumnParent column_parent,
- SlicePredicate predicate,
- ConsistencyLevel consistency_level)
- {
- this();
- this.key = key;
- this.column_parent = column_parent;
- this.predicate = predicate;
- this.consistency_level = consistency_level;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public get_slice_args(get_slice_args other) {
- if (other.isSetKey()) {
- this.key = org.apache.thrift.TBaseHelper.copyBinary(other.key);
-;
- }
- if (other.isSetColumn_parent()) {
- this.column_parent = new ColumnParent(other.column_parent);
- }
- if (other.isSetPredicate()) {
- this.predicate = new SlicePredicate(other.predicate);
- }
- if (other.isSetConsistency_level()) {
- this.consistency_level = other.consistency_level;
- }
- }
-
- public get_slice_args deepCopy() {
- return new get_slice_args(this);
- }
-
- @Override
- public void clear() {
- this.key = null;
- this.column_parent = null;
- this.predicate = null;
- this.consistency_level = org.apache.cassandra.thrift.ConsistencyLevel.ONE;
-
- }
-
- public byte[] getKey() {
- setKey(org.apache.thrift.TBaseHelper.rightSize(key));
- return key == null ? null : key.array();
- }
-
- public ByteBuffer bufferForKey() {
- return key;
- }
-
- public get_slice_args setKey(byte[] key) {
- setKey(key == null ? (ByteBuffer)null : ByteBuffer.wrap(key));
- return this;
- }
-
- public get_slice_args setKey(ByteBuffer key) {
- this.key = key;
- return this;
- }
-
- public void unsetKey() {
- this.key = null;
- }
-
- /** Returns true if field key is set (has been assigned a value) and false otherwise */
- public boolean isSetKey() {
- return this.key != null;
- }
-
- public void setKeyIsSet(boolean value) {
- if (!value) {
- this.key = null;
- }
- }
-
- public ColumnParent getColumn_parent() {
- return this.column_parent;
- }
-
- public get_slice_args setColumn_parent(ColumnParent column_parent) {
- this.column_parent = column_parent;
- return this;
- }
-
- public void unsetColumn_parent() {
- this.column_parent = null;
- }
-
- /** Returns true if field column_parent is set (has been assigned a value) and false otherwise */
- public boolean isSetColumn_parent() {
- return this.column_parent != null;
- }
-
- public void setColumn_parentIsSet(boolean value) {
- if (!value) {
- this.column_parent = null;
- }
- }
-
- public SlicePredicate getPredicate() {
- return this.predicate;
- }
-
- public get_slice_args setPredicate(SlicePredicate predicate) {
- this.predicate = predicate;
- return this;
- }
-
- public void unsetPredicate() {
- this.predicate = null;
- }
-
- /** Returns true if field predicate is set (has been assigned a value) and false otherwise */
- public boolean isSetPredicate() {
- return this.predicate != null;
- }
-
- public void setPredicateIsSet(boolean value) {
- if (!value) {
- this.predicate = null;
- }
- }
-
- /**
- *
- * @see ConsistencyLevel
- */
- public ConsistencyLevel getConsistency_level() {
- return this.consistency_level;
- }
-
- /**
- *
- * @see ConsistencyLevel
- */
- public get_slice_args setConsistency_level(ConsistencyLevel consistency_level) {
- this.consistency_level = consistency_level;
- return this;
- }
-
- public void unsetConsistency_level() {
- this.consistency_level = null;
- }
-
- /** Returns true if field consistency_level is set (has been assigned a value) and false otherwise */
- public boolean isSetConsistency_level() {
- return this.consistency_level != null;
- }
-
- public void setConsistency_levelIsSet(boolean value) {
- if (!value) {
- this.consistency_level = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case KEY:
- if (value == null) {
- unsetKey();
- } else {
- setKey((ByteBuffer)value);
- }
- break;
-
- case COLUMN_PARENT:
- if (value == null) {
- unsetColumn_parent();
- } else {
- setColumn_parent((ColumnParent)value);
- }
- break;
-
- case PREDICATE:
- if (value == null) {
- unsetPredicate();
- } else {
- setPredicate((SlicePredicate)value);
- }
- break;
-
- case CONSISTENCY_LEVEL:
- if (value == null) {
- unsetConsistency_level();
- } else {
- setConsistency_level((ConsistencyLevel)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case KEY:
- return getKey();
-
- case COLUMN_PARENT:
- return getColumn_parent();
-
- case PREDICATE:
- return getPredicate();
-
- case CONSISTENCY_LEVEL:
- return getConsistency_level();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case KEY:
- return isSetKey();
- case COLUMN_PARENT:
- return isSetColumn_parent();
- case PREDICATE:
- return isSetPredicate();
- case CONSISTENCY_LEVEL:
- return isSetConsistency_level();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof get_slice_args)
- return this.equals((get_slice_args)that);
- return false;
- }
-
- public boolean equals(get_slice_args that) {
- if (that == null)
- return false;
-
- boolean this_present_key = true && this.isSetKey();
- boolean that_present_key = true && that.isSetKey();
- if (this_present_key || that_present_key) {
- if (!(this_present_key && that_present_key))
- return false;
- if (!this.key.equals(that.key))
- return false;
- }
-
- boolean this_present_column_parent = true && this.isSetColumn_parent();
- boolean that_present_column_parent = true && that.isSetColumn_parent();
- if (this_present_column_parent || that_present_column_parent) {
- if (!(this_present_column_parent && that_present_column_parent))
- return false;
- if (!this.column_parent.equals(that.column_parent))
- return false;
- }
-
- boolean this_present_predicate = true && this.isSetPredicate();
- boolean that_present_predicate = true && that.isSetPredicate();
- if (this_present_predicate || that_present_predicate) {
- if (!(this_present_predicate && that_present_predicate))
- return false;
- if (!this.predicate.equals(that.predicate))
- return false;
- }
-
- boolean this_present_consistency_level = true && this.isSetConsistency_level();
- boolean that_present_consistency_level = true && that.isSetConsistency_level();
- if (this_present_consistency_level || that_present_consistency_level) {
- if (!(this_present_consistency_level && that_present_consistency_level))
- return false;
- if (!this.consistency_level.equals(that.consistency_level))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_key = true && (isSetKey());
- builder.append(present_key);
- if (present_key)
- builder.append(key);
-
- boolean present_column_parent = true && (isSetColumn_parent());
- builder.append(present_column_parent);
- if (present_column_parent)
- builder.append(column_parent);
-
- boolean present_predicate = true && (isSetPredicate());
- builder.append(present_predicate);
- if (present_predicate)
- builder.append(predicate);
-
- boolean present_consistency_level = true && (isSetConsistency_level());
- builder.append(present_consistency_level);
- if (present_consistency_level)
- builder.append(consistency_level.getValue());
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(get_slice_args other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetKey()).compareTo(other.isSetKey());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetKey()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.key, other.key);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetColumn_parent()).compareTo(other.isSetColumn_parent());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetColumn_parent()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.column_parent, other.column_parent);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetPredicate()).compareTo(other.isSetPredicate());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetPredicate()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.predicate, other.predicate);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetConsistency_level()).compareTo(other.isSetConsistency_level());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetConsistency_level()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.consistency_level, other.consistency_level);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("get_slice_args(");
- boolean first = true;
-
- sb.append("key:");
- if (this.key == null) {
- sb.append("null");
- } else {
- org.apache.thrift.TBaseHelper.toString(this.key, sb);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("column_parent:");
- if (this.column_parent == null) {
- sb.append("null");
- } else {
- sb.append(this.column_parent);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("predicate:");
- if (this.predicate == null) {
- sb.append("null");
- } else {
- sb.append(this.predicate);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("consistency_level:");
- if (this.consistency_level == null) {
- sb.append("null");
- } else {
- sb.append(this.consistency_level);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- if (key == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'key' was not present! Struct: " + toString());
- }
- if (column_parent == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'column_parent' was not present! Struct: " + toString());
- }
- if (predicate == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'predicate' was not present! Struct: " + toString());
- }
- if (consistency_level == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'consistency_level' was not present! Struct: " + toString());
- }
- // check for sub-struct validity
- if (column_parent != null) {
- column_parent.validate();
- }
- if (predicate != null) {
- predicate.validate();
- }
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class get_slice_argsStandardSchemeFactory implements SchemeFactory {
- public get_slice_argsStandardScheme getScheme() {
- return new get_slice_argsStandardScheme();
- }
- }
-
- private static class get_slice_argsStandardScheme extends StandardScheme<get_slice_args> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, get_slice_args struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // KEY
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.key = iprot.readBinary();
- struct.setKeyIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 2: // COLUMN_PARENT
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.column_parent = new ColumnParent();
- struct.column_parent.read(iprot);
- struct.setColumn_parentIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 3: // PREDICATE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.predicate = new SlicePredicate();
- struct.predicate.read(iprot);
- struct.setPredicateIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 4: // CONSISTENCY_LEVEL
- if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
- struct.consistency_level = ConsistencyLevel.findByValue(iprot.readI32());
- struct.setConsistency_levelIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, get_slice_args struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.key != null) {
- oprot.writeFieldBegin(KEY_FIELD_DESC);
- oprot.writeBinary(struct.key);
- oprot.writeFieldEnd();
- }
- if (struct.column_parent != null) {
- oprot.writeFieldBegin(COLUMN_PARENT_FIELD_DESC);
- struct.column_parent.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.predicate != null) {
- oprot.writeFieldBegin(PREDICATE_FIELD_DESC);
- struct.predicate.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.consistency_level != null) {
- oprot.writeFieldBegin(CONSISTENCY_LEVEL_FIELD_DESC);
- oprot.writeI32(struct.consistency_level.getValue());
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class get_slice_argsTupleSchemeFactory implements SchemeFactory {
- public get_slice_argsTupleScheme getScheme() {
- return new get_slice_argsTupleScheme();
- }
- }
-
- private static class get_slice_argsTupleScheme extends TupleScheme<get_slice_args> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, get_slice_args struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- oprot.writeBinary(struct.key);
- struct.column_parent.write(oprot);
- struct.predicate.write(oprot);
- oprot.writeI32(struct.consistency_level.getValue());
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, get_slice_args struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- struct.key = iprot.readBinary();
- struct.setKeyIsSet(true);
- struct.column_parent = new ColumnParent();
- struct.column_parent.read(iprot);
- struct.setColumn_parentIsSet(true);
- struct.predicate = new SlicePredicate();
- struct.predicate.read(iprot);
- struct.setPredicateIsSet(true);
- struct.consistency_level = ConsistencyLevel.findByValue(iprot.readI32());
- struct.setConsistency_levelIsSet(true);
- }
- }
-
- }
-
- public static class get_slice_result implements org.apache.thrift.TBase<get_slice_result, get_slice_result._Fields>, java.io.Serializable, Cloneable, Comparable<get_slice_result> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("get_slice_result");
-
- private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.LIST, (short)0);
- private static final org.apache.thrift.protocol.TField IRE_FIELD_DESC = new org.apache.thrift.protocol.TField("ire", org.apache.thrift.protocol.TType.STRUCT, (short)1);
- private static final org.apache.thrift.protocol.TField UE_FIELD_DESC = new org.apache.thrift.protocol.TField("ue", org.apache.thrift.protocol.TType.STRUCT, (short)2);
- private static final org.apache.thrift.protocol.TField TE_FIELD_DESC = new org.apache.thrift.protocol.TField("te", org.apache.thrift.protocol.TType.STRUCT, (short)3);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new get_slice_resultStandardSchemeFactory());
- schemes.put(TupleScheme.class, new get_slice_resultTupleSchemeFactory());
- }
-
- public List<ColumnOrSuperColumn> success; // required
- public InvalidRequestException ire; // required
- public UnavailableException ue; // required
- public TimedOutException te; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- SUCCESS((short)0, "success"),
- IRE((short)1, "ire"),
- UE((short)2, "ue"),
- TE((short)3, "te");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 0: // SUCCESS
- return SUCCESS;
- case 1: // IRE
- return IRE;
- case 2: // UE
- return UE;
- case 3: // TE
- return TE;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST,
- new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, ColumnOrSuperColumn.class))));
- tmpMap.put(_Fields.IRE, new org.apache.thrift.meta_data.FieldMetaData("ire", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- tmpMap.put(_Fields.UE, new org.apache.thrift.meta_data.FieldMetaData("ue", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- tmpMap.put(_Fields.TE, new org.apache.thrift.meta_data.FieldMetaData("te", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(get_slice_result.class, metaDataMap);
- }
-
- public get_slice_result() {
- }
-
- public get_slice_result(
- List<ColumnOrSuperColumn> success,
- InvalidRequestException ire,
- UnavailableException ue,
- TimedOutException te)
- {
- this();
- this.success = success;
- this.ire = ire;
- this.ue = ue;
- this.te = te;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public get_slice_result(get_slice_result other) {
- if (other.isSetSuccess()) {
- List<ColumnOrSuperColumn> __this__success = new ArrayList<ColumnOrSuperColumn>(other.success.size());
- for (ColumnOrSuperColumn other_element : other.success) {
- __this__success.add(new ColumnOrSuperColumn(other_element));
- }
- this.success = __this__success;
- }
- if (other.isSetIre()) {
- this.ire = new InvalidRequestException(other.ire);
- }
- if (other.isSetUe()) {
- this.ue = new UnavailableException(other.ue);
- }
- if (other.isSetTe()) {
- this.te = new TimedOutException(other.te);
- }
- }
-
- public get_slice_result deepCopy() {
- return new get_slice_result(this);
- }
-
- @Override
- public void clear() {
- this.success = null;
- this.ire = null;
- this.ue = null;
- this.te = null;
- }
-
- public int getSuccessSize() {
- return (this.success == null) ? 0 : this.success.size();
- }
-
- public java.util.Iterator<ColumnOrSuperColumn> getSuccessIterator() {
- return (this.success == null) ? null : this.success.iterator();
- }
-
- public void addToSuccess(ColumnOrSuperColumn elem) {
- if (this.success == null) {
- this.success = new ArrayList<ColumnOrSuperColumn>();
- }
- this.success.add(elem);
- }
-
- public List<ColumnOrSuperColumn> getSuccess() {
- return this.success;
- }
-
- public get_slice_result setSuccess(List<ColumnOrSuperColumn> success) {
- this.success = success;
- return this;
- }
-
- public void unsetSuccess() {
- this.success = null;
- }
-
- /** Returns true if field success is set (has been assigned a value) and false otherwise */
- public boolean isSetSuccess() {
- return this.success != null;
- }
-
- public void setSuccessIsSet(boolean value) {
- if (!value) {
- this.success = null;
- }
- }
-
- public InvalidRequestException getIre() {
- return this.ire;
- }
-
- public get_slice_result setIre(InvalidRequestException ire) {
- this.ire = ire;
- return this;
- }
-
- public void unsetIre() {
- this.ire = null;
- }
-
- /** Returns true if field ire is set (has been assigned a value) and false otherwise */
- public boolean isSetIre() {
- return this.ire != null;
- }
-
- public void setIreIsSet(boolean value) {
- if (!value) {
- this.ire = null;
- }
- }
-
- public UnavailableException getUe() {
- return this.ue;
- }
-
- public get_slice_result setUe(UnavailableException ue) {
- this.ue = ue;
- return this;
- }
-
- public void unsetUe() {
- this.ue = null;
- }
-
- /** Returns true if field ue is set (has been assigned a value) and false otherwise */
- public boolean isSetUe() {
- return this.ue != null;
- }
-
- public void setUeIsSet(boolean value) {
- if (!value) {
- this.ue = null;
- }
- }
-
- public TimedOutException getTe() {
- return this.te;
- }
-
- public get_slice_result setTe(TimedOutException te) {
- this.te = te;
- return this;
- }
-
- public void unsetTe() {
- this.te = null;
- }
-
- /** Returns true if field te is set (has been assigned a value) and false otherwise */
- public boolean isSetTe() {
- return this.te != null;
- }
-
- public void setTeIsSet(boolean value) {
- if (!value) {
- this.te = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case SUCCESS:
- if (value == null) {
- unsetSuccess();
- } else {
- setSuccess((List<ColumnOrSuperColumn>)value);
- }
- break;
-
- case IRE:
- if (value == null) {
- unsetIre();
- } else {
- setIre((InvalidRequestException)value);
- }
- break;
-
- case UE:
- if (value == null) {
- unsetUe();
- } else {
- setUe((UnavailableException)value);
- }
- break;
-
- case TE:
- if (value == null) {
- unsetTe();
- } else {
- setTe((TimedOutException)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case SUCCESS:
- return getSuccess();
-
- case IRE:
- return getIre();
-
- case UE:
- return getUe();
-
- case TE:
- return getTe();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case SUCCESS:
- return isSetSuccess();
- case IRE:
- return isSetIre();
- case UE:
- return isSetUe();
- case TE:
- return isSetTe();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof get_slice_result)
- return this.equals((get_slice_result)that);
- return false;
- }
-
- public boolean equals(get_slice_result that) {
- if (that == null)
- return false;
-
- boolean this_present_success = true && this.isSetSuccess();
- boolean that_present_success = true && that.isSetSuccess();
- if (this_present_success || that_present_success) {
- if (!(this_present_success && that_present_success))
- return false;
- if (!this.success.equals(that.success))
- return false;
- }
-
- boolean this_present_ire = true && this.isSetIre();
- boolean that_present_ire = true && that.isSetIre();
- if (this_present_ire || that_present_ire) {
- if (!(this_present_ire && that_present_ire))
- return false;
- if (!this.ire.equals(that.ire))
- return false;
- }
-
- boolean this_present_ue = true && this.isSetUe();
- boolean that_present_ue = true && that.isSetUe();
- if (this_present_ue || that_present_ue) {
- if (!(this_present_ue && that_present_ue))
- return false;
- if (!this.ue.equals(that.ue))
- return false;
- }
-
- boolean this_present_te = true && this.isSetTe();
- boolean that_present_te = true && that.isSetTe();
- if (this_present_te || that_present_te) {
- if (!(this_present_te && that_present_te))
- return false;
- if (!this.te.equals(that.te))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_success = true && (isSetSuccess());
- builder.append(present_success);
- if (present_success)
- builder.append(success);
-
- boolean present_ire = true && (isSetIre());
- builder.append(present_ire);
- if (present_ire)
- builder.append(ire);
-
- boolean present_ue = true && (isSetUe());
- builder.append(present_ue);
- if (present_ue)
- builder.append(ue);
-
- boolean present_te = true && (isSetTe());
- builder.append(present_te);
- if (present_te)
- builder.append(te);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(get_slice_result other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(other.isSetSuccess());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetSuccess()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, other.success);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetIre()).compareTo(other.isSetIre());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetIre()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.ire, other.ire);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetUe()).compareTo(other.isSetUe());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetUe()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.ue, other.ue);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetTe()).compareTo(other.isSetTe());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetTe()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.te, other.te);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("get_slice_result(");
- boolean first = true;
-
- sb.append("success:");
- if (this.success == null) {
- sb.append("null");
- } else {
- sb.append(this.success);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("ire:");
- if (this.ire == null) {
- sb.append("null");
- } else {
- sb.append(this.ire);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("ue:");
- if (this.ue == null) {
- sb.append("null");
- } else {
- sb.append(this.ue);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("te:");
- if (this.te == null) {
- sb.append("null");
- } else {
- sb.append(this.te);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class get_slice_resultStandardSchemeFactory implements SchemeFactory {
- public get_slice_resultStandardScheme getScheme() {
- return new get_slice_resultStandardScheme();
- }
- }
-
- private static class get_slice_resultStandardScheme extends StandardScheme<get_slice_result> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, get_slice_result struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 0: // SUCCESS
- if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
- {
- org.apache.thrift.protocol.TList _list224 = iprot.readListBegin();
- struct.success = new ArrayList<ColumnOrSuperColumn>(_list224.size);
- for (int _i225 = 0; _i225 < _list224.size; ++_i225)
- {
- ColumnOrSuperColumn _elem226;
- _elem226 = new ColumnOrSuperColumn();
- _elem226.read(iprot);
- struct.success.add(_elem226);
- }
- iprot.readListEnd();
- }
- struct.setSuccessIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 1: // IRE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.ire = new InvalidRequestException();
- struct.ire.read(iprot);
- struct.setIreIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 2: // UE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.ue = new UnavailableException();
- struct.ue.read(iprot);
- struct.setUeIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 3: // TE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.te = new TimedOutException();
- struct.te.read(iprot);
- struct.setTeIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, get_slice_result struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.success != null) {
- oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
- {
- oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.success.size()));
- for (ColumnOrSuperColumn _iter227 : struct.success)
- {
- _iter227.write(oprot);
- }
- oprot.writeListEnd();
- }
- oprot.writeFieldEnd();
- }
- if (struct.ire != null) {
- oprot.writeFieldBegin(IRE_FIELD_DESC);
- struct.ire.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.ue != null) {
- oprot.writeFieldBegin(UE_FIELD_DESC);
- struct.ue.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.te != null) {
- oprot.writeFieldBegin(TE_FIELD_DESC);
- struct.te.write(oprot);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class get_slice_resultTupleSchemeFactory implements SchemeFactory {
- public get_slice_resultTupleScheme getScheme() {
- return new get_slice_resultTupleScheme();
- }
- }
-
- private static class get_slice_resultTupleScheme extends TupleScheme<get_slice_result> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, get_slice_result struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- BitSet optionals = new BitSet();
- if (struct.isSetSuccess()) {
- optionals.set(0);
- }
- if (struct.isSetIre()) {
- optionals.set(1);
- }
- if (struct.isSetUe()) {
- optionals.set(2);
- }
- if (struct.isSetTe()) {
- optionals.set(3);
- }
- oprot.writeBitSet(optionals, 4);
- if (struct.isSetSuccess()) {
- {
- oprot.writeI32(struct.success.size());
- for (ColumnOrSuperColumn _iter228 : struct.success)
- {
- _iter228.write(oprot);
- }
- }
- }
- if (struct.isSetIre()) {
- struct.ire.write(oprot);
- }
- if (struct.isSetUe()) {
- struct.ue.write(oprot);
- }
- if (struct.isSetTe()) {
- struct.te.write(oprot);
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, get_slice_result struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- BitSet incoming = iprot.readBitSet(4);
- if (incoming.get(0)) {
- {
- org.apache.thrift.protocol.TList _list229 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
- struct.success = new ArrayList<ColumnOrSuperColumn>(_list229.size);
- for (int _i230 = 0; _i230 < _list229.size; ++_i230)
- {
- ColumnOrSuperColumn _elem231;
- _elem231 = new ColumnOrSuperColumn();
- _elem231.read(iprot);
- struct.success.add(_elem231);
- }
- }
- struct.setSuccessIsSet(true);
- }
- if (incoming.get(1)) {
- struct.ire = new InvalidRequestException();
- struct.ire.read(iprot);
- struct.setIreIsSet(true);
- }
- if (incoming.get(2)) {
- struct.ue = new UnavailableException();
- struct.ue.read(iprot);
- struct.setUeIsSet(true);
- }
- if (incoming.get(3)) {
- struct.te = new TimedOutException();
- struct.te.read(iprot);
- struct.setTeIsSet(true);
- }
- }
- }
-
- }
-
- public static class get_count_args implements org.apache.thrift.TBase<get_count_args, get_count_args._Fields>, java.io.Serializable, Cloneable, Comparable<get_count_args> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("get_count_args");
-
- private static final org.apache.thrift.protocol.TField KEY_FIELD_DESC = new org.apache.thrift.protocol.TField("key", org.apache.thrift.protocol.TType.STRING, (short)1);
- private static final org.apache.thrift.protocol.TField COLUMN_PARENT_FIELD_DESC = new org.apache.thrift.protocol.TField("column_parent", org.apache.thrift.protocol.TType.STRUCT, (short)2);
- private static final org.apache.thrift.protocol.TField PREDICATE_FIELD_DESC = new org.apache.thrift.protocol.TField("predicate", org.apache.thrift.protocol.TType.STRUCT, (short)3);
- private static final org.apache.thrift.protocol.TField CONSISTENCY_LEVEL_FIELD_DESC = new org.apache.thrift.protocol.TField("consistency_level", org.apache.thrift.protocol.TType.I32, (short)4);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new get_count_argsStandardSchemeFactory());
- schemes.put(TupleScheme.class, new get_count_argsTupleSchemeFactory());
- }
-
- public ByteBuffer key; // required
- public ColumnParent column_parent; // required
- public SlicePredicate predicate; // required
- /**
- *
- * @see ConsistencyLevel
- */
- public ConsistencyLevel consistency_level; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- KEY((short)1, "key"),
- COLUMN_PARENT((short)2, "column_parent"),
- PREDICATE((short)3, "predicate"),
- /**
- *
- * @see ConsistencyLevel
- */
- CONSISTENCY_LEVEL((short)4, "consistency_level");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // KEY
- return KEY;
- case 2: // COLUMN_PARENT
- return COLUMN_PARENT;
- case 3: // PREDICATE
- return PREDICATE;
- case 4: // CONSISTENCY_LEVEL
- return CONSISTENCY_LEVEL;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.KEY, new org.apache.thrift.meta_data.FieldMetaData("key", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true)));
- tmpMap.put(_Fields.COLUMN_PARENT, new org.apache.thrift.meta_data.FieldMetaData("column_parent", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, ColumnParent.class)));
- tmpMap.put(_Fields.PREDICATE, new org.apache.thrift.meta_data.FieldMetaData("predicate", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, SlicePredicate.class)));
- tmpMap.put(_Fields.CONSISTENCY_LEVEL, new org.apache.thrift.meta_data.FieldMetaData("consistency_level", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.EnumMetaData(org.apache.thrift.protocol.TType.ENUM, ConsistencyLevel.class)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(get_count_args.class, metaDataMap);
- }
-
- public get_count_args() {
- this.consistency_level = org.apache.cassandra.thrift.ConsistencyLevel.ONE;
-
- }
-
- public get_count_args(
- ByteBuffer key,
- ColumnParent column_parent,
- SlicePredicate predicate,
- ConsistencyLevel consistency_level)
- {
- this();
- this.key = key;
- this.column_parent = column_parent;
- this.predicate = predicate;
- this.consistency_level = consistency_level;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public get_count_args(get_count_args other) {
- if (other.isSetKey()) {
- this.key = org.apache.thrift.TBaseHelper.copyBinary(other.key);
-;
- }
- if (other.isSetColumn_parent()) {
- this.column_parent = new ColumnParent(other.column_parent);
- }
- if (other.isSetPredicate()) {
- this.predicate = new SlicePredicate(other.predicate);
- }
- if (other.isSetConsistency_level()) {
- this.consistency_level = other.consistency_level;
- }
- }
-
- public get_count_args deepCopy() {
- return new get_count_args(this);
- }
-
- @Override
- public void clear() {
- this.key = null;
- this.column_parent = null;
- this.predicate = null;
- this.consistency_level = org.apache.cassandra.thrift.ConsistencyLevel.ONE;
-
- }
-
- public byte[] getKey() {
- setKey(org.apache.thrift.TBaseHelper.rightSize(key));
- return key == null ? null : key.array();
- }
-
- public ByteBuffer bufferForKey() {
- return key;
- }
-
- public get_count_args setKey(byte[] key) {
- setKey(key == null ? (ByteBuffer)null : ByteBuffer.wrap(key));
- return this;
- }
-
- public get_count_args setKey(ByteBuffer key) {
- this.key = key;
- return this;
- }
-
- public void unsetKey() {
- this.key = null;
- }
-
- /** Returns true if field key is set (has been assigned a value) and false otherwise */
- public boolean isSetKey() {
- return this.key != null;
- }
-
- public void setKeyIsSet(boolean value) {
- if (!value) {
- this.key = null;
- }
- }
-
- public ColumnParent getColumn_parent() {
- return this.column_parent;
- }
-
- public get_count_args setColumn_parent(ColumnParent column_parent) {
- this.column_parent = column_parent;
- return this;
- }
-
- public void unsetColumn_parent() {
- this.column_parent = null;
- }
-
- /** Returns true if field column_parent is set (has been assigned a value) and false otherwise */
- public boolean isSetColumn_parent() {
- return this.column_parent != null;
- }
-
- public void setColumn_parentIsSet(boolean value) {
- if (!value) {
- this.column_parent = null;
- }
- }
-
- public SlicePredicate getPredicate() {
- return this.predicate;
- }
-
- public get_count_args setPredicate(SlicePredicate predicate) {
- this.predicate = predicate;
- return this;
- }
-
- public void unsetPredicate() {
- this.predicate = null;
- }
-
- /** Returns true if field predicate is set (has been assigned a value) and false otherwise */
- public boolean isSetPredicate() {
- return this.predicate != null;
- }
-
- public void setPredicateIsSet(boolean value) {
- if (!value) {
- this.predicate = null;
- }
- }
-
- /**
- *
- * @see ConsistencyLevel
- */
- public ConsistencyLevel getConsistency_level() {
- return this.consistency_level;
- }
-
- /**
- *
- * @see ConsistencyLevel
- */
- public get_count_args setConsistency_level(ConsistencyLevel consistency_level) {
- this.consistency_level = consistency_level;
- return this;
- }
-
- public void unsetConsistency_level() {
- this.consistency_level = null;
- }
-
- /** Returns true if field consistency_level is set (has been assigned a value) and false otherwise */
- public boolean isSetConsistency_level() {
- return this.consistency_level != null;
- }
-
- public void setConsistency_levelIsSet(boolean value) {
- if (!value) {
- this.consistency_level = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case KEY:
- if (value == null) {
- unsetKey();
- } else {
- setKey((ByteBuffer)value);
- }
- break;
-
- case COLUMN_PARENT:
- if (value == null) {
- unsetColumn_parent();
- } else {
- setColumn_parent((ColumnParent)value);
- }
- break;
-
- case PREDICATE:
- if (value == null) {
- unsetPredicate();
- } else {
- setPredicate((SlicePredicate)value);
- }
- break;
-
- case CONSISTENCY_LEVEL:
- if (value == null) {
- unsetConsistency_level();
- } else {
- setConsistency_level((ConsistencyLevel)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case KEY:
- return getKey();
-
- case COLUMN_PARENT:
- return getColumn_parent();
-
- case PREDICATE:
- return getPredicate();
-
- case CONSISTENCY_LEVEL:
- return getConsistency_level();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case KEY:
- return isSetKey();
- case COLUMN_PARENT:
- return isSetColumn_parent();
- case PREDICATE:
- return isSetPredicate();
- case CONSISTENCY_LEVEL:
- return isSetConsistency_level();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof get_count_args)
- return this.equals((get_count_args)that);
- return false;
- }
-
- public boolean equals(get_count_args that) {
- if (that == null)
- return false;
-
- boolean this_present_key = true && this.isSetKey();
- boolean that_present_key = true && that.isSetKey();
- if (this_present_key || that_present_key) {
- if (!(this_present_key && that_present_key))
- return false;
- if (!this.key.equals(that.key))
- return false;
- }
-
- boolean this_present_column_parent = true && this.isSetColumn_parent();
- boolean that_present_column_parent = true && that.isSetColumn_parent();
- if (this_present_column_parent || that_present_column_parent) {
- if (!(this_present_column_parent && that_present_column_parent))
- return false;
- if (!this.column_parent.equals(that.column_parent))
- return false;
- }
-
- boolean this_present_predicate = true && this.isSetPredicate();
- boolean that_present_predicate = true && that.isSetPredicate();
- if (this_present_predicate || that_present_predicate) {
- if (!(this_present_predicate && that_present_predicate))
- return false;
- if (!this.predicate.equals(that.predicate))
- return false;
- }
-
- boolean this_present_consistency_level = true && this.isSetConsistency_level();
- boolean that_present_consistency_level = true && that.isSetConsistency_level();
- if (this_present_consistency_level || that_present_consistency_level) {
- if (!(this_present_consistency_level && that_present_consistency_level))
- return false;
- if (!this.consistency_level.equals(that.consistency_level))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_key = true && (isSetKey());
- builder.append(present_key);
- if (present_key)
- builder.append(key);
-
- boolean present_column_parent = true && (isSetColumn_parent());
- builder.append(present_column_parent);
- if (present_column_parent)
- builder.append(column_parent);
-
- boolean present_predicate = true && (isSetPredicate());
- builder.append(present_predicate);
- if (present_predicate)
- builder.append(predicate);
-
- boolean present_consistency_level = true && (isSetConsistency_level());
- builder.append(present_consistency_level);
- if (present_consistency_level)
- builder.append(consistency_level.getValue());
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(get_count_args other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetKey()).compareTo(other.isSetKey());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetKey()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.key, other.key);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetColumn_parent()).compareTo(other.isSetColumn_parent());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetColumn_parent()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.column_parent, other.column_parent);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetPredicate()).compareTo(other.isSetPredicate());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetPredicate()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.predicate, other.predicate);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetConsistency_level()).compareTo(other.isSetConsistency_level());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetConsistency_level()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.consistency_level, other.consistency_level);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("get_count_args(");
- boolean first = true;
-
- sb.append("key:");
- if (this.key == null) {
- sb.append("null");
- } else {
- org.apache.thrift.TBaseHelper.toString(this.key, sb);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("column_parent:");
- if (this.column_parent == null) {
- sb.append("null");
- } else {
- sb.append(this.column_parent);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("predicate:");
- if (this.predicate == null) {
- sb.append("null");
- } else {
- sb.append(this.predicate);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("consistency_level:");
- if (this.consistency_level == null) {
- sb.append("null");
- } else {
- sb.append(this.consistency_level);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- if (key == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'key' was not present! Struct: " + toString());
- }
- if (column_parent == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'column_parent' was not present! Struct: " + toString());
- }
- if (predicate == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'predicate' was not present! Struct: " + toString());
- }
- if (consistency_level == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'consistency_level' was not present! Struct: " + toString());
- }
- // check for sub-struct validity
- if (column_parent != null) {
- column_parent.validate();
- }
- if (predicate != null) {
- predicate.validate();
- }
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class get_count_argsStandardSchemeFactory implements SchemeFactory {
- public get_count_argsStandardScheme getScheme() {
- return new get_count_argsStandardScheme();
- }
- }
-
- private static class get_count_argsStandardScheme extends StandardScheme<get_count_args> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, get_count_args struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // KEY
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.key = iprot.readBinary();
- struct.setKeyIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 2: // COLUMN_PARENT
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.column_parent = new ColumnParent();
- struct.column_parent.read(iprot);
- struct.setColumn_parentIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 3: // PREDICATE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.predicate = new SlicePredicate();
- struct.predicate.read(iprot);
- struct.setPredicateIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 4: // CONSISTENCY_LEVEL
- if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
- struct.consistency_level = ConsistencyLevel.findByValue(iprot.readI32());
- struct.setConsistency_levelIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, get_count_args struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.key != null) {
- oprot.writeFieldBegin(KEY_FIELD_DESC);
- oprot.writeBinary(struct.key);
- oprot.writeFieldEnd();
- }
- if (struct.column_parent != null) {
- oprot.writeFieldBegin(COLUMN_PARENT_FIELD_DESC);
- struct.column_parent.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.predicate != null) {
- oprot.writeFieldBegin(PREDICATE_FIELD_DESC);
- struct.predicate.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.consistency_level != null) {
- oprot.writeFieldBegin(CONSISTENCY_LEVEL_FIELD_DESC);
- oprot.writeI32(struct.consistency_level.getValue());
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class get_count_argsTupleSchemeFactory implements SchemeFactory {
- public get_count_argsTupleScheme getScheme() {
- return new get_count_argsTupleScheme();
- }
- }
-
- private static class get_count_argsTupleScheme extends TupleScheme<get_count_args> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, get_count_args struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- oprot.writeBinary(struct.key);
- struct.column_parent.write(oprot);
- struct.predicate.write(oprot);
- oprot.writeI32(struct.consistency_level.getValue());
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, get_count_args struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- struct.key = iprot.readBinary();
- struct.setKeyIsSet(true);
- struct.column_parent = new ColumnParent();
- struct.column_parent.read(iprot);
- struct.setColumn_parentIsSet(true);
- struct.predicate = new SlicePredicate();
- struct.predicate.read(iprot);
- struct.setPredicateIsSet(true);
- struct.consistency_level = ConsistencyLevel.findByValue(iprot.readI32());
- struct.setConsistency_levelIsSet(true);
- }
- }
-
- }
-
- public static class get_count_result implements org.apache.thrift.TBase<get_count_result, get_count_result._Fields>, java.io.Serializable, Cloneable, Comparable<get_count_result> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("get_count_result");
-
- private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.I32, (short)0);
- private static final org.apache.thrift.protocol.TField IRE_FIELD_DESC = new org.apache.thrift.protocol.TField("ire", org.apache.thrift.protocol.TType.STRUCT, (short)1);
- private static final org.apache.thrift.protocol.TField UE_FIELD_DESC = new org.apache.thrift.protocol.TField("ue", org.apache.thrift.protocol.TType.STRUCT, (short)2);
- private static final org.apache.thrift.protocol.TField TE_FIELD_DESC = new org.apache.thrift.protocol.TField("te", org.apache.thrift.protocol.TType.STRUCT, (short)3);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new get_count_resultStandardSchemeFactory());
- schemes.put(TupleScheme.class, new get_count_resultTupleSchemeFactory());
- }
-
- public int success; // required
- public InvalidRequestException ire; // required
- public UnavailableException ue; // required
- public TimedOutException te; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- SUCCESS((short)0, "success"),
- IRE((short)1, "ire"),
- UE((short)2, "ue"),
- TE((short)3, "te");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 0: // SUCCESS
- return SUCCESS;
- case 1: // IRE
- return IRE;
- case 2: // UE
- return UE;
- case 3: // TE
- return TE;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- private static final int __SUCCESS_ISSET_ID = 0;
- private byte __isset_bitfield = 0;
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32)));
- tmpMap.put(_Fields.IRE, new org.apache.thrift.meta_data.FieldMetaData("ire", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- tmpMap.put(_Fields.UE, new org.apache.thrift.meta_data.FieldMetaData("ue", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- tmpMap.put(_Fields.TE, new org.apache.thrift.meta_data.FieldMetaData("te", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(get_count_result.class, metaDataMap);
- }
-
- public get_count_result() {
- }
-
- public get_count_result(
- int success,
- InvalidRequestException ire,
- UnavailableException ue,
- TimedOutException te)
- {
- this();
- this.success = success;
- setSuccessIsSet(true);
- this.ire = ire;
- this.ue = ue;
- this.te = te;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public get_count_result(get_count_result other) {
- __isset_bitfield = other.__isset_bitfield;
- this.success = other.success;
- if (other.isSetIre()) {
- this.ire = new InvalidRequestException(other.ire);
- }
- if (other.isSetUe()) {
- this.ue = new UnavailableException(other.ue);
- }
- if (other.isSetTe()) {
- this.te = new TimedOutException(other.te);
- }
- }
-
- public get_count_result deepCopy() {
- return new get_count_result(this);
- }
-
- @Override
- public void clear() {
- setSuccessIsSet(false);
- this.success = 0;
- this.ire = null;
- this.ue = null;
- this.te = null;
- }
-
- public int getSuccess() {
- return this.success;
- }
-
- public get_count_result setSuccess(int success) {
- this.success = success;
- setSuccessIsSet(true);
- return this;
- }
-
- public void unsetSuccess() {
- __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __SUCCESS_ISSET_ID);
- }
-
- /** Returns true if field success is set (has been assigned a value) and false otherwise */
- public boolean isSetSuccess() {
- return EncodingUtils.testBit(__isset_bitfield, __SUCCESS_ISSET_ID);
- }
-
- public void setSuccessIsSet(boolean value) {
- __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __SUCCESS_ISSET_ID, value);
- }
-
- public InvalidRequestException getIre() {
- return this.ire;
- }
-
- public get_count_result setIre(InvalidRequestException ire) {
- this.ire = ire;
- return this;
- }
-
- public void unsetIre() {
- this.ire = null;
- }
-
- /** Returns true if field ire is set (has been assigned a value) and false otherwise */
- public boolean isSetIre() {
- return this.ire != null;
- }
-
- public void setIreIsSet(boolean value) {
- if (!value) {
- this.ire = null;
- }
- }
-
- public UnavailableException getUe() {
- return this.ue;
- }
-
- public get_count_result setUe(UnavailableException ue) {
- this.ue = ue;
- return this;
- }
-
- public void unsetUe() {
- this.ue = null;
- }
-
- /** Returns true if field ue is set (has been assigned a value) and false otherwise */
- public boolean isSetUe() {
- return this.ue != null;
- }
-
- public void setUeIsSet(boolean value) {
- if (!value) {
- this.ue = null;
- }
- }
-
- public TimedOutException getTe() {
- return this.te;
- }
-
- public get_count_result setTe(TimedOutException te) {
- this.te = te;
- return this;
- }
-
- public void unsetTe() {
- this.te = null;
- }
-
- /** Returns true if field te is set (has been assigned a value) and false otherwise */
- public boolean isSetTe() {
- return this.te != null;
- }
-
- public void setTeIsSet(boolean value) {
- if (!value) {
- this.te = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case SUCCESS:
- if (value == null) {
- unsetSuccess();
- } else {
- setSuccess((Integer)value);
- }
- break;
-
- case IRE:
- if (value == null) {
- unsetIre();
- } else {
- setIre((InvalidRequestException)value);
- }
- break;
-
- case UE:
- if (value == null) {
- unsetUe();
- } else {
- setUe((UnavailableException)value);
- }
- break;
-
- case TE:
- if (value == null) {
- unsetTe();
- } else {
- setTe((TimedOutException)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case SUCCESS:
- return Integer.valueOf(getSuccess());
-
- case IRE:
- return getIre();
-
- case UE:
- return getUe();
-
- case TE:
- return getTe();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case SUCCESS:
- return isSetSuccess();
- case IRE:
- return isSetIre();
- case UE:
- return isSetUe();
- case TE:
- return isSetTe();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof get_count_result)
- return this.equals((get_count_result)that);
- return false;
- }
-
- public boolean equals(get_count_result that) {
- if (that == null)
- return false;
-
- boolean this_present_success = true;
- boolean that_present_success = true;
- if (this_present_success || that_present_success) {
- if (!(this_present_success && that_present_success))
- return false;
- if (this.success != that.success)
- return false;
- }
-
- boolean this_present_ire = true && this.isSetIre();
- boolean that_present_ire = true && that.isSetIre();
- if (this_present_ire || that_present_ire) {
- if (!(this_present_ire && that_present_ire))
- return false;
- if (!this.ire.equals(that.ire))
- return false;
- }
-
- boolean this_present_ue = true && this.isSetUe();
- boolean that_present_ue = true && that.isSetUe();
- if (this_present_ue || that_present_ue) {
- if (!(this_present_ue && that_present_ue))
- return false;
- if (!this.ue.equals(that.ue))
- return false;
- }
-
- boolean this_present_te = true && this.isSetTe();
- boolean that_present_te = true && that.isSetTe();
- if (this_present_te || that_present_te) {
- if (!(this_present_te && that_present_te))
- return false;
- if (!this.te.equals(that.te))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_success = true;
- builder.append(present_success);
- if (present_success)
- builder.append(success);
-
- boolean present_ire = true && (isSetIre());
- builder.append(present_ire);
- if (present_ire)
- builder.append(ire);
-
- boolean present_ue = true && (isSetUe());
- builder.append(present_ue);
- if (present_ue)
- builder.append(ue);
-
- boolean present_te = true && (isSetTe());
- builder.append(present_te);
- if (present_te)
- builder.append(te);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(get_count_result other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(other.isSetSuccess());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetSuccess()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, other.success);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetIre()).compareTo(other.isSetIre());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetIre()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.ire, other.ire);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetUe()).compareTo(other.isSetUe());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetUe()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.ue, other.ue);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetTe()).compareTo(other.isSetTe());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetTe()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.te, other.te);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("get_count_result(");
- boolean first = true;
-
- sb.append("success:");
- sb.append(this.success);
- first = false;
- if (!first) sb.append(", ");
- sb.append("ire:");
- if (this.ire == null) {
- sb.append("null");
- } else {
- sb.append(this.ire);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("ue:");
- if (this.ue == null) {
- sb.append("null");
- } else {
- sb.append(this.ue);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("te:");
- if (this.te == null) {
- sb.append("null");
- } else {
- sb.append(this.te);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
- __isset_bitfield = 0;
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class get_count_resultStandardSchemeFactory implements SchemeFactory {
- public get_count_resultStandardScheme getScheme() {
- return new get_count_resultStandardScheme();
- }
- }
-
- private static class get_count_resultStandardScheme extends StandardScheme<get_count_result> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, get_count_result struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 0: // SUCCESS
- if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
- struct.success = iprot.readI32();
- struct.setSuccessIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 1: // IRE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.ire = new InvalidRequestException();
- struct.ire.read(iprot);
- struct.setIreIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 2: // UE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.ue = new UnavailableException();
- struct.ue.read(iprot);
- struct.setUeIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 3: // TE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.te = new TimedOutException();
- struct.te.read(iprot);
- struct.setTeIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, get_count_result struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.isSetSuccess()) {
- oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
- oprot.writeI32(struct.success);
- oprot.writeFieldEnd();
- }
- if (struct.ire != null) {
- oprot.writeFieldBegin(IRE_FIELD_DESC);
- struct.ire.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.ue != null) {
- oprot.writeFieldBegin(UE_FIELD_DESC);
- struct.ue.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.te != null) {
- oprot.writeFieldBegin(TE_FIELD_DESC);
- struct.te.write(oprot);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class get_count_resultTupleSchemeFactory implements SchemeFactory {
- public get_count_resultTupleScheme getScheme() {
- return new get_count_resultTupleScheme();
- }
- }
-
- private static class get_count_resultTupleScheme extends TupleScheme<get_count_result> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, get_count_result struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- BitSet optionals = new BitSet();
- if (struct.isSetSuccess()) {
- optionals.set(0);
- }
- if (struct.isSetIre()) {
- optionals.set(1);
- }
- if (struct.isSetUe()) {
- optionals.set(2);
- }
- if (struct.isSetTe()) {
- optionals.set(3);
- }
- oprot.writeBitSet(optionals, 4);
- if (struct.isSetSuccess()) {
- oprot.writeI32(struct.success);
- }
- if (struct.isSetIre()) {
- struct.ire.write(oprot);
- }
- if (struct.isSetUe()) {
- struct.ue.write(oprot);
- }
- if (struct.isSetTe()) {
- struct.te.write(oprot);
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, get_count_result struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- BitSet incoming = iprot.readBitSet(4);
- if (incoming.get(0)) {
- struct.success = iprot.readI32();
- struct.setSuccessIsSet(true);
- }
- if (incoming.get(1)) {
- struct.ire = new InvalidRequestException();
- struct.ire.read(iprot);
- struct.setIreIsSet(true);
- }
- if (incoming.get(2)) {
- struct.ue = new UnavailableException();
- struct.ue.read(iprot);
- struct.setUeIsSet(true);
- }
- if (incoming.get(3)) {
- struct.te = new TimedOutException();
- struct.te.read(iprot);
- struct.setTeIsSet(true);
- }
- }
- }
-
- }
-
- public static class multiget_slice_args implements org.apache.thrift.TBase<multiget_slice_args, multiget_slice_args._Fields>, java.io.Serializable, Cloneable, Comparable<multiget_slice_args> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("multiget_slice_args");
-
- private static final org.apache.thrift.protocol.TField KEYS_FIELD_DESC = new org.apache.thrift.protocol.TField("keys", org.apache.thrift.protocol.TType.LIST, (short)1);
- private static final org.apache.thrift.protocol.TField COLUMN_PARENT_FIELD_DESC = new org.apache.thrift.protocol.TField("column_parent", org.apache.thrift.protocol.TType.STRUCT, (short)2);
- private static final org.apache.thrift.protocol.TField PREDICATE_FIELD_DESC = new org.apache.thrift.protocol.TField("predicate", org.apache.thrift.protocol.TType.STRUCT, (short)3);
- private static final org.apache.thrift.protocol.TField CONSISTENCY_LEVEL_FIELD_DESC = new org.apache.thrift.protocol.TField("consistency_level", org.apache.thrift.protocol.TType.I32, (short)4);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new multiget_slice_argsStandardSchemeFactory());
- schemes.put(TupleScheme.class, new multiget_slice_argsTupleSchemeFactory());
- }
-
- public List<ByteBuffer> keys; // required
- public ColumnParent column_parent; // required
- public SlicePredicate predicate; // required
- /**
- *
- * @see ConsistencyLevel
- */
- public ConsistencyLevel consistency_level; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- KEYS((short)1, "keys"),
- COLUMN_PARENT((short)2, "column_parent"),
- PREDICATE((short)3, "predicate"),
- /**
- *
- * @see ConsistencyLevel
- */
- CONSISTENCY_LEVEL((short)4, "consistency_level");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // KEYS
- return KEYS;
- case 2: // COLUMN_PARENT
- return COLUMN_PARENT;
- case 3: // PREDICATE
- return PREDICATE;
- case 4: // CONSISTENCY_LEVEL
- return CONSISTENCY_LEVEL;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.KEYS, new org.apache.thrift.meta_data.FieldMetaData("keys", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true))));
- tmpMap.put(_Fields.COLUMN_PARENT, new org.apache.thrift.meta_data.FieldMetaData("column_parent", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, ColumnParent.class)));
- tmpMap.put(_Fields.PREDICATE, new org.apache.thrift.meta_data.FieldMetaData("predicate", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, SlicePredicate.class)));
- tmpMap.put(_Fields.CONSISTENCY_LEVEL, new org.apache.thrift.meta_data.FieldMetaData("consistency_level", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.EnumMetaData(org.apache.thrift.protocol.TType.ENUM, ConsistencyLevel.class)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(multiget_slice_args.class, metaDataMap);
- }
-
- public multiget_slice_args() {
- this.consistency_level = org.apache.cassandra.thrift.ConsistencyLevel.ONE;
-
- }
-
- public multiget_slice_args(
- List<ByteBuffer> keys,
- ColumnParent column_parent,
- SlicePredicate predicate,
- ConsistencyLevel consistency_level)
- {
- this();
- this.keys = keys;
- this.column_parent = column_parent;
- this.predicate = predicate;
- this.consistency_level = consistency_level;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public multiget_slice_args(multiget_slice_args other) {
- if (other.isSetKeys()) {
- List<ByteBuffer> __this__keys = new ArrayList<ByteBuffer>(other.keys);
- this.keys = __this__keys;
- }
- if (other.isSetColumn_parent()) {
- this.column_parent = new ColumnParent(other.column_parent);
- }
- if (other.isSetPredicate()) {
- this.predicate = new SlicePredicate(other.predicate);
- }
- if (other.isSetConsistency_level()) {
- this.consistency_level = other.consistency_level;
- }
- }
-
- public multiget_slice_args deepCopy() {
- return new multiget_slice_args(this);
- }
-
- @Override
- public void clear() {
- this.keys = null;
- this.column_parent = null;
- this.predicate = null;
- this.consistency_level = org.apache.cassandra.thrift.ConsistencyLevel.ONE;
-
- }
-
- public int getKeysSize() {
- return (this.keys == null) ? 0 : this.keys.size();
- }
-
- public java.util.Iterator<ByteBuffer> getKeysIterator() {
- return (this.keys == null) ? null : this.keys.iterator();
- }
-
- public void addToKeys(ByteBuffer elem) {
- if (this.keys == null) {
- this.keys = new ArrayList<ByteBuffer>();
- }
- this.keys.add(elem);
- }
-
- public List<ByteBuffer> getKeys() {
- return this.keys;
- }
-
- public multiget_slice_args setKeys(List<ByteBuffer> keys) {
- this.keys = keys;
- return this;
- }
-
- public void unsetKeys() {
- this.keys = null;
- }
-
- /** Returns true if field keys is set (has been assigned a value) and false otherwise */
- public boolean isSetKeys() {
- return this.keys != null;
- }
-
- public void setKeysIsSet(boolean value) {
- if (!value) {
- this.keys = null;
- }
- }
-
- public ColumnParent getColumn_parent() {
- return this.column_parent;
- }
-
- public multiget_slice_args setColumn_parent(ColumnParent column_parent) {
- this.column_parent = column_parent;
- return this;
- }
-
- public void unsetColumn_parent() {
- this.column_parent = null;
- }
-
- /** Returns true if field column_parent is set (has been assigned a value) and false otherwise */
- public boolean isSetColumn_parent() {
- return this.column_parent != null;
- }
-
- public void setColumn_parentIsSet(boolean value) {
- if (!value) {
- this.column_parent = null;
- }
- }
-
- public SlicePredicate getPredicate() {
- return this.predicate;
- }
-
- public multiget_slice_args setPredicate(SlicePredicate predicate) {
- this.predicate = predicate;
- return this;
- }
-
- public void unsetPredicate() {
- this.predicate = null;
- }
-
- /** Returns true if field predicate is set (has been assigned a value) and false otherwise */
- public boolean isSetPredicate() {
- return this.predicate != null;
- }
-
- public void setPredicateIsSet(boolean value) {
- if (!value) {
- this.predicate = null;
- }
- }
-
- /**
- *
- * @see ConsistencyLevel
- */
- public ConsistencyLevel getConsistency_level() {
- return this.consistency_level;
- }
-
- /**
- *
- * @see ConsistencyLevel
- */
- public multiget_slice_args setConsistency_level(ConsistencyLevel consistency_level) {
- this.consistency_level = consistency_level;
- return this;
- }
-
- public void unsetConsistency_level() {
- this.consistency_level = null;
- }
-
- /** Returns true if field consistency_level is set (has been assigned a value) and false otherwise */
- public boolean isSetConsistency_level() {
- return this.consistency_level != null;
- }
-
- public void setConsistency_levelIsSet(boolean value) {
- if (!value) {
- this.consistency_level = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case KEYS:
- if (value == null) {
- unsetKeys();
- } else {
- setKeys((List<ByteBuffer>)value);
- }
- break;
-
- case COLUMN_PARENT:
- if (value == null) {
- unsetColumn_parent();
- } else {
- setColumn_parent((ColumnParent)value);
- }
- break;
-
- case PREDICATE:
- if (value == null) {
- unsetPredicate();
- } else {
- setPredicate((SlicePredicate)value);
- }
- break;
-
- case CONSISTENCY_LEVEL:
- if (value == null) {
- unsetConsistency_level();
- } else {
- setConsistency_level((ConsistencyLevel)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case KEYS:
- return getKeys();
-
- case COLUMN_PARENT:
- return getColumn_parent();
-
- case PREDICATE:
- return getPredicate();
-
- case CONSISTENCY_LEVEL:
- return getConsistency_level();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case KEYS:
- return isSetKeys();
- case COLUMN_PARENT:
- return isSetColumn_parent();
- case PREDICATE:
- return isSetPredicate();
- case CONSISTENCY_LEVEL:
- return isSetConsistency_level();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof multiget_slice_args)
- return this.equals((multiget_slice_args)that);
- return false;
- }
-
- public boolean equals(multiget_slice_args that) {
- if (that == null)
- return false;
-
- boolean this_present_keys = true && this.isSetKeys();
- boolean that_present_keys = true && that.isSetKeys();
- if (this_present_keys || that_present_keys) {
- if (!(this_present_keys && that_present_keys))
- return false;
- if (!this.keys.equals(that.keys))
- return false;
- }
-
- boolean this_present_column_parent = true && this.isSetColumn_parent();
- boolean that_present_column_parent = true && that.isSetColumn_parent();
- if (this_present_column_parent || that_present_column_parent) {
- if (!(this_present_column_parent && that_present_column_parent))
- return false;
- if (!this.column_parent.equals(that.column_parent))
- return false;
- }
-
- boolean this_present_predicate = true && this.isSetPredicate();
- boolean that_present_predicate = true && that.isSetPredicate();
- if (this_present_predicate || that_present_predicate) {
- if (!(this_present_predicate && that_present_predicate))
- return false;
- if (!this.predicate.equals(that.predicate))
- return false;
- }
-
- boolean this_present_consistency_level = true && this.isSetConsistency_level();
- boolean that_present_consistency_level = true && that.isSetConsistency_level();
- if (this_present_consistency_level || that_present_consistency_level) {
- if (!(this_present_consistency_level && that_present_consistency_level))
- return false;
- if (!this.consistency_level.equals(that.consistency_level))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_keys = true && (isSetKeys());
- builder.append(present_keys);
- if (present_keys)
- builder.append(keys);
-
- boolean present_column_parent = true && (isSetColumn_parent());
- builder.append(present_column_parent);
- if (present_column_parent)
- builder.append(column_parent);
-
- boolean present_predicate = true && (isSetPredicate());
- builder.append(present_predicate);
- if (present_predicate)
- builder.append(predicate);
-
- boolean present_consistency_level = true && (isSetConsistency_level());
- builder.append(present_consistency_level);
- if (present_consistency_level)
- builder.append(consistency_level.getValue());
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(multiget_slice_args other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetKeys()).compareTo(other.isSetKeys());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetKeys()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.keys, other.keys);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetColumn_parent()).compareTo(other.isSetColumn_parent());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetColumn_parent()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.column_parent, other.column_parent);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetPredicate()).compareTo(other.isSetPredicate());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetPredicate()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.predicate, other.predicate);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetConsistency_level()).compareTo(other.isSetConsistency_level());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetConsistency_level()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.consistency_level, other.consistency_level);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("multiget_slice_args(");
- boolean first = true;
-
- sb.append("keys:");
- if (this.keys == null) {
- sb.append("null");
- } else {
- sb.append(this.keys);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("column_parent:");
- if (this.column_parent == null) {
- sb.append("null");
- } else {
- sb.append(this.column_parent);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("predicate:");
- if (this.predicate == null) {
- sb.append("null");
- } else {
- sb.append(this.predicate);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("consistency_level:");
- if (this.consistency_level == null) {
- sb.append("null");
- } else {
- sb.append(this.consistency_level);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- if (keys == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'keys' was not present! Struct: " + toString());
- }
- if (column_parent == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'column_parent' was not present! Struct: " + toString());
- }
- if (predicate == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'predicate' was not present! Struct: " + toString());
- }
- if (consistency_level == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'consistency_level' was not present! Struct: " + toString());
- }
- // check for sub-struct validity
- if (column_parent != null) {
- column_parent.validate();
- }
- if (predicate != null) {
- predicate.validate();
- }
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class multiget_slice_argsStandardSchemeFactory implements SchemeFactory {
- public multiget_slice_argsStandardScheme getScheme() {
- return new multiget_slice_argsStandardScheme();
- }
- }
-
- private static class multiget_slice_argsStandardScheme extends StandardScheme<multiget_slice_args> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, multiget_slice_args struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // KEYS
- if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
- {
- org.apache.thrift.protocol.TList _list232 = iprot.readListBegin();
- struct.keys = new ArrayList<ByteBuffer>(_list232.size);
- for (int _i233 = 0; _i233 < _list232.size; ++_i233)
- {
- ByteBuffer _elem234;
- _elem234 = iprot.readBinary();
- struct.keys.add(_elem234);
- }
- iprot.readListEnd();
- }
- struct.setKeysIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 2: // COLUMN_PARENT
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.column_parent = new ColumnParent();
- struct.column_parent.read(iprot);
- struct.setColumn_parentIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 3: // PREDICATE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.predicate = new SlicePredicate();
- struct.predicate.read(iprot);
- struct.setPredicateIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 4: // CONSISTENCY_LEVEL
- if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
- struct.consistency_level = ConsistencyLevel.findByValue(iprot.readI32());
- struct.setConsistency_levelIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, multiget_slice_args struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.keys != null) {
- oprot.writeFieldBegin(KEYS_FIELD_DESC);
- {
- oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, struct.keys.size()));
- for (ByteBuffer _iter235 : struct.keys)
- {
- oprot.writeBinary(_iter235);
- }
- oprot.writeListEnd();
- }
- oprot.writeFieldEnd();
- }
- if (struct.column_parent != null) {
- oprot.writeFieldBegin(COLUMN_PARENT_FIELD_DESC);
- struct.column_parent.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.predicate != null) {
- oprot.writeFieldBegin(PREDICATE_FIELD_DESC);
- struct.predicate.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.consistency_level != null) {
- oprot.writeFieldBegin(CONSISTENCY_LEVEL_FIELD_DESC);
- oprot.writeI32(struct.consistency_level.getValue());
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class multiget_slice_argsTupleSchemeFactory implements SchemeFactory {
- public multiget_slice_argsTupleScheme getScheme() {
- return new multiget_slice_argsTupleScheme();
- }
- }
-
- private static class multiget_slice_argsTupleScheme extends TupleScheme<multiget_slice_args> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, multiget_slice_args struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- {
- oprot.writeI32(struct.keys.size());
- for (ByteBuffer _iter236 : struct.keys)
- {
- oprot.writeBinary(_iter236);
- }
- }
- struct.column_parent.write(oprot);
- struct.predicate.write(oprot);
- oprot.writeI32(struct.consistency_level.getValue());
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, multiget_slice_args struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- {
- org.apache.thrift.protocol.TList _list237 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32());
- struct.keys = new ArrayList<ByteBuffer>(_list237.size);
- for (int _i238 = 0; _i238 < _list237.size; ++_i238)
- {
- ByteBuffer _elem239;
- _elem239 = iprot.readBinary();
- struct.keys.add(_elem239);
- }
- }
- struct.setKeysIsSet(true);
- struct.column_parent = new ColumnParent();
- struct.column_parent.read(iprot);
- struct.setColumn_parentIsSet(true);
- struct.predicate = new SlicePredicate();
- struct.predicate.read(iprot);
- struct.setPredicateIsSet(true);
- struct.consistency_level = ConsistencyLevel.findByValue(iprot.readI32());
- struct.setConsistency_levelIsSet(true);
- }
- }
-
- }
-
- public static class multiget_slice_result implements org.apache.thrift.TBase<multiget_slice_result, multiget_slice_result._Fields>, java.io.Serializable, Cloneable, Comparable<multiget_slice_result> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("multiget_slice_result");
-
- private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.MAP, (short)0);
- private static final org.apache.thrift.protocol.TField IRE_FIELD_DESC = new org.apache.thrift.protocol.TField("ire", org.apache.thrift.protocol.TType.STRUCT, (short)1);
- private static final org.apache.thrift.protocol.TField UE_FIELD_DESC = new org.apache.thrift.protocol.TField("ue", org.apache.thrift.protocol.TType.STRUCT, (short)2);
- private static final org.apache.thrift.protocol.TField TE_FIELD_DESC = new org.apache.thrift.protocol.TField("te", org.apache.thrift.protocol.TType.STRUCT, (short)3);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new multiget_slice_resultStandardSchemeFactory());
- schemes.put(TupleScheme.class, new multiget_slice_resultTupleSchemeFactory());
- }
-
- public Map<ByteBuffer,List<ColumnOrSuperColumn>> success; // required
- public InvalidRequestException ire; // required
- public UnavailableException ue; // required
- public TimedOutException te; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- SUCCESS((short)0, "success"),
- IRE((short)1, "ire"),
- UE((short)2, "ue"),
- TE((short)3, "te");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 0: // SUCCESS
- return SUCCESS;
- case 1: // IRE
- return IRE;
- case 2: // UE
- return UE;
- case 3: // TE
- return TE;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true),
- new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST,
- new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, ColumnOrSuperColumn.class)))));
- tmpMap.put(_Fields.IRE, new org.apache.thrift.meta_data.FieldMetaData("ire", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- tmpMap.put(_Fields.UE, new org.apache.thrift.meta_data.FieldMetaData("ue", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- tmpMap.put(_Fields.TE, new org.apache.thrift.meta_data.FieldMetaData("te", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(multiget_slice_result.class, metaDataMap);
- }
-
- public multiget_slice_result() {
- }
-
- public multiget_slice_result(
- Map<ByteBuffer,List<ColumnOrSuperColumn>> success,
- InvalidRequestException ire,
- UnavailableException ue,
- TimedOutException te)
- {
- this();
- this.success = success;
- this.ire = ire;
- this.ue = ue;
- this.te = te;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public multiget_slice_result(multiget_slice_result other) {
- if (other.isSetSuccess()) {
- Map<ByteBuffer,List<ColumnOrSuperColumn>> __this__success = new HashMap<ByteBuffer,List<ColumnOrSuperColumn>>(other.success.size());
- for (Map.Entry<ByteBuffer, List<ColumnOrSuperColumn>> other_element : other.success.entrySet()) {
-
- ByteBuffer other_element_key = other_element.getKey();
- List<ColumnOrSuperColumn> other_element_value = other_element.getValue();
-
- ByteBuffer __this__success_copy_key = org.apache.thrift.TBaseHelper.copyBinary(other_element_key);
-;
-
- List<ColumnOrSuperColumn> __this__success_copy_value = new ArrayList<ColumnOrSuperColumn>(other_element_value.size());
- for (ColumnOrSuperColumn other_element_value_element : other_element_value) {
- __this__success_copy_value.add(new ColumnOrSuperColumn(other_element_value_element));
- }
-
- __this__success.put(__this__success_copy_key, __this__success_copy_value);
- }
- this.success = __this__success;
- }
- if (other.isSetIre()) {
- this.ire = new InvalidRequestException(other.ire);
- }
- if (other.isSetUe()) {
- this.ue = new UnavailableException(other.ue);
- }
- if (other.isSetTe()) {
- this.te = new TimedOutException(other.te);
- }
- }
-
- public multiget_slice_result deepCopy() {
- return new multiget_slice_result(this);
- }
-
- @Override
- public void clear() {
- this.success = null;
- this.ire = null;
- this.ue = null;
- this.te = null;
- }
-
- public int getSuccessSize() {
- return (this.success == null) ? 0 : this.success.size();
- }
-
- public void putToSuccess(ByteBuffer key, List<ColumnOrSuperColumn> val) {
- if (this.success == null) {
- this.success = new HashMap<ByteBuffer,List<ColumnOrSuperColumn>>();
- }
- this.success.put(key, val);
- }
-
- public Map<ByteBuffer,List<ColumnOrSuperColumn>> getSuccess() {
- return this.success;
- }
-
- public multiget_slice_result setSuccess(Map<ByteBuffer,List<ColumnOrSuperColumn>> success) {
- this.success = success;
- return this;
- }
-
- public void unsetSuccess() {
- this.success = null;
- }
-
- /** Returns true if field success is set (has been assigned a value) and false otherwise */
- public boolean isSetSuccess() {
- return this.success != null;
- }
-
- public void setSuccessIsSet(boolean value) {
- if (!value) {
- this.success = null;
- }
- }
-
- public InvalidRequestException getIre() {
- return this.ire;
- }
-
- public multiget_slice_result setIre(InvalidRequestException ire) {
- this.ire = ire;
- return this;
- }
-
- public void unsetIre() {
- this.ire = null;
- }
-
- /** Returns true if field ire is set (has been assigned a value) and false otherwise */
- public boolean isSetIre() {
- return this.ire != null;
- }
-
- public void setIreIsSet(boolean value) {
- if (!value) {
- this.ire = null;
- }
- }
-
- public UnavailableException getUe() {
- return this.ue;
- }
-
- public multiget_slice_result setUe(UnavailableException ue) {
- this.ue = ue;
- return this;
- }
-
- public void unsetUe() {
- this.ue = null;
- }
-
- /** Returns true if field ue is set (has been assigned a value) and false otherwise */
- public boolean isSetUe() {
- return this.ue != null;
- }
-
- public void setUeIsSet(boolean value) {
- if (!value) {
- this.ue = null;
- }
- }
-
- public TimedOutException getTe() {
- return this.te;
- }
-
- public multiget_slice_result setTe(TimedOutException te) {
- this.te = te;
- return this;
- }
-
- public void unsetTe() {
- this.te = null;
- }
-
- /** Returns true if field te is set (has been assigned a value) and false otherwise */
- public boolean isSetTe() {
- return this.te != null;
- }
-
- public void setTeIsSet(boolean value) {
- if (!value) {
- this.te = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case SUCCESS:
- if (value == null) {
- unsetSuccess();
- } else {
- setSuccess((Map<ByteBuffer,List<ColumnOrSuperColumn>>)value);
- }
- break;
-
- case IRE:
- if (value == null) {
- unsetIre();
- } else {
- setIre((InvalidRequestException)value);
- }
- break;
-
- case UE:
- if (value == null) {
- unsetUe();
- } else {
- setUe((UnavailableException)value);
- }
- break;
-
- case TE:
- if (value == null) {
- unsetTe();
- } else {
- setTe((TimedOutException)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case SUCCESS:
- return getSuccess();
-
- case IRE:
- return getIre();
-
- case UE:
- return getUe();
-
- case TE:
- return getTe();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case SUCCESS:
- return isSetSuccess();
- case IRE:
- return isSetIre();
- case UE:
- return isSetUe();
- case TE:
- return isSetTe();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof multiget_slice_result)
- return this.equals((multiget_slice_result)that);
- return false;
- }
-
- public boolean equals(multiget_slice_result that) {
- if (that == null)
- return false;
-
- boolean this_present_success = true && this.isSetSuccess();
- boolean that_present_success = true && that.isSetSuccess();
- if (this_present_success || that_present_success) {
- if (!(this_present_success && that_present_success))
- return false;
- if (!this.success.equals(that.success))
- return false;
- }
-
- boolean this_present_ire = true && this.isSetIre();
- boolean that_present_ire = true && that.isSetIre();
- if (this_present_ire || that_present_ire) {
- if (!(this_present_ire && that_present_ire))
- return false;
- if (!this.ire.equals(that.ire))
- return false;
- }
-
- boolean this_present_ue = true && this.isSetUe();
- boolean that_present_ue = true && that.isSetUe();
- if (this_present_ue || that_present_ue) {
- if (!(this_present_ue && that_present_ue))
- return false;
- if (!this.ue.equals(that.ue))
- return false;
- }
-
- boolean this_present_te = true && this.isSetTe();
- boolean that_present_te = true && that.isSetTe();
- if (this_present_te || that_present_te) {
- if (!(this_present_te && that_present_te))
- return false;
- if (!this.te.equals(that.te))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_success = true && (isSetSuccess());
- builder.append(present_success);
- if (present_success)
- builder.append(success);
-
- boolean present_ire = true && (isSetIre());
- builder.append(present_ire);
- if (present_ire)
- builder.append(ire);
-
- boolean present_ue = true && (isSetUe());
- builder.append(present_ue);
- if (present_ue)
- builder.append(ue);
-
- boolean present_te = true && (isSetTe());
- builder.append(present_te);
- if (present_te)
- builder.append(te);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(multiget_slice_result other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(other.isSetSuccess());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetSuccess()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, other.success);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetIre()).compareTo(other.isSetIre());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetIre()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.ire, other.ire);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetUe()).compareTo(other.isSetUe());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetUe()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.ue, other.ue);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetTe()).compareTo(other.isSetTe());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetTe()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.te, other.te);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("multiget_slice_result(");
- boolean first = true;
-
- sb.append("success:");
- if (this.success == null) {
- sb.append("null");
- } else {
- sb.append(this.success);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("ire:");
- if (this.ire == null) {
- sb.append("null");
- } else {
- sb.append(this.ire);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("ue:");
- if (this.ue == null) {
- sb.append("null");
- } else {
- sb.append(this.ue);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("te:");
- if (this.te == null) {
- sb.append("null");
- } else {
- sb.append(this.te);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class multiget_slice_resultStandardSchemeFactory implements SchemeFactory {
- public multiget_slice_resultStandardScheme getScheme() {
- return new multiget_slice_resultStandardScheme();
- }
- }
-
- private static class multiget_slice_resultStandardScheme extends StandardScheme<multiget_slice_result> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, multiget_slice_result struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 0: // SUCCESS
- if (schemeField.type == org.apache.thrift.protocol.TType.MAP) {
- {
- org.apache.thrift.protocol.TMap _map240 = iprot.readMapBegin();
- struct.success = new HashMap<ByteBuffer,List<ColumnOrSuperColumn>>(2*_map240.size);
- for (int _i241 = 0; _i241 < _map240.size; ++_i241)
- {
- ByteBuffer _key242;
- List<ColumnOrSuperColumn> _val243;
- _key242 = iprot.readBinary();
- {
- org.apache.thrift.protocol.TList _list244 = iprot.readListBegin();
- _val243 = new ArrayList<ColumnOrSuperColumn>(_list244.size);
- for (int _i245 = 0; _i245 < _list244.size; ++_i245)
- {
- ColumnOrSuperColumn _elem246;
- _elem246 = new ColumnOrSuperColumn();
- _elem246.read(iprot);
- _val243.add(_elem246);
- }
- iprot.readListEnd();
- }
- struct.success.put(_key242, _val243);
- }
- iprot.readMapEnd();
- }
- struct.setSuccessIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 1: // IRE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.ire = new InvalidRequestException();
- struct.ire.read(iprot);
- struct.setIreIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 2: // UE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.ue = new UnavailableException();
- struct.ue.read(iprot);
- struct.setUeIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 3: // TE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.te = new TimedOutException();
- struct.te.read(iprot);
- struct.setTeIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, multiget_slice_result struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.success != null) {
- oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
- {
- oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.LIST, struct.success.size()));
- for (Map.Entry<ByteBuffer, List<ColumnOrSuperColumn>> _iter247 : struct.success.entrySet())
- {
- oprot.writeBinary(_iter247.getKey());
- {
- oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, _iter247.getValue().size()));
- for (ColumnOrSuperColumn _iter248 : _iter247.getValue())
- {
- _iter248.write(oprot);
- }
- oprot.writeListEnd();
- }
- }
- oprot.writeMapEnd();
- }
- oprot.writeFieldEnd();
- }
- if (struct.ire != null) {
- oprot.writeFieldBegin(IRE_FIELD_DESC);
- struct.ire.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.ue != null) {
- oprot.writeFieldBegin(UE_FIELD_DESC);
- struct.ue.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.te != null) {
- oprot.writeFieldBegin(TE_FIELD_DESC);
- struct.te.write(oprot);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class multiget_slice_resultTupleSchemeFactory implements SchemeFactory {
- public multiget_slice_resultTupleScheme getScheme() {
- return new multiget_slice_resultTupleScheme();
- }
- }
-
- private static class multiget_slice_resultTupleScheme extends TupleScheme<multiget_slice_result> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, multiget_slice_result struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- BitSet optionals = new BitSet();
- if (struct.isSetSuccess()) {
- optionals.set(0);
- }
- if (struct.isSetIre()) {
- optionals.set(1);
- }
- if (struct.isSetUe()) {
- optionals.set(2);
- }
- if (struct.isSetTe()) {
- optionals.set(3);
- }
- oprot.writeBitSet(optionals, 4);
- if (struct.isSetSuccess()) {
- {
- oprot.writeI32(struct.success.size());
- for (Map.Entry<ByteBuffer, List<ColumnOrSuperColumn>> _iter249 : struct.success.entrySet())
- {
- oprot.writeBinary(_iter249.getKey());
- {
- oprot.writeI32(_iter249.getValue().size());
- for (ColumnOrSuperColumn _iter250 : _iter249.getValue())
- {
- _iter250.write(oprot);
- }
- }
- }
- }
- }
- if (struct.isSetIre()) {
- struct.ire.write(oprot);
- }
- if (struct.isSetUe()) {
- struct.ue.write(oprot);
- }
- if (struct.isSetTe()) {
- struct.te.write(oprot);
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, multiget_slice_result struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- BitSet incoming = iprot.readBitSet(4);
- if (incoming.get(0)) {
- {
- org.apache.thrift.protocol.TMap _map251 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.LIST, iprot.readI32());
- struct.success = new HashMap<ByteBuffer,List<ColumnOrSuperColumn>>(2*_map251.size);
- for (int _i252 = 0; _i252 < _map251.size; ++_i252)
- {
- ByteBuffer _key253;
- List<ColumnOrSuperColumn> _val254;
- _key253 = iprot.readBinary();
- {
- org.apache.thrift.protocol.TList _list255 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
- _val254 = new ArrayList<ColumnOrSuperColumn>(_list255.size);
- for (int _i256 = 0; _i256 < _list255.size; ++_i256)
- {
- ColumnOrSuperColumn _elem257;
- _elem257 = new ColumnOrSuperColumn();
- _elem257.read(iprot);
- _val254.add(_elem257);
- }
- }
- struct.success.put(_key253, _val254);
- }
- }
- struct.setSuccessIsSet(true);
- }
- if (incoming.get(1)) {
- struct.ire = new InvalidRequestException();
- struct.ire.read(iprot);
- struct.setIreIsSet(true);
- }
- if (incoming.get(2)) {
- struct.ue = new UnavailableException();
- struct.ue.read(iprot);
- struct.setUeIsSet(true);
- }
- if (incoming.get(3)) {
- struct.te = new TimedOutException();
- struct.te.read(iprot);
- struct.setTeIsSet(true);
- }
- }
- }
-
- }
-
- public static class multiget_count_args implements org.apache.thrift.TBase<multiget_count_args, multiget_count_args._Fields>, java.io.Serializable, Cloneable, Comparable<multiget_count_args> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("multiget_count_args");
-
- private static final org.apache.thrift.protocol.TField KEYS_FIELD_DESC = new org.apache.thrift.protocol.TField("keys", org.apache.thrift.protocol.TType.LIST, (short)1);
- private static final org.apache.thrift.protocol.TField COLUMN_PARENT_FIELD_DESC = new org.apache.thrift.protocol.TField("column_parent", org.apache.thrift.protocol.TType.STRUCT, (short)2);
- private static final org.apache.thrift.protocol.TField PREDICATE_FIELD_DESC = new org.apache.thrift.protocol.TField("predicate", org.apache.thrift.protocol.TType.STRUCT, (short)3);
- private static final org.apache.thrift.protocol.TField CONSISTENCY_LEVEL_FIELD_DESC = new org.apache.thrift.protocol.TField("consistency_level", org.apache.thrift.protocol.TType.I32, (short)4);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new multiget_count_argsStandardSchemeFactory());
- schemes.put(TupleScheme.class, new multiget_count_argsTupleSchemeFactory());
- }
-
- public List<ByteBuffer> keys; // required
- public ColumnParent column_parent; // required
- public SlicePredicate predicate; // required
- /**
- *
- * @see ConsistencyLevel
- */
- public ConsistencyLevel consistency_level; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- KEYS((short)1, "keys"),
- COLUMN_PARENT((short)2, "column_parent"),
- PREDICATE((short)3, "predicate"),
- /**
- *
- * @see ConsistencyLevel
- */
- CONSISTENCY_LEVEL((short)4, "consistency_level");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // KEYS
- return KEYS;
- case 2: // COLUMN_PARENT
- return COLUMN_PARENT;
- case 3: // PREDICATE
- return PREDICATE;
- case 4: // CONSISTENCY_LEVEL
- return CONSISTENCY_LEVEL;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.KEYS, new org.apache.thrift.meta_data.FieldMetaData("keys", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true))));
- tmpMap.put(_Fields.COLUMN_PARENT, new org.apache.thrift.meta_data.FieldMetaData("column_parent", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, ColumnParent.class)));
- tmpMap.put(_Fields.PREDICATE, new org.apache.thrift.meta_data.FieldMetaData("predicate", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, SlicePredicate.class)));
- tmpMap.put(_Fields.CONSISTENCY_LEVEL, new org.apache.thrift.meta_data.FieldMetaData("consistency_level", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.EnumMetaData(org.apache.thrift.protocol.TType.ENUM, ConsistencyLevel.class)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(multiget_count_args.class, metaDataMap);
- }
-
- public multiget_count_args() {
- this.consistency_level = org.apache.cassandra.thrift.ConsistencyLevel.ONE;
-
- }
-
- public multiget_count_args(
- List<ByteBuffer> keys,
- ColumnParent column_parent,
- SlicePredicate predicate,
- ConsistencyLevel consistency_level)
- {
- this();
- this.keys = keys;
- this.column_parent = column_parent;
- this.predicate = predicate;
- this.consistency_level = consistency_level;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public multiget_count_args(multiget_count_args other) {
- if (other.isSetKeys()) {
- List<ByteBuffer> __this__keys = new ArrayList<ByteBuffer>(other.keys);
- this.keys = __this__keys;
- }
- if (other.isSetColumn_parent()) {
- this.column_parent = new ColumnParent(other.column_parent);
- }
- if (other.isSetPredicate()) {
- this.predicate = new SlicePredicate(other.predicate);
- }
- if (other.isSetConsistency_level()) {
- this.consistency_level = other.consistency_level;
- }
- }
-
- public multiget_count_args deepCopy() {
- return new multiget_count_args(this);
- }
-
- @Override
- public void clear() {
- this.keys = null;
- this.column_parent = null;
- this.predicate = null;
- this.consistency_level = org.apache.cassandra.thrift.ConsistencyLevel.ONE;
-
- }
-
- public int getKeysSize() {
- return (this.keys == null) ? 0 : this.keys.size();
- }
-
- public java.util.Iterator<ByteBuffer> getKeysIterator() {
- return (this.keys == null) ? null : this.keys.iterator();
- }
-
- public void addToKeys(ByteBuffer elem) {
- if (this.keys == null) {
- this.keys = new ArrayList<ByteBuffer>();
- }
- this.keys.add(elem);
- }
-
- public List<ByteBuffer> getKeys() {
- return this.keys;
- }
-
- public multiget_count_args setKeys(List<ByteBuffer> keys) {
- this.keys = keys;
- return this;
- }
-
- public void unsetKeys() {
- this.keys = null;
- }
-
- /** Returns true if field keys is set (has been assigned a value) and false otherwise */
- public boolean isSetKeys() {
- return this.keys != null;
- }
-
- public void setKeysIsSet(boolean value) {
- if (!value) {
- this.keys = null;
- }
- }
-
- public ColumnParent getColumn_parent() {
- return this.column_parent;
- }
-
- public multiget_count_args setColumn_parent(ColumnParent column_parent) {
- this.column_parent = column_parent;
- return this;
- }
-
- public void unsetColumn_parent() {
- this.column_parent = null;
- }
-
- /** Returns true if field column_parent is set (has been assigned a value) and false otherwise */
- public boolean isSetColumn_parent() {
- return this.column_parent != null;
- }
-
- public void setColumn_parentIsSet(boolean value) {
- if (!value) {
- this.column_parent = null;
- }
- }
-
- public SlicePredicate getPredicate() {
- return this.predicate;
- }
-
- public multiget_count_args setPredicate(SlicePredicate predicate) {
- this.predicate = predicate;
- return this;
- }
-
- public void unsetPredicate() {
- this.predicate = null;
- }
-
- /** Returns true if field predicate is set (has been assigned a value) and false otherwise */
- public boolean isSetPredicate() {
- return this.predicate != null;
- }
-
- public void setPredicateIsSet(boolean value) {
- if (!value) {
- this.predicate = null;
- }
- }
-
- /**
- *
- * @see ConsistencyLevel
- */
- public ConsistencyLevel getConsistency_level() {
- return this.consistency_level;
- }
-
- /**
- *
- * @see ConsistencyLevel
- */
- public multiget_count_args setConsistency_level(ConsistencyLevel consistency_level) {
- this.consistency_level = consistency_level;
- return this;
- }
-
- public void unsetConsistency_level() {
- this.consistency_level = null;
- }
-
- /** Returns true if field consistency_level is set (has been assigned a value) and false otherwise */
- public boolean isSetConsistency_level() {
- return this.consistency_level != null;
- }
-
- public void setConsistency_levelIsSet(boolean value) {
- if (!value) {
- this.consistency_level = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case KEYS:
- if (value == null) {
- unsetKeys();
- } else {
- setKeys((List<ByteBuffer>)value);
- }
- break;
-
- case COLUMN_PARENT:
- if (value == null) {
- unsetColumn_parent();
- } else {
- setColumn_parent((ColumnParent)value);
- }
- break;
-
- case PREDICATE:
- if (value == null) {
- unsetPredicate();
- } else {
- setPredicate((SlicePredicate)value);
- }
- break;
-
- case CONSISTENCY_LEVEL:
- if (value == null) {
- unsetConsistency_level();
- } else {
- setConsistency_level((ConsistencyLevel)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case KEYS:
- return getKeys();
-
- case COLUMN_PARENT:
- return getColumn_parent();
-
- case PREDICATE:
- return getPredicate();
-
- case CONSISTENCY_LEVEL:
- return getConsistency_level();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case KEYS:
- return isSetKeys();
- case COLUMN_PARENT:
- return isSetColumn_parent();
- case PREDICATE:
- return isSetPredicate();
- case CONSISTENCY_LEVEL:
- return isSetConsistency_level();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof multiget_count_args)
- return this.equals((multiget_count_args)that);
- return false;
- }
-
- public boolean equals(multiget_count_args that) {
- if (that == null)
- return false;
-
- boolean this_present_keys = true && this.isSetKeys();
- boolean that_present_keys = true && that.isSetKeys();
- if (this_present_keys || that_present_keys) {
- if (!(this_present_keys && that_present_keys))
- return false;
- if (!this.keys.equals(that.keys))
- return false;
- }
-
- boolean this_present_column_parent = true && this.isSetColumn_parent();
- boolean that_present_column_parent = true && that.isSetColumn_parent();
- if (this_present_column_parent || that_present_column_parent) {
- if (!(this_present_column_parent && that_present_column_parent))
- return false;
- if (!this.column_parent.equals(that.column_parent))
- return false;
- }
-
- boolean this_present_predicate = true && this.isSetPredicate();
- boolean that_present_predicate = true && that.isSetPredicate();
- if (this_present_predicate || that_present_predicate) {
- if (!(this_present_predicate && that_present_predicate))
- return false;
- if (!this.predicate.equals(that.predicate))
- return false;
- }
-
- boolean this_present_consistency_level = true && this.isSetConsistency_level();
- boolean that_present_consistency_level = true && that.isSetConsistency_level();
- if (this_present_consistency_level || that_present_consistency_level) {
- if (!(this_present_consistency_level && that_present_consistency_level))
- return false;
- if (!this.consistency_level.equals(that.consistency_level))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_keys = true && (isSetKeys());
- builder.append(present_keys);
- if (present_keys)
- builder.append(keys);
-
- boolean present_column_parent = true && (isSetColumn_parent());
- builder.append(present_column_parent);
- if (present_column_parent)
- builder.append(column_parent);
-
- boolean present_predicate = true && (isSetPredicate());
- builder.append(present_predicate);
- if (present_predicate)
- builder.append(predicate);
-
- boolean present_consistency_level = true && (isSetConsistency_level());
- builder.append(present_consistency_level);
- if (present_consistency_level)
- builder.append(consistency_level.getValue());
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(multiget_count_args other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetKeys()).compareTo(other.isSetKeys());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetKeys()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.keys, other.keys);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetColumn_parent()).compareTo(other.isSetColumn_parent());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetColumn_parent()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.column_parent, other.column_parent);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetPredicate()).compareTo(other.isSetPredicate());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetPredicate()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.predicate, other.predicate);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetConsistency_level()).compareTo(other.isSetConsistency_level());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetConsistency_level()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.consistency_level, other.consistency_level);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("multiget_count_args(");
- boolean first = true;
-
- sb.append("keys:");
- if (this.keys == null) {
- sb.append("null");
- } else {
- sb.append(this.keys);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("column_parent:");
- if (this.column_parent == null) {
- sb.append("null");
- } else {
- sb.append(this.column_parent);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("predicate:");
- if (this.predicate == null) {
- sb.append("null");
- } else {
- sb.append(this.predicate);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("consistency_level:");
- if (this.consistency_level == null) {
- sb.append("null");
- } else {
- sb.append(this.consistency_level);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- if (keys == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'keys' was not present! Struct: " + toString());
- }
- if (column_parent == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'column_parent' was not present! Struct: " + toString());
- }
- if (predicate == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'predicate' was not present! Struct: " + toString());
- }
- if (consistency_level == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'consistency_level' was not present! Struct: " + toString());
- }
- // check for sub-struct validity
- if (column_parent != null) {
- column_parent.validate();
- }
- if (predicate != null) {
- predicate.validate();
- }
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class multiget_count_argsStandardSchemeFactory implements SchemeFactory {
- public multiget_count_argsStandardScheme getScheme() {
- return new multiget_count_argsStandardScheme();
- }
- }
-
- private static class multiget_count_argsStandardScheme extends StandardScheme<multiget_count_args> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, multiget_count_args struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // KEYS
- if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
- {
- org.apache.thrift.protocol.TList _list258 = iprot.readListBegin();
- struct.keys = new ArrayList<ByteBuffer>(_list258.size);
- for (int _i259 = 0; _i259 < _list258.size; ++_i259)
- {
- ByteBuffer _elem260;
- _elem260 = iprot.readBinary();
- struct.keys.add(_elem260);
- }
- iprot.readListEnd();
- }
- struct.setKeysIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 2: // COLUMN_PARENT
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.column_parent = new ColumnParent();
- struct.column_parent.read(iprot);
- struct.setColumn_parentIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 3: // PREDICATE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.predicate = new SlicePredicate();
- struct.predicate.read(iprot);
- struct.setPredicateIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 4: // CONSISTENCY_LEVEL
- if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
- struct.consistency_level = ConsistencyLevel.findByValue(iprot.readI32());
- struct.setConsistency_levelIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, multiget_count_args struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.keys != null) {
- oprot.writeFieldBegin(KEYS_FIELD_DESC);
- {
- oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, struct.keys.size()));
- for (ByteBuffer _iter261 : struct.keys)
- {
- oprot.writeBinary(_iter261);
- }
- oprot.writeListEnd();
- }
- oprot.writeFieldEnd();
- }
- if (struct.column_parent != null) {
- oprot.writeFieldBegin(COLUMN_PARENT_FIELD_DESC);
- struct.column_parent.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.predicate != null) {
- oprot.writeFieldBegin(PREDICATE_FIELD_DESC);
- struct.predicate.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.consistency_level != null) {
- oprot.writeFieldBegin(CONSISTENCY_LEVEL_FIELD_DESC);
- oprot.writeI32(struct.consistency_level.getValue());
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class multiget_count_argsTupleSchemeFactory implements SchemeFactory {
- public multiget_count_argsTupleScheme getScheme() {
- return new multiget_count_argsTupleScheme();
- }
- }
-
- private static class multiget_count_argsTupleScheme extends TupleScheme<multiget_count_args> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, multiget_count_args struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- {
- oprot.writeI32(struct.keys.size());
- for (ByteBuffer _iter262 : struct.keys)
- {
- oprot.writeBinary(_iter262);
- }
- }
- struct.column_parent.write(oprot);
- struct.predicate.write(oprot);
- oprot.writeI32(struct.consistency_level.getValue());
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, multiget_count_args struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- {
- org.apache.thrift.protocol.TList _list263 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32());
- struct.keys = new ArrayList<ByteBuffer>(_list263.size);
- for (int _i264 = 0; _i264 < _list263.size; ++_i264)
- {
- ByteBuffer _elem265;
- _elem265 = iprot.readBinary();
- struct.keys.add(_elem265);
- }
- }
- struct.setKeysIsSet(true);
- struct.column_parent = new ColumnParent();
- struct.column_parent.read(iprot);
- struct.setColumn_parentIsSet(true);
- struct.predicate = new SlicePredicate();
- struct.predicate.read(iprot);
- struct.setPredicateIsSet(true);
- struct.consistency_level = ConsistencyLevel.findByValue(iprot.readI32());
- struct.setConsistency_levelIsSet(true);
- }
- }
-
- }
-
- public static class multiget_count_result implements org.apache.thrift.TBase<multiget_count_result, multiget_count_result._Fields>, java.io.Serializable, Cloneable, Comparable<multiget_count_result> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("multiget_count_result");
-
- private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.MAP, (short)0);
- private static final org.apache.thrift.protocol.TField IRE_FIELD_DESC = new org.apache.thrift.protocol.TField("ire", org.apache.thrift.protocol.TType.STRUCT, (short)1);
- private static final org.apache.thrift.protocol.TField UE_FIELD_DESC = new org.apache.thrift.protocol.TField("ue", org.apache.thrift.protocol.TType.STRUCT, (short)2);
- private static final org.apache.thrift.protocol.TField TE_FIELD_DESC = new org.apache.thrift.protocol.TField("te", org.apache.thrift.protocol.TType.STRUCT, (short)3);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new multiget_count_resultStandardSchemeFactory());
- schemes.put(TupleScheme.class, new multiget_count_resultTupleSchemeFactory());
- }
-
- public Map<ByteBuffer,Integer> success; // required
- public InvalidRequestException ire; // required
- public UnavailableException ue; // required
- public TimedOutException te; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- SUCCESS((short)0, "success"),
- IRE((short)1, "ire"),
- UE((short)2, "ue"),
- TE((short)3, "te");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 0: // SUCCESS
- return SUCCESS;
- case 1: // IRE
- return IRE;
- case 2: // UE
- return UE;
- case 3: // TE
- return TE;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true),
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32))));
- tmpMap.put(_Fields.IRE, new org.apache.thrift.meta_data.FieldMetaData("ire", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- tmpMap.put(_Fields.UE, new org.apache.thrift.meta_data.FieldMetaData("ue", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- tmpMap.put(_Fields.TE, new org.apache.thrift.meta_data.FieldMetaData("te", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(multiget_count_result.class, metaDataMap);
- }
-
- public multiget_count_result() {
- }
-
- public multiget_count_result(
- Map<ByteBuffer,Integer> success,
- InvalidRequestException ire,
- UnavailableException ue,
- TimedOutException te)
- {
- this();
- this.success = success;
- this.ire = ire;
- this.ue = ue;
- this.te = te;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public multiget_count_result(multiget_count_result other) {
- if (other.isSetSuccess()) {
- Map<ByteBuffer,Integer> __this__success = new HashMap<ByteBuffer,Integer>(other.success);
- this.success = __this__success;
- }
- if (other.isSetIre()) {
- this.ire = new InvalidRequestException(other.ire);
- }
- if (other.isSetUe()) {
- this.ue = new UnavailableException(other.ue);
- }
- if (other.isSetTe()) {
- this.te = new TimedOutException(other.te);
- }
- }
-
- public multiget_count_result deepCopy() {
- return new multiget_count_result(this);
- }
-
- @Override
- public void clear() {
- this.success = null;
- this.ire = null;
- this.ue = null;
- this.te = null;
- }
-
- public int getSuccessSize() {
- return (this.success == null) ? 0 : this.success.size();
- }
-
- public void putToSuccess(ByteBuffer key, int val) {
- if (this.success == null) {
- this.success = new HashMap<ByteBuffer,Integer>();
- }
- this.success.put(key, val);
- }
-
- public Map<ByteBuffer,Integer> getSuccess() {
- return this.success;
- }
-
- public multiget_count_result setSuccess(Map<ByteBuffer,Integer> success) {
- this.success = success;
- return this;
- }
-
- public void unsetSuccess() {
- this.success = null;
- }
-
- /** Returns true if field success is set (has been assigned a value) and false otherwise */
- public boolean isSetSuccess() {
- return this.success != null;
- }
-
- public void setSuccessIsSet(boolean value) {
- if (!value) {
- this.success = null;
- }
- }
-
- public InvalidRequestException getIre() {
- return this.ire;
- }
-
- public multiget_count_result setIre(InvalidRequestException ire) {
- this.ire = ire;
- return this;
- }
-
- public void unsetIre() {
- this.ire = null;
- }
-
- /** Returns true if field ire is set (has been assigned a value) and false otherwise */
- public boolean isSetIre() {
- return this.ire != null;
- }
-
- public void setIreIsSet(boolean value) {
- if (!value) {
- this.ire = null;
- }
- }
-
- public UnavailableException getUe() {
- return this.ue;
- }
-
- public multiget_count_result setUe(UnavailableException ue) {
- this.ue = ue;
- return this;
- }
-
- public void unsetUe() {
- this.ue = null;
- }
-
- /** Returns true if field ue is set (has been assigned a value) and false otherwise */
- public boolean isSetUe() {
- return this.ue != null;
- }
-
- public void setUeIsSet(boolean value) {
- if (!value) {
- this.ue = null;
- }
- }
-
- public TimedOutException getTe() {
- return this.te;
- }
-
- public multiget_count_result setTe(TimedOutException te) {
- this.te = te;
- return this;
- }
-
- public void unsetTe() {
- this.te = null;
- }
-
- /** Returns true if field te is set (has been assigned a value) and false otherwise */
- public boolean isSetTe() {
- return this.te != null;
- }
-
- public void setTeIsSet(boolean value) {
- if (!value) {
- this.te = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case SUCCESS:
- if (value == null) {
- unsetSuccess();
- } else {
- setSuccess((Map<ByteBuffer,Integer>)value);
- }
- break;
-
- case IRE:
- if (value == null) {
- unsetIre();
- } else {
- setIre((InvalidRequestException)value);
- }
- break;
-
- case UE:
- if (value == null) {
- unsetUe();
- } else {
- setUe((UnavailableException)value);
- }
- break;
-
- case TE:
- if (value == null) {
- unsetTe();
- } else {
- setTe((TimedOutException)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case SUCCESS:
- return getSuccess();
-
- case IRE:
- return getIre();
-
- case UE:
- return getUe();
-
- case TE:
- return getTe();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case SUCCESS:
- return isSetSuccess();
- case IRE:
- return isSetIre();
- case UE:
- return isSetUe();
- case TE:
- return isSetTe();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof multiget_count_result)
- return this.equals((multiget_count_result)that);
- return false;
- }
-
- public boolean equals(multiget_count_result that) {
- if (that == null)
- return false;
-
- boolean this_present_success = true && this.isSetSuccess();
- boolean that_present_success = true && that.isSetSuccess();
- if (this_present_success || that_present_success) {
- if (!(this_present_success && that_present_success))
- return false;
- if (!this.success.equals(that.success))
- return false;
- }
-
- boolean this_present_ire = true && this.isSetIre();
- boolean that_present_ire = true && that.isSetIre();
- if (this_present_ire || that_present_ire) {
- if (!(this_present_ire && that_present_ire))
- return false;
- if (!this.ire.equals(that.ire))
- return false;
- }
-
- boolean this_present_ue = true && this.isSetUe();
- boolean that_present_ue = true && that.isSetUe();
- if (this_present_ue || that_present_ue) {
- if (!(this_present_ue && that_present_ue))
- return false;
- if (!this.ue.equals(that.ue))
- return false;
- }
-
- boolean this_present_te = true && this.isSetTe();
- boolean that_present_te = true && that.isSetTe();
- if (this_present_te || that_present_te) {
- if (!(this_present_te && that_present_te))
- return false;
- if (!this.te.equals(that.te))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_success = true && (isSetSuccess());
- builder.append(present_success);
- if (present_success)
- builder.append(success);
-
- boolean present_ire = true && (isSetIre());
- builder.append(present_ire);
- if (present_ire)
- builder.append(ire);
-
- boolean present_ue = true && (isSetUe());
- builder.append(present_ue);
- if (present_ue)
- builder.append(ue);
-
- boolean present_te = true && (isSetTe());
- builder.append(present_te);
- if (present_te)
- builder.append(te);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(multiget_count_result other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(other.isSetSuccess());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetSuccess()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, other.success);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetIre()).compareTo(other.isSetIre());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetIre()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.ire, other.ire);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetUe()).compareTo(other.isSetUe());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetUe()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.ue, other.ue);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetTe()).compareTo(other.isSetTe());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetTe()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.te, other.te);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("multiget_count_result(");
- boolean first = true;
-
- sb.append("success:");
- if (this.success == null) {
- sb.append("null");
- } else {
- sb.append(this.success);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("ire:");
- if (this.ire == null) {
- sb.append("null");
- } else {
- sb.append(this.ire);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("ue:");
- if (this.ue == null) {
- sb.append("null");
- } else {
- sb.append(this.ue);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("te:");
- if (this.te == null) {
- sb.append("null");
- } else {
- sb.append(this.te);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class multiget_count_resultStandardSchemeFactory implements SchemeFactory {
- public multiget_count_resultStandardScheme getScheme() {
- return new multiget_count_resultStandardScheme();
- }
- }
-
- private static class multiget_count_resultStandardScheme extends StandardScheme<multiget_count_result> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, multiget_count_result struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 0: // SUCCESS
- if (schemeField.type == org.apache.thrift.protocol.TType.MAP) {
- {
- org.apache.thrift.protocol.TMap _map266 = iprot.readMapBegin();
- struct.success = new HashMap<ByteBuffer,Integer>(2*_map266.size);
- for (int _i267 = 0; _i267 < _map266.size; ++_i267)
- {
- ByteBuffer _key268;
- int _val269;
- _key268 = iprot.readBinary();
- _val269 = iprot.readI32();
- struct.success.put(_key268, _val269);
- }
- iprot.readMapEnd();
- }
- struct.setSuccessIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 1: // IRE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.ire = new InvalidRequestException();
- struct.ire.read(iprot);
- struct.setIreIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 2: // UE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.ue = new UnavailableException();
- struct.ue.read(iprot);
- struct.setUeIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 3: // TE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.te = new TimedOutException();
- struct.te.read(iprot);
- struct.setTeIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, multiget_count_result struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.success != null) {
- oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
- {
- oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.I32, struct.success.size()));
- for (Map.Entry<ByteBuffer, Integer> _iter270 : struct.success.entrySet())
- {
- oprot.writeBinary(_iter270.getKey());
- oprot.writeI32(_iter270.getValue());
- }
- oprot.writeMapEnd();
- }
- oprot.writeFieldEnd();
- }
- if (struct.ire != null) {
- oprot.writeFieldBegin(IRE_FIELD_DESC);
- struct.ire.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.ue != null) {
- oprot.writeFieldBegin(UE_FIELD_DESC);
- struct.ue.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.te != null) {
- oprot.writeFieldBegin(TE_FIELD_DESC);
- struct.te.write(oprot);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class multiget_count_resultTupleSchemeFactory implements SchemeFactory {
- public multiget_count_resultTupleScheme getScheme() {
- return new multiget_count_resultTupleScheme();
- }
- }
-
- private static class multiget_count_resultTupleScheme extends TupleScheme<multiget_count_result> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, multiget_count_result struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- BitSet optionals = new BitSet();
- if (struct.isSetSuccess()) {
- optionals.set(0);
- }
- if (struct.isSetIre()) {
- optionals.set(1);
- }
- if (struct.isSetUe()) {
- optionals.set(2);
- }
- if (struct.isSetTe()) {
- optionals.set(3);
- }
- oprot.writeBitSet(optionals, 4);
- if (struct.isSetSuccess()) {
- {
- oprot.writeI32(struct.success.size());
- for (Map.Entry<ByteBuffer, Integer> _iter271 : struct.success.entrySet())
- {
- oprot.writeBinary(_iter271.getKey());
- oprot.writeI32(_iter271.getValue());
- }
- }
- }
- if (struct.isSetIre()) {
- struct.ire.write(oprot);
- }
- if (struct.isSetUe()) {
- struct.ue.write(oprot);
- }
- if (struct.isSetTe()) {
- struct.te.write(oprot);
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, multiget_count_result struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- BitSet incoming = iprot.readBitSet(4);
- if (incoming.get(0)) {
- {
- org.apache.thrift.protocol.TMap _map272 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.I32, iprot.readI32());
- struct.success = new HashMap<ByteBuffer,Integer>(2*_map272.size);
- for (int _i273 = 0; _i273 < _map272.size; ++_i273)
- {
- ByteBuffer _key274;
- int _val275;
- _key274 = iprot.readBinary();
- _val275 = iprot.readI32();
- struct.success.put(_key274, _val275);
- }
- }
- struct.setSuccessIsSet(true);
- }
- if (incoming.get(1)) {
- struct.ire = new InvalidRequestException();
- struct.ire.read(iprot);
- struct.setIreIsSet(true);
- }
- if (incoming.get(2)) {
- struct.ue = new UnavailableException();
- struct.ue.read(iprot);
- struct.setUeIsSet(true);
- }
- if (incoming.get(3)) {
- struct.te = new TimedOutException();
- struct.te.read(iprot);
- struct.setTeIsSet(true);
- }
- }
- }
-
- }
-
- public static class get_range_slices_args implements org.apache.thrift.TBase<get_range_slices_args, get_range_slices_args._Fields>, java.io.Serializable, Cloneable, Comparable<get_range_slices_args> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("get_range_slices_args");
-
- private static final org.apache.thrift.protocol.TField COLUMN_PARENT_FIELD_DESC = new org.apache.thrift.protocol.TField("column_parent", org.apache.thrift.protocol.TType.STRUCT, (short)1);
- private static final org.apache.thrift.protocol.TField PREDICATE_FIELD_DESC = new org.apache.thrift.protocol.TField("predicate", org.apache.thrift.protocol.TType.STRUCT, (short)2);
- private static final org.apache.thrift.protocol.TField RANGE_FIELD_DESC = new org.apache.thrift.protocol.TField("range", org.apache.thrift.protocol.TType.STRUCT, (short)3);
- private static final org.apache.thrift.protocol.TField CONSISTENCY_LEVEL_FIELD_DESC = new org.apache.thrift.protocol.TField("consistency_level", org.apache.thrift.protocol.TType.I32, (short)4);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new get_range_slices_argsStandardSchemeFactory());
- schemes.put(TupleScheme.class, new get_range_slices_argsTupleSchemeFactory());
- }
-
- public ColumnParent column_parent; // required
- public SlicePredicate predicate; // required
- public KeyRange range; // required
- /**
- *
- * @see ConsistencyLevel
- */
- public ConsistencyLevel consistency_level; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- COLUMN_PARENT((short)1, "column_parent"),
- PREDICATE((short)2, "predicate"),
- RANGE((short)3, "range"),
- /**
- *
- * @see ConsistencyLevel
- */
- CONSISTENCY_LEVEL((short)4, "consistency_level");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // COLUMN_PARENT
- return COLUMN_PARENT;
- case 2: // PREDICATE
- return PREDICATE;
- case 3: // RANGE
- return RANGE;
- case 4: // CONSISTENCY_LEVEL
- return CONSISTENCY_LEVEL;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.COLUMN_PARENT, new org.apache.thrift.meta_data.FieldMetaData("column_parent", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, ColumnParent.class)));
- tmpMap.put(_Fields.PREDICATE, new org.apache.thrift.meta_data.FieldMetaData("predicate", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, SlicePredicate.class)));
- tmpMap.put(_Fields.RANGE, new org.apache.thrift.meta_data.FieldMetaData("range", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, KeyRange.class)));
- tmpMap.put(_Fields.CONSISTENCY_LEVEL, new org.apache.thrift.meta_data.FieldMetaData("consistency_level", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.EnumMetaData(org.apache.thrift.protocol.TType.ENUM, ConsistencyLevel.class)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(get_range_slices_args.class, metaDataMap);
- }
-
- public get_range_slices_args() {
- this.consistency_level = org.apache.cassandra.thrift.ConsistencyLevel.ONE;
-
- }
-
- public get_range_slices_args(
- ColumnParent column_parent,
- SlicePredicate predicate,
- KeyRange range,
- ConsistencyLevel consistency_level)
- {
- this();
- this.column_parent = column_parent;
- this.predicate = predicate;
- this.range = range;
- this.consistency_level = consistency_level;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public get_range_slices_args(get_range_slices_args other) {
- if (other.isSetColumn_parent()) {
- this.column_parent = new ColumnParent(other.column_parent);
- }
- if (other.isSetPredicate()) {
- this.predicate = new SlicePredicate(other.predicate);
- }
- if (other.isSetRange()) {
- this.range = new KeyRange(other.range);
- }
- if (other.isSetConsistency_level()) {
- this.consistency_level = other.consistency_level;
- }
- }
-
- public get_range_slices_args deepCopy() {
- return new get_range_slices_args(this);
- }
-
- @Override
- public void clear() {
- this.column_parent = null;
- this.predicate = null;
- this.range = null;
- this.consistency_level = org.apache.cassandra.thrift.ConsistencyLevel.ONE;
-
- }
-
- public ColumnParent getColumn_parent() {
- return this.column_parent;
- }
-
- public get_range_slices_args setColumn_parent(ColumnParent column_parent) {
- this.column_parent = column_parent;
- return this;
- }
-
- public void unsetColumn_parent() {
- this.column_parent = null;
- }
-
- /** Returns true if field column_parent is set (has been assigned a value) and false otherwise */
- public boolean isSetColumn_parent() {
- return this.column_parent != null;
- }
-
- public void setColumn_parentIsSet(boolean value) {
- if (!value) {
- this.column_parent = null;
- }
- }
-
- public SlicePredicate getPredicate() {
- return this.predicate;
- }
-
- public get_range_slices_args setPredicate(SlicePredicate predicate) {
- this.predicate = predicate;
- return this;
- }
-
- public void unsetPredicate() {
- this.predicate = null;
- }
-
- /** Returns true if field predicate is set (has been assigned a value) and false otherwise */
- public boolean isSetPredicate() {
- return this.predicate != null;
- }
-
- public void setPredicateIsSet(boolean value) {
- if (!value) {
- this.predicate = null;
- }
- }
-
- public KeyRange getRange() {
- return this.range;
- }
-
- public get_range_slices_args setRange(KeyRange range) {
- this.range = range;
- return this;
- }
-
- public void unsetRange() {
- this.range = null;
- }
-
- /** Returns true if field range is set (has been assigned a value) and false otherwise */
- public boolean isSetRange() {
- return this.range != null;
- }
-
- public void setRangeIsSet(boolean value) {
- if (!value) {
- this.range = null;
- }
- }
-
- /**
- *
- * @see ConsistencyLevel
- */
- public ConsistencyLevel getConsistency_level() {
- return this.consistency_level;
- }
-
- /**
- *
- * @see ConsistencyLevel
- */
- public get_range_slices_args setConsistency_level(ConsistencyLevel consistency_level) {
- this.consistency_level = consistency_level;
- return this;
- }
-
- public void unsetConsistency_level() {
- this.consistency_level = null;
- }
-
- /** Returns true if field consistency_level is set (has been assigned a value) and false otherwise */
- public boolean isSetConsistency_level() {
- return this.consistency_level != null;
- }
-
- public void setConsistency_levelIsSet(boolean value) {
- if (!value) {
- this.consistency_level = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case COLUMN_PARENT:
- if (value == null) {
- unsetColumn_parent();
- } else {
- setColumn_parent((ColumnParent)value);
- }
- break;
-
- case PREDICATE:
- if (value == null) {
- unsetPredicate();
- } else {
- setPredicate((SlicePredicate)value);
- }
- break;
-
- case RANGE:
- if (value == null) {
- unsetRange();
- } else {
- setRange((KeyRange)value);
- }
- break;
-
- case CONSISTENCY_LEVEL:
- if (value == null) {
- unsetConsistency_level();
- } else {
- setConsistency_level((ConsistencyLevel)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case COLUMN_PARENT:
- return getColumn_parent();
-
- case PREDICATE:
- return getPredicate();
-
- case RANGE:
- return getRange();
-
- case CONSISTENCY_LEVEL:
- return getConsistency_level();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case COLUMN_PARENT:
- return isSetColumn_parent();
- case PREDICATE:
- return isSetPredicate();
- case RANGE:
- return isSetRange();
- case CONSISTENCY_LEVEL:
- return isSetConsistency_level();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof get_range_slices_args)
- return this.equals((get_range_slices_args)that);
- return false;
- }
-
- public boolean equals(get_range_slices_args that) {
- if (that == null)
- return false;
-
- boolean this_present_column_parent = true && this.isSetColumn_parent();
- boolean that_present_column_parent = true && that.isSetColumn_parent();
- if (this_present_column_parent || that_present_column_parent) {
- if (!(this_present_column_parent && that_present_column_parent))
- return false;
- if (!this.column_parent.equals(that.column_parent))
- return false;
- }
-
- boolean this_present_predicate = true && this.isSetPredicate();
- boolean that_present_predicate = true && that.isSetPredicate();
- if (this_present_predicate || that_present_predicate) {
- if (!(this_present_predicate && that_present_predicate))
- return false;
- if (!this.predicate.equals(that.predicate))
- return false;
- }
-
- boolean this_present_range = true && this.isSetRange();
- boolean that_present_range = true && that.isSetRange();
- if (this_present_range || that_present_range) {
- if (!(this_present_range && that_present_range))
- return false;
- if (!this.range.equals(that.range))
- return false;
- }
-
- boolean this_present_consistency_level = true && this.isSetConsistency_level();
- boolean that_present_consistency_level = true && that.isSetConsistency_level();
- if (this_present_consistency_level || that_present_consistency_level) {
- if (!(this_present_consistency_level && that_present_consistency_level))
- return false;
- if (!this.consistency_level.equals(that.consistency_level))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_column_parent = true && (isSetColumn_parent());
- builder.append(present_column_parent);
- if (present_column_parent)
- builder.append(column_parent);
-
- boolean present_predicate = true && (isSetPredicate());
- builder.append(present_predicate);
- if (present_predicate)
- builder.append(predicate);
-
- boolean present_range = true && (isSetRange());
- builder.append(present_range);
- if (present_range)
- builder.append(range);
-
- boolean present_consistency_level = true && (isSetConsistency_level());
- builder.append(present_consistency_level);
- if (present_consistency_level)
- builder.append(consistency_level.getValue());
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(get_range_slices_args other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetColumn_parent()).compareTo(other.isSetColumn_parent());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetColumn_parent()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.column_parent, other.column_parent);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetPredicate()).compareTo(other.isSetPredicate());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetPredicate()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.predicate, other.predicate);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetRange()).compareTo(other.isSetRange());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetRange()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.range, other.range);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetConsistency_level()).compareTo(other.isSetConsistency_level());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetConsistency_level()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.consistency_level, other.consistency_level);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("get_range_slices_args(");
- boolean first = true;
-
- sb.append("column_parent:");
- if (this.column_parent == null) {
- sb.append("null");
- } else {
- sb.append(this.column_parent);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("predicate:");
- if (this.predicate == null) {
- sb.append("null");
- } else {
- sb.append(this.predicate);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("range:");
- if (this.range == null) {
- sb.append("null");
- } else {
- sb.append(this.range);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("consistency_level:");
- if (this.consistency_level == null) {
- sb.append("null");
- } else {
- sb.append(this.consistency_level);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- if (column_parent == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'column_parent' was not present! Struct: " + toString());
- }
- if (predicate == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'predicate' was not present! Struct: " + toString());
- }
- if (range == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'range' was not present! Struct: " + toString());
- }
- if (consistency_level == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'consistency_level' was not present! Struct: " + toString());
- }
- // check for sub-struct validity
- if (column_parent != null) {
- column_parent.validate();
- }
- if (predicate != null) {
- predicate.validate();
- }
- if (range != null) {
- range.validate();
- }
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class get_range_slices_argsStandardSchemeFactory implements SchemeFactory {
- public get_range_slices_argsStandardScheme getScheme() {
- return new get_range_slices_argsStandardScheme();
- }
- }
-
- private static class get_range_slices_argsStandardScheme extends StandardScheme<get_range_slices_args> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, get_range_slices_args struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // COLUMN_PARENT
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.column_parent = new ColumnParent();
- struct.column_parent.read(iprot);
- struct.setColumn_parentIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 2: // PREDICATE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.predicate = new SlicePredicate();
- struct.predicate.read(iprot);
- struct.setPredicateIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 3: // RANGE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.range = new KeyRange();
- struct.range.read(iprot);
- struct.setRangeIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 4: // CONSISTENCY_LEVEL
- if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
- struct.consistency_level = ConsistencyLevel.findByValue(iprot.readI32());
- struct.setConsistency_levelIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, get_range_slices_args struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.column_parent != null) {
- oprot.writeFieldBegin(COLUMN_PARENT_FIELD_DESC);
- struct.column_parent.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.predicate != null) {
- oprot.writeFieldBegin(PREDICATE_FIELD_DESC);
- struct.predicate.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.range != null) {
- oprot.writeFieldBegin(RANGE_FIELD_DESC);
- struct.range.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.consistency_level != null) {
- oprot.writeFieldBegin(CONSISTENCY_LEVEL_FIELD_DESC);
- oprot.writeI32(struct.consistency_level.getValue());
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class get_range_slices_argsTupleSchemeFactory implements SchemeFactory {
- public get_range_slices_argsTupleScheme getScheme() {
- return new get_range_slices_argsTupleScheme();
- }
- }
-
- private static class get_range_slices_argsTupleScheme extends TupleScheme<get_range_slices_args> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, get_range_slices_args struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- struct.column_parent.write(oprot);
- struct.predicate.write(oprot);
- struct.range.write(oprot);
- oprot.writeI32(struct.consistency_level.getValue());
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, get_range_slices_args struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- struct.column_parent = new ColumnParent();
- struct.column_parent.read(iprot);
- struct.setColumn_parentIsSet(true);
- struct.predicate = new SlicePredicate();
- struct.predicate.read(iprot);
- struct.setPredicateIsSet(true);
- struct.range = new KeyRange();
- struct.range.read(iprot);
- struct.setRangeIsSet(true);
- struct.consistency_level = ConsistencyLevel.findByValue(iprot.readI32());
- struct.setConsistency_levelIsSet(true);
- }
- }
-
- }
-
- public static class get_range_slices_result implements org.apache.thrift.TBase<get_range_slices_result, get_range_slices_result._Fields>, java.io.Serializable, Cloneable, Comparable<get_range_slices_result> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("get_range_slices_result");
-
- private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.LIST, (short)0);
- private static final org.apache.thrift.protocol.TField IRE_FIELD_DESC = new org.apache.thrift.protocol.TField("ire", org.apache.thrift.protocol.TType.STRUCT, (short)1);
- private static final org.apache.thrift.protocol.TField UE_FIELD_DESC = new org.apache.thrift.protocol.TField("ue", org.apache.thrift.protocol.TType.STRUCT, (short)2);
- private static final org.apache.thrift.protocol.TField TE_FIELD_DESC = new org.apache.thrift.protocol.TField("te", org.apache.thrift.protocol.TType.STRUCT, (short)3);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new get_range_slices_resultStandardSchemeFactory());
- schemes.put(TupleScheme.class, new get_range_slices_resultTupleSchemeFactory());
- }
-
- public List<KeySlice> success; // required
- public InvalidRequestException ire; // required
- public UnavailableException ue; // required
- public TimedOutException te; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- SUCCESS((short)0, "success"),
- IRE((short)1, "ire"),
- UE((short)2, "ue"),
- TE((short)3, "te");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 0: // SUCCESS
- return SUCCESS;
- case 1: // IRE
- return IRE;
- case 2: // UE
- return UE;
- case 3: // TE
- return TE;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST,
- new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, KeySlice.class))));
- tmpMap.put(_Fields.IRE, new org.apache.thrift.meta_data.FieldMetaData("ire", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- tmpMap.put(_Fields.UE, new org.apache.thrift.meta_data.FieldMetaData("ue", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- tmpMap.put(_Fields.TE, new org.apache.thrift.meta_data.FieldMetaData("te", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(get_range_slices_result.class, metaDataMap);
- }
-
- public get_range_slices_result() {
- }
-
- public get_range_slices_result(
- List<KeySlice> success,
- InvalidRequestException ire,
- UnavailableException ue,
- TimedOutException te)
- {
- this();
- this.success = success;
- this.ire = ire;
- this.ue = ue;
- this.te = te;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public get_range_slices_result(get_range_slices_result other) {
- if (other.isSetSuccess()) {
- List<KeySlice> __this__success = new ArrayList<KeySlice>(other.success.size());
- for (KeySlice other_element : other.success) {
- __this__success.add(new KeySlice(other_element));
- }
- this.success = __this__success;
- }
- if (other.isSetIre()) {
- this.ire = new InvalidRequestException(other.ire);
- }
- if (other.isSetUe()) {
- this.ue = new UnavailableException(other.ue);
- }
- if (other.isSetTe()) {
- this.te = new TimedOutException(other.te);
- }
- }
-
- public get_range_slices_result deepCopy() {
- return new get_range_slices_result(this);
- }
-
- @Override
- public void clear() {
- this.success = null;
- this.ire = null;
- this.ue = null;
- this.te = null;
- }
-
- public int getSuccessSize() {
- return (this.success == null) ? 0 : this.success.size();
- }
-
- public java.util.Iterator<KeySlice> getSuccessIterator() {
- return (this.success == null) ? null : this.success.iterator();
- }
-
- public void addToSuccess(KeySlice elem) {
- if (this.success == null) {
- this.success = new ArrayList<KeySlice>();
- }
- this.success.add(elem);
- }
-
- public List<KeySlice> getSuccess() {
- return this.success;
- }
-
- public get_range_slices_result setSuccess(List<KeySlice> success) {
- this.success = success;
- return this;
- }
-
- public void unsetSuccess() {
- this.success = null;
- }
-
- /** Returns true if field success is set (has been assigned a value) and false otherwise */
- public boolean isSetSuccess() {
- return this.success != null;
- }
-
- public void setSuccessIsSet(boolean value) {
- if (!value) {
- this.success = null;
- }
- }
-
- public InvalidRequestException getIre() {
- return this.ire;
- }
-
- public get_range_slices_result setIre(InvalidRequestException ire) {
- this.ire = ire;
- return this;
- }
-
- public void unsetIre() {
- this.ire = null;
- }
-
- /** Returns true if field ire is set (has been assigned a value) and false otherwise */
- public boolean isSetIre() {
- return this.ire != null;
- }
-
- public void setIreIsSet(boolean value) {
- if (!value) {
- this.ire = null;
- }
- }
-
- public UnavailableException getUe() {
- return this.ue;
- }
-
- public get_range_slices_result setUe(UnavailableException ue) {
- this.ue = ue;
- return this;
- }
-
- public void unsetUe() {
- this.ue = null;
- }
-
- /** Returns true if field ue is set (has been assigned a value) and false otherwise */
- public boolean isSetUe() {
- return this.ue != null;
- }
-
- public void setUeIsSet(boolean value) {
- if (!value) {
- this.ue = null;
- }
- }
-
- public TimedOutException getTe() {
- return this.te;
- }
-
- public get_range_slices_result setTe(TimedOutException te) {
- this.te = te;
- return this;
- }
-
- public void unsetTe() {
- this.te = null;
- }
-
- /** Returns true if field te is set (has been assigned a value) and false otherwise */
- public boolean isSetTe() {
- return this.te != null;
- }
-
- public void setTeIsSet(boolean value) {
- if (!value) {
- this.te = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case SUCCESS:
- if (value == null) {
- unsetSuccess();
- } else {
- setSuccess((List<KeySlice>)value);
- }
- break;
-
- case IRE:
- if (value == null) {
- unsetIre();
- } else {
- setIre((InvalidRequestException)value);
- }
- break;
-
- case UE:
- if (value == null) {
- unsetUe();
- } else {
- setUe((UnavailableException)value);
- }
- break;
-
- case TE:
- if (value == null) {
- unsetTe();
- } else {
- setTe((TimedOutException)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case SUCCESS:
- return getSuccess();
-
- case IRE:
- return getIre();
-
- case UE:
- return getUe();
-
- case TE:
- return getTe();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case SUCCESS:
- return isSetSuccess();
- case IRE:
- return isSetIre();
- case UE:
- return isSetUe();
- case TE:
- return isSetTe();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof get_range_slices_result)
- return this.equals((get_range_slices_result)that);
- return false;
- }
-
- public boolean equals(get_range_slices_result that) {
- if (that == null)
- return false;
-
- boolean this_present_success = true && this.isSetSuccess();
- boolean that_present_success = true && that.isSetSuccess();
- if (this_present_success || that_present_success) {
- if (!(this_present_success && that_present_success))
- return false;
- if (!this.success.equals(that.success))
- return false;
- }
-
- boolean this_present_ire = true && this.isSetIre();
- boolean that_present_ire = true && that.isSetIre();
- if (this_present_ire || that_present_ire) {
- if (!(this_present_ire && that_present_ire))
- return false;
- if (!this.ire.equals(that.ire))
- return false;
- }
-
- boolean this_present_ue = true && this.isSetUe();
- boolean that_present_ue = true && that.isSetUe();
- if (this_present_ue || that_present_ue) {
- if (!(this_present_ue && that_present_ue))
- return false;
- if (!this.ue.equals(that.ue))
- return false;
- }
-
- boolean this_present_te = true && this.isSetTe();
- boolean that_present_te = true && that.isSetTe();
- if (this_present_te || that_present_te) {
- if (!(this_present_te && that_present_te))
- return false;
- if (!this.te.equals(that.te))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_success = true && (isSetSuccess());
- builder.append(present_success);
- if (present_success)
- builder.append(success);
-
- boolean present_ire = true && (isSetIre());
- builder.append(present_ire);
- if (present_ire)
- builder.append(ire);
-
- boolean present_ue = true && (isSetUe());
- builder.append(present_ue);
- if (present_ue)
- builder.append(ue);
-
- boolean present_te = true && (isSetTe());
- builder.append(present_te);
- if (present_te)
- builder.append(te);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(get_range_slices_result other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(other.isSetSuccess());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetSuccess()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, other.success);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetIre()).compareTo(other.isSetIre());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetIre()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.ire, other.ire);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetUe()).compareTo(other.isSetUe());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetUe()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.ue, other.ue);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetTe()).compareTo(other.isSetTe());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetTe()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.te, other.te);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("get_range_slices_result(");
- boolean first = true;
-
- sb.append("success:");
- if (this.success == null) {
- sb.append("null");
- } else {
- sb.append(this.success);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("ire:");
- if (this.ire == null) {
- sb.append("null");
- } else {
- sb.append(this.ire);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("ue:");
- if (this.ue == null) {
- sb.append("null");
- } else {
- sb.append(this.ue);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("te:");
- if (this.te == null) {
- sb.append("null");
- } else {
- sb.append(this.te);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class get_range_slices_resultStandardSchemeFactory implements SchemeFactory {
- public get_range_slices_resultStandardScheme getScheme() {
- return new get_range_slices_resultStandardScheme();
- }
- }
-
- private static class get_range_slices_resultStandardScheme extends StandardScheme<get_range_slices_result> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, get_range_slices_result struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 0: // SUCCESS
- if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
- {
- org.apache.thrift.protocol.TList _list276 = iprot.readListBegin();
- struct.success = new ArrayList<KeySlice>(_list276.size);
- for (int _i277 = 0; _i277 < _list276.size; ++_i277)
- {
- KeySlice _elem278;
- _elem278 = new KeySlice();
- _elem278.read(iprot);
- struct.success.add(_elem278);
- }
- iprot.readListEnd();
- }
- struct.setSuccessIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 1: // IRE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.ire = new InvalidRequestException();
- struct.ire.read(iprot);
- struct.setIreIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 2: // UE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.ue = new UnavailableException();
- struct.ue.read(iprot);
- struct.setUeIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 3: // TE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.te = new TimedOutException();
- struct.te.read(iprot);
- struct.setTeIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, get_range_slices_result struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.success != null) {
- oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
- {
- oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.success.size()));
- for (KeySlice _iter279 : struct.success)
- {
- _iter279.write(oprot);
- }
- oprot.writeListEnd();
- }
- oprot.writeFieldEnd();
- }
- if (struct.ire != null) {
- oprot.writeFieldBegin(IRE_FIELD_DESC);
- struct.ire.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.ue != null) {
- oprot.writeFieldBegin(UE_FIELD_DESC);
- struct.ue.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.te != null) {
- oprot.writeFieldBegin(TE_FIELD_DESC);
- struct.te.write(oprot);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class get_range_slices_resultTupleSchemeFactory implements SchemeFactory {
- public get_range_slices_resultTupleScheme getScheme() {
- return new get_range_slices_resultTupleScheme();
- }
- }
-
- private static class get_range_slices_resultTupleScheme extends TupleScheme<get_range_slices_result> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, get_range_slices_result struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- BitSet optionals = new BitSet();
- if (struct.isSetSuccess()) {
- optionals.set(0);
- }
- if (struct.isSetIre()) {
- optionals.set(1);
- }
- if (struct.isSetUe()) {
- optionals.set(2);
- }
- if (struct.isSetTe()) {
- optionals.set(3);
- }
- oprot.writeBitSet(optionals, 4);
- if (struct.isSetSuccess()) {
- {
- oprot.writeI32(struct.success.size());
- for (KeySlice _iter280 : struct.success)
- {
- _iter280.write(oprot);
- }
- }
- }
- if (struct.isSetIre()) {
- struct.ire.write(oprot);
- }
- if (struct.isSetUe()) {
- struct.ue.write(oprot);
- }
- if (struct.isSetTe()) {
- struct.te.write(oprot);
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, get_range_slices_result struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- BitSet incoming = iprot.readBitSet(4);
- if (incoming.get(0)) {
- {
- org.apache.thrift.protocol.TList _list281 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
- struct.success = new ArrayList<KeySlice>(_list281.size);
- for (int _i282 = 0; _i282 < _list281.size; ++_i282)
- {
- KeySlice _elem283;
- _elem283 = new KeySlice();
- _elem283.read(iprot);
- struct.success.add(_elem283);
- }
- }
- struct.setSuccessIsSet(true);
- }
- if (incoming.get(1)) {
- struct.ire = new InvalidRequestException();
- struct.ire.read(iprot);
- struct.setIreIsSet(true);
- }
- if (incoming.get(2)) {
- struct.ue = new UnavailableException();
- struct.ue.read(iprot);
- struct.setUeIsSet(true);
- }
- if (incoming.get(3)) {
- struct.te = new TimedOutException();
- struct.te.read(iprot);
- struct.setTeIsSet(true);
- }
- }
- }
-
- }
-
- public static class get_paged_slice_args implements org.apache.thrift.TBase<get_paged_slice_args, get_paged_slice_args._Fields>, java.io.Serializable, Cloneable, Comparable<get_paged_slice_args> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("get_paged_slice_args");
-
- private static final org.apache.thrift.protocol.TField COLUMN_FAMILY_FIELD_DESC = new org.apache.thrift.protocol.TField("column_family", org.apache.thrift.protocol.TType.STRING, (short)1);
- private static final org.apache.thrift.protocol.TField RANGE_FIELD_DESC = new org.apache.thrift.protocol.TField("range", org.apache.thrift.protocol.TType.STRUCT, (short)2);
- private static final org.apache.thrift.protocol.TField START_COLUMN_FIELD_DESC = new org.apache.thrift.protocol.TField("start_column", org.apache.thrift.protocol.TType.STRING, (short)3);
- private static final org.apache.thrift.protocol.TField CONSISTENCY_LEVEL_FIELD_DESC = new org.apache.thrift.protocol.TField("consistency_level", org.apache.thrift.protocol.TType.I32, (short)4);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new get_paged_slice_argsStandardSchemeFactory());
- schemes.put(TupleScheme.class, new get_paged_slice_argsTupleSchemeFactory());
- }
-
- public String column_family; // required
- public KeyRange range; // required
- public ByteBuffer start_column; // required
- /**
- *
- * @see ConsistencyLevel
- */
- public ConsistencyLevel consistency_level; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- COLUMN_FAMILY((short)1, "column_family"),
- RANGE((short)2, "range"),
- START_COLUMN((short)3, "start_column"),
- /**
- *
- * @see ConsistencyLevel
- */
- CONSISTENCY_LEVEL((short)4, "consistency_level");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // COLUMN_FAMILY
- return COLUMN_FAMILY;
- case 2: // RANGE
- return RANGE;
- case 3: // START_COLUMN
- return START_COLUMN;
- case 4: // CONSISTENCY_LEVEL
- return CONSISTENCY_LEVEL;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.COLUMN_FAMILY, new org.apache.thrift.meta_data.FieldMetaData("column_family", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
- tmpMap.put(_Fields.RANGE, new org.apache.thrift.meta_data.FieldMetaData("range", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, KeyRange.class)));
- tmpMap.put(_Fields.START_COLUMN, new org.apache.thrift.meta_data.FieldMetaData("start_column", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true)));
- tmpMap.put(_Fields.CONSISTENCY_LEVEL, new org.apache.thrift.meta_data.FieldMetaData("consistency_level", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.EnumMetaData(org.apache.thrift.protocol.TType.ENUM, ConsistencyLevel.class)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(get_paged_slice_args.class, metaDataMap);
- }
-
- public get_paged_slice_args() {
- this.consistency_level = org.apache.cassandra.thrift.ConsistencyLevel.ONE;
-
- }
-
- public get_paged_slice_args(
- String column_family,
- KeyRange range,
- ByteBuffer start_column,
- ConsistencyLevel consistency_level)
- {
- this();
- this.column_family = column_family;
- this.range = range;
- this.start_column = start_column;
- this.consistency_level = consistency_level;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public get_paged_slice_args(get_paged_slice_args other) {
- if (other.isSetColumn_family()) {
- this.column_family = other.column_family;
- }
- if (other.isSetRange()) {
- this.range = new KeyRange(other.range);
- }
- if (other.isSetStart_column()) {
- this.start_column = org.apache.thrift.TBaseHelper.copyBinary(other.start_column);
-;
- }
- if (other.isSetConsistency_level()) {
- this.consistency_level = other.consistency_level;
- }
- }
-
- public get_paged_slice_args deepCopy() {
- return new get_paged_slice_args(this);
- }
-
- @Override
- public void clear() {
- this.column_family = null;
- this.range = null;
- this.start_column = null;
- this.consistency_level = org.apache.cassandra.thrift.ConsistencyLevel.ONE;
-
- }
-
- public String getColumn_family() {
- return this.column_family;
- }
-
- public get_paged_slice_args setColumn_family(String column_family) {
- this.column_family = column_family;
- return this;
- }
-
- public void unsetColumn_family() {
- this.column_family = null;
- }
-
- /** Returns true if field column_family is set (has been assigned a value) and false otherwise */
- public boolean isSetColumn_family() {
- return this.column_family != null;
- }
-
- public void setColumn_familyIsSet(boolean value) {
- if (!value) {
- this.column_family = null;
- }
- }
-
- public KeyRange getRange() {
- return this.range;
- }
-
- public get_paged_slice_args setRange(KeyRange range) {
- this.range = range;
- return this;
- }
-
- public void unsetRange() {
- this.range = null;
- }
-
- /** Returns true if field range is set (has been assigned a value) and false otherwise */
- public boolean isSetRange() {
- return this.range != null;
- }
-
- public void setRangeIsSet(boolean value) {
- if (!value) {
- this.range = null;
- }
- }
-
- public byte[] getStart_column() {
- setStart_column(org.apache.thrift.TBaseHelper.rightSize(start_column));
- return start_column == null ? null : start_column.array();
- }
-
- public ByteBuffer bufferForStart_column() {
- return start_column;
- }
-
- public get_paged_slice_args setStart_column(byte[] start_column) {
- setStart_column(start_column == null ? (ByteBuffer)null : ByteBuffer.wrap(start_column));
- return this;
- }
-
- public get_paged_slice_args setStart_column(ByteBuffer start_column) {
- this.start_column = start_column;
- return this;
- }
-
- public void unsetStart_column() {
- this.start_column = null;
- }
-
- /** Returns true if field start_column is set (has been assigned a value) and false otherwise */
- public boolean isSetStart_column() {
- return this.start_column != null;
- }
-
- public void setStart_columnIsSet(boolean value) {
- if (!value) {
- this.start_column = null;
- }
- }
-
- /**
- *
- * @see ConsistencyLevel
- */
- public ConsistencyLevel getConsistency_level() {
- return this.consistency_level;
- }
-
- /**
- *
- * @see ConsistencyLevel
- */
- public get_paged_slice_args setConsistency_level(ConsistencyLevel consistency_level) {
- this.consistency_level = consistency_level;
- return this;
- }
-
- public void unsetConsistency_level() {
- this.consistency_level = null;
- }
-
- /** Returns true if field consistency_level is set (has been assigned a value) and false otherwise */
- public boolean isSetConsistency_level() {
- return this.consistency_level != null;
- }
-
- public void setConsistency_levelIsSet(boolean value) {
- if (!value) {
- this.consistency_level = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case COLUMN_FAMILY:
- if (value == null) {
- unsetColumn_family();
- } else {
- setColumn_family((String)value);
- }
- break;
-
- case RANGE:
- if (value == null) {
- unsetRange();
- } else {
- setRange((KeyRange)value);
- }
- break;
-
- case START_COLUMN:
- if (value == null) {
- unsetStart_column();
- } else {
- setStart_column((ByteBuffer)value);
- }
- break;
-
- case CONSISTENCY_LEVEL:
- if (value == null) {
- unsetConsistency_level();
- } else {
- setConsistency_level((ConsistencyLevel)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case COLUMN_FAMILY:
- return getColumn_family();
-
- case RANGE:
- return getRange();
-
- case START_COLUMN:
- return getStart_column();
-
- case CONSISTENCY_LEVEL:
- return getConsistency_level();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case COLUMN_FAMILY:
- return isSetColumn_family();
- case RANGE:
- return isSetRange();
- case START_COLUMN:
- return isSetStart_column();
- case CONSISTENCY_LEVEL:
- return isSetConsistency_level();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof get_paged_slice_args)
- return this.equals((get_paged_slice_args)that);
- return false;
- }
-
- public boolean equals(get_paged_slice_args that) {
- if (that == null)
- return false;
-
- boolean this_present_column_family = true && this.isSetColumn_family();
- boolean that_present_column_family = true && that.isSetColumn_family();
- if (this_present_column_family || that_present_column_family) {
- if (!(this_present_column_family && that_present_column_family))
- return false;
- if (!this.column_family.equals(that.column_family))
- return false;
- }
-
- boolean this_present_range = true && this.isSetRange();
- boolean that_present_range = true && that.isSetRange();
- if (this_present_range || that_present_range) {
- if (!(this_present_range && that_present_range))
- return false;
- if (!this.range.equals(that.range))
- return false;
- }
-
- boolean this_present_start_column = true && this.isSetStart_column();
- boolean that_present_start_column = true && that.isSetStart_column();
- if (this_present_start_column || that_present_start_column) {
- if (!(this_present_start_column && that_present_start_column))
- return false;
- if (!this.start_column.equals(that.start_column))
- return false;
- }
-
- boolean this_present_consistency_level = true && this.isSetConsistency_level();
- boolean that_present_consistency_level = true && that.isSetConsistency_level();
- if (this_present_consistency_level || that_present_consistency_level) {
- if (!(this_present_consistency_level && that_present_consistency_level))
- return false;
- if (!this.consistency_level.equals(that.consistency_level))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_column_family = true && (isSetColumn_family());
- builder.append(present_column_family);
- if (present_column_family)
- builder.append(column_family);
-
- boolean present_range = true && (isSetRange());
- builder.append(present_range);
- if (present_range)
- builder.append(range);
-
- boolean present_start_column = true && (isSetStart_column());
- builder.append(present_start_column);
- if (present_start_column)
- builder.append(start_column);
-
- boolean present_consistency_level = true && (isSetConsistency_level());
- builder.append(present_consistency_level);
- if (present_consistency_level)
- builder.append(consistency_level.getValue());
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(get_paged_slice_args other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetColumn_family()).compareTo(other.isSetColumn_family());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetColumn_family()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.column_family, other.column_family);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetRange()).compareTo(other.isSetRange());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetRange()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.range, other.range);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetStart_column()).compareTo(other.isSetStart_column());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetStart_column()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.start_column, other.start_column);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetConsistency_level()).compareTo(other.isSetConsistency_level());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetConsistency_level()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.consistency_level, other.consistency_level);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("get_paged_slice_args(");
- boolean first = true;
-
- sb.append("column_family:");
- if (this.column_family == null) {
- sb.append("null");
- } else {
- sb.append(this.column_family);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("range:");
- if (this.range == null) {
- sb.append("null");
- } else {
- sb.append(this.range);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("start_column:");
- if (this.start_column == null) {
- sb.append("null");
- } else {
- org.apache.thrift.TBaseHelper.toString(this.start_column, sb);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("consistency_level:");
- if (this.consistency_level == null) {
- sb.append("null");
- } else {
- sb.append(this.consistency_level);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- if (column_family == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'column_family' was not present! Struct: " + toString());
- }
- if (range == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'range' was not present! Struct: " + toString());
- }
- if (start_column == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'start_column' was not present! Struct: " + toString());
- }
- if (consistency_level == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'consistency_level' was not present! Struct: " + toString());
- }
- // check for sub-struct validity
- if (range != null) {
- range.validate();
- }
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class get_paged_slice_argsStandardSchemeFactory implements SchemeFactory {
- public get_paged_slice_argsStandardScheme getScheme() {
- return new get_paged_slice_argsStandardScheme();
- }
- }
-
- private static class get_paged_slice_argsStandardScheme extends StandardScheme<get_paged_slice_args> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, get_paged_slice_args struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // COLUMN_FAMILY
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.column_family = iprot.readString();
- struct.setColumn_familyIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 2: // RANGE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.range = new KeyRange();
- struct.range.read(iprot);
- struct.setRangeIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 3: // START_COLUMN
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.start_column = iprot.readBinary();
- struct.setStart_columnIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 4: // CONSISTENCY_LEVEL
- if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
- struct.consistency_level = ConsistencyLevel.findByValue(iprot.readI32());
- struct.setConsistency_levelIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, get_paged_slice_args struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.column_family != null) {
- oprot.writeFieldBegin(COLUMN_FAMILY_FIELD_DESC);
- oprot.writeString(struct.column_family);
- oprot.writeFieldEnd();
- }
- if (struct.range != null) {
- oprot.writeFieldBegin(RANGE_FIELD_DESC);
- struct.range.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.start_column != null) {
- oprot.writeFieldBegin(START_COLUMN_FIELD_DESC);
- oprot.writeBinary(struct.start_column);
- oprot.writeFieldEnd();
- }
- if (struct.consistency_level != null) {
- oprot.writeFieldBegin(CONSISTENCY_LEVEL_FIELD_DESC);
- oprot.writeI32(struct.consistency_level.getValue());
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class get_paged_slice_argsTupleSchemeFactory implements SchemeFactory {
- public get_paged_slice_argsTupleScheme getScheme() {
- return new get_paged_slice_argsTupleScheme();
- }
- }
-
- private static class get_paged_slice_argsTupleScheme extends TupleScheme<get_paged_slice_args> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, get_paged_slice_args struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- oprot.writeString(struct.column_family);
- struct.range.write(oprot);
- oprot.writeBinary(struct.start_column);
- oprot.writeI32(struct.consistency_level.getValue());
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, get_paged_slice_args struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- struct.column_family = iprot.readString();
- struct.setColumn_familyIsSet(true);
- struct.range = new KeyRange();
- struct.range.read(iprot);
- struct.setRangeIsSet(true);
- struct.start_column = iprot.readBinary();
- struct.setStart_columnIsSet(true);
- struct.consistency_level = ConsistencyLevel.findByValue(iprot.readI32());
- struct.setConsistency_levelIsSet(true);
- }
- }
-
- }
-
- public static class get_paged_slice_result implements org.apache.thrift.TBase<get_paged_slice_result, get_paged_slice_result._Fields>, java.io.Serializable, Cloneable, Comparable<get_paged_slice_result> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("get_paged_slice_result");
-
- private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.LIST, (short)0);
- private static final org.apache.thrift.protocol.TField IRE_FIELD_DESC = new org.apache.thrift.protocol.TField("ire", org.apache.thrift.protocol.TType.STRUCT, (short)1);
- private static final org.apache.thrift.protocol.TField UE_FIELD_DESC = new org.apache.thrift.protocol.TField("ue", org.apache.thrift.protocol.TType.STRUCT, (short)2);
- private static final org.apache.thrift.protocol.TField TE_FIELD_DESC = new org.apache.thrift.protocol.TField("te", org.apache.thrift.protocol.TType.STRUCT, (short)3);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new get_paged_slice_resultStandardSchemeFactory());
- schemes.put(TupleScheme.class, new get_paged_slice_resultTupleSchemeFactory());
- }
-
- public List<KeySlice> success; // required
- public InvalidRequestException ire; // required
- public UnavailableException ue; // required
- public TimedOutException te; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- SUCCESS((short)0, "success"),
- IRE((short)1, "ire"),
- UE((short)2, "ue"),
- TE((short)3, "te");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 0: // SUCCESS
- return SUCCESS;
- case 1: // IRE
- return IRE;
- case 2: // UE
- return UE;
- case 3: // TE
- return TE;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST,
- new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, KeySlice.class))));
- tmpMap.put(_Fields.IRE, new org.apache.thrift.meta_data.FieldMetaData("ire", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- tmpMap.put(_Fields.UE, new org.apache.thrift.meta_data.FieldMetaData("ue", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- tmpMap.put(_Fields.TE, new org.apache.thrift.meta_data.FieldMetaData("te", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(get_paged_slice_result.class, metaDataMap);
- }
-
- public get_paged_slice_result() {
- }
-
- public get_paged_slice_result(
- List<KeySlice> success,
- InvalidRequestException ire,
- UnavailableException ue,
- TimedOutException te)
- {
- this();
- this.success = success;
- this.ire = ire;
- this.ue = ue;
- this.te = te;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public get_paged_slice_result(get_paged_slice_result other) {
- if (other.isSetSuccess()) {
- List<KeySlice> __this__success = new ArrayList<KeySlice>(other.success.size());
- for (KeySlice other_element : other.success) {
- __this__success.add(new KeySlice(other_element));
- }
- this.success = __this__success;
- }
- if (other.isSetIre()) {
- this.ire = new InvalidRequestException(other.ire);
- }
- if (other.isSetUe()) {
- this.ue = new UnavailableException(other.ue);
- }
- if (other.isSetTe()) {
- this.te = new TimedOutException(other.te);
- }
- }
-
- public get_paged_slice_result deepCopy() {
- return new get_paged_slice_result(this);
- }
-
- @Override
- public void clear() {
- this.success = null;
- this.ire = null;
- this.ue = null;
- this.te = null;
- }
-
- public int getSuccessSize() {
- return (this.success == null) ? 0 : this.success.size();
- }
-
- public java.util.Iterator<KeySlice> getSuccessIterator() {
- return (this.success == null) ? null : this.success.iterator();
- }
-
- public void addToSuccess(KeySlice elem) {
- if (this.success == null) {
- this.success = new ArrayList<KeySlice>();
- }
- this.success.add(elem);
- }
-
- public List<KeySlice> getSuccess() {
- return this.success;
- }
-
- public get_paged_slice_result setSuccess(List<KeySlice> success) {
- this.success = success;
- return this;
- }
-
- public void unsetSuccess() {
- this.success = null;
- }
-
- /** Returns true if field success is set (has been assigned a value) and false otherwise */
- public boolean isSetSuccess() {
- return this.success != null;
- }
-
- public void setSuccessIsSet(boolean value) {
- if (!value) {
- this.success = null;
- }
- }
-
- public InvalidRequestException getIre() {
- return this.ire;
- }
-
- public get_paged_slice_result setIre(InvalidRequestException ire) {
- this.ire = ire;
- return this;
- }
-
- public void unsetIre() {
- this.ire = null;
- }
-
- /** Returns true if field ire is set (has been assigned a value) and false otherwise */
- public boolean isSetIre() {
- return this.ire != null;
- }
-
- public void setIreIsSet(boolean value) {
- if (!value) {
- this.ire = null;
- }
- }
-
- public UnavailableException getUe() {
- return this.ue;
- }
-
- public get_paged_slice_result setUe(UnavailableException ue) {
- this.ue = ue;
- return this;
- }
-
- public void unsetUe() {
- this.ue = null;
- }
-
- /** Returns true if field ue is set (has been assigned a value) and false otherwise */
- public boolean isSetUe() {
- return this.ue != null;
- }
-
- public void setUeIsSet(boolean value) {
- if (!value) {
- this.ue = null;
- }
- }
-
- public TimedOutException getTe() {
- return this.te;
- }
-
- public get_paged_slice_result setTe(TimedOutException te) {
- this.te = te;
- return this;
- }
-
- public void unsetTe() {
- this.te = null;
- }
-
- /** Returns true if field te is set (has been assigned a value) and false otherwise */
- public boolean isSetTe() {
- return this.te != null;
- }
-
- public void setTeIsSet(boolean value) {
- if (!value) {
- this.te = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case SUCCESS:
- if (value == null) {
- unsetSuccess();
- } else {
- setSuccess((List<KeySlice>)value);
- }
- break;
-
- case IRE:
- if (value == null) {
- unsetIre();
- } else {
- setIre((InvalidRequestException)value);
- }
- break;
-
- case UE:
- if (value == null) {
- unsetUe();
- } else {
- setUe((UnavailableException)value);
- }
- break;
-
- case TE:
- if (value == null) {
- unsetTe();
- } else {
- setTe((TimedOutException)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case SUCCESS:
- return getSuccess();
-
- case IRE:
- return getIre();
-
- case UE:
- return getUe();
-
- case TE:
- return getTe();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case SUCCESS:
- return isSetSuccess();
- case IRE:
- return isSetIre();
- case UE:
- return isSetUe();
- case TE:
- return isSetTe();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof get_paged_slice_result)
- return this.equals((get_paged_slice_result)that);
- return false;
- }
-
- public boolean equals(get_paged_slice_result that) {
- if (that == null)
- return false;
-
- boolean this_present_success = true && this.isSetSuccess();
- boolean that_present_success = true && that.isSetSuccess();
- if (this_present_success || that_present_success) {
- if (!(this_present_success && that_present_success))
- return false;
- if (!this.success.equals(that.success))
- return false;
- }
-
- boolean this_present_ire = true && this.isSetIre();
- boolean that_present_ire = true && that.isSetIre();
- if (this_present_ire || that_present_ire) {
- if (!(this_present_ire && that_present_ire))
- return false;
- if (!this.ire.equals(that.ire))
- return false;
- }
-
- boolean this_present_ue = true && this.isSetUe();
- boolean that_present_ue = true && that.isSetUe();
- if (this_present_ue || that_present_ue) {
- if (!(this_present_ue && that_present_ue))
- return false;
- if (!this.ue.equals(that.ue))
- return false;
- }
-
- boolean this_present_te = true && this.isSetTe();
- boolean that_present_te = true && that.isSetTe();
- if (this_present_te || that_present_te) {
- if (!(this_present_te && that_present_te))
- return false;
- if (!this.te.equals(that.te))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_success = true && (isSetSuccess());
- builder.append(present_success);
- if (present_success)
- builder.append(success);
-
- boolean present_ire = true && (isSetIre());
- builder.append(present_ire);
- if (present_ire)
- builder.append(ire);
-
- boolean present_ue = true && (isSetUe());
- builder.append(present_ue);
- if (present_ue)
- builder.append(ue);
-
- boolean present_te = true && (isSetTe());
- builder.append(present_te);
- if (present_te)
- builder.append(te);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(get_paged_slice_result other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(other.isSetSuccess());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetSuccess()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, other.success);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetIre()).compareTo(other.isSetIre());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetIre()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.ire, other.ire);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetUe()).compareTo(other.isSetUe());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetUe()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.ue, other.ue);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetTe()).compareTo(other.isSetTe());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetTe()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.te, other.te);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("get_paged_slice_result(");
- boolean first = true;
-
- sb.append("success:");
- if (this.success == null) {
- sb.append("null");
- } else {
- sb.append(this.success);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("ire:");
- if (this.ire == null) {
- sb.append("null");
- } else {
- sb.append(this.ire);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("ue:");
- if (this.ue == null) {
- sb.append("null");
- } else {
- sb.append(this.ue);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("te:");
- if (this.te == null) {
- sb.append("null");
- } else {
- sb.append(this.te);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class get_paged_slice_resultStandardSchemeFactory implements SchemeFactory {
- public get_paged_slice_resultStandardScheme getScheme() {
- return new get_paged_slice_resultStandardScheme();
- }
- }
-
- private static class get_paged_slice_resultStandardScheme extends StandardScheme<get_paged_slice_result> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, get_paged_slice_result struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 0: // SUCCESS
- if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
- {
- org.apache.thrift.protocol.TList _list284 = iprot.readListBegin();
- struct.success = new ArrayList<KeySlice>(_list284.size);
- for (int _i285 = 0; _i285 < _list284.size; ++_i285)
- {
- KeySlice _elem286;
- _elem286 = new KeySlice();
- _elem286.read(iprot);
- struct.success.add(_elem286);
- }
- iprot.readListEnd();
- }
- struct.setSuccessIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 1: // IRE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.ire = new InvalidRequestException();
- struct.ire.read(iprot);
- struct.setIreIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 2: // UE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.ue = new UnavailableException();
- struct.ue.read(iprot);
- struct.setUeIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 3: // TE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.te = new TimedOutException();
- struct.te.read(iprot);
- struct.setTeIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, get_paged_slice_result struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.success != null) {
- oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
- {
- oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.success.size()));
- for (KeySlice _iter287 : struct.success)
- {
- _iter287.write(oprot);
- }
- oprot.writeListEnd();
- }
- oprot.writeFieldEnd();
- }
- if (struct.ire != null) {
- oprot.writeFieldBegin(IRE_FIELD_DESC);
- struct.ire.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.ue != null) {
- oprot.writeFieldBegin(UE_FIELD_DESC);
- struct.ue.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.te != null) {
- oprot.writeFieldBegin(TE_FIELD_DESC);
- struct.te.write(oprot);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class get_paged_slice_resultTupleSchemeFactory implements SchemeFactory {
- public get_paged_slice_resultTupleScheme getScheme() {
- return new get_paged_slice_resultTupleScheme();
- }
- }
-
- private static class get_paged_slice_resultTupleScheme extends TupleScheme<get_paged_slice_result> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, get_paged_slice_result struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- BitSet optionals = new BitSet();
- if (struct.isSetSuccess()) {
- optionals.set(0);
- }
- if (struct.isSetIre()) {
- optionals.set(1);
- }
- if (struct.isSetUe()) {
- optionals.set(2);
- }
- if (struct.isSetTe()) {
- optionals.set(3);
- }
- oprot.writeBitSet(optionals, 4);
- if (struct.isSetSuccess()) {
- {
- oprot.writeI32(struct.success.size());
- for (KeySlice _iter288 : struct.success)
- {
- _iter288.write(oprot);
- }
- }
- }
- if (struct.isSetIre()) {
- struct.ire.write(oprot);
- }
- if (struct.isSetUe()) {
- struct.ue.write(oprot);
- }
- if (struct.isSetTe()) {
- struct.te.write(oprot);
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, get_paged_slice_result struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- BitSet incoming = iprot.readBitSet(4);
- if (incoming.get(0)) {
- {
- org.apache.thrift.protocol.TList _list289 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
- struct.success = new ArrayList<KeySlice>(_list289.size);
- for (int _i290 = 0; _i290 < _list289.size; ++_i290)
- {
- KeySlice _elem291;
- _elem291 = new KeySlice();
- _elem291.read(iprot);
- struct.success.add(_elem291);
- }
- }
- struct.setSuccessIsSet(true);
- }
- if (incoming.get(1)) {
- struct.ire = new InvalidRequestException();
- struct.ire.read(iprot);
- struct.setIreIsSet(true);
- }
- if (incoming.get(2)) {
- struct.ue = new UnavailableException();
- struct.ue.read(iprot);
- struct.setUeIsSet(true);
- }
- if (incoming.get(3)) {
- struct.te = new TimedOutException();
- struct.te.read(iprot);
- struct.setTeIsSet(true);
- }
- }
- }
-
- }
-
- public static class get_indexed_slices_args implements org.apache.thrift.TBase<get_indexed_slices_args, get_indexed_slices_args._Fields>, java.io.Serializable, Cloneable, Comparable<get_indexed_slices_args> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("get_indexed_slices_args");
-
- private static final org.apache.thrift.protocol.TField COLUMN_PARENT_FIELD_DESC = new org.apache.thrift.protocol.TField("column_parent", org.apache.thrift.protocol.TType.STRUCT, (short)1);
- private static final org.apache.thrift.protocol.TField INDEX_CLAUSE_FIELD_DESC = new org.apache.thrift.protocol.TField("index_clause", org.apache.thrift.protocol.TType.STRUCT, (short)2);
- private static final org.apache.thrift.protocol.TField COLUMN_PREDICATE_FIELD_DESC = new org.apache.thrift.protocol.TField("column_predicate", org.apache.thrift.protocol.TType.STRUCT, (short)3);
- private static final org.apache.thrift.protocol.TField CONSISTENCY_LEVEL_FIELD_DESC = new org.apache.thrift.protocol.TField("consistency_level", org.apache.thrift.protocol.TType.I32, (short)4);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new get_indexed_slices_argsStandardSchemeFactory());
- schemes.put(TupleScheme.class, new get_indexed_slices_argsTupleSchemeFactory());
- }
-
- public ColumnParent column_parent; // required
- public IndexClause index_clause; // required
- public SlicePredicate column_predicate; // required
- /**
- *
- * @see ConsistencyLevel
- */
- public ConsistencyLevel consistency_level; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- COLUMN_PARENT((short)1, "column_parent"),
- INDEX_CLAUSE((short)2, "index_clause"),
- COLUMN_PREDICATE((short)3, "column_predicate"),
- /**
- *
- * @see ConsistencyLevel
- */
- CONSISTENCY_LEVEL((short)4, "consistency_level");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // COLUMN_PARENT
- return COLUMN_PARENT;
- case 2: // INDEX_CLAUSE
- return INDEX_CLAUSE;
- case 3: // COLUMN_PREDICATE
- return COLUMN_PREDICATE;
- case 4: // CONSISTENCY_LEVEL
- return CONSISTENCY_LEVEL;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.COLUMN_PARENT, new org.apache.thrift.meta_data.FieldMetaData("column_parent", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, ColumnParent.class)));
- tmpMap.put(_Fields.INDEX_CLAUSE, new org.apache.thrift.meta_data.FieldMetaData("index_clause", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, IndexClause.class)));
- tmpMap.put(_Fields.COLUMN_PREDICATE, new org.apache.thrift.meta_data.FieldMetaData("column_predicate", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, SlicePredicate.class)));
- tmpMap.put(_Fields.CONSISTENCY_LEVEL, new org.apache.thrift.meta_data.FieldMetaData("consistency_level", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.EnumMetaData(org.apache.thrift.protocol.TType.ENUM, ConsistencyLevel.class)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(get_indexed_slices_args.class, metaDataMap);
- }
-
- public get_indexed_slices_args() {
- this.consistency_level = org.apache.cassandra.thrift.ConsistencyLevel.ONE;
-
- }
-
- public get_indexed_slices_args(
- ColumnParent column_parent,
- IndexClause index_clause,
- SlicePredicate column_predicate,
- ConsistencyLevel consistency_level)
- {
- this();
- this.column_parent = column_parent;
- this.index_clause = index_clause;
- this.column_predicate = column_predicate;
- this.consistency_level = consistency_level;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public get_indexed_slices_args(get_indexed_slices_args other) {
- if (other.isSetColumn_parent()) {
- this.column_parent = new ColumnParent(other.column_parent);
- }
- if (other.isSetIndex_clause()) {
- this.index_clause = new IndexClause(other.index_clause);
- }
- if (other.isSetColumn_predicate()) {
- this.column_predicate = new SlicePredicate(other.column_predicate);
- }
- if (other.isSetConsistency_level()) {
- this.consistency_level = other.consistency_level;
- }
- }
-
- public get_indexed_slices_args deepCopy() {
- return new get_indexed_slices_args(this);
- }
-
- @Override
- public void clear() {
- this.column_parent = null;
- this.index_clause = null;
- this.column_predicate = null;
- this.consistency_level = org.apache.cassandra.thrift.ConsistencyLevel.ONE;
-
- }
-
- public ColumnParent getColumn_parent() {
- return this.column_parent;
- }
-
- public get_indexed_slices_args setColumn_parent(ColumnParent column_parent) {
- this.column_parent = column_parent;
- return this;
- }
-
- public void unsetColumn_parent() {
- this.column_parent = null;
- }
-
- /** Returns true if field column_parent is set (has been assigned a value) and false otherwise */
- public boolean isSetColumn_parent() {
- return this.column_parent != null;
- }
-
- public void setColumn_parentIsSet(boolean value) {
- if (!value) {
- this.column_parent = null;
- }
- }
-
- public IndexClause getIndex_clause() {
- return this.index_clause;
- }
-
- public get_indexed_slices_args setIndex_clause(IndexClause index_clause) {
- this.index_clause = index_clause;
- return this;
- }
-
- public void unsetIndex_clause() {
- this.index_clause = null;
- }
-
- /** Returns true if field index_clause is set (has been assigned a value) and false otherwise */
- public boolean isSetIndex_clause() {
- return this.index_clause != null;
- }
-
- public void setIndex_clauseIsSet(boolean value) {
- if (!value) {
- this.index_clause = null;
- }
- }
-
- public SlicePredicate getColumn_predicate() {
- return this.column_predicate;
- }
-
- public get_indexed_slices_args setColumn_predicate(SlicePredicate column_predicate) {
- this.column_predicate = column_predicate;
- return this;
- }
-
- public void unsetColumn_predicate() {
- this.column_predicate = null;
- }
-
- /** Returns true if field column_predicate is set (has been assigned a value) and false otherwise */
- public boolean isSetColumn_predicate() {
- return this.column_predicate != null;
- }
-
- public void setColumn_predicateIsSet(boolean value) {
- if (!value) {
- this.column_predicate = null;
- }
- }
-
- /**
- *
- * @see ConsistencyLevel
- */
- public ConsistencyLevel getConsistency_level() {
- return this.consistency_level;
- }
-
- /**
- *
- * @see ConsistencyLevel
- */
- public get_indexed_slices_args setConsistency_level(ConsistencyLevel consistency_level) {
- this.consistency_level = consistency_level;
- return this;
- }
-
- public void unsetConsistency_level() {
- this.consistency_level = null;
- }
-
- /** Returns true if field consistency_level is set (has been assigned a value) and false otherwise */
- public boolean isSetConsistency_level() {
- return this.consistency_level != null;
- }
-
- public void setConsistency_levelIsSet(boolean value) {
- if (!value) {
- this.consistency_level = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case COLUMN_PARENT:
- if (value == null) {
- unsetColumn_parent();
- } else {
- setColumn_parent((ColumnParent)value);
- }
- break;
-
- case INDEX_CLAUSE:
- if (value == null) {
- unsetIndex_clause();
- } else {
- setIndex_clause((IndexClause)value);
- }
- break;
-
- case COLUMN_PREDICATE:
- if (value == null) {
- unsetColumn_predicate();
- } else {
- setColumn_predicate((SlicePredicate)value);
- }
- break;
-
- case CONSISTENCY_LEVEL:
- if (value == null) {
- unsetConsistency_level();
- } else {
- setConsistency_level((ConsistencyLevel)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case COLUMN_PARENT:
- return getColumn_parent();
-
- case INDEX_CLAUSE:
- return getIndex_clause();
-
- case COLUMN_PREDICATE:
- return getColumn_predicate();
-
- case CONSISTENCY_LEVEL:
- return getConsistency_level();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case COLUMN_PARENT:
- return isSetColumn_parent();
- case INDEX_CLAUSE:
- return isSetIndex_clause();
- case COLUMN_PREDICATE:
- return isSetColumn_predicate();
- case CONSISTENCY_LEVEL:
- return isSetConsistency_level();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof get_indexed_slices_args)
- return this.equals((get_indexed_slices_args)that);
- return false;
- }
-
- public boolean equals(get_indexed_slices_args that) {
- if (that == null)
- return false;
-
- boolean this_present_column_parent = true && this.isSetColumn_parent();
- boolean that_present_column_parent = true && that.isSetColumn_parent();
- if (this_present_column_parent || that_present_column_parent) {
- if (!(this_present_column_parent && that_present_column_parent))
- return false;
- if (!this.column_parent.equals(that.column_parent))
- return false;
- }
-
- boolean this_present_index_clause = true && this.isSetIndex_clause();
- boolean that_present_index_clause = true && that.isSetIndex_clause();
- if (this_present_index_clause || that_present_index_clause) {
- if (!(this_present_index_clause && that_present_index_clause))
- return false;
- if (!this.index_clause.equals(that.index_clause))
- return false;
- }
-
- boolean this_present_column_predicate = true && this.isSetColumn_predicate();
- boolean that_present_column_predicate = true && that.isSetColumn_predicate();
- if (this_present_column_predicate || that_present_column_predicate) {
- if (!(this_present_column_predicate && that_present_column_predicate))
- return false;
- if (!this.column_predicate.equals(that.column_predicate))
- return false;
- }
-
- boolean this_present_consistency_level = true && this.isSetConsistency_level();
- boolean that_present_consistency_level = true && that.isSetConsistency_level();
- if (this_present_consistency_level || that_present_consistency_level) {
- if (!(this_present_consistency_level && that_present_consistency_level))
- return false;
- if (!this.consistency_level.equals(that.consistency_level))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_column_parent = true && (isSetColumn_parent());
- builder.append(present_column_parent);
- if (present_column_parent)
- builder.append(column_parent);
-
- boolean present_index_clause = true && (isSetIndex_clause());
- builder.append(present_index_clause);
- if (present_index_clause)
- builder.append(index_clause);
-
- boolean present_column_predicate = true && (isSetColumn_predicate());
- builder.append(present_column_predicate);
- if (present_column_predicate)
- builder.append(column_predicate);
-
- boolean present_consistency_level = true && (isSetConsistency_level());
- builder.append(present_consistency_level);
- if (present_consistency_level)
- builder.append(consistency_level.getValue());
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(get_indexed_slices_args other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetColumn_parent()).compareTo(other.isSetColumn_parent());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetColumn_parent()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.column_parent, other.column_parent);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetIndex_clause()).compareTo(other.isSetIndex_clause());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetIndex_clause()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.index_clause, other.index_clause);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetColumn_predicate()).compareTo(other.isSetColumn_predicate());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetColumn_predicate()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.column_predicate, other.column_predicate);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetConsistency_level()).compareTo(other.isSetConsistency_level());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetConsistency_level()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.consistency_level, other.consistency_level);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("get_indexed_slices_args(");
- boolean first = true;
-
- sb.append("column_parent:");
- if (this.column_parent == null) {
- sb.append("null");
- } else {
- sb.append(this.column_parent);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("index_clause:");
- if (this.index_clause == null) {
- sb.append("null");
- } else {
- sb.append(this.index_clause);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("column_predicate:");
- if (this.column_predicate == null) {
- sb.append("null");
- } else {
- sb.append(this.column_predicate);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("consistency_level:");
- if (this.consistency_level == null) {
- sb.append("null");
- } else {
- sb.append(this.consistency_level);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- if (column_parent == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'column_parent' was not present! Struct: " + toString());
- }
- if (index_clause == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'index_clause' was not present! Struct: " + toString());
- }
- if (column_predicate == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'column_predicate' was not present! Struct: " + toString());
- }
- if (consistency_level == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'consistency_level' was not present! Struct: " + toString());
- }
- // check for sub-struct validity
- if (column_parent != null) {
- column_parent.validate();
- }
- if (index_clause != null) {
- index_clause.validate();
- }
- if (column_predicate != null) {
- column_predicate.validate();
- }
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class get_indexed_slices_argsStandardSchemeFactory implements SchemeFactory {
- public get_indexed_slices_argsStandardScheme getScheme() {
- return new get_indexed_slices_argsStandardScheme();
- }
- }
-
- private static class get_indexed_slices_argsStandardScheme extends StandardScheme<get_indexed_slices_args> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, get_indexed_slices_args struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // COLUMN_PARENT
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.column_parent = new ColumnParent();
- struct.column_parent.read(iprot);
- struct.setColumn_parentIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 2: // INDEX_CLAUSE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.index_clause = new IndexClause();
- struct.index_clause.read(iprot);
- struct.setIndex_clauseIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 3: // COLUMN_PREDICATE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.column_predicate = new SlicePredicate();
- struct.column_predicate.read(iprot);
- struct.setColumn_predicateIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 4: // CONSISTENCY_LEVEL
- if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
- struct.consistency_level = ConsistencyLevel.findByValue(iprot.readI32());
- struct.setConsistency_levelIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, get_indexed_slices_args struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.column_parent != null) {
- oprot.writeFieldBegin(COLUMN_PARENT_FIELD_DESC);
- struct.column_parent.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.index_clause != null) {
- oprot.writeFieldBegin(INDEX_CLAUSE_FIELD_DESC);
- struct.index_clause.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.column_predicate != null) {
- oprot.writeFieldBegin(COLUMN_PREDICATE_FIELD_DESC);
- struct.column_predicate.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.consistency_level != null) {
- oprot.writeFieldBegin(CONSISTENCY_LEVEL_FIELD_DESC);
- oprot.writeI32(struct.consistency_level.getValue());
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class get_indexed_slices_argsTupleSchemeFactory implements SchemeFactory {
- public get_indexed_slices_argsTupleScheme getScheme() {
- return new get_indexed_slices_argsTupleScheme();
- }
- }
-
- private static class get_indexed_slices_argsTupleScheme extends TupleScheme<get_indexed_slices_args> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, get_indexed_slices_args struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- struct.column_parent.write(oprot);
- struct.index_clause.write(oprot);
- struct.column_predicate.write(oprot);
- oprot.writeI32(struct.consistency_level.getValue());
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, get_indexed_slices_args struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- struct.column_parent = new ColumnParent();
- struct.column_parent.read(iprot);
- struct.setColumn_parentIsSet(true);
- struct.index_clause = new IndexClause();
- struct.index_clause.read(iprot);
- struct.setIndex_clauseIsSet(true);
- struct.column_predicate = new SlicePredicate();
- struct.column_predicate.read(iprot);
- struct.setColumn_predicateIsSet(true);
- struct.consistency_level = ConsistencyLevel.findByValue(iprot.readI32());
- struct.setConsistency_levelIsSet(true);
- }
- }
-
- }
-
- public static class get_indexed_slices_result implements org.apache.thrift.TBase<get_indexed_slices_result, get_indexed_slices_result._Fields>, java.io.Serializable, Cloneable, Comparable<get_indexed_slices_result> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("get_indexed_slices_result");
-
- private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.LIST, (short)0);
- private static final org.apache.thrift.protocol.TField IRE_FIELD_DESC = new org.apache.thrift.protocol.TField("ire", org.apache.thrift.protocol.TType.STRUCT, (short)1);
- private static final org.apache.thrift.protocol.TField UE_FIELD_DESC = new org.apache.thrift.protocol.TField("ue", org.apache.thrift.protocol.TType.STRUCT, (short)2);
- private static final org.apache.thrift.protocol.TField TE_FIELD_DESC = new org.apache.thrift.protocol.TField("te", org.apache.thrift.protocol.TType.STRUCT, (short)3);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new get_indexed_slices_resultStandardSchemeFactory());
- schemes.put(TupleScheme.class, new get_indexed_slices_resultTupleSchemeFactory());
- }
-
- public List<KeySlice> success; // required
- public InvalidRequestException ire; // required
- public UnavailableException ue; // required
- public TimedOutException te; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- SUCCESS((short)0, "success"),
- IRE((short)1, "ire"),
- UE((short)2, "ue"),
- TE((short)3, "te");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 0: // SUCCESS
- return SUCCESS;
- case 1: // IRE
- return IRE;
- case 2: // UE
- return UE;
- case 3: // TE
- return TE;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST,
- new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, KeySlice.class))));
- tmpMap.put(_Fields.IRE, new org.apache.thrift.meta_data.FieldMetaData("ire", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- tmpMap.put(_Fields.UE, new org.apache.thrift.meta_data.FieldMetaData("ue", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- tmpMap.put(_Fields.TE, new org.apache.thrift.meta_data.FieldMetaData("te", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(get_indexed_slices_result.class, metaDataMap);
- }
-
- public get_indexed_slices_result() {
- }
-
- public get_indexed_slices_result(
- List<KeySlice> success,
- InvalidRequestException ire,
- UnavailableException ue,
- TimedOutException te)
- {
- this();
- this.success = success;
- this.ire = ire;
- this.ue = ue;
- this.te = te;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public get_indexed_slices_result(get_indexed_slices_result other) {
- if (other.isSetSuccess()) {
- List<KeySlice> __this__success = new ArrayList<KeySlice>(other.success.size());
- for (KeySlice other_element : other.success) {
- __this__success.add(new KeySlice(other_element));
- }
- this.success = __this__success;
- }
- if (other.isSetIre()) {
- this.ire = new InvalidRequestException(other.ire);
- }
- if (other.isSetUe()) {
- this.ue = new UnavailableException(other.ue);
- }
- if (other.isSetTe()) {
- this.te = new TimedOutException(other.te);
- }
- }
-
- public get_indexed_slices_result deepCopy() {
- return new get_indexed_slices_result(this);
- }
-
- @Override
- public void clear() {
- this.success = null;
- this.ire = null;
- this.ue = null;
- this.te = null;
- }
-
- public int getSuccessSize() {
- return (this.success == null) ? 0 : this.success.size();
- }
-
- public java.util.Iterator<KeySlice> getSuccessIterator() {
- return (this.success == null) ? null : this.success.iterator();
- }
-
- public void addToSuccess(KeySlice elem) {
- if (this.success == null) {
- this.success = new ArrayList<KeySlice>();
- }
- this.success.add(elem);
- }
-
- public List<KeySlice> getSuccess() {
- return this.success;
- }
-
- public get_indexed_slices_result setSuccess(List<KeySlice> success) {
- this.success = success;
- return this;
- }
-
- public void unsetSuccess() {
- this.success = null;
- }
-
- /** Returns true if field success is set (has been assigned a value) and false otherwise */
- public boolean isSetSuccess() {
- return this.success != null;
- }
-
- public void setSuccessIsSet(boolean value) {
- if (!value) {
- this.success = null;
- }
- }
-
- public InvalidRequestException getIre() {
- return this.ire;
- }
-
- public get_indexed_slices_result setIre(InvalidRequestException ire) {
- this.ire = ire;
- return this;
- }
-
- public void unsetIre() {
- this.ire = null;
- }
-
- /** Returns true if field ire is set (has been assigned a value) and false otherwise */
- public boolean isSetIre() {
- return this.ire != null;
- }
-
- public void setIreIsSet(boolean value) {
- if (!value) {
- this.ire = null;
- }
- }
-
- public UnavailableException getUe() {
- return this.ue;
- }
-
- public get_indexed_slices_result setUe(UnavailableException ue) {
- this.ue = ue;
- return this;
- }
-
- public void unsetUe() {
- this.ue = null;
- }
-
- /** Returns true if field ue is set (has been assigned a value) and false otherwise */
- public boolean isSetUe() {
- return this.ue != null;
- }
-
- public void setUeIsSet(boolean value) {
- if (!value) {
- this.ue = null;
- }
- }
-
- public TimedOutException getTe() {
- return this.te;
- }
-
- public get_indexed_slices_result setTe(TimedOutException te) {
- this.te = te;
- return this;
- }
-
- public void unsetTe() {
- this.te = null;
- }
-
- /** Returns true if field te is set (has been assigned a value) and false otherwise */
- public boolean isSetTe() {
- return this.te != null;
- }
-
- public void setTeIsSet(boolean value) {
- if (!value) {
- this.te = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case SUCCESS:
- if (value == null) {
- unsetSuccess();
- } else {
- setSuccess((List<KeySlice>)value);
- }
- break;
-
- case IRE:
- if (value == null) {
- unsetIre();
- } else {
- setIre((InvalidRequestException)value);
- }
- break;
-
- case UE:
- if (value == null) {
- unsetUe();
- } else {
- setUe((UnavailableException)value);
- }
- break;
-
- case TE:
- if (value == null) {
- unsetTe();
- } else {
- setTe((TimedOutException)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case SUCCESS:
- return getSuccess();
-
- case IRE:
- return getIre();
-
- case UE:
- return getUe();
-
- case TE:
- return getTe();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case SUCCESS:
- return isSetSuccess();
- case IRE:
- return isSetIre();
- case UE:
- return isSetUe();
- case TE:
- return isSetTe();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof get_indexed_slices_result)
- return this.equals((get_indexed_slices_result)that);
- return false;
- }
-
- public boolean equals(get_indexed_slices_result that) {
- if (that == null)
- return false;
-
- boolean this_present_success = true && this.isSetSuccess();
- boolean that_present_success = true && that.isSetSuccess();
- if (this_present_success || that_present_success) {
- if (!(this_present_success && that_present_success))
- return false;
- if (!this.success.equals(that.success))
- return false;
- }
-
- boolean this_present_ire = true && this.isSetIre();
- boolean that_present_ire = true && that.isSetIre();
- if (this_present_ire || that_present_ire) {
- if (!(this_present_ire && that_present_ire))
- return false;
- if (!this.ire.equals(that.ire))
- return false;
- }
-
- boolean this_present_ue = true && this.isSetUe();
- boolean that_present_ue = true && that.isSetUe();
- if (this_present_ue || that_present_ue) {
- if (!(this_present_ue && that_present_ue))
- return false;
- if (!this.ue.equals(that.ue))
- return false;
- }
-
- boolean this_present_te = true && this.isSetTe();
- boolean that_present_te = true && that.isSetTe();
- if (this_present_te || that_present_te) {
- if (!(this_present_te && that_present_te))
- return false;
- if (!this.te.equals(that.te))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_success = true && (isSetSuccess());
- builder.append(present_success);
- if (present_success)
- builder.append(success);
-
- boolean present_ire = true && (isSetIre());
- builder.append(present_ire);
- if (present_ire)
- builder.append(ire);
-
- boolean present_ue = true && (isSetUe());
- builder.append(present_ue);
- if (present_ue)
- builder.append(ue);
-
- boolean present_te = true && (isSetTe());
- builder.append(present_te);
- if (present_te)
- builder.append(te);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(get_indexed_slices_result other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(other.isSetSuccess());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetSuccess()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, other.success);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetIre()).compareTo(other.isSetIre());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetIre()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.ire, other.ire);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetUe()).compareTo(other.isSetUe());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetUe()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.ue, other.ue);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetTe()).compareTo(other.isSetTe());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetTe()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.te, other.te);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("get_indexed_slices_result(");
- boolean first = true;
-
- sb.append("success:");
- if (this.success == null) {
- sb.append("null");
- } else {
- sb.append(this.success);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("ire:");
- if (this.ire == null) {
- sb.append("null");
- } else {
- sb.append(this.ire);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("ue:");
- if (this.ue == null) {
- sb.append("null");
- } else {
- sb.append(this.ue);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("te:");
- if (this.te == null) {
- sb.append("null");
- } else {
- sb.append(this.te);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class get_indexed_slices_resultStandardSchemeFactory implements SchemeFactory {
- public get_indexed_slices_resultStandardScheme getScheme() {
- return new get_indexed_slices_resultStandardScheme();
- }
- }
-
- private static class get_indexed_slices_resultStandardScheme extends StandardScheme<get_indexed_slices_result> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, get_indexed_slices_result struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 0: // SUCCESS
- if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
- {
- org.apache.thrift.protocol.TList _list292 = iprot.readListBegin();
- struct.success = new ArrayList<KeySlice>(_list292.size);
- for (int _i293 = 0; _i293 < _list292.size; ++_i293)
- {
- KeySlice _elem294;
- _elem294 = new KeySlice();
- _elem294.read(iprot);
- struct.success.add(_elem294);
- }
- iprot.readListEnd();
- }
- struct.setSuccessIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 1: // IRE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.ire = new InvalidRequestException();
- struct.ire.read(iprot);
- struct.setIreIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 2: // UE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.ue = new UnavailableException();
- struct.ue.read(iprot);
- struct.setUeIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 3: // TE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.te = new TimedOutException();
- struct.te.read(iprot);
- struct.setTeIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, get_indexed_slices_result struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.success != null) {
- oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
- {
- oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.success.size()));
- for (KeySlice _iter295 : struct.success)
- {
- _iter295.write(oprot);
- }
- oprot.writeListEnd();
- }
- oprot.writeFieldEnd();
- }
- if (struct.ire != null) {
- oprot.writeFieldBegin(IRE_FIELD_DESC);
- struct.ire.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.ue != null) {
- oprot.writeFieldBegin(UE_FIELD_DESC);
- struct.ue.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.te != null) {
- oprot.writeFieldBegin(TE_FIELD_DESC);
- struct.te.write(oprot);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class get_indexed_slices_resultTupleSchemeFactory implements SchemeFactory {
- public get_indexed_slices_resultTupleScheme getScheme() {
- return new get_indexed_slices_resultTupleScheme();
- }
- }
-
- private static class get_indexed_slices_resultTupleScheme extends TupleScheme<get_indexed_slices_result> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, get_indexed_slices_result struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- BitSet optionals = new BitSet();
- if (struct.isSetSuccess()) {
- optionals.set(0);
- }
- if (struct.isSetIre()) {
- optionals.set(1);
- }
- if (struct.isSetUe()) {
- optionals.set(2);
- }
- if (struct.isSetTe()) {
- optionals.set(3);
- }
- oprot.writeBitSet(optionals, 4);
- if (struct.isSetSuccess()) {
- {
- oprot.writeI32(struct.success.size());
- for (KeySlice _iter296 : struct.success)
- {
- _iter296.write(oprot);
- }
- }
- }
- if (struct.isSetIre()) {
- struct.ire.write(oprot);
- }
- if (struct.isSetUe()) {
- struct.ue.write(oprot);
- }
- if (struct.isSetTe()) {
- struct.te.write(oprot);
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, get_indexed_slices_result struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- BitSet incoming = iprot.readBitSet(4);
- if (incoming.get(0)) {
- {
- org.apache.thrift.protocol.TList _list297 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
- struct.success = new ArrayList<KeySlice>(_list297.size);
- for (int _i298 = 0; _i298 < _list297.size; ++_i298)
- {
- KeySlice _elem299;
- _elem299 = new KeySlice();
- _elem299.read(iprot);
- struct.success.add(_elem299);
- }
- }
- struct.setSuccessIsSet(true);
- }
- if (incoming.get(1)) {
- struct.ire = new InvalidRequestException();
- struct.ire.read(iprot);
- struct.setIreIsSet(true);
- }
- if (incoming.get(2)) {
- struct.ue = new UnavailableException();
- struct.ue.read(iprot);
- struct.setUeIsSet(true);
- }
- if (incoming.get(3)) {
- struct.te = new TimedOutException();
- struct.te.read(iprot);
- struct.setTeIsSet(true);
- }
- }
- }
-
- }
-
- public static class insert_args implements org.apache.thrift.TBase<insert_args, insert_args._Fields>, java.io.Serializable, Cloneable, Comparable<insert_args> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("insert_args");
-
- private static final org.apache.thrift.protocol.TField KEY_FIELD_DESC = new org.apache.thrift.protocol.TField("key", org.apache.thrift.protocol.TType.STRING, (short)1);
- private static final org.apache.thrift.protocol.TField COLUMN_PARENT_FIELD_DESC = new org.apache.thrift.protocol.TField("column_parent", org.apache.thrift.protocol.TType.STRUCT, (short)2);
- private static final org.apache.thrift.protocol.TField COLUMN_FIELD_DESC = new org.apache.thrift.protocol.TField("column", org.apache.thrift.protocol.TType.STRUCT, (short)3);
- private static final org.apache.thrift.protocol.TField CONSISTENCY_LEVEL_FIELD_DESC = new org.apache.thrift.protocol.TField("consistency_level", org.apache.thrift.protocol.TType.I32, (short)4);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new insert_argsStandardSchemeFactory());
- schemes.put(TupleScheme.class, new insert_argsTupleSchemeFactory());
- }
-
- public ByteBuffer key; // required
- public ColumnParent column_parent; // required
- public Column column; // required
- /**
- *
- * @see ConsistencyLevel
- */
- public ConsistencyLevel consistency_level; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- KEY((short)1, "key"),
- COLUMN_PARENT((short)2, "column_parent"),
- COLUMN((short)3, "column"),
- /**
- *
- * @see ConsistencyLevel
- */
- CONSISTENCY_LEVEL((short)4, "consistency_level");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // KEY
- return KEY;
- case 2: // COLUMN_PARENT
- return COLUMN_PARENT;
- case 3: // COLUMN
- return COLUMN;
- case 4: // CONSISTENCY_LEVEL
- return CONSISTENCY_LEVEL;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.KEY, new org.apache.thrift.meta_data.FieldMetaData("key", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true)));
- tmpMap.put(_Fields.COLUMN_PARENT, new org.apache.thrift.meta_data.FieldMetaData("column_parent", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, ColumnParent.class)));
- tmpMap.put(_Fields.COLUMN, new org.apache.thrift.meta_data.FieldMetaData("column", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, Column.class)));
- tmpMap.put(_Fields.CONSISTENCY_LEVEL, new org.apache.thrift.meta_data.FieldMetaData("consistency_level", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.EnumMetaData(org.apache.thrift.protocol.TType.ENUM, ConsistencyLevel.class)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(insert_args.class, metaDataMap);
- }
-
- public insert_args() {
- this.consistency_level = org.apache.cassandra.thrift.ConsistencyLevel.ONE;
-
- }
-
- public insert_args(
- ByteBuffer key,
- ColumnParent column_parent,
- Column column,
- ConsistencyLevel consistency_level)
- {
- this();
- this.key = key;
- this.column_parent = column_parent;
- this.column = column;
- this.consistency_level = consistency_level;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public insert_args(insert_args other) {
- if (other.isSetKey()) {
- this.key = org.apache.thrift.TBaseHelper.copyBinary(other.key);
-;
- }
- if (other.isSetColumn_parent()) {
- this.column_parent = new ColumnParent(other.column_parent);
- }
- if (other.isSetColumn()) {
- this.column = new Column(other.column);
- }
- if (other.isSetConsistency_level()) {
- this.consistency_level = other.consistency_level;
- }
- }
-
- public insert_args deepCopy() {
- return new insert_args(this);
- }
-
- @Override
- public void clear() {
- this.key = null;
- this.column_parent = null;
- this.column = null;
- this.consistency_level = org.apache.cassandra.thrift.ConsistencyLevel.ONE;
-
- }
-
- public byte[] getKey() {
- setKey(org.apache.thrift.TBaseHelper.rightSize(key));
- return key == null ? null : key.array();
- }
-
- public ByteBuffer bufferForKey() {
- return key;
- }
-
- public insert_args setKey(byte[] key) {
- setKey(key == null ? (ByteBuffer)null : ByteBuffer.wrap(key));
- return this;
- }
-
- public insert_args setKey(ByteBuffer key) {
- this.key = key;
- return this;
- }
-
- public void unsetKey() {
- this.key = null;
- }
-
- /** Returns true if field key is set (has been assigned a value) and false otherwise */
- public boolean isSetKey() {
- return this.key != null;
- }
-
- public void setKeyIsSet(boolean value) {
- if (!value) {
- this.key = null;
- }
- }
-
- public ColumnParent getColumn_parent() {
- return this.column_parent;
- }
-
- public insert_args setColumn_parent(ColumnParent column_parent) {
- this.column_parent = column_parent;
- return this;
- }
-
- public void unsetColumn_parent() {
- this.column_parent = null;
- }
-
- /** Returns true if field column_parent is set (has been assigned a value) and false otherwise */
- public boolean isSetColumn_parent() {
- return this.column_parent != null;
- }
-
- public void setColumn_parentIsSet(boolean value) {
- if (!value) {
- this.column_parent = null;
- }
- }
-
- public Column getColumn() {
- return this.column;
- }
-
- public insert_args setColumn(Column column) {
- this.column = column;
- return this;
- }
-
- public void unsetColumn() {
- this.column = null;
- }
-
- /** Returns true if field column is set (has been assigned a value) and false otherwise */
- public boolean isSetColumn() {
- return this.column != null;
- }
-
- public void setColumnIsSet(boolean value) {
- if (!value) {
- this.column = null;
- }
- }
-
- /**
- *
- * @see ConsistencyLevel
- */
- public ConsistencyLevel getConsistency_level() {
- return this.consistency_level;
- }
-
- /**
- *
- * @see ConsistencyLevel
- */
- public insert_args setConsistency_level(ConsistencyLevel consistency_level) {
- this.consistency_level = consistency_level;
- return this;
- }
-
- public void unsetConsistency_level() {
- this.consistency_level = null;
- }
-
- /** Returns true if field consistency_level is set (has been assigned a value) and false otherwise */
- public boolean isSetConsistency_level() {
- return this.consistency_level != null;
- }
-
- public void setConsistency_levelIsSet(boolean value) {
- if (!value) {
- this.consistency_level = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case KEY:
- if (value == null) {
- unsetKey();
- } else {
- setKey((ByteBuffer)value);
- }
- break;
-
- case COLUMN_PARENT:
- if (value == null) {
- unsetColumn_parent();
- } else {
- setColumn_parent((ColumnParent)value);
- }
- break;
-
- case COLUMN:
- if (value == null) {
- unsetColumn();
- } else {
- setColumn((Column)value);
- }
- break;
-
- case CONSISTENCY_LEVEL:
- if (value == null) {
- unsetConsistency_level();
- } else {
- setConsistency_level((ConsistencyLevel)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case KEY:
- return getKey();
-
- case COLUMN_PARENT:
- return getColumn_parent();
-
- case COLUMN:
- return getColumn();
-
- case CONSISTENCY_LEVEL:
- return getConsistency_level();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case KEY:
- return isSetKey();
- case COLUMN_PARENT:
- return isSetColumn_parent();
- case COLUMN:
- return isSetColumn();
- case CONSISTENCY_LEVEL:
- return isSetConsistency_level();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof insert_args)
- return this.equals((insert_args)that);
- return false;
- }
-
- public boolean equals(insert_args that) {
- if (that == null)
- return false;
-
- boolean this_present_key = true && this.isSetKey();
- boolean that_present_key = true && that.isSetKey();
- if (this_present_key || that_present_key) {
- if (!(this_present_key && that_present_key))
- return false;
- if (!this.key.equals(that.key))
- return false;
- }
-
- boolean this_present_column_parent = true && this.isSetColumn_parent();
- boolean that_present_column_parent = true && that.isSetColumn_parent();
- if (this_present_column_parent || that_present_column_parent) {
- if (!(this_present_column_parent && that_present_column_parent))
- return false;
- if (!this.column_parent.equals(that.column_parent))
- return false;
- }
-
- boolean this_present_column = true && this.isSetColumn();
- boolean that_present_column = true && that.isSetColumn();
- if (this_present_column || that_present_column) {
- if (!(this_present_column && that_present_column))
- return false;
- if (!this.column.equals(that.column))
- return false;
- }
-
- boolean this_present_consistency_level = true && this.isSetConsistency_level();
- boolean that_present_consistency_level = true && that.isSetConsistency_level();
- if (this_present_consistency_level || that_present_consistency_level) {
- if (!(this_present_consistency_level && that_present_consistency_level))
- return false;
- if (!this.consistency_level.equals(that.consistency_level))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_key = true && (isSetKey());
- builder.append(present_key);
- if (present_key)
- builder.append(key);
-
- boolean present_column_parent = true && (isSetColumn_parent());
- builder.append(present_column_parent);
- if (present_column_parent)
- builder.append(column_parent);
-
- boolean present_column = true && (isSetColumn());
- builder.append(present_column);
- if (present_column)
- builder.append(column);
-
- boolean present_consistency_level = true && (isSetConsistency_level());
- builder.append(present_consistency_level);
- if (present_consistency_level)
- builder.append(consistency_level.getValue());
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(insert_args other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetKey()).compareTo(other.isSetKey());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetKey()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.key, other.key);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetColumn_parent()).compareTo(other.isSetColumn_parent());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetColumn_parent()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.column_parent, other.column_parent);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetColumn()).compareTo(other.isSetColumn());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetColumn()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.column, other.column);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetConsistency_level()).compareTo(other.isSetConsistency_level());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetConsistency_level()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.consistency_level, other.consistency_level);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("insert_args(");
- boolean first = true;
-
- sb.append("key:");
- if (this.key == null) {
- sb.append("null");
- } else {
- org.apache.thrift.TBaseHelper.toString(this.key, sb);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("column_parent:");
- if (this.column_parent == null) {
- sb.append("null");
- } else {
- sb.append(this.column_parent);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("column:");
- if (this.column == null) {
- sb.append("null");
- } else {
- sb.append(this.column);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("consistency_level:");
- if (this.consistency_level == null) {
- sb.append("null");
- } else {
- sb.append(this.consistency_level);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- if (key == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'key' was not present! Struct: " + toString());
- }
- if (column_parent == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'column_parent' was not present! Struct: " + toString());
- }
- if (column == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'column' was not present! Struct: " + toString());
- }
- if (consistency_level == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'consistency_level' was not present! Struct: " + toString());
- }
- // check for sub-struct validity
- if (column_parent != null) {
- column_parent.validate();
- }
- if (column != null) {
- column.validate();
- }
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class insert_argsStandardSchemeFactory implements SchemeFactory {
- public insert_argsStandardScheme getScheme() {
- return new insert_argsStandardScheme();
- }
- }
-
- private static class insert_argsStandardScheme extends StandardScheme<insert_args> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, insert_args struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // KEY
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.key = iprot.readBinary();
- struct.setKeyIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 2: // COLUMN_PARENT
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.column_parent = new ColumnParent();
- struct.column_parent.read(iprot);
- struct.setColumn_parentIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 3: // COLUMN
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.column = new Column();
- struct.column.read(iprot);
- struct.setColumnIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 4: // CONSISTENCY_LEVEL
- if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
- struct.consistency_level = ConsistencyLevel.findByValue(iprot.readI32());
- struct.setConsistency_levelIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, insert_args struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.key != null) {
- oprot.writeFieldBegin(KEY_FIELD_DESC);
- oprot.writeBinary(struct.key);
- oprot.writeFieldEnd();
- }
- if (struct.column_parent != null) {
- oprot.writeFieldBegin(COLUMN_PARENT_FIELD_DESC);
- struct.column_parent.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.column != null) {
- oprot.writeFieldBegin(COLUMN_FIELD_DESC);
- struct.column.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.consistency_level != null) {
- oprot.writeFieldBegin(CONSISTENCY_LEVEL_FIELD_DESC);
- oprot.writeI32(struct.consistency_level.getValue());
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class insert_argsTupleSchemeFactory implements SchemeFactory {
- public insert_argsTupleScheme getScheme() {
- return new insert_argsTupleScheme();
- }
- }
-
- private static class insert_argsTupleScheme extends TupleScheme<insert_args> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, insert_args struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- oprot.writeBinary(struct.key);
- struct.column_parent.write(oprot);
- struct.column.write(oprot);
- oprot.writeI32(struct.consistency_level.getValue());
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, insert_args struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- struct.key = iprot.readBinary();
- struct.setKeyIsSet(true);
- struct.column_parent = new ColumnParent();
- struct.column_parent.read(iprot);
- struct.setColumn_parentIsSet(true);
- struct.column = new Column();
- struct.column.read(iprot);
- struct.setColumnIsSet(true);
- struct.consistency_level = ConsistencyLevel.findByValue(iprot.readI32());
- struct.setConsistency_levelIsSet(true);
- }
- }
-
- }
-
- public static class insert_result implements org.apache.thrift.TBase<insert_result, insert_result._Fields>, java.io.Serializable, Cloneable, Comparable<insert_result> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("insert_result");
-
- private static final org.apache.thrift.protocol.TField IRE_FIELD_DESC = new org.apache.thrift.protocol.TField("ire", org.apache.thrift.protocol.TType.STRUCT, (short)1);
- private static final org.apache.thrift.protocol.TField UE_FIELD_DESC = new org.apache.thrift.protocol.TField("ue", org.apache.thrift.protocol.TType.STRUCT, (short)2);
- private static final org.apache.thrift.protocol.TField TE_FIELD_DESC = new org.apache.thrift.protocol.TField("te", org.apache.thrift.protocol.TType.STRUCT, (short)3);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new insert_resultStandardSchemeFactory());
- schemes.put(TupleScheme.class, new insert_resultTupleSchemeFactory());
- }
-
- public InvalidRequestException ire; // required
- public UnavailableException ue; // required
- public TimedOutException te; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- IRE((short)1, "ire"),
- UE((short)2, "ue"),
- TE((short)3, "te");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // IRE
- return IRE;
- case 2: // UE
- return UE;
- case 3: // TE
- return TE;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.IRE, new org.apache.thrift.meta_data.FieldMetaData("ire", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- tmpMap.put(_Fields.UE, new org.apache.thrift.meta_data.FieldMetaData("ue", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- tmpMap.put(_Fields.TE, new org.apache.thrift.meta_data.FieldMetaData("te", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(insert_result.class, metaDataMap);
- }
-
- public insert_result() {
- }
-
- public insert_result(
- InvalidRequestException ire,
- UnavailableException ue,
- TimedOutException te)
- {
- this();
- this.ire = ire;
- this.ue = ue;
- this.te = te;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public insert_result(insert_result other) {
- if (other.isSetIre()) {
- this.ire = new InvalidRequestException(other.ire);
- }
- if (other.isSetUe()) {
- this.ue = new UnavailableException(other.ue);
- }
- if (other.isSetTe()) {
- this.te = new TimedOutException(other.te);
- }
- }
-
- public insert_result deepCopy() {
- return new insert_result(this);
- }
-
- @Override
- public void clear() {
- this.ire = null;
- this.ue = null;
- this.te = null;
- }
-
- public InvalidRequestException getIre() {
- return this.ire;
- }
-
- public insert_result setIre(InvalidRequestException ire) {
- this.ire = ire;
- return this;
- }
-
- public void unsetIre() {
- this.ire = null;
- }
-
- /** Returns true if field ire is set (has been assigned a value) and false otherwise */
- public boolean isSetIre() {
- return this.ire != null;
- }
-
- public void setIreIsSet(boolean value) {
- if (!value) {
- this.ire = null;
- }
- }
-
- public UnavailableException getUe() {
- return this.ue;
- }
-
- public insert_result setUe(UnavailableException ue) {
- this.ue = ue;
- return this;
- }
-
- public void unsetUe() {
- this.ue = null;
- }
-
- /** Returns true if field ue is set (has been assigned a value) and false otherwise */
- public boolean isSetUe() {
- return this.ue != null;
- }
-
- public void setUeIsSet(boolean value) {
- if (!value) {
- this.ue = null;
- }
- }
-
- public TimedOutException getTe() {
- return this.te;
- }
-
- public insert_result setTe(TimedOutException te) {
- this.te = te;
- return this;
- }
-
- public void unsetTe() {
- this.te = null;
- }
-
- /** Returns true if field te is set (has been assigned a value) and false otherwise */
- public boolean isSetTe() {
- return this.te != null;
- }
-
- public void setTeIsSet(boolean value) {
- if (!value) {
- this.te = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case IRE:
- if (value == null) {
- unsetIre();
- } else {
- setIre((InvalidRequestException)value);
- }
- break;
-
- case UE:
- if (value == null) {
- unsetUe();
- } else {
- setUe((UnavailableException)value);
- }
- break;
-
- case TE:
- if (value == null) {
- unsetTe();
- } else {
- setTe((TimedOutException)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case IRE:
- return getIre();
-
- case UE:
- return getUe();
-
- case TE:
- return getTe();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case IRE:
- return isSetIre();
- case UE:
- return isSetUe();
- case TE:
- return isSetTe();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof insert_result)
- return this.equals((insert_result)that);
- return false;
- }
-
- public boolean equals(insert_result that) {
- if (that == null)
- return false;
-
- boolean this_present_ire = true && this.isSetIre();
- boolean that_present_ire = true && that.isSetIre();
- if (this_present_ire || that_present_ire) {
- if (!(this_present_ire && that_present_ire))
- return false;
- if (!this.ire.equals(that.ire))
- return false;
- }
-
- boolean this_present_ue = true && this.isSetUe();
- boolean that_present_ue = true && that.isSetUe();
- if (this_present_ue || that_present_ue) {
- if (!(this_present_ue && that_present_ue))
- return false;
- if (!this.ue.equals(that.ue))
- return false;
- }
-
- boolean this_present_te = true && this.isSetTe();
- boolean that_present_te = true && that.isSetTe();
- if (this_present_te || that_present_te) {
- if (!(this_present_te && that_present_te))
- return false;
- if (!this.te.equals(that.te))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_ire = true && (isSetIre());
- builder.append(present_ire);
- if (present_ire)
- builder.append(ire);
-
- boolean present_ue = true && (isSetUe());
- builder.append(present_ue);
- if (present_ue)
- builder.append(ue);
-
- boolean present_te = true && (isSetTe());
- builder.append(present_te);
- if (present_te)
- builder.append(te);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(insert_result other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetIre()).compareTo(other.isSetIre());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetIre()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.ire, other.ire);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetUe()).compareTo(other.isSetUe());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetUe()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.ue, other.ue);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetTe()).compareTo(other.isSetTe());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetTe()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.te, other.te);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("insert_result(");
- boolean first = true;
-
- sb.append("ire:");
- if (this.ire == null) {
- sb.append("null");
- } else {
- sb.append(this.ire);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("ue:");
- if (this.ue == null) {
- sb.append("null");
- } else {
- sb.append(this.ue);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("te:");
- if (this.te == null) {
- sb.append("null");
- } else {
- sb.append(this.te);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class insert_resultStandardSchemeFactory implements SchemeFactory {
- public insert_resultStandardScheme getScheme() {
- return new insert_resultStandardScheme();
- }
- }
-
- private static class insert_resultStandardScheme extends StandardScheme<insert_result> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, insert_result struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // IRE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.ire = new InvalidRequestException();
- struct.ire.read(iprot);
- struct.setIreIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 2: // UE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.ue = new UnavailableException();
- struct.ue.read(iprot);
- struct.setUeIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 3: // TE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.te = new TimedOutException();
- struct.te.read(iprot);
- struct.setTeIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, insert_result struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.ire != null) {
- oprot.writeFieldBegin(IRE_FIELD_DESC);
- struct.ire.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.ue != null) {
- oprot.writeFieldBegin(UE_FIELD_DESC);
- struct.ue.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.te != null) {
- oprot.writeFieldBegin(TE_FIELD_DESC);
- struct.te.write(oprot);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class insert_resultTupleSchemeFactory implements SchemeFactory {
- public insert_resultTupleScheme getScheme() {
- return new insert_resultTupleScheme();
- }
- }
-
- private static class insert_resultTupleScheme extends TupleScheme<insert_result> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, insert_result struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- BitSet optionals = new BitSet();
- if (struct.isSetIre()) {
- optionals.set(0);
- }
- if (struct.isSetUe()) {
- optionals.set(1);
- }
- if (struct.isSetTe()) {
- optionals.set(2);
- }
- oprot.writeBitSet(optionals, 3);
- if (struct.isSetIre()) {
- struct.ire.write(oprot);
- }
- if (struct.isSetUe()) {
- struct.ue.write(oprot);
- }
- if (struct.isSetTe()) {
- struct.te.write(oprot);
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, insert_result struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- BitSet incoming = iprot.readBitSet(3);
- if (incoming.get(0)) {
- struct.ire = new InvalidRequestException();
- struct.ire.read(iprot);
- struct.setIreIsSet(true);
- }
- if (incoming.get(1)) {
- struct.ue = new UnavailableException();
- struct.ue.read(iprot);
- struct.setUeIsSet(true);
- }
- if (incoming.get(2)) {
- struct.te = new TimedOutException();
- struct.te.read(iprot);
- struct.setTeIsSet(true);
- }
- }
- }
-
- }
-
- public static class add_args implements org.apache.thrift.TBase<add_args, add_args._Fields>, java.io.Serializable, Cloneable, Comparable<add_args> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("add_args");
-
- private static final org.apache.thrift.protocol.TField KEY_FIELD_DESC = new org.apache.thrift.protocol.TField("key", org.apache.thrift.protocol.TType.STRING, (short)1);
- private static final org.apache.thrift.protocol.TField COLUMN_PARENT_FIELD_DESC = new org.apache.thrift.protocol.TField("column_parent", org.apache.thrift.protocol.TType.STRUCT, (short)2);
- private static final org.apache.thrift.protocol.TField COLUMN_FIELD_DESC = new org.apache.thrift.protocol.TField("column", org.apache.thrift.protocol.TType.STRUCT, (short)3);
- private static final org.apache.thrift.protocol.TField CONSISTENCY_LEVEL_FIELD_DESC = new org.apache.thrift.protocol.TField("consistency_level", org.apache.thrift.protocol.TType.I32, (short)4);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new add_argsStandardSchemeFactory());
- schemes.put(TupleScheme.class, new add_argsTupleSchemeFactory());
- }
-
- public ByteBuffer key; // required
- public ColumnParent column_parent; // required
- public CounterColumn column; // required
- /**
- *
- * @see ConsistencyLevel
- */
- public ConsistencyLevel consistency_level; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- KEY((short)1, "key"),
- COLUMN_PARENT((short)2, "column_parent"),
- COLUMN((short)3, "column"),
- /**
- *
- * @see ConsistencyLevel
- */
- CONSISTENCY_LEVEL((short)4, "consistency_level");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // KEY
- return KEY;
- case 2: // COLUMN_PARENT
- return COLUMN_PARENT;
- case 3: // COLUMN
- return COLUMN;
- case 4: // CONSISTENCY_LEVEL
- return CONSISTENCY_LEVEL;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.KEY, new org.apache.thrift.meta_data.FieldMetaData("key", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true)));
- tmpMap.put(_Fields.COLUMN_PARENT, new org.apache.thrift.meta_data.FieldMetaData("column_parent", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, ColumnParent.class)));
- tmpMap.put(_Fields.COLUMN, new org.apache.thrift.meta_data.FieldMetaData("column", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, CounterColumn.class)));
- tmpMap.put(_Fields.CONSISTENCY_LEVEL, new org.apache.thrift.meta_data.FieldMetaData("consistency_level", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.EnumMetaData(org.apache.thrift.protocol.TType.ENUM, ConsistencyLevel.class)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(add_args.class, metaDataMap);
- }
-
- public add_args() {
- this.consistency_level = org.apache.cassandra.thrift.ConsistencyLevel.ONE;
-
- }
-
- public add_args(
- ByteBuffer key,
- ColumnParent column_parent,
- CounterColumn column,
- ConsistencyLevel consistency_level)
- {
- this();
- this.key = key;
- this.column_parent = column_parent;
- this.column = column;
- this.consistency_level = consistency_level;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public add_args(add_args other) {
- if (other.isSetKey()) {
- this.key = org.apache.thrift.TBaseHelper.copyBinary(other.key);
-;
- }
- if (other.isSetColumn_parent()) {
- this.column_parent = new ColumnParent(other.column_parent);
- }
- if (other.isSetColumn()) {
- this.column = new CounterColumn(other.column);
- }
- if (other.isSetConsistency_level()) {
- this.consistency_level = other.consistency_level;
- }
- }
-
- public add_args deepCopy() {
- return new add_args(this);
- }
-
- @Override
- public void clear() {
- this.key = null;
- this.column_parent = null;
- this.column = null;
- this.consistency_level = org.apache.cassandra.thrift.ConsistencyLevel.ONE;
-
- }
-
- public byte[] getKey() {
- setKey(org.apache.thrift.TBaseHelper.rightSize(key));
- return key == null ? null : key.array();
- }
-
- public ByteBuffer bufferForKey() {
- return key;
- }
-
- public add_args setKey(byte[] key) {
- setKey(key == null ? (ByteBuffer)null : ByteBuffer.wrap(key));
- return this;
- }
-
- public add_args setKey(ByteBuffer key) {
- this.key = key;
- return this;
- }
-
- public void unsetKey() {
- this.key = null;
- }
-
- /** Returns true if field key is set (has been assigned a value) and false otherwise */
- public boolean isSetKey() {
- return this.key != null;
- }
-
- public void setKeyIsSet(boolean value) {
- if (!value) {
- this.key = null;
- }
- }
-
- public ColumnParent getColumn_parent() {
- return this.column_parent;
- }
-
- public add_args setColumn_parent(ColumnParent column_parent) {
- this.column_parent = column_parent;
- return this;
- }
-
- public void unsetColumn_parent() {
- this.column_parent = null;
- }
-
- /** Returns true if field column_parent is set (has been assigned a value) and false otherwise */
- public boolean isSetColumn_parent() {
- return this.column_parent != null;
- }
-
- public void setColumn_parentIsSet(boolean value) {
- if (!value) {
- this.column_parent = null;
- }
- }
-
- public CounterColumn getColumn() {
- return this.column;
- }
-
- public add_args setColumn(CounterColumn column) {
- this.column = column;
- return this;
- }
-
- public void unsetColumn() {
- this.column = null;
- }
-
- /** Returns true if field column is set (has been assigned a value) and false otherwise */
- public boolean isSetColumn() {
- return this.column != null;
- }
-
- public void setColumnIsSet(boolean value) {
- if (!value) {
- this.column = null;
- }
- }
-
- /**
- *
- * @see ConsistencyLevel
- */
- public ConsistencyLevel getConsistency_level() {
- return this.consistency_level;
- }
-
- /**
- *
- * @see ConsistencyLevel
- */
- public add_args setConsistency_level(ConsistencyLevel consistency_level) {
- this.consistency_level = consistency_level;
- return this;
- }
-
- public void unsetConsistency_level() {
- this.consistency_level = null;
- }
-
- /** Returns true if field consistency_level is set (has been assigned a value) and false otherwise */
- public boolean isSetConsistency_level() {
- return this.consistency_level != null;
- }
-
- public void setConsistency_levelIsSet(boolean value) {
- if (!value) {
- this.consistency_level = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case KEY:
- if (value == null) {
- unsetKey();
- } else {
- setKey((ByteBuffer)value);
- }
- break;
-
- case COLUMN_PARENT:
- if (value == null) {
- unsetColumn_parent();
- } else {
- setColumn_parent((ColumnParent)value);
- }
- break;
-
- case COLUMN:
- if (value == null) {
- unsetColumn();
- } else {
- setColumn((CounterColumn)value);
- }
- break;
-
- case CONSISTENCY_LEVEL:
- if (value == null) {
- unsetConsistency_level();
- } else {
- setConsistency_level((ConsistencyLevel)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case KEY:
- return getKey();
-
- case COLUMN_PARENT:
- return getColumn_parent();
-
- case COLUMN:
- return getColumn();
-
- case CONSISTENCY_LEVEL:
- return getConsistency_level();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case KEY:
- return isSetKey();
- case COLUMN_PARENT:
- return isSetColumn_parent();
- case COLUMN:
- return isSetColumn();
- case CONSISTENCY_LEVEL:
- return isSetConsistency_level();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof add_args)
- return this.equals((add_args)that);
- return false;
- }
-
- public boolean equals(add_args that) {
- if (that == null)
- return false;
-
- boolean this_present_key = true && this.isSetKey();
- boolean that_present_key = true && that.isSetKey();
- if (this_present_key || that_present_key) {
- if (!(this_present_key && that_present_key))
- return false;
- if (!this.key.equals(that.key))
- return false;
- }
-
- boolean this_present_column_parent = true && this.isSetColumn_parent();
- boolean that_present_column_parent = true && that.isSetColumn_parent();
- if (this_present_column_parent || that_present_column_parent) {
- if (!(this_present_column_parent && that_present_column_parent))
- return false;
- if (!this.column_parent.equals(that.column_parent))
- return false;
- }
-
- boolean this_present_column = true && this.isSetColumn();
- boolean that_present_column = true && that.isSetColumn();
- if (this_present_column || that_present_column) {
- if (!(this_present_column && that_present_column))
- return false;
- if (!this.column.equals(that.column))
- return false;
- }
-
- boolean this_present_consistency_level = true && this.isSetConsistency_level();
- boolean that_present_consistency_level = true && that.isSetConsistency_level();
- if (this_present_consistency_level || that_present_consistency_level) {
- if (!(this_present_consistency_level && that_present_consistency_level))
- return false;
- if (!this.consistency_level.equals(that.consistency_level))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_key = true && (isSetKey());
- builder.append(present_key);
- if (present_key)
- builder.append(key);
-
- boolean present_column_parent = true && (isSetColumn_parent());
- builder.append(present_column_parent);
- if (present_column_parent)
- builder.append(column_parent);
-
- boolean present_column = true && (isSetColumn());
- builder.append(present_column);
- if (present_column)
- builder.append(column);
-
- boolean present_consistency_level = true && (isSetConsistency_level());
- builder.append(present_consistency_level);
- if (present_consistency_level)
- builder.append(consistency_level.getValue());
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(add_args other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetKey()).compareTo(other.isSetKey());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetKey()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.key, other.key);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetColumn_parent()).compareTo(other.isSetColumn_parent());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetColumn_parent()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.column_parent, other.column_parent);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetColumn()).compareTo(other.isSetColumn());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetColumn()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.column, other.column);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetConsistency_level()).compareTo(other.isSetConsistency_level());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetConsistency_level()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.consistency_level, other.consistency_level);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("add_args(");
- boolean first = true;
-
- sb.append("key:");
- if (this.key == null) {
- sb.append("null");
- } else {
- org.apache.thrift.TBaseHelper.toString(this.key, sb);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("column_parent:");
- if (this.column_parent == null) {
- sb.append("null");
- } else {
- sb.append(this.column_parent);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("column:");
- if (this.column == null) {
- sb.append("null");
- } else {
- sb.append(this.column);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("consistency_level:");
- if (this.consistency_level == null) {
- sb.append("null");
- } else {
- sb.append(this.consistency_level);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- if (key == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'key' was not present! Struct: " + toString());
- }
- if (column_parent == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'column_parent' was not present! Struct: " + toString());
- }
- if (column == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'column' was not present! Struct: " + toString());
- }
- if (consistency_level == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'consistency_level' was not present! Struct: " + toString());
- }
- // check for sub-struct validity
- if (column_parent != null) {
- column_parent.validate();
- }
- if (column != null) {
- column.validate();
- }
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class add_argsStandardSchemeFactory implements SchemeFactory {
- public add_argsStandardScheme getScheme() {
- return new add_argsStandardScheme();
- }
- }
-
- private static class add_argsStandardScheme extends StandardScheme<add_args> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, add_args struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // KEY
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.key = iprot.readBinary();
- struct.setKeyIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 2: // COLUMN_PARENT
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.column_parent = new ColumnParent();
- struct.column_parent.read(iprot);
- struct.setColumn_parentIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 3: // COLUMN
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.column = new CounterColumn();
- struct.column.read(iprot);
- struct.setColumnIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 4: // CONSISTENCY_LEVEL
- if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
- struct.consistency_level = ConsistencyLevel.findByValue(iprot.readI32());
- struct.setConsistency_levelIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, add_args struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.key != null) {
- oprot.writeFieldBegin(KEY_FIELD_DESC);
- oprot.writeBinary(struct.key);
- oprot.writeFieldEnd();
- }
- if (struct.column_parent != null) {
- oprot.writeFieldBegin(COLUMN_PARENT_FIELD_DESC);
- struct.column_parent.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.column != null) {
- oprot.writeFieldBegin(COLUMN_FIELD_DESC);
- struct.column.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.consistency_level != null) {
- oprot.writeFieldBegin(CONSISTENCY_LEVEL_FIELD_DESC);
- oprot.writeI32(struct.consistency_level.getValue());
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class add_argsTupleSchemeFactory implements SchemeFactory {
- public add_argsTupleScheme getScheme() {
- return new add_argsTupleScheme();
- }
- }
-
- private static class add_argsTupleScheme extends TupleScheme<add_args> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, add_args struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- oprot.writeBinary(struct.key);
- struct.column_parent.write(oprot);
- struct.column.write(oprot);
- oprot.writeI32(struct.consistency_level.getValue());
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, add_args struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- struct.key = iprot.readBinary();
- struct.setKeyIsSet(true);
- struct.column_parent = new ColumnParent();
- struct.column_parent.read(iprot);
- struct.setColumn_parentIsSet(true);
- struct.column = new CounterColumn();
- struct.column.read(iprot);
- struct.setColumnIsSet(true);
- struct.consistency_level = ConsistencyLevel.findByValue(iprot.readI32());
- struct.setConsistency_levelIsSet(true);
- }
- }
-
- }
-
- public static class add_result implements org.apache.thrift.TBase<add_result, add_result._Fields>, java.io.Serializable, Cloneable, Comparable<add_result> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("add_result");
-
- private static final org.apache.thrift.protocol.TField IRE_FIELD_DESC = new org.apache.thrift.protocol.TField("ire", org.apache.thrift.protocol.TType.STRUCT, (short)1);
- private static final org.apache.thrift.protocol.TField UE_FIELD_DESC = new org.apache.thrift.protocol.TField("ue", org.apache.thrift.protocol.TType.STRUCT, (short)2);
- private static final org.apache.thrift.protocol.TField TE_FIELD_DESC = new org.apache.thrift.protocol.TField("te", org.apache.thrift.protocol.TType.STRUCT, (short)3);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new add_resultStandardSchemeFactory());
- schemes.put(TupleScheme.class, new add_resultTupleSchemeFactory());
- }
-
- public InvalidRequestException ire; // required
- public UnavailableException ue; // required
- public TimedOutException te; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- IRE((short)1, "ire"),
- UE((short)2, "ue"),
- TE((short)3, "te");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // IRE
- return IRE;
- case 2: // UE
- return UE;
- case 3: // TE
- return TE;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.IRE, new org.apache.thrift.meta_data.FieldMetaData("ire", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- tmpMap.put(_Fields.UE, new org.apache.thrift.meta_data.FieldMetaData("ue", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- tmpMap.put(_Fields.TE, new org.apache.thrift.meta_data.FieldMetaData("te", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(add_result.class, metaDataMap);
- }
-
- public add_result() {
- }
-
- public add_result(
- InvalidRequestException ire,
- UnavailableException ue,
- TimedOutException te)
- {
- this();
- this.ire = ire;
- this.ue = ue;
- this.te = te;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public add_result(add_result other) {
- if (other.isSetIre()) {
- this.ire = new InvalidRequestException(other.ire);
- }
- if (other.isSetUe()) {
- this.ue = new UnavailableException(other.ue);
- }
- if (other.isSetTe()) {
- this.te = new TimedOutException(other.te);
- }
- }
-
- public add_result deepCopy() {
- return new add_result(this);
- }
-
- @Override
- public void clear() {
- this.ire = null;
- this.ue = null;
- this.te = null;
- }
-
- public InvalidRequestException getIre() {
- return this.ire;
- }
-
- public add_result setIre(InvalidRequestException ire) {
- this.ire = ire;
- return this;
- }
-
- public void unsetIre() {
- this.ire = null;
- }
-
- /** Returns true if field ire is set (has been assigned a value) and false otherwise */
- public boolean isSetIre() {
- return this.ire != null;
- }
-
- public void setIreIsSet(boolean value) {
- if (!value) {
- this.ire = null;
- }
- }
-
- public UnavailableException getUe() {
- return this.ue;
- }
-
- public add_result setUe(UnavailableException ue) {
- this.ue = ue;
- return this;
- }
-
- public void unsetUe() {
- this.ue = null;
- }
-
- /** Returns true if field ue is set (has been assigned a value) and false otherwise */
- public boolean isSetUe() {
- return this.ue != null;
- }
-
- public void setUeIsSet(boolean value) {
- if (!value) {
- this.ue = null;
- }
- }
-
- public TimedOutException getTe() {
- return this.te;
- }
-
- public add_result setTe(TimedOutException te) {
- this.te = te;
- return this;
- }
-
- public void unsetTe() {
- this.te = null;
- }
-
- /** Returns true if field te is set (has been assigned a value) and false otherwise */
- public boolean isSetTe() {
- return this.te != null;
- }
-
- public void setTeIsSet(boolean value) {
- if (!value) {
- this.te = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case IRE:
- if (value == null) {
- unsetIre();
- } else {
- setIre((InvalidRequestException)value);
- }
- break;
-
- case UE:
- if (value == null) {
- unsetUe();
- } else {
- setUe((UnavailableException)value);
- }
- break;
-
- case TE:
- if (value == null) {
- unsetTe();
- } else {
- setTe((TimedOutException)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case IRE:
- return getIre();
-
- case UE:
- return getUe();
-
- case TE:
- return getTe();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case IRE:
- return isSetIre();
- case UE:
- return isSetUe();
- case TE:
- return isSetTe();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof add_result)
- return this.equals((add_result)that);
- return false;
- }
-
- public boolean equals(add_result that) {
- if (that == null)
- return false;
-
- boolean this_present_ire = true && this.isSetIre();
- boolean that_present_ire = true && that.isSetIre();
- if (this_present_ire || that_present_ire) {
- if (!(this_present_ire && that_present_ire))
- return false;
- if (!this.ire.equals(that.ire))
- return false;
- }
-
- boolean this_present_ue = true && this.isSetUe();
- boolean that_present_ue = true && that.isSetUe();
- if (this_present_ue || that_present_ue) {
- if (!(this_present_ue && that_present_ue))
- return false;
- if (!this.ue.equals(that.ue))
- return false;
- }
-
- boolean this_present_te = true && this.isSetTe();
- boolean that_present_te = true && that.isSetTe();
- if (this_present_te || that_present_te) {
- if (!(this_present_te && that_present_te))
- return false;
- if (!this.te.equals(that.te))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_ire = true && (isSetIre());
- builder.append(present_ire);
- if (present_ire)
- builder.append(ire);
-
- boolean present_ue = true && (isSetUe());
- builder.append(present_ue);
- if (present_ue)
- builder.append(ue);
-
- boolean present_te = true && (isSetTe());
- builder.append(present_te);
- if (present_te)
- builder.append(te);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(add_result other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetIre()).compareTo(other.isSetIre());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetIre()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.ire, other.ire);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetUe()).compareTo(other.isSetUe());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetUe()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.ue, other.ue);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetTe()).compareTo(other.isSetTe());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetTe()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.te, other.te);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("add_result(");
- boolean first = true;
-
- sb.append("ire:");
- if (this.ire == null) {
- sb.append("null");
- } else {
- sb.append(this.ire);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("ue:");
- if (this.ue == null) {
- sb.append("null");
- } else {
- sb.append(this.ue);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("te:");
- if (this.te == null) {
- sb.append("null");
- } else {
- sb.append(this.te);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class add_resultStandardSchemeFactory implements SchemeFactory {
- public add_resultStandardScheme getScheme() {
- return new add_resultStandardScheme();
- }
- }
-
- private static class add_resultStandardScheme extends StandardScheme<add_result> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, add_result struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // IRE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.ire = new InvalidRequestException();
- struct.ire.read(iprot);
- struct.setIreIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 2: // UE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.ue = new UnavailableException();
- struct.ue.read(iprot);
- struct.setUeIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 3: // TE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.te = new TimedOutException();
- struct.te.read(iprot);
- struct.setTeIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, add_result struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.ire != null) {
- oprot.writeFieldBegin(IRE_FIELD_DESC);
- struct.ire.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.ue != null) {
- oprot.writeFieldBegin(UE_FIELD_DESC);
- struct.ue.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.te != null) {
- oprot.writeFieldBegin(TE_FIELD_DESC);
- struct.te.write(oprot);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class add_resultTupleSchemeFactory implements SchemeFactory {
- public add_resultTupleScheme getScheme() {
- return new add_resultTupleScheme();
- }
- }
-
- private static class add_resultTupleScheme extends TupleScheme<add_result> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, add_result struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- BitSet optionals = new BitSet();
- if (struct.isSetIre()) {
- optionals.set(0);
- }
- if (struct.isSetUe()) {
- optionals.set(1);
- }
- if (struct.isSetTe()) {
- optionals.set(2);
- }
- oprot.writeBitSet(optionals, 3);
- if (struct.isSetIre()) {
- struct.ire.write(oprot);
- }
- if (struct.isSetUe()) {
- struct.ue.write(oprot);
- }
- if (struct.isSetTe()) {
- struct.te.write(oprot);
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, add_result struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- BitSet incoming = iprot.readBitSet(3);
- if (incoming.get(0)) {
- struct.ire = new InvalidRequestException();
- struct.ire.read(iprot);
- struct.setIreIsSet(true);
- }
- if (incoming.get(1)) {
- struct.ue = new UnavailableException();
- struct.ue.read(iprot);
- struct.setUeIsSet(true);
- }
- if (incoming.get(2)) {
- struct.te = new TimedOutException();
- struct.te.read(iprot);
- struct.setTeIsSet(true);
- }
- }
- }
-
- }
-
- public static class cas_args implements org.apache.thrift.TBase<cas_args, cas_args._Fields>, java.io.Serializable, Cloneable, Comparable<cas_args> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("cas_args");
-
- private static final org.apache.thrift.protocol.TField KEY_FIELD_DESC = new org.apache.thrift.protocol.TField("key", org.apache.thrift.protocol.TType.STRING, (short)1);
- private static final org.apache.thrift.protocol.TField COLUMN_FAMILY_FIELD_DESC = new org.apache.thrift.protocol.TField("column_family", org.apache.thrift.protocol.TType.STRING, (short)2);
- private static final org.apache.thrift.protocol.TField EXPECTED_FIELD_DESC = new org.apache.thrift.protocol.TField("expected", org.apache.thrift.protocol.TType.LIST, (short)3);
- private static final org.apache.thrift.protocol.TField UPDATES_FIELD_DESC = new org.apache.thrift.protocol.TField("updates", org.apache.thrift.protocol.TType.LIST, (short)4);
- private static final org.apache.thrift.protocol.TField SERIAL_CONSISTENCY_LEVEL_FIELD_DESC = new org.apache.thrift.protocol.TField("serial_consistency_level", org.apache.thrift.protocol.TType.I32, (short)5);
- private static final org.apache.thrift.protocol.TField COMMIT_CONSISTENCY_LEVEL_FIELD_DESC = new org.apache.thrift.protocol.TField("commit_consistency_level", org.apache.thrift.protocol.TType.I32, (short)6);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new cas_argsStandardSchemeFactory());
- schemes.put(TupleScheme.class, new cas_argsTupleSchemeFactory());
- }
-
- public ByteBuffer key; // required
- public String column_family; // required
- public List<Column> expected; // required
- public List<Column> updates; // required
- /**
- *
- * @see ConsistencyLevel
- */
- public ConsistencyLevel serial_consistency_level; // required
- /**
- *
- * @see ConsistencyLevel
- */
- public ConsistencyLevel commit_consistency_level; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- KEY((short)1, "key"),
- COLUMN_FAMILY((short)2, "column_family"),
- EXPECTED((short)3, "expected"),
- UPDATES((short)4, "updates"),
- /**
- *
- * @see ConsistencyLevel
- */
- SERIAL_CONSISTENCY_LEVEL((short)5, "serial_consistency_level"),
- /**
- *
- * @see ConsistencyLevel
- */
- COMMIT_CONSISTENCY_LEVEL((short)6, "commit_consistency_level");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // KEY
- return KEY;
- case 2: // COLUMN_FAMILY
- return COLUMN_FAMILY;
- case 3: // EXPECTED
- return EXPECTED;
- case 4: // UPDATES
- return UPDATES;
- case 5: // SERIAL_CONSISTENCY_LEVEL
- return SERIAL_CONSISTENCY_LEVEL;
- case 6: // COMMIT_CONSISTENCY_LEVEL
- return COMMIT_CONSISTENCY_LEVEL;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.KEY, new org.apache.thrift.meta_data.FieldMetaData("key", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true)));
- tmpMap.put(_Fields.COLUMN_FAMILY, new org.apache.thrift.meta_data.FieldMetaData("column_family", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
- tmpMap.put(_Fields.EXPECTED, new org.apache.thrift.meta_data.FieldMetaData("expected", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST,
- new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, Column.class))));
- tmpMap.put(_Fields.UPDATES, new org.apache.thrift.meta_data.FieldMetaData("updates", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST,
- new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, Column.class))));
- tmpMap.put(_Fields.SERIAL_CONSISTENCY_LEVEL, new org.apache.thrift.meta_data.FieldMetaData("serial_consistency_level", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.EnumMetaData(org.apache.thrift.protocol.TType.ENUM, ConsistencyLevel.class)));
- tmpMap.put(_Fields.COMMIT_CONSISTENCY_LEVEL, new org.apache.thrift.meta_data.FieldMetaData("commit_consistency_level", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.EnumMetaData(org.apache.thrift.protocol.TType.ENUM, ConsistencyLevel.class)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(cas_args.class, metaDataMap);
- }
-
- public cas_args() {
- this.serial_consistency_level = org.apache.cassandra.thrift.ConsistencyLevel.SERIAL;
-
- this.commit_consistency_level = org.apache.cassandra.thrift.ConsistencyLevel.QUORUM;
-
- }
-
- public cas_args(
- ByteBuffer key,
- String column_family,
- List<Column> expected,
- List<Column> updates,
- ConsistencyLevel serial_consistency_level,
- ConsistencyLevel commit_consistency_level)
- {
- this();
- this.key = key;
- this.column_family = column_family;
- this.expected = expected;
- this.updates = updates;
- this.serial_consistency_level = serial_consistency_level;
- this.commit_consistency_level = commit_consistency_level;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public cas_args(cas_args other) {
- if (other.isSetKey()) {
- this.key = org.apache.thrift.TBaseHelper.copyBinary(other.key);
-;
- }
- if (other.isSetColumn_family()) {
- this.column_family = other.column_family;
- }
- if (other.isSetExpected()) {
- List<Column> __this__expected = new ArrayList<Column>(other.expected.size());
- for (Column other_element : other.expected) {
- __this__expected.add(new Column(other_element));
- }
- this.expected = __this__expected;
- }
- if (other.isSetUpdates()) {
- List<Column> __this__updates = new ArrayList<Column>(other.updates.size());
- for (Column other_element : other.updates) {
- __this__updates.add(new Column(other_element));
- }
- this.updates = __this__updates;
- }
- if (other.isSetSerial_consistency_level()) {
- this.serial_consistency_level = other.serial_consistency_level;
- }
- if (other.isSetCommit_consistency_level()) {
- this.commit_consistency_level = other.commit_consistency_level;
- }
- }
-
- public cas_args deepCopy() {
- return new cas_args(this);
- }
-
- @Override
- public void clear() {
- this.key = null;
- this.column_family = null;
- this.expected = null;
- this.updates = null;
- this.serial_consistency_level = org.apache.cassandra.thrift.ConsistencyLevel.SERIAL;
-
- this.commit_consistency_level = org.apache.cassandra.thrift.ConsistencyLevel.QUORUM;
-
- }
-
- public byte[] getKey() {
- setKey(org.apache.thrift.TBaseHelper.rightSize(key));
- return key == null ? null : key.array();
- }
-
- public ByteBuffer bufferForKey() {
- return key;
- }
-
- public cas_args setKey(byte[] key) {
- setKey(key == null ? (ByteBuffer)null : ByteBuffer.wrap(key));
- return this;
- }
-
- public cas_args setKey(ByteBuffer key) {
- this.key = key;
- return this;
- }
-
- public void unsetKey() {
- this.key = null;
- }
-
- /** Returns true if field key is set (has been assigned a value) and false otherwise */
- public boolean isSetKey() {
- return this.key != null;
- }
-
- public void setKeyIsSet(boolean value) {
- if (!value) {
- this.key = null;
- }
- }
-
- public String getColumn_family() {
- return this.column_family;
- }
-
- public cas_args setColumn_family(String column_family) {
- this.column_family = column_family;
- return this;
- }
-
- public void unsetColumn_family() {
- this.column_family = null;
- }
-
- /** Returns true if field column_family is set (has been assigned a value) and false otherwise */
- public boolean isSetColumn_family() {
- return this.column_family != null;
- }
-
- public void setColumn_familyIsSet(boolean value) {
- if (!value) {
- this.column_family = null;
- }
- }
-
- public int getExpectedSize() {
- return (this.expected == null) ? 0 : this.expected.size();
- }
-
- public java.util.Iterator<Column> getExpectedIterator() {
- return (this.expected == null) ? null : this.expected.iterator();
- }
-
- public void addToExpected(Column elem) {
- if (this.expected == null) {
- this.expected = new ArrayList<Column>();
- }
- this.expected.add(elem);
- }
-
- public List<Column> getExpected() {
- return this.expected;
- }
-
- public cas_args setExpected(List<Column> expected) {
- this.expected = expected;
- return this;
- }
-
- public void unsetExpected() {
- this.expected = null;
- }
-
- /** Returns true if field expected is set (has been assigned a value) and false otherwise */
- public boolean isSetExpected() {
- return this.expected != null;
- }
-
- public void setExpectedIsSet(boolean value) {
- if (!value) {
- this.expected = null;
- }
- }
-
- public int getUpdatesSize() {
- return (this.updates == null) ? 0 : this.updates.size();
- }
-
- public java.util.Iterator<Column> getUpdatesIterator() {
- return (this.updates == null) ? null : this.updates.iterator();
- }
-
- public void addToUpdates(Column elem) {
- if (this.updates == null) {
- this.updates = new ArrayList<Column>();
- }
- this.updates.add(elem);
- }
-
- public List<Column> getUpdates() {
- return this.updates;
- }
-
- public cas_args setUpdates(List<Column> updates) {
- this.updates = updates;
- return this;
- }
-
- public void unsetUpdates() {
- this.updates = null;
- }
-
- /** Returns true if field updates is set (has been assigned a value) and false otherwise */
- public boolean isSetUpdates() {
- return this.updates != null;
- }
-
- public void setUpdatesIsSet(boolean value) {
- if (!value) {
- this.updates = null;
- }
- }
-
- /**
- *
- * @see ConsistencyLevel
- */
- public ConsistencyLevel getSerial_consistency_level() {
- return this.serial_consistency_level;
- }
-
- /**
- *
- * @see ConsistencyLevel
- */
- public cas_args setSerial_consistency_level(ConsistencyLevel serial_consistency_level) {
- this.serial_consistency_level = serial_consistency_level;
- return this;
- }
-
- public void unsetSerial_consistency_level() {
- this.serial_consistency_level = null;
- }
-
- /** Returns true if field serial_consistency_level is set (has been assigned a value) and false otherwise */
- public boolean isSetSerial_consistency_level() {
- return this.serial_consistency_level != null;
- }
-
- public void setSerial_consistency_levelIsSet(boolean value) {
- if (!value) {
- this.serial_consistency_level = null;
- }
- }
-
- /**
- *
- * @see ConsistencyLevel
- */
- public ConsistencyLevel getCommit_consistency_level() {
- return this.commit_consistency_level;
- }
-
- /**
- *
- * @see ConsistencyLevel
- */
- public cas_args setCommit_consistency_level(ConsistencyLevel commit_consistency_level) {
- this.commit_consistency_level = commit_consistency_level;
- return this;
- }
-
- public void unsetCommit_consistency_level() {
- this.commit_consistency_level = null;
- }
-
- /** Returns true if field commit_consistency_level is set (has been assigned a value) and false otherwise */
- public boolean isSetCommit_consistency_level() {
- return this.commit_consistency_level != null;
- }
-
- public void setCommit_consistency_levelIsSet(boolean value) {
- if (!value) {
- this.commit_consistency_level = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case KEY:
- if (value == null) {
- unsetKey();
- } else {
- setKey((ByteBuffer)value);
- }
- break;
-
- case COLUMN_FAMILY:
- if (value == null) {
- unsetColumn_family();
- } else {
- setColumn_family((String)value);
- }
- break;
-
- case EXPECTED:
- if (value == null) {
- unsetExpected();
- } else {
- setExpected((List<Column>)value);
- }
- break;
-
- case UPDATES:
- if (value == null) {
- unsetUpdates();
- } else {
- setUpdates((List<Column>)value);
- }
- break;
-
- case SERIAL_CONSISTENCY_LEVEL:
- if (value == null) {
- unsetSerial_consistency_level();
- } else {
- setSerial_consistency_level((ConsistencyLevel)value);
- }
- break;
-
- case COMMIT_CONSISTENCY_LEVEL:
- if (value == null) {
- unsetCommit_consistency_level();
- } else {
- setCommit_consistency_level((ConsistencyLevel)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case KEY:
- return getKey();
-
- case COLUMN_FAMILY:
- return getColumn_family();
-
- case EXPECTED:
- return getExpected();
-
- case UPDATES:
- return getUpdates();
-
- case SERIAL_CONSISTENCY_LEVEL:
- return getSerial_consistency_level();
-
- case COMMIT_CONSISTENCY_LEVEL:
- return getCommit_consistency_level();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case KEY:
- return isSetKey();
- case COLUMN_FAMILY:
- return isSetColumn_family();
- case EXPECTED:
- return isSetExpected();
- case UPDATES:
- return isSetUpdates();
- case SERIAL_CONSISTENCY_LEVEL:
- return isSetSerial_consistency_level();
- case COMMIT_CONSISTENCY_LEVEL:
- return isSetCommit_consistency_level();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof cas_args)
- return this.equals((cas_args)that);
- return false;
- }
-
- public boolean equals(cas_args that) {
- if (that == null)
- return false;
-
- boolean this_present_key = true && this.isSetKey();
- boolean that_present_key = true && that.isSetKey();
- if (this_present_key || that_present_key) {
- if (!(this_present_key && that_present_key))
- return false;
- if (!this.key.equals(that.key))
- return false;
- }
-
- boolean this_present_column_family = true && this.isSetColumn_family();
- boolean that_present_column_family = true && that.isSetColumn_family();
- if (this_present_column_family || that_present_column_family) {
- if (!(this_present_column_family && that_present_column_family))
- return false;
- if (!this.column_family.equals(that.column_family))
- return false;
- }
-
- boolean this_present_expected = true && this.isSetExpected();
- boolean that_present_expected = true && that.isSetExpected();
- if (this_present_expected || that_present_expected) {
- if (!(this_present_expected && that_present_expected))
- return false;
- if (!this.expected.equals(that.expected))
- return false;
- }
-
- boolean this_present_updates = true && this.isSetUpdates();
- boolean that_present_updates = true && that.isSetUpdates();
- if (this_present_updates || that_present_updates) {
- if (!(this_present_updates && that_present_updates))
- return false;
- if (!this.updates.equals(that.updates))
- return false;
- }
-
- boolean this_present_serial_consistency_level = true && this.isSetSerial_consistency_level();
- boolean that_present_serial_consistency_level = true && that.isSetSerial_consistency_level();
- if (this_present_serial_consistency_level || that_present_serial_consistency_level) {
- if (!(this_present_serial_consistency_level && that_present_serial_consistency_level))
- return false;
- if (!this.serial_consistency_level.equals(that.serial_consistency_level))
- return false;
- }
-
- boolean this_present_commit_consistency_level = true && this.isSetCommit_consistency_level();
- boolean that_present_commit_consistency_level = true && that.isSetCommit_consistency_level();
- if (this_present_commit_consistency_level || that_present_commit_consistency_level) {
- if (!(this_present_commit_consistency_level && that_present_commit_consistency_level))
- return false;
- if (!this.commit_consistency_level.equals(that.commit_consistency_level))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_key = true && (isSetKey());
- builder.append(present_key);
- if (present_key)
- builder.append(key);
-
- boolean present_column_family = true && (isSetColumn_family());
- builder.append(present_column_family);
- if (present_column_family)
- builder.append(column_family);
-
- boolean present_expected = true && (isSetExpected());
- builder.append(present_expected);
- if (present_expected)
- builder.append(expected);
-
- boolean present_updates = true && (isSetUpdates());
- builder.append(present_updates);
- if (present_updates)
- builder.append(updates);
-
- boolean present_serial_consistency_level = true && (isSetSerial_consistency_level());
- builder.append(present_serial_consistency_level);
- if (present_serial_consistency_level)
- builder.append(serial_consistency_level.getValue());
-
- boolean present_commit_consistency_level = true && (isSetCommit_consistency_level());
- builder.append(present_commit_consistency_level);
- if (present_commit_consistency_level)
- builder.append(commit_consistency_level.getValue());
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(cas_args other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetKey()).compareTo(other.isSetKey());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetKey()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.key, other.key);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetColumn_family()).compareTo(other.isSetColumn_family());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetColumn_family()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.column_family, other.column_family);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetExpected()).compareTo(other.isSetExpected());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetExpected()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.expected, other.expected);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetUpdates()).compareTo(other.isSetUpdates());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetUpdates()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.updates, other.updates);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetSerial_consistency_level()).compareTo(other.isSetSerial_consistency_level());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetSerial_consistency_level()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.serial_consistency_level, other.serial_consistency_level);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetCommit_consistency_level()).compareTo(other.isSetCommit_consistency_level());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetCommit_consistency_level()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.commit_consistency_level, other.commit_consistency_level);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("cas_args(");
- boolean first = true;
-
- sb.append("key:");
- if (this.key == null) {
- sb.append("null");
- } else {
- org.apache.thrift.TBaseHelper.toString(this.key, sb);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("column_family:");
- if (this.column_family == null) {
- sb.append("null");
- } else {
- sb.append(this.column_family);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("expected:");
- if (this.expected == null) {
- sb.append("null");
- } else {
- sb.append(this.expected);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("updates:");
- if (this.updates == null) {
- sb.append("null");
- } else {
- sb.append(this.updates);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("serial_consistency_level:");
- if (this.serial_consistency_level == null) {
- sb.append("null");
- } else {
- sb.append(this.serial_consistency_level);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("commit_consistency_level:");
- if (this.commit_consistency_level == null) {
- sb.append("null");
- } else {
- sb.append(this.commit_consistency_level);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- if (key == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'key' was not present! Struct: " + toString());
- }
- if (column_family == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'column_family' was not present! Struct: " + toString());
- }
- if (serial_consistency_level == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'serial_consistency_level' was not present! Struct: " + toString());
- }
- if (commit_consistency_level == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'commit_consistency_level' was not present! Struct: " + toString());
- }
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class cas_argsStandardSchemeFactory implements SchemeFactory {
- public cas_argsStandardScheme getScheme() {
- return new cas_argsStandardScheme();
- }
- }
-
- private static class cas_argsStandardScheme extends StandardScheme<cas_args> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, cas_args struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // KEY
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.key = iprot.readBinary();
- struct.setKeyIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 2: // COLUMN_FAMILY
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.column_family = iprot.readString();
- struct.setColumn_familyIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 3: // EXPECTED
- if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
- {
- org.apache.thrift.protocol.TList _list300 = iprot.readListBegin();
- struct.expected = new ArrayList<Column>(_list300.size);
- for (int _i301 = 0; _i301 < _list300.size; ++_i301)
- {
- Column _elem302;
- _elem302 = new Column();
- _elem302.read(iprot);
- struct.expected.add(_elem302);
- }
- iprot.readListEnd();
- }
- struct.setExpectedIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 4: // UPDATES
- if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
- {
- org.apache.thrift.protocol.TList _list303 = iprot.readListBegin();
- struct.updates = new ArrayList<Column>(_list303.size);
- for (int _i304 = 0; _i304 < _list303.size; ++_i304)
- {
- Column _elem305;
- _elem305 = new Column();
- _elem305.read(iprot);
- struct.updates.add(_elem305);
- }
- iprot.readListEnd();
- }
- struct.setUpdatesIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 5: // SERIAL_CONSISTENCY_LEVEL
- if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
- struct.serial_consistency_level = ConsistencyLevel.findByValue(iprot.readI32());
- struct.setSerial_consistency_levelIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 6: // COMMIT_CONSISTENCY_LEVEL
- if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
- struct.commit_consistency_level = ConsistencyLevel.findByValue(iprot.readI32());
- struct.setCommit_consistency_levelIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, cas_args struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.key != null) {
- oprot.writeFieldBegin(KEY_FIELD_DESC);
- oprot.writeBinary(struct.key);
- oprot.writeFieldEnd();
- }
- if (struct.column_family != null) {
- oprot.writeFieldBegin(COLUMN_FAMILY_FIELD_DESC);
- oprot.writeString(struct.column_family);
- oprot.writeFieldEnd();
- }
- if (struct.expected != null) {
- oprot.writeFieldBegin(EXPECTED_FIELD_DESC);
- {
- oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.expected.size()));
- for (Column _iter306 : struct.expected)
- {
- _iter306.write(oprot);
- }
- oprot.writeListEnd();
- }
- oprot.writeFieldEnd();
- }
- if (struct.updates != null) {
- oprot.writeFieldBegin(UPDATES_FIELD_DESC);
- {
- oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.updates.size()));
- for (Column _iter307 : struct.updates)
- {
- _iter307.write(oprot);
- }
- oprot.writeListEnd();
- }
- oprot.writeFieldEnd();
- }
- if (struct.serial_consistency_level != null) {
- oprot.writeFieldBegin(SERIAL_CONSISTENCY_LEVEL_FIELD_DESC);
- oprot.writeI32(struct.serial_consistency_level.getValue());
- oprot.writeFieldEnd();
- }
- if (struct.commit_consistency_level != null) {
- oprot.writeFieldBegin(COMMIT_CONSISTENCY_LEVEL_FIELD_DESC);
- oprot.writeI32(struct.commit_consistency_level.getValue());
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class cas_argsTupleSchemeFactory implements SchemeFactory {
- public cas_argsTupleScheme getScheme() {
- return new cas_argsTupleScheme();
- }
- }
-
- private static class cas_argsTupleScheme extends TupleScheme<cas_args> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, cas_args struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- oprot.writeBinary(struct.key);
- oprot.writeString(struct.column_family);
- oprot.writeI32(struct.serial_consistency_level.getValue());
- oprot.writeI32(struct.commit_consistency_level.getValue());
- BitSet optionals = new BitSet();
- if (struct.isSetExpected()) {
- optionals.set(0);
- }
- if (struct.isSetUpdates()) {
- optionals.set(1);
- }
- oprot.writeBitSet(optionals, 2);
- if (struct.isSetExpected()) {
- {
- oprot.writeI32(struct.expected.size());
- for (Column _iter308 : struct.expected)
- {
- _iter308.write(oprot);
- }
- }
- }
- if (struct.isSetUpdates()) {
- {
- oprot.writeI32(struct.updates.size());
- for (Column _iter309 : struct.updates)
- {
- _iter309.write(oprot);
- }
- }
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, cas_args struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- struct.key = iprot.readBinary();
- struct.setKeyIsSet(true);
- struct.column_family = iprot.readString();
- struct.setColumn_familyIsSet(true);
- struct.serial_consistency_level = ConsistencyLevel.findByValue(iprot.readI32());
- struct.setSerial_consistency_levelIsSet(true);
- struct.commit_consistency_level = ConsistencyLevel.findByValue(iprot.readI32());
- struct.setCommit_consistency_levelIsSet(true);
- BitSet incoming = iprot.readBitSet(2);
- if (incoming.get(0)) {
- {
- org.apache.thrift.protocol.TList _list310 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
- struct.expected = new ArrayList<Column>(_list310.size);
- for (int _i311 = 0; _i311 < _list310.size; ++_i311)
- {
- Column _elem312;
- _elem312 = new Column();
- _elem312.read(iprot);
- struct.expected.add(_elem312);
- }
- }
- struct.setExpectedIsSet(true);
- }
- if (incoming.get(1)) {
- {
- org.apache.thrift.protocol.TList _list313 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
- struct.updates = new ArrayList<Column>(_list313.size);
- for (int _i314 = 0; _i314 < _list313.size; ++_i314)
- {
- Column _elem315;
- _elem315 = new Column();
- _elem315.read(iprot);
- struct.updates.add(_elem315);
- }
- }
- struct.setUpdatesIsSet(true);
- }
- }
- }
-
- }
-
- public static class cas_result implements org.apache.thrift.TBase<cas_result, cas_result._Fields>, java.io.Serializable, Cloneable, Comparable<cas_result> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("cas_result");
-
- private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRUCT, (short)0);
- private static final org.apache.thrift.protocol.TField IRE_FIELD_DESC = new org.apache.thrift.protocol.TField("ire", org.apache.thrift.protocol.TType.STRUCT, (short)1);
- private static final org.apache.thrift.protocol.TField UE_FIELD_DESC = new org.apache.thrift.protocol.TField("ue", org.apache.thrift.protocol.TType.STRUCT, (short)2);
- private static final org.apache.thrift.protocol.TField TE_FIELD_DESC = new org.apache.thrift.protocol.TField("te", org.apache.thrift.protocol.TType.STRUCT, (short)3);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new cas_resultStandardSchemeFactory());
- schemes.put(TupleScheme.class, new cas_resultTupleSchemeFactory());
- }
-
- public CASResult success; // required
- public InvalidRequestException ire; // required
- public UnavailableException ue; // required
- public TimedOutException te; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- SUCCESS((short)0, "success"),
- IRE((short)1, "ire"),
- UE((short)2, "ue"),
- TE((short)3, "te");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 0: // SUCCESS
- return SUCCESS;
- case 1: // IRE
- return IRE;
- case 2: // UE
- return UE;
- case 3: // TE
- return TE;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, CASResult.class)));
- tmpMap.put(_Fields.IRE, new org.apache.thrift.meta_data.FieldMetaData("ire", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- tmpMap.put(_Fields.UE, new org.apache.thrift.meta_data.FieldMetaData("ue", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- tmpMap.put(_Fields.TE, new org.apache.thrift.meta_data.FieldMetaData("te", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(cas_result.class, metaDataMap);
- }
-
- public cas_result() {
- }
-
- public cas_result(
- CASResult success,
- InvalidRequestException ire,
- UnavailableException ue,
- TimedOutException te)
- {
- this();
- this.success = success;
- this.ire = ire;
- this.ue = ue;
- this.te = te;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public cas_result(cas_result other) {
- if (other.isSetSuccess()) {
- this.success = new CASResult(other.success);
- }
- if (other.isSetIre()) {
- this.ire = new InvalidRequestException(other.ire);
- }
- if (other.isSetUe()) {
- this.ue = new UnavailableException(other.ue);
- }
- if (other.isSetTe()) {
- this.te = new TimedOutException(other.te);
- }
- }
-
- public cas_result deepCopy() {
- return new cas_result(this);
- }
-
- @Override
- public void clear() {
- this.success = null;
- this.ire = null;
- this.ue = null;
- this.te = null;
- }
-
- public CASResult getSuccess() {
- return this.success;
- }
-
- public cas_result setSuccess(CASResult success) {
- this.success = success;
- return this;
- }
-
- public void unsetSuccess() {
- this.success = null;
- }
-
- /** Returns true if field success is set (has been assigned a value) and false otherwise */
- public boolean isSetSuccess() {
- return this.success != null;
- }
-
- public void setSuccessIsSet(boolean value) {
- if (!value) {
- this.success = null;
- }
- }
-
- public InvalidRequestException getIre() {
- return this.ire;
- }
-
- public cas_result setIre(InvalidRequestException ire) {
- this.ire = ire;
- return this;
- }
-
- public void unsetIre() {
- this.ire = null;
- }
-
- /** Returns true if field ire is set (has been assigned a value) and false otherwise */
- public boolean isSetIre() {
- return this.ire != null;
- }
-
- public void setIreIsSet(boolean value) {
- if (!value) {
- this.ire = null;
- }
- }
-
- public UnavailableException getUe() {
- return this.ue;
- }
-
- public cas_result setUe(UnavailableException ue) {
- this.ue = ue;
- return this;
- }
-
- public void unsetUe() {
- this.ue = null;
- }
-
- /** Returns true if field ue is set (has been assigned a value) and false otherwise */
- public boolean isSetUe() {
- return this.ue != null;
- }
-
- public void setUeIsSet(boolean value) {
- if (!value) {
- this.ue = null;
- }
- }
-
- public TimedOutException getTe() {
- return this.te;
- }
-
- public cas_result setTe(TimedOutException te) {
- this.te = te;
- return this;
- }
-
- public void unsetTe() {
- this.te = null;
- }
-
- /** Returns true if field te is set (has been assigned a value) and false otherwise */
- public boolean isSetTe() {
- return this.te != null;
- }
-
- public void setTeIsSet(boolean value) {
- if (!value) {
- this.te = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case SUCCESS:
- if (value == null) {
- unsetSuccess();
- } else {
- setSuccess((CASResult)value);
- }
- break;
-
- case IRE:
- if (value == null) {
- unsetIre();
- } else {
- setIre((InvalidRequestException)value);
- }
- break;
-
- case UE:
- if (value == null) {
- unsetUe();
- } else {
- setUe((UnavailableException)value);
- }
- break;
-
- case TE:
- if (value == null) {
- unsetTe();
- } else {
- setTe((TimedOutException)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case SUCCESS:
- return getSuccess();
-
- case IRE:
- return getIre();
-
- case UE:
- return getUe();
-
- case TE:
- return getTe();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case SUCCESS:
- return isSetSuccess();
- case IRE:
- return isSetIre();
- case UE:
- return isSetUe();
- case TE:
- return isSetTe();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof cas_result)
- return this.equals((cas_result)that);
- return false;
- }
-
- public boolean equals(cas_result that) {
- if (that == null)
- return false;
-
- boolean this_present_success = true && this.isSetSuccess();
- boolean that_present_success = true && that.isSetSuccess();
- if (this_present_success || that_present_success) {
- if (!(this_present_success && that_present_success))
- return false;
- if (!this.success.equals(that.success))
- return false;
- }
-
- boolean this_present_ire = true && this.isSetIre();
- boolean that_present_ire = true && that.isSetIre();
- if (this_present_ire || that_present_ire) {
- if (!(this_present_ire && that_present_ire))
- return false;
- if (!this.ire.equals(that.ire))
- return false;
- }
-
- boolean this_present_ue = true && this.isSetUe();
- boolean that_present_ue = true && that.isSetUe();
- if (this_present_ue || that_present_ue) {
- if (!(this_present_ue && that_present_ue))
- return false;
- if (!this.ue.equals(that.ue))
- return false;
- }
-
- boolean this_present_te = true && this.isSetTe();
- boolean that_present_te = true && that.isSetTe();
- if (this_present_te || that_present_te) {
- if (!(this_present_te && that_present_te))
- return false;
- if (!this.te.equals(that.te))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_success = true && (isSetSuccess());
- builder.append(present_success);
- if (present_success)
- builder.append(success);
-
- boolean present_ire = true && (isSetIre());
- builder.append(present_ire);
- if (present_ire)
- builder.append(ire);
-
- boolean present_ue = true && (isSetUe());
- builder.append(present_ue);
- if (present_ue)
- builder.append(ue);
-
- boolean present_te = true && (isSetTe());
- builder.append(present_te);
- if (present_te)
- builder.append(te);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(cas_result other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(other.isSetSuccess());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetSuccess()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, other.success);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetIre()).compareTo(other.isSetIre());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetIre()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.ire, other.ire);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetUe()).compareTo(other.isSetUe());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetUe()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.ue, other.ue);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetTe()).compareTo(other.isSetTe());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetTe()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.te, other.te);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("cas_result(");
- boolean first = true;
-
- sb.append("success:");
- if (this.success == null) {
- sb.append("null");
- } else {
- sb.append(this.success);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("ire:");
- if (this.ire == null) {
- sb.append("null");
- } else {
- sb.append(this.ire);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("ue:");
- if (this.ue == null) {
- sb.append("null");
- } else {
- sb.append(this.ue);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("te:");
- if (this.te == null) {
- sb.append("null");
- } else {
- sb.append(this.te);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // check for sub-struct validity
- if (success != null) {
- success.validate();
- }
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class cas_resultStandardSchemeFactory implements SchemeFactory {
- public cas_resultStandardScheme getScheme() {
- return new cas_resultStandardScheme();
- }
- }
-
- private static class cas_resultStandardScheme extends StandardScheme<cas_result> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, cas_result struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 0: // SUCCESS
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.success = new CASResult();
- struct.success.read(iprot);
- struct.setSuccessIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 1: // IRE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.ire = new InvalidRequestException();
- struct.ire.read(iprot);
- struct.setIreIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 2: // UE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.ue = new UnavailableException();
- struct.ue.read(iprot);
- struct.setUeIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 3: // TE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.te = new TimedOutException();
- struct.te.read(iprot);
- struct.setTeIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, cas_result struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.success != null) {
- oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
- struct.success.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.ire != null) {
- oprot.writeFieldBegin(IRE_FIELD_DESC);
- struct.ire.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.ue != null) {
- oprot.writeFieldBegin(UE_FIELD_DESC);
- struct.ue.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.te != null) {
- oprot.writeFieldBegin(TE_FIELD_DESC);
- struct.te.write(oprot);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class cas_resultTupleSchemeFactory implements SchemeFactory {
- public cas_resultTupleScheme getScheme() {
- return new cas_resultTupleScheme();
- }
- }
-
- private static class cas_resultTupleScheme extends TupleScheme<cas_result> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, cas_result struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- BitSet optionals = new BitSet();
- if (struct.isSetSuccess()) {
- optionals.set(0);
- }
- if (struct.isSetIre()) {
- optionals.set(1);
- }
- if (struct.isSetUe()) {
- optionals.set(2);
- }
- if (struct.isSetTe()) {
- optionals.set(3);
- }
- oprot.writeBitSet(optionals, 4);
- if (struct.isSetSuccess()) {
- struct.success.write(oprot);
- }
- if (struct.isSetIre()) {
- struct.ire.write(oprot);
- }
- if (struct.isSetUe()) {
- struct.ue.write(oprot);
- }
- if (struct.isSetTe()) {
- struct.te.write(oprot);
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, cas_result struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- BitSet incoming = iprot.readBitSet(4);
- if (incoming.get(0)) {
- struct.success = new CASResult();
- struct.success.read(iprot);
- struct.setSuccessIsSet(true);
- }
- if (incoming.get(1)) {
- struct.ire = new InvalidRequestException();
- struct.ire.read(iprot);
- struct.setIreIsSet(true);
- }
- if (incoming.get(2)) {
- struct.ue = new UnavailableException();
- struct.ue.read(iprot);
- struct.setUeIsSet(true);
- }
- if (incoming.get(3)) {
- struct.te = new TimedOutException();
- struct.te.read(iprot);
- struct.setTeIsSet(true);
- }
- }
- }
-
- }
-
- public static class remove_args implements org.apache.thrift.TBase<remove_args, remove_args._Fields>, java.io.Serializable, Cloneable, Comparable<remove_args> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("remove_args");
-
- private static final org.apache.thrift.protocol.TField KEY_FIELD_DESC = new org.apache.thrift.protocol.TField("key", org.apache.thrift.protocol.TType.STRING, (short)1);
- private static final org.apache.thrift.protocol.TField COLUMN_PATH_FIELD_DESC = new org.apache.thrift.protocol.TField("column_path", org.apache.thrift.protocol.TType.STRUCT, (short)2);
- private static final org.apache.thrift.protocol.TField TIMESTAMP_FIELD_DESC = new org.apache.thrift.protocol.TField("timestamp", org.apache.thrift.protocol.TType.I64, (short)3);
- private static final org.apache.thrift.protocol.TField CONSISTENCY_LEVEL_FIELD_DESC = new org.apache.thrift.protocol.TField("consistency_level", org.apache.thrift.protocol.TType.I32, (short)4);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new remove_argsStandardSchemeFactory());
- schemes.put(TupleScheme.class, new remove_argsTupleSchemeFactory());
- }
-
- public ByteBuffer key; // required
- public ColumnPath column_path; // required
- public long timestamp; // required
- /**
- *
- * @see ConsistencyLevel
- */
- public ConsistencyLevel consistency_level; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- KEY((short)1, "key"),
- COLUMN_PATH((short)2, "column_path"),
- TIMESTAMP((short)3, "timestamp"),
- /**
- *
- * @see ConsistencyLevel
- */
- CONSISTENCY_LEVEL((short)4, "consistency_level");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // KEY
- return KEY;
- case 2: // COLUMN_PATH
- return COLUMN_PATH;
- case 3: // TIMESTAMP
- return TIMESTAMP;
- case 4: // CONSISTENCY_LEVEL
- return CONSISTENCY_LEVEL;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- private static final int __TIMESTAMP_ISSET_ID = 0;
- private byte __isset_bitfield = 0;
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.KEY, new org.apache.thrift.meta_data.FieldMetaData("key", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true)));
- tmpMap.put(_Fields.COLUMN_PATH, new org.apache.thrift.meta_data.FieldMetaData("column_path", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, ColumnPath.class)));
- tmpMap.put(_Fields.TIMESTAMP, new org.apache.thrift.meta_data.FieldMetaData("timestamp", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64)));
- tmpMap.put(_Fields.CONSISTENCY_LEVEL, new org.apache.thrift.meta_data.FieldMetaData("consistency_level", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.EnumMetaData(org.apache.thrift.protocol.TType.ENUM, ConsistencyLevel.class)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(remove_args.class, metaDataMap);
- }
-
- public remove_args() {
- this.consistency_level = org.apache.cassandra.thrift.ConsistencyLevel.ONE;
-
- }
-
- public remove_args(
- ByteBuffer key,
- ColumnPath column_path,
- long timestamp,
- ConsistencyLevel consistency_level)
- {
- this();
- this.key = key;
- this.column_path = column_path;
- this.timestamp = timestamp;
- setTimestampIsSet(true);
- this.consistency_level = consistency_level;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public remove_args(remove_args other) {
- __isset_bitfield = other.__isset_bitfield;
- if (other.isSetKey()) {
- this.key = org.apache.thrift.TBaseHelper.copyBinary(other.key);
-;
- }
- if (other.isSetColumn_path()) {
- this.column_path = new ColumnPath(other.column_path);
- }
- this.timestamp = other.timestamp;
- if (other.isSetConsistency_level()) {
- this.consistency_level = other.consistency_level;
- }
- }
-
- public remove_args deepCopy() {
- return new remove_args(this);
- }
-
- @Override
- public void clear() {
- this.key = null;
- this.column_path = null;
- setTimestampIsSet(false);
- this.timestamp = 0;
- this.consistency_level = org.apache.cassandra.thrift.ConsistencyLevel.ONE;
-
- }
-
- public byte[] getKey() {
- setKey(org.apache.thrift.TBaseHelper.rightSize(key));
- return key == null ? null : key.array();
- }
-
- public ByteBuffer bufferForKey() {
- return key;
- }
-
- public remove_args setKey(byte[] key) {
- setKey(key == null ? (ByteBuffer)null : ByteBuffer.wrap(key));
- return this;
- }
-
- public remove_args setKey(ByteBuffer key) {
- this.key = key;
- return this;
- }
-
- public void unsetKey() {
- this.key = null;
- }
-
- /** Returns true if field key is set (has been assigned a value) and false otherwise */
- public boolean isSetKey() {
- return this.key != null;
- }
-
- public void setKeyIsSet(boolean value) {
- if (!value) {
- this.key = null;
- }
- }
-
- public ColumnPath getColumn_path() {
- return this.column_path;
- }
-
- public remove_args setColumn_path(ColumnPath column_path) {
- this.column_path = column_path;
- return this;
- }
-
- public void unsetColumn_path() {
- this.column_path = null;
- }
-
- /** Returns true if field column_path is set (has been assigned a value) and false otherwise */
- public boolean isSetColumn_path() {
- return this.column_path != null;
- }
-
- public void setColumn_pathIsSet(boolean value) {
- if (!value) {
- this.column_path = null;
- }
- }
-
- public long getTimestamp() {
- return this.timestamp;
- }
-
- public remove_args setTimestamp(long timestamp) {
- this.timestamp = timestamp;
- setTimestampIsSet(true);
- return this;
- }
-
- public void unsetTimestamp() {
- __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __TIMESTAMP_ISSET_ID);
- }
-
- /** Returns true if field timestamp is set (has been assigned a value) and false otherwise */
- public boolean isSetTimestamp() {
- return EncodingUtils.testBit(__isset_bitfield, __TIMESTAMP_ISSET_ID);
- }
-
- public void setTimestampIsSet(boolean value) {
- __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __TIMESTAMP_ISSET_ID, value);
- }
-
- /**
- *
- * @see ConsistencyLevel
- */
- public ConsistencyLevel getConsistency_level() {
- return this.consistency_level;
- }
-
- /**
- *
- * @see ConsistencyLevel
- */
- public remove_args setConsistency_level(ConsistencyLevel consistency_level) {
- this.consistency_level = consistency_level;
- return this;
- }
-
- public void unsetConsistency_level() {
- this.consistency_level = null;
- }
-
- /** Returns true if field consistency_level is set (has been assigned a value) and false otherwise */
- public boolean isSetConsistency_level() {
- return this.consistency_level != null;
- }
-
- public void setConsistency_levelIsSet(boolean value) {
- if (!value) {
- this.consistency_level = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case KEY:
- if (value == null) {
- unsetKey();
- } else {
- setKey((ByteBuffer)value);
- }
- break;
-
- case COLUMN_PATH:
- if (value == null) {
- unsetColumn_path();
- } else {
- setColumn_path((ColumnPath)value);
- }
- break;
-
- case TIMESTAMP:
- if (value == null) {
- unsetTimestamp();
- } else {
- setTimestamp((Long)value);
- }
- break;
-
- case CONSISTENCY_LEVEL:
- if (value == null) {
- unsetConsistency_level();
- } else {
- setConsistency_level((ConsistencyLevel)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case KEY:
- return getKey();
-
- case COLUMN_PATH:
- return getColumn_path();
-
- case TIMESTAMP:
- return Long.valueOf(getTimestamp());
-
- case CONSISTENCY_LEVEL:
- return getConsistency_level();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case KEY:
- return isSetKey();
- case COLUMN_PATH:
- return isSetColumn_path();
- case TIMESTAMP:
- return isSetTimestamp();
- case CONSISTENCY_LEVEL:
- return isSetConsistency_level();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof remove_args)
- return this.equals((remove_args)that);
- return false;
- }
-
- public boolean equals(remove_args that) {
- if (that == null)
- return false;
-
- boolean this_present_key = true && this.isSetKey();
- boolean that_present_key = true && that.isSetKey();
- if (this_present_key || that_present_key) {
- if (!(this_present_key && that_present_key))
- return false;
- if (!this.key.equals(that.key))
- return false;
- }
-
- boolean this_present_column_path = true && this.isSetColumn_path();
- boolean that_present_column_path = true && that.isSetColumn_path();
- if (this_present_column_path || that_present_column_path) {
- if (!(this_present_column_path && that_present_column_path))
- return false;
- if (!this.column_path.equals(that.column_path))
- return false;
- }
-
- boolean this_present_timestamp = true;
- boolean that_present_timestamp = true;
- if (this_present_timestamp || that_present_timestamp) {
- if (!(this_present_timestamp && that_present_timestamp))
- return false;
- if (this.timestamp != that.timestamp)
- return false;
- }
-
- boolean this_present_consistency_level = true && this.isSetConsistency_level();
- boolean that_present_consistency_level = true && that.isSetConsistency_level();
- if (this_present_consistency_level || that_present_consistency_level) {
- if (!(this_present_consistency_level && that_present_consistency_level))
- return false;
- if (!this.consistency_level.equals(that.consistency_level))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_key = true && (isSetKey());
- builder.append(present_key);
- if (present_key)
- builder.append(key);
-
- boolean present_column_path = true && (isSetColumn_path());
- builder.append(present_column_path);
- if (present_column_path)
- builder.append(column_path);
-
- boolean present_timestamp = true;
- builder.append(present_timestamp);
- if (present_timestamp)
- builder.append(timestamp);
-
- boolean present_consistency_level = true && (isSetConsistency_level());
- builder.append(present_consistency_level);
- if (present_consistency_level)
- builder.append(consistency_level.getValue());
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(remove_args other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetKey()).compareTo(other.isSetKey());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetKey()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.key, other.key);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetColumn_path()).compareTo(other.isSetColumn_path());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetColumn_path()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.column_path, other.column_path);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetTimestamp()).compareTo(other.isSetTimestamp());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetTimestamp()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.timestamp, other.timestamp);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetConsistency_level()).compareTo(other.isSetConsistency_level());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetConsistency_level()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.consistency_level, other.consistency_level);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("remove_args(");
- boolean first = true;
-
- sb.append("key:");
- if (this.key == null) {
- sb.append("null");
- } else {
- org.apache.thrift.TBaseHelper.toString(this.key, sb);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("column_path:");
- if (this.column_path == null) {
- sb.append("null");
- } else {
- sb.append(this.column_path);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("timestamp:");
- sb.append(this.timestamp);
- first = false;
- if (!first) sb.append(", ");
- sb.append("consistency_level:");
- if (this.consistency_level == null) {
- sb.append("null");
- } else {
- sb.append(this.consistency_level);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- if (key == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'key' was not present! Struct: " + toString());
- }
- if (column_path == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'column_path' was not present! Struct: " + toString());
- }
- // alas, we cannot check 'timestamp' because it's a primitive and you chose the non-beans generator.
- // check for sub-struct validity
- if (column_path != null) {
- column_path.validate();
- }
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
- __isset_bitfield = 0;
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class remove_argsStandardSchemeFactory implements SchemeFactory {
- public remove_argsStandardScheme getScheme() {
- return new remove_argsStandardScheme();
- }
- }
-
- private static class remove_argsStandardScheme extends StandardScheme<remove_args> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, remove_args struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // KEY
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.key = iprot.readBinary();
- struct.setKeyIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 2: // COLUMN_PATH
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.column_path = new ColumnPath();
- struct.column_path.read(iprot);
- struct.setColumn_pathIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 3: // TIMESTAMP
- if (schemeField.type == org.apache.thrift.protocol.TType.I64) {
- struct.timestamp = iprot.readI64();
- struct.setTimestampIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 4: // CONSISTENCY_LEVEL
- if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
- struct.consistency_level = ConsistencyLevel.findByValue(iprot.readI32());
- struct.setConsistency_levelIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- if (!struct.isSetTimestamp()) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'timestamp' was not found in serialized data! Struct: " + toString());
- }
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, remove_args struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.key != null) {
- oprot.writeFieldBegin(KEY_FIELD_DESC);
- oprot.writeBinary(struct.key);
- oprot.writeFieldEnd();
- }
- if (struct.column_path != null) {
- oprot.writeFieldBegin(COLUMN_PATH_FIELD_DESC);
- struct.column_path.write(oprot);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldBegin(TIMESTAMP_FIELD_DESC);
- oprot.writeI64(struct.timestamp);
- oprot.writeFieldEnd();
- if (struct.consistency_level != null) {
- oprot.writeFieldBegin(CONSISTENCY_LEVEL_FIELD_DESC);
- oprot.writeI32(struct.consistency_level.getValue());
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class remove_argsTupleSchemeFactory implements SchemeFactory {
- public remove_argsTupleScheme getScheme() {
- return new remove_argsTupleScheme();
- }
- }
-
- private static class remove_argsTupleScheme extends TupleScheme<remove_args> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, remove_args struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- oprot.writeBinary(struct.key);
- struct.column_path.write(oprot);
- oprot.writeI64(struct.timestamp);
- BitSet optionals = new BitSet();
- if (struct.isSetConsistency_level()) {
- optionals.set(0);
- }
- oprot.writeBitSet(optionals, 1);
- if (struct.isSetConsistency_level()) {
- oprot.writeI32(struct.consistency_level.getValue());
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, remove_args struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- struct.key = iprot.readBinary();
- struct.setKeyIsSet(true);
- struct.column_path = new ColumnPath();
- struct.column_path.read(iprot);
- struct.setColumn_pathIsSet(true);
- struct.timestamp = iprot.readI64();
- struct.setTimestampIsSet(true);
- BitSet incoming = iprot.readBitSet(1);
- if (incoming.get(0)) {
- struct.consistency_level = ConsistencyLevel.findByValue(iprot.readI32());
- struct.setConsistency_levelIsSet(true);
- }
- }
- }
-
- }
-
- public static class remove_result implements org.apache.thrift.TBase<remove_result, remove_result._Fields>, java.io.Serializable, Cloneable, Comparable<remove_result> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("remove_result");
-
- private static final org.apache.thrift.protocol.TField IRE_FIELD_DESC = new org.apache.thrift.protocol.TField("ire", org.apache.thrift.protocol.TType.STRUCT, (short)1);
- private static final org.apache.thrift.protocol.TField UE_FIELD_DESC = new org.apache.thrift.protocol.TField("ue", org.apache.thrift.protocol.TType.STRUCT, (short)2);
- private static final org.apache.thrift.protocol.TField TE_FIELD_DESC = new org.apache.thrift.protocol.TField("te", org.apache.thrift.protocol.TType.STRUCT, (short)3);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new remove_resultStandardSchemeFactory());
- schemes.put(TupleScheme.class, new remove_resultTupleSchemeFactory());
- }
-
- public InvalidRequestException ire; // required
- public UnavailableException ue; // required
- public TimedOutException te; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- IRE((short)1, "ire"),
- UE((short)2, "ue"),
- TE((short)3, "te");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // IRE
- return IRE;
- case 2: // UE
- return UE;
- case 3: // TE
- return TE;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.IRE, new org.apache.thrift.meta_data.FieldMetaData("ire", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- tmpMap.put(_Fields.UE, new org.apache.thrift.meta_data.FieldMetaData("ue", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- tmpMap.put(_Fields.TE, new org.apache.thrift.meta_data.FieldMetaData("te", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(remove_result.class, metaDataMap);
- }
-
- public remove_result() {
- }
-
- public remove_result(
- InvalidRequestException ire,
- UnavailableException ue,
- TimedOutException te)
- {
- this();
- this.ire = ire;
- this.ue = ue;
- this.te = te;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public remove_result(remove_result other) {
- if (other.isSetIre()) {
- this.ire = new InvalidRequestException(other.ire);
- }
- if (other.isSetUe()) {
- this.ue = new UnavailableException(other.ue);
- }
- if (other.isSetTe()) {
- this.te = new TimedOutException(other.te);
- }
- }
-
- public remove_result deepCopy() {
- return new remove_result(this);
- }
-
- @Override
- public void clear() {
- this.ire = null;
- this.ue = null;
- this.te = null;
- }
-
- public InvalidRequestException getIre() {
- return this.ire;
- }
-
- public remove_result setIre(InvalidRequestException ire) {
- this.ire = ire;
- return this;
- }
-
- public void unsetIre() {
- this.ire = null;
- }
-
- /** Returns true if field ire is set (has been assigned a value) and false otherwise */
- public boolean isSetIre() {
- return this.ire != null;
- }
-
- public void setIreIsSet(boolean value) {
- if (!value) {
- this.ire = null;
- }
- }
-
- public UnavailableException getUe() {
- return this.ue;
- }
-
- public remove_result setUe(UnavailableException ue) {
- this.ue = ue;
- return this;
- }
-
- public void unsetUe() {
- this.ue = null;
- }
-
- /** Returns true if field ue is set (has been assigned a value) and false otherwise */
- public boolean isSetUe() {
- return this.ue != null;
- }
-
- public void setUeIsSet(boolean value) {
- if (!value) {
- this.ue = null;
- }
- }
-
- public TimedOutException getTe() {
- return this.te;
- }
-
- public remove_result setTe(TimedOutException te) {
- this.te = te;
- return this;
- }
-
- public void unsetTe() {
- this.te = null;
- }
-
- /** Returns true if field te is set (has been assigned a value) and false otherwise */
- public boolean isSetTe() {
- return this.te != null;
- }
-
- public void setTeIsSet(boolean value) {
- if (!value) {
- this.te = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case IRE:
- if (value == null) {
- unsetIre();
- } else {
- setIre((InvalidRequestException)value);
- }
- break;
-
- case UE:
- if (value == null) {
- unsetUe();
- } else {
- setUe((UnavailableException)value);
- }
- break;
-
- case TE:
- if (value == null) {
- unsetTe();
- } else {
- setTe((TimedOutException)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case IRE:
- return getIre();
-
- case UE:
- return getUe();
-
- case TE:
- return getTe();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case IRE:
- return isSetIre();
- case UE:
- return isSetUe();
- case TE:
- return isSetTe();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof remove_result)
- return this.equals((remove_result)that);
- return false;
- }
-
- public boolean equals(remove_result that) {
- if (that == null)
- return false;
-
- boolean this_present_ire = true && this.isSetIre();
- boolean that_present_ire = true && that.isSetIre();
- if (this_present_ire || that_present_ire) {
- if (!(this_present_ire && that_present_ire))
- return false;
- if (!this.ire.equals(that.ire))
- return false;
- }
-
- boolean this_present_ue = true && this.isSetUe();
- boolean that_present_ue = true && that.isSetUe();
- if (this_present_ue || that_present_ue) {
- if (!(this_present_ue && that_present_ue))
- return false;
- if (!this.ue.equals(that.ue))
- return false;
- }
-
- boolean this_present_te = true && this.isSetTe();
- boolean that_present_te = true && that.isSetTe();
- if (this_present_te || that_present_te) {
- if (!(this_present_te && that_present_te))
- return false;
- if (!this.te.equals(that.te))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_ire = true && (isSetIre());
- builder.append(present_ire);
- if (present_ire)
- builder.append(ire);
-
- boolean present_ue = true && (isSetUe());
- builder.append(present_ue);
- if (present_ue)
- builder.append(ue);
-
- boolean present_te = true && (isSetTe());
- builder.append(present_te);
- if (present_te)
- builder.append(te);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(remove_result other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetIre()).compareTo(other.isSetIre());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetIre()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.ire, other.ire);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetUe()).compareTo(other.isSetUe());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetUe()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.ue, other.ue);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetTe()).compareTo(other.isSetTe());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetTe()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.te, other.te);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("remove_result(");
- boolean first = true;
-
- sb.append("ire:");
- if (this.ire == null) {
- sb.append("null");
- } else {
- sb.append(this.ire);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("ue:");
- if (this.ue == null) {
- sb.append("null");
- } else {
- sb.append(this.ue);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("te:");
- if (this.te == null) {
- sb.append("null");
- } else {
- sb.append(this.te);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class remove_resultStandardSchemeFactory implements SchemeFactory {
- public remove_resultStandardScheme getScheme() {
- return new remove_resultStandardScheme();
- }
- }
-
- private static class remove_resultStandardScheme extends StandardScheme<remove_result> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, remove_result struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // IRE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.ire = new InvalidRequestException();
- struct.ire.read(iprot);
- struct.setIreIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 2: // UE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.ue = new UnavailableException();
- struct.ue.read(iprot);
- struct.setUeIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 3: // TE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.te = new TimedOutException();
- struct.te.read(iprot);
- struct.setTeIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, remove_result struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.ire != null) {
- oprot.writeFieldBegin(IRE_FIELD_DESC);
- struct.ire.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.ue != null) {
- oprot.writeFieldBegin(UE_FIELD_DESC);
- struct.ue.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.te != null) {
- oprot.writeFieldBegin(TE_FIELD_DESC);
- struct.te.write(oprot);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class remove_resultTupleSchemeFactory implements SchemeFactory {
- public remove_resultTupleScheme getScheme() {
- return new remove_resultTupleScheme();
- }
- }
-
- private static class remove_resultTupleScheme extends TupleScheme<remove_result> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, remove_result struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- BitSet optionals = new BitSet();
- if (struct.isSetIre()) {
- optionals.set(0);
- }
- if (struct.isSetUe()) {
- optionals.set(1);
- }
- if (struct.isSetTe()) {
- optionals.set(2);
- }
- oprot.writeBitSet(optionals, 3);
- if (struct.isSetIre()) {
- struct.ire.write(oprot);
- }
- if (struct.isSetUe()) {
- struct.ue.write(oprot);
- }
- if (struct.isSetTe()) {
- struct.te.write(oprot);
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, remove_result struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- BitSet incoming = iprot.readBitSet(3);
- if (incoming.get(0)) {
- struct.ire = new InvalidRequestException();
- struct.ire.read(iprot);
- struct.setIreIsSet(true);
- }
- if (incoming.get(1)) {
- struct.ue = new UnavailableException();
- struct.ue.read(iprot);
- struct.setUeIsSet(true);
- }
- if (incoming.get(2)) {
- struct.te = new TimedOutException();
- struct.te.read(iprot);
- struct.setTeIsSet(true);
- }
- }
- }
-
- }
-
- public static class remove_counter_args implements org.apache.thrift.TBase<remove_counter_args, remove_counter_args._Fields>, java.io.Serializable, Cloneable, Comparable<remove_counter_args> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("remove_counter_args");
-
- private static final org.apache.thrift.protocol.TField KEY_FIELD_DESC = new org.apache.thrift.protocol.TField("key", org.apache.thrift.protocol.TType.STRING, (short)1);
- private static final org.apache.thrift.protocol.TField PATH_FIELD_DESC = new org.apache.thrift.protocol.TField("path", org.apache.thrift.protocol.TType.STRUCT, (short)2);
- private static final org.apache.thrift.protocol.TField CONSISTENCY_LEVEL_FIELD_DESC = new org.apache.thrift.protocol.TField("consistency_level", org.apache.thrift.protocol.TType.I32, (short)3);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new remove_counter_argsStandardSchemeFactory());
- schemes.put(TupleScheme.class, new remove_counter_argsTupleSchemeFactory());
- }
-
- public ByteBuffer key; // required
- public ColumnPath path; // required
- /**
- *
- * @see ConsistencyLevel
- */
- public ConsistencyLevel consistency_level; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- KEY((short)1, "key"),
- PATH((short)2, "path"),
- /**
- *
- * @see ConsistencyLevel
- */
- CONSISTENCY_LEVEL((short)3, "consistency_level");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // KEY
- return KEY;
- case 2: // PATH
- return PATH;
- case 3: // CONSISTENCY_LEVEL
- return CONSISTENCY_LEVEL;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.KEY, new org.apache.thrift.meta_data.FieldMetaData("key", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true)));
- tmpMap.put(_Fields.PATH, new org.apache.thrift.meta_data.FieldMetaData("path", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, ColumnPath.class)));
- tmpMap.put(_Fields.CONSISTENCY_LEVEL, new org.apache.thrift.meta_data.FieldMetaData("consistency_level", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.EnumMetaData(org.apache.thrift.protocol.TType.ENUM, ConsistencyLevel.class)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(remove_counter_args.class, metaDataMap);
- }
-
- public remove_counter_args() {
- this.consistency_level = org.apache.cassandra.thrift.ConsistencyLevel.ONE;
-
- }
-
- public remove_counter_args(
- ByteBuffer key,
- ColumnPath path,
- ConsistencyLevel consistency_level)
- {
- this();
- this.key = key;
- this.path = path;
- this.consistency_level = consistency_level;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public remove_counter_args(remove_counter_args other) {
- if (other.isSetKey()) {
- this.key = org.apache.thrift.TBaseHelper.copyBinary(other.key);
-;
- }
- if (other.isSetPath()) {
- this.path = new ColumnPath(other.path);
- }
- if (other.isSetConsistency_level()) {
- this.consistency_level = other.consistency_level;
- }
- }
-
- public remove_counter_args deepCopy() {
- return new remove_counter_args(this);
- }
-
- @Override
- public void clear() {
- this.key = null;
- this.path = null;
- this.consistency_level = org.apache.cassandra.thrift.ConsistencyLevel.ONE;
-
- }
-
- public byte[] getKey() {
- setKey(org.apache.thrift.TBaseHelper.rightSize(key));
- return key == null ? null : key.array();
- }
-
- public ByteBuffer bufferForKey() {
- return key;
- }
-
- public remove_counter_args setKey(byte[] key) {
- setKey(key == null ? (ByteBuffer)null : ByteBuffer.wrap(key));
- return this;
- }
-
- public remove_counter_args setKey(ByteBuffer key) {
- this.key = key;
- return this;
- }
-
- public void unsetKey() {
- this.key = null;
- }
-
- /** Returns true if field key is set (has been assigned a value) and false otherwise */
- public boolean isSetKey() {
- return this.key != null;
- }
-
- public void setKeyIsSet(boolean value) {
- if (!value) {
- this.key = null;
- }
- }
-
- public ColumnPath getPath() {
- return this.path;
- }
-
- public remove_counter_args setPath(ColumnPath path) {
- this.path = path;
- return this;
- }
-
- public void unsetPath() {
- this.path = null;
- }
-
- /** Returns true if field path is set (has been assigned a value) and false otherwise */
- public boolean isSetPath() {
- return this.path != null;
- }
-
- public void setPathIsSet(boolean value) {
- if (!value) {
- this.path = null;
- }
- }
-
- /**
- *
- * @see ConsistencyLevel
- */
- public ConsistencyLevel getConsistency_level() {
- return this.consistency_level;
- }
-
- /**
- *
- * @see ConsistencyLevel
- */
- public remove_counter_args setConsistency_level(ConsistencyLevel consistency_level) {
- this.consistency_level = consistency_level;
- return this;
- }
-
- public void unsetConsistency_level() {
- this.consistency_level = null;
- }
-
- /** Returns true if field consistency_level is set (has been assigned a value) and false otherwise */
- public boolean isSetConsistency_level() {
- return this.consistency_level != null;
- }
-
- public void setConsistency_levelIsSet(boolean value) {
- if (!value) {
- this.consistency_level = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case KEY:
- if (value == null) {
- unsetKey();
- } else {
- setKey((ByteBuffer)value);
- }
- break;
-
- case PATH:
- if (value == null) {
- unsetPath();
- } else {
- setPath((ColumnPath)value);
- }
- break;
-
- case CONSISTENCY_LEVEL:
- if (value == null) {
- unsetConsistency_level();
- } else {
- setConsistency_level((ConsistencyLevel)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case KEY:
- return getKey();
-
- case PATH:
- return getPath();
-
- case CONSISTENCY_LEVEL:
- return getConsistency_level();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case KEY:
- return isSetKey();
- case PATH:
- return isSetPath();
- case CONSISTENCY_LEVEL:
- return isSetConsistency_level();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof remove_counter_args)
- return this.equals((remove_counter_args)that);
- return false;
- }
-
- public boolean equals(remove_counter_args that) {
- if (that == null)
- return false;
-
- boolean this_present_key = true && this.isSetKey();
- boolean that_present_key = true && that.isSetKey();
- if (this_present_key || that_present_key) {
- if (!(this_present_key && that_present_key))
- return false;
- if (!this.key.equals(that.key))
- return false;
- }
-
- boolean this_present_path = true && this.isSetPath();
- boolean that_present_path = true && that.isSetPath();
- if (this_present_path || that_present_path) {
- if (!(this_present_path && that_present_path))
- return false;
- if (!this.path.equals(that.path))
- return false;
- }
-
- boolean this_present_consistency_level = true && this.isSetConsistency_level();
- boolean that_present_consistency_level = true && that.isSetConsistency_level();
- if (this_present_consistency_level || that_present_consistency_level) {
- if (!(this_present_consistency_level && that_present_consistency_level))
- return false;
- if (!this.consistency_level.equals(that.consistency_level))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_key = true && (isSetKey());
- builder.append(present_key);
- if (present_key)
- builder.append(key);
-
- boolean present_path = true && (isSetPath());
- builder.append(present_path);
- if (present_path)
- builder.append(path);
-
- boolean present_consistency_level = true && (isSetConsistency_level());
- builder.append(present_consistency_level);
- if (present_consistency_level)
- builder.append(consistency_level.getValue());
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(remove_counter_args other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetKey()).compareTo(other.isSetKey());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetKey()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.key, other.key);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetPath()).compareTo(other.isSetPath());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetPath()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.path, other.path);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetConsistency_level()).compareTo(other.isSetConsistency_level());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetConsistency_level()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.consistency_level, other.consistency_level);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("remove_counter_args(");
- boolean first = true;
-
- sb.append("key:");
- if (this.key == null) {
- sb.append("null");
- } else {
- org.apache.thrift.TBaseHelper.toString(this.key, sb);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("path:");
- if (this.path == null) {
- sb.append("null");
- } else {
- sb.append(this.path);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("consistency_level:");
- if (this.consistency_level == null) {
- sb.append("null");
- } else {
- sb.append(this.consistency_level);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- if (key == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'key' was not present! Struct: " + toString());
- }
- if (path == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'path' was not present! Struct: " + toString());
- }
- if (consistency_level == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'consistency_level' was not present! Struct: " + toString());
- }
- // check for sub-struct validity
- if (path != null) {
- path.validate();
- }
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class remove_counter_argsStandardSchemeFactory implements SchemeFactory {
- public remove_counter_argsStandardScheme getScheme() {
- return new remove_counter_argsStandardScheme();
- }
- }
-
- private static class remove_counter_argsStandardScheme extends StandardScheme<remove_counter_args> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, remove_counter_args struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // KEY
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.key = iprot.readBinary();
- struct.setKeyIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 2: // PATH
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.path = new ColumnPath();
- struct.path.read(iprot);
- struct.setPathIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 3: // CONSISTENCY_LEVEL
- if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
- struct.consistency_level = ConsistencyLevel.findByValue(iprot.readI32());
- struct.setConsistency_levelIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, remove_counter_args struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.key != null) {
- oprot.writeFieldBegin(KEY_FIELD_DESC);
- oprot.writeBinary(struct.key);
- oprot.writeFieldEnd();
- }
- if (struct.path != null) {
- oprot.writeFieldBegin(PATH_FIELD_DESC);
- struct.path.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.consistency_level != null) {
- oprot.writeFieldBegin(CONSISTENCY_LEVEL_FIELD_DESC);
- oprot.writeI32(struct.consistency_level.getValue());
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class remove_counter_argsTupleSchemeFactory implements SchemeFactory {
- public remove_counter_argsTupleScheme getScheme() {
- return new remove_counter_argsTupleScheme();
- }
- }
-
- private static class remove_counter_argsTupleScheme extends TupleScheme<remove_counter_args> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, remove_counter_args struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- oprot.writeBinary(struct.key);
- struct.path.write(oprot);
- oprot.writeI32(struct.consistency_level.getValue());
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, remove_counter_args struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- struct.key = iprot.readBinary();
- struct.setKeyIsSet(true);
- struct.path = new ColumnPath();
- struct.path.read(iprot);
- struct.setPathIsSet(true);
- struct.consistency_level = ConsistencyLevel.findByValue(iprot.readI32());
- struct.setConsistency_levelIsSet(true);
- }
- }
-
- }
-
- public static class remove_counter_result implements org.apache.thrift.TBase<remove_counter_result, remove_counter_result._Fields>, java.io.Serializable, Cloneable, Comparable<remove_counter_result> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("remove_counter_result");
-
- private static final org.apache.thrift.protocol.TField IRE_FIELD_DESC = new org.apache.thrift.protocol.TField("ire", org.apache.thrift.protocol.TType.STRUCT, (short)1);
- private static final org.apache.thrift.protocol.TField UE_FIELD_DESC = new org.apache.thrift.protocol.TField("ue", org.apache.thrift.protocol.TType.STRUCT, (short)2);
- private static final org.apache.thrift.protocol.TField TE_FIELD_DESC = new org.apache.thrift.protocol.TField("te", org.apache.thrift.protocol.TType.STRUCT, (short)3);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new remove_counter_resultStandardSchemeFactory());
- schemes.put(TupleScheme.class, new remove_counter_resultTupleSchemeFactory());
- }
-
- public InvalidRequestException ire; // required
- public UnavailableException ue; // required
- public TimedOutException te; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- IRE((short)1, "ire"),
- UE((short)2, "ue"),
- TE((short)3, "te");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // IRE
- return IRE;
- case 2: // UE
- return UE;
- case 3: // TE
- return TE;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.IRE, new org.apache.thrift.meta_data.FieldMetaData("ire", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- tmpMap.put(_Fields.UE, new org.apache.thrift.meta_data.FieldMetaData("ue", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- tmpMap.put(_Fields.TE, new org.apache.thrift.meta_data.FieldMetaData("te", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(remove_counter_result.class, metaDataMap);
- }
-
- public remove_counter_result() {
- }
-
- public remove_counter_result(
- InvalidRequestException ire,
- UnavailableException ue,
- TimedOutException te)
- {
- this();
- this.ire = ire;
- this.ue = ue;
- this.te = te;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public remove_counter_result(remove_counter_result other) {
- if (other.isSetIre()) {
- this.ire = new InvalidRequestException(other.ire);
- }
- if (other.isSetUe()) {
- this.ue = new UnavailableException(other.ue);
- }
- if (other.isSetTe()) {
- this.te = new TimedOutException(other.te);
- }
- }
-
- public remove_counter_result deepCopy() {
- return new remove_counter_result(this);
- }
-
- @Override
- public void clear() {
- this.ire = null;
- this.ue = null;
- this.te = null;
- }
-
- public InvalidRequestException getIre() {
- return this.ire;
- }
-
- public remove_counter_result setIre(InvalidRequestException ire) {
- this.ire = ire;
- return this;
- }
-
- public void unsetIre() {
- this.ire = null;
- }
-
- /** Returns true if field ire is set (has been assigned a value) and false otherwise */
- public boolean isSetIre() {
- return this.ire != null;
- }
-
- public void setIreIsSet(boolean value) {
- if (!value) {
- this.ire = null;
- }
- }
-
- public UnavailableException getUe() {
- return this.ue;
- }
-
- public remove_counter_result setUe(UnavailableException ue) {
- this.ue = ue;
- return this;
- }
-
- public void unsetUe() {
- this.ue = null;
- }
-
- /** Returns true if field ue is set (has been assigned a value) and false otherwise */
- public boolean isSetUe() {
- return this.ue != null;
- }
-
- public void setUeIsSet(boolean value) {
- if (!value) {
- this.ue = null;
- }
- }
-
- public TimedOutException getTe() {
- return this.te;
- }
-
- public remove_counter_result setTe(TimedOutException te) {
- this.te = te;
- return this;
- }
-
- public void unsetTe() {
- this.te = null;
- }
-
- /** Returns true if field te is set (has been assigned a value) and false otherwise */
- public boolean isSetTe() {
- return this.te != null;
- }
-
- public void setTeIsSet(boolean value) {
- if (!value) {
- this.te = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case IRE:
- if (value == null) {
- unsetIre();
- } else {
- setIre((InvalidRequestException)value);
- }
- break;
-
- case UE:
- if (value == null) {
- unsetUe();
- } else {
- setUe((UnavailableException)value);
- }
- break;
-
- case TE:
- if (value == null) {
- unsetTe();
- } else {
- setTe((TimedOutException)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case IRE:
- return getIre();
-
- case UE:
- return getUe();
-
- case TE:
- return getTe();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case IRE:
- return isSetIre();
- case UE:
- return isSetUe();
- case TE:
- return isSetTe();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof remove_counter_result)
- return this.equals((remove_counter_result)that);
- return false;
- }
-
- public boolean equals(remove_counter_result that) {
- if (that == null)
- return false;
-
- boolean this_present_ire = true && this.isSetIre();
- boolean that_present_ire = true && that.isSetIre();
- if (this_present_ire || that_present_ire) {
- if (!(this_present_ire && that_present_ire))
- return false;
- if (!this.ire.equals(that.ire))
- return false;
- }
-
- boolean this_present_ue = true && this.isSetUe();
- boolean that_present_ue = true && that.isSetUe();
- if (this_present_ue || that_present_ue) {
- if (!(this_present_ue && that_present_ue))
- return false;
- if (!this.ue.equals(that.ue))
- return false;
- }
-
- boolean this_present_te = true && this.isSetTe();
- boolean that_present_te = true && that.isSetTe();
- if (this_present_te || that_present_te) {
- if (!(this_present_te && that_present_te))
- return false;
- if (!this.te.equals(that.te))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_ire = true && (isSetIre());
- builder.append(present_ire);
- if (present_ire)
- builder.append(ire);
-
- boolean present_ue = true && (isSetUe());
- builder.append(present_ue);
- if (present_ue)
- builder.append(ue);
-
- boolean present_te = true && (isSetTe());
- builder.append(present_te);
- if (present_te)
- builder.append(te);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(remove_counter_result other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetIre()).compareTo(other.isSetIre());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetIre()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.ire, other.ire);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetUe()).compareTo(other.isSetUe());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetUe()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.ue, other.ue);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetTe()).compareTo(other.isSetTe());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetTe()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.te, other.te);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("remove_counter_result(");
- boolean first = true;
-
- sb.append("ire:");
- if (this.ire == null) {
- sb.append("null");
- } else {
- sb.append(this.ire);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("ue:");
- if (this.ue == null) {
- sb.append("null");
- } else {
- sb.append(this.ue);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("te:");
- if (this.te == null) {
- sb.append("null");
- } else {
- sb.append(this.te);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class remove_counter_resultStandardSchemeFactory implements SchemeFactory {
- public remove_counter_resultStandardScheme getScheme() {
- return new remove_counter_resultStandardScheme();
- }
- }
-
- private static class remove_counter_resultStandardScheme extends StandardScheme<remove_counter_result> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, remove_counter_result struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // IRE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.ire = new InvalidRequestException();
- struct.ire.read(iprot);
- struct.setIreIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 2: // UE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.ue = new UnavailableException();
- struct.ue.read(iprot);
- struct.setUeIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 3: // TE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.te = new TimedOutException();
- struct.te.read(iprot);
- struct.setTeIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, remove_counter_result struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.ire != null) {
- oprot.writeFieldBegin(IRE_FIELD_DESC);
- struct.ire.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.ue != null) {
- oprot.writeFieldBegin(UE_FIELD_DESC);
- struct.ue.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.te != null) {
- oprot.writeFieldBegin(TE_FIELD_DESC);
- struct.te.write(oprot);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class remove_counter_resultTupleSchemeFactory implements SchemeFactory {
- public remove_counter_resultTupleScheme getScheme() {
- return new remove_counter_resultTupleScheme();
- }
- }
-
- private static class remove_counter_resultTupleScheme extends TupleScheme<remove_counter_result> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, remove_counter_result struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- BitSet optionals = new BitSet();
- if (struct.isSetIre()) {
- optionals.set(0);
- }
- if (struct.isSetUe()) {
- optionals.set(1);
- }
- if (struct.isSetTe()) {
- optionals.set(2);
- }
- oprot.writeBitSet(optionals, 3);
- if (struct.isSetIre()) {
- struct.ire.write(oprot);
- }
- if (struct.isSetUe()) {
- struct.ue.write(oprot);
- }
- if (struct.isSetTe()) {
- struct.te.write(oprot);
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, remove_counter_result struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- BitSet incoming = iprot.readBitSet(3);
- if (incoming.get(0)) {
- struct.ire = new InvalidRequestException();
- struct.ire.read(iprot);
- struct.setIreIsSet(true);
- }
- if (incoming.get(1)) {
- struct.ue = new UnavailableException();
- struct.ue.read(iprot);
- struct.setUeIsSet(true);
- }
- if (incoming.get(2)) {
- struct.te = new TimedOutException();
- struct.te.read(iprot);
- struct.setTeIsSet(true);
- }
- }
- }
-
- }
-
- public static class batch_mutate_args implements org.apache.thrift.TBase<batch_mutate_args, batch_mutate_args._Fields>, java.io.Serializable, Cloneable, Comparable<batch_mutate_args> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("batch_mutate_args");
-
- private static final org.apache.thrift.protocol.TField MUTATION_MAP_FIELD_DESC = new org.apache.thrift.protocol.TField("mutation_map", org.apache.thrift.protocol.TType.MAP, (short)1);
- private static final org.apache.thrift.protocol.TField CONSISTENCY_LEVEL_FIELD_DESC = new org.apache.thrift.protocol.TField("consistency_level", org.apache.thrift.protocol.TType.I32, (short)2);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new batch_mutate_argsStandardSchemeFactory());
- schemes.put(TupleScheme.class, new batch_mutate_argsTupleSchemeFactory());
- }
-
- public Map<ByteBuffer,Map<String,List<Mutation>>> mutation_map; // required
- /**
- *
- * @see ConsistencyLevel
- */
- public ConsistencyLevel consistency_level; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- MUTATION_MAP((short)1, "mutation_map"),
- /**
- *
- * @see ConsistencyLevel
- */
- CONSISTENCY_LEVEL((short)2, "consistency_level");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // MUTATION_MAP
- return MUTATION_MAP;
- case 2: // CONSISTENCY_LEVEL
- return CONSISTENCY_LEVEL;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.MUTATION_MAP, new org.apache.thrift.meta_data.FieldMetaData("mutation_map", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true),
- new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING),
- new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST,
- new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, Mutation.class))))));
- tmpMap.put(_Fields.CONSISTENCY_LEVEL, new org.apache.thrift.meta_data.FieldMetaData("consistency_level", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.EnumMetaData(org.apache.thrift.protocol.TType.ENUM, ConsistencyLevel.class)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(batch_mutate_args.class, metaDataMap);
- }
-
- public batch_mutate_args() {
- this.consistency_level = org.apache.cassandra.thrift.ConsistencyLevel.ONE;
-
- }
-
- public batch_mutate_args(
- Map<ByteBuffer,Map<String,List<Mutation>>> mutation_map,
- ConsistencyLevel consistency_level)
- {
- this();
- this.mutation_map = mutation_map;
- this.consistency_level = consistency_level;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public batch_mutate_args(batch_mutate_args other) {
- if (other.isSetMutation_map()) {
- Map<ByteBuffer,Map<String,List<Mutation>>> __this__mutation_map = new HashMap<ByteBuffer,Map<String,List<Mutation>>>(other.mutation_map.size());
- for (Map.Entry<ByteBuffer, Map<String,List<Mutation>>> other_element : other.mutation_map.entrySet()) {
-
- ByteBuffer other_element_key = other_element.getKey();
- Map<String,List<Mutation>> other_element_value = other_element.getValue();
-
- ByteBuffer __this__mutation_map_copy_key = org.apache.thrift.TBaseHelper.copyBinary(other_element_key);
-;
-
- Map<String,List<Mutation>> __this__mutation_map_copy_value = new HashMap<String,List<Mutation>>(other_element_value.size());
- for (Map.Entry<String, List<Mutation>> other_element_value_element : other_element_value.entrySet()) {
-
- String other_element_value_element_key = other_element_value_element.getKey();
- List<Mutation> other_element_value_element_value = other_element_value_element.getValue();
-
- String __this__mutation_map_copy_value_copy_key = other_element_value_element_key;
-
- List<Mutation> __this__mutation_map_copy_value_copy_value = new ArrayList<Mutation>(other_element_value_element_value.size());
- for (Mutation other_element_value_element_value_element : other_element_value_element_value) {
- __this__mutation_map_copy_value_copy_value.add(new Mutation(other_element_value_element_value_element));
- }
-
- __this__mutation_map_copy_value.put(__this__mutation_map_copy_value_copy_key, __this__mutation_map_copy_value_copy_value);
- }
-
- __this__mutation_map.put(__this__mutation_map_copy_key, __this__mutation_map_copy_value);
- }
- this.mutation_map = __this__mutation_map;
- }
- if (other.isSetConsistency_level()) {
- this.consistency_level = other.consistency_level;
- }
- }
-
- public batch_mutate_args deepCopy() {
- return new batch_mutate_args(this);
- }
-
- @Override
- public void clear() {
- this.mutation_map = null;
- this.consistency_level = org.apache.cassandra.thrift.ConsistencyLevel.ONE;
-
- }
-
- public int getMutation_mapSize() {
- return (this.mutation_map == null) ? 0 : this.mutation_map.size();
- }
-
- public void putToMutation_map(ByteBuffer key, Map<String,List<Mutation>> val) {
- if (this.mutation_map == null) {
- this.mutation_map = new HashMap<ByteBuffer,Map<String,List<Mutation>>>();
- }
- this.mutation_map.put(key, val);
- }
-
- public Map<ByteBuffer,Map<String,List<Mutation>>> getMutation_map() {
- return this.mutation_map;
- }
-
- public batch_mutate_args setMutation_map(Map<ByteBuffer,Map<String,List<Mutation>>> mutation_map) {
- this.mutation_map = mutation_map;
- return this;
- }
-
- public void unsetMutation_map() {
- this.mutation_map = null;
- }
-
- /** Returns true if field mutation_map is set (has been assigned a value) and false otherwise */
- public boolean isSetMutation_map() {
- return this.mutation_map != null;
- }
-
- public void setMutation_mapIsSet(boolean value) {
- if (!value) {
- this.mutation_map = null;
- }
- }
-
- /**
- *
- * @see ConsistencyLevel
- */
- public ConsistencyLevel getConsistency_level() {
- return this.consistency_level;
- }
-
- /**
- *
- * @see ConsistencyLevel
- */
- public batch_mutate_args setConsistency_level(ConsistencyLevel consistency_level) {
- this.consistency_level = consistency_level;
- return this;
- }
-
- public void unsetConsistency_level() {
- this.consistency_level = null;
- }
-
- /** Returns true if field consistency_level is set (has been assigned a value) and false otherwise */
- public boolean isSetConsistency_level() {
- return this.consistency_level != null;
- }
-
- public void setConsistency_levelIsSet(boolean value) {
- if (!value) {
- this.consistency_level = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case MUTATION_MAP:
- if (value == null) {
- unsetMutation_map();
- } else {
- setMutation_map((Map<ByteBuffer,Map<String,List<Mutation>>>)value);
- }
- break;
-
- case CONSISTENCY_LEVEL:
- if (value == null) {
- unsetConsistency_level();
- } else {
- setConsistency_level((ConsistencyLevel)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case MUTATION_MAP:
- return getMutation_map();
-
- case CONSISTENCY_LEVEL:
- return getConsistency_level();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case MUTATION_MAP:
- return isSetMutation_map();
- case CONSISTENCY_LEVEL:
- return isSetConsistency_level();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof batch_mutate_args)
- return this.equals((batch_mutate_args)that);
- return false;
- }
-
- public boolean equals(batch_mutate_args that) {
- if (that == null)
- return false;
-
- boolean this_present_mutation_map = true && this.isSetMutation_map();
- boolean that_present_mutation_map = true && that.isSetMutation_map();
- if (this_present_mutation_map || that_present_mutation_map) {
- if (!(this_present_mutation_map && that_present_mutation_map))
- return false;
- if (!this.mutation_map.equals(that.mutation_map))
- return false;
- }
-
- boolean this_present_consistency_level = true && this.isSetConsistency_level();
- boolean that_present_consistency_level = true && that.isSetConsistency_level();
- if (this_present_consistency_level || that_present_consistency_level) {
- if (!(this_present_consistency_level && that_present_consistency_level))
- return false;
- if (!this.consistency_level.equals(that.consistency_level))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_mutation_map = true && (isSetMutation_map());
- builder.append(present_mutation_map);
- if (present_mutation_map)
- builder.append(mutation_map);
-
- boolean present_consistency_level = true && (isSetConsistency_level());
- builder.append(present_consistency_level);
- if (present_consistency_level)
- builder.append(consistency_level.getValue());
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(batch_mutate_args other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetMutation_map()).compareTo(other.isSetMutation_map());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetMutation_map()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.mutation_map, other.mutation_map);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetConsistency_level()).compareTo(other.isSetConsistency_level());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetConsistency_level()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.consistency_level, other.consistency_level);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("batch_mutate_args(");
- boolean first = true;
-
- sb.append("mutation_map:");
- if (this.mutation_map == null) {
- sb.append("null");
- } else {
- sb.append(this.mutation_map);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("consistency_level:");
- if (this.consistency_level == null) {
- sb.append("null");
- } else {
- sb.append(this.consistency_level);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- if (mutation_map == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'mutation_map' was not present! Struct: " + toString());
- }
- if (consistency_level == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'consistency_level' was not present! Struct: " + toString());
- }
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class batch_mutate_argsStandardSchemeFactory implements SchemeFactory {
- public batch_mutate_argsStandardScheme getScheme() {
- return new batch_mutate_argsStandardScheme();
- }
- }
-
- private static class batch_mutate_argsStandardScheme extends StandardScheme<batch_mutate_args> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, batch_mutate_args struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // MUTATION_MAP
- if (schemeField.type == org.apache.thrift.protocol.TType.MAP) {
- {
- org.apache.thrift.protocol.TMap _map316 = iprot.readMapBegin();
- struct.mutation_map = new HashMap<ByteBuffer,Map<String,List<Mutation>>>(2*_map316.size);
- for (int _i317 = 0; _i317 < _map316.size; ++_i317)
- {
- ByteBuffer _key318;
- Map<String,List<Mutation>> _val319;
- _key318 = iprot.readBinary();
- {
- org.apache.thrift.protocol.TMap _map320 = iprot.readMapBegin();
- _val319 = new HashMap<String,List<Mutation>>(2*_map320.size);
- for (int _i321 = 0; _i321 < _map320.size; ++_i321)
- {
- String _key322;
- List<Mutation> _val323;
- _key322 = iprot.readString();
- {
- org.apache.thrift.protocol.TList _list324 = iprot.readListBegin();
- _val323 = new ArrayList<Mutation>(_list324.size);
- for (int _i325 = 0; _i325 < _list324.size; ++_i325)
- {
- Mutation _elem326;
- _elem326 = new Mutation();
- _elem326.read(iprot);
- _val323.add(_elem326);
- }
- iprot.readListEnd();
- }
- _val319.put(_key322, _val323);
- }
- iprot.readMapEnd();
- }
- struct.mutation_map.put(_key318, _val319);
- }
- iprot.readMapEnd();
- }
- struct.setMutation_mapIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 2: // CONSISTENCY_LEVEL
- if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
- struct.consistency_level = ConsistencyLevel.findByValue(iprot.readI32());
- struct.setConsistency_levelIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, batch_mutate_args struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.mutation_map != null) {
- oprot.writeFieldBegin(MUTATION_MAP_FIELD_DESC);
- {
- oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.MAP, struct.mutation_map.size()));
- for (Map.Entry<ByteBuffer, Map<String,List<Mutation>>> _iter327 : struct.mutation_map.entrySet())
- {
- oprot.writeBinary(_iter327.getKey());
- {
- oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.LIST, _iter327.getValue().size()));
- for (Map.Entry<String, List<Mutation>> _iter328 : _iter327.getValue().entrySet())
- {
- oprot.writeString(_iter328.getKey());
- {
- oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, _iter328.getValue().size()));
- for (Mutation _iter329 : _iter328.getValue())
- {
- _iter329.write(oprot);
- }
- oprot.writeListEnd();
- }
- }
- oprot.writeMapEnd();
- }
- }
- oprot.writeMapEnd();
- }
- oprot.writeFieldEnd();
- }
- if (struct.consistency_level != null) {
- oprot.writeFieldBegin(CONSISTENCY_LEVEL_FIELD_DESC);
- oprot.writeI32(struct.consistency_level.getValue());
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class batch_mutate_argsTupleSchemeFactory implements SchemeFactory {
- public batch_mutate_argsTupleScheme getScheme() {
- return new batch_mutate_argsTupleScheme();
- }
- }
-
- private static class batch_mutate_argsTupleScheme extends TupleScheme<batch_mutate_args> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, batch_mutate_args struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- {
- oprot.writeI32(struct.mutation_map.size());
- for (Map.Entry<ByteBuffer, Map<String,List<Mutation>>> _iter330 : struct.mutation_map.entrySet())
- {
- oprot.writeBinary(_iter330.getKey());
- {
- oprot.writeI32(_iter330.getValue().size());
- for (Map.Entry<String, List<Mutation>> _iter331 : _iter330.getValue().entrySet())
- {
- oprot.writeString(_iter331.getKey());
- {
- oprot.writeI32(_iter331.getValue().size());
- for (Mutation _iter332 : _iter331.getValue())
- {
- _iter332.write(oprot);
- }
- }
- }
- }
- }
- }
- oprot.writeI32(struct.consistency_level.getValue());
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, batch_mutate_args struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- {
- org.apache.thrift.protocol.TMap _map333 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.MAP, iprot.readI32());
- struct.mutation_map = new HashMap<ByteBuffer,Map<String,List<Mutation>>>(2*_map333.size);
- for (int _i334 = 0; _i334 < _map333.size; ++_i334)
- {
- ByteBuffer _key335;
- Map<String,List<Mutation>> _val336;
- _key335 = iprot.readBinary();
- {
- org.apache.thrift.protocol.TMap _map337 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.LIST, iprot.readI32());
- _val336 = new HashMap<String,List<Mutation>>(2*_map337.size);
- for (int _i338 = 0; _i338 < _map337.size; ++_i338)
- {
- String _key339;
- List<Mutation> _val340;
- _key339 = iprot.readString();
- {
- org.apache.thrift.protocol.TList _list341 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
- _val340 = new ArrayList<Mutation>(_list341.size);
- for (int _i342 = 0; _i342 < _list341.size; ++_i342)
- {
- Mutation _elem343;
- _elem343 = new Mutation();
- _elem343.read(iprot);
- _val340.add(_elem343);
- }
- }
- _val336.put(_key339, _val340);
- }
- }
- struct.mutation_map.put(_key335, _val336);
- }
- }
- struct.setMutation_mapIsSet(true);
- struct.consistency_level = ConsistencyLevel.findByValue(iprot.readI32());
- struct.setConsistency_levelIsSet(true);
- }
- }
-
- }
-
- public static class batch_mutate_result implements org.apache.thrift.TBase<batch_mutate_result, batch_mutate_result._Fields>, java.io.Serializable, Cloneable, Comparable<batch_mutate_result> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("batch_mutate_result");
-
- private static final org.apache.thrift.protocol.TField IRE_FIELD_DESC = new org.apache.thrift.protocol.TField("ire", org.apache.thrift.protocol.TType.STRUCT, (short)1);
- private static final org.apache.thrift.protocol.TField UE_FIELD_DESC = new org.apache.thrift.protocol.TField("ue", org.apache.thrift.protocol.TType.STRUCT, (short)2);
- private static final org.apache.thrift.protocol.TField TE_FIELD_DESC = new org.apache.thrift.protocol.TField("te", org.apache.thrift.protocol.TType.STRUCT, (short)3);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new batch_mutate_resultStandardSchemeFactory());
- schemes.put(TupleScheme.class, new batch_mutate_resultTupleSchemeFactory());
- }
-
- public InvalidRequestException ire; // required
- public UnavailableException ue; // required
- public TimedOutException te; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- IRE((short)1, "ire"),
- UE((short)2, "ue"),
- TE((short)3, "te");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // IRE
- return IRE;
- case 2: // UE
- return UE;
- case 3: // TE
- return TE;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.IRE, new org.apache.thrift.meta_data.FieldMetaData("ire", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- tmpMap.put(_Fields.UE, new org.apache.thrift.meta_data.FieldMetaData("ue", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- tmpMap.put(_Fields.TE, new org.apache.thrift.meta_data.FieldMetaData("te", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(batch_mutate_result.class, metaDataMap);
- }
-
- public batch_mutate_result() {
- }
-
- public batch_mutate_result(
- InvalidRequestException ire,
- UnavailableException ue,
- TimedOutException te)
- {
- this();
- this.ire = ire;
- this.ue = ue;
- this.te = te;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public batch_mutate_result(batch_mutate_result other) {
- if (other.isSetIre()) {
- this.ire = new InvalidRequestException(other.ire);
- }
- if (other.isSetUe()) {
- this.ue = new UnavailableException(other.ue);
- }
- if (other.isSetTe()) {
- this.te = new TimedOutException(other.te);
- }
- }
-
- public batch_mutate_result deepCopy() {
- return new batch_mutate_result(this);
- }
-
- @Override
- public void clear() {
- this.ire = null;
- this.ue = null;
- this.te = null;
- }
-
- public InvalidRequestException getIre() {
- return this.ire;
- }
-
- public batch_mutate_result setIre(InvalidRequestException ire) {
- this.ire = ire;
- return this;
- }
-
- public void unsetIre() {
- this.ire = null;
- }
-
- /** Returns true if field ire is set (has been assigned a value) and false otherwise */
- public boolean isSetIre() {
- return this.ire != null;
- }
-
- public void setIreIsSet(boolean value) {
- if (!value) {
- this.ire = null;
- }
- }
-
- public UnavailableException getUe() {
- return this.ue;
- }
-
- public batch_mutate_result setUe(UnavailableException ue) {
- this.ue = ue;
- return this;
- }
-
- public void unsetUe() {
- this.ue = null;
- }
-
- /** Returns true if field ue is set (has been assigned a value) and false otherwise */
- public boolean isSetUe() {
- return this.ue != null;
- }
-
- public void setUeIsSet(boolean value) {
- if (!value) {
- this.ue = null;
- }
- }
-
- public TimedOutException getTe() {
- return this.te;
- }
-
- public batch_mutate_result setTe(TimedOutException te) {
- this.te = te;
- return this;
- }
-
- public void unsetTe() {
- this.te = null;
- }
-
- /** Returns true if field te is set (has been assigned a value) and false otherwise */
- public boolean isSetTe() {
- return this.te != null;
- }
-
- public void setTeIsSet(boolean value) {
- if (!value) {
- this.te = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case IRE:
- if (value == null) {
- unsetIre();
- } else {
- setIre((InvalidRequestException)value);
- }
- break;
-
- case UE:
- if (value == null) {
- unsetUe();
- } else {
- setUe((UnavailableException)value);
- }
- break;
-
- case TE:
- if (value == null) {
- unsetTe();
- } else {
- setTe((TimedOutException)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case IRE:
- return getIre();
-
- case UE:
- return getUe();
-
- case TE:
- return getTe();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case IRE:
- return isSetIre();
- case UE:
- return isSetUe();
- case TE:
- return isSetTe();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof batch_mutate_result)
- return this.equals((batch_mutate_result)that);
- return false;
- }
-
- public boolean equals(batch_mutate_result that) {
- if (that == null)
- return false;
-
- boolean this_present_ire = true && this.isSetIre();
- boolean that_present_ire = true && that.isSetIre();
- if (this_present_ire || that_present_ire) {
- if (!(this_present_ire && that_present_ire))
- return false;
- if (!this.ire.equals(that.ire))
- return false;
- }
-
- boolean this_present_ue = true && this.isSetUe();
- boolean that_present_ue = true && that.isSetUe();
- if (this_present_ue || that_present_ue) {
- if (!(this_present_ue && that_present_ue))
- return false;
- if (!this.ue.equals(that.ue))
- return false;
- }
-
- boolean this_present_te = true && this.isSetTe();
- boolean that_present_te = true && that.isSetTe();
- if (this_present_te || that_present_te) {
- if (!(this_present_te && that_present_te))
- return false;
- if (!this.te.equals(that.te))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_ire = true && (isSetIre());
- builder.append(present_ire);
- if (present_ire)
- builder.append(ire);
-
- boolean present_ue = true && (isSetUe());
- builder.append(present_ue);
- if (present_ue)
- builder.append(ue);
-
- boolean present_te = true && (isSetTe());
- builder.append(present_te);
- if (present_te)
- builder.append(te);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(batch_mutate_result other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetIre()).compareTo(other.isSetIre());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetIre()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.ire, other.ire);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetUe()).compareTo(other.isSetUe());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetUe()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.ue, other.ue);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetTe()).compareTo(other.isSetTe());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetTe()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.te, other.te);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("batch_mutate_result(");
- boolean first = true;
-
- sb.append("ire:");
- if (this.ire == null) {
- sb.append("null");
- } else {
- sb.append(this.ire);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("ue:");
- if (this.ue == null) {
- sb.append("null");
- } else {
- sb.append(this.ue);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("te:");
- if (this.te == null) {
- sb.append("null");
- } else {
- sb.append(this.te);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class batch_mutate_resultStandardSchemeFactory implements SchemeFactory {
- public batch_mutate_resultStandardScheme getScheme() {
- return new batch_mutate_resultStandardScheme();
- }
- }
-
- private static class batch_mutate_resultStandardScheme extends StandardScheme<batch_mutate_result> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, batch_mutate_result struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // IRE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.ire = new InvalidRequestException();
- struct.ire.read(iprot);
- struct.setIreIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 2: // UE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.ue = new UnavailableException();
- struct.ue.read(iprot);
- struct.setUeIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 3: // TE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.te = new TimedOutException();
- struct.te.read(iprot);
- struct.setTeIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, batch_mutate_result struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.ire != null) {
- oprot.writeFieldBegin(IRE_FIELD_DESC);
- struct.ire.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.ue != null) {
- oprot.writeFieldBegin(UE_FIELD_DESC);
- struct.ue.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.te != null) {
- oprot.writeFieldBegin(TE_FIELD_DESC);
- struct.te.write(oprot);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class batch_mutate_resultTupleSchemeFactory implements SchemeFactory {
- public batch_mutate_resultTupleScheme getScheme() {
- return new batch_mutate_resultTupleScheme();
- }
- }
-
- private static class batch_mutate_resultTupleScheme extends TupleScheme<batch_mutate_result> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, batch_mutate_result struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- BitSet optionals = new BitSet();
- if (struct.isSetIre()) {
- optionals.set(0);
- }
- if (struct.isSetUe()) {
- optionals.set(1);
- }
- if (struct.isSetTe()) {
- optionals.set(2);
- }
- oprot.writeBitSet(optionals, 3);
- if (struct.isSetIre()) {
- struct.ire.write(oprot);
- }
- if (struct.isSetUe()) {
- struct.ue.write(oprot);
- }
- if (struct.isSetTe()) {
- struct.te.write(oprot);
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, batch_mutate_result struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- BitSet incoming = iprot.readBitSet(3);
- if (incoming.get(0)) {
- struct.ire = new InvalidRequestException();
- struct.ire.read(iprot);
- struct.setIreIsSet(true);
- }
- if (incoming.get(1)) {
- struct.ue = new UnavailableException();
- struct.ue.read(iprot);
- struct.setUeIsSet(true);
- }
- if (incoming.get(2)) {
- struct.te = new TimedOutException();
- struct.te.read(iprot);
- struct.setTeIsSet(true);
- }
- }
- }
-
- }
-
- public static class atomic_batch_mutate_args implements org.apache.thrift.TBase<atomic_batch_mutate_args, atomic_batch_mutate_args._Fields>, java.io.Serializable, Cloneable, Comparable<atomic_batch_mutate_args> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("atomic_batch_mutate_args");
-
- private static final org.apache.thrift.protocol.TField MUTATION_MAP_FIELD_DESC = new org.apache.thrift.protocol.TField("mutation_map", org.apache.thrift.protocol.TType.MAP, (short)1);
- private static final org.apache.thrift.protocol.TField CONSISTENCY_LEVEL_FIELD_DESC = new org.apache.thrift.protocol.TField("consistency_level", org.apache.thrift.protocol.TType.I32, (short)2);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new atomic_batch_mutate_argsStandardSchemeFactory());
- schemes.put(TupleScheme.class, new atomic_batch_mutate_argsTupleSchemeFactory());
- }
-
- public Map<ByteBuffer,Map<String,List<Mutation>>> mutation_map; // required
- /**
- *
- * @see ConsistencyLevel
- */
- public ConsistencyLevel consistency_level; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- MUTATION_MAP((short)1, "mutation_map"),
- /**
- *
- * @see ConsistencyLevel
- */
- CONSISTENCY_LEVEL((short)2, "consistency_level");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // MUTATION_MAP
- return MUTATION_MAP;
- case 2: // CONSISTENCY_LEVEL
- return CONSISTENCY_LEVEL;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.MUTATION_MAP, new org.apache.thrift.meta_data.FieldMetaData("mutation_map", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true),
- new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING),
- new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST,
- new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, Mutation.class))))));
- tmpMap.put(_Fields.CONSISTENCY_LEVEL, new org.apache.thrift.meta_data.FieldMetaData("consistency_level", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.EnumMetaData(org.apache.thrift.protocol.TType.ENUM, ConsistencyLevel.class)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(atomic_batch_mutate_args.class, metaDataMap);
- }
-
- public atomic_batch_mutate_args() {
- this.consistency_level = org.apache.cassandra.thrift.ConsistencyLevel.ONE;
-
- }
-
- public atomic_batch_mutate_args(
- Map<ByteBuffer,Map<String,List<Mutation>>> mutation_map,
- ConsistencyLevel consistency_level)
- {
- this();
- this.mutation_map = mutation_map;
- this.consistency_level = consistency_level;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public atomic_batch_mutate_args(atomic_batch_mutate_args other) {
- if (other.isSetMutation_map()) {
- Map<ByteBuffer,Map<String,List<Mutation>>> __this__mutation_map = new HashMap<ByteBuffer,Map<String,List<Mutation>>>(other.mutation_map.size());
- for (Map.Entry<ByteBuffer, Map<String,List<Mutation>>> other_element : other.mutation_map.entrySet()) {
-
- ByteBuffer other_element_key = other_element.getKey();
- Map<String,List<Mutation>> other_element_value = other_element.getValue();
-
- ByteBuffer __this__mutation_map_copy_key = org.apache.thrift.TBaseHelper.copyBinary(other_element_key);
-;
-
- Map<String,List<Mutation>> __this__mutation_map_copy_value = new HashMap<String,List<Mutation>>(other_element_value.size());
- for (Map.Entry<String, List<Mutation>> other_element_value_element : other_element_value.entrySet()) {
-
- String other_element_value_element_key = other_element_value_element.getKey();
- List<Mutation> other_element_value_element_value = other_element_value_element.getValue();
-
- String __this__mutation_map_copy_value_copy_key = other_element_value_element_key;
-
- List<Mutation> __this__mutation_map_copy_value_copy_value = new ArrayList<Mutation>(other_element_value_element_value.size());
- for (Mutation other_element_value_element_value_element : other_element_value_element_value) {
- __this__mutation_map_copy_value_copy_value.add(new Mutation(other_element_value_element_value_element));
- }
-
- __this__mutation_map_copy_value.put(__this__mutation_map_copy_value_copy_key, __this__mutation_map_copy_value_copy_value);
- }
-
- __this__mutation_map.put(__this__mutation_map_copy_key, __this__mutation_map_copy_value);
- }
- this.mutation_map = __this__mutation_map;
- }
- if (other.isSetConsistency_level()) {
- this.consistency_level = other.consistency_level;
- }
- }
-
- public atomic_batch_mutate_args deepCopy() {
- return new atomic_batch_mutate_args(this);
- }
-
- @Override
- public void clear() {
- this.mutation_map = null;
- this.consistency_level = org.apache.cassandra.thrift.ConsistencyLevel.ONE;
-
- }
-
- public int getMutation_mapSize() {
- return (this.mutation_map == null) ? 0 : this.mutation_map.size();
- }
-
- public void putToMutation_map(ByteBuffer key, Map<String,List<Mutation>> val) {
- if (this.mutation_map == null) {
- this.mutation_map = new HashMap<ByteBuffer,Map<String,List<Mutation>>>();
- }
- this.mutation_map.put(key, val);
- }
-
- public Map<ByteBuffer,Map<String,List<Mutation>>> getMutation_map() {
- return this.mutation_map;
- }
-
- public atomic_batch_mutate_args setMutation_map(Map<ByteBuffer,Map<String,List<Mutation>>> mutation_map) {
- this.mutation_map = mutation_map;
- return this;
- }
-
- public void unsetMutation_map() {
- this.mutation_map = null;
- }
-
- /** Returns true if field mutation_map is set (has been assigned a value) and false otherwise */
- public boolean isSetMutation_map() {
- return this.mutation_map != null;
- }
-
- public void setMutation_mapIsSet(boolean value) {
- if (!value) {
- this.mutation_map = null;
- }
- }
-
- /**
- *
- * @see ConsistencyLevel
- */
- public ConsistencyLevel getConsistency_level() {
- return this.consistency_level;
- }
-
- /**
- *
- * @see ConsistencyLevel
- */
- public atomic_batch_mutate_args setConsistency_level(ConsistencyLevel consistency_level) {
- this.consistency_level = consistency_level;
- return this;
- }
-
- public void unsetConsistency_level() {
- this.consistency_level = null;
- }
-
- /** Returns true if field consistency_level is set (has been assigned a value) and false otherwise */
- public boolean isSetConsistency_level() {
- return this.consistency_level != null;
- }
-
- public void setConsistency_levelIsSet(boolean value) {
- if (!value) {
- this.consistency_level = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case MUTATION_MAP:
- if (value == null) {
- unsetMutation_map();
- } else {
- setMutation_map((Map<ByteBuffer,Map<String,List<Mutation>>>)value);
- }
- break;
-
- case CONSISTENCY_LEVEL:
- if (value == null) {
- unsetConsistency_level();
- } else {
- setConsistency_level((ConsistencyLevel)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case MUTATION_MAP:
- return getMutation_map();
-
- case CONSISTENCY_LEVEL:
- return getConsistency_level();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case MUTATION_MAP:
- return isSetMutation_map();
- case CONSISTENCY_LEVEL:
- return isSetConsistency_level();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof atomic_batch_mutate_args)
- return this.equals((atomic_batch_mutate_args)that);
- return false;
- }
-
- public boolean equals(atomic_batch_mutate_args that) {
- if (that == null)
- return false;
-
- boolean this_present_mutation_map = true && this.isSetMutation_map();
- boolean that_present_mutation_map = true && that.isSetMutation_map();
- if (this_present_mutation_map || that_present_mutation_map) {
- if (!(this_present_mutation_map && that_present_mutation_map))
- return false;
- if (!this.mutation_map.equals(that.mutation_map))
- return false;
- }
-
- boolean this_present_consistency_level = true && this.isSetConsistency_level();
- boolean that_present_consistency_level = true && that.isSetConsistency_level();
- if (this_present_consistency_level || that_present_consistency_level) {
- if (!(this_present_consistency_level && that_present_consistency_level))
- return false;
- if (!this.consistency_level.equals(that.consistency_level))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_mutation_map = true && (isSetMutation_map());
- builder.append(present_mutation_map);
- if (present_mutation_map)
- builder.append(mutation_map);
-
- boolean present_consistency_level = true && (isSetConsistency_level());
- builder.append(present_consistency_level);
- if (present_consistency_level)
- builder.append(consistency_level.getValue());
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(atomic_batch_mutate_args other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetMutation_map()).compareTo(other.isSetMutation_map());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetMutation_map()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.mutation_map, other.mutation_map);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetConsistency_level()).compareTo(other.isSetConsistency_level());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetConsistency_level()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.consistency_level, other.consistency_level);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("atomic_batch_mutate_args(");
- boolean first = true;
-
- sb.append("mutation_map:");
- if (this.mutation_map == null) {
- sb.append("null");
- } else {
- sb.append(this.mutation_map);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("consistency_level:");
- if (this.consistency_level == null) {
- sb.append("null");
- } else {
- sb.append(this.consistency_level);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- if (mutation_map == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'mutation_map' was not present! Struct: " + toString());
- }
- if (consistency_level == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'consistency_level' was not present! Struct: " + toString());
- }
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class atomic_batch_mutate_argsStandardSchemeFactory implements SchemeFactory {
- public atomic_batch_mutate_argsStandardScheme getScheme() {
- return new atomic_batch_mutate_argsStandardScheme();
- }
- }
-
- private static class atomic_batch_mutate_argsStandardScheme extends StandardScheme<atomic_batch_mutate_args> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, atomic_batch_mutate_args struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // MUTATION_MAP
- if (schemeField.type == org.apache.thrift.protocol.TType.MAP) {
- {
- org.apache.thrift.protocol.TMap _map344 = iprot.readMapBegin();
- struct.mutation_map = new HashMap<ByteBuffer,Map<String,List<Mutation>>>(2*_map344.size);
- for (int _i345 = 0; _i345 < _map344.size; ++_i345)
- {
- ByteBuffer _key346;
- Map<String,List<Mutation>> _val347;
- _key346 = iprot.readBinary();
- {
- org.apache.thrift.protocol.TMap _map348 = iprot.readMapBegin();
- _val347 = new HashMap<String,List<Mutation>>(2*_map348.size);
- for (int _i349 = 0; _i349 < _map348.size; ++_i349)
- {
- String _key350;
- List<Mutation> _val351;
- _key350 = iprot.readString();
- {
- org.apache.thrift.protocol.TList _list352 = iprot.readListBegin();
- _val351 = new ArrayList<Mutation>(_list352.size);
- for (int _i353 = 0; _i353 < _list352.size; ++_i353)
- {
- Mutation _elem354;
- _elem354 = new Mutation();
- _elem354.read(iprot);
- _val351.add(_elem354);
- }
- iprot.readListEnd();
- }
- _val347.put(_key350, _val351);
- }
- iprot.readMapEnd();
- }
- struct.mutation_map.put(_key346, _val347);
- }
- iprot.readMapEnd();
- }
- struct.setMutation_mapIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 2: // CONSISTENCY_LEVEL
- if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
- struct.consistency_level = ConsistencyLevel.findByValue(iprot.readI32());
- struct.setConsistency_levelIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, atomic_batch_mutate_args struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.mutation_map != null) {
- oprot.writeFieldBegin(MUTATION_MAP_FIELD_DESC);
- {
- oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.MAP, struct.mutation_map.size()));
- for (Map.Entry<ByteBuffer, Map<String,List<Mutation>>> _iter355 : struct.mutation_map.entrySet())
- {
- oprot.writeBinary(_iter355.getKey());
- {
- oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.LIST, _iter355.getValue().size()));
- for (Map.Entry<String, List<Mutation>> _iter356 : _iter355.getValue().entrySet())
- {
- oprot.writeString(_iter356.getKey());
- {
- oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, _iter356.getValue().size()));
- for (Mutation _iter357 : _iter356.getValue())
- {
- _iter357.write(oprot);
- }
- oprot.writeListEnd();
- }
- }
- oprot.writeMapEnd();
- }
- }
- oprot.writeMapEnd();
- }
- oprot.writeFieldEnd();
- }
- if (struct.consistency_level != null) {
- oprot.writeFieldBegin(CONSISTENCY_LEVEL_FIELD_DESC);
- oprot.writeI32(struct.consistency_level.getValue());
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class atomic_batch_mutate_argsTupleSchemeFactory implements SchemeFactory {
- public atomic_batch_mutate_argsTupleScheme getScheme() {
- return new atomic_batch_mutate_argsTupleScheme();
- }
- }
-
- private static class atomic_batch_mutate_argsTupleScheme extends TupleScheme<atomic_batch_mutate_args> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, atomic_batch_mutate_args struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- {
- oprot.writeI32(struct.mutation_map.size());
- for (Map.Entry<ByteBuffer, Map<String,List<Mutation>>> _iter358 : struct.mutation_map.entrySet())
- {
- oprot.writeBinary(_iter358.getKey());
- {
- oprot.writeI32(_iter358.getValue().size());
- for (Map.Entry<String, List<Mutation>> _iter359 : _iter358.getValue().entrySet())
- {
- oprot.writeString(_iter359.getKey());
- {
- oprot.writeI32(_iter359.getValue().size());
- for (Mutation _iter360 : _iter359.getValue())
- {
- _iter360.write(oprot);
- }
- }
- }
- }
- }
- }
- oprot.writeI32(struct.consistency_level.getValue());
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, atomic_batch_mutate_args struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- {
- org.apache.thrift.protocol.TMap _map361 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.MAP, iprot.readI32());
- struct.mutation_map = new HashMap<ByteBuffer,Map<String,List<Mutation>>>(2*_map361.size);
- for (int _i362 = 0; _i362 < _map361.size; ++_i362)
- {
- ByteBuffer _key363;
- Map<String,List<Mutation>> _val364;
- _key363 = iprot.readBinary();
- {
- org.apache.thrift.protocol.TMap _map365 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.LIST, iprot.readI32());
- _val364 = new HashMap<String,List<Mutation>>(2*_map365.size);
- for (int _i366 = 0; _i366 < _map365.size; ++_i366)
- {
- String _key367;
- List<Mutation> _val368;
- _key367 = iprot.readString();
- {
- org.apache.thrift.protocol.TList _list369 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
- _val368 = new ArrayList<Mutation>(_list369.size);
- for (int _i370 = 0; _i370 < _list369.size; ++_i370)
- {
- Mutation _elem371;
- _elem371 = new Mutation();
- _elem371.read(iprot);
- _val368.add(_elem371);
- }
- }
- _val364.put(_key367, _val368);
- }
- }
- struct.mutation_map.put(_key363, _val364);
- }
- }
- struct.setMutation_mapIsSet(true);
- struct.consistency_level = ConsistencyLevel.findByValue(iprot.readI32());
- struct.setConsistency_levelIsSet(true);
- }
- }
-
- }
-
- public static class atomic_batch_mutate_result implements org.apache.thrift.TBase<atomic_batch_mutate_result, atomic_batch_mutate_result._Fields>, java.io.Serializable, Cloneable, Comparable<atomic_batch_mutate_result> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("atomic_batch_mutate_result");
-
- private static final org.apache.thrift.protocol.TField IRE_FIELD_DESC = new org.apache.thrift.protocol.TField("ire", org.apache.thrift.protocol.TType.STRUCT, (short)1);
- private static final org.apache.thrift.protocol.TField UE_FIELD_DESC = new org.apache.thrift.protocol.TField("ue", org.apache.thrift.protocol.TType.STRUCT, (short)2);
- private static final org.apache.thrift.protocol.TField TE_FIELD_DESC = new org.apache.thrift.protocol.TField("te", org.apache.thrift.protocol.TType.STRUCT, (short)3);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new atomic_batch_mutate_resultStandardSchemeFactory());
- schemes.put(TupleScheme.class, new atomic_batch_mutate_resultTupleSchemeFactory());
- }
-
- public InvalidRequestException ire; // required
- public UnavailableException ue; // required
- public TimedOutException te; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- IRE((short)1, "ire"),
- UE((short)2, "ue"),
- TE((short)3, "te");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // IRE
- return IRE;
- case 2: // UE
- return UE;
- case 3: // TE
- return TE;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.IRE, new org.apache.thrift.meta_data.FieldMetaData("ire", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- tmpMap.put(_Fields.UE, new org.apache.thrift.meta_data.FieldMetaData("ue", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- tmpMap.put(_Fields.TE, new org.apache.thrift.meta_data.FieldMetaData("te", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(atomic_batch_mutate_result.class, metaDataMap);
- }
-
- public atomic_batch_mutate_result() {
- }
-
- public atomic_batch_mutate_result(
- InvalidRequestException ire,
- UnavailableException ue,
- TimedOutException te)
- {
- this();
- this.ire = ire;
- this.ue = ue;
- this.te = te;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public atomic_batch_mutate_result(atomic_batch_mutate_result other) {
- if (other.isSetIre()) {
- this.ire = new InvalidRequestException(other.ire);
- }
- if (other.isSetUe()) {
- this.ue = new UnavailableException(other.ue);
- }
- if (other.isSetTe()) {
- this.te = new TimedOutException(other.te);
- }
- }
-
- public atomic_batch_mutate_result deepCopy() {
- return new atomic_batch_mutate_result(this);
- }
-
- @Override
- public void clear() {
- this.ire = null;
- this.ue = null;
- this.te = null;
- }
-
- public InvalidRequestException getIre() {
- return this.ire;
- }
-
- public atomic_batch_mutate_result setIre(InvalidRequestException ire) {
- this.ire = ire;
- return this;
- }
-
- public void unsetIre() {
- this.ire = null;
- }
-
- /** Returns true if field ire is set (has been assigned a value) and false otherwise */
- public boolean isSetIre() {
- return this.ire != null;
- }
-
- public void setIreIsSet(boolean value) {
- if (!value) {
- this.ire = null;
- }
- }
-
- public UnavailableException getUe() {
- return this.ue;
- }
-
- public atomic_batch_mutate_result setUe(UnavailableException ue) {
- this.ue = ue;
- return this;
- }
-
- public void unsetUe() {
- this.ue = null;
- }
-
- /** Returns true if field ue is set (has been assigned a value) and false otherwise */
- public boolean isSetUe() {
- return this.ue != null;
- }
-
- public void setUeIsSet(boolean value) {
- if (!value) {
- this.ue = null;
- }
- }
-
- public TimedOutException getTe() {
- return this.te;
- }
-
- public atomic_batch_mutate_result setTe(TimedOutException te) {
- this.te = te;
- return this;
- }
-
- public void unsetTe() {
- this.te = null;
- }
-
- /** Returns true if field te is set (has been assigned a value) and false otherwise */
- public boolean isSetTe() {
- return this.te != null;
- }
-
- public void setTeIsSet(boolean value) {
- if (!value) {
- this.te = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case IRE:
- if (value == null) {
- unsetIre();
- } else {
- setIre((InvalidRequestException)value);
- }
- break;
-
- case UE:
- if (value == null) {
- unsetUe();
- } else {
- setUe((UnavailableException)value);
- }
- break;
-
- case TE:
- if (value == null) {
- unsetTe();
- } else {
- setTe((TimedOutException)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case IRE:
- return getIre();
-
- case UE:
- return getUe();
-
- case TE:
- return getTe();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case IRE:
- return isSetIre();
- case UE:
- return isSetUe();
- case TE:
- return isSetTe();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof atomic_batch_mutate_result)
- return this.equals((atomic_batch_mutate_result)that);
- return false;
- }
-
- public boolean equals(atomic_batch_mutate_result that) {
- if (that == null)
- return false;
-
- boolean this_present_ire = true && this.isSetIre();
- boolean that_present_ire = true && that.isSetIre();
- if (this_present_ire || that_present_ire) {
- if (!(this_present_ire && that_present_ire))
- return false;
- if (!this.ire.equals(that.ire))
- return false;
- }
-
- boolean this_present_ue = true && this.isSetUe();
- boolean that_present_ue = true && that.isSetUe();
- if (this_present_ue || that_present_ue) {
- if (!(this_present_ue && that_present_ue))
- return false;
- if (!this.ue.equals(that.ue))
- return false;
- }
-
- boolean this_present_te = true && this.isSetTe();
- boolean that_present_te = true && that.isSetTe();
- if (this_present_te || that_present_te) {
- if (!(this_present_te && that_present_te))
- return false;
- if (!this.te.equals(that.te))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_ire = true && (isSetIre());
- builder.append(present_ire);
- if (present_ire)
- builder.append(ire);
-
- boolean present_ue = true && (isSetUe());
- builder.append(present_ue);
- if (present_ue)
- builder.append(ue);
-
- boolean present_te = true && (isSetTe());
- builder.append(present_te);
- if (present_te)
- builder.append(te);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(atomic_batch_mutate_result other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetIre()).compareTo(other.isSetIre());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetIre()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.ire, other.ire);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetUe()).compareTo(other.isSetUe());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetUe()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.ue, other.ue);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetTe()).compareTo(other.isSetTe());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetTe()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.te, other.te);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("atomic_batch_mutate_result(");
- boolean first = true;
-
- sb.append("ire:");
- if (this.ire == null) {
- sb.append("null");
- } else {
- sb.append(this.ire);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("ue:");
- if (this.ue == null) {
- sb.append("null");
- } else {
- sb.append(this.ue);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("te:");
- if (this.te == null) {
- sb.append("null");
- } else {
- sb.append(this.te);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class atomic_batch_mutate_resultStandardSchemeFactory implements SchemeFactory {
- public atomic_batch_mutate_resultStandardScheme getScheme() {
- return new atomic_batch_mutate_resultStandardScheme();
- }
- }
-
- private static class atomic_batch_mutate_resultStandardScheme extends StandardScheme<atomic_batch_mutate_result> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, atomic_batch_mutate_result struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // IRE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.ire = new InvalidRequestException();
- struct.ire.read(iprot);
- struct.setIreIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 2: // UE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.ue = new UnavailableException();
- struct.ue.read(iprot);
- struct.setUeIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 3: // TE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.te = new TimedOutException();
- struct.te.read(iprot);
- struct.setTeIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, atomic_batch_mutate_result struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.ire != null) {
- oprot.writeFieldBegin(IRE_FIELD_DESC);
- struct.ire.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.ue != null) {
- oprot.writeFieldBegin(UE_FIELD_DESC);
- struct.ue.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.te != null) {
- oprot.writeFieldBegin(TE_FIELD_DESC);
- struct.te.write(oprot);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class atomic_batch_mutate_resultTupleSchemeFactory implements SchemeFactory {
- public atomic_batch_mutate_resultTupleScheme getScheme() {
- return new atomic_batch_mutate_resultTupleScheme();
- }
- }
-
- private static class atomic_batch_mutate_resultTupleScheme extends TupleScheme<atomic_batch_mutate_result> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, atomic_batch_mutate_result struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- BitSet optionals = new BitSet();
- if (struct.isSetIre()) {
- optionals.set(0);
- }
- if (struct.isSetUe()) {
- optionals.set(1);
- }
- if (struct.isSetTe()) {
- optionals.set(2);
- }
- oprot.writeBitSet(optionals, 3);
- if (struct.isSetIre()) {
- struct.ire.write(oprot);
- }
- if (struct.isSetUe()) {
- struct.ue.write(oprot);
- }
- if (struct.isSetTe()) {
- struct.te.write(oprot);
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, atomic_batch_mutate_result struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- BitSet incoming = iprot.readBitSet(3);
- if (incoming.get(0)) {
- struct.ire = new InvalidRequestException();
- struct.ire.read(iprot);
- struct.setIreIsSet(true);
- }
- if (incoming.get(1)) {
- struct.ue = new UnavailableException();
- struct.ue.read(iprot);
- struct.setUeIsSet(true);
- }
- if (incoming.get(2)) {
- struct.te = new TimedOutException();
- struct.te.read(iprot);
- struct.setTeIsSet(true);
- }
- }
- }
-
- }
-
- public static class truncate_args implements org.apache.thrift.TBase<truncate_args, truncate_args._Fields>, java.io.Serializable, Cloneable, Comparable<truncate_args> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("truncate_args");
-
- private static final org.apache.thrift.protocol.TField CFNAME_FIELD_DESC = new org.apache.thrift.protocol.TField("cfname", org.apache.thrift.protocol.TType.STRING, (short)1);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new truncate_argsStandardSchemeFactory());
- schemes.put(TupleScheme.class, new truncate_argsTupleSchemeFactory());
- }
-
- public String cfname; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- CFNAME((short)1, "cfname");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // CFNAME
- return CFNAME;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.CFNAME, new org.apache.thrift.meta_data.FieldMetaData("cfname", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(truncate_args.class, metaDataMap);
- }
-
- public truncate_args() {
- }
-
- public truncate_args(
- String cfname)
- {
- this();
- this.cfname = cfname;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public truncate_args(truncate_args other) {
- if (other.isSetCfname()) {
- this.cfname = other.cfname;
- }
- }
-
- public truncate_args deepCopy() {
- return new truncate_args(this);
- }
-
- @Override
- public void clear() {
- this.cfname = null;
- }
-
- public String getCfname() {
- return this.cfname;
- }
-
- public truncate_args setCfname(String cfname) {
- this.cfname = cfname;
- return this;
- }
-
- public void unsetCfname() {
- this.cfname = null;
- }
-
- /** Returns true if field cfname is set (has been assigned a value) and false otherwise */
- public boolean isSetCfname() {
- return this.cfname != null;
- }
-
- public void setCfnameIsSet(boolean value) {
- if (!value) {
- this.cfname = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case CFNAME:
- if (value == null) {
- unsetCfname();
- } else {
- setCfname((String)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case CFNAME:
- return getCfname();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case CFNAME:
- return isSetCfname();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof truncate_args)
- return this.equals((truncate_args)that);
- return false;
- }
-
- public boolean equals(truncate_args that) {
- if (that == null)
- return false;
-
- boolean this_present_cfname = true && this.isSetCfname();
- boolean that_present_cfname = true && that.isSetCfname();
- if (this_present_cfname || that_present_cfname) {
- if (!(this_present_cfname && that_present_cfname))
- return false;
- if (!this.cfname.equals(that.cfname))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_cfname = true && (isSetCfname());
- builder.append(present_cfname);
- if (present_cfname)
- builder.append(cfname);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(truncate_args other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetCfname()).compareTo(other.isSetCfname());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetCfname()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.cfname, other.cfname);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("truncate_args(");
- boolean first = true;
-
- sb.append("cfname:");
- if (this.cfname == null) {
- sb.append("null");
- } else {
- sb.append(this.cfname);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- if (cfname == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'cfname' was not present! Struct: " + toString());
- }
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class truncate_argsStandardSchemeFactory implements SchemeFactory {
- public truncate_argsStandardScheme getScheme() {
- return new truncate_argsStandardScheme();
- }
- }
-
- private static class truncate_argsStandardScheme extends StandardScheme<truncate_args> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, truncate_args struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // CFNAME
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.cfname = iprot.readString();
- struct.setCfnameIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, truncate_args struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.cfname != null) {
- oprot.writeFieldBegin(CFNAME_FIELD_DESC);
- oprot.writeString(struct.cfname);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class truncate_argsTupleSchemeFactory implements SchemeFactory {
- public truncate_argsTupleScheme getScheme() {
- return new truncate_argsTupleScheme();
- }
- }
-
- private static class truncate_argsTupleScheme extends TupleScheme<truncate_args> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, truncate_args struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- oprot.writeString(struct.cfname);
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, truncate_args struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- struct.cfname = iprot.readString();
- struct.setCfnameIsSet(true);
- }
- }
-
- }
-
- public static class truncate_result implements org.apache.thrift.TBase<truncate_result, truncate_result._Fields>, java.io.Serializable, Cloneable, Comparable<truncate_result> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("truncate_result");
-
- private static final org.apache.thrift.protocol.TField IRE_FIELD_DESC = new org.apache.thrift.protocol.TField("ire", org.apache.thrift.protocol.TType.STRUCT, (short)1);
- private static final org.apache.thrift.protocol.TField UE_FIELD_DESC = new org.apache.thrift.protocol.TField("ue", org.apache.thrift.protocol.TType.STRUCT, (short)2);
- private static final org.apache.thrift.protocol.TField TE_FIELD_DESC = new org.apache.thrift.protocol.TField("te", org.apache.thrift.protocol.TType.STRUCT, (short)3);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new truncate_resultStandardSchemeFactory());
- schemes.put(TupleScheme.class, new truncate_resultTupleSchemeFactory());
- }
-
- public InvalidRequestException ire; // required
- public UnavailableException ue; // required
- public TimedOutException te; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- IRE((short)1, "ire"),
- UE((short)2, "ue"),
- TE((short)3, "te");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // IRE
- return IRE;
- case 2: // UE
- return UE;
- case 3: // TE
- return TE;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.IRE, new org.apache.thrift.meta_data.FieldMetaData("ire", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- tmpMap.put(_Fields.UE, new org.apache.thrift.meta_data.FieldMetaData("ue", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- tmpMap.put(_Fields.TE, new org.apache.thrift.meta_data.FieldMetaData("te", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(truncate_result.class, metaDataMap);
- }
-
- public truncate_result() {
- }
-
- public truncate_result(
- InvalidRequestException ire,
- UnavailableException ue,
- TimedOutException te)
- {
- this();
- this.ire = ire;
- this.ue = ue;
- this.te = te;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public truncate_result(truncate_result other) {
- if (other.isSetIre()) {
- this.ire = new InvalidRequestException(other.ire);
- }
- if (other.isSetUe()) {
- this.ue = new UnavailableException(other.ue);
- }
- if (other.isSetTe()) {
- this.te = new TimedOutException(other.te);
- }
- }
-
- public truncate_result deepCopy() {
- return new truncate_result(this);
- }
-
- @Override
- public void clear() {
- this.ire = null;
- this.ue = null;
- this.te = null;
- }
-
- public InvalidRequestException getIre() {
- return this.ire;
- }
-
- public truncate_result setIre(InvalidRequestException ire) {
- this.ire = ire;
- return this;
- }
-
- public void unsetIre() {
- this.ire = null;
- }
-
- /** Returns true if field ire is set (has been assigned a value) and false otherwise */
- public boolean isSetIre() {
- return this.ire != null;
- }
-
- public void setIreIsSet(boolean value) {
- if (!value) {
- this.ire = null;
- }
- }
-
- public UnavailableException getUe() {
- return this.ue;
- }
-
- public truncate_result setUe(UnavailableException ue) {
- this.ue = ue;
- return this;
- }
-
- public void unsetUe() {
- this.ue = null;
- }
-
- /** Returns true if field ue is set (has been assigned a value) and false otherwise */
- public boolean isSetUe() {
- return this.ue != null;
- }
-
- public void setUeIsSet(boolean value) {
- if (!value) {
- this.ue = null;
- }
- }
-
- public TimedOutException getTe() {
- return this.te;
- }
-
- public truncate_result setTe(TimedOutException te) {
- this.te = te;
- return this;
- }
-
- public void unsetTe() {
- this.te = null;
- }
-
- /** Returns true if field te is set (has been assigned a value) and false otherwise */
- public boolean isSetTe() {
- return this.te != null;
- }
-
- public void setTeIsSet(boolean value) {
- if (!value) {
- this.te = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case IRE:
- if (value == null) {
- unsetIre();
- } else {
- setIre((InvalidRequestException)value);
- }
- break;
-
- case UE:
- if (value == null) {
- unsetUe();
- } else {
- setUe((UnavailableException)value);
- }
- break;
-
- case TE:
- if (value == null) {
- unsetTe();
- } else {
- setTe((TimedOutException)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case IRE:
- return getIre();
-
- case UE:
- return getUe();
-
- case TE:
- return getTe();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case IRE:
- return isSetIre();
- case UE:
- return isSetUe();
- case TE:
- return isSetTe();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof truncate_result)
- return this.equals((truncate_result)that);
- return false;
- }
-
- public boolean equals(truncate_result that) {
- if (that == null)
- return false;
-
- boolean this_present_ire = true && this.isSetIre();
- boolean that_present_ire = true && that.isSetIre();
- if (this_present_ire || that_present_ire) {
- if (!(this_present_ire && that_present_ire))
- return false;
- if (!this.ire.equals(that.ire))
- return false;
- }
-
- boolean this_present_ue = true && this.isSetUe();
- boolean that_present_ue = true && that.isSetUe();
- if (this_present_ue || that_present_ue) {
- if (!(this_present_ue && that_present_ue))
- return false;
- if (!this.ue.equals(that.ue))
- return false;
- }
-
- boolean this_present_te = true && this.isSetTe();
- boolean that_present_te = true && that.isSetTe();
- if (this_present_te || that_present_te) {
- if (!(this_present_te && that_present_te))
- return false;
- if (!this.te.equals(that.te))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_ire = true && (isSetIre());
- builder.append(present_ire);
- if (present_ire)
- builder.append(ire);
-
- boolean present_ue = true && (isSetUe());
- builder.append(present_ue);
- if (present_ue)
- builder.append(ue);
-
- boolean present_te = true && (isSetTe());
- builder.append(present_te);
- if (present_te)
- builder.append(te);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(truncate_result other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetIre()).compareTo(other.isSetIre());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetIre()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.ire, other.ire);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetUe()).compareTo(other.isSetUe());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetUe()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.ue, other.ue);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetTe()).compareTo(other.isSetTe());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetTe()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.te, other.te);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("truncate_result(");
- boolean first = true;
-
- sb.append("ire:");
- if (this.ire == null) {
- sb.append("null");
- } else {
- sb.append(this.ire);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("ue:");
- if (this.ue == null) {
- sb.append("null");
- } else {
- sb.append(this.ue);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("te:");
- if (this.te == null) {
- sb.append("null");
- } else {
- sb.append(this.te);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class truncate_resultStandardSchemeFactory implements SchemeFactory {
- public truncate_resultStandardScheme getScheme() {
- return new truncate_resultStandardScheme();
- }
- }
-
- private static class truncate_resultStandardScheme extends StandardScheme<truncate_result> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, truncate_result struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // IRE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.ire = new InvalidRequestException();
- struct.ire.read(iprot);
- struct.setIreIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 2: // UE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.ue = new UnavailableException();
- struct.ue.read(iprot);
- struct.setUeIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 3: // TE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.te = new TimedOutException();
- struct.te.read(iprot);
- struct.setTeIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, truncate_result struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.ire != null) {
- oprot.writeFieldBegin(IRE_FIELD_DESC);
- struct.ire.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.ue != null) {
- oprot.writeFieldBegin(UE_FIELD_DESC);
- struct.ue.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.te != null) {
- oprot.writeFieldBegin(TE_FIELD_DESC);
- struct.te.write(oprot);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class truncate_resultTupleSchemeFactory implements SchemeFactory {
- public truncate_resultTupleScheme getScheme() {
- return new truncate_resultTupleScheme();
- }
- }
-
- private static class truncate_resultTupleScheme extends TupleScheme<truncate_result> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, truncate_result struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- BitSet optionals = new BitSet();
- if (struct.isSetIre()) {
- optionals.set(0);
- }
- if (struct.isSetUe()) {
- optionals.set(1);
- }
- if (struct.isSetTe()) {
- optionals.set(2);
- }
- oprot.writeBitSet(optionals, 3);
- if (struct.isSetIre()) {
- struct.ire.write(oprot);
- }
- if (struct.isSetUe()) {
- struct.ue.write(oprot);
- }
- if (struct.isSetTe()) {
- struct.te.write(oprot);
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, truncate_result struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- BitSet incoming = iprot.readBitSet(3);
- if (incoming.get(0)) {
- struct.ire = new InvalidRequestException();
- struct.ire.read(iprot);
- struct.setIreIsSet(true);
- }
- if (incoming.get(1)) {
- struct.ue = new UnavailableException();
- struct.ue.read(iprot);
- struct.setUeIsSet(true);
- }
- if (incoming.get(2)) {
- struct.te = new TimedOutException();
- struct.te.read(iprot);
- struct.setTeIsSet(true);
- }
- }
- }
-
- }
-
- public static class get_multi_slice_args implements org.apache.thrift.TBase<get_multi_slice_args, get_multi_slice_args._Fields>, java.io.Serializable, Cloneable, Comparable<get_multi_slice_args> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("get_multi_slice_args");
-
- private static final org.apache.thrift.protocol.TField REQUEST_FIELD_DESC = new org.apache.thrift.protocol.TField("request", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new get_multi_slice_argsStandardSchemeFactory());
- schemes.put(TupleScheme.class, new get_multi_slice_argsTupleSchemeFactory());
- }
-
- public MultiSliceRequest request; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- REQUEST((short)1, "request");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // REQUEST
- return REQUEST;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.REQUEST, new org.apache.thrift.meta_data.FieldMetaData("request", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, MultiSliceRequest.class)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(get_multi_slice_args.class, metaDataMap);
- }
-
- public get_multi_slice_args() {
- }
-
- public get_multi_slice_args(
- MultiSliceRequest request)
- {
- this();
- this.request = request;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public get_multi_slice_args(get_multi_slice_args other) {
- if (other.isSetRequest()) {
- this.request = new MultiSliceRequest(other.request);
- }
- }
-
- public get_multi_slice_args deepCopy() {
- return new get_multi_slice_args(this);
- }
-
- @Override
- public void clear() {
- this.request = null;
- }
-
- public MultiSliceRequest getRequest() {
- return this.request;
- }
-
- public get_multi_slice_args setRequest(MultiSliceRequest request) {
- this.request = request;
- return this;
- }
-
- public void unsetRequest() {
- this.request = null;
- }
-
- /** Returns true if field request is set (has been assigned a value) and false otherwise */
- public boolean isSetRequest() {
- return this.request != null;
- }
-
- public void setRequestIsSet(boolean value) {
- if (!value) {
- this.request = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case REQUEST:
- if (value == null) {
- unsetRequest();
- } else {
- setRequest((MultiSliceRequest)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case REQUEST:
- return getRequest();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case REQUEST:
- return isSetRequest();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof get_multi_slice_args)
- return this.equals((get_multi_slice_args)that);
- return false;
- }
-
- public boolean equals(get_multi_slice_args that) {
- if (that == null)
- return false;
-
- boolean this_present_request = true && this.isSetRequest();
- boolean that_present_request = true && that.isSetRequest();
- if (this_present_request || that_present_request) {
- if (!(this_present_request && that_present_request))
- return false;
- if (!this.request.equals(that.request))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_request = true && (isSetRequest());
- builder.append(present_request);
- if (present_request)
- builder.append(request);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(get_multi_slice_args other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetRequest()).compareTo(other.isSetRequest());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetRequest()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.request, other.request);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("get_multi_slice_args(");
- boolean first = true;
-
- sb.append("request:");
- if (this.request == null) {
- sb.append("null");
- } else {
- sb.append(this.request);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- if (request == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'request' was not present! Struct: " + toString());
- }
- // check for sub-struct validity
- if (request != null) {
- request.validate();
- }
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class get_multi_slice_argsStandardSchemeFactory implements SchemeFactory {
- public get_multi_slice_argsStandardScheme getScheme() {
- return new get_multi_slice_argsStandardScheme();
- }
- }
-
- private static class get_multi_slice_argsStandardScheme extends StandardScheme<get_multi_slice_args> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, get_multi_slice_args struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // REQUEST
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.request = new MultiSliceRequest();
- struct.request.read(iprot);
- struct.setRequestIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, get_multi_slice_args struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.request != null) {
- oprot.writeFieldBegin(REQUEST_FIELD_DESC);
- struct.request.write(oprot);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class get_multi_slice_argsTupleSchemeFactory implements SchemeFactory {
- public get_multi_slice_argsTupleScheme getScheme() {
- return new get_multi_slice_argsTupleScheme();
- }
- }
-
- private static class get_multi_slice_argsTupleScheme extends TupleScheme<get_multi_slice_args> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, get_multi_slice_args struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- struct.request.write(oprot);
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, get_multi_slice_args struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- struct.request = new MultiSliceRequest();
- struct.request.read(iprot);
- struct.setRequestIsSet(true);
- }
- }
-
- }
-
- public static class get_multi_slice_result implements org.apache.thrift.TBase<get_multi_slice_result, get_multi_slice_result._Fields>, java.io.Serializable, Cloneable, Comparable<get_multi_slice_result> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("get_multi_slice_result");
-
- private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.LIST, (short)0);
- private static final org.apache.thrift.protocol.TField IRE_FIELD_DESC = new org.apache.thrift.protocol.TField("ire", org.apache.thrift.protocol.TType.STRUCT, (short)1);
- private static final org.apache.thrift.protocol.TField UE_FIELD_DESC = new org.apache.thrift.protocol.TField("ue", org.apache.thrift.protocol.TType.STRUCT, (short)2);
- private static final org.apache.thrift.protocol.TField TE_FIELD_DESC = new org.apache.thrift.protocol.TField("te", org.apache.thrift.protocol.TType.STRUCT, (short)3);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new get_multi_slice_resultStandardSchemeFactory());
- schemes.put(TupleScheme.class, new get_multi_slice_resultTupleSchemeFactory());
- }
-
- public List<ColumnOrSuperColumn> success; // required
- public InvalidRequestException ire; // required
- public UnavailableException ue; // required
- public TimedOutException te; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- SUCCESS((short)0, "success"),
- IRE((short)1, "ire"),
- UE((short)2, "ue"),
- TE((short)3, "te");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 0: // SUCCESS
- return SUCCESS;
- case 1: // IRE
- return IRE;
- case 2: // UE
- return UE;
- case 3: // TE
- return TE;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST,
- new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, ColumnOrSuperColumn.class))));
- tmpMap.put(_Fields.IRE, new org.apache.thrift.meta_data.FieldMetaData("ire", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- tmpMap.put(_Fields.UE, new org.apache.thrift.meta_data.FieldMetaData("ue", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- tmpMap.put(_Fields.TE, new org.apache.thrift.meta_data.FieldMetaData("te", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(get_multi_slice_result.class, metaDataMap);
- }
-
- public get_multi_slice_result() {
- }
-
- public get_multi_slice_result(
- List<ColumnOrSuperColumn> success,
- InvalidRequestException ire,
- UnavailableException ue,
- TimedOutException te)
- {
- this();
- this.success = success;
- this.ire = ire;
- this.ue = ue;
- this.te = te;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public get_multi_slice_result(get_multi_slice_result other) {
- if (other.isSetSuccess()) {
- List<ColumnOrSuperColumn> __this__success = new ArrayList<ColumnOrSuperColumn>(other.success.size());
- for (ColumnOrSuperColumn other_element : other.success) {
- __this__success.add(new ColumnOrSuperColumn(other_element));
- }
- this.success = __this__success;
- }
- if (other.isSetIre()) {
- this.ire = new InvalidRequestException(other.ire);
- }
- if (other.isSetUe()) {
- this.ue = new UnavailableException(other.ue);
- }
- if (other.isSetTe()) {
- this.te = new TimedOutException(other.te);
- }
- }
-
- public get_multi_slice_result deepCopy() {
- return new get_multi_slice_result(this);
- }
-
- @Override
- public void clear() {
- this.success = null;
- this.ire = null;
- this.ue = null;
- this.te = null;
- }
-
- public int getSuccessSize() {
- return (this.success == null) ? 0 : this.success.size();
- }
-
- public java.util.Iterator<ColumnOrSuperColumn> getSuccessIterator() {
- return (this.success == null) ? null : this.success.iterator();
- }
-
- public void addToSuccess(ColumnOrSuperColumn elem) {
- if (this.success == null) {
- this.success = new ArrayList<ColumnOrSuperColumn>();
- }
- this.success.add(elem);
- }
-
- public List<ColumnOrSuperColumn> getSuccess() {
- return this.success;
- }
-
- public get_multi_slice_result setSuccess(List<ColumnOrSuperColumn> success) {
- this.success = success;
- return this;
- }
-
- public void unsetSuccess() {
- this.success = null;
- }
-
- /** Returns true if field success is set (has been assigned a value) and false otherwise */
- public boolean isSetSuccess() {
- return this.success != null;
- }
-
- public void setSuccessIsSet(boolean value) {
- if (!value) {
- this.success = null;
- }
- }
-
- public InvalidRequestException getIre() {
- return this.ire;
- }
-
- public get_multi_slice_result setIre(InvalidRequestException ire) {
- this.ire = ire;
- return this;
- }
-
- public void unsetIre() {
- this.ire = null;
- }
-
- /** Returns true if field ire is set (has been assigned a value) and false otherwise */
- public boolean isSetIre() {
- return this.ire != null;
- }
-
- public void setIreIsSet(boolean value) {
- if (!value) {
- this.ire = null;
- }
- }
-
- public UnavailableException getUe() {
- return this.ue;
- }
-
- public get_multi_slice_result setUe(UnavailableException ue) {
- this.ue = ue;
- return this;
- }
-
- public void unsetUe() {
- this.ue = null;
- }
-
- /** Returns true if field ue is set (has been assigned a value) and false otherwise */
- public boolean isSetUe() {
- return this.ue != null;
- }
-
- public void setUeIsSet(boolean value) {
- if (!value) {
- this.ue = null;
- }
- }
-
- public TimedOutException getTe() {
- return this.te;
- }
-
- public get_multi_slice_result setTe(TimedOutException te) {
- this.te = te;
- return this;
- }
-
- public void unsetTe() {
- this.te = null;
- }
-
- /** Returns true if field te is set (has been assigned a value) and false otherwise */
- public boolean isSetTe() {
- return this.te != null;
- }
-
- public void setTeIsSet(boolean value) {
- if (!value) {
- this.te = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case SUCCESS:
- if (value == null) {
- unsetSuccess();
- } else {
- setSuccess((List<ColumnOrSuperColumn>)value);
- }
- break;
-
- case IRE:
- if (value == null) {
- unsetIre();
- } else {
- setIre((InvalidRequestException)value);
- }
- break;
-
- case UE:
- if (value == null) {
- unsetUe();
- } else {
- setUe((UnavailableException)value);
- }
- break;
-
- case TE:
- if (value == null) {
- unsetTe();
- } else {
- setTe((TimedOutException)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case SUCCESS:
- return getSuccess();
-
- case IRE:
- return getIre();
-
- case UE:
- return getUe();
-
- case TE:
- return getTe();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case SUCCESS:
- return isSetSuccess();
- case IRE:
- return isSetIre();
- case UE:
- return isSetUe();
- case TE:
- return isSetTe();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof get_multi_slice_result)
- return this.equals((get_multi_slice_result)that);
- return false;
- }
-
- public boolean equals(get_multi_slice_result that) {
- if (that == null)
- return false;
-
- boolean this_present_success = true && this.isSetSuccess();
- boolean that_present_success = true && that.isSetSuccess();
- if (this_present_success || that_present_success) {
- if (!(this_present_success && that_present_success))
- return false;
- if (!this.success.equals(that.success))
- return false;
- }
-
- boolean this_present_ire = true && this.isSetIre();
- boolean that_present_ire = true && that.isSetIre();
- if (this_present_ire || that_present_ire) {
- if (!(this_present_ire && that_present_ire))
- return false;
- if (!this.ire.equals(that.ire))
- return false;
- }
-
- boolean this_present_ue = true && this.isSetUe();
- boolean that_present_ue = true && that.isSetUe();
- if (this_present_ue || that_present_ue) {
- if (!(this_present_ue && that_present_ue))
- return false;
- if (!this.ue.equals(that.ue))
- return false;
- }
-
- boolean this_present_te = true && this.isSetTe();
- boolean that_present_te = true && that.isSetTe();
- if (this_present_te || that_present_te) {
- if (!(this_present_te && that_present_te))
- return false;
- if (!this.te.equals(that.te))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_success = true && (isSetSuccess());
- builder.append(present_success);
- if (present_success)
- builder.append(success);
-
- boolean present_ire = true && (isSetIre());
- builder.append(present_ire);
- if (present_ire)
- builder.append(ire);
-
- boolean present_ue = true && (isSetUe());
- builder.append(present_ue);
- if (present_ue)
- builder.append(ue);
-
- boolean present_te = true && (isSetTe());
- builder.append(present_te);
- if (present_te)
- builder.append(te);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(get_multi_slice_result other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(other.isSetSuccess());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetSuccess()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, other.success);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetIre()).compareTo(other.isSetIre());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetIre()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.ire, other.ire);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetUe()).compareTo(other.isSetUe());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetUe()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.ue, other.ue);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetTe()).compareTo(other.isSetTe());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetTe()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.te, other.te);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("get_multi_slice_result(");
- boolean first = true;
-
- sb.append("success:");
- if (this.success == null) {
- sb.append("null");
- } else {
- sb.append(this.success);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("ire:");
- if (this.ire == null) {
- sb.append("null");
- } else {
- sb.append(this.ire);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("ue:");
- if (this.ue == null) {
- sb.append("null");
- } else {
- sb.append(this.ue);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("te:");
- if (this.te == null) {
- sb.append("null");
- } else {
- sb.append(this.te);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class get_multi_slice_resultStandardSchemeFactory implements SchemeFactory {
- public get_multi_slice_resultStandardScheme getScheme() {
- return new get_multi_slice_resultStandardScheme();
- }
- }
-
- private static class get_multi_slice_resultStandardScheme extends StandardScheme<get_multi_slice_result> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, get_multi_slice_result struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 0: // SUCCESS
- if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
- {
- org.apache.thrift.protocol.TList _list372 = iprot.readListBegin();
- struct.success = new ArrayList<ColumnOrSuperColumn>(_list372.size);
- for (int _i373 = 0; _i373 < _list372.size; ++_i373)
- {
- ColumnOrSuperColumn _elem374;
- _elem374 = new ColumnOrSuperColumn();
- _elem374.read(iprot);
- struct.success.add(_elem374);
- }
- iprot.readListEnd();
- }
- struct.setSuccessIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 1: // IRE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.ire = new InvalidRequestException();
- struct.ire.read(iprot);
- struct.setIreIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 2: // UE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.ue = new UnavailableException();
- struct.ue.read(iprot);
- struct.setUeIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 3: // TE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.te = new TimedOutException();
- struct.te.read(iprot);
- struct.setTeIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, get_multi_slice_result struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.success != null) {
- oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
- {
- oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.success.size()));
- for (ColumnOrSuperColumn _iter375 : struct.success)
- {
- _iter375.write(oprot);
- }
- oprot.writeListEnd();
- }
- oprot.writeFieldEnd();
- }
- if (struct.ire != null) {
- oprot.writeFieldBegin(IRE_FIELD_DESC);
- struct.ire.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.ue != null) {
- oprot.writeFieldBegin(UE_FIELD_DESC);
- struct.ue.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.te != null) {
- oprot.writeFieldBegin(TE_FIELD_DESC);
- struct.te.write(oprot);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class get_multi_slice_resultTupleSchemeFactory implements SchemeFactory {
- public get_multi_slice_resultTupleScheme getScheme() {
- return new get_multi_slice_resultTupleScheme();
- }
- }
-
- private static class get_multi_slice_resultTupleScheme extends TupleScheme<get_multi_slice_result> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, get_multi_slice_result struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- BitSet optionals = new BitSet();
- if (struct.isSetSuccess()) {
- optionals.set(0);
- }
- if (struct.isSetIre()) {
- optionals.set(1);
- }
- if (struct.isSetUe()) {
- optionals.set(2);
- }
- if (struct.isSetTe()) {
- optionals.set(3);
- }
- oprot.writeBitSet(optionals, 4);
- if (struct.isSetSuccess()) {
- {
- oprot.writeI32(struct.success.size());
- for (ColumnOrSuperColumn _iter376 : struct.success)
- {
- _iter376.write(oprot);
- }
- }
- }
- if (struct.isSetIre()) {
- struct.ire.write(oprot);
- }
- if (struct.isSetUe()) {
- struct.ue.write(oprot);
- }
- if (struct.isSetTe()) {
- struct.te.write(oprot);
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, get_multi_slice_result struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- BitSet incoming = iprot.readBitSet(4);
- if (incoming.get(0)) {
- {
- org.apache.thrift.protocol.TList _list377 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
- struct.success = new ArrayList<ColumnOrSuperColumn>(_list377.size);
- for (int _i378 = 0; _i378 < _list377.size; ++_i378)
- {
- ColumnOrSuperColumn _elem379;
- _elem379 = new ColumnOrSuperColumn();
- _elem379.read(iprot);
- struct.success.add(_elem379);
- }
- }
- struct.setSuccessIsSet(true);
- }
- if (incoming.get(1)) {
- struct.ire = new InvalidRequestException();
- struct.ire.read(iprot);
- struct.setIreIsSet(true);
- }
- if (incoming.get(2)) {
- struct.ue = new UnavailableException();
- struct.ue.read(iprot);
- struct.setUeIsSet(true);
- }
- if (incoming.get(3)) {
- struct.te = new TimedOutException();
- struct.te.read(iprot);
- struct.setTeIsSet(true);
- }
- }
- }
-
- }
-
- public static class describe_schema_versions_args implements org.apache.thrift.TBase<describe_schema_versions_args, describe_schema_versions_args._Fields>, java.io.Serializable, Cloneable, Comparable<describe_schema_versions_args> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("describe_schema_versions_args");
-
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new describe_schema_versions_argsStandardSchemeFactory());
- schemes.put(TupleScheme.class, new describe_schema_versions_argsTupleSchemeFactory());
- }
-
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-;
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(describe_schema_versions_args.class, metaDataMap);
- }
-
- public describe_schema_versions_args() {
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public describe_schema_versions_args(describe_schema_versions_args other) {
- }
-
- public describe_schema_versions_args deepCopy() {
- return new describe_schema_versions_args(this);
- }
-
- @Override
- public void clear() {
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof describe_schema_versions_args)
- return this.equals((describe_schema_versions_args)that);
- return false;
- }
-
- public boolean equals(describe_schema_versions_args that) {
- if (that == null)
- return false;
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(describe_schema_versions_args other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("describe_schema_versions_args(");
- boolean first = true;
-
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class describe_schema_versions_argsStandardSchemeFactory implements SchemeFactory {
- public describe_schema_versions_argsStandardScheme getScheme() {
- return new describe_schema_versions_argsStandardScheme();
- }
- }
-
- private static class describe_schema_versions_argsStandardScheme extends StandardScheme<describe_schema_versions_args> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, describe_schema_versions_args struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, describe_schema_versions_args struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class describe_schema_versions_argsTupleSchemeFactory implements SchemeFactory {
- public describe_schema_versions_argsTupleScheme getScheme() {
- return new describe_schema_versions_argsTupleScheme();
- }
- }
-
- private static class describe_schema_versions_argsTupleScheme extends TupleScheme<describe_schema_versions_args> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, describe_schema_versions_args struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, describe_schema_versions_args struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- }
- }
-
- }
-
- public static class describe_schema_versions_result implements org.apache.thrift.TBase<describe_schema_versions_result, describe_schema_versions_result._Fields>, java.io.Serializable, Cloneable, Comparable<describe_schema_versions_result> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("describe_schema_versions_result");
-
- private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.MAP, (short)0);
- private static final org.apache.thrift.protocol.TField IRE_FIELD_DESC = new org.apache.thrift.protocol.TField("ire", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new describe_schema_versions_resultStandardSchemeFactory());
- schemes.put(TupleScheme.class, new describe_schema_versions_resultTupleSchemeFactory());
- }
-
- public Map<String,List<String>> success; // required
- public InvalidRequestException ire; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- SUCCESS((short)0, "success"),
- IRE((short)1, "ire");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 0: // SUCCESS
- return SUCCESS;
- case 1: // IRE
- return IRE;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING),
- new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)))));
- tmpMap.put(_Fields.IRE, new org.apache.thrift.meta_data.FieldMetaData("ire", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(describe_schema_versions_result.class, metaDataMap);
- }
-
- public describe_schema_versions_result() {
- }
-
- public describe_schema_versions_result(
- Map<String,List<String>> success,
- InvalidRequestException ire)
- {
- this();
- this.success = success;
- this.ire = ire;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public describe_schema_versions_result(describe_schema_versions_result other) {
- if (other.isSetSuccess()) {
- Map<String,List<String>> __this__success = new HashMap<String,List<String>>(other.success.size());
- for (Map.Entry<String, List<String>> other_element : other.success.entrySet()) {
-
- String other_element_key = other_element.getKey();
- List<String> other_element_value = other_element.getValue();
-
- String __this__success_copy_key = other_element_key;
-
- List<String> __this__success_copy_value = new ArrayList<String>(other_element_value);
-
- __this__success.put(__this__success_copy_key, __this__success_copy_value);
- }
- this.success = __this__success;
- }
- if (other.isSetIre()) {
- this.ire = new InvalidRequestException(other.ire);
- }
- }
-
- public describe_schema_versions_result deepCopy() {
- return new describe_schema_versions_result(this);
- }
-
- @Override
- public void clear() {
- this.success = null;
- this.ire = null;
- }
-
- public int getSuccessSize() {
- return (this.success == null) ? 0 : this.success.size();
- }
-
- public void putToSuccess(String key, List<String> val) {
- if (this.success == null) {
- this.success = new HashMap<String,List<String>>();
- }
- this.success.put(key, val);
- }
-
- public Map<String,List<String>> getSuccess() {
- return this.success;
- }
-
- public describe_schema_versions_result setSuccess(Map<String,List<String>> success) {
- this.success = success;
- return this;
- }
-
- public void unsetSuccess() {
- this.success = null;
- }
-
- /** Returns true if field success is set (has been assigned a value) and false otherwise */
- public boolean isSetSuccess() {
- return this.success != null;
- }
-
- public void setSuccessIsSet(boolean value) {
- if (!value) {
- this.success = null;
- }
- }
-
- public InvalidRequestException getIre() {
- return this.ire;
- }
-
- public describe_schema_versions_result setIre(InvalidRequestException ire) {
- this.ire = ire;
- return this;
- }
-
- public void unsetIre() {
- this.ire = null;
- }
-
- /** Returns true if field ire is set (has been assigned a value) and false otherwise */
- public boolean isSetIre() {
- return this.ire != null;
- }
-
- public void setIreIsSet(boolean value) {
- if (!value) {
- this.ire = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case SUCCESS:
- if (value == null) {
- unsetSuccess();
- } else {
- setSuccess((Map<String,List<String>>)value);
- }
- break;
-
- case IRE:
- if (value == null) {
- unsetIre();
- } else {
- setIre((InvalidRequestException)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case SUCCESS:
- return getSuccess();
-
- case IRE:
- return getIre();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case SUCCESS:
- return isSetSuccess();
- case IRE:
- return isSetIre();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof describe_schema_versions_result)
- return this.equals((describe_schema_versions_result)that);
- return false;
- }
-
- public boolean equals(describe_schema_versions_result that) {
- if (that == null)
- return false;
-
- boolean this_present_success = true && this.isSetSuccess();
- boolean that_present_success = true && that.isSetSuccess();
- if (this_present_success || that_present_success) {
- if (!(this_present_success && that_present_success))
- return false;
- if (!this.success.equals(that.success))
- return false;
- }
-
- boolean this_present_ire = true && this.isSetIre();
- boolean that_present_ire = true && that.isSetIre();
- if (this_present_ire || that_present_ire) {
- if (!(this_present_ire && that_present_ire))
- return false;
- if (!this.ire.equals(that.ire))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_success = true && (isSetSuccess());
- builder.append(present_success);
- if (present_success)
- builder.append(success);
-
- boolean present_ire = true && (isSetIre());
- builder.append(present_ire);
- if (present_ire)
- builder.append(ire);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(describe_schema_versions_result other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(other.isSetSuccess());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetSuccess()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, other.success);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetIre()).compareTo(other.isSetIre());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetIre()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.ire, other.ire);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("describe_schema_versions_result(");
- boolean first = true;
-
- sb.append("success:");
- if (this.success == null) {
- sb.append("null");
- } else {
- sb.append(this.success);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("ire:");
- if (this.ire == null) {
- sb.append("null");
- } else {
- sb.append(this.ire);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class describe_schema_versions_resultStandardSchemeFactory implements SchemeFactory {
- public describe_schema_versions_resultStandardScheme getScheme() {
- return new describe_schema_versions_resultStandardScheme();
- }
- }
-
- private static class describe_schema_versions_resultStandardScheme extends StandardScheme<describe_schema_versions_result> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, describe_schema_versions_result struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 0: // SUCCESS
- if (schemeField.type == org.apache.thrift.protocol.TType.MAP) {
- {
- org.apache.thrift.protocol.TMap _map380 = iprot.readMapBegin();
- struct.success = new HashMap<String,List<String>>(2*_map380.size);
- for (int _i381 = 0; _i381 < _map380.size; ++_i381)
- {
- String _key382;
- List<String> _val383;
- _key382 = iprot.readString();
- {
- org.apache.thrift.protocol.TList _list384 = iprot.readListBegin();
- _val383 = new ArrayList<String>(_list384.size);
- for (int _i385 = 0; _i385 < _list384.size; ++_i385)
- {
- String _elem386;
- _elem386 = iprot.readString();
- _val383.add(_elem386);
- }
- iprot.readListEnd();
- }
- struct.success.put(_key382, _val383);
- }
- iprot.readMapEnd();
- }
- struct.setSuccessIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 1: // IRE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.ire = new InvalidRequestException();
- struct.ire.read(iprot);
- struct.setIreIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, describe_schema_versions_result struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.success != null) {
- oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
- {
- oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.LIST, struct.success.size()));
- for (Map.Entry<String, List<String>> _iter387 : struct.success.entrySet())
- {
- oprot.writeString(_iter387.getKey());
- {
- oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, _iter387.getValue().size()));
- for (String _iter388 : _iter387.getValue())
- {
- oprot.writeString(_iter388);
- }
- oprot.writeListEnd();
- }
- }
- oprot.writeMapEnd();
- }
- oprot.writeFieldEnd();
- }
- if (struct.ire != null) {
- oprot.writeFieldBegin(IRE_FIELD_DESC);
- struct.ire.write(oprot);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class describe_schema_versions_resultTupleSchemeFactory implements SchemeFactory {
- public describe_schema_versions_resultTupleScheme getScheme() {
- return new describe_schema_versions_resultTupleScheme();
- }
- }
-
- private static class describe_schema_versions_resultTupleScheme extends TupleScheme<describe_schema_versions_result> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, describe_schema_versions_result struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- BitSet optionals = new BitSet();
- if (struct.isSetSuccess()) {
- optionals.set(0);
- }
- if (struct.isSetIre()) {
- optionals.set(1);
- }
- oprot.writeBitSet(optionals, 2);
- if (struct.isSetSuccess()) {
- {
- oprot.writeI32(struct.success.size());
- for (Map.Entry<String, List<String>> _iter389 : struct.success.entrySet())
- {
- oprot.writeString(_iter389.getKey());
- {
- oprot.writeI32(_iter389.getValue().size());
- for (String _iter390 : _iter389.getValue())
- {
- oprot.writeString(_iter390);
- }
- }
- }
- }
- }
- if (struct.isSetIre()) {
- struct.ire.write(oprot);
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, describe_schema_versions_result struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- BitSet incoming = iprot.readBitSet(2);
- if (incoming.get(0)) {
- {
- org.apache.thrift.protocol.TMap _map391 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.LIST, iprot.readI32());
- struct.success = new HashMap<String,List<String>>(2*_map391.size);
- for (int _i392 = 0; _i392 < _map391.size; ++_i392)
- {
- String _key393;
- List<String> _val394;
- _key393 = iprot.readString();
- {
- org.apache.thrift.protocol.TList _list395 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32());
- _val394 = new ArrayList<String>(_list395.size);
- for (int _i396 = 0; _i396 < _list395.size; ++_i396)
- {
- String _elem397;
- _elem397 = iprot.readString();
- _val394.add(_elem397);
- }
- }
- struct.success.put(_key393, _val394);
- }
- }
- struct.setSuccessIsSet(true);
- }
- if (incoming.get(1)) {
- struct.ire = new InvalidRequestException();
- struct.ire.read(iprot);
- struct.setIreIsSet(true);
- }
- }
- }
-
- }
-
- public static class describe_keyspaces_args implements org.apache.thrift.TBase<describe_keyspaces_args, describe_keyspaces_args._Fields>, java.io.Serializable, Cloneable, Comparable<describe_keyspaces_args> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("describe_keyspaces_args");
-
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new describe_keyspaces_argsStandardSchemeFactory());
- schemes.put(TupleScheme.class, new describe_keyspaces_argsTupleSchemeFactory());
- }
-
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-;
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(describe_keyspaces_args.class, metaDataMap);
- }
-
- public describe_keyspaces_args() {
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public describe_keyspaces_args(describe_keyspaces_args other) {
- }
-
- public describe_keyspaces_args deepCopy() {
- return new describe_keyspaces_args(this);
- }
-
- @Override
- public void clear() {
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof describe_keyspaces_args)
- return this.equals((describe_keyspaces_args)that);
- return false;
- }
-
- public boolean equals(describe_keyspaces_args that) {
- if (that == null)
- return false;
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(describe_keyspaces_args other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("describe_keyspaces_args(");
- boolean first = true;
-
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class describe_keyspaces_argsStandardSchemeFactory implements SchemeFactory {
- public describe_keyspaces_argsStandardScheme getScheme() {
- return new describe_keyspaces_argsStandardScheme();
- }
- }
-
- private static class describe_keyspaces_argsStandardScheme extends StandardScheme<describe_keyspaces_args> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, describe_keyspaces_args struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, describe_keyspaces_args struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class describe_keyspaces_argsTupleSchemeFactory implements SchemeFactory {
- public describe_keyspaces_argsTupleScheme getScheme() {
- return new describe_keyspaces_argsTupleScheme();
- }
- }
-
- private static class describe_keyspaces_argsTupleScheme extends TupleScheme<describe_keyspaces_args> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, describe_keyspaces_args struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, describe_keyspaces_args struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- }
- }
-
- }
-
- public static class describe_keyspaces_result implements org.apache.thrift.TBase<describe_keyspaces_result, describe_keyspaces_result._Fields>, java.io.Serializable, Cloneable, Comparable<describe_keyspaces_result> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("describe_keyspaces_result");
-
- private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.LIST, (short)0);
- private static final org.apache.thrift.protocol.TField IRE_FIELD_DESC = new org.apache.thrift.protocol.TField("ire", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new describe_keyspaces_resultStandardSchemeFactory());
- schemes.put(TupleScheme.class, new describe_keyspaces_resultTupleSchemeFactory());
- }
-
- public List<KsDef> success; // required
- public InvalidRequestException ire; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- SUCCESS((short)0, "success"),
- IRE((short)1, "ire");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 0: // SUCCESS
- return SUCCESS;
- case 1: // IRE
- return IRE;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST,
- new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, KsDef.class))));
- tmpMap.put(_Fields.IRE, new org.apache.thrift.meta_data.FieldMetaData("ire", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(describe_keyspaces_result.class, metaDataMap);
- }
-
- public describe_keyspaces_result() {
- }
-
- public describe_keyspaces_result(
- List<KsDef> success,
- InvalidRequestException ire)
- {
- this();
- this.success = success;
- this.ire = ire;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public describe_keyspaces_result(describe_keyspaces_result other) {
- if (other.isSetSuccess()) {
- List<KsDef> __this__success = new ArrayList<KsDef>(other.success.size());
- for (KsDef other_element : other.success) {
- __this__success.add(new KsDef(other_element));
- }
- this.success = __this__success;
- }
- if (other.isSetIre()) {
- this.ire = new InvalidRequestException(other.ire);
- }
- }
-
- public describe_keyspaces_result deepCopy() {
- return new describe_keyspaces_result(this);
- }
-
- @Override
- public void clear() {
- this.success = null;
- this.ire = null;
- }
-
- public int getSuccessSize() {
- return (this.success == null) ? 0 : this.success.size();
- }
-
- public java.util.Iterator<KsDef> getSuccessIterator() {
- return (this.success == null) ? null : this.success.iterator();
- }
-
- public void addToSuccess(KsDef elem) {
- if (this.success == null) {
- this.success = new ArrayList<KsDef>();
- }
- this.success.add(elem);
- }
-
- public List<KsDef> getSuccess() {
- return this.success;
- }
-
- public describe_keyspaces_result setSuccess(List<KsDef> success) {
- this.success = success;
- return this;
- }
-
- public void unsetSuccess() {
- this.success = null;
- }
-
- /** Returns true if field success is set (has been assigned a value) and false otherwise */
- public boolean isSetSuccess() {
- return this.success != null;
- }
-
- public void setSuccessIsSet(boolean value) {
- if (!value) {
- this.success = null;
- }
- }
-
- public InvalidRequestException getIre() {
- return this.ire;
- }
-
- public describe_keyspaces_result setIre(InvalidRequestException ire) {
- this.ire = ire;
- return this;
- }
-
- public void unsetIre() {
- this.ire = null;
- }
-
- /** Returns true if field ire is set (has been assigned a value) and false otherwise */
- public boolean isSetIre() {
- return this.ire != null;
- }
-
- public void setIreIsSet(boolean value) {
- if (!value) {
- this.ire = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case SUCCESS:
- if (value == null) {
- unsetSuccess();
- } else {
- setSuccess((List<KsDef>)value);
- }
- break;
-
- case IRE:
- if (value == null) {
- unsetIre();
- } else {
- setIre((InvalidRequestException)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case SUCCESS:
- return getSuccess();
-
- case IRE:
- return getIre();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case SUCCESS:
- return isSetSuccess();
- case IRE:
- return isSetIre();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof describe_keyspaces_result)
- return this.equals((describe_keyspaces_result)that);
- return false;
- }
-
- public boolean equals(describe_keyspaces_result that) {
- if (that == null)
- return false;
-
- boolean this_present_success = true && this.isSetSuccess();
- boolean that_present_success = true && that.isSetSuccess();
- if (this_present_success || that_present_success) {
- if (!(this_present_success && that_present_success))
- return false;
- if (!this.success.equals(that.success))
- return false;
- }
-
- boolean this_present_ire = true && this.isSetIre();
- boolean that_present_ire = true && that.isSetIre();
- if (this_present_ire || that_present_ire) {
- if (!(this_present_ire && that_present_ire))
- return false;
- if (!this.ire.equals(that.ire))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_success = true && (isSetSuccess());
- builder.append(present_success);
- if (present_success)
- builder.append(success);
-
- boolean present_ire = true && (isSetIre());
- builder.append(present_ire);
- if (present_ire)
- builder.append(ire);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(describe_keyspaces_result other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(other.isSetSuccess());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetSuccess()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, other.success);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetIre()).compareTo(other.isSetIre());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetIre()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.ire, other.ire);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("describe_keyspaces_result(");
- boolean first = true;
-
- sb.append("success:");
- if (this.success == null) {
- sb.append("null");
- } else {
- sb.append(this.success);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("ire:");
- if (this.ire == null) {
- sb.append("null");
- } else {
- sb.append(this.ire);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class describe_keyspaces_resultStandardSchemeFactory implements SchemeFactory {
- public describe_keyspaces_resultStandardScheme getScheme() {
- return new describe_keyspaces_resultStandardScheme();
- }
- }
-
- private static class describe_keyspaces_resultStandardScheme extends StandardScheme<describe_keyspaces_result> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, describe_keyspaces_result struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 0: // SUCCESS
- if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
- {
- org.apache.thrift.protocol.TList _list398 = iprot.readListBegin();
- struct.success = new ArrayList<KsDef>(_list398.size);
- for (int _i399 = 0; _i399 < _list398.size; ++_i399)
- {
- KsDef _elem400;
- _elem400 = new KsDef();
- _elem400.read(iprot);
- struct.success.add(_elem400);
- }
- iprot.readListEnd();
- }
- struct.setSuccessIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 1: // IRE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.ire = new InvalidRequestException();
- struct.ire.read(iprot);
- struct.setIreIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, describe_keyspaces_result struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.success != null) {
- oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
- {
- oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.success.size()));
- for (KsDef _iter401 : struct.success)
- {
- _iter401.write(oprot);
- }
- oprot.writeListEnd();
- }
- oprot.writeFieldEnd();
- }
- if (struct.ire != null) {
- oprot.writeFieldBegin(IRE_FIELD_DESC);
- struct.ire.write(oprot);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class describe_keyspaces_resultTupleSchemeFactory implements SchemeFactory {
- public describe_keyspaces_resultTupleScheme getScheme() {
- return new describe_keyspaces_resultTupleScheme();
- }
- }
-
- private static class describe_keyspaces_resultTupleScheme extends TupleScheme<describe_keyspaces_result> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, describe_keyspaces_result struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- BitSet optionals = new BitSet();
- if (struct.isSetSuccess()) {
- optionals.set(0);
- }
- if (struct.isSetIre()) {
- optionals.set(1);
- }
- oprot.writeBitSet(optionals, 2);
- if (struct.isSetSuccess()) {
- {
- oprot.writeI32(struct.success.size());
- for (KsDef _iter402 : struct.success)
- {
- _iter402.write(oprot);
- }
- }
- }
- if (struct.isSetIre()) {
- struct.ire.write(oprot);
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, describe_keyspaces_result struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- BitSet incoming = iprot.readBitSet(2);
- if (incoming.get(0)) {
- {
- org.apache.thrift.protocol.TList _list403 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
- struct.success = new ArrayList<KsDef>(_list403.size);
- for (int _i404 = 0; _i404 < _list403.size; ++_i404)
- {
- KsDef _elem405;
- _elem405 = new KsDef();
- _elem405.read(iprot);
- struct.success.add(_elem405);
- }
- }
- struct.setSuccessIsSet(true);
- }
- if (incoming.get(1)) {
- struct.ire = new InvalidRequestException();
- struct.ire.read(iprot);
- struct.setIreIsSet(true);
- }
- }
- }
-
- }
-
- public static class describe_cluster_name_args implements org.apache.thrift.TBase<describe_cluster_name_args, describe_cluster_name_args._Fields>, java.io.Serializable, Cloneable, Comparable<describe_cluster_name_args> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("describe_cluster_name_args");
-
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new describe_cluster_name_argsStandardSchemeFactory());
- schemes.put(TupleScheme.class, new describe_cluster_name_argsTupleSchemeFactory());
- }
-
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-;
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(describe_cluster_name_args.class, metaDataMap);
- }
-
- public describe_cluster_name_args() {
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public describe_cluster_name_args(describe_cluster_name_args other) {
- }
-
- public describe_cluster_name_args deepCopy() {
- return new describe_cluster_name_args(this);
- }
-
- @Override
- public void clear() {
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof describe_cluster_name_args)
- return this.equals((describe_cluster_name_args)that);
- return false;
- }
-
- public boolean equals(describe_cluster_name_args that) {
- if (that == null)
- return false;
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(describe_cluster_name_args other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("describe_cluster_name_args(");
- boolean first = true;
-
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class describe_cluster_name_argsStandardSchemeFactory implements SchemeFactory {
- public describe_cluster_name_argsStandardScheme getScheme() {
- return new describe_cluster_name_argsStandardScheme();
- }
- }
-
- private static class describe_cluster_name_argsStandardScheme extends StandardScheme<describe_cluster_name_args> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, describe_cluster_name_args struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, describe_cluster_name_args struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class describe_cluster_name_argsTupleSchemeFactory implements SchemeFactory {
- public describe_cluster_name_argsTupleScheme getScheme() {
- return new describe_cluster_name_argsTupleScheme();
- }
- }
-
- private static class describe_cluster_name_argsTupleScheme extends TupleScheme<describe_cluster_name_args> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, describe_cluster_name_args struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, describe_cluster_name_args struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- }
- }
-
- }
-
- public static class describe_cluster_name_result implements org.apache.thrift.TBase<describe_cluster_name_result, describe_cluster_name_result._Fields>, java.io.Serializable, Cloneable, Comparable<describe_cluster_name_result> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("describe_cluster_name_result");
-
- private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRING, (short)0);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new describe_cluster_name_resultStandardSchemeFactory());
- schemes.put(TupleScheme.class, new describe_cluster_name_resultTupleSchemeFactory());
- }
-
- public String success; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- SUCCESS((short)0, "success");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 0: // SUCCESS
- return SUCCESS;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(describe_cluster_name_result.class, metaDataMap);
- }
-
- public describe_cluster_name_result() {
- }
-
- public describe_cluster_name_result(
- String success)
- {
- this();
- this.success = success;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public describe_cluster_name_result(describe_cluster_name_result other) {
- if (other.isSetSuccess()) {
- this.success = other.success;
- }
- }
-
- public describe_cluster_name_result deepCopy() {
- return new describe_cluster_name_result(this);
- }
-
- @Override
- public void clear() {
- this.success = null;
- }
-
- public String getSuccess() {
- return this.success;
- }
-
- public describe_cluster_name_result setSuccess(String success) {
- this.success = success;
- return this;
- }
-
- public void unsetSuccess() {
- this.success = null;
- }
-
- /** Returns true if field success is set (has been assigned a value) and false otherwise */
- public boolean isSetSuccess() {
- return this.success != null;
- }
-
- public void setSuccessIsSet(boolean value) {
- if (!value) {
- this.success = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case SUCCESS:
- if (value == null) {
- unsetSuccess();
- } else {
- setSuccess((String)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case SUCCESS:
- return getSuccess();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case SUCCESS:
- return isSetSuccess();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof describe_cluster_name_result)
- return this.equals((describe_cluster_name_result)that);
- return false;
- }
-
- public boolean equals(describe_cluster_name_result that) {
- if (that == null)
- return false;
-
- boolean this_present_success = true && this.isSetSuccess();
- boolean that_present_success = true && that.isSetSuccess();
- if (this_present_success || that_present_success) {
- if (!(this_present_success && that_present_success))
- return false;
- if (!this.success.equals(that.success))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_success = true && (isSetSuccess());
- builder.append(present_success);
- if (present_success)
- builder.append(success);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(describe_cluster_name_result other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(other.isSetSuccess());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetSuccess()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, other.success);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("describe_cluster_name_result(");
- boolean first = true;
-
- sb.append("success:");
- if (this.success == null) {
- sb.append("null");
- } else {
- sb.append(this.success);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class describe_cluster_name_resultStandardSchemeFactory implements SchemeFactory {
- public describe_cluster_name_resultStandardScheme getScheme() {
- return new describe_cluster_name_resultStandardScheme();
- }
- }
-
- private static class describe_cluster_name_resultStandardScheme extends StandardScheme<describe_cluster_name_result> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, describe_cluster_name_result struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 0: // SUCCESS
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.success = iprot.readString();
- struct.setSuccessIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, describe_cluster_name_result struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.success != null) {
- oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
- oprot.writeString(struct.success);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class describe_cluster_name_resultTupleSchemeFactory implements SchemeFactory {
- public describe_cluster_name_resultTupleScheme getScheme() {
- return new describe_cluster_name_resultTupleScheme();
- }
- }
-
- private static class describe_cluster_name_resultTupleScheme extends TupleScheme<describe_cluster_name_result> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, describe_cluster_name_result struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- BitSet optionals = new BitSet();
- if (struct.isSetSuccess()) {
- optionals.set(0);
- }
- oprot.writeBitSet(optionals, 1);
- if (struct.isSetSuccess()) {
- oprot.writeString(struct.success);
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, describe_cluster_name_result struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- BitSet incoming = iprot.readBitSet(1);
- if (incoming.get(0)) {
- struct.success = iprot.readString();
- struct.setSuccessIsSet(true);
- }
- }
- }
-
- }
-
- public static class describe_version_args implements org.apache.thrift.TBase<describe_version_args, describe_version_args._Fields>, java.io.Serializable, Cloneable, Comparable<describe_version_args> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("describe_version_args");
-
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new describe_version_argsStandardSchemeFactory());
- schemes.put(TupleScheme.class, new describe_version_argsTupleSchemeFactory());
- }
-
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-;
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(describe_version_args.class, metaDataMap);
- }
-
- public describe_version_args() {
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public describe_version_args(describe_version_args other) {
- }
-
- public describe_version_args deepCopy() {
- return new describe_version_args(this);
- }
-
- @Override
- public void clear() {
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof describe_version_args)
- return this.equals((describe_version_args)that);
- return false;
- }
-
- public boolean equals(describe_version_args that) {
- if (that == null)
- return false;
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(describe_version_args other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("describe_version_args(");
- boolean first = true;
-
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class describe_version_argsStandardSchemeFactory implements SchemeFactory {
- public describe_version_argsStandardScheme getScheme() {
- return new describe_version_argsStandardScheme();
- }
- }
-
- private static class describe_version_argsStandardScheme extends StandardScheme<describe_version_args> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, describe_version_args struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, describe_version_args struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class describe_version_argsTupleSchemeFactory implements SchemeFactory {
- public describe_version_argsTupleScheme getScheme() {
- return new describe_version_argsTupleScheme();
- }
- }
-
- private static class describe_version_argsTupleScheme extends TupleScheme<describe_version_args> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, describe_version_args struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, describe_version_args struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- }
- }
-
- }
-
- public static class describe_version_result implements org.apache.thrift.TBase<describe_version_result, describe_version_result._Fields>, java.io.Serializable, Cloneable, Comparable<describe_version_result> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("describe_version_result");
-
- private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRING, (short)0);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new describe_version_resultStandardSchemeFactory());
- schemes.put(TupleScheme.class, new describe_version_resultTupleSchemeFactory());
- }
-
- public String success; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- SUCCESS((short)0, "success");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 0: // SUCCESS
- return SUCCESS;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(describe_version_result.class, metaDataMap);
- }
-
- public describe_version_result() {
- }
-
- public describe_version_result(
- String success)
- {
- this();
- this.success = success;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public describe_version_result(describe_version_result other) {
- if (other.isSetSuccess()) {
- this.success = other.success;
- }
- }
-
- public describe_version_result deepCopy() {
- return new describe_version_result(this);
- }
-
- @Override
- public void clear() {
- this.success = null;
- }
-
- public String getSuccess() {
- return this.success;
- }
-
- public describe_version_result setSuccess(String success) {
- this.success = success;
- return this;
- }
-
- public void unsetSuccess() {
- this.success = null;
- }
-
- /** Returns true if field success is set (has been assigned a value) and false otherwise */
- public boolean isSetSuccess() {
- return this.success != null;
- }
-
- public void setSuccessIsSet(boolean value) {
- if (!value) {
- this.success = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case SUCCESS:
- if (value == null) {
- unsetSuccess();
- } else {
- setSuccess((String)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case SUCCESS:
- return getSuccess();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case SUCCESS:
- return isSetSuccess();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof describe_version_result)
- return this.equals((describe_version_result)that);
- return false;
- }
-
- public boolean equals(describe_version_result that) {
- if (that == null)
- return false;
-
- boolean this_present_success = true && this.isSetSuccess();
- boolean that_present_success = true && that.isSetSuccess();
- if (this_present_success || that_present_success) {
- if (!(this_present_success && that_present_success))
- return false;
- if (!this.success.equals(that.success))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_success = true && (isSetSuccess());
- builder.append(present_success);
- if (present_success)
- builder.append(success);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(describe_version_result other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(other.isSetSuccess());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetSuccess()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, other.success);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("describe_version_result(");
- boolean first = true;
-
- sb.append("success:");
- if (this.success == null) {
- sb.append("null");
- } else {
- sb.append(this.success);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class describe_version_resultStandardSchemeFactory implements SchemeFactory {
- public describe_version_resultStandardScheme getScheme() {
- return new describe_version_resultStandardScheme();
- }
- }
-
- private static class describe_version_resultStandardScheme extends StandardScheme<describe_version_result> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, describe_version_result struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 0: // SUCCESS
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.success = iprot.readString();
- struct.setSuccessIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, describe_version_result struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.success != null) {
- oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
- oprot.writeString(struct.success);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class describe_version_resultTupleSchemeFactory implements SchemeFactory {
- public describe_version_resultTupleScheme getScheme() {
- return new describe_version_resultTupleScheme();
- }
- }
-
- private static class describe_version_resultTupleScheme extends TupleScheme<describe_version_result> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, describe_version_result struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- BitSet optionals = new BitSet();
- if (struct.isSetSuccess()) {
- optionals.set(0);
- }
- oprot.writeBitSet(optionals, 1);
- if (struct.isSetSuccess()) {
- oprot.writeString(struct.success);
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, describe_version_result struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- BitSet incoming = iprot.readBitSet(1);
- if (incoming.get(0)) {
- struct.success = iprot.readString();
- struct.setSuccessIsSet(true);
- }
- }
- }
-
- }
-
- public static class describe_ring_args implements org.apache.thrift.TBase<describe_ring_args, describe_ring_args._Fields>, java.io.Serializable, Cloneable, Comparable<describe_ring_args> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("describe_ring_args");
-
- private static final org.apache.thrift.protocol.TField KEYSPACE_FIELD_DESC = new org.apache.thrift.protocol.TField("keyspace", org.apache.thrift.protocol.TType.STRING, (short)1);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new describe_ring_argsStandardSchemeFactory());
- schemes.put(TupleScheme.class, new describe_ring_argsTupleSchemeFactory());
- }
-
- public String keyspace; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- KEYSPACE((short)1, "keyspace");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // KEYSPACE
- return KEYSPACE;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.KEYSPACE, new org.apache.thrift.meta_data.FieldMetaData("keyspace", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(describe_ring_args.class, metaDataMap);
- }
-
- public describe_ring_args() {
- }
-
- public describe_ring_args(
- String keyspace)
- {
- this();
- this.keyspace = keyspace;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public describe_ring_args(describe_ring_args other) {
- if (other.isSetKeyspace()) {
- this.keyspace = other.keyspace;
- }
- }
-
- public describe_ring_args deepCopy() {
- return new describe_ring_args(this);
- }
-
- @Override
- public void clear() {
- this.keyspace = null;
- }
-
- public String getKeyspace() {
- return this.keyspace;
- }
-
- public describe_ring_args setKeyspace(String keyspace) {
- this.keyspace = keyspace;
- return this;
- }
-
- public void unsetKeyspace() {
- this.keyspace = null;
- }
-
- /** Returns true if field keyspace is set (has been assigned a value) and false otherwise */
- public boolean isSetKeyspace() {
- return this.keyspace != null;
- }
-
- public void setKeyspaceIsSet(boolean value) {
- if (!value) {
- this.keyspace = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case KEYSPACE:
- if (value == null) {
- unsetKeyspace();
- } else {
- setKeyspace((String)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case KEYSPACE:
- return getKeyspace();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case KEYSPACE:
- return isSetKeyspace();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof describe_ring_args)
- return this.equals((describe_ring_args)that);
- return false;
- }
-
- public boolean equals(describe_ring_args that) {
- if (that == null)
- return false;
-
- boolean this_present_keyspace = true && this.isSetKeyspace();
- boolean that_present_keyspace = true && that.isSetKeyspace();
- if (this_present_keyspace || that_present_keyspace) {
- if (!(this_present_keyspace && that_present_keyspace))
- return false;
- if (!this.keyspace.equals(that.keyspace))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_keyspace = true && (isSetKeyspace());
- builder.append(present_keyspace);
- if (present_keyspace)
- builder.append(keyspace);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(describe_ring_args other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetKeyspace()).compareTo(other.isSetKeyspace());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetKeyspace()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.keyspace, other.keyspace);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("describe_ring_args(");
- boolean first = true;
-
- sb.append("keyspace:");
- if (this.keyspace == null) {
- sb.append("null");
- } else {
- sb.append(this.keyspace);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- if (keyspace == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'keyspace' was not present! Struct: " + toString());
- }
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class describe_ring_argsStandardSchemeFactory implements SchemeFactory {
- public describe_ring_argsStandardScheme getScheme() {
- return new describe_ring_argsStandardScheme();
- }
- }
-
- private static class describe_ring_argsStandardScheme extends StandardScheme<describe_ring_args> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, describe_ring_args struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // KEYSPACE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.keyspace = iprot.readString();
- struct.setKeyspaceIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, describe_ring_args struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.keyspace != null) {
- oprot.writeFieldBegin(KEYSPACE_FIELD_DESC);
- oprot.writeString(struct.keyspace);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class describe_ring_argsTupleSchemeFactory implements SchemeFactory {
- public describe_ring_argsTupleScheme getScheme() {
- return new describe_ring_argsTupleScheme();
- }
- }
-
- private static class describe_ring_argsTupleScheme extends TupleScheme<describe_ring_args> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, describe_ring_args struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- oprot.writeString(struct.keyspace);
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, describe_ring_args struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- struct.keyspace = iprot.readString();
- struct.setKeyspaceIsSet(true);
- }
- }
-
- }
-
- public static class describe_ring_result implements org.apache.thrift.TBase<describe_ring_result, describe_ring_result._Fields>, java.io.Serializable, Cloneable, Comparable<describe_ring_result> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("describe_ring_result");
-
- private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.LIST, (short)0);
- private static final org.apache.thrift.protocol.TField IRE_FIELD_DESC = new org.apache.thrift.protocol.TField("ire", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new describe_ring_resultStandardSchemeFactory());
- schemes.put(TupleScheme.class, new describe_ring_resultTupleSchemeFactory());
- }
-
- public List<TokenRange> success; // required
- public InvalidRequestException ire; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- SUCCESS((short)0, "success"),
- IRE((short)1, "ire");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 0: // SUCCESS
- return SUCCESS;
- case 1: // IRE
- return IRE;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST,
- new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TokenRange.class))));
- tmpMap.put(_Fields.IRE, new org.apache.thrift.meta_data.FieldMetaData("ire", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(describe_ring_result.class, metaDataMap);
- }
-
- public describe_ring_result() {
- }
-
- public describe_ring_result(
- List<TokenRange> success,
- InvalidRequestException ire)
- {
- this();
- this.success = success;
- this.ire = ire;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public describe_ring_result(describe_ring_result other) {
- if (other.isSetSuccess()) {
- List<TokenRange> __this__success = new ArrayList<TokenRange>(other.success.size());
- for (TokenRange other_element : other.success) {
- __this__success.add(new TokenRange(other_element));
- }
- this.success = __this__success;
- }
- if (other.isSetIre()) {
- this.ire = new InvalidRequestException(other.ire);
- }
- }
-
- public describe_ring_result deepCopy() {
- return new describe_ring_result(this);
- }
-
- @Override
- public void clear() {
- this.success = null;
- this.ire = null;
- }
-
- public int getSuccessSize() {
- return (this.success == null) ? 0 : this.success.size();
- }
-
- public java.util.Iterator<TokenRange> getSuccessIterator() {
- return (this.success == null) ? null : this.success.iterator();
- }
-
- public void addToSuccess(TokenRange elem) {
- if (this.success == null) {
- this.success = new ArrayList<TokenRange>();
- }
- this.success.add(elem);
- }
-
- public List<TokenRange> getSuccess() {
- return this.success;
- }
-
- public describe_ring_result setSuccess(List<TokenRange> success) {
- this.success = success;
- return this;
- }
-
- public void unsetSuccess() {
- this.success = null;
- }
-
- /** Returns true if field success is set (has been assigned a value) and false otherwise */
- public boolean isSetSuccess() {
- return this.success != null;
- }
-
- public void setSuccessIsSet(boolean value) {
- if (!value) {
- this.success = null;
- }
- }
-
- public InvalidRequestException getIre() {
- return this.ire;
- }
-
- public describe_ring_result setIre(InvalidRequestException ire) {
- this.ire = ire;
- return this;
- }
-
- public void unsetIre() {
- this.ire = null;
- }
-
- /** Returns true if field ire is set (has been assigned a value) and false otherwise */
- public boolean isSetIre() {
- return this.ire != null;
- }
-
- public void setIreIsSet(boolean value) {
- if (!value) {
- this.ire = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case SUCCESS:
- if (value == null) {
- unsetSuccess();
- } else {
- setSuccess((List<TokenRange>)value);
- }
- break;
-
- case IRE:
- if (value == null) {
- unsetIre();
- } else {
- setIre((InvalidRequestException)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case SUCCESS:
- return getSuccess();
-
- case IRE:
- return getIre();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case SUCCESS:
- return isSetSuccess();
- case IRE:
- return isSetIre();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof describe_ring_result)
- return this.equals((describe_ring_result)that);
- return false;
- }
-
- public boolean equals(describe_ring_result that) {
- if (that == null)
- return false;
-
- boolean this_present_success = true && this.isSetSuccess();
- boolean that_present_success = true && that.isSetSuccess();
- if (this_present_success || that_present_success) {
- if (!(this_present_success && that_present_success))
- return false;
- if (!this.success.equals(that.success))
- return false;
- }
-
- boolean this_present_ire = true && this.isSetIre();
- boolean that_present_ire = true && that.isSetIre();
- if (this_present_ire || that_present_ire) {
- if (!(this_present_ire && that_present_ire))
- return false;
- if (!this.ire.equals(that.ire))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_success = true && (isSetSuccess());
- builder.append(present_success);
- if (present_success)
- builder.append(success);
-
- boolean present_ire = true && (isSetIre());
- builder.append(present_ire);
- if (present_ire)
- builder.append(ire);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(describe_ring_result other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(other.isSetSuccess());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetSuccess()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, other.success);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetIre()).compareTo(other.isSetIre());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetIre()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.ire, other.ire);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("describe_ring_result(");
- boolean first = true;
-
- sb.append("success:");
- if (this.success == null) {
- sb.append("null");
- } else {
- sb.append(this.success);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("ire:");
- if (this.ire == null) {
- sb.append("null");
- } else {
- sb.append(this.ire);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class describe_ring_resultStandardSchemeFactory implements SchemeFactory {
- public describe_ring_resultStandardScheme getScheme() {
- return new describe_ring_resultStandardScheme();
- }
- }
-
- private static class describe_ring_resultStandardScheme extends StandardScheme<describe_ring_result> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, describe_ring_result struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 0: // SUCCESS
- if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
- {
- org.apache.thrift.protocol.TList _list406 = iprot.readListBegin();
- struct.success = new ArrayList<TokenRange>(_list406.size);
- for (int _i407 = 0; _i407 < _list406.size; ++_i407)
- {
- TokenRange _elem408;
- _elem408 = new TokenRange();
- _elem408.read(iprot);
- struct.success.add(_elem408);
- }
- iprot.readListEnd();
- }
- struct.setSuccessIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 1: // IRE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.ire = new InvalidRequestException();
- struct.ire.read(iprot);
- struct.setIreIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, describe_ring_result struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.success != null) {
- oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
- {
- oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.success.size()));
- for (TokenRange _iter409 : struct.success)
- {
- _iter409.write(oprot);
- }
- oprot.writeListEnd();
- }
- oprot.writeFieldEnd();
- }
- if (struct.ire != null) {
- oprot.writeFieldBegin(IRE_FIELD_DESC);
- struct.ire.write(oprot);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class describe_ring_resultTupleSchemeFactory implements SchemeFactory {
- public describe_ring_resultTupleScheme getScheme() {
- return new describe_ring_resultTupleScheme();
- }
- }
-
- private static class describe_ring_resultTupleScheme extends TupleScheme<describe_ring_result> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, describe_ring_result struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- BitSet optionals = new BitSet();
- if (struct.isSetSuccess()) {
- optionals.set(0);
- }
- if (struct.isSetIre()) {
- optionals.set(1);
- }
- oprot.writeBitSet(optionals, 2);
- if (struct.isSetSuccess()) {
- {
- oprot.writeI32(struct.success.size());
- for (TokenRange _iter410 : struct.success)
- {
- _iter410.write(oprot);
- }
- }
- }
- if (struct.isSetIre()) {
- struct.ire.write(oprot);
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, describe_ring_result struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- BitSet incoming = iprot.readBitSet(2);
- if (incoming.get(0)) {
- {
- org.apache.thrift.protocol.TList _list411 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
- struct.success = new ArrayList<TokenRange>(_list411.size);
- for (int _i412 = 0; _i412 < _list411.size; ++_i412)
- {
- TokenRange _elem413;
- _elem413 = new TokenRange();
- _elem413.read(iprot);
- struct.success.add(_elem413);
- }
- }
- struct.setSuccessIsSet(true);
- }
- if (incoming.get(1)) {
- struct.ire = new InvalidRequestException();
- struct.ire.read(iprot);
- struct.setIreIsSet(true);
- }
- }
- }
-
- }
-
- public static class describe_local_ring_args implements org.apache.thrift.TBase<describe_local_ring_args, describe_local_ring_args._Fields>, java.io.Serializable, Cloneable, Comparable<describe_local_ring_args> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("describe_local_ring_args");
-
- private static final org.apache.thrift.protocol.TField KEYSPACE_FIELD_DESC = new org.apache.thrift.protocol.TField("keyspace", org.apache.thrift.protocol.TType.STRING, (short)1);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new describe_local_ring_argsStandardSchemeFactory());
- schemes.put(TupleScheme.class, new describe_local_ring_argsTupleSchemeFactory());
- }
-
- public String keyspace; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- KEYSPACE((short)1, "keyspace");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // KEYSPACE
- return KEYSPACE;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.KEYSPACE, new org.apache.thrift.meta_data.FieldMetaData("keyspace", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(describe_local_ring_args.class, metaDataMap);
- }
-
- public describe_local_ring_args() {
- }
-
- public describe_local_ring_args(
- String keyspace)
- {
- this();
- this.keyspace = keyspace;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public describe_local_ring_args(describe_local_ring_args other) {
- if (other.isSetKeyspace()) {
- this.keyspace = other.keyspace;
- }
- }
-
- public describe_local_ring_args deepCopy() {
- return new describe_local_ring_args(this);
- }
-
- @Override
- public void clear() {
- this.keyspace = null;
- }
-
- public String getKeyspace() {
- return this.keyspace;
- }
-
- public describe_local_ring_args setKeyspace(String keyspace) {
- this.keyspace = keyspace;
- return this;
- }
-
- public void unsetKeyspace() {
- this.keyspace = null;
- }
-
- /** Returns true if field keyspace is set (has been assigned a value) and false otherwise */
- public boolean isSetKeyspace() {
- return this.keyspace != null;
- }
-
- public void setKeyspaceIsSet(boolean value) {
- if (!value) {
- this.keyspace = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case KEYSPACE:
- if (value == null) {
- unsetKeyspace();
- } else {
- setKeyspace((String)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case KEYSPACE:
- return getKeyspace();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case KEYSPACE:
- return isSetKeyspace();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof describe_local_ring_args)
- return this.equals((describe_local_ring_args)that);
- return false;
- }
-
- public boolean equals(describe_local_ring_args that) {
- if (that == null)
- return false;
-
- boolean this_present_keyspace = true && this.isSetKeyspace();
- boolean that_present_keyspace = true && that.isSetKeyspace();
- if (this_present_keyspace || that_present_keyspace) {
- if (!(this_present_keyspace && that_present_keyspace))
- return false;
- if (!this.keyspace.equals(that.keyspace))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_keyspace = true && (isSetKeyspace());
- builder.append(present_keyspace);
- if (present_keyspace)
- builder.append(keyspace);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(describe_local_ring_args other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetKeyspace()).compareTo(other.isSetKeyspace());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetKeyspace()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.keyspace, other.keyspace);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("describe_local_ring_args(");
- boolean first = true;
-
- sb.append("keyspace:");
- if (this.keyspace == null) {
- sb.append("null");
- } else {
- sb.append(this.keyspace);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- if (keyspace == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'keyspace' was not present! Struct: " + toString());
- }
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class describe_local_ring_argsStandardSchemeFactory implements SchemeFactory {
- public describe_local_ring_argsStandardScheme getScheme() {
- return new describe_local_ring_argsStandardScheme();
- }
- }
-
- private static class describe_local_ring_argsStandardScheme extends StandardScheme<describe_local_ring_args> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, describe_local_ring_args struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // KEYSPACE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.keyspace = iprot.readString();
- struct.setKeyspaceIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, describe_local_ring_args struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.keyspace != null) {
- oprot.writeFieldBegin(KEYSPACE_FIELD_DESC);
- oprot.writeString(struct.keyspace);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class describe_local_ring_argsTupleSchemeFactory implements SchemeFactory {
- public describe_local_ring_argsTupleScheme getScheme() {
- return new describe_local_ring_argsTupleScheme();
- }
- }
-
- private static class describe_local_ring_argsTupleScheme extends TupleScheme<describe_local_ring_args> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, describe_local_ring_args struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- oprot.writeString(struct.keyspace);
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, describe_local_ring_args struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- struct.keyspace = iprot.readString();
- struct.setKeyspaceIsSet(true);
- }
- }
-
- }
-
- public static class describe_local_ring_result implements org.apache.thrift.TBase<describe_local_ring_result, describe_local_ring_result._Fields>, java.io.Serializable, Cloneable, Comparable<describe_local_ring_result> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("describe_local_ring_result");
-
- private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.LIST, (short)0);
- private static final org.apache.thrift.protocol.TField IRE_FIELD_DESC = new org.apache.thrift.protocol.TField("ire", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new describe_local_ring_resultStandardSchemeFactory());
- schemes.put(TupleScheme.class, new describe_local_ring_resultTupleSchemeFactory());
- }
-
- public List<TokenRange> success; // required
- public InvalidRequestException ire; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- SUCCESS((short)0, "success"),
- IRE((short)1, "ire");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 0: // SUCCESS
- return SUCCESS;
- case 1: // IRE
- return IRE;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST,
- new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TokenRange.class))));
- tmpMap.put(_Fields.IRE, new org.apache.thrift.meta_data.FieldMetaData("ire", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(describe_local_ring_result.class, metaDataMap);
- }
-
- public describe_local_ring_result() {
- }
-
- public describe_local_ring_result(
- List<TokenRange> success,
- InvalidRequestException ire)
- {
- this();
- this.success = success;
- this.ire = ire;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public describe_local_ring_result(describe_local_ring_result other) {
- if (other.isSetSuccess()) {
- List<TokenRange> __this__success = new ArrayList<TokenRange>(other.success.size());
- for (TokenRange other_element : other.success) {
- __this__success.add(new TokenRange(other_element));
- }
- this.success = __this__success;
- }
- if (other.isSetIre()) {
- this.ire = new InvalidRequestException(other.ire);
- }
- }
-
- public describe_local_ring_result deepCopy() {
- return new describe_local_ring_result(this);
- }
-
- @Override
- public void clear() {
- this.success = null;
- this.ire = null;
- }
-
- public int getSuccessSize() {
- return (this.success == null) ? 0 : this.success.size();
- }
-
- public java.util.Iterator<TokenRange> getSuccessIterator() {
- return (this.success == null) ? null : this.success.iterator();
- }
-
- public void addToSuccess(TokenRange elem) {
- if (this.success == null) {
- this.success = new ArrayList<TokenRange>();
- }
- this.success.add(elem);
- }
-
- public List<TokenRange> getSuccess() {
- return this.success;
- }
-
- public describe_local_ring_result setSuccess(List<TokenRange> success) {
- this.success = success;
- return this;
- }
-
- public void unsetSuccess() {
- this.success = null;
- }
-
- /** Returns true if field success is set (has been assigned a value) and false otherwise */
- public boolean isSetSuccess() {
- return this.success != null;
- }
-
- public void setSuccessIsSet(boolean value) {
- if (!value) {
- this.success = null;
- }
- }
-
- public InvalidRequestException getIre() {
- return this.ire;
- }
-
- public describe_local_ring_result setIre(InvalidRequestException ire) {
- this.ire = ire;
- return this;
- }
-
- public void unsetIre() {
- this.ire = null;
- }
-
- /** Returns true if field ire is set (has been assigned a value) and false otherwise */
- public boolean isSetIre() {
- return this.ire != null;
- }
-
- public void setIreIsSet(boolean value) {
- if (!value) {
- this.ire = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case SUCCESS:
- if (value == null) {
- unsetSuccess();
- } else {
- setSuccess((List<TokenRange>)value);
- }
- break;
-
- case IRE:
- if (value == null) {
- unsetIre();
- } else {
- setIre((InvalidRequestException)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case SUCCESS:
- return getSuccess();
-
- case IRE:
- return getIre();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case SUCCESS:
- return isSetSuccess();
- case IRE:
- return isSetIre();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof describe_local_ring_result)
- return this.equals((describe_local_ring_result)that);
- return false;
- }
-
- public boolean equals(describe_local_ring_result that) {
- if (that == null)
- return false;
-
- boolean this_present_success = true && this.isSetSuccess();
- boolean that_present_success = true && that.isSetSuccess();
- if (this_present_success || that_present_success) {
- if (!(this_present_success && that_present_success))
- return false;
- if (!this.success.equals(that.success))
- return false;
- }
-
- boolean this_present_ire = true && this.isSetIre();
- boolean that_present_ire = true && that.isSetIre();
- if (this_present_ire || that_present_ire) {
- if (!(this_present_ire && that_present_ire))
- return false;
- if (!this.ire.equals(that.ire))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_success = true && (isSetSuccess());
- builder.append(present_success);
- if (present_success)
- builder.append(success);
-
- boolean present_ire = true && (isSetIre());
- builder.append(present_ire);
- if (present_ire)
- builder.append(ire);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(describe_local_ring_result other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(other.isSetSuccess());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetSuccess()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, other.success);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetIre()).compareTo(other.isSetIre());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetIre()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.ire, other.ire);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("describe_local_ring_result(");
- boolean first = true;
-
- sb.append("success:");
- if (this.success == null) {
- sb.append("null");
- } else {
- sb.append(this.success);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("ire:");
- if (this.ire == null) {
- sb.append("null");
- } else {
- sb.append(this.ire);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class describe_local_ring_resultStandardSchemeFactory implements SchemeFactory {
- public describe_local_ring_resultStandardScheme getScheme() {
- return new describe_local_ring_resultStandardScheme();
- }
- }
-
- private static class describe_local_ring_resultStandardScheme extends StandardScheme<describe_local_ring_result> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, describe_local_ring_result struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 0: // SUCCESS
- if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
- {
- org.apache.thrift.protocol.TList _list414 = iprot.readListBegin();
- struct.success = new ArrayList<TokenRange>(_list414.size);
- for (int _i415 = 0; _i415 < _list414.size; ++_i415)
- {
- TokenRange _elem416;
- _elem416 = new TokenRange();
- _elem416.read(iprot);
- struct.success.add(_elem416);
- }
- iprot.readListEnd();
- }
- struct.setSuccessIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 1: // IRE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.ire = new InvalidRequestException();
- struct.ire.read(iprot);
- struct.setIreIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, describe_local_ring_result struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.success != null) {
- oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
- {
- oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.success.size()));
- for (TokenRange _iter417 : struct.success)
- {
- _iter417.write(oprot);
- }
- oprot.writeListEnd();
- }
- oprot.writeFieldEnd();
- }
- if (struct.ire != null) {
- oprot.writeFieldBegin(IRE_FIELD_DESC);
- struct.ire.write(oprot);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class describe_local_ring_resultTupleSchemeFactory implements SchemeFactory {
- public describe_local_ring_resultTupleScheme getScheme() {
- return new describe_local_ring_resultTupleScheme();
- }
- }
-
- private static class describe_local_ring_resultTupleScheme extends TupleScheme<describe_local_ring_result> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, describe_local_ring_result struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- BitSet optionals = new BitSet();
- if (struct.isSetSuccess()) {
- optionals.set(0);
- }
- if (struct.isSetIre()) {
- optionals.set(1);
- }
- oprot.writeBitSet(optionals, 2);
- if (struct.isSetSuccess()) {
- {
- oprot.writeI32(struct.success.size());
- for (TokenRange _iter418 : struct.success)
- {
- _iter418.write(oprot);
- }
- }
- }
- if (struct.isSetIre()) {
- struct.ire.write(oprot);
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, describe_local_ring_result struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- BitSet incoming = iprot.readBitSet(2);
- if (incoming.get(0)) {
- {
- org.apache.thrift.protocol.TList _list419 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
- struct.success = new ArrayList<TokenRange>(_list419.size);
- for (int _i420 = 0; _i420 < _list419.size; ++_i420)
- {
- TokenRange _elem421;
- _elem421 = new TokenRange();
- _elem421.read(iprot);
- struct.success.add(_elem421);
- }
- }
- struct.setSuccessIsSet(true);
- }
- if (incoming.get(1)) {
- struct.ire = new InvalidRequestException();
- struct.ire.read(iprot);
- struct.setIreIsSet(true);
- }
- }
- }
-
- }
-
- public static class describe_token_map_args implements org.apache.thrift.TBase<describe_token_map_args, describe_token_map_args._Fields>, java.io.Serializable, Cloneable, Comparable<describe_token_map_args> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("describe_token_map_args");
-
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new describe_token_map_argsStandardSchemeFactory());
- schemes.put(TupleScheme.class, new describe_token_map_argsTupleSchemeFactory());
- }
-
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-;
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(describe_token_map_args.class, metaDataMap);
- }
-
- public describe_token_map_args() {
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public describe_token_map_args(describe_token_map_args other) {
- }
-
- public describe_token_map_args deepCopy() {
- return new describe_token_map_args(this);
- }
-
- @Override
- public void clear() {
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof describe_token_map_args)
- return this.equals((describe_token_map_args)that);
- return false;
- }
-
- public boolean equals(describe_token_map_args that) {
- if (that == null)
- return false;
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(describe_token_map_args other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("describe_token_map_args(");
- boolean first = true;
-
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class describe_token_map_argsStandardSchemeFactory implements SchemeFactory {
- public describe_token_map_argsStandardScheme getScheme() {
- return new describe_token_map_argsStandardScheme();
- }
- }
-
- private static class describe_token_map_argsStandardScheme extends StandardScheme<describe_token_map_args> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, describe_token_map_args struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, describe_token_map_args struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class describe_token_map_argsTupleSchemeFactory implements SchemeFactory {
- public describe_token_map_argsTupleScheme getScheme() {
- return new describe_token_map_argsTupleScheme();
- }
- }
-
- private static class describe_token_map_argsTupleScheme extends TupleScheme<describe_token_map_args> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, describe_token_map_args struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, describe_token_map_args struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- }
- }
-
- }
-
- public static class describe_token_map_result implements org.apache.thrift.TBase<describe_token_map_result, describe_token_map_result._Fields>, java.io.Serializable, Cloneable, Comparable<describe_token_map_result> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("describe_token_map_result");
-
- private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.MAP, (short)0);
- private static final org.apache.thrift.protocol.TField IRE_FIELD_DESC = new org.apache.thrift.protocol.TField("ire", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new describe_token_map_resultStandardSchemeFactory());
- schemes.put(TupleScheme.class, new describe_token_map_resultTupleSchemeFactory());
- }
-
- public Map<String,String> success; // required
- public InvalidRequestException ire; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- SUCCESS((short)0, "success"),
- IRE((short)1, "ire");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 0: // SUCCESS
- return SUCCESS;
- case 1: // IRE
- return IRE;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING),
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))));
- tmpMap.put(_Fields.IRE, new org.apache.thrift.meta_data.FieldMetaData("ire", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(describe_token_map_result.class, metaDataMap);
- }
-
- public describe_token_map_result() {
- }
-
- public describe_token_map_result(
- Map<String,String> success,
- InvalidRequestException ire)
- {
- this();
- this.success = success;
- this.ire = ire;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public describe_token_map_result(describe_token_map_result other) {
- if (other.isSetSuccess()) {
- Map<String,String> __this__success = new HashMap<String,String>(other.success);
- this.success = __this__success;
- }
- if (other.isSetIre()) {
- this.ire = new InvalidRequestException(other.ire);
- }
- }
-
- public describe_token_map_result deepCopy() {
- return new describe_token_map_result(this);
- }
-
- @Override
- public void clear() {
- this.success = null;
- this.ire = null;
- }
-
- public int getSuccessSize() {
- return (this.success == null) ? 0 : this.success.size();
- }
-
- public void putToSuccess(String key, String val) {
- if (this.success == null) {
- this.success = new HashMap<String,String>();
- }
- this.success.put(key, val);
- }
-
- public Map<String,String> getSuccess() {
- return this.success;
- }
-
- public describe_token_map_result setSuccess(Map<String,String> success) {
- this.success = success;
- return this;
- }
-
- public void unsetSuccess() {
- this.success = null;
- }
-
- /** Returns true if field success is set (has been assigned a value) and false otherwise */
- public boolean isSetSuccess() {
- return this.success != null;
- }
-
- public void setSuccessIsSet(boolean value) {
- if (!value) {
- this.success = null;
- }
- }
-
- public InvalidRequestException getIre() {
- return this.ire;
- }
-
- public describe_token_map_result setIre(InvalidRequestException ire) {
- this.ire = ire;
- return this;
- }
-
- public void unsetIre() {
- this.ire = null;
- }
-
- /** Returns true if field ire is set (has been assigned a value) and false otherwise */
- public boolean isSetIre() {
- return this.ire != null;
- }
-
- public void setIreIsSet(boolean value) {
- if (!value) {
- this.ire = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case SUCCESS:
- if (value == null) {
- unsetSuccess();
- } else {
- setSuccess((Map<String,String>)value);
- }
- break;
-
- case IRE:
- if (value == null) {
- unsetIre();
- } else {
- setIre((InvalidRequestException)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case SUCCESS:
- return getSuccess();
-
- case IRE:
- return getIre();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case SUCCESS:
- return isSetSuccess();
- case IRE:
- return isSetIre();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof describe_token_map_result)
- return this.equals((describe_token_map_result)that);
- return false;
- }
-
- public boolean equals(describe_token_map_result that) {
- if (that == null)
- return false;
-
- boolean this_present_success = true && this.isSetSuccess();
- boolean that_present_success = true && that.isSetSuccess();
- if (this_present_success || that_present_success) {
- if (!(this_present_success && that_present_success))
- return false;
- if (!this.success.equals(that.success))
- return false;
- }
-
- boolean this_present_ire = true && this.isSetIre();
- boolean that_present_ire = true && that.isSetIre();
- if (this_present_ire || that_present_ire) {
- if (!(this_present_ire && that_present_ire))
- return false;
- if (!this.ire.equals(that.ire))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_success = true && (isSetSuccess());
- builder.append(present_success);
- if (present_success)
- builder.append(success);
-
- boolean present_ire = true && (isSetIre());
- builder.append(present_ire);
- if (present_ire)
- builder.append(ire);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(describe_token_map_result other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(other.isSetSuccess());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetSuccess()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, other.success);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetIre()).compareTo(other.isSetIre());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetIre()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.ire, other.ire);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("describe_token_map_result(");
- boolean first = true;
-
- sb.append("success:");
- if (this.success == null) {
- sb.append("null");
- } else {
- sb.append(this.success);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("ire:");
- if (this.ire == null) {
- sb.append("null");
- } else {
- sb.append(this.ire);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class describe_token_map_resultStandardSchemeFactory implements SchemeFactory {
- public describe_token_map_resultStandardScheme getScheme() {
- return new describe_token_map_resultStandardScheme();
- }
- }
-
- private static class describe_token_map_resultStandardScheme extends StandardScheme<describe_token_map_result> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, describe_token_map_result struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 0: // SUCCESS
- if (schemeField.type == org.apache.thrift.protocol.TType.MAP) {
- {
- org.apache.thrift.protocol.TMap _map422 = iprot.readMapBegin();
- struct.success = new HashMap<String,String>(2*_map422.size);
- for (int _i423 = 0; _i423 < _map422.size; ++_i423)
- {
- String _key424;
- String _val425;
- _key424 = iprot.readString();
- _val425 = iprot.readString();
- struct.success.put(_key424, _val425);
- }
- iprot.readMapEnd();
- }
- struct.setSuccessIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 1: // IRE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.ire = new InvalidRequestException();
- struct.ire.read(iprot);
- struct.setIreIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, describe_token_map_result struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.success != null) {
- oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
- {
- oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, struct.success.size()));
- for (Map.Entry<String, String> _iter426 : struct.success.entrySet())
- {
- oprot.writeString(_iter426.getKey());
- oprot.writeString(_iter426.getValue());
- }
- oprot.writeMapEnd();
- }
- oprot.writeFieldEnd();
- }
- if (struct.ire != null) {
- oprot.writeFieldBegin(IRE_FIELD_DESC);
- struct.ire.write(oprot);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class describe_token_map_resultTupleSchemeFactory implements SchemeFactory {
- public describe_token_map_resultTupleScheme getScheme() {
- return new describe_token_map_resultTupleScheme();
- }
- }
-
- private static class describe_token_map_resultTupleScheme extends TupleScheme<describe_token_map_result> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, describe_token_map_result struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- BitSet optionals = new BitSet();
- if (struct.isSetSuccess()) {
- optionals.set(0);
- }
- if (struct.isSetIre()) {
- optionals.set(1);
- }
- oprot.writeBitSet(optionals, 2);
- if (struct.isSetSuccess()) {
- {
- oprot.writeI32(struct.success.size());
- for (Map.Entry<String, String> _iter427 : struct.success.entrySet())
- {
- oprot.writeString(_iter427.getKey());
- oprot.writeString(_iter427.getValue());
- }
- }
- }
- if (struct.isSetIre()) {
- struct.ire.write(oprot);
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, describe_token_map_result struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- BitSet incoming = iprot.readBitSet(2);
- if (incoming.get(0)) {
- {
- org.apache.thrift.protocol.TMap _map428 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32());
- struct.success = new HashMap<String,String>(2*_map428.size);
- for (int _i429 = 0; _i429 < _map428.size; ++_i429)
- {
- String _key430;
- String _val431;
- _key430 = iprot.readString();
- _val431 = iprot.readString();
- struct.success.put(_key430, _val431);
- }
- }
- struct.setSuccessIsSet(true);
- }
- if (incoming.get(1)) {
- struct.ire = new InvalidRequestException();
- struct.ire.read(iprot);
- struct.setIreIsSet(true);
- }
- }
- }
-
- }
-
- public static class describe_partitioner_args implements org.apache.thrift.TBase<describe_partitioner_args, describe_partitioner_args._Fields>, java.io.Serializable, Cloneable, Comparable<describe_partitioner_args> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("describe_partitioner_args");
-
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new describe_partitioner_argsStandardSchemeFactory());
- schemes.put(TupleScheme.class, new describe_partitioner_argsTupleSchemeFactory());
- }
-
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-;
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(describe_partitioner_args.class, metaDataMap);
- }
-
- public describe_partitioner_args() {
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public describe_partitioner_args(describe_partitioner_args other) {
- }
-
- public describe_partitioner_args deepCopy() {
- return new describe_partitioner_args(this);
- }
-
- @Override
- public void clear() {
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof describe_partitioner_args)
- return this.equals((describe_partitioner_args)that);
- return false;
- }
-
- public boolean equals(describe_partitioner_args that) {
- if (that == null)
- return false;
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(describe_partitioner_args other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("describe_partitioner_args(");
- boolean first = true;
-
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class describe_partitioner_argsStandardSchemeFactory implements SchemeFactory {
- public describe_partitioner_argsStandardScheme getScheme() {
- return new describe_partitioner_argsStandardScheme();
- }
- }
-
- private static class describe_partitioner_argsStandardScheme extends StandardScheme<describe_partitioner_args> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, describe_partitioner_args struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, describe_partitioner_args struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class describe_partitioner_argsTupleSchemeFactory implements SchemeFactory {
- public describe_partitioner_argsTupleScheme getScheme() {
- return new describe_partitioner_argsTupleScheme();
- }
- }
-
- private static class describe_partitioner_argsTupleScheme extends TupleScheme<describe_partitioner_args> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, describe_partitioner_args struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, describe_partitioner_args struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- }
- }
-
- }
-
- public static class describe_partitioner_result implements org.apache.thrift.TBase<describe_partitioner_result, describe_partitioner_result._Fields>, java.io.Serializable, Cloneable, Comparable<describe_partitioner_result> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("describe_partitioner_result");
-
- private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRING, (short)0);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new describe_partitioner_resultStandardSchemeFactory());
- schemes.put(TupleScheme.class, new describe_partitioner_resultTupleSchemeFactory());
- }
-
- public String success; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- SUCCESS((short)0, "success");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 0: // SUCCESS
- return SUCCESS;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(describe_partitioner_result.class, metaDataMap);
- }
-
- public describe_partitioner_result() {
- }
-
- public describe_partitioner_result(
- String success)
- {
- this();
- this.success = success;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public describe_partitioner_result(describe_partitioner_result other) {
- if (other.isSetSuccess()) {
- this.success = other.success;
- }
- }
-
- public describe_partitioner_result deepCopy() {
- return new describe_partitioner_result(this);
- }
-
- @Override
- public void clear() {
- this.success = null;
- }
-
- public String getSuccess() {
- return this.success;
- }
-
- public describe_partitioner_result setSuccess(String success) {
- this.success = success;
- return this;
- }
-
- public void unsetSuccess() {
- this.success = null;
- }
-
- /** Returns true if field success is set (has been assigned a value) and false otherwise */
- public boolean isSetSuccess() {
- return this.success != null;
- }
-
- public void setSuccessIsSet(boolean value) {
- if (!value) {
- this.success = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case SUCCESS:
- if (value == null) {
- unsetSuccess();
- } else {
- setSuccess((String)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case SUCCESS:
- return getSuccess();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case SUCCESS:
- return isSetSuccess();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof describe_partitioner_result)
- return this.equals((describe_partitioner_result)that);
- return false;
- }
-
- public boolean equals(describe_partitioner_result that) {
- if (that == null)
- return false;
-
- boolean this_present_success = true && this.isSetSuccess();
- boolean that_present_success = true && that.isSetSuccess();
- if (this_present_success || that_present_success) {
- if (!(this_present_success && that_present_success))
- return false;
- if (!this.success.equals(that.success))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_success = true && (isSetSuccess());
- builder.append(present_success);
- if (present_success)
- builder.append(success);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(describe_partitioner_result other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(other.isSetSuccess());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetSuccess()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, other.success);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("describe_partitioner_result(");
- boolean first = true;
-
- sb.append("success:");
- if (this.success == null) {
- sb.append("null");
- } else {
- sb.append(this.success);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class describe_partitioner_resultStandardSchemeFactory implements SchemeFactory {
- public describe_partitioner_resultStandardScheme getScheme() {
- return new describe_partitioner_resultStandardScheme();
- }
- }
-
- private static class describe_partitioner_resultStandardScheme extends StandardScheme<describe_partitioner_result> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, describe_partitioner_result struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 0: // SUCCESS
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.success = iprot.readString();
- struct.setSuccessIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, describe_partitioner_result struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.success != null) {
- oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
- oprot.writeString(struct.success);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class describe_partitioner_resultTupleSchemeFactory implements SchemeFactory {
- public describe_partitioner_resultTupleScheme getScheme() {
- return new describe_partitioner_resultTupleScheme();
- }
- }
-
- private static class describe_partitioner_resultTupleScheme extends TupleScheme<describe_partitioner_result> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, describe_partitioner_result struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- BitSet optionals = new BitSet();
- if (struct.isSetSuccess()) {
- optionals.set(0);
- }
- oprot.writeBitSet(optionals, 1);
- if (struct.isSetSuccess()) {
- oprot.writeString(struct.success);
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, describe_partitioner_result struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- BitSet incoming = iprot.readBitSet(1);
- if (incoming.get(0)) {
- struct.success = iprot.readString();
- struct.setSuccessIsSet(true);
- }
- }
- }
-
- }
-
- public static class describe_snitch_args implements org.apache.thrift.TBase<describe_snitch_args, describe_snitch_args._Fields>, java.io.Serializable, Cloneable, Comparable<describe_snitch_args> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("describe_snitch_args");
-
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new describe_snitch_argsStandardSchemeFactory());
- schemes.put(TupleScheme.class, new describe_snitch_argsTupleSchemeFactory());
- }
-
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-;
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(describe_snitch_args.class, metaDataMap);
- }
-
- public describe_snitch_args() {
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public describe_snitch_args(describe_snitch_args other) {
- }
-
- public describe_snitch_args deepCopy() {
- return new describe_snitch_args(this);
- }
-
- @Override
- public void clear() {
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof describe_snitch_args)
- return this.equals((describe_snitch_args)that);
- return false;
- }
-
- public boolean equals(describe_snitch_args that) {
- if (that == null)
- return false;
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(describe_snitch_args other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("describe_snitch_args(");
- boolean first = true;
-
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class describe_snitch_argsStandardSchemeFactory implements SchemeFactory {
- public describe_snitch_argsStandardScheme getScheme() {
- return new describe_snitch_argsStandardScheme();
- }
- }
-
- private static class describe_snitch_argsStandardScheme extends StandardScheme<describe_snitch_args> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, describe_snitch_args struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, describe_snitch_args struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class describe_snitch_argsTupleSchemeFactory implements SchemeFactory {
- public describe_snitch_argsTupleScheme getScheme() {
- return new describe_snitch_argsTupleScheme();
- }
- }
-
- private static class describe_snitch_argsTupleScheme extends TupleScheme<describe_snitch_args> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, describe_snitch_args struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, describe_snitch_args struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- }
- }
-
- }
-
- public static class describe_snitch_result implements org.apache.thrift.TBase<describe_snitch_result, describe_snitch_result._Fields>, java.io.Serializable, Cloneable, Comparable<describe_snitch_result> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("describe_snitch_result");
-
- private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRING, (short)0);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new describe_snitch_resultStandardSchemeFactory());
- schemes.put(TupleScheme.class, new describe_snitch_resultTupleSchemeFactory());
- }
-
- public String success; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- SUCCESS((short)0, "success");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 0: // SUCCESS
- return SUCCESS;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(describe_snitch_result.class, metaDataMap);
- }
-
- public describe_snitch_result() {
- }
-
- public describe_snitch_result(
- String success)
- {
- this();
- this.success = success;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public describe_snitch_result(describe_snitch_result other) {
- if (other.isSetSuccess()) {
- this.success = other.success;
- }
- }
-
- public describe_snitch_result deepCopy() {
- return new describe_snitch_result(this);
- }
-
- @Override
- public void clear() {
- this.success = null;
- }
-
- public String getSuccess() {
- return this.success;
- }
-
- public describe_snitch_result setSuccess(String success) {
- this.success = success;
- return this;
- }
-
- public void unsetSuccess() {
- this.success = null;
- }
-
- /** Returns true if field success is set (has been assigned a value) and false otherwise */
- public boolean isSetSuccess() {
- return this.success != null;
- }
-
- public void setSuccessIsSet(boolean value) {
- if (!value) {
- this.success = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case SUCCESS:
- if (value == null) {
- unsetSuccess();
- } else {
- setSuccess((String)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case SUCCESS:
- return getSuccess();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case SUCCESS:
- return isSetSuccess();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof describe_snitch_result)
- return this.equals((describe_snitch_result)that);
- return false;
- }
-
- public boolean equals(describe_snitch_result that) {
- if (that == null)
- return false;
-
- boolean this_present_success = true && this.isSetSuccess();
- boolean that_present_success = true && that.isSetSuccess();
- if (this_present_success || that_present_success) {
- if (!(this_present_success && that_present_success))
- return false;
- if (!this.success.equals(that.success))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_success = true && (isSetSuccess());
- builder.append(present_success);
- if (present_success)
- builder.append(success);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(describe_snitch_result other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(other.isSetSuccess());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetSuccess()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, other.success);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("describe_snitch_result(");
- boolean first = true;
-
- sb.append("success:");
- if (this.success == null) {
- sb.append("null");
- } else {
- sb.append(this.success);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class describe_snitch_resultStandardSchemeFactory implements SchemeFactory {
- public describe_snitch_resultStandardScheme getScheme() {
- return new describe_snitch_resultStandardScheme();
- }
- }
-
- private static class describe_snitch_resultStandardScheme extends StandardScheme<describe_snitch_result> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, describe_snitch_result struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 0: // SUCCESS
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.success = iprot.readString();
- struct.setSuccessIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, describe_snitch_result struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.success != null) {
- oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
- oprot.writeString(struct.success);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class describe_snitch_resultTupleSchemeFactory implements SchemeFactory {
- public describe_snitch_resultTupleScheme getScheme() {
- return new describe_snitch_resultTupleScheme();
- }
- }
-
- private static class describe_snitch_resultTupleScheme extends TupleScheme<describe_snitch_result> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, describe_snitch_result struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- BitSet optionals = new BitSet();
- if (struct.isSetSuccess()) {
- optionals.set(0);
- }
- oprot.writeBitSet(optionals, 1);
- if (struct.isSetSuccess()) {
- oprot.writeString(struct.success);
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, describe_snitch_result struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- BitSet incoming = iprot.readBitSet(1);
- if (incoming.get(0)) {
- struct.success = iprot.readString();
- struct.setSuccessIsSet(true);
- }
- }
- }
-
- }
-
- public static class describe_keyspace_args implements org.apache.thrift.TBase<describe_keyspace_args, describe_keyspace_args._Fields>, java.io.Serializable, Cloneable, Comparable<describe_keyspace_args> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("describe_keyspace_args");
-
- private static final org.apache.thrift.protocol.TField KEYSPACE_FIELD_DESC = new org.apache.thrift.protocol.TField("keyspace", org.apache.thrift.protocol.TType.STRING, (short)1);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new describe_keyspace_argsStandardSchemeFactory());
- schemes.put(TupleScheme.class, new describe_keyspace_argsTupleSchemeFactory());
- }
-
- public String keyspace; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- KEYSPACE((short)1, "keyspace");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // KEYSPACE
- return KEYSPACE;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.KEYSPACE, new org.apache.thrift.meta_data.FieldMetaData("keyspace", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(describe_keyspace_args.class, metaDataMap);
- }
-
- public describe_keyspace_args() {
- }
-
- public describe_keyspace_args(
- String keyspace)
- {
- this();
- this.keyspace = keyspace;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public describe_keyspace_args(describe_keyspace_args other) {
- if (other.isSetKeyspace()) {
- this.keyspace = other.keyspace;
- }
- }
-
- public describe_keyspace_args deepCopy() {
- return new describe_keyspace_args(this);
- }
-
- @Override
- public void clear() {
- this.keyspace = null;
- }
-
- public String getKeyspace() {
- return this.keyspace;
- }
-
- public describe_keyspace_args setKeyspace(String keyspace) {
- this.keyspace = keyspace;
- return this;
- }
-
- public void unsetKeyspace() {
- this.keyspace = null;
- }
-
- /** Returns true if field keyspace is set (has been assigned a value) and false otherwise */
- public boolean isSetKeyspace() {
- return this.keyspace != null;
- }
-
- public void setKeyspaceIsSet(boolean value) {
- if (!value) {
- this.keyspace = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case KEYSPACE:
- if (value == null) {
- unsetKeyspace();
- } else {
- setKeyspace((String)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case KEYSPACE:
- return getKeyspace();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case KEYSPACE:
- return isSetKeyspace();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof describe_keyspace_args)
- return this.equals((describe_keyspace_args)that);
- return false;
- }
-
- public boolean equals(describe_keyspace_args that) {
- if (that == null)
- return false;
-
- boolean this_present_keyspace = true && this.isSetKeyspace();
- boolean that_present_keyspace = true && that.isSetKeyspace();
- if (this_present_keyspace || that_present_keyspace) {
- if (!(this_present_keyspace && that_present_keyspace))
- return false;
- if (!this.keyspace.equals(that.keyspace))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_keyspace = true && (isSetKeyspace());
- builder.append(present_keyspace);
- if (present_keyspace)
- builder.append(keyspace);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(describe_keyspace_args other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetKeyspace()).compareTo(other.isSetKeyspace());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetKeyspace()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.keyspace, other.keyspace);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("describe_keyspace_args(");
- boolean first = true;
-
- sb.append("keyspace:");
- if (this.keyspace == null) {
- sb.append("null");
- } else {
- sb.append(this.keyspace);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- if (keyspace == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'keyspace' was not present! Struct: " + toString());
- }
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class describe_keyspace_argsStandardSchemeFactory implements SchemeFactory {
- public describe_keyspace_argsStandardScheme getScheme() {
- return new describe_keyspace_argsStandardScheme();
- }
- }
-
- private static class describe_keyspace_argsStandardScheme extends StandardScheme<describe_keyspace_args> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, describe_keyspace_args struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // KEYSPACE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.keyspace = iprot.readString();
- struct.setKeyspaceIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, describe_keyspace_args struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.keyspace != null) {
- oprot.writeFieldBegin(KEYSPACE_FIELD_DESC);
- oprot.writeString(struct.keyspace);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class describe_keyspace_argsTupleSchemeFactory implements SchemeFactory {
- public describe_keyspace_argsTupleScheme getScheme() {
- return new describe_keyspace_argsTupleScheme();
- }
- }
-
- private static class describe_keyspace_argsTupleScheme extends TupleScheme<describe_keyspace_args> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, describe_keyspace_args struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- oprot.writeString(struct.keyspace);
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, describe_keyspace_args struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- struct.keyspace = iprot.readString();
- struct.setKeyspaceIsSet(true);
- }
- }
-
- }
-
- public static class describe_keyspace_result implements org.apache.thrift.TBase<describe_keyspace_result, describe_keyspace_result._Fields>, java.io.Serializable, Cloneable, Comparable<describe_keyspace_result> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("describe_keyspace_result");
-
- private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRUCT, (short)0);
- private static final org.apache.thrift.protocol.TField NFE_FIELD_DESC = new org.apache.thrift.protocol.TField("nfe", org.apache.thrift.protocol.TType.STRUCT, (short)1);
- private static final org.apache.thrift.protocol.TField IRE_FIELD_DESC = new org.apache.thrift.protocol.TField("ire", org.apache.thrift.protocol.TType.STRUCT, (short)2);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new describe_keyspace_resultStandardSchemeFactory());
- schemes.put(TupleScheme.class, new describe_keyspace_resultTupleSchemeFactory());
- }
-
- public KsDef success; // required
- public NotFoundException nfe; // required
- public InvalidRequestException ire; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- SUCCESS((short)0, "success"),
- NFE((short)1, "nfe"),
- IRE((short)2, "ire");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 0: // SUCCESS
- return SUCCESS;
- case 1: // NFE
- return NFE;
- case 2: // IRE
- return IRE;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, KsDef.class)));
- tmpMap.put(_Fields.NFE, new org.apache.thrift.meta_data.FieldMetaData("nfe", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- tmpMap.put(_Fields.IRE, new org.apache.thrift.meta_data.FieldMetaData("ire", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(describe_keyspace_result.class, metaDataMap);
- }
-
- public describe_keyspace_result() {
- }
-
- public describe_keyspace_result(
- KsDef success,
- NotFoundException nfe,
- InvalidRequestException ire)
- {
- this();
- this.success = success;
- this.nfe = nfe;
- this.ire = ire;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public describe_keyspace_result(describe_keyspace_result other) {
- if (other.isSetSuccess()) {
- this.success = new KsDef(other.success);
- }
- if (other.isSetNfe()) {
- this.nfe = new NotFoundException(other.nfe);
- }
- if (other.isSetIre()) {
- this.ire = new InvalidRequestException(other.ire);
- }
- }
-
- public describe_keyspace_result deepCopy() {
- return new describe_keyspace_result(this);
- }
-
- @Override
- public void clear() {
- this.success = null;
- this.nfe = null;
- this.ire = null;
- }
-
- public KsDef getSuccess() {
- return this.success;
- }
-
- public describe_keyspace_result setSuccess(KsDef success) {
- this.success = success;
- return this;
- }
-
- public void unsetSuccess() {
- this.success = null;
- }
-
- /** Returns true if field success is set (has been assigned a value) and false otherwise */
- public boolean isSetSuccess() {
- return this.success != null;
- }
-
- public void setSuccessIsSet(boolean value) {
- if (!value) {
- this.success = null;
- }
- }
-
- public NotFoundException getNfe() {
- return this.nfe;
- }
-
- public describe_keyspace_result setNfe(NotFoundException nfe) {
- this.nfe = nfe;
- return this;
- }
-
- public void unsetNfe() {
- this.nfe = null;
- }
-
- /** Returns true if field nfe is set (has been assigned a value) and false otherwise */
- public boolean isSetNfe() {
- return this.nfe != null;
- }
-
- public void setNfeIsSet(boolean value) {
- if (!value) {
- this.nfe = null;
- }
- }
-
- public InvalidRequestException getIre() {
- return this.ire;
- }
-
- public describe_keyspace_result setIre(InvalidRequestException ire) {
- this.ire = ire;
- return this;
- }
-
- public void unsetIre() {
- this.ire = null;
- }
-
- /** Returns true if field ire is set (has been assigned a value) and false otherwise */
- public boolean isSetIre() {
- return this.ire != null;
- }
-
- public void setIreIsSet(boolean value) {
- if (!value) {
- this.ire = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case SUCCESS:
- if (value == null) {
- unsetSuccess();
- } else {
- setSuccess((KsDef)value);
- }
- break;
-
- case NFE:
- if (value == null) {
- unsetNfe();
- } else {
- setNfe((NotFoundException)value);
- }
- break;
-
- case IRE:
- if (value == null) {
- unsetIre();
- } else {
- setIre((InvalidRequestException)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case SUCCESS:
- return getSuccess();
-
- case NFE:
- return getNfe();
-
- case IRE:
- return getIre();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case SUCCESS:
- return isSetSuccess();
- case NFE:
- return isSetNfe();
- case IRE:
- return isSetIre();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof describe_keyspace_result)
- return this.equals((describe_keyspace_result)that);
- return false;
- }
-
- public boolean equals(describe_keyspace_result that) {
- if (that == null)
- return false;
-
- boolean this_present_success = true && this.isSetSuccess();
- boolean that_present_success = true && that.isSetSuccess();
- if (this_present_success || that_present_success) {
- if (!(this_present_success && that_present_success))
- return false;
- if (!this.success.equals(that.success))
- return false;
- }
-
- boolean this_present_nfe = true && this.isSetNfe();
- boolean that_present_nfe = true && that.isSetNfe();
- if (this_present_nfe || that_present_nfe) {
- if (!(this_present_nfe && that_present_nfe))
- return false;
- if (!this.nfe.equals(that.nfe))
- return false;
- }
-
- boolean this_present_ire = true && this.isSetIre();
- boolean that_present_ire = true && that.isSetIre();
- if (this_present_ire || that_present_ire) {
- if (!(this_present_ire && that_present_ire))
- return false;
- if (!this.ire.equals(that.ire))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_success = true && (isSetSuccess());
- builder.append(present_success);
- if (present_success)
- builder.append(success);
-
- boolean present_nfe = true && (isSetNfe());
- builder.append(present_nfe);
- if (present_nfe)
- builder.append(nfe);
-
- boolean present_ire = true && (isSetIre());
- builder.append(present_ire);
- if (present_ire)
- builder.append(ire);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(describe_keyspace_result other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(other.isSetSuccess());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetSuccess()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, other.success);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetNfe()).compareTo(other.isSetNfe());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetNfe()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.nfe, other.nfe);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetIre()).compareTo(other.isSetIre());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetIre()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.ire, other.ire);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("describe_keyspace_result(");
- boolean first = true;
-
- sb.append("success:");
- if (this.success == null) {
- sb.append("null");
- } else {
- sb.append(this.success);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("nfe:");
- if (this.nfe == null) {
- sb.append("null");
- } else {
- sb.append(this.nfe);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("ire:");
- if (this.ire == null) {
- sb.append("null");
- } else {
- sb.append(this.ire);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // check for sub-struct validity
- if (success != null) {
- success.validate();
- }
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class describe_keyspace_resultStandardSchemeFactory implements SchemeFactory {
- public describe_keyspace_resultStandardScheme getScheme() {
- return new describe_keyspace_resultStandardScheme();
- }
- }
-
- private static class describe_keyspace_resultStandardScheme extends StandardScheme<describe_keyspace_result> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, describe_keyspace_result struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 0: // SUCCESS
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.success = new KsDef();
- struct.success.read(iprot);
- struct.setSuccessIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 1: // NFE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.nfe = new NotFoundException();
- struct.nfe.read(iprot);
- struct.setNfeIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 2: // IRE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.ire = new InvalidRequestException();
- struct.ire.read(iprot);
- struct.setIreIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, describe_keyspace_result struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.success != null) {
- oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
- struct.success.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.nfe != null) {
- oprot.writeFieldBegin(NFE_FIELD_DESC);
- struct.nfe.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.ire != null) {
- oprot.writeFieldBegin(IRE_FIELD_DESC);
- struct.ire.write(oprot);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class describe_keyspace_resultTupleSchemeFactory implements SchemeFactory {
- public describe_keyspace_resultTupleScheme getScheme() {
- return new describe_keyspace_resultTupleScheme();
- }
- }
-
- private static class describe_keyspace_resultTupleScheme extends TupleScheme<describe_keyspace_result> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, describe_keyspace_result struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- BitSet optionals = new BitSet();
- if (struct.isSetSuccess()) {
- optionals.set(0);
- }
- if (struct.isSetNfe()) {
- optionals.set(1);
- }
- if (struct.isSetIre()) {
- optionals.set(2);
- }
- oprot.writeBitSet(optionals, 3);
- if (struct.isSetSuccess()) {
- struct.success.write(oprot);
- }
- if (struct.isSetNfe()) {
- struct.nfe.write(oprot);
- }
- if (struct.isSetIre()) {
- struct.ire.write(oprot);
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, describe_keyspace_result struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- BitSet incoming = iprot.readBitSet(3);
- if (incoming.get(0)) {
- struct.success = new KsDef();
- struct.success.read(iprot);
- struct.setSuccessIsSet(true);
- }
- if (incoming.get(1)) {
- struct.nfe = new NotFoundException();
- struct.nfe.read(iprot);
- struct.setNfeIsSet(true);
- }
- if (incoming.get(2)) {
- struct.ire = new InvalidRequestException();
- struct.ire.read(iprot);
- struct.setIreIsSet(true);
- }
- }
- }
-
- }
-
- public static class describe_splits_args implements org.apache.thrift.TBase<describe_splits_args, describe_splits_args._Fields>, java.io.Serializable, Cloneable, Comparable<describe_splits_args> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("describe_splits_args");
-
- private static final org.apache.thrift.protocol.TField CF_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("cfName", org.apache.thrift.protocol.TType.STRING, (short)1);
- private static final org.apache.thrift.protocol.TField START_TOKEN_FIELD_DESC = new org.apache.thrift.protocol.TField("start_token", org.apache.thrift.protocol.TType.STRING, (short)2);
- private static final org.apache.thrift.protocol.TField END_TOKEN_FIELD_DESC = new org.apache.thrift.protocol.TField("end_token", org.apache.thrift.protocol.TType.STRING, (short)3);
- private static final org.apache.thrift.protocol.TField KEYS_PER_SPLIT_FIELD_DESC = new org.apache.thrift.protocol.TField("keys_per_split", org.apache.thrift.protocol.TType.I32, (short)4);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new describe_splits_argsStandardSchemeFactory());
- schemes.put(TupleScheme.class, new describe_splits_argsTupleSchemeFactory());
- }
-
- public String cfName; // required
- public String start_token; // required
- public String end_token; // required
- public int keys_per_split; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- CF_NAME((short)1, "cfName"),
- START_TOKEN((short)2, "start_token"),
- END_TOKEN((short)3, "end_token"),
- KEYS_PER_SPLIT((short)4, "keys_per_split");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // CF_NAME
- return CF_NAME;
- case 2: // START_TOKEN
- return START_TOKEN;
- case 3: // END_TOKEN
- return END_TOKEN;
- case 4: // KEYS_PER_SPLIT
- return KEYS_PER_SPLIT;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- private static final int __KEYS_PER_SPLIT_ISSET_ID = 0;
- private byte __isset_bitfield = 0;
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.CF_NAME, new org.apache.thrift.meta_data.FieldMetaData("cfName", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
- tmpMap.put(_Fields.START_TOKEN, new org.apache.thrift.meta_data.FieldMetaData("start_token", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
- tmpMap.put(_Fields.END_TOKEN, new org.apache.thrift.meta_data.FieldMetaData("end_token", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
- tmpMap.put(_Fields.KEYS_PER_SPLIT, new org.apache.thrift.meta_data.FieldMetaData("keys_per_split", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(describe_splits_args.class, metaDataMap);
- }
-
- public describe_splits_args() {
- }
-
- public describe_splits_args(
- String cfName,
- String start_token,
- String end_token,
- int keys_per_split)
- {
- this();
- this.cfName = cfName;
- this.start_token = start_token;
- this.end_token = end_token;
- this.keys_per_split = keys_per_split;
- setKeys_per_splitIsSet(true);
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public describe_splits_args(describe_splits_args other) {
- __isset_bitfield = other.__isset_bitfield;
- if (other.isSetCfName()) {
- this.cfName = other.cfName;
- }
- if (other.isSetStart_token()) {
- this.start_token = other.start_token;
- }
- if (other.isSetEnd_token()) {
- this.end_token = other.end_token;
- }
- this.keys_per_split = other.keys_per_split;
- }
-
- public describe_splits_args deepCopy() {
- return new describe_splits_args(this);
- }
-
- @Override
- public void clear() {
- this.cfName = null;
- this.start_token = null;
- this.end_token = null;
- setKeys_per_splitIsSet(false);
- this.keys_per_split = 0;
- }
-
- public String getCfName() {
- return this.cfName;
- }
-
- public describe_splits_args setCfName(String cfName) {
- this.cfName = cfName;
- return this;
- }
-
- public void unsetCfName() {
- this.cfName = null;
- }
-
- /** Returns true if field cfName is set (has been assigned a value) and false otherwise */
- public boolean isSetCfName() {
- return this.cfName != null;
- }
-
- public void setCfNameIsSet(boolean value) {
- if (!value) {
- this.cfName = null;
- }
- }
-
- public String getStart_token() {
- return this.start_token;
- }
-
- public describe_splits_args setStart_token(String start_token) {
- this.start_token = start_token;
- return this;
- }
-
- public void unsetStart_token() {
- this.start_token = null;
- }
-
- /** Returns true if field start_token is set (has been assigned a value) and false otherwise */
- public boolean isSetStart_token() {
- return this.start_token != null;
- }
-
- public void setStart_tokenIsSet(boolean value) {
- if (!value) {
- this.start_token = null;
- }
- }
-
- public String getEnd_token() {
- return this.end_token;
- }
-
- public describe_splits_args setEnd_token(String end_token) {
- this.end_token = end_token;
- return this;
- }
-
- public void unsetEnd_token() {
- this.end_token = null;
- }
-
- /** Returns true if field end_token is set (has been assigned a value) and false otherwise */
- public boolean isSetEnd_token() {
- return this.end_token != null;
- }
-
- public void setEnd_tokenIsSet(boolean value) {
- if (!value) {
- this.end_token = null;
- }
- }
-
- public int getKeys_per_split() {
- return this.keys_per_split;
- }
-
- public describe_splits_args setKeys_per_split(int keys_per_split) {
- this.keys_per_split = keys_per_split;
- setKeys_per_splitIsSet(true);
- return this;
- }
-
- public void unsetKeys_per_split() {
- __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __KEYS_PER_SPLIT_ISSET_ID);
- }
-
- /** Returns true if field keys_per_split is set (has been assigned a value) and false otherwise */
- public boolean isSetKeys_per_split() {
- return EncodingUtils.testBit(__isset_bitfield, __KEYS_PER_SPLIT_ISSET_ID);
- }
-
- public void setKeys_per_splitIsSet(boolean value) {
- __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __KEYS_PER_SPLIT_ISSET_ID, value);
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case CF_NAME:
- if (value == null) {
- unsetCfName();
- } else {
- setCfName((String)value);
- }
- break;
-
- case START_TOKEN:
- if (value == null) {
- unsetStart_token();
- } else {
- setStart_token((String)value);
- }
- break;
-
- case END_TOKEN:
- if (value == null) {
- unsetEnd_token();
- } else {
- setEnd_token((String)value);
- }
- break;
-
- case KEYS_PER_SPLIT:
- if (value == null) {
- unsetKeys_per_split();
- } else {
- setKeys_per_split((Integer)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case CF_NAME:
- return getCfName();
-
- case START_TOKEN:
- return getStart_token();
-
- case END_TOKEN:
- return getEnd_token();
-
- case KEYS_PER_SPLIT:
- return Integer.valueOf(getKeys_per_split());
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case CF_NAME:
- return isSetCfName();
- case START_TOKEN:
- return isSetStart_token();
- case END_TOKEN:
- return isSetEnd_token();
- case KEYS_PER_SPLIT:
- return isSetKeys_per_split();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof describe_splits_args)
- return this.equals((describe_splits_args)that);
- return false;
- }
-
- public boolean equals(describe_splits_args that) {
- if (that == null)
- return false;
-
- boolean this_present_cfName = true && this.isSetCfName();
- boolean that_present_cfName = true && that.isSetCfName();
- if (this_present_cfName || that_present_cfName) {
- if (!(this_present_cfName && that_present_cfName))
- return false;
- if (!this.cfName.equals(that.cfName))
- return false;
- }
-
- boolean this_present_start_token = true && this.isSetStart_token();
- boolean that_present_start_token = true && that.isSetStart_token();
- if (this_present_start_token || that_present_start_token) {
- if (!(this_present_start_token && that_present_start_token))
- return false;
- if (!this.start_token.equals(that.start_token))
- return false;
- }
-
- boolean this_present_end_token = true && this.isSetEnd_token();
- boolean that_present_end_token = true && that.isSetEnd_token();
- if (this_present_end_token || that_present_end_token) {
- if (!(this_present_end_token && that_present_end_token))
- return false;
- if (!this.end_token.equals(that.end_token))
- return false;
- }
-
- boolean this_present_keys_per_split = true;
- boolean that_present_keys_per_split = true;
- if (this_present_keys_per_split || that_present_keys_per_split) {
- if (!(this_present_keys_per_split && that_present_keys_per_split))
- return false;
- if (this.keys_per_split != that.keys_per_split)
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_cfName = true && (isSetCfName());
- builder.append(present_cfName);
- if (present_cfName)
- builder.append(cfName);
-
- boolean present_start_token = true && (isSetStart_token());
- builder.append(present_start_token);
- if (present_start_token)
- builder.append(start_token);
-
- boolean present_end_token = true && (isSetEnd_token());
- builder.append(present_end_token);
- if (present_end_token)
- builder.append(end_token);
-
- boolean present_keys_per_split = true;
- builder.append(present_keys_per_split);
- if (present_keys_per_split)
- builder.append(keys_per_split);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(describe_splits_args other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetCfName()).compareTo(other.isSetCfName());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetCfName()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.cfName, other.cfName);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetStart_token()).compareTo(other.isSetStart_token());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetStart_token()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.start_token, other.start_token);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetEnd_token()).compareTo(other.isSetEnd_token());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetEnd_token()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.end_token, other.end_token);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetKeys_per_split()).compareTo(other.isSetKeys_per_split());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetKeys_per_split()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.keys_per_split, other.keys_per_split);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("describe_splits_args(");
- boolean first = true;
-
- sb.append("cfName:");
- if (this.cfName == null) {
- sb.append("null");
- } else {
- sb.append(this.cfName);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("start_token:");
- if (this.start_token == null) {
- sb.append("null");
- } else {
- sb.append(this.start_token);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("end_token:");
- if (this.end_token == null) {
- sb.append("null");
- } else {
- sb.append(this.end_token);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("keys_per_split:");
- sb.append(this.keys_per_split);
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- if (cfName == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'cfName' was not present! Struct: " + toString());
- }
- if (start_token == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'start_token' was not present! Struct: " + toString());
- }
- if (end_token == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'end_token' was not present! Struct: " + toString());
- }
- // alas, we cannot check 'keys_per_split' because it's a primitive and you chose the non-beans generator.
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
- __isset_bitfield = 0;
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class describe_splits_argsStandardSchemeFactory implements SchemeFactory {
- public describe_splits_argsStandardScheme getScheme() {
- return new describe_splits_argsStandardScheme();
- }
- }
-
- private static class describe_splits_argsStandardScheme extends StandardScheme<describe_splits_args> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, describe_splits_args struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // CF_NAME
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.cfName = iprot.readString();
- struct.setCfNameIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 2: // START_TOKEN
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.start_token = iprot.readString();
- struct.setStart_tokenIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 3: // END_TOKEN
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.end_token = iprot.readString();
- struct.setEnd_tokenIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 4: // KEYS_PER_SPLIT
- if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
- struct.keys_per_split = iprot.readI32();
- struct.setKeys_per_splitIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- if (!struct.isSetKeys_per_split()) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'keys_per_split' was not found in serialized data! Struct: " + toString());
- }
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, describe_splits_args struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.cfName != null) {
- oprot.writeFieldBegin(CF_NAME_FIELD_DESC);
- oprot.writeString(struct.cfName);
- oprot.writeFieldEnd();
- }
- if (struct.start_token != null) {
- oprot.writeFieldBegin(START_TOKEN_FIELD_DESC);
- oprot.writeString(struct.start_token);
- oprot.writeFieldEnd();
- }
- if (struct.end_token != null) {
- oprot.writeFieldBegin(END_TOKEN_FIELD_DESC);
- oprot.writeString(struct.end_token);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldBegin(KEYS_PER_SPLIT_FIELD_DESC);
- oprot.writeI32(struct.keys_per_split);
- oprot.writeFieldEnd();
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class describe_splits_argsTupleSchemeFactory implements SchemeFactory {
- public describe_splits_argsTupleScheme getScheme() {
- return new describe_splits_argsTupleScheme();
- }
- }
-
- private static class describe_splits_argsTupleScheme extends TupleScheme<describe_splits_args> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, describe_splits_args struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- oprot.writeString(struct.cfName);
- oprot.writeString(struct.start_token);
- oprot.writeString(struct.end_token);
- oprot.writeI32(struct.keys_per_split);
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, describe_splits_args struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- struct.cfName = iprot.readString();
- struct.setCfNameIsSet(true);
- struct.start_token = iprot.readString();
- struct.setStart_tokenIsSet(true);
- struct.end_token = iprot.readString();
- struct.setEnd_tokenIsSet(true);
- struct.keys_per_split = iprot.readI32();
- struct.setKeys_per_splitIsSet(true);
- }
- }
-
- }
-
- public static class describe_splits_result implements org.apache.thrift.TBase<describe_splits_result, describe_splits_result._Fields>, java.io.Serializable, Cloneable, Comparable<describe_splits_result> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("describe_splits_result");
-
- private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.LIST, (short)0);
- private static final org.apache.thrift.protocol.TField IRE_FIELD_DESC = new org.apache.thrift.protocol.TField("ire", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new describe_splits_resultStandardSchemeFactory());
- schemes.put(TupleScheme.class, new describe_splits_resultTupleSchemeFactory());
- }
-
- public List<String> success; // required
- public InvalidRequestException ire; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- SUCCESS((short)0, "success"),
- IRE((short)1, "ire");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 0: // SUCCESS
- return SUCCESS;
- case 1: // IRE
- return IRE;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))));
- tmpMap.put(_Fields.IRE, new org.apache.thrift.meta_data.FieldMetaData("ire", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(describe_splits_result.class, metaDataMap);
- }
-
- public describe_splits_result() {
- }
-
- public describe_splits_result(
- List<String> success,
- InvalidRequestException ire)
- {
- this();
- this.success = success;
- this.ire = ire;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public describe_splits_result(describe_splits_result other) {
- if (other.isSetSuccess()) {
- List<String> __this__success = new ArrayList<String>(other.success);
- this.success = __this__success;
- }
- if (other.isSetIre()) {
- this.ire = new InvalidRequestException(other.ire);
- }
- }
-
- public describe_splits_result deepCopy() {
- return new describe_splits_result(this);
- }
-
- @Override
- public void clear() {
- this.success = null;
- this.ire = null;
- }
-
- public int getSuccessSize() {
- return (this.success == null) ? 0 : this.success.size();
- }
-
- public java.util.Iterator<String> getSuccessIterator() {
- return (this.success == null) ? null : this.success.iterator();
- }
-
- public void addToSuccess(String elem) {
- if (this.success == null) {
- this.success = new ArrayList<String>();
- }
- this.success.add(elem);
- }
-
- public List<String> getSuccess() {
- return this.success;
- }
-
- public describe_splits_result setSuccess(List<String> success) {
- this.success = success;
- return this;
- }
-
- public void unsetSuccess() {
- this.success = null;
- }
-
- /** Returns true if field success is set (has been assigned a value) and false otherwise */
- public boolean isSetSuccess() {
- return this.success != null;
- }
-
- public void setSuccessIsSet(boolean value) {
- if (!value) {
- this.success = null;
- }
- }
-
- public InvalidRequestException getIre() {
- return this.ire;
- }
-
- public describe_splits_result setIre(InvalidRequestException ire) {
- this.ire = ire;
- return this;
- }
-
- public void unsetIre() {
- this.ire = null;
- }
-
- /** Returns true if field ire is set (has been assigned a value) and false otherwise */
- public boolean isSetIre() {
- return this.ire != null;
- }
-
- public void setIreIsSet(boolean value) {
- if (!value) {
- this.ire = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case SUCCESS:
- if (value == null) {
- unsetSuccess();
- } else {
- setSuccess((List<String>)value);
- }
- break;
-
- case IRE:
- if (value == null) {
- unsetIre();
- } else {
- setIre((InvalidRequestException)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case SUCCESS:
- return getSuccess();
-
- case IRE:
- return getIre();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case SUCCESS:
- return isSetSuccess();
- case IRE:
- return isSetIre();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof describe_splits_result)
- return this.equals((describe_splits_result)that);
- return false;
- }
-
- public boolean equals(describe_splits_result that) {
- if (that == null)
- return false;
-
- boolean this_present_success = true && this.isSetSuccess();
- boolean that_present_success = true && that.isSetSuccess();
- if (this_present_success || that_present_success) {
- if (!(this_present_success && that_present_success))
- return false;
- if (!this.success.equals(that.success))
- return false;
- }
-
- boolean this_present_ire = true && this.isSetIre();
- boolean that_present_ire = true && that.isSetIre();
- if (this_present_ire || that_present_ire) {
- if (!(this_present_ire && that_present_ire))
- return false;
- if (!this.ire.equals(that.ire))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_success = true && (isSetSuccess());
- builder.append(present_success);
- if (present_success)
- builder.append(success);
-
- boolean present_ire = true && (isSetIre());
- builder.append(present_ire);
- if (present_ire)
- builder.append(ire);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(describe_splits_result other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(other.isSetSuccess());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetSuccess()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, other.success);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetIre()).compareTo(other.isSetIre());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetIre()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.ire, other.ire);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("describe_splits_result(");
- boolean first = true;
-
- sb.append("success:");
- if (this.success == null) {
- sb.append("null");
- } else {
- sb.append(this.success);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("ire:");
- if (this.ire == null) {
- sb.append("null");
- } else {
- sb.append(this.ire);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class describe_splits_resultStandardSchemeFactory implements SchemeFactory {
- public describe_splits_resultStandardScheme getScheme() {
- return new describe_splits_resultStandardScheme();
- }
- }
-
- private static class describe_splits_resultStandardScheme extends StandardScheme<describe_splits_result> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, describe_splits_result struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 0: // SUCCESS
- if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
- {
- org.apache.thrift.protocol.TList _list432 = iprot.readListBegin();
- struct.success = new ArrayList<String>(_list432.size);
- for (int _i433 = 0; _i433 < _list432.size; ++_i433)
- {
- String _elem434;
- _elem434 = iprot.readString();
- struct.success.add(_elem434);
- }
- iprot.readListEnd();
- }
- struct.setSuccessIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 1: // IRE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.ire = new InvalidRequestException();
- struct.ire.read(iprot);
- struct.setIreIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, describe_splits_result struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.success != null) {
- oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
- {
- oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, struct.success.size()));
- for (String _iter435 : struct.success)
- {
- oprot.writeString(_iter435);
- }
- oprot.writeListEnd();
- }
- oprot.writeFieldEnd();
- }
- if (struct.ire != null) {
- oprot.writeFieldBegin(IRE_FIELD_DESC);
- struct.ire.write(oprot);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class describe_splits_resultTupleSchemeFactory implements SchemeFactory {
- public describe_splits_resultTupleScheme getScheme() {
- return new describe_splits_resultTupleScheme();
- }
- }
-
- private static class describe_splits_resultTupleScheme extends TupleScheme<describe_splits_result> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, describe_splits_result struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- BitSet optionals = new BitSet();
- if (struct.isSetSuccess()) {
- optionals.set(0);
- }
- if (struct.isSetIre()) {
- optionals.set(1);
- }
- oprot.writeBitSet(optionals, 2);
- if (struct.isSetSuccess()) {
- {
- oprot.writeI32(struct.success.size());
- for (String _iter436 : struct.success)
- {
- oprot.writeString(_iter436);
- }
- }
- }
- if (struct.isSetIre()) {
- struct.ire.write(oprot);
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, describe_splits_result struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- BitSet incoming = iprot.readBitSet(2);
- if (incoming.get(0)) {
- {
- org.apache.thrift.protocol.TList _list437 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32());
- struct.success = new ArrayList<String>(_list437.size);
- for (int _i438 = 0; _i438 < _list437.size; ++_i438)
- {
- String _elem439;
- _elem439 = iprot.readString();
- struct.success.add(_elem439);
- }
- }
- struct.setSuccessIsSet(true);
- }
- if (incoming.get(1)) {
- struct.ire = new InvalidRequestException();
- struct.ire.read(iprot);
- struct.setIreIsSet(true);
- }
- }
- }
-
- }
-
- public static class trace_next_query_args implements org.apache.thrift.TBase<trace_next_query_args, trace_next_query_args._Fields>, java.io.Serializable, Cloneable, Comparable<trace_next_query_args> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("trace_next_query_args");
-
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new trace_next_query_argsStandardSchemeFactory());
- schemes.put(TupleScheme.class, new trace_next_query_argsTupleSchemeFactory());
- }
-
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-;
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(trace_next_query_args.class, metaDataMap);
- }
-
- public trace_next_query_args() {
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public trace_next_query_args(trace_next_query_args other) {
- }
-
- public trace_next_query_args deepCopy() {
- return new trace_next_query_args(this);
- }
-
- @Override
- public void clear() {
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof trace_next_query_args)
- return this.equals((trace_next_query_args)that);
- return false;
- }
-
- public boolean equals(trace_next_query_args that) {
- if (that == null)
- return false;
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(trace_next_query_args other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("trace_next_query_args(");
- boolean first = true;
-
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class trace_next_query_argsStandardSchemeFactory implements SchemeFactory {
- public trace_next_query_argsStandardScheme getScheme() {
- return new trace_next_query_argsStandardScheme();
- }
- }
-
- private static class trace_next_query_argsStandardScheme extends StandardScheme<trace_next_query_args> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, trace_next_query_args struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, trace_next_query_args struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class trace_next_query_argsTupleSchemeFactory implements SchemeFactory {
- public trace_next_query_argsTupleScheme getScheme() {
- return new trace_next_query_argsTupleScheme();
- }
- }
-
- private static class trace_next_query_argsTupleScheme extends TupleScheme<trace_next_query_args> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, trace_next_query_args struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, trace_next_query_args struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- }
- }
-
- }
-
- public static class trace_next_query_result implements org.apache.thrift.TBase<trace_next_query_result, trace_next_query_result._Fields>, java.io.Serializable, Cloneable, Comparable<trace_next_query_result> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("trace_next_query_result");
-
- private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRING, (short)0);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new trace_next_query_resultStandardSchemeFactory());
- schemes.put(TupleScheme.class, new trace_next_query_resultTupleSchemeFactory());
- }
-
- public ByteBuffer success; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- SUCCESS((short)0, "success");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 0: // SUCCESS
- return SUCCESS;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(trace_next_query_result.class, metaDataMap);
- }
-
- public trace_next_query_result() {
- }
-
- public trace_next_query_result(
- ByteBuffer success)
- {
- this();
- this.success = success;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public trace_next_query_result(trace_next_query_result other) {
- if (other.isSetSuccess()) {
- this.success = org.apache.thrift.TBaseHelper.copyBinary(other.success);
-;
- }
- }
-
- public trace_next_query_result deepCopy() {
- return new trace_next_query_result(this);
- }
-
- @Override
- public void clear() {
- this.success = null;
- }
-
- public byte[] getSuccess() {
- setSuccess(org.apache.thrift.TBaseHelper.rightSize(success));
- return success == null ? null : success.array();
- }
-
- public ByteBuffer bufferForSuccess() {
- return success;
- }
-
- public trace_next_query_result setSuccess(byte[] success) {
- setSuccess(success == null ? (ByteBuffer)null : ByteBuffer.wrap(success));
- return this;
- }
-
- public trace_next_query_result setSuccess(ByteBuffer success) {
- this.success = success;
- return this;
- }
-
- public void unsetSuccess() {
- this.success = null;
- }
-
- /** Returns true if field success is set (has been assigned a value) and false otherwise */
- public boolean isSetSuccess() {
- return this.success != null;
- }
-
- public void setSuccessIsSet(boolean value) {
- if (!value) {
- this.success = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case SUCCESS:
- if (value == null) {
- unsetSuccess();
- } else {
- setSuccess((ByteBuffer)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case SUCCESS:
- return getSuccess();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case SUCCESS:
- return isSetSuccess();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof trace_next_query_result)
- return this.equals((trace_next_query_result)that);
- return false;
- }
-
- public boolean equals(trace_next_query_result that) {
- if (that == null)
- return false;
-
- boolean this_present_success = true && this.isSetSuccess();
- boolean that_present_success = true && that.isSetSuccess();
- if (this_present_success || that_present_success) {
- if (!(this_present_success && that_present_success))
- return false;
- if (!this.success.equals(that.success))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_success = true && (isSetSuccess());
- builder.append(present_success);
- if (present_success)
- builder.append(success);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(trace_next_query_result other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(other.isSetSuccess());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetSuccess()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, other.success);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("trace_next_query_result(");
- boolean first = true;
-
- sb.append("success:");
- if (this.success == null) {
- sb.append("null");
- } else {
- org.apache.thrift.TBaseHelper.toString(this.success, sb);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class trace_next_query_resultStandardSchemeFactory implements SchemeFactory {
- public trace_next_query_resultStandardScheme getScheme() {
- return new trace_next_query_resultStandardScheme();
- }
- }
-
- private static class trace_next_query_resultStandardScheme extends StandardScheme<trace_next_query_result> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, trace_next_query_result struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 0: // SUCCESS
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.success = iprot.readBinary();
- struct.setSuccessIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, trace_next_query_result struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.success != null) {
- oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
- oprot.writeBinary(struct.success);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class trace_next_query_resultTupleSchemeFactory implements SchemeFactory {
- public trace_next_query_resultTupleScheme getScheme() {
- return new trace_next_query_resultTupleScheme();
- }
- }
-
- private static class trace_next_query_resultTupleScheme extends TupleScheme<trace_next_query_result> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, trace_next_query_result struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- BitSet optionals = new BitSet();
- if (struct.isSetSuccess()) {
- optionals.set(0);
- }
- oprot.writeBitSet(optionals, 1);
- if (struct.isSetSuccess()) {
- oprot.writeBinary(struct.success);
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, trace_next_query_result struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- BitSet incoming = iprot.readBitSet(1);
- if (incoming.get(0)) {
- struct.success = iprot.readBinary();
- struct.setSuccessIsSet(true);
- }
- }
- }
-
- }
-
- public static class describe_splits_ex_args implements org.apache.thrift.TBase<describe_splits_ex_args, describe_splits_ex_args._Fields>, java.io.Serializable, Cloneable, Comparable<describe_splits_ex_args> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("describe_splits_ex_args");
-
- private static final org.apache.thrift.protocol.TField CF_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("cfName", org.apache.thrift.protocol.TType.STRING, (short)1);
- private static final org.apache.thrift.protocol.TField START_TOKEN_FIELD_DESC = new org.apache.thrift.protocol.TField("start_token", org.apache.thrift.protocol.TType.STRING, (short)2);
- private static final org.apache.thrift.protocol.TField END_TOKEN_FIELD_DESC = new org.apache.thrift.protocol.TField("end_token", org.apache.thrift.protocol.TType.STRING, (short)3);
- private static final org.apache.thrift.protocol.TField KEYS_PER_SPLIT_FIELD_DESC = new org.apache.thrift.protocol.TField("keys_per_split", org.apache.thrift.protocol.TType.I32, (short)4);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new describe_splits_ex_argsStandardSchemeFactory());
- schemes.put(TupleScheme.class, new describe_splits_ex_argsTupleSchemeFactory());
- }
-
- public String cfName; // required
- public String start_token; // required
- public String end_token; // required
- public int keys_per_split; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- CF_NAME((short)1, "cfName"),
- START_TOKEN((short)2, "start_token"),
- END_TOKEN((short)3, "end_token"),
- KEYS_PER_SPLIT((short)4, "keys_per_split");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // CF_NAME
- return CF_NAME;
- case 2: // START_TOKEN
- return START_TOKEN;
- case 3: // END_TOKEN
- return END_TOKEN;
- case 4: // KEYS_PER_SPLIT
- return KEYS_PER_SPLIT;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- private static final int __KEYS_PER_SPLIT_ISSET_ID = 0;
- private byte __isset_bitfield = 0;
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.CF_NAME, new org.apache.thrift.meta_data.FieldMetaData("cfName", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
- tmpMap.put(_Fields.START_TOKEN, new org.apache.thrift.meta_data.FieldMetaData("start_token", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
- tmpMap.put(_Fields.END_TOKEN, new org.apache.thrift.meta_data.FieldMetaData("end_token", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
- tmpMap.put(_Fields.KEYS_PER_SPLIT, new org.apache.thrift.meta_data.FieldMetaData("keys_per_split", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(describe_splits_ex_args.class, metaDataMap);
- }
-
- public describe_splits_ex_args() {
- }
-
- public describe_splits_ex_args(
- String cfName,
- String start_token,
- String end_token,
- int keys_per_split)
- {
- this();
- this.cfName = cfName;
- this.start_token = start_token;
- this.end_token = end_token;
- this.keys_per_split = keys_per_split;
- setKeys_per_splitIsSet(true);
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public describe_splits_ex_args(describe_splits_ex_args other) {
- __isset_bitfield = other.__isset_bitfield;
- if (other.isSetCfName()) {
- this.cfName = other.cfName;
- }
- if (other.isSetStart_token()) {
- this.start_token = other.start_token;
- }
- if (other.isSetEnd_token()) {
- this.end_token = other.end_token;
- }
- this.keys_per_split = other.keys_per_split;
- }
-
- public describe_splits_ex_args deepCopy() {
- return new describe_splits_ex_args(this);
- }
-
- @Override
- public void clear() {
- this.cfName = null;
- this.start_token = null;
- this.end_token = null;
- setKeys_per_splitIsSet(false);
- this.keys_per_split = 0;
- }
-
- public String getCfName() {
- return this.cfName;
- }
-
- public describe_splits_ex_args setCfName(String cfName) {
- this.cfName = cfName;
- return this;
- }
-
- public void unsetCfName() {
- this.cfName = null;
- }
-
- /** Returns true if field cfName is set (has been assigned a value) and false otherwise */
- public boolean isSetCfName() {
- return this.cfName != null;
- }
-
- public void setCfNameIsSet(boolean value) {
- if (!value) {
- this.cfName = null;
- }
- }
-
- public String getStart_token() {
- return this.start_token;
- }
-
- public describe_splits_ex_args setStart_token(String start_token) {
- this.start_token = start_token;
- return this;
- }
-
- public void unsetStart_token() {
- this.start_token = null;
- }
-
- /** Returns true if field start_token is set (has been assigned a value) and false otherwise */
- public boolean isSetStart_token() {
- return this.start_token != null;
- }
-
- public void setStart_tokenIsSet(boolean value) {
- if (!value) {
- this.start_token = null;
- }
- }
-
- public String getEnd_token() {
- return this.end_token;
- }
-
- public describe_splits_ex_args setEnd_token(String end_token) {
- this.end_token = end_token;
- return this;
- }
-
- public void unsetEnd_token() {
- this.end_token = null;
- }
-
- /** Returns true if field end_token is set (has been assigned a value) and false otherwise */
- public boolean isSetEnd_token() {
- return this.end_token != null;
- }
-
- public void setEnd_tokenIsSet(boolean value) {
- if (!value) {
- this.end_token = null;
- }
- }
-
- public int getKeys_per_split() {
- return this.keys_per_split;
- }
-
- public describe_splits_ex_args setKeys_per_split(int keys_per_split) {
- this.keys_per_split = keys_per_split;
- setKeys_per_splitIsSet(true);
- return this;
- }
-
- public void unsetKeys_per_split() {
- __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __KEYS_PER_SPLIT_ISSET_ID);
- }
-
- /** Returns true if field keys_per_split is set (has been assigned a value) and false otherwise */
- public boolean isSetKeys_per_split() {
- return EncodingUtils.testBit(__isset_bitfield, __KEYS_PER_SPLIT_ISSET_ID);
- }
-
- public void setKeys_per_splitIsSet(boolean value) {
- __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __KEYS_PER_SPLIT_ISSET_ID, value);
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case CF_NAME:
- if (value == null) {
- unsetCfName();
- } else {
- setCfName((String)value);
- }
- break;
-
- case START_TOKEN:
- if (value == null) {
- unsetStart_token();
- } else {
- setStart_token((String)value);
- }
- break;
-
- case END_TOKEN:
- if (value == null) {
- unsetEnd_token();
- } else {
- setEnd_token((String)value);
- }
- break;
-
- case KEYS_PER_SPLIT:
- if (value == null) {
- unsetKeys_per_split();
- } else {
- setKeys_per_split((Integer)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case CF_NAME:
- return getCfName();
-
- case START_TOKEN:
- return getStart_token();
-
- case END_TOKEN:
- return getEnd_token();
-
- case KEYS_PER_SPLIT:
- return Integer.valueOf(getKeys_per_split());
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case CF_NAME:
- return isSetCfName();
- case START_TOKEN:
- return isSetStart_token();
- case END_TOKEN:
- return isSetEnd_token();
- case KEYS_PER_SPLIT:
- return isSetKeys_per_split();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof describe_splits_ex_args)
- return this.equals((describe_splits_ex_args)that);
- return false;
- }
-
- public boolean equals(describe_splits_ex_args that) {
- if (that == null)
- return false;
-
- boolean this_present_cfName = true && this.isSetCfName();
- boolean that_present_cfName = true && that.isSetCfName();
- if (this_present_cfName || that_present_cfName) {
- if (!(this_present_cfName && that_present_cfName))
- return false;
- if (!this.cfName.equals(that.cfName))
- return false;
- }
-
- boolean this_present_start_token = true && this.isSetStart_token();
- boolean that_present_start_token = true && that.isSetStart_token();
- if (this_present_start_token || that_present_start_token) {
- if (!(this_present_start_token && that_present_start_token))
- return false;
- if (!this.start_token.equals(that.start_token))
- return false;
- }
-
- boolean this_present_end_token = true && this.isSetEnd_token();
- boolean that_present_end_token = true && that.isSetEnd_token();
- if (this_present_end_token || that_present_end_token) {
- if (!(this_present_end_token && that_present_end_token))
- return false;
- if (!this.end_token.equals(that.end_token))
- return false;
- }
-
- boolean this_present_keys_per_split = true;
- boolean that_present_keys_per_split = true;
- if (this_present_keys_per_split || that_present_keys_per_split) {
- if (!(this_present_keys_per_split && that_present_keys_per_split))
- return false;
- if (this.keys_per_split != that.keys_per_split)
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_cfName = true && (isSetCfName());
- builder.append(present_cfName);
- if (present_cfName)
- builder.append(cfName);
-
- boolean present_start_token = true && (isSetStart_token());
- builder.append(present_start_token);
- if (present_start_token)
- builder.append(start_token);
-
- boolean present_end_token = true && (isSetEnd_token());
- builder.append(present_end_token);
- if (present_end_token)
- builder.append(end_token);
-
- boolean present_keys_per_split = true;
- builder.append(present_keys_per_split);
- if (present_keys_per_split)
- builder.append(keys_per_split);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(describe_splits_ex_args other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetCfName()).compareTo(other.isSetCfName());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetCfName()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.cfName, other.cfName);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetStart_token()).compareTo(other.isSetStart_token());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetStart_token()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.start_token, other.start_token);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetEnd_token()).compareTo(other.isSetEnd_token());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetEnd_token()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.end_token, other.end_token);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetKeys_per_split()).compareTo(other.isSetKeys_per_split());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetKeys_per_split()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.keys_per_split, other.keys_per_split);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("describe_splits_ex_args(");
- boolean first = true;
-
- sb.append("cfName:");
- if (this.cfName == null) {
- sb.append("null");
- } else {
- sb.append(this.cfName);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("start_token:");
- if (this.start_token == null) {
- sb.append("null");
- } else {
- sb.append(this.start_token);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("end_token:");
- if (this.end_token == null) {
- sb.append("null");
- } else {
- sb.append(this.end_token);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("keys_per_split:");
- sb.append(this.keys_per_split);
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- if (cfName == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'cfName' was not present! Struct: " + toString());
- }
- if (start_token == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'start_token' was not present! Struct: " + toString());
- }
- if (end_token == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'end_token' was not present! Struct: " + toString());
- }
- // alas, we cannot check 'keys_per_split' because it's a primitive and you chose the non-beans generator.
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
- __isset_bitfield = 0;
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class describe_splits_ex_argsStandardSchemeFactory implements SchemeFactory {
- public describe_splits_ex_argsStandardScheme getScheme() {
- return new describe_splits_ex_argsStandardScheme();
- }
- }
-
- private static class describe_splits_ex_argsStandardScheme extends StandardScheme<describe_splits_ex_args> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, describe_splits_ex_args struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // CF_NAME
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.cfName = iprot.readString();
- struct.setCfNameIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 2: // START_TOKEN
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.start_token = iprot.readString();
- struct.setStart_tokenIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 3: // END_TOKEN
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.end_token = iprot.readString();
- struct.setEnd_tokenIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 4: // KEYS_PER_SPLIT
- if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
- struct.keys_per_split = iprot.readI32();
- struct.setKeys_per_splitIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- if (!struct.isSetKeys_per_split()) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'keys_per_split' was not found in serialized data! Struct: " + toString());
- }
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, describe_splits_ex_args struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.cfName != null) {
- oprot.writeFieldBegin(CF_NAME_FIELD_DESC);
- oprot.writeString(struct.cfName);
- oprot.writeFieldEnd();
- }
- if (struct.start_token != null) {
- oprot.writeFieldBegin(START_TOKEN_FIELD_DESC);
- oprot.writeString(struct.start_token);
- oprot.writeFieldEnd();
- }
- if (struct.end_token != null) {
- oprot.writeFieldBegin(END_TOKEN_FIELD_DESC);
- oprot.writeString(struct.end_token);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldBegin(KEYS_PER_SPLIT_FIELD_DESC);
- oprot.writeI32(struct.keys_per_split);
- oprot.writeFieldEnd();
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class describe_splits_ex_argsTupleSchemeFactory implements SchemeFactory {
- public describe_splits_ex_argsTupleScheme getScheme() {
- return new describe_splits_ex_argsTupleScheme();
- }
- }
-
- private static class describe_splits_ex_argsTupleScheme extends TupleScheme<describe_splits_ex_args> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, describe_splits_ex_args struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- oprot.writeString(struct.cfName);
- oprot.writeString(struct.start_token);
- oprot.writeString(struct.end_token);
- oprot.writeI32(struct.keys_per_split);
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, describe_splits_ex_args struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- struct.cfName = iprot.readString();
- struct.setCfNameIsSet(true);
- struct.start_token = iprot.readString();
- struct.setStart_tokenIsSet(true);
- struct.end_token = iprot.readString();
- struct.setEnd_tokenIsSet(true);
- struct.keys_per_split = iprot.readI32();
- struct.setKeys_per_splitIsSet(true);
- }
- }
-
- }
-
- public static class describe_splits_ex_result implements org.apache.thrift.TBase<describe_splits_ex_result, describe_splits_ex_result._Fields>, java.io.Serializable, Cloneable, Comparable<describe_splits_ex_result> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("describe_splits_ex_result");
-
- private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.LIST, (short)0);
- private static final org.apache.thrift.protocol.TField IRE_FIELD_DESC = new org.apache.thrift.protocol.TField("ire", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new describe_splits_ex_resultStandardSchemeFactory());
- schemes.put(TupleScheme.class, new describe_splits_ex_resultTupleSchemeFactory());
- }
-
- public List<CfSplit> success; // required
- public InvalidRequestException ire; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- SUCCESS((short)0, "success"),
- IRE((short)1, "ire");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 0: // SUCCESS
- return SUCCESS;
- case 1: // IRE
- return IRE;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST,
- new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, CfSplit.class))));
- tmpMap.put(_Fields.IRE, new org.apache.thrift.meta_data.FieldMetaData("ire", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(describe_splits_ex_result.class, metaDataMap);
- }
-
- public describe_splits_ex_result() {
- }
-
- public describe_splits_ex_result(
- List<CfSplit> success,
- InvalidRequestException ire)
- {
- this();
- this.success = success;
- this.ire = ire;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public describe_splits_ex_result(describe_splits_ex_result other) {
- if (other.isSetSuccess()) {
- List<CfSplit> __this__success = new ArrayList<CfSplit>(other.success.size());
- for (CfSplit other_element : other.success) {
- __this__success.add(new CfSplit(other_element));
- }
- this.success = __this__success;
- }
- if (other.isSetIre()) {
- this.ire = new InvalidRequestException(other.ire);
- }
- }
-
- public describe_splits_ex_result deepCopy() {
- return new describe_splits_ex_result(this);
- }
-
- @Override
- public void clear() {
- this.success = null;
- this.ire = null;
- }
-
- public int getSuccessSize() {
- return (this.success == null) ? 0 : this.success.size();
- }
-
- public java.util.Iterator<CfSplit> getSuccessIterator() {
- return (this.success == null) ? null : this.success.iterator();
- }
-
- public void addToSuccess(CfSplit elem) {
- if (this.success == null) {
- this.success = new ArrayList<CfSplit>();
- }
- this.success.add(elem);
- }
-
- public List<CfSplit> getSuccess() {
- return this.success;
- }
-
- public describe_splits_ex_result setSuccess(List<CfSplit> success) {
- this.success = success;
- return this;
- }
-
- public void unsetSuccess() {
- this.success = null;
- }
-
- /** Returns true if field success is set (has been assigned a value) and false otherwise */
- public boolean isSetSuccess() {
- return this.success != null;
- }
-
- public void setSuccessIsSet(boolean value) {
- if (!value) {
- this.success = null;
- }
- }
-
- public InvalidRequestException getIre() {
- return this.ire;
- }
-
- public describe_splits_ex_result setIre(InvalidRequestException ire) {
- this.ire = ire;
- return this;
- }
-
- public void unsetIre() {
- this.ire = null;
- }
-
- /** Returns true if field ire is set (has been assigned a value) and false otherwise */
- public boolean isSetIre() {
- return this.ire != null;
- }
-
- public void setIreIsSet(boolean value) {
- if (!value) {
- this.ire = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case SUCCESS:
- if (value == null) {
- unsetSuccess();
- } else {
- setSuccess((List<CfSplit>)value);
- }
- break;
-
- case IRE:
- if (value == null) {
- unsetIre();
- } else {
- setIre((InvalidRequestException)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case SUCCESS:
- return getSuccess();
-
- case IRE:
- return getIre();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case SUCCESS:
- return isSetSuccess();
- case IRE:
- return isSetIre();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof describe_splits_ex_result)
- return this.equals((describe_splits_ex_result)that);
- return false;
- }
-
- public boolean equals(describe_splits_ex_result that) {
- if (that == null)
- return false;
-
- boolean this_present_success = true && this.isSetSuccess();
- boolean that_present_success = true && that.isSetSuccess();
- if (this_present_success || that_present_success) {
- if (!(this_present_success && that_present_success))
- return false;
- if (!this.success.equals(that.success))
- return false;
- }
-
- boolean this_present_ire = true && this.isSetIre();
- boolean that_present_ire = true && that.isSetIre();
- if (this_present_ire || that_present_ire) {
- if (!(this_present_ire && that_present_ire))
- return false;
- if (!this.ire.equals(that.ire))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_success = true && (isSetSuccess());
- builder.append(present_success);
- if (present_success)
- builder.append(success);
-
- boolean present_ire = true && (isSetIre());
- builder.append(present_ire);
- if (present_ire)
- builder.append(ire);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(describe_splits_ex_result other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(other.isSetSuccess());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetSuccess()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, other.success);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetIre()).compareTo(other.isSetIre());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetIre()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.ire, other.ire);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("describe_splits_ex_result(");
- boolean first = true;
-
- sb.append("success:");
- if (this.success == null) {
- sb.append("null");
- } else {
- sb.append(this.success);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("ire:");
- if (this.ire == null) {
- sb.append("null");
- } else {
- sb.append(this.ire);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class describe_splits_ex_resultStandardSchemeFactory implements SchemeFactory {
- public describe_splits_ex_resultStandardScheme getScheme() {
- return new describe_splits_ex_resultStandardScheme();
- }
- }
-
- private static class describe_splits_ex_resultStandardScheme extends StandardScheme<describe_splits_ex_result> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, describe_splits_ex_result struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 0: // SUCCESS
- if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
- {
- org.apache.thrift.protocol.TList _list440 = iprot.readListBegin();
- struct.success = new ArrayList<CfSplit>(_list440.size);
- for (int _i441 = 0; _i441 < _list440.size; ++_i441)
- {
- CfSplit _elem442;
- _elem442 = new CfSplit();
- _elem442.read(iprot);
- struct.success.add(_elem442);
- }
- iprot.readListEnd();
- }
- struct.setSuccessIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 1: // IRE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.ire = new InvalidRequestException();
- struct.ire.read(iprot);
- struct.setIreIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, describe_splits_ex_result struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.success != null) {
- oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
- {
- oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.success.size()));
- for (CfSplit _iter443 : struct.success)
- {
- _iter443.write(oprot);
- }
- oprot.writeListEnd();
- }
- oprot.writeFieldEnd();
- }
- if (struct.ire != null) {
- oprot.writeFieldBegin(IRE_FIELD_DESC);
- struct.ire.write(oprot);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class describe_splits_ex_resultTupleSchemeFactory implements SchemeFactory {
- public describe_splits_ex_resultTupleScheme getScheme() {
- return new describe_splits_ex_resultTupleScheme();
- }
- }
-
- private static class describe_splits_ex_resultTupleScheme extends TupleScheme<describe_splits_ex_result> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, describe_splits_ex_result struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- BitSet optionals = new BitSet();
- if (struct.isSetSuccess()) {
- optionals.set(0);
- }
- if (struct.isSetIre()) {
- optionals.set(1);
- }
- oprot.writeBitSet(optionals, 2);
- if (struct.isSetSuccess()) {
- {
- oprot.writeI32(struct.success.size());
- for (CfSplit _iter444 : struct.success)
- {
- _iter444.write(oprot);
- }
- }
- }
- if (struct.isSetIre()) {
- struct.ire.write(oprot);
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, describe_splits_ex_result struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- BitSet incoming = iprot.readBitSet(2);
- if (incoming.get(0)) {
- {
- org.apache.thrift.protocol.TList _list445 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
- struct.success = new ArrayList<CfSplit>(_list445.size);
- for (int _i446 = 0; _i446 < _list445.size; ++_i446)
- {
- CfSplit _elem447;
- _elem447 = new CfSplit();
- _elem447.read(iprot);
- struct.success.add(_elem447);
- }
- }
- struct.setSuccessIsSet(true);
- }
- if (incoming.get(1)) {
- struct.ire = new InvalidRequestException();
- struct.ire.read(iprot);
- struct.setIreIsSet(true);
- }
- }
- }
-
- }
-
- public static class system_add_column_family_args implements org.apache.thrift.TBase<system_add_column_family_args, system_add_column_family_args._Fields>, java.io.Serializable, Cloneable, Comparable<system_add_column_family_args> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("system_add_column_family_args");
-
- private static final org.apache.thrift.protocol.TField CF_DEF_FIELD_DESC = new org.apache.thrift.protocol.TField("cf_def", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new system_add_column_family_argsStandardSchemeFactory());
- schemes.put(TupleScheme.class, new system_add_column_family_argsTupleSchemeFactory());
- }
-
- public CfDef cf_def; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- CF_DEF((short)1, "cf_def");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // CF_DEF
- return CF_DEF;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.CF_DEF, new org.apache.thrift.meta_data.FieldMetaData("cf_def", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, CfDef.class)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(system_add_column_family_args.class, metaDataMap);
- }
-
- public system_add_column_family_args() {
- }
-
- public system_add_column_family_args(
- CfDef cf_def)
- {
- this();
- this.cf_def = cf_def;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public system_add_column_family_args(system_add_column_family_args other) {
- if (other.isSetCf_def()) {
- this.cf_def = new CfDef(other.cf_def);
- }
- }
-
- public system_add_column_family_args deepCopy() {
- return new system_add_column_family_args(this);
- }
-
- @Override
- public void clear() {
- this.cf_def = null;
- }
-
- public CfDef getCf_def() {
- return this.cf_def;
- }
-
- public system_add_column_family_args setCf_def(CfDef cf_def) {
- this.cf_def = cf_def;
- return this;
- }
-
- public void unsetCf_def() {
- this.cf_def = null;
- }
-
- /** Returns true if field cf_def is set (has been assigned a value) and false otherwise */
- public boolean isSetCf_def() {
- return this.cf_def != null;
- }
-
- public void setCf_defIsSet(boolean value) {
- if (!value) {
- this.cf_def = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case CF_DEF:
- if (value == null) {
- unsetCf_def();
- } else {
- setCf_def((CfDef)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case CF_DEF:
- return getCf_def();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case CF_DEF:
- return isSetCf_def();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof system_add_column_family_args)
- return this.equals((system_add_column_family_args)that);
- return false;
- }
-
- public boolean equals(system_add_column_family_args that) {
- if (that == null)
- return false;
-
- boolean this_present_cf_def = true && this.isSetCf_def();
- boolean that_present_cf_def = true && that.isSetCf_def();
- if (this_present_cf_def || that_present_cf_def) {
- if (!(this_present_cf_def && that_present_cf_def))
- return false;
- if (!this.cf_def.equals(that.cf_def))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_cf_def = true && (isSetCf_def());
- builder.append(present_cf_def);
- if (present_cf_def)
- builder.append(cf_def);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(system_add_column_family_args other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetCf_def()).compareTo(other.isSetCf_def());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetCf_def()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.cf_def, other.cf_def);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("system_add_column_family_args(");
- boolean first = true;
-
- sb.append("cf_def:");
- if (this.cf_def == null) {
- sb.append("null");
- } else {
- sb.append(this.cf_def);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- if (cf_def == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'cf_def' was not present! Struct: " + toString());
- }
- // check for sub-struct validity
- if (cf_def != null) {
- cf_def.validate();
- }
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class system_add_column_family_argsStandardSchemeFactory implements SchemeFactory {
- public system_add_column_family_argsStandardScheme getScheme() {
- return new system_add_column_family_argsStandardScheme();
- }
- }
-
- private static class system_add_column_family_argsStandardScheme extends StandardScheme<system_add_column_family_args> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, system_add_column_family_args struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // CF_DEF
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.cf_def = new CfDef();
- struct.cf_def.read(iprot);
- struct.setCf_defIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, system_add_column_family_args struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.cf_def != null) {
- oprot.writeFieldBegin(CF_DEF_FIELD_DESC);
- struct.cf_def.write(oprot);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class system_add_column_family_argsTupleSchemeFactory implements SchemeFactory {
- public system_add_column_family_argsTupleScheme getScheme() {
- return new system_add_column_family_argsTupleScheme();
- }
- }
-
- private static class system_add_column_family_argsTupleScheme extends TupleScheme<system_add_column_family_args> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, system_add_column_family_args struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- struct.cf_def.write(oprot);
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, system_add_column_family_args struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- struct.cf_def = new CfDef();
- struct.cf_def.read(iprot);
- struct.setCf_defIsSet(true);
- }
- }
-
- }
-
- public static class system_add_column_family_result implements org.apache.thrift.TBase<system_add_column_family_result, system_add_column_family_result._Fields>, java.io.Serializable, Cloneable, Comparable<system_add_column_family_result> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("system_add_column_family_result");
-
- private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRING, (short)0);
- private static final org.apache.thrift.protocol.TField IRE_FIELD_DESC = new org.apache.thrift.protocol.TField("ire", org.apache.thrift.protocol.TType.STRUCT, (short)1);
- private static final org.apache.thrift.protocol.TField SDE_FIELD_DESC = new org.apache.thrift.protocol.TField("sde", org.apache.thrift.protocol.TType.STRUCT, (short)2);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new system_add_column_family_resultStandardSchemeFactory());
- schemes.put(TupleScheme.class, new system_add_column_family_resultTupleSchemeFactory());
- }
-
- public String success; // required
- public InvalidRequestException ire; // required
- public SchemaDisagreementException sde; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- SUCCESS((short)0, "success"),
- IRE((short)1, "ire"),
- SDE((short)2, "sde");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 0: // SUCCESS
- return SUCCESS;
- case 1: // IRE
- return IRE;
- case 2: // SDE
- return SDE;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
- tmpMap.put(_Fields.IRE, new org.apache.thrift.meta_data.FieldMetaData("ire", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- tmpMap.put(_Fields.SDE, new org.apache.thrift.meta_data.FieldMetaData("sde", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(system_add_column_family_result.class, metaDataMap);
- }
-
- public system_add_column_family_result() {
- }
-
- public system_add_column_family_result(
- String success,
- InvalidRequestException ire,
- SchemaDisagreementException sde)
- {
- this();
- this.success = success;
- this.ire = ire;
- this.sde = sde;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public system_add_column_family_result(system_add_column_family_result other) {
- if (other.isSetSuccess()) {
- this.success = other.success;
- }
- if (other.isSetIre()) {
- this.ire = new InvalidRequestException(other.ire);
- }
- if (other.isSetSde()) {
- this.sde = new SchemaDisagreementException(other.sde);
- }
- }
-
- public system_add_column_family_result deepCopy() {
- return new system_add_column_family_result(this);
- }
-
- @Override
- public void clear() {
- this.success = null;
- this.ire = null;
- this.sde = null;
- }
-
- public String getSuccess() {
- return this.success;
- }
-
- public system_add_column_family_result setSuccess(String success) {
- this.success = success;
- return this;
- }
-
- public void unsetSuccess() {
- this.success = null;
- }
-
- /** Returns true if field success is set (has been assigned a value) and false otherwise */
- public boolean isSetSuccess() {
- return this.success != null;
- }
-
- public void setSuccessIsSet(boolean value) {
- if (!value) {
- this.success = null;
- }
- }
-
- public InvalidRequestException getIre() {
- return this.ire;
- }
-
- public system_add_column_family_result setIre(InvalidRequestException ire) {
- this.ire = ire;
- return this;
- }
-
- public void unsetIre() {
- this.ire = null;
- }
-
- /** Returns true if field ire is set (has been assigned a value) and false otherwise */
- public boolean isSetIre() {
- return this.ire != null;
- }
-
- public void setIreIsSet(boolean value) {
- if (!value) {
- this.ire = null;
- }
- }
-
- public SchemaDisagreementException getSde() {
- return this.sde;
- }
-
- public system_add_column_family_result setSde(SchemaDisagreementException sde) {
- this.sde = sde;
- return this;
- }
-
- public void unsetSde() {
- this.sde = null;
- }
-
- /** Returns true if field sde is set (has been assigned a value) and false otherwise */
- public boolean isSetSde() {
- return this.sde != null;
- }
-
- public void setSdeIsSet(boolean value) {
- if (!value) {
- this.sde = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case SUCCESS:
- if (value == null) {
- unsetSuccess();
- } else {
- setSuccess((String)value);
- }
- break;
-
- case IRE:
- if (value == null) {
- unsetIre();
- } else {
- setIre((InvalidRequestException)value);
- }
- break;
-
- case SDE:
- if (value == null) {
- unsetSde();
- } else {
- setSde((SchemaDisagreementException)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case SUCCESS:
- return getSuccess();
-
- case IRE:
- return getIre();
-
- case SDE:
- return getSde();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case SUCCESS:
- return isSetSuccess();
- case IRE:
- return isSetIre();
- case SDE:
- return isSetSde();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof system_add_column_family_result)
- return this.equals((system_add_column_family_result)that);
- return false;
- }
-
- public boolean equals(system_add_column_family_result that) {
- if (that == null)
- return false;
-
- boolean this_present_success = true && this.isSetSuccess();
- boolean that_present_success = true && that.isSetSuccess();
- if (this_present_success || that_present_success) {
- if (!(this_present_success && that_present_success))
- return false;
- if (!this.success.equals(that.success))
- return false;
- }
-
- boolean this_present_ire = true && this.isSetIre();
- boolean that_present_ire = true && that.isSetIre();
- if (this_present_ire || that_present_ire) {
- if (!(this_present_ire && that_present_ire))
- return false;
- if (!this.ire.equals(that.ire))
- return false;
- }
-
- boolean this_present_sde = true && this.isSetSde();
- boolean that_present_sde = true && that.isSetSde();
- if (this_present_sde || that_present_sde) {
- if (!(this_present_sde && that_present_sde))
- return false;
- if (!this.sde.equals(that.sde))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_success = true && (isSetSuccess());
- builder.append(present_success);
- if (present_success)
- builder.append(success);
-
- boolean present_ire = true && (isSetIre());
- builder.append(present_ire);
- if (present_ire)
- builder.append(ire);
-
- boolean present_sde = true && (isSetSde());
- builder.append(present_sde);
- if (present_sde)
- builder.append(sde);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(system_add_column_family_result other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(other.isSetSuccess());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetSuccess()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, other.success);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetIre()).compareTo(other.isSetIre());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetIre()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.ire, other.ire);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetSde()).compareTo(other.isSetSde());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetSde()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.sde, other.sde);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("system_add_column_family_result(");
- boolean first = true;
-
- sb.append("success:");
- if (this.success == null) {
- sb.append("null");
- } else {
- sb.append(this.success);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("ire:");
- if (this.ire == null) {
- sb.append("null");
- } else {
- sb.append(this.ire);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("sde:");
- if (this.sde == null) {
- sb.append("null");
- } else {
- sb.append(this.sde);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class system_add_column_family_resultStandardSchemeFactory implements SchemeFactory {
- public system_add_column_family_resultStandardScheme getScheme() {
- return new system_add_column_family_resultStandardScheme();
- }
- }
-
- private static class system_add_column_family_resultStandardScheme extends StandardScheme<system_add_column_family_result> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, system_add_column_family_result struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 0: // SUCCESS
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.success = iprot.readString();
- struct.setSuccessIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 1: // IRE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.ire = new InvalidRequestException();
- struct.ire.read(iprot);
- struct.setIreIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 2: // SDE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.sde = new SchemaDisagreementException();
- struct.sde.read(iprot);
- struct.setSdeIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, system_add_column_family_result struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.success != null) {
- oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
- oprot.writeString(struct.success);
- oprot.writeFieldEnd();
- }
- if (struct.ire != null) {
- oprot.writeFieldBegin(IRE_FIELD_DESC);
- struct.ire.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.sde != null) {
- oprot.writeFieldBegin(SDE_FIELD_DESC);
- struct.sde.write(oprot);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class system_add_column_family_resultTupleSchemeFactory implements SchemeFactory {
- public system_add_column_family_resultTupleScheme getScheme() {
- return new system_add_column_family_resultTupleScheme();
- }
- }
-
- private static class system_add_column_family_resultTupleScheme extends TupleScheme<system_add_column_family_result> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, system_add_column_family_result struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- BitSet optionals = new BitSet();
- if (struct.isSetSuccess()) {
- optionals.set(0);
- }
- if (struct.isSetIre()) {
- optionals.set(1);
- }
- if (struct.isSetSde()) {
- optionals.set(2);
- }
- oprot.writeBitSet(optionals, 3);
- if (struct.isSetSuccess()) {
- oprot.writeString(struct.success);
- }
- if (struct.isSetIre()) {
- struct.ire.write(oprot);
- }
- if (struct.isSetSde()) {
- struct.sde.write(oprot);
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, system_add_column_family_result struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- BitSet incoming = iprot.readBitSet(3);
- if (incoming.get(0)) {
- struct.success = iprot.readString();
- struct.setSuccessIsSet(true);
- }
- if (incoming.get(1)) {
- struct.ire = new InvalidRequestException();
- struct.ire.read(iprot);
- struct.setIreIsSet(true);
- }
- if (incoming.get(2)) {
- struct.sde = new SchemaDisagreementException();
- struct.sde.read(iprot);
- struct.setSdeIsSet(true);
- }
- }
- }
-
- }
-
- public static class system_drop_column_family_args implements org.apache.thrift.TBase<system_drop_column_family_args, system_drop_column_family_args._Fields>, java.io.Serializable, Cloneable, Comparable<system_drop_column_family_args> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("system_drop_column_family_args");
-
- private static final org.apache.thrift.protocol.TField COLUMN_FAMILY_FIELD_DESC = new org.apache.thrift.protocol.TField("column_family", org.apache.thrift.protocol.TType.STRING, (short)1);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new system_drop_column_family_argsStandardSchemeFactory());
- schemes.put(TupleScheme.class, new system_drop_column_family_argsTupleSchemeFactory());
- }
-
- public String column_family; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- COLUMN_FAMILY((short)1, "column_family");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // COLUMN_FAMILY
- return COLUMN_FAMILY;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.COLUMN_FAMILY, new org.apache.thrift.meta_data.FieldMetaData("column_family", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(system_drop_column_family_args.class, metaDataMap);
- }
-
- public system_drop_column_family_args() {
- }
-
- public system_drop_column_family_args(
- String column_family)
- {
- this();
- this.column_family = column_family;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public system_drop_column_family_args(system_drop_column_family_args other) {
- if (other.isSetColumn_family()) {
- this.column_family = other.column_family;
- }
- }
-
- public system_drop_column_family_args deepCopy() {
- return new system_drop_column_family_args(this);
- }
-
- @Override
- public void clear() {
- this.column_family = null;
- }
-
- public String getColumn_family() {
- return this.column_family;
- }
-
- public system_drop_column_family_args setColumn_family(String column_family) {
- this.column_family = column_family;
- return this;
- }
-
- public void unsetColumn_family() {
- this.column_family = null;
- }
-
- /** Returns true if field column_family is set (has been assigned a value) and false otherwise */
- public boolean isSetColumn_family() {
- return this.column_family != null;
- }
-
- public void setColumn_familyIsSet(boolean value) {
- if (!value) {
- this.column_family = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case COLUMN_FAMILY:
- if (value == null) {
- unsetColumn_family();
- } else {
- setColumn_family((String)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case COLUMN_FAMILY:
- return getColumn_family();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case COLUMN_FAMILY:
- return isSetColumn_family();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof system_drop_column_family_args)
- return this.equals((system_drop_column_family_args)that);
- return false;
- }
-
- public boolean equals(system_drop_column_family_args that) {
- if (that == null)
- return false;
-
- boolean this_present_column_family = true && this.isSetColumn_family();
- boolean that_present_column_family = true && that.isSetColumn_family();
- if (this_present_column_family || that_present_column_family) {
- if (!(this_present_column_family && that_present_column_family))
- return false;
- if (!this.column_family.equals(that.column_family))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_column_family = true && (isSetColumn_family());
- builder.append(present_column_family);
- if (present_column_family)
- builder.append(column_family);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(system_drop_column_family_args other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetColumn_family()).compareTo(other.isSetColumn_family());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetColumn_family()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.column_family, other.column_family);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("system_drop_column_family_args(");
- boolean first = true;
-
- sb.append("column_family:");
- if (this.column_family == null) {
- sb.append("null");
- } else {
- sb.append(this.column_family);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- if (column_family == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'column_family' was not present! Struct: " + toString());
- }
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class system_drop_column_family_argsStandardSchemeFactory implements SchemeFactory {
- public system_drop_column_family_argsStandardScheme getScheme() {
- return new system_drop_column_family_argsStandardScheme();
- }
- }
-
- private static class system_drop_column_family_argsStandardScheme extends StandardScheme<system_drop_column_family_args> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, system_drop_column_family_args struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // COLUMN_FAMILY
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.column_family = iprot.readString();
- struct.setColumn_familyIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, system_drop_column_family_args struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.column_family != null) {
- oprot.writeFieldBegin(COLUMN_FAMILY_FIELD_DESC);
- oprot.writeString(struct.column_family);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class system_drop_column_family_argsTupleSchemeFactory implements SchemeFactory {
- public system_drop_column_family_argsTupleScheme getScheme() {
- return new system_drop_column_family_argsTupleScheme();
- }
- }
-
- private static class system_drop_column_family_argsTupleScheme extends TupleScheme<system_drop_column_family_args> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, system_drop_column_family_args struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- oprot.writeString(struct.column_family);
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, system_drop_column_family_args struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- struct.column_family = iprot.readString();
- struct.setColumn_familyIsSet(true);
- }
- }
-
- }
-
- public static class system_drop_column_family_result implements org.apache.thrift.TBase<system_drop_column_family_result, system_drop_column_family_result._Fields>, java.io.Serializable, Cloneable, Comparable<system_drop_column_family_result> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("system_drop_column_family_result");
-
- private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRING, (short)0);
- private static final org.apache.thrift.protocol.TField IRE_FIELD_DESC = new org.apache.thrift.protocol.TField("ire", org.apache.thrift.protocol.TType.STRUCT, (short)1);
- private static final org.apache.thrift.protocol.TField SDE_FIELD_DESC = new org.apache.thrift.protocol.TField("sde", org.apache.thrift.protocol.TType.STRUCT, (short)2);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new system_drop_column_family_resultStandardSchemeFactory());
- schemes.put(TupleScheme.class, new system_drop_column_family_resultTupleSchemeFactory());
- }
-
- public String success; // required
- public InvalidRequestException ire; // required
- public SchemaDisagreementException sde; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- SUCCESS((short)0, "success"),
- IRE((short)1, "ire"),
- SDE((short)2, "sde");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 0: // SUCCESS
- return SUCCESS;
- case 1: // IRE
- return IRE;
- case 2: // SDE
- return SDE;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
- tmpMap.put(_Fields.IRE, new org.apache.thrift.meta_data.FieldMetaData("ire", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- tmpMap.put(_Fields.SDE, new org.apache.thrift.meta_data.FieldMetaData("sde", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(system_drop_column_family_result.class, metaDataMap);
- }
-
- public system_drop_column_family_result() {
- }
-
- public system_drop_column_family_result(
- String success,
- InvalidRequestException ire,
- SchemaDisagreementException sde)
- {
- this();
- this.success = success;
- this.ire = ire;
- this.sde = sde;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public system_drop_column_family_result(system_drop_column_family_result other) {
- if (other.isSetSuccess()) {
- this.success = other.success;
- }
- if (other.isSetIre()) {
- this.ire = new InvalidRequestException(other.ire);
- }
- if (other.isSetSde()) {
- this.sde = new SchemaDisagreementException(other.sde);
- }
- }
-
- public system_drop_column_family_result deepCopy() {
- return new system_drop_column_family_result(this);
- }
-
- @Override
- public void clear() {
- this.success = null;
- this.ire = null;
- this.sde = null;
- }
-
- public String getSuccess() {
- return this.success;
- }
-
- public system_drop_column_family_result setSuccess(String success) {
- this.success = success;
- return this;
- }
-
- public void unsetSuccess() {
- this.success = null;
- }
-
- /** Returns true if field success is set (has been assigned a value) and false otherwise */
- public boolean isSetSuccess() {
- return this.success != null;
- }
-
- public void setSuccessIsSet(boolean value) {
- if (!value) {
- this.success = null;
- }
- }
-
- public InvalidRequestException getIre() {
- return this.ire;
- }
-
- public system_drop_column_family_result setIre(InvalidRequestException ire) {
- this.ire = ire;
- return this;
- }
-
- public void unsetIre() {
- this.ire = null;
- }
-
- /** Returns true if field ire is set (has been assigned a value) and false otherwise */
- public boolean isSetIre() {
- return this.ire != null;
- }
-
- public void setIreIsSet(boolean value) {
- if (!value) {
- this.ire = null;
- }
- }
-
- public SchemaDisagreementException getSde() {
- return this.sde;
- }
-
- public system_drop_column_family_result setSde(SchemaDisagreementException sde) {
- this.sde = sde;
- return this;
- }
-
- public void unsetSde() {
- this.sde = null;
- }
-
- /** Returns true if field sde is set (has been assigned a value) and false otherwise */
- public boolean isSetSde() {
- return this.sde != null;
- }
-
- public void setSdeIsSet(boolean value) {
- if (!value) {
- this.sde = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case SUCCESS:
- if (value == null) {
- unsetSuccess();
- } else {
- setSuccess((String)value);
- }
- break;
-
- case IRE:
- if (value == null) {
- unsetIre();
- } else {
- setIre((InvalidRequestException)value);
- }
- break;
-
- case SDE:
- if (value == null) {
- unsetSde();
- } else {
- setSde((SchemaDisagreementException)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case SUCCESS:
- return getSuccess();
-
- case IRE:
- return getIre();
-
- case SDE:
- return getSde();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case SUCCESS:
- return isSetSuccess();
- case IRE:
- return isSetIre();
- case SDE:
- return isSetSde();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof system_drop_column_family_result)
- return this.equals((system_drop_column_family_result)that);
- return false;
- }
-
- public boolean equals(system_drop_column_family_result that) {
- if (that == null)
- return false;
-
- boolean this_present_success = true && this.isSetSuccess();
- boolean that_present_success = true && that.isSetSuccess();
- if (this_present_success || that_present_success) {
- if (!(this_present_success && that_present_success))
- return false;
- if (!this.success.equals(that.success))
- return false;
- }
-
- boolean this_present_ire = true && this.isSetIre();
- boolean that_present_ire = true && that.isSetIre();
- if (this_present_ire || that_present_ire) {
- if (!(this_present_ire && that_present_ire))
- return false;
- if (!this.ire.equals(that.ire))
- return false;
- }
-
- boolean this_present_sde = true && this.isSetSde();
- boolean that_present_sde = true && that.isSetSde();
- if (this_present_sde || that_present_sde) {
- if (!(this_present_sde && that_present_sde))
- return false;
- if (!this.sde.equals(that.sde))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_success = true && (isSetSuccess());
- builder.append(present_success);
- if (present_success)
- builder.append(success);
-
- boolean present_ire = true && (isSetIre());
- builder.append(present_ire);
- if (present_ire)
- builder.append(ire);
-
- boolean present_sde = true && (isSetSde());
- builder.append(present_sde);
- if (present_sde)
- builder.append(sde);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(system_drop_column_family_result other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(other.isSetSuccess());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetSuccess()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, other.success);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetIre()).compareTo(other.isSetIre());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetIre()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.ire, other.ire);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetSde()).compareTo(other.isSetSde());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetSde()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.sde, other.sde);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("system_drop_column_family_result(");
- boolean first = true;
-
- sb.append("success:");
- if (this.success == null) {
- sb.append("null");
- } else {
- sb.append(this.success);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("ire:");
- if (this.ire == null) {
- sb.append("null");
- } else {
- sb.append(this.ire);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("sde:");
- if (this.sde == null) {
- sb.append("null");
- } else {
- sb.append(this.sde);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class system_drop_column_family_resultStandardSchemeFactory implements SchemeFactory {
- public system_drop_column_family_resultStandardScheme getScheme() {
- return new system_drop_column_family_resultStandardScheme();
- }
- }
-
- private static class system_drop_column_family_resultStandardScheme extends StandardScheme<system_drop_column_family_result> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, system_drop_column_family_result struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 0: // SUCCESS
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.success = iprot.readString();
- struct.setSuccessIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 1: // IRE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.ire = new InvalidRequestException();
- struct.ire.read(iprot);
- struct.setIreIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 2: // SDE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.sde = new SchemaDisagreementException();
- struct.sde.read(iprot);
- struct.setSdeIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, system_drop_column_family_result struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.success != null) {
- oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
- oprot.writeString(struct.success);
- oprot.writeFieldEnd();
- }
- if (struct.ire != null) {
- oprot.writeFieldBegin(IRE_FIELD_DESC);
- struct.ire.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.sde != null) {
- oprot.writeFieldBegin(SDE_FIELD_DESC);
- struct.sde.write(oprot);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class system_drop_column_family_resultTupleSchemeFactory implements SchemeFactory {
- public system_drop_column_family_resultTupleScheme getScheme() {
- return new system_drop_column_family_resultTupleScheme();
- }
- }
-
- private static class system_drop_column_family_resultTupleScheme extends TupleScheme<system_drop_column_family_result> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, system_drop_column_family_result struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- BitSet optionals = new BitSet();
- if (struct.isSetSuccess()) {
- optionals.set(0);
- }
- if (struct.isSetIre()) {
- optionals.set(1);
- }
- if (struct.isSetSde()) {
- optionals.set(2);
- }
- oprot.writeBitSet(optionals, 3);
- if (struct.isSetSuccess()) {
- oprot.writeString(struct.success);
- }
- if (struct.isSetIre()) {
- struct.ire.write(oprot);
- }
- if (struct.isSetSde()) {
- struct.sde.write(oprot);
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, system_drop_column_family_result struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- BitSet incoming = iprot.readBitSet(3);
- if (incoming.get(0)) {
- struct.success = iprot.readString();
- struct.setSuccessIsSet(true);
- }
- if (incoming.get(1)) {
- struct.ire = new InvalidRequestException();
- struct.ire.read(iprot);
- struct.setIreIsSet(true);
- }
- if (incoming.get(2)) {
- struct.sde = new SchemaDisagreementException();
- struct.sde.read(iprot);
- struct.setSdeIsSet(true);
- }
- }
- }
-
- }
-
- public static class system_add_keyspace_args implements org.apache.thrift.TBase<system_add_keyspace_args, system_add_keyspace_args._Fields>, java.io.Serializable, Cloneable, Comparable<system_add_keyspace_args> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("system_add_keyspace_args");
-
- private static final org.apache.thrift.protocol.TField KS_DEF_FIELD_DESC = new org.apache.thrift.protocol.TField("ks_def", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new system_add_keyspace_argsStandardSchemeFactory());
- schemes.put(TupleScheme.class, new system_add_keyspace_argsTupleSchemeFactory());
- }
-
- public KsDef ks_def; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- KS_DEF((short)1, "ks_def");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // KS_DEF
- return KS_DEF;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.KS_DEF, new org.apache.thrift.meta_data.FieldMetaData("ks_def", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, KsDef.class)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(system_add_keyspace_args.class, metaDataMap);
- }
-
- public system_add_keyspace_args() {
- }
-
- public system_add_keyspace_args(
- KsDef ks_def)
- {
- this();
- this.ks_def = ks_def;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public system_add_keyspace_args(system_add_keyspace_args other) {
- if (other.isSetKs_def()) {
- this.ks_def = new KsDef(other.ks_def);
- }
- }
-
- public system_add_keyspace_args deepCopy() {
- return new system_add_keyspace_args(this);
- }
-
- @Override
- public void clear() {
- this.ks_def = null;
- }
-
- public KsDef getKs_def() {
- return this.ks_def;
- }
-
- public system_add_keyspace_args setKs_def(KsDef ks_def) {
- this.ks_def = ks_def;
- return this;
- }
-
- public void unsetKs_def() {
- this.ks_def = null;
- }
-
- /** Returns true if field ks_def is set (has been assigned a value) and false otherwise */
- public boolean isSetKs_def() {
- return this.ks_def != null;
- }
-
- public void setKs_defIsSet(boolean value) {
- if (!value) {
- this.ks_def = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case KS_DEF:
- if (value == null) {
- unsetKs_def();
- } else {
- setKs_def((KsDef)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case KS_DEF:
- return getKs_def();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case KS_DEF:
- return isSetKs_def();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof system_add_keyspace_args)
- return this.equals((system_add_keyspace_args)that);
- return false;
- }
-
- public boolean equals(system_add_keyspace_args that) {
- if (that == null)
- return false;
-
- boolean this_present_ks_def = true && this.isSetKs_def();
- boolean that_present_ks_def = true && that.isSetKs_def();
- if (this_present_ks_def || that_present_ks_def) {
- if (!(this_present_ks_def && that_present_ks_def))
- return false;
- if (!this.ks_def.equals(that.ks_def))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_ks_def = true && (isSetKs_def());
- builder.append(present_ks_def);
- if (present_ks_def)
- builder.append(ks_def);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(system_add_keyspace_args other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetKs_def()).compareTo(other.isSetKs_def());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetKs_def()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.ks_def, other.ks_def);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("system_add_keyspace_args(");
- boolean first = true;
-
- sb.append("ks_def:");
- if (this.ks_def == null) {
- sb.append("null");
- } else {
- sb.append(this.ks_def);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- if (ks_def == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'ks_def' was not present! Struct: " + toString());
- }
- // check for sub-struct validity
- if (ks_def != null) {
- ks_def.validate();
- }
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class system_add_keyspace_argsStandardSchemeFactory implements SchemeFactory {
- public system_add_keyspace_argsStandardScheme getScheme() {
- return new system_add_keyspace_argsStandardScheme();
- }
- }
-
- private static class system_add_keyspace_argsStandardScheme extends StandardScheme<system_add_keyspace_args> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, system_add_keyspace_args struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // KS_DEF
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.ks_def = new KsDef();
- struct.ks_def.read(iprot);
- struct.setKs_defIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, system_add_keyspace_args struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.ks_def != null) {
- oprot.writeFieldBegin(KS_DEF_FIELD_DESC);
- struct.ks_def.write(oprot);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class system_add_keyspace_argsTupleSchemeFactory implements SchemeFactory {
- public system_add_keyspace_argsTupleScheme getScheme() {
- return new system_add_keyspace_argsTupleScheme();
- }
- }
-
- private static class system_add_keyspace_argsTupleScheme extends TupleScheme<system_add_keyspace_args> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, system_add_keyspace_args struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- struct.ks_def.write(oprot);
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, system_add_keyspace_args struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- struct.ks_def = new KsDef();
- struct.ks_def.read(iprot);
- struct.setKs_defIsSet(true);
- }
- }
-
- }
-
- public static class system_add_keyspace_result implements org.apache.thrift.TBase<system_add_keyspace_result, system_add_keyspace_result._Fields>, java.io.Serializable, Cloneable, Comparable<system_add_keyspace_result> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("system_add_keyspace_result");
-
- private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRING, (short)0);
- private static final org.apache.thrift.protocol.TField IRE_FIELD_DESC = new org.apache.thrift.protocol.TField("ire", org.apache.thrift.protocol.TType.STRUCT, (short)1);
- private static final org.apache.thrift.protocol.TField SDE_FIELD_DESC = new org.apache.thrift.protocol.TField("sde", org.apache.thrift.protocol.TType.STRUCT, (short)2);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new system_add_keyspace_resultStandardSchemeFactory());
- schemes.put(TupleScheme.class, new system_add_keyspace_resultTupleSchemeFactory());
- }
-
- public String success; // required
- public InvalidRequestException ire; // required
- public SchemaDisagreementException sde; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- SUCCESS((short)0, "success"),
- IRE((short)1, "ire"),
- SDE((short)2, "sde");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 0: // SUCCESS
- return SUCCESS;
- case 1: // IRE
- return IRE;
- case 2: // SDE
- return SDE;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
- tmpMap.put(_Fields.IRE, new org.apache.thrift.meta_data.FieldMetaData("ire", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- tmpMap.put(_Fields.SDE, new org.apache.thrift.meta_data.FieldMetaData("sde", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(system_add_keyspace_result.class, metaDataMap);
- }
-
- public system_add_keyspace_result() {
- }
-
- public system_add_keyspace_result(
- String success,
- InvalidRequestException ire,
- SchemaDisagreementException sde)
- {
- this();
- this.success = success;
- this.ire = ire;
- this.sde = sde;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public system_add_keyspace_result(system_add_keyspace_result other) {
- if (other.isSetSuccess()) {
- this.success = other.success;
- }
- if (other.isSetIre()) {
- this.ire = new InvalidRequestException(other.ire);
- }
- if (other.isSetSde()) {
- this.sde = new SchemaDisagreementException(other.sde);
- }
- }
-
- public system_add_keyspace_result deepCopy() {
- return new system_add_keyspace_result(this);
- }
-
- @Override
- public void clear() {
- this.success = null;
- this.ire = null;
- this.sde = null;
- }
-
- public String getSuccess() {
- return this.success;
- }
-
- public system_add_keyspace_result setSuccess(String success) {
- this.success = success;
- return this;
- }
-
- public void unsetSuccess() {
- this.success = null;
- }
-
- /** Returns true if field success is set (has been assigned a value) and false otherwise */
- public boolean isSetSuccess() {
- return this.success != null;
- }
-
- public void setSuccessIsSet(boolean value) {
- if (!value) {
- this.success = null;
- }
- }
-
- public InvalidRequestException getIre() {
- return this.ire;
- }
-
- public system_add_keyspace_result setIre(InvalidRequestException ire) {
- this.ire = ire;
- return this;
- }
-
- public void unsetIre() {
- this.ire = null;
- }
-
- /** Returns true if field ire is set (has been assigned a value) and false otherwise */
- public boolean isSetIre() {
- return this.ire != null;
- }
-
- public void setIreIsSet(boolean value) {
- if (!value) {
- this.ire = null;
- }
- }
-
- public SchemaDisagreementException getSde() {
- return this.sde;
- }
-
- public system_add_keyspace_result setSde(SchemaDisagreementException sde) {
- this.sde = sde;
- return this;
- }
-
- public void unsetSde() {
- this.sde = null;
- }
-
- /** Returns true if field sde is set (has been assigned a value) and false otherwise */
- public boolean isSetSde() {
- return this.sde != null;
- }
-
- public void setSdeIsSet(boolean value) {
- if (!value) {
- this.sde = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case SUCCESS:
- if (value == null) {
- unsetSuccess();
- } else {
- setSuccess((String)value);
- }
- break;
-
- case IRE:
- if (value == null) {
- unsetIre();
- } else {
- setIre((InvalidRequestException)value);
- }
- break;
-
- case SDE:
- if (value == null) {
- unsetSde();
- } else {
- setSde((SchemaDisagreementException)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case SUCCESS:
- return getSuccess();
-
- case IRE:
- return getIre();
-
- case SDE:
- return getSde();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case SUCCESS:
- return isSetSuccess();
- case IRE:
- return isSetIre();
- case SDE:
- return isSetSde();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof system_add_keyspace_result)
- return this.equals((system_add_keyspace_result)that);
- return false;
- }
-
- public boolean equals(system_add_keyspace_result that) {
- if (that == null)
- return false;
-
- boolean this_present_success = true && this.isSetSuccess();
- boolean that_present_success = true && that.isSetSuccess();
- if (this_present_success || that_present_success) {
- if (!(this_present_success && that_present_success))
- return false;
- if (!this.success.equals(that.success))
- return false;
- }
-
- boolean this_present_ire = true && this.isSetIre();
- boolean that_present_ire = true && that.isSetIre();
- if (this_present_ire || that_present_ire) {
- if (!(this_present_ire && that_present_ire))
- return false;
- if (!this.ire.equals(that.ire))
- return false;
- }
-
- boolean this_present_sde = true && this.isSetSde();
- boolean that_present_sde = true && that.isSetSde();
- if (this_present_sde || that_present_sde) {
- if (!(this_present_sde && that_present_sde))
- return false;
- if (!this.sde.equals(that.sde))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_success = true && (isSetSuccess());
- builder.append(present_success);
- if (present_success)
- builder.append(success);
-
- boolean present_ire = true && (isSetIre());
- builder.append(present_ire);
- if (present_ire)
- builder.append(ire);
-
- boolean present_sde = true && (isSetSde());
- builder.append(present_sde);
- if (present_sde)
- builder.append(sde);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(system_add_keyspace_result other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(other.isSetSuccess());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetSuccess()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, other.success);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetIre()).compareTo(other.isSetIre());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetIre()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.ire, other.ire);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetSde()).compareTo(other.isSetSde());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetSde()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.sde, other.sde);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("system_add_keyspace_result(");
- boolean first = true;
-
- sb.append("success:");
- if (this.success == null) {
- sb.append("null");
- } else {
- sb.append(this.success);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("ire:");
- if (this.ire == null) {
- sb.append("null");
- } else {
- sb.append(this.ire);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("sde:");
- if (this.sde == null) {
- sb.append("null");
- } else {
- sb.append(this.sde);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class system_add_keyspace_resultStandardSchemeFactory implements SchemeFactory {
- public system_add_keyspace_resultStandardScheme getScheme() {
- return new system_add_keyspace_resultStandardScheme();
- }
- }
-
- private static class system_add_keyspace_resultStandardScheme extends StandardScheme<system_add_keyspace_result> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, system_add_keyspace_result struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 0: // SUCCESS
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.success = iprot.readString();
- struct.setSuccessIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 1: // IRE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.ire = new InvalidRequestException();
- struct.ire.read(iprot);
- struct.setIreIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 2: // SDE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.sde = new SchemaDisagreementException();
- struct.sde.read(iprot);
- struct.setSdeIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, system_add_keyspace_result struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.success != null) {
- oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
- oprot.writeString(struct.success);
- oprot.writeFieldEnd();
- }
- if (struct.ire != null) {
- oprot.writeFieldBegin(IRE_FIELD_DESC);
- struct.ire.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.sde != null) {
- oprot.writeFieldBegin(SDE_FIELD_DESC);
- struct.sde.write(oprot);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class system_add_keyspace_resultTupleSchemeFactory implements SchemeFactory {
- public system_add_keyspace_resultTupleScheme getScheme() {
- return new system_add_keyspace_resultTupleScheme();
- }
- }
-
- private static class system_add_keyspace_resultTupleScheme extends TupleScheme<system_add_keyspace_result> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, system_add_keyspace_result struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- BitSet optionals = new BitSet();
- if (struct.isSetSuccess()) {
- optionals.set(0);
- }
- if (struct.isSetIre()) {
- optionals.set(1);
- }
- if (struct.isSetSde()) {
- optionals.set(2);
- }
- oprot.writeBitSet(optionals, 3);
- if (struct.isSetSuccess()) {
- oprot.writeString(struct.success);
- }
- if (struct.isSetIre()) {
- struct.ire.write(oprot);
- }
- if (struct.isSetSde()) {
- struct.sde.write(oprot);
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, system_add_keyspace_result struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- BitSet incoming = iprot.readBitSet(3);
- if (incoming.get(0)) {
- struct.success = iprot.readString();
- struct.setSuccessIsSet(true);
- }
- if (incoming.get(1)) {
- struct.ire = new InvalidRequestException();
- struct.ire.read(iprot);
- struct.setIreIsSet(true);
- }
- if (incoming.get(2)) {
- struct.sde = new SchemaDisagreementException();
- struct.sde.read(iprot);
- struct.setSdeIsSet(true);
- }
- }
- }
-
- }
-
- public static class system_drop_keyspace_args implements org.apache.thrift.TBase<system_drop_keyspace_args, system_drop_keyspace_args._Fields>, java.io.Serializable, Cloneable, Comparable<system_drop_keyspace_args> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("system_drop_keyspace_args");
-
- private static final org.apache.thrift.protocol.TField KEYSPACE_FIELD_DESC = new org.apache.thrift.protocol.TField("keyspace", org.apache.thrift.protocol.TType.STRING, (short)1);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new system_drop_keyspace_argsStandardSchemeFactory());
- schemes.put(TupleScheme.class, new system_drop_keyspace_argsTupleSchemeFactory());
- }
-
- public String keyspace; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- KEYSPACE((short)1, "keyspace");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // KEYSPACE
- return KEYSPACE;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.KEYSPACE, new org.apache.thrift.meta_data.FieldMetaData("keyspace", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(system_drop_keyspace_args.class, metaDataMap);
- }
-
- public system_drop_keyspace_args() {
- }
-
- public system_drop_keyspace_args(
- String keyspace)
- {
- this();
- this.keyspace = keyspace;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public system_drop_keyspace_args(system_drop_keyspace_args other) {
- if (other.isSetKeyspace()) {
- this.keyspace = other.keyspace;
- }
- }
-
- public system_drop_keyspace_args deepCopy() {
- return new system_drop_keyspace_args(this);
- }
-
- @Override
- public void clear() {
- this.keyspace = null;
- }
-
- public String getKeyspace() {
- return this.keyspace;
- }
-
- public system_drop_keyspace_args setKeyspace(String keyspace) {
- this.keyspace = keyspace;
- return this;
- }
-
- public void unsetKeyspace() {
- this.keyspace = null;
- }
-
- /** Returns true if field keyspace is set (has been assigned a value) and false otherwise */
- public boolean isSetKeyspace() {
- return this.keyspace != null;
- }
-
- public void setKeyspaceIsSet(boolean value) {
- if (!value) {
- this.keyspace = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case KEYSPACE:
- if (value == null) {
- unsetKeyspace();
- } else {
- setKeyspace((String)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case KEYSPACE:
- return getKeyspace();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case KEYSPACE:
- return isSetKeyspace();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof system_drop_keyspace_args)
- return this.equals((system_drop_keyspace_args)that);
- return false;
- }
-
- public boolean equals(system_drop_keyspace_args that) {
- if (that == null)
- return false;
-
- boolean this_present_keyspace = true && this.isSetKeyspace();
- boolean that_present_keyspace = true && that.isSetKeyspace();
- if (this_present_keyspace || that_present_keyspace) {
- if (!(this_present_keyspace && that_present_keyspace))
- return false;
- if (!this.keyspace.equals(that.keyspace))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_keyspace = true && (isSetKeyspace());
- builder.append(present_keyspace);
- if (present_keyspace)
- builder.append(keyspace);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(system_drop_keyspace_args other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetKeyspace()).compareTo(other.isSetKeyspace());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetKeyspace()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.keyspace, other.keyspace);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("system_drop_keyspace_args(");
- boolean first = true;
-
- sb.append("keyspace:");
- if (this.keyspace == null) {
- sb.append("null");
- } else {
- sb.append(this.keyspace);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- if (keyspace == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'keyspace' was not present! Struct: " + toString());
- }
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class system_drop_keyspace_argsStandardSchemeFactory implements SchemeFactory {
- public system_drop_keyspace_argsStandardScheme getScheme() {
- return new system_drop_keyspace_argsStandardScheme();
- }
- }
-
- private static class system_drop_keyspace_argsStandardScheme extends StandardScheme<system_drop_keyspace_args> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, system_drop_keyspace_args struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // KEYSPACE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.keyspace = iprot.readString();
- struct.setKeyspaceIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, system_drop_keyspace_args struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.keyspace != null) {
- oprot.writeFieldBegin(KEYSPACE_FIELD_DESC);
- oprot.writeString(struct.keyspace);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class system_drop_keyspace_argsTupleSchemeFactory implements SchemeFactory {
- public system_drop_keyspace_argsTupleScheme getScheme() {
- return new system_drop_keyspace_argsTupleScheme();
- }
- }
-
- private static class system_drop_keyspace_argsTupleScheme extends TupleScheme<system_drop_keyspace_args> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, system_drop_keyspace_args struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- oprot.writeString(struct.keyspace);
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, system_drop_keyspace_args struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- struct.keyspace = iprot.readString();
- struct.setKeyspaceIsSet(true);
- }
- }
-
- }
-
- public static class system_drop_keyspace_result implements org.apache.thrift.TBase<system_drop_keyspace_result, system_drop_keyspace_result._Fields>, java.io.Serializable, Cloneable, Comparable<system_drop_keyspace_result> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("system_drop_keyspace_result");
-
- private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRING, (short)0);
- private static final org.apache.thrift.protocol.TField IRE_FIELD_DESC = new org.apache.thrift.protocol.TField("ire", org.apache.thrift.protocol.TType.STRUCT, (short)1);
- private static final org.apache.thrift.protocol.TField SDE_FIELD_DESC = new org.apache.thrift.protocol.TField("sde", org.apache.thrift.protocol.TType.STRUCT, (short)2);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new system_drop_keyspace_resultStandardSchemeFactory());
- schemes.put(TupleScheme.class, new system_drop_keyspace_resultTupleSchemeFactory());
- }
-
- public String success; // required
- public InvalidRequestException ire; // required
- public SchemaDisagreementException sde; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- SUCCESS((short)0, "success"),
- IRE((short)1, "ire"),
- SDE((short)2, "sde");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 0: // SUCCESS
- return SUCCESS;
- case 1: // IRE
- return IRE;
- case 2: // SDE
- return SDE;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
- tmpMap.put(_Fields.IRE, new org.apache.thrift.meta_data.FieldMetaData("ire", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- tmpMap.put(_Fields.SDE, new org.apache.thrift.meta_data.FieldMetaData("sde", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(system_drop_keyspace_result.class, metaDataMap);
- }
-
- public system_drop_keyspace_result() {
- }
-
- public system_drop_keyspace_result(
- String success,
- InvalidRequestException ire,
- SchemaDisagreementException sde)
- {
- this();
- this.success = success;
- this.ire = ire;
- this.sde = sde;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public system_drop_keyspace_result(system_drop_keyspace_result other) {
- if (other.isSetSuccess()) {
- this.success = other.success;
- }
- if (other.isSetIre()) {
- this.ire = new InvalidRequestException(other.ire);
- }
- if (other.isSetSde()) {
- this.sde = new SchemaDisagreementException(other.sde);
- }
- }
-
- public system_drop_keyspace_result deepCopy() {
- return new system_drop_keyspace_result(this);
- }
-
- @Override
- public void clear() {
- this.success = null;
- this.ire = null;
- this.sde = null;
- }
-
- public String getSuccess() {
- return this.success;
- }
-
- public system_drop_keyspace_result setSuccess(String success) {
- this.success = success;
- return this;
- }
-
- public void unsetSuccess() {
- this.success = null;
- }
-
- /** Returns true if field success is set (has been assigned a value) and false otherwise */
- public boolean isSetSuccess() {
- return this.success != null;
- }
-
- public void setSuccessIsSet(boolean value) {
- if (!value) {
- this.success = null;
- }
- }
-
- public InvalidRequestException getIre() {
- return this.ire;
- }
-
- public system_drop_keyspace_result setIre(InvalidRequestException ire) {
- this.ire = ire;
- return this;
- }
-
- public void unsetIre() {
- this.ire = null;
- }
-
- /** Returns true if field ire is set (has been assigned a value) and false otherwise */
- public boolean isSetIre() {
- return this.ire != null;
- }
-
- public void setIreIsSet(boolean value) {
- if (!value) {
- this.ire = null;
- }
- }
-
- public SchemaDisagreementException getSde() {
- return this.sde;
- }
-
- public system_drop_keyspace_result setSde(SchemaDisagreementException sde) {
- this.sde = sde;
- return this;
- }
-
- public void unsetSde() {
- this.sde = null;
- }
-
- /** Returns true if field sde is set (has been assigned a value) and false otherwise */
- public boolean isSetSde() {
- return this.sde != null;
- }
-
- public void setSdeIsSet(boolean value) {
- if (!value) {
- this.sde = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case SUCCESS:
- if (value == null) {
- unsetSuccess();
- } else {
- setSuccess((String)value);
- }
- break;
-
- case IRE:
- if (value == null) {
- unsetIre();
- } else {
- setIre((InvalidRequestException)value);
- }
- break;
-
- case SDE:
- if (value == null) {
- unsetSde();
- } else {
- setSde((SchemaDisagreementException)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case SUCCESS:
- return getSuccess();
-
- case IRE:
- return getIre();
-
- case SDE:
- return getSde();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case SUCCESS:
- return isSetSuccess();
- case IRE:
- return isSetIre();
- case SDE:
- return isSetSde();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof system_drop_keyspace_result)
- return this.equals((system_drop_keyspace_result)that);
- return false;
- }
-
- public boolean equals(system_drop_keyspace_result that) {
- if (that == null)
- return false;
-
- boolean this_present_success = true && this.isSetSuccess();
- boolean that_present_success = true && that.isSetSuccess();
- if (this_present_success || that_present_success) {
- if (!(this_present_success && that_present_success))
- return false;
- if (!this.success.equals(that.success))
- return false;
- }
-
- boolean this_present_ire = true && this.isSetIre();
- boolean that_present_ire = true && that.isSetIre();
- if (this_present_ire || that_present_ire) {
- if (!(this_present_ire && that_present_ire))
- return false;
- if (!this.ire.equals(that.ire))
- return false;
- }
-
- boolean this_present_sde = true && this.isSetSde();
- boolean that_present_sde = true && that.isSetSde();
- if (this_present_sde || that_present_sde) {
- if (!(this_present_sde && that_present_sde))
- return false;
- if (!this.sde.equals(that.sde))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_success = true && (isSetSuccess());
- builder.append(present_success);
- if (present_success)
- builder.append(success);
-
- boolean present_ire = true && (isSetIre());
- builder.append(present_ire);
- if (present_ire)
- builder.append(ire);
-
- boolean present_sde = true && (isSetSde());
- builder.append(present_sde);
- if (present_sde)
- builder.append(sde);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(system_drop_keyspace_result other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(other.isSetSuccess());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetSuccess()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, other.success);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetIre()).compareTo(other.isSetIre());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetIre()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.ire, other.ire);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetSde()).compareTo(other.isSetSde());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetSde()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.sde, other.sde);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("system_drop_keyspace_result(");
- boolean first = true;
-
- sb.append("success:");
- if (this.success == null) {
- sb.append("null");
- } else {
- sb.append(this.success);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("ire:");
- if (this.ire == null) {
- sb.append("null");
- } else {
- sb.append(this.ire);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("sde:");
- if (this.sde == null) {
- sb.append("null");
- } else {
- sb.append(this.sde);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class system_drop_keyspace_resultStandardSchemeFactory implements SchemeFactory {
- public system_drop_keyspace_resultStandardScheme getScheme() {
- return new system_drop_keyspace_resultStandardScheme();
- }
- }
-
- private static class system_drop_keyspace_resultStandardScheme extends StandardScheme<system_drop_keyspace_result> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, system_drop_keyspace_result struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 0: // SUCCESS
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.success = iprot.readString();
- struct.setSuccessIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 1: // IRE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.ire = new InvalidRequestException();
- struct.ire.read(iprot);
- struct.setIreIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 2: // SDE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.sde = new SchemaDisagreementException();
- struct.sde.read(iprot);
- struct.setSdeIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, system_drop_keyspace_result struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.success != null) {
- oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
- oprot.writeString(struct.success);
- oprot.writeFieldEnd();
- }
- if (struct.ire != null) {
- oprot.writeFieldBegin(IRE_FIELD_DESC);
- struct.ire.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.sde != null) {
- oprot.writeFieldBegin(SDE_FIELD_DESC);
- struct.sde.write(oprot);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class system_drop_keyspace_resultTupleSchemeFactory implements SchemeFactory {
- public system_drop_keyspace_resultTupleScheme getScheme() {
- return new system_drop_keyspace_resultTupleScheme();
- }
- }
-
- private static class system_drop_keyspace_resultTupleScheme extends TupleScheme<system_drop_keyspace_result> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, system_drop_keyspace_result struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- BitSet optionals = new BitSet();
- if (struct.isSetSuccess()) {
- optionals.set(0);
- }
- if (struct.isSetIre()) {
- optionals.set(1);
- }
- if (struct.isSetSde()) {
- optionals.set(2);
- }
- oprot.writeBitSet(optionals, 3);
- if (struct.isSetSuccess()) {
- oprot.writeString(struct.success);
- }
- if (struct.isSetIre()) {
- struct.ire.write(oprot);
- }
- if (struct.isSetSde()) {
- struct.sde.write(oprot);
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, system_drop_keyspace_result struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- BitSet incoming = iprot.readBitSet(3);
- if (incoming.get(0)) {
- struct.success = iprot.readString();
- struct.setSuccessIsSet(true);
- }
- if (incoming.get(1)) {
- struct.ire = new InvalidRequestException();
- struct.ire.read(iprot);
- struct.setIreIsSet(true);
- }
- if (incoming.get(2)) {
- struct.sde = new SchemaDisagreementException();
- struct.sde.read(iprot);
- struct.setSdeIsSet(true);
- }
- }
- }
-
- }
-
- public static class system_update_keyspace_args implements org.apache.thrift.TBase<system_update_keyspace_args, system_update_keyspace_args._Fields>, java.io.Serializable, Cloneable, Comparable<system_update_keyspace_args> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("system_update_keyspace_args");
-
- private static final org.apache.thrift.protocol.TField KS_DEF_FIELD_DESC = new org.apache.thrift.protocol.TField("ks_def", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new system_update_keyspace_argsStandardSchemeFactory());
- schemes.put(TupleScheme.class, new system_update_keyspace_argsTupleSchemeFactory());
- }
-
- public KsDef ks_def; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- KS_DEF((short)1, "ks_def");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // KS_DEF
- return KS_DEF;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.KS_DEF, new org.apache.thrift.meta_data.FieldMetaData("ks_def", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, KsDef.class)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(system_update_keyspace_args.class, metaDataMap);
- }
-
- public system_update_keyspace_args() {
- }
-
- public system_update_keyspace_args(
- KsDef ks_def)
- {
- this();
- this.ks_def = ks_def;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public system_update_keyspace_args(system_update_keyspace_args other) {
- if (other.isSetKs_def()) {
- this.ks_def = new KsDef(other.ks_def);
- }
- }
-
- public system_update_keyspace_args deepCopy() {
- return new system_update_keyspace_args(this);
- }
-
- @Override
- public void clear() {
- this.ks_def = null;
- }
-
- public KsDef getKs_def() {
- return this.ks_def;
- }
-
- public system_update_keyspace_args setKs_def(KsDef ks_def) {
- this.ks_def = ks_def;
- return this;
- }
-
- public void unsetKs_def() {
- this.ks_def = null;
- }
-
- /** Returns true if field ks_def is set (has been assigned a value) and false otherwise */
- public boolean isSetKs_def() {
- return this.ks_def != null;
- }
-
- public void setKs_defIsSet(boolean value) {
- if (!value) {
- this.ks_def = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case KS_DEF:
- if (value == null) {
- unsetKs_def();
- } else {
- setKs_def((KsDef)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case KS_DEF:
- return getKs_def();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case KS_DEF:
- return isSetKs_def();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof system_update_keyspace_args)
- return this.equals((system_update_keyspace_args)that);
- return false;
- }
-
- public boolean equals(system_update_keyspace_args that) {
- if (that == null)
- return false;
-
- boolean this_present_ks_def = true && this.isSetKs_def();
- boolean that_present_ks_def = true && that.isSetKs_def();
- if (this_present_ks_def || that_present_ks_def) {
- if (!(this_present_ks_def && that_present_ks_def))
- return false;
- if (!this.ks_def.equals(that.ks_def))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_ks_def = true && (isSetKs_def());
- builder.append(present_ks_def);
- if (present_ks_def)
- builder.append(ks_def);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(system_update_keyspace_args other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetKs_def()).compareTo(other.isSetKs_def());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetKs_def()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.ks_def, other.ks_def);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("system_update_keyspace_args(");
- boolean first = true;
-
- sb.append("ks_def:");
- if (this.ks_def == null) {
- sb.append("null");
- } else {
- sb.append(this.ks_def);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- if (ks_def == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'ks_def' was not present! Struct: " + toString());
- }
- // check for sub-struct validity
- if (ks_def != null) {
- ks_def.validate();
- }
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class system_update_keyspace_argsStandardSchemeFactory implements SchemeFactory {
- public system_update_keyspace_argsStandardScheme getScheme() {
- return new system_update_keyspace_argsStandardScheme();
- }
- }
-
- private static class system_update_keyspace_argsStandardScheme extends StandardScheme<system_update_keyspace_args> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, system_update_keyspace_args struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // KS_DEF
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.ks_def = new KsDef();
- struct.ks_def.read(iprot);
- struct.setKs_defIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, system_update_keyspace_args struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.ks_def != null) {
- oprot.writeFieldBegin(KS_DEF_FIELD_DESC);
- struct.ks_def.write(oprot);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class system_update_keyspace_argsTupleSchemeFactory implements SchemeFactory {
- public system_update_keyspace_argsTupleScheme getScheme() {
- return new system_update_keyspace_argsTupleScheme();
- }
- }
-
- private static class system_update_keyspace_argsTupleScheme extends TupleScheme<system_update_keyspace_args> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, system_update_keyspace_args struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- struct.ks_def.write(oprot);
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, system_update_keyspace_args struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- struct.ks_def = new KsDef();
- struct.ks_def.read(iprot);
- struct.setKs_defIsSet(true);
- }
- }
-
- }
-
- public static class system_update_keyspace_result implements org.apache.thrift.TBase<system_update_keyspace_result, system_update_keyspace_result._Fields>, java.io.Serializable, Cloneable, Comparable<system_update_keyspace_result> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("system_update_keyspace_result");
-
- private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRING, (short)0);
- private static final org.apache.thrift.protocol.TField IRE_FIELD_DESC = new org.apache.thrift.protocol.TField("ire", org.apache.thrift.protocol.TType.STRUCT, (short)1);
- private static final org.apache.thrift.protocol.TField SDE_FIELD_DESC = new org.apache.thrift.protocol.TField("sde", org.apache.thrift.protocol.TType.STRUCT, (short)2);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new system_update_keyspace_resultStandardSchemeFactory());
- schemes.put(TupleScheme.class, new system_update_keyspace_resultTupleSchemeFactory());
- }
-
- public String success; // required
- public InvalidRequestException ire; // required
- public SchemaDisagreementException sde; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- SUCCESS((short)0, "success"),
- IRE((short)1, "ire"),
- SDE((short)2, "sde");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 0: // SUCCESS
- return SUCCESS;
- case 1: // IRE
- return IRE;
- case 2: // SDE
- return SDE;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
- tmpMap.put(_Fields.IRE, new org.apache.thrift.meta_data.FieldMetaData("ire", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- tmpMap.put(_Fields.SDE, new org.apache.thrift.meta_data.FieldMetaData("sde", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(system_update_keyspace_result.class, metaDataMap);
- }
-
- public system_update_keyspace_result() {
- }
-
- public system_update_keyspace_result(
- String success,
- InvalidRequestException ire,
- SchemaDisagreementException sde)
- {
- this();
- this.success = success;
- this.ire = ire;
- this.sde = sde;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public system_update_keyspace_result(system_update_keyspace_result other) {
- if (other.isSetSuccess()) {
- this.success = other.success;
- }
- if (other.isSetIre()) {
- this.ire = new InvalidRequestException(other.ire);
- }
- if (other.isSetSde()) {
- this.sde = new SchemaDisagreementException(other.sde);
- }
- }
-
- public system_update_keyspace_result deepCopy() {
- return new system_update_keyspace_result(this);
- }
-
- @Override
- public void clear() {
- this.success = null;
- this.ire = null;
- this.sde = null;
- }
-
- public String getSuccess() {
- return this.success;
- }
-
- public system_update_keyspace_result setSuccess(String success) {
- this.success = success;
- return this;
- }
-
- public void unsetSuccess() {
- this.success = null;
- }
-
- /** Returns true if field success is set (has been assigned a value) and false otherwise */
- public boolean isSetSuccess() {
- return this.success != null;
- }
-
- public void setSuccessIsSet(boolean value) {
- if (!value) {
- this.success = null;
- }
- }
-
- public InvalidRequestException getIre() {
- return this.ire;
- }
-
- public system_update_keyspace_result setIre(InvalidRequestException ire) {
- this.ire = ire;
- return this;
- }
-
- public void unsetIre() {
- this.ire = null;
- }
-
- /** Returns true if field ire is set (has been assigned a value) and false otherwise */
- public boolean isSetIre() {
- return this.ire != null;
- }
-
- public void setIreIsSet(boolean value) {
- if (!value) {
- this.ire = null;
- }
- }
-
- public SchemaDisagreementException getSde() {
- return this.sde;
- }
-
- public system_update_keyspace_result setSde(SchemaDisagreementException sde) {
- this.sde = sde;
- return this;
- }
-
- public void unsetSde() {
- this.sde = null;
- }
-
- /** Returns true if field sde is set (has been assigned a value) and false otherwise */
- public boolean isSetSde() {
- return this.sde != null;
- }
-
- public void setSdeIsSet(boolean value) {
- if (!value) {
- this.sde = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case SUCCESS:
- if (value == null) {
- unsetSuccess();
- } else {
- setSuccess((String)value);
- }
- break;
-
- case IRE:
- if (value == null) {
- unsetIre();
- } else {
- setIre((InvalidRequestException)value);
- }
- break;
-
- case SDE:
- if (value == null) {
- unsetSde();
- } else {
- setSde((SchemaDisagreementException)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case SUCCESS:
- return getSuccess();
-
- case IRE:
- return getIre();
-
- case SDE:
- return getSde();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case SUCCESS:
- return isSetSuccess();
- case IRE:
- return isSetIre();
- case SDE:
- return isSetSde();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof system_update_keyspace_result)
- return this.equals((system_update_keyspace_result)that);
- return false;
- }
-
- public boolean equals(system_update_keyspace_result that) {
- if (that == null)
- return false;
-
- boolean this_present_success = true && this.isSetSuccess();
- boolean that_present_success = true && that.isSetSuccess();
- if (this_present_success || that_present_success) {
- if (!(this_present_success && that_present_success))
- return false;
- if (!this.success.equals(that.success))
- return false;
- }
-
- boolean this_present_ire = true && this.isSetIre();
- boolean that_present_ire = true && that.isSetIre();
- if (this_present_ire || that_present_ire) {
- if (!(this_present_ire && that_present_ire))
- return false;
- if (!this.ire.equals(that.ire))
- return false;
- }
-
- boolean this_present_sde = true && this.isSetSde();
- boolean that_present_sde = true && that.isSetSde();
- if (this_present_sde || that_present_sde) {
- if (!(this_present_sde && that_present_sde))
- return false;
- if (!this.sde.equals(that.sde))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_success = true && (isSetSuccess());
- builder.append(present_success);
- if (present_success)
- builder.append(success);
-
- boolean present_ire = true && (isSetIre());
- builder.append(present_ire);
- if (present_ire)
- builder.append(ire);
-
- boolean present_sde = true && (isSetSde());
- builder.append(present_sde);
- if (present_sde)
- builder.append(sde);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(system_update_keyspace_result other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(other.isSetSuccess());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetSuccess()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, other.success);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetIre()).compareTo(other.isSetIre());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetIre()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.ire, other.ire);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetSde()).compareTo(other.isSetSde());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetSde()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.sde, other.sde);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("system_update_keyspace_result(");
- boolean first = true;
-
- sb.append("success:");
- if (this.success == null) {
- sb.append("null");
- } else {
- sb.append(this.success);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("ire:");
- if (this.ire == null) {
- sb.append("null");
- } else {
- sb.append(this.ire);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("sde:");
- if (this.sde == null) {
- sb.append("null");
- } else {
- sb.append(this.sde);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class system_update_keyspace_resultStandardSchemeFactory implements SchemeFactory {
- public system_update_keyspace_resultStandardScheme getScheme() {
- return new system_update_keyspace_resultStandardScheme();
- }
- }
-
- private static class system_update_keyspace_resultStandardScheme extends StandardScheme<system_update_keyspace_result> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, system_update_keyspace_result struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 0: // SUCCESS
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.success = iprot.readString();
- struct.setSuccessIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 1: // IRE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.ire = new InvalidRequestException();
- struct.ire.read(iprot);
- struct.setIreIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 2: // SDE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.sde = new SchemaDisagreementException();
- struct.sde.read(iprot);
- struct.setSdeIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, system_update_keyspace_result struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.success != null) {
- oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
- oprot.writeString(struct.success);
- oprot.writeFieldEnd();
- }
- if (struct.ire != null) {
- oprot.writeFieldBegin(IRE_FIELD_DESC);
- struct.ire.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.sde != null) {
- oprot.writeFieldBegin(SDE_FIELD_DESC);
- struct.sde.write(oprot);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class system_update_keyspace_resultTupleSchemeFactory implements SchemeFactory {
- public system_update_keyspace_resultTupleScheme getScheme() {
- return new system_update_keyspace_resultTupleScheme();
- }
- }
-
- private static class system_update_keyspace_resultTupleScheme extends TupleScheme<system_update_keyspace_result> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, system_update_keyspace_result struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- BitSet optionals = new BitSet();
- if (struct.isSetSuccess()) {
- optionals.set(0);
- }
- if (struct.isSetIre()) {
- optionals.set(1);
- }
- if (struct.isSetSde()) {
- optionals.set(2);
- }
- oprot.writeBitSet(optionals, 3);
- if (struct.isSetSuccess()) {
- oprot.writeString(struct.success);
- }
- if (struct.isSetIre()) {
- struct.ire.write(oprot);
- }
- if (struct.isSetSde()) {
- struct.sde.write(oprot);
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, system_update_keyspace_result struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- BitSet incoming = iprot.readBitSet(3);
- if (incoming.get(0)) {
- struct.success = iprot.readString();
- struct.setSuccessIsSet(true);
- }
- if (incoming.get(1)) {
- struct.ire = new InvalidRequestException();
- struct.ire.read(iprot);
- struct.setIreIsSet(true);
- }
- if (incoming.get(2)) {
- struct.sde = new SchemaDisagreementException();
- struct.sde.read(iprot);
- struct.setSdeIsSet(true);
- }
- }
- }
-
- }
-
- public static class system_update_column_family_args implements org.apache.thrift.TBase<system_update_column_family_args, system_update_column_family_args._Fields>, java.io.Serializable, Cloneable, Comparable<system_update_column_family_args> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("system_update_column_family_args");
-
- private static final org.apache.thrift.protocol.TField CF_DEF_FIELD_DESC = new org.apache.thrift.protocol.TField("cf_def", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new system_update_column_family_argsStandardSchemeFactory());
- schemes.put(TupleScheme.class, new system_update_column_family_argsTupleSchemeFactory());
- }
-
- public CfDef cf_def; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- CF_DEF((short)1, "cf_def");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // CF_DEF
- return CF_DEF;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.CF_DEF, new org.apache.thrift.meta_data.FieldMetaData("cf_def", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, CfDef.class)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(system_update_column_family_args.class, metaDataMap);
- }
-
- public system_update_column_family_args() {
- }
-
- public system_update_column_family_args(
- CfDef cf_def)
- {
- this();
- this.cf_def = cf_def;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public system_update_column_family_args(system_update_column_family_args other) {
- if (other.isSetCf_def()) {
- this.cf_def = new CfDef(other.cf_def);
- }
- }
-
- public system_update_column_family_args deepCopy() {
- return new system_update_column_family_args(this);
- }
-
- @Override
- public void clear() {
- this.cf_def = null;
- }
-
- public CfDef getCf_def() {
- return this.cf_def;
- }
-
- public system_update_column_family_args setCf_def(CfDef cf_def) {
- this.cf_def = cf_def;
- return this;
- }
-
- public void unsetCf_def() {
- this.cf_def = null;
- }
-
- /** Returns true if field cf_def is set (has been assigned a value) and false otherwise */
- public boolean isSetCf_def() {
- return this.cf_def != null;
- }
-
- public void setCf_defIsSet(boolean value) {
- if (!value) {
- this.cf_def = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case CF_DEF:
- if (value == null) {
- unsetCf_def();
- } else {
- setCf_def((CfDef)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case CF_DEF:
- return getCf_def();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case CF_DEF:
- return isSetCf_def();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof system_update_column_family_args)
- return this.equals((system_update_column_family_args)that);
- return false;
- }
-
- public boolean equals(system_update_column_family_args that) {
- if (that == null)
- return false;
-
- boolean this_present_cf_def = true && this.isSetCf_def();
- boolean that_present_cf_def = true && that.isSetCf_def();
- if (this_present_cf_def || that_present_cf_def) {
- if (!(this_present_cf_def && that_present_cf_def))
- return false;
- if (!this.cf_def.equals(that.cf_def))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_cf_def = true && (isSetCf_def());
- builder.append(present_cf_def);
- if (present_cf_def)
- builder.append(cf_def);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(system_update_column_family_args other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetCf_def()).compareTo(other.isSetCf_def());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetCf_def()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.cf_def, other.cf_def);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("system_update_column_family_args(");
- boolean first = true;
-
- sb.append("cf_def:");
- if (this.cf_def == null) {
- sb.append("null");
- } else {
- sb.append(this.cf_def);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- if (cf_def == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'cf_def' was not present! Struct: " + toString());
- }
- // check for sub-struct validity
- if (cf_def != null) {
- cf_def.validate();
- }
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class system_update_column_family_argsStandardSchemeFactory implements SchemeFactory {
- public system_update_column_family_argsStandardScheme getScheme() {
- return new system_update_column_family_argsStandardScheme();
- }
- }
-
- private static class system_update_column_family_argsStandardScheme extends StandardScheme<system_update_column_family_args> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, system_update_column_family_args struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // CF_DEF
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.cf_def = new CfDef();
- struct.cf_def.read(iprot);
- struct.setCf_defIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, system_update_column_family_args struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.cf_def != null) {
- oprot.writeFieldBegin(CF_DEF_FIELD_DESC);
- struct.cf_def.write(oprot);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class system_update_column_family_argsTupleSchemeFactory implements SchemeFactory {
- public system_update_column_family_argsTupleScheme getScheme() {
- return new system_update_column_family_argsTupleScheme();
- }
- }
-
- private static class system_update_column_family_argsTupleScheme extends TupleScheme<system_update_column_family_args> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, system_update_column_family_args struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- struct.cf_def.write(oprot);
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, system_update_column_family_args struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- struct.cf_def = new CfDef();
- struct.cf_def.read(iprot);
- struct.setCf_defIsSet(true);
- }
- }
-
- }
-
- public static class system_update_column_family_result implements org.apache.thrift.TBase<system_update_column_family_result, system_update_column_family_result._Fields>, java.io.Serializable, Cloneable, Comparable<system_update_column_family_result> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("system_update_column_family_result");
-
- private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRING, (short)0);
- private static final org.apache.thrift.protocol.TField IRE_FIELD_DESC = new org.apache.thrift.protocol.TField("ire", org.apache.thrift.protocol.TType.STRUCT, (short)1);
- private static final org.apache.thrift.protocol.TField SDE_FIELD_DESC = new org.apache.thrift.protocol.TField("sde", org.apache.thrift.protocol.TType.STRUCT, (short)2);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new system_update_column_family_resultStandardSchemeFactory());
- schemes.put(TupleScheme.class, new system_update_column_family_resultTupleSchemeFactory());
- }
-
- public String success; // required
- public InvalidRequestException ire; // required
- public SchemaDisagreementException sde; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- SUCCESS((short)0, "success"),
- IRE((short)1, "ire"),
- SDE((short)2, "sde");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 0: // SUCCESS
- return SUCCESS;
- case 1: // IRE
- return IRE;
- case 2: // SDE
- return SDE;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
- tmpMap.put(_Fields.IRE, new org.apache.thrift.meta_data.FieldMetaData("ire", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- tmpMap.put(_Fields.SDE, new org.apache.thrift.meta_data.FieldMetaData("sde", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(system_update_column_family_result.class, metaDataMap);
- }
-
- public system_update_column_family_result() {
- }
-
- public system_update_column_family_result(
- String success,
- InvalidRequestException ire,
- SchemaDisagreementException sde)
- {
- this();
- this.success = success;
- this.ire = ire;
- this.sde = sde;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public system_update_column_family_result(system_update_column_family_result other) {
- if (other.isSetSuccess()) {
- this.success = other.success;
- }
- if (other.isSetIre()) {
- this.ire = new InvalidRequestException(other.ire);
- }
- if (other.isSetSde()) {
- this.sde = new SchemaDisagreementException(other.sde);
- }
- }
-
- public system_update_column_family_result deepCopy() {
- return new system_update_column_family_result(this);
- }
-
- @Override
- public void clear() {
- this.success = null;
- this.ire = null;
- this.sde = null;
- }
-
- public String getSuccess() {
- return this.success;
- }
-
- public system_update_column_family_result setSuccess(String success) {
- this.success = success;
- return this;
- }
-
- public void unsetSuccess() {
- this.success = null;
- }
-
- /** Returns true if field success is set (has been assigned a value) and false otherwise */
- public boolean isSetSuccess() {
- return this.success != null;
- }
-
- public void setSuccessIsSet(boolean value) {
- if (!value) {
- this.success = null;
- }
- }
-
- public InvalidRequestException getIre() {
- return this.ire;
- }
-
- public system_update_column_family_result setIre(InvalidRequestException ire) {
- this.ire = ire;
- return this;
- }
-
- public void unsetIre() {
- this.ire = null;
- }
-
- /** Returns true if field ire is set (has been assigned a value) and false otherwise */
- public boolean isSetIre() {
- return this.ire != null;
- }
-
- public void setIreIsSet(boolean value) {
- if (!value) {
- this.ire = null;
- }
- }
-
- public SchemaDisagreementException getSde() {
- return this.sde;
- }
-
- public system_update_column_family_result setSde(SchemaDisagreementException sde) {
- this.sde = sde;
- return this;
- }
-
- public void unsetSde() {
- this.sde = null;
- }
-
- /** Returns true if field sde is set (has been assigned a value) and false otherwise */
- public boolean isSetSde() {
- return this.sde != null;
- }
-
- public void setSdeIsSet(boolean value) {
- if (!value) {
- this.sde = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case SUCCESS:
- if (value == null) {
- unsetSuccess();
- } else {
- setSuccess((String)value);
- }
- break;
-
- case IRE:
- if (value == null) {
- unsetIre();
- } else {
- setIre((InvalidRequestException)value);
- }
- break;
-
- case SDE:
- if (value == null) {
- unsetSde();
- } else {
- setSde((SchemaDisagreementException)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case SUCCESS:
- return getSuccess();
-
- case IRE:
- return getIre();
-
- case SDE:
- return getSde();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case SUCCESS:
- return isSetSuccess();
- case IRE:
- return isSetIre();
- case SDE:
- return isSetSde();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof system_update_column_family_result)
- return this.equals((system_update_column_family_result)that);
- return false;
- }
-
- public boolean equals(system_update_column_family_result that) {
- if (that == null)
- return false;
-
- boolean this_present_success = true && this.isSetSuccess();
- boolean that_present_success = true && that.isSetSuccess();
- if (this_present_success || that_present_success) {
- if (!(this_present_success && that_present_success))
- return false;
- if (!this.success.equals(that.success))
- return false;
- }
-
- boolean this_present_ire = true && this.isSetIre();
- boolean that_present_ire = true && that.isSetIre();
- if (this_present_ire || that_present_ire) {
- if (!(this_present_ire && that_present_ire))
- return false;
- if (!this.ire.equals(that.ire))
- return false;
- }
-
- boolean this_present_sde = true && this.isSetSde();
- boolean that_present_sde = true && that.isSetSde();
- if (this_present_sde || that_present_sde) {
- if (!(this_present_sde && that_present_sde))
- return false;
- if (!this.sde.equals(that.sde))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_success = true && (isSetSuccess());
- builder.append(present_success);
- if (present_success)
- builder.append(success);
-
- boolean present_ire = true && (isSetIre());
- builder.append(present_ire);
- if (present_ire)
- builder.append(ire);
-
- boolean present_sde = true && (isSetSde());
- builder.append(present_sde);
- if (present_sde)
- builder.append(sde);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(system_update_column_family_result other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(other.isSetSuccess());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetSuccess()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, other.success);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetIre()).compareTo(other.isSetIre());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetIre()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.ire, other.ire);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetSde()).compareTo(other.isSetSde());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetSde()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.sde, other.sde);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("system_update_column_family_result(");
- boolean first = true;
-
- sb.append("success:");
- if (this.success == null) {
- sb.append("null");
- } else {
- sb.append(this.success);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("ire:");
- if (this.ire == null) {
- sb.append("null");
- } else {
- sb.append(this.ire);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("sde:");
- if (this.sde == null) {
- sb.append("null");
- } else {
- sb.append(this.sde);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class system_update_column_family_resultStandardSchemeFactory implements SchemeFactory {
- public system_update_column_family_resultStandardScheme getScheme() {
- return new system_update_column_family_resultStandardScheme();
- }
- }
-
- private static class system_update_column_family_resultStandardScheme extends StandardScheme<system_update_column_family_result> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, system_update_column_family_result struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 0: // SUCCESS
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.success = iprot.readString();
- struct.setSuccessIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 1: // IRE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.ire = new InvalidRequestException();
- struct.ire.read(iprot);
- struct.setIreIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 2: // SDE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.sde = new SchemaDisagreementException();
- struct.sde.read(iprot);
- struct.setSdeIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, system_update_column_family_result struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.success != null) {
- oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
- oprot.writeString(struct.success);
- oprot.writeFieldEnd();
- }
- if (struct.ire != null) {
- oprot.writeFieldBegin(IRE_FIELD_DESC);
- struct.ire.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.sde != null) {
- oprot.writeFieldBegin(SDE_FIELD_DESC);
- struct.sde.write(oprot);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class system_update_column_family_resultTupleSchemeFactory implements SchemeFactory {
- public system_update_column_family_resultTupleScheme getScheme() {
- return new system_update_column_family_resultTupleScheme();
- }
- }
-
- private static class system_update_column_family_resultTupleScheme extends TupleScheme<system_update_column_family_result> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, system_update_column_family_result struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- BitSet optionals = new BitSet();
- if (struct.isSetSuccess()) {
- optionals.set(0);
- }
- if (struct.isSetIre()) {
- optionals.set(1);
- }
- if (struct.isSetSde()) {
- optionals.set(2);
- }
- oprot.writeBitSet(optionals, 3);
- if (struct.isSetSuccess()) {
- oprot.writeString(struct.success);
- }
- if (struct.isSetIre()) {
- struct.ire.write(oprot);
- }
- if (struct.isSetSde()) {
- struct.sde.write(oprot);
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, system_update_column_family_result struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- BitSet incoming = iprot.readBitSet(3);
- if (incoming.get(0)) {
- struct.success = iprot.readString();
- struct.setSuccessIsSet(true);
- }
- if (incoming.get(1)) {
- struct.ire = new InvalidRequestException();
- struct.ire.read(iprot);
- struct.setIreIsSet(true);
- }
- if (incoming.get(2)) {
- struct.sde = new SchemaDisagreementException();
- struct.sde.read(iprot);
- struct.setSdeIsSet(true);
- }
- }
- }
-
- }
-
- public static class execute_cql_query_args implements org.apache.thrift.TBase<execute_cql_query_args, execute_cql_query_args._Fields>, java.io.Serializable, Cloneable, Comparable<execute_cql_query_args> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("execute_cql_query_args");
-
- private static final org.apache.thrift.protocol.TField QUERY_FIELD_DESC = new org.apache.thrift.protocol.TField("query", org.apache.thrift.protocol.TType.STRING, (short)1);
- private static final org.apache.thrift.protocol.TField COMPRESSION_FIELD_DESC = new org.apache.thrift.protocol.TField("compression", org.apache.thrift.protocol.TType.I32, (short)2);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new execute_cql_query_argsStandardSchemeFactory());
- schemes.put(TupleScheme.class, new execute_cql_query_argsTupleSchemeFactory());
- }
-
- public ByteBuffer query; // required
- /**
- *
- * @see Compression
- */
- public Compression compression; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- QUERY((short)1, "query"),
- /**
- *
- * @see Compression
- */
- COMPRESSION((short)2, "compression");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // QUERY
- return QUERY;
- case 2: // COMPRESSION
- return COMPRESSION;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.QUERY, new org.apache.thrift.meta_data.FieldMetaData("query", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true)));
- tmpMap.put(_Fields.COMPRESSION, new org.apache.thrift.meta_data.FieldMetaData("compression", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.EnumMetaData(org.apache.thrift.protocol.TType.ENUM, Compression.class)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(execute_cql_query_args.class, metaDataMap);
- }
-
- public execute_cql_query_args() {
- }
-
- public execute_cql_query_args(
- ByteBuffer query,
- Compression compression)
- {
- this();
- this.query = query;
- this.compression = compression;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public execute_cql_query_args(execute_cql_query_args other) {
- if (other.isSetQuery()) {
- this.query = org.apache.thrift.TBaseHelper.copyBinary(other.query);
-;
- }
- if (other.isSetCompression()) {
- this.compression = other.compression;
- }
- }
-
- public execute_cql_query_args deepCopy() {
- return new execute_cql_query_args(this);
- }
-
- @Override
- public void clear() {
- this.query = null;
- this.compression = null;
- }
-
- public byte[] getQuery() {
- setQuery(org.apache.thrift.TBaseHelper.rightSize(query));
- return query == null ? null : query.array();
- }
-
- public ByteBuffer bufferForQuery() {
- return query;
- }
-
- public execute_cql_query_args setQuery(byte[] query) {
- setQuery(query == null ? (ByteBuffer)null : ByteBuffer.wrap(query));
- return this;
- }
-
- public execute_cql_query_args setQuery(ByteBuffer query) {
- this.query = query;
- return this;
- }
-
- public void unsetQuery() {
- this.query = null;
- }
-
- /** Returns true if field query is set (has been assigned a value) and false otherwise */
- public boolean isSetQuery() {
- return this.query != null;
- }
-
- public void setQueryIsSet(boolean value) {
- if (!value) {
- this.query = null;
- }
- }
-
- /**
- *
- * @see Compression
- */
- public Compression getCompression() {
- return this.compression;
- }
-
- /**
- *
- * @see Compression
- */
- public execute_cql_query_args setCompression(Compression compression) {
- this.compression = compression;
- return this;
- }
-
- public void unsetCompression() {
- this.compression = null;
- }
-
- /** Returns true if field compression is set (has been assigned a value) and false otherwise */
- public boolean isSetCompression() {
- return this.compression != null;
- }
-
- public void setCompressionIsSet(boolean value) {
- if (!value) {
- this.compression = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case QUERY:
- if (value == null) {
- unsetQuery();
- } else {
- setQuery((ByteBuffer)value);
- }
- break;
-
- case COMPRESSION:
- if (value == null) {
- unsetCompression();
- } else {
- setCompression((Compression)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case QUERY:
- return getQuery();
-
- case COMPRESSION:
- return getCompression();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case QUERY:
- return isSetQuery();
- case COMPRESSION:
- return isSetCompression();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof execute_cql_query_args)
- return this.equals((execute_cql_query_args)that);
- return false;
- }
-
- public boolean equals(execute_cql_query_args that) {
- if (that == null)
- return false;
-
- boolean this_present_query = true && this.isSetQuery();
- boolean that_present_query = true && that.isSetQuery();
- if (this_present_query || that_present_query) {
- if (!(this_present_query && that_present_query))
- return false;
- if (!this.query.equals(that.query))
- return false;
- }
-
- boolean this_present_compression = true && this.isSetCompression();
- boolean that_present_compression = true && that.isSetCompression();
- if (this_present_compression || that_present_compression) {
- if (!(this_present_compression && that_present_compression))
- return false;
- if (!this.compression.equals(that.compression))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_query = true && (isSetQuery());
- builder.append(present_query);
- if (present_query)
- builder.append(query);
-
- boolean present_compression = true && (isSetCompression());
- builder.append(present_compression);
- if (present_compression)
- builder.append(compression.getValue());
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(execute_cql_query_args other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetQuery()).compareTo(other.isSetQuery());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetQuery()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.query, other.query);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetCompression()).compareTo(other.isSetCompression());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetCompression()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.compression, other.compression);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("execute_cql_query_args(");
- boolean first = true;
-
- sb.append("query:");
- if (this.query == null) {
- sb.append("null");
- } else {
- org.apache.thrift.TBaseHelper.toString(this.query, sb);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("compression:");
- if (this.compression == null) {
- sb.append("null");
- } else {
- sb.append(this.compression);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- if (query == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'query' was not present! Struct: " + toString());
- }
- if (compression == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'compression' was not present! Struct: " + toString());
- }
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class execute_cql_query_argsStandardSchemeFactory implements SchemeFactory {
- public execute_cql_query_argsStandardScheme getScheme() {
- return new execute_cql_query_argsStandardScheme();
- }
- }
-
- private static class execute_cql_query_argsStandardScheme extends StandardScheme<execute_cql_query_args> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, execute_cql_query_args struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // QUERY
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.query = iprot.readBinary();
- struct.setQueryIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 2: // COMPRESSION
- if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
- struct.compression = Compression.findByValue(iprot.readI32());
- struct.setCompressionIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, execute_cql_query_args struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.query != null) {
- oprot.writeFieldBegin(QUERY_FIELD_DESC);
- oprot.writeBinary(struct.query);
- oprot.writeFieldEnd();
- }
- if (struct.compression != null) {
- oprot.writeFieldBegin(COMPRESSION_FIELD_DESC);
- oprot.writeI32(struct.compression.getValue());
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class execute_cql_query_argsTupleSchemeFactory implements SchemeFactory {
- public execute_cql_query_argsTupleScheme getScheme() {
- return new execute_cql_query_argsTupleScheme();
- }
- }
-
- private static class execute_cql_query_argsTupleScheme extends TupleScheme<execute_cql_query_args> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, execute_cql_query_args struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- oprot.writeBinary(struct.query);
- oprot.writeI32(struct.compression.getValue());
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, execute_cql_query_args struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- struct.query = iprot.readBinary();
- struct.setQueryIsSet(true);
- struct.compression = Compression.findByValue(iprot.readI32());
- struct.setCompressionIsSet(true);
- }
- }
-
- }
-
- public static class execute_cql_query_result implements org.apache.thrift.TBase<execute_cql_query_result, execute_cql_query_result._Fields>, java.io.Serializable, Cloneable, Comparable<execute_cql_query_result> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("execute_cql_query_result");
-
- private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRUCT, (short)0);
- private static final org.apache.thrift.protocol.TField IRE_FIELD_DESC = new org.apache.thrift.protocol.TField("ire", org.apache.thrift.protocol.TType.STRUCT, (short)1);
- private static final org.apache.thrift.protocol.TField UE_FIELD_DESC = new org.apache.thrift.protocol.TField("ue", org.apache.thrift.protocol.TType.STRUCT, (short)2);
- private static final org.apache.thrift.protocol.TField TE_FIELD_DESC = new org.apache.thrift.protocol.TField("te", org.apache.thrift.protocol.TType.STRUCT, (short)3);
- private static final org.apache.thrift.protocol.TField SDE_FIELD_DESC = new org.apache.thrift.protocol.TField("sde", org.apache.thrift.protocol.TType.STRUCT, (short)4);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new execute_cql_query_resultStandardSchemeFactory());
- schemes.put(TupleScheme.class, new execute_cql_query_resultTupleSchemeFactory());
- }
-
- public CqlResult success; // required
- public InvalidRequestException ire; // required
- public UnavailableException ue; // required
- public TimedOutException te; // required
- public SchemaDisagreementException sde; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- SUCCESS((short)0, "success"),
- IRE((short)1, "ire"),
- UE((short)2, "ue"),
- TE((short)3, "te"),
- SDE((short)4, "sde");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 0: // SUCCESS
- return SUCCESS;
- case 1: // IRE
- return IRE;
- case 2: // UE
- return UE;
- case 3: // TE
- return TE;
- case 4: // SDE
- return SDE;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, CqlResult.class)));
- tmpMap.put(_Fields.IRE, new org.apache.thrift.meta_data.FieldMetaData("ire", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- tmpMap.put(_Fields.UE, new org.apache.thrift.meta_data.FieldMetaData("ue", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- tmpMap.put(_Fields.TE, new org.apache.thrift.meta_data.FieldMetaData("te", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- tmpMap.put(_Fields.SDE, new org.apache.thrift.meta_data.FieldMetaData("sde", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(execute_cql_query_result.class, metaDataMap);
- }
-
- public execute_cql_query_result() {
- }
-
- public execute_cql_query_result(
- CqlResult success,
- InvalidRequestException ire,
- UnavailableException ue,
- TimedOutException te,
- SchemaDisagreementException sde)
- {
- this();
- this.success = success;
- this.ire = ire;
- this.ue = ue;
- this.te = te;
- this.sde = sde;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public execute_cql_query_result(execute_cql_query_result other) {
- if (other.isSetSuccess()) {
- this.success = new CqlResult(other.success);
- }
- if (other.isSetIre()) {
- this.ire = new InvalidRequestException(other.ire);
- }
- if (other.isSetUe()) {
- this.ue = new UnavailableException(other.ue);
- }
- if (other.isSetTe()) {
- this.te = new TimedOutException(other.te);
- }
- if (other.isSetSde()) {
- this.sde = new SchemaDisagreementException(other.sde);
- }
- }
-
- public execute_cql_query_result deepCopy() {
- return new execute_cql_query_result(this);
- }
-
- @Override
- public void clear() {
- this.success = null;
- this.ire = null;
- this.ue = null;
- this.te = null;
- this.sde = null;
- }
-
- public CqlResult getSuccess() {
- return this.success;
- }
-
- public execute_cql_query_result setSuccess(CqlResult success) {
- this.success = success;
- return this;
- }
-
- public void unsetSuccess() {
- this.success = null;
- }
-
- /** Returns true if field success is set (has been assigned a value) and false otherwise */
- public boolean isSetSuccess() {
- return this.success != null;
- }
-
- public void setSuccessIsSet(boolean value) {
- if (!value) {
- this.success = null;
- }
- }
-
- public InvalidRequestException getIre() {
- return this.ire;
- }
-
- public execute_cql_query_result setIre(InvalidRequestException ire) {
- this.ire = ire;
- return this;
- }
-
- public void unsetIre() {
- this.ire = null;
- }
-
- /** Returns true if field ire is set (has been assigned a value) and false otherwise */
- public boolean isSetIre() {
- return this.ire != null;
- }
-
- public void setIreIsSet(boolean value) {
- if (!value) {
- this.ire = null;
- }
- }
-
- public UnavailableException getUe() {
- return this.ue;
- }
-
- public execute_cql_query_result setUe(UnavailableException ue) {
- this.ue = ue;
- return this;
- }
-
- public void unsetUe() {
- this.ue = null;
- }
-
- /** Returns true if field ue is set (has been assigned a value) and false otherwise */
- public boolean isSetUe() {
- return this.ue != null;
- }
-
- public void setUeIsSet(boolean value) {
- if (!value) {
- this.ue = null;
- }
- }
-
- public TimedOutException getTe() {
- return this.te;
- }
-
- public execute_cql_query_result setTe(TimedOutException te) {
- this.te = te;
- return this;
- }
-
- public void unsetTe() {
- this.te = null;
- }
-
- /** Returns true if field te is set (has been assigned a value) and false otherwise */
- public boolean isSetTe() {
- return this.te != null;
- }
-
- public void setTeIsSet(boolean value) {
- if (!value) {
- this.te = null;
- }
- }
-
- public SchemaDisagreementException getSde() {
- return this.sde;
- }
-
- public execute_cql_query_result setSde(SchemaDisagreementException sde) {
- this.sde = sde;
- return this;
- }
-
- public void unsetSde() {
- this.sde = null;
- }
-
- /** Returns true if field sde is set (has been assigned a value) and false otherwise */
- public boolean isSetSde() {
- return this.sde != null;
- }
-
- public void setSdeIsSet(boolean value) {
- if (!value) {
- this.sde = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case SUCCESS:
- if (value == null) {
- unsetSuccess();
- } else {
- setSuccess((CqlResult)value);
- }
- break;
-
- case IRE:
- if (value == null) {
- unsetIre();
- } else {
- setIre((InvalidRequestException)value);
- }
- break;
-
- case UE:
- if (value == null) {
- unsetUe();
- } else {
- setUe((UnavailableException)value);
- }
- break;
-
- case TE:
- if (value == null) {
- unsetTe();
- } else {
- setTe((TimedOutException)value);
- }
- break;
-
- case SDE:
- if (value == null) {
- unsetSde();
- } else {
- setSde((SchemaDisagreementException)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case SUCCESS:
- return getSuccess();
-
- case IRE:
- return getIre();
-
- case UE:
- return getUe();
-
- case TE:
- return getTe();
-
- case SDE:
- return getSde();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case SUCCESS:
- return isSetSuccess();
- case IRE:
- return isSetIre();
- case UE:
- return isSetUe();
- case TE:
- return isSetTe();
- case SDE:
- return isSetSde();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof execute_cql_query_result)
- return this.equals((execute_cql_query_result)that);
- return false;
- }
-
- public boolean equals(execute_cql_query_result that) {
- if (that == null)
- return false;
-
- boolean this_present_success = true && this.isSetSuccess();
- boolean that_present_success = true && that.isSetSuccess();
- if (this_present_success || that_present_success) {
- if (!(this_present_success && that_present_success))
- return false;
- if (!this.success.equals(that.success))
- return false;
- }
-
- boolean this_present_ire = true && this.isSetIre();
- boolean that_present_ire = true && that.isSetIre();
- if (this_present_ire || that_present_ire) {
- if (!(this_present_ire && that_present_ire))
- return false;
- if (!this.ire.equals(that.ire))
- return false;
- }
-
- boolean this_present_ue = true && this.isSetUe();
- boolean that_present_ue = true && that.isSetUe();
- if (this_present_ue || that_present_ue) {
- if (!(this_present_ue && that_present_ue))
- return false;
- if (!this.ue.equals(that.ue))
- return false;
- }
-
- boolean this_present_te = true && this.isSetTe();
- boolean that_present_te = true && that.isSetTe();
- if (this_present_te || that_present_te) {
- if (!(this_present_te && that_present_te))
- return false;
- if (!this.te.equals(that.te))
- return false;
- }
-
- boolean this_present_sde = true && this.isSetSde();
- boolean that_present_sde = true && that.isSetSde();
- if (this_present_sde || that_present_sde) {
- if (!(this_present_sde && that_present_sde))
- return false;
- if (!this.sde.equals(that.sde))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_success = true && (isSetSuccess());
- builder.append(present_success);
- if (present_success)
- builder.append(success);
-
- boolean present_ire = true && (isSetIre());
- builder.append(present_ire);
- if (present_ire)
- builder.append(ire);
-
- boolean present_ue = true && (isSetUe());
- builder.append(present_ue);
- if (present_ue)
- builder.append(ue);
-
- boolean present_te = true && (isSetTe());
- builder.append(present_te);
- if (present_te)
- builder.append(te);
-
- boolean present_sde = true && (isSetSde());
- builder.append(present_sde);
- if (present_sde)
- builder.append(sde);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(execute_cql_query_result other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(other.isSetSuccess());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetSuccess()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, other.success);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetIre()).compareTo(other.isSetIre());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetIre()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.ire, other.ire);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetUe()).compareTo(other.isSetUe());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetUe()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.ue, other.ue);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetTe()).compareTo(other.isSetTe());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetTe()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.te, other.te);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetSde()).compareTo(other.isSetSde());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetSde()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.sde, other.sde);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("execute_cql_query_result(");
- boolean first = true;
-
- sb.append("success:");
- if (this.success == null) {
- sb.append("null");
- } else {
- sb.append(this.success);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("ire:");
- if (this.ire == null) {
- sb.append("null");
- } else {
- sb.append(this.ire);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("ue:");
- if (this.ue == null) {
- sb.append("null");
- } else {
- sb.append(this.ue);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("te:");
- if (this.te == null) {
- sb.append("null");
- } else {
- sb.append(this.te);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("sde:");
- if (this.sde == null) {
- sb.append("null");
- } else {
- sb.append(this.sde);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // check for sub-struct validity
- if (success != null) {
- success.validate();
- }
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class execute_cql_query_resultStandardSchemeFactory implements SchemeFactory {
- public execute_cql_query_resultStandardScheme getScheme() {
- return new execute_cql_query_resultStandardScheme();
- }
- }
-
- private static class execute_cql_query_resultStandardScheme extends StandardScheme<execute_cql_query_result> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, execute_cql_query_result struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 0: // SUCCESS
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.success = new CqlResult();
- struct.success.read(iprot);
- struct.setSuccessIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 1: // IRE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.ire = new InvalidRequestException();
- struct.ire.read(iprot);
- struct.setIreIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 2: // UE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.ue = new UnavailableException();
- struct.ue.read(iprot);
- struct.setUeIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 3: // TE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.te = new TimedOutException();
- struct.te.read(iprot);
- struct.setTeIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 4: // SDE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.sde = new SchemaDisagreementException();
- struct.sde.read(iprot);
- struct.setSdeIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, execute_cql_query_result struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.success != null) {
- oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
- struct.success.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.ire != null) {
- oprot.writeFieldBegin(IRE_FIELD_DESC);
- struct.ire.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.ue != null) {
- oprot.writeFieldBegin(UE_FIELD_DESC);
- struct.ue.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.te != null) {
- oprot.writeFieldBegin(TE_FIELD_DESC);
- struct.te.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.sde != null) {
- oprot.writeFieldBegin(SDE_FIELD_DESC);
- struct.sde.write(oprot);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class execute_cql_query_resultTupleSchemeFactory implements SchemeFactory {
- public execute_cql_query_resultTupleScheme getScheme() {
- return new execute_cql_query_resultTupleScheme();
- }
- }
-
- private static class execute_cql_query_resultTupleScheme extends TupleScheme<execute_cql_query_result> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, execute_cql_query_result struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- BitSet optionals = new BitSet();
- if (struct.isSetSuccess()) {
- optionals.set(0);
- }
- if (struct.isSetIre()) {
- optionals.set(1);
- }
- if (struct.isSetUe()) {
- optionals.set(2);
- }
- if (struct.isSetTe()) {
- optionals.set(3);
- }
- if (struct.isSetSde()) {
- optionals.set(4);
- }
- oprot.writeBitSet(optionals, 5);
- if (struct.isSetSuccess()) {
- struct.success.write(oprot);
- }
- if (struct.isSetIre()) {
- struct.ire.write(oprot);
- }
- if (struct.isSetUe()) {
- struct.ue.write(oprot);
- }
- if (struct.isSetTe()) {
- struct.te.write(oprot);
- }
- if (struct.isSetSde()) {
- struct.sde.write(oprot);
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, execute_cql_query_result struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- BitSet incoming = iprot.readBitSet(5);
- if (incoming.get(0)) {
- struct.success = new CqlResult();
- struct.success.read(iprot);
- struct.setSuccessIsSet(true);
- }
- if (incoming.get(1)) {
- struct.ire = new InvalidRequestException();
- struct.ire.read(iprot);
- struct.setIreIsSet(true);
- }
- if (incoming.get(2)) {
- struct.ue = new UnavailableException();
- struct.ue.read(iprot);
- struct.setUeIsSet(true);
- }
- if (incoming.get(3)) {
- struct.te = new TimedOutException();
- struct.te.read(iprot);
- struct.setTeIsSet(true);
- }
- if (incoming.get(4)) {
- struct.sde = new SchemaDisagreementException();
- struct.sde.read(iprot);
- struct.setSdeIsSet(true);
- }
- }
- }
-
- }
-
- public static class execute_cql3_query_args implements org.apache.thrift.TBase<execute_cql3_query_args, execute_cql3_query_args._Fields>, java.io.Serializable, Cloneable, Comparable<execute_cql3_query_args> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("execute_cql3_query_args");
-
- private static final org.apache.thrift.protocol.TField QUERY_FIELD_DESC = new org.apache.thrift.protocol.TField("query", org.apache.thrift.protocol.TType.STRING, (short)1);
- private static final org.apache.thrift.protocol.TField COMPRESSION_FIELD_DESC = new org.apache.thrift.protocol.TField("compression", org.apache.thrift.protocol.TType.I32, (short)2);
- private static final org.apache.thrift.protocol.TField CONSISTENCY_FIELD_DESC = new org.apache.thrift.protocol.TField("consistency", org.apache.thrift.protocol.TType.I32, (short)3);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new execute_cql3_query_argsStandardSchemeFactory());
- schemes.put(TupleScheme.class, new execute_cql3_query_argsTupleSchemeFactory());
- }
-
- public ByteBuffer query; // required
- /**
- *
- * @see Compression
- */
- public Compression compression; // required
- /**
- *
- * @see ConsistencyLevel
- */
- public ConsistencyLevel consistency; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- QUERY((short)1, "query"),
- /**
- *
- * @see Compression
- */
- COMPRESSION((short)2, "compression"),
- /**
- *
- * @see ConsistencyLevel
- */
- CONSISTENCY((short)3, "consistency");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // QUERY
- return QUERY;
- case 2: // COMPRESSION
- return COMPRESSION;
- case 3: // CONSISTENCY
- return CONSISTENCY;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.QUERY, new org.apache.thrift.meta_data.FieldMetaData("query", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true)));
- tmpMap.put(_Fields.COMPRESSION, new org.apache.thrift.meta_data.FieldMetaData("compression", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.EnumMetaData(org.apache.thrift.protocol.TType.ENUM, Compression.class)));
- tmpMap.put(_Fields.CONSISTENCY, new org.apache.thrift.meta_data.FieldMetaData("consistency", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.EnumMetaData(org.apache.thrift.protocol.TType.ENUM, ConsistencyLevel.class)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(execute_cql3_query_args.class, metaDataMap);
- }
-
- public execute_cql3_query_args() {
- }
-
- public execute_cql3_query_args(
- ByteBuffer query,
- Compression compression,
- ConsistencyLevel consistency)
- {
- this();
- this.query = query;
- this.compression = compression;
- this.consistency = consistency;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public execute_cql3_query_args(execute_cql3_query_args other) {
- if (other.isSetQuery()) {
- this.query = org.apache.thrift.TBaseHelper.copyBinary(other.query);
-;
- }
- if (other.isSetCompression()) {
- this.compression = other.compression;
- }
- if (other.isSetConsistency()) {
- this.consistency = other.consistency;
- }
- }
-
- public execute_cql3_query_args deepCopy() {
- return new execute_cql3_query_args(this);
- }
-
- @Override
- public void clear() {
- this.query = null;
- this.compression = null;
- this.consistency = null;
- }
-
- public byte[] getQuery() {
- setQuery(org.apache.thrift.TBaseHelper.rightSize(query));
- return query == null ? null : query.array();
- }
-
- public ByteBuffer bufferForQuery() {
- return query;
- }
-
- public execute_cql3_query_args setQuery(byte[] query) {
- setQuery(query == null ? (ByteBuffer)null : ByteBuffer.wrap(query));
- return this;
- }
-
- public execute_cql3_query_args setQuery(ByteBuffer query) {
- this.query = query;
- return this;
- }
-
- public void unsetQuery() {
- this.query = null;
- }
-
- /** Returns true if field query is set (has been assigned a value) and false otherwise */
- public boolean isSetQuery() {
- return this.query != null;
- }
-
- public void setQueryIsSet(boolean value) {
- if (!value) {
- this.query = null;
- }
- }
-
- /**
- *
- * @see Compression
- */
- public Compression getCompression() {
- return this.compression;
- }
-
- /**
- *
- * @see Compression
- */
- public execute_cql3_query_args setCompression(Compression compression) {
- this.compression = compression;
- return this;
- }
-
- public void unsetCompression() {
- this.compression = null;
- }
-
- /** Returns true if field compression is set (has been assigned a value) and false otherwise */
- public boolean isSetCompression() {
- return this.compression != null;
- }
-
- public void setCompressionIsSet(boolean value) {
- if (!value) {
- this.compression = null;
- }
- }
-
- /**
- *
- * @see ConsistencyLevel
- */
- public ConsistencyLevel getConsistency() {
- return this.consistency;
- }
-
- /**
- *
- * @see ConsistencyLevel
- */
- public execute_cql3_query_args setConsistency(ConsistencyLevel consistency) {
- this.consistency = consistency;
- return this;
- }
-
- public void unsetConsistency() {
- this.consistency = null;
- }
-
- /** Returns true if field consistency is set (has been assigned a value) and false otherwise */
- public boolean isSetConsistency() {
- return this.consistency != null;
- }
-
- public void setConsistencyIsSet(boolean value) {
- if (!value) {
- this.consistency = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case QUERY:
- if (value == null) {
- unsetQuery();
- } else {
- setQuery((ByteBuffer)value);
- }
- break;
-
- case COMPRESSION:
- if (value == null) {
- unsetCompression();
- } else {
- setCompression((Compression)value);
- }
- break;
-
- case CONSISTENCY:
- if (value == null) {
- unsetConsistency();
- } else {
- setConsistency((ConsistencyLevel)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case QUERY:
- return getQuery();
-
- case COMPRESSION:
- return getCompression();
-
- case CONSISTENCY:
- return getConsistency();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case QUERY:
- return isSetQuery();
- case COMPRESSION:
- return isSetCompression();
- case CONSISTENCY:
- return isSetConsistency();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof execute_cql3_query_args)
- return this.equals((execute_cql3_query_args)that);
- return false;
- }
-
- public boolean equals(execute_cql3_query_args that) {
- if (that == null)
- return false;
-
- boolean this_present_query = true && this.isSetQuery();
- boolean that_present_query = true && that.isSetQuery();
- if (this_present_query || that_present_query) {
- if (!(this_present_query && that_present_query))
- return false;
- if (!this.query.equals(that.query))
- return false;
- }
-
- boolean this_present_compression = true && this.isSetCompression();
- boolean that_present_compression = true && that.isSetCompression();
- if (this_present_compression || that_present_compression) {
- if (!(this_present_compression && that_present_compression))
- return false;
- if (!this.compression.equals(that.compression))
- return false;
- }
-
- boolean this_present_consistency = true && this.isSetConsistency();
- boolean that_present_consistency = true && that.isSetConsistency();
- if (this_present_consistency || that_present_consistency) {
- if (!(this_present_consistency && that_present_consistency))
- return false;
- if (!this.consistency.equals(that.consistency))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_query = true && (isSetQuery());
- builder.append(present_query);
- if (present_query)
- builder.append(query);
-
- boolean present_compression = true && (isSetCompression());
- builder.append(present_compression);
- if (present_compression)
- builder.append(compression.getValue());
-
- boolean present_consistency = true && (isSetConsistency());
- builder.append(present_consistency);
- if (present_consistency)
- builder.append(consistency.getValue());
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(execute_cql3_query_args other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetQuery()).compareTo(other.isSetQuery());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetQuery()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.query, other.query);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetCompression()).compareTo(other.isSetCompression());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetCompression()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.compression, other.compression);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetConsistency()).compareTo(other.isSetConsistency());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetConsistency()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.consistency, other.consistency);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("execute_cql3_query_args(");
- boolean first = true;
-
- sb.append("query:");
- if (this.query == null) {
- sb.append("null");
- } else {
- org.apache.thrift.TBaseHelper.toString(this.query, sb);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("compression:");
- if (this.compression == null) {
- sb.append("null");
- } else {
- sb.append(this.compression);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("consistency:");
- if (this.consistency == null) {
- sb.append("null");
- } else {
- sb.append(this.consistency);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- if (query == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'query' was not present! Struct: " + toString());
- }
- if (compression == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'compression' was not present! Struct: " + toString());
- }
- if (consistency == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'consistency' was not present! Struct: " + toString());
- }
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class execute_cql3_query_argsStandardSchemeFactory implements SchemeFactory {
- public execute_cql3_query_argsStandardScheme getScheme() {
- return new execute_cql3_query_argsStandardScheme();
- }
- }
-
- private static class execute_cql3_query_argsStandardScheme extends StandardScheme<execute_cql3_query_args> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, execute_cql3_query_args struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // QUERY
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.query = iprot.readBinary();
- struct.setQueryIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 2: // COMPRESSION
- if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
- struct.compression = Compression.findByValue(iprot.readI32());
- struct.setCompressionIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 3: // CONSISTENCY
- if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
- struct.consistency = ConsistencyLevel.findByValue(iprot.readI32());
- struct.setConsistencyIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, execute_cql3_query_args struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.query != null) {
- oprot.writeFieldBegin(QUERY_FIELD_DESC);
- oprot.writeBinary(struct.query);
- oprot.writeFieldEnd();
- }
- if (struct.compression != null) {
- oprot.writeFieldBegin(COMPRESSION_FIELD_DESC);
- oprot.writeI32(struct.compression.getValue());
- oprot.writeFieldEnd();
- }
- if (struct.consistency != null) {
- oprot.writeFieldBegin(CONSISTENCY_FIELD_DESC);
- oprot.writeI32(struct.consistency.getValue());
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class execute_cql3_query_argsTupleSchemeFactory implements SchemeFactory {
- public execute_cql3_query_argsTupleScheme getScheme() {
- return new execute_cql3_query_argsTupleScheme();
- }
- }
-
- private static class execute_cql3_query_argsTupleScheme extends TupleScheme<execute_cql3_query_args> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, execute_cql3_query_args struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- oprot.writeBinary(struct.query);
- oprot.writeI32(struct.compression.getValue());
- oprot.writeI32(struct.consistency.getValue());
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, execute_cql3_query_args struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- struct.query = iprot.readBinary();
- struct.setQueryIsSet(true);
- struct.compression = Compression.findByValue(iprot.readI32());
- struct.setCompressionIsSet(true);
- struct.consistency = ConsistencyLevel.findByValue(iprot.readI32());
- struct.setConsistencyIsSet(true);
- }
- }
-
- }
-
- public static class execute_cql3_query_result implements org.apache.thrift.TBase<execute_cql3_query_result, execute_cql3_query_result._Fields>, java.io.Serializable, Cloneable, Comparable<execute_cql3_query_result> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("execute_cql3_query_result");
-
- private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRUCT, (short)0);
- private static final org.apache.thrift.protocol.TField IRE_FIELD_DESC = new org.apache.thrift.protocol.TField("ire", org.apache.thrift.protocol.TType.STRUCT, (short)1);
- private static final org.apache.thrift.protocol.TField UE_FIELD_DESC = new org.apache.thrift.protocol.TField("ue", org.apache.thrift.protocol.TType.STRUCT, (short)2);
- private static final org.apache.thrift.protocol.TField TE_FIELD_DESC = new org.apache.thrift.protocol.TField("te", org.apache.thrift.protocol.TType.STRUCT, (short)3);
- private static final org.apache.thrift.protocol.TField SDE_FIELD_DESC = new org.apache.thrift.protocol.TField("sde", org.apache.thrift.protocol.TType.STRUCT, (short)4);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new execute_cql3_query_resultStandardSchemeFactory());
- schemes.put(TupleScheme.class, new execute_cql3_query_resultTupleSchemeFactory());
- }
-
- public CqlResult success; // required
- public InvalidRequestException ire; // required
- public UnavailableException ue; // required
- public TimedOutException te; // required
- public SchemaDisagreementException sde; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- SUCCESS((short)0, "success"),
- IRE((short)1, "ire"),
- UE((short)2, "ue"),
- TE((short)3, "te"),
- SDE((short)4, "sde");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 0: // SUCCESS
- return SUCCESS;
- case 1: // IRE
- return IRE;
- case 2: // UE
- return UE;
- case 3: // TE
- return TE;
- case 4: // SDE
- return SDE;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, CqlResult.class)));
- tmpMap.put(_Fields.IRE, new org.apache.thrift.meta_data.FieldMetaData("ire", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- tmpMap.put(_Fields.UE, new org.apache.thrift.meta_data.FieldMetaData("ue", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- tmpMap.put(_Fields.TE, new org.apache.thrift.meta_data.FieldMetaData("te", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- tmpMap.put(_Fields.SDE, new org.apache.thrift.meta_data.FieldMetaData("sde", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(execute_cql3_query_result.class, metaDataMap);
- }
-
- public execute_cql3_query_result() {
- }
-
- public execute_cql3_query_result(
- CqlResult success,
- InvalidRequestException ire,
- UnavailableException ue,
- TimedOutException te,
- SchemaDisagreementException sde)
- {
- this();
- this.success = success;
- this.ire = ire;
- this.ue = ue;
- this.te = te;
- this.sde = sde;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public execute_cql3_query_result(execute_cql3_query_result other) {
- if (other.isSetSuccess()) {
- this.success = new CqlResult(other.success);
- }
- if (other.isSetIre()) {
- this.ire = new InvalidRequestException(other.ire);
- }
- if (other.isSetUe()) {
- this.ue = new UnavailableException(other.ue);
- }
- if (other.isSetTe()) {
- this.te = new TimedOutException(other.te);
- }
- if (other.isSetSde()) {
- this.sde = new SchemaDisagreementException(other.sde);
- }
- }
-
- public execute_cql3_query_result deepCopy() {
- return new execute_cql3_query_result(this);
- }
-
- @Override
- public void clear() {
- this.success = null;
- this.ire = null;
- this.ue = null;
- this.te = null;
- this.sde = null;
- }
-
- public CqlResult getSuccess() {
- return this.success;
- }
-
- public execute_cql3_query_result setSuccess(CqlResult success) {
- this.success = success;
- return this;
- }
-
- public void unsetSuccess() {
- this.success = null;
- }
-
- /** Returns true if field success is set (has been assigned a value) and false otherwise */
- public boolean isSetSuccess() {
- return this.success != null;
- }
-
- public void setSuccessIsSet(boolean value) {
- if (!value) {
- this.success = null;
- }
- }
-
- public InvalidRequestException getIre() {
- return this.ire;
- }
-
- public execute_cql3_query_result setIre(InvalidRequestException ire) {
- this.ire = ire;
- return this;
- }
-
- public void unsetIre() {
- this.ire = null;
- }
-
- /** Returns true if field ire is set (has been assigned a value) and false otherwise */
- public boolean isSetIre() {
- return this.ire != null;
- }
-
- public void setIreIsSet(boolean value) {
- if (!value) {
- this.ire = null;
- }
- }
-
- public UnavailableException getUe() {
- return this.ue;
- }
-
- public execute_cql3_query_result setUe(UnavailableException ue) {
- this.ue = ue;
- return this;
- }
-
- public void unsetUe() {
- this.ue = null;
- }
-
- /** Returns true if field ue is set (has been assigned a value) and false otherwise */
- public boolean isSetUe() {
- return this.ue != null;
- }
-
- public void setUeIsSet(boolean value) {
- if (!value) {
- this.ue = null;
- }
- }
-
- public TimedOutException getTe() {
- return this.te;
- }
-
- public execute_cql3_query_result setTe(TimedOutException te) {
- this.te = te;
- return this;
- }
-
- public void unsetTe() {
- this.te = null;
- }
-
- /** Returns true if field te is set (has been assigned a value) and false otherwise */
- public boolean isSetTe() {
- return this.te != null;
- }
-
- public void setTeIsSet(boolean value) {
- if (!value) {
- this.te = null;
- }
- }
-
- public SchemaDisagreementException getSde() {
- return this.sde;
- }
-
- public execute_cql3_query_result setSde(SchemaDisagreementException sde) {
- this.sde = sde;
- return this;
- }
-
- public void unsetSde() {
- this.sde = null;
- }
-
- /** Returns true if field sde is set (has been assigned a value) and false otherwise */
- public boolean isSetSde() {
- return this.sde != null;
- }
-
- public void setSdeIsSet(boolean value) {
- if (!value) {
- this.sde = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case SUCCESS:
- if (value == null) {
- unsetSuccess();
- } else {
- setSuccess((CqlResult)value);
- }
- break;
-
- case IRE:
- if (value == null) {
- unsetIre();
- } else {
- setIre((InvalidRequestException)value);
- }
- break;
-
- case UE:
- if (value == null) {
- unsetUe();
- } else {
- setUe((UnavailableException)value);
- }
- break;
-
- case TE:
- if (value == null) {
- unsetTe();
- } else {
- setTe((TimedOutException)value);
- }
- break;
-
- case SDE:
- if (value == null) {
- unsetSde();
- } else {
- setSde((SchemaDisagreementException)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case SUCCESS:
- return getSuccess();
-
- case IRE:
- return getIre();
-
- case UE:
- return getUe();
-
- case TE:
- return getTe();
-
- case SDE:
- return getSde();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case SUCCESS:
- return isSetSuccess();
- case IRE:
- return isSetIre();
- case UE:
- return isSetUe();
- case TE:
- return isSetTe();
- case SDE:
- return isSetSde();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof execute_cql3_query_result)
- return this.equals((execute_cql3_query_result)that);
- return false;
- }
-
- public boolean equals(execute_cql3_query_result that) {
- if (that == null)
- return false;
-
- boolean this_present_success = true && this.isSetSuccess();
- boolean that_present_success = true && that.isSetSuccess();
- if (this_present_success || that_present_success) {
- if (!(this_present_success && that_present_success))
- return false;
- if (!this.success.equals(that.success))
- return false;
- }
-
- boolean this_present_ire = true && this.isSetIre();
- boolean that_present_ire = true && that.isSetIre();
- if (this_present_ire || that_present_ire) {
- if (!(this_present_ire && that_present_ire))
- return false;
- if (!this.ire.equals(that.ire))
- return false;
- }
-
- boolean this_present_ue = true && this.isSetUe();
- boolean that_present_ue = true && that.isSetUe();
- if (this_present_ue || that_present_ue) {
- if (!(this_present_ue && that_present_ue))
- return false;
- if (!this.ue.equals(that.ue))
- return false;
- }
-
- boolean this_present_te = true && this.isSetTe();
- boolean that_present_te = true && that.isSetTe();
- if (this_present_te || that_present_te) {
- if (!(this_present_te && that_present_te))
- return false;
- if (!this.te.equals(that.te))
- return false;
- }
-
- boolean this_present_sde = true && this.isSetSde();
- boolean that_present_sde = true && that.isSetSde();
- if (this_present_sde || that_present_sde) {
- if (!(this_present_sde && that_present_sde))
- return false;
- if (!this.sde.equals(that.sde))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_success = true && (isSetSuccess());
- builder.append(present_success);
- if (present_success)
- builder.append(success);
-
- boolean present_ire = true && (isSetIre());
- builder.append(present_ire);
- if (present_ire)
- builder.append(ire);
-
- boolean present_ue = true && (isSetUe());
- builder.append(present_ue);
- if (present_ue)
- builder.append(ue);
-
- boolean present_te = true && (isSetTe());
- builder.append(present_te);
- if (present_te)
- builder.append(te);
-
- boolean present_sde = true && (isSetSde());
- builder.append(present_sde);
- if (present_sde)
- builder.append(sde);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(execute_cql3_query_result other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(other.isSetSuccess());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetSuccess()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, other.success);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetIre()).compareTo(other.isSetIre());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetIre()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.ire, other.ire);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetUe()).compareTo(other.isSetUe());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetUe()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.ue, other.ue);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetTe()).compareTo(other.isSetTe());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetTe()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.te, other.te);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetSde()).compareTo(other.isSetSde());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetSde()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.sde, other.sde);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("execute_cql3_query_result(");
- boolean first = true;
-
- sb.append("success:");
- if (this.success == null) {
- sb.append("null");
- } else {
- sb.append(this.success);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("ire:");
- if (this.ire == null) {
- sb.append("null");
- } else {
- sb.append(this.ire);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("ue:");
- if (this.ue == null) {
- sb.append("null");
- } else {
- sb.append(this.ue);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("te:");
- if (this.te == null) {
- sb.append("null");
- } else {
- sb.append(this.te);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("sde:");
- if (this.sde == null) {
- sb.append("null");
- } else {
- sb.append(this.sde);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // check for sub-struct validity
- if (success != null) {
- success.validate();
- }
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class execute_cql3_query_resultStandardSchemeFactory implements SchemeFactory {
- public execute_cql3_query_resultStandardScheme getScheme() {
- return new execute_cql3_query_resultStandardScheme();
- }
- }
-
- private static class execute_cql3_query_resultStandardScheme extends StandardScheme<execute_cql3_query_result> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, execute_cql3_query_result struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 0: // SUCCESS
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.success = new CqlResult();
- struct.success.read(iprot);
- struct.setSuccessIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 1: // IRE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.ire = new InvalidRequestException();
- struct.ire.read(iprot);
- struct.setIreIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 2: // UE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.ue = new UnavailableException();
- struct.ue.read(iprot);
- struct.setUeIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 3: // TE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.te = new TimedOutException();
- struct.te.read(iprot);
- struct.setTeIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 4: // SDE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.sde = new SchemaDisagreementException();
- struct.sde.read(iprot);
- struct.setSdeIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, execute_cql3_query_result struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.success != null) {
- oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
- struct.success.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.ire != null) {
- oprot.writeFieldBegin(IRE_FIELD_DESC);
- struct.ire.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.ue != null) {
- oprot.writeFieldBegin(UE_FIELD_DESC);
- struct.ue.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.te != null) {
- oprot.writeFieldBegin(TE_FIELD_DESC);
- struct.te.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.sde != null) {
- oprot.writeFieldBegin(SDE_FIELD_DESC);
- struct.sde.write(oprot);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class execute_cql3_query_resultTupleSchemeFactory implements SchemeFactory {
- public execute_cql3_query_resultTupleScheme getScheme() {
- return new execute_cql3_query_resultTupleScheme();
- }
- }
-
- private static class execute_cql3_query_resultTupleScheme extends TupleScheme<execute_cql3_query_result> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, execute_cql3_query_result struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- BitSet optionals = new BitSet();
- if (struct.isSetSuccess()) {
- optionals.set(0);
- }
- if (struct.isSetIre()) {
- optionals.set(1);
- }
- if (struct.isSetUe()) {
- optionals.set(2);
- }
- if (struct.isSetTe()) {
- optionals.set(3);
- }
- if (struct.isSetSde()) {
- optionals.set(4);
- }
- oprot.writeBitSet(optionals, 5);
- if (struct.isSetSuccess()) {
- struct.success.write(oprot);
- }
- if (struct.isSetIre()) {
- struct.ire.write(oprot);
- }
- if (struct.isSetUe()) {
- struct.ue.write(oprot);
- }
- if (struct.isSetTe()) {
- struct.te.write(oprot);
- }
- if (struct.isSetSde()) {
- struct.sde.write(oprot);
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, execute_cql3_query_result struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- BitSet incoming = iprot.readBitSet(5);
- if (incoming.get(0)) {
- struct.success = new CqlResult();
- struct.success.read(iprot);
- struct.setSuccessIsSet(true);
- }
- if (incoming.get(1)) {
- struct.ire = new InvalidRequestException();
- struct.ire.read(iprot);
- struct.setIreIsSet(true);
- }
- if (incoming.get(2)) {
- struct.ue = new UnavailableException();
- struct.ue.read(iprot);
- struct.setUeIsSet(true);
- }
- if (incoming.get(3)) {
- struct.te = new TimedOutException();
- struct.te.read(iprot);
- struct.setTeIsSet(true);
- }
- if (incoming.get(4)) {
- struct.sde = new SchemaDisagreementException();
- struct.sde.read(iprot);
- struct.setSdeIsSet(true);
- }
- }
- }
-
- }
-
- public static class prepare_cql_query_args implements org.apache.thrift.TBase<prepare_cql_query_args, prepare_cql_query_args._Fields>, java.io.Serializable, Cloneable, Comparable<prepare_cql_query_args> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("prepare_cql_query_args");
-
- private static final org.apache.thrift.protocol.TField QUERY_FIELD_DESC = new org.apache.thrift.protocol.TField("query", org.apache.thrift.protocol.TType.STRING, (short)1);
- private static final org.apache.thrift.protocol.TField COMPRESSION_FIELD_DESC = new org.apache.thrift.protocol.TField("compression", org.apache.thrift.protocol.TType.I32, (short)2);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new prepare_cql_query_argsStandardSchemeFactory());
- schemes.put(TupleScheme.class, new prepare_cql_query_argsTupleSchemeFactory());
- }
-
- public ByteBuffer query; // required
- /**
- *
- * @see Compression
- */
- public Compression compression; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- QUERY((short)1, "query"),
- /**
- *
- * @see Compression
- */
- COMPRESSION((short)2, "compression");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // QUERY
- return QUERY;
- case 2: // COMPRESSION
- return COMPRESSION;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.QUERY, new org.apache.thrift.meta_data.FieldMetaData("query", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true)));
- tmpMap.put(_Fields.COMPRESSION, new org.apache.thrift.meta_data.FieldMetaData("compression", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.EnumMetaData(org.apache.thrift.protocol.TType.ENUM, Compression.class)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(prepare_cql_query_args.class, metaDataMap);
- }
-
- public prepare_cql_query_args() {
- }
-
- public prepare_cql_query_args(
- ByteBuffer query,
- Compression compression)
- {
- this();
- this.query = query;
- this.compression = compression;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public prepare_cql_query_args(prepare_cql_query_args other) {
- if (other.isSetQuery()) {
- this.query = org.apache.thrift.TBaseHelper.copyBinary(other.query);
-;
- }
- if (other.isSetCompression()) {
- this.compression = other.compression;
- }
- }
-
- public prepare_cql_query_args deepCopy() {
- return new prepare_cql_query_args(this);
- }
-
- @Override
- public void clear() {
- this.query = null;
- this.compression = null;
- }
-
- public byte[] getQuery() {
- setQuery(org.apache.thrift.TBaseHelper.rightSize(query));
- return query == null ? null : query.array();
- }
-
- public ByteBuffer bufferForQuery() {
- return query;
- }
-
- public prepare_cql_query_args setQuery(byte[] query) {
- setQuery(query == null ? (ByteBuffer)null : ByteBuffer.wrap(query));
- return this;
- }
-
- public prepare_cql_query_args setQuery(ByteBuffer query) {
- this.query = query;
- return this;
- }
-
- public void unsetQuery() {
- this.query = null;
- }
-
- /** Returns true if field query is set (has been assigned a value) and false otherwise */
- public boolean isSetQuery() {
- return this.query != null;
- }
-
- public void setQueryIsSet(boolean value) {
- if (!value) {
- this.query = null;
- }
- }
-
- /**
- *
- * @see Compression
- */
- public Compression getCompression() {
- return this.compression;
- }
-
- /**
- *
- * @see Compression
- */
- public prepare_cql_query_args setCompression(Compression compression) {
- this.compression = compression;
- return this;
- }
-
- public void unsetCompression() {
- this.compression = null;
- }
-
- /** Returns true if field compression is set (has been assigned a value) and false otherwise */
- public boolean isSetCompression() {
- return this.compression != null;
- }
-
- public void setCompressionIsSet(boolean value) {
- if (!value) {
- this.compression = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case QUERY:
- if (value == null) {
- unsetQuery();
- } else {
- setQuery((ByteBuffer)value);
- }
- break;
-
- case COMPRESSION:
- if (value == null) {
- unsetCompression();
- } else {
- setCompression((Compression)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case QUERY:
- return getQuery();
-
- case COMPRESSION:
- return getCompression();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case QUERY:
- return isSetQuery();
- case COMPRESSION:
- return isSetCompression();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof prepare_cql_query_args)
- return this.equals((prepare_cql_query_args)that);
- return false;
- }
-
- public boolean equals(prepare_cql_query_args that) {
- if (that == null)
- return false;
-
- boolean this_present_query = true && this.isSetQuery();
- boolean that_present_query = true && that.isSetQuery();
- if (this_present_query || that_present_query) {
- if (!(this_present_query && that_present_query))
- return false;
- if (!this.query.equals(that.query))
- return false;
- }
-
- boolean this_present_compression = true && this.isSetCompression();
- boolean that_present_compression = true && that.isSetCompression();
- if (this_present_compression || that_present_compression) {
- if (!(this_present_compression && that_present_compression))
- return false;
- if (!this.compression.equals(that.compression))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_query = true && (isSetQuery());
- builder.append(present_query);
- if (present_query)
- builder.append(query);
-
- boolean present_compression = true && (isSetCompression());
- builder.append(present_compression);
- if (present_compression)
- builder.append(compression.getValue());
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(prepare_cql_query_args other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetQuery()).compareTo(other.isSetQuery());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetQuery()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.query, other.query);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetCompression()).compareTo(other.isSetCompression());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetCompression()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.compression, other.compression);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("prepare_cql_query_args(");
- boolean first = true;
-
- sb.append("query:");
- if (this.query == null) {
- sb.append("null");
- } else {
- org.apache.thrift.TBaseHelper.toString(this.query, sb);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("compression:");
- if (this.compression == null) {
- sb.append("null");
- } else {
- sb.append(this.compression);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- if (query == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'query' was not present! Struct: " + toString());
- }
- if (compression == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'compression' was not present! Struct: " + toString());
- }
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class prepare_cql_query_argsStandardSchemeFactory implements SchemeFactory {
- public prepare_cql_query_argsStandardScheme getScheme() {
- return new prepare_cql_query_argsStandardScheme();
- }
- }
-
- private static class prepare_cql_query_argsStandardScheme extends StandardScheme<prepare_cql_query_args> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, prepare_cql_query_args struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // QUERY
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.query = iprot.readBinary();
- struct.setQueryIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 2: // COMPRESSION
- if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
- struct.compression = Compression.findByValue(iprot.readI32());
- struct.setCompressionIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, prepare_cql_query_args struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.query != null) {
- oprot.writeFieldBegin(QUERY_FIELD_DESC);
- oprot.writeBinary(struct.query);
- oprot.writeFieldEnd();
- }
- if (struct.compression != null) {
- oprot.writeFieldBegin(COMPRESSION_FIELD_DESC);
- oprot.writeI32(struct.compression.getValue());
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class prepare_cql_query_argsTupleSchemeFactory implements SchemeFactory {
- public prepare_cql_query_argsTupleScheme getScheme() {
- return new prepare_cql_query_argsTupleScheme();
- }
- }
-
- private static class prepare_cql_query_argsTupleScheme extends TupleScheme<prepare_cql_query_args> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, prepare_cql_query_args struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- oprot.writeBinary(struct.query);
- oprot.writeI32(struct.compression.getValue());
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, prepare_cql_query_args struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- struct.query = iprot.readBinary();
- struct.setQueryIsSet(true);
- struct.compression = Compression.findByValue(iprot.readI32());
- struct.setCompressionIsSet(true);
- }
- }
-
- }
-
- public static class prepare_cql_query_result implements org.apache.thrift.TBase<prepare_cql_query_result, prepare_cql_query_result._Fields>, java.io.Serializable, Cloneable, Comparable<prepare_cql_query_result> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("prepare_cql_query_result");
-
- private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRUCT, (short)0);
- private static final org.apache.thrift.protocol.TField IRE_FIELD_DESC = new org.apache.thrift.protocol.TField("ire", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new prepare_cql_query_resultStandardSchemeFactory());
- schemes.put(TupleScheme.class, new prepare_cql_query_resultTupleSchemeFactory());
- }
-
- public CqlPreparedResult success; // required
- public InvalidRequestException ire; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- SUCCESS((short)0, "success"),
- IRE((short)1, "ire");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 0: // SUCCESS
- return SUCCESS;
- case 1: // IRE
- return IRE;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, CqlPreparedResult.class)));
- tmpMap.put(_Fields.IRE, new org.apache.thrift.meta_data.FieldMetaData("ire", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(prepare_cql_query_result.class, metaDataMap);
- }
-
- public prepare_cql_query_result() {
- }
-
- public prepare_cql_query_result(
- CqlPreparedResult success,
- InvalidRequestException ire)
- {
- this();
- this.success = success;
- this.ire = ire;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public prepare_cql_query_result(prepare_cql_query_result other) {
- if (other.isSetSuccess()) {
- this.success = new CqlPreparedResult(other.success);
- }
- if (other.isSetIre()) {
- this.ire = new InvalidRequestException(other.ire);
- }
- }
-
- public prepare_cql_query_result deepCopy() {
- return new prepare_cql_query_result(this);
- }
-
- @Override
- public void clear() {
- this.success = null;
- this.ire = null;
- }
-
- public CqlPreparedResult getSuccess() {
- return this.success;
- }
-
- public prepare_cql_query_result setSuccess(CqlPreparedResult success) {
- this.success = success;
- return this;
- }
-
- public void unsetSuccess() {
- this.success = null;
- }
-
- /** Returns true if field success is set (has been assigned a value) and false otherwise */
- public boolean isSetSuccess() {
- return this.success != null;
- }
-
- public void setSuccessIsSet(boolean value) {
- if (!value) {
- this.success = null;
- }
- }
-
- public InvalidRequestException getIre() {
- return this.ire;
- }
-
- public prepare_cql_query_result setIre(InvalidRequestException ire) {
- this.ire = ire;
- return this;
- }
-
- public void unsetIre() {
- this.ire = null;
- }
-
- /** Returns true if field ire is set (has been assigned a value) and false otherwise */
- public boolean isSetIre() {
- return this.ire != null;
- }
-
- public void setIreIsSet(boolean value) {
- if (!value) {
- this.ire = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case SUCCESS:
- if (value == null) {
- unsetSuccess();
- } else {
- setSuccess((CqlPreparedResult)value);
- }
- break;
-
- case IRE:
- if (value == null) {
- unsetIre();
- } else {
- setIre((InvalidRequestException)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case SUCCESS:
- return getSuccess();
-
- case IRE:
- return getIre();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case SUCCESS:
- return isSetSuccess();
- case IRE:
- return isSetIre();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof prepare_cql_query_result)
- return this.equals((prepare_cql_query_result)that);
- return false;
- }
-
- public boolean equals(prepare_cql_query_result that) {
- if (that == null)
- return false;
-
- boolean this_present_success = true && this.isSetSuccess();
- boolean that_present_success = true && that.isSetSuccess();
- if (this_present_success || that_present_success) {
- if (!(this_present_success && that_present_success))
- return false;
- if (!this.success.equals(that.success))
- return false;
- }
-
- boolean this_present_ire = true && this.isSetIre();
- boolean that_present_ire = true && that.isSetIre();
- if (this_present_ire || that_present_ire) {
- if (!(this_present_ire && that_present_ire))
- return false;
- if (!this.ire.equals(that.ire))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_success = true && (isSetSuccess());
- builder.append(present_success);
- if (present_success)
- builder.append(success);
-
- boolean present_ire = true && (isSetIre());
- builder.append(present_ire);
- if (present_ire)
- builder.append(ire);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(prepare_cql_query_result other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(other.isSetSuccess());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetSuccess()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, other.success);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetIre()).compareTo(other.isSetIre());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetIre()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.ire, other.ire);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("prepare_cql_query_result(");
- boolean first = true;
-
- sb.append("success:");
- if (this.success == null) {
- sb.append("null");
- } else {
- sb.append(this.success);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("ire:");
- if (this.ire == null) {
- sb.append("null");
- } else {
- sb.append(this.ire);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // check for sub-struct validity
- if (success != null) {
- success.validate();
- }
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class prepare_cql_query_resultStandardSchemeFactory implements SchemeFactory {
- public prepare_cql_query_resultStandardScheme getScheme() {
- return new prepare_cql_query_resultStandardScheme();
- }
- }
-
- private static class prepare_cql_query_resultStandardScheme extends StandardScheme<prepare_cql_query_result> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, prepare_cql_query_result struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 0: // SUCCESS
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.success = new CqlPreparedResult();
- struct.success.read(iprot);
- struct.setSuccessIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 1: // IRE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.ire = new InvalidRequestException();
- struct.ire.read(iprot);
- struct.setIreIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, prepare_cql_query_result struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.success != null) {
- oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
- struct.success.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.ire != null) {
- oprot.writeFieldBegin(IRE_FIELD_DESC);
- struct.ire.write(oprot);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class prepare_cql_query_resultTupleSchemeFactory implements SchemeFactory {
- public prepare_cql_query_resultTupleScheme getScheme() {
- return new prepare_cql_query_resultTupleScheme();
- }
- }
-
- private static class prepare_cql_query_resultTupleScheme extends TupleScheme<prepare_cql_query_result> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, prepare_cql_query_result struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- BitSet optionals = new BitSet();
- if (struct.isSetSuccess()) {
- optionals.set(0);
- }
- if (struct.isSetIre()) {
- optionals.set(1);
- }
- oprot.writeBitSet(optionals, 2);
- if (struct.isSetSuccess()) {
- struct.success.write(oprot);
- }
- if (struct.isSetIre()) {
- struct.ire.write(oprot);
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, prepare_cql_query_result struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- BitSet incoming = iprot.readBitSet(2);
- if (incoming.get(0)) {
- struct.success = new CqlPreparedResult();
- struct.success.read(iprot);
- struct.setSuccessIsSet(true);
- }
- if (incoming.get(1)) {
- struct.ire = new InvalidRequestException();
- struct.ire.read(iprot);
- struct.setIreIsSet(true);
- }
- }
- }
-
- }
-
- public static class prepare_cql3_query_args implements org.apache.thrift.TBase<prepare_cql3_query_args, prepare_cql3_query_args._Fields>, java.io.Serializable, Cloneable, Comparable<prepare_cql3_query_args> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("prepare_cql3_query_args");
-
- private static final org.apache.thrift.protocol.TField QUERY_FIELD_DESC = new org.apache.thrift.protocol.TField("query", org.apache.thrift.protocol.TType.STRING, (short)1);
- private static final org.apache.thrift.protocol.TField COMPRESSION_FIELD_DESC = new org.apache.thrift.protocol.TField("compression", org.apache.thrift.protocol.TType.I32, (short)2);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new prepare_cql3_query_argsStandardSchemeFactory());
- schemes.put(TupleScheme.class, new prepare_cql3_query_argsTupleSchemeFactory());
- }
-
- public ByteBuffer query; // required
- /**
- *
- * @see Compression
- */
- public Compression compression; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- QUERY((short)1, "query"),
- /**
- *
- * @see Compression
- */
- COMPRESSION((short)2, "compression");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // QUERY
- return QUERY;
- case 2: // COMPRESSION
- return COMPRESSION;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.QUERY, new org.apache.thrift.meta_data.FieldMetaData("query", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true)));
- tmpMap.put(_Fields.COMPRESSION, new org.apache.thrift.meta_data.FieldMetaData("compression", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.EnumMetaData(org.apache.thrift.protocol.TType.ENUM, Compression.class)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(prepare_cql3_query_args.class, metaDataMap);
- }
-
- public prepare_cql3_query_args() {
- }
-
- public prepare_cql3_query_args(
- ByteBuffer query,
- Compression compression)
- {
- this();
- this.query = query;
- this.compression = compression;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public prepare_cql3_query_args(prepare_cql3_query_args other) {
- if (other.isSetQuery()) {
- this.query = org.apache.thrift.TBaseHelper.copyBinary(other.query);
-;
- }
- if (other.isSetCompression()) {
- this.compression = other.compression;
- }
- }
-
- public prepare_cql3_query_args deepCopy() {
- return new prepare_cql3_query_args(this);
- }
-
- @Override
- public void clear() {
- this.query = null;
- this.compression = null;
- }
-
- public byte[] getQuery() {
- setQuery(org.apache.thrift.TBaseHelper.rightSize(query));
- return query == null ? null : query.array();
- }
-
- public ByteBuffer bufferForQuery() {
- return query;
- }
-
- public prepare_cql3_query_args setQuery(byte[] query) {
- setQuery(query == null ? (ByteBuffer)null : ByteBuffer.wrap(query));
- return this;
- }
-
- public prepare_cql3_query_args setQuery(ByteBuffer query) {
- this.query = query;
- return this;
- }
-
- public void unsetQuery() {
- this.query = null;
- }
-
- /** Returns true if field query is set (has been assigned a value) and false otherwise */
- public boolean isSetQuery() {
- return this.query != null;
- }
-
- public void setQueryIsSet(boolean value) {
- if (!value) {
- this.query = null;
- }
- }
-
- /**
- *
- * @see Compression
- */
- public Compression getCompression() {
- return this.compression;
- }
-
- /**
- *
- * @see Compression
- */
- public prepare_cql3_query_args setCompression(Compression compression) {
- this.compression = compression;
- return this;
- }
-
- public void unsetCompression() {
- this.compression = null;
- }
-
- /** Returns true if field compression is set (has been assigned a value) and false otherwise */
- public boolean isSetCompression() {
- return this.compression != null;
- }
-
- public void setCompressionIsSet(boolean value) {
- if (!value) {
- this.compression = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case QUERY:
- if (value == null) {
- unsetQuery();
- } else {
- setQuery((ByteBuffer)value);
- }
- break;
-
- case COMPRESSION:
- if (value == null) {
- unsetCompression();
- } else {
- setCompression((Compression)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case QUERY:
- return getQuery();
-
- case COMPRESSION:
- return getCompression();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case QUERY:
- return isSetQuery();
- case COMPRESSION:
- return isSetCompression();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof prepare_cql3_query_args)
- return this.equals((prepare_cql3_query_args)that);
- return false;
- }
-
- public boolean equals(prepare_cql3_query_args that) {
- if (that == null)
- return false;
-
- boolean this_present_query = true && this.isSetQuery();
- boolean that_present_query = true && that.isSetQuery();
- if (this_present_query || that_present_query) {
- if (!(this_present_query && that_present_query))
- return false;
- if (!this.query.equals(that.query))
- return false;
- }
-
- boolean this_present_compression = true && this.isSetCompression();
- boolean that_present_compression = true && that.isSetCompression();
- if (this_present_compression || that_present_compression) {
- if (!(this_present_compression && that_present_compression))
- return false;
- if (!this.compression.equals(that.compression))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_query = true && (isSetQuery());
- builder.append(present_query);
- if (present_query)
- builder.append(query);
-
- boolean present_compression = true && (isSetCompression());
- builder.append(present_compression);
- if (present_compression)
- builder.append(compression.getValue());
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(prepare_cql3_query_args other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetQuery()).compareTo(other.isSetQuery());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetQuery()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.query, other.query);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetCompression()).compareTo(other.isSetCompression());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetCompression()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.compression, other.compression);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("prepare_cql3_query_args(");
- boolean first = true;
-
- sb.append("query:");
- if (this.query == null) {
- sb.append("null");
- } else {
- org.apache.thrift.TBaseHelper.toString(this.query, sb);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("compression:");
- if (this.compression == null) {
- sb.append("null");
- } else {
- sb.append(this.compression);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- if (query == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'query' was not present! Struct: " + toString());
- }
- if (compression == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'compression' was not present! Struct: " + toString());
- }
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class prepare_cql3_query_argsStandardSchemeFactory implements SchemeFactory {
- public prepare_cql3_query_argsStandardScheme getScheme() {
- return new prepare_cql3_query_argsStandardScheme();
- }
- }
-
- private static class prepare_cql3_query_argsStandardScheme extends StandardScheme<prepare_cql3_query_args> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, prepare_cql3_query_args struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // QUERY
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.query = iprot.readBinary();
- struct.setQueryIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 2: // COMPRESSION
- if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
- struct.compression = Compression.findByValue(iprot.readI32());
- struct.setCompressionIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, prepare_cql3_query_args struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.query != null) {
- oprot.writeFieldBegin(QUERY_FIELD_DESC);
- oprot.writeBinary(struct.query);
- oprot.writeFieldEnd();
- }
- if (struct.compression != null) {
- oprot.writeFieldBegin(COMPRESSION_FIELD_DESC);
- oprot.writeI32(struct.compression.getValue());
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class prepare_cql3_query_argsTupleSchemeFactory implements SchemeFactory {
- public prepare_cql3_query_argsTupleScheme getScheme() {
- return new prepare_cql3_query_argsTupleScheme();
- }
- }
-
- private static class prepare_cql3_query_argsTupleScheme extends TupleScheme<prepare_cql3_query_args> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, prepare_cql3_query_args struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- oprot.writeBinary(struct.query);
- oprot.writeI32(struct.compression.getValue());
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, prepare_cql3_query_args struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- struct.query = iprot.readBinary();
- struct.setQueryIsSet(true);
- struct.compression = Compression.findByValue(iprot.readI32());
- struct.setCompressionIsSet(true);
- }
- }
-
- }
-
- public static class prepare_cql3_query_result implements org.apache.thrift.TBase<prepare_cql3_query_result, prepare_cql3_query_result._Fields>, java.io.Serializable, Cloneable, Comparable<prepare_cql3_query_result> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("prepare_cql3_query_result");
-
- private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRUCT, (short)0);
- private static final org.apache.thrift.protocol.TField IRE_FIELD_DESC = new org.apache.thrift.protocol.TField("ire", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new prepare_cql3_query_resultStandardSchemeFactory());
- schemes.put(TupleScheme.class, new prepare_cql3_query_resultTupleSchemeFactory());
- }
-
- public CqlPreparedResult success; // required
- public InvalidRequestException ire; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- SUCCESS((short)0, "success"),
- IRE((short)1, "ire");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 0: // SUCCESS
- return SUCCESS;
- case 1: // IRE
- return IRE;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, CqlPreparedResult.class)));
- tmpMap.put(_Fields.IRE, new org.apache.thrift.meta_data.FieldMetaData("ire", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(prepare_cql3_query_result.class, metaDataMap);
- }
-
- public prepare_cql3_query_result() {
- }
-
- public prepare_cql3_query_result(
- CqlPreparedResult success,
- InvalidRequestException ire)
- {
- this();
- this.success = success;
- this.ire = ire;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public prepare_cql3_query_result(prepare_cql3_query_result other) {
- if (other.isSetSuccess()) {
- this.success = new CqlPreparedResult(other.success);
- }
- if (other.isSetIre()) {
- this.ire = new InvalidRequestException(other.ire);
- }
- }
-
- public prepare_cql3_query_result deepCopy() {
- return new prepare_cql3_query_result(this);
- }
-
- @Override
- public void clear() {
- this.success = null;
- this.ire = null;
- }
-
- public CqlPreparedResult getSuccess() {
- return this.success;
- }
-
- public prepare_cql3_query_result setSuccess(CqlPreparedResult success) {
- this.success = success;
- return this;
- }
-
- public void unsetSuccess() {
- this.success = null;
- }
-
- /** Returns true if field success is set (has been assigned a value) and false otherwise */
- public boolean isSetSuccess() {
- return this.success != null;
- }
-
- public void setSuccessIsSet(boolean value) {
- if (!value) {
- this.success = null;
- }
- }
-
- public InvalidRequestException getIre() {
- return this.ire;
- }
-
- public prepare_cql3_query_result setIre(InvalidRequestException ire) {
- this.ire = ire;
- return this;
- }
-
- public void unsetIre() {
- this.ire = null;
- }
-
- /** Returns true if field ire is set (has been assigned a value) and false otherwise */
- public boolean isSetIre() {
- return this.ire != null;
- }
-
- public void setIreIsSet(boolean value) {
- if (!value) {
- this.ire = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case SUCCESS:
- if (value == null) {
- unsetSuccess();
- } else {
- setSuccess((CqlPreparedResult)value);
- }
- break;
-
- case IRE:
- if (value == null) {
- unsetIre();
- } else {
- setIre((InvalidRequestException)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case SUCCESS:
- return getSuccess();
-
- case IRE:
- return getIre();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case SUCCESS:
- return isSetSuccess();
- case IRE:
- return isSetIre();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof prepare_cql3_query_result)
- return this.equals((prepare_cql3_query_result)that);
- return false;
- }
-
- public boolean equals(prepare_cql3_query_result that) {
- if (that == null)
- return false;
-
- boolean this_present_success = true && this.isSetSuccess();
- boolean that_present_success = true && that.isSetSuccess();
- if (this_present_success || that_present_success) {
- if (!(this_present_success && that_present_success))
- return false;
- if (!this.success.equals(that.success))
- return false;
- }
-
- boolean this_present_ire = true && this.isSetIre();
- boolean that_present_ire = true && that.isSetIre();
- if (this_present_ire || that_present_ire) {
- if (!(this_present_ire && that_present_ire))
- return false;
- if (!this.ire.equals(that.ire))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_success = true && (isSetSuccess());
- builder.append(present_success);
- if (present_success)
- builder.append(success);
-
- boolean present_ire = true && (isSetIre());
- builder.append(present_ire);
- if (present_ire)
- builder.append(ire);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(prepare_cql3_query_result other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(other.isSetSuccess());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetSuccess()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, other.success);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetIre()).compareTo(other.isSetIre());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetIre()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.ire, other.ire);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("prepare_cql3_query_result(");
- boolean first = true;
-
- sb.append("success:");
- if (this.success == null) {
- sb.append("null");
- } else {
- sb.append(this.success);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("ire:");
- if (this.ire == null) {
- sb.append("null");
- } else {
- sb.append(this.ire);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // check for sub-struct validity
- if (success != null) {
- success.validate();
- }
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class prepare_cql3_query_resultStandardSchemeFactory implements SchemeFactory {
- public prepare_cql3_query_resultStandardScheme getScheme() {
- return new prepare_cql3_query_resultStandardScheme();
- }
- }
-
- private static class prepare_cql3_query_resultStandardScheme extends StandardScheme<prepare_cql3_query_result> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, prepare_cql3_query_result struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 0: // SUCCESS
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.success = new CqlPreparedResult();
- struct.success.read(iprot);
- struct.setSuccessIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 1: // IRE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.ire = new InvalidRequestException();
- struct.ire.read(iprot);
- struct.setIreIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, prepare_cql3_query_result struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.success != null) {
- oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
- struct.success.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.ire != null) {
- oprot.writeFieldBegin(IRE_FIELD_DESC);
- struct.ire.write(oprot);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class prepare_cql3_query_resultTupleSchemeFactory implements SchemeFactory {
- public prepare_cql3_query_resultTupleScheme getScheme() {
- return new prepare_cql3_query_resultTupleScheme();
- }
- }
-
- private static class prepare_cql3_query_resultTupleScheme extends TupleScheme<prepare_cql3_query_result> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, prepare_cql3_query_result struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- BitSet optionals = new BitSet();
- if (struct.isSetSuccess()) {
- optionals.set(0);
- }
- if (struct.isSetIre()) {
- optionals.set(1);
- }
- oprot.writeBitSet(optionals, 2);
- if (struct.isSetSuccess()) {
- struct.success.write(oprot);
- }
- if (struct.isSetIre()) {
- struct.ire.write(oprot);
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, prepare_cql3_query_result struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- BitSet incoming = iprot.readBitSet(2);
- if (incoming.get(0)) {
- struct.success = new CqlPreparedResult();
- struct.success.read(iprot);
- struct.setSuccessIsSet(true);
- }
- if (incoming.get(1)) {
- struct.ire = new InvalidRequestException();
- struct.ire.read(iprot);
- struct.setIreIsSet(true);
- }
- }
- }
-
- }
-
- public static class execute_prepared_cql_query_args implements org.apache.thrift.TBase<execute_prepared_cql_query_args, execute_prepared_cql_query_args._Fields>, java.io.Serializable, Cloneable, Comparable<execute_prepared_cql_query_args> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("execute_prepared_cql_query_args");
-
- private static final org.apache.thrift.protocol.TField ITEM_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("itemId", org.apache.thrift.protocol.TType.I32, (short)1);
- private static final org.apache.thrift.protocol.TField VALUES_FIELD_DESC = new org.apache.thrift.protocol.TField("values", org.apache.thrift.protocol.TType.LIST, (short)2);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new execute_prepared_cql_query_argsStandardSchemeFactory());
- schemes.put(TupleScheme.class, new execute_prepared_cql_query_argsTupleSchemeFactory());
- }
-
- public int itemId; // required
- public List<ByteBuffer> values; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- ITEM_ID((short)1, "itemId"),
- VALUES((short)2, "values");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // ITEM_ID
- return ITEM_ID;
- case 2: // VALUES
- return VALUES;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- private static final int __ITEMID_ISSET_ID = 0;
- private byte __isset_bitfield = 0;
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.ITEM_ID, new org.apache.thrift.meta_data.FieldMetaData("itemId", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32)));
- tmpMap.put(_Fields.VALUES, new org.apache.thrift.meta_data.FieldMetaData("values", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true))));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(execute_prepared_cql_query_args.class, metaDataMap);
- }
-
- public execute_prepared_cql_query_args() {
- }
-
- public execute_prepared_cql_query_args(
- int itemId,
- List<ByteBuffer> values)
- {
- this();
- this.itemId = itemId;
- setItemIdIsSet(true);
- this.values = values;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public execute_prepared_cql_query_args(execute_prepared_cql_query_args other) {
- __isset_bitfield = other.__isset_bitfield;
- this.itemId = other.itemId;
- if (other.isSetValues()) {
- List<ByteBuffer> __this__values = new ArrayList<ByteBuffer>(other.values);
- this.values = __this__values;
- }
- }
-
- public execute_prepared_cql_query_args deepCopy() {
- return new execute_prepared_cql_query_args(this);
- }
-
- @Override
- public void clear() {
- setItemIdIsSet(false);
- this.itemId = 0;
- this.values = null;
- }
-
- public int getItemId() {
- return this.itemId;
- }
-
- public execute_prepared_cql_query_args setItemId(int itemId) {
- this.itemId = itemId;
- setItemIdIsSet(true);
- return this;
- }
-
- public void unsetItemId() {
- __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __ITEMID_ISSET_ID);
- }
-
- /** Returns true if field itemId is set (has been assigned a value) and false otherwise */
- public boolean isSetItemId() {
- return EncodingUtils.testBit(__isset_bitfield, __ITEMID_ISSET_ID);
- }
-
- public void setItemIdIsSet(boolean value) {
- __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __ITEMID_ISSET_ID, value);
- }
-
- public int getValuesSize() {
- return (this.values == null) ? 0 : this.values.size();
- }
-
- public java.util.Iterator<ByteBuffer> getValuesIterator() {
- return (this.values == null) ? null : this.values.iterator();
- }
-
- public void addToValues(ByteBuffer elem) {
- if (this.values == null) {
- this.values = new ArrayList<ByteBuffer>();
- }
- this.values.add(elem);
- }
-
- public List<ByteBuffer> getValues() {
- return this.values;
- }
-
- public execute_prepared_cql_query_args setValues(List<ByteBuffer> values) {
- this.values = values;
- return this;
- }
-
- public void unsetValues() {
- this.values = null;
- }
-
- /** Returns true if field values is set (has been assigned a value) and false otherwise */
- public boolean isSetValues() {
- return this.values != null;
- }
-
- public void setValuesIsSet(boolean value) {
- if (!value) {
- this.values = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case ITEM_ID:
- if (value == null) {
- unsetItemId();
- } else {
- setItemId((Integer)value);
- }
- break;
-
- case VALUES:
- if (value == null) {
- unsetValues();
- } else {
- setValues((List<ByteBuffer>)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case ITEM_ID:
- return Integer.valueOf(getItemId());
-
- case VALUES:
- return getValues();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case ITEM_ID:
- return isSetItemId();
- case VALUES:
- return isSetValues();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof execute_prepared_cql_query_args)
- return this.equals((execute_prepared_cql_query_args)that);
- return false;
- }
-
- public boolean equals(execute_prepared_cql_query_args that) {
- if (that == null)
- return false;
-
- boolean this_present_itemId = true;
- boolean that_present_itemId = true;
- if (this_present_itemId || that_present_itemId) {
- if (!(this_present_itemId && that_present_itemId))
- return false;
- if (this.itemId != that.itemId)
- return false;
- }
-
- boolean this_present_values = true && this.isSetValues();
- boolean that_present_values = true && that.isSetValues();
- if (this_present_values || that_present_values) {
- if (!(this_present_values && that_present_values))
- return false;
- if (!this.values.equals(that.values))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_itemId = true;
- builder.append(present_itemId);
- if (present_itemId)
- builder.append(itemId);
-
- boolean present_values = true && (isSetValues());
- builder.append(present_values);
- if (present_values)
- builder.append(values);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(execute_prepared_cql_query_args other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetItemId()).compareTo(other.isSetItemId());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetItemId()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.itemId, other.itemId);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetValues()).compareTo(other.isSetValues());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetValues()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.values, other.values);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("execute_prepared_cql_query_args(");
- boolean first = true;
-
- sb.append("itemId:");
- sb.append(this.itemId);
- first = false;
- if (!first) sb.append(", ");
- sb.append("values:");
- if (this.values == null) {
- sb.append("null");
- } else {
- sb.append(this.values);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // alas, we cannot check 'itemId' because it's a primitive and you chose the non-beans generator.
- if (values == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'values' was not present! Struct: " + toString());
- }
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
- __isset_bitfield = 0;
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class execute_prepared_cql_query_argsStandardSchemeFactory implements SchemeFactory {
- public execute_prepared_cql_query_argsStandardScheme getScheme() {
- return new execute_prepared_cql_query_argsStandardScheme();
- }
- }
-
- private static class execute_prepared_cql_query_argsStandardScheme extends StandardScheme<execute_prepared_cql_query_args> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, execute_prepared_cql_query_args struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // ITEM_ID
- if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
- struct.itemId = iprot.readI32();
- struct.setItemIdIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 2: // VALUES
- if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
- {
- org.apache.thrift.protocol.TList _list448 = iprot.readListBegin();
- struct.values = new ArrayList<ByteBuffer>(_list448.size);
- for (int _i449 = 0; _i449 < _list448.size; ++_i449)
- {
- ByteBuffer _elem450;
- _elem450 = iprot.readBinary();
- struct.values.add(_elem450);
- }
- iprot.readListEnd();
- }
- struct.setValuesIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- if (!struct.isSetItemId()) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'itemId' was not found in serialized data! Struct: " + toString());
- }
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, execute_prepared_cql_query_args struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- oprot.writeFieldBegin(ITEM_ID_FIELD_DESC);
- oprot.writeI32(struct.itemId);
- oprot.writeFieldEnd();
- if (struct.values != null) {
- oprot.writeFieldBegin(VALUES_FIELD_DESC);
- {
- oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, struct.values.size()));
- for (ByteBuffer _iter451 : struct.values)
- {
- oprot.writeBinary(_iter451);
- }
- oprot.writeListEnd();
- }
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class execute_prepared_cql_query_argsTupleSchemeFactory implements SchemeFactory {
- public execute_prepared_cql_query_argsTupleScheme getScheme() {
- return new execute_prepared_cql_query_argsTupleScheme();
- }
- }
-
- private static class execute_prepared_cql_query_argsTupleScheme extends TupleScheme<execute_prepared_cql_query_args> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, execute_prepared_cql_query_args struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- oprot.writeI32(struct.itemId);
- {
- oprot.writeI32(struct.values.size());
- for (ByteBuffer _iter452 : struct.values)
- {
- oprot.writeBinary(_iter452);
- }
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, execute_prepared_cql_query_args struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- struct.itemId = iprot.readI32();
- struct.setItemIdIsSet(true);
- {
- org.apache.thrift.protocol.TList _list453 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32());
- struct.values = new ArrayList<ByteBuffer>(_list453.size);
- for (int _i454 = 0; _i454 < _list453.size; ++_i454)
- {
- ByteBuffer _elem455;
- _elem455 = iprot.readBinary();
- struct.values.add(_elem455);
- }
- }
- struct.setValuesIsSet(true);
- }
- }
-
- }
-
- public static class execute_prepared_cql_query_result implements org.apache.thrift.TBase<execute_prepared_cql_query_result, execute_prepared_cql_query_result._Fields>, java.io.Serializable, Cloneable, Comparable<execute_prepared_cql_query_result> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("execute_prepared_cql_query_result");
-
- private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRUCT, (short)0);
- private static final org.apache.thrift.protocol.TField IRE_FIELD_DESC = new org.apache.thrift.protocol.TField("ire", org.apache.thrift.protocol.TType.STRUCT, (short)1);
- private static final org.apache.thrift.protocol.TField UE_FIELD_DESC = new org.apache.thrift.protocol.TField("ue", org.apache.thrift.protocol.TType.STRUCT, (short)2);
- private static final org.apache.thrift.protocol.TField TE_FIELD_DESC = new org.apache.thrift.protocol.TField("te", org.apache.thrift.protocol.TType.STRUCT, (short)3);
- private static final org.apache.thrift.protocol.TField SDE_FIELD_DESC = new org.apache.thrift.protocol.TField("sde", org.apache.thrift.protocol.TType.STRUCT, (short)4);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new execute_prepared_cql_query_resultStandardSchemeFactory());
- schemes.put(TupleScheme.class, new execute_prepared_cql_query_resultTupleSchemeFactory());
- }
-
- public CqlResult success; // required
- public InvalidRequestException ire; // required
- public UnavailableException ue; // required
- public TimedOutException te; // required
- public SchemaDisagreementException sde; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- SUCCESS((short)0, "success"),
- IRE((short)1, "ire"),
- UE((short)2, "ue"),
- TE((short)3, "te"),
- SDE((short)4, "sde");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 0: // SUCCESS
- return SUCCESS;
- case 1: // IRE
- return IRE;
- case 2: // UE
- return UE;
- case 3: // TE
- return TE;
- case 4: // SDE
- return SDE;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, CqlResult.class)));
- tmpMap.put(_Fields.IRE, new org.apache.thrift.meta_data.FieldMetaData("ire", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- tmpMap.put(_Fields.UE, new org.apache.thrift.meta_data.FieldMetaData("ue", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- tmpMap.put(_Fields.TE, new org.apache.thrift.meta_data.FieldMetaData("te", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- tmpMap.put(_Fields.SDE, new org.apache.thrift.meta_data.FieldMetaData("sde", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(execute_prepared_cql_query_result.class, metaDataMap);
- }
-
- public execute_prepared_cql_query_result() {
- }
-
- public execute_prepared_cql_query_result(
- CqlResult success,
- InvalidRequestException ire,
- UnavailableException ue,
- TimedOutException te,
- SchemaDisagreementException sde)
- {
- this();
- this.success = success;
- this.ire = ire;
- this.ue = ue;
- this.te = te;
- this.sde = sde;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public execute_prepared_cql_query_result(execute_prepared_cql_query_result other) {
- if (other.isSetSuccess()) {
- this.success = new CqlResult(other.success);
- }
- if (other.isSetIre()) {
- this.ire = new InvalidRequestException(other.ire);
- }
- if (other.isSetUe()) {
- this.ue = new UnavailableException(other.ue);
- }
- if (other.isSetTe()) {
- this.te = new TimedOutException(other.te);
- }
- if (other.isSetSde()) {
- this.sde = new SchemaDisagreementException(other.sde);
- }
- }
-
- public execute_prepared_cql_query_result deepCopy() {
- return new execute_prepared_cql_query_result(this);
- }
-
- @Override
- public void clear() {
- this.success = null;
- this.ire = null;
- this.ue = null;
- this.te = null;
- this.sde = null;
- }
-
- public CqlResult getSuccess() {
- return this.success;
- }
-
- public execute_prepared_cql_query_result setSuccess(CqlResult success) {
- this.success = success;
- return this;
- }
-
- public void unsetSuccess() {
- this.success = null;
- }
-
- /** Returns true if field success is set (has been assigned a value) and false otherwise */
- public boolean isSetSuccess() {
- return this.success != null;
- }
-
- public void setSuccessIsSet(boolean value) {
- if (!value) {
- this.success = null;
- }
- }
-
- public InvalidRequestException getIre() {
- return this.ire;
- }
-
- public execute_prepared_cql_query_result setIre(InvalidRequestException ire) {
- this.ire = ire;
- return this;
- }
-
- public void unsetIre() {
- this.ire = null;
- }
-
- /** Returns true if field ire is set (has been assigned a value) and false otherwise */
- public boolean isSetIre() {
- return this.ire != null;
- }
-
- public void setIreIsSet(boolean value) {
- if (!value) {
- this.ire = null;
- }
- }
-
- public UnavailableException getUe() {
- return this.ue;
- }
-
- public execute_prepared_cql_query_result setUe(UnavailableException ue) {
- this.ue = ue;
- return this;
- }
-
- public void unsetUe() {
- this.ue = null;
- }
-
- /** Returns true if field ue is set (has been assigned a value) and false otherwise */
- public boolean isSetUe() {
- return this.ue != null;
- }
-
- public void setUeIsSet(boolean value) {
- if (!value) {
- this.ue = null;
- }
- }
-
- public TimedOutException getTe() {
- return this.te;
- }
-
- public execute_prepared_cql_query_result setTe(TimedOutException te) {
- this.te = te;
- return this;
- }
-
- public void unsetTe() {
- this.te = null;
- }
-
- /** Returns true if field te is set (has been assigned a value) and false otherwise */
- public boolean isSetTe() {
- return this.te != null;
- }
-
- public void setTeIsSet(boolean value) {
- if (!value) {
- this.te = null;
- }
- }
-
- public SchemaDisagreementException getSde() {
- return this.sde;
- }
-
- public execute_prepared_cql_query_result setSde(SchemaDisagreementException sde) {
- this.sde = sde;
- return this;
- }
-
- public void unsetSde() {
- this.sde = null;
- }
-
- /** Returns true if field sde is set (has been assigned a value) and false otherwise */
- public boolean isSetSde() {
- return this.sde != null;
- }
-
- public void setSdeIsSet(boolean value) {
- if (!value) {
- this.sde = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case SUCCESS:
- if (value == null) {
- unsetSuccess();
- } else {
- setSuccess((CqlResult)value);
- }
- break;
-
- case IRE:
- if (value == null) {
- unsetIre();
- } else {
- setIre((InvalidRequestException)value);
- }
- break;
-
- case UE:
- if (value == null) {
- unsetUe();
- } else {
- setUe((UnavailableException)value);
- }
- break;
-
- case TE:
- if (value == null) {
- unsetTe();
- } else {
- setTe((TimedOutException)value);
- }
- break;
-
- case SDE:
- if (value == null) {
- unsetSde();
- } else {
- setSde((SchemaDisagreementException)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case SUCCESS:
- return getSuccess();
-
- case IRE:
- return getIre();
-
- case UE:
- return getUe();
-
- case TE:
- return getTe();
-
- case SDE:
- return getSde();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case SUCCESS:
- return isSetSuccess();
- case IRE:
- return isSetIre();
- case UE:
- return isSetUe();
- case TE:
- return isSetTe();
- case SDE:
- return isSetSde();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof execute_prepared_cql_query_result)
- return this.equals((execute_prepared_cql_query_result)that);
- return false;
- }
-
- public boolean equals(execute_prepared_cql_query_result that) {
- if (that == null)
- return false;
-
- boolean this_present_success = true && this.isSetSuccess();
- boolean that_present_success = true && that.isSetSuccess();
- if (this_present_success || that_present_success) {
- if (!(this_present_success && that_present_success))
- return false;
- if (!this.success.equals(that.success))
- return false;
- }
-
- boolean this_present_ire = true && this.isSetIre();
- boolean that_present_ire = true && that.isSetIre();
- if (this_present_ire || that_present_ire) {
- if (!(this_present_ire && that_present_ire))
- return false;
- if (!this.ire.equals(that.ire))
- return false;
- }
-
- boolean this_present_ue = true && this.isSetUe();
- boolean that_present_ue = true && that.isSetUe();
- if (this_present_ue || that_present_ue) {
- if (!(this_present_ue && that_present_ue))
- return false;
- if (!this.ue.equals(that.ue))
- return false;
- }
-
- boolean this_present_te = true && this.isSetTe();
- boolean that_present_te = true && that.isSetTe();
- if (this_present_te || that_present_te) {
- if (!(this_present_te && that_present_te))
- return false;
- if (!this.te.equals(that.te))
- return false;
- }
-
- boolean this_present_sde = true && this.isSetSde();
- boolean that_present_sde = true && that.isSetSde();
- if (this_present_sde || that_present_sde) {
- if (!(this_present_sde && that_present_sde))
- return false;
- if (!this.sde.equals(that.sde))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_success = true && (isSetSuccess());
- builder.append(present_success);
- if (present_success)
- builder.append(success);
-
- boolean present_ire = true && (isSetIre());
- builder.append(present_ire);
- if (present_ire)
- builder.append(ire);
-
- boolean present_ue = true && (isSetUe());
- builder.append(present_ue);
- if (present_ue)
- builder.append(ue);
-
- boolean present_te = true && (isSetTe());
- builder.append(present_te);
- if (present_te)
- builder.append(te);
-
- boolean present_sde = true && (isSetSde());
- builder.append(present_sde);
- if (present_sde)
- builder.append(sde);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(execute_prepared_cql_query_result other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(other.isSetSuccess());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetSuccess()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, other.success);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetIre()).compareTo(other.isSetIre());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetIre()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.ire, other.ire);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetUe()).compareTo(other.isSetUe());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetUe()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.ue, other.ue);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetTe()).compareTo(other.isSetTe());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetTe()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.te, other.te);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetSde()).compareTo(other.isSetSde());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetSde()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.sde, other.sde);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("execute_prepared_cql_query_result(");
- boolean first = true;
-
- sb.append("success:");
- if (this.success == null) {
- sb.append("null");
- } else {
- sb.append(this.success);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("ire:");
- if (this.ire == null) {
- sb.append("null");
- } else {
- sb.append(this.ire);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("ue:");
- if (this.ue == null) {
- sb.append("null");
- } else {
- sb.append(this.ue);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("te:");
- if (this.te == null) {
- sb.append("null");
- } else {
- sb.append(this.te);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("sde:");
- if (this.sde == null) {
- sb.append("null");
- } else {
- sb.append(this.sde);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // check for sub-struct validity
- if (success != null) {
- success.validate();
- }
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class execute_prepared_cql_query_resultStandardSchemeFactory implements SchemeFactory {
- public execute_prepared_cql_query_resultStandardScheme getScheme() {
- return new execute_prepared_cql_query_resultStandardScheme();
- }
- }
-
- private static class execute_prepared_cql_query_resultStandardScheme extends StandardScheme<execute_prepared_cql_query_result> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, execute_prepared_cql_query_result struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 0: // SUCCESS
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.success = new CqlResult();
- struct.success.read(iprot);
- struct.setSuccessIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 1: // IRE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.ire = new InvalidRequestException();
- struct.ire.read(iprot);
- struct.setIreIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 2: // UE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.ue = new UnavailableException();
- struct.ue.read(iprot);
- struct.setUeIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 3: // TE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.te = new TimedOutException();
- struct.te.read(iprot);
- struct.setTeIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 4: // SDE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.sde = new SchemaDisagreementException();
- struct.sde.read(iprot);
- struct.setSdeIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, execute_prepared_cql_query_result struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.success != null) {
- oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
- struct.success.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.ire != null) {
- oprot.writeFieldBegin(IRE_FIELD_DESC);
- struct.ire.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.ue != null) {
- oprot.writeFieldBegin(UE_FIELD_DESC);
- struct.ue.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.te != null) {
- oprot.writeFieldBegin(TE_FIELD_DESC);
- struct.te.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.sde != null) {
- oprot.writeFieldBegin(SDE_FIELD_DESC);
- struct.sde.write(oprot);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class execute_prepared_cql_query_resultTupleSchemeFactory implements SchemeFactory {
- public execute_prepared_cql_query_resultTupleScheme getScheme() {
- return new execute_prepared_cql_query_resultTupleScheme();
- }
- }
-
- private static class execute_prepared_cql_query_resultTupleScheme extends TupleScheme<execute_prepared_cql_query_result> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, execute_prepared_cql_query_result struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- BitSet optionals = new BitSet();
- if (struct.isSetSuccess()) {
- optionals.set(0);
- }
- if (struct.isSetIre()) {
- optionals.set(1);
- }
- if (struct.isSetUe()) {
- optionals.set(2);
- }
- if (struct.isSetTe()) {
- optionals.set(3);
- }
- if (struct.isSetSde()) {
- optionals.set(4);
- }
- oprot.writeBitSet(optionals, 5);
- if (struct.isSetSuccess()) {
- struct.success.write(oprot);
- }
- if (struct.isSetIre()) {
- struct.ire.write(oprot);
- }
- if (struct.isSetUe()) {
- struct.ue.write(oprot);
- }
- if (struct.isSetTe()) {
- struct.te.write(oprot);
- }
- if (struct.isSetSde()) {
- struct.sde.write(oprot);
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, execute_prepared_cql_query_result struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- BitSet incoming = iprot.readBitSet(5);
- if (incoming.get(0)) {
- struct.success = new CqlResult();
- struct.success.read(iprot);
- struct.setSuccessIsSet(true);
- }
- if (incoming.get(1)) {
- struct.ire = new InvalidRequestException();
- struct.ire.read(iprot);
- struct.setIreIsSet(true);
- }
- if (incoming.get(2)) {
- struct.ue = new UnavailableException();
- struct.ue.read(iprot);
- struct.setUeIsSet(true);
- }
- if (incoming.get(3)) {
- struct.te = new TimedOutException();
- struct.te.read(iprot);
- struct.setTeIsSet(true);
- }
- if (incoming.get(4)) {
- struct.sde = new SchemaDisagreementException();
- struct.sde.read(iprot);
- struct.setSdeIsSet(true);
- }
- }
- }
-
- }
-
- public static class execute_prepared_cql3_query_args implements org.apache.thrift.TBase<execute_prepared_cql3_query_args, execute_prepared_cql3_query_args._Fields>, java.io.Serializable, Cloneable, Comparable<execute_prepared_cql3_query_args> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("execute_prepared_cql3_query_args");
-
- private static final org.apache.thrift.protocol.TField ITEM_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("itemId", org.apache.thrift.protocol.TType.I32, (short)1);
- private static final org.apache.thrift.protocol.TField VALUES_FIELD_DESC = new org.apache.thrift.protocol.TField("values", org.apache.thrift.protocol.TType.LIST, (short)2);
- private static final org.apache.thrift.protocol.TField CONSISTENCY_FIELD_DESC = new org.apache.thrift.protocol.TField("consistency", org.apache.thrift.protocol.TType.I32, (short)3);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new execute_prepared_cql3_query_argsStandardSchemeFactory());
- schemes.put(TupleScheme.class, new execute_prepared_cql3_query_argsTupleSchemeFactory());
- }
-
- public int itemId; // required
- public List<ByteBuffer> values; // required
- /**
- *
- * @see ConsistencyLevel
- */
- public ConsistencyLevel consistency; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- ITEM_ID((short)1, "itemId"),
- VALUES((short)2, "values"),
- /**
- *
- * @see ConsistencyLevel
- */
- CONSISTENCY((short)3, "consistency");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // ITEM_ID
- return ITEM_ID;
- case 2: // VALUES
- return VALUES;
- case 3: // CONSISTENCY
- return CONSISTENCY;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- private static final int __ITEMID_ISSET_ID = 0;
- private byte __isset_bitfield = 0;
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.ITEM_ID, new org.apache.thrift.meta_data.FieldMetaData("itemId", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32)));
- tmpMap.put(_Fields.VALUES, new org.apache.thrift.meta_data.FieldMetaData("values", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true))));
- tmpMap.put(_Fields.CONSISTENCY, new org.apache.thrift.meta_data.FieldMetaData("consistency", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.EnumMetaData(org.apache.thrift.protocol.TType.ENUM, ConsistencyLevel.class)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(execute_prepared_cql3_query_args.class, metaDataMap);
- }
-
- public execute_prepared_cql3_query_args() {
- }
-
- public execute_prepared_cql3_query_args(
- int itemId,
- List<ByteBuffer> values,
- ConsistencyLevel consistency)
- {
- this();
- this.itemId = itemId;
- setItemIdIsSet(true);
- this.values = values;
- this.consistency = consistency;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public execute_prepared_cql3_query_args(execute_prepared_cql3_query_args other) {
- __isset_bitfield = other.__isset_bitfield;
- this.itemId = other.itemId;
- if (other.isSetValues()) {
- List<ByteBuffer> __this__values = new ArrayList<ByteBuffer>(other.values);
- this.values = __this__values;
- }
- if (other.isSetConsistency()) {
- this.consistency = other.consistency;
- }
- }
-
- public execute_prepared_cql3_query_args deepCopy() {
- return new execute_prepared_cql3_query_args(this);
- }
-
- @Override
- public void clear() {
- setItemIdIsSet(false);
- this.itemId = 0;
- this.values = null;
- this.consistency = null;
- }
-
- public int getItemId() {
- return this.itemId;
- }
-
- public execute_prepared_cql3_query_args setItemId(int itemId) {
- this.itemId = itemId;
- setItemIdIsSet(true);
- return this;
- }
-
- public void unsetItemId() {
- __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __ITEMID_ISSET_ID);
- }
-
- /** Returns true if field itemId is set (has been assigned a value) and false otherwise */
- public boolean isSetItemId() {
- return EncodingUtils.testBit(__isset_bitfield, __ITEMID_ISSET_ID);
- }
-
- public void setItemIdIsSet(boolean value) {
- __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __ITEMID_ISSET_ID, value);
- }
-
- public int getValuesSize() {
- return (this.values == null) ? 0 : this.values.size();
- }
-
- public java.util.Iterator<ByteBuffer> getValuesIterator() {
- return (this.values == null) ? null : this.values.iterator();
- }
-
- public void addToValues(ByteBuffer elem) {
- if (this.values == null) {
- this.values = new ArrayList<ByteBuffer>();
- }
- this.values.add(elem);
- }
-
- public List<ByteBuffer> getValues() {
- return this.values;
- }
-
- public execute_prepared_cql3_query_args setValues(List<ByteBuffer> values) {
- this.values = values;
- return this;
- }
-
- public void unsetValues() {
- this.values = null;
- }
-
- /** Returns true if field values is set (has been assigned a value) and false otherwise */
- public boolean isSetValues() {
- return this.values != null;
- }
-
- public void setValuesIsSet(boolean value) {
- if (!value) {
- this.values = null;
- }
- }
-
- /**
- *
- * @see ConsistencyLevel
- */
- public ConsistencyLevel getConsistency() {
- return this.consistency;
- }
-
- /**
- *
- * @see ConsistencyLevel
- */
- public execute_prepared_cql3_query_args setConsistency(ConsistencyLevel consistency) {
- this.consistency = consistency;
- return this;
- }
-
- public void unsetConsistency() {
- this.consistency = null;
- }
-
- /** Returns true if field consistency is set (has been assigned a value) and false otherwise */
- public boolean isSetConsistency() {
- return this.consistency != null;
- }
-
- public void setConsistencyIsSet(boolean value) {
- if (!value) {
- this.consistency = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case ITEM_ID:
- if (value == null) {
- unsetItemId();
- } else {
- setItemId((Integer)value);
- }
- break;
-
- case VALUES:
- if (value == null) {
- unsetValues();
- } else {
- setValues((List<ByteBuffer>)value);
- }
- break;
-
- case CONSISTENCY:
- if (value == null) {
- unsetConsistency();
- } else {
- setConsistency((ConsistencyLevel)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case ITEM_ID:
- return Integer.valueOf(getItemId());
-
- case VALUES:
- return getValues();
-
- case CONSISTENCY:
- return getConsistency();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case ITEM_ID:
- return isSetItemId();
- case VALUES:
- return isSetValues();
- case CONSISTENCY:
- return isSetConsistency();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof execute_prepared_cql3_query_args)
- return this.equals((execute_prepared_cql3_query_args)that);
- return false;
- }
-
- public boolean equals(execute_prepared_cql3_query_args that) {
- if (that == null)
- return false;
-
- boolean this_present_itemId = true;
- boolean that_present_itemId = true;
- if (this_present_itemId || that_present_itemId) {
- if (!(this_present_itemId && that_present_itemId))
- return false;
- if (this.itemId != that.itemId)
- return false;
- }
-
- boolean this_present_values = true && this.isSetValues();
- boolean that_present_values = true && that.isSetValues();
- if (this_present_values || that_present_values) {
- if (!(this_present_values && that_present_values))
- return false;
- if (!this.values.equals(that.values))
- return false;
- }
-
- boolean this_present_consistency = true && this.isSetConsistency();
- boolean that_present_consistency = true && that.isSetConsistency();
- if (this_present_consistency || that_present_consistency) {
- if (!(this_present_consistency && that_present_consistency))
- return false;
- if (!this.consistency.equals(that.consistency))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_itemId = true;
- builder.append(present_itemId);
- if (present_itemId)
- builder.append(itemId);
-
- boolean present_values = true && (isSetValues());
- builder.append(present_values);
- if (present_values)
- builder.append(values);
-
- boolean present_consistency = true && (isSetConsistency());
- builder.append(present_consistency);
- if (present_consistency)
- builder.append(consistency.getValue());
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(execute_prepared_cql3_query_args other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetItemId()).compareTo(other.isSetItemId());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetItemId()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.itemId, other.itemId);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetValues()).compareTo(other.isSetValues());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetValues()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.values, other.values);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetConsistency()).compareTo(other.isSetConsistency());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetConsistency()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.consistency, other.consistency);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("execute_prepared_cql3_query_args(");
- boolean first = true;
-
- sb.append("itemId:");
- sb.append(this.itemId);
- first = false;
- if (!first) sb.append(", ");
- sb.append("values:");
- if (this.values == null) {
- sb.append("null");
- } else {
- sb.append(this.values);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("consistency:");
- if (this.consistency == null) {
- sb.append("null");
- } else {
- sb.append(this.consistency);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // alas, we cannot check 'itemId' because it's a primitive and you chose the non-beans generator.
- if (values == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'values' was not present! Struct: " + toString());
- }
- if (consistency == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'consistency' was not present! Struct: " + toString());
- }
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
- __isset_bitfield = 0;
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class execute_prepared_cql3_query_argsStandardSchemeFactory implements SchemeFactory {
- public execute_prepared_cql3_query_argsStandardScheme getScheme() {
- return new execute_prepared_cql3_query_argsStandardScheme();
- }
- }
-
- private static class execute_prepared_cql3_query_argsStandardScheme extends StandardScheme<execute_prepared_cql3_query_args> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, execute_prepared_cql3_query_args struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // ITEM_ID
- if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
- struct.itemId = iprot.readI32();
- struct.setItemIdIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 2: // VALUES
- if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
- {
- org.apache.thrift.protocol.TList _list456 = iprot.readListBegin();
- struct.values = new ArrayList<ByteBuffer>(_list456.size);
- for (int _i457 = 0; _i457 < _list456.size; ++_i457)
- {
- ByteBuffer _elem458;
- _elem458 = iprot.readBinary();
- struct.values.add(_elem458);
- }
- iprot.readListEnd();
- }
- struct.setValuesIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 3: // CONSISTENCY
- if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
- struct.consistency = ConsistencyLevel.findByValue(iprot.readI32());
- struct.setConsistencyIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- if (!struct.isSetItemId()) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'itemId' was not found in serialized data! Struct: " + toString());
- }
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, execute_prepared_cql3_query_args struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- oprot.writeFieldBegin(ITEM_ID_FIELD_DESC);
- oprot.writeI32(struct.itemId);
- oprot.writeFieldEnd();
- if (struct.values != null) {
- oprot.writeFieldBegin(VALUES_FIELD_DESC);
- {
- oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, struct.values.size()));
- for (ByteBuffer _iter459 : struct.values)
- {
- oprot.writeBinary(_iter459);
- }
- oprot.writeListEnd();
- }
- oprot.writeFieldEnd();
- }
- if (struct.consistency != null) {
- oprot.writeFieldBegin(CONSISTENCY_FIELD_DESC);
- oprot.writeI32(struct.consistency.getValue());
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class execute_prepared_cql3_query_argsTupleSchemeFactory implements SchemeFactory {
- public execute_prepared_cql3_query_argsTupleScheme getScheme() {
- return new execute_prepared_cql3_query_argsTupleScheme();
- }
- }
-
- private static class execute_prepared_cql3_query_argsTupleScheme extends TupleScheme<execute_prepared_cql3_query_args> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, execute_prepared_cql3_query_args struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- oprot.writeI32(struct.itemId);
- {
- oprot.writeI32(struct.values.size());
- for (ByteBuffer _iter460 : struct.values)
- {
- oprot.writeBinary(_iter460);
- }
- }
- oprot.writeI32(struct.consistency.getValue());
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, execute_prepared_cql3_query_args struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- struct.itemId = iprot.readI32();
- struct.setItemIdIsSet(true);
- {
- org.apache.thrift.protocol.TList _list461 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32());
- struct.values = new ArrayList<ByteBuffer>(_list461.size);
- for (int _i462 = 0; _i462 < _list461.size; ++_i462)
- {
- ByteBuffer _elem463;
- _elem463 = iprot.readBinary();
- struct.values.add(_elem463);
- }
- }
- struct.setValuesIsSet(true);
- struct.consistency = ConsistencyLevel.findByValue(iprot.readI32());
- struct.setConsistencyIsSet(true);
- }
- }
-
- }
-
- public static class execute_prepared_cql3_query_result implements org.apache.thrift.TBase<execute_prepared_cql3_query_result, execute_prepared_cql3_query_result._Fields>, java.io.Serializable, Cloneable, Comparable<execute_prepared_cql3_query_result> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("execute_prepared_cql3_query_result");
-
- private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRUCT, (short)0);
- private static final org.apache.thrift.protocol.TField IRE_FIELD_DESC = new org.apache.thrift.protocol.TField("ire", org.apache.thrift.protocol.TType.STRUCT, (short)1);
- private static final org.apache.thrift.protocol.TField UE_FIELD_DESC = new org.apache.thrift.protocol.TField("ue", org.apache.thrift.protocol.TType.STRUCT, (short)2);
- private static final org.apache.thrift.protocol.TField TE_FIELD_DESC = new org.apache.thrift.protocol.TField("te", org.apache.thrift.protocol.TType.STRUCT, (short)3);
- private static final org.apache.thrift.protocol.TField SDE_FIELD_DESC = new org.apache.thrift.protocol.TField("sde", org.apache.thrift.protocol.TType.STRUCT, (short)4);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new execute_prepared_cql3_query_resultStandardSchemeFactory());
- schemes.put(TupleScheme.class, new execute_prepared_cql3_query_resultTupleSchemeFactory());
- }
-
- public CqlResult success; // required
- public InvalidRequestException ire; // required
- public UnavailableException ue; // required
- public TimedOutException te; // required
- public SchemaDisagreementException sde; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- SUCCESS((short)0, "success"),
- IRE((short)1, "ire"),
- UE((short)2, "ue"),
- TE((short)3, "te"),
- SDE((short)4, "sde");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 0: // SUCCESS
- return SUCCESS;
- case 1: // IRE
- return IRE;
- case 2: // UE
- return UE;
- case 3: // TE
- return TE;
- case 4: // SDE
- return SDE;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, CqlResult.class)));
- tmpMap.put(_Fields.IRE, new org.apache.thrift.meta_data.FieldMetaData("ire", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- tmpMap.put(_Fields.UE, new org.apache.thrift.meta_data.FieldMetaData("ue", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- tmpMap.put(_Fields.TE, new org.apache.thrift.meta_data.FieldMetaData("te", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- tmpMap.put(_Fields.SDE, new org.apache.thrift.meta_data.FieldMetaData("sde", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(execute_prepared_cql3_query_result.class, metaDataMap);
- }
-
- public execute_prepared_cql3_query_result() {
- }
-
- public execute_prepared_cql3_query_result(
- CqlResult success,
- InvalidRequestException ire,
- UnavailableException ue,
- TimedOutException te,
- SchemaDisagreementException sde)
- {
- this();
- this.success = success;
- this.ire = ire;
- this.ue = ue;
- this.te = te;
- this.sde = sde;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public execute_prepared_cql3_query_result(execute_prepared_cql3_query_result other) {
- if (other.isSetSuccess()) {
- this.success = new CqlResult(other.success);
- }
- if (other.isSetIre()) {
- this.ire = new InvalidRequestException(other.ire);
- }
- if (other.isSetUe()) {
- this.ue = new UnavailableException(other.ue);
- }
- if (other.isSetTe()) {
- this.te = new TimedOutException(other.te);
- }
- if (other.isSetSde()) {
- this.sde = new SchemaDisagreementException(other.sde);
- }
- }
-
- public execute_prepared_cql3_query_result deepCopy() {
- return new execute_prepared_cql3_query_result(this);
- }
-
- @Override
- public void clear() {
- this.success = null;
- this.ire = null;
- this.ue = null;
- this.te = null;
- this.sde = null;
- }
-
- public CqlResult getSuccess() {
- return this.success;
- }
-
- public execute_prepared_cql3_query_result setSuccess(CqlResult success) {
- this.success = success;
- return this;
- }
-
- public void unsetSuccess() {
- this.success = null;
- }
-
- /** Returns true if field success is set (has been assigned a value) and false otherwise */
- public boolean isSetSuccess() {
- return this.success != null;
- }
-
- public void setSuccessIsSet(boolean value) {
- if (!value) {
- this.success = null;
- }
- }
-
- public InvalidRequestException getIre() {
- return this.ire;
- }
-
- public execute_prepared_cql3_query_result setIre(InvalidRequestException ire) {
- this.ire = ire;
- return this;
- }
-
- public void unsetIre() {
- this.ire = null;
- }
-
- /** Returns true if field ire is set (has been assigned a value) and false otherwise */
- public boolean isSetIre() {
- return this.ire != null;
- }
-
- public void setIreIsSet(boolean value) {
- if (!value) {
- this.ire = null;
- }
- }
-
- public UnavailableException getUe() {
- return this.ue;
- }
-
- public execute_prepared_cql3_query_result setUe(UnavailableException ue) {
- this.ue = ue;
- return this;
- }
-
- public void unsetUe() {
- this.ue = null;
- }
-
- /** Returns true if field ue is set (has been assigned a value) and false otherwise */
- public boolean isSetUe() {
- return this.ue != null;
- }
-
- public void setUeIsSet(boolean value) {
- if (!value) {
- this.ue = null;
- }
- }
-
- public TimedOutException getTe() {
- return this.te;
- }
-
- public execute_prepared_cql3_query_result setTe(TimedOutException te) {
- this.te = te;
- return this;
- }
-
- public void unsetTe() {
- this.te = null;
- }
-
- /** Returns true if field te is set (has been assigned a value) and false otherwise */
- public boolean isSetTe() {
- return this.te != null;
- }
-
- public void setTeIsSet(boolean value) {
- if (!value) {
- this.te = null;
- }
- }
-
- public SchemaDisagreementException getSde() {
- return this.sde;
- }
-
- public execute_prepared_cql3_query_result setSde(SchemaDisagreementException sde) {
- this.sde = sde;
- return this;
- }
-
- public void unsetSde() {
- this.sde = null;
- }
-
- /** Returns true if field sde is set (has been assigned a value) and false otherwise */
- public boolean isSetSde() {
- return this.sde != null;
- }
-
- public void setSdeIsSet(boolean value) {
- if (!value) {
- this.sde = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case SUCCESS:
- if (value == null) {
- unsetSuccess();
- } else {
- setSuccess((CqlResult)value);
- }
- break;
-
- case IRE:
- if (value == null) {
- unsetIre();
- } else {
- setIre((InvalidRequestException)value);
- }
- break;
-
- case UE:
- if (value == null) {
- unsetUe();
- } else {
- setUe((UnavailableException)value);
- }
- break;
-
- case TE:
- if (value == null) {
- unsetTe();
- } else {
- setTe((TimedOutException)value);
- }
- break;
-
- case SDE:
- if (value == null) {
- unsetSde();
- } else {
- setSde((SchemaDisagreementException)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case SUCCESS:
- return getSuccess();
-
- case IRE:
- return getIre();
-
- case UE:
- return getUe();
-
- case TE:
- return getTe();
-
- case SDE:
- return getSde();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case SUCCESS:
- return isSetSuccess();
- case IRE:
- return isSetIre();
- case UE:
- return isSetUe();
- case TE:
- return isSetTe();
- case SDE:
- return isSetSde();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof execute_prepared_cql3_query_result)
- return this.equals((execute_prepared_cql3_query_result)that);
- return false;
- }
-
- public boolean equals(execute_prepared_cql3_query_result that) {
- if (that == null)
- return false;
-
- boolean this_present_success = true && this.isSetSuccess();
- boolean that_present_success = true && that.isSetSuccess();
- if (this_present_success || that_present_success) {
- if (!(this_present_success && that_present_success))
- return false;
- if (!this.success.equals(that.success))
- return false;
- }
-
- boolean this_present_ire = true && this.isSetIre();
- boolean that_present_ire = true && that.isSetIre();
- if (this_present_ire || that_present_ire) {
- if (!(this_present_ire && that_present_ire))
- return false;
- if (!this.ire.equals(that.ire))
- return false;
- }
-
- boolean this_present_ue = true && this.isSetUe();
- boolean that_present_ue = true && that.isSetUe();
- if (this_present_ue || that_present_ue) {
- if (!(this_present_ue && that_present_ue))
- return false;
- if (!this.ue.equals(that.ue))
- return false;
- }
-
- boolean this_present_te = true && this.isSetTe();
- boolean that_present_te = true && that.isSetTe();
- if (this_present_te || that_present_te) {
- if (!(this_present_te && that_present_te))
- return false;
- if (!this.te.equals(that.te))
- return false;
- }
-
- boolean this_present_sde = true && this.isSetSde();
- boolean that_present_sde = true && that.isSetSde();
- if (this_present_sde || that_present_sde) {
- if (!(this_present_sde && that_present_sde))
- return false;
- if (!this.sde.equals(that.sde))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_success = true && (isSetSuccess());
- builder.append(present_success);
- if (present_success)
- builder.append(success);
-
- boolean present_ire = true && (isSetIre());
- builder.append(present_ire);
- if (present_ire)
- builder.append(ire);
-
- boolean present_ue = true && (isSetUe());
- builder.append(present_ue);
- if (present_ue)
- builder.append(ue);
-
- boolean present_te = true && (isSetTe());
- builder.append(present_te);
- if (present_te)
- builder.append(te);
-
- boolean present_sde = true && (isSetSde());
- builder.append(present_sde);
- if (present_sde)
- builder.append(sde);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(execute_prepared_cql3_query_result other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(other.isSetSuccess());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetSuccess()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, other.success);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetIre()).compareTo(other.isSetIre());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetIre()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.ire, other.ire);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetUe()).compareTo(other.isSetUe());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetUe()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.ue, other.ue);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetTe()).compareTo(other.isSetTe());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetTe()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.te, other.te);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetSde()).compareTo(other.isSetSde());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetSde()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.sde, other.sde);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("execute_prepared_cql3_query_result(");
- boolean first = true;
-
- sb.append("success:");
- if (this.success == null) {
- sb.append("null");
- } else {
- sb.append(this.success);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("ire:");
- if (this.ire == null) {
- sb.append("null");
- } else {
- sb.append(this.ire);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("ue:");
- if (this.ue == null) {
- sb.append("null");
- } else {
- sb.append(this.ue);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("te:");
- if (this.te == null) {
- sb.append("null");
- } else {
- sb.append(this.te);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("sde:");
- if (this.sde == null) {
- sb.append("null");
- } else {
- sb.append(this.sde);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // check for sub-struct validity
- if (success != null) {
- success.validate();
- }
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class execute_prepared_cql3_query_resultStandardSchemeFactory implements SchemeFactory {
- public execute_prepared_cql3_query_resultStandardScheme getScheme() {
- return new execute_prepared_cql3_query_resultStandardScheme();
- }
- }
-
- private static class execute_prepared_cql3_query_resultStandardScheme extends StandardScheme<execute_prepared_cql3_query_result> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, execute_prepared_cql3_query_result struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 0: // SUCCESS
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.success = new CqlResult();
- struct.success.read(iprot);
- struct.setSuccessIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 1: // IRE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.ire = new InvalidRequestException();
- struct.ire.read(iprot);
- struct.setIreIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 2: // UE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.ue = new UnavailableException();
- struct.ue.read(iprot);
- struct.setUeIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 3: // TE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.te = new TimedOutException();
- struct.te.read(iprot);
- struct.setTeIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 4: // SDE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.sde = new SchemaDisagreementException();
- struct.sde.read(iprot);
- struct.setSdeIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, execute_prepared_cql3_query_result struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.success != null) {
- oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
- struct.success.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.ire != null) {
- oprot.writeFieldBegin(IRE_FIELD_DESC);
- struct.ire.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.ue != null) {
- oprot.writeFieldBegin(UE_FIELD_DESC);
- struct.ue.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.te != null) {
- oprot.writeFieldBegin(TE_FIELD_DESC);
- struct.te.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.sde != null) {
- oprot.writeFieldBegin(SDE_FIELD_DESC);
- struct.sde.write(oprot);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class execute_prepared_cql3_query_resultTupleSchemeFactory implements SchemeFactory {
- public execute_prepared_cql3_query_resultTupleScheme getScheme() {
- return new execute_prepared_cql3_query_resultTupleScheme();
- }
- }
-
- private static class execute_prepared_cql3_query_resultTupleScheme extends TupleScheme<execute_prepared_cql3_query_result> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, execute_prepared_cql3_query_result struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- BitSet optionals = new BitSet();
- if (struct.isSetSuccess()) {
- optionals.set(0);
- }
- if (struct.isSetIre()) {
- optionals.set(1);
- }
- if (struct.isSetUe()) {
- optionals.set(2);
- }
- if (struct.isSetTe()) {
- optionals.set(3);
- }
- if (struct.isSetSde()) {
- optionals.set(4);
- }
- oprot.writeBitSet(optionals, 5);
- if (struct.isSetSuccess()) {
- struct.success.write(oprot);
- }
- if (struct.isSetIre()) {
- struct.ire.write(oprot);
- }
- if (struct.isSetUe()) {
- struct.ue.write(oprot);
- }
- if (struct.isSetTe()) {
- struct.te.write(oprot);
- }
- if (struct.isSetSde()) {
- struct.sde.write(oprot);
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, execute_prepared_cql3_query_result struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- BitSet incoming = iprot.readBitSet(5);
- if (incoming.get(0)) {
- struct.success = new CqlResult();
- struct.success.read(iprot);
- struct.setSuccessIsSet(true);
- }
- if (incoming.get(1)) {
- struct.ire = new InvalidRequestException();
- struct.ire.read(iprot);
- struct.setIreIsSet(true);
- }
- if (incoming.get(2)) {
- struct.ue = new UnavailableException();
- struct.ue.read(iprot);
- struct.setUeIsSet(true);
- }
- if (incoming.get(3)) {
- struct.te = new TimedOutException();
- struct.te.read(iprot);
- struct.setTeIsSet(true);
- }
- if (incoming.get(4)) {
- struct.sde = new SchemaDisagreementException();
- struct.sde.read(iprot);
- struct.setSdeIsSet(true);
- }
- }
- }
-
- }
-
- public static class set_cql_version_args implements org.apache.thrift.TBase<set_cql_version_args, set_cql_version_args._Fields>, java.io.Serializable, Cloneable, Comparable<set_cql_version_args> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("set_cql_version_args");
-
- private static final org.apache.thrift.protocol.TField VERSION_FIELD_DESC = new org.apache.thrift.protocol.TField("version", org.apache.thrift.protocol.TType.STRING, (short)1);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new set_cql_version_argsStandardSchemeFactory());
- schemes.put(TupleScheme.class, new set_cql_version_argsTupleSchemeFactory());
- }
-
- public String version; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- VERSION((short)1, "version");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // VERSION
- return VERSION;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.VERSION, new org.apache.thrift.meta_data.FieldMetaData("version", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(set_cql_version_args.class, metaDataMap);
- }
-
- public set_cql_version_args() {
- }
-
- public set_cql_version_args(
- String version)
- {
- this();
- this.version = version;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public set_cql_version_args(set_cql_version_args other) {
- if (other.isSetVersion()) {
- this.version = other.version;
- }
- }
-
- public set_cql_version_args deepCopy() {
- return new set_cql_version_args(this);
- }
-
- @Override
- public void clear() {
- this.version = null;
- }
-
- public String getVersion() {
- return this.version;
- }
-
- public set_cql_version_args setVersion(String version) {
- this.version = version;
- return this;
- }
-
- public void unsetVersion() {
- this.version = null;
- }
-
- /** Returns true if field version is set (has been assigned a value) and false otherwise */
- public boolean isSetVersion() {
- return this.version != null;
- }
-
- public void setVersionIsSet(boolean value) {
- if (!value) {
- this.version = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case VERSION:
- if (value == null) {
- unsetVersion();
- } else {
- setVersion((String)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case VERSION:
- return getVersion();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case VERSION:
- return isSetVersion();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof set_cql_version_args)
- return this.equals((set_cql_version_args)that);
- return false;
- }
-
- public boolean equals(set_cql_version_args that) {
- if (that == null)
- return false;
-
- boolean this_present_version = true && this.isSetVersion();
- boolean that_present_version = true && that.isSetVersion();
- if (this_present_version || that_present_version) {
- if (!(this_present_version && that_present_version))
- return false;
- if (!this.version.equals(that.version))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_version = true && (isSetVersion());
- builder.append(present_version);
- if (present_version)
- builder.append(version);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(set_cql_version_args other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetVersion()).compareTo(other.isSetVersion());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetVersion()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.version, other.version);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("set_cql_version_args(");
- boolean first = true;
-
- sb.append("version:");
- if (this.version == null) {
- sb.append("null");
- } else {
- sb.append(this.version);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- if (version == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'version' was not present! Struct: " + toString());
- }
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class set_cql_version_argsStandardSchemeFactory implements SchemeFactory {
- public set_cql_version_argsStandardScheme getScheme() {
- return new set_cql_version_argsStandardScheme();
- }
- }
-
- private static class set_cql_version_argsStandardScheme extends StandardScheme<set_cql_version_args> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, set_cql_version_args struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // VERSION
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.version = iprot.readString();
- struct.setVersionIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, set_cql_version_args struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.version != null) {
- oprot.writeFieldBegin(VERSION_FIELD_DESC);
- oprot.writeString(struct.version);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class set_cql_version_argsTupleSchemeFactory implements SchemeFactory {
- public set_cql_version_argsTupleScheme getScheme() {
- return new set_cql_version_argsTupleScheme();
- }
- }
-
- private static class set_cql_version_argsTupleScheme extends TupleScheme<set_cql_version_args> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, set_cql_version_args struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- oprot.writeString(struct.version);
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, set_cql_version_args struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- struct.version = iprot.readString();
- struct.setVersionIsSet(true);
- }
- }
-
- }
-
- public static class set_cql_version_result implements org.apache.thrift.TBase<set_cql_version_result, set_cql_version_result._Fields>, java.io.Serializable, Cloneable, Comparable<set_cql_version_result> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("set_cql_version_result");
-
- private static final org.apache.thrift.protocol.TField IRE_FIELD_DESC = new org.apache.thrift.protocol.TField("ire", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new set_cql_version_resultStandardSchemeFactory());
- schemes.put(TupleScheme.class, new set_cql_version_resultTupleSchemeFactory());
- }
-
- public InvalidRequestException ire; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- IRE((short)1, "ire");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // IRE
- return IRE;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.IRE, new org.apache.thrift.meta_data.FieldMetaData("ire", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(set_cql_version_result.class, metaDataMap);
- }
-
- public set_cql_version_result() {
- }
-
- public set_cql_version_result(
- InvalidRequestException ire)
- {
- this();
- this.ire = ire;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public set_cql_version_result(set_cql_version_result other) {
- if (other.isSetIre()) {
- this.ire = new InvalidRequestException(other.ire);
- }
- }
-
- public set_cql_version_result deepCopy() {
- return new set_cql_version_result(this);
- }
-
- @Override
- public void clear() {
- this.ire = null;
- }
-
- public InvalidRequestException getIre() {
- return this.ire;
- }
-
- public set_cql_version_result setIre(InvalidRequestException ire) {
- this.ire = ire;
- return this;
- }
-
- public void unsetIre() {
- this.ire = null;
- }
-
- /** Returns true if field ire is set (has been assigned a value) and false otherwise */
- public boolean isSetIre() {
- return this.ire != null;
- }
-
- public void setIreIsSet(boolean value) {
- if (!value) {
- this.ire = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case IRE:
- if (value == null) {
- unsetIre();
- } else {
- setIre((InvalidRequestException)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case IRE:
- return getIre();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case IRE:
- return isSetIre();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof set_cql_version_result)
- return this.equals((set_cql_version_result)that);
- return false;
- }
-
- public boolean equals(set_cql_version_result that) {
- if (that == null)
- return false;
-
- boolean this_present_ire = true && this.isSetIre();
- boolean that_present_ire = true && that.isSetIre();
- if (this_present_ire || that_present_ire) {
- if (!(this_present_ire && that_present_ire))
- return false;
- if (!this.ire.equals(that.ire))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_ire = true && (isSetIre());
- builder.append(present_ire);
- if (present_ire)
- builder.append(ire);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(set_cql_version_result other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetIre()).compareTo(other.isSetIre());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetIre()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.ire, other.ire);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("set_cql_version_result(");
- boolean first = true;
-
- sb.append("ire:");
- if (this.ire == null) {
- sb.append("null");
- } else {
- sb.append(this.ire);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class set_cql_version_resultStandardSchemeFactory implements SchemeFactory {
- public set_cql_version_resultStandardScheme getScheme() {
- return new set_cql_version_resultStandardScheme();
- }
- }
-
- private static class set_cql_version_resultStandardScheme extends StandardScheme<set_cql_version_result> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, set_cql_version_result struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // IRE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.ire = new InvalidRequestException();
- struct.ire.read(iprot);
- struct.setIreIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, set_cql_version_result struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.ire != null) {
- oprot.writeFieldBegin(IRE_FIELD_DESC);
- struct.ire.write(oprot);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class set_cql_version_resultTupleSchemeFactory implements SchemeFactory {
- public set_cql_version_resultTupleScheme getScheme() {
- return new set_cql_version_resultTupleScheme();
- }
- }
-
- private static class set_cql_version_resultTupleScheme extends TupleScheme<set_cql_version_result> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, set_cql_version_result struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- BitSet optionals = new BitSet();
- if (struct.isSetIre()) {
- optionals.set(0);
- }
- oprot.writeBitSet(optionals, 1);
- if (struct.isSetIre()) {
- struct.ire.write(oprot);
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, set_cql_version_result struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- BitSet incoming = iprot.readBitSet(1);
- if (incoming.get(0)) {
- struct.ire = new InvalidRequestException();
- struct.ire.read(iprot);
- struct.setIreIsSet(true);
- }
- }
- }
-
- }
-
-}
diff --git a/interface/thrift/gen-java/org/apache/cassandra/thrift/CfDef.java b/interface/thrift/gen-java/org/apache/cassandra/thrift/CfDef.java
deleted file mode 100644
index ec10050..0000000
--- a/interface/thrift/gen-java/org/apache/cassandra/thrift/CfDef.java
+++ /dev/null
@@ -1,4927 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.1)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- * @generated
- */
-package org.apache.cassandra.thrift;
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
-
-import org.apache.commons.lang3.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class CfDef implements org.apache.thrift.TBase<CfDef, CfDef._Fields>, java.io.Serializable, Cloneable, Comparable<CfDef> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("CfDef");
-
- private static final org.apache.thrift.protocol.TField KEYSPACE_FIELD_DESC = new org.apache.thrift.protocol.TField("keyspace", org.apache.thrift.protocol.TType.STRING, (short)1);
- private static final org.apache.thrift.protocol.TField NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("name", org.apache.thrift.protocol.TType.STRING, (short)2);
- private static final org.apache.thrift.protocol.TField COLUMN_TYPE_FIELD_DESC = new org.apache.thrift.protocol.TField("column_type", org.apache.thrift.protocol.TType.STRING, (short)3);
- private static final org.apache.thrift.protocol.TField COMPARATOR_TYPE_FIELD_DESC = new org.apache.thrift.protocol.TField("comparator_type", org.apache.thrift.protocol.TType.STRING, (short)5);
- private static final org.apache.thrift.protocol.TField SUBCOMPARATOR_TYPE_FIELD_DESC = new org.apache.thrift.protocol.TField("subcomparator_type", org.apache.thrift.protocol.TType.STRING, (short)6);
- private static final org.apache.thrift.protocol.TField COMMENT_FIELD_DESC = new org.apache.thrift.protocol.TField("comment", org.apache.thrift.protocol.TType.STRING, (short)8);
- private static final org.apache.thrift.protocol.TField READ_REPAIR_CHANCE_FIELD_DESC = new org.apache.thrift.protocol.TField("read_repair_chance", org.apache.thrift.protocol.TType.DOUBLE, (short)12);
- private static final org.apache.thrift.protocol.TField COLUMN_METADATA_FIELD_DESC = new org.apache.thrift.protocol.TField("column_metadata", org.apache.thrift.protocol.TType.LIST, (short)13);
- private static final org.apache.thrift.protocol.TField GC_GRACE_SECONDS_FIELD_DESC = new org.apache.thrift.protocol.TField("gc_grace_seconds", org.apache.thrift.protocol.TType.I32, (short)14);
- private static final org.apache.thrift.protocol.TField DEFAULT_VALIDATION_CLASS_FIELD_DESC = new org.apache.thrift.protocol.TField("default_validation_class", org.apache.thrift.protocol.TType.STRING, (short)15);
- private static final org.apache.thrift.protocol.TField ID_FIELD_DESC = new org.apache.thrift.protocol.TField("id", org.apache.thrift.protocol.TType.I32, (short)16);
- private static final org.apache.thrift.protocol.TField MIN_COMPACTION_THRESHOLD_FIELD_DESC = new org.apache.thrift.protocol.TField("min_compaction_threshold", org.apache.thrift.protocol.TType.I32, (short)17);
- private static final org.apache.thrift.protocol.TField MAX_COMPACTION_THRESHOLD_FIELD_DESC = new org.apache.thrift.protocol.TField("max_compaction_threshold", org.apache.thrift.protocol.TType.I32, (short)18);
- private static final org.apache.thrift.protocol.TField KEY_VALIDATION_CLASS_FIELD_DESC = new org.apache.thrift.protocol.TField("key_validation_class", org.apache.thrift.protocol.TType.STRING, (short)26);
- private static final org.apache.thrift.protocol.TField KEY_ALIAS_FIELD_DESC = new org.apache.thrift.protocol.TField("key_alias", org.apache.thrift.protocol.TType.STRING, (short)28);
- private static final org.apache.thrift.protocol.TField COMPACTION_STRATEGY_FIELD_DESC = new org.apache.thrift.protocol.TField("compaction_strategy", org.apache.thrift.protocol.TType.STRING, (short)29);
- private static final org.apache.thrift.protocol.TField COMPACTION_STRATEGY_OPTIONS_FIELD_DESC = new org.apache.thrift.protocol.TField("compaction_strategy_options", org.apache.thrift.protocol.TType.MAP, (short)30);
- private static final org.apache.thrift.protocol.TField COMPRESSION_OPTIONS_FIELD_DESC = new org.apache.thrift.protocol.TField("compression_options", org.apache.thrift.protocol.TType.MAP, (short)32);
- private static final org.apache.thrift.protocol.TField BLOOM_FILTER_FP_CHANCE_FIELD_DESC = new org.apache.thrift.protocol.TField("bloom_filter_fp_chance", org.apache.thrift.protocol.TType.DOUBLE, (short)33);
- private static final org.apache.thrift.protocol.TField CACHING_FIELD_DESC = new org.apache.thrift.protocol.TField("caching", org.apache.thrift.protocol.TType.STRING, (short)34);
- private static final org.apache.thrift.protocol.TField DCLOCAL_READ_REPAIR_CHANCE_FIELD_DESC = new org.apache.thrift.protocol.TField("dclocal_read_repair_chance", org.apache.thrift.protocol.TType.DOUBLE, (short)37);
- private static final org.apache.thrift.protocol.TField MEMTABLE_FLUSH_PERIOD_IN_MS_FIELD_DESC = new org.apache.thrift.protocol.TField("memtable_flush_period_in_ms", org.apache.thrift.protocol.TType.I32, (short)39);
- private static final org.apache.thrift.protocol.TField DEFAULT_TIME_TO_LIVE_FIELD_DESC = new org.apache.thrift.protocol.TField("default_time_to_live", org.apache.thrift.protocol.TType.I32, (short)40);
- private static final org.apache.thrift.protocol.TField SPECULATIVE_RETRY_FIELD_DESC = new org.apache.thrift.protocol.TField("speculative_retry", org.apache.thrift.protocol.TType.STRING, (short)42);
- private static final org.apache.thrift.protocol.TField TRIGGERS_FIELD_DESC = new org.apache.thrift.protocol.TField("triggers", org.apache.thrift.protocol.TType.LIST, (short)43);
- private static final org.apache.thrift.protocol.TField CELLS_PER_ROW_TO_CACHE_FIELD_DESC = new org.apache.thrift.protocol.TField("cells_per_row_to_cache", org.apache.thrift.protocol.TType.STRING, (short)44);
- private static final org.apache.thrift.protocol.TField MIN_INDEX_INTERVAL_FIELD_DESC = new org.apache.thrift.protocol.TField("min_index_interval", org.apache.thrift.protocol.TType.I32, (short)45);
- private static final org.apache.thrift.protocol.TField MAX_INDEX_INTERVAL_FIELD_DESC = new org.apache.thrift.protocol.TField("max_index_interval", org.apache.thrift.protocol.TType.I32, (short)46);
- private static final org.apache.thrift.protocol.TField ROW_CACHE_SIZE_FIELD_DESC = new org.apache.thrift.protocol.TField("row_cache_size", org.apache.thrift.protocol.TType.DOUBLE, (short)9);
- private static final org.apache.thrift.protocol.TField KEY_CACHE_SIZE_FIELD_DESC = new org.apache.thrift.protocol.TField("key_cache_size", org.apache.thrift.protocol.TType.DOUBLE, (short)11);
- private static final org.apache.thrift.protocol.TField ROW_CACHE_SAVE_PERIOD_IN_SECONDS_FIELD_DESC = new org.apache.thrift.protocol.TField("row_cache_save_period_in_seconds", org.apache.thrift.protocol.TType.I32, (short)19);
- private static final org.apache.thrift.protocol.TField KEY_CACHE_SAVE_PERIOD_IN_SECONDS_FIELD_DESC = new org.apache.thrift.protocol.TField("key_cache_save_period_in_seconds", org.apache.thrift.protocol.TType.I32, (short)20);
- private static final org.apache.thrift.protocol.TField MEMTABLE_FLUSH_AFTER_MINS_FIELD_DESC = new org.apache.thrift.protocol.TField("memtable_flush_after_mins", org.apache.thrift.protocol.TType.I32, (short)21);
- private static final org.apache.thrift.protocol.TField MEMTABLE_THROUGHPUT_IN_MB_FIELD_DESC = new org.apache.thrift.protocol.TField("memtable_throughput_in_mb", org.apache.thrift.protocol.TType.I32, (short)22);
- private static final org.apache.thrift.protocol.TField MEMTABLE_OPERATIONS_IN_MILLIONS_FIELD_DESC = new org.apache.thrift.protocol.TField("memtable_operations_in_millions", org.apache.thrift.protocol.TType.DOUBLE, (short)23);
- private static final org.apache.thrift.protocol.TField REPLICATE_ON_WRITE_FIELD_DESC = new org.apache.thrift.protocol.TField("replicate_on_write", org.apache.thrift.protocol.TType.BOOL, (short)24);
- private static final org.apache.thrift.protocol.TField MERGE_SHARDS_CHANCE_FIELD_DESC = new org.apache.thrift.protocol.TField("merge_shards_chance", org.apache.thrift.protocol.TType.DOUBLE, (short)25);
- private static final org.apache.thrift.protocol.TField ROW_CACHE_PROVIDER_FIELD_DESC = new org.apache.thrift.protocol.TField("row_cache_provider", org.apache.thrift.protocol.TType.STRING, (short)27);
- private static final org.apache.thrift.protocol.TField ROW_CACHE_KEYS_TO_SAVE_FIELD_DESC = new org.apache.thrift.protocol.TField("row_cache_keys_to_save", org.apache.thrift.protocol.TType.I32, (short)31);
- private static final org.apache.thrift.protocol.TField POPULATE_IO_CACHE_ON_FLUSH_FIELD_DESC = new org.apache.thrift.protocol.TField("populate_io_cache_on_flush", org.apache.thrift.protocol.TType.BOOL, (short)38);
- private static final org.apache.thrift.protocol.TField INDEX_INTERVAL_FIELD_DESC = new org.apache.thrift.protocol.TField("index_interval", org.apache.thrift.protocol.TType.I32, (short)41);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new CfDefStandardSchemeFactory());
- schemes.put(TupleScheme.class, new CfDefTupleSchemeFactory());
- }
-
- public String keyspace; // required
- public String name; // required
- public String column_type; // optional
- public String comparator_type; // optional
- public String subcomparator_type; // optional
- public String comment; // optional
- public double read_repair_chance; // optional
- public List<ColumnDef> column_metadata; // optional
- public int gc_grace_seconds; // optional
- public String default_validation_class; // optional
- public int id; // optional
- public int min_compaction_threshold; // optional
- public int max_compaction_threshold; // optional
- public String key_validation_class; // optional
- public ByteBuffer key_alias; // optional
- public String compaction_strategy; // optional
- public Map<String,String> compaction_strategy_options; // optional
- public Map<String,String> compression_options; // optional
- public double bloom_filter_fp_chance; // optional
- public String caching; // optional
- public double dclocal_read_repair_chance; // optional
- public int memtable_flush_period_in_ms; // optional
- public int default_time_to_live; // optional
- public String speculative_retry; // optional
- public List<TriggerDef> triggers; // optional
- public String cells_per_row_to_cache; // optional
- public int min_index_interval; // optional
- public int max_index_interval; // optional
- /**
- * @deprecated
- */
- public double row_cache_size; // optional
- /**
- * @deprecated
- */
- public double key_cache_size; // optional
- /**
- * @deprecated
- */
- public int row_cache_save_period_in_seconds; // optional
- /**
- * @deprecated
- */
- public int key_cache_save_period_in_seconds; // optional
- /**
- * @deprecated
- */
- public int memtable_flush_after_mins; // optional
- /**
- * @deprecated
- */
- public int memtable_throughput_in_mb; // optional
- /**
- * @deprecated
- */
- public double memtable_operations_in_millions; // optional
- /**
- * @deprecated
- */
- public boolean replicate_on_write; // optional
- /**
- * @deprecated
- */
- public double merge_shards_chance; // optional
- /**
- * @deprecated
- */
- public String row_cache_provider; // optional
- /**
- * @deprecated
- */
- public int row_cache_keys_to_save; // optional
- /**
- * @deprecated
- */
- public boolean populate_io_cache_on_flush; // optional
- /**
- * @deprecated
- */
- public int index_interval; // optional
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- KEYSPACE((short)1, "keyspace"),
- NAME((short)2, "name"),
- COLUMN_TYPE((short)3, "column_type"),
- COMPARATOR_TYPE((short)5, "comparator_type"),
- SUBCOMPARATOR_TYPE((short)6, "subcomparator_type"),
- COMMENT((short)8, "comment"),
- READ_REPAIR_CHANCE((short)12, "read_repair_chance"),
- COLUMN_METADATA((short)13, "column_metadata"),
- GC_GRACE_SECONDS((short)14, "gc_grace_seconds"),
- DEFAULT_VALIDATION_CLASS((short)15, "default_validation_class"),
- ID((short)16, "id"),
- MIN_COMPACTION_THRESHOLD((short)17, "min_compaction_threshold"),
- MAX_COMPACTION_THRESHOLD((short)18, "max_compaction_threshold"),
- KEY_VALIDATION_CLASS((short)26, "key_validation_class"),
- KEY_ALIAS((short)28, "key_alias"),
- COMPACTION_STRATEGY((short)29, "compaction_strategy"),
- COMPACTION_STRATEGY_OPTIONS((short)30, "compaction_strategy_options"),
- COMPRESSION_OPTIONS((short)32, "compression_options"),
- BLOOM_FILTER_FP_CHANCE((short)33, "bloom_filter_fp_chance"),
- CACHING((short)34, "caching"),
- DCLOCAL_READ_REPAIR_CHANCE((short)37, "dclocal_read_repair_chance"),
- MEMTABLE_FLUSH_PERIOD_IN_MS((short)39, "memtable_flush_period_in_ms"),
- DEFAULT_TIME_TO_LIVE((short)40, "default_time_to_live"),
- SPECULATIVE_RETRY((short)42, "speculative_retry"),
- TRIGGERS((short)43, "triggers"),
- CELLS_PER_ROW_TO_CACHE((short)44, "cells_per_row_to_cache"),
- MIN_INDEX_INTERVAL((short)45, "min_index_interval"),
- MAX_INDEX_INTERVAL((short)46, "max_index_interval"),
- /**
- * @deprecated
- */
- ROW_CACHE_SIZE((short)9, "row_cache_size"),
- /**
- * @deprecated
- */
- KEY_CACHE_SIZE((short)11, "key_cache_size"),
- /**
- * @deprecated
- */
- ROW_CACHE_SAVE_PERIOD_IN_SECONDS((short)19, "row_cache_save_period_in_seconds"),
- /**
- * @deprecated
- */
- KEY_CACHE_SAVE_PERIOD_IN_SECONDS((short)20, "key_cache_save_period_in_seconds"),
- /**
- * @deprecated
- */
- MEMTABLE_FLUSH_AFTER_MINS((short)21, "memtable_flush_after_mins"),
- /**
- * @deprecated
- */
- MEMTABLE_THROUGHPUT_IN_MB((short)22, "memtable_throughput_in_mb"),
- /**
- * @deprecated
- */
- MEMTABLE_OPERATIONS_IN_MILLIONS((short)23, "memtable_operations_in_millions"),
- /**
- * @deprecated
- */
- REPLICATE_ON_WRITE((short)24, "replicate_on_write"),
- /**
- * @deprecated
- */
- MERGE_SHARDS_CHANCE((short)25, "merge_shards_chance"),
- /**
- * @deprecated
- */
- ROW_CACHE_PROVIDER((short)27, "row_cache_provider"),
- /**
- * @deprecated
- */
- ROW_CACHE_KEYS_TO_SAVE((short)31, "row_cache_keys_to_save"),
- /**
- * @deprecated
- */
- POPULATE_IO_CACHE_ON_FLUSH((short)38, "populate_io_cache_on_flush"),
- /**
- * @deprecated
- */
- INDEX_INTERVAL((short)41, "index_interval");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // KEYSPACE
- return KEYSPACE;
- case 2: // NAME
- return NAME;
- case 3: // COLUMN_TYPE
- return COLUMN_TYPE;
- case 5: // COMPARATOR_TYPE
- return COMPARATOR_TYPE;
- case 6: // SUBCOMPARATOR_TYPE
- return SUBCOMPARATOR_TYPE;
- case 8: // COMMENT
- return COMMENT;
- case 12: // READ_REPAIR_CHANCE
- return READ_REPAIR_CHANCE;
- case 13: // COLUMN_METADATA
- return COLUMN_METADATA;
- case 14: // GC_GRACE_SECONDS
- return GC_GRACE_SECONDS;
- case 15: // DEFAULT_VALIDATION_CLASS
- return DEFAULT_VALIDATION_CLASS;
- case 16: // ID
- return ID;
- case 17: // MIN_COMPACTION_THRESHOLD
- return MIN_COMPACTION_THRESHOLD;
- case 18: // MAX_COMPACTION_THRESHOLD
- return MAX_COMPACTION_THRESHOLD;
- case 26: // KEY_VALIDATION_CLASS
- return KEY_VALIDATION_CLASS;
- case 28: // KEY_ALIAS
- return KEY_ALIAS;
- case 29: // COMPACTION_STRATEGY
- return COMPACTION_STRATEGY;
- case 30: // COMPACTION_STRATEGY_OPTIONS
- return COMPACTION_STRATEGY_OPTIONS;
- case 32: // COMPRESSION_OPTIONS
- return COMPRESSION_OPTIONS;
- case 33: // BLOOM_FILTER_FP_CHANCE
- return BLOOM_FILTER_FP_CHANCE;
- case 34: // CACHING
- return CACHING;
- case 37: // DCLOCAL_READ_REPAIR_CHANCE
- return DCLOCAL_READ_REPAIR_CHANCE;
- case 39: // MEMTABLE_FLUSH_PERIOD_IN_MS
- return MEMTABLE_FLUSH_PERIOD_IN_MS;
- case 40: // DEFAULT_TIME_TO_LIVE
- return DEFAULT_TIME_TO_LIVE;
- case 42: // SPECULATIVE_RETRY
- return SPECULATIVE_RETRY;
- case 43: // TRIGGERS
- return TRIGGERS;
- case 44: // CELLS_PER_ROW_TO_CACHE
- return CELLS_PER_ROW_TO_CACHE;
- case 45: // MIN_INDEX_INTERVAL
- return MIN_INDEX_INTERVAL;
- case 46: // MAX_INDEX_INTERVAL
- return MAX_INDEX_INTERVAL;
- case 9: // ROW_CACHE_SIZE
- return ROW_CACHE_SIZE;
- case 11: // KEY_CACHE_SIZE
- return KEY_CACHE_SIZE;
- case 19: // ROW_CACHE_SAVE_PERIOD_IN_SECONDS
- return ROW_CACHE_SAVE_PERIOD_IN_SECONDS;
- case 20: // KEY_CACHE_SAVE_PERIOD_IN_SECONDS
- return KEY_CACHE_SAVE_PERIOD_IN_SECONDS;
- case 21: // MEMTABLE_FLUSH_AFTER_MINS
- return MEMTABLE_FLUSH_AFTER_MINS;
- case 22: // MEMTABLE_THROUGHPUT_IN_MB
- return MEMTABLE_THROUGHPUT_IN_MB;
- case 23: // MEMTABLE_OPERATIONS_IN_MILLIONS
- return MEMTABLE_OPERATIONS_IN_MILLIONS;
- case 24: // REPLICATE_ON_WRITE
- return REPLICATE_ON_WRITE;
- case 25: // MERGE_SHARDS_CHANCE
- return MERGE_SHARDS_CHANCE;
- case 27: // ROW_CACHE_PROVIDER
- return ROW_CACHE_PROVIDER;
- case 31: // ROW_CACHE_KEYS_TO_SAVE
- return ROW_CACHE_KEYS_TO_SAVE;
- case 38: // POPULATE_IO_CACHE_ON_FLUSH
- return POPULATE_IO_CACHE_ON_FLUSH;
- case 41: // INDEX_INTERVAL
- return INDEX_INTERVAL;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- private static final int __READ_REPAIR_CHANCE_ISSET_ID = 0;
- private static final int __GC_GRACE_SECONDS_ISSET_ID = 1;
- private static final int __ID_ISSET_ID = 2;
- private static final int __MIN_COMPACTION_THRESHOLD_ISSET_ID = 3;
- private static final int __MAX_COMPACTION_THRESHOLD_ISSET_ID = 4;
- private static final int __BLOOM_FILTER_FP_CHANCE_ISSET_ID = 5;
- private static final int __DCLOCAL_READ_REPAIR_CHANCE_ISSET_ID = 6;
- private static final int __MEMTABLE_FLUSH_PERIOD_IN_MS_ISSET_ID = 7;
- private static final int __DEFAULT_TIME_TO_LIVE_ISSET_ID = 8;
- private static final int __MIN_INDEX_INTERVAL_ISSET_ID = 9;
- private static final int __MAX_INDEX_INTERVAL_ISSET_ID = 10;
- private static final int __ROW_CACHE_SIZE_ISSET_ID = 11;
- private static final int __KEY_CACHE_SIZE_ISSET_ID = 12;
- private static final int __ROW_CACHE_SAVE_PERIOD_IN_SECONDS_ISSET_ID = 13;
- private static final int __KEY_CACHE_SAVE_PERIOD_IN_SECONDS_ISSET_ID = 14;
- private static final int __MEMTABLE_FLUSH_AFTER_MINS_ISSET_ID = 15;
- private static final int __MEMTABLE_THROUGHPUT_IN_MB_ISSET_ID = 16;
- private static final int __MEMTABLE_OPERATIONS_IN_MILLIONS_ISSET_ID = 17;
- private static final int __REPLICATE_ON_WRITE_ISSET_ID = 18;
- private static final int __MERGE_SHARDS_CHANCE_ISSET_ID = 19;
- private static final int __ROW_CACHE_KEYS_TO_SAVE_ISSET_ID = 20;
- private static final int __POPULATE_IO_CACHE_ON_FLUSH_ISSET_ID = 21;
- private static final int __INDEX_INTERVAL_ISSET_ID = 22;
- private int __isset_bitfield = 0;
- private _Fields optionals[] = {_Fields.COLUMN_TYPE,_Fields.COMPARATOR_TYPE,_Fields.SUBCOMPARATOR_TYPE,_Fields.COMMENT,_Fields.READ_REPAIR_CHANCE,_Fields.COLUMN_METADATA,_Fields.GC_GRACE_SECONDS,_Fields.DEFAULT_VALIDATION_CLASS,_Fields.ID,_Fields.MIN_COMPACTION_THRESHOLD,_Fields.MAX_COMPACTION_THRESHOLD,_Fields.KEY_VALIDATION_CLASS,_Fields.KEY_ALIAS,_Fields.COMPACTION_STRATEGY,_Fields.COMPACTION_STRATEGY_OPTIONS,_Fields.COMPRESSION_OPTIONS,_Fields.BLOOM_FILTER_FP_CHANCE,_Fields.CACHING,_Fields.DCLOCAL_READ_REPAIR_CHANCE,_Fields.MEMTABLE_FLUSH_PERIOD_IN_MS,_Fields.DEFAULT_TIME_TO_LIVE,_Fields.SPECULATIVE_RETRY,_Fields.TRIGGERS,_Fields.CELLS_PER_ROW_TO_CACHE,_Fields.MIN_INDEX_INTERVAL,_Fields.MAX_INDEX_INTERVAL,_Fields.ROW_CACHE_SIZE,_Fields.KEY_CACHE_SIZE,_Fields.ROW_CACHE_SAVE_PERIOD_IN_SECONDS,_Fields.KEY_CACHE_SAVE_PERIOD_IN_SECONDS,_Fields.MEMTABLE_FLUSH_AFTER_MINS,_Fields.MEMTABLE_THROUGHPUT_IN_MB,_Fields.MEMTABLE_OPERATIONS_IN_MILLIONS,_Fields.REPLICATE_ON_WRITE,_Fields.MERGE_SHARDS_CHANCE,_Fields.ROW_CACHE_PROVIDER,_Fields.ROW_CACHE_KEYS_TO_SAVE,_Fields.POPULATE_IO_CACHE_ON_FLUSH,_Fields.INDEX_INTERVAL};
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.KEYSPACE, new org.apache.thrift.meta_data.FieldMetaData("keyspace", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
- tmpMap.put(_Fields.NAME, new org.apache.thrift.meta_data.FieldMetaData("name", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
- tmpMap.put(_Fields.COLUMN_TYPE, new org.apache.thrift.meta_data.FieldMetaData("column_type", org.apache.thrift.TFieldRequirementType.OPTIONAL,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
- tmpMap.put(_Fields.COMPARATOR_TYPE, new org.apache.thrift.meta_data.FieldMetaData("comparator_type", org.apache.thrift.TFieldRequirementType.OPTIONAL,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
- tmpMap.put(_Fields.SUBCOMPARATOR_TYPE, new org.apache.thrift.meta_data.FieldMetaData("subcomparator_type", org.apache.thrift.TFieldRequirementType.OPTIONAL,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
- tmpMap.put(_Fields.COMMENT, new org.apache.thrift.meta_data.FieldMetaData("comment", org.apache.thrift.TFieldRequirementType.OPTIONAL,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
- tmpMap.put(_Fields.READ_REPAIR_CHANCE, new org.apache.thrift.meta_data.FieldMetaData("read_repair_chance", org.apache.thrift.TFieldRequirementType.OPTIONAL,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.DOUBLE)));
- tmpMap.put(_Fields.COLUMN_METADATA, new org.apache.thrift.meta_data.FieldMetaData("column_metadata", org.apache.thrift.TFieldRequirementType.OPTIONAL,
- new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST,
- new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, ColumnDef.class))));
- tmpMap.put(_Fields.GC_GRACE_SECONDS, new org.apache.thrift.meta_data.FieldMetaData("gc_grace_seconds", org.apache.thrift.TFieldRequirementType.OPTIONAL,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32)));
- tmpMap.put(_Fields.DEFAULT_VALIDATION_CLASS, new org.apache.thrift.meta_data.FieldMetaData("default_validation_class", org.apache.thrift.TFieldRequirementType.OPTIONAL,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
- tmpMap.put(_Fields.ID, new org.apache.thrift.meta_data.FieldMetaData("id", org.apache.thrift.TFieldRequirementType.OPTIONAL,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32)));
- tmpMap.put(_Fields.MIN_COMPACTION_THRESHOLD, new org.apache.thrift.meta_data.FieldMetaData("min_compaction_threshold", org.apache.thrift.TFieldRequirementType.OPTIONAL,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32)));
- tmpMap.put(_Fields.MAX_COMPACTION_THRESHOLD, new org.apache.thrift.meta_data.FieldMetaData("max_compaction_threshold", org.apache.thrift.TFieldRequirementType.OPTIONAL,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32)));
- tmpMap.put(_Fields.KEY_VALIDATION_CLASS, new org.apache.thrift.meta_data.FieldMetaData("key_validation_class", org.apache.thrift.TFieldRequirementType.OPTIONAL,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
- tmpMap.put(_Fields.KEY_ALIAS, new org.apache.thrift.meta_data.FieldMetaData("key_alias", org.apache.thrift.TFieldRequirementType.OPTIONAL,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true)));
- tmpMap.put(_Fields.COMPACTION_STRATEGY, new org.apache.thrift.meta_data.FieldMetaData("compaction_strategy", org.apache.thrift.TFieldRequirementType.OPTIONAL,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
- tmpMap.put(_Fields.COMPACTION_STRATEGY_OPTIONS, new org.apache.thrift.meta_data.FieldMetaData("compaction_strategy_options", org.apache.thrift.TFieldRequirementType.OPTIONAL,
- new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING),
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))));
- tmpMap.put(_Fields.COMPRESSION_OPTIONS, new org.apache.thrift.meta_data.FieldMetaData("compression_options", org.apache.thrift.TFieldRequirementType.OPTIONAL,
- new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING),
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))));
- tmpMap.put(_Fields.BLOOM_FILTER_FP_CHANCE, new org.apache.thrift.meta_data.FieldMetaData("bloom_filter_fp_chance", org.apache.thrift.TFieldRequirementType.OPTIONAL,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.DOUBLE)));
- tmpMap.put(_Fields.CACHING, new org.apache.thrift.meta_data.FieldMetaData("caching", org.apache.thrift.TFieldRequirementType.OPTIONAL,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
- tmpMap.put(_Fields.DCLOCAL_READ_REPAIR_CHANCE, new org.apache.thrift.meta_data.FieldMetaData("dclocal_read_repair_chance", org.apache.thrift.TFieldRequirementType.OPTIONAL,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.DOUBLE)));
- tmpMap.put(_Fields.MEMTABLE_FLUSH_PERIOD_IN_MS, new org.apache.thrift.meta_data.FieldMetaData("memtable_flush_period_in_ms", org.apache.thrift.TFieldRequirementType.OPTIONAL,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32)));
- tmpMap.put(_Fields.DEFAULT_TIME_TO_LIVE, new org.apache.thrift.meta_data.FieldMetaData("default_time_to_live", org.apache.thrift.TFieldRequirementType.OPTIONAL,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32)));
- tmpMap.put(_Fields.SPECULATIVE_RETRY, new org.apache.thrift.meta_data.FieldMetaData("speculative_retry", org.apache.thrift.TFieldRequirementType.OPTIONAL,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
- tmpMap.put(_Fields.TRIGGERS, new org.apache.thrift.meta_data.FieldMetaData("triggers", org.apache.thrift.TFieldRequirementType.OPTIONAL,
- new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST,
- new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TriggerDef.class))));
- tmpMap.put(_Fields.CELLS_PER_ROW_TO_CACHE, new org.apache.thrift.meta_data.FieldMetaData("cells_per_row_to_cache", org.apache.thrift.TFieldRequirementType.OPTIONAL,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
- tmpMap.put(_Fields.MIN_INDEX_INTERVAL, new org.apache.thrift.meta_data.FieldMetaData("min_index_interval", org.apache.thrift.TFieldRequirementType.OPTIONAL,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32)));
- tmpMap.put(_Fields.MAX_INDEX_INTERVAL, new org.apache.thrift.meta_data.FieldMetaData("max_index_interval", org.apache.thrift.TFieldRequirementType.OPTIONAL,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32)));
- tmpMap.put(_Fields.ROW_CACHE_SIZE, new org.apache.thrift.meta_data.FieldMetaData("row_cache_size", org.apache.thrift.TFieldRequirementType.OPTIONAL,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.DOUBLE)));
- tmpMap.put(_Fields.KEY_CACHE_SIZE, new org.apache.thrift.meta_data.FieldMetaData("key_cache_size", org.apache.thrift.TFieldRequirementType.OPTIONAL,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.DOUBLE)));
- tmpMap.put(_Fields.ROW_CACHE_SAVE_PERIOD_IN_SECONDS, new org.apache.thrift.meta_data.FieldMetaData("row_cache_save_period_in_seconds", org.apache.thrift.TFieldRequirementType.OPTIONAL,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32)));
- tmpMap.put(_Fields.KEY_CACHE_SAVE_PERIOD_IN_SECONDS, new org.apache.thrift.meta_data.FieldMetaData("key_cache_save_period_in_seconds", org.apache.thrift.TFieldRequirementType.OPTIONAL,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32)));
- tmpMap.put(_Fields.MEMTABLE_FLUSH_AFTER_MINS, new org.apache.thrift.meta_data.FieldMetaData("memtable_flush_after_mins", org.apache.thrift.TFieldRequirementType.OPTIONAL,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32)));
- tmpMap.put(_Fields.MEMTABLE_THROUGHPUT_IN_MB, new org.apache.thrift.meta_data.FieldMetaData("memtable_throughput_in_mb", org.apache.thrift.TFieldRequirementType.OPTIONAL,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32)));
- tmpMap.put(_Fields.MEMTABLE_OPERATIONS_IN_MILLIONS, new org.apache.thrift.meta_data.FieldMetaData("memtable_operations_in_millions", org.apache.thrift.TFieldRequirementType.OPTIONAL,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.DOUBLE)));
- tmpMap.put(_Fields.REPLICATE_ON_WRITE, new org.apache.thrift.meta_data.FieldMetaData("replicate_on_write", org.apache.thrift.TFieldRequirementType.OPTIONAL,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.BOOL)));
- tmpMap.put(_Fields.MERGE_SHARDS_CHANCE, new org.apache.thrift.meta_data.FieldMetaData("merge_shards_chance", org.apache.thrift.TFieldRequirementType.OPTIONAL,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.DOUBLE)));
- tmpMap.put(_Fields.ROW_CACHE_PROVIDER, new org.apache.thrift.meta_data.FieldMetaData("row_cache_provider", org.apache.thrift.TFieldRequirementType.OPTIONAL,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
- tmpMap.put(_Fields.ROW_CACHE_KEYS_TO_SAVE, new org.apache.thrift.meta_data.FieldMetaData("row_cache_keys_to_save", org.apache.thrift.TFieldRequirementType.OPTIONAL,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32)));
- tmpMap.put(_Fields.POPULATE_IO_CACHE_ON_FLUSH, new org.apache.thrift.meta_data.FieldMetaData("populate_io_cache_on_flush", org.apache.thrift.TFieldRequirementType.OPTIONAL,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.BOOL)));
- tmpMap.put(_Fields.INDEX_INTERVAL, new org.apache.thrift.meta_data.FieldMetaData("index_interval", org.apache.thrift.TFieldRequirementType.OPTIONAL,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(CfDef.class, metaDataMap);
- }
-
- public CfDef() {
- this.column_type = "Standard";
-
- this.comparator_type = "BytesType";
-
- this.caching = "keys_only";
-
- this.dclocal_read_repair_chance = 0;
-
- this.speculative_retry = "NONE";
-
- this.cells_per_row_to_cache = "100";
-
- }
-
- public CfDef(
- String keyspace,
- String name)
- {
- this();
- this.keyspace = keyspace;
- this.name = name;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public CfDef(CfDef other) {
- __isset_bitfield = other.__isset_bitfield;
- if (other.isSetKeyspace()) {
- this.keyspace = other.keyspace;
- }
- if (other.isSetName()) {
- this.name = other.name;
- }
- if (other.isSetColumn_type()) {
- this.column_type = other.column_type;
- }
- if (other.isSetComparator_type()) {
- this.comparator_type = other.comparator_type;
- }
- if (other.isSetSubcomparator_type()) {
- this.subcomparator_type = other.subcomparator_type;
- }
- if (other.isSetComment()) {
- this.comment = other.comment;
- }
- this.read_repair_chance = other.read_repair_chance;
- if (other.isSetColumn_metadata()) {
- List<ColumnDef> __this__column_metadata = new ArrayList<ColumnDef>(other.column_metadata.size());
- for (ColumnDef other_element : other.column_metadata) {
- __this__column_metadata.add(new ColumnDef(other_element));
- }
- this.column_metadata = __this__column_metadata;
- }
- this.gc_grace_seconds = other.gc_grace_seconds;
- if (other.isSetDefault_validation_class()) {
- this.default_validation_class = other.default_validation_class;
- }
- this.id = other.id;
- this.min_compaction_threshold = other.min_compaction_threshold;
- this.max_compaction_threshold = other.max_compaction_threshold;
- if (other.isSetKey_validation_class()) {
- this.key_validation_class = other.key_validation_class;
- }
- if (other.isSetKey_alias()) {
- this.key_alias = org.apache.thrift.TBaseHelper.copyBinary(other.key_alias);
-;
- }
- if (other.isSetCompaction_strategy()) {
- this.compaction_strategy = other.compaction_strategy;
- }
- if (other.isSetCompaction_strategy_options()) {
- Map<String,String> __this__compaction_strategy_options = new HashMap<String,String>(other.compaction_strategy_options);
- this.compaction_strategy_options = __this__compaction_strategy_options;
- }
- if (other.isSetCompression_options()) {
- Map<String,String> __this__compression_options = new HashMap<String,String>(other.compression_options);
- this.compression_options = __this__compression_options;
- }
- this.bloom_filter_fp_chance = other.bloom_filter_fp_chance;
- if (other.isSetCaching()) {
- this.caching = other.caching;
- }
- this.dclocal_read_repair_chance = other.dclocal_read_repair_chance;
- this.memtable_flush_period_in_ms = other.memtable_flush_period_in_ms;
- this.default_time_to_live = other.default_time_to_live;
- if (other.isSetSpeculative_retry()) {
- this.speculative_retry = other.speculative_retry;
- }
- if (other.isSetTriggers()) {
- List<TriggerDef> __this__triggers = new ArrayList<TriggerDef>(other.triggers.size());
- for (TriggerDef other_element : other.triggers) {
- __this__triggers.add(new TriggerDef(other_element));
- }
- this.triggers = __this__triggers;
- }
- if (other.isSetCells_per_row_to_cache()) {
- this.cells_per_row_to_cache = other.cells_per_row_to_cache;
- }
- this.min_index_interval = other.min_index_interval;
- this.max_index_interval = other.max_index_interval;
- this.row_cache_size = other.row_cache_size;
- this.key_cache_size = other.key_cache_size;
- this.row_cache_save_period_in_seconds = other.row_cache_save_period_in_seconds;
- this.key_cache_save_period_in_seconds = other.key_cache_save_period_in_seconds;
- this.memtable_flush_after_mins = other.memtable_flush_after_mins;
- this.memtable_throughput_in_mb = other.memtable_throughput_in_mb;
- this.memtable_operations_in_millions = other.memtable_operations_in_millions;
- this.replicate_on_write = other.replicate_on_write;
- this.merge_shards_chance = other.merge_shards_chance;
- if (other.isSetRow_cache_provider()) {
- this.row_cache_provider = other.row_cache_provider;
- }
- this.row_cache_keys_to_save = other.row_cache_keys_to_save;
- this.populate_io_cache_on_flush = other.populate_io_cache_on_flush;
- this.index_interval = other.index_interval;
- }
-
- public CfDef deepCopy() {
- return new CfDef(this);
- }
-
- @Override
- public void clear() {
- this.keyspace = null;
- this.name = null;
- this.column_type = "Standard";
-
- this.comparator_type = "BytesType";
-
- this.subcomparator_type = null;
- this.comment = null;
- setRead_repair_chanceIsSet(false);
- this.read_repair_chance = 0.0;
- this.column_metadata = null;
- setGc_grace_secondsIsSet(false);
- this.gc_grace_seconds = 0;
- this.default_validation_class = null;
- setIdIsSet(false);
- this.id = 0;
- setMin_compaction_thresholdIsSet(false);
- this.min_compaction_threshold = 0;
- setMax_compaction_thresholdIsSet(false);
- this.max_compaction_threshold = 0;
- this.key_validation_class = null;
- this.key_alias = null;
- this.compaction_strategy = null;
- this.compaction_strategy_options = null;
- this.compression_options = null;
- setBloom_filter_fp_chanceIsSet(false);
- this.bloom_filter_fp_chance = 0.0;
- this.caching = "keys_only";
-
- this.dclocal_read_repair_chance = 0;
-
- setMemtable_flush_period_in_msIsSet(false);
- this.memtable_flush_period_in_ms = 0;
- setDefault_time_to_liveIsSet(false);
- this.default_time_to_live = 0;
- this.speculative_retry = "NONE";
-
- this.triggers = null;
- this.cells_per_row_to_cache = "100";
-
- setMin_index_intervalIsSet(false);
- this.min_index_interval = 0;
- setMax_index_intervalIsSet(false);
- this.max_index_interval = 0;
- setRow_cache_sizeIsSet(false);
- this.row_cache_size = 0.0;
- setKey_cache_sizeIsSet(false);
- this.key_cache_size = 0.0;
- setRow_cache_save_period_in_secondsIsSet(false);
- this.row_cache_save_period_in_seconds = 0;
- setKey_cache_save_period_in_secondsIsSet(false);
- this.key_cache_save_period_in_seconds = 0;
- setMemtable_flush_after_minsIsSet(false);
- this.memtable_flush_after_mins = 0;
- setMemtable_throughput_in_mbIsSet(false);
- this.memtable_throughput_in_mb = 0;
- setMemtable_operations_in_millionsIsSet(false);
- this.memtable_operations_in_millions = 0.0;
- setReplicate_on_writeIsSet(false);
- this.replicate_on_write = false;
- setMerge_shards_chanceIsSet(false);
- this.merge_shards_chance = 0.0;
- this.row_cache_provider = null;
- setRow_cache_keys_to_saveIsSet(false);
- this.row_cache_keys_to_save = 0;
- setPopulate_io_cache_on_flushIsSet(false);
- this.populate_io_cache_on_flush = false;
- setIndex_intervalIsSet(false);
- this.index_interval = 0;
- }
-
- public String getKeyspace() {
- return this.keyspace;
- }
-
- public CfDef setKeyspace(String keyspace) {
- this.keyspace = keyspace;
- return this;
- }
-
- public void unsetKeyspace() {
- this.keyspace = null;
- }
-
- /** Returns true if field keyspace is set (has been assigned a value) and false otherwise */
- public boolean isSetKeyspace() {
- return this.keyspace != null;
- }
-
- public void setKeyspaceIsSet(boolean value) {
- if (!value) {
- this.keyspace = null;
- }
- }
-
- public String getName() {
- return this.name;
- }
-
- public CfDef setName(String name) {
- this.name = name;
- return this;
- }
-
- public void unsetName() {
- this.name = null;
- }
-
- /** Returns true if field name is set (has been assigned a value) and false otherwise */
- public boolean isSetName() {
- return this.name != null;
- }
-
- public void setNameIsSet(boolean value) {
- if (!value) {
- this.name = null;
- }
- }
-
- public String getColumn_type() {
- return this.column_type;
- }
-
- public CfDef setColumn_type(String column_type) {
- this.column_type = column_type;
- return this;
- }
-
- public void unsetColumn_type() {
- this.column_type = null;
- }
-
- /** Returns true if field column_type is set (has been assigned a value) and false otherwise */
- public boolean isSetColumn_type() {
- return this.column_type != null;
- }
-
- public void setColumn_typeIsSet(boolean value) {
- if (!value) {
- this.column_type = null;
- }
- }
-
- public String getComparator_type() {
- return this.comparator_type;
- }
-
- public CfDef setComparator_type(String comparator_type) {
- this.comparator_type = comparator_type;
- return this;
- }
-
- public void unsetComparator_type() {
- this.comparator_type = null;
- }
-
- /** Returns true if field comparator_type is set (has been assigned a value) and false otherwise */
- public boolean isSetComparator_type() {
- return this.comparator_type != null;
- }
-
- public void setComparator_typeIsSet(boolean value) {
- if (!value) {
- this.comparator_type = null;
- }
- }
-
- public String getSubcomparator_type() {
- return this.subcomparator_type;
- }
-
- public CfDef setSubcomparator_type(String subcomparator_type) {
- this.subcomparator_type = subcomparator_type;
- return this;
- }
-
- public void unsetSubcomparator_type() {
- this.subcomparator_type = null;
- }
-
- /** Returns true if field subcomparator_type is set (has been assigned a value) and false otherwise */
- public boolean isSetSubcomparator_type() {
- return this.subcomparator_type != null;
- }
-
- public void setSubcomparator_typeIsSet(boolean value) {
- if (!value) {
- this.subcomparator_type = null;
- }
- }
-
- public String getComment() {
- return this.comment;
- }
-
- public CfDef setComment(String comment) {
- this.comment = comment;
- return this;
- }
-
- public void unsetComment() {
- this.comment = null;
- }
-
- /** Returns true if field comment is set (has been assigned a value) and false otherwise */
- public boolean isSetComment() {
- return this.comment != null;
- }
-
- public void setCommentIsSet(boolean value) {
- if (!value) {
- this.comment = null;
- }
- }
-
- public double getRead_repair_chance() {
- return this.read_repair_chance;
- }
-
- public CfDef setRead_repair_chance(double read_repair_chance) {
- this.read_repair_chance = read_repair_chance;
- setRead_repair_chanceIsSet(true);
- return this;
- }
-
- public void unsetRead_repair_chance() {
- __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __READ_REPAIR_CHANCE_ISSET_ID);
- }
-
- /** Returns true if field read_repair_chance is set (has been assigned a value) and false otherwise */
- public boolean isSetRead_repair_chance() {
- return EncodingUtils.testBit(__isset_bitfield, __READ_REPAIR_CHANCE_ISSET_ID);
- }
-
- public void setRead_repair_chanceIsSet(boolean value) {
- __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __READ_REPAIR_CHANCE_ISSET_ID, value);
- }
-
- public int getColumn_metadataSize() {
- return (this.column_metadata == null) ? 0 : this.column_metadata.size();
- }
-
- public java.util.Iterator<ColumnDef> getColumn_metadataIterator() {
- return (this.column_metadata == null) ? null : this.column_metadata.iterator();
- }
-
- public void addToColumn_metadata(ColumnDef elem) {
- if (this.column_metadata == null) {
- this.column_metadata = new ArrayList<ColumnDef>();
- }
- this.column_metadata.add(elem);
- }
-
- public List<ColumnDef> getColumn_metadata() {
- return this.column_metadata;
- }
-
- public CfDef setColumn_metadata(List<ColumnDef> column_metadata) {
- this.column_metadata = column_metadata;
- return this;
- }
-
- public void unsetColumn_metadata() {
- this.column_metadata = null;
- }
-
- /** Returns true if field column_metadata is set (has been assigned a value) and false otherwise */
- public boolean isSetColumn_metadata() {
- return this.column_metadata != null;
- }
-
- public void setColumn_metadataIsSet(boolean value) {
- if (!value) {
- this.column_metadata = null;
- }
- }
-
- public int getGc_grace_seconds() {
- return this.gc_grace_seconds;
- }
-
- public CfDef setGc_grace_seconds(int gc_grace_seconds) {
- this.gc_grace_seconds = gc_grace_seconds;
- setGc_grace_secondsIsSet(true);
- return this;
- }
-
- public void unsetGc_grace_seconds() {
- __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __GC_GRACE_SECONDS_ISSET_ID);
- }
-
- /** Returns true if field gc_grace_seconds is set (has been assigned a value) and false otherwise */
- public boolean isSetGc_grace_seconds() {
- return EncodingUtils.testBit(__isset_bitfield, __GC_GRACE_SECONDS_ISSET_ID);
- }
-
- public void setGc_grace_secondsIsSet(boolean value) {
- __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __GC_GRACE_SECONDS_ISSET_ID, value);
- }
-
- public String getDefault_validation_class() {
- return this.default_validation_class;
- }
-
- public CfDef setDefault_validation_class(String default_validation_class) {
- this.default_validation_class = default_validation_class;
- return this;
- }
-
- public void unsetDefault_validation_class() {
- this.default_validation_class = null;
- }
-
- /** Returns true if field default_validation_class is set (has been assigned a value) and false otherwise */
- public boolean isSetDefault_validation_class() {
- return this.default_validation_class != null;
- }
-
- public void setDefault_validation_classIsSet(boolean value) {
- if (!value) {
- this.default_validation_class = null;
- }
- }
-
- public int getId() {
- return this.id;
- }
-
- public CfDef setId(int id) {
- this.id = id;
- setIdIsSet(true);
- return this;
- }
-
- public void unsetId() {
- __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __ID_ISSET_ID);
- }
-
- /** Returns true if field id is set (has been assigned a value) and false otherwise */
- public boolean isSetId() {
- return EncodingUtils.testBit(__isset_bitfield, __ID_ISSET_ID);
- }
-
- public void setIdIsSet(boolean value) {
- __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __ID_ISSET_ID, value);
- }
-
- public int getMin_compaction_threshold() {
- return this.min_compaction_threshold;
- }
-
- public CfDef setMin_compaction_threshold(int min_compaction_threshold) {
- this.min_compaction_threshold = min_compaction_threshold;
- setMin_compaction_thresholdIsSet(true);
- return this;
- }
-
- public void unsetMin_compaction_threshold() {
- __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __MIN_COMPACTION_THRESHOLD_ISSET_ID);
- }
-
- /** Returns true if field min_compaction_threshold is set (has been assigned a value) and false otherwise */
- public boolean isSetMin_compaction_threshold() {
- return EncodingUtils.testBit(__isset_bitfield, __MIN_COMPACTION_THRESHOLD_ISSET_ID);
- }
-
- public void setMin_compaction_thresholdIsSet(boolean value) {
- __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __MIN_COMPACTION_THRESHOLD_ISSET_ID, value);
- }
-
- public int getMax_compaction_threshold() {
- return this.max_compaction_threshold;
- }
-
- public CfDef setMax_compaction_threshold(int max_compaction_threshold) {
- this.max_compaction_threshold = max_compaction_threshold;
- setMax_compaction_thresholdIsSet(true);
- return this;
- }
-
- public void unsetMax_compaction_threshold() {
- __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __MAX_COMPACTION_THRESHOLD_ISSET_ID);
- }
-
- /** Returns true if field max_compaction_threshold is set (has been assigned a value) and false otherwise */
- public boolean isSetMax_compaction_threshold() {
- return EncodingUtils.testBit(__isset_bitfield, __MAX_COMPACTION_THRESHOLD_ISSET_ID);
- }
-
- public void setMax_compaction_thresholdIsSet(boolean value) {
- __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __MAX_COMPACTION_THRESHOLD_ISSET_ID, value);
- }
-
- public String getKey_validation_class() {
- return this.key_validation_class;
- }
-
- public CfDef setKey_validation_class(String key_validation_class) {
- this.key_validation_class = key_validation_class;
- return this;
- }
-
- public void unsetKey_validation_class() {
- this.key_validation_class = null;
- }
-
- /** Returns true if field key_validation_class is set (has been assigned a value) and false otherwise */
- public boolean isSetKey_validation_class() {
- return this.key_validation_class != null;
- }
-
- public void setKey_validation_classIsSet(boolean value) {
- if (!value) {
- this.key_validation_class = null;
- }
- }
-
- public byte[] getKey_alias() {
- setKey_alias(org.apache.thrift.TBaseHelper.rightSize(key_alias));
- return key_alias == null ? null : key_alias.array();
- }
-
- public ByteBuffer bufferForKey_alias() {
- return key_alias;
- }
-
- public CfDef setKey_alias(byte[] key_alias) {
- setKey_alias(key_alias == null ? (ByteBuffer)null : ByteBuffer.wrap(key_alias));
- return this;
- }
-
- public CfDef setKey_alias(ByteBuffer key_alias) {
- this.key_alias = key_alias;
- return this;
- }
-
- public void unsetKey_alias() {
- this.key_alias = null;
- }
-
- /** Returns true if field key_alias is set (has been assigned a value) and false otherwise */
- public boolean isSetKey_alias() {
- return this.key_alias != null;
- }
-
- public void setKey_aliasIsSet(boolean value) {
- if (!value) {
- this.key_alias = null;
- }
- }
-
- public String getCompaction_strategy() {
- return this.compaction_strategy;
- }
-
- public CfDef setCompaction_strategy(String compaction_strategy) {
- this.compaction_strategy = compaction_strategy;
- return this;
- }
-
- public void unsetCompaction_strategy() {
- this.compaction_strategy = null;
- }
-
- /** Returns true if field compaction_strategy is set (has been assigned a value) and false otherwise */
- public boolean isSetCompaction_strategy() {
- return this.compaction_strategy != null;
- }
-
- public void setCompaction_strategyIsSet(boolean value) {
- if (!value) {
- this.compaction_strategy = null;
- }
- }
-
- public int getCompaction_strategy_optionsSize() {
- return (this.compaction_strategy_options == null) ? 0 : this.compaction_strategy_options.size();
- }
-
- public void putToCompaction_strategy_options(String key, String val) {
- if (this.compaction_strategy_options == null) {
- this.compaction_strategy_options = new HashMap<String,String>();
- }
- this.compaction_strategy_options.put(key, val);
- }
-
- public Map<String,String> getCompaction_strategy_options() {
- return this.compaction_strategy_options;
- }
-
- public CfDef setCompaction_strategy_options(Map<String,String> compaction_strategy_options) {
- this.compaction_strategy_options = compaction_strategy_options;
- return this;
- }
-
- public void unsetCompaction_strategy_options() {
- this.compaction_strategy_options = null;
- }
-
- /** Returns true if field compaction_strategy_options is set (has been assigned a value) and false otherwise */
- public boolean isSetCompaction_strategy_options() {
- return this.compaction_strategy_options != null;
- }
-
- public void setCompaction_strategy_optionsIsSet(boolean value) {
- if (!value) {
- this.compaction_strategy_options = null;
- }
- }
-
- public int getCompression_optionsSize() {
- return (this.compression_options == null) ? 0 : this.compression_options.size();
- }
-
- public void putToCompression_options(String key, String val) {
- if (this.compression_options == null) {
- this.compression_options = new HashMap<String,String>();
- }
- this.compression_options.put(key, val);
- }
-
- public Map<String,String> getCompression_options() {
- return this.compression_options;
- }
-
- public CfDef setCompression_options(Map<String,String> compression_options) {
- this.compression_options = compression_options;
- return this;
- }
-
- public void unsetCompression_options() {
- this.compression_options = null;
- }
-
- /** Returns true if field compression_options is set (has been assigned a value) and false otherwise */
- public boolean isSetCompression_options() {
- return this.compression_options != null;
- }
-
- public void setCompression_optionsIsSet(boolean value) {
- if (!value) {
- this.compression_options = null;
- }
- }
-
- public double getBloom_filter_fp_chance() {
- return this.bloom_filter_fp_chance;
- }
-
- public CfDef setBloom_filter_fp_chance(double bloom_filter_fp_chance) {
- this.bloom_filter_fp_chance = bloom_filter_fp_chance;
- setBloom_filter_fp_chanceIsSet(true);
- return this;
- }
-
- public void unsetBloom_filter_fp_chance() {
- __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __BLOOM_FILTER_FP_CHANCE_ISSET_ID);
- }
-
- /** Returns true if field bloom_filter_fp_chance is set (has been assigned a value) and false otherwise */
- public boolean isSetBloom_filter_fp_chance() {
- return EncodingUtils.testBit(__isset_bitfield, __BLOOM_FILTER_FP_CHANCE_ISSET_ID);
- }
-
- public void setBloom_filter_fp_chanceIsSet(boolean value) {
- __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __BLOOM_FILTER_FP_CHANCE_ISSET_ID, value);
- }
-
- public String getCaching() {
- return this.caching;
- }
-
- public CfDef setCaching(String caching) {
- this.caching = caching;
- return this;
- }
-
- public void unsetCaching() {
- this.caching = null;
- }
-
- /** Returns true if field caching is set (has been assigned a value) and false otherwise */
- public boolean isSetCaching() {
- return this.caching != null;
- }
-
- public void setCachingIsSet(boolean value) {
- if (!value) {
- this.caching = null;
- }
- }
-
- public double getDclocal_read_repair_chance() {
- return this.dclocal_read_repair_chance;
- }
-
- public CfDef setDclocal_read_repair_chance(double dclocal_read_repair_chance) {
- this.dclocal_read_repair_chance = dclocal_read_repair_chance;
- setDclocal_read_repair_chanceIsSet(true);
- return this;
- }
-
- public void unsetDclocal_read_repair_chance() {
- __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __DCLOCAL_READ_REPAIR_CHANCE_ISSET_ID);
- }
-
- /** Returns true if field dclocal_read_repair_chance is set (has been assigned a value) and false otherwise */
- public boolean isSetDclocal_read_repair_chance() {
- return EncodingUtils.testBit(__isset_bitfield, __DCLOCAL_READ_REPAIR_CHANCE_ISSET_ID);
- }
-
- public void setDclocal_read_repair_chanceIsSet(boolean value) {
- __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __DCLOCAL_READ_REPAIR_CHANCE_ISSET_ID, value);
- }
-
- public int getMemtable_flush_period_in_ms() {
- return this.memtable_flush_period_in_ms;
- }
-
- public CfDef setMemtable_flush_period_in_ms(int memtable_flush_period_in_ms) {
- this.memtable_flush_period_in_ms = memtable_flush_period_in_ms;
- setMemtable_flush_period_in_msIsSet(true);
- return this;
- }
-
- public void unsetMemtable_flush_period_in_ms() {
- __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __MEMTABLE_FLUSH_PERIOD_IN_MS_ISSET_ID);
- }
-
- /** Returns true if field memtable_flush_period_in_ms is set (has been assigned a value) and false otherwise */
- public boolean isSetMemtable_flush_period_in_ms() {
- return EncodingUtils.testBit(__isset_bitfield, __MEMTABLE_FLUSH_PERIOD_IN_MS_ISSET_ID);
- }
-
- public void setMemtable_flush_period_in_msIsSet(boolean value) {
- __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __MEMTABLE_FLUSH_PERIOD_IN_MS_ISSET_ID, value);
- }
-
- public int getDefault_time_to_live() {
- return this.default_time_to_live;
- }
-
- public CfDef setDefault_time_to_live(int default_time_to_live) {
- this.default_time_to_live = default_time_to_live;
- setDefault_time_to_liveIsSet(true);
- return this;
- }
-
- public void unsetDefault_time_to_live() {
- __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __DEFAULT_TIME_TO_LIVE_ISSET_ID);
- }
-
- /** Returns true if field default_time_to_live is set (has been assigned a value) and false otherwise */
- public boolean isSetDefault_time_to_live() {
- return EncodingUtils.testBit(__isset_bitfield, __DEFAULT_TIME_TO_LIVE_ISSET_ID);
- }
-
- public void setDefault_time_to_liveIsSet(boolean value) {
- __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __DEFAULT_TIME_TO_LIVE_ISSET_ID, value);
- }
-
- public String getSpeculative_retry() {
- return this.speculative_retry;
- }
-
- public CfDef setSpeculative_retry(String speculative_retry) {
- this.speculative_retry = speculative_retry;
- return this;
- }
-
- public void unsetSpeculative_retry() {
- this.speculative_retry = null;
- }
-
- /** Returns true if field speculative_retry is set (has been assigned a value) and false otherwise */
- public boolean isSetSpeculative_retry() {
- return this.speculative_retry != null;
- }
-
- public void setSpeculative_retryIsSet(boolean value) {
- if (!value) {
- this.speculative_retry = null;
- }
- }
-
- public int getTriggersSize() {
- return (this.triggers == null) ? 0 : this.triggers.size();
- }
-
- public java.util.Iterator<TriggerDef> getTriggersIterator() {
- return (this.triggers == null) ? null : this.triggers.iterator();
- }
-
- public void addToTriggers(TriggerDef elem) {
- if (this.triggers == null) {
- this.triggers = new ArrayList<TriggerDef>();
- }
- this.triggers.add(elem);
- }
-
- public List<TriggerDef> getTriggers() {
- return this.triggers;
- }
-
- public CfDef setTriggers(List<TriggerDef> triggers) {
- this.triggers = triggers;
- return this;
- }
-
- public void unsetTriggers() {
- this.triggers = null;
- }
-
- /** Returns true if field triggers is set (has been assigned a value) and false otherwise */
- public boolean isSetTriggers() {
- return this.triggers != null;
- }
-
- public void setTriggersIsSet(boolean value) {
- if (!value) {
- this.triggers = null;
- }
- }
-
- public String getCells_per_row_to_cache() {
- return this.cells_per_row_to_cache;
- }
-
- public CfDef setCells_per_row_to_cache(String cells_per_row_to_cache) {
- this.cells_per_row_to_cache = cells_per_row_to_cache;
- return this;
- }
-
- public void unsetCells_per_row_to_cache() {
- this.cells_per_row_to_cache = null;
- }
-
- /** Returns true if field cells_per_row_to_cache is set (has been assigned a value) and false otherwise */
- public boolean isSetCells_per_row_to_cache() {
- return this.cells_per_row_to_cache != null;
- }
-
- public void setCells_per_row_to_cacheIsSet(boolean value) {
- if (!value) {
- this.cells_per_row_to_cache = null;
- }
- }
-
- public int getMin_index_interval() {
- return this.min_index_interval;
- }
-
- public CfDef setMin_index_interval(int min_index_interval) {
- this.min_index_interval = min_index_interval;
- setMin_index_intervalIsSet(true);
- return this;
- }
-
- public void unsetMin_index_interval() {
- __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __MIN_INDEX_INTERVAL_ISSET_ID);
- }
-
- /** Returns true if field min_index_interval is set (has been assigned a value) and false otherwise */
- public boolean isSetMin_index_interval() {
- return EncodingUtils.testBit(__isset_bitfield, __MIN_INDEX_INTERVAL_ISSET_ID);
- }
-
- public void setMin_index_intervalIsSet(boolean value) {
- __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __MIN_INDEX_INTERVAL_ISSET_ID, value);
- }
-
- public int getMax_index_interval() {
- return this.max_index_interval;
- }
-
- public CfDef setMax_index_interval(int max_index_interval) {
- this.max_index_interval = max_index_interval;
- setMax_index_intervalIsSet(true);
- return this;
- }
-
- public void unsetMax_index_interval() {
- __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __MAX_INDEX_INTERVAL_ISSET_ID);
- }
-
- /** Returns true if field max_index_interval is set (has been assigned a value) and false otherwise */
- public boolean isSetMax_index_interval() {
- return EncodingUtils.testBit(__isset_bitfield, __MAX_INDEX_INTERVAL_ISSET_ID);
- }
-
- public void setMax_index_intervalIsSet(boolean value) {
- __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __MAX_INDEX_INTERVAL_ISSET_ID, value);
- }
-
- /**
- * @deprecated
- */
- public double getRow_cache_size() {
- return this.row_cache_size;
- }
-
- /**
- * @deprecated
- */
- public CfDef setRow_cache_size(double row_cache_size) {
- this.row_cache_size = row_cache_size;
- setRow_cache_sizeIsSet(true);
- return this;
- }
-
- public void unsetRow_cache_size() {
- __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __ROW_CACHE_SIZE_ISSET_ID);
- }
-
- /** Returns true if field row_cache_size is set (has been assigned a value) and false otherwise */
- public boolean isSetRow_cache_size() {
- return EncodingUtils.testBit(__isset_bitfield, __ROW_CACHE_SIZE_ISSET_ID);
- }
-
- public void setRow_cache_sizeIsSet(boolean value) {
- __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __ROW_CACHE_SIZE_ISSET_ID, value);
- }
-
- /**
- * @deprecated
- */
- public double getKey_cache_size() {
- return this.key_cache_size;
- }
-
- /**
- * @deprecated
- */
- public CfDef setKey_cache_size(double key_cache_size) {
- this.key_cache_size = key_cache_size;
- setKey_cache_sizeIsSet(true);
- return this;
- }
-
- public void unsetKey_cache_size() {
- __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __KEY_CACHE_SIZE_ISSET_ID);
- }
-
- /** Returns true if field key_cache_size is set (has been assigned a value) and false otherwise */
- public boolean isSetKey_cache_size() {
- return EncodingUtils.testBit(__isset_bitfield, __KEY_CACHE_SIZE_ISSET_ID);
- }
-
- public void setKey_cache_sizeIsSet(boolean value) {
- __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __KEY_CACHE_SIZE_ISSET_ID, value);
- }
-
- /**
- * @deprecated
- */
- public int getRow_cache_save_period_in_seconds() {
- return this.row_cache_save_period_in_seconds;
- }
-
- /**
- * @deprecated
- */
- public CfDef setRow_cache_save_period_in_seconds(int row_cache_save_period_in_seconds) {
- this.row_cache_save_period_in_seconds = row_cache_save_period_in_seconds;
- setRow_cache_save_period_in_secondsIsSet(true);
- return this;
- }
-
- public void unsetRow_cache_save_period_in_seconds() {
- __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __ROW_CACHE_SAVE_PERIOD_IN_SECONDS_ISSET_ID);
- }
-
- /** Returns true if field row_cache_save_period_in_seconds is set (has been assigned a value) and false otherwise */
- public boolean isSetRow_cache_save_period_in_seconds() {
- return EncodingUtils.testBit(__isset_bitfield, __ROW_CACHE_SAVE_PERIOD_IN_SECONDS_ISSET_ID);
- }
-
- public void setRow_cache_save_period_in_secondsIsSet(boolean value) {
- __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __ROW_CACHE_SAVE_PERIOD_IN_SECONDS_ISSET_ID, value);
- }
-
- /**
- * @deprecated
- */
- public int getKey_cache_save_period_in_seconds() {
- return this.key_cache_save_period_in_seconds;
- }
-
- /**
- * @deprecated
- */
- public CfDef setKey_cache_save_period_in_seconds(int key_cache_save_period_in_seconds) {
- this.key_cache_save_period_in_seconds = key_cache_save_period_in_seconds;
- setKey_cache_save_period_in_secondsIsSet(true);
- return this;
- }
-
- public void unsetKey_cache_save_period_in_seconds() {
- __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __KEY_CACHE_SAVE_PERIOD_IN_SECONDS_ISSET_ID);
- }
-
- /** Returns true if field key_cache_save_period_in_seconds is set (has been assigned a value) and false otherwise */
- public boolean isSetKey_cache_save_period_in_seconds() {
- return EncodingUtils.testBit(__isset_bitfield, __KEY_CACHE_SAVE_PERIOD_IN_SECONDS_ISSET_ID);
- }
-
- public void setKey_cache_save_period_in_secondsIsSet(boolean value) {
- __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __KEY_CACHE_SAVE_PERIOD_IN_SECONDS_ISSET_ID, value);
- }
-
- /**
- * @deprecated
- */
- public int getMemtable_flush_after_mins() {
- return this.memtable_flush_after_mins;
- }
-
- /**
- * @deprecated
- */
- public CfDef setMemtable_flush_after_mins(int memtable_flush_after_mins) {
- this.memtable_flush_after_mins = memtable_flush_after_mins;
- setMemtable_flush_after_minsIsSet(true);
- return this;
- }
-
- public void unsetMemtable_flush_after_mins() {
- __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __MEMTABLE_FLUSH_AFTER_MINS_ISSET_ID);
- }
-
- /** Returns true if field memtable_flush_after_mins is set (has been assigned a value) and false otherwise */
- public boolean isSetMemtable_flush_after_mins() {
- return EncodingUtils.testBit(__isset_bitfield, __MEMTABLE_FLUSH_AFTER_MINS_ISSET_ID);
- }
-
- public void setMemtable_flush_after_minsIsSet(boolean value) {
- __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __MEMTABLE_FLUSH_AFTER_MINS_ISSET_ID, value);
- }
-
- /**
- * @deprecated
- */
- public int getMemtable_throughput_in_mb() {
- return this.memtable_throughput_in_mb;
- }
-
- /**
- * @deprecated
- */
- public CfDef setMemtable_throughput_in_mb(int memtable_throughput_in_mb) {
- this.memtable_throughput_in_mb = memtable_throughput_in_mb;
- setMemtable_throughput_in_mbIsSet(true);
- return this;
- }
-
- public void unsetMemtable_throughput_in_mb() {
- __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __MEMTABLE_THROUGHPUT_IN_MB_ISSET_ID);
- }
-
- /** Returns true if field memtable_throughput_in_mb is set (has been assigned a value) and false otherwise */
- public boolean isSetMemtable_throughput_in_mb() {
- return EncodingUtils.testBit(__isset_bitfield, __MEMTABLE_THROUGHPUT_IN_MB_ISSET_ID);
- }
-
- public void setMemtable_throughput_in_mbIsSet(boolean value) {
- __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __MEMTABLE_THROUGHPUT_IN_MB_ISSET_ID, value);
- }
-
- /**
- * @deprecated
- */
- public double getMemtable_operations_in_millions() {
- return this.memtable_operations_in_millions;
- }
-
- /**
- * @deprecated
- */
- public CfDef setMemtable_operations_in_millions(double memtable_operations_in_millions) {
- this.memtable_operations_in_millions = memtable_operations_in_millions;
- setMemtable_operations_in_millionsIsSet(true);
- return this;
- }
-
- public void unsetMemtable_operations_in_millions() {
- __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __MEMTABLE_OPERATIONS_IN_MILLIONS_ISSET_ID);
- }
-
- /** Returns true if field memtable_operations_in_millions is set (has been assigned a value) and false otherwise */
- public boolean isSetMemtable_operations_in_millions() {
- return EncodingUtils.testBit(__isset_bitfield, __MEMTABLE_OPERATIONS_IN_MILLIONS_ISSET_ID);
- }
-
- public void setMemtable_operations_in_millionsIsSet(boolean value) {
- __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __MEMTABLE_OPERATIONS_IN_MILLIONS_ISSET_ID, value);
- }
-
- /**
- * @deprecated
- */
- public boolean isReplicate_on_write() {
- return this.replicate_on_write;
- }
-
- /**
- * @deprecated
- */
- public CfDef setReplicate_on_write(boolean replicate_on_write) {
- this.replicate_on_write = replicate_on_write;
- setReplicate_on_writeIsSet(true);
- return this;
- }
-
- public void unsetReplicate_on_write() {
- __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __REPLICATE_ON_WRITE_ISSET_ID);
- }
-
- /** Returns true if field replicate_on_write is set (has been assigned a value) and false otherwise */
- public boolean isSetReplicate_on_write() {
- return EncodingUtils.testBit(__isset_bitfield, __REPLICATE_ON_WRITE_ISSET_ID);
- }
-
- public void setReplicate_on_writeIsSet(boolean value) {
- __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __REPLICATE_ON_WRITE_ISSET_ID, value);
- }
-
- /**
- * @deprecated
- */
- public double getMerge_shards_chance() {
- return this.merge_shards_chance;
- }
-
- /**
- * @deprecated
- */
- public CfDef setMerge_shards_chance(double merge_shards_chance) {
- this.merge_shards_chance = merge_shards_chance;
- setMerge_shards_chanceIsSet(true);
- return this;
- }
-
- public void unsetMerge_shards_chance() {
- __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __MERGE_SHARDS_CHANCE_ISSET_ID);
- }
-
- /** Returns true if field merge_shards_chance is set (has been assigned a value) and false otherwise */
- public boolean isSetMerge_shards_chance() {
- return EncodingUtils.testBit(__isset_bitfield, __MERGE_SHARDS_CHANCE_ISSET_ID);
- }
-
- public void setMerge_shards_chanceIsSet(boolean value) {
- __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __MERGE_SHARDS_CHANCE_ISSET_ID, value);
- }
-
- /**
- * @deprecated
- */
- public String getRow_cache_provider() {
- return this.row_cache_provider;
- }
-
- /**
- * @deprecated
- */
- public CfDef setRow_cache_provider(String row_cache_provider) {
- this.row_cache_provider = row_cache_provider;
- return this;
- }
-
- public void unsetRow_cache_provider() {
- this.row_cache_provider = null;
- }
-
- /** Returns true if field row_cache_provider is set (has been assigned a value) and false otherwise */
- public boolean isSetRow_cache_provider() {
- return this.row_cache_provider != null;
- }
-
- public void setRow_cache_providerIsSet(boolean value) {
- if (!value) {
- this.row_cache_provider = null;
- }
- }
-
- /**
- * @deprecated
- */
- public int getRow_cache_keys_to_save() {
- return this.row_cache_keys_to_save;
- }
-
- /**
- * @deprecated
- */
- public CfDef setRow_cache_keys_to_save(int row_cache_keys_to_save) {
- this.row_cache_keys_to_save = row_cache_keys_to_save;
- setRow_cache_keys_to_saveIsSet(true);
- return this;
- }
-
- public void unsetRow_cache_keys_to_save() {
- __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __ROW_CACHE_KEYS_TO_SAVE_ISSET_ID);
- }
-
- /** Returns true if field row_cache_keys_to_save is set (has been assigned a value) and false otherwise */
- public boolean isSetRow_cache_keys_to_save() {
- return EncodingUtils.testBit(__isset_bitfield, __ROW_CACHE_KEYS_TO_SAVE_ISSET_ID);
- }
-
- public void setRow_cache_keys_to_saveIsSet(boolean value) {
- __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __ROW_CACHE_KEYS_TO_SAVE_ISSET_ID, value);
- }
-
- /**
- * @deprecated
- */
- public boolean isPopulate_io_cache_on_flush() {
- return this.populate_io_cache_on_flush;
- }
-
- /**
- * @deprecated
- */
- public CfDef setPopulate_io_cache_on_flush(boolean populate_io_cache_on_flush) {
- this.populate_io_cache_on_flush = populate_io_cache_on_flush;
- setPopulate_io_cache_on_flushIsSet(true);
- return this;
- }
-
- public void unsetPopulate_io_cache_on_flush() {
- __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __POPULATE_IO_CACHE_ON_FLUSH_ISSET_ID);
- }
-
- /** Returns true if field populate_io_cache_on_flush is set (has been assigned a value) and false otherwise */
- public boolean isSetPopulate_io_cache_on_flush() {
- return EncodingUtils.testBit(__isset_bitfield, __POPULATE_IO_CACHE_ON_FLUSH_ISSET_ID);
- }
-
- public void setPopulate_io_cache_on_flushIsSet(boolean value) {
- __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __POPULATE_IO_CACHE_ON_FLUSH_ISSET_ID, value);
- }
-
- /**
- * @deprecated
- */
- public int getIndex_interval() {
- return this.index_interval;
- }
-
- /**
- * @deprecated
- */
- public CfDef setIndex_interval(int index_interval) {
- this.index_interval = index_interval;
- setIndex_intervalIsSet(true);
- return this;
- }
-
- public void unsetIndex_interval() {
- __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __INDEX_INTERVAL_ISSET_ID);
- }
-
- /** Returns true if field index_interval is set (has been assigned a value) and false otherwise */
- public boolean isSetIndex_interval() {
- return EncodingUtils.testBit(__isset_bitfield, __INDEX_INTERVAL_ISSET_ID);
- }
-
- public void setIndex_intervalIsSet(boolean value) {
- __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __INDEX_INTERVAL_ISSET_ID, value);
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case KEYSPACE:
- if (value == null) {
- unsetKeyspace();
- } else {
- setKeyspace((String)value);
- }
- break;
-
- case NAME:
- if (value == null) {
- unsetName();
- } else {
- setName((String)value);
- }
- break;
-
- case COLUMN_TYPE:
- if (value == null) {
- unsetColumn_type();
- } else {
- setColumn_type((String)value);
- }
- break;
-
- case COMPARATOR_TYPE:
- if (value == null) {
- unsetComparator_type();
- } else {
- setComparator_type((String)value);
- }
- break;
-
- case SUBCOMPARATOR_TYPE:
- if (value == null) {
- unsetSubcomparator_type();
- } else {
- setSubcomparator_type((String)value);
- }
- break;
-
- case COMMENT:
- if (value == null) {
- unsetComment();
- } else {
- setComment((String)value);
- }
- break;
-
- case READ_REPAIR_CHANCE:
- if (value == null) {
- unsetRead_repair_chance();
- } else {
- setRead_repair_chance((Double)value);
- }
- break;
-
- case COLUMN_METADATA:
- if (value == null) {
- unsetColumn_metadata();
- } else {
- setColumn_metadata((List<ColumnDef>)value);
- }
- break;
-
- case GC_GRACE_SECONDS:
- if (value == null) {
- unsetGc_grace_seconds();
- } else {
- setGc_grace_seconds((Integer)value);
- }
- break;
-
- case DEFAULT_VALIDATION_CLASS:
- if (value == null) {
- unsetDefault_validation_class();
- } else {
- setDefault_validation_class((String)value);
- }
- break;
-
- case ID:
- if (value == null) {
- unsetId();
- } else {
- setId((Integer)value);
- }
- break;
-
- case MIN_COMPACTION_THRESHOLD:
- if (value == null) {
- unsetMin_compaction_threshold();
- } else {
- setMin_compaction_threshold((Integer)value);
- }
- break;
-
- case MAX_COMPACTION_THRESHOLD:
- if (value == null) {
- unsetMax_compaction_threshold();
- } else {
- setMax_compaction_threshold((Integer)value);
- }
- break;
-
- case KEY_VALIDATION_CLASS:
- if (value == null) {
- unsetKey_validation_class();
- } else {
- setKey_validation_class((String)value);
- }
- break;
-
- case KEY_ALIAS:
- if (value == null) {
- unsetKey_alias();
- } else {
- setKey_alias((ByteBuffer)value);
- }
- break;
-
- case COMPACTION_STRATEGY:
- if (value == null) {
- unsetCompaction_strategy();
- } else {
- setCompaction_strategy((String)value);
- }
- break;
-
- case COMPACTION_STRATEGY_OPTIONS:
- if (value == null) {
- unsetCompaction_strategy_options();
- } else {
- setCompaction_strategy_options((Map<String,String>)value);
- }
- break;
-
- case COMPRESSION_OPTIONS:
- if (value == null) {
- unsetCompression_options();
- } else {
- setCompression_options((Map<String,String>)value);
- }
- break;
-
- case BLOOM_FILTER_FP_CHANCE:
- if (value == null) {
- unsetBloom_filter_fp_chance();
- } else {
- setBloom_filter_fp_chance((Double)value);
- }
- break;
-
- case CACHING:
- if (value == null) {
- unsetCaching();
- } else {
- setCaching((String)value);
- }
- break;
-
- case DCLOCAL_READ_REPAIR_CHANCE:
- if (value == null) {
- unsetDclocal_read_repair_chance();
- } else {
- setDclocal_read_repair_chance((Double)value);
- }
- break;
-
- case MEMTABLE_FLUSH_PERIOD_IN_MS:
- if (value == null) {
- unsetMemtable_flush_period_in_ms();
- } else {
- setMemtable_flush_period_in_ms((Integer)value);
- }
- break;
-
- case DEFAULT_TIME_TO_LIVE:
- if (value == null) {
- unsetDefault_time_to_live();
- } else {
- setDefault_time_to_live((Integer)value);
- }
- break;
-
- case SPECULATIVE_RETRY:
- if (value == null) {
- unsetSpeculative_retry();
- } else {
- setSpeculative_retry((String)value);
- }
- break;
-
- case TRIGGERS:
- if (value == null) {
- unsetTriggers();
- } else {
- setTriggers((List<TriggerDef>)value);
- }
- break;
-
- case CELLS_PER_ROW_TO_CACHE:
- if (value == null) {
- unsetCells_per_row_to_cache();
- } else {
- setCells_per_row_to_cache((String)value);
- }
- break;
-
- case MIN_INDEX_INTERVAL:
- if (value == null) {
- unsetMin_index_interval();
- } else {
- setMin_index_interval((Integer)value);
- }
- break;
-
- case MAX_INDEX_INTERVAL:
- if (value == null) {
- unsetMax_index_interval();
- } else {
- setMax_index_interval((Integer)value);
- }
- break;
-
- case ROW_CACHE_SIZE:
- if (value == null) {
- unsetRow_cache_size();
- } else {
- setRow_cache_size((Double)value);
- }
- break;
-
- case KEY_CACHE_SIZE:
- if (value == null) {
- unsetKey_cache_size();
- } else {
- setKey_cache_size((Double)value);
- }
- break;
-
- case ROW_CACHE_SAVE_PERIOD_IN_SECONDS:
- if (value == null) {
- unsetRow_cache_save_period_in_seconds();
- } else {
- setRow_cache_save_period_in_seconds((Integer)value);
- }
- break;
-
- case KEY_CACHE_SAVE_PERIOD_IN_SECONDS:
- if (value == null) {
- unsetKey_cache_save_period_in_seconds();
- } else {
- setKey_cache_save_period_in_seconds((Integer)value);
- }
- break;
-
- case MEMTABLE_FLUSH_AFTER_MINS:
- if (value == null) {
- unsetMemtable_flush_after_mins();
- } else {
- setMemtable_flush_after_mins((Integer)value);
- }
- break;
-
- case MEMTABLE_THROUGHPUT_IN_MB:
- if (value == null) {
- unsetMemtable_throughput_in_mb();
- } else {
- setMemtable_throughput_in_mb((Integer)value);
- }
- break;
-
- case MEMTABLE_OPERATIONS_IN_MILLIONS:
- if (value == null) {
- unsetMemtable_operations_in_millions();
- } else {
- setMemtable_operations_in_millions((Double)value);
- }
- break;
-
- case REPLICATE_ON_WRITE:
- if (value == null) {
- unsetReplicate_on_write();
- } else {
- setReplicate_on_write((Boolean)value);
- }
- break;
-
- case MERGE_SHARDS_CHANCE:
- if (value == null) {
- unsetMerge_shards_chance();
- } else {
- setMerge_shards_chance((Double)value);
- }
- break;
-
- case ROW_CACHE_PROVIDER:
- if (value == null) {
- unsetRow_cache_provider();
- } else {
- setRow_cache_provider((String)value);
- }
- break;
-
- case ROW_CACHE_KEYS_TO_SAVE:
- if (value == null) {
- unsetRow_cache_keys_to_save();
- } else {
- setRow_cache_keys_to_save((Integer)value);
- }
- break;
-
- case POPULATE_IO_CACHE_ON_FLUSH:
- if (value == null) {
- unsetPopulate_io_cache_on_flush();
- } else {
- setPopulate_io_cache_on_flush((Boolean)value);
- }
- break;
-
- case INDEX_INTERVAL:
- if (value == null) {
- unsetIndex_interval();
- } else {
- setIndex_interval((Integer)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case KEYSPACE:
- return getKeyspace();
-
- case NAME:
- return getName();
-
- case COLUMN_TYPE:
- return getColumn_type();
-
- case COMPARATOR_TYPE:
- return getComparator_type();
-
- case SUBCOMPARATOR_TYPE:
- return getSubcomparator_type();
-
- case COMMENT:
- return getComment();
-
- case READ_REPAIR_CHANCE:
- return Double.valueOf(getRead_repair_chance());
-
- case COLUMN_METADATA:
- return getColumn_metadata();
-
- case GC_GRACE_SECONDS:
- return Integer.valueOf(getGc_grace_seconds());
-
- case DEFAULT_VALIDATION_CLASS:
- return getDefault_validation_class();
-
- case ID:
- return Integer.valueOf(getId());
-
- case MIN_COMPACTION_THRESHOLD:
- return Integer.valueOf(getMin_compaction_threshold());
-
- case MAX_COMPACTION_THRESHOLD:
- return Integer.valueOf(getMax_compaction_threshold());
-
- case KEY_VALIDATION_CLASS:
- return getKey_validation_class();
-
- case KEY_ALIAS:
- return getKey_alias();
-
- case COMPACTION_STRATEGY:
- return getCompaction_strategy();
-
- case COMPACTION_STRATEGY_OPTIONS:
- return getCompaction_strategy_options();
-
- case COMPRESSION_OPTIONS:
- return getCompression_options();
-
- case BLOOM_FILTER_FP_CHANCE:
- return Double.valueOf(getBloom_filter_fp_chance());
-
- case CACHING:
- return getCaching();
-
- case DCLOCAL_READ_REPAIR_CHANCE:
- return Double.valueOf(getDclocal_read_repair_chance());
-
- case MEMTABLE_FLUSH_PERIOD_IN_MS:
- return Integer.valueOf(getMemtable_flush_period_in_ms());
-
- case DEFAULT_TIME_TO_LIVE:
- return Integer.valueOf(getDefault_time_to_live());
-
- case SPECULATIVE_RETRY:
- return getSpeculative_retry();
-
- case TRIGGERS:
- return getTriggers();
-
- case CELLS_PER_ROW_TO_CACHE:
- return getCells_per_row_to_cache();
-
- case MIN_INDEX_INTERVAL:
- return Integer.valueOf(getMin_index_interval());
-
- case MAX_INDEX_INTERVAL:
- return Integer.valueOf(getMax_index_interval());
-
- case ROW_CACHE_SIZE:
- return Double.valueOf(getRow_cache_size());
-
- case KEY_CACHE_SIZE:
- return Double.valueOf(getKey_cache_size());
-
- case ROW_CACHE_SAVE_PERIOD_IN_SECONDS:
- return Integer.valueOf(getRow_cache_save_period_in_seconds());
-
- case KEY_CACHE_SAVE_PERIOD_IN_SECONDS:
- return Integer.valueOf(getKey_cache_save_period_in_seconds());
-
- case MEMTABLE_FLUSH_AFTER_MINS:
- return Integer.valueOf(getMemtable_flush_after_mins());
-
- case MEMTABLE_THROUGHPUT_IN_MB:
- return Integer.valueOf(getMemtable_throughput_in_mb());
-
- case MEMTABLE_OPERATIONS_IN_MILLIONS:
- return Double.valueOf(getMemtable_operations_in_millions());
-
- case REPLICATE_ON_WRITE:
- return Boolean.valueOf(isReplicate_on_write());
-
- case MERGE_SHARDS_CHANCE:
- return Double.valueOf(getMerge_shards_chance());
-
- case ROW_CACHE_PROVIDER:
- return getRow_cache_provider();
-
- case ROW_CACHE_KEYS_TO_SAVE:
- return Integer.valueOf(getRow_cache_keys_to_save());
-
- case POPULATE_IO_CACHE_ON_FLUSH:
- return Boolean.valueOf(isPopulate_io_cache_on_flush());
-
- case INDEX_INTERVAL:
- return Integer.valueOf(getIndex_interval());
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case KEYSPACE:
- return isSetKeyspace();
- case NAME:
- return isSetName();
- case COLUMN_TYPE:
- return isSetColumn_type();
- case COMPARATOR_TYPE:
- return isSetComparator_type();
- case SUBCOMPARATOR_TYPE:
- return isSetSubcomparator_type();
- case COMMENT:
- return isSetComment();
- case READ_REPAIR_CHANCE:
- return isSetRead_repair_chance();
- case COLUMN_METADATA:
- return isSetColumn_metadata();
- case GC_GRACE_SECONDS:
- return isSetGc_grace_seconds();
- case DEFAULT_VALIDATION_CLASS:
- return isSetDefault_validation_class();
- case ID:
- return isSetId();
- case MIN_COMPACTION_THRESHOLD:
- return isSetMin_compaction_threshold();
- case MAX_COMPACTION_THRESHOLD:
- return isSetMax_compaction_threshold();
- case KEY_VALIDATION_CLASS:
- return isSetKey_validation_class();
- case KEY_ALIAS:
- return isSetKey_alias();
- case COMPACTION_STRATEGY:
- return isSetCompaction_strategy();
- case COMPACTION_STRATEGY_OPTIONS:
- return isSetCompaction_strategy_options();
- case COMPRESSION_OPTIONS:
- return isSetCompression_options();
- case BLOOM_FILTER_FP_CHANCE:
- return isSetBloom_filter_fp_chance();
- case CACHING:
- return isSetCaching();
- case DCLOCAL_READ_REPAIR_CHANCE:
- return isSetDclocal_read_repair_chance();
- case MEMTABLE_FLUSH_PERIOD_IN_MS:
- return isSetMemtable_flush_period_in_ms();
- case DEFAULT_TIME_TO_LIVE:
- return isSetDefault_time_to_live();
- case SPECULATIVE_RETRY:
- return isSetSpeculative_retry();
- case TRIGGERS:
- return isSetTriggers();
- case CELLS_PER_ROW_TO_CACHE:
- return isSetCells_per_row_to_cache();
- case MIN_INDEX_INTERVAL:
- return isSetMin_index_interval();
- case MAX_INDEX_INTERVAL:
- return isSetMax_index_interval();
- case ROW_CACHE_SIZE:
- return isSetRow_cache_size();
- case KEY_CACHE_SIZE:
- return isSetKey_cache_size();
- case ROW_CACHE_SAVE_PERIOD_IN_SECONDS:
- return isSetRow_cache_save_period_in_seconds();
- case KEY_CACHE_SAVE_PERIOD_IN_SECONDS:
- return isSetKey_cache_save_period_in_seconds();
- case MEMTABLE_FLUSH_AFTER_MINS:
- return isSetMemtable_flush_after_mins();
- case MEMTABLE_THROUGHPUT_IN_MB:
- return isSetMemtable_throughput_in_mb();
- case MEMTABLE_OPERATIONS_IN_MILLIONS:
- return isSetMemtable_operations_in_millions();
- case REPLICATE_ON_WRITE:
- return isSetReplicate_on_write();
- case MERGE_SHARDS_CHANCE:
- return isSetMerge_shards_chance();
- case ROW_CACHE_PROVIDER:
- return isSetRow_cache_provider();
- case ROW_CACHE_KEYS_TO_SAVE:
- return isSetRow_cache_keys_to_save();
- case POPULATE_IO_CACHE_ON_FLUSH:
- return isSetPopulate_io_cache_on_flush();
- case INDEX_INTERVAL:
- return isSetIndex_interval();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof CfDef)
- return this.equals((CfDef)that);
- return false;
- }
-
- public boolean equals(CfDef that) {
- if (that == null)
- return false;
-
- boolean this_present_keyspace = true && this.isSetKeyspace();
- boolean that_present_keyspace = true && that.isSetKeyspace();
- if (this_present_keyspace || that_present_keyspace) {
- if (!(this_present_keyspace && that_present_keyspace))
- return false;
- if (!this.keyspace.equals(that.keyspace))
- return false;
- }
-
- boolean this_present_name = true && this.isSetName();
- boolean that_present_name = true && that.isSetName();
- if (this_present_name || that_present_name) {
- if (!(this_present_name && that_present_name))
- return false;
- if (!this.name.equals(that.name))
- return false;
- }
-
- boolean this_present_column_type = true && this.isSetColumn_type();
- boolean that_present_column_type = true && that.isSetColumn_type();
- if (this_present_column_type || that_present_column_type) {
- if (!(this_present_column_type && that_present_column_type))
- return false;
- if (!this.column_type.equals(that.column_type))
- return false;
- }
-
- boolean this_present_comparator_type = true && this.isSetComparator_type();
- boolean that_present_comparator_type = true && that.isSetComparator_type();
- if (this_present_comparator_type || that_present_comparator_type) {
- if (!(this_present_comparator_type && that_present_comparator_type))
- return false;
- if (!this.comparator_type.equals(that.comparator_type))
- return false;
- }
-
- boolean this_present_subcomparator_type = true && this.isSetSubcomparator_type();
- boolean that_present_subcomparator_type = true && that.isSetSubcomparator_type();
- if (this_present_subcomparator_type || that_present_subcomparator_type) {
- if (!(this_present_subcomparator_type && that_present_subcomparator_type))
- return false;
- if (!this.subcomparator_type.equals(that.subcomparator_type))
- return false;
- }
-
- boolean this_present_comment = true && this.isSetComment();
- boolean that_present_comment = true && that.isSetComment();
- if (this_present_comment || that_present_comment) {
- if (!(this_present_comment && that_present_comment))
- return false;
- if (!this.comment.equals(that.comment))
- return false;
- }
-
- boolean this_present_read_repair_chance = true && this.isSetRead_repair_chance();
- boolean that_present_read_repair_chance = true && that.isSetRead_repair_chance();
- if (this_present_read_repair_chance || that_present_read_repair_chance) {
- if (!(this_present_read_repair_chance && that_present_read_repair_chance))
- return false;
- if (this.read_repair_chance != that.read_repair_chance)
- return false;
- }
-
- boolean this_present_column_metadata = true && this.isSetColumn_metadata();
- boolean that_present_column_metadata = true && that.isSetColumn_metadata();
- if (this_present_column_metadata || that_present_column_metadata) {
- if (!(this_present_column_metadata && that_present_column_metadata))
- return false;
- if (!this.column_metadata.equals(that.column_metadata))
- return false;
- }
-
- boolean this_present_gc_grace_seconds = true && this.isSetGc_grace_seconds();
- boolean that_present_gc_grace_seconds = true && that.isSetGc_grace_seconds();
- if (this_present_gc_grace_seconds || that_present_gc_grace_seconds) {
- if (!(this_present_gc_grace_seconds && that_present_gc_grace_seconds))
- return false;
- if (this.gc_grace_seconds != that.gc_grace_seconds)
- return false;
- }
-
- boolean this_present_default_validation_class = true && this.isSetDefault_validation_class();
- boolean that_present_default_validation_class = true && that.isSetDefault_validation_class();
- if (this_present_default_validation_class || that_present_default_validation_class) {
- if (!(this_present_default_validation_class && that_present_default_validation_class))
- return false;
- if (!this.default_validation_class.equals(that.default_validation_class))
- return false;
- }
-
- boolean this_present_id = true && this.isSetId();
- boolean that_present_id = true && that.isSetId();
- if (this_present_id || that_present_id) {
- if (!(this_present_id && that_present_id))
- return false;
- if (this.id != that.id)
- return false;
- }
-
- boolean this_present_min_compaction_threshold = true && this.isSetMin_compaction_threshold();
- boolean that_present_min_compaction_threshold = true && that.isSetMin_compaction_threshold();
- if (this_present_min_compaction_threshold || that_present_min_compaction_threshold) {
- if (!(this_present_min_compaction_threshold && that_present_min_compaction_threshold))
- return false;
- if (this.min_compaction_threshold != that.min_compaction_threshold)
- return false;
- }
-
- boolean this_present_max_compaction_threshold = true && this.isSetMax_compaction_threshold();
- boolean that_present_max_compaction_threshold = true && that.isSetMax_compaction_threshold();
- if (this_present_max_compaction_threshold || that_present_max_compaction_threshold) {
- if (!(this_present_max_compaction_threshold && that_present_max_compaction_threshold))
- return false;
- if (this.max_compaction_threshold != that.max_compaction_threshold)
- return false;
- }
-
- boolean this_present_key_validation_class = true && this.isSetKey_validation_class();
- boolean that_present_key_validation_class = true && that.isSetKey_validation_class();
- if (this_present_key_validation_class || that_present_key_validation_class) {
- if (!(this_present_key_validation_class && that_present_key_validation_class))
- return false;
- if (!this.key_validation_class.equals(that.key_validation_class))
- return false;
- }
-
- boolean this_present_key_alias = true && this.isSetKey_alias();
- boolean that_present_key_alias = true && that.isSetKey_alias();
- if (this_present_key_alias || that_present_key_alias) {
- if (!(this_present_key_alias && that_present_key_alias))
- return false;
- if (!this.key_alias.equals(that.key_alias))
- return false;
- }
-
- boolean this_present_compaction_strategy = true && this.isSetCompaction_strategy();
- boolean that_present_compaction_strategy = true && that.isSetCompaction_strategy();
- if (this_present_compaction_strategy || that_present_compaction_strategy) {
- if (!(this_present_compaction_strategy && that_present_compaction_strategy))
- return false;
- if (!this.compaction_strategy.equals(that.compaction_strategy))
- return false;
- }
-
- boolean this_present_compaction_strategy_options = true && this.isSetCompaction_strategy_options();
- boolean that_present_compaction_strategy_options = true && that.isSetCompaction_strategy_options();
- if (this_present_compaction_strategy_options || that_present_compaction_strategy_options) {
- if (!(this_present_compaction_strategy_options && that_present_compaction_strategy_options))
- return false;
- if (!this.compaction_strategy_options.equals(that.compaction_strategy_options))
- return false;
- }
-
- boolean this_present_compression_options = true && this.isSetCompression_options();
- boolean that_present_compression_options = true && that.isSetCompression_options();
- if (this_present_compression_options || that_present_compression_options) {
- if (!(this_present_compression_options && that_present_compression_options))
- return false;
- if (!this.compression_options.equals(that.compression_options))
- return false;
- }
-
- boolean this_present_bloom_filter_fp_chance = true && this.isSetBloom_filter_fp_chance();
- boolean that_present_bloom_filter_fp_chance = true && that.isSetBloom_filter_fp_chance();
- if (this_present_bloom_filter_fp_chance || that_present_bloom_filter_fp_chance) {
- if (!(this_present_bloom_filter_fp_chance && that_present_bloom_filter_fp_chance))
- return false;
- if (this.bloom_filter_fp_chance != that.bloom_filter_fp_chance)
- return false;
- }
-
- boolean this_present_caching = true && this.isSetCaching();
- boolean that_present_caching = true && that.isSetCaching();
- if (this_present_caching || that_present_caching) {
- if (!(this_present_caching && that_present_caching))
- return false;
- if (!this.caching.equals(that.caching))
- return false;
- }
-
- boolean this_present_dclocal_read_repair_chance = true && this.isSetDclocal_read_repair_chance();
- boolean that_present_dclocal_read_repair_chance = true && that.isSetDclocal_read_repair_chance();
- if (this_present_dclocal_read_repair_chance || that_present_dclocal_read_repair_chance) {
- if (!(this_present_dclocal_read_repair_chance && that_present_dclocal_read_repair_chance))
- return false;
- if (this.dclocal_read_repair_chance != that.dclocal_read_repair_chance)
- return false;
- }
-
- boolean this_present_memtable_flush_period_in_ms = true && this.isSetMemtable_flush_period_in_ms();
- boolean that_present_memtable_flush_period_in_ms = true && that.isSetMemtable_flush_period_in_ms();
- if (this_present_memtable_flush_period_in_ms || that_present_memtable_flush_period_in_ms) {
- if (!(this_present_memtable_flush_period_in_ms && that_present_memtable_flush_period_in_ms))
- return false;
- if (this.memtable_flush_period_in_ms != that.memtable_flush_period_in_ms)
- return false;
- }
-
- boolean this_present_default_time_to_live = true && this.isSetDefault_time_to_live();
- boolean that_present_default_time_to_live = true && that.isSetDefault_time_to_live();
- if (this_present_default_time_to_live || that_present_default_time_to_live) {
- if (!(this_present_default_time_to_live && that_present_default_time_to_live))
- return false;
- if (this.default_time_to_live != that.default_time_to_live)
- return false;
- }
-
- boolean this_present_speculative_retry = true && this.isSetSpeculative_retry();
- boolean that_present_speculative_retry = true && that.isSetSpeculative_retry();
- if (this_present_speculative_retry || that_present_speculative_retry) {
- if (!(this_present_speculative_retry && that_present_speculative_retry))
- return false;
- if (!this.speculative_retry.equals(that.speculative_retry))
- return false;
- }
-
- boolean this_present_triggers = true && this.isSetTriggers();
- boolean that_present_triggers = true && that.isSetTriggers();
- if (this_present_triggers || that_present_triggers) {
- if (!(this_present_triggers && that_present_triggers))
- return false;
- if (!this.triggers.equals(that.triggers))
- return false;
- }
-
- boolean this_present_cells_per_row_to_cache = true && this.isSetCells_per_row_to_cache();
- boolean that_present_cells_per_row_to_cache = true && that.isSetCells_per_row_to_cache();
- if (this_present_cells_per_row_to_cache || that_present_cells_per_row_to_cache) {
- if (!(this_present_cells_per_row_to_cache && that_present_cells_per_row_to_cache))
- return false;
- if (!this.cells_per_row_to_cache.equals(that.cells_per_row_to_cache))
- return false;
- }
-
- boolean this_present_min_index_interval = true && this.isSetMin_index_interval();
- boolean that_present_min_index_interval = true && that.isSetMin_index_interval();
- if (this_present_min_index_interval || that_present_min_index_interval) {
- if (!(this_present_min_index_interval && that_present_min_index_interval))
- return false;
- if (this.min_index_interval != that.min_index_interval)
- return false;
- }
-
- boolean this_present_max_index_interval = true && this.isSetMax_index_interval();
- boolean that_present_max_index_interval = true && that.isSetMax_index_interval();
- if (this_present_max_index_interval || that_present_max_index_interval) {
- if (!(this_present_max_index_interval && that_present_max_index_interval))
- return false;
- if (this.max_index_interval != that.max_index_interval)
- return false;
- }
-
- boolean this_present_row_cache_size = true && this.isSetRow_cache_size();
- boolean that_present_row_cache_size = true && that.isSetRow_cache_size();
- if (this_present_row_cache_size || that_present_row_cache_size) {
- if (!(this_present_row_cache_size && that_present_row_cache_size))
- return false;
- if (this.row_cache_size != that.row_cache_size)
- return false;
- }
-
- boolean this_present_key_cache_size = true && this.isSetKey_cache_size();
- boolean that_present_key_cache_size = true && that.isSetKey_cache_size();
- if (this_present_key_cache_size || that_present_key_cache_size) {
- if (!(this_present_key_cache_size && that_present_key_cache_size))
- return false;
- if (this.key_cache_size != that.key_cache_size)
- return false;
- }
-
- boolean this_present_row_cache_save_period_in_seconds = true && this.isSetRow_cache_save_period_in_seconds();
- boolean that_present_row_cache_save_period_in_seconds = true && that.isSetRow_cache_save_period_in_seconds();
- if (this_present_row_cache_save_period_in_seconds || that_present_row_cache_save_period_in_seconds) {
- if (!(this_present_row_cache_save_period_in_seconds && that_present_row_cache_save_period_in_seconds))
- return false;
- if (this.row_cache_save_period_in_seconds != that.row_cache_save_period_in_seconds)
- return false;
- }
-
- boolean this_present_key_cache_save_period_in_seconds = true && this.isSetKey_cache_save_period_in_seconds();
- boolean that_present_key_cache_save_period_in_seconds = true && that.isSetKey_cache_save_period_in_seconds();
- if (this_present_key_cache_save_period_in_seconds || that_present_key_cache_save_period_in_seconds) {
- if (!(this_present_key_cache_save_period_in_seconds && that_present_key_cache_save_period_in_seconds))
- return false;
- if (this.key_cache_save_period_in_seconds != that.key_cache_save_period_in_seconds)
- return false;
- }
-
- boolean this_present_memtable_flush_after_mins = true && this.isSetMemtable_flush_after_mins();
- boolean that_present_memtable_flush_after_mins = true && that.isSetMemtable_flush_after_mins();
- if (this_present_memtable_flush_after_mins || that_present_memtable_flush_after_mins) {
- if (!(this_present_memtable_flush_after_mins && that_present_memtable_flush_after_mins))
- return false;
- if (this.memtable_flush_after_mins != that.memtable_flush_after_mins)
- return false;
- }
-
- boolean this_present_memtable_throughput_in_mb = true && this.isSetMemtable_throughput_in_mb();
- boolean that_present_memtable_throughput_in_mb = true && that.isSetMemtable_throughput_in_mb();
- if (this_present_memtable_throughput_in_mb || that_present_memtable_throughput_in_mb) {
- if (!(this_present_memtable_throughput_in_mb && that_present_memtable_throughput_in_mb))
- return false;
- if (this.memtable_throughput_in_mb != that.memtable_throughput_in_mb)
- return false;
- }
-
- boolean this_present_memtable_operations_in_millions = true && this.isSetMemtable_operations_in_millions();
- boolean that_present_memtable_operations_in_millions = true && that.isSetMemtable_operations_in_millions();
- if (this_present_memtable_operations_in_millions || that_present_memtable_operations_in_millions) {
- if (!(this_present_memtable_operations_in_millions && that_present_memtable_operations_in_millions))
- return false;
- if (this.memtable_operations_in_millions != that.memtable_operations_in_millions)
- return false;
- }
-
- boolean this_present_replicate_on_write = true && this.isSetReplicate_on_write();
- boolean that_present_replicate_on_write = true && that.isSetReplicate_on_write();
- if (this_present_replicate_on_write || that_present_replicate_on_write) {
- if (!(this_present_replicate_on_write && that_present_replicate_on_write))
- return false;
- if (this.replicate_on_write != that.replicate_on_write)
- return false;
- }
-
- boolean this_present_merge_shards_chance = true && this.isSetMerge_shards_chance();
- boolean that_present_merge_shards_chance = true && that.isSetMerge_shards_chance();
- if (this_present_merge_shards_chance || that_present_merge_shards_chance) {
- if (!(this_present_merge_shards_chance && that_present_merge_shards_chance))
- return false;
- if (this.merge_shards_chance != that.merge_shards_chance)
- return false;
- }
-
- boolean this_present_row_cache_provider = true && this.isSetRow_cache_provider();
- boolean that_present_row_cache_provider = true && that.isSetRow_cache_provider();
- if (this_present_row_cache_provider || that_present_row_cache_provider) {
- if (!(this_present_row_cache_provider && that_present_row_cache_provider))
- return false;
- if (!this.row_cache_provider.equals(that.row_cache_provider))
- return false;
- }
-
- boolean this_present_row_cache_keys_to_save = true && this.isSetRow_cache_keys_to_save();
- boolean that_present_row_cache_keys_to_save = true && that.isSetRow_cache_keys_to_save();
- if (this_present_row_cache_keys_to_save || that_present_row_cache_keys_to_save) {
- if (!(this_present_row_cache_keys_to_save && that_present_row_cache_keys_to_save))
- return false;
- if (this.row_cache_keys_to_save != that.row_cache_keys_to_save)
- return false;
- }
-
- boolean this_present_populate_io_cache_on_flush = true && this.isSetPopulate_io_cache_on_flush();
- boolean that_present_populate_io_cache_on_flush = true && that.isSetPopulate_io_cache_on_flush();
- if (this_present_populate_io_cache_on_flush || that_present_populate_io_cache_on_flush) {
- if (!(this_present_populate_io_cache_on_flush && that_present_populate_io_cache_on_flush))
- return false;
- if (this.populate_io_cache_on_flush != that.populate_io_cache_on_flush)
- return false;
- }
-
- boolean this_present_index_interval = true && this.isSetIndex_interval();
- boolean that_present_index_interval = true && that.isSetIndex_interval();
- if (this_present_index_interval || that_present_index_interval) {
- if (!(this_present_index_interval && that_present_index_interval))
- return false;
- if (this.index_interval != that.index_interval)
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_keyspace = true && (isSetKeyspace());
- builder.append(present_keyspace);
- if (present_keyspace)
- builder.append(keyspace);
-
- boolean present_name = true && (isSetName());
- builder.append(present_name);
- if (present_name)
- builder.append(name);
-
- boolean present_column_type = true && (isSetColumn_type());
- builder.append(present_column_type);
- if (present_column_type)
- builder.append(column_type);
-
- boolean present_comparator_type = true && (isSetComparator_type());
- builder.append(present_comparator_type);
- if (present_comparator_type)
- builder.append(comparator_type);
-
- boolean present_subcomparator_type = true && (isSetSubcomparator_type());
- builder.append(present_subcomparator_type);
- if (present_subcomparator_type)
- builder.append(subcomparator_type);
-
- boolean present_comment = true && (isSetComment());
- builder.append(present_comment);
- if (present_comment)
- builder.append(comment);
-
- boolean present_read_repair_chance = true && (isSetRead_repair_chance());
- builder.append(present_read_repair_chance);
- if (present_read_repair_chance)
- builder.append(read_repair_chance);
-
- boolean present_column_metadata = true && (isSetColumn_metadata());
- builder.append(present_column_metadata);
- if (present_column_metadata)
- builder.append(column_metadata);
-
- boolean present_gc_grace_seconds = true && (isSetGc_grace_seconds());
- builder.append(present_gc_grace_seconds);
- if (present_gc_grace_seconds)
- builder.append(gc_grace_seconds);
-
- boolean present_default_validation_class = true && (isSetDefault_validation_class());
- builder.append(present_default_validation_class);
- if (present_default_validation_class)
- builder.append(default_validation_class);
-
- boolean present_id = true && (isSetId());
- builder.append(present_id);
- if (present_id)
- builder.append(id);
-
- boolean present_min_compaction_threshold = true && (isSetMin_compaction_threshold());
- builder.append(present_min_compaction_threshold);
- if (present_min_compaction_threshold)
- builder.append(min_compaction_threshold);
-
- boolean present_max_compaction_threshold = true && (isSetMax_compaction_threshold());
- builder.append(present_max_compaction_threshold);
- if (present_max_compaction_threshold)
- builder.append(max_compaction_threshold);
-
- boolean present_key_validation_class = true && (isSetKey_validation_class());
- builder.append(present_key_validation_class);
- if (present_key_validation_class)
- builder.append(key_validation_class);
-
- boolean present_key_alias = true && (isSetKey_alias());
- builder.append(present_key_alias);
- if (present_key_alias)
- builder.append(key_alias);
-
- boolean present_compaction_strategy = true && (isSetCompaction_strategy());
- builder.append(present_compaction_strategy);
- if (present_compaction_strategy)
- builder.append(compaction_strategy);
-
- boolean present_compaction_strategy_options = true && (isSetCompaction_strategy_options());
- builder.append(present_compaction_strategy_options);
- if (present_compaction_strategy_options)
- builder.append(compaction_strategy_options);
-
- boolean present_compression_options = true && (isSetCompression_options());
- builder.append(present_compression_options);
- if (present_compression_options)
- builder.append(compression_options);
-
- boolean present_bloom_filter_fp_chance = true && (isSetBloom_filter_fp_chance());
- builder.append(present_bloom_filter_fp_chance);
- if (present_bloom_filter_fp_chance)
- builder.append(bloom_filter_fp_chance);
-
- boolean present_caching = true && (isSetCaching());
- builder.append(present_caching);
- if (present_caching)
- builder.append(caching);
-
- boolean present_dclocal_read_repair_chance = true && (isSetDclocal_read_repair_chance());
- builder.append(present_dclocal_read_repair_chance);
- if (present_dclocal_read_repair_chance)
- builder.append(dclocal_read_repair_chance);
-
- boolean present_memtable_flush_period_in_ms = true && (isSetMemtable_flush_period_in_ms());
- builder.append(present_memtable_flush_period_in_ms);
- if (present_memtable_flush_period_in_ms)
- builder.append(memtable_flush_period_in_ms);
-
- boolean present_default_time_to_live = true && (isSetDefault_time_to_live());
- builder.append(present_default_time_to_live);
- if (present_default_time_to_live)
- builder.append(default_time_to_live);
-
- boolean present_speculative_retry = true && (isSetSpeculative_retry());
- builder.append(present_speculative_retry);
- if (present_speculative_retry)
- builder.append(speculative_retry);
-
- boolean present_triggers = true && (isSetTriggers());
- builder.append(present_triggers);
- if (present_triggers)
- builder.append(triggers);
-
- boolean present_cells_per_row_to_cache = true && (isSetCells_per_row_to_cache());
- builder.append(present_cells_per_row_to_cache);
- if (present_cells_per_row_to_cache)
- builder.append(cells_per_row_to_cache);
-
- boolean present_min_index_interval = true && (isSetMin_index_interval());
- builder.append(present_min_index_interval);
- if (present_min_index_interval)
- builder.append(min_index_interval);
-
- boolean present_max_index_interval = true && (isSetMax_index_interval());
- builder.append(present_max_index_interval);
- if (present_max_index_interval)
- builder.append(max_index_interval);
-
- boolean present_row_cache_size = true && (isSetRow_cache_size());
- builder.append(present_row_cache_size);
- if (present_row_cache_size)
- builder.append(row_cache_size);
-
- boolean present_key_cache_size = true && (isSetKey_cache_size());
- builder.append(present_key_cache_size);
- if (present_key_cache_size)
- builder.append(key_cache_size);
-
- boolean present_row_cache_save_period_in_seconds = true && (isSetRow_cache_save_period_in_seconds());
- builder.append(present_row_cache_save_period_in_seconds);
- if (present_row_cache_save_period_in_seconds)
- builder.append(row_cache_save_period_in_seconds);
-
- boolean present_key_cache_save_period_in_seconds = true && (isSetKey_cache_save_period_in_seconds());
- builder.append(present_key_cache_save_period_in_seconds);
- if (present_key_cache_save_period_in_seconds)
- builder.append(key_cache_save_period_in_seconds);
-
- boolean present_memtable_flush_after_mins = true && (isSetMemtable_flush_after_mins());
- builder.append(present_memtable_flush_after_mins);
- if (present_memtable_flush_after_mins)
- builder.append(memtable_flush_after_mins);
-
- boolean present_memtable_throughput_in_mb = true && (isSetMemtable_throughput_in_mb());
- builder.append(present_memtable_throughput_in_mb);
- if (present_memtable_throughput_in_mb)
- builder.append(memtable_throughput_in_mb);
-
- boolean present_memtable_operations_in_millions = true && (isSetMemtable_operations_in_millions());
- builder.append(present_memtable_operations_in_millions);
- if (present_memtable_operations_in_millions)
- builder.append(memtable_operations_in_millions);
-
- boolean present_replicate_on_write = true && (isSetReplicate_on_write());
- builder.append(present_replicate_on_write);
- if (present_replicate_on_write)
- builder.append(replicate_on_write);
-
- boolean present_merge_shards_chance = true && (isSetMerge_shards_chance());
- builder.append(present_merge_shards_chance);
- if (present_merge_shards_chance)
- builder.append(merge_shards_chance);
-
- boolean present_row_cache_provider = true && (isSetRow_cache_provider());
- builder.append(present_row_cache_provider);
- if (present_row_cache_provider)
- builder.append(row_cache_provider);
-
- boolean present_row_cache_keys_to_save = true && (isSetRow_cache_keys_to_save());
- builder.append(present_row_cache_keys_to_save);
- if (present_row_cache_keys_to_save)
- builder.append(row_cache_keys_to_save);
-
- boolean present_populate_io_cache_on_flush = true && (isSetPopulate_io_cache_on_flush());
- builder.append(present_populate_io_cache_on_flush);
- if (present_populate_io_cache_on_flush)
- builder.append(populate_io_cache_on_flush);
-
- boolean present_index_interval = true && (isSetIndex_interval());
- builder.append(present_index_interval);
- if (present_index_interval)
- builder.append(index_interval);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(CfDef other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetKeyspace()).compareTo(other.isSetKeyspace());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetKeyspace()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.keyspace, other.keyspace);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetName()).compareTo(other.isSetName());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetName()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.name, other.name);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetColumn_type()).compareTo(other.isSetColumn_type());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetColumn_type()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.column_type, other.column_type);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetComparator_type()).compareTo(other.isSetComparator_type());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetComparator_type()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.comparator_type, other.comparator_type);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetSubcomparator_type()).compareTo(other.isSetSubcomparator_type());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetSubcomparator_type()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.subcomparator_type, other.subcomparator_type);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetComment()).compareTo(other.isSetComment());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetComment()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.comment, other.comment);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetRead_repair_chance()).compareTo(other.isSetRead_repair_chance());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetRead_repair_chance()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.read_repair_chance, other.read_repair_chance);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetColumn_metadata()).compareTo(other.isSetColumn_metadata());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetColumn_metadata()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.column_metadata, other.column_metadata);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetGc_grace_seconds()).compareTo(other.isSetGc_grace_seconds());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetGc_grace_seconds()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.gc_grace_seconds, other.gc_grace_seconds);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetDefault_validation_class()).compareTo(other.isSetDefault_validation_class());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetDefault_validation_class()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.default_validation_class, other.default_validation_class);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetId()).compareTo(other.isSetId());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetId()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.id, other.id);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetMin_compaction_threshold()).compareTo(other.isSetMin_compaction_threshold());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetMin_compaction_threshold()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.min_compaction_threshold, other.min_compaction_threshold);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetMax_compaction_threshold()).compareTo(other.isSetMax_compaction_threshold());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetMax_compaction_threshold()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.max_compaction_threshold, other.max_compaction_threshold);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetKey_validation_class()).compareTo(other.isSetKey_validation_class());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetKey_validation_class()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.key_validation_class, other.key_validation_class);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetKey_alias()).compareTo(other.isSetKey_alias());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetKey_alias()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.key_alias, other.key_alias);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetCompaction_strategy()).compareTo(other.isSetCompaction_strategy());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetCompaction_strategy()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.compaction_strategy, other.compaction_strategy);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetCompaction_strategy_options()).compareTo(other.isSetCompaction_strategy_options());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetCompaction_strategy_options()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.compaction_strategy_options, other.compaction_strategy_options);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetCompression_options()).compareTo(other.isSetCompression_options());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetCompression_options()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.compression_options, other.compression_options);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetBloom_filter_fp_chance()).compareTo(other.isSetBloom_filter_fp_chance());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetBloom_filter_fp_chance()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.bloom_filter_fp_chance, other.bloom_filter_fp_chance);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetCaching()).compareTo(other.isSetCaching());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetCaching()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.caching, other.caching);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetDclocal_read_repair_chance()).compareTo(other.isSetDclocal_read_repair_chance());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetDclocal_read_repair_chance()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.dclocal_read_repair_chance, other.dclocal_read_repair_chance);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetMemtable_flush_period_in_ms()).compareTo(other.isSetMemtable_flush_period_in_ms());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetMemtable_flush_period_in_ms()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.memtable_flush_period_in_ms, other.memtable_flush_period_in_ms);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetDefault_time_to_live()).compareTo(other.isSetDefault_time_to_live());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetDefault_time_to_live()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.default_time_to_live, other.default_time_to_live);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetSpeculative_retry()).compareTo(other.isSetSpeculative_retry());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetSpeculative_retry()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.speculative_retry, other.speculative_retry);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetTriggers()).compareTo(other.isSetTriggers());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetTriggers()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.triggers, other.triggers);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetCells_per_row_to_cache()).compareTo(other.isSetCells_per_row_to_cache());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetCells_per_row_to_cache()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.cells_per_row_to_cache, other.cells_per_row_to_cache);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetMin_index_interval()).compareTo(other.isSetMin_index_interval());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetMin_index_interval()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.min_index_interval, other.min_index_interval);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetMax_index_interval()).compareTo(other.isSetMax_index_interval());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetMax_index_interval()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.max_index_interval, other.max_index_interval);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetRow_cache_size()).compareTo(other.isSetRow_cache_size());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetRow_cache_size()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.row_cache_size, other.row_cache_size);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetKey_cache_size()).compareTo(other.isSetKey_cache_size());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetKey_cache_size()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.key_cache_size, other.key_cache_size);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetRow_cache_save_period_in_seconds()).compareTo(other.isSetRow_cache_save_period_in_seconds());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetRow_cache_save_period_in_seconds()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.row_cache_save_period_in_seconds, other.row_cache_save_period_in_seconds);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetKey_cache_save_period_in_seconds()).compareTo(other.isSetKey_cache_save_period_in_seconds());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetKey_cache_save_period_in_seconds()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.key_cache_save_period_in_seconds, other.key_cache_save_period_in_seconds);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetMemtable_flush_after_mins()).compareTo(other.isSetMemtable_flush_after_mins());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetMemtable_flush_after_mins()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.memtable_flush_after_mins, other.memtable_flush_after_mins);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetMemtable_throughput_in_mb()).compareTo(other.isSetMemtable_throughput_in_mb());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetMemtable_throughput_in_mb()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.memtable_throughput_in_mb, other.memtable_throughput_in_mb);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetMemtable_operations_in_millions()).compareTo(other.isSetMemtable_operations_in_millions());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetMemtable_operations_in_millions()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.memtable_operations_in_millions, other.memtable_operations_in_millions);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetReplicate_on_write()).compareTo(other.isSetReplicate_on_write());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetReplicate_on_write()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.replicate_on_write, other.replicate_on_write);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetMerge_shards_chance()).compareTo(other.isSetMerge_shards_chance());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetMerge_shards_chance()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.merge_shards_chance, other.merge_shards_chance);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetRow_cache_provider()).compareTo(other.isSetRow_cache_provider());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetRow_cache_provider()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.row_cache_provider, other.row_cache_provider);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetRow_cache_keys_to_save()).compareTo(other.isSetRow_cache_keys_to_save());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetRow_cache_keys_to_save()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.row_cache_keys_to_save, other.row_cache_keys_to_save);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetPopulate_io_cache_on_flush()).compareTo(other.isSetPopulate_io_cache_on_flush());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetPopulate_io_cache_on_flush()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.populate_io_cache_on_flush, other.populate_io_cache_on_flush);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetIndex_interval()).compareTo(other.isSetIndex_interval());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetIndex_interval()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.index_interval, other.index_interval);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("CfDef(");
- boolean first = true;
-
- sb.append("keyspace:");
- if (this.keyspace == null) {
- sb.append("null");
- } else {
- sb.append(this.keyspace);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("name:");
- if (this.name == null) {
- sb.append("null");
- } else {
- sb.append(this.name);
- }
- first = false;
- if (isSetColumn_type()) {
- if (!first) sb.append(", ");
- sb.append("column_type:");
- if (this.column_type == null) {
- sb.append("null");
- } else {
- sb.append(this.column_type);
- }
- first = false;
- }
- if (isSetComparator_type()) {
- if (!first) sb.append(", ");
- sb.append("comparator_type:");
- if (this.comparator_type == null) {
- sb.append("null");
- } else {
- sb.append(this.comparator_type);
- }
- first = false;
- }
- if (isSetSubcomparator_type()) {
- if (!first) sb.append(", ");
- sb.append("subcomparator_type:");
- if (this.subcomparator_type == null) {
- sb.append("null");
- } else {
- sb.append(this.subcomparator_type);
- }
- first = false;
- }
- if (isSetComment()) {
- if (!first) sb.append(", ");
- sb.append("comment:");
- if (this.comment == null) {
- sb.append("null");
- } else {
- sb.append(this.comment);
- }
- first = false;
- }
- if (isSetRead_repair_chance()) {
- if (!first) sb.append(", ");
- sb.append("read_repair_chance:");
- sb.append(this.read_repair_chance);
- first = false;
- }
- if (isSetColumn_metadata()) {
- if (!first) sb.append(", ");
- sb.append("column_metadata:");
- if (this.column_metadata == null) {
- sb.append("null");
- } else {
- sb.append(this.column_metadata);
- }
- first = false;
- }
- if (isSetGc_grace_seconds()) {
- if (!first) sb.append(", ");
- sb.append("gc_grace_seconds:");
- sb.append(this.gc_grace_seconds);
- first = false;
- }
- if (isSetDefault_validation_class()) {
- if (!first) sb.append(", ");
- sb.append("default_validation_class:");
- if (this.default_validation_class == null) {
- sb.append("null");
- } else {
- sb.append(this.default_validation_class);
- }
- first = false;
- }
- if (isSetId()) {
- if (!first) sb.append(", ");
- sb.append("id:");
- sb.append(this.id);
- first = false;
- }
- if (isSetMin_compaction_threshold()) {
- if (!first) sb.append(", ");
- sb.append("min_compaction_threshold:");
- sb.append(this.min_compaction_threshold);
- first = false;
- }
- if (isSetMax_compaction_threshold()) {
- if (!first) sb.append(", ");
- sb.append("max_compaction_threshold:");
- sb.append(this.max_compaction_threshold);
- first = false;
- }
- if (isSetKey_validation_class()) {
- if (!first) sb.append(", ");
- sb.append("key_validation_class:");
- if (this.key_validation_class == null) {
- sb.append("null");
- } else {
- sb.append(this.key_validation_class);
- }
- first = false;
- }
- if (isSetKey_alias()) {
- if (!first) sb.append(", ");
- sb.append("key_alias:");
- if (this.key_alias == null) {
- sb.append("null");
- } else {
- org.apache.thrift.TBaseHelper.toString(this.key_alias, sb);
- }
- first = false;
- }
- if (isSetCompaction_strategy()) {
- if (!first) sb.append(", ");
- sb.append("compaction_strategy:");
- if (this.compaction_strategy == null) {
- sb.append("null");
- } else {
- sb.append(this.compaction_strategy);
- }
- first = false;
- }
- if (isSetCompaction_strategy_options()) {
- if (!first) sb.append(", ");
- sb.append("compaction_strategy_options:");
- if (this.compaction_strategy_options == null) {
- sb.append("null");
- } else {
- sb.append(this.compaction_strategy_options);
- }
- first = false;
- }
- if (isSetCompression_options()) {
- if (!first) sb.append(", ");
- sb.append("compression_options:");
- if (this.compression_options == null) {
- sb.append("null");
- } else {
- sb.append(this.compression_options);
- }
- first = false;
- }
- if (isSetBloom_filter_fp_chance()) {
- if (!first) sb.append(", ");
- sb.append("bloom_filter_fp_chance:");
- sb.append(this.bloom_filter_fp_chance);
- first = false;
- }
- if (isSetCaching()) {
- if (!first) sb.append(", ");
- sb.append("caching:");
- if (this.caching == null) {
- sb.append("null");
- } else {
- sb.append(this.caching);
- }
- first = false;
- }
- if (isSetDclocal_read_repair_chance()) {
- if (!first) sb.append(", ");
- sb.append("dclocal_read_repair_chance:");
- sb.append(this.dclocal_read_repair_chance);
- first = false;
- }
- if (isSetMemtable_flush_period_in_ms()) {
- if (!first) sb.append(", ");
- sb.append("memtable_flush_period_in_ms:");
- sb.append(this.memtable_flush_period_in_ms);
- first = false;
- }
- if (isSetDefault_time_to_live()) {
- if (!first) sb.append(", ");
- sb.append("default_time_to_live:");
- sb.append(this.default_time_to_live);
- first = false;
- }
- if (isSetSpeculative_retry()) {
- if (!first) sb.append(", ");
- sb.append("speculative_retry:");
- if (this.speculative_retry == null) {
- sb.append("null");
- } else {
- sb.append(this.speculative_retry);
- }
- first = false;
- }
- if (isSetTriggers()) {
- if (!first) sb.append(", ");
- sb.append("triggers:");
- if (this.triggers == null) {
- sb.append("null");
- } else {
- sb.append(this.triggers);
- }
- first = false;
- }
- if (isSetCells_per_row_to_cache()) {
- if (!first) sb.append(", ");
- sb.append("cells_per_row_to_cache:");
- if (this.cells_per_row_to_cache == null) {
- sb.append("null");
- } else {
- sb.append(this.cells_per_row_to_cache);
- }
- first = false;
- }
- if (isSetMin_index_interval()) {
- if (!first) sb.append(", ");
- sb.append("min_index_interval:");
- sb.append(this.min_index_interval);
- first = false;
- }
- if (isSetMax_index_interval()) {
- if (!first) sb.append(", ");
- sb.append("max_index_interval:");
- sb.append(this.max_index_interval);
- first = false;
- }
- if (isSetRow_cache_size()) {
- if (!first) sb.append(", ");
- sb.append("row_cache_size:");
- sb.append(this.row_cache_size);
- first = false;
- }
- if (isSetKey_cache_size()) {
- if (!first) sb.append(", ");
- sb.append("key_cache_size:");
- sb.append(this.key_cache_size);
- first = false;
- }
- if (isSetRow_cache_save_period_in_seconds()) {
- if (!first) sb.append(", ");
- sb.append("row_cache_save_period_in_seconds:");
- sb.append(this.row_cache_save_period_in_seconds);
- first = false;
- }
- if (isSetKey_cache_save_period_in_seconds()) {
- if (!first) sb.append(", ");
- sb.append("key_cache_save_period_in_seconds:");
- sb.append(this.key_cache_save_period_in_seconds);
- first = false;
- }
- if (isSetMemtable_flush_after_mins()) {
- if (!first) sb.append(", ");
- sb.append("memtable_flush_after_mins:");
- sb.append(this.memtable_flush_after_mins);
- first = false;
- }
- if (isSetMemtable_throughput_in_mb()) {
- if (!first) sb.append(", ");
- sb.append("memtable_throughput_in_mb:");
- sb.append(this.memtable_throughput_in_mb);
- first = false;
- }
- if (isSetMemtable_operations_in_millions()) {
- if (!first) sb.append(", ");
- sb.append("memtable_operations_in_millions:");
- sb.append(this.memtable_operations_in_millions);
- first = false;
- }
- if (isSetReplicate_on_write()) {
- if (!first) sb.append(", ");
- sb.append("replicate_on_write:");
- sb.append(this.replicate_on_write);
- first = false;
- }
- if (isSetMerge_shards_chance()) {
- if (!first) sb.append(", ");
- sb.append("merge_shards_chance:");
- sb.append(this.merge_shards_chance);
- first = false;
- }
- if (isSetRow_cache_provider()) {
- if (!first) sb.append(", ");
- sb.append("row_cache_provider:");
- if (this.row_cache_provider == null) {
- sb.append("null");
- } else {
- sb.append(this.row_cache_provider);
- }
- first = false;
- }
- if (isSetRow_cache_keys_to_save()) {
- if (!first) sb.append(", ");
- sb.append("row_cache_keys_to_save:");
- sb.append(this.row_cache_keys_to_save);
- first = false;
- }
- if (isSetPopulate_io_cache_on_flush()) {
- if (!first) sb.append(", ");
- sb.append("populate_io_cache_on_flush:");
- sb.append(this.populate_io_cache_on_flush);
- first = false;
- }
- if (isSetIndex_interval()) {
- if (!first) sb.append(", ");
- sb.append("index_interval:");
- sb.append(this.index_interval);
- first = false;
- }
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- if (keyspace == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'keyspace' was not present! Struct: " + toString());
- }
- if (name == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'name' was not present! Struct: " + toString());
- }
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
- __isset_bitfield = 0;
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class CfDefStandardSchemeFactory implements SchemeFactory {
- public CfDefStandardScheme getScheme() {
- return new CfDefStandardScheme();
- }
- }
-
- private static class CfDefStandardScheme extends StandardScheme<CfDef> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, CfDef struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // KEYSPACE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.keyspace = iprot.readString();
- struct.setKeyspaceIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 2: // NAME
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.name = iprot.readString();
- struct.setNameIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 3: // COLUMN_TYPE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.column_type = iprot.readString();
- struct.setColumn_typeIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 5: // COMPARATOR_TYPE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.comparator_type = iprot.readString();
- struct.setComparator_typeIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 6: // SUBCOMPARATOR_TYPE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.subcomparator_type = iprot.readString();
- struct.setSubcomparator_typeIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 8: // COMMENT
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.comment = iprot.readString();
- struct.setCommentIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 12: // READ_REPAIR_CHANCE
- if (schemeField.type == org.apache.thrift.protocol.TType.DOUBLE) {
- struct.read_repair_chance = iprot.readDouble();
- struct.setRead_repair_chanceIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 13: // COLUMN_METADATA
- if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
- {
- org.apache.thrift.protocol.TList _list110 = iprot.readListBegin();
- struct.column_metadata = new ArrayList<ColumnDef>(_list110.size);
- for (int _i111 = 0; _i111 < _list110.size; ++_i111)
- {
- ColumnDef _elem112;
- _elem112 = new ColumnDef();
- _elem112.read(iprot);
- struct.column_metadata.add(_elem112);
- }
- iprot.readListEnd();
- }
- struct.setColumn_metadataIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 14: // GC_GRACE_SECONDS
- if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
- struct.gc_grace_seconds = iprot.readI32();
- struct.setGc_grace_secondsIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 15: // DEFAULT_VALIDATION_CLASS
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.default_validation_class = iprot.readString();
- struct.setDefault_validation_classIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 16: // ID
- if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
- struct.id = iprot.readI32();
- struct.setIdIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 17: // MIN_COMPACTION_THRESHOLD
- if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
- struct.min_compaction_threshold = iprot.readI32();
- struct.setMin_compaction_thresholdIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 18: // MAX_COMPACTION_THRESHOLD
- if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
- struct.max_compaction_threshold = iprot.readI32();
- struct.setMax_compaction_thresholdIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 26: // KEY_VALIDATION_CLASS
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.key_validation_class = iprot.readString();
- struct.setKey_validation_classIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 28: // KEY_ALIAS
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.key_alias = iprot.readBinary();
- struct.setKey_aliasIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 29: // COMPACTION_STRATEGY
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.compaction_strategy = iprot.readString();
- struct.setCompaction_strategyIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 30: // COMPACTION_STRATEGY_OPTIONS
- if (schemeField.type == org.apache.thrift.protocol.TType.MAP) {
- {
- org.apache.thrift.protocol.TMap _map113 = iprot.readMapBegin();
- struct.compaction_strategy_options = new HashMap<String,String>(2*_map113.size);
- for (int _i114 = 0; _i114 < _map113.size; ++_i114)
- {
- String _key115;
- String _val116;
- _key115 = iprot.readString();
- _val116 = iprot.readString();
- struct.compaction_strategy_options.put(_key115, _val116);
- }
- iprot.readMapEnd();
- }
- struct.setCompaction_strategy_optionsIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 32: // COMPRESSION_OPTIONS
- if (schemeField.type == org.apache.thrift.protocol.TType.MAP) {
- {
- org.apache.thrift.protocol.TMap _map117 = iprot.readMapBegin();
- struct.compression_options = new HashMap<String,String>(2*_map117.size);
- for (int _i118 = 0; _i118 < _map117.size; ++_i118)
- {
- String _key119;
- String _val120;
- _key119 = iprot.readString();
- _val120 = iprot.readString();
- struct.compression_options.put(_key119, _val120);
- }
- iprot.readMapEnd();
- }
- struct.setCompression_optionsIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 33: // BLOOM_FILTER_FP_CHANCE
- if (schemeField.type == org.apache.thrift.protocol.TType.DOUBLE) {
- struct.bloom_filter_fp_chance = iprot.readDouble();
- struct.setBloom_filter_fp_chanceIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 34: // CACHING
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.caching = iprot.readString();
- struct.setCachingIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 37: // DCLOCAL_READ_REPAIR_CHANCE
- if (schemeField.type == org.apache.thrift.protocol.TType.DOUBLE) {
- struct.dclocal_read_repair_chance = iprot.readDouble();
- struct.setDclocal_read_repair_chanceIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 39: // MEMTABLE_FLUSH_PERIOD_IN_MS
- if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
- struct.memtable_flush_period_in_ms = iprot.readI32();
- struct.setMemtable_flush_period_in_msIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 40: // DEFAULT_TIME_TO_LIVE
- if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
- struct.default_time_to_live = iprot.readI32();
- struct.setDefault_time_to_liveIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 42: // SPECULATIVE_RETRY
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.speculative_retry = iprot.readString();
- struct.setSpeculative_retryIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 43: // TRIGGERS
- if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
- {
- org.apache.thrift.protocol.TList _list121 = iprot.readListBegin();
- struct.triggers = new ArrayList<TriggerDef>(_list121.size);
- for (int _i122 = 0; _i122 < _list121.size; ++_i122)
- {
- TriggerDef _elem123;
- _elem123 = new TriggerDef();
- _elem123.read(iprot);
- struct.triggers.add(_elem123);
- }
- iprot.readListEnd();
- }
- struct.setTriggersIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 44: // CELLS_PER_ROW_TO_CACHE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.cells_per_row_to_cache = iprot.readString();
- struct.setCells_per_row_to_cacheIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 45: // MIN_INDEX_INTERVAL
- if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
- struct.min_index_interval = iprot.readI32();
- struct.setMin_index_intervalIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 46: // MAX_INDEX_INTERVAL
- if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
- struct.max_index_interval = iprot.readI32();
- struct.setMax_index_intervalIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 9: // ROW_CACHE_SIZE
- if (schemeField.type == org.apache.thrift.protocol.TType.DOUBLE) {
- struct.row_cache_size = iprot.readDouble();
- struct.setRow_cache_sizeIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 11: // KEY_CACHE_SIZE
- if (schemeField.type == org.apache.thrift.protocol.TType.DOUBLE) {
- struct.key_cache_size = iprot.readDouble();
- struct.setKey_cache_sizeIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 19: // ROW_CACHE_SAVE_PERIOD_IN_SECONDS
- if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
- struct.row_cache_save_period_in_seconds = iprot.readI32();
- struct.setRow_cache_save_period_in_secondsIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 20: // KEY_CACHE_SAVE_PERIOD_IN_SECONDS
- if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
- struct.key_cache_save_period_in_seconds = iprot.readI32();
- struct.setKey_cache_save_period_in_secondsIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 21: // MEMTABLE_FLUSH_AFTER_MINS
- if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
- struct.memtable_flush_after_mins = iprot.readI32();
- struct.setMemtable_flush_after_minsIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 22: // MEMTABLE_THROUGHPUT_IN_MB
- if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
- struct.memtable_throughput_in_mb = iprot.readI32();
- struct.setMemtable_throughput_in_mbIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 23: // MEMTABLE_OPERATIONS_IN_MILLIONS
- if (schemeField.type == org.apache.thrift.protocol.TType.DOUBLE) {
- struct.memtable_operations_in_millions = iprot.readDouble();
- struct.setMemtable_operations_in_millionsIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 24: // REPLICATE_ON_WRITE
- if (schemeField.type == org.apache.thrift.protocol.TType.BOOL) {
- struct.replicate_on_write = iprot.readBool();
- struct.setReplicate_on_writeIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 25: // MERGE_SHARDS_CHANCE
- if (schemeField.type == org.apache.thrift.protocol.TType.DOUBLE) {
- struct.merge_shards_chance = iprot.readDouble();
- struct.setMerge_shards_chanceIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 27: // ROW_CACHE_PROVIDER
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.row_cache_provider = iprot.readString();
- struct.setRow_cache_providerIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 31: // ROW_CACHE_KEYS_TO_SAVE
- if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
- struct.row_cache_keys_to_save = iprot.readI32();
- struct.setRow_cache_keys_to_saveIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 38: // POPULATE_IO_CACHE_ON_FLUSH
- if (schemeField.type == org.apache.thrift.protocol.TType.BOOL) {
- struct.populate_io_cache_on_flush = iprot.readBool();
- struct.setPopulate_io_cache_on_flushIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 41: // INDEX_INTERVAL
- if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
- struct.index_interval = iprot.readI32();
- struct.setIndex_intervalIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, CfDef struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.keyspace != null) {
- oprot.writeFieldBegin(KEYSPACE_FIELD_DESC);
- oprot.writeString(struct.keyspace);
- oprot.writeFieldEnd();
- }
- if (struct.name != null) {
- oprot.writeFieldBegin(NAME_FIELD_DESC);
- oprot.writeString(struct.name);
- oprot.writeFieldEnd();
- }
- if (struct.column_type != null) {
- if (struct.isSetColumn_type()) {
- oprot.writeFieldBegin(COLUMN_TYPE_FIELD_DESC);
- oprot.writeString(struct.column_type);
- oprot.writeFieldEnd();
- }
- }
- if (struct.comparator_type != null) {
- if (struct.isSetComparator_type()) {
- oprot.writeFieldBegin(COMPARATOR_TYPE_FIELD_DESC);
- oprot.writeString(struct.comparator_type);
- oprot.writeFieldEnd();
- }
- }
- if (struct.subcomparator_type != null) {
- if (struct.isSetSubcomparator_type()) {
- oprot.writeFieldBegin(SUBCOMPARATOR_TYPE_FIELD_DESC);
- oprot.writeString(struct.subcomparator_type);
- oprot.writeFieldEnd();
- }
- }
- if (struct.comment != null) {
- if (struct.isSetComment()) {
- oprot.writeFieldBegin(COMMENT_FIELD_DESC);
- oprot.writeString(struct.comment);
- oprot.writeFieldEnd();
- }
- }
- if (struct.isSetRow_cache_size()) {
- oprot.writeFieldBegin(ROW_CACHE_SIZE_FIELD_DESC);
- oprot.writeDouble(struct.row_cache_size);
- oprot.writeFieldEnd();
- }
- if (struct.isSetKey_cache_size()) {
- oprot.writeFieldBegin(KEY_CACHE_SIZE_FIELD_DESC);
- oprot.writeDouble(struct.key_cache_size);
- oprot.writeFieldEnd();
- }
- if (struct.isSetRead_repair_chance()) {
- oprot.writeFieldBegin(READ_REPAIR_CHANCE_FIELD_DESC);
- oprot.writeDouble(struct.read_repair_chance);
- oprot.writeFieldEnd();
- }
- if (struct.column_metadata != null) {
- if (struct.isSetColumn_metadata()) {
- oprot.writeFieldBegin(COLUMN_METADATA_FIELD_DESC);
- {
- oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.column_metadata.size()));
- for (ColumnDef _iter124 : struct.column_metadata)
- {
- _iter124.write(oprot);
- }
- oprot.writeListEnd();
- }
- oprot.writeFieldEnd();
- }
- }
- if (struct.isSetGc_grace_seconds()) {
- oprot.writeFieldBegin(GC_GRACE_SECONDS_FIELD_DESC);
- oprot.writeI32(struct.gc_grace_seconds);
- oprot.writeFieldEnd();
- }
- if (struct.default_validation_class != null) {
- if (struct.isSetDefault_validation_class()) {
- oprot.writeFieldBegin(DEFAULT_VALIDATION_CLASS_FIELD_DESC);
- oprot.writeString(struct.default_validation_class);
- oprot.writeFieldEnd();
- }
- }
- if (struct.isSetId()) {
- oprot.writeFieldBegin(ID_FIELD_DESC);
- oprot.writeI32(struct.id);
- oprot.writeFieldEnd();
- }
- if (struct.isSetMin_compaction_threshold()) {
- oprot.writeFieldBegin(MIN_COMPACTION_THRESHOLD_FIELD_DESC);
- oprot.writeI32(struct.min_compaction_threshold);
- oprot.writeFieldEnd();
- }
- if (struct.isSetMax_compaction_threshold()) {
- oprot.writeFieldBegin(MAX_COMPACTION_THRESHOLD_FIELD_DESC);
- oprot.writeI32(struct.max_compaction_threshold);
- oprot.writeFieldEnd();
- }
- if (struct.isSetRow_cache_save_period_in_seconds()) {
- oprot.writeFieldBegin(ROW_CACHE_SAVE_PERIOD_IN_SECONDS_FIELD_DESC);
- oprot.writeI32(struct.row_cache_save_period_in_seconds);
- oprot.writeFieldEnd();
- }
- if (struct.isSetKey_cache_save_period_in_seconds()) {
- oprot.writeFieldBegin(KEY_CACHE_SAVE_PERIOD_IN_SECONDS_FIELD_DESC);
- oprot.writeI32(struct.key_cache_save_period_in_seconds);
- oprot.writeFieldEnd();
- }
- if (struct.isSetMemtable_flush_after_mins()) {
- oprot.writeFieldBegin(MEMTABLE_FLUSH_AFTER_MINS_FIELD_DESC);
- oprot.writeI32(struct.memtable_flush_after_mins);
- oprot.writeFieldEnd();
- }
- if (struct.isSetMemtable_throughput_in_mb()) {
- oprot.writeFieldBegin(MEMTABLE_THROUGHPUT_IN_MB_FIELD_DESC);
- oprot.writeI32(struct.memtable_throughput_in_mb);
- oprot.writeFieldEnd();
- }
- if (struct.isSetMemtable_operations_in_millions()) {
- oprot.writeFieldBegin(MEMTABLE_OPERATIONS_IN_MILLIONS_FIELD_DESC);
- oprot.writeDouble(struct.memtable_operations_in_millions);
- oprot.writeFieldEnd();
- }
- if (struct.isSetReplicate_on_write()) {
- oprot.writeFieldBegin(REPLICATE_ON_WRITE_FIELD_DESC);
- oprot.writeBool(struct.replicate_on_write);
- oprot.writeFieldEnd();
- }
- if (struct.isSetMerge_shards_chance()) {
- oprot.writeFieldBegin(MERGE_SHARDS_CHANCE_FIELD_DESC);
- oprot.writeDouble(struct.merge_shards_chance);
- oprot.writeFieldEnd();
- }
- if (struct.key_validation_class != null) {
- if (struct.isSetKey_validation_class()) {
- oprot.writeFieldBegin(KEY_VALIDATION_CLASS_FIELD_DESC);
- oprot.writeString(struct.key_validation_class);
- oprot.writeFieldEnd();
- }
- }
- if (struct.row_cache_provider != null) {
- if (struct.isSetRow_cache_provider()) {
- oprot.writeFieldBegin(ROW_CACHE_PROVIDER_FIELD_DESC);
- oprot.writeString(struct.row_cache_provider);
- oprot.writeFieldEnd();
- }
- }
- if (struct.key_alias != null) {
- if (struct.isSetKey_alias()) {
- oprot.writeFieldBegin(KEY_ALIAS_FIELD_DESC);
- oprot.writeBinary(struct.key_alias);
- oprot.writeFieldEnd();
- }
- }
- if (struct.compaction_strategy != null) {
- if (struct.isSetCompaction_strategy()) {
- oprot.writeFieldBegin(COMPACTION_STRATEGY_FIELD_DESC);
- oprot.writeString(struct.compaction_strategy);
- oprot.writeFieldEnd();
- }
- }
- if (struct.compaction_strategy_options != null) {
- if (struct.isSetCompaction_strategy_options()) {
- oprot.writeFieldBegin(COMPACTION_STRATEGY_OPTIONS_FIELD_DESC);
- {
- oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, struct.compaction_strategy_options.size()));
- for (Map.Entry<String, String> _iter125 : struct.compaction_strategy_options.entrySet())
- {
- oprot.writeString(_iter125.getKey());
- oprot.writeString(_iter125.getValue());
- }
- oprot.writeMapEnd();
- }
- oprot.writeFieldEnd();
- }
- }
- if (struct.isSetRow_cache_keys_to_save()) {
- oprot.writeFieldBegin(ROW_CACHE_KEYS_TO_SAVE_FIELD_DESC);
- oprot.writeI32(struct.row_cache_keys_to_save);
- oprot.writeFieldEnd();
- }
- if (struct.compression_options != null) {
- if (struct.isSetCompression_options()) {
- oprot.writeFieldBegin(COMPRESSION_OPTIONS_FIELD_DESC);
- {
- oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, struct.compression_options.size()));
- for (Map.Entry<String, String> _iter126 : struct.compression_options.entrySet())
- {
- oprot.writeString(_iter126.getKey());
- oprot.writeString(_iter126.getValue());
- }
- oprot.writeMapEnd();
- }
- oprot.writeFieldEnd();
- }
- }
- if (struct.isSetBloom_filter_fp_chance()) {
- oprot.writeFieldBegin(BLOOM_FILTER_FP_CHANCE_FIELD_DESC);
- oprot.writeDouble(struct.bloom_filter_fp_chance);
- oprot.writeFieldEnd();
- }
- if (struct.caching != null) {
- if (struct.isSetCaching()) {
- oprot.writeFieldBegin(CACHING_FIELD_DESC);
- oprot.writeString(struct.caching);
- oprot.writeFieldEnd();
- }
- }
- if (struct.isSetDclocal_read_repair_chance()) {
- oprot.writeFieldBegin(DCLOCAL_READ_REPAIR_CHANCE_FIELD_DESC);
- oprot.writeDouble(struct.dclocal_read_repair_chance);
- oprot.writeFieldEnd();
- }
- if (struct.isSetPopulate_io_cache_on_flush()) {
- oprot.writeFieldBegin(POPULATE_IO_CACHE_ON_FLUSH_FIELD_DESC);
- oprot.writeBool(struct.populate_io_cache_on_flush);
- oprot.writeFieldEnd();
- }
- if (struct.isSetMemtable_flush_period_in_ms()) {
- oprot.writeFieldBegin(MEMTABLE_FLUSH_PERIOD_IN_MS_FIELD_DESC);
- oprot.writeI32(struct.memtable_flush_period_in_ms);
- oprot.writeFieldEnd();
- }
- if (struct.isSetDefault_time_to_live()) {
- oprot.writeFieldBegin(DEFAULT_TIME_TO_LIVE_FIELD_DESC);
- oprot.writeI32(struct.default_time_to_live);
- oprot.writeFieldEnd();
- }
- if (struct.isSetIndex_interval()) {
- oprot.writeFieldBegin(INDEX_INTERVAL_FIELD_DESC);
- oprot.writeI32(struct.index_interval);
- oprot.writeFieldEnd();
- }
- if (struct.speculative_retry != null) {
- if (struct.isSetSpeculative_retry()) {
- oprot.writeFieldBegin(SPECULATIVE_RETRY_FIELD_DESC);
- oprot.writeString(struct.speculative_retry);
- oprot.writeFieldEnd();
- }
- }
- if (struct.triggers != null) {
- if (struct.isSetTriggers()) {
- oprot.writeFieldBegin(TRIGGERS_FIELD_DESC);
- {
- oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.triggers.size()));
- for (TriggerDef _iter127 : struct.triggers)
- {
- _iter127.write(oprot);
- }
- oprot.writeListEnd();
- }
- oprot.writeFieldEnd();
- }
- }
- if (struct.cells_per_row_to_cache != null) {
- if (struct.isSetCells_per_row_to_cache()) {
- oprot.writeFieldBegin(CELLS_PER_ROW_TO_CACHE_FIELD_DESC);
- oprot.writeString(struct.cells_per_row_to_cache);
- oprot.writeFieldEnd();
- }
- }
- if (struct.isSetMin_index_interval()) {
- oprot.writeFieldBegin(MIN_INDEX_INTERVAL_FIELD_DESC);
- oprot.writeI32(struct.min_index_interval);
- oprot.writeFieldEnd();
- }
- if (struct.isSetMax_index_interval()) {
- oprot.writeFieldBegin(MAX_INDEX_INTERVAL_FIELD_DESC);
- oprot.writeI32(struct.max_index_interval);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class CfDefTupleSchemeFactory implements SchemeFactory {
- public CfDefTupleScheme getScheme() {
- return new CfDefTupleScheme();
- }
- }
-
- private static class CfDefTupleScheme extends TupleScheme<CfDef> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, CfDef struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- oprot.writeString(struct.keyspace);
- oprot.writeString(struct.name);
- BitSet optionals = new BitSet();
- if (struct.isSetColumn_type()) {
- optionals.set(0);
- }
- if (struct.isSetComparator_type()) {
- optionals.set(1);
- }
- if (struct.isSetSubcomparator_type()) {
- optionals.set(2);
- }
- if (struct.isSetComment()) {
- optionals.set(3);
- }
- if (struct.isSetRead_repair_chance()) {
- optionals.set(4);
- }
- if (struct.isSetColumn_metadata()) {
- optionals.set(5);
- }
- if (struct.isSetGc_grace_seconds()) {
- optionals.set(6);
- }
- if (struct.isSetDefault_validation_class()) {
- optionals.set(7);
- }
- if (struct.isSetId()) {
- optionals.set(8);
- }
- if (struct.isSetMin_compaction_threshold()) {
- optionals.set(9);
- }
- if (struct.isSetMax_compaction_threshold()) {
- optionals.set(10);
- }
- if (struct.isSetKey_validation_class()) {
- optionals.set(11);
- }
- if (struct.isSetKey_alias()) {
- optionals.set(12);
- }
- if (struct.isSetCompaction_strategy()) {
- optionals.set(13);
- }
- if (struct.isSetCompaction_strategy_options()) {
- optionals.set(14);
- }
- if (struct.isSetCompression_options()) {
- optionals.set(15);
- }
- if (struct.isSetBloom_filter_fp_chance()) {
- optionals.set(16);
- }
- if (struct.isSetCaching()) {
- optionals.set(17);
- }
- if (struct.isSetDclocal_read_repair_chance()) {
- optionals.set(18);
- }
- if (struct.isSetMemtable_flush_period_in_ms()) {
- optionals.set(19);
- }
- if (struct.isSetDefault_time_to_live()) {
- optionals.set(20);
- }
- if (struct.isSetSpeculative_retry()) {
- optionals.set(21);
- }
- if (struct.isSetTriggers()) {
- optionals.set(22);
- }
- if (struct.isSetCells_per_row_to_cache()) {
- optionals.set(23);
- }
- if (struct.isSetMin_index_interval()) {
- optionals.set(24);
- }
- if (struct.isSetMax_index_interval()) {
- optionals.set(25);
- }
- if (struct.isSetRow_cache_size()) {
- optionals.set(26);
- }
- if (struct.isSetKey_cache_size()) {
- optionals.set(27);
- }
- if (struct.isSetRow_cache_save_period_in_seconds()) {
- optionals.set(28);
- }
- if (struct.isSetKey_cache_save_period_in_seconds()) {
- optionals.set(29);
- }
- if (struct.isSetMemtable_flush_after_mins()) {
- optionals.set(30);
- }
- if (struct.isSetMemtable_throughput_in_mb()) {
- optionals.set(31);
- }
- if (struct.isSetMemtable_operations_in_millions()) {
- optionals.set(32);
- }
- if (struct.isSetReplicate_on_write()) {
- optionals.set(33);
- }
- if (struct.isSetMerge_shards_chance()) {
- optionals.set(34);
- }
- if (struct.isSetRow_cache_provider()) {
- optionals.set(35);
- }
- if (struct.isSetRow_cache_keys_to_save()) {
- optionals.set(36);
- }
- if (struct.isSetPopulate_io_cache_on_flush()) {
- optionals.set(37);
- }
- if (struct.isSetIndex_interval()) {
- optionals.set(38);
- }
- oprot.writeBitSet(optionals, 39);
- if (struct.isSetColumn_type()) {
- oprot.writeString(struct.column_type);
- }
- if (struct.isSetComparator_type()) {
- oprot.writeString(struct.comparator_type);
- }
- if (struct.isSetSubcomparator_type()) {
- oprot.writeString(struct.subcomparator_type);
- }
- if (struct.isSetComment()) {
- oprot.writeString(struct.comment);
- }
- if (struct.isSetRead_repair_chance()) {
- oprot.writeDouble(struct.read_repair_chance);
- }
- if (struct.isSetColumn_metadata()) {
- {
- oprot.writeI32(struct.column_metadata.size());
- for (ColumnDef _iter128 : struct.column_metadata)
- {
- _iter128.write(oprot);
- }
- }
- }
- if (struct.isSetGc_grace_seconds()) {
- oprot.writeI32(struct.gc_grace_seconds);
- }
- if (struct.isSetDefault_validation_class()) {
- oprot.writeString(struct.default_validation_class);
- }
- if (struct.isSetId()) {
- oprot.writeI32(struct.id);
- }
- if (struct.isSetMin_compaction_threshold()) {
- oprot.writeI32(struct.min_compaction_threshold);
- }
- if (struct.isSetMax_compaction_threshold()) {
- oprot.writeI32(struct.max_compaction_threshold);
- }
- if (struct.isSetKey_validation_class()) {
- oprot.writeString(struct.key_validation_class);
- }
- if (struct.isSetKey_alias()) {
- oprot.writeBinary(struct.key_alias);
- }
- if (struct.isSetCompaction_strategy()) {
- oprot.writeString(struct.compaction_strategy);
- }
- if (struct.isSetCompaction_strategy_options()) {
- {
- oprot.writeI32(struct.compaction_strategy_options.size());
- for (Map.Entry<String, String> _iter129 : struct.compaction_strategy_options.entrySet())
- {
- oprot.writeString(_iter129.getKey());
- oprot.writeString(_iter129.getValue());
- }
- }
- }
- if (struct.isSetCompression_options()) {
- {
- oprot.writeI32(struct.compression_options.size());
- for (Map.Entry<String, String> _iter130 : struct.compression_options.entrySet())
- {
- oprot.writeString(_iter130.getKey());
- oprot.writeString(_iter130.getValue());
- }
- }
- }
- if (struct.isSetBloom_filter_fp_chance()) {
- oprot.writeDouble(struct.bloom_filter_fp_chance);
- }
- if (struct.isSetCaching()) {
- oprot.writeString(struct.caching);
- }
- if (struct.isSetDclocal_read_repair_chance()) {
- oprot.writeDouble(struct.dclocal_read_repair_chance);
- }
- if (struct.isSetMemtable_flush_period_in_ms()) {
- oprot.writeI32(struct.memtable_flush_period_in_ms);
- }
- if (struct.isSetDefault_time_to_live()) {
- oprot.writeI32(struct.default_time_to_live);
- }
- if (struct.isSetSpeculative_retry()) {
- oprot.writeString(struct.speculative_retry);
- }
- if (struct.isSetTriggers()) {
- {
- oprot.writeI32(struct.triggers.size());
- for (TriggerDef _iter131 : struct.triggers)
- {
- _iter131.write(oprot);
- }
- }
- }
- if (struct.isSetCells_per_row_to_cache()) {
- oprot.writeString(struct.cells_per_row_to_cache);
- }
- if (struct.isSetMin_index_interval()) {
- oprot.writeI32(struct.min_index_interval);
- }
- if (struct.isSetMax_index_interval()) {
- oprot.writeI32(struct.max_index_interval);
- }
- if (struct.isSetRow_cache_size()) {
- oprot.writeDouble(struct.row_cache_size);
- }
- if (struct.isSetKey_cache_size()) {
- oprot.writeDouble(struct.key_cache_size);
- }
- if (struct.isSetRow_cache_save_period_in_seconds()) {
- oprot.writeI32(struct.row_cache_save_period_in_seconds);
- }
- if (struct.isSetKey_cache_save_period_in_seconds()) {
- oprot.writeI32(struct.key_cache_save_period_in_seconds);
- }
- if (struct.isSetMemtable_flush_after_mins()) {
- oprot.writeI32(struct.memtable_flush_after_mins);
- }
- if (struct.isSetMemtable_throughput_in_mb()) {
- oprot.writeI32(struct.memtable_throughput_in_mb);
- }
- if (struct.isSetMemtable_operations_in_millions()) {
- oprot.writeDouble(struct.memtable_operations_in_millions);
- }
- if (struct.isSetReplicate_on_write()) {
- oprot.writeBool(struct.replicate_on_write);
- }
- if (struct.isSetMerge_shards_chance()) {
- oprot.writeDouble(struct.merge_shards_chance);
- }
- if (struct.isSetRow_cache_provider()) {
- oprot.writeString(struct.row_cache_provider);
- }
- if (struct.isSetRow_cache_keys_to_save()) {
- oprot.writeI32(struct.row_cache_keys_to_save);
- }
- if (struct.isSetPopulate_io_cache_on_flush()) {
- oprot.writeBool(struct.populate_io_cache_on_flush);
- }
- if (struct.isSetIndex_interval()) {
- oprot.writeI32(struct.index_interval);
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, CfDef struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- struct.keyspace = iprot.readString();
- struct.setKeyspaceIsSet(true);
- struct.name = iprot.readString();
- struct.setNameIsSet(true);
- BitSet incoming = iprot.readBitSet(39);
- if (incoming.get(0)) {
- struct.column_type = iprot.readString();
- struct.setColumn_typeIsSet(true);
- }
- if (incoming.get(1)) {
- struct.comparator_type = iprot.readString();
- struct.setComparator_typeIsSet(true);
- }
- if (incoming.get(2)) {
- struct.subcomparator_type = iprot.readString();
- struct.setSubcomparator_typeIsSet(true);
- }
- if (incoming.get(3)) {
- struct.comment = iprot.readString();
- struct.setCommentIsSet(true);
- }
- if (incoming.get(4)) {
- struct.read_repair_chance = iprot.readDouble();
- struct.setRead_repair_chanceIsSet(true);
- }
- if (incoming.get(5)) {
- {
- org.apache.thrift.protocol.TList _list132 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
- struct.column_metadata = new ArrayList<ColumnDef>(_list132.size);
- for (int _i133 = 0; _i133 < _list132.size; ++_i133)
- {
- ColumnDef _elem134;
- _elem134 = new ColumnDef();
- _elem134.read(iprot);
- struct.column_metadata.add(_elem134);
- }
- }
- struct.setColumn_metadataIsSet(true);
- }
- if (incoming.get(6)) {
- struct.gc_grace_seconds = iprot.readI32();
- struct.setGc_grace_secondsIsSet(true);
- }
- if (incoming.get(7)) {
- struct.default_validation_class = iprot.readString();
- struct.setDefault_validation_classIsSet(true);
- }
- if (incoming.get(8)) {
- struct.id = iprot.readI32();
- struct.setIdIsSet(true);
- }
- if (incoming.get(9)) {
- struct.min_compaction_threshold = iprot.readI32();
- struct.setMin_compaction_thresholdIsSet(true);
- }
- if (incoming.get(10)) {
- struct.max_compaction_threshold = iprot.readI32();
- struct.setMax_compaction_thresholdIsSet(true);
- }
- if (incoming.get(11)) {
- struct.key_validation_class = iprot.readString();
- struct.setKey_validation_classIsSet(true);
- }
- if (incoming.get(12)) {
- struct.key_alias = iprot.readBinary();
- struct.setKey_aliasIsSet(true);
- }
- if (incoming.get(13)) {
- struct.compaction_strategy = iprot.readString();
- struct.setCompaction_strategyIsSet(true);
- }
- if (incoming.get(14)) {
- {
- org.apache.thrift.protocol.TMap _map135 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32());
- struct.compaction_strategy_options = new HashMap<String,String>(2*_map135.size);
- for (int _i136 = 0; _i136 < _map135.size; ++_i136)
- {
- String _key137;
- String _val138;
- _key137 = iprot.readString();
- _val138 = iprot.readString();
- struct.compaction_strategy_options.put(_key137, _val138);
- }
- }
- struct.setCompaction_strategy_optionsIsSet(true);
- }
- if (incoming.get(15)) {
- {
- org.apache.thrift.protocol.TMap _map139 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32());
- struct.compression_options = new HashMap<String,String>(2*_map139.size);
- for (int _i140 = 0; _i140 < _map139.size; ++_i140)
- {
- String _key141;
- String _val142;
- _key141 = iprot.readString();
- _val142 = iprot.readString();
- struct.compression_options.put(_key141, _val142);
- }
- }
- struct.setCompression_optionsIsSet(true);
- }
- if (incoming.get(16)) {
- struct.bloom_filter_fp_chance = iprot.readDouble();
- struct.setBloom_filter_fp_chanceIsSet(true);
- }
- if (incoming.get(17)) {
- struct.caching = iprot.readString();
- struct.setCachingIsSet(true);
- }
- if (incoming.get(18)) {
- struct.dclocal_read_repair_chance = iprot.readDouble();
- struct.setDclocal_read_repair_chanceIsSet(true);
- }
- if (incoming.get(19)) {
- struct.memtable_flush_period_in_ms = iprot.readI32();
- struct.setMemtable_flush_period_in_msIsSet(true);
- }
- if (incoming.get(20)) {
- struct.default_time_to_live = iprot.readI32();
- struct.setDefault_time_to_liveIsSet(true);
- }
- if (incoming.get(21)) {
- struct.speculative_retry = iprot.readString();
- struct.setSpeculative_retryIsSet(true);
- }
- if (incoming.get(22)) {
- {
- org.apache.thrift.protocol.TList _list143 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
- struct.triggers = new ArrayList<TriggerDef>(_list143.size);
- for (int _i144 = 0; _i144 < _list143.size; ++_i144)
- {
- TriggerDef _elem145;
- _elem145 = new TriggerDef();
- _elem145.read(iprot);
- struct.triggers.add(_elem145);
- }
- }
- struct.setTriggersIsSet(true);
- }
- if (incoming.get(23)) {
- struct.cells_per_row_to_cache = iprot.readString();
- struct.setCells_per_row_to_cacheIsSet(true);
- }
- if (incoming.get(24)) {
- struct.min_index_interval = iprot.readI32();
- struct.setMin_index_intervalIsSet(true);
- }
- if (incoming.get(25)) {
- struct.max_index_interval = iprot.readI32();
- struct.setMax_index_intervalIsSet(true);
- }
- if (incoming.get(26)) {
- struct.row_cache_size = iprot.readDouble();
- struct.setRow_cache_sizeIsSet(true);
- }
- if (incoming.get(27)) {
- struct.key_cache_size = iprot.readDouble();
- struct.setKey_cache_sizeIsSet(true);
- }
- if (incoming.get(28)) {
- struct.row_cache_save_period_in_seconds = iprot.readI32();
- struct.setRow_cache_save_period_in_secondsIsSet(true);
- }
- if (incoming.get(29)) {
- struct.key_cache_save_period_in_seconds = iprot.readI32();
- struct.setKey_cache_save_period_in_secondsIsSet(true);
- }
- if (incoming.get(30)) {
- struct.memtable_flush_after_mins = iprot.readI32();
- struct.setMemtable_flush_after_minsIsSet(true);
- }
- if (incoming.get(31)) {
- struct.memtable_throughput_in_mb = iprot.readI32();
- struct.setMemtable_throughput_in_mbIsSet(true);
- }
- if (incoming.get(32)) {
- struct.memtable_operations_in_millions = iprot.readDouble();
- struct.setMemtable_operations_in_millionsIsSet(true);
- }
- if (incoming.get(33)) {
- struct.replicate_on_write = iprot.readBool();
- struct.setReplicate_on_writeIsSet(true);
- }
- if (incoming.get(34)) {
- struct.merge_shards_chance = iprot.readDouble();
- struct.setMerge_shards_chanceIsSet(true);
- }
- if (incoming.get(35)) {
- struct.row_cache_provider = iprot.readString();
- struct.setRow_cache_providerIsSet(true);
- }
- if (incoming.get(36)) {
- struct.row_cache_keys_to_save = iprot.readI32();
- struct.setRow_cache_keys_to_saveIsSet(true);
- }
- if (incoming.get(37)) {
- struct.populate_io_cache_on_flush = iprot.readBool();
- struct.setPopulate_io_cache_on_flushIsSet(true);
- }
- if (incoming.get(38)) {
- struct.index_interval = iprot.readI32();
- struct.setIndex_intervalIsSet(true);
- }
- }
- }
-
-}
-
diff --git a/interface/thrift/gen-java/org/apache/cassandra/thrift/CfSplit.java b/interface/thrift/gen-java/org/apache/cassandra/thrift/CfSplit.java
deleted file mode 100644
index b654f86..0000000
--- a/interface/thrift/gen-java/org/apache/cassandra/thrift/CfSplit.java
+++ /dev/null
@@ -1,601 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.1)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- * @generated
- */
-package org.apache.cassandra.thrift;
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
-
-import org.apache.commons.lang3.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.EncodingUtils;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.EnumSet;
-import java.util.Collections;
-
-/**
- * Represents input splits used by hadoop ColumnFamilyRecordReaders
- */
-public class CfSplit implements org.apache.thrift.TBase<CfSplit, CfSplit._Fields>, java.io.Serializable, Cloneable, Comparable<CfSplit> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("CfSplit");
-
- private static final org.apache.thrift.protocol.TField START_TOKEN_FIELD_DESC = new org.apache.thrift.protocol.TField("start_token", org.apache.thrift.protocol.TType.STRING, (short)1);
- private static final org.apache.thrift.protocol.TField END_TOKEN_FIELD_DESC = new org.apache.thrift.protocol.TField("end_token", org.apache.thrift.protocol.TType.STRING, (short)2);
- private static final org.apache.thrift.protocol.TField ROW_COUNT_FIELD_DESC = new org.apache.thrift.protocol.TField("row_count", org.apache.thrift.protocol.TType.I64, (short)3);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new CfSplitStandardSchemeFactory());
- schemes.put(TupleScheme.class, new CfSplitTupleSchemeFactory());
- }
-
- public String start_token; // required
- public String end_token; // required
- public long row_count; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- START_TOKEN((short)1, "start_token"),
- END_TOKEN((short)2, "end_token"),
- ROW_COUNT((short)3, "row_count");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // START_TOKEN
- return START_TOKEN;
- case 2: // END_TOKEN
- return END_TOKEN;
- case 3: // ROW_COUNT
- return ROW_COUNT;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- private static final int __ROW_COUNT_ISSET_ID = 0;
- private byte __isset_bitfield = 0;
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.START_TOKEN, new org.apache.thrift.meta_data.FieldMetaData("start_token", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
- tmpMap.put(_Fields.END_TOKEN, new org.apache.thrift.meta_data.FieldMetaData("end_token", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
- tmpMap.put(_Fields.ROW_COUNT, new org.apache.thrift.meta_data.FieldMetaData("row_count", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(CfSplit.class, metaDataMap);
- }
-
- public CfSplit() {
- }
-
- public CfSplit(
- String start_token,
- String end_token,
- long row_count)
- {
- this();
- this.start_token = start_token;
- this.end_token = end_token;
- this.row_count = row_count;
- setRow_countIsSet(true);
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public CfSplit(CfSplit other) {
- __isset_bitfield = other.__isset_bitfield;
- if (other.isSetStart_token()) {
- this.start_token = other.start_token;
- }
- if (other.isSetEnd_token()) {
- this.end_token = other.end_token;
- }
- this.row_count = other.row_count;
- }
-
- public CfSplit deepCopy() {
- return new CfSplit(this);
- }
-
- @Override
- public void clear() {
- this.start_token = null;
- this.end_token = null;
- setRow_countIsSet(false);
- this.row_count = 0;
- }
-
- public String getStart_token() {
- return this.start_token;
- }
-
- public CfSplit setStart_token(String start_token) {
- this.start_token = start_token;
- return this;
- }
-
- public void unsetStart_token() {
- this.start_token = null;
- }
-
- /** Returns true if field start_token is set (has been assigned a value) and false otherwise */
- public boolean isSetStart_token() {
- return this.start_token != null;
- }
-
- public void setStart_tokenIsSet(boolean value) {
- if (!value) {
- this.start_token = null;
- }
- }
-
- public String getEnd_token() {
- return this.end_token;
- }
-
- public CfSplit setEnd_token(String end_token) {
- this.end_token = end_token;
- return this;
- }
-
- public void unsetEnd_token() {
- this.end_token = null;
- }
-
- /** Returns true if field end_token is set (has been assigned a value) and false otherwise */
- public boolean isSetEnd_token() {
- return this.end_token != null;
- }
-
- public void setEnd_tokenIsSet(boolean value) {
- if (!value) {
- this.end_token = null;
- }
- }
-
- public long getRow_count() {
- return this.row_count;
- }
-
- public CfSplit setRow_count(long row_count) {
- this.row_count = row_count;
- setRow_countIsSet(true);
- return this;
- }
-
- public void unsetRow_count() {
- __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __ROW_COUNT_ISSET_ID);
- }
-
- /** Returns true if field row_count is set (has been assigned a value) and false otherwise */
- public boolean isSetRow_count() {
- return EncodingUtils.testBit(__isset_bitfield, __ROW_COUNT_ISSET_ID);
- }
-
- public void setRow_countIsSet(boolean value) {
- __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __ROW_COUNT_ISSET_ID, value);
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case START_TOKEN:
- if (value == null) {
- unsetStart_token();
- } else {
- setStart_token((String)value);
- }
- break;
-
- case END_TOKEN:
- if (value == null) {
- unsetEnd_token();
- } else {
- setEnd_token((String)value);
- }
- break;
-
- case ROW_COUNT:
- if (value == null) {
- unsetRow_count();
- } else {
- setRow_count((Long)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case START_TOKEN:
- return getStart_token();
-
- case END_TOKEN:
- return getEnd_token();
-
- case ROW_COUNT:
- return Long.valueOf(getRow_count());
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case START_TOKEN:
- return isSetStart_token();
- case END_TOKEN:
- return isSetEnd_token();
- case ROW_COUNT:
- return isSetRow_count();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof CfSplit)
- return this.equals((CfSplit)that);
- return false;
- }
-
- public boolean equals(CfSplit that) {
- if (that == null)
- return false;
-
- boolean this_present_start_token = true && this.isSetStart_token();
- boolean that_present_start_token = true && that.isSetStart_token();
- if (this_present_start_token || that_present_start_token) {
- if (!(this_present_start_token && that_present_start_token))
- return false;
- if (!this.start_token.equals(that.start_token))
- return false;
- }
-
- boolean this_present_end_token = true && this.isSetEnd_token();
- boolean that_present_end_token = true && that.isSetEnd_token();
- if (this_present_end_token || that_present_end_token) {
- if (!(this_present_end_token && that_present_end_token))
- return false;
- if (!this.end_token.equals(that.end_token))
- return false;
- }
-
- boolean this_present_row_count = true;
- boolean that_present_row_count = true;
- if (this_present_row_count || that_present_row_count) {
- if (!(this_present_row_count && that_present_row_count))
- return false;
- if (this.row_count != that.row_count)
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_start_token = true && (isSetStart_token());
- builder.append(present_start_token);
- if (present_start_token)
- builder.append(start_token);
-
- boolean present_end_token = true && (isSetEnd_token());
- builder.append(present_end_token);
- if (present_end_token)
- builder.append(end_token);
-
- boolean present_row_count = true;
- builder.append(present_row_count);
- if (present_row_count)
- builder.append(row_count);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(CfSplit other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetStart_token()).compareTo(other.isSetStart_token());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetStart_token()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.start_token, other.start_token);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetEnd_token()).compareTo(other.isSetEnd_token());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetEnd_token()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.end_token, other.end_token);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetRow_count()).compareTo(other.isSetRow_count());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetRow_count()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.row_count, other.row_count);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("CfSplit(");
- boolean first = true;
-
- sb.append("start_token:");
- if (this.start_token == null) {
- sb.append("null");
- } else {
- sb.append(this.start_token);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("end_token:");
- if (this.end_token == null) {
- sb.append("null");
- } else {
- sb.append(this.end_token);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("row_count:");
- sb.append(this.row_count);
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- if (start_token == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'start_token' was not present! Struct: " + toString());
- }
- if (end_token == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'end_token' was not present! Struct: " + toString());
- }
- // alas, we cannot check 'row_count' because it's a primitive and you chose the non-beans generator.
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
- __isset_bitfield = 0;
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class CfSplitStandardSchemeFactory implements SchemeFactory {
- public CfSplitStandardScheme getScheme() {
- return new CfSplitStandardScheme();
- }
- }
-
- private static class CfSplitStandardScheme extends StandardScheme<CfSplit> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, CfSplit struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // START_TOKEN
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.start_token = iprot.readString();
- struct.setStart_tokenIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 2: // END_TOKEN
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.end_token = iprot.readString();
- struct.setEnd_tokenIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 3: // ROW_COUNT
- if (schemeField.type == org.apache.thrift.protocol.TType.I64) {
- struct.row_count = iprot.readI64();
- struct.setRow_countIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- if (!struct.isSetRow_count()) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'row_count' was not found in serialized data! Struct: " + toString());
- }
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, CfSplit struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.start_token != null) {
- oprot.writeFieldBegin(START_TOKEN_FIELD_DESC);
- oprot.writeString(struct.start_token);
- oprot.writeFieldEnd();
- }
- if (struct.end_token != null) {
- oprot.writeFieldBegin(END_TOKEN_FIELD_DESC);
- oprot.writeString(struct.end_token);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldBegin(ROW_COUNT_FIELD_DESC);
- oprot.writeI64(struct.row_count);
- oprot.writeFieldEnd();
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class CfSplitTupleSchemeFactory implements SchemeFactory {
- public CfSplitTupleScheme getScheme() {
- return new CfSplitTupleScheme();
- }
- }
-
- private static class CfSplitTupleScheme extends TupleScheme<CfSplit> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, CfSplit struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- oprot.writeString(struct.start_token);
- oprot.writeString(struct.end_token);
- oprot.writeI64(struct.row_count);
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, CfSplit struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- struct.start_token = iprot.readString();
- struct.setStart_tokenIsSet(true);
- struct.end_token = iprot.readString();
- struct.setEnd_tokenIsSet(true);
- struct.row_count = iprot.readI64();
- struct.setRow_countIsSet(true);
- }
- }
-
-}
-
diff --git a/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java b/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java
deleted file mode 100644
index e24a4e4..0000000
--- a/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java
+++ /dev/null
@@ -1,743 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.1)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- * @generated
- */
-package org.apache.cassandra.thrift;
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
-
-import org.apache.commons.lang3.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.EncodingUtils;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-
-/**
- * Basic unit of data within a ColumnFamily.
- * @param name, the name by which this column is set and retrieved. Maximum 64KB long.
- * @param value. The data associated with the name. Maximum 2GB long, but in practice you should limit it to small numbers of MB (since Thrift must read the full value into memory to operate on it).
- * @param timestamp. The timestamp is used for conflict detection/resolution when two columns with same name need to be compared.
- * @param ttl. An optional, positive delay (in seconds) after which the column will be automatically deleted.
- */
-public class Column implements org.apache.thrift.TBase<Column, Column._Fields>, java.io.Serializable, Cloneable, Comparable<Column> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("Column");
-
- private static final org.apache.thrift.protocol.TField NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("name", org.apache.thrift.protocol.TType.STRING, (short)1);
- private static final org.apache.thrift.protocol.TField VALUE_FIELD_DESC = new org.apache.thrift.protocol.TField("value", org.apache.thrift.protocol.TType.STRING, (short)2);
- private static final org.apache.thrift.protocol.TField TIMESTAMP_FIELD_DESC = new org.apache.thrift.protocol.TField("timestamp", org.apache.thrift.protocol.TType.I64, (short)3);
- private static final org.apache.thrift.protocol.TField TTL_FIELD_DESC = new org.apache.thrift.protocol.TField("ttl", org.apache.thrift.protocol.TType.I32, (short)4);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new ColumnStandardSchemeFactory());
- schemes.put(TupleScheme.class, new ColumnTupleSchemeFactory());
- }
-
- public ByteBuffer name; // required
- public ByteBuffer value; // optional
- public long timestamp; // optional
- public int ttl; // optional
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- NAME((short)1, "name"),
- VALUE((short)2, "value"),
- TIMESTAMP((short)3, "timestamp"),
- TTL((short)4, "ttl");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // NAME
- return NAME;
- case 2: // VALUE
- return VALUE;
- case 3: // TIMESTAMP
- return TIMESTAMP;
- case 4: // TTL
- return TTL;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- private static final int __TIMESTAMP_ISSET_ID = 0;
- private static final int __TTL_ISSET_ID = 1;
- private byte __isset_bitfield = 0;
- private _Fields optionals[] = {_Fields.VALUE,_Fields.TIMESTAMP,_Fields.TTL};
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.NAME, new org.apache.thrift.meta_data.FieldMetaData("name", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true)));
- tmpMap.put(_Fields.VALUE, new org.apache.thrift.meta_data.FieldMetaData("value", org.apache.thrift.TFieldRequirementType.OPTIONAL,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true)));
- tmpMap.put(_Fields.TIMESTAMP, new org.apache.thrift.meta_data.FieldMetaData("timestamp", org.apache.thrift.TFieldRequirementType.OPTIONAL,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64)));
- tmpMap.put(_Fields.TTL, new org.apache.thrift.meta_data.FieldMetaData("ttl", org.apache.thrift.TFieldRequirementType.OPTIONAL,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(Column.class, metaDataMap);
- }
-
- public Column() {
- }
-
- public Column(
- ByteBuffer name)
- {
- this();
- this.name = name;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public Column(Column other) {
- __isset_bitfield = other.__isset_bitfield;
- if (other.isSetName()) {
- this.name = org.apache.thrift.TBaseHelper.copyBinary(other.name);
-;
- }
- if (other.isSetValue()) {
- this.value = org.apache.thrift.TBaseHelper.copyBinary(other.value);
-;
- }
- this.timestamp = other.timestamp;
- this.ttl = other.ttl;
- }
-
- public Column deepCopy() {
- return new Column(this);
- }
-
- @Override
- public void clear() {
- this.name = null;
- this.value = null;
- setTimestampIsSet(false);
- this.timestamp = 0;
- setTtlIsSet(false);
- this.ttl = 0;
- }
-
- public byte[] getName() {
- setName(org.apache.thrift.TBaseHelper.rightSize(name));
- return name == null ? null : name.array();
- }
-
- public ByteBuffer bufferForName() {
- return name;
- }
-
- public Column setName(byte[] name) {
- setName(name == null ? (ByteBuffer)null : ByteBuffer.wrap(name));
- return this;
- }
-
- public Column setName(ByteBuffer name) {
- this.name = name;
- return this;
- }
-
- public void unsetName() {
- this.name = null;
- }
-
- /** Returns true if field name is set (has been assigned a value) and false otherwise */
- public boolean isSetName() {
- return this.name != null;
- }
-
- public void setNameIsSet(boolean value) {
- if (!value) {
- this.name = null;
- }
- }
-
- public byte[] getValue() {
- setValue(org.apache.thrift.TBaseHelper.rightSize(value));
- return value == null ? null : value.array();
- }
-
- public ByteBuffer bufferForValue() {
- return value;
- }
-
- public Column setValue(byte[] value) {
- setValue(value == null ? (ByteBuffer)null : ByteBuffer.wrap(value));
- return this;
- }
-
- public Column setValue(ByteBuffer value) {
- this.value = value;
- return this;
- }
-
- public void unsetValue() {
- this.value = null;
- }
-
- /** Returns true if field value is set (has been assigned a value) and false otherwise */
- public boolean isSetValue() {
- return this.value != null;
- }
-
- public void setValueIsSet(boolean value) {
- if (!value) {
- this.value = null;
- }
- }
-
- public long getTimestamp() {
- return this.timestamp;
- }
-
- public Column setTimestamp(long timestamp) {
- this.timestamp = timestamp;
- setTimestampIsSet(true);
- return this;
- }
-
- public void unsetTimestamp() {
- __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __TIMESTAMP_ISSET_ID);
- }
-
- /** Returns true if field timestamp is set (has been assigned a value) and false otherwise */
- public boolean isSetTimestamp() {
- return EncodingUtils.testBit(__isset_bitfield, __TIMESTAMP_ISSET_ID);
- }
-
- public void setTimestampIsSet(boolean value) {
- __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __TIMESTAMP_ISSET_ID, value);
- }
-
- public int getTtl() {
- return this.ttl;
- }
-
- public Column setTtl(int ttl) {
- this.ttl = ttl;
- setTtlIsSet(true);
- return this;
- }
-
- public void unsetTtl() {
- __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __TTL_ISSET_ID);
- }
-
- /** Returns true if field ttl is set (has been assigned a value) and false otherwise */
- public boolean isSetTtl() {
- return EncodingUtils.testBit(__isset_bitfield, __TTL_ISSET_ID);
- }
-
- public void setTtlIsSet(boolean value) {
- __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __TTL_ISSET_ID, value);
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case NAME:
- if (value == null) {
- unsetName();
- } else {
- setName((ByteBuffer)value);
- }
- break;
-
- case VALUE:
- if (value == null) {
- unsetValue();
- } else {
- setValue((ByteBuffer)value);
- }
- break;
-
- case TIMESTAMP:
- if (value == null) {
- unsetTimestamp();
- } else {
- setTimestamp((Long)value);
- }
- break;
-
- case TTL:
- if (value == null) {
- unsetTtl();
- } else {
- setTtl((Integer)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case NAME:
- return getName();
-
- case VALUE:
- return getValue();
-
- case TIMESTAMP:
- return Long.valueOf(getTimestamp());
-
- case TTL:
- return Integer.valueOf(getTtl());
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case NAME:
- return isSetName();
- case VALUE:
- return isSetValue();
- case TIMESTAMP:
- return isSetTimestamp();
- case TTL:
- return isSetTtl();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof Column)
- return this.equals((Column)that);
- return false;
- }
-
- public boolean equals(Column that) {
- if (that == null)
- return false;
-
- boolean this_present_name = true && this.isSetName();
- boolean that_present_name = true && that.isSetName();
- if (this_present_name || that_present_name) {
- if (!(this_present_name && that_present_name))
- return false;
- if (!this.name.equals(that.name))
- return false;
- }
-
- boolean this_present_value = true && this.isSetValue();
- boolean that_present_value = true && that.isSetValue();
- if (this_present_value || that_present_value) {
- if (!(this_present_value && that_present_value))
- return false;
- if (!this.value.equals(that.value))
- return false;
- }
-
- boolean this_present_timestamp = true && this.isSetTimestamp();
- boolean that_present_timestamp = true && that.isSetTimestamp();
- if (this_present_timestamp || that_present_timestamp) {
- if (!(this_present_timestamp && that_present_timestamp))
- return false;
- if (this.timestamp != that.timestamp)
- return false;
- }
-
- boolean this_present_ttl = true && this.isSetTtl();
- boolean that_present_ttl = true && that.isSetTtl();
- if (this_present_ttl || that_present_ttl) {
- if (!(this_present_ttl && that_present_ttl))
- return false;
- if (this.ttl != that.ttl)
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_name = true && (isSetName());
- builder.append(present_name);
- if (present_name)
- builder.append(name);
-
- boolean present_value = true && (isSetValue());
- builder.append(present_value);
- if (present_value)
- builder.append(value);
-
- boolean present_timestamp = true && (isSetTimestamp());
- builder.append(present_timestamp);
- if (present_timestamp)
- builder.append(timestamp);
-
- boolean present_ttl = true && (isSetTtl());
- builder.append(present_ttl);
- if (present_ttl)
- builder.append(ttl);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(Column other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetName()).compareTo(other.isSetName());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetName()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.name, other.name);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetValue()).compareTo(other.isSetValue());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetValue()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.value, other.value);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetTimestamp()).compareTo(other.isSetTimestamp());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetTimestamp()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.timestamp, other.timestamp);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetTtl()).compareTo(other.isSetTtl());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetTtl()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.ttl, other.ttl);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("Column(");
- boolean first = true;
-
- sb.append("name:");
- if (this.name == null) {
- sb.append("null");
- } else {
- org.apache.thrift.TBaseHelper.toString(this.name, sb);
- }
- first = false;
- if (isSetValue()) {
- if (!first) sb.append(", ");
- sb.append("value:");
- if (this.value == null) {
- sb.append("null");
- } else {
- org.apache.thrift.TBaseHelper.toString(this.value, sb);
- }
- first = false;
- }
- if (isSetTimestamp()) {
- if (!first) sb.append(", ");
- sb.append("timestamp:");
- sb.append(this.timestamp);
- first = false;
- }
- if (isSetTtl()) {
- if (!first) sb.append(", ");
- sb.append("ttl:");
- sb.append(this.ttl);
- first = false;
- }
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- if (name == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'name' was not present! Struct: " + toString());
- }
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
- __isset_bitfield = 0;
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class ColumnStandardSchemeFactory implements SchemeFactory {
- public ColumnStandardScheme getScheme() {
- return new ColumnStandardScheme();
- }
- }
-
- private static class ColumnStandardScheme extends StandardScheme<Column> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, Column struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // NAME
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.name = iprot.readBinary();
- struct.setNameIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 2: // VALUE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.value = iprot.readBinary();
- struct.setValueIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 3: // TIMESTAMP
- if (schemeField.type == org.apache.thrift.protocol.TType.I64) {
- struct.timestamp = iprot.readI64();
- struct.setTimestampIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 4: // TTL
- if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
- struct.ttl = iprot.readI32();
- struct.setTtlIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, Column struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.name != null) {
- oprot.writeFieldBegin(NAME_FIELD_DESC);
- oprot.writeBinary(struct.name);
- oprot.writeFieldEnd();
- }
- if (struct.value != null) {
- if (struct.isSetValue()) {
- oprot.writeFieldBegin(VALUE_FIELD_DESC);
- oprot.writeBinary(struct.value);
- oprot.writeFieldEnd();
- }
- }
- if (struct.isSetTimestamp()) {
- oprot.writeFieldBegin(TIMESTAMP_FIELD_DESC);
- oprot.writeI64(struct.timestamp);
- oprot.writeFieldEnd();
- }
- if (struct.isSetTtl()) {
- oprot.writeFieldBegin(TTL_FIELD_DESC);
- oprot.writeI32(struct.ttl);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class ColumnTupleSchemeFactory implements SchemeFactory {
- public ColumnTupleScheme getScheme() {
- return new ColumnTupleScheme();
- }
- }
-
- private static class ColumnTupleScheme extends TupleScheme<Column> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, Column struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- oprot.writeBinary(struct.name);
- BitSet optionals = new BitSet();
- if (struct.isSetValue()) {
- optionals.set(0);
- }
- if (struct.isSetTimestamp()) {
- optionals.set(1);
- }
- if (struct.isSetTtl()) {
- optionals.set(2);
- }
- oprot.writeBitSet(optionals, 3);
- if (struct.isSetValue()) {
- oprot.writeBinary(struct.value);
- }
- if (struct.isSetTimestamp()) {
- oprot.writeI64(struct.timestamp);
- }
- if (struct.isSetTtl()) {
- oprot.writeI32(struct.ttl);
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, Column struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- struct.name = iprot.readBinary();
- struct.setNameIsSet(true);
- BitSet incoming = iprot.readBitSet(3);
- if (incoming.get(0)) {
- struct.value = iprot.readBinary();
- struct.setValueIsSet(true);
- }
- if (incoming.get(1)) {
- struct.timestamp = iprot.readI64();
- struct.setTimestampIsSet(true);
- }
- if (incoming.get(2)) {
- struct.ttl = iprot.readI32();
- struct.setTtlIsSet(true);
- }
- }
- }
-
-}
-
diff --git a/interface/thrift/gen-java/org/apache/cassandra/thrift/ColumnDef.java b/interface/thrift/gen-java/org/apache/cassandra/thrift/ColumnDef.java
deleted file mode 100644
index 409e4ac..0000000
--- a/interface/thrift/gen-java/org/apache/cassandra/thrift/ColumnDef.java
+++ /dev/null
@@ -1,903 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.1)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- * @generated
- */
-package org.apache.cassandra.thrift;
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
-
-import org.apache.commons.lang3.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-
-public class ColumnDef implements org.apache.thrift.TBase<ColumnDef, ColumnDef._Fields>, java.io.Serializable, Cloneable, Comparable<ColumnDef> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("ColumnDef");
-
- private static final org.apache.thrift.protocol.TField NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("name", org.apache.thrift.protocol.TType.STRING, (short)1);
- private static final org.apache.thrift.protocol.TField VALIDATION_CLASS_FIELD_DESC = new org.apache.thrift.protocol.TField("validation_class", org.apache.thrift.protocol.TType.STRING, (short)2);
- private static final org.apache.thrift.protocol.TField INDEX_TYPE_FIELD_DESC = new org.apache.thrift.protocol.TField("index_type", org.apache.thrift.protocol.TType.I32, (short)3);
- private static final org.apache.thrift.protocol.TField INDEX_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("index_name", org.apache.thrift.protocol.TType.STRING, (short)4);
- private static final org.apache.thrift.protocol.TField INDEX_OPTIONS_FIELD_DESC = new org.apache.thrift.protocol.TField("index_options", org.apache.thrift.protocol.TType.MAP, (short)5);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new ColumnDefStandardSchemeFactory());
- schemes.put(TupleScheme.class, new ColumnDefTupleSchemeFactory());
- }
-
- public ByteBuffer name; // required
- public String validation_class; // required
- /**
- *
- * @see IndexType
- */
- public IndexType index_type; // optional
- public String index_name; // optional
- public Map<String,String> index_options; // optional
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- NAME((short)1, "name"),
- VALIDATION_CLASS((short)2, "validation_class"),
- /**
- *
- * @see IndexType
- */
- INDEX_TYPE((short)3, "index_type"),
- INDEX_NAME((short)4, "index_name"),
- INDEX_OPTIONS((short)5, "index_options");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // NAME
- return NAME;
- case 2: // VALIDATION_CLASS
- return VALIDATION_CLASS;
- case 3: // INDEX_TYPE
- return INDEX_TYPE;
- case 4: // INDEX_NAME
- return INDEX_NAME;
- case 5: // INDEX_OPTIONS
- return INDEX_OPTIONS;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- private _Fields optionals[] = {_Fields.INDEX_TYPE,_Fields.INDEX_NAME,_Fields.INDEX_OPTIONS};
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.NAME, new org.apache.thrift.meta_data.FieldMetaData("name", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true)));
- tmpMap.put(_Fields.VALIDATION_CLASS, new org.apache.thrift.meta_data.FieldMetaData("validation_class", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
- tmpMap.put(_Fields.INDEX_TYPE, new org.apache.thrift.meta_data.FieldMetaData("index_type", org.apache.thrift.TFieldRequirementType.OPTIONAL,
- new org.apache.thrift.meta_data.EnumMetaData(org.apache.thrift.protocol.TType.ENUM, IndexType.class)));
- tmpMap.put(_Fields.INDEX_NAME, new org.apache.thrift.meta_data.FieldMetaData("index_name", org.apache.thrift.TFieldRequirementType.OPTIONAL,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
- tmpMap.put(_Fields.INDEX_OPTIONS, new org.apache.thrift.meta_data.FieldMetaData("index_options", org.apache.thrift.TFieldRequirementType.OPTIONAL,
- new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING),
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(ColumnDef.class, metaDataMap);
- }
-
- public ColumnDef() {
- }
-
- public ColumnDef(
- ByteBuffer name,
- String validation_class)
- {
- this();
- this.name = name;
- this.validation_class = validation_class;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public ColumnDef(ColumnDef other) {
- if (other.isSetName()) {
- this.name = org.apache.thrift.TBaseHelper.copyBinary(other.name);
-;
- }
- if (other.isSetValidation_class()) {
- this.validation_class = other.validation_class;
- }
- if (other.isSetIndex_type()) {
- this.index_type = other.index_type;
- }
- if (other.isSetIndex_name()) {
- this.index_name = other.index_name;
- }
- if (other.isSetIndex_options()) {
- Map<String,String> __this__index_options = new HashMap<String,String>(other.index_options);
- this.index_options = __this__index_options;
- }
- }
-
- public ColumnDef deepCopy() {
- return new ColumnDef(this);
- }
-
- @Override
- public void clear() {
- this.name = null;
- this.validation_class = null;
- this.index_type = null;
- this.index_name = null;
- this.index_options = null;
- }
-
- public byte[] getName() {
- setName(org.apache.thrift.TBaseHelper.rightSize(name));
- return name == null ? null : name.array();
- }
-
- public ByteBuffer bufferForName() {
- return name;
- }
-
- public ColumnDef setName(byte[] name) {
- setName(name == null ? (ByteBuffer)null : ByteBuffer.wrap(name));
- return this;
- }
-
- public ColumnDef setName(ByteBuffer name) {
- this.name = name;
- return this;
- }
-
- public void unsetName() {
- this.name = null;
- }
-
- /** Returns true if field name is set (has been assigned a value) and false otherwise */
- public boolean isSetName() {
- return this.name != null;
- }
-
- public void setNameIsSet(boolean value) {
- if (!value) {
- this.name = null;
- }
- }
-
- public String getValidation_class() {
- return this.validation_class;
- }
-
- public ColumnDef setValidation_class(String validation_class) {
- this.validation_class = validation_class;
- return this;
- }
-
- public void unsetValidation_class() {
- this.validation_class = null;
- }
-
- /** Returns true if field validation_class is set (has been assigned a value) and false otherwise */
- public boolean isSetValidation_class() {
- return this.validation_class != null;
- }
-
- public void setValidation_classIsSet(boolean value) {
- if (!value) {
- this.validation_class = null;
- }
- }
-
- /**
- *
- * @see IndexType
- */
- public IndexType getIndex_type() {
- return this.index_type;
- }
-
- /**
- *
- * @see IndexType
- */
- public ColumnDef setIndex_type(IndexType index_type) {
- this.index_type = index_type;
- return this;
- }
-
- public void unsetIndex_type() {
- this.index_type = null;
- }
-
- /** Returns true if field index_type is set (has been assigned a value) and false otherwise */
- public boolean isSetIndex_type() {
- return this.index_type != null;
- }
-
- public void setIndex_typeIsSet(boolean value) {
- if (!value) {
- this.index_type = null;
- }
- }
-
- public String getIndex_name() {
- return this.index_name;
- }
-
- public ColumnDef setIndex_name(String index_name) {
- this.index_name = index_name;
- return this;
- }
-
- public void unsetIndex_name() {
- this.index_name = null;
- }
-
- /** Returns true if field index_name is set (has been assigned a value) and false otherwise */
- public boolean isSetIndex_name() {
- return this.index_name != null;
- }
-
- public void setIndex_nameIsSet(boolean value) {
- if (!value) {
- this.index_name = null;
- }
- }
-
- public int getIndex_optionsSize() {
- return (this.index_options == null) ? 0 : this.index_options.size();
- }
-
- public void putToIndex_options(String key, String val) {
- if (this.index_options == null) {
- this.index_options = new HashMap<String,String>();
- }
- this.index_options.put(key, val);
- }
-
- public Map<String,String> getIndex_options() {
- return this.index_options;
- }
-
- public ColumnDef setIndex_options(Map<String,String> index_options) {
- this.index_options = index_options;
- return this;
- }
-
- public void unsetIndex_options() {
- this.index_options = null;
- }
-
- /** Returns true if field index_options is set (has been assigned a value) and false otherwise */
- public boolean isSetIndex_options() {
- return this.index_options != null;
- }
-
- public void setIndex_optionsIsSet(boolean value) {
- if (!value) {
- this.index_options = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case NAME:
- if (value == null) {
- unsetName();
- } else {
- setName((ByteBuffer)value);
- }
- break;
-
- case VALIDATION_CLASS:
- if (value == null) {
- unsetValidation_class();
- } else {
- setValidation_class((String)value);
- }
- break;
-
- case INDEX_TYPE:
- if (value == null) {
- unsetIndex_type();
- } else {
- setIndex_type((IndexType)value);
- }
- break;
-
- case INDEX_NAME:
- if (value == null) {
- unsetIndex_name();
- } else {
- setIndex_name((String)value);
- }
- break;
-
- case INDEX_OPTIONS:
- if (value == null) {
- unsetIndex_options();
- } else {
- setIndex_options((Map<String,String>)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case NAME:
- return getName();
-
- case VALIDATION_CLASS:
- return getValidation_class();
-
- case INDEX_TYPE:
- return getIndex_type();
-
- case INDEX_NAME:
- return getIndex_name();
-
- case INDEX_OPTIONS:
- return getIndex_options();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case NAME:
- return isSetName();
- case VALIDATION_CLASS:
- return isSetValidation_class();
- case INDEX_TYPE:
- return isSetIndex_type();
- case INDEX_NAME:
- return isSetIndex_name();
- case INDEX_OPTIONS:
- return isSetIndex_options();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof ColumnDef)
- return this.equals((ColumnDef)that);
- return false;
- }
-
- public boolean equals(ColumnDef that) {
- if (that == null)
- return false;
-
- boolean this_present_name = true && this.isSetName();
- boolean that_present_name = true && that.isSetName();
- if (this_present_name || that_present_name) {
- if (!(this_present_name && that_present_name))
- return false;
- if (!this.name.equals(that.name))
- return false;
- }
-
- boolean this_present_validation_class = true && this.isSetValidation_class();
- boolean that_present_validation_class = true && that.isSetValidation_class();
- if (this_present_validation_class || that_present_validation_class) {
- if (!(this_present_validation_class && that_present_validation_class))
- return false;
- if (!this.validation_class.equals(that.validation_class))
- return false;
- }
-
- boolean this_present_index_type = true && this.isSetIndex_type();
- boolean that_present_index_type = true && that.isSetIndex_type();
- if (this_present_index_type || that_present_index_type) {
- if (!(this_present_index_type && that_present_index_type))
- return false;
- if (!this.index_type.equals(that.index_type))
- return false;
- }
-
- boolean this_present_index_name = true && this.isSetIndex_name();
- boolean that_present_index_name = true && that.isSetIndex_name();
- if (this_present_index_name || that_present_index_name) {
- if (!(this_present_index_name && that_present_index_name))
- return false;
- if (!this.index_name.equals(that.index_name))
- return false;
- }
-
- boolean this_present_index_options = true && this.isSetIndex_options();
- boolean that_present_index_options = true && that.isSetIndex_options();
- if (this_present_index_options || that_present_index_options) {
- if (!(this_present_index_options && that_present_index_options))
- return false;
- if (!this.index_options.equals(that.index_options))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_name = true && (isSetName());
- builder.append(present_name);
- if (present_name)
- builder.append(name);
-
- boolean present_validation_class = true && (isSetValidation_class());
- builder.append(present_validation_class);
- if (present_validation_class)
- builder.append(validation_class);
-
- boolean present_index_type = true && (isSetIndex_type());
- builder.append(present_index_type);
- if (present_index_type)
- builder.append(index_type.getValue());
-
- boolean present_index_name = true && (isSetIndex_name());
- builder.append(present_index_name);
- if (present_index_name)
- builder.append(index_name);
-
- boolean present_index_options = true && (isSetIndex_options());
- builder.append(present_index_options);
- if (present_index_options)
- builder.append(index_options);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(ColumnDef other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetName()).compareTo(other.isSetName());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetName()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.name, other.name);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetValidation_class()).compareTo(other.isSetValidation_class());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetValidation_class()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.validation_class, other.validation_class);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetIndex_type()).compareTo(other.isSetIndex_type());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetIndex_type()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.index_type, other.index_type);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetIndex_name()).compareTo(other.isSetIndex_name());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetIndex_name()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.index_name, other.index_name);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetIndex_options()).compareTo(other.isSetIndex_options());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetIndex_options()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.index_options, other.index_options);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("ColumnDef(");
- boolean first = true;
-
- sb.append("name:");
- if (this.name == null) {
- sb.append("null");
- } else {
- org.apache.thrift.TBaseHelper.toString(this.name, sb);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("validation_class:");
- if (this.validation_class == null) {
- sb.append("null");
- } else {
- sb.append(this.validation_class);
- }
- first = false;
- if (isSetIndex_type()) {
- if (!first) sb.append(", ");
- sb.append("index_type:");
- if (this.index_type == null) {
- sb.append("null");
- } else {
- sb.append(this.index_type);
- }
- first = false;
- }
- if (isSetIndex_name()) {
- if (!first) sb.append(", ");
- sb.append("index_name:");
- if (this.index_name == null) {
- sb.append("null");
- } else {
- sb.append(this.index_name);
- }
- first = false;
- }
- if (isSetIndex_options()) {
- if (!first) sb.append(", ");
- sb.append("index_options:");
- if (this.index_options == null) {
- sb.append("null");
- } else {
- sb.append(this.index_options);
- }
- first = false;
- }
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- if (name == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'name' was not present! Struct: " + toString());
- }
- if (validation_class == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'validation_class' was not present! Struct: " + toString());
- }
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class ColumnDefStandardSchemeFactory implements SchemeFactory {
- public ColumnDefStandardScheme getScheme() {
- return new ColumnDefStandardScheme();
- }
- }
-
- private static class ColumnDefStandardScheme extends StandardScheme<ColumnDef> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, ColumnDef struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // NAME
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.name = iprot.readBinary();
- struct.setNameIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 2: // VALIDATION_CLASS
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.validation_class = iprot.readString();
- struct.setValidation_classIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 3: // INDEX_TYPE
- if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
- struct.index_type = IndexType.findByValue(iprot.readI32());
- struct.setIndex_typeIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 4: // INDEX_NAME
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.index_name = iprot.readString();
- struct.setIndex_nameIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 5: // INDEX_OPTIONS
- if (schemeField.type == org.apache.thrift.protocol.TType.MAP) {
- {
- org.apache.thrift.protocol.TMap _map90 = iprot.readMapBegin();
- struct.index_options = new HashMap<String,String>(2*_map90.size);
- for (int _i91 = 0; _i91 < _map90.size; ++_i91)
- {
- String _key92;
- String _val93;
- _key92 = iprot.readString();
- _val93 = iprot.readString();
- struct.index_options.put(_key92, _val93);
- }
- iprot.readMapEnd();
- }
- struct.setIndex_optionsIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, ColumnDef struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.name != null) {
- oprot.writeFieldBegin(NAME_FIELD_DESC);
- oprot.writeBinary(struct.name);
- oprot.writeFieldEnd();
- }
- if (struct.validation_class != null) {
- oprot.writeFieldBegin(VALIDATION_CLASS_FIELD_DESC);
- oprot.writeString(struct.validation_class);
- oprot.writeFieldEnd();
- }
- if (struct.index_type != null) {
- if (struct.isSetIndex_type()) {
- oprot.writeFieldBegin(INDEX_TYPE_FIELD_DESC);
- oprot.writeI32(struct.index_type.getValue());
- oprot.writeFieldEnd();
- }
- }
- if (struct.index_name != null) {
- if (struct.isSetIndex_name()) {
- oprot.writeFieldBegin(INDEX_NAME_FIELD_DESC);
- oprot.writeString(struct.index_name);
- oprot.writeFieldEnd();
- }
- }
- if (struct.index_options != null) {
- if (struct.isSetIndex_options()) {
- oprot.writeFieldBegin(INDEX_OPTIONS_FIELD_DESC);
- {
- oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, struct.index_options.size()));
- for (Map.Entry<String, String> _iter94 : struct.index_options.entrySet())
- {
- oprot.writeString(_iter94.getKey());
- oprot.writeString(_iter94.getValue());
- }
- oprot.writeMapEnd();
- }
- oprot.writeFieldEnd();
- }
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class ColumnDefTupleSchemeFactory implements SchemeFactory {
- public ColumnDefTupleScheme getScheme() {
- return new ColumnDefTupleScheme();
- }
- }
-
- private static class ColumnDefTupleScheme extends TupleScheme<ColumnDef> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, ColumnDef struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- oprot.writeBinary(struct.name);
- oprot.writeString(struct.validation_class);
- BitSet optionals = new BitSet();
- if (struct.isSetIndex_type()) {
- optionals.set(0);
- }
- if (struct.isSetIndex_name()) {
- optionals.set(1);
- }
- if (struct.isSetIndex_options()) {
- optionals.set(2);
- }
- oprot.writeBitSet(optionals, 3);
- if (struct.isSetIndex_type()) {
- oprot.writeI32(struct.index_type.getValue());
- }
- if (struct.isSetIndex_name()) {
- oprot.writeString(struct.index_name);
- }
- if (struct.isSetIndex_options()) {
- {
- oprot.writeI32(struct.index_options.size());
- for (Map.Entry<String, String> _iter95 : struct.index_options.entrySet())
- {
- oprot.writeString(_iter95.getKey());
- oprot.writeString(_iter95.getValue());
- }
- }
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, ColumnDef struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- struct.name = iprot.readBinary();
- struct.setNameIsSet(true);
- struct.validation_class = iprot.readString();
- struct.setValidation_classIsSet(true);
- BitSet incoming = iprot.readBitSet(3);
- if (incoming.get(0)) {
- struct.index_type = IndexType.findByValue(iprot.readI32());
- struct.setIndex_typeIsSet(true);
- }
- if (incoming.get(1)) {
- struct.index_name = iprot.readString();
- struct.setIndex_nameIsSet(true);
- }
- if (incoming.get(2)) {
- {
- org.apache.thrift.protocol.TMap _map96 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32());
- struct.index_options = new HashMap<String,String>(2*_map96.size);
- for (int _i97 = 0; _i97 < _map96.size; ++_i97)
- {
- String _key98;
- String _val99;
- _key98 = iprot.readString();
- _val99 = iprot.readString();
- struct.index_options.put(_key98, _val99);
- }
- }
- struct.setIndex_optionsIsSet(true);
- }
- }
- }
-
-}
-
diff --git a/interface/thrift/gen-java/org/apache/cassandra/thrift/ColumnOrSuperColumn.java b/interface/thrift/gen-java/org/apache/cassandra/thrift/ColumnOrSuperColumn.java
deleted file mode 100644
index 261d93f..0000000
--- a/interface/thrift/gen-java/org/apache/cassandra/thrift/ColumnOrSuperColumn.java
+++ /dev/null
@@ -1,758 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.1)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- * @generated
- */
-package org.apache.cassandra.thrift;
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
-
-import org.apache.commons.lang3.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-
-/**
- * Methods for fetching rows/records from Cassandra will return either a single instance of ColumnOrSuperColumn or a list
- * of ColumnOrSuperColumns (get_slice()). If you're looking up a SuperColumn (or list of SuperColumns) then the resulting
- * instances of ColumnOrSuperColumn will have the requested SuperColumn in the attribute super_column. For queries resulting
- * in Columns, those values will be in the attribute column. This change was made between 0.3 and 0.4 to standardize on
- * single query methods that may return either a SuperColumn or Column.
- *
- * If the query was on a counter column family, you will either get a counter_column (instead of a column) or a
- * counter_super_column (instead of a super_column)
- *
- * @param column. The Column returned by get() or get_slice().
- * @param super_column. The SuperColumn returned by get() or get_slice().
- * @param counter_column. The Counterolumn returned by get() or get_slice().
- * @param counter_super_column. The CounterSuperColumn returned by get() or get_slice().
- */
-public class ColumnOrSuperColumn implements org.apache.thrift.TBase<ColumnOrSuperColumn, ColumnOrSuperColumn._Fields>, java.io.Serializable, Cloneable, Comparable<ColumnOrSuperColumn> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("ColumnOrSuperColumn");
-
- private static final org.apache.thrift.protocol.TField COLUMN_FIELD_DESC = new org.apache.thrift.protocol.TField("column", org.apache.thrift.protocol.TType.STRUCT, (short)1);
- private static final org.apache.thrift.protocol.TField SUPER_COLUMN_FIELD_DESC = new org.apache.thrift.protocol.TField("super_column", org.apache.thrift.protocol.TType.STRUCT, (short)2);
- private static final org.apache.thrift.protocol.TField COUNTER_COLUMN_FIELD_DESC = new org.apache.thrift.protocol.TField("counter_column", org.apache.thrift.protocol.TType.STRUCT, (short)3);
- private static final org.apache.thrift.protocol.TField COUNTER_SUPER_COLUMN_FIELD_DESC = new org.apache.thrift.protocol.TField("counter_super_column", org.apache.thrift.protocol.TType.STRUCT, (short)4);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new ColumnOrSuperColumnStandardSchemeFactory());
- schemes.put(TupleScheme.class, new ColumnOrSuperColumnTupleSchemeFactory());
- }
-
- public Column column; // optional
- public SuperColumn super_column; // optional
- public CounterColumn counter_column; // optional
- public CounterSuperColumn counter_super_column; // optional
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- COLUMN((short)1, "column"),
- SUPER_COLUMN((short)2, "super_column"),
- COUNTER_COLUMN((short)3, "counter_column"),
- COUNTER_SUPER_COLUMN((short)4, "counter_super_column");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // COLUMN
- return COLUMN;
- case 2: // SUPER_COLUMN
- return SUPER_COLUMN;
- case 3: // COUNTER_COLUMN
- return COUNTER_COLUMN;
- case 4: // COUNTER_SUPER_COLUMN
- return COUNTER_SUPER_COLUMN;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- private _Fields optionals[] = {_Fields.COLUMN,_Fields.SUPER_COLUMN,_Fields.COUNTER_COLUMN,_Fields.COUNTER_SUPER_COLUMN};
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.COLUMN, new org.apache.thrift.meta_data.FieldMetaData("column", org.apache.thrift.TFieldRequirementType.OPTIONAL,
- new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, Column.class)));
- tmpMap.put(_Fields.SUPER_COLUMN, new org.apache.thrift.meta_data.FieldMetaData("super_column", org.apache.thrift.TFieldRequirementType.OPTIONAL,
- new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, SuperColumn.class)));
- tmpMap.put(_Fields.COUNTER_COLUMN, new org.apache.thrift.meta_data.FieldMetaData("counter_column", org.apache.thrift.TFieldRequirementType.OPTIONAL,
- new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, CounterColumn.class)));
- tmpMap.put(_Fields.COUNTER_SUPER_COLUMN, new org.apache.thrift.meta_data.FieldMetaData("counter_super_column", org.apache.thrift.TFieldRequirementType.OPTIONAL,
- new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, CounterSuperColumn.class)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(ColumnOrSuperColumn.class, metaDataMap);
- }
-
- public ColumnOrSuperColumn() {
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public ColumnOrSuperColumn(ColumnOrSuperColumn other) {
- if (other.isSetColumn()) {
- this.column = new Column(other.column);
- }
- if (other.isSetSuper_column()) {
- this.super_column = new SuperColumn(other.super_column);
- }
- if (other.isSetCounter_column()) {
- this.counter_column = new CounterColumn(other.counter_column);
- }
- if (other.isSetCounter_super_column()) {
- this.counter_super_column = new CounterSuperColumn(other.counter_super_column);
- }
- }
-
- public ColumnOrSuperColumn deepCopy() {
- return new ColumnOrSuperColumn(this);
- }
-
- @Override
- public void clear() {
- this.column = null;
- this.super_column = null;
- this.counter_column = null;
- this.counter_super_column = null;
- }
-
- public Column getColumn() {
- return this.column;
- }
-
- public ColumnOrSuperColumn setColumn(Column column) {
- this.column = column;
- return this;
- }
-
- public void unsetColumn() {
- this.column = null;
- }
-
- /** Returns true if field column is set (has been assigned a value) and false otherwise */
- public boolean isSetColumn() {
- return this.column != null;
- }
-
- public void setColumnIsSet(boolean value) {
- if (!value) {
- this.column = null;
- }
- }
-
- public SuperColumn getSuper_column() {
- return this.super_column;
- }
-
- public ColumnOrSuperColumn setSuper_column(SuperColumn super_column) {
- this.super_column = super_column;
- return this;
- }
-
- public void unsetSuper_column() {
- this.super_column = null;
- }
-
- /** Returns true if field super_column is set (has been assigned a value) and false otherwise */
- public boolean isSetSuper_column() {
- return this.super_column != null;
- }
-
- public void setSuper_columnIsSet(boolean value) {
- if (!value) {
- this.super_column = null;
- }
- }
-
- public CounterColumn getCounter_column() {
- return this.counter_column;
- }
-
- public ColumnOrSuperColumn setCounter_column(CounterColumn counter_column) {
- this.counter_column = counter_column;
- return this;
- }
-
- public void unsetCounter_column() {
- this.counter_column = null;
- }
-
- /** Returns true if field counter_column is set (has been assigned a value) and false otherwise */
- public boolean isSetCounter_column() {
- return this.counter_column != null;
- }
-
- public void setCounter_columnIsSet(boolean value) {
- if (!value) {
- this.counter_column = null;
- }
- }
-
- public CounterSuperColumn getCounter_super_column() {
- return this.counter_super_column;
- }
-
- public ColumnOrSuperColumn setCounter_super_column(CounterSuperColumn counter_super_column) {
- this.counter_super_column = counter_super_column;
- return this;
- }
-
- public void unsetCounter_super_column() {
- this.counter_super_column = null;
- }
-
- /** Returns true if field counter_super_column is set (has been assigned a value) and false otherwise */
- public boolean isSetCounter_super_column() {
- return this.counter_super_column != null;
- }
-
- public void setCounter_super_columnIsSet(boolean value) {
- if (!value) {
- this.counter_super_column = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case COLUMN:
- if (value == null) {
- unsetColumn();
- } else {
- setColumn((Column)value);
- }
- break;
-
- case SUPER_COLUMN:
- if (value == null) {
- unsetSuper_column();
- } else {
- setSuper_column((SuperColumn)value);
- }
- break;
-
- case COUNTER_COLUMN:
- if (value == null) {
- unsetCounter_column();
- } else {
- setCounter_column((CounterColumn)value);
- }
- break;
-
- case COUNTER_SUPER_COLUMN:
- if (value == null) {
- unsetCounter_super_column();
- } else {
- setCounter_super_column((CounterSuperColumn)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case COLUMN:
- return getColumn();
-
- case SUPER_COLUMN:
- return getSuper_column();
-
- case COUNTER_COLUMN:
- return getCounter_column();
-
- case COUNTER_SUPER_COLUMN:
- return getCounter_super_column();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case COLUMN:
- return isSetColumn();
- case SUPER_COLUMN:
- return isSetSuper_column();
- case COUNTER_COLUMN:
- return isSetCounter_column();
- case COUNTER_SUPER_COLUMN:
- return isSetCounter_super_column();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof ColumnOrSuperColumn)
- return this.equals((ColumnOrSuperColumn)that);
- return false;
- }
-
- public boolean equals(ColumnOrSuperColumn that) {
- if (that == null)
- return false;
-
- boolean this_present_column = true && this.isSetColumn();
- boolean that_present_column = true && that.isSetColumn();
- if (this_present_column || that_present_column) {
- if (!(this_present_column && that_present_column))
- return false;
- if (!this.column.equals(that.column))
- return false;
- }
-
- boolean this_present_super_column = true && this.isSetSuper_column();
- boolean that_present_super_column = true && that.isSetSuper_column();
- if (this_present_super_column || that_present_super_column) {
- if (!(this_present_super_column && that_present_super_column))
- return false;
- if (!this.super_column.equals(that.super_column))
- return false;
- }
-
- boolean this_present_counter_column = true && this.isSetCounter_column();
- boolean that_present_counter_column = true && that.isSetCounter_column();
- if (this_present_counter_column || that_present_counter_column) {
- if (!(this_present_counter_column && that_present_counter_column))
- return false;
- if (!this.counter_column.equals(that.counter_column))
- return false;
- }
-
- boolean this_present_counter_super_column = true && this.isSetCounter_super_column();
- boolean that_present_counter_super_column = true && that.isSetCounter_super_column();
- if (this_present_counter_super_column || that_present_counter_super_column) {
- if (!(this_present_counter_super_column && that_present_counter_super_column))
- return false;
- if (!this.counter_super_column.equals(that.counter_super_column))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_column = true && (isSetColumn());
- builder.append(present_column);
- if (present_column)
- builder.append(column);
-
- boolean present_super_column = true && (isSetSuper_column());
- builder.append(present_super_column);
- if (present_super_column)
- builder.append(super_column);
-
- boolean present_counter_column = true && (isSetCounter_column());
- builder.append(present_counter_column);
- if (present_counter_column)
- builder.append(counter_column);
-
- boolean present_counter_super_column = true && (isSetCounter_super_column());
- builder.append(present_counter_super_column);
- if (present_counter_super_column)
- builder.append(counter_super_column);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(ColumnOrSuperColumn other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetColumn()).compareTo(other.isSetColumn());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetColumn()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.column, other.column);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetSuper_column()).compareTo(other.isSetSuper_column());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetSuper_column()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.super_column, other.super_column);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetCounter_column()).compareTo(other.isSetCounter_column());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetCounter_column()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.counter_column, other.counter_column);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetCounter_super_column()).compareTo(other.isSetCounter_super_column());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetCounter_super_column()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.counter_super_column, other.counter_super_column);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("ColumnOrSuperColumn(");
- boolean first = true;
-
- if (isSetColumn()) {
- sb.append("column:");
- if (this.column == null) {
- sb.append("null");
- } else {
- sb.append(this.column);
- }
- first = false;
- }
- if (isSetSuper_column()) {
- if (!first) sb.append(", ");
- sb.append("super_column:");
- if (this.super_column == null) {
- sb.append("null");
- } else {
- sb.append(this.super_column);
- }
- first = false;
- }
- if (isSetCounter_column()) {
- if (!first) sb.append(", ");
- sb.append("counter_column:");
- if (this.counter_column == null) {
- sb.append("null");
- } else {
- sb.append(this.counter_column);
- }
- first = false;
- }
- if (isSetCounter_super_column()) {
- if (!first) sb.append(", ");
- sb.append("counter_super_column:");
- if (this.counter_super_column == null) {
- sb.append("null");
- } else {
- sb.append(this.counter_super_column);
- }
- first = false;
- }
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // check for sub-struct validity
- if (column != null) {
- column.validate();
- }
- if (super_column != null) {
- super_column.validate();
- }
- if (counter_column != null) {
- counter_column.validate();
- }
- if (counter_super_column != null) {
- counter_super_column.validate();
- }
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class ColumnOrSuperColumnStandardSchemeFactory implements SchemeFactory {
- public ColumnOrSuperColumnStandardScheme getScheme() {
- return new ColumnOrSuperColumnStandardScheme();
- }
- }
-
- private static class ColumnOrSuperColumnStandardScheme extends StandardScheme<ColumnOrSuperColumn> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, ColumnOrSuperColumn struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // COLUMN
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.column = new Column();
- struct.column.read(iprot);
- struct.setColumnIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 2: // SUPER_COLUMN
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.super_column = new SuperColumn();
- struct.super_column.read(iprot);
- struct.setSuper_columnIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 3: // COUNTER_COLUMN
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.counter_column = new CounterColumn();
- struct.counter_column.read(iprot);
- struct.setCounter_columnIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 4: // COUNTER_SUPER_COLUMN
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.counter_super_column = new CounterSuperColumn();
- struct.counter_super_column.read(iprot);
- struct.setCounter_super_columnIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, ColumnOrSuperColumn struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.column != null) {
- if (struct.isSetColumn()) {
- oprot.writeFieldBegin(COLUMN_FIELD_DESC);
- struct.column.write(oprot);
- oprot.writeFieldEnd();
- }
- }
- if (struct.super_column != null) {
- if (struct.isSetSuper_column()) {
- oprot.writeFieldBegin(SUPER_COLUMN_FIELD_DESC);
- struct.super_column.write(oprot);
- oprot.writeFieldEnd();
- }
- }
- if (struct.counter_column != null) {
- if (struct.isSetCounter_column()) {
- oprot.writeFieldBegin(COUNTER_COLUMN_FIELD_DESC);
- struct.counter_column.write(oprot);
- oprot.writeFieldEnd();
- }
- }
- if (struct.counter_super_column != null) {
- if (struct.isSetCounter_super_column()) {
- oprot.writeFieldBegin(COUNTER_SUPER_COLUMN_FIELD_DESC);
- struct.counter_super_column.write(oprot);
- oprot.writeFieldEnd();
- }
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class ColumnOrSuperColumnTupleSchemeFactory implements SchemeFactory {
- public ColumnOrSuperColumnTupleScheme getScheme() {
- return new ColumnOrSuperColumnTupleScheme();
- }
- }
-
- private static class ColumnOrSuperColumnTupleScheme extends TupleScheme<ColumnOrSuperColumn> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, ColumnOrSuperColumn struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- BitSet optionals = new BitSet();
- if (struct.isSetColumn()) {
- optionals.set(0);
- }
- if (struct.isSetSuper_column()) {
- optionals.set(1);
- }
- if (struct.isSetCounter_column()) {
- optionals.set(2);
- }
- if (struct.isSetCounter_super_column()) {
- optionals.set(3);
- }
- oprot.writeBitSet(optionals, 4);
- if (struct.isSetColumn()) {
- struct.column.write(oprot);
- }
- if (struct.isSetSuper_column()) {
- struct.super_column.write(oprot);
- }
- if (struct.isSetCounter_column()) {
- struct.counter_column.write(oprot);
- }
- if (struct.isSetCounter_super_column()) {
- struct.counter_super_column.write(oprot);
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, ColumnOrSuperColumn struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- BitSet incoming = iprot.readBitSet(4);
- if (incoming.get(0)) {
- struct.column = new Column();
- struct.column.read(iprot);
- struct.setColumnIsSet(true);
- }
- if (incoming.get(1)) {
- struct.super_column = new SuperColumn();
- struct.super_column.read(iprot);
- struct.setSuper_columnIsSet(true);
- }
- if (incoming.get(2)) {
- struct.counter_column = new CounterColumn();
- struct.counter_column.read(iprot);
- struct.setCounter_columnIsSet(true);
- }
- if (incoming.get(3)) {
- struct.counter_super_column = new CounterSuperColumn();
- struct.counter_super_column.read(iprot);
- struct.setCounter_super_columnIsSet(true);
- }
- }
- }
-
-}
-
diff --git a/interface/thrift/gen-java/org/apache/cassandra/thrift/ColumnParent.java b/interface/thrift/gen-java/org/apache/cassandra/thrift/ColumnParent.java
deleted file mode 100644
index 73aff66..0000000
--- a/interface/thrift/gen-java/org/apache/cassandra/thrift/ColumnParent.java
+++ /dev/null
@@ -1,538 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.1)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- * @generated
- */
-package org.apache.cassandra.thrift;
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
-
-import org.apache.commons.lang3.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * ColumnParent is used when selecting groups of columns from the same ColumnFamily. In directory structure terms, imagine
- * ColumnParent as ColumnPath + '/../'.
- *
- * See also <a href="cassandra.html#Struct_ColumnPath">ColumnPath</a>
- */
-public class ColumnParent implements org.apache.thrift.TBase<ColumnParent, ColumnParent._Fields>, java.io.Serializable, Cloneable, Comparable<ColumnParent> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("ColumnParent");
-
- private static final org.apache.thrift.protocol.TField COLUMN_FAMILY_FIELD_DESC = new org.apache.thrift.protocol.TField("column_family", org.apache.thrift.protocol.TType.STRING, (short)3);
- private static final org.apache.thrift.protocol.TField SUPER_COLUMN_FIELD_DESC = new org.apache.thrift.protocol.TField("super_column", org.apache.thrift.protocol.TType.STRING, (short)4);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new ColumnParentStandardSchemeFactory());
- schemes.put(TupleScheme.class, new ColumnParentTupleSchemeFactory());
- }
-
- public String column_family; // required
- public ByteBuffer super_column; // optional
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- COLUMN_FAMILY((short)3, "column_family"),
- SUPER_COLUMN((short)4, "super_column");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 3: // COLUMN_FAMILY
- return COLUMN_FAMILY;
- case 4: // SUPER_COLUMN
- return SUPER_COLUMN;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- private _Fields optionals[] = {_Fields.SUPER_COLUMN};
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.COLUMN_FAMILY, new org.apache.thrift.meta_data.FieldMetaData("column_family", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
- tmpMap.put(_Fields.SUPER_COLUMN, new org.apache.thrift.meta_data.FieldMetaData("super_column", org.apache.thrift.TFieldRequirementType.OPTIONAL,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(ColumnParent.class, metaDataMap);
- }
-
- public ColumnParent() {
- }
-
- public ColumnParent(
- String column_family)
- {
- this();
- this.column_family = column_family;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public ColumnParent(ColumnParent other) {
- if (other.isSetColumn_family()) {
- this.column_family = other.column_family;
- }
- if (other.isSetSuper_column()) {
- this.super_column = org.apache.thrift.TBaseHelper.copyBinary(other.super_column);
-;
- }
- }
-
- public ColumnParent deepCopy() {
- return new ColumnParent(this);
- }
-
- @Override
- public void clear() {
- this.column_family = null;
- this.super_column = null;
- }
-
- public String getColumn_family() {
- return this.column_family;
- }
-
- public ColumnParent setColumn_family(String column_family) {
- this.column_family = column_family;
- return this;
- }
-
- public void unsetColumn_family() {
- this.column_family = null;
- }
-
- /** Returns true if field column_family is set (has been assigned a value) and false otherwise */
- public boolean isSetColumn_family() {
- return this.column_family != null;
- }
-
- public void setColumn_familyIsSet(boolean value) {
- if (!value) {
- this.column_family = null;
- }
- }
-
- public byte[] getSuper_column() {
- setSuper_column(org.apache.thrift.TBaseHelper.rightSize(super_column));
- return super_column == null ? null : super_column.array();
- }
-
- public ByteBuffer bufferForSuper_column() {
- return super_column;
- }
-
- public ColumnParent setSuper_column(byte[] super_column) {
- setSuper_column(super_column == null ? (ByteBuffer)null : ByteBuffer.wrap(super_column));
- return this;
- }
-
- public ColumnParent setSuper_column(ByteBuffer super_column) {
- this.super_column = super_column;
- return this;
- }
-
- public void unsetSuper_column() {
- this.super_column = null;
- }
-
- /** Returns true if field super_column is set (has been assigned a value) and false otherwise */
- public boolean isSetSuper_column() {
- return this.super_column != null;
- }
-
- public void setSuper_columnIsSet(boolean value) {
- if (!value) {
- this.super_column = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case COLUMN_FAMILY:
- if (value == null) {
- unsetColumn_family();
- } else {
- setColumn_family((String)value);
- }
- break;
-
- case SUPER_COLUMN:
- if (value == null) {
- unsetSuper_column();
- } else {
- setSuper_column((ByteBuffer)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case COLUMN_FAMILY:
- return getColumn_family();
-
- case SUPER_COLUMN:
- return getSuper_column();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case COLUMN_FAMILY:
- return isSetColumn_family();
- case SUPER_COLUMN:
- return isSetSuper_column();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof ColumnParent)
- return this.equals((ColumnParent)that);
- return false;
- }
-
- public boolean equals(ColumnParent that) {
- if (that == null)
- return false;
-
- boolean this_present_column_family = true && this.isSetColumn_family();
- boolean that_present_column_family = true && that.isSetColumn_family();
- if (this_present_column_family || that_present_column_family) {
- if (!(this_present_column_family && that_present_column_family))
- return false;
- if (!this.column_family.equals(that.column_family))
- return false;
- }
-
- boolean this_present_super_column = true && this.isSetSuper_column();
- boolean that_present_super_column = true && that.isSetSuper_column();
- if (this_present_super_column || that_present_super_column) {
- if (!(this_present_super_column && that_present_super_column))
- return false;
- if (!this.super_column.equals(that.super_column))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_column_family = true && (isSetColumn_family());
- builder.append(present_column_family);
- if (present_column_family)
- builder.append(column_family);
-
- boolean present_super_column = true && (isSetSuper_column());
- builder.append(present_super_column);
- if (present_super_column)
- builder.append(super_column);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(ColumnParent other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetColumn_family()).compareTo(other.isSetColumn_family());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetColumn_family()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.column_family, other.column_family);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetSuper_column()).compareTo(other.isSetSuper_column());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetSuper_column()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.super_column, other.super_column);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("ColumnParent(");
- boolean first = true;
-
- sb.append("column_family:");
- if (this.column_family == null) {
- sb.append("null");
- } else {
- sb.append(this.column_family);
- }
- first = false;
- if (isSetSuper_column()) {
- if (!first) sb.append(", ");
- sb.append("super_column:");
- if (this.super_column == null) {
- sb.append("null");
- } else {
- org.apache.thrift.TBaseHelper.toString(this.super_column, sb);
- }
- first = false;
- }
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- if (column_family == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'column_family' was not present! Struct: " + toString());
- }
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class ColumnParentStandardSchemeFactory implements SchemeFactory {
- public ColumnParentStandardScheme getScheme() {
- return new ColumnParentStandardScheme();
- }
- }
-
- private static class ColumnParentStandardScheme extends StandardScheme<ColumnParent> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, ColumnParent struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 3: // COLUMN_FAMILY
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.column_family = iprot.readString();
- struct.setColumn_familyIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 4: // SUPER_COLUMN
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.super_column = iprot.readBinary();
- struct.setSuper_columnIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, ColumnParent struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.column_family != null) {
- oprot.writeFieldBegin(COLUMN_FAMILY_FIELD_DESC);
- oprot.writeString(struct.column_family);
- oprot.writeFieldEnd();
- }
- if (struct.super_column != null) {
- if (struct.isSetSuper_column()) {
- oprot.writeFieldBegin(SUPER_COLUMN_FIELD_DESC);
- oprot.writeBinary(struct.super_column);
- oprot.writeFieldEnd();
- }
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class ColumnParentTupleSchemeFactory implements SchemeFactory {
- public ColumnParentTupleScheme getScheme() {
- return new ColumnParentTupleScheme();
- }
- }
-
- private static class ColumnParentTupleScheme extends TupleScheme<ColumnParent> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, ColumnParent struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- oprot.writeString(struct.column_family);
- BitSet optionals = new BitSet();
- if (struct.isSetSuper_column()) {
- optionals.set(0);
- }
- oprot.writeBitSet(optionals, 1);
- if (struct.isSetSuper_column()) {
- oprot.writeBinary(struct.super_column);
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, ColumnParent struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- struct.column_family = iprot.readString();
- struct.setColumn_familyIsSet(true);
- BitSet incoming = iprot.readBitSet(1);
- if (incoming.get(0)) {
- struct.super_column = iprot.readBinary();
- struct.setSuper_columnIsSet(true);
- }
- }
- }
-
-}
-
diff --git a/interface/thrift/gen-java/org/apache/cassandra/thrift/ColumnPath.java b/interface/thrift/gen-java/org/apache/cassandra/thrift/ColumnPath.java
deleted file mode 100644
index 997c1bb..0000000
--- a/interface/thrift/gen-java/org/apache/cassandra/thrift/ColumnPath.java
+++ /dev/null
@@ -1,648 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.1)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- * @generated
- */
-package org.apache.cassandra.thrift;
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
-
-import org.apache.commons.lang3.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-
-/**
- * The ColumnPath is the path to a single column in Cassandra. It might make sense to think of ColumnPath and
- * ColumnParent in terms of a directory structure.
- *
- * ColumnPath is used to looking up a single column.
- *
- * @param column_family. The name of the CF of the column being looked up.
- * @param super_column. The super column name.
- * @param column. The column name.
- */
-public class ColumnPath implements org.apache.thrift.TBase<ColumnPath, ColumnPath._Fields>, java.io.Serializable, Cloneable, Comparable<ColumnPath> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("ColumnPath");
-
- private static final org.apache.thrift.protocol.TField COLUMN_FAMILY_FIELD_DESC = new org.apache.thrift.protocol.TField("column_family", org.apache.thrift.protocol.TType.STRING, (short)3);
- private static final org.apache.thrift.protocol.TField SUPER_COLUMN_FIELD_DESC = new org.apache.thrift.protocol.TField("super_column", org.apache.thrift.protocol.TType.STRING, (short)4);
- private static final org.apache.thrift.protocol.TField COLUMN_FIELD_DESC = new org.apache.thrift.protocol.TField("column", org.apache.thrift.protocol.TType.STRING, (short)5);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new ColumnPathStandardSchemeFactory());
- schemes.put(TupleScheme.class, new ColumnPathTupleSchemeFactory());
- }
-
- public String column_family; // required
- public ByteBuffer super_column; // optional
- public ByteBuffer column; // optional
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- COLUMN_FAMILY((short)3, "column_family"),
- SUPER_COLUMN((short)4, "super_column"),
- COLUMN((short)5, "column");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 3: // COLUMN_FAMILY
- return COLUMN_FAMILY;
- case 4: // SUPER_COLUMN
- return SUPER_COLUMN;
- case 5: // COLUMN
- return COLUMN;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- private _Fields optionals[] = {_Fields.SUPER_COLUMN,_Fields.COLUMN};
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.COLUMN_FAMILY, new org.apache.thrift.meta_data.FieldMetaData("column_family", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
- tmpMap.put(_Fields.SUPER_COLUMN, new org.apache.thrift.meta_data.FieldMetaData("super_column", org.apache.thrift.TFieldRequirementType.OPTIONAL,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true)));
- tmpMap.put(_Fields.COLUMN, new org.apache.thrift.meta_data.FieldMetaData("column", org.apache.thrift.TFieldRequirementType.OPTIONAL,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(ColumnPath.class, metaDataMap);
- }
-
- public ColumnPath() {
- }
-
- public ColumnPath(
- String column_family)
- {
- this();
- this.column_family = column_family;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public ColumnPath(ColumnPath other) {
- if (other.isSetColumn_family()) {
- this.column_family = other.column_family;
- }
- if (other.isSetSuper_column()) {
- this.super_column = org.apache.thrift.TBaseHelper.copyBinary(other.super_column);
-;
- }
- if (other.isSetColumn()) {
- this.column = org.apache.thrift.TBaseHelper.copyBinary(other.column);
-;
- }
- }
-
- public ColumnPath deepCopy() {
- return new ColumnPath(this);
- }
-
- @Override
- public void clear() {
- this.column_family = null;
- this.super_column = null;
- this.column = null;
- }
-
- public String getColumn_family() {
- return this.column_family;
- }
-
- public ColumnPath setColumn_family(String column_family) {
- this.column_family = column_family;
- return this;
- }
-
- public void unsetColumn_family() {
- this.column_family = null;
- }
-
- /** Returns true if field column_family is set (has been assigned a value) and false otherwise */
- public boolean isSetColumn_family() {
- return this.column_family != null;
- }
-
- public void setColumn_familyIsSet(boolean value) {
- if (!value) {
- this.column_family = null;
- }
- }
-
- public byte[] getSuper_column() {
- setSuper_column(org.apache.thrift.TBaseHelper.rightSize(super_column));
- return super_column == null ? null : super_column.array();
- }
-
- public ByteBuffer bufferForSuper_column() {
- return super_column;
- }
-
- public ColumnPath setSuper_column(byte[] super_column) {
- setSuper_column(super_column == null ? (ByteBuffer)null : ByteBuffer.wrap(super_column));
- return this;
- }
-
- public ColumnPath setSuper_column(ByteBuffer super_column) {
- this.super_column = super_column;
- return this;
- }
-
- public void unsetSuper_column() {
- this.super_column = null;
- }
-
- /** Returns true if field super_column is set (has been assigned a value) and false otherwise */
- public boolean isSetSuper_column() {
- return this.super_column != null;
- }
-
- public void setSuper_columnIsSet(boolean value) {
- if (!value) {
- this.super_column = null;
- }
- }
-
- public byte[] getColumn() {
- setColumn(org.apache.thrift.TBaseHelper.rightSize(column));
- return column == null ? null : column.array();
- }
-
- public ByteBuffer bufferForColumn() {
- return column;
- }
-
- public ColumnPath setColumn(byte[] column) {
- setColumn(column == null ? (ByteBuffer)null : ByteBuffer.wrap(column));
- return this;
- }
-
- public ColumnPath setColumn(ByteBuffer column) {
- this.column = column;
- return this;
- }
-
- public void unsetColumn() {
- this.column = null;
- }
-
- /** Returns true if field column is set (has been assigned a value) and false otherwise */
- public boolean isSetColumn() {
- return this.column != null;
- }
-
- public void setColumnIsSet(boolean value) {
- if (!value) {
- this.column = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case COLUMN_FAMILY:
- if (value == null) {
- unsetColumn_family();
- } else {
- setColumn_family((String)value);
- }
- break;
-
- case SUPER_COLUMN:
- if (value == null) {
- unsetSuper_column();
- } else {
- setSuper_column((ByteBuffer)value);
- }
- break;
-
- case COLUMN:
- if (value == null) {
- unsetColumn();
- } else {
- setColumn((ByteBuffer)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case COLUMN_FAMILY:
- return getColumn_family();
-
- case SUPER_COLUMN:
- return getSuper_column();
-
- case COLUMN:
- return getColumn();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case COLUMN_FAMILY:
- return isSetColumn_family();
- case SUPER_COLUMN:
- return isSetSuper_column();
- case COLUMN:
- return isSetColumn();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof ColumnPath)
- return this.equals((ColumnPath)that);
- return false;
- }
-
- public boolean equals(ColumnPath that) {
- if (that == null)
- return false;
-
- boolean this_present_column_family = true && this.isSetColumn_family();
- boolean that_present_column_family = true && that.isSetColumn_family();
- if (this_present_column_family || that_present_column_family) {
- if (!(this_present_column_family && that_present_column_family))
- return false;
- if (!this.column_family.equals(that.column_family))
- return false;
- }
-
- boolean this_present_super_column = true && this.isSetSuper_column();
- boolean that_present_super_column = true && that.isSetSuper_column();
- if (this_present_super_column || that_present_super_column) {
- if (!(this_present_super_column && that_present_super_column))
- return false;
- if (!this.super_column.equals(that.super_column))
- return false;
- }
-
- boolean this_present_column = true && this.isSetColumn();
- boolean that_present_column = true && that.isSetColumn();
- if (this_present_column || that_present_column) {
- if (!(this_present_column && that_present_column))
- return false;
- if (!this.column.equals(that.column))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_column_family = true && (isSetColumn_family());
- builder.append(present_column_family);
- if (present_column_family)
- builder.append(column_family);
-
- boolean present_super_column = true && (isSetSuper_column());
- builder.append(present_super_column);
- if (present_super_column)
- builder.append(super_column);
-
- boolean present_column = true && (isSetColumn());
- builder.append(present_column);
- if (present_column)
- builder.append(column);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(ColumnPath other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetColumn_family()).compareTo(other.isSetColumn_family());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetColumn_family()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.column_family, other.column_family);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetSuper_column()).compareTo(other.isSetSuper_column());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetSuper_column()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.super_column, other.super_column);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetColumn()).compareTo(other.isSetColumn());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetColumn()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.column, other.column);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("ColumnPath(");
- boolean first = true;
-
- sb.append("column_family:");
- if (this.column_family == null) {
- sb.append("null");
- } else {
- sb.append(this.column_family);
- }
- first = false;
- if (isSetSuper_column()) {
- if (!first) sb.append(", ");
- sb.append("super_column:");
- if (this.super_column == null) {
- sb.append("null");
- } else {
- org.apache.thrift.TBaseHelper.toString(this.super_column, sb);
- }
- first = false;
- }
- if (isSetColumn()) {
- if (!first) sb.append(", ");
- sb.append("column:");
- if (this.column == null) {
- sb.append("null");
- } else {
- org.apache.thrift.TBaseHelper.toString(this.column, sb);
- }
- first = false;
- }
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- if (column_family == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'column_family' was not present! Struct: " + toString());
- }
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class ColumnPathStandardSchemeFactory implements SchemeFactory {
- public ColumnPathStandardScheme getScheme() {
- return new ColumnPathStandardScheme();
- }
- }
-
- private static class ColumnPathStandardScheme extends StandardScheme<ColumnPath> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, ColumnPath struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 3: // COLUMN_FAMILY
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.column_family = iprot.readString();
- struct.setColumn_familyIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 4: // SUPER_COLUMN
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.super_column = iprot.readBinary();
- struct.setSuper_columnIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 5: // COLUMN
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.column = iprot.readBinary();
- struct.setColumnIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, ColumnPath struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.column_family != null) {
- oprot.writeFieldBegin(COLUMN_FAMILY_FIELD_DESC);
- oprot.writeString(struct.column_family);
- oprot.writeFieldEnd();
- }
- if (struct.super_column != null) {
- if (struct.isSetSuper_column()) {
- oprot.writeFieldBegin(SUPER_COLUMN_FIELD_DESC);
- oprot.writeBinary(struct.super_column);
- oprot.writeFieldEnd();
- }
- }
- if (struct.column != null) {
- if (struct.isSetColumn()) {
- oprot.writeFieldBegin(COLUMN_FIELD_DESC);
- oprot.writeBinary(struct.column);
- oprot.writeFieldEnd();
- }
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class ColumnPathTupleSchemeFactory implements SchemeFactory {
- public ColumnPathTupleScheme getScheme() {
- return new ColumnPathTupleScheme();
- }
- }
-
- private static class ColumnPathTupleScheme extends TupleScheme<ColumnPath> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, ColumnPath struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- oprot.writeString(struct.column_family);
- BitSet optionals = new BitSet();
- if (struct.isSetSuper_column()) {
- optionals.set(0);
- }
- if (struct.isSetColumn()) {
- optionals.set(1);
- }
- oprot.writeBitSet(optionals, 2);
- if (struct.isSetSuper_column()) {
- oprot.writeBinary(struct.super_column);
- }
- if (struct.isSetColumn()) {
- oprot.writeBinary(struct.column);
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, ColumnPath struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- struct.column_family = iprot.readString();
- struct.setColumn_familyIsSet(true);
- BitSet incoming = iprot.readBitSet(2);
- if (incoming.get(0)) {
- struct.super_column = iprot.readBinary();
- struct.setSuper_columnIsSet(true);
- }
- if (incoming.get(1)) {
- struct.column = iprot.readBinary();
- struct.setColumnIsSet(true);
- }
- }
- }
-
-}
-
diff --git a/interface/thrift/gen-java/org/apache/cassandra/thrift/ColumnSlice.java b/interface/thrift/gen-java/org/apache/cassandra/thrift/ColumnSlice.java
deleted file mode 100644
index 67b88a3..0000000
--- a/interface/thrift/gen-java/org/apache/cassandra/thrift/ColumnSlice.java
+++ /dev/null
@@ -1,551 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.1)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- * @generated
- */
-package org.apache.cassandra.thrift;
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
-
-import org.apache.commons.lang3.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * The ColumnSlice is used to select a set of columns from inside a row.
- * If start or finish are unspecified they will default to the start-of
- * end-of value.
- * @param start. The start of the ColumnSlice inclusive
- * @param finish. The end of the ColumnSlice inclusive
- */
-public class ColumnSlice implements org.apache.thrift.TBase<ColumnSlice, ColumnSlice._Fields>, java.io.Serializable, Cloneable, Comparable<ColumnSlice> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("ColumnSlice");
-
- private static final org.apache.thrift.protocol.TField START_FIELD_DESC = new org.apache.thrift.protocol.TField("start", org.apache.thrift.protocol.TType.STRING, (short)1);
- private static final org.apache.thrift.protocol.TField FINISH_FIELD_DESC = new org.apache.thrift.protocol.TField("finish", org.apache.thrift.protocol.TType.STRING, (short)2);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new ColumnSliceStandardSchemeFactory());
- schemes.put(TupleScheme.class, new ColumnSliceTupleSchemeFactory());
- }
-
- public ByteBuffer start; // optional
- public ByteBuffer finish; // optional
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- START((short)1, "start"),
- FINISH((short)2, "finish");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // START
- return START;
- case 2: // FINISH
- return FINISH;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- private _Fields optionals[] = {_Fields.START,_Fields.FINISH};
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.START, new org.apache.thrift.meta_data.FieldMetaData("start", org.apache.thrift.TFieldRequirementType.OPTIONAL,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true)));
- tmpMap.put(_Fields.FINISH, new org.apache.thrift.meta_data.FieldMetaData("finish", org.apache.thrift.TFieldRequirementType.OPTIONAL,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(ColumnSlice.class, metaDataMap);
- }
-
- public ColumnSlice() {
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public ColumnSlice(ColumnSlice other) {
- if (other.isSetStart()) {
- this.start = org.apache.thrift.TBaseHelper.copyBinary(other.start);
-;
- }
- if (other.isSetFinish()) {
- this.finish = org.apache.thrift.TBaseHelper.copyBinary(other.finish);
-;
- }
- }
-
- public ColumnSlice deepCopy() {
- return new ColumnSlice(this);
- }
-
- @Override
- public void clear() {
- this.start = null;
- this.finish = null;
- }
-
- public byte[] getStart() {
- setStart(org.apache.thrift.TBaseHelper.rightSize(start));
- return start == null ? null : start.array();
- }
-
- public ByteBuffer bufferForStart() {
- return start;
- }
-
- public ColumnSlice setStart(byte[] start) {
- setStart(start == null ? (ByteBuffer)null : ByteBuffer.wrap(start));
- return this;
- }
-
- public ColumnSlice setStart(ByteBuffer start) {
- this.start = start;
- return this;
- }
-
- public void unsetStart() {
- this.start = null;
- }
-
- /** Returns true if field start is set (has been assigned a value) and false otherwise */
- public boolean isSetStart() {
- return this.start != null;
- }
-
- public void setStartIsSet(boolean value) {
- if (!value) {
- this.start = null;
- }
- }
-
- public byte[] getFinish() {
- setFinish(org.apache.thrift.TBaseHelper.rightSize(finish));
- return finish == null ? null : finish.array();
- }
-
- public ByteBuffer bufferForFinish() {
- return finish;
- }
-
- public ColumnSlice setFinish(byte[] finish) {
- setFinish(finish == null ? (ByteBuffer)null : ByteBuffer.wrap(finish));
- return this;
- }
-
- public ColumnSlice setFinish(ByteBuffer finish) {
- this.finish = finish;
- return this;
- }
-
- public void unsetFinish() {
- this.finish = null;
- }
-
- /** Returns true if field finish is set (has been assigned a value) and false otherwise */
- public boolean isSetFinish() {
- return this.finish != null;
- }
-
- public void setFinishIsSet(boolean value) {
- if (!value) {
- this.finish = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case START:
- if (value == null) {
- unsetStart();
- } else {
- setStart((ByteBuffer)value);
- }
- break;
-
- case FINISH:
- if (value == null) {
- unsetFinish();
- } else {
- setFinish((ByteBuffer)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case START:
- return getStart();
-
- case FINISH:
- return getFinish();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case START:
- return isSetStart();
- case FINISH:
- return isSetFinish();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof ColumnSlice)
- return this.equals((ColumnSlice)that);
- return false;
- }
-
- public boolean equals(ColumnSlice that) {
- if (that == null)
- return false;
-
- boolean this_present_start = true && this.isSetStart();
- boolean that_present_start = true && that.isSetStart();
- if (this_present_start || that_present_start) {
- if (!(this_present_start && that_present_start))
- return false;
- if (!this.start.equals(that.start))
- return false;
- }
-
- boolean this_present_finish = true && this.isSetFinish();
- boolean that_present_finish = true && that.isSetFinish();
- if (this_present_finish || that_present_finish) {
- if (!(this_present_finish && that_present_finish))
- return false;
- if (!this.finish.equals(that.finish))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_start = true && (isSetStart());
- builder.append(present_start);
- if (present_start)
- builder.append(start);
-
- boolean present_finish = true && (isSetFinish());
- builder.append(present_finish);
- if (present_finish)
- builder.append(finish);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(ColumnSlice other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetStart()).compareTo(other.isSetStart());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetStart()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.start, other.start);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetFinish()).compareTo(other.isSetFinish());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetFinish()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.finish, other.finish);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("ColumnSlice(");
- boolean first = true;
-
- if (isSetStart()) {
- sb.append("start:");
- if (this.start == null) {
- sb.append("null");
- } else {
- org.apache.thrift.TBaseHelper.toString(this.start, sb);
- }
- first = false;
- }
- if (isSetFinish()) {
- if (!first) sb.append(", ");
- sb.append("finish:");
- if (this.finish == null) {
- sb.append("null");
- } else {
- org.apache.thrift.TBaseHelper.toString(this.finish, sb);
- }
- first = false;
- }
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class ColumnSliceStandardSchemeFactory implements SchemeFactory {
- public ColumnSliceStandardScheme getScheme() {
- return new ColumnSliceStandardScheme();
- }
- }
-
- private static class ColumnSliceStandardScheme extends StandardScheme<ColumnSlice> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, ColumnSlice struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // START
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.start = iprot.readBinary();
- struct.setStartIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 2: // FINISH
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.finish = iprot.readBinary();
- struct.setFinishIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, ColumnSlice struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.start != null) {
- if (struct.isSetStart()) {
- oprot.writeFieldBegin(START_FIELD_DESC);
- oprot.writeBinary(struct.start);
- oprot.writeFieldEnd();
- }
- }
- if (struct.finish != null) {
- if (struct.isSetFinish()) {
- oprot.writeFieldBegin(FINISH_FIELD_DESC);
- oprot.writeBinary(struct.finish);
- oprot.writeFieldEnd();
- }
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class ColumnSliceTupleSchemeFactory implements SchemeFactory {
- public ColumnSliceTupleScheme getScheme() {
- return new ColumnSliceTupleScheme();
- }
- }
-
- private static class ColumnSliceTupleScheme extends TupleScheme<ColumnSlice> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, ColumnSlice struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- BitSet optionals = new BitSet();
- if (struct.isSetStart()) {
- optionals.set(0);
- }
- if (struct.isSetFinish()) {
- optionals.set(1);
- }
- oprot.writeBitSet(optionals, 2);
- if (struct.isSetStart()) {
- oprot.writeBinary(struct.start);
- }
- if (struct.isSetFinish()) {
- oprot.writeBinary(struct.finish);
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, ColumnSlice struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- BitSet incoming = iprot.readBitSet(2);
- if (incoming.get(0)) {
- struct.start = iprot.readBinary();
- struct.setStartIsSet(true);
- }
- if (incoming.get(1)) {
- struct.finish = iprot.readBinary();
- struct.setFinishIsSet(true);
- }
- }
- }
-
-}
-
diff --git a/interface/thrift/gen-java/org/apache/cassandra/thrift/Compression.java b/interface/thrift/gen-java/org/apache/cassandra/thrift/Compression.java
deleted file mode 100644
index acaf43f..0000000
--- a/interface/thrift/gen-java/org/apache/cassandra/thrift/Compression.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.1)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- * @generated
- */
-package org.apache.cassandra.thrift;
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
-
-
-import java.util.Map;
-import java.util.HashMap;
-import org.apache.thrift.TEnum;
-
-/**
- * CQL query compression
- */
-public enum Compression implements org.apache.thrift.TEnum {
- GZIP(1),
- NONE(2);
-
- private final int value;
-
- private Compression(int value) {
- this.value = value;
- }
-
- /**
- * Get the integer value of this enum value, as defined in the Thrift IDL.
- */
- public int getValue() {
- return value;
- }
-
- /**
- * Find a the enum type by its integer value, as defined in the Thrift IDL.
- * @return null if the value is not found.
- */
- public static Compression findByValue(int value) {
- switch (value) {
- case 1:
- return GZIP;
- case 2:
- return NONE;
- default:
- return null;
- }
- }
-}
diff --git a/interface/thrift/gen-java/org/apache/cassandra/thrift/ConsistencyLevel.java b/interface/thrift/gen-java/org/apache/cassandra/thrift/ConsistencyLevel.java
deleted file mode 100644
index ec5080a..0000000
--- a/interface/thrift/gen-java/org/apache/cassandra/thrift/ConsistencyLevel.java
+++ /dev/null
@@ -1,137 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.1)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- * @generated
- */
-package org.apache.cassandra.thrift;
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
-
-
-import java.util.Map;
-import java.util.HashMap;
-import org.apache.thrift.TEnum;
-
-/**
- * The ConsistencyLevel is an enum that controls both read and write
- * behavior based on the ReplicationFactor of the keyspace. The
- * different consistency levels have different meanings, depending on
- * if you're doing a write or read operation.
- *
- * If W + R > ReplicationFactor, where W is the number of nodes to
- * block for on write, and R the number to block for on reads, you
- * will have strongly consistent behavior; that is, readers will
- * always see the most recent write. Of these, the most interesting is
- * to do QUORUM reads and writes, which gives you consistency while
- * still allowing availability in the face of node failures up to half
- * of <ReplicationFactor>. Of course if latency is more important than
- * consistency then you can use lower values for either or both.
- *
- * Some ConsistencyLevels (ONE, TWO, THREE) refer to a specific number
- * of replicas rather than a logical concept that adjusts
- * automatically with the replication factor. Of these, only ONE is
- * commonly used; TWO and (even more rarely) THREE are only useful
- * when you care more about guaranteeing a certain level of
- * durability, than consistency.
- *
- * Write consistency levels make the following guarantees before reporting success to the client:
- * ANY Ensure that the write has been written once somewhere, including possibly being hinted in a non-target node.
- * ONE Ensure that the write has been written to at least 1 node's commit log and memory table
- * TWO Ensure that the write has been written to at least 2 node's commit log and memory table
- * THREE Ensure that the write has been written to at least 3 node's commit log and memory table
- * QUORUM Ensure that the write has been written to <ReplicationFactor> / 2 + 1 nodes
- * LOCAL_ONE Ensure that the write has been written to 1 node within the local datacenter (requires NetworkTopologyStrategy)
- * LOCAL_QUORUM Ensure that the write has been written to <ReplicationFactor> / 2 + 1 nodes, within the local datacenter (requires NetworkTopologyStrategy)
- * EACH_QUORUM Ensure that the write has been written to <ReplicationFactor> / 2 + 1 nodes in each datacenter (requires NetworkTopologyStrategy)
- * ALL Ensure that the write is written to <code><ReplicationFactor></code> nodes before responding to the client.
- *
- * Read consistency levels make the following guarantees before returning successful results to the client:
- * ANY Not supported. You probably want ONE instead.
- * ONE Returns the record obtained from a single replica.
- * TWO Returns the record with the most recent timestamp once two replicas have replied.
- * THREE Returns the record with the most recent timestamp once three replicas have replied.
- * QUORUM Returns the record with the most recent timestamp once a majority of replicas have replied.
- * LOCAL_ONE Returns the record with the most recent timestamp once a single replica within the local datacenter have replied.
- * LOCAL_QUORUM Returns the record with the most recent timestamp once a majority of replicas within the local datacenter have replied.
- * EACH_QUORUM Returns the record with the most recent timestamp once a majority of replicas within each datacenter have replied.
- * ALL Returns the record with the most recent timestamp once all replicas have replied (implies no replica may be down)..
- */
-public enum ConsistencyLevel implements org.apache.thrift.TEnum {
- ONE(1),
- QUORUM(2),
- LOCAL_QUORUM(3),
- EACH_QUORUM(4),
- ALL(5),
- ANY(6),
- TWO(7),
- THREE(8),
- SERIAL(9),
- LOCAL_SERIAL(10),
- LOCAL_ONE(11);
-
- private final int value;
-
- private ConsistencyLevel(int value) {
- this.value = value;
- }
-
- /**
- * Get the integer value of this enum value, as defined in the Thrift IDL.
- */
- public int getValue() {
- return value;
- }
-
- /**
- * Find a the enum type by its integer value, as defined in the Thrift IDL.
- * @return null if the value is not found.
- */
- public static ConsistencyLevel findByValue(int value) {
- switch (value) {
- case 1:
- return ONE;
- case 2:
- return QUORUM;
- case 3:
- return LOCAL_QUORUM;
- case 4:
- return EACH_QUORUM;
- case 5:
- return ALL;
- case 6:
- return ANY;
- case 7:
- return TWO;
- case 8:
- return THREE;
- case 9:
- return SERIAL;
- case 10:
- return LOCAL_SERIAL;
- case 11:
- return LOCAL_ONE;
- default:
- return null;
- }
- }
-}
diff --git a/interface/thrift/gen-java/org/apache/cassandra/thrift/CounterColumn.java b/interface/thrift/gen-java/org/apache/cassandra/thrift/CounterColumn.java
deleted file mode 100644
index cd9c593..0000000
--- a/interface/thrift/gen-java/org/apache/cassandra/thrift/CounterColumn.java
+++ /dev/null
@@ -1,520 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.1)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- * @generated
- */
-package org.apache.cassandra.thrift;
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
-
-import org.apache.commons.lang3.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class CounterColumn implements org.apache.thrift.TBase<CounterColumn, CounterColumn._Fields>, java.io.Serializable, Cloneable, Comparable<CounterColumn> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("CounterColumn");
-
- private static final org.apache.thrift.protocol.TField NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("name", org.apache.thrift.protocol.TType.STRING, (short)1);
- private static final org.apache.thrift.protocol.TField VALUE_FIELD_DESC = new org.apache.thrift.protocol.TField("value", org.apache.thrift.protocol.TType.I64, (short)2);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new CounterColumnStandardSchemeFactory());
- schemes.put(TupleScheme.class, new CounterColumnTupleSchemeFactory());
- }
-
- public ByteBuffer name; // required
- public long value; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- NAME((short)1, "name"),
- VALUE((short)2, "value");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // NAME
- return NAME;
- case 2: // VALUE
- return VALUE;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- private static final int __VALUE_ISSET_ID = 0;
- private byte __isset_bitfield = 0;
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.NAME, new org.apache.thrift.meta_data.FieldMetaData("name", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true)));
- tmpMap.put(_Fields.VALUE, new org.apache.thrift.meta_data.FieldMetaData("value", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(CounterColumn.class, metaDataMap);
- }
-
- public CounterColumn() {
- }
-
- public CounterColumn(
- ByteBuffer name,
- long value)
- {
- this();
- this.name = name;
- this.value = value;
- setValueIsSet(true);
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public CounterColumn(CounterColumn other) {
- __isset_bitfield = other.__isset_bitfield;
- if (other.isSetName()) {
- this.name = org.apache.thrift.TBaseHelper.copyBinary(other.name);
-;
- }
- this.value = other.value;
- }
-
- public CounterColumn deepCopy() {
- return new CounterColumn(this);
- }
-
- @Override
- public void clear() {
- this.name = null;
- setValueIsSet(false);
- this.value = 0;
- }
-
- public byte[] getName() {
- setName(org.apache.thrift.TBaseHelper.rightSize(name));
- return name == null ? null : name.array();
- }
-
- public ByteBuffer bufferForName() {
- return name;
- }
-
- public CounterColumn setName(byte[] name) {
- setName(name == null ? (ByteBuffer)null : ByteBuffer.wrap(name));
- return this;
- }
-
- public CounterColumn setName(ByteBuffer name) {
- this.name = name;
- return this;
- }
-
- public void unsetName() {
- this.name = null;
- }
-
- /** Returns true if field name is set (has been assigned a value) and false otherwise */
- public boolean isSetName() {
- return this.name != null;
- }
-
- public void setNameIsSet(boolean value) {
- if (!value) {
- this.name = null;
- }
- }
-
- public long getValue() {
- return this.value;
- }
-
- public CounterColumn setValue(long value) {
- this.value = value;
- setValueIsSet(true);
- return this;
- }
-
- public void unsetValue() {
- __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __VALUE_ISSET_ID);
- }
-
- /** Returns true if field value is set (has been assigned a value) and false otherwise */
- public boolean isSetValue() {
- return EncodingUtils.testBit(__isset_bitfield, __VALUE_ISSET_ID);
- }
-
- public void setValueIsSet(boolean value) {
- __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __VALUE_ISSET_ID, value);
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case NAME:
- if (value == null) {
- unsetName();
- } else {
- setName((ByteBuffer)value);
- }
- break;
-
- case VALUE:
- if (value == null) {
- unsetValue();
- } else {
- setValue((Long)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case NAME:
- return getName();
-
- case VALUE:
- return Long.valueOf(getValue());
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case NAME:
- return isSetName();
- case VALUE:
- return isSetValue();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof CounterColumn)
- return this.equals((CounterColumn)that);
- return false;
- }
-
- public boolean equals(CounterColumn that) {
- if (that == null)
- return false;
-
- boolean this_present_name = true && this.isSetName();
- boolean that_present_name = true && that.isSetName();
- if (this_present_name || that_present_name) {
- if (!(this_present_name && that_present_name))
- return false;
- if (!this.name.equals(that.name))
- return false;
- }
-
- boolean this_present_value = true;
- boolean that_present_value = true;
- if (this_present_value || that_present_value) {
- if (!(this_present_value && that_present_value))
- return false;
- if (this.value != that.value)
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_name = true && (isSetName());
- builder.append(present_name);
- if (present_name)
- builder.append(name);
-
- boolean present_value = true;
- builder.append(present_value);
- if (present_value)
- builder.append(value);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(CounterColumn other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetName()).compareTo(other.isSetName());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetName()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.name, other.name);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetValue()).compareTo(other.isSetValue());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetValue()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.value, other.value);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("CounterColumn(");
- boolean first = true;
-
- sb.append("name:");
- if (this.name == null) {
- sb.append("null");
- } else {
- org.apache.thrift.TBaseHelper.toString(this.name, sb);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("value:");
- sb.append(this.value);
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- if (name == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'name' was not present! Struct: " + toString());
- }
- // alas, we cannot check 'value' because it's a primitive and you chose the non-beans generator.
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
- __isset_bitfield = 0;
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class CounterColumnStandardSchemeFactory implements SchemeFactory {
- public CounterColumnStandardScheme getScheme() {
- return new CounterColumnStandardScheme();
- }
- }
-
- private static class CounterColumnStandardScheme extends StandardScheme<CounterColumn> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, CounterColumn struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // NAME
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.name = iprot.readBinary();
- struct.setNameIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 2: // VALUE
- if (schemeField.type == org.apache.thrift.protocol.TType.I64) {
- struct.value = iprot.readI64();
- struct.setValueIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- if (!struct.isSetValue()) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'value' was not found in serialized data! Struct: " + toString());
- }
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, CounterColumn struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.name != null) {
- oprot.writeFieldBegin(NAME_FIELD_DESC);
- oprot.writeBinary(struct.name);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldBegin(VALUE_FIELD_DESC);
- oprot.writeI64(struct.value);
- oprot.writeFieldEnd();
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class CounterColumnTupleSchemeFactory implements SchemeFactory {
- public CounterColumnTupleScheme getScheme() {
- return new CounterColumnTupleScheme();
- }
- }
-
- private static class CounterColumnTupleScheme extends TupleScheme<CounterColumn> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, CounterColumn struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- oprot.writeBinary(struct.name);
- oprot.writeI64(struct.value);
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, CounterColumn struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- struct.name = iprot.readBinary();
- struct.setNameIsSet(true);
- struct.value = iprot.readI64();
- struct.setValueIsSet(true);
- }
- }
-
-}
-
diff --git a/interface/thrift/gen-java/org/apache/cassandra/thrift/CounterSuperColumn.java b/interface/thrift/gen-java/org/apache/cassandra/thrift/CounterSuperColumn.java
deleted file mode 100644
index 96234c7..0000000
--- a/interface/thrift/gen-java/org/apache/cassandra/thrift/CounterSuperColumn.java
+++ /dev/null
@@ -1,576 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.1)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- * @generated
- */
-package org.apache.cassandra.thrift;
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
-
-import org.apache.commons.lang3.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class CounterSuperColumn implements org.apache.thrift.TBase<CounterSuperColumn, CounterSuperColumn._Fields>, java.io.Serializable, Cloneable, Comparable<CounterSuperColumn> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("CounterSuperColumn");
-
- private static final org.apache.thrift.protocol.TField NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("name", org.apache.thrift.protocol.TType.STRING, (short)1);
- private static final org.apache.thrift.protocol.TField COLUMNS_FIELD_DESC = new org.apache.thrift.protocol.TField("columns", org.apache.thrift.protocol.TType.LIST, (short)2);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new CounterSuperColumnStandardSchemeFactory());
- schemes.put(TupleScheme.class, new CounterSuperColumnTupleSchemeFactory());
- }
-
- public ByteBuffer name; // required
- public List<CounterColumn> columns; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- NAME((short)1, "name"),
- COLUMNS((short)2, "columns");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // NAME
- return NAME;
- case 2: // COLUMNS
- return COLUMNS;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.NAME, new org.apache.thrift.meta_data.FieldMetaData("name", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true)));
- tmpMap.put(_Fields.COLUMNS, new org.apache.thrift.meta_data.FieldMetaData("columns", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST,
- new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, CounterColumn.class))));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(CounterSuperColumn.class, metaDataMap);
- }
-
- public CounterSuperColumn() {
- }
-
- public CounterSuperColumn(
- ByteBuffer name,
- List<CounterColumn> columns)
- {
- this();
- this.name = name;
- this.columns = columns;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public CounterSuperColumn(CounterSuperColumn other) {
- if (other.isSetName()) {
- this.name = org.apache.thrift.TBaseHelper.copyBinary(other.name);
-;
- }
- if (other.isSetColumns()) {
- List<CounterColumn> __this__columns = new ArrayList<CounterColumn>(other.columns.size());
- for (CounterColumn other_element : other.columns) {
- __this__columns.add(new CounterColumn(other_element));
- }
- this.columns = __this__columns;
- }
- }
-
- public CounterSuperColumn deepCopy() {
- return new CounterSuperColumn(this);
- }
-
- @Override
- public void clear() {
- this.name = null;
- this.columns = null;
- }
-
- public byte[] getName() {
- setName(org.apache.thrift.TBaseHelper.rightSize(name));
- return name == null ? null : name.array();
- }
-
- public ByteBuffer bufferForName() {
- return name;
- }
-
- public CounterSuperColumn setName(byte[] name) {
- setName(name == null ? (ByteBuffer)null : ByteBuffer.wrap(name));
- return this;
- }
-
- public CounterSuperColumn setName(ByteBuffer name) {
- this.name = name;
- return this;
- }
-
- public void unsetName() {
- this.name = null;
- }
-
- /** Returns true if field name is set (has been assigned a value) and false otherwise */
- public boolean isSetName() {
- return this.name != null;
- }
-
- public void setNameIsSet(boolean value) {
- if (!value) {
- this.name = null;
- }
- }
-
- public int getColumnsSize() {
- return (this.columns == null) ? 0 : this.columns.size();
- }
-
- public java.util.Iterator<CounterColumn> getColumnsIterator() {
- return (this.columns == null) ? null : this.columns.iterator();
- }
-
- public void addToColumns(CounterColumn elem) {
- if (this.columns == null) {
- this.columns = new ArrayList<CounterColumn>();
- }
- this.columns.add(elem);
- }
-
- public List<CounterColumn> getColumns() {
- return this.columns;
- }
-
- public CounterSuperColumn setColumns(List<CounterColumn> columns) {
- this.columns = columns;
- return this;
- }
-
- public void unsetColumns() {
- this.columns = null;
- }
-
- /** Returns true if field columns is set (has been assigned a value) and false otherwise */
- public boolean isSetColumns() {
- return this.columns != null;
- }
-
- public void setColumnsIsSet(boolean value) {
- if (!value) {
- this.columns = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case NAME:
- if (value == null) {
- unsetName();
- } else {
- setName((ByteBuffer)value);
- }
- break;
-
- case COLUMNS:
- if (value == null) {
- unsetColumns();
- } else {
- setColumns((List<CounterColumn>)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case NAME:
- return getName();
-
- case COLUMNS:
- return getColumns();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case NAME:
- return isSetName();
- case COLUMNS:
- return isSetColumns();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof CounterSuperColumn)
- return this.equals((CounterSuperColumn)that);
- return false;
- }
-
- public boolean equals(CounterSuperColumn that) {
- if (that == null)
- return false;
-
- boolean this_present_name = true && this.isSetName();
- boolean that_present_name = true && that.isSetName();
- if (this_present_name || that_present_name) {
- if (!(this_present_name && that_present_name))
- return false;
- if (!this.name.equals(that.name))
- return false;
- }
-
- boolean this_present_columns = true && this.isSetColumns();
- boolean that_present_columns = true && that.isSetColumns();
- if (this_present_columns || that_present_columns) {
- if (!(this_present_columns && that_present_columns))
- return false;
- if (!this.columns.equals(that.columns))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_name = true && (isSetName());
- builder.append(present_name);
- if (present_name)
- builder.append(name);
-
- boolean present_columns = true && (isSetColumns());
- builder.append(present_columns);
- if (present_columns)
- builder.append(columns);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(CounterSuperColumn other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetName()).compareTo(other.isSetName());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetName()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.name, other.name);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetColumns()).compareTo(other.isSetColumns());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetColumns()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.columns, other.columns);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("CounterSuperColumn(");
- boolean first = true;
-
- sb.append("name:");
- if (this.name == null) {
- sb.append("null");
- } else {
- org.apache.thrift.TBaseHelper.toString(this.name, sb);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("columns:");
- if (this.columns == null) {
- sb.append("null");
- } else {
- sb.append(this.columns);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- if (name == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'name' was not present! Struct: " + toString());
- }
- if (columns == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'columns' was not present! Struct: " + toString());
- }
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class CounterSuperColumnStandardSchemeFactory implements SchemeFactory {
- public CounterSuperColumnStandardScheme getScheme() {
- return new CounterSuperColumnStandardScheme();
- }
- }
-
- private static class CounterSuperColumnStandardScheme extends StandardScheme<CounterSuperColumn> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, CounterSuperColumn struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // NAME
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.name = iprot.readBinary();
- struct.setNameIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 2: // COLUMNS
- if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
- {
- org.apache.thrift.protocol.TList _list8 = iprot.readListBegin();
- struct.columns = new ArrayList<CounterColumn>(_list8.size);
- for (int _i9 = 0; _i9 < _list8.size; ++_i9)
- {
- CounterColumn _elem10;
- _elem10 = new CounterColumn();
- _elem10.read(iprot);
- struct.columns.add(_elem10);
- }
- iprot.readListEnd();
- }
- struct.setColumnsIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, CounterSuperColumn struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.name != null) {
- oprot.writeFieldBegin(NAME_FIELD_DESC);
- oprot.writeBinary(struct.name);
- oprot.writeFieldEnd();
- }
- if (struct.columns != null) {
- oprot.writeFieldBegin(COLUMNS_FIELD_DESC);
- {
- oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.columns.size()));
- for (CounterColumn _iter11 : struct.columns)
- {
- _iter11.write(oprot);
- }
- oprot.writeListEnd();
- }
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class CounterSuperColumnTupleSchemeFactory implements SchemeFactory {
- public CounterSuperColumnTupleScheme getScheme() {
- return new CounterSuperColumnTupleScheme();
- }
- }
-
- private static class CounterSuperColumnTupleScheme extends TupleScheme<CounterSuperColumn> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, CounterSuperColumn struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- oprot.writeBinary(struct.name);
- {
- oprot.writeI32(struct.columns.size());
- for (CounterColumn _iter12 : struct.columns)
- {
- _iter12.write(oprot);
- }
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, CounterSuperColumn struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- struct.name = iprot.readBinary();
- struct.setNameIsSet(true);
- {
- org.apache.thrift.protocol.TList _list13 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
- struct.columns = new ArrayList<CounterColumn>(_list13.size);
- for (int _i14 = 0; _i14 < _list13.size; ++_i14)
- {
- CounterColumn _elem15;
- _elem15 = new CounterColumn();
- _elem15.read(iprot);
- struct.columns.add(_elem15);
- }
- }
- struct.setColumnsIsSet(true);
- }
- }
-
-}
-
diff --git a/interface/thrift/gen-java/org/apache/cassandra/thrift/CqlMetadata.java b/interface/thrift/gen-java/org/apache/cassandra/thrift/CqlMetadata.java
deleted file mode 100644
index e2dcfa3..0000000
--- a/interface/thrift/gen-java/org/apache/cassandra/thrift/CqlMetadata.java
+++ /dev/null
@@ -1,817 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.1)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- * @generated
- */
-package org.apache.cassandra.thrift;
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
-
-import org.apache.commons.lang3.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class CqlMetadata implements org.apache.thrift.TBase<CqlMetadata, CqlMetadata._Fields>, java.io.Serializable, Cloneable, Comparable<CqlMetadata> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("CqlMetadata");
-
- private static final org.apache.thrift.protocol.TField NAME_TYPES_FIELD_DESC = new org.apache.thrift.protocol.TField("name_types", org.apache.thrift.protocol.TType.MAP, (short)1);
- private static final org.apache.thrift.protocol.TField VALUE_TYPES_FIELD_DESC = new org.apache.thrift.protocol.TField("value_types", org.apache.thrift.protocol.TType.MAP, (short)2);
- private static final org.apache.thrift.protocol.TField DEFAULT_NAME_TYPE_FIELD_DESC = new org.apache.thrift.protocol.TField("default_name_type", org.apache.thrift.protocol.TType.STRING, (short)3);
- private static final org.apache.thrift.protocol.TField DEFAULT_VALUE_TYPE_FIELD_DESC = new org.apache.thrift.protocol.TField("default_value_type", org.apache.thrift.protocol.TType.STRING, (short)4);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new CqlMetadataStandardSchemeFactory());
- schemes.put(TupleScheme.class, new CqlMetadataTupleSchemeFactory());
- }
-
- public Map<ByteBuffer,String> name_types; // required
- public Map<ByteBuffer,String> value_types; // required
- public String default_name_type; // required
- public String default_value_type; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- NAME_TYPES((short)1, "name_types"),
- VALUE_TYPES((short)2, "value_types"),
- DEFAULT_NAME_TYPE((short)3, "default_name_type"),
- DEFAULT_VALUE_TYPE((short)4, "default_value_type");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // NAME_TYPES
- return NAME_TYPES;
- case 2: // VALUE_TYPES
- return VALUE_TYPES;
- case 3: // DEFAULT_NAME_TYPE
- return DEFAULT_NAME_TYPE;
- case 4: // DEFAULT_VALUE_TYPE
- return DEFAULT_VALUE_TYPE;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.NAME_TYPES, new org.apache.thrift.meta_data.FieldMetaData("name_types", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true),
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))));
- tmpMap.put(_Fields.VALUE_TYPES, new org.apache.thrift.meta_data.FieldMetaData("value_types", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true),
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))));
- tmpMap.put(_Fields.DEFAULT_NAME_TYPE, new org.apache.thrift.meta_data.FieldMetaData("default_name_type", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
- tmpMap.put(_Fields.DEFAULT_VALUE_TYPE, new org.apache.thrift.meta_data.FieldMetaData("default_value_type", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(CqlMetadata.class, metaDataMap);
- }
-
- public CqlMetadata() {
- }
-
- public CqlMetadata(
- Map<ByteBuffer,String> name_types,
- Map<ByteBuffer,String> value_types,
- String default_name_type,
- String default_value_type)
- {
- this();
- this.name_types = name_types;
- this.value_types = value_types;
- this.default_name_type = default_name_type;
- this.default_value_type = default_value_type;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public CqlMetadata(CqlMetadata other) {
- if (other.isSetName_types()) {
- Map<ByteBuffer,String> __this__name_types = new HashMap<ByteBuffer,String>(other.name_types);
- this.name_types = __this__name_types;
- }
- if (other.isSetValue_types()) {
- Map<ByteBuffer,String> __this__value_types = new HashMap<ByteBuffer,String>(other.value_types);
- this.value_types = __this__value_types;
- }
- if (other.isSetDefault_name_type()) {
- this.default_name_type = other.default_name_type;
- }
- if (other.isSetDefault_value_type()) {
- this.default_value_type = other.default_value_type;
- }
- }
-
- public CqlMetadata deepCopy() {
- return new CqlMetadata(this);
- }
-
- @Override
- public void clear() {
- this.name_types = null;
- this.value_types = null;
- this.default_name_type = null;
- this.default_value_type = null;
- }
-
- public int getName_typesSize() {
- return (this.name_types == null) ? 0 : this.name_types.size();
- }
-
- public void putToName_types(ByteBuffer key, String val) {
- if (this.name_types == null) {
- this.name_types = new HashMap<ByteBuffer,String>();
- }
- this.name_types.put(key, val);
- }
-
- public Map<ByteBuffer,String> getName_types() {
- return this.name_types;
- }
-
- public CqlMetadata setName_types(Map<ByteBuffer,String> name_types) {
- this.name_types = name_types;
- return this;
- }
-
- public void unsetName_types() {
- this.name_types = null;
- }
-
- /** Returns true if field name_types is set (has been assigned a value) and false otherwise */
- public boolean isSetName_types() {
- return this.name_types != null;
- }
-
- public void setName_typesIsSet(boolean value) {
- if (!value) {
- this.name_types = null;
- }
- }
-
- public int getValue_typesSize() {
- return (this.value_types == null) ? 0 : this.value_types.size();
- }
-
- public void putToValue_types(ByteBuffer key, String val) {
- if (this.value_types == null) {
- this.value_types = new HashMap<ByteBuffer,String>();
- }
- this.value_types.put(key, val);
- }
-
- public Map<ByteBuffer,String> getValue_types() {
- return this.value_types;
- }
-
- public CqlMetadata setValue_types(Map<ByteBuffer,String> value_types) {
- this.value_types = value_types;
- return this;
- }
-
- public void unsetValue_types() {
- this.value_types = null;
- }
-
- /** Returns true if field value_types is set (has been assigned a value) and false otherwise */
- public boolean isSetValue_types() {
- return this.value_types != null;
- }
-
- public void setValue_typesIsSet(boolean value) {
- if (!value) {
- this.value_types = null;
- }
- }
-
- public String getDefault_name_type() {
- return this.default_name_type;
- }
-
- public CqlMetadata setDefault_name_type(String default_name_type) {
- this.default_name_type = default_name_type;
- return this;
- }
-
- public void unsetDefault_name_type() {
- this.default_name_type = null;
- }
-
- /** Returns true if field default_name_type is set (has been assigned a value) and false otherwise */
- public boolean isSetDefault_name_type() {
- return this.default_name_type != null;
- }
-
- public void setDefault_name_typeIsSet(boolean value) {
- if (!value) {
- this.default_name_type = null;
- }
- }
-
- public String getDefault_value_type() {
- return this.default_value_type;
- }
-
- public CqlMetadata setDefault_value_type(String default_value_type) {
- this.default_value_type = default_value_type;
- return this;
- }
-
- public void unsetDefault_value_type() {
- this.default_value_type = null;
- }
-
- /** Returns true if field default_value_type is set (has been assigned a value) and false otherwise */
- public boolean isSetDefault_value_type() {
- return this.default_value_type != null;
- }
-
- public void setDefault_value_typeIsSet(boolean value) {
- if (!value) {
- this.default_value_type = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case NAME_TYPES:
- if (value == null) {
- unsetName_types();
- } else {
- setName_types((Map<ByteBuffer,String>)value);
- }
- break;
-
- case VALUE_TYPES:
- if (value == null) {
- unsetValue_types();
- } else {
- setValue_types((Map<ByteBuffer,String>)value);
- }
- break;
-
- case DEFAULT_NAME_TYPE:
- if (value == null) {
- unsetDefault_name_type();
- } else {
- setDefault_name_type((String)value);
- }
- break;
-
- case DEFAULT_VALUE_TYPE:
- if (value == null) {
- unsetDefault_value_type();
- } else {
- setDefault_value_type((String)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case NAME_TYPES:
- return getName_types();
-
- case VALUE_TYPES:
- return getValue_types();
-
- case DEFAULT_NAME_TYPE:
- return getDefault_name_type();
-
- case DEFAULT_VALUE_TYPE:
- return getDefault_value_type();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case NAME_TYPES:
- return isSetName_types();
- case VALUE_TYPES:
- return isSetValue_types();
- case DEFAULT_NAME_TYPE:
- return isSetDefault_name_type();
- case DEFAULT_VALUE_TYPE:
- return isSetDefault_value_type();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof CqlMetadata)
- return this.equals((CqlMetadata)that);
- return false;
- }
-
- public boolean equals(CqlMetadata that) {
- if (that == null)
- return false;
-
- boolean this_present_name_types = true && this.isSetName_types();
- boolean that_present_name_types = true && that.isSetName_types();
- if (this_present_name_types || that_present_name_types) {
- if (!(this_present_name_types && that_present_name_types))
- return false;
- if (!this.name_types.equals(that.name_types))
- return false;
- }
-
- boolean this_present_value_types = true && this.isSetValue_types();
- boolean that_present_value_types = true && that.isSetValue_types();
- if (this_present_value_types || that_present_value_types) {
- if (!(this_present_value_types && that_present_value_types))
- return false;
- if (!this.value_types.equals(that.value_types))
- return false;
- }
-
- boolean this_present_default_name_type = true && this.isSetDefault_name_type();
- boolean that_present_default_name_type = true && that.isSetDefault_name_type();
- if (this_present_default_name_type || that_present_default_name_type) {
- if (!(this_present_default_name_type && that_present_default_name_type))
- return false;
- if (!this.default_name_type.equals(that.default_name_type))
- return false;
- }
-
- boolean this_present_default_value_type = true && this.isSetDefault_value_type();
- boolean that_present_default_value_type = true && that.isSetDefault_value_type();
- if (this_present_default_value_type || that_present_default_value_type) {
- if (!(this_present_default_value_type && that_present_default_value_type))
- return false;
- if (!this.default_value_type.equals(that.default_value_type))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_name_types = true && (isSetName_types());
- builder.append(present_name_types);
- if (present_name_types)
- builder.append(name_types);
-
- boolean present_value_types = true && (isSetValue_types());
- builder.append(present_value_types);
- if (present_value_types)
- builder.append(value_types);
-
- boolean present_default_name_type = true && (isSetDefault_name_type());
- builder.append(present_default_name_type);
- if (present_default_name_type)
- builder.append(default_name_type);
-
- boolean present_default_value_type = true && (isSetDefault_value_type());
- builder.append(present_default_value_type);
- if (present_default_value_type)
- builder.append(default_value_type);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(CqlMetadata other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetName_types()).compareTo(other.isSetName_types());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetName_types()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.name_types, other.name_types);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetValue_types()).compareTo(other.isSetValue_types());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetValue_types()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.value_types, other.value_types);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetDefault_name_type()).compareTo(other.isSetDefault_name_type());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetDefault_name_type()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.default_name_type, other.default_name_type);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetDefault_value_type()).compareTo(other.isSetDefault_value_type());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetDefault_value_type()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.default_value_type, other.default_value_type);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("CqlMetadata(");
- boolean first = true;
-
- sb.append("name_types:");
- if (this.name_types == null) {
- sb.append("null");
- } else {
- sb.append(this.name_types);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("value_types:");
- if (this.value_types == null) {
- sb.append("null");
- } else {
- sb.append(this.value_types);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("default_name_type:");
- if (this.default_name_type == null) {
- sb.append("null");
- } else {
- sb.append(this.default_name_type);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("default_value_type:");
- if (this.default_value_type == null) {
- sb.append("null");
- } else {
- sb.append(this.default_value_type);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- if (name_types == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'name_types' was not present! Struct: " + toString());
- }
- if (value_types == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'value_types' was not present! Struct: " + toString());
- }
- if (default_name_type == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'default_name_type' was not present! Struct: " + toString());
- }
- if (default_value_type == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'default_value_type' was not present! Struct: " + toString());
- }
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class CqlMetadataStandardSchemeFactory implements SchemeFactory {
- public CqlMetadataStandardScheme getScheme() {
- return new CqlMetadataStandardScheme();
- }
- }
-
- private static class CqlMetadataStandardScheme extends StandardScheme<CqlMetadata> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, CqlMetadata struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // NAME_TYPES
- if (schemeField.type == org.apache.thrift.protocol.TType.MAP) {
- {
- org.apache.thrift.protocol.TMap _map172 = iprot.readMapBegin();
- struct.name_types = new HashMap<ByteBuffer,String>(2*_map172.size);
- for (int _i173 = 0; _i173 < _map172.size; ++_i173)
- {
- ByteBuffer _key174;
- String _val175;
- _key174 = iprot.readBinary();
- _val175 = iprot.readString();
- struct.name_types.put(_key174, _val175);
- }
- iprot.readMapEnd();
- }
- struct.setName_typesIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 2: // VALUE_TYPES
- if (schemeField.type == org.apache.thrift.protocol.TType.MAP) {
- {
- org.apache.thrift.protocol.TMap _map176 = iprot.readMapBegin();
- struct.value_types = new HashMap<ByteBuffer,String>(2*_map176.size);
- for (int _i177 = 0; _i177 < _map176.size; ++_i177)
- {
- ByteBuffer _key178;
- String _val179;
- _key178 = iprot.readBinary();
- _val179 = iprot.readString();
- struct.value_types.put(_key178, _val179);
- }
- iprot.readMapEnd();
- }
- struct.setValue_typesIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 3: // DEFAULT_NAME_TYPE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.default_name_type = iprot.readString();
- struct.setDefault_name_typeIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 4: // DEFAULT_VALUE_TYPE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.default_value_type = iprot.readString();
- struct.setDefault_value_typeIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, CqlMetadata struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.name_types != null) {
- oprot.writeFieldBegin(NAME_TYPES_FIELD_DESC);
- {
- oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, struct.name_types.size()));
- for (Map.Entry<ByteBuffer, String> _iter180 : struct.name_types.entrySet())
- {
- oprot.writeBinary(_iter180.getKey());
- oprot.writeString(_iter180.getValue());
- }
- oprot.writeMapEnd();
- }
- oprot.writeFieldEnd();
- }
- if (struct.value_types != null) {
- oprot.writeFieldBegin(VALUE_TYPES_FIELD_DESC);
- {
- oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, struct.value_types.size()));
- for (Map.Entry<ByteBuffer, String> _iter181 : struct.value_types.entrySet())
- {
- oprot.writeBinary(_iter181.getKey());
- oprot.writeString(_iter181.getValue());
- }
- oprot.writeMapEnd();
- }
- oprot.writeFieldEnd();
- }
- if (struct.default_name_type != null) {
- oprot.writeFieldBegin(DEFAULT_NAME_TYPE_FIELD_DESC);
- oprot.writeString(struct.default_name_type);
- oprot.writeFieldEnd();
- }
- if (struct.default_value_type != null) {
- oprot.writeFieldBegin(DEFAULT_VALUE_TYPE_FIELD_DESC);
- oprot.writeString(struct.default_value_type);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class CqlMetadataTupleSchemeFactory implements SchemeFactory {
- public CqlMetadataTupleScheme getScheme() {
- return new CqlMetadataTupleScheme();
- }
- }
-
- private static class CqlMetadataTupleScheme extends TupleScheme<CqlMetadata> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, CqlMetadata struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- {
- oprot.writeI32(struct.name_types.size());
- for (Map.Entry<ByteBuffer, String> _iter182 : struct.name_types.entrySet())
- {
- oprot.writeBinary(_iter182.getKey());
- oprot.writeString(_iter182.getValue());
- }
- }
- {
- oprot.writeI32(struct.value_types.size());
- for (Map.Entry<ByteBuffer, String> _iter183 : struct.value_types.entrySet())
- {
- oprot.writeBinary(_iter183.getKey());
- oprot.writeString(_iter183.getValue());
- }
- }
- oprot.writeString(struct.default_name_type);
- oprot.writeString(struct.default_value_type);
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, CqlMetadata struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- {
- org.apache.thrift.protocol.TMap _map184 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32());
- struct.name_types = new HashMap<ByteBuffer,String>(2*_map184.size);
- for (int _i185 = 0; _i185 < _map184.size; ++_i185)
- {
- ByteBuffer _key186;
- String _val187;
- _key186 = iprot.readBinary();
- _val187 = iprot.readString();
- struct.name_types.put(_key186, _val187);
- }
- }
- struct.setName_typesIsSet(true);
- {
- org.apache.thrift.protocol.TMap _map188 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32());
- struct.value_types = new HashMap<ByteBuffer,String>(2*_map188.size);
- for (int _i189 = 0; _i189 < _map188.size; ++_i189)
- {
- ByteBuffer _key190;
- String _val191;
- _key190 = iprot.readBinary();
- _val191 = iprot.readString();
- struct.value_types.put(_key190, _val191);
- }
- }
- struct.setValue_typesIsSet(true);
- struct.default_name_type = iprot.readString();
- struct.setDefault_name_typeIsSet(true);
- struct.default_value_type = iprot.readString();
- struct.setDefault_value_typeIsSet(true);
- }
- }
-
-}
-
diff --git a/interface/thrift/gen-java/org/apache/cassandra/thrift/CqlPreparedResult.java b/interface/thrift/gen-java/org/apache/cassandra/thrift/CqlPreparedResult.java
deleted file mode 100644
index b720a2d..0000000
--- a/interface/thrift/gen-java/org/apache/cassandra/thrift/CqlPreparedResult.java
+++ /dev/null
@@ -1,821 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.1)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- * @generated
- */
-package org.apache.cassandra.thrift;
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
-
-import org.apache.commons.lang3.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class CqlPreparedResult implements org.apache.thrift.TBase<CqlPreparedResult, CqlPreparedResult._Fields>, java.io.Serializable, Cloneable, Comparable<CqlPreparedResult> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("CqlPreparedResult");
-
- private static final org.apache.thrift.protocol.TField ITEM_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("itemId", org.apache.thrift.protocol.TType.I32, (short)1);
- private static final org.apache.thrift.protocol.TField COUNT_FIELD_DESC = new org.apache.thrift.protocol.TField("count", org.apache.thrift.protocol.TType.I32, (short)2);
- private static final org.apache.thrift.protocol.TField VARIABLE_TYPES_FIELD_DESC = new org.apache.thrift.protocol.TField("variable_types", org.apache.thrift.protocol.TType.LIST, (short)3);
- private static final org.apache.thrift.protocol.TField VARIABLE_NAMES_FIELD_DESC = new org.apache.thrift.protocol.TField("variable_names", org.apache.thrift.protocol.TType.LIST, (short)4);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new CqlPreparedResultStandardSchemeFactory());
- schemes.put(TupleScheme.class, new CqlPreparedResultTupleSchemeFactory());
- }
-
- public int itemId; // required
- public int count; // required
- public List<String> variable_types; // optional
- public List<String> variable_names; // optional
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- ITEM_ID((short)1, "itemId"),
- COUNT((short)2, "count"),
- VARIABLE_TYPES((short)3, "variable_types"),
- VARIABLE_NAMES((short)4, "variable_names");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // ITEM_ID
- return ITEM_ID;
- case 2: // COUNT
- return COUNT;
- case 3: // VARIABLE_TYPES
- return VARIABLE_TYPES;
- case 4: // VARIABLE_NAMES
- return VARIABLE_NAMES;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- private static final int __ITEMID_ISSET_ID = 0;
- private static final int __COUNT_ISSET_ID = 1;
- private byte __isset_bitfield = 0;
- private _Fields optionals[] = {_Fields.VARIABLE_TYPES,_Fields.VARIABLE_NAMES};
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.ITEM_ID, new org.apache.thrift.meta_data.FieldMetaData("itemId", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32)));
- tmpMap.put(_Fields.COUNT, new org.apache.thrift.meta_data.FieldMetaData("count", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32)));
- tmpMap.put(_Fields.VARIABLE_TYPES, new org.apache.thrift.meta_data.FieldMetaData("variable_types", org.apache.thrift.TFieldRequirementType.OPTIONAL,
- new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))));
- tmpMap.put(_Fields.VARIABLE_NAMES, new org.apache.thrift.meta_data.FieldMetaData("variable_names", org.apache.thrift.TFieldRequirementType.OPTIONAL,
- new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(CqlPreparedResult.class, metaDataMap);
- }
-
- public CqlPreparedResult() {
- }
-
- public CqlPreparedResult(
- int itemId,
- int count)
- {
- this();
- this.itemId = itemId;
- setItemIdIsSet(true);
- this.count = count;
- setCountIsSet(true);
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public CqlPreparedResult(CqlPreparedResult other) {
- __isset_bitfield = other.__isset_bitfield;
- this.itemId = other.itemId;
- this.count = other.count;
- if (other.isSetVariable_types()) {
- List<String> __this__variable_types = new ArrayList<String>(other.variable_types);
- this.variable_types = __this__variable_types;
- }
- if (other.isSetVariable_names()) {
- List<String> __this__variable_names = new ArrayList<String>(other.variable_names);
- this.variable_names = __this__variable_names;
- }
- }
-
- public CqlPreparedResult deepCopy() {
- return new CqlPreparedResult(this);
- }
-
- @Override
- public void clear() {
- setItemIdIsSet(false);
- this.itemId = 0;
- setCountIsSet(false);
- this.count = 0;
- this.variable_types = null;
- this.variable_names = null;
- }
-
- public int getItemId() {
- return this.itemId;
- }
-
- public CqlPreparedResult setItemId(int itemId) {
- this.itemId = itemId;
- setItemIdIsSet(true);
- return this;
- }
-
- public void unsetItemId() {
- __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __ITEMID_ISSET_ID);
- }
-
- /** Returns true if field itemId is set (has been assigned a value) and false otherwise */
- public boolean isSetItemId() {
- return EncodingUtils.testBit(__isset_bitfield, __ITEMID_ISSET_ID);
- }
-
- public void setItemIdIsSet(boolean value) {
- __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __ITEMID_ISSET_ID, value);
- }
-
- public int getCount() {
- return this.count;
- }
-
- public CqlPreparedResult setCount(int count) {
- this.count = count;
- setCountIsSet(true);
- return this;
- }
-
- public void unsetCount() {
- __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __COUNT_ISSET_ID);
- }
-
- /** Returns true if field count is set (has been assigned a value) and false otherwise */
- public boolean isSetCount() {
- return EncodingUtils.testBit(__isset_bitfield, __COUNT_ISSET_ID);
- }
-
- public void setCountIsSet(boolean value) {
- __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __COUNT_ISSET_ID, value);
- }
-
- public int getVariable_typesSize() {
- return (this.variable_types == null) ? 0 : this.variable_types.size();
- }
-
- public java.util.Iterator<String> getVariable_typesIterator() {
- return (this.variable_types == null) ? null : this.variable_types.iterator();
- }
-
- public void addToVariable_types(String elem) {
- if (this.variable_types == null) {
- this.variable_types = new ArrayList<String>();
- }
- this.variable_types.add(elem);
- }
-
- public List<String> getVariable_types() {
- return this.variable_types;
- }
-
- public CqlPreparedResult setVariable_types(List<String> variable_types) {
- this.variable_types = variable_types;
- return this;
- }
-
- public void unsetVariable_types() {
- this.variable_types = null;
- }
-
- /** Returns true if field variable_types is set (has been assigned a value) and false otherwise */
- public boolean isSetVariable_types() {
- return this.variable_types != null;
- }
-
- public void setVariable_typesIsSet(boolean value) {
- if (!value) {
- this.variable_types = null;
- }
- }
-
- public int getVariable_namesSize() {
- return (this.variable_names == null) ? 0 : this.variable_names.size();
- }
-
- public java.util.Iterator<String> getVariable_namesIterator() {
- return (this.variable_names == null) ? null : this.variable_names.iterator();
- }
-
- public void addToVariable_names(String elem) {
- if (this.variable_names == null) {
- this.variable_names = new ArrayList<String>();
- }
- this.variable_names.add(elem);
- }
-
- public List<String> getVariable_names() {
- return this.variable_names;
- }
-
- public CqlPreparedResult setVariable_names(List<String> variable_names) {
- this.variable_names = variable_names;
- return this;
- }
-
- public void unsetVariable_names() {
- this.variable_names = null;
- }
-
- /** Returns true if field variable_names is set (has been assigned a value) and false otherwise */
- public boolean isSetVariable_names() {
- return this.variable_names != null;
- }
-
- public void setVariable_namesIsSet(boolean value) {
- if (!value) {
- this.variable_names = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case ITEM_ID:
- if (value == null) {
- unsetItemId();
- } else {
- setItemId((Integer)value);
- }
- break;
-
- case COUNT:
- if (value == null) {
- unsetCount();
- } else {
- setCount((Integer)value);
- }
- break;
-
- case VARIABLE_TYPES:
- if (value == null) {
- unsetVariable_types();
- } else {
- setVariable_types((List<String>)value);
- }
- break;
-
- case VARIABLE_NAMES:
- if (value == null) {
- unsetVariable_names();
- } else {
- setVariable_names((List<String>)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case ITEM_ID:
- return Integer.valueOf(getItemId());
-
- case COUNT:
- return Integer.valueOf(getCount());
-
- case VARIABLE_TYPES:
- return getVariable_types();
-
- case VARIABLE_NAMES:
- return getVariable_names();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case ITEM_ID:
- return isSetItemId();
- case COUNT:
- return isSetCount();
- case VARIABLE_TYPES:
- return isSetVariable_types();
- case VARIABLE_NAMES:
- return isSetVariable_names();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof CqlPreparedResult)
- return this.equals((CqlPreparedResult)that);
- return false;
- }
-
- public boolean equals(CqlPreparedResult that) {
- if (that == null)
- return false;
-
- boolean this_present_itemId = true;
- boolean that_present_itemId = true;
- if (this_present_itemId || that_present_itemId) {
- if (!(this_present_itemId && that_present_itemId))
- return false;
- if (this.itemId != that.itemId)
- return false;
- }
-
- boolean this_present_count = true;
- boolean that_present_count = true;
- if (this_present_count || that_present_count) {
- if (!(this_present_count && that_present_count))
- return false;
- if (this.count != that.count)
- return false;
- }
-
- boolean this_present_variable_types = true && this.isSetVariable_types();
- boolean that_present_variable_types = true && that.isSetVariable_types();
- if (this_present_variable_types || that_present_variable_types) {
- if (!(this_present_variable_types && that_present_variable_types))
- return false;
- if (!this.variable_types.equals(that.variable_types))
- return false;
- }
-
- boolean this_present_variable_names = true && this.isSetVariable_names();
- boolean that_present_variable_names = true && that.isSetVariable_names();
- if (this_present_variable_names || that_present_variable_names) {
- if (!(this_present_variable_names && that_present_variable_names))
- return false;
- if (!this.variable_names.equals(that.variable_names))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_itemId = true;
- builder.append(present_itemId);
- if (present_itemId)
- builder.append(itemId);
-
- boolean present_count = true;
- builder.append(present_count);
- if (present_count)
- builder.append(count);
-
- boolean present_variable_types = true && (isSetVariable_types());
- builder.append(present_variable_types);
- if (present_variable_types)
- builder.append(variable_types);
-
- boolean present_variable_names = true && (isSetVariable_names());
- builder.append(present_variable_names);
- if (present_variable_names)
- builder.append(variable_names);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(CqlPreparedResult other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetItemId()).compareTo(other.isSetItemId());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetItemId()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.itemId, other.itemId);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetCount()).compareTo(other.isSetCount());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetCount()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.count, other.count);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetVariable_types()).compareTo(other.isSetVariable_types());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetVariable_types()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.variable_types, other.variable_types);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetVariable_names()).compareTo(other.isSetVariable_names());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetVariable_names()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.variable_names, other.variable_names);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("CqlPreparedResult(");
- boolean first = true;
-
- sb.append("itemId:");
- sb.append(this.itemId);
- first = false;
- if (!first) sb.append(", ");
- sb.append("count:");
- sb.append(this.count);
- first = false;
- if (isSetVariable_types()) {
- if (!first) sb.append(", ");
- sb.append("variable_types:");
- if (this.variable_types == null) {
- sb.append("null");
- } else {
- sb.append(this.variable_types);
- }
- first = false;
- }
- if (isSetVariable_names()) {
- if (!first) sb.append(", ");
- sb.append("variable_names:");
- if (this.variable_names == null) {
- sb.append("null");
- } else {
- sb.append(this.variable_names);
- }
- first = false;
- }
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // alas, we cannot check 'itemId' because it's a primitive and you chose the non-beans generator.
- // alas, we cannot check 'count' because it's a primitive and you chose the non-beans generator.
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
- __isset_bitfield = 0;
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class CqlPreparedResultStandardSchemeFactory implements SchemeFactory {
- public CqlPreparedResultStandardScheme getScheme() {
- return new CqlPreparedResultStandardScheme();
- }
- }
-
- private static class CqlPreparedResultStandardScheme extends StandardScheme<CqlPreparedResult> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, CqlPreparedResult struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // ITEM_ID
- if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
- struct.itemId = iprot.readI32();
- struct.setItemIdIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 2: // COUNT
- if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
- struct.count = iprot.readI32();
- struct.setCountIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 3: // VARIABLE_TYPES
- if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
- {
- org.apache.thrift.protocol.TList _list200 = iprot.readListBegin();
- struct.variable_types = new ArrayList<String>(_list200.size);
- for (int _i201 = 0; _i201 < _list200.size; ++_i201)
- {
- String _elem202;
- _elem202 = iprot.readString();
- struct.variable_types.add(_elem202);
- }
- iprot.readListEnd();
- }
- struct.setVariable_typesIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 4: // VARIABLE_NAMES
- if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
- {
- org.apache.thrift.protocol.TList _list203 = iprot.readListBegin();
- struct.variable_names = new ArrayList<String>(_list203.size);
- for (int _i204 = 0; _i204 < _list203.size; ++_i204)
- {
- String _elem205;
- _elem205 = iprot.readString();
- struct.variable_names.add(_elem205);
- }
- iprot.readListEnd();
- }
- struct.setVariable_namesIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- if (!struct.isSetItemId()) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'itemId' was not found in serialized data! Struct: " + toString());
- }
- if (!struct.isSetCount()) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'count' was not found in serialized data! Struct: " + toString());
- }
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, CqlPreparedResult struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- oprot.writeFieldBegin(ITEM_ID_FIELD_DESC);
- oprot.writeI32(struct.itemId);
- oprot.writeFieldEnd();
- oprot.writeFieldBegin(COUNT_FIELD_DESC);
- oprot.writeI32(struct.count);
- oprot.writeFieldEnd();
- if (struct.variable_types != null) {
- if (struct.isSetVariable_types()) {
- oprot.writeFieldBegin(VARIABLE_TYPES_FIELD_DESC);
- {
- oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, struct.variable_types.size()));
- for (String _iter206 : struct.variable_types)
- {
- oprot.writeString(_iter206);
- }
- oprot.writeListEnd();
- }
- oprot.writeFieldEnd();
- }
- }
- if (struct.variable_names != null) {
- if (struct.isSetVariable_names()) {
- oprot.writeFieldBegin(VARIABLE_NAMES_FIELD_DESC);
- {
- oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, struct.variable_names.size()));
- for (String _iter207 : struct.variable_names)
- {
- oprot.writeString(_iter207);
- }
- oprot.writeListEnd();
- }
- oprot.writeFieldEnd();
- }
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class CqlPreparedResultTupleSchemeFactory implements SchemeFactory {
- public CqlPreparedResultTupleScheme getScheme() {
- return new CqlPreparedResultTupleScheme();
- }
- }
-
- private static class CqlPreparedResultTupleScheme extends TupleScheme<CqlPreparedResult> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, CqlPreparedResult struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- oprot.writeI32(struct.itemId);
- oprot.writeI32(struct.count);
- BitSet optionals = new BitSet();
- if (struct.isSetVariable_types()) {
- optionals.set(0);
- }
- if (struct.isSetVariable_names()) {
- optionals.set(1);
- }
- oprot.writeBitSet(optionals, 2);
- if (struct.isSetVariable_types()) {
- {
- oprot.writeI32(struct.variable_types.size());
- for (String _iter208 : struct.variable_types)
- {
- oprot.writeString(_iter208);
- }
- }
- }
- if (struct.isSetVariable_names()) {
- {
- oprot.writeI32(struct.variable_names.size());
- for (String _iter209 : struct.variable_names)
- {
- oprot.writeString(_iter209);
- }
- }
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, CqlPreparedResult struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- struct.itemId = iprot.readI32();
- struct.setItemIdIsSet(true);
- struct.count = iprot.readI32();
- struct.setCountIsSet(true);
- BitSet incoming = iprot.readBitSet(2);
- if (incoming.get(0)) {
- {
- org.apache.thrift.protocol.TList _list210 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32());
- struct.variable_types = new ArrayList<String>(_list210.size);
- for (int _i211 = 0; _i211 < _list210.size; ++_i211)
- {
- String _elem212;
- _elem212 = iprot.readString();
- struct.variable_types.add(_elem212);
- }
- }
- struct.setVariable_typesIsSet(true);
- }
- if (incoming.get(1)) {
- {
- org.apache.thrift.protocol.TList _list213 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32());
- struct.variable_names = new ArrayList<String>(_list213.size);
- for (int _i214 = 0; _i214 < _list213.size; ++_i214)
- {
- String _elem215;
- _elem215 = iprot.readString();
- struct.variable_names.add(_elem215);
- }
- }
- struct.setVariable_namesIsSet(true);
- }
- }
- }
-
-}
-
diff --git a/interface/thrift/gen-java/org/apache/cassandra/thrift/CqlResult.java b/interface/thrift/gen-java/org/apache/cassandra/thrift/CqlResult.java
deleted file mode 100644
index 1cdbe07..0000000
--- a/interface/thrift/gen-java/org/apache/cassandra/thrift/CqlResult.java
+++ /dev/null
@@ -1,807 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.1)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- * @generated
- */
-package org.apache.cassandra.thrift;
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
-
-import org.apache.commons.lang3.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class CqlResult implements org.apache.thrift.TBase<CqlResult, CqlResult._Fields>, java.io.Serializable, Cloneable, Comparable<CqlResult> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("CqlResult");
-
- private static final org.apache.thrift.protocol.TField TYPE_FIELD_DESC = new org.apache.thrift.protocol.TField("type", org.apache.thrift.protocol.TType.I32, (short)1);
- private static final org.apache.thrift.protocol.TField ROWS_FIELD_DESC = new org.apache.thrift.protocol.TField("rows", org.apache.thrift.protocol.TType.LIST, (short)2);
- private static final org.apache.thrift.protocol.TField NUM_FIELD_DESC = new org.apache.thrift.protocol.TField("num", org.apache.thrift.protocol.TType.I32, (short)3);
- private static final org.apache.thrift.protocol.TField SCHEMA_FIELD_DESC = new org.apache.thrift.protocol.TField("schema", org.apache.thrift.protocol.TType.STRUCT, (short)4);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new CqlResultStandardSchemeFactory());
- schemes.put(TupleScheme.class, new CqlResultTupleSchemeFactory());
- }
-
- /**
- *
- * @see CqlResultType
- */
- public CqlResultType type; // required
- public List<CqlRow> rows; // optional
- public int num; // optional
- public CqlMetadata schema; // optional
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- /**
- *
- * @see CqlResultType
- */
- TYPE((short)1, "type"),
- ROWS((short)2, "rows"),
- NUM((short)3, "num"),
- SCHEMA((short)4, "schema");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // TYPE
- return TYPE;
- case 2: // ROWS
- return ROWS;
- case 3: // NUM
- return NUM;
- case 4: // SCHEMA
- return SCHEMA;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- private static final int __NUM_ISSET_ID = 0;
- private byte __isset_bitfield = 0;
- private _Fields optionals[] = {_Fields.ROWS,_Fields.NUM,_Fields.SCHEMA};
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.TYPE, new org.apache.thrift.meta_data.FieldMetaData("type", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.EnumMetaData(org.apache.thrift.protocol.TType.ENUM, CqlResultType.class)));
- tmpMap.put(_Fields.ROWS, new org.apache.thrift.meta_data.FieldMetaData("rows", org.apache.thrift.TFieldRequirementType.OPTIONAL,
- new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST,
- new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, CqlRow.class))));
- tmpMap.put(_Fields.NUM, new org.apache.thrift.meta_data.FieldMetaData("num", org.apache.thrift.TFieldRequirementType.OPTIONAL,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32)));
- tmpMap.put(_Fields.SCHEMA, new org.apache.thrift.meta_data.FieldMetaData("schema", org.apache.thrift.TFieldRequirementType.OPTIONAL,
- new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, CqlMetadata.class)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(CqlResult.class, metaDataMap);
- }
-
- public CqlResult() {
- }
-
- public CqlResult(
- CqlResultType type)
- {
- this();
- this.type = type;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public CqlResult(CqlResult other) {
- __isset_bitfield = other.__isset_bitfield;
- if (other.isSetType()) {
- this.type = other.type;
- }
- if (other.isSetRows()) {
- List<CqlRow> __this__rows = new ArrayList<CqlRow>(other.rows.size());
- for (CqlRow other_element : other.rows) {
- __this__rows.add(new CqlRow(other_element));
- }
- this.rows = __this__rows;
- }
- this.num = other.num;
- if (other.isSetSchema()) {
- this.schema = new CqlMetadata(other.schema);
- }
- }
-
- public CqlResult deepCopy() {
- return new CqlResult(this);
- }
-
- @Override
- public void clear() {
- this.type = null;
- this.rows = null;
- setNumIsSet(false);
- this.num = 0;
- this.schema = null;
- }
-
- /**
- *
- * @see CqlResultType
- */
- public CqlResultType getType() {
- return this.type;
- }
-
- /**
- *
- * @see CqlResultType
- */
- public CqlResult setType(CqlResultType type) {
- this.type = type;
- return this;
- }
-
- public void unsetType() {
- this.type = null;
- }
-
- /** Returns true if field type is set (has been assigned a value) and false otherwise */
- public boolean isSetType() {
- return this.type != null;
- }
-
- public void setTypeIsSet(boolean value) {
- if (!value) {
- this.type = null;
- }
- }
-
- public int getRowsSize() {
- return (this.rows == null) ? 0 : this.rows.size();
- }
-
- public java.util.Iterator<CqlRow> getRowsIterator() {
- return (this.rows == null) ? null : this.rows.iterator();
- }
-
- public void addToRows(CqlRow elem) {
- if (this.rows == null) {
- this.rows = new ArrayList<CqlRow>();
- }
- this.rows.add(elem);
- }
-
- public List<CqlRow> getRows() {
- return this.rows;
- }
-
- public CqlResult setRows(List<CqlRow> rows) {
- this.rows = rows;
- return this;
- }
-
- public void unsetRows() {
- this.rows = null;
- }
-
- /** Returns true if field rows is set (has been assigned a value) and false otherwise */
- public boolean isSetRows() {
- return this.rows != null;
- }
-
- public void setRowsIsSet(boolean value) {
- if (!value) {
- this.rows = null;
- }
- }
-
- public int getNum() {
- return this.num;
- }
-
- public CqlResult setNum(int num) {
- this.num = num;
- setNumIsSet(true);
- return this;
- }
-
- public void unsetNum() {
- __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __NUM_ISSET_ID);
- }
-
- /** Returns true if field num is set (has been assigned a value) and false otherwise */
- public boolean isSetNum() {
- return EncodingUtils.testBit(__isset_bitfield, __NUM_ISSET_ID);
- }
-
- public void setNumIsSet(boolean value) {
- __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __NUM_ISSET_ID, value);
- }
-
- public CqlMetadata getSchema() {
- return this.schema;
- }
-
- public CqlResult setSchema(CqlMetadata schema) {
- this.schema = schema;
- return this;
- }
-
- public void unsetSchema() {
- this.schema = null;
- }
-
- /** Returns true if field schema is set (has been assigned a value) and false otherwise */
- public boolean isSetSchema() {
- return this.schema != null;
- }
-
- public void setSchemaIsSet(boolean value) {
- if (!value) {
- this.schema = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case TYPE:
- if (value == null) {
- unsetType();
- } else {
- setType((CqlResultType)value);
- }
- break;
-
- case ROWS:
- if (value == null) {
- unsetRows();
- } else {
- setRows((List<CqlRow>)value);
- }
- break;
-
- case NUM:
- if (value == null) {
- unsetNum();
- } else {
- setNum((Integer)value);
- }
- break;
-
- case SCHEMA:
- if (value == null) {
- unsetSchema();
- } else {
- setSchema((CqlMetadata)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case TYPE:
- return getType();
-
- case ROWS:
- return getRows();
-
- case NUM:
- return Integer.valueOf(getNum());
-
- case SCHEMA:
- return getSchema();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case TYPE:
- return isSetType();
- case ROWS:
- return isSetRows();
- case NUM:
- return isSetNum();
- case SCHEMA:
- return isSetSchema();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof CqlResult)
- return this.equals((CqlResult)that);
- return false;
- }
-
- public boolean equals(CqlResult that) {
- if (that == null)
- return false;
-
- boolean this_present_type = true && this.isSetType();
- boolean that_present_type = true && that.isSetType();
- if (this_present_type || that_present_type) {
- if (!(this_present_type && that_present_type))
- return false;
- if (!this.type.equals(that.type))
- return false;
- }
-
- boolean this_present_rows = true && this.isSetRows();
- boolean that_present_rows = true && that.isSetRows();
- if (this_present_rows || that_present_rows) {
- if (!(this_present_rows && that_present_rows))
- return false;
- if (!this.rows.equals(that.rows))
- return false;
- }
-
- boolean this_present_num = true && this.isSetNum();
- boolean that_present_num = true && that.isSetNum();
- if (this_present_num || that_present_num) {
- if (!(this_present_num && that_present_num))
- return false;
- if (this.num != that.num)
- return false;
- }
-
- boolean this_present_schema = true && this.isSetSchema();
- boolean that_present_schema = true && that.isSetSchema();
- if (this_present_schema || that_present_schema) {
- if (!(this_present_schema && that_present_schema))
- return false;
- if (!this.schema.equals(that.schema))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_type = true && (isSetType());
- builder.append(present_type);
- if (present_type)
- builder.append(type.getValue());
-
- boolean present_rows = true && (isSetRows());
- builder.append(present_rows);
- if (present_rows)
- builder.append(rows);
-
- boolean present_num = true && (isSetNum());
- builder.append(present_num);
- if (present_num)
- builder.append(num);
-
- boolean present_schema = true && (isSetSchema());
- builder.append(present_schema);
- if (present_schema)
- builder.append(schema);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(CqlResult other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetType()).compareTo(other.isSetType());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetType()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.type, other.type);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetRows()).compareTo(other.isSetRows());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetRows()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.rows, other.rows);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetNum()).compareTo(other.isSetNum());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetNum()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.num, other.num);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetSchema()).compareTo(other.isSetSchema());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetSchema()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.schema, other.schema);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("CqlResult(");
- boolean first = true;
-
- sb.append("type:");
- if (this.type == null) {
- sb.append("null");
- } else {
- sb.append(this.type);
- }
- first = false;
- if (isSetRows()) {
- if (!first) sb.append(", ");
- sb.append("rows:");
- if (this.rows == null) {
- sb.append("null");
- } else {
- sb.append(this.rows);
- }
- first = false;
- }
- if (isSetNum()) {
- if (!first) sb.append(", ");
- sb.append("num:");
- sb.append(this.num);
- first = false;
- }
- if (isSetSchema()) {
- if (!first) sb.append(", ");
- sb.append("schema:");
- if (this.schema == null) {
- sb.append("null");
- } else {
- sb.append(this.schema);
- }
- first = false;
- }
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- if (type == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'type' was not present! Struct: " + toString());
- }
- // check for sub-struct validity
- if (schema != null) {
- schema.validate();
- }
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
- __isset_bitfield = 0;
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class CqlResultStandardSchemeFactory implements SchemeFactory {
- public CqlResultStandardScheme getScheme() {
- return new CqlResultStandardScheme();
- }
- }
-
- private static class CqlResultStandardScheme extends StandardScheme<CqlResult> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, CqlResult struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // TYPE
- if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
- struct.type = CqlResultType.findByValue(iprot.readI32());
- struct.setTypeIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 2: // ROWS
- if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
- {
- org.apache.thrift.protocol.TList _list192 = iprot.readListBegin();
- struct.rows = new ArrayList<CqlRow>(_list192.size);
- for (int _i193 = 0; _i193 < _list192.size; ++_i193)
- {
- CqlRow _elem194;
- _elem194 = new CqlRow();
- _elem194.read(iprot);
- struct.rows.add(_elem194);
- }
- iprot.readListEnd();
- }
- struct.setRowsIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 3: // NUM
- if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
- struct.num = iprot.readI32();
- struct.setNumIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 4: // SCHEMA
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.schema = new CqlMetadata();
- struct.schema.read(iprot);
- struct.setSchemaIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, CqlResult struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.type != null) {
- oprot.writeFieldBegin(TYPE_FIELD_DESC);
- oprot.writeI32(struct.type.getValue());
- oprot.writeFieldEnd();
- }
- if (struct.rows != null) {
- if (struct.isSetRows()) {
- oprot.writeFieldBegin(ROWS_FIELD_DESC);
- {
- oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.rows.size()));
- for (CqlRow _iter195 : struct.rows)
- {
- _iter195.write(oprot);
- }
- oprot.writeListEnd();
- }
- oprot.writeFieldEnd();
- }
- }
- if (struct.isSetNum()) {
- oprot.writeFieldBegin(NUM_FIELD_DESC);
- oprot.writeI32(struct.num);
- oprot.writeFieldEnd();
- }
- if (struct.schema != null) {
- if (struct.isSetSchema()) {
- oprot.writeFieldBegin(SCHEMA_FIELD_DESC);
- struct.schema.write(oprot);
- oprot.writeFieldEnd();
- }
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class CqlResultTupleSchemeFactory implements SchemeFactory {
- public CqlResultTupleScheme getScheme() {
- return new CqlResultTupleScheme();
- }
- }
-
- private static class CqlResultTupleScheme extends TupleScheme<CqlResult> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, CqlResult struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- oprot.writeI32(struct.type.getValue());
- BitSet optionals = new BitSet();
- if (struct.isSetRows()) {
- optionals.set(0);
- }
- if (struct.isSetNum()) {
- optionals.set(1);
- }
- if (struct.isSetSchema()) {
- optionals.set(2);
- }
- oprot.writeBitSet(optionals, 3);
- if (struct.isSetRows()) {
- {
- oprot.writeI32(struct.rows.size());
- for (CqlRow _iter196 : struct.rows)
- {
- _iter196.write(oprot);
- }
- }
- }
- if (struct.isSetNum()) {
- oprot.writeI32(struct.num);
- }
- if (struct.isSetSchema()) {
- struct.schema.write(oprot);
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, CqlResult struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- struct.type = CqlResultType.findByValue(iprot.readI32());
- struct.setTypeIsSet(true);
- BitSet incoming = iprot.readBitSet(3);
- if (incoming.get(0)) {
- {
- org.apache.thrift.protocol.TList _list197 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
- struct.rows = new ArrayList<CqlRow>(_list197.size);
- for (int _i198 = 0; _i198 < _list197.size; ++_i198)
- {
- CqlRow _elem199;
- _elem199 = new CqlRow();
- _elem199.read(iprot);
- struct.rows.add(_elem199);
- }
- }
- struct.setRowsIsSet(true);
- }
- if (incoming.get(1)) {
- struct.num = iprot.readI32();
- struct.setNumIsSet(true);
- }
- if (incoming.get(2)) {
- struct.schema = new CqlMetadata();
- struct.schema.read(iprot);
- struct.setSchemaIsSet(true);
- }
- }
- }
-
-}
-
diff --git a/interface/thrift/gen-java/org/apache/cassandra/thrift/CqlResultType.java b/interface/thrift/gen-java/org/apache/cassandra/thrift/CqlResultType.java
deleted file mode 100644
index 2928f68..0000000
--- a/interface/thrift/gen-java/org/apache/cassandra/thrift/CqlResultType.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.1)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- * @generated
- */
-package org.apache.cassandra.thrift;
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
-
-
-import java.util.Map;
-import java.util.HashMap;
-import org.apache.thrift.TEnum;
-
-public enum CqlResultType implements org.apache.thrift.TEnum {
- ROWS(1),
- VOID(2),
- INT(3);
-
- private final int value;
-
- private CqlResultType(int value) {
- this.value = value;
- }
-
- /**
- * Get the integer value of this enum value, as defined in the Thrift IDL.
- */
- public int getValue() {
- return value;
- }
-
- /**
- * Find a the enum type by its integer value, as defined in the Thrift IDL.
- * @return null if the value is not found.
- */
- public static CqlResultType findByValue(int value) {
- switch (value) {
- case 1:
- return ROWS;
- case 2:
- return VOID;
- case 3:
- return INT;
- default:
- return null;
- }
- }
-}
diff --git a/interface/thrift/gen-java/org/apache/cassandra/thrift/CqlRow.java b/interface/thrift/gen-java/org/apache/cassandra/thrift/CqlRow.java
deleted file mode 100644
index 7487ed7..0000000
--- a/interface/thrift/gen-java/org/apache/cassandra/thrift/CqlRow.java
+++ /dev/null
@@ -1,584 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.1)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- * @generated
- */
-package org.apache.cassandra.thrift;
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
-
-import org.apache.commons.lang3.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Row returned from a CQL query.
- *
- * This struct is used for both CQL2 and CQL3 queries. For CQL2, the partition key
- * is special-cased and is always returned. For CQL3, it is not special cased;
- * it will be included in the columns list if it was included in the SELECT and
- * the key field is always null.
- */
-public class CqlRow implements org.apache.thrift.TBase<CqlRow, CqlRow._Fields>, java.io.Serializable, Cloneable, Comparable<CqlRow> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("CqlRow");
-
- private static final org.apache.thrift.protocol.TField KEY_FIELD_DESC = new org.apache.thrift.protocol.TField("key", org.apache.thrift.protocol.TType.STRING, (short)1);
- private static final org.apache.thrift.protocol.TField COLUMNS_FIELD_DESC = new org.apache.thrift.protocol.TField("columns", org.apache.thrift.protocol.TType.LIST, (short)2);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new CqlRowStandardSchemeFactory());
- schemes.put(TupleScheme.class, new CqlRowTupleSchemeFactory());
- }
-
- public ByteBuffer key; // required
- public List<Column> columns; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- KEY((short)1, "key"),
- COLUMNS((short)2, "columns");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // KEY
- return KEY;
- case 2: // COLUMNS
- return COLUMNS;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.KEY, new org.apache.thrift.meta_data.FieldMetaData("key", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true)));
- tmpMap.put(_Fields.COLUMNS, new org.apache.thrift.meta_data.FieldMetaData("columns", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST,
- new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, Column.class))));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(CqlRow.class, metaDataMap);
- }
-
- public CqlRow() {
- }
-
- public CqlRow(
- ByteBuffer key,
- List<Column> columns)
- {
- this();
- this.key = key;
- this.columns = columns;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public CqlRow(CqlRow other) {
- if (other.isSetKey()) {
- this.key = org.apache.thrift.TBaseHelper.copyBinary(other.key);
-;
- }
- if (other.isSetColumns()) {
- List<Column> __this__columns = new ArrayList<Column>(other.columns.size());
- for (Column other_element : other.columns) {
- __this__columns.add(new Column(other_element));
- }
- this.columns = __this__columns;
- }
- }
-
- public CqlRow deepCopy() {
- return new CqlRow(this);
- }
-
- @Override
- public void clear() {
- this.key = null;
- this.columns = null;
- }
-
- public byte[] getKey() {
- setKey(org.apache.thrift.TBaseHelper.rightSize(key));
- return key == null ? null : key.array();
- }
-
- public ByteBuffer bufferForKey() {
- return key;
- }
-
- public CqlRow setKey(byte[] key) {
- setKey(key == null ? (ByteBuffer)null : ByteBuffer.wrap(key));
- return this;
- }
-
- public CqlRow setKey(ByteBuffer key) {
- this.key = key;
- return this;
- }
-
- public void unsetKey() {
- this.key = null;
- }
-
- /** Returns true if field key is set (has been assigned a value) and false otherwise */
- public boolean isSetKey() {
- return this.key != null;
- }
-
- public void setKeyIsSet(boolean value) {
- if (!value) {
- this.key = null;
- }
- }
-
- public int getColumnsSize() {
- return (this.columns == null) ? 0 : this.columns.size();
- }
-
- public java.util.Iterator<Column> getColumnsIterator() {
- return (this.columns == null) ? null : this.columns.iterator();
- }
-
- public void addToColumns(Column elem) {
- if (this.columns == null) {
- this.columns = new ArrayList<Column>();
- }
- this.columns.add(elem);
- }
-
- public List<Column> getColumns() {
- return this.columns;
- }
-
- public CqlRow setColumns(List<Column> columns) {
- this.columns = columns;
- return this;
- }
-
- public void unsetColumns() {
- this.columns = null;
- }
-
- /** Returns true if field columns is set (has been assigned a value) and false otherwise */
- public boolean isSetColumns() {
- return this.columns != null;
- }
-
- public void setColumnsIsSet(boolean value) {
- if (!value) {
- this.columns = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case KEY:
- if (value == null) {
- unsetKey();
- } else {
- setKey((ByteBuffer)value);
- }
- break;
-
- case COLUMNS:
- if (value == null) {
- unsetColumns();
- } else {
- setColumns((List<Column>)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case KEY:
- return getKey();
-
- case COLUMNS:
- return getColumns();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case KEY:
- return isSetKey();
- case COLUMNS:
- return isSetColumns();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof CqlRow)
- return this.equals((CqlRow)that);
- return false;
- }
-
- public boolean equals(CqlRow that) {
- if (that == null)
- return false;
-
- boolean this_present_key = true && this.isSetKey();
- boolean that_present_key = true && that.isSetKey();
- if (this_present_key || that_present_key) {
- if (!(this_present_key && that_present_key))
- return false;
- if (!this.key.equals(that.key))
- return false;
- }
-
- boolean this_present_columns = true && this.isSetColumns();
- boolean that_present_columns = true && that.isSetColumns();
- if (this_present_columns || that_present_columns) {
- if (!(this_present_columns && that_present_columns))
- return false;
- if (!this.columns.equals(that.columns))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_key = true && (isSetKey());
- builder.append(present_key);
- if (present_key)
- builder.append(key);
-
- boolean present_columns = true && (isSetColumns());
- builder.append(present_columns);
- if (present_columns)
- builder.append(columns);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(CqlRow other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetKey()).compareTo(other.isSetKey());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetKey()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.key, other.key);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetColumns()).compareTo(other.isSetColumns());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetColumns()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.columns, other.columns);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("CqlRow(");
- boolean first = true;
-
- sb.append("key:");
- if (this.key == null) {
- sb.append("null");
- } else {
- org.apache.thrift.TBaseHelper.toString(this.key, sb);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("columns:");
- if (this.columns == null) {
- sb.append("null");
- } else {
- sb.append(this.columns);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- if (key == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'key' was not present! Struct: " + toString());
- }
- if (columns == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'columns' was not present! Struct: " + toString());
- }
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class CqlRowStandardSchemeFactory implements SchemeFactory {
- public CqlRowStandardScheme getScheme() {
- return new CqlRowStandardScheme();
- }
- }
-
- private static class CqlRowStandardScheme extends StandardScheme<CqlRow> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, CqlRow struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // KEY
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.key = iprot.readBinary();
- struct.setKeyIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 2: // COLUMNS
- if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
- {
- org.apache.thrift.protocol.TList _list164 = iprot.readListBegin();
- struct.columns = new ArrayList<Column>(_list164.size);
- for (int _i165 = 0; _i165 < _list164.size; ++_i165)
- {
- Column _elem166;
- _elem166 = new Column();
- _elem166.read(iprot);
- struct.columns.add(_elem166);
- }
- iprot.readListEnd();
- }
- struct.setColumnsIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, CqlRow struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.key != null) {
- oprot.writeFieldBegin(KEY_FIELD_DESC);
- oprot.writeBinary(struct.key);
- oprot.writeFieldEnd();
- }
- if (struct.columns != null) {
- oprot.writeFieldBegin(COLUMNS_FIELD_DESC);
- {
- oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.columns.size()));
- for (Column _iter167 : struct.columns)
- {
- _iter167.write(oprot);
- }
- oprot.writeListEnd();
- }
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class CqlRowTupleSchemeFactory implements SchemeFactory {
- public CqlRowTupleScheme getScheme() {
- return new CqlRowTupleScheme();
- }
- }
-
- private static class CqlRowTupleScheme extends TupleScheme<CqlRow> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, CqlRow struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- oprot.writeBinary(struct.key);
- {
- oprot.writeI32(struct.columns.size());
- for (Column _iter168 : struct.columns)
- {
- _iter168.write(oprot);
- }
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, CqlRow struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- struct.key = iprot.readBinary();
- struct.setKeyIsSet(true);
- {
- org.apache.thrift.protocol.TList _list169 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
- struct.columns = new ArrayList<Column>(_list169.size);
- for (int _i170 = 0; _i170 < _list169.size; ++_i170)
- {
- Column _elem171;
- _elem171 = new Column();
- _elem171.read(iprot);
- struct.columns.add(_elem171);
- }
- }
- struct.setColumnsIsSet(true);
- }
- }
-
-}
-
diff --git a/interface/thrift/gen-java/org/apache/cassandra/thrift/Deletion.java b/interface/thrift/gen-java/org/apache/cassandra/thrift/Deletion.java
deleted file mode 100644
index c98e449..0000000
--- a/interface/thrift/gen-java/org/apache/cassandra/thrift/Deletion.java
+++ /dev/null
@@ -1,645 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.1)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- * @generated
- */
-package org.apache.cassandra.thrift;
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
-
-import org.apache.commons.lang3.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Note that the timestamp is only optional in case of counter deletion.
- */
-public class Deletion implements org.apache.thrift.TBase<Deletion, Deletion._Fields>, java.io.Serializable, Cloneable, Comparable<Deletion> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("Deletion");
-
- private static final org.apache.thrift.protocol.TField TIMESTAMP_FIELD_DESC = new org.apache.thrift.protocol.TField("timestamp", org.apache.thrift.protocol.TType.I64, (short)1);
- private static final org.apache.thrift.protocol.TField SUPER_COLUMN_FIELD_DESC = new org.apache.thrift.protocol.TField("super_column", org.apache.thrift.protocol.TType.STRING, (short)2);
- private static final org.apache.thrift.protocol.TField PREDICATE_FIELD_DESC = new org.apache.thrift.protocol.TField("predicate", org.apache.thrift.protocol.TType.STRUCT, (short)3);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new DeletionStandardSchemeFactory());
- schemes.put(TupleScheme.class, new DeletionTupleSchemeFactory());
- }
-
- public long timestamp; // optional
- public ByteBuffer super_column; // optional
- public SlicePredicate predicate; // optional
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- TIMESTAMP((short)1, "timestamp"),
- SUPER_COLUMN((short)2, "super_column"),
- PREDICATE((short)3, "predicate");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // TIMESTAMP
- return TIMESTAMP;
- case 2: // SUPER_COLUMN
- return SUPER_COLUMN;
- case 3: // PREDICATE
- return PREDICATE;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- private static final int __TIMESTAMP_ISSET_ID = 0;
- private byte __isset_bitfield = 0;
- private _Fields optionals[] = {_Fields.TIMESTAMP,_Fields.SUPER_COLUMN,_Fields.PREDICATE};
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.TIMESTAMP, new org.apache.thrift.meta_data.FieldMetaData("timestamp", org.apache.thrift.TFieldRequirementType.OPTIONAL,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64)));
- tmpMap.put(_Fields.SUPER_COLUMN, new org.apache.thrift.meta_data.FieldMetaData("super_column", org.apache.thrift.TFieldRequirementType.OPTIONAL,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true)));
- tmpMap.put(_Fields.PREDICATE, new org.apache.thrift.meta_data.FieldMetaData("predicate", org.apache.thrift.TFieldRequirementType.OPTIONAL,
- new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, SlicePredicate.class)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(Deletion.class, metaDataMap);
- }
-
- public Deletion() {
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public Deletion(Deletion other) {
- __isset_bitfield = other.__isset_bitfield;
- this.timestamp = other.timestamp;
- if (other.isSetSuper_column()) {
- this.super_column = org.apache.thrift.TBaseHelper.copyBinary(other.super_column);
-;
- }
- if (other.isSetPredicate()) {
- this.predicate = new SlicePredicate(other.predicate);
- }
- }
-
- public Deletion deepCopy() {
- return new Deletion(this);
- }
-
- @Override
- public void clear() {
- setTimestampIsSet(false);
- this.timestamp = 0;
- this.super_column = null;
- this.predicate = null;
- }
-
- public long getTimestamp() {
- return this.timestamp;
- }
-
- public Deletion setTimestamp(long timestamp) {
- this.timestamp = timestamp;
- setTimestampIsSet(true);
- return this;
- }
-
- public void unsetTimestamp() {
- __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __TIMESTAMP_ISSET_ID);
- }
-
- /** Returns true if field timestamp is set (has been assigned a value) and false otherwise */
- public boolean isSetTimestamp() {
- return EncodingUtils.testBit(__isset_bitfield, __TIMESTAMP_ISSET_ID);
- }
-
- public void setTimestampIsSet(boolean value) {
- __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __TIMESTAMP_ISSET_ID, value);
- }
-
- public byte[] getSuper_column() {
- setSuper_column(org.apache.thrift.TBaseHelper.rightSize(super_column));
- return super_column == null ? null : super_column.array();
- }
-
- public ByteBuffer bufferForSuper_column() {
- return super_column;
- }
-
- public Deletion setSuper_column(byte[] super_column) {
- setSuper_column(super_column == null ? (ByteBuffer)null : ByteBuffer.wrap(super_column));
- return this;
- }
-
- public Deletion setSuper_column(ByteBuffer super_column) {
- this.super_column = super_column;
- return this;
- }
-
- public void unsetSuper_column() {
- this.super_column = null;
- }
-
- /** Returns true if field super_column is set (has been assigned a value) and false otherwise */
- public boolean isSetSuper_column() {
- return this.super_column != null;
- }
-
- public void setSuper_columnIsSet(boolean value) {
- if (!value) {
- this.super_column = null;
- }
- }
-
- public SlicePredicate getPredicate() {
- return this.predicate;
- }
-
- public Deletion setPredicate(SlicePredicate predicate) {
- this.predicate = predicate;
- return this;
- }
-
- public void unsetPredicate() {
- this.predicate = null;
- }
-
- /** Returns true if field predicate is set (has been assigned a value) and false otherwise */
- public boolean isSetPredicate() {
- return this.predicate != null;
- }
-
- public void setPredicateIsSet(boolean value) {
- if (!value) {
- this.predicate = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case TIMESTAMP:
- if (value == null) {
- unsetTimestamp();
- } else {
- setTimestamp((Long)value);
- }
- break;
-
- case SUPER_COLUMN:
- if (value == null) {
- unsetSuper_column();
- } else {
- setSuper_column((ByteBuffer)value);
- }
- break;
-
- case PREDICATE:
- if (value == null) {
- unsetPredicate();
- } else {
- setPredicate((SlicePredicate)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case TIMESTAMP:
- return Long.valueOf(getTimestamp());
-
- case SUPER_COLUMN:
- return getSuper_column();
-
- case PREDICATE:
- return getPredicate();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case TIMESTAMP:
- return isSetTimestamp();
- case SUPER_COLUMN:
- return isSetSuper_column();
- case PREDICATE:
- return isSetPredicate();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof Deletion)
- return this.equals((Deletion)that);
- return false;
- }
-
- public boolean equals(Deletion that) {
- if (that == null)
- return false;
-
- boolean this_present_timestamp = true && this.isSetTimestamp();
- boolean that_present_timestamp = true && that.isSetTimestamp();
- if (this_present_timestamp || that_present_timestamp) {
- if (!(this_present_timestamp && that_present_timestamp))
- return false;
- if (this.timestamp != that.timestamp)
- return false;
- }
-
- boolean this_present_super_column = true && this.isSetSuper_column();
- boolean that_present_super_column = true && that.isSetSuper_column();
- if (this_present_super_column || that_present_super_column) {
- if (!(this_present_super_column && that_present_super_column))
- return false;
- if (!this.super_column.equals(that.super_column))
- return false;
- }
-
- boolean this_present_predicate = true && this.isSetPredicate();
- boolean that_present_predicate = true && that.isSetPredicate();
- if (this_present_predicate || that_present_predicate) {
- if (!(this_present_predicate && that_present_predicate))
- return false;
- if (!this.predicate.equals(that.predicate))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_timestamp = true && (isSetTimestamp());
- builder.append(present_timestamp);
- if (present_timestamp)
- builder.append(timestamp);
-
- boolean present_super_column = true && (isSetSuper_column());
- builder.append(present_super_column);
- if (present_super_column)
- builder.append(super_column);
-
- boolean present_predicate = true && (isSetPredicate());
- builder.append(present_predicate);
- if (present_predicate)
- builder.append(predicate);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(Deletion other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetTimestamp()).compareTo(other.isSetTimestamp());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetTimestamp()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.timestamp, other.timestamp);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetSuper_column()).compareTo(other.isSetSuper_column());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetSuper_column()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.super_column, other.super_column);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetPredicate()).compareTo(other.isSetPredicate());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetPredicate()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.predicate, other.predicate);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("Deletion(");
- boolean first = true;
-
- if (isSetTimestamp()) {
- sb.append("timestamp:");
- sb.append(this.timestamp);
- first = false;
- }
- if (isSetSuper_column()) {
- if (!first) sb.append(", ");
- sb.append("super_column:");
- if (this.super_column == null) {
- sb.append("null");
- } else {
- org.apache.thrift.TBaseHelper.toString(this.super_column, sb);
- }
- first = false;
- }
- if (isSetPredicate()) {
- if (!first) sb.append(", ");
- sb.append("predicate:");
- if (this.predicate == null) {
- sb.append("null");
- } else {
- sb.append(this.predicate);
- }
- first = false;
- }
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // check for sub-struct validity
- if (predicate != null) {
- predicate.validate();
- }
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
- __isset_bitfield = 0;
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class DeletionStandardSchemeFactory implements SchemeFactory {
- public DeletionStandardScheme getScheme() {
- return new DeletionStandardScheme();
- }
- }
-
- private static class DeletionStandardScheme extends StandardScheme<Deletion> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, Deletion struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // TIMESTAMP
- if (schemeField.type == org.apache.thrift.protocol.TType.I64) {
- struct.timestamp = iprot.readI64();
- struct.setTimestampIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 2: // SUPER_COLUMN
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.super_column = iprot.readBinary();
- struct.setSuper_columnIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 3: // PREDICATE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.predicate = new SlicePredicate();
- struct.predicate.read(iprot);
- struct.setPredicateIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, Deletion struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.isSetTimestamp()) {
- oprot.writeFieldBegin(TIMESTAMP_FIELD_DESC);
- oprot.writeI64(struct.timestamp);
- oprot.writeFieldEnd();
- }
- if (struct.super_column != null) {
- if (struct.isSetSuper_column()) {
- oprot.writeFieldBegin(SUPER_COLUMN_FIELD_DESC);
- oprot.writeBinary(struct.super_column);
- oprot.writeFieldEnd();
- }
- }
- if (struct.predicate != null) {
- if (struct.isSetPredicate()) {
- oprot.writeFieldBegin(PREDICATE_FIELD_DESC);
- struct.predicate.write(oprot);
- oprot.writeFieldEnd();
- }
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class DeletionTupleSchemeFactory implements SchemeFactory {
- public DeletionTupleScheme getScheme() {
- return new DeletionTupleScheme();
- }
- }
-
- private static class DeletionTupleScheme extends TupleScheme<Deletion> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, Deletion struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- BitSet optionals = new BitSet();
- if (struct.isSetTimestamp()) {
- optionals.set(0);
- }
- if (struct.isSetSuper_column()) {
- optionals.set(1);
- }
- if (struct.isSetPredicate()) {
- optionals.set(2);
- }
- oprot.writeBitSet(optionals, 3);
- if (struct.isSetTimestamp()) {
- oprot.writeI64(struct.timestamp);
- }
- if (struct.isSetSuper_column()) {
- oprot.writeBinary(struct.super_column);
- }
- if (struct.isSetPredicate()) {
- struct.predicate.write(oprot);
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, Deletion struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- BitSet incoming = iprot.readBitSet(3);
- if (incoming.get(0)) {
- struct.timestamp = iprot.readI64();
- struct.setTimestampIsSet(true);
- }
- if (incoming.get(1)) {
- struct.super_column = iprot.readBinary();
- struct.setSuper_columnIsSet(true);
- }
- if (incoming.get(2)) {
- struct.predicate = new SlicePredicate();
- struct.predicate.read(iprot);
- struct.setPredicateIsSet(true);
- }
- }
- }
-
-}
-
diff --git a/interface/thrift/gen-java/org/apache/cassandra/thrift/EndpointDetails.java b/interface/thrift/gen-java/org/apache/cassandra/thrift/EndpointDetails.java
deleted file mode 100644
index 69fcf58..0000000
--- a/interface/thrift/gen-java/org/apache/cassandra/thrift/EndpointDetails.java
+++ /dev/null
@@ -1,630 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.1)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- * @generated
- */
-package org.apache.cassandra.thrift;
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
-
-import org.apache.commons.lang3.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class EndpointDetails implements org.apache.thrift.TBase<EndpointDetails, EndpointDetails._Fields>, java.io.Serializable, Cloneable, Comparable<EndpointDetails> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("EndpointDetails");
-
- private static final org.apache.thrift.protocol.TField HOST_FIELD_DESC = new org.apache.thrift.protocol.TField("host", org.apache.thrift.protocol.TType.STRING, (short)1);
- private static final org.apache.thrift.protocol.TField DATACENTER_FIELD_DESC = new org.apache.thrift.protocol.TField("datacenter", org.apache.thrift.protocol.TType.STRING, (short)2);
- private static final org.apache.thrift.protocol.TField RACK_FIELD_DESC = new org.apache.thrift.protocol.TField("rack", org.apache.thrift.protocol.TType.STRING, (short)3);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new EndpointDetailsStandardSchemeFactory());
- schemes.put(TupleScheme.class, new EndpointDetailsTupleSchemeFactory());
- }
-
- public String host; // required
- public String datacenter; // required
- public String rack; // optional
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- HOST((short)1, "host"),
- DATACENTER((short)2, "datacenter"),
- RACK((short)3, "rack");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // HOST
- return HOST;
- case 2: // DATACENTER
- return DATACENTER;
- case 3: // RACK
- return RACK;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- private _Fields optionals[] = {_Fields.RACK};
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.HOST, new org.apache.thrift.meta_data.FieldMetaData("host", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
- tmpMap.put(_Fields.DATACENTER, new org.apache.thrift.meta_data.FieldMetaData("datacenter", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
- tmpMap.put(_Fields.RACK, new org.apache.thrift.meta_data.FieldMetaData("rack", org.apache.thrift.TFieldRequirementType.OPTIONAL,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(EndpointDetails.class, metaDataMap);
- }
-
- public EndpointDetails() {
- }
-
- public EndpointDetails(
- String host,
- String datacenter)
- {
- this();
- this.host = host;
- this.datacenter = datacenter;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public EndpointDetails(EndpointDetails other) {
- if (other.isSetHost()) {
- this.host = other.host;
- }
- if (other.isSetDatacenter()) {
- this.datacenter = other.datacenter;
- }
- if (other.isSetRack()) {
- this.rack = other.rack;
- }
- }
-
- public EndpointDetails deepCopy() {
- return new EndpointDetails(this);
- }
-
- @Override
- public void clear() {
- this.host = null;
- this.datacenter = null;
- this.rack = null;
- }
-
- public String getHost() {
- return this.host;
- }
-
- public EndpointDetails setHost(String host) {
- this.host = host;
- return this;
- }
-
- public void unsetHost() {
- this.host = null;
- }
-
- /** Returns true if field host is set (has been assigned a value) and false otherwise */
- public boolean isSetHost() {
- return this.host != null;
- }
-
- public void setHostIsSet(boolean value) {
- if (!value) {
- this.host = null;
- }
- }
-
- public String getDatacenter() {
- return this.datacenter;
- }
-
- public EndpointDetails setDatacenter(String datacenter) {
- this.datacenter = datacenter;
- return this;
- }
-
- public void unsetDatacenter() {
- this.datacenter = null;
- }
-
- /** Returns true if field datacenter is set (has been assigned a value) and false otherwise */
- public boolean isSetDatacenter() {
- return this.datacenter != null;
- }
-
- public void setDatacenterIsSet(boolean value) {
- if (!value) {
- this.datacenter = null;
- }
- }
-
- public String getRack() {
- return this.rack;
- }
-
- public EndpointDetails setRack(String rack) {
- this.rack = rack;
- return this;
- }
-
- public void unsetRack() {
- this.rack = null;
- }
-
- /** Returns true if field rack is set (has been assigned a value) and false otherwise */
- public boolean isSetRack() {
- return this.rack != null;
- }
-
- public void setRackIsSet(boolean value) {
- if (!value) {
- this.rack = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case HOST:
- if (value == null) {
- unsetHost();
- } else {
- setHost((String)value);
- }
- break;
-
- case DATACENTER:
- if (value == null) {
- unsetDatacenter();
- } else {
- setDatacenter((String)value);
- }
- break;
-
- case RACK:
- if (value == null) {
- unsetRack();
- } else {
- setRack((String)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case HOST:
- return getHost();
-
- case DATACENTER:
- return getDatacenter();
-
- case RACK:
- return getRack();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case HOST:
- return isSetHost();
- case DATACENTER:
- return isSetDatacenter();
- case RACK:
- return isSetRack();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof EndpointDetails)
- return this.equals((EndpointDetails)that);
- return false;
- }
-
- public boolean equals(EndpointDetails that) {
- if (that == null)
- return false;
-
- boolean this_present_host = true && this.isSetHost();
- boolean that_present_host = true && that.isSetHost();
- if (this_present_host || that_present_host) {
- if (!(this_present_host && that_present_host))
- return false;
- if (!this.host.equals(that.host))
- return false;
- }
-
- boolean this_present_datacenter = true && this.isSetDatacenter();
- boolean that_present_datacenter = true && that.isSetDatacenter();
- if (this_present_datacenter || that_present_datacenter) {
- if (!(this_present_datacenter && that_present_datacenter))
- return false;
- if (!this.datacenter.equals(that.datacenter))
- return false;
- }
-
- boolean this_present_rack = true && this.isSetRack();
- boolean that_present_rack = true && that.isSetRack();
- if (this_present_rack || that_present_rack) {
- if (!(this_present_rack && that_present_rack))
- return false;
- if (!this.rack.equals(that.rack))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_host = true && (isSetHost());
- builder.append(present_host);
- if (present_host)
- builder.append(host);
-
- boolean present_datacenter = true && (isSetDatacenter());
- builder.append(present_datacenter);
- if (present_datacenter)
- builder.append(datacenter);
-
- boolean present_rack = true && (isSetRack());
- builder.append(present_rack);
- if (present_rack)
- builder.append(rack);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(EndpointDetails other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetHost()).compareTo(other.isSetHost());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetHost()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.host, other.host);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetDatacenter()).compareTo(other.isSetDatacenter());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetDatacenter()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.datacenter, other.datacenter);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetRack()).compareTo(other.isSetRack());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetRack()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.rack, other.rack);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("EndpointDetails(");
- boolean first = true;
-
- sb.append("host:");
- if (this.host == null) {
- sb.append("null");
- } else {
- sb.append(this.host);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("datacenter:");
- if (this.datacenter == null) {
- sb.append("null");
- } else {
- sb.append(this.datacenter);
- }
- first = false;
- if (isSetRack()) {
- if (!first) sb.append(", ");
- sb.append("rack:");
- if (this.rack == null) {
- sb.append("null");
- } else {
- sb.append(this.rack);
- }
- first = false;
- }
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class EndpointDetailsStandardSchemeFactory implements SchemeFactory {
- public EndpointDetailsStandardScheme getScheme() {
- return new EndpointDetailsStandardScheme();
- }
- }
-
- private static class EndpointDetailsStandardScheme extends StandardScheme<EndpointDetails> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, EndpointDetails struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // HOST
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.host = iprot.readString();
- struct.setHostIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 2: // DATACENTER
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.datacenter = iprot.readString();
- struct.setDatacenterIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 3: // RACK
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.rack = iprot.readString();
- struct.setRackIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, EndpointDetails struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.host != null) {
- oprot.writeFieldBegin(HOST_FIELD_DESC);
- oprot.writeString(struct.host);
- oprot.writeFieldEnd();
- }
- if (struct.datacenter != null) {
- oprot.writeFieldBegin(DATACENTER_FIELD_DESC);
- oprot.writeString(struct.datacenter);
- oprot.writeFieldEnd();
- }
- if (struct.rack != null) {
- if (struct.isSetRack()) {
- oprot.writeFieldBegin(RACK_FIELD_DESC);
- oprot.writeString(struct.rack);
- oprot.writeFieldEnd();
- }
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class EndpointDetailsTupleSchemeFactory implements SchemeFactory {
- public EndpointDetailsTupleScheme getScheme() {
- return new EndpointDetailsTupleScheme();
- }
- }
-
- private static class EndpointDetailsTupleScheme extends TupleScheme<EndpointDetails> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, EndpointDetails struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- BitSet optionals = new BitSet();
- if (struct.isSetHost()) {
- optionals.set(0);
- }
- if (struct.isSetDatacenter()) {
- optionals.set(1);
- }
- if (struct.isSetRack()) {
- optionals.set(2);
- }
- oprot.writeBitSet(optionals, 3);
- if (struct.isSetHost()) {
- oprot.writeString(struct.host);
- }
- if (struct.isSetDatacenter()) {
- oprot.writeString(struct.datacenter);
- }
- if (struct.isSetRack()) {
- oprot.writeString(struct.rack);
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, EndpointDetails struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- BitSet incoming = iprot.readBitSet(3);
- if (incoming.get(0)) {
- struct.host = iprot.readString();
- struct.setHostIsSet(true);
- }
- if (incoming.get(1)) {
- struct.datacenter = iprot.readString();
- struct.setDatacenterIsSet(true);
- }
- if (incoming.get(2)) {
- struct.rack = iprot.readString();
- struct.setRackIsSet(true);
- }
- }
- }
-
-}
-
diff --git a/interface/thrift/gen-java/org/apache/cassandra/thrift/IndexClause.java b/interface/thrift/gen-java/org/apache/cassandra/thrift/IndexClause.java
deleted file mode 100644
index f3524b5..0000000
--- a/interface/thrift/gen-java/org/apache/cassandra/thrift/IndexClause.java
+++ /dev/null
@@ -1,681 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.1)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- * @generated
- */
-package org.apache.cassandra.thrift;
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
-
-import org.apache.commons.lang3.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * @deprecated use a KeyRange with row_filter in get_range_slices instead
- */
-public class IndexClause implements org.apache.thrift.TBase<IndexClause, IndexClause._Fields>, java.io.Serializable, Cloneable, Comparable<IndexClause> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("IndexClause");
-
- private static final org.apache.thrift.protocol.TField EXPRESSIONS_FIELD_DESC = new org.apache.thrift.protocol.TField("expressions", org.apache.thrift.protocol.TType.LIST, (short)1);
- private static final org.apache.thrift.protocol.TField START_KEY_FIELD_DESC = new org.apache.thrift.protocol.TField("start_key", org.apache.thrift.protocol.TType.STRING, (short)2);
- private static final org.apache.thrift.protocol.TField COUNT_FIELD_DESC = new org.apache.thrift.protocol.TField("count", org.apache.thrift.protocol.TType.I32, (short)3);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new IndexClauseStandardSchemeFactory());
- schemes.put(TupleScheme.class, new IndexClauseTupleSchemeFactory());
- }
-
- public List<IndexExpression> expressions; // required
- public ByteBuffer start_key; // required
- public int count; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- EXPRESSIONS((short)1, "expressions"),
- START_KEY((short)2, "start_key"),
- COUNT((short)3, "count");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // EXPRESSIONS
- return EXPRESSIONS;
- case 2: // START_KEY
- return START_KEY;
- case 3: // COUNT
- return COUNT;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- private static final int __COUNT_ISSET_ID = 0;
- private byte __isset_bitfield = 0;
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.EXPRESSIONS, new org.apache.thrift.meta_data.FieldMetaData("expressions", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST,
- new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, IndexExpression.class))));
- tmpMap.put(_Fields.START_KEY, new org.apache.thrift.meta_data.FieldMetaData("start_key", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true)));
- tmpMap.put(_Fields.COUNT, new org.apache.thrift.meta_data.FieldMetaData("count", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(IndexClause.class, metaDataMap);
- }
-
- public IndexClause() {
- this.count = 100;
-
- }
-
- public IndexClause(
- List<IndexExpression> expressions,
- ByteBuffer start_key,
- int count)
- {
- this();
- this.expressions = expressions;
- this.start_key = start_key;
- this.count = count;
- setCountIsSet(true);
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public IndexClause(IndexClause other) {
- __isset_bitfield = other.__isset_bitfield;
- if (other.isSetExpressions()) {
- List<IndexExpression> __this__expressions = new ArrayList<IndexExpression>(other.expressions.size());
- for (IndexExpression other_element : other.expressions) {
- __this__expressions.add(new IndexExpression(other_element));
- }
- this.expressions = __this__expressions;
- }
- if (other.isSetStart_key()) {
- this.start_key = org.apache.thrift.TBaseHelper.copyBinary(other.start_key);
-;
- }
- this.count = other.count;
- }
-
- public IndexClause deepCopy() {
- return new IndexClause(this);
- }
-
- @Override
- public void clear() {
- this.expressions = null;
- this.start_key = null;
- this.count = 100;
-
- }
-
- public int getExpressionsSize() {
- return (this.expressions == null) ? 0 : this.expressions.size();
- }
-
- public java.util.Iterator<IndexExpression> getExpressionsIterator() {
- return (this.expressions == null) ? null : this.expressions.iterator();
- }
-
- public void addToExpressions(IndexExpression elem) {
- if (this.expressions == null) {
- this.expressions = new ArrayList<IndexExpression>();
- }
- this.expressions.add(elem);
- }
-
- public List<IndexExpression> getExpressions() {
- return this.expressions;
- }
-
- public IndexClause setExpressions(List<IndexExpression> expressions) {
- this.expressions = expressions;
- return this;
- }
-
- public void unsetExpressions() {
- this.expressions = null;
- }
-
- /** Returns true if field expressions is set (has been assigned a value) and false otherwise */
- public boolean isSetExpressions() {
- return this.expressions != null;
- }
-
- public void setExpressionsIsSet(boolean value) {
- if (!value) {
- this.expressions = null;
- }
- }
-
- public byte[] getStart_key() {
- setStart_key(org.apache.thrift.TBaseHelper.rightSize(start_key));
- return start_key == null ? null : start_key.array();
- }
-
- public ByteBuffer bufferForStart_key() {
- return start_key;
- }
-
- public IndexClause setStart_key(byte[] start_key) {
- setStart_key(start_key == null ? (ByteBuffer)null : ByteBuffer.wrap(start_key));
- return this;
- }
-
- public IndexClause setStart_key(ByteBuffer start_key) {
- this.start_key = start_key;
- return this;
- }
-
- public void unsetStart_key() {
- this.start_key = null;
- }
-
- /** Returns true if field start_key is set (has been assigned a value) and false otherwise */
- public boolean isSetStart_key() {
- return this.start_key != null;
- }
-
- public void setStart_keyIsSet(boolean value) {
- if (!value) {
- this.start_key = null;
- }
- }
-
- public int getCount() {
- return this.count;
- }
-
- public IndexClause setCount(int count) {
- this.count = count;
- setCountIsSet(true);
- return this;
- }
-
- public void unsetCount() {
- __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __COUNT_ISSET_ID);
- }
-
- /** Returns true if field count is set (has been assigned a value) and false otherwise */
- public boolean isSetCount() {
- return EncodingUtils.testBit(__isset_bitfield, __COUNT_ISSET_ID);
- }
-
- public void setCountIsSet(boolean value) {
- __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __COUNT_ISSET_ID, value);
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case EXPRESSIONS:
- if (value == null) {
- unsetExpressions();
- } else {
- setExpressions((List<IndexExpression>)value);
- }
- break;
-
- case START_KEY:
- if (value == null) {
- unsetStart_key();
- } else {
- setStart_key((ByteBuffer)value);
- }
- break;
-
- case COUNT:
- if (value == null) {
- unsetCount();
- } else {
- setCount((Integer)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case EXPRESSIONS:
- return getExpressions();
-
- case START_KEY:
- return getStart_key();
-
- case COUNT:
- return Integer.valueOf(getCount());
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case EXPRESSIONS:
- return isSetExpressions();
- case START_KEY:
- return isSetStart_key();
- case COUNT:
- return isSetCount();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof IndexClause)
- return this.equals((IndexClause)that);
- return false;
- }
-
- public boolean equals(IndexClause that) {
- if (that == null)
- return false;
-
- boolean this_present_expressions = true && this.isSetExpressions();
- boolean that_present_expressions = true && that.isSetExpressions();
- if (this_present_expressions || that_present_expressions) {
- if (!(this_present_expressions && that_present_expressions))
- return false;
- if (!this.expressions.equals(that.expressions))
- return false;
- }
-
- boolean this_present_start_key = true && this.isSetStart_key();
- boolean that_present_start_key = true && that.isSetStart_key();
- if (this_present_start_key || that_present_start_key) {
- if (!(this_present_start_key && that_present_start_key))
- return false;
- if (!this.start_key.equals(that.start_key))
- return false;
- }
-
- boolean this_present_count = true;
- boolean that_present_count = true;
- if (this_present_count || that_present_count) {
- if (!(this_present_count && that_present_count))
- return false;
- if (this.count != that.count)
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_expressions = true && (isSetExpressions());
- builder.append(present_expressions);
- if (present_expressions)
- builder.append(expressions);
-
- boolean present_start_key = true && (isSetStart_key());
- builder.append(present_start_key);
- if (present_start_key)
- builder.append(start_key);
-
- boolean present_count = true;
- builder.append(present_count);
- if (present_count)
- builder.append(count);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(IndexClause other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetExpressions()).compareTo(other.isSetExpressions());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetExpressions()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.expressions, other.expressions);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetStart_key()).compareTo(other.isSetStart_key());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetStart_key()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.start_key, other.start_key);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetCount()).compareTo(other.isSetCount());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetCount()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.count, other.count);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("IndexClause(");
- boolean first = true;
-
- sb.append("expressions:");
- if (this.expressions == null) {
- sb.append("null");
- } else {
- sb.append(this.expressions);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("start_key:");
- if (this.start_key == null) {
- sb.append("null");
- } else {
- org.apache.thrift.TBaseHelper.toString(this.start_key, sb);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("count:");
- sb.append(this.count);
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- if (expressions == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'expressions' was not present! Struct: " + toString());
- }
- if (start_key == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'start_key' was not present! Struct: " + toString());
- }
- // alas, we cannot check 'count' because it's a primitive and you chose the non-beans generator.
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
- __isset_bitfield = 0;
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class IndexClauseStandardSchemeFactory implements SchemeFactory {
- public IndexClauseStandardScheme getScheme() {
- return new IndexClauseStandardScheme();
- }
- }
-
- private static class IndexClauseStandardScheme extends StandardScheme<IndexClause> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, IndexClause struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // EXPRESSIONS
- if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
- {
- org.apache.thrift.protocol.TList _list24 = iprot.readListBegin();
- struct.expressions = new ArrayList<IndexExpression>(_list24.size);
- for (int _i25 = 0; _i25 < _list24.size; ++_i25)
- {
- IndexExpression _elem26;
- _elem26 = new IndexExpression();
- _elem26.read(iprot);
- struct.expressions.add(_elem26);
- }
- iprot.readListEnd();
- }
- struct.setExpressionsIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 2: // START_KEY
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.start_key = iprot.readBinary();
- struct.setStart_keyIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 3: // COUNT
- if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
- struct.count = iprot.readI32();
- struct.setCountIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- if (!struct.isSetCount()) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'count' was not found in serialized data! Struct: " + toString());
- }
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, IndexClause struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.expressions != null) {
- oprot.writeFieldBegin(EXPRESSIONS_FIELD_DESC);
- {
- oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.expressions.size()));
- for (IndexExpression _iter27 : struct.expressions)
- {
- _iter27.write(oprot);
- }
- oprot.writeListEnd();
- }
- oprot.writeFieldEnd();
- }
- if (struct.start_key != null) {
- oprot.writeFieldBegin(START_KEY_FIELD_DESC);
- oprot.writeBinary(struct.start_key);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldBegin(COUNT_FIELD_DESC);
- oprot.writeI32(struct.count);
- oprot.writeFieldEnd();
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class IndexClauseTupleSchemeFactory implements SchemeFactory {
- public IndexClauseTupleScheme getScheme() {
- return new IndexClauseTupleScheme();
- }
- }
-
- private static class IndexClauseTupleScheme extends TupleScheme<IndexClause> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, IndexClause struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- {
- oprot.writeI32(struct.expressions.size());
- for (IndexExpression _iter28 : struct.expressions)
- {
- _iter28.write(oprot);
- }
- }
- oprot.writeBinary(struct.start_key);
- oprot.writeI32(struct.count);
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, IndexClause struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- {
- org.apache.thrift.protocol.TList _list29 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
- struct.expressions = new ArrayList<IndexExpression>(_list29.size);
- for (int _i30 = 0; _i30 < _list29.size; ++_i30)
- {
- IndexExpression _elem31;
- _elem31 = new IndexExpression();
- _elem31.read(iprot);
- struct.expressions.add(_elem31);
- }
- }
- struct.setExpressionsIsSet(true);
- struct.start_key = iprot.readBinary();
- struct.setStart_keyIsSet(true);
- struct.count = iprot.readI32();
- struct.setCountIsSet(true);
- }
- }
-
-}
-
diff --git a/interface/thrift/gen-java/org/apache/cassandra/thrift/IndexExpression.java b/interface/thrift/gen-java/org/apache/cassandra/thrift/IndexExpression.java
deleted file mode 100644
index 5062f2f..0000000
--- a/interface/thrift/gen-java/org/apache/cassandra/thrift/IndexExpression.java
+++ /dev/null
@@ -1,650 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.1)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- * @generated
- */
-package org.apache.cassandra.thrift;
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
-
-import org.apache.commons.lang3.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class IndexExpression implements org.apache.thrift.TBase<IndexExpression, IndexExpression._Fields>, java.io.Serializable, Cloneable, Comparable<IndexExpression> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("IndexExpression");
-
- private static final org.apache.thrift.protocol.TField COLUMN_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("column_name", org.apache.thrift.protocol.TType.STRING, (short)1);
- private static final org.apache.thrift.protocol.TField OP_FIELD_DESC = new org.apache.thrift.protocol.TField("op", org.apache.thrift.protocol.TType.I32, (short)2);
- private static final org.apache.thrift.protocol.TField VALUE_FIELD_DESC = new org.apache.thrift.protocol.TField("value", org.apache.thrift.protocol.TType.STRING, (short)3);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new IndexExpressionStandardSchemeFactory());
- schemes.put(TupleScheme.class, new IndexExpressionTupleSchemeFactory());
- }
-
- public ByteBuffer column_name; // required
- /**
- *
- * @see IndexOperator
- */
- public IndexOperator op; // required
- public ByteBuffer value; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- COLUMN_NAME((short)1, "column_name"),
- /**
- *
- * @see IndexOperator
- */
- OP((short)2, "op"),
- VALUE((short)3, "value");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // COLUMN_NAME
- return COLUMN_NAME;
- case 2: // OP
- return OP;
- case 3: // VALUE
- return VALUE;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.COLUMN_NAME, new org.apache.thrift.meta_data.FieldMetaData("column_name", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true)));
- tmpMap.put(_Fields.OP, new org.apache.thrift.meta_data.FieldMetaData("op", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.EnumMetaData(org.apache.thrift.protocol.TType.ENUM, IndexOperator.class)));
- tmpMap.put(_Fields.VALUE, new org.apache.thrift.meta_data.FieldMetaData("value", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(IndexExpression.class, metaDataMap);
- }
-
- public IndexExpression() {
- }
-
- public IndexExpression(
- ByteBuffer column_name,
- IndexOperator op,
- ByteBuffer value)
- {
- this();
- this.column_name = column_name;
- this.op = op;
- this.value = value;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public IndexExpression(IndexExpression other) {
- if (other.isSetColumn_name()) {
- this.column_name = org.apache.thrift.TBaseHelper.copyBinary(other.column_name);
-;
- }
- if (other.isSetOp()) {
- this.op = other.op;
- }
- if (other.isSetValue()) {
- this.value = org.apache.thrift.TBaseHelper.copyBinary(other.value);
-;
- }
- }
-
- public IndexExpression deepCopy() {
- return new IndexExpression(this);
- }
-
- @Override
- public void clear() {
- this.column_name = null;
- this.op = null;
- this.value = null;
- }
-
- public byte[] getColumn_name() {
- setColumn_name(org.apache.thrift.TBaseHelper.rightSize(column_name));
- return column_name == null ? null : column_name.array();
- }
-
- public ByteBuffer bufferForColumn_name() {
- return column_name;
- }
-
- public IndexExpression setColumn_name(byte[] column_name) {
- setColumn_name(column_name == null ? (ByteBuffer)null : ByteBuffer.wrap(column_name));
- return this;
- }
-
- public IndexExpression setColumn_name(ByteBuffer column_name) {
- this.column_name = column_name;
- return this;
- }
-
- public void unsetColumn_name() {
- this.column_name = null;
- }
-
- /** Returns true if field column_name is set (has been assigned a value) and false otherwise */
- public boolean isSetColumn_name() {
- return this.column_name != null;
- }
-
- public void setColumn_nameIsSet(boolean value) {
- if (!value) {
- this.column_name = null;
- }
- }
-
- /**
- *
- * @see IndexOperator
- */
- public IndexOperator getOp() {
- return this.op;
- }
-
- /**
- *
- * @see IndexOperator
- */
- public IndexExpression setOp(IndexOperator op) {
- this.op = op;
- return this;
- }
-
- public void unsetOp() {
- this.op = null;
- }
-
- /** Returns true if field op is set (has been assigned a value) and false otherwise */
- public boolean isSetOp() {
- return this.op != null;
- }
-
- public void setOpIsSet(boolean value) {
- if (!value) {
- this.op = null;
- }
- }
-
- public byte[] getValue() {
- setValue(org.apache.thrift.TBaseHelper.rightSize(value));
- return value == null ? null : value.array();
- }
-
- public ByteBuffer bufferForValue() {
- return value;
- }
-
- public IndexExpression setValue(byte[] value) {
- setValue(value == null ? (ByteBuffer)null : ByteBuffer.wrap(value));
- return this;
- }
-
- public IndexExpression setValue(ByteBuffer value) {
- this.value = value;
- return this;
- }
-
- public void unsetValue() {
- this.value = null;
- }
-
- /** Returns true if field value is set (has been assigned a value) and false otherwise */
- public boolean isSetValue() {
- return this.value != null;
- }
-
- public void setValueIsSet(boolean value) {
- if (!value) {
- this.value = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case COLUMN_NAME:
- if (value == null) {
- unsetColumn_name();
- } else {
- setColumn_name((ByteBuffer)value);
- }
- break;
-
- case OP:
- if (value == null) {
- unsetOp();
- } else {
- setOp((IndexOperator)value);
- }
- break;
-
- case VALUE:
- if (value == null) {
- unsetValue();
- } else {
- setValue((ByteBuffer)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case COLUMN_NAME:
- return getColumn_name();
-
- case OP:
- return getOp();
-
- case VALUE:
- return getValue();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case COLUMN_NAME:
- return isSetColumn_name();
- case OP:
- return isSetOp();
- case VALUE:
- return isSetValue();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof IndexExpression)
- return this.equals((IndexExpression)that);
- return false;
- }
-
- public boolean equals(IndexExpression that) {
- if (that == null)
- return false;
-
- boolean this_present_column_name = true && this.isSetColumn_name();
- boolean that_present_column_name = true && that.isSetColumn_name();
- if (this_present_column_name || that_present_column_name) {
- if (!(this_present_column_name && that_present_column_name))
- return false;
- if (!this.column_name.equals(that.column_name))
- return false;
- }
-
- boolean this_present_op = true && this.isSetOp();
- boolean that_present_op = true && that.isSetOp();
- if (this_present_op || that_present_op) {
- if (!(this_present_op && that_present_op))
- return false;
- if (!this.op.equals(that.op))
- return false;
- }
-
- boolean this_present_value = true && this.isSetValue();
- boolean that_present_value = true && that.isSetValue();
- if (this_present_value || that_present_value) {
- if (!(this_present_value && that_present_value))
- return false;
- if (!this.value.equals(that.value))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_column_name = true && (isSetColumn_name());
- builder.append(present_column_name);
- if (present_column_name)
- builder.append(column_name);
-
- boolean present_op = true && (isSetOp());
- builder.append(present_op);
- if (present_op)
- builder.append(op.getValue());
-
- boolean present_value = true && (isSetValue());
- builder.append(present_value);
- if (present_value)
- builder.append(value);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(IndexExpression other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetColumn_name()).compareTo(other.isSetColumn_name());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetColumn_name()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.column_name, other.column_name);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetOp()).compareTo(other.isSetOp());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetOp()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.op, other.op);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetValue()).compareTo(other.isSetValue());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetValue()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.value, other.value);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("IndexExpression(");
- boolean first = true;
-
- sb.append("column_name:");
- if (this.column_name == null) {
- sb.append("null");
- } else {
- org.apache.thrift.TBaseHelper.toString(this.column_name, sb);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("op:");
- if (this.op == null) {
- sb.append("null");
- } else {
- sb.append(this.op);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("value:");
- if (this.value == null) {
- sb.append("null");
- } else {
- org.apache.thrift.TBaseHelper.toString(this.value, sb);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- if (column_name == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'column_name' was not present! Struct: " + toString());
- }
- if (op == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'op' was not present! Struct: " + toString());
- }
- if (value == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'value' was not present! Struct: " + toString());
- }
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class IndexExpressionStandardSchemeFactory implements SchemeFactory {
- public IndexExpressionStandardScheme getScheme() {
- return new IndexExpressionStandardScheme();
- }
- }
-
- private static class IndexExpressionStandardScheme extends StandardScheme<IndexExpression> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, IndexExpression struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // COLUMN_NAME
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.column_name = iprot.readBinary();
- struct.setColumn_nameIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 2: // OP
- if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
- struct.op = IndexOperator.findByValue(iprot.readI32());
- struct.setOpIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 3: // VALUE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.value = iprot.readBinary();
- struct.setValueIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, IndexExpression struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.column_name != null) {
- oprot.writeFieldBegin(COLUMN_NAME_FIELD_DESC);
- oprot.writeBinary(struct.column_name);
- oprot.writeFieldEnd();
- }
- if (struct.op != null) {
- oprot.writeFieldBegin(OP_FIELD_DESC);
- oprot.writeI32(struct.op.getValue());
- oprot.writeFieldEnd();
- }
- if (struct.value != null) {
- oprot.writeFieldBegin(VALUE_FIELD_DESC);
- oprot.writeBinary(struct.value);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class IndexExpressionTupleSchemeFactory implements SchemeFactory {
- public IndexExpressionTupleScheme getScheme() {
- return new IndexExpressionTupleScheme();
- }
- }
-
- private static class IndexExpressionTupleScheme extends TupleScheme<IndexExpression> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, IndexExpression struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- oprot.writeBinary(struct.column_name);
- oprot.writeI32(struct.op.getValue());
- oprot.writeBinary(struct.value);
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, IndexExpression struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- struct.column_name = iprot.readBinary();
- struct.setColumn_nameIsSet(true);
- struct.op = IndexOperator.findByValue(iprot.readI32());
- struct.setOpIsSet(true);
- struct.value = iprot.readBinary();
- struct.setValueIsSet(true);
- }
- }
-
-}
-
diff --git a/interface/thrift/gen-java/org/apache/cassandra/thrift/IndexOperator.java b/interface/thrift/gen-java/org/apache/cassandra/thrift/IndexOperator.java
deleted file mode 100644
index 767d773..0000000
--- a/interface/thrift/gen-java/org/apache/cassandra/thrift/IndexOperator.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.1)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- * @generated
- */
-package org.apache.cassandra.thrift;
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
-
-
-import java.util.Map;
-import java.util.HashMap;
-import org.apache.thrift.TEnum;
-
-public enum IndexOperator implements org.apache.thrift.TEnum {
- EQ(0),
- GTE(1),
- GT(2),
- LTE(3),
- LT(4);
-
- private final int value;
-
- private IndexOperator(int value) {
- this.value = value;
- }
-
- /**
- * Get the integer value of this enum value, as defined in the Thrift IDL.
- */
- public int getValue() {
- return value;
- }
-
- /**
- * Find a the enum type by its integer value, as defined in the Thrift IDL.
- * @return null if the value is not found.
- */
- public static IndexOperator findByValue(int value) {
- switch (value) {
- case 0:
- return EQ;
- case 1:
- return GTE;
- case 2:
- return GT;
- case 3:
- return LTE;
- case 4:
- return LT;
- default:
- return null;
- }
- }
-}
diff --git a/interface/thrift/gen-java/org/apache/cassandra/thrift/IndexType.java b/interface/thrift/gen-java/org/apache/cassandra/thrift/IndexType.java
deleted file mode 100644
index e6a5e9b..0000000
--- a/interface/thrift/gen-java/org/apache/cassandra/thrift/IndexType.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.1)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- * @generated
- */
-package org.apache.cassandra.thrift;
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
-
-
-import java.util.Map;
-import java.util.HashMap;
-import org.apache.thrift.TEnum;
-
-public enum IndexType implements org.apache.thrift.TEnum {
- KEYS(0),
- CUSTOM(1),
- COMPOSITES(2);
-
- private final int value;
-
- private IndexType(int value) {
- this.value = value;
- }
-
- /**
- * Get the integer value of this enum value, as defined in the Thrift IDL.
- */
- public int getValue() {
- return value;
- }
-
- /**
- * Find a the enum type by its integer value, as defined in the Thrift IDL.
- * @return null if the value is not found.
- */
- public static IndexType findByValue(int value) {
- switch (value) {
- case 0:
- return KEYS;
- case 1:
- return CUSTOM;
- case 2:
- return COMPOSITES;
- default:
- return null;
- }
- }
-}
diff --git a/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java b/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java
deleted file mode 100644
index 6038a23..0000000
--- a/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java
+++ /dev/null
@@ -1,414 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.1)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- * @generated
- */
-package org.apache.cassandra.thrift;
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
-
-import org.apache.commons.lang3.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Invalid request could mean keyspace or column family does not exist, required parameters are missing, or a parameter is malformed.
- * why contains an associated error message.
- */
-public class InvalidRequestException extends TException implements org.apache.thrift.TBase<InvalidRequestException, InvalidRequestException._Fields>, java.io.Serializable, Cloneable, Comparable<InvalidRequestException> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("InvalidRequestException");
-
- private static final org.apache.thrift.protocol.TField WHY_FIELD_DESC = new org.apache.thrift.protocol.TField("why", org.apache.thrift.protocol.TType.STRING, (short)1);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new InvalidRequestExceptionStandardSchemeFactory());
- schemes.put(TupleScheme.class, new InvalidRequestExceptionTupleSchemeFactory());
- }
-
- public String why; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- WHY((short)1, "why");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // WHY
- return WHY;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.WHY, new org.apache.thrift.meta_data.FieldMetaData("why", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(InvalidRequestException.class, metaDataMap);
- }
-
- public InvalidRequestException() {
- }
-
- public InvalidRequestException(
- String why)
- {
- this();
- this.why = why;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public InvalidRequestException(InvalidRequestException other) {
- if (other.isSetWhy()) {
- this.why = other.why;
- }
- }
-
- public InvalidRequestException deepCopy() {
- return new InvalidRequestException(this);
- }
-
- @Override
- public void clear() {
- this.why = null;
- }
-
- public String getWhy() {
- return this.why;
- }
-
- public InvalidRequestException setWhy(String why) {
- this.why = why;
- return this;
- }
-
- public void unsetWhy() {
- this.why = null;
- }
-
- /** Returns true if field why is set (has been assigned a value) and false otherwise */
- public boolean isSetWhy() {
- return this.why != null;
- }
-
- public void setWhyIsSet(boolean value) {
- if (!value) {
- this.why = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case WHY:
- if (value == null) {
- unsetWhy();
- } else {
- setWhy((String)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case WHY:
- return getWhy();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case WHY:
- return isSetWhy();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof InvalidRequestException)
- return this.equals((InvalidRequestException)that);
- return false;
- }
-
- public boolean equals(InvalidRequestException that) {
- if (that == null)
- return false;
-
- boolean this_present_why = true && this.isSetWhy();
- boolean that_present_why = true && that.isSetWhy();
- if (this_present_why || that_present_why) {
- if (!(this_present_why && that_present_why))
- return false;
- if (!this.why.equals(that.why))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_why = true && (isSetWhy());
- builder.append(present_why);
- if (present_why)
- builder.append(why);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(InvalidRequestException other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetWhy()).compareTo(other.isSetWhy());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetWhy()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.why, other.why);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("InvalidRequestException(");
- boolean first = true;
-
- sb.append("why:");
- if (this.why == null) {
- sb.append("null");
- } else {
- sb.append(this.why);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- if (why == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'why' was not present! Struct: " + toString());
- }
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class InvalidRequestExceptionStandardSchemeFactory implements SchemeFactory {
- public InvalidRequestExceptionStandardScheme getScheme() {
- return new InvalidRequestExceptionStandardScheme();
- }
- }
-
- private static class InvalidRequestExceptionStandardScheme extends StandardScheme<InvalidRequestException> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, InvalidRequestException struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // WHY
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.why = iprot.readString();
- struct.setWhyIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, InvalidRequestException struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.why != null) {
- oprot.writeFieldBegin(WHY_FIELD_DESC);
- oprot.writeString(struct.why);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class InvalidRequestExceptionTupleSchemeFactory implements SchemeFactory {
- public InvalidRequestExceptionTupleScheme getScheme() {
- return new InvalidRequestExceptionTupleScheme();
- }
- }
-
- private static class InvalidRequestExceptionTupleScheme extends TupleScheme<InvalidRequestException> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, InvalidRequestException struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- oprot.writeString(struct.why);
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, InvalidRequestException struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- struct.why = iprot.readString();
- struct.setWhyIsSet(true);
- }
- }
-
-}
-
diff --git a/interface/thrift/gen-java/org/apache/cassandra/thrift/KeyCount.java b/interface/thrift/gen-java/org/apache/cassandra/thrift/KeyCount.java
deleted file mode 100644
index cbb5e51..0000000
--- a/interface/thrift/gen-java/org/apache/cassandra/thrift/KeyCount.java
+++ /dev/null
@@ -1,521 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.1)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- * @generated
- */
-package org.apache.cassandra.thrift;
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
-
-import org.apache.commons.lang3.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class KeyCount implements org.apache.thrift.TBase<KeyCount, KeyCount._Fields>, java.io.Serializable, Cloneable, Comparable<KeyCount> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("KeyCount");
-
- private static final org.apache.thrift.protocol.TField KEY_FIELD_DESC = new org.apache.thrift.protocol.TField("key", org.apache.thrift.protocol.TType.STRING, (short)1);
- private static final org.apache.thrift.protocol.TField COUNT_FIELD_DESC = new org.apache.thrift.protocol.TField("count", org.apache.thrift.protocol.TType.I32, (short)2);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new KeyCountStandardSchemeFactory());
- schemes.put(TupleScheme.class, new KeyCountTupleSchemeFactory());
- }
-
- public ByteBuffer key; // required
- public int count; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- KEY((short)1, "key"),
- COUNT((short)2, "count");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // KEY
- return KEY;
- case 2: // COUNT
- return COUNT;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- private static final int __COUNT_ISSET_ID = 0;
- private byte __isset_bitfield = 0;
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.KEY, new org.apache.thrift.meta_data.FieldMetaData("key", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true)));
- tmpMap.put(_Fields.COUNT, new org.apache.thrift.meta_data.FieldMetaData("count", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(KeyCount.class, metaDataMap);
- }
-
- public KeyCount() {
- }
-
- public KeyCount(
- ByteBuffer key,
- int count)
- {
- this();
- this.key = key;
- this.count = count;
- setCountIsSet(true);
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public KeyCount(KeyCount other) {
- __isset_bitfield = other.__isset_bitfield;
- if (other.isSetKey()) {
- this.key = org.apache.thrift.TBaseHelper.copyBinary(other.key);
-;
- }
- this.count = other.count;
- }
-
- public KeyCount deepCopy() {
- return new KeyCount(this);
- }
-
- @Override
- public void clear() {
- this.key = null;
- setCountIsSet(false);
- this.count = 0;
- }
-
- public byte[] getKey() {
- setKey(org.apache.thrift.TBaseHelper.rightSize(key));
- return key == null ? null : key.array();
- }
-
- public ByteBuffer bufferForKey() {
- return key;
- }
-
- public KeyCount setKey(byte[] key) {
- setKey(key == null ? (ByteBuffer)null : ByteBuffer.wrap(key));
- return this;
- }
-
- public KeyCount setKey(ByteBuffer key) {
- this.key = key;
- return this;
- }
-
- public void unsetKey() {
- this.key = null;
- }
-
- /** Returns true if field key is set (has been assigned a value) and false otherwise */
- public boolean isSetKey() {
- return this.key != null;
- }
-
- public void setKeyIsSet(boolean value) {
- if (!value) {
- this.key = null;
- }
- }
-
- public int getCount() {
- return this.count;
- }
-
- public KeyCount setCount(int count) {
- this.count = count;
- setCountIsSet(true);
- return this;
- }
-
- public void unsetCount() {
- __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __COUNT_ISSET_ID);
- }
-
- /** Returns true if field count is set (has been assigned a value) and false otherwise */
- public boolean isSetCount() {
- return EncodingUtils.testBit(__isset_bitfield, __COUNT_ISSET_ID);
- }
-
- public void setCountIsSet(boolean value) {
- __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __COUNT_ISSET_ID, value);
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case KEY:
- if (value == null) {
- unsetKey();
- } else {
- setKey((ByteBuffer)value);
- }
- break;
-
- case COUNT:
- if (value == null) {
- unsetCount();
- } else {
- setCount((Integer)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case KEY:
- return getKey();
-
- case COUNT:
- return Integer.valueOf(getCount());
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case KEY:
- return isSetKey();
- case COUNT:
- return isSetCount();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof KeyCount)
- return this.equals((KeyCount)that);
- return false;
- }
-
- public boolean equals(KeyCount that) {
- if (that == null)
- return false;
-
- boolean this_present_key = true && this.isSetKey();
- boolean that_present_key = true && that.isSetKey();
- if (this_present_key || that_present_key) {
- if (!(this_present_key && that_present_key))
- return false;
- if (!this.key.equals(that.key))
- return false;
- }
-
- boolean this_present_count = true;
- boolean that_present_count = true;
- if (this_present_count || that_present_count) {
- if (!(this_present_count && that_present_count))
- return false;
- if (this.count != that.count)
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_key = true && (isSetKey());
- builder.append(present_key);
- if (present_key)
- builder.append(key);
-
- boolean present_count = true;
- builder.append(present_count);
- if (present_count)
- builder.append(count);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(KeyCount other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetKey()).compareTo(other.isSetKey());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetKey()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.key, other.key);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetCount()).compareTo(other.isSetCount());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetCount()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.count, other.count);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("KeyCount(");
- boolean first = true;
-
- sb.append("key:");
- if (this.key == null) {
- sb.append("null");
- } else {
- org.apache.thrift.TBaseHelper.toString(this.key, sb);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("count:");
- sb.append(this.count);
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- if (key == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'key' was not present! Struct: " + toString());
- }
- // alas, we cannot check 'count' because it's a primitive and you chose the non-beans generator.
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
- __isset_bitfield = 0;
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class KeyCountStandardSchemeFactory implements SchemeFactory {
- public KeyCountStandardScheme getScheme() {
- return new KeyCountStandardScheme();
- }
- }
-
- private static class KeyCountStandardScheme extends StandardScheme<KeyCount> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, KeyCount struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // KEY
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.key = iprot.readBinary();
- struct.setKeyIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 2: // COUNT
- if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
- struct.count = iprot.readI32();
- struct.setCountIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- if (!struct.isSetCount()) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'count' was not found in serialized data! Struct: " + toString());
- }
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, KeyCount struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.key != null) {
- oprot.writeFieldBegin(KEY_FIELD_DESC);
- oprot.writeBinary(struct.key);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldBegin(COUNT_FIELD_DESC);
- oprot.writeI32(struct.count);
- oprot.writeFieldEnd();
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class KeyCountTupleSchemeFactory implements SchemeFactory {
- public KeyCountTupleScheme getScheme() {
- return new KeyCountTupleScheme();
- }
- }
-
- private static class KeyCountTupleScheme extends TupleScheme<KeyCount> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, KeyCount struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- oprot.writeBinary(struct.key);
- oprot.writeI32(struct.count);
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, KeyCount struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- struct.key = iprot.readBinary();
- struct.setKeyIsSet(true);
- struct.count = iprot.readI32();
- struct.setCountIsSet(true);
- }
- }
-
-}
-
diff --git a/interface/thrift/gen-java/org/apache/cassandra/thrift/KeyRange.java b/interface/thrift/gen-java/org/apache/cassandra/thrift/KeyRange.java
deleted file mode 100644
index 0168410..0000000
--- a/interface/thrift/gen-java/org/apache/cassandra/thrift/KeyRange.java
+++ /dev/null
@@ -1,1034 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.1)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- * @generated
- */
-package org.apache.cassandra.thrift;
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
-
-import org.apache.commons.lang3.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * The semantics of start keys and tokens are slightly different.
- * Keys are start-inclusive; tokens are start-exclusive. Token
- * ranges may also wrap -- that is, the end token may be less
- * than the start one. Thus, a range from keyX to keyX is a
- * one-element range, but a range from tokenY to tokenY is the
- * full ring.
- */
-public class KeyRange implements org.apache.thrift.TBase<KeyRange, KeyRange._Fields>, java.io.Serializable, Cloneable, Comparable<KeyRange> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("KeyRange");
-
- private static final org.apache.thrift.protocol.TField START_KEY_FIELD_DESC = new org.apache.thrift.protocol.TField("start_key", org.apache.thrift.protocol.TType.STRING, (short)1);
- private static final org.apache.thrift.protocol.TField END_KEY_FIELD_DESC = new org.apache.thrift.protocol.TField("end_key", org.apache.thrift.protocol.TType.STRING, (short)2);
- private static final org.apache.thrift.protocol.TField START_TOKEN_FIELD_DESC = new org.apache.thrift.protocol.TField("start_token", org.apache.thrift.protocol.TType.STRING, (short)3);
- private static final org.apache.thrift.protocol.TField END_TOKEN_FIELD_DESC = new org.apache.thrift.protocol.TField("end_token", org.apache.thrift.protocol.TType.STRING, (short)4);
- private static final org.apache.thrift.protocol.TField ROW_FILTER_FIELD_DESC = new org.apache.thrift.protocol.TField("row_filter", org.apache.thrift.protocol.TType.LIST, (short)6);
- private static final org.apache.thrift.protocol.TField COUNT_FIELD_DESC = new org.apache.thrift.protocol.TField("count", org.apache.thrift.protocol.TType.I32, (short)5);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new KeyRangeStandardSchemeFactory());
- schemes.put(TupleScheme.class, new KeyRangeTupleSchemeFactory());
- }
-
- public ByteBuffer start_key; // optional
- public ByteBuffer end_key; // optional
- public String start_token; // optional
- public String end_token; // optional
- public List<IndexExpression> row_filter; // optional
- public int count; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- START_KEY((short)1, "start_key"),
- END_KEY((short)2, "end_key"),
- START_TOKEN((short)3, "start_token"),
- END_TOKEN((short)4, "end_token"),
- ROW_FILTER((short)6, "row_filter"),
- COUNT((short)5, "count");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // START_KEY
- return START_KEY;
- case 2: // END_KEY
- return END_KEY;
- case 3: // START_TOKEN
- return START_TOKEN;
- case 4: // END_TOKEN
- return END_TOKEN;
- case 6: // ROW_FILTER
- return ROW_FILTER;
- case 5: // COUNT
- return COUNT;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- private static final int __COUNT_ISSET_ID = 0;
- private byte __isset_bitfield = 0;
- private _Fields optionals[] = {_Fields.START_KEY,_Fields.END_KEY,_Fields.START_TOKEN,_Fields.END_TOKEN,_Fields.ROW_FILTER};
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.START_KEY, new org.apache.thrift.meta_data.FieldMetaData("start_key", org.apache.thrift.TFieldRequirementType.OPTIONAL,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true)));
- tmpMap.put(_Fields.END_KEY, new org.apache.thrift.meta_data.FieldMetaData("end_key", org.apache.thrift.TFieldRequirementType.OPTIONAL,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true)));
- tmpMap.put(_Fields.START_TOKEN, new org.apache.thrift.meta_data.FieldMetaData("start_token", org.apache.thrift.TFieldRequirementType.OPTIONAL,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
- tmpMap.put(_Fields.END_TOKEN, new org.apache.thrift.meta_data.FieldMetaData("end_token", org.apache.thrift.TFieldRequirementType.OPTIONAL,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
- tmpMap.put(_Fields.ROW_FILTER, new org.apache.thrift.meta_data.FieldMetaData("row_filter", org.apache.thrift.TFieldRequirementType.OPTIONAL,
- new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST,
- new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, IndexExpression.class))));
- tmpMap.put(_Fields.COUNT, new org.apache.thrift.meta_data.FieldMetaData("count", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(KeyRange.class, metaDataMap);
- }
-
- public KeyRange() {
- this.count = 100;
-
- }
-
- public KeyRange(
- int count)
- {
- this();
- this.count = count;
- setCountIsSet(true);
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public KeyRange(KeyRange other) {
- __isset_bitfield = other.__isset_bitfield;
- if (other.isSetStart_key()) {
- this.start_key = org.apache.thrift.TBaseHelper.copyBinary(other.start_key);
-;
- }
- if (other.isSetEnd_key()) {
- this.end_key = org.apache.thrift.TBaseHelper.copyBinary(other.end_key);
-;
- }
- if (other.isSetStart_token()) {
- this.start_token = other.start_token;
- }
- if (other.isSetEnd_token()) {
- this.end_token = other.end_token;
- }
- if (other.isSetRow_filter()) {
- List<IndexExpression> __this__row_filter = new ArrayList<IndexExpression>(other.row_filter.size());
- for (IndexExpression other_element : other.row_filter) {
- __this__row_filter.add(new IndexExpression(other_element));
- }
- this.row_filter = __this__row_filter;
- }
- this.count = other.count;
- }
-
- public KeyRange deepCopy() {
- return new KeyRange(this);
- }
-
- @Override
- public void clear() {
- this.start_key = null;
- this.end_key = null;
- this.start_token = null;
- this.end_token = null;
- this.row_filter = null;
- this.count = 100;
-
- }
-
- public byte[] getStart_key() {
- setStart_key(org.apache.thrift.TBaseHelper.rightSize(start_key));
- return start_key == null ? null : start_key.array();
- }
-
- public ByteBuffer bufferForStart_key() {
- return start_key;
- }
-
- public KeyRange setStart_key(byte[] start_key) {
- setStart_key(start_key == null ? (ByteBuffer)null : ByteBuffer.wrap(start_key));
- return this;
- }
-
- public KeyRange setStart_key(ByteBuffer start_key) {
- this.start_key = start_key;
- return this;
- }
-
- public void unsetStart_key() {
- this.start_key = null;
- }
-
- /** Returns true if field start_key is set (has been assigned a value) and false otherwise */
- public boolean isSetStart_key() {
- return this.start_key != null;
- }
-
- public void setStart_keyIsSet(boolean value) {
- if (!value) {
- this.start_key = null;
- }
- }
-
- public byte[] getEnd_key() {
- setEnd_key(org.apache.thrift.TBaseHelper.rightSize(end_key));
- return end_key == null ? null : end_key.array();
- }
-
- public ByteBuffer bufferForEnd_key() {
- return end_key;
- }
-
- public KeyRange setEnd_key(byte[] end_key) {
- setEnd_key(end_key == null ? (ByteBuffer)null : ByteBuffer.wrap(end_key));
- return this;
- }
-
- public KeyRange setEnd_key(ByteBuffer end_key) {
- this.end_key = end_key;
- return this;
- }
-
- public void unsetEnd_key() {
- this.end_key = null;
- }
-
- /** Returns true if field end_key is set (has been assigned a value) and false otherwise */
- public boolean isSetEnd_key() {
- return this.end_key != null;
- }
-
- public void setEnd_keyIsSet(boolean value) {
- if (!value) {
- this.end_key = null;
- }
- }
-
- public String getStart_token() {
- return this.start_token;
- }
-
- public KeyRange setStart_token(String start_token) {
- this.start_token = start_token;
- return this;
- }
-
- public void unsetStart_token() {
- this.start_token = null;
- }
-
- /** Returns true if field start_token is set (has been assigned a value) and false otherwise */
- public boolean isSetStart_token() {
- return this.start_token != null;
- }
-
- public void setStart_tokenIsSet(boolean value) {
- if (!value) {
- this.start_token = null;
- }
- }
-
- public String getEnd_token() {
- return this.end_token;
- }
-
- public KeyRange setEnd_token(String end_token) {
- this.end_token = end_token;
- return this;
- }
-
- public void unsetEnd_token() {
- this.end_token = null;
- }
-
- /** Returns true if field end_token is set (has been assigned a value) and false otherwise */
- public boolean isSetEnd_token() {
- return this.end_token != null;
- }
-
- public void setEnd_tokenIsSet(boolean value) {
- if (!value) {
- this.end_token = null;
- }
- }
-
- public int getRow_filterSize() {
- return (this.row_filter == null) ? 0 : this.row_filter.size();
- }
-
- public java.util.Iterator<IndexExpression> getRow_filterIterator() {
- return (this.row_filter == null) ? null : this.row_filter.iterator();
- }
-
- public void addToRow_filter(IndexExpression elem) {
- if (this.row_filter == null) {
- this.row_filter = new ArrayList<IndexExpression>();
- }
- this.row_filter.add(elem);
- }
-
- public List<IndexExpression> getRow_filter() {
- return this.row_filter;
- }
-
- public KeyRange setRow_filter(List<IndexExpression> row_filter) {
- this.row_filter = row_filter;
- return this;
- }
-
- public void unsetRow_filter() {
- this.row_filter = null;
- }
-
- /** Returns true if field row_filter is set (has been assigned a value) and false otherwise */
- public boolean isSetRow_filter() {
- return this.row_filter != null;
- }
-
- public void setRow_filterIsSet(boolean value) {
- if (!value) {
- this.row_filter = null;
- }
- }
-
- public int getCount() {
- return this.count;
- }
-
- public KeyRange setCount(int count) {
- this.count = count;
- setCountIsSet(true);
- return this;
- }
-
- public void unsetCount() {
- __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __COUNT_ISSET_ID);
- }
-
- /** Returns true if field count is set (has been assigned a value) and false otherwise */
- public boolean isSetCount() {
- return EncodingUtils.testBit(__isset_bitfield, __COUNT_ISSET_ID);
- }
-
- public void setCountIsSet(boolean value) {
- __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __COUNT_ISSET_ID, value);
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case START_KEY:
- if (value == null) {
- unsetStart_key();
- } else {
- setStart_key((ByteBuffer)value);
- }
- break;
-
- case END_KEY:
- if (value == null) {
- unsetEnd_key();
- } else {
- setEnd_key((ByteBuffer)value);
- }
- break;
-
- case START_TOKEN:
- if (value == null) {
- unsetStart_token();
- } else {
- setStart_token((String)value);
- }
- break;
-
- case END_TOKEN:
- if (value == null) {
- unsetEnd_token();
- } else {
- setEnd_token((String)value);
- }
- break;
-
- case ROW_FILTER:
- if (value == null) {
- unsetRow_filter();
- } else {
- setRow_filter((List<IndexExpression>)value);
- }
- break;
-
- case COUNT:
- if (value == null) {
- unsetCount();
- } else {
- setCount((Integer)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case START_KEY:
- return getStart_key();
-
- case END_KEY:
- return getEnd_key();
-
- case START_TOKEN:
- return getStart_token();
-
- case END_TOKEN:
- return getEnd_token();
-
- case ROW_FILTER:
- return getRow_filter();
-
- case COUNT:
- return Integer.valueOf(getCount());
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case START_KEY:
- return isSetStart_key();
- case END_KEY:
- return isSetEnd_key();
- case START_TOKEN:
- return isSetStart_token();
- case END_TOKEN:
- return isSetEnd_token();
- case ROW_FILTER:
- return isSetRow_filter();
- case COUNT:
- return isSetCount();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof KeyRange)
- return this.equals((KeyRange)that);
- return false;
- }
-
- public boolean equals(KeyRange that) {
- if (that == null)
- return false;
-
- boolean this_present_start_key = true && this.isSetStart_key();
- boolean that_present_start_key = true && that.isSetStart_key();
- if (this_present_start_key || that_present_start_key) {
- if (!(this_present_start_key && that_present_start_key))
- return false;
- if (!this.start_key.equals(that.start_key))
- return false;
- }
-
- boolean this_present_end_key = true && this.isSetEnd_key();
- boolean that_present_end_key = true && that.isSetEnd_key();
- if (this_present_end_key || that_present_end_key) {
- if (!(this_present_end_key && that_present_end_key))
- return false;
- if (!this.end_key.equals(that.end_key))
- return false;
- }
-
- boolean this_present_start_token = true && this.isSetStart_token();
- boolean that_present_start_token = true && that.isSetStart_token();
- if (this_present_start_token || that_present_start_token) {
- if (!(this_present_start_token && that_present_start_token))
- return false;
- if (!this.start_token.equals(that.start_token))
- return false;
- }
-
- boolean this_present_end_token = true && this.isSetEnd_token();
- boolean that_present_end_token = true && that.isSetEnd_token();
- if (this_present_end_token || that_present_end_token) {
- if (!(this_present_end_token && that_present_end_token))
- return false;
- if (!this.end_token.equals(that.end_token))
- return false;
- }
-
- boolean this_present_row_filter = true && this.isSetRow_filter();
- boolean that_present_row_filter = true && that.isSetRow_filter();
- if (this_present_row_filter || that_present_row_filter) {
- if (!(this_present_row_filter && that_present_row_filter))
- return false;
- if (!this.row_filter.equals(that.row_filter))
- return false;
- }
-
- boolean this_present_count = true;
- boolean that_present_count = true;
- if (this_present_count || that_present_count) {
- if (!(this_present_count && that_present_count))
- return false;
- if (this.count != that.count)
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_start_key = true && (isSetStart_key());
- builder.append(present_start_key);
- if (present_start_key)
- builder.append(start_key);
-
- boolean present_end_key = true && (isSetEnd_key());
- builder.append(present_end_key);
- if (present_end_key)
- builder.append(end_key);
-
- boolean present_start_token = true && (isSetStart_token());
- builder.append(present_start_token);
- if (present_start_token)
- builder.append(start_token);
-
- boolean present_end_token = true && (isSetEnd_token());
- builder.append(present_end_token);
- if (present_end_token)
- builder.append(end_token);
-
- boolean present_row_filter = true && (isSetRow_filter());
- builder.append(present_row_filter);
- if (present_row_filter)
- builder.append(row_filter);
-
- boolean present_count = true;
- builder.append(present_count);
- if (present_count)
- builder.append(count);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(KeyRange other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetStart_key()).compareTo(other.isSetStart_key());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetStart_key()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.start_key, other.start_key);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetEnd_key()).compareTo(other.isSetEnd_key());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetEnd_key()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.end_key, other.end_key);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetStart_token()).compareTo(other.isSetStart_token());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetStart_token()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.start_token, other.start_token);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetEnd_token()).compareTo(other.isSetEnd_token());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetEnd_token()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.end_token, other.end_token);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetRow_filter()).compareTo(other.isSetRow_filter());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetRow_filter()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.row_filter, other.row_filter);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetCount()).compareTo(other.isSetCount());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetCount()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.count, other.count);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("KeyRange(");
- boolean first = true;
-
- if (isSetStart_key()) {
- sb.append("start_key:");
- if (this.start_key == null) {
- sb.append("null");
- } else {
- org.apache.thrift.TBaseHelper.toString(this.start_key, sb);
- }
- first = false;
- }
- if (isSetEnd_key()) {
- if (!first) sb.append(", ");
- sb.append("end_key:");
- if (this.end_key == null) {
- sb.append("null");
- } else {
- org.apache.thrift.TBaseHelper.toString(this.end_key, sb);
- }
- first = false;
- }
- if (isSetStart_token()) {
- if (!first) sb.append(", ");
- sb.append("start_token:");
- if (this.start_token == null) {
- sb.append("null");
- } else {
- sb.append(this.start_token);
- }
- first = false;
- }
- if (isSetEnd_token()) {
- if (!first) sb.append(", ");
- sb.append("end_token:");
- if (this.end_token == null) {
- sb.append("null");
- } else {
- sb.append(this.end_token);
- }
- first = false;
- }
- if (isSetRow_filter()) {
- if (!first) sb.append(", ");
- sb.append("row_filter:");
- if (this.row_filter == null) {
- sb.append("null");
- } else {
- sb.append(this.row_filter);
- }
- first = false;
- }
- if (!first) sb.append(", ");
- sb.append("count:");
- sb.append(this.count);
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // alas, we cannot check 'count' because it's a primitive and you chose the non-beans generator.
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
- __isset_bitfield = 0;
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class KeyRangeStandardSchemeFactory implements SchemeFactory {
- public KeyRangeStandardScheme getScheme() {
- return new KeyRangeStandardScheme();
- }
- }
-
- private static class KeyRangeStandardScheme extends StandardScheme<KeyRange> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, KeyRange struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // START_KEY
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.start_key = iprot.readBinary();
- struct.setStart_keyIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 2: // END_KEY
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.end_key = iprot.readBinary();
- struct.setEnd_keyIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 3: // START_TOKEN
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.start_token = iprot.readString();
- struct.setStart_tokenIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 4: // END_TOKEN
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.end_token = iprot.readString();
- struct.setEnd_tokenIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 6: // ROW_FILTER
- if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
- {
- org.apache.thrift.protocol.TList _list32 = iprot.readListBegin();
- struct.row_filter = new ArrayList<IndexExpression>(_list32.size);
- for (int _i33 = 0; _i33 < _list32.size; ++_i33)
- {
- IndexExpression _elem34;
- _elem34 = new IndexExpression();
- _elem34.read(iprot);
- struct.row_filter.add(_elem34);
- }
- iprot.readListEnd();
- }
- struct.setRow_filterIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 5: // COUNT
- if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
- struct.count = iprot.readI32();
- struct.setCountIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- if (!struct.isSetCount()) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'count' was not found in serialized data! Struct: " + toString());
- }
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, KeyRange struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.start_key != null) {
- if (struct.isSetStart_key()) {
- oprot.writeFieldBegin(START_KEY_FIELD_DESC);
- oprot.writeBinary(struct.start_key);
- oprot.writeFieldEnd();
- }
- }
- if (struct.end_key != null) {
- if (struct.isSetEnd_key()) {
- oprot.writeFieldBegin(END_KEY_FIELD_DESC);
- oprot.writeBinary(struct.end_key);
- oprot.writeFieldEnd();
- }
- }
- if (struct.start_token != null) {
- if (struct.isSetStart_token()) {
- oprot.writeFieldBegin(START_TOKEN_FIELD_DESC);
- oprot.writeString(struct.start_token);
- oprot.writeFieldEnd();
- }
- }
- if (struct.end_token != null) {
- if (struct.isSetEnd_token()) {
- oprot.writeFieldBegin(END_TOKEN_FIELD_DESC);
- oprot.writeString(struct.end_token);
- oprot.writeFieldEnd();
- }
- }
- oprot.writeFieldBegin(COUNT_FIELD_DESC);
- oprot.writeI32(struct.count);
- oprot.writeFieldEnd();
- if (struct.row_filter != null) {
- if (struct.isSetRow_filter()) {
- oprot.writeFieldBegin(ROW_FILTER_FIELD_DESC);
- {
- oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.row_filter.size()));
- for (IndexExpression _iter35 : struct.row_filter)
- {
- _iter35.write(oprot);
- }
- oprot.writeListEnd();
- }
- oprot.writeFieldEnd();
- }
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class KeyRangeTupleSchemeFactory implements SchemeFactory {
- public KeyRangeTupleScheme getScheme() {
- return new KeyRangeTupleScheme();
- }
- }
-
- private static class KeyRangeTupleScheme extends TupleScheme<KeyRange> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, KeyRange struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- oprot.writeI32(struct.count);
- BitSet optionals = new BitSet();
- if (struct.isSetStart_key()) {
- optionals.set(0);
- }
- if (struct.isSetEnd_key()) {
- optionals.set(1);
- }
- if (struct.isSetStart_token()) {
- optionals.set(2);
- }
- if (struct.isSetEnd_token()) {
- optionals.set(3);
- }
- if (struct.isSetRow_filter()) {
- optionals.set(4);
- }
- oprot.writeBitSet(optionals, 5);
- if (struct.isSetStart_key()) {
- oprot.writeBinary(struct.start_key);
- }
- if (struct.isSetEnd_key()) {
- oprot.writeBinary(struct.end_key);
- }
- if (struct.isSetStart_token()) {
- oprot.writeString(struct.start_token);
- }
- if (struct.isSetEnd_token()) {
- oprot.writeString(struct.end_token);
- }
- if (struct.isSetRow_filter()) {
- {
- oprot.writeI32(struct.row_filter.size());
- for (IndexExpression _iter36 : struct.row_filter)
- {
- _iter36.write(oprot);
- }
- }
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, KeyRange struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- struct.count = iprot.readI32();
- struct.setCountIsSet(true);
- BitSet incoming = iprot.readBitSet(5);
- if (incoming.get(0)) {
- struct.start_key = iprot.readBinary();
- struct.setStart_keyIsSet(true);
- }
- if (incoming.get(1)) {
- struct.end_key = iprot.readBinary();
- struct.setEnd_keyIsSet(true);
- }
- if (incoming.get(2)) {
- struct.start_token = iprot.readString();
- struct.setStart_tokenIsSet(true);
- }
- if (incoming.get(3)) {
- struct.end_token = iprot.readString();
- struct.setEnd_tokenIsSet(true);
- }
- if (incoming.get(4)) {
- {
- org.apache.thrift.protocol.TList _list37 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
- struct.row_filter = new ArrayList<IndexExpression>(_list37.size);
- for (int _i38 = 0; _i38 < _list37.size; ++_i38)
- {
- IndexExpression _elem39;
- _elem39 = new IndexExpression();
- _elem39.read(iprot);
- struct.row_filter.add(_elem39);
- }
- }
- struct.setRow_filterIsSet(true);
- }
- }
- }
-
-}
-
diff --git a/interface/thrift/gen-java/org/apache/cassandra/thrift/KeySlice.java b/interface/thrift/gen-java/org/apache/cassandra/thrift/KeySlice.java
deleted file mode 100644
index df4beb1..0000000
--- a/interface/thrift/gen-java/org/apache/cassandra/thrift/KeySlice.java
+++ /dev/null
@@ -1,583 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.1)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- * @generated
- */
-package org.apache.cassandra.thrift;
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
-
-import org.apache.commons.lang3.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * A KeySlice is key followed by the data it maps to. A collection of KeySlice is returned by the get_range_slice operation.
- *
- * @param key. a row key
- * @param columns. List of data represented by the key. Typically, the list is pared down to only the columns specified by
- * a SlicePredicate.
- */
-public class KeySlice implements org.apache.thrift.TBase<KeySlice, KeySlice._Fields>, java.io.Serializable, Cloneable, Comparable<KeySlice> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("KeySlice");
-
- private static final org.apache.thrift.protocol.TField KEY_FIELD_DESC = new org.apache.thrift.protocol.TField("key", org.apache.thrift.protocol.TType.STRING, (short)1);
- private static final org.apache.thrift.protocol.TField COLUMNS_FIELD_DESC = new org.apache.thrift.protocol.TField("columns", org.apache.thrift.protocol.TType.LIST, (short)2);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new KeySliceStandardSchemeFactory());
- schemes.put(TupleScheme.class, new KeySliceTupleSchemeFactory());
- }
-
- public ByteBuffer key; // required
- public List<ColumnOrSuperColumn> columns; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- KEY((short)1, "key"),
- COLUMNS((short)2, "columns");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // KEY
- return KEY;
- case 2: // COLUMNS
- return COLUMNS;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.KEY, new org.apache.thrift.meta_data.FieldMetaData("key", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true)));
- tmpMap.put(_Fields.COLUMNS, new org.apache.thrift.meta_data.FieldMetaData("columns", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST,
- new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, ColumnOrSuperColumn.class))));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(KeySlice.class, metaDataMap);
- }
-
- public KeySlice() {
- }
-
- public KeySlice(
- ByteBuffer key,
- List<ColumnOrSuperColumn> columns)
- {
- this();
- this.key = key;
- this.columns = columns;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public KeySlice(KeySlice other) {
- if (other.isSetKey()) {
- this.key = org.apache.thrift.TBaseHelper.copyBinary(other.key);
-;
- }
- if (other.isSetColumns()) {
- List<ColumnOrSuperColumn> __this__columns = new ArrayList<ColumnOrSuperColumn>(other.columns.size());
- for (ColumnOrSuperColumn other_element : other.columns) {
- __this__columns.add(new ColumnOrSuperColumn(other_element));
- }
- this.columns = __this__columns;
- }
- }
-
- public KeySlice deepCopy() {
- return new KeySlice(this);
- }
-
- @Override
- public void clear() {
- this.key = null;
- this.columns = null;
- }
-
- public byte[] getKey() {
- setKey(org.apache.thrift.TBaseHelper.rightSize(key));
- return key == null ? null : key.array();
- }
-
- public ByteBuffer bufferForKey() {
- return key;
- }
-
- public KeySlice setKey(byte[] key) {
- setKey(key == null ? (ByteBuffer)null : ByteBuffer.wrap(key));
- return this;
- }
-
- public KeySlice setKey(ByteBuffer key) {
- this.key = key;
- return this;
- }
-
- public void unsetKey() {
- this.key = null;
- }
-
- /** Returns true if field key is set (has been assigned a value) and false otherwise */
- public boolean isSetKey() {
- return this.key != null;
- }
-
- public void setKeyIsSet(boolean value) {
- if (!value) {
- this.key = null;
- }
- }
-
- public int getColumnsSize() {
- return (this.columns == null) ? 0 : this.columns.size();
- }
-
- public java.util.Iterator<ColumnOrSuperColumn> getColumnsIterator() {
- return (this.columns == null) ? null : this.columns.iterator();
- }
-
- public void addToColumns(ColumnOrSuperColumn elem) {
- if (this.columns == null) {
- this.columns = new ArrayList<ColumnOrSuperColumn>();
- }
- this.columns.add(elem);
- }
-
- public List<ColumnOrSuperColumn> getColumns() {
- return this.columns;
- }
-
- public KeySlice setColumns(List<ColumnOrSuperColumn> columns) {
- this.columns = columns;
- return this;
- }
-
- public void unsetColumns() {
- this.columns = null;
- }
-
- /** Returns true if field columns is set (has been assigned a value) and false otherwise */
- public boolean isSetColumns() {
- return this.columns != null;
- }
-
- public void setColumnsIsSet(boolean value) {
- if (!value) {
- this.columns = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case KEY:
- if (value == null) {
- unsetKey();
- } else {
- setKey((ByteBuffer)value);
- }
- break;
-
- case COLUMNS:
- if (value == null) {
- unsetColumns();
- } else {
- setColumns((List<ColumnOrSuperColumn>)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case KEY:
- return getKey();
-
- case COLUMNS:
- return getColumns();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case KEY:
- return isSetKey();
- case COLUMNS:
- return isSetColumns();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof KeySlice)
- return this.equals((KeySlice)that);
- return false;
- }
-
- public boolean equals(KeySlice that) {
- if (that == null)
- return false;
-
- boolean this_present_key = true && this.isSetKey();
- boolean that_present_key = true && that.isSetKey();
- if (this_present_key || that_present_key) {
- if (!(this_present_key && that_present_key))
- return false;
- if (!this.key.equals(that.key))
- return false;
- }
-
- boolean this_present_columns = true && this.isSetColumns();
- boolean that_present_columns = true && that.isSetColumns();
- if (this_present_columns || that_present_columns) {
- if (!(this_present_columns && that_present_columns))
- return false;
- if (!this.columns.equals(that.columns))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_key = true && (isSetKey());
- builder.append(present_key);
- if (present_key)
- builder.append(key);
-
- boolean present_columns = true && (isSetColumns());
- builder.append(present_columns);
- if (present_columns)
- builder.append(columns);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(KeySlice other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetKey()).compareTo(other.isSetKey());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetKey()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.key, other.key);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetColumns()).compareTo(other.isSetColumns());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetColumns()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.columns, other.columns);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("KeySlice(");
- boolean first = true;
-
- sb.append("key:");
- if (this.key == null) {
- sb.append("null");
- } else {
- org.apache.thrift.TBaseHelper.toString(this.key, sb);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("columns:");
- if (this.columns == null) {
- sb.append("null");
- } else {
- sb.append(this.columns);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- if (key == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'key' was not present! Struct: " + toString());
- }
- if (columns == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'columns' was not present! Struct: " + toString());
- }
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class KeySliceStandardSchemeFactory implements SchemeFactory {
- public KeySliceStandardScheme getScheme() {
- return new KeySliceStandardScheme();
- }
- }
-
- private static class KeySliceStandardScheme extends StandardScheme<KeySlice> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, KeySlice struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // KEY
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.key = iprot.readBinary();
- struct.setKeyIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 2: // COLUMNS
- if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
- {
- org.apache.thrift.protocol.TList _list40 = iprot.readListBegin();
- struct.columns = new ArrayList<ColumnOrSuperColumn>(_list40.size);
- for (int _i41 = 0; _i41 < _list40.size; ++_i41)
- {
- ColumnOrSuperColumn _elem42;
- _elem42 = new ColumnOrSuperColumn();
- _elem42.read(iprot);
- struct.columns.add(_elem42);
- }
- iprot.readListEnd();
- }
- struct.setColumnsIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, KeySlice struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.key != null) {
- oprot.writeFieldBegin(KEY_FIELD_DESC);
- oprot.writeBinary(struct.key);
- oprot.writeFieldEnd();
- }
- if (struct.columns != null) {
- oprot.writeFieldBegin(COLUMNS_FIELD_DESC);
- {
- oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.columns.size()));
- for (ColumnOrSuperColumn _iter43 : struct.columns)
- {
- _iter43.write(oprot);
- }
- oprot.writeListEnd();
- }
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class KeySliceTupleSchemeFactory implements SchemeFactory {
- public KeySliceTupleScheme getScheme() {
- return new KeySliceTupleScheme();
- }
- }
-
- private static class KeySliceTupleScheme extends TupleScheme<KeySlice> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, KeySlice struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- oprot.writeBinary(struct.key);
- {
- oprot.writeI32(struct.columns.size());
- for (ColumnOrSuperColumn _iter44 : struct.columns)
- {
- _iter44.write(oprot);
- }
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, KeySlice struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- struct.key = iprot.readBinary();
- struct.setKeyIsSet(true);
- {
- org.apache.thrift.protocol.TList _list45 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
- struct.columns = new ArrayList<ColumnOrSuperColumn>(_list45.size);
- for (int _i46 = 0; _i46 < _list45.size; ++_i46)
- {
- ColumnOrSuperColumn _elem47;
- _elem47 = new ColumnOrSuperColumn();
- _elem47.read(iprot);
- struct.columns.add(_elem47);
- }
- }
- struct.setColumnsIsSet(true);
- }
- }
-
-}
-
diff --git a/interface/thrift/gen-java/org/apache/cassandra/thrift/KsDef.java b/interface/thrift/gen-java/org/apache/cassandra/thrift/KsDef.java
deleted file mode 100644
index cd2a938..0000000
--- a/interface/thrift/gen-java/org/apache/cassandra/thrift/KsDef.java
+++ /dev/null
@@ -1,1047 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.1)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- * @generated
- */
-package org.apache.cassandra.thrift;
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
-
-import org.apache.commons.lang3.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class KsDef implements org.apache.thrift.TBase<KsDef, KsDef._Fields>, java.io.Serializable, Cloneable, Comparable<KsDef> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("KsDef");
-
- private static final org.apache.thrift.protocol.TField NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("name", org.apache.thrift.protocol.TType.STRING, (short)1);
- private static final org.apache.thrift.protocol.TField STRATEGY_CLASS_FIELD_DESC = new org.apache.thrift.protocol.TField("strategy_class", org.apache.thrift.protocol.TType.STRING, (short)2);
- private static final org.apache.thrift.protocol.TField STRATEGY_OPTIONS_FIELD_DESC = new org.apache.thrift.protocol.TField("strategy_options", org.apache.thrift.protocol.TType.MAP, (short)3);
- private static final org.apache.thrift.protocol.TField REPLICATION_FACTOR_FIELD_DESC = new org.apache.thrift.protocol.TField("replication_factor", org.apache.thrift.protocol.TType.I32, (short)4);
- private static final org.apache.thrift.protocol.TField CF_DEFS_FIELD_DESC = new org.apache.thrift.protocol.TField("cf_defs", org.apache.thrift.protocol.TType.LIST, (short)5);
- private static final org.apache.thrift.protocol.TField DURABLE_WRITES_FIELD_DESC = new org.apache.thrift.protocol.TField("durable_writes", org.apache.thrift.protocol.TType.BOOL, (short)6);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new KsDefStandardSchemeFactory());
- schemes.put(TupleScheme.class, new KsDefTupleSchemeFactory());
- }
-
- public String name; // required
- public String strategy_class; // required
- public Map<String,String> strategy_options; // optional
- /**
- * @deprecated ignored
- */
- public int replication_factor; // optional
- public List<CfDef> cf_defs; // required
- public boolean durable_writes; // optional
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- NAME((short)1, "name"),
- STRATEGY_CLASS((short)2, "strategy_class"),
- STRATEGY_OPTIONS((short)3, "strategy_options"),
- /**
- * @deprecated ignored
- */
- REPLICATION_FACTOR((short)4, "replication_factor"),
- CF_DEFS((short)5, "cf_defs"),
- DURABLE_WRITES((short)6, "durable_writes");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // NAME
- return NAME;
- case 2: // STRATEGY_CLASS
- return STRATEGY_CLASS;
- case 3: // STRATEGY_OPTIONS
- return STRATEGY_OPTIONS;
- case 4: // REPLICATION_FACTOR
- return REPLICATION_FACTOR;
- case 5: // CF_DEFS
- return CF_DEFS;
- case 6: // DURABLE_WRITES
- return DURABLE_WRITES;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- private static final int __REPLICATION_FACTOR_ISSET_ID = 0;
- private static final int __DURABLE_WRITES_ISSET_ID = 1;
- private byte __isset_bitfield = 0;
- private _Fields optionals[] = {_Fields.STRATEGY_OPTIONS,_Fields.REPLICATION_FACTOR,_Fields.DURABLE_WRITES};
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.NAME, new org.apache.thrift.meta_data.FieldMetaData("name", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
- tmpMap.put(_Fields.STRATEGY_CLASS, new org.apache.thrift.meta_data.FieldMetaData("strategy_class", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
- tmpMap.put(_Fields.STRATEGY_OPTIONS, new org.apache.thrift.meta_data.FieldMetaData("strategy_options", org.apache.thrift.TFieldRequirementType.OPTIONAL,
- new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING),
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))));
- tmpMap.put(_Fields.REPLICATION_FACTOR, new org.apache.thrift.meta_data.FieldMetaData("replication_factor", org.apache.thrift.TFieldRequirementType.OPTIONAL,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32)));
- tmpMap.put(_Fields.CF_DEFS, new org.apache.thrift.meta_data.FieldMetaData("cf_defs", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST,
- new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, CfDef.class))));
- tmpMap.put(_Fields.DURABLE_WRITES, new org.apache.thrift.meta_data.FieldMetaData("durable_writes", org.apache.thrift.TFieldRequirementType.OPTIONAL,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.BOOL)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(KsDef.class, metaDataMap);
- }
-
- public KsDef() {
- this.durable_writes = true;
-
- }
-
- public KsDef(
- String name,
- String strategy_class,
- List<CfDef> cf_defs)
- {
- this();
- this.name = name;
- this.strategy_class = strategy_class;
- this.cf_defs = cf_defs;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public KsDef(KsDef other) {
- __isset_bitfield = other.__isset_bitfield;
- if (other.isSetName()) {
- this.name = other.name;
- }
- if (other.isSetStrategy_class()) {
- this.strategy_class = other.strategy_class;
- }
- if (other.isSetStrategy_options()) {
- Map<String,String> __this__strategy_options = new HashMap<String,String>(other.strategy_options);
- this.strategy_options = __this__strategy_options;
- }
- this.replication_factor = other.replication_factor;
- if (other.isSetCf_defs()) {
- List<CfDef> __this__cf_defs = new ArrayList<CfDef>(other.cf_defs.size());
- for (CfDef other_element : other.cf_defs) {
- __this__cf_defs.add(new CfDef(other_element));
- }
- this.cf_defs = __this__cf_defs;
- }
- this.durable_writes = other.durable_writes;
- }
-
- public KsDef deepCopy() {
- return new KsDef(this);
- }
-
- @Override
- public void clear() {
- this.name = null;
- this.strategy_class = null;
- this.strategy_options = null;
- setReplication_factorIsSet(false);
- this.replication_factor = 0;
- this.cf_defs = null;
- this.durable_writes = true;
-
- }
-
- public String getName() {
- return this.name;
- }
-
- public KsDef setName(String name) {
- this.name = name;
- return this;
- }
-
- public void unsetName() {
- this.name = null;
- }
-
- /** Returns true if field name is set (has been assigned a value) and false otherwise */
- public boolean isSetName() {
- return this.name != null;
- }
-
- public void setNameIsSet(boolean value) {
- if (!value) {
- this.name = null;
- }
- }
-
- public String getStrategy_class() {
- return this.strategy_class;
- }
-
- public KsDef setStrategy_class(String strategy_class) {
- this.strategy_class = strategy_class;
- return this;
- }
-
- public void unsetStrategy_class() {
- this.strategy_class = null;
- }
-
- /** Returns true if field strategy_class is set (has been assigned a value) and false otherwise */
- public boolean isSetStrategy_class() {
- return this.strategy_class != null;
- }
-
- public void setStrategy_classIsSet(boolean value) {
- if (!value) {
- this.strategy_class = null;
- }
- }
-
- public int getStrategy_optionsSize() {
- return (this.strategy_options == null) ? 0 : this.strategy_options.size();
- }
-
- public void putToStrategy_options(String key, String val) {
- if (this.strategy_options == null) {
- this.strategy_options = new HashMap<String,String>();
- }
- this.strategy_options.put(key, val);
- }
-
- public Map<String,String> getStrategy_options() {
- return this.strategy_options;
- }
-
- public KsDef setStrategy_options(Map<String,String> strategy_options) {
- this.strategy_options = strategy_options;
- return this;
- }
-
- public void unsetStrategy_options() {
- this.strategy_options = null;
- }
-
- /** Returns true if field strategy_options is set (has been assigned a value) and false otherwise */
- public boolean isSetStrategy_options() {
- return this.strategy_options != null;
- }
-
- public void setStrategy_optionsIsSet(boolean value) {
- if (!value) {
- this.strategy_options = null;
- }
- }
-
- /**
- * @deprecated ignored
- */
- public int getReplication_factor() {
- return this.replication_factor;
- }
-
- /**
- * @deprecated ignored
- */
- public KsDef setReplication_factor(int replication_factor) {
- this.replication_factor = replication_factor;
- setReplication_factorIsSet(true);
- return this;
- }
-
- public void unsetReplication_factor() {
- __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __REPLICATION_FACTOR_ISSET_ID);
- }
-
- /** Returns true if field replication_factor is set (has been assigned a value) and false otherwise */
- public boolean isSetReplication_factor() {
- return EncodingUtils.testBit(__isset_bitfield, __REPLICATION_FACTOR_ISSET_ID);
- }
-
- public void setReplication_factorIsSet(boolean value) {
- __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __REPLICATION_FACTOR_ISSET_ID, value);
- }
-
- public int getCf_defsSize() {
- return (this.cf_defs == null) ? 0 : this.cf_defs.size();
- }
-
- public java.util.Iterator<CfDef> getCf_defsIterator() {
- return (this.cf_defs == null) ? null : this.cf_defs.iterator();
- }
-
- public void addToCf_defs(CfDef elem) {
- if (this.cf_defs == null) {
- this.cf_defs = new ArrayList<CfDef>();
- }
- this.cf_defs.add(elem);
- }
-
- public List<CfDef> getCf_defs() {
- return this.cf_defs;
- }
-
- public KsDef setCf_defs(List<CfDef> cf_defs) {
- this.cf_defs = cf_defs;
- return this;
- }
-
- public void unsetCf_defs() {
- this.cf_defs = null;
- }
-
- /** Returns true if field cf_defs is set (has been assigned a value) and false otherwise */
- public boolean isSetCf_defs() {
- return this.cf_defs != null;
- }
-
- public void setCf_defsIsSet(boolean value) {
- if (!value) {
- this.cf_defs = null;
- }
- }
-
- public boolean isDurable_writes() {
- return this.durable_writes;
- }
-
- public KsDef setDurable_writes(boolean durable_writes) {
- this.durable_writes = durable_writes;
- setDurable_writesIsSet(true);
- return this;
- }
-
- public void unsetDurable_writes() {
- __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __DURABLE_WRITES_ISSET_ID);
- }
-
- /** Returns true if field durable_writes is set (has been assigned a value) and false otherwise */
- public boolean isSetDurable_writes() {
- return EncodingUtils.testBit(__isset_bitfield, __DURABLE_WRITES_ISSET_ID);
- }
-
- public void setDurable_writesIsSet(boolean value) {
- __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __DURABLE_WRITES_ISSET_ID, value);
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case NAME:
- if (value == null) {
- unsetName();
- } else {
- setName((String)value);
- }
- break;
-
- case STRATEGY_CLASS:
- if (value == null) {
- unsetStrategy_class();
- } else {
- setStrategy_class((String)value);
- }
- break;
-
- case STRATEGY_OPTIONS:
- if (value == null) {
- unsetStrategy_options();
- } else {
- setStrategy_options((Map<String,String>)value);
- }
- break;
-
- case REPLICATION_FACTOR:
- if (value == null) {
- unsetReplication_factor();
- } else {
- setReplication_factor((Integer)value);
- }
- break;
-
- case CF_DEFS:
- if (value == null) {
- unsetCf_defs();
- } else {
- setCf_defs((List<CfDef>)value);
- }
- break;
-
- case DURABLE_WRITES:
- if (value == null) {
- unsetDurable_writes();
- } else {
- setDurable_writes((Boolean)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case NAME:
- return getName();
-
- case STRATEGY_CLASS:
- return getStrategy_class();
-
- case STRATEGY_OPTIONS:
- return getStrategy_options();
-
- case REPLICATION_FACTOR:
- return Integer.valueOf(getReplication_factor());
-
- case CF_DEFS:
- return getCf_defs();
-
- case DURABLE_WRITES:
- return Boolean.valueOf(isDurable_writes());
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case NAME:
- return isSetName();
- case STRATEGY_CLASS:
- return isSetStrategy_class();
- case STRATEGY_OPTIONS:
- return isSetStrategy_options();
- case REPLICATION_FACTOR:
- return isSetReplication_factor();
- case CF_DEFS:
- return isSetCf_defs();
- case DURABLE_WRITES:
- return isSetDurable_writes();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof KsDef)
- return this.equals((KsDef)that);
- return false;
- }
-
- public boolean equals(KsDef that) {
- if (that == null)
- return false;
-
- boolean this_present_name = true && this.isSetName();
- boolean that_present_name = true && that.isSetName();
- if (this_present_name || that_present_name) {
- if (!(this_present_name && that_present_name))
- return false;
- if (!this.name.equals(that.name))
- return false;
- }
-
- boolean this_present_strategy_class = true && this.isSetStrategy_class();
- boolean that_present_strategy_class = true && that.isSetStrategy_class();
- if (this_present_strategy_class || that_present_strategy_class) {
- if (!(this_present_strategy_class && that_present_strategy_class))
- return false;
- if (!this.strategy_class.equals(that.strategy_class))
- return false;
- }
-
- boolean this_present_strategy_options = true && this.isSetStrategy_options();
- boolean that_present_strategy_options = true && that.isSetStrategy_options();
- if (this_present_strategy_options || that_present_strategy_options) {
- if (!(this_present_strategy_options && that_present_strategy_options))
- return false;
- if (!this.strategy_options.equals(that.strategy_options))
- return false;
- }
-
- boolean this_present_replication_factor = true && this.isSetReplication_factor();
- boolean that_present_replication_factor = true && that.isSetReplication_factor();
- if (this_present_replication_factor || that_present_replication_factor) {
- if (!(this_present_replication_factor && that_present_replication_factor))
- return false;
- if (this.replication_factor != that.replication_factor)
- return false;
- }
-
- boolean this_present_cf_defs = true && this.isSetCf_defs();
- boolean that_present_cf_defs = true && that.isSetCf_defs();
- if (this_present_cf_defs || that_present_cf_defs) {
- if (!(this_present_cf_defs && that_present_cf_defs))
- return false;
- if (!this.cf_defs.equals(that.cf_defs))
- return false;
- }
-
- boolean this_present_durable_writes = true && this.isSetDurable_writes();
- boolean that_present_durable_writes = true && that.isSetDurable_writes();
- if (this_present_durable_writes || that_present_durable_writes) {
- if (!(this_present_durable_writes && that_present_durable_writes))
- return false;
- if (this.durable_writes != that.durable_writes)
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_name = true && (isSetName());
- builder.append(present_name);
- if (present_name)
- builder.append(name);
-
- boolean present_strategy_class = true && (isSetStrategy_class());
- builder.append(present_strategy_class);
- if (present_strategy_class)
- builder.append(strategy_class);
-
- boolean present_strategy_options = true && (isSetStrategy_options());
- builder.append(present_strategy_options);
- if (present_strategy_options)
- builder.append(strategy_options);
-
- boolean present_replication_factor = true && (isSetReplication_factor());
- builder.append(present_replication_factor);
- if (present_replication_factor)
- builder.append(replication_factor);
-
- boolean present_cf_defs = true && (isSetCf_defs());
- builder.append(present_cf_defs);
- if (present_cf_defs)
- builder.append(cf_defs);
-
- boolean present_durable_writes = true && (isSetDurable_writes());
- builder.append(present_durable_writes);
- if (present_durable_writes)
- builder.append(durable_writes);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(KsDef other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetName()).compareTo(other.isSetName());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetName()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.name, other.name);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetStrategy_class()).compareTo(other.isSetStrategy_class());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetStrategy_class()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.strategy_class, other.strategy_class);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetStrategy_options()).compareTo(other.isSetStrategy_options());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetStrategy_options()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.strategy_options, other.strategy_options);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetReplication_factor()).compareTo(other.isSetReplication_factor());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetReplication_factor()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.replication_factor, other.replication_factor);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetCf_defs()).compareTo(other.isSetCf_defs());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetCf_defs()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.cf_defs, other.cf_defs);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetDurable_writes()).compareTo(other.isSetDurable_writes());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetDurable_writes()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.durable_writes, other.durable_writes);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("KsDef(");
- boolean first = true;
-
- sb.append("name:");
- if (this.name == null) {
- sb.append("null");
- } else {
- sb.append(this.name);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("strategy_class:");
- if (this.strategy_class == null) {
- sb.append("null");
- } else {
- sb.append(this.strategy_class);
- }
- first = false;
- if (isSetStrategy_options()) {
- if (!first) sb.append(", ");
- sb.append("strategy_options:");
- if (this.strategy_options == null) {
- sb.append("null");
- } else {
- sb.append(this.strategy_options);
- }
- first = false;
- }
- if (isSetReplication_factor()) {
- if (!first) sb.append(", ");
- sb.append("replication_factor:");
- sb.append(this.replication_factor);
- first = false;
- }
- if (!first) sb.append(", ");
- sb.append("cf_defs:");
- if (this.cf_defs == null) {
- sb.append("null");
- } else {
- sb.append(this.cf_defs);
- }
- first = false;
- if (isSetDurable_writes()) {
- if (!first) sb.append(", ");
- sb.append("durable_writes:");
- sb.append(this.durable_writes);
- first = false;
- }
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- if (name == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'name' was not present! Struct: " + toString());
- }
- if (strategy_class == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'strategy_class' was not present! Struct: " + toString());
- }
- if (cf_defs == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'cf_defs' was not present! Struct: " + toString());
- }
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
- __isset_bitfield = 0;
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class KsDefStandardSchemeFactory implements SchemeFactory {
- public KsDefStandardScheme getScheme() {
- return new KsDefStandardScheme();
- }
- }
-
- private static class KsDefStandardScheme extends StandardScheme<KsDef> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, KsDef struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // NAME
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.name = iprot.readString();
- struct.setNameIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 2: // STRATEGY_CLASS
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.strategy_class = iprot.readString();
- struct.setStrategy_classIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 3: // STRATEGY_OPTIONS
- if (schemeField.type == org.apache.thrift.protocol.TType.MAP) {
- {
- org.apache.thrift.protocol.TMap _map146 = iprot.readMapBegin();
- struct.strategy_options = new HashMap<String,String>(2*_map146.size);
- for (int _i147 = 0; _i147 < _map146.size; ++_i147)
- {
- String _key148;
- String _val149;
- _key148 = iprot.readString();
- _val149 = iprot.readString();
- struct.strategy_options.put(_key148, _val149);
- }
- iprot.readMapEnd();
- }
- struct.setStrategy_optionsIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 4: // REPLICATION_FACTOR
- if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
- struct.replication_factor = iprot.readI32();
- struct.setReplication_factorIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 5: // CF_DEFS
- if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
- {
- org.apache.thrift.protocol.TList _list150 = iprot.readListBegin();
- struct.cf_defs = new ArrayList<CfDef>(_list150.size);
- for (int _i151 = 0; _i151 < _list150.size; ++_i151)
- {
- CfDef _elem152;
- _elem152 = new CfDef();
- _elem152.read(iprot);
- struct.cf_defs.add(_elem152);
- }
- iprot.readListEnd();
- }
- struct.setCf_defsIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 6: // DURABLE_WRITES
- if (schemeField.type == org.apache.thrift.protocol.TType.BOOL) {
- struct.durable_writes = iprot.readBool();
- struct.setDurable_writesIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, KsDef struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.name != null) {
- oprot.writeFieldBegin(NAME_FIELD_DESC);
- oprot.writeString(struct.name);
- oprot.writeFieldEnd();
- }
- if (struct.strategy_class != null) {
- oprot.writeFieldBegin(STRATEGY_CLASS_FIELD_DESC);
- oprot.writeString(struct.strategy_class);
- oprot.writeFieldEnd();
- }
- if (struct.strategy_options != null) {
- if (struct.isSetStrategy_options()) {
- oprot.writeFieldBegin(STRATEGY_OPTIONS_FIELD_DESC);
- {
- oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, struct.strategy_options.size()));
- for (Map.Entry<String, String> _iter153 : struct.strategy_options.entrySet())
- {
- oprot.writeString(_iter153.getKey());
- oprot.writeString(_iter153.getValue());
- }
- oprot.writeMapEnd();
- }
- oprot.writeFieldEnd();
- }
- }
- if (struct.isSetReplication_factor()) {
- oprot.writeFieldBegin(REPLICATION_FACTOR_FIELD_DESC);
- oprot.writeI32(struct.replication_factor);
- oprot.writeFieldEnd();
- }
- if (struct.cf_defs != null) {
- oprot.writeFieldBegin(CF_DEFS_FIELD_DESC);
- {
- oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.cf_defs.size()));
- for (CfDef _iter154 : struct.cf_defs)
- {
- _iter154.write(oprot);
- }
- oprot.writeListEnd();
- }
- oprot.writeFieldEnd();
- }
- if (struct.isSetDurable_writes()) {
- oprot.writeFieldBegin(DURABLE_WRITES_FIELD_DESC);
- oprot.writeBool(struct.durable_writes);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class KsDefTupleSchemeFactory implements SchemeFactory {
- public KsDefTupleScheme getScheme() {
- return new KsDefTupleScheme();
- }
- }
-
- private static class KsDefTupleScheme extends TupleScheme<KsDef> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, KsDef struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- oprot.writeString(struct.name);
- oprot.writeString(struct.strategy_class);
- {
- oprot.writeI32(struct.cf_defs.size());
- for (CfDef _iter155 : struct.cf_defs)
- {
- _iter155.write(oprot);
- }
- }
- BitSet optionals = new BitSet();
- if (struct.isSetStrategy_options()) {
- optionals.set(0);
- }
- if (struct.isSetReplication_factor()) {
- optionals.set(1);
- }
- if (struct.isSetDurable_writes()) {
- optionals.set(2);
- }
- oprot.writeBitSet(optionals, 3);
- if (struct.isSetStrategy_options()) {
- {
- oprot.writeI32(struct.strategy_options.size());
- for (Map.Entry<String, String> _iter156 : struct.strategy_options.entrySet())
- {
- oprot.writeString(_iter156.getKey());
- oprot.writeString(_iter156.getValue());
- }
- }
- }
- if (struct.isSetReplication_factor()) {
- oprot.writeI32(struct.replication_factor);
- }
- if (struct.isSetDurable_writes()) {
- oprot.writeBool(struct.durable_writes);
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, KsDef struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- struct.name = iprot.readString();
- struct.setNameIsSet(true);
- struct.strategy_class = iprot.readString();
- struct.setStrategy_classIsSet(true);
- {
- org.apache.thrift.protocol.TList _list157 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
- struct.cf_defs = new ArrayList<CfDef>(_list157.size);
- for (int _i158 = 0; _i158 < _list157.size; ++_i158)
- {
- CfDef _elem159;
- _elem159 = new CfDef();
- _elem159.read(iprot);
- struct.cf_defs.add(_elem159);
- }
- }
- struct.setCf_defsIsSet(true);
- BitSet incoming = iprot.readBitSet(3);
- if (incoming.get(0)) {
- {
- org.apache.thrift.protocol.TMap _map160 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32());
- struct.strategy_options = new HashMap<String,String>(2*_map160.size);
- for (int _i161 = 0; _i161 < _map160.size; ++_i161)
- {
- String _key162;
- String _val163;
- _key162 = iprot.readString();
- _val163 = iprot.readString();
- struct.strategy_options.put(_key162, _val163);
- }
- }
- struct.setStrategy_optionsIsSet(true);
- }
- if (incoming.get(1)) {
- struct.replication_factor = iprot.readI32();
- struct.setReplication_factorIsSet(true);
- }
- if (incoming.get(2)) {
- struct.durable_writes = iprot.readBool();
- struct.setDurable_writesIsSet(true);
- }
- }
- }
-
-}
-
diff --git a/interface/thrift/gen-java/org/apache/cassandra/thrift/MultiSliceRequest.java b/interface/thrift/gen-java/org/apache/cassandra/thrift/MultiSliceRequest.java
deleted file mode 100644
index 9d4878c..0000000
--- a/interface/thrift/gen-java/org/apache/cassandra/thrift/MultiSliceRequest.java
+++ /dev/null
@@ -1,1042 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.1)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- * @generated
- */
-package org.apache.cassandra.thrift;
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
-
-import org.apache.commons.lang3.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Used to perform multiple slices on a single row key in one rpc operation
- * @param key. The row key to be multi sliced
- * @param column_parent. The column family (super columns are unsupported)
- * @param column_slices. 0 to many ColumnSlice objects each will be used to select columns
- * @param reversed. Direction of slice
- * @param count. Maximum number of columns
- * @param consistency_level. Level to perform the operation at
- */
-public class MultiSliceRequest implements org.apache.thrift.TBase<MultiSliceRequest, MultiSliceRequest._Fields>, java.io.Serializable, Cloneable, Comparable<MultiSliceRequest> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("MultiSliceRequest");
-
- private static final org.apache.thrift.protocol.TField KEY_FIELD_DESC = new org.apache.thrift.protocol.TField("key", org.apache.thrift.protocol.TType.STRING, (short)1);
- private static final org.apache.thrift.protocol.TField COLUMN_PARENT_FIELD_DESC = new org.apache.thrift.protocol.TField("column_parent", org.apache.thrift.protocol.TType.STRUCT, (short)2);
- private static final org.apache.thrift.protocol.TField COLUMN_SLICES_FIELD_DESC = new org.apache.thrift.protocol.TField("column_slices", org.apache.thrift.protocol.TType.LIST, (short)3);
- private static final org.apache.thrift.protocol.TField REVERSED_FIELD_DESC = new org.apache.thrift.protocol.TField("reversed", org.apache.thrift.protocol.TType.BOOL, (short)4);
- private static final org.apache.thrift.protocol.TField COUNT_FIELD_DESC = new org.apache.thrift.protocol.TField("count", org.apache.thrift.protocol.TType.I32, (short)5);
- private static final org.apache.thrift.protocol.TField CONSISTENCY_LEVEL_FIELD_DESC = new org.apache.thrift.protocol.TField("consistency_level", org.apache.thrift.protocol.TType.I32, (short)6);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new MultiSliceRequestStandardSchemeFactory());
- schemes.put(TupleScheme.class, new MultiSliceRequestTupleSchemeFactory());
- }
-
- public ByteBuffer key; // optional
- public ColumnParent column_parent; // optional
- public List<ColumnSlice> column_slices; // optional
- public boolean reversed; // optional
- public int count; // optional
- /**
- *
- * @see ConsistencyLevel
- */
- public ConsistencyLevel consistency_level; // optional
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- KEY((short)1, "key"),
- COLUMN_PARENT((short)2, "column_parent"),
- COLUMN_SLICES((short)3, "column_slices"),
- REVERSED((short)4, "reversed"),
- COUNT((short)5, "count"),
- /**
- *
- * @see ConsistencyLevel
- */
- CONSISTENCY_LEVEL((short)6, "consistency_level");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // KEY
- return KEY;
- case 2: // COLUMN_PARENT
- return COLUMN_PARENT;
- case 3: // COLUMN_SLICES
- return COLUMN_SLICES;
- case 4: // REVERSED
- return REVERSED;
- case 5: // COUNT
- return COUNT;
- case 6: // CONSISTENCY_LEVEL
- return CONSISTENCY_LEVEL;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- private static final int __REVERSED_ISSET_ID = 0;
- private static final int __COUNT_ISSET_ID = 1;
- private byte __isset_bitfield = 0;
- private _Fields optionals[] = {_Fields.KEY,_Fields.COLUMN_PARENT,_Fields.COLUMN_SLICES,_Fields.REVERSED,_Fields.COUNT,_Fields.CONSISTENCY_LEVEL};
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.KEY, new org.apache.thrift.meta_data.FieldMetaData("key", org.apache.thrift.TFieldRequirementType.OPTIONAL,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true)));
- tmpMap.put(_Fields.COLUMN_PARENT, new org.apache.thrift.meta_data.FieldMetaData("column_parent", org.apache.thrift.TFieldRequirementType.OPTIONAL,
- new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, ColumnParent.class)));
- tmpMap.put(_Fields.COLUMN_SLICES, new org.apache.thrift.meta_data.FieldMetaData("column_slices", org.apache.thrift.TFieldRequirementType.OPTIONAL,
- new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST,
- new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, ColumnSlice.class))));
- tmpMap.put(_Fields.REVERSED, new org.apache.thrift.meta_data.FieldMetaData("reversed", org.apache.thrift.TFieldRequirementType.OPTIONAL,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.BOOL)));
- tmpMap.put(_Fields.COUNT, new org.apache.thrift.meta_data.FieldMetaData("count", org.apache.thrift.TFieldRequirementType.OPTIONAL,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32)));
- tmpMap.put(_Fields.CONSISTENCY_LEVEL, new org.apache.thrift.meta_data.FieldMetaData("consistency_level", org.apache.thrift.TFieldRequirementType.OPTIONAL,
- new org.apache.thrift.meta_data.EnumMetaData(org.apache.thrift.protocol.TType.ENUM, ConsistencyLevel.class)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(MultiSliceRequest.class, metaDataMap);
- }
-
- public MultiSliceRequest() {
- this.reversed = false;
-
- this.count = 1000;
-
- this.consistency_level = org.apache.cassandra.thrift.ConsistencyLevel.ONE;
-
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public MultiSliceRequest(MultiSliceRequest other) {
- __isset_bitfield = other.__isset_bitfield;
- if (other.isSetKey()) {
- this.key = org.apache.thrift.TBaseHelper.copyBinary(other.key);
-;
- }
- if (other.isSetColumn_parent()) {
- this.column_parent = new ColumnParent(other.column_parent);
- }
- if (other.isSetColumn_slices()) {
- List<ColumnSlice> __this__column_slices = new ArrayList<ColumnSlice>(other.column_slices.size());
- for (ColumnSlice other_element : other.column_slices) {
- __this__column_slices.add(new ColumnSlice(other_element));
- }
- this.column_slices = __this__column_slices;
- }
- this.reversed = other.reversed;
- this.count = other.count;
- if (other.isSetConsistency_level()) {
- this.consistency_level = other.consistency_level;
- }
- }
-
- public MultiSliceRequest deepCopy() {
- return new MultiSliceRequest(this);
- }
-
- @Override
- public void clear() {
- this.key = null;
- this.column_parent = null;
- this.column_slices = null;
- this.reversed = false;
-
- this.count = 1000;
-
- this.consistency_level = org.apache.cassandra.thrift.ConsistencyLevel.ONE;
-
- }
-
- public byte[] getKey() {
- setKey(org.apache.thrift.TBaseHelper.rightSize(key));
- return key == null ? null : key.array();
- }
-
- public ByteBuffer bufferForKey() {
- return key;
- }
-
- public MultiSliceRequest setKey(byte[] key) {
- setKey(key == null ? (ByteBuffer)null : ByteBuffer.wrap(key));
- return this;
- }
-
- public MultiSliceRequest setKey(ByteBuffer key) {
- this.key = key;
- return this;
- }
-
- public void unsetKey() {
- this.key = null;
- }
-
- /** Returns true if field key is set (has been assigned a value) and false otherwise */
- public boolean isSetKey() {
- return this.key != null;
- }
-
- public void setKeyIsSet(boolean value) {
- if (!value) {
- this.key = null;
- }
- }
-
- public ColumnParent getColumn_parent() {
- return this.column_parent;
- }
-
- public MultiSliceRequest setColumn_parent(ColumnParent column_parent) {
- this.column_parent = column_parent;
- return this;
- }
-
- public void unsetColumn_parent() {
- this.column_parent = null;
- }
-
- /** Returns true if field column_parent is set (has been assigned a value) and false otherwise */
- public boolean isSetColumn_parent() {
- return this.column_parent != null;
- }
-
- public void setColumn_parentIsSet(boolean value) {
- if (!value) {
- this.column_parent = null;
- }
- }
-
- public int getColumn_slicesSize() {
- return (this.column_slices == null) ? 0 : this.column_slices.size();
- }
-
- public java.util.Iterator<ColumnSlice> getColumn_slicesIterator() {
- return (this.column_slices == null) ? null : this.column_slices.iterator();
- }
-
- public void addToColumn_slices(ColumnSlice elem) {
- if (this.column_slices == null) {
- this.column_slices = new ArrayList<ColumnSlice>();
- }
- this.column_slices.add(elem);
- }
-
- public List<ColumnSlice> getColumn_slices() {
- return this.column_slices;
- }
-
- public MultiSliceRequest setColumn_slices(List<ColumnSlice> column_slices) {
- this.column_slices = column_slices;
- return this;
- }
-
- public void unsetColumn_slices() {
- this.column_slices = null;
- }
-
- /** Returns true if field column_slices is set (has been assigned a value) and false otherwise */
- public boolean isSetColumn_slices() {
- return this.column_slices != null;
- }
-
- public void setColumn_slicesIsSet(boolean value) {
- if (!value) {
- this.column_slices = null;
- }
- }
-
- public boolean isReversed() {
- return this.reversed;
- }
-
- public MultiSliceRequest setReversed(boolean reversed) {
- this.reversed = reversed;
- setReversedIsSet(true);
- return this;
- }
-
- public void unsetReversed() {
- __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __REVERSED_ISSET_ID);
- }
-
- /** Returns true if field reversed is set (has been assigned a value) and false otherwise */
- public boolean isSetReversed() {
- return EncodingUtils.testBit(__isset_bitfield, __REVERSED_ISSET_ID);
- }
-
- public void setReversedIsSet(boolean value) {
- __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __REVERSED_ISSET_ID, value);
- }
-
- public int getCount() {
- return this.count;
- }
-
- public MultiSliceRequest setCount(int count) {
- this.count = count;
- setCountIsSet(true);
- return this;
- }
-
- public void unsetCount() {
- __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __COUNT_ISSET_ID);
- }
-
- /** Returns true if field count is set (has been assigned a value) and false otherwise */
- public boolean isSetCount() {
- return EncodingUtils.testBit(__isset_bitfield, __COUNT_ISSET_ID);
- }
-
- public void setCountIsSet(boolean value) {
- __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __COUNT_ISSET_ID, value);
- }
-
- /**
- *
- * @see ConsistencyLevel
- */
- public ConsistencyLevel getConsistency_level() {
- return this.consistency_level;
- }
-
- /**
- *
- * @see ConsistencyLevel
- */
- public MultiSliceRequest setConsistency_level(ConsistencyLevel consistency_level) {
- this.consistency_level = consistency_level;
- return this;
- }
-
- public void unsetConsistency_level() {
- this.consistency_level = null;
- }
-
- /** Returns true if field consistency_level is set (has been assigned a value) and false otherwise */
- public boolean isSetConsistency_level() {
- return this.consistency_level != null;
- }
-
- public void setConsistency_levelIsSet(boolean value) {
- if (!value) {
- this.consistency_level = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case KEY:
- if (value == null) {
- unsetKey();
- } else {
- setKey((ByteBuffer)value);
- }
- break;
-
- case COLUMN_PARENT:
- if (value == null) {
- unsetColumn_parent();
- } else {
- setColumn_parent((ColumnParent)value);
- }
- break;
-
- case COLUMN_SLICES:
- if (value == null) {
- unsetColumn_slices();
- } else {
- setColumn_slices((List<ColumnSlice>)value);
- }
- break;
-
- case REVERSED:
- if (value == null) {
- unsetReversed();
- } else {
- setReversed((Boolean)value);
- }
- break;
-
- case COUNT:
- if (value == null) {
- unsetCount();
- } else {
- setCount((Integer)value);
- }
- break;
-
- case CONSISTENCY_LEVEL:
- if (value == null) {
- unsetConsistency_level();
- } else {
- setConsistency_level((ConsistencyLevel)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case KEY:
- return getKey();
-
- case COLUMN_PARENT:
- return getColumn_parent();
-
- case COLUMN_SLICES:
- return getColumn_slices();
-
- case REVERSED:
- return Boolean.valueOf(isReversed());
-
- case COUNT:
- return Integer.valueOf(getCount());
-
- case CONSISTENCY_LEVEL:
- return getConsistency_level();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case KEY:
- return isSetKey();
- case COLUMN_PARENT:
- return isSetColumn_parent();
- case COLUMN_SLICES:
- return isSetColumn_slices();
- case REVERSED:
- return isSetReversed();
- case COUNT:
- return isSetCount();
- case CONSISTENCY_LEVEL:
- return isSetConsistency_level();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof MultiSliceRequest)
- return this.equals((MultiSliceRequest)that);
- return false;
- }
-
- public boolean equals(MultiSliceRequest that) {
- if (that == null)
- return false;
-
- boolean this_present_key = true && this.isSetKey();
- boolean that_present_key = true && that.isSetKey();
- if (this_present_key || that_present_key) {
- if (!(this_present_key && that_present_key))
- return false;
- if (!this.key.equals(that.key))
- return false;
- }
-
- boolean this_present_column_parent = true && this.isSetColumn_parent();
- boolean that_present_column_parent = true && that.isSetColumn_parent();
- if (this_present_column_parent || that_present_column_parent) {
- if (!(this_present_column_parent && that_present_column_parent))
- return false;
- if (!this.column_parent.equals(that.column_parent))
- return false;
- }
-
- boolean this_present_column_slices = true && this.isSetColumn_slices();
- boolean that_present_column_slices = true && that.isSetColumn_slices();
- if (this_present_column_slices || that_present_column_slices) {
- if (!(this_present_column_slices && that_present_column_slices))
- return false;
- if (!this.column_slices.equals(that.column_slices))
- return false;
- }
-
- boolean this_present_reversed = true && this.isSetReversed();
- boolean that_present_reversed = true && that.isSetReversed();
- if (this_present_reversed || that_present_reversed) {
- if (!(this_present_reversed && that_present_reversed))
- return false;
- if (this.reversed != that.reversed)
- return false;
- }
-
- boolean this_present_count = true && this.isSetCount();
- boolean that_present_count = true && that.isSetCount();
- if (this_present_count || that_present_count) {
- if (!(this_present_count && that_present_count))
- return false;
- if (this.count != that.count)
- return false;
- }
-
- boolean this_present_consistency_level = true && this.isSetConsistency_level();
- boolean that_present_consistency_level = true && that.isSetConsistency_level();
- if (this_present_consistency_level || that_present_consistency_level) {
- if (!(this_present_consistency_level && that_present_consistency_level))
- return false;
- if (!this.consistency_level.equals(that.consistency_level))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_key = true && (isSetKey());
- builder.append(present_key);
- if (present_key)
- builder.append(key);
-
- boolean present_column_parent = true && (isSetColumn_parent());
- builder.append(present_column_parent);
- if (present_column_parent)
- builder.append(column_parent);
-
- boolean present_column_slices = true && (isSetColumn_slices());
- builder.append(present_column_slices);
- if (present_column_slices)
- builder.append(column_slices);
-
- boolean present_reversed = true && (isSetReversed());
- builder.append(present_reversed);
- if (present_reversed)
- builder.append(reversed);
-
- boolean present_count = true && (isSetCount());
- builder.append(present_count);
- if (present_count)
- builder.append(count);
-
- boolean present_consistency_level = true && (isSetConsistency_level());
- builder.append(present_consistency_level);
- if (present_consistency_level)
- builder.append(consistency_level.getValue());
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(MultiSliceRequest other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetKey()).compareTo(other.isSetKey());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetKey()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.key, other.key);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetColumn_parent()).compareTo(other.isSetColumn_parent());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetColumn_parent()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.column_parent, other.column_parent);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetColumn_slices()).compareTo(other.isSetColumn_slices());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetColumn_slices()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.column_slices, other.column_slices);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetReversed()).compareTo(other.isSetReversed());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetReversed()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.reversed, other.reversed);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetCount()).compareTo(other.isSetCount());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetCount()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.count, other.count);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetConsistency_level()).compareTo(other.isSetConsistency_level());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetConsistency_level()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.consistency_level, other.consistency_level);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("MultiSliceRequest(");
- boolean first = true;
-
- if (isSetKey()) {
- sb.append("key:");
- if (this.key == null) {
- sb.append("null");
- } else {
- org.apache.thrift.TBaseHelper.toString(this.key, sb);
- }
- first = false;
- }
- if (isSetColumn_parent()) {
- if (!first) sb.append(", ");
- sb.append("column_parent:");
- if (this.column_parent == null) {
- sb.append("null");
- } else {
- sb.append(this.column_parent);
- }
- first = false;
- }
- if (isSetColumn_slices()) {
- if (!first) sb.append(", ");
- sb.append("column_slices:");
- if (this.column_slices == null) {
- sb.append("null");
- } else {
- sb.append(this.column_slices);
- }
- first = false;
- }
- if (isSetReversed()) {
- if (!first) sb.append(", ");
- sb.append("reversed:");
- sb.append(this.reversed);
- first = false;
- }
- if (isSetCount()) {
- if (!first) sb.append(", ");
- sb.append("count:");
- sb.append(this.count);
- first = false;
- }
- if (isSetConsistency_level()) {
- if (!first) sb.append(", ");
- sb.append("consistency_level:");
- if (this.consistency_level == null) {
- sb.append("null");
- } else {
- sb.append(this.consistency_level);
- }
- first = false;
- }
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // check for sub-struct validity
- if (column_parent != null) {
- column_parent.validate();
- }
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
- __isset_bitfield = 0;
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class MultiSliceRequestStandardSchemeFactory implements SchemeFactory {
- public MultiSliceRequestStandardScheme getScheme() {
- return new MultiSliceRequestStandardScheme();
- }
- }
-
- private static class MultiSliceRequestStandardScheme extends StandardScheme<MultiSliceRequest> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, MultiSliceRequest struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // KEY
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.key = iprot.readBinary();
- struct.setKeyIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 2: // COLUMN_PARENT
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.column_parent = new ColumnParent();
- struct.column_parent.read(iprot);
- struct.setColumn_parentIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 3: // COLUMN_SLICES
- if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
- {
- org.apache.thrift.protocol.TList _list216 = iprot.readListBegin();
- struct.column_slices = new ArrayList<ColumnSlice>(_list216.size);
- for (int _i217 = 0; _i217 < _list216.size; ++_i217)
- {
- ColumnSlice _elem218;
- _elem218 = new ColumnSlice();
- _elem218.read(iprot);
- struct.column_slices.add(_elem218);
- }
- iprot.readListEnd();
- }
- struct.setColumn_slicesIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 4: // REVERSED
- if (schemeField.type == org.apache.thrift.protocol.TType.BOOL) {
- struct.reversed = iprot.readBool();
- struct.setReversedIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 5: // COUNT
- if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
- struct.count = iprot.readI32();
- struct.setCountIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 6: // CONSISTENCY_LEVEL
- if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
- struct.consistency_level = ConsistencyLevel.findByValue(iprot.readI32());
- struct.setConsistency_levelIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, MultiSliceRequest struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.key != null) {
- if (struct.isSetKey()) {
- oprot.writeFieldBegin(KEY_FIELD_DESC);
- oprot.writeBinary(struct.key);
- oprot.writeFieldEnd();
- }
- }
- if (struct.column_parent != null) {
- if (struct.isSetColumn_parent()) {
- oprot.writeFieldBegin(COLUMN_PARENT_FIELD_DESC);
- struct.column_parent.write(oprot);
- oprot.writeFieldEnd();
- }
- }
- if (struct.column_slices != null) {
- if (struct.isSetColumn_slices()) {
- oprot.writeFieldBegin(COLUMN_SLICES_FIELD_DESC);
- {
- oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.column_slices.size()));
- for (ColumnSlice _iter219 : struct.column_slices)
- {
- _iter219.write(oprot);
- }
- oprot.writeListEnd();
- }
- oprot.writeFieldEnd();
- }
- }
- if (struct.isSetReversed()) {
- oprot.writeFieldBegin(REVERSED_FIELD_DESC);
- oprot.writeBool(struct.reversed);
- oprot.writeFieldEnd();
- }
- if (struct.isSetCount()) {
- oprot.writeFieldBegin(COUNT_FIELD_DESC);
- oprot.writeI32(struct.count);
- oprot.writeFieldEnd();
- }
- if (struct.consistency_level != null) {
- if (struct.isSetConsistency_level()) {
- oprot.writeFieldBegin(CONSISTENCY_LEVEL_FIELD_DESC);
- oprot.writeI32(struct.consistency_level.getValue());
- oprot.writeFieldEnd();
- }
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class MultiSliceRequestTupleSchemeFactory implements SchemeFactory {
- public MultiSliceRequestTupleScheme getScheme() {
- return new MultiSliceRequestTupleScheme();
- }
- }
-
- private static class MultiSliceRequestTupleScheme extends TupleScheme<MultiSliceRequest> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, MultiSliceRequest struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- BitSet optionals = new BitSet();
- if (struct.isSetKey()) {
- optionals.set(0);
- }
- if (struct.isSetColumn_parent()) {
- optionals.set(1);
- }
- if (struct.isSetColumn_slices()) {
- optionals.set(2);
- }
- if (struct.isSetReversed()) {
- optionals.set(3);
- }
- if (struct.isSetCount()) {
- optionals.set(4);
- }
- if (struct.isSetConsistency_level()) {
- optionals.set(5);
- }
- oprot.writeBitSet(optionals, 6);
- if (struct.isSetKey()) {
- oprot.writeBinary(struct.key);
- }
- if (struct.isSetColumn_parent()) {
- struct.column_parent.write(oprot);
- }
- if (struct.isSetColumn_slices()) {
- {
- oprot.writeI32(struct.column_slices.size());
- for (ColumnSlice _iter220 : struct.column_slices)
- {
- _iter220.write(oprot);
- }
- }
- }
- if (struct.isSetReversed()) {
- oprot.writeBool(struct.reversed);
- }
- if (struct.isSetCount()) {
- oprot.writeI32(struct.count);
- }
- if (struct.isSetConsistency_level()) {
- oprot.writeI32(struct.consistency_level.getValue());
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, MultiSliceRequest struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- BitSet incoming = iprot.readBitSet(6);
- if (incoming.get(0)) {
- struct.key = iprot.readBinary();
- struct.setKeyIsSet(true);
- }
- if (incoming.get(1)) {
- struct.column_parent = new ColumnParent();
- struct.column_parent.read(iprot);
- struct.setColumn_parentIsSet(true);
- }
- if (incoming.get(2)) {
- {
- org.apache.thrift.protocol.TList _list221 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
- struct.column_slices = new ArrayList<ColumnSlice>(_list221.size);
- for (int _i222 = 0; _i222 < _list221.size; ++_i222)
- {
- ColumnSlice _elem223;
- _elem223 = new ColumnSlice();
- _elem223.read(iprot);
- struct.column_slices.add(_elem223);
- }
- }
- struct.setColumn_slicesIsSet(true);
- }
- if (incoming.get(3)) {
- struct.reversed = iprot.readBool();
- struct.setReversedIsSet(true);
- }
- if (incoming.get(4)) {
- struct.count = iprot.readI32();
- struct.setCountIsSet(true);
- }
- if (incoming.get(5)) {
- struct.consistency_level = ConsistencyLevel.findByValue(iprot.readI32());
- struct.setConsistency_levelIsSet(true);
- }
- }
- }
-
-}
-
diff --git a/interface/thrift/gen-java/org/apache/cassandra/thrift/Mutation.java b/interface/thrift/gen-java/org/apache/cassandra/thrift/Mutation.java
deleted file mode 100644
index 981d5a4..0000000
--- a/interface/thrift/gen-java/org/apache/cassandra/thrift/Mutation.java
+++ /dev/null
@@ -1,537 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.1)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- * @generated
- */
-package org.apache.cassandra.thrift;
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
-
-import org.apache.commons.lang3.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * A Mutation is either an insert (represented by filling column_or_supercolumn) or a deletion (represented by filling the deletion attribute).
- * @param column_or_supercolumn. An insert to a column or supercolumn (possibly counter column or supercolumn)
- * @param deletion. A deletion of a column or supercolumn
- */
-public class Mutation implements org.apache.thrift.TBase<Mutation, Mutation._Fields>, java.io.Serializable, Cloneable, Comparable<Mutation> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("Mutation");
-
- private static final org.apache.thrift.protocol.TField COLUMN_OR_SUPERCOLUMN_FIELD_DESC = new org.apache.thrift.protocol.TField("column_or_supercolumn", org.apache.thrift.protocol.TType.STRUCT, (short)1);
- private static final org.apache.thrift.protocol.TField DELETION_FIELD_DESC = new org.apache.thrift.protocol.TField("deletion", org.apache.thrift.protocol.TType.STRUCT, (short)2);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new MutationStandardSchemeFactory());
- schemes.put(TupleScheme.class, new MutationTupleSchemeFactory());
- }
-
- public ColumnOrSuperColumn column_or_supercolumn; // optional
- public Deletion deletion; // optional
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- COLUMN_OR_SUPERCOLUMN((short)1, "column_or_supercolumn"),
- DELETION((short)2, "deletion");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // COLUMN_OR_SUPERCOLUMN
- return COLUMN_OR_SUPERCOLUMN;
- case 2: // DELETION
- return DELETION;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- private _Fields optionals[] = {_Fields.COLUMN_OR_SUPERCOLUMN,_Fields.DELETION};
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.COLUMN_OR_SUPERCOLUMN, new org.apache.thrift.meta_data.FieldMetaData("column_or_supercolumn", org.apache.thrift.TFieldRequirementType.OPTIONAL,
- new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, ColumnOrSuperColumn.class)));
- tmpMap.put(_Fields.DELETION, new org.apache.thrift.meta_data.FieldMetaData("deletion", org.apache.thrift.TFieldRequirementType.OPTIONAL,
- new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, Deletion.class)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(Mutation.class, metaDataMap);
- }
-
- public Mutation() {
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public Mutation(Mutation other) {
- if (other.isSetColumn_or_supercolumn()) {
- this.column_or_supercolumn = new ColumnOrSuperColumn(other.column_or_supercolumn);
- }
- if (other.isSetDeletion()) {
- this.deletion = new Deletion(other.deletion);
- }
- }
-
- public Mutation deepCopy() {
- return new Mutation(this);
- }
-
- @Override
- public void clear() {
- this.column_or_supercolumn = null;
- this.deletion = null;
- }
-
- public ColumnOrSuperColumn getColumn_or_supercolumn() {
- return this.column_or_supercolumn;
- }
-
- public Mutation setColumn_or_supercolumn(ColumnOrSuperColumn column_or_supercolumn) {
- this.column_or_supercolumn = column_or_supercolumn;
- return this;
- }
-
- public void unsetColumn_or_supercolumn() {
- this.column_or_supercolumn = null;
- }
-
- /** Returns true if field column_or_supercolumn is set (has been assigned a value) and false otherwise */
- public boolean isSetColumn_or_supercolumn() {
- return this.column_or_supercolumn != null;
- }
-
- public void setColumn_or_supercolumnIsSet(boolean value) {
- if (!value) {
- this.column_or_supercolumn = null;
- }
- }
-
- public Deletion getDeletion() {
- return this.deletion;
- }
-
- public Mutation setDeletion(Deletion deletion) {
- this.deletion = deletion;
- return this;
- }
-
- public void unsetDeletion() {
- this.deletion = null;
- }
-
- /** Returns true if field deletion is set (has been assigned a value) and false otherwise */
- public boolean isSetDeletion() {
- return this.deletion != null;
- }
-
- public void setDeletionIsSet(boolean value) {
- if (!value) {
- this.deletion = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case COLUMN_OR_SUPERCOLUMN:
- if (value == null) {
- unsetColumn_or_supercolumn();
- } else {
- setColumn_or_supercolumn((ColumnOrSuperColumn)value);
- }
- break;
-
- case DELETION:
- if (value == null) {
- unsetDeletion();
- } else {
- setDeletion((Deletion)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case COLUMN_OR_SUPERCOLUMN:
- return getColumn_or_supercolumn();
-
- case DELETION:
- return getDeletion();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case COLUMN_OR_SUPERCOLUMN:
- return isSetColumn_or_supercolumn();
- case DELETION:
- return isSetDeletion();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof Mutation)
- return this.equals((Mutation)that);
- return false;
- }
-
- public boolean equals(Mutation that) {
- if (that == null)
- return false;
-
- boolean this_present_column_or_supercolumn = true && this.isSetColumn_or_supercolumn();
- boolean that_present_column_or_supercolumn = true && that.isSetColumn_or_supercolumn();
- if (this_present_column_or_supercolumn || that_present_column_or_supercolumn) {
- if (!(this_present_column_or_supercolumn && that_present_column_or_supercolumn))
- return false;
- if (!this.column_or_supercolumn.equals(that.column_or_supercolumn))
- return false;
- }
-
- boolean this_present_deletion = true && this.isSetDeletion();
- boolean that_present_deletion = true && that.isSetDeletion();
- if (this_present_deletion || that_present_deletion) {
- if (!(this_present_deletion && that_present_deletion))
- return false;
- if (!this.deletion.equals(that.deletion))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_column_or_supercolumn = true && (isSetColumn_or_supercolumn());
- builder.append(present_column_or_supercolumn);
- if (present_column_or_supercolumn)
- builder.append(column_or_supercolumn);
-
- boolean present_deletion = true && (isSetDeletion());
- builder.append(present_deletion);
- if (present_deletion)
- builder.append(deletion);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(Mutation other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetColumn_or_supercolumn()).compareTo(other.isSetColumn_or_supercolumn());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetColumn_or_supercolumn()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.column_or_supercolumn, other.column_or_supercolumn);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetDeletion()).compareTo(other.isSetDeletion());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetDeletion()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.deletion, other.deletion);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("Mutation(");
- boolean first = true;
-
- if (isSetColumn_or_supercolumn()) {
- sb.append("column_or_supercolumn:");
- if (this.column_or_supercolumn == null) {
- sb.append("null");
- } else {
- sb.append(this.column_or_supercolumn);
- }
- first = false;
- }
- if (isSetDeletion()) {
- if (!first) sb.append(", ");
- sb.append("deletion:");
- if (this.deletion == null) {
- sb.append("null");
- } else {
- sb.append(this.deletion);
- }
- first = false;
- }
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // check for sub-struct validity
- if (column_or_supercolumn != null) {
- column_or_supercolumn.validate();
- }
- if (deletion != null) {
- deletion.validate();
- }
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class MutationStandardSchemeFactory implements SchemeFactory {
- public MutationStandardScheme getScheme() {
- return new MutationStandardScheme();
- }
- }
-
- private static class MutationStandardScheme extends StandardScheme<Mutation> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, Mutation struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // COLUMN_OR_SUPERCOLUMN
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.column_or_supercolumn = new ColumnOrSuperColumn();
- struct.column_or_supercolumn.read(iprot);
- struct.setColumn_or_supercolumnIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 2: // DELETION
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.deletion = new Deletion();
- struct.deletion.read(iprot);
- struct.setDeletionIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, Mutation struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.column_or_supercolumn != null) {
- if (struct.isSetColumn_or_supercolumn()) {
- oprot.writeFieldBegin(COLUMN_OR_SUPERCOLUMN_FIELD_DESC);
- struct.column_or_supercolumn.write(oprot);
- oprot.writeFieldEnd();
- }
- }
- if (struct.deletion != null) {
- if (struct.isSetDeletion()) {
- oprot.writeFieldBegin(DELETION_FIELD_DESC);
- struct.deletion.write(oprot);
- oprot.writeFieldEnd();
- }
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class MutationTupleSchemeFactory implements SchemeFactory {
- public MutationTupleScheme getScheme() {
- return new MutationTupleScheme();
- }
- }
-
- private static class MutationTupleScheme extends TupleScheme<Mutation> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, Mutation struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- BitSet optionals = new BitSet();
- if (struct.isSetColumn_or_supercolumn()) {
- optionals.set(0);
- }
- if (struct.isSetDeletion()) {
- optionals.set(1);
- }
- oprot.writeBitSet(optionals, 2);
- if (struct.isSetColumn_or_supercolumn()) {
- struct.column_or_supercolumn.write(oprot);
- }
- if (struct.isSetDeletion()) {
- struct.deletion.write(oprot);
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, Mutation struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- BitSet incoming = iprot.readBitSet(2);
- if (incoming.get(0)) {
- struct.column_or_supercolumn = new ColumnOrSuperColumn();
- struct.column_or_supercolumn.read(iprot);
- struct.setColumn_or_supercolumnIsSet(true);
- }
- if (incoming.get(1)) {
- struct.deletion = new Deletion();
- struct.deletion.read(iprot);
- struct.setDeletionIsSet(true);
- }
- }
- }
-
-}
-
diff --git a/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java b/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java
deleted file mode 100644
index 0bd8cee..0000000
--- a/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java
+++ /dev/null
@@ -1,307 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.1)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- * @generated
- */
-package org.apache.cassandra.thrift;
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
-
-import org.apache.commons.lang3.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * A specific column was requested that does not exist.
- */
-public class NotFoundException extends TException implements org.apache.thrift.TBase<NotFoundException, NotFoundException._Fields>, java.io.Serializable, Cloneable, Comparable<NotFoundException> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("NotFoundException");
-
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new NotFoundExceptionStandardSchemeFactory());
- schemes.put(TupleScheme.class, new NotFoundExceptionTupleSchemeFactory());
- }
-
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-;
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(NotFoundException.class, metaDataMap);
- }
-
- public NotFoundException() {
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public NotFoundException(NotFoundException other) {
- }
-
- public NotFoundException deepCopy() {
- return new NotFoundException(this);
- }
-
- @Override
- public void clear() {
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof NotFoundException)
- return this.equals((NotFoundException)that);
- return false;
- }
-
- public boolean equals(NotFoundException that) {
- if (that == null)
- return false;
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(NotFoundException other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("NotFoundException(");
- boolean first = true;
-
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class NotFoundExceptionStandardSchemeFactory implements SchemeFactory {
- public NotFoundExceptionStandardScheme getScheme() {
- return new NotFoundExceptionStandardScheme();
- }
- }
-
- private static class NotFoundExceptionStandardScheme extends StandardScheme<NotFoundException> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, NotFoundException struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, NotFoundException struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class NotFoundExceptionTupleSchemeFactory implements SchemeFactory {
- public NotFoundExceptionTupleScheme getScheme() {
- return new NotFoundExceptionTupleScheme();
- }
- }
-
- private static class NotFoundExceptionTupleScheme extends TupleScheme<NotFoundException> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, NotFoundException struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, NotFoundException struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- }
- }
-
-}
-
diff --git a/interface/thrift/gen-java/org/apache/cassandra/thrift/SchemaDisagreementException.java b/interface/thrift/gen-java/org/apache/cassandra/thrift/SchemaDisagreementException.java
deleted file mode 100644
index 003b822..0000000
--- a/interface/thrift/gen-java/org/apache/cassandra/thrift/SchemaDisagreementException.java
+++ /dev/null
@@ -1,310 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.1)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- * @generated
- */
-package org.apache.cassandra.thrift;
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
-
-import org.apache.commons.lang3.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * NOTE: This up outdated exception left for backward compatibility reasons,
- * no actual schema agreement validation is done starting from Cassandra 1.2
- *
- * schemas are not in agreement across all nodes
- */
-public class SchemaDisagreementException extends TException implements org.apache.thrift.TBase<SchemaDisagreementException, SchemaDisagreementException._Fields>, java.io.Serializable, Cloneable, Comparable<SchemaDisagreementException> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("SchemaDisagreementException");
-
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new SchemaDisagreementExceptionStandardSchemeFactory());
- schemes.put(TupleScheme.class, new SchemaDisagreementExceptionTupleSchemeFactory());
- }
-
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-;
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(SchemaDisagreementException.class, metaDataMap);
- }
-
- public SchemaDisagreementException() {
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public SchemaDisagreementException(SchemaDisagreementException other) {
- }
-
- public SchemaDisagreementException deepCopy() {
- return new SchemaDisagreementException(this);
- }
-
- @Override
- public void clear() {
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof SchemaDisagreementException)
- return this.equals((SchemaDisagreementException)that);
- return false;
- }
-
- public boolean equals(SchemaDisagreementException that) {
- if (that == null)
- return false;
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(SchemaDisagreementException other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("SchemaDisagreementException(");
- boolean first = true;
-
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class SchemaDisagreementExceptionStandardSchemeFactory implements SchemeFactory {
- public SchemaDisagreementExceptionStandardScheme getScheme() {
- return new SchemaDisagreementExceptionStandardScheme();
- }
- }
-
- private static class SchemaDisagreementExceptionStandardScheme extends StandardScheme<SchemaDisagreementException> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, SchemaDisagreementException struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, SchemaDisagreementException struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class SchemaDisagreementExceptionTupleSchemeFactory implements SchemeFactory {
- public SchemaDisagreementExceptionTupleScheme getScheme() {
- return new SchemaDisagreementExceptionTupleScheme();
- }
- }
-
- private static class SchemaDisagreementExceptionTupleScheme extends TupleScheme<SchemaDisagreementException> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, SchemaDisagreementException struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, SchemaDisagreementException struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- }
- }
-
-}
-
diff --git a/interface/thrift/gen-java/org/apache/cassandra/thrift/SlicePredicate.java b/interface/thrift/gen-java/org/apache/cassandra/thrift/SlicePredicate.java
deleted file mode 100644
index 9d46680..0000000
--- a/interface/thrift/gen-java/org/apache/cassandra/thrift/SlicePredicate.java
+++ /dev/null
@@ -1,588 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.1)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- * @generated
- */
-package org.apache.cassandra.thrift;
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
-
-import org.apache.commons.lang3.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * A SlicePredicate is similar to a mathematic predicate (see http://en.wikipedia.org/wiki/Predicate_(mathematical_logic)),
- * which is described as "a property that the elements of a set have in common."
- *
- * SlicePredicate's in Cassandra are described with either a list of column_names or a SliceRange. If column_names is
- * specified, slice_range is ignored.
- *
- * @param column_name. A list of column names to retrieve. This can be used similar to Memcached's "multi-get" feature
- * to fetch N known column names. For instance, if you know you wish to fetch columns 'Joe', 'Jack',
- * and 'Jim' you can pass those column names as a list to fetch all three at once.
- * @param slice_range. A SliceRange describing how to range, order, and/or limit the slice.
- */
-public class SlicePredicate implements org.apache.thrift.TBase<SlicePredicate, SlicePredicate._Fields>, java.io.Serializable, Cloneable, Comparable<SlicePredicate> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("SlicePredicate");
-
- private static final org.apache.thrift.protocol.TField COLUMN_NAMES_FIELD_DESC = new org.apache.thrift.protocol.TField("column_names", org.apache.thrift.protocol.TType.LIST, (short)1);
- private static final org.apache.thrift.protocol.TField SLICE_RANGE_FIELD_DESC = new org.apache.thrift.protocol.TField("slice_range", org.apache.thrift.protocol.TType.STRUCT, (short)2);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new SlicePredicateStandardSchemeFactory());
- schemes.put(TupleScheme.class, new SlicePredicateTupleSchemeFactory());
- }
-
- public List<ByteBuffer> column_names; // optional
- public SliceRange slice_range; // optional
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- COLUMN_NAMES((short)1, "column_names"),
- SLICE_RANGE((short)2, "slice_range");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // COLUMN_NAMES
- return COLUMN_NAMES;
- case 2: // SLICE_RANGE
- return SLICE_RANGE;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- private _Fields optionals[] = {_Fields.COLUMN_NAMES,_Fields.SLICE_RANGE};
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.COLUMN_NAMES, new org.apache.thrift.meta_data.FieldMetaData("column_names", org.apache.thrift.TFieldRequirementType.OPTIONAL,
- new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true))));
- tmpMap.put(_Fields.SLICE_RANGE, new org.apache.thrift.meta_data.FieldMetaData("slice_range", org.apache.thrift.TFieldRequirementType.OPTIONAL,
- new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, SliceRange.class)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(SlicePredicate.class, metaDataMap);
- }
-
- public SlicePredicate() {
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public SlicePredicate(SlicePredicate other) {
- if (other.isSetColumn_names()) {
- List<ByteBuffer> __this__column_names = new ArrayList<ByteBuffer>(other.column_names);
- this.column_names = __this__column_names;
- }
- if (other.isSetSlice_range()) {
- this.slice_range = new SliceRange(other.slice_range);
- }
- }
-
- public SlicePredicate deepCopy() {
- return new SlicePredicate(this);
- }
-
- @Override
- public void clear() {
- this.column_names = null;
- this.slice_range = null;
- }
-
- public int getColumn_namesSize() {
- return (this.column_names == null) ? 0 : this.column_names.size();
- }
-
- public java.util.Iterator<ByteBuffer> getColumn_namesIterator() {
- return (this.column_names == null) ? null : this.column_names.iterator();
- }
-
- public void addToColumn_names(ByteBuffer elem) {
- if (this.column_names == null) {
- this.column_names = new ArrayList<ByteBuffer>();
- }
- this.column_names.add(elem);
- }
-
- public List<ByteBuffer> getColumn_names() {
- return this.column_names;
- }
-
- public SlicePredicate setColumn_names(List<ByteBuffer> column_names) {
- this.column_names = column_names;
- return this;
- }
-
- public void unsetColumn_names() {
- this.column_names = null;
- }
-
- /** Returns true if field column_names is set (has been assigned a value) and false otherwise */
- public boolean isSetColumn_names() {
- return this.column_names != null;
- }
-
- public void setColumn_namesIsSet(boolean value) {
- if (!value) {
- this.column_names = null;
- }
- }
-
- public SliceRange getSlice_range() {
- return this.slice_range;
- }
-
- public SlicePredicate setSlice_range(SliceRange slice_range) {
- this.slice_range = slice_range;
- return this;
- }
-
- public void unsetSlice_range() {
- this.slice_range = null;
- }
-
- /** Returns true if field slice_range is set (has been assigned a value) and false otherwise */
- public boolean isSetSlice_range() {
- return this.slice_range != null;
- }
-
- public void setSlice_rangeIsSet(boolean value) {
- if (!value) {
- this.slice_range = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case COLUMN_NAMES:
- if (value == null) {
- unsetColumn_names();
- } else {
- setColumn_names((List<ByteBuffer>)value);
- }
- break;
-
- case SLICE_RANGE:
- if (value == null) {
- unsetSlice_range();
- } else {
- setSlice_range((SliceRange)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case COLUMN_NAMES:
- return getColumn_names();
-
- case SLICE_RANGE:
- return getSlice_range();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case COLUMN_NAMES:
- return isSetColumn_names();
- case SLICE_RANGE:
- return isSetSlice_range();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof SlicePredicate)
- return this.equals((SlicePredicate)that);
- return false;
- }
-
- public boolean equals(SlicePredicate that) {
- if (that == null)
- return false;
-
- boolean this_present_column_names = true && this.isSetColumn_names();
- boolean that_present_column_names = true && that.isSetColumn_names();
- if (this_present_column_names || that_present_column_names) {
- if (!(this_present_column_names && that_present_column_names))
- return false;
- if (!this.column_names.equals(that.column_names))
- return false;
- }
-
- boolean this_present_slice_range = true && this.isSetSlice_range();
- boolean that_present_slice_range = true && that.isSetSlice_range();
- if (this_present_slice_range || that_present_slice_range) {
- if (!(this_present_slice_range && that_present_slice_range))
- return false;
- if (!this.slice_range.equals(that.slice_range))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_column_names = true && (isSetColumn_names());
- builder.append(present_column_names);
- if (present_column_names)
- builder.append(column_names);
-
- boolean present_slice_range = true && (isSetSlice_range());
- builder.append(present_slice_range);
- if (present_slice_range)
- builder.append(slice_range);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(SlicePredicate other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetColumn_names()).compareTo(other.isSetColumn_names());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetColumn_names()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.column_names, other.column_names);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetSlice_range()).compareTo(other.isSetSlice_range());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetSlice_range()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.slice_range, other.slice_range);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("SlicePredicate(");
- boolean first = true;
-
- if (isSetColumn_names()) {
- sb.append("column_names:");
- if (this.column_names == null) {
- sb.append("null");
- } else {
- sb.append(this.column_names);
- }
- first = false;
- }
- if (isSetSlice_range()) {
- if (!first) sb.append(", ");
- sb.append("slice_range:");
- if (this.slice_range == null) {
- sb.append("null");
- } else {
- sb.append(this.slice_range);
- }
- first = false;
- }
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // check for sub-struct validity
- if (slice_range != null) {
- slice_range.validate();
- }
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class SlicePredicateStandardSchemeFactory implements SchemeFactory {
- public SlicePredicateStandardScheme getScheme() {
- return new SlicePredicateStandardScheme();
- }
- }
-
- private static class SlicePredicateStandardScheme extends StandardScheme<SlicePredicate> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, SlicePredicate struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // COLUMN_NAMES
- if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
- {
- org.apache.thrift.protocol.TList _list16 = iprot.readListBegin();
- struct.column_names = new ArrayList<ByteBuffer>(_list16.size);
- for (int _i17 = 0; _i17 < _list16.size; ++_i17)
- {
- ByteBuffer _elem18;
- _elem18 = iprot.readBinary();
- struct.column_names.add(_elem18);
- }
- iprot.readListEnd();
- }
- struct.setColumn_namesIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 2: // SLICE_RANGE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.slice_range = new SliceRange();
- struct.slice_range.read(iprot);
- struct.setSlice_rangeIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, SlicePredicate struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.column_names != null) {
- if (struct.isSetColumn_names()) {
- oprot.writeFieldBegin(COLUMN_NAMES_FIELD_DESC);
- {
- oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, struct.column_names.size()));
- for (ByteBuffer _iter19 : struct.column_names)
- {
- oprot.writeBinary(_iter19);
- }
- oprot.writeListEnd();
- }
- oprot.writeFieldEnd();
- }
- }
- if (struct.slice_range != null) {
- if (struct.isSetSlice_range()) {
- oprot.writeFieldBegin(SLICE_RANGE_FIELD_DESC);
- struct.slice_range.write(oprot);
- oprot.writeFieldEnd();
- }
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class SlicePredicateTupleSchemeFactory implements SchemeFactory {
- public SlicePredicateTupleScheme getScheme() {
- return new SlicePredicateTupleScheme();
- }
- }
-
- private static class SlicePredicateTupleScheme extends TupleScheme<SlicePredicate> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, SlicePredicate struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- BitSet optionals = new BitSet();
- if (struct.isSetColumn_names()) {
- optionals.set(0);
- }
- if (struct.isSetSlice_range()) {
- optionals.set(1);
- }
- oprot.writeBitSet(optionals, 2);
- if (struct.isSetColumn_names()) {
- {
- oprot.writeI32(struct.column_names.size());
- for (ByteBuffer _iter20 : struct.column_names)
- {
- oprot.writeBinary(_iter20);
- }
- }
- }
- if (struct.isSetSlice_range()) {
- struct.slice_range.write(oprot);
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, SlicePredicate struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- BitSet incoming = iprot.readBitSet(2);
- if (incoming.get(0)) {
- {
- org.apache.thrift.protocol.TList _list21 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32());
- struct.column_names = new ArrayList<ByteBuffer>(_list21.size);
- for (int _i22 = 0; _i22 < _list21.size; ++_i22)
- {
- ByteBuffer _elem23;
- _elem23 = iprot.readBinary();
- struct.column_names.add(_elem23);
- }
- }
- struct.setColumn_namesIsSet(true);
- }
- if (incoming.get(1)) {
- struct.slice_range = new SliceRange();
- struct.slice_range.read(iprot);
- struct.setSlice_rangeIsSet(true);
- }
- }
- }
-
-}
-
diff --git a/interface/thrift/gen-java/org/apache/cassandra/thrift/SliceRange.java b/interface/thrift/gen-java/org/apache/cassandra/thrift/SliceRange.java
deleted file mode 100644
index 4b96c86..0000000
--- a/interface/thrift/gen-java/org/apache/cassandra/thrift/SliceRange.java
+++ /dev/null
@@ -1,749 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.1)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- * @generated
- */
-package org.apache.cassandra.thrift;
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
-
-import org.apache.commons.lang3.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * A slice range is a structure that stores basic range, ordering and limit information for a query that will return
- * multiple columns. It could be thought of as Cassandra's version of LIMIT and ORDER BY
- *
- * @param start. The column name to start the slice with. This attribute is not required, though there is no default value,
- * and can be safely set to '', i.e., an empty byte array, to start with the first column name. Otherwise, it
- * must a valid value under the rules of the Comparator defined for the given ColumnFamily.
- * @param finish. The column name to stop the slice at. This attribute is not required, though there is no default value,
- * and can be safely set to an empty byte array to not stop until 'count' results are seen. Otherwise, it
- * must also be a valid value to the ColumnFamily Comparator.
- * @param reversed. Whether the results should be ordered in reversed order. Similar to ORDER BY blah DESC in SQL.
- * @param count. How many columns to return. Similar to LIMIT in SQL. May be arbitrarily large, but Thrift will
- * materialize the whole result into memory before returning it to the client, so be aware that you may
- * be better served by iterating through slices by passing the last value of one call in as the 'start'
- * of the next instead of increasing 'count' arbitrarily large.
- */
-public class SliceRange implements org.apache.thrift.TBase<SliceRange, SliceRange._Fields>, java.io.Serializable, Cloneable, Comparable<SliceRange> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("SliceRange");
-
- private static final org.apache.thrift.protocol.TField START_FIELD_DESC = new org.apache.thrift.protocol.TField("start", org.apache.thrift.protocol.TType.STRING, (short)1);
- private static final org.apache.thrift.protocol.TField FINISH_FIELD_DESC = new org.apache.thrift.protocol.TField("finish", org.apache.thrift.protocol.TType.STRING, (short)2);
- private static final org.apache.thrift.protocol.TField REVERSED_FIELD_DESC = new org.apache.thrift.protocol.TField("reversed", org.apache.thrift.protocol.TType.BOOL, (short)3);
- private static final org.apache.thrift.protocol.TField COUNT_FIELD_DESC = new org.apache.thrift.protocol.TField("count", org.apache.thrift.protocol.TType.I32, (short)4);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new SliceRangeStandardSchemeFactory());
- schemes.put(TupleScheme.class, new SliceRangeTupleSchemeFactory());
- }
-
- public ByteBuffer start; // required
- public ByteBuffer finish; // required
- public boolean reversed; // required
- public int count; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- START((short)1, "start"),
- FINISH((short)2, "finish"),
- REVERSED((short)3, "reversed"),
- COUNT((short)4, "count");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // START
- return START;
- case 2: // FINISH
- return FINISH;
- case 3: // REVERSED
- return REVERSED;
- case 4: // COUNT
- return COUNT;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- private static final int __REVERSED_ISSET_ID = 0;
- private static final int __COUNT_ISSET_ID = 1;
- private byte __isset_bitfield = 0;
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.START, new org.apache.thrift.meta_data.FieldMetaData("start", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true)));
- tmpMap.put(_Fields.FINISH, new org.apache.thrift.meta_data.FieldMetaData("finish", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true)));
- tmpMap.put(_Fields.REVERSED, new org.apache.thrift.meta_data.FieldMetaData("reversed", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.BOOL)));
- tmpMap.put(_Fields.COUNT, new org.apache.thrift.meta_data.FieldMetaData("count", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(SliceRange.class, metaDataMap);
- }
-
- public SliceRange() {
- this.reversed = false;
-
- this.count = 100;
-
- }
-
- public SliceRange(
- ByteBuffer start,
- ByteBuffer finish,
- boolean reversed,
- int count)
- {
- this();
- this.start = start;
- this.finish = finish;
- this.reversed = reversed;
- setReversedIsSet(true);
- this.count = count;
- setCountIsSet(true);
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public SliceRange(SliceRange other) {
- __isset_bitfield = other.__isset_bitfield;
- if (other.isSetStart()) {
- this.start = org.apache.thrift.TBaseHelper.copyBinary(other.start);
-;
- }
- if (other.isSetFinish()) {
- this.finish = org.apache.thrift.TBaseHelper.copyBinary(other.finish);
-;
- }
- this.reversed = other.reversed;
- this.count = other.count;
- }
-
- public SliceRange deepCopy() {
- return new SliceRange(this);
- }
-
- @Override
- public void clear() {
- this.start = null;
- this.finish = null;
- this.reversed = false;
-
- this.count = 100;
-
- }
-
- public byte[] getStart() {
- setStart(org.apache.thrift.TBaseHelper.rightSize(start));
- return start == null ? null : start.array();
- }
-
- public ByteBuffer bufferForStart() {
- return start;
- }
-
- public SliceRange setStart(byte[] start) {
- setStart(start == null ? (ByteBuffer)null : ByteBuffer.wrap(start));
- return this;
- }
-
- public SliceRange setStart(ByteBuffer start) {
- this.start = start;
- return this;
- }
-
- public void unsetStart() {
- this.start = null;
- }
-
- /** Returns true if field start is set (has been assigned a value) and false otherwise */
- public boolean isSetStart() {
- return this.start != null;
- }
-
- public void setStartIsSet(boolean value) {
- if (!value) {
- this.start = null;
- }
- }
-
- public byte[] getFinish() {
- setFinish(org.apache.thrift.TBaseHelper.rightSize(finish));
- return finish == null ? null : finish.array();
- }
-
- public ByteBuffer bufferForFinish() {
- return finish;
- }
-
- public SliceRange setFinish(byte[] finish) {
- setFinish(finish == null ? (ByteBuffer)null : ByteBuffer.wrap(finish));
- return this;
- }
-
- public SliceRange setFinish(ByteBuffer finish) {
- this.finish = finish;
- return this;
- }
-
- public void unsetFinish() {
- this.finish = null;
- }
-
- /** Returns true if field finish is set (has been assigned a value) and false otherwise */
- public boolean isSetFinish() {
- return this.finish != null;
- }
-
- public void setFinishIsSet(boolean value) {
- if (!value) {
- this.finish = null;
- }
- }
-
- public boolean isReversed() {
- return this.reversed;
- }
-
- public SliceRange setReversed(boolean reversed) {
- this.reversed = reversed;
- setReversedIsSet(true);
- return this;
- }
-
- public void unsetReversed() {
- __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __REVERSED_ISSET_ID);
- }
-
- /** Returns true if field reversed is set (has been assigned a value) and false otherwise */
- public boolean isSetReversed() {
- return EncodingUtils.testBit(__isset_bitfield, __REVERSED_ISSET_ID);
- }
-
- public void setReversedIsSet(boolean value) {
- __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __REVERSED_ISSET_ID, value);
- }
-
- public int getCount() {
- return this.count;
- }
-
- public SliceRange setCount(int count) {
- this.count = count;
- setCountIsSet(true);
- return this;
- }
-
- public void unsetCount() {
- __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __COUNT_ISSET_ID);
- }
-
- /** Returns true if field count is set (has been assigned a value) and false otherwise */
- public boolean isSetCount() {
- return EncodingUtils.testBit(__isset_bitfield, __COUNT_ISSET_ID);
- }
-
- public void setCountIsSet(boolean value) {
- __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __COUNT_ISSET_ID, value);
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case START:
- if (value == null) {
- unsetStart();
- } else {
- setStart((ByteBuffer)value);
- }
- break;
-
- case FINISH:
- if (value == null) {
- unsetFinish();
- } else {
- setFinish((ByteBuffer)value);
- }
- break;
-
- case REVERSED:
- if (value == null) {
- unsetReversed();
- } else {
- setReversed((Boolean)value);
- }
- break;
-
- case COUNT:
- if (value == null) {
- unsetCount();
- } else {
- setCount((Integer)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case START:
- return getStart();
-
- case FINISH:
- return getFinish();
-
- case REVERSED:
- return Boolean.valueOf(isReversed());
-
- case COUNT:
- return Integer.valueOf(getCount());
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case START:
- return isSetStart();
- case FINISH:
- return isSetFinish();
- case REVERSED:
- return isSetReversed();
- case COUNT:
- return isSetCount();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof SliceRange)
- return this.equals((SliceRange)that);
- return false;
- }
-
- public boolean equals(SliceRange that) {
- if (that == null)
- return false;
-
- boolean this_present_start = true && this.isSetStart();
- boolean that_present_start = true && that.isSetStart();
- if (this_present_start || that_present_start) {
- if (!(this_present_start && that_present_start))
- return false;
- if (!this.start.equals(that.start))
- return false;
- }
-
- boolean this_present_finish = true && this.isSetFinish();
- boolean that_present_finish = true && that.isSetFinish();
- if (this_present_finish || that_present_finish) {
- if (!(this_present_finish && that_present_finish))
- return false;
- if (!this.finish.equals(that.finish))
- return false;
- }
-
- boolean this_present_reversed = true;
- boolean that_present_reversed = true;
- if (this_present_reversed || that_present_reversed) {
- if (!(this_present_reversed && that_present_reversed))
- return false;
- if (this.reversed != that.reversed)
- return false;
- }
-
- boolean this_present_count = true;
- boolean that_present_count = true;
- if (this_present_count || that_present_count) {
- if (!(this_present_count && that_present_count))
- return false;
- if (this.count != that.count)
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_start = true && (isSetStart());
- builder.append(present_start);
- if (present_start)
- builder.append(start);
-
- boolean present_finish = true && (isSetFinish());
- builder.append(present_finish);
- if (present_finish)
- builder.append(finish);
-
- boolean present_reversed = true;
- builder.append(present_reversed);
- if (present_reversed)
- builder.append(reversed);
-
- boolean present_count = true;
- builder.append(present_count);
- if (present_count)
- builder.append(count);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(SliceRange other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetStart()).compareTo(other.isSetStart());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetStart()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.start, other.start);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetFinish()).compareTo(other.isSetFinish());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetFinish()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.finish, other.finish);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetReversed()).compareTo(other.isSetReversed());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetReversed()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.reversed, other.reversed);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetCount()).compareTo(other.isSetCount());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetCount()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.count, other.count);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("SliceRange(");
- boolean first = true;
-
- sb.append("start:");
- if (this.start == null) {
- sb.append("null");
- } else {
- org.apache.thrift.TBaseHelper.toString(this.start, sb);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("finish:");
- if (this.finish == null) {
- sb.append("null");
- } else {
- org.apache.thrift.TBaseHelper.toString(this.finish, sb);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("reversed:");
- sb.append(this.reversed);
- first = false;
- if (!first) sb.append(", ");
- sb.append("count:");
- sb.append(this.count);
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- if (start == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'start' was not present! Struct: " + toString());
- }
- if (finish == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'finish' was not present! Struct: " + toString());
- }
- // alas, we cannot check 'reversed' because it's a primitive and you chose the non-beans generator.
- // alas, we cannot check 'count' because it's a primitive and you chose the non-beans generator.
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
- __isset_bitfield = 0;
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class SliceRangeStandardSchemeFactory implements SchemeFactory {
- public SliceRangeStandardScheme getScheme() {
- return new SliceRangeStandardScheme();
- }
- }
-
- private static class SliceRangeStandardScheme extends StandardScheme<SliceRange> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, SliceRange struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // START
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.start = iprot.readBinary();
- struct.setStartIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 2: // FINISH
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.finish = iprot.readBinary();
- struct.setFinishIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 3: // REVERSED
- if (schemeField.type == org.apache.thrift.protocol.TType.BOOL) {
- struct.reversed = iprot.readBool();
- struct.setReversedIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 4: // COUNT
- if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
- struct.count = iprot.readI32();
- struct.setCountIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- if (!struct.isSetReversed()) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'reversed' was not found in serialized data! Struct: " + toString());
- }
- if (!struct.isSetCount()) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'count' was not found in serialized data! Struct: " + toString());
- }
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, SliceRange struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.start != null) {
- oprot.writeFieldBegin(START_FIELD_DESC);
- oprot.writeBinary(struct.start);
- oprot.writeFieldEnd();
- }
- if (struct.finish != null) {
- oprot.writeFieldBegin(FINISH_FIELD_DESC);
- oprot.writeBinary(struct.finish);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldBegin(REVERSED_FIELD_DESC);
- oprot.writeBool(struct.reversed);
- oprot.writeFieldEnd();
- oprot.writeFieldBegin(COUNT_FIELD_DESC);
- oprot.writeI32(struct.count);
- oprot.writeFieldEnd();
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class SliceRangeTupleSchemeFactory implements SchemeFactory {
- public SliceRangeTupleScheme getScheme() {
- return new SliceRangeTupleScheme();
- }
- }
-
- private static class SliceRangeTupleScheme extends TupleScheme<SliceRange> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, SliceRange struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- oprot.writeBinary(struct.start);
- oprot.writeBinary(struct.finish);
- oprot.writeBool(struct.reversed);
- oprot.writeI32(struct.count);
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, SliceRange struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- struct.start = iprot.readBinary();
- struct.setStartIsSet(true);
- struct.finish = iprot.readBinary();
- struct.setFinishIsSet(true);
- struct.reversed = iprot.readBool();
- struct.setReversedIsSet(true);
- struct.count = iprot.readI32();
- struct.setCountIsSet(true);
- }
- }
-
-}
-
diff --git a/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java b/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java
deleted file mode 100644
index 37215bf..0000000
--- a/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java
+++ /dev/null
@@ -1,582 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.1)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- * @generated
- */
-package org.apache.cassandra.thrift;
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
-
-import org.apache.commons.lang3.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * A named list of columns.
- * @param name. see Column.name.
- * @param columns. A collection of standard Columns. The columns within a super column are defined in an adhoc manner.
- * Columns within a super column do not have to have matching structures (similarly named child columns).
- */
-public class SuperColumn implements org.apache.thrift.TBase<SuperColumn, SuperColumn._Fields>, java.io.Serializable, Cloneable, Comparable<SuperColumn> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("SuperColumn");
-
- private static final org.apache.thrift.protocol.TField NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("name", org.apache.thrift.protocol.TType.STRING, (short)1);
- private static final org.apache.thrift.protocol.TField COLUMNS_FIELD_DESC = new org.apache.thrift.protocol.TField("columns", org.apache.thrift.protocol.TType.LIST, (short)2);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new SuperColumnStandardSchemeFactory());
- schemes.put(TupleScheme.class, new SuperColumnTupleSchemeFactory());
- }
-
- public ByteBuffer name; // required
- public List<Column> columns; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- NAME((short)1, "name"),
- COLUMNS((short)2, "columns");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // NAME
- return NAME;
- case 2: // COLUMNS
- return COLUMNS;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.NAME, new org.apache.thrift.meta_data.FieldMetaData("name", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true)));
- tmpMap.put(_Fields.COLUMNS, new org.apache.thrift.meta_data.FieldMetaData("columns", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST,
- new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, Column.class))));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(SuperColumn.class, metaDataMap);
- }
-
- public SuperColumn() {
- }
-
- public SuperColumn(
- ByteBuffer name,
- List<Column> columns)
- {
- this();
- this.name = name;
- this.columns = columns;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public SuperColumn(SuperColumn other) {
- if (other.isSetName()) {
- this.name = org.apache.thrift.TBaseHelper.copyBinary(other.name);
-;
- }
- if (other.isSetColumns()) {
- List<Column> __this__columns = new ArrayList<Column>(other.columns.size());
- for (Column other_element : other.columns) {
- __this__columns.add(new Column(other_element));
- }
- this.columns = __this__columns;
- }
- }
-
- public SuperColumn deepCopy() {
- return new SuperColumn(this);
- }
-
- @Override
- public void clear() {
- this.name = null;
- this.columns = null;
- }
-
- public byte[] getName() {
- setName(org.apache.thrift.TBaseHelper.rightSize(name));
- return name == null ? null : name.array();
- }
-
- public ByteBuffer bufferForName() {
- return name;
- }
-
- public SuperColumn setName(byte[] name) {
- setName(name == null ? (ByteBuffer)null : ByteBuffer.wrap(name));
- return this;
- }
-
- public SuperColumn setName(ByteBuffer name) {
- this.name = name;
- return this;
- }
-
- public void unsetName() {
- this.name = null;
- }
-
- /** Returns true if field name is set (has been assigned a value) and false otherwise */
- public boolean isSetName() {
- return this.name != null;
- }
-
- public void setNameIsSet(boolean value) {
- if (!value) {
- this.name = null;
- }
- }
-
- public int getColumnsSize() {
- return (this.columns == null) ? 0 : this.columns.size();
- }
-
- public java.util.Iterator<Column> getColumnsIterator() {
- return (this.columns == null) ? null : this.columns.iterator();
- }
-
- public void addToColumns(Column elem) {
- if (this.columns == null) {
- this.columns = new ArrayList<Column>();
- }
- this.columns.add(elem);
- }
-
- public List<Column> getColumns() {
- return this.columns;
- }
-
- public SuperColumn setColumns(List<Column> columns) {
- this.columns = columns;
- return this;
- }
-
- public void unsetColumns() {
- this.columns = null;
- }
-
- /** Returns true if field columns is set (has been assigned a value) and false otherwise */
- public boolean isSetColumns() {
- return this.columns != null;
- }
-
- public void setColumnsIsSet(boolean value) {
- if (!value) {
- this.columns = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case NAME:
- if (value == null) {
- unsetName();
- } else {
- setName((ByteBuffer)value);
- }
- break;
-
- case COLUMNS:
- if (value == null) {
- unsetColumns();
- } else {
- setColumns((List<Column>)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case NAME:
- return getName();
-
- case COLUMNS:
- return getColumns();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case NAME:
- return isSetName();
- case COLUMNS:
- return isSetColumns();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof SuperColumn)
- return this.equals((SuperColumn)that);
- return false;
- }
-
- public boolean equals(SuperColumn that) {
- if (that == null)
- return false;
-
- boolean this_present_name = true && this.isSetName();
- boolean that_present_name = true && that.isSetName();
- if (this_present_name || that_present_name) {
- if (!(this_present_name && that_present_name))
- return false;
- if (!this.name.equals(that.name))
- return false;
- }
-
- boolean this_present_columns = true && this.isSetColumns();
- boolean that_present_columns = true && that.isSetColumns();
- if (this_present_columns || that_present_columns) {
- if (!(this_present_columns && that_present_columns))
- return false;
- if (!this.columns.equals(that.columns))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_name = true && (isSetName());
- builder.append(present_name);
- if (present_name)
- builder.append(name);
-
- boolean present_columns = true && (isSetColumns());
- builder.append(present_columns);
- if (present_columns)
- builder.append(columns);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(SuperColumn other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetName()).compareTo(other.isSetName());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetName()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.name, other.name);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetColumns()).compareTo(other.isSetColumns());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetColumns()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.columns, other.columns);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("SuperColumn(");
- boolean first = true;
-
- sb.append("name:");
- if (this.name == null) {
- sb.append("null");
- } else {
- org.apache.thrift.TBaseHelper.toString(this.name, sb);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("columns:");
- if (this.columns == null) {
- sb.append("null");
- } else {
- sb.append(this.columns);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- if (name == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'name' was not present! Struct: " + toString());
- }
- if (columns == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'columns' was not present! Struct: " + toString());
- }
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class SuperColumnStandardSchemeFactory implements SchemeFactory {
- public SuperColumnStandardScheme getScheme() {
- return new SuperColumnStandardScheme();
- }
- }
-
- private static class SuperColumnStandardScheme extends StandardScheme<SuperColumn> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, SuperColumn struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // NAME
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.name = iprot.readBinary();
- struct.setNameIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 2: // COLUMNS
- if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
- {
- org.apache.thrift.protocol.TList _list0 = iprot.readListBegin();
- struct.columns = new ArrayList<Column>(_list0.size);
- for (int _i1 = 0; _i1 < _list0.size; ++_i1)
- {
- Column _elem2;
- _elem2 = new Column();
- _elem2.read(iprot);
- struct.columns.add(_elem2);
- }
- iprot.readListEnd();
- }
- struct.setColumnsIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, SuperColumn struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.name != null) {
- oprot.writeFieldBegin(NAME_FIELD_DESC);
- oprot.writeBinary(struct.name);
- oprot.writeFieldEnd();
- }
- if (struct.columns != null) {
- oprot.writeFieldBegin(COLUMNS_FIELD_DESC);
- {
- oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.columns.size()));
- for (Column _iter3 : struct.columns)
- {
- _iter3.write(oprot);
- }
- oprot.writeListEnd();
- }
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class SuperColumnTupleSchemeFactory implements SchemeFactory {
- public SuperColumnTupleScheme getScheme() {
- return new SuperColumnTupleScheme();
- }
- }
-
- private static class SuperColumnTupleScheme extends TupleScheme<SuperColumn> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, SuperColumn struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- oprot.writeBinary(struct.name);
- {
- oprot.writeI32(struct.columns.size());
- for (Column _iter4 : struct.columns)
- {
- _iter4.write(oprot);
- }
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, SuperColumn struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- struct.name = iprot.readBinary();
- struct.setNameIsSet(true);
- {
- org.apache.thrift.protocol.TList _list5 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
- struct.columns = new ArrayList<Column>(_list5.size);
- for (int _i6 = 0; _i6 < _list5.size; ++_i6)
- {
- Column _elem7;
- _elem7 = new Column();
- _elem7.read(iprot);
- struct.columns.add(_elem7);
- }
- }
- struct.setColumnsIsSet(true);
- }
- }
-
-}
-
diff --git a/interface/thrift/gen-java/org/apache/cassandra/thrift/TimedOutException.java b/interface/thrift/gen-java/org/apache/cassandra/thrift/TimedOutException.java
deleted file mode 100644
index 2dafe85..0000000
--- a/interface/thrift/gen-java/org/apache/cassandra/thrift/TimedOutException.java
+++ /dev/null
@@ -1,671 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.1)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- * @generated
- */
-package org.apache.cassandra.thrift;
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
-
-import org.apache.commons.lang3.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * RPC timeout was exceeded. either a node failed mid-operation, or load was too high, or the requested op was too large.
- */
-public class TimedOutException extends TException implements org.apache.thrift.TBase<TimedOutException, TimedOutException._Fields>, java.io.Serializable, Cloneable, Comparable<TimedOutException> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TimedOutException");
-
- private static final org.apache.thrift.protocol.TField ACKNOWLEDGED_BY_FIELD_DESC = new org.apache.thrift.protocol.TField("acknowledged_by", org.apache.thrift.protocol.TType.I32, (short)1);
- private static final org.apache.thrift.protocol.TField ACKNOWLEDGED_BY_BATCHLOG_FIELD_DESC = new org.apache.thrift.protocol.TField("acknowledged_by_batchlog", org.apache.thrift.protocol.TType.BOOL, (short)2);
- private static final org.apache.thrift.protocol.TField PAXOS_IN_PROGRESS_FIELD_DESC = new org.apache.thrift.protocol.TField("paxos_in_progress", org.apache.thrift.protocol.TType.BOOL, (short)3);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new TimedOutExceptionStandardSchemeFactory());
- schemes.put(TupleScheme.class, new TimedOutExceptionTupleSchemeFactory());
- }
-
- /**
- * if a write operation was acknowledged by some replicas but not by enough to
- * satisfy the required ConsistencyLevel, the number of successful
- * replies will be given here. In case of atomic_batch_mutate method this field
- * will be set to -1 if the batch was written to the batchlog and to 0 if it wasn't.
- */
- public int acknowledged_by; // optional
- /**
- * in case of atomic_batch_mutate method this field tells if the batch
- * was written to the batchlog.
- */
- public boolean acknowledged_by_batchlog; // optional
- /**
- * for the CAS method, this field tells if we timed out during the paxos
- * protocol, as opposed to during the commit of our update
- */
- public boolean paxos_in_progress; // optional
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- /**
- * if a write operation was acknowledged by some replicas but not by enough to
- * satisfy the required ConsistencyLevel, the number of successful
- * replies will be given here. In case of atomic_batch_mutate method this field
- * will be set to -1 if the batch was written to the batchlog and to 0 if it wasn't.
- */
- ACKNOWLEDGED_BY((short)1, "acknowledged_by"),
- /**
- * in case of atomic_batch_mutate method this field tells if the batch
- * was written to the batchlog.
- */
- ACKNOWLEDGED_BY_BATCHLOG((short)2, "acknowledged_by_batchlog"),
- /**
- * for the CAS method, this field tells if we timed out during the paxos
- * protocol, as opposed to during the commit of our update
- */
- PAXOS_IN_PROGRESS((short)3, "paxos_in_progress");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // ACKNOWLEDGED_BY
- return ACKNOWLEDGED_BY;
- case 2: // ACKNOWLEDGED_BY_BATCHLOG
- return ACKNOWLEDGED_BY_BATCHLOG;
- case 3: // PAXOS_IN_PROGRESS
- return PAXOS_IN_PROGRESS;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- private static final int __ACKNOWLEDGED_BY_ISSET_ID = 0;
- private static final int __ACKNOWLEDGED_BY_BATCHLOG_ISSET_ID = 1;
- private static final int __PAXOS_IN_PROGRESS_ISSET_ID = 2;
- private byte __isset_bitfield = 0;
- private _Fields optionals[] = {_Fields.ACKNOWLEDGED_BY,_Fields.ACKNOWLEDGED_BY_BATCHLOG,_Fields.PAXOS_IN_PROGRESS};
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.ACKNOWLEDGED_BY, new org.apache.thrift.meta_data.FieldMetaData("acknowledged_by", org.apache.thrift.TFieldRequirementType.OPTIONAL,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32)));
- tmpMap.put(_Fields.ACKNOWLEDGED_BY_BATCHLOG, new org.apache.thrift.meta_data.FieldMetaData("acknowledged_by_batchlog", org.apache.thrift.TFieldRequirementType.OPTIONAL,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.BOOL)));
- tmpMap.put(_Fields.PAXOS_IN_PROGRESS, new org.apache.thrift.meta_data.FieldMetaData("paxos_in_progress", org.apache.thrift.TFieldRequirementType.OPTIONAL,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.BOOL)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TimedOutException.class, metaDataMap);
- }
-
- public TimedOutException() {
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public TimedOutException(TimedOutException other) {
- __isset_bitfield = other.__isset_bitfield;
- this.acknowledged_by = other.acknowledged_by;
- this.acknowledged_by_batchlog = other.acknowledged_by_batchlog;
- this.paxos_in_progress = other.paxos_in_progress;
- }
-
- public TimedOutException deepCopy() {
- return new TimedOutException(this);
- }
-
- @Override
- public void clear() {
- setAcknowledged_byIsSet(false);
- this.acknowledged_by = 0;
- setAcknowledged_by_batchlogIsSet(false);
- this.acknowledged_by_batchlog = false;
- setPaxos_in_progressIsSet(false);
- this.paxos_in_progress = false;
- }
-
- /**
- * if a write operation was acknowledged by some replicas but not by enough to
- * satisfy the required ConsistencyLevel, the number of successful
- * replies will be given here. In case of atomic_batch_mutate method this field
- * will be set to -1 if the batch was written to the batchlog and to 0 if it wasn't.
- */
- public int getAcknowledged_by() {
- return this.acknowledged_by;
- }
-
- /**
- * if a write operation was acknowledged by some replicas but not by enough to
- * satisfy the required ConsistencyLevel, the number of successful
- * replies will be given here. In case of atomic_batch_mutate method this field
- * will be set to -1 if the batch was written to the batchlog and to 0 if it wasn't.
- */
- public TimedOutException setAcknowledged_by(int acknowledged_by) {
- this.acknowledged_by = acknowledged_by;
- setAcknowledged_byIsSet(true);
- return this;
- }
-
- public void unsetAcknowledged_by() {
- __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __ACKNOWLEDGED_BY_ISSET_ID);
- }
-
- /** Returns true if field acknowledged_by is set (has been assigned a value) and false otherwise */
- public boolean isSetAcknowledged_by() {
- return EncodingUtils.testBit(__isset_bitfield, __ACKNOWLEDGED_BY_ISSET_ID);
- }
-
- public void setAcknowledged_byIsSet(boolean value) {
- __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __ACKNOWLEDGED_BY_ISSET_ID, value);
- }
-
- /**
- * in case of atomic_batch_mutate method this field tells if the batch
- * was written to the batchlog.
- */
- public boolean isAcknowledged_by_batchlog() {
- return this.acknowledged_by_batchlog;
- }
-
- /**
- * in case of atomic_batch_mutate method this field tells if the batch
- * was written to the batchlog.
- */
- public TimedOutException setAcknowledged_by_batchlog(boolean acknowledged_by_batchlog) {
- this.acknowledged_by_batchlog = acknowledged_by_batchlog;
- setAcknowledged_by_batchlogIsSet(true);
- return this;
- }
-
- public void unsetAcknowledged_by_batchlog() {
- __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __ACKNOWLEDGED_BY_BATCHLOG_ISSET_ID);
- }
-
- /** Returns true if field acknowledged_by_batchlog is set (has been assigned a value) and false otherwise */
- public boolean isSetAcknowledged_by_batchlog() {
- return EncodingUtils.testBit(__isset_bitfield, __ACKNOWLEDGED_BY_BATCHLOG_ISSET_ID);
- }
-
- public void setAcknowledged_by_batchlogIsSet(boolean value) {
- __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __ACKNOWLEDGED_BY_BATCHLOG_ISSET_ID, value);
- }
-
- /**
- * for the CAS method, this field tells if we timed out during the paxos
- * protocol, as opposed to during the commit of our update
- */
- public boolean isPaxos_in_progress() {
- return this.paxos_in_progress;
- }
-
- /**
- * for the CAS method, this field tells if we timed out during the paxos
- * protocol, as opposed to during the commit of our update
- */
- public TimedOutException setPaxos_in_progress(boolean paxos_in_progress) {
- this.paxos_in_progress = paxos_in_progress;
- setPaxos_in_progressIsSet(true);
- return this;
- }
-
- public void unsetPaxos_in_progress() {
- __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __PAXOS_IN_PROGRESS_ISSET_ID);
- }
-
- /** Returns true if field paxos_in_progress is set (has been assigned a value) and false otherwise */
- public boolean isSetPaxos_in_progress() {
- return EncodingUtils.testBit(__isset_bitfield, __PAXOS_IN_PROGRESS_ISSET_ID);
- }
-
- public void setPaxos_in_progressIsSet(boolean value) {
- __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __PAXOS_IN_PROGRESS_ISSET_ID, value);
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case ACKNOWLEDGED_BY:
- if (value == null) {
- unsetAcknowledged_by();
- } else {
- setAcknowledged_by((Integer)value);
- }
- break;
-
- case ACKNOWLEDGED_BY_BATCHLOG:
- if (value == null) {
- unsetAcknowledged_by_batchlog();
- } else {
- setAcknowledged_by_batchlog((Boolean)value);
- }
- break;
-
- case PAXOS_IN_PROGRESS:
- if (value == null) {
- unsetPaxos_in_progress();
- } else {
- setPaxos_in_progress((Boolean)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case ACKNOWLEDGED_BY:
- return Integer.valueOf(getAcknowledged_by());
-
- case ACKNOWLEDGED_BY_BATCHLOG:
- return Boolean.valueOf(isAcknowledged_by_batchlog());
-
- case PAXOS_IN_PROGRESS:
- return Boolean.valueOf(isPaxos_in_progress());
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case ACKNOWLEDGED_BY:
- return isSetAcknowledged_by();
- case ACKNOWLEDGED_BY_BATCHLOG:
- return isSetAcknowledged_by_batchlog();
- case PAXOS_IN_PROGRESS:
- return isSetPaxos_in_progress();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof TimedOutException)
- return this.equals((TimedOutException)that);
- return false;
- }
-
- public boolean equals(TimedOutException that) {
- if (that == null)
- return false;
-
- boolean this_present_acknowledged_by = true && this.isSetAcknowledged_by();
- boolean that_present_acknowledged_by = true && that.isSetAcknowledged_by();
- if (this_present_acknowledged_by || that_present_acknowledged_by) {
- if (!(this_present_acknowledged_by && that_present_acknowledged_by))
- return false;
- if (this.acknowledged_by != that.acknowledged_by)
- return false;
- }
-
- boolean this_present_acknowledged_by_batchlog = true && this.isSetAcknowledged_by_batchlog();
- boolean that_present_acknowledged_by_batchlog = true && that.isSetAcknowledged_by_batchlog();
- if (this_present_acknowledged_by_batchlog || that_present_acknowledged_by_batchlog) {
- if (!(this_present_acknowledged_by_batchlog && that_present_acknowledged_by_batchlog))
- return false;
- if (this.acknowledged_by_batchlog != that.acknowledged_by_batchlog)
- return false;
- }
-
- boolean this_present_paxos_in_progress = true && this.isSetPaxos_in_progress();
- boolean that_present_paxos_in_progress = true && that.isSetPaxos_in_progress();
- if (this_present_paxos_in_progress || that_present_paxos_in_progress) {
- if (!(this_present_paxos_in_progress && that_present_paxos_in_progress))
- return false;
- if (this.paxos_in_progress != that.paxos_in_progress)
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_acknowledged_by = true && (isSetAcknowledged_by());
- builder.append(present_acknowledged_by);
- if (present_acknowledged_by)
- builder.append(acknowledged_by);
-
- boolean present_acknowledged_by_batchlog = true && (isSetAcknowledged_by_batchlog());
- builder.append(present_acknowledged_by_batchlog);
- if (present_acknowledged_by_batchlog)
- builder.append(acknowledged_by_batchlog);
-
- boolean present_paxos_in_progress = true && (isSetPaxos_in_progress());
- builder.append(present_paxos_in_progress);
- if (present_paxos_in_progress)
- builder.append(paxos_in_progress);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(TimedOutException other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetAcknowledged_by()).compareTo(other.isSetAcknowledged_by());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetAcknowledged_by()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.acknowledged_by, other.acknowledged_by);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetAcknowledged_by_batchlog()).compareTo(other.isSetAcknowledged_by_batchlog());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetAcknowledged_by_batchlog()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.acknowledged_by_batchlog, other.acknowledged_by_batchlog);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetPaxos_in_progress()).compareTo(other.isSetPaxos_in_progress());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetPaxos_in_progress()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.paxos_in_progress, other.paxos_in_progress);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("TimedOutException(");
- boolean first = true;
-
- if (isSetAcknowledged_by()) {
- sb.append("acknowledged_by:");
- sb.append(this.acknowledged_by);
- first = false;
- }
- if (isSetAcknowledged_by_batchlog()) {
- if (!first) sb.append(", ");
- sb.append("acknowledged_by_batchlog:");
- sb.append(this.acknowledged_by_batchlog);
- first = false;
- }
- if (isSetPaxos_in_progress()) {
- if (!first) sb.append(", ");
- sb.append("paxos_in_progress:");
- sb.append(this.paxos_in_progress);
- first = false;
- }
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
- __isset_bitfield = 0;
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class TimedOutExceptionStandardSchemeFactory implements SchemeFactory {
- public TimedOutExceptionStandardScheme getScheme() {
- return new TimedOutExceptionStandardScheme();
- }
- }
-
- private static class TimedOutExceptionStandardScheme extends StandardScheme<TimedOutException> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, TimedOutException struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // ACKNOWLEDGED_BY
- if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
- struct.acknowledged_by = iprot.readI32();
- struct.setAcknowledged_byIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 2: // ACKNOWLEDGED_BY_BATCHLOG
- if (schemeField.type == org.apache.thrift.protocol.TType.BOOL) {
- struct.acknowledged_by_batchlog = iprot.readBool();
- struct.setAcknowledged_by_batchlogIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 3: // PAXOS_IN_PROGRESS
- if (schemeField.type == org.apache.thrift.protocol.TType.BOOL) {
- struct.paxos_in_progress = iprot.readBool();
- struct.setPaxos_in_progressIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, TimedOutException struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.isSetAcknowledged_by()) {
- oprot.writeFieldBegin(ACKNOWLEDGED_BY_FIELD_DESC);
- oprot.writeI32(struct.acknowledged_by);
- oprot.writeFieldEnd();
- }
- if (struct.isSetAcknowledged_by_batchlog()) {
- oprot.writeFieldBegin(ACKNOWLEDGED_BY_BATCHLOG_FIELD_DESC);
- oprot.writeBool(struct.acknowledged_by_batchlog);
- oprot.writeFieldEnd();
- }
- if (struct.isSetPaxos_in_progress()) {
- oprot.writeFieldBegin(PAXOS_IN_PROGRESS_FIELD_DESC);
- oprot.writeBool(struct.paxos_in_progress);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class TimedOutExceptionTupleSchemeFactory implements SchemeFactory {
- public TimedOutExceptionTupleScheme getScheme() {
- return new TimedOutExceptionTupleScheme();
- }
- }
-
- private static class TimedOutExceptionTupleScheme extends TupleScheme<TimedOutException> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, TimedOutException struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- BitSet optionals = new BitSet();
- if (struct.isSetAcknowledged_by()) {
- optionals.set(0);
- }
- if (struct.isSetAcknowledged_by_batchlog()) {
- optionals.set(1);
- }
- if (struct.isSetPaxos_in_progress()) {
- optionals.set(2);
- }
- oprot.writeBitSet(optionals, 3);
- if (struct.isSetAcknowledged_by()) {
- oprot.writeI32(struct.acknowledged_by);
- }
- if (struct.isSetAcknowledged_by_batchlog()) {
- oprot.writeBool(struct.acknowledged_by_batchlog);
- }
- if (struct.isSetPaxos_in_progress()) {
- oprot.writeBool(struct.paxos_in_progress);
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, TimedOutException struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- BitSet incoming = iprot.readBitSet(3);
- if (incoming.get(0)) {
- struct.acknowledged_by = iprot.readI32();
- struct.setAcknowledged_byIsSet(true);
- }
- if (incoming.get(1)) {
- struct.acknowledged_by_batchlog = iprot.readBool();
- struct.setAcknowledged_by_batchlogIsSet(true);
- }
- if (incoming.get(2)) {
- struct.paxos_in_progress = iprot.readBool();
- struct.setPaxos_in_progressIsSet(true);
- }
- }
- }
-
-}
-
diff --git a/interface/thrift/gen-java/org/apache/cassandra/thrift/TokenRange.java b/interface/thrift/gen-java/org/apache/cassandra/thrift/TokenRange.java
deleted file mode 100644
index 37d0f77..0000000
--- a/interface/thrift/gen-java/org/apache/cassandra/thrift/TokenRange.java
+++ /dev/null
@@ -1,990 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.1)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- * @generated
- */
-package org.apache.cassandra.thrift;
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
-
-import org.apache.commons.lang3.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * A TokenRange describes part of the Cassandra ring, it is a mapping from a range to
- * endpoints responsible for that range.
- * @param start_token The first token in the range
- * @param end_token The last token in the range
- * @param endpoints The endpoints responsible for the range (listed by their configured listen_address)
- * @param rpc_endpoints The endpoints responsible for the range (listed by their configured rpc_address)
- */
-public class TokenRange implements org.apache.thrift.TBase<TokenRange, TokenRange._Fields>, java.io.Serializable, Cloneable, Comparable<TokenRange> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TokenRange");
-
- private static final org.apache.thrift.protocol.TField START_TOKEN_FIELD_DESC = new org.apache.thrift.protocol.TField("start_token", org.apache.thrift.protocol.TType.STRING, (short)1);
- private static final org.apache.thrift.protocol.TField END_TOKEN_FIELD_DESC = new org.apache.thrift.protocol.TField("end_token", org.apache.thrift.protocol.TType.STRING, (short)2);
- private static final org.apache.thrift.protocol.TField ENDPOINTS_FIELD_DESC = new org.apache.thrift.protocol.TField("endpoints", org.apache.thrift.protocol.TType.LIST, (short)3);
- private static final org.apache.thrift.protocol.TField RPC_ENDPOINTS_FIELD_DESC = new org.apache.thrift.protocol.TField("rpc_endpoints", org.apache.thrift.protocol.TType.LIST, (short)4);
- private static final org.apache.thrift.protocol.TField ENDPOINT_DETAILS_FIELD_DESC = new org.apache.thrift.protocol.TField("endpoint_details", org.apache.thrift.protocol.TType.LIST, (short)5);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new TokenRangeStandardSchemeFactory());
- schemes.put(TupleScheme.class, new TokenRangeTupleSchemeFactory());
- }
-
- public String start_token; // required
- public String end_token; // required
- public List<String> endpoints; // required
- public List<String> rpc_endpoints; // optional
- public List<EndpointDetails> endpoint_details; // optional
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- START_TOKEN((short)1, "start_token"),
- END_TOKEN((short)2, "end_token"),
- ENDPOINTS((short)3, "endpoints"),
- RPC_ENDPOINTS((short)4, "rpc_endpoints"),
- ENDPOINT_DETAILS((short)5, "endpoint_details");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // START_TOKEN
- return START_TOKEN;
- case 2: // END_TOKEN
- return END_TOKEN;
- case 3: // ENDPOINTS
- return ENDPOINTS;
- case 4: // RPC_ENDPOINTS
- return RPC_ENDPOINTS;
- case 5: // ENDPOINT_DETAILS
- return ENDPOINT_DETAILS;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- private _Fields optionals[] = {_Fields.RPC_ENDPOINTS,_Fields.ENDPOINT_DETAILS};
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.START_TOKEN, new org.apache.thrift.meta_data.FieldMetaData("start_token", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
- tmpMap.put(_Fields.END_TOKEN, new org.apache.thrift.meta_data.FieldMetaData("end_token", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
- tmpMap.put(_Fields.ENDPOINTS, new org.apache.thrift.meta_data.FieldMetaData("endpoints", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))));
- tmpMap.put(_Fields.RPC_ENDPOINTS, new org.apache.thrift.meta_data.FieldMetaData("rpc_endpoints", org.apache.thrift.TFieldRequirementType.OPTIONAL,
- new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))));
- tmpMap.put(_Fields.ENDPOINT_DETAILS, new org.apache.thrift.meta_data.FieldMetaData("endpoint_details", org.apache.thrift.TFieldRequirementType.OPTIONAL,
- new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST,
- new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, EndpointDetails.class))));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TokenRange.class, metaDataMap);
- }
-
- public TokenRange() {
- }
-
- public TokenRange(
- String start_token,
- String end_token,
- List<String> endpoints)
- {
- this();
- this.start_token = start_token;
- this.end_token = end_token;
- this.endpoints = endpoints;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public TokenRange(TokenRange other) {
- if (other.isSetStart_token()) {
- this.start_token = other.start_token;
- }
- if (other.isSetEnd_token()) {
- this.end_token = other.end_token;
- }
- if (other.isSetEndpoints()) {
- List<String> __this__endpoints = new ArrayList<String>(other.endpoints);
- this.endpoints = __this__endpoints;
- }
- if (other.isSetRpc_endpoints()) {
- List<String> __this__rpc_endpoints = new ArrayList<String>(other.rpc_endpoints);
- this.rpc_endpoints = __this__rpc_endpoints;
- }
- if (other.isSetEndpoint_details()) {
- List<EndpointDetails> __this__endpoint_details = new ArrayList<EndpointDetails>(other.endpoint_details.size());
- for (EndpointDetails other_element : other.endpoint_details) {
- __this__endpoint_details.add(new EndpointDetails(other_element));
- }
- this.endpoint_details = __this__endpoint_details;
- }
- }
-
- public TokenRange deepCopy() {
- return new TokenRange(this);
- }
-
- @Override
- public void clear() {
- this.start_token = null;
- this.end_token = null;
- this.endpoints = null;
- this.rpc_endpoints = null;
- this.endpoint_details = null;
- }
-
- public String getStart_token() {
- return this.start_token;
- }
-
- public TokenRange setStart_token(String start_token) {
- this.start_token = start_token;
- return this;
- }
-
- public void unsetStart_token() {
- this.start_token = null;
- }
-
- /** Returns true if field start_token is set (has been assigned a value) and false otherwise */
- public boolean isSetStart_token() {
- return this.start_token != null;
- }
-
- public void setStart_tokenIsSet(boolean value) {
- if (!value) {
- this.start_token = null;
- }
- }
-
- public String getEnd_token() {
- return this.end_token;
- }
-
- public TokenRange setEnd_token(String end_token) {
- this.end_token = end_token;
- return this;
- }
-
- public void unsetEnd_token() {
- this.end_token = null;
- }
-
- /** Returns true if field end_token is set (has been assigned a value) and false otherwise */
- public boolean isSetEnd_token() {
- return this.end_token != null;
- }
-
- public void setEnd_tokenIsSet(boolean value) {
- if (!value) {
- this.end_token = null;
- }
- }
-
- public int getEndpointsSize() {
- return (this.endpoints == null) ? 0 : this.endpoints.size();
- }
-
- public java.util.Iterator<String> getEndpointsIterator() {
- return (this.endpoints == null) ? null : this.endpoints.iterator();
- }
-
- public void addToEndpoints(String elem) {
- if (this.endpoints == null) {
- this.endpoints = new ArrayList<String>();
- }
- this.endpoints.add(elem);
- }
-
- public List<String> getEndpoints() {
- return this.endpoints;
- }
-
- public TokenRange setEndpoints(List<String> endpoints) {
- this.endpoints = endpoints;
- return this;
- }
-
- public void unsetEndpoints() {
- this.endpoints = null;
- }
-
- /** Returns true if field endpoints is set (has been assigned a value) and false otherwise */
- public boolean isSetEndpoints() {
- return this.endpoints != null;
- }
-
- public void setEndpointsIsSet(boolean value) {
- if (!value) {
- this.endpoints = null;
- }
- }
-
- public int getRpc_endpointsSize() {
- return (this.rpc_endpoints == null) ? 0 : this.rpc_endpoints.size();
- }
-
- public java.util.Iterator<String> getRpc_endpointsIterator() {
- return (this.rpc_endpoints == null) ? null : this.rpc_endpoints.iterator();
- }
-
- public void addToRpc_endpoints(String elem) {
- if (this.rpc_endpoints == null) {
- this.rpc_endpoints = new ArrayList<String>();
- }
- this.rpc_endpoints.add(elem);
- }
-
- public List<String> getRpc_endpoints() {
- return this.rpc_endpoints;
- }
-
- public TokenRange setRpc_endpoints(List<String> rpc_endpoints) {
- this.rpc_endpoints = rpc_endpoints;
- return this;
- }
-
- public void unsetRpc_endpoints() {
- this.rpc_endpoints = null;
- }
-
- /** Returns true if field rpc_endpoints is set (has been assigned a value) and false otherwise */
- public boolean isSetRpc_endpoints() {
- return this.rpc_endpoints != null;
- }
-
- public void setRpc_endpointsIsSet(boolean value) {
- if (!value) {
- this.rpc_endpoints = null;
- }
- }
-
- public int getEndpoint_detailsSize() {
- return (this.endpoint_details == null) ? 0 : this.endpoint_details.size();
- }
-
- public java.util.Iterator<EndpointDetails> getEndpoint_detailsIterator() {
- return (this.endpoint_details == null) ? null : this.endpoint_details.iterator();
- }
-
- public void addToEndpoint_details(EndpointDetails elem) {
- if (this.endpoint_details == null) {
- this.endpoint_details = new ArrayList<EndpointDetails>();
- }
- this.endpoint_details.add(elem);
- }
-
- public List<EndpointDetails> getEndpoint_details() {
- return this.endpoint_details;
- }
-
- public TokenRange setEndpoint_details(List<EndpointDetails> endpoint_details) {
- this.endpoint_details = endpoint_details;
- return this;
- }
-
- public void unsetEndpoint_details() {
- this.endpoint_details = null;
- }
-
- /** Returns true if field endpoint_details is set (has been assigned a value) and false otherwise */
- public boolean isSetEndpoint_details() {
- return this.endpoint_details != null;
- }
-
- public void setEndpoint_detailsIsSet(boolean value) {
- if (!value) {
- this.endpoint_details = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case START_TOKEN:
- if (value == null) {
- unsetStart_token();
- } else {
- setStart_token((String)value);
- }
- break;
-
- case END_TOKEN:
- if (value == null) {
- unsetEnd_token();
- } else {
- setEnd_token((String)value);
- }
- break;
-
- case ENDPOINTS:
- if (value == null) {
- unsetEndpoints();
- } else {
- setEndpoints((List<String>)value);
- }
- break;
-
- case RPC_ENDPOINTS:
- if (value == null) {
- unsetRpc_endpoints();
- } else {
- setRpc_endpoints((List<String>)value);
- }
- break;
-
- case ENDPOINT_DETAILS:
- if (value == null) {
- unsetEndpoint_details();
- } else {
- setEndpoint_details((List<EndpointDetails>)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case START_TOKEN:
- return getStart_token();
-
- case END_TOKEN:
- return getEnd_token();
-
- case ENDPOINTS:
- return getEndpoints();
-
- case RPC_ENDPOINTS:
- return getRpc_endpoints();
-
- case ENDPOINT_DETAILS:
- return getEndpoint_details();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case START_TOKEN:
- return isSetStart_token();
- case END_TOKEN:
- return isSetEnd_token();
- case ENDPOINTS:
- return isSetEndpoints();
- case RPC_ENDPOINTS:
- return isSetRpc_endpoints();
- case ENDPOINT_DETAILS:
- return isSetEndpoint_details();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof TokenRange)
- return this.equals((TokenRange)that);
- return false;
- }
-
- public boolean equals(TokenRange that) {
- if (that == null)
- return false;
-
- boolean this_present_start_token = true && this.isSetStart_token();
- boolean that_present_start_token = true && that.isSetStart_token();
- if (this_present_start_token || that_present_start_token) {
- if (!(this_present_start_token && that_present_start_token))
- return false;
- if (!this.start_token.equals(that.start_token))
- return false;
- }
-
- boolean this_present_end_token = true && this.isSetEnd_token();
- boolean that_present_end_token = true && that.isSetEnd_token();
- if (this_present_end_token || that_present_end_token) {
- if (!(this_present_end_token && that_present_end_token))
- return false;
- if (!this.end_token.equals(that.end_token))
- return false;
- }
-
- boolean this_present_endpoints = true && this.isSetEndpoints();
- boolean that_present_endpoints = true && that.isSetEndpoints();
- if (this_present_endpoints || that_present_endpoints) {
- if (!(this_present_endpoints && that_present_endpoints))
- return false;
- if (!this.endpoints.equals(that.endpoints))
- return false;
- }
-
- boolean this_present_rpc_endpoints = true && this.isSetRpc_endpoints();
- boolean that_present_rpc_endpoints = true && that.isSetRpc_endpoints();
- if (this_present_rpc_endpoints || that_present_rpc_endpoints) {
- if (!(this_present_rpc_endpoints && that_present_rpc_endpoints))
- return false;
- if (!this.rpc_endpoints.equals(that.rpc_endpoints))
- return false;
- }
-
- boolean this_present_endpoint_details = true && this.isSetEndpoint_details();
- boolean that_present_endpoint_details = true && that.isSetEndpoint_details();
- if (this_present_endpoint_details || that_present_endpoint_details) {
- if (!(this_present_endpoint_details && that_present_endpoint_details))
- return false;
- if (!this.endpoint_details.equals(that.endpoint_details))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_start_token = true && (isSetStart_token());
- builder.append(present_start_token);
- if (present_start_token)
- builder.append(start_token);
-
- boolean present_end_token = true && (isSetEnd_token());
- builder.append(present_end_token);
- if (present_end_token)
- builder.append(end_token);
-
- boolean present_endpoints = true && (isSetEndpoints());
- builder.append(present_endpoints);
- if (present_endpoints)
- builder.append(endpoints);
-
- boolean present_rpc_endpoints = true && (isSetRpc_endpoints());
- builder.append(present_rpc_endpoints);
- if (present_rpc_endpoints)
- builder.append(rpc_endpoints);
-
- boolean present_endpoint_details = true && (isSetEndpoint_details());
- builder.append(present_endpoint_details);
- if (present_endpoint_details)
- builder.append(endpoint_details);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(TokenRange other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetStart_token()).compareTo(other.isSetStart_token());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetStart_token()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.start_token, other.start_token);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetEnd_token()).compareTo(other.isSetEnd_token());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetEnd_token()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.end_token, other.end_token);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetEndpoints()).compareTo(other.isSetEndpoints());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetEndpoints()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.endpoints, other.endpoints);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetRpc_endpoints()).compareTo(other.isSetRpc_endpoints());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetRpc_endpoints()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.rpc_endpoints, other.rpc_endpoints);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetEndpoint_details()).compareTo(other.isSetEndpoint_details());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetEndpoint_details()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.endpoint_details, other.endpoint_details);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("TokenRange(");
- boolean first = true;
-
- sb.append("start_token:");
- if (this.start_token == null) {
- sb.append("null");
- } else {
- sb.append(this.start_token);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("end_token:");
- if (this.end_token == null) {
- sb.append("null");
- } else {
- sb.append(this.end_token);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("endpoints:");
- if (this.endpoints == null) {
- sb.append("null");
- } else {
- sb.append(this.endpoints);
- }
- first = false;
- if (isSetRpc_endpoints()) {
- if (!first) sb.append(", ");
- sb.append("rpc_endpoints:");
- if (this.rpc_endpoints == null) {
- sb.append("null");
- } else {
- sb.append(this.rpc_endpoints);
- }
- first = false;
- }
- if (isSetEndpoint_details()) {
- if (!first) sb.append(", ");
- sb.append("endpoint_details:");
- if (this.endpoint_details == null) {
- sb.append("null");
- } else {
- sb.append(this.endpoint_details);
- }
- first = false;
- }
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- if (start_token == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'start_token' was not present! Struct: " + toString());
- }
- if (end_token == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'end_token' was not present! Struct: " + toString());
- }
- if (endpoints == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'endpoints' was not present! Struct: " + toString());
- }
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class TokenRangeStandardSchemeFactory implements SchemeFactory {
- public TokenRangeStandardScheme getScheme() {
- return new TokenRangeStandardScheme();
- }
- }
-
- private static class TokenRangeStandardScheme extends StandardScheme<TokenRange> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, TokenRange struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // START_TOKEN
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.start_token = iprot.readString();
- struct.setStart_tokenIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 2: // END_TOKEN
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.end_token = iprot.readString();
- struct.setEnd_tokenIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 3: // ENDPOINTS
- if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
- {
- org.apache.thrift.protocol.TList _list56 = iprot.readListBegin();
- struct.endpoints = new ArrayList<String>(_list56.size);
- for (int _i57 = 0; _i57 < _list56.size; ++_i57)
- {
- String _elem58;
- _elem58 = iprot.readString();
- struct.endpoints.add(_elem58);
- }
- iprot.readListEnd();
- }
- struct.setEndpointsIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 4: // RPC_ENDPOINTS
- if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
- {
- org.apache.thrift.protocol.TList _list59 = iprot.readListBegin();
- struct.rpc_endpoints = new ArrayList<String>(_list59.size);
- for (int _i60 = 0; _i60 < _list59.size; ++_i60)
- {
- String _elem61;
- _elem61 = iprot.readString();
- struct.rpc_endpoints.add(_elem61);
- }
- iprot.readListEnd();
- }
- struct.setRpc_endpointsIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 5: // ENDPOINT_DETAILS
- if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
- {
- org.apache.thrift.protocol.TList _list62 = iprot.readListBegin();
- struct.endpoint_details = new ArrayList<EndpointDetails>(_list62.size);
- for (int _i63 = 0; _i63 < _list62.size; ++_i63)
- {
- EndpointDetails _elem64;
- _elem64 = new EndpointDetails();
- _elem64.read(iprot);
- struct.endpoint_details.add(_elem64);
- }
- iprot.readListEnd();
- }
- struct.setEndpoint_detailsIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, TokenRange struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.start_token != null) {
- oprot.writeFieldBegin(START_TOKEN_FIELD_DESC);
- oprot.writeString(struct.start_token);
- oprot.writeFieldEnd();
- }
- if (struct.end_token != null) {
- oprot.writeFieldBegin(END_TOKEN_FIELD_DESC);
- oprot.writeString(struct.end_token);
- oprot.writeFieldEnd();
- }
- if (struct.endpoints != null) {
- oprot.writeFieldBegin(ENDPOINTS_FIELD_DESC);
- {
- oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, struct.endpoints.size()));
- for (String _iter65 : struct.endpoints)
- {
- oprot.writeString(_iter65);
- }
- oprot.writeListEnd();
- }
- oprot.writeFieldEnd();
- }
- if (struct.rpc_endpoints != null) {
- if (struct.isSetRpc_endpoints()) {
- oprot.writeFieldBegin(RPC_ENDPOINTS_FIELD_DESC);
- {
- oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, struct.rpc_endpoints.size()));
- for (String _iter66 : struct.rpc_endpoints)
- {
- oprot.writeString(_iter66);
- }
- oprot.writeListEnd();
- }
- oprot.writeFieldEnd();
- }
- }
- if (struct.endpoint_details != null) {
- if (struct.isSetEndpoint_details()) {
- oprot.writeFieldBegin(ENDPOINT_DETAILS_FIELD_DESC);
- {
- oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.endpoint_details.size()));
- for (EndpointDetails _iter67 : struct.endpoint_details)
- {
- _iter67.write(oprot);
- }
- oprot.writeListEnd();
- }
- oprot.writeFieldEnd();
- }
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class TokenRangeTupleSchemeFactory implements SchemeFactory {
- public TokenRangeTupleScheme getScheme() {
- return new TokenRangeTupleScheme();
- }
- }
-
- private static class TokenRangeTupleScheme extends TupleScheme<TokenRange> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, TokenRange struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- oprot.writeString(struct.start_token);
- oprot.writeString(struct.end_token);
- {
- oprot.writeI32(struct.endpoints.size());
- for (String _iter68 : struct.endpoints)
- {
- oprot.writeString(_iter68);
- }
- }
- BitSet optionals = new BitSet();
- if (struct.isSetRpc_endpoints()) {
- optionals.set(0);
- }
- if (struct.isSetEndpoint_details()) {
- optionals.set(1);
- }
- oprot.writeBitSet(optionals, 2);
- if (struct.isSetRpc_endpoints()) {
- {
- oprot.writeI32(struct.rpc_endpoints.size());
- for (String _iter69 : struct.rpc_endpoints)
- {
- oprot.writeString(_iter69);
- }
- }
- }
- if (struct.isSetEndpoint_details()) {
- {
- oprot.writeI32(struct.endpoint_details.size());
- for (EndpointDetails _iter70 : struct.endpoint_details)
- {
- _iter70.write(oprot);
- }
- }
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, TokenRange struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- struct.start_token = iprot.readString();
- struct.setStart_tokenIsSet(true);
- struct.end_token = iprot.readString();
- struct.setEnd_tokenIsSet(true);
- {
- org.apache.thrift.protocol.TList _list71 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32());
- struct.endpoints = new ArrayList<String>(_list71.size);
- for (int _i72 = 0; _i72 < _list71.size; ++_i72)
- {
- String _elem73;
- _elem73 = iprot.readString();
- struct.endpoints.add(_elem73);
- }
- }
- struct.setEndpointsIsSet(true);
- BitSet incoming = iprot.readBitSet(2);
- if (incoming.get(0)) {
- {
- org.apache.thrift.protocol.TList _list74 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32());
- struct.rpc_endpoints = new ArrayList<String>(_list74.size);
- for (int _i75 = 0; _i75 < _list74.size; ++_i75)
- {
- String _elem76;
- _elem76 = iprot.readString();
- struct.rpc_endpoints.add(_elem76);
- }
- }
- struct.setRpc_endpointsIsSet(true);
- }
- if (incoming.get(1)) {
- {
- org.apache.thrift.protocol.TList _list77 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
- struct.endpoint_details = new ArrayList<EndpointDetails>(_list77.size);
- for (int _i78 = 0; _i78 < _list77.size; ++_i78)
- {
- EndpointDetails _elem79;
- _elem79 = new EndpointDetails();
- _elem79.read(iprot);
- struct.endpoint_details.add(_elem79);
- }
- }
- struct.setEndpoint_detailsIsSet(true);
- }
- }
- }
-
-}
-
diff --git a/interface/thrift/gen-java/org/apache/cassandra/thrift/TriggerDef.java b/interface/thrift/gen-java/org/apache/cassandra/thrift/TriggerDef.java
deleted file mode 100644
index 32b0ac5..0000000
--- a/interface/thrift/gen-java/org/apache/cassandra/thrift/TriggerDef.java
+++ /dev/null
@@ -1,568 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.1)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- * @generated
- */
-package org.apache.cassandra.thrift;
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
-
-import org.apache.commons.lang3.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Describes a trigger.
- * `options` should include at least 'class' param.
- * Other options are not supported yet.
- */
-public class TriggerDef implements org.apache.thrift.TBase<TriggerDef, TriggerDef._Fields>, java.io.Serializable, Cloneable, Comparable<TriggerDef> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TriggerDef");
-
- private static final org.apache.thrift.protocol.TField NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("name", org.apache.thrift.protocol.TType.STRING, (short)1);
- private static final org.apache.thrift.protocol.TField OPTIONS_FIELD_DESC = new org.apache.thrift.protocol.TField("options", org.apache.thrift.protocol.TType.MAP, (short)2);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new TriggerDefStandardSchemeFactory());
- schemes.put(TupleScheme.class, new TriggerDefTupleSchemeFactory());
- }
-
- public String name; // required
- public Map<String,String> options; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- NAME((short)1, "name"),
- OPTIONS((short)2, "options");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // NAME
- return NAME;
- case 2: // OPTIONS
- return OPTIONS;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.NAME, new org.apache.thrift.meta_data.FieldMetaData("name", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
- tmpMap.put(_Fields.OPTIONS, new org.apache.thrift.meta_data.FieldMetaData("options", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING),
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TriggerDef.class, metaDataMap);
- }
-
- public TriggerDef() {
- }
-
- public TriggerDef(
- String name,
- Map<String,String> options)
- {
- this();
- this.name = name;
- this.options = options;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public TriggerDef(TriggerDef other) {
- if (other.isSetName()) {
- this.name = other.name;
- }
- if (other.isSetOptions()) {
- Map<String,String> __this__options = new HashMap<String,String>(other.options);
- this.options = __this__options;
- }
- }
-
- public TriggerDef deepCopy() {
- return new TriggerDef(this);
- }
-
- @Override
- public void clear() {
- this.name = null;
- this.options = null;
- }
-
- public String getName() {
- return this.name;
- }
-
- public TriggerDef setName(String name) {
- this.name = name;
- return this;
- }
-
- public void unsetName() {
- this.name = null;
- }
-
- /** Returns true if field name is set (has been assigned a value) and false otherwise */
- public boolean isSetName() {
- return this.name != null;
- }
-
- public void setNameIsSet(boolean value) {
- if (!value) {
- this.name = null;
- }
- }
-
- public int getOptionsSize() {
- return (this.options == null) ? 0 : this.options.size();
- }
-
- public void putToOptions(String key, String val) {
- if (this.options == null) {
- this.options = new HashMap<String,String>();
- }
- this.options.put(key, val);
- }
-
- public Map<String,String> getOptions() {
- return this.options;
- }
-
- public TriggerDef setOptions(Map<String,String> options) {
- this.options = options;
- return this;
- }
-
- public void unsetOptions() {
- this.options = null;
- }
-
- /** Returns true if field options is set (has been assigned a value) and false otherwise */
- public boolean isSetOptions() {
- return this.options != null;
- }
-
- public void setOptionsIsSet(boolean value) {
- if (!value) {
- this.options = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case NAME:
- if (value == null) {
- unsetName();
- } else {
- setName((String)value);
- }
- break;
-
- case OPTIONS:
- if (value == null) {
- unsetOptions();
- } else {
- setOptions((Map<String,String>)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case NAME:
- return getName();
-
- case OPTIONS:
- return getOptions();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case NAME:
- return isSetName();
- case OPTIONS:
- return isSetOptions();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof TriggerDef)
- return this.equals((TriggerDef)that);
- return false;
- }
-
- public boolean equals(TriggerDef that) {
- if (that == null)
- return false;
-
- boolean this_present_name = true && this.isSetName();
- boolean that_present_name = true && that.isSetName();
- if (this_present_name || that_present_name) {
- if (!(this_present_name && that_present_name))
- return false;
- if (!this.name.equals(that.name))
- return false;
- }
-
- boolean this_present_options = true && this.isSetOptions();
- boolean that_present_options = true && that.isSetOptions();
- if (this_present_options || that_present_options) {
- if (!(this_present_options && that_present_options))
- return false;
- if (!this.options.equals(that.options))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- boolean present_name = true && (isSetName());
- builder.append(present_name);
- if (present_name)
- builder.append(name);
-
- boolean present_options = true && (isSetOptions());
- builder.append(present_options);
- if (present_options)
- builder.append(options);
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(TriggerDef other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetName()).compareTo(other.isSetName());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetName()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.name, other.name);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetOptions()).compareTo(other.isSetOptions());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetOptions()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.options, other.options);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("TriggerDef(");
- boolean first = true;
-
- sb.append("name:");
- if (this.name == null) {
- sb.append("null");
- } else {
- sb.append(this.name);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("options:");
- if (this.options == null) {
- sb.append("null");
- } else {
- sb.append(this.options);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- if (name == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'name' was not present! Struct: " + toString());
- }
- if (options == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'options' was not present! Struct: " + toString());
- }
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class TriggerDefStandardSchemeFactory implements SchemeFactory {
- public TriggerDefStandardScheme getScheme() {
- return new TriggerDefStandardScheme();
- }
- }
-
- private static class TriggerDefStandardScheme extends StandardScheme<TriggerDef> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, TriggerDef struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // NAME
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.name = iprot.readString();
- struct.setNameIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 2: // OPTIONS
- if (schemeField.type == org.apache.thrift.protocol.TType.MAP) {
- {
- org.apache.thrift.protocol.TMap _map100 = iprot.readMapBegin();
- struct.options = new HashMap<String,String>(2*_map100.size);
- for (int _i101 = 0; _i101 < _map100.size; ++_i101)
- {
- String _key102;
- String _val103;
- _key102 = iprot.readString();
- _val103 = iprot.readString();
- struct.options.put(_key102, _val103);
- }
- iprot.readMapEnd();
- }
- struct.setOptionsIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, TriggerDef struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.name != null) {
- oprot.writeFieldBegin(NAME_FIELD_DESC);
- oprot.writeString(struct.name);
- oprot.writeFieldEnd();
- }
- if (struct.options != null) {
- oprot.writeFieldBegin(OPTIONS_FIELD_DESC);
- {
- oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, struct.options.size()));
- for (Map.Entry<String, String> _iter104 : struct.options.entrySet())
- {
- oprot.writeString(_iter104.getKey());
- oprot.writeString(_iter104.getValue());
- }
- oprot.writeMapEnd();
- }
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class TriggerDefTupleSchemeFactory implements SchemeFactory {
- public TriggerDefTupleScheme getScheme() {
- return new TriggerDefTupleScheme();
- }
- }
-
- private static class TriggerDefTupleScheme extends TupleScheme<TriggerDef> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, TriggerDef struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- oprot.writeString(struct.name);
- {
- oprot.writeI32(struct.options.size());
- for (Map.Entry<String, String> _iter105 : struct.options.entrySet())
- {
- oprot.writeString(_iter105.getKey());
- oprot.writeString(_iter105.getValue());
- }
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, TriggerDef struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- struct.name = iprot.readString();
- struct.setNameIsSet(true);
- {
- org.apache.thrift.protocol.TMap _map106 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32());
- struct.options = new HashMap<String,String>(2*_map106.size);
- for (int _i107 = 0; _i107 < _map106.size; ++_i107)
- {
- String _key108;
- String _val109;
- _key108 = iprot.readString();
- _val109 = iprot.readString();
- struct.options.put(_key108, _val109);
- }
- }
- struct.setOptionsIsSet(true);
- }
- }
-
-}
-
diff --git a/interface/thrift/gen-java/org/apache/cassandra/thrift/UnavailableException.java b/interface/thrift/gen-java/org/apache/cassandra/thrift/UnavailableException.java
deleted file mode 100644
index 23bfeed..0000000
--- a/interface/thrift/gen-java/org/apache/cassandra/thrift/UnavailableException.java
+++ /dev/null
@@ -1,307 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.1)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- * @generated
- */
-package org.apache.cassandra.thrift;
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
-
-import org.apache.commons.lang3.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Not all the replicas required could be created and/or read.
- */
-public class UnavailableException extends TException implements org.apache.thrift.TBase<UnavailableException, UnavailableException._Fields>, java.io.Serializable, Cloneable, Comparable<UnavailableException> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("UnavailableException");
-
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new UnavailableExceptionStandardSchemeFactory());
- schemes.put(TupleScheme.class, new UnavailableExceptionTupleSchemeFactory());
- }
-
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-;
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(UnavailableException.class, metaDataMap);
- }
-
- public UnavailableException() {
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public UnavailableException(UnavailableException other) {
- }
-
- public UnavailableException deepCopy() {
- return new UnavailableException(this);
- }
-
- @Override
- public void clear() {
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof UnavailableException)
- return this.equals((UnavailableException)that);
- return false;
- }
-
- public boolean equals(UnavailableException that) {
- if (that == null)
- return false;
-
- return true;
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
-
- return builder.toHashCode();
- }
-
- @Override
- public int compareTo(UnavailableException other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("UnavailableException(");
- boolean first = true;
-
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class UnavailableExceptionStandardSchemeFactory implements SchemeFactory {
- public UnavailableExceptionStandardScheme getScheme() {
- return new UnavailableExceptionStandardScheme();
- }
- }
-
- private static class UnavailableExceptionStandardScheme extends StandardScheme<UnavailableException> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, UnavailableException struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, UnavailableException struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class UnavailableExceptionTupleSchemeFactory implements SchemeFactory {
- public UnavailableExceptionTupleScheme getScheme() {
- return new UnavailableExceptionTupleScheme();
- }
- }
-
- private static class UnavailableExceptionTupleScheme extends TupleScheme<UnavailableException> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, UnavailableException struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, UnavailableException struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- }
- }
-
-}
-
diff --git a/interface/thrift/gen-java/org/apache/cassandra/thrift/cassandraConstants.java b/interface/thrift/gen-java/org/apache/cassandra/thrift/cassandraConstants.java
deleted file mode 100644
index f84243e..0000000
--- a/interface/thrift/gen-java/org/apache/cassandra/thrift/cassandraConstants.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.1)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- * @generated
- */
-package org.apache.cassandra.thrift;
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
-
-import org.apache.commons.lang3.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class cassandraConstants {
-
- public static final String VERSION = "20.1.0";
-
-}
diff --git a/lib/airline-0.6.jar b/lib/airline-0.6.jar
deleted file mode 100644
index a35ae79..0000000
--- a/lib/airline-0.6.jar
+++ /dev/null
Binary files differ
diff --git a/lib/airline-0.8.jar b/lib/airline-0.8.jar
new file mode 100644
index 0000000..a34c945
--- /dev/null
+++ b/lib/airline-0.8.jar
Binary files differ
diff --git a/lib/asm-5.0.4.jar b/lib/asm-5.0.4.jar
deleted file mode 100644
index cdb283d..0000000
--- a/lib/asm-5.0.4.jar
+++ /dev/null
Binary files differ
diff --git a/lib/asm-7.1.jar b/lib/asm-7.1.jar
new file mode 100644
index 0000000..355eb08
--- /dev/null
+++ b/lib/asm-7.1.jar
Binary files differ
diff --git a/lib/caffeine-2.2.6.jar b/lib/caffeine-2.2.6.jar
deleted file mode 100644
index 74b91bc..0000000
--- a/lib/caffeine-2.2.6.jar
+++ /dev/null
Binary files differ
diff --git a/lib/caffeine-2.3.5.jar b/lib/caffeine-2.3.5.jar
new file mode 100644
index 0000000..70cf178
--- /dev/null
+++ b/lib/caffeine-2.3.5.jar
Binary files differ
diff --git a/lib/cassandra-driver-core-3.0.1-shaded.jar b/lib/cassandra-driver-core-3.0.1-shaded.jar
deleted file mode 100644
index bc269a0..0000000
--- a/lib/cassandra-driver-core-3.0.1-shaded.jar
+++ /dev/null
Binary files differ
diff --git a/lib/cassandra-driver-core-3.9.0-shaded.jar b/lib/cassandra-driver-core-3.9.0-shaded.jar
new file mode 100644
index 0000000..1ecabbb
--- /dev/null
+++ b/lib/cassandra-driver-core-3.9.0-shaded.jar
Binary files differ
diff --git a/lib/cassandra-driver-internal-only-3.10.zip b/lib/cassandra-driver-internal-only-3.10.zip
deleted file mode 100644
index 22b877c..0000000
--- a/lib/cassandra-driver-internal-only-3.10.zip
+++ /dev/null
Binary files differ
diff --git a/lib/cassandra-driver-internal-only-3.11.0-bb96859b.zip b/lib/cassandra-driver-internal-only-3.11.0-bb96859b.zip
deleted file mode 100644
index d31abc3..0000000
--- a/lib/cassandra-driver-internal-only-3.11.0-bb96859b.zip
+++ /dev/null
Binary files differ
diff --git a/lib/cassandra-driver-internal-only-3.23.0.post0-1a184b99.zip b/lib/cassandra-driver-internal-only-3.23.0.post0-1a184b99.zip
new file mode 100644
index 0000000..9b25bc1
--- /dev/null
+++ b/lib/cassandra-driver-internal-only-3.23.0.post0-1a184b99.zip
Binary files differ
diff --git a/lib/chronicle-bytes-1.16.3.jar b/lib/chronicle-bytes-1.16.3.jar
new file mode 100644
index 0000000..1b7c3a0
--- /dev/null
+++ b/lib/chronicle-bytes-1.16.3.jar
Binary files differ
diff --git a/lib/chronicle-core-1.16.4.jar b/lib/chronicle-core-1.16.4.jar
new file mode 100644
index 0000000..0275a72
--- /dev/null
+++ b/lib/chronicle-core-1.16.4.jar
Binary files differ
diff --git a/lib/chronicle-queue-4.16.3.jar b/lib/chronicle-queue-4.16.3.jar
new file mode 100644
index 0000000..1def472
--- /dev/null
+++ b/lib/chronicle-queue-4.16.3.jar
Binary files differ
diff --git a/lib/chronicle-threads-1.16.0.jar b/lib/chronicle-threads-1.16.0.jar
new file mode 100644
index 0000000..b1f1228
--- /dev/null
+++ b/lib/chronicle-threads-1.16.0.jar
Binary files differ
diff --git a/lib/chronicle-wire-1.16.1.jar b/lib/chronicle-wire-1.16.1.jar
new file mode 100644
index 0000000..720d242
--- /dev/null
+++ b/lib/chronicle-wire-1.16.1.jar
Binary files differ
diff --git a/lib/compress-lzf-0.8.4.jar b/lib/compress-lzf-0.8.4.jar
deleted file mode 100644
index a712c24..0000000
--- a/lib/compress-lzf-0.8.4.jar
+++ /dev/null
Binary files differ
diff --git a/lib/concurrentlinkedhashmap-lru-1.4.jar b/lib/concurrentlinkedhashmap-lru-1.4.jar
deleted file mode 100644
index 572b258..0000000
--- a/lib/concurrentlinkedhashmap-lru-1.4.jar
+++ /dev/null
Binary files differ
diff --git a/lib/disruptor-3.0.1.jar b/lib/disruptor-3.0.1.jar
deleted file mode 100644
index 1899ed0..0000000
--- a/lib/disruptor-3.0.1.jar
+++ /dev/null
Binary files differ
diff --git a/lib/ecj-4.4.2.jar b/lib/ecj-4.4.2.jar
deleted file mode 100644
index d9411b3..0000000
--- a/lib/ecj-4.4.2.jar
+++ /dev/null
Binary files differ
diff --git a/lib/ecj-4.6.1.jar b/lib/ecj-4.6.1.jar
new file mode 100644
index 0000000..1c7e9ca
--- /dev/null
+++ b/lib/ecj-4.6.1.jar
Binary files differ
diff --git a/lib/geomet-0.1.0.zip b/lib/geomet-0.1.0.zip
new file mode 100644
index 0000000..9b446c0
--- /dev/null
+++ b/lib/geomet-0.1.0.zip
Binary files differ
diff --git a/lib/guava-18.0.jar b/lib/guava-18.0.jar
deleted file mode 100644
index 8f89e49..0000000
--- a/lib/guava-18.0.jar
+++ /dev/null
Binary files differ
diff --git a/lib/guava-27.0-jre.jar b/lib/guava-27.0-jre.jar
new file mode 100644
index 0000000..1bdd007
--- /dev/null
+++ b/lib/guava-27.0-jre.jar
Binary files differ
diff --git a/lib/hppc-0.5.4.jar b/lib/hppc-0.5.4.jar
deleted file mode 100644
index d84b83b..0000000
--- a/lib/hppc-0.5.4.jar
+++ /dev/null
Binary files differ
diff --git a/lib/hppc-0.8.1.jar b/lib/hppc-0.8.1.jar
new file mode 100644
index 0000000..39a7c24
--- /dev/null
+++ b/lib/hppc-0.8.1.jar
Binary files differ
diff --git a/lib/j2objc-annotations-1.3.jar b/lib/j2objc-annotations-1.3.jar
new file mode 100644
index 0000000..a429c72
--- /dev/null
+++ b/lib/j2objc-annotations-1.3.jar
Binary files differ
diff --git a/lib/jamm-0.3.0.jar b/lib/jamm-0.3.0.jar
deleted file mode 100644
index f770592..0000000
--- a/lib/jamm-0.3.0.jar
+++ /dev/null
Binary files differ
diff --git a/lib/jamm-0.3.2.jar b/lib/jamm-0.3.2.jar
new file mode 100644
index 0000000..e0a61a1
--- /dev/null
+++ b/lib/jamm-0.3.2.jar
Binary files differ
diff --git a/lib/jcl-over-slf4j-1.7.25.jar b/lib/jcl-over-slf4j-1.7.25.jar
new file mode 100644
index 0000000..8e7fec8
--- /dev/null
+++ b/lib/jcl-over-slf4j-1.7.25.jar
Binary files differ
diff --git a/lib/jcl-over-slf4j-1.7.7.jar b/lib/jcl-over-slf4j-1.7.7.jar
deleted file mode 100644
index ed8d4dd..0000000
--- a/lib/jcl-over-slf4j-1.7.7.jar
+++ /dev/null
Binary files differ
diff --git a/lib/jcommander-1.30.jar b/lib/jcommander-1.30.jar
new file mode 100644
index 0000000..ec6c420
--- /dev/null
+++ b/lib/jcommander-1.30.jar
Binary files differ
diff --git a/lib/joda-time-2.4.jar b/lib/joda-time-2.4.jar
deleted file mode 100644
index ace67d7..0000000
--- a/lib/joda-time-2.4.jar
+++ /dev/null
Binary files differ
diff --git a/lib/jvm-attach-api-1.5.jar b/lib/jvm-attach-api-1.5.jar
new file mode 100644
index 0000000..e9b5809
--- /dev/null
+++ b/lib/jvm-attach-api-1.5.jar
Binary files differ
diff --git a/lib/libthrift-0.9.2.jar b/lib/libthrift-0.9.2.jar
deleted file mode 100644
index 39143a5..0000000
--- a/lib/libthrift-0.9.2.jar
+++ /dev/null
Binary files differ
diff --git a/lib/licenses/LICENSE-2.0.txt b/lib/licenses/LICENSE-2.0.txt
new file mode 100644
index 0000000..7a4a3ea
--- /dev/null
+++ b/lib/licenses/LICENSE-2.0.txt
@@ -0,0 +1,202 @@
+
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright [yyyy] [name of copyright owner]
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
\ No newline at end of file
diff --git a/lib/licenses/airline-0.6.txt b/lib/licenses/airline-0.8.txt
similarity index 100%
rename from lib/licenses/airline-0.6.txt
rename to lib/licenses/airline-0.8.txt
diff --git a/lib/licenses/asm-5.0.4.txt b/lib/licenses/asm-6.2.txt
similarity index 100%
rename from lib/licenses/asm-5.0.4.txt
rename to lib/licenses/asm-6.2.txt
diff --git a/lib/licenses/caffeine-2.2.6.txt b/lib/licenses/caffeine-2.2.6.txt
deleted file mode 100644
index d645695..0000000
--- a/lib/licenses/caffeine-2.2.6.txt
+++ /dev/null
@@ -1,202 +0,0 @@
-
- Apache License
- Version 2.0, January 2004
- http://www.apache.org/licenses/
-
- TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
- 1. Definitions.
-
- "License" shall mean the terms and conditions for use, reproduction,
- and distribution as defined by Sections 1 through 9 of this document.
-
- "Licensor" shall mean the copyright owner or entity authorized by
- the copyright owner that is granting the License.
-
- "Legal Entity" shall mean the union of the acting entity and all
- other entities that control, are controlled by, or are under common
- control with that entity. For the purposes of this definition,
- "control" means (i) the power, direct or indirect, to cause the
- direction or management of such entity, whether by contract or
- otherwise, or (ii) ownership of fifty percent (50%) or more of the
- outstanding shares, or (iii) beneficial ownership of such entity.
-
- "You" (or "Your") shall mean an individual or Legal Entity
- exercising permissions granted by this License.
-
- "Source" form shall mean the preferred form for making modifications,
- including but not limited to software source code, documentation
- source, and configuration files.
-
- "Object" form shall mean any form resulting from mechanical
- transformation or translation of a Source form, including but
- not limited to compiled object code, generated documentation,
- and conversions to other media types.
-
- "Work" shall mean the work of authorship, whether in Source or
- Object form, made available under the License, as indicated by a
- copyright notice that is included in or attached to the work
- (an example is provided in the Appendix below).
-
- "Derivative Works" shall mean any work, whether in Source or Object
- form, that is based on (or derived from) the Work and for which the
- editorial revisions, annotations, elaborations, or other modifications
- represent, as a whole, an original work of authorship. For the purposes
- of this License, Derivative Works shall not include works that remain
- separable from, or merely link (or bind by name) to the interfaces of,
- the Work and Derivative Works thereof.
-
- "Contribution" shall mean any work of authorship, including
- the original version of the Work and any modifications or additions
- to that Work or Derivative Works thereof, that is intentionally
- submitted to Licensor for inclusion in the Work by the copyright owner
- or by an individual or Legal Entity authorized to submit on behalf of
- the copyright owner. For the purposes of this definition, "submitted"
- means any form of electronic, verbal, or written communication sent
- to the Licensor or its representatives, including but not limited to
- communication on electronic mailing lists, source code control systems,
- and issue tracking systems that are managed by, or on behalf of, the
- Licensor for the purpose of discussing and improving the Work, but
- excluding communication that is conspicuously marked or otherwise
- designated in writing by the copyright owner as "Not a Contribution."
-
- "Contributor" shall mean Licensor and any individual or Legal Entity
- on behalf of whom a Contribution has been received by Licensor and
- subsequently incorporated within the Work.
-
- 2. Grant of Copyright License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- copyright license to reproduce, prepare Derivative Works of,
- publicly display, publicly perform, sublicense, and distribute the
- Work and such Derivative Works in Source or Object form.
-
- 3. Grant of Patent License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- (except as stated in this section) patent license to make, have made,
- use, offer to sell, sell, import, and otherwise transfer the Work,
- where such license applies only to those patent claims licensable
- by such Contributor that are necessarily infringed by their
- Contribution(s) alone or by combination of their Contribution(s)
- with the Work to which such Contribution(s) was submitted. If You
- institute patent litigation against any entity (including a
- cross-claim or counterclaim in a lawsuit) alleging that the Work
- or a Contribution incorporated within the Work constitutes direct
- or contributory patent infringement, then any patent licenses
- granted to You under this License for that Work shall terminate
- as of the date such litigation is filed.
-
- 4. Redistribution. You may reproduce and distribute copies of the
- Work or Derivative Works thereof in any medium, with or without
- modifications, and in Source or Object form, provided that You
- meet the following conditions:
-
- (a) You must give any other recipients of the Work or
- Derivative Works a copy of this License; and
-
- (b) You must cause any modified files to carry prominent notices
- stating that You changed the files; and
-
- (c) You must retain, in the Source form of any Derivative Works
- that You distribute, all copyright, patent, trademark, and
- attribution notices from the Source form of the Work,
- excluding those notices that do not pertain to any part of
- the Derivative Works; and
-
- (d) If the Work includes a "NOTICE" text file as part of its
- distribution, then any Derivative Works that You distribute must
- include a readable copy of the attribution notices contained
- within such NOTICE file, excluding those notices that do not
- pertain to any part of the Derivative Works, in at least one
- of the following places: within a NOTICE text file distributed
- as part of the Derivative Works; within the Source form or
- documentation, if provided along with the Derivative Works; or,
- within a display generated by the Derivative Works, if and
- wherever such third-party notices normally appear. The contents
- of the NOTICE file are for informational purposes only and
- do not modify the License. You may add Your own attribution
- notices within Derivative Works that You distribute, alongside
- or as an addendum to the NOTICE text from the Work, provided
- that such additional attribution notices cannot be construed
- as modifying the License.
-
- You may add Your own copyright statement to Your modifications and
- may provide additional or different license terms and conditions
- for use, reproduction, or distribution of Your modifications, or
- for any such Derivative Works as a whole, provided Your use,
- reproduction, and distribution of the Work otherwise complies with
- the conditions stated in this License.
-
- 5. Submission of Contributions. Unless You explicitly state otherwise,
- any Contribution intentionally submitted for inclusion in the Work
- by You to the Licensor shall be under the terms and conditions of
- this License, without any additional terms or conditions.
- Notwithstanding the above, nothing herein shall supersede or modify
- the terms of any separate license agreement you may have executed
- with Licensor regarding such Contributions.
-
- 6. Trademarks. This License does not grant permission to use the trade
- names, trademarks, service marks, or product names of the Licensor,
- except as required for reasonable and customary use in describing the
- origin of the Work and reproducing the content of the NOTICE file.
-
- 7. Disclaimer of Warranty. Unless required by applicable law or
- agreed to in writing, Licensor provides the Work (and each
- Contributor provides its Contributions) on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
- implied, including, without limitation, any warranties or conditions
- of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
- PARTICULAR PURPOSE. You are solely responsible for determining the
- appropriateness of using or redistributing the Work and assume any
- risks associated with Your exercise of permissions under this License.
-
- 8. Limitation of Liability. In no event and under no legal theory,
- whether in tort (including negligence), contract, or otherwise,
- unless required by applicable law (such as deliberate and grossly
- negligent acts) or agreed to in writing, shall any Contributor be
- liable to You for damages, including any direct, indirect, special,
- incidental, or consequential damages of any character arising as a
- result of this License or out of the use or inability to use the
- Work (including but not limited to damages for loss of goodwill,
- work stoppage, computer failure or malfunction, or any and all
- other commercial damages or losses), even if such Contributor
- has been advised of the possibility of such damages.
-
- 9. Accepting Warranty or Additional Liability. While redistributing
- the Work or Derivative Works thereof, You may choose to offer,
- and charge a fee for, acceptance of support, warranty, indemnity,
- or other liability obligations and/or rights consistent with this
- License. However, in accepting such obligations, You may act only
- on Your own behalf and on Your sole responsibility, not on behalf
- of any other Contributor, and only if You agree to indemnify,
- defend, and hold each Contributor harmless for any liability
- incurred by, or claims asserted against, such Contributor by reason
- of your accepting any such warranty or additional liability.
-
- END OF TERMS AND CONDITIONS
-
- APPENDIX: How to apply the Apache License to your work.
-
- To apply the Apache License to your work, attach the following
- boilerplate notice, with the fields enclosed by brackets "[]"
- replaced with your own identifying information. (Don't include
- the brackets!) The text should be enclosed in the appropriate
- comment syntax for the file format. We also recommend that a
- file or class name and description of purpose be included on the
- same "printed page" as the copyright notice for easier
- identification within third-party archives.
-
- Copyright [yyyy] [name of copyright owner]
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
diff --git a/lib/licenses/airline-0.6.txt b/lib/licenses/caffeine-2.3.5.txt
similarity index 100%
copy from lib/licenses/airline-0.6.txt
copy to lib/licenses/caffeine-2.3.5.txt
diff --git a/lib/licenses/chronicle-bytes-1.16.3.txt b/lib/licenses/chronicle-bytes-1.16.3.txt
new file mode 100644
index 0000000..d8a262e
--- /dev/null
+++ b/lib/licenses/chronicle-bytes-1.16.3.txt
@@ -0,0 +1,14 @@
+
+== Copyright 2016 higherfrequencytrading.com
+
+Licensed under the *Apache License, Version 2.0* (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
diff --git a/lib/licenses/chronicle-core-1.16.3-SNAPSHOT.txt b/lib/licenses/chronicle-core-1.16.3-SNAPSHOT.txt
new file mode 100644
index 0000000..d8a262e
--- /dev/null
+++ b/lib/licenses/chronicle-core-1.16.3-SNAPSHOT.txt
@@ -0,0 +1,14 @@
+
+== Copyright 2016 higherfrequencytrading.com
+
+Licensed under the *Apache License, Version 2.0* (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
diff --git a/lib/licenses/chronicle-queue-4.16.3.txt b/lib/licenses/chronicle-queue-4.16.3.txt
new file mode 100644
index 0000000..d8a262e
--- /dev/null
+++ b/lib/licenses/chronicle-queue-4.16.3.txt
@@ -0,0 +1,14 @@
+
+== Copyright 2016 higherfrequencytrading.com
+
+Licensed under the *Apache License, Version 2.0* (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
diff --git a/lib/licenses/chronicle-threads-1.16.0.txt b/lib/licenses/chronicle-threads-1.16.0.txt
new file mode 100644
index 0000000..d8a262e
--- /dev/null
+++ b/lib/licenses/chronicle-threads-1.16.0.txt
@@ -0,0 +1,14 @@
+
+== Copyright 2016 higherfrequencytrading.com
+
+Licensed under the *Apache License, Version 2.0* (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
diff --git a/lib/licenses/chronicle-wire-1.16.1.txt b/lib/licenses/chronicle-wire-1.16.1.txt
new file mode 100644
index 0000000..d8a262e
--- /dev/null
+++ b/lib/licenses/chronicle-wire-1.16.1.txt
@@ -0,0 +1,14 @@
+
+== Copyright 2016 higherfrequencytrading.com
+
+Licensed under the *Apache License, Version 2.0* (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
diff --git a/lib/licenses/concurrentlinkedhashmap-lru-1.4.txt b/lib/licenses/concurrentlinkedhashmap-lru-1.4.txt
deleted file mode 100644
index d645695..0000000
--- a/lib/licenses/concurrentlinkedhashmap-lru-1.4.txt
+++ /dev/null
@@ -1,202 +0,0 @@
-
- Apache License
- Version 2.0, January 2004
- http://www.apache.org/licenses/
-
- TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
- 1. Definitions.
-
- "License" shall mean the terms and conditions for use, reproduction,
- and distribution as defined by Sections 1 through 9 of this document.
-
- "Licensor" shall mean the copyright owner or entity authorized by
- the copyright owner that is granting the License.
-
- "Legal Entity" shall mean the union of the acting entity and all
- other entities that control, are controlled by, or are under common
- control with that entity. For the purposes of this definition,
- "control" means (i) the power, direct or indirect, to cause the
- direction or management of such entity, whether by contract or
- otherwise, or (ii) ownership of fifty percent (50%) or more of the
- outstanding shares, or (iii) beneficial ownership of such entity.
-
- "You" (or "Your") shall mean an individual or Legal Entity
- exercising permissions granted by this License.
-
- "Source" form shall mean the preferred form for making modifications,
- including but not limited to software source code, documentation
- source, and configuration files.
-
- "Object" form shall mean any form resulting from mechanical
- transformation or translation of a Source form, including but
- not limited to compiled object code, generated documentation,
- and conversions to other media types.
-
- "Work" shall mean the work of authorship, whether in Source or
- Object form, made available under the License, as indicated by a
- copyright notice that is included in or attached to the work
- (an example is provided in the Appendix below).
-
- "Derivative Works" shall mean any work, whether in Source or Object
- form, that is based on (or derived from) the Work and for which the
- editorial revisions, annotations, elaborations, or other modifications
- represent, as a whole, an original work of authorship. For the purposes
- of this License, Derivative Works shall not include works that remain
- separable from, or merely link (or bind by name) to the interfaces of,
- the Work and Derivative Works thereof.
-
- "Contribution" shall mean any work of authorship, including
- the original version of the Work and any modifications or additions
- to that Work or Derivative Works thereof, that is intentionally
- submitted to Licensor for inclusion in the Work by the copyright owner
- or by an individual or Legal Entity authorized to submit on behalf of
- the copyright owner. For the purposes of this definition, "submitted"
- means any form of electronic, verbal, or written communication sent
- to the Licensor or its representatives, including but not limited to
- communication on electronic mailing lists, source code control systems,
- and issue tracking systems that are managed by, or on behalf of, the
- Licensor for the purpose of discussing and improving the Work, but
- excluding communication that is conspicuously marked or otherwise
- designated in writing by the copyright owner as "Not a Contribution."
-
- "Contributor" shall mean Licensor and any individual or Legal Entity
- on behalf of whom a Contribution has been received by Licensor and
- subsequently incorporated within the Work.
-
- 2. Grant of Copyright License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- copyright license to reproduce, prepare Derivative Works of,
- publicly display, publicly perform, sublicense, and distribute the
- Work and such Derivative Works in Source or Object form.
-
- 3. Grant of Patent License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- (except as stated in this section) patent license to make, have made,
- use, offer to sell, sell, import, and otherwise transfer the Work,
- where such license applies only to those patent claims licensable
- by such Contributor that are necessarily infringed by their
- Contribution(s) alone or by combination of their Contribution(s)
- with the Work to which such Contribution(s) was submitted. If You
- institute patent litigation against any entity (including a
- cross-claim or counterclaim in a lawsuit) alleging that the Work
- or a Contribution incorporated within the Work constitutes direct
- or contributory patent infringement, then any patent licenses
- granted to You under this License for that Work shall terminate
- as of the date such litigation is filed.
-
- 4. Redistribution. You may reproduce and distribute copies of the
- Work or Derivative Works thereof in any medium, with or without
- modifications, and in Source or Object form, provided that You
- meet the following conditions:
-
- (a) You must give any other recipients of the Work or
- Derivative Works a copy of this License; and
-
- (b) You must cause any modified files to carry prominent notices
- stating that You changed the files; and
-
- (c) You must retain, in the Source form of any Derivative Works
- that You distribute, all copyright, patent, trademark, and
- attribution notices from the Source form of the Work,
- excluding those notices that do not pertain to any part of
- the Derivative Works; and
-
- (d) If the Work includes a "NOTICE" text file as part of its
- distribution, then any Derivative Works that You distribute must
- include a readable copy of the attribution notices contained
- within such NOTICE file, excluding those notices that do not
- pertain to any part of the Derivative Works, in at least one
- of the following places: within a NOTICE text file distributed
- as part of the Derivative Works; within the Source form or
- documentation, if provided along with the Derivative Works; or,
- within a display generated by the Derivative Works, if and
- wherever such third-party notices normally appear. The contents
- of the NOTICE file are for informational purposes only and
- do not modify the License. You may add Your own attribution
- notices within Derivative Works that You distribute, alongside
- or as an addendum to the NOTICE text from the Work, provided
- that such additional attribution notices cannot be construed
- as modifying the License.
-
- You may add Your own copyright statement to Your modifications and
- may provide additional or different license terms and conditions
- for use, reproduction, or distribution of Your modifications, or
- for any such Derivative Works as a whole, provided Your use,
- reproduction, and distribution of the Work otherwise complies with
- the conditions stated in this License.
-
- 5. Submission of Contributions. Unless You explicitly state otherwise,
- any Contribution intentionally submitted for inclusion in the Work
- by You to the Licensor shall be under the terms and conditions of
- this License, without any additional terms or conditions.
- Notwithstanding the above, nothing herein shall supersede or modify
- the terms of any separate license agreement you may have executed
- with Licensor regarding such Contributions.
-
- 6. Trademarks. This License does not grant permission to use the trade
- names, trademarks, service marks, or product names of the Licensor,
- except as required for reasonable and customary use in describing the
- origin of the Work and reproducing the content of the NOTICE file.
-
- 7. Disclaimer of Warranty. Unless required by applicable law or
- agreed to in writing, Licensor provides the Work (and each
- Contributor provides its Contributions) on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
- implied, including, without limitation, any warranties or conditions
- of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
- PARTICULAR PURPOSE. You are solely responsible for determining the
- appropriateness of using or redistributing the Work and assume any
- risks associated with Your exercise of permissions under this License.
-
- 8. Limitation of Liability. In no event and under no legal theory,
- whether in tort (including negligence), contract, or otherwise,
- unless required by applicable law (such as deliberate and grossly
- negligent acts) or agreed to in writing, shall any Contributor be
- liable to You for damages, including any direct, indirect, special,
- incidental, or consequential damages of any character arising as a
- result of this License or out of the use or inability to use the
- Work (including but not limited to damages for loss of goodwill,
- work stoppage, computer failure or malfunction, or any and all
- other commercial damages or losses), even if such Contributor
- has been advised of the possibility of such damages.
-
- 9. Accepting Warranty or Additional Liability. While redistributing
- the Work or Derivative Works thereof, You may choose to offer,
- and charge a fee for, acceptance of support, warranty, indemnity,
- or other liability obligations and/or rights consistent with this
- License. However, in accepting such obligations, You may act only
- on Your own behalf and on Your sole responsibility, not on behalf
- of any other Contributor, and only if You agree to indemnify,
- defend, and hold each Contributor harmless for any liability
- incurred by, or claims asserted against, such Contributor by reason
- of your accepting any such warranty or additional liability.
-
- END OF TERMS AND CONDITIONS
-
- APPENDIX: How to apply the Apache License to your work.
-
- To apply the Apache License to your work, attach the following
- boilerplate notice, with the fields enclosed by brackets "[]"
- replaced with your own identifying information. (Don't include
- the brackets!) The text should be enclosed in the appropriate
- comment syntax for the file format. We also recommend that a
- file or class name and description of purpose be included on the
- same "printed page" as the copyright notice for easier
- identification within third-party archives.
-
- Copyright [yyyy] [name of copyright owner]
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
diff --git a/lib/licenses/disruptor-3.0.1.txt b/lib/licenses/disruptor-3.0.1.txt
deleted file mode 100644
index 50086f8..0000000
--- a/lib/licenses/disruptor-3.0.1.txt
+++ /dev/null
@@ -1,201 +0,0 @@
- Apache License
- Version 2.0, January 2004
- http://www.apache.org/licenses/
-
- TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
- 1. Definitions.
-
- "License" shall mean the terms and conditions for use, reproduction,
- and distribution as defined by Sections 1 through 9 of this document.
-
- "Licensor" shall mean the copyright owner or entity authorized by
- the copyright owner that is granting the License.
-
- "Legal Entity" shall mean the union of the acting entity and all
- other entities that control, are controlled by, or are under common
- control with that entity. For the purposes of this definition,
- "control" means (i) the power, direct or indirect, to cause the
- direction or management of such entity, whether by contract or
- otherwise, or (ii) ownership of fifty percent (50%) or more of the
- outstanding shares, or (iii) beneficial ownership of such entity.
-
- "You" (or "Your") shall mean an individual or Legal Entity
- exercising permissions granted by this License.
-
- "Source" form shall mean the preferred form for making modifications,
- including but not limited to software source code, documentation
- source, and configuration files.
-
- "Object" form shall mean any form resulting from mechanical
- transformation or translation of a Source form, including but
- not limited to compiled object code, generated documentation,
- and conversions to other media types.
-
- "Work" shall mean the work of authorship, whether in Source or
- Object form, made available under the License, as indicated by a
- copyright notice that is included in or attached to the work
- (an example is provided in the Appendix below).
-
- "Derivative Works" shall mean any work, whether in Source or Object
- form, that is based on (or derived from) the Work and for which the
- editorial revisions, annotations, elaborations, or other modifications
- represent, as a whole, an original work of authorship. For the purposes
- of this License, Derivative Works shall not include works that remain
- separable from, or merely link (or bind by name) to the interfaces of,
- the Work and Derivative Works thereof.
-
- "Contribution" shall mean any work of authorship, including
- the original version of the Work and any modifications or additions
- to that Work or Derivative Works thereof, that is intentionally
- submitted to Licensor for inclusion in the Work by the copyright owner
- or by an individual or Legal Entity authorized to submit on behalf of
- the copyright owner. For the purposes of this definition, "submitted"
- means any form of electronic, verbal, or written communication sent
- to the Licensor or its representatives, including but not limited to
- communication on electronic mailing lists, source code control systems,
- and issue tracking systems that are managed by, or on behalf of, the
- Licensor for the purpose of discussing and improving the Work, but
- excluding communication that is conspicuously marked or otherwise
- designated in writing by the copyright owner as "Not a Contribution."
-
- "Contributor" shall mean Licensor and any individual or Legal Entity
- on behalf of whom a Contribution has been received by Licensor and
- subsequently incorporated within the Work.
-
- 2. Grant of Copyright License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- copyright license to reproduce, prepare Derivative Works of,
- publicly display, publicly perform, sublicense, and distribute the
- Work and such Derivative Works in Source or Object form.
-
- 3. Grant of Patent License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- (except as stated in this section) patent license to make, have made,
- use, offer to sell, sell, import, and otherwise transfer the Work,
- where such license applies only to those patent claims licensable
- by such Contributor that are necessarily infringed by their
- Contribution(s) alone or by combination of their Contribution(s)
- with the Work to which such Contribution(s) was submitted. If You
- institute patent litigation against any entity (including a
- cross-claim or counterclaim in a lawsuit) alleging that the Work
- or a Contribution incorporated within the Work constitutes direct
- or contributory patent infringement, then any patent licenses
- granted to You under this License for that Work shall terminate
- as of the date such litigation is filed.
-
- 4. Redistribution. You may reproduce and distribute copies of the
- Work or Derivative Works thereof in any medium, with or without
- modifications, and in Source or Object form, provided that You
- meet the following conditions:
-
- (a) You must give any other recipients of the Work or
- Derivative Works a copy of this License; and
-
- (b) You must cause any modified files to carry prominent notices
- stating that You changed the files; and
-
- (c) You must retain, in the Source form of any Derivative Works
- that You distribute, all copyright, patent, trademark, and
- attribution notices from the Source form of the Work,
- excluding those notices that do not pertain to any part of
- the Derivative Works; and
-
- (d) If the Work includes a "NOTICE" text file as part of its
- distribution, then any Derivative Works that You distribute must
- include a readable copy of the attribution notices contained
- within such NOTICE file, excluding those notices that do not
- pertain to any part of the Derivative Works, in at least one
- of the following places: within a NOTICE text file distributed
- as part of the Derivative Works; within the Source form or
- documentation, if provided along with the Derivative Works; or,
- within a display generated by the Derivative Works, if and
- wherever such third-party notices normally appear. The contents
- of the NOTICE file are for informational purposes only and
- do not modify the License. You may add Your own attribution
- notices within Derivative Works that You distribute, alongside
- or as an addendum to the NOTICE text from the Work, provided
- that such additional attribution notices cannot be construed
- as modifying the License.
-
- You may add Your own copyright statement to Your modifications and
- may provide additional or different license terms and conditions
- for use, reproduction, or distribution of Your modifications, or
- for any such Derivative Works as a whole, provided Your use,
- reproduction, and distribution of the Work otherwise complies with
- the conditions stated in this License.
-
- 5. Submission of Contributions. Unless You explicitly state otherwise,
- any Contribution intentionally submitted for inclusion in the Work
- by You to the Licensor shall be under the terms and conditions of
- this License, without any additional terms or conditions.
- Notwithstanding the above, nothing herein shall supersede or modify
- the terms of any separate license agreement you may have executed
- with Licensor regarding such Contributions.
-
- 6. Trademarks. This License does not grant permission to use the trade
- names, trademarks, service marks, or product names of the Licensor,
- except as required for reasonable and customary use in describing the
- origin of the Work and reproducing the content of the NOTICE file.
-
- 7. Disclaimer of Warranty. Unless required by applicable law or
- agreed to in writing, Licensor provides the Work (and each
- Contributor provides its Contributions) on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
- implied, including, without limitation, any warranties or conditions
- of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
- PARTICULAR PURPOSE. You are solely responsible for determining the
- appropriateness of using or redistributing the Work and assume any
- risks associated with Your exercise of permissions under this License.
-
- 8. Limitation of Liability. In no event and under no legal theory,
- whether in tort (including negligence), contract, or otherwise,
- unless required by applicable law (such as deliberate and grossly
- negligent acts) or agreed to in writing, shall any Contributor be
- liable to You for damages, including any direct, indirect, special,
- incidental, or consequential damages of any character arising as a
- result of this License or out of the use or inability to use the
- Work (including but not limited to damages for loss of goodwill,
- work stoppage, computer failure or malfunction, or any and all
- other commercial damages or losses), even if such Contributor
- has been advised of the possibility of such damages.
-
- 9. Accepting Warranty or Additional Liability. While redistributing
- the Work or Derivative Works thereof, You may choose to offer,
- and charge a fee for, acceptance of support, warranty, indemnity,
- or other liability obligations and/or rights consistent with this
- License. However, in accepting such obligations, You may act only
- on Your own behalf and on Your sole responsibility, not on behalf
- of any other Contributor, and only if You agree to indemnify,
- defend, and hold each Contributor harmless for any liability
- incurred by, or claims asserted against, such Contributor by reason
- of your accepting any such warranty or additional liability.
-
- END OF TERMS AND CONDITIONS
-
- APPENDIX: How to apply the Apache License to your work.
-
- To apply the Apache License to your work, attach the following
- boilerplate notice, with the fields enclosed by brackets "[]"
- replaced with your own identifying information. (Don't include
- the brackets!) The text should be enclosed in the appropriate
- comment syntax for the file format. We also recommend that a
- file or class name and description of purpose be included on the
- same "printed page" as the copyright notice for easier
- identification within third-party archives.
-
- Copyright [yyyy] [name of copyright owner]
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
\ No newline at end of file
diff --git a/lib/licenses/ecj-4.4.2.txt b/lib/licenses/ecj-4.6.1.txt
similarity index 100%
rename from lib/licenses/ecj-4.4.2.txt
rename to lib/licenses/ecj-4.6.1.txt
diff --git a/lib/licenses/geom-0.1.0.txt b/lib/licenses/geom-0.1.0.txt
new file mode 100644
index 0000000..5c304d1
--- /dev/null
+++ b/lib/licenses/geom-0.1.0.txt
@@ -0,0 +1,201 @@
+Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "{}"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright {yyyy} {name of copyright owner}
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
diff --git a/lib/licenses/airline-0.6.txt b/lib/licenses/guava-23.3-jre.txt
similarity index 100%
copy from lib/licenses/airline-0.6.txt
copy to lib/licenses/guava-23.3-jre.txt
diff --git a/lib/licenses/jackson-core-asl-1.9.13.txt b/lib/licenses/j2objc-annotations-1.3.txt
similarity index 100%
rename from lib/licenses/jackson-core-asl-1.9.13.txt
rename to lib/licenses/j2objc-annotations-1.3.txt
diff --git a/lib/licenses/airline-0.6.txt b/lib/licenses/jackson-annotations-2.9.5.txt
similarity index 100%
copy from lib/licenses/airline-0.6.txt
copy to lib/licenses/jackson-annotations-2.9.5.txt
diff --git a/lib/licenses/airline-0.6.txt b/lib/licenses/jackson-core-2.9.5.txt
similarity index 100%
copy from lib/licenses/airline-0.6.txt
copy to lib/licenses/jackson-core-2.9.5.txt
diff --git a/lib/licenses/airline-0.6.txt b/lib/licenses/jackson-databind-2.9.5.txt
similarity index 100%
copy from lib/licenses/airline-0.6.txt
copy to lib/licenses/jackson-databind-2.9.5.txt
diff --git a/lib/licenses/jackson-mapper-asl-1.9.13.txt b/lib/licenses/jackson-mapper-asl-1.9.13.txt
deleted file mode 100644
index d645695..0000000
--- a/lib/licenses/jackson-mapper-asl-1.9.13.txt
+++ /dev/null
@@ -1,202 +0,0 @@
-
- Apache License
- Version 2.0, January 2004
- http://www.apache.org/licenses/
-
- TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
- 1. Definitions.
-
- "License" shall mean the terms and conditions for use, reproduction,
- and distribution as defined by Sections 1 through 9 of this document.
-
- "Licensor" shall mean the copyright owner or entity authorized by
- the copyright owner that is granting the License.
-
- "Legal Entity" shall mean the union of the acting entity and all
- other entities that control, are controlled by, or are under common
- control with that entity. For the purposes of this definition,
- "control" means (i) the power, direct or indirect, to cause the
- direction or management of such entity, whether by contract or
- otherwise, or (ii) ownership of fifty percent (50%) or more of the
- outstanding shares, or (iii) beneficial ownership of such entity.
-
- "You" (or "Your") shall mean an individual or Legal Entity
- exercising permissions granted by this License.
-
- "Source" form shall mean the preferred form for making modifications,
- including but not limited to software source code, documentation
- source, and configuration files.
-
- "Object" form shall mean any form resulting from mechanical
- transformation or translation of a Source form, including but
- not limited to compiled object code, generated documentation,
- and conversions to other media types.
-
- "Work" shall mean the work of authorship, whether in Source or
- Object form, made available under the License, as indicated by a
- copyright notice that is included in or attached to the work
- (an example is provided in the Appendix below).
-
- "Derivative Works" shall mean any work, whether in Source or Object
- form, that is based on (or derived from) the Work and for which the
- editorial revisions, annotations, elaborations, or other modifications
- represent, as a whole, an original work of authorship. For the purposes
- of this License, Derivative Works shall not include works that remain
- separable from, or merely link (or bind by name) to the interfaces of,
- the Work and Derivative Works thereof.
-
- "Contribution" shall mean any work of authorship, including
- the original version of the Work and any modifications or additions
- to that Work or Derivative Works thereof, that is intentionally
- submitted to Licensor for inclusion in the Work by the copyright owner
- or by an individual or Legal Entity authorized to submit on behalf of
- the copyright owner. For the purposes of this definition, "submitted"
- means any form of electronic, verbal, or written communication sent
- to the Licensor or its representatives, including but not limited to
- communication on electronic mailing lists, source code control systems,
- and issue tracking systems that are managed by, or on behalf of, the
- Licensor for the purpose of discussing and improving the Work, but
- excluding communication that is conspicuously marked or otherwise
- designated in writing by the copyright owner as "Not a Contribution."
-
- "Contributor" shall mean Licensor and any individual or Legal Entity
- on behalf of whom a Contribution has been received by Licensor and
- subsequently incorporated within the Work.
-
- 2. Grant of Copyright License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- copyright license to reproduce, prepare Derivative Works of,
- publicly display, publicly perform, sublicense, and distribute the
- Work and such Derivative Works in Source or Object form.
-
- 3. Grant of Patent License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- (except as stated in this section) patent license to make, have made,
- use, offer to sell, sell, import, and otherwise transfer the Work,
- where such license applies only to those patent claims licensable
- by such Contributor that are necessarily infringed by their
- Contribution(s) alone or by combination of their Contribution(s)
- with the Work to which such Contribution(s) was submitted. If You
- institute patent litigation against any entity (including a
- cross-claim or counterclaim in a lawsuit) alleging that the Work
- or a Contribution incorporated within the Work constitutes direct
- or contributory patent infringement, then any patent licenses
- granted to You under this License for that Work shall terminate
- as of the date such litigation is filed.
-
- 4. Redistribution. You may reproduce and distribute copies of the
- Work or Derivative Works thereof in any medium, with or without
- modifications, and in Source or Object form, provided that You
- meet the following conditions:
-
- (a) You must give any other recipients of the Work or
- Derivative Works a copy of this License; and
-
- (b) You must cause any modified files to carry prominent notices
- stating that You changed the files; and
-
- (c) You must retain, in the Source form of any Derivative Works
- that You distribute, all copyright, patent, trademark, and
- attribution notices from the Source form of the Work,
- excluding those notices that do not pertain to any part of
- the Derivative Works; and
-
- (d) If the Work includes a "NOTICE" text file as part of its
- distribution, then any Derivative Works that You distribute must
- include a readable copy of the attribution notices contained
- within such NOTICE file, excluding those notices that do not
- pertain to any part of the Derivative Works, in at least one
- of the following places: within a NOTICE text file distributed
- as part of the Derivative Works; within the Source form or
- documentation, if provided along with the Derivative Works; or,
- within a display generated by the Derivative Works, if and
- wherever such third-party notices normally appear. The contents
- of the NOTICE file are for informational purposes only and
- do not modify the License. You may add Your own attribution
- notices within Derivative Works that You distribute, alongside
- or as an addendum to the NOTICE text from the Work, provided
- that such additional attribution notices cannot be construed
- as modifying the License.
-
- You may add Your own copyright statement to Your modifications and
- may provide additional or different license terms and conditions
- for use, reproduction, or distribution of Your modifications, or
- for any such Derivative Works as a whole, provided Your use,
- reproduction, and distribution of the Work otherwise complies with
- the conditions stated in this License.
-
- 5. Submission of Contributions. Unless You explicitly state otherwise,
- any Contribution intentionally submitted for inclusion in the Work
- by You to the Licensor shall be under the terms and conditions of
- this License, without any additional terms or conditions.
- Notwithstanding the above, nothing herein shall supersede or modify
- the terms of any separate license agreement you may have executed
- with Licensor regarding such Contributions.
-
- 6. Trademarks. This License does not grant permission to use the trade
- names, trademarks, service marks, or product names of the Licensor,
- except as required for reasonable and customary use in describing the
- origin of the Work and reproducing the content of the NOTICE file.
-
- 7. Disclaimer of Warranty. Unless required by applicable law or
- agreed to in writing, Licensor provides the Work (and each
- Contributor provides its Contributions) on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
- implied, including, without limitation, any warranties or conditions
- of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
- PARTICULAR PURPOSE. You are solely responsible for determining the
- appropriateness of using or redistributing the Work and assume any
- risks associated with Your exercise of permissions under this License.
-
- 8. Limitation of Liability. In no event and under no legal theory,
- whether in tort (including negligence), contract, or otherwise,
- unless required by applicable law (such as deliberate and grossly
- negligent acts) or agreed to in writing, shall any Contributor be
- liable to You for damages, including any direct, indirect, special,
- incidental, or consequential damages of any character arising as a
- result of this License or out of the use or inability to use the
- Work (including but not limited to damages for loss of goodwill,
- work stoppage, computer failure or malfunction, or any and all
- other commercial damages or losses), even if such Contributor
- has been advised of the possibility of such damages.
-
- 9. Accepting Warranty or Additional Liability. While redistributing
- the Work or Derivative Works thereof, You may choose to offer,
- and charge a fee for, acceptance of support, warranty, indemnity,
- or other liability obligations and/or rights consistent with this
- License. However, in accepting such obligations, You may act only
- on Your own behalf and on Your sole responsibility, not on behalf
- of any other Contributor, and only if You agree to indemnify,
- defend, and hold each Contributor harmless for any liability
- incurred by, or claims asserted against, such Contributor by reason
- of your accepting any such warranty or additional liability.
-
- END OF TERMS AND CONDITIONS
-
- APPENDIX: How to apply the Apache License to your work.
-
- To apply the Apache License to your work, attach the following
- boilerplate notice, with the fields enclosed by brackets "[]"
- replaced with your own identifying information. (Don't include
- the brackets!) The text should be enclosed in the appropriate
- comment syntax for the file format. We also recommend that a
- file or class name and description of purpose be included on the
- same "printed page" as the copyright notice for easier
- identification within third-party archives.
-
- Copyright [yyyy] [name of copyright owner]
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
diff --git a/lib/licenses/jamm-0.3.0.txt b/lib/licenses/jamm-0.3.0.txt
deleted file mode 100644
index d645695..0000000
--- a/lib/licenses/jamm-0.3.0.txt
+++ /dev/null
@@ -1,202 +0,0 @@
-
- Apache License
- Version 2.0, January 2004
- http://www.apache.org/licenses/
-
- TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
- 1. Definitions.
-
- "License" shall mean the terms and conditions for use, reproduction,
- and distribution as defined by Sections 1 through 9 of this document.
-
- "Licensor" shall mean the copyright owner or entity authorized by
- the copyright owner that is granting the License.
-
- "Legal Entity" shall mean the union of the acting entity and all
- other entities that control, are controlled by, or are under common
- control with that entity. For the purposes of this definition,
- "control" means (i) the power, direct or indirect, to cause the
- direction or management of such entity, whether by contract or
- otherwise, or (ii) ownership of fifty percent (50%) or more of the
- outstanding shares, or (iii) beneficial ownership of such entity.
-
- "You" (or "Your") shall mean an individual or Legal Entity
- exercising permissions granted by this License.
-
- "Source" form shall mean the preferred form for making modifications,
- including but not limited to software source code, documentation
- source, and configuration files.
-
- "Object" form shall mean any form resulting from mechanical
- transformation or translation of a Source form, including but
- not limited to compiled object code, generated documentation,
- and conversions to other media types.
-
- "Work" shall mean the work of authorship, whether in Source or
- Object form, made available under the License, as indicated by a
- copyright notice that is included in or attached to the work
- (an example is provided in the Appendix below).
-
- "Derivative Works" shall mean any work, whether in Source or Object
- form, that is based on (or derived from) the Work and for which the
- editorial revisions, annotations, elaborations, or other modifications
- represent, as a whole, an original work of authorship. For the purposes
- of this License, Derivative Works shall not include works that remain
- separable from, or merely link (or bind by name) to the interfaces of,
- the Work and Derivative Works thereof.
-
- "Contribution" shall mean any work of authorship, including
- the original version of the Work and any modifications or additions
- to that Work or Derivative Works thereof, that is intentionally
- submitted to Licensor for inclusion in the Work by the copyright owner
- or by an individual or Legal Entity authorized to submit on behalf of
- the copyright owner. For the purposes of this definition, "submitted"
- means any form of electronic, verbal, or written communication sent
- to the Licensor or its representatives, including but not limited to
- communication on electronic mailing lists, source code control systems,
- and issue tracking systems that are managed by, or on behalf of, the
- Licensor for the purpose of discussing and improving the Work, but
- excluding communication that is conspicuously marked or otherwise
- designated in writing by the copyright owner as "Not a Contribution."
-
- "Contributor" shall mean Licensor and any individual or Legal Entity
- on behalf of whom a Contribution has been received by Licensor and
- subsequently incorporated within the Work.
-
- 2. Grant of Copyright License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- copyright license to reproduce, prepare Derivative Works of,
- publicly display, publicly perform, sublicense, and distribute the
- Work and such Derivative Works in Source or Object form.
-
- 3. Grant of Patent License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- (except as stated in this section) patent license to make, have made,
- use, offer to sell, sell, import, and otherwise transfer the Work,
- where such license applies only to those patent claims licensable
- by such Contributor that are necessarily infringed by their
- Contribution(s) alone or by combination of their Contribution(s)
- with the Work to which such Contribution(s) was submitted. If You
- institute patent litigation against any entity (including a
- cross-claim or counterclaim in a lawsuit) alleging that the Work
- or a Contribution incorporated within the Work constitutes direct
- or contributory patent infringement, then any patent licenses
- granted to You under this License for that Work shall terminate
- as of the date such litigation is filed.
-
- 4. Redistribution. You may reproduce and distribute copies of the
- Work or Derivative Works thereof in any medium, with or without
- modifications, and in Source or Object form, provided that You
- meet the following conditions:
-
- (a) You must give any other recipients of the Work or
- Derivative Works a copy of this License; and
-
- (b) You must cause any modified files to carry prominent notices
- stating that You changed the files; and
-
- (c) You must retain, in the Source form of any Derivative Works
- that You distribute, all copyright, patent, trademark, and
- attribution notices from the Source form of the Work,
- excluding those notices that do not pertain to any part of
- the Derivative Works; and
-
- (d) If the Work includes a "NOTICE" text file as part of its
- distribution, then any Derivative Works that You distribute must
- include a readable copy of the attribution notices contained
- within such NOTICE file, excluding those notices that do not
- pertain to any part of the Derivative Works, in at least one
- of the following places: within a NOTICE text file distributed
- as part of the Derivative Works; within the Source form or
- documentation, if provided along with the Derivative Works; or,
- within a display generated by the Derivative Works, if and
- wherever such third-party notices normally appear. The contents
- of the NOTICE file are for informational purposes only and
- do not modify the License. You may add Your own attribution
- notices within Derivative Works that You distribute, alongside
- or as an addendum to the NOTICE text from the Work, provided
- that such additional attribution notices cannot be construed
- as modifying the License.
-
- You may add Your own copyright statement to Your modifications and
- may provide additional or different license terms and conditions
- for use, reproduction, or distribution of Your modifications, or
- for any such Derivative Works as a whole, provided Your use,
- reproduction, and distribution of the Work otherwise complies with
- the conditions stated in this License.
-
- 5. Submission of Contributions. Unless You explicitly state otherwise,
- any Contribution intentionally submitted for inclusion in the Work
- by You to the Licensor shall be under the terms and conditions of
- this License, without any additional terms or conditions.
- Notwithstanding the above, nothing herein shall supersede or modify
- the terms of any separate license agreement you may have executed
- with Licensor regarding such Contributions.
-
- 6. Trademarks. This License does not grant permission to use the trade
- names, trademarks, service marks, or product names of the Licensor,
- except as required for reasonable and customary use in describing the
- origin of the Work and reproducing the content of the NOTICE file.
-
- 7. Disclaimer of Warranty. Unless required by applicable law or
- agreed to in writing, Licensor provides the Work (and each
- Contributor provides its Contributions) on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
- implied, including, without limitation, any warranties or conditions
- of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
- PARTICULAR PURPOSE. You are solely responsible for determining the
- appropriateness of using or redistributing the Work and assume any
- risks associated with Your exercise of permissions under this License.
-
- 8. Limitation of Liability. In no event and under no legal theory,
- whether in tort (including negligence), contract, or otherwise,
- unless required by applicable law (such as deliberate and grossly
- negligent acts) or agreed to in writing, shall any Contributor be
- liable to You for damages, including any direct, indirect, special,
- incidental, or consequential damages of any character arising as a
- result of this License or out of the use or inability to use the
- Work (including but not limited to damages for loss of goodwill,
- work stoppage, computer failure or malfunction, or any and all
- other commercial damages or losses), even if such Contributor
- has been advised of the possibility of such damages.
-
- 9. Accepting Warranty or Additional Liability. While redistributing
- the Work or Derivative Works thereof, You may choose to offer,
- and charge a fee for, acceptance of support, warranty, indemnity,
- or other liability obligations and/or rights consistent with this
- License. However, in accepting such obligations, You may act only
- on Your own behalf and on Your sole responsibility, not on behalf
- of any other Contributor, and only if You agree to indemnify,
- defend, and hold each Contributor harmless for any liability
- incurred by, or claims asserted against, such Contributor by reason
- of your accepting any such warranty or additional liability.
-
- END OF TERMS AND CONDITIONS
-
- APPENDIX: How to apply the Apache License to your work.
-
- To apply the Apache License to your work, attach the following
- boilerplate notice, with the fields enclosed by brackets "[]"
- replaced with your own identifying information. (Don't include
- the brackets!) The text should be enclosed in the appropriate
- comment syntax for the file format. We also recommend that a
- file or class name and description of purpose be included on the
- same "printed page" as the copyright notice for easier
- identification within third-party archives.
-
- Copyright [yyyy] [name of copyright owner]
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
diff --git a/lib/licenses/libthrift-0.9.2.txt b/lib/licenses/jamm-0.3.2.txt
similarity index 100%
rename from lib/licenses/libthrift-0.9.2.txt
rename to lib/licenses/jamm-0.3.2.txt
diff --git a/lib/licenses/jcl-over-slf4j-1.7.25.txt b/lib/licenses/jcl-over-slf4j-1.7.25.txt
new file mode 100644
index 0000000..315bd49
--- /dev/null
+++ b/lib/licenses/jcl-over-slf4j-1.7.25.txt
@@ -0,0 +1,24 @@
+Copyright (c) 2004-2017 QOS.ch
+All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+
+
diff --git a/lib/licenses/jcl-over-slf4j-1.7.7.txt b/lib/licenses/jcl-over-slf4j-1.7.7.txt
deleted file mode 100644
index dbfc769..0000000
--- a/lib/licenses/jcl-over-slf4j-1.7.7.txt
+++ /dev/null
@@ -1,20 +0,0 @@
-Copyright (c) 2004-2008 QOS.ch
-All rights reserved.
-
-Permission is hereby granted, free of charge, to any person obtaining a
-copy of this software and associated documentation files (the "Software"),
-to deal in the Software without restriction, including without limitation
-the rights to use, copy, modify, merge, publish, distribute, sublicense,
-and/or sell copies of the Software, and to permit persons to whom the
-Software is furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
-THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-DEALINGS IN THE SOFTWARE.
diff --git a/lib/licenses/log4j-over-slf4j-1.7.25.txt b/lib/licenses/log4j-over-slf4j-1.7.25.txt
new file mode 100644
index 0000000..315bd49
--- /dev/null
+++ b/lib/licenses/log4j-over-slf4j-1.7.25.txt
@@ -0,0 +1,24 @@
+Copyright (c) 2004-2017 QOS.ch
+All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+
+
diff --git a/lib/licenses/log4j-over-slf4j-1.7.7.txt b/lib/licenses/log4j-over-slf4j-1.7.7.txt
deleted file mode 100644
index dbfc769..0000000
--- a/lib/licenses/log4j-over-slf4j-1.7.7.txt
+++ /dev/null
@@ -1,20 +0,0 @@
-Copyright (c) 2004-2008 QOS.ch
-All rights reserved.
-
-Permission is hereby granted, free of charge, to any person obtaining a
-copy of this software and associated documentation files (the "Software"),
-to deal in the Software without restriction, including without limitation
-the rights to use, copy, modify, merge, publish, distribute, sublicense,
-and/or sell copies of the Software, and to permit persons to whom the
-Software is furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
-THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-DEALINGS IN THE SOFTWARE.
diff --git a/lib/licenses/logback-classic-1.1.3.txt b/lib/licenses/logback-classic-1.2.3.txt
similarity index 100%
rename from lib/licenses/logback-classic-1.1.3.txt
rename to lib/licenses/logback-classic-1.2.3.txt
diff --git a/lib/licenses/logback-core-1.1.3.txt b/lib/licenses/logback-core-1.2.3.txt
similarity index 100%
rename from lib/licenses/logback-core-1.1.3.txt
rename to lib/licenses/logback-core-1.2.3.txt
diff --git a/lib/licenses/lz4-1.3.0.txt b/lib/licenses/lz4-1.4.0.txt
similarity index 100%
rename from lib/licenses/lz4-1.3.0.txt
rename to lib/licenses/lz4-1.4.0.txt
diff --git a/lib/licenses/guava-18.0.txt b/lib/licenses/netty-4.1.50.txt
similarity index 100%
rename from lib/licenses/guava-18.0.txt
rename to lib/licenses/netty-4.1.50.txt
diff --git a/lib/licenses/netty-all-4.0.44.Final.txt b/lib/licenses/netty-all-4.0.44.Final.txt
deleted file mode 100644
index d645695..0000000
--- a/lib/licenses/netty-all-4.0.44.Final.txt
+++ /dev/null
@@ -1,202 +0,0 @@
-
- Apache License
- Version 2.0, January 2004
- http://www.apache.org/licenses/
-
- TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
- 1. Definitions.
-
- "License" shall mean the terms and conditions for use, reproduction,
- and distribution as defined by Sections 1 through 9 of this document.
-
- "Licensor" shall mean the copyright owner or entity authorized by
- the copyright owner that is granting the License.
-
- "Legal Entity" shall mean the union of the acting entity and all
- other entities that control, are controlled by, or are under common
- control with that entity. For the purposes of this definition,
- "control" means (i) the power, direct or indirect, to cause the
- direction or management of such entity, whether by contract or
- otherwise, or (ii) ownership of fifty percent (50%) or more of the
- outstanding shares, or (iii) beneficial ownership of such entity.
-
- "You" (or "Your") shall mean an individual or Legal Entity
- exercising permissions granted by this License.
-
- "Source" form shall mean the preferred form for making modifications,
- including but not limited to software source code, documentation
- source, and configuration files.
-
- "Object" form shall mean any form resulting from mechanical
- transformation or translation of a Source form, including but
- not limited to compiled object code, generated documentation,
- and conversions to other media types.
-
- "Work" shall mean the work of authorship, whether in Source or
- Object form, made available under the License, as indicated by a
- copyright notice that is included in or attached to the work
- (an example is provided in the Appendix below).
-
- "Derivative Works" shall mean any work, whether in Source or Object
- form, that is based on (or derived from) the Work and for which the
- editorial revisions, annotations, elaborations, or other modifications
- represent, as a whole, an original work of authorship. For the purposes
- of this License, Derivative Works shall not include works that remain
- separable from, or merely link (or bind by name) to the interfaces of,
- the Work and Derivative Works thereof.
-
- "Contribution" shall mean any work of authorship, including
- the original version of the Work and any modifications or additions
- to that Work or Derivative Works thereof, that is intentionally
- submitted to Licensor for inclusion in the Work by the copyright owner
- or by an individual or Legal Entity authorized to submit on behalf of
- the copyright owner. For the purposes of this definition, "submitted"
- means any form of electronic, verbal, or written communication sent
- to the Licensor or its representatives, including but not limited to
- communication on electronic mailing lists, source code control systems,
- and issue tracking systems that are managed by, or on behalf of, the
- Licensor for the purpose of discussing and improving the Work, but
- excluding communication that is conspicuously marked or otherwise
- designated in writing by the copyright owner as "Not a Contribution."
-
- "Contributor" shall mean Licensor and any individual or Legal Entity
- on behalf of whom a Contribution has been received by Licensor and
- subsequently incorporated within the Work.
-
- 2. Grant of Copyright License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- copyright license to reproduce, prepare Derivative Works of,
- publicly display, publicly perform, sublicense, and distribute the
- Work and such Derivative Works in Source or Object form.
-
- 3. Grant of Patent License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- (except as stated in this section) patent license to make, have made,
- use, offer to sell, sell, import, and otherwise transfer the Work,
- where such license applies only to those patent claims licensable
- by such Contributor that are necessarily infringed by their
- Contribution(s) alone or by combination of their Contribution(s)
- with the Work to which such Contribution(s) was submitted. If You
- institute patent litigation against any entity (including a
- cross-claim or counterclaim in a lawsuit) alleging that the Work
- or a Contribution incorporated within the Work constitutes direct
- or contributory patent infringement, then any patent licenses
- granted to You under this License for that Work shall terminate
- as of the date such litigation is filed.
-
- 4. Redistribution. You may reproduce and distribute copies of the
- Work or Derivative Works thereof in any medium, with or without
- modifications, and in Source or Object form, provided that You
- meet the following conditions:
-
- (a) You must give any other recipients of the Work or
- Derivative Works a copy of this License; and
-
- (b) You must cause any modified files to carry prominent notices
- stating that You changed the files; and
-
- (c) You must retain, in the Source form of any Derivative Works
- that You distribute, all copyright, patent, trademark, and
- attribution notices from the Source form of the Work,
- excluding those notices that do not pertain to any part of
- the Derivative Works; and
-
- (d) If the Work includes a "NOTICE" text file as part of its
- distribution, then any Derivative Works that You distribute must
- include a readable copy of the attribution notices contained
- within such NOTICE file, excluding those notices that do not
- pertain to any part of the Derivative Works, in at least one
- of the following places: within a NOTICE text file distributed
- as part of the Derivative Works; within the Source form or
- documentation, if provided along with the Derivative Works; or,
- within a display generated by the Derivative Works, if and
- wherever such third-party notices normally appear. The contents
- of the NOTICE file are for informational purposes only and
- do not modify the License. You may add Your own attribution
- notices within Derivative Works that You distribute, alongside
- or as an addendum to the NOTICE text from the Work, provided
- that such additional attribution notices cannot be construed
- as modifying the License.
-
- You may add Your own copyright statement to Your modifications and
- may provide additional or different license terms and conditions
- for use, reproduction, or distribution of Your modifications, or
- for any such Derivative Works as a whole, provided Your use,
- reproduction, and distribution of the Work otherwise complies with
- the conditions stated in this License.
-
- 5. Submission of Contributions. Unless You explicitly state otherwise,
- any Contribution intentionally submitted for inclusion in the Work
- by You to the Licensor shall be under the terms and conditions of
- this License, without any additional terms or conditions.
- Notwithstanding the above, nothing herein shall supersede or modify
- the terms of any separate license agreement you may have executed
- with Licensor regarding such Contributions.
-
- 6. Trademarks. This License does not grant permission to use the trade
- names, trademarks, service marks, or product names of the Licensor,
- except as required for reasonable and customary use in describing the
- origin of the Work and reproducing the content of the NOTICE file.
-
- 7. Disclaimer of Warranty. Unless required by applicable law or
- agreed to in writing, Licensor provides the Work (and each
- Contributor provides its Contributions) on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
- implied, including, without limitation, any warranties or conditions
- of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
- PARTICULAR PURPOSE. You are solely responsible for determining the
- appropriateness of using or redistributing the Work and assume any
- risks associated with Your exercise of permissions under this License.
-
- 8. Limitation of Liability. In no event and under no legal theory,
- whether in tort (including negligence), contract, or otherwise,
- unless required by applicable law (such as deliberate and grossly
- negligent acts) or agreed to in writing, shall any Contributor be
- liable to You for damages, including any direct, indirect, special,
- incidental, or consequential damages of any character arising as a
- result of this License or out of the use or inability to use the
- Work (including but not limited to damages for loss of goodwill,
- work stoppage, computer failure or malfunction, or any and all
- other commercial damages or losses), even if such Contributor
- has been advised of the possibility of such damages.
-
- 9. Accepting Warranty or Additional Liability. While redistributing
- the Work or Derivative Works thereof, You may choose to offer,
- and charge a fee for, acceptance of support, warranty, indemnity,
- or other liability obligations and/or rights consistent with this
- License. However, in accepting such obligations, You may act only
- on Your own behalf and on Your sole responsibility, not on behalf
- of any other Contributor, and only if You agree to indemnify,
- defend, and hold each Contributor harmless for any liability
- incurred by, or claims asserted against, such Contributor by reason
- of your accepting any such warranty or additional liability.
-
- END OF TERMS AND CONDITIONS
-
- APPENDIX: How to apply the Apache License to your work.
-
- To apply the Apache License to your work, attach the following
- boilerplate notice, with the fields enclosed by brackets "[]"
- replaced with your own identifying information. (Don't include
- the brackets!) The text should be enclosed in the appropriate
- comment syntax for the file format. We also recommend that a
- file or class name and description of purpose be included on the
- same "printed page" as the copyright notice for easier
- identification within third-party archives.
-
- Copyright [yyyy] [name of copyright owner]
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
diff --git a/lib/licenses/netty-tcnative-2.0.31.txt b/lib/licenses/netty-tcnative-2.0.31.txt
new file mode 100644
index 0000000..261eeb9
--- /dev/null
+++ b/lib/licenses/netty-tcnative-2.0.31.txt
@@ -0,0 +1,201 @@
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright [yyyy] [name of copyright owner]
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
diff --git a/lib/licenses/ohc-0.4.4.txt b/lib/licenses/ohc-0.5.1.txt
similarity index 100%
rename from lib/licenses/ohc-0.4.4.txt
rename to lib/licenses/ohc-0.5.1.txt
diff --git a/lib/licenses/psjava-0.1.19.txt b/lib/licenses/psjava-0.1.19.txt
new file mode 100644
index 0000000..5acf4c1
--- /dev/null
+++ b/lib/licenses/psjava-0.1.19.txt
@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2013 psjava authors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
diff --git a/lib/licenses/slf4j-api-1.7.25.txt b/lib/licenses/slf4j-api-1.7.25.txt
new file mode 100644
index 0000000..315bd49
--- /dev/null
+++ b/lib/licenses/slf4j-api-1.7.25.txt
@@ -0,0 +1,24 @@
+Copyright (c) 2004-2017 QOS.ch
+All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+
+
diff --git a/lib/licenses/slf4j-api-1.7.7.txt b/lib/licenses/slf4j-api-1.7.7.txt
deleted file mode 100644
index dbfc769..0000000
--- a/lib/licenses/slf4j-api-1.7.7.txt
+++ /dev/null
@@ -1,20 +0,0 @@
-Copyright (c) 2004-2008 QOS.ch
-All rights reserved.
-
-Permission is hereby granted, free of charge, to any person obtaining a
-copy of this software and associated documentation files (the "Software"),
-to deal in the Software without restriction, including without limitation
-the rights to use, copy, modify, merge, publish, distribute, sublicense,
-and/or sell copies of the Software, and to permit persons to whom the
-Software is furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
-THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-DEALINGS IN THE SOFTWARE.
diff --git a/lib/licenses/snappy-java-1.1.1.7.txt b/lib/licenses/snappy-java-1.1.2.6.txt
similarity index 100%
rename from lib/licenses/snappy-java-1.1.1.7.txt
rename to lib/licenses/snappy-java-1.1.2.6.txt
diff --git a/lib/licenses/thrift-server-0.3.7.txt b/lib/licenses/thrift-server-0.3.7.txt
deleted file mode 100644
index d645695..0000000
--- a/lib/licenses/thrift-server-0.3.7.txt
+++ /dev/null
@@ -1,202 +0,0 @@
-
- Apache License
- Version 2.0, January 2004
- http://www.apache.org/licenses/
-
- TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
- 1. Definitions.
-
- "License" shall mean the terms and conditions for use, reproduction,
- and distribution as defined by Sections 1 through 9 of this document.
-
- "Licensor" shall mean the copyright owner or entity authorized by
- the copyright owner that is granting the License.
-
- "Legal Entity" shall mean the union of the acting entity and all
- other entities that control, are controlled by, or are under common
- control with that entity. For the purposes of this definition,
- "control" means (i) the power, direct or indirect, to cause the
- direction or management of such entity, whether by contract or
- otherwise, or (ii) ownership of fifty percent (50%) or more of the
- outstanding shares, or (iii) beneficial ownership of such entity.
-
- "You" (or "Your") shall mean an individual or Legal Entity
- exercising permissions granted by this License.
-
- "Source" form shall mean the preferred form for making modifications,
- including but not limited to software source code, documentation
- source, and configuration files.
-
- "Object" form shall mean any form resulting from mechanical
- transformation or translation of a Source form, including but
- not limited to compiled object code, generated documentation,
- and conversions to other media types.
-
- "Work" shall mean the work of authorship, whether in Source or
- Object form, made available under the License, as indicated by a
- copyright notice that is included in or attached to the work
- (an example is provided in the Appendix below).
-
- "Derivative Works" shall mean any work, whether in Source or Object
- form, that is based on (or derived from) the Work and for which the
- editorial revisions, annotations, elaborations, or other modifications
- represent, as a whole, an original work of authorship. For the purposes
- of this License, Derivative Works shall not include works that remain
- separable from, or merely link (or bind by name) to the interfaces of,
- the Work and Derivative Works thereof.
-
- "Contribution" shall mean any work of authorship, including
- the original version of the Work and any modifications or additions
- to that Work or Derivative Works thereof, that is intentionally
- submitted to Licensor for inclusion in the Work by the copyright owner
- or by an individual or Legal Entity authorized to submit on behalf of
- the copyright owner. For the purposes of this definition, "submitted"
- means any form of electronic, verbal, or written communication sent
- to the Licensor or its representatives, including but not limited to
- communication on electronic mailing lists, source code control systems,
- and issue tracking systems that are managed by, or on behalf of, the
- Licensor for the purpose of discussing and improving the Work, but
- excluding communication that is conspicuously marked or otherwise
- designated in writing by the copyright owner as "Not a Contribution."
-
- "Contributor" shall mean Licensor and any individual or Legal Entity
- on behalf of whom a Contribution has been received by Licensor and
- subsequently incorporated within the Work.
-
- 2. Grant of Copyright License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- copyright license to reproduce, prepare Derivative Works of,
- publicly display, publicly perform, sublicense, and distribute the
- Work and such Derivative Works in Source or Object form.
-
- 3. Grant of Patent License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- (except as stated in this section) patent license to make, have made,
- use, offer to sell, sell, import, and otherwise transfer the Work,
- where such license applies only to those patent claims licensable
- by such Contributor that are necessarily infringed by their
- Contribution(s) alone or by combination of their Contribution(s)
- with the Work to which such Contribution(s) was submitted. If You
- institute patent litigation against any entity (including a
- cross-claim or counterclaim in a lawsuit) alleging that the Work
- or a Contribution incorporated within the Work constitutes direct
- or contributory patent infringement, then any patent licenses
- granted to You under this License for that Work shall terminate
- as of the date such litigation is filed.
-
- 4. Redistribution. You may reproduce and distribute copies of the
- Work or Derivative Works thereof in any medium, with or without
- modifications, and in Source or Object form, provided that You
- meet the following conditions:
-
- (a) You must give any other recipients of the Work or
- Derivative Works a copy of this License; and
-
- (b) You must cause any modified files to carry prominent notices
- stating that You changed the files; and
-
- (c) You must retain, in the Source form of any Derivative Works
- that You distribute, all copyright, patent, trademark, and
- attribution notices from the Source form of the Work,
- excluding those notices that do not pertain to any part of
- the Derivative Works; and
-
- (d) If the Work includes a "NOTICE" text file as part of its
- distribution, then any Derivative Works that You distribute must
- include a readable copy of the attribution notices contained
- within such NOTICE file, excluding those notices that do not
- pertain to any part of the Derivative Works, in at least one
- of the following places: within a NOTICE text file distributed
- as part of the Derivative Works; within the Source form or
- documentation, if provided along with the Derivative Works; or,
- within a display generated by the Derivative Works, if and
- wherever such third-party notices normally appear. The contents
- of the NOTICE file are for informational purposes only and
- do not modify the License. You may add Your own attribution
- notices within Derivative Works that You distribute, alongside
- or as an addendum to the NOTICE text from the Work, provided
- that such additional attribution notices cannot be construed
- as modifying the License.
-
- You may add Your own copyright statement to Your modifications and
- may provide additional or different license terms and conditions
- for use, reproduction, or distribution of Your modifications, or
- for any such Derivative Works as a whole, provided Your use,
- reproduction, and distribution of the Work otherwise complies with
- the conditions stated in this License.
-
- 5. Submission of Contributions. Unless You explicitly state otherwise,
- any Contribution intentionally submitted for inclusion in the Work
- by You to the Licensor shall be under the terms and conditions of
- this License, without any additional terms or conditions.
- Notwithstanding the above, nothing herein shall supersede or modify
- the terms of any separate license agreement you may have executed
- with Licensor regarding such Contributions.
-
- 6. Trademarks. This License does not grant permission to use the trade
- names, trademarks, service marks, or product names of the Licensor,
- except as required for reasonable and customary use in describing the
- origin of the Work and reproducing the content of the NOTICE file.
-
- 7. Disclaimer of Warranty. Unless required by applicable law or
- agreed to in writing, Licensor provides the Work (and each
- Contributor provides its Contributions) on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
- implied, including, without limitation, any warranties or conditions
- of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
- PARTICULAR PURPOSE. You are solely responsible for determining the
- appropriateness of using or redistributing the Work and assume any
- risks associated with Your exercise of permissions under this License.
-
- 8. Limitation of Liability. In no event and under no legal theory,
- whether in tort (including negligence), contract, or otherwise,
- unless required by applicable law (such as deliberate and grossly
- negligent acts) or agreed to in writing, shall any Contributor be
- liable to You for damages, including any direct, indirect, special,
- incidental, or consequential damages of any character arising as a
- result of this License or out of the use or inability to use the
- Work (including but not limited to damages for loss of goodwill,
- work stoppage, computer failure or malfunction, or any and all
- other commercial damages or losses), even if such Contributor
- has been advised of the possibility of such damages.
-
- 9. Accepting Warranty or Additional Liability. While redistributing
- the Work or Derivative Works thereof, You may choose to offer,
- and charge a fee for, acceptance of support, warranty, indemnity,
- or other liability obligations and/or rights consistent with this
- License. However, in accepting such obligations, You may act only
- on Your own behalf and on Your sole responsibility, not on behalf
- of any other Contributor, and only if You agree to indemnify,
- defend, and hold each Contributor harmless for any liability
- incurred by, or claims asserted against, such Contributor by reason
- of your accepting any such warranty or additional liability.
-
- END OF TERMS AND CONDITIONS
-
- APPENDIX: How to apply the Apache License to your work.
-
- To apply the Apache License to your work, attach the following
- boilerplate notice, with the fields enclosed by brackets "[]"
- replaced with your own identifying information. (Don't include
- the brackets!) The text should be enclosed in the appropriate
- comment syntax for the file format. We also recommend that a
- file or class name and description of purpose be included on the
- same "printed page" as the copyright notice for easier
- identification within third-party archives.
-
- Copyright [yyyy] [name of copyright owner]
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
diff --git a/lib/licenses/zstd-jni-1.3.8-3.txt b/lib/licenses/zstd-jni-1.3.8-3.txt
new file mode 100644
index 0000000..66abb8a
--- /dev/null
+++ b/lib/licenses/zstd-jni-1.3.8-3.txt
@@ -0,0 +1,26 @@
+Zstd-jni: JNI bindings to Zstd Library
+
+Copyright (c) 2015-present, Luben Karavelov/ All rights reserved.
+
+BSD License
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+ list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice, this
+ list of conditions and the following disclaimer in the documentation and/or
+ other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/lib/log4j-over-slf4j-1.7.25.jar b/lib/log4j-over-slf4j-1.7.25.jar
new file mode 100644
index 0000000..ba241a4
--- /dev/null
+++ b/lib/log4j-over-slf4j-1.7.25.jar
Binary files differ
diff --git a/lib/log4j-over-slf4j-1.7.7.jar b/lib/log4j-over-slf4j-1.7.7.jar
deleted file mode 100644
index d2a102e..0000000
--- a/lib/log4j-over-slf4j-1.7.7.jar
+++ /dev/null
Binary files differ
diff --git a/lib/logback-classic-1.1.3.jar b/lib/logback-classic-1.1.3.jar
deleted file mode 100644
index 2aa10a5..0000000
--- a/lib/logback-classic-1.1.3.jar
+++ /dev/null
Binary files differ
diff --git a/lib/logback-classic-1.2.3.jar b/lib/logback-classic-1.2.3.jar
new file mode 100644
index 0000000..bed00c0
--- /dev/null
+++ b/lib/logback-classic-1.2.3.jar
Binary files differ
diff --git a/lib/logback-core-1.1.3.jar b/lib/logback-core-1.1.3.jar
deleted file mode 100644
index 996b722..0000000
--- a/lib/logback-core-1.1.3.jar
+++ /dev/null
Binary files differ
diff --git a/lib/logback-core-1.2.3.jar b/lib/logback-core-1.2.3.jar
new file mode 100644
index 0000000..487b395
--- /dev/null
+++ b/lib/logback-core-1.2.3.jar
Binary files differ
diff --git a/lib/lz4-1.3.0.jar b/lib/lz4-1.3.0.jar
deleted file mode 100644
index 0fb0109..0000000
--- a/lib/lz4-1.3.0.jar
+++ /dev/null
Binary files differ
diff --git a/lib/lz4-java-1.7.1.jar b/lib/lz4-java-1.7.1.jar
new file mode 100644
index 0000000..95f57ca
--- /dev/null
+++ b/lib/lz4-java-1.7.1.jar
Binary files differ
diff --git a/lib/mxdump-0.14.jar b/lib/mxdump-0.14.jar
new file mode 100644
index 0000000..a2b177c
--- /dev/null
+++ b/lib/mxdump-0.14.jar
Binary files differ
diff --git a/lib/netty-all-4.0.44.Final.jar b/lib/netty-all-4.0.44.Final.jar
deleted file mode 100644
index 9c5bda5..0000000
--- a/lib/netty-all-4.0.44.Final.jar
+++ /dev/null
Binary files differ
diff --git a/lib/netty-all-4.1.50.Final.jar b/lib/netty-all-4.1.50.Final.jar
new file mode 100644
index 0000000..f8b1557
--- /dev/null
+++ b/lib/netty-all-4.1.50.Final.jar
Binary files differ
diff --git a/lib/netty-tcnative-boringssl-static-2.0.31.Final.jar b/lib/netty-tcnative-boringssl-static-2.0.31.Final.jar
new file mode 100644
index 0000000..582c582
--- /dev/null
+++ b/lib/netty-tcnative-boringssl-static-2.0.31.Final.jar
Binary files differ
diff --git a/lib/ohc-core-0.4.4.jar b/lib/ohc-core-0.4.4.jar
deleted file mode 100644
index 6d0f558..0000000
--- a/lib/ohc-core-0.4.4.jar
+++ /dev/null
Binary files differ
diff --git a/lib/ohc-core-0.5.1.jar b/lib/ohc-core-0.5.1.jar
new file mode 100644
index 0000000..20f535f
--- /dev/null
+++ b/lib/ohc-core-0.5.1.jar
Binary files differ
diff --git a/lib/ohc-core-j8-0.4.4.jar b/lib/ohc-core-j8-0.4.4.jar
deleted file mode 100644
index f0f9452..0000000
--- a/lib/ohc-core-j8-0.4.4.jar
+++ /dev/null
Binary files differ
diff --git a/lib/ohc-core-j8-0.5.1.jar b/lib/ohc-core-j8-0.5.1.jar
new file mode 100644
index 0000000..566cfb2
--- /dev/null
+++ b/lib/ohc-core-j8-0.5.1.jar
Binary files differ
diff --git a/lib/psjava-0.1.19.jar b/lib/psjava-0.1.19.jar
new file mode 100644
index 0000000..6652b95
--- /dev/null
+++ b/lib/psjava-0.1.19.jar
Binary files differ
diff --git a/lib/sigar-bin/libsigar-ppc64le-linux.so b/lib/sigar-bin/libsigar-ppc64le-linux.so
new file mode 100644
index 0000000..717ae9a
--- /dev/null
+++ b/lib/sigar-bin/libsigar-ppc64le-linux.so
Binary files differ
diff --git a/lib/six-1.12.0-py2.py3-none-any.zip b/lib/six-1.12.0-py2.py3-none-any.zip
new file mode 100644
index 0000000..78ba903
--- /dev/null
+++ b/lib/six-1.12.0-py2.py3-none-any.zip
Binary files differ
diff --git a/lib/six-1.7.3-py2.py3-none-any.zip b/lib/six-1.7.3-py2.py3-none-any.zip
deleted file mode 100644
index e077898..0000000
--- a/lib/six-1.7.3-py2.py3-none-any.zip
+++ /dev/null
Binary files differ
diff --git a/lib/sjk-cli-0.14.jar b/lib/sjk-cli-0.14.jar
new file mode 100644
index 0000000..bdc7f86
--- /dev/null
+++ b/lib/sjk-cli-0.14.jar
Binary files differ
diff --git a/lib/sjk-core-0.14.jar b/lib/sjk-core-0.14.jar
new file mode 100644
index 0000000..28fe34f
--- /dev/null
+++ b/lib/sjk-core-0.14.jar
Binary files differ
diff --git a/lib/sjk-json-0.14.jar b/lib/sjk-json-0.14.jar
new file mode 100644
index 0000000..82bf43b
--- /dev/null
+++ b/lib/sjk-json-0.14.jar
Binary files differ
diff --git a/lib/sjk-stacktrace-0.14.jar b/lib/sjk-stacktrace-0.14.jar
new file mode 100644
index 0000000..b6c9242
--- /dev/null
+++ b/lib/sjk-stacktrace-0.14.jar
Binary files differ
diff --git a/lib/slf4j-api-1.7.25.jar b/lib/slf4j-api-1.7.25.jar
new file mode 100644
index 0000000..0143c09
--- /dev/null
+++ b/lib/slf4j-api-1.7.25.jar
Binary files differ
diff --git a/lib/slf4j-api-1.7.7.jar b/lib/slf4j-api-1.7.7.jar
deleted file mode 100644
index bebabd9..0000000
--- a/lib/slf4j-api-1.7.7.jar
+++ /dev/null
Binary files differ
diff --git a/lib/snappy-java-1.1.1.7.jar b/lib/snappy-java-1.1.1.7.jar
deleted file mode 100644
index 2bbd1fc..0000000
--- a/lib/snappy-java-1.1.1.7.jar
+++ /dev/null
Binary files differ
diff --git a/lib/snappy-java-1.1.2.6.jar b/lib/snappy-java-1.1.2.6.jar
new file mode 100644
index 0000000..5c354d1
--- /dev/null
+++ b/lib/snappy-java-1.1.2.6.jar
Binary files differ
diff --git a/lib/thrift-server-0.3.7.jar b/lib/thrift-server-0.3.7.jar
deleted file mode 100644
index 1231618..0000000
--- a/lib/thrift-server-0.3.7.jar
+++ /dev/null
Binary files differ
diff --git a/lib/zstd-jni-1.3.8-5.jar b/lib/zstd-jni-1.3.8-5.jar
new file mode 100644
index 0000000..f68ce66
--- /dev/null
+++ b/lib/zstd-jni-1.3.8-5.jar
Binary files differ
diff --git a/pylib/Dockerfile.ubuntu.py2 b/pylib/Dockerfile.ubuntu.py2
new file mode 100644
index 0000000..93016f0
--- /dev/null
+++ b/pylib/Dockerfile.ubuntu.py2
@@ -0,0 +1,2 @@
+FROM ubuntu:bionic
+RUN apt-get update && apt-get install -y python-minimal
diff --git a/pylib/Dockerfile.ubuntu.py3 b/pylib/Dockerfile.ubuntu.py3
new file mode 100644
index 0000000..7bbb715
--- /dev/null
+++ b/pylib/Dockerfile.ubuntu.py3
@@ -0,0 +1,2 @@
+FROM ubuntu:bionic
+RUN apt-get update && apt-get install -y python3-minimal && update-alternatives --install /usr/bin/python python /usr/bin/python3.6 1
diff --git a/pylib/Dockerfile.ubuntu.py37 b/pylib/Dockerfile.ubuntu.py37
new file mode 100644
index 0000000..7b08508
--- /dev/null
+++ b/pylib/Dockerfile.ubuntu.py37
@@ -0,0 +1,2 @@
+FROM ubuntu:bionic
+RUN apt-get update && apt-get install -y python3.7-minimal && update-alternatives --install /usr/bin/python python /usr/bin/python3.7 1
diff --git a/pylib/Dockerfile.ubuntu.py38 b/pylib/Dockerfile.ubuntu.py38
new file mode 100644
index 0000000..74f9e3c
--- /dev/null
+++ b/pylib/Dockerfile.ubuntu.py38
@@ -0,0 +1,2 @@
+FROM ubuntu:bionic
+RUN apt-get update && apt-get install -y python3.8-minimal && update-alternatives --install /usr/bin/python python /usr/bin/python3.8 1
diff --git a/pylib/README.asc b/pylib/README.asc
new file mode 100644
index 0000000..b0b6c7d
--- /dev/null
+++ b/pylib/README.asc
@@ -0,0 +1,35 @@
+== Overview
+
+This directory contains code primarily for cqlsh. cqlsh uses cqlshlib in this directory.
+Currently, cqlshlib supports Python 2 as well as Python 3. Support for Python 3 is relatively
+new.
+
+== Requirements
+. Python 3 and 2.7 (for cqlsh)
+. virtualenv
+. Docker (optional)
+
+== Running tests
+
+In order to run tests for cqlshlib, run cassandra-cqlsh-tests.sh in this directory. It will
+automatically setup a virtualenv with the appropriate version of Python and run tests inside it.
+
+There are Dockerfiles that can be used to test whether cqlsh works with a default, barebones
+Python installation. Assuming Cassandra's source is checked out at `$CASSANDRA_DIR`. To test, first
+build the Docker image containing the barebones Python installation -
+
+ $ docker build . --file Dockerfile.ubuntu.py3 -t ubuntu-lts-py3
+
+Next, run cqlsh inside the newly built image -
+
+ $ docker run -v $CASSANDRA_DIR:/code -it ubuntu-lts-py3:latest /code/bin/cqlsh host.docker.internal
+
+If `host.docker.internal` isn't supported, then you can use `--net="host"` with `docker run`:
+
+ $ docker run --net="host" -v $CASSANDRA_DIR:/code -it ubuntu-lts-py3:latest /code/bin/cqlsh
+
+This will try to spawn a cqlsh instance inside the Docker container running Ubuntu LTS (18.04)
+with minimal Python installation. It will try to connect to the Cassandra instance running on the
+Docker host at port 9042. If you have Cassandra running elsewhere, replace host.docker.internal
+with the IP / hostname as usual. Please ensure that the IP / host is accessible from _within_ the
+Docker container.
\ No newline at end of file
diff --git a/pylib/cassandra-cqlsh-tests.sh b/pylib/cassandra-cqlsh-tests.sh
index 8174636..2013507 100755
--- a/pylib/cassandra-cqlsh-tests.sh
+++ b/pylib/cassandra-cqlsh-tests.sh
@@ -7,12 +7,22 @@
################################
WORKSPACE=$1
+PYTHON_VERSION=$2
if [ "${WORKSPACE}" = "" ]; then
echo "Specify Cassandra source directory"
exit
fi
+if [ "${PYTHON_VERSION}" = "" ]; then
+ PYTHON_VERSION=python3
+fi
+
+if [ "${PYTHON_VERSION}" != "python3" -a "${PYTHON_VERSION}" != "python2" ]; then
+ echo "Specify Python version python3 or python2"
+ exit
+fi
+
export PYTHONIOENCODING="utf-8"
export PYTHONUNBUFFERED=true
export CASS_DRIVER_NO_EXTENSIONS=true
@@ -22,7 +32,17 @@
export CCM_CONFIG_DIR=${WORKSPACE}/.ccm
export NUM_TOKENS="32"
export CASSANDRA_DIR=${WORKSPACE}
-export TESTSUITE_NAME="cqlshlib.python2.jdk8"
+export TESTSUITE_NAME="cqlshlib.${PYTHON_VERSION}"
+
+if [ -z "$CASSANDRA_USE_JDK11" ]; then
+ export CASSANDRA_USE_JDK11=false
+fi
+
+if [ "$CASSANDRA_USE_JDK11" = true ] ; then
+ TESTSUITE_NAME="${TESTSUITE_NAME}.jdk11"
+else
+ TESTSUITE_NAME="${TESTSUITE_NAME}.jdk8"
+fi
# Loop to prevent failure due to maven-ant-tasks not downloading a jar..
for x in $(seq 1 3); do
@@ -40,8 +60,9 @@
# Set up venv with dtest dependencies
set -e # enable immediate exit if venv setup fails
-virtualenv --python=python2 --no-site-packages venv
+virtualenv --python=$PYTHON_VERSION venv
source venv/bin/activate
+
pip install -r ${CASSANDRA_DIR}/pylib/requirements.txt
pip freeze
@@ -63,10 +84,11 @@
ccm remove test || true # in case an old ccm cluster is left behind
ccm create test -n 1 --install-dir=${CASSANDRA_DIR}
ccm updateconf "enable_user_defined_functions: true"
+ccm updateconf "enable_scripted_user_defined_functions: true"
version_from_build=$(ccm node1 versionfrombuild)
export pre_or_post_cdc=$(python -c """from distutils.version import LooseVersion
-print \"postcdc\" if LooseVersion(\"${version_from_build}\") >= \"3.8\" else \"precdc\"
+print (\"postcdc\" if LooseVersion(\"${version_from_build}\") >= \"3.8\" else \"precdc\")
""")
case "${pre_or_post_cdc}" in
postcdc)
diff --git a/pylib/cqlshlib/copyutil.py b/pylib/cqlshlib/copyutil.py
index b91bb76..169a6e0 100644
--- a/pylib/cqlshlib/copyutil.py
+++ b/pylib/cqlshlib/copyutil.py
@@ -16,7 +16,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-import ConfigParser
+from __future__ import unicode_literals
import csv
import datetime
import json
@@ -26,6 +26,8 @@
import platform
import random
import re
+import signal
+import six
import struct
import sys
import threading
@@ -36,25 +38,30 @@
from calendar import timegm
from collections import defaultdict, namedtuple
from decimal import Decimal
-from Queue import Queue
from random import randint
-from StringIO import StringIO
+from io import BytesIO, StringIO
from select import select
from uuid import UUID
-from util import profile_on, profile_off
+from .util import profile_on, profile_off
+
+from six import ensure_str, ensure_text
+from six.moves import configparser
+from six.moves import range
+from six.moves.queue import Queue
from cassandra import OperationTimedOut
from cassandra.cluster import Cluster, DefaultConnection
-from cassandra.cqltypes import ReversedType, UserType
+from cassandra.cqltypes import ReversedType, UserType, BytesType
from cassandra.metadata import protect_name, protect_names, protect_value
from cassandra.policies import RetryPolicy, WhiteListRoundRobinPolicy, DCAwareRoundRobinPolicy, FallthroughRetryPolicy
from cassandra.query import BatchStatement, BatchType, SimpleStatement, tuple_factory
from cassandra.util import Date, Time
+from cqlshlib.util import profile_on, profile_off
-from cql3handling import CqlRuleSet
-from displaying import NO_COLOR_MAP
-from formatting import format_value_default, CqlType, DateTimeFormat, EMPTY, get_formatter, BlobType
-from sslhandling import ssl_settings
+from cqlshlib.cql3handling import CqlRuleSet
+from cqlshlib.displaying import NO_COLOR_MAP
+from cqlshlib.formatting import format_value_default, CqlType, DateTimeFormat, EMPTY, get_formatter, BlobType
+from cqlshlib.sslhandling import ssl_settings
PROFILE_ON = False
STRACE_ON = False
@@ -79,7 +86,7 @@
def printmsg(msg, eol='\n', encoding='utf8'):
- sys.stdout.write(msg.encode(encoding))
+ sys.stdout.write(msg)
sys.stdout.write(eol)
sys.stdout.flush()
@@ -138,8 +145,8 @@
try:
msg = self.pending_messages.get()
self.pipe.send(msg)
- except Exception, e:
- printmsg('%s: %s' % (e.__class__.__name__, e.message))
+ except Exception as e:
+ printmsg('%s: %s' % (e.__class__.__name__, e.message if hasattr(e, 'message') else str(e)))
feeding_thread = threading.Thread(target=feed)
feeding_thread.setDaemon(True)
@@ -160,7 +167,7 @@
A group of one way channels for sending messages.
"""
def __init__(self, num_channels):
- self.pipes = [OneWayPipe() for _ in xrange(num_channels)]
+ self.pipes = [OneWayPipe() for _ in range(num_channels)]
self.channels = [SendingChannel(p) for p in self.pipes]
self.num_channels = num_channels
@@ -177,7 +184,7 @@
A group of one way channels for receiving messages.
"""
def __init__(self, num_channels):
- self.pipes = [OneWayPipe() for _ in xrange(num_channels)]
+ self.pipes = [OneWayPipe() for _ in range(num_channels)]
self.channels = [ReceivingChannel(p) for p in self.pipes]
self._readers = [p.reader for p in self.pipes]
self._rlocks = [p.rlock for p in self.pipes]
@@ -276,7 +283,7 @@
if not os.path.isfile(config_file):
return opts
- configs = ConfigParser.RawConfigParser()
+ configs = configparser.RawConfigParser()
configs.readfp(open(config_file))
ret = dict()
@@ -305,8 +312,8 @@
"""
Convert all option values to valid string literals unless they are path names
"""
- return dict([(k, v.decode('string_escape') if k not in ['errfile', 'ratefile'] else v)
- for k, v, in opts.iteritems()])
+ return dict([(k, v if k not in ['errfile', 'ratefile'] else v)
+ for k, v, in opts.items()])
def parse_options(self, opts, direction):
"""
@@ -319,9 +326,9 @@
opts = self.clean_options(self.maybe_read_config_file(opts, direction))
dialect_options = dict()
- dialect_options['quotechar'] = opts.pop('quote', '"')
- dialect_options['escapechar'] = opts.pop('escape', '\\')
- dialect_options['delimiter'] = opts.pop('delimiter', ',')
+ dialect_options['quotechar'] = ensure_str(opts.pop('quote', '"'))
+ dialect_options['escapechar'] = ensure_str(opts.pop('escape', '\\'))
+ dialect_options['delimiter'] = ensure_str(opts.pop('delimiter', ','))
if dialect_options['quotechar'] == dialect_options['escapechar']:
dialect_options['doublequote'] = True
del dialect_options['escapechar']
@@ -329,7 +336,7 @@
dialect_options['doublequote'] = False
copy_options = dict()
- copy_options['nullval'] = opts.pop('null', '')
+ copy_options['nullval'] = ensure_str(opts.pop('null', ''))
copy_options['header'] = bool(opts.pop('header', '').lower() == 'true')
copy_options['encoding'] = opts.pop('encoding', 'utf8')
copy_options['maxrequests'] = int(opts.pop('maxrequests', 6))
@@ -351,7 +358,7 @@
copy_options['consistencylevel'] = shell.consistency_level
copy_options['decimalsep'] = opts.pop('decimalsep', '.')
copy_options['thousandssep'] = opts.pop('thousandssep', '')
- copy_options['boolstyle'] = [s.strip() for s in opts.pop('boolstyle', 'True, False').split(',')]
+ copy_options['boolstyle'] = [ensure_str(s.strip()) for s in opts.pop('boolstyle', 'True, False').split(',')]
copy_options['numprocesses'] = int(opts.pop('numprocesses', self.get_num_processes(16)))
copy_options['begintoken'] = opts.pop('begintoken', '')
copy_options['endtoken'] = opts.pop('endtoken', '')
@@ -494,7 +501,9 @@
cql_version=shell.conn.cql_version,
config_file=self.config_file,
protocol_version=self.protocol_version,
- debug=shell.debug
+ debug=shell.debug,
+ coverage=shell.coverage,
+ coveragerc_path=shell.coveragerc_path
)
def validate_columns(self):
@@ -534,7 +543,7 @@
self.columns = columns
self.options = options
self.header = options.copy['header']
- self.max_output_size = long(options.copy['maxoutputsize'])
+ self.max_output_size = int(options.copy['maxoutputsize'])
self.current_dest = None
self.num_files = 0
@@ -577,10 +586,10 @@
return CsvDest(output=sys.stdout, close=False)
else:
try:
- ret = CsvDest(output=open(source_name, 'wb'), close=True)
+ ret = CsvDest(output=open(source_name, 'w'), close=True)
self.num_files += 1
return ret
- except IOError, e:
+ except IOError as e:
self.shell.printerr("Can't open %r for writing: %s" % (source_name, e))
return None
@@ -604,7 +613,7 @@
if (self.num_written + num) > self.max_output_size:
num_remaining = self.max_output_size - self.num_written
last_switch = 0
- for i, row in enumerate(filter(None, data.split(os.linesep))):
+ for i, row in enumerate([_f for _f in data.split(os.linesep) if _f]):
if i == num_remaining:
self._next_dest()
last_switch = i
@@ -625,8 +634,8 @@
CopyTask.__init__(self, shell, ks, table, columns, fname, opts, protocol_version, config_file, 'to')
options = self.options
- self.begin_token = long(options.copy['begintoken']) if options.copy['begintoken'] else None
- self.end_token = long(options.copy['endtoken']) if options.copy['endtoken'] else None
+ self.begin_token = int(options.copy['begintoken']) if options.copy['begintoken'] else None
+ self.end_token = int(options.copy['endtoken']) if options.copy['endtoken'] else None
self.writer = ExportWriter(fname, shell, columns, options)
def run(self):
@@ -637,7 +646,7 @@
shell = self.shell
if self.options.unrecognized:
- shell.printerr('Unrecognized COPY TO options: %s' % ', '.join(self.options.unrecognized.keys()))
+ shell.printerr('Unrecognized COPY TO options: %s' % ', '.join(list(self.options.unrecognized.keys())))
return
if not self.validate_columns():
@@ -650,11 +659,11 @@
if not self.writer.open():
return 0
- columns = u"[" + u", ".join(self.columns) + u"]"
- self.printmsg(u"\nStarting copy of %s.%s with columns %s." % (self.ks, self.table, columns), encoding=self.encoding)
+ columns = "[" + ", ".join(self.columns) + "]"
+ self.printmsg("\nStarting copy of %s.%s with columns %s." % (self.ks, self.table, columns), encoding=self.encoding)
params = self.make_params()
- for i in xrange(self.num_processes):
+ for i in range(self.num_processes):
self.processes.append(ExportProcess(self.update_params(params, i)))
self.start_processes()
@@ -695,15 +704,15 @@
"""
ret = (prev, curr)
if begin_token:
- if ret[1] < begin_token:
+ if curr < begin_token:
return None
- elif ret[0] < begin_token:
- ret = (begin_token, ret[1])
+ elif (prev is None) or (prev < begin_token):
+ ret = (begin_token, curr)
if end_token:
- if ret[0] > end_token:
+ if (ret[0] is not None) and (ret[0] > end_token):
return None
- elif ret[1] > end_token:
+ elif (curr is not None) and (curr > end_token):
ret = (ret[0], end_token)
return ret
@@ -730,7 +739,7 @@
ranges[(begin_token, end_token)] = make_range_data()
return ranges
- ring = shell.get_ring(self.ks).items()
+ ring = list(shell.get_ring(self.ks).items())
ring.sort()
if not ring:
@@ -760,6 +769,9 @@
# For the last ring interval we query the same replicas that hold the first token in the ring
if previous is not None and (not end_token or previous < end_token):
ranges[(previous, end_token)] = first_range_data
+ elif previous is None and (not end_token or previous < end_token):
+ previous = begin_token if begin_token else min_token
+ ranges[(previous, end_token)] = first_range_data
if not ranges:
shell.printerr('Found no ranges to query, check begin and end tokens: %s - %s' % (begin_token, end_token))
@@ -809,7 +821,7 @@
total_requests = len(ranges)
max_attempts = self.options.copy['maxattempts']
- self.send_work(ranges, ranges.keys())
+ self.send_work(ranges, list(ranges.keys()))
num_processes = len(processes)
succeeded = 0
@@ -881,8 +893,8 @@
"""
def make_source(fname):
try:
- return open(fname, 'rb')
- except IOError, e:
+ return open(fname, 'r')
+ except IOError as e:
raise IOError("Can't open %r for reading: %s" % (fname, e))
for path in paths.split(','):
@@ -913,14 +925,14 @@
self.close_current_source()
while self.current_source is None:
try:
- self.current_source = self.sources.next()
+ self.current_source = next(self.sources)
if self.current_source:
self.num_sources += 1
except StopIteration:
return False
if self.header:
- self.current_source.next()
+ next(self.current_source)
return True
@@ -939,9 +951,9 @@
return []
rows = []
- for i in xrange(min(max_rows, self.chunk_size)):
+ for i in range(min(max_rows, self.chunk_size)):
try:
- row = self.current_source.next()
+ row = next(self.current_source)
self.num_read += 1
if 0 <= self.max_rows < self.num_read:
@@ -955,7 +967,7 @@
self.next_source()
break
- return filter(None, rows)
+ return [_f for _f in rows if _f]
class PipeReader(object):
@@ -977,7 +989,7 @@
def read_rows(self, max_rows):
rows = []
- for i in xrange(min(max_rows, self.chunk_size)):
+ for i in range(min(max_rows, self.chunk_size)):
row = self.inpipe.recv()
if row is None:
self.exhausted = True
@@ -1143,14 +1155,14 @@
shell = self.shell
if self.options.unrecognized:
- shell.printerr('Unrecognized COPY FROM options: %s' % ', '.join(self.options.unrecognized.keys()))
+ shell.printerr('Unrecognized COPY FROM options: %s' % ', '.join(list(self.options.unrecognized.keys())))
return
if not self.validate_columns():
return 0
- columns = u"[" + u", ".join(self.valid_columns) + u"]"
- self.printmsg(u"\nStarting copy of %s.%s with columns %s." % (self.ks, self.table, columns), encoding=self.encoding)
+ columns = "[" + ", ".join(self.valid_columns) + "]"
+ self.printmsg("\nStarting copy of %s.%s with columns %s." % (self.ks, self.table, columns), encoding=self.encoding)
try:
params = self.make_params()
@@ -1172,8 +1184,8 @@
if pr:
profile_off(pr, file_name='parent_profile_%d.txt' % (os.getpid(),))
- except Exception, exc:
- shell.printerr(unicode(exc))
+ except Exception as exc:
+ shell.printerr(str(exc))
if shell.debug:
traceback.print_exc()
return 0
@@ -1195,7 +1207,7 @@
self.outmsg.channels[-1].send(None)
if shell.tty:
- print
+ print()
def import_records(self):
"""
@@ -1280,7 +1292,7 @@
A process that reads from import sources and sends chunks to worker processes.
"""
def __init__(self, inpipe, outpipe, worker_pipes, fname, options, parent_cluster):
- mp.Process.__init__(self, target=self.run)
+ super(FeedingProcess, self).__init__(target=self.run)
self.inpipe = inpipe
self.outpipe = outpipe
self.worker_pipes = worker_pipes
@@ -1329,8 +1341,8 @@
reader = self.reader
try:
reader.start()
- except IOError, exc:
- self.outmsg.send(ImportTaskError(exc.__class__.__name__, exc.message))
+ except IOError as exc:
+ self.outmsg.send(ImportTaskError(exc.__class__.__name__, exc.message if hasattr(exc, 'message') else str(exc)))
channels = self.worker_channels
max_pending_chunks = self.max_pending_chunks
@@ -1338,7 +1350,7 @@
failed_attempts = 0
while not reader.exhausted:
- channels_eligible = filter(lambda c: c.num_pending() < max_pending_chunks, channels)
+ channels_eligible = [c for c in channels if c.num_pending() < max_pending_chunks]
if not channels_eligible:
failed_attempts += 1
delay = randint(1, pow(2, failed_attempts))
@@ -1358,8 +1370,8 @@
rows = reader.read_rows(max_rows)
if rows:
sent += self.send_chunk(ch, rows)
- except Exception, exc:
- self.outmsg.send(ImportTaskError(exc.__class__.__name__, exc.message))
+ except Exception as exc:
+ self.outmsg.send(ImportTaskError(exc.__class__.__name__, exc.message if hasattr(exc, 'message') else str(exc)))
if reader.exhausted:
break
@@ -1392,7 +1404,7 @@
"""
def __init__(self, params, target):
- mp.Process.__init__(self, target=target)
+ super(ChildProcess, self).__init__(target=target)
self.inpipe = params['inpipe']
self.outpipe = params['outpipe']
self.inmsg = None # must be initialized after fork on Windows
@@ -1425,6 +1437,12 @@
self.test_failures = json.loads(os.environ.get('CQLSH_COPY_TEST_FAILURES', ''))
else:
self.test_failures = None
+ # attributes for coverage
+ self.coverage = params['coverage']
+ self.coveragerc_path = params['coveragerc_path']
+ self.coverage_collection = None
+ self.sigterm_handler = None
+ self.sighup_handler = None
def on_fork(self):
"""
@@ -1442,6 +1460,31 @@
self.inmsg.close()
self.outmsg.close()
+ def start_coverage(self):
+ import coverage
+ self.coverage_collection = coverage.Coverage(config_file=self.coveragerc_path)
+ self.coverage_collection.start()
+
+ # save current handlers for SIGTERM and SIGHUP
+ self.sigterm_handler = signal.getsignal(signal.SIGTERM)
+ self.sighup_handler = signal.getsignal(signal.SIGTERM)
+
+ def handle_sigterm():
+ self.stop_coverage()
+ self.close()
+ self.terminate()
+
+ # set custom handler for SIGHUP and SIGTERM
+ # needed to make sure coverage data is saved
+ signal.signal(signal.SIGTERM, handle_sigterm)
+ signal.signal(signal.SIGHUP, handle_sigterm)
+
+ def stop_coverage(self):
+ self.coverage_collection.stop()
+ self.coverage_collection.save()
+ signal.signal(signal.SIGTERM, self.sigterm_handler)
+ signal.signal(signal.SIGHUP, self.sighup_handler)
+
class ExpBackoffRetryPolicy(RetryPolicy):
"""
@@ -1547,9 +1590,13 @@
self.options = options
def run(self):
+ if self.coverage:
+ self.start_coverage()
try:
self.inner_run()
finally:
+ if self.coverage:
+ self.stop_coverage()
self.close()
def inner_run(self):
@@ -1582,7 +1629,7 @@
if print_traceback and sys.exc_info()[1] == err:
traceback.print_exc()
else:
- msg = unicode(err)
+ msg = str(err)
return msg
def report_error(self, err, token_range):
@@ -1606,7 +1653,7 @@
self.attach_callbacks(token_range, future, session)
def num_requests(self):
- return sum(session.num_requests() for session in self.hosts_to_sessions.values())
+ return sum(session.num_requests() for session in list(self.hosts_to_sessions.values()))
def get_session(self, hosts, token_range):
"""
@@ -1626,7 +1673,7 @@
for host in hosts:
try:
ret = self.connect(host)
- except Exception, e:
+ except Exception as e:
errors.append(self.get_error_message(e))
if ret:
@@ -1639,7 +1686,7 @@
return None
def connect(self, host):
- if host in self.hosts_to_sessions.keys():
+ if host in list(self.hosts_to_sessions.keys()):
session = self.hosts_to_sessions[host]
session.add_request()
return session
@@ -1687,17 +1734,18 @@
return # no rows in this range
try:
- output = StringIO()
+ output = StringIO() if six.PY3 else BytesIO()
writer = csv.writer(output, **self.options.dialect)
for row in rows:
- writer.writerow(map(self.format_value, row, cql_types))
+ print("cqlshlib.copyutil.ExportProcess.write_rows_to_csv(): writing row")
+ writer.writerow(list(map(self.format_value, row, cql_types)))
data = (output.getvalue(), len(rows))
self.send((token_range, data))
output.close()
- except Exception, e:
+ except Exception as e:
self.report_error(e, token_range)
def format_value(self, val, cqltype):
@@ -1712,15 +1760,16 @@
if not hasattr(cqltype, 'precision'):
cqltype.precision = self.double_precision if cqltype.type_name == 'double' else self.float_precision
- return formatter(val, cqltype=cqltype,
- encoding=self.encoding, colormap=NO_COLOR_MAP, date_time_format=self.date_time_format,
- float_precision=cqltype.precision, nullval=self.nullval, quote=False,
- decimal_sep=self.decimal_sep, thousands_sep=self.thousands_sep,
- boolean_styles=self.boolean_styles)
+ formatted = formatter(val, cqltype=cqltype,
+ encoding=self.encoding, colormap=NO_COLOR_MAP, date_time_format=self.date_time_format,
+ float_precision=cqltype.precision, nullval=self.nullval, quote=False,
+ decimal_sep=self.decimal_sep, thousands_sep=self.thousands_sep,
+ boolean_styles=self.boolean_styles)
+ return formatted if six.PY3 else formatted.encode('utf8')
def close(self):
ChildProcess.close(self)
- for session in self.hosts_to_sessions.values():
+ for session in list(self.hosts_to_sessions.values()):
session.shutdown()
def prepare_query(self, partition_key, token_range, attempts):
@@ -1782,6 +1831,22 @@
pass
+class ImmutableDict(frozenset):
+ """
+ Immutable dictionary implementation to represent map types.
+ We need to pass BoundStatement.bind() a dict() because it calls iteritems(),
+ except we can't create a dict with another dict as the key, hence we use a class
+ that adds iteritems to a frozen set of tuples (which is how dict are normally made
+ immutable in python).
+ Must be declared in the top level of the module to be available for pickling.
+ """
+ iteritems = frozenset.__iter__
+
+ def items(self):
+ for k, v in self.iteritems():
+ yield k, v
+
+
class ImportConversion(object):
"""
A class for converting strings to values when importing from csv, used by ImportProcess,
@@ -1840,7 +1905,7 @@
select_query = 'SELECT * FROM %s.%s WHERE %s' % (protect_name(parent.ks),
protect_name(parent.table),
where_clause)
- return parent.session.prepare(select_query)
+ return parent.session.prepare(ensure_str(select_query))
@staticmethod
def unprotect(v):
@@ -1868,23 +1933,26 @@
return converters.get(t.typename, convert_unknown)(v, ct=t)
def convert_blob(v, **_):
- return BlobType(v[2:].decode("hex"))
+ if sys.version_info.major >= 3:
+ return bytes.fromhex(v[2:])
+ else:
+ return BlobType(v[2:].decode("hex"))
def convert_text(v, **_):
- return v
+ return ensure_str(v)
def convert_uuid(v, **_):
return UUID(v)
def convert_bool(v, **_):
- return True if v.lower() == self.boolean_styles[0].lower() else False
+ return True if v.lower() == ensure_str(self.boolean_styles[0]).lower() else False
def get_convert_integer_fcn(adapter=int):
"""
Return a slow and a fast integer conversion function depending on self.thousands_sep
"""
if self.thousands_sep:
- return lambda v, ct=cql_type: adapter(v.replace(self.thousands_sep, ''))
+ return lambda v, ct=cql_type: adapter(v.replace(self.thousands_sep, ensure_str('')))
else:
return lambda v, ct=cql_type: adapter(v)
@@ -1892,12 +1960,14 @@
"""
Return a slow and a fast decimal conversion function depending on self.thousands_sep and self.decimal_sep
"""
+ empty_str = ensure_str('')
+ dot_str = ensure_str('.')
if self.thousands_sep and self.decimal_sep:
- return lambda v, ct=cql_type: adapter(v.replace(self.thousands_sep, '').replace(self.decimal_sep, '.'))
+ return lambda v, ct=cql_type: adapter(v.replace(self.thousands_sep, empty_str).replace(self.decimal_sep, dot_str))
elif self.thousands_sep:
- return lambda v, ct=cql_type: adapter(v.replace(self.thousands_sep, ''))
+ return lambda v, ct=cql_type: adapter(v.replace(self.thousands_sep, empty_str))
elif self.decimal_sep:
- return lambda v, ct=cql_type: adapter(v.replace(self.decimal_sep, '.'))
+ return lambda v, ct=cql_type: adapter(v.replace(self.decimal_sep, dot_str))
else:
return lambda v, ct=cql_type: adapter(v)
@@ -1949,14 +2019,20 @@
return ret
# this should match all possible CQL and CQLSH datetime formats
- p = re.compile("(\d{4})\-(\d{2})\-(\d{2})\s?(?:'T')?" + # YYYY-MM-DD[( |'T')]
- "(?:(\d{2}):(\d{2})(?::(\d{2})(?:\.(\d{1,6}))?))?" + # [HH:MM[:SS[.NNNNNN]]]
- "(?:([+\-])(\d{2}):?(\d{2}))?") # [(+|-)HH[:]MM]]
+ p = re.compile(r"(\d{4})\-(\d{2})\-(\d{2})\s?(?:'T')?" # YYYY-MM-DD[( |'T')]
+ + r"(?:(\d{2}):(\d{2})(?::(\d{2})(?:\.(\d{1,6}))?))?" # [HH:MM[:SS[.NNNNNN]]]
+ + r"(?:([+\-])(\d{2}):?(\d{2}))?") # [(+|-)HH[:]MM]]
def convert_datetime(val, **_):
try:
- tval = time.strptime(val, self.date_time_format)
- return timegm(tval) * 1e3 # scale seconds to millis for the raw value
+ if six.PY2:
+ # Python 2 implementation
+ tval = time.strptime(val, self.date_time_format)
+ return timegm(tval) * 1e3 # scale seconds to millis for the raw value
+ else:
+ # Python 3 implementation
+ dtval = datetime.datetime.strptime(val, self.date_time_format)
+ return dtval.timestamp() * 1000
except ValueError:
pass # if it's not in the default format we try CQL formats
@@ -1986,7 +2062,7 @@
offset = -time.timezone
# scale seconds to millis for the raw value
- return ((timegm(tval) + offset) * 1e3) + milliseconds
+ return ((timegm(tval) + offset) * 1000) + milliseconds
def convert_date(v, **_):
return Date(v)
@@ -2005,16 +2081,12 @@
def convert_map(val, ct=cql_type):
"""
- We need to pass to BoundStatement.bind() a dict() because it calls iteritems(),
- except we can't create a dict with another dict as the key, hence we use a class
- that adds iteritems to a frozen set of tuples (which is how dict are normally made
- immutable in python).
+ See ImmutableDict above for a discussion of why a special object is needed here.
"""
- class ImmutableDict(frozenset):
- iteritems = frozenset.__iter__
-
+ split_format_str = ensure_str('{%s}')
+ sep = ensure_str(':')
return ImmutableDict(frozenset((convert_mandatory(ct.subtypes[0], v[0]), convert(ct.subtypes[1], v[1]))
- for v in [split('{%s}' % vv, sep=':') for vv in split(val)]))
+ for v in [split(split_format_str % vv, sep=sep) for vv in split(val)]))
def convert_user_type(val, ct=cql_type):
"""
@@ -2025,7 +2097,9 @@
Also note that it is possible that the subfield names in the csv are in the
wrong order, so we must sort them according to ct.fieldnames, see CASSANDRA-12959.
"""
- vals = [v for v in [split('{%s}' % vv, sep=':') for vv in split(val)]]
+ split_format_str = ensure_str('{%s}')
+ sep = ensure_str(':')
+ vals = [v for v in [split(split_format_str % vv, sep=sep) for vv in split(val)]]
dict_vals = dict((unprotect(v[0]), v[1]) for v in vals)
sorted_converted_vals = [(n, convert(t, dict_vals[n]) if n in dict_vals else self.get_null_val())
for n, t in zip(ct.fieldnames, ct.subtypes)]
@@ -2053,11 +2127,11 @@
'ascii': convert_text,
'float': get_convert_decimal_fcn(),
'double': get_convert_decimal_fcn(),
- 'bigint': get_convert_integer_fcn(adapter=long),
+ 'bigint': get_convert_integer_fcn(adapter=int),
'int': get_convert_integer_fcn(),
'varint': get_convert_integer_fcn(),
'inet': convert_text,
- 'counter': get_convert_integer_fcn(adapter=long),
+ 'counter': get_convert_integer_fcn(adapter=int),
'timestamp': convert_datetime,
'timeuuid': convert_uuid,
'date': convert_date,
@@ -2082,7 +2156,7 @@
or "NULL" otherwise. Note that for counters we never use prepared statements, so we
only check is_counter when use_prepared_statements is false.
"""
- return None if self.use_prepared_statements else ("0" if self.is_counter else "NULL")
+ return None if self.use_prepared_statements else (ensure_str("0") if self.is_counter else ensure_str("NULL"))
def convert_row(self, row):
"""
@@ -2102,7 +2176,7 @@
def convert(c, v):
try:
return c(v) if v != self.nullval else self.get_null_val()
- except Exception, e:
+ except Exception as e:
# if we could not convert an empty string, then self.nullval has been set to a marker
# because the user needs to import empty strings, except that the converters for some types
# will fail to convert an empty string, in this case the null value should be inserted
@@ -2112,7 +2186,7 @@
if self.debug:
traceback.print_exc()
- raise ParseError("Failed to parse %s : %s" % (val, e.message))
+ raise ParseError("Failed to parse %s : %s" % (v, e.message if hasattr(e, 'message') else str(e)))
return [convert(conv, val) for conv, val in zip(converters, row)]
@@ -2149,6 +2223,7 @@
val = serialize(i, row[i])
length = len(val)
pk_values.append(struct.pack(">H%dsB" % length, length, val, 0))
+
return b"".join(pk_values)
if len(partition_key_indexes) == 1:
@@ -2204,7 +2279,7 @@
def filter_replicas(self, hosts):
shuffled = tuple(sorted(hosts, key=lambda k: random.random()))
- return filter(lambda r: r.is_up is not False and r.datacenter == self.local_dc, shuffled) if hosts else ()
+ return [r for r in shuffled if r.is_up is not False and r.datacenter == self.local_dc] if hosts else ()
class FastTokenAwarePolicy(DCAwareRoundRobinPolicy):
@@ -2237,7 +2312,7 @@
return conn.in_flight < min(conn.max_request_id, self.max_inflight_messages)
return True
- for i in xrange(self.max_backoff_attempts):
+ for i in range(self.max_backoff_attempts):
for r in filter(replica_is_not_overloaded, replicas):
yield r
@@ -2269,7 +2344,7 @@
ChildProcess.__init__(self, params=params, target=self.run)
self.skip_columns = params['skip_columns']
- self.valid_columns = [c.encode(self.encoding) for c in params['valid_columns']]
+ self.valid_columns = [c for c in params['valid_columns']]
self.skip_column_indexes = [i for i, c in enumerate(self.columns) if c in self.skip_columns]
options = params['options']
@@ -2312,6 +2387,9 @@
return self._session
def run(self):
+ if self.coverage:
+ self.start_coverage()
+
try:
pr = profile_on() if PROFILE_ON else None
@@ -2321,10 +2399,12 @@
if pr:
profile_off(pr, file_name='worker_profile_%d.txt' % (os.getpid(),))
- except Exception, exc:
+ except Exception as exc:
self.report_error(exc)
finally:
+ if self.coverage:
+ self.stop_coverage()
self.close()
def close(self):
@@ -2361,6 +2441,7 @@
if self.ttl >= 0:
query += 'USING TTL %s' % (self.ttl,)
make_statement = self.wrap_make_statement(self.make_non_prepared_batch_statement)
+ query = ensure_str(query)
conv = ImportConversion(self, table_meta, prepared_statement)
tm = TokenMap(self.ks, self.hostname, self.local_dc, self.session)
@@ -2399,15 +2480,15 @@
# causes the statement to be None, then we should not report the error so that we can test
# the parent process handling missing batches from child processes
- except Exception, exc:
+ except Exception as exc:
self.report_error(exc, chunk, chunk['rows'])
def wrap_make_statement(self, inner_make_statement):
def make_statement(query, conv, chunk, batch, replicas):
try:
return inner_make_statement(query, conv, batch, replicas)
- except Exception, exc:
- print "Failed to make batch statement: {}".format(exc)
+ except Exception as exc:
+ print("Failed to make batch statement: {}".format(exc))
self.report_error(exc, chunk, batch['rows'])
return None
@@ -2428,12 +2509,12 @@
set_clause = []
for i, value in enumerate(row):
if i in conv.primary_key_indexes:
- where_clause.append("%s=%s" % (self.valid_columns[i], value))
+ where_clause.append(ensure_text("{}={}").format(self.valid_columns[i], ensure_text(value)))
else:
- set_clause.append("%s=%s+%s" % (self.valid_columns[i], self.valid_columns[i], value))
+ set_clause.append(ensure_text("{}={}+{}").format(self.valid_columns[i], self.valid_columns[i], ensure_text(value)))
- full_query_text = query % (','.join(set_clause), ' AND '.join(where_clause))
- statement.add(full_query_text)
+ full_query_text = query % (ensure_text(',').join(set_clause), ensure_text(' AND ').join(where_clause))
+ statement.add(ensure_str(full_query_text))
return statement
def make_prepared_batch_statement(self, query, _, batch, replicas):
@@ -2457,7 +2538,8 @@
statement = BatchStatement(batch_type=BatchType.UNLOGGED, consistency_level=self.consistency_level)
statement.replicas = replicas
statement.keyspace = self.ks
- statement._statements_and_parameters = [(False, query % (','.join(r),), ()) for r in batch['rows']]
+ field_sep = b',' if six.PY2 else ','
+ statement._statements_and_parameters = [(False, query % (field_sep.join(r),), ()) for r in batch['rows']]
return statement
def convert_rows(self, conv, chunk):
@@ -2477,14 +2559,14 @@
def convert_row(r):
try:
return conv.convert_row(r)
- except Exception, err:
- errors[err.message].append(r)
+ except Exception as err:
+ errors[err.message if hasattr(err, 'message') else str(err)].append(r)
return None
- converted_rows = filter(None, [convert_row(r) for r in rows])
+ converted_rows = [_f for _f in [convert_row(r) for r in rows] if _f]
if errors:
- for msg, rows in errors.iteritems():
+ for msg, rows in errors.items():
self.report_error(ParseError(msg), chunk, rows)
return converted_rows
@@ -2549,27 +2631,27 @@
try:
pk = get_row_partition_key_values(row)
rows_by_ring_pos[get_ring_pos(ring, pk_to_token_value(pk))].append(row)
- except Exception, e:
- errors[e.message].append(row)
+ except Exception as e:
+ errors[e.message if hasattr(e, 'message') else str(e)].append(row)
if errors:
- for msg, rows in errors.iteritems():
+ for msg, rows in errors.items():
self.report_error(ParseError(msg), chunk, rows)
replicas = tm.replicas
filter_replicas = tm.filter_replicas
rows_by_replica = defaultdict(list)
- for ring_pos, rows in rows_by_ring_pos.iteritems():
+ for ring_pos, rows in rows_by_ring_pos.items():
if len(rows) > min_batch_size:
- for i in xrange(0, len(rows), max_batch_size):
+ for i in range(0, len(rows), max_batch_size):
yield filter_replicas(replicas[ring_pos]), make_batch(chunk['id'], rows[i:i + max_batch_size])
else:
# select only the first valid replica to guarantee more overlap or none at all
- rows_by_replica[filter_replicas(replicas[ring_pos])[:1]].extend(rows)
+ rows_by_replica[tuple(filter_replicas(replicas[ring_pos])[:1])].extend(rows) # TODO: revisit tuple wrapper
# Now send the batches by replica
- for replicas, rows in rows_by_replica.iteritems():
- for i in xrange(0, len(rows), max_batch_size):
+ for replicas, rows in rows_by_replica.items():
+ for i in range(0, len(rows), max_batch_size):
yield replicas, make_batch(chunk['id'], rows[i:i + max_batch_size])
def result_callback(self, _, batch, chunk):
@@ -2590,7 +2672,8 @@
def report_error(self, err, chunk=None, rows=None, attempts=1, final=True):
if self.debug and sys.exc_info()[1] == err:
traceback.print_exc()
- self.outmsg.send(ImportTaskError(err.__class__.__name__, err.message, rows, attempts, final))
+ err_msg = err.message if hasattr(err, 'message') else str(err)
+ self.outmsg.send(ImportTaskError(err.__class__.__name__, err_msg, rows, attempts, final))
if final and chunk is not None:
self.update_chunk(rows, chunk)
diff --git a/pylib/cqlshlib/cql3handling.py b/pylib/cqlshlib/cql3handling.py
index ae5bc8a..b2403a7 100644
--- a/pylib/cqlshlib/cql3handling.py
+++ b/pylib/cqlshlib/cql3handling.py
@@ -14,15 +14,14 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from .cqlhandling import CqlParsingRuleSet, Hint
from cassandra.metadata import maybe_escape_name
-
+from cqlshlib import helptopics
+from cqlshlib.cqlhandling import CqlParsingRuleSet, Hint
simple_cql_types = set(('ascii', 'bigint', 'blob', 'boolean', 'counter', 'date', 'decimal', 'double', 'duration', 'float',
'inet', 'int', 'smallint', 'text', 'time', 'timestamp', 'timeuuid', 'tinyint', 'uuid', 'varchar', 'varint'))
simple_cql_types.difference_update(('set', 'map', 'list'))
-from . import helptopics
cqldocs = helptopics.CQL3HelpTopics()
@@ -35,8 +34,8 @@
return 'Unexpected table structure; may not translate correctly to CQL. ' + self.msg
-SYSTEM_KEYSPACES = ('system', 'system_schema', 'system_traces', 'system_auth', 'system_distributed')
-NONALTERBALE_KEYSPACES = ('system', 'system_schema')
+SYSTEM_KEYSPACES = ('system', 'system_schema', 'system_traces', 'system_auth', 'system_distributed', 'system_views', 'system_virtual_schema')
+NONALTERBALE_KEYSPACES = ('system', 'system_schema', 'system_views', 'system_virtual_schema')
class Cql3ParsingRuleSet(CqlParsingRuleSet):
@@ -44,15 +43,15 @@
columnfamily_layout_options = (
('bloom_filter_fp_chance', None),
('comment', None),
- ('dclocal_read_repair_chance', 'local_read_repair_chance'),
('gc_grace_seconds', None),
('min_index_interval', None),
('max_index_interval', None),
- ('read_repair_chance', None),
('default_time_to_live', None),
('speculative_retry', None),
+ ('additional_write_policy', None),
('memtable_flush_period_in_ms', None),
- ('cdc', None)
+ ('cdc', None),
+ ('read_repair', None),
)
columnfamily_layout_map_options = (
@@ -80,6 +79,33 @@
'SERIAL'
)
+ size_tiered_compaction_strategy_options = (
+ 'min_sstable_size',
+ 'min_threshold',
+ 'bucket_high',
+ 'bucket_low'
+ )
+
+ leveled_compaction_strategy_options = (
+ 'sstable_size_in_mb',
+ 'fanout_size'
+ )
+
+ date_tiered_compaction_strategy_options = (
+ 'base_time_seconds',
+ 'max_sstable_age_days',
+ 'min_threshold',
+ 'max_window_size_seconds',
+ 'timestamp_resolution'
+ )
+
+ time_window_compaction_strategy_options = (
+ 'compaction_window_unit',
+ 'compaction_window_size',
+ 'min_threshold',
+ 'timestamp_resolution'
+ )
+
@classmethod
def escape_value(cls, value):
if value is None:
@@ -154,7 +180,7 @@
<colon> ::= ":" ;
<star> ::= "*" ;
<endtoken> ::= ";" ;
-<op> ::= /[-+=,().]/ ;
+<op> ::= /[-+=%/,().]/ ;
<cmp> ::= /[<>!]=?/ ;
<brackets> ::= /[][{}]/ ;
@@ -415,19 +441,19 @@
optname = ctxt.get_binding('propname')[-1]
if optname != 'replication':
return ()
- keysseen = map(dequote_value, ctxt.get_binding('propmapkey', ()))
- valsseen = map(dequote_value, ctxt.get_binding('propmapval', ()))
+ keysseen = list(map(dequote_value, ctxt.get_binding('propmapkey', ())))
+ valsseen = list(map(dequote_value, ctxt.get_binding('propmapval', ())))
for k, v in zip(keysseen, valsseen):
if k == 'class':
repclass = v
break
else:
return ["'class'"]
- if repclass in CqlRuleSet.replication_factor_strategies:
+ if repclass == 'SimpleStrategy':
opts = set(('replication_factor',))
elif repclass == 'NetworkTopologyStrategy':
return [Hint('<dc_name>')]
- return map(escape_value, opts.difference(keysseen))
+ return list(map(escape_value, opts.difference(keysseen)))
def ks_prop_val_mapval_completer(ctxt, cass):
@@ -436,7 +462,7 @@
return ()
currentkey = dequote_value(ctxt.get_binding('propmapkey')[-1])
if currentkey == 'class':
- return map(escape_value, CqlRuleSet.replication_strategies)
+ return list(map(escape_value, CqlRuleSet.replication_strategies))
return [Hint('<term>')]
@@ -444,15 +470,15 @@
optname = ctxt.get_binding('propname')[-1]
if optname != 'replication':
return [',']
- keysseen = map(dequote_value, ctxt.get_binding('propmapkey', ()))
- valsseen = map(dequote_value, ctxt.get_binding('propmapval', ()))
+ keysseen = list(map(dequote_value, ctxt.get_binding('propmapkey', ())))
+ valsseen = list(map(dequote_value, ctxt.get_binding('propmapval', ())))
for k, v in zip(keysseen, valsseen):
if k == 'class':
repclass = v
break
else:
return [',']
- if repclass in CqlRuleSet.replication_factor_strategies:
+ if repclass == 'SimpleStrategy':
if 'replication_factor' not in keysseen:
return [',']
if repclass == 'NetworkTopologyStrategy' and len(keysseen) == 1:
@@ -461,8 +487,8 @@
def cf_prop_name_completer(ctxt, cass):
- return [c[0] for c in (CqlRuleSet.columnfamily_layout_options +
- CqlRuleSet.columnfamily_layout_map_options)]
+ return [c[0] for c in (CqlRuleSet.columnfamily_layout_options
+ + CqlRuleSet.columnfamily_layout_map_options)]
def cf_prop_val_completer(ctxt, cass):
@@ -476,14 +502,15 @@
return ["{'keys': '"]
if any(this_opt == opt[0] for opt in CqlRuleSet.obsolete_cf_options):
return ["'<obsolete_option>'"]
- if this_opt in ('read_repair_chance', 'bloom_filter_fp_chance',
- 'dclocal_read_repair_chance'):
+ if this_opt == 'bloom_filter_fp_chance':
return [Hint('<float_between_0_and_1>')]
if this_opt in ('min_compaction_threshold', 'max_compaction_threshold',
'gc_grace_seconds', 'min_index_interval', 'max_index_interval'):
return [Hint('<integer>')]
if this_opt in ('cdc'):
return [Hint('<true|false>')]
+ if this_opt in ('read_repair'):
+ return [Hint('<\'none\'|\'blocking\'>')]
return [Hint('<option_value>')]
@@ -494,13 +521,13 @@
break
else:
return ()
- keysseen = map(dequote_value, ctxt.get_binding('propmapkey', ()))
- valsseen = map(dequote_value, ctxt.get_binding('propmapval', ()))
- pairsseen = dict(zip(keysseen, valsseen))
+ keysseen = list(map(dequote_value, ctxt.get_binding('propmapkey', ())))
+ valsseen = list(map(dequote_value, ctxt.get_binding('propmapval', ())))
+ pairsseen = dict(list(zip(keysseen, valsseen)))
if optname == 'compression':
- return map(escape_value, set(subopts).difference(keysseen))
+ return list(map(escape_value, set(subopts).difference(keysseen)))
if optname == 'caching':
- return map(escape_value, set(subopts).difference(keysseen))
+ return list(map(escape_value, set(subopts).difference(keysseen)))
if optname == 'compaction':
opts = set(subopts)
try:
@@ -509,27 +536,15 @@
return ["'class'"]
csc = csc.split('.')[-1]
if csc == 'SizeTieredCompactionStrategy':
- opts.add('min_sstable_size')
- opts.add('min_threshold')
- opts.add('bucket_high')
- opts.add('bucket_low')
+ opts = opts.union(set(CqlRuleSet.size_tiered_compaction_strategy_options))
elif csc == 'LeveledCompactionStrategy':
- opts.add('sstable_size_in_mb')
- opts.add('fanout_size')
+ opts = opts.union(set(CqlRuleSet.leveled_compaction_strategy_options))
elif csc == 'DateTieredCompactionStrategy':
- opts.add('base_time_seconds')
- opts.add('max_sstable_age_days')
- opts.add('min_threshold')
- opts.add('max_window_size_seconds')
- opts.add('timestamp_resolution')
+ opts = opts.union(set(CqlRuleSet.date_tiered_compaction_strategy_options))
elif csc == 'TimeWindowCompactionStrategy':
- opts.add('compaction_window_unit')
- opts.add('compaction_window_size')
- opts.add('min_threshold')
- opts.add('max_threshold')
- opts.add('timestamp_resolution')
+ opts = opts.union(set(CqlRuleSet.time_window_compaction_strategy_options))
- return map(escape_value, opts)
+ return list(map(escape_value, opts))
return ()
@@ -538,11 +553,11 @@
key = dequote_value(ctxt.get_binding('propmapkey')[-1])
if opt == 'compaction':
if key == 'class':
- return map(escape_value, CqlRuleSet.available_compaction_classes)
+ return list(map(escape_value, CqlRuleSet.available_compaction_classes))
return [Hint('<option_value>')]
elif opt == 'compression':
if key == 'sstable_compression':
- return map(escape_value, CqlRuleSet.available_compression_classes)
+ return list(map(escape_value, CqlRuleSet.available_compression_classes))
return [Hint('<option_value>')]
elif opt == 'caching':
if key == 'rows_per_partition':
@@ -568,19 +583,19 @@
@completer_for('keyspaceName', 'ksname')
def ks_name_completer(ctxt, cass):
- return map(maybe_escape_name, cass.get_keyspace_names())
+ return list(map(maybe_escape_name, cass.get_keyspace_names()))
@completer_for('nonSystemKeyspaceName', 'ksname')
def non_system_ks_name_completer(ctxt, cass):
ksnames = [n for n in cass.get_keyspace_names() if n not in SYSTEM_KEYSPACES]
- return map(maybe_escape_name, ksnames)
+ return list(map(maybe_escape_name, ksnames))
@completer_for('alterableKeyspaceName', 'ksname')
def alterable_ks_name_completer(ctxt, cass):
ksnames = [n for n in cass.get_keyspace_names() if n not in NONALTERBALE_KEYSPACES]
- return map(maybe_escape_name, ksnames)
+ return list(map(maybe_escape_name, ksnames))
def cf_ks_name_completer(ctxt, cass):
@@ -613,7 +628,7 @@
if ks is None:
return ()
raise
- return map(maybe_escape_name, cfnames)
+ return list(map(maybe_escape_name, cfnames))
@completer_for('materializedViewName', 'mvname')
@@ -627,7 +642,7 @@
if ks is None:
return ()
raise
- return map(maybe_escape_name, mvnames)
+ return list(map(maybe_escape_name, mvnames))
completer_for('userTypeName', 'ksname')(cf_ks_name_completer)
@@ -645,7 +660,7 @@
if ks is None:
return ()
raise
- return map(maybe_escape_name, utnames)
+ return list(map(maybe_escape_name, utnames))
completer_for('userTypeName', 'utname')(ut_name_completer)
@@ -708,7 +723,7 @@
;
<udtSubfieldSelection> ::= <identifier> "." <identifier>
;
-<selector> ::= [colname]=<cident>
+<selector> ::= [colname]=<cident> ( "[" ( <term> ( ".." <term> "]" )? | <term> ".." ) )?
| <udtSubfieldSelection>
| "WRITETIME" "(" [colname]=<cident> ")"
| "TTL" "(" [colname]=<cident> ")"
@@ -736,7 +751,7 @@
if ks is None:
return ()
raise
- return map(maybe_escape_name, udfnames)
+ return list(map(maybe_escape_name, udfnames))
def uda_name_completer(ctxt, cass):
@@ -749,7 +764,7 @@
if ks is None:
return ()
raise
- return map(maybe_escape_name, udanames)
+ return list(map(maybe_escape_name, udanames))
def udf_uda_name_completer(ctxt, cass):
@@ -762,7 +777,7 @@
if ks is None:
return ()
raise
- return map(maybe_escape_name, functionnames)
+ return list(map(maybe_escape_name, functionnames))
def ref_udf_name_completer(ctxt, cass):
@@ -770,7 +785,7 @@
udanames = cass.get_userfunction_names(None)
except Exception:
return ()
- return map(maybe_escape_name, udanames)
+ return list(map(maybe_escape_name, udanames))
completer_for('functionAggregateName', 'ksname')(cf_ks_name_completer)
@@ -828,7 +843,7 @@
def select_relation_lhs_completer(ctxt, cass):
layout = get_table_meta(ctxt, cass)
filterable = set()
- already_filtered_on = map(dequote_name, ctxt.get_binding('rel_lhs', ()))
+ already_filtered_on = list(map(dequote_name, ctxt.get_binding('rel_lhs', ())))
for num in range(0, len(layout.partition_key)):
if num == 0 or layout.partition_key[num - 1].name in already_filtered_on:
filterable.add(layout.partition_key[num].name)
@@ -839,9 +854,9 @@
filterable.add(layout.clustering_key[num].name)
else:
break
- for idx in layout.indexes.itervalues():
+ for idx in layout.indexes.values():
filterable.add(idx.index_options["target"])
- return map(maybe_escape_name, filterable)
+ return list(map(maybe_escape_name, filterable))
explain_completion('selector', 'colname')
@@ -864,9 +879,9 @@
def regular_column_names(table_meta):
if not table_meta or not table_meta.columns:
return []
- regular_columns = list(set(table_meta.columns.keys()) -
- set([key.name for key in table_meta.partition_key]) -
- set([key.name for key in table_meta.clustering_key]))
+ regular_columns = list(set(table_meta.columns.keys())
+ - set([key.name for key in table_meta.partition_key])
+ - set([key.name for key in table_meta.clustering_key]))
return regular_columns
@@ -879,13 +894,13 @@
if k.name not in colnames:
return [maybe_escape_name(k.name)]
normalcols = set(regular_column_names(layout)) - colnames
- return map(maybe_escape_name, normalcols)
+ return list(map(maybe_escape_name, normalcols))
@completer_for('insertStatement', 'newval')
def insert_newval_completer(ctxt, cass):
layout = get_table_meta(ctxt, cass)
- insertcols = map(dequote_name, ctxt.get_binding('colname'))
+ insertcols = list(map(dequote_name, ctxt.get_binding('colname')))
valuesdone = ctxt.get_binding('newval', ())
if len(valuesdone) >= len(insertcols):
return []
@@ -958,7 +973,7 @@
@completer_for('assignment', 'updatecol')
def update_col_completer(ctxt, cass):
layout = get_table_meta(ctxt, cass)
- return map(maybe_escape_name, regular_column_names(layout))
+ return list(map(maybe_escape_name, regular_column_names(layout)))
@completer_for('assignment', 'update_rhs')
@@ -1100,7 +1115,7 @@
@completer_for('deleteSelector', 'delcol')
def delete_delcol_completer(ctxt, cass):
layout = get_table_meta(ctxt, cass)
- return map(maybe_escape_name, regular_column_names(layout))
+ return list(map(maybe_escape_name, regular_column_names(layout)))
syntax_rules += r'''
@@ -1181,7 +1196,7 @@
@completer_for('cfamOrdering', 'ordercol')
def create_cf_clustering_order_colname_completer(ctxt, cass):
- colnames = map(dequote_name, ctxt.get_binding('newcolname', ()))
+ colnames = list(map(dequote_name, ctxt.get_binding('newcolname', ())))
# Definitely some of these aren't valid for ordering, but I'm not sure
# precisely which are. This is good enough for now
return colnames
@@ -1211,7 +1226,7 @@
def create_cf_pkdef_declaration_completer(ctxt, cass):
cols_declared = ctxt.get_binding('newcolname')
pieces_already = ctxt.get_binding('ptkey', ())
- pieces_already = map(dequote_name, pieces_already)
+ pieces_already = list(map(dequote_name, pieces_already))
while cols_declared[0] in pieces_already:
cols_declared = cols_declared[1:]
if len(cols_declared) < 2:
@@ -1223,7 +1238,7 @@
def create_cf_composite_key_declaration_completer(ctxt, cass):
cols_declared = ctxt.get_binding('newcolname')
pieces_already = ctxt.get_binding('ptkey', ()) + ctxt.get_binding('pkey', ())
- pieces_already = map(dequote_name, pieces_already)
+ pieces_already = list(map(dequote_name, pieces_already))
while cols_declared[0] in pieces_already:
cols_declared = cols_declared[1:]
if len(cols_declared) < 2:
@@ -1309,9 +1324,9 @@
def create_index_col_completer(ctxt, cass):
""" Return the columns for which an index doesn't exist yet. """
layout = get_table_meta(ctxt, cass)
- idx_targets = [idx.index_options["target"] for idx in layout.indexes.itervalues()]
- colnames = [cd.name for cd in layout.columns.values() if cd.name not in idx_targets]
- return map(maybe_escape_name, colnames)
+ idx_targets = [idx.index_options["target"] for idx in layout.indexes.values()]
+ colnames = [cd.name for cd in list(layout.columns.values()) if cd.name not in idx_targets]
+ return list(map(maybe_escape_name, colnames))
syntax_rules += r'''
@@ -1369,7 +1384,7 @@
if ks is None:
return ()
raise
- return map(maybe_escape_name, idxnames)
+ return list(map(maybe_escape_name, idxnames))
syntax_rules += r'''
@@ -1397,14 +1412,14 @@
def alter_table_col_completer(ctxt, cass):
layout = get_table_meta(ctxt, cass)
cols = [str(md) for md in layout.columns]
- return map(maybe_escape_name, cols)
+ return list(map(maybe_escape_name, cols))
@completer_for('alterTypeInstructions', 'existcol')
def alter_type_field_completer(ctxt, cass):
layout = get_ut_layout(ctxt, cass)
fields = [tuple[0] for tuple in layout]
- return map(maybe_escape_name, fields)
+ return list(map(maybe_escape_name, fields))
explain_completion('alterInstructions', 'newcol', '<new_column_name>')
@@ -1456,6 +1471,8 @@
| "OPTIONS" "=" <mapLiteral>
| "SUPERUSER" "=" <boolean>
| "LOGIN" "=" <boolean>
+ | "ACCESS" "TO" "DATACENTERS" <setLiteral>
+ | "ACCESS" "TO" "ALL" "DATACENTERS"
;
<dropRoleStatement> ::= "DROP" "ROLE" <rolename>
@@ -1539,7 +1556,7 @@
return [Hint('<username>')]
session = cass.session
- return [maybe_quote(row.values()[0].replace("'", "''")) for row in session.execute("LIST USERS")]
+ return [maybe_quote(list(row.values())[0].replace("'", "''")) for row in session.execute("LIST USERS")]
@completer_for('rolename', 'role')
@@ -1578,7 +1595,7 @@
@completer_for('dropTriggerStatement', 'triggername')
def drop_trigger_completer(ctxt, cass):
names = get_trigger_names(ctxt, cass)
- return map(maybe_escape_name, names)
+ return list(map(maybe_escape_name, names))
# END SYNTAX/COMPLETION RULE DEFINITIONS
diff --git a/pylib/cqlshlib/cqlhandling.py b/pylib/cqlshlib/cqlhandling.py
index 51d9726..08d5828 100644
--- a/pylib/cqlshlib/cqlhandling.py
+++ b/pylib/cqlshlib/cqlhandling.py
@@ -18,8 +18,9 @@
# i.e., stuff that's not necessarily cqlsh-specific
import traceback
+
from cassandra.metadata import cql_keywords_reserved
-from . import pylexotron, util
+from cqlshlib import pylexotron, util
Hint = pylexotron.Hint
@@ -30,6 +31,7 @@
'DeflateCompressor',
'SnappyCompressor',
'LZ4Compressor',
+ 'ZstdCompressor',
)
available_compaction_classes = (
@@ -41,17 +43,9 @@
replication_strategies = (
'SimpleStrategy',
- 'OldNetworkTopologyStrategy',
'NetworkTopologyStrategy'
)
- replication_factor_strategies = (
- 'SimpleStrategy',
- 'org.apache.cassandra.locator.SimpleStrategy',
- 'OldNetworkTopologyStrategy',
- 'org.apache.cassandra.locator.OldNetworkTopologyStrategy'
- )
-
def __init__(self, *args, **kwargs):
pylexotron.ParsingRuleSet.__init__(self, *args, **kwargs)
@@ -74,7 +68,7 @@
if cass is None:
return ()
return f(ctxt, cass)
- completerwrapper.func_name = 'completerwrapper_on_' + f.func_name
+ completerwrapper.__name__ = 'completerwrapper_on_' + f.__name__
self.register_completer(completerwrapper, rulename, symname)
return completerwrapper
return registrator
@@ -108,12 +102,6 @@
# operations on tokens (like .lower()). See CASSANDRA-9083
# for one example of this.
str_token = t[1]
- if isinstance(str_token, unicode):
- try:
- str_token = str_token.encode('ascii')
- t = (t[0], str_token) + t[2:]
- except UnicodeEncodeError:
- pass
curstmt.append(t)
if t[0] == 'endtoken':
@@ -204,7 +192,7 @@
f = lambda s: s and dequoter(s).lower().startswith(partial)
else:
f = lambda s: s and dequoter(s).startswith(partial)
- candidates = filter(f, strcompletes)
+ candidates = list(filter(f, strcompletes))
if prefix is not None:
# dequote, re-escape, strip quotes: gets us the right quoted text
@@ -215,7 +203,7 @@
# the above process can result in an empty string; this doesn't help for
# completions
- candidates = filter(None, candidates)
+ candidates = [_f for _f in candidates if _f]
# prefix a space when desirable for pleasant cql formatting
if tokens:
@@ -257,7 +245,7 @@
init_bindings = {'cassandra_conn': cassandra_conn}
if debug:
init_bindings['*DEBUG*'] = True
- print "cql_complete(%r, partial=%r)" % (text, partial)
+ print("cql_complete(%r, partial=%r)" % (text, partial))
completions, hints = self.cql_complete_single(text, partial, init_bindings,
startsymbol=startsymbol)
@@ -269,12 +257,12 @@
if len(completions) == 1 and len(hints) == 0:
c = completions[0]
if debug:
- print "** Got one completion: %r. Checking for further matches...\n" % (c,)
+ print("** Got one completion: %r. Checking for further matches...\n" % (c,))
if not c.isspace():
new_c = self.cql_complete_multiple(text, c, init_bindings, startsymbol=startsymbol)
completions = [new_c]
if debug:
- print "** New list of completions: %r" % (completions,)
+ print("** New list of completions: %r" % (completions,))
return hints + completions
@@ -285,18 +273,18 @@
startsymbol=startsymbol)
except Exception:
if debug:
- print "** completion expansion had a problem:"
+ print("** completion expansion had a problem:")
traceback.print_exc()
return first
if hints:
if not first[-1].isspace():
first += ' '
if debug:
- print "** completion expansion found hints: %r" % (hints,)
+ print("** completion expansion found hints: %r" % (hints,))
return first
if len(completions) == 1 and completions[0] != '':
if debug:
- print "** Got another completion: %r." % (completions[0],)
+ print("** Got another completion: %r." % (completions[0],))
if completions[0][0] in (',', ')', ':') and first[-1] == ' ':
first = first[:-1]
first += completions[0]
@@ -307,10 +295,10 @@
if common_prefix[0] in (',', ')', ':') and first[-1] == ' ':
first = first[:-1]
if debug:
- print "** Got a partial completion: %r." % (common_prefix,)
+ print("** Got a partial completion: %r." % (common_prefix,))
return first + common_prefix
if debug:
- print "** New total completion: %r. Checking for further matches...\n" % (first,)
+ print("** New total completion: %r. Checking for further matches...\n" % (first,))
return self.cql_complete_multiple(text, first, init_bindings, startsymbol=startsymbol)
@staticmethod
diff --git a/pylib/cqlshlib/cqlshhandling.py b/pylib/cqlshlib/cqlshhandling.py
index 9545876..aa1fbc0 100644
--- a/pylib/cqlshlib/cqlshhandling.py
+++ b/pylib/cqlshlib/cqlshhandling.py
@@ -15,7 +15,8 @@
# limitations under the License.
import os
-import cqlhandling
+
+from cqlshlib import cqlhandling
# we want the cql parser to understand our cqlsh-specific commands too
my_commands_ending_with_newline = (
@@ -77,12 +78,12 @@
cqlsh_describe_cmd_syntax_rules = r'''
<describeCommand> ::= ( "DESCRIBE" | "DESC" )
- ( "FUNCTIONS"
+ ( ( "FUNCTIONS"
| "FUNCTION" udf=<anyFunctionName>
| "AGGREGATES"
| "AGGREGATE" uda=<userAggregateName>
| "KEYSPACES"
- | "KEYSPACE" ksname=<keyspaceName>?
+ | "ONLY"? "KEYSPACE" ksname=<keyspaceName>?
| ( "COLUMNFAMILY" | "TABLE" ) cf=<columnFamilyName>
| "INDEX" idx=<indexName>
| "MATERIALIZED" "VIEW" mv=<materializedViewName>
@@ -91,7 +92,9 @@
| "CLUSTER"
| "TYPES"
| "TYPE" ut=<userTypeName>
- | (ksname=<keyspaceName> | cf=<columnFamilyName> | idx=<indexName> | mv=<materializedViewName>))
+ | (ksname=<keyspaceName> | cf=<columnFamilyName> | idx=<indexName> | mv=<materializedViewName>)
+ ) ("WITH" "INTERNALS")?
+ )
;
'''
@@ -112,6 +115,7 @@
| "SERIAL"
| "LOCAL_SERIAL"
| "LOCAL_ONE"
+ | "NODE_LOCAL"
;
'''
@@ -239,7 +243,7 @@
contents = os.listdir(exhead or '.')
except OSError:
return ()
- matches = filter(lambda f: f.startswith(tail), contents)
+ matches = [f for f in contents if f.startswith(tail)]
annotated = []
for f in matches:
match = os.path.join(head, f)
@@ -266,7 +270,7 @@
@cqlsh_syntax_completer('copyCommand', 'colnames')
def complete_copy_column_names(ctxt, cqlsh):
- existcols = map(cqlsh.cql_unprotect_name, ctxt.get_binding('colnames', ()))
+ existcols = list(map(cqlsh.cql_unprotect_name, ctxt.get_binding('colnames', ())))
ks = cqlsh.cql_unprotect_name(ctxt.get_binding('ksname', None))
cf = cqlsh.cql_unprotect_name(ctxt.get_binding('cfname'))
colnames = cqlsh.get_column_names(ks, cf)
@@ -287,7 +291,7 @@
@cqlsh_syntax_completer('copyOption', 'optnames')
def complete_copy_options(ctxt, cqlsh):
- optnames = map(str.upper, ctxt.get_binding('optnames', ()))
+ optnames = list(map(str.upper, ctxt.get_binding('optnames', ())))
direction = ctxt.get_binding('dir').upper()
if direction == 'FROM':
opts = set(COPY_COMMON_OPTIONS + COPY_FROM_OPTIONS) - set(optnames)
diff --git a/pylib/cqlshlib/displaying.py b/pylib/cqlshlib/displaying.py
index 424d633..ef076f7 100644
--- a/pylib/cqlshlib/displaying.py
+++ b/pylib/cqlshlib/displaying.py
@@ -14,6 +14,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+from __future__ import unicode_literals
+
from collections import defaultdict
RED = '\033[0;1;31m'
@@ -41,7 +43,7 @@
return val
-class FormattedValue:
+class FormattedValue(object):
def __init__(self, strval, coloredval=None, displaywidth=None):
self.strval = strval
diff --git a/pylib/cqlshlib/formatting.py b/pylib/cqlshlib/formatting.py
index 9927aa1..a8ee51d 100644
--- a/pylib/cqlshlib/formatting.py
+++ b/pylib/cqlshlib/formatting.py
@@ -14,6 +14,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+from __future__ import unicode_literals
+
import binascii
import calendar
import datetime
@@ -21,15 +23,17 @@
import os
import re
import sys
-import six
import platform
-import wcwidth
+
+from six import ensure_text
from collections import defaultdict
-from displaying import colorme, get_str, FormattedValue, DEFAULT_VALUE_COLORS, NO_COLOR_MAP
+
from cassandra.cqltypes import EMPTY
from cassandra.util import datetime_from_timestamp
-from util import UTC
+from . import wcwidth
+from .displaying import colorme, get_str, FormattedValue, DEFAULT_VALUE_COLORS, NO_COLOR_MAP
+from .util import UTC
is_win = platform.system() == 'Windows'
@@ -204,7 +208,7 @@
def format_value_default(val, colormap, **_):
- val = str(val)
+ val = ensure_text(str(val))
escapedval = val.replace('\\', '\\\\')
bval = controlchars_re.sub(_show_control_chars, escapedval)
return bval if colormap is NO_COLOR_MAP else color_text(bval, colormap)
@@ -218,7 +222,6 @@
def format_value(val, cqltype, **kwargs):
if val == EMPTY:
return format_value_default('', **kwargs)
-
formatter = get_formatter(val, cqltype)
return formatter(val, cqltype=cqltype, **kwargs)
@@ -236,6 +239,7 @@
return f
return registrator
+
class BlobType(object):
def __init__(self, val):
self.val = val
@@ -243,9 +247,10 @@
def __str__(self):
return str(self.val)
+
@formatter_for('BlobType')
def format_value_blob(val, colormap, **_):
- bval = '0x' + binascii.hexlify(val)
+ bval = ensure_text('0x') + ensure_text(binascii.hexlify(val))
return colorme(bval, colormap, 'blob')
@@ -255,7 +260,7 @@
def format_python_formatted_type(val, colormap, color, quote=False):
- bval = str(val)
+ bval = ensure_text(str(val))
if quote:
bval = "'%s'" % bval
return colorme(bval, colormap, color)
@@ -325,6 +330,7 @@
def format_integer_type(val, colormap, thousands_sep=None, **_):
# base-10 only for now; support others?
bval = format_integer_with_thousands_sep(val, thousands_sep) if thousands_sep else str(val)
+ bval = ensure_text(bval)
return colorme(bval, colormap, 'int')
@@ -359,7 +365,7 @@
if date_time_format.milliseconds_only:
bval = round_microseconds(bval)
else:
- bval = str(val)
+ bval = ensure_text(str(val))
if quote:
bval = "'%s'" % bval
@@ -385,7 +391,7 @@
return '%d' % (seconds * 1000.0)
-microseconds_regex = re.compile("(.*)(?:\.(\d{1,6}))(.*)")
+microseconds_regex = re.compile(r"(.*)(?:\.(\d{1,6}))(.*)")
def round_microseconds(val):
@@ -460,14 +466,14 @@
For example, if we need to read 3 more bytes the first byte will start with 1110.
"""
- first_byte = buf.next()
+ first_byte = next(buf)
if (first_byte >> 7) == 0:
return first_byte
size = number_of_extra_bytes_to_read(first_byte)
retval = first_byte & (0xff >> size)
for i in range(size):
- b = buf.next()
+ b = next(buf)
retval <<= 8
retval |= b & 0xff
@@ -484,15 +490,14 @@
@formatter_for('str')
def format_value_text(val, encoding, colormap, quote=False, **_):
- escapedval = val.replace(u'\\', u'\\\\')
+ escapedval = val.replace('\\', '\\\\')
if quote:
escapedval = escapedval.replace("'", "''")
escapedval = unicode_controlchars_re.sub(_show_control_chars, escapedval)
- bval = escapedval.encode(encoding, 'backslashreplace')
+ bval = escapedval
if quote:
- bval = "'%s'" % bval
-
- return bval if colormap is NO_COLOR_MAP else color_text(bval, colormap, wcwidth.wcswidth(bval.decode(encoding)))
+ bval = "'{}'".format(bval)
+ return bval if colormap is NO_COLOR_MAP else color_text(bval, colormap, wcwidth.wcswidth(bval))
# name alias
@@ -539,7 +544,7 @@
@formatter_for('set')
def format_value_set(val, cqltype, encoding, colormap, date_time_format, float_precision, nullval,
decimal_sep, thousands_sep, boolean_styles, **_):
- return format_simple_collection(sorted(val), cqltype, '{', '}', encoding, colormap,
+ return format_simple_collection(val, cqltype, '{', '}', encoding, colormap,
date_time_format, float_precision, nullval,
decimal_sep, thousands_sep, boolean_styles)
@@ -591,7 +596,7 @@
def format_field_name(name):
return format_value_text(name, encoding=encoding, colormap=colormap, quote=False)
- subs = [(format_field_name(k), format_field_value(v, t)) for ((k, v), t) in zip(val._asdict().items(),
+ subs = [(format_field_name(k), format_field_value(v, t)) for ((k, v), t) in zip(list(val._asdict().items()),
cqltype.sub_types)]
bval = '{' + ', '.join(get_str(k) + ': ' + get_str(v) for (k, v) in subs) + '}'
if colormap is NO_COLOR_MAP:
diff --git a/pylib/cqlshlib/pylexotron.py b/pylib/cqlshlib/pylexotron.py
index 7b11eac..69f31dc 100644
--- a/pylib/cqlshlib/pylexotron.py
+++ b/pylib/cqlshlib/pylexotron.py
@@ -15,7 +15,8 @@
# limitations under the License.
import re
-from .saferscanner import SaferScanner
+
+from cqlshlib.saferscanner import SaferScanner
class LexingError(Exception):
@@ -107,15 +108,6 @@
return ' '.join([t[1] for t in tokens])
# low end of span for first token, to high end of span for last token
orig_text = orig[tokens[0][2][0]:tokens[-1][2][1]]
-
- # Convert all unicode tokens to ascii, where possible. This
- # helps avoid problems with performing unicode-incompatible
- # operations on tokens (like .lower()). See CASSANDRA-9083
- # for one example of this.
- try:
- orig_text = orig_text.encode('ascii')
- except UnicodeEncodeError:
- pass
return orig_text
def __repr__(self):
@@ -146,7 +138,7 @@
except KeyError:
return False
if debugging:
- print "Trying completer %r with %r" % (completer, ctxt)
+ print("Trying completer %r with %r" % (completer, ctxt))
try:
new_compls = completer(ctxt)
except Exception:
@@ -155,7 +147,7 @@
traceback.print_exc()
return False
if debugging:
- print "got %r" % (new_compls,)
+ print("got %r" % (new_compls,))
completions.update(new_compls)
return True
@@ -284,7 +276,7 @@
try:
terminal_matcher.__init__(self, eval(text))
except SyntaxError:
- print "bad syntax %r" % (text,)
+ print("bad syntax %r" % (text,))
def match(self, ctxt, completions):
if ctxt.remainder:
@@ -359,7 +351,7 @@
(r'[@()|?*;]', lambda s, t: t),
(r'\s+', None),
(r'#[^\n]*', None),
- ], re.I | re.S)
+ ], re.I | re.S | re.U)
def __init__(self):
self.ruleset = {}
@@ -382,7 +374,7 @@
tokeniter = iter(tokens)
for t in tokeniter:
if isinstance(t, tuple) and t[0] in ('reference', 'junk'):
- assign = tokeniter.next()
+ assign = next(tokeniter)
if assign != '::=':
raise ValueError('Unexpected token %r; expected "::="' % (assign,))
name = t[1]
@@ -405,7 +397,7 @@
@classmethod
def read_rule_tokens_until(cls, endtoks, tokeniter):
- if isinstance(endtoks, basestring):
+ if isinstance(endtoks, str):
endtoks = (endtoks,)
counttarget = None
if isinstance(endtoks, int):
@@ -419,7 +411,7 @@
if t in endtoks:
if len(mybranches) == 1:
return cls.mkrule(mybranches[0])
- return choice(map(cls.mkrule, mybranches))
+ return choice(list(map(cls.mkrule, mybranches)))
if isinstance(t, tuple):
if t[0] == 'reference':
t = rule_reference(t[1])
@@ -441,7 +433,7 @@
elif t == '*':
t = repeat(myrules.pop(-1))
elif t == '@':
- x = tokeniter.next()
+ x = next(tokeniter)
if not isinstance(x, tuple) or x[0] != 'litstring':
raise ValueError("Unexpected token %r following '@'" % (x,))
t = case_match(x[1])
@@ -455,7 +447,7 @@
if countsofar == counttarget:
if len(mybranches) == 1:
return cls.mkrule(mybranches[0])
- return choice(map(cls.mkrule, mybranches))
+ return choice(list(map(cls.mkrule, mybranches)))
raise ValueError('Unexpected end of rule tokens')
def append_rules(self, rulestr):
@@ -474,7 +466,7 @@
return None
return lambda s, t: (name, t, s.match.span())
regexes = [(p.pattern(), make_handler(name)) for (name, p) in self.terminals]
- return SaferScanner(regexes, re.I | re.S).scan
+ return SaferScanner(regexes, re.I | re.S | re.U).scan
def lex(self, text):
if self.scanner is None:
diff --git a/pylib/cqlshlib/saferscanner.py b/pylib/cqlshlib/saferscanner.py
index 2a05608..8949321 100644
--- a/pylib/cqlshlib/saferscanner.py
+++ b/pylib/cqlshlib/saferscanner.py
@@ -19,24 +19,12 @@
# regex in-pattern flags. Any of those can break correct operation of Scanner.
import re
+import six
from sre_constants import BRANCH, SUBPATTERN, GROUPREF, GROUPREF_IGNORE, GROUPREF_EXISTS
+from sys import version_info
-class SaferScanner(re.Scanner):
-
- def __init__(self, lexicon, flags=0):
- self.lexicon = lexicon
- p = []
- s = re.sre_parse.Pattern()
- s.flags = flags
- for phrase, action in lexicon:
- p.append(re.sre_parse.SubPattern(s, [
- (SUBPATTERN, (len(p) + 1, self.subpat(phrase, flags))),
- ]))
- s.groups = len(p) + 1
- p = re.sre_parse.SubPattern(s, [(BRANCH, (None, p))])
- self.p = p
- self.scanner = re.sre_compile.compile(p)
+class SaferScannerBase(re.Scanner):
@classmethod
def subpat(cls, phrase, flags):
@@ -60,3 +48,56 @@
if sub.pattern.flags ^ flags:
raise ValueError("RE flag setting not allowed in SaferScanner lexicon (%s)" % (bin(sub.pattern.flags),))
return re.sre_parse.SubPattern(sub.pattern, scrubbedsub)
+
+
+class Py2SaferScanner(SaferScannerBase):
+
+ def __init__(self, lexicon, flags=0):
+ self.lexicon = lexicon
+ p = []
+ s = re.sre_parse.Pattern()
+ s.flags = flags
+ for phrase, action in lexicon:
+ p.append(re.sre_parse.SubPattern(s, [
+ (SUBPATTERN, (len(p) + 1, self.subpat(phrase, flags))),
+ ]))
+ s.groups = len(p) + 1
+ p = re.sre_parse.SubPattern(s, [(BRANCH, (None, p))])
+ self.p = p
+ self.scanner = re.sre_compile.compile(p)
+
+
+class Py36SaferScanner(SaferScannerBase):
+
+ def __init__(self, lexicon, flags=0):
+ self.lexicon = lexicon
+ p = []
+ s = re.sre_parse.Pattern()
+ s.flags = flags
+ for phrase, action in lexicon:
+ gid = s.opengroup()
+ p.append(re.sre_parse.SubPattern(s, [(SUBPATTERN, (gid, 0, 0, re.sre_parse.parse(phrase, flags))), ]))
+ s.closegroup(gid, p[-1])
+ p = re.sre_parse.SubPattern(s, [(BRANCH, (None, p))])
+ self.p = p
+ self.scanner = re.sre_compile.compile(p)
+
+
+class Py38SaferScanner(SaferScannerBase):
+
+ def __init__(self, lexicon, flags=0):
+ self.lexicon = lexicon
+ p = []
+ s = re.sre_parse.State()
+ s.flags = flags
+ for phrase, action in lexicon:
+ gid = s.opengroup()
+ p.append(re.sre_parse.SubPattern(s, [(SUBPATTERN, (gid, 0, 0, re.sre_parse.parse(phrase, flags))), ]))
+ s.closegroup(gid, p[-1])
+ p = re.sre_parse.SubPattern(s, [(BRANCH, (None, p))])
+ self.p = p
+ self.scanner = re.sre_compile.compile(p)
+
+
+SaferScanner = Py36SaferScanner if six.PY3 else Py2SaferScanner
+SaferScanner = Py38SaferScanner if version_info >= (3, 8) else SaferScanner
diff --git a/pylib/cqlshlib/sslhandling.py b/pylib/cqlshlib/sslhandling.py
index 8765ffa..b6a1369 100644
--- a/pylib/cqlshlib/sslhandling.py
+++ b/pylib/cqlshlib/sslhandling.py
@@ -16,9 +16,10 @@
import os
import sys
-import ConfigParser
import ssl
+from six.moves import configparser
+
def ssl_settings(host, config_file, env=os.environ):
"""
@@ -38,13 +39,13 @@
either in the config file or as an environment variable.
Environment variables override any options set in cqlsh config file.
"""
- configs = ConfigParser.SafeConfigParser()
+ configs = configparser.SafeConfigParser()
configs.read(config_file)
def get_option(section, option):
try:
return configs.get(section, option)
- except ConfigParser.Error:
+ except configparser.Error:
return None
ssl_validate = env.get('SSL_VALIDATE')
diff --git a/pylib/cqlshlib/test/__init__.py b/pylib/cqlshlib/test/__init__.py
index ba8f373..4bb037e 100644
--- a/pylib/cqlshlib/test/__init__.py
+++ b/pylib/cqlshlib/test/__init__.py
@@ -15,6 +15,3 @@
# limitations under the License.
from .cassconnect import create_db, remove_db
-
-setUp = create_db
-tearDown = remove_db
diff --git a/pylib/cqlshlib/test/ansi_colors.py b/pylib/cqlshlib/test/ansi_colors.py
index b0bc738..9fc3411 100644
--- a/pylib/cqlshlib/test/ansi_colors.py
+++ b/pylib/cqlshlib/test/ansi_colors.py
@@ -14,31 +14,35 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+from __future__ import unicode_literals
+
import re
+import six
-LIGHT = 010
+LIGHT = 0o10
-ansi_CSI = '\033['
+
+ansi_CSI = '\x1b['
ansi_seq = re.compile(re.escape(ansi_CSI) + r'(?P<params>[\x20-\x3f]*)(?P<final>[\x40-\x7e])')
ansi_cmd_SGR = 'm' # set graphics rendition
color_defs = (
(000, 'k', 'black'),
- (001, 'r', 'dark red'),
- (002, 'g', 'dark green'),
- (003, 'w', 'brown', 'dark yellow'),
- (004, 'b', 'dark blue'),
- (005, 'm', 'dark magenta', 'dark purple'),
- (006, 'c', 'dark cyan'),
- (007, 'n', 'light grey', 'light gray', 'neutral', 'dark white'),
- (010, 'B', 'dark grey', 'dark gray', 'light black'),
- (011, 'R', 'red', 'light red'),
- (012, 'G', 'green', 'light green'),
- (013, 'Y', 'yellow', 'light yellow'),
- (014, 'B', 'blue', 'light blue'),
- (015, 'M', 'magenta', 'purple', 'light magenta', 'light purple'),
- (016, 'C', 'cyan', 'light cyan'),
- (017, 'W', 'white', 'light white'),
+ (0o01, 'r', 'dark red'),
+ (0o02, 'g', 'dark green'),
+ (0o03, 'w', 'brown', 'dark yellow'),
+ (0o04, 'b', 'dark blue'),
+ (0o05, 'm', 'dark magenta', 'dark purple'),
+ (0o06, 'c', 'dark cyan'),
+ (0o07, 'n', 'light grey', 'light gray', 'neutral', 'dark white'),
+ (0o10, 'B', 'dark grey', 'dark gray', 'light black'),
+ (0o11, 'R', 'red', 'light red'),
+ (0o12, 'G', 'green', 'light green'),
+ (0o13, 'Y', 'yellow', 'light yellow'),
+ (0o14, 'B', 'blue', 'light blue'),
+ (0o15, 'M', 'magenta', 'purple', 'light magenta', 'light purple'),
+ (0o16, 'C', 'cyan', 'light cyan'),
+ (0o17, 'W', 'white', 'light white'),
)
colors_by_num = {}
@@ -61,7 +65,7 @@
for c in nameset:
colors_by_name[c] = colorcode
-class ColoredChar:
+class ColoredChar(object):
def __init__(self, c, colorcode):
self.c = c
self._colorcode = colorcode
@@ -76,8 +80,8 @@
return getattr(self.c, name)
def ansi_color(self):
- clr = str(30 + (07 & self._colorcode))
- if self._colorcode & 010:
+ clr = str(30 + (0o7 & self._colorcode))
+ if self._colorcode & 0o10:
clr = '1;' + clr
return clr
@@ -100,11 +104,11 @@
def colortag(self):
return lookup_letter_from_code(self._colorcode)
-class ColoredText:
+class ColoredText(object):
def __init__(self, source=''):
- if isinstance(source, basestring):
+ if isinstance(source, six.text_type):
plain, colors = self.parse_ansi_colors(source)
- self.chars = map(ColoredChar, plain, colors)
+ self.chars = list(map(ColoredChar, plain, colors))
else:
# expected that source is an iterable of ColoredChars (or duck-typed as such)
self.chars = tuple(source)
@@ -149,7 +153,7 @@
@staticmethod
def parse_sgr_param(curclr, paramstr):
oldclr = curclr
- args = map(int, paramstr.split(';'))
+ args = list(map(int, paramstr.split(';')))
for a in args:
if a == 0:
curclr = lookup_colorcode('neutral')
diff --git a/pylib/cqlshlib/test/basecase.py b/pylib/cqlshlib/test/basecase.py
index d393769..f398511 100644
--- a/pylib/cqlshlib/test/basecase.py
+++ b/pylib/cqlshlib/test/basecase.py
@@ -17,7 +17,7 @@
import os
import sys
import logging
-from itertools import izip
+
from os.path import dirname, join, normpath
cqlshlog = logging.getLogger('test_cqlsh')
@@ -46,13 +46,27 @@
class BaseTestCase(unittest.TestCase):
def assertNicelyFormattedTableHeader(self, line, msg=None):
- return self.assertRegexpMatches(line, r'^ +\w+( +\| \w+)*\s*$', msg=msg)
+ return self.assertRegex(line, r'^ +\w+( +\| \w+)*\s*$', msg=msg)
def assertNicelyFormattedTableRule(self, line, msg=None):
- return self.assertRegexpMatches(line, r'^-+(\+-+)*\s*$', msg=msg)
+ return self.assertRegex(line, r'^-+(\+-+)*\s*$', msg=msg)
def assertNicelyFormattedTableData(self, line, msg=None):
- return self.assertRegexpMatches(line, r'^ .* \| ', msg=msg)
+ return self.assertRegex(line, r'^ .* \| ', msg=msg)
+
+ def assertRegex(self, text, regex, msg=None):
+ """Call assertRegexpMatches() if in Python 2"""
+ if hasattr(unittest.TestCase, 'assertRegex'):
+ return super().assertRegex(text, regex, msg)
+ else:
+ return self.assertRegexpMatches(text, regex, msg)
+
+ def assertNotRegex(self, text, regex, msg=None):
+ """Call assertNotRegexpMatches() if in Python 2"""
+ if hasattr(unittest.TestCase, 'assertNotRegex'):
+ return super().assertNotRegex(text, regex, msg)
+ else:
+ return self.assertNotRegexpMatches(text, regex, msg)
def dedent(s):
lines = [ln.rstrip() for ln in s.splitlines()]
@@ -63,4 +77,4 @@
return '\n'.join(line[minspace:] for line in lines)
def at_a_time(i, num):
- return izip(*([iter(i)] * num))
+ return zip(*([iter(i)] * num))
diff --git a/pylib/cqlshlib/test/cassconnect.py b/pylib/cqlshlib/test/cassconnect.py
index 501850c..c9571f2 100644
--- a/pylib/cqlshlib/test/cassconnect.py
+++ b/pylib/cqlshlib/test/cassconnect.py
@@ -14,12 +14,13 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import with_statement
import contextlib
-import tempfile
+import io
import os.path
-from .basecase import cql, cqlsh, cqlshlog, TEST_HOST, TEST_PORT, rundir, policy, quote_name
+import tempfile
+
+from .basecase import TEST_HOST, TEST_PORT, cql, cqlsh, cqlshlog, policy, quote_name, rundir
from .run_cqlsh import run_cqlsh, call_cqlsh
test_keyspace_init = os.path.join(rundir, 'test_keyspace_init.cql')
@@ -43,12 +44,12 @@
return os.path.basename(tempfile.mktemp(prefix='CqlshTests_'))
def create_keyspace(cursor):
- ksname = make_ks_name()
+ ksname = make_ks_name().lower()
qksname = quote_name(ksname)
cursor.execute('''
CREATE KEYSPACE %s WITH replication =
{'class': 'SimpleStrategy', 'replication_factor': 1};
- ''' % quote_name(ksname))
+ ''' % qksname)
cursor.execute('USE %s;' % qksname)
TEST_KEYSPACES_CREATED.append(ksname)
return ksname
@@ -63,11 +64,11 @@
def execute_cql_commands(cursor, source, logprefix='INIT: '):
for cql in split_cql_commands(source):
- cqlshlog.debug(logprefix + cql)
+ cqlshlog.debug((logprefix + cql).encode("utf-8"))
cursor.execute(cql)
def execute_cql_file(cursor, fname):
- with open(fname) as f:
+ with io.open(fname, "r", encoding="utf-8") as f:
return execute_cql_commands(cursor, f.read())
def create_db():
@@ -116,8 +117,6 @@
c = conn.connect(ks)
# increase default timeout to fix flacky tests, see CASSANDRA-12481
c.default_timeout = 60.0
- # if ks is not None:
- # c.execute('USE %s;' % quote_name(c, ks))
yield c
finally:
conn.shutdown()
@@ -137,4 +136,6 @@
def testcall_cqlsh(keyspace=None, **kwargs):
if keyspace is None:
keyspace = get_keyspace()
+ if ('input' in kwargs.keys() and isinstance(kwargs['input'], str)):
+ kwargs['input'] = kwargs['input'].encode('utf-8')
return call_cqlsh(keyspace=keyspace, **kwargs)
diff --git a/pylib/cqlshlib/test/run_cqlsh.py b/pylib/cqlshlib/test/run_cqlsh.py
index fa010fe..1c90d58 100644
--- a/pylib/cqlshlib/test/run_cqlsh.py
+++ b/pylib/cqlshlib/test/run_cqlsh.py
@@ -16,6 +16,8 @@
# NOTE: this testing tool is *nix specific
+from __future__ import unicode_literals
+
import os
import sys
import re
@@ -32,7 +34,7 @@
return sys.platform in ("cygwin", "win32")
if is_win():
- from winpty import WinPty
+ from .winpty import WinPty
DEFAULT_PREFIX = ''
else:
import pty
@@ -41,6 +43,24 @@
DEFAULT_CQLSH_PROMPT = DEFAULT_PREFIX + '(\S+@)?cqlsh(:\S+)?> '
DEFAULT_CQLSH_TERM = 'xterm'
+def get_smm_sequence(term='xterm'):
+ """
+ Return the set meta mode (smm) sequence, if any.
+ On more recent Linux systems, xterm emits the smm sequence
+ before each prompt.
+ """
+ result = ''
+ if not is_win():
+ tput_proc = subprocess.Popen(['tput', '-T{}'.format(term), 'smm'], stdout=subprocess.PIPE)
+ tput_stdout = tput_proc.communicate()[0]
+ if (tput_stdout and (tput_stdout != b'')):
+ result = tput_stdout
+ if isinstance(result, bytes):
+ result = result.decode("utf-8")
+ return result
+
+DEFAULT_SMM_SEQUENCE = get_smm_sequence()
+
cqlshlog = basecase.cqlshlog
def set_controlling_pty(master, slave):
@@ -103,7 +123,7 @@
if is_win():
try:
import eventlet
- except ImportError, e:
+ except ImportError as e:
sys.exit("evenlet library required to run cqlshlib tests on Windows")
def timing_out(seconds):
@@ -171,22 +191,33 @@
return self.proc.wait()
def send_tty(self, data):
+ if not isinstance(data, bytes):
+ data = data.encode("utf-8")
os.write(self.childpty, data)
def send_pipe(self, data):
self.proc.stdin.write(data)
def read_tty(self, blksize, timeout=None):
- return os.read(self.childpty, blksize)
+ buf = os.read(self.childpty, blksize)
+ if isinstance(buf, bytes):
+ buf = buf.decode("utf-8")
+ return buf
def read_pipe(self, blksize, timeout=None):
- return self.proc.stdout.read(blksize)
+ buf = self.proc.stdout.read(blksize)
+ if isinstance(buf, bytes):
+ buf = buf.decode("utf-8")
+ return buf
def read_winpty(self, blksize, timeout=None):
- return self.winpty.read(blksize, timeout)
+ buf = self.winpty.read(blksize, timeout)
+ if isinstance(buf, bytes):
+ buf = buf.decode("utf-8")
+ return buf
def read_until(self, until, blksize=4096, timeout=None,
- flags=0, ptty_timeout=None):
+ flags=0, ptty_timeout=None, replace=[]):
if not isinstance(until, re._pattern_type):
until = re.compile(until, flags)
@@ -196,6 +227,9 @@
with timing_out(timeout):
while True:
val = self.read(blksize, ptty_timeout)
+ for replace_target in replace:
+ if (replace_target != ''):
+ val = val.replace(replace_target, '')
cqlshlog.debug("read %r from subproc" % (val,))
if val == '':
raise EOFError("'until' pattern %r not found" % (until.pattern,))
@@ -252,16 +286,21 @@
env.setdefault('TERM', 'xterm')
env.setdefault('CQLSH_NO_BUNDLED', os.environ.get('CQLSH_NO_BUNDLED', ''))
env.setdefault('PYTHONPATH', os.environ.get('PYTHONPATH', ''))
+ coverage = False
+ if ('CQLSH_COVERAGE' in env.keys()):
+ coverage = True
args = tuple(args) + (host, str(port))
if cqlver is not None:
args += ('--cqlversion', str(cqlver))
if keyspace is not None:
- args += ('--keyspace', keyspace)
+ args += ('--keyspace', keyspace.lower())
if tty and is_win():
args += ('--tty',)
args += ('--encoding', 'utf-8')
if win_force_colors:
args += ('--color',)
+ if coverage:
+ args += ('--coverage',)
self.keyspace = keyspace
ProcRunner.__init__(self, path, tty=tty, args=args, env=env, **kwargs)
self.prompt = prompt
@@ -270,8 +309,8 @@
else:
self.output_header = self.read_to_next_prompt()
- def read_to_next_prompt(self):
- return self.read_until(self.prompt, timeout=10.0, ptty_timeout=3)
+ def read_to_next_prompt(self, timeout=10.0):
+ return self.read_until(self.prompt, timeout=timeout, ptty_timeout=3, replace=[DEFAULT_SMM_SEQUENCE,])
def read_up_to_timeout(self, timeout, blksize=4096):
output = ProcRunner.read_up_to_timeout(self, timeout, blksize=blksize)
@@ -309,4 +348,6 @@
c = CqlshRunner(**kwargs)
output, _ = c.proc.communicate(proginput)
result = c.close()
+ if isinstance(output, bytes):
+ output = output.decode("utf-8")
return output, result
diff --git a/pylib/cqlshlib/test/test_copyutil.py b/pylib/cqlshlib/test/test_copyutil.py
new file mode 100644
index 0000000..18b167a
--- /dev/null
+++ b/pylib/cqlshlib/test/test_copyutil.py
@@ -0,0 +1,116 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# to configure behavior, define $CQL_TEST_HOST to the destination address
+# and $CQL_TEST_PORT to the associated port.
+
+
+import unittest
+
+from cassandra.metadata import MIN_LONG, Murmur3Token, TokenMap
+from cassandra.policies import SimpleConvictionPolicy
+from cassandra.pool import Host
+from unittest.mock import Mock
+
+from cqlshlib.copyutil import ExportTask
+
+
+class CopyTaskTest(unittest.TestCase):
+
+ def setUp(self):
+ # set up default test data
+ self.ks = 'testks'
+ self.table = 'testtable'
+ self.columns = ['a', 'b']
+ self.fname = 'test_fname'
+ self.opts = {}
+ self.protocol_version = 0
+ self.config_file = 'test_config'
+ self.hosts = [
+ Host('10.0.0.1', SimpleConvictionPolicy, 9000),
+ Host('10.0.0.2', SimpleConvictionPolicy, 9000),
+ Host('10.0.0.3', SimpleConvictionPolicy, 9000),
+ Host('10.0.0.4', SimpleConvictionPolicy, 9000)
+ ]
+
+ def mock_shell(self):
+ """
+ Set up a mock Shell so we can unit test ExportTask internals
+ """
+ shell = Mock()
+ shell.conn = Mock()
+ shell.conn.get_control_connection_host.return_value = self.hosts[0]
+ shell.get_column_names.return_value = self.columns
+ shell.debug = False
+ return shell
+
+
+class TestExportTask(CopyTaskTest):
+
+ def _test_get_ranges_murmur3_base(self, opts, expected_ranges):
+ """
+ Set up a mock shell with a simple token map to test the ExportTask get_ranges function.
+ """
+ shell = self.mock_shell()
+ shell.conn.metadata.partitioner = 'Murmur3Partitioner'
+ # token range for a cluster of 4 nodes with replication factor 3
+ shell.get_ring.return_value = {
+ Murmur3Token(-9223372036854775808): self.hosts[0:3],
+ Murmur3Token(-4611686018427387904): self.hosts[1:4],
+ Murmur3Token(0): [self.hosts[2], self.hosts[3], self.hosts[0]],
+ Murmur3Token(4611686018427387904): [self.hosts[3], self.hosts[0], self.hosts[1]]
+ }
+ # merge override options with standard options
+ overridden_opts = dict(self.opts)
+ for k,v in opts.items():
+ overridden_opts[k] = v
+ export_task = ExportTask(shell, self.ks, self.table, self.columns, self.fname, overridden_opts, self.protocol_version, self.config_file)
+ assert export_task.get_ranges() == expected_ranges
+
+ def test_get_ranges_murmur3(self):
+ """
+ Test behavior of ExportTask internal get_ranges function
+ """
+
+ # return empty dict and print error if begin_token < min_token
+ self._test_get_ranges_murmur3_base({'begintoken': MIN_LONG - 1}, {})
+
+ # return empty dict and print error if begin_token < min_token
+ self._test_get_ranges_murmur3_base({'begintoken': 1, 'endtoken': -1}, {})
+
+ # simple case of a single range
+ expected_ranges = {(1,2): {'hosts': ('10.0.0.4', '10.0.0.1', '10.0.0.2'), 'attempts': 0, 'rows': 0, 'workerno': -1}}
+ self._test_get_ranges_murmur3_base({'begintoken': 1, 'endtoken': 2}, expected_ranges)
+
+ # simple case of two contiguous ranges
+ expected_ranges = {
+ (-4611686018427387903,0): {'hosts': ('10.0.0.3', '10.0.0.4', '10.0.0.1'), 'attempts': 0, 'rows': 0, 'workerno': -1},
+ (0,1): {'hosts': ('10.0.0.4', '10.0.0.1', '10.0.0.2'), 'attempts': 0, 'rows': 0, 'workerno': -1}
+ }
+ self._test_get_ranges_murmur3_base({'begintoken': -4611686018427387903, 'endtoken': 1}, expected_ranges)
+
+ # specify a begintoken only (endtoken defaults to None)
+ expected_ranges = {
+ (4611686018427387905,None): {'hosts': ('10.0.0.1', '10.0.0.2', '10.0.0.3'), 'attempts': 0, 'rows': 0, 'workerno': -1}
+ }
+ self._test_get_ranges_murmur3_base({'begintoken': 4611686018427387905}, expected_ranges)
+
+ # specify an endtoken only (begintoken defaults to None)
+ expected_ranges = {
+ (None, MIN_LONG + 1): {'hosts': ('10.0.0.2', '10.0.0.3', '10.0.0.4'), 'attempts': 0, 'rows': 0, 'workerno': -1}
+ }
+ self._test_get_ranges_murmur3_base({'endtoken': MIN_LONG + 1}, expected_ranges)
+
diff --git a/pylib/cqlshlib/test/test_cql_parsing.py b/pylib/cqlshlib/test/test_cql_parsing.py
index ad60c9b..10be99f 100644
--- a/pylib/cqlshlib/test/test_cql_parsing.py
+++ b/pylib/cqlshlib/test/test_cql_parsing.py
@@ -15,12 +15,12 @@
# limitations under the License.
# to configure behavior, define $CQL_TEST_HOST to the destination address
-# for Thrift connections, and $CQL_TEST_PORT to the associated port.
+# and $CQL_TEST_PORT to the associated port.
from unittest import TestCase
from operator import itemgetter
-from ..cql3handling import CqlRuleSet
+from cqlshlib.cql3handling import CqlRuleSet
class TestCqlParsing(TestCase):
diff --git a/pylib/cqlshlib/test/test_cqlsh_commands.py b/pylib/cqlshlib/test/test_cqlsh_commands.py
deleted file mode 100644
index 0b12882..0000000
--- a/pylib/cqlshlib/test/test_cqlsh_commands.py
+++ /dev/null
@@ -1,39 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# to configure behavior, define $CQL_TEST_HOST to the destination address
-# for Thrift connections, and $CQL_TEST_PORT to the associated port.
-
-from .basecase import BaseTestCase, cqlsh
-
-class TestCqlshCommands(BaseTestCase):
- def setUp(self):
- pass
-
- def tearDown(self):
- pass
-
- def test_show(self):
- pass
-
- def test_describe(self):
- pass
-
- def test_exit(self):
- pass
-
- def test_help(self):
- pass
diff --git a/pylib/cqlshlib/test/test_cqlsh_completion.py b/pylib/cqlshlib/test/test_cqlsh_completion.py
index df4f7e8..d432f22 100644
--- a/pylib/cqlshlib/test/test_cqlsh_completion.py
+++ b/pylib/cqlshlib/test/test_cqlsh_completion.py
@@ -15,13 +15,15 @@
# limitations under the License.
# to configure behavior, define $CQL_TEST_HOST to the destination address
-# for Thrift connections, and $CQL_TEST_PORT to the associated port.
+# and $CQL_TEST_PORT to the associated port.
-from __future__ import with_statement
+import locale
+import os
import re
-from .basecase import BaseTestCase, cqlsh
-from .cassconnect import testrun_cqlsh
+from .basecase import BaseTestCase, cqlsh, cqlshlog
+from .cassconnect import create_db, remove_db, testrun_cqlsh
+from .run_cqlsh import TimeoutError
import unittest
import sys
@@ -41,8 +43,22 @@
@unittest.skipIf(sys.platform == "win32", 'Tab completion tests not supported on Windows')
class CqlshCompletionCase(BaseTestCase):
+ @classmethod
+ def setUpClass(cls):
+ create_db()
+
+ @classmethod
+ def tearDownClass(cls):
+ remove_db()
+
def setUp(self):
- self.cqlsh_runner = testrun_cqlsh(cqlver=None, env={'COLUMNS': '100000'})
+ env = os.environ
+ env['COLUMNS'] = '100000'
+ if (locale.getpreferredencoding() != 'UTF-8'):
+ env['LC_CTYPE'] = 'en_US.utf8'
+ if ('PATH' in os.environ.keys()):
+ env['PATH'] = os.environ['PATH']
+ self.cqlsh_runner = testrun_cqlsh(cqlver=None, env=env)
self.cqlsh = self.cqlsh_runner.__enter__()
def tearDown(self):
@@ -81,14 +97,14 @@
prompt_regex = self.cqlsh.prompt.lstrip() + re.escape(inputstring)
msg = ('Double-tab completion '
'does not print prompt for input "{}"'.format(inputstring))
- self.assertRegexpMatches(choice_lines[-1], prompt_regex, msg=msg)
+ self.assertRegex(choice_lines[-1], prompt_regex, msg=msg)
choice_lines = [line.strip() for line in choice_lines[:-1]]
choice_lines = [line for line in choice_lines if line]
if split_completed_lines:
- completed_lines = map(set, (completion_separation_re.split(line.strip())
- for line in choice_lines))
+ completed_lines = list(map(set, (completion_separation_re.split(line.strip())
+ for line in choice_lines)))
if not completed_lines:
return set()
@@ -132,8 +148,14 @@
other_choices_ok=other_choices_ok,
split_completed_lines=split_completed_lines)
finally:
- self.cqlsh.send(CTRL_C) # cancel any current line
- self.cqlsh.read_to_next_prompt()
+ try:
+ self.cqlsh.send(CTRL_C) # cancel any current line
+ self.cqlsh.read_to_next_prompt(timeout=1.0)
+ except TimeoutError:
+ # retry once
+ self.cqlsh.send(CTRL_C)
+ self.cqlsh.read_to_next_prompt(timeout=10.0)
+
def strategies(self):
return self.module.CqlRuleSet.replication_strategies
@@ -345,8 +367,6 @@
choices=[',', 'WHERE'])
self.trycompletions("UPDATE empty_table SET lonelycol = 'eggs' WHERE ",
choices=['TOKEN(', 'lonelykey'])
- self.trycompletions("UPDATE empty_table SET lonelycol = 'eggs' WHERE ",
- choices=['TOKEN(', 'lonelykey'])
self.trycompletions("UPDATE empty_table SET lonelycol = 'eggs' WHERE lonel",
immediate='ykey ')
@@ -392,7 +412,7 @@
'twenty_rows_composite_table',
'utf8_with_special_chars',
'system_traces.', 'songs',
- '"' + self.cqlsh.keyspace + '".'],
+ self.cqlsh.keyspace + '.'],
other_choices_ok=True)
self.trycompletions('DELETE FROM ',
@@ -407,7 +427,7 @@
'system_traces.', 'songs',
'system_auth.', 'system_distributed.',
'system_schema.', 'system_traces.',
- '"' + self.cqlsh.keyspace + '".'],
+ self.cqlsh.keyspace + '.'],
other_choices_ok=True)
self.trycompletions('DELETE FROM twenty_rows_composite_table ',
choices=['USING', 'WHERE'])
@@ -529,13 +549,13 @@
self.trycompletions('DROP K', immediate='EYSPACE ')
quoted_keyspace = '"' + self.cqlsh.keyspace + '"'
self.trycompletions('DROP KEYSPACE ',
- choices=['IF', quoted_keyspace])
+ choices=['IF', self.cqlsh.keyspace])
self.trycompletions('DROP KEYSPACE ' + quoted_keyspace,
choices=[';'])
self.trycompletions('DROP KEYSPACE I',
- immediate='F EXISTS ' + quoted_keyspace + ';')
+ immediate='F EXISTS ' + self.cqlsh.keyspace + ' ;')
def create_columnfamily_table_template(self, name):
"""Parameterized test for CREATE COLUMNFAMILY and CREATE TABLE. Since
@@ -544,11 +564,11 @@
prefix = 'CREATE ' + name + ' '
quoted_keyspace = '"' + self.cqlsh.keyspace + '"'
self.trycompletions(prefix + '',
- choices=['IF', quoted_keyspace, '<new_table_name>'])
+ choices=['IF', self.cqlsh.keyspace, '<new_table_name>'])
self.trycompletions(prefix + 'IF ',
immediate='NOT EXISTS ')
self.trycompletions(prefix + 'IF NOT EXISTS ',
- choices=['<new_table_name>', quoted_keyspace])
+ choices=['<new_table_name>', self.cqlsh.keyspace])
self.trycompletions(prefix + 'IF NOT EXISTS new_table ',
immediate='( ')
@@ -589,23 +609,21 @@
self.trycompletions(prefix + ' new_table (col_a int PRIMARY KEY) WITH ',
choices=['bloom_filter_fp_chance', 'compaction',
'compression',
- 'dclocal_read_repair_chance',
'default_time_to_live', 'gc_grace_seconds',
'max_index_interval',
'memtable_flush_period_in_ms',
- 'read_repair_chance', 'CLUSTERING',
+ 'CLUSTERING',
'COMPACT', 'caching', 'comment',
- 'min_index_interval', 'speculative_retry', 'cdc'])
+ 'min_index_interval', 'speculative_retry', 'additional_write_policy', 'cdc', 'read_repair'])
self.trycompletions(prefix + ' new_table (col_a int PRIMARY KEY) WITH ',
choices=['bloom_filter_fp_chance', 'compaction',
'compression',
- 'dclocal_read_repair_chance',
'default_time_to_live', 'gc_grace_seconds',
'max_index_interval',
'memtable_flush_period_in_ms',
- 'read_repair_chance', 'CLUSTERING',
+ 'CLUSTERING',
'COMPACT', 'caching', 'comment',
- 'min_index_interval', 'speculative_retry', 'cdc'])
+ 'min_index_interval', 'speculative_retry', 'additional_write_policy', 'cdc', 'read_repair'])
self.trycompletions(prefix + ' new_table (col_a int PRIMARY KEY) WITH bloom_filter_fp_chance ',
immediate='= ')
self.trycompletions(prefix + ' new_table (col_a int PRIMARY KEY) WITH bloom_filter_fp_chance = ',
@@ -647,13 +665,12 @@
+ "{'class': 'SizeTieredCompactionStrategy'} AND ",
choices=['bloom_filter_fp_chance', 'compaction',
'compression',
- 'dclocal_read_repair_chance',
'default_time_to_live', 'gc_grace_seconds',
'max_index_interval',
'memtable_flush_period_in_ms',
- 'read_repair_chance', 'CLUSTERING',
+ 'CLUSTERING',
'COMPACT', 'caching', 'comment',
- 'min_index_interval', 'speculative_retry', 'cdc'])
+ 'min_index_interval', 'speculative_retry', 'additional_write_policy', 'cdc', 'read_repair'])
self.trycompletions(prefix + " new_table (col_a int PRIMARY KEY) WITH compaction = "
+ "{'class': 'DateTieredCompactionStrategy', '",
choices=['base_time_seconds', 'max_sstable_age_days',
@@ -697,7 +714,7 @@
'utf8_with_special_chars',
'system_traces.', 'songs',
'system_distributed.',
- '"' + self.cqlsh.keyspace + '".'],
+ self.cqlsh.keyspace + '.'],
other_choices_ok=True)
self.trycompletions('DESC TYPE ',
@@ -720,7 +737,7 @@
'fbestsong',
'fmax',
'fmin',
- '"' + self.cqlsh.keyspace + '".'],
+ self.cqlsh.keyspace + '.'],
other_choices_ok=True)
self.trycompletions('DESC AGGREGATE ',
@@ -730,7 +747,7 @@
'system_distributed.',
'aggmin',
'aggmax',
- '"' + self.cqlsh.keyspace + '".'],
+ self.cqlsh.keyspace + '.'],
other_choices_ok=True)
# Unfortunately these commented tests will not work. This is due to the keyspace name containing quotes;
@@ -803,3 +820,11 @@
def test_complete_in_drop_index(self):
pass
+
+ def test_complete_in_alter_keyspace(self):
+ self.trycompletions('ALTER KEY', 'SPACE ')
+ self.trycompletions('ALTER KEYSPACE ', '', choices=[self.cqlsh.keyspace, 'system_auth',
+ 'system_distributed', 'system_traces'])
+ self.trycompletions('ALTER KEYSPACE system_trac', "es WITH replication = {'class': '")
+ self.trycompletions("ALTER KEYSPACE system_traces WITH replication = {'class': '", '',
+ choices=['NetworkTopologyStrategy', 'SimpleStrategy'])
diff --git a/pylib/cqlshlib/test/test_cqlsh_invocation.py b/pylib/cqlshlib/test/test_cqlsh_invocation.py
deleted file mode 100644
index 67fa76f..0000000
--- a/pylib/cqlshlib/test/test_cqlsh_invocation.py
+++ /dev/null
@@ -1,78 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# to configure behavior, define $CQL_TEST_HOST to the destination address
-# for Thrift connections, and $CQL_TEST_PORT to the associated port.
-
-from .basecase import BaseTestCase
-
-class TestCqlshInvocation(BaseTestCase):
- def setUp(self):
- pass
-
- def tearDown(self):
- pass
-
- def test_normal_run(self):
- pass
-
- def test_python_interpreter_location(self):
- pass
-
- def test_color_capability_detection(self):
- pass
-
- def test_colored_output(self):
- pass
-
- def test_color_cmdline_option(self):
- pass
-
- def test_debug_option(self):
- pass
-
- def test_connection_args(self):
- pass
-
- def test_connection_config(self):
- pass
-
- def test_connection_envvars(self):
- pass
-
- def test_command_history(self):
- pass
-
- def test_missing_dependencies(self):
- pass
-
- def test_completekey_config(self):
- pass
-
- def test_ctrl_c(self):
- pass
-
- def test_eof(self):
- pass
-
- def test_output_encoding_detection(self):
- pass
-
- def test_output_encoding(self):
- pass
-
- def test_retries(self):
- pass
diff --git a/pylib/cqlshlib/test/test_cqlsh_output.py b/pylib/cqlshlib/test/test_cqlsh_output.py
index f57c734..accf3ed 100644
--- a/pylib/cqlshlib/test/test_cqlsh_output.py
+++ b/pylib/cqlshlib/test/test_cqlsh_output.py
@@ -15,37 +15,57 @@
# limitations under the License.
# to configure behavior, define $CQL_TEST_HOST to the destination address
-# for Thrift connections, and $CQL_TEST_PORT to the associated port.
+# and $CQL_TEST_PORT to the associated port.
-from __future__ import with_statement
+from __future__ import unicode_literals, with_statement
+import locale
+import os
import re
-from itertools import izip
-from .basecase import (BaseTestCase, cqlshlog, dedent, at_a_time, cqlsh,
- TEST_HOST, TEST_PORT)
-from .cassconnect import (get_keyspace, testrun_cqlsh, testcall_cqlsh,
- cassandra_cursor, split_cql_commands, quote_name)
-from .ansi_colors import (ColoredText, lookup_colorcode, lookup_colorname,
- lookup_colorletter, ansi_seq)
-import unittest
import sys
+import unittest
+
+from .basecase import (BaseTestCase, TEST_HOST, TEST_PORT,
+ at_a_time, cqlsh, cqlshlog, dedent)
+from .cassconnect import (cassandra_cursor, create_db, get_keyspace,
+ quote_name, remove_db, split_cql_commands,
+ testcall_cqlsh, testrun_cqlsh)
+from .ansi_colors import (ColoredText, ansi_seq, lookup_colorcode,
+ lookup_colorname, lookup_colorletter)
CONTROL_C = '\x03'
CONTROL_D = '\x04'
class TestCqlshOutput(BaseTestCase):
+ @classmethod
+ def setUpClass(cls):
+ create_db()
+
+ @classmethod
+ def tearDownClass(cls):
+ remove_db()
+
def setUp(self):
- pass
+ env = os.environ
+ env['COLUMNS'] = '100000'
+ # carry forward or override locale LC_CTYPE for UTF-8 encoding
+ if (locale.getpreferredencoding() != 'UTF-8'):
+ env['LC_CTYPE'] = 'en_US.utf8'
+ else:
+ env['LC_CTYPE'] = os.environ.get('LC_CTYPE', 'en_US.utf8')
+ if ('PATH' in os.environ.keys()):
+ env['PATH'] = os.environ['PATH']
+ self.default_env = env
def tearDown(self):
pass
def assertNoHasColors(self, text, msg=None):
- self.assertNotRegexpMatches(text, ansi_seq, msg='ANSI CSI sequence found in %r' % text)
+ self.assertNotRegex(text, ansi_seq, msg='ANSI CSI sequence found in %r' % text)
def assertHasColors(self, text, msg=None):
- self.assertRegexpMatches(text, ansi_seq, msg=msg)
+ self.assertRegex(text, ansi_seq, msg=msg)
def assertColored(self, coloredtext, colorname):
wanted_colorcode = lookup_colorcode(colorname)
@@ -57,7 +77,7 @@
% (coloredtext, num, lookup_colorname(ccolor), colorname))
def assertColorFromTags(self, coloredtext, tags):
- for (char, tag) in izip(coloredtext, tags):
+ for (char, tag) in zip(coloredtext, tags):
if char.isspace():
continue
if tag.isspace():
@@ -67,12 +87,14 @@
'Actually got: %s\ncolor code: %s'
% (tags, coloredtext.colored_version(), coloredtext.colortags()))
- def assertQueriesGiveColoredOutput(self, queries_and_expected_outputs, **kwargs):
+ def assertQueriesGiveColoredOutput(self, queries_and_expected_outputs, env=None, **kwargs):
"""
Allow queries and expected output to be specified in structured tuples,
along with expected color information.
"""
- with testrun_cqlsh(tty=True, **kwargs) as c:
+ if env is None:
+ env = self.default_env
+ with testrun_cqlsh(tty=True, env=env, **kwargs) as c:
for query, expected in queries_and_expected_outputs:
cqlshlog.debug('Testing %r' % (query,))
output = c.cmd_and_response(query).lstrip("\r\n")
@@ -83,10 +105,25 @@
self.assertEqual(outputline.plain().rstrip(), plain)
self.assertColorFromTags(outputline, colorcodes)
+ def strip_read_repair_chance(self, describe_statement):
+ """
+ Remove read_repair_chance and dclocal_read_repair_chance options
+ from output of DESCRIBE statements. The resulting string may be
+ reused as a CREATE statement.
+ Useful after CASSANDRA-13910, which removed read_repair_chance
+ options from CREATE statements but did not remove them completely
+ from the system.
+ """
+ describe_statement = re.sub(r"( AND)? (dclocal_)?read_repair_chance = [\d\.]+", "", describe_statement)
+ describe_statement = re.sub(r"WITH[\s]*;", "", describe_statement)
+ return describe_statement
+
def test_no_color_output(self):
+ env = self.default_env
for termname in ('', 'dumb', 'vt100'):
cqlshlog.debug('TERM=%r' % termname)
- with testrun_cqlsh(tty=True, env={'TERM': termname},
+ env['TERM'] = termname
+ with testrun_cqlsh(tty=True, env=env,
win_force_colors=False) as c:
c.send('select * from has_all_types;\n')
self.assertNoHasColors(c.read_to_next_prompt())
@@ -96,15 +133,17 @@
self.assertNoHasColors(c.read_to_next_prompt())
def test_no_prompt_or_colors_output(self):
+ env = self.default_env
for termname in ('', 'dumb', 'vt100', 'xterm'):
cqlshlog.debug('TERM=%r' % termname)
+ env['TERM'] = termname
query = 'select * from has_all_types limit 1;'
- output, result = testcall_cqlsh(prompt=None, env={'TERM': termname},
+ output, result = testcall_cqlsh(prompt=None, env=env,
tty=False, input=query + '\n')
output = output.splitlines()
for line in output:
self.assertNoHasColors(line)
- self.assertNotRegexpMatches(line, r'^cqlsh\S*>')
+ self.assertNotRegex(line, r'^cqlsh\S*>')
self.assertEqual(len(output), 6,
msg='output: %r' % '\n'.join(output))
self.assertEqual(output[0], '')
@@ -115,9 +154,12 @@
self.assertEqual(output[5].strip(), '(1 rows)')
def test_color_output(self):
+ env = self.default_env
for termname in ('xterm', 'unknown-garbage'):
cqlshlog.debug('TERM=%r' % termname)
- with testrun_cqlsh(tty=True, env={'TERM': termname}) as c:
+ env['TERMNAME'] = termname
+ env['TERM'] = termname
+ with testrun_cqlsh(tty=True, env=env) as c:
c.send('select * from has_all_types;\n')
self.assertHasColors(c.read_to_next_prompt())
c.send('select count(*) from has_all_types;\n')
@@ -349,6 +391,8 @@
))
def test_timestamp_output(self):
+ env = self.default_env
+ env['TZ'] = 'Etc/UTC'
self.assertQueriesGiveColoredOutput((
('''select timestampcol from has_all_types where num = 0;''', """
timestampcol
@@ -362,9 +406,10 @@
(1 rows)
nnnnnnnn
"""),
- ), env={'TZ': 'Etc/UTC'})
+ ), env=env)
try:
import pytz # test only if pytz is available on PYTHONPATH
+ env['TZ'] = 'America/Sao_Paulo'
self.assertQueriesGiveColoredOutput((
('''select timestampcol from has_all_types where num = 0;''', """
timestampcol
@@ -378,7 +423,7 @@
(1 rows)
nnnnnnnn
"""),
- ), env={'TZ': 'America/Sao_Paulo'})
+ ), env=env)
except ImportError:
pass
@@ -470,8 +515,10 @@
# terminals, but the color-checking machinery here will still treat
# it as one character, so those won't seem to line up visually either.
+ env = self.default_env
+ env['LANG'] = 'en_US.UTF-8'
self.assertQueriesGiveColoredOutput((
- ("select * from utf8_with_special_chars where k in (0, 1, 2, 3, 4, 5, 6);", u"""
+ ("select * from utf8_with_special_chars where k in (0, 1, 2, 3, 4, 5, 6);", """
k | val
R MMM
---+-------------------------------
@@ -494,8 +541,8 @@
(7 rows)
nnnnnnnn
- """.encode('utf-8')),
- ), env={'LANG': 'en_US.UTF-8'})
+ """),
+ ), env=env)
def test_blob_output(self):
self.assertQueriesGiveColoredOutput((
@@ -516,11 +563,11 @@
(4 rows)
nnnnnnnn
- """),
+ """, ),
))
def test_prompt(self):
- with testrun_cqlsh(tty=True, keyspace=None) as c:
+ with testrun_cqlsh(tty=True, keyspace=None, env=self.default_env) as c:
self.assertTrue(c.output_header.splitlines()[-1].endswith('cqlsh> '))
c.send('\n')
@@ -552,7 +599,7 @@
"RRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRR")
def test_describe_keyspace_output(self):
- with testrun_cqlsh(tty=True) as c:
+ with testrun_cqlsh(tty=True, env=self.default_env) as c:
ks = get_keyspace()
qks = quote_name(ks)
for cmd in ('describe keyspace', 'desc keyspace'):
@@ -572,19 +619,20 @@
with cassandra_cursor() as curs:
try:
for stmt in statements:
+ stmt = self.strip_read_repair_chance(stmt)
cqlshlog.debug('TEST EXEC: %s' % stmt)
curs.execute(stmt)
finally:
curs.execute('use system')
if do_drop:
- curs.execute('drop keyspace %s' % quote_name(new_ks_name))
+ curs.execute('drop keyspace {}'.format(new_ks_name))
def check_describe_keyspace_output(self, output, qksname):
expected_bits = [r'(?im)^CREATE KEYSPACE %s WITH\b' % re.escape(qksname),
r';\s*$',
r'\breplication = {\'class\':']
for expr in expected_bits:
- self.assertRegexpMatches(output, expr)
+ self.assertRegex(output, expr)
def test_describe_columnfamily_output(self):
# we can change these to regular expressions if/when it makes sense
@@ -592,7 +640,6 @@
# note columns are now comparator-ordered instead of original-order.
table_desc3 = dedent("""
-
CREATE TABLE %s.has_all_types (
num int PRIMARY KEY,
asciicol ascii,
@@ -610,43 +657,40 @@
uuidcol uuid,
varcharcol text,
varintcol varint
- ) WITH bloom_filter_fp_chance = 0.01
+ ) WITH additional_write_policy = '99p'
+ AND bloom_filter_fp_chance = 0.01
AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
AND cdc = false
AND comment = ''
AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}
- AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
+ AND compression = {'chunk_length_in_kb': '16', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
AND crc_check_chance = 1.0
- AND dclocal_read_repair_chance = 0.1
AND default_time_to_live = 0
+ AND extensions = {}
AND gc_grace_seconds = 864000
AND max_index_interval = 2048
AND memtable_flush_period_in_ms = 0
AND min_index_interval = 128
- AND read_repair_chance = 0.0
- AND speculative_retry = '99PERCENTILE';
+ AND read_repair = 'BLOCKING'
+ AND speculative_retry = '99p';""" % quote_name(get_keyspace()))
- """ % quote_name(get_keyspace()))
-
- with testrun_cqlsh(tty=True) as c:
+ with testrun_cqlsh(tty=True, env=self.default_env) as c:
for cmdword in ('describe table', 'desc columnfamily'):
for semicolon in (';', ''):
output = c.cmd_and_response('%s has_all_types%s' % (cmdword, semicolon))
self.assertNoHasColors(output)
- self.assertSequenceEqual(output.split('\n'), table_desc3.split('\n'))
+ self.assertSequenceEqual(dedent(output).split('\n'), table_desc3.split('\n'))
def test_describe_columnfamilies_output(self):
output_re = r'''
- \n
- Keyspace [ ] (?P<ksname> \S+ ) \n
+ \n? Keyspace [ ] (?P<ksname> \S+ ) \n
-----------* \n
- (?P<cfnames> .*? )
- \n
- '''
+ (?P<cfnames> ( ( ["']?\w+["']? [^\w\n]* )+ \n )* )
+ '''
ks = get_keyspace()
- with testrun_cqlsh(tty=True, keyspace=None) as c:
+ with testrun_cqlsh(tty=True, keyspace=None, env=self.default_env) as c:
# when not in a keyspace
for cmdword in ('DESCRIBE COLUMNFAMILIES', 'desc tables'):
@@ -654,7 +698,7 @@
ksnames = []
output = c.cmd_and_response(cmdword + semicolon)
self.assertNoHasColors(output)
- self.assertRegexpMatches(output, '(?xs) ^ ( %s )+ $' % output_re)
+ self.assertRegex(output, '(?xs) ^ ( %s )+ $' % output_re)
for section in re.finditer('(?xs)' + output_re, output):
ksname = section.group('ksname')
@@ -686,24 +730,25 @@
\n
Cluster: [ ] (?P<clustername> .* ) \n
Partitioner: [ ] (?P<partitionername> .* ) \n
+ Snitch: [ ] (?P<snitchname> .* ) \n
\n
'''
ringinfo_re = r'''
Range[ ]ownership: \n
(
- [ ] .*? [ ][ ] \[ ( \d+ \. ){3} \d+ \] \n
+ [ ] .*? [ ][ ] \[ ( \d+ \. ){3} \d+ : \d+ \] \n
)+
\n
'''
- with testrun_cqlsh(tty=True, keyspace=None) as c:
+ with testrun_cqlsh(tty=True, keyspace=None, env=self.default_env) as c:
# not in a keyspace
for semicolon in ('', ';'):
output = c.cmd_and_response('describe cluster' + semicolon)
self.assertNoHasColors(output)
- self.assertRegexpMatches(output, output_re + '$')
+ self.assertRegex(output, output_re + '$')
c.send('USE %s;\n' % quote_name(get_keyspace()))
c.read_to_next_prompt()
@@ -711,32 +756,33 @@
for semicolon in ('', ';'):
output = c.cmd_and_response('describe cluster' + semicolon)
self.assertNoHasColors(output)
- self.assertRegexpMatches(output, output_re + ringinfo_re + '$')
+ self.assertRegex(output, output_re + ringinfo_re + '$')
def test_describe_schema_output(self):
- with testrun_cqlsh(tty=True) as c:
+ with testrun_cqlsh(tty=True, env=self.default_env) as c:
for semicolon in ('', ';'):
output = c.cmd_and_response('desc full schema' + semicolon)
self.assertNoHasColors(output)
- self.assertRegexpMatches(output, '^\nCREATE KEYSPACE')
- self.assertIn("\nCREATE KEYSPACE system WITH replication = {'class': 'LocalStrategy'} AND durable_writes = true;\n",
+ # Since CASSANDRA-7622 'DESC FULL SCHEMA' also shows all VIRTUAL keyspaces
+ self.assertIn('VIRTUAL KEYSPACE system_virtual_schema', output)
+ self.assertIn("\nCREATE KEYSPACE system_auth WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'} AND durable_writes = true;\n",
output)
- self.assertRegexpMatches(output, ';\s*$')
+ self.assertRegex(output, '.*\s*$')
def test_show_output(self):
- with testrun_cqlsh(tty=True) as c:
+ with testrun_cqlsh(tty=True, env=self.default_env) as c:
output = c.cmd_and_response('show version;')
- self.assertRegexpMatches(output,
+ self.assertRegex(output,
'^\[cqlsh \S+ \| Cassandra \S+ \| CQL spec \S+ \| Native protocol \S+\]$')
output = c.cmd_and_response('show host;')
self.assertHasColors(output)
- self.assertRegexpMatches(output, '^Connected to .* at %s:%d\.$'
+ self.assertRegex(output, '^Connected to .* at %s:%d\.$'
% (re.escape(TEST_HOST), TEST_PORT))
@unittest.skipIf(sys.platform == "win32", 'EOF signaling not supported on Windows')
def test_eof_prints_newline(self):
- with testrun_cqlsh(tty=True) as c:
+ with testrun_cqlsh(tty=True, env=self.default_env) as c:
c.send(CONTROL_D)
out = c.read_lines(1)[0].replace('\r', '')
self.assertEqual(out, '\n')
@@ -746,7 +792,7 @@
def test_exit_prints_no_newline(self):
for semicolon in ('', ';'):
- with testrun_cqlsh(tty=True) as c:
+ with testrun_cqlsh(tty=True, env=self.default_env) as c:
cmd = 'exit%s\n' % semicolon
c.send(cmd)
if c.realtty:
@@ -757,7 +803,7 @@
self.assertIn(type(cm.exception), (EOFError, OSError))
def test_help_types(self):
- with testrun_cqlsh(tty=True) as c:
+ with testrun_cqlsh(tty=True, env=self.default_env) as c:
c.cmd_and_response('help types')
def test_help(self):
@@ -801,6 +847,7 @@
nnnnnnnn
"""),
))
+
self.assertQueriesGiveColoredOutput((
("select phone_numbers from users;", r"""
phone_numbers
@@ -833,6 +880,7 @@
nnnnnnnn
"""),
))
+
self.assertQueriesGiveColoredOutput((
("select tags from songs;", r"""
tags
diff --git a/pylib/cqlshlib/test/test_cqlsh_parsing.py b/pylib/cqlshlib/test/test_cqlsh_parsing.py
deleted file mode 100644
index 7e7f08b..0000000
--- a/pylib/cqlshlib/test/test_cqlsh_parsing.py
+++ /dev/null
@@ -1,26 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# to configure behavior, define $CQL_TEST_HOST to the destination address
-# for Thrift connections, and $CQL_TEST_PORT to the associated port.
-
-from unittest import TestCase
-
-
-class TestCqlshParsing(TestCase):
- def test_describe(self):
- pass
-
diff --git a/pylib/cqlshlib/test/test_keyspace_init.cql b/pylib/cqlshlib/test/test_keyspace_init.cql
index c64163a..26e8dae 100644
--- a/pylib/cqlshlib/test/test_keyspace_init.cql
+++ b/pylib/cqlshlib/test/test_keyspace_init.cql
@@ -63,12 +63,12 @@
-CREATE COLUMNFAMILY dynamic_columns (
+CREATE TABLE dynamic_columns (
somekey int,
column1 float,
value text,
PRIMARY KEY(somekey, column1)
-) WITH COMPACT STORAGE;
+);
INSERT INTO dynamic_columns (somekey, column1, value) VALUES (1, 1.2, 'one point two');
INSERT INTO dynamic_columns (somekey, column1, value) VALUES (2, 2.3, 'two point three');
diff --git a/pylib/cqlshlib/test/winpty.py b/pylib/cqlshlib/test/winpty.py
index 0db9ec3..f197aa5 100644
--- a/pylib/cqlshlib/test/winpty.py
+++ b/pylib/cqlshlib/test/winpty.py
@@ -15,11 +15,11 @@
# limitations under the License.
from threading import Thread
-from cStringIO import StringIO
-from Queue import Queue, Empty
+from six import StringIO
+from six.moves.queue import Queue, Empty
-class WinPty:
+class WinPty(object):
def __init__(self, stdin):
self._s = stdin
@@ -47,4 +47,4 @@
count = count + 1
except Empty:
pass
- return buf.getvalue()
+ return buf.getvalue()
\ No newline at end of file
diff --git a/pylib/cqlshlib/tracing.py b/pylib/cqlshlib/tracing.py
index 5b55367..0f1988a 100644
--- a/pylib/cqlshlib/tracing.py
+++ b/pylib/cqlshlib/tracing.py
@@ -14,11 +14,12 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from cqlshlib.displaying import MAGENTA
from datetime import datetime, timedelta
-from formatting import CqlType
import time
+
from cassandra.query import QueryTrace, TraceUnavailable
+from cqlshlib.displaying import MAGENTA
+from cqlshlib.formatting import CqlType
def print_trace_session(shell, session, session_id, partial_session=False):
@@ -45,8 +46,8 @@
return
names = ['activity', 'timestamp', 'source', 'source_elapsed', 'client']
- formatted_names = map(shell.myformat_colname, names)
- formatted_values = [map(shell.myformat_value, row) for row in rows]
+ formatted_names = list(map(shell.myformat_colname, names))
+ formatted_values = [list(map(shell.myformat_value, row)) for row in rows]
shell.writeresult('')
shell.writeresult('Tracing session: ', color=MAGENTA, newline=False)
diff --git a/pylib/cqlshlib/util.py b/pylib/cqlshlib/util.py
index 3ee128d..82a332f 100644
--- a/pylib/cqlshlib/util.py
+++ b/pylib/cqlshlib/util.py
@@ -19,9 +19,9 @@
import codecs
import pstats
-from itertools import izip
+
from datetime import timedelta, tzinfo
-from StringIO import StringIO
+from six import StringIO
try:
from line_profiler import LineProfiler
@@ -77,7 +77,7 @@
"""
common = []
- for cgroup in izip(*strs):
+ for cgroup in zip(*strs):
if all(x == cgroup[0] for x in cgroup[1:]):
common.append(cgroup[0])
else:
@@ -161,6 +161,6 @@
ret = s.getvalue()
if file_name:
with open(file_name, 'w') as f:
- print "Writing to %s\n" % (f.name, )
+ print("Writing to %s\n" % (f.name, ))
f.write(ret)
return ret
diff --git a/pylib/cqlshlib/wcwidth.py b/pylib/cqlshlib/wcwidth.py
index 985fd41..0be3af2 100644
--- a/pylib/cqlshlib/wcwidth.py
+++ b/pylib/cqlshlib/wcwidth.py
@@ -85,7 +85,7 @@
if ucs < table[0][0] or ucs > table[max][1]:
return 0
while max >= min:
- mid = (min + max) / 2
+ mid = int((min + max) / 2)
if ucs > table[mid][1]:
min = mid + 1
elif ucs < table[mid][0]:
@@ -251,20 +251,21 @@
# if we arrive here, ucs is not a combining or C0/C1 control character
- return 1 + \
- int(ucs >= 0x1100 and
- (ucs <= 0x115f or # Hangul Jamo init. consonants
- ucs == 0x2329 or ucs == 0x232a or
- (ucs >= 0x2e80 and ucs <= 0xa4cf and
- ucs != 0x303f) or # CJK ... Yi
- (ucs >= 0xac00 and ucs <= 0xd7a3) or # Hangul Syllables
- (ucs >= 0xf900 and ucs <= 0xfaff) or # CJK Compatibility Ideographs
- (ucs >= 0xfe10 and ucs <= 0xfe19) or # Vertical forms
- (ucs >= 0xfe30 and ucs <= 0xfe6f) or # CJK Compatibility Forms
- (ucs >= 0xff00 and ucs <= 0xff60) or # Fullwidth Forms
- (ucs >= 0xffe0 and ucs <= 0xffe6) or
- (ucs >= 0x20000 and ucs <= 0x2fffd) or
- (ucs >= 0x30000 and ucs <= 0x3fffd)))
+ return 1 + int(
+ ucs >= 0x1100
+ and (ucs <= 0x115f # Hangul Jamo init. consonants
+ or ucs == 0x2329 or ucs == 0x232a
+ or (ucs >= 0x2e80 and ucs <= 0xa4cf
+ and ucs != 0x303f) # CJK ... Yi
+ or (ucs >= 0xac00 and ucs <= 0xd7a3) # Hangul Syllables
+ or (ucs >= 0xf900 and ucs <= 0xfaff) # CJK Compatibility Ideographs
+ or (ucs >= 0xfe10 and ucs <= 0xfe19) # Vertical forms
+ or (ucs >= 0xfe30 and ucs <= 0xfe6f) # CJK Compatibility Forms
+ or (ucs >= 0xff00 and ucs <= 0xff60) # Fullwidth Forms
+ or (ucs >= 0xffe0 and ucs <= 0xffe6)
+ or (ucs >= 0x20000 and ucs <= 0x2fffd)
+ or (ucs >= 0x30000 and ucs <= 0x3fffd))
+ )
def mk_wcswidth(pwcs):
@@ -313,7 +314,7 @@
def wcswidth(s):
- return mk_wcswidth(map(ord, s))
+ return mk_wcswidth(list(map(ord, s)))
def wcwidth_cjk(c):
@@ -321,7 +322,7 @@
def wcswidth_cjk(s):
- return mk_wcswidth_cjk(map(ord, s))
+ return mk_wcswidth_cjk(list(map(ord, s)))
if __name__ == "__main__":
@@ -341,7 +342,7 @@
('COMBINING PALATALIZED HOOK BELOW', 0),
('COMBINING GRAVE ACCENT', 0),
)
- nonprinting = u'\r\n\t\a\b\f\v\x7f'
+ nonprinting = '\r\n\t\a\b\f\v\x7f'
import unicodedata
@@ -366,13 +367,13 @@
assert mk_wcwidth(0x10ffff) == 1
assert mk_wcwidth(0x3fffd) == 2
- teststr = u'B\0ig br\u00f8wn moose\ub143\u200b'
+ teststr = 'B\0ig br\u00f8wn moose\ub143\u200b'
calculatedwidth = wcswidth(teststr)
assert calculatedwidth == 17, 'expected 17, got %d' % calculatedwidth
calculatedwidth = wcswidth_cjk(teststr)
assert calculatedwidth == 18, 'expected 18, got %d' % calculatedwidth
- assert wcswidth(u'foobar\u200b\a') < 0
+ assert wcswidth('foobar\u200b\a') < 0
- print 'tests pass.'
+ print('tests pass.')
diff --git a/pylib/requirements.txt b/pylib/requirements.txt
index a9b6217..cdfa566 100644
--- a/pylib/requirements.txt
+++ b/pylib/requirements.txt
@@ -2,10 +2,11 @@
# cythonizing the driver, perhaps only on old pips.
# http://datastax.github.io/python-driver/installation.html#cython-based-extensions
futures
-six
+six>=0.12.0
-e git+https://github.com/datastax/python-driver.git@cassandra-test#egg=cassandra-driver
# Used ccm version is tracked by cassandra-test branch in ccm repo. Please create a PR there for fixes or upgrades to new releases.
-e git+https://github.com/riptano/ccm.git@cassandra-test#egg=ccm
+coverage
cql
decorator
docopt
@@ -17,5 +18,4 @@
parse
pycodestyle
psutil
-pycassa
thrift==0.9.3
diff --git a/redhat/cassandra b/redhat/cassandra
index 97a0447..77622b6 100644
--- a/redhat/cassandra
+++ b/redhat/cassandra
@@ -34,7 +34,7 @@
CASSANDRA_PROG=/usr/sbin/cassandra
# The first existing directory is used for JAVA_HOME if needed.
-JVM_SEARCH_DIRS="/usr/lib/jvm/jre /usr/lib/jvm/jre-1.7.* /usr/lib/jvm/java-1.7.*/jre"
+JVM_SEARCH_DIRS="/usr/lib/jvm/jre /usr/lib/jvm/jre-1.8.* /usr/lib/jvm/java-1.8.*/jre"
# Read configuration variable file if it is present
[ -r /etc/default/$NAME ] && . /etc/default/$NAME
diff --git a/redhat/cassandra.in.sh b/redhat/cassandra.in.sh
index ca71782..ecca949 100644
--- a/redhat/cassandra.in.sh
+++ b/redhat/cassandra.in.sh
@@ -23,8 +23,84 @@
# set JVM javaagent opts to avoid warnings/errors
-if [ "$JVM_VENDOR" != "OpenJDK" -o "$JVM_VERSION" \> "1.6.0" ] \
- || [ "$JVM_VERSION" = "1.6.0" -a "$JVM_PATCH_VERSION" -ge 23 ]
-then
- JAVA_AGENT="$JAVA_AGENT -javaagent:$CASSANDRA_HOME/lib/jamm-0.3.0.jar"
+JAVA_AGENT="$JAVA_AGENT -javaagent:$CASSANDRA_HOME/lib/jamm-0.3.2.jar"
+
+
+#
+# Java executable and per-Java version JVM settings
+#
+
+# Use JAVA_HOME if set, otherwise look for java in PATH
+if [ -n "$JAVA_HOME" ]; then
+ # Why we can't have nice things: Solaris combines x86 and x86_64
+ # installations in the same tree, using an unconventional path for the
+ # 64bit JVM. Since we prefer 64bit, search the alternate path first,
+ # (see https://issues.apache.org/jira/browse/CASSANDRA-4638).
+ for java in "$JAVA_HOME"/bin/amd64/java "$JAVA_HOME"/bin/java; do
+ if [ -x "$java" ]; then
+ JAVA="$java"
+ break
+ fi
+ done
+else
+ JAVA=java
fi
+
+if [ -z $JAVA ] ; then
+ echo Unable to find java executable. Check JAVA_HOME and PATH environment variables. >&2
+ exit 1;
+fi
+
+# Determine the sort of JVM we'll be running on.
+java_ver_output=`"${JAVA:-java}" -version 2>&1`
+jvmver=`echo "$java_ver_output" | grep '[openjdk|java] version' | awk -F'"' 'NR==1 {print $2}' | cut -d\- -f1`
+JVM_VERSION=${jvmver%_*}
+
+JAVA_VERSION=11
+if [ "$JVM_VERSION" = "1.8.0" ] ; then
+ JVM_PATCH_VERSION=${jvmver#*_}
+ if [ "$JVM_VERSION" \< "1.8" ] || [ "$JVM_VERSION" \> "1.8.2" ] ; then
+ echo "Cassandra 4.0 requires either Java 8 (update 151 or newer) or Java 11 (or newer). Java $JVM_VERSION is not supported."
+ exit 1;
+ fi
+ if [ "$JVM_PATCH_VERSION" -lt 151 ] ; then
+ echo "Cassandra 4.0 requires either Java 8 (update 151 or newer) or Java 11 (or newer). Java 8 update $JVM_PATCH_VERSION is not supported."
+ exit 1;
+ fi
+ JAVA_VERSION=8
+elif [ "$JVM_VERSION" \< "11" ] ; then
+ echo "Cassandra 4.0 requires either Java 8 (update 151 or newer) or Java 11 (or newer)."
+ exit 1;
+fi
+
+jvm=`echo "$java_ver_output" | grep -A 1 '[openjdk|java] version' | awk 'NR==2 {print $1}'`
+case "$jvm" in
+ OpenJDK)
+ JVM_VENDOR=OpenJDK
+ # this will be "64-Bit" or "32-Bit"
+ JVM_ARCH=`echo "$java_ver_output" | awk 'NR==3 {print $2}'`
+ ;;
+ "Java(TM)")
+ JVM_VENDOR=Oracle
+ # this will be "64-Bit" or "32-Bit"
+ JVM_ARCH=`echo "$java_ver_output" | awk 'NR==3 {print $3}'`
+ ;;
+ *)
+ # Help fill in other JVM values
+ JVM_VENDOR=other
+ JVM_ARCH=unknown
+ ;;
+esac
+
+# Read user-defined JVM options from jvm-server.options file
+JVM_OPTS_FILE=$CASSANDRA_CONF/jvm${jvmoptions_variant:--clients}.options
+if [ $JAVA_VERSION -ge 11 ] ; then
+ JVM_DEP_OPTS_FILE=$CASSANDRA_CONF/jvm11${jvmoptions_variant:--clients}.options
+else
+ JVM_DEP_OPTS_FILE=$CASSANDRA_CONF/jvm8${jvmoptions_variant:--clients}.options
+fi
+
+for opt in `grep "^-" $JVM_OPTS_FILE` `grep "^-" $JVM_DEP_OPTS_FILE`
+do
+ JVM_OPTS="$JVM_OPTS $opt"
+done
diff --git a/redhat/cassandra.spec b/redhat/cassandra.spec
index 63bfd9a..f42ab83 100644
--- a/redhat/cassandra.spec
+++ b/redhat/cassandra.spec
@@ -8,7 +8,9 @@
%global username cassandra
-%define relname apache-cassandra-%{version}
+# input of ~alphaN, ~betaN, ~rcN package versions need to retain upstream '-alphaN, etc' version for sources
+%define upstream_version %(echo %{version} | sed -r 's/~/-/g')
+%define relname apache-cassandra-%{upstream_version}
Name: cassandra
Version: %{version}
@@ -46,7 +48,7 @@
%build
export LANG=en_US.UTF-8
export JAVA_TOOL_OPTIONS="-Dfile.encoding=UTF-8"
-ant clean jar -Dversion=%{version}
+ant clean jar -Dversion=%{upstream_version}
%install
%{__rm} -rf %{buildroot}
@@ -68,8 +70,8 @@
( cd pylib && python2.7 setup.py install --no-compile --root %{buildroot}; )
# patches for data and log paths
-patch -p1 < debian/patches/001cassandra_yaml_dirs.dpatch
-patch -p1 < debian/patches/002cassandra_logdir_fix.dpatch
+patch -p1 < debian/patches/cassandra_yaml_dirs.diff
+patch -p1 < debian/patches/cassandra_logdir_fix.diff
# uncomment hints_directory path
sed -i 's/^# hints_directory:/hints_directory:/' conf/cassandra.yaml
@@ -98,14 +100,16 @@
# copy stress jar
cp -p build/tools/lib/stress.jar %{buildroot}/usr/share/%{username}/
+# copy fqltool jar
+cp -p build/tools/lib/fqltool.jar %{buildroot}/usr/share/%{username}/
+
# copy binaries
mv bin/cassandra %{buildroot}/usr/sbin/
cp -p bin/* %{buildroot}/usr/bin/
cp -p tools/bin/* %{buildroot}/usr/bin/
-# copy cassandra, thrift jars
-cp build/apache-cassandra-%{version}.jar %{buildroot}/usr/share/%{username}/
-cp build/apache-cassandra-thrift-%{version}.jar %{buildroot}/usr/share/%{username}/
+# copy cassandra jar
+cp build/apache-cassandra-%{upstream_version}.jar %{buildroot}/usr/share/%{username}/
%clean
%{__rm} -rf %{buildroot}
@@ -119,10 +123,12 @@
%files
%defattr(0644,root,root,0755)
%doc CHANGES.txt LICENSE.txt README.asc NEWS.txt NOTICE.txt CASSANDRA-14092.txt
+%attr(755,root,root) %{_bindir}/auditlogviewer
%attr(755,root,root) %{_bindir}/cassandra-stress
%attr(755,root,root) %{_bindir}/cqlsh
%attr(755,root,root) %{_bindir}/cqlsh.py
%attr(755,root,root) %{_bindir}/debug-cql
+%attr(755,root,root) %{_bindir}/fqltool
%attr(755,root,root) %{_bindir}/nodetool
%attr(755,root,root) %{_bindir}/sstableloader
%attr(755,root,root) %{_bindir}/sstablescrub
@@ -174,6 +180,8 @@
%attr(755,root,root) %{_bindir}/sstableofflinerelevel
%attr(755,root,root) %{_bindir}/sstablerepairedset
%attr(755,root,root) %{_bindir}/sstablesplit
+%attr(755,root,root) %{_bindir}/auditlogviewer
+%attr(755,root,root) %{_bindir}/fqltool
%changelog
diff --git a/src/antlr/Cql.g b/src/antlr/Cql.g
index a11f2fd..272c63b 100644
--- a/src/antlr/Cql.g
+++ b/src/antlr/Cql.g
@@ -28,28 +28,25 @@
@header {
package org.apache.cassandra.cql3;
- import java.util.ArrayList;
- import java.util.Arrays;
import java.util.Collections;
import java.util.EnumSet;
import java.util.HashSet;
- import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.cassandra.auth.*;
- import org.apache.cassandra.config.ColumnDefinition;
- import org.apache.cassandra.cql3.*;
- import org.apache.cassandra.cql3.restrictions.CustomIndexExpression;
- import org.apache.cassandra.cql3.statements.*;
- import org.apache.cassandra.cql3.selection.*;
+ import org.apache.cassandra.cql3.conditions.*;
import org.apache.cassandra.cql3.functions.*;
- import org.apache.cassandra.db.marshal.CollectionType;
+ import org.apache.cassandra.cql3.restrictions.CustomIndexExpression;
+ import org.apache.cassandra.cql3.selection.*;
+ import org.apache.cassandra.cql3.statements.*;
+ import org.apache.cassandra.cql3.statements.schema.*;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.exceptions.InvalidRequestException;
import org.apache.cassandra.exceptions.SyntaxException;
+ import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.utils.Pair;
}
@@ -94,8 +91,6 @@
@lexer::header {
package org.apache.cassandra.cql3;
-
- import org.apache.cassandra.exceptions.SyntaxException;
}
@lexer::members {
@@ -134,6 +129,6 @@
}
}
-query returns [ParsedStatement stmnt]
+query returns [CQLStatement.Raw stmnt]
: st=cqlStatement (';')* EOF { $stmnt = st; }
;
diff --git a/src/antlr/Lexer.g b/src/antlr/Lexer.g
index 1c52d4c..e9ca5eb 100644
--- a/src/antlr/Lexer.g
+++ b/src/antlr/Lexer.g
@@ -86,11 +86,14 @@
K_DELETE: D E L E T E;
K_IN: I N;
K_CREATE: C R E A T E;
+K_SCHEMA: S C H E M A;
K_KEYSPACE: ( K E Y S P A C E
- | S C H E M A );
+ | K_SCHEMA );
K_KEYSPACES: K E Y S P A C E S;
K_COLUMNFAMILY:( C O L U M N F A M I L Y
| T A B L E );
+K_TABLES: ( C O L U M N F A M I L I E S
+ | T A B L E S );
K_MATERIALIZED:M A T E R I A L I Z E D;
K_VIEW: V I E W;
K_INDEX: I N D E X;
@@ -108,6 +111,7 @@
K_RENAME: R E N A M E;
K_ADD: A D D;
K_TYPE: T Y P E;
+K_TYPES: T Y P E S;
K_COMPACT: C O M P A C T;
K_STORAGE: S T O R A G E;
K_ORDER: O R D E R;
@@ -120,6 +124,9 @@
K_IS: I S;
K_CONTAINS: C O N T A I N S;
K_GROUP: G R O U P;
+K_CLUSTER: C L U S T E R;
+K_INTERNALS: I N T E R N A L S;
+K_ONLY: O N L Y;
K_GRANT: G R A N T;
K_ALL: A L L;
@@ -145,6 +152,8 @@
K_LOGIN: L O G I N;
K_NOLOGIN: N O L O G I N;
K_OPTIONS: O P T I O N S;
+K_ACCESS: A C C E S S;
+K_DATACENTERS: D A T A C E N T E R S;
K_CLUSTERING: C L U S T E R I N G;
K_ASCII: A S C I I;
@@ -176,8 +185,10 @@
K_MAP: M A P;
K_LIST: L I S T;
-K_NAN: N A N;
-K_INFINITY: I N F I N I T Y;
+K_POSITIVE_NAN: N A N;
+K_NEGATIVE_NAN: '-' N A N;
+K_POSITIVE_INFINITY: I N F I N I T Y;
+K_NEGATIVE_INFINITY: '-' I N F I N I T Y;
K_TUPLE: T U P L E;
K_TRIGGER: T R I G G E R;
@@ -187,6 +198,7 @@
K_FUNCTION: F U N C T I O N;
K_FUNCTIONS: F U N C T I O N S;
K_AGGREGATE: A G G R E G A T E;
+K_AGGREGATES: A G G R E G A T E S;
K_SFUNC: S F U N C;
K_STYPE: S T Y P E;
K_FINALFUNC: F I N A L F U N C;
@@ -302,13 +314,18 @@
: '?'
;
+RANGE
+ : '..'
+ ;
+
/*
* Normally a lexer only emits one token at a time, but ours is tricked out
* to support multiple (see @lexer::members near the top of the grammar).
*/
FLOAT
- : INTEGER EXPONENT
- | INTEGER '.' DIGIT* EXPONENT?
+ : (INTEGER '.' RANGE) => INTEGER '.'
+ | (INTEGER RANGE) => INTEGER {$type = INTEGER;}
+ | INTEGER ('.' DIGIT*)? EXPONENT?
;
/*
diff --git a/src/antlr/Parser.g b/src/antlr/Parser.g
index 26074b8..4b7e5bf 100644
--- a/src/antlr/Parser.g
+++ b/src/antlr/Parser.g
@@ -136,9 +136,9 @@
return res;
}
- public void addRawUpdate(List<Pair<ColumnDefinition.Raw, Operation.RawUpdate>> operations, ColumnDefinition.Raw key, Operation.RawUpdate update)
+ public void addRawUpdate(List<Pair<ColumnMetadata.Raw, Operation.RawUpdate>> operations, ColumnMetadata.Raw key, Operation.RawUpdate update)
{
- for (Pair<ColumnDefinition.Raw, Operation.RawUpdate> p : operations)
+ for (Pair<ColumnMetadata.Raw, Operation.RawUpdate> p : operations)
{
if (p.left.equals(key) && !p.right.isCompatibleWith(update))
addRecognitionError("Multiple incompatible setting of column " + key);
@@ -204,8 +204,8 @@
/** STATEMENTS **/
-cqlStatement returns [ParsedStatement stmt]
- @after{ if (stmt != null) stmt.setBoundVariables(bindVariables); }
+cqlStatement returns [CQLStatement.Raw stmt]
+ @after{ if (stmt != null) stmt.setBindVariables(bindVariables); }
: st1= selectStatement { $stmt = st1; }
| st2= insertStatement { $stmt = st2; }
| st3= updateStatement { $stmt = st3; }
@@ -246,6 +246,7 @@
| st38=createMaterializedViewStatement { $stmt = st38; }
| st39=dropMaterializedViewStatement { $stmt = st39; }
| st40=alterMaterializedViewStatement { $stmt = st40; }
+ | st41=describeStatement { $stmt = st41; }
;
/*
@@ -263,17 +264,16 @@
*/
selectStatement returns [SelectStatement.RawStatement expr]
@init {
- boolean isDistinct = false;
Term.Raw limit = null;
Term.Raw perPartitionLimit = null;
- Map<ColumnDefinition.Raw, Boolean> orderings = new LinkedHashMap<>();
- List<ColumnDefinition.Raw> groups = new ArrayList<>();
+ Map<ColumnMetadata.Raw, Boolean> orderings = new LinkedHashMap<>();
+ List<ColumnMetadata.Raw> groups = new ArrayList<>();
boolean allowFiltering = false;
boolean isJson = false;
}
: K_SELECT
- ( K_JSON { isJson = true; } )?
- ( ( K_DISTINCT { isDistinct = true; } )? sclause=selectClause )
+ // json is a valid column name. By consequence, we need to resolve the ambiguity for "json - json"
+ ( (K_JSON selectClause)=> K_JSON { isJson = true; } )? sclause=selectClause
K_FROM cf=columnFamilyName
( K_WHERE wclause=whereClause )?
( K_GROUP K_BY groupByClause[groups] ( ',' groupByClause[groups] )* )?
@@ -284,15 +284,22 @@
{
SelectStatement.Parameters params = new SelectStatement.Parameters(orderings,
groups,
- isDistinct,
+ $sclause.isDistinct,
allowFiltering,
isJson);
WhereClause where = wclause == null ? WhereClause.empty() : wclause.build();
- $expr = new SelectStatement.RawStatement(cf, params, sclause, where, limit, perPartitionLimit);
+ $expr = new SelectStatement.RawStatement(cf, params, $sclause.selectors, where, limit, perPartitionLimit);
}
;
-selectClause returns [List<RawSelector> expr]
+selectClause returns [boolean isDistinct, List<RawSelector> selectors]
+ @init{ $isDistinct = false; }
+ // distinct is a valid column name. By consequence, we need to resolve the ambiguity for "distinct - distinct"
+ : (K_DISTINCT selectors)=> K_DISTINCT s=selectors { $isDistinct = true; $selectors = s; }
+ | s=selectors { $selectors = s; }
+ ;
+
+selectors returns [List<RawSelector> expr]
: t1=selector { $expr = new ArrayList<RawSelector>(); $expr.add(t1); } (',' tN=selector { $expr.add(tN); })*
| '\*' { $expr = Collections.<RawSelector>emptyList();}
;
@@ -302,28 +309,136 @@
: us=unaliasedSelector (K_AS c=noncol_ident { alias = c; })? { $s = new RawSelector(us, alias); }
;
+unaliasedSelector returns [Selectable.Raw s]
+ : a=selectionAddition {$s = a;}
+ ;
+
+selectionAddition returns [Selectable.Raw s]
+ : l=selectionMultiplication {$s = l;}
+ ( '+' r=selectionMultiplication {$s = Selectable.WithFunction.Raw.newOperation('+', $s, r);}
+ | '-' r=selectionMultiplication {$s = Selectable.WithFunction.Raw.newOperation('-', $s, r);}
+ )*
+ ;
+
+selectionMultiplication returns [Selectable.Raw s]
+ : l=selectionGroup {$s = l;}
+ ( '\*' r=selectionGroup {$s = Selectable.WithFunction.Raw.newOperation('*', $s, r);}
+ | '/' r=selectionGroup {$s = Selectable.WithFunction.Raw.newOperation('/', $s, r);}
+ | '%' r=selectionGroup {$s = Selectable.WithFunction.Raw.newOperation('\%', $s, r);}
+ )*
+ ;
+
+selectionGroup returns [Selectable.Raw s]
+ : (selectionGroupWithField)=> f=selectionGroupWithField { $s=f; }
+ | g=selectionGroupWithoutField { $s=g; }
+ | '-' g=selectionGroup {$s = Selectable.WithFunction.Raw.newNegation(g);}
+ ;
+
+selectionGroupWithField returns [Selectable.Raw s]
+ : g=selectionGroupWithoutField m=selectorModifier[g] {$s = m;}
+ ;
+
+selectorModifier[Selectable.Raw receiver] returns [Selectable.Raw s]
+ : f=fieldSelectorModifier[receiver] m=selectorModifier[f] { $s = m; }
+ | '[' ss=collectionSubSelection[receiver] ']' m=selectorModifier[ss] { $s = m; }
+ | { $s = receiver; }
+ ;
+
+fieldSelectorModifier[Selectable.Raw receiver] returns [Selectable.Raw s]
+ : '.' fi=fident { $s = new Selectable.WithFieldSelection.Raw(receiver, fi); }
+ ;
+
+collectionSubSelection [Selectable.Raw receiver] returns [Selectable.Raw s]
+ @init { boolean isSlice=false; }
+ : ( t1=term ( { isSlice=true; } RANGE (t2=term)? )?
+ | RANGE { isSlice=true; } t2=term
+ ) {
+ $s = isSlice
+ ? new Selectable.WithSliceSelection.Raw(receiver, t1, t2)
+ : new Selectable.WithElementSelection.Raw(receiver, t1);
+ }
+ ;
+
+selectionGroupWithoutField returns [Selectable.Raw s]
+ @init { Selectable.Raw tmp = null; }
+ @after { $s = tmp; }
+ : sn=simpleUnaliasedSelector { tmp=sn; }
+ | (selectionTypeHint)=> h=selectionTypeHint { tmp=h; }
+ | t=selectionTupleOrNestedSelector { tmp=t; }
+ | l=selectionList { tmp=l; }
+ | m=selectionMapOrSet { tmp=m; }
+ // UDTs are equivalent to maps from the syntax point of view, so the final decision will be done in Selectable.WithMapOrUdt
+ ;
+
+selectionTypeHint returns [Selectable.Raw s]
+ : '(' ct=comparatorType ')' a=selectionGroupWithoutField { $s = new Selectable.WithTypeHint.Raw(ct, a); }
+ ;
+
+selectionList returns [Selectable.Raw s]
+ @init { List<Selectable.Raw> l = new ArrayList<>(); }
+ @after { $s = new Selectable.WithList.Raw(l); }
+ : '[' ( t1=unaliasedSelector { l.add(t1); } ( ',' tn=unaliasedSelector { l.add(tn); } )* )? ']'
+ ;
+
+selectionMapOrSet returns [Selectable.Raw s]
+ : '{' t1=unaliasedSelector ( m=selectionMap[t1] { $s = m; } | st=selectionSet[t1] { $s = st; }) '}'
+ | '{' '}' { $s = new Selectable.WithSet.Raw(Collections.emptyList());}
+ ;
+
+selectionMap [Selectable.Raw k1] returns [Selectable.Raw s]
+ @init { List<Pair<Selectable.Raw, Selectable.Raw>> m = new ArrayList<>(); }
+ @after { $s = new Selectable.WithMapOrUdt.Raw(m); }
+ : ':' v1=unaliasedSelector { m.add(Pair.create(k1, v1)); } ( ',' kn=unaliasedSelector ':' vn=unaliasedSelector { m.add(Pair.create(kn, vn)); } )*
+ ;
+
+selectionSet [Selectable.Raw t1] returns [Selectable.Raw s]
+ @init { List<Selectable.Raw> l = new ArrayList<>(); l.add(t1); }
+ @after { $s = new Selectable.WithSet.Raw(l); }
+ : ( ',' tn=unaliasedSelector { l.add(tn); } )*
+ ;
+
+selectionTupleOrNestedSelector returns [Selectable.Raw s]
+ @init { List<Selectable.Raw> l = new ArrayList<>(); }
+ @after { $s = new Selectable.BetweenParenthesesOrWithTuple.Raw(l); }
+ : '(' t1=unaliasedSelector { l.add(t1); } (',' tn=unaliasedSelector { l.add(tn); } )* ')'
+ ;
+
/*
* A single selection. The core of it is selecting a column, but we also allow any term and function, as well as
* sub-element selection for UDT.
*/
-unaliasedSelector returns [Selectable.Raw s]
- @init { Selectable.Raw tmp = null; }
- : ( c=cident { tmp = c; }
- | v=value { tmp = new Selectable.WithTerm.Raw(v); }
- | '(' ct=comparatorType ')' v=value { tmp = new Selectable.WithTerm.Raw(new TypeCast(ct, v)); }
- | K_COUNT '(' '\*' ')' { tmp = Selectable.WithFunction.Raw.newCountRowsFunction(); }
- | K_WRITETIME '(' c=cident ')' { tmp = new Selectable.WritetimeOrTTL.Raw(c, true); }
- | K_TTL '(' c=cident ')' { tmp = new Selectable.WritetimeOrTTL.Raw(c, false); }
- | K_CAST '(' sn=unaliasedSelector K_AS t=native_type ')' {tmp = new Selectable.WithCast.Raw(sn, t);}
- | f=functionName args=selectionFunctionArgs { tmp = new Selectable.WithFunction.Raw(f, args); }
- ) ( '.' fi=fident { tmp = new Selectable.WithFieldSelection.Raw(tmp, fi); } )* { $s = tmp; }
+simpleUnaliasedSelector returns [Selectable.Raw s]
+ : c=sident { $s = c; }
+ | l=selectionLiteral { $s = new Selectable.WithTerm.Raw(l); }
+ | f=selectionFunction { $s = f; }
+ ;
+
+selectionFunction returns [Selectable.Raw s]
+ : K_COUNT '(' '\*' ')' { $s = Selectable.WithFunction.Raw.newCountRowsFunction(); }
+ | K_WRITETIME '(' c=cident ')' { $s = new Selectable.WritetimeOrTTL.Raw(c, true); }
+ | K_TTL '(' c=cident ')' { $s = new Selectable.WritetimeOrTTL.Raw(c, false); }
+ | K_CAST '(' sn=unaliasedSelector K_AS t=native_type ')' {$s = new Selectable.WithCast.Raw(sn, t);}
+ | f=functionName args=selectionFunctionArgs { $s = new Selectable.WithFunction.Raw(f, args); }
+ ;
+
+selectionLiteral returns [Term.Raw value]
+ : c=constant { $value = c; }
+ | K_NULL { $value = Constants.NULL_LITERAL; }
+ | ':' id=noncol_ident { $value = newBindVariables(id); }
+ | QMARK { $value = newBindVariables(null); }
;
selectionFunctionArgs returns [List<Selectable.Raw> a]
- : '(' ')' { $a = Collections.emptyList(); }
- | '(' s1=unaliasedSelector { List<Selectable.Raw> args = new ArrayList<Selectable.Raw>(); args.add(s1); }
- ( ',' sn=unaliasedSelector { args.add(sn); } )*
- ')' { $a = args; }
+ @init{ $a = new ArrayList<>(); }
+ : '(' (s1=unaliasedSelector { $a.add(s1); }
+ ( ',' sn=unaliasedSelector { $a.add(sn); } )*)?
+ ')'
+ ;
+
+sident returns [Selectable.Raw id]
+ : t=IDENT { $id = Selectable.RawIdentifier.forUnquoted($t.text); }
+ | t=QUOTED_NAME { $id = Selectable.RawIdentifier.forQuoted($t.text); }
+ | k=unreserved_keyword { $id = Selectable.RawIdentifier.forUnquoted(k); }
;
whereClause returns [WhereClause.Builder clause]
@@ -337,18 +452,18 @@
;
customIndexExpression [WhereClause.Builder clause]
- @init{IndexName name = new IndexName();}
+ @init{QualifiedName name = new QualifiedName();}
: 'expr(' idxName[name] ',' t=term ')' { clause.add(new CustomIndexExpression(name, t));}
;
-orderByClause[Map<ColumnDefinition.Raw, Boolean> orderings]
+orderByClause[Map<ColumnMetadata.Raw, Boolean> orderings]
@init{
boolean reversed = false;
}
: c=cident (K_ASC | K_DESC { reversed = true; })? { orderings.put(c, reversed); }
;
-groupByClause[List<ColumnDefinition.Raw> groups]
+groupByClause[List<ColumnMetadata.Raw> groups]
: c=cident { groups.add(c); }
;
@@ -364,10 +479,10 @@
| K_JSON st2=jsonInsertStatement[cf] { $expr = st2; })
;
-normalInsertStatement [CFName cf] returns [UpdateStatement.ParsedInsert expr]
+normalInsertStatement [QualifiedName qn] returns [UpdateStatement.ParsedInsert expr]
@init {
Attributes.Raw attrs = new Attributes.Raw();
- List<ColumnDefinition.Raw> columnNames = new ArrayList<>();
+ List<ColumnMetadata.Raw> columnNames = new ArrayList<>();
List<Term.Raw> values = new ArrayList<>();
boolean ifNotExists = false;
}
@@ -377,11 +492,11 @@
( K_IF K_NOT K_EXISTS { ifNotExists = true; } )?
( usingClause[attrs] )?
{
- $expr = new UpdateStatement.ParsedInsert(cf, attrs, columnNames, values, ifNotExists);
+ $expr = new UpdateStatement.ParsedInsert(qn, attrs, columnNames, values, ifNotExists);
}
;
-jsonInsertStatement [CFName cf] returns [UpdateStatement.ParsedInsertJson expr]
+jsonInsertStatement [QualifiedName qn] returns [UpdateStatement.ParsedInsertJson expr]
@init {
Attributes.Raw attrs = new Attributes.Raw();
boolean ifNotExists = false;
@@ -392,7 +507,7 @@
( K_IF K_NOT K_EXISTS { ifNotExists = true; } )?
( usingClause[attrs] )?
{
- $expr = new UpdateStatement.ParsedInsertJson(cf, attrs, val, defaultUnset, ifNotExists);
+ $expr = new UpdateStatement.ParsedInsertJson(qn, attrs, val, defaultUnset, ifNotExists);
}
;
@@ -421,7 +536,7 @@
updateStatement returns [UpdateStatement.ParsedUpdate expr]
@init {
Attributes.Raw attrs = new Attributes.Raw();
- List<Pair<ColumnDefinition.Raw, Operation.RawUpdate>> operations = new ArrayList<>();
+ List<Pair<ColumnMetadata.Raw, Operation.RawUpdate>> operations = new ArrayList<>();
boolean ifExists = false;
}
: K_UPDATE cf=columnFamilyName
@@ -434,13 +549,13 @@
attrs,
operations,
wclause.build(),
- conditions == null ? Collections.<Pair<ColumnDefinition.Raw, ColumnCondition.Raw>>emptyList() : conditions,
+ conditions == null ? Collections.<Pair<ColumnMetadata.Raw, ColumnCondition.Raw>>emptyList() : conditions,
ifExists);
}
;
-updateConditions returns [List<Pair<ColumnDefinition.Raw, ColumnCondition.Raw>> conditions]
- @init { conditions = new ArrayList<Pair<ColumnDefinition.Raw, ColumnCondition.Raw>>(); }
+updateConditions returns [List<Pair<ColumnMetadata.Raw, ColumnCondition.Raw>> conditions]
+ @init { conditions = new ArrayList<Pair<ColumnMetadata.Raw, ColumnCondition.Raw>>(); }
: columnCondition[conditions] ( K_AND columnCondition[conditions] )*
;
@@ -468,7 +583,7 @@
attrs,
columnDeletions,
wclause.build(),
- conditions == null ? Collections.<Pair<ColumnDefinition.Raw, ColumnCondition.Raw>>emptyList() : conditions,
+ conditions == null ? Collections.<Pair<ColumnMetadata.Raw, ColumnCondition.Raw>>emptyList() : conditions,
ifExists);
}
;
@@ -535,12 +650,12 @@
| d=deleteStatement { $statement = d; }
;
-createAggregateStatement returns [CreateAggregateStatement expr]
+createAggregateStatement returns [CreateAggregateStatement.Raw stmt]
@init {
boolean orReplace = false;
boolean ifNotExists = false;
- List<CQL3Type.Raw> argsTypes = new ArrayList<>();
+ List<CQL3Type.Raw> argTypes = new ArrayList<>();
}
: K_CREATE (K_OR K_REPLACE { orReplace = true; })?
K_AGGREGATE
@@ -548,8 +663,8 @@
fn=functionName
'('
(
- v=comparatorType { argsTypes.add(v); }
- ( ',' v=comparatorType { argsTypes.add(v); } )*
+ v=comparatorType { argTypes.add(v); }
+ ( ',' v=comparatorType { argTypes.add(v); } )*
)?
')'
K_SFUNC sfunc = allowedFunctionName
@@ -560,14 +675,14 @@
(
K_INITCOND ival = term
)?
- { $expr = new CreateAggregateStatement(fn, argsTypes, sfunc, stype, ffunc, ival, orReplace, ifNotExists); }
+ { $stmt = new CreateAggregateStatement.Raw(fn, argTypes, stype, sfunc, ffunc, ival, orReplace, ifNotExists); }
;
-dropAggregateStatement returns [DropAggregateStatement expr]
+dropAggregateStatement returns [DropAggregateStatement.Raw stmt]
@init {
boolean ifExists = false;
- List<CQL3Type.Raw> argsTypes = new ArrayList<>();
- boolean argsPresent = false;
+ List<CQL3Type.Raw> argTypes = new ArrayList<>();
+ boolean argsSpecified = false;
}
: K_DROP K_AGGREGATE
(K_IF K_EXISTS { ifExists = true; } )?
@@ -575,22 +690,22 @@
(
'('
(
- v=comparatorType { argsTypes.add(v); }
- ( ',' v=comparatorType { argsTypes.add(v); } )*
+ v=comparatorType { argTypes.add(v); }
+ ( ',' v=comparatorType { argTypes.add(v); } )*
)?
')'
- { argsPresent = true; }
+ { argsSpecified = true; }
)?
- { $expr = new DropAggregateStatement(fn, argsTypes, argsPresent, ifExists); }
+ { $stmt = new DropAggregateStatement.Raw(fn, argTypes, argsSpecified, ifExists); }
;
-createFunctionStatement returns [CreateFunctionStatement expr]
+createFunctionStatement returns [CreateFunctionStatement.Raw stmt]
@init {
boolean orReplace = false;
boolean ifNotExists = false;
- List<ColumnIdentifier> argsNames = new ArrayList<>();
- List<CQL3Type.Raw> argsTypes = new ArrayList<>();
+ List<ColumnIdentifier> argNames = new ArrayList<>();
+ List<CQL3Type.Raw> argTypes = new ArrayList<>();
boolean calledOnNullInput = false;
}
: K_CREATE (K_OR K_REPLACE { orReplace = true; })?
@@ -599,23 +714,24 @@
fn=functionName
'('
(
- k=noncol_ident v=comparatorType { argsNames.add(k); argsTypes.add(v); }
- ( ',' k=noncol_ident v=comparatorType { argsNames.add(k); argsTypes.add(v); } )*
+ k=noncol_ident v=comparatorType { argNames.add(k); argTypes.add(v); }
+ ( ',' k=noncol_ident v=comparatorType { argNames.add(k); argTypes.add(v); } )*
)?
')'
( (K_RETURNS K_NULL) | (K_CALLED { calledOnNullInput=true; })) K_ON K_NULL K_INPUT
- K_RETURNS rt = comparatorType
+ K_RETURNS returnType = comparatorType
K_LANGUAGE language = IDENT
K_AS body = STRING_LITERAL
- { $expr = new CreateFunctionStatement(fn, $language.text.toLowerCase(), $body.text,
- argsNames, argsTypes, rt, calledOnNullInput, orReplace, ifNotExists); }
+ { $stmt = new CreateFunctionStatement.Raw(
+ fn, argNames, argTypes, returnType, calledOnNullInput, $language.text.toLowerCase(), $body.text, orReplace, ifNotExists);
+ }
;
-dropFunctionStatement returns [DropFunctionStatement expr]
+dropFunctionStatement returns [DropFunctionStatement.Raw stmt]
@init {
boolean ifExists = false;
- List<CQL3Type.Raw> argsTypes = new ArrayList<>();
- boolean argsPresent = false;
+ List<CQL3Type.Raw> argTypes = new ArrayList<>();
+ boolean argsSpecified = false;
}
: K_DROP K_FUNCTION
(K_IF K_EXISTS { ifExists = true; } )?
@@ -623,69 +739,71 @@
(
'('
(
- v=comparatorType { argsTypes.add(v); }
- ( ',' v=comparatorType { argsTypes.add(v); } )*
+ v=comparatorType { argTypes.add(v); }
+ ( ',' v=comparatorType { argTypes.add(v); } )*
)?
')'
- { argsPresent = true; }
+ { argsSpecified = true; }
)?
- { $expr = new DropFunctionStatement(fn, argsTypes, argsPresent, ifExists); }
+ { $stmt = new DropFunctionStatement.Raw(fn, argTypes, argsSpecified, ifExists); }
;
/**
* CREATE KEYSPACE [IF NOT EXISTS] <KEYSPACE> WITH attr1 = value1 AND attr2 = value2;
*/
-createKeyspaceStatement returns [CreateKeyspaceStatement expr]
+createKeyspaceStatement returns [CreateKeyspaceStatement.Raw stmt]
@init {
KeyspaceAttributes attrs = new KeyspaceAttributes();
boolean ifNotExists = false;
}
: K_CREATE K_KEYSPACE (K_IF K_NOT K_EXISTS { ifNotExists = true; } )? ks=keyspaceName
- K_WITH properties[attrs] { $expr = new CreateKeyspaceStatement(ks, attrs, ifNotExists); }
+ K_WITH properties[attrs] { $stmt = new CreateKeyspaceStatement.Raw(ks, attrs, ifNotExists); }
;
/**
- * CREATE COLUMNFAMILY [IF NOT EXISTS] <CF> (
+ * CREATE TABLE [IF NOT EXISTS] <CF> (
* <name1> <type>,
* <name2> <type>,
* <name3> <type>
* ) WITH <property> = <value> AND ...;
*/
-createTableStatement returns [CreateTableStatement.RawStatement expr]
+createTableStatement returns [CreateTableStatement.Raw stmt]
@init { boolean ifNotExists = false; }
: K_CREATE K_COLUMNFAMILY (K_IF K_NOT K_EXISTS { ifNotExists = true; } )?
- cf=columnFamilyName { $expr = new CreateTableStatement.RawStatement(cf, ifNotExists); }
- cfamDefinition[expr]
+ cf=columnFamilyName { $stmt = new CreateTableStatement.Raw(cf, ifNotExists); }
+ tableDefinition[stmt]
;
-cfamDefinition[CreateTableStatement.RawStatement expr]
- : '(' cfamColumns[expr] ( ',' cfamColumns[expr]? )* ')'
- ( K_WITH cfamProperty[expr.properties] ( K_AND cfamProperty[expr.properties] )*)?
+tableDefinition[CreateTableStatement.Raw stmt]
+ : '(' tableColumns[stmt] ( ',' tableColumns[stmt]? )* ')'
+ ( K_WITH tableProperty[stmt] ( K_AND tableProperty[stmt] )*)?
;
-cfamColumns[CreateTableStatement.RawStatement expr]
- : k=ident v=comparatorType { boolean isStatic=false; } (K_STATIC {isStatic = true;})? { $expr.addDefinition(k, v, isStatic); }
- (K_PRIMARY K_KEY { $expr.addKeyAliases(Collections.singletonList(k)); })?
- | K_PRIMARY K_KEY '(' pkDef[expr] (',' c=ident { $expr.addColumnAlias(c); } )* ')'
+tableColumns[CreateTableStatement.Raw stmt]
+ @init { boolean isStatic = false; }
+ : k=ident v=comparatorType (K_STATIC { isStatic = true; })? { $stmt.addColumn(k, v, isStatic); }
+ (K_PRIMARY K_KEY { $stmt.setPartitionKeyColumn(k); })?
+ | K_PRIMARY K_KEY '(' tablePartitionKey[stmt] (',' c=ident { $stmt.markClusteringColumn(c); } )* ')'
;
-pkDef[CreateTableStatement.RawStatement expr]
- : k=ident { $expr.addKeyAliases(Collections.singletonList(k)); }
- | '(' { List<ColumnIdentifier> l = new ArrayList<ColumnIdentifier>(); } k1=ident { l.add(k1); } ( ',' kn=ident { l.add(kn); } )* ')' { $expr.addKeyAliases(l); }
+tablePartitionKey[CreateTableStatement.Raw stmt]
+ @init {List<ColumnIdentifier> l = new ArrayList<ColumnIdentifier>();}
+ @after{ $stmt.setPartitionKeyColumns(l); }
+ : k1=ident { l.add(k1);}
+ | '(' k1=ident { l.add(k1); } ( ',' kn=ident { l.add(kn); } )* ')'
;
-cfamProperty[CFProperties props]
- : property[props.properties]
- | K_COMPACT K_STORAGE { $props.setCompactStorage(); }
- | K_CLUSTERING K_ORDER K_BY '(' cfamOrdering[props] (',' cfamOrdering[props])* ')'
+tableProperty[CreateTableStatement.Raw stmt]
+ : property[stmt.attrs]
+ | K_COMPACT K_STORAGE { throw new SyntaxException("COMPACT STORAGE tables are not allowed starting with version 4.0"); }
+ | K_CLUSTERING K_ORDER K_BY '(' tableClusteringOrder[stmt] (',' tableClusteringOrder[stmt])* ')'
;
-cfamOrdering[CFProperties props]
- @init{ boolean reversed=false; }
- : k=ident (K_ASC | K_DESC { reversed=true;} ) { $props.setOrdering(k, reversed); }
+tableClusteringOrder[CreateTableStatement.Raw stmt]
+ @init{ boolean ascending = true; }
+ : k=ident (K_ASC | K_DESC { ascending = false; } ) { $stmt.extendClusteringOrder(k, ascending); }
;
-
/**
* CREATE TYPE foo (
* <name1> <type1>,
@@ -693,34 +811,33 @@
* ....
* )
*/
-createTypeStatement returns [CreateTypeStatement expr]
+createTypeStatement returns [CreateTypeStatement.Raw stmt]
@init { boolean ifNotExists = false; }
: K_CREATE K_TYPE (K_IF K_NOT K_EXISTS { ifNotExists = true; } )?
- tn=userTypeName { $expr = new CreateTypeStatement(tn, ifNotExists); }
- '(' typeColumns[expr] ( ',' typeColumns[expr]? )* ')'
+ tn=userTypeName { $stmt = new CreateTypeStatement.Raw(tn, ifNotExists); }
+ '(' typeColumns[stmt] ( ',' typeColumns[stmt]? )* ')'
;
-typeColumns[CreateTypeStatement expr]
- : k=fident v=comparatorType { $expr.addDefinition(k, v); }
+typeColumns[CreateTypeStatement.Raw stmt]
+ : k=fident v=comparatorType { $stmt.addField(k, v); }
;
-
/**
* CREATE INDEX [IF NOT EXISTS] [indexName] ON <columnFamily> (<columnName>);
* CREATE CUSTOM INDEX [IF NOT EXISTS] [indexName] ON <columnFamily> (<columnName>) USING <indexClass>;
*/
-createIndexStatement returns [CreateIndexStatement expr]
+createIndexStatement returns [CreateIndexStatement.Raw stmt]
@init {
- IndexPropDefs props = new IndexPropDefs();
+ IndexAttributes props = new IndexAttributes();
boolean ifNotExists = false;
- IndexName name = new IndexName();
+ QualifiedName name = new QualifiedName();
List<IndexTarget.Raw> targets = new ArrayList<>();
}
: K_CREATE (K_CUSTOM { props.isCustom = true; })? K_INDEX (K_IF K_NOT K_EXISTS { ifNotExists = true; } )?
(idxName[name])? K_ON cf=columnFamilyName '(' (indexIdent[targets] (',' indexIdent[targets])*)? ')'
(K_USING cls=STRING_LITERAL { props.customClass = $cls.text; })?
(K_WITH properties[props])?
- { $expr = new CreateIndexStatement(cf, name, targets, props, ifNotExists); }
+ { $stmt = new CreateIndexStatement.Raw(cf, name, targets, props, ifNotExists); }
;
indexIdent [List<IndexTarget.Raw> targets]
@@ -739,107 +856,114 @@
* PRIMARY KEY (<pkColumns>)
* WITH <property> = <value> AND ...;
*/
-createMaterializedViewStatement returns [CreateViewStatement expr]
+createMaterializedViewStatement returns [CreateViewStatement.Raw stmt]
@init {
boolean ifNotExists = false;
- List<ColumnDefinition.Raw> partitionKeys = new ArrayList<>();
- List<ColumnDefinition.Raw> compositeKeys = new ArrayList<>();
}
: K_CREATE K_MATERIALIZED K_VIEW (K_IF K_NOT K_EXISTS { ifNotExists = true; })? cf=columnFamilyName K_AS
- K_SELECT sclause=selectClause K_FROM basecf=columnFamilyName
+ K_SELECT sclause=selectors K_FROM basecf=columnFamilyName
(K_WHERE wclause=whereClause)?
- K_PRIMARY K_KEY (
- '(' '(' k1=cident { partitionKeys.add(k1); } ( ',' kn=cident { partitionKeys.add(kn); } )* ')' ( ',' c1=cident { compositeKeys.add(c1); } )* ')'
- | '(' k1=cident { partitionKeys.add(k1); } ( ',' cn=cident { compositeKeys.add(cn); } )* ')'
- )
{
WhereClause where = wclause == null ? WhereClause.empty() : wclause.build();
- $expr = new CreateViewStatement(cf, basecf, sclause, where, partitionKeys, compositeKeys, ifNotExists);
+ $stmt = new CreateViewStatement.Raw(basecf, cf, sclause, where, ifNotExists);
}
- ( K_WITH cfamProperty[expr.properties] ( K_AND cfamProperty[expr.properties] )*)?
+ viewPrimaryKey[stmt]
+ ( K_WITH viewProperty[stmt] ( K_AND viewProperty[stmt] )*)?
+ ;
+
+viewPrimaryKey[CreateViewStatement.Raw stmt]
+ : K_PRIMARY K_KEY '(' viewPartitionKey[stmt] (',' c=ident { $stmt.markClusteringColumn(c); } )* ')'
+ ;
+
+viewPartitionKey[CreateViewStatement.Raw stmt]
+ @init {List<ColumnIdentifier> l = new ArrayList<ColumnIdentifier>();}
+ @after{ $stmt.setPartitionKeyColumns(l); }
+ : k1=ident { l.add(k1);}
+ | '(' k1=ident { l.add(k1); } ( ',' kn=ident { l.add(kn); } )* ')'
+ ;
+
+viewProperty[CreateViewStatement.Raw stmt]
+ : property[stmt.attrs]
+ | K_COMPACT K_STORAGE { throw new SyntaxException("COMPACT STORAGE tables are not allowed starting with version 4.0"); }
+ | K_CLUSTERING K_ORDER K_BY '(' viewClusteringOrder[stmt] (',' viewClusteringOrder[stmt])* ')'
+ ;
+
+viewClusteringOrder[CreateViewStatement.Raw stmt]
+ @init{ boolean ascending = true; }
+ : k=ident (K_ASC | K_DESC { ascending = false; } ) { $stmt.extendClusteringOrder(k, ascending); }
;
/**
* CREATE TRIGGER triggerName ON columnFamily USING 'triggerClass';
*/
-createTriggerStatement returns [CreateTriggerStatement expr]
+createTriggerStatement returns [CreateTriggerStatement.Raw stmt]
@init {
boolean ifNotExists = false;
}
: K_CREATE K_TRIGGER (K_IF K_NOT K_EXISTS { ifNotExists = true; } )? (name=ident)
K_ON cf=columnFamilyName K_USING cls=STRING_LITERAL
- { $expr = new CreateTriggerStatement(cf, name.toString(), $cls.text, ifNotExists); }
+ { $stmt = new CreateTriggerStatement.Raw(cf, name.toString(), $cls.text, ifNotExists); }
;
/**
* DROP TRIGGER [IF EXISTS] triggerName ON columnFamily;
*/
-dropTriggerStatement returns [DropTriggerStatement expr]
+dropTriggerStatement returns [DropTriggerStatement.Raw stmt]
@init { boolean ifExists = false; }
: K_DROP K_TRIGGER (K_IF K_EXISTS { ifExists = true; } )? (name=ident) K_ON cf=columnFamilyName
- { $expr = new DropTriggerStatement(cf, name.toString(), ifExists); }
+ { $stmt = new DropTriggerStatement.Raw(cf, name.toString(), ifExists); }
;
/**
* ALTER KEYSPACE <KS> WITH <property> = <value>;
*/
-alterKeyspaceStatement returns [AlterKeyspaceStatement expr]
+alterKeyspaceStatement returns [AlterKeyspaceStatement.Raw stmt]
@init { KeyspaceAttributes attrs = new KeyspaceAttributes(); }
: K_ALTER K_KEYSPACE ks=keyspaceName
- K_WITH properties[attrs] { $expr = new AlterKeyspaceStatement(ks, attrs); }
+ K_WITH properties[attrs] { $stmt = new AlterKeyspaceStatement.Raw(ks, attrs); }
;
/**
- * ALTER COLUMN FAMILY <CF> ALTER <column> TYPE <newtype>;
- * ALTER COLUMN FAMILY <CF> ADD <column> <newtype>; | ALTER COLUMN FAMILY <CF> ADD (<column> <newtype>,<column1> <newtype1>..... <column n> <newtype n>)
- * ALTER COLUMN FAMILY <CF> DROP <column>; | ALTER COLUMN FAMILY <CF> DROP ( <column>,<column1>.....<column n>)
- * ALTER COLUMN FAMILY <CF> WITH <property> = <value>;
- * ALTER COLUMN FAMILY <CF> RENAME <column> TO <column>;
+ * ALTER TABLE <table> ALTER <column> TYPE <newtype>;
+ * ALTER TABLE <table> ADD <column> <newtype>; | ALTER TABLE <table> ADD (<column> <newtype>,<column1> <newtype1>..... <column n> <newtype n>)
+ * ALTER TABLE <table> DROP <column>; | ALTER TABLE <table> DROP ( <column>,<column1>.....<column n>)
+ * ALTER TABLE <table> RENAME <column> TO <column>;
+ * ALTER TABLE <table> WITH <property> = <value>;
*/
-alterTableStatement returns [AlterTableStatement expr]
- @init {
- AlterTableStatement.Type type = null;
- TableAttributes attrs = new TableAttributes();
- Map<ColumnDefinition.Raw, ColumnDefinition.Raw> renames = new HashMap<ColumnDefinition.Raw, ColumnDefinition.Raw>();
- List<AlterTableStatementColumn> colNameList = new ArrayList<AlterTableStatementColumn>();
- Long deleteTimestamp = null;
- }
- : K_ALTER K_COLUMNFAMILY cf=columnFamilyName
- ( K_ALTER id=schema_cident K_TYPE v=comparatorType { type = AlterTableStatement.Type.ALTER; } { colNameList.add(new AlterTableStatementColumn(id,v)); }
- | K_ADD ( (aid=schema_cident v=comparatorType b1=cfisStatic { colNameList.add(new AlterTableStatementColumn(aid,v,b1)); })
- | ('(' id1=schema_cident v1=comparatorType b1=cfisStatic { colNameList.add(new AlterTableStatementColumn(id1,v1,b1)); }
- ( ',' idn=schema_cident vn=comparatorType bn=cfisStatic { colNameList.add(new AlterTableStatementColumn(idn,vn,bn)); } )* ')' ) ) { type = AlterTableStatement.Type.ADD; }
- | K_DROP K_COMPACT K_STORAGE { type = AlterTableStatement.Type.DROP_COMPACT_STORAGE; }
- | K_DROP ( ( id=schema_cident { colNameList.add(new AlterTableStatementColumn(id)); }
- | ('(' id1=schema_cident { colNameList.add(new AlterTableStatementColumn(id1)); }
- ( ',' idn=schema_cident { colNameList.add(new AlterTableStatementColumn(idn)); } )* ')') )
- ( K_USING K_TIMESTAMP t=INTEGER { deleteTimestamp = Long.parseLong(Constants.Literal.integer($t.text).getText()); })? ) { type = AlterTableStatement.Type.DROP; }
- | K_WITH properties[attrs] { type = AlterTableStatement.Type.OPTS; }
- | K_RENAME { type = AlterTableStatement.Type.RENAME; }
- id1=schema_cident K_TO toId1=schema_cident { renames.put(id1, toId1); }
- ( K_AND idn=schema_cident K_TO toIdn=schema_cident { renames.put(idn, toIdn); } )*
- )
- {
- $expr = new AlterTableStatement(cf, type, colNameList, attrs, renames, deleteTimestamp);
- }
+alterTableStatement returns [AlterTableStatement.Raw stmt]
+ : K_ALTER K_COLUMNFAMILY cf=columnFamilyName { $stmt = new AlterTableStatement.Raw(cf); }
+ (
+ K_ALTER id=cident K_TYPE v=comparatorType { $stmt.alter(id, v); }
+
+ | K_ADD ( id=schema_cident v=comparatorType b=isStaticColumn { $stmt.add(id, v, b); }
+ | ('(' id1=schema_cident v1=comparatorType b1=isStaticColumn { $stmt.add(id1, v1, b1); }
+ ( ',' idn=schema_cident vn=comparatorType bn=isStaticColumn { $stmt.add(idn, vn, bn); } )* ')') )
+
+ | K_DROP ( id=schema_cident { $stmt.drop(id); }
+ | ('(' id1=schema_cident { $stmt.drop(id1); }
+ ( ',' idn=schema_cident { $stmt.drop(idn); } )* ')') )
+ ( K_USING K_TIMESTAMP t=INTEGER { $stmt.timestamp(Long.parseLong(Constants.Literal.integer($t.text).getText())); } )?
+
+ | K_RENAME id1=schema_cident K_TO toId1=schema_cident { $stmt.rename(id1, toId1); }
+ ( K_AND idn=schema_cident K_TO toIdn=schema_cident { $stmt.rename(idn, toIdn); } )*
+
+ | K_WITH properties[$stmt.attrs] { $stmt.attrs(); }
+ )
;
-cfisStatic returns [boolean isStaticColumn]
- @init{
- boolean isStatic = false;
- }
- : (K_STATIC { isStatic=true; })? { $isStaticColumn = isStatic;
- }
+isStaticColumn returns [boolean isStaticColumn]
+ @init { boolean isStatic = false; }
+ : (K_STATIC { isStatic=true; })? { $isStaticColumn = isStatic; }
;
-alterMaterializedViewStatement returns [AlterViewStatement expr]
+alterMaterializedViewStatement returns [AlterViewStatement.Raw stmt]
@init {
TableAttributes attrs = new TableAttributes();
}
: K_ALTER K_MATERIALIZED K_VIEW name=columnFamilyName
K_WITH properties[attrs]
{
- $expr = new AlterViewStatement(name, attrs);
+ $stmt = new AlterViewStatement.Raw(name, attrs);
}
;
@@ -849,59 +973,58 @@
* ALTER TYPE <name> ADD <field> <newtype>;
* ALTER TYPE <name> RENAME <field> TO <newtype> AND ...;
*/
-alterTypeStatement returns [AlterTypeStatement expr]
- : K_ALTER K_TYPE name=userTypeName
- ( K_ALTER f=fident K_TYPE v=comparatorType { $expr = AlterTypeStatement.alter(name, f, v); }
- | K_ADD f=fident v=comparatorType { $expr = AlterTypeStatement.addition(name, f, v); }
- | K_RENAME
- { Map<FieldIdentifier, FieldIdentifier> renames = new HashMap<>(); }
- id1=fident K_TO toId1=fident { renames.put(id1, toId1); }
- ( K_AND idn=fident K_TO toIdn=fident { renames.put(idn, toIdn); } )*
- { $expr = AlterTypeStatement.renames(name, renames); }
- )
- ;
+alterTypeStatement returns [AlterTypeStatement.Raw stmt]
+ : K_ALTER K_TYPE name=userTypeName { $stmt = new AlterTypeStatement.Raw(name); }
+ (
+ K_ALTER f=fident K_TYPE v=comparatorType { $stmt.alter(f, v); }
+ | K_ADD f=fident v=comparatorType { $stmt.add(f, v); }
+
+ | K_RENAME f1=fident K_TO toF1=fident { $stmt.rename(f1, toF1); }
+ ( K_AND fn=fident K_TO toFn=fident { $stmt.rename(fn, toFn); } )*
+ )
+ ;
/**
* DROP KEYSPACE [IF EXISTS] <KSP>;
*/
-dropKeyspaceStatement returns [DropKeyspaceStatement ksp]
+dropKeyspaceStatement returns [DropKeyspaceStatement.Raw stmt]
@init { boolean ifExists = false; }
- : K_DROP K_KEYSPACE (K_IF K_EXISTS { ifExists = true; } )? ks=keyspaceName { $ksp = new DropKeyspaceStatement(ks, ifExists); }
+ : K_DROP K_KEYSPACE (K_IF K_EXISTS { ifExists = true; } )? ks=keyspaceName { $stmt = new DropKeyspaceStatement.Raw(ks, ifExists); }
;
/**
- * DROP COLUMNFAMILY [IF EXISTS] <CF>;
+ * DROP TABLE [IF EXISTS] <table>;
*/
-dropTableStatement returns [DropTableStatement stmt]
+dropTableStatement returns [DropTableStatement.Raw stmt]
@init { boolean ifExists = false; }
- : K_DROP K_COLUMNFAMILY (K_IF K_EXISTS { ifExists = true; } )? cf=columnFamilyName { $stmt = new DropTableStatement(cf, ifExists); }
+ : K_DROP K_COLUMNFAMILY (K_IF K_EXISTS { ifExists = true; } )? name=columnFamilyName { $stmt = new DropTableStatement.Raw(name, ifExists); }
;
/**
* DROP TYPE <name>;
*/
-dropTypeStatement returns [DropTypeStatement stmt]
+dropTypeStatement returns [DropTypeStatement.Raw stmt]
@init { boolean ifExists = false; }
- : K_DROP K_TYPE (K_IF K_EXISTS { ifExists = true; } )? name=userTypeName { $stmt = new DropTypeStatement(name, ifExists); }
+ : K_DROP K_TYPE (K_IF K_EXISTS { ifExists = true; } )? name=userTypeName { $stmt = new DropTypeStatement.Raw(name, ifExists); }
;
/**
* DROP INDEX [IF EXISTS] <INDEX_NAME>
*/
-dropIndexStatement returns [DropIndexStatement expr]
+dropIndexStatement returns [DropIndexStatement.Raw stmt]
@init { boolean ifExists = false; }
: K_DROP K_INDEX (K_IF K_EXISTS { ifExists = true; } )? index=indexName
- { $expr = new DropIndexStatement(index, ifExists); }
+ { $stmt = new DropIndexStatement.Raw(index, ifExists); }
;
/**
* DROP MATERIALIZED VIEW [IF EXISTS] <view_name>
*/
-dropMaterializedViewStatement returns [DropViewStatement expr]
+dropMaterializedViewStatement returns [DropViewStatement.Raw stmt]
@init { boolean ifExists = false; }
: K_DROP K_MATERIALIZED K_VIEW (K_IF K_EXISTS { ifExists = true; } )? cf=columnFamilyName
- { $expr = new DropViewStatement(cf, ifExists); }
+ { $stmt = new DropViewStatement.Raw(cf, ifExists); }
;
/**
@@ -994,7 +1117,7 @@
: K_ALL K_KEYSPACES { $res = DataResource.root(); }
| K_KEYSPACE ks = keyspaceName { $res = DataResource.keyspace($ks.id); }
| ( K_COLUMNFAMILY )? cf = columnFamilyName
- { $res = DataResource.table($cf.name.getKeyspace(), $cf.name.getColumnFamily()); }
+ { $res = DataResource.table($cf.name.getKeyspace(), $cf.name.getName()); }
;
jmxResource returns [JMXResource res]
@@ -1044,7 +1167,7 @@
( K_WITH userPassword[opts] )?
( K_SUPERUSER { superuser = true; } | K_NOSUPERUSER { superuser = false; } )?
{ opts.setOption(IRoleManager.Option.SUPERUSER, superuser);
- $stmt = new CreateRoleStatement(name, opts, ifNotExists); }
+ $stmt = new CreateRoleStatement(name, opts, DCPermissions.all(), ifNotExists); }
;
/**
@@ -1059,7 +1182,7 @@
( K_WITH userPassword[opts] )?
( K_SUPERUSER { opts.setOption(IRoleManager.Option.SUPERUSER, true); }
| K_NOSUPERUSER { opts.setOption(IRoleManager.Option.SUPERUSER, false); } ) ?
- { $stmt = new AlterRoleStatement(name, opts); }
+ { $stmt = new AlterRoleStatement(name, opts, null); }
;
/**
@@ -1092,10 +1215,11 @@
createRoleStatement returns [CreateRoleStatement stmt]
@init {
RoleOptions opts = new RoleOptions();
+ DCPermissions.Builder dcperms = DCPermissions.builder();
boolean ifNotExists = false;
}
: K_CREATE K_ROLE (K_IF K_NOT K_EXISTS { ifNotExists = true; })? name=userOrRoleName
- ( K_WITH roleOptions[opts] )?
+ ( K_WITH roleOptions[opts, dcperms] )?
{
// set defaults if they weren't explictly supplied
if (!opts.getLogin().isPresent())
@@ -1106,7 +1230,7 @@
{
opts.setOption(IRoleManager.Option.SUPERUSER, false);
}
- $stmt = new CreateRoleStatement(name, opts, ifNotExists);
+ $stmt = new CreateRoleStatement(name, opts, dcperms.build(), ifNotExists);
}
;
@@ -1122,10 +1246,11 @@
alterRoleStatement returns [AlterRoleStatement stmt]
@init {
RoleOptions opts = new RoleOptions();
+ DCPermissions.Builder dcperms = DCPermissions.builder();
}
: K_ALTER K_ROLE name=userOrRoleName
- ( K_WITH roleOptions[opts] )?
- { $stmt = new AlterRoleStatement(name, opts); }
+ ( K_WITH roleOptions[opts, dcperms] )?
+ { $stmt = new AlterRoleStatement(name, opts, dcperms.isModified() ? dcperms.build() : null); }
;
/**
@@ -1153,15 +1278,21 @@
{ $stmt = new ListRolesStatement(grantee, recursive); }
;
-roleOptions[RoleOptions opts]
- : roleOption[opts] (K_AND roleOption[opts])*
+roleOptions[RoleOptions opts, DCPermissions.Builder dcperms]
+ : roleOption[opts, dcperms] (K_AND roleOption[opts, dcperms])*
;
-roleOption[RoleOptions opts]
+roleOption[RoleOptions opts, DCPermissions.Builder dcperms]
: K_PASSWORD '=' v=STRING_LITERAL { opts.setOption(IRoleManager.Option.PASSWORD, $v.text); }
- | K_OPTIONS '=' m=mapLiteral { opts.setOption(IRoleManager.Option.OPTIONS, convertPropertyMap(m)); }
+ | K_OPTIONS '=' m=fullMapLiteral { opts.setOption(IRoleManager.Option.OPTIONS, convertPropertyMap(m)); }
| K_SUPERUSER '=' b=BOOLEAN { opts.setOption(IRoleManager.Option.SUPERUSER, Boolean.valueOf($b.text)); }
| K_LOGIN '=' b=BOOLEAN { opts.setOption(IRoleManager.Option.LOGIN, Boolean.valueOf($b.text)); }
+ | K_ACCESS K_TO K_ALL K_DATACENTERS { dcperms.all(); }
+ | K_ACCESS K_TO K_DATACENTERS '{' dcPermission[dcperms] (',' dcPermission[dcperms])* '}'
+ ;
+
+dcPermission[DCPermissions.Builder builder]
+ : dc=STRING_LITERAL { builder.add($dc.text); }
;
// for backwards compatibility in CREATE/ALTER USER, this has no '='
@@ -1169,6 +1300,47 @@
: K_PASSWORD v=STRING_LITERAL { opts.setOption(IRoleManager.Option.PASSWORD, $v.text); }
;
+/**
+ * DESCRIBE statement(s)
+ *
+ * Must be in sync with the javadoc for org.apache.cassandra.cql3.statements.DescribeStatement and the
+ * cqlsh syntax definition in for cqlsh_describe_cmd_syntax_rules pylib/cqlshlib/cqlshhandling.py.
+ */
+describeStatement returns [DescribeStatement stmt]
+ @init {
+ boolean fullSchema = false;
+ boolean pending = false;
+ boolean config = false;
+ boolean only = false;
+ QualifiedName gen = new QualifiedName();
+ }
+ : ( K_DESCRIBE | K_DESC )
+ ( (K_CLUSTER)=> K_CLUSTER { $stmt = DescribeStatement.cluster(); }
+ | (K_FULL { fullSchema=true; })? K_SCHEMA { $stmt = DescribeStatement.schema(fullSchema); }
+ | (K_KEYSPACES)=> K_KEYSPACES { $stmt = DescribeStatement.keyspaces(); }
+ | (K_ONLY { only=true; })? K_KEYSPACE ( ks=keyspaceName )?
+ { $stmt = DescribeStatement.keyspace(ks, only); }
+ | (K_TABLES) => K_TABLES { $stmt = DescribeStatement.tables(); }
+ | K_COLUMNFAMILY cf=columnFamilyName { $stmt = DescribeStatement.table(cf.getKeyspace(), cf.getName()); }
+ | K_INDEX idx=columnFamilyName { $stmt = DescribeStatement.index(idx.getKeyspace(), idx.getName()); }
+ | K_MATERIALIZED K_VIEW view=columnFamilyName { $stmt = DescribeStatement.view(view.getKeyspace(), view.getName()); }
+ | (K_TYPES) => K_TYPES { $stmt = DescribeStatement.types(); }
+ | K_TYPE tn=userTypeName { $stmt = DescribeStatement.type(tn.getKeyspace(), tn.getStringTypeName()); }
+ | (K_FUNCTIONS) => K_FUNCTIONS { $stmt = DescribeStatement.functions(); }
+ | K_FUNCTION fn=functionName { $stmt = DescribeStatement.function(fn.keyspace, fn.name); }
+ | (K_AGGREGATES) => K_AGGREGATES { $stmt = DescribeStatement.aggregates(); }
+ | K_AGGREGATE ag=functionName { $stmt = DescribeStatement.aggregate(ag.keyspace, ag.name); }
+ | ( ( ksT=IDENT { gen.setKeyspace($ksT.text, false);}
+ | ksT=QUOTED_NAME { gen.setKeyspace($ksT.text, true);}
+ | ksK=unreserved_keyword { gen.setKeyspace(ksK, false);} ) '.' )?
+ ( tT=IDENT { gen.setName($tT.text, false);}
+ | tT=QUOTED_NAME { gen.setName($tT.text, true);}
+ | tK=unreserved_keyword { gen.setName(tK, false);} )
+ { $stmt = DescribeStatement.generic(gen.getKeyspace(), gen.getName()); }
+ )
+ ( K_WITH K_INTERNALS { $stmt.withInternalDetails(); } )?
+ ;
+
/** DEFINITIONS **/
// Column Identifiers. These need to be treated differently from other
@@ -1177,17 +1349,17 @@
// Also, we need to support the internal of the super column map (for backward
// compatibility) which is empty (we only want to allow this is in data manipulation
// queries, not in schema defition etc).
-cident returns [ColumnDefinition.Raw id]
- : EMPTY_QUOTED_NAME { $id = ColumnDefinition.Raw.forQuoted(""); }
- | t=IDENT { $id = ColumnDefinition.Raw.forUnquoted($t.text); }
- | t=QUOTED_NAME { $id = ColumnDefinition.Raw.forQuoted($t.text); }
- | k=unreserved_keyword { $id = ColumnDefinition.Raw.forUnquoted(k); }
+cident returns [ColumnMetadata.Raw id]
+ : EMPTY_QUOTED_NAME { $id = ColumnMetadata.Raw.forQuoted(""); }
+ | t=IDENT { $id = ColumnMetadata.Raw.forUnquoted($t.text); }
+ | t=QUOTED_NAME { $id = ColumnMetadata.Raw.forQuoted($t.text); }
+ | k=unreserved_keyword { $id = ColumnMetadata.Raw.forUnquoted(k); }
;
-schema_cident returns [ColumnDefinition.Raw id]
- : t=IDENT { $id = ColumnDefinition.Raw.forUnquoted($t.text); }
- | t=QUOTED_NAME { $id = ColumnDefinition.Raw.forQuoted($t.text); }
- | k=unreserved_keyword { $id = ColumnDefinition.Raw.forUnquoted(k); }
+schema_cident returns [ColumnMetadata.Raw id]
+ : t=IDENT { $id = ColumnMetadata.Raw.forUnquoted($t.text); }
+ | t=QUOTED_NAME { $id = ColumnMetadata.Raw.forQuoted($t.text); }
+ | k=unreserved_keyword { $id = ColumnMetadata.Raw.forUnquoted(k); }
;
// Column identifiers where the comparator is known to be text
@@ -1212,17 +1384,17 @@
// Keyspace & Column family names
keyspaceName returns [String id]
- @init { CFName name = new CFName(); }
+ @init { QualifiedName name = new QualifiedName(); }
: ksName[name] { $id = name.getKeyspace(); }
;
-indexName returns [IndexName name]
- @init { $name = new IndexName(); }
+indexName returns [QualifiedName name]
+ @init { $name = new QualifiedName(); }
: (ksName[name] '.')? idxName[name]
;
-columnFamilyName returns [CFName name]
- @init { $name = new CFName(); }
+columnFamilyName returns [QualifiedName name]
+ @init { $name = new QualifiedName(); }
: (ksName[name] '.')? cfName[name]
;
@@ -1235,24 +1407,24 @@
: roleName[role] {$name = role;}
;
-ksName[KeyspaceElementName name]
+ksName[QualifiedName name]
: t=IDENT { $name.setKeyspace($t.text, false);}
| t=QUOTED_NAME { $name.setKeyspace($t.text, true);}
| k=unreserved_keyword { $name.setKeyspace(k, false);}
| QMARK {addRecognitionError("Bind variables cannot be used for keyspace names");}
;
-cfName[CFName name]
- : t=IDENT { $name.setColumnFamily($t.text, false); }
- | t=QUOTED_NAME { $name.setColumnFamily($t.text, true); }
- | k=unreserved_keyword { $name.setColumnFamily(k, false); }
+cfName[QualifiedName name]
+ : t=IDENT { $name.setName($t.text, false); }
+ | t=QUOTED_NAME { $name.setName($t.text, true); }
+ | k=unreserved_keyword { $name.setName(k, false); }
| QMARK {addRecognitionError("Bind variables cannot be used for table names");}
;
-idxName[IndexName name]
- : t=IDENT { $name.setIndex($t.text, false); }
- | t=QUOTED_NAME { $name.setIndex($t.text, true);}
- | k=unreserved_keyword { $name.setIndex(k, false); }
+idxName[QualifiedName name]
+ : t=IDENT { $name.setName($t.text, false); }
+ | t=QUOTED_NAME { $name.setName($t.text, true);}
+ | k=unreserved_keyword { $name.setName(k, false); }
| QMARK {addRecognitionError("Bind variables cannot be used for index names");}
;
@@ -1272,34 +1444,49 @@
| t=DURATION { $constant = Constants.Literal.duration($t.text);}
| t=UUID { $constant = Constants.Literal.uuid($t.text); }
| t=HEXNUMBER { $constant = Constants.Literal.hex($t.text); }
- | { String sign=""; } ('-' {sign = "-"; } )? t=(K_NAN | K_INFINITY) { $constant = Constants.Literal.floatingPoint(sign + $t.text); }
+ | ((K_POSITIVE_NAN | K_NEGATIVE_NAN) { $constant = Constants.Literal.floatingPoint("NaN"); }
+ | K_POSITIVE_INFINITY { $constant = Constants.Literal.floatingPoint("Infinity"); }
+ | K_NEGATIVE_INFINITY { $constant = Constants.Literal.floatingPoint("-Infinity"); })
;
-mapLiteral returns [Maps.Literal map]
- : '{' { List<Pair<Term.Raw, Term.Raw>> m = new ArrayList<Pair<Term.Raw, Term.Raw>>(); }
- ( k1=term ':' v1=term { m.add(Pair.create(k1, v1)); } ( ',' kn=term ':' vn=term { m.add(Pair.create(kn, vn)); } )* )?
- '}' { $map = new Maps.Literal(m); }
+fullMapLiteral returns [Maps.Literal map]
+ @init { List<Pair<Term.Raw, Term.Raw>> m = new ArrayList<Pair<Term.Raw, Term.Raw>>();}
+ @after{ $map = new Maps.Literal(m); }
+ : '{' ( k1=term ':' v1=term { m.add(Pair.create(k1, v1)); } ( ',' kn=term ':' vn=term { m.add(Pair.create(kn, vn)); } )* )?
+ '}'
;
setOrMapLiteral[Term.Raw t] returns [Term.Raw value]
- : ':' v=term { List<Pair<Term.Raw, Term.Raw>> m = new ArrayList<Pair<Term.Raw, Term.Raw>>(); m.add(Pair.create(t, v)); }
- ( ',' kn=term ':' vn=term { m.add(Pair.create(kn, vn)); } )*
- { $value = new Maps.Literal(m); }
- | { List<Term.Raw> s = new ArrayList<Term.Raw>(); s.add(t); }
- ( ',' tn=term { s.add(tn); } )*
- { $value = new Sets.Literal(s); }
+ : m=mapLiteral[t] { $value=m; }
+ | s=setLiteral[t] { $value=s; }
+ ;
+
+setLiteral[Term.Raw t] returns [Term.Raw value]
+ @init { List<Term.Raw> s = new ArrayList<Term.Raw>(); s.add(t); }
+ @after { $value = new Sets.Literal(s); }
+ : ( ',' tn=term { s.add(tn); } )*
+ ;
+
+mapLiteral[Term.Raw k] returns [Term.Raw value]
+ @init { List<Pair<Term.Raw, Term.Raw>> m = new ArrayList<Pair<Term.Raw, Term.Raw>>(); }
+ @after { $value = new Maps.Literal(m); }
+ : ':' v=term { m.add(Pair.create(k, v)); } ( ',' kn=term ':' vn=term { m.add(Pair.create(kn, vn)); } )*
;
collectionLiteral returns [Term.Raw value]
- : '[' { List<Term.Raw> l = new ArrayList<Term.Raw>(); }
- ( t1=term { l.add(t1); } ( ',' tn=term { l.add(tn); } )* )?
- ']' { $value = new Lists.Literal(l); }
+ : l=listLiteral { $value = l; }
| '{' t=term v=setOrMapLiteral[t] { $value = v; } '}'
// Note that we have an ambiguity between maps and set for "{}". So we force it to a set literal,
// and deal with it later based on the type of the column (SetLiteral.java).
| '{' '}' { $value = new Sets.Literal(Collections.<Term.Raw>emptyList()); }
;
+listLiteral returns [Term.Raw value]
+ @init {List<Term.Raw> l = new ArrayList<Term.Raw>();}
+ @after {$value = new Lists.Literal(l);}
+ : '[' ( t1=term { l.add(t1); } ( ',' tn=term { l.add(tn); } )* )? ']' { $value = new Lists.Literal(l); }
+ ;
+
usertypeLiteral returns [UserTypes.Literal ut]
@init{ Map<FieldIdentifier, Term.Raw> m = new HashMap<>(); }
@after{ $ut = new UserTypes.Literal(m); }
@@ -1354,23 +1541,47 @@
;
term returns [Term.Raw term]
- : v=value { $term = v; }
- | f=function { $term = f; }
- | '(' c=comparatorType ')' t=term { $term = new TypeCast(c, t); }
+ : t=termAddition { $term = t; }
;
-columnOperation[List<Pair<ColumnDefinition.Raw, Operation.RawUpdate>> operations]
+termAddition returns [Term.Raw term]
+ : l=termMultiplication {$term = l;}
+ ( '+' r=termMultiplication {$term = FunctionCall.Raw.newOperation('+', $term, r);}
+ | '-' r=termMultiplication {$term = FunctionCall.Raw.newOperation('-', $term, r);}
+ )*
+ ;
+
+termMultiplication returns [Term.Raw term]
+ : l=termGroup {$term = l;}
+ ( '\*' r=termGroup {$term = FunctionCall.Raw.newOperation('*', $term, r);}
+ | '/' r=termGroup {$term = FunctionCall.Raw.newOperation('/', $term, r);}
+ | '%' r=termGroup {$term = FunctionCall.Raw.newOperation('\%', $term, r);}
+ )*
+ ;
+
+termGroup returns [Term.Raw term]
+ : t=simpleTerm { $term = t; }
+ | '-' t=simpleTerm { $term = FunctionCall.Raw.newNegation(t); }
+ ;
+
+simpleTerm returns [Term.Raw term]
+ : v=value { $term = v; }
+ | f=function { $term = f; }
+ | '(' c=comparatorType ')' t=simpleTerm { $term = new TypeCast(c, t); }
+ ;
+
+columnOperation[List<Pair<ColumnMetadata.Raw, Operation.RawUpdate>> operations]
: key=cident columnOperationDifferentiator[operations, key]
;
-columnOperationDifferentiator[List<Pair<ColumnDefinition.Raw, Operation.RawUpdate>> operations, ColumnDefinition.Raw key]
+columnOperationDifferentiator[List<Pair<ColumnMetadata.Raw, Operation.RawUpdate>> operations, ColumnMetadata.Raw key]
: '=' normalColumnOperation[operations, key]
| shorthandColumnOperation[operations, key]
| '[' k=term ']' collectionColumnOperation[operations, key, k]
| '.' field=fident udtColumnOperation[operations, key, field]
;
-normalColumnOperation[List<Pair<ColumnDefinition.Raw, Operation.RawUpdate>> operations, ColumnDefinition.Raw key]
+normalColumnOperation[List<Pair<ColumnMetadata.Raw, Operation.RawUpdate>> operations, ColumnMetadata.Raw key]
: t=term ('+' c=cident )?
{
if (c == null)
@@ -1400,28 +1611,28 @@
}
;
-shorthandColumnOperation[List<Pair<ColumnDefinition.Raw, Operation.RawUpdate>> operations, ColumnDefinition.Raw key]
+shorthandColumnOperation[List<Pair<ColumnMetadata.Raw, Operation.RawUpdate>> operations, ColumnMetadata.Raw key]
: sig=('+=' | '-=') t=term
{
addRawUpdate(operations, key, $sig.text.equals("+=") ? new Operation.Addition(t) : new Operation.Substraction(t));
}
;
-collectionColumnOperation[List<Pair<ColumnDefinition.Raw, Operation.RawUpdate>> operations, ColumnDefinition.Raw key, Term.Raw k]
+collectionColumnOperation[List<Pair<ColumnMetadata.Raw, Operation.RawUpdate>> operations, ColumnMetadata.Raw key, Term.Raw k]
: '=' t=term
{
addRawUpdate(operations, key, new Operation.SetElement(k, t));
}
;
-udtColumnOperation[List<Pair<ColumnDefinition.Raw, Operation.RawUpdate>> operations, ColumnDefinition.Raw key, FieldIdentifier field]
+udtColumnOperation[List<Pair<ColumnMetadata.Raw, Operation.RawUpdate>> operations, ColumnMetadata.Raw key, FieldIdentifier field]
: '=' t=term
{
addRawUpdate(operations, key, new Operation.SetField(field, t));
}
;
-columnCondition[List<Pair<ColumnDefinition.Raw, ColumnCondition.Raw>> conditions]
+columnCondition[List<Pair<ColumnMetadata.Raw, ColumnCondition.Raw>> conditions]
// Note: we'll reject duplicates later
: key=cident
( op=relationType t=term { conditions.add(Pair.create(key, ColumnCondition.Raw.simpleCondition(t, op))); }
@@ -1452,7 +1663,7 @@
property[PropertyDefinitions props]
: k=noncol_ident '=' simple=propertyValue { try { $props.addProperty(k.toString(), simple); } catch (SyntaxException e) { addRecognitionError(e.getMessage()); } }
- | k=noncol_ident '=' map=mapLiteral { try { $props.addProperty(k.toString(), convertPropertyMap(map)); } catch (SyntaxException e) { addRecognitionError(e.getMessage()); } }
+ | k=noncol_ident '=' map=fullMapLiteral { try { $props.addProperty(k.toString(), convertPropertyMap(map)); } catch (SyntaxException e) { addRecognitionError(e.getMessage()); } }
;
propertyValue returns [String str]
@@ -1479,8 +1690,7 @@
{ $clauses.add(new SingleColumnRelation(name, Operator.IN, marker)); }
| name=cident K_IN inValues=singleColumnInValues
{ $clauses.add(SingleColumnRelation.createInRelation($name.id, inValues)); }
- | name=cident K_CONTAINS { Operator rt = Operator.CONTAINS; } (K_KEY { rt = Operator.CONTAINS_KEY; })?
- t=term { $clauses.add(new SingleColumnRelation(name, rt, t)); }
+ | name=cident rt=containsOperator t=term { $clauses.add(new SingleColumnRelation(name, rt, t)); }
| name=cident '[' key=term ']' type=relationType t=term { $clauses.add(new SingleColumnRelation(name, key, type, t)); }
| ids=tupleOfIdentifiers
( K_IN
@@ -1505,13 +1715,17 @@
| '(' relation[$clauses] ')'
;
+containsOperator returns [Operator o]
+ : K_CONTAINS { o = Operator.CONTAINS; } (K_KEY { o = Operator.CONTAINS_KEY; })?
+ ;
+
inMarker returns [AbstractMarker.INRaw marker]
: QMARK { $marker = newINBindVariables(null); }
| ':' name=noncol_ident { $marker = newINBindVariables(name); }
;
-tupleOfIdentifiers returns [List<ColumnDefinition.Raw> ids]
- @init { $ids = new ArrayList<ColumnDefinition.Raw>(); }
+tupleOfIdentifiers returns [List<ColumnMetadata.Raw> ids]
+ @init { $ids = new ArrayList<ColumnMetadata.Raw>(); }
: '(' n1=cident { $ids.add(n1); } (',' ni=cident { $ids.add(ni); })* ')'
;
@@ -1548,7 +1762,7 @@
| K_FROZEN '<' f=comparatorType '>'
{
try {
- $t = CQL3Type.Raw.frozen(f);
+ $t = f.freeze();
} catch (InvalidRequestException e) {
addRecognitionError(e.getMessage());
}
@@ -1603,9 +1817,9 @@
;
tuple_type returns [CQL3Type.Raw t]
- : K_TUPLE '<' { List<CQL3Type.Raw> types = new ArrayList<>(); }
- t1=comparatorType { types.add(t1); } (',' tn=comparatorType { types.add(tn); })*
- '>' { $t = CQL3Type.Raw.tuple(types); }
+ @init {List<CQL3Type.Raw> types = new ArrayList<>();}
+ @after {$t = CQL3Type.Raw.tuple(types);}
+ : K_TUPLE '<' t1=comparatorType { types.add(t1); } (',' tn=comparatorType { types.add(tn); })* '>'
;
username
@@ -1640,10 +1854,13 @@
basic_unreserved_keyword returns [String str]
: k=( K_KEYS
| K_AS
+ | K_CLUSTER
| K_CLUSTERING
| K_COMPACT
| K_STORAGE
+ | K_TABLES
| K_TYPE
+ | K_TYPES
| K_VALUES
| K_MAP
| K_LIST
@@ -1666,12 +1883,15 @@
| K_CUSTOM
| K_TRIGGER
| K_CONTAINS
+ | K_INTERNALS
+ | K_ONLY
| K_STATIC
| K_FROZEN
| K_TUPLE
| K_FUNCTION
| K_FUNCTIONS
| K_AGGREGATE
+ | K_AGGREGATES
| K_SFUNC
| K_STYPE
| K_FINALFUNC
diff --git a/src/java/org/apache/cassandra/audit/AuditEvent.java b/src/java/org/apache/cassandra/audit/AuditEvent.java
new file mode 100644
index 0000000..b21fe58
--- /dev/null
+++ b/src/java/org/apache/cassandra/audit/AuditEvent.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.audit;
+
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.cassandra.diag.DiagnosticEvent;
+import org.apache.cassandra.diag.DiagnosticEventService;
+
+/**
+ * {@Link AuditLogEntry} wrapper to expose audit events as {@link DiagnosticEvent}s.
+ */
+public final class AuditEvent extends DiagnosticEvent
+{
+ private final AuditLogEntry entry;
+
+ private AuditEvent(AuditLogEntry entry)
+ {
+ this.entry = entry;
+ }
+
+ static void create(AuditLogEntry entry)
+ {
+ if (isEnabled(entry.getType()))
+ DiagnosticEventService.instance().publish(new AuditEvent(entry));
+ }
+
+ private static boolean isEnabled(AuditLogEntryType type)
+ {
+ return DiagnosticEventService.instance().isEnabled(AuditEvent.class, type);
+ }
+
+ public Enum<?> getType()
+ {
+ return entry.getType();
+ }
+
+ public String getSource()
+ {
+ return entry.getSource().toString(true);
+ }
+
+ public AuditLogEntry getEntry()
+ {
+ return entry;
+ }
+
+ public Map<String, Serializable> toMap()
+ {
+ HashMap<String, Serializable> ret = new HashMap<>();
+ if (entry.getKeyspace() != null) ret.put("keyspace", entry.getKeyspace());
+ if (entry.getOperation() != null) ret.put("operation", entry.getOperation());
+ if (entry.getScope() != null) ret.put("scope", entry.getScope());
+ if (entry.getUser() != null) ret.put("user", entry.getUser());
+ return ret;
+ }
+}
diff --git a/src/java/org/apache/cassandra/audit/AuditLogContext.java b/src/java/org/apache/cassandra/audit/AuditLogContext.java
new file mode 100644
index 0000000..9b44cf3
--- /dev/null
+++ b/src/java/org/apache/cassandra/audit/AuditLogContext.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.audit;
+
+public class AuditLogContext
+{
+ public final AuditLogEntryType auditLogEntryType;
+ public final String keyspace;
+ public final String scope;
+
+ public AuditLogContext(AuditLogEntryType auditLogEntryType)
+ {
+ this(auditLogEntryType, null, null);
+ }
+
+ public AuditLogContext(AuditLogEntryType auditLogEntryType, String keyspace)
+ {
+ this(auditLogEntryType, keyspace, null);
+ }
+
+ public AuditLogContext(AuditLogEntryType auditLogEntryType, String keyspace, String scope)
+ {
+ this.auditLogEntryType = auditLogEntryType;
+ this.keyspace = keyspace;
+ this.scope = scope;
+ }
+}
diff --git a/src/java/org/apache/cassandra/audit/AuditLogEntry.java b/src/java/org/apache/cassandra/audit/AuditLogEntry.java
new file mode 100644
index 0000000..4d3b867
--- /dev/null
+++ b/src/java/org/apache/cassandra/audit/AuditLogEntry.java
@@ -0,0 +1,320 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.audit;
+
+import java.net.InetSocketAddress;
+import java.net.UnknownHostException;
+import java.util.UUID;
+import javax.annotation.Nullable;
+
+import org.apache.commons.lang3.StringUtils;
+
+import org.apache.cassandra.auth.AuthenticatedUser;
+import org.apache.cassandra.cql3.CQLStatement;
+import org.apache.cassandra.cql3.QueryOptions;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.service.QueryState;
+import org.apache.cassandra.utils.FBUtilities;
+
+public class AuditLogEntry
+{
+ private final InetAddressAndPort host = FBUtilities.getBroadcastAddressAndPort();
+ private final InetAddressAndPort source;
+ private final String user;
+ private final long timestamp;
+ private final AuditLogEntryType type;
+ private final UUID batch;
+ private final String keyspace;
+ private final String scope;
+ private final String operation;
+ private final QueryOptions options;
+ private final QueryState state;
+
+ private AuditLogEntry(AuditLogEntryType type,
+ InetAddressAndPort source,
+ String user,
+ long timestamp,
+ UUID batch,
+ String keyspace,
+ String scope,
+ String operation,
+ QueryOptions options,
+ QueryState state)
+ {
+ this.type = type;
+ this.source = source;
+ this.user = user;
+ this.timestamp = timestamp;
+ this.batch = batch;
+ this.keyspace = keyspace;
+ this.scope = scope;
+ this.operation = operation;
+ this.options = options;
+ this.state = state;
+ }
+
+ String getLogString()
+ {
+ StringBuilder builder = new StringBuilder(100);
+ builder.append("user:").append(user)
+ .append("|host:").append(host)
+ .append("|source:").append(source.address);
+ if (source.port > 0)
+ {
+ builder.append("|port:").append(source.port);
+ }
+
+ builder.append("|timestamp:").append(timestamp)
+ .append("|type:").append(type)
+ .append("|category:").append(type.getCategory());
+
+ if (batch != null)
+ {
+ builder.append("|batch:").append(batch);
+ }
+ if (StringUtils.isNotBlank(keyspace))
+ {
+ builder.append("|ks:").append(keyspace);
+ }
+ if (StringUtils.isNotBlank(scope))
+ {
+ builder.append("|scope:").append(scope);
+ }
+ if (StringUtils.isNotBlank(operation))
+ {
+ builder.append("|operation:").append(operation);
+ }
+ return builder.toString();
+ }
+
+ public InetAddressAndPort getHost()
+ {
+ return host;
+ }
+
+ public InetAddressAndPort getSource()
+ {
+ return source;
+ }
+
+ public String getUser()
+ {
+ return user;
+ }
+
+ public long getTimestamp()
+ {
+ return timestamp;
+ }
+
+ public AuditLogEntryType getType()
+ {
+ return type;
+ }
+
+ public UUID getBatch()
+ {
+ return batch;
+ }
+
+ public String getKeyspace()
+ {
+ return keyspace;
+ }
+
+ public String getScope()
+ {
+ return scope;
+ }
+
+ public String getOperation()
+ {
+ return operation;
+ }
+
+ public QueryOptions getOptions()
+ {
+ return options;
+ }
+
+ public QueryState getState()
+ {
+ return state;
+ }
+
+ public static class Builder
+ {
+ private static final InetAddressAndPort DEFAULT_SOURCE;
+
+ static
+ {
+ try
+ {
+ DEFAULT_SOURCE = InetAddressAndPort.getByNameOverrideDefaults("0.0.0.0", 0);
+ }
+ catch (UnknownHostException e)
+ {
+
+ throw new RuntimeException("failed to create default source address", e);
+ }
+ }
+
+ private static final String DEFAULT_OPERATION = StringUtils.EMPTY;
+
+ private AuditLogEntryType type;
+ private InetAddressAndPort source;
+ private String user;
+ private long timestamp;
+ private UUID batch;
+ private String keyspace;
+ private String scope;
+ private String operation;
+ private QueryOptions options;
+ private QueryState state;
+
+ public Builder(QueryState queryState)
+ {
+ state = queryState;
+
+ ClientState clientState = queryState.getClientState();
+
+ if (clientState != null)
+ {
+ if (clientState.getRemoteAddress() != null)
+ {
+ InetSocketAddress addr = clientState.getRemoteAddress();
+ source = InetAddressAndPort.getByAddressOverrideDefaults(addr.getAddress(), addr.getPort());
+ }
+
+ if (clientState.getUser() != null)
+ {
+ user = clientState.getUser().getName();
+ }
+ keyspace = clientState.getRawKeyspace();
+ }
+ else
+ {
+ source = DEFAULT_SOURCE;
+ user = AuthenticatedUser.SYSTEM_USER.getName();
+ }
+
+ timestamp = System.currentTimeMillis();
+ }
+
+ public Builder(AuditLogEntry entry)
+ {
+ type = entry.type;
+ source = entry.source;
+ user = entry.user;
+ timestamp = entry.timestamp;
+ batch = entry.batch;
+ keyspace = entry.keyspace;
+ scope = entry.scope;
+ operation = entry.operation;
+ options = entry.options;
+ state = entry.state;
+ }
+
+ public Builder setType(AuditLogEntryType type)
+ {
+ this.type = type;
+ return this;
+ }
+
+ public Builder(AuditLogEntryType type)
+ {
+ this.type = type;
+ operation = DEFAULT_OPERATION;
+ }
+
+ public Builder setUser(String user)
+ {
+ this.user = user;
+ return this;
+ }
+
+ public Builder setBatch(UUID batch)
+ {
+ this.batch = batch;
+ return this;
+ }
+
+ public Builder setTimestamp(long timestampMillis)
+ {
+ this.timestamp = timestampMillis;
+ return this;
+ }
+
+ public Builder setKeyspace(QueryState queryState, @Nullable CQLStatement statement)
+ {
+ keyspace = statement != null && statement.getAuditLogContext().keyspace != null
+ ? statement.getAuditLogContext().keyspace
+ : queryState.getClientState().getRawKeyspace();
+ return this;
+ }
+
+ public Builder setKeyspace(String keyspace)
+ {
+ this.keyspace = keyspace;
+ return this;
+ }
+
+ public Builder setKeyspace(CQLStatement statement)
+ {
+ this.keyspace = statement.getAuditLogContext().keyspace;
+ return this;
+ }
+
+ public Builder setScope(CQLStatement statement)
+ {
+ this.scope = statement.getAuditLogContext().scope;
+ return this;
+ }
+
+ public Builder setOperation(String operation)
+ {
+ this.operation = operation;
+ return this;
+ }
+
+ public void appendToOperation(String str)
+ {
+ if (StringUtils.isNotBlank(str))
+ {
+ if (operation.isEmpty())
+ operation = str;
+ else
+ operation = operation.concat("; ").concat(str);
+ }
+ }
+
+ public Builder setOptions(QueryOptions options)
+ {
+ this.options = options;
+ return this;
+ }
+
+ public AuditLogEntry build()
+ {
+ timestamp = timestamp > 0 ? timestamp : System.currentTimeMillis();
+ return new AuditLogEntry(type, source, user, timestamp, batch, keyspace, scope, operation, options, state);
+ }
+ }
+}
+
diff --git a/src/java/org/apache/cassandra/audit/AuditLogEntryCategory.java b/src/java/org/apache/cassandra/audit/AuditLogEntryCategory.java
new file mode 100644
index 0000000..616658c
--- /dev/null
+++ b/src/java/org/apache/cassandra/audit/AuditLogEntryCategory.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.audit;
+
+/**
+ * Enum to categorize AuditLogEntries
+ */
+public enum AuditLogEntryCategory
+{
+ QUERY, DML, DDL, DCL, OTHER, AUTH, ERROR, PREPARE
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/audit/AuditLogEntryType.java b/src/java/org/apache/cassandra/audit/AuditLogEntryType.java
new file mode 100644
index 0000000..ccf0169
--- /dev/null
+++ b/src/java/org/apache/cassandra/audit/AuditLogEntryType.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.audit;
+
+public enum AuditLogEntryType
+{
+ /*
+ * CQL Audit Log Entry Types
+ */
+
+ SELECT(AuditLogEntryCategory.QUERY),
+ UPDATE(AuditLogEntryCategory.DML),
+ DELETE(AuditLogEntryCategory.DML),
+ TRUNCATE(AuditLogEntryCategory.DDL),
+ CREATE_KEYSPACE(AuditLogEntryCategory.DDL),
+ ALTER_KEYSPACE(AuditLogEntryCategory.DDL),
+ DROP_KEYSPACE(AuditLogEntryCategory.DDL),
+ CREATE_TABLE(AuditLogEntryCategory.DDL),
+ DROP_TABLE(AuditLogEntryCategory.DDL),
+ PREPARE_STATEMENT(AuditLogEntryCategory.PREPARE),
+ DROP_TRIGGER(AuditLogEntryCategory.DDL),
+ LIST_USERS(AuditLogEntryCategory.DCL),
+ CREATE_INDEX(AuditLogEntryCategory.DDL),
+ DROP_INDEX(AuditLogEntryCategory.DDL),
+ GRANT(AuditLogEntryCategory.DCL),
+ REVOKE(AuditLogEntryCategory.DCL),
+ CREATE_TYPE(AuditLogEntryCategory.DDL),
+ DROP_AGGREGATE(AuditLogEntryCategory.DDL),
+ ALTER_VIEW(AuditLogEntryCategory.DDL),
+ CREATE_VIEW(AuditLogEntryCategory.DDL),
+ DROP_ROLE(AuditLogEntryCategory.DCL),
+ CREATE_FUNCTION(AuditLogEntryCategory.DDL),
+ ALTER_TABLE(AuditLogEntryCategory.DDL),
+ BATCH(AuditLogEntryCategory.DML),
+ CREATE_AGGREGATE(AuditLogEntryCategory.DDL),
+ DROP_VIEW(AuditLogEntryCategory.DDL),
+ DROP_TYPE(AuditLogEntryCategory.DDL),
+ DROP_FUNCTION(AuditLogEntryCategory.DDL),
+ ALTER_ROLE(AuditLogEntryCategory.DCL),
+ CREATE_TRIGGER(AuditLogEntryCategory.DDL),
+ LIST_ROLES(AuditLogEntryCategory.DCL),
+ LIST_PERMISSIONS(AuditLogEntryCategory.DCL),
+ ALTER_TYPE(AuditLogEntryCategory.DDL),
+ CREATE_ROLE(AuditLogEntryCategory.DCL),
+ USE_KEYSPACE(AuditLogEntryCategory.OTHER),
+ DESCRIBE(AuditLogEntryCategory.OTHER),
+
+ /*
+ * Common Audit Log Entry Types
+ */
+
+ REQUEST_FAILURE(AuditLogEntryCategory.ERROR),
+ LOGIN_ERROR(AuditLogEntryCategory.AUTH),
+ UNAUTHORIZED_ATTEMPT(AuditLogEntryCategory.AUTH),
+ LOGIN_SUCCESS(AuditLogEntryCategory.AUTH);
+
+ private final AuditLogEntryCategory category;
+
+ AuditLogEntryType(AuditLogEntryCategory category)
+ {
+ this.category = category;
+ }
+
+ public AuditLogEntryCategory getCategory()
+ {
+ return category;
+ }
+}
diff --git a/src/java/org/apache/cassandra/audit/AuditLogFilter.java b/src/java/org/apache/cassandra/audit/AuditLogFilter.java
new file mode 100644
index 0000000..163114d
--- /dev/null
+++ b/src/java/org/apache/cassandra/audit/AuditLogFilter.java
@@ -0,0 +1,162 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.audit;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import com.google.common.collect.ImmutableSet;
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class AuditLogFilter
+{
+ private static final Logger logger = LoggerFactory.getLogger(AuditLogFilter.class);
+
+ private static ImmutableSet<String> EMPTY_FILTERS = ImmutableSet.of();
+
+ private final ImmutableSet<String> excludedKeyspaces;
+ private final ImmutableSet<String> includedKeyspaces;
+ private final ImmutableSet<String> excludedCategories;
+ private final ImmutableSet<String> includedCategories;
+ private final ImmutableSet<String> includedUsers;
+ private final ImmutableSet<String> excludedUsers;
+
+ private AuditLogFilter(ImmutableSet<String> excludedKeyspaces, ImmutableSet<String> includedKeyspaces, ImmutableSet<String> excludedCategories, ImmutableSet<String> includedCategories, ImmutableSet<String> excludedUsers, ImmutableSet<String> includedUsers)
+ {
+ this.excludedKeyspaces = excludedKeyspaces;
+ this.includedKeyspaces = includedKeyspaces;
+ this.excludedCategories = excludedCategories;
+ this.includedCategories = includedCategories;
+ this.includedUsers = includedUsers;
+ this.excludedUsers = excludedUsers;
+ }
+
+ /**
+ * (Re-)Loads filters from config. Called during startup as well as JMX invocations.
+ */
+ public static AuditLogFilter create(AuditLogOptions auditLogOptions)
+ {
+ logger.trace("Loading AuditLog filters");
+
+ IncludeExcludeHolder keyspaces = loadInputSets(auditLogOptions.included_keyspaces, auditLogOptions.excluded_keyspaces);
+ IncludeExcludeHolder categories = loadInputSets(auditLogOptions.included_categories, auditLogOptions.excluded_categories);
+ IncludeExcludeHolder users = loadInputSets(auditLogOptions.included_users, auditLogOptions.excluded_users);
+
+ return new AuditLogFilter(keyspaces.excludedSet, keyspaces.includedSet,
+ categories.excludedSet, categories.includedSet,
+ users.excludedSet, users.includedSet);
+ }
+
+ /**
+ * Constructs mutually exclusive sets of included and excluded data. When there is a conflict,
+ * the entry is put into the excluded set (and removed fron the included).
+ */
+ private static IncludeExcludeHolder loadInputSets(String includedInput, String excludedInput)
+ {
+ final ImmutableSet<String> excludedSet;
+ if (StringUtils.isEmpty(excludedInput))
+ {
+ excludedSet = EMPTY_FILTERS;
+ }
+ else
+ {
+ String[] excludes = excludedInput.split(",");
+ ImmutableSet.Builder<String> builder = ImmutableSet.builderWithExpectedSize(excludes.length);
+ for (String exclude : excludes)
+ {
+ if (!exclude.isEmpty())
+ {
+ builder.add(exclude);
+ }
+ }
+ excludedSet = builder.build();
+ }
+
+ final ImmutableSet<String> includedSet;
+ if (StringUtils.isEmpty(includedInput))
+ {
+ includedSet = EMPTY_FILTERS;
+ }
+ else
+ {
+ String[] includes = includedInput.split(",");
+ ImmutableSet.Builder<String> builder = ImmutableSet.builderWithExpectedSize(includes.length);
+ for (String include : includes)
+ {
+ //Ensure both included and excluded sets are mutually exclusive
+ if (!include.isEmpty() && !excludedSet.contains(include))
+ {
+ builder.add(include);
+ }
+ }
+ includedSet = builder.build();
+ }
+
+ return new IncludeExcludeHolder(includedSet, excludedSet);
+ }
+
+ /**
+ * Simple struct to hold inclusion/exclusion sets.
+ */
+ private static class IncludeExcludeHolder
+ {
+ private final ImmutableSet<String> includedSet;
+ private final ImmutableSet<String> excludedSet;
+
+ private IncludeExcludeHolder(ImmutableSet<String> includedSet, ImmutableSet<String> excludedSet)
+ {
+ this.includedSet = includedSet;
+ this.excludedSet = excludedSet;
+ }
+ }
+
+ /**
+ * Checks whether a give AuditLog Entry is filtered or not
+ *
+ * @param auditLogEntry AuditLogEntry to verify
+ * @return true if it is filtered, false otherwise
+ */
+ boolean isFiltered(AuditLogEntry auditLogEntry)
+ {
+ return isFiltered(auditLogEntry.getKeyspace(), includedKeyspaces, excludedKeyspaces)
+ || isFiltered(auditLogEntry.getType().getCategory().toString(), includedCategories, excludedCategories)
+ || isFiltered(auditLogEntry.getUser(), includedUsers, excludedUsers);
+ }
+
+ /**
+ * Checks whether given input is being filtered or not.
+ * If excludeSet does not contain any items, by default nothing is excluded (unless there are
+ * entries in the includeSet).
+ * If includeSet does not contain any items, by default everything is included
+ * If an input is part of both includeSet and excludeSet, excludeSet takes the priority over includeSet
+ *
+ * @param input Input to be checked for filtereing based on includeSet and excludeSet
+ * @param includeSet Include filtering set
+ * @param excludeSet Exclude filtering set
+ * @return true if the input is filtered, false when the input is not filtered
+ */
+ static boolean isFiltered(String input, Set<String> includeSet, Set<String> excludeSet)
+ {
+ if (!excludeSet.isEmpty() && excludeSet.contains(input))
+ return true;
+
+ return !(includeSet.isEmpty() || includeSet.contains(input));
+ }
+}
diff --git a/src/java/org/apache/cassandra/audit/AuditLogManager.java b/src/java/org/apache/cassandra/audit/AuditLogManager.java
new file mode 100644
index 0000000..bd6d10d
--- /dev/null
+++ b/src/java/org/apache/cassandra/audit/AuditLogManager.java
@@ -0,0 +1,331 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.audit;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+
+import javax.annotation.Nullable;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.auth.AuthEvents;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.config.ParameterizedClass;
+import org.apache.cassandra.cql3.CQLStatement;
+import org.apache.cassandra.cql3.QueryEvents;
+import org.apache.cassandra.cql3.QueryOptions;
+import org.apache.cassandra.cql3.statements.BatchStatement;
+import org.apache.cassandra.exceptions.AuthenticationException;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.exceptions.PreparedQueryNotFoundException;
+import org.apache.cassandra.exceptions.UnauthorizedException;
+import org.apache.cassandra.service.QueryState;
+import org.apache.cassandra.transport.Message;
+import org.apache.cassandra.transport.messages.ResultMessage;
+import org.apache.cassandra.utils.FBUtilities;
+
+/**
+ * Central location for managing the logging of client/user-initated actions (like queries, log in commands, and so on).
+ *
+ */
+public class AuditLogManager implements QueryEvents.Listener, AuthEvents.Listener
+{
+ private static final Logger logger = LoggerFactory.getLogger(AuditLogManager.class);
+ public static final AuditLogManager instance = new AuditLogManager();
+
+ // auditLogger can write anywhere, as it's pluggable (logback, BinLog, DiagnosticEvents, etc ...)
+ private volatile IAuditLogger auditLogger;
+
+ private volatile AuditLogFilter filter;
+
+ private AuditLogManager()
+ {
+ final AuditLogOptions auditLogOptions = DatabaseDescriptor.getAuditLoggingOptions();
+
+ if (auditLogOptions.enabled)
+ {
+ logger.info("Audit logging is enabled.");
+ auditLogger = getAuditLogger(auditLogOptions.logger);
+ }
+ else
+ {
+ logger.debug("Audit logging is disabled.");
+ auditLogger = new NoOpAuditLogger(Collections.emptyMap());
+ }
+
+ filter = AuditLogFilter.create(auditLogOptions);
+ }
+
+ public void initialize()
+ {
+ if (DatabaseDescriptor.getAuditLoggingOptions().enabled)
+ registerAsListener();
+ }
+
+ private IAuditLogger getAuditLogger(ParameterizedClass logger) throws ConfigurationException
+ {
+ if (logger.class_name != null)
+ {
+ return FBUtilities.newAuditLogger(logger.class_name, logger.parameters == null ? Collections.emptyMap() : logger.parameters);
+ }
+
+ return FBUtilities.newAuditLogger(BinAuditLogger.class.getName(), Collections.emptyMap());
+ }
+
+ @VisibleForTesting
+ public IAuditLogger getLogger()
+ {
+ return auditLogger;
+ }
+
+ public boolean isEnabled()
+ {
+ return auditLogger.isEnabled();
+ }
+
+ /**
+ * Logs AudigLogEntry to standard audit logger
+ * @param logEntry AuditLogEntry to be logged
+ */
+ private void log(AuditLogEntry logEntry)
+ {
+ if (!filter.isFiltered(logEntry))
+ {
+ auditLogger.log(logEntry);
+ }
+ }
+
+ private void log(AuditLogEntry logEntry, Exception e)
+ {
+ AuditLogEntry.Builder builder = new AuditLogEntry.Builder(logEntry);
+
+ if (e instanceof UnauthorizedException)
+ {
+ builder.setType(AuditLogEntryType.UNAUTHORIZED_ATTEMPT);
+ }
+ else if (e instanceof AuthenticationException)
+ {
+ builder.setType(AuditLogEntryType.LOGIN_ERROR);
+ }
+ else
+ {
+ builder.setType(AuditLogEntryType.REQUEST_FAILURE);
+ }
+
+ builder.appendToOperation(e.getMessage());
+
+ log(builder.build());
+ }
+
+ /**
+ * Disables AuditLog, designed to be invoked only via JMX/ Nodetool, not from anywhere else in the codepath.
+ */
+ public synchronized void disableAuditLog()
+ {
+ unregisterAsListener();
+ IAuditLogger oldLogger = auditLogger;
+ auditLogger = new NoOpAuditLogger(Collections.emptyMap());
+ oldLogger.stop();
+ }
+
+ /**
+ * Enables AuditLog, designed to be invoked only via JMX/ Nodetool, not from anywhere else in the codepath.
+ * @param auditLogOptions AuditLogOptions to be used for enabling AuditLog
+ * @throws ConfigurationException It can throw configuration exception when provided logger class does not exist in the classpath
+ */
+ public synchronized void enable(AuditLogOptions auditLogOptions) throws ConfigurationException
+ {
+ // always reload the filters
+ filter = AuditLogFilter.create(auditLogOptions);
+
+ // next, check to see if we're changing the logging implementation; if not, keep the same instance and bail.
+ // note: auditLogger should never be null
+ IAuditLogger oldLogger = auditLogger;
+ if (oldLogger.getClass().getSimpleName().equals(auditLogOptions.logger.class_name))
+ return;
+
+ auditLogger = getAuditLogger(auditLogOptions.logger);
+
+ // note that we might already be registered here and we rely on the fact that Query/AuthEvents have a Set of listeners
+ registerAsListener();
+
+ // ensure oldLogger's stop() is called after we swap it with new logger,
+ // otherwise, we might be calling log() on the stopped logger.
+ oldLogger.stop();
+ }
+
+ private void registerAsListener()
+ {
+ QueryEvents.instance.registerListener(this);
+ AuthEvents.instance.registerListener(this);
+ }
+
+ private void unregisterAsListener()
+ {
+ QueryEvents.instance.unregisterListener(this);
+ AuthEvents.instance.unregisterListener(this);
+ }
+
+ public void querySuccess(CQLStatement statement, String query, QueryOptions options, QueryState state, long queryTime, Message.Response response)
+ {
+ AuditLogEntry entry = new AuditLogEntry.Builder(state).setType(statement.getAuditLogContext().auditLogEntryType)
+ .setOperation(query)
+ .setTimestamp(queryTime)
+ .setScope(statement)
+ .setKeyspace(state, statement)
+ .setOptions(options)
+ .build();
+ log(entry);
+ }
+
+ public void queryFailure(CQLStatement stmt, String query, QueryOptions options, QueryState state, Exception cause)
+ {
+ AuditLogEntry entry = new AuditLogEntry.Builder(state).setOperation(query)
+ .setOptions(options)
+ .build();
+ log(entry, cause);
+ }
+
+ public void executeSuccess(CQLStatement statement, String query, QueryOptions options, QueryState state, long queryTime, Message.Response response)
+ {
+ AuditLogEntry entry = new AuditLogEntry.Builder(state).setType(statement.getAuditLogContext().auditLogEntryType)
+ .setOperation(query)
+ .setTimestamp(queryTime)
+ .setScope(statement)
+ .setKeyspace(state, statement)
+ .setOptions(options)
+ .build();
+ log(entry);
+ }
+
+ public void executeFailure(CQLStatement statement, String query, QueryOptions options, QueryState state, Exception cause)
+ {
+ AuditLogEntry entry = null;
+ if (cause instanceof PreparedQueryNotFoundException)
+ {
+ entry = new AuditLogEntry.Builder(state).setOperation(query == null ? "null" : query)
+ .setOptions(options)
+ .build();
+ }
+ else if (statement != null)
+ {
+ entry = new AuditLogEntry.Builder(state).setOperation(query == null ? statement.toString() : query)
+ .setType(statement.getAuditLogContext().auditLogEntryType)
+ .setScope(statement)
+ .setKeyspace(state, statement)
+ .setOptions(options)
+ .build();
+ }
+ if (entry != null)
+ log(entry, cause);
+ }
+
+ public void batchSuccess(BatchStatement.Type batchType, List<? extends CQLStatement> statements, List<String> queries, List<List<ByteBuffer>> values, QueryOptions options, QueryState state, long queryTime, Message.Response response)
+ {
+ List<AuditLogEntry> entries = buildEntriesForBatch(statements, queries, state, options, queryTime);
+ for (AuditLogEntry auditLogEntry : entries)
+ {
+ log(auditLogEntry);
+ }
+ }
+
+ public void batchFailure(BatchStatement.Type batchType, List<? extends CQLStatement> statements, List<String> queries, List<List<ByteBuffer>> values, QueryOptions options, QueryState state, Exception cause)
+ {
+ String auditMessage = String.format("BATCH of %d statements at consistency %s", statements.size(), options.getConsistency());
+ AuditLogEntry entry = new AuditLogEntry.Builder(state).setOperation(auditMessage)
+ .setOptions(options)
+ .setType(AuditLogEntryType.BATCH)
+ .build();
+ log(entry, cause);
+ }
+
+ private static List<AuditLogEntry> buildEntriesForBatch(List<? extends CQLStatement> statements, List<String> queries, QueryState state, QueryOptions options, long queryStartTimeMillis)
+ {
+ List<AuditLogEntry> auditLogEntries = new ArrayList<>(statements.size() + 1);
+ UUID batchId = UUID.randomUUID();
+ String queryString = String.format("BatchId:[%s] - BATCH of [%d] statements", batchId, statements.size());
+ AuditLogEntry entry = new AuditLogEntry.Builder(state)
+ .setOperation(queryString)
+ .setOptions(options)
+ .setTimestamp(queryStartTimeMillis)
+ .setBatch(batchId)
+ .setType(AuditLogEntryType.BATCH)
+ .build();
+ auditLogEntries.add(entry);
+
+ for (int i = 0; i < statements.size(); i++)
+ {
+ CQLStatement statement = statements.get(i);
+ entry = new AuditLogEntry.Builder(state)
+ .setType(statement.getAuditLogContext().auditLogEntryType)
+ .setOperation(queries.get(i))
+ .setTimestamp(queryStartTimeMillis)
+ .setScope(statement)
+ .setKeyspace(state, statement)
+ .setOptions(options)
+ .setBatch(batchId)
+ .build();
+ auditLogEntries.add(entry);
+ }
+
+ return auditLogEntries;
+ }
+
+ public void prepareSuccess(CQLStatement statement, String query, QueryState state, long queryTime, ResultMessage.Prepared response)
+ {
+ AuditLogEntry entry = new AuditLogEntry.Builder(state).setOperation(query)
+ .setType(AuditLogEntryType.PREPARE_STATEMENT)
+ .setScope(statement)
+ .setKeyspace(statement)
+ .build();
+ log(entry);
+ }
+
+ public void prepareFailure(@Nullable CQLStatement stmt, @Nullable String query, QueryState state, Exception cause)
+ {
+ AuditLogEntry entry = new AuditLogEntry.Builder(state).setOperation(query)
+// .setKeyspace(keyspace) // todo: do we need this? very much special case compared to the others
+ .setType(AuditLogEntryType.PREPARE_STATEMENT)
+ .build();
+ log(entry, cause);
+ }
+
+ public void authSuccess(QueryState state)
+ {
+ AuditLogEntry entry = new AuditLogEntry.Builder(state).setOperation("LOGIN SUCCESSFUL")
+ .setType(AuditLogEntryType.LOGIN_SUCCESS)
+ .build();
+ log(entry);
+ }
+
+ public void authFailure(QueryState state, Exception cause)
+ {
+ AuditLogEntry entry = new AuditLogEntry.Builder(state).setOperation("LOGIN FAILURE")
+ .setType(AuditLogEntryType.LOGIN_ERROR)
+ .build();
+ log(entry, cause);
+ }
+}
diff --git a/src/java/org/apache/cassandra/audit/AuditLogOptions.java b/src/java/org/apache/cassandra/audit/AuditLogOptions.java
new file mode 100644
index 0000000..e8691df
--- /dev/null
+++ b/src/java/org/apache/cassandra/audit/AuditLogOptions.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.audit;
+
+import java.util.Collections;
+import java.util.Map;
+
+import org.apache.commons.lang3.StringUtils;
+
+import org.apache.cassandra.config.ParameterizedClass;
+import org.apache.cassandra.utils.binlog.BinLogOptions;
+
+public class AuditLogOptions extends BinLogOptions
+{
+ public volatile boolean enabled = false;
+ public ParameterizedClass logger = new ParameterizedClass(BinAuditLogger.class.getSimpleName(), Collections.emptyMap());
+ public String included_keyspaces = StringUtils.EMPTY;
+ // CASSANDRA-14498: By default, system, system_schema and system_virtual_schema are excluded, but these can be included via cassandra.yaml
+ public String excluded_keyspaces = "system,system_schema,system_virtual_schema";
+ public String included_categories = StringUtils.EMPTY;
+ public String excluded_categories = StringUtils.EMPTY;
+ public String included_users = StringUtils.EMPTY;
+ public String excluded_users = StringUtils.EMPTY;
+
+ /**
+ * AuditLogs directory can be configured using `cassandra.logdir.audit` or default is set to `cassandra.logdir` + /audit/
+ */
+ public String audit_logs_dir = System.getProperty("cassandra.logdir.audit",
+ System.getProperty("cassandra.logdir",".")+"/audit/");
+
+ public String toString()
+ {
+ return "AuditLogOptions{" +
+ "enabled=" + enabled +
+ ", logger='" + logger + '\'' +
+ ", included_keyspaces='" + included_keyspaces + '\'' +
+ ", excluded_keyspaces='" + excluded_keyspaces + '\'' +
+ ", included_categories='" + included_categories + '\'' +
+ ", excluded_categories='" + excluded_categories + '\'' +
+ ", included_users='" + included_users + '\'' +
+ ", excluded_users='" + excluded_users + '\'' +
+ ", audit_logs_dir='" + audit_logs_dir + '\'' +
+ ", archive_command='" + archive_command + '\'' +
+ ", roll_cycle='" + roll_cycle + '\'' +
+ ", block=" + block +
+ ", max_queue_weight=" + max_queue_weight +
+ ", max_log_size=" + max_log_size +
+ '}';
+ }
+}
diff --git a/src/java/org/apache/cassandra/audit/BinAuditLogger.java b/src/java/org/apache/cassandra/audit/BinAuditLogger.java
new file mode 100644
index 0000000..95a53f1
--- /dev/null
+++ b/src/java/org/apache/cassandra/audit/BinAuditLogger.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.audit;
+
+import java.nio.file.Paths;
+import java.util.Map;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.primitives.Ints;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import net.openhft.chronicle.wire.WireOut;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.utils.ObjectSizes;
+import org.apache.cassandra.utils.binlog.BinLog;
+import org.apache.cassandra.utils.concurrent.WeightedQueue;
+
+public class BinAuditLogger implements IAuditLogger
+{
+ public static final long CURRENT_VERSION = 0;
+ public static final String AUDITLOG_TYPE = "audit";
+ public static final String AUDITLOG_MESSAGE = "message";
+ private static final Logger logger = LoggerFactory.getLogger(BinAuditLogger.class);
+
+ private volatile BinLog binLog;
+
+ public BinAuditLogger(Map<String, String> params)
+ {
+ AuditLogOptions auditLoggingOptions = DatabaseDescriptor.getAuditLoggingOptions();
+
+ this.binLog = new BinLog.Builder().path(Paths.get(auditLoggingOptions.audit_logs_dir))
+ .rollCycle(auditLoggingOptions.roll_cycle)
+ .blocking(auditLoggingOptions.block)
+ .maxQueueWeight(auditLoggingOptions.max_queue_weight)
+ .maxLogSize(auditLoggingOptions.max_log_size)
+ .archiveCommand(auditLoggingOptions.archive_command)
+ .maxArchiveRetries(auditLoggingOptions.max_archive_retries)
+ .build(false);
+ }
+
+ /**
+ * Stop the audit log leaving behind any generated files.
+ */
+ public synchronized void stop()
+ {
+ try
+ {
+ logger.info("Deactivation of audit log requested.");
+ if (binLog != null)
+ {
+ logger.info("Stopping audit logger");
+ binLog.stop();
+ binLog = null;
+ }
+ }
+ catch (Exception e)
+ {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public boolean isEnabled()
+ {
+ return binLog != null;
+ }
+
+ @Override
+ public void log(AuditLogEntry auditLogEntry)
+ {
+ BinLog binLog = this.binLog;
+ if (binLog == null || auditLogEntry == null)
+ {
+ return;
+ }
+ binLog.logRecord(new Message(auditLogEntry.getLogString()));
+ }
+
+
+ @VisibleForTesting
+ public static class Message extends BinLog.ReleaseableWriteMarshallable implements WeightedQueue.Weighable
+ {
+ private final String message;
+
+ public Message(String message)
+ {
+ this.message = message;
+ }
+
+ protected long version()
+ {
+ return CURRENT_VERSION;
+ }
+
+ protected String type()
+ {
+ return AUDITLOG_TYPE;
+ }
+
+ @Override
+ public void writeMarshallablePayload(WireOut wire)
+ {
+ wire.write(AUDITLOG_MESSAGE).text(message);
+ }
+
+ @Override
+ public void release()
+ {
+
+ }
+
+ @Override
+ public int weight()
+ {
+ return Ints.checkedCast(ObjectSizes.sizeOf(message));
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/audit/DiagnosticEventAuditLogger.java b/src/java/org/apache/cassandra/audit/DiagnosticEventAuditLogger.java
new file mode 100644
index 0000000..9a154ae
--- /dev/null
+++ b/src/java/org/apache/cassandra/audit/DiagnosticEventAuditLogger.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.audit;
+
+import java.util.Map;
+
+import org.apache.cassandra.diag.DiagnosticEventService;
+
+public class DiagnosticEventAuditLogger implements IAuditLogger
+{
+ public DiagnosticEventAuditLogger(Map<String, String> params)
+ {
+
+ }
+
+ public void log(AuditLogEntry logMessage)
+ {
+ AuditEvent.create(logMessage);
+ }
+
+ public boolean isEnabled()
+ {
+ return DiagnosticEventService.instance().isDiagnosticsEnabled();
+ }
+
+ public void stop()
+ {
+
+ }
+}
diff --git a/src/java/org/apache/cassandra/audit/FileAuditLogger.java b/src/java/org/apache/cassandra/audit/FileAuditLogger.java
new file mode 100644
index 0000000..a5fffcb
--- /dev/null
+++ b/src/java/org/apache/cassandra/audit/FileAuditLogger.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.audit;
+
+import java.util.Map;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Synchronous, file-based audit logger; just uses the standard logging mechansim.
+ */
+public class FileAuditLogger implements IAuditLogger
+{
+ protected static final Logger logger = LoggerFactory.getLogger(FileAuditLogger.class);
+
+ private volatile boolean enabled;
+
+ public FileAuditLogger(Map<String, String> params)
+ {
+ enabled = true;
+ }
+
+ @Override
+ public boolean isEnabled()
+ {
+ return enabled;
+ }
+
+ @Override
+ public void log(AuditLogEntry auditLogEntry)
+ {
+ // don't bother with the volatile read of enabled here. just go ahead and log, other components
+ // will check the enbaled field.
+ logger.info(auditLogEntry.getLogString());
+ }
+
+ @Override
+ public void stop()
+ {
+ enabled = false;
+ }
+}
diff --git a/src/java/org/apache/cassandra/audit/IAuditLogger.java b/src/java/org/apache/cassandra/audit/IAuditLogger.java
new file mode 100644
index 0000000..9ab4c08
--- /dev/null
+++ b/src/java/org/apache/cassandra/audit/IAuditLogger.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.audit;
+
+public interface IAuditLogger
+{
+ boolean isEnabled();
+
+ /**
+ * Logs AuditLogEntry. This method might be called after {@link #stop()},
+ * hence implementations need to handle the race condition.
+ */
+ void log(AuditLogEntry auditLogEntry);
+
+ /**
+ * Stop and cleanup any resources of IAuditLogger implementations. Please note that
+ * {@link #log(AuditLogEntry)} might be called after being stopped.
+ */
+ void stop();
+}
diff --git a/src/java/org/apache/cassandra/audit/NoOpAuditLogger.java b/src/java/org/apache/cassandra/audit/NoOpAuditLogger.java
new file mode 100644
index 0000000..8f159d0
--- /dev/null
+++ b/src/java/org/apache/cassandra/audit/NoOpAuditLogger.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.audit;
+
+import java.util.Map;
+
+/**
+ * No-Op implementation of {@link IAuditLogger} to be used as a default audit logger when audit logging is disabled.
+ */
+public class NoOpAuditLogger implements IAuditLogger
+{
+ public NoOpAuditLogger(Map<String, String> params)
+ {
+
+ }
+
+ @Override
+ public boolean isEnabled()
+ {
+ return false;
+ }
+
+ @Override
+ public void log(AuditLogEntry logMessage)
+ {
+
+ }
+
+ @Override
+ public void stop()
+ {
+
+ }
+}
diff --git a/src/java/org/apache/cassandra/auth/AllowAllNetworkAuthorizer.java b/src/java/org/apache/cassandra/auth/AllowAllNetworkAuthorizer.java
new file mode 100644
index 0000000..17c04d5
--- /dev/null
+++ b/src/java/org/apache/cassandra/auth/AllowAllNetworkAuthorizer.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.auth;
+
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+
+public class AllowAllNetworkAuthorizer implements INetworkAuthorizer
+{
+ public void setup() {}
+
+ public DCPermissions authorize(RoleResource role)
+ {
+ return DCPermissions.all();
+ }
+
+ public void setRoleDatacenters(RoleResource role, DCPermissions permissions)
+ {
+ throw new InvalidRequestException("ACCESS TO DATACENTERS operations not supported by AllowAllNetworkAuthorizer");
+ }
+
+ public void drop(RoleResource role) {}
+
+ public void validateConfiguration() throws ConfigurationException {}
+
+ @Override
+ public boolean requireAuthorization()
+ {
+ return false;
+ }
+}
diff --git a/src/java/org/apache/cassandra/auth/AuthCache.java b/src/java/org/apache/cassandra/auth/AuthCache.java
index 80664d1..4bf15c1 100644
--- a/src/java/org/apache/cassandra/auth/AuthCache.java
+++ b/src/java/org/apache/cassandra/auth/AuthCache.java
@@ -18,26 +18,21 @@
package org.apache.cassandra.auth;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
-import java.util.function.Consumer;
+import java.util.function.BooleanSupplier;
import java.util.function.Function;
-import java.util.function.Supplier;
+import java.util.function.IntConsumer;
+import java.util.function.IntSupplier;
-import com.google.common.base.Throwables;
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.CacheLoader;
-import com.google.common.cache.LoadingCache;
-import com.google.common.util.concurrent.ListenableFuture;
-import com.google.common.util.concurrent.ListenableFutureTask;
-import com.google.common.util.concurrent.UncheckedExecutionException;
+import com.google.common.util.concurrent.MoreExecutors;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.github.benmanes.caffeine.cache.Caffeine;
+import com.github.benmanes.caffeine.cache.LoadingCache;
import org.apache.cassandra.utils.MBeanWrapper;
-import org.apache.cassandra.concurrent.DebuggableThreadPoolExecutor;
+import static com.google.common.base.Preconditions.checkNotNull;
public class AuthCache<K, V> implements AuthCacheMBean
{
@@ -45,85 +40,111 @@
private static final String MBEAN_NAME_BASE = "org.apache.cassandra.auth:type=";
- private volatile LoadingCache<K, V> cache;
- private ThreadPoolExecutor cacheRefreshExecutor;
+ /**
+ * Underlying cache. LoadingCache will call underlying load function on {@link #get} if key is not present
+ */
+ protected volatile LoadingCache<K, V> cache;
- private final String name;
- private final Consumer<Integer> setValidityDelegate;
- private final Supplier<Integer> getValidityDelegate;
- private final Consumer<Integer> setUpdateIntervalDelegate;
- private final Supplier<Integer> getUpdateIntervalDelegate;
- private final Consumer<Integer> setMaxEntriesDelegate;
- private final Supplier<Integer> getMaxEntriesDelegate;
- private final Function<K, V> loadFunction;
- private final Supplier<Boolean> enableCache;
+ private String name;
+ private IntConsumer setValidityDelegate;
+ private IntSupplier getValidityDelegate;
+ private IntConsumer setUpdateIntervalDelegate;
+ private IntSupplier getUpdateIntervalDelegate;
+ private IntConsumer setMaxEntriesDelegate;
+ private IntSupplier getMaxEntriesDelegate;
+ private Function<K, V> loadFunction;
+ private BooleanSupplier enableCache;
+ /**
+ * @param name Used for MBean
+ * @param setValidityDelegate Used to set cache validity period. See {@link Policy#expireAfterWrite()}
+ * @param getValidityDelegate Getter for validity period
+ * @param setUpdateIntervalDelegate Used to set cache update interval. See {@link Policy#refreshAfterWrite()}
+ * @param getUpdateIntervalDelegate Getter for update interval
+ * @param setMaxEntriesDelegate Used to set max # entries in cache. See {@link com.github.benmanes.caffeine.cache.Policy.Eviction#setMaximum(long)}
+ * @param getMaxEntriesDelegate Getter for max entries.
+ * @param loadFunction Function to load the cache. Called on {@link #get(Object)}
+ * @param cacheEnabledDelegate Used to determine if cache is enabled.
+ */
protected AuthCache(String name,
- Consumer<Integer> setValidityDelegate,
- Supplier<Integer> getValidityDelegate,
- Consumer<Integer> setUpdateIntervalDelegate,
- Supplier<Integer> getUpdateIntervalDelegate,
- Consumer<Integer> setMaxEntriesDelegate,
- Supplier<Integer> getMaxEntriesDelegate,
+ IntConsumer setValidityDelegate,
+ IntSupplier getValidityDelegate,
+ IntConsumer setUpdateIntervalDelegate,
+ IntSupplier getUpdateIntervalDelegate,
+ IntConsumer setMaxEntriesDelegate,
+ IntSupplier getMaxEntriesDelegate,
Function<K, V> loadFunction,
- Supplier<Boolean> enableCache)
+ BooleanSupplier cacheEnabledDelegate)
{
- this.name = name;
- this.setValidityDelegate = setValidityDelegate;
- this.getValidityDelegate = getValidityDelegate;
- this.setUpdateIntervalDelegate = setUpdateIntervalDelegate;
- this.getUpdateIntervalDelegate = getUpdateIntervalDelegate;
- this.setMaxEntriesDelegate = setMaxEntriesDelegate;
- this.getMaxEntriesDelegate = getMaxEntriesDelegate;
- this.loadFunction = loadFunction;
- this.enableCache = enableCache;
+ this.name = checkNotNull(name);
+ this.setValidityDelegate = checkNotNull(setValidityDelegate);
+ this.getValidityDelegate = checkNotNull(getValidityDelegate);
+ this.setUpdateIntervalDelegate = checkNotNull(setUpdateIntervalDelegate);
+ this.getUpdateIntervalDelegate = checkNotNull(getUpdateIntervalDelegate);
+ this.setMaxEntriesDelegate = checkNotNull(setMaxEntriesDelegate);
+ this.getMaxEntriesDelegate = checkNotNull(getMaxEntriesDelegate);
+ this.loadFunction = checkNotNull(loadFunction);
+ this.enableCache = checkNotNull(cacheEnabledDelegate);
init();
}
+ /**
+ * Do setup for the cache and MBean.
+ */
protected void init()
{
- this.cacheRefreshExecutor = new DebuggableThreadPoolExecutor(name + "Refresh", Thread.NORM_PRIORITY)
- {
- protected void afterExecute(Runnable r, Throwable t)
- {
- // empty to avoid logging on background updates
- }
- };
- this.cache = initCache(null);
+ cache = initCache(null);
MBeanWrapper.instance.registerMBean(this, getObjectName());
}
+ protected void unregisterMBean()
+ {
+ MBeanWrapper.instance.unregisterMBean(getObjectName(), MBeanWrapper.OnException.LOG);
+ }
+
protected String getObjectName()
{
return MBEAN_NAME_BASE + name;
}
+ /**
+ * Retrieve a value from the cache. Will call {@link LoadingCache#get(Object)} which will
+ * "load" the value if it's not present, thus populating the key.
+ * @param k
+ * @return The current value of {@code K} if cached or loaded.
+ *
+ * See {@link LoadingCache#get(Object)} for possible exceptions.
+ */
public V get(K k)
{
if (cache == null)
return loadFunction.apply(k);
- try {
- return cache.get(k);
- }
- catch (ExecutionException | UncheckedExecutionException e)
- {
- Throwables.propagateIfInstanceOf(e.getCause(), RuntimeException.class);
- throw Throwables.propagate(e);
- }
+ return cache.get(k);
}
+ /**
+ * Invalidate the entire cache.
+ */
public void invalidate()
{
cache = initCache(null);
}
+ /**
+ * Invalidate a key
+ * @param k key to invalidate
+ */
public void invalidate(K k)
{
if (cache != null)
cache.invalidate(k);
}
+ /**
+ * Time in milliseconds that a value in the cache will expire after.
+ * @param validityPeriod in milliseconds
+ */
public void setValidity(int validityPeriod)
{
if (Boolean.getBoolean("cassandra.disable_auth_caches_remote_configuration"))
@@ -135,9 +156,13 @@
public int getValidity()
{
- return getValidityDelegate.get();
+ return getValidityDelegate.getAsInt();
}
+ /**
+ * Time in milliseconds after which an entry in the cache should be refreshed (it's load function called again)
+ * @param updateInterval in milliseconds
+ */
public void setUpdateInterval(int updateInterval)
{
if (Boolean.getBoolean("cassandra.disable_auth_caches_remote_configuration"))
@@ -149,9 +174,13 @@
public int getUpdateInterval()
{
- return getUpdateIntervalDelegate.get();
+ return getUpdateIntervalDelegate.getAsInt();
}
+ /**
+ * Set maximum number of entries in the cache.
+ * @param maxEntries
+ */
public void setMaxEntries(int maxEntries)
{
if (Boolean.getBoolean("cassandra.disable_auth_caches_remote_configuration"))
@@ -163,12 +192,19 @@
public int getMaxEntries()
{
- return getMaxEntriesDelegate.get();
+ return getMaxEntriesDelegate.getAsInt();
}
- private LoadingCache<K, V> initCache(LoadingCache<K, V> existing)
+ /**
+ * (Re-)initialise the underlying cache. Will update validity, max entries, and update interval if
+ * any have changed. The underlying {@link LoadingCache} will be initiated based on the provided {@code loadFunction}.
+ * Note: If you need some unhandled cache setting to be set you should extend {@link AuthCache} and override this method.
+ * @param existing If not null will only update cache update validity, max entries, and update interval.
+ * @return New {@link LoadingCache} if existing was null, otherwise the existing {@code cache}
+ */
+ protected LoadingCache<K, V> initCache(LoadingCache<K, V> existing)
{
- if (!enableCache.get())
+ if (!enableCache.getAsBoolean())
return null;
if (getValidity() <= 0)
@@ -177,36 +213,22 @@
logger.info("(Re)initializing {} (validity period/update interval/max entries) ({}/{}/{})",
name, getValidity(), getUpdateInterval(), getMaxEntries());
- LoadingCache<K, V> newcache = CacheBuilder.newBuilder()
- .refreshAfterWrite(getUpdateInterval(), TimeUnit.MILLISECONDS)
- .expireAfterWrite(getValidity(), TimeUnit.MILLISECONDS)
- .maximumSize(getMaxEntries())
- .build(new CacheLoader<K, V>()
- {
- public V load(K k)
- {
- return loadFunction.apply(k);
- }
+ if (existing == null) {
+ return Caffeine.newBuilder()
+ .refreshAfterWrite(getUpdateInterval(), TimeUnit.MILLISECONDS)
+ .expireAfterWrite(getValidity(), TimeUnit.MILLISECONDS)
+ .maximumSize(getMaxEntries())
+ .executor(MoreExecutors.directExecutor())
+ .build(loadFunction::apply);
+ }
- public ListenableFuture<V> reload(final K k, final V oldV)
- {
- ListenableFutureTask<V> task = ListenableFutureTask.create(() -> {
- try
- {
- return loadFunction.apply(k);
- }
- catch (Exception e)
- {
- logger.trace("Error performing async refresh of auth data in {}", name, e);
- throw e;
- }
- });
- cacheRefreshExecutor.execute(task);
- return task;
- }
- });
- if (existing != null)
- newcache.putAll(existing.asMap());
- return newcache;
+ // Always set as mandatory
+ cache.policy().refreshAfterWrite().ifPresent(policy ->
+ policy.setExpiresAfter(getUpdateInterval(), TimeUnit.MILLISECONDS));
+ cache.policy().expireAfterWrite().ifPresent(policy ->
+ policy.setExpiresAfter(getValidity(), TimeUnit.MILLISECONDS));
+ cache.policy().eviction().ifPresent(policy ->
+ policy.setMaximum(getMaxEntries()));
+ return cache;
}
}
diff --git a/src/java/org/apache/cassandra/auth/AuthConfig.java b/src/java/org/apache/cassandra/auth/AuthConfig.java
index c389ae4..cc38296 100644
--- a/src/java/org/apache/cassandra/auth/AuthConfig.java
+++ b/src/java/org/apache/cassandra/auth/AuthConfig.java
@@ -94,13 +94,16 @@
// authenticator
- IInternodeAuthenticator internodeAuthenticator;
if (conf.internode_authenticator != null)
- internodeAuthenticator = FBUtilities.construct(conf.internode_authenticator, "internode_authenticator");
- else
- internodeAuthenticator = new AllowAllInternodeAuthenticator();
+ DatabaseDescriptor.setInternodeAuthenticator(FBUtilities.construct(conf.internode_authenticator, "internode_authenticator"));
- DatabaseDescriptor.setInternodeAuthenticator(internodeAuthenticator);
+ // network authorizer
+ INetworkAuthorizer networkAuthorizer = FBUtilities.newNetworkAuthorizer(conf.network_authorizer);
+ DatabaseDescriptor.setNetworkAuthorizer(networkAuthorizer);
+ if (networkAuthorizer.requireAuthorization() && !authenticator.requireAuthentication())
+ {
+ throw new ConfigurationException(conf.network_authorizer + " can't be used with " + conf.authenticator, false);
+ }
// Validate at last to have authenticator, authorizer, role-manager and internode-auth setup
// in case these rely on each other.
@@ -108,6 +111,7 @@
authenticator.validateConfiguration();
authorizer.validateConfiguration();
roleManager.validateConfiguration();
- internodeAuthenticator.validateConfiguration();
+ networkAuthorizer.validateConfiguration();
+ DatabaseDescriptor.getInternodeAuthenticator().validateConfiguration();
}
}
diff --git a/src/java/org/apache/cassandra/auth/AuthEvents.java b/src/java/org/apache/cassandra/auth/AuthEvents.java
new file mode 100644
index 0000000..e4b6048
--- /dev/null
+++ b/src/java/org/apache/cassandra/auth/AuthEvents.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.auth;
+
+import java.util.Set;
+import java.util.concurrent.CopyOnWriteArraySet;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.cql3.QueryEvents;
+import org.apache.cassandra.service.QueryState;
+
+public class AuthEvents
+{
+ private static final Logger logger = LoggerFactory.getLogger(QueryEvents.class);
+
+ public static final AuthEvents instance = new AuthEvents();
+
+ private final Set<Listener> listeners = new CopyOnWriteArraySet<>();
+
+ @VisibleForTesting
+ public int listenerCount()
+ {
+ return listeners.size();
+ }
+
+ public void registerListener(Listener listener)
+ {
+ listeners.add(listener);
+ }
+
+ public void unregisterListener(Listener listener)
+ {
+ listeners.remove(listener);
+ }
+
+ public void notifyAuthSuccess(QueryState state)
+ {
+ try
+ {
+ for (Listener listener : listeners)
+ listener.authSuccess(state);
+ }
+ catch (Exception e)
+ {
+ logger.error("Failed notifying listeners", e);
+ }
+ }
+
+ public void notifyAuthFailure(QueryState state, Exception cause)
+ {
+ try
+ {
+ for (Listener listener : listeners)
+ listener.authFailure(state, cause);
+ }
+ catch (Exception e)
+ {
+ logger.error("Failed notifying listeners", e);
+ }
+ }
+
+ public static interface Listener
+ {
+ void authSuccess(QueryState state);
+ void authFailure(QueryState state, Exception cause);
+ }
+}
diff --git a/src/java/org/apache/cassandra/auth/AuthKeyspace.java b/src/java/org/apache/cassandra/auth/AuthKeyspace.java
index afa94a6..a57257c 100644
--- a/src/java/org/apache/cassandra/auth/AuthKeyspace.java
+++ b/src/java/org/apache/cassandra/auth/AuthKeyspace.java
@@ -19,12 +19,16 @@
import java.util.concurrent.TimeUnit;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.SchemaConstants;
+import org.apache.cassandra.cql3.statements.schema.CreateTableStatement;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.schema.SchemaConstants;
import org.apache.cassandra.schema.KeyspaceMetadata;
import org.apache.cassandra.schema.KeyspaceParams;
import org.apache.cassandra.schema.Tables;
+import static java.lang.String.format;
+
public final class AuthKeyspace
{
private AuthKeyspace()
@@ -37,62 +41,75 @@
* generation and document your change here.
*
* gen 0: original definition in 3.0
+ * gen 1: compression chunk length reduced to 16KiB, memtable_flush_period_in_ms now unset on all tables in 4.0
*/
- public static final long GENERATION = 0;
+ public static final long GENERATION = 1;
public static final String ROLES = "roles";
public static final String ROLE_MEMBERS = "role_members";
public static final String ROLE_PERMISSIONS = "role_permissions";
public static final String RESOURCE_ROLE_INDEX = "resource_role_permissons_index";
+ public static final String NETWORK_PERMISSIONS = "network_permissions";
public static final long SUPERUSER_SETUP_DELAY = Long.getLong("cassandra.superuser_setup_delay_ms", 10000);
- private static final CFMetaData Roles =
- compile(ROLES,
- "role definitions",
- "CREATE TABLE %s ("
- + "role text,"
- + "is_superuser boolean,"
- + "can_login boolean,"
- + "salted_hash text,"
- + "member_of set<text>,"
- + "PRIMARY KEY(role))");
+ private static final TableMetadata Roles =
+ parse(ROLES,
+ "role definitions",
+ "CREATE TABLE %s ("
+ + "role text,"
+ + "is_superuser boolean,"
+ + "can_login boolean,"
+ + "salted_hash text,"
+ + "member_of set<text>,"
+ + "PRIMARY KEY(role))");
- private static final CFMetaData RoleMembers =
- compile(ROLE_MEMBERS,
- "role memberships lookup table",
- "CREATE TABLE %s ("
- + "role text,"
- + "member text,"
- + "PRIMARY KEY(role, member))");
+ private static final TableMetadata RoleMembers =
+ parse(ROLE_MEMBERS,
+ "role memberships lookup table",
+ "CREATE TABLE %s ("
+ + "role text,"
+ + "member text,"
+ + "PRIMARY KEY(role, member))");
- private static final CFMetaData RolePermissions =
- compile(ROLE_PERMISSIONS,
- "permissions granted to db roles",
- "CREATE TABLE %s ("
- + "role text,"
- + "resource text,"
- + "permissions set<text>,"
- + "PRIMARY KEY(role, resource))");
+ private static final TableMetadata RolePermissions =
+ parse(ROLE_PERMISSIONS,
+ "permissions granted to db roles",
+ "CREATE TABLE %s ("
+ + "role text,"
+ + "resource text,"
+ + "permissions set<text>,"
+ + "PRIMARY KEY(role, resource))");
- private static final CFMetaData ResourceRoleIndex =
- compile(RESOURCE_ROLE_INDEX,
- "index of db roles with permissions granted on a resource",
- "CREATE TABLE %s ("
- + "resource text,"
- + "role text,"
- + "PRIMARY KEY(resource, role))");
+ private static final TableMetadata ResourceRoleIndex =
+ parse(RESOURCE_ROLE_INDEX,
+ "index of db roles with permissions granted on a resource",
+ "CREATE TABLE %s ("
+ + "resource text,"
+ + "role text,"
+ + "PRIMARY KEY(resource, role))");
+ private static final TableMetadata NetworkPermissions =
+ parse(NETWORK_PERMISSIONS,
+ "user network permissions",
+ "CREATE TABLE %s ("
+ + "role text, "
+ + "dcs frozen<set<text>>, "
+ + "PRIMARY KEY(role))");
- private static CFMetaData compile(String name, String description, String schema)
+ private static TableMetadata parse(String name, String description, String cql)
{
- return CFMetaData.compile(String.format(schema, name), SchemaConstants.AUTH_KEYSPACE_NAME)
- .comment(description)
- .gcGraceSeconds((int) TimeUnit.DAYS.toSeconds(90));
+ return CreateTableStatement.parse(format(cql, name), SchemaConstants.AUTH_KEYSPACE_NAME)
+ .id(TableId.forSystemTable(SchemaConstants.AUTH_KEYSPACE_NAME, name))
+ .comment(description)
+ .gcGraceSeconds((int) TimeUnit.DAYS.toSeconds(90))
+ .build();
}
public static KeyspaceMetadata metadata()
{
- return KeyspaceMetadata.create(SchemaConstants.AUTH_KEYSPACE_NAME, KeyspaceParams.simple(1), Tables.of(Roles, RoleMembers, RolePermissions, ResourceRoleIndex));
+ return KeyspaceMetadata.create(SchemaConstants.AUTH_KEYSPACE_NAME,
+ KeyspaceParams.simple(1),
+ Tables.of(Roles, RoleMembers, RolePermissions, ResourceRoleIndex, NetworkPermissions));
}
}
diff --git a/src/java/org/apache/cassandra/auth/AuthMigrationListener.java b/src/java/org/apache/cassandra/auth/AuthMigrationListener.java
deleted file mode 100644
index 64fe7c6..0000000
--- a/src/java/org/apache/cassandra/auth/AuthMigrationListener.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.auth;
-
-import java.util.List;
-
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.db.marshal.AbstractType;
-import org.apache.cassandra.service.MigrationListener;
-
-/**
- * MigrationListener implementation that cleans up permissions on dropped resources.
- */
-public class AuthMigrationListener extends MigrationListener
-{
- public void onDropKeyspace(String ksName)
- {
- DatabaseDescriptor.getAuthorizer().revokeAllOn(DataResource.keyspace(ksName));
- DatabaseDescriptor.getAuthorizer().revokeAllOn(FunctionResource.keyspace(ksName));
- }
-
- public void onDropColumnFamily(String ksName, String cfName)
- {
- DatabaseDescriptor.getAuthorizer().revokeAllOn(DataResource.table(ksName, cfName));
- }
-
- public void onDropFunction(String ksName, String functionName, List<AbstractType<?>> argTypes)
- {
- DatabaseDescriptor.getAuthorizer()
- .revokeAllOn(FunctionResource.function(ksName, functionName, argTypes));
- }
-
- public void onDropAggregate(String ksName, String aggregateName, List<AbstractType<?>> argTypes)
- {
- DatabaseDescriptor.getAuthorizer()
- .revokeAllOn(FunctionResource.function(ksName, aggregateName, argTypes));
- }
-}
diff --git a/src/java/org/apache/cassandra/auth/AuthSchemaChangeListener.java b/src/java/org/apache/cassandra/auth/AuthSchemaChangeListener.java
new file mode 100644
index 0000000..6c21d7b
--- /dev/null
+++ b/src/java/org/apache/cassandra/auth/AuthSchemaChangeListener.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.auth;
+
+import java.util.List;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.schema.SchemaChangeListener;
+
+/**
+ * SchemaChangeListener implementation that cleans up permissions on dropped resources.
+ */
+public class AuthSchemaChangeListener extends SchemaChangeListener
+{
+ public void onDropKeyspace(String ksName)
+ {
+ DatabaseDescriptor.getAuthorizer().revokeAllOn(DataResource.keyspace(ksName));
+ DatabaseDescriptor.getAuthorizer().revokeAllOn(FunctionResource.keyspace(ksName));
+ }
+
+ public void onDropTable(String ksName, String cfName)
+ {
+ DatabaseDescriptor.getAuthorizer().revokeAllOn(DataResource.table(ksName, cfName));
+ }
+
+ public void onDropFunction(String ksName, String functionName, List<AbstractType<?>> argTypes)
+ {
+ DatabaseDescriptor.getAuthorizer()
+ .revokeAllOn(FunctionResource.function(ksName, functionName, argTypes));
+ }
+
+ public void onDropAggregate(String ksName, String aggregateName, List<AbstractType<?>> argTypes)
+ {
+ DatabaseDescriptor.getAuthorizer()
+ .revokeAllOn(FunctionResource.function(ksName, aggregateName, argTypes));
+ }
+}
diff --git a/src/java/org/apache/cassandra/auth/AuthenticatedUser.java b/src/java/org/apache/cassandra/auth/AuthenticatedUser.java
index 5e57308..9f22bea 100644
--- a/src/java/org/apache/cassandra/auth/AuthenticatedUser.java
+++ b/src/java/org/apache/cassandra/auth/AuthenticatedUser.java
@@ -22,6 +22,7 @@
import com.google.common.base.Objects;
import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.dht.Datacenters;
/**
* Returned from IAuthenticator#authenticate(), represents an authenticated user everywhere internally.
@@ -39,6 +40,7 @@
// User-level permissions cache.
private static final PermissionsCache permissionsCache = new PermissionsCache(DatabaseDescriptor.getAuthorizer());
+ private static final NetworkAuthCache networkAuthCache = new NetworkAuthCache(DatabaseDescriptor.getNetworkAuthorizer());
private final String name;
// primary Role of the logged in user
@@ -82,7 +84,7 @@
/**
* Some internal operations are performed on behalf of Cassandra itself, in those cases
* the system user should be used where an identity is required
- * see CreateRoleStatement#execute() and overrides of SchemaAlteringStatement#grantPermissionsToCreator()
+ * see CreateRoleStatement#execute() and overrides of AlterSchemaStatement#createdResources()
*/
public boolean isSystem()
{
@@ -92,18 +94,51 @@
/**
* Get the roles that have been granted to the user via the IRoleManager
*
- * @return a list of roles that have been granted to the user
+ * @return a set of identifiers for the roles that have been granted to the user
*/
public Set<RoleResource> getRoles()
{
return Roles.getRoles(role);
}
+ /**
+ * Get the detailed info on roles granted to the user via IRoleManager
+ *
+ * @return a set of Role objects detailing the roles granted to the user
+ */
+ public Set<Role> getRoleDetails()
+ {
+ return Roles.getRoleDetails(role);
+ }
+
public Set<Permission> getPermissions(IResource resource)
{
return permissionsCache.getPermissions(this, resource);
}
+ /**
+ * Check whether this user has login privileges.
+ * LOGIN is not inherited from granted roles, so must be directly granted to the primary role for this user
+ *
+ * @return true if the user is permitted to login, false otherwise.
+ */
+ public boolean canLogin()
+ {
+ return Roles.canLogin(getPrimaryRole());
+ }
+
+ /**
+ * Verify that there is not DC level restriction on this user accessing this node.
+ * Further extends the login privilege check by verifying that the primary role for this user is permitted
+ * to perform operations in the local (to this node) datacenter. Like LOGIN, this is not inherited from
+ * granted roles.
+ * @return true if the user is permitted to access nodes in this node's datacenter, false otherwise
+ */
+ public boolean hasLocalAccess()
+ {
+ return networkAuthCache.get(this.getPrimaryRole()).canAccess(Datacenters.thisDatacenter());
+ }
+
@Override
public String toString()
{
@@ -129,4 +164,5 @@
{
return Objects.hashCode(name);
}
+
}
diff --git a/src/java/org/apache/cassandra/auth/CassandraAuthorizer.java b/src/java/org/apache/cassandra/auth/CassandraAuthorizer.java
index d4253d1..37ad60a 100644
--- a/src/java/org/apache/cassandra/auth/CassandraAuthorizer.java
+++ b/src/java/org/apache/cassandra/auth/CassandraAuthorizer.java
@@ -18,9 +18,7 @@
package org.apache.cassandra.auth;
import java.util.*;
-import java.util.concurrent.TimeUnit;
-import com.google.common.base.Predicate;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
@@ -28,10 +26,7 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.cassandra.concurrent.ScheduledExecutors;
import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.config.Schema;
-import org.apache.cassandra.config.SchemaConstants;
import org.apache.cassandra.cql3.*;
import org.apache.cassandra.cql3.statements.BatchStatement;
import org.apache.cassandra.cql3.statements.ModificationStatement;
@@ -39,13 +34,8 @@
import org.apache.cassandra.db.ConsistencyLevel;
import org.apache.cassandra.db.marshal.UTF8Type;
import org.apache.cassandra.exceptions.*;
-import org.apache.cassandra.serializers.SetSerializer;
-import org.apache.cassandra.serializers.UTF8Serializer;
+import org.apache.cassandra.schema.SchemaConstants;
import org.apache.cassandra.service.ClientState;
-
-import org.apache.cassandra.cql3.QueryOptions;
-import org.apache.cassandra.cql3.QueryProcessor;
-import org.apache.cassandra.cql3.UntypedResultSet;
import org.apache.cassandra.service.QueryState;
import org.apache.cassandra.transport.messages.ResultMessage;
import org.apache.cassandra.utils.ByteBufferUtil;
@@ -63,12 +53,7 @@
private static final String RESOURCE = "resource";
private static final String PERMISSIONS = "permissions";
- // used during upgrades to perform authz on mixed clusters
- public static final String USERNAME = "username";
- public static final String USER_PERMISSIONS = "permissions";
-
private SelectStatement authorizeRoleStatement;
- private SelectStatement legacyAuthorizeRoleStatement;
public CassandraAuthorizer()
{
@@ -85,9 +70,10 @@
Set<Permission> permissions = EnumSet.noneOf(Permission.class);
- for (RoleResource role: user.getRoles())
- addPermissionsForRole(permissions, resource, role);
-
+ // Even though we only care about the RoleResource here, we use getRoleDetails as
+ // it saves a Set creation in RolesCache
+ for (Role role: user.getRoleDetails())
+ addPermissionsForRole(permissions, resource, role.resource);
return permissions;
}
catch (RequestExecutionException | RequestValidationException e)
@@ -134,7 +120,7 @@
AuthKeyspace.RESOURCE_ROLE_INDEX,
escape(row.getString("resource")),
escape(revokee.getRoleName())),
- ClientState.forInternalCalls()).statement);
+ ClientState.forInternalCalls()));
}
@@ -142,13 +128,13 @@
SchemaConstants.AUTH_KEYSPACE_NAME,
AuthKeyspace.ROLE_PERMISSIONS,
escape(revokee.getRoleName())),
- ClientState.forInternalCalls()).statement);
+ ClientState.forInternalCalls()));
executeLoggedBatch(statements);
}
catch (RequestExecutionException | RequestValidationException e)
{
- logger.warn("CassandraAuthorizer failed to revoke all permissions of {}: {}", revokee.getRoleName(), e);
+ logger.warn(String.format("CassandraAuthorizer failed to revoke all permissions of %s", revokee.getRoleName()), e);
}
}
@@ -172,36 +158,31 @@
AuthKeyspace.ROLE_PERMISSIONS,
escape(row.getString("role")),
escape(droppedResource.getName())),
- ClientState.forInternalCalls()).statement);
+ ClientState.forInternalCalls()));
}
statements.add(QueryProcessor.getStatement(String.format("DELETE FROM %s.%s WHERE resource = '%s'",
SchemaConstants.AUTH_KEYSPACE_NAME,
AuthKeyspace.RESOURCE_ROLE_INDEX,
escape(droppedResource.getName())),
- ClientState.forInternalCalls()).statement);
+ ClientState.forInternalCalls()));
executeLoggedBatch(statements);
}
catch (RequestExecutionException | RequestValidationException e)
{
- logger.warn("CassandraAuthorizer failed to revoke all permissions on {}: {}", droppedResource, e);
- return;
+ logger.warn(String.format("CassandraAuthorizer failed to revoke all permissions on %s", droppedResource), e);
}
}
private void executeLoggedBatch(List<CQLStatement> statements)
throws RequestExecutionException, RequestValidationException
{
- BatchStatement batch = new BatchStatement(0,
- BatchStatement.Type.LOGGED,
+ BatchStatement batch = new BatchStatement(BatchStatement.Type.LOGGED,
+ VariableSpecifications.empty(),
Lists.newArrayList(Iterables.filter(statements, ModificationStatement.class)),
Attributes.none());
- QueryProcessor.instance.processBatch(batch,
- QueryState.forInternalCalls(),
- BatchQueryOptions.withoutPerStatementVariables(QueryOptions.DEFAULT),
- System.nanoTime());
-
+ processBatch(batch);
}
// Add every permission on the resource granted to the role
@@ -212,19 +193,8 @@
Lists.newArrayList(ByteBufferUtil.bytes(role.getRoleName()),
ByteBufferUtil.bytes(resource.getName())));
- SelectStatement statement;
- // If it exists, read from the legacy user permissions table to handle the case where the cluster
- // is being upgraded and so is running with mixed versions of the authz schema
- if (Schema.instance.getCFMetaData(SchemaConstants.AUTH_KEYSPACE_NAME, USER_PERMISSIONS) == null)
- statement = authorizeRoleStatement;
- else
- {
- // If the permissions table was initialised only after the statement got prepared, re-prepare (CASSANDRA-12813)
- if (legacyAuthorizeRoleStatement == null)
- legacyAuthorizeRoleStatement = prepare(USERNAME, USER_PERMISSIONS);
- statement = legacyAuthorizeRoleStatement;
- }
- ResultMessage.Rows rows = statement.execute(QueryState.forInternalCalls(), options, System.nanoTime());
+ ResultMessage.Rows rows = select(authorizeRoleStatement, options);
+
UntypedResultSet result = UntypedResultSet.create(rows.result);
if (!result.isEmpty() && result.one().has(PERMISSIONS))
@@ -300,11 +270,7 @@
throws RequestExecutionException
{
Set<PermissionDetails> details = new HashSet<>();
- // If it exists, try the legacy user permissions table first. This is to handle the case
- // where the cluster is being upgraded and so is running with mixed versions of the perms table
- boolean useLegacyTable = Schema.instance.getCFMetaData(SchemaConstants.AUTH_KEYSPACE_NAME, USER_PERMISSIONS) != null;
- String entityColumnName = useLegacyTable ? USERNAME : ROLE;
- for (UntypedResultSet.Row row : process(buildListQuery(resource, role, useLegacyTable)))
+ for (UntypedResultSet.Row row : process(buildListQuery(resource, role)))
{
if (row.has(PERMISSIONS))
{
@@ -312,7 +278,7 @@
{
Permission permission = Permission.valueOf(p);
if (permissions.contains(permission))
- details.add(new PermissionDetails(row.getString(entityColumnName),
+ details.add(new PermissionDetails(row.getString(ROLE),
Resources.fromName(row.getString(RESOURCE)),
permission));
}
@@ -321,11 +287,9 @@
return details;
}
- private String buildListQuery(IResource resource, RoleResource grantee, boolean useLegacyTable)
+ private String buildListQuery(IResource resource, RoleResource grantee)
{
- String tableName = useLegacyTable ? USER_PERMISSIONS : AuthKeyspace.ROLE_PERMISSIONS;
- String entityName = useLegacyTable ? USERNAME : ROLE;
- List<String> vars = Lists.newArrayList(SchemaConstants.AUTH_KEYSPACE_NAME, tableName);
+ List<String> vars = Lists.newArrayList(SchemaConstants.AUTH_KEYSPACE_NAME, AuthKeyspace.ROLE_PERMISSIONS);
List<String> conditions = new ArrayList<>();
if (resource != null)
@@ -336,11 +300,11 @@
if (grantee != null)
{
- conditions.add(entityName + " = '%s'");
+ conditions.add(ROLE + " = '%s'");
vars.add(escape(grantee.getRoleName()));
}
- String query = "SELECT " + entityName + ", resource, permissions FROM %s.%s";
+ String query = "SELECT " + ROLE + ", resource, permissions FROM %s.%s";
if (!conditions.isEmpty())
query += " WHERE " + StringUtils.join(conditions, " AND ");
@@ -364,21 +328,6 @@
public void setup()
{
authorizeRoleStatement = prepare(ROLE, AuthKeyspace.ROLE_PERMISSIONS);
-
- // If old user permissions table exists, migrate the legacy authz data to the new table
- // The delay is to give the node a chance to see its peers before attempting the conversion
- if (Schema.instance.getCFMetaData(SchemaConstants.AUTH_KEYSPACE_NAME, "permissions") != null)
- {
- legacyAuthorizeRoleStatement = prepare(USERNAME, USER_PERMISSIONS);
-
- ScheduledExecutors.optionalTasks.schedule(new Runnable()
- {
- public void run()
- {
- convertLegacyData();
- }
- }, AuthKeyspace.SUPERUSER_SETUP_DELAY, TimeUnit.MILLISECONDS);
- }
}
private SelectStatement prepare(String entityname, String permissionsTable)
@@ -387,72 +336,7 @@
SchemaConstants.AUTH_KEYSPACE_NAME,
permissionsTable,
entityname);
- return (SelectStatement) QueryProcessor.getStatement(query, ClientState.forInternalCalls()).statement;
- }
-
- /**
- * Copy legacy authz data from the system_auth.permissions table to the new system_auth.role_permissions table and
- * also insert entries into the reverse lookup table.
- * In theory, we could simply rename the existing table as the schema is structurally the same, but this would
- * break mixed clusters during a rolling upgrade.
- * This setup is not performed if AllowAllAuthenticator is configured (see Auth#setup).
- */
- private void convertLegacyData()
- {
- try
- {
- if (Schema.instance.getCFMetaData("system_auth", "permissions") != null)
- {
- logger.info("Converting legacy permissions data");
- CQLStatement insertStatement =
- QueryProcessor.getStatement(String.format("INSERT INTO %s.%s (role, resource, permissions) " +
- "VALUES (?, ?, ?)",
- SchemaConstants.AUTH_KEYSPACE_NAME,
- AuthKeyspace.ROLE_PERMISSIONS),
- ClientState.forInternalCalls()).statement;
- CQLStatement indexStatement =
- QueryProcessor.getStatement(String.format("INSERT INTO %s.%s (resource, role) VALUES (?,?)",
- SchemaConstants.AUTH_KEYSPACE_NAME,
- AuthKeyspace.RESOURCE_ROLE_INDEX),
- ClientState.forInternalCalls()).statement;
-
- UntypedResultSet permissions = process("SELECT * FROM system_auth.permissions");
- for (UntypedResultSet.Row row : permissions)
- {
- final IResource resource = Resources.fromName(row.getString("resource"));
- Predicate<String> isApplicable = new Predicate<String>()
- {
- public boolean apply(String s)
- {
- return resource.applicablePermissions().contains(Permission.valueOf(s));
- }
- };
- SetSerializer<String> serializer = SetSerializer.getInstance(UTF8Serializer.instance, UTF8Type.instance);
- Set<String> originalPerms = serializer.deserialize(row.getBytes("permissions"));
- Set<String> filteredPerms = ImmutableSet.copyOf(Iterables.filter(originalPerms, isApplicable));
- insertStatement.execute(QueryState.forInternalCalls(),
- QueryOptions.forInternalCalls(ConsistencyLevel.ONE,
- Lists.newArrayList(row.getBytes("username"),
- row.getBytes("resource"),
- serializer.serialize(filteredPerms))),
- System.nanoTime());
-
- indexStatement.execute(QueryState.forInternalCalls(),
- QueryOptions.forInternalCalls(ConsistencyLevel.ONE,
- Lists.newArrayList(row.getBytes("resource"),
- row.getBytes("username"))),
- System.nanoTime());
-
- }
- logger.info("Completed conversion of legacy permissions");
- }
- }
- catch (Exception e)
- {
- logger.info("Unable to complete conversion of legacy permissions data (perhaps not enough nodes are upgraded yet). " +
- "Conversion should not be considered complete");
- logger.trace("Conversion error", e);
- }
+ return (SelectStatement) QueryProcessor.getStatement(query, ClientState.forInternalCalls());
}
// We only worry about one character ('). Make sure it's properly escaped.
@@ -461,8 +345,21 @@
return StringUtils.replace(name, "'", "''");
}
- private UntypedResultSet process(String query) throws RequestExecutionException
+ ResultMessage.Rows select(SelectStatement statement, QueryOptions options)
+ {
+ return statement.execute(QueryState.forInternalCalls(), options, System.nanoTime());
+ }
+
+ UntypedResultSet process(String query) throws RequestExecutionException
{
return QueryProcessor.process(query, ConsistencyLevel.LOCAL_ONE);
}
+
+ void processBatch(BatchStatement statement)
+ {
+ QueryProcessor.instance.processBatch(statement,
+ QueryState.forInternalCalls(),
+ BatchQueryOptions.withoutPerStatementVariables(QueryOptions.DEFAULT),
+ System.nanoTime());
+ }
}
diff --git a/src/java/org/apache/cassandra/auth/CassandraNetworkAuthorizer.java b/src/java/org/apache/cassandra/auth/CassandraNetworkAuthorizer.java
new file mode 100644
index 0000000..6fdcd69
--- /dev/null
+++ b/src/java/org/apache/cassandra/auth/CassandraNetworkAuthorizer.java
@@ -0,0 +1,157 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.auth;
+
+import java.util.Set;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Lists;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.QueryOptions;
+import org.apache.cassandra.cql3.QueryProcessor;
+import org.apache.cassandra.cql3.UntypedResultSet;
+import org.apache.cassandra.cql3.statements.SelectStatement;
+import org.apache.cassandra.db.ConsistencyLevel;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.schema.SchemaConstants;
+import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.service.QueryState;
+import org.apache.cassandra.transport.messages.ResultMessage;
+import org.apache.cassandra.utils.ByteBufferUtil;
+
+public class CassandraNetworkAuthorizer implements INetworkAuthorizer
+{
+ private SelectStatement authorizeUserStatement = null;
+
+ public void setup()
+ {
+ String query = String.format("SELECT dcs FROM %s.%s WHERE role = ?",
+ SchemaConstants.AUTH_KEYSPACE_NAME,
+ AuthKeyspace.NETWORK_PERMISSIONS);
+ authorizeUserStatement = (SelectStatement) QueryProcessor.getStatement(query, ClientState.forInternalCalls());
+ }
+
+ @VisibleForTesting
+ ResultMessage.Rows select(SelectStatement statement, QueryOptions options)
+ {
+ return statement.execute(QueryState.forInternalCalls(), options, System.nanoTime());
+ }
+
+ @VisibleForTesting
+ void process(String query)
+ {
+ QueryProcessor.process(query, ConsistencyLevel.LOCAL_ONE);
+ }
+
+ private Set<String> getAuthorizedDcs(String name)
+ {
+ QueryOptions options = QueryOptions.forInternalCalls(ConsistencyLevel.LOCAL_ONE,
+ Lists.newArrayList(ByteBufferUtil.bytes(name)));
+
+ ResultMessage.Rows rows = select(authorizeUserStatement, options);
+ UntypedResultSet result = UntypedResultSet.create(rows.result);
+ Set<String> dcs = null;
+ if (!result.isEmpty() && result.one().has("dcs"))
+ {
+ dcs = result.one().getFrozenSet("dcs", UTF8Type.instance);
+ }
+ return dcs;
+ }
+
+ public DCPermissions authorize(RoleResource role)
+ {
+ if (!Roles.canLogin(role))
+ {
+ return DCPermissions.none();
+ }
+ if (Roles.hasSuperuserStatus(role))
+ {
+ return DCPermissions.all();
+ }
+
+ Set<String> dcs = getAuthorizedDcs(role.getName());
+
+ if (dcs == null || dcs.isEmpty())
+ {
+ return DCPermissions.all();
+ }
+ else
+ {
+ return DCPermissions.subset(dcs);
+ }
+ }
+
+ private static String getSetString(DCPermissions permissions)
+ {
+ if (permissions.restrictsAccess())
+ {
+ StringBuilder builder = new StringBuilder();
+ builder.append('{');
+ boolean first = true;
+ for (String dc: permissions.allowedDCs())
+ {
+ if (first)
+ {
+ first = false;
+ }
+ else
+ {
+ builder.append(", ");
+ }
+ builder.append('\'');
+ builder.append(dc);
+ builder.append('\'');
+ }
+ builder.append('}');
+ return builder.toString();
+ }
+ else
+ {
+ return "{}";
+ }
+ }
+
+ public void setRoleDatacenters(RoleResource role, DCPermissions permissions)
+ {
+ String query = String.format("UPDATE %s.%s SET dcs = %s WHERE role = '%s'",
+ SchemaConstants.AUTH_KEYSPACE_NAME,
+ AuthKeyspace.NETWORK_PERMISSIONS,
+ getSetString(permissions),
+ role.getName());
+
+ process(query);
+ }
+
+ public void drop(RoleResource role)
+ {
+ String query = String.format("DELETE FROM %s.%s WHERE role = '%s'",
+ SchemaConstants.AUTH_KEYSPACE_NAME,
+ AuthKeyspace.NETWORK_PERMISSIONS,
+ role.getName());
+
+ process(query);
+ }
+
+ public void validateConfiguration() throws ConfigurationException
+ {
+ // noop
+ }
+}
diff --git a/src/java/org/apache/cassandra/auth/CassandraRoleManager.java b/src/java/org/apache/cassandra/auth/CassandraRoleManager.java
index 38862d9..f4c9428 100644
--- a/src/java/org/apache/cassandra/auth/CassandraRoleManager.java
+++ b/src/java/org/apache/cassandra/auth/CassandraRoleManager.java
@@ -20,11 +20,14 @@
import java.util.*;
import java.util.concurrent.Callable;
import java.util.concurrent.TimeUnit;
+import java.util.function.Function;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
-import com.google.common.base.*;
-import com.google.common.base.Objects;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Strings;
import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Iterables;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -32,14 +35,12 @@
import org.apache.cassandra.concurrent.ScheduledExecutors;
import org.apache.cassandra.config.Config;
import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.config.Schema;
-import org.apache.cassandra.config.SchemaConstants;
+import org.apache.cassandra.schema.SchemaConstants;
import org.apache.cassandra.cql3.*;
import org.apache.cassandra.cql3.statements.SelectStatement;
import org.apache.cassandra.db.ConsistencyLevel;
import org.apache.cassandra.db.marshal.UTF8Type;
import org.apache.cassandra.exceptions.*;
-import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.service.ClientState;
import org.apache.cassandra.service.QueryState;
import org.apache.cassandra.service.StorageService;
@@ -80,41 +81,24 @@
static final String DEFAULT_SUPERUSER_PASSWORD = "cassandra";
// Transform a row in the AuthKeyspace.ROLES to a Role instance
- private static final Function<UntypedResultSet.Row, Role> ROW_TO_ROLE = new Function<UntypedResultSet.Row, Role>()
+ private static final Function<UntypedResultSet.Row, Role> ROW_TO_ROLE = row ->
{
- public Role apply(UntypedResultSet.Row row)
+ try
{
- try
- {
- return new Role(row.getString("role"),
- row.getBoolean("is_superuser"),
- row.getBoolean("can_login"),
- row.has("member_of") ? row.getSet("member_of", UTF8Type.instance)
- : Collections.<String>emptySet());
- }
- // Failing to deserialize a boolean in is_superuser or can_login will throw an NPE
- catch (NullPointerException e)
- {
- logger.warn("An invalid value has been detected in the {} table for role {}. If you are " +
- "unable to login, you may need to disable authentication and confirm " +
- "that values in that table are accurate", AuthKeyspace.ROLES, row.getString("role"));
- throw new RuntimeException(String.format("Invalid metadata has been detected for role %s", row.getString("role")), e);
- }
-
+ return new Role(row.getString("role"),
+ row.getBoolean("is_superuser"),
+ row.getBoolean("can_login"),
+ Collections.emptyMap(),
+ row.has("member_of") ? row.getSet("member_of", UTF8Type.instance)
+ : Collections.<String>emptySet());
}
- };
-
- public static final String LEGACY_USERS_TABLE = "users";
- // Transform a row in the legacy system_auth.users table to a Role instance,
- // used to fallback to previous schema on a mixed cluster during an upgrade
- private static final Function<UntypedResultSet.Row, Role> LEGACY_ROW_TO_ROLE = new Function<UntypedResultSet.Row, Role>()
- {
- public Role apply(UntypedResultSet.Row row)
+ // Failing to deserialize a boolean in is_superuser or can_login will throw an NPE
+ catch (NullPointerException e)
{
- return new Role(row.getString("name"),
- row.getBoolean("super"),
- true,
- Collections.<String>emptySet());
+ logger.warn("An invalid value has been detected in the {} table for role {}. If you are " +
+ "unable to login, you may need to disable authentication and confirm " +
+ "that values in that table are accurate", AuthKeyspace.ROLES, row.getString("role"));
+ throw new RuntimeException(String.format("Invalid metadata has been detected for role %s", row.getString("role")), e);
}
};
@@ -132,11 +116,7 @@
return rounds;
}
- // NullObject returned when a supplied role name not found in AuthKeyspace.ROLES
- private static final Role NULL_ROLE = new Role(null, false, false, Collections.<String>emptySet());
-
private SelectStatement loadRoleStatement;
- private SelectStatement legacySelectUserStatement;
private final Set<Option> supportedOptions;
private final Set<Option> alterableOptions;
@@ -159,27 +139,10 @@
loadRoleStatement = (SelectStatement) prepare("SELECT * from %s.%s WHERE role = ?",
SchemaConstants.AUTH_KEYSPACE_NAME,
AuthKeyspace.ROLES);
- // If the old users table exists, we may need to migrate the legacy authn
- // data to the new table. We also need to prepare a statement to read from
- // it, so we can continue to use the old tables while the cluster is upgraded.
- // Otherwise, we may need to create a default superuser role to enable others
- // to be added.
- if (Schema.instance.getCFMetaData(SchemaConstants.AUTH_KEYSPACE_NAME, "users") != null)
- {
- legacySelectUserStatement = prepareLegacySelectUserStatement();
-
- scheduleSetupTask(() -> {
- convertLegacyData();
- return null;
- });
- }
- else
- {
- scheduleSetupTask(() -> {
- setupDefaultRole();
- return null;
- });
- }
+ scheduleSetupTask(() -> {
+ setupDefaultRole();
+ return null;
+ });
}
public Set<Option> supportedOptions()
@@ -226,8 +189,7 @@
{
// Unlike most of the other data access methods here, this does not use a
// prepared statement in order to allow the set of assignments to be variable.
- String assignments = Joiner.on(',').join(Iterables.filter(optionsToAssignments(options.getOptions()),
- Predicates.notNull()));
+ String assignments = optionsToAssignments(options.getOptions());
if (!Strings.isNullOrEmpty(assignments))
{
process(String.format("UPDATE %s.%s SET %s WHERE role = '%s'",
@@ -277,29 +239,33 @@
consistencyForRole(role.getRoleName()));
}
- public Set<RoleResource> getRoles(RoleResource grantee, boolean includeInherited) throws RequestValidationException, RequestExecutionException
+ public Set<RoleResource> getRoles(RoleResource grantee, boolean includeInherited)
+ throws RequestValidationException, RequestExecutionException
{
- Set<RoleResource> roles = new HashSet<>();
- Role role = getRole(grantee.getRoleName());
- if (!role.equals(NULL_ROLE))
- {
- roles.add(RoleResource.role(role.name));
- collectRoles(role, roles, includeInherited);
- }
- return roles;
+ return collectRoles(getRole(grantee.getRoleName()),
+ includeInherited,
+ filter())
+ .map(r -> r.resource)
+ .collect(Collectors.toSet());
+ }
+
+ public Set<Role> getRoleDetails(RoleResource grantee)
+ {
+ return collectRoles(getRole(grantee.getRoleName()),
+ true,
+ filter())
+ .collect(Collectors.toSet());
}
public Set<RoleResource> getAllRoles() throws RequestValidationException, RequestExecutionException
{
- UntypedResultSet rows = process(String.format("SELECT role from %s.%s", SchemaConstants.AUTH_KEYSPACE_NAME, AuthKeyspace.ROLES), ConsistencyLevel.QUORUM);
- Iterable<RoleResource> roles = Iterables.transform(rows, new Function<UntypedResultSet.Row, RoleResource>()
- {
- public RoleResource apply(UntypedResultSet.Row row)
- {
- return RoleResource.role(row.getString("role"));
- }
- });
- return ImmutableSet.<RoleResource>builder().addAll(roles).build();
+ ImmutableSet.Builder<RoleResource> builder = ImmutableSet.builder();
+ UntypedResultSet rows = process(String.format("SELECT role from %s.%s",
+ SchemaConstants.AUTH_KEYSPACE_NAME,
+ AuthKeyspace.ROLES),
+ ConsistencyLevel.QUORUM);
+ rows.forEach(row -> builder.add(RoleResource.role(row.getString("role"))));
+ return builder.build();
}
public boolean isSuper(RoleResource role)
@@ -335,7 +301,7 @@
public boolean isExistingRole(RoleResource role)
{
- return getRole(role.getRoleName()) != NULL_ROLE;
+ return !Roles.isNullRole(getRole(role.getRoleName()));
}
public Set<? extends IResource> protectedResources()
@@ -389,101 +355,28 @@
|| !QueryProcessor.process(allUsersQuery, ConsistencyLevel.QUORUM).isEmpty();
}
- private void scheduleSetupTask(final Callable<Void> setupTask)
+ protected void scheduleSetupTask(final Callable<Void> setupTask)
{
// The delay is to give the node a chance to see its peers before attempting the operation
- ScheduledExecutors.optionalTasks.schedule(new Runnable()
- {
- public void run()
+ ScheduledExecutors.optionalTasks.schedule(() -> {
+ isClusterReady = true;
+ try
{
- // If not all nodes are on 2.2, we don't want to initialize the role manager as this will confuse 2.1
- // nodes (see CASSANDRA-9761 for details). So we re-schedule the setup for later, hoping that the upgrade
- // will be finished by then.
- if (!MessagingService.instance().areAllNodesAtLeast22())
- {
- logger.trace("Not all nodes are upgraded to a version that supports Roles yet, rescheduling setup task");
- scheduleSetupTask(setupTask);
- return;
- }
-
- isClusterReady = true;
- try
- {
- setupTask.call();
- }
- catch (Exception e)
- {
- logger.info("Setup task failed with error, rescheduling");
- scheduleSetupTask(setupTask);
- }
+ setupTask.call();
+ }
+ catch (Exception e)
+ {
+ logger.info("Setup task failed with error, rescheduling");
+ scheduleSetupTask(setupTask);
}
}, AuthKeyspace.SUPERUSER_SETUP_DELAY, TimeUnit.MILLISECONDS);
}
- /*
- * Copy legacy auth data from the system_auth.users & system_auth.credentials tables to
- * the new system_auth.roles table. This setup is not performed if AllowAllAuthenticator
- * is configured (see Auth#setup).
- */
- private void convertLegacyData() throws Exception
- {
- try
- {
- // read old data at QUORUM as it may contain the data for the default superuser
- if (Schema.instance.getCFMetaData("system_auth", "users") != null)
- {
- logger.info("Converting legacy users");
- UntypedResultSet users = QueryProcessor.process("SELECT * FROM system_auth.users",
- ConsistencyLevel.QUORUM);
- for (UntypedResultSet.Row row : users)
- {
- RoleOptions options = new RoleOptions();
- options.setOption(Option.SUPERUSER, row.getBoolean("super"));
- options.setOption(Option.LOGIN, true);
- createRole(null, RoleResource.role(row.getString("name")), options);
- }
- logger.info("Completed conversion of legacy users");
- }
-
- if (Schema.instance.getCFMetaData("system_auth", "credentials") != null)
- {
- logger.info("Migrating legacy credentials data to new system table");
- UntypedResultSet credentials = QueryProcessor.process("SELECT * FROM system_auth.credentials",
- ConsistencyLevel.QUORUM);
- for (UntypedResultSet.Row row : credentials)
- {
- // Write the password directly into the table to avoid doubly encrypting it
- QueryProcessor.process(String.format("UPDATE %s.%s SET salted_hash = '%s' WHERE role = '%s'",
- SchemaConstants.AUTH_KEYSPACE_NAME,
- AuthKeyspace.ROLES,
- row.getString("salted_hash"),
- row.getString("username")),
- consistencyForRole(row.getString("username")));
- }
- logger.info("Completed conversion of legacy credentials");
- }
- }
- catch (Exception e)
- {
- logger.info("Unable to complete conversion of legacy auth data (perhaps not enough nodes are upgraded yet). " +
- "Conversion should not be considered complete");
- logger.trace("Conversion error", e);
- throw e;
- }
- }
-
- private SelectStatement prepareLegacySelectUserStatement()
- {
- return (SelectStatement) prepare("SELECT * FROM %s.%s WHERE name = ?",
- SchemaConstants.AUTH_KEYSPACE_NAME,
- LEGACY_USERS_TABLE);
- }
-
private CQLStatement prepare(String template, String keyspace, String table)
{
try
{
- return QueryProcessor.parseStatement(String.format(template, keyspace, table)).prepare(ClientState.forInternalCalls()).statement;
+ return QueryProcessor.parseStatement(String.format(template, keyspace, table)).prepare(ClientState.forInternalCalls());
}
catch (RequestValidationException e)
{
@@ -491,55 +384,42 @@
}
}
- /*
- * Retrieve all roles granted to the given role. includeInherited specifies
- * whether to include only those roles granted directly or all inherited roles.
- */
- private void collectRoles(Role role, Set<RoleResource> collected, boolean includeInherited) throws RequestValidationException, RequestExecutionException
+ private Stream<Role> collectRoles(Role role, boolean includeInherited, Predicate<String> distinctFilter)
{
- for (String memberOf : role.memberOf)
- {
- Role granted = getRole(memberOf);
- if (granted.equals(NULL_ROLE))
- continue;
- collected.add(RoleResource.role(granted.name));
- if (includeInherited)
- collectRoles(granted, collected, true);
- }
+ if (Roles.isNullRole(role))
+ return Stream.empty();
+
+ if (!includeInherited)
+ return Stream.concat(Stream.of(role), role.memberOf.stream().map(this::getRole));
+
+
+ return Stream.concat(Stream.of(role),
+ role.memberOf.stream()
+ .filter(distinctFilter)
+ .flatMap(r -> collectRoles(getRole(r), true, distinctFilter)));
+ }
+
+ // Used as a stateful filtering function when recursively collecting granted roles
+ private static Predicate<String> filter()
+ {
+ final Set<String> seen = new HashSet<>();
+ return seen::add;
}
/*
* Get a single Role instance given the role name. This never returns null, instead it
- * uses the null object NULL_ROLE when a role with the given name cannot be found. So
+ * uses a null object when a role with the given name cannot be found. So
* it's always safe to call methods on the returned object without risk of NPE.
*/
private Role getRole(String name)
{
- // If it exists, try the legacy users table in case the cluster
- // is in the process of being upgraded and so is running with mixed
- // versions of the authn schema.
- if (Schema.instance.getCFMetaData(SchemaConstants.AUTH_KEYSPACE_NAME, "users") == null)
- return getRoleFromTable(name, loadRoleStatement, ROW_TO_ROLE);
- else
- {
- if (legacySelectUserStatement == null)
- legacySelectUserStatement = prepareLegacySelectUserStatement();
- return getRoleFromTable(name, legacySelectUserStatement, LEGACY_ROW_TO_ROLE);
- }
- }
-
- private Role getRoleFromTable(String name, SelectStatement statement, Function<UntypedResultSet.Row, Role> function)
- throws RequestExecutionException, RequestValidationException
- {
- ResultMessage.Rows rows =
- statement.execute(QueryState.forInternalCalls(),
- QueryOptions.forInternalCalls(consistencyForRole(name),
- Collections.singletonList(ByteBufferUtil.bytes(name))),
- System.nanoTime());
+ QueryOptions options = QueryOptions.forInternalCalls(consistencyForRole(name),
+ Collections.singletonList(ByteBufferUtil.bytes(name)));
+ ResultMessage.Rows rows = select(loadRoleStatement, options);
if (rows.result.isEmpty())
- return NULL_ROLE;
+ return Roles.nullRole();
- return function.apply(UntypedResultSet.create(rows.result).one());
+ return ROW_TO_ROLE.apply(UntypedResultSet.create(rows.result).one());
}
/*
@@ -588,27 +468,26 @@
* Convert a map of Options from a CREATE/ALTER statement into
* assignment clauses used to construct a CQL UPDATE statement
*/
- private Iterable<String> optionsToAssignments(Map<Option, Object> options)
+ private String optionsToAssignments(Map<Option, Object> options)
{
- return Iterables.transform(
- options.entrySet(),
- new Function<Map.Entry<Option, Object>, String>()
- {
- public String apply(Map.Entry<Option, Object> entry)
- {
- switch (entry.getKey())
- {
- case LOGIN:
- return String.format("can_login = %s", entry.getValue());
- case SUPERUSER:
- return String.format("is_superuser = %s", entry.getValue());
- case PASSWORD:
- return String.format("salted_hash = '%s'", escape(hashpw((String) entry.getValue())));
- default:
- return null;
- }
- }
- });
+ return options.entrySet()
+ .stream()
+ .map(entry ->
+ {
+ switch (entry.getKey())
+ {
+ case LOGIN:
+ return String.format("can_login = %s", entry.getValue());
+ case SUPERUSER:
+ return String.format("is_superuser = %s", entry.getValue());
+ case PASSWORD:
+ return String.format("salted_hash = '%s'", escape(hashpw((String) entry.getValue())));
+ default:
+ return null;
+ }
+ })
+ .filter(Objects::nonNull)
+ .collect(Collectors.joining(","));
}
protected static ConsistencyLevel consistencyForRole(String role)
@@ -634,7 +513,9 @@
* This shouldn't be used during setup as this will directly return an error if the manager is not setup yet. Setup tasks
* should use QueryProcessor.process directly.
*/
- private UntypedResultSet process(String query, ConsistencyLevel consistencyLevel) throws RequestValidationException, RequestExecutionException
+ @VisibleForTesting
+ UntypedResultSet process(String query, ConsistencyLevel consistencyLevel)
+ throws RequestValidationException, RequestExecutionException
{
if (!isClusterReady)
throw new InvalidRequestException("Cannot process role related query as the role manager isn't yet setup. "
@@ -644,36 +525,10 @@
return QueryProcessor.process(query, consistencyLevel);
}
- private static final class Role
+ @VisibleForTesting
+ ResultMessage.Rows select(SelectStatement statement, QueryOptions options)
{
- private String name;
- private final boolean isSuper;
- private final boolean canLogin;
- private Set<String> memberOf;
-
- private Role(String name, boolean isSuper, boolean canLogin, Set<String> memberOf)
- {
- this.name = name;
- this.isSuper = isSuper;
- this.canLogin = canLogin;
- this.memberOf = memberOf;
- }
-
- public boolean equals(Object o)
- {
- if (this == o)
- return true;
-
- if (!(o instanceof Role))
- return false;
-
- Role r = (Role) o;
- return Objects.equal(name, r.name);
- }
-
- public int hashCode()
- {
- return Objects.hashCode(name);
- }
+ return statement.execute(QueryState.forInternalCalls(), options, System.nanoTime());
}
+
}
diff --git a/src/java/org/apache/cassandra/auth/DCPermissions.java b/src/java/org/apache/cassandra/auth/DCPermissions.java
new file mode 100644
index 0000000..d04242d
--- /dev/null
+++ b/src/java/org/apache/cassandra/auth/DCPermissions.java
@@ -0,0 +1,223 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.auth;
+
+import java.util.HashSet;
+import java.util.Set;
+import java.util.StringJoiner;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Sets;
+
+import org.apache.cassandra.dht.Datacenters;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+
+public abstract class DCPermissions
+{
+ /**
+ * returns true if the user can access the given dc
+ */
+ public abstract boolean canAccess(String dc);
+
+ /**
+ * Indicates whether the permissions object explicitly allow access to
+ * some dcs (true) or if it implicitly allows access to all dcs (false)
+ */
+ public abstract boolean restrictsAccess();
+ public abstract Set<String> allowedDCs();
+ public abstract void validate();
+
+ private static class SubsetPermissions extends DCPermissions
+ {
+ private final Set<String> subset;
+
+ public SubsetPermissions(Set<String> subset)
+ {
+ Preconditions.checkNotNull(subset);
+ this.subset = subset;
+ }
+
+ public boolean canAccess(String dc)
+ {
+ return subset.contains(dc);
+ }
+
+ public boolean restrictsAccess()
+ {
+ return true;
+ }
+
+ public Set<String> allowedDCs()
+ {
+ return ImmutableSet.copyOf(subset);
+ }
+
+ public boolean equals(Object o)
+ {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ SubsetPermissions that = (SubsetPermissions) o;
+
+ return subset.equals(that.subset);
+ }
+
+ public int hashCode()
+ {
+ return subset.hashCode();
+ }
+
+ public String toString()
+ {
+ StringJoiner joiner = new StringJoiner(", ");
+ subset.forEach(joiner::add);
+ return joiner.toString();
+ }
+
+ public void validate()
+ {
+ Set<String> unknownDcs = Sets.difference(subset, Datacenters.getValidDatacenters());
+ if (!unknownDcs.isEmpty())
+ {
+ throw new InvalidRequestException(String.format("Invalid value(s) for DATACENTERS '%s'," +
+ "All values must be valid datacenters", subset));
+ }
+ }
+ }
+
+ private static final DCPermissions ALL = new DCPermissions()
+ {
+ public boolean canAccess(String dc)
+ {
+ return true;
+ }
+
+ public boolean restrictsAccess()
+ {
+ return false;
+ }
+
+ public Set<String> allowedDCs()
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ public String toString()
+ {
+ return "ALL";
+ }
+
+ public void validate()
+ {
+
+ }
+ };
+
+ private static final DCPermissions NONE = new DCPermissions()
+ {
+ public boolean canAccess(String dc)
+ {
+ return false;
+ }
+
+ public boolean restrictsAccess()
+ {
+ return true;
+ }
+
+ public Set<String> allowedDCs()
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ public String toString()
+ {
+ return "n/a";
+ }
+
+ public void validate()
+ {
+ throw new UnsupportedOperationException();
+ }
+ };
+
+ public static DCPermissions all()
+ {
+ return ALL;
+ }
+
+ public static DCPermissions none()
+ {
+ return NONE;
+ }
+
+ public static DCPermissions subset(Set<String> dcs)
+ {
+ return new SubsetPermissions(dcs);
+ }
+
+ public static DCPermissions subset(String... dcs)
+ {
+ return subset(Sets.newHashSet(dcs));
+ }
+
+ public static class Builder
+ {
+ private Set<String> dcs = new HashSet<>();
+ private boolean isAll = false;
+ private boolean modified = false;
+
+ public void add(String dc)
+ {
+ Preconditions.checkArgument(!isAll, "All has been set");
+ dcs.add(dc);
+ modified = true;
+ }
+
+ public void all()
+ {
+ Preconditions.checkArgument(dcs.isEmpty(), "DCs have already been set");
+ isAll = true;
+ modified = true;
+ }
+
+ public boolean isModified()
+ {
+ return modified;
+ }
+
+ public DCPermissions build()
+ {
+ if (dcs.isEmpty())
+ {
+ return DCPermissions.all();
+ }
+ else
+ {
+ return subset(dcs);
+ }
+ }
+ }
+
+ public static Builder builder()
+ {
+ return new Builder();
+ }
+}
diff --git a/src/java/org/apache/cassandra/auth/DataResource.java b/src/java/org/apache/cassandra/auth/DataResource.java
index 0aa24db..c3f5b32 100644
--- a/src/java/org/apache/cassandra/auth/DataResource.java
+++ b/src/java/org/apache/cassandra/auth/DataResource.java
@@ -23,7 +23,7 @@
import com.google.common.collect.Sets;
import org.apache.commons.lang3.StringUtils;
-import org.apache.cassandra.config.Schema;
+import org.apache.cassandra.schema.Schema;
/**
* The primary type of resource in Cassandra.
@@ -211,7 +211,7 @@
case KEYSPACE:
return Schema.instance.getKeyspaces().contains(keyspace);
case TABLE:
- return Schema.instance.getCFMetaData(keyspace, table) != null;
+ return Schema.instance.getTableMetadata(keyspace, table) != null;
}
throw new AssertionError();
}
diff --git a/src/java/org/apache/cassandra/auth/FunctionResource.java b/src/java/org/apache/cassandra/auth/FunctionResource.java
index 01a4de5..61c6a29 100644
--- a/src/java/org/apache/cassandra/auth/FunctionResource.java
+++ b/src/java/org/apache/cassandra/auth/FunctionResource.java
@@ -28,7 +28,7 @@
import com.google.common.collect.Sets;
import org.apache.commons.lang3.StringUtils;
-import org.apache.cassandra.config.Schema;
+import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.cql3.CQL3Type;
import org.apache.cassandra.cql3.functions.Function;
import org.apache.cassandra.cql3.functions.FunctionName;
@@ -134,6 +134,11 @@
return new FunctionResource(keyspace, name, argTypes);
}
+ public static FunctionResource function(Function function)
+ {
+ return new FunctionResource(function.name().keyspace, function.name().name, function.argTypes());
+ }
+
/**
* Creates a FunctionResource representing a specific, keyspace-scoped function.
* This variant is used to create an instance during parsing of a CQL statement.
@@ -150,13 +155,18 @@
if (keyspace == null)
throw new InvalidRequestException("In this context function name must be " +
"explictly qualified by a keyspace");
- List<AbstractType<?>> abstractTypes = new ArrayList<>();
+ List<AbstractType<?>> abstractTypes = new ArrayList<>(argTypes.size());
for (CQL3Type.Raw cqlType : argTypes)
abstractTypes.add(cqlType.prepare(keyspace).getType());
return new FunctionResource(keyspace, name, abstractTypes);
}
+ public static FunctionResource functionFromCql(FunctionName name, List<CQL3Type.Raw> argTypes)
+ {
+ return functionFromCql(name.keyspace, name.name, argTypes);
+ }
+
/**
* Parses a resource name into a FunctionResource instance.
*
diff --git a/src/java/org/apache/cassandra/auth/IAuthenticator.java b/src/java/org/apache/cassandra/auth/IAuthenticator.java
index ccbdb75..80ea719 100644
--- a/src/java/org/apache/cassandra/auth/IAuthenticator.java
+++ b/src/java/org/apache/cassandra/auth/IAuthenticator.java
@@ -21,6 +21,8 @@
import java.util.Map;
import java.util.Set;
+import javax.security.cert.X509Certificate;
+
import org.apache.cassandra.exceptions.AuthenticationException;
import org.apache.cassandra.exceptions.ConfigurationException;
@@ -65,12 +67,26 @@
SaslNegotiator newSaslNegotiator(InetAddress clientAddress);
/**
- * For implementations which support the Thrift login method that accepts arbitrary
- * key/value pairs containing credentials data.
- * Also used by CQL native protocol v1, in which username and password are sent from
- * client to server in a {@link org.apache.cassandra.transport.messages.CredentialsMessage}
- * Implementations where support for Thrift and CQL protocol v1 is not required should make
- * this an unsupported operation.
+ * Provide a SASL handler to perform authentication for an single connection. SASL
+ * is a stateful protocol, so a new instance must be used for each authentication
+ * attempt. This method accepts certificates as well. Authentication strategies can
+ * override this method to gain access to client's certificate chain, if present.
+ * @param clientAddress the IP address of the client whom we wish to authenticate, or null
+ * if an internal client (one not connected over the remote transport).
+ * @param certificates the peer's X509 Certificate chain, if present.
+ * @return org.apache.cassandra.auth.IAuthenticator.SaslNegotiator implementation
+ * (see {@link org.apache.cassandra.auth.PasswordAuthenticator.PlainTextSaslAuthenticator})
+ */
+ default SaslNegotiator newSaslNegotiator(InetAddress clientAddress, X509Certificate[] certificates)
+ {
+ return newSaslNegotiator(clientAddress);
+ }
+
+ /**
+ * A legacy method that is still used by JMX authentication.
+ *
+ * You should implement this for having JMX authentication through your
+ * authenticator.
*
* Should never return null - always throw AuthenticationException instead.
* Returning AuthenticatedUser.ANONYMOUS_USER is an option as well if authentication is not required.
@@ -89,7 +105,7 @@
public interface SaslNegotiator
{
/**
- * Evaluates the client response data and generates a byte[] reply which may be a further challenge or purely
+ * Evaluates the client response data and generates a byte[] response which may be a further challenge or purely
* informational in the case that the negotiation is completed on this round.
*
* This method is called each time a {@link org.apache.cassandra.transport.messages.AuthResponse} is received
diff --git a/src/java/org/apache/cassandra/auth/INetworkAuthorizer.java b/src/java/org/apache/cassandra/auth/INetworkAuthorizer.java
new file mode 100644
index 0000000..4582b5e
--- /dev/null
+++ b/src/java/org/apache/cassandra/auth/INetworkAuthorizer.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.auth;
+
+import org.apache.cassandra.exceptions.ConfigurationException;
+
+public interface INetworkAuthorizer
+{
+ /**
+ * Whether or not the authorizer will attempt authorization.
+ * If false the authorizer will not be called for authorization of resources.
+ */
+ default boolean requireAuthorization()
+ {
+ return true;
+ }
+
+ /**
+ * Setup is called once upon system startup to initialize the INetworkAuthorizer.
+ *
+ * For example, use this method to create any required keyspaces/column families.
+ */
+ void setup();
+
+ /**
+ * Returns the dc permissions associated with the given role
+ */
+ DCPermissions authorize(RoleResource role);
+
+ void setRoleDatacenters(RoleResource role, DCPermissions permissions);
+
+ /**
+ * Called when a role is deleted, so any corresponding network auth
+ * data can also be cleaned up
+ */
+ void drop(RoleResource role);
+
+ /**
+ * Validates configuration of IAuthorizer implementation (if configurable).
+ *
+ * @throws ConfigurationException when there is a configuration error.
+ */
+ void validateConfiguration() throws ConfigurationException;
+}
diff --git a/src/java/org/apache/cassandra/auth/IRoleManager.java b/src/java/org/apache/cassandra/auth/IRoleManager.java
index b27681d..1d47bee 100644
--- a/src/java/org/apache/cassandra/auth/IRoleManager.java
+++ b/src/java/org/apache/cassandra/auth/IRoleManager.java
@@ -19,6 +19,7 @@
import java.util.Map;
import java.util.Set;
+import java.util.stream.Collectors;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.exceptions.RequestExecutionException;
@@ -135,6 +136,23 @@
Set<RoleResource> getRoles(RoleResource grantee, boolean includeInherited) throws RequestValidationException, RequestExecutionException;
/**
+ * Used to retrieve detailed role info on the full set of roles granted to a grantee.
+ * This method was not part of the V1 IRoleManager API, so a default impl is supplied which uses the V1
+ * methods to retrieve the detailed role info for the grantee. This is essentially what clients of this interface
+ * would have to do themselves. Implementations can provide optimized versions of this method where the details
+ * can be retrieved more efficiently.
+ *
+ * @param grantee identifies the role whose granted roles are retrieved
+ * @return A set of Role objects detailing the roles granted to the grantee, either directly or through inheritance.
+ */
+ default Set<Role> getRoleDetails(RoleResource grantee)
+ {
+ return getRoles(grantee, true).stream()
+ .map(roleResource -> Roles.fromRoleResource(roleResource, this))
+ .collect(Collectors.toSet());
+ }
+
+ /**
* Called during the execution of an unqualified LIST ROLES query.
* Returns the total set of distinct roles in the system.
*
diff --git a/src/java/org/apache/cassandra/auth/NetworkAuthCache.java b/src/java/org/apache/cassandra/auth/NetworkAuthCache.java
new file mode 100644
index 0000000..6b3c74e
--- /dev/null
+++ b/src/java/org/apache/cassandra/auth/NetworkAuthCache.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.auth;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+
+public class NetworkAuthCache extends AuthCache<RoleResource, DCPermissions>
+{
+ public NetworkAuthCache(INetworkAuthorizer authorizer)
+ {
+ super("NetworkAuthCache",
+ DatabaseDescriptor::setRolesValidity,
+ DatabaseDescriptor::getRolesValidity,
+ DatabaseDescriptor::setRolesUpdateInterval,
+ DatabaseDescriptor::getRolesUpdateInterval,
+ DatabaseDescriptor::setRolesCacheMaxEntries,
+ DatabaseDescriptor::getRolesCacheMaxEntries,
+ authorizer::authorize,
+ () -> DatabaseDescriptor.getAuthenticator().requireAuthentication());
+ }
+}
diff --git a/src/java/org/apache/cassandra/auth/PasswordAuthenticator.java b/src/java/org/apache/cassandra/auth/PasswordAuthenticator.java
index 4bd3696..9da99a9 100644
--- a/src/java/org/apache/cassandra/auth/PasswordAuthenticator.java
+++ b/src/java/org/apache/cassandra/auth/PasswordAuthenticator.java
@@ -29,15 +29,14 @@
import org.slf4j.LoggerFactory;
import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.config.Schema;
-import org.apache.cassandra.config.SchemaConstants;
+import org.apache.cassandra.exceptions.RequestExecutionException;
+import org.apache.cassandra.schema.SchemaConstants;
import org.apache.cassandra.cql3.QueryOptions;
import org.apache.cassandra.cql3.QueryProcessor;
import org.apache.cassandra.cql3.UntypedResultSet;
import org.apache.cassandra.cql3.statements.SelectStatement;
import org.apache.cassandra.exceptions.AuthenticationException;
import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.exceptions.RequestExecutionException;
import org.apache.cassandra.service.ClientState;
import org.apache.cassandra.service.QueryState;
import org.apache.cassandra.transport.messages.ResultMessage;
@@ -69,9 +68,6 @@
static final byte NUL = 0;
private SelectStatement authenticateStatement;
- public static final String LEGACY_CREDENTIALS_TABLE = "credentials";
- private SelectStatement legacyAuthenticateStatement;
-
private CredentialsCache cache;
// No anonymous access.
@@ -103,17 +99,15 @@
return new AuthenticatedUser(username);
}
- private String queryHashedPassword(String username) throws AuthenticationException
+ private String queryHashedPassword(String username)
{
try
{
- SelectStatement authenticationStatement = authenticationStatement();
-
ResultMessage.Rows rows =
- authenticationStatement.execute(QueryState.forInternalCalls(),
- QueryOptions.forInternalCalls(consistencyForRole(username),
- Lists.newArrayList(ByteBufferUtil.bytes(username))),
- System.nanoTime());
+ authenticateStatement.execute(QueryState.forInternalCalls(),
+ QueryOptions.forInternalCalls(consistencyForRole(username),
+ Lists.newArrayList(ByteBufferUtil.bytes(username))),
+ System.nanoTime());
// If either a non-existent role name was supplied, or no credentials
// were found for that role we don't want to cache the result so we throw
@@ -133,25 +127,6 @@
}
}
- /**
- * If the legacy users table exists try to verify credentials there. This is to handle the case
- * where the cluster is being upgraded and so is running with mixed versions of the authn tables
- */
- private SelectStatement authenticationStatement()
- {
- if (Schema.instance.getCFMetaData(SchemaConstants.AUTH_KEYSPACE_NAME, LEGACY_CREDENTIALS_TABLE) == null)
- return authenticateStatement;
- else
- {
- // the statement got prepared, we to try preparing it again.
- // If the credentials was initialised only after statement got prepared, re-prepare (CASSANDRA-12813).
- if (legacyAuthenticateStatement == null)
- prepareLegacyAuthenticateStatement();
- return legacyAuthenticateStatement;
- }
- }
-
-
public Set<DataResource> protectedResources()
{
// Also protected by CassandraRoleManager, but the duplication doesn't hurt and is more explicit
@@ -170,21 +145,9 @@
AuthKeyspace.ROLES);
authenticateStatement = prepare(query);
- if (Schema.instance.getCFMetaData(SchemaConstants.AUTH_KEYSPACE_NAME, LEGACY_CREDENTIALS_TABLE) != null)
- prepareLegacyAuthenticateStatement();
-
cache = new CredentialsCache(this);
}
- private void prepareLegacyAuthenticateStatement()
- {
- String query = String.format("SELECT %s from %s.%s WHERE username = ?",
- SALTED_HASH,
- SchemaConstants.AUTH_KEYSPACE_NAME,
- LEGACY_CREDENTIALS_TABLE);
- legacyAuthenticateStatement = prepare(query);
- }
-
public AuthenticatedUser legacyAuthenticate(Map<String, String> credentials) throws AuthenticationException
{
String username = credentials.get(USERNAME_KEY);
@@ -205,7 +168,7 @@
private static SelectStatement prepare(String query)
{
- return (SelectStatement) QueryProcessor.getStatement(query, ClientState.forInternalCalls()).statement;
+ return (SelectStatement) QueryProcessor.getStatement(query, ClientState.forInternalCalls());
}
private class PlainTextSaslAuthenticator implements SaslNegotiator
diff --git a/src/java/org/apache/cassandra/auth/PermissionsCache.java b/src/java/org/apache/cassandra/auth/PermissionsCache.java
index 981ede8..a33f5d1 100644
--- a/src/java/org/apache/cassandra/auth/PermissionsCache.java
+++ b/src/java/org/apache/cassandra/auth/PermissionsCache.java
@@ -22,7 +22,7 @@
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.utils.Pair;
-public class PermissionsCache extends AuthCache<Pair<AuthenticatedUser, IResource>, Set<Permission>> implements PermissionsCacheMBean
+public class PermissionsCache extends AuthCache<Pair<AuthenticatedUser, IResource>, Set<Permission>>
{
public PermissionsCache(IAuthorizer authorizer)
{
diff --git a/src/java/org/apache/cassandra/auth/PermissionsCacheMBean.java b/src/java/org/apache/cassandra/auth/PermissionsCacheMBean.java
deleted file mode 100644
index d370d06..0000000
--- a/src/java/org/apache/cassandra/auth/PermissionsCacheMBean.java
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.auth;
-
-/**
- * Retained since CASSANDRA-7715 for backwards compatibility of MBean interface
- * classes. This should be removed in the next major version (4.0)
- */
-public interface PermissionsCacheMBean extends AuthCacheMBean
-{
-}
diff --git a/src/java/org/apache/cassandra/auth/Role.java b/src/java/org/apache/cassandra/auth/Role.java
new file mode 100644
index 0000000..e98cc7d
--- /dev/null
+++ b/src/java/org/apache/cassandra/auth/Role.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.auth;
+
+import java.util.Map;
+import java.util.Set;
+
+import com.google.common.base.Objects;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+
+public class Role
+{
+ /**
+ * Represents a user or group in the auth subsystem.
+ * Roles may be members of other roles, but circular graphs of roles are not permitted.
+ * The reason that memberOf is a Set<String> and not Set<Role> is to simplify loading
+ * for IRoleManager implementations (in particular, CassandraRoleManager)
+ */
+
+ public final RoleResource resource ;
+ public final boolean isSuper;
+ public final boolean canLogin;
+ public final Set<String> memberOf;
+ public final Map<String, String> options;
+
+ public Role(String name, boolean isSuperUser, boolean canLogin, Map<String, String> options, Set<String> memberOf)
+ {
+ this.resource = RoleResource.role(name);
+ this.isSuper = isSuperUser;
+ this.canLogin = canLogin;
+ this.memberOf = ImmutableSet.copyOf(memberOf);
+ this.options = ImmutableMap.copyOf(options);
+ }
+
+ public boolean equals(Object o)
+ {
+ if (this == o)
+ return true;
+
+ if (!(o instanceof Role))
+ return false;
+
+ Role r = (Role)o;
+ return Objects.equal(resource, r.resource)
+ && Objects.equal(isSuper, r.isSuper)
+ && Objects.equal(canLogin, r.canLogin)
+ && Objects.equal(memberOf, r.memberOf)
+ && Objects.equal(options, r.options);
+ }
+
+ public int hashCode()
+ {
+ return Objects.hashCode(resource, isSuper, canLogin, memberOf, options);
+ }
+}
diff --git a/src/java/org/apache/cassandra/auth/Roles.java b/src/java/org/apache/cassandra/auth/Roles.java
index 2b1ff6e..527451e 100644
--- a/src/java/org/apache/cassandra/auth/Roles.java
+++ b/src/java/org/apache/cassandra/auth/Roles.java
@@ -17,7 +17,13 @@
*/
package org.apache.cassandra.auth;
+import java.util.Collections;
+import java.util.Map;
import java.util.Set;
+import java.util.function.BooleanSupplier;
+import java.util.stream.Collectors;
+
+import com.google.common.annotations.VisibleForTesting;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -30,18 +36,56 @@
{
private static final Logger logger = LoggerFactory.getLogger(Roles.class);
- private static final RolesCache cache = new RolesCache(DatabaseDescriptor.getRoleManager());
+ private static final Role NO_ROLE = new Role("", false, false, Collections.emptyMap(), Collections.emptySet());
+
+ private static RolesCache cache;
+ static
+ {
+ initRolesCache(DatabaseDescriptor.getRoleManager(),
+ () -> DatabaseDescriptor.getAuthenticator().requireAuthentication());
+ }
+
+ @VisibleForTesting
+ public static void initRolesCache(IRoleManager roleManager, BooleanSupplier enableCache)
+ {
+ if (cache != null)
+ cache.unregisterMBean();
+ cache = new RolesCache(roleManager, enableCache);
+ }
+
+ @VisibleForTesting
+ public static void clearCache()
+ {
+ cache.invalidate();
+ }
/**
- * Get all roles granted to the supplied Role, including both directly granted
+ * Identify all roles granted to the supplied Role, including both directly granted
* and inherited roles.
- * The returned roles may be cached if {@code roles_validity_in_ms > 0}
+ * This method is used where we mainly just care about *which* roles are granted to a given role,
+ * including when looking up or listing permissions for a role on a given resource.
*
* @param primaryRole the Role
* @return set of all granted Roles for the primary Role
*/
public static Set<RoleResource> getRoles(RoleResource primaryRole)
{
+ return cache.getRoleResources(primaryRole);
+ }
+
+ /**
+ * Get detailed info on all the roles granted to the role identified by the supplied RoleResource.
+ * This includes superuser status and login privileges for the primary role and all roles granted directly
+ * to it or inherited.
+ * The returnred roles may be cached if roles_validity_in_ms > 0
+ * This method is used where we need to know specific attributes of the collection of granted roles, i.e.
+ * when checking for superuser status which may be inherited from *any* granted role.
+ *
+ * @param primaryRole identifies the role
+ * @return set of detailed info for all of the roles granted to the primary
+ */
+ public static Set<Role> getRoleDetails(RoleResource primaryRole)
+ {
return cache.getRoles(primaryRole);
}
@@ -56,10 +100,10 @@
{
try
{
- IRoleManager roleManager = DatabaseDescriptor.getRoleManager();
- for (RoleResource r : cache.getRoles(role))
- if (roleManager.isSuper(r))
+ for (Role r : getRoleDetails(role))
+ if (r.isSuper)
return true;
+
return false;
}
catch (RequestExecutionException e)
@@ -68,4 +112,88 @@
throw new UnauthorizedException("Unable to perform authorization of super-user permission: " + e.getMessage(), e);
}
}
+
+ /**
+ * Returns true if the supplied role has the login privilege. This cannot be inherited, so
+ * returns true iff the named role has that bit set.
+ * @param role the role identifier
+ * @return true if the role has the canLogin privilege, false otherwise
+ */
+ public static boolean canLogin(final RoleResource role)
+ {
+ try
+ {
+ for (Role r : getRoleDetails(role))
+ if (r.resource.equals(role))
+ return r.canLogin;
+
+ return false;
+ }
+ catch (RequestExecutionException e)
+ {
+ logger.debug("Failed to authorize {} for login permission", role.getRoleName());
+ throw new UnauthorizedException("Unable to perform authorization of login permission: " + e.getMessage(), e);
+ }
+ }
+
+ /**
+ * Returns the map of custom options for the named role. These options are not inherited from granted roles, but
+ * are set directly.
+ * @param role the role identifier
+ * @return map of option_name -> value. If no options are set for the named role, the map will be empty
+ * but never null.
+ */
+ public static Map<String, String> getOptions(RoleResource role)
+ {
+ for (Role r : getRoleDetails(role))
+ if (r.resource.equals(role))
+ return r.options;
+
+ return NO_ROLE.options;
+ }
+
+ /**
+ * Return the NullObject Role instance which can be safely used to indicate no information is available
+ * when querying for a specific named role.
+ * @return singleton null role object
+ */
+ public static Role nullRole()
+ {
+ return NO_ROLE;
+ }
+
+ /**
+ * Just a convenience method which compares a role instance with the null object version, indicating if the
+ * return from some query/lookup method was a valid Role or indicates that the role does not exist.
+ * @param role
+ * @return true if the supplied role is the null role instance, false otherwise.
+ */
+ public static boolean isNullRole(Role role)
+ {
+ return NO_ROLE.equals(role);
+ }
+
+
+ /**
+ * Constructs a Role object from a RoleResource, using the methods of the supplied IRoleManager.
+ * This is used by the default implementation of IRoleManager#getRoleDetails so that IRoleManager impls
+ * which don't implement an optimized getRoleDetails remain compatible. Depending on the IRoleManager
+ * implementation this could be quite heavyweight, so should not be used on any hot path.
+ *
+ * @param resource identifies the role
+ * @param roleManager provides lookup functions to retrieve role info
+ * @return Role object including superuser status, login privilege, custom options and the set of roles
+ * granted to identified role.
+ */
+ public static Role fromRoleResource(RoleResource resource, IRoleManager roleManager)
+ {
+ return new Role(resource.getName(),
+ roleManager.isSuper(resource),
+ roleManager.canLogin(resource),
+ roleManager.getCustomOptions(resource),
+ roleManager.getRoles(resource, false)
+ .stream()
+ .map(RoleResource::getRoleName)
+ .collect(Collectors.toSet()));
+ }
}
diff --git a/src/java/org/apache/cassandra/auth/RolesCache.java b/src/java/org/apache/cassandra/auth/RolesCache.java
index a02828a..d01de63 100644
--- a/src/java/org/apache/cassandra/auth/RolesCache.java
+++ b/src/java/org/apache/cassandra/auth/RolesCache.java
@@ -18,12 +18,14 @@
package org.apache.cassandra.auth;
import java.util.Set;
+import java.util.function.BooleanSupplier;
+import java.util.stream.Collectors;
import org.apache.cassandra.config.DatabaseDescriptor;
-public class RolesCache extends AuthCache<RoleResource, Set<RoleResource>> implements RolesCacheMBean
+public class RolesCache extends AuthCache<RoleResource, Set<Role>>
{
- public RolesCache(IRoleManager roleManager)
+ public RolesCache(IRoleManager roleManager, BooleanSupplier enableCache)
{
super("RolesCache",
DatabaseDescriptor::setRolesValidity,
@@ -32,12 +34,32 @@
DatabaseDescriptor::getRolesUpdateInterval,
DatabaseDescriptor::setRolesCacheMaxEntries,
DatabaseDescriptor::getRolesCacheMaxEntries,
- (r) -> roleManager.getRoles(r, true),
- () -> DatabaseDescriptor.getAuthenticator().requireAuthentication());
+ roleManager::getRoleDetails,
+ enableCache);
}
- public Set<RoleResource> getRoles(RoleResource role)
+ /**
+ * Read or return from the cache the Set of the RoleResources identifying the roles granted to the primary resource
+ * @see Roles#getRoles(RoleResource)
+ * @param primaryRole identifier for the primary role
+ * @return the set of identifiers of all the roles granted to (directly or through inheritance) the primary role
+ */
+ Set<RoleResource> getRoleResources(RoleResource primaryRole)
{
- return get(role);
+ return get(primaryRole).stream()
+ .map(r -> r.resource)
+ .collect(Collectors.toSet());
+ }
+
+ /**
+ * Read or return from cache the set of Role objects representing the roles granted to the primary resource
+ * @see Roles#getRoleDetails(RoleResource)
+ * @param primaryRole identifier for the primary role
+ * @return the set of Role objects containing info of all roles granted to (directly or through inheritance)
+ * the primary role.
+ */
+ Set<Role> getRoles(RoleResource primaryRole)
+ {
+ return get(primaryRole);
}
}
diff --git a/src/java/org/apache/cassandra/auth/RolesCacheMBean.java b/src/java/org/apache/cassandra/auth/RolesCacheMBean.java
deleted file mode 100644
index 06482d7..0000000
--- a/src/java/org/apache/cassandra/auth/RolesCacheMBean.java
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.auth;
-
-/**
- * Retained since CASSANDRA-7715 for backwards compatibility of MBean interface
- * classes. This should be removed in the next major version (4.0)
- */
-public interface RolesCacheMBean extends AuthCacheMBean
-{
-}
diff --git a/src/java/org/apache/cassandra/auth/jmx/AuthorizationProxy.java b/src/java/org/apache/cassandra/auth/jmx/AuthorizationProxy.java
index ebc1763..68cff0c 100644
--- a/src/java/org/apache/cassandra/auth/jmx/AuthorizationProxy.java
+++ b/src/java/org/apache/cassandra/auth/jmx/AuthorizationProxy.java
@@ -23,8 +23,9 @@
import java.security.AccessController;
import java.security.Principal;
import java.util.Set;
+import java.util.function.BooleanSupplier;
import java.util.function.Function;
-import java.util.function.Supplier;
+import java.util.function.Predicate;
import java.util.stream.Collectors;
import javax.management.MBeanServer;
import javax.management.MalformedObjectNameException;
@@ -51,7 +52,7 @@
*
* Because an ObjectName may contain wildcards, meaning it represents a set of individual MBeans,
* JMX resources don't fit well with the hierarchical approach modelled by other IResource
- * implementations and utilised by ClientState::ensureHasPermission etc. To enable grants to use
+ * implementations and utilised by ClientState::ensurePermission etc. To enable grants to use
* pattern-type ObjectNames, this class performs its own custom matching and filtering of resources
* rather than pushing that down to the configured IAuthorizer. To that end, during authorization
* it pulls back all permissions for the active subject, filtering them to retain only grants on
@@ -109,7 +110,7 @@
Used to check whether the Role associated with the authenticated Subject has superuser
status. By default, just delegates to Roles::hasSuperuserStatus, but can be overridden for testing.
*/
- protected Function<RoleResource, Boolean> isSuperuser = Roles::hasSuperuserStatus;
+ protected Predicate<RoleResource> isSuperuser = Roles::hasSuperuserStatus;
/*
Used to retrieve the set of all permissions granted to a given role. By default, this fetches
@@ -122,7 +123,7 @@
Used to decide whether authorization is enabled or not, usually this depends on the configured
IAuthorizer, but can be overridden for testing.
*/
- protected Supplier<Boolean> isAuthzRequired = () -> DatabaseDescriptor.getAuthorizer().requireAuthorization();
+ protected BooleanSupplier isAuthzRequired = () -> DatabaseDescriptor.getAuthorizer().requireAuthorization();
/*
Used to find matching MBeans when the invocation target is a pattern type ObjectName.
@@ -134,7 +135,7 @@
Used to determine whether auth setup has completed so we know whether the expect the IAuthorizer
to be ready. Can be overridden for testing.
*/
- protected Supplier<Boolean> isAuthSetupComplete = () -> StorageService.instance.isAuthSetupComplete();
+ protected BooleanSupplier isAuthSetupComplete = () -> StorageService.instance.isAuthSetupComplete();
@Override
public Object invoke(Object proxy, Method method, Object[] args)
@@ -187,14 +188,14 @@
methodName,
subject == null ? "" :subject.toString().replaceAll("\\n", " "));
- if (!isAuthSetupComplete.get())
+ if (!isAuthSetupComplete.getAsBoolean())
{
logger.trace("Auth setup is not complete, refusing access");
return false;
}
// Permissive authorization is enabled
- if (!isAuthzRequired.get())
+ if (!isAuthzRequired.getAsBoolean())
return true;
// Allow operations performed locally on behalf of the connector server itself
@@ -219,7 +220,7 @@
// might choose to associate with the Subject following successful authentication
RoleResource userResource = RoleResource.role(principals.iterator().next().getName());
// A role with superuser status can do anything
- if (isSuperuser.apply(userResource))
+ if (isSuperuser.test(userResource))
return true;
// The method being invoked may be a method on an MBean, or it could belong
diff --git a/src/java/org/apache/cassandra/batchlog/Batch.java b/src/java/org/apache/cassandra/batchlog/Batch.java
index e91e3ca..fb6c5d5 100644
--- a/src/java/org/apache/cassandra/batchlog/Batch.java
+++ b/src/java/org/apache/cassandra/batchlog/Batch.java
@@ -90,7 +90,7 @@
size += sizeofUnsignedVInt(batch.decodedMutations.size());
for (Mutation mutation : batch.decodedMutations)
{
- int mutationSize = (int) Mutation.serializer.serializedSize(mutation, version);
+ int mutationSize = mutation.serializedSize(version);
size += sizeofUnsignedVInt(mutationSize);
size += mutationSize;
}
@@ -108,7 +108,7 @@
out.writeUnsignedVInt(batch.decodedMutations.size());
for (Mutation mutation : batch.decodedMutations)
{
- out.writeUnsignedVInt(Mutation.serializer.serializedSize(mutation, version));
+ out.writeUnsignedVInt(mutation.serializedSize(version));
Mutation.serializer.serialize(mutation, out, version);
}
}
diff --git a/src/java/org/apache/cassandra/batchlog/BatchRemoveVerbHandler.java b/src/java/org/apache/cassandra/batchlog/BatchRemoveVerbHandler.java
index 3c3fcec..3443cab 100644
--- a/src/java/org/apache/cassandra/batchlog/BatchRemoveVerbHandler.java
+++ b/src/java/org/apache/cassandra/batchlog/BatchRemoveVerbHandler.java
@@ -20,11 +20,13 @@
import java.util.UUID;
import org.apache.cassandra.net.IVerbHandler;
-import org.apache.cassandra.net.MessageIn;
+import org.apache.cassandra.net.Message;
public final class BatchRemoveVerbHandler implements IVerbHandler<UUID>
{
- public void doVerb(MessageIn<UUID> message, int id)
+ public static final BatchRemoveVerbHandler instance = new BatchRemoveVerbHandler();
+
+ public void doVerb(Message<UUID> message)
{
BatchlogManager.remove(message.payload);
}
diff --git a/src/java/org/apache/cassandra/batchlog/BatchStoreVerbHandler.java b/src/java/org/apache/cassandra/batchlog/BatchStoreVerbHandler.java
index 4bc878c..77335cb 100644
--- a/src/java/org/apache/cassandra/batchlog/BatchStoreVerbHandler.java
+++ b/src/java/org/apache/cassandra/batchlog/BatchStoreVerbHandler.java
@@ -17,16 +17,17 @@
*/
package org.apache.cassandra.batchlog;
-import org.apache.cassandra.db.WriteResponse;
import org.apache.cassandra.net.IVerbHandler;
-import org.apache.cassandra.net.MessageIn;
+import org.apache.cassandra.net.Message;
import org.apache.cassandra.net.MessagingService;
public final class BatchStoreVerbHandler implements IVerbHandler<Batch>
{
- public void doVerb(MessageIn<Batch> message, int id)
+ public static final BatchStoreVerbHandler instance = new BatchStoreVerbHandler();
+
+ public void doVerb(Message<Batch> message)
{
BatchlogManager.store(message.payload);
- MessagingService.instance().sendReply(WriteResponse.createMessage(), id, message.from);
+ MessagingService.instance().send(message.emptyResponse(), message.from());
}
}
diff --git a/src/java/org/apache/cassandra/batchlog/BatchlogManager.java b/src/java/org/apache/cassandra/batchlog/BatchlogManager.java
index 3c0ad56..f140332 100644
--- a/src/java/org/apache/cassandra/batchlog/BatchlogManager.java
+++ b/src/java/org/apache/cassandra/batchlog/BatchlogManager.java
@@ -18,14 +18,12 @@
package org.apache.cassandra.batchlog;
import java.io.IOException;
-import java.net.InetAddress;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
-import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
@@ -33,25 +31,21 @@
import java.util.concurrent.Future;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledThreadPoolExecutor;
-import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.Collections2;
import com.google.common.collect.Iterables;
-import com.google.common.collect.ListMultimap;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Multimap;
import com.google.common.util.concurrent.RateLimiter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.concurrent.DebuggableScheduledThreadPoolExecutor;
import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.config.SchemaConstants;
import org.apache.cassandra.cql3.UntypedResultSet;
import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.ConsistencyLevel;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.db.Mutation;
import org.apache.cassandra.db.SystemKeyspace;
@@ -67,9 +61,16 @@
import org.apache.cassandra.hints.HintsService;
import org.apache.cassandra.io.util.DataInputBuffer;
import org.apache.cassandra.io.util.DataOutputBuffer;
-import org.apache.cassandra.net.MessageIn;
-import org.apache.cassandra.net.MessageOut;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.Replica;
+import org.apache.cassandra.locator.ReplicaLayout;
+import org.apache.cassandra.locator.ReplicaPlan;
+import org.apache.cassandra.locator.Replicas;
+import org.apache.cassandra.net.Message;
+import org.apache.cassandra.net.MessageFlag;
import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.schema.SchemaConstants;
+import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.service.WriteResponseHandler;
import org.apache.cassandra.utils.ExecutorUtils;
@@ -78,8 +79,10 @@
import org.apache.cassandra.utils.UUIDGen;
import static com.google.common.collect.Iterables.transform;
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
import static org.apache.cassandra.cql3.QueryProcessor.executeInternal;
import static org.apache.cassandra.cql3.QueryProcessor.executeInternalWithPaging;
+import static org.apache.cassandra.net.Verb.MUTATION_REQ;
public class BatchlogManager implements BatchlogManagerMBean
{
@@ -89,7 +92,7 @@
private static final Logger logger = LoggerFactory.getLogger(BatchlogManager.class);
public static final BatchlogManager instance = new BatchlogManager();
- public static final long BATCHLOG_REPLAY_TIMEOUT = Long.getLong("cassandra.batchlog.replay_timeout_in_ms", DatabaseDescriptor.getWriteRpcTimeout() * 2);
+ public static final long BATCHLOG_REPLAY_TIMEOUT = Long.getLong("cassandra.batchlog.replay_timeout_in_ms", DatabaseDescriptor.getWriteRpcTimeout(MILLISECONDS) * 2);
private volatile long totalBatchesReplayed = 0; // no concurrency protection necessary as only written by replay thread.
private volatile UUID lastReplayedUuid = UUIDGen.minTimeUUID(0);
@@ -97,6 +100,8 @@
// Single-thread executor service for scheduling and serializing log replay.
private final ScheduledExecutorService batchlogTasks;
+ private final RateLimiter rateLimiter = RateLimiter.create(Double.MAX_VALUE);
+
public BatchlogManager()
{
ScheduledThreadPoolExecutor executor = new DebuggableScheduledThreadPoolExecutor("BatchlogTasks");
@@ -111,7 +116,7 @@
batchlogTasks.scheduleWithFixedDelay(this::replayFailedBatches,
StorageService.RING_DELAY,
REPLAY_INTERVAL,
- TimeUnit.MILLISECONDS);
+ MILLISECONDS);
}
public void shutdownAndWait(long timeout, TimeUnit unit) throws InterruptedException, TimeoutException
@@ -200,14 +205,13 @@
// rate limit is in bytes per second. Uses Double.MAX_VALUE if disabled (set to 0 in cassandra.yaml).
// max rate is scaled by the number of nodes in the cluster (same as for HHOM - see CASSANDRA-5272).
- int endpointsCount = StorageService.instance.getTokenMetadata().getAllEndpoints().size();
+ int endpointsCount = StorageService.instance.getTokenMetadata().getSizeOfAllEndpoints();
if (endpointsCount <= 0)
{
logger.trace("Replay cancelled as there are no peers in the ring.");
return;
}
- int throttleInKB = DatabaseDescriptor.getBatchlogReplayThrottleInKB() / endpointsCount;
- RateLimiter rateLimiter = RateLimiter.create(throttleInKB == 0 ? Double.MAX_VALUE : throttleInKB * 1024);
+ setRate(DatabaseDescriptor.getBatchlogReplayThrottleInKB());
UUID limitUuid = UUIDGen.maxTimeUUID(System.currentTimeMillis() - getBatchlogTimeout());
ColumnFamilyStore store = Keyspace.open(SchemaConstants.SYSTEM_KEYSPACE_NAME).getColumnFamilyStore(SystemKeyspace.BATCHES);
@@ -224,6 +228,27 @@
logger.trace("Finished replayFailedBatches");
}
+ /**
+ * Sets the rate for the current rate limiter. When {@code throttleInKB} is 0, this sets the rate to
+ * {@link Double#MAX_VALUE} bytes per second.
+ *
+ * @param throttleInKB throughput to set in KB per second
+ */
+ public void setRate(final int throttleInKB)
+ {
+ int endpointsCount = StorageService.instance.getTokenMetadata().getSizeOfAllEndpoints();
+ if (endpointsCount > 0)
+ {
+ int endpointThrottleInKB = throttleInKB / endpointsCount;
+ double throughput = endpointThrottleInKB == 0 ? Double.MAX_VALUE : endpointThrottleInKB * 1024.0;
+ if (rateLimiter.getRate() != throughput)
+ {
+ logger.debug("Updating batchlog replay throttle to {} KB/s, {} KB/s per endpoint", throttleInKB, endpointThrottleInKB);
+ rateLimiter.setRate(throughput);
+ }
+ }
+ }
+
// read less rows (batches) per page if they are very large
static int calculatePageSize(ColumnFamilyStore store)
{
@@ -239,8 +264,10 @@
int positionInPage = 0;
ArrayList<ReplayingBatch> unfinishedBatches = new ArrayList<>(pageSize);
- Set<InetAddress> hintedNodes = new HashSet<>();
+ Set<InetAddressAndPort> hintedNodes = new HashSet<>();
Set<UUID> replayedBatches = new HashSet<>();
+ Exception caughtException = null;
+ int skipped = 0;
// Sending out batches for replay without waiting for them, so that one stuck batch doesn't affect others
for (UntypedResultSet.Row row : batches)
@@ -262,8 +289,10 @@
}
catch (IOException e)
{
- logger.warn("Skipped batch replay of {} due to {}", id, e);
+ logger.warn("Skipped batch replay of {} due to {}", id, e.getMessage());
+ caughtException = e;
remove(id);
+ ++skipped;
}
if (++positionInPage == pageSize)
@@ -277,6 +306,9 @@
finishAndClearBatches(unfinishedBatches, hintedNodes, replayedBatches);
+ if (caughtException != null)
+ logger.warn(String.format("Encountered %d unexpected exceptions while sending out batches", skipped), caughtException);
+
// to preserve batch guarantees, we must ensure that hints (if any) have made it to disk, before deleting the batches
HintsService.instance.flushAndFsyncBlockingly(transform(hintedNodes, StorageService.instance::getHostIdForEndpoint));
@@ -284,7 +316,7 @@
replayedBatches.forEach(BatchlogManager::remove);
}
- private void finishAndClearBatches(ArrayList<ReplayingBatch> batches, Set<InetAddress> hintedNodes, Set<UUID> replayedBatches)
+ private void finishAndClearBatches(ArrayList<ReplayingBatch> batches, Set<InetAddressAndPort> hintedNodes, Set<UUID> replayedBatches)
{
// schedule hints for timed out deliveries
for (ReplayingBatch batch : batches)
@@ -319,7 +351,7 @@
this.replayedBytes = addMutations(version, serializedMutations);
}
- public int replay(RateLimiter rateLimiter, Set<InetAddress> hintedNodes) throws IOException
+ public int replay(RateLimiter rateLimiter, Set<InetAddressAndPort> hintedNodes) throws IOException
{
logger.trace("Replaying batch {}", id);
@@ -327,7 +359,7 @@
return 0;
int gcgs = gcgs(mutations);
- if (TimeUnit.MILLISECONDS.toSeconds(writtenAt) + gcgs <= FBUtilities.nowInSeconds())
+ if (MILLISECONDS.toSeconds(writtenAt) + gcgs <= FBUtilities.nowInSeconds())
return 0;
replayHandlers = sendReplays(mutations, writtenAt, hintedNodes);
@@ -337,7 +369,7 @@
return replayHandlers.size();
}
- public void finish(Set<InetAddress> hintedNodes)
+ public void finish(Set<InetAddressAndPort> hintedNodes)
{
for (int i = 0; i < replayHandlers.size(); i++)
{
@@ -377,20 +409,20 @@
// truncated.
private void addMutation(Mutation mutation)
{
- for (UUID cfId : mutation.getColumnFamilyIds())
- if (writtenAt <= SystemKeyspace.getTruncatedAt(cfId))
- mutation = mutation.without(cfId);
+ for (TableId tableId : mutation.getTableIds())
+ if (writtenAt <= SystemKeyspace.getTruncatedAt(tableId))
+ mutation = mutation.without(tableId);
if (!mutation.isEmpty())
mutations.add(mutation);
}
- private void writeHintsForUndeliveredEndpoints(int startFrom, Set<InetAddress> hintedNodes)
+ private void writeHintsForUndeliveredEndpoints(int startFrom, Set<InetAddressAndPort> hintedNodes)
{
int gcgs = gcgs(mutations);
// expired
- if (TimeUnit.MILLISECONDS.toSeconds(writtenAt) + gcgs <= FBUtilities.nowInSeconds())
+ if (MILLISECONDS.toSeconds(writtenAt) + gcgs <= FBUtilities.nowInSeconds())
return;
for (int i = startFrom; i < replayHandlers.size(); i++)
@@ -401,7 +433,7 @@
if (handler != null)
{
hintedNodes.addAll(handler.undelivered);
- HintsService.instance.write(transform(handler.undelivered, StorageService.instance::getHostIdForEndpoint),
+ HintsService.instance.write(Collections2.transform(handler.undelivered, StorageService.instance::getHostIdForEndpoint),
Hint.create(undeliveredMutation, writtenAt));
}
}
@@ -409,7 +441,7 @@
private static List<ReplayWriteResponseHandler<Mutation>> sendReplays(List<Mutation> mutations,
long writtenAt,
- Set<InetAddress> hintedNodes)
+ Set<InetAddressAndPort> hintedNodes)
{
List<ReplayWriteResponseHandler<Mutation>> handlers = new ArrayList<>(mutations.size());
for (Mutation mutation : mutations)
@@ -429,37 +461,41 @@
*/
private static ReplayWriteResponseHandler<Mutation> sendSingleReplayMutation(final Mutation mutation,
long writtenAt,
- Set<InetAddress> hintedNodes)
+ Set<InetAddressAndPort> hintedNodes)
{
- Set<InetAddress> liveEndpoints = new HashSet<>();
String ks = mutation.getKeyspaceName();
+ Keyspace keyspace = Keyspace.open(ks);
Token tk = mutation.key().getToken();
- for (InetAddress endpoint : StorageService.instance.getNaturalAndPendingEndpoints(ks, tk))
+ // TODO: this logic could do with revisiting at some point, as it is unclear what its rationale is
+ // we perform a local write, ignoring errors and inline in this thread (potentially slowing replay down)
+ // effectively bumping CL for locally owned writes and also potentially stalling log replay if an error occurs
+ // once we decide how it should work, it can also probably be simplified, and avoid constructing a ReplicaPlan directly
+ ReplicaLayout.ForTokenWrite liveAndDown = ReplicaLayout.forTokenWriteLiveAndDown(keyspace, tk);
+ Replicas.temporaryAssertFull(liveAndDown.all()); // TODO in CASSANDRA-14549
+
+ Replica selfReplica = liveAndDown.all().selfIfPresent();
+ if (selfReplica != null)
+ mutation.apply();
+
+ ReplicaLayout.ForTokenWrite liveRemoteOnly = liveAndDown.filter(
+ r -> FailureDetector.isReplicaAlive.test(r) && r != selfReplica);
+
+ for (Replica replica : liveAndDown.all())
{
- if (endpoint.equals(FBUtilities.getBroadcastAddress()))
- {
- mutation.apply();
- }
- else if (FailureDetector.instance.isAlive(endpoint))
- {
- liveEndpoints.add(endpoint); // will try delivering directly instead of writing a hint.
- }
- else
- {
- hintedNodes.add(endpoint);
- HintsService.instance.write(StorageService.instance.getHostIdForEndpoint(endpoint),
- Hint.create(mutation, writtenAt));
- }
+ if (replica == selfReplica || liveRemoteOnly.all().contains(replica))
+ continue;
+ hintedNodes.add(replica.endpoint());
+ HintsService.instance.write(StorageService.instance.getHostIdForEndpoint(replica.endpoint()),
+ Hint.create(mutation, writtenAt));
}
- if (liveEndpoints.isEmpty())
- return null;
-
- ReplayWriteResponseHandler<Mutation> handler = new ReplayWriteResponseHandler<>(liveEndpoints, System.nanoTime());
- MessageOut<Mutation> message = mutation.createMessage();
- for (InetAddress endpoint : liveEndpoints)
- MessagingService.instance().sendRR(message, endpoint, handler, false);
+ ReplicaPlan.ForTokenWrite replicaPlan = new ReplicaPlan.ForTokenWrite(keyspace, ConsistencyLevel.ONE,
+ liveRemoteOnly.pending(), liveRemoteOnly.all(), liveRemoteOnly.all(), liveRemoteOnly.all());
+ ReplayWriteResponseHandler<Mutation> handler = new ReplayWriteResponseHandler<>(replicaPlan, System.nanoTime());
+ Message<Mutation> message = Message.outWithFlag(MUTATION_REQ, mutation, MessageFlag.CALL_BACK_ON_FAILURE);
+ for (Replica replica : liveRemoteOnly.all())
+ MessagingService.instance().sendWriteWithCallback(message, replica, handler, false);
return handler;
}
@@ -473,120 +509,31 @@
/**
* A wrapper of WriteResponseHandler that stores the addresses of the endpoints from
- * which we did not receive a successful reply.
+ * which we did not receive a successful response.
*/
private static class ReplayWriteResponseHandler<T> extends WriteResponseHandler<T>
{
- private final Set<InetAddress> undelivered = Collections.newSetFromMap(new ConcurrentHashMap<>());
+ private final Set<InetAddressAndPort> undelivered = Collections.newSetFromMap(new ConcurrentHashMap<>());
- ReplayWriteResponseHandler(Collection<InetAddress> writeEndpoints, long queryStartNanoTime)
+ ReplayWriteResponseHandler(ReplicaPlan.ForTokenWrite replicaPlan, long queryStartNanoTime)
{
- super(writeEndpoints, Collections.<InetAddress>emptySet(), null, null, null, WriteType.UNLOGGED_BATCH, queryStartNanoTime);
- undelivered.addAll(writeEndpoints);
+ super(replicaPlan, null, WriteType.UNLOGGED_BATCH, queryStartNanoTime);
+ Iterables.addAll(undelivered, replicaPlan.contacts().endpoints());
}
@Override
- protected int totalBlockFor()
+ protected int blockFor()
{
- return this.naturalEndpoints.size();
+ return this.replicaPlan.contacts().size();
}
@Override
- public void response(MessageIn<T> m)
+ public void onResponse(Message<T> m)
{
- boolean removed = undelivered.remove(m == null ? FBUtilities.getBroadcastAddress() : m.from);
+ boolean removed = undelivered.remove(m == null ? FBUtilities.getBroadcastAddressAndPort() : m.from());
assert removed;
- super.response(m);
+ super.onResponse(m);
}
}
}
-
- public static class EndpointFilter
- {
- private final String localRack;
- private final Multimap<String, InetAddress> endpoints;
-
- public EndpointFilter(String localRack, Multimap<String, InetAddress> endpoints)
- {
- this.localRack = localRack;
- this.endpoints = endpoints;
- }
-
- /**
- * @return list of candidates for batchlog hosting. If possible these will be two nodes from different racks.
- */
- public Collection<InetAddress> filter()
- {
- // special case for single-node data centers
- if (endpoints.values().size() == 1)
- return endpoints.values();
-
- // strip out dead endpoints and localhost
- ListMultimap<String, InetAddress> validated = ArrayListMultimap.create();
- for (Map.Entry<String, InetAddress> entry : endpoints.entries())
- if (isValid(entry.getValue()))
- validated.put(entry.getKey(), entry.getValue());
-
- if (validated.size() <= 2)
- return validated.values();
-
- if (validated.size() - validated.get(localRack).size() >= 2)
- {
- // we have enough endpoints in other racks
- validated.removeAll(localRack);
- }
-
- if (validated.keySet().size() == 1)
- {
- /*
- * we have only 1 `other` rack to select replicas from (whether it be the local rack or a single non-local rack)
- * pick two random nodes from there; we are guaranteed to have at least two nodes in the single remaining rack
- * because of the preceding if block.
- */
- List<InetAddress> otherRack = Lists.newArrayList(validated.values());
- shuffle(otherRack);
- return otherRack.subList(0, 2);
- }
-
- // randomize which racks we pick from if more than 2 remaining
- Collection<String> racks;
- if (validated.keySet().size() == 2)
- {
- racks = validated.keySet();
- }
- else
- {
- racks = Lists.newArrayList(validated.keySet());
- shuffle((List<String>) racks);
- }
-
- // grab a random member of up to two racks
- List<InetAddress> result = new ArrayList<>(2);
- for (String rack : Iterables.limit(racks, 2))
- {
- List<InetAddress> rackMembers = validated.get(rack);
- result.add(rackMembers.get(getRandomInt(rackMembers.size())));
- }
-
- return result;
- }
-
- @VisibleForTesting
- protected boolean isValid(InetAddress input)
- {
- return !input.equals(FBUtilities.getBroadcastAddress()) && FailureDetector.instance.isAlive(input);
- }
-
- @VisibleForTesting
- protected int getRandomInt(int bound)
- {
- return ThreadLocalRandom.current().nextInt(bound);
- }
-
- @VisibleForTesting
- protected void shuffle(List<?> list)
- {
- Collections.shuffle(list);
- }
- }
}
diff --git a/src/java/org/apache/cassandra/batchlog/LegacyBatchlogMigrator.java b/src/java/org/apache/cassandra/batchlog/LegacyBatchlogMigrator.java
deleted file mode 100644
index 4592488..0000000
--- a/src/java/org/apache/cassandra/batchlog/LegacyBatchlogMigrator.java
+++ /dev/null
@@ -1,199 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.batchlog;
-
-import java.io.IOException;
-import java.net.InetAddress;
-import java.nio.ByteBuffer;
-import java.util.*;
-import java.util.concurrent.TimeUnit;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.cassandra.config.SchemaConstants;
-import org.apache.cassandra.cql3.QueryProcessor;
-import org.apache.cassandra.cql3.UntypedResultSet;
-import org.apache.cassandra.db.*;
-import org.apache.cassandra.db.marshal.UUIDType;
-import org.apache.cassandra.db.partitions.PartitionUpdate;
-import org.apache.cassandra.exceptions.WriteFailureException;
-import org.apache.cassandra.exceptions.WriteTimeoutException;
-import org.apache.cassandra.io.util.DataInputBuffer;
-import org.apache.cassandra.io.util.DataOutputBuffer;
-import org.apache.cassandra.net.MessagingService;
-import org.apache.cassandra.service.AbstractWriteResponseHandler;
-import org.apache.cassandra.service.WriteResponseHandler;
-import org.apache.cassandra.utils.FBUtilities;
-import org.apache.cassandra.utils.UUIDGen;
-
-public final class LegacyBatchlogMigrator
-{
- private static final Logger logger = LoggerFactory.getLogger(LegacyBatchlogMigrator.class);
-
- private LegacyBatchlogMigrator()
- {
- // static class
- }
-
- @SuppressWarnings("deprecation")
- public static void migrate()
- {
- ColumnFamilyStore store = Keyspace.open(SchemaConstants.SYSTEM_KEYSPACE_NAME).getColumnFamilyStore(SystemKeyspace.LEGACY_BATCHLOG);
-
- // nothing to migrate
- if (store.isEmpty())
- return;
-
- logger.info("Migrating legacy batchlog to new storage");
-
- int convertedBatches = 0;
- String query = String.format("SELECT id, data, written_at, version FROM %s.%s",
- SchemaConstants.SYSTEM_KEYSPACE_NAME,
- SystemKeyspace.LEGACY_BATCHLOG);
-
- int pageSize = BatchlogManager.calculatePageSize(store);
-
- UntypedResultSet rows = QueryProcessor.executeInternalWithPaging(query, pageSize);
- for (UntypedResultSet.Row row : rows)
- {
- if (apply(row, convertedBatches))
- convertedBatches++;
- }
-
- if (convertedBatches > 0)
- Keyspace.openAndGetStore(SystemKeyspace.LegacyBatchlog).truncateBlocking();
- }
-
- @SuppressWarnings("deprecation")
- public static boolean isLegacyBatchlogMutation(Mutation mutation)
- {
- return mutation.getKeyspaceName().equals(SchemaConstants.SYSTEM_KEYSPACE_NAME)
- && mutation.getPartitionUpdate(SystemKeyspace.LegacyBatchlog.cfId) != null;
- }
-
- @SuppressWarnings("deprecation")
- public static void handleLegacyMutation(Mutation mutation)
- {
- PartitionUpdate update = mutation.getPartitionUpdate(SystemKeyspace.LegacyBatchlog.cfId);
- logger.trace("Applying legacy batchlog mutation {}", update);
- update.forEach(row -> apply(UntypedResultSet.Row.fromInternalRow(update.metadata(), update.partitionKey(), row), -1));
- }
-
- private static boolean apply(UntypedResultSet.Row row, long counter)
- {
- UUID id = row.getUUID("id");
- long timestamp = id.version() == 1 ? UUIDGen.unixTimestamp(id) : row.getLong("written_at");
- int version = row.has("version") ? row.getInt("version") : MessagingService.VERSION_12;
-
- if (id.version() != 1)
- id = UUIDGen.getTimeUUID(timestamp, counter);
-
- logger.trace("Converting mutation at {}", timestamp);
-
- try (DataInputBuffer in = new DataInputBuffer(row.getBytes("data"), false))
- {
- int numMutations = in.readInt();
- List<Mutation> mutations = new ArrayList<>(numMutations);
- for (int i = 0; i < numMutations; i++)
- mutations.add(Mutation.serializer.deserialize(in, version));
-
- BatchlogManager.store(Batch.createLocal(id, TimeUnit.MILLISECONDS.toMicros(timestamp), mutations));
- return true;
- }
- catch (Throwable t)
- {
- logger.error("Failed to convert mutation {} at timestamp {}", id, timestamp, t);
- return false;
- }
- }
-
- public static void syncWriteToBatchlog(WriteResponseHandler<?> handler, Batch batch, Collection<InetAddress> endpoints)
- throws WriteTimeoutException, WriteFailureException
- {
- for (InetAddress target : endpoints)
- {
- logger.trace("Sending legacy batchlog store request {} to {} for {} mutations", batch.id, target, batch.size());
-
- int targetVersion = MessagingService.instance().getVersion(target);
- MessagingService.instance().sendRR(getStoreMutation(batch, targetVersion).createMessage(MessagingService.Verb.MUTATION),
- target,
- handler,
- false);
- }
- }
-
- public static void asyncRemoveFromBatchlog(Collection<InetAddress> endpoints, UUID uuid, long queryStartNanoTime)
- {
- AbstractWriteResponseHandler<IMutation> handler = new WriteResponseHandler<>(endpoints,
- Collections.<InetAddress>emptyList(),
- ConsistencyLevel.ANY,
- Keyspace.open(SchemaConstants.SYSTEM_KEYSPACE_NAME),
- null,
- WriteType.SIMPLE,
- queryStartNanoTime);
- Mutation mutation = getRemoveMutation(uuid);
-
- for (InetAddress target : endpoints)
- {
- logger.trace("Sending legacy batchlog remove request {} to {}", uuid, target);
- MessagingService.instance().sendRR(mutation.createMessage(MessagingService.Verb.MUTATION), target, handler, false);
- }
- }
-
- static void store(Batch batch, int version)
- {
- getStoreMutation(batch, version).apply();
- }
-
- @SuppressWarnings("deprecation")
- static Mutation getStoreMutation(Batch batch, int version)
- {
- PartitionUpdate.SimpleBuilder builder = PartitionUpdate.simpleBuilder(SystemKeyspace.LegacyBatchlog, batch.id);
- builder.row()
- .timestamp(batch.creationTime)
- .add("written_at", new Date(batch.creationTime / 1000))
- .add("data", getSerializedMutations(version, batch.decodedMutations))
- .add("version", version);
- return builder.buildAsMutation();
- }
-
- @SuppressWarnings("deprecation")
- private static Mutation getRemoveMutation(UUID uuid)
- {
- return new Mutation(PartitionUpdate.fullPartitionDelete(SystemKeyspace.LegacyBatchlog,
- UUIDType.instance.decompose(uuid),
- FBUtilities.timestampMicros(),
- FBUtilities.nowInSeconds()));
- }
-
- private static ByteBuffer getSerializedMutations(int version, Collection<Mutation> mutations)
- {
- try (DataOutputBuffer buf = new DataOutputBuffer())
- {
- buf.writeInt(mutations.size());
- for (Mutation mutation : mutations)
- Mutation.serializer.serialize(mutation, buf, version);
- return buf.buffer();
- }
- catch (IOException e)
- {
- throw new RuntimeException(e);
- }
- }
-}
diff --git a/src/java/org/apache/cassandra/cache/AutoSavingCache.java b/src/java/org/apache/cassandra/cache/AutoSavingCache.java
index 9410ba6..34f056a 100644
--- a/src/java/org/apache/cassandra/cache/AutoSavingCache.java
+++ b/src/java/org/apache/cassandra/cache/AutoSavingCache.java
@@ -34,10 +34,11 @@
import com.google.common.util.concurrent.MoreExecutors;
import org.apache.cassandra.concurrent.ScheduledExecutors;
-import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.config.Schema;
-import org.apache.cassandra.config.SchemaConstants;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.SchemaConstants;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.compaction.CompactionInfo;
import org.apache.cassandra.db.compaction.CompactionManager;
@@ -80,8 +81,10 @@
* Sticking with "d" is fine for 3.0 since it has never been released or used by another version
*
* "e" introduced with CASSANDRA-11206, omits IndexInfo from key-cache, stores offset into index-file
+ *
+ * "f" introduced with CASSANDRA-9425, changes "keyspace.table.index" in cache keys to TableMetadata.id+TableMetadata.indexName
*/
- private static final String CURRENT_VERSION = "e";
+ private static final String CURRENT_VERSION = "f";
private static volatile IStreamFactory streamFactory = new IStreamFactory()
{
@@ -202,20 +205,24 @@
UUID schemaVersion = new UUID(in.readLong(), in.readLong());
if (!schemaVersion.equals(Schema.instance.getVersion()))
throw new RuntimeException("Cache schema version "
- + schemaVersion.toString()
+ + schemaVersion
+ " does not match current schema version "
+ Schema.instance.getVersion());
ArrayDeque<Future<Pair<K, V>>> futures = new ArrayDeque<Future<Pair<K, V>>>();
while (in.available() > 0)
{
- //ksname and cfname are serialized by the serializers in CacheService
+ //tableId and indexName are serialized by the serializers in CacheService
//That is delegated there because there are serializer specific conditions
//where a cache key is skipped and not written
- String ksname = in.readUTF();
- String cfname = in.readUTF();
+ TableId tableId = TableId.deserialize(in);
+ String indexName = in.readUTF();
+ if (indexName.isEmpty())
+ indexName = null;
- ColumnFamilyStore cfs = Schema.instance.getColumnFamilyStoreIncludingIndexes(Pair.create(ksname, cfname));
+ ColumnFamilyStore cfs = Schema.instance.getColumnFamilyStoreInstance(tableId);
+ if (indexName != null && cfs != null)
+ cfs = cfs.indexManager.getIndexByName(indexName).getBackingTable().orElse(null);
Future<Pair<K, V>> entryFuture = cacheLoader.deserialize(in, cfs);
// Key cache entry can return null, if the SSTable doesn't exist.
@@ -309,12 +316,12 @@
else
type = OperationType.UNKNOWN;
- info = new CompactionInfo(CFMetaData.createFake(SchemaConstants.SYSTEM_KEYSPACE_NAME, cacheType.toString()),
- type,
- 0,
- keysEstimate,
- Unit.KEYS,
- UUIDGen.getTimeUUID());
+ info = CompactionInfo.withoutSSTables(TableMetadata.minimal(SchemaConstants.SYSTEM_KEYSPACE_NAME, cacheType.toString()),
+ type,
+ 0,
+ keysEstimate,
+ Unit.KEYS,
+ UUIDGen.getTimeUUID());
}
public CacheService.CacheType cacheType()
@@ -360,9 +367,11 @@
{
K key = keyIterator.next();
- ColumnFamilyStore cfs = Schema.instance.getColumnFamilyStoreIncludingIndexes(key.ksAndCFName);
+ ColumnFamilyStore cfs = Schema.instance.getColumnFamilyStoreInstance(key.tableId);
if (cfs == null)
continue; // the table or 2i has been dropped.
+ if (key.indexName != null)
+ cfs = cfs.indexManager.getIndexByName(key.indexName).getBackingTable().orElse(null);
cacheLoader.serialize(key, writer, cfs);
diff --git a/src/java/org/apache/cassandra/cache/CacheKey.java b/src/java/org/apache/cassandra/cache/CacheKey.java
index 0e82990..ccab672 100644
--- a/src/java/org/apache/cassandra/cache/CacheKey.java
+++ b/src/java/org/apache/cassandra/cache/CacheKey.java
@@ -17,14 +17,30 @@
*/
package org.apache.cassandra.cache;
-import org.apache.cassandra.utils.Pair;
+import java.util.Objects;
+
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.schema.TableMetadata;
public abstract class CacheKey implements IMeasurableMemory
{
- public final Pair<String, String> ksAndCFName;
+ public final TableId tableId;
+ public final String indexName;
- public CacheKey(Pair<String, String> ksAndCFName)
+ protected CacheKey(TableId tableId, String indexName)
{
- this.ksAndCFName = ksAndCFName;
+ this.tableId = tableId;
+ this.indexName = indexName;
+ }
+
+ public CacheKey(TableMetadata metadata)
+ {
+ this(metadata.id, metadata.indexName().orElse(null));
+ }
+
+ public boolean sameTable(TableMetadata tableMetadata)
+ {
+ return tableId.equals(tableMetadata.id)
+ && Objects.equals(indexName, tableMetadata.indexName().orElse(null));
}
}
diff --git a/src/java/org/apache/cassandra/cache/CaffeineCache.java b/src/java/org/apache/cassandra/cache/CaffeineCache.java
new file mode 100644
index 0000000..d51ea84
--- /dev/null
+++ b/src/java/org/apache/cassandra/cache/CaffeineCache.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cache;
+
+import static com.google.common.base.Preconditions.checkState;
+
+import java.util.Iterator;
+
+import com.google.common.util.concurrent.MoreExecutors;
+
+import com.github.benmanes.caffeine.cache.Cache;
+import com.github.benmanes.caffeine.cache.Caffeine;
+import com.github.benmanes.caffeine.cache.Policy.Eviction;
+import com.github.benmanes.caffeine.cache.Weigher;
+
+/**
+ * An adapter from a Caffeine cache to the ICache interface. This provides an on-heap cache using
+ * the W-TinyLFU eviction policy (http://arxiv.org/pdf/1512.00727.pdf), which has a higher hit rate
+ * than an LRU.
+ */
+public class CaffeineCache<K extends IMeasurableMemory, V extends IMeasurableMemory> implements ICache<K, V>
+{
+ private final Cache<K, V> cache;
+ private final Eviction<K, V> policy;
+
+ private CaffeineCache(Cache<K, V> cache)
+ {
+ this.cache = cache;
+ this.policy = cache.policy().eviction().orElseThrow(() ->
+ new IllegalArgumentException("Expected a size bounded cache"));
+ checkState(policy.isWeighted(), "Expected a weighted cache");
+ }
+
+ /**
+ * Initialize a cache with initial capacity with weightedCapacity
+ */
+ public static <K extends IMeasurableMemory, V extends IMeasurableMemory> CaffeineCache<K, V> create(long weightedCapacity, Weigher<K, V> weigher)
+ {
+ Cache<K, V> cache = Caffeine.newBuilder()
+ .maximumWeight(weightedCapacity)
+ .weigher(weigher)
+ .executor(MoreExecutors.directExecutor())
+ .build();
+ return new CaffeineCache<>(cache);
+ }
+
+ public static <K extends IMeasurableMemory, V extends IMeasurableMemory> CaffeineCache<K, V> create(long weightedCapacity)
+ {
+ return create(weightedCapacity, (key, value) -> {
+ long size = key.unsharedHeapSize() + value.unsharedHeapSize();
+ if (size > Integer.MAX_VALUE) {
+ throw new IllegalArgumentException("Serialized size cannot be more than 2GB/Integer.MAX_VALUE");
+ }
+ return (int) size;
+ });
+ }
+
+ public long capacity()
+ {
+ return policy.getMaximum();
+ }
+
+ public void setCapacity(long capacity)
+ {
+ policy.setMaximum(capacity);
+ }
+
+ public boolean isEmpty()
+ {
+ return cache.asMap().isEmpty();
+ }
+
+ public int size()
+ {
+ return cache.asMap().size();
+ }
+
+ public long weightedSize()
+ {
+ return policy.weightedSize().getAsLong();
+ }
+
+ public void clear()
+ {
+ cache.invalidateAll();
+ }
+
+ public V get(K key)
+ {
+ return cache.getIfPresent(key);
+ }
+
+ public void put(K key, V value)
+ {
+ cache.put(key, value);
+ }
+
+ public boolean putIfAbsent(K key, V value)
+ {
+ return cache.asMap().putIfAbsent(key, value) == null;
+ }
+
+ public boolean replace(K key, V old, V value)
+ {
+ return cache.asMap().replace(key, old, value);
+ }
+
+ public void remove(K key)
+ {
+ cache.invalidate(key);
+ }
+
+ public Iterator<K> keyIterator()
+ {
+ return cache.asMap().keySet().iterator();
+ }
+
+ public Iterator<K> hotKeyIterator(int n)
+ {
+ return policy.hottest(n).keySet().iterator();
+ }
+
+ public boolean containsKey(K key)
+ {
+ return cache.asMap().containsKey(key);
+ }
+}
diff --git a/src/java/org/apache/cassandra/cache/ChunkCache.java b/src/java/org/apache/cassandra/cache/ChunkCache.java
index 4e7f848..0edb681 100644
--- a/src/java/org/apache/cassandra/cache/ChunkCache.java
+++ b/src/java/org/apache/cassandra/cache/ChunkCache.java
@@ -29,11 +29,10 @@
import com.google.common.util.concurrent.MoreExecutors;
import com.github.benmanes.caffeine.cache.*;
-import com.codahale.metrics.Timer;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.io.sstable.CorruptSSTableException;
import org.apache.cassandra.io.util.*;
-import org.apache.cassandra.metrics.CacheMissMetrics;
+import org.apache.cassandra.metrics.ChunkCacheMetrics;
import org.apache.cassandra.utils.memory.BufferPool;
public class ChunkCache
@@ -47,7 +46,7 @@
public static final ChunkCache instance = enabled ? new ChunkCache() : null;
private final LoadingCache<Key, Buffer> cache;
- public final CacheMissMetrics metrics;
+ public final ChunkCacheMetrics metrics;
static class Key
{
@@ -135,29 +134,25 @@
}
}
- public ChunkCache()
+ private ChunkCache()
{
+ metrics = new ChunkCacheMetrics(this);
cache = Caffeine.newBuilder()
- .maximumWeight(cacheSize)
- .executor(MoreExecutors.directExecutor())
- .weigher((key, buffer) -> ((Buffer) buffer).buffer.capacity())
- .removalListener(this)
- .build(this);
- metrics = new CacheMissMetrics("ChunkCache", this);
+ .maximumWeight(cacheSize)
+ .executor(MoreExecutors.directExecutor())
+ .weigher((key, buffer) -> ((Buffer) buffer).buffer.capacity())
+ .removalListener(this)
+ .recordStats(() -> metrics)
+ .build(this);
}
@Override
- public Buffer load(Key key) throws Exception
+ public Buffer load(Key key)
{
- ChunkReader rebufferer = key.file;
- metrics.misses.mark();
- try (Timer.Context ctx = metrics.missLatency.time())
- {
- ByteBuffer buffer = BufferPool.get(key.file.chunkSize(), key.file.preferredBufferType());
- assert buffer != null;
- rebufferer.readChunk(key.position, buffer);
- return new Buffer(buffer, key.position);
- }
+ ByteBuffer buffer = BufferPool.get(key.file.chunkSize(), key.file.preferredBufferType());
+ assert buffer != null;
+ key.file.readChunk(key.position, buffer);
+ return new Buffer(buffer, key.position);
}
@Override
@@ -229,7 +224,6 @@
{
try
{
- metrics.requests.mark();
long pageAlignedPos = position & alignmentMask;
Buffer buf;
do
@@ -290,7 +284,7 @@
@Override
public String toString()
{
- return "CachingRebufferer:" + source.toString();
+ return "CachingRebufferer:" + source;
}
}
diff --git a/src/java/org/apache/cassandra/cache/ConcurrentLinkedHashCache.java b/src/java/org/apache/cassandra/cache/ConcurrentLinkedHashCache.java
deleted file mode 100644
index bb14055..0000000
--- a/src/java/org/apache/cassandra/cache/ConcurrentLinkedHashCache.java
+++ /dev/null
@@ -1,133 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.cache;
-
-import java.util.Iterator;
-
-import com.googlecode.concurrentlinkedhashmap.ConcurrentLinkedHashMap;
-import com.googlecode.concurrentlinkedhashmap.EntryWeigher;
-
-/** Wrapper so CLHM can implement ICache interface.
- * (this is what you get for making library classes final.) */
-public class ConcurrentLinkedHashCache<K extends IMeasurableMemory, V extends IMeasurableMemory> implements ICache<K, V>
-{
- public static final int DEFAULT_CONCURENCY_LEVEL = 64;
- private final ConcurrentLinkedHashMap<K, V> map;
-
- private ConcurrentLinkedHashCache(ConcurrentLinkedHashMap<K, V> map)
- {
- this.map = map;
- }
-
- /**
- * Initialize a cache with initial capacity with weightedCapacity
- */
- public static <K extends IMeasurableMemory, V extends IMeasurableMemory> ConcurrentLinkedHashCache<K, V> create(long weightedCapacity, EntryWeigher<K, V> entryWeiger)
- {
- ConcurrentLinkedHashMap<K, V> map = new ConcurrentLinkedHashMap.Builder<K, V>()
- .weigher(entryWeiger)
- .maximumWeightedCapacity(weightedCapacity)
- .concurrencyLevel(DEFAULT_CONCURENCY_LEVEL)
- .build();
-
- return new ConcurrentLinkedHashCache<>(map);
- }
-
- public static <K extends IMeasurableMemory, V extends IMeasurableMemory> ConcurrentLinkedHashCache<K, V> create(long weightedCapacity)
- {
- return create(weightedCapacity, new EntryWeigher<K, V>()
- {
- public int weightOf(K key, V value)
- {
- long size = key.unsharedHeapSize() + value.unsharedHeapSize();
- assert size <= Integer.MAX_VALUE : "Serialized size cannot be more than 2GB/Integer.MAX_VALUE";
- return (int) size;
- }
- });
- }
-
- public long capacity()
- {
- return map.capacity();
- }
-
- public void setCapacity(long capacity)
- {
- map.setCapacity(capacity);
- }
-
- public boolean isEmpty()
- {
- return map.isEmpty();
- }
-
- public int size()
- {
- return map.size();
- }
-
- public long weightedSize()
- {
- return map.weightedSize();
- }
-
- public void clear()
- {
- map.clear();
- }
-
- public V get(K key)
- {
- return map.get(key);
- }
-
- public void put(K key, V value)
- {
- map.put(key, value);
- }
-
- public boolean putIfAbsent(K key, V value)
- {
- return map.putIfAbsent(key, value) == null;
- }
-
- public boolean replace(K key, V old, V value)
- {
- return map.replace(key, old, value);
- }
-
- public void remove(K key)
- {
- map.remove(key);
- }
-
- public Iterator<K> keyIterator()
- {
- return map.keySet().iterator();
- }
-
- public Iterator<K> hotKeyIterator(int n)
- {
- return map.descendingKeySetWithLimit(n).iterator();
- }
-
- public boolean containsKey(K key)
- {
- return map.containsKey(key);
- }
-}
diff --git a/src/java/org/apache/cassandra/cache/CounterCacheKey.java b/src/java/org/apache/cassandra/cache/CounterCacheKey.java
index 8b173bf..dc3ce4e 100644
--- a/src/java/org/apache/cassandra/cache/CounterCacheKey.java
+++ b/src/java/org/apache/cassandra/cache/CounterCacheKey.java
@@ -17,35 +17,56 @@
*/
package org.apache.cassandra.cache;
+import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.Arrays;
+import java.util.List;
+import java.util.Objects;
-import org.apache.cassandra.config.ColumnDefinition;
import org.apache.cassandra.db.*;
-import org.apache.cassandra.db.rows.CellPath;
+import org.apache.cassandra.db.filter.ClusteringIndexFilter;
+import org.apache.cassandra.db.filter.ClusteringIndexNamesFilter;
+import org.apache.cassandra.db.filter.ColumnFilter;
import org.apache.cassandra.db.marshal.CompositeType;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.db.rows.CellPath;
+import org.apache.cassandra.db.rows.RowIterator;
+import org.apache.cassandra.db.rows.UnfilteredRowIterators;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.schema.TableMetadataRef;
import org.apache.cassandra.utils.*;
public final class CounterCacheKey extends CacheKey
{
- private static final long EMPTY_SIZE = ObjectSizes.measure(new CounterCacheKey(null, ByteBufferUtil.EMPTY_BYTE_BUFFER, ByteBuffer.allocate(1)));
+ private static final long EMPTY_SIZE = ObjectSizes.measure(new CounterCacheKey(TableMetadata.builder("ks", "tab")
+ .addPartitionKeyColumn("pk", UTF8Type.instance)
+ .build(), ByteBufferUtil.EMPTY_BYTE_BUFFER, ByteBuffer.allocate(1)));
- public final byte[] partitionKey;
- public final byte[] cellName;
+ private final byte[] partitionKey;
+ private final byte[] cellName;
- public CounterCacheKey(Pair<String, String> ksAndCFName, ByteBuffer partitionKey, ByteBuffer cellName)
+ private CounterCacheKey(TableMetadata tableMetadata, byte[] partitionKey, byte[] cellName)
{
- super(ksAndCFName);
- this.partitionKey = ByteBufferUtil.getArray(partitionKey);
- this.cellName = ByteBufferUtil.getArray(cellName);
+ super(tableMetadata);
+ this.partitionKey = partitionKey;
+ this.cellName = cellName;
}
- public static CounterCacheKey create(Pair<String, String> ksAndCFName, ByteBuffer partitionKey, Clustering clustering, ColumnDefinition c, CellPath path)
+ private CounterCacheKey(TableMetadata tableMetadata, ByteBuffer partitionKey, ByteBuffer cellName)
{
- return new CounterCacheKey(ksAndCFName, partitionKey, makeCellName(clustering, c, path));
+ this(tableMetadata, ByteBufferUtil.getArray(partitionKey), ByteBufferUtil.getArray(cellName));
}
- private static ByteBuffer makeCellName(Clustering clustering, ColumnDefinition c, CellPath path)
+ public static CounterCacheKey create(TableMetadata tableMetadata, ByteBuffer partitionKey, Clustering clustering, ColumnMetadata c, CellPath path)
+ {
+ return new CounterCacheKey(tableMetadata, partitionKey, makeCellName(clustering, c, path));
+ }
+
+ private static ByteBuffer makeCellName(Clustering clustering, ColumnMetadata c, CellPath path)
{
int cs = clustering.size();
ByteBuffer[] values = new ByteBuffer[cs + 1 + (path == null ? 0 : path.size())];
@@ -58,6 +79,75 @@
return CompositeType.build(values);
}
+ public ByteBuffer partitionKey()
+ {
+ return ByteBuffer.wrap(partitionKey);
+ }
+
+ /**
+ * Reads the value of the counter represented by this key.
+ *
+ * @param cfs the store for the table this is a key of.
+ * @return the value for the counter represented by this key, or {@code null} if there
+ * is not such counter.
+ */
+ public ByteBuffer readCounterValue(ColumnFamilyStore cfs)
+ {
+ TableMetadata metadata = cfs.metadata();
+ assert metadata.id.equals(tableId) && Objects.equals(metadata.indexName().orElse(null), indexName);
+
+ DecoratedKey key = cfs.decorateKey(partitionKey());
+
+ int clusteringSize = metadata.comparator.size();
+ List<ByteBuffer> buffers = CompositeType.splitName(ByteBuffer.wrap(cellName));
+ assert buffers.size() >= clusteringSize + 1; // See makeCellName above
+
+ Clustering clustering = Clustering.make(buffers.subList(0, clusteringSize).toArray(new ByteBuffer[clusteringSize]));
+ ColumnMetadata column = metadata.getColumn(buffers.get(clusteringSize));
+ // This can theoretically happen if a column is dropped after the cache is saved and we
+ // try to load it. Not point if failing in any case, just skip the value.
+ if (column == null)
+ return null;
+
+ CellPath path = column.isComplex() ? CellPath.create(buffers.get(buffers.size() - 1)) : null;
+
+ int nowInSec = FBUtilities.nowInSeconds();
+ ColumnFilter.Builder builder = ColumnFilter.selectionBuilder();
+ if (path == null)
+ builder.add(column);
+ else
+ builder.select(column, path);
+
+ ClusteringIndexFilter filter = new ClusteringIndexNamesFilter(FBUtilities.singleton(clustering, metadata.comparator), false);
+ SinglePartitionReadCommand cmd = SinglePartitionReadCommand.create(metadata, nowInSec, key, builder.build(), filter);
+ try (ReadExecutionController controller = cmd.executionController();
+ RowIterator iter = UnfilteredRowIterators.filter(cmd.queryMemtableAndDisk(cfs, controller), nowInSec))
+ {
+ ByteBuffer value = null;
+ if (column.isStatic())
+ value = iter.staticRow().getCell(column).value();
+ else if (iter.hasNext())
+ value = iter.next().getCell(column).value();
+
+ return value;
+ }
+ }
+
+ public void write(DataOutputPlus out)
+ throws IOException
+ {
+ ByteBufferUtil.writeWithLength(partitionKey, out);
+ ByteBufferUtil.writeWithLength(cellName, out);
+ }
+
+ public static CounterCacheKey read(TableMetadata tableMetadata, DataInputPlus in)
+ throws IOException
+ {
+ return new CounterCacheKey(tableMetadata,
+ ByteBufferUtil.readBytesWithLength(in),
+ ByteBufferUtil.readBytesWithLength(in));
+ }
+
public long unsharedHeapSize()
{
return EMPTY_SIZE
@@ -68,8 +158,9 @@
@Override
public String toString()
{
- return String.format("CounterCacheKey(%s, %s, %s)",
- ksAndCFName,
+ TableMetadataRef tableRef = Schema.instance.getTableMetadataRef(tableId);
+ return String.format("CounterCacheKey(%s, %s, %s, %s)",
+ tableRef, indexName,
ByteBufferUtil.bytesToHex(ByteBuffer.wrap(partitionKey)),
ByteBufferUtil.bytesToHex(ByteBuffer.wrap(cellName)));
}
@@ -77,7 +168,7 @@
@Override
public int hashCode()
{
- return Arrays.deepHashCode(new Object[]{ksAndCFName, partitionKey, cellName});
+ return Arrays.deepHashCode(new Object[]{tableId, indexName, partitionKey, cellName});
}
@Override
@@ -91,7 +182,8 @@
CounterCacheKey cck = (CounterCacheKey) o;
- return ksAndCFName.equals(cck.ksAndCFName)
+ return tableId.equals(cck.tableId)
+ && Objects.equals(indexName, cck.indexName)
&& Arrays.equals(partitionKey, cck.partitionKey)
&& Arrays.equals(cellName, cck.cellName);
}
diff --git a/src/java/org/apache/cassandra/cache/InstrumentingCache.java b/src/java/org/apache/cassandra/cache/InstrumentingCache.java
index c8728fd..e28766f 100644
--- a/src/java/org/apache/cassandra/cache/InstrumentingCache.java
+++ b/src/java/org/apache/cassandra/cache/InstrumentingCache.java
@@ -56,9 +56,10 @@
public V get(K key)
{
V v = map.get(key);
- metrics.requests.mark();
if (v != null)
metrics.hits.mark();
+ else
+ metrics.misses.mark();
return v;
}
diff --git a/src/java/org/apache/cassandra/cache/KeyCacheKey.java b/src/java/org/apache/cassandra/cache/KeyCacheKey.java
index 222622c..ac6f1f9 100644
--- a/src/java/org/apache/cassandra/cache/KeyCacheKey.java
+++ b/src/java/org/apache/cassandra/cache/KeyCacheKey.java
@@ -19,26 +19,29 @@
import java.nio.ByteBuffer;
import java.util.Arrays;
+import java.util.Objects;
+import org.apache.cassandra.db.marshal.UTF8Type;
import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.ObjectSizes;
-import org.apache.cassandra.utils.Pair;
public class KeyCacheKey extends CacheKey
{
public final Descriptor desc;
- private static final long EMPTY_SIZE = ObjectSizes.measure(new KeyCacheKey(null, null, ByteBufferUtil.EMPTY_BYTE_BUFFER));
+ private static final long EMPTY_SIZE = ObjectSizes.measure(new KeyCacheKey(TableMetadata.builder("ks", "tab")
+ .addPartitionKeyColumn("pk", UTF8Type.instance)
+ .build(), null, ByteBufferUtil.EMPTY_BYTE_BUFFER));
// keeping an array instead of a ByteBuffer lowers the overhead of the key cache working set,
// without extra copies on lookup since client-provided key ByteBuffers will be array-backed already
public final byte[] key;
- public KeyCacheKey(Pair<String, String> ksAndCFName, Descriptor desc, ByteBuffer key)
+ public KeyCacheKey(TableMetadata tableMetadata, Descriptor desc, ByteBuffer key)
{
-
- super(ksAndCFName);
+ super(tableMetadata);
this.desc = desc;
this.key = ByteBufferUtil.getArray(key);
assert this.key != null;
@@ -62,13 +65,17 @@
KeyCacheKey that = (KeyCacheKey) o;
- return ksAndCFName.equals(that.ksAndCFName) && desc.equals(that.desc) && Arrays.equals(key, that.key);
+ return tableId.equals(that.tableId)
+ && Objects.equals(indexName, that.indexName)
+ && desc.equals(that.desc)
+ && Arrays.equals(key, that.key);
}
@Override
public int hashCode()
{
- int result = ksAndCFName.hashCode();
+ int result = tableId.hashCode();
+ result = 31 * result + Objects.hashCode(indexName);
result = 31 * result + desc.hashCode();
result = 31 * result + Arrays.hashCode(key);
return result;
diff --git a/src/java/org/apache/cassandra/cache/OHCProvider.java b/src/java/org/apache/cassandra/cache/OHCProvider.java
index 1ea2b78..afdc872 100644
--- a/src/java/org/apache/cassandra/cache/OHCProvider.java
+++ b/src/java/org/apache/cassandra/cache/OHCProvider.java
@@ -28,7 +28,7 @@
import org.apache.cassandra.io.util.DataOutputBuffer;
import org.apache.cassandra.io.util.DataOutputBufferFixed;
import org.apache.cassandra.io.util.RebufferingInputStream;
-import org.apache.cassandra.utils.Pair;
+import org.apache.cassandra.schema.TableId;
import org.caffinitas.ohc.OHCache;
import org.caffinitas.ohc.OHCacheBuilder;
@@ -125,12 +125,10 @@
private static KeySerializer instance = new KeySerializer();
public void serialize(RowCacheKey rowCacheKey, ByteBuffer buf)
{
- @SuppressWarnings("resource")
- DataOutputBuffer dataOutput = new DataOutputBufferFixed(buf);
- try
+ try (DataOutputBuffer dataOutput = new DataOutputBufferFixed(buf))
{
- dataOutput.writeUTF(rowCacheKey.ksAndCFName.left);
- dataOutput.writeUTF(rowCacheKey.ksAndCFName.right);
+ rowCacheKey.tableId.serialize(dataOutput);
+ dataOutput.writeUTF(rowCacheKey.indexName != null ? rowCacheKey.indexName : "");
}
catch (IOException e)
{
@@ -142,14 +140,14 @@
public RowCacheKey deserialize(ByteBuffer buf)
{
- @SuppressWarnings("resource")
- DataInputBuffer dataInput = new DataInputBuffer(buf, false);
- String ksName = null;
- String cfName = null;
- try
+ TableId tableId = null;
+ String indexName = null;
+ try (DataInputBuffer dataInput = new DataInputBuffer(buf, false))
{
- ksName = dataInput.readUTF();
- cfName = dataInput.readUTF();
+ tableId = TableId.deserialize(dataInput);
+ indexName = dataInput.readUTF();
+ if (indexName.isEmpty())
+ indexName = null;
}
catch (IOException e)
{
@@ -157,15 +155,15 @@
}
byte[] key = new byte[buf.getInt()];
buf.get(key);
- return new RowCacheKey(Pair.create(ksName, cfName), key);
+ return new RowCacheKey(tableId, indexName, key);
}
public int serializedSize(RowCacheKey rowCacheKey)
{
- return TypeSizes.sizeof(rowCacheKey.ksAndCFName.left)
- + TypeSizes.sizeof(rowCacheKey.ksAndCFName.right)
- + 4
- + rowCacheKey.key.length;
+ return rowCacheKey.tableId.serializedSize()
+ + TypeSizes.sizeof(rowCacheKey.indexName != null ? rowCacheKey.indexName : "")
+ + 4
+ + rowCacheKey.key.length;
}
}
diff --git a/src/java/org/apache/cassandra/cache/RowCacheKey.java b/src/java/org/apache/cassandra/cache/RowCacheKey.java
index e02db42..bbf289a 100644
--- a/src/java/org/apache/cassandra/cache/RowCacheKey.java
+++ b/src/java/org/apache/cassandra/cache/RowCacheKey.java
@@ -19,32 +19,41 @@
import java.nio.ByteBuffer;
import java.util.Arrays;
+import java.util.Objects;
+
+import com.google.common.annotations.VisibleForTesting;
import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.schema.TableMetadataRef;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.ObjectSizes;
-import org.apache.cassandra.utils.Pair;
public final class RowCacheKey extends CacheKey
{
public final byte[] key;
- private static final long EMPTY_SIZE = ObjectSizes.measure(new RowCacheKey(null, ByteBufferUtil.EMPTY_BYTE_BUFFER));
+ private static final long EMPTY_SIZE = ObjectSizes.measure(new RowCacheKey(null, null, new byte[0]));
- public RowCacheKey(Pair<String, String> ksAndCFName, byte[] key)
+ public RowCacheKey(TableId tableId, String indexName, byte[] key)
{
- super(ksAndCFName);
+ super(tableId, indexName);
this.key = key;
}
- public RowCacheKey(Pair<String, String> ksAndCFName, DecoratedKey key)
+ public RowCacheKey(TableMetadata metadata, DecoratedKey key)
{
- this(ksAndCFName, key.getKey());
+ super(metadata);
+ this.key = ByteBufferUtil.getArray(key.getKey());
+ assert this.key != null;
}
- public RowCacheKey(Pair<String, String> ksAndCFName, ByteBuffer key)
+ @VisibleForTesting
+ public RowCacheKey(TableId tableId, String indexName, ByteBuffer key)
{
- super(ksAndCFName);
+ super(tableId, indexName);
this.key = ByteBufferUtil.getArray(key);
assert this.key != null;
}
@@ -62,13 +71,16 @@
RowCacheKey that = (RowCacheKey) o;
- return ksAndCFName.equals(that.ksAndCFName) && Arrays.equals(key, that.key);
+ return tableId.equals(that.tableId)
+ && Objects.equals(indexName, that.indexName)
+ && Arrays.equals(key, that.key);
}
@Override
public int hashCode()
{
- int result = ksAndCFName.hashCode();
+ int result = tableId.hashCode();
+ result = 31 * result + Objects.hashCode(indexName);
result = 31 * result + (key != null ? Arrays.hashCode(key) : 0);
return result;
}
@@ -76,6 +88,7 @@
@Override
public String toString()
{
- return String.format("RowCacheKey(ksAndCFName:%s, key:%s)", ksAndCFName, Arrays.toString(key));
+ TableMetadataRef tableRef = Schema.instance.getTableMetadataRef(tableId);
+ return String.format("RowCacheKey(%s, %s, key:%s)", tableRef, indexName, Arrays.toString(key));
}
}
diff --git a/src/java/org/apache/cassandra/cache/SerializingCache.java b/src/java/org/apache/cassandra/cache/SerializingCache.java
index 0ece686..55c20ec 100644
--- a/src/java/org/apache/cassandra/cache/SerializingCache.java
+++ b/src/java/org/apache/cassandra/cache/SerializingCache.java
@@ -17,21 +17,22 @@
*/
package org.apache.cassandra.cache;
-import java.io.IOException;
-import java.util.Iterator;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.googlecode.concurrentlinkedhashmap.ConcurrentLinkedHashMap;
-import com.googlecode.concurrentlinkedhashmap.EvictionListener;
-import com.googlecode.concurrentlinkedhashmap.Weigher;
+import com.github.benmanes.caffeine.cache.Cache;
+import com.github.benmanes.caffeine.cache.Caffeine;
+import com.github.benmanes.caffeine.cache.Weigher;
import org.apache.cassandra.io.ISerializer;
import org.apache.cassandra.io.util.MemoryInputStream;
import org.apache.cassandra.io.util.MemoryOutputStream;
import org.apache.cassandra.io.util.WrappedDataOutputStreamPlus;
import org.apache.cassandra.utils.FBUtilities;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Iterator;
+
+import com.google.common.util.concurrent.MoreExecutors;
/**
* Serializes cache values off-heap.
@@ -40,49 +41,42 @@
{
private static final Logger logger = LoggerFactory.getLogger(SerializingCache.class);
- private static final int DEFAULT_CONCURENCY_LEVEL = 64;
-
- private final ConcurrentLinkedHashMap<K, RefCountedMemory> map;
+ private final Cache<K, RefCountedMemory> cache;
private final ISerializer<V> serializer;
- private SerializingCache(long capacity, Weigher<RefCountedMemory> weigher, ISerializer<V> serializer)
+ private SerializingCache(long capacity, Weigher<K, RefCountedMemory> weigher, ISerializer<V> serializer)
{
this.serializer = serializer;
- EvictionListener<K,RefCountedMemory> listener = new EvictionListener<K, RefCountedMemory>()
- {
- public void onEviction(K k, RefCountedMemory mem)
- {
- mem.unreference();
- }
- };
-
- this.map = new ConcurrentLinkedHashMap.Builder<K, RefCountedMemory>()
+ this.cache = Caffeine.newBuilder()
.weigher(weigher)
- .maximumWeightedCapacity(capacity)
- .concurrencyLevel(DEFAULT_CONCURENCY_LEVEL)
- .listener(listener)
+ .maximumWeight(capacity)
+ .executor(MoreExecutors.directExecutor())
+ .removalListener((key, mem, cause) -> {
+ if (cause.wasEvicted()) {
+ mem.unreference();
+ }
+ })
.build();
}
- public static <K, V> SerializingCache<K, V> create(long weightedCapacity, Weigher<RefCountedMemory> weigher, ISerializer<V> serializer)
+ public static <K, V> SerializingCache<K, V> create(long weightedCapacity, Weigher<K, RefCountedMemory> weigher, ISerializer<V> serializer)
{
return new SerializingCache<>(weightedCapacity, weigher, serializer);
}
public static <K, V> SerializingCache<K, V> create(long weightedCapacity, ISerializer<V> serializer)
{
- return create(weightedCapacity, new Weigher<RefCountedMemory>()
- {
- public int weightOf(RefCountedMemory value)
- {
- long size = value.size();
- assert size < Integer.MAX_VALUE : "Serialized size cannot be more than 2GB";
- return (int) size;
+ return create(weightedCapacity, (key, value) -> {
+ long size = value.size();
+ if (size > Integer.MAX_VALUE) {
+ throw new IllegalArgumentException("Serialized size must not be more than 2GB");
}
+ return (int) size;
}, serializer);
}
+ @SuppressWarnings("resource")
private V deserialize(RefCountedMemory mem)
{
try
@@ -96,6 +90,7 @@
}
}
+ @SuppressWarnings("resource")
private RefCountedMemory serialize(V value)
{
long serializedSize = serializer.serializedSize(value);
@@ -126,38 +121,38 @@
public long capacity()
{
- return map.capacity();
+ return cache.policy().eviction().get().getMaximum();
}
public void setCapacity(long capacity)
{
- map.setCapacity(capacity);
+ cache.policy().eviction().get().setMaximum(capacity);
}
public boolean isEmpty()
{
- return map.isEmpty();
+ return cache.asMap().isEmpty();
}
public int size()
{
- return map.size();
+ return cache.asMap().size();
}
public long weightedSize()
{
- return map.weightedSize();
+ return cache.policy().eviction().get().weightedSize().getAsLong();
}
public void clear()
{
- map.clear();
+ cache.invalidateAll();
}
@SuppressWarnings("resource")
public V get(K key)
{
- RefCountedMemory mem = map.get(key);
+ RefCountedMemory mem = cache.getIfPresent(key);
if (mem == null)
return null;
if (!mem.reference())
@@ -182,7 +177,7 @@
RefCountedMemory old;
try
{
- old = map.put(key, mem);
+ old = cache.asMap().put(key, mem);
}
catch (Throwable t)
{
@@ -204,7 +199,7 @@
RefCountedMemory old;
try
{
- old = map.putIfAbsent(key, mem);
+ old = cache.asMap().putIfAbsent(key, mem);
}
catch (Throwable t)
{
@@ -221,8 +216,8 @@
@SuppressWarnings("resource")
public boolean replace(K key, V oldToReplace, V value)
{
- // if there is no old value in our map, we fail
- RefCountedMemory old = map.get(key);
+ // if there is no old value in our cache, we fail
+ RefCountedMemory old = cache.getIfPresent(key);
if (old == null)
return false;
@@ -245,7 +240,7 @@
boolean success;
try
{
- success = map.replace(key, old, mem);
+ success = cache.asMap().replace(key, old, mem);
}
catch (Throwable t)
{
@@ -263,23 +258,23 @@
public void remove(K key)
{
@SuppressWarnings("resource")
- RefCountedMemory mem = map.remove(key);
+ RefCountedMemory mem = cache.asMap().remove(key);
if (mem != null)
mem.unreference();
}
public Iterator<K> keyIterator()
{
- return map.keySet().iterator();
+ return cache.asMap().keySet().iterator();
}
public Iterator<K> hotKeyIterator(int n)
{
- return map.descendingKeySetWithLimit(n).iterator();
+ return cache.policy().eviction().get().hottest(n).keySet().iterator();
}
public boolean containsKey(K key)
{
- return map.containsKey(key);
+ return cache.asMap().containsKey(key);
}
}
diff --git a/src/java/org/apache/cassandra/cache/SerializingCacheProvider.java b/src/java/org/apache/cassandra/cache/SerializingCacheProvider.java
index 1119295..813f6fe 100644
--- a/src/java/org/apache/cassandra/cache/SerializingCacheProvider.java
+++ b/src/java/org/apache/cassandra/cache/SerializingCacheProvider.java
@@ -33,8 +33,8 @@
return SerializingCache.create(DatabaseDescriptor.getRowCacheSizeInMB() * 1024 * 1024, new RowCacheSerializer());
}
- // Package protected for tests
- static class RowCacheSerializer implements ISerializer<IRowCacheEntry>
+ // Package Public: used by external Row Cache plugins
+ public static class RowCacheSerializer implements ISerializer<IRowCacheEntry>
{
public void serialize(IRowCacheEntry entry, DataOutputPlus out) throws IOException
{
diff --git a/src/java/org/apache/cassandra/client/RingCache.java b/src/java/org/apache/cassandra/client/RingCache.java
deleted file mode 100644
index 5196bce..0000000
--- a/src/java/org/apache/cassandra/client/RingCache.java
+++ /dev/null
@@ -1,121 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.client;
-
-import java.io.IOException;
-import java.net.InetAddress;
-import java.net.UnknownHostException;
-import java.nio.ByteBuffer;
-import java.util.List;
-
-import org.apache.cassandra.dht.IPartitioner;
-import org.apache.cassandra.dht.Range;
-import org.apache.cassandra.dht.Token;
-import org.apache.cassandra.hadoop.ConfigHelper;
-import org.apache.cassandra.thrift.Cassandra;
-import org.apache.cassandra.thrift.TokenRange;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.thrift.TException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.collect.ArrayListMultimap;
-import com.google.common.collect.Multimap;
-
-/**
- * A class for caching the ring map at the client. For usage example, see
- * test/unit/org.apache.cassandra.client.TestRingCache.java.
- * TODO: doing a naive linear search of the token map
- */
-public class RingCache
-{
- final private static Logger logger = LoggerFactory.getLogger(RingCache.class);
-
- private final IPartitioner partitioner;
- private final Configuration conf;
-
- private Multimap<Range<Token>, InetAddress> rangeMap;
-
- public RingCache(Configuration conf)
- {
- this.conf = conf;
- this.partitioner = ConfigHelper.getOutputPartitioner(conf);
- refreshEndpointMap();
- }
-
- public void refreshEndpointMap()
- {
- try
- {
- Cassandra.Client client = ConfigHelper.getClientFromOutputAddressList(conf);
-
- String keyspace = ConfigHelper.getOutputKeyspace(conf);
- List<TokenRange> ring = ConfigHelper.getOutputLocalDCOnly(conf)
- ? client.describe_local_ring(keyspace)
- : client.describe_ring(keyspace);
- rangeMap = ArrayListMultimap.create();
-
- for (TokenRange range : ring)
- {
- Token left = partitioner.getTokenFactory().fromString(range.start_token);
- Token right = partitioner.getTokenFactory().fromString(range.end_token);
- Range<Token> r = new Range<Token>(left, right);
- for (String host : range.endpoints)
- {
- try
- {
- rangeMap.put(r, InetAddress.getByName(host));
- } catch (UnknownHostException e)
- {
- throw new AssertionError(e); // host strings are IPs
- }
- }
- }
- }
- catch (IOException e)
- {
- throw new RuntimeException(e);
- }
- catch (TException e)
- {
- logger.trace("Error contacting seed list {} {}", ConfigHelper.getOutputInitialAddress(conf), e.getMessage());
- }
- }
-
- /** ListMultimap promises to return a List for get(K) */
- public List<InetAddress> getEndpoint(Range<Token> range)
- {
- return (List<InetAddress>) rangeMap.get(range);
- }
-
- public List<InetAddress> getEndpoint(ByteBuffer key)
- {
- return getEndpoint(getRange(key));
- }
-
- public Range<Token> getRange(ByteBuffer key)
- {
- // TODO: naive linear search of the token map
- Token t = partitioner.getToken(key);
- for (Range<Token> range : rangeMap.keySet())
- if (range.contains(t))
- return range;
-
- throw new RuntimeException("Invalid token information returned by describe_ring: " + rangeMap);
- }
-}
diff --git a/src/java/org/apache/cassandra/concurrent/AbstractLocalAwareExecutorService.java b/src/java/org/apache/cassandra/concurrent/AbstractLocalAwareExecutorService.java
index 4b1fe05..6bc904c 100644
--- a/src/java/org/apache/cassandra/concurrent/AbstractLocalAwareExecutorService.java
+++ b/src/java/org/apache/cassandra/concurrent/AbstractLocalAwareExecutorService.java
@@ -166,7 +166,7 @@
}
catch (Throwable t)
{
- logger.error("Uncaught exception on thread {}", Thread.currentThread(), t);
+ logger.error(String.format("Uncaught exception on thread %s", Thread.currentThread()), t);
result = t;
failure = true;
if (t instanceof CorruptSSTableException)
diff --git a/src/java/org/apache/cassandra/concurrent/DebuggableThreadPoolExecutor.java b/src/java/org/apache/cassandra/concurrent/DebuggableThreadPoolExecutor.java
index 92cbbf4..a2de775 100644
--- a/src/java/org/apache/cassandra/concurrent/DebuggableThreadPoolExecutor.java
+++ b/src/java/org/apache/cassandra/concurrent/DebuggableThreadPoolExecutor.java
@@ -17,7 +17,21 @@
*/
package org.apache.cassandra.concurrent;
-import java.util.concurrent.*;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CancellationException;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.FutureTask;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.RejectedExecutionException;
+import java.util.concurrent.RejectedExecutionHandler;
+import java.util.concurrent.RunnableFuture;
+import java.util.concurrent.SynchronousQueue;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -147,7 +161,7 @@
{
super.execute(locals == null || command instanceof LocalSessionWrapper
? command
- : new LocalSessionWrapper<Object>(command, locals));
+ : LocalSessionWrapper.create(command, null, locals));
}
public void maybeExecuteImmediately(Runnable command)
@@ -160,7 +174,7 @@
public void execute(Runnable command)
{
super.execute(isTracing() && !(command instanceof LocalSessionWrapper)
- ? new LocalSessionWrapper<Object>(Executors.callable(command, null))
+ ? LocalSessionWrapper.create(command)
: command);
}
@@ -168,9 +182,9 @@
protected <T> RunnableFuture<T> newTaskFor(Runnable runnable, T result)
{
if (isTracing() && !(runnable instanceof LocalSessionWrapper))
- {
- return new LocalSessionWrapper<T>(Executors.callable(runnable, result));
- }
+ return LocalSessionWrapper.create(runnable, result);
+ if (runnable instanceof RunnableFuture)
+ return new ForwardingRunnableFuture<>((RunnableFuture) runnable, result);
return super.newTaskFor(runnable, result);
}
@@ -178,9 +192,7 @@
protected <T> RunnableFuture<T> newTaskFor(Callable<T> callable)
{
if (isTracing() && !(callable instanceof LocalSessionWrapper))
- {
- return new LocalSessionWrapper<T>(callable);
- }
+ return LocalSessionWrapper.create(callable);
return super.newTaskFor(callable);
}
@@ -213,6 +225,18 @@
super.beforeExecute(t, r);
}
+ @Override
+ public int getActiveTaskCount()
+ {
+ return getActiveCount();
+ }
+
+ @Override
+ public int getPendingTaskCount()
+ {
+ return getQueue().size();
+ }
+
/**
* Send @param t and any exception wrapped by @param r to the default uncaught exception handler,
* or log them if none such is set up
@@ -246,16 +270,32 @@
*/
public static Throwable extractThrowable(Runnable runnable)
{
- // Check for exceptions wrapped by FutureTask. We do this by calling get(), which will
+ // Check for exceptions wrapped by FutureTask or tasks which wrap FutureTask (HasDelegateFuture interface)
+ Throwable throwable = null;
+ if (runnable instanceof Future<?>)
+ {
+ throwable = extractThrowable(((Future<?>) runnable));
+ }
+ if (throwable == null && runnable instanceof HasDelegateFuture)
+ {
+ throwable = extractThrowable(((HasDelegateFuture) runnable).getDelegate());
+ }
+
+ return throwable;
+ }
+
+ private static Throwable extractThrowable(Future<?> future)
+ {
+ // Check for exceptions wrapped by Future. We do this by calling get(), which will
// cause it to throw any saved exception.
//
// Complicating things, calling get() on a ScheduledFutureTask will block until the task
// is cancelled. Hence, the extra isDone check beforehand.
- if ((runnable instanceof Future<?>) && ((Future<?>) runnable).isDone())
+ if (future.isDone())
{
try
{
- ((Future<?>) runnable).get();
+ future.get();
}
catch (InterruptedException e)
{
@@ -270,30 +310,65 @@
return e.getCause();
}
}
-
return null;
}
/**
+ * If a task wraps a {@link Future} then it should implement this interface to expose the underlining future for
+ * {@link #extractThrowable(Runnable)} to handle.
+ */
+ private interface HasDelegateFuture
+ {
+ Future<?> getDelegate();
+ }
+
+ /**
* Used to wrap a Runnable or Callable passed to submit or execute so we can clone the ExecutorLocals and move
* them into the worker thread.
*
+ * The {@link DebuggableThreadPoolExecutor#afterExecute(java.lang.Runnable, java.lang.Throwable)}
+ * method is called after the runnable completes, which will then call {@link #extractThrowable(Runnable)} to
+ * attempt to get the "hidden" throwable from a task which implements {@link Future}. The problem is that {@link LocalSessionWrapper}
+ * expects that the {@link Callable} provided to it will throw; which is not true for {@link RunnableFuture} tasks;
+ * the expected semantic in this case is to have the LocalSessionWrapper future be successful and a new implementation
+ * {@link FutureLocalSessionWrapper} is created to expose the underline {@link Future} for {@link #extractThrowable(Runnable)}.
+ *
+ * If a task is a {@link Runnable} the create family of methods should be called rather than {@link Executors#callable(Runnable)}
+ * since they will handle the case where the task is also a future, and will make sure the {@link #extractThrowable(Runnable)}
+ * is able to detect the task's underline exception.
+ *
* @param <T>
*/
private static class LocalSessionWrapper<T> extends FutureTask<T>
{
private final ExecutorLocals locals;
- public LocalSessionWrapper(Callable<T> callable)
+ private LocalSessionWrapper(Callable<T> callable, ExecutorLocals locals)
{
super(callable);
- locals = ExecutorLocals.create();
+ this.locals = locals;
}
- public LocalSessionWrapper(Runnable command, ExecutorLocals locals)
+ static LocalSessionWrapper<Object> create(Runnable command)
{
- super(command, null);
- this.locals = locals;
+ return create(command, null, ExecutorLocals.create());
+ }
+
+ static <T> LocalSessionWrapper<T> create(Runnable command, T result)
+ {
+ return create(command, result, ExecutorLocals.create());
+ }
+
+ static <T> LocalSessionWrapper<T> create(Runnable command, T result, ExecutorLocals locals)
+ {
+ if (command instanceof RunnableFuture)
+ return new FutureLocalSessionWrapper<>((RunnableFuture) command, result, locals);
+ return new LocalSessionWrapper<>(Executors.callable(command, result), locals);
+ }
+
+ static <T> LocalSessionWrapper<T> create(Callable<T> command)
+ {
+ return new LocalSessionWrapper<>(command, ExecutorLocals.create());
}
private void setupContext()
@@ -306,4 +381,46 @@
ExecutorLocals.set(null);
}
}
+
+ private static class FutureLocalSessionWrapper<T> extends LocalSessionWrapper<T> implements HasDelegateFuture
+ {
+ private final RunnableFuture<T> delegate;
+
+ private FutureLocalSessionWrapper(RunnableFuture command, T result, ExecutorLocals locals)
+ {
+ super(() -> {
+ command.run();
+ return result;
+ }, locals);
+ this.delegate = command;
+ }
+
+ public Future<T> getDelegate()
+ {
+ return delegate;
+ }
+ }
+
+ /**
+ * Similar to {@link FutureLocalSessionWrapper}, this class wraps a {@link Future} and will be success
+ * if the underline future is marked as failed; the main difference is that this class does not setup
+ * {@link ExecutorLocals}.
+ *
+ * @param <T>
+ */
+ private static class ForwardingRunnableFuture<T> extends FutureTask<T> implements HasDelegateFuture
+ {
+ private final RunnableFuture<T> delegate;
+
+ public ForwardingRunnableFuture(RunnableFuture<T> delegate, T result)
+ {
+ super(delegate, result);
+ this.delegate = delegate;
+ }
+
+ public Future<T> getDelegate()
+ {
+ return delegate;
+ }
+ }
}
diff --git a/src/java/org/apache/cassandra/concurrent/ImmediateExecutor.java b/src/java/org/apache/cassandra/concurrent/ImmediateExecutor.java
new file mode 100644
index 0000000..10c369c
--- /dev/null
+++ b/src/java/org/apache/cassandra/concurrent/ImmediateExecutor.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.concurrent;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.AbstractExecutorService;
+import java.util.concurrent.TimeUnit;
+
+public class ImmediateExecutor extends AbstractExecutorService implements LocalAwareExecutorService
+{
+ public static final ImmediateExecutor INSTANCE = new ImmediateExecutor();
+
+ private ImmediateExecutor() {}
+
+ public void execute(Runnable command, ExecutorLocals locals)
+ {
+ command.run();
+ }
+
+ public void maybeExecuteImmediately(Runnable command)
+ {
+ command.run();
+ }
+
+ public void execute(Runnable command)
+ {
+ command.run();
+ }
+
+ public int getActiveTaskCount() { return 0; }
+ public long getCompletedTaskCount() { return 0; }
+ public int getPendingTaskCount() { return 0; }
+ public int getCorePoolSize() { return 0; }
+ public int getMaximumPoolSize() { return 0; }
+ public void setCorePoolSize(int newCorePoolSize) { throw new IllegalArgumentException("Cannot resize ImmediateExecutor"); }
+ public void setMaximumPoolSize(int newMaximumPoolSize) { throw new IllegalArgumentException("Cannot resize ImmediateExecutor"); }
+
+ public void shutdown() { }
+ public List<Runnable> shutdownNow() { return Collections.emptyList(); }
+ public boolean isShutdown() { return false; }
+ public boolean isTerminated() { return false; }
+ public boolean awaitTermination(long timeout, TimeUnit unit) { return true; }
+}
diff --git a/src/java/org/apache/cassandra/concurrent/JMXConfigurableThreadPoolExecutor.java b/src/java/org/apache/cassandra/concurrent/JMXConfigurableThreadPoolExecutor.java
deleted file mode 100644
index 7fe6c07..0000000
--- a/src/java/org/apache/cassandra/concurrent/JMXConfigurableThreadPoolExecutor.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.concurrent;
-
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.TimeUnit;
-
-public class JMXConfigurableThreadPoolExecutor extends JMXEnabledThreadPoolExecutor implements JMXConfigurableThreadPoolExecutorMBean
-{
-
- public JMXConfigurableThreadPoolExecutor(int corePoolSize,
- long keepAliveTime,
- TimeUnit unit,
- BlockingQueue<Runnable> workQueue,
- NamedThreadFactory threadFactory,
- String jmxPath)
- {
- super(corePoolSize, keepAliveTime, unit, workQueue, threadFactory, jmxPath);
- }
-
-}
diff --git a/src/java/org/apache/cassandra/concurrent/JMXConfigurableThreadPoolExecutorMBean.java b/src/java/org/apache/cassandra/concurrent/JMXConfigurableThreadPoolExecutorMBean.java
deleted file mode 100644
index 948fb2c..0000000
--- a/src/java/org/apache/cassandra/concurrent/JMXConfigurableThreadPoolExecutorMBean.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.concurrent;
-
-public interface JMXConfigurableThreadPoolExecutorMBean extends JMXEnabledThreadPoolExecutorMBean
-{
- void setCorePoolSize(int n);
-
- int getCorePoolSize();
-}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/concurrent/JMXEnabledSingleThreadExecutor.java b/src/java/org/apache/cassandra/concurrent/JMXEnabledSingleThreadExecutor.java
new file mode 100644
index 0000000..ed54b3e
--- /dev/null
+++ b/src/java/org/apache/cassandra/concurrent/JMXEnabledSingleThreadExecutor.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.concurrent;
+
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.TimeUnit;
+
+public class JMXEnabledSingleThreadExecutor extends JMXEnabledThreadPoolExecutor
+{
+ public JMXEnabledSingleThreadExecutor(String threadPoolName, String jmxPath)
+ {
+ super(1, Integer.MAX_VALUE, TimeUnit.SECONDS, new LinkedBlockingQueue<>(), new SingleThreadFactory(threadPoolName), jmxPath);
+ }
+
+ @Override
+ public void setCoreThreads(int number)
+ {
+ throw new UnsupportedOperationException("Cannot change core pool size for single threaded executor.");
+ }
+
+ @Override
+ public void setMaximumThreads(int number)
+ {
+ throw new UnsupportedOperationException("Cannot change max threads for single threaded executor.");
+ }
+
+ @Override
+ public void setMaximumPoolSize(int newMaximumPoolSize)
+ {
+ setMaximumThreads(newMaximumPoolSize);
+ }
+
+ public boolean isExecutedBy(Thread test)
+ {
+ return getThreadFactory().thread == test;
+ }
+
+ public SingleThreadFactory getThreadFactory()
+ {
+ return (SingleThreadFactory) super.getThreadFactory();
+ }
+
+ public void setThreadFactory(ThreadFactory threadFactory)
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ private static class SingleThreadFactory extends NamedThreadFactory
+ {
+ private volatile Thread thread;
+ SingleThreadFactory(String id)
+ {
+ super(id);
+ }
+
+ @Override
+ public Thread newThread(Runnable r)
+ {
+ Thread thread = super.newThread(r);
+ this.thread = thread;
+ return thread;
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/concurrent/JMXEnabledThreadPoolExecutor.java b/src/java/org/apache/cassandra/concurrent/JMXEnabledThreadPoolExecutor.java
index ae51aff..4283d4f 100644
--- a/src/java/org/apache/cassandra/concurrent/JMXEnabledThreadPoolExecutor.java
+++ b/src/java/org/apache/cassandra/concurrent/JMXEnabledThreadPoolExecutor.java
@@ -20,6 +20,7 @@
import java.util.List;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.RejectedExecutionHandler;
import java.util.concurrent.TimeUnit;
import org.apache.cassandra.metrics.ThreadPoolMetrics;
@@ -76,15 +77,23 @@
{
super(corePoolSize, maxPoolSize, keepAliveTime, unit, workQueue, threadFactory);
super.prestartAllCoreThreads();
- metrics = new ThreadPoolMetrics(this, jmxPath, threadFactory.id);
+ metrics = new ThreadPoolMetrics(this, jmxPath, threadFactory.id).register();
mbeanName = "org.apache.cassandra." + jmxPath + ":type=" + threadFactory.id;
MBeanWrapper.instance.registerMBean(this, mbeanName);
}
- public JMXEnabledThreadPoolExecutor(Stage stage)
+ public JMXEnabledThreadPoolExecutor(int corePoolSize,
+ int maxPoolSize,
+ long keepAliveTime,
+ TimeUnit unit,
+ BlockingQueue<Runnable> workQueue,
+ NamedThreadFactory threadFactory,
+ String jmxPath,
+ RejectedExecutionHandler rejectedExecutionHandler)
{
- this(stage.getJmxName(), stage.getJmxType());
+ this(corePoolSize, maxPoolSize, keepAliveTime, unit, workQueue, threadFactory, jmxPath);
+ setRejectedExecutionHandler(rejectedExecutionHandler);
}
private void unregisterMBean()
@@ -119,9 +128,6 @@
return super.shutdownNow();
}
-
-
-
public int getTotalBlockedTasks()
{
return (int) metrics.totalBlocked.getCount();
@@ -132,27 +138,39 @@
return (int) metrics.currentBlocked.getCount();
}
+ @Deprecated
public int getCoreThreads()
{
return getCorePoolSize();
}
+ @Deprecated
public void setCoreThreads(int number)
{
setCorePoolSize(number);
}
+ @Deprecated
public int getMaximumThreads()
{
return getMaximumPoolSize();
}
+ @Deprecated
public void setMaximumThreads(int number)
{
setMaximumPoolSize(number);
}
@Override
+ public void setMaximumPoolSize(int newMaximumPoolSize)
+ {
+ if (newMaximumPoolSize < getCorePoolSize())
+ throw new IllegalArgumentException("maximum pool size cannot be less than core pool size");
+ super.setMaximumPoolSize(newMaximumPoolSize);
+ }
+
+ @Override
protected void onInitialRejection(Runnable task)
{
metrics.totalBlocked.inc();
diff --git a/src/java/org/apache/cassandra/concurrent/JMXEnabledThreadPoolExecutorMBean.java b/src/java/org/apache/cassandra/concurrent/JMXEnabledThreadPoolExecutorMBean.java
index fb964ae..c2959df 100644
--- a/src/java/org/apache/cassandra/concurrent/JMXEnabledThreadPoolExecutorMBean.java
+++ b/src/java/org/apache/cassandra/concurrent/JMXEnabledThreadPoolExecutorMBean.java
@@ -17,26 +17,33 @@
*/
package org.apache.cassandra.concurrent;
-
-public interface JMXEnabledThreadPoolExecutorMBean
+public interface JMXEnabledThreadPoolExecutorMBean extends ResizableThreadPool
{
/**
* Returns core pool size of thread pool.
+ * Deprecated, use getCorePoolSize instead.
*/
+ @Deprecated
public int getCoreThreads();
/**
* Allows user to resize core pool size of the thread pool.
+ * Deprecated, use setCorePoolSize instead.
*/
+ @Deprecated
public void setCoreThreads(int number);
/**
* Returns maximum pool size of thread pool.
+ * Deprecated, use getMaximumThreads instead.
*/
+ @Deprecated
public int getMaximumThreads();
/**
* Allows user to resize maximum size of the thread pool.
+ * Deprecated, use setMaximumThreads instead.
*/
+ @Deprecated
public void setMaximumThreads(int number);
}
diff --git a/src/java/org/apache/cassandra/concurrent/LocalAwareExecutorService.java b/src/java/org/apache/cassandra/concurrent/LocalAwareExecutorService.java
index 5577d59..d6ac8e4 100644
--- a/src/java/org/apache/cassandra/concurrent/LocalAwareExecutorService.java
+++ b/src/java/org/apache/cassandra/concurrent/LocalAwareExecutorService.java
@@ -23,12 +23,55 @@
import java.util.concurrent.ExecutorService;
-public interface LocalAwareExecutorService extends ExecutorService
+public interface LocalAwareExecutorService extends ExecutorService, ResizableThreadPool
{
// we need a way to inject a TraceState directly into the Executor context without going through
// the global Tracing sessions; see CASSANDRA-5668
- public void execute(Runnable command, ExecutorLocals locals);
+ void execute(Runnable command, ExecutorLocals locals);
// permits executing in the context of the submitting thread
- public void maybeExecuteImmediately(Runnable command);
+ void maybeExecuteImmediately(Runnable command);
+
+ /**
+ * Returns the approximate number of threads that are actively
+ * executing tasks.
+ *
+ * @return the number of threads
+ */
+ int getActiveTaskCount();
+
+ /**
+ * Returns the approximate total number of tasks that have
+ * completed execution. Because the states of tasks and threads
+ * may change dynamically during computation, the returned value
+ * is only an approximation, but one that does not ever decrease
+ * across successive calls.
+ *
+ * @return the number of tasks
+ */
+ long getCompletedTaskCount();
+
+ /**
+ * Returns the approximate total of tasks waiting to be executed.
+ * Because the states of tasks and threads
+ * may change dynamically during computation, the returned value
+ * is only an approximation, but one that does not ever decrease
+ * across successive calls.
+ *
+ * @return the number of tasks
+ */
+ int getPendingTaskCount();
+
+ default int getMaxTasksQueued()
+ {
+ return -1;
+ }
+
+ interface MaximumPoolSizeListener
+ {
+ /**
+ * Listener to follow changes to the maximum pool size
+ */
+ void onUpdateMaximumPoolSize(int maximumPoolSize);
+ }
}
diff --git a/src/java/org/apache/cassandra/concurrent/NamedThreadFactory.java b/src/java/org/apache/cassandra/concurrent/NamedThreadFactory.java
index 93d0c52..bcf686f 100644
--- a/src/java/org/apache/cassandra/concurrent/NamedThreadFactory.java
+++ b/src/java/org/apache/cassandra/concurrent/NamedThreadFactory.java
@@ -24,6 +24,7 @@
import io.netty.util.concurrent.FastThreadLocal;
import io.netty.util.concurrent.FastThreadLocalThread;
+import org.apache.cassandra.utils.memory.BufferPool;
/**
* This class is an implementation of the <i>ThreadFactory</i> interface. This
@@ -35,6 +36,10 @@
{
private static volatile String globalPrefix;
public static void setGlobalPrefix(String prefix) { globalPrefix = prefix; }
+ public static String globalPrefix() {
+ String prefix = globalPrefix;
+ return prefix == null ? "" : prefix;
+ }
public final String id;
private final int priority;
@@ -70,26 +75,6 @@
return thread;
}
- /**
- * Ensures that {@link FastThreadLocal#remove() FastThreadLocal.remove()} is called when the {@link Runnable#run()}
- * method of the given {@link Runnable} instance completes to ensure cleanup of {@link FastThreadLocal} instances.
- * This is especially important for direct byte buffers allocated locally for a thread.
- */
- public static Runnable threadLocalDeallocator(Runnable r)
- {
- return () ->
- {
- try
- {
- r.run();
- }
- finally
- {
- FastThreadLocal.removeAll();
- }
- };
- }
-
private static final AtomicInteger threadCounter = new AtomicInteger();
@VisibleForTesting
@@ -116,7 +101,7 @@
public static Thread createThread(ThreadGroup threadGroup, Runnable runnable, String name, boolean daemon)
{
String prefix = globalPrefix;
- Thread thread = new FastThreadLocalThread(threadGroup, threadLocalDeallocator(runnable), prefix != null ? prefix + name : name);
+ Thread thread = new FastThreadLocalThread(threadGroup, runnable, prefix != null ? prefix + name : name);
thread.setDaemon(daemon);
return thread;
}
diff --git a/src/java/org/apache/cassandra/concurrent/ResizableThreadPool.java b/src/java/org/apache/cassandra/concurrent/ResizableThreadPool.java
new file mode 100644
index 0000000..bd3b8ea
--- /dev/null
+++ b/src/java/org/apache/cassandra/concurrent/ResizableThreadPool.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.concurrent;
+
+public interface ResizableThreadPool
+{
+ /**
+ * Returns maximum pool size of thread pool.
+ */
+ public int getCorePoolSize();
+
+ /**
+ * Allows user to resize maximum size of the thread pool.
+ */
+ public void setCorePoolSize(int newCorePoolSize);
+
+ /**
+ * Returns maximum pool size of thread pool.
+ */
+ public int getMaximumPoolSize();
+
+ /**
+ * Allows user to resize maximum size of the thread pool.
+ */
+ public void setMaximumPoolSize(int newMaximumPoolSize);
+}
diff --git a/src/java/org/apache/cassandra/concurrent/SEPExecutor.java b/src/java/org/apache/cassandra/concurrent/SEPExecutor.java
index 323724a..ee939a2 100644
--- a/src/java/org/apache/cassandra/concurrent/SEPExecutor.java
+++ b/src/java/org/apache/cassandra/concurrent/SEPExecutor.java
@@ -21,26 +21,34 @@
import java.util.List;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
-import org.apache.cassandra.metrics.SEPMetrics;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.metrics.ThreadPoolMetrics;
+import org.apache.cassandra.utils.MBeanWrapper;
import org.apache.cassandra.utils.concurrent.SimpleCondition;
import org.apache.cassandra.utils.concurrent.WaitQueue;
import static org.apache.cassandra.concurrent.SEPWorker.Work;
-public class SEPExecutor extends AbstractLocalAwareExecutorService
+public class SEPExecutor extends AbstractLocalAwareExecutorService implements SEPExecutorMBean
{
+ private static final Logger logger = LoggerFactory.getLogger(SEPExecutor.class);
private final SharedExecutorPool pool;
- public final int maxWorkers;
+ private final AtomicInteger maximumPoolSize;
+ private final MaximumPoolSizeListener maximumPoolSizeListener;
public final String name;
- private final int maxTasksQueued;
- private final SEPMetrics metrics;
+ private final String mbeanName;
+ public final int maxTasksQueued;
+ private final ThreadPoolMetrics metrics;
// stores both a set of work permits and task permits:
// bottom 32 bits are number of queued tasks, in the range [0..maxTasksQueued] (initially 0)
- // top 32 bits are number of work permits available in the range [0..maxWorkers] (initially maxWorkers)
+ // top 32 bits are number of work permits available in the range [-resizeDelta..maximumPoolSize] (initially maximumPoolSize)
private final AtomicLong permits = new AtomicLong();
// producers wait on this when there is no room on the queue
@@ -53,14 +61,17 @@
// TODO: see if other queue implementations might improve throughput
protected final ConcurrentLinkedQueue<FutureTask<?>> tasks = new ConcurrentLinkedQueue<>();
- SEPExecutor(SharedExecutorPool pool, int maxWorkers, int maxTasksQueued, String jmxPath, String name)
+ SEPExecutor(SharedExecutorPool pool, int maximumPoolSize, MaximumPoolSizeListener maximumPoolSizeListener, int maxTasksQueued, String jmxPath, String name)
{
this.pool = pool;
this.name = name;
- this.maxWorkers = maxWorkers;
+ this.mbeanName = "org.apache.cassandra." + jmxPath + ":type=" + name;
+ this.maximumPoolSize = new AtomicInteger(maximumPoolSize);
+ this.maximumPoolSizeListener = maximumPoolSizeListener;
this.maxTasksQueued = maxTasksQueued;
- this.permits.set(combine(0, maxWorkers));
- this.metrics = new SEPMetrics(this, jmxPath, name);
+ this.permits.set(combine(0, maximumPoolSize));
+ this.metrics = new ThreadPoolMetrics(this, jmxPath, name).register();
+ MBeanWrapper.instance.registerMBean(this, mbeanName);
}
protected void onCompletion()
@@ -68,6 +79,12 @@
completedTasks.incrementAndGet();
}
+ @Override
+ public int getMaxTasksQueued()
+ {
+ return maxTasksQueued;
+ }
+
// schedules another worker for this pool if there is work outstanding and there are no spinning threads that
// will self-assign to it in the immediate future
boolean maybeSchedule()
@@ -129,21 +146,44 @@
}
}
+ public enum TakeTaskPermitResult
+ {
+ NONE_AVAILABLE, // No task permits available
+ TOOK_PERMIT, // Took a permit and reduced task permits
+ RETURNED_WORK_PERMIT // Detected pool shrinking and returned work permit ahead of SEPWorker exit.
+ };
+
// takes permission to perform a task, if any are available; once taken it is guaranteed
// that a proceeding call to tasks.poll() will return some work
- boolean takeTaskPermit()
+ TakeTaskPermitResult takeTaskPermit(boolean checkForWorkPermitOvercommit)
{
+ TakeTaskPermitResult result;
while (true)
{
long current = permits.get();
+ long updated;
+ int workPermits = workPermits(current);
int taskPermits = taskPermits(current);
- if (taskPermits == 0)
- return false;
- if (permits.compareAndSet(current, updateTaskPermits(current, taskPermits - 1)))
+ if (workPermits < 0 && checkForWorkPermitOvercommit)
+ {
+ // Work permits are negative when the pool is reducing in size. Atomically
+ // adjust the number of work permits so there is no race of multiple SEPWorkers
+ // exiting. On conflicting update, recheck.
+ result = TakeTaskPermitResult.RETURNED_WORK_PERMIT;
+ updated = updateWorkPermits(current, workPermits + 1);
+ }
+ else
+ {
+ if (taskPermits == 0)
+ return TakeTaskPermitResult.NONE_AVAILABLE;
+ result = TakeTaskPermitResult.TOOK_PERMIT;
+ updated = updateTaskPermits(current, taskPermits - 1);
+ }
+ if (permits.compareAndSet(current, updated))
{
if (taskPermits == maxTasksQueued && hasRoom.hasWaiters())
hasRoom.signalAll();
- return true;
+ return result;
}
}
}
@@ -157,7 +197,7 @@
long current = permits.get();
int workPermits = workPermits(current);
int taskPermits = taskPermits(current);
- if (workPermits == 0 || taskPermits == 0)
+ if (workPermits <= 0 || taskPermits == 0)
return false;
if (permits.compareAndSet(current, combine(taskPermits - taskDelta, workPermits - 1)))
{
@@ -176,11 +216,7 @@
long current = permits.get();
int workPermits = workPermits(current);
if (permits.compareAndSet(current, updateWorkPermits(current, workPermits + 1)))
- {
- if (shuttingDown && workPermits + 1 == maxWorkers)
- shutdown.signalAll();
- break;
- }
+ return;
}
}
@@ -210,23 +246,24 @@
public synchronized void shutdown()
{
+ if (shuttingDown)
+ return;
shuttingDown = true;
pool.executors.remove(this);
- if (getActiveCount() == 0 && getPendingTasks() == 0)
+ if (getActiveTaskCount() == 0)
shutdown.signalAll();
// release metrics
metrics.release();
+ MBeanWrapper.instance.unregisterMBean(mbeanName);
}
public synchronized List<Runnable> shutdownNow()
{
shutdown();
List<Runnable> aborted = new ArrayList<>();
- while (takeTaskPermit())
+ while (takeTaskPermit(false) == TakeTaskPermitResult.TOOK_PERMIT)
aborted.add(tasks.poll());
- if (getActiveCount() == 0)
- shutdown.signalAll();
return aborted;
}
@@ -246,19 +283,61 @@
return isTerminated();
}
- public long getPendingTasks()
+ @Override
+ public int getPendingTaskCount()
{
return taskPermits(permits.get());
}
- public long getCompletedTasks()
+ @Override
+ public long getCompletedTaskCount()
{
return completedTasks.get();
}
- public int getActiveCount()
+ public int getActiveTaskCount()
{
- return maxWorkers - workPermits(permits.get());
+ return maximumPoolSize.get() - workPermits(permits.get());
+ }
+
+ public int getCorePoolSize()
+ {
+ return 0;
+ }
+
+ public void setCorePoolSize(int newCorePoolSize)
+ {
+ throw new IllegalArgumentException("Cannot resize core pool size of SEPExecutor");
+ }
+
+ @Override
+ public int getMaximumPoolSize()
+ {
+ return maximumPoolSize.get();
+ }
+
+ @Override
+ public synchronized void setMaximumPoolSize(int newMaximumPoolSize)
+ {
+ final int oldMaximumPoolSize = maximumPoolSize.get();
+
+ if (newMaximumPoolSize < 0)
+ {
+ throw new IllegalArgumentException("Maximum number of workers must not be negative");
+ }
+
+ int deltaWorkPermits = newMaximumPoolSize - oldMaximumPoolSize;
+ if (!maximumPoolSize.compareAndSet(oldMaximumPoolSize, newMaximumPoolSize))
+ {
+ throw new IllegalStateException("Maximum pool size has been changed while resizing");
+ }
+
+ if (deltaWorkPermits == 0)
+ return;
+
+ permits.updateAndGet(cur -> updateWorkPermits(cur, workPermits(cur) + deltaWorkPermits));
+ logger.info("Resized {} maximum pool size from {} to {}", name, oldMaximumPoolSize, newMaximumPoolSize);
+ maximumPoolSizeListener.onUpdateMaximumPoolSize(newMaximumPoolSize);
}
private static int taskPermits(long both)
@@ -266,9 +345,9 @@
return (int) both;
}
- private static int workPermits(long both)
+ private static int workPermits(long both) // may be negative if resizing
{
- return (int) (both >>> 32);
+ return (int) (both >> 32); // sign extending right shift
}
private static long updateTaskPermits(long prev, int taskPermits)
diff --git a/src/java/org/apache/cassandra/concurrent/SEPExecutorMBean.java b/src/java/org/apache/cassandra/concurrent/SEPExecutorMBean.java
new file mode 100644
index 0000000..67de02e
--- /dev/null
+++ b/src/java/org/apache/cassandra/concurrent/SEPExecutorMBean.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.concurrent;
+
+public interface SEPExecutorMBean extends ResizableThreadPool
+{
+}
diff --git a/src/java/org/apache/cassandra/concurrent/SEPWorker.java b/src/java/org/apache/cassandra/concurrent/SEPWorker.java
index cd7af63..de5185d 100644
--- a/src/java/org/apache/cassandra/concurrent/SEPWorker.java
+++ b/src/java/org/apache/cassandra/concurrent/SEPWorker.java
@@ -17,6 +17,7 @@
*/
package org.apache.cassandra.concurrent;
+import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
import java.util.concurrent.locks.LockSupport;
@@ -27,6 +28,9 @@
import io.netty.util.concurrent.FastThreadLocalThread;
import org.apache.cassandra.utils.JVMStabilityInspector;
+import static org.apache.cassandra.concurrent.SEPExecutor.TakeTaskPermitResult.RETURNED_WORK_PERMIT;
+import static org.apache.cassandra.concurrent.SEPExecutor.TakeTaskPermitResult.TOOK_PERMIT;
+
final class SEPWorker extends AtomicReference<SEPWorker.Work> implements Runnable
{
private static final Logger logger = LoggerFactory.getLogger(SEPWorker.class);
@@ -102,6 +106,8 @@
// if we do have tasks assigned, nobody will change our state so we can simply set it to WORKING
// (which is also a state that will never be interrupted externally)
set(Work.WORKING);
+ boolean shutdown;
+ SEPExecutor.TakeTaskPermitResult status = null; // make sure set if shutdown check short circuits
while (true)
{
// before we process any task, we maybe schedule a new worker _to our executor only_; this
@@ -113,16 +119,28 @@
task.run();
task = null;
- // if we're shutting down, or we fail to take a permit, we don't perform any more work
- if (!assigned.takeTaskPermit())
+ if (shutdown = assigned.shuttingDown)
break;
+
+ if (TOOK_PERMIT != (status = assigned.takeTaskPermit(true)))
+ break;
+
task = assigned.tasks.poll();
}
// return our work permit, and maybe signal shutdown
- assigned.returnWorkPermit();
+ if (status != RETURNED_WORK_PERMIT)
+ assigned.returnWorkPermit();
+
+ if (shutdown)
+ {
+ if (assigned.getActiveTaskCount() == 0)
+ assigned.shutdown.signalAll();
+ return;
+ }
assigned = null;
+
// try to immediately reassign ourselves some work; if we fail, start spinning
if (!selfAssign())
startSpinning();
@@ -131,24 +149,22 @@
catch (Throwable t)
{
JVMStabilityInspector.inspectThrowable(t);
- if (task != null)
- logger.error("Failed to execute task, unexpected exception killed worker: {}", t);
- else
- logger.error("Unexpected exception killed worker: {}", t);
- }
- finally
- {
- if (assigned != null)
- assigned.returnWorkPermit();
-
- do
+ while (true)
{
if (get().assigned != null)
{
- get().assigned.returnWorkPermit();
+ assigned = get().assigned;
set(Work.WORKING);
}
- } while (!assign(Work.STOPPED, true));
+ if (assign(Work.STOPPED, true))
+ break;
+ }
+ if (assigned != null)
+ assigned.returnWorkPermit();
+ if (task != null)
+ logger.error("Failed to execute task, unexpected exception killed worker", t);
+ else
+ logger.error("Unexpected exception killed worker", t);
}
}
@@ -238,7 +254,7 @@
// we should always have a thread about to wake up, but most threads are sleeping
long sleep = 10000L * pool.spinningCount.get();
sleep = Math.min(1000000, sleep);
- sleep *= Math.random();
+ sleep *= ThreadLocalRandom.current().nextDouble();
sleep = Math.max(10000, sleep);
long start = System.nanoTime();
diff --git a/src/java/org/apache/cassandra/concurrent/ScheduledExecutors.java b/src/java/org/apache/cassandra/concurrent/ScheduledExecutors.java
index 3da4569..c549c4d 100644
--- a/src/java/org/apache/cassandra/concurrent/ScheduledExecutors.java
+++ b/src/java/org/apache/cassandra/concurrent/ScheduledExecutors.java
@@ -17,11 +17,15 @@
*/
package org.apache.cassandra.concurrent;
+import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableList;
+
+import org.apache.cassandra.utils.ExecutorUtils;
import org.apache.cassandra.utils.ExecutorUtils;
diff --git a/src/java/org/apache/cassandra/concurrent/SharedExecutorPool.java b/src/java/org/apache/cassandra/concurrent/SharedExecutorPool.java
index f309b46..28a994c 100644
--- a/src/java/org/apache/cassandra/concurrent/SharedExecutorPool.java
+++ b/src/java/org/apache/cassandra/concurrent/SharedExecutorPool.java
@@ -17,12 +17,12 @@
*/
package org.apache.cassandra.concurrent;
-import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentSkipListMap;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.LockSupport;
@@ -110,15 +110,19 @@
public synchronized LocalAwareExecutorService newExecutor(int maxConcurrency, int maxQueuedTasks, String jmxPath, String name)
{
- SEPExecutor executor = new SEPExecutor(this, maxConcurrency, maxQueuedTasks, jmxPath, name);
+ return newExecutor(maxConcurrency, i -> {}, maxQueuedTasks, jmxPath, name);
+ }
+
+ public LocalAwareExecutorService newExecutor(int maxConcurrency, LocalAwareExecutorService.MaximumPoolSizeListener maximumPoolSizeListener, int maxQueuedTasks, String jmxPath, String name)
+ {
+ SEPExecutor executor = new SEPExecutor(this, maxConcurrency, maximumPoolSizeListener, maxQueuedTasks, jmxPath, name);
executors.add(executor);
return executor;
}
- public synchronized void shutdownAndWait(long timeout, TimeUnit unit) throws InterruptedException
+ public synchronized void shutdownAndWait(long timeout, TimeUnit unit) throws InterruptedException, TimeoutException
{
shuttingDown = true;
- List<SEPExecutor> executors = new ArrayList<>(this.executors);
for (SEPExecutor executor : executors)
executor.shutdownNow();
@@ -126,10 +130,14 @@
long until = System.nanoTime() + unit.toNanos(timeout);
for (SEPExecutor executor : executors)
+ {
executor.shutdown.await(until - System.nanoTime(), TimeUnit.NANOSECONDS);
+ if (!executor.isTerminated())
+ throw new TimeoutException(executor.name + " not terminated");
+ }
}
- private void terminateWorkers()
+ void terminateWorkers()
{
assert shuttingDown;
diff --git a/src/java/org/apache/cassandra/concurrent/Stage.java b/src/java/org/apache/cassandra/concurrent/Stage.java
index ccb1565..d2ebe8d 100644
--- a/src/java/org/apache/cassandra/concurrent/Stage.java
+++ b/src/java/org/apache/cassandra/concurrent/Stage.java
@@ -18,66 +18,261 @@
package org.apache.cassandra.concurrent;
import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.RejectedExecutionHandler;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.function.IntSupplier;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
-import com.google.common.base.Predicate;
-import com.google.common.collect.Iterables;
+import com.google.common.annotations.VisibleForTesting;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.net.Verb;
+import org.apache.cassandra.utils.ExecutorUtils;
+
+import org.apache.cassandra.utils.FBUtilities;
+
+import static java.util.stream.Collectors.toMap;
public enum Stage
{
- READ,
- MUTATION,
- COUNTER_MUTATION,
- VIEW_MUTATION,
- GOSSIP,
- REQUEST_RESPONSE,
- ANTI_ENTROPY,
- MIGRATION,
- MISC,
- TRACING,
- INTERNAL_RESPONSE,
- READ_REPAIR;
+ READ ("ReadStage", "request", DatabaseDescriptor::getConcurrentReaders, DatabaseDescriptor::setConcurrentReaders, Stage::multiThreadedLowSignalStage),
+ MUTATION ("MutationStage", "request", DatabaseDescriptor::getConcurrentWriters, DatabaseDescriptor::setConcurrentWriters, Stage::multiThreadedLowSignalStage),
+ COUNTER_MUTATION ("CounterMutationStage", "request", DatabaseDescriptor::getConcurrentCounterWriters, DatabaseDescriptor::setConcurrentCounterWriters, Stage::multiThreadedLowSignalStage),
+ VIEW_MUTATION ("ViewMutationStage", "request", DatabaseDescriptor::getConcurrentViewWriters, DatabaseDescriptor::setConcurrentViewWriters, Stage::multiThreadedLowSignalStage),
+ GOSSIP ("GossipStage", "internal", () -> 1, null, Stage::singleThreadedStage),
+ REQUEST_RESPONSE ("RequestResponseStage", "request", FBUtilities::getAvailableProcessors, null, Stage::multiThreadedLowSignalStage),
+ ANTI_ENTROPY ("AntiEntropyStage", "internal", () -> 1, null, Stage::singleThreadedStage),
+ MIGRATION ("MigrationStage", "internal", () -> 1, null, Stage::singleThreadedStage),
+ MISC ("MiscStage", "internal", () -> 1, null, Stage::singleThreadedStage),
+ TRACING ("TracingStage", "internal", () -> 1, null, Stage::tracingExecutor),
+ INTERNAL_RESPONSE ("InternalResponseStage", "internal", FBUtilities::getAvailableProcessors, null, Stage::multiThreadedStage),
+ IMMEDIATE ("ImmediateStage", "internal", () -> 0, null, Stage::immediateExecutor);
- public static Iterable<Stage> jmxEnabledStages()
+
+ public static final long KEEP_ALIVE_SECONDS = 60; // seconds to keep "extra" threads alive for when idle
+ public final String jmxName;
+ private final Supplier<LocalAwareExecutorService> initialiser;
+ private volatile LocalAwareExecutorService executor = null;
+
+ Stage(String jmxName, String jmxType, IntSupplier numThreads, LocalAwareExecutorService.MaximumPoolSizeListener onSetMaximumPoolSize, ExecutorServiceInitialiser initialiser)
{
- return Iterables.filter(Arrays.asList(values()), new Predicate<Stage>()
+ this.jmxName = jmxName;
+ this.initialiser = () -> initialiser.init(jmxName, jmxType, numThreads.getAsInt(), onSetMaximumPoolSize);
+ }
+
+ private static String normalizeName(String stageName)
+ {
+ // Handle discrepancy between JMX names and actual pool names
+ String upperStageName = stageName.toUpperCase();
+ if (upperStageName.endsWith("STAGE"))
{
- public boolean apply(Stage stage)
+ upperStageName = upperStageName.substring(0, stageName.length() - 5);
+ }
+ return upperStageName;
+ }
+
+ private static final Map<String,Stage> nameMap = Arrays.stream(values())
+ .collect(toMap(s -> Stage.normalizeName(s.jmxName),
+ s -> s));
+
+ public static Stage fromPoolName(String stageName)
+ {
+ String upperStageName = normalizeName(stageName);
+
+ Stage result = nameMap.get(upperStageName);
+ if (result != null)
+ return result;
+
+ try
+ {
+ return valueOf(upperStageName);
+ }
+ catch (IllegalArgumentException e)
+ {
+ switch(upperStageName) // Handle discrepancy between configuration file and stage names
{
- return stage != TRACING;
+ case "CONCURRENT_READS":
+ return READ;
+ case "CONCURRENT_WRITERS":
+ return MUTATION;
+ case "CONCURRENT_COUNTER_WRITES":
+ return COUNTER_MUTATION;
+ case "CONCURRENT_MATERIALIZED_VIEW_WRITES":
+ return VIEW_MUTATION;
+ default:
+ throw new IllegalStateException("Must be one of " + Arrays.stream(values())
+ .map(Enum::toString)
+ .collect(Collectors.joining(",")));
}
- });
- }
-
- public String getJmxType()
- {
- switch (this)
- {
- case ANTI_ENTROPY:
- case GOSSIP:
- case MIGRATION:
- case MISC:
- case TRACING:
- case INTERNAL_RESPONSE:
- return "internal";
- case MUTATION:
- case COUNTER_MUTATION:
- case VIEW_MUTATION:
- case READ:
- case REQUEST_RESPONSE:
- case READ_REPAIR:
- return "request";
- default:
- throw new AssertionError("Unknown stage " + this);
}
}
- public String getJmxName()
+ // Convenience functions to execute on this stage
+ public void execute(Runnable command) { executor().execute(command); }
+ public void execute(Runnable command, ExecutorLocals locals) { executor().execute(command, locals); }
+ public void maybeExecuteImmediately(Runnable command) { executor().maybeExecuteImmediately(command); };
+ public <T> Future<T> submit(Callable<T> task) { return executor().submit(task); }
+ public Future<?> submit(Runnable task) { return executor().submit(task); }
+ public <T> Future<T> submit(Runnable task, T result) { return executor().submit(task, result); }
+
+ public LocalAwareExecutorService executor()
{
- String name = "";
- for (String word : toString().split("_"))
+ if (executor == null)
{
- name += word.substring(0, 1) + word.substring(1).toLowerCase();
+ synchronized (this)
+ {
+ if (executor == null)
+ {
+ executor = initialiser.get();
+ }
+ }
}
- return name + "Stage";
+ return executor;
+ }
+
+ private static List<ExecutorService> executors()
+ {
+ return Stream.of(Stage.values())
+ .map(Stage::executor)
+ .collect(Collectors.toList());
+ }
+
+ /**
+ * This method shuts down all registered stages.
+ */
+ public static void shutdownNow()
+ {
+ ExecutorUtils.shutdownNow(executors());
+ }
+
+ @VisibleForTesting
+ public static void shutdownAndWait(long timeout, TimeUnit units) throws InterruptedException, TimeoutException
+ {
+ List<ExecutorService> executors = executors();
+ ExecutorUtils.shutdownNow(executors);
+ ExecutorUtils.awaitTermination(timeout, units, executors);
+ }
+
+ static LocalAwareExecutorService tracingExecutor(String jmxName, String jmxType, int numThreads, LocalAwareExecutorService.MaximumPoolSizeListener onSetMaximumPoolSize)
+ {
+ RejectedExecutionHandler reh = (r, executor) -> MessagingService.instance().metrics.recordSelfDroppedMessage(Verb._TRACE);
+ return new TracingExecutor(1,
+ 1,
+ KEEP_ALIVE_SECONDS,
+ TimeUnit.SECONDS,
+ new ArrayBlockingQueue<>(1000),
+ new NamedThreadFactory(jmxName),
+ reh);
+ }
+
+ static LocalAwareExecutorService multiThreadedStage(String jmxName, String jmxType, int numThreads, LocalAwareExecutorService.MaximumPoolSizeListener onSetMaximumPoolSize)
+ {
+ return new JMXEnabledThreadPoolExecutor(numThreads,
+ KEEP_ALIVE_SECONDS,
+ TimeUnit.SECONDS,
+ new LinkedBlockingQueue<>(),
+ new NamedThreadFactory(jmxName),
+ jmxType);
+ }
+
+ static LocalAwareExecutorService multiThreadedLowSignalStage(String jmxName, String jmxType, int numThreads, LocalAwareExecutorService.MaximumPoolSizeListener onSetMaximumPoolSize)
+ {
+ return SharedExecutorPool.SHARED.newExecutor(numThreads, onSetMaximumPoolSize, Integer.MAX_VALUE, jmxType, jmxName);
+ }
+
+ static LocalAwareExecutorService singleThreadedStage(String jmxName, String jmxType, int numThreads, LocalAwareExecutorService.MaximumPoolSizeListener onSetMaximumPoolSize)
+ {
+ return new JMXEnabledSingleThreadExecutor(jmxName, jmxType);
+ }
+
+ static LocalAwareExecutorService immediateExecutor(String jmxName, String jmxType, int numThreads, LocalAwareExecutorService.MaximumPoolSizeListener onSetMaximumPoolSize)
+ {
+ return ImmediateExecutor.INSTANCE;
+ }
+
+ @FunctionalInterface
+ public interface ExecutorServiceInitialiser
+ {
+ public LocalAwareExecutorService init(String jmxName, String jmxType, int numThreads, LocalAwareExecutorService.MaximumPoolSizeListener onSetMaximumPoolSize);
+ }
+
+ /**
+ * Returns core thread pool size
+ */
+ public int getCorePoolSize()
+ {
+ return executor().getCorePoolSize();
+ }
+
+ /**
+ * Allows user to resize core thread pool size
+ */
+ public void setCorePoolSize(int newCorePoolSize)
+ {
+ executor().setCorePoolSize(newCorePoolSize);
+ }
+
+ /**
+ * Returns maximum pool size of thread pool.
+ */
+ public int getMaximumPoolSize()
+ {
+ return executor().getMaximumPoolSize();
+ }
+
+ /**
+ * Allows user to resize maximum size of the thread pool.
+ */
+ public void setMaximumPoolSize(int newMaximumPoolSize)
+ {
+ executor().setMaximumPoolSize(newMaximumPoolSize);
+ }
+
+ /**
+ * The executor used for tracing.
+ */
+ private static class TracingExecutor extends ThreadPoolExecutor implements LocalAwareExecutorService
+ {
+ TracingExecutor(int corePoolSize, int maximumPoolSize, long keepAliveTime, TimeUnit unit, BlockingQueue<Runnable> workQueue, ThreadFactory threadFactory, RejectedExecutionHandler handler)
+ {
+ super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, threadFactory, handler);
+ }
+
+ public void execute(Runnable command, ExecutorLocals locals)
+ {
+ assert locals == null;
+ super.execute(command);
+ }
+
+ public void maybeExecuteImmediately(Runnable command)
+ {
+ execute(command);
+ }
+
+ @Override
+ public int getActiveTaskCount()
+ {
+ return getActiveCount();
+ }
+
+ @Override
+ public int getPendingTaskCount()
+ {
+ return getQueue().size();
+ }
}
}
diff --git a/src/java/org/apache/cassandra/concurrent/StageManager.java b/src/java/org/apache/cassandra/concurrent/StageManager.java
deleted file mode 100644
index 857c5b7..0000000
--- a/src/java/org/apache/cassandra/concurrent/StageManager.java
+++ /dev/null
@@ -1,144 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.concurrent;
-
-import java.util.EnumMap;
-import java.util.concurrent.*;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.cassandra.net.MessagingService;
-import org.apache.cassandra.utils.ExecutorUtils;
-import org.apache.cassandra.utils.FBUtilities;
-
-import static org.apache.cassandra.config.DatabaseDescriptor.*;
-
-
-/**
- * This class manages executor services for Messages recieved: each Message requests
- * running on a specific "stage" for concurrency control; hence the Map approach,
- * even though stages (executors) are not created dynamically.
- */
-public class StageManager
-{
- private static final Logger logger = LoggerFactory.getLogger(StageManager.class);
-
- private static final EnumMap<Stage, LocalAwareExecutorService> stages = new EnumMap<Stage, LocalAwareExecutorService>(Stage.class);
-
- public static final long KEEPALIVE = 60; // seconds to keep "extra" threads alive for when idle
-
- static
- {
- stages.put(Stage.MUTATION, multiThreadedLowSignalStage(Stage.MUTATION, getConcurrentWriters()));
- stages.put(Stage.COUNTER_MUTATION, multiThreadedLowSignalStage(Stage.COUNTER_MUTATION, getConcurrentCounterWriters()));
- stages.put(Stage.VIEW_MUTATION, multiThreadedLowSignalStage(Stage.VIEW_MUTATION, getConcurrentViewWriters()));
- stages.put(Stage.READ, multiThreadedLowSignalStage(Stage.READ, getConcurrentReaders()));
- stages.put(Stage.REQUEST_RESPONSE, multiThreadedLowSignalStage(Stage.REQUEST_RESPONSE, FBUtilities.getAvailableProcessors()));
- stages.put(Stage.INTERNAL_RESPONSE, multiThreadedStage(Stage.INTERNAL_RESPONSE, FBUtilities.getAvailableProcessors()));
- // the rest are all single-threaded
- stages.put(Stage.GOSSIP, new JMXEnabledThreadPoolExecutor(Stage.GOSSIP));
- stages.put(Stage.ANTI_ENTROPY, new JMXEnabledThreadPoolExecutor(Stage.ANTI_ENTROPY));
- stages.put(Stage.MIGRATION, new JMXEnabledThreadPoolExecutor(Stage.MIGRATION));
- stages.put(Stage.MISC, new JMXEnabledThreadPoolExecutor(Stage.MISC));
- stages.put(Stage.READ_REPAIR, multiThreadedStage(Stage.READ_REPAIR, FBUtilities.getAvailableProcessors()));
- stages.put(Stage.TRACING, tracingExecutor());
- }
-
- private static LocalAwareExecutorService tracingExecutor()
- {
- RejectedExecutionHandler reh = new RejectedExecutionHandler()
- {
- public void rejectedExecution(Runnable r, ThreadPoolExecutor executor)
- {
- MessagingService.instance().incrementDroppedMessages(MessagingService.Verb._TRACE);
- }
- };
- return new TracingExecutor(1,
- 1,
- KEEPALIVE,
- TimeUnit.SECONDS,
- new ArrayBlockingQueue<Runnable>(1000),
- new NamedThreadFactory(Stage.TRACING.getJmxName()),
- reh);
- }
-
- private static JMXEnabledThreadPoolExecutor multiThreadedStage(Stage stage, int numThreads)
- {
- return new JMXEnabledThreadPoolExecutor(numThreads,
- KEEPALIVE,
- TimeUnit.SECONDS,
- new LinkedBlockingQueue<Runnable>(),
- new NamedThreadFactory(stage.getJmxName()),
- stage.getJmxType());
- }
-
- private static LocalAwareExecutorService multiThreadedLowSignalStage(Stage stage, int numThreads)
- {
- return SharedExecutorPool.SHARED.newExecutor(numThreads, Integer.MAX_VALUE, stage.getJmxType(), stage.getJmxName());
- }
-
- /**
- * Retrieve a stage from the StageManager
- * @param stage name of the stage to be retrieved.
- */
- public static LocalAwareExecutorService getStage(Stage stage)
- {
- return stages.get(stage);
- }
-
- /**
- * This method shuts down all registered stages.
- */
- public static void shutdownNow()
- {
- for (Stage stage : Stage.values())
- {
- StageManager.stages.get(stage).shutdownNow();
- }
- }
-
- @VisibleForTesting
- public static void shutdownAndWait(long timeout, TimeUnit unit) throws InterruptedException, TimeoutException
- {
- ExecutorUtils.shutdownNowAndWait(timeout, unit, StageManager.stages.values());
- }
-
- /**
- * The executor used for tracing.
- */
- private static class TracingExecutor extends ThreadPoolExecutor implements LocalAwareExecutorService
- {
- public TracingExecutor(int corePoolSize, int maximumPoolSize, long keepAliveTime, TimeUnit unit, BlockingQueue<Runnable> workQueue, ThreadFactory threadFactory, RejectedExecutionHandler handler)
- {
- super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, threadFactory, handler);
- }
-
- public void execute(Runnable command, ExecutorLocals locals)
- {
- assert locals == null;
- super.execute(command);
- }
-
- public void maybeExecuteImmediately(Runnable command)
- {
- execute(command);
- }
- }
-}
diff --git a/src/java/org/apache/cassandra/config/CFMetaData.java b/src/java/org/apache/cassandra/config/CFMetaData.java
deleted file mode 100644
index 7f6a1b9..0000000
--- a/src/java/org/apache/cassandra/config/CFMetaData.java
+++ /dev/null
@@ -1,1644 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.config;
-
-import java.io.IOException;
-import java.lang.reflect.Constructor;
-import java.lang.reflect.InvocationTargetException;
-import java.nio.ByteBuffer;
-import java.util.*;
-import java.util.concurrent.ThreadLocalRandom;
-import java.util.concurrent.TimeUnit;
-import java.util.regex.Pattern;
-import java.util.stream.Collectors;
-
-import javax.annotation.Nullable;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.MoreObjects;
-import com.google.common.base.Objects;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Iterators;
-import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
-import org.apache.commons.lang3.ArrayUtils;
-import org.apache.commons.lang3.builder.HashCodeBuilder;
-import org.apache.commons.lang3.builder.ToStringBuilder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.cassandra.auth.DataResource;
-import org.apache.cassandra.cql3.ColumnIdentifier;
-import org.apache.cassandra.cql3.QueryProcessor;
-import org.apache.cassandra.cql3.SuperColumnCompatibility;
-import org.apache.cassandra.cql3.statements.CFStatement;
-import org.apache.cassandra.cql3.statements.CreateTableStatement;
-import org.apache.cassandra.db.*;
-import org.apache.cassandra.db.compaction.AbstractCompactionStrategy;
-import org.apache.cassandra.db.filter.ColumnFilter;
-import org.apache.cassandra.db.marshal.*;
-import org.apache.cassandra.dht.IPartitioner;
-import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.exceptions.InvalidRequestException;
-import org.apache.cassandra.io.util.DataInputPlus;
-import org.apache.cassandra.io.util.DataOutputPlus;
-import org.apache.cassandra.schema.*;
-import org.apache.cassandra.utils.*;
-import org.github.jamm.Unmetered;
-
-/**
- * This class can be tricky to modify. Please read http://wiki.apache.org/cassandra/ConfigurationNotes for how to do so safely.
- */
-@Unmetered
-public final class CFMetaData
-{
- public enum Flag
- {
- SUPER, COUNTER, DENSE, COMPOUND
- }
-
- private static final Pattern PATTERN_WORD_CHARS = Pattern.compile("\\w+");
-
- private static final Logger logger = LoggerFactory.getLogger(CFMetaData.class);
-
- public static final Serializer serializer = new Serializer();
-
- //REQUIRED
- public final UUID cfId; // internal id, never exposed to user
- public final String ksName; // name of keyspace
- public final String cfName; // name of this column family
- public final Pair<String, String> ksAndCFName;
- public final byte[] ksAndCFBytes;
-
- private final boolean isCounter;
- private final boolean isView;
- private final boolean isIndex;
-
- public volatile ClusteringComparator comparator; // bytes, long, timeuuid, utf8, etc. This is built directly from clusteringColumns
- public final IPartitioner partitioner; // partitioner the table uses
- private volatile AbstractType<?> keyValidator;
-
- private final Serializers serializers;
-
- // non-final, for now
- private volatile ImmutableSet<Flag> flags;
- private volatile boolean isDense;
- private volatile boolean isCompound;
- private volatile boolean isSuper;
-
- public volatile TableParams params = TableParams.DEFAULT;
-
- private volatile Map<ByteBuffer, DroppedColumn> droppedColumns = new HashMap<>();
- private volatile Triggers triggers = Triggers.none();
- private volatile Indexes indexes = Indexes.none();
-
- /*
- * All CQL3 columns definition are stored in the columnMetadata map.
- * On top of that, we keep separated collection of each kind of definition, to
- * 1) allow easy access to each kind and 2) for the partition key and
- * clustering key ones, those list are ordered by the "component index" of the
- * elements.
- */
- private volatile Map<ByteBuffer, ColumnDefinition> columnMetadata = new HashMap<>();
- private volatile List<ColumnDefinition> partitionKeyColumns; // Always of size keyValidator.componentsCount, null padded if necessary
- private volatile List<ColumnDefinition> clusteringColumns; // Of size comparator.componentsCount or comparator.componentsCount -1, null padded if necessary
- private volatile PartitionColumns partitionColumns; // Always non-PK, non-clustering columns
-
- // For dense tables, this alias the single non-PK column the table contains (since it can only have one). We keep
- // that as convenience to access that column more easily (but we could replace calls by partitionColumns().iterator().next()
- // for those tables in practice).
- private volatile ColumnDefinition compactValueColumn;
-
- private volatile Set<ColumnDefinition> hiddenColumns;
-
- public final DataResource resource;
-
- //For hot path serialization it's often easier to store this info here
- private volatile ColumnFilter allColumnFilter;
-
- /**
- * These two columns are "virtual" (e.g. not persisted together with schema).
- *
- * They are stored here to avoid re-creating during SELECT and UPDATE queries, where
- * they are used to allow presenting supercolumn families in the CQL-compatible
- * format. See {@link SuperColumnCompatibility} for more details.
- **/
- private volatile ColumnDefinition superCfKeyColumn;
- private volatile ColumnDefinition superCfValueColumn;
-
- /** Caches a non-compact version of the metadata for compact tables to be used with the NO_COMPACT protocol option. */
- private volatile CFMetaData nonCompactCopy = null;
-
- public boolean isSuperColumnKeyColumn(ColumnDefinition cd)
- {
- return cd.name.equals(superCfKeyColumn.name);
- }
-
- public boolean isSuperColumnValueColumn(ColumnDefinition cd)
- {
- return cd.name.equals(superCfValueColumn.name);
- }
-
- public ColumnDefinition superColumnValueColumn()
- {
- return superCfValueColumn;
- }
-
- public ColumnDefinition superColumnKeyColumn() { return superCfKeyColumn; }
-
- /*
- * All of these methods will go away once CFMetaData becomes completely immutable.
- */
- public CFMetaData params(TableParams params)
- {
- this.params = params;
- return this;
- }
-
- public CFMetaData bloomFilterFpChance(double prop)
- {
- params = TableParams.builder(params).bloomFilterFpChance(prop).build();
- return this;
- }
-
- public CFMetaData caching(CachingParams prop)
- {
- params = TableParams.builder(params).caching(prop).build();
- return this;
- }
-
- public CFMetaData comment(String prop)
- {
- params = TableParams.builder(params).comment(prop).build();
- return this;
- }
-
- public CFMetaData compaction(CompactionParams prop)
- {
- params = TableParams.builder(params).compaction(prop).build();
- return this;
- }
-
- public CFMetaData compression(CompressionParams prop)
- {
- params = TableParams.builder(params).compression(prop).build();
- return this;
- }
-
- public CFMetaData dcLocalReadRepairChance(double prop)
- {
- params = TableParams.builder(params).dcLocalReadRepairChance(prop).build();
- return this;
- }
-
- public CFMetaData defaultTimeToLive(int prop)
- {
- params = TableParams.builder(params).defaultTimeToLive(prop).build();
- return this;
- }
-
- public CFMetaData gcGraceSeconds(int prop)
- {
- params = TableParams.builder(params).gcGraceSeconds(prop).build();
- return this;
- }
-
- public CFMetaData maxIndexInterval(int prop)
- {
- params = TableParams.builder(params).maxIndexInterval(prop).build();
- return this;
- }
-
- public CFMetaData memtableFlushPeriod(int prop)
- {
- params = TableParams.builder(params).memtableFlushPeriodInMs(prop).build();
- return this;
- }
-
- public CFMetaData minIndexInterval(int prop)
- {
- params = TableParams.builder(params).minIndexInterval(prop).build();
- return this;
- }
-
- public CFMetaData readRepairChance(double prop)
- {
- params = TableParams.builder(params).readRepairChance(prop).build();
- return this;
- }
-
- public CFMetaData crcCheckChance(double prop)
- {
- params = TableParams.builder(params).crcCheckChance(prop).build();
- return this;
- }
-
- public CFMetaData speculativeRetry(SpeculativeRetryParam prop)
- {
- params = TableParams.builder(params).speculativeRetry(prop).build();
- return this;
- }
-
- public CFMetaData extensions(Map<String, ByteBuffer> extensions)
- {
- params = TableParams.builder(params).extensions(extensions).build();
- return this;
- }
-
- public CFMetaData droppedColumns(Map<ByteBuffer, DroppedColumn> cols)
- {
- droppedColumns = cols;
- return this;
- }
-
- public CFMetaData triggers(Triggers prop)
- {
- triggers = prop;
- return this;
- }
-
- public CFMetaData indexes(Indexes indexes)
- {
- this.indexes = indexes;
- return this;
- }
-
- private CFMetaData(String keyspace,
- String name,
- UUID cfId,
- boolean isSuper,
- boolean isCounter,
- boolean isDense,
- boolean isCompound,
- boolean isView,
- List<ColumnDefinition> partitionKeyColumns,
- List<ColumnDefinition> clusteringColumns,
- PartitionColumns partitionColumns,
- IPartitioner partitioner,
- ColumnDefinition superCfKeyColumn,
- ColumnDefinition superCfValueColumn)
- {
- this.cfId = cfId;
- this.ksName = keyspace;
- this.cfName = name;
- ksAndCFName = Pair.create(keyspace, name);
- byte[] ksBytes = FBUtilities.toWriteUTFBytes(ksName);
- byte[] cfBytes = FBUtilities.toWriteUTFBytes(cfName);
- ksAndCFBytes = Arrays.copyOf(ksBytes, ksBytes.length + cfBytes.length);
- System.arraycopy(cfBytes, 0, ksAndCFBytes, ksBytes.length, cfBytes.length);
-
- this.isDense = isSuper ? (isDense || SuperColumnCompatibility.recalculateIsDense(partitionColumns.regulars)) : isDense;
-
- this.isCompound = isCompound;
- this.isSuper = isSuper;
- this.isCounter = isCounter;
- this.isView = isView;
-
- EnumSet<Flag> flags = EnumSet.noneOf(Flag.class);
- if (isSuper)
- flags.add(Flag.SUPER);
- if (isCounter)
- flags.add(Flag.COUNTER);
- if (isDense)
- flags.add(Flag.DENSE);
- if (isCompound)
- flags.add(Flag.COMPOUND);
- this.flags = Sets.immutableEnumSet(flags);
-
- isIndex = cfName.contains(".");
-
- assert partitioner != null : "This assertion failure is probably due to accessing Schema.instance " +
- "from client-mode tools - See CASSANDRA-8143.";
- this.partitioner = partitioner;
-
- // A compact table should always have a clustering
- assert isCQLTable() || !clusteringColumns.isEmpty() : String.format("For table %s.%s, isDense=%b, isCompound=%b, clustering=%s", ksName, cfName, isDense, isCompound, clusteringColumns);
-
- // All tables should have a partition key
- assert !partitionKeyColumns.isEmpty() : String.format("Have no partition keys for table %s.%s", ksName, cfName);
-
- this.partitionKeyColumns = partitionKeyColumns;
- this.clusteringColumns = clusteringColumns;
- this.partitionColumns = partitionColumns;
-
- this.superCfKeyColumn = superCfKeyColumn;
- this.superCfValueColumn = superCfValueColumn;
-
- //This needs to happen before serializers are set
- //because they use comparator.subtypes()
- rebuild();
-
- this.serializers = new Serializers(this);
- this.resource = DataResource.table(ksName, cfName);
- }
-
- // This rebuild informations that are intrinsically duplicate of the table definition but
- // are kept because they are often useful in a different format.
- private void rebuild()
- {
- // A non-compact copy will be created lazily
- this.nonCompactCopy = null;
-
- if (isCompactTable())
- {
- this.compactValueColumn = isSuper() ?
- SuperColumnCompatibility.getCompactValueColumn(partitionColumns) :
- CompactTables.getCompactValueColumn(partitionColumns);
- }
-
- Map<ByteBuffer, ColumnDefinition> newColumnMetadata = Maps.newHashMapWithExpectedSize(partitionKeyColumns.size() + clusteringColumns.size() + partitionColumns.size());
-
- if (isSuper() && isDense())
- {
- CompactTables.DefaultNames defaultNames = SuperColumnCompatibility.columnNameGenerator(partitionKeyColumns, clusteringColumns, partitionColumns);
- if (superCfKeyColumn == null)
- superCfKeyColumn = SuperColumnCompatibility.getSuperCfKeyColumn(this, clusteringColumns, defaultNames);
- if (superCfValueColumn == null)
- superCfValueColumn = SuperColumnCompatibility.getSuperCfValueColumn(this, partitionColumns, superCfKeyColumn, defaultNames);
-
- for (ColumnDefinition def : partitionKeyColumns)
- newColumnMetadata.put(def.name.bytes, def);
- newColumnMetadata.put(clusteringColumns.get(0).name.bytes, clusteringColumns.get(0));
- newColumnMetadata.put(superCfKeyColumn.name.bytes, SuperColumnCompatibility.getSuperCfSschemaRepresentation(superCfKeyColumn));
- newColumnMetadata.put(superCfValueColumn.name.bytes, superCfValueColumn);
- newColumnMetadata.put(compactValueColumn.name.bytes, compactValueColumn);
- clusteringColumns = Arrays.asList(clusteringColumns().get(0));
- partitionColumns = PartitionColumns.of(compactValueColumn);
- }
- else
- {
- for (ColumnDefinition def : partitionKeyColumns)
- newColumnMetadata.put(def.name.bytes, def);
- for (ColumnDefinition def : clusteringColumns)
- newColumnMetadata.put(def.name.bytes, def);
- for (ColumnDefinition def : partitionColumns)
- newColumnMetadata.put(def.name.bytes, def);
- }
- this.columnMetadata = newColumnMetadata;
-
- List<AbstractType<?>> keyTypes = extractTypes(partitionKeyColumns);
- this.keyValidator = keyTypes.size() == 1 ? keyTypes.get(0) : CompositeType.getInstance(keyTypes);
-
- if (isSuper())
- this.comparator = new ClusteringComparator(clusteringColumns.get(0).type);
- else
- this.comparator = new ClusteringComparator(extractTypes(clusteringColumns));
-
- Set<ColumnDefinition> hiddenColumns;
- if (isCompactTable() && isDense && CompactTables.hasEmptyCompactValue(this))
- {
- hiddenColumns = Collections.singleton(compactValueColumn);
- }
- else if (isCompactTable() && !isDense && !isSuper)
- {
- hiddenColumns = Sets.newHashSetWithExpectedSize(clusteringColumns.size() + 1);
- hiddenColumns.add(compactValueColumn);
- hiddenColumns.addAll(clusteringColumns);
-
- }
- else
- {
- hiddenColumns = Collections.emptySet();
- }
- this.hiddenColumns = hiddenColumns;
-
- this.allColumnFilter = ColumnFilter.all(this);
- }
-
- public Indexes getIndexes()
- {
- return indexes;
- }
-
- public ColumnFilter getAllColumnFilter()
- {
- return allColumnFilter;
- }
-
- public static CFMetaData create(String ksName,
- String name,
- UUID cfId,
- boolean isDense,
- boolean isCompound,
- boolean isSuper,
- boolean isCounter,
- boolean isView,
- List<ColumnDefinition> columns,
- IPartitioner partitioner)
- {
- List<ColumnDefinition> partitions = new ArrayList<>();
- List<ColumnDefinition> clusterings = new ArrayList<>();
- PartitionColumns.Builder builder = PartitionColumns.builder();
-
- for (ColumnDefinition column : columns)
- {
- switch (column.kind)
- {
- case PARTITION_KEY:
- partitions.add(column);
- break;
- case CLUSTERING:
- clusterings.add(column);
- break;
- default:
- builder.add(column);
- break;
- }
- }
-
- Collections.sort(partitions);
- Collections.sort(clusterings);
-
- return new CFMetaData(ksName,
- name,
- cfId,
- isSuper,
- isCounter,
- isDense,
- isCompound,
- isView,
- partitions,
- clusterings,
- builder.build(),
- partitioner,
- null,
- null);
- }
-
- public static List<AbstractType<?>> extractTypes(Iterable<ColumnDefinition> clusteringColumns)
- {
- List<AbstractType<?>> types = new ArrayList<>();
- for (ColumnDefinition def : clusteringColumns)
- types.add(def.type);
- return types;
- }
-
- public Set<Flag> flags()
- {
- return flags;
- }
-
- /**
- * There is a couple of places in the code where we need a CFMetaData object and don't have one readily available
- * and know that only the keyspace and name matter. This creates such "fake" metadata. Use only if you know what
- * you're doing.
- */
- public static CFMetaData createFake(String keyspace, String name)
- {
- return CFMetaData.Builder.create(keyspace, name).addPartitionKey("key", BytesType.instance).build();
- }
-
- public Triggers getTriggers()
- {
- return triggers;
- }
-
- // Compiles a system metadata
- public static CFMetaData compile(String cql, String keyspace)
- {
- CFStatement parsed = (CFStatement)QueryProcessor.parseStatement(cql);
- parsed.prepareKeyspace(keyspace);
- CreateTableStatement statement = (CreateTableStatement) ((CreateTableStatement.RawStatement) parsed).prepare(Types.none()).statement;
-
- return statement.metadataBuilder()
- .withId(generateLegacyCfId(keyspace, statement.columnFamily()))
- .build()
- .params(statement.params())
- .readRepairChance(0.0)
- .dcLocalReadRepairChance(0.0)
- .gcGraceSeconds(0)
- .memtableFlushPeriod((int) TimeUnit.HOURS.toMillis(1));
- }
-
- /**
- * Generates deterministic UUID from keyspace/columnfamily name pair.
- * This is used to generate the same UUID for {@code C* version < 2.1}
- *
- * Since 2.1, this is only used for system columnfamilies and tests.
- */
- public static UUID generateLegacyCfId(String ksName, String cfName)
- {
- return UUID.nameUUIDFromBytes(ArrayUtils.addAll(ksName.getBytes(), cfName.getBytes()));
- }
-
- public CFMetaData reloadIndexMetadataProperties(CFMetaData parent)
- {
- TableParams.Builder indexParams = TableParams.builder(parent.params);
-
- // Depends on parent's cache setting, turn on its index CF's cache.
- // Row caching is never enabled; see CASSANDRA-5732
- if (parent.params.caching.cacheKeys())
- indexParams.caching(CachingParams.CACHE_KEYS);
- else
- indexParams.caching(CachingParams.CACHE_NOTHING);
-
- indexParams.readRepairChance(0.0)
- .dcLocalReadRepairChance(0.0)
- .gcGraceSeconds(0);
-
- return params(indexParams.build());
- }
-
- /**
- * Returns a cached non-compact version of this table. Cached version has to be invalidated
- * every time the table is rebuilt.
- */
- public CFMetaData asNonCompact()
- {
- assert isCompactTable() : "Can't get non-compact version of a CQL table";
-
- // Note that this is racy, but re-computing the non-compact copy a few times on first uses isn't a big deal so
- // we don't bother.
- if (nonCompactCopy == null)
- {
- nonCompactCopy = copyOpts(new CFMetaData(ksName,
- cfName,
- cfId,
- false,
- isCounter,
- false,
- true,
- isView,
- copy(partitionKeyColumns),
- copy(clusteringColumns),
- copy(partitionColumns),
- partitioner,
- superCfKeyColumn,
- superCfValueColumn),
- this);
- }
-
- return nonCompactCopy;
- }
-
- public CFMetaData copy()
- {
- return copy(cfId);
- }
-
- /**
- * Clones the CFMetaData, but sets a different cfId
- *
- * @param newCfId the cfId for the cloned CFMetaData
- * @return the cloned CFMetaData instance with the new cfId
- */
- public CFMetaData copy(UUID newCfId)
- {
- return copyOpts(new CFMetaData(ksName,
- cfName,
- newCfId,
- isSuper(),
- isCounter(),
- isDense(),
- isCompound(),
- isView(),
- copy(partitionKeyColumns),
- copy(clusteringColumns),
- copy(partitionColumns),
- partitioner,
- superCfKeyColumn,
- superCfValueColumn),
- this);
- }
-
- public CFMetaData copy(IPartitioner partitioner)
- {
- return copyOpts(new CFMetaData(ksName,
- cfName,
- cfId,
- isSuper,
- isCounter,
- isDense,
- isCompound,
- isView,
- copy(partitionKeyColumns),
- copy(clusteringColumns),
- copy(partitionColumns),
- partitioner,
- superCfKeyColumn,
- superCfValueColumn),
- this);
- }
-
- public CFMetaData copyWithNewCompactValueType(AbstractType<?> type)
- {
- assert isDense && compactValueColumn.type instanceof EmptyType && partitionColumns.size() == 1;
- return copyOpts(new CFMetaData(ksName,
- cfName,
- cfId,
- isSuper,
- isCounter,
- isDense,
- isCompound,
- isView,
- copy(partitionKeyColumns),
- copy(clusteringColumns),
- PartitionColumns.of(compactValueColumn.withNewType(type)),
- partitioner,
- superCfKeyColumn,
- superCfValueColumn),
- this);
- }
-
-
- private static List<ColumnDefinition> copy(List<ColumnDefinition> l)
- {
- List<ColumnDefinition> copied = new ArrayList<>(l.size());
- for (ColumnDefinition cd : l)
- copied.add(cd.copy());
- return copied;
- }
-
- private static PartitionColumns copy(PartitionColumns columns)
- {
- PartitionColumns.Builder newColumns = PartitionColumns.builder();
- for (ColumnDefinition cd : columns)
- newColumns.add(cd.copy());
- return newColumns.build();
- }
-
- @VisibleForTesting
- public static CFMetaData copyOpts(CFMetaData newCFMD, CFMetaData oldCFMD)
- {
- return newCFMD.params(oldCFMD.params)
- .droppedColumns(new HashMap<>(oldCFMD.droppedColumns))
- .triggers(oldCFMD.triggers)
- .indexes(oldCFMD.indexes);
- }
-
- /**
- * generate a column family name for an index corresponding to the given column.
- * This is NOT the same as the index's name! This is only used in sstable filenames and is not exposed to users.
- *
- * @param info A definition of the column with index
- *
- * @return name of the index ColumnFamily
- */
- public String indexColumnFamilyName(IndexMetadata info)
- {
- // TODO simplify this when info.index_name is guaranteed to be set
- return cfName + Directories.SECONDARY_INDEX_NAME_SEPARATOR + info.name;
- }
-
- /**
- * true if this CFS contains secondary index data.
- */
- public boolean isIndex()
- {
- return isIndex;
- }
-
- public DecoratedKey decorateKey(ByteBuffer key)
- {
- return partitioner.decorateKey(key);
- }
-
- public Map<ByteBuffer, ColumnDefinition> getColumnMetadata()
- {
- return columnMetadata;
- }
-
- /**
- *
- * @return The name of the parent cf if this is a seconday index
- */
- public String getParentColumnFamilyName()
- {
- return isIndex ? cfName.substring(0, cfName.indexOf('.')) : null;
- }
-
- public ReadRepairDecision newReadRepairDecision()
- {
- double chance = ThreadLocalRandom.current().nextDouble();
- if (params.readRepairChance > chance)
- return ReadRepairDecision.GLOBAL;
-
- if (params.dcLocalReadRepairChance > chance)
- return ReadRepairDecision.DC_LOCAL;
-
- return ReadRepairDecision.NONE;
- }
-
- public AbstractType<?> getColumnDefinitionNameComparator(ColumnDefinition.Kind kind)
- {
- return (isSuper() && kind == ColumnDefinition.Kind.REGULAR) || (isStaticCompactTable() && kind == ColumnDefinition.Kind.STATIC)
- ? thriftColumnNameType()
- : UTF8Type.instance;
- }
-
- public AbstractType<?> getKeyValidator()
- {
- return keyValidator;
- }
-
- public Collection<ColumnDefinition> allColumns()
- {
- return columnMetadata.values();
- }
-
- private Iterator<ColumnDefinition> nonPkColumnIterator()
- {
- final boolean noNonPkColumns = isCompactTable() && CompactTables.hasEmptyCompactValue(this) && !isSuper();
- if (noNonPkColumns)
- {
- return Collections.<ColumnDefinition>emptyIterator();
- }
- else if (isStaticCompactTable())
- {
- return partitionColumns.statics.selectOrderIterator();
- }
- else if (isSuper())
- {
- if (isDense)
- return Iterators.forArray(superCfKeyColumn, superCfValueColumn);
- else
- return Iterators.filter(partitionColumns.iterator(), (c) -> !c.type.isCollection());
- }
- else
- return partitionColumns().selectOrderIterator();
- }
-
- // An iterator over all column definitions but that respect the order of a SELECT *.
- // This also hides the clustering/regular columns for a non-CQL3 non-dense table for backward compatibility
- // sake (those are accessible through thrift but not through CQL currently) and exposes the key and value
- // columns for supercolumn family.
- public Iterator<ColumnDefinition> allColumnsInSelectOrder()
- {
- return new AbstractIterator<ColumnDefinition>()
- {
- private final Iterator<ColumnDefinition> partitionKeyIter = partitionKeyColumns.iterator();
- private final Iterator<ColumnDefinition> clusteringIter = isStaticCompactTable() ? Collections.<ColumnDefinition>emptyIterator() : clusteringColumns.iterator();
- private final Iterator<ColumnDefinition> otherColumns = nonPkColumnIterator();
-
- protected ColumnDefinition computeNext()
- {
- if (partitionKeyIter.hasNext())
- return partitionKeyIter.next();
-
- if (clusteringIter.hasNext())
- return clusteringIter.next();
-
- return otherColumns.hasNext() ? otherColumns.next() : endOfData();
- }
- };
- }
-
- public Iterable<ColumnDefinition> primaryKeyColumns()
- {
- return Iterables.concat(partitionKeyColumns, clusteringColumns);
- }
-
- public List<ColumnDefinition> partitionKeyColumns()
- {
- return partitionKeyColumns;
- }
-
- public List<ColumnDefinition> clusteringColumns()
- {
- return clusteringColumns;
- }
-
- public PartitionColumns partitionColumns()
- {
- return partitionColumns;
- }
-
- public ColumnDefinition compactValueColumn()
- {
- return compactValueColumn;
- }
-
- public ClusteringComparator getKeyValidatorAsClusteringComparator()
- {
- boolean isCompound = keyValidator instanceof CompositeType;
- List<AbstractType<?>> types = isCompound
- ? ((CompositeType) keyValidator).types
- : Collections.<AbstractType<?>>singletonList(keyValidator);
- return new ClusteringComparator(types);
- }
-
- public static ByteBuffer serializePartitionKey(ClusteringPrefix keyAsClustering)
- {
- // TODO: we should stop using Clustering for partition keys. Maybe we can add
- // a few methods to DecoratedKey so we don't have to (note that while using a Clustering
- // allows to use buildBound(), it's actually used for partition keys only when every restriction
- // is an equal, so we could easily create a specific method for keys for that.
- if (keyAsClustering.size() == 1)
- return keyAsClustering.get(0);
-
- ByteBuffer[] values = new ByteBuffer[keyAsClustering.size()];
- for (int i = 0; i < keyAsClustering.size(); i++)
- values[i] = keyAsClustering.get(i);
- return CompositeType.build(values);
- }
-
- public Map<ByteBuffer, DroppedColumn> getDroppedColumns()
- {
- return droppedColumns;
- }
-
- public ColumnDefinition getDroppedColumnDefinition(ByteBuffer name)
- {
- return getDroppedColumnDefinition(name, false);
- }
-
- /**
- * Returns a "fake" ColumnDefinition corresponding to the dropped column {@code name}
- * of {@code null} if there is no such dropped column.
- *
- * @param name - the column name
- * @param isStatic - whether the column was a static column, if known
- */
- public ColumnDefinition getDroppedColumnDefinition(ByteBuffer name, boolean isStatic)
- {
- DroppedColumn dropped = droppedColumns.get(name);
- if (dropped == null)
- return null;
-
- // We need the type for deserialization purpose. If we don't have the type however,
- // it means that it's a dropped column from before 3.0, and in that case using
- // BytesType is fine for what we'll be using it for, even if that's a hack.
- AbstractType<?> type = dropped.type == null ? BytesType.instance : dropped.type;
- return isStatic
- ? ColumnDefinition.staticDef(this, name, type)
- : ColumnDefinition.regularDef(this, name, type);
- }
-
- @Override
- public boolean equals(Object o)
- {
- if (this == o)
- return true;
-
- if (!(o instanceof CFMetaData))
- return false;
-
- CFMetaData other = (CFMetaData) o;
-
- return Objects.equal(cfId, other.cfId)
- && Objects.equal(flags, other.flags)
- && Objects.equal(ksName, other.ksName)
- && Objects.equal(cfName, other.cfName)
- && Objects.equal(params, other.params)
- && Objects.equal(comparator, other.comparator)
- && Objects.equal(keyValidator, other.keyValidator)
- && Objects.equal(columnMetadata, other.columnMetadata)
- && Objects.equal(droppedColumns, other.droppedColumns)
- && Objects.equal(triggers, other.triggers)
- && Objects.equal(indexes, other.indexes);
- }
-
- @Override
- public int hashCode()
- {
- return new HashCodeBuilder(29, 1597)
- .append(cfId)
- .append(ksName)
- .append(cfName)
- .append(flags)
- .append(comparator)
- .append(params)
- .append(keyValidator)
- .append(columnMetadata)
- .append(droppedColumns)
- .append(triggers)
- .append(indexes)
- .toHashCode();
- }
-
- /**
- * Updates CFMetaData in-place to match cfm
- *
- * @return true if any change was made which impacts queries/updates on the table,
- * e.g. any columns or indexes were added, removed, or altered; otherwise, false is returned.
- * Used to determine whether prepared statements against this table need to be re-prepared.
- * @throws ConfigurationException if ks/cf names or cf ids didn't match
- */
- @VisibleForTesting
- public boolean apply(CFMetaData cfm) throws ConfigurationException
- {
- logger.debug("applying {} to {}", cfm, this);
-
- validateCompatibility(cfm);
-
- partitionKeyColumns = cfm.partitionKeyColumns;
- clusteringColumns = cfm.clusteringColumns;
-
- boolean changeAffectsStatements = !partitionColumns.equals(cfm.partitionColumns);
- partitionColumns = cfm.partitionColumns;
- superCfKeyColumn = cfm.superCfKeyColumn;
- superCfValueColumn = cfm.superCfValueColumn;
-
- isDense = cfm.isDense;
- isCompound = cfm.isCompound;
- isSuper = cfm.isSuper;
-
- flags = cfm.flags;
-
- rebuild();
-
- // compaction thresholds are checked by ThriftValidation. We shouldn't be doing
- // validation on the apply path; it's too late for that.
-
- params = cfm.params;
-
- if (!cfm.droppedColumns.isEmpty())
- droppedColumns = cfm.droppedColumns;
-
- triggers = cfm.triggers;
-
- changeAffectsStatements |= !indexes.equals(cfm.indexes);
- indexes = cfm.indexes;
-
- logger.debug("application result is {}", this);
-
- return changeAffectsStatements;
- }
-
- public void validateCompatibility(CFMetaData cfm) throws ConfigurationException
- {
- // validate
- if (!cfm.ksName.equals(ksName))
- throw new ConfigurationException(String.format("Keyspace mismatch (found %s; expected %s)",
- cfm.ksName, ksName));
- if (!cfm.cfName.equals(cfName))
- throw new ConfigurationException(String.format("Column family mismatch (found %s; expected %s)",
- cfm.cfName, cfName));
- if (!cfm.cfId.equals(cfId))
- throw new ConfigurationException(String.format("Column family ID mismatch (found %s; expected %s)",
- cfm.cfId, cfId));
- }
-
-
- public static Class<? extends AbstractCompactionStrategy> createCompactionStrategy(String className) throws ConfigurationException
- {
- className = className.contains(".") ? className : "org.apache.cassandra.db.compaction." + className;
- Class<AbstractCompactionStrategy> strategyClass = FBUtilities.classForName(className, "compaction strategy");
- if (!AbstractCompactionStrategy.class.isAssignableFrom(strategyClass))
- throw new ConfigurationException(String.format("Specified compaction strategy class (%s) is not derived from AbstractCompactionStrategy", className));
-
- return strategyClass;
- }
-
- public static AbstractCompactionStrategy createCompactionStrategyInstance(ColumnFamilyStore cfs,
- CompactionParams compactionParams)
- {
- try
- {
- Constructor<? extends AbstractCompactionStrategy> constructor =
- compactionParams.klass().getConstructor(ColumnFamilyStore.class, Map.class);
- return constructor.newInstance(cfs, compactionParams.options());
- }
- catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException | InstantiationException e)
- {
- throw new RuntimeException(e);
- }
- }
-
- /**
- * Returns the ColumnDefinition for {@code name}.
- */
- public ColumnDefinition getColumnDefinition(ColumnIdentifier name)
- {
- return getColumnDefinition(name.bytes);
- }
-
- // In general it is preferable to work with ColumnIdentifier to make it
- // clear that we are talking about a CQL column, not a cell name, but there
- // is a few cases where all we have is a ByteBuffer (when dealing with IndexExpression
- // for instance) so...
- public ColumnDefinition getColumnDefinition(ByteBuffer name)
- {
- return columnMetadata.get(name);
- }
-
- // Returns only columns that are supposed to be visible through CQL layer
- public ColumnDefinition getColumnDefinitionForCQL(ColumnIdentifier name)
- {
- return getColumnDefinitionForCQL(name.bytes);
- }
-
- public ColumnDefinition getColumnDefinitionForCQL(ByteBuffer name)
- {
- ColumnDefinition cd = getColumnDefinition(name);
- return hiddenColumns.contains(cd) ? null : cd;
- }
-
- public static boolean isNameValid(String name)
- {
- return name != null && !name.isEmpty()
- && name.length() <= SchemaConstants.NAME_LENGTH && PATTERN_WORD_CHARS.matcher(name).matches();
- }
-
- public CFMetaData validate() throws ConfigurationException
- {
- rebuild();
-
- if (!isNameValid(ksName))
- throw new ConfigurationException(String.format("Keyspace name must not be empty, more than %s characters long, or contain non-alphanumeric-underscore characters (got \"%s\")", SchemaConstants.NAME_LENGTH, ksName));
- if (!isNameValid(cfName))
- throw new ConfigurationException(String.format("ColumnFamily name must not be empty, more than %s characters long, or contain non-alphanumeric-underscore characters (got \"%s\")", SchemaConstants.NAME_LENGTH, cfName));
-
- params.validate();
-
- for (int i = 0; i < comparator.size(); i++)
- {
- if (comparator.subtype(i) instanceof CounterColumnType)
- throw new ConfigurationException("CounterColumnType is not a valid comparator");
- }
- if (keyValidator instanceof CounterColumnType)
- throw new ConfigurationException("CounterColumnType is not a valid key validator");
-
- // Mixing counter with non counter columns is not supported (#2614)
- if (isCounter())
- {
- for (ColumnDefinition def : partitionColumns())
- if (!(def.type instanceof CounterColumnType) && (!isSuper() || isSuperColumnValueColumn(def)))
- throw new ConfigurationException("Cannot add a non counter column (" + def + ") in a counter column family");
- }
- else
- {
- for (ColumnDefinition def : allColumns())
- if (def.type instanceof CounterColumnType)
- throw new ConfigurationException("Cannot add a counter column (" + def.name + ") in a non counter column family");
- }
-
- if (!indexes.isEmpty() && isSuper())
- throw new ConfigurationException("Secondary indexes are not supported on super column families");
-
- // initialize a set of names NOT in the CF under consideration
- KeyspaceMetadata ksm = Schema.instance.getKSMetaData(ksName);
-
- Set<String> indexNames = ksm == null ? new HashSet<>() : ksm.existingIndexNames(cfName);
- for (IndexMetadata index : indexes)
- {
- // check index names against this CF _and_ globally
- if (indexNames.contains(index.name))
- throw new ConfigurationException("Duplicate index name " + index.name);
- indexNames.add(index.name);
-
- index.validate(this);
- }
-
- return this;
- }
-
- // The comparator to validate the definition name with thrift.
- public AbstractType<?> thriftColumnNameType()
- {
- if (isSuper())
- {
- ColumnDefinition def = compactValueColumn();
- assert def != null && def.type instanceof MapType;
- return ((MapType)def.type).nameComparator();
- }
-
- assert isStaticCompactTable();
- return clusteringColumns.get(0).type;
- }
-
- public CFMetaData addColumnDefinition(ColumnDefinition def) throws ConfigurationException
- {
- if (columnMetadata.containsKey(def.name.bytes))
- throw new ConfigurationException(String.format("Cannot add column %s, a column with the same name already exists", def.name));
-
- return addOrReplaceColumnDefinition(def);
- }
-
- // This method doesn't check if a def of the same name already exist and should only be used when we
- // know this cannot happen.
- public CFMetaData addOrReplaceColumnDefinition(ColumnDefinition def)
- {
- // Adds the definition and rebuild what is necessary. We could call rebuild() but it's not too hard to
- // only rebuild the necessary bits.
- switch (def.kind)
- {
- case PARTITION_KEY:
- partitionKeyColumns.set(def.position(), def);
- break;
- case CLUSTERING:
- clusteringColumns.set(def.position(), def);
- break;
- case REGULAR:
- case STATIC:
- PartitionColumns.Builder builder = PartitionColumns.builder();
- for (ColumnDefinition column : partitionColumns)
- if (!column.name.equals(def.name))
- builder.add(column);
- builder.add(def);
- partitionColumns = builder.build();
- // If dense, we must have modified the compact value since that's the only one we can have.
- if (isDense())
- this.compactValueColumn = def;
- break;
- }
- this.columnMetadata.put(def.name.bytes, def);
- return this;
- }
-
- public boolean removeColumnDefinition(ColumnDefinition def)
- {
- assert !def.isPartitionKey();
- boolean removed = columnMetadata.remove(def.name.bytes) != null;
- if (removed)
- partitionColumns = partitionColumns.without(def);
- return removed;
- }
-
- /**
- * Adds the column definition as a dropped column, recording the drop with the provided timestamp.
- */
- public void recordColumnDrop(ColumnDefinition def, long timeMicros)
- {
- recordColumnDrop(def, timeMicros, true);
- }
-
- @VisibleForTesting
- public void recordColumnDrop(ColumnDefinition def, long timeMicros, boolean preserveKind)
- {
- droppedColumns.put(def.name.bytes, new DroppedColumn(def.name.toString(), preserveKind ? def.kind : null, def.type, timeMicros));
- }
-
- public void renameColumn(ColumnIdentifier from, ColumnIdentifier to) throws InvalidRequestException
- {
- ColumnDefinition def = getColumnDefinitionForCQL(from);
-
- if (def == null)
- throw new InvalidRequestException(String.format("Cannot rename unknown column %s in keyspace %s", from, cfName));
-
- if (getColumnDefinition(to) != null)
- throw new InvalidRequestException(String.format("Cannot rename column %s to %s in keyspace %s; another column of that name already exist", from, to, cfName));
-
- if (def.isPartOfCellName(isCQLTable(), isSuper()) && !isDense())
- {
- throw new InvalidRequestException(String.format("Cannot rename non PRIMARY KEY part %s", from));
- }
-
- if (!getIndexes().isEmpty())
- {
- ColumnFamilyStore store = Keyspace.openAndGetStore(this);
- Set<IndexMetadata> dependentIndexes = store.indexManager.getDependentIndexes(def);
- if (!dependentIndexes.isEmpty())
- throw new InvalidRequestException(String.format("Cannot rename column %s because it has " +
- "dependent secondary indexes (%s)",
- from,
- dependentIndexes.stream()
- .map(i -> i.name)
- .collect(Collectors.joining(","))));
- }
-
- if (isSuper() && isDense())
- {
- if (isSuperColumnKeyColumn(def))
- {
- columnMetadata.remove(superCfKeyColumn.name.bytes);
- superCfKeyColumn = superCfKeyColumn.withNewName(to);
- columnMetadata.put(superCfKeyColumn.name.bytes, SuperColumnCompatibility.getSuperCfSschemaRepresentation(superCfKeyColumn));
- }
- else if (isSuperColumnValueColumn(def))
- {
- columnMetadata.remove(superCfValueColumn.name.bytes);
- superCfValueColumn = superCfValueColumn.withNewName(to);
- columnMetadata.put(superCfValueColumn.name.bytes, superCfValueColumn);
- }
- else
- addOrReplaceColumnDefinition(def.withNewName(to));
- }
- else
- {
- addOrReplaceColumnDefinition(def.withNewName(to));
- }
-
-
- // removeColumnDefinition doesn't work for partition key (expectedly) but renaming one is fine so we still
- // want to update columnMetadata.
- if (def.isPartitionKey())
- columnMetadata.remove(def.name.bytes);
- else
- removeColumnDefinition(def);
- }
-
- public boolean isCQLTable()
- {
- return !isSuper() && !isDense() && isCompound();
- }
-
- public boolean isCompactTable()
- {
- return !isCQLTable();
- }
-
- public boolean isStaticCompactTable()
- {
- return !isSuper() && !isDense() && !isCompound();
- }
-
- /**
- * Returns whether this CFMetaData can be returned to thrift.
- */
- public boolean isThriftCompatible()
- {
- return isCompactTable();
- }
-
- public boolean hasStaticColumns()
- {
- return !partitionColumns.statics.isEmpty();
- }
-
- public boolean hasCollectionColumns()
- {
- for (ColumnDefinition def : partitionColumns())
- if (def.type instanceof CollectionType && def.type.isMultiCell())
- return true;
- return false;
- }
-
- public boolean hasComplexColumns()
- {
- for (ColumnDefinition def : partitionColumns())
- if (def.isComplex())
- return true;
- return false;
- }
-
- public boolean hasDroppedCollectionColumns()
- {
- for (DroppedColumn def : getDroppedColumns().values())
- if (def.type instanceof CollectionType && def.type.isMultiCell())
- return true;
- return false;
- }
-
- public boolean isSuper()
- {
- return isSuper;
- }
-
- public boolean isCounter()
- {
- return isCounter;
- }
-
- // We call dense a CF for which each component of the comparator is a clustering column, i.e. no
- // component is used to store a regular column names. In other words, non-composite static "thrift"
- // and CQL3 CF are *not* dense.
- public boolean isDense()
- {
- return isDense;
- }
-
- public boolean isCompound()
- {
- return isCompound;
- }
-
- public boolean isView()
- {
- return isView;
- }
-
- /**
- * A table with strict liveness filters/ignores rows without PK liveness info,
- * effectively tying the row liveness to its primary key liveness.
- *
- * Currently this is only used by views with normal base column as PK column
- * so updates to other columns do not make the row live when the base column
- * is not live. See CASSANDRA-11500.
- */
- public boolean enforceStrictLiveness()
- {
- return isView && Keyspace.open(ksName).viewManager.getByName(cfName).enforceStrictLiveness();
- }
-
- public Serializers serializers()
- {
- return serializers;
- }
-
- public AbstractType<?> makeLegacyDefaultValidator()
- {
- if (isCounter())
- return CounterColumnType.instance;
- else if (isCompactTable())
- return isSuper() ? ((MapType)compactValueColumn().type).valueComparator() : compactValueColumn().type;
- else
- return BytesType.instance;
- }
-
- public static Set<Flag> flagsFromStrings(Set<String> strings)
- {
- return strings.stream()
- .map(String::toUpperCase)
- .map(Flag::valueOf)
- .collect(Collectors.toSet());
- }
-
- public static Set<String> flagsToStrings(Set<Flag> flags)
- {
- return flags.stream()
- .map(Flag::toString)
- .map(String::toLowerCase)
- .collect(Collectors.toSet());
- }
-
-
- @Override
- public String toString()
- {
- return new ToStringBuilder(this)
- .append("cfId", cfId)
- .append("ksName", ksName)
- .append("cfName", cfName)
- .append("flags", flags)
- .append("params", params)
- .append("comparator", comparator)
- .append("partitionColumns", partitionColumns)
- .append("partitionKeyColumns", partitionKeyColumns)
- .append("clusteringColumns", clusteringColumns)
- .append("keyValidator", keyValidator)
- .append("columnMetadata", columnMetadata.values())
- .append("droppedColumns", droppedColumns)
- .append("triggers", triggers)
- .append("indexes", indexes)
- .toString();
- }
-
- public static class Builder
- {
- private final String keyspace;
- private final String table;
- private final boolean isDense;
- private final boolean isCompound;
- private final boolean isSuper;
- private final boolean isCounter;
- private final boolean isView;
- private Optional<IPartitioner> partitioner;
-
- private UUID tableId;
-
- private final List<Pair<ColumnIdentifier, AbstractType>> partitionKeys = new ArrayList<>();
- private final List<Pair<ColumnIdentifier, AbstractType>> clusteringColumns = new ArrayList<>();
- private final List<Pair<ColumnIdentifier, AbstractType>> staticColumns = new ArrayList<>();
- private final List<Pair<ColumnIdentifier, AbstractType>> regularColumns = new ArrayList<>();
-
- private Builder(String keyspace, String table, boolean isDense, boolean isCompound, boolean isSuper, boolean isCounter, boolean isView)
- {
- this.keyspace = keyspace;
- this.table = table;
- this.isDense = isDense;
- this.isCompound = isCompound;
- this.isSuper = isSuper;
- this.isCounter = isCounter;
- this.isView = isView;
- this.partitioner = Optional.empty();
- }
-
- public static Builder create(String keyspace, String table)
- {
- return create(keyspace, table, false, true, false);
- }
-
- public static Builder create(String keyspace, String table, boolean isDense, boolean isCompound, boolean isCounter)
- {
- return create(keyspace, table, isDense, isCompound, false, isCounter);
- }
-
- public static Builder create(String keyspace, String table, boolean isDense, boolean isCompound, boolean isSuper, boolean isCounter)
- {
- return new Builder(keyspace, table, isDense, isCompound, isSuper, isCounter, false);
- }
-
- public static Builder createView(String keyspace, String table)
- {
- return new Builder(keyspace, table, false, true, false, false, true);
- }
-
- public static Builder createDense(String keyspace, String table, boolean isCompound, boolean isCounter)
- {
- return create(keyspace, table, true, isCompound, isCounter);
- }
-
- public static Builder createSuper(String keyspace, String table, boolean isCounter)
- {
- return create(keyspace, table, true, true, true, isCounter);
- }
-
- public Builder withPartitioner(IPartitioner partitioner)
- {
- this.partitioner = Optional.ofNullable(partitioner);
- return this;
- }
-
- public Builder withId(UUID tableId)
- {
- this.tableId = tableId;
- return this;
- }
-
- public Builder addPartitionKey(String name, AbstractType type)
- {
- return addPartitionKey(ColumnIdentifier.getInterned(name, false), type);
- }
-
- public Builder addPartitionKey(ColumnIdentifier name, AbstractType type)
- {
- this.partitionKeys.add(Pair.create(name, type));
- return this;
- }
-
- public Builder addClusteringColumn(String name, AbstractType type)
- {
- return addClusteringColumn(ColumnIdentifier.getInterned(name, false), type);
- }
-
- public Builder addClusteringColumn(ColumnIdentifier name, AbstractType type)
- {
- this.clusteringColumns.add(Pair.create(name, type));
- return this;
- }
-
- public Builder addRegularColumn(String name, AbstractType type)
- {
- return addRegularColumn(ColumnIdentifier.getInterned(name, false), type);
- }
-
- public Builder addRegularColumn(ColumnIdentifier name, AbstractType type)
- {
- this.regularColumns.add(Pair.create(name, type));
- return this;
- }
-
- public boolean hasRegulars()
- {
- return !this.regularColumns.isEmpty();
- }
-
- public Builder addStaticColumn(String name, AbstractType type)
- {
- return addStaticColumn(ColumnIdentifier.getInterned(name, false), type);
- }
-
- public Builder addStaticColumn(ColumnIdentifier name, AbstractType type)
- {
- this.staticColumns.add(Pair.create(name, type));
- return this;
- }
-
- public Set<String> usedColumnNames()
- {
- Set<String> usedNames = Sets.newHashSetWithExpectedSize(partitionKeys.size() + clusteringColumns.size() + staticColumns.size() + regularColumns.size());
- for (Pair<ColumnIdentifier, AbstractType> p : partitionKeys)
- usedNames.add(p.left.toString());
- for (Pair<ColumnIdentifier, AbstractType> p : clusteringColumns)
- usedNames.add(p.left.toString());
- for (Pair<ColumnIdentifier, AbstractType> p : staticColumns)
- usedNames.add(p.left.toString());
- for (Pair<ColumnIdentifier, AbstractType> p : regularColumns)
- usedNames.add(p.left.toString());
- return usedNames;
- }
-
- public CFMetaData build()
- {
- if (tableId == null)
- tableId = UUIDGen.getTimeUUID();
-
- List<ColumnDefinition> partitions = new ArrayList<>(partitionKeys.size());
- List<ColumnDefinition> clusterings = new ArrayList<>(clusteringColumns.size());
- PartitionColumns.Builder builder = PartitionColumns.builder();
-
- for (int i = 0; i < partitionKeys.size(); i++)
- {
- Pair<ColumnIdentifier, AbstractType> p = partitionKeys.get(i);
- partitions.add(new ColumnDefinition(keyspace, table, p.left, p.right, i, ColumnDefinition.Kind.PARTITION_KEY));
- }
-
- for (int i = 0; i < clusteringColumns.size(); i++)
- {
- Pair<ColumnIdentifier, AbstractType> p = clusteringColumns.get(i);
- clusterings.add(new ColumnDefinition(keyspace, table, p.left, p.right, i, ColumnDefinition.Kind.CLUSTERING));
- }
-
- for (Pair<ColumnIdentifier, AbstractType> p : regularColumns)
- builder.add(new ColumnDefinition(keyspace, table, p.left, p.right, ColumnDefinition.NO_POSITION, ColumnDefinition.Kind.REGULAR));
-
- for (Pair<ColumnIdentifier, AbstractType> p : staticColumns)
- builder.add(new ColumnDefinition(keyspace, table, p.left, p.right, ColumnDefinition.NO_POSITION, ColumnDefinition.Kind.STATIC));
-
- return new CFMetaData(keyspace,
- table,
- tableId,
- isSuper,
- isCounter,
- isDense,
- isCompound,
- isView,
- partitions,
- clusterings,
- builder.build(),
- partitioner.orElseGet(DatabaseDescriptor::getPartitioner),
- null,
- null);
- }
- }
-
- public static class Serializer
- {
- public void serialize(CFMetaData metadata, DataOutputPlus out, int version) throws IOException
- {
- UUIDSerializer.serializer.serialize(metadata.cfId, out, version);
- }
-
- public CFMetaData deserialize(DataInputPlus in, int version) throws IOException
- {
- UUID cfId = UUIDSerializer.serializer.deserialize(in, version);
- CFMetaData metadata = Schema.instance.getCFMetaData(cfId);
- if (metadata == null)
- {
- String message = String.format("Couldn't find table for cfId %s. If a table was just " +
- "created, this is likely due to the schema not being fully propagated. Please wait for schema " +
- "agreement on table creation.", cfId);
- throw new UnknownColumnFamilyException(message, cfId);
- }
-
- return metadata;
- }
-
- public long serializedSize(CFMetaData metadata, int version)
- {
- return UUIDSerializer.serializer.serializedSize(metadata.cfId, version);
- }
- }
-
- public static class DroppedColumn
- {
- // we only allow dropping REGULAR columns, from CQL-native tables, so the names are always of UTF8Type
- public final String name;
- public final AbstractType<?> type;
-
- // drop timestamp, in microseconds, yet with millisecond granularity
- public final long droppedTime;
-
- @Nullable
- public final ColumnDefinition.Kind kind;
-
- public DroppedColumn(String name, ColumnDefinition.Kind kind, AbstractType<?> type, long droppedTime)
- {
- this.name = name;
- this.kind = kind;
- this.type = type;
- this.droppedTime = droppedTime;
- }
-
- @Override
- public boolean equals(Object o)
- {
- if (this == o)
- return true;
-
- if (!(o instanceof DroppedColumn))
- return false;
-
- DroppedColumn dc = (DroppedColumn) o;
-
- return name.equals(dc.name)
- && kind == dc.kind
- && type.equals(dc.type)
- && droppedTime == dc.droppedTime;
- }
-
- @Override
- public int hashCode()
- {
- return Objects.hashCode(name, kind, type, droppedTime);
- }
-
- @Override
- public String toString()
- {
- return MoreObjects.toStringHelper(this)
- .add("name", name)
- .add("kind", kind)
- .add("type", type)
- .add("droppedTime", droppedTime)
- .toString();
- }
- }
-}
diff --git a/src/java/org/apache/cassandra/config/ColumnDefinition.java b/src/java/org/apache/cassandra/config/ColumnDefinition.java
deleted file mode 100644
index 0d6bdc8..0000000
--- a/src/java/org/apache/cassandra/config/ColumnDefinition.java
+++ /dev/null
@@ -1,657 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.config;
-
-import java.nio.ByteBuffer;
-import java.util.*;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Function;
-import com.google.common.base.MoreObjects;
-import com.google.common.base.Objects;
-import com.google.common.collect.Collections2;
-
-import org.apache.cassandra.cql3.*;
-import org.apache.cassandra.cql3.selection.Selectable;
-import org.apache.cassandra.cql3.selection.Selector;
-import org.apache.cassandra.cql3.selection.SimpleSelector;
-import org.apache.cassandra.db.rows.*;
-import org.apache.cassandra.db.marshal.*;
-import org.apache.cassandra.exceptions.InvalidRequestException;
-import org.apache.cassandra.serializers.MarshalException;
-import org.apache.cassandra.utils.ByteBufferUtil;
-import org.github.jamm.Unmetered;
-
-@Unmetered
-public class ColumnDefinition extends ColumnSpecification implements Selectable, Comparable<ColumnDefinition>
-{
- public static final Comparator<Object> asymmetricColumnDataComparator =
- (a, b) -> ((ColumnData) a).column().compareTo((ColumnDefinition) b);
-
- public static final int NO_POSITION = -1;
-
- public enum ClusteringOrder
- {
- ASC, DESC, NONE
- }
-
- /*
- * The type of CQL3 column this definition represents.
- * There is 4 main type of CQL3 columns: those parts of the partition key,
- * those parts of the clustering columns and amongst the others, regular and
- * static ones.
- *
- * Note that thrift only knows about definitions of type REGULAR (and
- * the ones whose position == NO_POSITION (-1)).
- */
- public enum Kind
- {
- // NOTE: if adding a new type, must modify comparisonOrder
- PARTITION_KEY,
- CLUSTERING,
- REGULAR,
- STATIC;
-
- public boolean isPrimaryKeyKind()
- {
- return this == PARTITION_KEY || this == CLUSTERING;
- }
-
- }
-
- public final Kind kind;
-
- /*
- * If the column is a partition key or clustering column, its position relative to
- * other columns of the same kind. Otherwise, NO_POSITION (-1).
- *
- * Note that partition key and clustering columns are numbered separately so
- * the first clustering column is 0.
- */
- private final int position;
-
- private final Comparator<CellPath> cellPathComparator;
- private final Comparator<Object> asymmetricCellPathComparator;
- private final Comparator<? super Cell> cellComparator;
-
- private int hash;
-
- /**
- * These objects are compared frequently, so we encode several of their comparison components
- * into a single long value so that this can be done efficiently
- */
- private final long comparisonOrder;
-
- private static long comparisonOrder(Kind kind, boolean isComplex, long position, ColumnIdentifier name)
- {
- assert position >= 0 && position < 1 << 12;
- return (((long) kind.ordinal()) << 61)
- | (isComplex ? 1L << 60 : 0)
- | (position << 48)
- | (name.prefixComparison >>> 16);
- }
-
- public static ColumnDefinition partitionKeyDef(CFMetaData cfm, ByteBuffer name, AbstractType<?> type, int position)
- {
- return new ColumnDefinition(cfm, name, type, position, Kind.PARTITION_KEY);
- }
-
- public static ColumnDefinition partitionKeyDef(String ksName, String cfName, String name, AbstractType<?> type, int position)
- {
- return new ColumnDefinition(ksName, cfName, ColumnIdentifier.getInterned(name, true), type, position, Kind.PARTITION_KEY);
- }
-
- public static ColumnDefinition clusteringDef(CFMetaData cfm, ByteBuffer name, AbstractType<?> type, int position)
- {
- return new ColumnDefinition(cfm, name, type, position, Kind.CLUSTERING);
- }
-
- public static ColumnDefinition clusteringDef(String ksName, String cfName, String name, AbstractType<?> type, int position)
- {
- return new ColumnDefinition(ksName, cfName, ColumnIdentifier.getInterned(name, true), type, position, Kind.CLUSTERING);
- }
-
- public static ColumnDefinition regularDef(CFMetaData cfm, ByteBuffer name, AbstractType<?> type)
- {
- return new ColumnDefinition(cfm, name, type, NO_POSITION, Kind.REGULAR);
- }
-
- public static ColumnDefinition regularDef(String ksName, String cfName, String name, AbstractType<?> type)
- {
- return new ColumnDefinition(ksName, cfName, ColumnIdentifier.getInterned(name, true), type, NO_POSITION, Kind.REGULAR);
- }
-
- public static ColumnDefinition staticDef(CFMetaData cfm, ByteBuffer name, AbstractType<?> type)
- {
- return new ColumnDefinition(cfm, name, type, NO_POSITION, Kind.STATIC);
- }
-
- public static ColumnDefinition staticDef(String ksName, String cfName, String name, AbstractType<?> type)
- {
- return new ColumnDefinition(ksName, cfName, ColumnIdentifier.getInterned(name, true), type, NO_POSITION, Kind.STATIC);
- }
-
- public ColumnDefinition(CFMetaData cfm, ByteBuffer name, AbstractType<?> type, int position, Kind kind)
- {
- this(cfm.ksName,
- cfm.cfName,
- ColumnIdentifier.getInterned(name, cfm.getColumnDefinitionNameComparator(kind)),
- type,
- position,
- kind);
- }
-
- @VisibleForTesting
- public ColumnDefinition(String ksName,
- String cfName,
- ColumnIdentifier name,
- AbstractType<?> type,
- int position,
- Kind kind)
- {
- super(ksName, cfName, name, type);
- assert name != null && type != null && kind != null;
- assert (position == NO_POSITION) == !kind.isPrimaryKeyKind(); // The position really only make sense for partition and clustering columns (and those must have one),
- // so make sure we don't sneak it for something else since it'd breaks equals()
- this.kind = kind;
- this.position = position;
- this.cellPathComparator = makeCellPathComparator(kind, type);
- this.cellComparator = cellPathComparator == null ? ColumnData.comparator : (a, b) -> cellPathComparator.compare(a.path(), b.path());
- this.asymmetricCellPathComparator = cellPathComparator == null ? null : (a, b) -> cellPathComparator.compare(((Cell)a).path(), (CellPath) b);
- this.comparisonOrder = comparisonOrder(kind, isComplex(), Math.max(0, position), name);
- }
-
- private static Comparator<CellPath> makeCellPathComparator(Kind kind, AbstractType<?> type)
- {
- if (kind.isPrimaryKeyKind() || !type.isMultiCell())
- return null;
-
- AbstractType<?> nameComparator = type.isCollection()
- ? ((CollectionType) type).nameComparator()
- : ((UserType) type).nameComparator();
-
-
- return new Comparator<CellPath>()
- {
- public int compare(CellPath path1, CellPath path2)
- {
- if (path1.size() == 0 || path2.size() == 0)
- {
- if (path1 == CellPath.BOTTOM)
- return path2 == CellPath.BOTTOM ? 0 : -1;
- if (path1 == CellPath.TOP)
- return path2 == CellPath.TOP ? 0 : 1;
- return path2 == CellPath.BOTTOM ? 1 : -1;
- }
-
- // This will get more complicated once we have non-frozen UDT and nested collections
- assert path1.size() == 1 && path2.size() == 1;
- return nameComparator.compare(path1.get(0), path2.get(0));
- }
- };
- }
-
- public ColumnDefinition copy()
- {
- return new ColumnDefinition(ksName, cfName, name, type, position, kind);
- }
-
- public ColumnDefinition withNewName(ColumnIdentifier newName)
- {
- return new ColumnDefinition(ksName, cfName, newName, type, position, kind);
- }
-
- public ColumnDefinition withNewType(AbstractType<?> newType)
- {
- return new ColumnDefinition(ksName, cfName, name, newType, position, kind);
- }
-
- public boolean isPartitionKey()
- {
- return kind == Kind.PARTITION_KEY;
- }
-
- public boolean isClusteringColumn()
- {
- return kind == Kind.CLUSTERING;
- }
-
- public boolean isStatic()
- {
- return kind == Kind.STATIC;
- }
-
- public boolean isRegular()
- {
- return kind == Kind.REGULAR;
- }
-
- public ClusteringOrder clusteringOrder()
- {
- if (!isClusteringColumn())
- return ClusteringOrder.NONE;
-
- return type.isReversed() ? ClusteringOrder.DESC : ClusteringOrder.ASC;
- }
-
- public int position()
- {
- return position;
- }
-
- @Override
- public boolean equals(Object o)
- {
- if (this == o)
- return true;
-
- if (!(o instanceof ColumnDefinition))
- return false;
-
- ColumnDefinition cd = (ColumnDefinition) o;
-
- return Objects.equal(ksName, cd.ksName)
- && Objects.equal(cfName, cd.cfName)
- && Objects.equal(name, cd.name)
- && Objects.equal(type, cd.type)
- && Objects.equal(kind, cd.kind)
- && Objects.equal(position, cd.position);
- }
-
- @Override
- public int hashCode()
- {
- // This achieves the same as Objects.hashcode, but avoids the object array allocation
- // which features significantly in the allocation profile and caches the result.
- int result = hash;
- if(result == 0)
- {
- result = 31 + (ksName == null ? 0 : ksName.hashCode());
- result = 31 * result + (cfName == null ? 0 : cfName.hashCode());
- result = 31 * result + (name == null ? 0 : name.hashCode());
- result = 31 * result + (type == null ? 0 : type.hashCode());
- result = 31 * result + (kind == null ? 0 : kind.hashCode());
- result = 31 * result + position;
- hash = result;
- }
- return result;
- }
-
- @Override
- public String toString()
- {
- return name.toString();
- }
-
- public String debugString()
- {
- return MoreObjects.toStringHelper(this)
- .add("name", name)
- .add("type", type)
- .add("kind", kind)
- .add("position", position)
- .toString();
- }
-
- public boolean isPrimaryKeyColumn()
- {
- return kind.isPrimaryKeyKind();
- }
-
- /**
- * Whether the name of this definition is serialized in the cell nane, i.e. whether
- * it's not just a non-stored CQL metadata.
- */
- public boolean isPartOfCellName(boolean isCQL3Table, boolean isSuper)
- {
- // When converting CQL3 tables to thrift, any regular or static column ends up in the cell name.
- // When it's a compact table however, the REGULAR definition is the name for the cell value of "dynamic"
- // column (so it's not part of the cell name) and it's static columns that ends up in the cell name.
- if (isCQL3Table)
- return kind == Kind.REGULAR || kind == Kind.STATIC;
- else if (isSuper)
- return kind == Kind.REGULAR;
- else
- return kind == Kind.STATIC;
- }
-
- /**
- * Converts the specified column definitions into column identifiers.
- *
- * @param definitions the column definitions to convert.
- * @return the column identifiers corresponding to the specified definitions
- */
- public static Collection<ColumnIdentifier> toIdentifiers(Collection<ColumnDefinition> definitions)
- {
- return Collections2.transform(definitions, new Function<ColumnDefinition, ColumnIdentifier>()
- {
- @Override
- public ColumnIdentifier apply(ColumnDefinition columnDef)
- {
- return columnDef.name;
- }
- });
- }
-
- public int compareTo(ColumnDefinition other)
- {
- if (this == other)
- return 0;
-
- if (comparisonOrder != other.comparisonOrder)
- return Long.compare(comparisonOrder, other.comparisonOrder);
-
- return this.name.compareTo(other.name);
- }
-
- public Comparator<CellPath> cellPathComparator()
- {
- return cellPathComparator;
- }
-
- public Comparator<Object> asymmetricCellPathComparator()
- {
- return asymmetricCellPathComparator;
- }
-
- public Comparator<? super Cell> cellComparator()
- {
- return cellComparator;
- }
-
- public boolean isComplex()
- {
- return cellPathComparator != null;
- }
-
- public boolean isSimple()
- {
- return !isComplex();
- }
-
- public CellPath.Serializer cellPathSerializer()
- {
- // Collections are our only complex so far, so keep it simple
- return CollectionType.cellPathSerializer;
- }
-
- public void validateCell(Cell cell)
- {
- if (cell.isTombstone())
- {
- if (cell.value().hasRemaining())
- throw new MarshalException("A tombstone should not have a value");
- if (cell.path() != null)
- validateCellPath(cell.path());
- }
- else if(type.isUDT())
- {
- // To validate a non-frozen UDT field, both the path and the value
- // are needed, the path being an index into an array of value types.
- ((UserType)type).validateCell(cell);
- }
- else
- {
- type.validateCellValue(cell.value());
- if (cell.path() != null)
- validateCellPath(cell.path());
- }
- }
-
- private void validateCellPath(CellPath path)
- {
- if (!isComplex())
- throw new MarshalException("Only complex cells should have a cell path");
-
- assert type.isMultiCell();
- if (type.isCollection())
- ((CollectionType)type).nameComparator().validate(path.get(0));
- else
- ((UserType)type).nameComparator().validate(path.get(0));
- }
-
- public static String toCQLString(Iterable<ColumnDefinition> defs)
- {
- return toCQLString(defs.iterator());
- }
-
- public static String toCQLString(Iterator<ColumnDefinition> defs)
- {
- if (!defs.hasNext())
- return "";
-
- StringBuilder sb = new StringBuilder();
- sb.append(defs.next().name);
- while (defs.hasNext())
- sb.append(", ").append(defs.next().name);
- return sb.toString();
- }
-
- /**
- * The type of the cell values for cell belonging to this column.
- *
- * This is the same than the column type, except for non-frozen collections where it's the 'valueComparator'
- * of the collection.
- *
- * This method should not be used to get value type of non-frozon UDT.
- */
- public AbstractType<?> cellValueType()
- {
- assert !(type instanceof UserType && type.isMultiCell());
- return type instanceof CollectionType && type.isMultiCell()
- ? ((CollectionType)type).valueComparator()
- : type;
- }
-
-
- public boolean isCounterColumn()
- {
- if (type instanceof CollectionType) // for thrift
- return ((CollectionType) type).valueComparator().isCounter();
- return type.isCounter();
- }
-
- public Selector.Factory newSelectorFactory(CFMetaData cfm, AbstractType<?> expectedType, List<ColumnDefinition> defs, VariableSpecifications boundNames) throws InvalidRequestException
- {
- return SimpleSelector.newFactory(this, addAndGetIndex(this, defs));
- }
-
- public AbstractType<?> getExactTypeIfKnown(String keyspace)
- {
- return type;
- }
-
- /**
- * Because Thrift-created tables may have a non-text comparator, we cannot determine the proper 'key' until
- * we know the comparator. ColumnDefinition.Raw is a placeholder that can be converted to a real ColumnIdentifier
- * once the comparator is known with prepare(). This should only be used with identifiers that are actual
- * column names. See CASSANDRA-8178 for more background.
- */
- public static abstract class Raw extends Selectable.Raw
- {
- /**
- * Creates a {@code ColumnDefinition.Raw} from an unquoted identifier string.
- */
- public static Raw forUnquoted(String text)
- {
- return new Literal(text, false);
- }
-
- /**
- * Creates a {@code ColumnDefinition.Raw} from a quoted identifier string.
- */
- public static Raw forQuoted(String text)
- {
- return new Literal(text, true);
- }
-
- /**
- * Creates a {@code ColumnDefinition.Raw} from a pre-existing {@code ColumnDefinition}
- * (useful in the rare cases where we already have the column but need
- * a {@code ColumnDefinition.Raw} for typing purposes).
- */
- public static Raw forColumn(ColumnDefinition column)
- {
- return new ForColumn(column);
- }
-
- /**
- * Get the identifier corresponding to this raw column, without assuming this is an
- * existing column (unlike {@link #prepare}).
- */
- public abstract ColumnIdentifier getIdentifier(CFMetaData cfm);
-
- public abstract String rawText();
-
- @Override
- public abstract ColumnDefinition prepare(CFMetaData cfm);
-
- @Override
- public boolean processesSelection()
- {
- return false;
- }
-
- @Override
- public final int hashCode()
- {
- return toString().hashCode();
- }
-
- @Override
- public final boolean equals(Object o)
- {
- if(!(o instanceof Raw))
- return false;
-
- Raw that = (Raw)o;
- return this.toString().equals(that.toString());
- }
-
- private static class Literal extends Raw
- {
- private final String text;
-
- public Literal(String rawText, boolean keepCase)
- {
- this.text = keepCase ? rawText : rawText.toLowerCase(Locale.US);
- }
-
- public ColumnIdentifier getIdentifier(CFMetaData cfm)
- {
- if (!cfm.isStaticCompactTable())
- return ColumnIdentifier.getInterned(text, true);
-
- AbstractType<?> thriftColumnNameType = cfm.thriftColumnNameType();
- if (thriftColumnNameType instanceof UTF8Type)
- return ColumnIdentifier.getInterned(text, true);
-
- // We have a Thrift-created table with a non-text comparator. Check if we have a match column, otherwise assume we should use
- // thriftColumnNameType
- ByteBuffer bufferName = ByteBufferUtil.bytes(text);
- for (ColumnDefinition def : cfm.allColumns())
- {
- if (def.name.bytes.equals(bufferName))
- return def.name;
- }
- return ColumnIdentifier.getInterned(thriftColumnNameType, thriftColumnNameType.fromString(text), text);
- }
-
- public ColumnDefinition prepare(CFMetaData cfm)
- {
- if (!cfm.isStaticCompactTable())
- return find(cfm);
-
- AbstractType<?> thriftColumnNameType = cfm.thriftColumnNameType();
- if (thriftColumnNameType instanceof UTF8Type)
- return find(cfm);
-
- // We have a Thrift-created table with a non-text comparator. Check if we have a match column, otherwise assume we should use
- // thriftColumnNameType
- ByteBuffer bufferName = ByteBufferUtil.bytes(text);
- for (ColumnDefinition def : cfm.allColumns())
- {
- if (def.name.bytes.equals(bufferName))
- return def;
- }
- return find(thriftColumnNameType.fromString(text), cfm);
- }
-
- private ColumnDefinition find(CFMetaData cfm)
- {
- return find(ByteBufferUtil.bytes(text), cfm);
- }
-
- private ColumnDefinition find(ByteBuffer id, CFMetaData cfm)
- {
- ColumnDefinition def = cfm.getColumnDefinitionForCQL(id);
- if (def == null)
- throw new InvalidRequestException(String.format("Undefined column name %s", toString()));
- return def;
- }
-
- public String rawText()
- {
- return text;
- }
-
- @Override
- public String toString()
- {
- return ColumnIdentifier.maybeQuote(text);
- }
- }
-
- // Use internally in the rare case where we need a ColumnDefinition.Raw for type-checking but
- // actually already have the column itself.
- private static class ForColumn extends Raw
- {
- private final ColumnDefinition column;
-
- private ForColumn(ColumnDefinition column)
- {
- this.column = column;
- }
-
- public ColumnIdentifier getIdentifier(CFMetaData cfm)
- {
- return column.name;
- }
-
- public ColumnDefinition prepare(CFMetaData cfm)
- {
- assert cfm.getColumnDefinition(column.name) != null; // Sanity check that we're not doing something crazy
- return column;
- }
-
- public String rawText()
- {
- return column.name.toString();
- }
-
- @Override
- public String toString()
- {
- return column.name.toCQLString();
- }
- }
- }
-
-
-
-}
diff --git a/src/java/org/apache/cassandra/config/Config.java b/src/java/org/apache/cassandra/config/Config.java
index 322f1f5..63cfff4 100644
--- a/src/java/org/apache/cassandra/config/Config.java
+++ b/src/java/org/apache/cassandra/config/Config.java
@@ -33,6 +33,10 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.cassandra.audit.AuditLogOptions;
+import org.apache.cassandra.fql.FullQueryLoggerOptions;
+import org.apache.cassandra.db.ConsistencyLevel;
+
/**
* A class that contains configuration properties for the cassandra node it runs within.
*
@@ -51,6 +55,7 @@
public String authenticator;
public String authorizer;
public String role_manager;
+ public String network_authorizer;
public volatile int permissions_validity_in_ms = 2000;
public volatile int permissions_cache_max_entries = 1000;
public volatile int permissions_update_interval_in_ms = -1;
@@ -81,6 +86,10 @@
public int num_tokens = 1;
/** Triggers automatic allocation of tokens if set, using the replication strategy of the referenced keyspace */
public String allocate_tokens_for_keyspace = null;
+ /** Triggers automatic allocation of tokens if set, based on the provided replica count for a datacenter */
+ public Integer allocate_tokens_for_local_replication_factor = null;
+
+ public long native_transport_idle_timeout_in_ms = 0L;
public volatile long request_timeout_in_ms = 10000L;
@@ -96,15 +105,10 @@
public volatile long truncate_request_timeout_in_ms = 60000L;
- /**
- * @deprecated use {@link this#streaming_keep_alive_period_in_secs} instead
- */
- @Deprecated
- public int streaming_socket_timeout_in_ms = 86400000; //24 hours
-
+ public Integer streaming_connections_per_host = 1;
public Integer streaming_keep_alive_period_in_secs = 300; //5 minutes
- public boolean cross_node_timeout = false;
+ public boolean cross_node_timeout = true;
public volatile long slow_query_log_timeout_in_ms = 500L;
@@ -124,7 +128,11 @@
public Float memtable_cleanup_threshold = null;
// Limit the maximum depth of repair session merkle trees
- public volatile int repair_session_max_tree_depth = 18;
+ @Deprecated
+ public volatile Integer repair_session_max_tree_depth = null;
+ public volatile Integer repair_session_space_in_mb = null;
+
+ public volatile boolean use_offheap_merkle_trees = true;
public int storage_port = 7000;
public int ssl_storage_port = 7001;
@@ -135,38 +143,58 @@
public boolean listen_on_broadcast_address = false;
public String internode_authenticator;
- /* intentionally left set to true, despite being set to false in stock 2.2 cassandra.yaml
- we don't want to surprise Thrift users who have the setting blank in the yaml during 2.1->2.2 upgrade */
- public boolean start_rpc = true;
+ /*
+ * RPC address and interface refer to the address/interface used for the native protocol used to communicate with
+ * clients. It's still called RPC in some places even though Thrift RPC is gone. If you see references to native
+ * address or native port it's derived from the RPC address configuration.
+ *
+ * native_transport_port is the port that is paired with RPC address to bind on.
+ */
public String rpc_address;
public String rpc_interface;
public boolean rpc_interface_prefer_ipv6 = false;
public String broadcast_rpc_address;
- public int rpc_port = 9160;
- public int rpc_listen_backlog = 50;
- public String rpc_server_type = "sync";
public boolean rpc_keepalive = true;
- public int rpc_min_threads = 16;
- public int rpc_max_threads = Integer.MAX_VALUE;
- public Integer rpc_send_buff_size_in_bytes;
- public Integer rpc_recv_buff_size_in_bytes;
- public int internode_send_buff_size_in_bytes = 0;
- public int internode_recv_buff_size_in_bytes = 0;
- public boolean start_native_transport = false;
+ public Integer internode_max_message_size_in_bytes;
+
+ public int internode_socket_send_buffer_size_in_bytes = 0;
+ public int internode_socket_receive_buffer_size_in_bytes = 0;
+
+ // TODO: derive defaults from system memory settings?
+ public int internode_application_send_queue_capacity_in_bytes = 1 << 22; // 4MiB
+ public int internode_application_send_queue_reserve_endpoint_capacity_in_bytes = 1 << 27; // 128MiB
+ public int internode_application_send_queue_reserve_global_capacity_in_bytes = 1 << 29; // 512MiB
+
+ public int internode_application_receive_queue_capacity_in_bytes = 1 << 22; // 4MiB
+ public int internode_application_receive_queue_reserve_endpoint_capacity_in_bytes = 1 << 27; // 128MiB
+ public int internode_application_receive_queue_reserve_global_capacity_in_bytes = 1 << 29; // 512MiB
+
+ // Defensive settings for protecting Cassandra from true network partitions. See (CASSANDRA-14358) for details.
+ // The amount of time to wait for internode tcp connections to establish.
+ public int internode_tcp_connect_timeout_in_ms = 2000;
+ // The amount of time unacknowledged data is allowed on a connection before we throw out the connection
+ // Note this is only supported on Linux + epoll, and it appears to behave oddly above a setting of 30000
+ // (it takes much longer than 30s) as of Linux 4.12. If you want something that high set this to 0
+ // (which picks up the OS default) and configure the net.ipv4.tcp_retries2 sysctl to be ~8.
+ public int internode_tcp_user_timeout_in_ms = 30000;
+
+ public boolean start_native_transport = true;
public int native_transport_port = 9042;
public Integer native_transport_port_ssl = null;
public int native_transport_max_threads = 128;
public int native_transport_max_frame_size_in_mb = 256;
public volatile long native_transport_max_concurrent_connections = -1L;
public volatile long native_transport_max_concurrent_connections_per_ip = -1L;
- public boolean native_transport_flush_in_batches_legacy = true;
+ public boolean native_transport_flush_in_batches_legacy = false;
+ public volatile boolean native_transport_allow_older_protocols = true;
+ public int native_transport_frame_block_size_in_kb = 32;
public volatile long native_transport_max_concurrent_requests_in_bytes_per_ip = -1L;
public volatile long native_transport_max_concurrent_requests_in_bytes = -1L;
- public Integer native_transport_max_negotiable_protocol_version = Integer.MIN_VALUE;
-
@Deprecated
- public int thrift_max_message_length_in_mb = 16;
+ public Integer native_transport_max_negotiable_protocol_version = null;
+
+
/**
* Max size of values in SSTables, in MegaBytes.
* Default is the same as the native protocol frame limit: 256Mb.
@@ -174,13 +202,12 @@
*/
public int max_value_size_in_mb = 256;
- public int thrift_framed_transport_size_in_mb = 15;
public boolean snapshot_before_compaction = false;
public boolean auto_snapshot = true;
/* if the size of columns or super-columns are more than this, indexing will kick in */
public int column_index_size_in_kb = 64;
- public int column_index_cache_size_in_kb = 2;
+ public volatile int column_index_cache_size_in_kb = 2;
public volatile int batch_size_warn_threshold_in_kb = 5;
public volatile int batch_size_fail_threshold_in_kb = 50;
public Integer unlogged_batch_across_partitions_warn_threshold = 10;
@@ -189,6 +216,8 @@
public volatile int compaction_large_partition_warning_threshold_mb = 100;
public int min_free_space_per_drive_in_mb = 50;
+ public volatile int concurrent_materialized_view_builders = 1;
+
/**
* @deprecated retry support removed on CASSANDRA-10992
*/
@@ -206,11 +235,18 @@
public String commitlog_directory;
public Integer commitlog_total_space_in_mb;
public CommitLogSync commitlog_sync;
+
+ /**
+ * @deprecated since 4.0 This value was near useless, and we're not using it anymore
+ */
public double commitlog_sync_batch_window_in_ms = Double.NaN;
+ public double commitlog_sync_group_window_in_ms = Double.NaN;
public int commitlog_sync_period_in_ms;
public int commitlog_segment_size_in_mb = 32;
public ParameterizedClass commitlog_compression;
+ public FlushCompression flush_compression = FlushCompression.fast;
public int commitlog_max_compression_buffers_in_pool = 3;
+ public Integer periodic_commitlog_sync_lag_block_in_ms;
public TransparentDataEncryptionOptions transparent_data_encryption_options = new TransparentDataEncryptionOptions();
public Integer max_mutation_size_in_kb;
@@ -230,32 +266,25 @@
public int dynamic_snitch_reset_interval_in_ms = 600000;
public double dynamic_snitch_badness_threshold = 0.1;
- public String request_scheduler;
- public RequestSchedulerId request_scheduler_id;
- public RequestSchedulerOptions request_scheduler_options;
-
public EncryptionOptions.ServerEncryptionOptions server_encryption_options = new EncryptionOptions.ServerEncryptionOptions();
- public EncryptionOptions.ClientEncryptionOptions client_encryption_options = new EncryptionOptions.ClientEncryptionOptions();
- // this encOptions is for backward compatibility (a warning is logged by DatabaseDescriptor)
- public EncryptionOptions.ServerEncryptionOptions encryption_options;
+ public EncryptionOptions client_encryption_options = new EncryptionOptions();
public InternodeCompression internode_compression = InternodeCompression.none;
- @Deprecated
- public Integer index_interval = null;
-
public int hinted_handoff_throttle_in_kb = 1024;
public int batchlog_replay_throttle_in_kb = 1024;
public int max_hints_delivery_threads = 2;
public int hints_flush_period_in_ms = 10000;
public int max_hints_file_size_in_mb = 128;
public ParameterizedClass hints_compression;
- public int sstable_preemptive_open_interval_in_mb = 50;
public volatile boolean incremental_backups = false;
public boolean trickle_fsync = false;
public int trickle_fsync_interval_in_kb = 10240;
+ public volatile int sstable_preemptive_open_interval_in_mb = 50;
+
+ public volatile boolean key_cache_migrate_during_compaction = true;
public Long key_cache_size_in_mb = null;
public volatile int key_cache_save_period = 14400;
public volatile int key_cache_keys_to_save = Integer.MAX_VALUE;
@@ -285,7 +314,8 @@
*/
public Boolean file_cache_round_up;
- public boolean buffer_pool_use_heap_if_exhausted = true;
+ @Deprecated
+ public boolean buffer_pool_use_heap_if_exhausted;
public DiskOptimizationStrategy disk_optimization_strategy = DiskOptimizationStrategy.ssd;
@@ -304,16 +334,23 @@
public volatile int index_summary_resize_interval_in_minutes = 60;
public int gc_log_threshold_in_ms = 200;
- public int gc_warn_threshold_in_ms = 0;
+ public int gc_warn_threshold_in_ms = 1000;
// TTL for different types of trace events.
public int tracetype_query_ttl = (int) TimeUnit.DAYS.toSeconds(1);
public int tracetype_repair_ttl = (int) TimeUnit.DAYS.toSeconds(7);
+ /**
+ * Maintain statistics on whether writes achieve the ideal consistency level
+ * before expiring and becoming hints
+ */
+ public volatile ConsistencyLevel ideal_consistency_level = null;
+
/*
- * Strategy to use for coalescing messages in OutboundTcpConnection.
+ * Strategy to use for coalescing messages in {@link OutboundConnections}.
* Can be fixed, movingaverage, timehorizon, disabled. Setting is case and leading/trailing
- * whitespace insensitive. You can also specify a subclass of CoalescingStrategies.CoalescingStrategy by name.
+ * whitespace insensitive. You can also specify a subclass of
+ * {@link org.apache.cassandra.utils.CoalescingStrategies.CoalescingStrategy} by name.
*/
public String otc_coalescing_strategy = "DISABLED";
@@ -327,12 +364,6 @@
public int otc_coalescing_window_us = otc_coalescing_window_us_default;
public int otc_coalescing_enough_coalesced_messages = 8;
- /**
- * Backlog expiration interval in milliseconds for the OutboundTcpConnection.
- */
- public static final int otc_backlog_expiration_interval_ms_default = 200;
- public volatile int otc_backlog_expiration_interval_ms = otc_backlog_expiration_interval_ms_default;
-
public int windows_timer_interval = 0;
/**
@@ -340,18 +371,15 @@
* Defaults to 1/256th of the heap size or 10MB, whichever is greater.
*/
public Long prepared_statements_cache_size_mb = null;
- /**
- * Size of the Thrift prepared statements cache in MB.
- * Defaults to 1/256th of the heap size or 10MB, whichever is greater.
- */
- public Long thrift_prepared_statements_cache_size_mb = null;
public boolean enable_user_defined_functions = false;
public boolean enable_scripted_user_defined_functions = false;
- public boolean enable_materialized_views = true;
+ public boolean enable_materialized_views = false;
- public boolean enable_sasi_indexes = true;
+ public boolean enable_transient_replication = false;
+
+ public boolean enable_sasi_indexes = false;
/**
* Optionally disable asynchronous UDF execution.
@@ -387,6 +415,85 @@
public volatile boolean back_pressure_enabled = false;
public volatile ParameterizedClass back_pressure_strategy;
+ public volatile int concurrent_validations;
+ public RepairCommandPoolFullStrategy repair_command_pool_full_strategy = RepairCommandPoolFullStrategy.queue;
+ public int repair_command_pool_size = concurrent_validations;
+
+ /**
+ * When a node first starts up it intially considers all other peers as DOWN and is disconnected from all of them.
+ * To be useful as a coordinator (and not introduce latency penalties on restart) this node must have successfully
+ * opened all three internode TCP connections (gossip, small, and large messages) before advertising to clients.
+ * Due to this, by default, Casssandra will prime these internode TCP connections and wait for all but a single
+ * node to be DOWN/disconnected in the local datacenter before offering itself as a coordinator, subject to a
+ * timeout. See CASSANDRA-13993 and CASSANDRA-14297 for more details.
+ *
+ * We provide two tunables to control this behavior as some users may want to block until all datacenters are
+ * available (global QUORUM/EACH_QUORUM), some users may not want to block at all (clients that already work
+ * around the problem), and some users may want to prime the connections but not delay startup.
+ *
+ * block_for_peers_timeout_in_secs: controls how long this node will wait to connect to peers. To completely disable
+ * any startup connectivity checks set this to -1. To trigger the internode connections but immediately continue
+ * startup, set this to to 0. The default is 10 seconds.
+ *
+ * block_for_peers_in_remote_dcs: controls if this node will consider remote datacenters to wait for. The default
+ * is to _not_ wait on remote datacenters.
+ */
+ public int block_for_peers_timeout_in_secs = 10;
+ public boolean block_for_peers_in_remote_dcs = false;
+
+ public volatile boolean automatic_sstable_upgrade = false;
+ public volatile int max_concurrent_automatic_sstable_upgrades = 1;
+ public boolean stream_entire_sstables = true;
+
+ public volatile AuditLogOptions audit_logging_options = new AuditLogOptions();
+ public volatile FullQueryLoggerOptions full_query_logging_options = new FullQueryLoggerOptions();
+
+ public CorruptedTombstoneStrategy corrupted_tombstone_strategy = CorruptedTombstoneStrategy.disabled;
+
+ public volatile boolean diagnostic_events_enabled = false;
+
+ /**
+ * flags for enabling tracking repaired state of data during reads
+ * separate flags for range & single partition reads as single partition reads are only tracked
+ * when CL > 1 and a digest mismatch occurs. Currently, range queries don't use digests so if
+ * enabled for range reads, all such reads will include repaired data tracking. As this adds
+ * some overhead, operators may wish to disable it whilst still enabling it for partition reads
+ */
+ public volatile boolean repaired_data_tracking_for_range_reads_enabled = false;
+ public volatile boolean repaired_data_tracking_for_partition_reads_enabled = false;
+ /* If true, unconfirmed mismatches (those which cannot be considered conclusive proof of out of
+ * sync repaired data due to the presence of pending repair sessions, or unrepaired partition
+ * deletes) will increment a metric, distinct from confirmed mismatches. If false, unconfirmed
+ * mismatches are simply ignored by the coordinator.
+ * This is purely to allow operators to avoid potential signal:noise issues as these types of
+ * mismatches are considerably less actionable than their confirmed counterparts. Setting this
+ * to true only disables the incrementing of the counters when an unconfirmed mismatch is found
+ * and has no other effect on the collection or processing of the repaired data.
+ */
+ public volatile boolean report_unconfirmed_repaired_data_mismatches = false;
+ /*
+ * If true, when a repaired data mismatch is detected at read time or during a preview repair,
+ * a snapshot request will be issued to each particpating replica. These are limited at the replica level
+ * so that only a single snapshot per-table per-day can be taken via this method.
+ */
+ public volatile boolean snapshot_on_repaired_data_mismatch = false;
+
+ /**
+ * number of seconds to set nowInSec into the future when performing validation previews against repaired data
+ * this (attempts) to prevent a race where validations on different machines are started on different sides of
+ * a tombstone being compacted away
+ */
+ public volatile int validation_preview_purge_head_start_in_sec = 60 * 60;
+
+ /**
+ * The intial capacity for creating RangeTombstoneList.
+ */
+ public volatile int initial_range_tombstone_list_allocation_size = 1;
+ /**
+ * The growth factor to enlarge a RangeTombstoneList.
+ */
+ public volatile double range_tombstone_list_growth_factor = 1.5;
+
/**
* @deprecated migrate to {@link DatabaseDescriptor#isClientInitialized()}
*/
@@ -414,6 +521,8 @@
public volatile boolean check_for_duplicate_rows_during_reads = true;
public volatile boolean check_for_duplicate_rows_during_compaction = true;
+ public boolean autocompaction_on_startup_enabled = Boolean.parseBoolean(System.getProperty("cassandra.autocompaction_on_startup_enabled", "true"));
+
/**
* Client mode means that the process is a pure client, that uses C* code base but does
* not read or write local C* database files.
@@ -439,8 +548,17 @@
public enum CommitLogSync
{
periodic,
- batch
+ batch,
+ group
}
+
+ public enum FlushCompression
+ {
+ none,
+ fast,
+ table
+ }
+
public enum InternodeCompression
{
all, none, dc
@@ -486,17 +604,25 @@
die_immediate
}
- public enum RequestSchedulerId
- {
- keyspace
- }
-
public enum DiskOptimizationStrategy
{
ssd,
spinning
}
+ public enum RepairCommandPoolFullStrategy
+ {
+ queue,
+ reject
+ }
+
+ public enum CorruptedTombstoneStrategy
+ {
+ disabled,
+ warn,
+ exception
+ }
+
private static final List<String> SENSITIVE_KEYS = new ArrayList<String>() {{
add("client_encryption_options");
add("server_encryption_options");
diff --git a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
index 86d2287..5f94710 100644
--- a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
+++ b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
@@ -26,9 +26,12 @@
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.*;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Function;
import java.util.function.Supplier;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableSet;
import com.google.common.primitives.Ints;
import com.google.common.primitives.Longs;
@@ -36,13 +39,22 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.cassandra.audit.AuditLogOptions;
+import org.apache.cassandra.fql.FullQueryLoggerOptions;
+import org.apache.cassandra.auth.AllowAllInternodeAuthenticator;
import org.apache.cassandra.auth.AuthConfig;
import org.apache.cassandra.auth.IAuthenticator;
import org.apache.cassandra.auth.IAuthorizer;
import org.apache.cassandra.auth.IInternodeAuthenticator;
+import org.apache.cassandra.auth.INetworkAuthorizer;
import org.apache.cassandra.auth.IRoleManager;
import org.apache.cassandra.config.Config.CommitLogSync;
-import org.apache.cassandra.config.Config.RequestSchedulerId;
+import org.apache.cassandra.config.EncryptionOptions.ServerEncryptionOptions.InternodeEncryption;
+import org.apache.cassandra.db.ConsistencyLevel;
+import org.apache.cassandra.db.commitlog.AbstractCommitLogSegmentManager;
+import org.apache.cassandra.db.commitlog.CommitLog;
+import org.apache.cassandra.db.commitlog.CommitLogSegmentManagerCDC;
+import org.apache.cassandra.db.commitlog.CommitLogSegmentManagerStandard;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.io.FSWriteError;
@@ -53,24 +65,31 @@
import org.apache.cassandra.locator.DynamicEndpointSnitch;
import org.apache.cassandra.locator.EndpointSnitchInfo;
import org.apache.cassandra.locator.IEndpointSnitch;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.Replica;
import org.apache.cassandra.locator.SeedProvider;
import org.apache.cassandra.net.BackPressureStrategy;
import org.apache.cassandra.net.RateBasedBackPressure;
-import org.apache.cassandra.scheduler.IRequestScheduler;
-import org.apache.cassandra.scheduler.NoScheduler;
import org.apache.cassandra.security.EncryptionContext;
+import org.apache.cassandra.security.SSLFactory;
import org.apache.cassandra.service.CacheService.CacheType;
-import org.apache.cassandra.thrift.ThriftServer.ThriftServerType;
-import org.apache.cassandra.transport.ProtocolVersion;
-import org.apache.cassandra.transport.ProtocolVersionLimit;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.commons.lang3.StringUtils;
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
import static org.apache.cassandra.io.util.FileUtils.ONE_GB;
+import static org.apache.cassandra.io.util.FileUtils.ONE_MB;
public class DatabaseDescriptor
{
+ static
+ {
+ // This static block covers most usages
+ FBUtilities.preventIllegalAccessWarnings();
+ System.setProperty("io.netty.transport.estimateSizeOnSubmit", "false");
+ }
+
private static final Logger logger = LoggerFactory.getLogger(DatabaseDescriptor.class);
/**
@@ -81,13 +100,18 @@
private static Config conf;
+ /**
+ * Request timeouts can not be less than below defined value (see CASSANDRA-9375)
+ */
+ static final long LOWEST_ACCEPTED_TIMEOUT = 10L;
+
private static IEndpointSnitch snitch;
private static InetAddress listenAddress; // leave null so we can fall through to getLocalHost
private static InetAddress broadcastAddress;
private static InetAddress rpcAddress;
private static InetAddress broadcastRpcAddress;
private static SeedProvider seedProvider;
- private static IInternodeAuthenticator internodeAuthenticator;
+ private static IInternodeAuthenticator internodeAuthenticator = new AllowAllInternodeAuthenticator();
/* Hashing strategy Random or OPHF */
private static IPartitioner partitioner;
@@ -97,23 +121,19 @@
private static IAuthenticator authenticator;
private static IAuthorizer authorizer;
+ private static INetworkAuthorizer networkAuthorizer;
// Don't initialize the role manager until applying config. The options supported by CassandraRoleManager
// depend on the configured IAuthenticator, so defer creating it until that's been set.
private static IRoleManager roleManager;
- private static IRequestScheduler requestScheduler;
- private static RequestSchedulerId requestSchedulerId;
- private static RequestSchedulerOptions requestSchedulerOptions;
-
private static long preparedStatementsCacheSizeInMB;
- private static long thriftPreparedStatementsCacheSizeInMB;
private static long keyCacheSizeInMB;
private static long counterCacheSizeInMB;
private static long indexSummaryCapacityInMB;
private static String localDC;
- private static Comparator<InetAddress> localComparator;
+ private static Comparator<Replica> localComparator;
private static EncryptionContext encryptionContext;
private static boolean hasLoggedConfig;
@@ -126,12 +146,18 @@
private static final int searchConcurrencyFactor = Integer.parseInt(System.getProperty(Config.PROPERTY_PREFIX + "search_concurrency_factor", "1"));
- private static final boolean disableSTCSInL0 = Boolean.getBoolean(Config.PROPERTY_PREFIX + "disable_stcs_in_l0");
+ private static volatile boolean disableSTCSInL0 = Boolean.getBoolean(Config.PROPERTY_PREFIX + "disable_stcs_in_l0");
private static final boolean unsafeSystem = Boolean.getBoolean(Config.PROPERTY_PREFIX + "unsafesystem");
// turns some warnings into exceptions for testing
private static final boolean strictRuntimeChecks = Boolean.getBoolean("cassandra.strict.runtime.checks");
+ public static volatile boolean allowUnlimitedConcurrentValidations = Boolean.getBoolean("cassandra.allow_unlimited_concurrent_validations");
+
+ private static Function<CommitLog, AbstractCommitLogSegmentManager> commitLogSegmentMgrProvider = c -> DatabaseDescriptor.isCDCEnabled()
+ ? new CommitLogSegmentManagerCDC(c, DatabaseDescriptor.getCommitLogLocation())
+ : new CommitLogSegmentManagerStandard(c, DatabaseDescriptor.getCommitLogLocation());
+
public static void daemonInitialization() throws ConfigurationException
{
daemonInitialization(DatabaseDescriptor::loadConfig);
@@ -321,27 +347,29 @@
private static void applyAll() throws ConfigurationException
{
+ //InetAddressAndPort cares that applySimpleConfig runs first
applySimpleConfig();
applyPartitioner();
applyAddressConfig();
- applyThriftHSHA();
-
applySnitch();
- applyRequestScheduler();
-
applyInitialTokens();
applySeedProvider();
applyEncryptionContext();
+
+ applySslContextHotReload();
}
private static void applySimpleConfig()
{
+ //Doing this first before all other things in case other pieces of config want to construct
+ //InetAddressAndPort and get the right defaults
+ InetAddressAndPort.initializeDefaultPort(getStoragePort());
if (conf.commitlog_sync == null)
{
@@ -350,15 +378,23 @@
if (conf.commitlog_sync == Config.CommitLogSync.batch)
{
- if (Double.isNaN(conf.commitlog_sync_batch_window_in_ms) || conf.commitlog_sync_batch_window_in_ms <= 0d)
- {
- throw new ConfigurationException("Missing value for commitlog_sync_batch_window_in_ms: positive double value expected.", false);
- }
- else if (conf.commitlog_sync_period_in_ms != 0)
+ if (conf.commitlog_sync_period_in_ms != 0)
{
throw new ConfigurationException("Batch sync specified, but commitlog_sync_period_in_ms found. Only specify commitlog_sync_batch_window_in_ms when using batch sync", false);
}
- logger.debug("Syncing log with a batch window of {}", conf.commitlog_sync_batch_window_in_ms);
+ logger.debug("Syncing log with batch mode");
+ }
+ else if (conf.commitlog_sync == CommitLogSync.group)
+ {
+ if (Double.isNaN(conf.commitlog_sync_group_window_in_ms) || conf.commitlog_sync_group_window_in_ms <= 0d)
+ {
+ throw new ConfigurationException("Missing value for commitlog_sync_group_window_in_ms: positive double value expected.", false);
+ }
+ else if (conf.commitlog_sync_period_in_ms != 0)
+ {
+ throw new ConfigurationException("Group sync specified, but commitlog_sync_period_in_ms found. Only specify commitlog_sync_group_window_in_ms when using group sync", false);
+ }
+ logger.debug("Syncing log with a group window of {}", conf.commitlog_sync_period_in_ms);
}
else
{
@@ -442,19 +478,47 @@
else
logger.info("Global memtable off-heap threshold is enabled at {}MB", conf.memtable_offheap_space_in_mb);
- if (conf.repair_session_max_tree_depth < 10)
- throw new ConfigurationException("repair_session_max_tree_depth should not be < 10, but was " + conf.repair_session_max_tree_depth);
- if (conf.repair_session_max_tree_depth > 20)
- logger.warn("repair_session_max_tree_depth of " + conf.repair_session_max_tree_depth + " > 20 could lead to excessive memory usage");
+ if (conf.repair_session_max_tree_depth != null)
+ {
+ logger.warn("repair_session_max_tree_depth has been deprecated and should be removed from cassandra.yaml. Use repair_session_space_in_mb instead");
+ if (conf.repair_session_max_tree_depth < 10)
+ throw new ConfigurationException("repair_session_max_tree_depth should not be < 10, but was " + conf.repair_session_max_tree_depth);
+ if (conf.repair_session_max_tree_depth > 20)
+ logger.warn("repair_session_max_tree_depth of " + conf.repair_session_max_tree_depth + " > 20 could lead to excessive memory usage");
+ }
+ else
+ {
+ conf.repair_session_max_tree_depth = 20;
+ }
- if (conf.thrift_framed_transport_size_in_mb <= 0)
- throw new ConfigurationException("thrift_framed_transport_size_in_mb must be positive, but was " + conf.thrift_framed_transport_size_in_mb, false);
+ if (conf.repair_session_space_in_mb == null)
+ conf.repair_session_space_in_mb = Math.max(1, (int) (Runtime.getRuntime().maxMemory() / (16 * 1048576)));
- if (conf.native_transport_max_frame_size_in_mb <= 0)
- throw new ConfigurationException("native_transport_max_frame_size_in_mb must be positive, but was " + conf.native_transport_max_frame_size_in_mb, false);
- else if (conf.native_transport_max_frame_size_in_mb >= 2048)
- throw new ConfigurationException("native_transport_max_frame_size_in_mb must be smaller than 2048, but was "
- + conf.native_transport_max_frame_size_in_mb, false);
+ if (conf.repair_session_space_in_mb < 1)
+ throw new ConfigurationException("repair_session_space_in_mb must be > 0, but was " + conf.repair_session_space_in_mb);
+ else if (conf.repair_session_space_in_mb > (int) (Runtime.getRuntime().maxMemory() / (4 * 1048576)))
+ logger.warn("A repair_session_space_in_mb of " + conf.repair_session_space_in_mb + " megabytes is likely to cause heap pressure");
+
+ checkForLowestAcceptedTimeouts(conf);
+
+ checkValidForByteConversion(conf.native_transport_max_frame_size_in_mb,
+ "native_transport_max_frame_size_in_mb", ByteUnit.MEBI_BYTES);
+
+ checkValidForByteConversion(conf.column_index_size_in_kb,
+ "column_index_size_in_kb", ByteUnit.KIBI_BYTES);
+
+ checkValidForByteConversion(conf.column_index_cache_size_in_kb,
+ "column_index_cache_size_in_kb", ByteUnit.KIBI_BYTES);
+
+ checkValidForByteConversion(conf.batch_size_warn_threshold_in_kb,
+ "batch_size_warn_threshold_in_kb", ByteUnit.KIBI_BYTES);
+
+ checkValidForByteConversion(conf.native_transport_frame_block_size_in_kb,
+ "native_transport_frame_block_size_in_kb", ByteUnit.KIBI_BYTES);
+
+ if (conf.native_transport_max_negotiable_protocol_version != null)
+ logger.warn("The configuration option native_transport_max_negotiable_protocol_version has been deprecated " +
+ "and should be removed from cassandra.yaml as it has no longer has any effect.");
// if data dirs, commitlog dir, or saved caches dir are set in cassandra.yaml, use that. Otherwise,
// use -Dcassandra.storagedir (set in cassandra-env.sh) as the parent dir for data/, commitlog/, and saved_caches/
@@ -478,67 +542,60 @@
conf.native_transport_max_concurrent_requests_in_bytes_per_ip = Runtime.getRuntime().maxMemory() / 40;
}
- if (conf.cdc_raw_directory == null)
- {
- conf.cdc_raw_directory = storagedirFor("cdc_raw");
- }
-
if (conf.commitlog_total_space_in_mb == null)
{
- int preferredSize = 8192;
- int minSize = 0;
+ final int preferredSizeInMB = 8192;
try
{
// use 1/4 of available space. See discussion on #10013 and #10199
- minSize = Ints.saturatedCast((guessFileStore(conf.commitlog_directory).getTotalSpace() / 1048576) / 4);
- }
- catch (IOException e)
- {
- logger.debug("Error checking disk space", e);
- throw new ConfigurationException(String.format("Unable to check disk space available to %s. Perhaps the Cassandra user does not have the necessary permissions",
- conf.commitlog_directory), e);
- }
- if (minSize < preferredSize)
- {
- logger.warn("Small commitlog volume detected at {}; setting commitlog_total_space_in_mb to {}. You can override this in cassandra.yaml",
- conf.commitlog_directory, minSize);
- conf.commitlog_total_space_in_mb = minSize;
- }
- else
- {
- conf.commitlog_total_space_in_mb = preferredSize;
- }
- }
+ final long totalSpaceInBytes = guessFileStore(conf.commitlog_directory).getTotalSpace();
+ conf.commitlog_total_space_in_mb = calculateDefaultSpaceInMB("commitlog",
+ conf.commitlog_directory,
+ "commitlog_total_space_in_mb",
+ preferredSizeInMB,
+ totalSpaceInBytes, 1, 4);
- if (conf.cdc_total_space_in_mb == 0)
- {
- int preferredSize = 4096;
- int minSize = 0;
- try
- {
- // use 1/8th of available space. See discussion on #10013 and #10199 on the CL, taking half that for CDC
- minSize = Ints.saturatedCast((guessFileStore(conf.cdc_raw_directory).getTotalSpace() / 1048576) / 8);
}
catch (IOException e)
{
logger.debug("Error checking disk space", e);
- throw new ConfigurationException(String.format("Unable to check disk space available to %s. Perhaps the Cassandra user does not have the necessary permissions",
- conf.cdc_raw_directory), e);
- }
- if (minSize < preferredSize)
- {
- logger.warn("Small cdc volume detected at {}; setting cdc_total_space_in_mb to {}. You can override this in cassandra.yaml",
- conf.cdc_raw_directory, minSize);
- conf.cdc_total_space_in_mb = minSize;
- }
- else
- {
- conf.cdc_total_space_in_mb = preferredSize;
+ throw new ConfigurationException(String.format("Unable to check disk space available to '%s'. Perhaps the Cassandra user does not have the necessary permissions",
+ conf.commitlog_directory), e);
}
}
if (conf.cdc_enabled)
{
+ // Windows memory-mapped CommitLog files is incompatible with CDC as we hard-link files in cdc_raw. Confirm we don't have both enabled.
+ if (FBUtilities.isWindows && conf.commitlog_compression == null)
+ throw new ConfigurationException("Cannot enable cdc on Windows with uncompressed commitlog.");
+
+ if (conf.cdc_raw_directory == null)
+ {
+ conf.cdc_raw_directory = storagedirFor("cdc_raw");
+ }
+
+ if (conf.cdc_total_space_in_mb == 0)
+ {
+ final int preferredSizeInMB = 4096;
+ try
+ {
+ // use 1/8th of available space. See discussion on #10013 and #10199 on the CL, taking half that for CDC
+ final long totalSpaceInBytes = guessFileStore(conf.cdc_raw_directory).getTotalSpace();
+ conf.cdc_total_space_in_mb = calculateDefaultSpaceInMB("cdc",
+ conf.cdc_raw_directory,
+ "cdc_total_space_in_mb",
+ preferredSizeInMB,
+ totalSpaceInBytes, 1, 8);
+ }
+ catch (IOException e)
+ {
+ logger.debug("Error checking disk space", e);
+ throw new ConfigurationException(String.format("Unable to check disk space available to '%s'. Perhaps the Cassandra user does not have the necessary permissions",
+ conf.cdc_raw_directory), e);
+ }
+ }
+
logger.info("cdc_enabled is true. Starting casssandra node with Change-Data-Capture enabled.");
}
@@ -616,6 +673,12 @@
if (conf.concurrent_compactors <= 0)
throw new ConfigurationException("concurrent_compactors should be strictly greater than 0, but was " + conf.concurrent_compactors, false);
+ applyConcurrentValidations(conf);
+ applyRepairCommandPoolSize(conf);
+
+ if (conf.concurrent_materialized_view_builders <= 0)
+ throw new ConfigurationException("concurrent_materialized_view_builders should be strictly greater than 0, but was " + conf.concurrent_materialized_view_builders, false);
+
if (conf.num_tokens > MAX_NUM_TOKENS)
throw new ConfigurationException(String.format("A maximum number of %d tokens per node is supported", MAX_NUM_TOKENS), false);
@@ -637,22 +700,6 @@
try
{
- // if thrift_prepared_statements_cache_size_mb option was set to "auto" then size of the cache should be "max(1/256 of Heap (in MB), 10MB)"
- thriftPreparedStatementsCacheSizeInMB = (conf.thrift_prepared_statements_cache_size_mb == null)
- ? Math.max(10, (int) (Runtime.getRuntime().maxMemory() / 1024 / 1024 / 256))
- : conf.thrift_prepared_statements_cache_size_mb;
-
- if (thriftPreparedStatementsCacheSizeInMB <= 0)
- throw new NumberFormatException(); // to escape duplicating error message
- }
- catch (NumberFormatException e)
- {
- throw new ConfigurationException("thrift_prepared_statements_cache_size_mb option was set incorrectly to '"
- + conf.thrift_prepared_statements_cache_size_mb + "', supported values are <integer> >= 0.", false);
- }
-
- try
- {
// if key_cache_size_in_mb option was set to "auto" then size of the cache should be "min(5% of Heap (in MB), 100MB)
keyCacheSizeInMB = (conf.key_cache_size_in_mb == null)
? Math.min(Math.max(1, (int) (Runtime.getRuntime().totalMemory() * 0.05 / 1024 / 1024)), 100)
@@ -692,16 +739,6 @@
throw new ConfigurationException("index_summary_capacity_in_mb option was set incorrectly to '"
+ conf.index_summary_capacity_in_mb + "', it should be a non-negative integer.", false);
- if (conf.index_interval != null)
- logger.warn("index_interval has been deprecated and should be removed from cassandra.yaml");
-
- if(conf.encryption_options != null)
- {
- logger.warn("Please rename encryption_options as server_encryption_options in the yaml");
- //operate under the assumption that server_encryption_options is not set in yaml rather than both
- conf.server_encryption_options = conf.encryption_options;
- }
-
if (conf.user_defined_function_fail_timeout < 0)
throw new ConfigurationException("user_defined_function_fail_timeout must not be negative", false);
if (conf.user_defined_function_warn_timeout < 0)
@@ -725,26 +762,11 @@
// native transport encryption options
if (conf.native_transport_port_ssl != null
&& conf.native_transport_port_ssl != conf.native_transport_port
- && !conf.client_encryption_options.enabled)
+ && !conf.client_encryption_options.isEnabled())
{
throw new ConfigurationException("Encryption must be enabled in client_encryption_options for native_transport_port_ssl", false);
}
- // If max protocol version has been set, just validate it's within an acceptable range
- if (conf.native_transport_max_negotiable_protocol_version != Integer.MIN_VALUE)
- {
- try
- {
- ProtocolVersion.decode(conf.native_transport_max_negotiable_protocol_version, ProtocolVersionLimit.SERVER_DEFAULT);
- logger.info("Native transport max negotiable version statically limited to {}", conf.native_transport_max_negotiable_protocol_version);
- }
- catch (Exception e)
- {
- throw new ConfigurationException("Invalid setting for native_transport_max_negotiable_protocol_version; " +
- ProtocolVersion.invalidVersionMessage(conf.native_transport_max_negotiable_protocol_version));
- }
- }
-
if (conf.max_value_size_in_mb <= 0)
throw new ConfigurationException("max_value_size_in_mb must be positive", false);
else if (conf.max_value_size_in_mb >= 2048)
@@ -787,6 +809,51 @@
if (conf.otc_coalescing_enough_coalesced_messages <= 0)
throw new ConfigurationException("otc_coalescing_enough_coalesced_messages must be positive", false);
+
+ Integer maxMessageSize = conf.internode_max_message_size_in_bytes;
+ if (maxMessageSize != null)
+ {
+ if (maxMessageSize > conf.internode_application_receive_queue_reserve_endpoint_capacity_in_bytes)
+ throw new ConfigurationException("internode_max_message_size_in_mb must no exceed internode_application_receive_queue_reserve_endpoint_capacity_in_bytes", false);
+
+ if (maxMessageSize > conf.internode_application_receive_queue_reserve_global_capacity_in_bytes)
+ throw new ConfigurationException("internode_max_message_size_in_mb must no exceed internode_application_receive_queue_reserve_global_capacity_in_bytes", false);
+
+ if (maxMessageSize > conf.internode_application_send_queue_reserve_endpoint_capacity_in_bytes)
+ throw new ConfigurationException("internode_max_message_size_in_mb must no exceed internode_application_send_queue_reserve_endpoint_capacity_in_bytes", false);
+
+ if (maxMessageSize > conf.internode_application_send_queue_reserve_global_capacity_in_bytes)
+ throw new ConfigurationException("internode_max_message_size_in_mb must no exceed internode_application_send_queue_reserve_global_capacity_in_bytes", false);
+ }
+ else
+ {
+ conf.internode_max_message_size_in_bytes =
+ Math.min(conf.internode_application_receive_queue_reserve_endpoint_capacity_in_bytes,
+ conf.internode_application_send_queue_reserve_endpoint_capacity_in_bytes);
+ }
+
+ validateMaxConcurrentAutoUpgradeTasksConf(conf.max_concurrent_automatic_sstable_upgrades);
+ }
+
+ @VisibleForTesting
+ static void applyConcurrentValidations(Config config)
+ {
+ if (config.concurrent_validations < 1)
+ {
+ config.concurrent_validations = config.concurrent_compactors;
+ }
+ else if (config.concurrent_validations > config.concurrent_compactors && !allowUnlimitedConcurrentValidations)
+ {
+ throw new ConfigurationException("To set concurrent_validations > concurrent_compactors, " +
+ "set the system property cassandra.allow_unlimited_concurrent_validations=true");
+ }
+ }
+
+ @VisibleForTesting
+ static void applyRepairCommandPoolSize(Config config)
+ {
+ if (config.repair_command_pool_size < 1)
+ config.repair_command_pool_size = config.concurrent_validations;
}
private static String storagedirFor(String type)
@@ -802,6 +869,23 @@
return storagedir;
}
+ static int calculateDefaultSpaceInMB(String type, String path, String setting, int preferredSizeInMB, long totalSpaceInBytes, long totalSpaceNumerator, long totalSpaceDenominator)
+ {
+ final long totalSizeInMB = totalSpaceInBytes / ONE_MB;
+ final int minSizeInMB = Ints.saturatedCast(totalSpaceNumerator * totalSizeInMB / totalSpaceDenominator);
+
+ if (minSizeInMB < preferredSizeInMB)
+ {
+ logger.warn("Small {} volume detected at '{}'; setting {} to {}. You can override this in cassandra.yaml",
+ type, path, setting, minSizeInMB);
+ return minSizeInMB;
+ }
+ else
+ {
+ return preferredSizeInMB;
+ }
+ }
+
public static void applyAddressConfig() throws ConfigurationException
{
applyAddressConfig(conf);
@@ -876,7 +960,7 @@
}
else
{
- rpcAddress = FBUtilities.getLocalAddress();
+ rpcAddress = FBUtilities.getJustLocalAddress();
}
/* RPC address to broadcast */
@@ -902,18 +986,6 @@
}
}
- public static void applyThriftHSHA()
- {
- // fail early instead of OOMing (see CASSANDRA-8116)
- if (ThriftServerType.HSHA.equals(conf.rpc_server_type) && conf.rpc_max_threads == Integer.MAX_VALUE)
- throw new ConfigurationException("The hsha rpc_server_type is not compatible with an rpc_max_threads " +
- "setting of 'unlimited'. Please see the comments in cassandra.yaml " +
- "for rpc_server_type and rpc_max_threads.",
- false);
- if (ThriftServerType.HSHA.equals(conf.rpc_server_type) && conf.rpc_max_threads > (FBUtilities.getAvailableProcessors() * 2 + 1024))
- logger.warn("rpc_max_threads setting of {} may be too high for the hsha server and cause unnecessary thread contention, reducing performance", conf.rpc_max_threads);
- }
-
public static void applyEncryptionContext()
{
// always attempt to load the cipher factory, as we could be in the situation where the user has disabled encryption,
@@ -921,6 +993,18 @@
encryptionContext = new EncryptionContext(conf.transparent_data_encryption_options);
}
+ public static void applySslContextHotReload()
+ {
+ try
+ {
+ SSLFactory.initHotReloading(conf.server_encryption_options, conf.client_encryption_options, false);
+ }
+ catch(IOException e)
+ {
+ throw new ConfigurationException("Failed to initialize SSL hot reloading", e);
+ }
+ }
+
public static void applySeedProvider()
{
// load the seeds for node contact points
@@ -942,6 +1026,57 @@
throw new ConfigurationException("The seed provider lists no seeds.", false);
}
+ @VisibleForTesting
+ static void checkForLowestAcceptedTimeouts(Config conf)
+ {
+ if(conf.read_request_timeout_in_ms < LOWEST_ACCEPTED_TIMEOUT)
+ {
+ logInfo("read_request_timeout_in_ms", conf.read_request_timeout_in_ms, LOWEST_ACCEPTED_TIMEOUT);
+ conf.read_request_timeout_in_ms = LOWEST_ACCEPTED_TIMEOUT;
+ }
+
+ if(conf.range_request_timeout_in_ms < LOWEST_ACCEPTED_TIMEOUT)
+ {
+ logInfo("range_request_timeout_in_ms", conf.range_request_timeout_in_ms, LOWEST_ACCEPTED_TIMEOUT);
+ conf.range_request_timeout_in_ms = LOWEST_ACCEPTED_TIMEOUT;
+ }
+
+ if(conf.request_timeout_in_ms < LOWEST_ACCEPTED_TIMEOUT)
+ {
+ logInfo("request_timeout_in_ms", conf.request_timeout_in_ms, LOWEST_ACCEPTED_TIMEOUT);
+ conf.request_timeout_in_ms = LOWEST_ACCEPTED_TIMEOUT;
+ }
+
+ if(conf.write_request_timeout_in_ms < LOWEST_ACCEPTED_TIMEOUT)
+ {
+ logInfo("write_request_timeout_in_ms", conf.write_request_timeout_in_ms, LOWEST_ACCEPTED_TIMEOUT);
+ conf.write_request_timeout_in_ms = LOWEST_ACCEPTED_TIMEOUT;
+ }
+
+ if(conf.cas_contention_timeout_in_ms < LOWEST_ACCEPTED_TIMEOUT)
+ {
+ logInfo("cas_contention_timeout_in_ms", conf.cas_contention_timeout_in_ms, LOWEST_ACCEPTED_TIMEOUT);
+ conf.cas_contention_timeout_in_ms = LOWEST_ACCEPTED_TIMEOUT;
+ }
+
+ if(conf.counter_write_request_timeout_in_ms < LOWEST_ACCEPTED_TIMEOUT)
+ {
+ logInfo("counter_write_request_timeout_in_ms", conf.counter_write_request_timeout_in_ms, LOWEST_ACCEPTED_TIMEOUT);
+ conf.counter_write_request_timeout_in_ms = LOWEST_ACCEPTED_TIMEOUT;
+ }
+
+ if(conf.truncate_request_timeout_in_ms < LOWEST_ACCEPTED_TIMEOUT)
+ {
+ logInfo("truncate_request_timeout_in_ms", conf.truncate_request_timeout_in_ms, LOWEST_ACCEPTED_TIMEOUT);
+ conf.truncate_request_timeout_in_ms = LOWEST_ACCEPTED_TIMEOUT;
+ }
+ }
+
+ private static void logInfo(String property, long actualValue, long lowestAcceptedValue)
+ {
+ logger.info("found {}::{} less than lowest acceptable value {}, continuing with {}", property, actualValue, lowestAcceptedValue, lowestAcceptedValue);
+ }
+
public static void applyInitialTokens()
{
if (conf.initial_token != null)
@@ -955,47 +1090,6 @@
}
}
- // Maybe safe for clients + tools
- public static void applyRequestScheduler()
- {
- /* Request Scheduler setup */
- requestSchedulerOptions = conf.request_scheduler_options;
- if (conf.request_scheduler != null)
- {
- try
- {
- if (requestSchedulerOptions == null)
- {
- requestSchedulerOptions = new RequestSchedulerOptions();
- }
- Class<?> cls = Class.forName(conf.request_scheduler);
- requestScheduler = (IRequestScheduler) cls.getConstructor(RequestSchedulerOptions.class).newInstance(requestSchedulerOptions);
- }
- catch (ClassNotFoundException e)
- {
- throw new ConfigurationException("Invalid Request Scheduler class " + conf.request_scheduler, false);
- }
- catch (Exception e)
- {
- throw new ConfigurationException("Unable to instantiate request scheduler", e);
- }
- }
- else
- {
- requestScheduler = new NoScheduler();
- }
-
- if (conf.request_scheduler_id == RequestSchedulerId.keyspace)
- {
- requestSchedulerId = conf.request_scheduler_id;
- }
- else
- {
- // Default to Keyspace
- requestSchedulerId = RequestSchedulerId.keyspace;
- }
- }
-
// definitely not safe for tools + clients - implicitly instantiates StorageService
public static void applySnitch()
{
@@ -1007,37 +1101,40 @@
snitch = createEndpointSnitch(conf.dynamic_snitch, conf.endpoint_snitch);
EndpointSnitchInfo.create();
- localDC = snitch.getDatacenter(FBUtilities.getBroadcastAddress());
- localComparator = new Comparator<InetAddress>()
- {
- public int compare(InetAddress endpoint1, InetAddress endpoint2)
- {
- boolean local1 = localDC.equals(snitch.getDatacenter(endpoint1));
- boolean local2 = localDC.equals(snitch.getDatacenter(endpoint2));
- if (local1 && !local2)
- return -1;
- if (local2 && !local1)
- return 1;
- return 0;
- }
+ localDC = snitch.getLocalDatacenter();
+ localComparator = (replica1, replica2) -> {
+ boolean local1 = localDC.equals(snitch.getDatacenter(replica1));
+ boolean local2 = localDC.equals(snitch.getDatacenter(replica2));
+ if (local1 && !local2)
+ return -1;
+ if (local2 && !local1)
+ return 1;
+ return 0;
};
}
// definitely not safe for tools + clients - implicitly instantiates schema
public static void applyPartitioner()
{
+ applyPartitioner(conf);
+ }
+
+ public static void applyPartitioner(Config conf)
+ {
/* Hashing strategy */
if (conf.partitioner == null)
{
throw new ConfigurationException("Missing directive: partitioner", false);
}
+ String name = conf.partitioner;
try
{
- partitioner = FBUtilities.newPartitioner(System.getProperty(Config.PROPERTY_PREFIX + "partitioner", conf.partitioner));
+ name = System.getProperty(Config.PROPERTY_PREFIX + "partitioner", conf.partitioner);
+ partitioner = FBUtilities.newPartitioner(name);
}
catch (Exception e)
{
- throw new ConfigurationException("Invalid partitioner class " + conf.partitioner, false);
+ throw new ConfigurationException("Invalid partitioner class " + name, e);
}
paritionerName = partitioner.getClass().getCanonicalName();
@@ -1069,9 +1166,17 @@
catch (IOException e)
{
if (e instanceof NoSuchFileException)
+ {
path = path.getParent();
+ if (path == null)
+ {
+ throw new ConfigurationException("Unable to find filesystem for '" + dir + "'.");
+ }
+ }
else
+ {
throw e;
+ }
}
}
}
@@ -1104,6 +1209,16 @@
DatabaseDescriptor.authorizer = authorizer;
}
+ public static INetworkAuthorizer getNetworkAuthorizer()
+ {
+ return networkAuthorizer;
+ }
+
+ public static void setNetworkAuthorizer(INetworkAuthorizer networkAuthorizer)
+ {
+ DatabaseDescriptor.networkAuthorizer = networkAuthorizer;
+ }
+
public static IRoleManager getRoleManager()
{
return roleManager;
@@ -1210,11 +1325,6 @@
return conf.credentials_cache_max_entries = maxEntries;
}
- public static int getThriftFramedTransportSize()
- {
- return conf.thrift_framed_transport_size_in_mb * 1024 * 1024;
- }
-
public static int getMaxValueSize()
{
return conf.max_value_size_in_mb * 1024 * 1024;
@@ -1294,51 +1404,52 @@
snitch = eps;
}
- public static IRequestScheduler getRequestScheduler()
- {
- return requestScheduler;
- }
-
- public static RequestSchedulerOptions getRequestSchedulerOptions()
- {
- return requestSchedulerOptions;
- }
-
- public static RequestSchedulerId getRequestSchedulerId()
- {
- return requestSchedulerId;
- }
-
public static int getColumnIndexSize()
{
- return conf.column_index_size_in_kb * 1024;
+ return (int) ByteUnit.KIBI_BYTES.toBytes(conf.column_index_size_in_kb);
+ }
+
+ public static int getColumnIndexSizeInKB()
+ {
+ return conf.column_index_size_in_kb;
}
@VisibleForTesting
public static void setColumnIndexSize(int val)
{
+ checkValidForByteConversion(val, "column_index_size_in_kb", ByteUnit.KIBI_BYTES);
conf.column_index_size_in_kb = val;
}
public static int getColumnIndexCacheSize()
{
- return conf.column_index_cache_size_in_kb * 1024;
+ return (int) ByteUnit.KIBI_BYTES.toBytes(conf.column_index_cache_size_in_kb);
}
- @VisibleForTesting
+ public static int getColumnIndexCacheSizeInKB()
+ {
+ return conf.column_index_cache_size_in_kb;
+ }
+
public static void setColumnIndexCacheSize(int val)
{
+ checkValidForByteConversion(val, "column_index_cache_size_in_kb", ByteUnit.KIBI_BYTES);
conf.column_index_cache_size_in_kb = val;
}
public static int getBatchSizeWarnThreshold()
{
- return conf.batch_size_warn_threshold_in_kb * 1024;
+ return (int) ByteUnit.KIBI_BYTES.toBytes(conf.batch_size_warn_threshold_in_kb);
+ }
+
+ public static int getBatchSizeWarnThresholdInKB()
+ {
+ return conf.batch_size_warn_threshold_in_kb;
}
public static long getBatchSizeFailThreshold()
{
- return conf.batch_size_fail_threshold_in_kb * 1024L;
+ return ByteUnit.KIBI_BYTES.toBytes(conf.batch_size_fail_threshold_in_kb);
}
public static int getBatchSizeFailThresholdInKB()
@@ -1353,6 +1464,7 @@
public static void setBatchSizeWarnThresholdInKB(int threshold)
{
+ checkValidForByteConversion(threshold, "batch_size_warn_threshold_in_kb", ByteUnit.KIBI_BYTES);
conf.batch_size_warn_threshold_in_kb = threshold;
}
@@ -1371,6 +1483,11 @@
return System.getProperty(Config.PROPERTY_PREFIX + "allocate_tokens_for_keyspace", conf.allocate_tokens_for_keyspace);
}
+ public static Integer getAllocateTokensForLocalRf()
+ {
+ return conf.allocate_tokens_for_local_replication_factor;
+ }
+
public static Collection<String> tokensFromString(String tokenString)
{
List<String> tokens = new ArrayList<String>();
@@ -1385,14 +1502,14 @@
return conf.num_tokens;
}
- public static InetAddress getReplaceAddress()
+ public static InetAddressAndPort getReplaceAddress()
{
try
{
if (System.getProperty(Config.PROPERTY_PREFIX + "replace_address", null) != null)
- return InetAddress.getByName(System.getProperty(Config.PROPERTY_PREFIX + "replace_address", null));
+ return InetAddressAndPort.getByName(System.getProperty(Config.PROPERTY_PREFIX + "replace_address", null));
else if (System.getProperty(Config.PROPERTY_PREFIX + "replace_address_first_boot", null) != null)
- return InetAddress.getByName(System.getProperty(Config.PROPERTY_PREFIX + "replace_address_first_boot", null));
+ return InetAddressAndPort.getByName(System.getProperty(Config.PROPERTY_PREFIX + "replace_address_first_boot", null));
return null;
}
catch (UnknownHostException e)
@@ -1432,19 +1549,19 @@
return Integer.parseInt(System.getProperty(Config.PROPERTY_PREFIX + "ssl_storage_port", Integer.toString(conf.ssl_storage_port)));
}
- public static int getRpcPort()
+ public static long nativeTransportIdleTimeout()
{
- return Integer.parseInt(System.getProperty(Config.PROPERTY_PREFIX + "rpc_port", Integer.toString(conf.rpc_port)));
+ return conf.native_transport_idle_timeout_in_ms;
}
- public static int getRpcListenBacklog()
+ public static void setNativeTransportIdleTimeout(long nativeTransportTimeout)
{
- return conf.rpc_listen_backlog;
+ conf.native_transport_idle_timeout_in_ms = nativeTransportTimeout;
}
- public static long getRpcTimeout()
+ public static long getRpcTimeout(TimeUnit unit)
{
- return conf.request_timeout_in_ms;
+ return unit.convert(conf.request_timeout_in_ms, MILLISECONDS);
}
public static void setRpcTimeout(long timeOutInMillis)
@@ -1452,9 +1569,9 @@
conf.request_timeout_in_ms = timeOutInMillis;
}
- public static long getReadRpcTimeout()
+ public static long getReadRpcTimeout(TimeUnit unit)
{
- return conf.read_request_timeout_in_ms;
+ return unit.convert(conf.read_request_timeout_in_ms, MILLISECONDS);
}
public static void setReadRpcTimeout(long timeOutInMillis)
@@ -1462,9 +1579,9 @@
conf.read_request_timeout_in_ms = timeOutInMillis;
}
- public static long getRangeRpcTimeout()
+ public static long getRangeRpcTimeout(TimeUnit unit)
{
- return conf.range_request_timeout_in_ms;
+ return unit.convert(conf.range_request_timeout_in_ms, MILLISECONDS);
}
public static void setRangeRpcTimeout(long timeOutInMillis)
@@ -1472,9 +1589,9 @@
conf.range_request_timeout_in_ms = timeOutInMillis;
}
- public static long getWriteRpcTimeout()
+ public static long getWriteRpcTimeout(TimeUnit unit)
{
- return conf.write_request_timeout_in_ms;
+ return unit.convert(conf.write_request_timeout_in_ms, MILLISECONDS);
}
public static void setWriteRpcTimeout(long timeOutInMillis)
@@ -1482,9 +1599,9 @@
conf.write_request_timeout_in_ms = timeOutInMillis;
}
- public static long getCounterWriteRpcTimeout()
+ public static long getCounterWriteRpcTimeout(TimeUnit unit)
{
- return conf.counter_write_request_timeout_in_ms;
+ return unit.convert(conf.counter_write_request_timeout_in_ms, MILLISECONDS);
}
public static void setCounterWriteRpcTimeout(long timeOutInMillis)
@@ -1492,9 +1609,9 @@
conf.counter_write_request_timeout_in_ms = timeOutInMillis;
}
- public static long getCasContentionTimeout()
+ public static long getCasContentionTimeout(TimeUnit unit)
{
- return conf.cas_contention_timeout_in_ms;
+ return unit.convert(conf.cas_contention_timeout_in_ms, MILLISECONDS);
}
public static void setCasContentionTimeout(long timeOutInMillis)
@@ -1502,9 +1619,9 @@
conf.cas_contention_timeout_in_ms = timeOutInMillis;
}
- public static long getTruncateRpcTimeout()
+ public static long getTruncateRpcTimeout(TimeUnit unit)
{
- return conf.truncate_request_timeout_in_ms;
+ return unit.convert(conf.truncate_request_timeout_in_ms, MILLISECONDS);
}
public static void setTruncateRpcTimeout(long timeOutInMillis)
@@ -1517,22 +1634,32 @@
return conf.cross_node_timeout;
}
- public static long getSlowQueryTimeout()
+ public static void setCrossNodeTimeout(boolean crossNodeTimeout)
{
- return conf.slow_query_log_timeout_in_ms;
+ conf.cross_node_timeout = crossNodeTimeout;
+ }
+
+ public static long getSlowQueryTimeout(TimeUnit units)
+ {
+ return units.convert(conf.slow_query_log_timeout_in_ms, MILLISECONDS);
}
/**
* @return the minimum configured {read, write, range, truncate, misc} timeout
*/
- public static long getMinRpcTimeout()
+ public static long getMinRpcTimeout(TimeUnit unit)
{
- return Longs.min(getRpcTimeout(),
- getReadRpcTimeout(),
- getRangeRpcTimeout(),
- getWriteRpcTimeout(),
- getCounterWriteRpcTimeout(),
- getTruncateRpcTimeout());
+ return Longs.min(getRpcTimeout(unit),
+ getReadRpcTimeout(unit),
+ getRangeRpcTimeout(unit),
+ getWriteRpcTimeout(unit),
+ getCounterWriteRpcTimeout(unit),
+ getTruncateRpcTimeout(unit));
+ }
+
+ public static long getPingTimeout(TimeUnit unit)
+ {
+ return unit.convert(getBlockForPeersTimeoutInSeconds(), TimeUnit.SECONDS);
}
public static double getPhiConvictThreshold()
@@ -1550,21 +1677,57 @@
return conf.concurrent_reads;
}
+ public static void setConcurrentReaders(int concurrent_reads)
+ {
+ if (concurrent_reads < 0)
+ {
+ throw new IllegalArgumentException("Concurrent reads must be non-negative");
+ }
+ conf.concurrent_reads = concurrent_reads;
+ }
+
public static int getConcurrentWriters()
{
return conf.concurrent_writes;
}
+ public static void setConcurrentWriters(int concurrent_writers)
+ {
+ if (concurrent_writers < 0)
+ {
+ throw new IllegalArgumentException("Concurrent reads must be non-negative");
+ }
+ conf.concurrent_writes = concurrent_writers;
+ }
+
public static int getConcurrentCounterWriters()
{
return conf.concurrent_counter_writes;
}
+ public static void setConcurrentCounterWriters(int concurrent_counter_writes)
+ {
+ if (concurrent_counter_writes < 0)
+ {
+ throw new IllegalArgumentException("Concurrent reads must be non-negative");
+ }
+ conf.concurrent_counter_writes = concurrent_counter_writes;
+ }
+
public static int getConcurrentViewWriters()
{
return conf.concurrent_materialized_view_writes;
}
+ public static void setConcurrentViewWriters(int concurrent_materialized_view_writes)
+ {
+ if (concurrent_materialized_view_writes < 0)
+ {
+ throw new IllegalArgumentException("Concurrent reads must be non-negative");
+ }
+ conf.concurrent_materialized_view_writes = concurrent_materialized_view_writes;
+ }
+
public static int getFlushWriters()
{
return conf.memtable_flush_writers;
@@ -1590,11 +1753,32 @@
conf.compaction_throughput_mb_per_sec = value;
}
- public static long getCompactionLargePartitionWarningThreshold() { return conf.compaction_large_partition_warning_threshold_mb * 1024L * 1024L; }
+ public static long getCompactionLargePartitionWarningThreshold() { return ByteUnit.MEBI_BYTES.toBytes(conf.compaction_large_partition_warning_threshold_mb); }
+
+ public static int getConcurrentValidations()
+ {
+ return conf.concurrent_validations;
+ }
+
+ public static void setConcurrentValidations(int value)
+ {
+ value = value > 0 ? value : Integer.MAX_VALUE;
+ conf.concurrent_validations = value;
+ }
+
+ public static int getConcurrentViewBuilders()
+ {
+ return conf.concurrent_materialized_view_builders;
+ }
+
+ public static void setConcurrentViewBuilders(int value)
+ {
+ conf.concurrent_materialized_view_builders = value;
+ }
public static long getMinFreeSpacePerDriveInBytes()
{
- return conf.min_free_space_per_drive_in_mb * 1024L * 1024L;
+ return ByteUnit.MEBI_BYTES.toBytes(conf.min_free_space_per_drive_in_mb);
}
public static boolean getDisableSTCSInL0()
@@ -1602,6 +1786,11 @@
return disableSTCSInL0;
}
+ public static void setDisableSTCSInL0(boolean disabled)
+ {
+ disableSTCSInL0 = disabled;
+ }
+
public static int getStreamThroughputOutboundMegabitsPerSec()
{
return conf.stream_throughput_outbound_megabits_per_sec;
@@ -1648,6 +1837,16 @@
conf.commitlog_compression = compressor;
}
+ public static Config.FlushCompression getFlushCompression()
+ {
+ return conf.flush_compression;
+ }
+
+ public static void setFlushCompression(Config.FlushCompression compression)
+ {
+ conf.flush_compression = compression;
+ }
+
/**
* Maximum number of buffers in the compression pool. The default value is 3, it should not be set lower than that
* (one segment in compression, one written to, one in reserve); delays in compression may cause the log to use
@@ -1665,7 +1864,7 @@
public static int getMaxMutationSize()
{
- return conf.max_mutation_size_in_kb * 1024;
+ return (int) ByteUnit.KIBI_BYTES.toBytes(conf.max_mutation_size_in_kb);
}
public static int getTombstoneWarnThreshold()
@@ -1693,7 +1892,7 @@
*/
public static int getCommitLogSegmentSize()
{
- return conf.commitlog_segment_size_in_mb * 1024 * 1024;
+ return (int) ByteUnit.MEBI_BYTES.toBytes(conf.commitlog_segment_size_in_mb);
}
public static void setCommitLogSegmentSize(int sizeMegabytes)
@@ -1706,9 +1905,19 @@
return conf.saved_caches_directory;
}
- public static Set<InetAddress> getSeeds()
+ public static Set<InetAddressAndPort> getSeeds()
{
- return ImmutableSet.<InetAddress>builder().addAll(seedProvider.getSeeds()).build();
+ return ImmutableSet.<InetAddressAndPort>builder().addAll(seedProvider.getSeeds()).build();
+ }
+
+ public static SeedProvider getSeedProvider()
+ {
+ return seedProvider;
+ }
+
+ public static void setSeedProvider(SeedProvider newSeedProvider)
+ {
+ seedProvider = newSeedProvider;
}
public static InetAddress getListenAddress()
@@ -1716,6 +1925,11 @@
return listenAddress;
}
+ public static void setListenAddress(InetAddress newlistenAddress)
+ {
+ listenAddress = newlistenAddress;
+ }
+
public static InetAddress getBroadcastAddress()
{
return broadcastAddress;
@@ -1726,6 +1940,16 @@
return conf.listen_on_broadcast_address;
}
+ public static void setShouldListenOnBroadcastAddress(boolean shouldListenOnBroadcastAddress)
+ {
+ conf.listen_on_broadcast_address = shouldListenOnBroadcastAddress;
+ }
+
+ public static void setListenOnBroadcastAddress(boolean listen_on_broadcast_address)
+ {
+ conf.listen_on_broadcast_address = listen_on_broadcast_address;
+ }
+
public static IInternodeAuthenticator getInternodeAuthenticator()
{
return internodeAuthenticator;
@@ -1733,6 +1957,7 @@
public static void setInternodeAuthenticator(IInternodeAuthenticator internodeAuthenticator)
{
+ Preconditions.checkNotNull(internodeAuthenticator);
DatabaseDescriptor.internodeAuthenticator = internodeAuthenticator;
}
@@ -1741,11 +1966,12 @@
broadcastAddress = broadcastAdd;
}
- public static boolean startRpc()
- {
- return conf.start_rpc;
- }
-
+ /**
+ * This is the address used to bind for the native protocol to communicate with clients. Most usages in the code
+ * refer to it as native address although some places still call it RPC address. It's not thrift RPC anymore
+ * so native is more appropriate. The address alone is not enough to uniquely identify this instance because
+ * multiple instances might use the same interface with different ports.
+ */
public static InetAddress getRpcAddress()
{
return rpcAddress;
@@ -1757,51 +1983,92 @@
}
/**
- * May be null, please use {@link FBUtilities#getBroadcastRpcAddress()} instead.
+ * This is the address used to reach this instance for the native protocol to communicate with clients. Most usages in the code
+ * refer to it as native address although some places still call it RPC address. It's not thrift RPC anymore
+ * so native is more appropriate. The address alone is not enough to uniquely identify this instance because
+ * multiple instances might use the same interface with different ports.
+ *
+ * May be null, please use {@link FBUtilities#getBroadcastNativeAddressAndPort()} instead.
*/
public static InetAddress getBroadcastRpcAddress()
{
return broadcastRpcAddress;
}
- public static String getRpcServerType()
- {
- return conf.rpc_server_type;
- }
-
public static boolean getRpcKeepAlive()
{
return conf.rpc_keepalive;
}
- public static Integer getRpcMinThreads()
+ public static int getInternodeSocketSendBufferSizeInBytes()
{
- return conf.rpc_min_threads;
+ return conf.internode_socket_send_buffer_size_in_bytes;
}
- public static Integer getRpcMaxThreads()
+ public static int getInternodeSocketReceiveBufferSizeInBytes()
{
- return conf.rpc_max_threads;
+ return conf.internode_socket_receive_buffer_size_in_bytes;
}
- public static Integer getRpcSendBufferSize()
+ public static int getInternodeApplicationSendQueueCapacityInBytes()
{
- return conf.rpc_send_buff_size_in_bytes;
+ return conf.internode_application_send_queue_capacity_in_bytes;
}
- public static Integer getRpcRecvBufferSize()
+ public static int getInternodeApplicationSendQueueReserveEndpointCapacityInBytes()
{
- return conf.rpc_recv_buff_size_in_bytes;
+ return conf.internode_application_send_queue_reserve_endpoint_capacity_in_bytes;
}
- public static int getInternodeSendBufferSize()
+ public static int getInternodeApplicationSendQueueReserveGlobalCapacityInBytes()
{
- return conf.internode_send_buff_size_in_bytes;
+ return conf.internode_application_send_queue_reserve_global_capacity_in_bytes;
}
- public static int getInternodeRecvBufferSize()
+ public static int getInternodeApplicationReceiveQueueCapacityInBytes()
{
- return conf.internode_recv_buff_size_in_bytes;
+ return conf.internode_application_receive_queue_capacity_in_bytes;
+ }
+
+ public static int getInternodeApplicationReceiveQueueReserveEndpointCapacityInBytes()
+ {
+ return conf.internode_application_receive_queue_reserve_endpoint_capacity_in_bytes;
+ }
+
+ public static int getInternodeApplicationReceiveQueueReserveGlobalCapacityInBytes()
+ {
+ return conf.internode_application_receive_queue_reserve_global_capacity_in_bytes;
+ }
+
+ public static int getInternodeTcpConnectTimeoutInMS()
+ {
+ return conf.internode_tcp_connect_timeout_in_ms;
+ }
+
+ public static void setInternodeTcpConnectTimeoutInMS(int value)
+ {
+ conf.internode_tcp_connect_timeout_in_ms = value;
+ }
+
+ public static int getInternodeTcpUserTimeoutInMS()
+ {
+ return conf.internode_tcp_user_timeout_in_ms;
+ }
+
+ public static void setInternodeTcpUserTimeoutInMS(int value)
+ {
+ conf.internode_tcp_user_timeout_in_ms = value;
+ }
+
+ public static int getInternodeMaxMessageSizeInBytes()
+ {
+ return conf.internode_max_message_size_in_bytes;
+ }
+
+ @VisibleForTesting
+ public static void setInternodeMaxMessageSizeInBytes(int value)
+ {
+ conf.internode_max_message_size_in_bytes = value;
}
public static boolean startNativeTransport()
@@ -1809,6 +2076,10 @@
return conf.start_native_transport;
}
+ /**
+ * This is the port used with RPC address for the native protocol to communicate with clients. Now that thrift RPC
+ * is no longer in use there is no RPC port.
+ */
public static int getNativeTransportPort()
{
return Integer.parseInt(System.getProperty(Config.PROPERTY_PREFIX + "native_transport_port", Integer.toString(conf.native_transport_port)));
@@ -1836,9 +2107,14 @@
return conf.native_transport_max_threads;
}
+ public static void setNativeTransportMaxThreads(int max_threads)
+ {
+ conf.native_transport_max_threads = max_threads;
+ }
+
public static int getNativeTransportMaxFrameSize()
{
- return conf.native_transport_max_frame_size_in_mb * 1024 * 1024;
+ return (int) ByteUnit.MEBI_BYTES.toBytes(conf.native_transport_max_frame_size_in_mb);
}
public static long getNativeTransportMaxConcurrentConnections()
@@ -1866,19 +2142,29 @@
return conf.native_transport_flush_in_batches_legacy;
}
- public static int getNativeProtocolMaxVersionOverride()
+ public static boolean getNativeTransportAllowOlderProtocols()
{
- return conf.native_transport_max_negotiable_protocol_version;
+ return conf.native_transport_allow_older_protocols;
}
- public static double getCommitLogSyncBatchWindow()
+ public static void setNativeTransportAllowOlderProtocols(boolean isEnabled)
{
- return conf.commitlog_sync_batch_window_in_ms;
+ conf.native_transport_allow_older_protocols = isEnabled;
}
- public static void setCommitLogSyncBatchWindow(double windowMillis)
+ public static int getNativeTransportFrameBlockSize()
{
- conf.commitlog_sync_batch_window_in_ms = windowMillis;
+ return (int) ByteUnit.KIBI_BYTES.toBytes(conf.native_transport_frame_block_size_in_kb);
+ }
+
+ public static double getCommitLogSyncGroupWindow()
+ {
+ return conf.commitlog_sync_group_window_in_ms;
+ }
+
+ public static void setCommitLogSyncGroupWindow(double windowMillis)
+ {
+ conf.commitlog_sync_group_window_in_ms = windowMillis;
}
public static long getNativeTransportMaxConcurrentRequestsInBytesPerIp()
@@ -1906,6 +2192,14 @@
return conf.commitlog_sync_period_in_ms;
}
+ public static long getPeriodicCommitLogSyncBlock()
+ {
+ Integer blockMillis = conf.periodic_commitlog_sync_lag_block_in_ms;
+ return blockMillis == null
+ ? (long)(getCommitLogSyncPeriod() * 1.5)
+ : blockMillis;
+ }
+
public static void setCommitLogSyncPeriod(int periodMillis)
{
conf.commitlog_sync_period_in_ms = periodMillis;
@@ -2066,29 +2360,45 @@
conf.dynamic_snitch_badness_threshold = dynamicBadnessThreshold;
}
- public static EncryptionOptions.ServerEncryptionOptions getServerEncryptionOptions()
+ public static EncryptionOptions.ServerEncryptionOptions getInternodeMessagingEncyptionOptions()
{
return conf.server_encryption_options;
}
- public static EncryptionOptions.ClientEncryptionOptions getClientEncryptionOptions()
+ public static void setInternodeMessagingEncyptionOptions(EncryptionOptions.ServerEncryptionOptions encryptionOptions)
+ {
+ conf.server_encryption_options = encryptionOptions;
+ }
+
+ public static EncryptionOptions getNativeProtocolEncryptionOptions()
{
return conf.client_encryption_options;
}
+ @VisibleForTesting
+ public static void updateNativeProtocolEncryptionOptions(Function<EncryptionOptions, EncryptionOptions> update)
+ {
+ conf.client_encryption_options = update.apply(conf.client_encryption_options);
+ }
+
public static int getHintedHandoffThrottleInKB()
{
return conf.hinted_handoff_throttle_in_kb;
}
+ public static void setHintedHandoffThrottleInKB(int throttleInKB)
+ {
+ conf.hinted_handoff_throttle_in_kb = throttleInKB;
+ }
+
public static int getBatchlogReplayThrottleInKB()
{
return conf.batchlog_replay_throttle_in_kb;
}
- public static void setHintedHandoffThrottleInKB(int throttleInKB)
+ public static void setBatchlogReplayThrottleInKB(int throttleInKB)
{
- conf.hinted_handoff_throttle_in_kb = throttleInKB;
+ conf.batchlog_replay_throttle_in_kb = throttleInKB;
}
public static int getMaxHintsDeliveryThreads()
@@ -2103,7 +2413,7 @@
public static long getMaxHintsFileSize()
{
- return conf.max_hints_file_size_in_mb * 1024L * 1024L;
+ return ByteUnit.MEBI_BYTES.toBytes(conf.max_hints_file_size_in_mb);
}
public static ParameterizedClass getHintsCompression()
@@ -2150,11 +2460,6 @@
return conf.file_cache_round_up;
}
- public static boolean getBufferPoolUseHeapIfExhausted()
- {
- return conf.buffer_pool_use_heap_if_exhausted;
- }
-
public static DiskOptimizationStrategy getDiskOptimizationStrategy()
{
return diskOptimizationStrategy;
@@ -2170,11 +2475,21 @@
return conf.commitlog_total_space_in_mb;
}
- public static int getSSTablePreempiveOpenIntervalInMB()
+ public static boolean shouldMigrateKeycacheOnCompaction()
+ {
+ return conf.key_cache_migrate_during_compaction;
+ }
+
+ public static void setMigrateKeycacheOnCompaction(boolean migrateCacheEntry)
+ {
+ conf.key_cache_migrate_during_compaction = migrateCacheEntry;
+ }
+
+ public static int getSSTablePreemptiveOpenIntervalInMB()
{
return FBUtilities.isWindows ? -1 : conf.sstable_preemptive_open_interval_in_mb;
}
- public static void setSSTablePreempiveOpenIntervalInMB(int mb)
+ public static void setSSTablePreemptiveOpenIntervalInMB(int mb)
{
conf.sstable_preemptive_open_interval_in_mb = mb;
}
@@ -2280,32 +2595,27 @@
conf.counter_cache_keys_to_save = counterCacheKeysToSave;
}
- public static void setStreamingSocketTimeout(int value)
- {
- conf.streaming_socket_timeout_in_ms = value;
- }
-
- /**
- * @deprecated use {@link this#getStreamingKeepAlivePeriod()} instead
- * @return streaming_socket_timeout_in_ms property
- */
- @Deprecated
- public static int getStreamingSocketTimeout()
- {
- return conf.streaming_socket_timeout_in_ms;
- }
-
public static int getStreamingKeepAlivePeriod()
{
return conf.streaming_keep_alive_period_in_secs;
}
+ public static int getStreamingConnectionsPerHost()
+ {
+ return conf.streaming_connections_per_host;
+ }
+
+ public static boolean streamEntireSSTables()
+ {
+ return conf.stream_entire_sstables;
+ }
+
public static String getLocalDataCenter()
{
return localDC;
}
- public static Comparator<InetAddress> getLocalComparator()
+ public static Comparator<Replica> getLocalComparator()
{
return localComparator;
}
@@ -2315,6 +2625,11 @@
return conf.internode_compression;
}
+ public static void setInternodeCompression(Config.InternodeCompression compression)
+ {
+ conf.internode_compression = compression;
+ }
+
public static boolean getInterDCTcpNoDelay()
{
return conf.inter_dc_tcp_nodelay;
@@ -2335,11 +2650,6 @@
return conf.memtable_allocation_type;
}
- public static Float getMemtableCleanupThreshold()
- {
- return conf.memtable_cleanup_threshold;
- }
-
public static int getRepairSessionMaxTreeDepth()
{
return conf.repair_session_max_tree_depth;
@@ -2356,6 +2666,28 @@
conf.repair_session_max_tree_depth = depth;
}
+ public static int getRepairSessionSpaceInMegabytes()
+ {
+ return conf.repair_session_space_in_mb;
+ }
+
+ public static void setRepairSessionSpaceInMegabytes(int sizeInMegabytes)
+ {
+ if (sizeInMegabytes < 1)
+ throw new ConfigurationException("Cannot set repair_session_space_in_mb to " + sizeInMegabytes +
+ " < 1 megabyte");
+ else if (sizeInMegabytes > (int) (Runtime.getRuntime().maxMemory() / (4 * 1048576)))
+ logger.warn("A repair_session_space_in_mb of " + conf.repair_session_space_in_mb +
+ " megabytes is likely to cause heap pressure.");
+
+ conf.repair_session_space_in_mb = sizeInMegabytes;
+ }
+
+ public static Float getMemtableCleanupThreshold()
+ {
+ return conf.memtable_cleanup_threshold;
+ }
+
public static int getIndexSummaryResizeIntervalInMinutes()
{
return conf.index_summary_resize_interval_in_minutes;
@@ -2387,36 +2719,6 @@
return conf.tracetype_query_ttl;
}
- public static String getOtcCoalescingStrategy()
- {
- return conf.otc_coalescing_strategy;
- }
-
- public static int getOtcCoalescingWindow()
- {
- return conf.otc_coalescing_window_us;
- }
-
- public static int getOtcCoalescingEnoughCoalescedMessages()
- {
- return conf.otc_coalescing_enough_coalesced_messages;
- }
-
- public static void setOtcCoalescingEnoughCoalescedMessages(int otc_coalescing_enough_coalesced_messages)
- {
- conf.otc_coalescing_enough_coalesced_messages = otc_coalescing_enough_coalesced_messages;
- }
-
- public static int getOtcBacklogExpirationInterval()
- {
- return conf.otc_backlog_expiration_interval_ms;
- }
-
- public static void setOtcBacklogExpirationInterval(int intervalInMillis)
- {
- conf.otc_backlog_expiration_interval_ms = intervalInMillis;
- }
-
public static int getWindowsTimerInterval()
{
return conf.windows_timer_interval;
@@ -2427,11 +2729,6 @@
return preparedStatementsCacheSizeInMB;
}
- public static long getThriftPreparedStatementsCacheSizeMB()
- {
- return thriftPreparedStatementsCacheSizeInMB;
- }
-
public static boolean enableUserDefinedFunctions()
{
return conf.enable_user_defined_functions;
@@ -2482,6 +2779,16 @@
conf.enable_sasi_indexes = enableSASIIndexes;
}
+ public static boolean isTransientReplicationEnabled()
+ {
+ return conf.enable_transient_replication;
+ }
+
+ public static void setTransientReplicationEnabledUnsafe(boolean enabled)
+ {
+ conf.enable_transient_replication = enabled;
+ }
+
public static long getUserDefinedFunctionFailTimeout()
{
return conf.user_defined_function_fail_timeout;
@@ -2522,6 +2829,7 @@
return conf.cdc_enabled;
}
+ @VisibleForTesting
public static void setCDCEnabled(boolean cdc_enabled)
{
conf.cdc_enabled = cdc_enabled;
@@ -2574,6 +2882,16 @@
return conf.back_pressure_enabled;
}
+ public static boolean diagnosticEventsEnabled()
+ {
+ return conf.diagnostic_events_enabled;
+ }
+
+ public static void setDiagnosticEventsEnabled(boolean enabled)
+ {
+ conf.diagnostic_events_enabled = enabled;
+ }
+
@VisibleForTesting
public static void setBackPressureStrategy(BackPressureStrategy strategy)
{
@@ -2585,9 +2903,122 @@
return backPressureStrategy;
}
- public static boolean strictRuntimeChecks()
+ public static ConsistencyLevel getIdealConsistencyLevel()
{
- return strictRuntimeChecks;
+ return conf.ideal_consistency_level;
+ }
+
+ public static void setIdealConsistencyLevel(ConsistencyLevel cl)
+ {
+ conf.ideal_consistency_level = cl;
+ }
+
+ public static int getRepairCommandPoolSize()
+ {
+ return conf.repair_command_pool_size;
+ }
+
+ public static Config.RepairCommandPoolFullStrategy getRepairCommandPoolFullStrategy()
+ {
+ return conf.repair_command_pool_full_strategy;
+ }
+
+ public static FullQueryLoggerOptions getFullQueryLogOptions()
+ {
+ return conf.full_query_logging_options;
+ }
+
+ public static boolean getBlockForPeersInRemoteDatacenters()
+ {
+ return conf.block_for_peers_in_remote_dcs;
+ }
+
+ public static int getBlockForPeersTimeoutInSeconds()
+ {
+ return conf.block_for_peers_timeout_in_secs;
+ }
+
+ public static boolean automaticSSTableUpgrade()
+ {
+ return conf.automatic_sstable_upgrade;
+ }
+
+ public static void setAutomaticSSTableUpgradeEnabled(boolean enabled)
+ {
+ if (conf.automatic_sstable_upgrade != enabled)
+ logger.debug("Changing automatic_sstable_upgrade to {}", enabled);
+ conf.automatic_sstable_upgrade = enabled;
+ }
+
+ public static int maxConcurrentAutoUpgradeTasks()
+ {
+ return conf.max_concurrent_automatic_sstable_upgrades;
+ }
+
+ public static void setMaxConcurrentAutoUpgradeTasks(int value)
+ {
+ if (conf.max_concurrent_automatic_sstable_upgrades != value)
+ logger.debug("Changing max_concurrent_automatic_sstable_upgrades to {}", value);
+ validateMaxConcurrentAutoUpgradeTasksConf(value);
+ conf.max_concurrent_automatic_sstable_upgrades = value;
+ }
+
+ private static void validateMaxConcurrentAutoUpgradeTasksConf(int value)
+ {
+ if (value < 0)
+ throw new ConfigurationException("max_concurrent_automatic_sstable_upgrades can't be negative");
+ if (value > getConcurrentCompactors())
+ logger.warn("max_concurrent_automatic_sstable_upgrades ({}) is larger than concurrent_compactors ({})", value, getConcurrentCompactors());
+ }
+
+ public static AuditLogOptions getAuditLoggingOptions()
+ {
+ return conf.audit_logging_options;
+ }
+
+ public static void setAuditLoggingOptions(AuditLogOptions auditLoggingOptions)
+ {
+ conf.audit_logging_options = auditLoggingOptions;
+ }
+
+ public static Config.CorruptedTombstoneStrategy getCorruptedTombstoneStrategy()
+ {
+ return conf.corrupted_tombstone_strategy;
+ }
+
+ public static void setCorruptedTombstoneStrategy(Config.CorruptedTombstoneStrategy strategy)
+ {
+ conf.corrupted_tombstone_strategy = strategy;
+ }
+
+ public static boolean getRepairedDataTrackingForRangeReadsEnabled()
+ {
+ return conf.repaired_data_tracking_for_range_reads_enabled;
+ }
+
+ public static void setRepairedDataTrackingForRangeReadsEnabled(boolean enabled)
+ {
+ conf.repaired_data_tracking_for_range_reads_enabled = enabled;
+ }
+
+ public static boolean getRepairedDataTrackingForPartitionReadsEnabled()
+ {
+ return conf.repaired_data_tracking_for_partition_reads_enabled;
+ }
+
+ public static void setRepairedDataTrackingForPartitionReadsEnabled(boolean enabled)
+ {
+ conf.repaired_data_tracking_for_partition_reads_enabled = enabled;
+ }
+
+ public static boolean snapshotOnRepairedDataMismatch()
+ {
+ return conf.snapshot_on_repaired_data_mismatch;
+ }
+
+ public static void setSnapshotOnRepairedDataMismatch(boolean enabled)
+ {
+ conf.snapshot_on_repaired_data_mismatch = enabled;
}
public static boolean snapshotOnDuplicateRowDetection()
@@ -2600,6 +3031,90 @@
conf.snapshot_on_duplicate_row_detection = enabled;
}
+ public static boolean reportUnconfirmedRepairedDataMismatches()
+ {
+ return conf.report_unconfirmed_repaired_data_mismatches;
+ }
+
+ public static void reportUnconfirmedRepairedDataMismatches(boolean enabled)
+ {
+ conf.report_unconfirmed_repaired_data_mismatches = enabled;
+ }
+
+ public static boolean strictRuntimeChecks()
+ {
+ return strictRuntimeChecks;
+ }
+
+ public static boolean useOffheapMerkleTrees()
+ {
+ return conf.use_offheap_merkle_trees;
+ }
+
+ public static void useOffheapMerkleTrees(boolean value)
+ {
+ logger.info("Setting use_offheap_merkle_trees to {}", value);
+ conf.use_offheap_merkle_trees = value;
+ }
+
+ public static Function<CommitLog, AbstractCommitLogSegmentManager> getCommitLogSegmentMgrProvider()
+ {
+ return commitLogSegmentMgrProvider;
+ }
+
+ public static void setCommitLogSegmentMgrProvider(Function<CommitLog, AbstractCommitLogSegmentManager> provider)
+ {
+ commitLogSegmentMgrProvider = provider;
+ }
+
+ /**
+ * Class that primarily tracks overflow thresholds during conversions
+ */
+ private enum ByteUnit {
+ KIBI_BYTES(2048 * 1024, 1024),
+ MEBI_BYTES(2048, 1024 * 1024);
+
+ private final int overflowThreshold;
+ private final int multiplier;
+
+ ByteUnit(int t, int m)
+ {
+ this.overflowThreshold = t;
+ this.multiplier = m;
+ }
+
+ public int overflowThreshold()
+ {
+ return overflowThreshold;
+ }
+
+ public boolean willOverflowInBytes(int val)
+ {
+ return val >= overflowThreshold;
+ }
+
+ public long toBytes(int val)
+ {
+ return val * multiplier;
+ }
+ }
+
+ /**
+ * Ensures passed in configuration value is positive and will not overflow when converted to Bytes
+ */
+ private static void checkValidForByteConversion(int val, final String name, final ByteUnit unit)
+ {
+ if (val < 0 || unit.willOverflowInBytes(val))
+ throw new ConfigurationException(String.format("%s must be positive value < %d, but was %d",
+ name, unit.overflowThreshold(), val), false);
+ }
+
+ public static int getValidationPreviewPurgeHeadStartInSec()
+ {
+ int seconds = conf.validation_preview_purge_head_start_in_sec;
+ return Math.max(seconds, 0);
+ }
+
public static boolean checkForDuplicateRowsDuringReads()
{
return conf.check_for_duplicate_rows_during_reads;
@@ -2620,4 +3135,28 @@
conf.check_for_duplicate_rows_during_compaction = enabled;
}
+ public static int getInitialRangeTombstoneListAllocationSize()
+ {
+ return conf.initial_range_tombstone_list_allocation_size;
+ }
+
+ public static void setInitialRangeTombstoneListAllocationSize(int size)
+ {
+ conf.initial_range_tombstone_list_allocation_size = size;
+ }
+
+ public static double getRangeTombstoneListGrowthFactor()
+ {
+ return conf.range_tombstone_list_growth_factor;
+ }
+
+ public static void setRangeTombstoneListGrowthFactor(double resizeFactor)
+ {
+ conf.range_tombstone_list_growth_factor = resizeFactor;
+ }
+
+ public static boolean getAutocompactionOnStartupEnabled()
+ {
+ return conf.autocompaction_on_startup_enabled;
+ }
}
diff --git a/src/java/org/apache/cassandra/config/EncryptionOptions.java b/src/java/org/apache/cassandra/config/EncryptionOptions.java
index d662871..8ccf6d2 100644
--- a/src/java/org/apache/cassandra/config/EncryptionOptions.java
+++ b/src/java/org/apache/cassandra/config/EncryptionOptions.java
@@ -17,33 +17,412 @@
*/
package org.apache.cassandra.config;
-import javax.net.ssl.SSLSocketFactory;
+import java.util.List;
+import java.util.Objects;
-public abstract class EncryptionOptions
+import com.google.common.collect.ImmutableList;
+
+import org.apache.cassandra.locator.IEndpointSnitch;
+import org.apache.cassandra.locator.InetAddressAndPort;
+
+public class EncryptionOptions
{
- public String keystore = "conf/.keystore";
- public String keystore_password = "cassandra";
- public String truststore = "conf/.truststore";
- public String truststore_password = "cassandra";
- public String[] cipher_suites = ((SSLSocketFactory)SSLSocketFactory.getDefault()).getDefaultCipherSuites();
- public String protocol = "TLS";
- public String algorithm = "SunX509";
- public String store_type = "JKS";
- public boolean require_client_auth = false;
- public boolean require_endpoint_verification = false;
+ public final String keystore;
+ public final String keystore_password;
+ public final String truststore;
+ public final String truststore_password;
+ public final List<String> cipher_suites;
+ public final String protocol;
+ public final String algorithm;
+ public final String store_type;
+ public final boolean require_client_auth;
+ public final boolean require_endpoint_verification;
+ // ServerEncryptionOptions does not use the enabled flag at all instead using the existing
+ // internode_encryption option. So we force this private and expose through isEnabled
+ // so users of ServerEncryptionOptions can't accidentally use this when they should use isEnabled
+ // Long term we need to refactor ClientEncryptionOptions and ServerEncyrptionOptions to be separate
+ // classes so we can choose appropriate configuration for each.
+ // See CASSANDRA-15262 and CASSANDRA-15146
+ private boolean enabled;
+ public final Boolean optional;
- public static class ClientEncryptionOptions extends EncryptionOptions
+ public EncryptionOptions()
{
- public boolean enabled = false;
- public boolean optional = false;
+ keystore = "conf/.keystore";
+ keystore_password = "cassandra";
+ truststore = "conf/.truststore";
+ truststore_password = "cassandra";
+ cipher_suites = ImmutableList.of();
+ protocol = "TLS";
+ algorithm = null;
+ store_type = "JKS";
+ require_client_auth = false;
+ require_endpoint_verification = false;
+ enabled = false;
+ optional = true;
+ }
+
+ public EncryptionOptions(String keystore, String keystore_password, String truststore, String truststore_password, List<String> cipher_suites, String protocol, String algorithm, String store_type, boolean require_client_auth, boolean require_endpoint_verification, boolean enabled, Boolean optional)
+ {
+ this.keystore = keystore;
+ this.keystore_password = keystore_password;
+ this.truststore = truststore;
+ this.truststore_password = truststore_password;
+ this.cipher_suites = cipher_suites;
+ this.protocol = protocol;
+ this.algorithm = algorithm;
+ this.store_type = store_type;
+ this.require_client_auth = require_client_auth;
+ this.require_endpoint_verification = require_endpoint_verification;
+ this.enabled = enabled;
+ if (optional != null) {
+ this.optional = optional;
+ } else {
+ // If someone is asking for an _insecure_ connection and not explicitly telling us to refuse
+ // encrypted connections we assume they would like to be able to transition to encrypted connections
+ // in the future.
+ this.optional = !enabled;
+ }
+ }
+
+ public EncryptionOptions(EncryptionOptions options)
+ {
+ keystore = options.keystore;
+ keystore_password = options.keystore_password;
+ truststore = options.truststore;
+ truststore_password = options.truststore_password;
+ cipher_suites = options.cipher_suites;
+ protocol = options.protocol;
+ algorithm = options.algorithm;
+ store_type = options.store_type;
+ require_client_auth = options.require_client_auth;
+ require_endpoint_verification = options.require_endpoint_verification;
+ enabled = options.enabled;
+ if (options.optional != null) {
+ optional = options.optional;
+ } else {
+ // If someone is asking for an _insecure_ connection and not explicitly telling us to refuse
+ // encrypted connections we assume they would like to be able to transition to encrypted connections
+ // in the future.
+ optional = !enabled;
+ }
+ }
+
+ /**
+ * Indicates if the channel should be encrypted. Client and Server uses different logic to determine this
+ *
+ * @return if the channel should be encrypted
+ */
+ public boolean isEnabled() {
+ return this.enabled;
+ }
+
+ /**
+ * Sets if encryption should be enabled for this channel. Note that this should only be called by
+ * the configuration parser or tests. It is public only for that purpose, mutating enabled state
+ * is probably a bad idea.
+ * @param enabled
+ */
+ public void setEnabled(boolean enabled) {
+ this.enabled = enabled;
+ }
+
+ public EncryptionOptions withKeyStore(String keystore)
+ {
+ return new EncryptionOptions(keystore, keystore_password, truststore, truststore_password, cipher_suites,
+ protocol, algorithm, store_type, require_client_auth, require_endpoint_verification,
+ enabled, optional);
+ }
+
+ public EncryptionOptions withKeyStorePassword(String keystore_password)
+ {
+ return new EncryptionOptions(keystore, keystore_password, truststore, truststore_password, cipher_suites,
+ protocol, algorithm, store_type, require_client_auth, require_endpoint_verification,
+ enabled, optional);
+ }
+
+ public EncryptionOptions withTrustStore(String truststore)
+ {
+ return new EncryptionOptions(keystore, keystore_password, truststore, truststore_password, cipher_suites,
+ protocol, algorithm, store_type, require_client_auth, require_endpoint_verification,
+ enabled, optional);
+ }
+
+ public EncryptionOptions withTrustStorePassword(String truststore_password)
+ {
+ return new EncryptionOptions(keystore, keystore_password, truststore, truststore_password, cipher_suites,
+ protocol, algorithm, store_type, require_client_auth, require_endpoint_verification,
+ enabled, optional);
+ }
+
+ public EncryptionOptions withCipherSuites(List<String> cipher_suites)
+ {
+ return new EncryptionOptions(keystore, keystore_password, truststore, truststore_password, cipher_suites,
+ protocol, algorithm, store_type, require_client_auth, require_endpoint_verification,
+ enabled, optional);
+ }
+
+ public EncryptionOptions withCipherSuites(String ... cipher_suites)
+ {
+ return new EncryptionOptions(keystore, keystore_password, truststore, truststore_password, ImmutableList.copyOf(cipher_suites),
+ protocol, algorithm, store_type, require_client_auth, require_endpoint_verification,
+ enabled, optional);
+ }
+
+ public EncryptionOptions withProtocol(String protocol)
+ {
+ return new EncryptionOptions(keystore, keystore_password, truststore, truststore_password, cipher_suites,
+ protocol, algorithm, store_type, require_client_auth, require_endpoint_verification,
+ enabled, optional);
+ }
+
+ public EncryptionOptions withAlgorithm(String algorithm)
+ {
+ return new EncryptionOptions(keystore, keystore_password, truststore, truststore_password, cipher_suites,
+ protocol, algorithm, store_type, require_client_auth, require_endpoint_verification,
+ enabled, optional);
+ }
+
+ public EncryptionOptions withStoreType(String store_type)
+ {
+ return new EncryptionOptions(keystore, keystore_password, truststore, truststore_password, cipher_suites,
+ protocol, algorithm, store_type, require_client_auth, require_endpoint_verification,
+ enabled, optional);
+ }
+
+ public EncryptionOptions withRequireClientAuth(boolean require_client_auth)
+ {
+ return new EncryptionOptions(keystore, keystore_password, truststore, truststore_password, cipher_suites,
+ protocol, algorithm, store_type, require_client_auth, require_endpoint_verification,
+ enabled, optional);
+ }
+
+ public EncryptionOptions withRequireEndpointVerification(boolean require_endpoint_verification)
+ {
+ return new EncryptionOptions(keystore, keystore_password, truststore, truststore_password, cipher_suites,
+ protocol, algorithm, store_type, require_client_auth, require_endpoint_verification,
+ enabled, optional);
+ }
+
+ public EncryptionOptions withEnabled(boolean enabled)
+ {
+ return new EncryptionOptions(keystore, keystore_password, truststore, truststore_password, cipher_suites,
+ protocol, algorithm, store_type, require_client_auth, require_endpoint_verification,
+ enabled, optional);
+ }
+
+ public EncryptionOptions withOptional(boolean optional)
+ {
+ return new EncryptionOptions(keystore, keystore_password, truststore, truststore_password, cipher_suites,
+ protocol, algorithm, store_type, require_client_auth, require_endpoint_verification,
+ enabled, optional);
+ }
+
+ /**
+ * The method is being mainly used to cache SslContexts therefore, we only consider
+ * fields that would make a difference when the TrustStore or KeyStore files are updated
+ */
+ @Override
+ public boolean equals(Object o)
+ {
+ if (o == this)
+ return true;
+ if (o == null || getClass() != o.getClass())
+ return false;
+
+ EncryptionOptions opt = (EncryptionOptions)o;
+ return enabled == opt.enabled &&
+ optional == opt.optional &&
+ require_client_auth == opt.require_client_auth &&
+ require_endpoint_verification == opt.require_endpoint_verification &&
+ Objects.equals(keystore, opt.keystore) &&
+ Objects.equals(keystore_password, opt.keystore_password) &&
+ Objects.equals(truststore, opt.truststore) &&
+ Objects.equals(truststore_password, opt.truststore_password) &&
+ Objects.equals(protocol, opt.protocol) &&
+ Objects.equals(algorithm, opt.algorithm) &&
+ Objects.equals(store_type, opt.store_type) &&
+ Objects.equals(cipher_suites, opt.cipher_suites);
+ }
+
+ /**
+ * The method is being mainly used to cache SslContexts therefore, we only consider
+ * fields that would make a difference when the TrustStore or KeyStore files are updated
+ */
+ @Override
+ public int hashCode()
+ {
+ int result = 0;
+ result += 31 * (keystore == null ? 0 : keystore.hashCode());
+ result += 31 * (keystore_password == null ? 0 : keystore_password.hashCode());
+ result += 31 * (truststore == null ? 0 : truststore.hashCode());
+ result += 31 * (truststore_password == null ? 0 : truststore_password.hashCode());
+ result += 31 * (protocol == null ? 0 : protocol.hashCode());
+ result += 31 * (algorithm == null ? 0 : algorithm.hashCode());
+ result += 31 * (store_type == null ? 0 : store_type.hashCode());
+ result += 31 * Boolean.hashCode(enabled);
+ result += 31 * Boolean.hashCode(optional);
+ result += 31 * (cipher_suites == null ? 0 : cipher_suites.hashCode());
+ result += 31 * Boolean.hashCode(require_client_auth);
+ result += 31 * Boolean.hashCode(require_endpoint_verification);
+ return result;
}
public static class ServerEncryptionOptions extends EncryptionOptions
{
- public static enum InternodeEncryption
+ public enum InternodeEncryption
{
all, none, dc, rack
}
- public InternodeEncryption internode_encryption = InternodeEncryption.none;
+
+ public final InternodeEncryption internode_encryption;
+ public final boolean enable_legacy_ssl_storage_port;
+
+ public ServerEncryptionOptions()
+ {
+ this.internode_encryption = InternodeEncryption.none;
+ this.enable_legacy_ssl_storage_port = false;
+ }
+
+ public ServerEncryptionOptions(String keystore, String keystore_password, String truststore, String truststore_password, List<String> cipher_suites, String protocol, String algorithm, String store_type, boolean require_client_auth, boolean require_endpoint_verification, Boolean optional, InternodeEncryption internode_encryption, boolean enable_legacy_ssl_storage_port)
+ {
+ super(keystore, keystore_password, truststore, truststore_password, cipher_suites, protocol, algorithm, store_type, require_client_auth, require_endpoint_verification, internode_encryption != InternodeEncryption.none, optional);
+ this.internode_encryption = internode_encryption;
+ this.enable_legacy_ssl_storage_port = enable_legacy_ssl_storage_port;
+ }
+
+ public ServerEncryptionOptions(ServerEncryptionOptions options)
+ {
+ super(options);
+ this.internode_encryption = options.internode_encryption;
+ this.enable_legacy_ssl_storage_port = options.enable_legacy_ssl_storage_port;
+ }
+
+ public boolean isEnabled() {
+ return this.internode_encryption != InternodeEncryption.none;
+ }
+
+ public boolean shouldEncrypt(InetAddressAndPort endpoint)
+ {
+ IEndpointSnitch snitch = DatabaseDescriptor.getEndpointSnitch();
+ switch (internode_encryption)
+ {
+ case none:
+ return false; // if nothing needs to be encrypted then return immediately.
+ case all:
+ break;
+ case dc:
+ if (snitch.getDatacenter(endpoint).equals(snitch.getLocalDatacenter()))
+ return false;
+ break;
+ case rack:
+ // for rack then check if the DC's are the same.
+ if (snitch.getRack(endpoint).equals(snitch.getLocalRack())
+ && snitch.getDatacenter(endpoint).equals(snitch.getLocalDatacenter()))
+ return false;
+ break;
+ }
+ return true;
+ }
+
+
+ public ServerEncryptionOptions withKeyStore(String keystore)
+ {
+ return new ServerEncryptionOptions(keystore, keystore_password, truststore, truststore_password, cipher_suites,
+ protocol, algorithm, store_type, require_client_auth, require_endpoint_verification,
+ optional, internode_encryption, enable_legacy_ssl_storage_port);
+ }
+
+ public ServerEncryptionOptions withKeyStorePassword(String keystore_password)
+ {
+ return new ServerEncryptionOptions(keystore, keystore_password, truststore, truststore_password, cipher_suites,
+ protocol, algorithm, store_type, require_client_auth, require_endpoint_verification,
+ optional, internode_encryption, enable_legacy_ssl_storage_port);
+ }
+
+ public ServerEncryptionOptions withTrustStore(String truststore)
+ {
+ return new ServerEncryptionOptions(keystore, keystore_password, truststore, truststore_password, cipher_suites,
+ protocol, algorithm, store_type, require_client_auth, require_endpoint_verification,
+ optional, internode_encryption, enable_legacy_ssl_storage_port);
+ }
+
+ public ServerEncryptionOptions withTrustStorePassword(String truststore_password)
+ {
+ return new ServerEncryptionOptions(keystore, keystore_password, truststore, truststore_password, cipher_suites,
+ protocol, algorithm, store_type, require_client_auth, require_endpoint_verification,
+ optional, internode_encryption, enable_legacy_ssl_storage_port);
+ }
+
+ public ServerEncryptionOptions withCipherSuites(List<String> cipher_suites)
+ {
+ return new ServerEncryptionOptions(keystore, keystore_password, truststore, truststore_password, cipher_suites,
+ protocol, algorithm, store_type, require_client_auth, require_endpoint_verification,
+ optional, internode_encryption, enable_legacy_ssl_storage_port);
+ }
+
+ public ServerEncryptionOptions withCipherSuites(String ... cipher_suites)
+ {
+ return new ServerEncryptionOptions(keystore, keystore_password, truststore, truststore_password, ImmutableList.copyOf(cipher_suites),
+ protocol, algorithm, store_type, require_client_auth, require_endpoint_verification,
+ optional, internode_encryption, enable_legacy_ssl_storage_port);
+ }
+
+ public ServerEncryptionOptions withProtocol(String protocol)
+ {
+ return new ServerEncryptionOptions(keystore, keystore_password, truststore, truststore_password, cipher_suites,
+ protocol, algorithm, store_type, require_client_auth, require_endpoint_verification,
+ optional, internode_encryption, enable_legacy_ssl_storage_port);
+ }
+
+ public ServerEncryptionOptions withAlgorithm(String algorithm)
+ {
+ return new ServerEncryptionOptions(keystore, keystore_password, truststore, truststore_password, cipher_suites,
+ protocol, algorithm, store_type, require_client_auth, require_endpoint_verification,
+ optional, internode_encryption, enable_legacy_ssl_storage_port);
+ }
+
+ public ServerEncryptionOptions withStoreType(String store_type)
+ {
+ return new ServerEncryptionOptions(keystore, keystore_password, truststore, truststore_password, cipher_suites,
+ protocol, algorithm, store_type, require_client_auth, require_endpoint_verification,
+ optional, internode_encryption, enable_legacy_ssl_storage_port);
+ }
+
+ public ServerEncryptionOptions withRequireClientAuth(boolean require_client_auth)
+ {
+ return new ServerEncryptionOptions(keystore, keystore_password, truststore, truststore_password, cipher_suites,
+ protocol, algorithm, store_type, require_client_auth, require_endpoint_verification,
+ optional, internode_encryption, enable_legacy_ssl_storage_port);
+ }
+
+ public ServerEncryptionOptions withRequireEndpointVerification(boolean require_endpoint_verification)
+ {
+ return new ServerEncryptionOptions(keystore, keystore_password, truststore, truststore_password, cipher_suites,
+ protocol, algorithm, store_type, require_client_auth, require_endpoint_verification,
+ optional, internode_encryption, enable_legacy_ssl_storage_port);
+ }
+
+ public ServerEncryptionOptions withOptional(boolean optional)
+ {
+ return new ServerEncryptionOptions(keystore, keystore_password, truststore, truststore_password, cipher_suites,
+ protocol, algorithm, store_type, require_client_auth, require_endpoint_verification,
+ optional, internode_encryption, enable_legacy_ssl_storage_port);
+ }
+
+ public ServerEncryptionOptions withInternodeEncryption(InternodeEncryption internode_encryption)
+ {
+ return new ServerEncryptionOptions(keystore, keystore_password, truststore, truststore_password, cipher_suites,
+ protocol, algorithm, store_type, require_client_auth, require_endpoint_verification,
+ optional, internode_encryption, enable_legacy_ssl_storage_port);
+ }
+
+ public ServerEncryptionOptions withLegacySslStoragePort(boolean enable_legacy_ssl_storage_port)
+ {
+ return new ServerEncryptionOptions(keystore, keystore_password, truststore, truststore_password, cipher_suites,
+ protocol, algorithm, store_type, require_client_auth, require_endpoint_verification,
+ optional, internode_encryption, enable_legacy_ssl_storage_port);
+ }
+
}
}
diff --git a/src/java/org/apache/cassandra/config/ReadRepairDecision.java b/src/java/org/apache/cassandra/config/ReadRepairDecision.java
deleted file mode 100644
index 1b4b648..0000000
--- a/src/java/org/apache/cassandra/config/ReadRepairDecision.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.config;
-
-public enum ReadRepairDecision
-{
- NONE, GLOBAL, DC_LOCAL;
-}
diff --git a/src/java/org/apache/cassandra/config/RequestSchedulerOptions.java b/src/java/org/apache/cassandra/config/RequestSchedulerOptions.java
deleted file mode 100644
index dacf405..0000000
--- a/src/java/org/apache/cassandra/config/RequestSchedulerOptions.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.config;
-
-import java.util.Map;
-
-/**
- *
- */
-public class RequestSchedulerOptions
-{
- public static final Integer DEFAULT_THROTTLE_LIMIT = 80;
- public static final Integer DEFAULT_WEIGHT = 1;
-
- public Integer throttle_limit = DEFAULT_THROTTLE_LIMIT;
- public Integer default_weight = DEFAULT_WEIGHT;
- public Map<String, Integer> weights;
-}
diff --git a/src/java/org/apache/cassandra/config/Schema.java b/src/java/org/apache/cassandra/config/Schema.java
deleted file mode 100644
index 253a66b..0000000
--- a/src/java/org/apache/cassandra/config/Schema.java
+++ /dev/null
@@ -1,855 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.config;
-
-import java.util.*;
-import java.util.stream.Collectors;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Sets;
-import org.cliffc.high_scale_lib.NonBlockingHashMap;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.cassandra.cql3.functions.*;
-import org.apache.cassandra.db.ColumnFamilyStore;
-import org.apache.cassandra.db.Keyspace;
-import org.apache.cassandra.db.SystemKeyspace;
-import org.apache.cassandra.db.commitlog.CommitLog;
-import org.apache.cassandra.db.compaction.CompactionManager;
-import org.apache.cassandra.db.marshal.AbstractType;
-import org.apache.cassandra.db.marshal.UserType;
-import org.apache.cassandra.gms.Gossiper;
-import org.apache.cassandra.index.Index;
-import org.apache.cassandra.io.sstable.Descriptor;
-import org.apache.cassandra.locator.LocalStrategy;
-import org.apache.cassandra.schema.*;
-import org.apache.cassandra.service.MigrationManager;
-import org.apache.cassandra.utils.ConcurrentBiMap;
-import org.apache.cassandra.utils.Pair;
-
-public class Schema
-{
- private static final Logger logger = LoggerFactory.getLogger(Schema.class);
-
- public static final Schema instance = new Schema();
-
- /* metadata map for faster keyspace lookup */
- private final Map<String, KeyspaceMetadata> keyspaces = new NonBlockingHashMap<>();
-
- /* Keyspace objects, one per keyspace. Only one instance should ever exist for any given keyspace. */
- private final Map<String, Keyspace> keyspaceInstances = new NonBlockingHashMap<>();
-
- /* metadata map for faster ColumnFamily lookup */
- private final ConcurrentBiMap<Pair<String, String>, UUID> cfIdMap = new ConcurrentBiMap<>();
-
- private volatile UUID version;
- private volatile UUID altVersion;
-
- /**
- * Initialize empty schema object and load the hardcoded system tables
- */
- public Schema()
- {
- if (DatabaseDescriptor.isDaemonInitialized() || DatabaseDescriptor.isToolInitialized())
- {
- load(SchemaKeyspace.metadata());
- load(SystemKeyspace.metadata());
- }
- }
-
- /**
- * load keyspace (keyspace) definitions, but do not initialize the keyspace instances.
- * Schema version may be updated as the result.
- */
- public Schema loadFromDisk()
- {
- return loadFromDisk(true);
- }
-
- /**
- * Load schema definitions from disk.
- *
- * @param updateVersion true if schema version needs to be updated
- */
- public Schema loadFromDisk(boolean updateVersion)
- {
- load(SchemaKeyspace.fetchNonSystemKeyspaces());
- if (updateVersion)
- updateVersion();
- return this;
- }
-
- /**
- * Load up non-system keyspaces
- *
- * @param keyspaceDefs The non-system keyspace definitions
- *
- * @return self to support chaining calls
- */
- public Schema load(Iterable<KeyspaceMetadata> keyspaceDefs)
- {
- keyspaceDefs.forEach(this::load);
- return this;
- }
-
- /**
- * Load specific keyspace into Schema
- *
- * @param keyspaceDef The keyspace to load up
- *
- * @return self to support chaining calls
- */
- public Schema load(KeyspaceMetadata keyspaceDef)
- {
- keyspaceDef.tables.forEach(this::load);
- keyspaceDef.views.forEach(this::load);
- setKeyspaceMetadata(keyspaceDef);
- return this;
- }
-
- /**
- * Get keyspace instance by name
- *
- * @param keyspaceName The name of the keyspace
- *
- * @return Keyspace object or null if keyspace was not found
- */
- public Keyspace getKeyspaceInstance(String keyspaceName)
- {
- return keyspaceInstances.get(keyspaceName);
- }
-
- /**
- * Retrieve a CFS by name even if that CFS is an index
- *
- * An index is identified by looking for '.' in the CF name and separating to find the base table
- * containing the index
- * @param ksNameAndCFName
- * @return The named CFS or null if the keyspace, base table, or index don't exist
- */
- public ColumnFamilyStore getColumnFamilyStoreIncludingIndexes(Pair<String, String> ksNameAndCFName)
- {
- String ksName = ksNameAndCFName.left;
- String cfName = ksNameAndCFName.right;
- Pair<String, String> baseTable;
-
- /*
- * Split does special case a one character regex, and it looks like it can detect
- * if you use two characters to escape '.', but it still allocates a useless array.
- */
- int indexOfSeparator = cfName.indexOf('.');
- if (indexOfSeparator > -1)
- baseTable = Pair.create(ksName, cfName.substring(0, indexOfSeparator));
- else
- baseTable = ksNameAndCFName;
-
- UUID cfId = cfIdMap.get(baseTable);
- if (cfId == null)
- return null;
-
- Keyspace ks = keyspaceInstances.get(ksName);
- if (ks == null)
- return null;
-
- ColumnFamilyStore baseCFS = ks.getColumnFamilyStore(cfId);
-
- //Not an index
- if (indexOfSeparator == -1)
- return baseCFS;
-
- if (baseCFS == null)
- return null;
-
- Index index = baseCFS.indexManager.getIndexByName(cfName.substring(indexOfSeparator + 1, cfName.length()));
- if (index == null)
- return null;
-
- //Shouldn't ask for a backing table if there is none so just throw?
- //Or should it return null?
- return index.getBackingTable().get();
- }
-
- public ColumnFamilyStore getColumnFamilyStoreInstance(UUID cfId)
- {
- Pair<String, String> pair = cfIdMap.inverse().get(cfId);
- if (pair == null)
- return null;
- Keyspace instance = getKeyspaceInstance(pair.left);
- if (instance == null)
- return null;
- return instance.getColumnFamilyStore(cfId);
- }
-
- /**
- * Store given Keyspace instance to the schema
- *
- * @param keyspace The Keyspace instance to store
- *
- * @throws IllegalArgumentException if Keyspace is already stored
- */
- public void storeKeyspaceInstance(Keyspace keyspace)
- {
- if (keyspaceInstances.containsKey(keyspace.getName()))
- throw new IllegalArgumentException(String.format("Keyspace %s was already initialized.", keyspace.getName()));
-
- keyspaceInstances.put(keyspace.getName(), keyspace);
- }
-
- /**
- * Remove keyspace from schema
- *
- * @param keyspaceName The name of the keyspace to remove
- *
- * @return removed keyspace instance or null if it wasn't found
- */
- public Keyspace removeKeyspaceInstance(String keyspaceName)
- {
- return keyspaceInstances.remove(keyspaceName);
- }
-
- /**
- * Remove keyspace definition from system
- *
- * @param ksm The keyspace definition to remove
- */
- public void clearKeyspaceMetadata(KeyspaceMetadata ksm)
- {
- keyspaces.remove(ksm.name);
- }
-
- /**
- * Given a keyspace name and column family name, get the column family
- * meta data. If the keyspace name or column family name is not valid
- * this function returns null.
- *
- * @param keyspaceName The keyspace name
- * @param cfName The ColumnFamily name
- *
- * @return ColumnFamily Metadata object or null if it wasn't found
- */
- public CFMetaData getCFMetaData(String keyspaceName, String cfName)
- {
- assert keyspaceName != null;
-
- KeyspaceMetadata ksm = keyspaces.get(keyspaceName);
- return ksm == null
- ? null
- : ksm.getTableOrViewNullable(cfName);
- }
-
- /**
- * Get ColumnFamily metadata by its identifier
- *
- * @param cfId The ColumnFamily identifier
- *
- * @return metadata about ColumnFamily
- */
- public CFMetaData getCFMetaData(UUID cfId)
- {
- Pair<String,String> cf = getCF(cfId);
- return (cf == null) ? null : getCFMetaData(cf.left, cf.right);
- }
-
- public CFMetaData getCFMetaData(Descriptor descriptor)
- {
- return getCFMetaData(descriptor.ksname, descriptor.cfname);
- }
-
- public int getNumberOfTables()
- {
- return cfIdMap.size();
- }
-
- public ViewDefinition getView(String keyspaceName, String viewName)
- {
- assert keyspaceName != null;
- KeyspaceMetadata ksm = keyspaces.get(keyspaceName);
- return (ksm == null) ? null : ksm.views.getNullable(viewName);
- }
-
- /**
- * Get metadata about keyspace by its name
- *
- * @param keyspaceName The name of the keyspace
- *
- * @return The keyspace metadata or null if it wasn't found
- */
- public KeyspaceMetadata getKSMetaData(String keyspaceName)
- {
- assert keyspaceName != null;
- return keyspaces.get(keyspaceName);
- }
-
- private Set<String> getNonSystemKeyspacesSet()
- {
- return Sets.difference(keyspaces.keySet(), SchemaConstants.LOCAL_SYSTEM_KEYSPACE_NAMES);
- }
-
- /**
- * @return collection of the non-system keyspaces (note that this count as system only the
- * non replicated keyspaces, so keyspace like system_traces which are replicated are actually
- * returned. See getUserKeyspace() below if you don't want those)
- */
- public List<String> getNonSystemKeyspaces()
- {
- return ImmutableList.copyOf(getNonSystemKeyspacesSet());
- }
-
- /**
- * @return a collection of keyspaces that do not use LocalStrategy for replication
- */
- public List<String> getNonLocalStrategyKeyspaces()
- {
- return keyspaces.values().stream()
- .filter(keyspace -> keyspace.params.replication.klass != LocalStrategy.class)
- .map(keyspace -> keyspace.name)
- .collect(Collectors.toList());
- }
-
- /**
- * @return collection of the user defined keyspaces
- */
- public List<String> getUserKeyspaces()
- {
- return ImmutableList.copyOf(Sets.difference(getNonSystemKeyspacesSet(), SchemaConstants.REPLICATED_SYSTEM_KEYSPACE_NAMES));
- }
-
- public Keyspaces getReplicatedKeyspaces()
- {
- Keyspaces.Builder builder = Keyspaces.builder();
-
- keyspaces.values()
- .stream()
- .filter(k -> !SchemaConstants.isLocalSystemKeyspace(k.name))
- .forEach(builder::add);
-
- return builder.build();
- }
-
- /**
- * Get metadata about keyspace inner ColumnFamilies
- *
- * @param keyspaceName The name of the keyspace
- *
- * @return metadata about ColumnFamilies the belong to the given keyspace
- */
- public Iterable<CFMetaData> getTablesAndViews(String keyspaceName)
- {
- assert keyspaceName != null;
- KeyspaceMetadata ksm = keyspaces.get(keyspaceName);
- assert ksm != null;
- return ksm.tablesAndViews();
- }
-
- /**
- * @return collection of the all keyspace names registered in the system (system and non-system)
- */
- public Set<String> getKeyspaces()
- {
- return keyspaces.keySet();
- }
-
- public Keyspaces getKeyspaces(Set<String> includedKeyspaceNames)
- {
- Keyspaces.Builder builder = Keyspaces.builder();
- keyspaces.values()
- .stream()
- .filter(k -> includedKeyspaceNames.contains(k.name))
- .forEach(builder::add);
- return builder.build();
- }
-
- /**
- * Update (or insert) new keyspace definition
- *
- * @param ksm The metadata about keyspace
- */
- public void setKeyspaceMetadata(KeyspaceMetadata ksm)
- {
- assert ksm != null;
-
- keyspaces.put(ksm.name, ksm);
- Keyspace keyspace = getKeyspaceInstance(ksm.name);
- if (keyspace != null)
- keyspace.setMetadata(ksm);
- }
-
- /* ColumnFamily query/control methods */
-
- /**
- * @param cfId The identifier of the ColumnFamily to lookup
- * @return The (ksname,cfname) pair for the given id, or null if it has been dropped.
- */
- public Pair<String,String> getCF(UUID cfId)
- {
- return cfIdMap.inverse().get(cfId);
- }
-
- /**
- * @param ksAndCFName The identifier of the ColumnFamily to lookup
- * @return true if the KS and CF pair is a known one, false otherwise.
- */
- public boolean hasCF(Pair<String, String> ksAndCFName)
- {
- return cfIdMap.containsKey(ksAndCFName);
- }
-
- /**
- * Lookup keyspace/ColumnFamily identifier
- *
- * @param ksName The keyspace name
- * @param cfName The ColumnFamily name
- *
- * @return The id for the given (ksname,cfname) pair, or null if it has been dropped.
- */
- public UUID getId(String ksName, String cfName)
- {
- return cfIdMap.get(Pair.create(ksName, cfName));
- }
-
- /**
- * Load individual ColumnFamily Definition to the schema
- * (to make ColumnFamily lookup faster)
- *
- * @param cfm The ColumnFamily definition to load
- */
- public void load(CFMetaData cfm)
- {
- Pair<String, String> key = Pair.create(cfm.ksName, cfm.cfName);
-
- if (cfIdMap.containsKey(key))
- throw new RuntimeException(String.format("Attempting to load already loaded table %s.%s", cfm.ksName, cfm.cfName));
-
- logger.debug("Adding {} to cfIdMap", cfm);
- cfIdMap.put(key, cfm.cfId);
- }
-
- /**
- * Load individual View Definition to the schema
- * (to make View lookup faster)
- *
- * @param view The View definition to load
- */
- public void load(ViewDefinition view)
- {
- CFMetaData cfm = view.metadata;
- Pair<String, String> key = Pair.create(cfm.ksName, cfm.cfName);
-
- if (cfIdMap.containsKey(key))
- throw new RuntimeException(String.format("Attempting to load already loaded view %s.%s", cfm.ksName, cfm.cfName));
-
- logger.debug("Adding {} to cfIdMap", cfm);
- cfIdMap.put(key, cfm.cfId);
- }
-
- /**
- * Used for ColumnFamily data eviction out from the schema
- *
- * @param cfm The ColumnFamily Definition to evict
- */
- public void unload(CFMetaData cfm)
- {
- cfIdMap.remove(Pair.create(cfm.ksName, cfm.cfName));
- }
-
- /**
- * Used for View eviction from the schema
- *
- * @param view The view definition to evict
- */
- private void unload(ViewDefinition view)
- {
- cfIdMap.remove(Pair.create(view.ksName, view.viewName));
- }
-
- /* Function helpers */
-
- /**
- * Get all function overloads with the specified name
- *
- * @param name fully qualified function name
- * @return an empty list if the keyspace or the function name are not found;
- * a non-empty collection of {@link Function} otherwise
- */
- public Collection<Function> getFunctions(FunctionName name)
- {
- if (!name.hasKeyspace())
- throw new IllegalArgumentException(String.format("Function name must be fully quallified: got %s", name));
-
- KeyspaceMetadata ksm = getKSMetaData(name.keyspace);
- return ksm == null
- ? Collections.emptyList()
- : ksm.functions.get(name);
- }
-
- /**
- * Find the function with the specified name
- *
- * @param name fully qualified function name
- * @param argTypes function argument types
- * @return an empty {@link Optional} if the keyspace or the function name are not found;
- * a non-empty optional of {@link Function} otherwise
- */
- public Optional<Function> findFunction(FunctionName name, List<AbstractType<?>> argTypes)
- {
- if (!name.hasKeyspace())
- throw new IllegalArgumentException(String.format("Function name must be fully quallified: got %s", name));
-
- KeyspaceMetadata ksm = getKSMetaData(name.keyspace);
- return ksm == null
- ? Optional.empty()
- : ksm.functions.find(name, argTypes);
- }
-
- /* Version control */
-
- /**
- * The schema version to announce.
- * This will be either the "real" schema version including the {@code cdc} column,
- * if no node in the cluster is running at 3.0, or a 3.0 compatible
- * schema version, with the {@code cdc} column excluded, if at least one node is
- * running 3.0.
- *
- * @return "current" schema version
- */
- public UUID getVersion()
- {
- return Gossiper.instance.isEnabled() && Gossiper.instance.isAnyNodeOn30()
- ? altVersion
- : version;
- }
-
- /**
- * The 3.11 schema version, always includes the {@code cdc} column.
- */
- public UUID getRealVersion()
- {
- return version;
- }
-
- /**
- * The "alternative" schema version, compatible to 3.0, always excludes the
- * {@code cdc} column.
- */
- public UUID getAltVersion()
- {
- return altVersion;
- }
-
- /**
- * Checks whether the given schema version is the same as the current local schema
- * version, either the 3.0 compatible or "real" one.
- */
- public boolean isSameVersion(UUID schemaVersion)
- {
- return schemaVersion != null
- && (schemaVersion.equals(version) || schemaVersion.equals(altVersion));
- }
-
- /**
- * Checks whether the current schema is empty.
- */
- public boolean isEmpty()
- {
- return SchemaConstants.emptyVersion.equals(version);
- }
-
- /**
- * Read schema from system keyspace and calculate MD5 digest of every row, resulting digest
- * will be converted into UUID which would act as content-based version of the schema.
- *
- * 3.11 note: we calculate the "real" schema version and the 3.0 compatible schema
- * version here.
- */
- public void updateVersion()
- {
- Pair<UUID, UUID> mixedVersions = SchemaKeyspace.calculateSchemaDigest();
- version = mixedVersions.left;
- altVersion = mixedVersions.right;
- SystemKeyspace.updateSchemaVersion(getVersion());
- }
-
- /**
- * Like updateVersion, but also announces via gossip
- *
- * 3.11 note: we announce the "current" schema version, which can be either the 3.0
- * compatible one, if at least one node is still running 3.0, or the "real" schema version.
- */
- public void updateVersionAndAnnounce()
- {
- updateVersion();
- UUID current = getVersion();
- MigrationManager.passiveAnnounce(current, current == getAltVersion());
- }
-
- /**
- * Clear all KS/CF metadata and reset version.
- */
- public synchronized void clear()
- {
- for (String keyspaceName : getNonSystemKeyspaces())
- {
- KeyspaceMetadata ksm = getKSMetaData(keyspaceName);
- ksm.tables.forEach(this::unload);
- ksm.views.forEach(this::unload);
- clearKeyspaceMetadata(ksm);
- }
-
- updateVersionAndAnnounce();
- }
-
- public void addKeyspace(KeyspaceMetadata ksm)
- {
- assert getKSMetaData(ksm.name) == null;
- load(ksm);
-
- Keyspace.open(ksm.name);
- MigrationManager.instance.notifyCreateKeyspace(ksm);
- }
-
- public void updateKeyspace(String ksName, KeyspaceParams newParams)
- {
- KeyspaceMetadata ksm = update(ksName, ks -> ks.withSwapped(newParams));
- MigrationManager.instance.notifyUpdateKeyspace(ksm);
- }
-
- public void dropKeyspace(String ksName)
- {
- KeyspaceMetadata ksm = Schema.instance.getKSMetaData(ksName);
- String snapshotName = Keyspace.getTimestampedSnapshotNameWithPrefix(ksName, ColumnFamilyStore.SNAPSHOT_DROP_PREFIX);
-
- CompactionManager.instance.interruptCompactionFor(ksm.tablesAndViews(), true);
-
- Keyspace keyspace = Keyspace.open(ksm.name);
-
- // remove all cfs from the keyspace instance.
- List<UUID> droppedCfs = new ArrayList<>();
- for (CFMetaData cfm : ksm.tablesAndViews())
- {
- ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfm.cfName);
-
- unload(cfm);
-
- if (DatabaseDescriptor.isAutoSnapshot())
- cfs.snapshot(snapshotName);
- Keyspace.open(ksm.name).dropCf(cfm.cfId);
-
- droppedCfs.add(cfm.cfId);
- }
-
- // remove the keyspace from the static instances.
- Keyspace.clear(ksm.name);
- clearKeyspaceMetadata(ksm);
-
- Keyspace.writeOrder.awaitNewBarrier();
-
- // force a new segment in the CL
- CommitLog.instance.forceRecycleAllSegments(droppedCfs);
-
- MigrationManager.instance.notifyDropKeyspace(ksm);
- }
-
- public void addTable(CFMetaData cfm)
- {
- assert getCFMetaData(cfm.ksName, cfm.cfName) == null;
-
- // Make sure the keyspace is initialized
- // and init the new CF before switching the KSM to the new one
- // to avoid races as in CASSANDRA-10761
- Keyspace.open(cfm.ksName).initCf(cfm, true);
- // Update the keyspaces map with the updated metadata
- update(cfm.ksName, ks -> ks.withSwapped(ks.tables.with(cfm)));
- // Update the table ID <-> table name map (cfIdMap)
- load(cfm);
- MigrationManager.instance.notifyCreateColumnFamily(cfm);
- }
-
- public void updateTable(CFMetaData table)
- {
- CFMetaData current = getCFMetaData(table.ksName, table.cfName);
- assert current != null;
- boolean changeAffectsStatements = current.apply(table);
-
- Keyspace keyspace = Keyspace.open(current.ksName);
- keyspace.getColumnFamilyStore(current.cfName).reload();
- MigrationManager.instance.notifyUpdateColumnFamily(current, changeAffectsStatements);
- }
-
- public void dropTable(String ksName, String tableName)
- {
- KeyspaceMetadata oldKsm = getKSMetaData(ksName);
- assert oldKsm != null;
- ColumnFamilyStore cfs = Keyspace.open(ksName).getColumnFamilyStore(tableName);
- assert cfs != null;
-
- // make sure all the indexes are dropped, or else.
- cfs.indexManager.markAllIndexesRemoved();
-
- // reinitialize the keyspace.
- CFMetaData cfm = oldKsm.tables.get(tableName).get();
- KeyspaceMetadata newKsm = oldKsm.withSwapped(oldKsm.tables.without(tableName));
-
- unload(cfm);
- setKeyspaceMetadata(newKsm);
-
- CompactionManager.instance.interruptCompactionFor(Collections.singleton(cfm), true);
-
- if (DatabaseDescriptor.isAutoSnapshot())
- cfs.snapshot(Keyspace.getTimestampedSnapshotNameWithPrefix(cfs.name, ColumnFamilyStore.SNAPSHOT_DROP_PREFIX));
- Keyspace.open(ksName).dropCf(cfm.cfId);
- MigrationManager.instance.notifyDropColumnFamily(cfm);
-
- CommitLog.instance.forceRecycleAllSegments(Collections.singleton(cfm.cfId));
- }
-
- public void addView(ViewDefinition view)
- {
- assert getCFMetaData(view.ksName, view.viewName) == null;
-
- Keyspace keyspace = Keyspace.open(view.ksName);
-
- // Make sure the keyspace is initialized and initialize the table.
- keyspace.initCf(view.metadata, true);
- // Update the keyspaces map with the updated metadata
- update(view.ksName, ks -> ks.withSwapped(ks.views.with(view)));
- // Update the table ID <-> table name map (cfIdMap)
- load(view);
-
- keyspace.viewManager.reload();
- MigrationManager.instance.notifyCreateView(view);
- }
-
- public void updateView(ViewDefinition view)
- {
- ViewDefinition current = getKSMetaData(view.ksName).views.get(view.viewName).get();
- boolean changeAffectsStatements = current.metadata.apply(view.metadata);
-
- Keyspace keyspace = Keyspace.open(current.ksName);
- keyspace.getColumnFamilyStore(current.viewName).reload();
- Keyspace.open(current.ksName).viewManager.update(current.viewName);
- MigrationManager.instance.notifyUpdateView(current, changeAffectsStatements);
- }
-
- public void dropView(String ksName, String viewName)
- {
- KeyspaceMetadata oldKsm = getKSMetaData(ksName);
- assert oldKsm != null;
- ColumnFamilyStore cfs = Keyspace.open(ksName).getColumnFamilyStore(viewName);
- assert cfs != null;
-
- // make sure all the indexes are dropped, or else.
- cfs.indexManager.markAllIndexesRemoved();
-
- // reinitialize the keyspace.
- ViewDefinition view = oldKsm.views.get(viewName).get();
- KeyspaceMetadata newKsm = oldKsm.withSwapped(oldKsm.views.without(viewName));
-
- unload(view);
- setKeyspaceMetadata(newKsm);
-
- CompactionManager.instance.interruptCompactionFor(Collections.singleton(view.metadata), true);
-
- if (DatabaseDescriptor.isAutoSnapshot())
- cfs.snapshot(Keyspace.getTimestampedSnapshotName(cfs.name));
- Keyspace.open(ksName).dropCf(view.metadata.cfId);
- Keyspace.open(ksName).viewManager.reload();
- MigrationManager.instance.notifyDropView(view);
-
- CommitLog.instance.forceRecycleAllSegments(Collections.singleton(view.metadata.cfId));
- }
-
- public void addType(UserType ut)
- {
- update(ut.keyspace, ks -> ks.withSwapped(ks.types.with(ut)));
- MigrationManager.instance.notifyCreateUserType(ut);
- }
-
- public void updateType(UserType ut)
- {
- update(ut.keyspace, ks -> ks.withSwapped(ks.types.without(ut.name).with(ut)));
- MigrationManager.instance.notifyUpdateUserType(ut);
- }
-
- public void dropType(UserType ut)
- {
- update(ut.keyspace, ks -> ks.withSwapped(ks.types.without(ut.name)));
- MigrationManager.instance.notifyDropUserType(ut);
- }
-
- public void addFunction(UDFunction udf)
- {
- update(udf.name().keyspace, ks -> ks.withSwapped(ks.functions.with(udf)));
- MigrationManager.instance.notifyCreateFunction(udf);
- }
-
- public void updateFunction(UDFunction udf)
- {
- update(udf.name().keyspace, ks -> ks.withSwapped(ks.functions.without(udf.name(), udf.argTypes()).with(udf)));
- MigrationManager.instance.notifyUpdateFunction(udf);
- }
-
- public void dropFunction(UDFunction udf)
- {
- update(udf.name().keyspace, ks -> ks.withSwapped(ks.functions.without(udf.name(), udf.argTypes())));
- MigrationManager.instance.notifyDropFunction(udf);
- }
-
- public void addAggregate(UDAggregate uda)
- {
- update(uda.name().keyspace, ks -> ks.withSwapped(ks.functions.with(uda)));
- MigrationManager.instance.notifyCreateAggregate(uda);
- }
-
- public void updateAggregate(UDAggregate uda)
- {
- update(uda.name().keyspace, ks -> ks.withSwapped(ks.functions.without(uda.name(), uda.argTypes()).with(uda)));
- MigrationManager.instance.notifyUpdateAggregate(uda);
- }
-
- public void dropAggregate(UDAggregate uda)
- {
- update(uda.name().keyspace, ks -> ks.withSwapped(ks.functions.without(uda.name(), uda.argTypes())));
- MigrationManager.instance.notifyDropAggregate(uda);
- }
-
- private synchronized KeyspaceMetadata update(String keyspaceName, java.util.function.Function<KeyspaceMetadata, KeyspaceMetadata> transformation)
- {
- KeyspaceMetadata current = getKSMetaData(keyspaceName);
- if (current == null)
- throw new IllegalStateException(String.format("Keyspace %s doesn't exist", keyspaceName));
-
- KeyspaceMetadata transformed = transformation.apply(current);
- setKeyspaceMetadata(transformed);
-
- return transformed;
- }
-
- /**
- * Converts the given schema version to a string. Returns {@code unknown}, if {@code version} is {@code null}
- * or {@code "(empty)"}, if {@code version} refers to an {@link SchemaConstants#emptyVersion empty) schema.
- */
- public static String schemaVersionToString(UUID version)
- {
- return version == null
- ? "unknown"
- : SchemaConstants.emptyVersion.equals(version)
- ? "(empty)"
- : version.toString();
- }
-}
diff --git a/src/java/org/apache/cassandra/config/SchemaConstants.java b/src/java/org/apache/cassandra/config/SchemaConstants.java
deleted file mode 100644
index 9e60b60..0000000
--- a/src/java/org/apache/cassandra/config/SchemaConstants.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.config;
-
-import java.security.MessageDigest;
-import java.security.NoSuchAlgorithmException;
-import java.util.Set;
-import java.util.UUID;
-
-import com.google.common.collect.ImmutableSet;
-
-public final class SchemaConstants
-{
- public static final String SYSTEM_KEYSPACE_NAME = "system";
- public static final String SCHEMA_KEYSPACE_NAME = "system_schema";
-
- public static final String TRACE_KEYSPACE_NAME = "system_traces";
- public static final String AUTH_KEYSPACE_NAME = "system_auth";
- public static final String DISTRIBUTED_KEYSPACE_NAME = "system_distributed";
-
- /* system keyspace names (the ones with LocalStrategy replication strategy) */
- public static final Set<String> LOCAL_SYSTEM_KEYSPACE_NAMES =
- ImmutableSet.of(SYSTEM_KEYSPACE_NAME, SCHEMA_KEYSPACE_NAME);
-
- /* replicate system keyspace names (the ones with a "true" replication strategy) */
- public static final Set<String> REPLICATED_SYSTEM_KEYSPACE_NAMES =
- ImmutableSet.of(TRACE_KEYSPACE_NAME, AUTH_KEYSPACE_NAME, DISTRIBUTED_KEYSPACE_NAME);
- /**
- * longest permissible KS or CF name. Our main concern is that filename not be more than 255 characters;
- * the filename will contain both the KS and CF names. Since non-schema-name components only take up
- * ~64 characters, we could allow longer names than this, but on Windows, the entire path should be not greater than
- * 255 characters, so a lower limit here helps avoid problems. See CASSANDRA-4110.
- */
- public static final int NAME_LENGTH = 48;
-
- // 59adb24e-f3cd-3e02-97f0-5b395827453f
- public static final UUID emptyVersion;
-
- static
- {
- try
- {
- emptyVersion = UUID.nameUUIDFromBytes(MessageDigest.getInstance("MD5").digest());
- }
- catch (NoSuchAlgorithmException e)
- {
- throw new AssertionError();
- }
- }
-
- /**
- * @return whether or not the keyspace is a really system one (w/ LocalStrategy, unmodifiable, hardcoded)
- */
- public static boolean isLocalSystemKeyspace(String keyspaceName)
- {
- return LOCAL_SYSTEM_KEYSPACE_NAMES.contains(keyspaceName.toLowerCase());
- }
-
- /**
- * @return whether or not the keyspace is a replicated system ks (system_auth, system_traces, system_distributed)
- */
- public static boolean isReplicatedSystemKeyspace(String keyspaceName)
- {
- return REPLICATED_SYSTEM_KEYSPACE_NAMES.contains(keyspaceName.toLowerCase());
- }
-}
diff --git a/src/java/org/apache/cassandra/config/ViewDefinition.java b/src/java/org/apache/cassandra/config/ViewDefinition.java
deleted file mode 100644
index 77cbcc9..0000000
--- a/src/java/org/apache/cassandra/config/ViewDefinition.java
+++ /dev/null
@@ -1,166 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.config;
-
-import java.util.List;
-import java.util.Objects;
-import java.util.UUID;
-import java.util.stream.Collectors;
-
-import org.antlr.runtime.*;
-import org.apache.cassandra.cql3.*;
-import org.apache.cassandra.cql3.statements.SelectStatement;
-import org.apache.cassandra.db.view.View;
-import org.apache.cassandra.exceptions.SyntaxException;
-import org.apache.commons.lang3.builder.HashCodeBuilder;
-import org.apache.commons.lang3.builder.ToStringBuilder;
-
-public class ViewDefinition
-{
- public final String ksName;
- public final String viewName;
- public final UUID baseTableId;
- public final String baseTableName;
- public final boolean includeAllColumns;
- public final CFMetaData metadata;
-
- public SelectStatement.RawStatement select;
- public String whereClause;
-
- public ViewDefinition(ViewDefinition def)
- {
- this(def.ksName, def.viewName, def.baseTableId, def.baseTableName, def.includeAllColumns, def.select, def.whereClause, def.metadata);
- }
-
- /**
- * @param viewName Name of the view
- * @param baseTableId Internal ID of the table which this view is based off of
- * @param includeAllColumns Whether to include all columns or not
- */
- public ViewDefinition(String ksName, String viewName, UUID baseTableId, String baseTableName, boolean includeAllColumns, SelectStatement.RawStatement select, String whereClause, CFMetaData metadata)
- {
- this.ksName = ksName;
- this.viewName = viewName;
- this.baseTableId = baseTableId;
- this.baseTableName = baseTableName;
- this.includeAllColumns = includeAllColumns;
- this.select = select;
- this.whereClause = whereClause;
- this.metadata = metadata;
- }
-
- /**
- * @return true if the view specified by this definition will include the column, false otherwise
- */
- public boolean includes(ColumnIdentifier column)
- {
- return metadata.getColumnDefinition(column) != null;
- }
-
- public ViewDefinition copy()
- {
- return new ViewDefinition(ksName, viewName, baseTableId, baseTableName, includeAllColumns, select, whereClause, metadata.copy());
- }
-
- public CFMetaData baseTableMetadata()
- {
- return Schema.instance.getCFMetaData(baseTableId);
- }
-
- @Override
- public boolean equals(Object o)
- {
- if (this == o)
- return true;
-
- if (!(o instanceof ViewDefinition))
- return false;
-
- ViewDefinition other = (ViewDefinition) o;
- return Objects.equals(ksName, other.ksName)
- && Objects.equals(viewName, other.viewName)
- && Objects.equals(baseTableId, other.baseTableId)
- && Objects.equals(includeAllColumns, other.includeAllColumns)
- && Objects.equals(whereClause, other.whereClause)
- && Objects.equals(metadata, other.metadata);
- }
-
- @Override
- public int hashCode()
- {
- return new HashCodeBuilder(29, 1597)
- .append(ksName)
- .append(viewName)
- .append(baseTableId)
- .append(includeAllColumns)
- .append(whereClause)
- .append(metadata)
- .toHashCode();
- }
-
- @Override
- public String toString()
- {
- return new ToStringBuilder(this)
- .append("ksName", ksName)
- .append("viewName", viewName)
- .append("baseTableId", baseTableId)
- .append("baseTableName", baseTableName)
- .append("includeAllColumns", includeAllColumns)
- .append("whereClause", whereClause)
- .append("metadata", metadata)
- .toString();
- }
-
- /**
- * Replace the column 'from' with 'to' in this materialized view definition's partition,
- * clustering, or included columns.
- * @param from the existing column
- * @param to the new column
- */
- public void renameColumn(ColumnIdentifier from, ColumnIdentifier to)
- {
- metadata.renameColumn(from, to);
-
- // convert whereClause to Relations, rename ids in Relations, then convert back to whereClause
- List<Relation> relations = whereClauseToRelations(whereClause);
- ColumnDefinition.Raw fromRaw = ColumnDefinition.Raw.forQuoted(from.toString());
- ColumnDefinition.Raw toRaw = ColumnDefinition.Raw.forQuoted(to.toString());
- List<Relation> newRelations = relations.stream()
- .map(r -> r.renameIdentifier(fromRaw, toRaw))
- .collect(Collectors.toList());
-
- this.whereClause = View.relationsToWhereClause(newRelations);
- String rawSelect = View.buildSelectStatement(baseTableName, metadata.allColumns(), whereClause);
- this.select = (SelectStatement.RawStatement) QueryProcessor.parseStatement(rawSelect);
- }
-
- private static List<Relation> whereClauseToRelations(String whereClause)
- {
- try
- {
- List<Relation> relations = CQLFragmentParser.parseAnyUnhandled(CqlParser::whereClause, whereClause).build().relations;
-
- return relations;
- }
- catch (RecognitionException | SyntaxException exc)
- {
- throw new RuntimeException("Unexpected error parsing materialized view's where clause while handling column rename: ", exc);
- }
- }
-}
diff --git a/src/java/org/apache/cassandra/cql3/AbstractConditions.java b/src/java/org/apache/cassandra/cql3/AbstractConditions.java
deleted file mode 100644
index 530d2b1..0000000
--- a/src/java/org/apache/cassandra/cql3/AbstractConditions.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.cql3;
-
-import java.util.List;
-
-import org.apache.cassandra.config.ColumnDefinition;
-import org.apache.cassandra.cql3.functions.Function;
-
-/**
- * Base class for <code>Conditions</code> classes.
- *
- */
-abstract class AbstractConditions implements Conditions
-{
- public void addFunctionsTo(List<Function> functions)
- {
- }
-
- public Iterable<ColumnDefinition> getColumns()
- {
- return null;
- }
-
- public boolean isEmpty()
- {
- return false;
- }
-
- public boolean appliesToStaticColumns()
- {
- return false;
- }
-
- public boolean appliesToRegularColumns()
- {
- return false;
- }
-
- public boolean isIfExists()
- {
- return false;
- }
-
- public boolean isIfNotExists()
- {
- return false;
- }
-}
diff --git a/src/java/org/apache/cassandra/cql3/AbstractMarker.java b/src/java/org/apache/cassandra/cql3/AbstractMarker.java
index a7fb073..19ce26c 100644
--- a/src/java/org/apache/cassandra/cql3/AbstractMarker.java
+++ b/src/java/org/apache/cassandra/cql3/AbstractMarker.java
@@ -58,7 +58,7 @@
*/
public static class Raw extends Term.Raw
{
- private final int bindIndex;
+ protected final int bindIndex;
public Raw(int bindIndex)
{
@@ -105,11 +105,6 @@
{
return "?";
}
-
- public int bindIndex()
- {
- return bindIndex;
- }
}
/** A MultiColumnRaw version of AbstractMarker.Raw */
@@ -145,7 +140,7 @@
*
* Because a single type is used, a List is used to represent the values.
*/
- public static class INRaw extends Raw
+ public static final class INRaw extends Raw
{
public INRaw(int bindIndex)
{
@@ -159,9 +154,9 @@
}
@Override
- public AbstractMarker prepare(String keyspace, ColumnSpecification receiver) throws InvalidRequestException
+ public Lists.Marker prepare(String keyspace, ColumnSpecification receiver) throws InvalidRequestException
{
- return new Lists.Marker(bindIndex(), makeInReceiver(receiver));
+ return new Lists.Marker(bindIndex, makeInReceiver(receiver));
}
}
}
diff --git a/src/java/org/apache/cassandra/cql3/Attributes.java b/src/java/org/apache/cassandra/cql3/Attributes.java
index d4e230f..e841828 100644
--- a/src/java/org/apache/cassandra/cql3/Attributes.java
+++ b/src/java/org/apache/cassandra/cql3/Attributes.java
@@ -20,15 +20,17 @@
import java.nio.ByteBuffer;
import java.util.List;
-import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.cql3.functions.Function;
import org.apache.cassandra.db.ExpirationDateOverflowHandling;
import org.apache.cassandra.db.LivenessInfo;
import org.apache.cassandra.db.marshal.Int32Type;
import org.apache.cassandra.db.marshal.LongType;
import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.serializers.MarshalException;
import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.commons.lang3.builder.ToStringBuilder;
+import org.apache.commons.lang3.builder.ToStringStyle;
/**
* Utility class for the Parser to gather attributes for modification
@@ -100,7 +102,7 @@
return LongType.instance.compose(tval);
}
- public int getTimeToLive(QueryOptions options, CFMetaData metadata) throws InvalidRequestException
+ public int getTimeToLive(QueryOptions options, TableMetadata metadata) throws InvalidRequestException
{
if (timeToLive == null)
{
@@ -169,4 +171,10 @@
return new ColumnSpecification(ksName, cfName, new ColumnIdentifier("[ttl]", true), Int32Type.instance);
}
}
+
+ @Override
+ public String toString()
+ {
+ return ToStringBuilder.reflectionToString(this, ToStringStyle.SHORT_PREFIX_STYLE);
+ }
}
diff --git a/src/java/org/apache/cassandra/cql3/BatchQueryOptions.java b/src/java/org/apache/cassandra/cql3/BatchQueryOptions.java
index db7fa39..ac8f179 100644
--- a/src/java/org/apache/cassandra/cql3/BatchQueryOptions.java
+++ b/src/java/org/apache/cassandra/cql3/BatchQueryOptions.java
@@ -26,6 +26,8 @@
import org.apache.cassandra.db.ConsistencyLevel;
import org.apache.cassandra.service.QueryState;
+import org.apache.commons.lang3.builder.ToStringBuilder;
+import org.apache.commons.lang3.builder.ToStringStyle;
public abstract class BatchQueryOptions
{
@@ -62,6 +64,11 @@
return wrapped.getConsistency();
}
+ public String getKeyspace()
+ {
+ return wrapped.getKeyspace();
+ }
+
public ConsistencyLevel getSerialConsistency()
{
return wrapped.getSerialConsistency();
@@ -77,6 +84,11 @@
return wrapped.getTimestamp(state);
}
+ public int getNowInSeconds(QueryState state)
+ {
+ return wrapped.getNowInSeconds(state);
+ }
+
private static class WithoutPerStatementVariables extends BatchQueryOptions
{
private WithoutPerStatementVariables(QueryOptions wrapped, List<Object> queryOrIdList)
@@ -136,4 +148,10 @@
return getQueryOrIdList().get(i) instanceof MD5Digest;
}
}
+
+ @Override
+ public String toString()
+ {
+ return ToStringBuilder.reflectionToString(this, ToStringStyle.SHORT_PREFIX_STYLE);
+ }
}
diff --git a/src/java/org/apache/cassandra/cql3/CFName.java b/src/java/org/apache/cassandra/cql3/CFName.java
deleted file mode 100644
index 3f4a118..0000000
--- a/src/java/org/apache/cassandra/cql3/CFName.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.cql3;
-
-public class CFName extends KeyspaceElementName
-{
- private String cfName;
-
- public void setColumnFamily(String cf, boolean keepCase)
- {
- cfName = toInternalName(cf, keepCase);
- }
-
- public String getColumnFamily()
- {
- return cfName;
- }
-
- @Override
- public String toString()
- {
- return super.toString() + cfName;
- }
-}
diff --git a/src/java/org/apache/cassandra/cql3/CQL3Type.java b/src/java/org/apache/cassandra/cql3/CQL3Type.java
index 095d536..8f8df42 100644
--- a/src/java/org/apache/cassandra/cql3/CQL3Type.java
+++ b/src/java/org/apache/cassandra/cql3/CQL3Type.java
@@ -24,7 +24,7 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.cassandra.config.Schema;
+import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.db.marshal.*;
import org.apache.cassandra.db.marshal.CollectionType.Kind;
import org.apache.cassandra.exceptions.InvalidRequestException;
@@ -37,6 +37,8 @@
import org.apache.cassandra.transport.ProtocolVersion;
import org.apache.cassandra.utils.ByteBufferUtil;
+import static java.util.stream.Collectors.toList;
+
public interface CQL3Type
{
static final Logger logger = LoggerFactory.getLogger(CQL3Type.class);
@@ -88,7 +90,7 @@
private final AbstractType<?> type;
- private Native(AbstractType<?> type)
+ Native(AbstractType<?> type)
{
this.type = type;
}
@@ -465,15 +467,25 @@
@Override
public String toString()
{
+ return toString(true);
+ }
+
+ public String toString(boolean withFrozen)
+ {
StringBuilder sb = new StringBuilder();
- sb.append("frozen<tuple<");
+ if (withFrozen)
+ sb.append("frozen<");
+ sb.append("tuple<");
for (int i = 0; i < type.size(); i++)
{
if (i > 0)
sb.append(", ");
sb.append(type.type(i).asCQL3Type());
}
- sb.append(">>");
+ sb.append('>');
+ if (withFrozen)
+ sb.append('>');
+
return sb.toString();
}
}
@@ -482,7 +494,12 @@
// actual type used, so Raw is a "not yet prepared" CQL3Type.
public abstract class Raw
{
- protected boolean frozen = false;
+ protected final boolean frozen;
+
+ protected Raw(boolean frozen)
+ {
+ this.frozen = frozen;
+ }
public abstract boolean supportsFreezing();
@@ -491,11 +508,6 @@
return this.frozen;
}
- public boolean canBeNonFrozen()
- {
- return true;
- }
-
public boolean isDuration()
{
return false;
@@ -516,7 +528,7 @@
return null;
}
- public void freeze() throws InvalidRequestException
+ public Raw freeze()
{
String message = String.format("frozen<> is only allowed on collections, tuples, and user-defined types (got %s)", this);
throw new InvalidRequestException(message);
@@ -524,7 +536,7 @@
public CQL3Type prepare(String keyspace)
{
- KeyspaceMetadata ksm = Schema.instance.getKSMetaData(keyspace);
+ KeyspaceMetadata ksm = Schema.instance.getKeyspaceMetadata(keyspace);
if (ksm == null)
throw new ConfigurationException(String.format("Keyspace %s doesn't exist", keyspace));
return prepare(keyspace, ksm.types);
@@ -544,46 +556,41 @@
public static Raw from(CQL3Type type)
{
- return new RawType(type);
+ return new RawType(type, false);
}
public static Raw userType(UTName name)
{
- return new RawUT(name);
+ return new RawUT(name, false);
}
public static Raw map(CQL3Type.Raw t1, CQL3Type.Raw t2)
{
- return new RawCollection(CollectionType.Kind.MAP, t1, t2);
+ return new RawCollection(CollectionType.Kind.MAP, t1, t2, false);
}
public static Raw list(CQL3Type.Raw t)
{
- return new RawCollection(CollectionType.Kind.LIST, null, t);
+ return new RawCollection(CollectionType.Kind.LIST, null, t, false);
}
public static Raw set(CQL3Type.Raw t)
{
- return new RawCollection(CollectionType.Kind.SET, null, t);
+ return new RawCollection(CollectionType.Kind.SET, null, t, false);
}
public static Raw tuple(List<CQL3Type.Raw> ts)
{
- return new RawTuple(ts);
- }
-
- public static Raw frozen(CQL3Type.Raw t) throws InvalidRequestException
- {
- t.freeze();
- return t;
+ return new RawTuple(ts, false);
}
private static class RawType extends Raw
{
private final CQL3Type type;
- private RawType(CQL3Type type)
+ private RawType(CQL3Type type, boolean frozen)
{
+ super(frozen);
this.type = type;
}
@@ -620,20 +627,28 @@
private final CQL3Type.Raw keys;
private final CQL3Type.Raw values;
- private RawCollection(CollectionType.Kind kind, CQL3Type.Raw keys, CQL3Type.Raw values)
+ private RawCollection(CollectionType.Kind kind, CQL3Type.Raw keys, CQL3Type.Raw values, boolean frozen)
{
+ super(frozen);
this.kind = kind;
this.keys = keys;
this.values = values;
}
- public void freeze() throws InvalidRequestException
+ @Override
+ public RawCollection freeze()
{
- if (keys != null && keys.supportsFreezing())
- keys.freeze();
- if (values != null && values.supportsFreezing())
- values.freeze();
- frozen = true;
+ CQL3Type.Raw frozenKeys =
+ null != keys && keys.supportsFreezing()
+ ? keys.freeze()
+ : keys;
+
+ CQL3Type.Raw frozenValues =
+ null != values && values.supportsFreezing()
+ ? values.freeze()
+ : values;
+
+ return new RawCollection(kind, frozenKeys, frozenValues, true);
}
public boolean supportsFreezing()
@@ -663,7 +678,7 @@
if (!frozen && values.supportsFreezing() && !values.frozen)
throwNestedNonFrozenError(values);
- // we represent Thrift supercolumns as maps, internally, and we do allow counters in supercolumns. Thus,
+ // we represent supercolumns as maps, internally, and we do allow counters in supercolumns. Thus,
// for internal type parsing (think schema) we have to make an exception and allow counters as (map) values
if (values.isCounter() && !isInternal)
throw new InvalidRequestException("Counters are not allowed inside collections: " + this);
@@ -729,8 +744,9 @@
{
private final UTName name;
- private RawUT(UTName name)
+ private RawUT(UTName name, boolean frozen)
{
+ super(frozen);
this.name = name;
}
@@ -739,14 +755,10 @@
return name.getKeyspace();
}
- public void freeze()
+ @Override
+ public RawUT freeze()
{
- frozen = true;
- }
-
- public boolean canBeNonFrozen()
- {
- return true;
+ return new RawUT(name, true);
}
public CQL3Type prepare(String keyspace, Types udts) throws InvalidRequestException
@@ -803,8 +815,9 @@
{
private final List<CQL3Type.Raw> types;
- private RawTuple(List<CQL3Type.Raw> types)
+ private RawTuple(List<CQL3Type.Raw> types, boolean frozen)
{
+ super(frozen);
this.types = types;
}
@@ -813,22 +826,22 @@
return true;
}
- public void freeze() throws InvalidRequestException
+ @Override
+ public RawTuple freeze()
{
- for (CQL3Type.Raw t : types)
- if (t.supportsFreezing())
- t.freeze();
-
- frozen = true;
+ List<CQL3Type.Raw> frozenTypes =
+ types.stream()
+ .map(t -> t.supportsFreezing() ? t.freeze() : t)
+ .collect(toList());
+ return new RawTuple(frozenTypes, true);
}
public CQL3Type prepare(String keyspace, Types udts) throws InvalidRequestException
{
- if (!frozen)
- freeze();
+ RawTuple raw = frozen ? this : freeze();
- List<AbstractType<?>> ts = new ArrayList<>(types.size());
- for (CQL3Type.Raw t : types)
+ List<AbstractType<?>> ts = new ArrayList<>(raw.types.size());
+ for (CQL3Type.Raw t : raw.types)
{
if (t.isCounter())
throw new InvalidRequestException("Counters are not allowed inside tuples");
diff --git a/src/java/org/apache/cassandra/cql3/CQLStatement.java b/src/java/org/apache/cassandra/cql3/CQLStatement.java
index 901ecd4..c34e27f 100644
--- a/src/java/org/apache/cassandra/cql3/CQLStatement.java
+++ b/src/java/org/apache/cassandra/cql3/CQLStatement.java
@@ -17,8 +17,11 @@
*/
package org.apache.cassandra.cql3;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.cassandra.audit.AuditLogContext;
import org.apache.cassandra.cql3.functions.Function;
-import org.apache.cassandra.exceptions.*;
import org.apache.cassandra.service.ClientState;
import org.apache.cassandra.service.QueryState;
import org.apache.cassandra.transport.messages.ResultMessage;
@@ -26,24 +29,46 @@
public interface CQLStatement
{
/**
- * Returns the number of bound terms in this statement.
+ * Returns all bind variables for the statement
*/
- public int getBoundTerms();
+ default List<ColumnSpecification> getBindVariables()
+ {
+ return Collections.emptyList();
+ }
+
+ /**
+ * Returns an array with the same length as the number of partition key columns for the table corresponding
+ * to table. Each short in the array represents the bind index of the marker that holds the value for that
+ * partition key column. If there are no bind markers for any of the partition key columns, null is returned.
+ */
+ default short[] getPartitionKeyBindVariableIndexes()
+ {
+ return null;
+ }
+
+ /**
+ * Return an Iterable over all of the functions (both native and user-defined) used by any component of the statement
+ *
+ * @return functions all functions found (may contain duplicates)
+ */
+ default Iterable<Function> getFunctions()
+ {
+ return Collections.emptyList();
+ }
/**
* Perform any access verification necessary for the statement.
*
* @param state the current client state
*/
- public void checkAccess(ClientState state) throws UnauthorizedException, InvalidRequestException;
+ public void authorize(ClientState state);
/**
- * Perform additional validation required by the statment.
- * To be overriden by subclasses if needed.
+ * Perform additional validation required by the statment. To be overriden by subclasses if needed.
*
* @param state the current client state
*/
- public void validate(ClientState state) throws RequestValidationException;
+ public void validate(ClientState state);
/**
* Execute the statement and return the resulting result or null if there is no result.
@@ -52,19 +77,37 @@
* @param options options for this query (consistency, variables, pageSize, ...)
* @param queryStartNanoTime the timestamp returned by System.nanoTime() when this statement was received
*/
- public ResultMessage execute(QueryState state, QueryOptions options, long queryStartNanoTime) throws RequestValidationException, RequestExecutionException;
+ public ResultMessage execute(QueryState state, QueryOptions options, long queryStartNanoTime);
/**
* Variant of execute used for internal query against the system tables, and thus only query the local node.
*
* @param state the current query state
*/
- public ResultMessage executeInternal(QueryState state, QueryOptions options) throws RequestValidationException, RequestExecutionException;
+ public ResultMessage executeLocally(QueryState state, QueryOptions options);
/**
- * Return an Iterable over all of the functions (both native and user-defined) used by any component
- * of the statement
- * @return functions all functions found (may contain duplicates)
+ * Provides the context needed for audit logging statements.
*/
- public Iterable<Function> getFunctions();
+ AuditLogContext getAuditLogContext();
+
+ /**
+ * Whether or not this CQL Statement has LWT conditions
+ */
+ default public boolean hasConditions()
+ {
+ return false;
+ }
+
+ public static abstract class Raw
+ {
+ protected VariableSpecifications bindVariables;
+
+ public void setBindVariables(List<ColumnIdentifier> variables)
+ {
+ bindVariables = new VariableSpecifications(variables);
+ }
+
+ public abstract CQLStatement prepare(ClientState state);
+ }
}
diff --git a/src/java/org/apache/cassandra/cql3/ColumnCondition.java b/src/java/org/apache/cassandra/cql3/ColumnCondition.java
deleted file mode 100644
index 43b0135..0000000
--- a/src/java/org/apache/cassandra/cql3/ColumnCondition.java
+++ /dev/null
@@ -1,1124 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.cql3;
-
-import java.nio.ByteBuffer;
-import java.util.*;
-
-import com.google.common.collect.Iterators;
-
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
-import org.apache.cassandra.cql3.Term.Terminal;
-import org.apache.cassandra.cql3.functions.Function;
-import org.apache.cassandra.cql3.statements.RequestValidations;
-import org.apache.cassandra.db.rows.*;
-import org.apache.cassandra.db.marshal.*;
-import org.apache.cassandra.exceptions.InvalidRequestException;
-import org.apache.cassandra.transport.ProtocolVersion;
-import org.apache.cassandra.utils.ByteBufferUtil;
-
-import static org.apache.cassandra.cql3.statements.RequestValidations.checkFalse;
-import static org.apache.cassandra.cql3.statements.RequestValidations.invalidRequest;
-
-/**
- * A CQL3 condition on the value of a column or collection element. For example, "UPDATE .. IF a = 0".
- */
-public class ColumnCondition
-{
- public final ColumnDefinition column;
-
- // For collection, when testing the equality of a specific element, null otherwise.
- private final Term collectionElement;
-
- // For UDT, when testing the equality of a specific field, null otherwise.
- private final FieldIdentifier field;
-
- private final Term value; // a single value or a marker for a list of IN values
- private final List<Term> inValues;
-
- public final Operator operator;
-
- private ColumnCondition(ColumnDefinition column, Term collectionElement, FieldIdentifier field, Term value, List<Term> inValues, Operator op)
- {
- this.column = column;
- this.collectionElement = collectionElement;
- this.field = field;
- this.value = value;
- this.inValues = inValues;
- this.operator = op;
-
- assert field == null || collectionElement == null;
- if (operator != Operator.IN)
- assert this.inValues == null;
- }
-
- // Public for SuperColumn tables support only
- public Term value()
- {
- return value;
- }
-
- public static ColumnCondition condition(ColumnDefinition column, Term value, Operator op)
- {
- return new ColumnCondition(column, null, null, value, null, op);
- }
-
- public static ColumnCondition condition(ColumnDefinition column, Term collectionElement, Term value, Operator op)
- {
- return new ColumnCondition(column, collectionElement, null, value, null, op);
- }
-
- public static ColumnCondition condition(ColumnDefinition column, FieldIdentifier udtField, Term value, Operator op)
- {
- return new ColumnCondition(column, null, udtField, value, null, op);
- }
-
- public static ColumnCondition inCondition(ColumnDefinition column, List<Term> inValues)
- {
- return new ColumnCondition(column, null, null, null, inValues, Operator.IN);
- }
-
- public static ColumnCondition inCondition(ColumnDefinition column, Term collectionElement, List<Term> inValues)
- {
- return new ColumnCondition(column, collectionElement, null, null, inValues, Operator.IN);
- }
-
- public static ColumnCondition inCondition(ColumnDefinition column, FieldIdentifier udtField, List<Term> inValues)
- {
- return new ColumnCondition(column, null, udtField, null, inValues, Operator.IN);
- }
-
- public static ColumnCondition inCondition(ColumnDefinition column, Term inMarker)
- {
- return new ColumnCondition(column, null, null, inMarker, null, Operator.IN);
- }
-
- public static ColumnCondition inCondition(ColumnDefinition column, Term collectionElement, Term inMarker)
- {
- return new ColumnCondition(column, collectionElement, null, inMarker, null, Operator.IN);
- }
-
- public static ColumnCondition inCondition(ColumnDefinition column, FieldIdentifier udtField, Term inMarker)
- {
- return new ColumnCondition(column, null, udtField, inMarker, null, Operator.IN);
- }
-
- public void addFunctionsTo(List<Function> functions)
- {
- if (collectionElement != null)
- collectionElement.addFunctionsTo(functions);
- if (value != null)
- value.addFunctionsTo(functions);
- if (inValues != null)
- for (Term value : inValues)
- if (value != null)
- value.addFunctionsTo(functions);
- }
-
- /**
- * Collects the column specification for the bind variables of this operation.
- *
- * @param boundNames the list of column specification where to collect the
- * bind variables of this term in.
- */
- public void collectMarkerSpecification(VariableSpecifications boundNames)
- {
- if (collectionElement != null)
- collectionElement.collectMarkerSpecification(boundNames);
-
- if ((operator == Operator.IN) && inValues != null)
- {
- for (Term value : inValues)
- value.collectMarkerSpecification(boundNames);
- }
- else
- {
- value.collectMarkerSpecification(boundNames);
- }
- }
-
- public ColumnCondition.Bound bind(QueryOptions options) throws InvalidRequestException
- {
- boolean isInCondition = operator == Operator.IN;
- if (column.type instanceof CollectionType)
- {
- if (collectionElement != null)
- return isInCondition ? new ElementAccessInBound(this, options) : new ElementAccessBound(this, options);
- else
- return isInCondition ? new CollectionInBound(this, options) : new CollectionBound(this, options);
- }
- else if (column.type.isUDT())
- {
- if (field != null)
- return isInCondition ? new UDTFieldAccessInBound(this, options) : new UDTFieldAccessBound(this, options);
- else
- return isInCondition ? new UDTInBound(this, options) : new UDTBound(this, options);
- }
-
- return isInCondition ? new SimpleInBound(this, options) : new SimpleBound(this, options);
- }
-
- public static abstract class Bound
- {
- public final ColumnDefinition column;
- public final Operator operator;
-
- protected Bound(ColumnDefinition column, Operator operator)
- {
- this.column = column;
- this.operator = operator;
- }
-
- /**
- * Validates whether this condition applies to {@code current}.
- */
- public abstract boolean appliesTo(Row row) throws InvalidRequestException;
-
- public ByteBuffer getCollectionElementValue()
- {
- return null;
- }
-
- protected boolean isSatisfiedByValue(ByteBuffer value, Cell c, AbstractType<?> type, Operator operator) throws InvalidRequestException
- {
- return compareWithOperator(operator, type, value, c == null ? null : c.value());
- }
-
- /** Returns true if the operator is satisfied (i.e. "otherValue operator value == true"), false otherwise. */
- protected boolean compareWithOperator(Operator operator, AbstractType<?> type, ByteBuffer value, ByteBuffer otherValue) throws InvalidRequestException
- {
- if (value == ByteBufferUtil.UNSET_BYTE_BUFFER)
- throw new InvalidRequestException("Invalid 'unset' value in condition");
- if (value == null)
- {
- switch (operator)
- {
- case EQ:
- return otherValue == null;
- case NEQ:
- return otherValue != null;
- default:
- throw new InvalidRequestException(String.format("Invalid comparison with null for operator \"%s\"", operator));
- }
- }
- else if (otherValue == null)
- {
- // the condition value is not null, so only NEQ can return true
- return operator == Operator.NEQ;
- }
- return operator.isSatisfiedBy(type, otherValue, value);
- }
- }
-
- private static Cell getCell(Row row, ColumnDefinition column)
- {
- // If we're asking for a given cell, and we didn't got any row from our read, it's
- // the same as not having said cell.
- return row == null ? null : row.getCell(column);
- }
-
- private static Cell getCell(Row row, ColumnDefinition column, CellPath path)
- {
- // If we're asking for a given cell, and we didn't got any row from our read, it's
- // the same as not having said cell.
- return row == null ? null : row.getCell(column, path);
- }
-
- private static Iterator<Cell> getCells(Row row, ColumnDefinition column)
- {
- // If we're asking for a complex cells, and we didn't got any row from our read, it's
- // the same as not having any cells for that column.
- if (row == null)
- return Collections.<Cell>emptyIterator();
-
- ComplexColumnData complexData = row.getComplexColumnData(column);
- return complexData == null ? Collections.<Cell>emptyIterator() : complexData.iterator();
- }
-
- private static boolean evaluateComparisonWithOperator(int comparison, Operator operator)
- {
- // called when comparison != 0
- switch (operator)
- {
- case EQ:
- return false;
- case LT:
- case LTE:
- return comparison < 0;
- case GT:
- case GTE:
- return comparison > 0;
- case NEQ:
- return true;
- default:
- throw new AssertionError();
- }
- }
-
- private static ByteBuffer cellValueAtIndex(Iterator<Cell> iter, int index)
- {
- int adv = Iterators.advance(iter, index);
- if (adv == index && iter.hasNext())
- return iter.next().value();
- else
- return null;
- }
-
- /**
- * A condition on a single non-collection column. This does not support IN operators (see SimpleInBound).
- */
- static class SimpleBound extends Bound
- {
- public final ByteBuffer value;
-
- private SimpleBound(ColumnCondition condition, QueryOptions options) throws InvalidRequestException
- {
- super(condition.column, condition.operator);
- assert !(column.type instanceof CollectionType) && condition.field == null;
- assert condition.operator != Operator.IN;
- this.value = condition.value.bindAndGet(options);
- }
-
- public boolean appliesTo(Row row) throws InvalidRequestException
- {
- return isSatisfiedByValue(value, getCell(row, column), column.type, operator);
- }
- }
-
- /**
- * An IN condition on a single non-collection column.
- */
- static class SimpleInBound extends Bound
- {
- public final List<ByteBuffer> inValues;
-
- private SimpleInBound(ColumnCondition condition, QueryOptions options) throws InvalidRequestException
- {
- super(condition.column, condition.operator);
- assert !(column.type instanceof CollectionType) && condition.field == null;
- assert condition.operator == Operator.IN;
- if (condition.inValues == null)
- {
- Terminal terminal = condition.value.bind(options);
-
- if (terminal == null)
- throw new InvalidRequestException("Invalid null list in IN condition");
-
- if (terminal == Constants.UNSET_VALUE)
- throw new InvalidRequestException("Invalid 'unset' value in condition");
-
- this.inValues = ((Lists.Value) terminal).getElements();
- }
- else
- {
- this.inValues = new ArrayList<>(condition.inValues.size());
- for (Term value : condition.inValues)
- {
- ByteBuffer buffer = value.bindAndGet(options);
- if (buffer != ByteBufferUtil.UNSET_BYTE_BUFFER)
- this.inValues.add(value.bindAndGet(options));
- }
- }
- }
-
- public boolean appliesTo(Row row) throws InvalidRequestException
- {
- Cell c = getCell(row, column);
- for (ByteBuffer value : inValues)
- {
- if (isSatisfiedByValue(value, c, column.type, Operator.EQ))
- return true;
- }
- return false;
- }
- }
-
- /** A condition on an element of a collection column. IN operators are not supported here, see ElementAccessInBound. */
- static class ElementAccessBound extends Bound
- {
- public final ByteBuffer collectionElement;
- public final ByteBuffer value;
-
- private ElementAccessBound(ColumnCondition condition, QueryOptions options) throws InvalidRequestException
- {
- super(condition.column, condition.operator);
- assert column.type instanceof CollectionType && condition.collectionElement != null;
- assert condition.operator != Operator.IN;
- this.collectionElement = condition.collectionElement.bindAndGet(options);
- this.value = condition.value.bindAndGet(options);
- }
-
- public boolean appliesTo(Row row) throws InvalidRequestException
- {
- if (collectionElement == null)
- throw new InvalidRequestException("Invalid null value for " + (column.type instanceof MapType ? "map" : "list") + " element access");
-
- if (column.type instanceof MapType)
- {
- MapType mapType = (MapType) column.type;
- if (column.type.isMultiCell())
- {
- Cell cell = getCell(row, column, CellPath.create(collectionElement));
- return isSatisfiedByValue(value, cell, ((MapType) column.type).getValuesType(), operator);
- }
- else
- {
- Cell cell = getCell(row, column);
- ByteBuffer mapElementValue = mapType.getSerializer().getSerializedValue(cell.value(), collectionElement, mapType.getKeysType());
- return compareWithOperator(operator, mapType.getValuesType(), value, mapElementValue);
- }
- }
-
- // sets don't have element access, so it's a list
- ListType listType = (ListType) column.type;
- if (column.type.isMultiCell())
- {
- ByteBuffer columnValue = cellValueAtIndex(getCells(row, column), getListIndex(collectionElement));
- return compareWithOperator(operator, ((ListType)column.type).getElementsType(), value, columnValue);
- }
- else
- {
- Cell cell = getCell(row, column);
- ByteBuffer listElementValue = listType.getSerializer().getElement(cell.value(), getListIndex(collectionElement));
- return compareWithOperator(operator, listType.getElementsType(), value, listElementValue);
- }
- }
-
- static int getListIndex(ByteBuffer collectionElement) throws InvalidRequestException
- {
- int idx = ByteBufferUtil.toInt(collectionElement);
- if (idx < 0)
- throw new InvalidRequestException(String.format("Invalid negative list index %d", idx));
- return idx;
- }
-
- public ByteBuffer getCollectionElementValue()
- {
- return collectionElement;
- }
- }
-
- static class ElementAccessInBound extends Bound
- {
- public final ByteBuffer collectionElement;
- public final List<ByteBuffer> inValues;
-
- private ElementAccessInBound(ColumnCondition condition, QueryOptions options) throws InvalidRequestException
- {
- super(condition.column, condition.operator);
- assert column.type instanceof CollectionType && condition.collectionElement != null;
- this.collectionElement = condition.collectionElement.bindAndGet(options);
-
- if (condition.inValues == null)
- {
- Terminal terminal = condition.value.bind(options);
- if (terminal == Constants.UNSET_VALUE)
- throw new InvalidRequestException("Invalid 'unset' value in condition");
- this.inValues = ((Lists.Value) terminal).getElements();
- }
- else
- {
- this.inValues = new ArrayList<>(condition.inValues.size());
- for (Term value : condition.inValues)
- {
- ByteBuffer buffer = value.bindAndGet(options);
- // We want to ignore unset values
- if (buffer != ByteBufferUtil.UNSET_BYTE_BUFFER)
- this.inValues.add(buffer);
- }
- }
- }
-
- public boolean appliesTo(Row row) throws InvalidRequestException
- {
- if (collectionElement == null)
- throw new InvalidRequestException("Invalid null value for " + (column.type instanceof MapType ? "map" : "list") + " element access");
-
- ByteBuffer cellValue;
- AbstractType<?> valueType;
- if (column.type instanceof MapType)
- {
- MapType mapType = (MapType) column.type;
- valueType = mapType.getValuesType();
- if (column.type.isMultiCell())
- {
- Cell cell = getCell(row, column, CellPath.create(collectionElement));
- cellValue = cell == null ? null : cell.value();
- }
- else
- {
- Cell cell = getCell(row, column);
- cellValue = cell == null
- ? null
- : mapType.getSerializer().getSerializedValue(cell.value(), collectionElement, mapType.getKeysType());
- }
- }
- else // ListType
- {
- ListType listType = (ListType) column.type;
- valueType = listType.getElementsType();
- if (column.type.isMultiCell())
- {
- cellValue = cellValueAtIndex(getCells(row, column), ElementAccessBound.getListIndex(collectionElement));
- }
- else
- {
- Cell cell = getCell(row, column);
- cellValue = cell == null
- ? null
- : listType.getSerializer().getElement(cell.value(), ElementAccessBound.getListIndex(collectionElement));
- }
- }
-
- for (ByteBuffer value : inValues)
- {
- if (compareWithOperator(Operator.EQ, valueType, value, cellValue))
- return true;
- }
- return false;
- }
- }
-
- /** A condition on an entire collection column. IN operators are not supported here, see CollectionInBound. */
- static class CollectionBound extends Bound
- {
- private final Term.Terminal value;
-
- private CollectionBound(ColumnCondition condition, QueryOptions options) throws InvalidRequestException
- {
- super(condition.column, condition.operator);
- assert column.type.isCollection() && condition.collectionElement == null;
- assert condition.operator != Operator.IN;
- this.value = condition.value.bind(options);
- }
-
- public boolean appliesTo(Row row) throws InvalidRequestException
- {
- CollectionType type = (CollectionType)column.type;
-
- if (type.isMultiCell())
- {
- Iterator<Cell> iter = getCells(row, column);
- if (value == null)
- {
- if (operator == Operator.EQ)
- return !iter.hasNext();
- else if (operator == Operator.NEQ)
- return iter.hasNext();
- else
- throw new InvalidRequestException(String.format("Invalid comparison with null for operator \"%s\"", operator));
- }
-
- return valueAppliesTo(type, iter, value, operator);
- }
-
- // frozen collections
- Cell cell = getCell(row, column);
- if (value == null)
- {
- if (operator == Operator.EQ)
- return cell == null;
- else if (operator == Operator.NEQ)
- return cell != null;
- else
- throw new InvalidRequestException(String.format("Invalid comparison with null for operator \"%s\"", operator));
- }
- else if (cell == null) // cell is null but condition has a value
- {
- return false;
- }
-
- // make sure we use v3 serialization format for comparison
- ByteBuffer conditionValue;
- if (type.kind == CollectionType.Kind.LIST)
- conditionValue = ((Lists.Value) value).get(ProtocolVersion.V3);
- else if (type.kind == CollectionType.Kind.SET)
- conditionValue = ((Sets.Value) value).get(ProtocolVersion.V3);
- else
- conditionValue = ((Maps.Value) value).get(ProtocolVersion.V3);
-
- return compareWithOperator(operator, type, conditionValue, cell.value());
- }
-
- static boolean valueAppliesTo(CollectionType type, Iterator<Cell> iter, Term.Terminal value, Operator operator)
- {
- if (value == null)
- return !iter.hasNext();
-
- switch (type.kind)
- {
- case LIST:
- List<ByteBuffer> valueList = ((Lists.Value) value).elements;
- return listAppliesTo((ListType)type, iter, valueList, operator);
- case SET:
- Set<ByteBuffer> valueSet = ((Sets.Value) value).elements;
- return setAppliesTo((SetType)type, iter, valueSet, operator);
- case MAP:
- Map<ByteBuffer, ByteBuffer> valueMap = ((Maps.Value) value).map;
- return mapAppliesTo((MapType)type, iter, valueMap, operator);
- }
- throw new AssertionError();
- }
-
- private static boolean setOrListAppliesTo(AbstractType<?> type, Iterator<Cell> iter, Iterator<ByteBuffer> conditionIter, Operator operator, boolean isSet)
- {
- while(iter.hasNext())
- {
- if (!conditionIter.hasNext())
- return (operator == Operator.GT) || (operator == Operator.GTE) || (operator == Operator.NEQ);
-
- // for lists we use the cell value; for sets we use the cell name
- ByteBuffer cellValue = isSet ? iter.next().path().get(0) : iter.next().value();
- int comparison = type.compare(cellValue, conditionIter.next());
- if (comparison != 0)
- return evaluateComparisonWithOperator(comparison, operator);
- }
-
- if (conditionIter.hasNext())
- return (operator == Operator.LT) || (operator == Operator.LTE) || (operator == Operator.NEQ);
-
- // they're equal
- return operator == Operator.EQ || operator == Operator.LTE || operator == Operator.GTE;
- }
-
- static boolean listAppliesTo(ListType type, Iterator<Cell> iter, List<ByteBuffer> elements, Operator operator)
- {
- return setOrListAppliesTo(type.getElementsType(), iter, elements.iterator(), operator, false);
- }
-
- static boolean setAppliesTo(SetType type, Iterator<Cell> iter, Set<ByteBuffer> elements, Operator operator)
- {
- ArrayList<ByteBuffer> sortedElements = new ArrayList<>(elements.size());
- sortedElements.addAll(elements);
- Collections.sort(sortedElements, type.getElementsType());
- return setOrListAppliesTo(type.getElementsType(), iter, sortedElements.iterator(), operator, true);
- }
-
- static boolean mapAppliesTo(MapType type, Iterator<Cell> iter, Map<ByteBuffer, ByteBuffer> elements, Operator operator)
- {
- Iterator<Map.Entry<ByteBuffer, ByteBuffer>> conditionIter = elements.entrySet().iterator();
- while(iter.hasNext())
- {
- if (!conditionIter.hasNext())
- return (operator == Operator.GT) || (operator == Operator.GTE) || (operator == Operator.NEQ);
-
- Map.Entry<ByteBuffer, ByteBuffer> conditionEntry = conditionIter.next();
- Cell c = iter.next();
-
- // compare the keys
- int comparison = type.getKeysType().compare(c.path().get(0), conditionEntry.getKey());
- if (comparison != 0)
- return evaluateComparisonWithOperator(comparison, operator);
-
- // compare the values
- comparison = type.getValuesType().compare(c.value(), conditionEntry.getValue());
- if (comparison != 0)
- return evaluateComparisonWithOperator(comparison, operator);
- }
-
- if (conditionIter.hasNext())
- return (operator == Operator.LT) || (operator == Operator.LTE) || (operator == Operator.NEQ);
-
- // they're equal
- return operator == Operator.EQ || operator == Operator.LTE || operator == Operator.GTE;
- }
- }
-
- public static class CollectionInBound extends Bound
- {
- private final List<Term.Terminal> inValues;
-
- private CollectionInBound(ColumnCondition condition, QueryOptions options) throws InvalidRequestException
- {
- super(condition.column, condition.operator);
- assert column.type instanceof CollectionType && condition.collectionElement == null;
- assert condition.operator == Operator.IN;
- inValues = new ArrayList<>();
- if (condition.inValues == null)
- {
- // We have a list of serialized collections that need to be deserialized for later comparisons
- CollectionType collectionType = (CollectionType) column.type;
- Lists.Marker inValuesMarker = (Lists.Marker) condition.value;
- Terminal terminal = inValuesMarker.bind(options);
-
- if (terminal == null)
- throw new InvalidRequestException("Invalid null list in IN condition");
-
- if (terminal == Constants.UNSET_VALUE)
- throw new InvalidRequestException("Invalid 'unset' value in condition");
-
- if (column.type instanceof ListType)
- {
- ListType deserializer = ListType.getInstance(collectionType.valueComparator(), false);
- for (ByteBuffer buffer : ((Lists.Value) terminal).elements)
- {
- if (buffer == ByteBufferUtil.UNSET_BYTE_BUFFER)
- continue;
-
- if (buffer == null)
- this.inValues.add(null);
- else
- this.inValues.add(Lists.Value.fromSerialized(buffer, deserializer, options.getProtocolVersion()));
- }
- }
- else if (column.type instanceof MapType)
- {
- MapType deserializer = MapType.getInstance(collectionType.nameComparator(), collectionType.valueComparator(), false);
- for (ByteBuffer buffer : ((Lists.Value) terminal).elements)
- {
- if (buffer == ByteBufferUtil.UNSET_BYTE_BUFFER)
- continue;
-
- if (buffer == null)
- this.inValues.add(null);
- else
- this.inValues.add(Maps.Value.fromSerialized(buffer, deserializer, options.getProtocolVersion()));
- }
- }
- else if (column.type instanceof SetType)
- {
- SetType deserializer = SetType.getInstance(collectionType.valueComparator(), false);
- for (ByteBuffer buffer : ((Lists.Value) terminal).elements)
- {
- if (buffer == ByteBufferUtil.UNSET_BYTE_BUFFER)
- continue;
-
- if (buffer == null)
- this.inValues.add(null);
- else
- this.inValues.add(Sets.Value.fromSerialized(buffer, deserializer, options.getProtocolVersion()));
- }
- }
- }
- else
- {
- for (Term value : condition.inValues)
- {
- Terminal terminal = value.bind(options);
- if (terminal != Constants.UNSET_VALUE)
- this.inValues.add(terminal);
- }
- }
- }
-
- public boolean appliesTo(Row row) throws InvalidRequestException
- {
- CollectionType type = (CollectionType)column.type;
- if (type.isMultiCell())
- {
- // copy iterator contents so that we can properly reuse them for each comparison with an IN value
- for (Term.Terminal value : inValues)
- {
- if (CollectionBound.valueAppliesTo(type, getCells(row, column), value, Operator.EQ))
- return true;
- }
- return false;
- }
- else
- {
- Cell cell = getCell(row, column);
- for (Term.Terminal value : inValues)
- {
- if (value == null)
- {
- if (cell == null)
- return true;
- }
- else if (type.compare(value.get(ProtocolVersion.V3), cell.value()) == 0)
- {
- return true;
- }
- }
- return false;
- }
- }
- }
-
- /** A condition on a UDT field. IN operators are not supported here, see UDTFieldAccessInBound. */
- static class UDTFieldAccessBound extends Bound
- {
- public final FieldIdentifier field;
- public final ByteBuffer value;
-
- private UDTFieldAccessBound(ColumnCondition condition, QueryOptions options) throws InvalidRequestException
- {
- super(condition.column, condition.operator);
- assert column.type.isUDT() && condition.field != null;
- assert condition.operator != Operator.IN;
- this.field = condition.field;
- this.value = condition.value.bindAndGet(options);
- }
-
- public boolean appliesTo(Row row) throws InvalidRequestException
- {
- UserType userType = (UserType) column.type;
- int fieldPosition = userType.fieldPosition(field);
- assert fieldPosition >= 0;
-
- ByteBuffer cellValue;
- if (column.type.isMultiCell())
- {
- Cell cell = getCell(row, column, userType.cellPathForField(field));
- cellValue = cell == null ? null : cell.value();
- }
- else
- {
- Cell cell = getCell(row, column);
- cellValue = cell == null
- ? null
- : userType.split(cell.value())[fieldPosition];
- }
- return compareWithOperator(operator, userType.fieldType(fieldPosition), value, cellValue);
- }
- }
-
- /** An IN condition on a UDT field. For example: IF user.name IN ('a', 'b') */
- static class UDTFieldAccessInBound extends Bound
- {
- public final FieldIdentifier field;
- public final List<ByteBuffer> inValues;
-
- private UDTFieldAccessInBound(ColumnCondition condition, QueryOptions options) throws InvalidRequestException
- {
- super(condition.column, condition.operator);
- assert column.type.isUDT() && condition.field != null;
- this.field = condition.field;
-
- if (condition.inValues == null)
- this.inValues = ((Lists.Value) condition.value.bind(options)).getElements();
- else
- {
- this.inValues = new ArrayList<>(condition.inValues.size());
- for (Term value : condition.inValues)
- this.inValues.add(value.bindAndGet(options));
- }
- }
-
- public boolean appliesTo(Row row) throws InvalidRequestException
- {
- UserType userType = (UserType) column.type;
- int fieldPosition = userType.fieldPosition(field);
- assert fieldPosition >= 0;
-
- ByteBuffer cellValue;
- if (column.type.isMultiCell())
- {
- Cell cell = getCell(row, column, userType.cellPathForField(field));
- cellValue = cell == null ? null : cell.value();
- }
- else
- {
- Cell cell = getCell(row, column);
- cellValue = cell == null ? null : userType.split(getCell(row, column).value())[fieldPosition];
- }
-
- AbstractType<?> valueType = userType.fieldType(fieldPosition);
- for (ByteBuffer value : inValues)
- {
- if (compareWithOperator(Operator.EQ, valueType, value, cellValue))
- return true;
- }
- return false;
- }
- }
-
- /** A non-IN condition on an entire UDT. For example: IF user = {name: 'joe', age: 42}). */
- static class UDTBound extends Bound
- {
- private final ByteBuffer value;
- private final ProtocolVersion protocolVersion;
-
- private UDTBound(ColumnCondition condition, QueryOptions options) throws InvalidRequestException
- {
- super(condition.column, condition.operator);
- assert column.type.isUDT() && condition.field == null;
- assert condition.operator != Operator.IN;
- protocolVersion = options.getProtocolVersion();
- value = condition.value.bindAndGet(options);
- }
-
- public boolean appliesTo(Row row) throws InvalidRequestException
- {
- UserType userType = (UserType) column.type;
- ByteBuffer rowValue;
- if (userType.isMultiCell())
- {
- Iterator<Cell> iter = getCells(row, column);
- rowValue = iter.hasNext() ? userType.serializeForNativeProtocol(iter, protocolVersion) : null;
- }
- else
- {
- Cell cell = getCell(row, column);
- rowValue = cell == null ? null : cell.value();
- }
-
- if (value == null)
- {
- if (operator == Operator.EQ)
- return rowValue == null;
- else if (operator == Operator.NEQ)
- return rowValue != null;
- else
- throw new InvalidRequestException(String.format("Invalid comparison with null for operator \"%s\"", operator));
- }
-
- return compareWithOperator(operator, userType, value, rowValue);
- }
- }
-
- /** An IN condition on an entire UDT. For example: IF user IN ({name: 'joe', age: 42}, {name: 'bob', age: 23}). */
- public static class UDTInBound extends Bound
- {
- private final List<ByteBuffer> inValues;
- private final ProtocolVersion protocolVersion;
-
- private UDTInBound(ColumnCondition condition, QueryOptions options) throws InvalidRequestException
- {
- super(condition.column, condition.operator);
- assert column.type.isUDT() && condition.field == null;
- assert condition.operator == Operator.IN;
- protocolVersion = options.getProtocolVersion();
- inValues = new ArrayList<>();
- if (condition.inValues == null)
- {
- Lists.Marker inValuesMarker = (Lists.Marker) condition.value;
- Terminal terminal = inValuesMarker.bind(options);
- if (terminal == null)
- throw new InvalidRequestException("Invalid null list in IN condition");
-
- if (terminal == Constants.UNSET_VALUE)
- throw new InvalidRequestException("Invalid 'unset' value in condition");
-
- for (ByteBuffer buffer : ((Lists.Value)terminal).elements)
- this.inValues.add(buffer);
- }
- else
- {
- for (Term value : condition.inValues)
- this.inValues.add(value.bindAndGet(options));
- }
- }
-
- public boolean appliesTo(Row row) throws InvalidRequestException
- {
- UserType userType = (UserType) column.type;
- ByteBuffer rowValue;
- if (userType.isMultiCell())
- {
- Iterator<Cell> cells = getCells(row, column);
- rowValue = cells.hasNext() ? userType.serializeForNativeProtocol(cells, protocolVersion) : null;
- }
- else
- {
- Cell cell = getCell(row, column);
- rowValue = cell == null ? null : cell.value();
- }
-
- for (ByteBuffer value : inValues)
- {
- if (value == null || rowValue == null)
- {
- if (value == rowValue) // both null
- return true;
- }
- else if (userType.compare(value, rowValue) == 0)
- {
- return true;
- }
- }
- return false;
- }
- }
-
- public static class Raw
- {
- private final Term.Raw value;
- private final List<Term.Raw> inValues;
- private final AbstractMarker.INRaw inMarker;
-
- // Can be null, only used with the syntax "IF m[e] = ..." (in which case it's 'e')
- private final Term.Raw collectionElement;
-
- // Can be null, only used with the syntax "IF udt.field = ..." (in which case it's 'field')
- private final FieldIdentifier udtField;
-
- private final Operator operator;
-
- private Raw(Term.Raw value, List<Term.Raw> inValues, AbstractMarker.INRaw inMarker, Term.Raw collectionElement,
- FieldIdentifier udtField, Operator op)
- {
- this.value = value;
- this.inValues = inValues;
- this.inMarker = inMarker;
- this.collectionElement = collectionElement;
- this.udtField = udtField;
- this.operator = op;
- }
-
- /** A condition on a column. For example: "IF col = 'foo'" */
- public static Raw simpleCondition(Term.Raw value, Operator op)
- {
- return new Raw(value, null, null, null, null, op);
- }
-
- /** An IN condition on a column. For example: "IF col IN ('foo', 'bar', ...)" */
- public static Raw simpleInCondition(List<Term.Raw> inValues)
- {
- return new Raw(null, inValues, null, null, null, Operator.IN);
- }
-
- /** An IN condition on a column with a single marker. For example: "IF col IN ?" */
- public static Raw simpleInCondition(AbstractMarker.INRaw inMarker)
- {
- return new Raw(null, null, inMarker, null, null, Operator.IN);
- }
-
- /** A condition on a collection element. For example: "IF col['key'] = 'foo'" */
- public static Raw collectionCondition(Term.Raw value, Term.Raw collectionElement, Operator op)
- {
- return new Raw(value, null, null, collectionElement, null, op);
- }
-
- /** An IN condition on a collection element. For example: "IF col['key'] IN ('foo', 'bar', ...)" */
- public static Raw collectionInCondition(Term.Raw collectionElement, List<Term.Raw> inValues)
- {
- return new Raw(null, inValues, null, collectionElement, null, Operator.IN);
- }
-
- /** An IN condition on a collection element with a single marker. For example: "IF col['key'] IN ?" */
- public static Raw collectionInCondition(Term.Raw collectionElement, AbstractMarker.INRaw inMarker)
- {
- return new Raw(null, null, inMarker, collectionElement, null, Operator.IN);
- }
-
- /** A condition on a UDT field. For example: "IF col.field = 'foo'" */
- public static Raw udtFieldCondition(Term.Raw value, FieldIdentifier udtField, Operator op)
- {
- return new Raw(value, null, null, null, udtField, op);
- }
-
- /** An IN condition on a collection element. For example: "IF col.field IN ('foo', 'bar', ...)" */
- public static Raw udtFieldInCondition(FieldIdentifier udtField, List<Term.Raw> inValues)
- {
- return new Raw(null, inValues, null, null, udtField, Operator.IN);
- }
-
- /** An IN condition on a collection element with a single marker. For example: "IF col.field IN ?" */
- public static Raw udtFieldInCondition(FieldIdentifier udtField, AbstractMarker.INRaw inMarker)
- {
- return new Raw(null, null, inMarker, null, udtField, Operator.IN);
- }
-
- public ColumnCondition prepare(String keyspace, ColumnDefinition receiver, CFMetaData cfm) throws InvalidRequestException
- {
- if (receiver.type instanceof CounterColumnType)
- throw new InvalidRequestException("Conditions on counters are not supported");
-
- if (collectionElement != null)
- {
- if (!(receiver.type.isCollection()))
- throw new InvalidRequestException(String.format("Invalid element access syntax for non-collection column %s", receiver.name));
-
- ColumnSpecification elementSpec, valueSpec;
- switch ((((CollectionType) receiver.type).kind))
- {
- case LIST:
- elementSpec = Lists.indexSpecOf(receiver);
- valueSpec = Lists.valueSpecOf(receiver);
- break;
- case MAP:
- elementSpec = Maps.keySpecOf(receiver);
- valueSpec = Maps.valueSpecOf(receiver);
- break;
- case SET:
- throw new InvalidRequestException(String.format("Invalid element access syntax for set column %s", receiver.name));
- default:
- throw new AssertionError();
- }
-
- if (operator == Operator.IN)
- {
- if (inValues == null)
- return ColumnCondition.inCondition(receiver, collectionElement.prepare(keyspace, elementSpec), inMarker.prepare(keyspace, valueSpec));
- List<Term> terms = new ArrayList<>(inValues.size());
- for (Term.Raw value : inValues)
- terms.add(value.prepare(keyspace, valueSpec));
- return ColumnCondition.inCondition(receiver, collectionElement.prepare(keyspace, elementSpec), terms);
- }
- else
- {
- validateOperationOnDurations(valueSpec.type);
- return ColumnCondition.condition(receiver, collectionElement.prepare(keyspace, elementSpec), value.prepare(keyspace, valueSpec), operator);
- }
- }
- else if (udtField != null)
- {
- UserType userType = (UserType) receiver.type;
- int fieldPosition = userType.fieldPosition(udtField);
- if (fieldPosition == -1)
- throw new InvalidRequestException(String.format("Unknown field %s for column %s", udtField, receiver.name));
-
- ColumnSpecification fieldReceiver = UserTypes.fieldSpecOf(receiver, fieldPosition);
- if (operator == Operator.IN)
- {
- if (inValues == null)
- return ColumnCondition.inCondition(receiver, udtField, inMarker.prepare(keyspace, fieldReceiver));
-
- List<Term> terms = new ArrayList<>(inValues.size());
- for (Term.Raw value : inValues)
- terms.add(value.prepare(keyspace, fieldReceiver));
- return ColumnCondition.inCondition(receiver, udtField, terms);
- }
- else
- {
- validateOperationOnDurations(fieldReceiver.type);
- return ColumnCondition.condition(receiver, udtField, value.prepare(keyspace, fieldReceiver), operator);
- }
- }
- else
- {
- if (operator == Operator.IN)
- {
- if (inValues == null)
- return ColumnCondition.inCondition(receiver, inMarker.prepare(keyspace, receiver));
- List<Term> terms = new ArrayList<>(inValues.size());
- for (Term.Raw value : inValues)
- terms.add(value.prepare(keyspace, receiver));
- return ColumnCondition.inCondition(receiver, terms);
- }
- else
- {
- validateOperationOnDurations(receiver.type);
- return ColumnCondition.condition(receiver, value.prepare(keyspace, receiver), operator);
- }
- }
- }
-
- private void validateOperationOnDurations(AbstractType<?> type)
- {
- if (type.referencesDuration() && operator.isSlice())
- {
- checkFalse(type.isCollection(), "Slice conditions are not supported on collections containing durations");
- checkFalse(type.isTuple(), "Slice conditions are not supported on tuples containing durations");
- checkFalse(type.isUDT(), "Slice conditions are not supported on UDTs containing durations");
- throw invalidRequest("Slice conditions are not supported on durations", operator);
- }
- }
- }
-}
diff --git a/src/java/org/apache/cassandra/cql3/ColumnConditions.java b/src/java/org/apache/cassandra/cql3/ColumnConditions.java
deleted file mode 100644
index 5ec4cb4..0000000
--- a/src/java/org/apache/cassandra/cql3/ColumnConditions.java
+++ /dev/null
@@ -1,170 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.cql3;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-import org.apache.cassandra.config.ColumnDefinition;
-import org.apache.cassandra.cql3.functions.Function;
-import org.apache.cassandra.cql3.statements.CQL3CasRequest;
-import org.apache.cassandra.db.Clustering;
-
-/**
- * A set of <code>ColumnCondition</code>s.
- *
- */
-public final class ColumnConditions extends AbstractConditions
-{
- /**
- * The conditions on regular columns.
- */
- private final List<ColumnCondition> columnConditions;
-
- /**
- * The conditions on static columns
- */
- private final List<ColumnCondition> staticConditions;
-
- /**
- * Creates a new <code>ColumnConditions</code> instance for the specified builder.
- */
- private ColumnConditions(Builder builder)
- {
- this.columnConditions = builder.columnConditions;
- this.staticConditions = builder.staticConditions;
- }
-
- @Override
- public boolean appliesToStaticColumns()
- {
- return !staticConditions.isEmpty();
- }
-
- @Override
- public boolean appliesToRegularColumns()
- {
- return !columnConditions.isEmpty();
- }
-
- @Override
- public Collection<ColumnDefinition> getColumns()
- {
- return Stream.concat(columnConditions.stream(), staticConditions.stream())
- .map(e -> e.column)
- .collect(Collectors.toList());
- }
-
- @Override
- public boolean isEmpty()
- {
- return columnConditions.isEmpty() && staticConditions.isEmpty();
- }
-
- /**
- * Adds the conditions to the specified CAS request.
- *
- * @param request the request
- * @param clustering the clustering prefix
- * @param options the query options
- */
- public void addConditionsTo(CQL3CasRequest request,
- Clustering clustering,
- QueryOptions options)
- {
- if (!columnConditions.isEmpty())
- request.addConditions(clustering, columnConditions, options);
- if (!staticConditions.isEmpty())
- request.addConditions(Clustering.STATIC_CLUSTERING, staticConditions, options);
- }
-
- @Override
- public void addFunctionsTo(List<Function> functions)
- {
- columnConditions.forEach(p -> p.addFunctionsTo(functions));
- staticConditions.forEach(p -> p.addFunctionsTo(functions));
- }
-
- // Public for SuperColumn tables support only
- public Collection<ColumnCondition> columnConditions()
- {
- return this.columnConditions;
- }
-
- /**
- * Creates a new <code>Builder</code> for <code>ColumnConditions</code>.
- * @return a new <code>Builder</code> for <code>ColumnConditions</code>
- */
- public static Builder newBuilder()
- {
- return new Builder();
- }
-
- /**
- * A <code>Builder</code> for <code>ColumnConditions</code>.
- *
- */
- public static final class Builder
- {
- /**
- * The conditions on regular columns.
- */
- private List<ColumnCondition> columnConditions = Collections.emptyList();
-
- /**
- * The conditions on static columns
- */
- private List<ColumnCondition> staticConditions = Collections.emptyList();
-
- /**
- * Adds the specified <code>ColumnCondition</code> to this set of conditions.
- * @param condition the condition to add
- */
- public Builder add(ColumnCondition condition)
- {
- List<ColumnCondition> conds = null;
- if (condition.column.isStatic())
- {
- if (staticConditions.isEmpty())
- staticConditions = new ArrayList<>();
- conds = staticConditions;
- }
- else
- {
- if (columnConditions.isEmpty())
- columnConditions = new ArrayList<>();
- conds = columnConditions;
- }
- conds.add(condition);
- return this;
- }
-
- public ColumnConditions build()
- {
- return new ColumnConditions(this);
- }
-
- private Builder()
- {
- }
- }
-}
diff --git a/src/java/org/apache/cassandra/cql3/ColumnIdentifier.java b/src/java/org/apache/cassandra/cql3/ColumnIdentifier.java
index ecf44e8..e7bf7b9 100644
--- a/src/java/org/apache/cassandra/cql3/ColumnIdentifier.java
+++ b/src/java/org/apache/cassandra/cql3/ColumnIdentifier.java
@@ -23,7 +23,6 @@
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.MapMaker;
import org.apache.cassandra.cache.IMeasurableMemory;
@@ -41,7 +40,7 @@
{
private static final Pattern PATTERN_DOUBLE_QUOTE = Pattern.compile("\"", Pattern.LITERAL);
private static final String ESCAPED_DOUBLE_QUOTE = Matcher.quoteReplacement("\"\"");
-
+
public final ByteBuffer bytes;
private final String text;
/**
@@ -225,7 +224,6 @@
return ByteBufferUtil.compareUnsigned(this.bytes, that.bytes);
}
- @VisibleForTesting
public static String maybeQuote(String text)
{
if (UNQUOTED_IDENTIFIER.matcher(text).matches() && !ReservedKeywords.isReserved(text))
diff --git a/src/java/org/apache/cassandra/cql3/Conditions.java b/src/java/org/apache/cassandra/cql3/Conditions.java
deleted file mode 100644
index 16fa4aa..0000000
--- a/src/java/org/apache/cassandra/cql3/Conditions.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.cql3;
-
-import java.util.List;
-
-import org.apache.cassandra.config.ColumnDefinition;
-import org.apache.cassandra.cql3.functions.Function;
-import org.apache.cassandra.cql3.statements.CQL3CasRequest;
-import org.apache.cassandra.db.Clustering;
-
-/**
- * Conditions that can be applied to a mutation statement.
- *
- */
-public interface Conditions
-{
- /**
- * An EMPTY condition
- */
- static final Conditions EMPTY_CONDITION = ColumnConditions.newBuilder().build();
-
- /**
- * IF EXISTS condition
- */
- static final Conditions IF_EXISTS_CONDITION = new IfExistsCondition();
-
- /**
- * IF NOT EXISTS condition
- */
- static final Conditions IF_NOT_EXISTS_CONDITION = new IfNotExistsCondition();
-
- /**
- * Adds the functions used by the conditions to the specified list.
- * @param functions the list to add to
- */
- void addFunctionsTo(List<Function> functions);
-
- /**
- * Returns the column definitions to which apply the conditions.
- * @return the column definitions to which apply the conditions.
- */
- Iterable<ColumnDefinition> getColumns();
-
- /**
- * Checks if this <code>Conditions</code> is empty.
- * @return <code>true</code> if this <code>Conditions</code> is empty, <code>false</code> otherwise.
- */
- boolean isEmpty();
-
- /**
- * Checks if this is a IF EXIST condition.
- * @return <code>true</code> if this is a IF EXIST condition, <code>false</code> otherwise.
- */
- boolean isIfExists();
-
- /**
- * Checks if this is a IF NOT EXIST condition.
- * @return <code>true</code> if this is a IF NOT EXIST condition, <code>false</code> otherwise.
- */
- boolean isIfNotExists();
-
- /**
- * Checks if some of the conditions apply to static columns.
- *
- * @return <code>true</code> if some of the conditions apply to static columns, <code>false</code> otherwise.
- */
- boolean appliesToStaticColumns();
-
- /**
- * Checks if some of the conditions apply to regular columns.
- *
- * @return <code>true</code> if some of the conditions apply to regular columns, <code>false</code> otherwise.
- */
- boolean appliesToRegularColumns();
-
- /**
- * Adds the conditions to the specified CAS request.
- *
- * @param request the request
- * @param clustering the clustering prefix
- * @param options the query options
- */
- public void addConditionsTo(CQL3CasRequest request,
- Clustering clustering,
- QueryOptions options);
-}
diff --git a/src/java/org/apache/cassandra/cql3/Constants.java b/src/java/org/apache/cassandra/cql3/Constants.java
index d650c44..6dce3a3 100644
--- a/src/java/org/apache/cassandra/cql3/Constants.java
+++ b/src/java/org/apache/cassandra/cql3/Constants.java
@@ -17,11 +17,13 @@
*/
package org.apache.cassandra.cql3;
+import java.math.BigDecimal;
+import java.math.BigInteger;
import java.nio.ByteBuffer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.marshal.*;
import org.apache.cassandra.exceptions.InvalidRequestException;
@@ -38,7 +40,54 @@
public enum Type
{
- STRING, INTEGER, UUID, FLOAT, BOOLEAN, HEX, DURATION;
+ STRING,
+ INTEGER
+ {
+ public AbstractType<?> getPreferedTypeFor(String text)
+ {
+ // We only try to determine the smallest possible type between int, long and BigInteger
+ BigInteger b = new BigInteger(text);
+
+ if (b.equals(BigInteger.valueOf(b.intValue())))
+ return Int32Type.instance;
+
+ if (b.equals(BigInteger.valueOf(b.longValue())))
+ return LongType.instance;
+
+ return IntegerType.instance;
+ }
+ },
+ UUID,
+ FLOAT
+ {
+ public AbstractType<?> getPreferedTypeFor(String text)
+ {
+ if ("NaN".equals(text) || "-NaN".equals(text) || "Infinity".equals(text) || "-Infinity".equals(text))
+ return DoubleType.instance;
+
+ // We only try to determine the smallest possible type between double and BigDecimal
+ BigDecimal b = new BigDecimal(text);
+
+ if (b.compareTo(BigDecimal.valueOf(b.doubleValue())) == 0)
+ return DoubleType.instance;
+
+ return DecimalType.instance;
+ }
+ },
+ BOOLEAN,
+ HEX,
+ DURATION;
+
+ /**
+ * Returns the exact type for the specified text
+ *
+ * @param text the text for which the type must be determined
+ * @return the exact type or {@code null} if it is not known.
+ */
+ public AbstractType<?> getPreferedTypeFor(String text)
+ {
+ return null;
+ }
}
private static class UnsetLiteral extends Term.Raw
@@ -119,12 +168,14 @@
{
private final Type type;
private final String text;
+ private final AbstractType<?> preferedType;
private Literal(Type type, String text)
{
assert type != null && text != null;
this.type = type;
this.text = text;
+ this.preferedType = type.getPreferedTypeFor(text);
}
public static Literal string(String text)
@@ -204,6 +255,11 @@
return AssignmentTestable.TestResult.WEAKLY_ASSIGNABLE;
CQL3Type.Native nt = (CQL3Type.Native)receiverType;
+
+ // If the receiver type match the prefered type we can straight away return an exact match
+ if (nt.getType().equals(preferedType))
+ return AssignmentTestable.TestResult.EXACT_MATCH;
+
switch (type)
{
case STRING:
@@ -333,8 +389,7 @@
public static class Marker extends AbstractMarker
{
- // Constructor is public only for the SuperColumn tables support
- public Marker(int bindIndex, ColumnSpecification receiver)
+ protected Marker(int bindIndex, ColumnSpecification receiver)
{
super(bindIndex, receiver);
assert !receiver.type.isCollection();
@@ -369,7 +424,7 @@
public static class Setter extends Operation
{
- public Setter(ColumnDefinition column, Term t)
+ public Setter(ColumnMetadata column, Term t)
{
super(column, t);
}
@@ -386,7 +441,7 @@
public static class Adder extends Operation
{
- public Adder(ColumnDefinition column, Term t)
+ public Adder(ColumnMetadata column, Term t)
{
super(column, t);
}
@@ -406,7 +461,7 @@
public static class Substracter extends Operation
{
- public Substracter(ColumnDefinition column, Term t)
+ public Substracter(ColumnMetadata column, Term t)
{
super(column, t);
}
@@ -431,7 +486,7 @@
// duplicating this further
public static class Deleter extends Operation
{
- public Deleter(ColumnDefinition column)
+ public Deleter(ColumnMetadata column)
{
super(column, null);
}
diff --git a/src/java/org/apache/cassandra/cql3/CqlBuilder.java b/src/java/org/apache/cassandra/cql3/CqlBuilder.java
new file mode 100644
index 0000000..d3ca1da
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/CqlBuilder.java
@@ -0,0 +1,226 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3;
+
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.TreeMap;
+
+import org.apache.cassandra.cql3.functions.FunctionName;
+import org.apache.cassandra.db.marshal.AbstractType;
+
+/**
+ * Utility class to facilitate the creation of the CQL representation of {@code SchemaElements}.
+ */
+public final class CqlBuilder
+{
+ @FunctionalInterface
+ public static interface Appender<T>
+ {
+ public void appendTo(CqlBuilder builder, T obj);
+ }
+
+ /**
+ * The new line character
+ */
+ private static final char NEW_LINE = '\n';
+
+ private static final String INDENTATION = " ";
+
+ private final StringBuilder builder;
+
+ private int indent;
+
+ private boolean isNewLine = false;
+
+ public CqlBuilder()
+ {
+ this(64);
+ }
+
+ public CqlBuilder(int capacity)
+ {
+ builder = new StringBuilder(capacity);
+ }
+
+ public CqlBuilder append(String str)
+ {
+ indentIfNeeded();
+ builder.append(str);
+ return this;
+ }
+
+ public CqlBuilder appendQuotingIfNeeded(String str)
+ {
+ return append(ColumnIdentifier.maybeQuote(str));
+ }
+
+ public CqlBuilder appendWithSingleQuotes(String str)
+ {
+ indentIfNeeded();
+
+ builder.append('\'')
+ .append(str.replaceAll("'", "''"))
+ .append('\'');
+
+ return this;
+ }
+
+ public CqlBuilder append(char c)
+ {
+ indentIfNeeded();
+ builder.append(c);
+ return this;
+ }
+
+ public CqlBuilder append(boolean b)
+ {
+ indentIfNeeded();
+ builder.append(b);
+ return this;
+ }
+
+ public CqlBuilder append(int i)
+ {
+ indentIfNeeded();
+ builder.append(i);
+ return this;
+ }
+
+ public CqlBuilder append(long l)
+ {
+ indentIfNeeded();
+ builder.append(l);
+ return this;
+ }
+
+ public CqlBuilder append(float f)
+ {
+ indentIfNeeded();
+ builder.append(f);
+ return this;
+ }
+
+ public CqlBuilder append(double d)
+ {
+ indentIfNeeded();
+ builder.append(d);
+ return this;
+ }
+
+ public CqlBuilder newLine()
+ {
+ builder.append(NEW_LINE);
+ isNewLine = true;
+ return this;
+ }
+
+ public CqlBuilder append(AbstractType<?> type)
+ {
+ return append(type.asCQL3Type().toString());
+ }
+
+ public CqlBuilder append(ColumnIdentifier column)
+ {
+ return append(column.toCQLString());
+ }
+
+ public CqlBuilder append(FunctionName name)
+ {
+ name.appendCqlTo(this);
+ return this;
+ }
+
+ public CqlBuilder append(Map<String, String> map)
+ {
+ return append(map, true);
+ }
+
+ public CqlBuilder append(Map<String, String> map, boolean quoteValue)
+ {
+ indentIfNeeded();
+
+ builder.append('{');
+
+ Iterator<Entry<String, String>> iter = new TreeMap<>(map).entrySet()
+ .iterator();
+ while(iter.hasNext())
+ {
+ Entry<String, String> e = iter.next();
+ appendWithSingleQuotes(e.getKey());
+ builder.append(": ");
+ if (quoteValue)
+ appendWithSingleQuotes(e.getValue());
+ else
+ builder.append(e.getValue());
+
+ if (iter.hasNext())
+ builder.append(", ");
+ }
+ builder.append('}');
+ return this;
+ }
+
+ public <T> CqlBuilder appendWithSeparators(Iterable<T> iterable, Appender<T> appender, String separator)
+ {
+ return appendWithSeparators(iterable.iterator(), appender, separator);
+ }
+
+ public <T> CqlBuilder appendWithSeparators(Iterator<T> iter, Appender<T> appender, String separator)
+ {
+ while (iter.hasNext())
+ {
+ appender.appendTo(this, iter.next());
+ if (iter.hasNext())
+ {
+ append(separator);
+ }
+ }
+ return this;
+ }
+
+ public CqlBuilder increaseIndent()
+ {
+ indent++;
+ return this;
+ }
+
+ public CqlBuilder decreaseIndent()
+ {
+ if (indent > 0)
+ indent--;
+
+ return this;
+ }
+
+ private void indentIfNeeded()
+ {
+ if (isNewLine)
+ {
+ for (int i = 0; i < indent; i++)
+ builder.append(INDENTATION);
+ isNewLine = false;
+ }
+ }
+
+ public String toString()
+ {
+ return builder.toString();
+ }
+}
diff --git a/src/java/org/apache/cassandra/cql3/CustomPayloadMirroringQueryHandler.java b/src/java/org/apache/cassandra/cql3/CustomPayloadMirroringQueryHandler.java
index 643c54b..13aa7f5 100644
--- a/src/java/org/apache/cassandra/cql3/CustomPayloadMirroringQueryHandler.java
+++ b/src/java/org/apache/cassandra/cql3/CustomPayloadMirroringQueryHandler.java
@@ -21,7 +21,7 @@
import java.util.Map;
import org.apache.cassandra.cql3.statements.BatchStatement;
-import org.apache.cassandra.cql3.statements.ParsedStatement;
+import org.apache.cassandra.service.ClientState;
import org.apache.cassandra.service.QueryState;
import org.apache.cassandra.transport.messages.ResultMessage;
import org.apache.cassandra.utils.MD5Digest;
@@ -35,34 +35,34 @@
{
static QueryProcessor queryProcessor = QueryProcessor.instance;
- public ResultMessage process(String query,
+ public CQLStatement parse(String query, QueryState state, QueryOptions options)
+ {
+ return queryProcessor.parse(query, state, options);
+ }
+
+ public ResultMessage process(CQLStatement statement,
QueryState state,
QueryOptions options,
Map<String, ByteBuffer> customPayload,
long queryStartNanoTime)
{
- ResultMessage result = queryProcessor.process(query, state, options, customPayload, queryStartNanoTime);
+ ResultMessage result = queryProcessor.process(statement, state, options, customPayload, queryStartNanoTime);
result.setCustomPayload(customPayload);
return result;
}
- public ResultMessage.Prepared prepare(String query, QueryState state, Map<String, ByteBuffer> customPayload)
+ public ResultMessage.Prepared prepare(String query, ClientState clientState, Map<String, ByteBuffer> customPayload)
{
- ResultMessage.Prepared prepared = queryProcessor.prepare(query, state, customPayload);
+ ResultMessage.Prepared prepared = queryProcessor.prepare(query, clientState, customPayload);
prepared.setCustomPayload(customPayload);
return prepared;
}
- public ParsedStatement.Prepared getPrepared(MD5Digest id)
+ public QueryProcessor.Prepared getPrepared(MD5Digest id)
{
return queryProcessor.getPrepared(id);
}
- public ParsedStatement.Prepared getPreparedForThrift(Integer id)
- {
- return queryProcessor.getPreparedForThrift(id);
- }
-
public ResultMessage processPrepared(CQLStatement statement,
QueryState state,
QueryOptions options,
diff --git a/src/java/org/apache/cassandra/cql3/Duration.java b/src/java/org/apache/cassandra/cql3/Duration.java
index 48f8850..520d195 100644
--- a/src/java/org/apache/cassandra/cql3/Duration.java
+++ b/src/java/org/apache/cassandra/cql3/Duration.java
@@ -17,6 +17,9 @@
*/
package org.apache.cassandra.cql3;
+import java.util.Calendar;
+import java.util.Locale;
+import java.util.TimeZone;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -26,6 +29,7 @@
import static org.apache.cassandra.cql3.statements.RequestValidations.checkTrue;
import static org.apache.cassandra.cql3.statements.RequestValidations.invalidRequest;
+import static org.apache.commons.lang3.time.DateUtils.MILLIS_PER_DAY;
/**
* Represents a duration. A durations store separately months, days, and seconds due to the fact that
@@ -265,6 +269,53 @@
return nanoseconds;
}
+ /**
+ * Adds this duration to the specified time in milliseconds.
+ * @param timeInMillis the time to which the duration must be added
+ * @return the specified time plus this duration
+ */
+ public long addTo(long timeInMillis)
+ {
+ return add(timeInMillis, months, days, nanoseconds);
+ }
+
+ /**
+ * Substracts this duration from the specified time in milliseconds.
+ * @param timeInMillis the time from which the duration must be substracted
+ * @return the specified time minus this duration
+ */
+ public long substractFrom(long timeInMillis)
+ {
+ return add(timeInMillis, -months, -days, -nanoseconds);
+ }
+
+ /**
+ * Adds the specified months, days and nanoseconds to the specified time in milliseconds.
+ *
+ * @param timeInMillis the time to which the months, days and nanoseconds must be added
+ * @param months the number of months to add
+ * @param days the number of days to add
+ * @param nanoseconds the number of nanoseconds to add
+ * @return the specified time plus the months, days and nanoseconds
+ */
+ private static long add(long timeInMillis, int months, int days, long nanoseconds)
+ {
+ // If the duration does not contains any months we can can ignore daylight saving,
+ // as time zones are not supported, and simply look at the milliseconds
+ if (months == 0)
+ {
+ long durationInMillis = (days * MILLIS_PER_DAY) + (nanoseconds / NANOS_PER_MILLI);
+ return timeInMillis + durationInMillis;
+ }
+
+ Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC"), Locale.US);
+ calendar.setTimeInMillis(timeInMillis);
+ calendar.add(Calendar.MONTH, months);
+ calendar.add(Calendar.DAY_OF_MONTH, days);
+ calendar.add(Calendar.MILLISECOND, (int) (nanoseconds / NANOS_PER_MILLI));
+ return calendar.getTimeInMillis();
+ }
+
@Override
public int hashCode()
{
@@ -309,6 +360,24 @@
}
/**
+ * Checks if that duration has a day precision (nothing bellow the day level).
+ * @return {@code true} if that duration has a day precision, {@code false} otherwise
+ */
+ public boolean hasDayPrecision()
+ {
+ return getNanoseconds() == 0;
+ }
+
+ /**
+ * Checks if that duration has a millisecond precision (nothing bellow the millisecond level).
+ * @return {@code true} if that duration has a millisecond precision, {@code false} otherwise
+ */
+ public boolean hasMillisecondPrecision()
+ {
+ return getNanoseconds() % NANOS_PER_MILLI == 0;
+ }
+
+ /**
* Appends the result of the division to the specified builder if the dividend is not zero.
*
* @param builder the builder to append to
diff --git a/src/java/org/apache/cassandra/cql3/FieldIdentifier.java b/src/java/org/apache/cassandra/cql3/FieldIdentifier.java
index 9f72fc4..bdde98d 100644
--- a/src/java/org/apache/cassandra/cql3/FieldIdentifier.java
+++ b/src/java/org/apache/cassandra/cql3/FieldIdentifier.java
@@ -41,7 +41,7 @@
*/
public static FieldIdentifier forUnquoted(String text)
{
- return new FieldIdentifier(convert(text.toLowerCase(Locale.US)));
+ return new FieldIdentifier(convert(text == null ? null : text.toLowerCase(Locale.US)));
}
/**
diff --git a/src/java/org/apache/cassandra/cql3/IfExistsCondition.java b/src/java/org/apache/cassandra/cql3/IfExistsCondition.java
deleted file mode 100644
index a24d8c0..0000000
--- a/src/java/org/apache/cassandra/cql3/IfExistsCondition.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.cql3;
-
-import org.apache.cassandra.cql3.statements.CQL3CasRequest;
-import org.apache.cassandra.db.Clustering;
-
-final class IfExistsCondition extends AbstractConditions
-{
- @Override
- public void addConditionsTo(CQL3CasRequest request, Clustering clustering, QueryOptions options)
- {
- request.addExist(clustering);
- }
-
- @Override
- public boolean isIfExists()
- {
- return true;
- }
-}
diff --git a/src/java/org/apache/cassandra/cql3/IfNotExistsCondition.java b/src/java/org/apache/cassandra/cql3/IfNotExistsCondition.java
deleted file mode 100644
index 05cb864..0000000
--- a/src/java/org/apache/cassandra/cql3/IfNotExistsCondition.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.cql3;
-
-import org.apache.cassandra.cql3.statements.CQL3CasRequest;
-import org.apache.cassandra.db.Clustering;
-
-final class IfNotExistsCondition extends AbstractConditions
-{
- @Override
- public void addConditionsTo(CQL3CasRequest request, Clustering clustering, QueryOptions options)
- {
- request.addNotExist(clustering);
- }
-
- @Override
- public boolean isIfNotExists()
- {
- return true;
- }
-}
diff --git a/src/java/org/apache/cassandra/cql3/IndexName.java b/src/java/org/apache/cassandra/cql3/IndexName.java
deleted file mode 100644
index d7ff8ff..0000000
--- a/src/java/org/apache/cassandra/cql3/IndexName.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.cql3;
-
-public final class IndexName extends KeyspaceElementName
-{
- private String idxName;
-
- public void setIndex(String idx, boolean keepCase)
- {
- idxName = toInternalName(idx, keepCase);
- }
-
- public String getIdx()
- {
- return idxName;
- }
-
- public CFName getCfName()
- {
- CFName cfName = new CFName();
- if (hasKeyspace())
- cfName.setKeyspace(getKeyspace(), true);
- return cfName;
- }
-
- @Override
- public String toString()
- {
- return super.toString() + idxName;
- }
-}
diff --git a/src/java/org/apache/cassandra/cql3/Json.java b/src/java/org/apache/cassandra/cql3/Json.java
index af004a8..ea0a49c 100644
--- a/src/java/org/apache/cassandra/cql3/Json.java
+++ b/src/java/org/apache/cassandra/cql3/Json.java
@@ -20,10 +20,10 @@
import java.io.IOException;
import java.util.*;
-import com.fasterxml.jackson.core.io.JsonStringEncoder;
+import com.fasterxml.jackson.core.util.BufferRecyclers;
import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.cql3.functions.Function;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.db.marshal.UTF8Type;
@@ -42,7 +42,7 @@
*/
public static String quoteAsJsonString(String s)
{
- return new String(JsonStringEncoder.getInstance().quoteAsString(s));
+ return new String(BufferRecyclers.getJsonStringEncoder().quoteAsString(s));
}
public static Object decodeJson(String json)
@@ -59,7 +59,7 @@
public interface Raw
{
- public Prepared prepareAndCollectMarkers(CFMetaData metadata, Collection<ColumnDefinition> receivers, VariableSpecifications boundNames);
+ public Prepared prepareAndCollectMarkers(TableMetadata metadata, Collection<ColumnMetadata> receivers, VariableSpecifications boundNames);
}
/**
@@ -75,7 +75,7 @@
this.text = text;
}
- public Prepared prepareAndCollectMarkers(CFMetaData metadata, Collection<ColumnDefinition> receivers, VariableSpecifications boundNames)
+ public Prepared prepareAndCollectMarkers(TableMetadata metadata, Collection<ColumnMetadata> receivers, VariableSpecifications boundNames)
{
return new PreparedLiteral(parseJson(text, receivers));
}
@@ -94,15 +94,15 @@
this.bindIndex = bindIndex;
}
- public Prepared prepareAndCollectMarkers(CFMetaData metadata, Collection<ColumnDefinition> receivers, VariableSpecifications boundNames)
+ public Prepared prepareAndCollectMarkers(TableMetadata metadata, Collection<ColumnMetadata> receivers, VariableSpecifications boundNames)
{
boundNames.add(bindIndex, makeReceiver(metadata));
return new PreparedMarker(bindIndex, receivers);
}
- private ColumnSpecification makeReceiver(CFMetaData metadata)
+ private ColumnSpecification makeReceiver(TableMetadata metadata)
{
- return new ColumnSpecification(metadata.ksName, metadata.cfName, JSON_COLUMN_ID, UTF8Type.instance);
+ return new ColumnSpecification(metadata.keyspace, metadata.name, JSON_COLUMN_ID, UTF8Type.instance);
}
}
@@ -111,7 +111,7 @@
*/
public static abstract class Prepared
{
- public abstract Term.Raw getRawTermForColumn(ColumnDefinition def, boolean defaultUnset);
+ public abstract Term.Raw getRawTermForColumn(ColumnMetadata def, boolean defaultUnset);
}
/**
@@ -126,7 +126,7 @@
this.columnMap = columnMap;
}
- public Term.Raw getRawTermForColumn(ColumnDefinition def, boolean defaultUnset)
+ public Term.Raw getRawTermForColumn(ColumnMetadata def, boolean defaultUnset)
{
Term value = columnMap.get(def.name);
return value == null
@@ -141,15 +141,15 @@
private static class PreparedMarker extends Prepared
{
private final int bindIndex;
- private final Collection<ColumnDefinition> columns;
+ private final Collection<ColumnMetadata> columns;
- public PreparedMarker(int bindIndex, Collection<ColumnDefinition> columns)
+ public PreparedMarker(int bindIndex, Collection<ColumnMetadata> columns)
{
this.bindIndex = bindIndex;
this.columns = columns;
}
- public RawDelayedColumnValue getRawTermForColumn(ColumnDefinition def, boolean defaultUnset)
+ public RawDelayedColumnValue getRawTermForColumn(ColumnMetadata def, boolean defaultUnset)
{
return new RawDelayedColumnValue(this, def, defaultUnset);
}
@@ -199,10 +199,10 @@
private static class RawDelayedColumnValue extends Term.Raw
{
private final PreparedMarker marker;
- private final ColumnDefinition column;
+ private final ColumnMetadata column;
private final boolean defaultUnset;
- public RawDelayedColumnValue(PreparedMarker prepared, ColumnDefinition column, boolean defaultUnset)
+ public RawDelayedColumnValue(PreparedMarker prepared, ColumnMetadata column, boolean defaultUnset)
{
this.marker = prepared;
this.column = column;
@@ -238,10 +238,10 @@
private static class DelayedColumnValue extends Term.NonTerminal
{
private final PreparedMarker marker;
- private final ColumnDefinition column;
+ private final ColumnMetadata column;
private final boolean defaultUnset;
- public DelayedColumnValue(PreparedMarker prepared, ColumnDefinition column, boolean defaultUnset)
+ public DelayedColumnValue(PreparedMarker prepared, ColumnMetadata column, boolean defaultUnset)
{
this.marker = prepared;
this.column = column;
@@ -278,7 +278,7 @@
/**
* Given a JSON string, return a map of columns to their values for the insert.
*/
- public static Map<ColumnIdentifier, Term> parseJson(String jsonString, Collection<ColumnDefinition> expectedReceivers)
+ public static Map<ColumnIdentifier, Term> parseJson(String jsonString, Collection<ColumnMetadata> expectedReceivers)
{
try
{
diff --git a/src/java/org/apache/cassandra/cql3/KeyspaceElementName.java b/src/java/org/apache/cassandra/cql3/KeyspaceElementName.java
deleted file mode 100644
index 0a68997..0000000
--- a/src/java/org/apache/cassandra/cql3/KeyspaceElementName.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.cql3;
-
-import java.util.Locale;
-
-/**
- * Base class for the names of the keyspace elements (e.g. table, index ...)
- */
-abstract class KeyspaceElementName
-{
- /**
- * The keyspace name as stored internally.
- */
- private String ksName;
-
- /**
- * Sets the keyspace.
- *
- * @param ks the keyspace name
- * @param keepCase <code>true</code> if the case must be kept, <code>false</code> otherwise.
- */
- public final void setKeyspace(String ks, boolean keepCase)
- {
- ksName = toInternalName(ks, keepCase);
- }
-
- /**
- * Checks if the keyspace is specified.
- * @return <code>true</code> if the keyspace is specified, <code>false</code> otherwise.
- */
- public final boolean hasKeyspace()
- {
- return ksName != null;
- }
-
- public final String getKeyspace()
- {
- return ksName;
- }
-
- /**
- * Converts the specified name into the name used internally.
- *
- * @param name the name
- * @param keepCase <code>true</code> if the case must be kept, <code>false</code> otherwise.
- * @return the name used internally.
- */
- protected static String toInternalName(String name, boolean keepCase)
- {
- return keepCase ? name : name.toLowerCase(Locale.US);
- }
-
- @Override
- public String toString()
- {
- return hasKeyspace() ? (getKeyspace() + ".") : "";
- }
-}
diff --git a/src/java/org/apache/cassandra/cql3/Lists.java b/src/java/org/apache/cassandra/cql3/Lists.java
index 6a2a5a5..4a68df9 100644
--- a/src/java/org/apache/cassandra/cql3/Lists.java
+++ b/src/java/org/apache/cassandra/cql3/Lists.java
@@ -22,12 +22,14 @@
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
+import java.util.Objects;
+import java.util.Optional;
import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Collectors;
+import java.util.stream.StreamSupport;
+import org.apache.cassandra.schema.ColumnMetadata;
import com.google.common.annotations.VisibleForTesting;
-
-import org.apache.cassandra.config.ColumnDefinition;
import org.apache.cassandra.cql3.functions.Function;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.rows.*;
@@ -55,7 +57,67 @@
public static ColumnSpecification valueSpecOf(ColumnSpecification column)
{
- return new ColumnSpecification(column.ksName, column.cfName, new ColumnIdentifier("value(" + column.name + ")", true), ((ListType)column.type).getElementsType());
+ return new ColumnSpecification(column.ksName, column.cfName, new ColumnIdentifier("value(" + column.name + ")", true), ((ListType<?>)column.type).getElementsType());
+ }
+
+ /**
+ * Tests that the list with the specified elements can be assigned to the specified column.
+ *
+ * @param receiver the receiving column
+ * @param elements the list elements
+ */
+ public static AssignmentTestable.TestResult testListAssignment(ColumnSpecification receiver,
+ List<? extends AssignmentTestable> elements)
+ {
+ if (!(receiver.type instanceof ListType))
+ return AssignmentTestable.TestResult.NOT_ASSIGNABLE;
+
+ // If there is no elements, we can't say it's an exact match (an empty list if fundamentally polymorphic).
+ if (elements.isEmpty())
+ return AssignmentTestable.TestResult.WEAKLY_ASSIGNABLE;
+
+ ColumnSpecification valueSpec = valueSpecOf(receiver);
+ return AssignmentTestable.TestResult.testAll(receiver.ksName, valueSpec, elements);
+ }
+
+ /**
+ * Create a <code>String</code> representation of the list containing the specified elements.
+ *
+ * @param elements the list elements
+ * @return a <code>String</code> representation of the list
+ */
+ public static String listToString(List<?> elements)
+ {
+ return listToString(elements, Object::toString);
+ }
+
+ /**
+ * Create a <code>String</code> representation of the list from the specified items associated to
+ * the list elements.
+ *
+ * @param items items associated to the list elements
+ * @param mapper the mapper used to map the items to the <code>String</code> representation of the list elements
+ * @return a <code>String</code> representation of the list
+ */
+ public static <T> String listToString(Iterable<T> items, java.util.function.Function<T, String> mapper)
+ {
+ return StreamSupport.stream(items.spliterator(), false)
+ .map(e -> mapper.apply(e))
+ .collect(Collectors.joining(", ", "[", "]"));
+ }
+
+ /**
+ * Returns the exact ListType from the items if it can be known.
+ *
+ * @param items the items mapped to the list elements
+ * @param mapper the mapper used to retrieve the element types from the items
+ * @return the exact ListType from the items if it can be known or <code>null</code>
+ */
+ public static <T> AbstractType<?> getExactListTypeIfKnown(List<T> items,
+ java.util.function.Function<T, AbstractType<?>> mapper)
+ {
+ Optional<AbstractType<?>> type = items.stream().map(mapper).filter(Objects::nonNull).findFirst();
+ return type.isPresent() ? ListType.getInstance(type.get(), false) : null;
}
public static class Literal extends Term.Raw
@@ -105,32 +167,18 @@
public AssignmentTestable.TestResult testAssignment(String keyspace, ColumnSpecification receiver)
{
- if (!(receiver.type instanceof ListType))
- return AssignmentTestable.TestResult.NOT_ASSIGNABLE;
-
- // If there is no elements, we can't say it's an exact match (an empty list if fundamentally polymorphic).
- if (elements.isEmpty())
- return AssignmentTestable.TestResult.WEAKLY_ASSIGNABLE;
-
- ColumnSpecification valueSpec = Lists.valueSpecOf(receiver);
- return AssignmentTestable.TestResult.testAll(keyspace, valueSpec, elements);
+ return testListAssignment(receiver, elements);
}
@Override
public AbstractType<?> getExactTypeIfKnown(String keyspace)
{
- for (Term.Raw term : elements)
- {
- AbstractType<?> type = term.getExactTypeIfKnown(keyspace);
- if (type != null)
- return ListType.getInstance(type, false);
- }
- return null;
+ return getExactListTypeIfKnown(elements, p -> p.getExactTypeIfKnown(keyspace));
}
public String getText()
{
- return elements.stream().map(Term.Raw::getText).collect(Collectors.joining(", ", "[", "]"));
+ return listToString(elements, Term.Raw::getText);
}
}
@@ -330,7 +378,7 @@
public static class Setter extends Operation
{
- public Setter(ColumnDefinition column, Term t)
+ public Setter(ColumnMetadata column, Term t)
{
super(column, t);
}
@@ -348,7 +396,7 @@
}
}
- private static int existingSize(Row row, ColumnDefinition column)
+ private static int existingSize(Row row, ColumnMetadata column)
{
if (row == null)
return 0;
@@ -361,7 +409,7 @@
{
private final Term idx;
- public SetterByIndex(ColumnDefinition column, Term idx, Term t)
+ public SetterByIndex(ColumnMetadata column, Term idx, Term t)
{
super(column, t);
this.idx = idx;
@@ -411,7 +459,7 @@
public static class Appender extends Operation
{
- public Appender(ColumnDefinition column, Term t)
+ public Appender(ColumnMetadata column, Term t)
{
super(column, t);
}
@@ -423,7 +471,7 @@
doAppend(value, column, params);
}
- static void doAppend(Term.Terminal value, ColumnDefinition column, UpdateParameters params) throws InvalidRequestException
+ static void doAppend(Term.Terminal value, ColumnMetadata column, UpdateParameters params) throws InvalidRequestException
{
if (column.type.isMultiCell())
{
@@ -451,7 +499,7 @@
public static class Prepender extends Operation
{
- public Prepender(ColumnDefinition column, Term t)
+ public Prepender(ColumnMetadata column, Term t)
{
super(column, t);
}
@@ -487,7 +535,7 @@
public static class Discarder extends Operation
{
- public Discarder(ColumnDefinition column, Term t)
+ public Discarder(ColumnMetadata column, Term t)
{
super(column, t);
}
@@ -525,7 +573,7 @@
public static class DiscarderByIndex extends Operation
{
- public DiscarderByIndex(ColumnDefinition column, Term idx)
+ public DiscarderByIndex(ColumnMetadata column, Term idx)
{
super(column, idx);
}
diff --git a/src/java/org/apache/cassandra/cql3/Maps.java b/src/java/org/apache/cassandra/cql3/Maps.java
index 5faa8cc..e02169e 100644
--- a/src/java/org/apache/cassandra/cql3/Maps.java
+++ b/src/java/org/apache/cassandra/cql3/Maps.java
@@ -23,7 +23,7 @@
import java.util.*;
import java.util.stream.Collectors;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.cql3.functions.Function;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.rows.*;
@@ -44,12 +44,89 @@
public static ColumnSpecification keySpecOf(ColumnSpecification column)
{
- return new ColumnSpecification(column.ksName, column.cfName, new ColumnIdentifier("key(" + column.name + ")", true), ((MapType)column.type).getKeysType());
+ return new ColumnSpecification(column.ksName, column.cfName, new ColumnIdentifier("key(" + column.name + ")", true), ((MapType<? , ?>)column.type).getKeysType());
}
public static ColumnSpecification valueSpecOf(ColumnSpecification column)
{
- return new ColumnSpecification(column.ksName, column.cfName, new ColumnIdentifier("value(" + column.name + ")", true), ((MapType)column.type).getValuesType());
+ return new ColumnSpecification(column.ksName, column.cfName, new ColumnIdentifier("value(" + column.name + ")", true), ((MapType<?, ?>)column.type).getValuesType());
+ }
+
+ /**
+ * Tests that the map with the specified entries can be assigned to the specified column.
+ *
+ * @param receiver the receiving column
+ * @param entries the map entries
+ */
+ public static <T extends AssignmentTestable> AssignmentTestable.TestResult testMapAssignment(ColumnSpecification receiver,
+ List<Pair<T, T>> entries)
+ {
+ ColumnSpecification keySpec = keySpecOf(receiver);
+ ColumnSpecification valueSpec = valueSpecOf(receiver);
+
+ // It's an exact match if all are exact match, but is not assignable as soon as any is non assignable.
+ AssignmentTestable.TestResult res = AssignmentTestable.TestResult.EXACT_MATCH;
+ for (Pair<T, T> entry : entries)
+ {
+ AssignmentTestable.TestResult t1 = entry.left.testAssignment(receiver.ksName, keySpec);
+ AssignmentTestable.TestResult t2 = entry.right.testAssignment(receiver.ksName, valueSpec);
+ if (t1 == AssignmentTestable.TestResult.NOT_ASSIGNABLE || t2 == AssignmentTestable.TestResult.NOT_ASSIGNABLE)
+ return AssignmentTestable.TestResult.NOT_ASSIGNABLE;
+ if (t1 != AssignmentTestable.TestResult.EXACT_MATCH || t2 != AssignmentTestable.TestResult.EXACT_MATCH)
+ res = AssignmentTestable.TestResult.WEAKLY_ASSIGNABLE;
+ }
+ return res;
+ }
+
+ /**
+ * Create a <code>String</code> representation of the list containing the specified elements.
+ *
+ * @param entries the list elements
+ * @return a <code>String</code> representation of the list
+ */
+ public static <T> String mapToString(List<Pair<T, T>> entries)
+ {
+ return mapToString(entries, Object::toString);
+ }
+
+ /**
+ * Create a <code>String</code> representation of the map from the specified items associated to
+ * the map entries.
+ *
+ * @param items items associated to the map entries
+ * @param mapper the mapper used to map the items to the <code>String</code> representation of the map entries
+ * @return a <code>String</code> representation of the map
+ */
+ public static <T> String mapToString(List<Pair<T, T>> items,
+ java.util.function.Function<T, String> mapper)
+ {
+ return items.stream()
+ .map(p -> String.format("%s: %s", mapper.apply(p.left), mapper.apply(p.right)))
+ .collect(Collectors.joining(", ", "{", "}"));
+ }
+
+ /**
+ * Returns the exact MapType from the entries if it can be known.
+ *
+ * @param entries the entries
+ * @param mapper the mapper used to retrieve the key and value types from the entries
+ * @return the exact MapType from the entries if it can be known or <code>null</code>
+ */
+ public static <T> AbstractType<?> getExactMapTypeIfKnown(List<Pair<T, T>> entries,
+ java.util.function.Function<T, AbstractType<?>> mapper)
+ {
+ AbstractType<?> keyType = null;
+ AbstractType<?> valueType = null;
+ for (Pair<T, T> entry : entries)
+ {
+ if (keyType == null)
+ keyType = mapper.apply(entry.left);
+ if (valueType == null)
+ valueType = mapper.apply(entry.right);
+ if (keyType != null && valueType != null)
+ return MapType.getInstance(keyType, valueType, false);
+ }
+ return null;
}
public static class Literal extends Term.Raw
@@ -82,7 +159,7 @@
values.put(k, v);
}
- DelayedValue value = new DelayedValue(((MapType)receiver.type).getKeysType(), values);
+ DelayedValue value = new DelayedValue(((MapType<?, ?>)receiver.type).getKeysType(), values);
return allTerminal ? value.bind(QueryOptions.DEFAULT) : value;
}
@@ -104,51 +181,18 @@
public AssignmentTestable.TestResult testAssignment(String keyspace, ColumnSpecification receiver)
{
- if (!(receiver.type instanceof MapType))
- return AssignmentTestable.TestResult.NOT_ASSIGNABLE;
-
- // If there is no elements, we can't say it's an exact match (an empty map if fundamentally polymorphic).
- if (entries.isEmpty())
- return AssignmentTestable.TestResult.WEAKLY_ASSIGNABLE;
-
- ColumnSpecification keySpec = Maps.keySpecOf(receiver);
- ColumnSpecification valueSpec = Maps.valueSpecOf(receiver);
- // It's an exact match if all are exact match, but is not assignable as soon as any is non assignable.
- AssignmentTestable.TestResult res = AssignmentTestable.TestResult.EXACT_MATCH;
- for (Pair<Term.Raw, Term.Raw> entry : entries)
- {
- AssignmentTestable.TestResult t1 = entry.left.testAssignment(keyspace, keySpec);
- AssignmentTestable.TestResult t2 = entry.right.testAssignment(keyspace, valueSpec);
- if (t1 == AssignmentTestable.TestResult.NOT_ASSIGNABLE || t2 == AssignmentTestable.TestResult.NOT_ASSIGNABLE)
- return AssignmentTestable.TestResult.NOT_ASSIGNABLE;
- if (t1 != AssignmentTestable.TestResult.EXACT_MATCH || t2 != AssignmentTestable.TestResult.EXACT_MATCH)
- res = AssignmentTestable.TestResult.WEAKLY_ASSIGNABLE;
- }
- return res;
+ return testMapAssignment(receiver, entries);
}
@Override
public AbstractType<?> getExactTypeIfKnown(String keyspace)
{
- AbstractType<?> keyType = null;
- AbstractType<?> valueType = null;
- for (Pair<Term.Raw, Term.Raw> entry : entries)
- {
- if (keyType == null)
- keyType = entry.left.getExactTypeIfKnown(keyspace);
- if (valueType == null)
- valueType = entry.right.getExactTypeIfKnown(keyspace);
- if (keyType != null && valueType != null)
- return MapType.getInstance(keyType, valueType, false);
- }
- return null;
+ return getExactMapTypeIfKnown(entries, p -> p.getExactTypeIfKnown(keyspace));
}
public String getText()
{
- return entries.stream()
- .map(entry -> String.format("%s: %s", entry.left.getText(), entry.right.getText()))
- .collect(Collectors.joining(", ", "{", "}"));
+ return mapToString(entries, Term.Raw::getText);
}
}
@@ -284,7 +328,7 @@
public static class Setter extends Operation
{
- public Setter(ColumnDefinition column, Term t)
+ public Setter(ColumnMetadata column, Term t)
{
super(column, t);
}
@@ -306,7 +350,7 @@
{
private final Term k;
- public SetterByKey(ColumnDefinition column, Term k, Term t)
+ public SetterByKey(ColumnMetadata column, Term k, Term t)
{
super(column, t);
this.k = k;
@@ -342,94 +386,9 @@
}
}
- // Currently only used internally counters support in SuperColumn families.
- // Addition on the element level inside the collections are otherwise not supported in the CQL.
- public static class AdderByKey extends Operation
- {
- private final Term k;
-
- public AdderByKey(ColumnDefinition column, Term t, Term k)
- {
- super(column, t);
- this.k = k;
- }
-
- @Override
- public void collectMarkerSpecification(VariableSpecifications boundNames)
- {
- super.collectMarkerSpecification(boundNames);
- k.collectMarkerSpecification(boundNames);
- }
-
- public void execute(DecoratedKey partitionKey, UpdateParameters params) throws InvalidRequestException
- {
- assert column.type.isMultiCell() : "Attempted to set a value for a single key on a frozen map";
-
- ByteBuffer key = k.bindAndGet(params.options);
- ByteBuffer value = t.bindAndGet(params.options);
-
- if (key == null)
- throw new InvalidRequestException("Invalid null map key");
- if (key == ByteBufferUtil.UNSET_BYTE_BUFFER)
- throw new InvalidRequestException("Invalid unset map key");
-
- if (value == null)
- throw new InvalidRequestException("Invalid null value for counter increment");
- if (value == ByteBufferUtil.UNSET_BYTE_BUFFER)
- return;
-
- long increment = ByteBufferUtil.toLong(value);
- params.addCounter(column, increment, CellPath.create(key));
- }
- }
-
- // Currently only used internally counters support in SuperColumn families.
- // Addition on the element level inside the collections are otherwise not supported in the CQL.
- public static class SubtracterByKey extends Operation
- {
- private final Term k;
-
- public SubtracterByKey(ColumnDefinition column, Term t, Term k)
- {
- super(column, t);
- this.k = k;
- }
-
- @Override
- public void collectMarkerSpecification(VariableSpecifications boundNames)
- {
- super.collectMarkerSpecification(boundNames);
- k.collectMarkerSpecification(boundNames);
- }
-
- public void execute(DecoratedKey partitionKey, UpdateParameters params) throws InvalidRequestException
- {
- assert column.type.isMultiCell() : "Attempted to set a value for a single key on a frozen map";
-
- ByteBuffer key = k.bindAndGet(params.options);
- ByteBuffer value = t.bindAndGet(params.options);
-
- if (key == null)
- throw new InvalidRequestException("Invalid null map key");
- if (key == ByteBufferUtil.UNSET_BYTE_BUFFER)
- throw new InvalidRequestException("Invalid unset map key");
-
- if (value == null)
- throw new InvalidRequestException("Invalid null value for counter increment");
- if (value == ByteBufferUtil.UNSET_BYTE_BUFFER)
- return;
-
- long increment = ByteBufferUtil.toLong(value);
- if (increment == Long.MIN_VALUE)
- throw new InvalidRequestException("The negation of " + increment + " overflows supported counter precision (signed 8 bytes integer)");
-
- params.addCounter(column, -increment, CellPath.create(key));
- }
- }
-
public static class Putter extends Operation
{
- public Putter(ColumnDefinition column, Term t)
+ public Putter(ColumnMetadata column, Term t)
{
super(column, t);
}
@@ -442,7 +401,7 @@
doPut(value, column, params);
}
- static void doPut(Term.Terminal value, ColumnDefinition column, UpdateParameters params) throws InvalidRequestException
+ static void doPut(Term.Terminal value, ColumnMetadata column, UpdateParameters params) throws InvalidRequestException
{
if (column.type.isMultiCell())
{
@@ -466,7 +425,7 @@
public static class DiscarderByKey extends Operation
{
- public DiscarderByKey(ColumnDefinition column, Term k)
+ public DiscarderByKey(ColumnMetadata column, Term k)
{
super(column, k);
}
diff --git a/src/java/org/apache/cassandra/cql3/MultiColumnRelation.java b/src/java/org/apache/cassandra/cql3/MultiColumnRelation.java
index fb86e7b..89d69ed 100644
--- a/src/java/org/apache/cassandra/cql3/MultiColumnRelation.java
+++ b/src/java/org/apache/cassandra/cql3/MultiColumnRelation.java
@@ -19,15 +19,15 @@
import java.util.ArrayList;
import java.util.List;
+import java.util.Objects;
import java.util.stream.Collectors;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.cql3.Term.MultiColumnRaw;
import org.apache.cassandra.cql3.Term.Raw;
import org.apache.cassandra.cql3.restrictions.MultiColumnRestriction;
import org.apache.cassandra.cql3.restrictions.Restriction;
-import org.apache.cassandra.cql3.restrictions.SingleColumnRestriction;
import org.apache.cassandra.cql3.statements.Bound;
import org.apache.cassandra.exceptions.InvalidRequestException;
@@ -47,7 +47,7 @@
*/
public class MultiColumnRelation extends Relation
{
- private final List<ColumnDefinition.Raw> entities;
+ private final List<ColumnMetadata.Raw> entities;
/** A Tuples.Literal or Tuples.Raw marker */
private final Term.MultiColumnRaw valuesOrMarker;
@@ -57,7 +57,7 @@
private final Tuples.INRaw inMarker;
- private MultiColumnRelation(List<ColumnDefinition.Raw> entities, Operator relationType, Term.MultiColumnRaw valuesOrMarker, List<? extends Term.MultiColumnRaw> inValues, Tuples.INRaw inMarker)
+ private MultiColumnRelation(List<ColumnMetadata.Raw> entities, Operator relationType, Term.MultiColumnRaw valuesOrMarker, List<? extends Term.MultiColumnRaw> inValues, Tuples.INRaw inMarker)
{
this.entities = entities;
this.relationType = relationType;
@@ -77,7 +77,7 @@
* @param valuesOrMarker a Tuples.Literal instance or a Tuples.Raw marker
* @return a new <code>MultiColumnRelation</code> instance
*/
- public static MultiColumnRelation createNonInRelation(List<ColumnDefinition.Raw> entities, Operator relationType, Term.MultiColumnRaw valuesOrMarker)
+ public static MultiColumnRelation createNonInRelation(List<ColumnMetadata.Raw> entities, Operator relationType, Term.MultiColumnRaw valuesOrMarker)
{
assert relationType != Operator.IN;
return new MultiColumnRelation(entities, relationType, valuesOrMarker, null, null);
@@ -90,7 +90,7 @@
* @param inValues a list of Tuples.Literal instances or a Tuples.Raw markers
* @return a new <code>MultiColumnRelation</code> instance
*/
- public static MultiColumnRelation createInRelation(List<ColumnDefinition.Raw> entities, List<? extends Term.MultiColumnRaw> inValues)
+ public static MultiColumnRelation createInRelation(List<ColumnMetadata.Raw> entities, List<? extends Term.MultiColumnRaw> inValues)
{
return new MultiColumnRelation(entities, Operator.IN, null, inValues, null);
}
@@ -102,12 +102,12 @@
* @param inMarker a single IN marker
* @return a new <code>MultiColumnRelation</code> instance
*/
- public static MultiColumnRelation createSingleMarkerInRelation(List<ColumnDefinition.Raw> entities, Tuples.INRaw inMarker)
+ public static MultiColumnRelation createSingleMarkerInRelation(List<ColumnMetadata.Raw> entities, Tuples.INRaw inMarker)
{
return new MultiColumnRelation(entities, Operator.IN, null, null, inMarker);
}
- public List<ColumnDefinition.Raw> getEntities()
+ public List<ColumnMetadata.Raw> getEntities()
{
return entities;
}
@@ -134,23 +134,21 @@
}
@Override
- protected Restriction newEQRestriction(CFMetaData cfm,
- VariableSpecifications boundNames) throws InvalidRequestException
+ protected Restriction newEQRestriction(TableMetadata table, VariableSpecifications boundNames)
{
- List<ColumnDefinition> receivers = receivers(cfm);
- Term term = toTerm(receivers, getValue(), cfm.ksName, boundNames);
+ List<ColumnMetadata> receivers = receivers(table);
+ Term term = toTerm(receivers, getValue(), table.keyspace, boundNames);
return new MultiColumnRestriction.EQRestriction(receivers, term);
}
@Override
- protected Restriction newINRestriction(CFMetaData cfm,
- VariableSpecifications boundNames) throws InvalidRequestException
+ protected Restriction newINRestriction(TableMetadata table, VariableSpecifications boundNames)
{
- List<ColumnDefinition> receivers = receivers(cfm);
- List<Term> terms = toTerms(receivers, inValues, cfm.ksName, boundNames);
+ List<ColumnMetadata> receivers = receivers(table);
+ List<Term> terms = toTerms(receivers, inValues, table.keyspace, boundNames);
if (terms == null)
{
- Term term = toTerm(receivers, getValue(), cfm.ksName, boundNames);
+ Term term = toTerm(receivers, getValue(), table.keyspace, boundNames);
return new MultiColumnRestriction.InRestrictionWithMarker(receivers, (AbstractMarker) term);
}
@@ -161,34 +159,28 @@
}
@Override
- protected Restriction newSliceRestriction(CFMetaData cfm,
- VariableSpecifications boundNames,
- Bound bound,
- boolean inclusive) throws InvalidRequestException
+ protected Restriction newSliceRestriction(TableMetadata table, VariableSpecifications boundNames, Bound bound, boolean inclusive)
{
- List<ColumnDefinition> receivers = receivers(cfm);
- Term term = toTerm(receivers, getValue(), cfm.ksName, boundNames);
+ List<ColumnMetadata> receivers = receivers(table);
+ Term term = toTerm(receivers(table), getValue(), table.keyspace, boundNames);
return new MultiColumnRestriction.SliceRestriction(receivers, bound, inclusive, term);
}
@Override
- protected Restriction newContainsRestriction(CFMetaData cfm,
- VariableSpecifications boundNames,
- boolean isKey) throws InvalidRequestException
+ protected Restriction newContainsRestriction(TableMetadata table, VariableSpecifications boundNames, boolean isKey)
{
throw invalidRequest("%s cannot be used for multi-column relations", operator());
}
@Override
- protected Restriction newIsNotRestriction(CFMetaData cfm,
- VariableSpecifications boundNames) throws InvalidRequestException
+ protected Restriction newIsNotRestriction(TableMetadata table, VariableSpecifications boundNames)
{
// this is currently disallowed by the grammar
throw new AssertionError(String.format("%s cannot be used for multi-column relations", operator()));
}
@Override
- protected Restriction newLikeRestriction(CFMetaData cfm, VariableSpecifications boundNames, Operator operator) throws InvalidRequestException
+ protected Restriction newLikeRestriction(TableMetadata table, VariableSpecifications boundNames, Operator operator)
{
throw invalidRequest("%s cannot be used for multi-column relations", operator());
}
@@ -204,13 +196,13 @@
return term;
}
- protected List<ColumnDefinition> receivers(CFMetaData cfm) throws InvalidRequestException
+ protected List<ColumnMetadata> receivers(TableMetadata table) throws InvalidRequestException
{
- List<ColumnDefinition> names = new ArrayList<>(getEntities().size());
+ List<ColumnMetadata> names = new ArrayList<>(getEntities().size());
int previousPosition = -1;
- for (ColumnDefinition.Raw raw : getEntities())
+ for (ColumnMetadata.Raw raw : getEntities())
{
- ColumnDefinition def = raw.prepare(cfm);
+ ColumnMetadata def = raw.prepare(table);
checkTrue(def.isClusteringColumn(), "Multi-column relations can only be applied to clustering columns but was applied to: %s", def.name);
checkFalse(names.contains(def), "Column \"%s\" appeared twice in a relation: %s", def.name, this);
@@ -224,12 +216,12 @@
return names;
}
- public Relation renameIdentifier(ColumnDefinition.Raw from, ColumnDefinition.Raw to)
+ public Relation renameIdentifier(ColumnMetadata.Raw from, ColumnMetadata.Raw to)
{
if (!entities.contains(from))
return this;
- List<ColumnDefinition.Raw> newEntities = entities.stream().map(e -> e.equals(from) ? to : e).collect(Collectors.toList());
+ List<ColumnMetadata.Raw> newEntities = entities.stream().map(e -> e.equals(from) ? to : e).collect(Collectors.toList());
return new MultiColumnRelation(newEntities, operator(), valuesOrMarker, inValues, inMarker);
}
@@ -251,63 +243,25 @@
}
@Override
- public Relation toSuperColumnAdapter()
+ public int hashCode()
{
- return new SuperColumnMultiColumnRelation(entities, relationType, valuesOrMarker, inValues, inMarker);
+ return Objects.hash(relationType, entities, valuesOrMarker, inValues, inMarker);
}
- /**
- * Required for SuperColumn compatibility, in order to map the SuperColumn key restrictions from the regular
- * column to the collection key one.
- */
- private class SuperColumnMultiColumnRelation extends MultiColumnRelation
+ @Override
+ public boolean equals(Object o)
{
- private SuperColumnMultiColumnRelation(List<ColumnDefinition.Raw> entities, Operator relationType, MultiColumnRaw valuesOrMarker, List<? extends MultiColumnRaw> inValues, Tuples.INRaw inMarker)
- {
- super(entities, relationType, valuesOrMarker, inValues, inMarker);
- }
+ if (this == o)
+ return true;
- @Override
- protected Restriction newSliceRestriction(CFMetaData cfm,
- VariableSpecifications boundNames,
- Bound bound,
- boolean inclusive) throws InvalidRequestException
- {
- assert cfm.isSuper() && cfm.isDense();
- List<ColumnDefinition> receivers = receivers(cfm);
- Term term = toTerm(receivers, getValue(), cfm.ksName, boundNames);
- return new SingleColumnRestriction.SuperColumnMultiSliceRestriction(receivers.get(0), bound, inclusive, term);
- }
+ if (!(o instanceof MultiColumnRelation))
+ return false;
- @Override
- protected Restriction newEQRestriction(CFMetaData cfm,
- VariableSpecifications boundNames) throws InvalidRequestException
- {
- assert cfm.isSuper() && cfm.isDense();
- List<ColumnDefinition> receivers = receivers(cfm);
- Term term = toTerm(receivers, getValue(), cfm.ksName, boundNames);
- return new SingleColumnRestriction.SuperColumnMultiEQRestriction(receivers.get(0), term);
- }
-
- @Override
- protected List<ColumnDefinition> receivers(CFMetaData cfm) throws InvalidRequestException
- {
- assert cfm.isSuper() && cfm.isDense();
- List<ColumnDefinition> names = new ArrayList<>(getEntities().size());
-
- for (ColumnDefinition.Raw raw : getEntities())
- {
- ColumnDefinition def = raw.prepare(cfm);
-
- checkTrue(def.isClusteringColumn() ||
- cfm.isSuperColumnKeyColumn(def),
- "Multi-column relations can only be applied to clustering columns but was applied to: %s", def.name);
-
- checkFalse(names.contains(def), "Column \"%s\" appeared twice in a relation: %s", def.name, this);
-
- names.add(def);
- }
- return names;
- }
+ MultiColumnRelation mcr = (MultiColumnRelation) o;
+ return Objects.equals(entities, mcr.entities)
+ && Objects.equals(relationType, mcr.relationType)
+ && Objects.equals(valuesOrMarker, mcr.valuesOrMarker)
+ && Objects.equals(inValues, mcr.inValues)
+ && Objects.equals(inMarker, mcr.inMarker);
}
}
diff --git a/src/java/org/apache/cassandra/cql3/Operation.java b/src/java/org/apache/cassandra/cql3/Operation.java
index c005701..85214f1 100644
--- a/src/java/org/apache/cassandra/cql3/Operation.java
+++ b/src/java/org/apache/cassandra/cql3/Operation.java
@@ -19,12 +19,12 @@
import java.util.List;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
import org.apache.cassandra.cql3.functions.Function;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.marshal.*;
import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableMetadata;
/**
* An UPDATE or DELETE operation.
@@ -43,13 +43,13 @@
public abstract class Operation
{
// the column the operation applies to
- public final ColumnDefinition column;
+ public final ColumnMetadata column;
// Term involved in the operation. In theory this should not be here since some operation
// may require none of more than one term, but most need 1 so it simplify things a bit.
protected final Term t;
- protected Operation(ColumnDefinition column, Term t)
+ protected Operation(ColumnMetadata column, Term t)
{
assert column != null;
this.column = column;
@@ -109,10 +109,12 @@
* It returns an Operation which can be though as post-preparation well-typed
* Operation.
*
+ *
+ * @param metadata
* @param receiver the column this operation applies to.
* @return the prepared update operation.
*/
- public Operation prepare(CFMetaData cfm, ColumnDefinition receiver) throws InvalidRequestException;
+ public Operation prepare(TableMetadata metadata, ColumnMetadata receiver) throws InvalidRequestException;
/**
* @return whether this operation can be applied alongside the {@code
@@ -133,7 +135,7 @@
/**
* The name of the column affected by this delete operation.
*/
- public ColumnDefinition.Raw affectedColumn();
+ public ColumnMetadata.Raw affectedColumn();
/**
* This method validates the operation (i.e. validate it is well typed)
@@ -144,9 +146,10 @@
* Operation.
*
* @param receiver the "column" this operation applies to.
+ * @param metadata
* @return the prepared delete operation.
*/
- public Operation prepare(String keyspace, ColumnDefinition receiver, CFMetaData cfm) throws InvalidRequestException;
+ public Operation prepare(String keyspace, ColumnMetadata receiver, TableMetadata metadata) throws InvalidRequestException;
}
public static class SetValue implements RawUpdate
@@ -158,9 +161,9 @@
this.value = value;
}
- public Operation prepare(CFMetaData cfm, ColumnDefinition receiver) throws InvalidRequestException
+ public Operation prepare(TableMetadata metadata, ColumnMetadata receiver) throws InvalidRequestException
{
- Term v = value.prepare(cfm.ksName, receiver);
+ Term v = value.prepare(metadata.keyspace, receiver);
if (receiver.type instanceof CounterColumnType)
throw new InvalidRequestException(String.format("Cannot set the value of counter column %s (counters can only be incremented/decremented, not set)", receiver.name));
@@ -197,11 +200,6 @@
// it's stupid and 2) the result would seem random to the user.
return false;
}
-
- public Term.Raw value()
- {
- return value;
- }
}
public static class SetElement implements RawUpdate
@@ -215,7 +213,7 @@
this.value = value;
}
- public Operation prepare(CFMetaData cfm, ColumnDefinition receiver) throws InvalidRequestException
+ public Operation prepare(TableMetadata metadata, ColumnMetadata receiver) throws InvalidRequestException
{
if (!(receiver.type instanceof CollectionType))
throw new InvalidRequestException(String.format("Invalid operation (%s) for non collection column %s", toString(receiver), receiver.name));
@@ -225,14 +223,14 @@
switch (((CollectionType)receiver.type).kind)
{
case LIST:
- Term idx = selector.prepare(cfm.ksName, Lists.indexSpecOf(receiver));
- Term lval = value.prepare(cfm.ksName, Lists.valueSpecOf(receiver));
+ Term idx = selector.prepare(metadata.keyspace, Lists.indexSpecOf(receiver));
+ Term lval = value.prepare(metadata.keyspace, Lists.valueSpecOf(receiver));
return new Lists.SetterByIndex(receiver, idx, lval);
case SET:
throw new InvalidRequestException(String.format("Invalid operation (%s) for set column %s", toString(receiver), receiver.name));
case MAP:
- Term key = selector.prepare(cfm.ksName, Maps.keySpecOf(receiver));
- Term mval = value.prepare(cfm.ksName, Maps.valueSpecOf(receiver));
+ Term key = selector.prepare(metadata.keyspace, Maps.keySpecOf(receiver));
+ Term mval = value.prepare(metadata.keyspace, Maps.valueSpecOf(receiver));
return new Maps.SetterByKey(receiver, key, mval);
}
throw new AssertionError();
@@ -262,7 +260,7 @@
this.value = value;
}
- public Operation prepare(CFMetaData cfm, ColumnDefinition receiver) throws InvalidRequestException
+ public Operation prepare(TableMetadata metadata, ColumnMetadata receiver) throws InvalidRequestException
{
if (!receiver.type.isUDT())
throw new InvalidRequestException(String.format("Invalid operation (%s) for non-UDT column %s", toString(receiver), receiver.name));
@@ -273,7 +271,7 @@
if (fieldPosition == -1)
throw new InvalidRequestException(String.format("UDT column %s does not have a field named %s", receiver.name, field));
- Term val = value.prepare(cfm.ksName, UserTypes.fieldSpecOf(receiver, fieldPosition));
+ Term val = value.prepare(metadata.keyspace, UserTypes.fieldSpecOf(receiver, fieldPosition));
return new UserTypes.SetterByField(receiver, field, val);
}
@@ -291,72 +289,6 @@
}
}
- // Currently only used internally counters support in SuperColumn families.
- // Addition on the element level inside the collections are otherwise not supported in the CQL.
- public static class ElementAddition implements RawUpdate
- {
- private final Term.Raw selector;
- private final Term.Raw value;
-
- public ElementAddition(Term.Raw selector, Term.Raw value)
- {
- this.selector = selector;
- this.value = value;
- }
-
- public Operation prepare(CFMetaData cfm, ColumnDefinition receiver) throws InvalidRequestException
- {
- assert receiver.type instanceof MapType;
- Term k = selector.prepare(cfm.ksName, Maps.keySpecOf(receiver));
- Term v = value.prepare(cfm.ksName, Maps.valueSpecOf(receiver));
-
- return new Maps.AdderByKey(receiver, v, k);
- }
-
- protected String toString(ColumnSpecification column)
- {
- return String.format("%s = %s + %s", column.name, column.name, value);
- }
-
- public boolean isCompatibleWith(RawUpdate other)
- {
- return !(other instanceof SetValue);
- }
- }
-
- // Currently only used internally counters support in SuperColumn families.
- // Addition on the element level inside the collections are otherwise not supported in the CQL.
- public static class ElementSubtraction implements RawUpdate
- {
- private final Term.Raw selector;
- private final Term.Raw value;
-
- public ElementSubtraction(Term.Raw selector, Term.Raw value)
- {
- this.selector = selector;
- this.value = value;
- }
-
- public Operation prepare(CFMetaData cfm, ColumnDefinition receiver) throws InvalidRequestException
- {
- assert receiver.type instanceof MapType;
- Term k = selector.prepare(cfm.ksName, Maps.keySpecOf(receiver));
- Term v = value.prepare(cfm.ksName, Maps.valueSpecOf(receiver));
-
- return new Maps.SubtracterByKey(receiver, v, k);
- }
-
- protected String toString(ColumnSpecification column)
- {
- return String.format("%s = %s + %s", column.name, column.name, value);
- }
-
- public boolean isCompatibleWith(RawUpdate other)
- {
- return !(other instanceof SetValue);
- }
- }
-
public static class Addition implements RawUpdate
{
private final Term.Raw value;
@@ -366,15 +298,16 @@
this.value = value;
}
- public Operation prepare(CFMetaData cfm, ColumnDefinition receiver) throws InvalidRequestException
+ public Operation prepare(TableMetadata metadata, ColumnMetadata receiver) throws InvalidRequestException
{
- Term v = value.prepare(cfm.ksName, receiver);
-
if (!(receiver.type instanceof CollectionType))
{
+ if (receiver.type instanceof TupleType)
+ throw new InvalidRequestException(String.format("Invalid operation (%s) for tuple column %s", toString(receiver), receiver.name));
+
if (!(receiver.type instanceof CounterColumnType))
throw new InvalidRequestException(String.format("Invalid operation (%s) for non counter column %s", toString(receiver), receiver.name));
- return new Constants.Adder(receiver, v);
+ return new Constants.Adder(receiver, value.prepare(metadata.keyspace, receiver));
}
else if (!(receiver.type.isMultiCell()))
throw new InvalidRequestException(String.format("Invalid operation (%s) for frozen collection column %s", toString(receiver), receiver.name));
@@ -382,11 +315,21 @@
switch (((CollectionType)receiver.type).kind)
{
case LIST:
- return new Lists.Appender(receiver, v);
+ return new Lists.Appender(receiver, value.prepare(metadata.keyspace, receiver));
case SET:
- return new Sets.Adder(receiver, v);
+ return new Sets.Adder(receiver, value.prepare(metadata.keyspace, receiver));
case MAP:
- return new Maps.Putter(receiver, v);
+ Term term;
+ try
+ {
+ term = value.prepare(metadata.keyspace, receiver);
+ }
+ catch (InvalidRequestException e)
+ {
+ throw new InvalidRequestException(String.format("Value for a map addition has to be a map, but was: '%s'", value));
+ }
+
+ return new Maps.Putter(receiver, term);
}
throw new AssertionError();
}
@@ -400,11 +343,6 @@
{
return !(other instanceof SetValue);
}
-
- public Term.Raw value()
- {
- return value;
- }
}
public static class Substraction implements RawUpdate
@@ -416,13 +354,13 @@
this.value = value;
}
- public Operation prepare(CFMetaData cfm, ColumnDefinition receiver) throws InvalidRequestException
+ public Operation prepare(TableMetadata metadata, ColumnMetadata receiver) throws InvalidRequestException
{
if (!(receiver.type instanceof CollectionType))
{
if (!(receiver.type instanceof CounterColumnType))
throw new InvalidRequestException(String.format("Invalid operation (%s) for non counter column %s", toString(receiver), receiver.name));
- return new Constants.Substracter(receiver, value.prepare(cfm.ksName, receiver));
+ return new Constants.Substracter(receiver, value.prepare(metadata.keyspace, receiver));
}
else if (!(receiver.type.isMultiCell()))
throw new InvalidRequestException(String.format("Invalid operation (%s) for frozen collection column %s", toString(receiver), receiver.name));
@@ -430,16 +368,25 @@
switch (((CollectionType)receiver.type).kind)
{
case LIST:
- return new Lists.Discarder(receiver, value.prepare(cfm.ksName, receiver));
+ return new Lists.Discarder(receiver, value.prepare(metadata.keyspace, receiver));
case SET:
- return new Sets.Discarder(receiver, value.prepare(cfm.ksName, receiver));
+ return new Sets.Discarder(receiver, value.prepare(metadata.keyspace, receiver));
case MAP:
// The value for a map subtraction is actually a set
ColumnSpecification vr = new ColumnSpecification(receiver.ksName,
receiver.cfName,
receiver.name,
SetType.getInstance(((MapType)receiver.type).getKeysType(), false));
- return new Sets.Discarder(receiver, value.prepare(cfm.ksName, vr));
+ Term term;
+ try
+ {
+ term = value.prepare(metadata.keyspace, vr);
+ }
+ catch (InvalidRequestException e)
+ {
+ throw new InvalidRequestException(String.format("Value for a map substraction has to be a set, but was: '%s'", value));
+ }
+ return new Sets.Discarder(receiver, term);
}
throw new AssertionError();
}
@@ -453,11 +400,6 @@
{
return !(other instanceof SetValue);
}
-
- public Term.Raw value()
- {
- return value;
- }
}
public static class Prepend implements RawUpdate
@@ -469,9 +411,9 @@
this.value = value;
}
- public Operation prepare(CFMetaData cfm, ColumnDefinition receiver) throws InvalidRequestException
+ public Operation prepare(TableMetadata metadata, ColumnMetadata receiver) throws InvalidRequestException
{
- Term v = value.prepare(cfm.ksName, receiver);
+ Term v = value.prepare(metadata.keyspace, receiver);
if (!(receiver.type instanceof ListType))
throw new InvalidRequestException(String.format("Invalid operation (%s) for non list column %s", toString(receiver), receiver.name));
@@ -494,19 +436,19 @@
public static class ColumnDeletion implements RawDeletion
{
- private final ColumnDefinition.Raw id;
+ private final ColumnMetadata.Raw id;
- public ColumnDeletion(ColumnDefinition.Raw id)
+ public ColumnDeletion(ColumnMetadata.Raw id)
{
this.id = id;
}
- public ColumnDefinition.Raw affectedColumn()
+ public ColumnMetadata.Raw affectedColumn()
{
return id;
}
- public Operation prepare(String keyspace, ColumnDefinition receiver, CFMetaData cfm) throws InvalidRequestException
+ public Operation prepare(String keyspace, ColumnMetadata receiver, TableMetadata metadata) throws InvalidRequestException
{
// No validation, deleting a column is always "well typed"
return new Constants.Deleter(receiver);
@@ -515,21 +457,21 @@
public static class ElementDeletion implements RawDeletion
{
- private final ColumnDefinition.Raw id;
+ private final ColumnMetadata.Raw id;
private final Term.Raw element;
- public ElementDeletion(ColumnDefinition.Raw id, Term.Raw element)
+ public ElementDeletion(ColumnMetadata.Raw id, Term.Raw element)
{
this.id = id;
this.element = element;
}
- public ColumnDefinition.Raw affectedColumn()
+ public ColumnMetadata.Raw affectedColumn()
{
return id;
}
- public Operation prepare(String keyspace, ColumnDefinition receiver, CFMetaData cfm) throws InvalidRequestException
+ public Operation prepare(String keyspace, ColumnMetadata receiver, TableMetadata metadata) throws InvalidRequestException
{
if (!(receiver.type.isCollection()))
throw new InvalidRequestException(String.format("Invalid deletion operation for non collection column %s", receiver.name));
@@ -554,21 +496,21 @@
public static class FieldDeletion implements RawDeletion
{
- private final ColumnDefinition.Raw id;
+ private final ColumnMetadata.Raw id;
private final FieldIdentifier field;
- public FieldDeletion(ColumnDefinition.Raw id, FieldIdentifier field)
+ public FieldDeletion(ColumnMetadata.Raw id, FieldIdentifier field)
{
this.id = id;
this.field = field;
}
- public ColumnDefinition.Raw affectedColumn()
+ public ColumnMetadata.Raw affectedColumn()
{
return id;
}
- public Operation prepare(String keyspace, ColumnDefinition receiver, CFMetaData cfm) throws InvalidRequestException
+ public Operation prepare(String keyspace, ColumnMetadata receiver, TableMetadata metadata) throws InvalidRequestException
{
if (!receiver.type.isUDT())
throw new InvalidRequestException(String.format("Invalid field deletion operation for non-UDT column %s", receiver.name));
diff --git a/src/java/org/apache/cassandra/cql3/Operator.java b/src/java/org/apache/cassandra/cql3/Operator.java
index 8c04bef..1acedee 100644
--- a/src/java/org/apache/cassandra/cql3/Operator.java
+++ b/src/java/org/apache/cassandra/cql3/Operator.java
@@ -37,6 +37,12 @@
{
return "=";
}
+
+ @Override
+ public boolean isSatisfiedBy(AbstractType<?> type, ByteBuffer leftOperand, ByteBuffer rightOperand)
+ {
+ return type.compareForCQL(leftOperand, rightOperand) == 0;
+ }
},
LT(4)
{
@@ -45,6 +51,12 @@
{
return "<";
}
+
+ @Override
+ public boolean isSatisfiedBy(AbstractType<?> type, ByteBuffer leftOperand, ByteBuffer rightOperand)
+ {
+ return type.compareForCQL(leftOperand, rightOperand) < 0;
+ }
},
LTE(3)
{
@@ -53,6 +65,12 @@
{
return "<=";
}
+
+ @Override
+ public boolean isSatisfiedBy(AbstractType<?> type, ByteBuffer leftOperand, ByteBuffer rightOperand)
+ {
+ return type.compareForCQL(leftOperand, rightOperand) <= 0;
+ }
},
GTE(1)
{
@@ -61,6 +79,12 @@
{
return ">=";
}
+
+ @Override
+ public boolean isSatisfiedBy(AbstractType<?> type, ByteBuffer leftOperand, ByteBuffer rightOperand)
+ {
+ return type.compareForCQL(leftOperand, rightOperand) >= 0;
+ }
},
GT(2)
{
@@ -69,12 +93,56 @@
{
return ">";
}
+
+ @Override
+ public boolean isSatisfiedBy(AbstractType<?> type, ByteBuffer leftOperand, ByteBuffer rightOperand)
+ {
+ return type.compareForCQL(leftOperand, rightOperand) > 0;
+ }
},
IN(7)
{
+ @Override
+ public String toString()
+ {
+ return "IN";
+ }
+
+ public boolean isSatisfiedBy(AbstractType<?> type, ByteBuffer leftOperand, ByteBuffer rightOperand)
+ {
+ List<?> inValues = ListType.getInstance(type, false).getSerializer().deserialize(rightOperand);
+ return inValues.contains(type.getSerializer().deserialize(leftOperand));
+ }
},
CONTAINS(5)
{
+ @Override
+ public String toString()
+ {
+ return "CONTAINS";
+ }
+
+ @Override
+ public boolean isSatisfiedBy(AbstractType<?> type, ByteBuffer leftOperand, ByteBuffer rightOperand)
+ {
+ switch(((CollectionType<?>) type).kind)
+ {
+ case LIST :
+ ListType<?> listType = (ListType<?>) type;
+ List<?> list = listType.getSerializer().deserialize(leftOperand);
+ return list.contains(listType.getElementsType().getSerializer().deserialize(rightOperand));
+ case SET:
+ SetType<?> setType = (SetType<?>) type;
+ Set<?> set = setType.getSerializer().deserialize(leftOperand);
+ return set.contains(setType.getElementsType().getSerializer().deserialize(rightOperand));
+ case MAP:
+ MapType<?, ?> mapType = (MapType<?, ?>) type;
+ Map<?, ?> map = mapType.getSerializer().deserialize(leftOperand);
+ return map.containsValue(mapType.getValuesType().getSerializer().deserialize(rightOperand));
+ default:
+ throw new AssertionError();
+ }
+ }
},
CONTAINS_KEY(6)
{
@@ -83,6 +151,14 @@
{
return "CONTAINS KEY";
}
+
+ @Override
+ public boolean isSatisfiedBy(AbstractType<?> type, ByteBuffer leftOperand, ByteBuffer rightOperand)
+ {
+ MapType<?, ?> mapType = (MapType<?, ?>) type;
+ Map<?, ?> map = mapType.getSerializer().deserialize(leftOperand);
+ return map.containsKey(mapType.getKeysType().getSerializer().deserialize(rightOperand));
+ }
},
NEQ(8)
{
@@ -91,6 +167,13 @@
{
return "!=";
}
+
+ @Override
+ public boolean isSatisfiedBy(AbstractType<?> type, ByteBuffer leftOperand, ByteBuffer rightOperand)
+ {
+ return type.compareForCQL(leftOperand, rightOperand) != 0;
+
+ }
},
IS_NOT(9)
{
@@ -99,6 +182,12 @@
{
return "IS NOT";
}
+
+ @Override
+ public boolean isSatisfiedBy(AbstractType<?> type, ByteBuffer leftOperand, ByteBuffer rightOperand)
+ {
+ throw new UnsupportedOperationException();
+ }
},
LIKE_PREFIX(10)
{
@@ -107,6 +196,12 @@
{
return "LIKE '<term>%'";
}
+
+ @Override
+ public boolean isSatisfiedBy(AbstractType<?> type, ByteBuffer leftOperand, ByteBuffer rightOperand)
+ {
+ return ByteBufferUtil.startsWith(leftOperand, rightOperand);
+ }
},
LIKE_SUFFIX(11)
{
@@ -115,6 +210,12 @@
{
return "LIKE '%<term>'";
}
+
+ @Override
+ public boolean isSatisfiedBy(AbstractType<?> type, ByteBuffer leftOperand, ByteBuffer rightOperand)
+ {
+ return ByteBufferUtil.endsWith(leftOperand, rightOperand);
+ }
},
LIKE_CONTAINS(12)
{
@@ -123,6 +224,12 @@
{
return "LIKE '%<term>%'";
}
+
+ @Override
+ public boolean isSatisfiedBy(AbstractType<?> type, ByteBuffer leftOperand, ByteBuffer rightOperand)
+ {
+ return ByteBufferUtil.contains(leftOperand, rightOperand);
+ }
},
LIKE_MATCHES(13)
{
@@ -131,6 +238,11 @@
{
return "LIKE '<term>'";
}
+
+ public boolean isSatisfiedBy(AbstractType<?> type, ByteBuffer leftOperand, ByteBuffer rightOperand)
+ {
+ return ByteBufferUtil.contains(leftOperand, rightOperand);
+ }
},
LIKE(14)
{
@@ -139,6 +251,12 @@
{
return "LIKE";
}
+
+ @Override
+ public boolean isSatisfiedBy(AbstractType<?> type, ByteBuffer leftOperand, ByteBuffer rightOperand)
+ {
+ throw new UnsupportedOperationException();
+ }
};
/**
@@ -190,59 +308,8 @@
/**
* Whether 2 values satisfy this operator (given the type they should be compared with).
- *
- * @throws AssertionError for CONTAINS and CONTAINS_KEY as this doesn't support those operators yet
*/
- public boolean isSatisfiedBy(AbstractType<?> type, ByteBuffer leftOperand, ByteBuffer rightOperand)
- {
- switch (this)
- {
- case EQ:
- return type.compareForCQL(leftOperand, rightOperand) == 0;
- case LT:
- return type.compareForCQL(leftOperand, rightOperand) < 0;
- case LTE:
- return type.compareForCQL(leftOperand, rightOperand) <= 0;
- case GT:
- return type.compareForCQL(leftOperand, rightOperand) > 0;
- case GTE:
- return type.compareForCQL(leftOperand, rightOperand) >= 0;
- case NEQ:
- return type.compareForCQL(leftOperand, rightOperand) != 0;
- case IN:
- List inValues = ((List) ListType.getInstance(type, false).getSerializer().deserialize(rightOperand));
- return inValues.contains(type.getSerializer().deserialize(leftOperand));
- case CONTAINS:
- if (type instanceof ListType)
- {
- List list = (List) type.getSerializer().deserialize(leftOperand);
- return list.contains(((ListType) type).getElementsType().getSerializer().deserialize(rightOperand));
- }
- else if (type instanceof SetType)
- {
- Set set = (Set) type.getSerializer().deserialize(leftOperand);
- return set.contains(((SetType) type).getElementsType().getSerializer().deserialize(rightOperand));
- }
- else // MapType
- {
- Map map = (Map) type.getSerializer().deserialize(leftOperand);
- return map.containsValue(((MapType) type).getValuesType().getSerializer().deserialize(rightOperand));
- }
- case CONTAINS_KEY:
- Map map = (Map) type.getSerializer().deserialize(leftOperand);
- return map.containsKey(((MapType) type).getKeysType().getSerializer().deserialize(rightOperand));
- case LIKE_PREFIX:
- return ByteBufferUtil.startsWith(leftOperand, rightOperand);
- case LIKE_SUFFIX:
- return ByteBufferUtil.endsWith(leftOperand, rightOperand);
- case LIKE_MATCHES:
- case LIKE_CONTAINS:
- return ByteBufferUtil.contains(leftOperand, rightOperand);
- default:
- // we shouldn't get LIKE, CONTAINS, CONTAINS KEY, or IS NOT here
- throw new AssertionError();
- }
- }
+ public abstract boolean isSatisfiedBy(AbstractType<?> type, ByteBuffer leftOperand, ByteBuffer rightOperand);
public int serializedSize()
{
@@ -263,4 +330,13 @@
{
return this.name();
}
+
+ /**
+ * Checks if this operator is an IN operator.
+ * @return {@code true} if this operator is an IN operator, {@code false} otherwise.
+ */
+ public boolean isIN()
+ {
+ return this == IN;
+ }
}
diff --git a/src/java/org/apache/cassandra/cql3/QualifiedName.java b/src/java/org/apache/cassandra/cql3/QualifiedName.java
new file mode 100644
index 0000000..fb2e110
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/QualifiedName.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3;
+
+import java.util.Locale;
+import java.util.Objects;
+
+/**
+ * Class for the names of the keyspace-prefixed elements (e.g. table, index, view names)
+ */
+public class QualifiedName
+{
+ /**
+ * The keyspace name as stored internally.
+ */
+ private String keyspace;
+ private String name;
+
+ public QualifiedName()
+ {
+ }
+
+ public QualifiedName(String keyspace, String name)
+ {
+ this.keyspace = keyspace;
+ this.name = name;
+ }
+
+ /**
+ * Sets the keyspace.
+ *
+ * @param ks the keyspace name
+ * @param keepCase <code>true</code> if the case must be kept, <code>false</code> otherwise.
+ */
+ public final void setKeyspace(String ks, boolean keepCase)
+ {
+ keyspace = toInternalName(ks, keepCase);
+ }
+
+ /**
+ * Checks if the keyspace is specified.
+ * @return <code>true</code> if the keyspace is specified, <code>false</code> otherwise.
+ */
+ public final boolean hasKeyspace()
+ {
+ return keyspace != null;
+ }
+
+ public final String getKeyspace()
+ {
+ return keyspace;
+ }
+
+ public void setName(String cf, boolean keepCase)
+ {
+ name = toInternalName(cf, keepCase);
+ }
+
+ public String getName()
+ {
+ return name;
+ }
+
+ @Override
+ public String toString()
+ {
+ return hasKeyspace()
+ ? String.format("%s.%s", keyspace, name)
+ : name;
+ }
+
+ @Override
+ public int hashCode()
+ {
+ return Objects.hash(keyspace, name);
+ }
+
+ public boolean equals(Object o)
+ {
+ if (this == o)
+ return true;
+
+ if (!(o instanceof QualifiedName))
+ return false;
+
+ QualifiedName qn = (QualifiedName) o;
+ return Objects.equals(keyspace, qn.keyspace) && name.equals(qn.name);
+ }
+
+ /**
+ * Converts the specified name into the name used internally.
+ *
+ * @param name the name
+ * @param keepCase <code>true</code> if the case must be kept, <code>false</code> otherwise.
+ * @return the name used internally.
+ */
+ private static String toInternalName(String name, boolean keepCase)
+ {
+ return keepCase ? name : name.toLowerCase(Locale.US);
+ }
+}
diff --git a/src/java/org/apache/cassandra/cql3/QueryEvents.java b/src/java/org/apache/cassandra/cql3/QueryEvents.java
new file mode 100644
index 0000000..86ef8a5
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/QueryEvents.java
@@ -0,0 +1,295 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cql3;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.CopyOnWriteArraySet;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Supplier;
+
+import javax.annotation.Nullable;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.cql3.statements.BatchStatement;
+import org.apache.cassandra.service.QueryState;
+import org.apache.cassandra.transport.Message;
+import org.apache.cassandra.transport.messages.ResultMessage;
+import org.apache.cassandra.utils.JVMStabilityInspector;
+import org.apache.cassandra.utils.NoSpamLogger;
+
+public class QueryEvents
+{
+ private static final Logger logger = LoggerFactory.getLogger(QueryEvents.class);
+ private static final NoSpamLogger noSpam1m = NoSpamLogger.getLogger(logger, 1, TimeUnit.MINUTES);
+ public static final QueryEvents instance = new QueryEvents();
+
+ private final Set<Listener> listeners = new CopyOnWriteArraySet<>();
+
+ @VisibleForTesting
+ public int listenerCount()
+ {
+ return listeners.size();
+ }
+
+ public void registerListener(Listener listener)
+ {
+ listeners.add(listener);
+ }
+
+ public void unregisterListener(Listener listener)
+ {
+ listeners.remove(listener);
+ }
+
+ public void notifyQuerySuccess(CQLStatement statement,
+ String query,
+ QueryOptions options,
+ QueryState state,
+ long queryTime,
+ Message.Response response)
+ {
+ try
+ {
+ for (Listener listener : listeners)
+ listener.querySuccess(statement, query, options, state, queryTime, response);
+ }
+ catch (Throwable t)
+ {
+ noSpam1m.error("Failed notifying listeners", t);
+ JVMStabilityInspector.inspectThrowable(t);
+ }
+ }
+
+ public void notifyQueryFailure(CQLStatement statement,
+ String query,
+ QueryOptions options,
+ QueryState state,
+ Exception cause)
+ {
+ try
+ {
+ for (Listener listener : listeners)
+ listener.queryFailure(statement, query, options, state, cause);
+ }
+ catch (Throwable t)
+ {
+ noSpam1m.error("Failed notifying listeners", t);
+ JVMStabilityInspector.inspectThrowable(t);
+ }
+ }
+
+ public void notifyExecuteSuccess(CQLStatement statement,
+ String query,
+ QueryOptions options,
+ QueryState state,
+ long queryTime,
+ Message.Response response)
+ {
+ try
+ {
+ for (Listener listener : listeners)
+ listener.executeSuccess(statement, query, options, state, queryTime, response);
+ }
+ catch (Throwable t)
+ {
+ noSpam1m.error("Failed notifying listeners", t);
+ JVMStabilityInspector.inspectThrowable(t);
+ }
+ }
+
+ public void notifyExecuteFailure(QueryHandler.Prepared prepared,
+ QueryOptions options,
+ QueryState state,
+ Exception cause)
+ {
+ CQLStatement statement = prepared != null ? prepared.statement : null;
+ String query = prepared != null ? prepared.rawCQLStatement : null;
+ try
+ {
+ for (Listener listener : listeners)
+ listener.executeFailure(statement, query, options, state, cause);
+ }
+ catch (Throwable t)
+ {
+ noSpam1m.error("Failed notifying listeners", t);
+ JVMStabilityInspector.inspectThrowable(t);
+ }
+ }
+
+ public void notifyBatchSuccess(BatchStatement.Type batchType,
+ List<? extends CQLStatement> statements,
+ List<String> queries,
+ List<List<ByteBuffer>> values,
+ QueryOptions options,
+ QueryState state,
+ long queryTime,
+ Message.Response response)
+ {
+ try
+ {
+ for (Listener listener : listeners)
+ listener.batchSuccess(batchType, statements, queries, values, options, state, queryTime, response);
+ }
+ catch (Throwable t)
+ {
+ noSpam1m.error("Failed notifying listeners", t);
+ JVMStabilityInspector.inspectThrowable(t);
+ }
+ }
+
+ public void notifyBatchFailure(List<QueryHandler.Prepared> prepared,
+ BatchStatement.Type batchType,
+ List<Object> queryOrIdList,
+ List<List<ByteBuffer>> values,
+ QueryOptions options,
+ QueryState state,
+ Exception cause)
+ {
+ if (hasListeners())
+ {
+ List<CQLStatement> statements = new ArrayList<>(queryOrIdList.size());
+ List<String> queries = new ArrayList<>(queryOrIdList.size());
+ if (prepared != null)
+ {
+ prepared.forEach(p -> {
+ statements.add(p.statement);
+ queries.add(p.rawCQLStatement);
+ });
+ }
+ try
+ {
+ for (Listener listener : listeners)
+ listener.batchFailure(batchType, statements, queries, values, options, state, cause);
+ }
+ catch (Throwable t)
+ {
+ noSpam1m.error("Failed notifying listeners", t);
+ JVMStabilityInspector.inspectThrowable(t);
+ }
+ }
+ }
+
+ public void notifyPrepareSuccess(Supplier<QueryHandler.Prepared> preparedProvider,
+ String query,
+ QueryState state,
+ long queryTime,
+ ResultMessage.Prepared response)
+ {
+ if (hasListeners())
+ {
+ QueryHandler.Prepared prepared = preparedProvider.get();
+ if (prepared != null)
+ {
+ try
+ {
+ for (Listener listener : listeners)
+ listener.prepareSuccess(prepared.statement, query, state, queryTime, response);
+ }
+ catch (Throwable t)
+ {
+ noSpam1m.error("Failed notifying listeners", t);
+ JVMStabilityInspector.inspectThrowable(t);
+ }
+ }
+ else
+ {
+ // this means that queryHandler.prepare was successful, but then immediately after we can't find the prepared query in the cache, should be very rare
+ notifyPrepareFailure(null, query, state, new RuntimeException("Successfully prepared, but could not find prepared statement for " + response.statementId));
+ }
+ }
+ }
+
+ public void notifyPrepareFailure(@Nullable CQLStatement statement, String query, QueryState state, Exception cause)
+ {
+ try
+ {
+ for (Listener listener : listeners)
+ listener.prepareFailure(statement, query, state, cause);
+ }
+ catch (Throwable t)
+ {
+ noSpam1m.error("Failed notifying listeners", t);
+ JVMStabilityInspector.inspectThrowable(t);
+ }
+ }
+
+ public boolean hasListeners()
+ {
+ return !listeners.isEmpty();
+ }
+
+ public static interface Listener
+ {
+ default void querySuccess(CQLStatement statement,
+ String query,
+ QueryOptions options,
+ QueryState state,
+ long queryTime,
+ Message.Response response) {}
+ default void queryFailure(@Nullable CQLStatement statement,
+ String query,
+ QueryOptions options,
+ QueryState state,
+ Exception cause) {}
+
+ default void executeSuccess(CQLStatement statement,
+ String query,
+ QueryOptions options,
+ QueryState state,
+ long queryTime,
+ Message.Response response) {}
+ default void executeFailure(@Nullable CQLStatement statement,
+ @Nullable String query,
+ QueryOptions options,
+ QueryState state,
+ Exception cause) {}
+
+ default void batchSuccess(BatchStatement.Type batchType,
+ List<? extends CQLStatement> statements,
+ List<String> queries,
+ List<List<ByteBuffer>> values,
+ QueryOptions options,
+ QueryState state,
+ long queryTime,
+ Message.Response response) {}
+ default void batchFailure(BatchStatement.Type batchType,
+ List<? extends CQLStatement> statements,
+ List<String> queries,
+ List<List<ByteBuffer>> values,
+ QueryOptions options,
+ QueryState state,
+ Exception cause) {}
+
+ default void prepareSuccess(CQLStatement statement,
+ String query,
+ QueryState state,
+ long queryTime,
+ ResultMessage.Prepared response) {}
+ default void prepareFailure(@Nullable CQLStatement statement,
+ String query,
+ QueryState state,
+ Exception cause) {}
+ }
+}
diff --git a/src/java/org/apache/cassandra/cql3/QueryHandler.java b/src/java/org/apache/cassandra/cql3/QueryHandler.java
index 2108d4c..2231484 100644
--- a/src/java/org/apache/cassandra/cql3/QueryHandler.java
+++ b/src/java/org/apache/cassandra/cql3/QueryHandler.java
@@ -21,28 +21,28 @@
import java.util.Map;
import org.apache.cassandra.cql3.statements.BatchStatement;
-import org.apache.cassandra.cql3.statements.ParsedStatement;
import org.apache.cassandra.exceptions.RequestExecutionException;
import org.apache.cassandra.exceptions.RequestValidationException;
+import org.apache.cassandra.service.ClientState;
import org.apache.cassandra.service.QueryState;
import org.apache.cassandra.transport.messages.ResultMessage;
import org.apache.cassandra.utils.MD5Digest;
public interface QueryHandler
{
- ResultMessage process(String query,
+ CQLStatement parse(String queryString, QueryState queryState, QueryOptions options);
+
+ ResultMessage process(CQLStatement statement,
QueryState state,
QueryOptions options,
Map<String, ByteBuffer> customPayload,
long queryStartNanoTime) throws RequestExecutionException, RequestValidationException;
ResultMessage.Prepared prepare(String query,
- QueryState state,
+ ClientState clientState,
Map<String, ByteBuffer> customPayload) throws RequestValidationException;
- ParsedStatement.Prepared getPrepared(MD5Digest id);
-
- ParsedStatement.Prepared getPreparedForThrift(Integer id);
+ QueryHandler.Prepared getPrepared(MD5Digest id);
ResultMessage processPrepared(CQLStatement statement,
QueryState state,
@@ -55,4 +55,30 @@
BatchQueryOptions options,
Map<String, ByteBuffer> customPayload,
long queryStartNanoTime) throws RequestExecutionException, RequestValidationException;
+
+ public static class Prepared
+ {
+ public final CQLStatement statement;
+
+ public final MD5Digest resultMetadataId;
+
+ /**
+ * Contains the CQL statement source if the statement has been "regularly" perpared via
+ * {@link QueryHandler#prepare(String, ClientState, Map)}.
+ * Other usages of this class may or may not contain the CQL statement source.
+ */
+ public final String rawCQLStatement;
+
+ public Prepared(CQLStatement statement)
+ {
+ this(statement, "");
+ }
+
+ public Prepared(CQLStatement statement, String rawCQLStatement)
+ {
+ this.statement = statement;
+ this.rawCQLStatement = rawCQLStatement;
+ this.resultMetadataId = ResultSet.ResultMetadata.fromPrepared(statement).getResultMetadataId();
+ }
+ }
}
diff --git a/src/java/org/apache/cassandra/cql3/QueryOptions.java b/src/java/org/apache/cassandra/cql3/QueryOptions.java
index a29c6ff..d3b1a03 100644
--- a/src/java/org/apache/cassandra/cql3/QueryOptions.java
+++ b/src/java/org/apache/cassandra/cql3/QueryOptions.java
@@ -24,7 +24,7 @@
import io.netty.buffer.ByteBuf;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.db.ConsistencyLevel;
import org.apache.cassandra.db.marshal.UTF8Type;
import org.apache.cassandra.exceptions.InvalidRequestException;
@@ -35,6 +35,8 @@
import org.apache.cassandra.transport.ProtocolException;
import org.apache.cassandra.transport.ProtocolVersion;
import org.apache.cassandra.utils.Pair;
+import org.apache.commons.lang3.builder.ToStringBuilder;
+import org.apache.commons.lang3.builder.ToStringStyle;
/**
* Options for a query.
@@ -42,7 +44,7 @@
public abstract class QueryOptions
{
public static final QueryOptions DEFAULT = new DefaultQueryOptions(ConsistencyLevel.ONE,
- Collections.<ByteBuffer>emptyList(),
+ Collections.emptyList(),
false,
SpecificOptions.DEFAULT,
ProtocolVersion.CURRENT);
@@ -52,11 +54,6 @@
// A cache of bind values parsed as JSON, see getJsonColumnValue for details.
private List<Map<ColumnIdentifier, Term>> jsonValuesCache;
- public static QueryOptions fromThrift(ConsistencyLevel consistency, List<ByteBuffer> values)
- {
- return new DefaultQueryOptions(consistency, values, false, SpecificOptions.DEFAULT, ProtocolVersion.V3);
- }
-
public static QueryOptions forInternalCalls(ConsistencyLevel consistency, List<ByteBuffer> values)
{
return new DefaultQueryOptions(consistency, values, false, SpecificOptions.DEFAULT, ProtocolVersion.V3);
@@ -72,9 +69,34 @@
return new DefaultQueryOptions(null, null, true, null, protocolVersion);
}
- public static QueryOptions create(ConsistencyLevel consistency, List<ByteBuffer> values, boolean skipMetadata, int pageSize, PagingState pagingState, ConsistencyLevel serialConsistency, ProtocolVersion version)
+ public static QueryOptions create(ConsistencyLevel consistency,
+ List<ByteBuffer> values,
+ boolean skipMetadata,
+ int pageSize,
+ PagingState pagingState,
+ ConsistencyLevel serialConsistency,
+ ProtocolVersion version,
+ String keyspace)
{
- return new DefaultQueryOptions(consistency, values, skipMetadata, new SpecificOptions(pageSize, pagingState, serialConsistency, Long.MIN_VALUE), version);
+ return create(consistency, values, skipMetadata, pageSize, pagingState, serialConsistency, version, keyspace, Long.MIN_VALUE, Integer.MIN_VALUE);
+ }
+
+ public static QueryOptions create(ConsistencyLevel consistency,
+ List<ByteBuffer> values,
+ boolean skipMetadata,
+ int pageSize,
+ PagingState pagingState,
+ ConsistencyLevel serialConsistency,
+ ProtocolVersion version,
+ String keyspace,
+ long timestamp,
+ int nowInSeconds)
+ {
+ return new DefaultQueryOptions(consistency,
+ values,
+ skipMetadata,
+ new SpecificOptions(pageSize, pagingState, serialConsistency, timestamp, keyspace, nowInSeconds),
+ version);
}
public static QueryOptions addColumnSpecifications(QueryOptions options, List<ColumnSpecification> columnSpecs)
@@ -91,11 +113,11 @@
*
* This is functionally equivalent to:
* {@code Json.parseJson(UTF8Type.instance.getSerializer().deserialize(getValues().get(bindIndex)), expectedReceivers).get(columnName)}
- * but this cache the result of parsing the JSON so that while this might be called for multiple columns on the same {@code bindIndex}
+ * but this caches the result of parsing the JSON, so that while this might be called for multiple columns on the same {@code bindIndex}
* value, the underlying JSON value is only parsed/processed once.
*
- * Note: this is a bit more involved in CQL specifics than this class generally is but we as we need to cache this per-query and in an object
- * that is available when we bind values, this is the easier place to have this.
+ * Note: this is a bit more involved in CQL specifics than this class generally is, but as we need to cache this per-query and in an object
+ * that is available when we bind values, this is the easiest place to have this.
*
* @param bindIndex the index of the bind value that should be interpreted as a JSON value.
* @param columnName the name of the column we want the value of.
@@ -106,7 +128,7 @@
* @return the value correspong to column {@code columnName} in the (JSON) bind value at index {@code bindIndex}. This may return null if the
* JSON value has no value for this column.
*/
- public Term getJsonColumnValue(int bindIndex, ColumnIdentifier columnName, Collection<ColumnDefinition> expectedReceivers) throws InvalidRequestException
+ public Term getJsonColumnValue(int bindIndex, ColumnIdentifier columnName, Collection<ColumnMetadata> expectedReceivers) throws InvalidRequestException
{
if (jsonValuesCache == null)
jsonValuesCache = new ArrayList<>(Collections.<Map<ColumnIdentifier, Term>>nCopies(getValues().size(), null));
@@ -141,7 +163,7 @@
*
* <p>The column specifications will be present only for prepared statements.</p>
*
- * <p>Invoke the {@link hasColumnSpecifications} method before invoking this method in order to ensure that this
+ * <p>Invoke the {@link #hasColumnSpecifications} method before invoking this method in order to ensure that this
* <code>QueryOptions</code> contains the column specifications.</p>
*
* @return the option names
@@ -177,9 +199,17 @@
return tstamp != Long.MIN_VALUE ? tstamp : state.getTimestamp();
}
+ public int getNowInSeconds(QueryState state)
+ {
+ int nowInSeconds = getSpecificOptions().nowInSeconds;
+ return nowInSeconds != Integer.MIN_VALUE ? nowInSeconds : state.getNowInSeconds();
+ }
+
+ /** The keyspace that this query is bound to, or null if not relevant. */
+ public String getKeyspace() { return getSpecificOptions().keyspace; }
+
/**
- * The protocol version for the query. Will be 3 if the object don't come from
- * a native protocol request (i.e. it's been allocated locally or by CQL-over-thrift).
+ * The protocol version for the query.
*/
public abstract ProtocolVersion getProtocolVersion();
@@ -320,7 +350,7 @@
{
super.prepare(specs);
- orderedValues = new ArrayList<ByteBuffer>(specs.size());
+ orderedValues = new ArrayList<>(specs.size());
for (int i = 0; i < specs.size(); i++)
{
String name = specs.get(i).name.toString();
@@ -347,19 +377,28 @@
// Options that are likely to not be present in most queries
static class SpecificOptions
{
- private static final SpecificOptions DEFAULT = new SpecificOptions(-1, null, null, Long.MIN_VALUE);
+ private static final SpecificOptions DEFAULT = new SpecificOptions(-1, null, null, Long.MIN_VALUE, null, Integer.MIN_VALUE);
private final int pageSize;
private final PagingState state;
private final ConsistencyLevel serialConsistency;
private final long timestamp;
+ private final String keyspace;
+ private final int nowInSeconds;
- private SpecificOptions(int pageSize, PagingState state, ConsistencyLevel serialConsistency, long timestamp)
+ private SpecificOptions(int pageSize,
+ PagingState state,
+ ConsistencyLevel serialConsistency,
+ long timestamp,
+ String keyspace,
+ int nowInSeconds)
{
this.pageSize = pageSize;
this.state = state;
this.serialConsistency = serialConsistency == null ? ConsistencyLevel.SERIAL : serialConsistency;
this.timestamp = timestamp;
+ this.keyspace = keyspace;
+ this.nowInSeconds = nowInSeconds;
}
}
@@ -374,7 +413,9 @@
PAGING_STATE,
SERIAL_CONSISTENCY,
TIMESTAMP,
- NAMES_FOR_VALUES;
+ NAMES_FOR_VALUES,
+ KEYSPACE,
+ NOW_IN_SECONDS;
private static final Flag[] ALL_VALUES = values();
@@ -429,7 +470,7 @@
if (!flags.isEmpty())
{
int pageSize = flags.contains(Flag.PAGE_SIZE) ? body.readInt() : -1;
- PagingState pagingState = flags.contains(Flag.PAGING_STATE) ? PagingState.deserialize(CBUtil.readValue(body), version) : null;
+ PagingState pagingState = flags.contains(Flag.PAGING_STATE) ? PagingState.deserialize(CBUtil.readValueNoCopy(body), version) : null;
ConsistencyLevel serialConsistency = flags.contains(Flag.SERIAL_CONSISTENCY) ? CBUtil.readConsistencyLevel(body) : ConsistencyLevel.SERIAL;
long timestamp = Long.MIN_VALUE;
if (flags.contains(Flag.TIMESTAMP))
@@ -439,9 +480,11 @@
throw new ProtocolException(String.format("Out of bound timestamp, must be in [%d, %d] (got %d)", Long.MIN_VALUE + 1, Long.MAX_VALUE, ts));
timestamp = ts;
}
-
- options = new SpecificOptions(pageSize, pagingState, serialConsistency, timestamp);
+ String keyspace = flags.contains(Flag.KEYSPACE) ? CBUtil.readString(body) : null;
+ int nowInSeconds = flags.contains(Flag.NOW_IN_SECONDS) ? body.readInt() : Integer.MIN_VALUE;
+ options = new SpecificOptions(pageSize, pagingState, serialConsistency, timestamp, keyspace, nowInSeconds);
}
+
DefaultQueryOptions opts = new DefaultQueryOptions(consistency, values, skipMetadata, options, version);
return names == null ? opts : new OptionsWithNames(opts, names);
}
@@ -450,7 +493,7 @@
{
CBUtil.writeConsistencyLevel(options.getConsistency(), dest);
- EnumSet<Flag> flags = gatherFlags(options);
+ EnumSet<Flag> flags = gatherFlags(options, version);
if (version.isGreaterOrEqualTo(ProtocolVersion.V5))
dest.writeInt(Flag.serialize(flags));
else
@@ -466,6 +509,10 @@
CBUtil.writeConsistencyLevel(options.getSerialConsistency(), dest);
if (flags.contains(Flag.TIMESTAMP))
dest.writeLong(options.getSpecificOptions().timestamp);
+ if (flags.contains(Flag.KEYSPACE))
+ CBUtil.writeAsciiString(options.getSpecificOptions().keyspace, dest);
+ if (flags.contains(Flag.NOW_IN_SECONDS))
+ dest.writeInt(options.getSpecificOptions().nowInSeconds);
// Note that we don't really have to bother with NAMES_FOR_VALUES server side,
// and in fact we never really encode QueryOptions, only decode them, so we
@@ -478,7 +525,7 @@
size += CBUtil.sizeOfConsistencyLevel(options.getConsistency());
- EnumSet<Flag> flags = gatherFlags(options);
+ EnumSet<Flag> flags = gatherFlags(options, version);
size += (version.isGreaterOrEqualTo(ProtocolVersion.V5) ? 4 : 1);
if (flags.contains(Flag.VALUES))
@@ -491,11 +538,15 @@
size += CBUtil.sizeOfConsistencyLevel(options.getSerialConsistency());
if (flags.contains(Flag.TIMESTAMP))
size += 8;
+ if (flags.contains(Flag.KEYSPACE))
+ size += CBUtil.sizeOfAsciiString(options.getSpecificOptions().keyspace);
+ if (flags.contains(Flag.NOW_IN_SECONDS))
+ size += 4;
return size;
}
- private EnumSet<Flag> gatherFlags(QueryOptions options)
+ private EnumSet<Flag> gatherFlags(QueryOptions options, ProtocolVersion version)
{
EnumSet<Flag> flags = EnumSet.noneOf(Flag.class);
if (options.getValues().size() > 0)
@@ -510,7 +561,22 @@
flags.add(Flag.SERIAL_CONSISTENCY);
if (options.getSpecificOptions().timestamp != Long.MIN_VALUE)
flags.add(Flag.TIMESTAMP);
+
+ if (version.isGreaterOrEqualTo(ProtocolVersion.V5))
+ {
+ if (options.getSpecificOptions().keyspace != null)
+ flags.add(Flag.KEYSPACE);
+ if (options.getSpecificOptions().nowInSeconds != Integer.MIN_VALUE)
+ flags.add(Flag.NOW_IN_SECONDS);
+ }
+
return flags;
}
}
+
+ @Override
+ public String toString()
+ {
+ return ToStringBuilder.reflectionToString(this, ToStringStyle.SHORT_PREFIX_STYLE);
+ }
}
diff --git a/src/java/org/apache/cassandra/cql3/QueryProcessor.java b/src/java/org/apache/cassandra/cql3/QueryProcessor.java
index 0c60735..0b94ec0 100644
--- a/src/java/org/apache/cassandra/cql3/QueryProcessor.java
+++ b/src/java/org/apache/cassandra/cql3/QueryProcessor.java
@@ -24,20 +24,22 @@
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
+import com.github.benmanes.caffeine.cache.Cache;
+import com.github.benmanes.caffeine.cache.Caffeine;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Predicate;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Iterators;
+import com.google.common.collect.*;
import com.google.common.primitives.Ints;
+import com.google.common.util.concurrent.MoreExecutors;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.googlecode.concurrentlinkedhashmap.ConcurrentLinkedHashMap;
import org.antlr.runtime.*;
import org.apache.cassandra.concurrent.ScheduledExecutors;
import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.config.Schema;
-import org.apache.cassandra.config.SchemaConstants;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.SchemaChangeListener;
+import org.apache.cassandra.schema.SchemaConstants;
import org.apache.cassandra.cql3.functions.Function;
import org.apache.cassandra.cql3.functions.FunctionName;
import org.apache.cassandra.cql3.statements.*;
@@ -50,7 +52,6 @@
import org.apache.cassandra.metrics.CQLMetrics;
import org.apache.cassandra.service.*;
import org.apache.cassandra.service.pager.QueryPager;
-import org.apache.cassandra.thrift.ThriftClientState;
import org.apache.cassandra.tracing.Tracing;
import org.apache.cassandra.transport.ProtocolVersion;
import org.apache.cassandra.transport.messages.ResultMessage;
@@ -60,62 +61,50 @@
public class QueryProcessor implements QueryHandler
{
- public static final CassandraVersion CQL_VERSION = new CassandraVersion("3.4.4");
+ public static final CassandraVersion CQL_VERSION = new CassandraVersion("3.4.5");
public static final QueryProcessor instance = new QueryProcessor();
private static final Logger logger = LoggerFactory.getLogger(QueryProcessor.class);
- private static final ConcurrentLinkedHashMap<MD5Digest, ParsedStatement.Prepared> preparedStatements;
- private static final ConcurrentLinkedHashMap<Integer, ParsedStatement.Prepared> thriftPreparedStatements;
+ private static final Cache<MD5Digest, Prepared> preparedStatements;
// A map for prepared statements used internally (which we don't want to mix with user statement, in particular we don't
// bother with expiration on those.
- private static final ConcurrentMap<String, ParsedStatement.Prepared> internalStatements = new ConcurrentHashMap<>();
+ private static final ConcurrentMap<String, Prepared> internalStatements = new ConcurrentHashMap<>();
// Direct calls to processStatement do not increment the preparedStatementsExecuted/regularStatementsExecuted
// counters. Callers of processStatement are responsible for correctly notifying metrics
public static final CQLMetrics metrics = new CQLMetrics();
private static final AtomicInteger lastMinuteEvictionsCount = new AtomicInteger(0);
- private static final AtomicInteger thriftLastMinuteEvictionsCount = new AtomicInteger(0);
static
{
- preparedStatements = new ConcurrentLinkedHashMap.Builder<MD5Digest, ParsedStatement.Prepared>()
- .maximumWeightedCapacity(capacityToBytes(DatabaseDescriptor.getPreparedStatementsCacheSizeMB()))
+ preparedStatements = Caffeine.newBuilder()
+ .executor(MoreExecutors.directExecutor())
+ .maximumWeight(capacityToBytes(DatabaseDescriptor.getPreparedStatementsCacheSizeMB()))
.weigher(QueryProcessor::measure)
- .listener((md5Digest, prepared) -> {
- metrics.preparedStatementsEvicted.inc();
- lastMinuteEvictionsCount.incrementAndGet();
- SystemKeyspace.removePreparedStatement(md5Digest);
+ .removalListener((key, prepared, cause) -> {
+ MD5Digest md5Digest = (MD5Digest) key;
+ if (cause.wasEvicted())
+ {
+ metrics.preparedStatementsEvicted.inc();
+ lastMinuteEvictionsCount.incrementAndGet();
+ SystemKeyspace.removePreparedStatement(md5Digest);
+ }
}).build();
- thriftPreparedStatements = new ConcurrentLinkedHashMap.Builder<Integer, ParsedStatement.Prepared>()
- .maximumWeightedCapacity(capacityToBytes(DatabaseDescriptor.getThriftPreparedStatementsCacheSizeMB()))
- .weigher(QueryProcessor::measure)
- .listener((integer, prepared) -> {
- metrics.preparedStatementsEvicted.inc();
- thriftLastMinuteEvictionsCount.incrementAndGet();
- })
- .build();
-
ScheduledExecutors.scheduledTasks.scheduleAtFixedRate(() -> {
long count = lastMinuteEvictionsCount.getAndSet(0);
if (count > 0)
logger.warn("{} prepared statements discarded in the last minute because cache limit reached ({} MB)",
count,
DatabaseDescriptor.getPreparedStatementsCacheSizeMB());
- count = thriftLastMinuteEvictionsCount.getAndSet(0);
- if (count > 0)
- logger.warn("{} prepared Thrift statements discarded in the last minute because cache limit reached ({} MB)",
- count,
- DatabaseDescriptor.getThriftPreparedStatementsCacheSizeMB());
}, 1, 1, TimeUnit.MINUTES);
- logger.info("Initialized prepared statement caches with {} MB (native) and {} MB (Thrift)",
- DatabaseDescriptor.getPreparedStatementsCacheSizeMB(),
- DatabaseDescriptor.getThriftPreparedStatementsCacheSizeMB());
+ logger.info("Initialized prepared statement caches with {} MB",
+ DatabaseDescriptor.getPreparedStatementsCacheSizeMB());
}
private static long capacityToBytes(long cacheSizeMB)
@@ -125,21 +114,19 @@
public static int preparedStatementsCount()
{
- return preparedStatements.size() + thriftPreparedStatements.size();
+ return preparedStatements.asMap().size();
}
// Work around initialization dependency
- private static enum InternalStateInstance
+ private enum InternalStateInstance
{
INSTANCE;
- private final QueryState queryState;
+ private final ClientState clientState;
InternalStateInstance()
{
- ClientState state = ClientState.forInternalCalls();
- state.setKeyspace(SchemaConstants.SYSTEM_KEYSPACE_NAME);
- this.queryState = new QueryState(state);
+ clientState = ClientState.forInternalCalls(SchemaConstants.SYSTEM_KEYSPACE_NAME);
}
}
@@ -152,7 +139,7 @@
try
{
clientState.setKeyspace(useKeyspaceAndCQL.left);
- prepare(useKeyspaceAndCQL.right, clientState, false);
+ prepare(useKeyspaceAndCQL.right, clientState);
count++;
}
catch (RequestValidationException e)
@@ -170,8 +157,7 @@
@VisibleForTesting
public static void clearPreparedStatements(boolean memoryOnly)
{
- preparedStatements.clear();
- thriftPreparedStatements.clear();
+ preparedStatements.invalidateAll();
if (!memoryOnly)
SystemKeyspace.resetPreparedStatements();
}
@@ -179,22 +165,17 @@
@VisibleForTesting
public static QueryState internalQueryState()
{
- return InternalStateInstance.INSTANCE.queryState;
+ return new QueryState(InternalStateInstance.INSTANCE.clientState);
}
private QueryProcessor()
{
- MigrationManager.instance.register(new MigrationSubscriber());
+ Schema.instance.registerListener(new StatementInvalidatingListener());
}
- public ParsedStatement.Prepared getPrepared(MD5Digest id)
+ public Prepared getPrepared(MD5Digest id)
{
- return preparedStatements.get(id);
- }
-
- public ParsedStatement.Prepared getPreparedForThrift(Integer id)
- {
- return thriftPreparedStatements.get(id);
+ return preparedStatements.getIfPresent(id);
}
public static void validateKey(ByteBuffer key) throws InvalidRequestException
@@ -219,35 +200,51 @@
{
logger.trace("Process {} @CL.{}", statement, options.getConsistency());
ClientState clientState = queryState.getClientState();
- statement.checkAccess(clientState);
+ statement.authorize(clientState);
statement.validate(clientState);
- ResultMessage result = statement.execute(queryState, options, queryStartNanoTime);
+ ResultMessage result;
+ if (options.getConsistency() == ConsistencyLevel.NODE_LOCAL)
+ {
+ assert Boolean.getBoolean("cassandra.enable_nodelocal_queries") : "Node local consistency level is highly dangerous and should be used only for debugging purposes";
+ assert statement instanceof SelectStatement : "Only SELECT statements are permitted for node-local execution";
+ logger.info("Statement {} executed with NODE_LOCAL consistency level.", statement);
+ result = statement.executeLocally(queryState, options);
+ }
+ else
+ {
+ result = statement.execute(queryState, options, queryStartNanoTime);
+ }
return result == null ? new ResultMessage.Void() : result;
}
public static ResultMessage process(String queryString, ConsistencyLevel cl, QueryState queryState, long queryStartNanoTime)
throws RequestExecutionException, RequestValidationException
{
- return instance.process(queryString, queryState, QueryOptions.forInternalCalls(cl, Collections.<ByteBuffer>emptyList()), queryStartNanoTime);
+ QueryOptions options = QueryOptions.forInternalCalls(cl, Collections.<ByteBuffer>emptyList());
+ CQLStatement statement = instance.parse(queryString, queryState, options);
+ return instance.process(statement, queryState, options, queryStartNanoTime);
}
- public ResultMessage process(String query,
+ public CQLStatement parse(String queryString, QueryState queryState, QueryOptions options)
+ {
+ return getStatement(queryString, queryState.getClientState().cloneWithKeyspaceIfSet(options.getKeyspace()));
+ }
+
+ public ResultMessage process(CQLStatement statement,
QueryState state,
QueryOptions options,
Map<String, ByteBuffer> customPayload,
long queryStartNanoTime) throws RequestExecutionException, RequestValidationException
{
- return process(query, state, options, queryStartNanoTime);
+ return process(statement, state, options, queryStartNanoTime);
}
- public ResultMessage process(String queryString, QueryState queryState, QueryOptions options, long queryStartNanoTime)
+ public ResultMessage process(CQLStatement prepared, QueryState queryState, QueryOptions options, long queryStartNanoTime)
throws RequestExecutionException, RequestValidationException
{
- ParsedStatement.Prepared p = getStatement(queryString, queryState.getClientState());
- options.prepare(p.boundNames);
- CQLStatement prepared = p.statement;
- if (prepared.getBoundTerms() != options.getValues().size())
+ options.prepare(prepared.getBindVariables());
+ if (prepared.getBindVariables().size() != options.getValues().size())
throw new InvalidRequestException("Invalid amount of bind variables");
if (!queryState.getClientState().isInternal)
@@ -256,9 +253,9 @@
return processStatement(prepared, queryState, options, queryStartNanoTime);
}
- public static ParsedStatement.Prepared parseStatement(String queryStr, QueryState queryState) throws RequestValidationException
+ public static CQLStatement parseStatement(String queryStr, ClientState clientState) throws RequestValidationException
{
- return getStatement(queryStr, queryState.getClientState());
+ return getStatement(queryStr, clientState);
}
public static UntypedResultSet process(String query, ConsistencyLevel cl) throws RequestExecutionException
@@ -268,7 +265,10 @@
public static UntypedResultSet process(String query, ConsistencyLevel cl, List<ByteBuffer> values) throws RequestExecutionException
{
- ResultMessage result = instance.process(query, QueryState.forInternalCalls(), QueryOptions.forInternalCalls(cl, values), System.nanoTime());
+ QueryState queryState = QueryState.forInternalCalls();
+ QueryOptions options = QueryOptions.forInternalCalls(cl, values);
+ CQLStatement statement = instance.parse(query, queryState, options);
+ ResultMessage result = instance.process(statement, queryState, options, System.nanoTime());
if (result instanceof ResultMessage.Rows)
return UntypedResultSet.create(((ResultMessage.Rows)result).result);
else
@@ -276,43 +276,45 @@
}
@VisibleForTesting
- public static QueryOptions makeInternalOptions(ParsedStatement.Prepared prepared, Object[] values)
+ public static QueryOptions makeInternalOptions(CQLStatement prepared, Object[] values)
{
return makeInternalOptions(prepared, values, ConsistencyLevel.ONE);
}
- private static QueryOptions makeInternalOptions(ParsedStatement.Prepared prepared, Object[] values, ConsistencyLevel cl)
+ private static QueryOptions makeInternalOptions(CQLStatement prepared, Object[] values, ConsistencyLevel cl)
{
- if (prepared.boundNames.size() != values.length)
- throw new IllegalArgumentException(String.format("Invalid number of values. Expecting %d but got %d", prepared.boundNames.size(), values.length));
+ if (prepared.getBindVariables().size() != values.length)
+ throw new IllegalArgumentException(String.format("Invalid number of values. Expecting %d but got %d", prepared.getBindVariables().size(), values.length));
List<ByteBuffer> boundValues = new ArrayList<>(values.length);
for (int i = 0; i < values.length; i++)
{
Object value = values[i];
- AbstractType type = prepared.boundNames.get(i).type;
+ AbstractType type = prepared.getBindVariables().get(i).type;
boundValues.add(value instanceof ByteBuffer || value == null ? (ByteBuffer)value : type.decompose(value));
}
return QueryOptions.forInternalCalls(cl, boundValues);
}
- public static ParsedStatement.Prepared prepareInternal(String query) throws RequestValidationException
+ public static Prepared prepareInternal(String query) throws RequestValidationException
{
- ParsedStatement.Prepared prepared = internalStatements.get(query);
+ Prepared prepared = internalStatements.get(query);
if (prepared != null)
return prepared;
// Note: if 2 threads prepare the same query, we'll live so don't bother synchronizing
- prepared = parseStatement(query, internalQueryState());
- prepared.statement.validate(internalQueryState().getClientState());
- internalStatements.putIfAbsent(query, prepared);
+ CQLStatement statement = parseStatement(query, internalQueryState().getClientState());
+ statement.validate(internalQueryState().getClientState());
+
+ prepared = new Prepared(statement);
+ internalStatements.put(query, prepared);
return prepared;
}
public static UntypedResultSet executeInternal(String query, Object... values)
{
- ParsedStatement.Prepared prepared = prepareInternal(query);
- ResultMessage result = prepared.statement.executeInternal(internalQueryState(), makeInternalOptions(prepared, values));
+ Prepared prepared = prepareInternal(query);
+ ResultMessage result = prepared.statement.executeLocally(internalQueryState(), makeInternalOptions(prepared.statement, values));
if (result instanceof ResultMessage.Rows)
return UntypedResultSet.create(((ResultMessage.Rows)result).result);
else
@@ -330,8 +332,8 @@
{
try
{
- ParsedStatement.Prepared prepared = prepareInternal(query);
- ResultMessage result = prepared.statement.execute(state, makeInternalOptions(prepared, values, cl), System.nanoTime());
+ Prepared prepared = prepareInternal(query);
+ ResultMessage result = prepared.statement.execute(state, makeInternalOptions(prepared.statement, values, cl), System.nanoTime());
if (result instanceof ResultMessage.Rows)
return UntypedResultSet.create(((ResultMessage.Rows)result).result);
else
@@ -345,24 +347,24 @@
public static UntypedResultSet executeInternalWithPaging(String query, int pageSize, Object... values)
{
- ParsedStatement.Prepared prepared = prepareInternal(query);
+ Prepared prepared = prepareInternal(query);
if (!(prepared.statement instanceof SelectStatement))
throw new IllegalArgumentException("Only SELECTs can be paged");
SelectStatement select = (SelectStatement)prepared.statement;
- QueryPager pager = select.getQuery(makeInternalOptions(prepared, values), FBUtilities.nowInSeconds()).getPager(null, ProtocolVersion.CURRENT);
+ QueryPager pager = select.getQuery(makeInternalOptions(prepared.statement, values), FBUtilities.nowInSeconds()).getPager(null, ProtocolVersion.CURRENT);
return UntypedResultSet.create(select, pager, pageSize);
}
/**
- * Same than executeInternal, but to use for queries we know are only executed once so that the
+ * Same than executeLocally, but to use for queries we know are only executed once so that the
* created statement object is not cached.
*/
public static UntypedResultSet executeOnceInternal(String query, Object... values)
{
- ParsedStatement.Prepared prepared = parseStatement(query, internalQueryState());
- prepared.statement.validate(internalQueryState().getClientState());
- ResultMessage result = prepared.statement.executeInternal(internalQueryState(), makeInternalOptions(prepared, values));
+ CQLStatement statement = parseStatement(query, internalQueryState().getClientState());
+ statement.validate(internalQueryState().getClientState());
+ ResultMessage result = statement.executeLocally(internalQueryState(), makeInternalOptions(statement, values));
if (result instanceof ResultMessage.Rows)
return UntypedResultSet.create(((ResultMessage.Rows)result).result);
else
@@ -370,16 +372,16 @@
}
/**
- * A special version of executeInternal that takes the time used as "now" for the query in argument.
+ * A special version of executeLocally that takes the time used as "now" for the query in argument.
* Note that this only make sense for Selects so this only accept SELECT statements and is only useful in rare
* cases.
*/
public static UntypedResultSet executeInternalWithNow(int nowInSec, long queryStartNanoTime, String query, Object... values)
{
- ParsedStatement.Prepared prepared = prepareInternal(query);
+ Prepared prepared = prepareInternal(query);
assert prepared.statement instanceof SelectStatement;
SelectStatement select = (SelectStatement)prepared.statement;
- ResultMessage result = select.executeInternal(internalQueryState(), makeInternalOptions(prepared, values), nowInSec, queryStartNanoTime);
+ ResultMessage result = select.executeInternal(internalQueryState(), makeInternalOptions(prepared.statement, values), nowInSec, queryStartNanoTime);
assert result instanceof ResultMessage.Rows;
return UntypedResultSet.create(((ResultMessage.Rows)result).result);
}
@@ -393,39 +395,33 @@
{
try (PartitionIterator iter = partitions)
{
- SelectStatement ss = (SelectStatement) getStatement(query, null).statement;
+ SelectStatement ss = (SelectStatement) getStatement(query, null);
ResultSet cqlRows = ss.process(iter, FBUtilities.nowInSeconds());
return UntypedResultSet.create(cqlRows);
}
}
public ResultMessage.Prepared prepare(String query,
- QueryState state,
+ ClientState clientState,
Map<String, ByteBuffer> customPayload) throws RequestValidationException
{
- return prepare(query, state);
+ return prepare(query, clientState);
}
- public ResultMessage.Prepared prepare(String queryString, QueryState queryState)
+ public static ResultMessage.Prepared prepare(String queryString, ClientState clientState)
{
- ClientState cState = queryState.getClientState();
- return prepare(queryString, cState, cState instanceof ThriftClientState);
- }
-
- public static ResultMessage.Prepared prepare(String queryString, ClientState clientState, boolean forThrift)
- {
- ResultMessage.Prepared existing = getStoredPreparedStatement(queryString, clientState.getRawKeyspace(), forThrift);
+ ResultMessage.Prepared existing = getStoredPreparedStatement(queryString, clientState.getRawKeyspace());
if (existing != null)
return existing;
- ParsedStatement.Prepared prepared = getStatement(queryString, clientState);
- prepared.rawCQLStatement = queryString;
- int boundTerms = prepared.statement.getBoundTerms();
+ CQLStatement statement = getStatement(queryString, clientState);
+ Prepared prepared = new Prepared(statement, queryString);
+
+ int boundTerms = statement.getBindVariables().size();
if (boundTerms > FBUtilities.MAX_UNSIGNED_SHORT)
throw new InvalidRequestException(String.format("Too many markers(?). %d markers exceed the allowed maximum of %d", boundTerms, FBUtilities.MAX_UNSIGNED_SHORT));
- assert boundTerms == prepared.boundNames.size();
- return storePreparedStatement(queryString, clientState.getRawKeyspace(), prepared, forThrift);
+ return storePreparedStatement(queryString, clientState.getRawKeyspace(), prepared);
}
private static MD5Digest computeId(String queryString, String keyspace)
@@ -434,69 +430,40 @@
return MD5Digest.compute(toHash);
}
- private static Integer computeThriftId(String queryString, String keyspace)
- {
- String toHash = keyspace == null ? queryString : keyspace + queryString;
- return toHash.hashCode();
- }
-
- private static ResultMessage.Prepared getStoredPreparedStatement(String queryString, String keyspace, boolean forThrift)
+ private static ResultMessage.Prepared getStoredPreparedStatement(String queryString, String keyspace)
throws InvalidRequestException
{
- if (forThrift)
- {
- Integer thriftStatementId = computeThriftId(queryString, keyspace);
- ParsedStatement.Prepared existing = thriftPreparedStatements.get(thriftStatementId);
- if (existing == null)
- return null;
+ MD5Digest statementId = computeId(queryString, keyspace);
+ Prepared existing = preparedStatements.getIfPresent(statementId);
+ if (existing == null)
+ return null;
- checkTrue(queryString.equals(existing.rawCQLStatement),
- String.format("MD5 hash collision: query with the same MD5 hash was already prepared. \n Existing: '%s'", existing.rawCQLStatement));
- return ResultMessage.Prepared.forThrift(thriftStatementId, existing.boundNames);
- }
- else
- {
- MD5Digest statementId = computeId(queryString, keyspace);
- ParsedStatement.Prepared existing = preparedStatements.get(statementId);
- if (existing == null)
- return null;
+ checkTrue(queryString.equals(existing.rawCQLStatement),
+ String.format("MD5 hash collision: query with the same MD5 hash was already prepared. \n Existing: '%s'", existing.rawCQLStatement));
- checkTrue(queryString.equals(existing.rawCQLStatement),
- String.format("MD5 hash collision: query with the same MD5 hash was already prepared. \n Existing: '%s'", existing.rawCQLStatement));
- return new ResultMessage.Prepared(statementId, existing);
- }
+ ResultSet.PreparedMetadata preparedMetadata = ResultSet.PreparedMetadata.fromPrepared(existing.statement);
+ ResultSet.ResultMetadata resultMetadata = ResultSet.ResultMetadata.fromPrepared(existing.statement);
+ return new ResultMessage.Prepared(statementId, resultMetadata.getResultMetadataId(), preparedMetadata, resultMetadata);
}
- private static ResultMessage.Prepared storePreparedStatement(String queryString, String keyspace, ParsedStatement.Prepared prepared, boolean forThrift)
+ private static ResultMessage.Prepared storePreparedStatement(String queryString, String keyspace, Prepared prepared)
throws InvalidRequestException
{
// Concatenate the current keyspace so we don't mix prepared statements between keyspace (#5352).
// (if the keyspace is null, queryString has to have a fully-qualified keyspace so it's fine.
long statementSize = ObjectSizes.measureDeep(prepared.statement);
// don't execute the statement if it's bigger than the allowed threshold
- if (forThrift)
- {
- if (statementSize > capacityToBytes(DatabaseDescriptor.getThriftPreparedStatementsCacheSizeMB()))
- throw new InvalidRequestException(String.format("Prepared statement of size %d bytes is larger than allowed maximum of %d MB: %s...",
- statementSize,
- DatabaseDescriptor.getThriftPreparedStatementsCacheSizeMB(),
- queryString.substring(0, 200)));
- Integer statementId = computeThriftId(queryString, keyspace);
- thriftPreparedStatements.put(statementId, prepared);
- return ResultMessage.Prepared.forThrift(statementId, prepared.boundNames);
- }
- else
- {
- if (statementSize > capacityToBytes(DatabaseDescriptor.getPreparedStatementsCacheSizeMB()))
- throw new InvalidRequestException(String.format("Prepared statement of size %d bytes is larger than allowed maximum of %d MB: %s...",
- statementSize,
- DatabaseDescriptor.getPreparedStatementsCacheSizeMB(),
- queryString.substring(0, 200)));
- MD5Digest statementId = computeId(queryString, keyspace);
- preparedStatements.put(statementId, prepared);
- SystemKeyspace.writePreparedStatement(keyspace, statementId, queryString);
- return new ResultMessage.Prepared(statementId, prepared);
- }
+ if (statementSize > capacityToBytes(DatabaseDescriptor.getPreparedStatementsCacheSizeMB()))
+ throw new InvalidRequestException(String.format("Prepared statement of size %d bytes is larger than allowed maximum of %d MB: %s...",
+ statementSize,
+ DatabaseDescriptor.getPreparedStatementsCacheSizeMB(),
+ queryString.substring(0, 200)));
+ MD5Digest statementId = computeId(queryString, keyspace);
+ preparedStatements.put(statementId, prepared);
+ SystemKeyspace.writePreparedStatement(keyspace, statementId, queryString);
+ ResultSet.PreparedMetadata preparedMetadata = ResultSet.PreparedMetadata.fromPrepared(prepared.statement);
+ ResultSet.ResultMetadata resultMetadata = ResultSet.ResultMetadata.fromPrepared(prepared.statement);
+ return new ResultMessage.Prepared(statementId, resultMetadata.getResultMetadataId(), preparedMetadata, resultMetadata);
}
public ResultMessage processPrepared(CQLStatement statement,
@@ -514,15 +481,14 @@
{
List<ByteBuffer> variables = options.getValues();
// Check to see if there are any bound variables to verify
- if (!(variables.isEmpty() && (statement.getBoundTerms() == 0)))
+ if (!(variables.isEmpty() && statement.getBindVariables().isEmpty()))
{
- if (variables.size() != statement.getBoundTerms())
+ if (variables.size() != statement.getBindVariables().size())
throw new InvalidRequestException(String.format("there were %d markers(?) in CQL but %d bound variables",
- statement.getBoundTerms(),
+ statement.getBindVariables().size(),
variables.size()));
// at this point there is a match in count between markers and variables that is non-zero
-
if (logger.isTraceEnabled())
for (int i = 0; i < variables.size(); i++)
logger.trace("[{}] '{}'", i+1, variables.get(i));
@@ -545,32 +511,32 @@
public ResultMessage processBatch(BatchStatement batch, QueryState queryState, BatchQueryOptions options, long queryStartNanoTime)
throws RequestExecutionException, RequestValidationException
{
- ClientState clientState = queryState.getClientState();
- batch.checkAccess(clientState);
+ ClientState clientState = queryState.getClientState().cloneWithKeyspaceIfSet(options.getKeyspace());
+ batch.authorize(clientState);
batch.validate();
batch.validate(clientState);
return batch.execute(queryState, options, queryStartNanoTime);
}
- public static ParsedStatement.Prepared getStatement(String queryStr, ClientState clientState)
+ public static CQLStatement getStatement(String queryStr, ClientState clientState)
throws RequestValidationException
{
Tracing.trace("Parsing {}", queryStr);
- ParsedStatement statement = parseStatement(queryStr);
+ CQLStatement.Raw statement = parseStatement(queryStr);
// Set keyspace for statement that require login
- if (statement instanceof CFStatement)
- ((CFStatement)statement).prepareKeyspace(clientState);
+ if (statement instanceof QualifiedStatement)
+ ((QualifiedStatement) statement).setKeyspace(clientState);
Tracing.trace("Preparing statement");
return statement.prepare(clientState);
}
- public static <T extends ParsedStatement> T parseStatement(String queryStr, Class<T> klass, String type) throws SyntaxException
+ public static <T extends CQLStatement.Raw> T parseStatement(String queryStr, Class<T> klass, String type) throws SyntaxException
{
try
{
- ParsedStatement stmt = parseStatement(queryStr);
+ CQLStatement.Raw stmt = parseStatement(queryStr);
if (!klass.isAssignableFrom(stmt.getClass()))
throw new IllegalArgumentException("Invalid query, must be a " + type + " statement but was: " + stmt.getClass());
@@ -582,7 +548,7 @@
throw new IllegalArgumentException(e.getMessage(), e);
}
}
- public static ParsedStatement parseStatement(String queryStr) throws SyntaxException
+ public static CQLStatement.Raw parseStatement(String queryStr) throws SyntaxException
{
try
{
@@ -606,7 +572,7 @@
}
}
- private static int measure(Object key, ParsedStatement.Prepared value)
+ private static int measure(Object key, Prepared value)
{
return Ints.checkedCast(ObjectSizes.measureDeep(key) + ObjectSizes.measureDeep(value));
}
@@ -620,23 +586,22 @@
internalStatements.clear();
}
- private static class MigrationSubscriber extends MigrationListener
+ private static class StatementInvalidatingListener extends SchemaChangeListener
{
private static void removeInvalidPreparedStatements(String ksName, String cfName)
{
removeInvalidPreparedStatements(internalStatements.values().iterator(), ksName, cfName);
- removeInvalidPersistentPreparedStatements(preparedStatements.entrySet().iterator(), ksName, cfName);
- removeInvalidPreparedStatements(thriftPreparedStatements.values().iterator(), ksName, cfName);
+ removeInvalidPersistentPreparedStatements(preparedStatements.asMap().entrySet().iterator(), ksName, cfName);
}
private static void removeInvalidPreparedStatementsForFunction(String ksName, String functionName)
{
Predicate<Function> matchesFunction = f -> ksName.equals(f.name().keyspace) && functionName.equals(f.name().name);
- for (Iterator<Map.Entry<MD5Digest, ParsedStatement.Prepared>> iter = preparedStatements.entrySet().iterator();
+ for (Iterator<Map.Entry<MD5Digest, Prepared>> iter = preparedStatements.asMap().entrySet().iterator();
iter.hasNext();)
{
- Map.Entry<MD5Digest, ParsedStatement.Prepared> pstmt = iter.next();
+ Map.Entry<MD5Digest, Prepared> pstmt = iter.next();
if (Iterables.any(pstmt.getValue().statement.getFunctions(), matchesFunction))
{
SystemKeyspace.removePreparedStatement(pstmt.getKey());
@@ -647,17 +612,14 @@
Iterators.removeIf(internalStatements.values().iterator(),
statement -> Iterables.any(statement.statement.getFunctions(), matchesFunction));
-
- Iterators.removeIf(thriftPreparedStatements.values().iterator(),
- statement -> Iterables.any(statement.statement.getFunctions(), matchesFunction));
}
- private static void removeInvalidPersistentPreparedStatements(Iterator<Map.Entry<MD5Digest, ParsedStatement.Prepared>> iterator,
+ private static void removeInvalidPersistentPreparedStatements(Iterator<Map.Entry<MD5Digest, Prepared>> iterator,
String ksName, String cfName)
{
while (iterator.hasNext())
{
- Map.Entry<MD5Digest, ParsedStatement.Prepared> entry = iterator.next();
+ Map.Entry<MD5Digest, Prepared> entry = iterator.next();
if (shouldInvalidate(ksName, cfName, entry.getValue().statement))
{
SystemKeyspace.removePreparedStatement(entry.getKey());
@@ -666,7 +628,7 @@
}
}
- private static void removeInvalidPreparedStatements(Iterator<ParsedStatement.Prepared> iterator, String ksName, String cfName)
+ private static void removeInvalidPreparedStatements(Iterator<Prepared> iterator, String ksName, String cfName)
{
while (iterator.hasNext())
{
@@ -724,18 +686,18 @@
{
// in case there are other overloads, we have to remove all overloads since argument type
// matching may change (due to type casting)
- if (Schema.instance.getKSMetaData(ksName).functions.get(new FunctionName(ksName, functionName)).size() > 1)
+ if (Schema.instance.getKeyspaceMetadata(ksName).functions.get(new FunctionName(ksName, functionName)).size() > 1)
removeInvalidPreparedStatementsForFunction(ksName, functionName);
}
- public void onUpdateColumnFamily(String ksName, String cfName, boolean affectsStatements)
+ public void onAlterTable(String ksName, String cfName, boolean affectsStatements)
{
logger.trace("Column definitions for {}.{} changed, invalidating related prepared statements", ksName, cfName);
if (affectsStatements)
removeInvalidPreparedStatements(ksName, cfName);
}
- public void onUpdateFunction(String ksName, String functionName, List<AbstractType<?>> argTypes)
+ public void onAlterFunction(String ksName, String functionName, List<AbstractType<?>> argTypes)
{
// Updating a function may imply we've changed the body of the function, so we need to invalid statements so that
// the new definition is picked (the function is resolved at preparation time).
@@ -744,7 +706,7 @@
removeInvalidPreparedStatementsForFunction(ksName, functionName);
}
- public void onUpdateAggregate(String ksName, String aggregateName, List<AbstractType<?>> argTypes)
+ public void onAlterAggregate(String ksName, String aggregateName, List<AbstractType<?>> argTypes)
{
// Updating a function may imply we've changed the body of the function, so we need to invalid statements so that
// the new definition is picked (the function is resolved at preparation time).
@@ -759,7 +721,7 @@
removeInvalidPreparedStatements(ksName, null);
}
- public void onDropColumnFamily(String ksName, String cfName)
+ public void onDropTable(String ksName, String cfName)
{
logger.trace("Table {}.{} was dropped, invalidating related prepared statements", ksName, cfName);
removeInvalidPreparedStatements(ksName, cfName);
diff --git a/src/java/org/apache/cassandra/cql3/Relation.java b/src/java/org/apache/cassandra/cql3/Relation.java
index 9537ca1..0dcb3fa 100644
--- a/src/java/org/apache/cassandra/cql3/Relation.java
+++ b/src/java/org/apache/cassandra/cql3/Relation.java
@@ -20,8 +20,8 @@
import java.util.ArrayList;
import java.util.List;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.cql3.restrictions.Restriction;
import org.apache.cassandra.cql3.statements.Bound;
import org.apache.cassandra.exceptions.InvalidRequestException;
@@ -132,100 +132,83 @@
/**
* Converts this <code>Relation</code> into a <code>Restriction</code>.
*
- * @param cfm the Column Family meta data
+ * @param table the Column Family meta data
* @param boundNames the variables specification where to collect the bind variables
* @return the <code>Restriction</code> corresponding to this <code>Relation</code>
* @throws InvalidRequestException if this <code>Relation</code> is not valid
*/
- public final Restriction toRestriction(CFMetaData cfm,
- VariableSpecifications boundNames) throws InvalidRequestException
+ public final Restriction toRestriction(TableMetadata table, VariableSpecifications boundNames)
{
switch (relationType)
{
- case EQ: return newEQRestriction(cfm, boundNames);
- case LT: return newSliceRestriction(cfm, boundNames, Bound.END, false);
- case LTE: return newSliceRestriction(cfm, boundNames, Bound.END, true);
- case GTE: return newSliceRestriction(cfm, boundNames, Bound.START, true);
- case GT: return newSliceRestriction(cfm, boundNames, Bound.START, false);
- case IN: return newINRestriction(cfm, boundNames);
- case CONTAINS: return newContainsRestriction(cfm, boundNames, false);
- case CONTAINS_KEY: return newContainsRestriction(cfm, boundNames, true);
- case IS_NOT: return newIsNotRestriction(cfm, boundNames);
+ case EQ: return newEQRestriction(table, boundNames);
+ case LT: return newSliceRestriction(table, boundNames, Bound.END, false);
+ case LTE: return newSliceRestriction(table, boundNames, Bound.END, true);
+ case GTE: return newSliceRestriction(table, boundNames, Bound.START, true);
+ case GT: return newSliceRestriction(table, boundNames, Bound.START, false);
+ case IN: return newINRestriction(table, boundNames);
+ case CONTAINS: return newContainsRestriction(table, boundNames, false);
+ case CONTAINS_KEY: return newContainsRestriction(table, boundNames, true);
+ case IS_NOT: return newIsNotRestriction(table, boundNames);
case LIKE_PREFIX:
case LIKE_SUFFIX:
case LIKE_CONTAINS:
case LIKE_MATCHES:
case LIKE:
- return newLikeRestriction(cfm, boundNames, relationType);
+ return newLikeRestriction(table, boundNames, relationType);
default: throw invalidRequest("Unsupported \"!=\" relation: %s", this);
}
}
/**
- * Required for SuperColumn compatibility, creates an adapter Relation that remaps all restrictions required for
- * SuperColumn tables.
- */
- public Relation toSuperColumnAdapter()
- {
- throw invalidRequest("Unsupported operation (" + this + ") on super column family");
- }
-
- /**
* Creates a new EQ restriction instance.
*
- * @param cfm the Column Family meta data
+ * @param table the table meta data
* @param boundNames the variables specification where to collect the bind variables
* @return a new EQ restriction instance.
* @throws InvalidRequestException if the relation cannot be converted into an EQ restriction.
*/
- protected abstract Restriction newEQRestriction(CFMetaData cfm,
- VariableSpecifications boundNames) throws InvalidRequestException;
+ protected abstract Restriction newEQRestriction(TableMetadata table, VariableSpecifications boundNames);
/**
* Creates a new IN restriction instance.
*
- * @param cfm the Column Family meta data
+ * @param table the table meta data
* @param boundNames the variables specification where to collect the bind variables
* @return a new IN restriction instance
* @throws InvalidRequestException if the relation cannot be converted into an IN restriction.
*/
- protected abstract Restriction newINRestriction(CFMetaData cfm,
- VariableSpecifications boundNames) throws InvalidRequestException;
+ protected abstract Restriction newINRestriction(TableMetadata table, VariableSpecifications boundNames);
/**
* Creates a new Slice restriction instance.
*
- * @param cfm the Column Family meta data
+ * @param table the table meta data
* @param boundNames the variables specification where to collect the bind variables
* @param bound the slice bound
* @param inclusive <code>true</code> if the bound is included.
* @return a new slice restriction instance
* @throws InvalidRequestException if the <code>Relation</code> is not valid
*/
- protected abstract Restriction newSliceRestriction(CFMetaData cfm,
+ protected abstract Restriction newSliceRestriction(TableMetadata table,
VariableSpecifications boundNames,
Bound bound,
- boolean inclusive) throws InvalidRequestException;
+ boolean inclusive);
/**
* Creates a new Contains restriction instance.
*
- * @param cfm the Column Family meta data
+ * @param table the table meta data
* @param boundNames the variables specification where to collect the bind variables
* @param isKey <code>true</code> if the restriction to create is a CONTAINS KEY
* @return a new Contains <code>Restriction</code> instance
* @throws InvalidRequestException if the <code>Relation</code> is not valid
*/
- protected abstract Restriction newContainsRestriction(CFMetaData cfm,
- VariableSpecifications boundNames,
- boolean isKey) throws InvalidRequestException;
+ protected abstract Restriction newContainsRestriction(TableMetadata table, VariableSpecifications boundNames, boolean isKey);
- protected abstract Restriction newIsNotRestriction(CFMetaData cfm,
- VariableSpecifications boundNames) throws InvalidRequestException;
+ protected abstract Restriction newIsNotRestriction(TableMetadata table, VariableSpecifications boundNames);
- protected abstract Restriction newLikeRestriction(CFMetaData cfm,
- VariableSpecifications boundNames,
- Operator operator) throws InvalidRequestException;
+ protected abstract Restriction newLikeRestriction(TableMetadata table, VariableSpecifications boundNames, Operator operator);
/**
* Converts the specified <code>Raw</code> into a <code>Term</code>.
@@ -240,8 +223,7 @@
protected abstract Term toTerm(List<? extends ColumnSpecification> receivers,
Term.Raw raw,
String keyspace,
- VariableSpecifications boundNames)
- throws InvalidRequestException;
+ VariableSpecifications boundNames);
/**
* Converts the specified <code>Raw</code> terms into a <code>Term</code>s.
@@ -256,12 +238,12 @@
protected final List<Term> toTerms(List<? extends ColumnSpecification> receivers,
List<? extends Term.Raw> raws,
String keyspace,
- VariableSpecifications boundNames) throws InvalidRequestException
+ VariableSpecifications boundNames)
{
if (raws == null)
return null;
- List<Term> terms = new ArrayList<>();
+ List<Term> terms = new ArrayList<>(raws.size());
for (int i = 0, m = raws.size(); i < m; i++)
terms.add(toTerm(receivers, raws.get(i), keyspace, boundNames));
@@ -275,5 +257,5 @@
* @return this object, if the old identifier is not in the set of entities that this relation covers; otherwise
* a new Relation with "from" replaced by "to" is returned.
*/
- public abstract Relation renameIdentifier(ColumnDefinition.Raw from, ColumnDefinition.Raw to);
+ public abstract Relation renameIdentifier(ColumnMetadata.Raw from, ColumnMetadata.Raw to);
}
diff --git a/src/java/org/apache/cassandra/cql3/ResultSet.java b/src/java/org/apache/cassandra/cql3/ResultSet.java
index 7d9d18d..4f60445 100644
--- a/src/java/org/apache/cassandra/cql3/ResultSet.java
+++ b/src/java/org/apache/cassandra/cql3/ResultSet.java
@@ -18,20 +18,28 @@
package org.apache.cassandra.cql3;
import java.nio.ByteBuffer;
-import java.util.*;
+import java.nio.charset.StandardCharsets;
+import java.security.MessageDigest;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.Objects;
+
+import com.google.common.annotations.VisibleForTesting;
import io.netty.buffer.ByteBuf;
-
-import org.apache.cassandra.transport.*;
+import org.apache.cassandra.cql3.statements.SelectStatement;
import org.apache.cassandra.db.marshal.AbstractType;
-import org.apache.cassandra.db.marshal.ReversedType;
-import org.apache.cassandra.thrift.Column;
-import org.apache.cassandra.thrift.CqlMetadata;
-import org.apache.cassandra.thrift.CqlResult;
-import org.apache.cassandra.thrift.CqlResultType;
-import org.apache.cassandra.thrift.CqlRow;
-import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.service.pager.PagingState;
+import org.apache.cassandra.transport.CBCodec;
+import org.apache.cassandra.transport.CBUtil;
+import org.apache.cassandra.transport.DataType;
+import org.apache.cassandra.transport.ProtocolVersion;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.MD5Digest;
public class ResultSet
{
@@ -40,14 +48,14 @@
public final ResultMetadata metadata;
public final List<List<ByteBuffer>> rows;
- public ResultSet(List<ColumnSpecification> metadata)
+ public ResultSet(ResultMetadata resultMetadata)
{
- this(new ResultMetadata(metadata), new ArrayList<List<ByteBuffer>>());
+ this(resultMetadata, new ArrayList<List<ByteBuffer>>());
}
- public ResultSet(ResultMetadata metadata, List<List<ByteBuffer>> rows)
+ public ResultSet(ResultMetadata resultMetadata, List<List<ByteBuffer>> rows)
{
- this.metadata = metadata;
+ this.metadata = resultMetadata;
this.rows = rows;
}
@@ -95,44 +103,6 @@
}
}
- public CqlResult toThriftResult()
- {
- assert metadata.names != null;
-
- String UTF8 = "UTF8Type";
- CqlMetadata schema = new CqlMetadata(new HashMap<ByteBuffer, String>(),
- new HashMap<ByteBuffer, String>(),
- // The 2 following ones shouldn't be needed in CQL3
- UTF8, UTF8);
-
- for (int i = 0; i < metadata.columnCount; i++)
- {
- ColumnSpecification spec = metadata.names.get(i);
- ByteBuffer colName = ByteBufferUtil.bytes(spec.name.toString());
- schema.name_types.put(colName, UTF8);
- AbstractType<?> normalizedType = spec.type instanceof ReversedType ? ((ReversedType)spec.type).baseType : spec.type;
- schema.value_types.put(colName, normalizedType.toString());
-
- }
-
- List<CqlRow> cqlRows = new ArrayList<CqlRow>(rows.size());
- for (List<ByteBuffer> row : rows)
- {
- List<Column> thriftCols = new ArrayList<Column>(metadata.columnCount);
- for (int i = 0; i < metadata.columnCount; i++)
- {
- Column col = new Column(ByteBufferUtil.bytes(metadata.names.get(i).name.toString()));
- col.setValue(row.get(i));
- thriftCols.add(col);
- }
- // The key of CqlRow shoudn't be needed in CQL3
- cqlRows.add(new CqlRow(ByteBufferUtil.EMPTY_BYTE_BUFFER, thriftCols));
- }
- CqlResult res = new CqlResult(CqlResultType.ROWS);
- res.setRows(cqlRows).setSchema(schema);
- return res;
- }
-
@Override
public String toString()
{
@@ -223,7 +193,7 @@
{
public static final CBCodec<ResultMetadata> codec = new Codec();
- public static final ResultMetadata EMPTY = new ResultMetadata(EnumSet.of(Flag.NO_METADATA), null, 0, null);
+ public static final ResultMetadata EMPTY = new ResultMetadata(MD5Digest.compute(new byte[0]), EnumSet.of(Flag.NO_METADATA), null, 0, null);
private final EnumSet<Flag> flags;
// Please note that columnCount can actually be smaller than names, even if names is not null. This is
@@ -233,16 +203,34 @@
public final List<ColumnSpecification> names;
private final int columnCount;
private PagingState pagingState;
+ private final MD5Digest resultMetadataId;
- public ResultMetadata(List<ColumnSpecification> names)
+ public ResultMetadata(MD5Digest digest, List<ColumnSpecification> names)
{
- this(EnumSet.noneOf(Flag.class), names, names.size(), null);
+ this(digest, EnumSet.noneOf(Flag.class), names, names.size(), null);
if (!names.isEmpty() && ColumnSpecification.allInSameTable(names))
flags.add(Flag.GLOBAL_TABLES_SPEC);
}
- private ResultMetadata(EnumSet<Flag> flags, List<ColumnSpecification> names, int columnCount, PagingState pagingState)
+ // Problem is that we compute the metadata from the columns on creation;
+ // when re-preparing we create the intermediate object
+ public ResultMetadata(List<ColumnSpecification> names)
{
+ this(names, null);
+ }
+
+ // Problem is that we compute the metadata from the columns on creation;
+ // when re-preparing we create the intermediate object
+ public ResultMetadata(List<ColumnSpecification> names, PagingState pagingState)
+ {
+ this(computeResultMetadataId(names), EnumSet.noneOf(Flag.class), names, names.size(), pagingState);
+ if (!names.isEmpty() && ColumnSpecification.allInSameTable(names))
+ flags.add(Flag.GLOBAL_TABLES_SPEC);
+ }
+
+ private ResultMetadata(MD5Digest resultMetadataId, EnumSet<Flag> flags, List<ColumnSpecification> names, int columnCount, PagingState pagingState)
+ {
+ this.resultMetadataId = resultMetadataId;
this.flags = flags;
this.names = names;
this.columnCount = columnCount;
@@ -251,12 +239,7 @@
public ResultMetadata copy()
{
- return new ResultMetadata(EnumSet.copyOf(flags), names, columnCount, pagingState);
- }
-
- public int getColumnCount()
- {
- return columnCount;
+ return new ResultMetadata(resultMetadataId, EnumSet.copyOf(flags), names, columnCount, pagingState);
}
/**
@@ -274,16 +257,35 @@
return names == null ? columnCount : names.size();
}
+ @VisibleForTesting
+ public EnumSet<Flag> getFlags()
+ {
+ return flags;
+ }
+
+ @VisibleForTesting
+ public int getColumnCount()
+ {
+ return columnCount;
+ }
+
+ @VisibleForTesting
+ public PagingState getPagingState()
+ {
+ return pagingState;
+ }
+
/**
- * Adds the specified column which will not be serialized.
+ * Adds the specified columns which will not be serialized.
*
- * @param name the column
+ * @param columns the columns
*/
- public void addNonSerializedColumn(ColumnSpecification name)
+ public ResultMetadata addNonSerializedColumns(Collection<? extends ColumnSpecification> columns)
{
// See comment above. Because columnCount doesn't account the newly added name, it
// won't be serialized.
- names.add(name);
+ names.addAll(columns);
+ return this;
}
public void setHasMorePages(PagingState pagingState)
@@ -300,6 +302,24 @@
flags.add(Flag.NO_METADATA);
}
+ public void setMetadataChanged()
+ {
+ flags.add(Flag.METADATA_CHANGED);
+ }
+
+ public MD5Digest getResultMetadataId()
+ {
+ return resultMetadataId;
+ }
+
+ public static ResultMetadata fromPrepared(CQLStatement statement)
+ {
+ if (statement instanceof SelectStatement)
+ return ((SelectStatement)statement).getResultMetadata();
+
+ return ResultSet.ResultMetadata.EMPTY;
+ }
+
@Override
public boolean equals(Object other)
{
@@ -356,12 +376,21 @@
EnumSet<Flag> flags = Flag.deserialize(iflags);
+ MD5Digest resultMetadataId = null;
+ if (flags.contains(Flag.METADATA_CHANGED))
+ {
+ assert version.isGreaterOrEqualTo(ProtocolVersion.V5) : "MetadataChanged flag is not supported before native protocol v5";
+ assert !flags.contains(Flag.NO_METADATA) : "MetadataChanged and NoMetadata are mutually exclusive flags";
+
+ resultMetadataId = MD5Digest.wrap(CBUtil.readBytes(body));
+ }
+
PagingState state = null;
if (flags.contains(Flag.HAS_MORE_PAGES))
- state = PagingState.deserialize(CBUtil.readValue(body), version);
+ state = PagingState.deserialize(CBUtil.readValueNoCopy(body), version);
if (flags.contains(Flag.NO_METADATA))
- return new ResultMetadata(flags, null, columnCount, state);
+ return new ResultMetadata(null, flags, null, columnCount, state);
boolean globalTablesSpec = flags.contains(Flag.GLOBAL_TABLES_SPEC);
@@ -383,7 +412,7 @@
AbstractType type = DataType.toType(DataType.codec.decodeOne(body, version));
names.add(new ColumnSpecification(ksName, cfName, colName, type));
}
- return new ResultMetadata(flags, names, names.size(), state);
+ return new ResultMetadata(resultMetadataId, flags, names, names.size(), state);
}
public void encode(ResultMetadata m, ByteBuf dest, ProtocolVersion version)
@@ -391,7 +420,7 @@
boolean noMetadata = m.flags.contains(Flag.NO_METADATA);
boolean globalTablesSpec = m.flags.contains(Flag.GLOBAL_TABLES_SPEC);
boolean hasMorePages = m.flags.contains(Flag.HAS_MORE_PAGES);
-
+ boolean metadataChanged = m.flags.contains(Flag.METADATA_CHANGED);
assert version.isGreaterThan(ProtocolVersion.V1) || (!hasMorePages && !noMetadata)
: "version = " + version + ", flags = " + m.flags;
@@ -401,12 +430,18 @@
if (hasMorePages)
CBUtil.writeValue(m.pagingState.serialize(version), dest);
+ if (version.isGreaterOrEqualTo(ProtocolVersion.V5) && metadataChanged)
+ {
+ assert !noMetadata : "MetadataChanged and NoMetadata are mutually exclusive flags";
+ CBUtil.writeBytes(m.getResultMetadataId().bytes, dest);
+ }
+
if (!noMetadata)
{
if (globalTablesSpec)
{
- CBUtil.writeString(m.names.get(0).ksName, dest);
- CBUtil.writeString(m.names.get(0).cfName, dest);
+ CBUtil.writeAsciiString(m.names.get(0).ksName, dest);
+ CBUtil.writeAsciiString(m.names.get(0).cfName, dest);
}
for (int i = 0; i < m.columnCount; i++)
@@ -414,10 +449,10 @@
ColumnSpecification name = m.names.get(i);
if (!globalTablesSpec)
{
- CBUtil.writeString(name.ksName, dest);
- CBUtil.writeString(name.cfName, dest);
+ CBUtil.writeAsciiString(name.ksName, dest);
+ CBUtil.writeAsciiString(name.cfName, dest);
}
- CBUtil.writeString(name.name.toString(), dest);
+ CBUtil.writeAsciiString(name.name.toString(), dest);
DataType.codec.writeOne(DataType.fromType(name.type, version), dest, version);
}
}
@@ -428,17 +463,21 @@
boolean noMetadata = m.flags.contains(Flag.NO_METADATA);
boolean globalTablesSpec = m.flags.contains(Flag.GLOBAL_TABLES_SPEC);
boolean hasMorePages = m.flags.contains(Flag.HAS_MORE_PAGES);
+ boolean metadataChanged = m.flags.contains(Flag.METADATA_CHANGED);
int size = 8;
if (hasMorePages)
size += CBUtil.sizeOfValue(m.pagingState.serializedSize(version));
+ if (version.isGreaterOrEqualTo(ProtocolVersion.V5) && metadataChanged)
+ size += CBUtil.sizeOfBytes(m.getResultMetadataId().bytes);
+
if (!noMetadata)
{
if (globalTablesSpec)
{
- size += CBUtil.sizeOfString(m.names.get(0).ksName);
- size += CBUtil.sizeOfString(m.names.get(0).cfName);
+ size += CBUtil.sizeOfAsciiString(m.names.get(0).ksName);
+ size += CBUtil.sizeOfAsciiString(m.names.get(0).cfName);
}
for (int i = 0; i < m.columnCount; i++)
@@ -446,10 +485,10 @@
ColumnSpecification name = m.names.get(i);
if (!globalTablesSpec)
{
- size += CBUtil.sizeOfString(name.ksName);
- size += CBUtil.sizeOfString(name.cfName);
+ size += CBUtil.sizeOfAsciiString(name.ksName);
+ size += CBUtil.sizeOfAsciiString(name.cfName);
}
- size += CBUtil.sizeOfString(name.name.toString());
+ size += CBUtil.sizeOfAsciiString(name.name.toString());
size += DataType.codec.oneSerializedSize(DataType.fromType(name.type, version), version);
}
}
@@ -534,6 +573,11 @@
return sb.toString();
}
+ public static PreparedMetadata fromPrepared(CQLStatement statement)
+ {
+ return new PreparedMetadata(statement.getBindVariables(), statement.getPartitionKeyBindVariableIndexes());
+ }
+
private static class Codec implements CBCodec<PreparedMetadata>
{
public PreparedMetadata decode(ByteBuf body, ProtocolVersion version)
@@ -602,18 +646,18 @@
if (globalTablesSpec)
{
- CBUtil.writeString(m.names.get(0).ksName, dest);
- CBUtil.writeString(m.names.get(0).cfName, dest);
+ CBUtil.writeAsciiString(m.names.get(0).ksName, dest);
+ CBUtil.writeAsciiString(m.names.get(0).cfName, dest);
}
for (ColumnSpecification name : m.names)
{
if (!globalTablesSpec)
{
- CBUtil.writeString(name.ksName, dest);
- CBUtil.writeString(name.cfName, dest);
+ CBUtil.writeAsciiString(name.ksName, dest);
+ CBUtil.writeAsciiString(name.cfName, dest);
}
- CBUtil.writeString(name.name.toString(), dest);
+ CBUtil.writeAsciiString(name.name.toString(), dest);
DataType.codec.writeOne(DataType.fromType(name.type, version), dest, version);
}
}
@@ -624,8 +668,8 @@
int size = 8;
if (globalTablesSpec)
{
- size += CBUtil.sizeOfString(m.names.get(0).ksName);
- size += CBUtil.sizeOfString(m.names.get(0).cfName);
+ size += CBUtil.sizeOfAsciiString(m.names.get(0).ksName);
+ size += CBUtil.sizeOfAsciiString(m.names.get(0).cfName);
}
if (m.partitionKeyBindIndexes != null && version.isGreaterOrEqualTo(ProtocolVersion.V4))
@@ -635,10 +679,10 @@
{
if (!globalTablesSpec)
{
- size += CBUtil.sizeOfString(name.ksName);
- size += CBUtil.sizeOfString(name.cfName);
+ size += CBUtil.sizeOfAsciiString(name.ksName);
+ size += CBUtil.sizeOfAsciiString(name.cfName);
}
- size += CBUtil.sizeOfString(name.name.toString());
+ size += CBUtil.sizeOfAsciiString(name.name.toString());
size += DataType.codec.oneSerializedSize(DataType.fromType(name.type, version), version);
}
return size;
@@ -651,7 +695,8 @@
// The order of that enum matters!!
GLOBAL_TABLES_SPEC,
HAS_MORE_PAGES,
- NO_METADATA;
+ NO_METADATA,
+ METADATA_CHANGED;
public static EnumSet<Flag> deserialize(int flags)
{
@@ -673,4 +718,28 @@
return i;
}
}
+
+ static MD5Digest computeResultMetadataId(List<ColumnSpecification> columnSpecifications)
+ {
+ // still using the MD5 MessageDigest thread local here instead of a HashingUtils/Guava
+ // Hasher, as ResultSet will need to be changed alongside other usages of MD5
+ // in the native transport/protocol and it seems to make more sense to do that
+ // then than as part of the Guava Hasher refactor which is focused on non-client
+ // protocol digests
+ MessageDigest md = MD5Digest.threadLocalMD5Digest();
+
+ if (columnSpecifications != null)
+ {
+ for (ColumnSpecification cs : columnSpecifications)
+ {
+ md.update(cs.name.bytes.duplicate());
+ md.update((byte) 0);
+ md.update(cs.type.toString().getBytes(StandardCharsets.UTF_8));
+ md.update((byte) 0);
+ md.update((byte) 0);
+ }
+ }
+
+ return MD5Digest.wrap(md.digest());
+ }
}
diff --git a/src/java/org/apache/cassandra/cql3/RoleName.java b/src/java/org/apache/cassandra/cql3/RoleName.java
index ce81fa9..b50c17d 100644
--- a/src/java/org/apache/cassandra/cql3/RoleName.java
+++ b/src/java/org/apache/cassandra/cql3/RoleName.java
@@ -25,7 +25,7 @@
public void setName(String name, boolean keepCase)
{
- this.name = keepCase ? name : name.toLowerCase(Locale.US);
+ this.name = keepCase ? name : (name == null ? name : name.toLowerCase(Locale.US));
}
public boolean hasName()
diff --git a/src/java/org/apache/cassandra/cql3/SchemaElement.java b/src/java/org/apache/cassandra/cql3/SchemaElement.java
new file mode 100644
index 0000000..ec0dbee
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/SchemaElement.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3;
+
+import java.util.Comparator;
+import java.util.Locale;
+
+/**
+ * A schema element (keyspace, udt, udf, uda, table, index, view).
+ */
+public interface SchemaElement
+{
+ /**
+ * Comparator used to sort {@code Describable} name.
+ */
+ Comparator<SchemaElement> NAME_COMPARATOR = (o1, o2) -> o1.elementName().compareToIgnoreCase(o2.elementName());
+
+ enum SchemaElementType
+ {
+ KEYSPACE,
+ TYPE,
+ FUNCTION,
+ AGGREGATE,
+ TABLE,
+ INDEX,
+ MATERIALIZED_VIEW;
+
+ @Override
+ public String toString()
+ {
+ return super.toString().toLowerCase(Locale.US);
+ }
+ }
+
+ /**
+ * Return the schema element type
+ *
+ * @return the schema element type
+ */
+ SchemaElementType elementType();
+
+ /**
+ * Returns the CQL name of the keyspace to which this schema element belong.
+ *
+ * @return the keyspace name.
+ */
+ String elementKeyspace();
+
+ /**
+ * Returns the CQL name of this schema element.
+ *
+ * @return the name of this schema element.
+ */
+ String elementName();
+
+ default String elementNameQuotedIfNeeded()
+ {
+ String name = elementName();
+ if (elementType() == SchemaElementType.FUNCTION
+ || elementType() == SchemaElementType.AGGREGATE)
+ {
+ int index = name.indexOf('(');
+ return ColumnIdentifier.maybeQuote(name.substring(0, index)) + name.substring(index);
+ }
+
+ return ColumnIdentifier.maybeQuote(name);
+ }
+
+ default String elementKeyspaceQuotedIfNeeded()
+ {
+ return ColumnIdentifier.maybeQuote(elementKeyspace());
+ }
+
+ /**
+ * Returns a CQL representation of this element
+ *
+ * @param withInternals if the internals part of the CQL should be exposed.
+ * @return a CQL representation of this element
+ */
+ String toCqlString(boolean withInternals);
+}
diff --git a/src/java/org/apache/cassandra/cql3/Sets.java b/src/java/org/apache/cassandra/cql3/Sets.java
index d17a771..2920ed7 100644
--- a/src/java/org/apache/cassandra/cql3/Sets.java
+++ b/src/java/org/apache/cassandra/cql3/Sets.java
@@ -22,8 +22,9 @@
import java.nio.ByteBuffer;
import java.util.*;
import java.util.stream.Collectors;
+import java.util.stream.StreamSupport;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.cql3.functions.Function;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.rows.*;
@@ -43,7 +44,73 @@
public static ColumnSpecification valueSpecOf(ColumnSpecification column)
{
- return new ColumnSpecification(column.ksName, column.cfName, new ColumnIdentifier("value(" + column.name + ")", true), ((SetType)column.type).getElementsType());
+ return new ColumnSpecification(column.ksName, column.cfName, new ColumnIdentifier("value(" + column.name + ")", true), ((SetType<?>)column.type).getElementsType());
+ }
+
+ /**
+ * Tests that the set with the specified elements can be assigned to the specified column.
+ *
+ * @param receiver the receiving column
+ * @param elements the set elements
+ */
+ public static AssignmentTestable.TestResult testSetAssignment(ColumnSpecification receiver,
+ List<? extends AssignmentTestable> elements)
+ {
+ if (!(receiver.type instanceof SetType))
+ {
+ // We've parsed empty maps as a set literal to break the ambiguity so handle that case now
+ if (receiver.type instanceof MapType && elements.isEmpty())
+ return AssignmentTestable.TestResult.WEAKLY_ASSIGNABLE;
+
+ return AssignmentTestable.TestResult.NOT_ASSIGNABLE;
+ }
+
+ // If there is no elements, we can't say it's an exact match (an empty set if fundamentally polymorphic).
+ if (elements.isEmpty())
+ return AssignmentTestable.TestResult.WEAKLY_ASSIGNABLE;
+
+ ColumnSpecification valueSpec = valueSpecOf(receiver);
+ return AssignmentTestable.TestResult.testAll(receiver.ksName, valueSpec, elements);
+ }
+
+ /**
+ * Create a <code>String</code> representation of the set containing the specified elements.
+ *
+ * @param elements the set elements
+ * @return a <code>String</code> representation of the set
+ */
+ public static String setToString(List<?> elements)
+ {
+ return setToString(elements, Object::toString);
+ }
+
+ /**
+ * Create a <code>String</code> representation of the set from the specified items associated to
+ * the set elements.
+ *
+ * @param items items associated to the set elements
+ * @param mapper the mapper used to map the items to the <code>String</code> representation of the set elements
+ * @return a <code>String</code> representation of the set
+ */
+ public static <T> String setToString(Iterable<T> items, java.util.function.Function<T, String> mapper)
+ {
+ return StreamSupport.stream(items.spliterator(), false)
+ .map(e -> mapper.apply(e))
+ .collect(Collectors.joining(", ", "{", "}"));
+ }
+
+ /**
+ * Returns the exact SetType from the items if it can be known.
+ *
+ * @param items the items mapped to the set elements
+ * @param mapper the mapper used to retrieve the element types from the items
+ * @return the exact SetType from the items if it can be known or <code>null</code>
+ */
+ public static <T> AbstractType<?> getExactSetTypeIfKnown(List<T> items,
+ java.util.function.Function<T, AbstractType<?>> mapper)
+ {
+ Optional<AbstractType<?>> type = items.stream().map(mapper).filter(Objects::nonNull).findFirst();
+ return type.isPresent() ? SetType.getInstance(type.get(), false) : null;
}
public static class Literal extends Term.Raw
@@ -105,38 +172,18 @@
public AssignmentTestable.TestResult testAssignment(String keyspace, ColumnSpecification receiver)
{
- if (!(receiver.type instanceof SetType))
- {
- // We've parsed empty maps as a set literal to break the ambiguity so handle that case now
- if (receiver.type instanceof MapType && elements.isEmpty())
- return AssignmentTestable.TestResult.WEAKLY_ASSIGNABLE;
-
- return AssignmentTestable.TestResult.NOT_ASSIGNABLE;
- }
-
- // If there is no elements, we can't say it's an exact match (an empty set if fundamentally polymorphic).
- if (elements.isEmpty())
- return AssignmentTestable.TestResult.WEAKLY_ASSIGNABLE;
-
- ColumnSpecification valueSpec = Sets.valueSpecOf(receiver);
- return AssignmentTestable.TestResult.testAll(keyspace, valueSpec, elements);
+ return testSetAssignment(receiver, elements);
}
@Override
public AbstractType<?> getExactTypeIfKnown(String keyspace)
{
- for (Term.Raw term : elements)
- {
- AbstractType<?> type = term.getExactTypeIfKnown(keyspace);
- if (type != null)
- return SetType.getInstance(type, false);
- }
- return null;
+ return getExactSetTypeIfKnown(elements, p -> p.getExactTypeIfKnown(keyspace));
}
public String getText()
{
- return elements.stream().map(Term.Raw::getText).collect(Collectors.joining(", ", "{", "}"));
+ return setToString(elements, Term.Raw::getText);
}
}
@@ -254,7 +301,7 @@
public static class Setter extends Operation
{
- public Setter(ColumnDefinition column, Term t)
+ public Setter(ColumnMetadata column, Term t)
{
super(column, t);
}
@@ -274,7 +321,7 @@
public static class Adder extends Operation
{
- public Adder(ColumnDefinition column, Term t)
+ public Adder(ColumnMetadata column, Term t)
{
super(column, t);
}
@@ -287,7 +334,7 @@
doAdd(value, column, params);
}
- static void doAdd(Term.Terminal value, ColumnDefinition column, UpdateParameters params) throws InvalidRequestException
+ static void doAdd(Term.Terminal value, ColumnMetadata column, UpdateParameters params) throws InvalidRequestException
{
if (column.type.isMultiCell())
{
@@ -316,7 +363,7 @@
// Note that this is reused for Map subtraction too (we subtract a set from a map)
public static class Discarder extends Operation
{
- public Discarder(ColumnDefinition column, Term t)
+ public Discarder(ColumnMetadata column, Term t)
{
super(column, t);
}
@@ -341,7 +388,7 @@
public static class ElementDiscarder extends Operation
{
- public ElementDiscarder(ColumnDefinition column, Term k)
+ public ElementDiscarder(ColumnMetadata column, Term k)
{
super(column, k);
}
diff --git a/src/java/org/apache/cassandra/cql3/SingleColumnRelation.java b/src/java/org/apache/cassandra/cql3/SingleColumnRelation.java
index 2e9b41f..bf453d7 100644
--- a/src/java/org/apache/cassandra/cql3/SingleColumnRelation.java
+++ b/src/java/org/apache/cassandra/cql3/SingleColumnRelation.java
@@ -20,9 +20,10 @@
import java.util.Collections;
import java.util.ArrayList;
import java.util.List;
+import java.util.Objects;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.cql3.Term.Raw;
import org.apache.cassandra.cql3.restrictions.Restriction;
import org.apache.cassandra.cql3.restrictions.SingleColumnRestriction;
@@ -41,14 +42,14 @@
* a value (term). For example, {@code <key> > "start" or "colname1" = "somevalue"}.
*
*/
-public class SingleColumnRelation extends Relation
+public final class SingleColumnRelation extends Relation
{
- private final ColumnDefinition.Raw entity;
+ private final ColumnMetadata.Raw entity;
private final Term.Raw mapKey;
private final Term.Raw value;
private final List<Term.Raw> inValues;
- private SingleColumnRelation(ColumnDefinition.Raw entity, Term.Raw mapKey, Operator type, Term.Raw value, List<Term.Raw> inValues)
+ private SingleColumnRelation(ColumnMetadata.Raw entity, Term.Raw mapKey, Operator type, Term.Raw value, List<Term.Raw> inValues)
{
this.entity = entity;
this.mapKey = mapKey;
@@ -68,7 +69,7 @@
* @param type the type that describes how this entity relates to the value.
* @param value the value being compared.
*/
- public SingleColumnRelation(ColumnDefinition.Raw entity, Term.Raw mapKey, Operator type, Term.Raw value)
+ public SingleColumnRelation(ColumnMetadata.Raw entity, Term.Raw mapKey, Operator type, Term.Raw value)
{
this(entity, mapKey, type, value, null);
}
@@ -80,7 +81,7 @@
* @param type the type that describes how this entity relates to the value.
* @param value the value being compared.
*/
- public SingleColumnRelation(ColumnDefinition.Raw entity, Operator type, Term.Raw value)
+ public SingleColumnRelation(ColumnMetadata.Raw entity, Operator type, Term.Raw value)
{
this(entity, null, type, value);
}
@@ -95,12 +96,12 @@
return inValues;
}
- public static SingleColumnRelation createInRelation(ColumnDefinition.Raw entity, List<Term.Raw> inValues)
+ public static SingleColumnRelation createInRelation(ColumnMetadata.Raw entity, List<Term.Raw> inValues)
{
return new SingleColumnRelation(entity, null, Operator.IN, null, inValues);
}
- public ColumnDefinition.Raw getEntity()
+ public ColumnMetadata.Raw getEntity()
{
return entity;
}
@@ -134,7 +135,7 @@
}
}
- public Relation renameIdentifier(ColumnDefinition.Raw from, ColumnDefinition.Raw to)
+ public Relation renameIdentifier(ColumnMetadata.Raw from, ColumnMetadata.Raw to)
{
return entity.equals(from)
? new SingleColumnRelation(to, mapKey, operator(), value, inValues)
@@ -149,37 +150,58 @@
entityAsString = String.format("%s[%s]", entityAsString, mapKey);
if (isIN())
- return String.format("%s IN %s", entityAsString, inValues);
+ return String.format("%s IN %s", entityAsString, Tuples.tupleToString(inValues));
return String.format("%s %s %s", entityAsString, relationType, value);
}
@Override
- protected Restriction newEQRestriction(CFMetaData cfm,
- VariableSpecifications boundNames) throws InvalidRequestException
+ public int hashCode()
{
- ColumnDefinition columnDef = entity.prepare(cfm);
+ return Objects.hash(relationType, entity, mapKey, value, inValues);
+ }
+
+ @Override
+ public boolean equals(Object o)
+ {
+ if (this == o)
+ return true;
+
+ if (!(o instanceof SingleColumnRelation))
+ return false;
+
+ SingleColumnRelation scr = (SingleColumnRelation) o;
+ return Objects.equals(entity, scr.entity)
+ && Objects.equals(relationType, scr.relationType)
+ && Objects.equals(mapKey, scr.mapKey)
+ && Objects.equals(value, scr.value)
+ && Objects.equals(inValues, scr.inValues);
+ }
+
+ @Override
+ protected Restriction newEQRestriction(TableMetadata table, VariableSpecifications boundNames)
+ {
+ ColumnMetadata columnDef = entity.prepare(table);
if (mapKey == null)
{
- Term term = toTerm(toReceivers(columnDef), value, cfm.ksName, boundNames);
+ Term term = toTerm(toReceivers(columnDef), value, table.keyspace, boundNames);
return new SingleColumnRestriction.EQRestriction(columnDef, term);
}
List<? extends ColumnSpecification> receivers = toReceivers(columnDef);
- Term entryKey = toTerm(Collections.singletonList(receivers.get(0)), mapKey, cfm.ksName, boundNames);
- Term entryValue = toTerm(Collections.singletonList(receivers.get(1)), value, cfm.ksName, boundNames);
+ Term entryKey = toTerm(Collections.singletonList(receivers.get(0)), mapKey, table.keyspace, boundNames);
+ Term entryValue = toTerm(Collections.singletonList(receivers.get(1)), value, table.keyspace, boundNames);
return new SingleColumnRestriction.ContainsRestriction(columnDef, entryKey, entryValue);
}
@Override
- protected Restriction newINRestriction(CFMetaData cfm,
- VariableSpecifications boundNames) throws InvalidRequestException
+ protected Restriction newINRestriction(TableMetadata table, VariableSpecifications boundNames)
{
- ColumnDefinition columnDef = entity.prepare(cfm);
+ ColumnMetadata columnDef = entity.prepare(table);
List<? extends ColumnSpecification> receivers = toReceivers(columnDef);
- List<Term> terms = toTerms(receivers, inValues, cfm.ksName, boundNames);
+ List<Term> terms = toTerms(receivers, inValues, table.keyspace, boundNames);
if (terms == null)
{
- Term term = toTerm(receivers, value, cfm.ksName, boundNames);
+ Term term = toTerm(receivers, value, table.keyspace, boundNames);
return new SingleColumnRestriction.InRestrictionWithMarker(columnDef, (Lists.Marker) term);
}
@@ -191,12 +213,12 @@
}
@Override
- protected Restriction newSliceRestriction(CFMetaData cfm,
+ protected Restriction newSliceRestriction(TableMetadata table,
VariableSpecifications boundNames,
Bound bound,
- boolean inclusive) throws InvalidRequestException
+ boolean inclusive)
{
- ColumnDefinition columnDef = entity.prepare(cfm);
+ ColumnMetadata columnDef = entity.prepare(table);
if (columnDef.type.referencesDuration())
{
@@ -206,38 +228,38 @@
throw invalidRequest("Slice restrictions are not supported on duration columns");
}
- Term term = toTerm(toReceivers(columnDef), value, cfm.ksName, boundNames);
+ Term term = toTerm(toReceivers(columnDef), value, table.keyspace, boundNames);
return new SingleColumnRestriction.SliceRestriction(columnDef, bound, inclusive, term);
}
@Override
- protected Restriction newContainsRestriction(CFMetaData cfm,
+ protected Restriction newContainsRestriction(TableMetadata table,
VariableSpecifications boundNames,
boolean isKey) throws InvalidRequestException
{
- ColumnDefinition columnDef = entity.prepare(cfm);
- Term term = toTerm(toReceivers(columnDef), value, cfm.ksName, boundNames);
+ ColumnMetadata columnDef = entity.prepare(table);
+ Term term = toTerm(toReceivers(columnDef), value, table.keyspace, boundNames);
return new SingleColumnRestriction.ContainsRestriction(columnDef, term, isKey);
}
@Override
- protected Restriction newIsNotRestriction(CFMetaData cfm,
+ protected Restriction newIsNotRestriction(TableMetadata table,
VariableSpecifications boundNames) throws InvalidRequestException
{
- ColumnDefinition columnDef = entity.prepare(cfm);
+ ColumnMetadata columnDef = entity.prepare(table);
// currently enforced by the grammar
assert value == Constants.NULL_LITERAL : "Expected null literal for IS NOT relation: " + this.toString();
return new SingleColumnRestriction.IsNotNullRestriction(columnDef);
}
@Override
- protected Restriction newLikeRestriction(CFMetaData cfm, VariableSpecifications boundNames, Operator operator) throws InvalidRequestException
+ protected Restriction newLikeRestriction(TableMetadata table, VariableSpecifications boundNames, Operator operator)
{
if (mapKey != null)
throw invalidRequest("%s can't be used with collections.", operator());
- ColumnDefinition columnDef = entity.prepare(cfm);
- Term term = toTerm(toReceivers(columnDef), value, cfm.ksName, boundNames);
+ ColumnMetadata columnDef = entity.prepare(table);
+ Term term = toTerm(toReceivers(columnDef), value, table.keyspace, boundNames);
return new SingleColumnRestriction.LikeRestriction(columnDef, operator, term);
}
@@ -248,7 +270,7 @@
* @return the receivers for the specified relation.
* @throws InvalidRequestException if the relation is invalid
*/
- private List<? extends ColumnSpecification> toReceivers(ColumnDefinition columnDef) throws InvalidRequestException
+ private List<? extends ColumnSpecification> toReceivers(ColumnMetadata columnDef) throws InvalidRequestException
{
ColumnSpecification receiver = columnDef;
@@ -323,78 +345,4 @@
{
return isEQ() || isLIKE() || (isIN() && inValues != null && inValues.size() == 1);
}
-
- @Override
- public Relation toSuperColumnAdapter()
- {
- return new SuperColumnSingleColumnRelation(entity, mapKey, relationType, value);
- }
-
- /**
- * Required for SuperColumn compatibility, in order to map the SuperColumn key restrictions from the regular
- * column to the collection key one.
- */
- private class SuperColumnSingleColumnRelation extends SingleColumnRelation
- {
- SuperColumnSingleColumnRelation(ColumnDefinition.Raw entity, Raw mapKey, Operator type, Raw value)
- {
- super(entity, mapKey, type, value, inValues);
- }
-
- @Override
- public Restriction newSliceRestriction(CFMetaData cfm,
- VariableSpecifications boundNames,
- Bound bound,
- boolean inclusive) throws InvalidRequestException
- {
- ColumnDefinition columnDef = entity.prepare(cfm);
- if (cfm.isSuperColumnKeyColumn(columnDef))
- {
- Term term = toTerm(toReceivers(columnDef), value, cfm.ksName, boundNames);
- return new SingleColumnRestriction.SuperColumnKeySliceRestriction(cfm.superColumnKeyColumn(), bound, inclusive, term);
- }
- else
- {
- return super.newSliceRestriction(cfm, boundNames, bound, inclusive);
- }
- }
-
- @Override
- protected Restriction newEQRestriction(CFMetaData cfm,
- VariableSpecifications boundNames) throws InvalidRequestException
- {
- ColumnDefinition columnDef = entity.prepare(cfm);
- if (cfm.isSuperColumnKeyColumn(columnDef))
- {
- Term term = toTerm(toReceivers(columnDef), value, cfm.ksName, boundNames);
- return new SingleColumnRestriction.SuperColumnKeyEQRestriction(cfm.superColumnKeyColumn(), term);
- }
- else
- {
- return super.newEQRestriction(cfm, boundNames);
- }
- }
-
- @Override
- protected Restriction newINRestriction(CFMetaData cfm,
- VariableSpecifications boundNames) throws InvalidRequestException
- {
- ColumnDefinition columnDef = entity.prepare(cfm);
- if (cfm.isSuperColumnKeyColumn(columnDef))
- {
- List<? extends ColumnSpecification> receivers = Collections.singletonList(cfm.superColumnKeyColumn());
- List<Term> terms = toTerms(receivers, inValues, cfm.ksName, boundNames);
- if (terms == null)
- {
- Term term = toTerm(receivers, value, cfm.ksName, boundNames);
- return new SingleColumnRestriction.SuperColumnKeyINRestrictionWithMarkers(cfm.superColumnKeyColumn(), (Lists.Marker) term);
- }
- return new SingleColumnRestriction.SuperColumnKeyINRestrictionWithValues(cfm.superColumnKeyColumn(), terms);
- }
- else
- {
- return super.newINRestriction(cfm, boundNames);
- }
- }
- }
}
diff --git a/src/java/org/apache/cassandra/cql3/SuperColumnCompatibility.java b/src/java/org/apache/cassandra/cql3/SuperColumnCompatibility.java
deleted file mode 100644
index 1fe0af0..0000000
--- a/src/java/org/apache/cassandra/cql3/SuperColumnCompatibility.java
+++ /dev/null
@@ -1,763 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.cql3;
-
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Set;
-import java.util.TreeSet;
-
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
-import org.apache.cassandra.cql3.restrictions.SingleColumnRestriction;
-import org.apache.cassandra.cql3.restrictions.SingleRestriction;
-import org.apache.cassandra.cql3.restrictions.TermSlice;
-import org.apache.cassandra.cql3.selection.Selection;
-import org.apache.cassandra.cql3.statements.Bound;
-import org.apache.cassandra.db.Clustering;
-import org.apache.cassandra.db.Columns;
-import org.apache.cassandra.db.CompactTables;
-import org.apache.cassandra.db.PartitionColumns;
-import org.apache.cassandra.db.filter.ColumnFilter;
-import org.apache.cassandra.db.filter.RowFilter;
-import org.apache.cassandra.db.marshal.AbstractType;
-import org.apache.cassandra.db.marshal.MapType;
-import org.apache.cassandra.db.marshal.UTF8Type;
-import org.apache.cassandra.db.rows.Cell;
-import org.apache.cassandra.db.rows.CellPath;
-import org.apache.cassandra.db.rows.ComplexColumnData;
-import org.apache.cassandra.db.rows.Row;
-import org.apache.cassandra.db.rows.RowIterator;
-import org.apache.cassandra.transport.ProtocolVersion;
-import org.apache.cassandra.utils.ByteBufferUtil;
-import org.apache.cassandra.utils.FBUtilities;
-import org.apache.cassandra.utils.Pair;
-
-import static org.apache.cassandra.cql3.statements.RequestValidations.checkFalse;
-import static org.apache.cassandra.cql3.statements.RequestValidations.checkNotNull;
-import static org.apache.cassandra.cql3.statements.RequestValidations.checkTrue;
-import static org.apache.cassandra.cql3.statements.RequestValidations.invalidRequest;
-import static org.apache.cassandra.cql3.statements.SelectStatement.getComponents;
-
-/**
- * Class incapsulating the helper logic to handle SELECT / UPDATE / INSERT special-cases related
- * to SuperColumn tables in applicable scenarios.
- *
- * SuperColumn families have a special layout and are represented as a Map internally. These tables
- * have two special columns (called `column2` and `value` by default):
- *
- * * `column2`, {@link CFMetaData#superCfValueColumn}, a key of the SuperColumn map, exposed as a
- * REGULAR column, but stored in schema tables as a CLUSTERING column to make a distinction from
- * the SC value column in case of renames.
- * * `value`, {@link CFMetaData#compactValueColumn()}, a value of the SuperColumn map, exposed and
- * stored as a REGULAR column
- *
- * These columns have to be translated to this internal representation as key and value, correspondingly.
- *
- * In CQL terms, the SuperColumn families is encoded with:
- *
- * CREATE TABLE super (
- * key [key_validation_class],
- * super_column_name [comparator],
- * [column_metadata_1] [type1],
- * ...,
- * [column_metadata_n] [type1],
- * "" map<[sub_comparator], [default_validation_class]>
- * PRIMARY KEY (key, super_column_name)
- * )
- *
- * In other words, every super column is encoded by a row. That row has one column for each defined
- * "column_metadata", but it also has a special map column (whose name is the empty string as this is
- * guaranteed to never conflict with a user-defined "column_metadata") which stores the super column
- * "dynamic" sub-columns.
- *
- * On write path, `column2` and `value` columns are translated to the key and value of the
- * underlying map. During the read, the inverse conversion is done. Deletes are converted into
- * discards by the key in the underlying map. Counters are handled by translating an update to a
- * counter update with a cell path. See {@link SuperColumnRestrictions} for the details.
- *
- * Since non-dense SuperColumn families do not modify the contents of the internal map through in CQL
- * and do not expose this via CQL either, reads, writes and deletes are handled normally.
- *
- * Sidenote: a _dense_ SuperColumn Familiy is the one that has no added REGULAR columns.
- */
-public class SuperColumnCompatibility
-{
- // We use an empty value for the 1) this can't conflict with a user-defined column and 2) this actually
- // validate with any comparator which makes it convenient for columnDefinitionComparator().
- public static final ByteBuffer SUPER_COLUMN_MAP_COLUMN = ByteBufferUtil.EMPTY_BYTE_BUFFER;
- public static final String SUPER_COLUMN_MAP_COLUMN_STR = UTF8Type.instance.compose(SUPER_COLUMN_MAP_COLUMN);
-
- /**
- * Dense flag might have been incorrectly set if the node was upgraded from 2.x before CASSANDRA-12373.
- *
- * For 3.x created tables, the flag is set correctly in ThriftConversion code.
- */
- public static boolean recalculateIsDense(Columns columns)
- {
- return columns.size() == 1 && columns.getComplex(0).name.toString().isEmpty();
- }
-
- /**
- * For _dense_ SuperColumn Families, the supercolumn key column has to be translated to the collection subselection
- * query in order to avoid reading an entire collection and then filtering out the results.
- */
- public static ColumnFilter getColumnFilter(CFMetaData cfm, QueryOptions queryOptions, SuperColumnRestrictions restrictions)
- {
- assert cfm.isSuper() && cfm.isDense();
-
- ColumnFilter.Builder builder = ColumnFilter.selectionBuilder();
- builder.add(cfm.compactValueColumn());
-
- if (restrictions.keySliceRestriction != null)
- {
- SingleColumnRestriction.SuperColumnKeySliceRestriction restriction = restrictions.keySliceRestriction;
- TermSlice slice = restriction.slice;
-
- ByteBuffer start = slice.hasBound(Bound.START) ? slice.bound(Bound.START).bindAndGet(queryOptions) : null;
- ByteBuffer end = slice.hasBound(Bound.END) ? slice.bound(Bound.END).bindAndGet(queryOptions) : null;
-
- builder.slice(cfm.compactValueColumn(),
- start == null ? CellPath.BOTTOM : CellPath.create(start),
- end == null ? CellPath.TOP : CellPath.create(end));
- }
- else if (restrictions.keyEQRestriction != null)
- {
- SingleColumnRestriction.SuperColumnKeyEQRestriction restriction = restrictions.keyEQRestriction;
- ByteBuffer value = restriction.bindValue(queryOptions);
- builder.select(cfm.compactValueColumn(), CellPath.create(value));
- }
- else if (restrictions.keyINRestriction != null)
- {
- SingleColumnRestriction.SuperColumnKeyINRestriction cast = restrictions.keyINRestriction;
- Set<ByteBuffer> keyINRestrictionValues = new TreeSet<ByteBuffer>(((MapType) cfm.compactValueColumn().type).getKeysType());
- keyINRestrictionValues.addAll(cast.getValues(queryOptions));
-
- for (ByteBuffer value : keyINRestrictionValues)
- builder.select(cfm.compactValueColumn(), CellPath.create(value));
- }
- else if (restrictions.multiEQRestriction != null)
- {
- SingleColumnRestriction.SuperColumnMultiEQRestriction restriction = restrictions.multiEQRestriction;
- ByteBuffer value = restriction.secondValue;
- builder.select(cfm.compactValueColumn(), CellPath.create(value));
- }
-
- return builder.build();
- }
-
- /**
- * For _dense_ SuperColumn Families.
- *
- * On read path, instead of writing row per map, we have to write a row per key/value pair in map.
- *
- * For example:
- *
- * | partition-key | clustering-key | { key1: value1, key2: value2 } |
- *
- * Will be translated to:
- *
- * | partition-key | clustering-key | key1 | value1 |
- * | partition-key | clustering-key | key2 | value2 |
- *
- */
- public static void processPartition(CFMetaData cfm, Selection selection, RowIterator partition, Selection.ResultSetBuilder result, ProtocolVersion protocolVersion,
- SuperColumnRestrictions restrictions, QueryOptions queryOptions)
- {
- assert cfm.isDense();
- ByteBuffer[] keyComponents = getComponents(cfm, partition.partitionKey());
-
- int nowInSeconds = FBUtilities.nowInSeconds();
- while (partition.hasNext())
- {
- Row row = partition.next();
-
- ComplexColumnData ccd = row.getComplexColumnData(cfm.compactValueColumn());
-
- if (ccd == null)
- continue;
-
- Iterator<Cell> cellIter = ccd.iterator();
-
- outer:
- while (cellIter.hasNext())
- {
- Cell cell = cellIter.next();
- ByteBuffer superColumnKey = cell.path().get(0);
-
- if (restrictions != null)
- {
- // Slice on SuperColumn key
- if (restrictions.keySliceRestriction != null)
- {
- for (Bound bound : Bound.values())
- {
- if (restrictions.keySliceRestriction.hasBound(bound) &&
- !restrictions.keySliceRestriction.isInclusive(bound))
- {
- ByteBuffer excludedValue = restrictions.keySliceRestriction.bindValue(queryOptions);
- if (excludedValue.equals(superColumnKey))
- continue outer;
- }
- }
- }
-
- // Multi-column restriction on clustering+SuperColumn key
- if (restrictions.multiSliceRestriction != null &&
- cfm.comparator.compare(row.clustering(), Clustering.make(restrictions.multiSliceRestriction.firstValue)) == 0)
- {
- AbstractType t = ((MapType) cfm.compactValueColumn().type).getKeysType();
- int cmp = t.compare(superColumnKey, restrictions.multiSliceRestriction.secondValue);
-
- if ((cmp == 0 && !restrictions.multiSliceRestriction.trueInclusive) || // EQ
- (restrictions.multiSliceRestriction.hasBound(Bound.END) && cmp > 0) || // LT
- (restrictions.multiSliceRestriction.hasBound(Bound.START) && cmp < 0)) // GT
- continue outer;
- }
- }
-
- Row staticRow = partition.staticRow();
- result.newRow(partition.partitionKey(), staticRow.clustering());
-
- for (ColumnDefinition def : selection.getColumns())
- {
- if (cfm.isSuperColumnKeyColumn(def))
- {
- result.add(superColumnKey);
- }
- else if (cfm.isSuperColumnValueColumn(def))
- {
- result.add(cell, nowInSeconds);
- }
- else
- {
- switch (def.kind)
- {
- case PARTITION_KEY:
- result.add(keyComponents[def.position()]);
- break;
- case CLUSTERING:
- result.add(row.clustering().get(def.position()));
- break;
- case REGULAR:
- case STATIC:
- throw new AssertionError(String.format("Invalid column '%s' found in SuperColumn table", def.name.toString()));
- }
- }
- }
- }
- }
- }
-
- /**
- * For _dense_ SuperColumn Families.
- *
- * On the write path, we have to do combine the columns into a key/value pair:
- *
- * So inserting a row:
- *
- * | partition-key | clustering-key | key1 | value1 |
- *
- * Would result into:
- *
- * | partition-key | clustering-key | {key1: value1} |
- *
- * or adding / overwriting the value for `key1`.
- */
- public static void prepareInsertOperations(CFMetaData cfm,
- List<ColumnDefinition.Raw> columnNames,
- WhereClause.Builder whereClause,
- List<Term.Raw> columnValues,
- VariableSpecifications boundNames,
- Operations operations)
- {
- List<ColumnDefinition> defs = new ArrayList<>(columnNames.size());
- for (int i = 0; i < columnNames.size(); i++)
- {
- ColumnDefinition id = columnNames.get(i).prepare(cfm);
- defs.add(id);
- }
-
- prepareInsertOperations(cfm, defs, boundNames, columnValues, whereClause, operations);
- }
-
- /**
- * For _dense_ SuperColumn Families.
- *
- * {@link #prepareInsertOperations(CFMetaData, List, VariableSpecifications, List, WhereClause.Builder, Operations)},
- * but for INSERT JSON queries
- */
- public static void prepareInsertJSONOperations(CFMetaData cfm,
- List<ColumnDefinition> defs,
- VariableSpecifications boundNames,
- Json.Prepared prepared,
- WhereClause.Builder whereClause,
- Operations operations)
- {
- List<Term.Raw> columnValues = new ArrayList<>(defs.size());
- for (ColumnDefinition def : defs)
- columnValues.add(prepared.getRawTermForColumn(def, true));
-
- prepareInsertOperations(cfm, defs, boundNames, columnValues, whereClause, operations);
- }
-
- private static void prepareInsertOperations(CFMetaData cfm,
- List<ColumnDefinition> defs,
- VariableSpecifications boundNames,
- List<Term.Raw> columnValues,
- WhereClause.Builder whereClause,
- Operations operations)
- {
- assert cfm.isDense();
- assert defs.size() == columnValues.size();
-
- Term.Raw superColumnKey = null;
- Term.Raw superColumnValue = null;
-
- for (int i = 0, size = defs.size(); i < size; i++)
- {
- ColumnDefinition def = defs.get(i);
- Term.Raw raw = columnValues.get(i);
-
- if (cfm.isSuperColumnKeyColumn(def))
- {
- superColumnKey = raw;
- collectMarkerSpecifications(raw, boundNames, def);
- }
- else if (cfm.isSuperColumnValueColumn(def))
- {
- superColumnValue = raw;
- collectMarkerSpecifications(raw, boundNames, def);
- }
- else if (def.isPrimaryKeyColumn())
- {
- whereClause.add(new SingleColumnRelation(ColumnDefinition.Raw.forColumn(def), Operator.EQ, raw));
- }
- else
- {
- throw invalidRequest("Invalid column {} in where clause");
- }
- }
-
- checkTrue(superColumnValue != null,
- "Column value is mandatory for SuperColumn tables");
- checkTrue(superColumnKey != null,
- "Column key is mandatory for SuperColumn tables");
-
- Operation operation = new Operation.SetElement(superColumnKey, superColumnValue).prepare(cfm, cfm.compactValueColumn());
- operations.add(operation);
- }
-
- /**
- * Collect the marker specifications for the bound columns manually, since the operations on a column are
- * converted to the operations on the collection element.
- */
- private static void collectMarkerSpecifications(Term.Raw raw, VariableSpecifications boundNames, ColumnDefinition def)
- {
- if (raw instanceof AbstractMarker.Raw)
- boundNames.add(((AbstractMarker.Raw) raw).bindIndex(), def);
- }
-
- /**
- * For _dense_ SuperColumn Families.
- *
- * During UPDATE operation, the update by clustering (with correponding relation in WHERE clause)
- * has to be substituted with an update to the map that backs the given SuperColumn.
- *
- * For example, an update such as:
- *
- * UPDATE ... SET value = 'value1' WHERE key = 'pk' AND column1 = 'ck' AND column2 = 'mk'
- *
- * Will update the value under key 'mk' in the map, backing the SuperColumn, located in the row
- * with clustering 'ck' in the partition with key 'pk'.
- */
- public static WhereClause prepareUpdateOperations(CFMetaData cfm,
- WhereClause whereClause,
- List<Pair<ColumnDefinition.Raw, Operation.RawUpdate>> updates,
- VariableSpecifications boundNames,
- Operations operations)
- {
- assert cfm.isDense();
- Term.Raw superColumnKey = null;
- Term.Raw superColumnValue = null;
-
- List<Relation> newRelations = new ArrayList<>(whereClause.relations.size());
- for (int i = 0; i < whereClause.relations.size(); i++)
- {
- SingleColumnRelation relation = (SingleColumnRelation) whereClause.relations.get(i);
- ColumnDefinition def = relation.getEntity().prepare(cfm);
-
- if (cfm.isSuperColumnKeyColumn(def))
- {
- superColumnKey = relation.getValue();
- collectMarkerSpecifications(superColumnKey, boundNames, def);
- }
- else
- {
- newRelations.add(relation);
- }
- }
-
- checkTrue(superColumnKey != null,
- "Column key is mandatory for SuperColumn tables");
-
- for (Pair<ColumnDefinition.Raw, Operation.RawUpdate> entry : updates)
- {
- ColumnDefinition def = entry.left.prepare(cfm);
-
- if (!cfm.isSuperColumnValueColumn(def))
- throw invalidRequest("Column `%s` of type `%s` found in SET part", def.name, def.type.asCQL3Type());
-
- Operation operation;
-
- if (entry.right instanceof Operation.Addition)
- {
- Operation.Addition op = (Operation.Addition) entry.right;
- superColumnValue = op.value();
-
- operation = new Operation.ElementAddition(superColumnKey, superColumnValue).prepare(cfm, cfm.compactValueColumn());
- }
- else if (entry.right instanceof Operation.Substraction)
- {
- Operation.Substraction op = (Operation.Substraction) entry.right;
- superColumnValue = op.value();
-
- operation = new Operation.ElementSubtraction(superColumnKey, superColumnValue).prepare(cfm, cfm.compactValueColumn());
- }
- else if (entry.right instanceof Operation.SetValue)
- {
- Operation.SetValue op = (Operation.SetValue) entry.right;
- superColumnValue = op.value();
-
- operation = new Operation.SetElement(superColumnKey, superColumnValue).prepare(cfm, cfm.compactValueColumn());
- }
- else
- {
- throw invalidRequest("Invalid operation `%s` on column `%s` of type `%s` found in SET part", entry.right, def.name, def.type.asCQL3Type());
- }
-
- collectMarkerSpecifications(superColumnValue, boundNames, def);
- operations.add(operation);
- }
-
- checkTrue(superColumnValue != null,
- "Column value is mandatory for SuperColumn tables");
-
- return newRelations.size() != whereClause.relations.size() ? whereClause.copy(newRelations) : whereClause;
- }
-
- /**
- * Rebuilds LWT conditions on SuperColumn _value_ column.
- *
- * Conditions have to be changed to correspond the internal representation of SuperColumn value, since it's not
- * a separate column, but a value in a hidden compact value column.
- */
- public static Conditions rebuildLWTColumnConditions(Conditions conditions, CFMetaData cfm, WhereClause whereClause)
- {
- if (conditions.isEmpty() || conditions.isIfExists() || conditions.isIfNotExists())
- return conditions;
-
- ColumnConditions.Builder builder = ColumnConditions.newBuilder();
- Collection<ColumnCondition> columnConditions = ((ColumnConditions) conditions).columnConditions();
-
- Pair<ColumnDefinition, Relation> superColumnKeyRelation = SuperColumnCompatibility.getSuperColumnKeyRelation(whereClause.relations, cfm);
-
- checkNotNull(superColumnKeyRelation,
- "Lightweight transactions on SuperColumn tables are only supported with supplied SuperColumn key");
-
- for (ColumnCondition columnCondition : columnConditions)
- {
- checkTrue(cfm.isSuperColumnValueColumn(columnCondition.column),
- "Lightweight transactions are only supported on the value column of SuperColumn tables");
-
- Term.Raw value = superColumnKeyRelation.right.getValue();
- Term collectionElemnt = value instanceof AbstractMarker.Raw ?
- new Constants.Marker(((AbstractMarker.Raw) value).bindIndex(),
- superColumnKeyRelation.left) :
- value.prepare(cfm.ksName, superColumnKeyRelation.left);
- builder.add(ColumnCondition.condition(cfm.compactValueColumn(),
- collectionElemnt,
- columnCondition.value(), columnCondition.operator));
- }
-
- return builder.build();
- }
-
- /**
- * Returns a relation on the SuperColumn key
- */
- private static Pair<ColumnDefinition, Relation> getSuperColumnKeyRelation(List<Relation> relations, CFMetaData cfm)
- {
- for (int i = 0; i < relations.size(); i++)
- {
- SingleColumnRelation relation = (SingleColumnRelation) relations.get(i);
- ColumnDefinition def = relation.getEntity().prepare(cfm);
-
- if (cfm.isSuperColumnKeyColumn(def))
- return Pair.create(def, relation);
- }
- return null;
- }
-
- /**
- * For _dense_ SuperColumn Families.
- *
- * Delete, when the "regular" columns are present, have to be translated into
- * deletion of value in the internal map by key.
- *
- * For example, delete such as:
- *
- * DELETE FROM ... WHERE key = 'pk' AND column1 = 'ck' AND column2 = 'mk'
- *
- * Will delete a value under 'mk' from the map, located in the row with clustering key 'ck' in the partition
- * with key 'pk'.
- */
- public static WhereClause prepareDeleteOperations(CFMetaData cfm,
- WhereClause whereClause,
- VariableSpecifications boundNames,
- Operations operations)
- {
- assert cfm.isDense();
- List<Relation> newRelations = new ArrayList<>(whereClause.relations.size());
-
- for (int i = 0; i < whereClause.relations.size(); i++)
- {
- Relation orig = whereClause.relations.get(i);
-
- checkFalse(orig.isMultiColumn(),
- "Multi-column relations cannot be used in WHERE clauses for UPDATE and DELETE statements: %s", orig);
- checkFalse(orig.onToken(),
- "Token relations cannot be used in WHERE clauses for UPDATE and DELETE statements: %s", orig);
-
- SingleColumnRelation relation = (SingleColumnRelation) orig;
- ColumnDefinition def = relation.getEntity().prepare(cfm);
-
- if (cfm.isSuperColumnKeyColumn(def))
- {
- Term.Raw value = relation.getValue();
-
- if (value instanceof AbstractMarker.Raw)
- boundNames.add(((AbstractMarker.Raw) value).bindIndex(), def);
-
- Operation operation = new Maps.DiscarderByKey(cfm.compactValueColumn(), value.prepare(cfm.ksName, def));
- operations.add(operation);
- }
- else
- {
- newRelations.add(relation);
- }
- }
-
- return newRelations.size() != whereClause.relations.size() ? whereClause.copy(newRelations) : whereClause;
- }
-
- /**
- * Create a column name generator for SuperColumns
- */
- public static CompactTables.DefaultNames columnNameGenerator(List<ColumnDefinition> partitionKeyColumns,
- List<ColumnDefinition> clusteringColumns,
- PartitionColumns partitionColumns)
- {
- Set<String> names = new HashSet<>();
- // If the clustering column was renamed, the supercolumn key's default nname still can't be `column1` (SuperColumn
- // key renames are handled separately by looking up an existing column).
- names.add("column1");
- for (ColumnDefinition columnDefinition: partitionKeyColumns)
- names.add(columnDefinition.name.toString());
- for (ColumnDefinition columnDefinition: clusteringColumns)
- names.add(columnDefinition.name.toString());
- for (ColumnDefinition columnDefinition: partitionColumns)
- names.add(columnDefinition.name.toString());
-
- return CompactTables.defaultNameGenerator(names);
- }
-
- /**
- * Find a SuperColumn key column if it's available (for example, when it was renamed) or create one with a default name.
- */
- public static ColumnDefinition getSuperCfKeyColumn(CFMetaData cfm, List<ColumnDefinition> clusteringColumns, CompactTables.DefaultNames defaultNames)
- {
- assert cfm.isDense();
-
- MapType mapType = (MapType) cfm.compactValueColumn().type;
- // Pre CASSANDRA-12373 3.x-created supercolumn family
- if (clusteringColumns.size() == 1)
- {
- // create a new one with a default name
- ColumnIdentifier identifier = ColumnIdentifier.getInterned(defaultNames.defaultClusteringName(), true);
- return new ColumnDefinition(cfm.ksName, cfm.cfName, identifier, mapType.getKeysType(), ColumnDefinition.NO_POSITION, ColumnDefinition.Kind.REGULAR);
- }
-
- // Upgrade path: table created in 2.x, handle pre-created columns and/or renames.
- assert clusteringColumns.size() == 2 : clusteringColumns;
- ColumnDefinition cd = clusteringColumns.get(1);
-
- assert cd.type.equals(mapType.getKeysType()) : cd.type + " != " + mapType.getKeysType();
- return new ColumnDefinition(cfm.ksName, cfm.cfName, cd.name, mapType.getKeysType(), ColumnDefinition.NO_POSITION, ColumnDefinition.Kind.REGULAR);
- }
-
- /**
- * Find a SuperColumn value column if it's available (for example, when it was renamed) or create one with a default name.
- */
- public static ColumnDefinition getSuperCfValueColumn(CFMetaData cfm, PartitionColumns partitionColumns, ColumnDefinition superCfKeyColumn, CompactTables.DefaultNames defaultNames)
- {
- assert cfm.isDense();
-
- MapType mapType = (MapType) cfm.compactValueColumn().type;
- for (ColumnDefinition def: partitionColumns.regulars)
- {
- if (!def.name.bytes.equals(SUPER_COLUMN_MAP_COLUMN) && def.type.equals(mapType.getValuesType()) && !def.equals(superCfKeyColumn))
- return def;
- }
-
- ColumnIdentifier identifier = ColumnIdentifier.getInterned(defaultNames.defaultCompactValueName(), true);
- return new ColumnDefinition(cfm.ksName, cfm.cfName, identifier, mapType.getValuesType(), ColumnDefinition.NO_POSITION, ColumnDefinition.Kind.REGULAR);
- }
-
- /**
- * SuperColumn key is stored in {@link CFMetaData#columnMetadata} as a clustering column (to make sure we can make
- * a distinction between the SuperColumn key and SuperColumn value columns, especially when they have the same type
- * and were renamed), but exposed as {@link CFMetaData#superCfKeyColumn} as a regular column to be compatible with
- * the storage engine.
- *
- * This remapping is necessary to facilitate the column metadata part.
- */
- public static ColumnDefinition getSuperCfSschemaRepresentation(ColumnDefinition superCfKeyColumn)
- {
- return new ColumnDefinition(superCfKeyColumn.ksName, superCfKeyColumn.cfName, superCfKeyColumn.name, superCfKeyColumn.type, 1, ColumnDefinition.Kind.CLUSTERING);
- }
-
- public static boolean isSuperColumnMapColumn(ColumnDefinition column)
- {
- return column.isRegular() && column.name.bytes.equals(SuperColumnCompatibility.SUPER_COLUMN_MAP_COLUMN);
- }
-
- public static ColumnDefinition getCompactValueColumn(PartitionColumns columns)
- {
- for (ColumnDefinition column : columns.regulars)
- {
- if (isSuperColumnMapColumn(column))
- return column;
- }
- throw new AssertionError("Invalid super column table definition, no 'dynamic' map column");
- }
-
- /**
- * Restrictions are the trickiest part of the SuperColumn integration.
- * See specific docs on each field. For the purpose of this doc, the "default" column names are used,
- * `column2` and `value`. Detailed description and semantics of these fields can be found in this class'
- * header comment.
- */
- public static class SuperColumnRestrictions
- {
- /**
- * Restrictions in the form of:
- * ... AND (column1, column2) > ('value1', 1)
- * Multi-column restrictions. `column1` will be handled normally by the clustering bounds,
- * and `column2` value has to be "saved" and filtered out in `processPartition`, as there's no
- * direct mapping of multi-column restrictions to clustering + cell path. The first row
- * is special-cased to make sure the semantics of multi-column restrictions are preserved.
- */
- private final SingleColumnRestriction.SuperColumnMultiSliceRestriction multiSliceRestriction;
-
- /**
- * Restrictions in the form of:
- * ... AND (column1, column2) = ('value1', 1)
- * Multi-column restriction with EQ does have a direct mapping: `column1` will be handled
- * normally by the clustering bounds, and the `column2` will be special-cased by the
- * {@link #getColumnFilter(CFMetaData, QueryOptions, SuperColumnRestrictions)} as a collection path lookup.
- */
- private final SingleColumnRestriction.SuperColumnMultiEQRestriction multiEQRestriction;
-
- /**
- * Restrictions in the form of:
- * ... AND column2 >= 5
- * For non-filtering cases (when the preceding clustering column and a partition key are
- * restricted), will be handled in {@link #getColumnFilter(CFMetaData, QueryOptions, SuperColumnRestrictions)}
- * like an inclusive bounds lookup.
- *
- * For the restrictions taking a form of
- * ... AND column2 > 5
- * (non-inclusive ones), the items that match `=` will be filtered out
- * by {@link #processPartition(CFMetaData, Selection, RowIterator, Selection.ResultSetBuilder, ProtocolVersion, SuperColumnRestrictions, QueryOptions)}
- *
- * Unfortunately, there are no good ways to do it other than here:
- * {@link RowFilter} can't be used in this case, since the complex collection cells are not yet rows by that
- * point.
- * {@link ColumnFilter} (which is used for inclusive slices) can't be changed to support exclusive slices as it would
- * require a protocol change in order to add a Kind. So exclusive slices are a combination of inclusive plus
- * an ad-hoc filter.
- */
- private final SingleColumnRestriction.SuperColumnKeySliceRestriction keySliceRestriction;
-
- /**
- * Restrictions in the form of:
- * ... AND column2 IN (1, 2, 3)
- * For non-filtering cases (when the preceeding clustering column and a partition key are
- * restricted), are handled in {@link #getColumnFilter(CFMetaData, QueryOptions, SuperColumnRestrictions)} by
- * adding multiple collection paths to the {@link ColumnFilter}
- */
- private final SingleColumnRestriction.SuperColumnKeyINRestriction keyINRestriction;
-
- /**
- * Restrictions in the form of:
- * ... AND column2 = 1
- * For non-filtering cases (when the preceeding clustering column and a partition key are
- * restricted), will be handled by converting the restriction to the column filter on
- * the collection key in {@link #getColumnFilter(CFMetaData, QueryOptions, SuperColumnRestrictions)}
- */
- private final SingleColumnRestriction.SuperColumnKeyEQRestriction keyEQRestriction;
-
- public SuperColumnRestrictions(Iterator<SingleRestriction> restrictions)
- {
- // In order to keep the fields final, assignments have to be done outside the loop
- SingleColumnRestriction.SuperColumnMultiSliceRestriction multiSliceRestriction = null;
- SingleColumnRestriction.SuperColumnKeySliceRestriction keySliceRestriction = null;
- SingleColumnRestriction.SuperColumnKeyINRestriction keyINRestriction = null;
- SingleColumnRestriction.SuperColumnMultiEQRestriction multiEQRestriction = null;
- SingleColumnRestriction.SuperColumnKeyEQRestriction keyEQRestriction = null;
-
- while (restrictions.hasNext())
- {
- SingleRestriction restriction = restrictions.next();
-
- if (restriction instanceof SingleColumnRestriction.SuperColumnMultiSliceRestriction)
- multiSliceRestriction = (SingleColumnRestriction.SuperColumnMultiSliceRestriction) restriction;
- else if (restriction instanceof SingleColumnRestriction.SuperColumnKeySliceRestriction)
- keySliceRestriction = (SingleColumnRestriction.SuperColumnKeySliceRestriction) restriction;
- else if (restriction instanceof SingleColumnRestriction.SuperColumnKeyINRestriction)
- keyINRestriction = (SingleColumnRestriction.SuperColumnKeyINRestriction) restriction;
- else if (restriction instanceof SingleColumnRestriction.SuperColumnMultiEQRestriction)
- multiEQRestriction = (SingleColumnRestriction.SuperColumnMultiEQRestriction) restriction;
- else if (restriction instanceof SingleColumnRestriction.SuperColumnKeyEQRestriction)
- keyEQRestriction = (SingleColumnRestriction.SuperColumnKeyEQRestriction) restriction;
- }
-
- this.multiSliceRestriction = multiSliceRestriction;
- this.keySliceRestriction = keySliceRestriction;
- this.keyINRestriction = keyINRestriction;
- this.multiEQRestriction = multiEQRestriction;
- this.keyEQRestriction = keyEQRestriction;
- }
- }
-}
diff --git a/src/java/org/apache/cassandra/cql3/Term.java b/src/java/org/apache/cassandra/cql3/Term.java
index 11b9860..f536baa 100644
--- a/src/java/org/apache/cassandra/cql3/Term.java
+++ b/src/java/org/apache/cassandra/cql3/Term.java
@@ -70,6 +70,14 @@
*/
public abstract boolean containsBindMarker();
+ /**
+ * Whether that term is terminal (this is a shortcut for {@code this instanceof Term.Terminal}).
+ */
+ default public boolean isTerminal()
+ {
+ return false; // overriden below by Terminal
+ }
+
public void addFunctionsTo(List<Function> functions);
/**
@@ -116,6 +124,18 @@
{
return getText();
}
+
+ @Override
+ public int hashCode()
+ {
+ return getText().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object o)
+ {
+ return this == o || (o instanceof Raw && getText().equals(((Raw) o).getText()));
+ }
}
public abstract class MultiColumnRaw extends Term.Raw
@@ -153,6 +173,12 @@
return false;
}
+ @Override
+ public boolean isTerminal()
+ {
+ return true;
+ }
+
/**
* @return the serialized value of this terminal.
* @param protocolVersion
diff --git a/src/java/org/apache/cassandra/cql3/Terms.java b/src/java/org/apache/cassandra/cql3/Terms.java
index 7d3948a..33ce2e9 100644
--- a/src/java/org/apache/cassandra/cql3/Terms.java
+++ b/src/java/org/apache/cassandra/cql3/Terms.java
@@ -18,17 +18,247 @@
package org.apache.cassandra.cql3;
import java.nio.ByteBuffer;
-import java.util.List;
+import java.util.*;
+import org.apache.cassandra.cql3.Term.MultiItemTerminal;
+import org.apache.cassandra.cql3.Term.Terminal;
import org.apache.cassandra.cql3.functions.Function;
-import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.*;
+import org.apache.cassandra.transport.ProtocolVersion;
-public class Terms
+/**
+ * A set of {@code Terms}
+ */
+public interface Terms
{
+ /**
+ * The {@code List} returned when the list was not set.
+ */
+ @SuppressWarnings("rawtypes")
+ public static final List UNSET_LIST = new AbstractList()
+ {
+ @Override
+ public Object get(int index)
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public int size()
+ {
+ return 0;
+ }
+ };
+
+ /**
+ * Adds all functions (native and user-defined) used by any of the terms to the specified list.
+ * @param functions the list to add to
+ */
+ public void addFunctionsTo(List<Function> functions);
+
+ /**
+ * Collects the column specifications for the bind variables in the terms.
+ * This is obviously a no-op if the terms are Terminal.
+ *
+ * @param boundNames the variables specification where to collect the
+ * bind variables of the terms in.
+ */
+ public void collectMarkerSpecification(VariableSpecifications boundNames);
+
+ /**
+ * Bind the values in these terms to the values contained in {@code options}.
+ * This is obviously a no-op if the term is Terminal.
+ *
+ * @param options the values to bind markers to.
+ * @return the result of binding all the variables of these NonTerminals or an {@code UNSET_LIST} if the term
+ * was unset.
+ */
+ public List<Terminal> bind(QueryOptions options);
+
+
+ public List<ByteBuffer> bindAndGet(QueryOptions options);
+
+ /**
+ * Creates a {@code Terms} for the specified list marker.
+ *
+ * @param marker the list marker
+ * @param type the element type
+ * @return a {@code Terms} for the specified list marker
+ */
+ public static Terms ofListMarker(final Lists.Marker marker, final AbstractType<?> type)
+ {
+ return new Terms()
+ {
+ @Override
+ public void addFunctionsTo(List<Function> functions)
+ {
+ }
+
+ @Override
+ public void collectMarkerSpecification(VariableSpecifications boundNames)
+ {
+ marker.collectMarkerSpecification(boundNames);
+ }
+
+ @Override
+ public List<ByteBuffer> bindAndGet(QueryOptions options)
+ {
+ Terminal terminal = marker.bind(options);
+
+ if (terminal == null)
+ return null;
+
+ if (terminal == Constants.UNSET_VALUE)
+ return UNSET_LIST;
+
+ return ((MultiItemTerminal) terminal).getElements();
+ }
+
+ @Override
+ public List<Terminal> bind(QueryOptions options)
+ {
+ Terminal terminal = marker.bind(options);
+
+ if (terminal == null)
+ return null;
+
+ if (terminal == Constants.UNSET_VALUE)
+ return UNSET_LIST;
+
+ java.util.function.Function<ByteBuffer, Term.Terminal> deserializer = deserializer(options.getProtocolVersion());
+
+ List<ByteBuffer> boundValues = ((MultiItemTerminal) terminal).getElements();
+ List<Term.Terminal> values = new ArrayList<>(boundValues.size());
+ for (int i = 0, m = boundValues.size(); i < m; i++)
+ {
+ ByteBuffer buffer = boundValues.get(i);
+ Term.Terminal value = buffer == null ? null : deserializer.apply(buffer);
+ values.add(value);
+ }
+ return values;
+ }
+
+ public java.util.function.Function<ByteBuffer, Term.Terminal> deserializer(ProtocolVersion version)
+ {
+ if (type.isCollection())
+ {
+ switch (((CollectionType<?>) type).kind)
+ {
+ case LIST:
+ return e -> Lists.Value.fromSerialized(e, (ListType<?>) type, version);
+ case SET:
+ return e -> Sets.Value.fromSerialized(e, (SetType<?>) type, version);
+ case MAP:
+ return e -> Maps.Value.fromSerialized(e, (MapType<?, ?>) type, version);
+ }
+ throw new AssertionError();
+ }
+ return e -> new Constants.Value(e);
+ }
+ };
+ }
+
+ /**
+ * Creates a {@code Terms} containing a single {@code Term}.
+ *
+ * @param term the {@code Term}
+ * @return a {@code Terms} containing a single {@code Term}.
+ */
+ public static Terms of(final Term term)
+ {
+ return new Terms()
+ {
+ @Override
+ public void addFunctionsTo(List<Function> functions)
+ {
+ term.addFunctionsTo(functions);
+ }
+
+ @Override
+ public void collectMarkerSpecification(VariableSpecifications boundNames)
+ {
+ term.collectMarkerSpecification(boundNames);
+ }
+
+ @Override
+ public List<ByteBuffer> bindAndGet(QueryOptions options)
+ {
+ return Collections.singletonList(term.bindAndGet(options));
+ }
+
+ @Override
+ public List<Terminal> bind(QueryOptions options)
+ {
+ return Collections.singletonList(term.bind(options));
+ }
+ };
+ }
+
+ /**
+ * Creates a {@code Terms} containing a set of {@code Term}.
+ *
+ * @param term the {@code Term}
+ * @return a {@code Terms} containing a set of {@code Term}.
+ */
+ public static Terms of(final List<Term> terms)
+ {
+ return new Terms()
+ {
+ @Override
+ public void addFunctionsTo(List<Function> functions)
+ {
+ addFunctions(terms, functions);
+ }
+
+ @Override
+ public void collectMarkerSpecification(VariableSpecifications boundNames)
+ {
+ for (int i = 0, m = terms.size(); i <m; i++)
+ {
+ Term term = terms.get(i);
+ term.collectMarkerSpecification(boundNames);
+ }
+ }
+
+ @Override
+ public List<Terminal> bind(QueryOptions options)
+ {
+ int size = terms.size();
+ List<Terminal> terminals = new ArrayList<>(size);
+ for (int i = 0; i < size; i++)
+ {
+ Term term = terms.get(i);
+ terminals.add(term.bind(options));
+ }
+ return terminals;
+ }
+
+ @Override
+ public List<ByteBuffer> bindAndGet(QueryOptions options)
+ {
+ int size = terms.size();
+ List<ByteBuffer> buffers = new ArrayList<>(size);
+ for (int i = 0; i < size; i++)
+ {
+ Term term = terms.get(i);
+ buffers.add(term.bindAndGet(options));
+ }
+ return buffers;
+ }
+ };
+ }
+
+ /**
+ * Adds all functions (native and user-defined) of the specified terms to the list.
+ * @param functions the list to add to
+ */
public static void addFunctions(Iterable<Term> terms, List<Function> functions)
{
- if (terms != null)
- terms.forEach(t -> t.addFunctionsTo(functions));
+ for (Term term : terms)
+ {
+ if (term != null)
+ term.addFunctionsTo(functions);
+ }
}
public static ByteBuffer asBytes(String keyspace, String term, AbstractType type)
diff --git a/src/java/org/apache/cassandra/cql3/TokenRelation.java b/src/java/org/apache/cassandra/cql3/TokenRelation.java
index 42464ef..0919c50 100644
--- a/src/java/org/apache/cassandra/cql3/TokenRelation.java
+++ b/src/java/org/apache/cassandra/cql3/TokenRelation.java
@@ -20,12 +20,13 @@
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
+import java.util.Objects;
import java.util.stream.Collectors;
import com.google.common.base.Joiner;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.cql3.Term.Raw;
import org.apache.cassandra.cql3.restrictions.Restriction;
import org.apache.cassandra.cql3.restrictions.TokenRestriction;
@@ -47,11 +48,11 @@
*/
public final class TokenRelation extends Relation
{
- private final List<ColumnDefinition.Raw> entities;
+ private final List<ColumnMetadata.Raw> entities;
private final Term.Raw value;
- public TokenRelation(List<ColumnDefinition.Raw> entities, Operator type, Term.Raw value)
+ public TokenRelation(List<ColumnMetadata.Raw> entities, Operator type, Term.Raw value)
{
this.entities = entities;
this.relationType = type;
@@ -75,44 +76,44 @@
}
@Override
- protected Restriction newEQRestriction(CFMetaData cfm, VariableSpecifications boundNames) throws InvalidRequestException
+ protected Restriction newEQRestriction(TableMetadata table, VariableSpecifications boundNames)
{
- List<ColumnDefinition> columnDefs = getColumnDefinitions(cfm);
- Term term = toTerm(toReceivers(cfm, columnDefs), value, cfm.ksName, boundNames);
- return new TokenRestriction.EQRestriction(cfm, columnDefs, term);
+ List<ColumnMetadata> columnDefs = getColumnDefinitions(table);
+ Term term = toTerm(toReceivers(table, columnDefs), value, table.keyspace, boundNames);
+ return new TokenRestriction.EQRestriction(table, columnDefs, term);
}
@Override
- protected Restriction newINRestriction(CFMetaData cfm, VariableSpecifications boundNames) throws InvalidRequestException
+ protected Restriction newINRestriction(TableMetadata table, VariableSpecifications boundNames)
{
throw invalidRequest("%s cannot be used with the token function", operator());
}
@Override
- protected Restriction newSliceRestriction(CFMetaData cfm,
+ protected Restriction newSliceRestriction(TableMetadata table,
VariableSpecifications boundNames,
Bound bound,
- boolean inclusive) throws InvalidRequestException
+ boolean inclusive)
{
- List<ColumnDefinition> columnDefs = getColumnDefinitions(cfm);
- Term term = toTerm(toReceivers(cfm, columnDefs), value, cfm.ksName, boundNames);
- return new TokenRestriction.SliceRestriction(cfm, columnDefs, bound, inclusive, term);
+ List<ColumnMetadata> columnDefs = getColumnDefinitions(table);
+ Term term = toTerm(toReceivers(table, columnDefs), value, table.keyspace, boundNames);
+ return new TokenRestriction.SliceRestriction(table, columnDefs, bound, inclusive, term);
}
@Override
- protected Restriction newContainsRestriction(CFMetaData cfm, VariableSpecifications boundNames, boolean isKey) throws InvalidRequestException
+ protected Restriction newContainsRestriction(TableMetadata table, VariableSpecifications boundNames, boolean isKey)
{
throw invalidRequest("%s cannot be used with the token function", operator());
}
@Override
- protected Restriction newIsNotRestriction(CFMetaData cfm, VariableSpecifications boundNames) throws InvalidRequestException
+ protected Restriction newIsNotRestriction(TableMetadata table, VariableSpecifications boundNames)
{
throw invalidRequest("%s cannot be used with the token function", operator());
}
@Override
- protected Restriction newLikeRestriction(CFMetaData cfm, VariableSpecifications boundNames, Operator operator) throws InvalidRequestException
+ protected Restriction newLikeRestriction(TableMetadata table, VariableSpecifications boundNames, Operator operator)
{
throw invalidRequest("%s cannot be used with the token function", operator);
}
@@ -128,12 +129,12 @@
return term;
}
- public Relation renameIdentifier(ColumnDefinition.Raw from, ColumnDefinition.Raw to)
+ public Relation renameIdentifier(ColumnMetadata.Raw from, ColumnMetadata.Raw to)
{
if (!entities.contains(from))
return this;
- List<ColumnDefinition.Raw> newEntities = entities.stream().map(e -> e.equals(from) ? to : e).collect(Collectors.toList());
+ List<ColumnMetadata.Raw> newEntities = entities.stream().map(e -> e.equals(from) ? to : e).collect(Collectors.toList());
return new TokenRelation(newEntities, operator(), value);
}
@@ -143,51 +144,70 @@
return String.format("token%s %s %s", Tuples.tupleToString(entities), relationType, value);
}
+ @Override
+ public int hashCode()
+ {
+ return Objects.hash(relationType, entities, value);
+ }
+
+ @Override
+ public boolean equals(Object o)
+ {
+ if (this == o)
+ return true;
+
+ if (!(o instanceof TokenRelation))
+ return false;
+
+ TokenRelation tr = (TokenRelation) o;
+ return relationType.equals(tr.relationType) && entities.equals(tr.entities) && value.equals(tr.value);
+ }
+
/**
* Returns the definition of the columns to which apply the token restriction.
*
- * @param cfm the column family metadata
+ * @param table the table metadata
* @return the definition of the columns to which apply the token restriction.
* @throws InvalidRequestException if the entity cannot be resolved
*/
- private List<ColumnDefinition> getColumnDefinitions(CFMetaData cfm) throws InvalidRequestException
+ private List<ColumnMetadata> getColumnDefinitions(TableMetadata table)
{
- List<ColumnDefinition> columnDefs = new ArrayList<>(entities.size());
- for ( ColumnDefinition.Raw raw : entities)
- columnDefs.add(raw.prepare(cfm));
+ List<ColumnMetadata> columnDefs = new ArrayList<>(entities.size());
+ for ( ColumnMetadata.Raw raw : entities)
+ columnDefs.add(raw.prepare(table));
return columnDefs;
}
/**
* Returns the receivers for this relation.
*
- * @param cfm the Column Family meta data
+ * @param table the table meta data
* @param columnDefs the column definitions
* @return the receivers for the specified relation.
* @throws InvalidRequestException if the relation is invalid
*/
- private static List<? extends ColumnSpecification> toReceivers(CFMetaData cfm,
- List<ColumnDefinition> columnDefs)
+ private static List<? extends ColumnSpecification> toReceivers(TableMetadata table,
+ List<ColumnMetadata> columnDefs)
throws InvalidRequestException
{
- if (!columnDefs.equals(cfm.partitionKeyColumns()))
+ if (!columnDefs.equals(table.partitionKeyColumns()))
{
- checkTrue(columnDefs.containsAll(cfm.partitionKeyColumns()),
+ checkTrue(columnDefs.containsAll(table.partitionKeyColumns()),
"The token() function must be applied to all partition key components or none of them");
checkContainsNoDuplicates(columnDefs, "The token() function contains duplicate partition key components");
- checkContainsOnly(columnDefs, cfm.partitionKeyColumns(), "The token() function must contains only partition key components");
+ checkContainsOnly(columnDefs, table.partitionKeyColumns(), "The token() function must contains only partition key components");
throw invalidRequest("The token function arguments must be in the partition key order: %s",
- Joiner.on(", ").join(ColumnDefinition.toIdentifiers(cfm.partitionKeyColumns())));
+ Joiner.on(", ").join(ColumnMetadata.toIdentifiers(table.partitionKeyColumns())));
}
- ColumnDefinition firstColumn = columnDefs.get(0);
+ ColumnMetadata firstColumn = columnDefs.get(0);
return Collections.singletonList(new ColumnSpecification(firstColumn.ksName,
firstColumn.cfName,
new ColumnIdentifier("partition key token", true),
- cfm.partitioner.getTokenValidator()));
+ table.partitioner.getTokenValidator()));
}
}
diff --git a/src/java/org/apache/cassandra/cql3/Tuples.java b/src/java/org/apache/cassandra/cql3/Tuples.java
index 01f3466..317e192 100644
--- a/src/java/org/apache/cassandra/cql3/Tuples.java
+++ b/src/java/org/apache/cassandra/cql3/Tuples.java
@@ -22,6 +22,7 @@
import java.util.Arrays;
import java.util.List;
import java.util.stream.Collectors;
+import java.util.stream.StreamSupport;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -33,6 +34,8 @@
import org.apache.cassandra.transport.ProtocolVersion;
import org.apache.cassandra.utils.ByteBufferUtil;
+import static org.apache.cassandra.cql3.statements.RequestValidations.invalidRequest;
+
/**
* Static helper methods and classes for tuples.
*/
@@ -65,7 +68,12 @@
public Term prepare(String keyspace, ColumnSpecification receiver) throws InvalidRequestException
{
- validateAssignableTo(keyspace, receiver);
+ // The parser cannot differentiate between a tuple with one element and a term between parenthesis.
+ // By consequence, we need to wait until we know the target type to determine which one it is.
+ if (elements.size() == 1 && !checkIfTupleType(receiver.type))
+ return elements.get(0).prepare(keyspace, receiver);
+
+ validateTupleAssignableTo(receiver, elements);
List<Term> values = new ArrayList<>(elements.size());
boolean allTerminal = true;
@@ -102,38 +110,14 @@
return allTerminal ? value.bind(QueryOptions.DEFAULT) : value;
}
- private void validateAssignableTo(String keyspace, ColumnSpecification receiver) throws InvalidRequestException
- {
- if (!checkIfTupleType(receiver.type))
- throw new InvalidRequestException(String.format("Invalid tuple type literal for %s of type %s", receiver.name, receiver.type.asCQL3Type()));
-
- TupleType tt = getTupleType(receiver.type);
- for (int i = 0; i < elements.size(); i++)
- {
- if (i >= tt.size())
- {
- throw new InvalidRequestException(String.format("Invalid tuple literal for %s: too many elements. Type %s expects %d but got %d",
- receiver.name, tt.asCQL3Type(), tt.size(), elements.size()));
- }
-
- Term.Raw value = elements.get(i);
- ColumnSpecification spec = componentSpecOf(receiver, i);
- if (!value.testAssignment(keyspace, spec).isAssignable())
- throw new InvalidRequestException(String.format("Invalid tuple literal for %s: component %d is not of type %s", receiver.name, i, spec.type.asCQL3Type()));
- }
- }
-
public AssignmentTestable.TestResult testAssignment(String keyspace, ColumnSpecification receiver)
{
- try
- {
- validateAssignableTo(keyspace, receiver);
- return AssignmentTestable.TestResult.WEAKLY_ASSIGNABLE;
- }
- catch (InvalidRequestException e)
- {
- return AssignmentTestable.TestResult.NOT_ASSIGNABLE;
- }
+ // The parser cannot differentiate between a tuple with one element and a term between parenthesis.
+ // By consequence, we need to wait until we know the target type to determine which one it is.
+ if (elements.size() == 1 && !checkIfTupleType(receiver.type))
+ return elements.get(0).testAssignment(keyspace, receiver);
+
+ return testTupleAssignment(receiver, elements);
}
@Override
@@ -152,7 +136,7 @@
public String getText()
{
- return elements.stream().map(Term.Raw::getText).collect(Collectors.joining(", ", "(", ")"));
+ return tupleToString(elements, Term.Raw::getText);
}
}
@@ -436,18 +420,101 @@
}
}
- public static String tupleToString(List<?> items)
+ /**
+ * Create a <code>String</code> representation of the tuple containing the specified elements.
+ *
+ * @param elements the tuple elements
+ * @return a <code>String</code> representation of the tuple
+ */
+ public static String tupleToString(List<?> elements)
{
+ return tupleToString(elements, Object::toString);
+ }
- StringBuilder sb = new StringBuilder("(");
- for (int i = 0; i < items.size(); i++)
+ /**
+ * Create a <code>String</code> representation of the tuple from the specified items associated to
+ * the tuples elements.
+ *
+ * @param items items associated to the tuple elements
+ * @param mapper the mapper used to map the items to the <code>String</code> representation of the tuple elements
+ * @return a <code>String</code> representation of the tuple
+ */
+ public static <T> String tupleToString(Iterable<T> items, java.util.function.Function<T, String> mapper)
+ {
+ return StreamSupport.stream(items.spliterator(), false)
+ .map(e -> mapper.apply(e))
+ .collect(Collectors.joining(", ", "(", ")"));
+ }
+
+ /**
+ * Returns the exact TupleType from the items if it can be known.
+ *
+ * @param items the items mapped to the tuple elements
+ * @param mapper the mapper used to retrieve the element types from the items
+ * @return the exact TupleType from the items if it can be known or <code>null</code>
+ */
+ public static <T> AbstractType<?> getExactTupleTypeIfKnown(List<T> items,
+ java.util.function.Function<T, AbstractType<?>> mapper)
+ {
+ List<AbstractType<?>> types = new ArrayList<>(items.size());
+ for (T item : items)
{
- sb.append(items.get(i));
- if (i < items.size() - 1)
- sb.append(", ");
+ AbstractType<?> type = mapper.apply(item);
+ if (type == null)
+ return null;
+ types.add(type);
}
- sb.append(')');
- return sb.toString();
+ return new TupleType(types);
+ }
+
+ /**
+ * Checks if the tuple with the specified elements can be assigned to the specified column.
+ *
+ * @param receiver the receiving column
+ * @param elements the tuple elements
+ * @throws InvalidRequestException if the tuple cannot be assigned to the specified column.
+ */
+ public static void validateTupleAssignableTo(ColumnSpecification receiver,
+ List<? extends AssignmentTestable> elements)
+ {
+ if (!checkIfTupleType(receiver.type))
+ throw invalidRequest("Invalid tuple type literal for %s of type %s", receiver.name, receiver.type.asCQL3Type());
+
+ TupleType tt = getTupleType(receiver.type);
+ for (int i = 0; i < elements.size(); i++)
+ {
+ if (i >= tt.size())
+ {
+ throw invalidRequest("Invalid tuple literal for %s: too many elements. Type %s expects %d but got %d",
+ receiver.name, tt.asCQL3Type(), tt.size(), elements.size());
+ }
+
+ AssignmentTestable value = elements.get(i);
+ ColumnSpecification spec = componentSpecOf(receiver, i);
+ if (!value.testAssignment(receiver.ksName, spec).isAssignable())
+ throw invalidRequest("Invalid tuple literal for %s: component %d is not of type %s",
+ receiver.name, i, spec.type.asCQL3Type());
+ }
+ }
+
+ /**
+ * Tests that the tuple with the specified elements can be assigned to the specified column.
+ *
+ * @param receiver the receiving column
+ * @param elements the tuple elements
+ */
+ public static AssignmentTestable.TestResult testTupleAssignment(ColumnSpecification receiver,
+ List<? extends AssignmentTestable> elements)
+ {
+ try
+ {
+ validateTupleAssignableTo(receiver, elements);
+ return AssignmentTestable.TestResult.WEAKLY_ASSIGNABLE;
+ }
+ catch (InvalidRequestException e)
+ {
+ return AssignmentTestable.TestResult.NOT_ASSIGNABLE;
+ }
}
public static boolean checkIfTupleType(AbstractType<?> tuple)
diff --git a/src/java/org/apache/cassandra/cql3/UntypedResultSet.java b/src/java/org/apache/cassandra/cql3/UntypedResultSet.java
index c551d42..5de4eae 100644
--- a/src/java/org/apache/cassandra/cql3/UntypedResultSet.java
+++ b/src/java/org/apache/cassandra/cql3/UntypedResultSet.java
@@ -24,18 +24,17 @@
import com.google.common.annotations.VisibleForTesting;
-import org.apache.cassandra.service.ClientState;
-import org.apache.cassandra.transport.ProtocolVersion;
-import org.apache.cassandra.utils.AbstractIterator;
-
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.cql3.statements.SelectStatement;
import org.apache.cassandra.db.*;
-import org.apache.cassandra.db.rows.*;
-import org.apache.cassandra.db.partitions.PartitionIterator;
import org.apache.cassandra.db.marshal.*;
+import org.apache.cassandra.db.partitions.PartitionIterator;
+import org.apache.cassandra.db.rows.*;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.service.ClientState;
import org.apache.cassandra.service.pager.QueryPager;
+import org.apache.cassandra.transport.ProtocolVersion;
+import org.apache.cassandra.utils.AbstractIterator;
import org.apache.cassandra.utils.FBUtilities;
/** a utility for doing internal cql-based queries */
@@ -123,71 +122,6 @@
}
}
- /**
- * Pager that calls `execute` rather than `executeInternal`
- */
- private static class FromDistributedPager extends UntypedResultSet
- {
- private final SelectStatement select;
- private final ConsistencyLevel cl;
- private final ClientState clientState;
- private final QueryPager pager;
- private final int pageSize;
- private final List<ColumnSpecification> metadata;
-
- private FromDistributedPager(SelectStatement select,
- ConsistencyLevel cl,
- ClientState clientState,
- QueryPager pager, int pageSize)
- {
- this.select = select;
- this.cl = cl;
- this.clientState = clientState;
- this.pager = pager;
- this.pageSize = pageSize;
- this.metadata = select.getResultMetadata().requestNames();
- }
-
- public int size()
- {
- throw new UnsupportedOperationException();
- }
-
- public Row one()
- {
- throw new UnsupportedOperationException();
- }
-
- public Iterator<Row> iterator()
- {
- return new AbstractIterator<Row>()
- {
- private Iterator<List<ByteBuffer>> currentPage;
-
- protected Row computeNext()
- {
- int nowInSec = FBUtilities.nowInSeconds();
- while (currentPage == null || !currentPage.hasNext())
- {
- if (pager.isExhausted())
- return endOfData();
-
- try (PartitionIterator iter = pager.fetchPage(pageSize, cl, clientState, System.nanoTime()))
- {
- currentPage = select.process(iter, nowInSec).rows.iterator();
- }
- }
- return new Row(metadata, currentPage.next());
- }
- };
- }
-
- public List<ColumnSpecification> metadata()
- {
- return metadata;
- }
- }
-
private static class FromResultList extends UntypedResultSet
{
private final List<Map<String, ByteBuffer>> cqlRows;
@@ -286,6 +220,71 @@
}
}
+ /**
+ * Pager that calls `execute` rather than `executeInternal`
+ */
+ private static class FromDistributedPager extends UntypedResultSet
+ {
+ private final SelectStatement select;
+ private final ConsistencyLevel cl;
+ private final ClientState clientState;
+ private final QueryPager pager;
+ private final int pageSize;
+ private final List<ColumnSpecification> metadata;
+
+ private FromDistributedPager(SelectStatement select,
+ ConsistencyLevel cl,
+ ClientState clientState,
+ QueryPager pager, int pageSize)
+ {
+ this.select = select;
+ this.cl = cl;
+ this.clientState = clientState;
+ this.pager = pager;
+ this.pageSize = pageSize;
+ this.metadata = select.getResultMetadata().requestNames();
+ }
+
+ public int size()
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ public Row one()
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ public Iterator<Row> iterator()
+ {
+ return new AbstractIterator<Row>()
+ {
+ private Iterator<List<ByteBuffer>> currentPage;
+
+ protected Row computeNext()
+ {
+ int nowInSec = FBUtilities.nowInSeconds();
+ while (currentPage == null || !currentPage.hasNext())
+ {
+ if (pager.isExhausted())
+ return endOfData();
+
+ try (PartitionIterator iter = pager.fetchPage(pageSize, cl, clientState, System.nanoTime()))
+ {
+ currentPage = select.process(iter, nowInSec).rows.iterator();
+ }
+ }
+ return new Row(metadata, currentPage.next());
+ }
+ };
+ }
+
+ public List<ColumnSpecification> metadata()
+ {
+ return metadata;
+ }
+ }
+
public static class Row
{
private final Map<String, ByteBuffer> data = new HashMap<>();
@@ -303,19 +302,19 @@
data.put(names.get(i).name.toString(), columns.get(i));
}
- public static Row fromInternalRow(CFMetaData metadata, DecoratedKey key, org.apache.cassandra.db.rows.Row row)
+ public static Row fromInternalRow(TableMetadata metadata, DecoratedKey key, org.apache.cassandra.db.rows.Row row)
{
Map<String, ByteBuffer> data = new HashMap<>();
ByteBuffer[] keyComponents = SelectStatement.getComponents(metadata, key);
- for (ColumnDefinition def : metadata.partitionKeyColumns())
+ for (ColumnMetadata def : metadata.partitionKeyColumns())
data.put(def.name.toString(), keyComponents[def.position()]);
Clustering clustering = row.clustering();
- for (ColumnDefinition def : metadata.clusteringColumns())
+ for (ColumnMetadata def : metadata.clusteringColumns())
data.put(def.name.toString(), clustering.get(def.position()));
- for (ColumnDefinition def : metadata.partitionColumns())
+ for (ColumnMetadata def : metadata.regularAndStaticColumns())
{
if (def.isSimple())
{
diff --git a/src/java/org/apache/cassandra/cql3/UpdateParameters.java b/src/java/org/apache/cassandra/cql3/UpdateParameters.java
index ab61a0d..740cd91 100644
--- a/src/java/org/apache/cassandra/cql3/UpdateParameters.java
+++ b/src/java/org/apache/cassandra/cql3/UpdateParameters.java
@@ -20,23 +20,22 @@
import java.nio.ByteBuffer;
import java.util.Map;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.context.CounterContext;
import org.apache.cassandra.db.filter.ColumnFilter;
import org.apache.cassandra.db.partitions.Partition;
import org.apache.cassandra.db.rows.*;
import org.apache.cassandra.exceptions.InvalidRequestException;
-import org.apache.cassandra.utils.FBUtilities;
/**
* Groups the parameters of an update query, and make building updates easier.
*/
public class UpdateParameters
{
- public final CFMetaData metadata;
- public final PartitionColumns updatedColumns;
+ public final TableMetadata metadata;
+ public final RegularAndStaticColumns updatedColumns;
public final QueryOptions options;
private final int nowInSec;
@@ -54,10 +53,11 @@
// The builder currently in use. Will alias either staticBuilder or regularBuilder, which are themselves built lazily.
private Row.Builder builder;
- public UpdateParameters(CFMetaData metadata,
- PartitionColumns updatedColumns,
+ public UpdateParameters(TableMetadata metadata,
+ RegularAndStaticColumns updatedColumns,
QueryOptions options,
long timestamp,
+ int nowInSec,
int ttl,
Map<DecoratedKey, Partition> prefetchedRows)
throws InvalidRequestException
@@ -66,7 +66,7 @@
this.updatedColumns = updatedColumns;
this.options = options;
- this.nowInSec = FBUtilities.nowInSeconds();
+ this.nowInSec = nowInSec;
this.timestamp = timestamp;
this.ttl = ttl;
@@ -84,8 +84,7 @@
{
if (metadata.isDense() && !metadata.isCompound())
{
- // If it's a COMPACT STORAGE table with a single clustering column, the clustering value is
- // translated in Thrift to the full Thrift column name, and for backward compatibility we
+ // If it's a COMPACT STORAGE table with a single clustering column and for backward compatibility we
// don't want to allow that to be empty (even though this would be fine for the storage engine).
assert clustering.size() == 1;
ByteBuffer value = clustering.get(0);
@@ -96,13 +95,13 @@
if (clustering == Clustering.STATIC_CLUSTERING)
{
if (staticBuilder == null)
- staticBuilder = BTreeRow.unsortedBuilder(nowInSec);
+ staticBuilder = BTreeRow.unsortedBuilder();
builder = staticBuilder;
}
else
{
if (regularBuilder == null)
- regularBuilder = BTreeRow.unsortedBuilder(nowInSec);
+ regularBuilder = BTreeRow.unsortedBuilder();
builder = regularBuilder;
}
@@ -122,31 +121,30 @@
public void addRowDeletion()
{
// For compact tables, at the exclusion of the static row (of static compact tables), each row ever has a single column,
- // the "compact" one. As such, deleting the row or deleting that single cell is equivalent. We favor the later however
- // because that makes it easier when translating back to the old format layout (for thrift and pre-3.0 backward
- // compatibility) as we don't have to special case for the row deletion. This is also in line with what we used to do pre-3.0.
- if (metadata.isCompactTable() && builder.clustering() != Clustering.STATIC_CLUSTERING && !metadata.isSuper())
- addTombstone(metadata.compactValueColumn());
+ // the "compact" one. As such, deleting the row or deleting that single cell is equivalent. We favor the later
+ // for backward compatibility (thought it doesn't truly matter anymore).
+ if (metadata.isCompactTable() && builder.clustering() != Clustering.STATIC_CLUSTERING)
+ addTombstone(metadata.compactValueColumn);
else
builder.addRowDeletion(Row.Deletion.regular(deletionTime));
}
- public void addTombstone(ColumnDefinition column) throws InvalidRequestException
+ public void addTombstone(ColumnMetadata column) throws InvalidRequestException
{
addTombstone(column, null);
}
- public void addTombstone(ColumnDefinition column, CellPath path) throws InvalidRequestException
+ public void addTombstone(ColumnMetadata column, CellPath path) throws InvalidRequestException
{
builder.addCell(BufferCell.tombstone(column, timestamp, nowInSec, path));
}
- public void addCell(ColumnDefinition column, ByteBuffer value) throws InvalidRequestException
+ public void addCell(ColumnMetadata column, ByteBuffer value) throws InvalidRequestException
{
addCell(column, null, value);
}
- public void addCell(ColumnDefinition column, CellPath path, ByteBuffer value) throws InvalidRequestException
+ public void addCell(ColumnMetadata column, CellPath path, ByteBuffer value) throws InvalidRequestException
{
Cell cell = ttl == LivenessInfo.NO_TTL
? BufferCell.live(column, timestamp, value, path)
@@ -154,12 +152,7 @@
builder.addCell(cell);
}
- public void addCounter(ColumnDefinition column, long increment) throws InvalidRequestException
- {
- addCounter(column, increment, null);
- }
-
- public void addCounter(ColumnDefinition column, long increment, CellPath path) throws InvalidRequestException
+ public void addCounter(ColumnMetadata column, long increment) throws InvalidRequestException
{
assert ttl == LivenessInfo.NO_TTL;
@@ -175,15 +168,15 @@
//
// We set counterid to a special value to differentiate between regular pre-2.0 local shards from pre-2.1 era
// and "counter update" temporary state cells. Please see CounterContext.createUpdate() for further details.
- builder.addCell(BufferCell.live(column, timestamp, CounterContext.instance().createUpdate(increment), path));
+ builder.addCell(BufferCell.live(column, timestamp, CounterContext.instance().createUpdate(increment)));
}
- public void setComplexDeletionTime(ColumnDefinition column)
+ public void setComplexDeletionTime(ColumnMetadata column)
{
builder.addComplexDeletion(column, deletionTime);
}
- public void setComplexDeletionTimeForOverwrite(ColumnDefinition column)
+ public void setComplexDeletionTimeForOverwrite(ColumnMetadata column)
{
builder.addComplexDeletion(column, new DeletionTime(deletionTime.markedForDeleteAt() - 1, deletionTime.localDeletionTime()));
}
@@ -237,7 +230,7 @@
if (prefetchedRow == null)
return pendingMutations;
- return Rows.merge(prefetchedRow, pendingMutations, nowInSec)
+ return Rows.merge(prefetchedRow, pendingMutations)
.purge(DeletionPurger.PURGE_ALL, nowInSec, metadata.enforceStrictLiveness());
}
}
diff --git a/src/java/org/apache/cassandra/cql3/UserTypes.java b/src/java/org/apache/cassandra/cql3/UserTypes.java
index 4edd27f..b023a8a 100644
--- a/src/java/org/apache/cassandra/cql3/UserTypes.java
+++ b/src/java/org/apache/cassandra/cql3/UserTypes.java
@@ -19,8 +19,9 @@
import java.nio.ByteBuffer;
import java.util.*;
+import java.util.stream.Collectors;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.cql3.functions.Function;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.marshal.*;
@@ -47,6 +48,77 @@
ut.fieldType(field));
}
+ public static <T extends AssignmentTestable> void validateUserTypeAssignableTo(ColumnSpecification receiver,
+ Map<FieldIdentifier, T> entries)
+ {
+ if (!receiver.type.isUDT())
+ throw new InvalidRequestException(String.format("Invalid user type literal for %s of type %s", receiver, receiver.type.asCQL3Type()));
+
+ UserType ut = (UserType) receiver.type;
+ for (int i = 0; i < ut.size(); i++)
+ {
+ FieldIdentifier field = ut.fieldName(i);
+ T value = entries.get(field);
+ if (value == null)
+ continue;
+
+ ColumnSpecification fieldSpec = fieldSpecOf(receiver, i);
+ if (!value.testAssignment(receiver.ksName, fieldSpec).isAssignable())
+ {
+ throw new InvalidRequestException(String.format("Invalid user type literal for %s: field %s is not of type %s",
+ receiver, field, fieldSpec.type.asCQL3Type()));
+ }
+ }
+ }
+
+ /**
+ * Tests that the map with the specified entries can be assigned to the specified column.
+ *
+ * @param receiver the receiving column
+ * @param entries the map entries
+ */
+ public static <T extends AssignmentTestable> AssignmentTestable.TestResult testUserTypeAssignment(ColumnSpecification receiver,
+ Map<FieldIdentifier, T> entries)
+ {
+ try
+ {
+ validateUserTypeAssignableTo(receiver, entries);
+ return AssignmentTestable.TestResult.WEAKLY_ASSIGNABLE;
+ }
+ catch (InvalidRequestException e)
+ {
+ return AssignmentTestable.TestResult.NOT_ASSIGNABLE;
+ }
+ }
+
+ /**
+ * Create a {@code String} representation of the user type from the specified items associated to
+ * the user type entries.
+ *
+ * @param items items associated to the user type entries
+ * @return a {@code String} representation of the user type
+ */
+ public static <T> String userTypeToString(Map<FieldIdentifier, T> items)
+ {
+ return userTypeToString(items, Object::toString);
+ }
+
+ /**
+ * Create a {@code String} representation of the user type from the specified items associated to
+ * the user type entries.
+ *
+ * @param items items associated to the user type entries
+ * @return a {@code String} representation of the user type
+ */
+ public static <T> String userTypeToString(Map<FieldIdentifier, T> items,
+ java.util.function.Function<T, String> mapper)
+ {
+ return items.entrySet()
+ .stream()
+ .map(p -> String.format("%s: %s", p.getKey(), mapper.apply(p.getValue())))
+ .collect(Collectors.joining(", ", "{", "}"));
+ }
+
public static class Literal extends Term.Raw
{
public final Map<FieldIdentifier, Term.Raw> entries;
@@ -117,15 +189,7 @@
public AssignmentTestable.TestResult testAssignment(String keyspace, ColumnSpecification receiver)
{
- try
- {
- validateAssignableTo(keyspace, receiver);
- return AssignmentTestable.TestResult.WEAKLY_ASSIGNABLE;
- }
- catch (InvalidRequestException e)
- {
- return AssignmentTestable.TestResult.NOT_ASSIGNABLE;
- }
+ return testUserTypeAssignment(receiver, entries);
}
public AbstractType<?> getExactTypeIfKnown(String keyspace)
@@ -135,18 +199,7 @@
public String getText()
{
- StringBuilder sb = new StringBuilder();
- sb.append("{");
- Iterator<Map.Entry<FieldIdentifier, Term.Raw>> iter = entries.entrySet().iterator();
- while (iter.hasNext())
- {
- Map.Entry<FieldIdentifier, Term.Raw> entry = iter.next();
- sb.append(entry.getKey()).append(": ").append(entry.getValue().getText());
- if (iter.hasNext())
- sb.append(", ");
- }
- sb.append("}");
- return sb.toString();
+ return userTypeToString(entries, Term.Raw::getText);
}
}
@@ -273,7 +326,7 @@
public static class Setter extends Operation
{
- public Setter(ColumnDefinition column, Term t)
+ public Setter(ColumnMetadata column, Term t)
{
super(column, t);
}
@@ -319,7 +372,7 @@
{
private final FieldIdentifier field;
- public SetterByField(ColumnDefinition column, FieldIdentifier field, Term t)
+ public SetterByField(ColumnMetadata column, FieldIdentifier field, Term t)
{
super(column, t);
this.field = field;
@@ -346,7 +399,7 @@
{
private final FieldIdentifier field;
- public DeleterByField(ColumnDefinition column, FieldIdentifier field)
+ public DeleterByField(ColumnMetadata column, FieldIdentifier field)
{
super(column, null);
this.field = field;
diff --git a/src/java/org/apache/cassandra/cql3/Validation.java b/src/java/org/apache/cassandra/cql3/Validation.java
new file mode 100644
index 0000000..34a4027
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/Validation.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3;
+
+import java.nio.ByteBuffer;
+
+import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.serializers.MarshalException;
+import org.apache.cassandra.utils.FBUtilities;
+
+/**
+ * A collection of static validation functions reused across statements.
+ *
+ * Note: this hosts functions that were historically in ThriftValidation, but
+ * it's not necessary clear that this is the best place to have this (this is
+ * certainly not horrible either though).
+ */
+public abstract class Validation
+{
+
+ /**
+ * Validates a (full serialized) partition key.
+ *
+ * @param metadata the metadata for the table of which to check the key.
+ * @param key the serialized partition key to check.
+ *
+ * @throws InvalidRequestException if the provided {@code key} is invalid.
+ */
+ public static void validateKey(TableMetadata metadata, ByteBuffer key)
+ {
+ if (key == null || key.remaining() == 0)
+ throw new InvalidRequestException("Key may not be empty");
+
+ // check that key can be handled by FBUtilities.writeShortByteArray
+ if (key.remaining() > FBUtilities.MAX_UNSIGNED_SHORT)
+ {
+ throw new InvalidRequestException("Key length of " + key.remaining() +
+ " is longer than maximum of " +
+ FBUtilities.MAX_UNSIGNED_SHORT);
+ }
+
+ try
+ {
+ metadata.partitionKeyType.validate(key);
+ }
+ catch (MarshalException e)
+ {
+ throw new InvalidRequestException(e.getMessage());
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/cql3/VariableSpecifications.java b/src/java/org/apache/cassandra/cql3/VariableSpecifications.java
index 24f71e4..e58290e 100644
--- a/src/java/org/apache/cassandra/cql3/VariableSpecifications.java
+++ b/src/java/org/apache/cassandra/cql3/VariableSpecifications.java
@@ -17,24 +17,24 @@
*/
package org.apache.cassandra.cql3;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
-
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableMetadata;
+
public class VariableSpecifications
{
private final List<ColumnIdentifier> variableNames;
- private final ColumnSpecification[] specs;
- private final ColumnDefinition[] targetColumns;
+ private final List<ColumnSpecification> specs;
+ private final ColumnMetadata[] targetColumns;
public VariableSpecifications(List<ColumnIdentifier> variableNames)
{
this.variableNames = variableNames;
- this.specs = new ColumnSpecification[variableNames.size()];
- this.targetColumns = new ColumnDefinition[variableNames.size()];
+ this.specs = Arrays.asList(new ColumnSpecification[variableNames.size()]);
+ this.targetColumns = new ColumnMetadata[variableNames.size()];
}
/**
@@ -43,36 +43,36 @@
*/
public static VariableSpecifications empty()
{
- return new VariableSpecifications(Collections.<ColumnIdentifier> emptyList());
+ return new VariableSpecifications(Collections.emptyList());
}
- public int size()
+ public boolean isEmpty()
{
- return variableNames.size();
+ return variableNames.isEmpty();
}
- public List<ColumnSpecification> getSpecifications()
+ public List<ColumnSpecification> getBindVariables()
{
- return Arrays.asList(specs);
+ return specs;
}
/**
* Returns an array with the same length as the number of partition key columns for the table corresponding
- * to cfm. Each short in the array represents the bind index of the marker that holds the value for that
+ * to table. Each short in the array represents the bind index of the marker that holds the value for that
* partition key column. If there are no bind markers for any of the partition key columns, null is returned.
*
* Callers of this method should ensure that all statements operate on the same table.
*/
- public short[] getPartitionKeyBindIndexes(CFMetaData cfm)
+ public short[] getPartitionKeyBindVariableIndexes(TableMetadata metadata)
{
- short[] partitionKeyPositions = new short[cfm.partitionKeyColumns().size()];
+ short[] partitionKeyPositions = new short[metadata.partitionKeyColumns().size()];
boolean[] set = new boolean[partitionKeyPositions.length];
for (int i = 0; i < targetColumns.length; i++)
{
- ColumnDefinition targetColumn = targetColumns[i];
+ ColumnMetadata targetColumn = targetColumns[i];
if (targetColumn != null && targetColumn.isPartitionKey())
{
- assert targetColumn.ksName.equals(cfm.ksName) && targetColumn.cfName.equals(cfm.cfName);
+ assert targetColumn.ksName.equals(metadata.keyspace) && targetColumn.cfName.equals(metadata.name);
partitionKeyPositions[targetColumn.position()] = (short) i;
set[targetColumn.position()] = true;
}
@@ -87,19 +87,19 @@
public void add(int bindIndex, ColumnSpecification spec)
{
- if (spec instanceof ColumnDefinition)
- targetColumns[bindIndex] = (ColumnDefinition) spec;
+ if (spec instanceof ColumnMetadata)
+ targetColumns[bindIndex] = (ColumnMetadata) spec;
ColumnIdentifier bindMarkerName = variableNames.get(bindIndex);
// Use the user name, if there is one
if (bindMarkerName != null)
spec = new ColumnSpecification(spec.ksName, spec.cfName, bindMarkerName, spec.type);
- specs[bindIndex] = spec;
+ specs.set(bindIndex, spec);
}
@Override
public String toString()
{
- return Arrays.toString(specs);
+ return specs.toString();
}
}
diff --git a/src/java/org/apache/cassandra/cql3/WhereClause.java b/src/java/org/apache/cassandra/cql3/WhereClause.java
index c56c8e0..87041f9 100644
--- a/src/java/org/apache/cassandra/cql3/WhereClause.java
+++ b/src/java/org/apache/cassandra/cql3/WhereClause.java
@@ -15,18 +15,24 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.cassandra.cql3;
import java.util.List;
+import java.util.Objects;
import com.google.common.collect.ImmutableList;
+import org.antlr.runtime.RecognitionException;
+import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.cql3.restrictions.CustomIndexExpression;
+import static java.lang.String.join;
+
+import static com.google.common.collect.Iterables.concat;
+import static com.google.common.collect.Iterables.transform;
+
public final class WhereClause
{
-
private static final WhereClause EMPTY = new WhereClause(new Builder());
public final List<Relation> relations;
@@ -34,13 +40,8 @@
private WhereClause(Builder builder)
{
- this(builder.relations.build(), builder.expressions.build());
- }
-
- private WhereClause(List<Relation> relations, List<CustomIndexExpression> expressions)
- {
- this.relations = relations;
- this.expressions = expressions;
+ relations = builder.relations.build();
+ expressions = builder.expressions.build();
}
public static WhereClause empty()
@@ -48,16 +49,62 @@
return EMPTY;
}
- public WhereClause copy(List<Relation> newRelations)
- {
- return new WhereClause(newRelations, expressions);
- }
-
public boolean containsCustomExpressions()
{
return !expressions.isEmpty();
}
+ /**
+ * Renames identifiers in all relations
+ * @param from the old identifier
+ * @param to the new identifier
+ * @return a new WhereClause with with "from" replaced by "to" in all relations
+ */
+ public WhereClause renameIdentifier(ColumnMetadata.Raw from, ColumnMetadata.Raw to)
+ {
+ WhereClause.Builder builder = new WhereClause.Builder();
+
+ relations.stream()
+ .map(r -> r.renameIdentifier(from, to))
+ .forEach(builder::add);
+
+ expressions.forEach(builder::add);
+
+ return builder.build();
+ }
+
+ public static WhereClause parse(String cql) throws RecognitionException
+ {
+ return CQLFragmentParser.parseAnyUnhandled(CqlParser::whereClause, cql).build();
+ }
+
+ @Override
+ public String toString()
+ {
+ return join(" AND ",
+ concat(transform(relations, Relation::toString),
+ transform(expressions, CustomIndexExpression::toString)));
+ }
+
+ @Override
+ public boolean equals(Object o)
+ {
+ if (this == o)
+ return true;
+
+ if (!(o instanceof WhereClause))
+ return false;
+
+ WhereClause wc = (WhereClause) o;
+ return relations.equals(wc.relations) && expressions.equals(wc.expressions);
+ }
+
+ @Override
+ public int hashCode()
+ {
+ return Objects.hash(relations, expressions);
+ }
+
public static final class Builder
{
ImmutableList.Builder<Relation> relations = new ImmutableList.Builder<>();
diff --git a/src/java/org/apache/cassandra/cql3/conditions/AbstractConditions.java b/src/java/org/apache/cassandra/cql3/conditions/AbstractConditions.java
new file mode 100644
index 0000000..0e2646e
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/conditions/AbstractConditions.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.conditions;
+
+import java.util.List;
+
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.cql3.functions.Function;
+
+/**
+ * Base class for <code>Conditions</code> classes.
+ *
+ */
+abstract class AbstractConditions implements Conditions
+{
+ public void addFunctionsTo(List<Function> functions)
+ {
+ }
+
+ public Iterable<ColumnMetadata> getColumns()
+ {
+ return null;
+ }
+
+ public boolean isEmpty()
+ {
+ return false;
+ }
+
+ public boolean appliesToStaticColumns()
+ {
+ return false;
+ }
+
+ public boolean appliesToRegularColumns()
+ {
+ return false;
+ }
+
+ public boolean isIfExists()
+ {
+ return false;
+ }
+
+ public boolean isIfNotExists()
+ {
+ return false;
+ }
+}
diff --git a/src/java/org/apache/cassandra/cql3/conditions/ColumnCondition.java b/src/java/org/apache/cassandra/cql3/conditions/ColumnCondition.java
new file mode 100644
index 0000000..aa5c10d
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/conditions/ColumnCondition.java
@@ -0,0 +1,864 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.conditions;
+
+import java.nio.ByteBuffer;
+import java.util.*;
+
+import com.google.common.collect.Iterators;
+
+import org.apache.cassandra.cql3.*;
+import org.apache.cassandra.cql3.Term.Terminal;
+import org.apache.cassandra.cql3.functions.Function;
+import org.apache.cassandra.db.rows.*;
+import org.apache.cassandra.db.marshal.*;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.transport.ProtocolVersion;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.commons.lang3.builder.ToStringBuilder;
+import org.apache.commons.lang3.builder.ToStringStyle;
+
+import static org.apache.cassandra.cql3.statements.RequestValidations.*;
+
+/**
+ * A CQL3 condition on the value of a column or collection element. For example, "UPDATE .. IF a = 0".
+ */
+public abstract class ColumnCondition
+{
+ public final ColumnMetadata column;
+ public final Operator operator;
+ private final Terms terms;
+
+ private ColumnCondition(ColumnMetadata column, Operator op, Terms terms)
+ {
+ this.column = column;
+ this.operator = op;
+ this.terms = terms;
+ }
+
+ /**
+ * Adds functions for the bind variables of this operation.
+ *
+ * @param functions the list of functions to get add
+ */
+ public void addFunctionsTo(List<Function> functions)
+ {
+ terms.addFunctionsTo(functions);
+ }
+
+ /**
+ * Collects the column specification for the bind variables of this operation.
+ *
+ * @param boundNames the list of column specification where to collect the
+ * bind variables of this term in.
+ */
+ public void collectMarkerSpecification(VariableSpecifications boundNames)
+ {
+ terms.collectMarkerSpecification(boundNames);
+ }
+
+ public abstract ColumnCondition.Bound bind(QueryOptions options);
+
+ protected final List<ByteBuffer> bindAndGetTerms(QueryOptions options)
+ {
+ return filterUnsetValuesIfNeeded(checkValues(terms.bindAndGet(options)));
+ }
+
+ protected final List<Terminal> bindTerms(QueryOptions options)
+ {
+ return filterUnsetValuesIfNeeded(checkValues(terms.bind(options)));
+ }
+
+ /**
+ * Checks that the output of a bind operations on {@code Terms} is a valid one.
+ * @param values the list to check
+ * @return the input list
+ */
+ private <T> List<T> checkValues(List<T> values)
+ {
+ checkFalse(values == null && operator.isIN(), "Invalid null list in IN condition");
+ checkFalse(values == Terms.UNSET_LIST, "Invalid 'unset' value in condition");
+ return values;
+ }
+
+ private <T> List<T> filterUnsetValuesIfNeeded(List<T> values)
+ {
+ if (!operator.isIN())
+ return values;
+
+ List<T> filtered = new ArrayList<>(values.size());
+ for (int i = 0, m = values.size(); i < m; i++)
+ {
+ T value = values.get(i);
+ // The value can be ByteBuffer or Constants.Value so we need to check the 2 type of UNSET
+ if (value != ByteBufferUtil.UNSET_BYTE_BUFFER && value != Constants.UNSET_VALUE)
+ filtered.add(value);
+ }
+ return filtered;
+ }
+
+ /**
+ * Simple condition (e.g. <pre>IF v = 1</pre>).
+ */
+ private static final class SimpleColumnCondition extends ColumnCondition
+ {
+ public SimpleColumnCondition(ColumnMetadata column, Operator op, Terms values)
+ {
+ super(column, op, values);
+ }
+
+ public Bound bind(QueryOptions options)
+ {
+ if (column.type.isCollection() && column.type.isMultiCell())
+ return new MultiCellCollectionBound(column, operator, bindTerms(options));
+
+ if (column.type.isUDT() && column.type.isMultiCell())
+ return new MultiCellUdtBound(column, operator, bindAndGetTerms(options), options.getProtocolVersion());
+
+ return new SimpleBound(column, operator, bindAndGetTerms(options));
+ }
+ }
+
+ /**
+ * A condition on a collection element (e.g. <pre>IF l[1] = 1</pre>).
+ */
+ private static class CollectionElementCondition extends ColumnCondition
+ {
+ private final Term collectionElement;
+
+ public CollectionElementCondition(ColumnMetadata column, Term collectionElement, Operator op, Terms values)
+ {
+ super(column, op, values);
+ this.collectionElement = collectionElement;
+ }
+
+ public void addFunctionsTo(List<Function> functions)
+ {
+ collectionElement.addFunctionsTo(functions);
+ super.addFunctionsTo(functions);
+ }
+
+ public void collectMarkerSpecification(VariableSpecifications boundNames)
+ {
+ collectionElement.collectMarkerSpecification(boundNames);
+ super.collectMarkerSpecification(boundNames);
+ }
+
+ public Bound bind(QueryOptions options)
+ {
+ return new ElementAccessBound(column, collectionElement.bindAndGet(options), operator, bindAndGetTerms(options));
+ }
+ }
+
+ /**
+ * A condition on a UDT field (e.g. <pre>IF v.a = 1</pre>).
+ */
+ private final static class UDTFieldCondition extends ColumnCondition
+ {
+ private final FieldIdentifier udtField;
+
+ public UDTFieldCondition(ColumnMetadata column, FieldIdentifier udtField, Operator op, Terms values)
+ {
+ super(column, op, values);
+ assert udtField != null;
+ this.udtField = udtField;
+ }
+
+ public Bound bind(QueryOptions options)
+ {
+ return new UDTFieldAccessBound(column, udtField, operator, bindAndGetTerms(options));
+ }
+ }
+
+ /**
+ * A regular column, simple condition.
+ */
+ public static ColumnCondition condition(ColumnMetadata column, Operator op, Terms terms)
+ {
+ return new SimpleColumnCondition(column, op, terms);
+ }
+
+ /**
+ * A collection column, simple condition.
+ */
+ public static ColumnCondition condition(ColumnMetadata column, Term collectionElement, Operator op, Terms terms)
+ {
+ return new CollectionElementCondition(column, collectionElement, op, terms);
+ }
+
+ /**
+ * A UDT column, simple condition.
+ */
+ public static ColumnCondition condition(ColumnMetadata column, FieldIdentifier udtField, Operator op, Terms terms)
+ {
+ return new UDTFieldCondition(column, udtField, op, terms);
+ }
+
+ public static abstract class Bound
+ {
+ public final ColumnMetadata column;
+ public final Operator comparisonOperator;
+
+ protected Bound(ColumnMetadata column, Operator operator)
+ {
+ this.column = column;
+ // If the operator is an IN we want to compare the value using an EQ.
+ this.comparisonOperator = operator.isIN() ? Operator.EQ : operator;
+ }
+
+ /**
+ * Validates whether this condition applies to {@code current}.
+ */
+ public abstract boolean appliesTo(Row row);
+
+ public ByteBuffer getCollectionElementValue()
+ {
+ return null;
+ }
+
+ /** Returns true if the operator is satisfied (i.e. "otherValue operator value == true"), false otherwise. */
+ protected static boolean compareWithOperator(Operator operator, AbstractType<?> type, ByteBuffer value, ByteBuffer otherValue)
+ {
+ if (value == ByteBufferUtil.UNSET_BYTE_BUFFER)
+ throw invalidRequest("Invalid 'unset' value in condition");
+
+ if (value == null)
+ {
+ switch (operator)
+ {
+ case EQ:
+ return otherValue == null;
+ case NEQ:
+ return otherValue != null;
+ default:
+ throw invalidRequest("Invalid comparison with null for operator \"%s\"", operator);
+ }
+ }
+ else if (otherValue == null)
+ {
+ // the condition value is not null, so only NEQ can return true
+ return operator == Operator.NEQ;
+ }
+ return operator.isSatisfiedBy(type, otherValue, value);
+ }
+ }
+
+ protected static final Cell getCell(Row row, ColumnMetadata column)
+ {
+ // If we're asking for a given cell, and we didn't got any row from our read, it's
+ // the same as not having said cell.
+ return row == null ? null : row.getCell(column);
+ }
+
+ protected static final Cell getCell(Row row, ColumnMetadata column, CellPath path)
+ {
+ // If we're asking for a given cell, and we didn't got any row from our read, it's
+ // the same as not having said cell.
+ return row == null ? null : row.getCell(column, path);
+ }
+
+ protected static final Iterator<Cell> getCells(Row row, ColumnMetadata column)
+ {
+ // If we're asking for a complex cells, and we didn't got any row from our read, it's
+ // the same as not having any cells for that column.
+ if (row == null)
+ return Collections.<Cell>emptyIterator();
+
+ ComplexColumnData complexData = row.getComplexColumnData(column);
+ return complexData == null ? Collections.<Cell>emptyIterator() : complexData.iterator();
+ }
+
+ protected static final boolean evaluateComparisonWithOperator(int comparison, Operator operator)
+ {
+ // called when comparison != 0
+ switch (operator)
+ {
+ case EQ:
+ return false;
+ case LT:
+ case LTE:
+ return comparison < 0;
+ case GT:
+ case GTE:
+ return comparison > 0;
+ case NEQ:
+ return true;
+ default:
+ throw new AssertionError();
+ }
+ }
+
+ /**
+ * A condition on a single non-collection column.
+ */
+ private static final class SimpleBound extends Bound
+ {
+ /**
+ * The condition values
+ */
+ private final List<ByteBuffer> values;
+
+ private SimpleBound(ColumnMetadata column, Operator operator, List<ByteBuffer> values)
+ {
+ super(column, operator);
+ this.values = values;
+ }
+
+ @Override
+ public boolean appliesTo(Row row)
+ {
+ return isSatisfiedBy(rowValue(row));
+ }
+
+ private ByteBuffer rowValue(Row row)
+ {
+ Cell c = getCell(row, column);
+ return c == null ? null : c.value();
+ }
+
+ private boolean isSatisfiedBy(ByteBuffer rowValue)
+ {
+ for (ByteBuffer value : values)
+ {
+ if (compareWithOperator(comparisonOperator, column.type, value, rowValue))
+ return true;
+ }
+ return false;
+ }
+ }
+
+ /**
+ * A condition on an element of a collection column.
+ */
+ private static final class ElementAccessBound extends Bound
+ {
+ /**
+ * The collection element
+ */
+ private final ByteBuffer collectionElement;
+
+ /**
+ * The conditions values.
+ */
+ private final List<ByteBuffer> values;
+
+ private ElementAccessBound(ColumnMetadata column,
+ ByteBuffer collectionElement,
+ Operator operator,
+ List<ByteBuffer> values)
+ {
+ super(column, operator);
+
+ this.collectionElement = collectionElement;
+ this.values = values;
+ }
+
+ @Override
+ public boolean appliesTo(Row row)
+ {
+ boolean isMap = column.type instanceof MapType;
+
+ if (collectionElement == null)
+ throw invalidRequest("Invalid null value for %s element access", isMap ? "map" : "list");
+
+ if (isMap)
+ {
+ MapType<?, ?> mapType = (MapType<?, ?>) column.type;
+ ByteBuffer rowValue = rowMapValue(mapType, row);
+ return isSatisfiedBy(mapType.getKeysType(), rowValue);
+ }
+
+ ListType<?> listType = (ListType<?>) column.type;
+ ByteBuffer rowValue = rowListValue(listType, row);
+ return isSatisfiedBy(listType.getElementsType(), rowValue);
+ }
+
+ private ByteBuffer rowMapValue(MapType<?, ?> type, Row row)
+ {
+ if (column.type.isMultiCell())
+ {
+ Cell cell = getCell(row, column, CellPath.create(collectionElement));
+ return cell == null ? null : cell.value();
+ }
+
+ Cell cell = getCell(row, column);
+ return cell == null
+ ? null
+ : type.getSerializer().getSerializedValue(cell.value(), collectionElement, type.getKeysType());
+ }
+
+ private ByteBuffer rowListValue(ListType<?> type, Row row)
+ {
+ if (column.type.isMultiCell())
+ return cellValueAtIndex(getCells(row, column), getListIndex(collectionElement));
+
+ Cell cell = getCell(row, column);
+ return cell == null
+ ? null
+ : type.getSerializer().getElement(cell.value(), getListIndex(collectionElement));
+ }
+
+ private static ByteBuffer cellValueAtIndex(Iterator<Cell> iter, int index)
+ {
+ int adv = Iterators.advance(iter, index);
+ if (adv == index && iter.hasNext())
+ return iter.next().value();
+
+ return null;
+ }
+
+ private boolean isSatisfiedBy(AbstractType<?> valueType, ByteBuffer rowValue)
+ {
+ for (ByteBuffer value : values)
+ {
+ if (compareWithOperator(comparisonOperator, valueType, value, rowValue))
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ public ByteBuffer getCollectionElementValue()
+ {
+ return collectionElement;
+ }
+
+ private static int getListIndex(ByteBuffer collectionElement)
+ {
+ int idx = ByteBufferUtil.toInt(collectionElement);
+ checkFalse(idx < 0, "Invalid negative list index %d", idx);
+ return idx;
+ }
+ }
+
+ /**
+ * A condition on an entire collection column.
+ */
+ private static final class MultiCellCollectionBound extends Bound
+ {
+ private final List<Term.Terminal> values;
+
+ public MultiCellCollectionBound(ColumnMetadata column, Operator operator, List<Term.Terminal> values)
+ {
+ super(column, operator);
+ assert column.type.isMultiCell();
+ this.values = values;
+ }
+
+ public boolean appliesTo(Row row)
+ {
+ CollectionType<?> type = (CollectionType<?>)column.type;
+
+ // copy iterator contents so that we can properly reuse them for each comparison with an IN value
+ for (Term.Terminal value : values)
+ {
+ Iterator<Cell> iter = getCells(row, column);
+ if (value == null)
+ {
+ if (comparisonOperator == Operator.EQ)
+ {
+ if (!iter.hasNext())
+ return true;
+ continue;
+ }
+
+ if (comparisonOperator == Operator.NEQ)
+ return iter.hasNext();
+
+ throw invalidRequest("Invalid comparison with null for operator \"%s\"", comparisonOperator);
+ }
+
+ if (valueAppliesTo(type, iter, value, comparisonOperator))
+ return true;
+ }
+ return false;
+ }
+
+ private static boolean valueAppliesTo(CollectionType<?> type, Iterator<Cell> iter, Term.Terminal value, Operator operator)
+ {
+ if (value == null)
+ return !iter.hasNext();
+
+ switch (type.kind)
+ {
+ case LIST:
+ List<ByteBuffer> valueList = ((Lists.Value) value).elements;
+ return listAppliesTo((ListType<?>)type, iter, valueList, operator);
+ case SET:
+ Set<ByteBuffer> valueSet = ((Sets.Value) value).elements;
+ return setAppliesTo((SetType<?>)type, iter, valueSet, operator);
+ case MAP:
+ Map<ByteBuffer, ByteBuffer> valueMap = ((Maps.Value) value).map;
+ return mapAppliesTo((MapType<?, ?>)type, iter, valueMap, operator);
+ }
+ throw new AssertionError();
+ }
+
+ private static boolean setOrListAppliesTo(AbstractType<?> type, Iterator<Cell> iter, Iterator<ByteBuffer> conditionIter, Operator operator, boolean isSet)
+ {
+ while(iter.hasNext())
+ {
+ if (!conditionIter.hasNext())
+ return (operator == Operator.GT) || (operator == Operator.GTE) || (operator == Operator.NEQ);
+
+ // for lists we use the cell value; for sets we use the cell name
+ ByteBuffer cellValue = isSet ? iter.next().path().get(0) : iter.next().value();
+ int comparison = type.compare(cellValue, conditionIter.next());
+ if (comparison != 0)
+ return evaluateComparisonWithOperator(comparison, operator);
+ }
+
+ if (conditionIter.hasNext())
+ return (operator == Operator.LT) || (operator == Operator.LTE) || (operator == Operator.NEQ);
+
+ // they're equal
+ return operator == Operator.EQ || operator == Operator.LTE || operator == Operator.GTE;
+ }
+
+ private static boolean listAppliesTo(ListType<?> type, Iterator<Cell> iter, List<ByteBuffer> elements, Operator operator)
+ {
+ return setOrListAppliesTo(type.getElementsType(), iter, elements.iterator(), operator, false);
+ }
+
+ private static boolean setAppliesTo(SetType<?> type, Iterator<Cell> iter, Set<ByteBuffer> elements, Operator operator)
+ {
+ ArrayList<ByteBuffer> sortedElements = new ArrayList<>(elements);
+ Collections.sort(sortedElements, type.getElementsType());
+ return setOrListAppliesTo(type.getElementsType(), iter, sortedElements.iterator(), operator, true);
+ }
+
+ private static boolean mapAppliesTo(MapType<?, ?> type, Iterator<Cell> iter, Map<ByteBuffer, ByteBuffer> elements, Operator operator)
+ {
+ Iterator<Map.Entry<ByteBuffer, ByteBuffer>> conditionIter = elements.entrySet().iterator();
+ while(iter.hasNext())
+ {
+ if (!conditionIter.hasNext())
+ return (operator == Operator.GT) || (operator == Operator.GTE) || (operator == Operator.NEQ);
+
+ Map.Entry<ByteBuffer, ByteBuffer> conditionEntry = conditionIter.next();
+ Cell c = iter.next();
+
+ // compare the keys
+ int comparison = type.getKeysType().compare(c.path().get(0), conditionEntry.getKey());
+ if (comparison != 0)
+ return evaluateComparisonWithOperator(comparison, operator);
+
+ // compare the values
+ comparison = type.getValuesType().compare(c.value(), conditionEntry.getValue());
+ if (comparison != 0)
+ return evaluateComparisonWithOperator(comparison, operator);
+ }
+
+ if (conditionIter.hasNext())
+ return (operator == Operator.LT) || (operator == Operator.LTE) || (operator == Operator.NEQ);
+
+ // they're equal
+ return operator == Operator.EQ || operator == Operator.LTE || operator == Operator.GTE;
+ }
+ }
+
+ /**
+ * A condition on a UDT field
+ */
+ private static final class UDTFieldAccessBound extends Bound
+ {
+ /**
+ * The UDT field.
+ */
+ private final FieldIdentifier field;
+
+ /**
+ * The conditions values.
+ */
+ private final List<ByteBuffer> values;
+
+ private UDTFieldAccessBound(ColumnMetadata column, FieldIdentifier field, Operator operator, List<ByteBuffer> values)
+ {
+ super(column, operator);
+ assert column.type.isUDT() && field != null;
+ this.field = field;
+ this.values = values;
+ }
+
+ @Override
+ public boolean appliesTo(Row row)
+ {
+ return isSatisfiedBy(rowValue(row));
+ }
+
+ private ByteBuffer rowValue(Row row)
+ {
+ UserType userType = (UserType) column.type;
+
+ if (column.type.isMultiCell())
+ {
+ Cell cell = getCell(row, column, userType.cellPathForField(field));
+ return cell == null ? null : cell.value();
+ }
+
+ Cell cell = getCell(row, column);
+ return cell == null
+ ? null
+ : userType.split(cell.value())[userType.fieldPosition(field)];
+ }
+
+ private boolean isSatisfiedBy(ByteBuffer rowValue)
+ {
+ UserType userType = (UserType) column.type;
+ int fieldPosition = userType.fieldPosition(field);
+ AbstractType<?> valueType = userType.fieldType(fieldPosition);
+ for (ByteBuffer value : values)
+ {
+ if (compareWithOperator(comparisonOperator, valueType, value, rowValue))
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ public String toString()
+ {
+ return ToStringBuilder.reflectionToString(this, ToStringStyle.SHORT_PREFIX_STYLE);
+ }
+ }
+
+ /**
+ * A condition on an entire UDT.
+ */
+ private static final class MultiCellUdtBound extends Bound
+ {
+ /**
+ * The conditions values.
+ */
+ private final List<ByteBuffer> values;
+
+ /**
+ * The protocol version
+ */
+ private final ProtocolVersion protocolVersion;
+
+ private MultiCellUdtBound(ColumnMetadata column, Operator op, List<ByteBuffer> values, ProtocolVersion protocolVersion)
+ {
+ super(column, op);
+ assert column.type.isMultiCell();
+ this.values = values;
+ this.protocolVersion = protocolVersion;
+ }
+
+ @Override
+ public boolean appliesTo(Row row)
+ {
+ return isSatisfiedBy(rowValue(row));
+ }
+
+ private final ByteBuffer rowValue(Row row)
+ {
+ UserType userType = (UserType) column.type;
+ Iterator<Cell> iter = getCells(row, column);
+ return iter.hasNext() ? userType.serializeForNativeProtocol(iter, protocolVersion) : null;
+ }
+
+ private boolean isSatisfiedBy(ByteBuffer rowValue)
+ {
+ for (ByteBuffer value : values)
+ {
+ if (compareWithOperator(comparisonOperator, column.type, value, rowValue))
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ public String toString()
+ {
+ return ToStringBuilder.reflectionToString(this, ToStringStyle.SHORT_PREFIX_STYLE);
+ }
+ }
+
+ public static class Raw
+ {
+ private final Term.Raw value;
+ private final List<Term.Raw> inValues;
+ private final AbstractMarker.INRaw inMarker;
+
+ // Can be null, only used with the syntax "IF m[e] = ..." (in which case it's 'e')
+ private final Term.Raw collectionElement;
+
+ // Can be null, only used with the syntax "IF udt.field = ..." (in which case it's 'field')
+ private final FieldIdentifier udtField;
+
+ private final Operator operator;
+
+ private Raw(Term.Raw value, List<Term.Raw> inValues, AbstractMarker.INRaw inMarker, Term.Raw collectionElement,
+ FieldIdentifier udtField, Operator op)
+ {
+ this.value = value;
+ this.inValues = inValues;
+ this.inMarker = inMarker;
+ this.collectionElement = collectionElement;
+ this.udtField = udtField;
+ this.operator = op;
+ }
+
+ /** A condition on a column. For example: "IF col = 'foo'" */
+ public static Raw simpleCondition(Term.Raw value, Operator op)
+ {
+ return new Raw(value, null, null, null, null, op);
+ }
+
+ /** An IN condition on a column. For example: "IF col IN ('foo', 'bar', ...)" */
+ public static Raw simpleInCondition(List<Term.Raw> inValues)
+ {
+ return new Raw(null, inValues, null, null, null, Operator.IN);
+ }
+
+ /** An IN condition on a column with a single marker. For example: "IF col IN ?" */
+ public static Raw simpleInCondition(AbstractMarker.INRaw inMarker)
+ {
+ return new Raw(null, null, inMarker, null, null, Operator.IN);
+ }
+
+ /** A condition on a collection element. For example: "IF col['key'] = 'foo'" */
+ public static Raw collectionCondition(Term.Raw value, Term.Raw collectionElement, Operator op)
+ {
+ return new Raw(value, null, null, collectionElement, null, op);
+ }
+
+ /** An IN condition on a collection element. For example: "IF col['key'] IN ('foo', 'bar', ...)" */
+ public static Raw collectionInCondition(Term.Raw collectionElement, List<Term.Raw> inValues)
+ {
+ return new Raw(null, inValues, null, collectionElement, null, Operator.IN);
+ }
+
+ /** An IN condition on a collection element with a single marker. For example: "IF col['key'] IN ?" */
+ public static Raw collectionInCondition(Term.Raw collectionElement, AbstractMarker.INRaw inMarker)
+ {
+ return new Raw(null, null, inMarker, collectionElement, null, Operator.IN);
+ }
+
+ /** A condition on a UDT field. For example: "IF col.field = 'foo'" */
+ public static Raw udtFieldCondition(Term.Raw value, FieldIdentifier udtField, Operator op)
+ {
+ return new Raw(value, null, null, null, udtField, op);
+ }
+
+ /** An IN condition on a collection element. For example: "IF col.field IN ('foo', 'bar', ...)" */
+ public static Raw udtFieldInCondition(FieldIdentifier udtField, List<Term.Raw> inValues)
+ {
+ return new Raw(null, inValues, null, null, udtField, Operator.IN);
+ }
+
+ /** An IN condition on a collection element with a single marker. For example: "IF col.field IN ?" */
+ public static Raw udtFieldInCondition(FieldIdentifier udtField, AbstractMarker.INRaw inMarker)
+ {
+ return new Raw(null, null, inMarker, null, udtField, Operator.IN);
+ }
+
+ public ColumnCondition prepare(String keyspace, ColumnMetadata receiver, TableMetadata cfm)
+ {
+ if (receiver.type instanceof CounterColumnType)
+ throw invalidRequest("Conditions on counters are not supported");
+
+ if (collectionElement != null)
+ {
+ if (!(receiver.type.isCollection()))
+ throw invalidRequest("Invalid element access syntax for non-collection column %s", receiver.name);
+
+ ColumnSpecification elementSpec, valueSpec;
+ switch ((((CollectionType<?>) receiver.type).kind))
+ {
+ case LIST:
+ elementSpec = Lists.indexSpecOf(receiver);
+ valueSpec = Lists.valueSpecOf(receiver);
+ break;
+ case MAP:
+ elementSpec = Maps.keySpecOf(receiver);
+ valueSpec = Maps.valueSpecOf(receiver);
+ break;
+ case SET:
+ throw invalidRequest("Invalid element access syntax for set column %s", receiver.name);
+ default:
+ throw new AssertionError();
+ }
+
+ validateOperationOnDurations(valueSpec.type);
+ return condition(receiver, collectionElement.prepare(keyspace, elementSpec), operator, prepareTerms(keyspace, valueSpec));
+ }
+
+ if (udtField != null)
+ {
+ UserType userType = (UserType) receiver.type;
+ int fieldPosition = userType.fieldPosition(udtField);
+ if (fieldPosition == -1)
+ throw invalidRequest("Unknown field %s for column %s", udtField, receiver.name);
+
+ ColumnSpecification fieldReceiver = UserTypes.fieldSpecOf(receiver, fieldPosition);
+ validateOperationOnDurations(fieldReceiver.type);
+ return condition(receiver, udtField, operator, prepareTerms(keyspace, fieldReceiver));
+ }
+
+ validateOperationOnDurations(receiver.type);
+ return condition(receiver, operator, prepareTerms(keyspace, receiver));
+ }
+
+ private Terms prepareTerms(String keyspace, ColumnSpecification receiver)
+ {
+ if (operator.isIN())
+ {
+ return inValues == null ? Terms.ofListMarker(inMarker.prepare(keyspace, receiver), receiver.type)
+ : Terms.of(prepareTerms(keyspace, receiver, inValues));
+ }
+
+ return Terms.of(value.prepare(keyspace, receiver));
+ }
+
+ private static List<Term> prepareTerms(String keyspace, ColumnSpecification receiver, List<Term.Raw> raws)
+ {
+ List<Term> terms = new ArrayList<>(raws.size());
+ for (int i = 0, m = raws.size(); i < m; i++)
+ {
+ Term.Raw raw = raws.get(i);
+ terms.add(raw.prepare(keyspace, receiver));
+ }
+ return terms;
+ }
+
+ private void validateOperationOnDurations(AbstractType<?> type)
+ {
+ if (type.referencesDuration() && operator.isSlice())
+ {
+ checkFalse(type.isCollection(), "Slice conditions are not supported on collections containing durations");
+ checkFalse(type.isTuple(), "Slice conditions are not supported on tuples containing durations");
+ checkFalse(type.isUDT(), "Slice conditions are not supported on UDTs containing durations");
+ throw invalidRequest("Slice conditions ( %s ) are not supported on durations", operator);
+ }
+ }
+
+ public Term.Raw getValue()
+ {
+ return value;
+ }
+
+ @Override
+ public String toString()
+ {
+ return ToStringBuilder.reflectionToString(this, ToStringStyle.SHORT_PREFIX_STYLE);
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/cql3/conditions/ColumnConditions.java b/src/java/org/apache/cassandra/cql3/conditions/ColumnConditions.java
new file mode 100644
index 0000000..ba82643
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/conditions/ColumnConditions.java
@@ -0,0 +1,173 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.conditions;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import org.apache.cassandra.cql3.QueryOptions;
+import org.apache.cassandra.cql3.functions.Function;
+import org.apache.cassandra.cql3.statements.CQL3CasRequest;
+import org.apache.cassandra.db.Clustering;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.commons.lang3.builder.ToStringBuilder;
+import org.apache.commons.lang3.builder.ToStringStyle;
+
+/**
+ * A set of <code>ColumnCondition</code>s.
+ *
+ */
+public final class ColumnConditions extends AbstractConditions
+{
+ /**
+ * The conditions on regular columns.
+ */
+ private final List<ColumnCondition> columnConditions;
+
+ /**
+ * The conditions on static columns
+ */
+ private final List<ColumnCondition> staticConditions;
+
+ /**
+ * Creates a new <code>ColumnConditions</code> instance for the specified builder.
+ */
+ private ColumnConditions(Builder builder)
+ {
+ this.columnConditions = builder.columnConditions;
+ this.staticConditions = builder.staticConditions;
+ }
+
+ @Override
+ public boolean appliesToStaticColumns()
+ {
+ return !staticConditions.isEmpty();
+ }
+
+ @Override
+ public boolean appliesToRegularColumns()
+ {
+ return !columnConditions.isEmpty();
+ }
+
+ @Override
+ public Collection<ColumnMetadata> getColumns()
+ {
+ return Stream.concat(columnConditions.stream(), staticConditions.stream())
+ .map(e -> e.column)
+ .collect(Collectors.toList());
+ }
+
+ @Override
+ public boolean isEmpty()
+ {
+ return columnConditions.isEmpty() && staticConditions.isEmpty();
+ }
+
+ /**
+ * Adds the conditions to the specified CAS request.
+ *
+ * @param request the request
+ * @param clustering the clustering prefix
+ * @param options the query options
+ */
+ public void addConditionsTo(CQL3CasRequest request,
+ Clustering clustering,
+ QueryOptions options)
+ {
+ if (!columnConditions.isEmpty())
+ request.addConditions(clustering, columnConditions, options);
+ if (!staticConditions.isEmpty())
+ request.addConditions(Clustering.STATIC_CLUSTERING, staticConditions, options);
+ }
+
+ @Override
+ public void addFunctionsTo(List<Function> functions)
+ {
+ columnConditions.forEach(p -> p.addFunctionsTo(functions));
+ staticConditions.forEach(p -> p.addFunctionsTo(functions));
+ }
+
+ /**
+ * Creates a new <code>Builder</code> for <code>ColumnConditions</code>.
+ * @return a new <code>Builder</code> for <code>ColumnConditions</code>
+ */
+ public static Builder newBuilder()
+ {
+ return new Builder();
+ }
+
+ /**
+ * A <code>Builder</code> for <code>ColumnConditions</code>.
+ *
+ */
+ public static final class Builder
+ {
+ /**
+ * The conditions on regular columns.
+ */
+ private List<ColumnCondition> columnConditions = Collections.emptyList();
+
+ /**
+ * The conditions on static columns
+ */
+ private List<ColumnCondition> staticConditions = Collections.emptyList();
+
+ /**
+ * Adds the specified <code>ColumnCondition</code> to this set of conditions.
+ * @param condition the condition to add
+ */
+ public Builder add(ColumnCondition condition)
+ {
+ List<ColumnCondition> conds;
+ if (condition.column.isStatic())
+ {
+ if (staticConditions.isEmpty())
+ staticConditions = new ArrayList<>();
+ conds = staticConditions;
+ }
+ else
+ {
+ if (columnConditions.isEmpty())
+ columnConditions = new ArrayList<>();
+ conds = columnConditions;
+ }
+ conds.add(condition);
+ return this;
+ }
+
+ public ColumnConditions build()
+ {
+ return new ColumnConditions(this);
+ }
+
+ private Builder()
+ {
+ }
+ }
+
+ @Override
+ public String toString()
+ {
+ return ToStringBuilder.reflectionToString(this, ToStringStyle.SHORT_PREFIX_STYLE);
+ }
+}
diff --git a/src/java/org/apache/cassandra/cql3/conditions/Conditions.java b/src/java/org/apache/cassandra/cql3/conditions/Conditions.java
new file mode 100644
index 0000000..1622be0
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/conditions/Conditions.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.conditions;
+
+import java.util.List;
+
+import org.apache.cassandra.cql3.QueryOptions;
+import org.apache.cassandra.cql3.functions.Function;
+import org.apache.cassandra.cql3.statements.CQL3CasRequest;
+import org.apache.cassandra.db.Clustering;
+import org.apache.cassandra.schema.ColumnMetadata;
+
+/**
+ * Conditions that can be applied to a mutation statement.
+ *
+ */
+public interface Conditions
+{
+ /**
+ * An EMPTY condition
+ */
+ static final Conditions EMPTY_CONDITION = ColumnConditions.newBuilder().build();
+
+ /**
+ * IF EXISTS condition
+ */
+ static final Conditions IF_EXISTS_CONDITION = new IfExistsCondition();
+
+ /**
+ * IF NOT EXISTS condition
+ */
+ static final Conditions IF_NOT_EXISTS_CONDITION = new IfNotExistsCondition();
+
+ /**
+ * Adds the functions used by the conditions to the specified list.
+ * @param functions the list to add to
+ */
+ void addFunctionsTo(List<Function> functions);
+
+ /**
+ * Returns the column definitions to which apply the conditions.
+ * @return the column definitions to which apply the conditions.
+ */
+ Iterable<ColumnMetadata> getColumns();
+
+ /**
+ * Checks if this <code>Conditions</code> is empty.
+ * @return <code>true</code> if this <code>Conditions</code> is empty, <code>false</code> otherwise.
+ */
+ boolean isEmpty();
+
+ /**
+ * Checks if this is a IF EXIST condition.
+ * @return <code>true</code> if this is a IF EXIST condition, <code>false</code> otherwise.
+ */
+ boolean isIfExists();
+
+ /**
+ * Checks if this is a IF NOT EXIST condition.
+ * @return <code>true</code> if this is a IF NOT EXIST condition, <code>false</code> otherwise.
+ */
+ boolean isIfNotExists();
+
+ /**
+ * Checks if some of the conditions apply to static columns.
+ *
+ * @return <code>true</code> if some of the conditions apply to static columns, <code>false</code> otherwise.
+ */
+ boolean appliesToStaticColumns();
+
+ /**
+ * Checks if some of the conditions apply to regular columns.
+ *
+ * @return <code>true</code> if some of the conditions apply to regular columns, <code>false</code> otherwise.
+ */
+ boolean appliesToRegularColumns();
+
+ /**
+ * Adds the conditions to the specified CAS request.
+ *
+ * @param request the request
+ * @param clustering the clustering prefix
+ * @param options the query options
+ */
+ public void addConditionsTo(CQL3CasRequest request,
+ Clustering clustering,
+ QueryOptions options);
+}
diff --git a/src/java/org/apache/cassandra/cql3/conditions/IfExistsCondition.java b/src/java/org/apache/cassandra/cql3/conditions/IfExistsCondition.java
new file mode 100644
index 0000000..fa8822d
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/conditions/IfExistsCondition.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.conditions;
+
+import org.apache.cassandra.cql3.QueryOptions;
+import org.apache.cassandra.cql3.statements.CQL3CasRequest;
+import org.apache.cassandra.db.Clustering;
+
+final class IfExistsCondition extends AbstractConditions
+{
+ @Override
+ public void addConditionsTo(CQL3CasRequest request, Clustering clustering, QueryOptions options)
+ {
+ request.addExist(clustering);
+ }
+
+ @Override
+ public boolean isIfExists()
+ {
+ return true;
+ }
+}
diff --git a/src/java/org/apache/cassandra/cql3/conditions/IfNotExistsCondition.java b/src/java/org/apache/cassandra/cql3/conditions/IfNotExistsCondition.java
new file mode 100644
index 0000000..ede9119
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/conditions/IfNotExistsCondition.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.conditions;
+
+import org.apache.cassandra.cql3.QueryOptions;
+import org.apache.cassandra.cql3.statements.CQL3CasRequest;
+import org.apache.cassandra.db.Clustering;
+
+final class IfNotExistsCondition extends AbstractConditions
+{
+ @Override
+ public void addConditionsTo(CQL3CasRequest request, Clustering clustering, QueryOptions options)
+ {
+ request.addNotExist(clustering);
+ }
+
+ @Override
+ public boolean isIfNotExists()
+ {
+ return true;
+ }
+}
diff --git a/src/java/org/apache/cassandra/cql3/functions/AbstractFunction.java b/src/java/org/apache/cassandra/cql3/functions/AbstractFunction.java
index aa7555f..940f0a4 100644
--- a/src/java/org/apache/cassandra/cql3/functions/AbstractFunction.java
+++ b/src/java/org/apache/cassandra/cql3/functions/AbstractFunction.java
@@ -17,15 +17,23 @@
*/
package org.apache.cassandra.cql3.functions;
+import java.nio.ByteBuffer;
import java.util.List;
import com.google.common.base.Objects;
import org.apache.cassandra.cql3.AssignmentTestable;
+import org.apache.cassandra.cql3.CQL3Type;
+import org.apache.cassandra.cql3.CQL3Type.Tuple;
import org.apache.cassandra.cql3.ColumnSpecification;
+import org.apache.cassandra.cql3.CqlBuilder;
+import org.apache.cassandra.cql3.CqlBuilder.Appender;
import org.apache.cassandra.db.marshal.AbstractType;
+
import org.apache.commons.lang3.text.StrBuilder;
+import static java.util.stream.Collectors.toList;
+
/**
* Base class for our native/hardcoded functions.
*/
@@ -57,6 +65,14 @@
return returnType;
}
+ public List<String> argumentsList()
+ {
+ return argTypes().stream()
+ .map(AbstractType::asCQL3Type)
+ .map(CQL3Type::toString)
+ .collect(toList());
+ }
+
@Override
public boolean equals(Object o)
{
@@ -74,7 +90,7 @@
functions.add(this);
}
- public boolean hasReferenceTo(Function function)
+ public boolean referencesUserType(ByteBuffer name)
{
return false;
}
@@ -105,16 +121,41 @@
@Override
public String toString()
{
- StringBuilder sb = new StringBuilder();
- sb.append(name).append(" : (");
- for (int i = 0; i < argTypes.size(); i++)
- {
- if (i > 0)
- sb.append(", ");
- sb.append(argTypes.get(i).asCQL3Type());
- }
- sb.append(") -> ").append(returnType.asCQL3Type());
- return sb.toString();
+ return new CqlBuilder().append(name)
+ .append(" : (")
+ .appendWithSeparators(argTypes, (b, t) -> b.append(toCqlString(t)), ", ")
+ .append(") -> ")
+ .append(returnType)
+ .toString();
+ }
+
+ public String elementKeyspace()
+ {
+ return name.keyspace;
+ }
+
+ public String elementName()
+ {
+ return new CqlBuilder().append(name.name)
+ .append('(')
+ .appendWithSeparators(argTypes, (b, t) -> b.append(toCqlString(t)), ", ")
+ .append(')')
+ .toString();
+ }
+
+ /**
+ * Converts the specified type into its CQL representation.
+ *
+ * <p>For user function and aggregates tuples need to be handle in a special way as they are frozen by nature
+ * but the frozen keyword should not appear in their CQL definition.</p>
+ *
+ * @param type the type
+ * @return the CQL representation of the specified type
+ */
+ protected String toCqlString(AbstractType<?> type)
+ {
+ return type.isTuple() ? ((Tuple) type.asCQL3Type()).toString(false)
+ : type.asCQL3Type().toString();
}
@Override
diff --git a/src/java/org/apache/cassandra/cql3/functions/CastFcts.java b/src/java/org/apache/cassandra/cql3/functions/CastFcts.java
index 9f825ee..81986c8 100644
--- a/src/java/org/apache/cassandra/cql3/functions/CastFcts.java
+++ b/src/java/org/apache/cassandra/cql3/functions/CastFcts.java
@@ -46,6 +46,8 @@
import org.apache.cassandra.db.marshal.UUIDType;
import org.apache.cassandra.transport.ProtocolVersion;
+import static org.apache.cassandra.cql3.functions.TimeFcts.*;
+
import org.apache.commons.lang3.text.WordUtils;
/**
@@ -93,14 +95,14 @@
functions.add(CastAsTextFunction.create(BooleanType.instance, AsciiType.instance));
functions.add(CastAsTextFunction.create(BooleanType.instance, UTF8Type.instance));
- functions.add(CassandraFunctionWrapper.create(TimeUUIDType.instance, SimpleDateType.instance, TimeFcts.timeUuidtoDate));
- functions.add(CassandraFunctionWrapper.create(TimeUUIDType.instance, TimestampType.instance, TimeFcts.timeUuidToTimestamp));
+ functions.add(CassandraFunctionWrapper.create(TimeUUIDType.instance, SimpleDateType.instance, toDate(TimeUUIDType.instance)));
+ functions.add(CassandraFunctionWrapper.create(TimeUUIDType.instance, TimestampType.instance, toTimestamp(TimeUUIDType.instance)));
functions.add(CastAsTextFunction.create(TimeUUIDType.instance, AsciiType.instance));
functions.add(CastAsTextFunction.create(TimeUUIDType.instance, UTF8Type.instance));
- functions.add(CassandraFunctionWrapper.create(TimestampType.instance, SimpleDateType.instance, TimeFcts.timestampToDate));
+ functions.add(CassandraFunctionWrapper.create(TimestampType.instance, SimpleDateType.instance, toDate(TimestampType.instance)));
functions.add(CastAsTextFunction.create(TimestampType.instance, AsciiType.instance));
functions.add(CastAsTextFunction.create(TimestampType.instance, UTF8Type.instance));
- functions.add(CassandraFunctionWrapper.create(SimpleDateType.instance, TimestampType.instance, TimeFcts.dateToTimestamp));
+ functions.add(CassandraFunctionWrapper.create(SimpleDateType.instance, TimestampType.instance, toTimestamp(SimpleDateType.instance)));
functions.add(CastAsTextFunction.create(SimpleDateType.instance, AsciiType.instance));
functions.add(CastAsTextFunction.create(SimpleDateType.instance, UTF8Type.instance));
functions.add(CastAsTextFunction.create(TimeType.instance, AsciiType.instance));
diff --git a/src/java/org/apache/cassandra/cql3/functions/Function.java b/src/java/org/apache/cassandra/cql3/functions/Function.java
index 5d258af..e13e906 100644
--- a/src/java/org/apache/cassandra/cql3/functions/Function.java
+++ b/src/java/org/apache/cassandra/cql3/functions/Function.java
@@ -17,10 +17,13 @@
*/
package org.apache.cassandra.cql3.functions;
+import java.nio.ByteBuffer;
import java.util.List;
+import java.util.Optional;
import org.apache.cassandra.cql3.AssignmentTestable;
import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.schema.Difference;
import org.github.jamm.Unmetered;
@Unmetered
@@ -46,7 +49,7 @@
public void addFunctionsTo(List<Function> functions);
- public boolean hasReferenceTo(Function function);
+ public boolean referencesUserType(ByteBuffer name);
/**
* Returns the name of the function to use within a ResultSet.
@@ -55,4 +58,9 @@
* @return the name of the function to use within a ResultSet
*/
public String columnName(List<String> columnNames);
+
+ public default Optional<Difference> compare(Function other)
+ {
+ throw new UnsupportedOperationException();
+ }
}
diff --git a/src/java/org/apache/cassandra/cql3/functions/FunctionCall.java b/src/java/org/apache/cassandra/cql3/functions/FunctionCall.java
index 8dcb3da..e0dae52 100644
--- a/src/java/org/apache/cassandra/cql3/functions/FunctionCall.java
+++ b/src/java/org/apache/cassandra/cql3/functions/FunctionCall.java
@@ -19,6 +19,8 @@
import java.nio.ByteBuffer;
import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
import java.util.List;
import java.util.stream.Collectors;
@@ -30,6 +32,8 @@
import org.apache.cassandra.transport.ProtocolVersion;
import org.apache.cassandra.utils.ByteBufferUtil;
+import static org.apache.cassandra.cql3.statements.RequestValidations.invalidRequest;
+
public class FunctionCall extends Term.NonTerminal
{
private final ScalarFunction fun;
@@ -132,26 +136,45 @@
this.terms = terms;
}
+ public static Raw newOperation(char operator, Term.Raw left, Term.Raw right)
+ {
+ FunctionName name = OperationFcts.getFunctionNameFromOperator(operator);
+ return new Raw(name, Arrays.asList(left, right));
+ }
+
+ public static Raw newNegation(Term.Raw raw)
+ {
+ FunctionName name = FunctionName.nativeFunction(OperationFcts.NEGATION_FUNCTION_NAME);
+ return new Raw(name, Collections.singletonList(raw));
+ }
+
public Term prepare(String keyspace, ColumnSpecification receiver) throws InvalidRequestException
{
Function fun = FunctionResolver.get(keyspace, name, terms, receiver.ksName, receiver.cfName, receiver.type);
if (fun == null)
- throw new InvalidRequestException(String.format("Unknown function %s called", name));
+ throw invalidRequest("Unknown function %s called", name);
if (fun.isAggregate())
- throw new InvalidRequestException("Aggregation function are not supported in the where clause");
+ throw invalidRequest("Aggregation function are not supported in the where clause");
ScalarFunction scalarFun = (ScalarFunction) fun;
// Functions.get() will complain if no function "name" type check with the provided arguments.
// We still have to validate that the return type matches however
if (!scalarFun.testAssignment(keyspace, receiver).isAssignable())
- throw new InvalidRequestException(String.format("Type error: cannot assign result of function %s (type %s) to %s (type %s)",
- scalarFun.name(), scalarFun.returnType().asCQL3Type(),
- receiver.name, receiver.type.asCQL3Type()));
+ {
+ if (OperationFcts.isOperation(name))
+ throw invalidRequest("Type error: cannot assign result of operation %s (type %s) to %s (type %s)",
+ OperationFcts.getOperator(scalarFun.name()), scalarFun.returnType().asCQL3Type(),
+ receiver.name, receiver.type.asCQL3Type());
+
+ throw invalidRequest("Type error: cannot assign result of function %s (type %s) to %s (type %s)",
+ scalarFun.name(), scalarFun.returnType().asCQL3Type(),
+ receiver.name, receiver.type.asCQL3Type());
+ }
if (fun.argTypes().size() != terms.size())
- throw new InvalidRequestException(String.format("Incorrect number of arguments specified for function %s (expected %d, found %d)",
- fun, fun.argTypes().size(), terms.size()));
+ throw invalidRequest("Incorrect number of arguments specified for function %s (expected %d, found %d)",
+ fun, fun.argTypes().size(), terms.size());
List<Term> parameters = new ArrayList<>(terms.size());
for (int i = 0; i < terms.size(); i++)
diff --git a/src/java/org/apache/cassandra/cql3/functions/FunctionName.java b/src/java/org/apache/cassandra/cql3/functions/FunctionName.java
index aa980e9..472a7ff 100644
--- a/src/java/org/apache/cassandra/cql3/functions/FunctionName.java
+++ b/src/java/org/apache/cassandra/cql3/functions/FunctionName.java
@@ -19,10 +19,14 @@
import com.google.common.base.Objects;
-import org.apache.cassandra.config.SchemaConstants;
+import org.apache.cassandra.cql3.CqlBuilder;
+import org.apache.cassandra.schema.SchemaConstants;
public final class FunctionName
{
+ // We special case the token function because that's the only function which name is a reserved keyword
+ private static final FunctionName TOKEN_FUNCTION_NAME = FunctionName.nativeFunction("token");
+
public final String keyspace;
public final String name;
@@ -79,4 +83,21 @@
{
return keyspace == null ? name : keyspace + "." + name;
}
+
+ public void appendCqlTo(CqlBuilder builder)
+ {
+ if (equalsNativeFunction(TOKEN_FUNCTION_NAME))
+ {
+ builder.append(name);
+ }
+ else
+ {
+ if (keyspace != null)
+ {
+ builder.appendQuotingIfNeeded(keyspace)
+ .append('.');
+ }
+ builder.appendQuotingIfNeeded(name);
+ }
+ }
}
diff --git a/src/java/org/apache/cassandra/cql3/functions/FunctionResolver.java b/src/java/org/apache/cassandra/cql3/functions/FunctionResolver.java
index 9e0b706..ae5d17e 100644
--- a/src/java/org/apache/cassandra/cql3/functions/FunctionResolver.java
+++ b/src/java/org/apache/cassandra/cql3/functions/FunctionResolver.java
@@ -21,12 +21,16 @@
import java.util.Collection;
import java.util.List;
-import org.apache.cassandra.config.Schema;
-import org.apache.cassandra.cql3.*;
+import org.apache.cassandra.cql3.AbstractMarker;
+import org.apache.cassandra.cql3.AssignmentTestable;
+import org.apache.cassandra.cql3.ColumnIdentifier;
+import org.apache.cassandra.cql3.ColumnSpecification;
+import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.exceptions.InvalidRequestException;
import static java.util.stream.Collectors.joining;
+import static org.apache.cassandra.cql3.statements.RequestValidations.invalidRequest;
public final class FunctionResolver
{
@@ -65,38 +69,7 @@
AbstractType<?> receiverType)
throws InvalidRequestException
{
- if (name.equalsNativeFunction(TOKEN_FUNCTION_NAME))
- return new TokenFct(Schema.instance.getCFMetaData(receiverKs, receiverCf));
-
- // The toJson() function can accept any type of argument, so instances of it are not pre-declared. Instead,
- // we create new instances as needed while handling selectors (which is the only place that toJson() is supported,
- // due to needing to know the argument types in advance).
- if (name.equalsNativeFunction(ToJsonFct.NAME))
- throw new InvalidRequestException("toJson() may only be used within the selection clause of SELECT statements");
-
- // Similarly, we can only use fromJson when we know the receiver type (such as inserts)
- if (name.equalsNativeFunction(FromJsonFct.NAME))
- {
- if (receiverType == null)
- throw new InvalidRequestException("fromJson() cannot be used in the selection clause of a SELECT statement");
- return FromJsonFct.getInstance(receiverType);
- }
-
- Collection<Function> candidates;
- if (!name.hasKeyspace())
- {
- // function name not fully qualified
- candidates = new ArrayList<>();
- // add 'SYSTEM' (native) candidates
- candidates.addAll(Schema.instance.getFunctions(name.asNativeFunction()));
- // add 'current keyspace' candidates
- candidates.addAll(Schema.instance.getFunctions(new FunctionName(keyspace, name.name)));
- }
- else
- {
- // function name is fully qualified (keyspace + name)
- candidates = Schema.instance.getFunctions(name);
- }
+ Collection<Function> candidates = collectCandidates(keyspace, name, receiverKs, receiverCf, receiverType);
if (candidates.isEmpty())
return null;
@@ -109,36 +82,139 @@
return fun;
}
+ return pickBestMatch(keyspace, name, providedArgs, receiverKs, receiverCf, receiverType, candidates);
+ }
+
+ private static Collection<Function> collectCandidates(String keyspace,
+ FunctionName name,
+ String receiverKs,
+ String receiverCf,
+ AbstractType<?> receiverType)
+ {
+ Collection<Function> candidates = new ArrayList<>();
+
+ if (name.equalsNativeFunction(TOKEN_FUNCTION_NAME))
+ candidates.add(new TokenFct(Schema.instance.getTableMetadata(receiverKs, receiverCf)));
+
+ // The toJson() function can accept any type of argument, so instances of it are not pre-declared. Instead,
+ // we create new instances as needed while handling selectors (which is the only place that toJson() is supported,
+ // due to needing to know the argument types in advance).
+ if (name.equalsNativeFunction(ToJsonFct.NAME))
+ throw new InvalidRequestException("toJson() may only be used within the selection clause of SELECT statements");
+
+ // Similarly, we can only use fromJson when we know the receiver type (such as inserts)
+ if (name.equalsNativeFunction(FromJsonFct.NAME))
+ {
+ if (receiverType == null)
+ throw new InvalidRequestException("fromJson() cannot be used in the selection clause of a SELECT statement");
+ candidates.add(FromJsonFct.getInstance(receiverType));
+ }
+
+ if (!name.hasKeyspace())
+ {
+ // function name not fully qualified
+ // add 'SYSTEM' (native) candidates
+ candidates.addAll(Schema.instance.getFunctions(name.asNativeFunction()));
+ // add 'current keyspace' candidates
+ candidates.addAll(Schema.instance.getFunctions(new FunctionName(keyspace, name.name)));
+ }
+ else
+ {
+ // function name is fully qualified (keyspace + name)
+ candidates.addAll(Schema.instance.getFunctions(name));
+ }
+
+ return candidates;
+ }
+
+ private static Function pickBestMatch(String keyspace,
+ FunctionName name,
+ List<? extends AssignmentTestable> providedArgs,
+ String receiverKs,
+ String receiverCf, AbstractType<?> receiverType,
+ Collection<Function> candidates)
+ {
List<Function> compatibles = null;
for (Function toTest : candidates)
{
- AssignmentTestable.TestResult r = matchAguments(keyspace, toTest, providedArgs, receiverKs, receiverCf);
- switch (r)
+ if (matchReturnType(toTest, receiverType))
{
- case EXACT_MATCH:
- // We always favor exact matches
- return toTest;
- case WEAKLY_ASSIGNABLE:
- if (compatibles == null)
- compatibles = new ArrayList<>();
- compatibles.add(toTest);
- break;
+ AssignmentTestable.TestResult r = matchAguments(keyspace, toTest, providedArgs, receiverKs, receiverCf);
+ switch (r)
+ {
+ case EXACT_MATCH:
+ // We always favor exact matches
+ return toTest;
+ case WEAKLY_ASSIGNABLE:
+ if (compatibles == null)
+ compatibles = new ArrayList<>();
+ compatibles.add(toTest);
+ break;
+ }
}
}
if (compatibles == null)
{
- throw new InvalidRequestException(String.format("Invalid call to function %s, none of its type signatures match (known type signatures: %s)",
- name, format(candidates)));
+ if (OperationFcts.isOperation(name))
+ throw invalidRequest("the '%s' operation is not supported between %s and %s",
+ OperationFcts.getOperator(name), providedArgs.get(0), providedArgs.get(1));
+
+ throw invalidRequest("Invalid call to function %s, none of its type signatures match (known type signatures: %s)",
+ name, format(candidates));
}
if (compatibles.size() > 1)
- throw new InvalidRequestException(String.format("Ambiguous call to function %s (can be matched by following signatures: %s): use type casts to disambiguate",
- name, format(compatibles)));
+ {
+ if (OperationFcts.isOperation(name))
+ {
+ if (receiverType != null && !containsMarkers(providedArgs))
+ {
+ for (Function toTest : compatibles)
+ {
+ List<AbstractType<?>> argTypes = toTest.argTypes();
+ if (receiverType.equals(argTypes.get(0)) && receiverType.equals(argTypes.get(1)))
+ return toTest;
+ }
+ }
+ throw invalidRequest("Ambiguous '%s' operation with args %s and %s: use type casts to disambiguate",
+ OperationFcts.getOperator(name), providedArgs.get(0), providedArgs.get(1));
+ }
+
+ if (OperationFcts.isNegation(name))
+ throw invalidRequest("Ambiguous negation: use type casts to disambiguate");
+
+ throw invalidRequest("Ambiguous call to function %s (can be matched by following signatures: %s): use type casts to disambiguate",
+ name, format(compatibles));
+ }
return compatibles.get(0);
}
+ /**
+ * Checks if at least one of the specified arguments is a marker.
+ *
+ * @param args the arguments to check
+ * @return {@code true} if if at least one of the specified arguments is a marker, {@code false} otherwise
+ */
+ private static boolean containsMarkers(List<? extends AssignmentTestable> args)
+ {
+ return args.stream().anyMatch(AbstractMarker.Raw.class::isInstance);
+ }
+
+ /**
+ * Checks that the return type of the specified function can be assigned to the specified receiver.
+ *
+ * @param fun the function to check
+ * @param receiverType the receiver type
+ * @return {@code true} if the return type of the specified function can be assigned to the specified receiver,
+ * {@code false} otherwise.
+ */
+ private static boolean matchReturnType(Function fun, AbstractType<?> receiverType)
+ {
+ return receiverType == null || fun.returnType().testAssignment(receiverType).isAssignable();
+ }
+
// This method and matchArguments are somewhat duplicate, but this method allows us to provide more precise errors in the common
// case where there is no override for a given function. This is thus probably worth the minor code duplication.
private static void validateTypes(String keyspace,
@@ -146,10 +222,10 @@
List<? extends AssignmentTestable> providedArgs,
String receiverKs,
String receiverCf)
- throws InvalidRequestException
{
if (providedArgs.size() != fun.argTypes().size())
- throw new InvalidRequestException(String.format("Invalid number of arguments in call to function %s: %d required but %d provided", fun.name(), fun.argTypes().size(), providedArgs.size()));
+ throw invalidRequest("Invalid number of arguments in call to function %s: %d required but %d provided",
+ fun.name(), fun.argTypes().size(), providedArgs.size());
for (int i = 0; i < providedArgs.size(); i++)
{
@@ -162,7 +238,8 @@
ColumnSpecification expected = makeArgSpec(receiverKs, receiverCf, fun, i);
if (!provided.testAssignment(keyspace, expected).isAssignable())
- throw new InvalidRequestException(String.format("Type error: %s cannot be passed as argument %d of function %s of type %s", provided, i, fun.name(), expected.type.asCQL3Type()));
+ throw invalidRequest("Type error: %s cannot be passed as argument %d of function %s of type %s",
+ provided, i, fun.name(), expected.type.asCQL3Type());
}
}
diff --git a/src/java/org/apache/cassandra/cql3/functions/JavaBasedUDFunction.java b/src/java/org/apache/cassandra/cql3/functions/JavaBasedUDFunction.java
index feb17e3..d2bac5f 100644
--- a/src/java/org/apache/cassandra/cql3/functions/JavaBasedUDFunction.java
+++ b/src/java/org/apache/cassandra/cql3/functions/JavaBasedUDFunction.java
@@ -44,9 +44,9 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.datastax.driver.core.TypeCodec;
import org.apache.cassandra.concurrent.NamedThreadFactory;
import org.apache.cassandra.cql3.ColumnIdentifier;
+import org.apache.cassandra.cql3.functions.types.TypeCodec;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.exceptions.InvalidRequestException;
import org.apache.cassandra.transport.ProtocolVersion;
@@ -70,7 +70,7 @@
private static final Pattern JAVA_LANG_PREFIX = Pattern.compile("\\bjava\\.lang\\.");
- static final Logger logger = LoggerFactory.getLogger(JavaBasedUDFunction.class);
+ private static final Logger logger = LoggerFactory.getLogger(JavaBasedUDFunction.class);
private static final AtomicInteger classSequence = new AtomicInteger();
@@ -187,6 +187,8 @@
private final JavaUDF javaUDF;
+ private static final Pattern patternJavaDriver = Pattern.compile("com\\.datastax\\.driver\\.core\\.");
+
JavaBasedUDFunction(FunctionName name, List<ColumnIdentifier> argNames, List<AbstractType<?>> argTypes,
AbstractType<?> returnType, boolean calledOnNullInput, String body)
{
@@ -223,7 +225,7 @@
break;
case "body":
lineOffset = countNewlines(javaSourceBuilder);
- s = body;
+ s = patternJavaDriver.matcher(body).replaceAll("org.apache.cassandra.cql3.functions.types.");
break;
case "arguments":
s = generateArguments(javaParamTypes, argNames, false);
@@ -257,10 +259,10 @@
EcjCompilationUnit compilationUnit = new EcjCompilationUnit(javaSource, targetClassName);
Compiler compiler = new Compiler(compilationUnit,
- errorHandlingPolicy,
- compilerOptions,
- compilationUnit,
- problemFactory);
+ errorHandlingPolicy,
+ compilerOptions,
+ compilationUnit,
+ problemFactory);
compiler.compile(new ICompilationUnit[]{ compilationUnit });
if (compilationUnit.problemList != null && !compilationUnit.problemList.isEmpty())
@@ -582,6 +584,7 @@
return findType(result.toString());
}
+ @SuppressWarnings("resource")
private NameEnvironmentAnswer findType(String className)
{
if (className.equals(this.className))
diff --git a/src/java/org/apache/cassandra/cql3/functions/JavaUDF.java b/src/java/org/apache/cassandra/cql3/functions/JavaUDF.java
index fab29f3..3134da9 100644
--- a/src/java/org/apache/cassandra/cql3/functions/JavaUDF.java
+++ b/src/java/org/apache/cassandra/cql3/functions/JavaUDF.java
@@ -21,7 +21,7 @@
import java.nio.ByteBuffer;
import java.util.List;
-import com.datastax.driver.core.TypeCodec;
+import org.apache.cassandra.cql3.functions.types.TypeCodec;
import org.apache.cassandra.transport.ProtocolVersion;
/**
diff --git a/src/java/org/apache/cassandra/cql3/functions/OperationFcts.java b/src/java/org/apache/cassandra/cql3/functions/OperationFcts.java
new file mode 100644
index 0000000..4994660
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/functions/OperationFcts.java
@@ -0,0 +1,464 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.functions;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+import org.apache.cassandra.schema.SchemaConstants;
+import org.apache.cassandra.db.marshal.*;
+import org.apache.cassandra.exceptions.OperationExecutionException;
+import org.apache.cassandra.transport.ProtocolVersion;
+
+/**
+ * Operation functions (Mathematics).
+ *
+ */
+public final class OperationFcts
+{
+ private static enum OPERATION
+ {
+ ADDITION('+', "_add")
+ {
+ protected ByteBuffer executeOnNumerics(NumberType<?> resultType,
+ NumberType<?> leftType,
+ ByteBuffer left,
+ NumberType<?> rightType,
+ ByteBuffer right)
+ {
+ return resultType.add(leftType, left, rightType, right);
+ }
+
+ @Override
+ protected ByteBuffer executeOnTemporals(TemporalType<?> type,
+ ByteBuffer temporal,
+ ByteBuffer duration)
+ {
+ return type.addDuration(temporal, duration);
+ }
+ },
+ SUBSTRACTION('-', "_substract")
+ {
+ protected ByteBuffer executeOnNumerics(NumberType<?> resultType,
+ NumberType<?> leftType,
+ ByteBuffer left,
+ NumberType<?> rightType,
+ ByteBuffer right)
+ {
+ return resultType.substract(leftType, left, rightType, right);
+ }
+
+ @Override
+ protected ByteBuffer executeOnTemporals(TemporalType<?> type,
+ ByteBuffer temporal,
+ ByteBuffer duration)
+ {
+ return type.substractDuration(temporal, duration);
+ }
+ },
+ MULTIPLICATION('*', "_multiply")
+ {
+ protected ByteBuffer executeOnNumerics(NumberType<?> resultType,
+ NumberType<?> leftType,
+ ByteBuffer left,
+ NumberType<?> rightType,
+ ByteBuffer right)
+ {
+ return resultType.multiply(leftType, left, rightType, right);
+ }
+ },
+ DIVISION('/', "_divide")
+ {
+ protected ByteBuffer executeOnNumerics(NumberType<?> resultType,
+ NumberType<?> leftType,
+ ByteBuffer left,
+ NumberType<?> rightType,
+ ByteBuffer right)
+ {
+ return resultType.divide(leftType, left, rightType, right);
+ }
+ },
+ MODULO('%', "_modulo")
+ {
+ protected ByteBuffer executeOnNumerics(NumberType<?> resultType,
+ NumberType<?> leftType,
+ ByteBuffer left,
+ NumberType<?> rightType,
+ ByteBuffer right)
+ {
+ return resultType.mod(leftType, left, rightType, right);
+ }
+ };
+
+ /**
+ * The operator symbol.
+ */
+ private final char symbol;
+
+ /**
+ * The name of the function associated to this operation
+ */
+ private final String functionName;
+
+ private OPERATION(char symbol, String functionName)
+ {
+ this.symbol = symbol;
+ this.functionName = functionName;
+ }
+
+ /**
+ * Executes the operation between the specified numeric operand.
+ *
+ * @param resultType the result ype of the operation
+ * @param leftType the type of the left operand
+ * @param left the left operand
+ * @param rightType the type of the right operand
+ * @param right the right operand
+ * @return the operation result
+ */
+ protected abstract ByteBuffer executeOnNumerics(NumberType<?> resultType,
+ NumberType<?> leftType,
+ ByteBuffer left,
+ NumberType<?> rightType,
+ ByteBuffer right);
+
+ /**
+ * Executes the operation on the specified temporal operand.
+ *
+ * @param type the temporal type
+ * @param temporal the temporal value
+ * @param duration the duration
+ * @return the operation result
+ */
+ protected ByteBuffer executeOnTemporals(TemporalType<?> type,
+ ByteBuffer temporal,
+ ByteBuffer duration)
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ /**
+ * Returns the {@code OPERATOR} associated to the specified function.
+ * @param functionName the function name
+ * @return the {@code OPERATOR} associated to the specified function
+ */
+ public static OPERATION fromFunctionName(String functionName)
+ {
+ for (OPERATION operator : values())
+ {
+ if (operator.functionName.equals(functionName))
+ return operator;
+ }
+ return null;
+ }
+
+ /**
+ * Returns the {@code OPERATOR} with the specified symbol.
+ * @param functionName the function name
+ * @return the {@code OPERATOR} with the specified symbol
+ */
+ public static OPERATION fromSymbol(char symbol)
+ {
+ for (OPERATION operator : values())
+ {
+ if (operator.symbol == symbol)
+ return operator;
+ }
+ return null;
+ }
+ }
+
+ /**
+ * The name of the function used to perform negations
+ */
+ public static final String NEGATION_FUNCTION_NAME = "_negate";
+
+ public static Collection<Function> all()
+ {
+ List<Function> functions = new ArrayList<>();
+
+ final NumberType<?>[] numericTypes = new NumberType[] { ByteType.instance,
+ ShortType.instance,
+ Int32Type.instance,
+ LongType.instance,
+ FloatType.instance,
+ DoubleType.instance,
+ DecimalType.instance,
+ IntegerType.instance,
+ CounterColumnType.instance};
+
+ for (NumberType<?> left : numericTypes)
+ {
+ for (NumberType<?> right : numericTypes)
+ {
+ NumberType<?> returnType = returnType(left, right);
+ for (OPERATION operation : OPERATION.values())
+ functions.add(new NumericOperationFunction(returnType, left, operation, right));
+ }
+ functions.add(new NumericNegationFunction(left));
+ }
+
+ for (OPERATION operation : new OPERATION[] {OPERATION.ADDITION, OPERATION.SUBSTRACTION})
+ {
+ functions.add(new TemporalOperationFunction(TimestampType.instance, operation));
+ functions.add(new TemporalOperationFunction(SimpleDateType.instance, operation));
+ }
+
+ return functions;
+ }
+
+ /**
+ * Checks if the function with the specified name is an operation.
+ *
+ * @param function the function name
+ * @return {@code true} if the function is an operation, {@code false} otherwise.
+ */
+ public static boolean isOperation(FunctionName function)
+ {
+ return SchemaConstants.SYSTEM_KEYSPACE_NAME.equals(function.keyspace)
+ && OPERATION.fromFunctionName(function.name) != null;
+ }
+
+ /**
+ * Checks if the function with the specified name is a negation.
+ *
+ * @param function the function name
+ * @return {@code true} if the function is an negation, {@code false} otherwise.
+ */
+ public static boolean isNegation(FunctionName function)
+ {
+ return SchemaConstants.SYSTEM_KEYSPACE_NAME.equals(function.keyspace)&& NEGATION_FUNCTION_NAME.equals(function.name);
+ }
+
+ /**
+ * Returns the operator associated to the specified function.
+ *
+ * @return the operator associated to the specified function.
+ */
+ public static char getOperator(FunctionName function)
+ {
+ assert SchemaConstants.SYSTEM_KEYSPACE_NAME.equals(function.keyspace);
+ return OPERATION.fromFunctionName(function.name).symbol;
+ }
+
+ /**
+ * Returns the name of the function associated to the specified operator.
+ *
+ * @param operator the operator
+ * @return the name of the function associated to the specified operator
+ */
+ public static FunctionName getFunctionNameFromOperator(char operator)
+ {
+ return FunctionName.nativeFunction(OPERATION.fromSymbol(operator).functionName);
+ }
+
+ /**
+ * Determine the return type for an operation between the specified types.
+ *
+ * @param left the type of the left operand
+ * @param right the type of the right operand
+ * @return the return type for an operation between the specified types
+ */
+ private static NumberType<?> returnType(NumberType<?> left, NumberType<?> right)
+ {
+ boolean isFloatingPoint = left.isFloatingPoint() || right.isFloatingPoint();
+ int size = Math.max(size(left), size(right));
+ return isFloatingPoint
+ ? floatPointType(size)
+ : integerType(size);
+ }
+
+ /**
+ * Returns the number of bytes used to represent a value of this type.
+ * @return the number of bytes used to represent a value of this type or {@code Integer.MAX} if the number of bytes
+ * is not limited.
+ */
+ private static int size(NumberType<?> type)
+ {
+ int size = type.valueLengthIfFixed();
+
+ if (size > 0)
+ return size;
+
+ // tinyint and smallint type are not fixed length types even if they should be.
+ // So we need to handle them in a special way.
+ if (type == ByteType.instance)
+ return 1;
+
+ if (type == ShortType.instance)
+ return 2;
+
+ if (type.isCounter())
+ return LongType.instance.valueLengthIfFixed();
+
+ return Integer.MAX_VALUE;
+ }
+
+ private static NumberType<?> floatPointType(int size)
+ {
+ switch (size)
+ {
+ case 4: return FloatType.instance;
+ case 8: return DoubleType.instance;
+ default: return DecimalType.instance;
+ }
+ }
+
+ private static NumberType<?> integerType(int size)
+ {
+ switch (size)
+ {
+ case 1: return ByteType.instance;
+ case 2: return ShortType.instance;
+ case 4: return Int32Type.instance;
+ case 8: return LongType.instance;
+ default: return IntegerType.instance;
+ }
+ }
+
+ /**
+ * The class must not be instantiated.
+ */
+ private OperationFcts()
+ {
+ }
+
+ /**
+ * Base class for functions that execute operations.
+ */
+ private static abstract class OperationFunction extends NativeScalarFunction
+ {
+ private final OPERATION operation;
+
+ public OperationFunction(AbstractType<?> returnType,
+ AbstractType<?> left,
+ OPERATION operation,
+ AbstractType<?> right)
+ {
+ super(operation.functionName, returnType, left, right);
+ this.operation = operation;
+ }
+
+ @Override
+ public final String columnName(List<String> columnNames)
+ {
+ return String.format("%s %s %s", columnNames.get(0), getOperator(), columnNames.get(1));
+ }
+
+ public final ByteBuffer execute(ProtocolVersion protocolVersion, List<ByteBuffer> parameters)
+ {
+ ByteBuffer left = parameters.get(0);
+ ByteBuffer right = parameters.get(1);
+ if (left == null || !left.hasRemaining() || right == null || !right.hasRemaining())
+ return null;
+
+ try
+ {
+ return doExecute(left, operation, right);
+ }
+ catch (Exception e)
+ {
+ throw OperationExecutionException.create(getOperator(), argTypes, e);
+ }
+ }
+
+ protected abstract ByteBuffer doExecute(ByteBuffer left, OPERATION operation, ByteBuffer right);
+
+ /**
+ * Returns the operator symbol.
+ * @return the operator symbol
+ */
+ private final char getOperator()
+ {
+ return operation.symbol;
+ }
+ }
+
+ /**
+ * Function that execute operations on numbers.
+ */
+ private static class NumericOperationFunction extends OperationFunction
+ {
+ public NumericOperationFunction(NumberType<?> returnType,
+ NumberType<?> left,
+ OPERATION operation,
+ NumberType<?> right)
+ {
+ super(returnType, left, operation, right);
+ }
+
+ @Override
+ protected ByteBuffer doExecute(ByteBuffer left, OPERATION operation, ByteBuffer right)
+ {
+ NumberType<?> leftType = (NumberType<?>) argTypes().get(0);
+ NumberType<?> rightType = (NumberType<?>) argTypes().get(1);
+ NumberType<?> resultType = (NumberType<?>) returnType();
+
+ return operation.executeOnNumerics(resultType, leftType, left, rightType, right);
+ }
+ }
+
+ /**
+ * Function that execute operations on temporals (timestamp, date, ...).
+ */
+ private static class TemporalOperationFunction extends OperationFunction
+ {
+ public TemporalOperationFunction(TemporalType<?> type,
+ OPERATION operation)
+ {
+ super(type, type, operation, DurationType.instance);
+ }
+
+ @Override
+ protected ByteBuffer doExecute(ByteBuffer left, OPERATION operation, ByteBuffer right)
+ {
+ TemporalType<?> resultType = (TemporalType<?>) returnType();
+ return operation.executeOnTemporals(resultType, left, right);
+ }
+ }
+
+ /**
+ * Function that negate a number.
+ */
+ private static class NumericNegationFunction extends NativeScalarFunction
+ {
+ public NumericNegationFunction(NumberType<?> inputType)
+ {
+ super(NEGATION_FUNCTION_NAME, inputType, inputType);
+ }
+
+ @Override
+ public final String columnName(List<String> columnNames)
+ {
+ return String.format("-%s", columnNames.get(0));
+ }
+
+ public final ByteBuffer execute(ProtocolVersion protocolVersion, List<ByteBuffer> parameters)
+ {
+ ByteBuffer input = parameters.get(0);
+ if (input == null)
+ return null;
+
+ NumberType<?> inputType = (NumberType<?>) argTypes().get(0);
+
+ return inputType.negate(input);
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/cql3/functions/ScriptBasedUDFunction.java b/src/java/org/apache/cassandra/cql3/functions/ScriptBasedUDFunction.java
index 41035a4..d7e5eb8 100644
--- a/src/java/org/apache/cassandra/cql3/functions/ScriptBasedUDFunction.java
+++ b/src/java/org/apache/cassandra/cql3/functions/ScriptBasedUDFunction.java
@@ -73,6 +73,14 @@
"jdk.internal.org.objectweb.asm.commons",
"jdk.nashorn.internal.runtime",
"jdk.nashorn.internal.runtime.linker",
+ // Nashorn / Java 11
+ "java.lang.ref",
+ "java.io",
+ "java.util.function",
+ "jdk.dynalink.linker",
+ "jdk.internal.org.objectweb.asm",
+ "jdk.internal.reflect",
+ "jdk.nashorn.internal.scripts",
// following required by Java Driver
"java.math",
"java.nio",
@@ -81,8 +89,9 @@
"com.google.common.collect",
"com.google.common.reflect",
// following required by UDF
- "com.datastax.driver.core",
- "com.datastax.driver.core.utils"
+ "org.apache.cassandra.cql3.functions.types",
+ "org.apache.cassandra.cql3.functions.types.exceptions",
+ "org.apache.cassandra.cql3.functions.types.utils"
};
// use a JVM standard ExecutorService as DebuggableThreadPoolExecutor references internal
diff --git a/src/java/org/apache/cassandra/cql3/functions/TimeFcts.java b/src/java/org/apache/cassandra/cql3/functions/TimeFcts.java
index e682dcd..f029e59 100644
--- a/src/java/org/apache/cassandra/cql3/functions/TimeFcts.java
+++ b/src/java/org/apache/cassandra/cql3/functions/TimeFcts.java
@@ -19,7 +19,6 @@
import java.nio.ByteBuffer;
import java.util.Collection;
-import java.util.Date;
import java.util.List;
import com.google.common.collect.ImmutableList;
@@ -37,26 +36,34 @@
public static Collection<Function> all()
{
- return ImmutableList.of(nowFct,
+ return ImmutableList.of(now("now", TimeUUIDType.instance),
+ now("currenttimeuuid", TimeUUIDType.instance),
+ now("currenttimestamp", TimestampType.instance),
+ now("currentdate", SimpleDateType.instance),
+ now("currenttime", TimeType.instance),
minTimeuuidFct,
maxTimeuuidFct,
dateOfFct,
unixTimestampOfFct,
- timeUuidtoDate,
- timeUuidToTimestamp,
- timeUuidToUnixTimestamp,
- timestampToUnixTimestamp,
- timestampToDate,
- dateToUnixTimestamp,
- dateToTimestamp);
+ toDate(TimeUUIDType.instance),
+ toTimestamp(TimeUUIDType.instance),
+ toUnixTimestamp(TimeUUIDType.instance),
+ toUnixTimestamp(TimestampType.instance),
+ toDate(TimestampType.instance),
+ toUnixTimestamp(SimpleDateType.instance),
+ toTimestamp(SimpleDateType.instance));
}
- public static final Function nowFct = new NativeScalarFunction("now", TimeUUIDType.instance)
+ public static final Function now(final String name, final TemporalType<?> type)
{
- public ByteBuffer execute(ProtocolVersion protocolVersion, List<ByteBuffer> parameters)
+ return new NativeScalarFunction(name, type)
{
- return ByteBuffer.wrap(UUIDGen.getTimeUUIDBytes());
- }
+ @Override
+ public ByteBuffer execute(ProtocolVersion protocolVersion, List<ByteBuffer> parameters)
+ {
+ return type.now();
+ }
+ };
};
public static final Function minTimeuuidFct = new NativeScalarFunction("mintimeuuid", TimeUUIDType.instance, TimestampType.instance)
@@ -134,114 +141,66 @@
}
};
- /**
- * Function that convert a value of <code>TIMEUUID</code> into a value of type <code>DATE</code>.
- */
- public static final NativeScalarFunction timeUuidtoDate = new NativeScalarFunction("todate", SimpleDateType.instance, TimeUUIDType.instance)
- {
- public ByteBuffer execute(ProtocolVersion protocolVersion, List<ByteBuffer> parameters)
- {
- ByteBuffer bb = parameters.get(0);
- if (bb == null)
- return null;
+ /**
+ * Creates a function that convert a value of the specified type into a <code>DATE</code>.
+ * @param type the temporal type
+ * @return a function that convert a value of the specified type into a <code>DATE</code>.
+ */
+ public static final NativeScalarFunction toDate(final TemporalType<?> type)
+ {
+ return new NativeScalarFunction("todate", SimpleDateType.instance, type)
+ {
+ public ByteBuffer execute(ProtocolVersion protocolVersion, List<ByteBuffer> parameters)
+ {
+ ByteBuffer bb = parameters.get(0);
+ if (bb == null || !bb.hasRemaining())
+ return null;
- long timeInMillis = UUIDGen.unixTimestamp(UUIDGen.getUUID(bb));
- return SimpleDateType.instance.fromTimeInMillis(timeInMillis);
- }
- };
-
- /**
- * Function that convert a value of type <code>TIMEUUID</code> into a value of type <code>TIMESTAMP</code>.
- */
- public static final NativeScalarFunction timeUuidToTimestamp = new NativeScalarFunction("totimestamp", TimestampType.instance, TimeUUIDType.instance)
- {
- public ByteBuffer execute(ProtocolVersion protocolVersion, List<ByteBuffer> parameters)
- {
- ByteBuffer bb = parameters.get(0);
- if (bb == null)
- return null;
-
- long timeInMillis = UUIDGen.unixTimestamp(UUIDGen.getUUID(bb));
- return TimestampType.instance.fromTimeInMillis(timeInMillis);
- }
- };
-
- /**
- * Function that convert a value of type <code>TIMEUUID</code> into an UNIX timestamp.
- */
- public static final NativeScalarFunction timeUuidToUnixTimestamp = new NativeScalarFunction("tounixtimestamp", LongType.instance, TimeUUIDType.instance)
- {
- public ByteBuffer execute(ProtocolVersion protocolVersion, List<ByteBuffer> parameters)
- {
- ByteBuffer bb = parameters.get(0);
- if (bb == null)
- return null;
-
- return ByteBufferUtil.bytes(UUIDGen.unixTimestamp(UUIDGen.getUUID(bb)));
- }
- };
-
- /**
- * Function that convert a value of type <code>TIMESTAMP</code> into an UNIX timestamp.
- */
- public static final NativeScalarFunction timestampToUnixTimestamp = new NativeScalarFunction("tounixtimestamp", LongType.instance, TimestampType.instance)
- {
- public ByteBuffer execute(ProtocolVersion protocolVersion, List<ByteBuffer> parameters)
- {
- ByteBuffer bb = parameters.get(0);
- if (bb == null)
- return null;
-
- Date date = TimestampType.instance.compose(bb);
- return date == null ? null : ByteBufferUtil.bytes(date.getTime());
- }
- };
+ long millis = type.toTimeInMillis(bb);
+ return SimpleDateType.instance.fromTimeInMillis(millis);
+ }
+ };
+ }
/**
- * Function that convert a value of type <code>TIMESTAMP</code> into a <code>DATE</code>.
+ * Creates a function that convert a value of the specified type into a <code>TIMESTAMP</code>.
+ * @param type the temporal type
+ * @return a function that convert a value of the specified type into a <code>TIMESTAMP</code>.
*/
- public static final NativeScalarFunction timestampToDate = new NativeScalarFunction("todate", SimpleDateType.instance, TimestampType.instance)
+ public static final NativeScalarFunction toTimestamp(final TemporalType<?> type)
{
- public ByteBuffer execute(ProtocolVersion protocolVersion, List<ByteBuffer> parameters)
+ return new NativeScalarFunction("totimestamp", TimestampType.instance, type)
{
- ByteBuffer bb = parameters.get(0);
- if (bb == null)
- return null;
+ public ByteBuffer execute(ProtocolVersion protocolVersion, List<ByteBuffer> parameters)
+ {
+ ByteBuffer bb = parameters.get(0);
+ if (bb == null || !bb.hasRemaining())
+ return null;
- Date date = TimestampType.instance.compose(bb);
- return date == null ? null : SimpleDateType.instance.fromTimeInMillis(date.getTime());
- }
- };
+ long millis = type.toTimeInMillis(bb);
+ return TimestampType.instance.fromTimeInMillis(millis);
+ }
+ };
+ }
- /**
- * Function that convert a value of type <code>TIMESTAMP</code> into a <code>DATE</code>.
- */
- public static final NativeScalarFunction dateToTimestamp = new NativeScalarFunction("totimestamp", TimestampType.instance, SimpleDateType.instance)
- {
- public ByteBuffer execute(ProtocolVersion protocolVersion, List<ByteBuffer> parameters)
- {
- ByteBuffer bb = parameters.get(0);
- if (bb == null)
- return null;
+ /**
+ * Creates a function that convert a value of the specified type into an UNIX timestamp.
+ * @param type the temporal type
+ * @return a function that convert a value of the specified type into an UNIX timestamp.
+ */
+ public static final NativeScalarFunction toUnixTimestamp(final TemporalType<?> type)
+ {
+ return new NativeScalarFunction("tounixtimestamp", LongType.instance, type)
+ {
+ public ByteBuffer execute(ProtocolVersion protocolVersion, List<ByteBuffer> parameters)
+ {
+ ByteBuffer bb = parameters.get(0);
+ if (bb == null || !bb.hasRemaining())
+ return null;
- long millis = SimpleDateType.instance.toTimeInMillis(bb);
- return TimestampType.instance.fromTimeInMillis(millis);
- }
- };
-
- /**
- * Function that convert a value of type <code>DATE</code> into an UNIX timestamp.
- */
- public static final NativeScalarFunction dateToUnixTimestamp = new NativeScalarFunction("tounixtimestamp", LongType.instance, SimpleDateType.instance)
- {
- public ByteBuffer execute(ProtocolVersion protocolVersion, List<ByteBuffer> parameters)
- {
- ByteBuffer bb = parameters.get(0);
- if (bb == null)
- return null;
-
- return ByteBufferUtil.bytes(SimpleDateType.instance.toTimeInMillis(bb));
- }
- };
+ return ByteBufferUtil.bytes(type.toTimeInMillis(bb));
+ }
+ };
+ }
}
diff --git a/src/java/org/apache/cassandra/cql3/functions/TokenFct.java b/src/java/org/apache/cassandra/cql3/functions/TokenFct.java
index 1907641..e93084f 100644
--- a/src/java/org/apache/cassandra/cql3/functions/TokenFct.java
+++ b/src/java/org/apache/cassandra/cql3/functions/TokenFct.java
@@ -20,8 +20,8 @@
import java.nio.ByteBuffer;
import java.util.List;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.db.CBuilder;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.exceptions.InvalidRequestException;
@@ -29,26 +29,26 @@
public class TokenFct extends NativeScalarFunction
{
- private final CFMetaData cfm;
+ private final TableMetadata metadata;
- public TokenFct(CFMetaData cfm)
+ public TokenFct(TableMetadata metadata)
{
- super("token", cfm.partitioner.getTokenValidator(), getKeyTypes(cfm));
- this.cfm = cfm;
+ super("token", metadata.partitioner.getTokenValidator(), getKeyTypes(metadata));
+ this.metadata = metadata;
}
- private static AbstractType[] getKeyTypes(CFMetaData cfm)
+ private static AbstractType[] getKeyTypes(TableMetadata metadata)
{
- AbstractType[] types = new AbstractType[cfm.partitionKeyColumns().size()];
+ AbstractType[] types = new AbstractType[metadata.partitionKeyColumns().size()];
int i = 0;
- for (ColumnDefinition def : cfm.partitionKeyColumns())
+ for (ColumnMetadata def : metadata.partitionKeyColumns())
types[i++] = def.type;
return types;
}
public ByteBuffer execute(ProtocolVersion protocolVersion, List<ByteBuffer> parameters) throws InvalidRequestException
{
- CBuilder builder = CBuilder.create(cfm.getKeyValidatorAsClusteringComparator());
+ CBuilder builder = CBuilder.create(metadata.partitionKeyAsClusteringComparator());
for (int i = 0; i < parameters.size(); i++)
{
ByteBuffer bb = parameters.get(i);
@@ -56,6 +56,6 @@
return null;
builder.add(bb);
}
- return cfm.partitioner.getTokenFactory().toByteArray(cfm.partitioner.getToken(CFMetaData.serializePartitionKey(builder.build())));
+ return metadata.partitioner.getTokenFactory().toByteArray(metadata.partitioner.getToken(builder.build().serializeAsPartitionKey()));
}
}
diff --git a/src/java/org/apache/cassandra/cql3/functions/UDAggregate.java b/src/java/org/apache/cassandra/cql3/functions/UDAggregate.java
index 1a3174c..db5859f 100644
--- a/src/java/org/apache/cassandra/cql3/functions/UDAggregate.java
+++ b/src/java/org/apache/cassandra/cql3/functions/UDAggregate.java
@@ -21,20 +21,29 @@
import java.util.*;
import com.google.common.base.Objects;
+import com.google.common.collect.Lists;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.datastax.driver.core.TypeCodec;
+import org.apache.cassandra.cql3.CqlBuilder;
+import org.apache.cassandra.cql3.SchemaElement;
+import org.apache.cassandra.cql3.functions.types.TypeCodec;
import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.UserType;
+import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.schema.Difference;
import org.apache.cassandra.schema.Functions;
import org.apache.cassandra.tracing.Tracing;
import org.apache.cassandra.transport.ProtocolVersion;
+import static com.google.common.collect.Iterables.any;
+import static com.google.common.collect.Iterables.transform;
+
/**
* Base class for user-defined-aggregates.
*/
-public class UDAggregate extends AbstractFunction implements AggregateFunction
+public class UDAggregate extends AbstractFunction implements AggregateFunction, SchemaElement
{
protected static final Logger logger = LoggerFactory.getLogger(UDAggregate.class);
@@ -55,13 +64,13 @@
super(name, argTypes, returnType);
this.stateFunction = stateFunc;
this.finalFunction = finalFunc;
- this.stateType = stateFunc != null ? stateFunc.returnType() : null;
- this.stateTypeCodec = stateType != null ? UDHelper.codecFor(UDHelper.driverType(stateType)) : null;
- this.returnTypeCodec = returnType != null ? UDHelper.codecFor(UDHelper.driverType(returnType)) : null;
+ this.stateType = stateFunc.returnType();
+ this.stateTypeCodec = UDHelper.codecFor(UDHelper.driverType(stateType));
+ this.returnTypeCodec = UDHelper.codecFor(UDHelper.driverType(returnType));
this.initcond = initcond;
}
- public static UDAggregate create(Functions functions,
+ public static UDAggregate create(Collection<UDFunction> functions,
FunctionName name,
List<AbstractType<?>> argTypes,
AbstractType<?> returnType,
@@ -69,7 +78,6 @@
FunctionName finalFunc,
AbstractType<?> stateType,
ByteBuffer initcond)
- throws InvalidRequestException
{
List<AbstractType<?>> stateTypes = new ArrayList<>(argTypes.size() + 1);
stateTypes.add(stateType);
@@ -78,27 +86,17 @@
return new UDAggregate(name,
argTypes,
returnType,
- resolveScalar(functions, name, stateFunc, stateTypes),
- finalFunc != null ? resolveScalar(functions, name, finalFunc, finalTypes) : null,
+ findFunction(name, functions, stateFunc, stateTypes),
+ null == finalFunc ? null : findFunction(name, functions, finalFunc, finalTypes),
initcond);
}
- public static UDAggregate createBroken(FunctionName name,
- List<AbstractType<?>> argTypes,
- AbstractType<?> returnType,
- ByteBuffer initcond,
- InvalidRequestException reason)
+ private static UDFunction findFunction(FunctionName udaName, Collection<UDFunction> functions, FunctionName name, List<AbstractType<?>> arguments)
{
- return new UDAggregate(name, argTypes, returnType, null, null, initcond)
- {
- public Aggregate newAggregate() throws InvalidRequestException
- {
- throw new InvalidRequestException(String.format("Aggregate '%s' exists but hasn't been loaded successfully for the following reason: %s. "
- + "Please see the server log for more details",
- this,
- reason.getMessage()));
- }
- };
+ return functions.stream()
+ .filter(f -> f.name().equals(name) && Functions.typesMatch(f.argTypes(), arguments))
+ .findFirst()
+ .orElseThrow(() -> new ConfigurationException(String.format("Unable to find function %s referenced by UDA %s", name, udaName)));
}
public boolean hasReferenceTo(Function function)
@@ -107,16 +105,37 @@
}
@Override
+ public boolean referencesUserType(ByteBuffer name)
+ {
+ return any(argTypes(), t -> t.referencesUserType(name))
+ || returnType.referencesUserType(name)
+ || (null != stateType && stateType.referencesUserType(name))
+ || stateFunction.referencesUserType(name)
+ || (null != finalFunction && finalFunction.referencesUserType(name));
+ }
+
+ public UDAggregate withUpdatedUserType(Collection<UDFunction> udfs, UserType udt)
+ {
+ if (!referencesUserType(udt.name))
+ return this;
+
+ return new UDAggregate(name,
+ Lists.newArrayList(transform(argTypes, t -> t.withUpdatedUserType(udt))),
+ returnType.withUpdatedUserType(udt),
+ findFunction(name, udfs, stateFunction.name(), stateFunction.argTypes()),
+ null == finalFunction ? null : findFunction(name, udfs, finalFunction.name(), finalFunction.argTypes()),
+ initcond);
+ }
+
+ @Override
public void addFunctionsTo(List<Function> functions)
{
functions.add(this);
- if (stateFunction != null)
- {
- stateFunction.addFunctionsTo(functions);
- if (finalFunction != null)
- finalFunction.addFunctionsTo(functions);
- }
+ stateFunction.addFunctionsTo(functions);
+
+ if (finalFunction != null)
+ finalFunction.addFunctionsTo(functions);
}
public boolean isAggregate()
@@ -214,23 +233,6 @@
};
}
- private static ScalarFunction resolveScalar(Functions functions, FunctionName aName, FunctionName fName, List<AbstractType<?>> argTypes) throws InvalidRequestException
- {
- Optional<Function> fun = functions.find(fName, argTypes);
- if (!fun.isPresent())
- throw new InvalidRequestException(String.format("Referenced state function '%s %s' for aggregate '%s' does not exist",
- fName,
- Arrays.toString(UDHelper.driverTypes(argTypes)),
- aName));
-
- if (!(fun.get() instanceof ScalarFunction))
- throw new InvalidRequestException(String.format("Referenced state function '%s %s' for aggregate '%s' is not a scalar function",
- fName,
- Arrays.toString(UDHelper.driverTypes(argTypes)),
- aName));
- return (ScalarFunction) fun.get();
- }
-
@Override
public boolean equals(Object o)
{
@@ -238,13 +240,83 @@
return false;
UDAggregate that = (UDAggregate) o;
- return Objects.equal(name, that.name)
- && Functions.typesMatch(argTypes, that.argTypes)
- && Functions.typesMatch(returnType, that.returnType)
+ return equalsWithoutTypesAndFunctions(that)
+ && argTypes.equals(that.argTypes)
+ && returnType.equals(that.returnType)
&& Objects.equal(stateFunction, that.stateFunction)
&& Objects.equal(finalFunction, that.finalFunction)
- && ((stateType == that.stateType) || ((stateType != null) && stateType.equals(that.stateType, true))) // ignore freezing
- && Objects.equal(initcond, that.initcond);
+ && ((stateType == that.stateType) || ((stateType != null) && stateType.equals(that.stateType)));
+ }
+
+ private boolean equalsWithoutTypesAndFunctions(UDAggregate other)
+ {
+ return name.equals(other.name)
+ && argTypes.size() == other.argTypes.size()
+ && Objects.equal(initcond, other.initcond);
+ }
+
+ @Override
+ public Optional<Difference> compare(Function function)
+ {
+ if (!(function instanceof UDAggregate))
+ throw new IllegalArgumentException();
+
+ UDAggregate other = (UDAggregate) function;
+
+ if (!equalsWithoutTypesAndFunctions(other)
+ || ((null == finalFunction) != (null == other.finalFunction))
+ || ((null == stateType) != (null == other.stateType)))
+ return Optional.of(Difference.SHALLOW);
+
+ boolean differsDeeply = false;
+
+ if (null != finalFunction && !finalFunction.equals(other.finalFunction))
+ {
+ if (finalFunction.name().equals(other.finalFunction.name()))
+ differsDeeply = true;
+ else
+ return Optional.of(Difference.SHALLOW);
+ }
+
+ if (null != stateType && !stateType.equals(other.stateType))
+ {
+ if (stateType.asCQL3Type().toString().equals(other.stateType.asCQL3Type().toString()))
+ differsDeeply = true;
+ else
+ return Optional.of(Difference.SHALLOW);
+ }
+
+ if (!returnType.equals(other.returnType))
+ {
+ if (returnType.asCQL3Type().toString().equals(other.returnType.asCQL3Type().toString()))
+ differsDeeply = true;
+ else
+ return Optional.of(Difference.SHALLOW);
+ }
+
+ for (int i = 0; i < argTypes().size(); i++)
+ {
+ AbstractType<?> thisType = argTypes.get(i);
+ AbstractType<?> thatType = other.argTypes.get(i);
+
+ if (!thisType.equals(thatType))
+ {
+ if (thisType.asCQL3Type().toString().equals(thatType.asCQL3Type().toString()))
+ differsDeeply = true;
+ else
+ return Optional.of(Difference.SHALLOW);
+ }
+ }
+
+ if (!stateFunction.equals(other.stateFunction))
+ {
+ if (stateFunction.name().equals(other.stateFunction.name()))
+ differsDeeply = true;
+ else
+ return Optional.of(Difference.SHALLOW);
+ }
+
+ return differsDeeply ? Optional.of(Difference.DEEP) : Optional.empty();
}
@Override
@@ -252,4 +324,41 @@
{
return Objects.hashCode(name, Functions.typeHashCode(argTypes), Functions.typeHashCode(returnType), stateFunction, finalFunction, stateType, initcond);
}
+
+ @Override
+ public SchemaElementType elementType()
+ {
+ return SchemaElementType.AGGREGATE;
+ }
+
+ @Override
+ public String toCqlString(boolean withInternals)
+ {
+ CqlBuilder builder = new CqlBuilder();
+ builder.append("CREATE AGGREGATE ")
+ .append(name())
+ .append('(')
+ .appendWithSeparators(argTypes, (b, t) -> b.append(toCqlString(t)), ", ")
+ .append(')')
+ .newLine()
+ .increaseIndent()
+ .append("SFUNC ")
+ .append(stateFunction().name().name)
+ .newLine()
+ .append("STYPE ")
+ .append(toCqlString(stateType()));
+
+ if (finalFunction() != null)
+ builder.newLine()
+ .append("FINALFUNC ")
+ .append(finalFunction().name().name);
+
+ if (initialCondition() != null)
+ builder.newLine()
+ .append("INITCOND ")
+ .append(stateType().asCQL3Type().toCQLLiteral(initialCondition(), ProtocolVersion.CURRENT));
+
+ return builder.append(";")
+ .toString();
+ }
}
diff --git a/src/java/org/apache/cassandra/cql3/functions/UDFByteCodeVerifier.java b/src/java/org/apache/cassandra/cql3/functions/UDFByteCodeVerifier.java
index 234aed9..e3b461c 100644
--- a/src/java/org/apache/cassandra/cql3/functions/UDFByteCodeVerifier.java
+++ b/src/java/org/apache/cassandra/cql3/functions/UDFByteCodeVerifier.java
@@ -47,7 +47,7 @@
public static final String JAVA_UDF_NAME = JavaUDF.class.getName().replace('.', '/');
public static final String OBJECT_NAME = Object.class.getName().replace('.', '/');
- public static final String CTOR_SIG = "(Lcom/datastax/driver/core/TypeCodec;[Lcom/datastax/driver/core/TypeCodec;Lorg/apache/cassandra/cql3/functions/UDFContext;)V";
+ public static final String CTOR_SIG = "(Lorg/apache/cassandra/cql3/functions/types/TypeCodec;[Lorg/apache/cassandra/cql3/functions/types/TypeCodec;Lorg/apache/cassandra/cql3/functions/UDFContext;)V";
private final Set<String> disallowedClasses = new HashSet<>();
private final Multimap<String, String> disallowedMethodCalls = HashMultimap.create();
@@ -84,7 +84,7 @@
{
String clsNameSl = clsName.replace('.', '/');
Set<String> errors = new TreeSet<>(); // it's a TreeSet for unit tests
- ClassVisitor classVisitor = new ClassVisitor(Opcodes.ASM5)
+ ClassVisitor classVisitor = new ClassVisitor(Opcodes.ASM7)
{
public FieldVisitor visitField(int access, String name, String desc, String signature, Object value)
{
diff --git a/src/java/org/apache/cassandra/cql3/functions/UDFContext.java b/src/java/org/apache/cassandra/cql3/functions/UDFContext.java
index 4465aec..bb298ef 100644
--- a/src/java/org/apache/cassandra/cql3/functions/UDFContext.java
+++ b/src/java/org/apache/cassandra/cql3/functions/UDFContext.java
@@ -18,8 +18,8 @@
package org.apache.cassandra.cql3.functions;
-import com.datastax.driver.core.TupleValue;
-import com.datastax.driver.core.UDTValue;
+import org.apache.cassandra.cql3.functions.types.TupleValue;
+import org.apache.cassandra.cql3.functions.types.UDTValue;
/**
* Provides context information for a particular user defined function.
diff --git a/src/java/org/apache/cassandra/cql3/functions/UDFContextImpl.java b/src/java/org/apache/cassandra/cql3/functions/UDFContextImpl.java
index 00625cd..d4bdf20 100644
--- a/src/java/org/apache/cassandra/cql3/functions/UDFContextImpl.java
+++ b/src/java/org/apache/cassandra/cql3/functions/UDFContextImpl.java
@@ -23,16 +23,11 @@
import java.util.Map;
import java.util.Optional;
-import com.datastax.driver.core.DataType;
-import com.datastax.driver.core.TupleType;
-import com.datastax.driver.core.TupleValue;
-import com.datastax.driver.core.TypeCodec;
-import com.datastax.driver.core.UDTValue;
-import com.datastax.driver.core.UserType;
import org.apache.cassandra.cql3.ColumnIdentifier;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.schema.CQLTypeParser;
import org.apache.cassandra.schema.KeyspaceMetadata;
+import org.apache.cassandra.cql3.functions.types.*;
import org.apache.cassandra.utils.ByteBufferUtil;
/**
diff --git a/src/java/org/apache/cassandra/cql3/functions/UDFunction.java b/src/java/org/apache/cassandra/cql3/functions/UDFunction.java
index 6928a06..bceb085 100644
--- a/src/java/org/apache/cassandra/cql3/functions/UDFunction.java
+++ b/src/java/org/apache/cassandra/cql3/functions/UDFunction.java
@@ -27,6 +27,7 @@
import java.util.Enumeration;
import java.util.HashSet;
import java.util.List;
+import java.util.Optional;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
@@ -37,32 +38,34 @@
import java.util.concurrent.TimeoutException;
import com.google.common.base.Objects;
+import com.google.common.collect.Lists;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.datastax.driver.core.DataType;
-import com.datastax.driver.core.TypeCodec;
-import com.datastax.driver.core.UserType;
import org.apache.cassandra.config.Config;
import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.config.Schema;
import org.apache.cassandra.cql3.ColumnIdentifier;
+import org.apache.cassandra.cql3.CqlBuilder;
+import org.apache.cassandra.cql3.SchemaElement;
+import org.apache.cassandra.cql3.functions.types.DataType;
+import org.apache.cassandra.cql3.functions.types.TypeCodec;
import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.UserType;
import org.apache.cassandra.exceptions.FunctionExecutionException;
import org.apache.cassandra.exceptions.InvalidRequestException;
-import org.apache.cassandra.schema.Functions;
-import org.apache.cassandra.schema.KeyspaceMetadata;
+import org.apache.cassandra.schema.*;
import org.apache.cassandra.service.ClientWarn;
-import org.apache.cassandra.service.MigrationManager;
import org.apache.cassandra.tracing.Tracing;
import org.apache.cassandra.transport.ProtocolVersion;
-import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.JVMStabilityInspector;
+import static com.google.common.collect.Iterables.any;
+import static com.google.common.collect.Iterables.transform;
+
/**
* Base class for User Defined Functions.
*/
-public abstract class UDFunction extends AbstractFunction implements ScalarFunction
+public abstract class UDFunction extends AbstractFunction implements ScalarFunction, SchemaElement
{
protected static final Logger logger = LoggerFactory.getLogger(UDFunction.class);
@@ -94,7 +97,6 @@
//
private static final String[] allowedPatterns =
{
- "com/datastax/driver/core/",
"com/google/common/reflect/TypeToken",
"java/io/IOException.class",
"java/io/Serializable.class",
@@ -111,6 +113,7 @@
"java/text/",
"java/time/",
"java/util/",
+ "org/apache/cassandra/cql3/functions/types/",
"org/apache/cassandra/cql3/functions/JavaUDF.class",
"org/apache/cassandra/cql3/functions/UDFContext.class",
"org/apache/cassandra/exceptions/",
@@ -212,11 +215,29 @@
this.argCodecs = UDHelper.codecsFor(argDataTypes);
this.returnCodec = UDHelper.codecFor(returnDataType);
this.calledOnNullInput = calledOnNullInput;
- KeyspaceMetadata keyspaceMetadata = Schema.instance.getKSMetaData(name.keyspace);
+ KeyspaceMetadata keyspaceMetadata = Schema.instance.getKeyspaceMetadata(name.keyspace);
this.udfContext = new UDFContextImpl(argNames, argCodecs, returnCodec,
keyspaceMetadata);
}
+ public static UDFunction tryCreate(FunctionName name,
+ List<ColumnIdentifier> argNames,
+ List<AbstractType<?>> argTypes,
+ AbstractType<?> returnType,
+ boolean calledOnNullInput,
+ String language,
+ String body)
+ {
+ try
+ {
+ return create(name, argNames, argTypes, returnType, calledOnNullInput, language, body);
+ }
+ catch (InvalidRequestException e)
+ {
+ return createBrokenFunction(name, argNames, argTypes, returnType, calledOnNullInput, language, body, e);
+ }
+ }
+
public static UDFunction create(FunctionName name,
List<ColumnIdentifier> argNames,
List<AbstractType<?>> argTypes,
@@ -225,7 +246,7 @@
String language,
String body)
{
- UDFunction.assertUdfsEnabled(language);
+ assertUdfsEnabled(language);
switch (language)
{
@@ -281,6 +302,48 @@
};
}
+ @Override
+ public SchemaElementType elementType()
+ {
+ return SchemaElementType.FUNCTION;
+ }
+
+ @Override
+ public String toCqlString(boolean withInternals)
+ {
+ CqlBuilder builder = new CqlBuilder();
+ builder.append("CREATE FUNCTION ")
+ .append(name())
+ .append("(");
+
+ for (int i = 0, m = argNames().size(); i < m; i++)
+ {
+ if (i > 0)
+ builder.append(", ");
+ builder.append(argNames().get(i))
+ .append(' ')
+ .append(toCqlString(argTypes().get(i)));
+ }
+
+ builder.append(')')
+ .newLine()
+ .increaseIndent()
+ .append(isCalledOnNullInput() ? "CALLED" : "RETURNS NULL")
+ .append(" ON NULL INPUT")
+ .newLine()
+ .append("RETURNS ")
+ .append(toCqlString(returnType()))
+ .newLine()
+ .append("LANGUAGE ")
+ .append(language())
+ .newLine()
+ .append("AS $$")
+ .append(body())
+ .append("$$;");
+
+ return builder.toString();
+ }
+
public final ByteBuffer execute(ProtocolVersion protocolVersion, List<ByteBuffer> parameters)
{
assertUdfsEnabled(language);
@@ -402,7 +465,7 @@
}
/**
- * Like {@link #executeAsync(int, List)} but the first parameter is already in non-serialized form.
+ * Like {@link #executeAsync(ProtocolVersion, List)} but the first parameter is already in non-serialized form.
* Remaining parameters (2nd paramters and all others) are in {@code parameters}.
* This is used to prevent superfluous (de)serialization of the state of aggregates.
* Means: scalar functions of aggregates are called using this variant.
@@ -585,18 +648,83 @@
}
@Override
+ public boolean referencesUserType(ByteBuffer name)
+ {
+ return any(argTypes(), t -> t.referencesUserType(name)) || returnType.referencesUserType(name);
+ }
+
+ public UDFunction withUpdatedUserType(UserType udt)
+ {
+ if (!referencesUserType(udt.name))
+ return this;
+
+ return tryCreate(name,
+ argNames,
+ Lists.newArrayList(transform(argTypes, t -> t.withUpdatedUserType(udt))),
+ returnType.withUpdatedUserType(udt),
+ calledOnNullInput,
+ language,
+ body);
+ }
+
+ @Override
public boolean equals(Object o)
{
if (!(o instanceof UDFunction))
return false;
UDFunction that = (UDFunction)o;
- return Objects.equal(name, that.name)
- && Objects.equal(argNames, that.argNames)
- && Functions.typesMatch(argTypes, that.argTypes)
- && Functions.typesMatch(returnType, that.returnType)
- && Objects.equal(language, that.language)
- && Objects.equal(body, that.body);
+ return equalsWithoutTypes(that)
+ && argTypes.equals(that.argTypes)
+ && returnType.equals(that.returnType);
+ }
+
+ private boolean equalsWithoutTypes(UDFunction other)
+ {
+ return name.equals(other.name)
+ && argTypes.size() == other.argTypes.size()
+ && argNames.equals(other.argNames)
+ && body.equals(other.body)
+ && language.equals(other.language)
+ && calledOnNullInput == other.calledOnNullInput;
+ }
+
+ @Override
+ public Optional<Difference> compare(Function function)
+ {
+ if (!(function instanceof UDFunction))
+ throw new IllegalArgumentException();
+
+ UDFunction other = (UDFunction) function;
+
+ if (!equalsWithoutTypes(other))
+ return Optional.of(Difference.SHALLOW);
+
+ boolean typesDifferDeeply = false;
+
+ if (!returnType.equals(other.returnType))
+ {
+ if (returnType.asCQL3Type().toString().equals(other.returnType.asCQL3Type().toString()))
+ typesDifferDeeply = true;
+ else
+ return Optional.of(Difference.SHALLOW);
+ }
+
+ for (int i = 0; i < argTypes().size(); i++)
+ {
+ AbstractType<?> thisType = argTypes.get(i);
+ AbstractType<?> thatType = other.argTypes.get(i);
+
+ if (!thisType.equals(thatType))
+ {
+ if (thisType.asCQL3Type().toString().equals(thatType.asCQL3Type().toString()))
+ typesDifferDeeply = true;
+ else
+ return Optional.of(Difference.SHALLOW);
+ }
+ }
+
+ return typesDifferDeeply ? Optional.of(Difference.DEEP) : Optional.empty();
}
@Override
@@ -605,37 +733,6 @@
return Objects.hashCode(name, Functions.typeHashCode(argTypes), Functions.typeHashCode(returnType), returnType, language, body);
}
- public void userTypeUpdated(String ksName, String typeName)
- {
- boolean updated = false;
-
- for (int i = 0; i < argCodecs.length; i++)
- {
- DataType dataType = argCodecs[i].getCqlType();
- if (dataType instanceof UserType)
- {
- UserType userType = (UserType) dataType;
- if (userType.getKeyspace().equals(ksName) && userType.getTypeName().equals(typeName))
- {
- KeyspaceMetadata ksm = Schema.instance.getKSMetaData(ksName);
- assert ksm != null;
-
- org.apache.cassandra.db.marshal.UserType ut = ksm.types.get(ByteBufferUtil.bytes(typeName)).get();
-
- DataType newUserType = UDHelper.driverType(ut);
- argCodecs[i] = UDHelper.codecFor(newUserType);
-
- argTypes.set(i, ut);
-
- updated = true;
- }
- }
- }
-
- if (updated)
- MigrationManager.announceNewFunction(this, true);
- }
-
private static class UDFClassLoader extends ClassLoader
{
// insecureClassLoader is the C* class loader
diff --git a/src/java/org/apache/cassandra/cql3/functions/UDHelper.java b/src/java/org/apache/cassandra/cql3/functions/UDHelper.java
index ddeae41..8c145d9 100644
--- a/src/java/org/apache/cassandra/cql3/functions/UDHelper.java
+++ b/src/java/org/apache/cassandra/cql3/functions/UDHelper.java
@@ -17,19 +17,14 @@
*/
package org.apache.cassandra.cql3.functions;
-import java.lang.invoke.MethodHandle;
-import java.lang.invoke.MethodHandles;
-import java.lang.reflect.Method;
import java.nio.ByteBuffer;
import java.util.List;
import com.google.common.reflect.TypeToken;
-import com.datastax.driver.core.CodecRegistry;
-import com.datastax.driver.core.DataType;
-import com.datastax.driver.core.TypeCodec;
-import com.datastax.driver.core.exceptions.InvalidTypeException;
import org.apache.cassandra.cql3.CQL3Type;
+import org.apache.cassandra.cql3.functions.types.*;
+import org.apache.cassandra.cql3.functions.types.exceptions.InvalidTypeException;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.transport.ProtocolVersion;
@@ -38,24 +33,7 @@
*/
public final class UDHelper
{
- // TODO make these c'tors and methods public in Java-Driver - see https://datastax-oss.atlassian.net/browse/JAVA-502
- private static final MethodHandle methodParseOne;
- private static final CodecRegistry codecRegistry;
- static
- {
- try
- {
- Class<?> cls = Class.forName("com.datastax.driver.core.DataTypeClassNameParser");
- Method m = cls.getDeclaredMethod("parseOne", String.class, com.datastax.driver.core.ProtocolVersion.class, CodecRegistry.class);
- m.setAccessible(true);
- methodParseOne = MethodHandles.lookup().unreflect(m);
- codecRegistry = new CodecRegistry();
- }
- catch (Exception e)
- {
- throw new RuntimeException(e);
- }
- }
+ private static final CodecRegistry codecRegistry = new CodecRegistry();
static TypeCodec<Object>[] codecsFor(DataType[] dataType)
{
@@ -71,7 +49,7 @@
}
/**
- * Construct an array containing the Java classes for the given Java Driver {@link com.datastax.driver.core.DataType}s.
+ * Construct an array containing the Java classes for the given {@link DataType}s.
*
* @param dataTypes array with UDF argument types
* @param calledOnNullInput whether to allow {@code null} as an argument value
@@ -108,11 +86,11 @@
}
/**
- * Construct an array containing the Java Driver {@link com.datastax.driver.core.DataType}s for the
+ * Construct an array containing the {@link DataType}s for the
* C* internal types.
*
* @param abstractTypes list with UDF argument types
- * @return array with argument types as {@link com.datastax.driver.core.DataType}
+ * @return array with argument types as {@link DataType}
*/
public static DataType[] driverTypes(List<AbstractType<?>> abstractTypes)
{
@@ -123,7 +101,7 @@
}
/**
- * Returns the Java Driver {@link com.datastax.driver.core.DataType} for the C* internal type.
+ * Returns the {@link DataType} for the C* internal type.
*/
public static DataType driverType(AbstractType abstractType)
{
@@ -134,34 +112,22 @@
public static DataType driverTypeFromAbstractType(String abstractTypeDef)
{
- try
- {
- return (DataType) methodParseOne.invoke(abstractTypeDef,
- com.datastax.driver.core.ProtocolVersion.fromInt(ProtocolVersion.CURRENT.asInt()),
- codecRegistry);
- }
- catch (RuntimeException | Error e)
- {
- // immediately rethrow these...
- throw e;
- }
- catch (Throwable e)
- {
- throw new RuntimeException("cannot parse driver type " + abstractTypeDef, e);
- }
+ return DataTypeClassNameParser.parseOne(abstractTypeDef,
+ ProtocolVersion.CURRENT,
+ codecRegistry);
}
public static Object deserialize(TypeCodec<?> codec, ProtocolVersion protocolVersion, ByteBuffer value)
{
- return codec.deserialize(value, com.datastax.driver.core.ProtocolVersion.fromInt(protocolVersion.asInt()));
+ return codec.deserialize(value, protocolVersion);
}
public static ByteBuffer serialize(TypeCodec<?> codec, ProtocolVersion protocolVersion, Object value)
{
if (!codec.getJavaType().getRawType().isAssignableFrom(value.getClass()))
- throw new InvalidTypeException("Invalid value for CQL type " + codec.getCqlType().getName().toString());
+ throw new InvalidTypeException("Invalid value for CQL type " + codec.getCqlType().getName());
- return ((TypeCodec)codec).serialize(value, com.datastax.driver.core.ProtocolVersion.fromInt(protocolVersion.asInt()));
+ return ((TypeCodec)codec).serialize(value, protocolVersion);
}
public static Class<?> asJavaClass(TypeCodec<?> codec)
diff --git a/src/java/org/apache/cassandra/cql3/functions/types/AbstractAddressableByIndexData.java b/src/java/org/apache/cassandra/cql3/functions/types/AbstractAddressableByIndexData.java
new file mode 100644
index 0000000..0e98c89
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/functions/types/AbstractAddressableByIndexData.java
@@ -0,0 +1,330 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.functions.types;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.net.InetAddress;
+import java.nio.ByteBuffer;
+import java.util.*;
+
+import com.google.common.reflect.TypeToken;
+
+import org.apache.cassandra.transport.ProtocolVersion;
+
+abstract class AbstractAddressableByIndexData<T extends SettableByIndexData<T>>
+extends AbstractGettableByIndexData implements SettableByIndexData<T>
+{
+
+ final ByteBuffer[] values;
+
+ AbstractAddressableByIndexData(ProtocolVersion protocolVersion, int size)
+ {
+ super(protocolVersion);
+ this.values = new ByteBuffer[size];
+ }
+
+ @SuppressWarnings("unchecked")
+ T setValue(int i, ByteBuffer value)
+ {
+ values[i] = value;
+ return (T) this;
+ }
+
+ @Override
+ protected ByteBuffer getValue(int i)
+ {
+ return values[i];
+ }
+
+ @Override
+ public T setBool(int i, boolean v)
+ {
+ TypeCodec<Boolean> codec = codecFor(i, Boolean.class);
+ ByteBuffer bb;
+ if (codec instanceof TypeCodec.PrimitiveBooleanCodec)
+ bb = ((TypeCodec.PrimitiveBooleanCodec) codec).serializeNoBoxing(v, protocolVersion);
+ else bb = codec.serialize(v, protocolVersion);
+ return setValue(i, bb);
+ }
+
+ @Override
+ public T setByte(int i, byte v)
+ {
+ TypeCodec<Byte> codec = codecFor(i, Byte.class);
+ ByteBuffer bb;
+ if (codec instanceof TypeCodec.PrimitiveByteCodec)
+ bb = ((TypeCodec.PrimitiveByteCodec) codec).serializeNoBoxing(v, protocolVersion);
+ else bb = codec.serialize(v, protocolVersion);
+ return setValue(i, bb);
+ }
+
+ @Override
+ public T setShort(int i, short v)
+ {
+ TypeCodec<Short> codec = codecFor(i, Short.class);
+ ByteBuffer bb;
+ if (codec instanceof TypeCodec.PrimitiveShortCodec)
+ bb = ((TypeCodec.PrimitiveShortCodec) codec).serializeNoBoxing(v, protocolVersion);
+ else bb = codec.serialize(v, protocolVersion);
+ return setValue(i, bb);
+ }
+
+ @Override
+ public T setInt(int i, int v)
+ {
+ TypeCodec<Integer> codec = codecFor(i, Integer.class);
+ ByteBuffer bb;
+ if (codec instanceof TypeCodec.PrimitiveIntCodec)
+ bb = ((TypeCodec.PrimitiveIntCodec) codec).serializeNoBoxing(v, protocolVersion);
+ else bb = codec.serialize(v, protocolVersion);
+ return setValue(i, bb);
+ }
+
+ @Override
+ public T setLong(int i, long v)
+ {
+ TypeCodec<Long> codec = codecFor(i, Long.class);
+ ByteBuffer bb;
+ if (codec instanceof TypeCodec.PrimitiveLongCodec)
+ bb = ((TypeCodec.PrimitiveLongCodec) codec).serializeNoBoxing(v, protocolVersion);
+ else bb = codec.serialize(v, protocolVersion);
+ return setValue(i, bb);
+ }
+
+ @Override
+ public T setTimestamp(int i, Date v)
+ {
+ return setValue(i, codecFor(i, Date.class).serialize(v, protocolVersion));
+ }
+
+ @Override
+ public T setDate(int i, LocalDate v)
+ {
+ return setValue(i, codecFor(i, LocalDate.class).serialize(v, protocolVersion));
+ }
+
+ @Override
+ public T setTime(int i, long v)
+ {
+ TypeCodec<Long> codec = codecFor(i, Long.class);
+ ByteBuffer bb;
+ if (codec instanceof TypeCodec.PrimitiveLongCodec)
+ bb = ((TypeCodec.PrimitiveLongCodec) codec).serializeNoBoxing(v, protocolVersion);
+ else bb = codec.serialize(v, protocolVersion);
+ return setValue(i, bb);
+ }
+
+ @Override
+ public T setFloat(int i, float v)
+ {
+ TypeCodec<Float> codec = codecFor(i, Float.class);
+ ByteBuffer bb;
+ if (codec instanceof TypeCodec.PrimitiveFloatCodec)
+ bb = ((TypeCodec.PrimitiveFloatCodec) codec).serializeNoBoxing(v, protocolVersion);
+ else bb = codec.serialize(v, protocolVersion);
+ return setValue(i, bb);
+ }
+
+ @Override
+ public T setDouble(int i, double v)
+ {
+ TypeCodec<Double> codec = codecFor(i, Double.class);
+ ByteBuffer bb;
+ if (codec instanceof TypeCodec.PrimitiveDoubleCodec)
+ bb = ((TypeCodec.PrimitiveDoubleCodec) codec).serializeNoBoxing(v, protocolVersion);
+ else bb = codec.serialize(v, protocolVersion);
+ return setValue(i, bb);
+ }
+
+ @Override
+ public T setString(int i, String v)
+ {
+ return setValue(i, codecFor(i, String.class).serialize(v, protocolVersion));
+ }
+
+ @Override
+ public T setBytes(int i, ByteBuffer v)
+ {
+ return setValue(i, codecFor(i, ByteBuffer.class).serialize(v, protocolVersion));
+ }
+
+ @Override
+ public T setBytesUnsafe(int i, ByteBuffer v)
+ {
+ return setValue(i, v == null ? null : v.duplicate());
+ }
+
+ @Override
+ public T setVarint(int i, BigInteger v)
+ {
+ return setValue(i, codecFor(i, BigInteger.class).serialize(v, protocolVersion));
+ }
+
+ @Override
+ public T setDecimal(int i, BigDecimal v)
+ {
+ return setValue(i, codecFor(i, BigDecimal.class).serialize(v, protocolVersion));
+ }
+
+ @Override
+ public T setUUID(int i, UUID v)
+ {
+ return setValue(i, codecFor(i, UUID.class).serialize(v, protocolVersion));
+ }
+
+ @Override
+ public T setInet(int i, InetAddress v)
+ {
+ return setValue(i, codecFor(i, InetAddress.class).serialize(v, protocolVersion));
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public <E> T setList(int i, List<E> v)
+ {
+ return setValue(i, codecFor(i).serialize(v, protocolVersion));
+ }
+
+ @Override
+ public <E> T setList(int i, List<E> v, Class<E> elementsClass)
+ {
+ return setValue(i, codecFor(i, TypeTokens.listOf(elementsClass)).serialize(v, protocolVersion));
+ }
+
+ @Override
+ public <E> T setList(int i, List<E> v, TypeToken<E> elementsType)
+ {
+ return setValue(i, codecFor(i, TypeTokens.listOf(elementsType)).serialize(v, protocolVersion));
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public <K, V> T setMap(int i, Map<K, V> v)
+ {
+ return setValue(i, codecFor(i).serialize(v, protocolVersion));
+ }
+
+ @Override
+ public <K, V> T setMap(int i, Map<K, V> v, Class<K> keysClass, Class<V> valuesClass)
+ {
+ return setValue(
+ i, codecFor(i, TypeTokens.mapOf(keysClass, valuesClass)).serialize(v, protocolVersion));
+ }
+
+ @Override
+ public <K, V> T setMap(int i, Map<K, V> v, TypeToken<K> keysType, TypeToken<V> valuesType)
+ {
+ return setValue(
+ i, codecFor(i, TypeTokens.mapOf(keysType, valuesType)).serialize(v, protocolVersion));
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public <E> T setSet(int i, Set<E> v)
+ {
+ return setValue(i, codecFor(i).serialize(v, protocolVersion));
+ }
+
+ @Override
+ public <E> T setSet(int i, Set<E> v, Class<E> elementsClass)
+ {
+ return setValue(i, codecFor(i, TypeTokens.setOf(elementsClass)).serialize(v, protocolVersion));
+ }
+
+ @Override
+ public <E> T setSet(int i, Set<E> v, TypeToken<E> elementsType)
+ {
+ return setValue(i, codecFor(i, TypeTokens.setOf(elementsType)).serialize(v, protocolVersion));
+ }
+
+ @Override
+ public T setUDTValue(int i, UDTValue v)
+ {
+ return setValue(i, codecFor(i, UDTValue.class).serialize(v, protocolVersion));
+ }
+
+ @Override
+ public T setTupleValue(int i, TupleValue v)
+ {
+ return setValue(i, codecFor(i, TupleValue.class).serialize(v, protocolVersion));
+ }
+
+ @Override
+ public <V> T set(int i, V v, Class<V> targetClass)
+ {
+ return set(i, v, codecFor(i, targetClass));
+ }
+
+ @Override
+ public <V> T set(int i, V v, TypeToken<V> targetType)
+ {
+ return set(i, v, codecFor(i, targetType));
+ }
+
+ @Override
+ public <V> T set(int i, V v, TypeCodec<V> codec)
+ {
+ checkType(i, codec.getCqlType().getName());
+ return setValue(i, codec.serialize(v, protocolVersion));
+ }
+
+ @Override
+ public T setToNull(int i)
+ {
+ return setValue(i, null);
+ }
+
+ @Override
+ public boolean equals(Object o)
+ {
+ if (!(o instanceof AbstractAddressableByIndexData)) return false;
+
+ AbstractAddressableByIndexData<?> that = (AbstractAddressableByIndexData<?>) o;
+ if (values.length != that.values.length) return false;
+
+ if (this.protocolVersion != that.protocolVersion) return false;
+
+ // Deserializing each value is slightly inefficient, but comparing
+ // the bytes could in theory be wrong (for varint for instance, 2 values
+ // can have different binary representation but be the same value due to
+ // leading zeros). So we don't take any risk.
+ for (int i = 0; i < values.length; i++)
+ {
+ DataType thisType = getType(i);
+ DataType thatType = that.getType(i);
+ if (!thisType.equals(thatType)) return false;
+
+ Object thisValue = this.codecFor(i).deserialize(this.values[i], this.protocolVersion);
+ Object thatValue = that.codecFor(i).deserialize(that.values[i], that.protocolVersion);
+ if (!Objects.equals(thisValue, thatValue)) return false;
+ }
+ return true;
+ }
+
+ @Override
+ public int hashCode()
+ {
+ // Same as equals
+ int hash = 31;
+ for (int i = 0; i < values.length; i++)
+ hash +=
+ values[i] == null ? 1 : codecFor(i).deserialize(values[i], protocolVersion).hashCode();
+ return hash;
+ }
+}
diff --git a/src/java/org/apache/cassandra/cql3/functions/types/AbstractData.java b/src/java/org/apache/cassandra/cql3/functions/types/AbstractData.java
new file mode 100644
index 0000000..adb7c0e
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/functions/types/AbstractData.java
@@ -0,0 +1,677 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.functions.types;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.net.InetAddress;
+import java.nio.ByteBuffer;
+import java.util.*;
+
+import com.google.common.reflect.TypeToken;
+
+import org.apache.cassandra.transport.ProtocolVersion;
+
+// We don't want to expose this one: it's less useful externally and it's a bit ugly to expose
+// anyway (but it's convenient).
+abstract class AbstractData<T extends SettableData<T>> extends AbstractGettableData
+implements SettableData<T>
+{
+
+ private final T wrapped;
+ final ByteBuffer[] values;
+
+ // Ugly, we could probably clean that: it is currently needed however because we sometimes
+ // want wrapped to be 'this' (UDTValue), and sometimes some other object (in BoundStatement).
+ @SuppressWarnings("unchecked")
+ protected AbstractData(ProtocolVersion protocolVersion, int size)
+ {
+ super(protocolVersion);
+ this.wrapped = (T) this;
+ this.values = new ByteBuffer[size];
+ }
+
+ protected AbstractData(ProtocolVersion protocolVersion, T wrapped, int size)
+ {
+ this(protocolVersion, wrapped, new ByteBuffer[size]);
+ }
+
+ protected AbstractData(ProtocolVersion protocolVersion, T wrapped, ByteBuffer[] values)
+ {
+ super(protocolVersion);
+ this.wrapped = wrapped;
+ this.values = values;
+ }
+
+ protected abstract int[] getAllIndexesOf(String name);
+
+ private T setValue(int i, ByteBuffer value)
+ {
+ values[i] = value;
+ return wrapped;
+ }
+
+ @Override
+ protected ByteBuffer getValue(int i)
+ {
+ return values[i];
+ }
+
+ @Override
+ protected int getIndexOf(String name)
+ {
+ return getAllIndexesOf(name)[0];
+ }
+
+ @Override
+ public T setBool(int i, boolean v)
+ {
+ TypeCodec<Boolean> codec = codecFor(i, Boolean.class);
+ ByteBuffer bb;
+ if (codec instanceof TypeCodec.PrimitiveBooleanCodec)
+ bb = ((TypeCodec.PrimitiveBooleanCodec) codec).serializeNoBoxing(v, protocolVersion);
+ else bb = codec.serialize(v, protocolVersion);
+ return setValue(i, bb);
+ }
+
+ @Override
+ public T setBool(String name, boolean v)
+ {
+ for (int i : getAllIndexesOf(name))
+ {
+ setBool(i, v);
+ }
+ return wrapped;
+ }
+
+ @Override
+ public T setByte(int i, byte v)
+ {
+ TypeCodec<Byte> codec = codecFor(i, Byte.class);
+ ByteBuffer bb;
+ if (codec instanceof TypeCodec.PrimitiveByteCodec)
+ bb = ((TypeCodec.PrimitiveByteCodec) codec).serializeNoBoxing(v, protocolVersion);
+ else bb = codec.serialize(v, protocolVersion);
+ return setValue(i, bb);
+ }
+
+ @Override
+ public T setByte(String name, byte v)
+ {
+ for (int i : getAllIndexesOf(name))
+ {
+ setByte(i, v);
+ }
+ return wrapped;
+ }
+
+ @Override
+ public T setShort(int i, short v)
+ {
+ TypeCodec<Short> codec = codecFor(i, Short.class);
+ ByteBuffer bb;
+ if (codec instanceof TypeCodec.PrimitiveShortCodec)
+ bb = ((TypeCodec.PrimitiveShortCodec) codec).serializeNoBoxing(v, protocolVersion);
+ else bb = codec.serialize(v, protocolVersion);
+ return setValue(i, bb);
+ }
+
+ @Override
+ public T setShort(String name, short v)
+ {
+ for (int i : getAllIndexesOf(name))
+ {
+ setShort(i, v);
+ }
+ return wrapped;
+ }
+
+ @Override
+ public T setInt(int i, int v)
+ {
+ TypeCodec<Integer> codec = codecFor(i, Integer.class);
+ ByteBuffer bb;
+ if (codec instanceof TypeCodec.PrimitiveIntCodec)
+ bb = ((TypeCodec.PrimitiveIntCodec) codec).serializeNoBoxing(v, protocolVersion);
+ else bb = codec.serialize(v, protocolVersion);
+ return setValue(i, bb);
+ }
+
+ @Override
+ public T setInt(String name, int v)
+ {
+ for (int i : getAllIndexesOf(name))
+ {
+ setInt(i, v);
+ }
+ return wrapped;
+ }
+
+ @Override
+ public T setLong(int i, long v)
+ {
+ TypeCodec<Long> codec = codecFor(i, Long.class);
+ ByteBuffer bb;
+ if (codec instanceof TypeCodec.PrimitiveLongCodec)
+ bb = ((TypeCodec.PrimitiveLongCodec) codec).serializeNoBoxing(v, protocolVersion);
+ else bb = codec.serialize(v, protocolVersion);
+ return setValue(i, bb);
+ }
+
+ @Override
+ public T setLong(String name, long v)
+ {
+ for (int i : getAllIndexesOf(name))
+ {
+ setLong(i, v);
+ }
+ return wrapped;
+ }
+
+ @Override
+ public T setTimestamp(int i, Date v)
+ {
+ return setValue(i, codecFor(i, Date.class).serialize(v, protocolVersion));
+ }
+
+ @Override
+ public T setTimestamp(String name, Date v)
+ {
+ for (int i : getAllIndexesOf(name))
+ {
+ setTimestamp(i, v);
+ }
+ return wrapped;
+ }
+
+ @Override
+ public T setDate(int i, LocalDate v)
+ {
+ return setValue(i, codecFor(i, LocalDate.class).serialize(v, protocolVersion));
+ }
+
+ @Override
+ public T setDate(String name, LocalDate v)
+ {
+ for (int i : getAllIndexesOf(name))
+ {
+ setDate(i, v);
+ }
+ return wrapped;
+ }
+
+ @Override
+ public T setTime(int i, long v)
+ {
+ TypeCodec<Long> codec = codecFor(i, Long.class);
+ ByteBuffer bb;
+ if (codec instanceof TypeCodec.PrimitiveLongCodec)
+ bb = ((TypeCodec.PrimitiveLongCodec) codec).serializeNoBoxing(v, protocolVersion);
+ else bb = codec.serialize(v, protocolVersion);
+ return setValue(i, bb);
+ }
+
+ @Override
+ public T setTime(String name, long v)
+ {
+ for (int i : getAllIndexesOf(name))
+ {
+ setTime(i, v);
+ }
+ return wrapped;
+ }
+
+ @Override
+ public T setFloat(int i, float v)
+ {
+ TypeCodec<Float> codec = codecFor(i, Float.class);
+ ByteBuffer bb;
+ if (codec instanceof TypeCodec.PrimitiveFloatCodec)
+ bb = ((TypeCodec.PrimitiveFloatCodec) codec).serializeNoBoxing(v, protocolVersion);
+ else bb = codec.serialize(v, protocolVersion);
+ return setValue(i, bb);
+ }
+
+ @Override
+ public T setFloat(String name, float v)
+ {
+ for (int i : getAllIndexesOf(name))
+ {
+ setFloat(i, v);
+ }
+ return wrapped;
+ }
+
+ @Override
+ public T setDouble(int i, double v)
+ {
+ TypeCodec<Double> codec = codecFor(i, Double.class);
+ ByteBuffer bb;
+ if (codec instanceof TypeCodec.PrimitiveDoubleCodec)
+ bb = ((TypeCodec.PrimitiveDoubleCodec) codec).serializeNoBoxing(v, protocolVersion);
+ else bb = codec.serialize(v, protocolVersion);
+ return setValue(i, bb);
+ }
+
+ @Override
+ public T setDouble(String name, double v)
+ {
+ for (int i : getAllIndexesOf(name))
+ {
+ setDouble(i, v);
+ }
+ return wrapped;
+ }
+
+ @Override
+ public T setString(int i, String v)
+ {
+ return setValue(i, codecFor(i, String.class).serialize(v, protocolVersion));
+ }
+
+ @Override
+ public T setString(String name, String v)
+ {
+ for (int i : getAllIndexesOf(name))
+ {
+ setString(i, v);
+ }
+ return wrapped;
+ }
+
+ @Override
+ public T setBytes(int i, ByteBuffer v)
+ {
+ return setValue(i, codecFor(i, ByteBuffer.class).serialize(v, protocolVersion));
+ }
+
+ @Override
+ public T setBytes(String name, ByteBuffer v)
+ {
+ for (int i : getAllIndexesOf(name))
+ {
+ setBytes(i, v);
+ }
+ return wrapped;
+ }
+
+ @Override
+ public T setBytesUnsafe(int i, ByteBuffer v)
+ {
+ return setValue(i, v == null ? null : v.duplicate());
+ }
+
+ @Override
+ public T setBytesUnsafe(String name, ByteBuffer v)
+ {
+ ByteBuffer value = v == null ? null : v.duplicate();
+ for (int i : getAllIndexesOf(name))
+ {
+ setValue(i, value);
+ }
+ return wrapped;
+ }
+
+ @Override
+ public T setVarint(int i, BigInteger v)
+ {
+ return setValue(i, codecFor(i, BigInteger.class).serialize(v, protocolVersion));
+ }
+
+ @Override
+ public T setVarint(String name, BigInteger v)
+ {
+ for (int i : getAllIndexesOf(name))
+ {
+ setVarint(i, v);
+ }
+ return wrapped;
+ }
+
+ @Override
+ public T setDecimal(int i, BigDecimal v)
+ {
+ return setValue(i, codecFor(i, BigDecimal.class).serialize(v, protocolVersion));
+ }
+
+ @Override
+ public T setDecimal(String name, BigDecimal v)
+ {
+ for (int i : getAllIndexesOf(name))
+ {
+ setDecimal(i, v);
+ }
+ return wrapped;
+ }
+
+ @Override
+ public T setUUID(int i, UUID v)
+ {
+ return setValue(i, codecFor(i, UUID.class).serialize(v, protocolVersion));
+ }
+
+ @Override
+ public T setUUID(String name, UUID v)
+ {
+ for (int i : getAllIndexesOf(name))
+ {
+ setUUID(i, v);
+ }
+ return wrapped;
+ }
+
+ @Override
+ public T setInet(int i, InetAddress v)
+ {
+ return setValue(i, codecFor(i, InetAddress.class).serialize(v, protocolVersion));
+ }
+
+ @Override
+ public T setInet(String name, InetAddress v)
+ {
+ for (int i : getAllIndexesOf(name))
+ {
+ setInet(i, v);
+ }
+ return wrapped;
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public <E> T setList(int i, List<E> v)
+ {
+ return setValue(i, codecFor(i).serialize(v, protocolVersion));
+ }
+
+ @Override
+ public <E> T setList(int i, List<E> v, Class<E> elementsClass)
+ {
+ return setValue(i, codecFor(i, TypeTokens.listOf(elementsClass)).serialize(v, protocolVersion));
+ }
+
+ @Override
+ public <E> T setList(int i, List<E> v, TypeToken<E> elementsType)
+ {
+ return setValue(i, codecFor(i, TypeTokens.listOf(elementsType)).serialize(v, protocolVersion));
+ }
+
+ @Override
+ public <E> T setList(String name, List<E> v)
+ {
+ for (int i : getAllIndexesOf(name))
+ {
+ setList(i, v);
+ }
+ return wrapped;
+ }
+
+ @Override
+ public <E> T setList(String name, List<E> v, Class<E> elementsClass)
+ {
+ for (int i : getAllIndexesOf(name))
+ {
+ setList(i, v, elementsClass);
+ }
+ return wrapped;
+ }
+
+ @Override
+ public <E> T setList(String name, List<E> v, TypeToken<E> elementsType)
+ {
+ for (int i : getAllIndexesOf(name))
+ {
+ setList(i, v, elementsType);
+ }
+ return wrapped;
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public <K, V> T setMap(int i, Map<K, V> v)
+ {
+ return setValue(i, codecFor(i).serialize(v, protocolVersion));
+ }
+
+ @Override
+ public <K, V> T setMap(int i, Map<K, V> v, Class<K> keysClass, Class<V> valuesClass)
+ {
+ return setValue(
+ i, codecFor(i, TypeTokens.mapOf(keysClass, valuesClass)).serialize(v, protocolVersion));
+ }
+
+ @Override
+ public <K, V> T setMap(int i, Map<K, V> v, TypeToken<K> keysType, TypeToken<V> valuesType)
+ {
+ return setValue(
+ i, codecFor(i, TypeTokens.mapOf(keysType, valuesType)).serialize(v, protocolVersion));
+ }
+
+ @Override
+ public <K, V> T setMap(String name, Map<K, V> v)
+ {
+ for (int i : getAllIndexesOf(name))
+ {
+ setMap(i, v);
+ }
+ return wrapped;
+ }
+
+ @Override
+ public <K, V> T setMap(String name, Map<K, V> v, Class<K> keysClass, Class<V> valuesClass)
+ {
+ for (int i : getAllIndexesOf(name))
+ {
+ setMap(i, v, keysClass, valuesClass);
+ }
+ return wrapped;
+ }
+
+ @Override
+ public <K, V> T setMap(String name, Map<K, V> v, TypeToken<K> keysType, TypeToken<V> valuesType)
+ {
+ for (int i : getAllIndexesOf(name))
+ {
+ setMap(i, v, keysType, valuesType);
+ }
+ return wrapped;
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public <E> T setSet(int i, Set<E> v)
+ {
+ return setValue(i, codecFor(i).serialize(v, protocolVersion));
+ }
+
+ @Override
+ public <E> T setSet(int i, Set<E> v, Class<E> elementsClass)
+ {
+ return setValue(i, codecFor(i, TypeTokens.setOf(elementsClass)).serialize(v, protocolVersion));
+ }
+
+ @Override
+ public <E> T setSet(int i, Set<E> v, TypeToken<E> elementsType)
+ {
+ return setValue(i, codecFor(i, TypeTokens.setOf(elementsType)).serialize(v, protocolVersion));
+ }
+
+ @Override
+ public <E> T setSet(String name, Set<E> v)
+ {
+ for (int i : getAllIndexesOf(name))
+ {
+ setSet(i, v);
+ }
+ return wrapped;
+ }
+
+ @Override
+ public <E> T setSet(String name, Set<E> v, Class<E> elementsClass)
+ {
+ for (int i : getAllIndexesOf(name))
+ {
+ setSet(i, v, elementsClass);
+ }
+ return wrapped;
+ }
+
+ @Override
+ public <E> T setSet(String name, Set<E> v, TypeToken<E> elementsType)
+ {
+ for (int i : getAllIndexesOf(name))
+ {
+ setSet(i, v, elementsType);
+ }
+ return wrapped;
+ }
+
+ @Override
+ public T setUDTValue(int i, UDTValue v)
+ {
+ return setValue(i, codecFor(i, UDTValue.class).serialize(v, protocolVersion));
+ }
+
+ @Override
+ public T setUDTValue(String name, UDTValue v)
+ {
+ for (int i : getAllIndexesOf(name))
+ {
+ setUDTValue(i, v);
+ }
+ return wrapped;
+ }
+
+ @Override
+ public T setTupleValue(int i, TupleValue v)
+ {
+ return setValue(i, codecFor(i, TupleValue.class).serialize(v, protocolVersion));
+ }
+
+ @Override
+ public T setTupleValue(String name, TupleValue v)
+ {
+ for (int i : getAllIndexesOf(name))
+ {
+ setTupleValue(i, v);
+ }
+ return wrapped;
+ }
+
+ @Override
+ public <V> T set(int i, V v, Class<V> targetClass)
+ {
+ return set(i, v, codecFor(i, targetClass));
+ }
+
+ @Override
+ public <V> T set(String name, V v, Class<V> targetClass)
+ {
+ for (int i : getAllIndexesOf(name))
+ {
+ set(i, v, targetClass);
+ }
+ return wrapped;
+ }
+
+ @Override
+ public <V> T set(int i, V v, TypeToken<V> targetType)
+ {
+ return set(i, v, codecFor(i, targetType));
+ }
+
+ @Override
+ public <V> T set(String name, V v, TypeToken<V> targetType)
+ {
+ for (int i : getAllIndexesOf(name))
+ {
+ set(i, v, targetType);
+ }
+ return wrapped;
+ }
+
+ @Override
+ public <V> T set(int i, V v, TypeCodec<V> codec)
+ {
+ checkType(i, codec.getCqlType().getName());
+ return setValue(i, codec.serialize(v, protocolVersion));
+ }
+
+ @Override
+ public <V> T set(String name, V v, TypeCodec<V> codec)
+ {
+ for (int i : getAllIndexesOf(name))
+ {
+ set(i, v, codec);
+ }
+ return wrapped;
+ }
+
+ @Override
+ public T setToNull(int i)
+ {
+ return setValue(i, null);
+ }
+
+ @Override
+ public T setToNull(String name)
+ {
+ for (int i : getAllIndexesOf(name))
+ {
+ setToNull(i);
+ }
+ return wrapped;
+ }
+
+ @Override
+ public boolean equals(Object o)
+ {
+ if (!(o instanceof AbstractData)) return false;
+
+ AbstractData<?> that = (AbstractData<?>) o;
+ if (values.length != that.values.length) return false;
+
+ if (this.protocolVersion != that.protocolVersion) return false;
+
+ // Deserializing each value is slightly inefficient, but comparing
+ // the bytes could in theory be wrong (for varint for instance, 2 values
+ // can have different binary representation but be the same value due to
+ // leading zeros). So we don't take any risk.
+ for (int i = 0; i < values.length; i++)
+ {
+ DataType thisType = getType(i);
+ DataType thatType = that.getType(i);
+ if (!thisType.equals(thatType)) return false;
+
+ Object thisValue = this.codecFor(i).deserialize(this.values[i], this.protocolVersion);
+ Object thatValue = that.codecFor(i).deserialize(that.values[i], that.protocolVersion);
+ if (!Objects.equals(thisValue, thatValue)) return false;
+ }
+ return true;
+ }
+
+ @Override
+ public int hashCode()
+ {
+ // Same as equals
+ int hash = 31;
+ for (int i = 0; i < values.length; i++)
+ hash +=
+ values[i] == null ? 1 : codecFor(i).deserialize(values[i], protocolVersion).hashCode();
+ return hash;
+ }
+}
diff --git a/src/java/org/apache/cassandra/cql3/functions/types/AbstractGettableByIndexData.java b/src/java/org/apache/cassandra/cql3/functions/types/AbstractGettableByIndexData.java
new file mode 100644
index 0000000..1552309
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/functions/types/AbstractGettableByIndexData.java
@@ -0,0 +1,418 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.functions.types;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.net.InetAddress;
+import java.nio.ByteBuffer;
+import java.util.*;
+
+import com.google.common.reflect.TypeToken;
+
+import org.apache.cassandra.transport.ProtocolVersion;
+import org.apache.cassandra.cql3.functions.types.exceptions.InvalidTypeException;
+
+abstract class AbstractGettableByIndexData implements GettableByIndexData
+{
+
+ protected final ProtocolVersion protocolVersion;
+
+ AbstractGettableByIndexData(ProtocolVersion protocolVersion)
+ {
+ this.protocolVersion = protocolVersion;
+ }
+
+ /**
+ * Returns the type for the value at index {@code i}.
+ *
+ * @param i the index of the type to fetch.
+ * @return the type of the value at index {@code i}.
+ * @throws IndexOutOfBoundsException if {@code i} is not a valid index.
+ */
+ protected abstract DataType getType(int i);
+
+ /**
+ * Returns the name corresponding to the value at index {@code i}.
+ *
+ * @param i the index of the name to fetch.
+ * @return the name corresponding to the value at index {@code i}.
+ * @throws IndexOutOfBoundsException if {@code i} is not a valid index.
+ */
+ protected abstract String getName(int i);
+
+ /**
+ * Returns the value at index {@code i}.
+ *
+ * @param i the index to fetch.
+ * @return the value at index {@code i}.
+ * @throws IndexOutOfBoundsException if {@code i} is not a valid index.
+ */
+ protected abstract ByteBuffer getValue(int i);
+
+ protected abstract CodecRegistry getCodecRegistry();
+
+ protected <T> TypeCodec<T> codecFor(int i)
+ {
+ return getCodecRegistry().codecFor(getType(i));
+ }
+
+ protected <T> TypeCodec<T> codecFor(int i, Class<T> javaClass)
+ {
+ return getCodecRegistry().codecFor(getType(i), javaClass);
+ }
+
+ protected <T> TypeCodec<T> codecFor(int i, TypeToken<T> javaType)
+ {
+ return getCodecRegistry().codecFor(getType(i), javaType);
+ }
+
+ protected <T> TypeCodec<T> codecFor(int i, T value)
+ {
+ return getCodecRegistry().codecFor(getType(i), value);
+ }
+
+ void checkType(int i, DataType.Name actual)
+ {
+ DataType.Name expected = getType(i).getName();
+ if (!actual.isCompatibleWith(expected))
+ throw new InvalidTypeException(
+ String.format("Value %s is of type %s, not %s", getName(i), expected, actual));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public boolean isNull(int i)
+ {
+ return getValue(i) == null;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public boolean getBool(int i)
+ {
+ ByteBuffer value = getValue(i);
+ TypeCodec<Boolean> codec = codecFor(i, Boolean.class);
+ if (codec instanceof TypeCodec.PrimitiveBooleanCodec)
+ return ((TypeCodec.PrimitiveBooleanCodec) codec).deserializeNoBoxing(value, protocolVersion);
+ else return codec.deserialize(value, protocolVersion);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public byte getByte(int i)
+ {
+ ByteBuffer value = getValue(i);
+ TypeCodec<Byte> codec = codecFor(i, Byte.class);
+ if (codec instanceof TypeCodec.PrimitiveByteCodec)
+ return ((TypeCodec.PrimitiveByteCodec) codec).deserializeNoBoxing(value, protocolVersion);
+ else return codec.deserialize(value, protocolVersion);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public short getShort(int i)
+ {
+ ByteBuffer value = getValue(i);
+ TypeCodec<Short> codec = codecFor(i, Short.class);
+ if (codec instanceof TypeCodec.PrimitiveShortCodec)
+ return ((TypeCodec.PrimitiveShortCodec) codec).deserializeNoBoxing(value, protocolVersion);
+ else return codec.deserialize(value, protocolVersion);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public int getInt(int i)
+ {
+ ByteBuffer value = getValue(i);
+ TypeCodec<Integer> codec = codecFor(i, Integer.class);
+ if (codec instanceof TypeCodec.PrimitiveIntCodec)
+ return ((TypeCodec.PrimitiveIntCodec) codec).deserializeNoBoxing(value, protocolVersion);
+ else return codec.deserialize(value, protocolVersion);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public long getLong(int i)
+ {
+ ByteBuffer value = getValue(i);
+ TypeCodec<Long> codec = codecFor(i, Long.class);
+ if (codec instanceof TypeCodec.PrimitiveLongCodec)
+ return ((TypeCodec.PrimitiveLongCodec) codec).deserializeNoBoxing(value, protocolVersion);
+ else return codec.deserialize(value, protocolVersion);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public Date getTimestamp(int i)
+ {
+ ByteBuffer value = getValue(i);
+ return codecFor(i, Date.class).deserialize(value, protocolVersion);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public LocalDate getDate(int i)
+ {
+ ByteBuffer value = getValue(i);
+ return codecFor(i, LocalDate.class).deserialize(value, protocolVersion);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public long getTime(int i)
+ {
+ ByteBuffer value = getValue(i);
+ TypeCodec<Long> codec = codecFor(i, Long.class);
+ if (codec instanceof TypeCodec.PrimitiveLongCodec)
+ return ((TypeCodec.PrimitiveLongCodec) codec).deserializeNoBoxing(value, protocolVersion);
+ else return codec.deserialize(value, protocolVersion);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public float getFloat(int i)
+ {
+ ByteBuffer value = getValue(i);
+ TypeCodec<Float> codec = codecFor(i, Float.class);
+ if (codec instanceof TypeCodec.PrimitiveFloatCodec)
+ return ((TypeCodec.PrimitiveFloatCodec) codec).deserializeNoBoxing(value, protocolVersion);
+ else return codec.deserialize(value, protocolVersion);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public double getDouble(int i)
+ {
+ ByteBuffer value = getValue(i);
+ TypeCodec<Double> codec = codecFor(i, Double.class);
+ if (codec instanceof TypeCodec.PrimitiveDoubleCodec)
+ return ((TypeCodec.PrimitiveDoubleCodec) codec).deserializeNoBoxing(value, protocolVersion);
+ else return codec.deserialize(value, protocolVersion);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public ByteBuffer getBytesUnsafe(int i)
+ {
+ ByteBuffer value = getValue(i);
+ if (value == null) return null;
+ return value.duplicate();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public ByteBuffer getBytes(int i)
+ {
+ ByteBuffer value = getValue(i);
+ return codecFor(i, ByteBuffer.class).deserialize(value, protocolVersion);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public String getString(int i)
+ {
+ ByteBuffer value = getValue(i);
+ return codecFor(i, String.class).deserialize(value, protocolVersion);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public BigInteger getVarint(int i)
+ {
+ ByteBuffer value = getValue(i);
+ return codecFor(i, BigInteger.class).deserialize(value, protocolVersion);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public BigDecimal getDecimal(int i)
+ {
+ ByteBuffer value = getValue(i);
+ return codecFor(i, BigDecimal.class).deserialize(value, protocolVersion);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public UUID getUUID(int i)
+ {
+ ByteBuffer value = getValue(i);
+ return codecFor(i, UUID.class).deserialize(value, protocolVersion);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public InetAddress getInet(int i)
+ {
+ ByteBuffer value = getValue(i);
+ return codecFor(i, InetAddress.class).deserialize(value, protocolVersion);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ @SuppressWarnings("unchecked")
+ public <T> List<T> getList(int i, Class<T> elementsClass)
+ {
+ return getList(i, TypeToken.of(elementsClass));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ @SuppressWarnings("unchecked")
+ public <T> List<T> getList(int i, TypeToken<T> elementsType)
+ {
+ ByteBuffer value = getValue(i);
+ TypeToken<List<T>> javaType = TypeTokens.listOf(elementsType);
+ return codecFor(i, javaType).deserialize(value, protocolVersion);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ @SuppressWarnings("unchecked")
+ public <T> Set<T> getSet(int i, Class<T> elementsClass)
+ {
+ return getSet(i, TypeToken.of(elementsClass));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ @SuppressWarnings("unchecked")
+ public <T> Set<T> getSet(int i, TypeToken<T> elementsType)
+ {
+ ByteBuffer value = getValue(i);
+ TypeToken<Set<T>> javaType = TypeTokens.setOf(elementsType);
+ return codecFor(i, javaType).deserialize(value, protocolVersion);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ @SuppressWarnings("unchecked")
+ public <K, V> Map<K, V> getMap(int i, Class<K> keysClass, Class<V> valuesClass)
+ {
+ return getMap(i, TypeToken.of(keysClass), TypeToken.of(valuesClass));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ @SuppressWarnings("unchecked")
+ public <K, V> Map<K, V> getMap(int i, TypeToken<K> keysType, TypeToken<V> valuesType)
+ {
+ ByteBuffer value = getValue(i);
+ TypeToken<Map<K, V>> javaType = TypeTokens.mapOf(keysType, valuesType);
+ return codecFor(i, javaType).deserialize(value, protocolVersion);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ @SuppressWarnings("unchecked")
+ public UDTValue getUDTValue(int i)
+ {
+ ByteBuffer value = getValue(i);
+ return codecFor(i, UDTValue.class).deserialize(value, protocolVersion);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ @SuppressWarnings("unchecked")
+ public TupleValue getTupleValue(int i)
+ {
+ ByteBuffer value = getValue(i);
+ return codecFor(i, TupleValue.class).deserialize(value, protocolVersion);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public Object getObject(int i)
+ {
+ return get(i, codecFor(i));
+ }
+
+ @Override
+ public <T> T get(int i, Class<T> targetClass)
+ {
+ return get(i, codecFor(i, targetClass));
+ }
+
+ @Override
+ public <T> T get(int i, TypeToken<T> targetType)
+ {
+ return get(i, codecFor(i, targetType));
+ }
+
+ @Override
+ public <T> T get(int i, TypeCodec<T> codec)
+ {
+ checkType(i, codec.getCqlType().getName());
+ ByteBuffer value = getValue(i);
+ return codec.deserialize(value, protocolVersion);
+ }
+}
diff --git a/src/java/org/apache/cassandra/cql3/functions/types/AbstractGettableData.java b/src/java/org/apache/cassandra/cql3/functions/types/AbstractGettableData.java
new file mode 100644
index 0000000..2ac7d98
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/functions/types/AbstractGettableData.java
@@ -0,0 +1,325 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.functions.types;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.net.InetAddress;
+import java.nio.ByteBuffer;
+import java.util.*;
+
+import com.google.common.reflect.TypeToken;
+
+import org.apache.cassandra.transport.ProtocolVersion;
+
+public abstract class AbstractGettableData extends AbstractGettableByIndexData
+implements GettableData
+{
+
+ /**
+ * Creates a new AbstractGettableData object.
+ *
+ * @param protocolVersion the protocol version in which values returned by {@link #getValue} will
+ * be returned. This must be a protocol version supported by this driver. In general, the
+ * correct value will be the value returned by {@code ProtocolOptions#getProtocolVersion}.
+ * @throws IllegalArgumentException if {@code protocolVersion} is not a valid protocol version.
+ */
+ AbstractGettableData(ProtocolVersion protocolVersion)
+ {
+ super(protocolVersion);
+ }
+
+ /**
+ * Returns the index corresponding to a given name.
+ *
+ * @param name the name for which to return the index of.
+ * @return the index for the value coressponding to {@code name}.
+ * @throws IllegalArgumentException if {@code name} is not valid name for this object.
+ */
+ protected abstract int getIndexOf(String name);
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public boolean isNull(String name)
+ {
+ return isNull(getIndexOf(name));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public boolean getBool(String name)
+ {
+ return getBool(getIndexOf(name));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public byte getByte(String name)
+ {
+ return getByte(getIndexOf(name));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public short getShort(String name)
+ {
+ return getShort(getIndexOf(name));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public int getInt(String name)
+ {
+ return getInt(getIndexOf(name));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public long getLong(String name)
+ {
+ return getLong(getIndexOf(name));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public Date getTimestamp(String name)
+ {
+ return getTimestamp(getIndexOf(name));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public LocalDate getDate(String name)
+ {
+ return getDate(getIndexOf(name));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public long getTime(String name)
+ {
+ return getTime(getIndexOf(name));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public float getFloat(String name)
+ {
+ return getFloat(getIndexOf(name));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public double getDouble(String name)
+ {
+ return getDouble(getIndexOf(name));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public ByteBuffer getBytesUnsafe(String name)
+ {
+ return getBytesUnsafe(getIndexOf(name));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public ByteBuffer getBytes(String name)
+ {
+ return getBytes(getIndexOf(name));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public String getString(String name)
+ {
+ return getString(getIndexOf(name));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public BigInteger getVarint(String name)
+ {
+ return getVarint(getIndexOf(name));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public BigDecimal getDecimal(String name)
+ {
+ return getDecimal(getIndexOf(name));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public UUID getUUID(String name)
+ {
+ return getUUID(getIndexOf(name));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public InetAddress getInet(String name)
+ {
+ return getInet(getIndexOf(name));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public <T> List<T> getList(String name, Class<T> elementsClass)
+ {
+ return getList(getIndexOf(name), elementsClass);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public <T> List<T> getList(String name, TypeToken<T> elementsType)
+ {
+ return getList(getIndexOf(name), elementsType);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public <T> Set<T> getSet(String name, Class<T> elementsClass)
+ {
+ return getSet(getIndexOf(name), elementsClass);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public <T> Set<T> getSet(String name, TypeToken<T> elementsType)
+ {
+ return getSet(getIndexOf(name), elementsType);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public <K, V> Map<K, V> getMap(String name, Class<K> keysClass, Class<V> valuesClass)
+ {
+ return getMap(getIndexOf(name), keysClass, valuesClass);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public <K, V> Map<K, V> getMap(String name, TypeToken<K> keysType, TypeToken<V> valuesType)
+ {
+ return getMap(getIndexOf(name), keysType, valuesType);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public UDTValue getUDTValue(String name)
+ {
+ return getUDTValue(getIndexOf(name));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public TupleValue getTupleValue(String name)
+ {
+ return getTupleValue(getIndexOf(name));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public Object getObject(String name)
+ {
+ return getObject(getIndexOf(name));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public <T> T get(String name, Class<T> targetClass)
+ {
+ return get(getIndexOf(name), targetClass);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public <T> T get(String name, TypeToken<T> targetType)
+ {
+ return get(getIndexOf(name), targetType);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public <T> T get(String name, TypeCodec<T> codec)
+ {
+ return get(getIndexOf(name), codec);
+ }
+}
diff --git a/src/java/org/apache/cassandra/cql3/functions/types/CodecRegistry.java b/src/java/org/apache/cassandra/cql3/functions/types/CodecRegistry.java
new file mode 100644
index 0000000..a979a57
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/functions/types/CodecRegistry.java
@@ -0,0 +1,885 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.functions.types;
+
+import java.lang.reflect.ParameterizedType;
+import java.lang.reflect.Type;
+import java.nio.ByteBuffer;
+import java.util.*;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.ExecutionException;
+
+import com.google.common.cache.*;
+import com.google.common.reflect.TypeToken;
+import com.google.common.util.concurrent.UncheckedExecutionException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.cql3.functions.types.exceptions.CodecNotFoundException;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static org.apache.cassandra.cql3.functions.types.DataType.Name.*;
+
+/**
+ * A registry for {@link TypeCodec}s. When the driver needs to serialize or deserialize a Java type
+ * to/from CQL, it will lookup in the registry for a suitable codec. The registry is initialized
+ * with default codecs that handle basic conversions (e.g. CQL {@code text} to {@code
+ * java.lang.String}), and users can add their own. Complex codecs can also be generated on-the-fly
+ * from simpler ones (more details below).
+ *
+ * <h3>Creating a registry </h3>
+ * <p>
+ * By default, the driver uses {@code CodecRegistry#DEFAULT_INSTANCE}, a shareable, JVM-wide
+ * instance initialized with built-in codecs for all the base CQL types. The only reason to create
+ * your own instances is if you have multiple {@code Cluster} objects that use different sets of
+ * codecs. In that case, use {@code
+ * Cluster.Builder#withCodecRegistry(CodecRegistry)} to associate the
+ * registry with the cluster:
+ *
+ * <pre>{@code
+ * CodecRegistry myCodecRegistry = new CodecRegistry();
+ * myCodecRegistry.register(myCodec1, myCodec2, myCodec3);
+ * Cluster cluster = Cluster.builder().withCodecRegistry(myCodecRegistry).build();
+ *
+ * // To retrieve the registry later:
+ * CodecRegistry registry = cluster.getConfiguration().getCodecRegistry();
+ * }</pre>
+ * <p>
+ * {@code CodecRegistry} instances are thread-safe.
+ *
+ * <p>It is possible to turn on log messages by setting the {@code
+ * CodecRegistry} logger level to {@code TRACE}. Beware that the registry
+ * can be very verbose at this log level.
+ *
+ * <h3>Registering and using custom codecs </h3>
+ * <p>
+ * To create a custom codec, write a class that extends {@link TypeCodec}, create an instance, and
+ * pass it to one of the {@link #register(TypeCodec) register} methods; for example, one could
+ * create a codec that maps CQL timestamps to JDK8's {@code java.time.LocalDate}:
+ *
+ * <pre>{@code
+ * class LocalDateCodec extends TypeCodec<java.time.LocalDate> {
+ * ...
+ * }
+ * myCodecRegistry.register(new LocalDateCodec());
+ * }</pre>
+ * <p>
+ * The conversion will be available to:
+ *
+ * <ul>
+ * <li>all driver types that implement {@link GettableByIndexData}, {@link GettableByNameData},
+ * {@link SettableByIndexData} and/or {@link SettableByNameData}. Namely: {@code Row}, {@code
+ * BoundStatement}, {@link UDTValue} and {@link TupleValue};
+ * <li>{@code SimpleStatement#SimpleStatement(String, Object...) simple statements};
+ * <li>statements created with the {@code querybuilder.QueryBuilder Query
+ * builder}.
+ * </ul>
+ *
+ * <p>Example:
+ *
+ * <pre>{@code
+ * Row row = session.executeQuery("select date from some_table where pk = 1").one();
+ * java.time.LocalDate date = row.get(0, java.time.LocalDate.class); // uses LocalDateCodec registered above
+ * }</pre>
+ * <p>
+ * You can also bypass the codec registry by passing a standalone codec instance to methods such as
+ * {@link GettableByIndexData#get(int, TypeCodec)}.
+ *
+ * <h3>Codec generation </h3>
+ * <p>
+ * When a {@code CodecRegistry} cannot find a suitable codec among existing ones, it will attempt to
+ * create it on-the-fly. It can manage:
+ *
+ * <ul>
+ * <li>collections (lists, sets and maps) of known types. For example, if you registered a codec
+ * for JDK8's {@code java.time.LocalDate} like in the example above, you get {@code
+ * List<LocalDate>>} and {@code Set<LocalDate>>} handled for free, as well as all {@code Map}
+ * types whose keys and/or values are {@code java.time.LocalDate}. This works recursively for
+ * nested collections;
+ * <li>{@link UserType user types}, mapped to {@link UDTValue} objects. Custom codecs are
+ * available recursively to the UDT's fields, so if one of your fields is a {@code timestamp}
+ * you can use your {@code LocalDateCodec} to retrieve it as a {@code java.time.LocalDate};
+ * <li>{@link TupleType tuple types}, mapped to {@link TupleValue} (with the same rules for nested
+ * fields);
+ * <li>{@link DataType.CustomType custom types}, mapped to {@code
+ * ByteBuffer}.
+ * </ul>
+ * <p>
+ * If the codec registry encounters a mapping that it can't handle automatically, a {@link
+ * CodecNotFoundException} is thrown; you'll need to register a custom codec for it.
+ *
+ * <h3>Performance and caching </h3>
+ * <p>
+ * Whenever possible, the registry will cache the result of a codec lookup for a specific type
+ * mapping, including any generated codec. For example, if you registered {@code LocalDateCodec} and
+ * ask the registry for a codec to convert a CQL {@code list<timestamp>} to a Java {@code
+ * List<LocalDate>}:
+ *
+ * <ol>
+ * <li>the first lookup will generate a {@code TypeCodec<List<LocalDate>>} from {@code
+ * LocalDateCodec}, and put it in the cache;
+ * <li>the second lookup will hit the cache directly, and reuse the previously generated instance.
+ * </ol>
+ * <p>
+ * The javadoc for each {@link #codecFor(DataType) codecFor} variant specifies whether the result
+ * can be cached or not.
+ *
+ * <h3>Codec order </h3>
+ * <p>
+ * When the registry looks up a codec, the rules of precedence are:
+ *
+ * <ul>
+ * <li>if a result was previously cached for that mapping, it is returned;
+ * <li>otherwise, the registry checks the list of built-in codecs – the default ones – and the
+ * ones that were explicitly registered (in the order that they were registered). It calls
+ * each codec's {@code accepts} methods to determine if it can handle the mapping, and if so
+ * returns it;
+ * <li>otherwise, the registry tries to generate a codec, according to the rules outlined above.
+ * </ul>
+ * <p>
+ * It is currently impossible to override an existing codec. If you try to do so, {@link
+ * #register(TypeCodec)} will log a warning and ignore it.
+ */
+public final class CodecRegistry
+{
+
+ private static final Logger logger = LoggerFactory.getLogger(CodecRegistry.class);
+
+ private static final Map<DataType.Name, TypeCodec<?>> BUILT_IN_CODECS_MAP =
+ new EnumMap<>(DataType.Name.class);
+
+ static
+ {
+ BUILT_IN_CODECS_MAP.put(DataType.Name.ASCII, TypeCodec.ascii());
+ BUILT_IN_CODECS_MAP.put(DataType.Name.BIGINT, TypeCodec.bigint());
+ BUILT_IN_CODECS_MAP.put(DataType.Name.BLOB, TypeCodec.blob());
+ BUILT_IN_CODECS_MAP.put(DataType.Name.BOOLEAN, TypeCodec.cboolean());
+ BUILT_IN_CODECS_MAP.put(DataType.Name.COUNTER, TypeCodec.counter());
+ BUILT_IN_CODECS_MAP.put(DataType.Name.DECIMAL, TypeCodec.decimal());
+ BUILT_IN_CODECS_MAP.put(DataType.Name.DOUBLE, TypeCodec.cdouble());
+ BUILT_IN_CODECS_MAP.put(DataType.Name.FLOAT, TypeCodec.cfloat());
+ BUILT_IN_CODECS_MAP.put(DataType.Name.INET, TypeCodec.inet());
+ BUILT_IN_CODECS_MAP.put(DataType.Name.INT, TypeCodec.cint());
+ BUILT_IN_CODECS_MAP.put(DataType.Name.TEXT, TypeCodec.varchar());
+ BUILT_IN_CODECS_MAP.put(DataType.Name.TIMESTAMP, TypeCodec.timestamp());
+ BUILT_IN_CODECS_MAP.put(DataType.Name.UUID, TypeCodec.uuid());
+ BUILT_IN_CODECS_MAP.put(DataType.Name.VARCHAR, TypeCodec.varchar());
+ BUILT_IN_CODECS_MAP.put(DataType.Name.VARINT, TypeCodec.varint());
+ BUILT_IN_CODECS_MAP.put(DataType.Name.TIMEUUID, TypeCodec.timeUUID());
+ BUILT_IN_CODECS_MAP.put(DataType.Name.SMALLINT, TypeCodec.smallInt());
+ BUILT_IN_CODECS_MAP.put(DataType.Name.TINYINT, TypeCodec.tinyInt());
+ BUILT_IN_CODECS_MAP.put(DataType.Name.DATE, TypeCodec.date());
+ BUILT_IN_CODECS_MAP.put(DataType.Name.TIME, TypeCodec.time());
+ BUILT_IN_CODECS_MAP.put(DataType.Name.DURATION, TypeCodec.duration());
+ }
+
+ // roughly sorted by popularity
+ private static final TypeCodec<?>[] BUILT_IN_CODECS =
+ new TypeCodec<?>[]{
+ TypeCodec
+ .varchar(), // must be declared before AsciiCodec so it gets chosen when CQL type not
+ // available
+ TypeCodec
+ .uuid(), // must be declared before TimeUUIDCodec so it gets chosen when CQL type not
+ // available
+ TypeCodec.timeUUID(),
+ TypeCodec.timestamp(),
+ TypeCodec.cint(),
+ TypeCodec.bigint(),
+ TypeCodec.blob(),
+ TypeCodec.cdouble(),
+ TypeCodec.cfloat(),
+ TypeCodec.decimal(),
+ TypeCodec.varint(),
+ TypeCodec.inet(),
+ TypeCodec.cboolean(),
+ TypeCodec.smallInt(),
+ TypeCodec.tinyInt(),
+ TypeCodec.date(),
+ TypeCodec.time(),
+ TypeCodec.duration(),
+ TypeCodec.counter(),
+ TypeCodec.ascii()
+ };
+
+ /**
+ * Cache key for the codecs cache.
+ */
+ private static final class CacheKey
+ {
+
+ private final DataType cqlType;
+
+ private final TypeToken<?> javaType;
+
+ CacheKey(DataType cqlType, TypeToken<?> javaType)
+ {
+ this.javaType = javaType;
+ this.cqlType = cqlType;
+ }
+
+ @Override
+ public boolean equals(Object o)
+ {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ CacheKey cacheKey = (CacheKey) o;
+ return Objects.equals(cqlType, cacheKey.cqlType)
+ && Objects.equals(javaType, cacheKey.javaType);
+ }
+
+ @Override
+ public int hashCode()
+ {
+ return Objects.hash(cqlType, javaType);
+ }
+ }
+
+ /**
+ * Cache loader for the codecs cache.
+ */
+ private class TypeCodecCacheLoader extends CacheLoader<CacheKey, TypeCodec<?>>
+ {
+ @Override
+ public TypeCodec<?> load(CacheKey cacheKey)
+ {
+ checkNotNull(cacheKey.cqlType, "Parameter cqlType cannot be null");
+ if (logger.isTraceEnabled())
+ logger.trace(
+ "Loading codec into cache: [{} <-> {}]",
+ CodecRegistry.toString(cacheKey.cqlType),
+ CodecRegistry.toString(cacheKey.javaType));
+ for (TypeCodec<?> codec : codecs)
+ {
+ if (codec.accepts(cacheKey.cqlType)
+ && (cacheKey.javaType == null || codec.accepts(cacheKey.javaType)))
+ {
+ logger.trace("Already existing codec found: {}", codec);
+ return codec;
+ }
+ }
+ return createCodec(cacheKey.cqlType, cacheKey.javaType);
+ }
+ }
+
+ /**
+ * A complexity-based weigher for the codecs cache. Weights are computed mainly according to the
+ * CQL type:
+ *
+ * <ol>
+ * <li>Manually-registered codecs always weigh 0;
+ * <li>Codecs for primitive types weigh 0;
+ * <li>Codecs for collections weigh the total weight of their inner types + the weight of their
+ * level of deepness;
+ * <li>Codecs for UDTs and tuples weigh the total weight of their inner types + the weight of
+ * their level of deepness, but cannot weigh less than 1;
+ * <li>Codecs for custom (non-CQL) types weigh 1.
+ * </ol>
+ * <p>
+ * A consequence of this algorithm is that codecs for primitive types and codecs for all "shallow"
+ * collections thereof are never evicted.
+ */
+ private class TypeCodecWeigher implements Weigher<CacheKey, TypeCodec<?>>
+ {
+
+ @Override
+ public int weigh(CacheKey key, TypeCodec<?> value)
+ {
+ return codecs.contains(value) ? 0 : weigh(value.cqlType, 0);
+ }
+
+ private int weigh(DataType cqlType, int level)
+ {
+ switch (cqlType.getName())
+ {
+ case LIST:
+ case SET:
+ case MAP:
+ {
+ int weight = level;
+ for (DataType eltType : cqlType.getTypeArguments())
+ {
+ weight += weigh(eltType, level + 1);
+ }
+ return weight;
+ }
+ case UDT:
+ {
+ int weight = level;
+ for (UserType.Field field : ((UserType) cqlType))
+ {
+ weight += weigh(field.getType(), level + 1);
+ }
+ return weight == 0 ? 1 : weight;
+ }
+ case TUPLE:
+ {
+ int weight = level;
+ for (DataType componentType : ((TupleType) cqlType).getComponentTypes())
+ {
+ weight += weigh(componentType, level + 1);
+ }
+ return weight == 0 ? 1 : weight;
+ }
+ case CUSTOM:
+ return 1;
+ default:
+ return 0;
+ }
+ }
+ }
+
+ /**
+ * Simple removal listener for the codec cache (can be used for debugging purposes by setting the
+ * {@code CodecRegistry} logger level to {@code TRACE}.
+ */
+ private static class TypeCodecRemovalListener implements RemovalListener<CacheKey, TypeCodec<?>>
+ {
+ @Override
+ public void onRemoval(RemovalNotification<CacheKey, TypeCodec<?>> notification)
+ {
+ logger.trace(
+ "Evicting codec from cache: {} (cause: {})",
+ notification.getValue(),
+ notification.getCause());
+ }
+ }
+
+ /**
+ * The list of user-registered codecs.
+ */
+ private final CopyOnWriteArrayList<TypeCodec<?>> codecs;
+
+ /**
+ * A LoadingCache to serve requests for codecs whenever possible. The cache can be used as long as
+ * at least the CQL type is known.
+ */
+ private final LoadingCache<CacheKey, TypeCodec<?>> cache;
+
+ /**
+ * Creates a new instance initialized with built-in codecs for all the base CQL types.
+ */
+ public CodecRegistry()
+ {
+ this.codecs = new CopyOnWriteArrayList<>();
+ this.cache = defaultCacheBuilder().build(new TypeCodecCacheLoader());
+ }
+
+ private CacheBuilder<CacheKey, TypeCodec<?>> defaultCacheBuilder()
+ {
+ CacheBuilder<CacheKey, TypeCodec<?>> builder =
+ CacheBuilder.newBuilder()
+ // lists, sets and maps of 20 primitive types = 20 + 20 + 20*20 = 440 codecs,
+ // so let's start with roughly 1/4 of that
+ .initialCapacity(100)
+ .maximumWeight(1000)
+ .weigher(new TypeCodecWeigher());
+ if (logger.isTraceEnabled())
+ // do not bother adding a listener if it will be ineffective
+ builder = builder.removalListener(new TypeCodecRemovalListener());
+ return builder;
+ }
+
+ /**
+ * Register the given codec with this registry.
+ *
+ * <p>This method will log a warning and ignore the codec if it collides with a previously
+ * registered one. Note that this check is not done in a completely thread-safe manner; codecs
+ * should typically be registered at application startup, not in a highly concurrent context (if a
+ * race condition occurs, the worst possible outcome is that no warning gets logged, and the codec
+ * gets registered but will never actually be used).
+ *
+ * @param newCodec The codec to add to the registry.
+ * @return this CodecRegistry (for method chaining).
+ */
+ public CodecRegistry register(TypeCodec<?> newCodec)
+ {
+ for (TypeCodec<?> oldCodec : BUILT_IN_CODECS)
+ {
+ if (oldCodec.accepts(newCodec.getCqlType()) && oldCodec.accepts(newCodec.getJavaType()))
+ {
+ logger.warn(
+ "Ignoring codec {} because it collides with previously registered codec {}",
+ newCodec,
+ oldCodec);
+ return this;
+ }
+ }
+ for (TypeCodec<?> oldCodec : codecs)
+ {
+ if (oldCodec.accepts(newCodec.getCqlType()) && oldCodec.accepts(newCodec.getJavaType()))
+ {
+ logger.warn(
+ "Ignoring codec {} because it collides with previously registered codec {}",
+ newCodec,
+ oldCodec);
+ return this;
+ }
+ }
+ CacheKey key = new CacheKey(newCodec.getCqlType(), newCodec.getJavaType());
+ TypeCodec<?> existing = cache.getIfPresent(key);
+ if (existing != null)
+ {
+ logger.warn(
+ "Ignoring codec {} because it collides with previously generated codec {}",
+ newCodec,
+ existing);
+ return this;
+ }
+ this.codecs.add(newCodec);
+ return this;
+ }
+
+ /**
+ * Register the given codecs with this registry.
+ *
+ * @param codecs The codecs to add to the registry.
+ * @return this CodecRegistry (for method chaining).
+ * @see #register(TypeCodec)
+ */
+ public CodecRegistry register(TypeCodec<?>... codecs)
+ {
+ for (TypeCodec<?> codec : codecs) register(codec);
+ return this;
+ }
+
+ /**
+ * Register the given codecs with this registry.
+ *
+ * @param codecs The codecs to add to the registry.
+ * @return this CodecRegistry (for method chaining).
+ * @see #register(TypeCodec)
+ */
+ public CodecRegistry register(Iterable<? extends TypeCodec<?>> codecs)
+ {
+ for (TypeCodec<?> codec : codecs) register(codec);
+ return this;
+ }
+
+ /**
+ * Returns a {@link TypeCodec codec} that accepts the given value.
+ *
+ * <p>This method takes an arbitrary Java object and tries to locate a suitable codec for it.
+ * Codecs must perform a {@link TypeCodec#accepts(Object) runtime inspection} of the object to
+ * determine if they can accept it or not, which, depending on the implementations, can be
+ * expensive; besides, the resulting codec cannot be cached. Therefore there might be a
+ * performance penalty when using this method.
+ *
+ * <p>Furthermore, this method returns the first matching codec, regardless of its accepted CQL
+ * type. It should be reserved for situations where the target CQL type is not available or
+ * unknown. In the Java driver, this happens mainly when serializing a value in a {@code
+ * SimpleStatement#SimpleStatement(String, Object...) SimpleStatement} or in the {@code
+ * querybuilder.QueryBuilder}, where no CQL type information is
+ * available.
+ *
+ * <p>Codecs returned by this method are <em>NOT</em> cached (see the {@link CodecRegistry
+ * top-level documentation} of this class for more explanations about caching).
+ *
+ * @param value The value the codec should accept; must not be {@code null}.
+ * @return A suitable codec.
+ * @throws CodecNotFoundException if a suitable codec cannot be found.
+ */
+ public <T> TypeCodec<T> codecFor(T value)
+ {
+ return findCodec(null, value);
+ }
+
+ /**
+ * Returns a {@link TypeCodec codec} that accepts the given {@link DataType CQL type}.
+ *
+ * <p>This method returns the first matching codec, regardless of its accepted Java type. It
+ * should be reserved for situations where the Java type is not available or unknown. In the Java
+ * driver, this happens mainly when deserializing a value using the {@link
+ * GettableByIndexData#getObject(int) getObject} method.
+ *
+ * <p>Codecs returned by this method are cached (see the {@link CodecRegistry top-level
+ * documentation} of this class for more explanations about caching).
+ *
+ * @param cqlType The {@link DataType CQL type} the codec should accept; must not be {@code null}.
+ * @return A suitable codec.
+ * @throws CodecNotFoundException if a suitable codec cannot be found.
+ */
+ public <T> TypeCodec<T> codecFor(DataType cqlType) throws CodecNotFoundException
+ {
+ return lookupCodec(cqlType, null);
+ }
+
+ /**
+ * Returns a {@link TypeCodec codec} that accepts the given {@link DataType CQL type} and the
+ * given Java class.
+ *
+ * <p>This method can only handle raw (non-parameterized) Java types. For parameterized types, use
+ * {@link #codecFor(DataType, TypeToken)} instead.
+ *
+ * <p>Codecs returned by this method are cached (see the {@link CodecRegistry top-level
+ * documentation} of this class for more explanations about caching).
+ *
+ * @param cqlType The {@link DataType CQL type} the codec should accept; must not be {@code null}.
+ * @param javaType The Java type the codec should accept; can be {@code null}.
+ * @return A suitable codec.
+ * @throws CodecNotFoundException if a suitable codec cannot be found.
+ */
+ public <T> TypeCodec<T> codecFor(DataType cqlType, Class<T> javaType)
+ throws CodecNotFoundException
+ {
+ return codecFor(cqlType, TypeToken.of(javaType));
+ }
+
+ /**
+ * Returns a {@link TypeCodec codec} that accepts the given {@link DataType CQL type} and the
+ * given Java type.
+ *
+ * <p>This method handles parameterized types thanks to Guava's {@link TypeToken} API.
+ *
+ * <p>Codecs returned by this method are cached (see the {@link CodecRegistry top-level
+ * documentation} of this class for more explanations about caching).
+ *
+ * @param cqlType The {@link DataType CQL type} the codec should accept; must not be {@code null}.
+ * @param javaType The {@link TypeToken Java type} the codec should accept; can be {@code null}.
+ * @return A suitable codec.
+ * @throws CodecNotFoundException if a suitable codec cannot be found.
+ */
+ public <T> TypeCodec<T> codecFor(DataType cqlType, TypeToken<T> javaType)
+ throws CodecNotFoundException
+ {
+ return lookupCodec(cqlType, javaType);
+ }
+
+ /**
+ * Returns a {@link TypeCodec codec} that accepts the given {@link DataType CQL type} and the
+ * given value.
+ *
+ * <p>This method takes an arbitrary Java object and tries to locate a suitable codec for it.
+ * Codecs must perform a {@link TypeCodec#accepts(Object) runtime inspection} of the object to
+ * determine if they can accept it or not, which, depending on the implementations, can be
+ * expensive; besides, the resulting codec cannot be cached. Therefore there might be a
+ * performance penalty when using this method.
+ *
+ * <p>Codecs returned by this method are <em>NOT</em> cached (see the {@link CodecRegistry
+ * top-level documentation} of this class for more explanations about caching).
+ *
+ * @param cqlType The {@link DataType CQL type} the codec should accept; can be {@code null}.
+ * @param value The value the codec should accept; must not be {@code null}.
+ * @return A suitable codec.
+ * @throws CodecNotFoundException if a suitable codec cannot be found.
+ */
+ public <T> TypeCodec<T> codecFor(DataType cqlType, T value)
+ {
+ return findCodec(cqlType, value);
+ }
+
+ @SuppressWarnings("unchecked")
+ private <T> TypeCodec<T> lookupCodec(DataType cqlType, TypeToken<T> javaType)
+ {
+ checkNotNull(cqlType, "Parameter cqlType cannot be null");
+ TypeCodec<?> codec = BUILT_IN_CODECS_MAP.get(cqlType.getName());
+ if (codec != null && (javaType == null || codec.accepts(javaType)))
+ {
+ logger.trace("Returning built-in codec {}", codec);
+ return (TypeCodec<T>) codec;
+ }
+ if (logger.isTraceEnabled())
+ logger.trace("Querying cache for codec [{} <-> {}]", toString(cqlType), toString(javaType));
+ try
+ {
+ CacheKey cacheKey = new CacheKey(cqlType, javaType);
+ codec = cache.get(cacheKey);
+ }
+ catch (UncheckedExecutionException e)
+ {
+ if (e.getCause() instanceof CodecNotFoundException)
+ {
+ throw (CodecNotFoundException) e.getCause();
+ }
+ throw new CodecNotFoundException(e.getCause());
+ }
+ catch (RuntimeException | ExecutionException e)
+ {
+ throw new CodecNotFoundException(e.getCause());
+ }
+ logger.trace("Returning cached codec {}", codec);
+ return (TypeCodec<T>) codec;
+ }
+
+ @SuppressWarnings("unchecked")
+ private <T> TypeCodec<T> findCodec(DataType cqlType, TypeToken<T> javaType)
+ {
+ checkNotNull(cqlType, "Parameter cqlType cannot be null");
+ if (logger.isTraceEnabled())
+ logger.trace("Looking for codec [{} <-> {}]", toString(cqlType), toString(javaType));
+
+ // Look at the built-in codecs first
+ for (TypeCodec<?> codec : BUILT_IN_CODECS)
+ {
+ if (codec.accepts(cqlType) && (javaType == null || codec.accepts(javaType)))
+ {
+ logger.trace("Built-in codec found: {}", codec);
+ return (TypeCodec<T>) codec;
+ }
+ }
+
+ // Look at the user-registered codecs next
+ for (TypeCodec<?> codec : codecs)
+ {
+ if (codec.accepts(cqlType) && (javaType == null || codec.accepts(javaType)))
+ {
+ logger.trace("Already registered codec found: {}", codec);
+ return (TypeCodec<T>) codec;
+ }
+ }
+ return createCodec(cqlType, javaType);
+ }
+
+ @SuppressWarnings("unchecked")
+ private <T> TypeCodec<T> findCodec(DataType cqlType, T value)
+ {
+ checkNotNull(value, "Parameter value cannot be null");
+ if (logger.isTraceEnabled())
+ logger.trace("Looking for codec [{} <-> {}]", toString(cqlType), value.getClass());
+
+ // Look at the built-in codecs first
+ for (TypeCodec<?> codec : BUILT_IN_CODECS)
+ {
+ if ((cqlType == null || codec.accepts(cqlType)) && codec.accepts(value))
+ {
+ logger.trace("Built-in codec found: {}", codec);
+ return (TypeCodec<T>) codec;
+ }
+ }
+
+ // Look at the user-registered codecs next
+ for (TypeCodec<?> codec : codecs)
+ {
+ if ((cqlType == null || codec.accepts(cqlType)) && codec.accepts(value))
+ {
+ logger.trace("Already registered codec found: {}", codec);
+ return (TypeCodec<T>) codec;
+ }
+ }
+ return createCodec(cqlType, value);
+ }
+
+ private <T> TypeCodec<T> createCodec(DataType cqlType, TypeToken<T> javaType)
+ {
+ TypeCodec<T> codec = maybeCreateCodec(cqlType, javaType);
+ if (codec == null) throw notFound(cqlType, javaType);
+ // double-check that the created codec satisfies the initial request
+ // this check can fail specially when creating codecs for collections
+ // e.g. if B extends A and there is a codec registered for A and
+ // we request a codec for List<B>, the registry would generate a codec for List<A>
+ if (!codec.accepts(cqlType) || (javaType != null && !codec.accepts(javaType)))
+ throw notFound(cqlType, javaType);
+ logger.trace("Codec created: {}", codec);
+ return codec;
+ }
+
+ private <T> TypeCodec<T> createCodec(DataType cqlType, T value)
+ {
+ TypeCodec<T> codec = maybeCreateCodec(cqlType, value);
+ if (codec == null) throw notFound(cqlType, TypeToken.of(value.getClass()));
+ // double-check that the created codec satisfies the initial request
+ if ((cqlType != null && !codec.accepts(cqlType)) || !codec.accepts(value))
+ throw notFound(cqlType, TypeToken.of(value.getClass()));
+ logger.trace("Codec created: {}", codec);
+ return codec;
+ }
+
+ @SuppressWarnings("unchecked")
+ private <T> TypeCodec<T> maybeCreateCodec(DataType cqlType, TypeToken<T> javaType)
+ {
+ checkNotNull(cqlType);
+
+ if (cqlType.getName() == LIST
+ && (javaType == null || List.class.isAssignableFrom(javaType.getRawType())))
+ {
+ TypeToken<?> elementType = null;
+ if (javaType != null && javaType.getType() instanceof ParameterizedType)
+ {
+ Type[] typeArguments = ((ParameterizedType) javaType.getType()).getActualTypeArguments();
+ elementType = TypeToken.of(typeArguments[0]);
+ }
+ TypeCodec<?> eltCodec = findCodec(cqlType.getTypeArguments().get(0), elementType);
+ return (TypeCodec<T>) TypeCodec.list(eltCodec);
+ }
+
+ if (cqlType.getName() == SET
+ && (javaType == null || Set.class.isAssignableFrom(javaType.getRawType())))
+ {
+ TypeToken<?> elementType = null;
+ if (javaType != null && javaType.getType() instanceof ParameterizedType)
+ {
+ Type[] typeArguments = ((ParameterizedType) javaType.getType()).getActualTypeArguments();
+ elementType = TypeToken.of(typeArguments[0]);
+ }
+ TypeCodec<?> eltCodec = findCodec(cqlType.getTypeArguments().get(0), elementType);
+ return (TypeCodec<T>) TypeCodec.set(eltCodec);
+ }
+
+ if (cqlType.getName() == MAP
+ && (javaType == null || Map.class.isAssignableFrom(javaType.getRawType())))
+ {
+ TypeToken<?> keyType = null;
+ TypeToken<?> valueType = null;
+ if (javaType != null && javaType.getType() instanceof ParameterizedType)
+ {
+ Type[] typeArguments = ((ParameterizedType) javaType.getType()).getActualTypeArguments();
+ keyType = TypeToken.of(typeArguments[0]);
+ valueType = TypeToken.of(typeArguments[1]);
+ }
+ TypeCodec<?> keyCodec = findCodec(cqlType.getTypeArguments().get(0), keyType);
+ TypeCodec<?> valueCodec = findCodec(cqlType.getTypeArguments().get(1), valueType);
+ return (TypeCodec<T>) TypeCodec.map(keyCodec, valueCodec);
+ }
+
+ if (cqlType instanceof TupleType
+ && (javaType == null || TupleValue.class.isAssignableFrom(javaType.getRawType())))
+ {
+ return (TypeCodec<T>) TypeCodec.tuple((TupleType) cqlType);
+ }
+
+ if (cqlType instanceof UserType
+ && (javaType == null || UDTValue.class.isAssignableFrom(javaType.getRawType())))
+ {
+ return (TypeCodec<T>) TypeCodec.userType((UserType) cqlType);
+ }
+
+ if (cqlType instanceof DataType.CustomType
+ && (javaType == null || ByteBuffer.class.isAssignableFrom(javaType.getRawType())))
+ {
+ return (TypeCodec<T>) TypeCodec.custom((DataType.CustomType) cqlType);
+ }
+
+ return null;
+ }
+
+ @SuppressWarnings({ "unchecked", "rawtypes" })
+ private <T> TypeCodec<T> maybeCreateCodec(DataType cqlType, T value)
+ {
+ checkNotNull(value);
+
+ if ((cqlType == null || cqlType.getName() == LIST) && value instanceof List)
+ {
+ List list = (List) value;
+ if (list.isEmpty())
+ {
+ DataType elementType =
+ (cqlType == null || cqlType.getTypeArguments().isEmpty())
+ ? DataType.blob()
+ : cqlType.getTypeArguments().get(0);
+ return (TypeCodec<T>) TypeCodec.list(findCodec(elementType, (TypeToken) null));
+ }
+ else
+ {
+ DataType elementType =
+ (cqlType == null || cqlType.getTypeArguments().isEmpty())
+ ? null
+ : cqlType.getTypeArguments().get(0);
+ return (TypeCodec<T>) TypeCodec.list(findCodec(elementType, list.iterator().next()));
+ }
+ }
+
+ if ((cqlType == null || cqlType.getName() == SET) && value instanceof Set)
+ {
+ Set set = (Set) value;
+ if (set.isEmpty())
+ {
+ DataType elementType =
+ (cqlType == null || cqlType.getTypeArguments().isEmpty())
+ ? DataType.blob()
+ : cqlType.getTypeArguments().get(0);
+ return (TypeCodec<T>) TypeCodec.set(findCodec(elementType, (TypeToken) null));
+ }
+ else
+ {
+ DataType elementType =
+ (cqlType == null || cqlType.getTypeArguments().isEmpty())
+ ? null
+ : cqlType.getTypeArguments().get(0);
+ return (TypeCodec<T>) TypeCodec.set(findCodec(elementType, set.iterator().next()));
+ }
+ }
+
+ if ((cqlType == null || cqlType.getName() == MAP) && value instanceof Map)
+ {
+ Map map = (Map) value;
+ if (map.isEmpty())
+ {
+ DataType keyType =
+ (cqlType == null || cqlType.getTypeArguments().size() < 1)
+ ? DataType.blob()
+ : cqlType.getTypeArguments().get(0);
+ DataType valueType =
+ (cqlType == null || cqlType.getTypeArguments().size() < 2)
+ ? DataType.blob()
+ : cqlType.getTypeArguments().get(1);
+ return (TypeCodec<T>) TypeCodec.map(
+ findCodec(keyType, (TypeToken) null), findCodec(valueType, (TypeToken) null));
+ }
+ else
+ {
+ DataType keyType =
+ (cqlType == null || cqlType.getTypeArguments().size() < 1)
+ ? null
+ : cqlType.getTypeArguments().get(0);
+ DataType valueType =
+ (cqlType == null || cqlType.getTypeArguments().size() < 2)
+ ? null
+ : cqlType.getTypeArguments().get(1);
+ Map.Entry entry = (Map.Entry) map.entrySet().iterator().next();
+ return (TypeCodec<T>)
+ TypeCodec.map(
+ findCodec(keyType, entry.getKey()), findCodec(valueType, entry.getValue()));
+ }
+ }
+
+ if ((cqlType == null || cqlType.getName() == DataType.Name.TUPLE)
+ && value instanceof TupleValue)
+ {
+ return (TypeCodec<T>)
+ TypeCodec.tuple(cqlType == null ? ((TupleValue) value).getType() : (TupleType) cqlType);
+ }
+
+ if ((cqlType == null || cqlType.getName() == DataType.Name.UDT) && value instanceof UDTValue)
+ {
+ return (TypeCodec<T>)
+ TypeCodec.userType(cqlType == null ? ((UDTValue) value).getType() : (UserType) cqlType);
+ }
+
+ if ((cqlType instanceof DataType.CustomType)
+ && value instanceof ByteBuffer)
+ {
+ return (TypeCodec<T>) TypeCodec.custom((DataType.CustomType) cqlType);
+ }
+
+ return null;
+ }
+
+ private static CodecNotFoundException notFound(DataType cqlType, TypeToken<?> javaType)
+ {
+ String msg =
+ String.format(
+ "Codec not found for requested operation: [%s <-> %s]",
+ toString(cqlType), toString(javaType));
+ return new CodecNotFoundException(msg);
+ }
+
+ private static String toString(Object value)
+ {
+ return value == null ? "ANY" : value.toString();
+ }
+}
diff --git a/src/java/org/apache/cassandra/cql3/functions/types/CodecUtils.java b/src/java/org/apache/cassandra/cql3/functions/types/CodecUtils.java
new file mode 100644
index 0000000..4e97e5c
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/functions/types/CodecUtils.java
@@ -0,0 +1,266 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.functions.types;
+
+import java.nio.ByteBuffer;
+
+import org.apache.cassandra.transport.ProtocolVersion;
+
+/**
+ * A set of utility methods to deal with type conversion and serialization.
+ */
+public final class CodecUtils
+{
+
+ private static final long MAX_CQL_LONG_VALUE = ((1L << 32) - 1);
+
+ private static final long EPOCH_AS_CQL_LONG = (1L << 31);
+
+ private CodecUtils()
+ {
+ }
+
+ /**
+ * Utility method that "packs" together a list of {@link ByteBuffer}s containing serialized
+ * collection elements. Mainly intended for use with collection codecs when serializing
+ * collections.
+ *
+ * @param buffers the collection elements
+ * @param elements the total number of elements
+ * @param version the protocol version to use
+ * @return The serialized collection
+ */
+ public static ByteBuffer pack(ByteBuffer[] buffers, int elements, ProtocolVersion version)
+ {
+ int size = 0;
+ for (ByteBuffer bb : buffers)
+ {
+ int elemSize = sizeOfValue(bb, version);
+ size += elemSize;
+ }
+ ByteBuffer result = ByteBuffer.allocate(sizeOfCollectionSize(version) + size);
+ writeSize(result, elements, version);
+ for (ByteBuffer bb : buffers) writeValue(result, bb, version);
+ return (ByteBuffer) result.flip();
+ }
+
+ /**
+ * Utility method that reads a size value. Mainly intended for collection codecs when
+ * deserializing CQL collections.
+ *
+ * @param input The ByteBuffer to read from.
+ * @param version The protocol version to use.
+ * @return The size value.
+ */
+ static int readSize(ByteBuffer input, ProtocolVersion version)
+ {
+ switch (version)
+ {
+ case V1:
+ case V2:
+ return getUnsignedShort(input);
+ case V3:
+ case V4:
+ case V5:
+ return input.getInt();
+ default:
+ throw new IllegalArgumentException(String.valueOf(version));
+ }
+ }
+
+ /**
+ * Utility method that writes a size value. Mainly intended for collection codecs when serializing
+ * CQL collections.
+ *
+ * @param output The ByteBuffer to write to.
+ * @param size The collection size.
+ * @param version The protocol version to use.
+ */
+ private static void writeSize(ByteBuffer output, int size, ProtocolVersion version)
+ {
+ switch (version)
+ {
+ case V1:
+ case V2:
+ if (size > 65535)
+ throw new IllegalArgumentException(
+ String.format(
+ "Native protocol version %d supports up to 65535 elements in any collection - but collection contains %d elements",
+ version.asInt(), size));
+ output.putShort((short) size);
+ break;
+ case V3:
+ case V4:
+ case V5:
+ output.putInt(size);
+ break;
+ default:
+ throw new IllegalArgumentException(String.valueOf(version));
+ }
+ }
+
+ /**
+ * Utility method that reads a value. Mainly intended for collection codecs when deserializing CQL
+ * collections.
+ *
+ * @param input The ByteBuffer to read from.
+ * @param version The protocol version to use.
+ * @return The collection element.
+ */
+ public static ByteBuffer readValue(ByteBuffer input, ProtocolVersion version)
+ {
+ int size = readSize(input, version);
+ return size < 0 ? null : readBytes(input, size);
+ }
+
+ /**
+ * Utility method that writes a value. Mainly intended for collection codecs when deserializing
+ * CQL collections.
+ *
+ * @param output The ByteBuffer to write to.
+ * @param value The value to write.
+ * @param version The protocol version to use.
+ */
+ public static void writeValue(ByteBuffer output, ByteBuffer value, ProtocolVersion version)
+ {
+ switch (version)
+ {
+ case V1:
+ case V2:
+ assert value != null;
+ output.putShort((short) value.remaining());
+ output.put(value.duplicate());
+ break;
+ case V3:
+ case V4:
+ case V5:
+ if (value == null)
+ {
+ output.putInt(-1);
+ }
+ else
+ {
+ output.putInt(value.remaining());
+ output.put(value.duplicate());
+ }
+ break;
+ default:
+ throw new IllegalArgumentException(String.valueOf(version));
+ }
+ }
+
+ /**
+ * Read {@code length} bytes from {@code bb} into a new ByteBuffer.
+ *
+ * @param bb The ByteBuffer to read.
+ * @param length The number of bytes to read.
+ * @return The read bytes.
+ */
+ public static ByteBuffer readBytes(ByteBuffer bb, int length)
+ {
+ ByteBuffer copy = bb.duplicate();
+ copy.limit(copy.position() + length);
+ bb.position(bb.position() + length);
+ return copy;
+ }
+
+ /**
+ * Converts an "unsigned" int read from a DATE value into a signed int.
+ *
+ * <p>The protocol encodes DATE values as <em>unsigned</em> ints with the Epoch in the middle of
+ * the range (2^31). This method handles the conversion from an "unsigned" to a signed int.
+ */
+ static int fromUnsignedToSignedInt(int unsigned)
+ {
+ return unsigned + Integer.MIN_VALUE; // this relies on overflow for "negative" values
+ }
+
+ /**
+ * Converts an int into an "unsigned" int suitable to be written as a DATE value.
+ *
+ * <p>The protocol encodes DATE values as <em>unsigned</em> ints with the Epoch in the middle of
+ * the range (2^31). This method handles the conversion from a signed to an "unsigned" int.
+ */
+ static int fromSignedToUnsignedInt(int signed)
+ {
+ return signed - Integer.MIN_VALUE;
+ }
+
+ /**
+ * Convert from a raw CQL long representing a numeric DATE literal to the number of days since the
+ * Epoch. In CQL, numeric DATE literals are longs (unsigned integers actually) between 0 and 2^32
+ * - 1, with the epoch in the middle; this method re-centers the epoch at 0.
+ *
+ * @param raw The CQL date value to convert.
+ * @return The number of days since the Epoch corresponding to the given raw value.
+ * @throws IllegalArgumentException if the value is out of range.
+ */
+ static int fromCqlDateToDaysSinceEpoch(long raw)
+ {
+ if (raw < 0 || raw > MAX_CQL_LONG_VALUE)
+ throw new IllegalArgumentException(
+ String.format(
+ "Numeric literals for DATE must be between 0 and %d (got %d)",
+ MAX_CQL_LONG_VALUE, raw));
+ return (int) (raw - EPOCH_AS_CQL_LONG);
+ }
+
+ private static int sizeOfCollectionSize(ProtocolVersion version)
+ {
+ switch (version)
+ {
+ case V1:
+ case V2:
+ return 2;
+ case V3:
+ case V4:
+ case V5:
+ return 4;
+ default:
+ throw new IllegalArgumentException(String.valueOf(version));
+ }
+ }
+
+ private static int sizeOfValue(ByteBuffer value, ProtocolVersion version)
+ {
+ switch (version)
+ {
+ case V1:
+ case V2:
+ int elemSize = value.remaining();
+ if (elemSize > 65535)
+ throw new IllegalArgumentException(
+ String.format(
+ "Native protocol version %d supports only elements with size up to 65535 bytes - but element size is %d bytes",
+ version.asInt(), elemSize));
+ return 2 + elemSize;
+ case V3:
+ case V4:
+ case V5:
+ return value == null ? 4 : 4 + value.remaining();
+ default:
+ throw new IllegalArgumentException(String.valueOf(version));
+ }
+ }
+
+ private static int getUnsignedShort(ByteBuffer bb)
+ {
+ int length = (bb.get() & 0xFF) << 8;
+ return length | (bb.get() & 0xFF);
+ }
+}
diff --git a/src/java/org/apache/cassandra/cql3/functions/types/DataType.java b/src/java/org/apache/cassandra/cql3/functions/types/DataType.java
new file mode 100644
index 0000000..5412720
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/functions/types/DataType.java
@@ -0,0 +1,703 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.functions.types;
+
+import java.util.*;
+
+import com.google.common.collect.ImmutableList;
+
+import org.apache.cassandra.transport.ProtocolVersion;
+
+/**
+ * Data types supported by cassandra.
+ */
+public abstract class DataType
+{
+
+ /**
+ * The CQL type name.
+ */
+ public enum Name
+ {
+ CUSTOM(0),
+ ASCII(1),
+ BIGINT(2),
+ BLOB(3),
+ BOOLEAN(4),
+ COUNTER(5),
+ DECIMAL(6),
+ DOUBLE(7),
+ FLOAT(8),
+ INT(9),
+ TEXT(10)
+ {
+ @Override
+ public boolean isCompatibleWith(Name that)
+ {
+ return this == that || that == VARCHAR;
+ }
+ },
+ TIMESTAMP(11),
+ UUID(12),
+ VARCHAR(13)
+ {
+ @Override
+ public boolean isCompatibleWith(Name that)
+ {
+ return this == that || that == TEXT;
+ }
+ },
+ VARINT(14),
+ TIMEUUID(15),
+ INET(16),
+ DATE(17, ProtocolVersion.V4),
+ TIME(18, ProtocolVersion.V4),
+ SMALLINT(19, ProtocolVersion.V4),
+ TINYINT(20, ProtocolVersion.V4),
+ DURATION(21, ProtocolVersion.V5),
+ LIST(32),
+ MAP(33),
+ SET(34),
+ UDT(48, ProtocolVersion.V3),
+ TUPLE(49, ProtocolVersion.V3);
+
+ final int protocolId;
+
+ final ProtocolVersion minProtocolVersion;
+
+ private static final Name[] nameToIds;
+
+ static
+ {
+ int maxCode = -1;
+ for (Name name : Name.values()) maxCode = Math.max(maxCode, name.protocolId);
+ nameToIds = new Name[maxCode + 1];
+ for (Name name : Name.values())
+ {
+ if (nameToIds[name.protocolId] != null) throw new IllegalStateException("Duplicate Id");
+ nameToIds[name.protocolId] = name;
+ }
+ }
+
+ Name(int protocolId)
+ {
+ this(protocolId, ProtocolVersion.V1);
+ }
+
+ Name(int protocolId, ProtocolVersion minProtocolVersion)
+ {
+ this.protocolId = protocolId;
+ this.minProtocolVersion = minProtocolVersion;
+ }
+
+ /**
+ * Return {@code true} if the provided Name is equal to this one, or if they are aliases for
+ * each other, and {@code false} otherwise.
+ *
+ * @param that the Name to compare with the current one.
+ * @return {@code true} if the provided Name is equal to this one, or if they are aliases for
+ * each other, and {@code false} otherwise.
+ */
+ public boolean isCompatibleWith(Name that)
+ {
+ return this == that;
+ }
+
+ @Override
+ public String toString()
+ {
+ return super.toString().toLowerCase();
+ }
+ }
+
+ private static final Map<Name, DataType> primitiveTypeMap =
+ new EnumMap<>(Name.class);
+
+ static
+ {
+ primitiveTypeMap.put(Name.ASCII, new DataType.NativeType(Name.ASCII));
+ primitiveTypeMap.put(Name.BIGINT, new DataType.NativeType(Name.BIGINT));
+ primitiveTypeMap.put(Name.BLOB, new DataType.NativeType(Name.BLOB));
+ primitiveTypeMap.put(Name.BOOLEAN, new DataType.NativeType(Name.BOOLEAN));
+ primitiveTypeMap.put(Name.COUNTER, new DataType.NativeType(Name.COUNTER));
+ primitiveTypeMap.put(Name.DECIMAL, new DataType.NativeType(Name.DECIMAL));
+ primitiveTypeMap.put(Name.DOUBLE, new DataType.NativeType(Name.DOUBLE));
+ primitiveTypeMap.put(Name.FLOAT, new DataType.NativeType(Name.FLOAT));
+ primitiveTypeMap.put(Name.INET, new DataType.NativeType(Name.INET));
+ primitiveTypeMap.put(Name.INT, new DataType.NativeType(Name.INT));
+ primitiveTypeMap.put(Name.TEXT, new DataType.NativeType(Name.TEXT));
+ primitiveTypeMap.put(Name.TIMESTAMP, new DataType.NativeType(Name.TIMESTAMP));
+ primitiveTypeMap.put(Name.UUID, new DataType.NativeType(Name.UUID));
+ primitiveTypeMap.put(Name.VARCHAR, new DataType.NativeType(Name.VARCHAR));
+ primitiveTypeMap.put(Name.VARINT, new DataType.NativeType(Name.VARINT));
+ primitiveTypeMap.put(Name.TIMEUUID, new DataType.NativeType(Name.TIMEUUID));
+ primitiveTypeMap.put(Name.SMALLINT, new DataType.NativeType(Name.SMALLINT));
+ primitiveTypeMap.put(Name.TINYINT, new DataType.NativeType(Name.TINYINT));
+ primitiveTypeMap.put(Name.DATE, new DataType.NativeType(Name.DATE));
+ primitiveTypeMap.put(Name.TIME, new DataType.NativeType(Name.TIME));
+ primitiveTypeMap.put(Name.DURATION, new DataType.NativeType(Name.DURATION));
+ }
+
+ protected final DataType.Name name;
+
+ protected DataType(DataType.Name name)
+ {
+ this.name = name;
+ }
+
+ /**
+ * Returns the ASCII type.
+ *
+ * @return The ASCII type.
+ */
+ public static DataType ascii()
+ {
+ return primitiveTypeMap.get(Name.ASCII);
+ }
+
+ /**
+ * Returns the BIGINT type.
+ *
+ * @return The BIGINT type.
+ */
+ public static DataType bigint()
+ {
+ return primitiveTypeMap.get(Name.BIGINT);
+ }
+
+ /**
+ * Returns the BLOB type.
+ *
+ * @return The BLOB type.
+ */
+ public static DataType blob()
+ {
+ return primitiveTypeMap.get(Name.BLOB);
+ }
+
+ /**
+ * Returns the BOOLEAN type.
+ *
+ * @return The BOOLEAN type.
+ */
+ public static DataType cboolean()
+ {
+ return primitiveTypeMap.get(Name.BOOLEAN);
+ }
+
+ /**
+ * Returns the COUNTER type.
+ *
+ * @return The COUNTER type.
+ */
+ public static DataType counter()
+ {
+ return primitiveTypeMap.get(Name.COUNTER);
+ }
+
+ /**
+ * Returns the DECIMAL type.
+ *
+ * @return The DECIMAL type.
+ */
+ public static DataType decimal()
+ {
+ return primitiveTypeMap.get(Name.DECIMAL);
+ }
+
+ /**
+ * Returns the DOUBLE type.
+ *
+ * @return The DOUBLE type.
+ */
+ public static DataType cdouble()
+ {
+ return primitiveTypeMap.get(Name.DOUBLE);
+ }
+
+ /**
+ * Returns the FLOAT type.
+ *
+ * @return The FLOAT type.
+ */
+ public static DataType cfloat()
+ {
+ return primitiveTypeMap.get(Name.FLOAT);
+ }
+
+ /**
+ * Returns the INET type.
+ *
+ * @return The INET type.
+ */
+ public static DataType inet()
+ {
+ return primitiveTypeMap.get(Name.INET);
+ }
+
+ /**
+ * Returns the TINYINT type.
+ *
+ * @return The TINYINT type.
+ */
+ public static DataType tinyint()
+ {
+ return primitiveTypeMap.get(Name.TINYINT);
+ }
+
+ /**
+ * Returns the SMALLINT type.
+ *
+ * @return The SMALLINT type.
+ */
+ public static DataType smallint()
+ {
+ return primitiveTypeMap.get(Name.SMALLINT);
+ }
+
+ /**
+ * Returns the INT type.
+ *
+ * @return The INT type.
+ */
+ public static DataType cint()
+ {
+ return primitiveTypeMap.get(Name.INT);
+ }
+
+ /**
+ * Returns the TEXT type.
+ *
+ * @return The TEXT type.
+ */
+ public static DataType text()
+ {
+ return primitiveTypeMap.get(Name.TEXT);
+ }
+
+ /**
+ * Returns the TIMESTAMP type.
+ *
+ * @return The TIMESTAMP type.
+ */
+ public static DataType timestamp()
+ {
+ return primitiveTypeMap.get(Name.TIMESTAMP);
+ }
+
+ /**
+ * Returns the DATE type.
+ *
+ * @return The DATE type.
+ */
+ public static DataType date()
+ {
+ return primitiveTypeMap.get(Name.DATE);
+ }
+
+ /**
+ * Returns the TIME type.
+ *
+ * @return The TIME type.
+ */
+ public static DataType time()
+ {
+ return primitiveTypeMap.get(Name.TIME);
+ }
+
+ /**
+ * Returns the UUID type.
+ *
+ * @return The UUID type.
+ */
+ public static DataType uuid()
+ {
+ return primitiveTypeMap.get(Name.UUID);
+ }
+
+ /**
+ * Returns the VARCHAR type.
+ *
+ * @return The VARCHAR type.
+ */
+ public static DataType varchar()
+ {
+ return primitiveTypeMap.get(Name.VARCHAR);
+ }
+
+ /**
+ * Returns the VARINT type.
+ *
+ * @return The VARINT type.
+ */
+ public static DataType varint()
+ {
+ return primitiveTypeMap.get(Name.VARINT);
+ }
+
+ /**
+ * Returns the TIMEUUID type.
+ *
+ * @return The TIMEUUID type.
+ */
+ public static DataType timeuuid()
+ {
+ return primitiveTypeMap.get(Name.TIMEUUID);
+ }
+
+ /**
+ * Returns the type of lists of {@code elementType} elements.
+ *
+ * @param elementType the type of the list elements.
+ * @param frozen whether the list is frozen.
+ * @return the type of lists of {@code elementType} elements.
+ */
+ public static CollectionType list(DataType elementType, boolean frozen)
+ {
+ return new DataType.CollectionType(Name.LIST, ImmutableList.of(elementType), frozen);
+ }
+
+ /**
+ * Returns the type of "not frozen" lists of {@code elementType} elements.
+ *
+ * <p>This is a shorthand for {@code list(elementType, false);}.
+ *
+ * @param elementType the type of the list elements.
+ * @return the type of "not frozen" lists of {@code elementType} elements.
+ */
+ public static CollectionType list(DataType elementType)
+ {
+ return list(elementType, false);
+ }
+
+ /**
+ * Returns the type of sets of {@code elementType} elements.
+ *
+ * @param elementType the type of the set elements.
+ * @param frozen whether the set is frozen.
+ * @return the type of sets of {@code elementType} elements.
+ */
+ public static CollectionType set(DataType elementType, boolean frozen)
+ {
+ return new DataType.CollectionType(Name.SET, ImmutableList.of(elementType), frozen);
+ }
+
+ /**
+ * Returns the type of "not frozen" sets of {@code elementType} elements.
+ *
+ * <p>This is a shorthand for {@code set(elementType, false);}.
+ *
+ * @param elementType the type of the set elements.
+ * @return the type of "not frozen" sets of {@code elementType} elements.
+ */
+ public static CollectionType set(DataType elementType)
+ {
+ return set(elementType, false);
+ }
+
+ /**
+ * Returns the type of maps of {@code keyType} to {@code valueType} elements.
+ *
+ * @param keyType the type of the map keys.
+ * @param valueType the type of the map values.
+ * @param frozen whether the map is frozen.
+ * @return the type of maps of {@code keyType} to {@code valueType} elements.
+ */
+ public static CollectionType map(DataType keyType, DataType valueType, boolean frozen)
+ {
+ return new DataType.CollectionType(Name.MAP, ImmutableList.of(keyType, valueType), frozen);
+ }
+
+ /**
+ * Returns the type of "not frozen" maps of {@code keyType} to {@code valueType} elements.
+ *
+ * <p>This is a shorthand for {@code map(keyType, valueType, false);}.
+ *
+ * @param keyType the type of the map keys.
+ * @param valueType the type of the map values.
+ * @return the type of "not frozen" maps of {@code keyType} to {@code valueType} elements.
+ */
+ public static CollectionType map(DataType keyType, DataType valueType)
+ {
+ return map(keyType, valueType, false);
+ }
+
+ /**
+ * Returns a Custom type.
+ *
+ * <p>A custom type is defined by the name of the class used on the Cassandra side to implement
+ * it. Note that the support for custom types by the driver is limited.
+ *
+ * <p>The use of custom types is rarely useful and is thus not encouraged.
+ *
+ * @param typeClassName the server-side fully qualified class name for the type.
+ * @return the custom type for {@code typeClassName}.
+ */
+ public static DataType.CustomType custom(String typeClassName)
+ {
+ if (typeClassName == null) throw new NullPointerException();
+ return new DataType.CustomType(Name.CUSTOM, typeClassName);
+ }
+
+ /**
+ * Returns the Duration type, introduced in Cassandra 3.10.
+ *
+ * <p>Note that a Duration type does not have a native representation in CQL, and technically, is
+ * merely a special {@link DataType#custom(String) custom type} from the driver's point of view.
+ *
+ * @return the Duration type. The returned instance is a singleton.
+ */
+ public static DataType duration()
+ {
+ return primitiveTypeMap.get(Name.DURATION);
+ }
+
+ /**
+ * Returns the name of that type.
+ *
+ * @return the name of that type.
+ */
+ public Name getName()
+ {
+ return name;
+ }
+
+ /**
+ * Returns whether this data type is frozen.
+ *
+ * <p>This applies to User Defined Types, tuples and nested collections. Frozen types are
+ * serialized as a single value in Cassandra's storage engine, whereas non-frozen types are stored
+ * in a form that allows updates to individual subfields.
+ *
+ * @return whether this data type is frozen.
+ */
+ public abstract boolean isFrozen();
+
+ /**
+ * Returns whether this data type represent a CQL {@link
+ * DataType.CollectionType collection type}, that is, a list, set or map.
+ *
+ * @return whether this data type name represent the name of a collection type.
+ */
+ public boolean isCollection()
+ {
+ return this instanceof CollectionType;
+ }
+
+ /**
+ * Returns the type arguments of this type.
+ *
+ * <p>Note that only the collection types (LIST, MAP, SET) have type arguments. For the other
+ * types, this will return an empty list.
+ *
+ * <p>For the collection types:
+ *
+ * <ul>
+ * <li>For lists and sets, this method returns one argument, the type of the elements.
+ * <li>For maps, this method returns two arguments, the first one is the type of the map keys,
+ * the second one is the type of the map values.
+ * </ul>
+ *
+ * @return an immutable list containing the type arguments of this type.
+ */
+ public List<DataType> getTypeArguments()
+ {
+ return Collections.emptyList();
+ }
+
+ /**
+ * Returns a String representation of this data type suitable for inclusion as a parameter type in
+ * a function or aggregate signature.
+ *
+ * <p>In such places, the String representation might vary from the canonical one as returned by
+ * {@link #toString()}; e.g. the {@code frozen} keyword is not accepted.
+ *
+ * @return a String representation of this data type suitable for inclusion as a parameter type in
+ * a function or aggregate signature.
+ */
+ public String asFunctionParameterString()
+ {
+ return toString();
+ }
+
+ /**
+ * Instances of this class represent CQL native types, also known as CQL primitive types.
+ */
+ public static class NativeType extends DataType
+ {
+
+ private NativeType(DataType.Name name)
+ {
+ super(name);
+ }
+
+ @Override
+ public boolean isFrozen()
+ {
+ return false;
+ }
+
+ @Override
+ public final int hashCode()
+ {
+ return (name == Name.TEXT) ? Name.VARCHAR.hashCode() : name.hashCode();
+ }
+
+ @Override
+ public final boolean equals(Object o)
+ {
+ if (!(o instanceof DataType.NativeType)) return false;
+
+ NativeType that = (DataType.NativeType) o;
+ return this.name.isCompatibleWith(that.name);
+ }
+
+ @Override
+ public String toString()
+ {
+ return name.toString();
+ }
+ }
+
+ /**
+ * Instances of this class represent collection types, that is, lists, sets or maps.
+ */
+ public static class CollectionType extends DataType
+ {
+
+ private final List<DataType> typeArguments;
+ private final boolean frozen;
+
+ private CollectionType(DataType.Name name, List<DataType> typeArguments, boolean frozen)
+ {
+ super(name);
+ this.typeArguments = typeArguments;
+ this.frozen = frozen;
+ }
+
+ @Override
+ public boolean isFrozen()
+ {
+ return frozen;
+ }
+
+ @Override
+ public List<DataType> getTypeArguments()
+ {
+ return typeArguments;
+ }
+
+ @Override
+ public final int hashCode()
+ {
+ return Objects.hash(name, typeArguments);
+ }
+
+ @Override
+ public final boolean equals(Object o)
+ {
+ if (!(o instanceof DataType.CollectionType)) return false;
+
+ DataType.CollectionType d = (DataType.CollectionType) o;
+ return name == d.name && typeArguments.equals(d.typeArguments);
+ }
+
+ @Override
+ public String toString()
+ {
+ if (name == Name.MAP)
+ {
+ String template = frozen ? "frozen<%s<%s, %s>>" : "%s<%s, %s>";
+ return String.format(template, name, typeArguments.get(0), typeArguments.get(1));
+ }
+ else
+ {
+ String template = frozen ? "frozen<%s<%s>>" : "%s<%s>";
+ return String.format(template, name, typeArguments.get(0));
+ }
+ }
+
+ @Override
+ public String asFunctionParameterString()
+ {
+ if (name == Name.MAP)
+ {
+ String template = "%s<%s, %s>";
+ return String.format(
+ template,
+ name,
+ typeArguments.get(0).asFunctionParameterString(),
+ typeArguments.get(1).asFunctionParameterString());
+ }
+ else
+ {
+ String template = "%s<%s>";
+ return String.format(template, name, typeArguments.get(0).asFunctionParameterString());
+ }
+ }
+ }
+
+ /**
+ * A "custom" type is a type that cannot be expressed as a CQL type.
+ *
+ * <p>Each custom type is merely identified by the fully qualified {@code
+ * #getCustomTypeClassName() class name} that represents this type server-side.
+ *
+ * <p>The driver provides a minimal support for such types through instances of this class.
+ *
+ * <p>A codec for custom types can be obtained via {@link TypeCodec#custom(DataType.CustomType)}.
+ */
+ public static class CustomType extends DataType
+ {
+
+ private final String customClassName;
+
+ private CustomType(DataType.Name name, String className)
+ {
+ super(name);
+ this.customClassName = className;
+ }
+
+ @Override
+ public boolean isFrozen()
+ {
+ return false;
+ }
+
+ @Override
+ public final int hashCode()
+ {
+ return Objects.hash(name, customClassName);
+ }
+
+ @Override
+ public final boolean equals(Object o)
+ {
+ if (!(o instanceof DataType.CustomType)) return false;
+
+ DataType.CustomType d = (DataType.CustomType) o;
+ return name == d.name && Objects.equals(customClassName, d.customClassName);
+ }
+
+ @Override
+ public String toString()
+ {
+ return String.format("'%s'", customClassName);
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/cql3/functions/types/DataTypeClassNameParser.java b/src/java/org/apache/cassandra/cql3/functions/types/DataTypeClassNameParser.java
new file mode 100644
index 0000000..7064ba2
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/functions/types/DataTypeClassNameParser.java
@@ -0,0 +1,396 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.functions.types;
+
+import java.util.*;
+
+import com.google.common.collect.ImmutableMap;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.transport.ProtocolVersion;
+import org.apache.cassandra.cql3.functions.types.exceptions.DriverInternalError;
+import org.apache.cassandra.cql3.functions.types.utils.Bytes;
+
+/*
+ * Parse data types from schema tables, for Cassandra 3.0 and above.
+ * In these versions, data types appear as class names, like "org.apache.cassandra.db.marshal.AsciiType"
+ * or "org.apache.cassandra.db.marshal.TupleType(org.apache.cassandra.db.marshal.Int32Type,org.apache.cassandra.db.marshal.Int32Type)".
+ *
+ * This is modified (and simplified) from Cassandra's TypeParser class to suit
+ * our needs. In particular it's not very efficient, but it doesn't really matter
+ * since it's rarely used and never in a critical path.
+ *
+ * Note that those methods all throw DriverInternalError when there is a parsing
+ * problem because in theory we'll only parse class names coming from Cassandra and
+ * so there shouldn't be anything wrong with them.
+ */
+public class DataTypeClassNameParser
+{
+ private static final Logger logger = LoggerFactory.getLogger(DataTypeClassNameParser.class);
+
+ private static final String REVERSED_TYPE = "org.apache.cassandra.db.marshal.ReversedType";
+ private static final String FROZEN_TYPE = "org.apache.cassandra.db.marshal.FrozenType";
+ private static final String LIST_TYPE = "org.apache.cassandra.db.marshal.ListType";
+ private static final String SET_TYPE = "org.apache.cassandra.db.marshal.SetType";
+ private static final String MAP_TYPE = "org.apache.cassandra.db.marshal.MapType";
+ private static final String UDT_TYPE = "org.apache.cassandra.db.marshal.UserType";
+ private static final String TUPLE_TYPE = "org.apache.cassandra.db.marshal.TupleType";
+ private static final String DURATION_TYPE = "org.apache.cassandra.db.marshal.DurationType";
+
+ private static final ImmutableMap<String, DataType> cassTypeToDataType =
+ new ImmutableMap.Builder<String, DataType>()
+ .put("org.apache.cassandra.db.marshal.AsciiType", DataType.ascii())
+ .put("org.apache.cassandra.db.marshal.LongType", DataType.bigint())
+ .put("org.apache.cassandra.db.marshal.BytesType", DataType.blob())
+ .put("org.apache.cassandra.db.marshal.BooleanType", DataType.cboolean())
+ .put("org.apache.cassandra.db.marshal.CounterColumnType", DataType.counter())
+ .put("org.apache.cassandra.db.marshal.DecimalType", DataType.decimal())
+ .put("org.apache.cassandra.db.marshal.DoubleType", DataType.cdouble())
+ .put("org.apache.cassandra.db.marshal.FloatType", DataType.cfloat())
+ .put("org.apache.cassandra.db.marshal.InetAddressType", DataType.inet())
+ .put("org.apache.cassandra.db.marshal.Int32Type", DataType.cint())
+ .put("org.apache.cassandra.db.marshal.UTF8Type", DataType.text())
+ .put("org.apache.cassandra.db.marshal.TimestampType", DataType.timestamp())
+ .put("org.apache.cassandra.db.marshal.SimpleDateType", DataType.date())
+ .put("org.apache.cassandra.db.marshal.TimeType", DataType.time())
+ .put("org.apache.cassandra.db.marshal.UUIDType", DataType.uuid())
+ .put("org.apache.cassandra.db.marshal.IntegerType", DataType.varint())
+ .put("org.apache.cassandra.db.marshal.TimeUUIDType", DataType.timeuuid())
+ .put("org.apache.cassandra.db.marshal.ByteType", DataType.tinyint())
+ .put("org.apache.cassandra.db.marshal.ShortType", DataType.smallint())
+ .put(DURATION_TYPE, DataType.duration())
+ .build();
+
+ public static DataType parseOne(
+ String className, ProtocolVersion protocolVersion, CodecRegistry codecRegistry)
+ {
+ boolean frozen = false;
+ if (isReversed(className))
+ {
+ // Just skip the ReversedType part, we don't care
+ className = getNestedClassName(className);
+ }
+ else if (isFrozen(className))
+ {
+ frozen = true;
+ className = getNestedClassName(className);
+ }
+
+ Parser parser = new Parser(className, 0);
+ String next = parser.parseNextName();
+
+ if (next.startsWith(LIST_TYPE))
+ return DataType.list(
+ parseOne(parser.getTypeParameters().get(0), protocolVersion, codecRegistry), frozen);
+
+ if (next.startsWith(SET_TYPE))
+ return DataType.set(
+ parseOne(parser.getTypeParameters().get(0), protocolVersion, codecRegistry), frozen);
+
+ if (next.startsWith(MAP_TYPE))
+ {
+ List<String> params = parser.getTypeParameters();
+ return DataType.map(
+ parseOne(params.get(0), protocolVersion, codecRegistry),
+ parseOne(params.get(1), protocolVersion, codecRegistry),
+ frozen);
+ }
+
+ if (frozen)
+ logger.warn(
+ "Got o.a.c.db.marshal.FrozenType for something else than a collection, "
+ + "this driver version might be too old for your version of Cassandra");
+
+ if (isUserType(next))
+ {
+ ++parser.idx; // skipping '('
+
+ String keyspace = parser.readOne();
+ parser.skipBlankAndComma();
+ String typeName =
+ TypeCodec.varchar()
+ .deserialize(Bytes.fromHexString("0x" + parser.readOne()), protocolVersion);
+ parser.skipBlankAndComma();
+ Map<String, String> rawFields = parser.getNameAndTypeParameters();
+ List<UserType.Field> fields = new ArrayList<>(rawFields.size());
+ for (Map.Entry<String, String> entry : rawFields.entrySet())
+ fields.add(
+ new UserType.Field(
+ entry.getKey(), parseOne(entry.getValue(), protocolVersion, codecRegistry)));
+ // create a frozen UserType since C* 2.x UDTs are always frozen.
+ return new UserType(keyspace, typeName, true, fields, protocolVersion, codecRegistry);
+ }
+
+ if (isTupleType(next))
+ {
+ List<String> rawTypes = parser.getTypeParameters();
+ List<DataType> types = new ArrayList<>(rawTypes.size());
+ for (String rawType : rawTypes)
+ {
+ types.add(parseOne(rawType, protocolVersion, codecRegistry));
+ }
+ return new TupleType(types, protocolVersion, codecRegistry);
+ }
+
+ DataType type = cassTypeToDataType.get(next);
+ return type == null ? DataType.custom(className) : type;
+ }
+
+ public static boolean isReversed(String className)
+ {
+ return className.startsWith(REVERSED_TYPE);
+ }
+
+ public static boolean isFrozen(String className)
+ {
+ return className.startsWith(FROZEN_TYPE);
+ }
+
+ private static String getNestedClassName(String className)
+ {
+ Parser p = new Parser(className, 0);
+ p.parseNextName();
+ List<String> l = p.getTypeParameters();
+ if (l.size() != 1) throw new IllegalStateException();
+ className = l.get(0);
+ return className;
+ }
+
+ private static boolean isUserType(String className)
+ {
+ return className.startsWith(UDT_TYPE);
+ }
+
+ private static boolean isTupleType(String className)
+ {
+ return className.startsWith(TUPLE_TYPE);
+ }
+
+ private static class Parser
+ {
+
+ private final String str;
+ private int idx;
+
+ private Parser(String str, int idx)
+ {
+ this.str = str;
+ this.idx = idx;
+ }
+
+ String parseNextName()
+ {
+ skipBlank();
+ return readNextIdentifier();
+ }
+
+ String readOne()
+ {
+ String name = parseNextName();
+ String args = readRawArguments();
+ return name + args;
+ }
+
+ // Assumes we have just read a class name and read it's potential arguments
+ // blindly. I.e. it assume that either parsing is done or that we're on a '('
+ // and this reads everything up until the corresponding closing ')'. It
+ // returns everything read, including the enclosing parenthesis.
+ private String readRawArguments()
+ {
+ skipBlank();
+
+ if (isEOS() || str.charAt(idx) == ')' || str.charAt(idx) == ',') return "";
+
+ if (str.charAt(idx) != '(')
+ throw new IllegalStateException(
+ String.format(
+ "Expecting char %d of %s to be '(' but '%c' found", idx, str, str.charAt(idx)));
+
+ int i = idx;
+ int open = 1;
+ while (open > 0)
+ {
+ ++idx;
+
+ if (isEOS()) throw new IllegalStateException("Non closed parenthesis");
+
+ if (str.charAt(idx) == '(')
+ {
+ open++;
+ }
+ else if (str.charAt(idx) == ')')
+ {
+ open--;
+ }
+ }
+ // we've stopped at the last closing ')' so move past that
+ ++idx;
+ return str.substring(i, idx);
+ }
+
+ List<String> getTypeParameters()
+ {
+ List<String> list = new ArrayList<>();
+
+ if (isEOS()) return list;
+
+ if (str.charAt(idx) != '(') throw new IllegalStateException();
+
+ ++idx; // skipping '('
+
+ while (skipBlankAndComma())
+ {
+ if (str.charAt(idx) == ')')
+ {
+ ++idx;
+ return list;
+ }
+
+ try
+ {
+ list.add(readOne());
+ }
+ catch (DriverInternalError e)
+ {
+ throw new DriverInternalError(
+ String.format("Exception while parsing '%s' around char %d", str, idx), e);
+ }
+ }
+ throw new DriverInternalError(
+ String.format(
+ "Syntax error parsing '%s' at char %d: unexpected end of string", str, idx));
+ }
+
+ // Must be at the start of the first parameter to read
+ Map<String, String> getNameAndTypeParameters()
+ {
+ // The order of the hashmap matters for UDT
+ Map<String, String> map = new LinkedHashMap<>();
+
+ while (skipBlankAndComma())
+ {
+ if (str.charAt(idx) == ')')
+ {
+ ++idx;
+ return map;
+ }
+
+ String bbHex = readNextIdentifier();
+ String name = null;
+ try
+ {
+ name =
+ TypeCodec.varchar()
+ .deserialize(Bytes.fromHexString("0x" + bbHex), ProtocolVersion.CURRENT);
+ }
+ catch (NumberFormatException e)
+ {
+ throwSyntaxError(e.getMessage());
+ }
+
+ skipBlank();
+ if (str.charAt(idx) != ':') throwSyntaxError("expecting ':' token");
+
+ ++idx;
+ skipBlank();
+ try
+ {
+ map.put(name, readOne());
+ }
+ catch (DriverInternalError e)
+ {
+ throw new DriverInternalError(
+ String.format("Exception while parsing '%s' around char %d", str, idx), e);
+ }
+ }
+ throw new DriverInternalError(
+ String.format(
+ "Syntax error parsing '%s' at char %d: unexpected end of string", str, idx));
+ }
+
+ private void throwSyntaxError(String msg)
+ {
+ throw new DriverInternalError(
+ String.format("Syntax error parsing '%s' at char %d: %s", str, idx, msg));
+ }
+
+ private boolean isEOS()
+ {
+ return isEOS(str, idx);
+ }
+
+ private static boolean isEOS(String str, int i)
+ {
+ return i >= str.length();
+ }
+
+ private void skipBlank()
+ {
+ idx = skipBlank(str, idx);
+ }
+
+ private static int skipBlank(String str, int i)
+ {
+ while (!isEOS(str, i) && ParseUtils.isBlank(str.charAt(i))) ++i;
+
+ return i;
+ }
+
+ // skip all blank and at best one comma, return true if there not EOS
+ private boolean skipBlankAndComma()
+ {
+ boolean commaFound = false;
+ while (!isEOS())
+ {
+ int c = str.charAt(idx);
+ if (c == ',')
+ {
+ if (commaFound) return true;
+ else commaFound = true;
+ }
+ else if (!ParseUtils.isBlank(c))
+ {
+ return true;
+ }
+ ++idx;
+ }
+ return false;
+ }
+
+ // left idx positioned on the character stopping the read
+ String readNextIdentifier()
+ {
+ int i = idx;
+ while (!isEOS() && ParseUtils.isIdentifierChar(str.charAt(idx))) ++idx;
+
+ return str.substring(i, idx);
+ }
+
+ @Override
+ public String toString()
+ {
+ return str.substring(0, idx)
+ + '['
+ + (idx == str.length() ? "" : str.charAt(idx))
+ + ']'
+ + str.substring(idx + 1);
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/cql3/functions/types/Duration.java b/src/java/org/apache/cassandra/cql3/functions/types/Duration.java
new file mode 100644
index 0000000..71d3e93
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/functions/types/Duration.java
@@ -0,0 +1,654 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.functions.types;
+
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import com.google.common.base.Objects;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+/**
+ * Represents a duration. A duration stores separately months, days, and seconds due to the fact
+ * that the number of days in a month varies, and a day can have 23 or 25 hours if a daylight saving
+ * is involved.
+ */
+public final class Duration
+{
+
+ private static final long NANOS_PER_MICRO = 1000L;
+ private static final long NANOS_PER_MILLI = 1000 * NANOS_PER_MICRO;
+ private static final long NANOS_PER_SECOND = 1000 * NANOS_PER_MILLI;
+ private static final long NANOS_PER_MINUTE = 60 * NANOS_PER_SECOND;
+ private static final long NANOS_PER_HOUR = 60 * NANOS_PER_MINUTE;
+ private static final int DAYS_PER_WEEK = 7;
+ private static final int MONTHS_PER_YEAR = 12;
+
+ /**
+ * The Regexp used to parse the duration provided as String.
+ */
+ private static final Pattern STANDARD_PATTERN =
+ Pattern.compile(
+ "\\G(\\d+)(y|Y|mo|MO|mO|Mo|w|W|d|D|h|H|s|S|ms|MS|mS|Ms|us|US|uS|Us|µs|µS|ns|NS|nS|Ns|m|M)");
+
+ /**
+ * The Regexp used to parse the duration when provided in the ISO 8601 format with designators.
+ */
+ private static final Pattern ISO8601_PATTERN =
+ Pattern.compile("P((\\d+)Y)?((\\d+)M)?((\\d+)D)?(T((\\d+)H)?((\\d+)M)?((\\d+)S)?)?");
+
+ /**
+ * The Regexp used to parse the duration when provided in the ISO 8601 format with designators.
+ */
+ private static final Pattern ISO8601_WEEK_PATTERN = Pattern.compile("P(\\d+)W");
+
+ /**
+ * The Regexp used to parse the duration when provided in the ISO 8601 alternative format.
+ */
+ private static final Pattern ISO8601_ALTERNATIVE_PATTERN =
+ Pattern.compile("P(\\d{4})-(\\d{2})-(\\d{2})T(\\d{2}):(\\d{2}):(\\d{2})");
+
+ /**
+ * The number of months.
+ */
+ private final int months;
+
+ /**
+ * The number of days.
+ */
+ private final int days;
+
+ /**
+ * The number of nanoseconds.
+ */
+ private final long nanoseconds;
+
+ private Duration(int months, int days, long nanoseconds)
+ {
+ // Makes sure that all the values are negative if one of them is
+ if ((months < 0 || days < 0 || nanoseconds < 0)
+ && ((months > 0 || days > 0 || nanoseconds > 0)))
+ {
+ throw new IllegalArgumentException(
+ String.format(
+ "All values must be either negative or positive, got %d months, %d days, %d nanoseconds",
+ months, days, nanoseconds));
+ }
+ this.months = months;
+ this.days = days;
+ this.nanoseconds = nanoseconds;
+ }
+
+ /**
+ * Creates a duration with the given number of months, days and nanoseconds.
+ *
+ * <p>A duration can be negative. In this case, all the non zero values must be negative.
+ *
+ * @param months the number of months
+ * @param days the number of days
+ * @param nanoseconds the number of nanoseconds
+ * @throws IllegalArgumentException if the values are not all negative or all positive
+ */
+ public static Duration newInstance(int months, int days, long nanoseconds)
+ {
+ return new Duration(months, days, nanoseconds);
+ }
+
+ /**
+ * Converts a <code>String</code> into a duration.
+ *
+ * <p>The accepted formats are:
+ *
+ * <ul>
+ * <li>multiple digits followed by a time unit like: 12h30m where the time unit can be:
+ * <ul>
+ * <li>{@code y}: years
+ * <li>{@code m}: months
+ * <li>{@code w}: weeks
+ * <li>{@code d}: days
+ * <li>{@code h}: hours
+ * <li>{@code m}: minutes
+ * <li>{@code s}: seconds
+ * <li>{@code ms}: milliseconds
+ * <li>{@code us} or {@code µs}: microseconds
+ * <li>{@code ns}: nanoseconds
+ * </ul>
+ * <li>ISO 8601 format: P[n]Y[n]M[n]DT[n]H[n]M[n]S or P[n]W
+ * <li>ISO 8601 alternative format: P[YYYY]-[MM]-[DD]T[hh]:[mm]:[ss]
+ * </ul>
+ *
+ * @param input the <code>String</code> to convert
+ * @return a {@link Duration}
+ */
+ public static Duration from(String input)
+ {
+ boolean isNegative = input.startsWith("-");
+ String source = isNegative ? input.substring(1) : input;
+
+ if (source.startsWith("P"))
+ {
+ if (source.endsWith("W")) return parseIso8601WeekFormat(isNegative, source);
+
+ if (source.contains("-")) return parseIso8601AlternativeFormat(isNegative, source);
+
+ return parseIso8601Format(isNegative, source);
+ }
+ return parseStandardFormat(isNegative, source);
+ }
+
+ private static Duration parseIso8601Format(boolean isNegative, String source)
+ {
+ Matcher matcher = ISO8601_PATTERN.matcher(source);
+ if (!matcher.matches())
+ throw new IllegalArgumentException(
+ String.format("Unable to convert '%s' to a duration", source));
+
+ Builder builder = new Builder(isNegative);
+ if (matcher.group(1) != null) builder.addYears(groupAsLong(matcher, 2));
+
+ if (matcher.group(3) != null) builder.addMonths(groupAsLong(matcher, 4));
+
+ if (matcher.group(5) != null) builder.addDays(groupAsLong(matcher, 6));
+
+ // Checks if the String contains time information
+ if (matcher.group(7) != null)
+ {
+ if (matcher.group(8) != null) builder.addHours(groupAsLong(matcher, 9));
+
+ if (matcher.group(10) != null) builder.addMinutes(groupAsLong(matcher, 11));
+
+ if (matcher.group(12) != null) builder.addSeconds(groupAsLong(matcher, 13));
+ }
+ return builder.build();
+ }
+
+ private static Duration parseIso8601AlternativeFormat(boolean isNegative, String source)
+ {
+ Matcher matcher = ISO8601_ALTERNATIVE_PATTERN.matcher(source);
+ if (!matcher.matches())
+ throw new IllegalArgumentException(
+ String.format("Unable to convert '%s' to a duration", source));
+
+ return new Builder(isNegative)
+ .addYears(groupAsLong(matcher, 1))
+ .addMonths(groupAsLong(matcher, 2))
+ .addDays(groupAsLong(matcher, 3))
+ .addHours(groupAsLong(matcher, 4))
+ .addMinutes(groupAsLong(matcher, 5))
+ .addSeconds(groupAsLong(matcher, 6))
+ .build();
+ }
+
+ private static Duration parseIso8601WeekFormat(boolean isNegative, String source)
+ {
+ Matcher matcher = ISO8601_WEEK_PATTERN.matcher(source);
+ if (!matcher.matches())
+ throw new IllegalArgumentException(
+ String.format("Unable to convert '%s' to a duration", source));
+
+ return new Builder(isNegative).addWeeks(groupAsLong(matcher, 1)).build();
+ }
+
+ private static Duration parseStandardFormat(boolean isNegative, String source)
+ {
+ Matcher matcher = STANDARD_PATTERN.matcher(source);
+ if (!matcher.find())
+ throw new IllegalArgumentException(
+ String.format("Unable to convert '%s' to a duration", source));
+
+ Builder builder = new Builder(isNegative);
+ boolean done;
+
+ do
+ {
+ long number = groupAsLong(matcher, 1);
+ String symbol = matcher.group(2);
+ add(builder, number, symbol);
+ done = matcher.end() == source.length();
+ } while (matcher.find());
+
+ if (!done)
+ throw new IllegalArgumentException(
+ String.format("Unable to convert '%s' to a duration", source));
+
+ return builder.build();
+ }
+
+ private static long groupAsLong(Matcher matcher, int group)
+ {
+ return Long.parseLong(matcher.group(group));
+ }
+
+ private static Builder add(Builder builder, long number, String symbol)
+ {
+ String s = symbol.toLowerCase();
+ if (s.equals("y"))
+ {
+ return builder.addYears(number);
+ }
+ else if (s.equals("mo"))
+ {
+ return builder.addMonths(number);
+ }
+ else if (s.equals("w"))
+ {
+ return builder.addWeeks(number);
+ }
+ else if (s.equals("d"))
+ {
+ return builder.addDays(number);
+ }
+ else if (s.equals("h"))
+ {
+ return builder.addHours(number);
+ }
+ else if (s.equals("m"))
+ {
+ return builder.addMinutes(number);
+ }
+ else if (s.equals("s"))
+ {
+ return builder.addSeconds(number);
+ }
+ else if (s.equals("ms"))
+ {
+ return builder.addMillis(number);
+ }
+ else if (s.equals("us") || s.equals("µs"))
+ {
+ return builder.addMicros(number);
+ }
+ else if (s.equals("ns"))
+ {
+ return builder.addNanos(number);
+ }
+ throw new IllegalArgumentException(String.format("Unknown duration symbol '%s'", symbol));
+ }
+
+ /**
+ * Appends the result of the division to the specified builder if the dividend is not zero.
+ *
+ * @param builder the builder to append to
+ * @param dividend the dividend
+ * @param divisor the divisor
+ * @param unit the time unit to append after the result of the division
+ * @return the remainder of the division
+ */
+ private static long append(StringBuilder builder, long dividend, long divisor, String unit)
+ {
+ if (dividend == 0 || dividend < divisor) return dividend;
+
+ builder.append(dividend / divisor).append(unit);
+ return dividend % divisor;
+ }
+
+ /**
+ * Returns the number of months in this duration.
+ *
+ * @return the number of months in this duration.
+ */
+ public int getMonths()
+ {
+ return months;
+ }
+
+ /**
+ * Returns the number of days in this duration.
+ *
+ * @return the number of days in this duration.
+ */
+ public int getDays()
+ {
+ return days;
+ }
+
+ /**
+ * Returns the number of nanoseconds in this duration.
+ *
+ * @return the number of months in this duration.
+ */
+ public long getNanoseconds()
+ {
+ return nanoseconds;
+ }
+
+ @Override
+ public int hashCode()
+ {
+ return Objects.hashCode(days, months, nanoseconds);
+ }
+
+ @Override
+ public boolean equals(Object obj)
+ {
+ if (!(obj instanceof Duration)) return false;
+
+ Duration other = (Duration) obj;
+ return days == other.days && months == other.months && nanoseconds == other.nanoseconds;
+ }
+
+ @Override
+ public String toString()
+ {
+ StringBuilder builder = new StringBuilder();
+
+ if (months < 0 || days < 0 || nanoseconds < 0) builder.append('-');
+
+ long remainder = append(builder, Math.abs(months), MONTHS_PER_YEAR, "y");
+ append(builder, remainder, 1, "mo");
+
+ append(builder, Math.abs(days), 1, "d");
+
+ if (nanoseconds != 0)
+ {
+ remainder = append(builder, Math.abs(nanoseconds), NANOS_PER_HOUR, "h");
+ remainder = append(builder, remainder, NANOS_PER_MINUTE, "m");
+ remainder = append(builder, remainder, NANOS_PER_SECOND, "s");
+ remainder = append(builder, remainder, NANOS_PER_MILLI, "ms");
+ remainder = append(builder, remainder, NANOS_PER_MICRO, "us");
+ append(builder, remainder, 1, "ns");
+ }
+ return builder.toString();
+ }
+
+ private static class Builder
+ {
+ /**
+ * {@code true} if the duration is a negative one, {@code false} otherwise.
+ */
+ private final boolean isNegative;
+
+ /**
+ * The number of months.
+ */
+ private int months;
+
+ /**
+ * The number of days.
+ */
+ private int days;
+
+ /**
+ * The number of nanoseconds.
+ */
+ private long nanoseconds;
+
+ /**
+ * We need to make sure that the values for each units are provided in order.
+ */
+ private int currentUnitIndex;
+
+ public Builder(boolean isNegative)
+ {
+ this.isNegative = isNegative;
+ }
+
+ /**
+ * Adds the specified amount of years.
+ *
+ * @param numberOfYears the number of years to add.
+ * @return this {@code Builder}
+ */
+ public Builder addYears(long numberOfYears)
+ {
+ validateOrder(1);
+ validateMonths(numberOfYears, MONTHS_PER_YEAR);
+ months += numberOfYears * MONTHS_PER_YEAR;
+ return this;
+ }
+
+ /**
+ * Adds the specified amount of months.
+ *
+ * @param numberOfMonths the number of months to add.
+ * @return this {@code Builder}
+ */
+ public Builder addMonths(long numberOfMonths)
+ {
+ validateOrder(2);
+ validateMonths(numberOfMonths, 1);
+ months += numberOfMonths;
+ return this;
+ }
+
+ /**
+ * Adds the specified amount of weeks.
+ *
+ * @param numberOfWeeks the number of weeks to add.
+ * @return this {@code Builder}
+ */
+ public Builder addWeeks(long numberOfWeeks)
+ {
+ validateOrder(3);
+ validateDays(numberOfWeeks, DAYS_PER_WEEK);
+ days += numberOfWeeks * DAYS_PER_WEEK;
+ return this;
+ }
+
+ /**
+ * Adds the specified amount of days.
+ *
+ * @param numberOfDays the number of days to add.
+ * @return this {@code Builder}
+ */
+ public Builder addDays(long numberOfDays)
+ {
+ validateOrder(4);
+ validateDays(numberOfDays, 1);
+ days += numberOfDays;
+ return this;
+ }
+
+ /**
+ * Adds the specified amount of hours.
+ *
+ * @param numberOfHours the number of hours to add.
+ * @return this {@code Builder}
+ */
+ public Builder addHours(long numberOfHours)
+ {
+ validateOrder(5);
+ validateNanos(numberOfHours, NANOS_PER_HOUR);
+ nanoseconds += numberOfHours * NANOS_PER_HOUR;
+ return this;
+ }
+
+ /**
+ * Adds the specified amount of minutes.
+ *
+ * @param numberOfMinutes the number of minutes to add.
+ * @return this {@code Builder}
+ */
+ public Builder addMinutes(long numberOfMinutes)
+ {
+ validateOrder(6);
+ validateNanos(numberOfMinutes, NANOS_PER_MINUTE);
+ nanoseconds += numberOfMinutes * NANOS_PER_MINUTE;
+ return this;
+ }
+
+ /**
+ * Adds the specified amount of seconds.
+ *
+ * @param numberOfSeconds the number of seconds to add.
+ * @return this {@code Builder}
+ */
+ public Builder addSeconds(long numberOfSeconds)
+ {
+ validateOrder(7);
+ validateNanos(numberOfSeconds, NANOS_PER_SECOND);
+ nanoseconds += numberOfSeconds * NANOS_PER_SECOND;
+ return this;
+ }
+
+ /**
+ * Adds the specified amount of milliseconds.
+ *
+ * @param numberOfMillis the number of milliseconds to add.
+ * @return this {@code Builder}
+ */
+ public Builder addMillis(long numberOfMillis)
+ {
+ validateOrder(8);
+ validateNanos(numberOfMillis, NANOS_PER_MILLI);
+ nanoseconds += numberOfMillis * NANOS_PER_MILLI;
+ return this;
+ }
+
+ /**
+ * Adds the specified amount of microseconds.
+ *
+ * @param numberOfMicros the number of microseconds to add.
+ * @return this {@code Builder}
+ */
+ public Builder addMicros(long numberOfMicros)
+ {
+ validateOrder(9);
+ validateNanos(numberOfMicros, NANOS_PER_MICRO);
+ nanoseconds += numberOfMicros * NANOS_PER_MICRO;
+ return this;
+ }
+
+ /**
+ * Adds the specified amount of nanoseconds.
+ *
+ * @param numberOfNanos the number of nanoseconds to add.
+ * @return this {@code Builder}
+ */
+ public Builder addNanos(long numberOfNanos)
+ {
+ validateOrder(10);
+ validateNanos(numberOfNanos, 1);
+ nanoseconds += numberOfNanos;
+ return this;
+ }
+
+ /**
+ * Validates that the total number of months can be stored.
+ *
+ * @param units the number of units that need to be added
+ * @param monthsPerUnit the number of days per unit
+ */
+ private void validateMonths(long units, int monthsPerUnit)
+ {
+ validate(units, (Integer.MAX_VALUE - months) / monthsPerUnit, "months");
+ }
+
+ /**
+ * Validates that the total number of days can be stored.
+ *
+ * @param units the number of units that need to be added
+ * @param daysPerUnit the number of days per unit
+ */
+ private void validateDays(long units, int daysPerUnit)
+ {
+ validate(units, (Integer.MAX_VALUE - days) / daysPerUnit, "days");
+ }
+
+ /**
+ * Validates that the total number of nanoseconds can be stored.
+ *
+ * @param units the number of units that need to be added
+ * @param nanosPerUnit the number of nanoseconds per unit
+ */
+ private void validateNanos(long units, long nanosPerUnit)
+ {
+ validate(units, (Long.MAX_VALUE - nanoseconds) / nanosPerUnit, "nanoseconds");
+ }
+
+ /**
+ * Validates that the specified amount is less than the limit.
+ *
+ * @param units the number of units to check
+ * @param limit the limit on the number of units
+ * @param unitName the unit name
+ */
+ private void validate(long units, long limit, String unitName)
+ {
+ checkArgument(
+ units <= limit,
+ "Invalid duration. The total number of %s must be less or equal to %s",
+ unitName,
+ Integer.MAX_VALUE);
+ }
+
+ /**
+ * Validates that the duration values are added in the proper order.
+ *
+ * @param unitIndex the unit index (e.g. years=1, months=2, ...)
+ */
+ private void validateOrder(int unitIndex)
+ {
+ if (unitIndex == currentUnitIndex)
+ throw new IllegalArgumentException(
+ String.format(
+ "Invalid duration. The %s are specified multiple times", getUnitName(unitIndex)));
+
+ if (unitIndex <= currentUnitIndex)
+ throw new IllegalArgumentException(
+ String.format(
+ "Invalid duration. The %s should be after %s",
+ getUnitName(currentUnitIndex), getUnitName(unitIndex)));
+
+ currentUnitIndex = unitIndex;
+ }
+
+ /**
+ * Returns the name of the unit corresponding to the specified index.
+ *
+ * @param unitIndex the unit index
+ * @return the name of the unit corresponding to the specified index.
+ */
+ private String getUnitName(int unitIndex)
+ {
+ switch (unitIndex)
+ {
+ case 1:
+ return "years";
+ case 2:
+ return "months";
+ case 3:
+ return "weeks";
+ case 4:
+ return "days";
+ case 5:
+ return "hours";
+ case 6:
+ return "minutes";
+ case 7:
+ return "seconds";
+ case 8:
+ return "milliseconds";
+ case 9:
+ return "microseconds";
+ case 10:
+ return "nanoseconds";
+ default:
+ throw new AssertionError("unknown unit index: " + unitIndex);
+ }
+ }
+
+ public Duration build()
+ {
+ return isNegative
+ ? new Duration(-months, -days, -nanoseconds)
+ : new Duration(months, days, nanoseconds);
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/cql3/functions/types/GettableByIndexData.java b/src/java/org/apache/cassandra/cql3/functions/types/GettableByIndexData.java
new file mode 100644
index 0000000..bcc5585
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/functions/types/GettableByIndexData.java
@@ -0,0 +1,592 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.functions.types;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.net.InetAddress;
+import java.nio.ByteBuffer;
+import java.util.*;
+
+import com.google.common.reflect.TypeToken;
+
+import org.apache.cassandra.cql3.functions.types.exceptions.CodecNotFoundException;
+import org.apache.cassandra.cql3.functions.types.exceptions.InvalidTypeException;
+
+/**
+ * Collection of (typed) CQL values that can be retrieved by index (starting at zero).
+ */
+public interface GettableByIndexData
+{
+
+ /**
+ * Returns whether the {@code i}th value is NULL.
+ *
+ * @param i the index ({@code 0 <= i < size()}) of the value to check.
+ * @return whether the {@code i}th value is NULL.
+ * @throws IndexOutOfBoundsException if {@code i} is not a valid index for this object.
+ */
+ public boolean isNull(int i);
+
+ /**
+ * Returns the {@code i}th value as a boolean.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to convert the underlying CQL
+ * type to a Java {@code boolean} (for CQL type {@code boolean}, this will be the built-in codec).
+ *
+ * @param i the index ({@code 0 <= i < size()}) to retrieve.
+ * @return the boolean value of the {@code i}th element. If the value is NULL, {@code false} is
+ * returned. If you need to distinguish NULL and false values, check first with {@link
+ * #isNull(int)} or use {@code get(i, Boolean.class)}.
+ * @throws IndexOutOfBoundsException if {@code i} is not a valid index for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the element's CQL
+ * type to a boolean.
+ */
+ public boolean getBool(int i);
+
+ /**
+ * Returns the {@code i}th value as a byte.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to convert the underlying CQL
+ * type to a Java {@code byte} (for CQL type {@code tinyint}, this will be the built-in codec).
+ *
+ * @param i the index ({@code 0 <= i < size()}) to retrieve.
+ * @return the value of the {@code i}th element as a byte. If the value is NULL, {@code 0} is
+ * returned. If you need to distinguish NULL and 0, check first with {@link #isNull(int)} or
+ * use {@code get(i, Byte.class)}.
+ * @throws IndexOutOfBoundsException if {@code i} is not a valid index for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the element's CQL
+ * type to a byte.
+ */
+ public byte getByte(int i);
+
+ /**
+ * Returns the {@code i}th value as a short.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to convert the underlying CQL
+ * type to a Java {@code short} (for CQL type {@code smallint}, this will be the built-in codec).
+ *
+ * @param i the index ({@code 0 <= i < size()}) to retrieve.
+ * @return the value of the {@code i}th element as a short. If the value is NULL, {@code 0} is
+ * returned. If you need to distinguish NULL and 0, check first with {@link #isNull(int)} or
+ * use {@code get(i, Short.class)}.
+ * @throws IndexOutOfBoundsException if {@code i} is not a valid index for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the element's CQL
+ * type to a short.
+ */
+ public short getShort(int i);
+
+ /**
+ * Returns the {@code i}th value as an integer.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to convert the underlying CQL
+ * type to a Java {@code int} (for CQL type {@code int}, this will be the built-in codec).
+ *
+ * @param i the index ({@code 0 <= i < size()}) to retrieve.
+ * @return the value of the {@code i}th element as an integer. If the value is NULL, {@code 0} is
+ * returned. If you need to distinguish NULL and 0, check first with {@link #isNull(int)} or
+ * use {@code get(i, Integer.class)}.
+ * @throws IndexOutOfBoundsException if {@code i} is not a valid index for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the element's CQL
+ * type to an int.
+ */
+ public int getInt(int i);
+
+ /**
+ * Returns the {@code i}th value as a long.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to convert the underlying CQL
+ * type to a Java {@code byte} (for CQL types {@code bigint} and {@code counter}, this will be the
+ * built-in codec).
+ *
+ * @param i the index ({@code 0 <= i < size()}) to retrieve.
+ * @return the value of the {@code i}th element as a long. If the value is NULL, {@code 0L} is
+ * returned. If you need to distinguish NULL and 0L, check first with {@link #isNull(int)} or
+ * use {@code get(i, Long.class)}.
+ * @throws IndexOutOfBoundsException if {@code i} is not a valid index for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the element's CQL
+ * type to a long.
+ */
+ public long getLong(int i);
+
+ /**
+ * Returns the {@code i}th value as a date.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to convert the underlying CQL
+ * type to a {@code Date} (for CQL type {@code timestamp}, this will be the built-in codec).
+ *
+ * @param i the index ({@code 0 <= i < size()}) to retrieve.
+ * @return the value of the {@code i}th element as a data. If the value is NULL, {@code null} is
+ * returned.
+ * @throws IndexOutOfBoundsException if {@code i} is not a valid index for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the element's CQL
+ * type to a {@code Date}.
+ */
+ public Date getTimestamp(int i);
+
+ /**
+ * Returns the {@code i}th value as a date (without time).
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to convert the underlying CQL
+ * type to a {@link LocalDate} (for CQL type {@code date}, this will be the built-in codec).
+ *
+ * @param i the index ({@code 0 <= i < size()}) to retrieve.
+ * @return the value of the {@code i}th element as an date. If the value is NULL, {@code null} is
+ * returned.
+ * @throws IndexOutOfBoundsException if {@code i} is not a valid index for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the element's CQL
+ * type to a {@code LocalDate}.
+ */
+ public LocalDate getDate(int i);
+
+ /**
+ * Returns the {@code i}th value as a long in nanoseconds since midnight.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to convert the underlying CQL
+ * type to a Java {@code long} (for CQL type {@code time}, this will be the built-in codec).
+ *
+ * @param i the index ({@code 0 <= i < size()}) to retrieve.
+ * @return the value of the {@code i}th element as a long. If the value is NULL, {@code 0L} is
+ * returned.
+ * @throws IndexOutOfBoundsException if {@code i} is not a valid index for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the element's CQL
+ * type to a long.
+ */
+ public long getTime(int i);
+
+ /**
+ * Returns the {@code i}th value as a float.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to convert the underlying CQL
+ * type to a Java {@code float} (for CQL type {@code float}, this will be the built-in codec).
+ *
+ * @param i the index ({@code 0 <= i < size()}) to retrieve.
+ * @return the value of the {@code i}th element as a float. If the value is NULL, {@code 0.0f} is
+ * returned. If you need to distinguish NULL and 0.0f, check first with {@link #isNull(int)}
+ * or use {@code get(i, Float.class)}.
+ * @throws IndexOutOfBoundsException if {@code i} is not a valid index for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the element's CQL
+ * type to a float.
+ */
+ public float getFloat(int i);
+
+ /**
+ * Returns the {@code i}th value as a double.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to convert the underlying CQL
+ * type to a Java {@code double} (for CQL type {@code double}, this will be the built-in codec).
+ *
+ * @param i the index ({@code 0 <= i < size()}) to retrieve.
+ * @return the value of the {@code i}th element as a double. If the value is NULL, {@code 0.0} is
+ * returned. If you need to distinguish NULL and 0.0, check first with {@link #isNull(int)} or
+ * use {@code get(i, Double.class)}.
+ * @throws IndexOutOfBoundsException if {@code i} is not a valid index for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the element's CQL
+ * type to a double.
+ */
+ public double getDouble(int i);
+
+ /**
+ * Returns the {@code i}th value as a {@code ByteBuffer}.
+ *
+ * <p>This method does not use any codec; it returns a copy of the binary representation of the
+ * value. It is up to the caller to convert the returned value appropriately.
+ *
+ * @param i the index ({@code 0 <= i < size()}) to retrieve.
+ * @return the value of the {@code i}th element as a ByteBuffer. If the value is NULL, {@code
+ * null} is returned.
+ * @throws IndexOutOfBoundsException if {@code i} is not a valid index for this object.
+ */
+ public ByteBuffer getBytesUnsafe(int i);
+
+ /**
+ * Returns the {@code i}th value as a byte array.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to convert the underlying CQL
+ * type to a Java {@code ByteBuffer} (for CQL type {@code blob}, this will be the built-in codec).
+ *
+ * @param i the index ({@code 0 <= i < size()}) to retrieve.
+ * @return the value of the {@code i}th element as a byte array. If the value is NULL, {@code
+ * null} is returned.
+ * @throws IndexOutOfBoundsException if {@code i} is not a valid index for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the element's CQL
+ * type to a {@code ByteBuffer}.
+ */
+ public ByteBuffer getBytes(int i);
+
+ /**
+ * Returns the {@code i}th value as a string.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to convert the underlying CQL
+ * type to a Java string (for CQL types {@code text}, {@code varchar} and {@code ascii}, this will
+ * be the built-in codec).
+ *
+ * @param i the index ({@code 0 <= i < size()}) to retrieve.
+ * @return the value of the {@code i}th element as a string. If the value is NULL, {@code null} is
+ * returned.
+ * @throws IndexOutOfBoundsException if {@code i} is not a valid index for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the element's CQL
+ * type to a string.
+ */
+ public String getString(int i);
+
+ /**
+ * Returns the {@code i}th value as a variable length integer.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to convert the underlying CQL
+ * type to a {@code BigInteger} (for CQL type {@code varint}, this will be the built-in codec).
+ *
+ * @param i the index ({@code 0 <= i < size()}) to retrieve.
+ * @return the value of the {@code i}th element as a variable length integer. If the value is
+ * NULL, {@code null} is returned.
+ * @throws IndexOutOfBoundsException if {@code i} is not a valid index for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the element's CQL
+ * type to a {@code BigInteger}.
+ */
+ public BigInteger getVarint(int i);
+
+ /**
+ * Returns the {@code i}th value as a variable length decimal.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to convert the underlying CQL
+ * type to a {@code BigDecimal} (for CQL type {@code decimal}, this will be the built-in codec).
+ *
+ * @param i the index ({@code 0 <= i < size()}) to retrieve.
+ * @return the value of the {@code i}th element as a variable length decimal. If the value is
+ * NULL, {@code null} is returned.
+ * @throws IndexOutOfBoundsException if {@code i} is not a valid index for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the element's CQL
+ * type to a {@code BigDecimal}.
+ */
+ public BigDecimal getDecimal(int i);
+
+ /**
+ * Returns the {@code i}th value as a UUID.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to convert the underlying CQL
+ * type to a {@code UUID} (for CQL types {@code uuid} and {@code timeuuid}, this will be the
+ * built-in codec).
+ *
+ * @param i the index ({@code 0 <= i < size()}) to retrieve.
+ * @return the value of the {@code i}th element as a UUID. If the value is NULL, {@code null} is
+ * returned.
+ * @throws IndexOutOfBoundsException if {@code i} is not a valid index for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the element's CQL
+ * type to a {@code UUID}.
+ */
+ public UUID getUUID(int i);
+
+ /**
+ * Returns the {@code i}th value as an InetAddress.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to convert the underlying CQL
+ * type to an {@code InetAddress} (for CQL type {@code inet}, this will be the built-in codec).
+ *
+ * @param i the index ({@code 0 <= i < size()}) to retrieve.
+ * @return the value of the {@code i}th element as an InetAddress. If the value is NULL, {@code
+ * null} is returned.
+ * @throws IndexOutOfBoundsException if {@code i} is not a valid index for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the element's CQL
+ * type to a {@code InetAddress}.
+ */
+ public InetAddress getInet(int i);
+
+ /**
+ * Returns the {@code i}th value as a list.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to convert the underlying CQL
+ * type to a list of the specified type.
+ *
+ * <p>If the type of the elements is generic, use {@link #getList(int, TypeToken)}.
+ *
+ * <p>Implementation note: the actual {@link List} implementation will depend on the {@link
+ * TypeCodec codec} being used; therefore, callers should make no assumptions concerning its
+ * mutability nor its thread-safety. Furthermore, the behavior of this method in respect to CQL
+ * {@code NULL} values is also codec-dependent. By default, the driver will return mutable
+ * instances, and a CQL {@code NULL} will be mapped to an empty collection (note that Cassandra
+ * makes no distinction between {@code NULL} and an empty collection).
+ *
+ * @param i the index ({@code 0 <= i < size()}) to retrieve.
+ * @param elementsClass the class for the elements of the list to retrieve.
+ * @return the value of the {@code i}th element as a list of {@code T} objects.
+ * @throws IndexOutOfBoundsException if {@code i} is not a valid index for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the element's CQL
+ * type to a list.
+ */
+ public <T> List<T> getList(int i, Class<T> elementsClass);
+
+ /**
+ * Returns the {@code i}th value as a list.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to convert the underlying CQL
+ * type to a list of the specified type.
+ *
+ * <p>Use this variant with nested collections, which produce a generic element type:
+ *
+ * <pre>
+ * {@code List<List<String>> l = row.getList(1, new TypeToken<List<String>>() {});}
+ * </pre>
+ *
+ * <p>Implementation note: the actual {@link List} implementation will depend on the {@link
+ * TypeCodec codec} being used; therefore, callers should make no assumptions concerning its
+ * mutability nor its thread-safety. Furthermore, the behavior of this method in respect to CQL
+ * {@code NULL} values is also codec-dependent. By default, the driver will return mutable
+ * instances, and a CQL {@code NULL} will mapped to an empty collection (note that Cassandra makes
+ * no distinction between {@code NULL} and an empty collection).
+ *
+ * @param i the index ({@code 0 <= i < size()}) to retrieve.
+ * @param elementsType the type of the elements of the list to retrieve.
+ * @return the value of the {@code i}th element as a list of {@code T} objects.
+ * @throws IndexOutOfBoundsException if {@code i} is not a valid index for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the element's CQL
+ * type to a list.
+ */
+ public <T> List<T> getList(int i, TypeToken<T> elementsType);
+
+ /**
+ * Returns the {@code i}th value as a set.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to convert the underlying CQL
+ * type to a set of the specified type.
+ *
+ * <p>If the type of the elements is generic, use {@link #getSet(int, TypeToken)}.
+ *
+ * <p>Implementation note: the actual {@link Set} implementation will depend on the {@link
+ * TypeCodec codec} being used; therefore, callers should make no assumptions concerning its
+ * mutability nor its thread-safety. Furthermore, the behavior of this method in respect to CQL
+ * {@code NULL} values is also codec-dependent. By default, the driver will return mutable
+ * instances, and a CQL {@code NULL} will mapped to an empty collection (note that Cassandra makes
+ * no distinction between {@code NULL} and an empty collection).
+ *
+ * @param i the index ({@code 0 <= i < size()}) to retrieve.
+ * @param elementsClass the class for the elements of the set to retrieve.
+ * @return the value of the {@code i}th element as a set of {@code T} objects.
+ * @throws IndexOutOfBoundsException if {@code i} is not a valid index for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the element's CQL
+ * type to a set.
+ */
+ public <T> Set<T> getSet(int i, Class<T> elementsClass);
+
+ /**
+ * Returns the {@code i}th value as a set.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to convert the underlying CQL
+ * type to a set of the specified type.
+ *
+ * <p>Use this variant with nested collections, which produce a generic element type:
+ *
+ * <pre>
+ * {@code Set<List<String>> l = row.getSet(1, new TypeToken<List<String>>() {});}
+ * </pre>
+ *
+ * <p>Implementation note: the actual {@link Set} implementation will depend on the {@link
+ * TypeCodec codec} being used; therefore, callers should make no assumptions concerning its
+ * mutability nor its thread-safety. Furthermore, the behavior of this method in respect to CQL
+ * {@code NULL} values is also codec-dependent. By default, the driver will return mutable
+ * instances, and a CQL {@code NULL} will mapped to an empty collection (note that Cassandra makes
+ * no distinction between {@code NULL} and an empty collection).
+ *
+ * @param i the index ({@code 0 <= i < size()}) to retrieve.
+ * @param elementsType the type for the elements of the set to retrieve.
+ * @return the value of the {@code i}th element as a set of {@code T} objects.
+ * @throws IndexOutOfBoundsException if {@code i} is not a valid index for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the element's CQL
+ * type to a set.
+ */
+ public <T> Set<T> getSet(int i, TypeToken<T> elementsType);
+
+ /**
+ * Returns the {@code i}th value as a map.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to convert the underlying CQL
+ * type to a map of the specified types.
+ *
+ * <p>If the type of the keys and/or values is generic, use {@link #getMap(int, TypeToken,
+ * TypeToken)}.
+ *
+ * <p>Implementation note: the actual {@link Map} implementation will depend on the {@link
+ * TypeCodec codec} being used; therefore, callers should make no assumptions concerning its
+ * mutability nor its thread-safety. Furthermore, the behavior of this method in respect to CQL
+ * {@code NULL} values is also codec-dependent. By default, the driver will return mutable
+ * instances, and a CQL {@code NULL} will mapped to an empty collection (note that Cassandra makes
+ * no distinction between {@code NULL} and an empty collection).
+ *
+ * @param i the index ({@code 0 <= i < size()}) to retrieve.
+ * @param keysClass the class for the keys of the map to retrieve.
+ * @param valuesClass the class for the values of the map to retrieve.
+ * @return the value of the {@code i}th element as a map of {@code K} to {@code V} objects.
+ * @throws IndexOutOfBoundsException if {@code i} is not a valid index for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the element's CQL
+ * type to a map.
+ */
+ public <K, V> Map<K, V> getMap(int i, Class<K> keysClass, Class<V> valuesClass);
+
+ /**
+ * Returns the {@code i}th value as a map.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to convert the underlying CQL
+ * type to a map of the specified types.
+ *
+ * <p>Use this variant with nested collections, which produce a generic element type:
+ *
+ * <pre>
+ * {@code Map<Int, List<String>> l = row.getMap(1, TypeToken.of(Integer.class), new TypeToken<List<String>>() {});}
+ * </pre>
+ *
+ * <p>Implementation note: the actual {@link Map} implementation will depend on the {@link
+ * TypeCodec codec} being used; therefore, callers should make no assumptions concerning its
+ * mutability nor its thread-safety. Furthermore, the behavior of this method in respect to CQL
+ * {@code NULL} values is also codec-dependent. By default, the driver will return mutable
+ * instances, and a CQL {@code NULL} will mapped to an empty collection (note that Cassandra makes
+ * no distinction between {@code NULL} and an empty collection).
+ *
+ * @param i the index ({@code 0 <= i < size()}) to retrieve.
+ * @param keysType the type for the keys of the map to retrieve.
+ * @param valuesType the type for the values of the map to retrieve.
+ * @return the value of the {@code i}th element as a map of {@code K} to {@code V} objects.
+ * @throws IndexOutOfBoundsException if {@code i} is not a valid index for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the element's CQL
+ * type to a map.
+ */
+ public <K, V> Map<K, V> getMap(int i, TypeToken<K> keysType, TypeToken<V> valuesType);
+
+ /**
+ * Return the {@code i}th value as a UDT value.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to convert the underlying CQL
+ * type to a {@code UDTValue} (if the CQL type is a UDT, the registry will generate a codec
+ * automatically).
+ *
+ * @param i the index ({@code 0 <= i < size()}) to retrieve.
+ * @return the value of the {@code i}th element as a UDT value. If the value is NULL, then {@code
+ * null} will be returned.
+ * @throws IndexOutOfBoundsException if {@code i} is not a valid index for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the element's CQL
+ * type to a {@code UDTValue}.
+ */
+ public UDTValue getUDTValue(int i);
+
+ /**
+ * Return the {@code i}th value as a tuple value.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to convert the underlying CQL
+ * type to a {@code TupleValue} (if the CQL type is a tuple, the registry will generate a codec
+ * automatically).
+ *
+ * @param i the index ({@code 0 <= i < size()}) to retrieve.
+ * @return the value of the {@code i}th element as a tuple value. If the value is NULL, then
+ * {@code null} will be returned.
+ * @throws IndexOutOfBoundsException if {@code i} is not a valid index for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the element's CQL
+ * type to a {@code TupleValue}.
+ */
+ public TupleValue getTupleValue(int i);
+
+ /**
+ * Returns the {@code i}th value as the Java type matching its CQL type.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find the first codec that handles the
+ * underlying CQL type. The Java type of the returned object will be determined by the codec that
+ * was selected.
+ *
+ * <p>Use this method to dynamically inspect elements when types aren't known in advance, for
+ * instance if you're writing a generic row logger. If you know the target Java type, it is
+ * generally preferable to use typed getters, such as the ones for built-in types ({@link
+ * #getBool(int)}, {@link #getInt(int)}, etc.), or {@link #get(int, Class)} and {@link #get(int,
+ * TypeToken)} for custom types.
+ *
+ * @param i the index to retrieve.
+ * @return the value of the {@code i}th value as the Java type matching its CQL type.
+ * @throws IndexOutOfBoundsException if {@code i} is not a valid index for this object.
+ * @see CodecRegistry#codecFor(DataType)
+ */
+ public Object getObject(int i);
+
+ /**
+ * Returns the {@code i}th value converted to the given Java type.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to convert the underlying CQL
+ * type to the given Java type.
+ *
+ * <p>If the target type is generic, use {@link #get(int, TypeToken)}.
+ *
+ * <p>Implementation note: the actual object returned by this method will depend on the {@link
+ * TypeCodec codec} being used; therefore, callers should make no assumptions concerning its
+ * mutability nor its thread-safety. Furthermore, the behavior of this method in respect to CQL
+ * {@code NULL} values is also codec-dependent; by default, a CQL {@code NULL} value translates to
+ * {@code null} for simple CQL types, UDTs and tuples, and to empty collections for all CQL
+ * collection types.
+ *
+ * @param i the index to retrieve.
+ * @param targetClass The Java type the value should be converted to.
+ * @return the value of the {@code i}th value converted to the given Java type.
+ * @throws IndexOutOfBoundsException if {@code i} is not a valid index for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the element's CQL
+ * type to {@code targetClass}.
+ */
+ <T> T get(int i, Class<T> targetClass);
+
+ /**
+ * Returns the {@code i}th value converted to the given Java type.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to convert the underlying CQL
+ * type to the given Java type.
+ *
+ * <p>Implementation note: the actual object returned by this method will depend on the {@link
+ * TypeCodec codec} being used; therefore, callers should make no assumptions concerning its
+ * mutability nor its thread-safety. Furthermore, the behavior of this method in respect to CQL
+ * {@code NULL} values is also codec-dependent; by default, a CQL {@code NULL} value translates to
+ * {@code null} for simple CQL types, UDTs and tuples, and to empty collections for all CQL
+ * collection types.
+ *
+ * @param i the index to retrieve.
+ * @param targetType The Java type the value should be converted to.
+ * @return the value of the {@code i}th value converted to the given Java type.
+ * @throws IndexOutOfBoundsException if {@code i} is not a valid index for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the element's CQL
+ * type to {@code targetType}.
+ */
+ <T> T get(int i, TypeToken<T> targetType);
+
+ /**
+ * Returns the {@code i}th value converted using the given {@link TypeCodec}.
+ *
+ * <p>This method entirely bypasses the {@link CodecRegistry} and forces the driver to use the
+ * given codec instead. This can be useful if the codec would collide with a previously registered
+ * one, or if you want to use the codec just once without registering it.
+ *
+ * <p>It is the caller's responsibility to ensure that the given codec {@link
+ * TypeCodec#accepts(DataType) accepts} the underlying CQL type; failing to do so may result in
+ * {@link InvalidTypeException}s being thrown.
+ *
+ * <p>Implementation note: the actual object returned by this method will depend on the {@link
+ * TypeCodec codec} being used; therefore, callers should make no assumptions concerning its
+ * mutability nor its thread-safety. Furthermore, the behavior of this method in respect to CQL
+ * {@code NULL} values is also codec-dependent; by default, a CQL {@code NULL} value translates to
+ * {@code null} for simple CQL types, UDTs and tuples, and to empty collections for all CQL
+ * collection types.
+ *
+ * @param i the index to retrieve.
+ * @param codec The {@link TypeCodec} to use to deserialize the value; may not be {@code null}.
+ * @return the value of the {@code i}th value converted using the given {@link TypeCodec}.
+ * @throws InvalidTypeException if the given codec does not {@link TypeCodec#accepts(DataType)
+ * accept} the underlying CQL type.
+ * @throws IndexOutOfBoundsException if {@code i} is not a valid index for this object.
+ */
+ <T> T get(int i, TypeCodec<T> codec);
+}
diff --git a/src/java/org/apache/cassandra/cql3/functions/types/GettableByNameData.java b/src/java/org/apache/cassandra/cql3/functions/types/GettableByNameData.java
new file mode 100644
index 0000000..c578ff4
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/functions/types/GettableByNameData.java
@@ -0,0 +1,593 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.functions.types;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.net.InetAddress;
+import java.nio.ByteBuffer;
+import java.util.*;
+
+import com.google.common.reflect.TypeToken;
+
+import org.apache.cassandra.cql3.functions.types.exceptions.CodecNotFoundException;
+import org.apache.cassandra.cql3.functions.types.exceptions.InvalidTypeException;
+
+/**
+ * Collection of (typed) CQL values that can be retrieved by name.
+ */
+public interface GettableByNameData
+{
+
+ /**
+ * Returns whether the value for {@code name} is NULL.
+ *
+ * @param name the name to check.
+ * @return whether the value for {@code name} is NULL.
+ * @throws IllegalArgumentException if {@code name} is not valid name for this object.
+ */
+ public boolean isNull(String name);
+
+ /**
+ * Returns the value for {@code name} as a boolean.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to convert the underlying CQL
+ * type to a Java {@code boolean} (for CQL type {@code boolean}, this will be the built-in codec).
+ *
+ * @param name the name to retrieve.
+ * @return the boolean value for {@code name}. If the value is NULL, {@code false} is returned. If
+ * you need to distinguish NULL and false values, check first with {@link #isNull(String)} or
+ * use {@code get(name, Boolean.class)}.
+ * @throws IllegalArgumentException if {@code name} is not valid name for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the underlying CQL
+ * type to a boolean.
+ */
+ public boolean getBool(String name);
+
+ /**
+ * Returns the value for {@code name} as a byte.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to convert the underlying CQL
+ * type to a Java {@code byte} (for CQL type {@code tinyint}, this will be the built-in codec).
+ *
+ * @param name the name to retrieve.
+ * @return the value for {@code name} as a byte. If the value is NULL, {@code 0} is returned. If
+ * you need to distinguish NULL and 0, check first with {@link #isNull(String)} or use {@code
+ * get(name, Byte.class)}. {@code 0} is returned.
+ * @throws IllegalArgumentException if {@code name} is not valid name for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the underlying CQL
+ * type to a byte.
+ */
+ public byte getByte(String name);
+
+ /**
+ * Returns the value for {@code name} as a short.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to convert the underlying CQL
+ * type to a Java {@code short} (for CQL type {@code smallint}, this will be the built-in codec).
+ *
+ * @param name the name to retrieve.
+ * @return the value for {@code name} as a short. If the value is NULL, {@code 0} is returned. If
+ * you need to distinguish NULL and 0, check first with {@link #isNull(String)} or use {@code
+ * get(name, Short.class)}. {@code 0} is returned.
+ * @throws IllegalArgumentException if {@code name} is not valid name for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the underlying CQL
+ * type to a short.
+ */
+ public short getShort(String name);
+
+ /**
+ * Returns the value for {@code name} as an integer.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to convert the underlying CQL
+ * type to a Java {@code int} (for CQL type {@code int}, this will be the built-in codec).
+ *
+ * @param name the name to retrieve.
+ * @return the value for {@code name} as an integer. If the value is NULL, {@code 0} is returned.
+ * If you need to distinguish NULL and 0, check first with {@link #isNull(String)} or use
+ * {@code get(name, Integer.class)}. {@code 0} is returned.
+ * @throws IllegalArgumentException if {@code name} is not valid name for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the underlying CQL
+ * type to an int.
+ */
+ public int getInt(String name);
+
+ /**
+ * Returns the value for {@code name} as a long.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to convert the underlying CQL
+ * type to a Java {@code byte} (for CQL types {@code bigint} and {@code counter}, this will be the
+ * built-in codec).
+ *
+ * @param name the name to retrieve.
+ * @return the value for {@code name} as a long. If the value is NULL, {@code 0L} is returned. If
+ * you need to distinguish NULL and 0L, check first with {@link #isNull(String)} or use {@code
+ * get(name, Long.class)}.
+ * @throws IllegalArgumentException if {@code name} is not valid name for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the underlying CQL
+ * type to a long.
+ */
+ public long getLong(String name);
+
+ /**
+ * Returns the value for {@code name} as a date.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to convert the underlying CQL
+ * type to a {@code Date} (for CQL type {@code timestamp}, this will be the built-in codec).
+ *
+ * @param name the name to retrieve.
+ * @return the value for {@code name} as a date. If the value is NULL, {@code null} is returned.
+ * @throws IllegalArgumentException if {@code name} is not valid name for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the underlying CQL
+ * type to a {@code Date}.
+ */
+ public Date getTimestamp(String name);
+
+ /**
+ * Returns the value for {@code name} as a date (without time).
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to convert the underlying CQL
+ * type to a {@link LocalDate} (for CQL type {@code date}, this will be the built-in codec).
+ *
+ * @param name the name to retrieve.
+ * @return the value for {@code name} as a date. If the value is NULL, {@code null} is returned.
+ * @throws IllegalArgumentException if {@code name} is not valid name for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the underlying CQL
+ * type to a {@code LocalDate}.
+ */
+ public LocalDate getDate(String name);
+
+ /**
+ * Returns the value for {@code name} as a long in nanoseconds since midnight.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to convert the underlying CQL
+ * type to a Java {@code long} (for CQL type {@code time}, this will be the built-in codec).
+ *
+ * @param name the name to retrieve.
+ * @return the value for {@code name} as a long. If the value is NULL, {@code 0L} is returned.
+ * @throws IllegalArgumentException if {@code name} is not valid name for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the underlying CQL
+ * type to a long.
+ */
+ public long getTime(String name);
+
+ /**
+ * Returns the value for {@code name} as a float.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to convert the underlying CQL
+ * type to a Java {@code float} (for CQL type {@code float}, this will be the built-in codec).
+ *
+ * @param name the name to retrieve.
+ * @return the value for {@code name} as a float. If the value is NULL, {@code 0.0f} is returned.
+ * If you need to distinguish NULL and 0.0f, check first with {@link #isNull(String)} or use
+ * {@code get(name, Float.class)}.
+ * @throws IllegalArgumentException if {@code name} is not valid name for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the underlying CQL
+ * type to a float.
+ */
+ public float getFloat(String name);
+
+ /**
+ * Returns the value for {@code name} as a double.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to convert the underlying CQL
+ * type to a Java {@code double} (for CQL type {@code double}, this will be the built-in codec).
+ *
+ * @param name the name to retrieve.
+ * @return the value for {@code name} as a double. If the value is NULL, {@code 0.0} is returned.
+ * If you need to distinguish NULL and 0.0, check first with {@link #isNull(String)} or use
+ * {@code get(name, Double.class)}.
+ * @throws IllegalArgumentException if {@code name} is not valid name for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the underlying CQL
+ * type to a double.
+ */
+ public double getDouble(String name);
+
+ /**
+ * Returns the value for {@code name} as a ByteBuffer.
+ *
+ * <p>This method does not use any codec; it returns a copy of the binary representation of the
+ * value. It is up to the caller to convert the returned value appropriately.
+ *
+ * <p>Note: this method always return the bytes composing the value, even if the column is not of
+ * type BLOB. That is, this method never throw an InvalidTypeException. However, if the type is
+ * not BLOB, it is up to the caller to handle the returned value correctly.
+ *
+ * @param name the name to retrieve.
+ * @return the value for {@code name} as a ByteBuffer. If the value is NULL, {@code null} is
+ * returned.
+ * @throws IllegalArgumentException if {@code name} is not valid name for this object.
+ */
+ public ByteBuffer getBytesUnsafe(String name);
+
+ /**
+ * Returns the value for {@code name} as a byte array.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to convert the underlying CQL
+ * type to a Java {@code ByteBuffer} (for CQL type {@code blob}, this will be the built-in codec).
+ *
+ * @param name the name to retrieve.
+ * @return the value for {@code name} as a byte array. If the value is NULL, {@code null} is
+ * returned.
+ * @throws IllegalArgumentException if {@code name} is not valid name for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the underlying CQL
+ * type to a {@code ByteBuffer}.
+ */
+ public ByteBuffer getBytes(String name);
+
+ /**
+ * Returns the value for {@code name} as a string.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to convert the underlying CQL
+ * type to a Java string (for CQL types {@code text}, {@code varchar} and {@code ascii}, this will
+ * be the built-in codec).
+ *
+ * @param name the name to retrieve.
+ * @return the value for {@code name} as a string. If the value is NULL, {@code null} is returned.
+ * @throws IllegalArgumentException if {@code name} is not valid name for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the underlying CQL
+ * type to a string.
+ */
+ public String getString(String name);
+
+ /**
+ * Returns the value for {@code name} as a variable length integer.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to convert the underlying CQL
+ * type to a {@code BigInteger} (for CQL type {@code varint}, this will be the built-in codec).
+ *
+ * @param name the name to retrieve.
+ * @return the value for {@code name} as a variable length integer. If the value is NULL, {@code
+ * null} is returned.
+ * @throws IllegalArgumentException if {@code name} is not valid name for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the underlying CQL
+ * type to a {@code BigInteger}.
+ */
+ public BigInteger getVarint(String name);
+
+ /**
+ * Returns the value for {@code name} as a variable length decimal.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to convert the underlying CQL
+ * type to a {@code BigDecimal} (for CQL type {@code decimal}, this will be the built-in codec).
+ *
+ * @param name the name to retrieve.
+ * @return the value for {@code name} as a variable length decimal. If the value is NULL, {@code
+ * null} is returned.
+ * @throws IllegalArgumentException if {@code name} is not valid name for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the underlying CQL
+ * type to a {@code BigDecimal}.
+ */
+ public BigDecimal getDecimal(String name);
+
+ /**
+ * Returns the value for {@code name} as a UUID.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to convert the underlying CQL
+ * type to a {@code UUID} (for CQL types {@code uuid} and {@code timeuuid}, this will be the
+ * built-in codec).
+ *
+ * @param name the name to retrieve.
+ * @return the value for {@code name} as a UUID. If the value is NULL, {@code null} is returned.
+ * @throws IllegalArgumentException if {@code name} is not valid name for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the underlying CQL
+ * type to a {@code UUID}.
+ */
+ public UUID getUUID(String name);
+
+ /**
+ * Returns the value for {@code name} as an InetAddress.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to convert the underlying CQL
+ * type to an {@code InetAddress} (for CQL type {@code inet}, this will be the built-in codec).
+ *
+ * @param name the name to retrieve.
+ * @return the value for {@code name} as an InetAddress. If the value is NULL, {@code null} is
+ * returned.
+ * @throws IllegalArgumentException if {@code name} is not valid name for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the underlying CQL
+ * type to a {@code InetAddress}.
+ */
+ public InetAddress getInet(String name);
+
+ /**
+ * Returns the value for {@code name} as a list.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to convert the underlying CQL
+ * type to a list of the specified type.
+ *
+ * <p>If the type of the elements is generic, use {@link #getList(String, TypeToken)}.
+ *
+ * <p>Implementation note: the actual {@link List} implementation will depend on the {@link
+ * TypeCodec codec} being used; therefore, callers should make no assumptions concerning its
+ * mutability nor its thread-safety. Furthermore, the behavior of this method in respect to CQL
+ * {@code NULL} values is also codec-dependent. By default, the driver will return mutable
+ * instances, and a CQL {@code NULL} will mapped to an empty collection (note that Cassandra makes
+ * no distinction between {@code NULL} and an empty collection).
+ *
+ * @param name the name to retrieve.
+ * @param elementsClass the class for the elements of the list to retrieve.
+ * @return the value of the {@code i}th element as a list of {@code T} objects.
+ * @throws IllegalArgumentException if {@code name} is not valid name for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the underlying CQL
+ * type to a list.
+ */
+ public <T> List<T> getList(String name, Class<T> elementsClass);
+
+ /**
+ * Returns the value for {@code name} as a list.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to convert the underlying CQL
+ * type to a list of the specified type.
+ *
+ * <p>Use this variant with nested collections, which produce a generic element type:
+ *
+ * <pre>
+ * {@code List<List<String>> l = row.getList("theColumn", new TypeToken<List<String>>() {});}
+ * </pre>
+ *
+ * <p>Implementation note: the actual {@link List} implementation will depend on the {@link
+ * TypeCodec codec} being used; therefore, callers should make no assumptions concerning its
+ * mutability nor its thread-safety. Furthermore, the behavior of this method in respect to CQL
+ * {@code NULL} values is also codec-dependent. By default, the driver will return mutable
+ * instances, and a CQL {@code NULL} will mapped to an empty collection (note that Cassandra makes
+ * no distinction between {@code NULL} and an empty collection).
+ *
+ * @param name the name to retrieve.
+ * @param elementsType the type for the elements of the list to retrieve.
+ * @return the value of the {@code i}th element as a list of {@code T} objects.
+ * @throws IllegalArgumentException if {@code name} is not valid name for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the underlying CQL
+ * type to a list.
+ */
+ public <T> List<T> getList(String name, TypeToken<T> elementsType);
+
+ /**
+ * Returns the value for {@code name} as a set.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to convert the underlying CQL
+ * type to a set of the specified type.
+ *
+ * <p>If the type of the elements is generic, use {@link #getSet(String, TypeToken)}.
+ *
+ * <p>Implementation note: the actual {@link Set} implementation will depend on the {@link
+ * TypeCodec codec} being used; therefore, callers should make no assumptions concerning its
+ * mutability nor its thread-safety. Furthermore, the behavior of this method in respect to CQL
+ * {@code NULL} values is also codec-dependent. By default, the driver will return mutable
+ * instances, and a CQL {@code NULL} will mapped to an empty collection (note that Cassandra makes
+ * no distinction between {@code NULL} and an empty collection).
+ *
+ * @param name the name to retrieve.
+ * @param elementsClass the class for the elements of the set to retrieve.
+ * @return the value of the {@code i}th element as a set of {@code T} objects.
+ * @throws IllegalArgumentException if {@code name} is not valid name for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the underlying CQL
+ * type to a set.
+ */
+ public <T> Set<T> getSet(String name, Class<T> elementsClass);
+
+ /**
+ * Returns the value for {@code name} as a set.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to convert the underlying CQL
+ * type to a set of the specified type.
+ *
+ * <p>Use this variant with nested collections, which produce a generic element type:
+ *
+ * <pre>
+ * {@code Set<List<String>> l = row.getSet("theColumn", new TypeToken<List<String>>() {});}
+ * </pre>
+ *
+ * <p>Implementation note: the actual {@link Set} implementation will depend on the {@link
+ * TypeCodec codec} being used; therefore, callers should make no assumptions concerning its
+ * mutability nor its thread-safety. Furthermore, the behavior of this method in respect to CQL
+ * {@code NULL} values is also codec-dependent. By default, the driver will return mutable
+ * instances, and a CQL {@code NULL} will mapped to an empty collection (note that Cassandra makes
+ * no distinction between {@code NULL} and an empty collection).
+ *
+ * @param name the name to retrieve.
+ * @param elementsType the type for the elements of the set to retrieve.
+ * @return the value of the {@code i}th element as a set of {@code T} objects.
+ * @throws IllegalArgumentException if {@code name} is not valid name for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the underlying CQL
+ * type to a set.
+ */
+ public <T> Set<T> getSet(String name, TypeToken<T> elementsType);
+
+ /**
+ * Returns the value for {@code name} as a map.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to convert the underlying CQL
+ * type to a map of the specified types.
+ *
+ * <p>If the type of the keys and/or values is generic, use {@link #getMap(String, TypeToken,
+ * TypeToken)}.
+ *
+ * <p>Implementation note: the actual {@link Map} implementation will depend on the {@link
+ * TypeCodec codec} being used; therefore, callers should make no assumptions concerning its
+ * mutability nor its thread-safety. Furthermore, the behavior of this method in respect to CQL
+ * {@code NULL} values is also codec-dependent. By default, the driver will return mutable
+ * instances, and a CQL {@code NULL} will mapped to an empty collection (note that Cassandra makes
+ * no distinction between {@code NULL} and an empty collection).
+ *
+ * @param name the name to retrieve.
+ * @param keysClass the class for the keys of the map to retrieve.
+ * @param valuesClass the class for the values of the map to retrieve.
+ * @return the value of {@code name} as a map of {@code K} to {@code V} objects.
+ * @throws IllegalArgumentException if {@code name} is not valid name for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the underlying CQL
+ * type to a map.
+ */
+ public <K, V> Map<K, V> getMap(String name, Class<K> keysClass, Class<V> valuesClass);
+
+ /**
+ * Returns the value for {@code name} as a map.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to convert the underlying CQL
+ * type to a map of the specified types.
+ *
+ * <p>Use this variant with nested collections, which produce a generic element type:
+ *
+ * <pre>
+ * {@code Map<Int, List<String>> l = row.getMap("theColumn", TypeToken.of(Integer.class), new TypeToken<List<String>>() {});}
+ * </pre>
+ *
+ * <p>Implementation note: the actual {@link Map} implementation will depend on the {@link
+ * TypeCodec codec} being used; therefore, callers should make no assumptions concerning its
+ * mutability nor its thread-safety. Furthermore, the behavior of this method in respect to CQL
+ * {@code NULL} values is also codec-dependent. By default, the driver will return mutable
+ * instances, and a CQL {@code NULL} will mapped to an empty collection (note that Cassandra makes
+ * no distinction between {@code NULL} and an empty collection).
+ *
+ * @param name the name to retrieve.
+ * @param keysType the class for the keys of the map to retrieve.
+ * @param valuesType the class for the values of the map to retrieve.
+ * @return the value of {@code name} as a map of {@code K} to {@code V} objects.
+ * @throws IllegalArgumentException if {@code name} is not valid name for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the underlying CQL
+ * type to a map.
+ */
+ public <K, V> Map<K, V> getMap(String name, TypeToken<K> keysType, TypeToken<V> valuesType);
+
+ /**
+ * Return the value for {@code name} as a UDT value.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to convert the underlying CQL
+ * type to a {@code UDTValue} (if the CQL type is a UDT, the registry will generate a codec
+ * automatically).
+ *
+ * @param name the name to retrieve.
+ * @return the value of {@code name} as a UDT value. If the value is NULL, then {@code null} will
+ * be returned.
+ * @throws IllegalArgumentException if {@code name} is not valid name for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the underlying CQL
+ * type to a {@code UDTValue}.
+ */
+ public UDTValue getUDTValue(String name);
+
+ /**
+ * Return the value for {@code name} as a tuple value.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to convert the underlying CQL
+ * type to a {@code TupleValue} (if the CQL type is a tuple, the registry will generate a codec
+ * automatically).
+ *
+ * @param name the name to retrieve.
+ * @return the value of {@code name} as a tuple value. If the value is NULL, then {@code null}
+ * will be returned.
+ * @throws IllegalArgumentException if {@code name} is not valid name for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the underlying CQL
+ * type to a {@code TupleValue}.
+ */
+ public TupleValue getTupleValue(String name);
+
+ /**
+ * Returns the value for {@code name} as the Java type matching its CQL type.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find the first codec that handles the
+ * underlying CQL type. The Java type of the returned object will be determined by the codec that
+ * was selected.
+ *
+ * <p>Use this method to dynamically inspect elements when types aren't known in advance, for
+ * instance if you're writing a generic row logger. If you know the target Java type, it is
+ * generally preferable to use typed getters, such as the ones for built-in types ({@link
+ * #getBool(String)}, {@link #getInt(String)}, etc.), or {@link #get(String, Class)} and {@link
+ * #get(String, TypeToken)} for custom types.
+ *
+ * @param name the name to retrieve.
+ * @return the value of {@code name} as the Java type matching its CQL type. If the value is NULL
+ * and is a simple type, UDT or tuple, {@code null} is returned. If it is NULL and is a
+ * collection type, an empty (immutable) collection is returned.
+ * @throws IllegalArgumentException if {@code name} is not a valid name for this object.
+ * @see CodecRegistry#codecFor(DataType)
+ */
+ Object getObject(String name);
+
+ /**
+ * Returns the value for {@code name} converted to the given Java type.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to convert the underlying CQL
+ * type to the given Java type.
+ *
+ * <p>If the target type is generic, use {@link #get(String, TypeToken)}.
+ *
+ * <p>Implementation note: the actual object returned by this method will depend on the {@link
+ * TypeCodec codec} being used; therefore, callers should make no assumptions concerning its
+ * mutability nor its thread-safety. Furthermore, the behavior of this method in respect to CQL
+ * {@code NULL} values is also codec-dependent; by default, a CQL {@code NULL} value translates to
+ * {@code null} for simple CQL types, UDTs and tuples, and to empty collections for all CQL
+ * collection types.
+ *
+ * @param name the name to retrieve.
+ * @param targetClass The Java type the value should be converted to.
+ * @return the value for {@code name} value converted to the given Java type.
+ * @throws IllegalArgumentException if {@code name} is not a valid name for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the underlying CQL
+ * type to {@code targetClass}.
+ */
+ <T> T get(String name, Class<T> targetClass);
+
+ /**
+ * Returns the value for {@code name} converted to the given Java type.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to convert the underlying CQL
+ * type to the given Java type.
+ *
+ * <p>Implementation note: the actual object returned by this method will depend on the {@link
+ * TypeCodec codec} being used; therefore, callers should make no assumptions concerning its
+ * mutability nor its thread-safety. Furthermore, the behavior of this method in respect to CQL
+ * {@code NULL} values is also codec-dependent; by default, a CQL {@code NULL} value translates to
+ * {@code null} for simple CQL types, UDTs and tuples, and to empty collections for all CQL
+ * collection types.
+ *
+ * @param name the name to retrieve.
+ * @param targetType The Java type the value should be converted to.
+ * @return the value for {@code name} value converted to the given Java type.
+ * @throws IllegalArgumentException if {@code name} is not a valid name for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the underlying CQL
+ * type to {@code targetType}.
+ */
+ <T> T get(String name, TypeToken<T> targetType);
+
+ /**
+ * Returns the value for {@code name} converted using the given {@link TypeCodec}.
+ *
+ * <p>This method entirely bypasses the {@link CodecRegistry} and forces the driver to use the
+ * given codec instead. This can be useful if the codec would collide with a previously registered
+ * one, or if you want to use the codec just once without registering it.
+ *
+ * <p>It is the caller's responsibility to ensure that the given codec {@link
+ * TypeCodec#accepts(DataType) accepts} the underlying CQL type; failing to do so may result in
+ * {@link InvalidTypeException}s being thrown.
+ *
+ * <p>Implementation note: the actual object returned by this method will depend on the {@link
+ * TypeCodec codec} being used; therefore, callers should make no assumptions concerning its
+ * mutability nor its thread-safety. Furthermore, the behavior of this method in respect to CQL
+ * {@code NULL} values is also codec-dependent; by default, a CQL {@code NULL} value translates to
+ * {@code null} for simple CQL types, UDTs and tuples, and to empty collections for all CQL
+ * collection types.
+ *
+ * @param name the name to retrieve.
+ * @param codec The {@link TypeCodec} to use to deserialize the value; may not be {@code null}.
+ * @return the value of the {@code i}th value converted using the given {@link TypeCodec}.
+ * @throws InvalidTypeException if the given codec does not {@link TypeCodec#accepts(DataType)
+ * accept} the underlying CQL type.
+ * @throws IndexOutOfBoundsException if {@code i} is not a valid index for this object.
+ */
+ <T> T get(String name, TypeCodec<T> codec);
+}
diff --git a/src/java/org/apache/cassandra/cql3/functions/types/GettableData.java b/src/java/org/apache/cassandra/cql3/functions/types/GettableData.java
new file mode 100644
index 0000000..e8f2b72
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/functions/types/GettableData.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.functions.types;
+
+/**
+ * Collection of (typed) CQL values that can be retrieved either by index (starting at zero) or by
+ * name.
+ */
+public interface GettableData extends GettableByIndexData, GettableByNameData
+{
+}
diff --git a/src/java/org/apache/cassandra/cql3/functions/types/LocalDate.java b/src/java/org/apache/cassandra/cql3/functions/types/LocalDate.java
new file mode 100644
index 0000000..dead6ec
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/functions/types/LocalDate.java
@@ -0,0 +1,212 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.functions.types;
+
+import java.util.*;
+import java.util.concurrent.TimeUnit;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+/**
+ * A date with no time components, no time zone, in the ISO 8601 calendar.
+ *
+ * <p>Note that ISO 8601 has a number of differences with the default gregorian calendar used in
+ * Java:
+ *
+ * <ul>
+ * <li>it uses a proleptic gregorian calendar, meaning that it's gregorian indefinitely back in
+ * the past (there is no gregorian change);
+ * <li>there is a year 0.
+ * </ul>
+ *
+ * <p>This class implements these differences, so that year/month/day fields match exactly the ones
+ * in CQL string literals.
+ *
+ * @since 2.2
+ */
+public final class LocalDate
+{
+
+ private static final TimeZone UTC = TimeZone.getTimeZone("UTC");
+
+ private final long millisSinceEpoch;
+ private final int daysSinceEpoch;
+
+ // This gets initialized lazily if we ever need it. Once set, it is effectively immutable.
+ private volatile GregorianCalendar calendar;
+
+ private LocalDate(int daysSinceEpoch)
+ {
+ this.daysSinceEpoch = daysSinceEpoch;
+ this.millisSinceEpoch = TimeUnit.DAYS.toMillis(daysSinceEpoch);
+ }
+
+ /**
+ * Builds a new instance from a number of days since January 1st, 1970 GMT.
+ *
+ * @param daysSinceEpoch the number of days.
+ * @return the new instance.
+ */
+ static LocalDate fromDaysSinceEpoch(int daysSinceEpoch)
+ {
+ return new LocalDate(daysSinceEpoch);
+ }
+
+ /**
+ * Builds a new instance from a number of milliseconds since January 1st, 1970 GMT. Note that if
+ * the given number does not correspond to a whole number of days, it will be rounded towards 0.
+ *
+ * @param millisSinceEpoch the number of milliseconds since January 1st, 1970 GMT.
+ * @return the new instance.
+ * @throws IllegalArgumentException if the date is not in the range [-5877641-06-23;
+ * 5881580-07-11].
+ */
+ static LocalDate fromMillisSinceEpoch(long millisSinceEpoch)
+ throws IllegalArgumentException
+ {
+ long daysSinceEpoch = TimeUnit.MILLISECONDS.toDays(millisSinceEpoch);
+ checkArgument(
+ daysSinceEpoch >= Integer.MIN_VALUE && daysSinceEpoch <= Integer.MAX_VALUE,
+ "Date should be in the range [-5877641-06-23; 5881580-07-11]");
+
+ return new LocalDate((int) daysSinceEpoch);
+ }
+
+ /**
+ * Returns the number of days since January 1st, 1970 GMT.
+ *
+ * @return the number of days.
+ */
+ int getDaysSinceEpoch()
+ {
+ return daysSinceEpoch;
+ }
+
+ /**
+ * Returns the year.
+ *
+ * @return the year.
+ */
+ public int getYear()
+ {
+ GregorianCalendar c = getCalendar();
+ int year = c.get(Calendar.YEAR);
+ if (c.get(Calendar.ERA) == GregorianCalendar.BC) year = -year + 1;
+ return year;
+ }
+
+ /**
+ * Returns the month.
+ *
+ * @return the month. It is 1-based, e.g. 1 for January.
+ */
+ public int getMonth()
+ {
+ return getCalendar().get(Calendar.MONTH) + 1;
+ }
+
+ /**
+ * Returns the day in the month.
+ *
+ * @return the day in the month.
+ */
+ public int getDay()
+ {
+ return getCalendar().get(Calendar.DAY_OF_MONTH);
+ }
+
+ /**
+ * Return a new {@link LocalDate} with the specified (signed) amount of time added to (or
+ * subtracted from) the given {@link Calendar} field, based on the calendar's rules.
+ *
+ * <p>Note that adding any amount to a field smaller than {@link Calendar#DAY_OF_MONTH} will
+ * remain without effect, as this class does not keep time components.
+ *
+ * <p>See {@link Calendar} javadocs for more information.
+ *
+ * @param field a {@link Calendar} field to modify.
+ * @param amount the amount of date or time to be added to the field.
+ * @return a new {@link LocalDate} with the specified (signed) amount of time added to (or
+ * subtracted from) the given {@link Calendar} field.
+ * @throws IllegalArgumentException if the new date is not in the range [-5877641-06-23;
+ * 5881580-07-11].
+ */
+ public LocalDate add(int field, int amount)
+ {
+ GregorianCalendar newCalendar = isoCalendar();
+ newCalendar.setTimeInMillis(millisSinceEpoch);
+ newCalendar.add(field, amount);
+ LocalDate newDate = fromMillisSinceEpoch(newCalendar.getTimeInMillis());
+ newDate.calendar = newCalendar;
+ return newDate;
+ }
+
+ @Override
+ public boolean equals(Object o)
+ {
+ if (this == o) return true;
+
+ if (o instanceof LocalDate)
+ {
+ LocalDate that = (LocalDate) o;
+ return this.daysSinceEpoch == that.daysSinceEpoch;
+ }
+ return false;
+ }
+
+ @Override
+ public int hashCode()
+ {
+ return daysSinceEpoch;
+ }
+
+ @Override
+ public String toString()
+ {
+ return String.format("%d-%s-%s", getYear(), pad2(getMonth()), pad2(getDay()));
+ }
+
+ private static String pad2(int i)
+ {
+ String s = Integer.toString(i);
+ return s.length() == 2 ? s : '0' + s;
+ }
+
+ private GregorianCalendar getCalendar()
+ {
+ // Two threads can race and both create a calendar. This is not a problem.
+ if (calendar == null)
+ {
+
+ // Use a local variable to only expose after we're done mutating it.
+ GregorianCalendar tmp = isoCalendar();
+ tmp.setTimeInMillis(millisSinceEpoch);
+
+ calendar = tmp;
+ }
+ return calendar;
+ }
+
+ // This matches what Cassandra uses server side (from Joda Time's LocalDate)
+ private static GregorianCalendar isoCalendar()
+ {
+ GregorianCalendar calendar = new GregorianCalendar(UTC);
+ calendar.setGregorianChange(new Date(Long.MIN_VALUE));
+ return calendar;
+ }
+}
diff --git a/src/java/org/apache/cassandra/cql3/functions/types/Metadata.java b/src/java/org/apache/cassandra/cql3/functions/types/Metadata.java
new file mode 100644
index 0000000..24ab05b
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/functions/types/Metadata.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.functions.types;
+
+import org.apache.cassandra.cql3.ColumnIdentifier;
+
+/**
+ * Keeps metadata on the connected cluster, including known nodes and schema definitions.
+ */
+public class Metadata
+{
+ /*
+ * Deal with case sensitivity for a given element id (keyspace, table, column, etc.)
+ *
+ * This method is used to convert identifiers provided by the client (through methods such as getKeyspace(String)),
+ * to the format used internally by the driver.
+ *
+ * We expect client-facing APIs to behave like cqlsh, that is:
+ * - identifiers that are mixed-case or contain special characters should be quoted.
+ * - unquoted identifiers will be lowercased: getKeyspace("Foo") will look for a keyspace named "foo"
+ */
+ static String handleId(String id)
+ {
+ // Shouldn't really happen for this method, but no reason to fail here
+ if (id == null) return null;
+
+ boolean isAlphanumericLowCase = true;
+ boolean isAlphanumeric = true;
+ for (int i = 0; i < id.length(); i++)
+ {
+ char c = id.charAt(i);
+ if (c >= 65 && c <= 90)
+ { // A-Z
+ isAlphanumericLowCase = false;
+ }
+ else if (!((c >= 48 && c <= 57) // 0-9
+ || (c == 95) // _ (underscore)
+ || (c >= 97 && c <= 122) // a-z
+ ))
+ {
+ isAlphanumeric = false;
+ isAlphanumericLowCase = false;
+ break;
+ }
+ }
+
+ if (isAlphanumericLowCase)
+ {
+ return id;
+ }
+ if (isAlphanumeric)
+ {
+ return id.toLowerCase();
+ }
+
+ // Check if it's enclosed in quotes. If it is, remove them and unescape internal double quotes
+ return ParseUtils.unDoubleQuote(id);
+ }
+
+ /**
+ * Quotes a CQL identifier if necessary.
+ *
+ * <p>This is similar to {@link #quote(String)}, except that it won't quote the input string if it
+ * can safely be used as-is. For example:
+ *
+ * <ul>
+ * <li>{@code quoteIfNecessary("foo").equals("foo")} (no need to quote).
+ * <li>{@code quoteIfNecessary("Foo").equals("\"Foo\"")} (identifier is mixed case so case
+ * sensitivity is required)
+ * <li>{@code quoteIfNecessary("foo bar").equals("\"foo bar\"")} (identifier contains special
+ * characters)
+ * <li>{@code quoteIfNecessary("table").equals("\"table\"")} (identifier is a reserved CQL
+ * keyword)
+ * </ul>
+ *
+ * @param id the "internal" form of the identifier. That is, the identifier as it would appear in
+ * Cassandra system tables (such as {@code system_schema.tables}, {@code
+ * system_schema.columns}, etc.)
+ * @return the identifier as it would appear in a CQL query string. This is also how you need to
+ * pass it to public driver methods, such as {@code #getKeyspace(String)}.
+ */
+ static String quoteIfNecessary(String id)
+ {
+ return ColumnIdentifier.maybeQuote(id);
+ }
+
+ /**
+ * Quote a keyspace, table or column identifier to make it case sensitive.
+ *
+ * <p>CQL identifiers, including keyspace, table and column ones, are case insensitive by default.
+ * Case sensitive identifiers can however be provided by enclosing the identifier in double quotes
+ * (see the <a href="http://cassandra.apache.org/doc/cql3/CQL.html#identifiers">CQL
+ * documentation</a> for details). If you are using case sensitive identifiers, this method can be
+ * used to enclose such identifiers in double quotes, making them case sensitive.
+ *
+ * <p>Note that <a
+ * href="https://docs.datastax.com/en/cql/3.0/cql/cql_reference/keywords_r.html">reserved CQL
+ * keywords</a> should also be quoted. You can check if a given identifier is a reserved keyword
+ * by calling {@code #isReservedCqlKeyword(String)}.
+ *
+ * @param id the keyspace or table identifier.
+ * @return {@code id} enclosed in double-quotes, for use in methods like {@code #getReplicas},
+ * {@code #getKeyspace}, {@code KeyspaceMetadata#getTable} or even {@code
+ * Cluster#connect(String)}.
+ */
+ public static String quote(String id)
+ {
+ return ParseUtils.doubleQuote(id);
+ }
+}
diff --git a/src/java/org/apache/cassandra/cql3/functions/types/ParseUtils.java b/src/java/org/apache/cassandra/cql3/functions/types/ParseUtils.java
new file mode 100644
index 0000000..8972bee
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/functions/types/ParseUtils.java
@@ -0,0 +1,625 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.functions.types;
+
+import java.text.*;
+import java.util.Date;
+import java.util.TimeZone;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Simple utility class used to help parsing CQL values (mainly UDT and collection ones).
+ */
+public abstract class ParseUtils
+{
+
+ /**
+ * Valid ISO-8601 patterns for CQL timestamp literals.
+ */
+ private static final String[] iso8601Patterns =
+ new String[]{
+ "yyyy-MM-dd HH:mm",
+ "yyyy-MM-dd HH:mm:ss",
+ "yyyy-MM-dd HH:mmZ",
+ "yyyy-MM-dd HH:mm:ssZ",
+ "yyyy-MM-dd HH:mm:ss.SSS",
+ "yyyy-MM-dd HH:mm:ss.SSSZ",
+ "yyyy-MM-dd'T'HH:mm",
+ "yyyy-MM-dd'T'HH:mmZ",
+ "yyyy-MM-dd'T'HH:mm:ss",
+ "yyyy-MM-dd'T'HH:mm:ssZ",
+ "yyyy-MM-dd'T'HH:mm:ss.SSS",
+ "yyyy-MM-dd'T'HH:mm:ss.SSSZ",
+ "yyyy-MM-dd",
+ "yyyy-MM-ddZ"
+ };
+
+ /**
+ * Returns the index of the first character in toParse from idx that is not a "space".
+ *
+ * @param toParse the string to skip space on.
+ * @param idx the index to start skipping space from.
+ * @return the index of the first character in toParse from idx that is not a "space.
+ */
+ static int skipSpaces(String toParse, int idx)
+ {
+ while (isBlank(toParse.charAt(idx))) ++idx;
+ return idx;
+ }
+
+ /**
+ * Assuming that idx points to the beginning of a CQL value in toParse, returns the index of the
+ * first character after this value.
+ *
+ * @param toParse the string to skip a value form.
+ * @param idx the index to start parsing a value from.
+ * @return the index ending the CQL value starting at {@code idx}.
+ * @throws IllegalArgumentException if idx doesn't point to the start of a valid CQL value.
+ */
+ static int skipCQLValue(String toParse, int idx)
+ {
+ if (idx >= toParse.length()) throw new IllegalArgumentException();
+
+ if (isBlank(toParse.charAt(idx))) throw new IllegalArgumentException();
+
+ int cbrackets = 0;
+ int sbrackets = 0;
+ int parens = 0;
+ boolean inString = false;
+
+ do
+ {
+ char c = toParse.charAt(idx);
+ if (inString)
+ {
+ if (c == '\'')
+ {
+ if (idx + 1 < toParse.length() && toParse.charAt(idx + 1) == '\'')
+ {
+ ++idx; // this is an escaped quote, skip it
+ }
+ else
+ {
+ inString = false;
+ if (cbrackets == 0 && sbrackets == 0 && parens == 0) return idx + 1;
+ }
+ }
+ // Skip any other character
+ }
+ else if (c == '\'')
+ {
+ inString = true;
+ }
+ else if (c == '{')
+ {
+ ++cbrackets;
+ }
+ else if (c == '[')
+ {
+ ++sbrackets;
+ }
+ else if (c == '(')
+ {
+ ++parens;
+ }
+ else if (c == '}')
+ {
+ if (cbrackets == 0) return idx;
+
+ --cbrackets;
+ if (cbrackets == 0 && sbrackets == 0 && parens == 0) return idx + 1;
+ }
+ else if (c == ']')
+ {
+ if (sbrackets == 0) return idx;
+
+ --sbrackets;
+ if (cbrackets == 0 && sbrackets == 0 && parens == 0) return idx + 1;
+ }
+ else if (c == ')')
+ {
+ if (parens == 0) return idx;
+
+ --parens;
+ if (cbrackets == 0 && sbrackets == 0 && parens == 0) return idx + 1;
+ }
+ else if (isBlank(c) || !isIdentifierChar(c))
+ {
+ if (cbrackets == 0 && sbrackets == 0 && parens == 0) return idx;
+ }
+ } while (++idx < toParse.length());
+
+ if (inString || cbrackets != 0 || sbrackets != 0 || parens != 0)
+ throw new IllegalArgumentException();
+ return idx;
+ }
+
+ /**
+ * Assuming that idx points to the beginning of a CQL identifier in toParse, returns the index of
+ * the first character after this identifier.
+ *
+ * @param toParse the string to skip an identifier from.
+ * @param idx the index to start parsing an identifier from.
+ * @return the index ending the CQL identifier starting at {@code idx}.
+ * @throws IllegalArgumentException if idx doesn't point to the start of a valid CQL identifier.
+ */
+ static int skipCQLId(String toParse, int idx)
+ {
+ if (idx >= toParse.length()) throw new IllegalArgumentException();
+
+ char c = toParse.charAt(idx);
+ if (isIdentifierChar(c))
+ {
+ while (idx < toParse.length() && isIdentifierChar(toParse.charAt(idx))) idx++;
+ return idx;
+ }
+
+ if (c != '"') throw new IllegalArgumentException();
+
+ while (++idx < toParse.length())
+ {
+ c = toParse.charAt(idx);
+ if (c != '"') continue;
+
+ if (idx + 1 < toParse.length() && toParse.charAt(idx + 1) == '\"')
+ ++idx; // this is an escaped double quote, skip it
+ else return idx + 1;
+ }
+ throw new IllegalArgumentException();
+ }
+
+ /**
+ * Return {@code true} if the given character is allowed in a CQL identifier, that is, if it is in
+ * the range: {@code [0..9a..zA..Z-+._&]}.
+ *
+ * @param c The character to inspect.
+ * @return {@code true} if the given character is allowed in a CQL identifier, {@code false}
+ * otherwise.
+ */
+ static boolean isIdentifierChar(int c)
+ {
+ return (c >= '0' && c <= '9')
+ || (c >= 'a' && c <= 'z')
+ || (c >= 'A' && c <= 'Z')
+ || c == '-'
+ || c == '+'
+ || c == '.'
+ || c == '_'
+ || c == '&';
+ }
+
+ /**
+ * Return {@code true} if the given character is a valid whitespace character in CQL, that is, if
+ * it is a regular space, a tabulation sign, or a new line sign.
+ *
+ * @param c The character to inspect.
+ * @return {@code true} if the given character is a valid whitespace character, {@code false}
+ * otherwise.
+ */
+ static boolean isBlank(int c)
+ {
+ return c == ' ' || c == '\t' || c == '\n';
+ }
+
+ /**
+ * Check whether the given string corresponds to a valid CQL long literal. Long literals are
+ * composed solely by digits, but can have an optional leading minus sign.
+ *
+ * @param str The string to inspect.
+ * @return {@code true} if the given string corresponds to a valid CQL integer literal, {@code
+ * false} otherwise.
+ */
+ static boolean isLongLiteral(String str)
+ {
+ if (str == null || str.isEmpty()) return false;
+ char[] chars = str.toCharArray();
+ for (int i = 0; i < chars.length; i++)
+ {
+ char c = chars[i];
+ if ((c < '0' && (i != 0 || c != '-')) || c > '9') return false;
+ }
+ return true;
+ }
+
+ /**
+ * Return {@code true} if the given string is surrounded by single quotes, and {@code false}
+ * otherwise.
+ *
+ * @param value The string to inspect.
+ * @return {@code true} if the given string is surrounded by single quotes, and {@code false}
+ * otherwise.
+ */
+ static boolean isQuoted(String value)
+ {
+ return isQuoted(value, '\'');
+ }
+
+ /**
+ * Quote the given string; single quotes are escaped. If the given string is null, this method
+ * returns a quoted empty string ({@code ''}).
+ *
+ * @param value The value to quote.
+ * @return The quoted string.
+ */
+ public static String quote(String value)
+ {
+ return quote(value, '\'');
+ }
+
+ /**
+ * Unquote the given string if it is quoted; single quotes are unescaped. If the given string is
+ * not quoted, it is returned without any modification.
+ *
+ * @param value The string to unquote.
+ * @return The unquoted string.
+ */
+ static String unquote(String value)
+ {
+ return unquote(value, '\'');
+ }
+
+ /**
+ * Double quote the given string; double quotes are escaped. If the given string is null, this
+ * method returns a quoted empty string ({@code ""}).
+ *
+ * @param value The value to double quote.
+ * @return The double quoted string.
+ */
+ static String doubleQuote(String value)
+ {
+ return quote(value, '"');
+ }
+
+ /**
+ * Unquote the given string if it is double quoted; double quotes are unescaped. If the given
+ * string is not double quoted, it is returned without any modification.
+ *
+ * @param value The string to un-double quote.
+ * @return The un-double quoted string.
+ */
+ static String unDoubleQuote(String value)
+ {
+ return unquote(value, '"');
+ }
+
+ /**
+ * Parse the given string as a date, using one of the accepted ISO-8601 date patterns.
+ *
+ * <p>This method is adapted from Apache Commons {@code DateUtils.parseStrictly()} method (that is
+ * used Cassandra side to parse date strings)..
+ *
+ * @throws ParseException If the given string is not a valid ISO-8601 date.
+ * @see <a href="https://cassandra.apache.org/doc/cql3/CQL-2.2.html#usingtimestamps">'Working with
+ * timestamps' section of CQL specification</a>
+ */
+ static Date parseDate(String str) throws ParseException
+ {
+ SimpleDateFormat parser = new SimpleDateFormat();
+ parser.setLenient(false);
+ // set a default timezone for patterns that do not provide one
+ parser.setTimeZone(TimeZone.getTimeZone("UTC"));
+ // Java 6 has very limited support for ISO-8601 time zone formats,
+ // so we need to transform the string first
+ // so that accepted patterns are correctly handled,
+ // such as Z for UTC, or "+00:00" instead of "+0000".
+ // Note: we cannot use the X letter in the pattern
+ // because it has been introduced in Java 7.
+ str = str.replaceAll("(\\+|\\-)(\\d\\d):(\\d\\d)$", "$1$2$3");
+ str = str.replaceAll("Z$", "+0000");
+ ParsePosition pos = new ParsePosition(0);
+ for (String parsePattern : iso8601Patterns)
+ {
+ parser.applyPattern(parsePattern);
+ pos.setIndex(0);
+ Date date = parser.parse(str, pos);
+ if (date != null && pos.getIndex() == str.length())
+ {
+ return date;
+ }
+ }
+ throw new ParseException("Unable to parse the date: " + str, -1);
+ }
+
+ /**
+ * Parse the given string as a date, using the supplied date pattern.
+ *
+ * <p>This method is adapted from Apache Commons {@code DateUtils.parseStrictly()} method (that is
+ * used Cassandra side to parse date strings)..
+ *
+ * @throws ParseException If the given string cannot be parsed with the given pattern.
+ * @see <a href="https://cassandra.apache.org/doc/cql3/CQL-2.2.html#usingtimestamps">'Working with
+ * timestamps' section of CQL specification</a>
+ */
+ static Date parseDate(String str, String pattern) throws ParseException
+ {
+ SimpleDateFormat parser = new SimpleDateFormat();
+ parser.setLenient(false);
+ // set a default timezone for patterns that do not provide one
+ parser.setTimeZone(TimeZone.getTimeZone("UTC"));
+ // Java 6 has very limited support for ISO-8601 time zone formats,
+ // so we need to transform the string first
+ // so that accepted patterns are correctly handled,
+ // such as Z for UTC, or "+00:00" instead of "+0000".
+ // Note: we cannot use the X letter in the pattern
+ // because it has been introduced in Java 7.
+ str = str.replaceAll("(\\+|\\-)(\\d\\d):(\\d\\d)$", "$1$2$3");
+ str = str.replaceAll("Z$", "+0000");
+ ParsePosition pos = new ParsePosition(0);
+ parser.applyPattern(pattern);
+ pos.setIndex(0);
+ Date date = parser.parse(str, pos);
+ if (date != null && pos.getIndex() == str.length())
+ {
+ return date;
+ }
+ throw new ParseException("Unable to parse the date: " + str, -1);
+ }
+
+ /**
+ * Parse the given string as a time, using the following time pattern: {@code
+ * hh:mm:ss[.fffffffff]}.
+ *
+ * <p>This method is loosely based on {@code java.sql.Timestamp}.
+ *
+ * @param str The string to parse.
+ * @return A long value representing the number of nanoseconds since midnight.
+ * @throws ParseException if the string cannot be parsed.
+ * @see <a href="https://cassandra.apache.org/doc/cql3/CQL-2.2.html#usingtime">'Working with time'
+ * section of CQL specification</a>
+ */
+ static long parseTime(String str) throws ParseException
+ {
+ String nanos_s;
+
+ long hour;
+ long minute;
+ long second;
+ long a_nanos = 0;
+
+ String formatError = "Timestamp format must be hh:mm:ss[.fffffffff]";
+ String zeros = "000000000";
+
+ if (str == null) throw new IllegalArgumentException(formatError);
+ str = str.trim();
+
+ // Parse the time
+ int firstColon = str.indexOf(':');
+ int secondColon = str.indexOf(':', firstColon + 1);
+
+ // Convert the time; default missing nanos
+ if (firstColon > 0 && secondColon > 0 && secondColon < str.length() - 1)
+ {
+ int period = str.indexOf('.', secondColon + 1);
+ hour = Integer.parseInt(str.substring(0, firstColon));
+ if (hour < 0 || hour >= 24) throw new IllegalArgumentException("Hour out of bounds.");
+
+ minute = Integer.parseInt(str.substring(firstColon + 1, secondColon));
+ if (minute < 0 || minute >= 60) throw new IllegalArgumentException("Minute out of bounds.");
+
+ if (period > 0 && period < str.length() - 1)
+ {
+ second = Integer.parseInt(str.substring(secondColon + 1, period));
+ if (second < 0 || second >= 60) throw new IllegalArgumentException("Second out of bounds.");
+
+ nanos_s = str.substring(period + 1);
+ if (nanos_s.length() > 9) throw new IllegalArgumentException(formatError);
+ if (!Character.isDigit(nanos_s.charAt(0))) throw new IllegalArgumentException(formatError);
+ nanos_s = nanos_s + zeros.substring(0, 9 - nanos_s.length());
+ a_nanos = Integer.parseInt(nanos_s);
+ }
+ else if (period > 0) throw new ParseException(formatError, -1);
+ else
+ {
+ second = Integer.parseInt(str.substring(secondColon + 1));
+ if (second < 0 || second >= 60) throw new ParseException("Second out of bounds.", -1);
+ }
+ }
+ else throw new ParseException(formatError, -1);
+
+ long rawTime = 0;
+ rawTime += TimeUnit.HOURS.toNanos(hour);
+ rawTime += TimeUnit.MINUTES.toNanos(minute);
+ rawTime += TimeUnit.SECONDS.toNanos(second);
+ rawTime += a_nanos;
+ return rawTime;
+ }
+
+ /**
+ * Format the given long value as a CQL time literal, using the following time pattern: {@code
+ * hh:mm:ss[.fffffffff]}.
+ *
+ * @param value A long value representing the number of nanoseconds since midnight.
+ * @return The formatted value.
+ * @see <a href="https://cassandra.apache.org/doc/cql3/CQL-2.2.html#usingtime">'Working with time'
+ * section of CQL specification</a>
+ */
+ static String formatTime(long value)
+ {
+ int nano = (int) (value % 1000000000);
+ value -= nano;
+ value /= 1000000000;
+ int seconds = (int) (value % 60);
+ value -= seconds;
+ value /= 60;
+ int minutes = (int) (value % 60);
+ value -= minutes;
+ value /= 60;
+ int hours = (int) (value % 24);
+ value -= hours;
+ value /= 24;
+ assert (value == 0);
+ StringBuilder sb = new StringBuilder();
+ leftPadZeros(hours, 2, sb);
+ sb.append(':');
+ leftPadZeros(minutes, 2, sb);
+ sb.append(':');
+ leftPadZeros(seconds, 2, sb);
+ sb.append('.');
+ leftPadZeros(nano, 9, sb);
+ return sb.toString();
+ }
+
+ /**
+ * Return {@code true} if the given string is surrounded by the quote character given, and {@code
+ * false} otherwise.
+ *
+ * @param value The string to inspect.
+ * @return {@code true} if the given string is surrounded by the quote character, and {@code
+ * false} otherwise.
+ */
+ private static boolean isQuoted(String value, char quoteChar)
+ {
+ return value != null
+ && value.length() > 1
+ && value.charAt(0) == quoteChar
+ && value.charAt(value.length() - 1) == quoteChar;
+ }
+
+ /**
+ * @param quoteChar " or '
+ * @return A quoted empty string.
+ */
+ private static String emptyQuoted(char quoteChar)
+ {
+ // don't handle non quote characters, this is done so that these are interned and don't create
+ // repeated empty quoted strings.
+ assert quoteChar == '"' || quoteChar == '\'';
+ if (quoteChar == '"') return "\"\"";
+ else return "''";
+ }
+
+ /**
+ * Quotes text and escapes any existing quotes in the text. {@code String.replace()} is a bit too
+ * inefficient (see JAVA-67, JAVA-1262).
+ *
+ * @param text The text.
+ * @param quoteChar The character to use as a quote.
+ * @return The text with surrounded in quotes with all existing quotes escaped with (i.e. '
+ * becomes '')
+ */
+ private static String quote(String text, char quoteChar)
+ {
+ if (text == null || text.isEmpty()) return emptyQuoted(quoteChar);
+
+ int nbMatch = 0;
+ int start = -1;
+ do
+ {
+ start = text.indexOf(quoteChar, start + 1);
+ if (start != -1) ++nbMatch;
+ } while (start != -1);
+
+ // no quotes found that need to be escaped, simply surround in quotes and return.
+ if (nbMatch == 0) return quoteChar + text + quoteChar;
+
+ // 2 for beginning and end quotes.
+ // length for original text
+ // nbMatch for escape characters to add to quotes to be escaped.
+ int newLength = 2 + text.length() + nbMatch;
+ char[] result = new char[newLength];
+ result[0] = quoteChar;
+ result[newLength - 1] = quoteChar;
+ int newIdx = 1;
+ for (int i = 0; i < text.length(); i++)
+ {
+ char c = text.charAt(i);
+ if (c == quoteChar)
+ {
+ // escape quote with another occurrence.
+ result[newIdx++] = c;
+ result[newIdx++] = c;
+ }
+ else
+ {
+ result[newIdx++] = c;
+ }
+ }
+ return new String(result);
+ }
+
+ /**
+ * Unquotes text and unescapes non surrounding quotes. {@code String.replace()} is a bit too
+ * inefficient (see JAVA-67, JAVA-1262).
+ *
+ * @param text The text
+ * @param quoteChar The character to use as a quote.
+ * @return The text with surrounding quotes removed and non surrounding quotes unescaped (i.e. ''
+ * becomes ')
+ */
+ private static String unquote(String text, char quoteChar)
+ {
+ if (!isQuoted(text, quoteChar)) return text;
+
+ if (text.length() == 2) return "";
+
+ String search = emptyQuoted(quoteChar);
+ int nbMatch = 0;
+ int start = -1;
+ do
+ {
+ start = text.indexOf(search, start + 2);
+ // ignore the second to last character occurrence, as the last character is a quote.
+ if (start != -1 && start != text.length() - 2) ++nbMatch;
+ } while (start != -1);
+
+ // no escaped quotes found, simply remove surrounding quotes and return.
+ if (nbMatch == 0) return text.substring(1, text.length() - 1);
+
+ // length of the new string will be its current length - the number of occurrences.
+ int newLength = text.length() - nbMatch - 2;
+ char[] result = new char[newLength];
+ int newIdx = 0;
+ // track whenever a quoteChar is encountered and the previous character is not a quoteChar.
+ boolean firstFound = false;
+ for (int i = 1; i < text.length() - 1; i++)
+ {
+ char c = text.charAt(i);
+ if (c == quoteChar)
+ {
+ if (firstFound)
+ {
+ // The previous character was a quoteChar, don't add this to result, this action in
+ // effect removes consecutive quotes.
+ firstFound = false;
+ }
+ else
+ {
+ // found a quoteChar and the previous character was not a quoteChar, include in result.
+ firstFound = true;
+ result[newIdx++] = c;
+ }
+ }
+ else
+ {
+ // non quoteChar encountered, include in result.
+ result[newIdx++] = c;
+ firstFound = false;
+ }
+ }
+ return new String(result);
+ }
+
+ private static void leftPadZeros(int value, int digits, StringBuilder sb)
+ {
+ sb.append(String.format("%0" + digits + 'd', value));
+ }
+
+ private ParseUtils()
+ {
+ }
+}
diff --git a/src/java/org/apache/cassandra/cql3/functions/types/SettableByIndexData.java b/src/java/org/apache/cassandra/cql3/functions/types/SettableByIndexData.java
new file mode 100644
index 0000000..a9d0898
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/functions/types/SettableByIndexData.java
@@ -0,0 +1,583 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.functions.types;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.net.InetAddress;
+import java.nio.ByteBuffer;
+import java.util.*;
+
+import com.google.common.reflect.TypeToken;
+
+import org.apache.cassandra.cql3.functions.types.exceptions.CodecNotFoundException;
+import org.apache.cassandra.cql3.functions.types.exceptions.InvalidTypeException;
+
+/**
+ * Collection of (typed) CQL values that can be set by index (starting at zero).
+ */
+public interface SettableByIndexData<T extends SettableByIndexData<T>>
+{
+
+ /**
+ * Sets the {@code i}th value to the provided boolean.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to handle the conversion to the
+ * underlying CQL type (for CQL type {@code boolean}, this will be the built-in codec).
+ *
+ * @param i the index of the value to set.
+ * @param v the value to set. To set the value to NULL, use {@link #setToNull(int)} or {@code
+ * set(i, v, Boolean.class)}
+ * @return this object.
+ * @throws IndexOutOfBoundsException if {@code i} is not a valid index for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the value to the
+ * underlying CQL type.
+ */
+ public T setBool(int i, boolean v);
+
+ /**
+ * Set the {@code i}th value to the provided byte.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to handle the conversion to the
+ * underlying CQL type (for CQL type {@code tinyint}, this will be the built-in codec).
+ *
+ * @param i the index of the value to set.
+ * @param v the value to set. To set the value to NULL, use {@link #setToNull(int)} or {@code
+ * set(i, v, Byte.class)}
+ * @return this object.
+ * @throws IndexOutOfBoundsException if {@code i} is not a valid index for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the value to the
+ * underlying CQL type.
+ */
+ public T setByte(int i, byte v);
+
+ /**
+ * Set the {@code i}th value to the provided short.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to handle the conversion to the
+ * underlying CQL type (for CQL type {@code smallint}, this will be the built-in codec).
+ *
+ * @param i the index of the value to set.
+ * @param v the value to set. To set the value to NULL, use {@link #setToNull(int)} or {@code
+ * set(i, v, Short.class)}
+ * @return this object.
+ * @throws IndexOutOfBoundsException if {@code i} is not a valid index for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the value to the
+ * underlying CQL type.
+ */
+ public T setShort(int i, short v);
+
+ /**
+ * Set the {@code i}th value to the provided integer.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to handle the conversion to the
+ * underlying CQL type (for CQL type {@code int}, this will be the built-in codec).
+ *
+ * @param i the index of the value to set.
+ * @param v the value to set. To set the value to NULL, use {@link #setToNull(int)} or {@code
+ * set(i, v, Integer.class)}
+ * @return this object.
+ * @throws IndexOutOfBoundsException if {@code i} is not a valid index for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the value to the
+ * underlying CQL type.
+ */
+ public T setInt(int i, int v);
+
+ /**
+ * Sets the {@code i}th value to the provided long.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to handle the conversion to the
+ * underlying CQL type (for CQL type {@code bigint}, this will be the built-in codec).
+ *
+ * @param i the index of the value to set.
+ * @param v the value to set. To set the value to NULL, use {@link #setToNull(int)} or {@code
+ * set(i, v, Long.class)}
+ * @return this object.
+ * @throws IndexOutOfBoundsException if {@code i} is not a valid index for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the value to the
+ * underlying CQL type.
+ */
+ public T setLong(int i, long v);
+
+ /**
+ * Set the {@code i}th value to the provided date.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to handle the conversion to the
+ * underlying CQL type (for CQL type {@code timestamp}, this will be the built-in codec).
+ *
+ * @param i the index of the value to set.
+ * @param v the value to set.
+ * @return this object.
+ * @throws IndexOutOfBoundsException if {@code i} is not a valid index for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the value to the
+ * underlying CQL type.
+ */
+ public T setTimestamp(int i, Date v);
+
+ /**
+ * Set the {@code i}th value to the provided date (without time).
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to handle the conversion to the
+ * underlying CQL type (for CQL type {@code date}, this will be the built-in codec).
+ *
+ * @param i the index of the value to set.
+ * @param v the value to set.
+ * @return this object.
+ * @throws IndexOutOfBoundsException if {@code i} is not a valid index for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the value to the
+ * underlying CQL type.
+ */
+ public T setDate(int i, LocalDate v);
+
+ /**
+ * Set the {@code i}th value to the provided time as a long in nanoseconds since midnight.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to handle the conversion to the
+ * underlying CQL type (for CQL type {@code time}, this will be the built-in codec).
+ *
+ * @param i the index of the value to set.
+ * @param v the value to set.
+ * @return this object.
+ * @throws IndexOutOfBoundsException if {@code i} is not a valid index for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the value to the
+ * underlying CQL type.
+ */
+ public T setTime(int i, long v);
+
+ /**
+ * Sets the {@code i}th value to the provided float.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to handle the conversion to the
+ * underlying CQL type (for CQL type {@code float}, this will be the built-in codec).
+ *
+ * @param i the index of the value to set.
+ * @param v the value to set. To set the value to NULL, use {@link #setToNull(int)} or {@code
+ * set(i, v, Float.class)}
+ * @return this object.
+ * @throws IndexOutOfBoundsException if {@code i} is not a valid index for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the value to the
+ * underlying CQL type.
+ */
+ public T setFloat(int i, float v);
+
+ /**
+ * Sets the {@code i}th value to the provided double.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to handle the conversion to the
+ * underlying CQL type (for CQL type {@code double}, this will be the built-in codec).
+ *
+ * @param i the index of the value to set.
+ * @param v the value to set. To set the value to NULL, use {@link #setToNull(int)} or {@code
+ * set(i, v, Double.class)}.
+ * @return this object.
+ * @throws IndexOutOfBoundsException if {@code i} is not a valid index for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the value to the
+ * underlying CQL type.
+ */
+ public T setDouble(int i, double v);
+
+ /**
+ * Sets the {@code i}th value to the provided string.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to handle the conversion to the
+ * underlying CQL type (for CQL types {@code text}, {@code varchar} and {@code ascii}, this will
+ * be the built-in codec).
+ *
+ * @param i the index of the value to set.
+ * @param v the value to set.
+ * @return this object.
+ * @throws IndexOutOfBoundsException if {@code i} is not a valid index for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the value to the
+ * underlying CQL type.
+ */
+ public T setString(int i, String v);
+
+ /**
+ * Sets the {@code i}th value to the provided byte buffer.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to handle the conversion to the
+ * underlying CQL type (for CQL type {@code blob}, this will be the built-in codec).
+ *
+ * @param i the index of the value to set.
+ * @param v the value to set.
+ * @return this object.
+ * @throws IndexOutOfBoundsException if {@code i} is not a valid index for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the value to the
+ * underlying CQL type.
+ */
+ public T setBytes(int i, ByteBuffer v);
+
+ /**
+ * Sets the {@code i}th value to the provided byte buffer.
+ *
+ * <p>This method does not use any codec; it sets the value in its binary form directly. If you
+ * insert data that is not compatible with the underlying CQL type, you will get an {@code
+ * InvalidQueryException} at execute time.
+ *
+ * @param i the index of the value to set.
+ * @param v the value to set.
+ * @return this object.
+ * @throws IndexOutOfBoundsException if {@code i} is not a valid index for this object.
+ */
+ public T setBytesUnsafe(int i, ByteBuffer v);
+
+ /**
+ * Sets the {@code i}th value to the provided big integer.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to handle the conversion to the
+ * underlying CQL type (for CQL type {@code varint}, this will be the built-in codec).
+ *
+ * @param i the index of the value to set.
+ * @param v the value to set.
+ * @return this object.
+ * @throws IndexOutOfBoundsException if {@code i} is not a valid index for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the value to the
+ * underlying CQL type.
+ */
+ public T setVarint(int i, BigInteger v);
+
+ /**
+ * Sets the {@code i}th value to the provided big decimal.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to handle the conversion to the
+ * underlying CQL type (for CQL type {@code decimal}, this will be the built-in codec).
+ *
+ * @param i the index of the value to set.
+ * @param v the value to set.
+ * @return this object.
+ * @throws IndexOutOfBoundsException if {@code i} is not a valid index for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the value to the
+ * underlying CQL type.
+ */
+ public T setDecimal(int i, BigDecimal v);
+
+ /**
+ * Sets the {@code i}th value to the provided UUID.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to handle the conversion to the
+ * underlying CQL type (for CQL types {@code uuid} and {@code timeuuid}, this will be the built-in
+ * codec).
+ *
+ * @param i the index of the value to set.
+ * @param v the value to set.
+ * @return this object.
+ * @throws IndexOutOfBoundsException if {@code i} is not a valid index for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the value to the
+ * underlying CQL type.
+ */
+ public T setUUID(int i, UUID v);
+
+ /**
+ * Sets the {@code i}th value to the provided inet address.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to handle the conversion to the
+ * underlying CQL type (for CQL type {@code inet}, this will be the built-in codec).
+ *
+ * @param i the index of the value to set.
+ * @param v the value to set.
+ * @return this object.
+ * @throws IndexOutOfBoundsException if {@code i} is not a valid index for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the value to the
+ * underlying CQL type.
+ */
+ public T setInet(int i, InetAddress v);
+
+ /**
+ * Sets the {@code i}th value to the provided list.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to handle the conversion to the
+ * underlying CQL type (the type of the elements in the Java list is not considered). If two or
+ * more codecs target that CQL type, the one that was first registered will be used. For this
+ * reason, it is generally preferable to use the more deterministic methods {@link #setList(int,
+ * List, Class)} or {@link #setList(int, List, TypeToken)}.
+ *
+ * @param i the index of the value to set.
+ * @param v the value to set. Note that {@code null} values inside collections are not supported
+ * by CQL.
+ * @return this object.
+ * @throws IndexOutOfBoundsException if {@code i} is not a valid index for this object.
+ * @throws NullPointerException if {@code v} contains null values. Nulls are not supported in
+ * collections by CQL.
+ * @throws CodecNotFoundException if there is no registered codec to convert the value to the
+ * underlying CQL type.
+ */
+ public <E> T setList(int i, List<E> v);
+
+ /**
+ * Sets the {@code i}th value to the provided list, which elements are of the provided Java class.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to handle the conversion of lists
+ * of the given Java type to the underlying CQL type.
+ *
+ * <p>If the type of the elements is generic, use {@link #setList(int, List, TypeToken)}.
+ *
+ * @param i the index of the value to set.
+ * @param v the value to set. Note that {@code null} values inside collections are not supported
+ * by CQL.
+ * @param elementsClass the class for the elements of the list.
+ * @return this object.
+ * @throws IndexOutOfBoundsException if {@code i} is not a valid index for this object.
+ * @throws NullPointerException if {@code v} contains null values. Nulls are not supported in
+ * collections by CQL.
+ * @throws CodecNotFoundException if there is no registered codec to convert the value to the
+ * underlying CQL type.
+ */
+ public <E> T setList(int i, List<E> v, Class<E> elementsClass);
+
+ /**
+ * Sets the {@code i}th value to the provided list, which elements are of the provided Java type.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to handle the conversion of lists
+ * of the given Java type to the underlying CQL type.
+ *
+ * @param i the index of the value to set.
+ * @param v the value to set. Note that {@code null} values inside collections are not supported
+ * by CQL.
+ * @param elementsType the type for the elements of the list.
+ * @return this object.
+ * @throws IndexOutOfBoundsException if {@code i} is not a valid index for this object.
+ * @throws NullPointerException if {@code v} contains null values. Nulls are not supported in
+ * collections by CQL.
+ * @throws CodecNotFoundException if there is no registered codec to convert the value to the
+ * underlying CQL type.
+ */
+ public <E> T setList(int i, List<E> v, TypeToken<E> elementsType);
+
+ /**
+ * Sets the {@code i}th value to the provided map.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to handle the conversion to the
+ * underlying CQL type (the type of the elements in the Java map is not considered). If two or
+ * more codecs target that CQL type, the one that was first registered will be used. For this
+ * reason, it is generally preferable to use the more deterministic methods {@link #setMap(int,
+ * Map, Class, Class)} or {@link #setMap(int, Map, TypeToken, TypeToken)}.
+ *
+ * @param i the index of the value to set.
+ * @param v the value to set. Note that {@code null} values inside collections are not supported
+ * by CQL.
+ * @return this object.
+ * @throws IndexOutOfBoundsException if {@code i} is not a valid index for this object.
+ * @throws NullPointerException if {@code v} contains null values. Nulls are not supported in
+ * collections by CQL.
+ * @throws CodecNotFoundException if there is no registered codec to convert the value to the
+ * underlying CQL type.
+ */
+ public <K, V> T setMap(int i, Map<K, V> v);
+
+ /**
+ * Sets the {@code i}th value to the provided map, which keys and values are of the provided Java
+ * classes.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to handle the conversion of lists
+ * of the given Java types to the underlying CQL type.
+ *
+ * <p>If the type of the keys or values is generic, use {@link #setMap(int, Map, TypeToken,
+ * TypeToken)}.
+ *
+ * @param i the index of the value to set.
+ * @param v the value to set. Note that {@code null} values inside collections are not supported
+ * by CQL.
+ * @param keysClass the class for the keys of the map.
+ * @param valuesClass the class for the values of the map.
+ * @return this object.
+ * @throws IndexOutOfBoundsException if {@code i} is not a valid index for this object.
+ * @throws NullPointerException if {@code v} contains null values. Nulls are not supported in
+ * collections by CQL.
+ * @throws CodecNotFoundException if there is no registered codec to convert the value to the
+ * underlying CQL type.
+ */
+ public <K, V> T setMap(int i, Map<K, V> v, Class<K> keysClass, Class<V> valuesClass);
+
+ /**
+ * Sets the {@code i}th value to the provided map, which keys and values are of the provided Java
+ * types.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to handle the conversion of lists
+ * of the given Java types to the underlying CQL type.
+ *
+ * @param i the index of the value to set.
+ * @param v the value to set. Note that {@code null} values inside collections are not supported
+ * by CQL.
+ * @param keysType the type for the keys of the map.
+ * @param valuesType the type for the values of the map.
+ * @return this object.
+ * @throws IndexOutOfBoundsException if {@code i} is not a valid index for this object.
+ * @throws NullPointerException if {@code v} contains null values. Nulls are not supported in
+ * collections by CQL.
+ * @throws CodecNotFoundException if there is no registered codec to convert the value to the
+ * underlying CQL type.
+ */
+ public <K, V> T setMap(int i, Map<K, V> v, TypeToken<K> keysType, TypeToken<V> valuesType);
+
+ /**
+ * Sets the {@code i}th value to the provided set.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to handle the conversion to the
+ * underlying CQL type (the type of the elements in the Java set is not considered). If two or
+ * more codecs target that CQL type, the one that was first registered will be used. For this
+ * reason, it is generally preferable to use the more deterministic methods {@link #setSet(int,
+ * Set, Class)} or {@link #setSet(int, Set, TypeToken)}.
+ *
+ * @param i the index of the value to set.
+ * @param v the value to set. Note that {@code null} values inside collections are not supported
+ * by CQL.
+ * @return this object.
+ * @throws IndexOutOfBoundsException if {@code i} is not a valid index for this object.
+ * @throws NullPointerException if {@code v} contains null values. Nulls are not supported in
+ * collections by CQL.
+ * @throws CodecNotFoundException if there is no registered codec to convert the value to the
+ * underlying CQL type.
+ */
+ public <E> T setSet(int i, Set<E> v);
+
+ /**
+ * Sets the {@code i}th value to the provided set, which elements are of the provided Java class.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to handle the conversion of sets
+ * of the given Java type to the underlying CQL type.
+ *
+ * <p>If the type of the elements is generic, use {@link #setSet(int, Set, TypeToken)}.
+ *
+ * @param i the index of the value to set.
+ * @param v the value to set. Note that {@code null} values inside collections are not supported
+ * by CQL.
+ * @param elementsClass the class for the elements of the set.
+ * @return this object.
+ * @throws IndexOutOfBoundsException if {@code i} is not a valid index for this object.
+ * @throws NullPointerException if {@code v} contains null values. Nulls are not supported in
+ * collections by CQL.
+ * @throws CodecNotFoundException if there is no registered codec to convert the value to the
+ * underlying CQL type.
+ */
+ public <E> T setSet(int i, Set<E> v, Class<E> elementsClass);
+
+ /**
+ * Sets the {@code i}th value to the provided set, which elements are of the provided Java type.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to handle the conversion of sets
+ * of the given Java type to the underlying CQL type.
+ *
+ * @param i the index of the value to set.
+ * @param v the value to set. Note that {@code null} values inside collections are not supported
+ * by CQL.
+ * @param elementsType the type for the elements of the set.
+ * @return this object.
+ * @throws IndexOutOfBoundsException if {@code i} is not a valid index for this object.
+ * @throws NullPointerException if {@code v} contains null values. Nulls are not supported in
+ * collections by CQL.
+ * @throws CodecNotFoundException if there is no registered codec to convert the value to the
+ * underlying CQL type.
+ */
+ public <E> T setSet(int i, Set<E> v, TypeToken<E> elementsType);
+
+ /**
+ * Sets the {@code i}th value to the provided UDT value.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to handle the conversion of
+ * {@code UDTValue} to the underlying CQL type.
+ *
+ * @param i the index of the value to set.
+ * @param v the value to set.
+ * @return this object.
+ * @throws IndexOutOfBoundsException if {@code i} is not a valid index for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the value to the
+ * underlying CQL type.
+ */
+ public T setUDTValue(int i, UDTValue v);
+
+ /**
+ * Sets the {@code i}th value to the provided tuple value.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to handle the conversion of
+ * {@code TupleValue} to the underlying CQL type.
+ *
+ * @param i the index of the value to set.
+ * @param v the value to set.
+ * @return this object.
+ * @throws IndexOutOfBoundsException if {@code i} is not a valid index for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the value to the
+ * underlying CQL type.
+ */
+ public T setTupleValue(int i, TupleValue v);
+
+ /**
+ * Sets the {@code i}th value to {@code null}.
+ *
+ * <p>This is mainly intended for CQL types which map to native Java types.
+ *
+ * @param i the index of the value to set.
+ * @return this object.
+ * @throws IndexOutOfBoundsException if {@code i} is not a valid index for this object.
+ */
+ public T setToNull(int i);
+
+ /**
+ * Sets the {@code i}th value to the provided value of the provided Java class.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to handle the conversion of the
+ * provided Java class to the underlying CQL type.
+ *
+ * <p>If the Java type is generic, use {@link #set(int, Object, TypeToken)} instead.
+ *
+ * @param i the index of the value to set.
+ * @param v the value to set; may be {@code null}.
+ * @param targetClass The Java class to convert to; must not be {@code null};
+ * @return this object.
+ * @throws IndexOutOfBoundsException if {@code i} is not a valid index for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the value to the
+ * underlying CQL type.
+ */
+ <V> T set(int i, V v, Class<V> targetClass);
+
+ /**
+ * Sets the {@code i}th value to the provided value of the provided Java type.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to handle the conversion of the
+ * provided Java type to the underlying CQL type.
+ *
+ * @param i the index of the value to set.
+ * @param v the value to set; may be {@code null}.
+ * @param targetType The Java type to convert to; must not be {@code null};
+ * @return this object.
+ * @throws IndexOutOfBoundsException if {@code i} is not a valid index for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the value to the
+ * underlying CQL type.
+ */
+ <V> T set(int i, V v, TypeToken<V> targetType);
+
+ /**
+ * Sets the {@code i}th value to the provided value, converted using the given {@link TypeCodec}.
+ *
+ * <p>This method entirely bypasses the {@link CodecRegistry} and forces the driver to use the
+ * given codec instead. This can be useful if the codec would collide with a previously registered
+ * one, or if you want to use the codec just once without registering it.
+ *
+ * <p>It is the caller's responsibility to ensure that the given codec {@link
+ * TypeCodec#accepts(DataType) accepts} the underlying CQL type; failing to do so may result in
+ * {@link InvalidTypeException}s being thrown.
+ *
+ * @param i the index of the value to set.
+ * @param v the value to set; may be {@code null}.
+ * @param codec The {@link TypeCodec} to use to serialize the value; may not be {@code null}.
+ * @return this object.
+ * @throws InvalidTypeException if the given codec does not {@link TypeCodec#accepts(DataType)
+ * accept} the underlying CQL type.
+ * @throws IndexOutOfBoundsException if {@code i} is not a valid index for this object.
+ */
+ <V> T set(int i, V v, TypeCodec<V> codec);
+}
diff --git a/src/java/org/apache/cassandra/cql3/functions/types/SettableByNameData.java b/src/java/org/apache/cassandra/cql3/functions/types/SettableByNameData.java
new file mode 100644
index 0000000..514ba60
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/functions/types/SettableByNameData.java
@@ -0,0 +1,620 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.functions.types;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.net.InetAddress;
+import java.nio.ByteBuffer;
+import java.util.*;
+
+import com.google.common.reflect.TypeToken;
+
+import org.apache.cassandra.cql3.functions.types.exceptions.CodecNotFoundException;
+import org.apache.cassandra.cql3.functions.types.exceptions.InvalidTypeException;
+
+/**
+ * Collection of (typed) CQL values that can set by name.
+ */
+public interface SettableByNameData<T extends SettableData<T>>
+{
+
+ /**
+ * Sets the value for (all occurrences of) variable {@code name} to the provided boolean.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to handle the conversion to the
+ * underlying CQL type (for CQL type {@code boolean}, this will be the built-in codec).
+ *
+ * @param name the name of the value to set; if {@code name} is present multiple times, all its
+ * values are set.
+ * @param v the value to set. To set the value to NULL, use {@link #setToNull(String)} or {@code
+ * set(name, v, Boolean.class)}.
+ * @return this object.
+ * @throws IllegalArgumentException if {@code name} is not a valid name for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the value to the
+ * underlying CQL type.
+ */
+ public T setBool(String name, boolean v);
+
+ /**
+ * Sets the value for (all occurrences of) variable {@code name} to the provided byte.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to handle the conversion to the
+ * underlying CQL type (for CQL type {@code tinyint}, this will be the built-in codec).
+ *
+ * @param name the name of the value to set; if {@code name} is present multiple times, all its
+ * values are set.
+ * @param v the value to set. To set the value to NULL, use {@link #setToNull(String)} or {@code
+ * set(name, v, Byte.class)}.
+ * @return this object.
+ * @throws IllegalArgumentException if {@code name} is not a valid name for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the value to the
+ * underlying CQL type.
+ */
+ public T setByte(String name, byte v);
+
+ /**
+ * Sets the value for (all occurrences of) variable {@code name} to the provided short.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to handle the conversion to the
+ * underlying CQL type (for CQL type {@code smallint}, this will be the built-in codec).
+ *
+ * @param name the name of the value to set; if {@code name} is present multiple times, all its
+ * values are set.
+ * @param v the value to set. To set the value to NULL, use {@link #setToNull(String)} or {@code
+ * set(name, v, Short.class)}.
+ * @return this object.
+ * @throws IllegalArgumentException if {@code name} is not a valid name for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the value to the
+ * underlying CQL type.
+ */
+ public T setShort(String name, short v);
+
+ /**
+ * Sets the value for (all occurrences of) variable {@code name} to the provided integer.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to handle the conversion to the
+ * underlying CQL type (for CQL type {@code int}, this will be the built-in codec).
+ *
+ * @param name the name of the value to set; if {@code name} is present multiple times, all its
+ * values are set.
+ * @param v the value to set. To set the value to NULL, use {@link #setToNull(String)} or {@code
+ * set(name, v, Integer.class)}.
+ * @return this object.
+ * @throws IllegalArgumentException if {@code name} is not a valid name for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the value to the
+ * underlying CQL type.
+ */
+ public T setInt(String name, int v);
+
+ /**
+ * Sets the value for (all occurrences of) variable {@code name} to the provided long.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to handle the conversion to the
+ * underlying CQL type (for CQL type {@code bigint}, this will be the built-in codec).
+ *
+ * @param name the name of the value to set; if {@code name} is present multiple times, all its
+ * values are set.
+ * @param v the value to set. To set the value to NULL, use {@link #setToNull(String)} or {@code
+ * set(name, v, Long.class)}.
+ * @return this object.
+ * @throws IllegalArgumentException if {@code name} is not a valid name for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the value to the
+ * underlying CQL type.
+ */
+ public T setLong(String name, long v);
+
+ /**
+ * Sets the value for (all occurrences of) variable {@code name} to the provided date.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to handle the conversion to the
+ * underlying CQL type (for CQL type {@code timestamp}, this will be the built-in codec).
+ *
+ * @param name the name of the value to set; if {@code name} is present multiple times, all its
+ * values are set.
+ * @param v the value to set.
+ * @return this object.
+ * @throws IllegalArgumentException if {@code name} is not a valid name for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the value to the
+ * underlying CQL type.
+ */
+ public T setTimestamp(String name, Date v);
+
+ /**
+ * Sets the value for (all occurrences of) variable {@code name} to the provided date (without
+ * time).
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to handle the conversion to the
+ * underlying CQL type (for CQL type {@code date}, this will be the built-in codec).
+ *
+ * @param name the name of the value to set; if {@code name} is present multiple times, all its
+ * values are set.
+ * @param v the value to set.
+ * @return this object.
+ * @throws IllegalArgumentException if {@code name} is not a valid name for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the value to the
+ * underlying CQL type.
+ */
+ public T setDate(String name, LocalDate v);
+
+ /**
+ * Sets the value for (all occurrences of) variable {@code name} to the provided time as a long in
+ * nanoseconds since midnight.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to handle the conversion to the
+ * underlying CQL type (for CQL type {@code time}, this will be the built-in codec).
+ *
+ * @param name the name of the value to set; if {@code name} is present multiple times, all its
+ * values are set.
+ * @param v the value to set.
+ * @return this object.
+ * @throws IllegalArgumentException if {@code name} is not a valid name for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the value to the
+ * underlying CQL type.
+ */
+ public T setTime(String name, long v);
+
+ /**
+ * Sets the value for (all occurrences of) variable {@code name} to the provided float.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to handle the conversion to the
+ * underlying CQL type (for CQL type {@code float}, this will be the built-in codec).
+ *
+ * @param name the name of the value to set; if {@code name} is present multiple times, all its
+ * values are set.
+ * @param v the value to set. To set the value to NULL, use {@link #setToNull(String)} or {@code
+ * set(name, v, Float.class)}.
+ * @return this object.
+ * @throws IllegalArgumentException if {@code name} is not a valid name for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the value to the
+ * underlying CQL type.
+ */
+ public T setFloat(String name, float v);
+
+ /**
+ * Sets the value for (all occurrences of) variable {@code name} to the provided double.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to handle the conversion to the
+ * underlying CQL type (for CQL type {@code double}, this will be the built-in codec).
+ *
+ * @param name the name of the value to set; if {@code name} is present multiple times, all its
+ * values are set.
+ * @param v the value to set. To set the value to NULL, use {@link #setToNull(String)} or {@code
+ * set(name, v, Double.class)}.
+ * @return this object.
+ * @throws IllegalArgumentException if {@code name} is not a valid name for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the value to the
+ * underlying CQL type.
+ */
+ public T setDouble(String name, double v);
+
+ /**
+ * Sets the value for (all occurrences of) variable {@code name} to the provided string.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to handle the conversion to the
+ * underlying CQL type (for CQL types {@code text}, {@code varchar} and {@code ascii}, this will
+ * be the built-in codec).
+ *
+ * @param name the name of the value to set; if {@code name} is present multiple times, all its
+ * values are set.
+ * @param v the value to set.
+ * @return this object.
+ * @throws IllegalArgumentException if {@code name} is not a valid name for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the value to the
+ * underlying CQL type.
+ */
+ public T setString(String name, String v);
+
+ /**
+ * Sets the value for (all occurrences of) variable {@code name} to the provided byte buffer.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to handle the conversion to the
+ * underlying CQL type (for CQL type {@code blob}, this will be the built-in codec).
+ *
+ * @param name the name of the value to set; if {@code name} is present multiple times, all its
+ * values are set.
+ * @param v the value to set.
+ * @return this object.
+ * @throws IllegalArgumentException if {@code name} is not a valid name for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the value to the
+ * underlying CQL type.
+ */
+ public T setBytes(String name, ByteBuffer v);
+
+ /**
+ * Sets the value for (all occurrences of) variable {@code name} to the provided byte buffer.
+ *
+ * <p>This method does not use any codec; it sets the value in its binary form directly. If you
+ * insert data that is not compatible with the underlying CQL type, you will get an {@code
+ * InvalidQueryException} at execute time.
+ *
+ * @param name the name of the value to set; if {@code name} is present multiple times, all its
+ * values are set.
+ * @param v the value to set.
+ * @return this object.
+ * @throws IllegalArgumentException if {@code name} is not a valid name for this object.
+ */
+ public T setBytesUnsafe(String name, ByteBuffer v);
+
+ /**
+ * Sets the value for (all occurrences of) variable {@code name} to the provided big integer.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to handle the conversion to the
+ * underlying CQL type (for CQL type {@code varint}, this will be the built-in codec).
+ *
+ * @param name the name of the value to set; if {@code name} is present multiple times, all its
+ * values are set.
+ * @param v the value to set.
+ * @return this object.
+ * @throws IllegalArgumentException if {@code name} is not a valid name for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the value to the
+ * underlying CQL type.
+ */
+ public T setVarint(String name, BigInteger v);
+
+ /**
+ * Sets the value for (all occurrences of) variable {@code name} to the provided big decimal.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to handle the conversion to the
+ * underlying CQL type (for CQL type {@code decimal}, this will be the built-in codec).
+ *
+ * @param name the name of the value to set; if {@code name} is present multiple times, all its
+ * values are set.
+ * @param v the value to set.
+ * @return this object.
+ * @throws IllegalArgumentException if {@code name} is not a valid name for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the value to the
+ * underlying CQL type.
+ */
+ public T setDecimal(String name, BigDecimal v);
+
+ /**
+ * Sets the value for (all occurrences of) variable {@code name} to the provided UUID.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to handle the conversion to the
+ * underlying CQL type (for CQL types {@code uuid} and {@code timeuuid}, this will be the built-in
+ * codec).
+ *
+ * @param name the name of the value to set; if {@code name} is present multiple times, all its
+ * values are set.
+ * @param v the value to set.
+ * @return this object.
+ * @throws IllegalArgumentException if {@code name} is not a valid name for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the value to the
+ * underlying CQL type.
+ */
+ public T setUUID(String name, UUID v);
+
+ /**
+ * Sets the value for (all occurrences of) variable {@code name} to the provided inet address.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to handle the conversion to the
+ * underlying CQL type (for CQL type {@code inet}, this will be the built-in codec).
+ *
+ * @param name the name of the value to set; if {@code name} is present multiple times, all its
+ * values are set.
+ * @param v the value to set.
+ * @return this object.
+ * @throws IllegalArgumentException if {@code name} is not a valid name for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the value to the
+ * underlying CQL type.
+ */
+ public T setInet(String name, InetAddress v);
+
+ /**
+ * Sets the value for (all occurrences of) variable {@code name} to the provided list.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to handle the conversion to the
+ * underlying CQL type (the type of the elements in the Java list is not considered). If two or
+ * more codecs target that CQL type, the one that was first registered will be used. For this
+ * reason, it is generally preferable to use the more deterministic methods {@link
+ * #setList(String, List, Class)} or {@link #setList(String, List, TypeToken)}.
+ *
+ * @param name the name of the value to set; if {@code name} is present multiple times, all its
+ * values are set.
+ * @param v the value to set. Note that {@code null} values inside collections are not supported
+ * by CQL.
+ * @return this object.
+ * @throws IllegalArgumentException if {@code name} is not a valid name for this object.
+ * @throws NullPointerException if {@code v} contains null values. Nulls are not supported in
+ * collections by CQL.
+ * @throws CodecNotFoundException if there is no registered codec to convert the value to the
+ * underlying CQL type.
+ */
+ public <E> T setList(String name, List<E> v);
+
+ /**
+ * Sets the value for (all occurrences of) variable {@code name} to the provided list, which
+ * elements are of the provided Java class.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to handle the conversion of lists
+ * of the given Java type to the underlying CQL type.
+ *
+ * <p>If the type of the elements is generic, use {@link #setList(String, List, TypeToken)}.
+ *
+ * @param name the name of the value to set; if {@code name} is present multiple
+ * @param v the value to set. Note that {@code null} values inside collections are not supported
+ * by CQL.
+ * @param elementsClass the class for the elements of the list.
+ * @return this object.
+ * @throws IllegalArgumentException if {@code name} is not a valid name for this object.
+ * @throws NullPointerException if {@code v} contains null values. Nulls are not supported in
+ * collections by CQL.
+ * @throws CodecNotFoundException if there is no registered codec to convert the value to the
+ * underlying CQL type.
+ */
+ public <E> T setList(String name, List<E> v, Class<E> elementsClass);
+
+ /**
+ * Sets the value for (all occurrences of) variable {@code name} to the provided list, which
+ * elements are of the provided Java type.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to handle the conversion of lists
+ * of the given Java type to the underlying CQL type.
+ *
+ * @param name the name of the value to set; if {@code name} is present multiple
+ * @param v the value to set. Note that {@code null} values inside collections are not supported
+ * by CQL.
+ * @param elementsType the type for the elements of the list.
+ * @return this object.
+ * @throws IllegalArgumentException if {@code name} is not a valid name for this object.
+ * @throws NullPointerException if {@code v} contains null values. Nulls are not supported in
+ * collections by CQL.
+ * @throws CodecNotFoundException if there is no registered codec to convert the value to the
+ * underlying CQL type.
+ */
+ public <E> T setList(String name, List<E> v, TypeToken<E> elementsType);
+
+ /**
+ * Sets the value for (all occurrences of) variable {@code name} to the provided map.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to handle the conversion to the
+ * underlying CQL type (the type of the elements in the Java map is not considered). If two or
+ * more codecs target that CQL type, the one that was first registered will be used. For this
+ * reason, it is generally preferable to use the more deterministic methods {@link #setMap(String,
+ * Map, Class, Class)} or {@link #setMap(String, Map, TypeToken, TypeToken)}.
+ *
+ * @param name the name of the value to set; if {@code name} is present multiple times, all its
+ * values are set.
+ * @param v the value to set. Note that {@code null} values inside collections are not supported
+ * by CQL.
+ * @return this object.
+ * @throws IllegalArgumentException if {@code name} is not a valid name for this object.
+ * @throws NullPointerException if {@code v} contains null values. Nulls are not supported in
+ * collections by CQL.
+ * @throws CodecNotFoundException if there is no registered codec to convert the value to the
+ * underlying CQL type.
+ */
+ public <K, V> T setMap(String name, Map<K, V> v);
+
+ /**
+ * Sets the value for (all occurrences of) variable {@code name} to the provided map, which keys
+ * and values are of the provided Java classes.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to handle the conversion of lists
+ * of the given Java types to the underlying CQL type.
+ *
+ * <p>If the type of the keys or values is generic, use {@link #setMap(String, Map, TypeToken,
+ * TypeToken)}.
+ *
+ * @param name the name of the value to set; if {@code name} is present multiple times, all its
+ * values are set.
+ * @param v the value to set. Note that {@code null} values inside collections are not supported
+ * by CQL.
+ * @param keysClass the class for the keys of the map.
+ * @param valuesClass the class for the values of the map.
+ * @return this object.
+ * @throws IllegalArgumentException if {@code name} is not a valid name for this object.
+ * @throws NullPointerException if {@code v} contains null values. Nulls are not supported in
+ * collections by CQL.
+ * @throws CodecNotFoundException if there is no registered codec to convert the value to the
+ * underlying CQL type.
+ */
+ public <K, V> T setMap(String name, Map<K, V> v, Class<K> keysClass, Class<V> valuesClass);
+
+ /**
+ * Sets the value for (all occurrences of) variable {@code name} to the provided map, which keys
+ * and values are of the provided Java types.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to handle the conversion of lists
+ * of the given Java types to the underlying CQL type.
+ *
+ * @param name the name of the value to set; if {@code name} is present multiple times, all its
+ * values are set.
+ * @param v the value to set. Note that {@code null} values inside collections are not supported
+ * by CQL.
+ * @param keysType the type for the keys of the map.
+ * @param valuesType the type for the values of the map.
+ * @return this object.
+ * @throws IllegalArgumentException if {@code name} is not a valid name for this object.
+ * @throws NullPointerException if {@code v} contains null values. Nulls are not supported in
+ * collections by CQL.
+ * @throws CodecNotFoundException if there is no registered codec to convert the value to the
+ * underlying CQL type.
+ */
+ public <K, V> T setMap(String name, Map<K, V> v, TypeToken<K> keysType, TypeToken<V> valuesType);
+
+ /**
+ * Sets the value for (all occurrences of) variable {@code name} to the provided set.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to handle the conversion to the
+ * underlying CQL type (the type of the elements in the Java set is not considered). If two or
+ * more codecs target that CQL type, the one that was first registered will be used. For this
+ * reason, it is generally preferable to use the more deterministic methods {@link #setSet(String,
+ * Set, Class)} or {@link #setSet(String, Set, TypeToken)}.
+ *
+ * @param name the name of the value to set; if {@code name} is present multiple times, all its
+ * values are set.
+ * @param v the value to set. Note that {@code null} values inside collections are not supported
+ * by CQL.
+ * @return this object.
+ * @throws IllegalArgumentException if {@code name} is not a valid name for this object.
+ * @throws NullPointerException if {@code v} contains null values. Nulls are not supported in
+ * collections by CQL.
+ * @throws CodecNotFoundException if there is no registered codec to convert the value to the
+ * underlying CQL type.
+ */
+ public <E> T setSet(String name, Set<E> v);
+
+ /**
+ * Sets the value for (all occurrences of) variable {@code name} to the provided set, which
+ * elements are of the provided Java class.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to handle the conversion of sets
+ * of the given Java type to the underlying CQL type.
+ *
+ * <p>If the type of the elements is generic, use {@link #setSet(String, Set, TypeToken)}.
+ *
+ * @param name the name of the value to set; if {@code name} is present multiple
+ * @param v the value to set. Note that {@code null} values inside collections are not supported
+ * by CQL.
+ * @param elementsClass the class for the elements of the set.
+ * @return this object.
+ * @throws IllegalArgumentException if {@code name} is not a valid name for this object.
+ * @throws NullPointerException if {@code v} contains null values. Nulls are not supported in
+ * collections by CQL.
+ * @throws CodecNotFoundException if there is no registered codec to convert the value to the
+ * underlying CQL type.
+ */
+ public <E> T setSet(String name, Set<E> v, Class<E> elementsClass);
+
+ /**
+ * Sets the value for (all occurrences of) variable {@code name} to the provided set, which
+ * elements are of the provided Java type.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to handle the conversion of sets
+ * of the given Java type to the underlying CQL type.
+ *
+ * @param name the name of the value to set; if {@code name} is present multiple
+ * @param v the value to set. Note that {@code null} values inside collections are not supported
+ * by CQL.
+ * @param elementsType the type for the elements of the set.
+ * @return this object.
+ * @throws IllegalArgumentException if {@code name} is not a valid name for this object.
+ * @throws NullPointerException if {@code v} contains null values. Nulls are not supported in
+ * collections by CQL.
+ * @throws CodecNotFoundException if there is no registered codec to convert the value to the
+ * underlying CQL type.
+ */
+ public <E> T setSet(String name, Set<E> v, TypeToken<E> elementsType);
+
+ /**
+ * Sets the value for (all occurrences of) variable {@code name} to the provided UDT value.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to handle the conversion of
+ * {@code UDTValue} to the underlying CQL type.
+ *
+ * @param name the name of the value to set; if {@code name} is present multiple times, all its
+ * values are set.
+ * @param v the value to set.
+ * @return this object.
+ * @throws IllegalArgumentException if {@code name} is not a valid name for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the value to the
+ * underlying CQL type.
+ */
+ public T setUDTValue(String name, UDTValue v);
+
+ /**
+ * Sets the value for (all occurrences of) variable {@code name} to the provided tuple value.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to handle the conversion of
+ * {@code TupleValue} to the underlying CQL type.
+ *
+ * @param name the name of the value to set; if {@code name} is present multiple times, all its
+ * values are set.
+ * @param v the value to set.
+ * @return this object.
+ * @throws IllegalArgumentException if {@code name} is not a valid name for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the value to the
+ * underlying CQL type.
+ */
+ public T setTupleValue(String name, TupleValue v);
+
+ /**
+ * Sets the value for (all occurrences of) variable {@code name} to {@code null}.
+ *
+ * <p>This is mainly intended for CQL types which map to native Java types.
+ *
+ * @param name the name of the value to set; if {@code name} is present multiple times, all its
+ * values are set.
+ * @return this object.
+ * @throws IllegalArgumentException if {@code name} is not a valid name for this object.
+ */
+ public T setToNull(String name);
+
+ /**
+ * Sets the value for (all occurrences of) variable {@code name} to the provided value of the
+ * provided Java class.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to handle the conversion of the
+ * provided Java class to the underlying CQL type.
+ *
+ * <p>If the Java type is generic, use {@link #set(String, Object, TypeToken)} instead.
+ *
+ * @param name the name of the value to set; if {@code name} is present multiple times, all its
+ * values are set.
+ * @param v the value to set; may be {@code null}.
+ * @param targetClass The Java class to convert to; must not be {@code null};
+ * @return this object.
+ * @throws IllegalArgumentException if {@code name} is not a valid name for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the value to the
+ * underlying CQL type.
+ */
+ <V> T set(String name, V v, Class<V> targetClass);
+
+ /**
+ * Sets the value for (all occurrences of) variable {@code name} to the provided value of the
+ * provided Java type.
+ *
+ * <p>This method uses the {@link CodecRegistry} to find a codec to handle the conversion of the
+ * provided Java type to the underlying CQL type.
+ *
+ * @param name the name of the value to set; if {@code name} is present multiple times, all its
+ * values are set.
+ * @param v the value to set; may be {@code null}.
+ * @param targetType The Java type to convert to; must not be {@code null};
+ * @return this object.
+ * @throws IllegalArgumentException if {@code name} is not a valid name for this object.
+ * @throws CodecNotFoundException if there is no registered codec to convert the value to the
+ * underlying CQL type.
+ */
+ <V> T set(String name, V v, TypeToken<V> targetType);
+
+ /**
+ * Sets the value for (all occurrences of) variable {@code name} to the provided value, converted
+ * using the given {@link TypeCodec}.
+ *
+ * <p>This method entirely bypasses the {@link CodecRegistry} and forces the driver to use the
+ * given codec instead. This can be useful if the codec would collide with a previously registered
+ * one, or if you want to use the codec just once without registering it.
+ *
+ * <p>It is the caller's responsibility to ensure that the given codec {@link
+ * TypeCodec#accepts(DataType) accepts} the underlying CQL type; failing to do so may result in
+ * {@link InvalidTypeException}s being thrown.
+ *
+ * @param name the name of the value to set; if {@code name} is present multiple times, all its
+ * values are set.
+ * @param v the value to set; may be {@code null}.
+ * @param codec The {@link TypeCodec} to use to serialize the value; may not be {@code null}.
+ * @return this object.
+ * @throws InvalidTypeException if the given codec does not {@link TypeCodec#accepts(DataType)
+ * accept} the underlying CQL type.
+ * @throws IllegalArgumentException if {@code name} is not a valid name for this object.
+ */
+ <V> T set(String name, V v, TypeCodec<V> codec);
+}
diff --git a/src/java/org/apache/cassandra/cql3/functions/types/SettableData.java b/src/java/org/apache/cassandra/cql3/functions/types/SettableData.java
new file mode 100644
index 0000000..a60e738
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/functions/types/SettableData.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.functions.types;
+
+/**
+ * Collection of (typed) CQL values that can be set either by index (starting at zero) or by name.
+ */
+public interface SettableData<T extends SettableData<T>>
+extends SettableByIndexData<T>, SettableByNameData<T>
+{
+}
diff --git a/src/java/org/apache/cassandra/cql3/functions/types/TupleType.java b/src/java/org/apache/cassandra/cql3/functions/types/TupleType.java
new file mode 100644
index 0000000..8b8f452
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/functions/types/TupleType.java
@@ -0,0 +1,201 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.functions.types;
+
+import java.util.Arrays;
+import java.util.List;
+
+import com.google.common.collect.ImmutableList;
+
+import org.apache.cassandra.transport.ProtocolVersion;
+import org.apache.cassandra.cql3.functions.types.exceptions.InvalidTypeException;
+
+/**
+ * A tuple type.
+ *
+ * <p>A tuple type is a essentially a list of types.
+ */
+public class TupleType extends DataType
+{
+
+ private final List<DataType> types;
+ private final ProtocolVersion protocolVersion;
+ private final CodecRegistry codecRegistry;
+
+ TupleType(List<DataType> types, ProtocolVersion protocolVersion, CodecRegistry codecRegistry)
+ {
+ super(DataType.Name.TUPLE);
+ this.types = ImmutableList.copyOf(types);
+ this.protocolVersion = protocolVersion;
+ this.codecRegistry = codecRegistry;
+ }
+
+ /**
+ * Creates a "disconnected" tuple type (<b>you should prefer {@code
+ * Metadata#newTupleType(DataType...) cluster.getMetadata().newTupleType(...)} whenever
+ * possible</b>).
+ *
+ * <p>This method is only exposed for situations where you don't have a {@code Cluster} instance
+ * available. If you create a type with this method and use it with a {@code Cluster} later, you
+ * won't be able to set tuple fields with custom codecs registered against the cluster, or you
+ * might get errors if the protocol versions don't match.
+ *
+ * @param protocolVersion the protocol version to use.
+ * @param codecRegistry the codec registry to use.
+ * @param types the types for the tuple type.
+ * @return the newly created tuple type.
+ */
+ public static TupleType of(
+ ProtocolVersion protocolVersion, CodecRegistry codecRegistry, DataType... types)
+ {
+ return new TupleType(Arrays.asList(types), protocolVersion, codecRegistry);
+ }
+
+ /**
+ * The (immutable) list of types composing this tuple type.
+ *
+ * @return the (immutable) list of types composing this tuple type.
+ */
+ List<DataType> getComponentTypes()
+ {
+ return types;
+ }
+
+ /**
+ * Returns a new empty value for this tuple type.
+ *
+ * @return an empty (with all component to {@code null}) value for this user type definition.
+ */
+ public TupleValue newValue()
+ {
+ return new TupleValue(this);
+ }
+
+ /**
+ * Returns a new value for this tuple type that uses the provided values for the components.
+ *
+ * <p>The numbers of values passed to this method must correspond to the number of components in
+ * this tuple type. The {@code i}th parameter value will then be assigned to the {@code i}th
+ * component of the resulting tuple value.
+ *
+ * @param values the values to use for the component of the resulting tuple.
+ * @return a new tuple values based on the provided values.
+ * @throws IllegalArgumentException if the number of {@code values} provided does not correspond
+ * to the number of components in this tuple type.
+ * @throws InvalidTypeException if any of the provided value is not of the correct type for the
+ * component.
+ */
+ public TupleValue newValue(Object... values)
+ {
+ if (values.length != types.size())
+ throw new IllegalArgumentException(
+ String.format(
+ "Invalid number of values. Expecting %d but got %d", types.size(), values.length));
+
+ TupleValue t = newValue();
+ for (int i = 0; i < values.length; i++)
+ {
+ DataType dataType = types.get(i);
+ if (values[i] == null) t.setValue(i, null);
+ else
+ t.setValue(
+ i, codecRegistry.codecFor(dataType, values[i]).serialize(values[i], protocolVersion));
+ }
+ return t;
+ }
+
+ @Override
+ public boolean isFrozen()
+ {
+ return true;
+ }
+
+ /**
+ * Return the protocol version that has been used to deserialize this tuple type, or that will be
+ * used to serialize it. In most cases this should be the version currently in use by the cluster
+ * instance that this tuple type belongs to, as reported by {@code
+ * ProtocolOptions#getProtocolVersion()}.
+ *
+ * @return the protocol version that has been used to deserialize this tuple type, or that will be
+ * used to serialize it.
+ */
+ ProtocolVersion getProtocolVersion()
+ {
+ return protocolVersion;
+ }
+
+ CodecRegistry getCodecRegistry()
+ {
+ return codecRegistry;
+ }
+
+ @Override
+ public int hashCode()
+ {
+ return Arrays.hashCode(new Object[]{ name, types });
+ }
+
+ @Override
+ public boolean equals(Object o)
+ {
+ if (!(o instanceof TupleType)) return false;
+
+ TupleType d = (TupleType) o;
+ return name == d.name && types.equals(d.types);
+ }
+
+ /**
+ * Return {@code true} if this tuple type contains the given tuple type, and {@code false}
+ * otherwise.
+ *
+ * <p>A tuple type is said to contain another one if the latter has fewer components than the
+ * former, but all of them are of the same type. E.g. the type {@code tuple<int, text>} is
+ * contained by the type {@code tuple<int, text, double>}.
+ *
+ * <p>A contained type can be seen as a "partial" view of a containing type, where the missing
+ * components are supposed to be {@code null}.
+ *
+ * @param other the tuple type to compare against the current one
+ * @return {@code true} if this tuple type contains the given tuple type, and {@code false}
+ * otherwise.
+ */
+ public boolean contains(TupleType other)
+ {
+ if (this.equals(other)) return true;
+ if (other.types.size() > this.types.size()) return false;
+ return types.subList(0, other.types.size()).equals(other.types);
+ }
+
+ @Override
+ public String toString()
+ {
+ return "frozen<" + asFunctionParameterString() + '>';
+ }
+
+ @Override
+ public String asFunctionParameterString()
+ {
+ StringBuilder sb = new StringBuilder();
+ for (DataType type : types)
+ {
+ sb.append(sb.length() == 0 ? "tuple<" : ", ");
+ sb.append(type.asFunctionParameterString());
+ }
+ return sb.append('>').toString();
+ }
+}
diff --git a/src/java/org/apache/cassandra/cql3/functions/types/TupleValue.java b/src/java/org/apache/cassandra/cql3/functions/types/TupleValue.java
new file mode 100644
index 0000000..68348e3
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/functions/types/TupleValue.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.functions.types;
+
+/**
+ * A value for a Tuple.
+ */
+public class TupleValue extends AbstractAddressableByIndexData<TupleValue>
+{
+
+ private final TupleType type;
+
+ /**
+ * Builds a new value for a tuple.
+ *
+ * @param type the {@link TupleType} instance defining this tuple's components.
+ */
+ TupleValue(TupleType type)
+ {
+ super(type.getProtocolVersion(), type.getComponentTypes().size());
+ this.type = type;
+ }
+
+ protected DataType getType(int i)
+ {
+ return type.getComponentTypes().get(i);
+ }
+
+ @Override
+ protected String getName(int i)
+ {
+ // This is used for error messages
+ return "component " + i;
+ }
+
+ @Override
+ protected CodecRegistry getCodecRegistry()
+ {
+ return type.getCodecRegistry();
+ }
+
+ /**
+ * The tuple type this is a value of.
+ *
+ * @return The tuple type this is a value of.
+ */
+ public TupleType getType()
+ {
+ return type;
+ }
+
+ @Override
+ public boolean equals(Object o)
+ {
+ if (!(o instanceof TupleValue)) return false;
+
+ TupleValue that = (TupleValue) o;
+ if (!type.equals(that.type)) return false;
+
+ return super.equals(o);
+ }
+
+ @Override
+ public int hashCode()
+ {
+ return super.hashCode();
+ }
+
+ @Override
+ public String toString()
+ {
+ StringBuilder sb = new StringBuilder();
+ TypeCodec<Object> codec = getCodecRegistry().codecFor(type);
+ sb.append(codec.format(this));
+ return sb.toString();
+ }
+}
diff --git a/src/java/org/apache/cassandra/cql3/functions/types/TypeCodec.java b/src/java/org/apache/cassandra/cql3/functions/types/TypeCodec.java
new file mode 100644
index 0000000..a728a1c
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/functions/types/TypeCodec.java
@@ -0,0 +1,3103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.functions.types;
+
+import java.io.DataInput;
+import java.io.IOException;
+import java.lang.reflect.Type;
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.nio.BufferUnderflowException;
+import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
+import java.text.ParseException;
+import java.util.*;
+import java.util.regex.Pattern;
+
+import com.google.common.io.ByteArrayDataOutput;
+import com.google.common.io.ByteStreams;
+import com.google.common.reflect.TypeToken;
+
+import org.apache.cassandra.transport.ProtocolVersion;
+import org.apache.cassandra.cql3.functions.types.DataType.CollectionType;
+import org.apache.cassandra.cql3.functions.types.DataType.Name;
+import org.apache.cassandra.cql3.functions.types.exceptions.InvalidTypeException;
+import org.apache.cassandra.cql3.functions.types.utils.Bytes;
+import org.apache.cassandra.utils.vint.VIntCoding;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkNotNull;
+import static org.apache.cassandra.cql3.functions.types.DataType.*;
+
+/**
+ * A Codec that can serialize and deserialize to and from a given {@link #getCqlType() CQL type} and
+ * a given {@link #getJavaType() Java Type}.
+ *
+ * <p>
+ *
+ * <h3>Serializing and deserializing</h3>
+ *
+ * <p>Two methods handle the serialization and deserialization of Java types into CQL types
+ * according to the native protocol specifications:
+ *
+ * <ol>
+ * <li>{@link #serialize(Object, ProtocolVersion)}: used to serialize from the codec's Java type
+ * to a {@link ByteBuffer} instance corresponding to the codec's CQL type;
+ * <li>{@link #deserialize(ByteBuffer, ProtocolVersion)}: used to deserialize a {@link ByteBuffer}
+ * instance corresponding to the codec's CQL type to the codec's Java type.
+ * </ol>
+ *
+ * <p>
+ *
+ * <h3>Formatting and parsing</h3>
+ *
+ * <p>Two methods handle the formatting and parsing of Java types into CQL strings:
+ *
+ * <ol>
+ * <li>{@link #format(Object)}: formats the Java type handled by the codec as a CQL string;
+ * <li>{@link #parse(String)}; parses a CQL string into the Java type handled by the codec.
+ * </ol>
+ *
+ * <p>
+ *
+ * <h3>Inspection</h3>
+ *
+ * <p>Codecs also have the following inspection methods:
+ *
+ * <p>
+ *
+ * <ol>
+ * <li>{@link #accepts(DataType)}: returns true if the codec can deserialize the given CQL type;
+ * <li>{@link #accepts(TypeToken)}: returns true if the codec can serialize the given Java type;
+ * <li>{@link #accepts(Object)}; returns true if the codec can serialize the given object.
+ * </ol>
+ *
+ * <p>
+ *
+ * <h3>Implementation notes</h3>
+ *
+ * <p>
+ *
+ * <ol>
+ * <li>TypeCodec implementations <em>must</em> be thread-safe.
+ * <li>TypeCodec implementations <em>must</em> perform fast and never block.
+ * <li>TypeCodec implementations <em>must</em> support all native protocol versions; it is not
+ * possible to use different codecs for the same types but under different protocol versions.
+ * <li>TypeCodec implementations must comply with the native protocol specifications; failing to
+ * do so will result in unexpected results and could cause the driver to crash.
+ * <li>TypeCodec implementations <em>should</em> be stateless and immutable.
+ * <li>TypeCodec implementations <em>should</em> interpret {@code null} values and empty
+ * ByteBuffers (i.e. <code>{@link ByteBuffer#remaining()} == 0</code>) in a
+ * <em>reasonable</em> way; usually, {@code NULL} CQL values should map to {@code null}
+ * references, but exceptions exist; e.g. for varchar types, a {@code NULL} CQL value maps to
+ * a {@code null} reference, whereas an empty buffer maps to an empty String. For collection
+ * types, it is also admitted that {@code NULL} CQL values map to empty Java collections
+ * instead of {@code null} references. In any case, the codec's behavior in respect to {@code
+ * null} values and empty ByteBuffers should be clearly documented.
+ * <li>TypeCodec implementations that wish to handle Java primitive types <em>must</em> be
+ * instantiated with the wrapper Java class instead, and implement the appropriate interface
+ * (e.g. {@link TypeCodec.PrimitiveBooleanCodec} for primitive {@code
+ * boolean} types; there is one such interface for each Java primitive type).
+ * <li>When deserializing, TypeCodec implementations should not consume {@link ByteBuffer}
+ * instances by performing relative read operations that modify their current position; codecs
+ * should instead prefer absolute read methods, or, if necessary, they should {@link
+ * ByteBuffer#duplicate() duplicate} their byte buffers prior to reading them.
+ * </ol>
+ *
+ * @param <T> The codec's Java type
+ */
+public abstract class TypeCodec<T>
+{
+
+ /**
+ * Return the default codec for the CQL type {@code boolean}. The returned codec maps the CQL type
+ * {@code boolean} into the Java type {@link Boolean}. The returned instance is a singleton.
+ *
+ * @return the default codec for CQL type {@code boolean}.
+ */
+ public static PrimitiveBooleanCodec cboolean()
+ {
+ return BooleanCodec.instance;
+ }
+
+ /**
+ * Return the default codec for the CQL type {@code tinyint}. The returned codec maps the CQL type
+ * {@code tinyint} into the Java type {@link Byte}. The returned instance is a singleton.
+ *
+ * @return the default codec for CQL type {@code tinyint}.
+ */
+ public static PrimitiveByteCodec tinyInt()
+ {
+ return TinyIntCodec.instance;
+ }
+
+ /**
+ * Return the default codec for the CQL type {@code smallint}. The returned codec maps the CQL
+ * type {@code smallint} into the Java type {@link Short}. The returned instance is a singleton.
+ *
+ * @return the default codec for CQL type {@code smallint}.
+ */
+ public static PrimitiveShortCodec smallInt()
+ {
+ return SmallIntCodec.instance;
+ }
+
+ /**
+ * Return the default codec for the CQL type {@code int}. The returned codec maps the CQL type
+ * {@code int} into the Java type {@link Integer}. The returned instance is a singleton.
+ *
+ * @return the default codec for CQL type {@code int}.
+ */
+ public static PrimitiveIntCodec cint()
+ {
+ return IntCodec.instance;
+ }
+
+ /**
+ * Return the default codec for the CQL type {@code bigint}. The returned codec maps the CQL type
+ * {@code bigint} into the Java type {@link Long}. The returned instance is a singleton.
+ *
+ * @return the default codec for CQL type {@code bigint}.
+ */
+ public static PrimitiveLongCodec bigint()
+ {
+ return BigintCodec.instance;
+ }
+
+ /**
+ * Return the default codec for the CQL type {@code counter}. The returned codec maps the CQL type
+ * {@code counter} into the Java type {@link Long}. The returned instance is a singleton.
+ *
+ * @return the default codec for CQL type {@code counter}.
+ */
+ public static PrimitiveLongCodec counter()
+ {
+ return CounterCodec.instance;
+ }
+
+ /**
+ * Return the default codec for the CQL type {@code float}. The returned codec maps the CQL type
+ * {@code float} into the Java type {@link Float}. The returned instance is a singleton.
+ *
+ * @return the default codec for CQL type {@code float}.
+ */
+ public static PrimitiveFloatCodec cfloat()
+ {
+ return FloatCodec.instance;
+ }
+
+ /**
+ * Return the default codec for the CQL type {@code double}. The returned codec maps the CQL type
+ * {@code double} into the Java type {@link Double}. The returned instance is a singleton.
+ *
+ * @return the default codec for CQL type {@code double}.
+ */
+ public static PrimitiveDoubleCodec cdouble()
+ {
+ return DoubleCodec.instance;
+ }
+
+ /**
+ * Return the default codec for the CQL type {@code varint}. The returned codec maps the CQL type
+ * {@code varint} into the Java type {@link BigInteger}. The returned instance is a singleton.
+ *
+ * @return the default codec for CQL type {@code varint}.
+ */
+ public static TypeCodec<BigInteger> varint()
+ {
+ return VarintCodec.instance;
+ }
+
+ /**
+ * Return the default codec for the CQL type {@code decimal}. The returned codec maps the CQL type
+ * {@code decimal} into the Java type {@link BigDecimal}. The returned instance is a singleton.
+ *
+ * @return the default codec for CQL type {@code decimal}.
+ */
+ public static TypeCodec<BigDecimal> decimal()
+ {
+ return DecimalCodec.instance;
+ }
+
+ /**
+ * Return the default codec for the CQL type {@code ascii}. The returned codec maps the CQL type
+ * {@code ascii} into the Java type {@link String}. The returned instance is a singleton.
+ *
+ * @return the default codec for CQL type {@code ascii}.
+ */
+ public static TypeCodec<String> ascii()
+ {
+ return AsciiCodec.instance;
+ }
+
+ /**
+ * Return the default codec for the CQL type {@code varchar}. The returned codec maps the CQL type
+ * {@code varchar} into the Java type {@link String}. The returned instance is a singleton.
+ *
+ * @return the default codec for CQL type {@code varchar}.
+ */
+ public static TypeCodec<String> varchar()
+ {
+ return VarcharCodec.instance;
+ }
+
+ /**
+ * Return the default codec for the CQL type {@code blob}. The returned codec maps the CQL type
+ * {@code blob} into the Java type {@link ByteBuffer}. The returned instance is a singleton.
+ *
+ * @return the default codec for CQL type {@code blob}.
+ */
+ public static TypeCodec<ByteBuffer> blob()
+ {
+ return BlobCodec.instance;
+ }
+
+ /**
+ * Return the default codec for the CQL type {@code date}. The returned codec maps the CQL type
+ * {@code date} into the Java type {@link LocalDate}. The returned instance is a singleton.
+ *
+ * @return the default codec for CQL type {@code date}.
+ */
+ public static TypeCodec<LocalDate> date()
+ {
+ return DateCodec.instance;
+ }
+
+ /**
+ * Return the default codec for the CQL type {@code time}. The returned codec maps the CQL type
+ * {@code time} into the Java type {@link Long}. The returned instance is a singleton.
+ *
+ * @return the default codec for CQL type {@code time}.
+ */
+ public static PrimitiveLongCodec time()
+ {
+ return TimeCodec.instance;
+ }
+
+ /**
+ * Return the default codec for the CQL type {@code timestamp}. The returned codec maps the CQL
+ * type {@code timestamp} into the Java type {@link Date}. The returned instance is a singleton.
+ *
+ * @return the default codec for CQL type {@code timestamp}.
+ */
+ public static TypeCodec<Date> timestamp()
+ {
+ return TimestampCodec.instance;
+ }
+
+ /**
+ * Return the default codec for the CQL type {@code uuid}. The returned codec maps the CQL type
+ * {@code uuid} into the Java type {@link UUID}. The returned instance is a singleton.
+ *
+ * @return the default codec for CQL type {@code uuid}.
+ */
+ public static TypeCodec<UUID> uuid()
+ {
+ return UUIDCodec.instance;
+ }
+
+ /**
+ * Return the default codec for the CQL type {@code timeuuid}. The returned codec maps the CQL
+ * type {@code timeuuid} into the Java type {@link UUID}. The returned instance is a singleton.
+ *
+ * @return the default codec for CQL type {@code timeuuid}.
+ */
+ public static TypeCodec<UUID> timeUUID()
+ {
+ return TimeUUIDCodec.instance;
+ }
+
+ /**
+ * Return the default codec for the CQL type {@code inet}. The returned codec maps the CQL type
+ * {@code inet} into the Java type {@link InetAddress}. The returned instance is a singleton.
+ *
+ * @return the default codec for CQL type {@code inet}.
+ */
+ public static TypeCodec<InetAddress> inet()
+ {
+ return InetCodec.instance;
+ }
+
+ /**
+ * Return a newly-created codec for the CQL type {@code list} whose element type is determined by
+ * the given element codec. The returned codec maps the CQL type {@code list} into the Java type
+ * {@link List}. This method does not cache returned instances and returns a newly-allocated
+ * object at each invocation.
+ *
+ * @param elementCodec the codec that will handle elements of this list.
+ * @return A newly-created codec for CQL type {@code list}.
+ */
+ public static <T> TypeCodec<List<T>> list(TypeCodec<T> elementCodec)
+ {
+ return new ListCodec<>(elementCodec);
+ }
+
+ /**
+ * Return a newly-created codec for the CQL type {@code set} whose element type is determined by
+ * the given element codec. The returned codec maps the CQL type {@code set} into the Java type
+ * {@link Set}. This method does not cache returned instances and returns a newly-allocated object
+ * at each invocation.
+ *
+ * @param elementCodec the codec that will handle elements of this set.
+ * @return A newly-created codec for CQL type {@code set}.
+ */
+ public static <T> TypeCodec<Set<T>> set(TypeCodec<T> elementCodec)
+ {
+ return new SetCodec<>(elementCodec);
+ }
+
+ /**
+ * Return a newly-created codec for the CQL type {@code map} whose key type and value type are
+ * determined by the given codecs. The returned codec maps the CQL type {@code map} into the Java
+ * type {@link Map}. This method does not cache returned instances and returns a newly-allocated
+ * object at each invocation.
+ *
+ * @param keyCodec the codec that will handle keys of this map.
+ * @param valueCodec the codec that will handle values of this map.
+ * @return A newly-created codec for CQL type {@code map}.
+ */
+ public static <K, V> TypeCodec<Map<K, V>> map(TypeCodec<K> keyCodec, TypeCodec<V> valueCodec)
+ {
+ return new MapCodec<>(keyCodec, valueCodec);
+ }
+
+ /**
+ * Return a newly-created codec for the given user-defined CQL type. The returned codec maps the
+ * user-defined type into the Java type {@link UDTValue}. This method does not cache returned
+ * instances and returns a newly-allocated object at each invocation.
+ *
+ * @param type the user-defined type this codec should handle.
+ * @return A newly-created codec for the given user-defined CQL type.
+ */
+ public static TypeCodec<UDTValue> userType(UserType type)
+ {
+ return new UDTCodec(type);
+ }
+
+ /**
+ * Return a newly-created codec for the given CQL tuple type. The returned codec maps the tuple
+ * type into the Java type {@link TupleValue}. This method does not cache returned instances and
+ * returns a newly-allocated object at each invocation.
+ *
+ * @param type the tuple type this codec should handle.
+ * @return A newly-created codec for the given CQL tuple type.
+ */
+ public static TypeCodec<TupleValue> tuple(TupleType type)
+ {
+ return new TupleCodec(type);
+ }
+
+ /**
+ * Return a newly-created codec for the given CQL custom type.
+ *
+ * <p>The returned codec maps the custom type into the Java type {@link ByteBuffer}, thus
+ * providing a (very lightweight) support for Cassandra types that do not have a CQL equivalent.
+ *
+ * <p>Note that the returned codec assumes that CQL literals for the given custom type are
+ * expressed in binary form as well, e.g. {@code 0xcafebabe}. If this is not the case, <em>the
+ * returned codec might be unable to {@link #parse(String) parse} and {@link #format(Object)
+ * format} literals for this type</em>. This is notoriously true for types inheriting from {@code
+ * org.apache.cassandra.db.marshal.AbstractCompositeType}, whose CQL literals are actually
+ * expressed as quoted strings.
+ *
+ * <p>This method does not cache returned instances and returns a newly-allocated object at each
+ * invocation.
+ *
+ * @param type the custom type this codec should handle.
+ * @return A newly-created codec for the given CQL custom type.
+ */
+ public static TypeCodec<ByteBuffer> custom(DataType.CustomType type)
+ {
+ return new CustomCodec(type);
+ }
+
+ /**
+ * Returns the default codec for the {@link DataType#duration() Duration type}.
+ *
+ * <p>This codec maps duration types to the driver's built-in {@link Duration} class, thus
+ * providing a more user-friendly mapping than the low-level mapping provided by regular {@link
+ * #custom(DataType.CustomType) custom type codecs}.
+ *
+ * <p>The returned instance is a singleton.
+ *
+ * @return the default codec for the Duration type.
+ */
+ public static TypeCodec<Duration> duration()
+ {
+ return DurationCodec.instance;
+ }
+
+ private final TypeToken<T> javaType;
+
+ final DataType cqlType;
+
+ /**
+ * This constructor can only be used for non parameterized types. For parameterized ones, please
+ * use {@link #TypeCodec(DataType, TypeToken)} instead.
+ *
+ * @param javaClass The Java class this codec serializes from and deserializes to.
+ */
+ protected TypeCodec(DataType cqlType, Class<T> javaClass)
+ {
+ this(cqlType, TypeToken.of(javaClass));
+ }
+
+ protected TypeCodec(DataType cqlType, TypeToken<T> javaType)
+ {
+ checkNotNull(cqlType, "cqlType cannot be null");
+ checkNotNull(javaType, "javaType cannot be null");
+ checkArgument(
+ !javaType.isPrimitive(),
+ "Cannot create a codec for a primitive Java type (%s), please use the wrapper type instead",
+ javaType);
+ this.cqlType = cqlType;
+ this.javaType = javaType;
+ }
+
+ /**
+ * Return the Java type that this codec deserializes to and serializes from.
+ *
+ * @return The Java type this codec deserializes to and serializes from.
+ */
+ public TypeToken<T> getJavaType()
+ {
+ return javaType;
+ }
+
+ /**
+ * Return the CQL type that this codec deserializes from and serializes to.
+ *
+ * @return The Java type this codec deserializes from and serializes to.
+ */
+ public DataType getCqlType()
+ {
+ return cqlType;
+ }
+
+ /**
+ * Serialize the given value according to the CQL type handled by this codec.
+ *
+ * <p>Implementation notes:
+ *
+ * <ol>
+ * <li>Null values should be gracefully handled and no exception should be raised; these should
+ * be considered as the equivalent of a NULL CQL value;
+ * <li>Codecs for CQL collection types should not permit null elements;
+ * <li>Codecs for CQL collection types should treat a {@code null} input as the equivalent of an
+ * empty collection.
+ * </ol>
+ *
+ * @param value An instance of T; may be {@code null}.
+ * @param protocolVersion the protocol version to use when serializing {@code bytes}. In most
+ * cases, the proper value to provide for this argument is the value returned by {@code
+ * ProtocolOptions#getProtocolVersion} (which is the protocol version in use by the driver).
+ * @return A {@link ByteBuffer} instance containing the serialized form of T
+ * @throws InvalidTypeException if the given value does not have the expected type
+ */
+ public abstract ByteBuffer serialize(T value, ProtocolVersion protocolVersion)
+ throws InvalidTypeException;
+
+ /**
+ * Deserialize the given {@link ByteBuffer} instance according to the CQL type handled by this
+ * codec.
+ *
+ * <p>Implementation notes:
+ *
+ * <ol>
+ * <li>Null or empty buffers should be gracefully handled and no exception should be raised;
+ * these should be considered as the equivalent of a NULL CQL value and, in most cases,
+ * should map to {@code null} or a default value for the corresponding Java type, if
+ * applicable;
+ * <li>Codecs for CQL collection types should clearly document whether they return immutable
+ * collections or not (note that the driver's default collection codecs return
+ * <em>mutable</em> collections);
+ * <li>Codecs for CQL collection types should avoid returning {@code null}; they should return
+ * empty collections instead (the driver's default collection codecs all comply with this
+ * rule).
+ * <li>The provided {@link ByteBuffer} should never be consumed by read operations that modify
+ * its current position; if necessary, {@link ByteBuffer#duplicate()} duplicate} it before
+ * consuming.
+ * </ol>
+ *
+ * @param bytes A {@link ByteBuffer} instance containing the serialized form of T; may be {@code
+ * null} or empty.
+ * @param protocolVersion the protocol version to use when serializing {@code bytes}. In most
+ * cases, the proper value to provide for this argument is the value returned by {@code
+ * ProtocolOptions#getProtocolVersion} (which is the protocol version in use by the driver).
+ * @return An instance of T
+ * @throws InvalidTypeException if the given {@link ByteBuffer} instance cannot be deserialized
+ */
+ public abstract T deserialize(ByteBuffer bytes, ProtocolVersion protocolVersion)
+ throws InvalidTypeException;
+
+ /**
+ * Parse the given CQL literal into an instance of the Java type handled by this codec.
+ *
+ * <p>Implementors should take care of unquoting and unescaping the given CQL string where
+ * applicable. Null values and empty Strings should be accepted, as well as the string {@code
+ * "NULL"}; in most cases, implementations should interpret these inputs has equivalent to a
+ * {@code null} reference.
+ *
+ * <p>Implementing this method is not strictly mandatory: internally, the driver only uses it to
+ * parse the INITCOND when building the metadata of an aggregate function (and in most cases it
+ * will use a built-in codec, unless the INITCOND has a custom type).
+ *
+ * @param value The CQL string to parse, may be {@code null} or empty.
+ * @return An instance of T; may be {@code null} on a {@code null input}.
+ * @throws InvalidTypeException if the given value cannot be parsed into the expected type
+ */
+ public abstract T parse(String value) throws InvalidTypeException;
+
+ /**
+ * Format the given value as a valid CQL literal according to the CQL type handled by this codec.
+ *
+ * <p>Implementors should take care of quoting and escaping the resulting CQL literal where
+ * applicable. Null values should be accepted; in most cases, implementations should return the
+ * CQL keyword {@code "NULL"} for {@code null} inputs.
+ *
+ * <p>Implementing this method is not strictly mandatory. It is used:
+ *
+ * <ol>
+ * <li>in the query builder, when values are inlined in the query string (see {@code
+ * querybuilder.BuiltStatement} for a detailed explanation of when
+ * this happens);
+ * <li>in the {@code QueryLogger}, if parameter logging is enabled;
+ * <li>to format the INITCOND in {@code AggregateMetadata#asCQLQuery(boolean)};
+ * <li>in the {@code toString()} implementation of some objects ({@link UDTValue}, {@link
+ * TupleValue}, and the internal representation of a {@code ROWS} response), which may
+ * appear in driver logs.
+ * </ol>
+ * <p>
+ * If you choose not to implement this method, you should not throw an exception but instead
+ * return a constant string (for example "XxxCodec.format not implemented").
+ *
+ * @param value An instance of T; may be {@code null}.
+ * @return CQL string
+ * @throws InvalidTypeException if the given value does not have the expected type
+ */
+ public abstract String format(T value) throws InvalidTypeException;
+
+ /**
+ * Return {@code true} if this codec is capable of serializing the given {@code javaType}.
+ *
+ * <p>The implementation is <em>invariant</em> with respect to the passed argument (through the
+ * usage of {@link TypeToken#equals(Object)} and <em>it's strongly recommended not to modify this
+ * behavior</em>. This means that a codec will only ever return {@code true} for the
+ * <em>exact</em> Java type that it has been created for.
+ *
+ * <p>If the argument represents a Java primitive type, its wrapper type is considered instead.
+ *
+ * @param javaType The Java type this codec should serialize from and deserialize to; cannot be
+ * {@code null}.
+ * @return {@code true} if the codec is capable of serializing the given {@code javaType}, and
+ * {@code false} otherwise.
+ * @throws NullPointerException if {@code javaType} is {@code null}.
+ */
+ public boolean accepts(TypeToken<?> javaType)
+ {
+ checkNotNull(javaType, "Parameter javaType cannot be null");
+ return this.javaType.equals(javaType.wrap());
+ }
+
+ /**
+ * Return {@code true} if this codec is capable of serializing the given {@code javaType}.
+ *
+ * <p>This implementation simply calls {@link #accepts(TypeToken)}.
+ *
+ * @param javaType The Java type this codec should serialize from and deserialize to; cannot be
+ * {@code null}.
+ * @return {@code true} if the codec is capable of serializing the given {@code javaType}, and
+ * {@code false} otherwise.
+ * @throws NullPointerException if {@code javaType} is {@code null}.
+ */
+ public boolean accepts(Class<?> javaType)
+ {
+ checkNotNull(javaType, "Parameter javaType cannot be null");
+ return accepts(TypeToken.of(javaType));
+ }
+
+ /**
+ * Return {@code true} if this codec is capable of deserializing the given {@code cqlType}.
+ *
+ * @param cqlType The CQL type this codec should deserialize from and serialize to; cannot be
+ * {@code null}.
+ * @return {@code true} if the codec is capable of deserializing the given {@code cqlType}, and
+ * {@code false} otherwise.
+ * @throws NullPointerException if {@code cqlType} is {@code null}.
+ */
+ public boolean accepts(DataType cqlType)
+ {
+ checkNotNull(cqlType, "Parameter cqlType cannot be null");
+ return this.cqlType.equals(cqlType);
+ }
+
+ /**
+ * Return {@code true} if this codec is capable of serializing the given object. Note that the
+ * object's Java type is inferred from the object' runtime (raw) type, contrary to {@link
+ * #accepts(TypeToken)} which is capable of handling generic types.
+ *
+ * <p>This method is intended mostly to be used by the QueryBuilder when no type information is
+ * available when the codec is used.
+ *
+ * <p>Implementation notes:
+ *
+ * <ol>
+ * <li>The default implementation is <em>covariant</em> with respect to the passed argument
+ * (through the usage of {@code TypeToken#isAssignableFrom(TypeToken)} or {@link
+ * TypeToken#isSupertypeOf(Type)}) and <em>it's strongly recommended not to modify this
+ * behavior</em>. This means that, by default, a codec will accept <em>any subtype</em> of
+ * the Java type that it has been created for.
+ * <li>The base implementation provided here can only handle non-parameterized types; codecs
+ * handling parameterized types, such as collection types, must override this method and
+ * perform some sort of "manual" inspection of the actual type parameters.
+ * <li>Similarly, codecs that only accept a partial subset of all possible values must override
+ * this method and manually inspect the object to check if it complies or not with the
+ * codec's limitations.
+ * </ol>
+ *
+ * @param value The Java type this codec should serialize from and deserialize to; cannot be
+ * {@code null}.
+ * @return {@code true} if the codec is capable of serializing the given {@code javaType}, and
+ * {@code false} otherwise.
+ * @throws NullPointerException if {@code value} is {@code null}.
+ */
+ public boolean accepts(Object value)
+ {
+ checkNotNull(value, "Parameter value cannot be null");
+ return this.javaType.isSupertypeOf(TypeToken.of(value.getClass()));
+ }
+
+ @Override
+ public String toString()
+ {
+ return String.format("%s [%s <-> %s]", this.getClass().getSimpleName(), cqlType, javaType);
+ }
+
+ /**
+ * A codec that is capable of handling primitive booleans, thus avoiding the overhead of boxing
+ * and unboxing such primitives.
+ */
+ public abstract static class PrimitiveBooleanCodec extends TypeCodec<Boolean>
+ {
+
+ PrimitiveBooleanCodec(DataType cqlType)
+ {
+ super(cqlType, Boolean.class);
+ }
+
+ public abstract ByteBuffer serializeNoBoxing(boolean v, ProtocolVersion protocolVersion);
+
+ public abstract boolean deserializeNoBoxing(ByteBuffer v, ProtocolVersion protocolVersion);
+
+ @Override
+ public ByteBuffer serialize(Boolean value, ProtocolVersion protocolVersion)
+ {
+ return value == null ? null : serializeNoBoxing(value, protocolVersion);
+ }
+
+ @Override
+ public Boolean deserialize(ByteBuffer bytes, ProtocolVersion protocolVersion)
+ {
+ return bytes == null || bytes.remaining() == 0
+ ? null
+ : deserializeNoBoxing(bytes, protocolVersion);
+ }
+ }
+
+ /**
+ * A codec that is capable of handling primitive bytes, thus avoiding the overhead of boxing and
+ * unboxing such primitives.
+ */
+ public abstract static class PrimitiveByteCodec extends TypeCodec<Byte>
+ {
+
+ PrimitiveByteCodec(DataType cqlType)
+ {
+ super(cqlType, Byte.class);
+ }
+
+ public abstract ByteBuffer serializeNoBoxing(byte v, ProtocolVersion protocolVersion);
+
+ public abstract byte deserializeNoBoxing(ByteBuffer v, ProtocolVersion protocolVersion);
+
+ @Override
+ public ByteBuffer serialize(Byte value, ProtocolVersion protocolVersion)
+ {
+ return value == null ? null : serializeNoBoxing(value, protocolVersion);
+ }
+
+ @Override
+ public Byte deserialize(ByteBuffer bytes, ProtocolVersion protocolVersion)
+ {
+ return bytes == null || bytes.remaining() == 0
+ ? null
+ : deserializeNoBoxing(bytes, protocolVersion);
+ }
+ }
+
+ /**
+ * A codec that is capable of handling primitive shorts, thus avoiding the overhead of boxing and
+ * unboxing such primitives.
+ */
+ public abstract static class PrimitiveShortCodec extends TypeCodec<Short>
+ {
+
+ PrimitiveShortCodec(DataType cqlType)
+ {
+ super(cqlType, Short.class);
+ }
+
+ public abstract ByteBuffer serializeNoBoxing(short v, ProtocolVersion protocolVersion);
+
+ public abstract short deserializeNoBoxing(ByteBuffer v, ProtocolVersion protocolVersion);
+
+ @Override
+ public ByteBuffer serialize(Short value, ProtocolVersion protocolVersion)
+ {
+ return value == null ? null : serializeNoBoxing(value, protocolVersion);
+ }
+
+ @Override
+ public Short deserialize(ByteBuffer bytes, ProtocolVersion protocolVersion)
+ {
+ return bytes == null || bytes.remaining() == 0
+ ? null
+ : deserializeNoBoxing(bytes, protocolVersion);
+ }
+ }
+
+ /**
+ * A codec that is capable of handling primitive ints, thus avoiding the overhead of boxing and
+ * unboxing such primitives.
+ */
+ public abstract static class PrimitiveIntCodec extends TypeCodec<Integer>
+ {
+
+ PrimitiveIntCodec(DataType cqlType)
+ {
+ super(cqlType, Integer.class);
+ }
+
+ public abstract ByteBuffer serializeNoBoxing(int v, ProtocolVersion protocolVersion);
+
+ public abstract int deserializeNoBoxing(ByteBuffer v, ProtocolVersion protocolVersion);
+
+ @Override
+ public ByteBuffer serialize(Integer value, ProtocolVersion protocolVersion)
+ {
+ return value == null ? null : serializeNoBoxing(value, protocolVersion);
+ }
+
+ @Override
+ public Integer deserialize(ByteBuffer bytes, ProtocolVersion protocolVersion)
+ {
+ return bytes == null || bytes.remaining() == 0
+ ? null
+ : deserializeNoBoxing(bytes, protocolVersion);
+ }
+ }
+
+ /**
+ * A codec that is capable of handling primitive longs, thus avoiding the overhead of boxing and
+ * unboxing such primitives.
+ */
+ public abstract static class PrimitiveLongCodec extends TypeCodec<Long>
+ {
+
+ PrimitiveLongCodec(DataType cqlType)
+ {
+ super(cqlType, Long.class);
+ }
+
+ public abstract ByteBuffer serializeNoBoxing(long v, ProtocolVersion protocolVersion);
+
+ public abstract long deserializeNoBoxing(ByteBuffer v, ProtocolVersion protocolVersion);
+
+ @Override
+ public ByteBuffer serialize(Long value, ProtocolVersion protocolVersion)
+ {
+ return value == null ? null : serializeNoBoxing(value, protocolVersion);
+ }
+
+ @Override
+ public Long deserialize(ByteBuffer bytes, ProtocolVersion protocolVersion)
+ {
+ return bytes == null || bytes.remaining() == 0
+ ? null
+ : deserializeNoBoxing(bytes, protocolVersion);
+ }
+ }
+
+ /**
+ * A codec that is capable of handling primitive floats, thus avoiding the overhead of boxing and
+ * unboxing such primitives.
+ */
+ public abstract static class PrimitiveFloatCodec extends TypeCodec<Float>
+ {
+
+ PrimitiveFloatCodec(DataType cqlType)
+ {
+ super(cqlType, Float.class);
+ }
+
+ public abstract ByteBuffer serializeNoBoxing(float v, ProtocolVersion protocolVersion);
+
+ public abstract float deserializeNoBoxing(ByteBuffer v, ProtocolVersion protocolVersion);
+
+ @Override
+ public ByteBuffer serialize(Float value, ProtocolVersion protocolVersion)
+ {
+ return value == null ? null : serializeNoBoxing(value, protocolVersion);
+ }
+
+ @Override
+ public Float deserialize(ByteBuffer bytes, ProtocolVersion protocolVersion)
+ {
+ return bytes == null || bytes.remaining() == 0
+ ? null
+ : deserializeNoBoxing(bytes, protocolVersion);
+ }
+ }
+
+ /**
+ * A codec that is capable of handling primitive doubles, thus avoiding the overhead of boxing and
+ * unboxing such primitives.
+ */
+ public abstract static class PrimitiveDoubleCodec extends TypeCodec<Double>
+ {
+
+ PrimitiveDoubleCodec(DataType cqlType)
+ {
+ super(cqlType, Double.class);
+ }
+
+ public abstract ByteBuffer serializeNoBoxing(double v, ProtocolVersion protocolVersion);
+
+ public abstract double deserializeNoBoxing(ByteBuffer v, ProtocolVersion protocolVersion);
+
+ @Override
+ public ByteBuffer serialize(Double value, ProtocolVersion protocolVersion)
+ {
+ return value == null ? null : serializeNoBoxing(value, protocolVersion);
+ }
+
+ @Override
+ public Double deserialize(ByteBuffer bytes, ProtocolVersion protocolVersion)
+ {
+ return bytes == null || bytes.remaining() == 0
+ ? null
+ : deserializeNoBoxing(bytes, protocolVersion);
+ }
+ }
+
+ /**
+ * Base class for codecs handling CQL string types such as {@link DataType#varchar()}, {@link
+ * DataType#text()} or {@link DataType#ascii()}.
+ */
+ private abstract static class StringCodec extends TypeCodec<String>
+ {
+
+ private final Charset charset;
+
+ private StringCodec(DataType cqlType, Charset charset)
+ {
+ super(cqlType, String.class);
+ this.charset = charset;
+ }
+
+ @Override
+ public String parse(String value)
+ {
+ if (value == null || value.isEmpty() || value.equalsIgnoreCase("NULL")) return null;
+ if (!ParseUtils.isQuoted(value))
+ throw new InvalidTypeException("text or varchar values must be enclosed by single quotes");
+
+ return ParseUtils.unquote(value);
+ }
+
+ @Override
+ public String format(String value)
+ {
+ if (value == null) return "NULL";
+ return ParseUtils.quote(value);
+ }
+
+ @Override
+ public ByteBuffer serialize(String value, ProtocolVersion protocolVersion)
+ {
+ return value == null ? null : ByteBuffer.wrap(value.getBytes(charset));
+ }
+
+ /**
+ * {@inheritDoc}
+ *
+ * <p>Implementation note: this method treats {@code null}s and empty buffers differently: the
+ * formers are mapped to {@code null}s while the latters are mapped to empty strings.
+ */
+ @Override
+ public String deserialize(ByteBuffer bytes, ProtocolVersion protocolVersion)
+ {
+ if (bytes == null) return null;
+ if (bytes.remaining() == 0) return "";
+ return new String(Bytes.getArray(bytes), charset);
+ }
+ }
+
+ /**
+ * This codec maps a CQL {@link DataType#varchar()} to a Java {@link String}. Note that this codec
+ * also handles {@link DataType#text()}, which is merely an alias for {@link DataType#varchar()}.
+ */
+ private static class VarcharCodec extends StringCodec
+ {
+
+ private static final VarcharCodec instance = new VarcharCodec();
+
+ private VarcharCodec()
+ {
+ super(DataType.varchar(), Charset.forName("UTF-8"));
+ }
+ }
+
+ /**
+ * This codec maps a CQL {@link DataType#ascii()} to a Java {@link String}.
+ */
+ private static class AsciiCodec extends StringCodec
+ {
+
+ private static final AsciiCodec instance = new AsciiCodec();
+
+ private static final Pattern ASCII_PATTERN = Pattern.compile("^\\p{ASCII}*$");
+
+ private AsciiCodec()
+ {
+ super(DataType.ascii(), Charset.forName("US-ASCII"));
+ }
+
+ @Override
+ public ByteBuffer serialize(String value, ProtocolVersion protocolVersion)
+ {
+ if (value != null && !ASCII_PATTERN.matcher(value).matches())
+ {
+ throw new InvalidTypeException(String.format("%s is not a valid ASCII String", value));
+ }
+ return super.serialize(value, protocolVersion);
+ }
+
+ @Override
+ public String format(String value)
+ {
+ if (value != null && !ASCII_PATTERN.matcher(value).matches())
+ {
+ throw new InvalidTypeException(String.format("%s is not a valid ASCII String", value));
+ }
+ return super.format(value);
+ }
+ }
+
+ /**
+ * Base class for codecs handling CQL 8-byte integer types such as {@link DataType#bigint()},
+ * {@link DataType#counter()} or {@link DataType#time()}.
+ */
+ private abstract static class LongCodec extends PrimitiveLongCodec
+ {
+
+ private LongCodec(DataType cqlType)
+ {
+ super(cqlType);
+ }
+
+ @Override
+ public Long parse(String value)
+ {
+ try
+ {
+ return value == null || value.isEmpty() || value.equalsIgnoreCase("NULL")
+ ? null
+ : Long.parseLong(value);
+ }
+ catch (NumberFormatException e)
+ {
+ throw new InvalidTypeException(
+ String.format("Cannot parse 64-bits long value from \"%s\"", value));
+ }
+ }
+
+ @Override
+ public String format(Long value)
+ {
+ if (value == null) return "NULL";
+ return Long.toString(value);
+ }
+
+ @Override
+ public ByteBuffer serializeNoBoxing(long value, ProtocolVersion protocolVersion)
+ {
+ ByteBuffer bb = ByteBuffer.allocate(8);
+ bb.putLong(0, value);
+ return bb;
+ }
+
+ @Override
+ public long deserializeNoBoxing(ByteBuffer bytes, ProtocolVersion protocolVersion)
+ {
+ if (bytes == null || bytes.remaining() == 0) return 0;
+ if (bytes.remaining() != 8)
+ throw new InvalidTypeException(
+ "Invalid 64-bits long value, expecting 8 bytes but got " + bytes.remaining());
+
+ return bytes.getLong(bytes.position());
+ }
+ }
+
+ /**
+ * This codec maps a CQL {@link DataType#bigint()} to a Java {@link Long}.
+ */
+ private static class BigintCodec extends LongCodec
+ {
+
+ private static final BigintCodec instance = new BigintCodec();
+
+ private BigintCodec()
+ {
+ super(DataType.bigint());
+ }
+ }
+
+ /**
+ * This codec maps a CQL {@link DataType#counter()} to a Java {@link Long}.
+ */
+ private static class CounterCodec extends LongCodec
+ {
+
+ private static final CounterCodec instance = new CounterCodec();
+
+ private CounterCodec()
+ {
+ super(DataType.counter());
+ }
+ }
+
+ /**
+ * This codec maps a CQL {@link DataType#blob()} to a Java {@link ByteBuffer}.
+ */
+ private static class BlobCodec extends TypeCodec<ByteBuffer>
+ {
+
+ private static final BlobCodec instance = new BlobCodec();
+
+ private BlobCodec()
+ {
+ super(DataType.blob(), ByteBuffer.class);
+ }
+
+ @Override
+ public ByteBuffer parse(String value)
+ {
+ return value == null || value.isEmpty() || value.equalsIgnoreCase("NULL")
+ ? null
+ : Bytes.fromHexString(value);
+ }
+
+ @Override
+ public String format(ByteBuffer value)
+ {
+ if (value == null) return "NULL";
+ return Bytes.toHexString(value);
+ }
+
+ @Override
+ public ByteBuffer serialize(ByteBuffer value, ProtocolVersion protocolVersion)
+ {
+ return value == null ? null : value.duplicate();
+ }
+
+ @Override
+ public ByteBuffer deserialize(ByteBuffer bytes, ProtocolVersion protocolVersion)
+ {
+ return bytes == null ? null : bytes.duplicate();
+ }
+ }
+
+ /**
+ * This codec maps a CQL {@link DataType#custom(String) custom} type to a Java {@link ByteBuffer}.
+ * Note that no instance of this codec is part of the default set of codecs used by the Java
+ * driver; instances of this codec must be manually registered.
+ */
+ private static class CustomCodec extends TypeCodec<ByteBuffer>
+ {
+
+ private CustomCodec(DataType custom)
+ {
+ super(custom, ByteBuffer.class);
+ assert custom.getName() == Name.CUSTOM;
+ }
+
+ @Override
+ public ByteBuffer parse(String value)
+ {
+ return value == null || value.isEmpty() || value.equalsIgnoreCase("NULL")
+ ? null
+ : Bytes.fromHexString(value);
+ }
+
+ @Override
+ public String format(ByteBuffer value)
+ {
+ if (value == null) return "NULL";
+ return Bytes.toHexString(value);
+ }
+
+ @Override
+ public ByteBuffer serialize(ByteBuffer value, ProtocolVersion protocolVersion)
+ {
+ return value == null ? null : value.duplicate();
+ }
+
+ @Override
+ public ByteBuffer deserialize(ByteBuffer bytes, ProtocolVersion protocolVersion)
+ {
+ return bytes == null ? null : bytes.duplicate();
+ }
+ }
+
+ /**
+ * This codec maps a CQL {@link DataType#cboolean()} to a Java {@link Boolean}.
+ */
+ private static class BooleanCodec extends PrimitiveBooleanCodec
+ {
+
+ private static final ByteBuffer TRUE = ByteBuffer.wrap(new byte[]{ 1 });
+ private static final ByteBuffer FALSE = ByteBuffer.wrap(new byte[]{ 0 });
+
+ private static final BooleanCodec instance = new BooleanCodec();
+
+ private BooleanCodec()
+ {
+ super(DataType.cboolean());
+ }
+
+ @Override
+ public Boolean parse(String value)
+ {
+ if (value == null || value.isEmpty() || value.equalsIgnoreCase("NULL")) return null;
+ if (value.equalsIgnoreCase(Boolean.FALSE.toString())) return false;
+ if (value.equalsIgnoreCase(Boolean.TRUE.toString())) return true;
+
+ throw new InvalidTypeException(
+ String.format("Cannot parse boolean value from \"%s\"", value));
+ }
+
+ @Override
+ public String format(Boolean value)
+ {
+ if (value == null) return "NULL";
+ return value ? "true" : "false";
+ }
+
+ @Override
+ public ByteBuffer serializeNoBoxing(boolean value, ProtocolVersion protocolVersion)
+ {
+ return value ? TRUE.duplicate() : FALSE.duplicate();
+ }
+
+ @Override
+ public boolean deserializeNoBoxing(ByteBuffer bytes, ProtocolVersion protocolVersion)
+ {
+ if (bytes == null || bytes.remaining() == 0) return false;
+ if (bytes.remaining() != 1)
+ throw new InvalidTypeException(
+ "Invalid boolean value, expecting 1 byte but got " + bytes.remaining());
+
+ return bytes.get(bytes.position()) != 0;
+ }
+ }
+
+ /**
+ * This codec maps a CQL {@link DataType#decimal()} to a Java {@link BigDecimal}.
+ */
+ private static class DecimalCodec extends TypeCodec<BigDecimal>
+ {
+
+ private static final DecimalCodec instance = new DecimalCodec();
+
+ private DecimalCodec()
+ {
+ super(DataType.decimal(), BigDecimal.class);
+ }
+
+ @Override
+ public BigDecimal parse(String value)
+ {
+ try
+ {
+ return value == null || value.isEmpty() || value.equalsIgnoreCase("NULL")
+ ? null
+ : new BigDecimal(value);
+ }
+ catch (NumberFormatException e)
+ {
+ throw new InvalidTypeException(
+ String.format("Cannot parse decimal value from \"%s\"", value));
+ }
+ }
+
+ @Override
+ public String format(BigDecimal value)
+ {
+ if (value == null) return "NULL";
+ return value.toString();
+ }
+
+ @Override
+ public ByteBuffer serialize(BigDecimal value, ProtocolVersion protocolVersion)
+ {
+ if (value == null) return null;
+ BigInteger bi = value.unscaledValue();
+ int scale = value.scale();
+ byte[] bibytes = bi.toByteArray();
+
+ ByteBuffer bytes = ByteBuffer.allocate(4 + bibytes.length);
+ bytes.putInt(scale);
+ bytes.put(bibytes);
+ bytes.rewind();
+ return bytes;
+ }
+
+ @Override
+ public BigDecimal deserialize(ByteBuffer bytes, ProtocolVersion protocolVersion)
+ {
+ if (bytes == null || bytes.remaining() == 0) return null;
+ if (bytes.remaining() < 4)
+ throw new InvalidTypeException(
+ "Invalid decimal value, expecting at least 4 bytes but got " + bytes.remaining());
+
+ bytes = bytes.duplicate();
+ int scale = bytes.getInt();
+ byte[] bibytes = new byte[bytes.remaining()];
+ bytes.get(bibytes);
+
+ BigInteger bi = new BigInteger(bibytes);
+ return new BigDecimal(bi, scale);
+ }
+ }
+
+ /**
+ * This codec maps a CQL {@link DataType#cdouble()} to a Java {@link Double}.
+ */
+ private static class DoubleCodec extends PrimitiveDoubleCodec
+ {
+
+ private static final DoubleCodec instance = new DoubleCodec();
+
+ private DoubleCodec()
+ {
+ super(DataType.cdouble());
+ }
+
+ @Override
+ public Double parse(String value)
+ {
+ try
+ {
+ return value == null || value.isEmpty() || value.equalsIgnoreCase("NULL")
+ ? null
+ : Double.parseDouble(value);
+ }
+ catch (NumberFormatException e)
+ {
+ throw new InvalidTypeException(
+ String.format("Cannot parse 64-bits double value from \"%s\"", value));
+ }
+ }
+
+ @Override
+ public String format(Double value)
+ {
+ if (value == null) return "NULL";
+ return Double.toString(value);
+ }
+
+ @Override
+ public ByteBuffer serializeNoBoxing(double value, ProtocolVersion protocolVersion)
+ {
+ ByteBuffer bb = ByteBuffer.allocate(8);
+ bb.putDouble(0, value);
+ return bb;
+ }
+
+ @Override
+ public double deserializeNoBoxing(ByteBuffer bytes, ProtocolVersion protocolVersion)
+ {
+ if (bytes == null || bytes.remaining() == 0) return 0;
+ if (bytes.remaining() != 8)
+ throw new InvalidTypeException(
+ "Invalid 64-bits double value, expecting 8 bytes but got " + bytes.remaining());
+
+ return bytes.getDouble(bytes.position());
+ }
+ }
+
+ /**
+ * This codec maps a CQL {@link DataType#cfloat()} to a Java {@link Float}.
+ */
+ private static class FloatCodec extends PrimitiveFloatCodec
+ {
+
+ private static final FloatCodec instance = new FloatCodec();
+
+ private FloatCodec()
+ {
+ super(DataType.cfloat());
+ }
+
+ @Override
+ public Float parse(String value)
+ {
+ try
+ {
+ return value == null || value.isEmpty() || value.equalsIgnoreCase("NULL")
+ ? null
+ : Float.parseFloat(value);
+ }
+ catch (NumberFormatException e)
+ {
+ throw new InvalidTypeException(
+ String.format("Cannot parse 32-bits float value from \"%s\"", value));
+ }
+ }
+
+ @Override
+ public String format(Float value)
+ {
+ if (value == null) return "NULL";
+ return Float.toString(value);
+ }
+
+ @Override
+ public ByteBuffer serializeNoBoxing(float value, ProtocolVersion protocolVersion)
+ {
+ ByteBuffer bb = ByteBuffer.allocate(4);
+ bb.putFloat(0, value);
+ return bb;
+ }
+
+ @Override
+ public float deserializeNoBoxing(ByteBuffer bytes, ProtocolVersion protocolVersion)
+ {
+ if (bytes == null || bytes.remaining() == 0) return 0;
+ if (bytes.remaining() != 4)
+ throw new InvalidTypeException(
+ "Invalid 32-bits float value, expecting 4 bytes but got " + bytes.remaining());
+
+ return bytes.getFloat(bytes.position());
+ }
+ }
+
+ /**
+ * This codec maps a CQL {@link DataType#inet()} to a Java {@link InetAddress}.
+ */
+ private static class InetCodec extends TypeCodec<InetAddress>
+ {
+
+ private static final InetCodec instance = new InetCodec();
+
+ private InetCodec()
+ {
+ super(DataType.inet(), InetAddress.class);
+ }
+
+ @Override
+ public InetAddress parse(String value)
+ {
+ if (value == null || value.isEmpty() || value.equalsIgnoreCase("NULL")) return null;
+
+ value = value.trim();
+ if (!ParseUtils.isQuoted(value))
+ throw new InvalidTypeException(
+ String.format("inet values must be enclosed in single quotes (\"%s\")", value));
+ try
+ {
+ return InetAddress.getByName(value.substring(1, value.length() - 1));
+ }
+ catch (Exception e)
+ {
+ throw new InvalidTypeException(String.format("Cannot parse inet value from \"%s\"", value));
+ }
+ }
+
+ @Override
+ public String format(InetAddress value)
+ {
+ if (value == null) return "NULL";
+ return '\'' + value.getHostAddress() + '\'';
+ }
+
+ @Override
+ public ByteBuffer serialize(InetAddress value, ProtocolVersion protocolVersion)
+ {
+ return value == null ? null : ByteBuffer.wrap(value.getAddress());
+ }
+
+ @Override
+ public InetAddress deserialize(ByteBuffer bytes, ProtocolVersion protocolVersion)
+ {
+ if (bytes == null || bytes.remaining() == 0) return null;
+ try
+ {
+ return InetAddress.getByAddress(Bytes.getArray(bytes));
+ }
+ catch (UnknownHostException e)
+ {
+ throw new InvalidTypeException(
+ "Invalid bytes for inet value, got " + bytes.remaining() + " bytes");
+ }
+ }
+ }
+
+ /**
+ * This codec maps a CQL {@link DataType#tinyint()} to a Java {@link Byte}.
+ */
+ private static class TinyIntCodec extends PrimitiveByteCodec
+ {
+
+ private static final TinyIntCodec instance = new TinyIntCodec();
+
+ private TinyIntCodec()
+ {
+ super(tinyint());
+ }
+
+ @Override
+ public Byte parse(String value)
+ {
+ try
+ {
+ return value == null || value.isEmpty() || value.equalsIgnoreCase("NULL")
+ ? null
+ : Byte.parseByte(value);
+ }
+ catch (NumberFormatException e)
+ {
+ throw new InvalidTypeException(
+ String.format("Cannot parse 8-bits int value from \"%s\"", value));
+ }
+ }
+
+ @Override
+ public String format(Byte value)
+ {
+ if (value == null) return "NULL";
+ return Byte.toString(value);
+ }
+
+ @Override
+ public ByteBuffer serializeNoBoxing(byte value, ProtocolVersion protocolVersion)
+ {
+ ByteBuffer bb = ByteBuffer.allocate(1);
+ bb.put(0, value);
+ return bb;
+ }
+
+ @Override
+ public byte deserializeNoBoxing(ByteBuffer bytes, ProtocolVersion protocolVersion)
+ {
+ if (bytes == null || bytes.remaining() == 0) return 0;
+ if (bytes.remaining() != 1)
+ throw new InvalidTypeException(
+ "Invalid 8-bits integer value, expecting 1 byte but got " + bytes.remaining());
+
+ return bytes.get(bytes.position());
+ }
+ }
+
+ /**
+ * This codec maps a CQL {@link DataType#smallint()} to a Java {@link Short}.
+ */
+ private static class SmallIntCodec extends PrimitiveShortCodec
+ {
+
+ private static final SmallIntCodec instance = new SmallIntCodec();
+
+ private SmallIntCodec()
+ {
+ super(smallint());
+ }
+
+ @Override
+ public Short parse(String value)
+ {
+ try
+ {
+ return value == null || value.isEmpty() || value.equalsIgnoreCase("NULL")
+ ? null
+ : Short.parseShort(value);
+ }
+ catch (NumberFormatException e)
+ {
+ throw new InvalidTypeException(
+ String.format("Cannot parse 16-bits int value from \"%s\"", value));
+ }
+ }
+
+ @Override
+ public String format(Short value)
+ {
+ if (value == null) return "NULL";
+ return Short.toString(value);
+ }
+
+ @Override
+ public ByteBuffer serializeNoBoxing(short value, ProtocolVersion protocolVersion)
+ {
+ ByteBuffer bb = ByteBuffer.allocate(2);
+ bb.putShort(0, value);
+ return bb;
+ }
+
+ @Override
+ public short deserializeNoBoxing(ByteBuffer bytes, ProtocolVersion protocolVersion)
+ {
+ if (bytes == null || bytes.remaining() == 0) return 0;
+ if (bytes.remaining() != 2)
+ throw new InvalidTypeException(
+ "Invalid 16-bits integer value, expecting 2 bytes but got " + bytes.remaining());
+
+ return bytes.getShort(bytes.position());
+ }
+ }
+
+ /**
+ * This codec maps a CQL {@link DataType#cint()} to a Java {@link Integer}.
+ */
+ private static class IntCodec extends PrimitiveIntCodec
+ {
+
+ private static final IntCodec instance = new IntCodec();
+
+ private IntCodec()
+ {
+ super(DataType.cint());
+ }
+
+ @Override
+ public Integer parse(String value)
+ {
+ try
+ {
+ return value == null || value.isEmpty() || value.equalsIgnoreCase("NULL")
+ ? null
+ : Integer.parseInt(value);
+ }
+ catch (NumberFormatException e)
+ {
+ throw new InvalidTypeException(
+ String.format("Cannot parse 32-bits int value from \"%s\"", value));
+ }
+ }
+
+ @Override
+ public String format(Integer value)
+ {
+ if (value == null) return "NULL";
+ return Integer.toString(value);
+ }
+
+ @Override
+ public ByteBuffer serializeNoBoxing(int value, ProtocolVersion protocolVersion)
+ {
+ ByteBuffer bb = ByteBuffer.allocate(4);
+ bb.putInt(0, value);
+ return bb;
+ }
+
+ @Override
+ public int deserializeNoBoxing(ByteBuffer bytes, ProtocolVersion protocolVersion)
+ {
+ if (bytes == null || bytes.remaining() == 0) return 0;
+ if (bytes.remaining() != 4)
+ throw new InvalidTypeException(
+ "Invalid 32-bits integer value, expecting 4 bytes but got " + bytes.remaining());
+
+ return bytes.getInt(bytes.position());
+ }
+ }
+
+ /**
+ * This codec maps a CQL {@link DataType#timestamp()} to a Java {@link Date}.
+ */
+ private static class TimestampCodec extends TypeCodec<Date>
+ {
+
+ private static final TimestampCodec instance = new TimestampCodec();
+
+ private TimestampCodec()
+ {
+ super(DataType.timestamp(), Date.class);
+ }
+
+ @Override
+ public Date parse(String value)
+ {
+ if (value == null || value.isEmpty() || value.equalsIgnoreCase("NULL")) return null;
+ // strip enclosing single quotes, if any
+ if (ParseUtils.isQuoted(value)) value = ParseUtils.unquote(value);
+
+ if (ParseUtils.isLongLiteral(value))
+ {
+ try
+ {
+ return new Date(Long.parseLong(value));
+ }
+ catch (NumberFormatException e)
+ {
+ throw new InvalidTypeException(
+ String.format("Cannot parse timestamp value from \"%s\"", value));
+ }
+ }
+
+ try
+ {
+ return ParseUtils.parseDate(value);
+ }
+ catch (ParseException e)
+ {
+ throw new InvalidTypeException(
+ String.format("Cannot parse timestamp value from \"%s\"", value));
+ }
+ }
+
+ @Override
+ public String format(Date value)
+ {
+ if (value == null) return "NULL";
+ return Long.toString(value.getTime());
+ }
+
+ @Override
+ public ByteBuffer serialize(Date value, ProtocolVersion protocolVersion)
+ {
+ return value == null
+ ? null
+ : BigintCodec.instance.serializeNoBoxing(value.getTime(), protocolVersion);
+ }
+
+ @Override
+ public Date deserialize(ByteBuffer bytes, ProtocolVersion protocolVersion)
+ {
+ return bytes == null || bytes.remaining() == 0
+ ? null
+ : new Date(BigintCodec.instance.deserializeNoBoxing(bytes, protocolVersion));
+ }
+ }
+
+ /**
+ * This codec maps a CQL {@link DataType#date()} to the custom {@link LocalDate} class.
+ */
+ private static class DateCodec extends TypeCodec<LocalDate>
+ {
+
+ private static final DateCodec instance = new DateCodec();
+
+ private static final String pattern = "yyyy-MM-dd";
+
+ private DateCodec()
+ {
+ super(DataType.date(), LocalDate.class);
+ }
+
+ @Override
+ public LocalDate parse(String value)
+ {
+ if (value == null || value.isEmpty() || value.equalsIgnoreCase("NULL")) return null;
+
+ // single quotes are optional for long literals, mandatory for date patterns
+ // strip enclosing single quotes, if any
+ if (ParseUtils.isQuoted(value)) value = ParseUtils.unquote(value);
+
+ if (ParseUtils.isLongLiteral(value))
+ {
+ long unsigned;
+ try
+ {
+ unsigned = Long.parseLong(value);
+ }
+ catch (NumberFormatException e)
+ {
+ throw new InvalidTypeException(
+ String.format("Cannot parse date value from \"%s\"", value), e);
+ }
+ try
+ {
+ int days = CodecUtils.fromCqlDateToDaysSinceEpoch(unsigned);
+ return LocalDate.fromDaysSinceEpoch(days);
+ }
+ catch (IllegalArgumentException e)
+ {
+ throw new InvalidTypeException(
+ String.format("Cannot parse date value from \"%s\"", value), e);
+ }
+ }
+
+ try
+ {
+ Date date = ParseUtils.parseDate(value, pattern);
+ return LocalDate.fromMillisSinceEpoch(date.getTime());
+ }
+ catch (ParseException e)
+ {
+ throw new InvalidTypeException(
+ String.format("Cannot parse date value from \"%s\"", value), e);
+ }
+ }
+
+ @Override
+ public String format(LocalDate value)
+ {
+ if (value == null) return "NULL";
+ return ParseUtils.quote(value.toString());
+ }
+
+ @Override
+ public ByteBuffer serialize(LocalDate value, ProtocolVersion protocolVersion)
+ {
+ if (value == null) return null;
+ int unsigned = CodecUtils.fromSignedToUnsignedInt(value.getDaysSinceEpoch());
+ return IntCodec.instance.serializeNoBoxing(unsigned, protocolVersion);
+ }
+
+ @Override
+ public LocalDate deserialize(ByteBuffer bytes, ProtocolVersion protocolVersion)
+ {
+ if (bytes == null || bytes.remaining() == 0) return null;
+ int unsigned = IntCodec.instance.deserializeNoBoxing(bytes, protocolVersion);
+ int signed = CodecUtils.fromUnsignedToSignedInt(unsigned);
+ return LocalDate.fromDaysSinceEpoch(signed);
+ }
+ }
+
+ /**
+ * This codec maps a CQL {@link DataType#time()} to a Java {@link Long}.
+ */
+ private static class TimeCodec extends LongCodec
+ {
+
+ private static final TimeCodec instance = new TimeCodec();
+
+ private TimeCodec()
+ {
+ super(DataType.time());
+ }
+
+ @Override
+ public Long parse(String value)
+ {
+ if (value == null || value.isEmpty() || value.equalsIgnoreCase("NULL")) return null;
+
+ // enclosing single quotes required, even for long literals
+ if (!ParseUtils.isQuoted(value))
+ throw new InvalidTypeException("time values must be enclosed by single quotes");
+ value = value.substring(1, value.length() - 1);
+
+ if (ParseUtils.isLongLiteral(value))
+ {
+ try
+ {
+ return Long.parseLong(value);
+ }
+ catch (NumberFormatException e)
+ {
+ throw new InvalidTypeException(
+ String.format("Cannot parse time value from \"%s\"", value), e);
+ }
+ }
+
+ try
+ {
+ return ParseUtils.parseTime(value);
+ }
+ catch (ParseException e)
+ {
+ throw new InvalidTypeException(
+ String.format("Cannot parse time value from \"%s\"", value), e);
+ }
+ }
+
+ @Override
+ public String format(Long value)
+ {
+ if (value == null) return "NULL";
+ return ParseUtils.quote(ParseUtils.formatTime(value));
+ }
+ }
+
+ /**
+ * Base class for codecs handling CQL UUID types such as {@link DataType#uuid()} and {@link
+ * DataType#timeuuid()}.
+ */
+ private abstract static class AbstractUUIDCodec extends TypeCodec<UUID>
+ {
+
+ private AbstractUUIDCodec(DataType cqlType)
+ {
+ super(cqlType, UUID.class);
+ }
+
+ @Override
+ public UUID parse(String value)
+ {
+ try
+ {
+ return value == null || value.isEmpty() || value.equalsIgnoreCase("NULL")
+ ? null
+ : UUID.fromString(value);
+ }
+ catch (IllegalArgumentException e)
+ {
+ throw new InvalidTypeException(
+ String.format("Cannot parse UUID value from \"%s\"", value), e);
+ }
+ }
+
+ @Override
+ public String format(UUID value)
+ {
+ if (value == null) return "NULL";
+ return value.toString();
+ }
+
+ @Override
+ public ByteBuffer serialize(UUID value, ProtocolVersion protocolVersion)
+ {
+ if (value == null) return null;
+ ByteBuffer bb = ByteBuffer.allocate(16);
+ bb.putLong(0, value.getMostSignificantBits());
+ bb.putLong(8, value.getLeastSignificantBits());
+ return bb;
+ }
+
+ @Override
+ public UUID deserialize(ByteBuffer bytes, ProtocolVersion protocolVersion)
+ {
+ return bytes == null || bytes.remaining() == 0
+ ? null
+ : new UUID(bytes.getLong(bytes.position()), bytes.getLong(bytes.position() + 8));
+ }
+ }
+
+ /**
+ * This codec maps a CQL {@link DataType#uuid()} to a Java {@link UUID}.
+ */
+ private static class UUIDCodec extends AbstractUUIDCodec
+ {
+
+ private static final UUIDCodec instance = new UUIDCodec();
+
+ private UUIDCodec()
+ {
+ super(DataType.uuid());
+ }
+ }
+
+ /**
+ * This codec maps a CQL {@link DataType#timeuuid()} to a Java {@link UUID}.
+ */
+ private static class TimeUUIDCodec extends AbstractUUIDCodec
+ {
+
+ private static final TimeUUIDCodec instance = new TimeUUIDCodec();
+
+ private TimeUUIDCodec()
+ {
+ super(timeuuid());
+ }
+
+ @Override
+ public String format(UUID value)
+ {
+ if (value == null) return "NULL";
+ if (value.version() != 1)
+ throw new InvalidTypeException(
+ String.format("%s is not a Type 1 (time-based) UUID", value));
+ return super.format(value);
+ }
+
+ @Override
+ public ByteBuffer serialize(UUID value, ProtocolVersion protocolVersion)
+ {
+ if (value == null) return null;
+ if (value.version() != 1)
+ throw new InvalidTypeException(
+ String.format("%s is not a Type 1 (time-based) UUID", value));
+ return super.serialize(value, protocolVersion);
+ }
+ }
+
+ /**
+ * This codec maps a CQL {@link DataType#varint()} to a Java {@link BigInteger}.
+ */
+ private static class VarintCodec extends TypeCodec<BigInteger>
+ {
+
+ private static final VarintCodec instance = new VarintCodec();
+
+ private VarintCodec()
+ {
+ super(DataType.varint(), BigInteger.class);
+ }
+
+ @Override
+ public BigInteger parse(String value)
+ {
+ try
+ {
+ return value == null || value.isEmpty() || value.equalsIgnoreCase("NULL")
+ ? null
+ : new BigInteger(value);
+ }
+ catch (NumberFormatException e)
+ {
+ throw new InvalidTypeException(
+ String.format("Cannot parse varint value from \"%s\"", value), e);
+ }
+ }
+
+ @Override
+ public String format(BigInteger value)
+ {
+ if (value == null) return "NULL";
+ return value.toString();
+ }
+
+ @Override
+ public ByteBuffer serialize(BigInteger value, ProtocolVersion protocolVersion)
+ {
+ return value == null ? null : ByteBuffer.wrap(value.toByteArray());
+ }
+
+ @Override
+ public BigInteger deserialize(ByteBuffer bytes, ProtocolVersion protocolVersion)
+ {
+ return bytes == null || bytes.remaining() == 0 ? null : new BigInteger(Bytes.getArray(bytes));
+ }
+ }
+
+ /**
+ * Base class for codecs mapping CQL {@link DataType#list(DataType) lists} and {@link
+ * DataType#set(DataType) sets} to Java collections.
+ */
+ public abstract static class AbstractCollectionCodec<E, C extends Collection<E>>
+ extends TypeCodec<C>
+ {
+
+ final TypeCodec<E> eltCodec;
+
+ AbstractCollectionCodec(
+ CollectionType cqlType, TypeToken<C> javaType, TypeCodec<E> eltCodec)
+ {
+ super(cqlType, javaType);
+ checkArgument(
+ cqlType.getName() == Name.LIST || cqlType.getName() == Name.SET,
+ "Expecting list or set type, got %s",
+ cqlType);
+ this.eltCodec = eltCodec;
+ }
+
+ @Override
+ public ByteBuffer serialize(C value, ProtocolVersion protocolVersion)
+ {
+ if (value == null) return null;
+ int i = 0;
+ ByteBuffer[] bbs = new ByteBuffer[value.size()];
+ for (E elt : value)
+ {
+ if (elt == null)
+ {
+ throw new NullPointerException("Collection elements cannot be null");
+ }
+ ByteBuffer bb;
+ try
+ {
+ bb = eltCodec.serialize(elt, protocolVersion);
+ }
+ catch (ClassCastException e)
+ {
+ throw new InvalidTypeException(
+ String.format(
+ "Invalid type for %s element, expecting %s but got %s",
+ cqlType, eltCodec.getJavaType(), elt.getClass()),
+ e);
+ }
+ bbs[i++] = bb;
+ }
+ return CodecUtils.pack(bbs, value.size(), protocolVersion);
+ }
+
+ @Override
+ public C deserialize(ByteBuffer bytes, ProtocolVersion protocolVersion)
+ {
+ if (bytes == null || bytes.remaining() == 0) return newInstance(0);
+ try
+ {
+ ByteBuffer input = bytes.duplicate();
+ int size = CodecUtils.readSize(input, protocolVersion);
+ C coll = newInstance(size);
+ for (int i = 0; i < size; i++)
+ {
+ ByteBuffer databb = CodecUtils.readValue(input, protocolVersion);
+ coll.add(eltCodec.deserialize(databb, protocolVersion));
+ }
+ return coll;
+ }
+ catch (BufferUnderflowException e)
+ {
+ throw new InvalidTypeException("Not enough bytes to deserialize collection", e);
+ }
+ }
+
+ @Override
+ public String format(C value)
+ {
+ if (value == null) return "NULL";
+ StringBuilder sb = new StringBuilder();
+ sb.append(getOpeningChar());
+ int i = 0;
+ for (E v : value)
+ {
+ if (i++ != 0) sb.append(',');
+ sb.append(eltCodec.format(v));
+ }
+ sb.append(getClosingChar());
+ return sb.toString();
+ }
+
+ @Override
+ public C parse(String value)
+ {
+ if (value == null || value.isEmpty() || value.equalsIgnoreCase("NULL")) return null;
+
+ int idx = ParseUtils.skipSpaces(value, 0);
+ if (value.charAt(idx++) != getOpeningChar())
+ throw new InvalidTypeException(
+ String.format(
+ "Cannot parse collection value from \"%s\", at character %d expecting '%s' but got '%c'",
+ value, idx, getOpeningChar(), value.charAt(idx)));
+
+ idx = ParseUtils.skipSpaces(value, idx);
+
+ if (value.charAt(idx) == getClosingChar()) return newInstance(0);
+
+ C l = newInstance(10);
+ while (idx < value.length())
+ {
+ int n;
+ try
+ {
+ n = ParseUtils.skipCQLValue(value, idx);
+ }
+ catch (IllegalArgumentException e)
+ {
+ throw new InvalidTypeException(
+ String.format(
+ "Cannot parse collection value from \"%s\", invalid CQL value at character %d",
+ value, idx),
+ e);
+ }
+
+ l.add(eltCodec.parse(value.substring(idx, n)));
+ idx = n;
+
+ idx = ParseUtils.skipSpaces(value, idx);
+ if (value.charAt(idx) == getClosingChar()) return l;
+ if (value.charAt(idx++) != ',')
+ throw new InvalidTypeException(
+ String.format(
+ "Cannot parse collection value from \"%s\", at character %d expecting ',' but got '%c'",
+ value, idx, value.charAt(idx)));
+
+ idx = ParseUtils.skipSpaces(value, idx);
+ }
+ throw new InvalidTypeException(
+ String.format(
+ "Malformed collection value \"%s\", missing closing '%s'", value, getClosingChar()));
+ }
+
+ @Override
+ public boolean accepts(Object value)
+ {
+ if (getJavaType().getRawType().isAssignableFrom(value.getClass()))
+ {
+ // runtime type ok, now check element type
+ Collection<?> coll = (Collection<?>) value;
+ if (coll.isEmpty()) return true;
+ Object elt = coll.iterator().next();
+ return eltCodec.accepts(elt);
+ }
+ return false;
+ }
+
+ /**
+ * Return a new instance of {@code C} with the given estimated size.
+ *
+ * @param size The estimated size of the collection to create.
+ * @return new instance of {@code C} with the given estimated size.
+ */
+ protected abstract C newInstance(int size);
+
+ /**
+ * Return the opening character to use when formatting values as CQL literals.
+ *
+ * @return The opening character to use when formatting values as CQL literals.
+ */
+ private char getOpeningChar()
+ {
+ return cqlType.getName() == Name.LIST ? '[' : '{';
+ }
+
+ /**
+ * Return the closing character to use when formatting values as CQL literals.
+ *
+ * @return The closing character to use when formatting values as CQL literals.
+ */
+ private char getClosingChar()
+ {
+ return cqlType.getName() == Name.LIST ? ']' : '}';
+ }
+ }
+
+ /**
+ * This codec maps a CQL {@link DataType#list(DataType) list type} to a Java {@link List}.
+ * Implementation note: this codec returns mutable, non thread-safe {@link ArrayList} instances.
+ */
+ private static class ListCodec<T> extends AbstractCollectionCodec<T, List<T>>
+ {
+
+ private ListCodec(TypeCodec<T> eltCodec)
+ {
+ super(
+ DataType.list(eltCodec.getCqlType()),
+ TypeTokens.listOf(eltCodec.getJavaType()),
+ eltCodec);
+ }
+
+ @Override
+ protected List<T> newInstance(int size)
+ {
+ return new ArrayList<>(size);
+ }
+ }
+
+ /**
+ * This codec maps a CQL {@link DataType#set(DataType) set type} to a Java {@link Set}.
+ * Implementation note: this codec returns mutable, non thread-safe {@link LinkedHashSet}
+ * instances.
+ */
+ private static class SetCodec<T> extends AbstractCollectionCodec<T, Set<T>>
+ {
+
+ private SetCodec(TypeCodec<T> eltCodec)
+ {
+ super(DataType.set(eltCodec.cqlType), TypeTokens.setOf(eltCodec.getJavaType()), eltCodec);
+ }
+
+ @Override
+ protected Set<T> newInstance(int size)
+ {
+ return new LinkedHashSet<>(size);
+ }
+ }
+
+ /**
+ * Base class for codecs mapping CQL {@link DataType#map(DataType, DataType) maps} to a Java
+ * {@link Map}.
+ */
+ public abstract static class AbstractMapCodec<K, V> extends TypeCodec<Map<K, V>>
+ {
+
+ final TypeCodec<K> keyCodec;
+
+ final TypeCodec<V> valueCodec;
+
+ AbstractMapCodec(TypeCodec<K> keyCodec, TypeCodec<V> valueCodec)
+ {
+ super(
+ DataType.map(keyCodec.getCqlType(), valueCodec.getCqlType()),
+ TypeTokens.mapOf(keyCodec.getJavaType(), valueCodec.getJavaType()));
+ this.keyCodec = keyCodec;
+ this.valueCodec = valueCodec;
+ }
+
+ @Override
+ public boolean accepts(Object value)
+ {
+ if (value instanceof Map)
+ {
+ // runtime type ok, now check key and value types
+ Map<?, ?> map = (Map<?, ?>) value;
+ if (map.isEmpty()) return true;
+ Map.Entry<?, ?> entry = map.entrySet().iterator().next();
+ return keyCodec.accepts(entry.getKey()) && valueCodec.accepts(entry.getValue());
+ }
+ return false;
+ }
+
+ @Override
+ public Map<K, V> parse(String value)
+ {
+ if (value == null || value.isEmpty() || value.equalsIgnoreCase("NULL")) return null;
+
+ int idx = ParseUtils.skipSpaces(value, 0);
+ if (value.charAt(idx++) != '{')
+ throw new InvalidTypeException(
+ String.format(
+ "cannot parse map value from \"%s\", at character %d expecting '{' but got '%c'",
+ value, idx, value.charAt(idx)));
+
+ idx = ParseUtils.skipSpaces(value, idx);
+
+ if (value.charAt(idx) == '}') return newInstance(0);
+
+ Map<K, V> m = new HashMap<>();
+ while (idx < value.length())
+ {
+ int n;
+ try
+ {
+ n = ParseUtils.skipCQLValue(value, idx);
+ }
+ catch (IllegalArgumentException e)
+ {
+ throw new InvalidTypeException(
+ String.format(
+ "Cannot parse map value from \"%s\", invalid CQL value at character %d",
+ value, idx),
+ e);
+ }
+
+ K k = keyCodec.parse(value.substring(idx, n));
+ idx = n;
+
+ idx = ParseUtils.skipSpaces(value, idx);
+ if (value.charAt(idx++) != ':')
+ throw new InvalidTypeException(
+ String.format(
+ "Cannot parse map value from \"%s\", at character %d expecting ':' but got '%c'",
+ value, idx, value.charAt(idx)));
+ idx = ParseUtils.skipSpaces(value, idx);
+
+ try
+ {
+ n = ParseUtils.skipCQLValue(value, idx);
+ }
+ catch (IllegalArgumentException e)
+ {
+ throw new InvalidTypeException(
+ String.format(
+ "Cannot parse map value from \"%s\", invalid CQL value at character %d",
+ value, idx),
+ e);
+ }
+
+ V v = valueCodec.parse(value.substring(idx, n));
+ idx = n;
+
+ m.put(k, v);
+
+ idx = ParseUtils.skipSpaces(value, idx);
+ if (value.charAt(idx) == '}') return m;
+ if (value.charAt(idx++) != ',')
+ throw new InvalidTypeException(
+ String.format(
+ "Cannot parse map value from \"%s\", at character %d expecting ',' but got '%c'",
+ value, idx, value.charAt(idx)));
+
+ idx = ParseUtils.skipSpaces(value, idx);
+ }
+ throw new InvalidTypeException(
+ String.format("Malformed map value \"%s\", missing closing '}'", value));
+ }
+
+ @Override
+ public String format(Map<K, V> value)
+ {
+ if (value == null) return "NULL";
+ StringBuilder sb = new StringBuilder();
+ sb.append('{');
+ int i = 0;
+ for (Map.Entry<K, V> e : value.entrySet())
+ {
+ if (i++ != 0) sb.append(',');
+ sb.append(keyCodec.format(e.getKey()));
+ sb.append(':');
+ sb.append(valueCodec.format(e.getValue()));
+ }
+ sb.append('}');
+ return sb.toString();
+ }
+
+ @Override
+ public ByteBuffer serialize(Map<K, V> value, ProtocolVersion protocolVersion)
+ {
+ if (value == null) return null;
+ int i = 0;
+ ByteBuffer[] bbs = new ByteBuffer[2 * value.size()];
+ for (Map.Entry<K, V> entry : value.entrySet())
+ {
+ ByteBuffer bbk;
+ K key = entry.getKey();
+ if (key == null)
+ {
+ throw new NullPointerException("Map keys cannot be null");
+ }
+ try
+ {
+ bbk = keyCodec.serialize(key, protocolVersion);
+ }
+ catch (ClassCastException e)
+ {
+ throw new InvalidTypeException(
+ String.format(
+ "Invalid type for map key, expecting %s but got %s",
+ keyCodec.getJavaType(), key.getClass()),
+ e);
+ }
+ ByteBuffer bbv;
+ V v = entry.getValue();
+ if (v == null)
+ {
+ throw new NullPointerException("Map values cannot be null");
+ }
+ try
+ {
+ bbv = valueCodec.serialize(v, protocolVersion);
+ }
+ catch (ClassCastException e)
+ {
+ throw new InvalidTypeException(
+ String.format(
+ "Invalid type for map value, expecting %s but got %s",
+ valueCodec.getJavaType(), v.getClass()),
+ e);
+ }
+ bbs[i++] = bbk;
+ bbs[i++] = bbv;
+ }
+ return CodecUtils.pack(bbs, value.size(), protocolVersion);
+ }
+
+ @Override
+ public Map<K, V> deserialize(ByteBuffer bytes, ProtocolVersion protocolVersion)
+ {
+ if (bytes == null || bytes.remaining() == 0) return newInstance(0);
+ try
+ {
+ ByteBuffer input = bytes.duplicate();
+ int n = CodecUtils.readSize(input, protocolVersion);
+ Map<K, V> m = newInstance(n);
+ for (int i = 0; i < n; i++)
+ {
+ ByteBuffer kbb = CodecUtils.readValue(input, protocolVersion);
+ ByteBuffer vbb = CodecUtils.readValue(input, protocolVersion);
+ m.put(
+ keyCodec.deserialize(kbb, protocolVersion),
+ valueCodec.deserialize(vbb, protocolVersion));
+ }
+ return m;
+ }
+ catch (BufferUnderflowException e)
+ {
+ throw new InvalidTypeException("Not enough bytes to deserialize a map", e);
+ }
+ }
+
+ /**
+ * Return a new {@link Map} instance with the given estimated size.
+ *
+ * @param size The estimated size of the collection to create.
+ * @return A new {@link Map} instance with the given estimated size.
+ */
+ protected abstract Map<K, V> newInstance(int size);
+ }
+
+ /**
+ * This codec maps a CQL {@link DataType#map(DataType, DataType) map type} to a Java {@link Map}.
+ * Implementation note: this codec returns mutable, non thread-safe {@link LinkedHashMap}
+ * instances.
+ */
+ private static class MapCodec<K, V> extends AbstractMapCodec<K, V>
+ {
+
+ private MapCodec(TypeCodec<K> keyCodec, TypeCodec<V> valueCodec)
+ {
+ super(keyCodec, valueCodec);
+ }
+
+ @Override
+ protected Map<K, V> newInstance(int size)
+ {
+ return new LinkedHashMap<>(size);
+ }
+ }
+
+ /**
+ * Base class for codecs mapping CQL {@link UserType user-defined types} (UDTs) to Java objects.
+ * It can serve as a base class for codecs dealing with direct UDT-to-Pojo mappings.
+ *
+ * @param <T> The Java type that the UDT will be mapped to.
+ */
+ public abstract static class AbstractUDTCodec<T> extends TypeCodec<T>
+ {
+
+ protected final UserType definition;
+
+ AbstractUDTCodec(UserType definition, Class<T> javaClass)
+ {
+ this(definition, TypeToken.of(javaClass));
+ }
+
+ AbstractUDTCodec(UserType definition, TypeToken<T> javaType)
+ {
+ super(definition, javaType);
+ this.definition = definition;
+ }
+
+ @Override
+ public ByteBuffer serialize(T value, ProtocolVersion protocolVersion)
+ {
+ if (value == null) return null;
+ int size = 0;
+ int length = definition.size();
+ ByteBuffer[] elements = new ByteBuffer[length];
+ int i = 0;
+ for (UserType.Field field : definition)
+ {
+ elements[i] =
+ serializeField(value, Metadata.quoteIfNecessary(field.getName()), protocolVersion);
+ size += 4 + (elements[i] == null ? 0 : elements[i].remaining());
+ i++;
+ }
+ ByteBuffer result = ByteBuffer.allocate(size);
+ for (ByteBuffer bb : elements)
+ {
+ if (bb == null)
+ {
+ result.putInt(-1);
+ }
+ else
+ {
+ result.putInt(bb.remaining());
+ result.put(bb.duplicate());
+ }
+ }
+ return (ByteBuffer) result.flip();
+ }
+
+ @Override
+ public T deserialize(ByteBuffer bytes, ProtocolVersion protocolVersion)
+ {
+ if (bytes == null) return null;
+ // empty byte buffers will result in empty values
+ try
+ {
+ ByteBuffer input = bytes.duplicate();
+ T value = newInstance();
+ for (UserType.Field field : definition)
+ {
+ if (!input.hasRemaining()) break;
+ int n = input.getInt();
+ ByteBuffer element = n < 0 ? null : CodecUtils.readBytes(input, n);
+ value =
+ deserializeAndSetField(
+ element, value, Metadata.quoteIfNecessary(field.getName()), protocolVersion);
+ }
+ return value;
+ }
+ catch (BufferUnderflowException e)
+ {
+ throw new InvalidTypeException("Not enough bytes to deserialize a UDT", e);
+ }
+ }
+
+ @Override
+ public String format(T value)
+ {
+ if (value == null) return "NULL";
+ StringBuilder sb = new StringBuilder("{");
+ int i = 0;
+ for (UserType.Field field : definition)
+ {
+ if (i > 0) sb.append(',');
+ sb.append(Metadata.quoteIfNecessary(field.getName()));
+ sb.append(':');
+ sb.append(formatField(value, Metadata.quoteIfNecessary(field.getName())));
+ i += 1;
+ }
+ sb.append('}');
+ return sb.toString();
+ }
+
+ @Override
+ public T parse(String value)
+ {
+ if (value == null || value.isEmpty() || value.equals("NULL")) return null;
+
+ T v = newInstance();
+
+ int idx = ParseUtils.skipSpaces(value, 0);
+ if (value.charAt(idx++) != '{')
+ throw new InvalidTypeException(
+ String.format(
+ "Cannot parse UDT value from \"%s\", at character %d expecting '{' but got '%c'",
+ value, idx, value.charAt(idx)));
+
+ idx = ParseUtils.skipSpaces(value, idx);
+
+ if (value.charAt(idx) == '}') return v;
+
+ while (idx < value.length())
+ {
+
+ int n;
+ try
+ {
+ n = ParseUtils.skipCQLId(value, idx);
+ }
+ catch (IllegalArgumentException e)
+ {
+ throw new InvalidTypeException(
+ String.format(
+ "Cannot parse UDT value from \"%s\", cannot parse a CQL identifier at character %d",
+ value, idx),
+ e);
+ }
+ String name = value.substring(idx, n);
+ idx = n;
+
+ if (!definition.contains(name))
+ throw new InvalidTypeException(
+ String.format("Unknown field %s in value \"%s\"", name, value));
+
+ idx = ParseUtils.skipSpaces(value, idx);
+ if (value.charAt(idx++) != ':')
+ throw new InvalidTypeException(
+ String.format(
+ "Cannot parse UDT value from \"%s\", at character %d expecting ':' but got '%c'",
+ value, idx, value.charAt(idx)));
+ idx = ParseUtils.skipSpaces(value, idx);
+
+ try
+ {
+ n = ParseUtils.skipCQLValue(value, idx);
+ }
+ catch (IllegalArgumentException e)
+ {
+ throw new InvalidTypeException(
+ String.format(
+ "Cannot parse UDT value from \"%s\", invalid CQL value at character %d",
+ value, idx),
+ e);
+ }
+
+ String input = value.substring(idx, n);
+ v = parseAndSetField(input, v, name);
+ idx = n;
+
+ idx = ParseUtils.skipSpaces(value, idx);
+ if (value.charAt(idx) == '}') return v;
+ if (value.charAt(idx) != ',')
+ throw new InvalidTypeException(
+ String.format(
+ "Cannot parse UDT value from \"%s\", at character %d expecting ',' but got '%c'",
+ value, idx, value.charAt(idx)));
+ ++idx; // skip ','
+
+ idx = ParseUtils.skipSpaces(value, idx);
+ }
+ throw new InvalidTypeException(
+ String.format("Malformed UDT value \"%s\", missing closing '}'", value));
+ }
+
+ /**
+ * Return a new instance of {@code T}.
+ *
+ * @return A new instance of {@code T}.
+ */
+ protected abstract T newInstance();
+
+ /**
+ * Serialize an individual field in an object, as part of serializing the whole object to a CQL
+ * UDT (see {@link #serialize(Object, ProtocolVersion)}).
+ *
+ * @param source The object to read the field from.
+ * @param fieldName The name of the field. Note that if it is case-sensitive or contains special
+ * characters, it will be double-quoted (i.e. the string will contain actual quote
+ * characters, as in {@code "\"foobar\""}).
+ * @param protocolVersion The protocol version to use.
+ * @return The serialized field, or {@code null} if that field should be ignored.
+ */
+ protected abstract ByteBuffer serializeField(
+ T source, String fieldName, ProtocolVersion protocolVersion);
+
+ /**
+ * Deserialize an individual field and set it on an object, as part of deserializing the whole
+ * object from a CQL UDT (see {@link #deserialize(ByteBuffer, ProtocolVersion)}).
+ *
+ * @param input The serialized form of the field.
+ * @param target The object to set the field on.
+ * @param fieldName The name of the field. Note that if it is case-sensitive or contains special
+ * characters, it will be double-quoted (i.e. the string will contain actual quote
+ * characters, as in {@code "\"foobar\""}).
+ * @param protocolVersion The protocol version to use.
+ * @return The target object with the field set. In most cases this should be the same as {@code
+ * target}, but if you're dealing with immutable types you'll need to return a different
+ * instance.
+ */
+ protected abstract T deserializeAndSetField(
+ ByteBuffer input, T target, String fieldName, ProtocolVersion protocolVersion);
+
+ /**
+ * Format an individual field in an object as a CQL literal, as part of formatting the whole
+ * object (see {@link #format(Object)}).
+ *
+ * @param source The object to read the field from.
+ * @param fieldName The name of the field. Note that if it is case-sensitive or contains special
+ * characters, it will be double-quoted (i.e. the string will contain actual quote
+ * characters, as in {@code "\"foobar\""}).
+ * @return The formatted value.
+ */
+ protected abstract String formatField(T source, String fieldName);
+
+ /**
+ * Parse an individual field and set it on an object, as part of parsing the whole object (see
+ * {@link #parse(String)}).
+ *
+ * @param input The String to parse the field from.
+ * @param target The value to write to.
+ * @param fieldName The name of the field. Note that if it is case-sensitive or contains special
+ * characters, it will be double-quoted (i.e. the string will contain actual quote
+ * characters, as in {@code "\"foobar\""}).
+ * @return The target object with the field set. In most cases this should be the same as {@code
+ * target}, but if you're dealing with immutable types you'll need to return a different
+ * instance.
+ */
+ protected abstract T parseAndSetField(String input, T target, String fieldName);
+ }
+
+ /**
+ * This codec maps a CQL {@link UserType} to a {@link UDTValue}.
+ */
+ private static class UDTCodec extends AbstractUDTCodec<UDTValue>
+ {
+
+ private UDTCodec(UserType definition)
+ {
+ super(definition, UDTValue.class);
+ }
+
+ @Override
+ public boolean accepts(Object value)
+ {
+ return super.accepts(value) && ((UDTValue) value).getType().equals(definition);
+ }
+
+ @Override
+ protected UDTValue newInstance()
+ {
+ return definition.newValue();
+ }
+
+ @Override
+ protected ByteBuffer serializeField(
+ UDTValue source, String fieldName, ProtocolVersion protocolVersion)
+ {
+ return source.getBytesUnsafe(fieldName);
+ }
+
+ @Override
+ protected UDTValue deserializeAndSetField(
+ ByteBuffer input, UDTValue target, String fieldName, ProtocolVersion protocolVersion)
+ {
+ return target.setBytesUnsafe(fieldName, input);
+ }
+
+ @Override
+ protected String formatField(UDTValue source, String fieldName)
+ {
+ DataType elementType = definition.getFieldType(fieldName);
+ TypeCodec<Object> codec = definition.getCodecRegistry().codecFor(elementType);
+ return codec.format(source.get(fieldName, codec.getJavaType()));
+ }
+
+ @Override
+ protected UDTValue parseAndSetField(String input, UDTValue target, String fieldName)
+ {
+ DataType elementType = definition.getFieldType(fieldName);
+ TypeCodec<Object> codec = definition.getCodecRegistry().codecFor(elementType);
+ target.set(fieldName, codec.parse(input), codec.getJavaType());
+ return target;
+ }
+ }
+
+ /**
+ * Base class for codecs mapping CQL {@link TupleType tuples} to Java objects. It can serve as a
+ * base class for codecs dealing with direct tuple-to-Pojo mappings.
+ *
+ * @param <T> The Java type that this codec handles.
+ */
+ public abstract static class AbstractTupleCodec<T> extends TypeCodec<T>
+ {
+
+ protected final TupleType definition;
+
+ AbstractTupleCodec(TupleType definition, Class<T> javaClass)
+ {
+ this(definition, TypeToken.of(javaClass));
+ }
+
+ AbstractTupleCodec(TupleType definition, TypeToken<T> javaType)
+ {
+ super(definition, javaType);
+ this.definition = definition;
+ }
+
+ @Override
+ public boolean accepts(DataType cqlType)
+ {
+ // a tuple codec should accept tuple values of a different type,
+ // provided that the latter is contained in this codec's type.
+ return super.accepts(cqlType) && definition.contains((TupleType) cqlType);
+ }
+
+ @Override
+ public ByteBuffer serialize(T value, ProtocolVersion protocolVersion)
+ {
+ if (value == null) return null;
+ int size = 0;
+ int length = definition.getComponentTypes().size();
+ ByteBuffer[] elements = new ByteBuffer[length];
+ for (int i = 0; i < length; i++)
+ {
+ elements[i] = serializeField(value, i, protocolVersion);
+ size += 4 + (elements[i] == null ? 0 : elements[i].remaining());
+ }
+ ByteBuffer result = ByteBuffer.allocate(size);
+ for (ByteBuffer bb : elements)
+ {
+ if (bb == null)
+ {
+ result.putInt(-1);
+ }
+ else
+ {
+ result.putInt(bb.remaining());
+ result.put(bb.duplicate());
+ }
+ }
+ return (ByteBuffer) result.flip();
+ }
+
+ @Override
+ public T deserialize(ByteBuffer bytes, ProtocolVersion protocolVersion)
+ {
+ if (bytes == null) return null;
+ // empty byte buffers will result in empty values
+ try
+ {
+ ByteBuffer input = bytes.duplicate();
+ T value = newInstance();
+ int i = 0;
+ while (input.hasRemaining() && i < definition.getComponentTypes().size())
+ {
+ int n = input.getInt();
+ ByteBuffer element = n < 0 ? null : CodecUtils.readBytes(input, n);
+ value = deserializeAndSetField(element, value, i++, protocolVersion);
+ }
+ return value;
+ }
+ catch (BufferUnderflowException e)
+ {
+ throw new InvalidTypeException("Not enough bytes to deserialize a tuple", e);
+ }
+ }
+
+ @Override
+ public String format(T value)
+ {
+ if (value == null) return "NULL";
+ StringBuilder sb = new StringBuilder("(");
+ int length = definition.getComponentTypes().size();
+ for (int i = 0; i < length; i++)
+ {
+ if (i > 0) sb.append(',');
+ sb.append(formatField(value, i));
+ }
+ sb.append(')');
+ return sb.toString();
+ }
+
+ @Override
+ public T parse(String value)
+ {
+ if (value == null || value.isEmpty() || value.equalsIgnoreCase("NULL")) return null;
+
+ T v = newInstance();
+
+ int idx = ParseUtils.skipSpaces(value, 0);
+ if (value.charAt(idx++) != '(')
+ throw new InvalidTypeException(
+ String.format(
+ "Cannot parse tuple value from \"%s\", at character %d expecting '(' but got '%c'",
+ value, idx, value.charAt(idx)));
+
+ idx = ParseUtils.skipSpaces(value, idx);
+
+ if (value.charAt(idx) == ')') return v;
+
+ int i = 0;
+ while (idx < value.length())
+ {
+ int n;
+ try
+ {
+ n = ParseUtils.skipCQLValue(value, idx);
+ }
+ catch (IllegalArgumentException e)
+ {
+ throw new InvalidTypeException(
+ String.format(
+ "Cannot parse tuple value from \"%s\", invalid CQL value at character %d",
+ value, idx),
+ e);
+ }
+
+ String input = value.substring(idx, n);
+ v = parseAndSetField(input, v, i);
+ idx = n;
+ i += 1;
+
+ idx = ParseUtils.skipSpaces(value, idx);
+ if (value.charAt(idx) == ')') return v;
+ if (value.charAt(idx) != ',')
+ throw new InvalidTypeException(
+ String.format(
+ "Cannot parse tuple value from \"%s\", at character %d expecting ',' but got '%c'",
+ value, idx, value.charAt(idx)));
+ ++idx; // skip ','
+
+ idx = ParseUtils.skipSpaces(value, idx);
+ }
+ throw new InvalidTypeException(
+ String.format("Malformed tuple value \"%s\", missing closing ')'", value));
+ }
+
+ /**
+ * Return a new instance of {@code T}.
+ *
+ * @return A new instance of {@code T}.
+ */
+ protected abstract T newInstance();
+
+ /**
+ * Serialize an individual field in an object, as part of serializing the whole object to a CQL
+ * tuple (see {@link #serialize(Object, ProtocolVersion)}).
+ *
+ * @param source The object to read the field from.
+ * @param index The index of the field.
+ * @param protocolVersion The protocol version to use.
+ * @return The serialized field, or {@code null} if that field should be ignored.
+ */
+ protected abstract ByteBuffer serializeField(
+ T source, int index, ProtocolVersion protocolVersion);
+
+ /**
+ * Deserialize an individual field and set it on an object, as part of deserializing the whole
+ * object from a CQL tuple (see {@link #deserialize(ByteBuffer, ProtocolVersion)}).
+ *
+ * @param input The serialized form of the field.
+ * @param target The object to set the field on.
+ * @param index The index of the field.
+ * @param protocolVersion The protocol version to use.
+ * @return The target object with the field set. In most cases this should be the same as {@code
+ * target}, but if you're dealing with immutable types you'll need to return a different
+ * instance.
+ */
+ protected abstract T deserializeAndSetField(
+ ByteBuffer input, T target, int index, ProtocolVersion protocolVersion);
+
+ /**
+ * Format an individual field in an object as a CQL literal, as part of formatting the whole
+ * object (see {@link #format(Object)}).
+ *
+ * @param source The object to read the field from.
+ * @param index The index of the field.
+ * @return The formatted value.
+ */
+ protected abstract String formatField(T source, int index);
+
+ /**
+ * Parse an individual field and set it on an object, as part of parsing the whole object (see
+ * {@link #parse(String)}).
+ *
+ * @param input The String to parse the field from.
+ * @param target The value to write to.
+ * @param index The index of the field.
+ * @return The target object with the field set. In most cases this should be the same as {@code
+ * target}, but if you're dealing with immutable types you'll need to return a different
+ * instance.
+ */
+ protected abstract T parseAndSetField(String input, T target, int index);
+ }
+
+ /**
+ * This codec maps a CQL {@link TupleType tuple} to a {@link TupleValue}.
+ */
+ private static class TupleCodec extends AbstractTupleCodec<TupleValue>
+ {
+
+ private TupleCodec(TupleType definition)
+ {
+ super(definition, TupleValue.class);
+ }
+
+ @Override
+ public boolean accepts(Object value)
+ {
+ // a tuple codec should accept tuple values of a different type,
+ // provided that the latter is contained in this codec's type.
+ return super.accepts(value) && definition.contains(((TupleValue) value).getType());
+ }
+
+ @Override
+ protected TupleValue newInstance()
+ {
+ return definition.newValue();
+ }
+
+ @Override
+ protected ByteBuffer serializeField(
+ TupleValue source, int index, ProtocolVersion protocolVersion)
+ {
+ if (index >= source.values.length) return null;
+ return source.getBytesUnsafe(index);
+ }
+
+ @Override
+ protected TupleValue deserializeAndSetField(
+ ByteBuffer input, TupleValue target, int index, ProtocolVersion protocolVersion)
+ {
+ if (index >= target.values.length) return target;
+ return target.setBytesUnsafe(index, input);
+ }
+
+ @Override
+ protected String formatField(TupleValue value, int index)
+ {
+ DataType elementType = definition.getComponentTypes().get(index);
+ TypeCodec<Object> codec = definition.getCodecRegistry().codecFor(elementType);
+ return codec.format(value.get(index, codec.getJavaType()));
+ }
+
+ @Override
+ protected TupleValue parseAndSetField(String input, TupleValue target, int index)
+ {
+ DataType elementType = definition.getComponentTypes().get(index);
+ TypeCodec<Object> codec = definition.getCodecRegistry().codecFor(elementType);
+ target.set(index, codec.parse(input), codec.getJavaType());
+ return target;
+ }
+ }
+
+ private static class DurationCodec extends TypeCodec<Duration>
+ {
+
+ private static final DurationCodec instance = new DurationCodec();
+
+ private DurationCodec()
+ {
+ super(DataType.duration(), Duration.class);
+ }
+
+ @Override
+ public ByteBuffer serialize(Duration duration, ProtocolVersion protocolVersion)
+ throws InvalidTypeException
+ {
+ if (duration == null) return null;
+ long months = duration.getMonths();
+ long days = duration.getDays();
+ long nanoseconds = duration.getNanoseconds();
+ int size =
+ VIntCoding.computeVIntSize(months)
+ + VIntCoding.computeVIntSize(days)
+ + VIntCoding.computeVIntSize(nanoseconds);
+ ByteArrayDataOutput out = ByteStreams.newDataOutput(size);
+ try
+ {
+ VIntCoding.writeVInt(months, out);
+ VIntCoding.writeVInt(days, out);
+ VIntCoding.writeVInt(nanoseconds, out);
+ }
+ catch (IOException e)
+ {
+ // cannot happen
+ throw new AssertionError();
+ }
+ return ByteBuffer.wrap(out.toByteArray());
+ }
+
+ @Override
+ public Duration deserialize(ByteBuffer bytes, ProtocolVersion protocolVersion)
+ throws InvalidTypeException
+ {
+ if (bytes == null || bytes.remaining() == 0)
+ {
+ return null;
+ }
+ else
+ {
+ DataInput in = ByteStreams.newDataInput(Bytes.getArray(bytes));
+ try
+ {
+ int months = (int) VIntCoding.readVInt(in);
+ int days = (int) VIntCoding.readVInt(in);
+ long nanoseconds = VIntCoding.readVInt(in);
+ return Duration.newInstance(months, days, nanoseconds);
+ }
+ catch (IOException e)
+ {
+ // cannot happen
+ throw new AssertionError();
+ }
+ }
+ }
+
+ @Override
+ public Duration parse(String value) throws InvalidTypeException
+ {
+ if (value == null || value.isEmpty() || value.equalsIgnoreCase("NULL")) return null;
+ return Duration.from(value);
+ }
+
+ @Override
+ public String format(Duration value) throws InvalidTypeException
+ {
+ if (value == null) return "NULL";
+ return value.toString();
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/cql3/functions/types/TypeTokens.java b/src/java/org/apache/cassandra/cql3/functions/types/TypeTokens.java
new file mode 100644
index 0000000..aba7466
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/functions/types/TypeTokens.java
@@ -0,0 +1,157 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.functions.types;
+
+import java.util.*;
+
+import com.google.common.reflect.TypeParameter;
+import com.google.common.reflect.TypeToken;
+
+/**
+ * Utility methods to create {@code TypeToken} instances.
+ */
+public final class TypeTokens
+{
+ private TypeTokens()
+ {
+ }
+
+ /**
+ * Create a {@link TypeToken} that represents a {@link List} whose elements are of the given type.
+ *
+ * @param eltType The list element type.
+ * @param <T> The list element type.
+ * @return A {@link TypeToken} that represents a {@link List} whose elements are of the given
+ * type.
+ */
+ public static <T> TypeToken<List<T>> listOf(Class<T> eltType)
+ {
+ // @formatter:off
+ return new TypeToken<List<T>>()
+ {
+ }.where(new TypeParameter<T>()
+ {
+ }, eltType);
+ // @formatter:on
+ }
+
+ /**
+ * Create a {@link TypeToken} that represents a {@link List} whose elements are of the given type.
+ *
+ * @param eltType The list element type.
+ * @param <T> The list element type.
+ * @return A {@link TypeToken} that represents a {@link List} whose elements are of the given
+ * type.
+ */
+ public static <T> TypeToken<List<T>> listOf(TypeToken<T> eltType)
+ {
+ // @formatter:off
+ return new TypeToken<List<T>>()
+ {
+ }.where(new TypeParameter<T>()
+ {
+ }, eltType);
+ // @formatter:on
+ }
+
+ /**
+ * Create a {@link TypeToken} that represents a {@link Set} whose elements are of the given type.
+ *
+ * @param eltType The set element type.
+ * @param <T> The set element type.
+ * @return A {@link TypeToken} that represents a {@link Set} whose elements are of the given type.
+ */
+ public static <T> TypeToken<Set<T>> setOf(Class<T> eltType)
+ {
+ // @formatter:off
+ return new TypeToken<Set<T>>()
+ {
+ }.where(new TypeParameter<T>()
+ {
+ }, eltType);
+ // @formatter:on
+ }
+
+ /**
+ * Create a {@link TypeToken} that represents a {@link Set} whose elements are of the given type.
+ *
+ * @param eltType The set element type.
+ * @param <T> The set element type.
+ * @return A {@link TypeToken} that represents a {@link Set} whose elements are of the given type.
+ */
+ public static <T> TypeToken<Set<T>> setOf(TypeToken<T> eltType)
+ {
+ // @formatter:off
+ return new TypeToken<Set<T>>()
+ {
+ }.where(new TypeParameter<T>()
+ {
+ }, eltType);
+ // @formatter:on
+ }
+
+ /**
+ * Create a {@link TypeToken} that represents a {@link Map} whose keys and values are of the given
+ * key and value types.
+ *
+ * @param keyType The map key type.
+ * @param valueType The map value type
+ * @param <K> The map key type.
+ * @param <V> The map value type
+ * @return A {@link TypeToken} that represents a {@link Map} whose keys and values are of the
+ * given key and value types
+ */
+ public static <K, V> TypeToken<Map<K, V>> mapOf(Class<K> keyType, Class<V> valueType)
+ {
+ // @formatter:off
+ return new TypeToken<Map<K, V>>()
+ {
+ }.where(new TypeParameter<K>()
+ {
+ }, keyType)
+ .where(new TypeParameter<V>()
+ {
+ }, valueType);
+ // @formatter:on
+ }
+
+ /**
+ * Create a {@link TypeToken} that represents a {@link Map} whose keys and values are of the given
+ * key and value types.
+ *
+ * @param keyType The map key type.
+ * @param valueType The map value type
+ * @param <K> The map key type.
+ * @param <V> The map value type
+ * @return A {@link TypeToken} that represents a {@link Map} whose keys and values are of the
+ * given key and value types
+ */
+ public static <K, V> TypeToken<Map<K, V>> mapOf(TypeToken<K> keyType, TypeToken<V> valueType)
+ {
+ // @formatter:off
+ return new TypeToken<Map<K, V>>()
+ {
+ }.where(new TypeParameter<K>()
+ {
+ }, keyType)
+ .where(new TypeParameter<V>()
+ {
+ }, valueType);
+ // @formatter:on
+ }
+}
diff --git a/src/java/org/apache/cassandra/cql3/functions/types/UDTValue.java b/src/java/org/apache/cassandra/cql3/functions/types/UDTValue.java
new file mode 100644
index 0000000..3b27d67
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/functions/types/UDTValue.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.functions.types;
+
+/**
+ * A value for a User Defined Type.
+ */
+public class UDTValue extends AbstractData<UDTValue>
+{
+
+ private final UserType definition;
+
+ UDTValue(UserType definition)
+ {
+ super(definition.getProtocolVersion(), definition.size());
+ this.definition = definition;
+ }
+
+ @Override
+ protected DataType getType(int i)
+ {
+ return definition.byIdx[i].getType();
+ }
+
+ @Override
+ protected String getName(int i)
+ {
+ return definition.byIdx[i].getName();
+ }
+
+ @Override
+ protected CodecRegistry getCodecRegistry()
+ {
+ return definition.getCodecRegistry();
+ }
+
+ @Override
+ protected int[] getAllIndexesOf(String name)
+ {
+ int[] indexes = definition.byName.get(Metadata.handleId(name));
+ if (indexes == null)
+ throw new IllegalArgumentException(name + " is not a field defined in this UDT");
+ return indexes;
+ }
+
+ /**
+ * The UDT this is a value of.
+ *
+ * @return the UDT this is a value of.
+ */
+ public UserType getType()
+ {
+ return definition;
+ }
+
+ @Override
+ public boolean equals(Object o)
+ {
+ if (!(o instanceof UDTValue)) return false;
+
+ UDTValue that = (UDTValue) o;
+ if (!definition.equals(that.definition)) return false;
+
+ return super.equals(o);
+ }
+
+ @Override
+ public int hashCode()
+ {
+ return super.hashCode();
+ }
+
+ @Override
+ public String toString()
+ {
+ StringBuilder sb = new StringBuilder();
+ TypeCodec<Object> codec = getCodecRegistry().codecFor(definition);
+ sb.append(codec.format(this));
+ return sb.toString();
+ }
+}
diff --git a/src/java/org/apache/cassandra/cql3/functions/types/UserType.java b/src/java/org/apache/cassandra/cql3/functions/types/UserType.java
new file mode 100644
index 0000000..1a24e22
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/functions/types/UserType.java
@@ -0,0 +1,318 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.functions.types;
+
+import java.util.*;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Iterators;
+
+import org.apache.cassandra.transport.ProtocolVersion;
+
+/**
+ * A User Defined Type (UDT).
+ *
+ * <p>A UDT is a essentially a named collection of fields (with a name and a type).
+ */
+public class UserType extends DataType implements Iterable<UserType.Field>
+{
+
+ private final String keyspace;
+ private final String typeName;
+ private final boolean frozen;
+ private final ProtocolVersion protocolVersion;
+
+ // can be null, if this object is being constructed from a response message
+ // see Responses.Result.Rows.Metadata.decode()
+ private final CodecRegistry codecRegistry;
+
+ // Note that we don't expose the order of fields, from an API perspective this is a map
+ // of String->Field, but internally we care about the order because the serialization format
+ // of UDT expects a particular order.
+ final Field[] byIdx;
+ // For a given name, we can only have one field with that name, so we don't need a int[] in
+ // practice. However, storing one element arrays save allocations in UDTValue.getAllIndexesOf
+ // implementation.
+ final Map<String, int[]> byName;
+
+ private UserType(
+ Name name,
+ String keyspace,
+ String typeName,
+ boolean frozen,
+ ProtocolVersion protocolVersion,
+ CodecRegistry codecRegistry,
+ Field[] byIdx,
+ Map<String, int[]> byName)
+ {
+ super(name);
+ this.keyspace = keyspace;
+ this.typeName = typeName;
+ this.frozen = frozen;
+ this.protocolVersion = protocolVersion;
+ this.codecRegistry = codecRegistry;
+ this.byIdx = byIdx;
+ this.byName = byName;
+ }
+
+ UserType(
+ String keyspace,
+ String typeName,
+ boolean frozen,
+ Collection<Field> fields,
+ ProtocolVersion protocolVersion,
+ CodecRegistry codecRegistry)
+ {
+ this(
+ DataType.Name.UDT,
+ keyspace,
+ typeName,
+ frozen,
+ protocolVersion,
+ codecRegistry,
+ fields.toArray(new Field[fields.size()]),
+ mapByName(fields));
+ }
+
+ private static ImmutableMap<String, int[]> mapByName(Collection<Field> fields)
+ {
+ ImmutableMap.Builder<String, int[]> builder = new ImmutableMap.Builder<>();
+ int i = 0;
+ for (Field field : fields)
+ {
+ builder.put(field.getName(), new int[]{ i });
+ i += 1;
+ }
+ return builder.build();
+ }
+
+ /**
+ * Returns a new empty value for this user type definition.
+ *
+ * @return an empty value for this user type definition.
+ */
+ public UDTValue newValue()
+ {
+ return new UDTValue(this);
+ }
+
+ /**
+ * The name of the keyspace this UDT is part of.
+ *
+ * @return the name of the keyspace this UDT is part of.
+ */
+ public String getKeyspace()
+ {
+ return keyspace;
+ }
+
+ /**
+ * The name of this user type.
+ *
+ * @return the name of this user type.
+ */
+ public String getTypeName()
+ {
+ return typeName;
+ }
+
+ /**
+ * Returns the number of fields in this UDT.
+ *
+ * @return the number of fields in this UDT.
+ */
+ public int size()
+ {
+ return byIdx.length;
+ }
+
+ /**
+ * Returns whether this UDT contains a given field.
+ *
+ * @param name the name to check. Note that {@code name} obey the usual CQL identifier rules: it
+ * should be quoted if it denotes a case sensitive identifier (you can use {@link
+ * Metadata#quote} for the quoting).
+ * @return {@code true} if this UDT contains a field named {@code name}, {@code false} otherwise.
+ */
+ public boolean contains(String name)
+ {
+ return byName.containsKey(Metadata.handleId(name));
+ }
+
+ /**
+ * Returns an iterator over the fields of this UDT.
+ *
+ * @return an iterator over the fields of this UDT.
+ */
+ @Override
+ public Iterator<Field> iterator()
+ {
+ return Iterators.forArray(byIdx);
+ }
+
+ /**
+ * Returns the type of a given field.
+ *
+ * @param name the name of the field. Note that {@code name} obey the usual CQL identifier rules:
+ * it should be quoted if it denotes a case sensitive identifier (you can use {@link
+ * Metadata#quote} for the quoting).
+ * @return the type of field {@code name} if this UDT has a field of this name, {@code null}
+ * otherwise.
+ * @throws IllegalArgumentException if {@code name} is not a field of this UDT definition.
+ */
+ DataType getFieldType(String name)
+ {
+ int[] idx = byName.get(Metadata.handleId(name));
+ if (idx == null)
+ throw new IllegalArgumentException(name + " is not a field defined in this definition");
+
+ return byIdx[idx[0]].getType();
+ }
+
+ @Override
+ public boolean isFrozen()
+ {
+ return frozen;
+ }
+
+ public UserType copy(boolean newFrozen)
+ {
+ if (newFrozen == frozen)
+ {
+ return this;
+ }
+ else
+ {
+ return new UserType(
+ name, keyspace, typeName, newFrozen, protocolVersion, codecRegistry, byIdx, byName);
+ }
+ }
+
+ @Override
+ public int hashCode()
+ {
+ int result = name.hashCode();
+ result = 31 * result + keyspace.hashCode();
+ result = 31 * result + typeName.hashCode();
+ result = 31 * result + Arrays.hashCode(byIdx);
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object o)
+ {
+ if (!(o instanceof UserType)) return false;
+
+ UserType other = (UserType) o;
+
+ // Note: we don't test byName because it's redundant with byIdx in practice,
+ // but also because the map holds 'int[]' which don't have proper equal.
+ return name.equals(other.name)
+ && keyspace.equals(other.keyspace)
+ && typeName.equals(other.typeName)
+ && Arrays.equals(byIdx, other.byIdx);
+ }
+
+ /**
+ * Return the protocol version that has been used to deserialize this UDT, or that will be used to
+ * serialize it. In most cases this should be the version currently in use by the cluster instance
+ * that this UDT belongs to, as reported by {@code ProtocolOptions#getProtocolVersion()}.
+ *
+ * @return the protocol version that has been used to deserialize this UDT, or that will be used
+ * to serialize it.
+ */
+ ProtocolVersion getProtocolVersion()
+ {
+ return protocolVersion;
+ }
+
+ CodecRegistry getCodecRegistry()
+ {
+ return codecRegistry;
+ }
+
+ @Override
+ public String toString()
+ {
+ String str =
+ Metadata.quoteIfNecessary(getKeyspace()) + '.' + Metadata.quoteIfNecessary(getTypeName());
+ return isFrozen() ? "frozen<" + str + '>' : str;
+ }
+
+ @Override
+ public String asFunctionParameterString()
+ {
+ return Metadata.quoteIfNecessary(getTypeName());
+ }
+
+ /**
+ * A UDT field.
+ */
+ public static class Field
+ {
+ private final String name;
+ private final DataType type;
+
+ Field(String name, DataType type)
+ {
+ this.name = name;
+ this.type = type;
+ }
+
+ /**
+ * Returns the name of the field.
+ *
+ * @return the name of the field.
+ */
+ public String getName()
+ {
+ return name;
+ }
+
+ /**
+ * Returns the type of the field.
+ *
+ * @return the type of the field.
+ */
+ public DataType getType()
+ {
+ return type;
+ }
+
+ @Override
+ public final int hashCode()
+ {
+ return Arrays.hashCode(new Object[]{ name, type });
+ }
+
+ @Override
+ public final boolean equals(Object o)
+ {
+ if (!(o instanceof Field)) return false;
+
+ Field other = (Field) o;
+ return name.equals(other.name) && type.equals(other.type);
+ }
+
+ @Override
+ public String toString()
+ {
+ return Metadata.quoteIfNecessary(name) + ' ' + type;
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/cql3/functions/types/exceptions/CodecNotFoundException.java b/src/java/org/apache/cassandra/cql3/functions/types/exceptions/CodecNotFoundException.java
new file mode 100644
index 0000000..e71a916
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/functions/types/exceptions/CodecNotFoundException.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.functions.types.exceptions;
+
+/**
+ * Thrown when a suitable {@link org.apache.cassandra.cql3.functions.types.TypeCodec} cannot be found by {@link
+ * org.apache.cassandra.cql3.functions.types.CodecRegistry} instances.
+ */
+@SuppressWarnings("serial")
+public class CodecNotFoundException extends DriverException
+{
+
+ public CodecNotFoundException(String msg)
+ {
+ this(msg, null);
+ }
+
+ public CodecNotFoundException(Throwable cause)
+ {
+ this(null, cause);
+ }
+
+ private CodecNotFoundException(String msg, Throwable cause)
+ {
+ super(msg, cause);
+ }
+}
diff --git a/src/java/org/apache/cassandra/cql3/functions/types/exceptions/DriverException.java b/src/java/org/apache/cassandra/cql3/functions/types/exceptions/DriverException.java
new file mode 100644
index 0000000..597e795
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/functions/types/exceptions/DriverException.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.functions.types.exceptions;
+
+/**
+ * Top level class for exceptions thrown by the driver.
+ */
+public class DriverException extends RuntimeException
+{
+
+ private static final long serialVersionUID = 0;
+
+ DriverException(String message)
+ {
+ super(message);
+ }
+
+ DriverException(String message, Throwable cause)
+ {
+ super(message, cause);
+ }
+}
diff --git a/src/java/org/apache/cassandra/cql3/functions/types/exceptions/DriverInternalError.java b/src/java/org/apache/cassandra/cql3/functions/types/exceptions/DriverInternalError.java
new file mode 100644
index 0000000..95c3bda
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/functions/types/exceptions/DriverInternalError.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.functions.types.exceptions;
+
+/**
+ * An unexpected error happened internally.
+ *
+ * <p>This should never be raised and indicates a bug (either in the driver or in Cassandra).
+ */
+public class DriverInternalError extends DriverException
+{
+
+ private static final long serialVersionUID = 0;
+
+ public DriverInternalError(String message)
+ {
+ super(message);
+ }
+
+ public DriverInternalError(String message, Throwable cause)
+ {
+ super(message, cause);
+ }
+}
diff --git a/src/java/org/apache/cassandra/cql3/functions/types/exceptions/InvalidTypeException.java b/src/java/org/apache/cassandra/cql3/functions/types/exceptions/InvalidTypeException.java
new file mode 100644
index 0000000..b536c24
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/functions/types/exceptions/InvalidTypeException.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.functions.types.exceptions;
+
+/**
+ * Thrown when a {@link org.apache.cassandra.cql3.functions.types.TypeCodec} is unable to perform the requested
+ * operation (serialization, deserialization, parsing or formatting) because the object or the byte
+ * buffer content being processed does not comply with the expected Java and/or CQL type.
+ */
+public class InvalidTypeException extends DriverException
+{
+
+ private static final long serialVersionUID = 0;
+
+ public InvalidTypeException(String msg)
+ {
+ super(msg);
+ }
+
+ public InvalidTypeException(String msg, Throwable cause)
+ {
+ super(msg, cause);
+ }
+}
diff --git a/src/java/org/apache/cassandra/cql3/functions/types/package-info.java b/src/java/org/apache/cassandra/cql3/functions/types/package-info.java
new file mode 100644
index 0000000..affa60d
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/functions/types/package-info.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Contains pieces of the Java Driver that are needed to handle data types
+ * in C* User-Defined-Functions/Aggregates.
+ * <p>
+ * The code has been copied from the Java Driver source but not especially
+ * adopted. Existing UDFs may rely on certain classes and interfaces, so
+ * changing interfaces and classes in this package must be performed very
+ * carefully to not break those existing UDFs.
+ * <p>
+ * Some of the functionality in this package is probably duplicated,
+ * especially the type parsing and value formatting/parsing code is.
+ * <p>
+ * But referencing code outside this package can break UDFs as the UDF
+ * sandbox can prevent the use of code outside this package.
+ * <p>
+ * Comments in the classes in this package have been left as they were in
+ * the Java Driver.
+ */
+package org.apache.cassandra.cql3.functions.types;
diff --git a/src/java/org/apache/cassandra/cql3/functions/types/utils/Bytes.java b/src/java/org/apache/cassandra/cql3/functions/types/utils/Bytes.java
new file mode 100644
index 0000000..c3d6ccd
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/functions/types/utils/Bytes.java
@@ -0,0 +1,219 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.functions.types.utils;
+
+import java.lang.reflect.Constructor;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+
+/**
+ * Simple utility methods to make working with bytes (blob) easier.
+ */
+public final class Bytes
+{
+
+ private Bytes()
+ {
+ }
+
+ private static final byte[] charToByte = new byte[256];
+ private static final char[] byteToChar = new char[16];
+
+ static
+ {
+ for (char c = 0; c < charToByte.length; ++c)
+ {
+ if (c >= '0' && c <= '9') charToByte[c] = (byte) (c - '0');
+ else if (c >= 'A' && c <= 'F') charToByte[c] = (byte) (c - 'A' + 10);
+ else if (c >= 'a' && c <= 'f') charToByte[c] = (byte) (c - 'a' + 10);
+ else charToByte[c] = (byte) -1;
+ }
+
+ for (int i = 0; i < 16; ++i)
+ {
+ byteToChar[i] = Integer.toHexString(i).charAt(0);
+ }
+ }
+
+ /*
+ * We use reflexion to get access to a String protected constructor
+ * (if available) so we can build avoid copy when creating hex strings.
+ * That's stolen from Cassandra's code.
+ */
+ private static final Constructor<String> stringConstructor;
+
+ static
+ {
+ Constructor<String> c;
+ try
+ {
+ c = String.class.getDeclaredConstructor(int.class, int.class, char[].class);
+ c.setAccessible(true);
+ }
+ catch (Exception e)
+ {
+ c = null;
+ }
+ stringConstructor = c;
+ }
+
+ private static String wrapCharArray(char[] c)
+ {
+ if (c == null) return null;
+
+ String s = null;
+ if (stringConstructor != null)
+ {
+ try
+ {
+ s = stringConstructor.newInstance(0, c.length, c);
+ }
+ catch (Exception e)
+ {
+ // Swallowing as we'll just use a copying constructor
+ }
+ }
+ return s == null ? new String(c) : s;
+ }
+
+ /**
+ * Converts a blob to its CQL hex string representation.
+ *
+ * <p>A CQL blob string representation consist of the hexadecimal representation of the blob bytes
+ * prefixed by "0x".
+ *
+ * @param bytes the blob/bytes to convert to a string.
+ * @return the CQL string representation of {@code bytes}. If {@code bytes} is {@code null}, this
+ * method returns {@code null}.
+ */
+ public static String toHexString(ByteBuffer bytes)
+ {
+ if (bytes == null) return null;
+
+ if (bytes.remaining() == 0) return "0x";
+
+ char[] array = new char[2 * (bytes.remaining() + 1)];
+ array[0] = '0';
+ array[1] = 'x';
+ return toRawHexString(bytes, array, 2);
+ }
+
+ /**
+ * Converts a blob to its CQL hex string representation.
+ *
+ * <p>A CQL blob string representation consist of the hexadecimal representation of the blob bytes
+ * prefixed by "0x".
+ *
+ * @param byteArray the blob/bytes array to convert to a string.
+ * @return the CQL string representation of {@code bytes}. If {@code bytes} is {@code null}, this
+ * method returns {@code null}.
+ */
+ public static String toHexString(byte[] byteArray)
+ {
+ return toHexString(ByteBuffer.wrap(byteArray));
+ }
+
+ /**
+ * Parse an hex string representing a CQL blob.
+ *
+ * <p>The input should be a valid representation of a CQL blob, i.e. it must start by "0x"
+ * followed by the hexadecimal representation of the blob bytes.
+ *
+ * @param str the CQL blob string representation to parse.
+ * @return the bytes corresponding to {@code str}. If {@code str} is {@code null}, this method
+ * returns {@code null}.
+ * @throws IllegalArgumentException if {@code str} is not a valid CQL blob string.
+ */
+ public static ByteBuffer fromHexString(String str)
+ {
+ if ((str.length() & 1) == 1)
+ throw new IllegalArgumentException(
+ "A CQL blob string must have an even length (since one byte is always 2 hexadecimal character)");
+
+ if (str.charAt(0) != '0' || str.charAt(1) != 'x')
+ throw new IllegalArgumentException("A CQL blob string must start with \"0x\"");
+
+ return ByteBuffer.wrap(fromRawHexString(str, 2));
+ }
+
+ /**
+ * Extract the content of the provided {@code ByteBuffer} as a byte array.
+ *
+ * <p>This method work with any type of {@code ByteBuffer} (direct and non-direct ones), but when
+ * the {@code ByteBuffer} is backed by an array, this method will try to avoid copy when possible.
+ * As a consequence, changes to the returned byte array may or may not reflect into the initial
+ * {@code ByteBuffer}.
+ *
+ * @param bytes the buffer whose content to extract.
+ * @return a byte array with the content of {@code bytes}. That array may be the array backing
+ * {@code bytes} if this can avoid a copy.
+ */
+ public static byte[] getArray(ByteBuffer bytes)
+ {
+ int length = bytes.remaining();
+
+ if (bytes.hasArray())
+ {
+ int boff = bytes.arrayOffset() + bytes.position();
+ if (boff == 0 && length == bytes.array().length) return bytes.array();
+ else return Arrays.copyOfRange(bytes.array(), boff, boff + length);
+ }
+ // else, DirectByteBuffer.get() is the fastest route
+ byte[] array = new byte[length];
+ bytes.duplicate().get(array);
+ return array;
+ }
+
+ private static String toRawHexString(ByteBuffer bytes, char[] array, int offset)
+ {
+ int size = bytes.remaining();
+ int bytesOffset = bytes.position();
+ assert array.length >= offset + 2 * size;
+ for (int i = 0; i < size; i++)
+ {
+ int bint = bytes.get(i + bytesOffset);
+ array[offset + i * 2] = byteToChar[(bint & 0xf0) >> 4];
+ array[offset + 1 + i * 2] = byteToChar[bint & 0x0f];
+ }
+ return wrapCharArray(array);
+ }
+
+ /**
+ * Converts a CQL hex string representation into a byte array.
+ *
+ * <p>A CQL blob string representation consist of the hexadecimal representation of the blob
+ * bytes.
+ *
+ * @param str the string converted in hex representation.
+ * @param strOffset he offset for starting the string conversion
+ * @return the byte array which the String was representing.
+ */
+ private static byte[] fromRawHexString(String str, int strOffset)
+ {
+ byte[] bytes = new byte[(str.length() - strOffset) / 2];
+ for (int i = 0; i < bytes.length; i++)
+ {
+ byte halfByte1 = charToByte[str.charAt(strOffset + i * 2)];
+ byte halfByte2 = charToByte[str.charAt(strOffset + i * 2 + 1)];
+ if (halfByte1 == -1 || halfByte2 == -1)
+ throw new IllegalArgumentException("Non-hex characters in " + str);
+ bytes[i] = (byte) ((halfByte1 << 4) | halfByte2);
+ }
+ return bytes;
+ }
+}
diff --git a/src/java/org/apache/cassandra/cql3/restrictions/ClusteringColumnRestrictions.java b/src/java/org/apache/cassandra/cql3/restrictions/ClusteringColumnRestrictions.java
index a8cc6bd..265d354 100644
--- a/src/java/org/apache/cassandra/cql3/restrictions/ClusteringColumnRestrictions.java
+++ b/src/java/org/apache/cassandra/cql3/restrictions/ClusteringColumnRestrictions.java
@@ -19,14 +19,14 @@
import java.util.*;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.cql3.QueryOptions;
import org.apache.cassandra.cql3.statements.Bound;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.filter.RowFilter;
import org.apache.cassandra.exceptions.InvalidRequestException;
-import org.apache.cassandra.index.SecondaryIndexManager;
+import org.apache.cassandra.index.IndexRegistry;
import org.apache.cassandra.utils.btree.BTreeSet;
import static org.apache.cassandra.cql3.statements.RequestValidations.checkFalse;
@@ -47,14 +47,14 @@
*/
private final boolean allowFiltering;
- public ClusteringColumnRestrictions(CFMetaData cfm)
+ public ClusteringColumnRestrictions(TableMetadata table)
{
- this(cfm, false);
+ this(table, false);
}
- public ClusteringColumnRestrictions(CFMetaData cfm, boolean allowFiltering)
+ public ClusteringColumnRestrictions(TableMetadata table, boolean allowFiltering)
{
- this(cfm.comparator, new RestrictionSet(), allowFiltering);
+ this(table.comparator, new RestrictionSet(), allowFiltering);
}
private ClusteringColumnRestrictions(ClusteringComparator comparator,
@@ -74,8 +74,8 @@
if (!isEmpty() && !allowFiltering)
{
SingleRestriction lastRestriction = restrictions.lastRestriction();
- ColumnDefinition lastRestrictionStart = lastRestriction.getFirstColumn();
- ColumnDefinition newRestrictionStart = restriction.getFirstColumn();
+ ColumnMetadata lastRestrictionStart = lastRestriction.getFirstColumn();
+ ColumnMetadata newRestrictionStart = restriction.getFirstColumn();
checkFalse(lastRestriction.isSlice() && newRestrictionStart.position() > lastRestrictionStart.position(),
"Clustering column \"%s\" cannot be restricted (preceding column \"%s\" is restricted by a non-EQ relation)",
@@ -199,7 +199,7 @@
@Override
public void addRowFilterTo(RowFilter filter,
- SecondaryIndexManager indexManager,
+ IndexRegistry indexRegistry,
QueryOptions options) throws InvalidRequestException
{
int position = 0;
@@ -207,9 +207,9 @@
for (SingleRestriction restriction : restrictions)
{
// We ignore all the clustering columns that can be handled by slices.
- if (handleInFilter(restriction, position) || restriction.hasSupportingIndex(indexManager))
+ if (handleInFilter(restriction, position) || restriction.hasSupportingIndex(indexRegistry))
{
- restriction.addRowFilterTo(filter, indexManager, options);
+ restriction.addRowFilterTo(filter, indexRegistry, options);
continue;
}
@@ -223,8 +223,4 @@
return restriction.isContains() || restriction.isLIKE() || index != restriction.getFirstColumn().position();
}
- public Iterator<SingleRestriction> iterator()
- {
- return restrictions.iterator();
- }
}
diff --git a/src/java/org/apache/cassandra/cql3/restrictions/CustomIndexExpression.java b/src/java/org/apache/cassandra/cql3/restrictions/CustomIndexExpression.java
index eb91928..539715c 100644
--- a/src/java/org/apache/cassandra/cql3/restrictions/CustomIndexExpression.java
+++ b/src/java/org/apache/cassandra/cql3/restrictions/CustomIndexExpression.java
@@ -15,44 +15,68 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.cassandra.cql3.restrictions;
-import org.apache.cassandra.config.CFMetaData;
+import java.util.Objects;
+
import org.apache.cassandra.cql3.*;
import org.apache.cassandra.db.filter.RowFilter;
import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.schema.TableMetadata;
public class CustomIndexExpression
{
private final ColumnIdentifier valueColId = new ColumnIdentifier("custom index expression", false);
- public final IndexName targetIndex;
+ public final QualifiedName targetIndex;
public final Term.Raw valueRaw;
private Term value;
- public CustomIndexExpression(IndexName targetIndex, Term.Raw value)
+ public CustomIndexExpression(QualifiedName targetIndex, Term.Raw value)
{
this.targetIndex = targetIndex;
this.valueRaw = value;
}
- public void prepareValue(CFMetaData cfm, AbstractType<?> expressionType, VariableSpecifications boundNames)
+ public void prepareValue(TableMetadata table, AbstractType<?> expressionType, VariableSpecifications boundNames)
{
- ColumnSpecification spec = new ColumnSpecification(cfm.ksName, cfm.ksName, valueColId, expressionType);
- value = valueRaw.prepare(cfm.ksName, spec);
+ ColumnSpecification spec = new ColumnSpecification(table.keyspace, table.keyspace, valueColId, expressionType);
+ value = valueRaw.prepare(table.keyspace, spec);
value.collectMarkerSpecification(boundNames);
}
- public void addToRowFilter(RowFilter filter,
- CFMetaData cfm,
- QueryOptions options)
+ public void addToRowFilter(RowFilter filter, TableMetadata table, QueryOptions options)
{
- filter.addCustomIndexExpression(cfm,
- cfm.getIndexes()
- .get(targetIndex.getIdx())
- .orElseThrow(() -> IndexRestrictions.indexNotFound(targetIndex, cfm)),
+ filter.addCustomIndexExpression(table,
+ table.indexes
+ .get(targetIndex.getName())
+ .orElseThrow(() -> IndexRestrictions.indexNotFound(targetIndex, table)),
value.bindAndGet(options));
}
+
+ @Override
+ public String toString()
+ {
+ return String.format("expr(%s,%s)", targetIndex, valueRaw);
+ }
+
+ @Override
+ public int hashCode()
+ {
+ return Objects.hash(targetIndex, valueRaw);
+ }
+
+ @Override
+ public boolean equals(Object o)
+ {
+ if (this == o)
+ return true;
+
+ if (!(o instanceof CustomIndexExpression))
+ return false;
+
+ CustomIndexExpression cie = (CustomIndexExpression) o;
+ return targetIndex.equals(cie.targetIndex) && valueRaw.equals(cie.valueRaw);
+ }
}
diff --git a/src/java/org/apache/cassandra/cql3/restrictions/IndexRestrictions.java b/src/java/org/apache/cassandra/cql3/restrictions/IndexRestrictions.java
index c7f6b5f..fd89d1b 100644
--- a/src/java/org/apache/cassandra/cql3/restrictions/IndexRestrictions.java
+++ b/src/java/org/apache/cassandra/cql3/restrictions/IndexRestrictions.java
@@ -21,14 +21,16 @@
import java.util.ArrayList;
import java.util.List;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.cql3.IndexName;
+import org.apache.cassandra.cql3.QualifiedName;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.commons.lang3.builder.ToStringBuilder;
+import org.apache.commons.lang3.builder.ToStringStyle;
public class IndexRestrictions
{
- public static final String INDEX_NOT_FOUND = "Invalid index expression, index %s not found for %s.%s";
- public static final String INVALID_INDEX = "Target index %s cannot be used to query %s.%s";
+ public static final String INDEX_NOT_FOUND = "Invalid index expression, index %s not found for %s";
+ public static final String INVALID_INDEX = "Target index %s cannot be used to query %s";
public static final String CUSTOM_EXPRESSION_NOT_SUPPORTED = "Index %s does not support custom expressions";
public static final String NON_CUSTOM_INDEX_IN_EXPRESSION = "Only CUSTOM indexes may be used in custom index expressions, %s is not valid";
public static final String MULTIPLE_EXPRESSIONS = "Multiple custom index expressions in a single query are not supported";
@@ -61,23 +63,29 @@
return customExpressions;
}
- static InvalidRequestException invalidIndex(IndexName indexName, CFMetaData cfm)
+ static InvalidRequestException invalidIndex(QualifiedName indexName, TableMetadata table)
{
- return new InvalidRequestException(String.format(INVALID_INDEX, indexName.getIdx(), cfm.ksName, cfm.cfName));
+ return new InvalidRequestException(String.format(INVALID_INDEX, indexName.getName(), table));
}
- static InvalidRequestException indexNotFound(IndexName indexName, CFMetaData cfm)
+ static InvalidRequestException indexNotFound(QualifiedName indexName, TableMetadata table)
{
- return new InvalidRequestException(String.format(INDEX_NOT_FOUND,indexName.getIdx(), cfm.ksName, cfm.cfName));
+ return new InvalidRequestException(String.format(INDEX_NOT_FOUND, indexName.getName(), table));
}
- static InvalidRequestException nonCustomIndexInExpression(IndexName indexName)
+ static InvalidRequestException nonCustomIndexInExpression(QualifiedName indexName)
{
- return new InvalidRequestException(String.format(NON_CUSTOM_INDEX_IN_EXPRESSION, indexName.getIdx()));
+ return new InvalidRequestException(String.format(NON_CUSTOM_INDEX_IN_EXPRESSION, indexName.getName()));
}
- static InvalidRequestException customExpressionNotSupported(IndexName indexName)
+ static InvalidRequestException customExpressionNotSupported(QualifiedName indexName)
{
- return new InvalidRequestException(String.format(CUSTOM_EXPRESSION_NOT_SUPPORTED, indexName.getIdx()));
+ return new InvalidRequestException(String.format(CUSTOM_EXPRESSION_NOT_SUPPORTED, indexName.getName()));
+ }
+
+ @Override
+ public String toString()
+ {
+ return ToStringBuilder.reflectionToString(this, ToStringStyle.SHORT_PREFIX_STYLE);
}
}
diff --git a/src/java/org/apache/cassandra/cql3/restrictions/MultiColumnRestriction.java b/src/java/org/apache/cassandra/cql3/restrictions/MultiColumnRestriction.java
index b0cbdff..4c6ce2f 100644
--- a/src/java/org/apache/cassandra/cql3/restrictions/MultiColumnRestriction.java
+++ b/src/java/org/apache/cassandra/cql3/restrictions/MultiColumnRestriction.java
@@ -20,7 +20,7 @@
import java.nio.ByteBuffer;
import java.util.*;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.cql3.*;
import org.apache.cassandra.cql3.Term.Terminal;
import org.apache.cassandra.cql3.functions.Function;
@@ -28,7 +28,9 @@
import org.apache.cassandra.db.MultiCBuilder;
import org.apache.cassandra.db.filter.RowFilter;
import org.apache.cassandra.index.Index;
-import org.apache.cassandra.index.SecondaryIndexManager;
+import org.apache.cassandra.index.IndexRegistry;
+import org.apache.commons.lang3.builder.ToStringBuilder;
+import org.apache.commons.lang3.builder.ToStringStyle;
import static org.apache.cassandra.cql3.statements.RequestValidations.checkFalse;
import static org.apache.cassandra.cql3.statements.RequestValidations.checkNotNull;
@@ -40,9 +42,9 @@
/**
* The columns to which the restriction apply.
*/
- protected final List<ColumnDefinition> columnDefs;
+ protected final List<ColumnMetadata> columnDefs;
- public MultiColumnRestriction(List<ColumnDefinition> columnDefs)
+ public MultiColumnRestriction(List<ColumnMetadata> columnDefs)
{
this.columnDefs = columnDefs;
}
@@ -54,19 +56,19 @@
}
@Override
- public ColumnDefinition getFirstColumn()
+ public ColumnMetadata getFirstColumn()
{
return columnDefs.get(0);
}
@Override
- public ColumnDefinition getLastColumn()
+ public ColumnMetadata getLastColumn()
{
return columnDefs.get(columnDefs.size() - 1);
}
@Override
- public List<ColumnDefinition> getColumnDefs()
+ public List<ColumnMetadata> getColumnDefs()
{
return columnDefs;
}
@@ -96,22 +98,22 @@
*/
protected final String getColumnsInCommons(Restriction otherRestriction)
{
- Set<ColumnDefinition> commons = new HashSet<>(getColumnDefs());
+ Set<ColumnMetadata> commons = new HashSet<>(getColumnDefs());
commons.retainAll(otherRestriction.getColumnDefs());
StringBuilder builder = new StringBuilder();
- for (ColumnDefinition columnDefinition : commons)
+ for (ColumnMetadata columnMetadata : commons)
{
if (builder.length() != 0)
builder.append(" ,");
- builder.append(columnDefinition.name);
+ builder.append(columnMetadata.name);
}
return builder.toString();
}
@Override
- public final boolean hasSupportingIndex(SecondaryIndexManager indexManager)
+ public final boolean hasSupportingIndex(IndexRegistry indexRegistry)
{
- for (Index index : indexManager.listIndexes())
+ for (Index index : indexRegistry.listIndexes())
if (isSupportedBy(index))
return true;
return false;
@@ -130,13 +132,19 @@
{
protected final Term value;
- public EQRestriction(List<ColumnDefinition> columnDefs, Term value)
+ public EQRestriction(List<ColumnMetadata> columnDefs, Term value)
{
super(columnDefs);
this.value = value;
}
@Override
+ public boolean isEQ()
+ {
+ return true;
+ }
+
+ @Override
public void addFunctionsTo(List<Function> functions)
{
value.addFunctionsTo(functions);
@@ -158,7 +166,7 @@
@Override
protected boolean isSupportedBy(Index index)
{
- for(ColumnDefinition column : columnDefs)
+ for(ColumnMetadata column : columnDefs)
if (index.supportsExpression(column, Operator.EQ))
return true;
return false;
@@ -178,14 +186,14 @@
}
@Override
- public final void addRowFilterTo(RowFilter filter, SecondaryIndexManager indexMananger, QueryOptions options)
+ public final void addRowFilterTo(RowFilter filter, IndexRegistry indexRegistry, QueryOptions options)
{
Tuples.Value t = ((Tuples.Value) value.bind(options));
List<ByteBuffer> values = t.getElements();
for (int i = 0, m = columnDefs.size(); i < m; i++)
{
- ColumnDefinition columnDef = columnDefs.get(i);
+ ColumnMetadata columnDef = columnDefs.get(i);
filter.add(columnDef, Operator.EQ, values.get(i));
}
}
@@ -193,7 +201,7 @@
public abstract static class INRestriction extends MultiColumnRestriction
{
- public INRestriction(List<ColumnDefinition> columnDefs)
+ public INRestriction(List<ColumnMetadata> columnDefs)
{
super(columnDefs);
}
@@ -208,7 +216,7 @@
builder.addAllElementsToAll(splitInValues);
if (builder.containsNull())
- throw invalidRequest("Invalid null value in condition for columns: %s", ColumnDefinition.toIdentifiers(columnDefs));
+ throw invalidRequest("Invalid null value in condition for columns: %s", ColumnMetadata.toIdentifiers(columnDefs));
return builder;
}
@@ -228,7 +236,7 @@
@Override
protected boolean isSupportedBy(Index index)
{
- for (ColumnDefinition column: columnDefs)
+ for (ColumnMetadata column: columnDefs)
if (index.supportsExpression(column, Operator.IN))
return true;
return false;
@@ -236,7 +244,7 @@
@Override
public final void addRowFilterTo(RowFilter filter,
- SecondaryIndexManager indexManager,
+ IndexRegistry indexRegistry,
QueryOptions options)
{
throw invalidRequest("IN restrictions are not supported on indexed columns");
@@ -253,7 +261,7 @@
{
protected final List<Term> values;
- public InRestrictionWithValues(List<ColumnDefinition> columnDefs, List<Term> values)
+ public InRestrictionWithValues(List<ColumnMetadata> columnDefs, List<Term> values)
{
super(columnDefs);
this.values = values;
@@ -292,7 +300,7 @@
{
protected final AbstractMarker marker;
- public InRestrictionWithMarker(List<ColumnDefinition> columnDefs, AbstractMarker marker)
+ public InRestrictionWithMarker(List<ColumnMetadata> columnDefs, AbstractMarker marker)
{
super(columnDefs);
this.marker = marker;
@@ -323,12 +331,12 @@
{
private final TermSlice slice;
- public SliceRestriction(List<ColumnDefinition> columnDefs, Bound bound, boolean inclusive, Term term)
+ public SliceRestriction(List<ColumnMetadata> columnDefs, Bound bound, boolean inclusive, Term term)
{
this(columnDefs, TermSlice.newInstance(bound, inclusive, term));
}
- SliceRestriction(List<ColumnDefinition> columnDefs, TermSlice slice)
+ SliceRestriction(List<ColumnMetadata> columnDefs, TermSlice slice)
{
super(columnDefs);
this.slice = slice;
@@ -360,7 +368,7 @@
for (int i = 0, m = columnDefs.size(); i < m; i++)
{
- ColumnDefinition column = columnDefs.get(i);
+ ColumnMetadata column = columnDefs.get(i);
Bound b = bound.reverseIfNeeded(column);
// For mixed order columns, we need to create additional slices when 2 columns are in reverse order
@@ -410,7 +418,7 @@
@Override
protected boolean isSupportedBy(Index index)
{
- for(ColumnDefinition def : columnDefs)
+ for(ColumnMetadata def : columnDefs)
if (slice.isSupportedBy(def, index))
return true;
return false;
@@ -443,7 +451,7 @@
if (!getFirstColumn().equals(otherRestriction.getFirstColumn()))
{
- ColumnDefinition column = getFirstColumn().position() > otherRestriction.getFirstColumn().position()
+ ColumnMetadata column = getFirstColumn().position() > otherRestriction.getFirstColumn().position()
? getFirstColumn() : otherRestriction.getFirstColumn();
throw invalidRequest("Column \"%s\" cannot be restricted by two inequalities not starting with the same column",
@@ -458,14 +466,14 @@
getColumnsInCommons(otherRestriction));
SliceRestriction otherSlice = (SliceRestriction) otherRestriction;
- List<ColumnDefinition> newColumnDefs = columnDefs.size() >= otherSlice.columnDefs.size() ? columnDefs : otherSlice.columnDefs;
+ List<ColumnMetadata> newColumnDefs = columnDefs.size() >= otherSlice.columnDefs.size() ? columnDefs : otherSlice.columnDefs;
return new SliceRestriction(newColumnDefs, slice.merge(otherSlice.slice));
}
@Override
public final void addRowFilterTo(RowFilter filter,
- SecondaryIndexManager indexManager,
+ IndexRegistry indexRegistry,
QueryOptions options)
{
throw invalidRequest("Multi-column slice restrictions cannot be used for filtering.");
@@ -507,7 +515,7 @@
public static class NotNullRestriction extends MultiColumnRestriction
{
- public NotNullRestriction(List<ColumnDefinition> columnDefs)
+ public NotNullRestriction(List<ColumnMetadata> columnDefs)
{
super(columnDefs);
assert columnDefs.size() == 1;
@@ -540,7 +548,7 @@
@Override
protected boolean isSupportedBy(Index index)
{
- for(ColumnDefinition column : columnDefs)
+ for(ColumnMetadata column : columnDefs)
if (index.supportsExpression(column, Operator.IS_NOT))
return true;
return false;
@@ -553,9 +561,15 @@
}
@Override
- public final void addRowFilterTo(RowFilter filter, SecondaryIndexManager indexMananger, QueryOptions options)
+ public final void addRowFilterTo(RowFilter filter, IndexRegistry indexRegistry, QueryOptions options)
{
throw new UnsupportedOperationException("Secondary indexes do not support IS NOT NULL restrictions");
}
}
+
+ @Override
+ public String toString()
+ {
+ return ToStringBuilder.reflectionToString(this, ToStringStyle.SHORT_PREFIX_STYLE);
+ }
}
diff --git a/src/java/org/apache/cassandra/cql3/restrictions/PartitionKeyRestrictions.java b/src/java/org/apache/cassandra/cql3/restrictions/PartitionKeyRestrictions.java
index 1ff45d0..b1edf94 100644
--- a/src/java/org/apache/cassandra/cql3/restrictions/PartitionKeyRestrictions.java
+++ b/src/java/org/apache/cassandra/cql3/restrictions/PartitionKeyRestrictions.java
@@ -20,7 +20,7 @@
import java.nio.ByteBuffer;
import java.util.List;
-import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.cql3.QueryOptions;
import org.apache.cassandra.cql3.statements.Bound;
@@ -53,16 +53,16 @@
/**
* checks if specified restrictions require filtering
*
- * @param cfm column family metadata
+ * @param table column family metadata
* @return <code>true</code> if filtering is required, <code>false</code> otherwise
*/
- public boolean needFiltering(CFMetaData cfm);
+ public boolean needFiltering(TableMetadata table);
/**
* Checks if the partition key has unrestricted components.
*
- * @param cfm column family metadata
+ * @param table column family metadata
* @return <code>true</code> if the partition key has unrestricted components, <code>false</code> otherwise.
*/
- public boolean hasUnrestrictedPartitionKeyComponents(CFMetaData cfm);
+ public boolean hasUnrestrictedPartitionKeyComponents(TableMetadata table);
}
diff --git a/src/java/org/apache/cassandra/cql3/restrictions/PartitionKeySingleRestrictionSet.java b/src/java/org/apache/cassandra/cql3/restrictions/PartitionKeySingleRestrictionSet.java
index f2b427d..5bb3242 100644
--- a/src/java/org/apache/cassandra/cql3/restrictions/PartitionKeySingleRestrictionSet.java
+++ b/src/java/org/apache/cassandra/cql3/restrictions/PartitionKeySingleRestrictionSet.java
@@ -20,14 +20,14 @@
import java.nio.ByteBuffer;
import java.util.*;
-import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.cql3.QueryOptions;
import org.apache.cassandra.cql3.statements.Bound;
import org.apache.cassandra.db.ClusteringComparator;
import org.apache.cassandra.db.ClusteringPrefix;
import org.apache.cassandra.db.MultiCBuilder;
import org.apache.cassandra.db.filter.RowFilter;
-import org.apache.cassandra.index.SecondaryIndexManager;
+import org.apache.cassandra.index.IndexRegistry;
/**
* A set of single restrictions on the partition key.
@@ -59,7 +59,7 @@
{
List<ByteBuffer> l = new ArrayList<>(clusterings.size());
for (ClusteringPrefix clustering : clusterings)
- l.add(CFMetaData.serializePartitionKey(clustering));
+ l.add(clustering.serializeAsPartitionKey());
return l;
}
@@ -121,29 +121,29 @@
@Override
public void addRowFilterTo(RowFilter filter,
- SecondaryIndexManager indexManager,
+ IndexRegistry indexRegistry,
QueryOptions options)
{
for (SingleRestriction restriction : restrictions)
{
- restriction.addRowFilterTo(filter, indexManager, options);
+ restriction.addRowFilterTo(filter, indexRegistry, options);
}
}
@Override
- public boolean needFiltering(CFMetaData cfm)
+ public boolean needFiltering(TableMetadata table)
{
if (isEmpty())
return false;
// slice or has unrestricted key component
- return hasUnrestrictedPartitionKeyComponents(cfm) || hasSlice() || hasContains();
+ return hasUnrestrictedPartitionKeyComponents(table) || hasSlice() || hasContains();
}
@Override
- public boolean hasUnrestrictedPartitionKeyComponents(CFMetaData cfm)
+ public boolean hasUnrestrictedPartitionKeyComponents(TableMetadata table)
{
- return size() < cfm.partitionKeyColumns().size();
+ return size() < table.partitionKeyColumns().size();
}
@Override
diff --git a/src/java/org/apache/cassandra/cql3/restrictions/Restriction.java b/src/java/org/apache/cassandra/cql3/restrictions/Restriction.java
index fc7f5bc..91dedad 100644
--- a/src/java/org/apache/cassandra/cql3/restrictions/Restriction.java
+++ b/src/java/org/apache/cassandra/cql3/restrictions/Restriction.java
@@ -19,11 +19,11 @@
import java.util.List;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.cql3.QueryOptions;
import org.apache.cassandra.cql3.functions.Function;
import org.apache.cassandra.db.filter.RowFilter;
-import org.apache.cassandra.index.SecondaryIndexManager;
+import org.apache.cassandra.index.IndexRegistry;
/**
* <p>Implementation of this class must be immutable.</p>
@@ -39,19 +39,19 @@
* Returns the definition of the first column.
* @return the definition of the first column.
*/
- public ColumnDefinition getFirstColumn();
+ public ColumnMetadata getFirstColumn();
/**
* Returns the definition of the last column.
* @return the definition of the last column.
*/
- public ColumnDefinition getLastColumn();
+ public ColumnMetadata getLastColumn();
/**
* Returns the column definitions in position order.
* @return the column definitions in position order.
*/
- public List<ColumnDefinition> getColumnDefs();
+ public List<ColumnMetadata> getColumnDefs();
/**
* Adds all functions (native and user-defined) used by any component of the restriction
@@ -63,19 +63,19 @@
/**
* Check if the restriction is on indexed columns.
*
- * @param indexManager the index manager
+ * @param indexRegistry the index registry
* @return <code>true</code> if the restriction is on indexed columns, <code>false</code>
*/
- public boolean hasSupportingIndex(SecondaryIndexManager indexManager);
+ public boolean hasSupportingIndex(IndexRegistry indexRegistry);
/**
* Adds to the specified row filter the expressions corresponding to this <code>Restriction</code>.
*
* @param filter the row filter to add expressions to
- * @param indexManager the secondary index manager
+ * @param indexRegistry the index registry
* @param options the query options
*/
public void addRowFilterTo(RowFilter filter,
- SecondaryIndexManager indexManager,
+ IndexRegistry indexRegistry,
QueryOptions options);
}
diff --git a/src/java/org/apache/cassandra/cql3/restrictions/RestrictionSet.java b/src/java/org/apache/cassandra/cql3/restrictions/RestrictionSet.java
index a0816d2..427c396 100644
--- a/src/java/org/apache/cassandra/cql3/restrictions/RestrictionSet.java
+++ b/src/java/org/apache/cassandra/cql3/restrictions/RestrictionSet.java
@@ -21,13 +21,15 @@
import com.google.common.collect.AbstractIterator;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.cql3.QueryOptions;
import org.apache.cassandra.cql3.functions.Function;
import org.apache.cassandra.cql3.restrictions.SingleColumnRestriction.ContainsRestriction;
import org.apache.cassandra.db.filter.RowFilter;
import org.apache.cassandra.exceptions.InvalidRequestException;
-import org.apache.cassandra.index.SecondaryIndexManager;
+import org.apache.cassandra.index.IndexRegistry;
+import org.apache.commons.lang3.builder.ToStringBuilder;
+import org.apache.commons.lang3.builder.ToStringStyle;
/**
* Sets of column restrictions.
@@ -39,10 +41,10 @@
/**
* The comparator used to sort the <code>Restriction</code>s.
*/
- private static final Comparator<ColumnDefinition> COLUMN_DEFINITION_COMPARATOR = new Comparator<ColumnDefinition>()
+ private static final Comparator<ColumnMetadata> COLUMN_DEFINITION_COMPARATOR = new Comparator<ColumnMetadata>()
{
@Override
- public int compare(ColumnDefinition column, ColumnDefinition otherColumn)
+ public int compare(ColumnMetadata column, ColumnMetadata otherColumn)
{
int value = Integer.compare(column.position(), otherColumn.position());
return value != 0 ? value : column.name.bytes.compareTo(otherColumn.name.bytes);
@@ -52,7 +54,7 @@
/**
* The restrictions per column.
*/
- protected final TreeMap<ColumnDefinition, SingleRestriction> restrictions;
+ protected final TreeMap<ColumnMetadata, SingleRestriction> restrictions;
/**
* {@code true} if it contains multi-column restrictions, {@code false} otherwise.
@@ -61,10 +63,10 @@
public RestrictionSet()
{
- this(new TreeMap<ColumnDefinition, SingleRestriction>(COLUMN_DEFINITION_COMPARATOR), false);
+ this(new TreeMap<ColumnMetadata, SingleRestriction>(COLUMN_DEFINITION_COMPARATOR), false);
}
- private RestrictionSet(TreeMap<ColumnDefinition, SingleRestriction> restrictions,
+ private RestrictionSet(TreeMap<ColumnMetadata, SingleRestriction> restrictions,
boolean hasMultiColumnRestrictions)
{
this.restrictions = restrictions;
@@ -72,14 +74,14 @@
}
@Override
- public void addRowFilterTo(RowFilter filter, SecondaryIndexManager indexManager, QueryOptions options) throws InvalidRequestException
+ public void addRowFilterTo(RowFilter filter, IndexRegistry indexRegistry, QueryOptions options) throws InvalidRequestException
{
for (Restriction restriction : restrictions.values())
- restriction.addRowFilterTo(filter, indexManager, options);
+ restriction.addRowFilterTo(filter, indexRegistry, options);
}
@Override
- public List<ColumnDefinition> getColumnDefs()
+ public List<ColumnMetadata> getColumnDefs()
{
return new ArrayList<>(restrictions.keySet());
}
@@ -108,9 +110,9 @@
* @param kind the column kind
* @return {@code true} if one of the restrictions applies to a column of the specific kind, {@code false} otherwise.
*/
- public boolean hasRestrictionFor(ColumnDefinition.Kind kind)
+ public boolean hasRestrictionFor(ColumnMetadata.Kind kind)
{
- for (ColumnDefinition column : restrictions.keySet())
+ for (ColumnMetadata column : restrictions.keySet())
{
if (column.kind == kind)
return true;
@@ -127,19 +129,19 @@
public RestrictionSet addRestriction(SingleRestriction restriction)
{
// RestrictionSet is immutable so we need to clone the restrictions map.
- TreeMap<ColumnDefinition, SingleRestriction> newRestrictions = new TreeMap<>(this.restrictions);
+ TreeMap<ColumnMetadata, SingleRestriction> newRestrictions = new TreeMap<>(this.restrictions);
return new RestrictionSet(mergeRestrictions(newRestrictions, restriction), hasMultiColumnRestrictions || restriction.isMultiColumn());
}
- private TreeMap<ColumnDefinition, SingleRestriction> mergeRestrictions(TreeMap<ColumnDefinition, SingleRestriction> restrictions,
- SingleRestriction restriction)
+ private TreeMap<ColumnMetadata, SingleRestriction> mergeRestrictions(TreeMap<ColumnMetadata, SingleRestriction> restrictions,
+ SingleRestriction restriction)
{
- Collection<ColumnDefinition> columnDefs = restriction.getColumnDefs();
+ Collection<ColumnMetadata> columnDefs = restriction.getColumnDefs();
Set<SingleRestriction> existingRestrictions = getRestrictions(columnDefs);
if (existingRestrictions.isEmpty())
{
- for (ColumnDefinition columnDef : columnDefs)
+ for (ColumnMetadata columnDef : columnDefs)
restrictions.put(columnDef, restriction);
}
else
@@ -148,7 +150,7 @@
{
SingleRestriction newRestriction = mergeRestrictions(existing, restriction);
- for (ColumnDefinition columnDef : columnDefs)
+ for (ColumnMetadata columnDef : columnDefs)
restrictions.put(columnDef, newRestriction);
}
}
@@ -157,7 +159,7 @@
}
@Override
- public Set<Restriction> getRestrictions(ColumnDefinition columnDef)
+ public Set<Restriction> getRestrictions(ColumnMetadata columnDef)
{
Restriction existing = restrictions.get(columnDef);
return existing == null ? Collections.emptySet() : Collections.singleton(existing);
@@ -169,10 +171,10 @@
* @param columnDefs the column definitions
* @return all the restrictions applied to the specified columns
*/
- private Set<SingleRestriction> getRestrictions(Collection<ColumnDefinition> columnDefs)
+ private Set<SingleRestriction> getRestrictions(Collection<ColumnMetadata> columnDefs)
{
Set<SingleRestriction> set = new HashSet<>();
- for (ColumnDefinition columnDef : columnDefs)
+ for (ColumnMetadata columnDef : columnDefs)
{
SingleRestriction existing = restrictions.get(columnDef);
if (existing != null)
@@ -182,11 +184,11 @@
}
@Override
- public final boolean hasSupportingIndex(SecondaryIndexManager indexManager)
+ public final boolean hasSupportingIndex(IndexRegistry indexRegistry)
{
for (Restriction restriction : restrictions.values())
{
- if (restriction.hasSupportingIndex(indexManager))
+ if (restriction.hasSupportingIndex(indexRegistry))
return true;
}
return false;
@@ -198,19 +200,19 @@
* @param columnDef the column for which the next one need to be found
* @return the column after the specified one.
*/
- ColumnDefinition nextColumn(ColumnDefinition columnDef)
+ ColumnMetadata nextColumn(ColumnMetadata columnDef)
{
return restrictions.tailMap(columnDef, false).firstKey();
}
@Override
- public ColumnDefinition getFirstColumn()
+ public ColumnMetadata getFirstColumn()
{
return isEmpty() ? null : this.restrictions.firstKey();
}
@Override
- public ColumnDefinition getLastColumn()
+ public ColumnMetadata getLastColumn()
{
return isEmpty() ? null : this.restrictions.lastKey();
}
@@ -354,4 +356,10 @@
return endOfData();
}
}
+
+ @Override
+ public String toString()
+ {
+ return ToStringBuilder.reflectionToString(this, ToStringStyle.SHORT_PREFIX_STYLE);
+ }
}
diff --git a/src/java/org/apache/cassandra/cql3/restrictions/RestrictionSetWrapper.java b/src/java/org/apache/cassandra/cql3/restrictions/RestrictionSetWrapper.java
index 5157de0..9803adc 100644
--- a/src/java/org/apache/cassandra/cql3/restrictions/RestrictionSetWrapper.java
+++ b/src/java/org/apache/cassandra/cql3/restrictions/RestrictionSetWrapper.java
@@ -20,11 +20,14 @@
import java.util.List;
import java.util.Set;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.commons.lang3.builder.ToStringBuilder;
+import org.apache.commons.lang3.builder.ToStringStyle;
+
+import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.cql3.QueryOptions;
import org.apache.cassandra.cql3.functions.Function;
import org.apache.cassandra.db.filter.RowFilter;
-import org.apache.cassandra.index.SecondaryIndexManager;
+import org.apache.cassandra.index.IndexRegistry;
/**
* A <code>RestrictionSet</code> wrapper that can be extended to allow to modify the <code>RestrictionSet</code>
@@ -43,13 +46,13 @@
}
public void addRowFilterTo(RowFilter filter,
- SecondaryIndexManager indexManager,
+ IndexRegistry indexRegistry,
QueryOptions options)
{
- restrictions.addRowFilterTo(filter, indexManager, options);
+ restrictions.addRowFilterTo(filter, indexRegistry, options);
}
- public List<ColumnDefinition> getColumnDefs()
+ public List<ColumnMetadata> getColumnDefs()
{
return restrictions.getColumnDefs();
}
@@ -69,17 +72,17 @@
return restrictions.size();
}
- public boolean hasSupportingIndex(SecondaryIndexManager indexManager)
+ public boolean hasSupportingIndex(IndexRegistry indexRegistry)
{
- return restrictions.hasSupportingIndex(indexManager);
+ return restrictions.hasSupportingIndex(indexRegistry);
}
- public ColumnDefinition getFirstColumn()
+ public ColumnMetadata getFirstColumn()
{
return restrictions.getFirstColumn();
}
- public ColumnDefinition getLastColumn()
+ public ColumnMetadata getLastColumn()
{
return restrictions.getLastColumn();
}
@@ -104,8 +107,14 @@
return restrictions.hasOnlyEqualityRestrictions();
}
- public Set<Restriction> getRestrictions(ColumnDefinition columnDef)
+ public Set<Restriction> getRestrictions(ColumnMetadata columnDef)
{
return restrictions.getRestrictions(columnDef);
}
+
+ @Override
+ public String toString()
+ {
+ return ToStringBuilder.reflectionToString(this, ToStringStyle.SHORT_PREFIX_STYLE);
+ }
}
diff --git a/src/java/org/apache/cassandra/cql3/restrictions/Restrictions.java b/src/java/org/apache/cassandra/cql3/restrictions/Restrictions.java
index 5d11e9f..77e0dd9 100644
--- a/src/java/org/apache/cassandra/cql3/restrictions/Restrictions.java
+++ b/src/java/org/apache/cassandra/cql3/restrictions/Restrictions.java
@@ -19,7 +19,7 @@
import java.util.Set;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.schema.ColumnMetadata;
/**
* Sets of restrictions
@@ -32,7 +32,7 @@
* @param columnDef the column definition
* @return the restrictions applied to the specified column
*/
- Set<Restriction> getRestrictions(ColumnDefinition columnDef);
+ Set<Restriction> getRestrictions(ColumnMetadata columnDef);
/**
* Checks if this <code>Restrictions</code> is empty or not.
diff --git a/src/java/org/apache/cassandra/cql3/restrictions/SingleColumnRestriction.java b/src/java/org/apache/cassandra/cql3/restrictions/SingleColumnRestriction.java
index 09c02ed..1b3482b 100644
--- a/src/java/org/apache/cassandra/cql3/restrictions/SingleColumnRestriction.java
+++ b/src/java/org/apache/cassandra/cql3/restrictions/SingleColumnRestriction.java
@@ -22,16 +22,15 @@
import java.util.Collections;
import java.util.List;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.cql3.*;
import org.apache.cassandra.cql3.Term.Terminal;
import org.apache.cassandra.cql3.functions.Function;
import org.apache.cassandra.cql3.statements.Bound;
import org.apache.cassandra.db.MultiCBuilder;
import org.apache.cassandra.db.filter.RowFilter;
-import org.apache.cassandra.exceptions.InvalidRequestException;
import org.apache.cassandra.index.Index;
-import org.apache.cassandra.index.SecondaryIndexManager;
+import org.apache.cassandra.index.IndexRegistry;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.Pair;
@@ -46,35 +45,35 @@
/**
* The definition of the column to which apply the restriction.
*/
- protected final ColumnDefinition columnDef;
+ protected final ColumnMetadata columnDef;
- public SingleColumnRestriction(ColumnDefinition columnDef)
+ public SingleColumnRestriction(ColumnMetadata columnDef)
{
this.columnDef = columnDef;
}
@Override
- public List<ColumnDefinition> getColumnDefs()
+ public List<ColumnMetadata> getColumnDefs()
{
return Collections.singletonList(columnDef);
}
@Override
- public ColumnDefinition getFirstColumn()
+ public ColumnMetadata getFirstColumn()
{
return columnDef;
}
@Override
- public ColumnDefinition getLastColumn()
+ public ColumnMetadata getLastColumn()
{
return columnDef;
}
@Override
- public boolean hasSupportingIndex(SecondaryIndexManager indexManager)
+ public boolean hasSupportingIndex(IndexRegistry indexRegistry)
{
- for (Index index : indexManager.listIndexes())
+ for (Index index : indexRegistry.listIndexes())
if (isSupportedBy(index))
return true;
@@ -122,11 +121,11 @@
*/
protected abstract boolean isSupportedBy(Index index);
- public static class EQRestriction extends SingleColumnRestriction
+ public static final class EQRestriction extends SingleColumnRestriction
{
- public final Term value;
+ private final Term value;
- public EQRestriction(ColumnDefinition columnDef, Term value)
+ public EQRestriction(ColumnMetadata columnDef, Term value)
{
super(columnDef);
this.value = value;
@@ -152,7 +151,7 @@
@Override
public void addRowFilterTo(RowFilter filter,
- SecondaryIndexManager indexManager,
+ IndexRegistry indexRegistry,
QueryOptions options)
{
filter.add(columnDef, Operator.EQ, value.bindAndGet(options));
@@ -188,7 +187,7 @@
public static abstract class INRestriction extends SingleColumnRestriction
{
- public INRestriction(ColumnDefinition columnDef)
+ public INRestriction(ColumnMetadata columnDef)
{
super(columnDef);
}
@@ -216,7 +215,7 @@
@Override
public void addRowFilterTo(RowFilter filter,
- SecondaryIndexManager indexManager,
+ IndexRegistry indexRegistry,
QueryOptions options)
{
throw invalidRequest("IN restrictions are not supported on indexed columns");
@@ -235,7 +234,7 @@
{
protected final List<Term> values;
- public InRestrictionWithValues(ColumnDefinition columnDef, List<Term> values)
+ public InRestrictionWithValues(ColumnMetadata columnDef, List<Term> values)
{
super(columnDef);
this.values = values;
@@ -273,7 +272,7 @@
{
protected final AbstractMarker marker;
- public InRestrictionWithMarker(ColumnDefinition columnDef, AbstractMarker marker)
+ public InRestrictionWithMarker(ColumnMetadata columnDef, AbstractMarker marker)
{
super(columnDef);
this.marker = marker;
@@ -309,9 +308,9 @@
public static class SliceRestriction extends SingleColumnRestriction
{
- public final TermSlice slice;
+ private final TermSlice slice;
- public SliceRestriction(ColumnDefinition columnDef, Bound bound, boolean inclusive, Term term)
+ public SliceRestriction(ColumnMetadata columnDef, Bound bound, boolean inclusive, Term term)
{
super(columnDef);
slice = TermSlice.newInstance(bound, inclusive, term);
@@ -386,7 +385,7 @@
}
@Override
- public void addRowFilterTo(RowFilter filter, SecondaryIndexManager indexManager, QueryOptions options)
+ public void addRowFilterTo(RowFilter filter, IndexRegistry indexRegistry, QueryOptions options)
{
for (Bound b : Bound.values())
if (hasBound(b))
@@ -405,7 +404,7 @@
return String.format("SLICE%s", slice);
}
- SliceRestriction(ColumnDefinition columnDef, TermSlice slice)
+ private SliceRestriction(ColumnMetadata columnDef, TermSlice slice)
{
super(columnDef);
this.slice = slice;
@@ -420,7 +419,7 @@
private List<Term> entryKeys = new ArrayList<>(); // for map[key] = value
private List<Term> entryValues = new ArrayList<>(); // for map[key] = value
- public ContainsRestriction(ColumnDefinition columnDef, Term t, boolean isKey)
+ public ContainsRestriction(ColumnMetadata columnDef, Term t, boolean isKey)
{
super(columnDef);
if (isKey)
@@ -429,7 +428,7 @@
values.add(t);
}
- public ContainsRestriction(ColumnDefinition columnDef, Term mapKey, Term mapValue)
+ public ContainsRestriction(ColumnMetadata columnDef, Term mapKey, Term mapValue)
{
super(columnDef);
entryKeys.add(mapKey);
@@ -476,7 +475,7 @@
}
@Override
- public void addRowFilterTo(RowFilter filter, SecondaryIndexManager indexManager, QueryOptions options)
+ public void addRowFilterTo(RowFilter filter, IndexRegistry indexRegistry, QueryOptions options)
{
for (ByteBuffer value : bindAndGet(values, options))
filter.add(columnDef, Operator.CONTAINS, value);
@@ -584,7 +583,7 @@
to.entryValues.addAll(from.entryValues);
}
- private ContainsRestriction(ColumnDefinition columnDef)
+ private ContainsRestriction(ColumnMetadata columnDef)
{
super(columnDef);
}
@@ -592,7 +591,7 @@
public static final class IsNotNullRestriction extends SingleColumnRestriction
{
- public IsNotNullRestriction(ColumnDefinition columnDef)
+ public IsNotNullRestriction(ColumnMetadata columnDef)
{
super(columnDef);
}
@@ -616,7 +615,7 @@
@Override
public void addRowFilterTo(RowFilter filter,
- SecondaryIndexManager indexManager,
+ IndexRegistry indexRegistry,
QueryOptions options)
{
throw new UnsupportedOperationException("Secondary indexes do not support IS NOT NULL restrictions");
@@ -653,7 +652,7 @@
private final Operator operator;
private final Term value;
- public LikeRestriction(ColumnDefinition columnDef, Operator operator, Term value)
+ public LikeRestriction(ColumnMetadata columnDef, Operator operator, Term value)
{
super(columnDef);
this.operator = operator;
@@ -692,16 +691,16 @@
@Override
public void addRowFilterTo(RowFilter filter,
- SecondaryIndexManager indexManager,
+ IndexRegistry indexRegistry,
QueryOptions options)
{
Pair<Operator, ByteBuffer> operation = makeSpecific(value.bindAndGet(options));
// there must be a suitable INDEX for LIKE_XXX expressions
RowFilter.SimpleExpression expression = filter.add(columnDef, operation.left, operation.right);
- indexManager.getBestIndexFor(expression)
- .orElseThrow(() -> invalidRequest("%s is only supported on properly indexed columns",
- expression));
+ indexRegistry.getBestIndexFor(expression)
+ .orElseThrow(() -> invalidRequest("%s is only supported on properly indexed columns",
+ expression));
}
@Override
@@ -777,202 +776,4 @@
return Pair.create(operator, newValue);
}
}
-
- /**
- * Super Column Compatibiltiy
- */
-
- public static class SuperColumnMultiEQRestriction extends EQRestriction
- {
- public ByteBuffer firstValue;
- public ByteBuffer secondValue;
-
- public SuperColumnMultiEQRestriction(ColumnDefinition columnDef, Term value)
- {
- super(columnDef, value);
- }
-
- @Override
- public MultiCBuilder appendTo(MultiCBuilder builder, QueryOptions options)
- {
- Term term = value.bind(options);
-
- assert (term instanceof Tuples.Value);
- firstValue = ((Tuples.Value)term).getElements().get(0);
- secondValue = ((Tuples.Value)term).getElements().get(1);
-
- builder.addElementToAll(firstValue);
- checkFalse(builder.containsNull(), "Invalid null value in condition for column %s", columnDef.name);
- checkFalse(builder.containsUnset(), "Invalid unset value for column %s", columnDef.name);
- return builder;
- }
- }
-
- public static class SuperColumnMultiSliceRestriction extends SliceRestriction
- {
- public ByteBuffer firstValue;
- public ByteBuffer secondValue;
-
- // These are here to avoid polluting SliceRestriction
- public final Bound bound;
- public final boolean trueInclusive;
- public SuperColumnMultiSliceRestriction(ColumnDefinition columnDef, Bound bound, boolean inclusive, Term term)
- {
- super(columnDef, bound, true, term);
- this.bound = bound;
- this.trueInclusive = inclusive;
-
- }
-
- @Override
- public MultiCBuilder appendBoundTo(MultiCBuilder builder, Bound bound, QueryOptions options)
- {
- Bound b = bound.reverseIfNeeded(getFirstColumn());
-
- if (!hasBound(b))
- return builder;
-
- Term term = slice.bound(b);
-
- assert (term instanceof Tuples.Value);
- firstValue = ((Tuples.Value)term).getElements().get(0);
- secondValue = ((Tuples.Value)term).getElements().get(1);
-
- checkBindValueSet(firstValue, "Invalid unset value for column %s", columnDef.name);
- checkBindValueSet(secondValue, "Invalid unset value for column %s", columnDef.name);
- return builder.addElementToAll(firstValue);
-
- }
- }
-
- public static final class SuperColumnKeyEQRestriction extends EQRestriction
- {
- public SuperColumnKeyEQRestriction(ColumnDefinition columnDef, Term value)
- {
- super(columnDef, value);
- }
-
- public ByteBuffer bindValue(QueryOptions options)
- {
- return value.bindAndGet(options);
- }
-
- @Override
- public MultiCBuilder appendBoundTo(MultiCBuilder builder, Bound bound, QueryOptions options)
- {
- // no-op
- return builder;
- }
-
- @Override
- public void addRowFilterTo(RowFilter filter, SecondaryIndexManager indexManager, QueryOptions options) throws InvalidRequestException
- {
- // no-op
- }
- }
-
- public static abstract class SuperColumnKeyINRestriction extends INRestriction
- {
- public SuperColumnKeyINRestriction(ColumnDefinition columnDef)
- {
- super(columnDef);
- }
-
- @Override
- public MultiCBuilder appendTo(MultiCBuilder builder, QueryOptions options)
- {
- // no-op
- return builder;
- }
-
- @Override
- public void addRowFilterTo(RowFilter filter,
- SecondaryIndexManager indexManager,
- QueryOptions options) throws InvalidRequestException
- {
- // no-op
- }
-
- public void addFunctionsTo(List<Function> functions)
- {
- // no-op
- }
-
- MultiColumnRestriction toMultiColumnRestriction()
- {
- // no-op
- return null;
- }
-
- public abstract List<ByteBuffer> getValues(QueryOptions options) throws InvalidRequestException;
- }
-
- public static class SuperColumnKeyINRestrictionWithMarkers extends SuperColumnKeyINRestriction
- {
- protected final AbstractMarker marker;
-
- public SuperColumnKeyINRestrictionWithMarkers(ColumnDefinition columnDef, AbstractMarker marker)
- {
- super(columnDef);
- this.marker = marker;
- }
-
- public List<ByteBuffer> getValues(QueryOptions options) throws InvalidRequestException
- {
- Terminal term = marker.bind(options);
- checkNotNull(term, "Invalid null value for column %s", columnDef.name);
- checkFalse(term == Constants.UNSET_VALUE, "Invalid unset value for column %s", columnDef.name);
- Term.MultiItemTerminal lval = (Term.MultiItemTerminal) term;
- return lval.getElements();
- }
- }
-
- public static class SuperColumnKeyINRestrictionWithValues extends SuperColumnKeyINRestriction
- {
- private final List<Term> values;
-
- public SuperColumnKeyINRestrictionWithValues(ColumnDefinition columnDef, List<Term> values)
- {
- super(columnDef);
- this.values = values;
- }
-
- public List<ByteBuffer> getValues(QueryOptions options) throws InvalidRequestException
- {
- List<ByteBuffer> buffers = new ArrayList<>(values.size());
- for (Term value : values)
- buffers.add(value.bindAndGet(options));
- return buffers;
- }
- }
-
- public static class SuperColumnKeySliceRestriction extends SliceRestriction
- {
- // These are here to avoid polluting SliceRestriction
- private Term term;
-
- public SuperColumnKeySliceRestriction(ColumnDefinition columnDef, Bound bound, boolean inclusive, Term term)
- {
- super(columnDef, bound, inclusive, term);
- this.term = term;
- }
-
- public ByteBuffer bindValue(QueryOptions options)
- {
- return term.bindAndGet(options);
- }
-
- @Override
- public MultiCBuilder appendBoundTo(MultiCBuilder builder, Bound bound, QueryOptions options)
- {
- // no-op
- return builder;
- }
-
- @Override
- public void addRowFilterTo(RowFilter filter, SecondaryIndexManager indexManager, QueryOptions options) throws InvalidRequestException
- {
- // no-op
- }
- }
}
diff --git a/src/java/org/apache/cassandra/cql3/restrictions/StatementRestrictions.java b/src/java/org/apache/cassandra/cql3/restrictions/StatementRestrictions.java
index 7636ecc..317e415 100644
--- a/src/java/org/apache/cassandra/cql3/restrictions/StatementRestrictions.java
+++ b/src/java/org/apache/cassandra/cql3/restrictions/StatementRestrictions.java
@@ -21,11 +21,7 @@
import java.util.*;
import com.google.common.base.Joiner;
-import com.google.common.collect.Iterators;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
-import org.apache.cassandra.config.ColumnDefinition.Kind;
import org.apache.cassandra.cql3.*;
import org.apache.cassandra.cql3.functions.Function;
import org.apache.cassandra.cql3.statements.Bound;
@@ -36,9 +32,12 @@
import org.apache.cassandra.dht.*;
import org.apache.cassandra.exceptions.InvalidRequestException;
import org.apache.cassandra.index.Index;
-import org.apache.cassandra.index.SecondaryIndexManager;
-import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.index.IndexRegistry;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.utils.btree.BTreeSet;
+import org.apache.commons.lang3.builder.ToStringBuilder;
+import org.apache.commons.lang3.builder.ToStringStyle;
import static org.apache.cassandra.cql3.statements.RequestValidations.checkFalse;
import static org.apache.cassandra.cql3.statements.RequestValidations.checkNotNull;
@@ -62,7 +61,7 @@
/**
* The Column Family meta data
*/
- public final CFMetaData cfm;
+ public final TableMetadata table;
/**
* Restrictions on partitioning columns
@@ -79,7 +78,7 @@
*/
private RestrictionSet nonPrimaryKeyRestrictions;
- private Set<ColumnDefinition> notNullColumns;
+ private Set<ColumnMetadata> notNullColumns;
/**
* The restrictions used to build the row filter
@@ -106,43 +105,53 @@
* Creates a new empty <code>StatementRestrictions</code>.
*
* @param type the type of statement
- * @param cfm the column family meta data
+ * @param table the column family meta data
* @return a new empty <code>StatementRestrictions</code>.
*/
- public static StatementRestrictions empty(StatementType type, CFMetaData cfm)
+ public static StatementRestrictions empty(StatementType type, TableMetadata table)
{
- return new StatementRestrictions(type, cfm, false);
+ return new StatementRestrictions(type, table, false);
}
- private StatementRestrictions(StatementType type, CFMetaData cfm, boolean allowFiltering)
+ private StatementRestrictions(StatementType type, TableMetadata table, boolean allowFiltering)
{
this.type = type;
- this.cfm = cfm;
- this.partitionKeyRestrictions = new PartitionKeySingleRestrictionSet(cfm.getKeyValidatorAsClusteringComparator());
- this.clusteringColumnsRestrictions = new ClusteringColumnRestrictions(cfm, allowFiltering);
+ this.table = table;
+ this.partitionKeyRestrictions = new PartitionKeySingleRestrictionSet(table.partitionKeyAsClusteringComparator());
+ this.clusteringColumnsRestrictions = new ClusteringColumnRestrictions(table, allowFiltering);
this.nonPrimaryKeyRestrictions = new RestrictionSet();
this.notNullColumns = new HashSet<>();
}
public StatementRestrictions(StatementType type,
- CFMetaData cfm,
+ TableMetadata table,
WhereClause whereClause,
VariableSpecifications boundNames,
boolean selectsOnlyStaticColumns,
- boolean selectsComplexColumn,
boolean allowFiltering,
boolean forView)
{
- this(type, cfm, allowFiltering);
+ this(type, table, whereClause, boundNames, selectsOnlyStaticColumns, type.allowUseOfSecondaryIndices(), allowFiltering, forView);
+ }
- ColumnFamilyStore cfs;
- SecondaryIndexManager secondaryIndexManager = null;
+ /*
+ * We want to override allowUseOfSecondaryIndices flag from the StatementType for MV statements
+ * to avoid initing the Keyspace and SecondaryIndexManager.
+ */
+ public StatementRestrictions(StatementType type,
+ TableMetadata table,
+ WhereClause whereClause,
+ VariableSpecifications boundNames,
+ boolean selectsOnlyStaticColumns,
+ boolean allowUseOfSecondaryIndices,
+ boolean allowFiltering,
+ boolean forView)
+ {
+ this(type, table, allowFiltering);
+ IndexRegistry indexRegistry = null;
if (type.allowUseOfSecondaryIndices())
- {
- cfs = Keyspace.open(cfm.ksName).getColumnFamilyStore(cfm.cfName);
- secondaryIndexManager = cfs.indexManager;
- }
+ indexRegistry = IndexRegistry.obtain(table);
/*
* WHERE clause. For a given entity, rules are:
@@ -160,14 +169,13 @@
if (!forView)
throw new InvalidRequestException("Unsupported restriction: " + relation);
- for (ColumnDefinition def : relation.toRestriction(cfm, boundNames).getColumnDefs())
- this.notNullColumns.add(def);
+ this.notNullColumns.addAll(relation.toRestriction(table, boundNames).getColumnDefs());
}
else if (relation.isLIKE())
{
- Restriction restriction = relation.toRestriction(cfm, boundNames);
+ Restriction restriction = relation.toRestriction(table, boundNames);
- if (!type.allowUseOfSecondaryIndices() || !restriction.hasSupportingIndex(secondaryIndexManager))
+ if (!type.allowUseOfSecondaryIndices() || !restriction.hasSupportingIndex(indexRegistry))
throw new InvalidRequestException(String.format("LIKE restriction is only supported on properly " +
"indexed columns. %s is not valid.",
relation.toString()));
@@ -176,28 +184,25 @@
}
else
{
- if (cfm.isSuper() && cfm.isDense() && !relation.onToken())
- addRestriction(relation.toSuperColumnAdapter().toRestriction(cfm, boundNames));
- else
- addRestriction(relation.toRestriction(cfm, boundNames));
+ addRestriction(relation.toRestriction(table, boundNames));
}
}
- hasRegularColumnsRestrictions = nonPrimaryKeyRestrictions.hasRestrictionFor(Kind.REGULAR);
+ hasRegularColumnsRestrictions = nonPrimaryKeyRestrictions.hasRestrictionFor(ColumnMetadata.Kind.REGULAR);
boolean hasQueriableClusteringColumnIndex = false;
boolean hasQueriableIndex = false;
- if (type.allowUseOfSecondaryIndices())
+ if (allowUseOfSecondaryIndices)
{
if (whereClause.containsCustomExpressions())
- processCustomIndexExpressions(whereClause.expressions, boundNames, secondaryIndexManager);
+ processCustomIndexExpressions(whereClause.expressions, boundNames, indexRegistry);
- hasQueriableClusteringColumnIndex = clusteringColumnsRestrictions.hasSupportingIndex(secondaryIndexManager);
+ hasQueriableClusteringColumnIndex = clusteringColumnsRestrictions.hasSupportingIndex(indexRegistry);
hasQueriableIndex = !filterRestrictions.getCustomIndexExpressions().isEmpty()
|| hasQueriableClusteringColumnIndex
- || partitionKeyRestrictions.hasSupportingIndex(secondaryIndexManager)
- || nonPrimaryKeyRestrictions.hasSupportingIndex(secondaryIndexManager);
+ || partitionKeyRestrictions.hasSupportingIndex(indexRegistry)
+ || nonPrimaryKeyRestrictions.hasSupportingIndex(indexRegistry);
}
// At this point, the select statement if fully constructed, but we still have a few things to validate
@@ -205,7 +210,7 @@
// Some but not all of the partition key columns have been specified;
// hence we need turn these restrictions into a row filter.
- if (usesSecondaryIndexing || partitionKeyRestrictions.needFiltering(cfm))
+ if (usesSecondaryIndexing || partitionKeyRestrictions.needFiltering(table))
filterRestrictions.add(partitionKeyRestrictions);
if (selectsOnlyStaticColumns && hasClusteringColumnsRestrictions())
@@ -229,7 +234,6 @@
processClusteringColumnsRestrictions(hasQueriableIndex,
selectsOnlyStaticColumns,
- selectsComplexColumn,
forView,
allowFiltering);
@@ -247,22 +251,15 @@
if (!type.allowNonPrimaryKeyInWhereClause())
{
Collection<ColumnIdentifier> nonPrimaryKeyColumns =
- ColumnDefinition.toIdentifiers(nonPrimaryKeyRestrictions.getColumnDefs());
+ ColumnMetadata.toIdentifiers(nonPrimaryKeyRestrictions.getColumnDefs());
throw invalidRequest("Non PRIMARY KEY columns found in where clause: %s ",
Joiner.on(", ").join(nonPrimaryKeyColumns));
}
if (hasQueriableIndex)
- {
usesSecondaryIndexing = true;
- }
- else if (!allowFiltering && !cfm.isSuper())
- {
+ else if (!allowFiltering)
throw invalidRequest(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE);
- }
-
- checkFalse(clusteringColumnsRestrictions.isEmpty() && cfm.isSuper(),
- "Filtering is not supported on SuperColumn tables");
filterRestrictions.add(nonPrimaryKeyRestrictions);
}
@@ -273,7 +270,7 @@
private void addRestriction(Restriction restriction)
{
- ColumnDefinition def = restriction.getFirstColumn();
+ ColumnMetadata def = restriction.getFirstColumn();
if (def.isPartitionKey())
partitionKeyRestrictions = partitionKeyRestrictions.mergeWith(restriction);
else if (def.isClusteringColumn())
@@ -300,19 +297,19 @@
* by an IS NOT NULL restriction will be included, otherwise they will not be included (unless another restriction
* applies to them).
*/
- public Set<ColumnDefinition> nonPKRestrictedColumns(boolean includeNotNullRestrictions)
+ public Set<ColumnMetadata> nonPKRestrictedColumns(boolean includeNotNullRestrictions)
{
- Set<ColumnDefinition> columns = new HashSet<>();
+ Set<ColumnMetadata> columns = new HashSet<>();
for (Restrictions r : filterRestrictions.getRestrictions())
{
- for (ColumnDefinition def : r.getColumnDefs())
+ for (ColumnMetadata def : r.getColumnDefs())
if (!def.isPrimaryKeyColumn())
columns.add(def);
}
if (includeNotNullRestrictions)
{
- for (ColumnDefinition def : notNullColumns)
+ for (ColumnMetadata def : notNullColumns)
{
if (!def.isPrimaryKeyColumn())
columns.add(def);
@@ -325,7 +322,7 @@
/**
* @return the set of columns that have an IS NOT NULL restriction on them
*/
- public Set<ColumnDefinition> notNullColumns()
+ public Set<ColumnMetadata> notNullColumns()
{
return notNullColumns;
}
@@ -333,7 +330,7 @@
/**
* @return true if column is restricted by some restriction, false otherwise
*/
- public boolean isRestricted(ColumnDefinition column)
+ public boolean isRestricted(ColumnMetadata column)
{
if (notNullColumns.contains(column))
return true;
@@ -369,7 +366,7 @@
* @return <code>true</code> if the specified column is restricted by an EQ restiction, <code>false</code>
* otherwise.
*/
- public boolean isColumnRestrictedByEq(ColumnDefinition columnDef)
+ public boolean isColumnRestrictedByEq(ColumnMetadata columnDef)
{
Set<Restriction> restrictions = getRestrictions(columnDef.kind).getRestrictions(columnDef);
return restrictions.stream()
@@ -383,7 +380,7 @@
* @param kind the column type
* @return the <code>Restrictions</code> for the specified type of columns
*/
- private Restrictions getRestrictions(ColumnDefinition.Kind kind)
+ private Restrictions getRestrictions(ColumnMetadata.Kind kind)
{
switch (kind)
{
@@ -410,7 +407,7 @@
checkFalse(partitionKeyRestrictions.isOnToken(),
"The token function cannot be used in WHERE clauses for %s statements", type);
- if (partitionKeyRestrictions.hasUnrestrictedPartitionKeyComponents(cfm))
+ if (partitionKeyRestrictions.hasUnrestrictedPartitionKeyComponents(table))
throw invalidRequest("Some partition key parts are missing: %s",
Joiner.on(", ").join(getPartitionKeyUnrestrictedComponents()));
@@ -425,7 +422,7 @@
if (partitionKeyRestrictions.isOnToken())
isKeyRange = true;
- if (partitionKeyRestrictions.isEmpty() && partitionKeyRestrictions.hasUnrestrictedPartitionKeyComponents(cfm))
+ if (partitionKeyRestrictions.isEmpty() && partitionKeyRestrictions.hasUnrestrictedPartitionKeyComponents(table))
{
isKeyRange = true;
usesSecondaryIndexing = hasQueriableIndex;
@@ -437,7 +434,7 @@
// - Is it queriable without 2ndary index, which is always more efficient
// If a component of the partition key is restricted by a relation, all preceding
// components must have a EQ. Only the last partition key component can be in IN relation.
- if (partitionKeyRestrictions.needFiltering(cfm))
+ if (partitionKeyRestrictions.needFiltering(table))
{
if (!allowFiltering && !forView && !hasQueriableIndex)
throw new InvalidRequestException(REQUIRES_ALLOW_FILTERING_MESSAGE);
@@ -471,9 +468,9 @@
*/
private Collection<ColumnIdentifier> getPartitionKeyUnrestrictedComponents()
{
- List<ColumnDefinition> list = new ArrayList<>(cfm.partitionKeyColumns());
+ List<ColumnMetadata> list = new ArrayList<>(table.partitionKeyColumns());
list.removeAll(partitionKeyRestrictions.getColumnDefs());
- return ColumnDefinition.toIdentifiers(list);
+ return ColumnMetadata.toIdentifiers(list);
}
/**
@@ -504,11 +501,9 @@
* @param hasQueriableIndex <code>true</code> if some of the queried data are indexed, <code>false</code> otherwise
* @param selectsOnlyStaticColumns <code>true</code> if the selected or modified columns are all statics,
* <code>false</code> otherwise.
- * @param selectsComplexColumn <code>true</code> if the query should return a collection column
*/
private void processClusteringColumnsRestrictions(boolean hasQueriableIndex,
boolean selectsOnlyStaticColumns,
- boolean selectsComplexColumn,
boolean forView,
boolean allowFiltering)
{
@@ -516,7 +511,7 @@
"Slice restrictions are not supported on the clustering columns in %s statements", type);
if (!type.allowClusteringColumnSlices()
- && (!cfm.isCompactTable() || (cfm.isCompactTable() && !hasClusteringColumnsRestrictions())))
+ && (!table.isCompactTable() || (table.isCompactTable() && !hasClusteringColumnsRestrictions())))
{
if (!selectsOnlyStaticColumns && hasUnrestrictedClusteringColumns())
throw invalidRequest("Some clustering keys are missing: %s",
@@ -524,8 +519,6 @@
}
else
{
- checkFalse(clusteringColumnsRestrictions.hasIN() && selectsComplexColumn,
- "Cannot restrict clustering columns by IN relations when a collection is selected by the query");
checkFalse(clusteringColumnsRestrictions.hasContains() && !hasQueriableIndex && !allowFiltering,
"Clustering columns can only be restricted with CONTAINS with a secondary index or filtering");
@@ -537,13 +530,13 @@
}
else if (!allowFiltering)
{
- List<ColumnDefinition> clusteringColumns = cfm.clusteringColumns();
- List<ColumnDefinition> restrictedColumns = new LinkedList<>(clusteringColumnsRestrictions.getColumnDefs());
+ List<ColumnMetadata> clusteringColumns = table.clusteringColumns();
+ List<ColumnMetadata> restrictedColumns = new LinkedList<>(clusteringColumnsRestrictions.getColumnDefs());
for (int i = 0, m = restrictedColumns.size(); i < m; i++)
{
- ColumnDefinition clusteringColumn = clusteringColumns.get(i);
- ColumnDefinition restrictedColumn = restrictedColumns.get(i);
+ ColumnMetadata clusteringColumn = clusteringColumns.get(i);
+ ColumnMetadata restrictedColumn = restrictedColumns.get(i);
if (!clusteringColumn.equals(restrictedColumn))
{
@@ -565,9 +558,9 @@
*/
private Collection<ColumnIdentifier> getUnrestrictedClusteringColumns()
{
- List<ColumnDefinition> missingClusteringColumns = new ArrayList<>(cfm.clusteringColumns());
+ List<ColumnMetadata> missingClusteringColumns = new ArrayList<>(table.clusteringColumns());
missingClusteringColumns.removeAll(new LinkedList<>(clusteringColumnsRestrictions.getColumnDefs()));
- return ColumnDefinition.toIdentifiers(missingClusteringColumns);
+ return ColumnMetadata.toIdentifiers(missingClusteringColumns);
}
/**
@@ -576,34 +569,27 @@
*/
private boolean hasUnrestrictedClusteringColumns()
{
- return cfm.clusteringColumns().size() != clusteringColumnsRestrictions.size();
+ return table.clusteringColumns().size() != clusteringColumnsRestrictions.size();
}
private void processCustomIndexExpressions(List<CustomIndexExpression> expressions,
VariableSpecifications boundNames,
- SecondaryIndexManager indexManager)
+ IndexRegistry indexRegistry)
{
- if (!MessagingService.instance().areAllNodesAtLeast30())
- throw new InvalidRequestException("Please upgrade all nodes to at least 3.0 before using custom index expressions");
-
if (expressions.size() > 1)
throw new InvalidRequestException(IndexRestrictions.MULTIPLE_EXPRESSIONS);
CustomIndexExpression expression = expressions.get(0);
- CFName cfName = expression.targetIndex.getCfName();
- if (cfName.hasKeyspace()
- && !expression.targetIndex.getKeyspace().equals(cfm.ksName))
- throw IndexRestrictions.invalidIndex(expression.targetIndex, cfm);
+ QualifiedName name = expression.targetIndex;
- if (cfName.getColumnFamily() != null && !cfName.getColumnFamily().equals(cfm.cfName))
- throw IndexRestrictions.invalidIndex(expression.targetIndex, cfm);
+ if (name.hasKeyspace() && !name.getKeyspace().equals(table.keyspace))
+ throw IndexRestrictions.invalidIndex(expression.targetIndex, table);
- if (!cfm.getIndexes().has(expression.targetIndex.getIdx()))
- throw IndexRestrictions.indexNotFound(expression.targetIndex, cfm);
+ if (!table.indexes.has(expression.targetIndex.getName()))
+ throw IndexRestrictions.indexNotFound(expression.targetIndex, table);
- Index index = indexManager.getIndex(cfm.getIndexes().get(expression.targetIndex.getIdx()).get());
-
+ Index index = indexRegistry.getIndex(table.indexes.get(expression.targetIndex.getName()).get());
if (!index.getIndexMetadata().isCustom())
throw IndexRestrictions.nonCustomIndexInExpression(expression.targetIndex);
@@ -611,22 +597,22 @@
if (expressionType == null)
throw IndexRestrictions.customExpressionNotSupported(expression.targetIndex);
- expression.prepareValue(cfm, expressionType, boundNames);
+ expression.prepareValue(table, expressionType, boundNames);
filterRestrictions.add(expression);
}
- public RowFilter getRowFilter(SecondaryIndexManager indexManager, QueryOptions options)
+ public RowFilter getRowFilter(IndexRegistry indexRegistry, QueryOptions options)
{
if (filterRestrictions.isEmpty())
return RowFilter.NONE;
RowFilter filter = RowFilter.create();
for (Restrictions restrictions : filterRestrictions.getRestrictions())
- restrictions.addRowFilterTo(filter, indexManager, options);
+ restrictions.addRowFilterTo(filter, indexRegistry, options);
for (CustomIndexExpression expression : filterRestrictions.getCustomIndexExpressions())
- expression.addToRowFilter(filter, cfm, options);
+ expression.addToRowFilter(filter, table, options);
return filter;
}
@@ -663,7 +649,7 @@
*/
public AbstractBounds<PartitionPosition> getPartitionKeyBounds(QueryOptions options)
{
- IPartitioner p = cfm.partitioner;
+ IPartitioner p = table.partitioner;
if (partitionKeyRestrictions.isOnToken())
{
@@ -678,7 +664,7 @@
{
// Deal with unrestricted partition key components (special-casing is required to deal with 2i queries on the
// first component of a composite partition key) queries that filter on the partition key.
- if (partitionKeyRestrictions.needFiltering(cfm))
+ if (partitionKeyRestrictions.needFiltering(table))
return new Range<>(p.getMinimumToken().minKeyBound(), p.getMinimumToken().maxKeyBound());
ByteBuffer startKeyBytes = getPartitionKeyBound(Bound.START, options);
@@ -764,8 +750,8 @@
// If this is a names command and the table is a static compact one, then as far as CQL is concerned we have
// only a single row which internally correspond to the static parts. In which case we want to return an empty
// set (since that's what ClusteringIndexNamesFilter expects).
- if (cfm.isStaticCompactTable())
- return BTreeSet.empty(cfm.comparator);
+ if (table.isStaticCompactTable())
+ return BTreeSet.empty(table.comparator);
return clusteringColumnsRestrictions.valuesAsClustering(options);
}
@@ -792,7 +778,7 @@
// For static compact tables we want to ignore the fake clustering column (note that if we weren't special casing,
// this would mean a 'SELECT *' on a static compact table would query whole partitions, even though we'll only return
// the static part as far as CQL is concerned. This is thus mostly an optimization to use the query-by-name path).
- int numberOfClusteringColumns = cfm.isStaticCompactTable() ? 0 : cfm.clusteringColumns().size();
+ int numberOfClusteringColumns = table.isStaticCompactTable() ? 0 : table.clusteringColumns().size();
// it is a range query if it has at least one the column alias for which no relation is defined or is not EQ or IN.
return clusteringColumnsRestrictions.size() < numberOfClusteringColumns
|| !clusteringColumnsRestrictions.hasOnlyEqualityRestrictions();
@@ -829,7 +815,7 @@
public boolean hasAllPKColumnsRestrictedByEqualities()
{
return !isPartitionKeyRestrictionsOnToken()
- && !partitionKeyRestrictions.hasUnrestrictedPartitionKeyComponents(cfm)
+ && !partitionKeyRestrictions.hasUnrestrictedPartitionKeyComponents(table)
&& (partitionKeyRestrictions.hasOnlyEqualityRestrictions())
&& !hasUnrestrictedClusteringColumns()
&& (clusteringColumnsRestrictions.hasOnlyEqualityRestrictions());
@@ -843,15 +829,10 @@
{
return hasRegularColumnsRestrictions;
}
-
- private SuperColumnCompatibility.SuperColumnRestrictions cached;
- public SuperColumnCompatibility.SuperColumnRestrictions getSuperColumnRestrictions()
+
+ @Override
+ public String toString()
{
- assert cfm.isSuper() && cfm.isDense();
-
- if (cached == null)
- cached = new SuperColumnCompatibility.SuperColumnRestrictions(Iterators.concat(clusteringColumnsRestrictions.iterator(),
- nonPrimaryKeyRestrictions.iterator()));
- return cached;
+ return ToStringBuilder.reflectionToString(this, ToStringStyle.SHORT_PREFIX_STYLE);
}
}
diff --git a/src/java/org/apache/cassandra/cql3/restrictions/TermSlice.java b/src/java/org/apache/cassandra/cql3/restrictions/TermSlice.java
index c7a6daf..100fcef 100644
--- a/src/java/org/apache/cassandra/cql3/restrictions/TermSlice.java
+++ b/src/java/org/apache/cassandra/cql3/restrictions/TermSlice.java
@@ -19,14 +19,14 @@
import java.util.List;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.cql3.Operator;
import org.apache.cassandra.cql3.Term;
import org.apache.cassandra.cql3.functions.Function;
import org.apache.cassandra.cql3.statements.Bound;
import org.apache.cassandra.index.Index;
-public final class TermSlice
+final class TermSlice
{
/**
* The slice boundaries.
@@ -155,7 +155,7 @@
* @return <code>true</code> this type of <code>TermSlice</code> is supported by the specified index,
* <code>false</code> otherwise.
*/
- public boolean isSupportedBy(ColumnDefinition column, Index index)
+ public boolean isSupportedBy(ColumnMetadata column, Index index)
{
boolean supported = false;
diff --git a/src/java/org/apache/cassandra/cql3/restrictions/TokenFilter.java b/src/java/org/apache/cassandra/cql3/restrictions/TokenFilter.java
index 13b3e3e..437b17c 100644
--- a/src/java/org/apache/cassandra/cql3/restrictions/TokenFilter.java
+++ b/src/java/org/apache/cassandra/cql3/restrictions/TokenFilter.java
@@ -25,8 +25,8 @@
import com.google.common.collect.Range;
import com.google.common.collect.RangeSet;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.cql3.QueryOptions;
import org.apache.cassandra.cql3.functions.Function;
import org.apache.cassandra.cql3.statements.Bound;
@@ -34,7 +34,7 @@
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.exceptions.InvalidRequestException;
-import org.apache.cassandra.index.SecondaryIndexManager;
+import org.apache.cassandra.index.IndexRegistry;
import static org.apache.cassandra.cql3.statements.Bound.END;
import static org.apache.cassandra.cql3.statements.Bound.START;
@@ -78,7 +78,7 @@
}
@Override
- public Set<Restriction> getRestrictions(ColumnDefinition columnDef)
+ public Set<Restriction> getRestrictions(ColumnMetadata columnDef)
{
Set<Restriction> set = new HashSet<>();
set.addAll(restrictions.getRestrictions(columnDef));
@@ -248,19 +248,19 @@
}
@Override
- public ColumnDefinition getFirstColumn()
+ public ColumnMetadata getFirstColumn()
{
return restrictions.getFirstColumn();
}
@Override
- public ColumnDefinition getLastColumn()
+ public ColumnMetadata getLastColumn()
{
return restrictions.getLastColumn();
}
@Override
- public List<ColumnDefinition> getColumnDefs()
+ public List<ColumnMetadata> getColumnDefs()
{
return restrictions.getColumnDefs();
}
@@ -272,15 +272,15 @@
}
@Override
- public boolean hasSupportingIndex(SecondaryIndexManager indexManager)
+ public boolean hasSupportingIndex(IndexRegistry indexRegistry)
{
- return restrictions.hasSupportingIndex(indexManager);
+ return restrictions.hasSupportingIndex(indexRegistry);
}
@Override
- public void addRowFilterTo(RowFilter filter, SecondaryIndexManager indexManager, QueryOptions options)
+ public void addRowFilterTo(RowFilter filter, IndexRegistry indexRegistry, QueryOptions options)
{
- restrictions.addRowFilterTo(filter, indexManager, options);
+ restrictions.addRowFilterTo(filter, indexRegistry, options);
}
@Override
@@ -296,15 +296,15 @@
}
@Override
- public boolean needFiltering(CFMetaData cfm)
+ public boolean needFiltering(TableMetadata table)
{
- return restrictions.needFiltering(cfm);
+ return restrictions.needFiltering(table);
}
@Override
- public boolean hasUnrestrictedPartitionKeyComponents(CFMetaData cfm)
+ public boolean hasUnrestrictedPartitionKeyComponents(TableMetadata table)
{
- return restrictions.hasUnrestrictedPartitionKeyComponents(cfm);
+ return restrictions.hasUnrestrictedPartitionKeyComponents(table);
}
@Override
diff --git a/src/java/org/apache/cassandra/cql3/restrictions/TokenRestriction.java b/src/java/org/apache/cassandra/cql3/restrictions/TokenRestriction.java
index 82b27dd..e71b177 100644
--- a/src/java/org/apache/cassandra/cql3/restrictions/TokenRestriction.java
+++ b/src/java/org/apache/cassandra/cql3/restrictions/TokenRestriction.java
@@ -22,15 +22,15 @@
import com.google.common.base.Joiner;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.cql3.QueryOptions;
import org.apache.cassandra.cql3.Term;
import org.apache.cassandra.cql3.functions.Function;
import org.apache.cassandra.cql3.statements.Bound;
import org.apache.cassandra.db.filter.RowFilter;
import org.apache.cassandra.exceptions.InvalidRequestException;
-import org.apache.cassandra.index.SecondaryIndexManager;
+import org.apache.cassandra.index.IndexRegistry;
import static org.apache.cassandra.cql3.statements.RequestValidations.invalidRequest;
@@ -42,16 +42,16 @@
/**
* The definition of the columns to which apply the token restriction.
*/
- protected final List<ColumnDefinition> columnDefs;
+ protected final List<ColumnMetadata> columnDefs;
- protected final CFMetaData metadata;
+ protected final TableMetadata metadata;
/**
* Creates a new <code>TokenRestriction</code> that apply to the specified columns.
*
* @param columnDefs the definition of the columns to which apply the token restriction
*/
- public TokenRestriction(CFMetaData metadata, List<ColumnDefinition> columnDefs)
+ public TokenRestriction(TableMetadata metadata, List<ColumnMetadata> columnDefs)
{
this.columnDefs = columnDefs;
this.metadata = metadata;
@@ -68,7 +68,7 @@
}
@Override
- public Set<Restriction> getRestrictions(ColumnDefinition columnDef)
+ public Set<Restriction> getRestrictions(ColumnMetadata columnDef)
{
return Collections.singleton(this);
}
@@ -80,7 +80,7 @@
}
@Override
- public boolean needFiltering(CFMetaData cfm)
+ public boolean needFiltering(TableMetadata table)
{
return false;
}
@@ -92,37 +92,37 @@
}
@Override
- public boolean hasUnrestrictedPartitionKeyComponents(CFMetaData cfm)
+ public boolean hasUnrestrictedPartitionKeyComponents(TableMetadata table)
{
return false;
}
@Override
- public List<ColumnDefinition> getColumnDefs()
+ public List<ColumnMetadata> getColumnDefs()
{
return columnDefs;
}
@Override
- public ColumnDefinition getFirstColumn()
+ public ColumnMetadata getFirstColumn()
{
return columnDefs.get(0);
}
@Override
- public ColumnDefinition getLastColumn()
+ public ColumnMetadata getLastColumn()
{
return columnDefs.get(columnDefs.size() - 1);
}
@Override
- public boolean hasSupportingIndex(SecondaryIndexManager secondaryIndexManager)
+ public boolean hasSupportingIndex(IndexRegistry indexRegistry)
{
return false;
}
@Override
- public void addRowFilterTo(RowFilter filter, SecondaryIndexManager indexManager, QueryOptions options)
+ public void addRowFilterTo(RowFilter filter, IndexRegistry indexRegistry, QueryOptions options)
{
throw new UnsupportedOperationException("Index expression cannot be created for token restriction");
}
@@ -146,7 +146,7 @@
*/
protected final String getColumnNamesAsString()
{
- return Joiner.on(", ").join(ColumnDefinition.toIdentifiers(columnDefs));
+ return Joiner.on(", ").join(ColumnMetadata.toIdentifiers(columnDefs));
}
@Override
@@ -176,16 +176,16 @@
if (restriction instanceof PartitionKeyRestrictions)
return (PartitionKeyRestrictions) restriction;
- return new PartitionKeySingleRestrictionSet(metadata.getKeyValidatorAsClusteringComparator()).mergeWith(restriction);
+ return new PartitionKeySingleRestrictionSet(metadata.partitionKeyAsClusteringComparator()).mergeWith(restriction);
}
public static final class EQRestriction extends TokenRestriction
{
private final Term value;
- public EQRestriction(CFMetaData cfm, List<ColumnDefinition> columnDefs, Term value)
+ public EQRestriction(TableMetadata table, List<ColumnMetadata> columnDefs, Term value)
{
- super(cfm, columnDefs);
+ super(table, columnDefs);
this.value = value;
}
@@ -199,7 +199,7 @@
protected PartitionKeyRestrictions doMergeWith(TokenRestriction otherRestriction) throws InvalidRequestException
{
throw invalidRequest("%s cannot be restricted by more than one relation if it includes an Equal",
- Joiner.on(", ").join(ColumnDefinition.toIdentifiers(columnDefs)));
+ Joiner.on(", ").join(ColumnMetadata.toIdentifiers(columnDefs)));
}
@Override
@@ -236,9 +236,9 @@
{
private final TermSlice slice;
- public SliceRestriction(CFMetaData cfm, List<ColumnDefinition> columnDefs, Bound bound, boolean inclusive, Term term)
+ public SliceRestriction(TableMetadata table, List<ColumnMetadata> columnDefs, Bound bound, boolean inclusive, Term term)
{
- super(cfm, columnDefs);
+ super(table, columnDefs);
slice = TermSlice.newInstance(bound, inclusive, term);
}
@@ -301,7 +301,7 @@
throw invalidRequest("More than one restriction was found for the end bound on %s",
getColumnNamesAsString());
- return new SliceRestriction(metadata, columnDefs, slice.merge(otherSlice.slice));
+ return new SliceRestriction(metadata, columnDefs, slice.merge(otherSlice.slice));
}
@Override
@@ -309,9 +309,9 @@
{
return String.format("SLICE%s", slice);
}
- private SliceRestriction(CFMetaData cfm, List<ColumnDefinition> columnDefs, TermSlice slice)
+ private SliceRestriction(TableMetadata table, List<ColumnMetadata> columnDefs, TermSlice slice)
{
- super(cfm, columnDefs);
+ super(table, columnDefs);
this.slice = slice;
}
}
diff --git a/src/java/org/apache/cassandra/cql3/selection/AbstractFunctionSelector.java b/src/java/org/apache/cassandra/cql3/selection/AbstractFunctionSelector.java
index 498cf0f..d420857 100644
--- a/src/java/org/apache/cassandra/cql3/selection/AbstractFunctionSelector.java
+++ b/src/java/org/apache/cassandra/cql3/selection/AbstractFunctionSelector.java
@@ -21,12 +21,15 @@
import java.util.Arrays;
import java.util.List;
+import com.google.common.collect.Iterables;
+
import org.apache.commons.lang3.text.StrBuilder;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.cql3.ColumnSpecification;
import org.apache.cassandra.cql3.QueryOptions;
import org.apache.cassandra.cql3.functions.Function;
import org.apache.cassandra.cql3.statements.RequestValidations;
+import org.apache.cassandra.db.filter.ColumnFilter;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.exceptions.InvalidRequestException;
@@ -70,7 +73,7 @@
if (tmpMapping.getMappings().get(resultsColumn).isEmpty())
// add a null mapping for cases where there are no
// further selectors, such as no-arg functions and count
- mapping.addMapping(resultsColumn, (ColumnDefinition)null);
+ mapping.addMapping(resultsColumn, (ColumnMetadata)null);
else
// collate the mapped columns from the child factories & add those
mapping.addMapping(resultsColumn, tmpMapping.getMappings().values());
@@ -102,6 +105,19 @@
{
return fun.isAggregate() || factories.doesAggregation();
}
+
+ @Override
+ public boolean areAllFetchedColumnsKnown()
+ {
+ return Iterables.all(factories, f -> f.areAllFetchedColumnsKnown());
+ }
+
+ @Override
+ public void addFetchedColumns(ColumnFilter.Builder builder)
+ {
+ for (Selector.Factory factory : factories)
+ factory.addFetchedColumns(builder);
+ }
};
}
@@ -112,6 +128,13 @@
this.args = Arrays.asList(new ByteBuffer[argSelectors.size()]);
}
+ @Override
+ public void addFetchedColumns(ColumnFilter.Builder builder)
+ {
+ for (Selector selector : argSelectors)
+ selector.addFetchedColumns(builder);
+ }
+
// Sets a given arg value. We should use that instead of directly setting the args list for the
// sake of validation.
protected void setArg(int i, ByteBuffer value) throws InvalidRequestException
diff --git a/src/java/org/apache/cassandra/cql3/selection/AggregateFunctionSelector.java b/src/java/org/apache/cassandra/cql3/selection/AggregateFunctionSelector.java
index e3e5328..a9df220 100644
--- a/src/java/org/apache/cassandra/cql3/selection/AggregateFunctionSelector.java
+++ b/src/java/org/apache/cassandra/cql3/selection/AggregateFunctionSelector.java
@@ -22,7 +22,6 @@
import org.apache.cassandra.cql3.functions.AggregateFunction;
import org.apache.cassandra.cql3.functions.Function;
-import org.apache.cassandra.cql3.selection.Selection.ResultSetBuilder;
import org.apache.cassandra.exceptions.InvalidRequestException;
import org.apache.cassandra.transport.ProtocolVersion;
diff --git a/src/java/org/apache/cassandra/cql3/selection/AliasedSelectable.java b/src/java/org/apache/cassandra/cql3/selection/AliasedSelectable.java
new file mode 100644
index 0000000..3cf0029
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/selection/AliasedSelectable.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.selection;
+
+import java.util.List;
+import java.util.function.Predicate;
+
+import org.apache.cassandra.cql3.ColumnIdentifier;
+import org.apache.cassandra.cql3.ColumnSpecification;
+import org.apache.cassandra.cql3.VariableSpecifications;
+import org.apache.cassandra.cql3.selection.Selector.Factory;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableMetadata;
+
+/**
+ * A {@code Selectable} with an alias.
+ */
+final class AliasedSelectable implements Selectable
+{
+ /**
+ * The selectable
+ */
+ private final Selectable selectable;
+
+ /**
+ * The alias associated to the selectable.
+ */
+ private final ColumnIdentifier alias;
+
+ public AliasedSelectable(Selectable selectable, ColumnIdentifier alias)
+ {
+ this.selectable = selectable;
+ this.alias = alias;
+ }
+
+ @Override
+ public TestResult testAssignment(String keyspace, ColumnSpecification receiver)
+ {
+ return selectable.testAssignment(keyspace, receiver);
+ }
+
+ @Override
+ public Factory newSelectorFactory(TableMetadata table,
+ AbstractType<?> expectedType,
+ List<ColumnMetadata> defs,
+ VariableSpecifications boundNames)
+ {
+ final Factory delegate = selectable.newSelectorFactory(table, expectedType, defs, boundNames);
+ final ColumnSpecification columnSpec = delegate.getColumnSpecification(table).withAlias(alias);
+
+ return new ForwardingFactory()
+ {
+ @Override
+ protected Factory delegate()
+ {
+ return delegate;
+ }
+
+ @Override
+ public ColumnSpecification getColumnSpecification(TableMetadata table)
+ {
+ return columnSpec;
+ }
+ };
+ }
+
+ @Override
+ public AbstractType<?> getExactTypeIfKnown(String keyspace)
+ {
+ return selectable.getExactTypeIfKnown(keyspace);
+ }
+
+ @Override
+ public boolean selectColumns(Predicate<ColumnMetadata> predicate)
+ {
+ return selectable.selectColumns(predicate);
+ }
+}
diff --git a/src/java/org/apache/cassandra/cql3/selection/CollectionFactory.java b/src/java/org/apache/cassandra/cql3/selection/CollectionFactory.java
new file mode 100644
index 0000000..816980d
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/selection/CollectionFactory.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.selection;
+
+import java.util.List;
+
+import org.apache.cassandra.cql3.ColumnSpecification;
+import org.apache.cassandra.cql3.functions.Function;
+import org.apache.cassandra.cql3.selection.Selector.Factory;
+import org.apache.cassandra.db.filter.ColumnFilter.Builder;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.schema.ColumnMetadata;
+
+/**
+ * A base <code>Selector.Factory</code> for collections or tuples.
+ */
+abstract class CollectionFactory extends Factory
+{
+ /**
+ * The collection or tuple type.
+ */
+ private final AbstractType<?> type;
+
+ /**
+ * The collection or tuple element factories.
+ */
+ private final SelectorFactories factories;
+
+ public CollectionFactory(AbstractType<?> type, SelectorFactories factories)
+ {
+ this.type = type;
+ this.factories = factories;
+ }
+
+ protected final AbstractType<?> getReturnType()
+ {
+ return type;
+ }
+
+ @Override
+ public final void addFunctionsTo(List<Function> functions)
+ {
+ factories.addFunctionsTo(functions);
+ }
+
+ @Override
+ public final boolean isAggregateSelectorFactory()
+ {
+ return factories.doesAggregation();
+ }
+
+ @Override
+ public final boolean isWritetimeSelectorFactory()
+ {
+ return factories.containsWritetimeSelectorFactory();
+ }
+
+ @Override
+ public final boolean isTTLSelectorFactory()
+ {
+ return factories.containsTTLSelectorFactory();
+ }
+
+ @Override
+ boolean areAllFetchedColumnsKnown()
+ {
+ return factories.areAllFetchedColumnsKnown();
+ }
+
+ @Override
+ void addFetchedColumns(Builder builder)
+ {
+ factories.addFetchedColumns(builder);
+ }
+
+ protected final void addColumnMapping(SelectionColumnMapping mapping, ColumnSpecification resultsColumn)
+ {
+ SelectionColumnMapping tmpMapping = SelectionColumnMapping.newMapping();
+ for (Factory factory : factories)
+ factory.addColumnMapping(tmpMapping, resultsColumn);
+
+ if (tmpMapping.getMappings().get(resultsColumn).isEmpty())
+ // add a null mapping for cases where the collection is empty
+ mapping.addMapping(resultsColumn, (ColumnMetadata)null);
+ else
+ // collate the mapped columns from the child factories & add those
+ mapping.addMapping(resultsColumn, tmpMapping.getMappings().values());
+ }
+}
diff --git a/src/java/org/apache/cassandra/cql3/selection/ColumnFilterFactory.java b/src/java/org/apache/cassandra/cql3/selection/ColumnFilterFactory.java
new file mode 100644
index 0000000..2e5d0df
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/selection/ColumnFilterFactory.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.selection;
+
+import java.util.List;
+import java.util.Set;
+
+import org.apache.cassandra.db.filter.ColumnFilter;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableMetadata;
+
+/**
+ * Factory for {@code ColumnFilter} instances.
+ * <p>This class is used to abstract the fact that depending on the selection clause the {@code ColumnFilter} instances
+ * can be computed at prepartion time (if all the requested columns are known) or must be computed at execution time.</p>
+ */
+abstract class ColumnFilterFactory
+{
+ /**
+ * Returns the {@code ColumnFilter} instance corresponding to the specified selectors.
+ * @param selectors the selectors for which the {@code ColumnFilter} must be created.
+ * @return the {@code ColumnFilter} instance corresponding to the specified selectors
+ */
+ abstract ColumnFilter newInstance(List<Selector> selectors);
+
+ public static ColumnFilterFactory wildcard(TableMetadata table)
+ {
+ return new PrecomputedColumnFilter(ColumnFilter.all(table));
+ }
+
+ public static ColumnFilterFactory fromColumns(TableMetadata table,
+ List<ColumnMetadata> selectedColumns,
+ Set<ColumnMetadata> orderingColumns,
+ Set<ColumnMetadata> nonPKRestrictedColumns)
+ {
+ ColumnFilter.Builder builder = ColumnFilter.allRegularColumnsBuilder(table);
+ builder.addAll(selectedColumns);
+ builder.addAll(orderingColumns);
+ // we'll also need to fetch any column on which we have a restriction (so we can apply said restriction)
+ builder.addAll(nonPKRestrictedColumns);
+ return new PrecomputedColumnFilter(builder.build());
+ }
+
+ /**
+ * Creates a new {@code ColumnFilterFactory} instance from the specified {@code SelectorFactories}.
+ *
+ * @param table the table metadata
+ * @param factories the {@code SelectorFactories}
+ * @param orderingColumns the columns used for ordering
+ * @param nonPKRestrictedColumns the non primary key columns that have been resticted in the WHERE clause
+ * @return a new {@code ColumnFilterFactory} instance
+ */
+ public static ColumnFilterFactory fromSelectorFactories(TableMetadata table,
+ SelectorFactories factories,
+ Set<ColumnMetadata> orderingColumns,
+ Set<ColumnMetadata> nonPKRestrictedColumns)
+ {
+ if (factories.areAllFetchedColumnsKnown())
+ {
+ ColumnFilter.Builder builder = ColumnFilter.allRegularColumnsBuilder(table);
+ factories.addFetchedColumns(builder);
+ builder.addAll(orderingColumns);
+ // we'll also need to fetch any column on which we have a restriction (so we can apply said restriction)
+ builder.addAll(nonPKRestrictedColumns);
+ return new PrecomputedColumnFilter(builder.build());
+ }
+
+ return new OnRequestColumnFilterFactory(table, nonPKRestrictedColumns);
+ }
+
+ /**
+ * A factory that always return the same pre-computed {@code ColumnFilter}.
+ */
+ private static class PrecomputedColumnFilter extends ColumnFilterFactory
+ {
+ /**
+ * The precomputed {@code ColumnFilter}
+ */
+ private final ColumnFilter columnFilter;
+
+ public PrecomputedColumnFilter(ColumnFilter columnFilter)
+ {
+ this.columnFilter = columnFilter;
+ }
+
+ @Override
+ public ColumnFilter newInstance(List<Selector> selectors)
+ {
+ return columnFilter;
+ }
+ }
+
+ /**
+ * A factory that will computed the {@code ColumnFilter} on request.
+ */
+ private static class OnRequestColumnFilterFactory extends ColumnFilterFactory
+ {
+ private final TableMetadata table;
+ private final Set<ColumnMetadata> nonPKRestrictedColumns;
+
+ public OnRequestColumnFilterFactory(TableMetadata table, Set<ColumnMetadata> nonPKRestrictedColumns)
+ {
+ this.table = table;
+ this.nonPKRestrictedColumns = nonPKRestrictedColumns;
+ }
+
+ @Override
+ public ColumnFilter newInstance(List<Selector> selectors)
+ {
+ ColumnFilter.Builder builder = ColumnFilter.allRegularColumnsBuilder(table);
+ for (int i = 0, m = selectors.size(); i < m; i++)
+ selectors.get(i).addFetchedColumns(builder);
+
+ // we'll also need to fetch any column on which we have a restriction (so we can apply said restriction)
+ builder.addAll(nonPKRestrictedColumns);
+ return builder.build();
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/cql3/selection/ElementsSelector.java b/src/java/org/apache/cassandra/cql3/selection/ElementsSelector.java
new file mode 100644
index 0000000..9427c51
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/selection/ElementsSelector.java
@@ -0,0 +1,325 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.selection;
+
+import java.nio.ByteBuffer;
+
+import org.apache.cassandra.cql3.ColumnSpecification;
+import org.apache.cassandra.cql3.QueryOptions;
+import org.apache.cassandra.cql3.Term;
+import org.apache.cassandra.cql3.selection.SimpleSelector.SimpleSelectorFactory;
+import org.apache.cassandra.db.filter.ColumnFilter;
+import org.apache.cassandra.db.marshal.*;
+import org.apache.cassandra.db.rows.CellPath;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.transport.ProtocolVersion;
+import org.apache.cassandra.utils.ByteBufferUtil;
+
+/**
+ * Selector class handling element (c[x]) and slice (c[x..y]) selections over collections.
+ */
+abstract class ElementsSelector extends Selector
+{
+ protected final Selector selected;
+
+ protected ElementsSelector(Selector selected)
+ {
+ this.selected = selected;
+ }
+
+ private static boolean isUnset(ByteBuffer bb)
+ {
+ return bb == ByteBufferUtil.UNSET_BYTE_BUFFER;
+ }
+
+ // For sets and maps, return the type corresponding to the element of a selection (that is, x in c[x]).
+ private static AbstractType<?> keyType(CollectionType<?> type)
+ {
+ return type.nameComparator();
+ }
+
+ // For sets and maps, return the type corresponding to the result of a selection (that is, c[x] in c[x]).
+ public static AbstractType<?> valueType(CollectionType<?> type)
+ {
+ return type instanceof MapType ? type.valueComparator() : type.nameComparator();
+ }
+
+ private static abstract class AbstractFactory extends Factory
+ {
+ protected final String name;
+ protected final Selector.Factory factory;
+ protected final CollectionType<?> type;
+
+ protected AbstractFactory(String name, Selector.Factory factory, CollectionType<?> type)
+ {
+ this.name = name;
+ this.factory = factory;
+ this.type = type;
+ }
+
+ protected String getColumnName()
+ {
+ return name;
+ }
+
+ protected void addColumnMapping(SelectionColumnMapping mapping, ColumnSpecification resultsColumn)
+ {
+ factory.addColumnMapping(mapping, resultsColumn);
+ }
+
+ public boolean isAggregateSelectorFactory()
+ {
+ return factory.isAggregateSelectorFactory();
+ }
+ }
+
+ /**
+ * Creates a {@code Selector.Factory} for the selection of an element of a collection.
+ *
+ * @param name a string representing the selection the factory is for. Something like "c[x]".
+ * @param factory the {@code Selector.Factory} corresponding to the collection on which an element
+ * is selected.
+ * @param type the type of the collection.
+ * @param key the element within the value represented by {@code factory} that is selected.
+ * @return the created factory.
+ */
+ public static Factory newElementFactory(String name, Selector.Factory factory, CollectionType<?> type, final Term key)
+ {
+ return new AbstractFactory(name, factory, type)
+ {
+ protected AbstractType<?> getReturnType()
+ {
+ return valueType(type);
+ }
+
+ public Selector newInstance(QueryOptions options) throws InvalidRequestException
+ {
+ ByteBuffer keyValue = key.bindAndGet(options);
+ if (keyValue == null)
+ throw new InvalidRequestException("Invalid null value for element selection on " + factory.getColumnName());
+ if (keyValue == ByteBufferUtil.UNSET_BYTE_BUFFER)
+ throw new InvalidRequestException("Invalid unset value for element selection on " + factory.getColumnName());
+ return new ElementSelector(factory.newInstance(options), keyValue);
+ }
+
+ public boolean areAllFetchedColumnsKnown()
+ {
+ // If we known all the fetched columns, it means that we don't have to wait execution to create
+ // the ColumnFilter (through addFetchedColumns below).
+ // That's the case if either there is no particular subselection
+ // to add, or if there is one but the selected key is terminal. In other words,
+ // we known all the fetched columns if all the feched columns of the factory are known and either:
+ // 1) the type is frozen (in which case there isn't subselection to do).
+ // 2) the factory (the left-hand-side) isn't a simple column selection (here again, no
+ // subselection we can do).
+ // 3) the element selected is terminal.
+ return factory.areAllFetchedColumnsKnown()
+ && (!type.isMultiCell() || !factory.isSimpleSelectorFactory() || key.isTerminal());
+ }
+
+ public void addFetchedColumns(ColumnFilter.Builder builder)
+ {
+ if (!type.isMultiCell() || !factory.isSimpleSelectorFactory())
+ {
+ factory.addFetchedColumns(builder);
+ return;
+ }
+
+ ColumnMetadata column = ((SimpleSelectorFactory) factory).getColumn();
+ builder.select(column, CellPath.create(((Term.Terminal)key).get(ProtocolVersion.V3)));
+ }
+ };
+ }
+
+ /**
+ * Creates a {@code Selector.Factory} for the selection of a slice of a collection.
+ *
+ * @param name a string representing the selection the factory is for. Something like "c[x..y]".
+ * @param factory the {@code Selector.Factory} corresponding to the collection on which a slice
+ * is selected.
+ * @param type the type of the collection.
+ * @param from the starting bound of the selected slice. This cannot be {@code null} but can be
+ * {@code Constants.UNSET_VALUE} if the slice doesn't have a start.
+ * @param to the ending bound of the selected slice. This cannot be {@code null} but can be
+ * {@code Constants.UNSET_VALUE} if the slice doesn't have an end.
+ * @return the created factory.
+ */
+ public static Factory newSliceFactory(String name, Selector.Factory factory, CollectionType<?> type, final Term from, final Term to)
+ {
+ return new AbstractFactory(name, factory, type)
+ {
+ protected AbstractType<?> getReturnType()
+ {
+ return type;
+ }
+
+ public Selector newInstance(QueryOptions options) throws InvalidRequestException
+ {
+ ByteBuffer fromValue = from.bindAndGet(options);
+ ByteBuffer toValue = to.bindAndGet(options);
+ // Note that we use UNSET values to represent no bound, so null is truly invalid
+ if (fromValue == null || toValue == null)
+ throw new InvalidRequestException("Invalid null value for slice selection on " + factory.getColumnName());
+ return new SliceSelector(factory.newInstance(options), from.bindAndGet(options), to.bindAndGet(options));
+ }
+
+ public boolean areAllFetchedColumnsKnown()
+ {
+ // If we known all the fetched columns, it means that we don't have to wait execution to create
+ // the ColumnFilter (through addFetchedColumns below).
+ // That's the case if either there is no particular subselection
+ // to add, or if there is one but the selected bound are terminal. In other words,
+ // we known all the fetched columns if all the feched columns of the factory are known and either:
+ // 1) the type is frozen (in which case there isn't subselection to do).
+ // 2) the factory (the left-hand-side) isn't a simple column selection (here again, no
+ // subselection we can do).
+ // 3) the bound of the selected slice are terminal.
+ return factory.areAllFetchedColumnsKnown()
+ && (!type.isMultiCell() || !factory.isSimpleSelectorFactory() || (from.isTerminal() && to.isTerminal()));
+ }
+
+ public void addFetchedColumns(ColumnFilter.Builder builder)
+ {
+ if (!type.isMultiCell() || !factory.isSimpleSelectorFactory())
+ {
+ factory.addFetchedColumns(builder);
+ return;
+ }
+
+ ColumnMetadata column = ((SimpleSelectorFactory) factory).getColumn();
+ ByteBuffer fromBB = ((Term.Terminal)from).get(ProtocolVersion.V3);
+ ByteBuffer toBB = ((Term.Terminal)to).get(ProtocolVersion.V3);
+ builder.slice(column, isUnset(fromBB) ? CellPath.BOTTOM : CellPath.create(fromBB), isUnset(toBB) ? CellPath.TOP : CellPath.create(toBB));
+ }
+ };
+ }
+
+ public ByteBuffer getOutput(ProtocolVersion protocolVersion) throws InvalidRequestException
+ {
+ ByteBuffer value = selected.getOutput(protocolVersion);
+ return value == null ? null : extractSelection(value);
+ }
+
+ protected abstract ByteBuffer extractSelection(ByteBuffer collection);
+
+ public void addInput(ProtocolVersion protocolVersion, ResultSetBuilder rs) throws InvalidRequestException
+ {
+ selected.addInput(protocolVersion, rs);
+ }
+
+ public void reset()
+ {
+ selected.reset();
+ }
+
+ private static class ElementSelector extends ElementsSelector
+ {
+ private final CollectionType<?> type;
+ private final ByteBuffer key;
+
+ private ElementSelector(Selector selected, ByteBuffer key)
+ {
+ super(selected);
+ assert selected.getType() instanceof MapType || selected.getType() instanceof SetType : "this shouldn't have passed validation in Selectable";
+ this.type = (CollectionType<?>) selected.getType();
+ this.key = key;
+ }
+
+ public void addFetchedColumns(ColumnFilter.Builder builder)
+ {
+ if (type.isMultiCell() && selected instanceof SimpleSelector)
+ {
+ ColumnMetadata column = ((SimpleSelector)selected).column;
+ builder.select(column, CellPath.create(key));
+ }
+ else
+ {
+ selected.addFetchedColumns(builder);
+ }
+ }
+
+ protected ByteBuffer extractSelection(ByteBuffer collection)
+ {
+ return type.getSerializer().getSerializedValue(collection, key, keyType(type));
+ }
+
+ public AbstractType<?> getType()
+ {
+ return valueType(type);
+ }
+
+ @Override
+ public String toString()
+ {
+ return String.format("%s[%s]", selected, keyType(type).getString(key));
+ }
+ }
+
+ private static class SliceSelector extends ElementsSelector
+ {
+ private final CollectionType<?> type;
+
+ // Note that neither from nor to can be null, but they can both be ByteBufferUtil.UNSET_BYTE_BUFFER to represent no particular bound
+ private final ByteBuffer from;
+ private final ByteBuffer to;
+
+ private SliceSelector(Selector selected, ByteBuffer from, ByteBuffer to)
+ {
+ super(selected);
+ assert selected.getType() instanceof MapType || selected.getType() instanceof SetType : "this shouldn't have passed validation in Selectable";
+ assert from != null && to != null : "We can have unset buffers, but not nulls";
+ this.type = (CollectionType<?>) selected.getType();
+ this.from = from;
+ this.to = to;
+ }
+
+ public void addFetchedColumns(ColumnFilter.Builder builder)
+ {
+ if (type.isMultiCell() && selected instanceof SimpleSelector)
+ {
+ ColumnMetadata column = ((SimpleSelector)selected).column;
+ builder.slice(column, isUnset(from) ? CellPath.BOTTOM : CellPath.create(from), isUnset(to) ? CellPath.TOP : CellPath.create(to));
+ }
+ else
+ {
+ selected.addFetchedColumns(builder);
+ }
+ }
+
+ protected ByteBuffer extractSelection(ByteBuffer collection)
+ {
+ return type.getSerializer().getSliceFromSerialized(collection, from, to, type.nameComparator(), type.isFrozenCollection());
+ }
+
+ public AbstractType<?> getType()
+ {
+ return type;
+ }
+
+ @Override
+ public String toString()
+ {
+ boolean fromUnset = isUnset(from);
+ boolean toUnset = isUnset(to);
+ return fromUnset && toUnset
+ ? selected.toString()
+ : String.format("%s[%s..%s]", selected, fromUnset ? "" : keyType(type).getString(from), toUnset ? "" : keyType(type).getString(to));
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/cql3/selection/FieldSelector.java b/src/java/org/apache/cassandra/cql3/selection/FieldSelector.java
index d4b74ae..c67fc03 100644
--- a/src/java/org/apache/cassandra/cql3/selection/FieldSelector.java
+++ b/src/java/org/apache/cassandra/cql3/selection/FieldSelector.java
@@ -21,7 +21,7 @@
import org.apache.cassandra.cql3.ColumnSpecification;
import org.apache.cassandra.cql3.QueryOptions;
-import org.apache.cassandra.cql3.selection.Selection.ResultSetBuilder;
+import org.apache.cassandra.db.filter.ColumnFilter;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.db.marshal.UserType;
import org.apache.cassandra.exceptions.InvalidRequestException;
@@ -61,9 +61,24 @@
{
return factory.isAggregateSelectorFactory();
}
+
+ public boolean areAllFetchedColumnsKnown()
+ {
+ return factory.areAllFetchedColumnsKnown();
+ }
+
+ public void addFetchedColumns(ColumnFilter.Builder builder)
+ {
+ factory.addFetchedColumns(builder);
+ }
};
}
+ public void addFetchedColumns(ColumnFilter.Builder builder)
+ {
+ selected.addFetchedColumns(builder);
+ }
+
public void addInput(ProtocolVersion protocolVersion, ResultSetBuilder rs) throws InvalidRequestException
{
selected.addInput(protocolVersion, rs);
diff --git a/src/java/org/apache/cassandra/cql3/selection/ForwardingFactory.java b/src/java/org/apache/cassandra/cql3/selection/ForwardingFactory.java
new file mode 100644
index 0000000..cf41e31
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/selection/ForwardingFactory.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.selection;
+
+import java.util.List;
+
+import org.apache.cassandra.cql3.ColumnSpecification;
+import org.apache.cassandra.cql3.QueryOptions;
+import org.apache.cassandra.cql3.functions.Function;
+import org.apache.cassandra.cql3.selection.Selector.Factory;
+import org.apache.cassandra.db.filter.ColumnFilter.Builder;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+
+/**
+ * A <code>Selector.Factory</code> which forwards all its method calls to another factory.
+ * Subclasses should override one or more methods to modify the behavior of the backing factory as desired per
+ * the decorator pattern.
+ */
+abstract class ForwardingFactory extends Factory
+{
+ /**
+ * Returns the backing delegate instance that methods are forwarded to.
+ */
+ protected abstract Factory delegate();
+
+ public Selector newInstance(QueryOptions options) throws InvalidRequestException
+ {
+ return delegate().newInstance(options);
+ }
+
+ protected String getColumnName()
+ {
+ return delegate().getColumnName();
+ }
+
+ protected AbstractType<?> getReturnType()
+ {
+ return delegate().getReturnType();
+ }
+
+ protected void addColumnMapping(SelectionColumnMapping mapping, ColumnSpecification resultsColumn)
+ {
+ delegate().addColumnMapping(mapping, resultsColumn);
+ }
+
+ @Override
+ public void addFunctionsTo(List<Function> functions)
+ {
+ delegate().addFunctionsTo(functions);
+ }
+
+ @Override
+ public boolean isAggregateSelectorFactory()
+ {
+ return delegate().isAggregateSelectorFactory();
+ }
+
+ @Override
+ public boolean isWritetimeSelectorFactory()
+ {
+ return delegate().isWritetimeSelectorFactory();
+ }
+
+ @Override
+ public boolean isTTLSelectorFactory()
+ {
+ return delegate().isTTLSelectorFactory();
+ }
+
+ @Override
+ public boolean isSimpleSelectorFactory()
+ {
+ return delegate().isSimpleSelectorFactory();
+ }
+
+ @Override
+ public boolean isSimpleSelectorFactoryFor(int index)
+ {
+ return delegate().isSimpleSelectorFactoryFor(index);
+ }
+
+ @Override
+ boolean areAllFetchedColumnsKnown()
+ {
+ return delegate().areAllFetchedColumnsKnown();
+ }
+
+ @Override
+ void addFetchedColumns(Builder builder)
+ {
+ delegate().addFetchedColumns(builder);
+ }
+}
diff --git a/src/java/org/apache/cassandra/cql3/selection/ListSelector.java b/src/java/org/apache/cassandra/cql3/selection/ListSelector.java
new file mode 100644
index 0000000..a8c5d5c
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/selection/ListSelector.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.selection;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.cassandra.cql3.QueryOptions;
+import org.apache.cassandra.cql3.Lists;
+import org.apache.cassandra.db.filter.ColumnFilter.Builder;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.serializers.CollectionSerializer;
+import org.apache.cassandra.transport.ProtocolVersion;
+
+/**
+ * <code>Selector</code> for literal list (e.g. [min(value), max(value), count(value)]).
+ *
+ */
+final class ListSelector extends Selector
+{
+ /**
+ * The list type.
+ */
+ private final AbstractType<?> type;
+
+ /**
+ * The list elements
+ */
+ private final List<Selector> elements;
+
+ public static Factory newFactory(final AbstractType<?> type, final SelectorFactories factories)
+ {
+ return new CollectionFactory(type, factories)
+ {
+ protected String getColumnName()
+ {
+ return Lists.listToString(factories, Factory::getColumnName);
+ }
+
+ public Selector newInstance(final QueryOptions options)
+ {
+ return new ListSelector(type, factories.newInstances(options));
+ }
+ };
+ }
+
+ @Override
+ public void addFetchedColumns(Builder builder)
+ {
+ for (int i = 0, m = elements.size(); i < m; i++)
+ elements.get(i).addFetchedColumns(builder);
+ }
+
+ public void addInput(ProtocolVersion protocolVersion, ResultSetBuilder rs) throws InvalidRequestException
+ {
+ for (int i = 0, m = elements.size(); i < m; i++)
+ elements.get(i).addInput(protocolVersion, rs);
+ }
+
+ public ByteBuffer getOutput(ProtocolVersion protocolVersion) throws InvalidRequestException
+ {
+ List<ByteBuffer> buffers = new ArrayList<>(elements.size());
+ for (int i = 0, m = elements.size(); i < m; i++)
+ {
+ buffers.add(elements.get(i).getOutput(protocolVersion));
+ }
+ return CollectionSerializer.pack(buffers, buffers.size(), protocolVersion);
+ }
+
+ public void reset()
+ {
+ for (int i = 0, m = elements.size(); i < m; i++)
+ elements.get(i).reset();
+ }
+
+ public AbstractType<?> getType()
+ {
+ return type;
+ }
+
+ @Override
+ public String toString()
+ {
+ return Lists.listToString(elements);
+ }
+
+ private ListSelector(AbstractType<?> type, List<Selector> elements)
+ {
+ this.type = type;
+ this.elements = elements;
+ }
+}
diff --git a/src/java/org/apache/cassandra/cql3/selection/MapSelector.java b/src/java/org/apache/cassandra/cql3/selection/MapSelector.java
new file mode 100644
index 0000000..09424bd
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/selection/MapSelector.java
@@ -0,0 +1,227 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.selection;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.stream.Collectors;
+
+import org.apache.cassandra.cql3.ColumnSpecification;
+import org.apache.cassandra.cql3.Maps;
+import org.apache.cassandra.cql3.QueryOptions;
+import org.apache.cassandra.cql3.functions.Function;
+import org.apache.cassandra.db.filter.ColumnFilter.Builder;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.MapType;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.serializers.CollectionSerializer;
+import org.apache.cassandra.transport.ProtocolVersion;
+import org.apache.cassandra.utils.Pair;
+
+/**
+ * <code>Selector</code> for literal map (e.g. {'min' : min(value), 'max' : max(value), 'count' : count(value)}).
+ *
+ */
+final class MapSelector extends Selector
+{
+ /**
+ * The map type.
+ */
+ private final MapType<?, ?> type;
+
+ /**
+ * The map elements
+ */
+ private final List<Pair<Selector, Selector>> elements;
+
+ public static Factory newFactory(final AbstractType<?> type, final List<Pair<Factory, Factory>> factories)
+ {
+ return new Factory()
+ {
+ protected String getColumnName()
+ {
+ return Maps.mapToString(factories, Factory::getColumnName);
+ }
+
+ protected AbstractType<?> getReturnType()
+ {
+ return type;
+ }
+
+ protected final void addColumnMapping(SelectionColumnMapping mapping, ColumnSpecification resultsColumn)
+ {
+ SelectionColumnMapping tmpMapping = SelectionColumnMapping.newMapping();
+ for (Pair<Factory, Factory> entry : factories)
+ {
+ entry.left.addColumnMapping(tmpMapping, resultsColumn);
+ entry.right.addColumnMapping(tmpMapping, resultsColumn);
+ }
+
+ if (tmpMapping.getMappings().get(resultsColumn).isEmpty())
+ // add a null mapping for cases where the collection is empty
+ mapping.addMapping(resultsColumn, (ColumnMetadata)null);
+ else
+ // collate the mapped columns from the child factories & add those
+ mapping.addMapping(resultsColumn, tmpMapping.getMappings().values());
+ }
+
+ public Selector newInstance(final QueryOptions options)
+ {
+ return new MapSelector(type,
+ factories.stream()
+ .map(p -> Pair.create(p.left.newInstance(options),
+ p.right.newInstance(options)))
+ .collect(Collectors.toList()));
+ }
+
+ @Override
+ public boolean isAggregateSelectorFactory()
+ {
+ for (Pair<Factory, Factory> entry : factories)
+ {
+ if (entry.left.isAggregateSelectorFactory() || entry.right.isAggregateSelectorFactory())
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ public void addFunctionsTo(List<Function> functions)
+ {
+ for (Pair<Factory, Factory> entry : factories)
+ {
+ entry.left.addFunctionsTo(functions);
+ entry.right.addFunctionsTo(functions);
+ }
+ }
+
+ @Override
+ public boolean isWritetimeSelectorFactory()
+ {
+ for (Pair<Factory, Factory> entry : factories)
+ {
+ if (entry.left.isWritetimeSelectorFactory() || entry.right.isWritetimeSelectorFactory())
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ public boolean isTTLSelectorFactory()
+ {
+ for (Pair<Factory, Factory> entry : factories)
+ {
+ if (entry.left.isTTLSelectorFactory() || entry.right.isTTLSelectorFactory())
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ boolean areAllFetchedColumnsKnown()
+ {
+ for (Pair<Factory, Factory> entry : factories)
+ {
+ if (!entry.left.areAllFetchedColumnsKnown() || !entry.right.areAllFetchedColumnsKnown())
+ return false;
+ }
+ return true;
+ }
+
+ @Override
+ void addFetchedColumns(Builder builder)
+ {
+ for (Pair<Factory, Factory> entry : factories)
+ {
+ entry.left.addFetchedColumns(builder);
+ entry.right.addFetchedColumns(builder);
+ }
+ }
+ };
+ }
+
+ @Override
+ public void addFetchedColumns(Builder builder)
+ {
+ for (int i = 0, m = elements.size(); i < m; i++)
+ {
+ Pair<Selector, Selector> pair = elements.get(i);
+ pair.left.addFetchedColumns(builder);
+ pair.right.addFetchedColumns(builder);
+ }
+ }
+
+ public void addInput(ProtocolVersion protocolVersion, ResultSetBuilder rs) throws InvalidRequestException
+ {
+ for (int i = 0, m = elements.size(); i < m; i++)
+ {
+ Pair<Selector, Selector> pair = elements.get(i);
+ pair.left.addInput(protocolVersion, rs);
+ pair.right.addInput(protocolVersion, rs);
+ }
+ }
+
+ public ByteBuffer getOutput(ProtocolVersion protocolVersion) throws InvalidRequestException
+ {
+ Map<ByteBuffer, ByteBuffer> map = new TreeMap<>(type.getKeysType());
+ for (int i = 0, m = elements.size(); i < m; i++)
+ {
+ Pair<Selector, Selector> pair = elements.get(i);
+ map.put(pair.left.getOutput(protocolVersion), pair.right.getOutput(protocolVersion));
+ }
+
+ List<ByteBuffer> buffers = new ArrayList<>(elements.size() * 2);
+ for (Map.Entry<ByteBuffer, ByteBuffer> entry : map.entrySet())
+ {
+ buffers.add(entry.getKey());
+ buffers.add(entry.getValue());
+ }
+ return CollectionSerializer.pack(buffers, elements.size(), protocolVersion);
+ }
+
+ public void reset()
+ {
+ for (int i = 0, m = elements.size(); i < m; i++)
+ {
+ Pair<Selector, Selector> pair = elements.get(i);
+ pair.left.reset();
+ pair.right.reset();
+ }
+ }
+
+ public AbstractType<?> getType()
+ {
+ return type;
+ }
+
+ @Override
+ public String toString()
+ {
+ return Maps.mapToString(elements);
+ }
+
+ private MapSelector(AbstractType<?> type, List<Pair<Selector, Selector>> elements)
+ {
+ this.type = (MapType<?, ?>) type;
+ this.elements = elements;
+ }
+}
diff --git a/src/java/org/apache/cassandra/cql3/selection/RawSelector.java b/src/java/org/apache/cassandra/cql3/selection/RawSelector.java
index 7d5543f..910b1d1 100644
--- a/src/java/org/apache/cassandra/cql3/selection/RawSelector.java
+++ b/src/java/org/apache/cassandra/cql3/selection/RawSelector.java
@@ -20,11 +20,11 @@
import java.util.List;
-import org.apache.cassandra.config.CFMetaData;
+import com.google.common.collect.Lists;
+
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.cql3.ColumnIdentifier;
-import com.google.common.base.Function;
-import com.google.common.collect.Lists;
public class RawSelector
{
@@ -43,19 +43,14 @@
* @param raws the <code>RawSelector</code>s to converts.
* @return a list of <code>Selectable</code>s
*/
- public static List<Selectable> toSelectables(List<RawSelector> raws, final CFMetaData cfm)
+ public static List<Selectable> toSelectables(List<RawSelector> raws, final TableMetadata table)
{
- return Lists.transform(raws, new Function<RawSelector, Selectable>()
- {
- public Selectable apply(RawSelector raw)
- {
- return raw.selectable.prepare(cfm);
- }
- });
+ return Lists.transform(raws, raw -> raw.prepare(table));
}
- public boolean processesSelection()
+ private Selectable prepare(TableMetadata table)
{
- return selectable.processesSelection();
+ Selectable s = selectable.prepare(table);
+ return alias != null ? new AliasedSelectable(s, alias) : s;
}
}
diff --git a/src/java/org/apache/cassandra/cql3/selection/ResultSetBuilder.java b/src/java/org/apache/cassandra/cql3/selection/ResultSetBuilder.java
new file mode 100644
index 0000000..9b7abe1
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/selection/ResultSetBuilder.java
@@ -0,0 +1,171 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.selection;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.cassandra.cql3.ResultSet;
+import org.apache.cassandra.cql3.ResultSet.ResultMetadata;
+import org.apache.cassandra.cql3.selection.Selection.Selectors;
+import org.apache.cassandra.db.Clustering;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.aggregation.GroupMaker;
+import org.apache.cassandra.db.context.CounterContext;
+import org.apache.cassandra.db.rows.Cell;
+import org.apache.cassandra.utils.ByteBufferUtil;
+
+public final class ResultSetBuilder
+{
+ private final ResultSet resultSet;
+
+ /**
+ * As multiple thread can access a <code>Selection</code> instance each <code>ResultSetBuilder</code> will use
+ * its own <code>Selectors</code> instance.
+ */
+ private final Selectors selectors;
+
+ /**
+ * The <code>GroupMaker</code> used to build the aggregates.
+ */
+ private final GroupMaker groupMaker;
+
+ /*
+ * We'll build CQL3 row one by one.
+ * The currentRow is the values for the (CQL3) columns we've fetched.
+ * We also collect timestamps and ttls for the case where the writetime and
+ * ttl functions are used. Note that we might collect timestamp and/or ttls
+ * we don't care about, but since the array below are allocated just once,
+ * it doesn't matter performance wise.
+ */
+ List<ByteBuffer> current;
+ final long[] timestamps;
+ final int[] ttls;
+
+ public ResultSetBuilder(ResultMetadata metadata, Selectors selectors)
+ {
+ this(metadata, selectors, null);
+ }
+
+ public ResultSetBuilder(ResultMetadata metadata, Selectors selectors, GroupMaker groupMaker)
+ {
+ this.resultSet = new ResultSet(metadata.copy(), new ArrayList<List<ByteBuffer>>());
+ this.selectors = selectors;
+ this.groupMaker = groupMaker;
+ this.timestamps = selectors.collectTimestamps() ? new long[selectors.numberOfFetchedColumns()] : null;
+ this.ttls = selectors.collectTTLs() ? new int[selectors.numberOfFetchedColumns()] : null;
+
+ // We use MIN_VALUE to indicate no timestamp and -1 for no ttl
+ if (timestamps != null)
+ Arrays.fill(timestamps, Long.MIN_VALUE);
+ if (ttls != null)
+ Arrays.fill(ttls, -1);
+ }
+
+ public void add(ByteBuffer v)
+ {
+ current.add(v);
+ }
+
+ public void add(Cell c, int nowInSec)
+ {
+ if (c == null)
+ {
+ current.add(null);
+ return;
+ }
+
+ current.add(value(c));
+
+ if (timestamps != null)
+ timestamps[current.size() - 1] = c.timestamp();
+
+ if (ttls != null)
+ ttls[current.size() - 1] = remainingTTL(c, nowInSec);
+ }
+
+ private int remainingTTL(Cell c, int nowInSec)
+ {
+ if (!c.isExpiring())
+ return -1;
+
+ int remaining = c.localDeletionTime() - nowInSec;
+ return remaining >= 0 ? remaining : -1;
+ }
+
+ private ByteBuffer value(Cell c)
+ {
+ return c.isCounterCell()
+ ? ByteBufferUtil.bytes(CounterContext.instance().total(c.value()))
+ : c.value();
+ }
+
+ /**
+ * Notifies this <code>Builder</code> that a new row is being processed.
+ *
+ * @param partitionKey the partition key of the new row
+ * @param clustering the clustering of the new row
+ */
+ public void newRow(DecoratedKey partitionKey, Clustering clustering)
+ {
+ // The groupMaker needs to be called for each row
+ boolean isNewAggregate = groupMaker == null || groupMaker.isNewGroup(partitionKey, clustering);
+ if (current != null)
+ {
+ selectors.addInputRow(this);
+ if (isNewAggregate)
+ {
+ resultSet.addRow(getOutputRow());
+ selectors.reset();
+ }
+ }
+ current = new ArrayList<>(selectors.numberOfFetchedColumns());
+
+ // Timestamps and TTLs are arrays per row, we must null them out between rows
+ if (timestamps != null)
+ Arrays.fill(timestamps, Long.MIN_VALUE);
+ if (ttls != null)
+ Arrays.fill(ttls, -1);
+ }
+
+ /**
+ * Builds the <code>ResultSet</code>
+ */
+ public ResultSet build()
+ {
+ if (current != null)
+ {
+ selectors.addInputRow(this);
+ resultSet.addRow(getOutputRow());
+ selectors.reset();
+ current = null;
+ }
+
+ // For aggregates we need to return a row even it no records have been found
+ if (resultSet.isEmpty() && groupMaker != null && groupMaker.returnAtLeastOneRow())
+ resultSet.addRow(getOutputRow());
+ return resultSet;
+ }
+
+ private List<ByteBuffer> getOutputRow()
+ {
+ return selectors.getOutputRow();
+ }
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/cql3/selection/ScalarFunctionSelector.java b/src/java/org/apache/cassandra/cql3/selection/ScalarFunctionSelector.java
index c05cdaa..de74678 100644
--- a/src/java/org/apache/cassandra/cql3/selection/ScalarFunctionSelector.java
+++ b/src/java/org/apache/cassandra/cql3/selection/ScalarFunctionSelector.java
@@ -22,21 +22,11 @@
import org.apache.cassandra.cql3.functions.Function;
import org.apache.cassandra.cql3.functions.ScalarFunction;
-import org.apache.cassandra.cql3.selection.Selection.ResultSetBuilder;
import org.apache.cassandra.exceptions.InvalidRequestException;
import org.apache.cassandra.transport.ProtocolVersion;
final class ScalarFunctionSelector extends AbstractFunctionSelector<ScalarFunction>
{
- public boolean isAggregate()
- {
- // We cannot just return true as it is possible to have a scalar function wrapping an aggregation function
- if (argSelectors.isEmpty())
- return false;
-
- return argSelectors.get(0).isAggregate();
- }
-
public void addInput(ProtocolVersion protocolVersion, ResultSetBuilder rs) throws InvalidRequestException
{
for (int i = 0, m = argSelectors.size(); i < m; i++)
diff --git a/src/java/org/apache/cassandra/cql3/selection/Selectable.java b/src/java/org/apache/cassandra/cql3/selection/Selectable.java
index 80e2ae8..220bb89 100644
--- a/src/java/org/apache/cassandra/cql3/selection/Selectable.java
+++ b/src/java/org/apache/cassandra/cql3/selection/Selectable.java
@@ -18,21 +18,27 @@
*/
package org.apache.cassandra.cql3.selection;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
+import java.util.*;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
import org.apache.commons.lang3.text.StrBuilder;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
+
import org.apache.cassandra.cql3.*;
import org.apache.cassandra.cql3.functions.*;
+import org.apache.cassandra.cql3.selection.Selector.Factory;
import org.apache.cassandra.db.marshal.*;
import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.utils.Pair;
+
+import static org.apache.cassandra.cql3.selection.SelectorFactories.createFactoriesAndCollectColumnDefinitions;
+import static org.apache.cassandra.cql3.statements.RequestValidations.invalidRequest;
public interface Selectable extends AssignmentTestable
{
- public Selector.Factory newSelectorFactory(CFMetaData cfm, AbstractType<?> expectedType, List<ColumnDefinition> defs, VariableSpecifications boundNames);
+ public Selector.Factory newSelectorFactory(TableMetadata table, AbstractType<?> expectedType, List<ColumnMetadata> defs, VariableSpecifications boundNames);
/**
* The type of the {@code Selectable} if it can be infered.
@@ -46,6 +52,39 @@
*/
public AbstractType<?> getExactTypeIfKnown(String keyspace);
+ /**
+ * Checks if this {@code Selectable} select columns matching the specified predicate.
+ * @return {@code true} if this {@code Selectable} select columns matching the specified predicate,
+ * {@code false} otherwise.
+ */
+ public boolean selectColumns(Predicate<ColumnMetadata> predicate);
+
+ /**
+ * Checks if the specified Selectables select columns matching the specified predicate.
+ * @param selectables the selectables to check.
+ * @return {@code true} if the specified Selectables select columns matching the specified predicate,
+ {@code false} otherwise.
+ */
+ public static boolean selectColumns(List<Selectable> selectables, Predicate<ColumnMetadata> predicate)
+ {
+ for (Selectable selectable : selectables)
+ {
+ if (selectable.selectColumns(predicate))
+ return true;
+ }
+ return false;
+ }
+
+ /**
+ * Checks if any processing is performed on the selected columns, {@code false} otherwise.
+ * @return {@code true} if any processing is performed on the selected columns, {@code false} otherwise.
+ */
+ public default boolean processesSelection()
+ {
+ // ColumnMetadata is the only case that returns false and override this
+ return true;
+ }
+
// Term.Raw overrides this since some literals can be WEAKLY_ASSIGNABLE
default public TestResult testAssignment(String keyspace, ColumnSpecification receiver)
{
@@ -53,7 +92,7 @@
return type == null ? TestResult.NOT_ASSIGNABLE : type.testAssignment(keyspace, receiver);
}
- default int addAndGetIndex(ColumnDefinition def, List<ColumnDefinition> l)
+ default int addAndGetIndex(ColumnMetadata def, List<ColumnMetadata> l)
{
int idx = l.indexOf(def);
if (idx < 0)
@@ -64,18 +103,20 @@
return idx;
}
+ default ColumnSpecification specForElementOrSlice(Selectable selected, ColumnSpecification receiver, String selectionType)
+ {
+ switch (((CollectionType)receiver.type).kind)
+ {
+ case LIST: throw new InvalidRequestException(String.format("%s selection is only allowed on sets and maps, but %s is a list", selectionType, selected));
+ case SET: return Sets.valueSpecOf(receiver);
+ case MAP: return Maps.keySpecOf(receiver);
+ default: throw new AssertionError();
+ }
+ }
+
public static abstract class Raw
{
- public abstract Selectable prepare(CFMetaData cfm);
-
- /**
- * Returns true if any processing is performed on the selected column.
- **/
- public boolean processesSelection()
- {
- // ColumnIdentifier is the only case that returns false and override this
- return true;
- }
+ public abstract Selectable prepare(TableMetadata table);
}
public static class WithTerm implements Selectable
@@ -106,7 +147,7 @@
return rawTerm.testAssignment(keyspace, receiver);
}
- public Selector.Factory newSelectorFactory(CFMetaData cfm, AbstractType<?> expectedType, List<ColumnDefinition> defs, VariableSpecifications boundNames) throws InvalidRequestException
+ public Selector.Factory newSelectorFactory(TableMetadata table, AbstractType<?> expectedType, List<ColumnMetadata> defs, VariableSpecifications boundNames) throws InvalidRequestException
{
/*
* expectedType will be null if we have no constraint on what the type should be. For instance, if this term is a bind marker:
@@ -128,7 +169,7 @@
* Lastly, note that if the term is a terminal literal, we don't have to check it's compatibility with 'expectedType' as any incompatibility
* would have been found at preparation time.
*/
- AbstractType<?> type = getExactTypeIfKnown(cfm.ksName);
+ AbstractType<?> type = getExactTypeIfKnown(table.keyspace);
if (type == null)
{
type = expectedType;
@@ -140,7 +181,7 @@
// selection will have this name. Which isn't terribly helpful, but it's unclear how to provide
// something a lot more helpful and in practice user can bind those markers by position or, even better,
// use bind markers.
- Term term = rawTerm.prepare(cfm.ksName, new ColumnSpecification(cfm.ksName, cfm.cfName, bindMarkerNameInSelection, type));
+ Term term = rawTerm.prepare(table.keyspace, new ColumnSpecification(table.keyspace, table.name, bindMarkerNameInSelection, type));
term.collectMarkerSpecification(boundNames);
return TermSelector.newFactory(rawTerm.getText(), term, type);
}
@@ -152,9 +193,15 @@
}
@Override
+ public boolean selectColumns(Predicate<ColumnMetadata> predicate)
+ {
+ return false;
+ }
+
+ @Override
public String toString()
{
- return rawTerm.toString();
+ return rawTerm.getText();
}
public static class Raw extends Selectable.Raw
@@ -166,7 +213,7 @@
this.term = term;
}
- public Selectable prepare(CFMetaData cfm)
+ public Selectable prepare(TableMetadata table)
{
return new WithTerm(term);
}
@@ -175,10 +222,10 @@
public static class WritetimeOrTTL implements Selectable
{
- public final ColumnDefinition column;
+ public final ColumnMetadata column;
public final boolean isWritetime;
- public WritetimeOrTTL(ColumnDefinition column, boolean isWritetime)
+ public WritetimeOrTTL(ColumnMetadata column, boolean isWritetime)
{
this.column = column;
this.isWritetime = isWritetime;
@@ -190,9 +237,9 @@
return (isWritetime ? "writetime" : "ttl") + "(" + column.name + ")";
}
- public Selector.Factory newSelectorFactory(CFMetaData cfm,
+ public Selector.Factory newSelectorFactory(TableMetadata table,
AbstractType<?> expectedType,
- List<ColumnDefinition> defs,
+ List<ColumnMetadata> defs,
VariableSpecifications boundNames)
{
if (column.isPrimaryKeyColumn())
@@ -212,20 +259,26 @@
return isWritetime ? LongType.instance : Int32Type.instance;
}
+ @Override
+ public boolean selectColumns(Predicate<ColumnMetadata> predicate)
+ {
+ return predicate.test(column);
+ }
+
public static class Raw extends Selectable.Raw
{
- private final ColumnDefinition.Raw id;
+ private final ColumnMetadata.Raw id;
private final boolean isWritetime;
- public Raw(ColumnDefinition.Raw id, boolean isWritetime)
+ public Raw(ColumnMetadata.Raw id, boolean isWritetime)
{
this.id = id;
this.isWritetime = isWritetime;
}
- public WritetimeOrTTL prepare(CFMetaData cfm)
+ public WritetimeOrTTL prepare(TableMetadata table)
{
- return new WritetimeOrTTL(id.prepare(cfm), isWritetime);
+ return new WritetimeOrTTL(id.prepare(table), isWritetime);
}
}
}
@@ -244,19 +297,21 @@
@Override
public String toString()
{
- return new StrBuilder().append(function.name())
- .append("(")
- .appendWithSeparators(args, ", ")
- .append(")")
- .toString();
+ return function.columnName(args.stream().map(Object::toString).collect(Collectors.toList()));
}
- public Selector.Factory newSelectorFactory(CFMetaData cfm, AbstractType<?> expectedType, List<ColumnDefinition> defs, VariableSpecifications boundNames)
+ public Selector.Factory newSelectorFactory(TableMetadata table, AbstractType<?> expectedType, List<ColumnMetadata> defs, VariableSpecifications boundNames)
{
- SelectorFactories factories = SelectorFactories.createFactoriesAndCollectColumnDefinitions(args, function.argTypes(), cfm, defs, boundNames);
+ SelectorFactories factories = SelectorFactories.createFactoriesAndCollectColumnDefinitions(args, function.argTypes(), table, defs, boundNames);
return AbstractFunctionSelector.newFactory(function, factories);
}
+ @Override
+ public boolean selectColumns(Predicate<ColumnMetadata> predicate)
+ {
+ return Selectable.selectColumns(args, predicate);
+ }
+
public AbstractType<?> getExactTypeIfKnown(String keyspace)
{
return function.returnType();
@@ -279,11 +334,23 @@
Collections.emptyList());
}
- public Selectable prepare(CFMetaData cfm)
+ public static Raw newOperation(char operator, Selectable.Raw left, Selectable.Raw right)
+ {
+ return new Raw(OperationFcts.getFunctionNameFromOperator(operator),
+ Arrays.asList(left, right));
+ }
+
+ public static Raw newNegation(Selectable.Raw arg)
+ {
+ return new Raw(FunctionName.nativeFunction(OperationFcts.NEGATION_FUNCTION_NAME),
+ Collections.singletonList(arg));
+ }
+
+ public Selectable prepare(TableMetadata table)
{
List<Selectable> preparedArgs = new ArrayList<>(args.size());
for (Selectable.Raw arg : args)
- preparedArgs.add(arg.prepare(cfm));
+ preparedArgs.add(arg.prepare(table));
FunctionName name = functionName;
// We need to circumvent the normal function lookup process for toJson() because instances of the function
@@ -306,7 +373,7 @@
preparedArgs = Collections.emptyList();
}
- Function fun = FunctionResolver.get(cfm.ksName, name, preparedArgs, cfm.ksName, cfm.cfName, null);
+ Function fun = FunctionResolver.get(table.keyspace, name, preparedArgs, table.keyspace, table.name, null);
if (fun == null)
throw new InvalidRequestException(String.format("Unknown function '%s'", functionName));
@@ -338,9 +405,9 @@
.toString();
}
- public Selector.Factory newSelectorFactory(CFMetaData cfm, AbstractType<?> expectedType, List<ColumnDefinition> defs, VariableSpecifications boundNames)
+ public Selector.Factory newSelectorFactory(TableMetadata table, AbstractType<?> expectedType, List<ColumnMetadata> defs, VariableSpecifications boundNames)
{
- SelectorFactories factories = SelectorFactories.createFactoriesAndCollectColumnDefinitions(args, null, cfm, defs, boundNames);
+ SelectorFactories factories = SelectorFactories.createFactoriesAndCollectColumnDefinitions(args, null, table, defs, boundNames);
Function fun = ToJsonFct.getInstance(factories.getReturnTypes());
return AbstractFunctionSelector.newFactory(fun, factories);
}
@@ -349,6 +416,12 @@
{
return UTF8Type.instance;
}
+
+ @Override
+ public boolean selectColumns(Predicate<ColumnMetadata> predicate)
+ {
+ return Selectable.selectColumns(args, predicate);
+ }
}
public static class WithCast implements Selectable
@@ -368,10 +441,10 @@
return String.format("cast(%s as %s)", arg, type.toString().toLowerCase());
}
- public Selector.Factory newSelectorFactory(CFMetaData cfm, AbstractType<?> expectedType, List<ColumnDefinition> defs, VariableSpecifications boundNames)
+ public Selector.Factory newSelectorFactory(TableMetadata table, AbstractType<?> expectedType, List<ColumnMetadata> defs, VariableSpecifications boundNames)
{
List<Selectable> args = Collections.singletonList(arg);
- SelectorFactories factories = SelectorFactories.createFactoriesAndCollectColumnDefinitions(args, null, cfm, defs, boundNames);
+ SelectorFactories factories = SelectorFactories.createFactoriesAndCollectColumnDefinitions(args, null, table, defs, boundNames);
Selector.Factory factory = factories.get(0);
@@ -380,7 +453,7 @@
return factory;
FunctionName name = FunctionName.nativeFunction(CastFcts.getFunctionName(type));
- Function fun = FunctionResolver.get(cfm.ksName, name, args, cfm.ksName, cfm.cfName, null);
+ Function fun = FunctionResolver.get(table.keyspace, name, args, table.keyspace, table.name, null);
if (fun == null)
{
@@ -396,6 +469,12 @@
return type.getType();
}
+ @Override
+ public boolean selectColumns(Predicate<ColumnMetadata> predicate)
+ {
+ return arg.selectColumns(predicate);
+ }
+
public static class Raw extends Selectable.Raw
{
private final CQL3Type type;
@@ -407,13 +486,16 @@
this.type = type;
}
- public WithCast prepare(CFMetaData cfm)
+ public WithCast prepare(TableMetadata table)
{
- return new WithCast(arg.prepare(cfm), type);
+ return new WithCast(arg.prepare(table), type);
}
}
}
+ /**
+ * Represents the selection of the field of a UDT (eg. t.f).
+ */
public static class WithFieldSelection implements Selectable
{
public final Selectable selected;
@@ -431,10 +513,19 @@
return String.format("%s.%s", selected, field);
}
- public Selector.Factory newSelectorFactory(CFMetaData cfm, AbstractType<?> expectedType, List<ColumnDefinition> defs, VariableSpecifications boundNames)
+ public Selector.Factory newSelectorFactory(TableMetadata table, AbstractType<?> expectedType, List<ColumnMetadata> defs, VariableSpecifications boundNames)
{
- Selector.Factory factory = selected.newSelectorFactory(cfm, null, defs, boundNames);
- AbstractType<?> type = factory.getColumnSpecification(cfm).type;
+ AbstractType<?> expectedUdtType = null;
+
+ // If the UDT is between parentheses, we know that it is not a tuple with a single element.
+ if (selected instanceof BetweenParenthesesOrWithTuple)
+ {
+ BetweenParenthesesOrWithTuple betweenParentheses = (BetweenParenthesesOrWithTuple) selected;
+ expectedUdtType = betweenParentheses.selectables.get(0).getExactTypeIfKnown(table.keyspace);
+ }
+
+ Selector.Factory factory = selected.newSelectorFactory(table, expectedUdtType, defs, boundNames);
+ AbstractType<?> type = factory.getReturnType();
if (!type.isUDT())
{
throw new InvalidRequestException(
@@ -468,6 +559,12 @@
return ut.fieldType(fieldIndex);
}
+ @Override
+ public boolean selectColumns(Predicate<ColumnMetadata> predicate)
+ {
+ return selected.selectColumns(predicate);
+ }
+
public static class Raw extends Selectable.Raw
{
private final Selectable.Raw selected;
@@ -479,9 +576,820 @@
this.field = field;
}
- public WithFieldSelection prepare(CFMetaData cfm)
+ public WithFieldSelection prepare(TableMetadata table)
{
- return new WithFieldSelection(selected.prepare(cfm), field);
+ return new WithFieldSelection(selected.prepare(table), field);
+ }
+ }
+ }
+
+ /**
+ * {@code Selectable} for {@code Selectable} between parentheses or tuples.
+ * <p>The parser cannot differentiate between a single element between parentheses or a single element tuple.
+ * By consequence, we are forced to wait until the type is known to be able to differentiate them.</p>
+ */
+ public static class BetweenParenthesesOrWithTuple implements Selectable
+ {
+ /**
+ * The tuple elements or the element between the parentheses
+ */
+ private final List<Selectable> selectables;
+
+ public BetweenParenthesesOrWithTuple(List<Selectable> selectables)
+ {
+ this.selectables = selectables;
+ }
+
+ @Override
+ public TestResult testAssignment(String keyspace, ColumnSpecification receiver)
+ {
+ if (selectables.size() == 1 && !receiver.type.isTuple())
+ return selectables.get(0).testAssignment(keyspace, receiver);
+
+ return Tuples.testTupleAssignment(receiver, selectables);
+ }
+
+ @Override
+ public Factory newSelectorFactory(TableMetadata cfm,
+ AbstractType<?> expectedType,
+ List<ColumnMetadata> defs,
+ VariableSpecifications boundNames)
+ {
+ AbstractType<?> type = getExactTypeIfKnown(cfm.keyspace);
+ if (type == null)
+ {
+ type = expectedType;
+ if (type == null)
+ throw invalidRequest("Cannot infer type for term %s in selection clause (try using a cast to force a type)",
+ this);
+ }
+
+ if (selectables.size() == 1 && !type.isTuple())
+ return newBetweenParenthesesSelectorFactory(cfm, expectedType, defs, boundNames);
+
+ return newTupleSelectorFactory(cfm, (TupleType) type, defs, boundNames);
+ }
+
+ private Factory newBetweenParenthesesSelectorFactory(TableMetadata cfm,
+ AbstractType<?> expectedType,
+ List<ColumnMetadata> defs,
+ VariableSpecifications boundNames)
+ {
+ Selectable selectable = selectables.get(0);
+ final Factory factory = selectable.newSelectorFactory(cfm, expectedType, defs, boundNames);
+
+ return new ForwardingFactory()
+ {
+ protected Factory delegate()
+ {
+ return factory;
+ }
+
+ protected String getColumnName()
+ {
+ return String.format("(%s)", factory.getColumnName());
+ }
+ };
+ }
+
+ private Factory newTupleSelectorFactory(TableMetadata cfm,
+ TupleType tupleType,
+ List<ColumnMetadata> defs,
+ VariableSpecifications boundNames)
+ {
+ SelectorFactories factories = createFactoriesAndCollectColumnDefinitions(selectables,
+ tupleType.allTypes(),
+ cfm,
+ defs,
+ boundNames);
+
+ return TupleSelector.newFactory(tupleType, factories);
+ }
+
+ @Override
+ public AbstractType<?> getExactTypeIfKnown(String keyspace)
+ {
+ // If there is only one element we cannot know if it is an element between parentheses or a tuple
+ // with only one element. By consequence, we need to force the user to specify the type.
+ if (selectables.size() == 1)
+ return null;
+
+ return Tuples.getExactTupleTypeIfKnown(selectables, p -> p.getExactTypeIfKnown(keyspace));
+ }
+
+ @Override
+ public boolean selectColumns(Predicate<ColumnMetadata> predicate)
+ {
+ return Selectable.selectColumns(selectables, predicate);
+ }
+
+ @Override
+ public String toString()
+ {
+ return Tuples.tupleToString(selectables);
+ }
+
+ public static class Raw extends Selectable.Raw
+ {
+ private final List<Selectable.Raw> raws;
+
+ public Raw(List<Selectable.Raw> raws)
+ {
+ this.raws = raws;
+ }
+
+ public Selectable prepare(TableMetadata cfm)
+ {
+ return new BetweenParenthesesOrWithTuple(raws.stream().map(p -> p.prepare(cfm)).collect(Collectors.toList()));
+ }
+ }
+ }
+
+ /**
+ * <code>Selectable</code> for literal Lists.
+ */
+ public static class WithList implements Selectable
+ {
+ /**
+ * The list elements
+ */
+ private final List<Selectable> selectables;
+
+ public WithList(List<Selectable> selectables)
+ {
+ this.selectables = selectables;
+ }
+
+ @Override
+ public TestResult testAssignment(String keyspace, ColumnSpecification receiver)
+ {
+ return Lists.testListAssignment(receiver, selectables);
+ }
+
+ @Override
+ public Factory newSelectorFactory(TableMetadata cfm,
+ AbstractType<?> expectedType,
+ List<ColumnMetadata> defs,
+ VariableSpecifications boundNames)
+ {
+ AbstractType<?> type = getExactTypeIfKnown(cfm.keyspace);
+ if (type == null)
+ {
+ type = expectedType;
+ if (type == null)
+ throw invalidRequest("Cannot infer type for term %s in selection clause (try using a cast to force a type)",
+ this);
+ }
+
+ ListType<?> listType = (ListType<?>) type;
+
+ List<AbstractType<?>> expectedTypes = new ArrayList<>(selectables.size());
+ for (int i = 0, m = selectables.size(); i < m; i++)
+ expectedTypes.add(listType.getElementsType());
+
+ SelectorFactories factories = createFactoriesAndCollectColumnDefinitions(selectables,
+ expectedTypes,
+ cfm,
+ defs,
+ boundNames);
+ return ListSelector.newFactory(type, factories);
+ }
+
+ @Override
+ public AbstractType<?> getExactTypeIfKnown(String keyspace)
+ {
+ return Lists.getExactListTypeIfKnown(selectables, p -> p.getExactTypeIfKnown(keyspace));
+ }
+
+ @Override
+ public boolean selectColumns(Predicate<ColumnMetadata> predicate)
+ {
+ return Selectable.selectColumns(selectables, predicate);
+ }
+
+ @Override
+ public String toString()
+ {
+ return Lists.listToString(selectables);
+ }
+
+ public static class Raw extends Selectable.Raw
+ {
+ private final List<Selectable.Raw> raws;
+
+ public Raw(List<Selectable.Raw> raws)
+ {
+ this.raws = raws;
+ }
+
+ public Selectable prepare(TableMetadata cfm)
+ {
+ return new WithList(raws.stream().map(p -> p.prepare(cfm)).collect(Collectors.toList()));
+ }
+ }
+ }
+
+ /**
+ * <code>Selectable</code> for literal Sets.
+ */
+ public static class WithSet implements Selectable
+ {
+ /**
+ * The set elements
+ */
+ private final List<Selectable> selectables;
+
+ public WithSet(List<Selectable> selectables)
+ {
+ this.selectables = selectables;
+ }
+
+ @Override
+ public TestResult testAssignment(String keyspace, ColumnSpecification receiver)
+ {
+ return Sets.testSetAssignment(receiver, selectables);
+ }
+
+ @Override
+ public Factory newSelectorFactory(TableMetadata cfm,
+ AbstractType<?> expectedType,
+ List<ColumnMetadata> defs,
+ VariableSpecifications boundNames)
+ {
+ AbstractType<?> type = getExactTypeIfKnown(cfm.keyspace);
+ if (type == null)
+ {
+ type = expectedType;
+ if (type == null)
+ throw invalidRequest("Cannot infer type for term %s in selection clause (try using a cast to force a type)",
+ this);
+ }
+
+ // The parser treats empty Maps as Sets so if the type is a MapType we know that the Map is empty
+ if (type instanceof MapType)
+ return MapSelector.newFactory(type, Collections.emptyList());
+
+ SetType<?> setType = (SetType<?>) type;
+
+ if (setType.getElementsType() == DurationType.instance)
+ throw invalidRequest("Durations are not allowed inside sets: %s", setType.asCQL3Type());
+
+ List<AbstractType<?>> expectedTypes = new ArrayList<>(selectables.size());
+ for (int i = 0, m = selectables.size(); i < m; i++)
+ expectedTypes.add(setType.getElementsType());
+
+ SelectorFactories factories = createFactoriesAndCollectColumnDefinitions(selectables,
+ expectedTypes,
+ cfm,
+ defs,
+ boundNames);
+
+ return SetSelector.newFactory(type, factories);
+ }
+
+ @Override
+ public AbstractType<?> getExactTypeIfKnown(String keyspace)
+ {
+ return Sets.getExactSetTypeIfKnown(selectables, p -> p.getExactTypeIfKnown(keyspace));
+ }
+
+ @Override
+ public boolean selectColumns(Predicate<ColumnMetadata> predicate)
+ {
+ return Selectable.selectColumns(selectables, predicate);
+ }
+
+ @Override
+ public String toString()
+ {
+ return Sets.setToString(selectables);
+ }
+
+ public static class Raw extends Selectable.Raw
+ {
+ private final List<Selectable.Raw> raws;
+
+ public Raw(List<Selectable.Raw> raws)
+ {
+ this.raws = raws;
+ }
+
+ public Selectable prepare(TableMetadata cfm)
+ {
+ return new WithSet(raws.stream().map(p -> p.prepare(cfm)).collect(Collectors.toList()));
+ }
+ }
+ }
+
+ /**
+ * {@code Selectable} for literal Maps or UDTs.
+ * <p>The parser cannot differentiate between a Map or a UDT in the selection cause because a
+ * {@code ColumnMetadata} is equivalent to a {@code FieldIdentifier} from a syntax point of view.
+ * By consequence, we are forced to wait until the type is known to be able to differentiate them.</p>
+ */
+ public static class WithMapOrUdt implements Selectable
+ {
+ /**
+ * The column family metadata. We need to store them to be able to build the proper data once the type has been
+ * identified.
+ */
+ private final TableMetadata cfm;
+
+ /**
+ * The Map or UDT raw elements.
+ */
+ private final List<Pair<Selectable.Raw, Selectable.Raw>> raws;
+
+ public WithMapOrUdt(TableMetadata cfm, List<Pair<Selectable.Raw, Selectable.Raw>> raws)
+ {
+ this.cfm = cfm;
+ this.raws = raws;
+ }
+
+ @Override
+ public TestResult testAssignment(String keyspace, ColumnSpecification receiver)
+ {
+ return receiver.type.isUDT() ? UserTypes.testUserTypeAssignment(receiver, getUdtFields((UserType) receiver.type))
+ : Maps.testMapAssignment(receiver, getMapEntries(cfm));
+ }
+
+ @Override
+ public Factory newSelectorFactory(TableMetadata cfm,
+ AbstractType<?> expectedType,
+ List<ColumnMetadata> defs,
+ VariableSpecifications boundNames)
+ {
+ AbstractType<?> type = getExactTypeIfKnown(cfm.keyspace);
+ if (type == null)
+ {
+ type = expectedType;
+ if (type == null)
+ throw invalidRequest("Cannot infer type for term %s in selection clause (try using a cast to force a type)",
+ this);
+ }
+
+ if (type.isUDT())
+ return newUdtSelectorFactory(cfm, expectedType, defs, boundNames);
+
+ return newMapSelectorFactory(cfm, defs, boundNames, type);
+ }
+
+ private Factory newMapSelectorFactory(TableMetadata cfm,
+ List<ColumnMetadata> defs,
+ VariableSpecifications boundNames,
+ AbstractType<?> type)
+ {
+ MapType<?, ?> mapType = (MapType<?, ?>) type;
+
+ if (mapType.getKeysType() == DurationType.instance)
+ throw invalidRequest("Durations are not allowed as map keys: %s", mapType.asCQL3Type());
+
+ return MapSelector.newFactory(type, getMapEntries(cfm).stream()
+ .map(p -> Pair.create(p.left.newSelectorFactory(cfm, mapType.getKeysType(), defs, boundNames),
+ p.right.newSelectorFactory(cfm, mapType.getValuesType(), defs, boundNames)))
+ .collect(Collectors.toList()));
+ }
+
+ private Factory newUdtSelectorFactory(TableMetadata cfm,
+ AbstractType<?> expectedType,
+ List<ColumnMetadata> defs,
+ VariableSpecifications boundNames)
+ {
+ UserType ut = (UserType) expectedType;
+ Map<FieldIdentifier, Factory> factories = new LinkedHashMap<>(ut.size());
+
+ for (Pair<Selectable.Raw, Selectable.Raw> raw : raws)
+ {
+ if (!(raw.left instanceof RawIdentifier))
+ throw invalidRequest("%s is not a valid field identifier of type %s ",
+ raw.left,
+ ut.getNameAsString());
+
+ FieldIdentifier fieldName = ((RawIdentifier) raw.left).toFieldIdentifier();
+ int fieldPosition = ut.fieldPosition(fieldName);
+
+ if (fieldPosition == -1)
+ throw invalidRequest("Unknown field '%s' in value of user defined type %s",
+ fieldName,
+ ut.getNameAsString());
+
+ AbstractType<?> fieldType = ut.fieldType(fieldPosition);
+ factories.put(fieldName,
+ raw.right.prepare(cfm).newSelectorFactory(cfm, fieldType, defs, boundNames));
+ }
+
+ return UserTypeSelector.newFactory(expectedType, factories);
+ }
+
+ @Override
+ public AbstractType<?> getExactTypeIfKnown(String keyspace)
+ {
+ // Lets force the user to specify the type.
+ return null;
+ }
+
+ @Override
+ public boolean selectColumns(Predicate<ColumnMetadata> predicate)
+ {
+ for (Pair<Selectable.Raw, Selectable.Raw> raw : raws)
+ {
+ if (!(raw.left instanceof RawIdentifier) && raw.left.prepare(cfm).selectColumns(predicate))
+ return true;
+
+ if (!raw.right.prepare(cfm).selectColumns(predicate))
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ public String toString()
+ {
+ return raws.stream()
+ .map(p -> String.format("%s: %s",
+ p.left instanceof RawIdentifier ? p.left : p.left.prepare(cfm),
+ p.right.prepare(cfm)))
+ .collect(Collectors.joining(", ", "{", "}"));
+ }
+
+ private List<Pair<Selectable, Selectable>> getMapEntries(TableMetadata cfm)
+ {
+ return raws.stream()
+ .map(p -> Pair.create(p.left.prepare(cfm), p.right.prepare(cfm)))
+ .collect(Collectors.toList());
+ }
+
+ private Map<FieldIdentifier, Selectable> getUdtFields(UserType ut)
+ {
+ Map<FieldIdentifier, Selectable> fields = new LinkedHashMap<>(ut.size());
+
+ for (Pair<Selectable.Raw, Selectable.Raw> raw : raws)
+ {
+ if (!(raw.left instanceof RawIdentifier))
+ throw invalidRequest("%s is not a valid field identifier of type %s ",
+ raw.left,
+ ut.getNameAsString());
+
+ FieldIdentifier fieldName = ((RawIdentifier) raw.left).toFieldIdentifier();
+ int fieldPosition = ut.fieldPosition(fieldName);
+
+ if (fieldPosition == -1)
+ throw invalidRequest("Unknown field '%s' in value of user defined type %s",
+ fieldName,
+ ut.getNameAsString());
+
+ fields.put(fieldName, raw.right.prepare(cfm));
+ }
+
+ return fields;
+ }
+
+ public static class Raw extends Selectable.Raw
+ {
+ private final List<Pair<Selectable.Raw, Selectable.Raw>> raws;
+
+ public Raw(List<Pair<Selectable.Raw, Selectable.Raw>> raws)
+ {
+ this.raws = raws;
+ }
+
+ public Selectable prepare(TableMetadata cfm)
+ {
+ return new WithMapOrUdt(cfm, raws);
+ }
+ }
+ }
+
+ /**
+ * <code>Selectable</code> for type hints (e.g. (int) ?).
+ */
+ public static class WithTypeHint implements Selectable
+ {
+
+ /**
+ * The name of the type as specified in the query.
+ */
+ private final String typeName;
+
+ /**
+ * The type specified by the hint.
+ */
+ private final AbstractType<?> type;
+
+ /**
+ * The selectable to which the hint should be applied.
+ */
+ private final Selectable selectable;
+
+ public WithTypeHint(String typeName, AbstractType<?> type, Selectable selectable)
+ {
+ this.typeName = typeName;
+ this.type = type;
+ this.selectable = selectable;
+ }
+
+ @Override
+ public TestResult testAssignment(String keyspace, ColumnSpecification receiver)
+ {
+ if (receiver.type.equals(type))
+ return AssignmentTestable.TestResult.EXACT_MATCH;
+ else if (receiver.type.isValueCompatibleWith(type))
+ return AssignmentTestable.TestResult.WEAKLY_ASSIGNABLE;
+ else
+ return AssignmentTestable.TestResult.NOT_ASSIGNABLE;
+ }
+
+ @Override
+ public Factory newSelectorFactory(TableMetadata cfm,
+ AbstractType<?> expectedType,
+ List<ColumnMetadata> defs,
+ VariableSpecifications boundNames)
+ {
+ final ColumnSpecification receiver = new ColumnSpecification(cfm.keyspace, cfm.name, new ColumnIdentifier(toString(), true), type);
+
+ if (!selectable.testAssignment(cfm.keyspace, receiver).isAssignable())
+ throw new InvalidRequestException(String.format("Cannot assign value %s to %s of type %s", this, receiver.name, receiver.type.asCQL3Type()));
+
+ final Factory factory = selectable.newSelectorFactory(cfm, type, defs, boundNames);
+
+ return new ForwardingFactory()
+ {
+ protected Factory delegate()
+ {
+ return factory;
+ }
+
+ protected AbstractType<?> getReturnType()
+ {
+ return type;
+ }
+
+ protected String getColumnName()
+ {
+ return String.format("(%s)%s", typeName, factory.getColumnName());
+ }
+ };
+ }
+
+ @Override
+ public AbstractType<?> getExactTypeIfKnown(String keyspace)
+ {
+ return type;
+ }
+
+ @Override
+ public boolean selectColumns(Predicate<ColumnMetadata> predicate)
+ {
+ return selectable.selectColumns(predicate);
+ }
+
+ @Override
+ public String toString()
+ {
+ return String.format("(%s)%s", typeName, selectable);
+ }
+
+ public static class Raw extends Selectable.Raw
+ {
+ private final CQL3Type.Raw typeRaw;
+
+ private final Selectable.Raw raw;
+
+ public Raw( CQL3Type.Raw typeRaw, Selectable.Raw raw)
+ {
+ this.typeRaw = typeRaw;
+ this.raw = raw;
+ }
+
+ public Selectable prepare(TableMetadata cfm)
+ {
+ Selectable selectable = raw.prepare(cfm);
+ AbstractType<?> type = this.typeRaw.prepare(cfm.keyspace).getType();
+ if (type.isFreezable())
+ type = type.freeze();
+ return new WithTypeHint(typeRaw.toString(), type, selectable);
+ }
+ }
+ }
+
+ /**
+ * In the selection clause, the parser cannot differentiate between Maps and UDTs as a column identifier and field
+ * identifier have the same syntax. By consequence, we need to wait until the type is known to create the proper
+ * Object: {@code ColumnMetadata} or {@code FieldIdentifier}.
+ */
+ public static final class RawIdentifier extends Selectable.Raw
+ {
+ private final String text;
+
+ private final boolean quoted;
+
+ /**
+ * Creates a {@code RawIdentifier} from an unquoted identifier string.
+ */
+ public static Raw forUnquoted(String text)
+ {
+ return new RawIdentifier(text, false);
+ }
+
+ /**
+ * Creates a {@code RawIdentifier} from a quoted identifier string.
+ */
+ public static Raw forQuoted(String text)
+ {
+ return new RawIdentifier(text, true);
+ }
+
+ private RawIdentifier(String text, boolean quoted)
+ {
+ this.text = text;
+ this.quoted = quoted;
+ }
+
+ @Override
+ public Selectable prepare(TableMetadata cfm)
+ {
+ ColumnMetadata.Raw raw = quoted ? ColumnMetadata.Raw.forQuoted(text)
+ : ColumnMetadata.Raw.forUnquoted(text);
+ return raw.prepare(cfm);
+ }
+
+ public FieldIdentifier toFieldIdentifier()
+ {
+ return quoted ? FieldIdentifier.forQuoted(text)
+ : FieldIdentifier.forUnquoted(text);
+ }
+
+ @Override
+ public String toString()
+ {
+ return text;
+ }
+ }
+
+ /**
+ * Represents the selection of an element of a collection (eg. c[x]).
+ */
+ public static class WithElementSelection implements Selectable
+ {
+ public final Selectable selected;
+ // Note that we can't yet prepare the Term.Raw yet as we need the ColumnSpecificiation corresponding to Selectable, which
+ // we'll only know in newSelectorFactory due to functions (which needs the defs passed to newSelectorFactory to resolve which
+ // function is called). Note that this doesn't really matter performance wise since the factories are still created during
+ // preparation of the corresponding SelectStatement.
+ public final Term.Raw element;
+
+ private WithElementSelection(Selectable selected, Term.Raw element)
+ {
+ assert element != null;
+ this.selected = selected;
+ this.element = element;
+ }
+
+ @Override
+ public String toString()
+ {
+ return String.format("%s[%s]", selected, element);
+ }
+
+ public Selector.Factory newSelectorFactory(TableMetadata cfm, AbstractType<?> expectedType, List<ColumnMetadata> defs, VariableSpecifications boundNames)
+ {
+ Selector.Factory factory = selected.newSelectorFactory(cfm, null, defs, boundNames);
+ ColumnSpecification receiver = factory.getColumnSpecification(cfm);
+
+ if (!(receiver.type instanceof CollectionType))
+ throw new InvalidRequestException(String.format("Invalid element selection: %s is of type %s is not a collection", selected, receiver.type.asCQL3Type()));
+
+ ColumnSpecification boundSpec = specForElementOrSlice(selected, receiver, "Element");
+
+ Term elt = element.prepare(cfm.keyspace, boundSpec);
+ elt.collectMarkerSpecification(boundNames);
+ return ElementsSelector.newElementFactory(toString(), factory, (CollectionType)receiver.type, elt);
+ }
+
+ public AbstractType<?> getExactTypeIfKnown(String keyspace)
+ {
+ AbstractType<?> selectedType = selected.getExactTypeIfKnown(keyspace);
+ if (selectedType == null || !(selectedType instanceof CollectionType))
+ return null;
+
+ return ElementsSelector.valueType((CollectionType) selectedType);
+ }
+
+ @Override
+ public boolean selectColumns(Predicate<ColumnMetadata> predicate)
+ {
+ return selected.selectColumns(predicate);
+ }
+
+ public static class Raw extends Selectable.Raw
+ {
+ private final Selectable.Raw selected;
+ private final Term.Raw element;
+
+ public Raw(Selectable.Raw selected, Term.Raw element)
+ {
+ this.selected = selected;
+ this.element = element;
+ }
+
+ public WithElementSelection prepare(TableMetadata cfm)
+ {
+ return new WithElementSelection(selected.prepare(cfm), element);
+ }
+
+ @Override
+ public String toString()
+ {
+ return String.format("%s[%s]", selected, element);
+ }
+ }
+ }
+
+ /**
+ * Represents the selection of a slice of a collection (eg. c[x..y]).
+ */
+ public static class WithSliceSelection implements Selectable
+ {
+ public final Selectable selected;
+ // Note that we can't yet prepare the Term.Raw yet as we need the ColumnSpecificiation corresponding to Selectable, which
+ // we'll only know in newSelectorFactory due to functions (which needs the defs passed to newSelectorFactory to resolve which
+ // function is called). Note that this doesn't really matter performance wise since the factories are still created during
+ // preparation of the corresponding SelectStatement.
+ // Both from and to can be null if they haven't been provided
+ public final Term.Raw from;
+ public final Term.Raw to;
+
+ private WithSliceSelection(Selectable selected, Term.Raw from, Term.Raw to)
+ {
+ this.selected = selected;
+ this.from = from;
+ this.to = to;
+ }
+
+ @Override
+ public String toString()
+ {
+ return String.format("%s[%s..%s]", selected, from == null ? "" : from, to == null ? "" : to);
+ }
+
+ public Selector.Factory newSelectorFactory(TableMetadata cfm, AbstractType<?> expectedType, List<ColumnMetadata> defs, VariableSpecifications boundNames)
+ {
+ // Note that a slice gives you the same type as the collection you applied it to, so we can pass expectedType for selected directly
+ Selector.Factory factory = selected.newSelectorFactory(cfm, expectedType, defs, boundNames);
+ ColumnSpecification receiver = factory.getColumnSpecification(cfm);
+
+ if (!(receiver.type instanceof CollectionType))
+ throw new InvalidRequestException(String.format("Invalid slice selection: %s of type %s is not a collection", selected, receiver.type.asCQL3Type()));
+
+ ColumnSpecification boundSpec = specForElementOrSlice(selected, receiver, "Slice");
+
+ // If from or to are null, this means the user didn't provide on in the syntax (we had c[x..] or c[..x]).
+ // The equivalent of doing this when preparing values would be to use UNSET.
+ Term f = from == null ? Constants.UNSET_VALUE : from.prepare(cfm.keyspace, boundSpec);
+ Term t = to == null ? Constants.UNSET_VALUE : to.prepare(cfm.keyspace, boundSpec);
+ f.collectMarkerSpecification(boundNames);
+ t.collectMarkerSpecification(boundNames);
+ return ElementsSelector.newSliceFactory(toString(), factory, (CollectionType)receiver.type, f, t);
+ }
+
+ public AbstractType<?> getExactTypeIfKnown(String keyspace)
+ {
+ AbstractType<?> selectedType = selected.getExactTypeIfKnown(keyspace);
+ if (selectedType == null || !(selectedType instanceof CollectionType))
+ return null;
+
+ return selectedType;
+ }
+
+ @Override
+ public boolean selectColumns(Predicate<ColumnMetadata> predicate)
+ {
+ return selected.selectColumns(predicate);
+ }
+
+ public static class Raw extends Selectable.Raw
+ {
+ private final Selectable.Raw selected;
+ // Both from and to can be null if they haven't been provided
+ private final Term.Raw from;
+ private final Term.Raw to;
+
+ public Raw(Selectable.Raw selected, Term.Raw from, Term.Raw to)
+ {
+ this.selected = selected;
+ this.from = from;
+ this.to = to;
+ }
+
+ public WithSliceSelection prepare(TableMetadata cfm)
+ {
+ return new WithSliceSelection(selected.prepare(cfm), from, to);
+ }
+
+ @Override
+ public String toString()
+ {
+ return String.format("%s[%s..%s]", selected, from == null ? "" : from, to == null ? "" : to);
}
}
}
diff --git a/src/java/org/apache/cassandra/cql3/selection/Selection.java b/src/java/org/apache/cassandra/cql3/selection/Selection.java
index 7e9b716..fd6a6cc 100644
--- a/src/java/org/apache/cassandra/cql3/selection/Selection.java
+++ b/src/java/org/apache/cassandra/cql3/selection/Selection.java
@@ -26,55 +26,52 @@
import com.google.common.collect.Iterators;
import com.google.common.collect.Lists;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
import org.apache.cassandra.cql3.*;
import org.apache.cassandra.cql3.functions.Function;
-import org.apache.cassandra.db.Clustering;
-import org.apache.cassandra.db.DecoratedKey;
-import org.apache.cassandra.db.aggregation.AggregationSpecification;
-import org.apache.cassandra.db.aggregation.GroupMaker;
-import org.apache.cassandra.db.context.CounterContext;
+import org.apache.cassandra.db.filter.ColumnFilter;
import org.apache.cassandra.db.marshal.UTF8Type;
-import org.apache.cassandra.db.rows.Cell;
import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.transport.ProtocolVersion;
-import org.apache.cassandra.utils.ByteBufferUtil;
public abstract class Selection
{
/**
* A predicate that returns <code>true</code> for static columns.
*/
- private static final Predicate<ColumnDefinition> STATIC_COLUMN_FILTER = new Predicate<ColumnDefinition>()
- {
- public boolean apply(ColumnDefinition def)
- {
- return def.isStatic();
- }
- };
+ private static final Predicate<ColumnMetadata> STATIC_COLUMN_FILTER = (column) -> column.isStatic();
- private final CFMetaData cfm;
- private final List<ColumnDefinition> columns;
+ private final TableMetadata table;
+ private final List<ColumnMetadata> columns;
private final SelectionColumnMapping columnMapping;
- private final ResultSet.ResultMetadata metadata;
- private final boolean collectTimestamps;
- private final boolean collectTTLs;
- // Columns used to order the result set for multi-partition queries
- private Map<ColumnDefinition, Integer> orderingIndex;
+ protected final ResultSet.ResultMetadata metadata;
+ protected final ColumnFilterFactory columnFilterFactory;
+ protected final boolean isJson;
- protected Selection(CFMetaData cfm,
- List<ColumnDefinition> columns,
+ // Columns used to order the result set for JSON queries with post ordering.
+ protected final List<ColumnMetadata> orderingColumns;
+
+ protected Selection(TableMetadata table,
+ List<ColumnMetadata> selectedColumns,
+ Set<ColumnMetadata> orderingColumns,
SelectionColumnMapping columnMapping,
- boolean collectTimestamps,
- boolean collectTTLs)
+ ColumnFilterFactory columnFilterFactory,
+ boolean isJson)
{
- this.cfm = cfm;
- this.columns = columns;
+ this.table = table;
+ this.columns = selectedColumns;
this.columnMapping = columnMapping;
this.metadata = new ResultSet.ResultMetadata(columnMapping.getColumnSpecifications());
- this.collectTimestamps = collectTimestamps;
- this.collectTTLs = collectTTLs;
+ this.columnFilterFactory = columnFilterFactory;
+ this.isJson = isJson;
+
+ // If we order post-query, the sorted column needs to be in the ResultSet for sorting,
+ // even if we don't ultimately ship them to the client (CASSANDRA-4911).
+ this.columns.addAll(orderingColumns);
+ this.metadata.addNonSerializedColumns(orderingColumns);
+
+ this.orderingColumns = orderingColumns.isEmpty() ? Collections.emptyList() : new ArrayList<>(orderingColumns);
}
// Overriden by SimpleSelection when appropriate.
@@ -89,7 +86,7 @@
*/
public boolean containsStaticColumns()
{
- if (cfm.isStaticCompactTable() || !cfm.hasStaticColumns())
+ if (table.isStaticCompactTable() || !table.hasStaticColumns())
return false;
if (isWildcard())
@@ -99,59 +96,24 @@
}
/**
- * Checks if this selection contains only static columns.
- * @return <code>true</code> if this selection contains only static columns, <code>false</code> otherwise;
+ * Returns the corresponding column index used for post query ordering
+ * @param c ordering column
+ * @return
*/
- public boolean containsOnlyStaticColumns()
- {
- if (!containsStaticColumns())
- return false;
-
- if (isWildcard())
- return false;
-
- for (ColumnDefinition def : getColumns())
- {
- if (!def.isPartitionKey() && !def.isStatic())
- return false;
- }
-
- return true;
- }
-
- /**
- * Checks if this selection contains a complex column.
- *
- * @return <code>true</code> if this selection contains a multicell collection or UDT, <code>false</code> otherwise.
- */
- public boolean containsAComplexColumn()
- {
- for (ColumnDefinition def : getColumns())
- if (def.isComplex())
- return true;
-
- return false;
- }
-
- public Map<ColumnDefinition, Integer> getOrderingIndex(boolean isJson)
+ public Integer getOrderingIndex(ColumnMetadata c)
{
if (!isJson)
- return orderingIndex;
+ return getResultSetIndex(c);
// If we order post-query in json, the first and only column that we ship to the client is the json column.
// In that case, we should keep ordering columns around to perform the ordering, then these columns will
// be placed after the json column. As a consequence of where the colums are placed, we should give the
// ordering index a value based on their position in the json encoding and discard the original index.
// (CASSANDRA-14286)
- int columnIndex = 1;
- Map<ColumnDefinition, Integer> jsonOrderingIndex = new LinkedHashMap<>(orderingIndex.size());
- for (ColumnDefinition column : orderingIndex.keySet())
- jsonOrderingIndex.put(column, columnIndex++);
-
- return jsonOrderingIndex;
+ return orderingColumns.indexOf(c) + 1;
}
- public ResultSet.ResultMetadata getResultMetadata(boolean isJson)
+ public ResultSet.ResultMetadata getResultMetadata()
{
if (!isJson)
return metadata;
@@ -159,83 +121,110 @@
ColumnSpecification firstColumn = metadata.names.get(0);
ColumnSpecification jsonSpec = new ColumnSpecification(firstColumn.ksName, firstColumn.cfName, Json.JSON_COLUMN_ID, UTF8Type.instance);
ResultSet.ResultMetadata resultMetadata = new ResultSet.ResultMetadata(Lists.newArrayList(jsonSpec));
- if (orderingIndex != null)
- {
- for (ColumnDefinition orderingColumn : orderingIndex.keySet())
- resultMetadata.addNonSerializedColumn(orderingColumn);
- }
+ resultMetadata.addNonSerializedColumns(orderingColumns);
return resultMetadata;
}
- public static Selection wildcard(CFMetaData cfm)
+ public static Selection wildcard(TableMetadata table, boolean isJson)
{
- List<ColumnDefinition> all = new ArrayList<>(cfm.allColumns().size());
- Iterators.addAll(all, cfm.allColumnsInSelectOrder());
- return new SimpleSelection(cfm, all, true);
+ List<ColumnMetadata> all = new ArrayList<>(table.columns().size());
+ Iterators.addAll(all, table.allColumnsInSelectOrder());
+ return new SimpleSelection(table, all, Collections.emptySet(), true, isJson);
}
- public static Selection wildcardWithGroupBy(CFMetaData cfm, VariableSpecifications boundNames)
+ public static Selection wildcardWithGroupBy(TableMetadata table,
+ VariableSpecifications boundNames,
+ boolean isJson)
{
- List<RawSelector> rawSelectors = new ArrayList<>(cfm.allColumns().size());
- Iterator<ColumnDefinition> iter = cfm.allColumnsInSelectOrder();
- while (iter.hasNext())
- {
- ColumnDefinition.Raw raw = ColumnDefinition.Raw.forColumn(iter.next());
- rawSelectors.add(new RawSelector(raw, null));
- }
- return fromSelectors(cfm, rawSelectors, boundNames, true);
+ return fromSelectors(table,
+ Lists.newArrayList(table.allColumnsInSelectOrder()),
+ boundNames,
+ Collections.emptySet(),
+ Collections.emptySet(),
+ true,
+ isJson);
}
- public static Selection forColumns(CFMetaData cfm, List<ColumnDefinition> columns)
+ public static Selection forColumns(TableMetadata table, List<ColumnMetadata> columns)
{
- return new SimpleSelection(cfm, columns, false);
- }
-
- public void addColumnForOrdering(ColumnDefinition c)
- {
- if (orderingIndex == null)
- orderingIndex = new LinkedHashMap<>();
-
- int index = getResultSetIndex(c);
-
- if (index < 0)
- index = addOrderingColumn(c);
-
- orderingIndex.put(c, index);
- }
-
- protected int addOrderingColumn(ColumnDefinition c)
- {
- columns.add(c);
- metadata.addNonSerializedColumn(c);
- return columns.size() - 1;
+ return new SimpleSelection(table, columns, Collections.emptySet(), false, false);
}
public void addFunctionsTo(List<Function> functions)
{
}
- private static boolean processesSelection(List<RawSelector> rawSelectors)
+ private static boolean processesSelection(List<Selectable> selectables)
{
- for (RawSelector rawSelector : rawSelectors)
+ for (Selectable selectable : selectables)
{
- if (rawSelector.processesSelection())
+ if (selectable.processesSelection())
return true;
}
return false;
}
- public static Selection fromSelectors(CFMetaData cfm, List<RawSelector> rawSelectors, VariableSpecifications boundNames, boolean hasGroupBy)
+ public static Selection fromSelectors(TableMetadata table,
+ List<Selectable> selectables,
+ VariableSpecifications boundNames,
+ Set<ColumnMetadata> orderingColumns,
+ Set<ColumnMetadata> nonPKRestrictedColumns,
+ boolean hasGroupBy,
+ boolean isJson)
{
- List<ColumnDefinition> defs = new ArrayList<>();
+ List<ColumnMetadata> selectedColumns = new ArrayList<>();
SelectorFactories factories =
- SelectorFactories.createFactoriesAndCollectColumnDefinitions(RawSelector.toSelectables(rawSelectors, cfm), null, cfm, defs, boundNames);
- SelectionColumnMapping mapping = collectColumnMappings(cfm, rawSelectors, factories);
+ SelectorFactories.createFactoriesAndCollectColumnDefinitions(selectables, null, table, selectedColumns, boundNames);
+ SelectionColumnMapping mapping = collectColumnMappings(table, factories);
- return (processesSelection(rawSelectors) || rawSelectors.size() != defs.size() || hasGroupBy)
- ? new SelectionWithProcessing(cfm, defs, mapping, factories)
- : new SimpleSelection(cfm, defs, mapping, false);
+ Set<ColumnMetadata> filteredOrderingColumns = filterOrderingColumns(orderingColumns,
+ selectedColumns,
+ factories,
+ isJson);
+
+ return (processesSelection(selectables) || selectables.size() != selectedColumns.size() || hasGroupBy)
+ ? new SelectionWithProcessing(table,
+ selectedColumns,
+ filteredOrderingColumns,
+ nonPKRestrictedColumns,
+ mapping,
+ factories,
+ isJson)
+ : new SimpleSelection(table,
+ selectedColumns,
+ filteredOrderingColumns,
+ nonPKRestrictedColumns,
+ mapping,
+ isJson);
+ }
+
+ /**
+ * Removes the ordering columns that are already selected.
+ *
+ * @param orderingColumns the columns used to order the results
+ * @param selectedColumns the selected columns
+ * @param factories the factory used to create the selectors
+ * @return the ordering columns that are not part of the selection
+ */
+ private static Set<ColumnMetadata> filterOrderingColumns(Set<ColumnMetadata> orderingColumns,
+ List<ColumnMetadata> selectedColumns,
+ SelectorFactories factories,
+ boolean isJson)
+ {
+ // CASSANDRA-14286
+ if (isJson)
+ return orderingColumns;
+ Set<ColumnMetadata> filteredOrderingColumns = new LinkedHashSet<>(orderingColumns.size());
+ for (ColumnMetadata orderingColumn : orderingColumns)
+ {
+ int index = selectedColumns.indexOf(orderingColumn);
+ if (index >= 0 && factories.indexOfSimpleSelectorFactory(index) >= 0)
+ continue;
+
+ filteredOrderingColumns.add(orderingColumn);
+ }
+ return filteredOrderingColumns;
}
/**
@@ -243,7 +232,7 @@
* @param c the column
* @return the index of the specified column within the resultset or -1
*/
- public int getResultSetIndex(ColumnDefinition c)
+ public int getResultSetIndex(ColumnMetadata c)
{
return getColumnIndex(c);
}
@@ -253,36 +242,29 @@
* @param c the column
* @return the index of the specified column or -1
*/
- protected final int getColumnIndex(ColumnDefinition c)
+ protected final int getColumnIndex(ColumnMetadata c)
{
- for (int i = 0, m = columns.size(); i < m; i++)
- if (columns.get(i).name.equals(c.name))
- return i;
- return -1;
+ return columns.indexOf(c);
}
- private static SelectionColumnMapping collectColumnMappings(CFMetaData cfm,
- List<RawSelector> rawSelectors,
+ private static SelectionColumnMapping collectColumnMappings(TableMetadata table,
SelectorFactories factories)
{
SelectionColumnMapping selectionColumns = SelectionColumnMapping.newMapping();
- Iterator<RawSelector> iter = rawSelectors.iterator();
for (Selector.Factory factory : factories)
{
- ColumnSpecification colSpec = factory.getColumnSpecification(cfm);
- ColumnIdentifier alias = iter.next().alias;
- factory.addColumnMapping(selectionColumns,
- alias == null ? colSpec : colSpec.withAlias(alias));
+ ColumnSpecification colSpec = factory.getColumnSpecification(table);
+ factory.addColumnMapping(selectionColumns, colSpec);
}
return selectionColumns;
}
- protected abstract Selectors newSelectors(QueryOptions options) throws InvalidRequestException;
+ public abstract Selectors newSelectors(QueryOptions options);
/**
* @return the list of CQL3 columns value this SelectionClause needs.
*/
- public List<ColumnDefinition> getColumns()
+ public List<ColumnMetadata> getColumns()
{
return columns;
}
@@ -295,17 +277,6 @@
return columnMapping;
}
- public ResultSetBuilder resultSetBuilder(QueryOptions options, boolean isJson)
- {
- return new ResultSetBuilder(options, isJson);
- }
-
- public ResultSetBuilder resultSetBuilder(QueryOptions options, boolean isJson, AggregationSpecification aggregationSpec)
- {
- return aggregationSpec == null ? new ResultSetBuilder(options, isJson)
- : new ResultSetBuilder(options, isJson, aggregationSpec.newGroupMaker());
- }
-
public abstract boolean isAggregate();
@Override
@@ -315,25 +286,37 @@
.add("columns", columns)
.add("columnMapping", columnMapping)
.add("metadata", metadata)
- .add("collectTimestamps", collectTimestamps)
- .add("collectTTLs", collectTTLs)
.toString();
}
- public static List<ByteBuffer> rowToJson(List<ByteBuffer> row, ProtocolVersion protocolVersion, ResultSet.ResultMetadata metadata)
+ private static List<ByteBuffer> rowToJson(List<ByteBuffer> row,
+ ProtocolVersion protocolVersion,
+ ResultSet.ResultMetadata metadata,
+ List<ColumnMetadata> orderingColumns)
{
+ ByteBuffer[] jsonRow = new ByteBuffer[orderingColumns.size() + 1];
StringBuilder sb = new StringBuilder("{");
- for (int i = 0; i < metadata.getColumnCount(); i++)
+ for (int i = 0; i < metadata.names.size(); i++)
{
+ ColumnSpecification spec = metadata.names.get(i);
+ ByteBuffer buffer = row.get(i);
+
+ // If it is an ordering column we need to keep it in case we need it for post ordering
+ int index = orderingColumns.indexOf(spec);
+ if (index >= 0)
+ jsonRow[index + 1] = buffer;
+
+ // If the column is only used for ordering we can stop here.
+ if (i >= metadata.getColumnCount())
+ continue;
+
if (i > 0)
sb.append(", ");
- ColumnSpecification spec = metadata.names.get(i);
String columnName = spec.name.toString();
if (!columnName.equals(columnName.toLowerCase(Locale.US)))
columnName = "\"" + columnName + "\"";
- ByteBuffer buffer = row.get(i);
sb.append('"');
sb.append(Json.quoteAsJsonString(columnName));
sb.append("\": ");
@@ -343,182 +326,52 @@
sb.append(spec.type.toJSONString(buffer, protocolVersion));
}
sb.append("}");
- List<ByteBuffer> jsonRow = new ArrayList<>();
- jsonRow.add(UTF8Type.instance.getSerializer().serialize(sb.toString()));
- return jsonRow;
+
+ jsonRow[0] = UTF8Type.instance.getSerializer().serialize(sb.toString());
+ return Arrays.asList(jsonRow);
}
- public class ResultSetBuilder
+ public static interface Selectors
{
- private final ResultSet resultSet;
- private final ProtocolVersion protocolVersion;
+ /**
+ * Returns the {@code ColumnFilter} corresponding to those selectors
+ * @return the {@code ColumnFilter} corresponding to those selectors
+ */
+ public ColumnFilter getColumnFilter();
/**
- * As multiple thread can access a <code>Selection</code> instance each <code>ResultSetBuilder</code> will use
- * its own <code>Selectors</code> instance.
+ * Checks if one of the selectors perform some aggregations.
+ * @return {@code true} if one of the selectors perform some aggregations, {@code false} otherwise.
*/
- private final Selectors selectors;
-
- /**
- * The <code>GroupMaker</code> used to build the aggregates.
- */
- private final GroupMaker groupMaker;
-
- /*
- * We'll build CQL3 row one by one.
- * The currentRow is the values for the (CQL3) columns we've fetched.
- * We also collect timestamps and ttls for the case where the writetime and
- * ttl functions are used. Note that we might collect timestamp and/or ttls
- * we don't care about, but since the array below are allocated just once,
- * it doesn't matter performance wise.
- */
- List<ByteBuffer> current;
- final long[] timestamps;
- final int[] ttls;
-
- private final boolean isJson;
-
- private ResultSetBuilder(QueryOptions options, boolean isJson)
- {
- this(options, isJson, null);
- }
-
- private ResultSetBuilder(QueryOptions options, boolean isJson, GroupMaker groupMaker)
- {
- this.resultSet = new ResultSet(getResultMetadata(isJson).copy(), new ArrayList<List<ByteBuffer>>());
- this.protocolVersion = options.getProtocolVersion();
- this.selectors = newSelectors(options);
- this.groupMaker = groupMaker;
- this.timestamps = collectTimestamps ? new long[columns.size()] : null;
- this.ttls = collectTTLs ? new int[columns.size()] : null;
- this.isJson = isJson;
-
- // We use MIN_VALUE to indicate no timestamp and -1 for no ttl
- if (timestamps != null)
- Arrays.fill(timestamps, Long.MIN_VALUE);
- if (ttls != null)
- Arrays.fill(ttls, -1);
- }
-
- public void add(ByteBuffer v)
- {
- current.add(v);
- }
-
- public void add(Cell c, int nowInSec)
- {
- if (c == null)
- {
- current.add(null);
- return;
- }
-
- current.add(value(c));
-
- if (timestamps != null)
- timestamps[current.size() - 1] = c.timestamp();
-
- if (ttls != null)
- ttls[current.size() - 1] = remainingTTL(c, nowInSec);
- }
-
- private int remainingTTL(Cell c, int nowInSec)
- {
- if (!c.isExpiring())
- return -1;
-
- int remaining = c.localDeletionTime() - nowInSec;
- return remaining >= 0 ? remaining : -1;
- }
-
- private ByteBuffer value(Cell c)
- {
- return c.isCounterCell()
- ? ByteBufferUtil.bytes(CounterContext.instance().total(c.value()))
- : c.value();
- }
-
- /**
- * Notifies this <code>Builder</code> that a new row is being processed.
- *
- * @param partitionKey the partition key of the new row
- * @param clustering the clustering of the new row
- */
- public void newRow(DecoratedKey partitionKey, Clustering clustering)
- {
- // The groupMaker needs to be called for each row
- boolean isNewAggregate = groupMaker == null || groupMaker.isNewGroup(partitionKey, clustering);
- if (current != null)
- {
- selectors.addInputRow(protocolVersion, this);
- if (isNewAggregate)
- {
- resultSet.addRow(getOutputRow());
- selectors.reset();
- }
- }
- current = new ArrayList<>(columns.size());
-
- // Timestamps and TTLs are arrays per row, we must null them out between rows
- if (timestamps != null)
- Arrays.fill(timestamps, Long.MIN_VALUE);
- if (ttls != null)
- Arrays.fill(ttls, -1);
- }
-
- /**
- * Builds the <code>ResultSet</code>
- */
- public ResultSet build()
- {
- if (current != null)
- {
- selectors.addInputRow(protocolVersion, this);
- resultSet.addRow(getOutputRow());
- selectors.reset();
- current = null;
- }
-
- // For aggregates we need to return a row even it no records have been found
- if (resultSet.isEmpty() && groupMaker != null && groupMaker.returnAtLeastOneRow())
- resultSet.addRow(getOutputRow());
- return resultSet;
- }
-
- private List<ByteBuffer> getOutputRow()
- {
- List<ByteBuffer> outputRow = selectors.getOutputRow(protocolVersion);
- if (isJson)
- {
- // Keep all columns around for possible post-query ordering. (CASSANDRA-14286)
- List<ByteBuffer> jsonRow = rowToJson(outputRow, protocolVersion, metadata);
-
- // Keep ordering columns around for possible post-query ordering. (CASSANDRA-14286)
- if (orderingIndex != null)
- {
- for (Integer orderingColumnIndex : orderingIndex.values())
- jsonRow.add(outputRow.get(orderingColumnIndex));
- }
- outputRow = jsonRow;
- }
- return outputRow;
- }
- }
-
- private static interface Selectors
- {
public boolean isAggregate();
/**
+ * Returns the number of fetched columns
+ * @return the number of fetched columns
+ */
+ public int numberOfFetchedColumns();
+
+ /**
+ * Checks if one of the selectors collect TTLs.
+ * @return {@code true} if one of the selectors collect TTLs, {@code false} otherwise.
+ */
+ public boolean collectTTLs();
+
+ /**
+ * Checks if one of the selectors collect timestamps.
+ * @return {@code true} if one of the selectors collect timestamps, {@code false} otherwise.
+ */
+ public boolean collectTimestamps();
+
+ /**
* Adds the current row of the specified <code>ResultSetBuilder</code>.
*
- * @param protocolVersion
* @param rs the <code>ResultSetBuilder</code>
* @throws InvalidRequestException
*/
- public void addInputRow(ProtocolVersion protocolVersion, ResultSetBuilder rs) throws InvalidRequestException;
+ public void addInputRow(ResultSetBuilder rs);
- public List<ByteBuffer> getOutputRow(ProtocolVersion protocolVersion) throws InvalidRequestException;
+ public List<ByteBuffer> getOutputRow();
public void reset();
}
@@ -528,22 +381,52 @@
{
private final boolean isWildcard;
- public SimpleSelection(CFMetaData cfm, List<ColumnDefinition> columns, boolean isWildcard)
+ public SimpleSelection(TableMetadata table,
+ List<ColumnMetadata> selectedColumns,
+ Set<ColumnMetadata> orderingColumns,
+ boolean isWildcard,
+ boolean isJson)
{
- this(cfm, columns, SelectionColumnMapping.simpleMapping(columns), isWildcard);
+ this(table,
+ selectedColumns,
+ orderingColumns,
+ SelectionColumnMapping.simpleMapping(selectedColumns),
+ isWildcard ? ColumnFilterFactory.wildcard(table)
+ : ColumnFilterFactory.fromColumns(table, selectedColumns, orderingColumns, Collections.emptySet()),
+ isWildcard,
+ isJson);
}
- public SimpleSelection(CFMetaData cfm,
- List<ColumnDefinition> columns,
- SelectionColumnMapping metadata,
- boolean isWildcard)
+ public SimpleSelection(TableMetadata table,
+ List<ColumnMetadata> selectedColumns,
+ Set<ColumnMetadata> orderingColumns,
+ Set<ColumnMetadata> nonPKRestrictedColumns,
+ SelectionColumnMapping mapping,
+ boolean isJson)
+ {
+ this(table,
+ selectedColumns,
+ orderingColumns,
+ mapping,
+ ColumnFilterFactory.fromColumns(table, selectedColumns, orderingColumns, nonPKRestrictedColumns),
+ false,
+ isJson);
+ }
+
+ private SimpleSelection(TableMetadata table,
+ List<ColumnMetadata> selectedColumns,
+ Set<ColumnMetadata> orderingColumns,
+ SelectionColumnMapping mapping,
+ ColumnFilterFactory columnFilterFactory,
+ boolean isWildcard,
+ boolean isJson)
{
/*
* In theory, even a simple selection could have multiple time the same column, so we
* could filter those duplicate out of columns. But since we're very unlikely to
* get much duplicate in practice, it's more efficient not to bother.
*/
- super(cfm, columns, metadata, false, false);
+ super(table, selectedColumns, orderingColumns, mapping, columnFilterFactory, isJson);
this.isWildcard = isWildcard;
}
@@ -558,7 +441,7 @@
return false;
}
- protected Selectors newSelectors(QueryOptions options)
+ public Selectors newSelectors(QueryOptions options)
{
return new Selectors()
{
@@ -569,12 +452,14 @@
current = null;
}
- public List<ByteBuffer> getOutputRow(ProtocolVersion protocolVersion)
+ public List<ByteBuffer> getOutputRow()
{
+ if (isJson)
+ return rowToJson(current, options.getProtocolVersion(), metadata, orderingColumns);
return current;
}
- public void addInputRow(ProtocolVersion protocolVersion, ResultSetBuilder rs) throws InvalidRequestException
+ public void addInputRow(ResultSetBuilder rs) throws InvalidRequestException
{
current = rs.current;
}
@@ -583,6 +468,32 @@
{
return false;
}
+
+ @Override
+ public int numberOfFetchedColumns()
+ {
+ return getColumns().size();
+ }
+
+ @Override
+ public boolean collectTTLs()
+ {
+ return false;
+ }
+
+ @Override
+ public boolean collectTimestamps()
+ {
+ return false;
+ }
+
+ @Override
+ public ColumnFilter getColumnFilter()
+ {
+ // In the case of simple selection we know that the ColumnFilter has already been computed and
+ // that by consequence the selectors argument has not impact on the output.
+ return columnFilterFactory.newInstance(null);
+ }
};
}
}
@@ -590,19 +501,32 @@
private static class SelectionWithProcessing extends Selection
{
private final SelectorFactories factories;
+ private final boolean collectTimestamps;
+ private final boolean collectTTLs;
- public SelectionWithProcessing(CFMetaData cfm,
- List<ColumnDefinition> columns,
+ public SelectionWithProcessing(TableMetadata table,
+ List<ColumnMetadata> columns,
+ Set<ColumnMetadata> orderingColumns,
+ Set<ColumnMetadata> nonPKRestrictedColumns,
SelectionColumnMapping metadata,
- SelectorFactories factories) throws InvalidRequestException
+ SelectorFactories factories,
+ boolean isJson)
{
- super(cfm,
+ super(table,
columns,
+ orderingColumns,
metadata,
- factories.containsWritetimeSelectorFactory(),
- factories.containsTTLSelectorFactory());
+ ColumnFilterFactory.fromSelectorFactories(table, factories, orderingColumns, nonPKRestrictedColumns),
+ isJson);
this.factories = factories;
+ this.collectTimestamps = factories.containsWritetimeSelectorFactory();
+ this.collectTTLs = factories.containsTTLSelectorFactory();;
+
+ for (ColumnMetadata orderingColumn : orderingColumns)
+ {
+ factories.addSelectorForOrdering(orderingColumn, getColumnIndex(orderingColumn));
+ }
}
@Override
@@ -612,26 +536,9 @@
}
@Override
- public int getResultSetIndex(ColumnDefinition c)
+ public int getResultSetIndex(ColumnMetadata c)
{
- int index = getColumnIndex(c);
-
- if (index < 0)
- return -1;
-
- for (int i = 0, m = factories.size(); i < m; i++)
- if (factories.get(i).isSimpleSelectorFactory(index))
- return i;
-
- return -1;
- }
-
- @Override
- protected int addOrderingColumn(ColumnDefinition c)
- {
- int index = super.addOrderingColumn(c);
- factories.addSelectorForOrdering(c, index);
- return factories.size() - 1;
+ return factories.indexOfSimpleSelectorFactory(super.getResultSetIndex(c));
}
public boolean isAggregate()
@@ -639,7 +546,7 @@
return factories.doesAggregation();
}
- protected Selectors newSelectors(final QueryOptions options) throws InvalidRequestException
+ public Selectors newSelectors(final QueryOptions options) throws InvalidRequestException
{
return new Selectors()
{
@@ -656,20 +563,44 @@
return factories.doesAggregation();
}
- public List<ByteBuffer> getOutputRow(ProtocolVersion protocolVersion) throws InvalidRequestException
+ public List<ByteBuffer> getOutputRow()
{
List<ByteBuffer> outputRow = new ArrayList<>(selectors.size());
for (Selector selector: selectors)
- outputRow.add(selector.getOutput(protocolVersion));
+ outputRow.add(selector.getOutput(options.getProtocolVersion()));
- return outputRow;
+ return isJson ? rowToJson(outputRow, options.getProtocolVersion(), metadata, orderingColumns) : outputRow;
}
- public void addInputRow(ProtocolVersion protocolVersion, ResultSetBuilder rs) throws InvalidRequestException
+ public void addInputRow(ResultSetBuilder rs) throws InvalidRequestException
{
for (Selector selector : selectors)
- selector.addInput(protocolVersion, rs);
+ selector.addInput(options.getProtocolVersion(), rs);
+ }
+
+ @Override
+ public int numberOfFetchedColumns()
+ {
+ return getColumns().size();
+ }
+
+ @Override
+ public boolean collectTTLs()
+ {
+ return collectTTLs;
+ }
+
+ @Override
+ public boolean collectTimestamps()
+ {
+ return collectTimestamps;
+ }
+
+ @Override
+ public ColumnFilter getColumnFilter()
+ {
+ return columnFilterFactory.newInstance(selectors);
}
};
}
diff --git a/src/java/org/apache/cassandra/cql3/selection/SelectionColumnMapping.java b/src/java/org/apache/cassandra/cql3/selection/SelectionColumnMapping.java
index 5072066..5d3727f 100644
--- a/src/java/org/apache/cassandra/cql3/selection/SelectionColumnMapping.java
+++ b/src/java/org/apache/cassandra/cql3/selection/SelectionColumnMapping.java
@@ -26,7 +26,7 @@
import com.google.common.base.Objects;
import com.google.common.collect.*;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.cql3.ColumnSpecification;
/**
@@ -37,8 +37,8 @@
*/
public class SelectionColumnMapping implements SelectionColumns
{
- private final ArrayList<ColumnSpecification> columnSpecifications;
- private final HashMultimap<ColumnSpecification, ColumnDefinition> columnMappings;
+ private final List<ColumnSpecification> columnSpecifications;
+ private final Multimap<ColumnSpecification, ColumnMetadata> columnMappings;
private SelectionColumnMapping()
{
@@ -51,15 +51,15 @@
return new SelectionColumnMapping();
}
- protected static SelectionColumnMapping simpleMapping(Iterable<ColumnDefinition> columnDefinitions)
+ protected static SelectionColumnMapping simpleMapping(Iterable<ColumnMetadata> columnDefinitions)
{
SelectionColumnMapping mapping = new SelectionColumnMapping();
- for (ColumnDefinition def: columnDefinitions)
+ for (ColumnMetadata def: columnDefinitions)
mapping.addMapping(def, def);
return mapping;
}
- protected SelectionColumnMapping addMapping(ColumnSpecification colSpec, ColumnDefinition column)
+ protected SelectionColumnMapping addMapping(ColumnSpecification colSpec, ColumnMetadata column)
{
columnSpecifications.add(colSpec);
// functions without arguments do not map to any column, so don't
@@ -69,7 +69,7 @@
return this;
}
- protected SelectionColumnMapping addMapping(ColumnSpecification colSpec, Iterable<ColumnDefinition> columns)
+ protected SelectionColumnMapping addMapping(ColumnSpecification colSpec, Iterable<ColumnMetadata> columns)
{
columnSpecifications.add(colSpec);
columnMappings.putAll(colSpec, columns);
@@ -83,7 +83,7 @@
return Lists.newArrayList(columnSpecifications);
}
- public Multimap<ColumnSpecification, ColumnDefinition> getMappings()
+ public Multimap<ColumnSpecification, ColumnMetadata> getMappings()
{
return Multimaps.unmodifiableMultimap(columnMappings);
}
diff --git a/src/java/org/apache/cassandra/cql3/selection/SelectionColumns.java b/src/java/org/apache/cassandra/cql3/selection/SelectionColumns.java
index 151a2f3..f4a8593 100644
--- a/src/java/org/apache/cassandra/cql3/selection/SelectionColumns.java
+++ b/src/java/org/apache/cassandra/cql3/selection/SelectionColumns.java
@@ -24,7 +24,7 @@
import com.google.common.collect.Multimap;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.cql3.ColumnSpecification;
/**
@@ -34,5 +34,5 @@
public interface SelectionColumns
{
List<ColumnSpecification> getColumnSpecifications();
- Multimap<ColumnSpecification, ColumnDefinition> getMappings();
+ Multimap<ColumnSpecification, ColumnMetadata> getMappings();
}
diff --git a/src/java/org/apache/cassandra/cql3/selection/Selector.java b/src/java/org/apache/cassandra/cql3/selection/Selector.java
index 922b57f..3262b9c 100644
--- a/src/java/org/apache/cassandra/cql3/selection/Selector.java
+++ b/src/java/org/apache/cassandra/cql3/selection/Selector.java
@@ -20,12 +20,12 @@
import java.nio.ByteBuffer;
import java.util.List;
-import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.cql3.ColumnIdentifier;
import org.apache.cassandra.cql3.ColumnSpecification;
import org.apache.cassandra.cql3.QueryOptions;
import org.apache.cassandra.cql3.functions.Function;
-import org.apache.cassandra.cql3.selection.Selection.ResultSetBuilder;
+import org.apache.cassandra.db.filter.ColumnFilter;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.exceptions.InvalidRequestException;
import org.apache.cassandra.transport.ProtocolVersion;
@@ -51,13 +51,13 @@
* Returns the column specification corresponding to the output value of the selector instances created by
* this factory.
*
- * @param cfm the column family meta data
+ * @param table the table meta data
* @return a column specification
*/
- public final ColumnSpecification getColumnSpecification(CFMetaData cfm)
+ public ColumnSpecification getColumnSpecification(TableMetadata table)
{
- return new ColumnSpecification(cfm.ksName,
- cfm.cfName,
+ return new ColumnSpecification(table.keyspace,
+ table.name,
new ColumnIdentifier(getColumnName(), true), // note that the name is not necessarily
// a true column name so we shouldn't intern it
getReturnType());
@@ -106,13 +106,25 @@
}
/**
+ * Checks if this factory creates <code>Selector</code>s that simply return a column value.
+ *
+ * @param index the column index
+ * @return <code>true</code> if this factory creates <code>Selector</code>s that simply return a column value,
+ * <code>false</code> otherwise.
+ */
+ public boolean isSimpleSelectorFactory()
+ {
+ return false;
+ }
+
+ /**
* Checks if this factory creates <code>Selector</code>s that simply return the specified column.
*
* @param index the column index
* @return <code>true</code> if this factory creates <code>Selector</code>s that simply return
* the specified column, <code>false</code> otherwise.
*/
- public boolean isSimpleSelectorFactory(int index)
+ public boolean isSimpleSelectorFactoryFor(int index)
{
return false;
}
@@ -144,9 +156,33 @@
* by the Selector are to be mapped
*/
protected abstract void addColumnMapping(SelectionColumnMapping mapping, ColumnSpecification resultsColumn);
+
+ /**
+ * Checks if all the columns fetched by the selector created by this factory are known
+ * @return {@code true} if all the columns fetched by the selector created by this factory are known,
+ * {@code false} otherwise.
+ */
+ abstract boolean areAllFetchedColumnsKnown();
+
+ /**
+ * Adds the columns fetched by the selector created by this factory to the provided builder, assuming the
+ * factory is terminal (i.e. that {@code isTerminal() == true}).
+ *
+ * @param builder the column builder to add fetched columns (and potential subselection) to.
+ * @throws AssertionError if the method is called on a factory where {@code isTerminal()} returns {@code false}.
+ */
+ abstract void addFetchedColumns(ColumnFilter.Builder builder);
}
/**
+ * Add to the provided builder the column (and potential subselections) to fetch for this
+ * selection.
+ *
+ * @param builder the builder to add columns and subselections to.
+ */
+ public abstract void addFetchedColumns(ColumnFilter.Builder builder);
+
+ /**
* Add the current value from the specified <code>ResultSetBuilder</code>.
*
* @param protocolVersion protocol version used for serialization
@@ -172,17 +208,6 @@
public abstract AbstractType<?> getType();
/**
- * Checks if this <code>Selector</code> is creating aggregates.
- *
- * @return <code>true</code> if this <code>Selector</code> is creating aggregates <code>false</code>
- * otherwise.
- */
- public boolean isAggregate()
- {
- return false;
- }
-
- /**
* Reset the internal state of this <code>Selector</code>.
*/
public abstract void reset();
diff --git a/src/java/org/apache/cassandra/cql3/selection/SelectorFactories.java b/src/java/org/apache/cassandra/cql3/selection/SelectorFactories.java
index 41bf193..7f4bcb3 100644
--- a/src/java/org/apache/cassandra/cql3/selection/SelectorFactories.java
+++ b/src/java/org/apache/cassandra/cql3/selection/SelectorFactories.java
@@ -21,12 +21,13 @@
import com.google.common.collect.Lists;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.cql3.QueryOptions;
import org.apache.cassandra.cql3.VariableSpecifications;
import org.apache.cassandra.cql3.functions.Function;
import org.apache.cassandra.cql3.selection.Selector.Factory;
+import org.apache.cassandra.db.filter.ColumnFilter.Builder;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.exceptions.InvalidRequestException;
@@ -63,7 +64,7 @@
* is any such expectations, or {@code null} otherwise. This will be {@code null} when called on
* the top-level selectables, but may not be for selectable nested within a function for instance
* (as the argument selectable will be expected to be of the type expected by the function).
- * @param cfm the Column Family Definition
+ * @param table the table Definition
* @param defs the collector parameter for the column definitions
* @param boundNames the collector for the specification of bound markers in the selection
* @return a new <code>SelectorFactories</code> instance
@@ -71,18 +72,18 @@
*/
public static SelectorFactories createFactoriesAndCollectColumnDefinitions(List<Selectable> selectables,
List<AbstractType<?>> expectedTypes,
- CFMetaData cfm,
- List<ColumnDefinition> defs,
+ TableMetadata table,
+ List<ColumnMetadata> defs,
VariableSpecifications boundNames)
throws InvalidRequestException
{
- return new SelectorFactories(selectables, expectedTypes, cfm, defs, boundNames);
+ return new SelectorFactories(selectables, expectedTypes, table, defs, boundNames);
}
private SelectorFactories(List<Selectable> selectables,
List<AbstractType<?>> expectedTypes,
- CFMetaData cfm,
- List<ColumnDefinition> defs,
+ TableMetadata table,
+ List<ColumnMetadata> defs,
VariableSpecifications boundNames)
throws InvalidRequestException
{
@@ -92,7 +93,7 @@
{
Selectable selectable = selectables.get(i);
AbstractType<?> expectedType = expectedTypes == null ? null : expectedTypes.get(i);
- Factory factory = selectable.newSelectorFactory(cfm, expectedType, defs, boundNames);
+ Factory factory = selectable.newSelectorFactory(table, expectedType, defs, boundNames);
containsWritetimeFactory |= factory.isWritetimeSelectorFactory();
containsTTLFactory |= factory.isTTLSelectorFactory();
if (factory.isAggregateSelectorFactory())
@@ -118,11 +119,27 @@
}
/**
+ * Returns the index of the {@code SimpleSelector.Factory} for the specified column.
+ *
+ * @param columnIndex the index of the column
+ * @return the index of the {@code SimpleSelector.Factory} for the specified column or -1 if it does not exist.
+ */
+ public int indexOfSimpleSelectorFactory(int columnIndex)
+ {
+ for (int i = 0, m = factories.size(); i < m; i++)
+ {
+ if (factories.get(i).isSimpleSelectorFactoryFor(columnIndex))
+ return i;
+ }
+ return -1;
+ }
+
+ /**
* Adds a new <code>Selector.Factory</code> for a column that is needed only for ORDER BY purposes.
* @param def the column that is needed for ordering
* @param index the index of the column definition in the Selection's list of columns
*/
- public void addSelectorForOrdering(ColumnDefinition def, int index)
+ public void addSelectorForOrdering(ColumnMetadata def, int index)
{
factories.add(SimpleSelector.newFactory(def, index));
}
@@ -211,6 +228,22 @@
});
}
+ boolean areAllFetchedColumnsKnown()
+ {
+ for (Factory factory : factories)
+ {
+ if (!factory.areAllFetchedColumnsKnown())
+ return false;
+ }
+ return true;
+ }
+
+ void addFetchedColumns(Builder builder)
+ {
+ for (Factory factory : factories)
+ factory.addFetchedColumns(builder);
+ }
+
/**
* Returns the number of factories.
* @return the number of factories
diff --git a/src/java/org/apache/cassandra/cql3/selection/SetSelector.java b/src/java/org/apache/cassandra/cql3/selection/SetSelector.java
new file mode 100644
index 0000000..6693121
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/selection/SetSelector.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.selection;
+
+import java.nio.ByteBuffer;
+import java.util.List;
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.apache.cassandra.cql3.QueryOptions;
+import org.apache.cassandra.cql3.Sets;
+import org.apache.cassandra.db.filter.ColumnFilter.Builder;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.SetType;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.serializers.CollectionSerializer;
+import org.apache.cassandra.transport.ProtocolVersion;
+
+/**
+ * <code>Selector</code> for literal set (e.g. {min(value), max(value), count(value)}).
+ *
+ */
+final class SetSelector extends Selector
+{
+ /**
+ * The set type.
+ */
+ private final SetType<?> type;
+
+ /**
+ * The set elements
+ */
+ private final List<Selector> elements;
+
+ public static Factory newFactory(final AbstractType<?> type, final SelectorFactories factories)
+ {
+ return new CollectionFactory(type, factories)
+ {
+ protected String getColumnName()
+ {
+ return Sets.setToString(factories, Factory::getColumnName);
+ }
+
+ public Selector newInstance(final QueryOptions options)
+ {
+ return new SetSelector(type, factories.newInstances(options));
+ }
+ };
+ }
+
+ @Override
+ public void addFetchedColumns(Builder builder)
+ {
+ for (int i = 0, m = elements.size(); i < m; i++)
+ elements.get(i).addFetchedColumns(builder);
+ }
+
+ public void addInput(ProtocolVersion protocolVersion, ResultSetBuilder rs) throws InvalidRequestException
+ {
+ for (int i = 0, m = elements.size(); i < m; i++)
+ elements.get(i).addInput(protocolVersion, rs);
+ }
+
+ public ByteBuffer getOutput(ProtocolVersion protocolVersion) throws InvalidRequestException
+ {
+ Set<ByteBuffer> buffers = new TreeSet<>(type.getElementsType());
+ for (int i = 0, m = elements.size(); i < m; i++)
+ {
+ buffers.add(elements.get(i).getOutput(protocolVersion));
+ }
+ return CollectionSerializer.pack(buffers, buffers.size(), protocolVersion);
+ }
+
+ public void reset()
+ {
+ for (int i = 0, m = elements.size(); i < m; i++)
+ elements.get(i).reset();
+ }
+
+ public AbstractType<?> getType()
+ {
+ return type;
+ }
+
+ @Override
+ public String toString()
+ {
+ return Sets.setToString(elements);
+ }
+
+ private SetSelector(AbstractType<?> type, List<Selector> elements)
+ {
+ this.type = (SetType<?>) type;
+ this.elements = elements;
+ }
+}
diff --git a/src/java/org/apache/cassandra/cql3/selection/SimpleSelector.java b/src/java/org/apache/cassandra/cql3/selection/SimpleSelector.java
index 8d5a305..31b1911 100644
--- a/src/java/org/apache/cassandra/cql3/selection/SimpleSelector.java
+++ b/src/java/org/apache/cassandra/cql3/selection/SimpleSelector.java
@@ -19,55 +19,97 @@
import java.nio.ByteBuffer;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.cql3.ColumnSpecification;
import org.apache.cassandra.cql3.QueryOptions;
-import org.apache.cassandra.cql3.selection.Selection.ResultSetBuilder;
+import org.apache.cassandra.db.filter.ColumnFilter;
+import org.apache.cassandra.db.filter.ColumnFilter.Builder;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.exceptions.InvalidRequestException;
import org.apache.cassandra.transport.ProtocolVersion;
public final class SimpleSelector extends Selector
{
- private final String columnName;
+ /**
+ * The Factory for {@code SimpleSelector}.
+ */
+ public static final class SimpleSelectorFactory extends Factory
+ {
+ private final int idx;
+
+ private final ColumnMetadata column;
+
+ private SimpleSelectorFactory(int idx, ColumnMetadata def)
+ {
+ this.idx = idx;
+ this.column = def;
+ }
+
+ @Override
+ protected String getColumnName()
+ {
+ return column.name.toString();
+ }
+
+ @Override
+ protected AbstractType<?> getReturnType()
+ {
+ return column.type;
+ }
+
+ protected void addColumnMapping(SelectionColumnMapping mapping, ColumnSpecification resultColumn)
+ {
+ mapping.addMapping(resultColumn, column);
+ }
+
+ @Override
+ public Selector newInstance(QueryOptions options)
+ {
+ return new SimpleSelector(column, idx);
+ }
+
+ @Override
+ public boolean isSimpleSelectorFactory()
+ {
+ return true;
+ }
+
+ @Override
+ public boolean isSimpleSelectorFactoryFor(int index)
+ {
+ return index == idx;
+ }
+
+ public boolean areAllFetchedColumnsKnown()
+ {
+ return true;
+ }
+
+ public void addFetchedColumns(ColumnFilter.Builder builder)
+ {
+ builder.add(column);
+ }
+
+ public ColumnMetadata getColumn()
+ {
+ return column;
+ }
+ }
+
+ public final ColumnMetadata column;
private final int idx;
- private final AbstractType<?> type;
private ByteBuffer current;
private boolean isSet;
- public static Factory newFactory(final ColumnDefinition def, final int idx)
+ public static Factory newFactory(final ColumnMetadata def, final int idx)
{
- return new Factory()
- {
- @Override
- protected String getColumnName()
- {
- return def.name.toString();
- }
+ return new SimpleSelectorFactory(idx, def);
+ }
- @Override
- protected AbstractType<?> getReturnType()
- {
- return def.type;
- }
-
- protected void addColumnMapping(SelectionColumnMapping mapping, ColumnSpecification resultColumn)
- {
- mapping.addMapping(resultColumn, def);
- }
-
- @Override
- public Selector newInstance(QueryOptions options)
- {
- return new SimpleSelector(def.name.toString(), idx, def.type);
- }
-
- @Override
- public boolean isSimpleSelectorFactory(int index)
- {
- return index == idx;
- }
- };
+ @Override
+ public void addFetchedColumns(Builder builder)
+ {
+ builder.add(column);
}
@Override
@@ -96,19 +138,18 @@
@Override
public AbstractType<?> getType()
{
- return type;
+ return column.type;
}
@Override
public String toString()
{
- return columnName;
+ return column.name.toString();
}
- private SimpleSelector(String columnName, int idx, AbstractType<?> type)
+ private SimpleSelector(ColumnMetadata column, int idx)
{
- this.columnName = columnName;
+ this.column = column;
this.idx = idx;
- this.type = type;
}
}
diff --git a/src/java/org/apache/cassandra/cql3/selection/TermSelector.java b/src/java/org/apache/cassandra/cql3/selection/TermSelector.java
index 2b0e975..321cd27 100644
--- a/src/java/org/apache/cassandra/cql3/selection/TermSelector.java
+++ b/src/java/org/apache/cassandra/cql3/selection/TermSelector.java
@@ -19,10 +19,11 @@
import java.nio.ByteBuffer;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.cql3.ColumnSpecification;
import org.apache.cassandra.cql3.QueryOptions;
import org.apache.cassandra.cql3.Term;
+import org.apache.cassandra.db.filter.ColumnFilter;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.exceptions.InvalidRequestException;
import org.apache.cassandra.transport.ProtocolVersion;
@@ -54,13 +55,22 @@
protected void addColumnMapping(SelectionColumnMapping mapping, ColumnSpecification resultColumn)
{
- mapping.addMapping(resultColumn, (ColumnDefinition)null);
+ mapping.addMapping(resultColumn, (ColumnMetadata)null);
}
public Selector newInstance(QueryOptions options)
{
return new TermSelector(term.bindAndGet(options), type);
}
+
+ public void addFetchedColumns(ColumnFilter.Builder builder)
+ {
+ }
+
+ public boolean areAllFetchedColumnsKnown()
+ {
+ return true;
+ }
};
}
@@ -70,7 +80,11 @@
this.type = type;
}
- public void addInput(ProtocolVersion protocolVersion, Selection.ResultSetBuilder rs) throws InvalidRequestException
+ public void addFetchedColumns(ColumnFilter.Builder builder)
+ {
+ }
+
+ public void addInput(ProtocolVersion protocolVersion, ResultSetBuilder rs) throws InvalidRequestException
{
}
diff --git a/src/java/org/apache/cassandra/cql3/selection/TupleSelector.java b/src/java/org/apache/cassandra/cql3/selection/TupleSelector.java
new file mode 100644
index 0000000..898085b
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/selection/TupleSelector.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.selection;
+
+import java.nio.ByteBuffer;
+import java.util.List;
+
+import org.apache.cassandra.cql3.QueryOptions;
+import org.apache.cassandra.cql3.Tuples;
+import org.apache.cassandra.db.filter.ColumnFilter.Builder;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.TupleType;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.transport.ProtocolVersion;
+
+/**
+ * <code>Selector</code> for literal tuples (e.g. (min(value), max(value), count(value))).
+ *
+ */
+final class TupleSelector extends Selector
+{
+ /**
+ * The tuple type.
+ */
+ private final AbstractType<?> type;
+
+ /**
+ * The tuple elements
+ */
+ private final List<Selector> elements;
+
+ public static Factory newFactory(final AbstractType<?> type, final SelectorFactories factories)
+ {
+ return new CollectionFactory(type, factories)
+ {
+ protected String getColumnName()
+ {
+ return Tuples.tupleToString(factories, Factory::getColumnName);
+ }
+
+ public Selector newInstance(final QueryOptions options)
+ {
+ return new TupleSelector(type, factories.newInstances(options));
+ }
+ };
+ }
+
+ @Override
+ public void addFetchedColumns(Builder builder)
+ {
+ for (int i = 0, m = elements.size(); i < m; i++)
+ elements.get(i).addFetchedColumns(builder);
+ }
+
+ public void addInput(ProtocolVersion protocolVersion, ResultSetBuilder rs) throws InvalidRequestException
+ {
+ for (int i = 0, m = elements.size(); i < m; i++)
+ elements.get(i).addInput(protocolVersion, rs);
+ }
+
+ public ByteBuffer getOutput(ProtocolVersion protocolVersion) throws InvalidRequestException
+ {
+ ByteBuffer[] buffers = new ByteBuffer[elements.size()];
+ for (int i = 0, m = elements.size(); i < m; i++)
+ {
+ buffers[i] = elements.get(i).getOutput(protocolVersion);
+ }
+ return TupleType.buildValue(buffers);
+ }
+
+ public void reset()
+ {
+ for (int i = 0, m = elements.size(); i < m; i++)
+ elements.get(i).reset();
+ }
+
+ public AbstractType<?> getType()
+ {
+ return type;
+ }
+
+ @Override
+ public String toString()
+ {
+ return Tuples.tupleToString(elements);
+ }
+
+ private TupleSelector(AbstractType<?> type, List<Selector> elements)
+ {
+ this.type = type;
+ this.elements = elements;
+ }
+}
diff --git a/src/java/org/apache/cassandra/cql3/selection/UserTypeSelector.java b/src/java/org/apache/cassandra/cql3/selection/UserTypeSelector.java
new file mode 100644
index 0000000..61faf8d
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/selection/UserTypeSelector.java
@@ -0,0 +1,202 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.selection;
+
+import java.nio.ByteBuffer;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.cql3.ColumnSpecification;
+import org.apache.cassandra.cql3.FieldIdentifier;
+import org.apache.cassandra.cql3.QueryOptions;
+import org.apache.cassandra.cql3.UserTypes;
+import org.apache.cassandra.cql3.functions.Function;
+import org.apache.cassandra.db.filter.ColumnFilter;
+import org.apache.cassandra.db.filter.ColumnFilter.Builder;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.TupleType;
+import org.apache.cassandra.db.marshal.UserType;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.transport.ProtocolVersion;
+
+/**
+ * <code>Selector</code> for literal map (e.g. {'min' : min(value), 'max' : max(value), 'count' : count(value)}).
+ *
+ */
+final class UserTypeSelector extends Selector
+{
+ /**
+ * The map type.
+ */
+ private final AbstractType<?> type;
+
+ /**
+ * The user type fields
+ */
+ private final Map<FieldIdentifier, Selector> fields;
+
+ public static Factory newFactory(final AbstractType<?> type, final Map<FieldIdentifier, Factory> factories)
+ {
+ return new Factory()
+ {
+ protected String getColumnName()
+ {
+ return UserTypes.userTypeToString(factories, Factory::getColumnName);
+ }
+
+ protected AbstractType<?> getReturnType()
+ {
+ return type;
+ }
+
+ protected final void addColumnMapping(SelectionColumnMapping mapping, ColumnSpecification resultsColumn)
+ {
+ SelectionColumnMapping tmpMapping = SelectionColumnMapping.newMapping();
+ for (Factory factory : factories.values())
+ {
+ factory.addColumnMapping(tmpMapping, resultsColumn);
+ }
+
+ if (tmpMapping.getMappings().get(resultsColumn).isEmpty())
+ // add a null mapping for cases where the collection is empty
+ mapping.addMapping(resultsColumn, (ColumnMetadata)null);
+ else
+ // collate the mapped columns from the child factories & add those
+ mapping.addMapping(resultsColumn, tmpMapping.getMappings().values());
+ }
+
+ public Selector newInstance(final QueryOptions options)
+ {
+ Map<FieldIdentifier, Selector> fields = new HashMap<>(factories.size());
+ for (Entry<FieldIdentifier, Factory> factory : factories.entrySet())
+ fields.put(factory.getKey(), factory.getValue().newInstance(options));
+
+ return new UserTypeSelector(type, fields);
+ }
+
+ @Override
+ public boolean isAggregateSelectorFactory()
+ {
+ for (Factory factory : factories.values())
+ {
+ if (factory.isAggregateSelectorFactory())
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ public void addFunctionsTo(List<Function> functions)
+ {
+ for (Factory factory : factories.values())
+ factory.addFunctionsTo(functions);
+ }
+
+ @Override
+ public boolean isWritetimeSelectorFactory()
+ {
+ for (Factory factory : factories.values())
+ {
+ if (factory.isWritetimeSelectorFactory())
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ public boolean isTTLSelectorFactory()
+ {
+ for (Factory factory : factories.values())
+ {
+ if (factory.isTTLSelectorFactory())
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ boolean areAllFetchedColumnsKnown()
+ {
+ for (Factory factory : factories.values())
+ {
+ if (!factory.areAllFetchedColumnsKnown())
+ return false;
+ }
+ return true;
+ }
+
+ @Override
+ void addFetchedColumns(Builder builder)
+ {
+ for (Factory factory : factories.values())
+ factory.addFetchedColumns(builder);
+ }
+ };
+ }
+
+ public void addFetchedColumns(ColumnFilter.Builder builder)
+ {
+ for (Selector field : fields.values())
+ field.addFetchedColumns(builder);
+ }
+
+ public void addInput(ProtocolVersion protocolVersion, ResultSetBuilder rs) throws InvalidRequestException
+ {
+ for (Selector field : fields.values())
+ field.addInput(protocolVersion, rs);
+ }
+
+ public ByteBuffer getOutput(ProtocolVersion protocolVersion) throws InvalidRequestException
+ {
+ UserType userType = (UserType) type;
+ ByteBuffer[] buffers = new ByteBuffer[userType.size()];
+ for (int i = 0, m = userType.size(); i < m; i++)
+ {
+ Selector selector = fields.get(userType.fieldName(i));
+ if (selector != null)
+ buffers[i] = selector.getOutput(protocolVersion);
+ }
+ return TupleType.buildValue(buffers);
+ }
+
+ public void reset()
+ {
+ for (Selector field : fields.values())
+ field.reset();
+ }
+
+ public AbstractType<?> getType()
+ {
+ return type;
+ }
+
+ @Override
+ public String toString()
+ {
+ return UserTypes.userTypeToString(fields);
+ }
+
+ private UserTypeSelector(AbstractType<?> type, Map<FieldIdentifier, Selector> fields)
+ {
+ this.type = type;
+ this.fields = fields;
+ }
+}
diff --git a/src/java/org/apache/cassandra/cql3/selection/WritetimeOrTTLSelector.java b/src/java/org/apache/cassandra/cql3/selection/WritetimeOrTTLSelector.java
index 939f8c2..95586f2 100644
--- a/src/java/org/apache/cassandra/cql3/selection/WritetimeOrTTLSelector.java
+++ b/src/java/org/apache/cassandra/cql3/selection/WritetimeOrTTLSelector.java
@@ -19,10 +19,10 @@
import java.nio.ByteBuffer;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.cql3.QueryOptions;
import org.apache.cassandra.cql3.ColumnSpecification;
-import org.apache.cassandra.cql3.selection.Selection.ResultSetBuilder;
+import org.apache.cassandra.db.filter.ColumnFilter;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.db.marshal.Int32Type;
import org.apache.cassandra.db.marshal.LongType;
@@ -31,13 +31,13 @@
final class WritetimeOrTTLSelector extends Selector
{
- private final String columnName;
+ private final ColumnMetadata column;
private final int idx;
private final boolean isWritetime;
private ByteBuffer current;
private boolean isSet;
- public static Factory newFactory(final ColumnDefinition def, final int idx, final boolean isWritetime)
+ public static Factory newFactory(final ColumnMetadata def, final int idx, final boolean isWritetime)
{
return new Factory()
{
@@ -58,7 +58,7 @@
public Selector newInstance(QueryOptions options)
{
- return new WritetimeOrTTLSelector(def.name.toString(), idx, isWritetime);
+ return new WritetimeOrTTLSelector(def, idx, isWritetime);
}
public boolean isWritetimeSelectorFactory()
@@ -70,9 +70,24 @@
{
return !isWritetime;
}
+
+ public boolean areAllFetchedColumnsKnown()
+ {
+ return true;
+ }
+
+ public void addFetchedColumns(ColumnFilter.Builder builder)
+ {
+ builder.add(def);
+ }
};
}
+ public void addFetchedColumns(ColumnFilter.Builder builder)
+ {
+ builder.add(column);
+ }
+
public void addInput(ProtocolVersion protocolVersion, ResultSetBuilder rs)
{
if (isSet)
@@ -111,14 +126,13 @@
@Override
public String toString()
{
- return columnName;
+ return column.name.toString();
}
- private WritetimeOrTTLSelector(String columnName, int idx, boolean isWritetime)
+ private WritetimeOrTTLSelector(ColumnMetadata column, int idx, boolean isWritetime)
{
- this.columnName = columnName;
+ this.column = column;
this.idx = idx;
this.isWritetime = isWritetime;
}
-
}
diff --git a/src/java/org/apache/cassandra/cql3/statements/AlterKeyspaceStatement.java b/src/java/org/apache/cassandra/cql3/statements/AlterKeyspaceStatement.java
deleted file mode 100644
index 76c8d2f..0000000
--- a/src/java/org/apache/cassandra/cql3/statements/AlterKeyspaceStatement.java
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.cql3.statements;
-
-import org.apache.cassandra.auth.Permission;
-import org.apache.cassandra.config.Schema;
-import org.apache.cassandra.config.SchemaConstants;
-import org.apache.cassandra.exceptions.*;
-import org.apache.cassandra.locator.LocalStrategy;
-import org.apache.cassandra.schema.KeyspaceMetadata;
-import org.apache.cassandra.schema.KeyspaceParams;
-import org.apache.cassandra.service.ClientState;
-import org.apache.cassandra.service.MigrationManager;
-import org.apache.cassandra.service.QueryState;
-import org.apache.cassandra.transport.Event;
-
-public class AlterKeyspaceStatement extends SchemaAlteringStatement
-{
- private final String name;
- private final KeyspaceAttributes attrs;
-
- public AlterKeyspaceStatement(String name, KeyspaceAttributes attrs)
- {
- super();
- this.name = name;
- this.attrs = attrs;
- }
-
- @Override
- public String keyspace()
- {
- return name;
- }
-
- public void checkAccess(ClientState state) throws UnauthorizedException, InvalidRequestException
- {
- state.hasKeyspaceAccess(name, Permission.ALTER);
- }
-
- public void validate(ClientState state) throws RequestValidationException
- {
- KeyspaceMetadata ksm = Schema.instance.getKSMetaData(name);
- if (ksm == null)
- throw new InvalidRequestException("Unknown keyspace " + name);
- if (SchemaConstants.isLocalSystemKeyspace(ksm.name))
- throw new InvalidRequestException("Cannot alter system keyspace");
-
- attrs.validate();
-
- if (attrs.getReplicationStrategyClass() == null && !attrs.getReplicationOptions().isEmpty())
- throw new ConfigurationException("Missing replication strategy class");
-
- if (attrs.getReplicationStrategyClass() != null)
- {
- // The strategy is validated through KSMetaData.validate() in announceKeyspaceUpdate below.
- // However, for backward compatibility with thrift, this doesn't validate unexpected options yet,
- // so doing proper validation here.
- KeyspaceParams params = attrs.asAlteredKeyspaceParams(ksm.params);
- params.validate(name);
- if (params.replication.klass.equals(LocalStrategy.class))
- throw new ConfigurationException("Unable to use given strategy class: LocalStrategy is reserved for internal use.");
- }
- }
-
- public Event.SchemaChange announceMigration(QueryState queryState, boolean isLocalOnly) throws RequestValidationException
- {
- KeyspaceMetadata oldKsm = Schema.instance.getKSMetaData(name);
- // In the (very) unlikely case the keyspace was dropped since validate()
- if (oldKsm == null)
- throw new InvalidRequestException("Unknown keyspace " + name);
-
- KeyspaceMetadata newKsm = oldKsm.withSwapped(attrs.asAlteredKeyspaceParams(oldKsm.params));
- MigrationManager.announceKeyspaceUpdate(newKsm, isLocalOnly);
- return new Event.SchemaChange(Event.SchemaChange.Change.UPDATED, keyspace());
- }
-}
diff --git a/src/java/org/apache/cassandra/cql3/statements/AlterRoleStatement.java b/src/java/org/apache/cassandra/cql3/statements/AlterRoleStatement.java
index 6134741..7a748e8 100644
--- a/src/java/org/apache/cassandra/cql3/statements/AlterRoleStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/AlterRoleStatement.java
@@ -17,6 +17,8 @@
*/
package org.apache.cassandra.cql3.statements;
+import org.apache.cassandra.audit.AuditLogContext;
+import org.apache.cassandra.audit.AuditLogEntryType;
import org.apache.cassandra.auth.*;
import org.apache.cassandra.auth.IRoleManager.Option;
import org.apache.cassandra.config.DatabaseDescriptor;
@@ -24,32 +26,46 @@
import org.apache.cassandra.exceptions.*;
import org.apache.cassandra.service.ClientState;
import org.apache.cassandra.transport.messages.ResultMessage;
+import org.apache.commons.lang3.builder.ToStringBuilder;
+import org.apache.commons.lang3.builder.ToStringStyle;
public class AlterRoleStatement extends AuthenticationStatement
{
private final RoleResource role;
private final RoleOptions opts;
+ final DCPermissions dcPermissions;
public AlterRoleStatement(RoleName name, RoleOptions opts)
{
+ this(name, opts, null);
+ }
+
+ public AlterRoleStatement(RoleName name, RoleOptions opts, DCPermissions dcPermissions)
+ {
this.role = RoleResource.role(name.getName());
this.opts = opts;
+ this.dcPermissions = dcPermissions;
}
public void validate(ClientState state) throws RequestValidationException
{
opts.validate();
- if (opts.isEmpty())
+ if (dcPermissions != null)
+ {
+ dcPermissions.validate();
+ }
+
+ if (opts.isEmpty() && dcPermissions == null)
throw new InvalidRequestException("ALTER [ROLE|USER] can't be empty");
- // validate login here before checkAccess to avoid leaking user existence to anonymous users.
+ // validate login here before authorize to avoid leaking user existence to anonymous users.
state.ensureNotAnonymous();
if (!DatabaseDescriptor.getRoleManager().isExistingRole(role))
throw new InvalidRequestException(String.format("%s doesn't exist", role.getRoleName()));
}
- public void checkAccess(ClientState state) throws UnauthorizedException
+ public void authorize(ClientState state) throws UnauthorizedException
{
AuthenticatedUser user = state.getUser();
boolean isSuper = user.isSuper();
@@ -85,6 +101,20 @@
{
if (!opts.isEmpty())
DatabaseDescriptor.getRoleManager().alterRole(state.getUser(), role, opts);
+ if (dcPermissions != null)
+ DatabaseDescriptor.getNetworkAuthorizer().setRoleDatacenters(role, dcPermissions);
return null;
}
+
+ @Override
+ public String toString()
+ {
+ return ToStringBuilder.reflectionToString(this, ToStringStyle.SHORT_PREFIX_STYLE);
+ }
+
+ @Override
+ public AuditLogContext getAuditLogContext()
+ {
+ return new AuditLogContext(AuditLogEntryType.ALTER_ROLE);
+ }
}
diff --git a/src/java/org/apache/cassandra/cql3/statements/AlterTableStatement.java b/src/java/org/apache/cassandra/cql3/statements/AlterTableStatement.java
deleted file mode 100644
index 3e0674a..0000000
--- a/src/java/org/apache/cassandra/cql3/statements/AlterTableStatement.java
+++ /dev/null
@@ -1,354 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.cql3.statements;
-
-import java.util.*;
-import java.util.stream.Collectors;
-
-import com.google.common.collect.Iterables;
-
-import org.apache.cassandra.auth.Permission;
-import org.apache.cassandra.config.*;
-import org.apache.cassandra.cql3.*;
-import org.apache.cassandra.db.ColumnFamilyStore;
-import org.apache.cassandra.db.Keyspace;
-import org.apache.cassandra.db.marshal.AbstractType;
-import org.apache.cassandra.db.marshal.BytesType;
-import org.apache.cassandra.db.marshal.EmptyType;
-import org.apache.cassandra.db.view.View;
-import org.apache.cassandra.exceptions.*;
-import org.apache.cassandra.schema.IndexMetadata;
-import org.apache.cassandra.schema.Indexes;
-import org.apache.cassandra.schema.TableParams;
-import org.apache.cassandra.service.ClientState;
-import org.apache.cassandra.service.MigrationManager;
-import org.apache.cassandra.service.QueryState;
-import org.apache.cassandra.transport.Event;
-
-import static org.apache.cassandra.thrift.ThriftValidation.validateColumnFamily;
-
-public class AlterTableStatement extends SchemaAlteringStatement
-{
- public enum Type
- {
- ADD, ALTER, DROP, DROP_COMPACT_STORAGE, OPTS, RENAME
- }
-
- public final Type oType;
- private final TableAttributes attrs;
- private final Map<ColumnDefinition.Raw, ColumnDefinition.Raw> renames;
- private final List<AlterTableStatementColumn> colNameList;
- private final Long deleteTimestamp;
-
- public AlterTableStatement(CFName name,
- Type type,
- List<AlterTableStatementColumn> colDataList,
- TableAttributes attrs,
- Map<ColumnDefinition.Raw, ColumnDefinition.Raw> renames,
- Long deleteTimestamp)
- {
- super(name);
- this.oType = type;
- this.colNameList = colDataList;
- this.attrs = attrs;
- this.renames = renames;
- this.deleteTimestamp = deleteTimestamp;
- }
-
- public void checkAccess(ClientState state) throws UnauthorizedException, InvalidRequestException
- {
- state.hasColumnFamilyAccess(keyspace(), columnFamily(), Permission.ALTER);
- }
-
- public void validate(ClientState state)
- {
- // validated in announceMigration()
- }
-
- public Event.SchemaChange announceMigration(QueryState queryState, boolean isLocalOnly) throws RequestValidationException
- {
- CFMetaData meta = validateColumnFamily(keyspace(), columnFamily());
- if (meta.isView())
- throw new InvalidRequestException("Cannot use ALTER TABLE on Materialized View");
-
- CFMetaData cfm;
- ColumnIdentifier columnName = null;
- ColumnDefinition def = null;
- CQL3Type.Raw dataType = null;
- boolean isStatic = false;
- CQL3Type validator = null;
-
- List<ViewDefinition> viewUpdates = null;
- Iterable<ViewDefinition> views = View.findAll(keyspace(), columnFamily());
-
- switch (oType)
- {
- case ALTER:
- cfm = null;
- for (AlterTableStatementColumn colData : colNameList)
- {
- columnName = colData.getColumnName().getIdentifier(meta);
- def = meta.getColumnDefinition(columnName);
- dataType = colData.getColumnType();
- validator = dataType.prepare(keyspace());
-
- // We do not support altering of types and only allow this to for people who have already one
- // through the upgrade of 2.x CQL-created SSTables with Thrift writes, affected by CASSANDRA-15778.
- if (meta.isDense()
- && meta.compactValueColumn().equals(def)
- && meta.compactValueColumn().type instanceof EmptyType
- && validator != null)
- {
- if (validator.getType() instanceof BytesType)
- cfm = meta.copyWithNewCompactValueType(validator.getType());
- else
- throw new InvalidRequestException(String.format("Compact value type can only be changed to BytesType, but %s was given.",
- validator.getType()));
- }
- }
-
- if (cfm == null)
- throw new InvalidRequestException("Altering of types is not allowed");
- else
- break;
- case ADD:
- if (meta.isDense())
- throw new InvalidRequestException("Cannot add new column to a COMPACT STORAGE table");
-
- cfm = meta.copy();
-
- for (AlterTableStatementColumn colData : colNameList)
- {
- columnName = colData.getColumnName().getIdentifier(cfm);
- def = cfm.getColumnDefinition(columnName);
- dataType = colData.getColumnType();
- assert dataType != null;
- isStatic = colData.getStaticType();
- validator = dataType.prepare(keyspace());
-
-
- if (isStatic)
- {
- if (!cfm.isCompound())
- throw new InvalidRequestException("Static columns are not allowed in COMPACT STORAGE tables");
- if (cfm.clusteringColumns().isEmpty())
- throw new InvalidRequestException("Static columns are only useful (and thus allowed) if the table has at least one clustering column");
- }
-
- if (def != null)
- {
- switch (def.kind)
- {
- case PARTITION_KEY:
- case CLUSTERING:
- throw new InvalidRequestException(String.format("Invalid column name %s because it conflicts with a PRIMARY KEY part", columnName));
- default:
- throw new InvalidRequestException(String.format("Invalid column name %s because it conflicts with an existing column", columnName));
- }
- }
-
- AbstractType<?> type = validator.getType();
- if (type.isCollection() && type.isMultiCell())
- {
- if (!cfm.isCompound())
- throw new InvalidRequestException("Cannot use non-frozen collections in COMPACT STORAGE tables");
- if (cfm.isSuper())
- throw new InvalidRequestException("Cannot use non-frozen collections with super column families");
- }
-
- ColumnDefinition toAdd = isStatic
- ? ColumnDefinition.staticDef(cfm, columnName.bytes, type)
- : ColumnDefinition.regularDef(cfm, columnName.bytes, type);
-
- CFMetaData.DroppedColumn droppedColumn = meta.getDroppedColumns().get(columnName.bytes);
- if (null != droppedColumn)
- {
- if (droppedColumn.kind != toAdd.kind)
- {
- String message =
- String.format("Cannot re-add previously dropped column '%s' of kind %s, incompatible with previous kind %s",
- columnName,
- toAdd.kind,
- droppedColumn.kind == null ? "UNKNOWN" : droppedColumn.kind);
- throw new InvalidRequestException(message);
- }
- // After #8099, not safe to re-add columns of incompatible types - until *maybe* deser logic with dropped
- // columns is pushed deeper down the line. The latter would still be problematic in cases of schema races.
- if (!type.isValueCompatibleWith(droppedColumn.type))
- {
- String message =
- String.format("Cannot re-add previously dropped column '%s' of type %s, incompatible with previous type %s",
- columnName,
- type.asCQL3Type(),
- droppedColumn.type.asCQL3Type());
- throw new InvalidRequestException(message);
- }
-
- // Cannot re-add a dropped counter column. See #7831.
- if (meta.isCounter())
- throw new InvalidRequestException(String.format("Cannot re-add previously dropped counter column %s", columnName));
- }
-
- cfm.addColumnDefinition(toAdd);
-
- // Adding a column to a table which has an include all view requires the column to be added to the view as well
- if (!isStatic)
- {
- for (ViewDefinition view : views)
- {
- if (view.includeAllColumns)
- {
- ViewDefinition viewCopy = view.copy();
- viewCopy.metadata.addColumnDefinition(ColumnDefinition.regularDef(viewCopy.metadata, columnName.bytes, type));
- if (viewUpdates == null)
- viewUpdates = new ArrayList<>();
- viewUpdates.add(viewCopy);
- }
- }
- }
- }
- break;
-
- case DROP:
- if (!meta.isCQLTable())
- throw new InvalidRequestException("Cannot drop columns from a non-CQL3 table");
-
- cfm = meta.copy();
-
- for (AlterTableStatementColumn colData : colNameList)
- {
- columnName = colData.getColumnName().getIdentifier(cfm);
- def = cfm.getColumnDefinition(columnName);
-
- if (def == null)
- throw new InvalidRequestException(String.format("Column %s was not found in table %s", columnName, columnFamily()));
-
- switch (def.kind)
- {
- case PARTITION_KEY:
- case CLUSTERING:
- throw new InvalidRequestException(String.format("Cannot drop PRIMARY KEY part %s", columnName));
- case REGULAR:
- case STATIC:
- ColumnDefinition toDelete = null;
- for (ColumnDefinition columnDef : cfm.partitionColumns())
- {
- if (columnDef.name.equals(columnName))
- {
- toDelete = columnDef;
- break;
- }
- }
- assert toDelete != null;
- cfm.removeColumnDefinition(toDelete);
- cfm.recordColumnDrop(toDelete, deleteTimestamp == null ? queryState.getTimestamp() : deleteTimestamp);
- break;
- }
-
- // If the dropped column is required by any secondary indexes
- // we reject the operation, as the indexes must be dropped first
- Indexes allIndexes = cfm.getIndexes();
- if (!allIndexes.isEmpty())
- {
- ColumnFamilyStore store = Keyspace.openAndGetStore(cfm);
- Set<IndexMetadata> dependentIndexes = store.indexManager.getDependentIndexes(def);
- if (!dependentIndexes.isEmpty())
- throw new InvalidRequestException(String.format("Cannot drop column %s because it has " +
- "dependent secondary indexes (%s)",
- def,
- dependentIndexes.stream()
- .map(i -> i.name)
- .collect(Collectors.joining(","))));
- }
-
- if (!Iterables.isEmpty(views))
- throw new InvalidRequestException(String.format("Cannot drop column %s on base table %s with materialized views.",
- columnName.toString(),
- columnFamily()));
- }
- break;
- case DROP_COMPACT_STORAGE:
- if (!meta.isCompactTable())
- throw new InvalidRequestException("Cannot DROP COMPACT STORAGE on table without COMPACT STORAGE");
-
- cfm = meta.asNonCompact();
- break;
- case OPTS:
- if (attrs == null)
- throw new InvalidRequestException("ALTER TABLE WITH invoked, but no parameters found");
- attrs.validate();
-
- cfm = meta.copy();
-
- TableParams params = attrs.asAlteredTableParams(cfm.params);
-
- if (!Iterables.isEmpty(views) && params.gcGraceSeconds == 0)
- {
- throw new InvalidRequestException("Cannot alter gc_grace_seconds of the base table of a " +
- "materialized view to 0, since this value is used to TTL " +
- "undelivered updates. Setting gc_grace_seconds too low might " +
- "cause undelivered updates to expire " +
- "before being replayed.");
- }
-
- if (meta.isCounter() && params.defaultTimeToLive > 0)
- throw new InvalidRequestException("Cannot set default_time_to_live on a table with counters");
-
- cfm.params(params);
-
- break;
- case RENAME:
- cfm = meta.copy();
-
- for (Map.Entry<ColumnDefinition.Raw, ColumnDefinition.Raw> entry : renames.entrySet())
- {
- ColumnIdentifier from = entry.getKey().getIdentifier(cfm);
- ColumnIdentifier to = entry.getValue().getIdentifier(cfm);
- cfm.renameColumn(from, to);
-
- // If the view includes a renamed column, it must be renamed in the view table and the definition.
- for (ViewDefinition view : views)
- {
- if (!view.includes(from)) continue;
-
- ViewDefinition viewCopy = view.copy();
- ColumnIdentifier viewFrom = entry.getKey().getIdentifier(viewCopy.metadata);
- ColumnIdentifier viewTo = entry.getValue().getIdentifier(viewCopy.metadata);
- viewCopy.renameColumn(viewFrom, viewTo);
-
- if (viewUpdates == null)
- viewUpdates = new ArrayList<>();
- viewUpdates.add(viewCopy);
- }
- }
- break;
- default:
- throw new InvalidRequestException("Can not alter table: unknown option type " + oType);
- }
-
- MigrationManager.announceColumnFamilyUpdate(cfm, viewUpdates, isLocalOnly);
- return new Event.SchemaChange(Event.SchemaChange.Change.UPDATED, Event.SchemaChange.Target.TABLE, keyspace(), columnFamily());
- }
-
- public String toString()
- {
- return String.format("AlterTableStatement(name=%s, type=%s)",
- cfName,
- oType);
- }
-}
diff --git a/src/java/org/apache/cassandra/cql3/statements/AlterTableStatementColumn.java b/src/java/org/apache/cassandra/cql3/statements/AlterTableStatementColumn.java
deleted file mode 100644
index 7dea565..0000000
--- a/src/java/org/apache/cassandra/cql3/statements/AlterTableStatementColumn.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.cql3.statements;
-
-import org.apache.cassandra.config.ColumnDefinition;
-import org.apache.cassandra.cql3.CQL3Type;
-
-/**
- * Stores a column name and optionally type for an Alter Table statement definition.
- *
- * This is used by AlterTableStatement to store the added, altered or dropped columns.
- */
-public class AlterTableStatementColumn
-{
- private final CQL3Type.Raw dataType;
- private final ColumnDefinition.Raw colName;
- private final Boolean isStatic;
-
- public AlterTableStatementColumn(ColumnDefinition.Raw colName, CQL3Type.Raw dataType, boolean isStatic)
- {
- assert colName != null;
- this.dataType = dataType; // will be null when dropping columns, and never null otherwise (for ADD and ALTER).
- this.colName = colName;
- this.isStatic = isStatic;
- }
-
- public AlterTableStatementColumn(ColumnDefinition.Raw colName, CQL3Type.Raw dataType)
- {
- this(colName, dataType, false);
- }
-
- public AlterTableStatementColumn(ColumnDefinition.Raw colName)
- {
- this(colName, null, false);
- }
-
- public CQL3Type.Raw getColumnType()
- {
- return dataType;
- }
-
- public ColumnDefinition.Raw getColumnName()
- {
- return colName;
- }
-
- public Boolean getStaticType()
- {
- return isStatic;
- }
-}
diff --git a/src/java/org/apache/cassandra/cql3/statements/AlterTypeStatement.java b/src/java/org/apache/cassandra/cql3/statements/AlterTypeStatement.java
deleted file mode 100644
index 71d19fa..0000000
--- a/src/java/org/apache/cassandra/cql3/statements/AlterTypeStatement.java
+++ /dev/null
@@ -1,306 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.cql3.statements;
-
-import java.nio.ByteBuffer;
-import java.util.*;
-
-import org.apache.cassandra.auth.Permission;
-import org.apache.cassandra.config.*;
-import org.apache.cassandra.cql3.*;
-import org.apache.cassandra.db.marshal.*;
-import org.apache.cassandra.exceptions.*;
-import org.apache.cassandra.schema.KeyspaceMetadata;
-import org.apache.cassandra.service.ClientState;
-import org.apache.cassandra.service.MigrationManager;
-import org.apache.cassandra.service.QueryState;
-import org.apache.cassandra.transport.Event;
-
-public abstract class AlterTypeStatement extends SchemaAlteringStatement
-{
- protected final UTName name;
-
- protected AlterTypeStatement(UTName name)
- {
- this.name = name;
- }
-
- @Override
- public void prepareKeyspace(ClientState state) throws InvalidRequestException
- {
- if (!name.hasKeyspace())
- name.setKeyspace(state.getKeyspace());
-
- if (name.getKeyspace() == null)
- throw new InvalidRequestException("You need to be logged in a keyspace or use a fully qualified user type name");
- }
-
- protected abstract UserType makeUpdatedType(UserType toUpdate, KeyspaceMetadata ksm) throws InvalidRequestException;
-
- public static AlterTypeStatement addition(UTName name, FieldIdentifier fieldName, CQL3Type.Raw type)
- {
- return new Add(name, fieldName, type);
- }
-
- public static AlterTypeStatement alter(UTName name, FieldIdentifier fieldName, CQL3Type.Raw type)
- {
- throw new InvalidRequestException("Altering of types is not allowed");
- }
-
- public static AlterTypeStatement renames(UTName name, Map<FieldIdentifier, FieldIdentifier> renames)
- {
- return new Renames(name, renames);
- }
-
- public void checkAccess(ClientState state) throws UnauthorizedException, InvalidRequestException
- {
- state.hasKeyspaceAccess(keyspace(), Permission.ALTER);
- }
-
- public void validate(ClientState state) throws RequestValidationException
- {
- // Validation is left to announceMigration as it's easier to do it while constructing the updated type.
- // It doesn't really change anything anyway.
- }
-
- @Override
- public String keyspace()
- {
- return name.getKeyspace();
- }
-
- public Event.SchemaChange announceMigration(QueryState queryState, boolean isLocalOnly) throws InvalidRequestException, ConfigurationException
- {
- KeyspaceMetadata ksm = Schema.instance.getKSMetaData(name.getKeyspace());
- if (ksm == null)
- throw new InvalidRequestException(String.format("Cannot alter type in unknown keyspace %s", name.getKeyspace()));
-
- UserType toUpdate =
- ksm.types.get(name.getUserTypeName())
- .orElseThrow(() -> new InvalidRequestException(String.format("No user type named %s exists.", name)));
-
- UserType updated = makeUpdatedType(toUpdate, ksm);
-
- // Now, we need to announce the type update to basically change it for new tables using this type,
- // but we also need to find all existing user types and CF using it and change them.
- MigrationManager.announceTypeUpdate(updated, isLocalOnly);
-
- for (CFMetaData cfm : ksm.tables)
- {
- CFMetaData copy = cfm.copy();
- boolean modified = false;
- for (ColumnDefinition def : copy.allColumns())
- modified |= updateDefinition(copy, def, toUpdate.keyspace, toUpdate.name, updated);
- if (modified)
- MigrationManager.announceColumnFamilyUpdate(copy, isLocalOnly);
- }
-
- for (ViewDefinition view : ksm.views)
- {
- ViewDefinition copy = view.copy();
- boolean modified = false;
- for (ColumnDefinition def : copy.metadata.allColumns())
- modified |= updateDefinition(copy.metadata, def, toUpdate.keyspace, toUpdate.name, updated);
- if (modified)
- MigrationManager.announceViewUpdate(copy, isLocalOnly);
- }
-
- // Other user types potentially using the updated type
- for (UserType ut : ksm.types)
- {
- // Re-updating the type we've just updated would be harmless but useless so we avoid it.
- // Besides, we use the occasion to drop the old version of the type if it's a type rename
- if (ut.keyspace.equals(toUpdate.keyspace) && ut.name.equals(toUpdate.name))
- {
- if (!ut.keyspace.equals(updated.keyspace) || !ut.name.equals(updated.name))
- MigrationManager.announceTypeDrop(ut);
- continue;
- }
- AbstractType<?> upd = updateWith(ut, toUpdate.keyspace, toUpdate.name, updated);
- if (upd != null)
- MigrationManager.announceTypeUpdate((UserType) upd, isLocalOnly);
- }
- return new Event.SchemaChange(Event.SchemaChange.Change.UPDATED, Event.SchemaChange.Target.TYPE, keyspace(), name.getStringTypeName());
- }
-
- private boolean updateDefinition(CFMetaData cfm, ColumnDefinition def, String keyspace, ByteBuffer toReplace, UserType updated)
- {
- AbstractType<?> t = updateWith(def.type, keyspace, toReplace, updated);
- if (t == null)
- return false;
-
- // We need to update this validator ...
- cfm.addOrReplaceColumnDefinition(def.withNewType(t));
- return true;
- }
-
- // Update the provided type were all instance of a given userType is replaced by a new version
- // Note that this methods reaches inside other UserType, CompositeType and CollectionType.
- private static AbstractType<?> updateWith(AbstractType<?> type, String keyspace, ByteBuffer toReplace, UserType updated)
- {
- if (type instanceof UserType)
- {
- UserType ut = (UserType)type;
-
- // If it's directly the type we've updated, then just use the new one.
- if (keyspace.equals(ut.keyspace) && toReplace.equals(ut.name))
- return type.isMultiCell() ? updated : updated.freeze();
-
- // Otherwise, check for nesting
- List<AbstractType<?>> updatedTypes = updateTypes(ut.fieldTypes(), keyspace, toReplace, updated);
- return updatedTypes == null ? null : new UserType(ut.keyspace, ut.name, new ArrayList<>(ut.fieldNames()), updatedTypes, type.isMultiCell());
- }
- else if (type instanceof TupleType)
- {
- TupleType tt = (TupleType)type;
- List<AbstractType<?>> updatedTypes = updateTypes(tt.allTypes(), keyspace, toReplace, updated);
- return updatedTypes == null ? null : new TupleType(updatedTypes);
- }
- else if (type instanceof CompositeType)
- {
- CompositeType ct = (CompositeType)type;
- List<AbstractType<?>> updatedTypes = updateTypes(ct.types, keyspace, toReplace, updated);
- return updatedTypes == null ? null : CompositeType.getInstance(updatedTypes);
- }
- else if (type instanceof CollectionType)
- {
- if (type instanceof ListType)
- {
- AbstractType<?> t = updateWith(((ListType)type).getElementsType(), keyspace, toReplace, updated);
- if (t == null)
- return null;
- return ListType.getInstance(t, type.isMultiCell());
- }
- else if (type instanceof SetType)
- {
- AbstractType<?> t = updateWith(((SetType)type).getElementsType(), keyspace, toReplace, updated);
- if (t == null)
- return null;
- return SetType.getInstance(t, type.isMultiCell());
- }
- else
- {
- assert type instanceof MapType;
- MapType mt = (MapType)type;
- AbstractType<?> k = updateWith(mt.getKeysType(), keyspace, toReplace, updated);
- AbstractType<?> v = updateWith(mt.getValuesType(), keyspace, toReplace, updated);
- if (k == null && v == null)
- return null;
- return MapType.getInstance(k == null ? mt.getKeysType() : k, v == null ? mt.getValuesType() : v, type.isMultiCell());
- }
- }
- else
- {
- return null;
- }
- }
-
- private static List<AbstractType<?>> updateTypes(List<AbstractType<?>> toUpdate, String keyspace, ByteBuffer toReplace, UserType updated)
- {
- // But this can also be nested.
- List<AbstractType<?>> updatedTypes = null;
- for (int i = 0; i < toUpdate.size(); i++)
- {
- AbstractType<?> t = updateWith(toUpdate.get(i), keyspace, toReplace, updated);
- if (t == null)
- continue;
-
- if (updatedTypes == null)
- updatedTypes = new ArrayList<>(toUpdate);
-
- updatedTypes.set(i, t);
- }
- return updatedTypes;
- }
-
- protected void checkTypeNotUsedByAggregate(KeyspaceMetadata ksm)
- {
- ksm.functions.udas().filter(aggregate -> aggregate.initialCondition() != null && aggregate.stateType().referencesUserType(name.getStringTypeName()))
- .findAny()
- .ifPresent((aggregate) -> {
- throw new InvalidRequestException(String.format("Cannot alter user type %s as it is still used as an INITCOND by aggregate %s", name, aggregate));
- });
- }
-
- private static class Add extends AlterTypeStatement
- {
- private final FieldIdentifier fieldName;
- private final CQL3Type.Raw type;
-
- public Add(UTName name, FieldIdentifier fieldName, CQL3Type.Raw type)
- {
- super(name);
- this.fieldName = fieldName;
- this.type = type;
- }
-
- protected UserType makeUpdatedType(UserType toUpdate, KeyspaceMetadata ksm) throws InvalidRequestException
- {
- if (toUpdate.fieldPosition(fieldName) >= 0)
- throw new InvalidRequestException(String.format("Cannot add new field %s to type %s: a field of the same name already exists", fieldName, name));
-
- List<FieldIdentifier> newNames = new ArrayList<>(toUpdate.size() + 1);
- newNames.addAll(toUpdate.fieldNames());
- newNames.add(fieldName);
-
- AbstractType<?> addType = type.prepare(keyspace()).getType();
- if (addType.referencesUserType(toUpdate.getNameAsString()))
- throw new InvalidRequestException(String.format("Cannot add new field %s of type %s to type %s as this would create a circular reference", fieldName, type, name));
-
- List<AbstractType<?>> newTypes = new ArrayList<>(toUpdate.size() + 1);
- newTypes.addAll(toUpdate.fieldTypes());
- newTypes.add(addType);
-
- return new UserType(toUpdate.keyspace, toUpdate.name, newNames, newTypes, toUpdate.isMultiCell());
- }
- }
-
- private static class Renames extends AlterTypeStatement
- {
- private final Map<FieldIdentifier, FieldIdentifier> renames;
-
- public Renames(UTName name, Map<FieldIdentifier, FieldIdentifier> renames)
- {
- super(name);
- this.renames = renames;
- }
-
- protected UserType makeUpdatedType(UserType toUpdate, KeyspaceMetadata ksm) throws InvalidRequestException
- {
- checkTypeNotUsedByAggregate(ksm);
-
- List<FieldIdentifier> newNames = new ArrayList<>(toUpdate.fieldNames());
- List<AbstractType<?>> newTypes = new ArrayList<>(toUpdate.fieldTypes());
-
- for (Map.Entry<FieldIdentifier, FieldIdentifier> entry : renames.entrySet())
- {
- FieldIdentifier from = entry.getKey();
- FieldIdentifier to = entry.getValue();
- int idx = toUpdate.fieldPosition(from);
- if (idx < 0)
- throw new InvalidRequestException(String.format("Unknown field %s in type %s", from, name));
- newNames.set(idx, to);
- }
-
- UserType updated = new UserType(toUpdate.keyspace, toUpdate.name, newNames, newTypes, toUpdate.isMultiCell());
- CreateTypeStatement.checkForDuplicateNames(updated);
- return updated;
- }
-
- }
-}
diff --git a/src/java/org/apache/cassandra/cql3/statements/AlterViewStatement.java b/src/java/org/apache/cassandra/cql3/statements/AlterViewStatement.java
deleted file mode 100644
index ea87cfd..0000000
--- a/src/java/org/apache/cassandra/cql3/statements/AlterViewStatement.java
+++ /dev/null
@@ -1,97 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.cql3.statements;
-
-import org.apache.cassandra.auth.Permission;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.Schema;
-import org.apache.cassandra.config.ViewDefinition;
-import org.apache.cassandra.cql3.CFName;
-import org.apache.cassandra.db.view.View;
-import org.apache.cassandra.exceptions.InvalidRequestException;
-import org.apache.cassandra.exceptions.RequestValidationException;
-import org.apache.cassandra.exceptions.UnauthorizedException;
-import org.apache.cassandra.schema.TableParams;
-import org.apache.cassandra.service.ClientState;
-import org.apache.cassandra.service.MigrationManager;
-import org.apache.cassandra.service.QueryState;
-import org.apache.cassandra.transport.Event;
-
-import static org.apache.cassandra.thrift.ThriftValidation.validateColumnFamily;
-
-public class AlterViewStatement extends SchemaAlteringStatement
-{
- private final TableAttributes attrs;
-
- public AlterViewStatement(CFName name, TableAttributes attrs)
- {
- super(name);
- this.attrs = attrs;
- }
-
- public void checkAccess(ClientState state) throws UnauthorizedException, InvalidRequestException
- {
- CFMetaData baseTable = View.findBaseTable(keyspace(), columnFamily());
- if (baseTable != null)
- state.hasColumnFamilyAccess(keyspace(), baseTable.cfName, Permission.ALTER);
- }
-
- public void validate(ClientState state)
- {
- // validated in announceMigration()
- }
-
- public Event.SchemaChange announceMigration(QueryState queryState, boolean isLocalOnly) throws RequestValidationException
- {
- CFMetaData meta = validateColumnFamily(keyspace(), columnFamily());
- if (!meta.isView())
- throw new InvalidRequestException("Cannot use ALTER MATERIALIZED VIEW on Table");
-
- ViewDefinition viewCopy = Schema.instance.getView(keyspace(), columnFamily()).copy();
-
- if (attrs == null)
- throw new InvalidRequestException("ALTER MATERIALIZED VIEW WITH invoked, but no parameters found");
-
- attrs.validate();
-
- TableParams params = attrs.asAlteredTableParams(viewCopy.metadata.params);
- if (params.gcGraceSeconds == 0)
- {
- throw new InvalidRequestException("Cannot alter gc_grace_seconds of a materialized view to 0, since this " +
- "value is used to TTL undelivered updates. Setting gc_grace_seconds too " +
- "low might cause undelivered updates to expire before being replayed.");
- }
-
- if (params.defaultTimeToLive > 0)
- {
- throw new InvalidRequestException("Cannot set or alter default_time_to_live for a materialized view. " +
- "Data in a materialized view always expire at the same time than " +
- "the corresponding data in the parent table.");
- }
-
- viewCopy.metadata.params(params);
-
- MigrationManager.announceViewUpdate(viewCopy, isLocalOnly);
- return new Event.SchemaChange(Event.SchemaChange.Change.UPDATED, Event.SchemaChange.Target.TABLE, keyspace(), columnFamily());
- }
-
- public String toString()
- {
- return String.format("AlterViewStatement(name=%s)", cfName);
- }
-}
diff --git a/src/java/org/apache/cassandra/cql3/statements/AuthenticationStatement.java b/src/java/org/apache/cassandra/cql3/statements/AuthenticationStatement.java
index 646e0f4..a8cbaa7 100644
--- a/src/java/org/apache/cassandra/cql3/statements/AuthenticationStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/AuthenticationStatement.java
@@ -28,17 +28,11 @@
import org.apache.cassandra.service.QueryState;
import org.apache.cassandra.transport.messages.ResultMessage;
-public abstract class AuthenticationStatement extends ParsedStatement implements CQLStatement
+public abstract class AuthenticationStatement extends CQLStatement.Raw implements CQLStatement
{
- @Override
- public Prepared prepare(ClientState clientState)
+ public AuthenticationStatement prepare(ClientState state)
{
- return new Prepared(this);
- }
-
- public int getBoundTerms()
- {
- return 0;
+ return this;
}
public ResultMessage execute(QueryState state, QueryOptions options, long queryStartNanoTime)
@@ -49,9 +43,9 @@
public abstract ResultMessage execute(ClientState state) throws RequestExecutionException, RequestValidationException;
- public ResultMessage executeInternal(QueryState state, QueryOptions options)
+ public ResultMessage executeLocally(QueryState state, QueryOptions options)
{
- // executeInternal is for local query only, thus altering users doesn't make sense and is not supported
+ // executeLocally is for local query only, thus altering users doesn't make sense and is not supported
throw new UnsupportedOperationException();
}
@@ -59,7 +53,7 @@
{
try
{
- state.ensureHasPermission(required, resource);
+ state.ensurePermission(required, resource);
}
catch (UnauthorizedException e)
{
diff --git a/src/java/org/apache/cassandra/cql3/statements/AuthorizationStatement.java b/src/java/org/apache/cassandra/cql3/statements/AuthorizationStatement.java
index 90fee36..46285c6 100644
--- a/src/java/org/apache/cassandra/cql3/statements/AuthorizationStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/AuthorizationStatement.java
@@ -17,7 +17,6 @@
*/
package org.apache.cassandra.cql3.statements;
-
import org.apache.cassandra.auth.DataResource;
import org.apache.cassandra.auth.IResource;
import org.apache.cassandra.cql3.CQLStatement;
@@ -28,18 +27,14 @@
import org.apache.cassandra.service.ClientState;
import org.apache.cassandra.service.QueryState;
import org.apache.cassandra.transport.messages.ResultMessage;
+import org.apache.commons.lang3.builder.ToStringBuilder;
+import org.apache.commons.lang3.builder.ToStringStyle;
-public abstract class AuthorizationStatement extends ParsedStatement implements CQLStatement
+public abstract class AuthorizationStatement extends CQLStatement.Raw implements CQLStatement
{
- @Override
- public Prepared prepare(ClientState clientState)
+ public AuthorizationStatement prepare(ClientState state)
{
- return new Prepared(this);
- }
-
- public int getBoundTerms()
- {
- return 0;
+ return this;
}
public ResultMessage execute(QueryState state, QueryOptions options, long queryStartNanoTime)
@@ -50,9 +45,9 @@
public abstract ResultMessage execute(ClientState state) throws RequestValidationException, RequestExecutionException;
- public ResultMessage executeInternal(QueryState state, QueryOptions options)
+ public ResultMessage executeLocally(QueryState state, QueryOptions options)
{
- // executeInternal is for local query only, thus altering permission doesn't make sense and is not supported
+ // executeLocally is for local query only, thus altering permission doesn't make sense and is not supported
throw new UnsupportedOperationException();
}
@@ -66,4 +61,10 @@
}
return resource;
}
+
+ @Override
+ public String toString()
+ {
+ return ToStringBuilder.reflectionToString(this, ToStringStyle.SHORT_PREFIX_STYLE);
+ }
}
diff --git a/src/java/org/apache/cassandra/cql3/statements/BatchStatement.java b/src/java/org/apache/cassandra/cql3/statements/BatchStatement.java
index aaa9b1a..c165969 100644
--- a/src/java/org/apache/cassandra/cql3/statements/BatchStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/BatchStatement.java
@@ -27,14 +27,18 @@
import org.slf4j.LoggerFactory;
import org.slf4j.helpers.MessageFormatter;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.audit.AuditLogContext;
+import org.apache.cassandra.audit.AuditLogEntryType;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.*;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.partitions.PartitionUpdate;
import org.apache.cassandra.db.rows.RowIterator;
import org.apache.cassandra.exceptions.*;
+import org.apache.cassandra.metrics.BatchMetrics;
import org.apache.cassandra.service.*;
import org.apache.cassandra.tracing.Tracing;
import org.apache.cassandra.transport.messages.ResultMessage;
@@ -42,6 +46,8 @@
import org.apache.cassandra.utils.NoSpamLogger;
import org.apache.cassandra.utils.Pair;
+import static java.util.function.Predicate.isEqual;
+
import static org.apache.cassandra.cql3.statements.RequestValidations.checkFalse;
/**
@@ -54,19 +60,21 @@
LOGGED, UNLOGGED, COUNTER
}
- private final int boundTerms;
public final Type type;
+ private final VariableSpecifications bindVariables;
private final List<ModificationStatement> statements;
// Columns modified for each table (keyed by the table ID)
- private final Map<UUID, PartitionColumns> updatedColumns;
+ private final Map<TableId, RegularAndStaticColumns> updatedColumns;
// Columns on which there is conditions. Note that if there is any, then the batch can only be on a single partition (and thus table).
- private final PartitionColumns conditionColumns;
+ private final RegularAndStaticColumns conditionColumns;
private final boolean updatesRegularRows;
private final boolean updatesStaticRow;
private final Attributes attrs;
private final boolean hasConditions;
+ private final boolean updatesVirtualTables;
+
private static final Logger logger = LoggerFactory.getLogger(BatchStatement.class);
private static final String UNLOGGED_BATCH_WARNING = "Unlogged batch covering {} partitions detected " +
@@ -79,31 +87,34 @@
"tables involved in an atomic batch might cause batchlog " +
"entries to expire before being replayed.";
+ public static final BatchMetrics metrics = new BatchMetrics();
+
/**
- * Creates a new BatchStatement from a list of statements and a
- * Thrift consistency level.
+ * Creates a new BatchStatement.
*
* @param type type of the batch
- * @param statements a list of UpdateStatements
+ * @param statements the list of statements in the batch
* @param attrs additional attributes for statement (CL, timestamp, timeToLive)
*/
- public BatchStatement(int boundTerms, Type type, List<ModificationStatement> statements, Attributes attrs)
+ public BatchStatement(Type type, VariableSpecifications bindVariables, List<ModificationStatement> statements, Attributes attrs)
{
- this.boundTerms = boundTerms;
this.type = type;
+ this.bindVariables = bindVariables;
this.statements = statements;
this.attrs = attrs;
boolean hasConditions = false;
MultiTableColumnsBuilder regularBuilder = new MultiTableColumnsBuilder();
- PartitionColumns.Builder conditionBuilder = PartitionColumns.builder();
+ RegularAndStaticColumns.Builder conditionBuilder = RegularAndStaticColumns.builder();
boolean updateRegular = false;
boolean updateStatic = false;
+ boolean updatesVirtualTables = false;
for (ModificationStatement stmt : statements)
{
- regularBuilder.addAll(stmt.cfm, stmt.updatedColumns());
+ regularBuilder.addAll(stmt.metadata(), stmt.updatedColumns());
updateRegular |= stmt.updatesRegularRows();
+ updatesVirtualTables |= stmt.isVirtual();
if (stmt.hasConditions())
{
hasConditions = true;
@@ -117,8 +128,29 @@
this.updatesRegularRows = updateRegular;
this.updatesStaticRow = updateStatic;
this.hasConditions = hasConditions;
+ this.updatesVirtualTables = updatesVirtualTables;
}
+ @Override
+ public List<ColumnSpecification> getBindVariables()
+ {
+ return bindVariables.getBindVariables();
+ }
+
+ @Override
+ public short[] getPartitionKeyBindVariableIndexes()
+ {
+ boolean affectsMultipleTables =
+ !statements.isEmpty() && !statements.stream().map(s -> s.metadata().id).allMatch(isEqual(statements.get(0).metadata().id));
+
+ // Use the TableMetadata of the first statement for partition key bind indexes. If the statements affect
+ // multiple tables, we won't send partition key bind indexes.
+ return (affectsMultipleTables || statements.isEmpty())
+ ? null
+ : bindVariables.getPartitionKeyBindVariableIndexes(statements.get(0).metadata());
+ }
+
+ @Override
public Iterable<org.apache.cassandra.cql3.functions.Function> getFunctions()
{
List<org.apache.cassandra.cql3.functions.Function> functions = new ArrayList<>();
@@ -127,15 +159,10 @@
return functions;
}
- public int getBoundTerms()
- {
- return boundTerms;
- }
-
- public void checkAccess(ClientState state) throws InvalidRequestException, UnauthorizedException
+ public void authorize(ClientState state) throws InvalidRequestException, UnauthorizedException
{
for (ModificationStatement statement : statements)
- statement.checkAccess(state);
+ statement.authorize(state);
}
// Validates a prepared batch statement without validating its nested statements.
@@ -157,29 +184,46 @@
boolean hasCounters = false;
boolean hasNonCounters = false;
+ boolean hasVirtualTables = false;
+ boolean hasRegularTables = false;
+
for (ModificationStatement statement : statements)
{
- if (timestampSet && statement.isCounter())
- throw new InvalidRequestException("Cannot provide custom timestamp for a BATCH containing counters");
-
if (timestampSet && statement.isTimestampSet())
throw new InvalidRequestException("Timestamp must be set either on BATCH or individual statements");
- if (isCounter() && !statement.isCounter())
- throw new InvalidRequestException("Cannot include non-counter statement in a counter batch");
-
- if (isLogged() && statement.isCounter())
- throw new InvalidRequestException("Cannot include a counter statement in a logged batch");
-
if (statement.isCounter())
hasCounters = true;
else
hasNonCounters = true;
+
+ if (statement.isVirtual())
+ hasVirtualTables = true;
+ else
+ hasRegularTables = true;
}
+ if (timestampSet && hasCounters)
+ throw new InvalidRequestException("Cannot provide custom timestamp for a BATCH containing counters");
+
+ if (isCounter() && hasNonCounters)
+ throw new InvalidRequestException("Cannot include non-counter statement in a counter batch");
+
if (hasCounters && hasNonCounters)
throw new InvalidRequestException("Counter and non-counter mutations cannot exist in the same batch");
+ if (isLogged() && hasCounters)
+ throw new InvalidRequestException("Cannot include a counter statement in a logged batch");
+
+ if (isLogged() && hasVirtualTables)
+ throw new InvalidRequestException("Cannot include a virtual table statement in a logged batch");
+
+ if (hasVirtualTables && hasRegularTables)
+ throw new InvalidRequestException("Mutations for virtual and regular tables cannot exist in the same batch");
+
+ if (hasConditions && hasVirtualTables)
+ throw new InvalidRequestException("Conditional BATCH statements cannot include mutations for virtual tables");
+
if (hasConditions)
{
String ksName = null;
@@ -217,23 +261,26 @@
return statements;
}
- private Collection<? extends IMutation> getMutations(BatchQueryOptions options, boolean local, long now, long queryStartNanoTime)
- throws RequestExecutionException, RequestValidationException
+ private List<? extends IMutation> getMutations(BatchQueryOptions options,
+ boolean local,
+ long batchTimestamp,
+ int nowInSeconds,
+ long queryStartNanoTime)
{
Set<String> tablesWithZeroGcGs = null;
- UpdatesCollector collector = new UpdatesCollector(updatedColumns, updatedRows());
+ BatchUpdatesCollector collector = new BatchUpdatesCollector(updatedColumns, updatedRows());
for (int i = 0; i < statements.size(); i++)
{
ModificationStatement statement = statements.get(i);
- if (isLogged() && statement.cfm.params.gcGraceSeconds == 0)
+ if (isLogged() && statement.metadata().params.gcGraceSeconds == 0)
{
if (tablesWithZeroGcGs == null)
tablesWithZeroGcGs = new HashSet<>();
- tablesWithZeroGcGs.add(String.format("%s.%s", statement.cfm.ksName, statement.cfm.cfName));
+ tablesWithZeroGcGs.add(statement.metadata.toString());
}
QueryOptions statementOptions = options.forStatement(i);
- long timestamp = attrs.getTimestamp(now, statementOptions);
- statement.addUpdates(collector, statementOptions, local, timestamp, queryStartNanoTime);
+ long timestamp = attrs.getTimestamp(batchTimestamp, statementOptions);
+ statement.addUpdates(collector, statementOptions, local, timestamp, nowInSeconds, queryStartNanoTime);
}
if (tablesWithZeroGcGs != null)
@@ -244,8 +291,6 @@
ClientWarn.instance.warn(MessageFormatter.arrayFormat(LOGGED_BATCH_LOW_GCGS_WARNING, new Object[] { suffix, tablesWithZeroGcGs })
.getMessage());
}
-
- collector.validateIndexedColumns();
return collector.toMutations();
}
@@ -259,7 +304,7 @@
/**
* Checks batch size to ensure threshold is met. If not, a warning is logged.
*
- * @param updates - the batch mutations.
+ * @param mutations - the batch mutations.
*/
private static void verifyBatchSize(Collection<? extends IMutation> mutations) throws InvalidRequestException
{
@@ -267,14 +312,8 @@
if (mutations.size() <= 1)
return;
- long size = 0;
long warnThreshold = DatabaseDescriptor.getBatchSizeWarnThreshold();
-
- for (IMutation mutation : mutations)
- {
- for (PartitionUpdate update : mutation.getPartitionUpdates())
- size += update.dataSize();
- }
+ long size = IMutation.dataSize(mutations);
if (size > warnThreshold)
{
@@ -282,7 +321,7 @@
for (IMutation mutation : mutations)
{
for (PartitionUpdate update : mutation.getPartitionUpdates())
- tableNames.add(String.format("%s.%s", update.metadata().ksName, update.metadata().cfName));
+ tableNames.add(update.metadata().toString());
}
long failThreshold = DatabaseDescriptor.getBatchSizeFailThreshold();
@@ -318,7 +357,7 @@
{
keySet.add(update.partitionKey());
- tableNames.add(String.format("%s.%s", update.metadata().ksName, update.metadata().cfName));
+ tableNames.add(update.metadata().toString());
}
}
@@ -336,19 +375,16 @@
}
- public ResultMessage execute(QueryState queryState, QueryOptions options, long queryStartNanoTime) throws RequestExecutionException, RequestValidationException
+ public ResultMessage execute(QueryState queryState, QueryOptions options, long queryStartNanoTime)
{
return execute(queryState, BatchQueryOptions.withoutPerStatementVariables(options), queryStartNanoTime);
}
- public ResultMessage execute(QueryState queryState, BatchQueryOptions options, long queryStartNanoTime) throws RequestExecutionException, RequestValidationException
+ public ResultMessage execute(QueryState queryState, BatchQueryOptions options, long queryStartNanoTime)
{
- return execute(queryState, options, false, options.getTimestamp(queryState), queryStartNanoTime);
- }
+ long timestamp = options.getTimestamp(queryState);
+ int nowInSeconds = options.getNowInSeconds(queryState);
- private ResultMessage execute(QueryState queryState, BatchQueryOptions options, boolean local, long now, long queryStartNanoTime)
- throws RequestExecutionException, RequestValidationException
- {
if (options.getConsistency() == null)
throw new InvalidRequestException("Invalid empty consistency level");
if (options.getSerialConsistency() == null)
@@ -357,11 +393,15 @@
if (hasConditions)
return executeWithConditions(options, queryState, queryStartNanoTime);
- executeWithoutConditions(getMutations(options, local, now, queryStartNanoTime), options.getConsistency(), queryStartNanoTime);
+ if (updatesVirtualTables)
+ executeInternalWithoutCondition(queryState, options, queryStartNanoTime);
+ else
+ executeWithoutConditions(getMutations(options, false, timestamp, nowInSeconds, queryStartNanoTime), options.getConsistency(), queryStartNanoTime);
+
return new ResultMessage.Void();
}
- private void executeWithoutConditions(Collection<? extends IMutation> mutations, ConsistencyLevel cl, long queryStartNanoTime) throws RequestExecutionException, RequestValidationException
+ private void executeWithoutConditions(List<? extends IMutation> mutations, ConsistencyLevel cl, long queryStartNanoTime) throws RequestExecutionException, RequestValidationException
{
if (mutations.isEmpty())
return;
@@ -369,19 +409,31 @@
verifyBatchSize(mutations);
verifyBatchType(mutations);
+ updatePartitionsPerBatchMetrics(mutations.size());
+
boolean mutateAtomic = (isLogged() && mutations.size() > 1);
StorageProxy.mutateWithTriggers(mutations, cl, mutateAtomic, queryStartNanoTime);
}
- private ResultMessage executeWithConditions(BatchQueryOptions options, QueryState state, long queryStartNanoTime)
- throws RequestExecutionException, RequestValidationException
+ private void updatePartitionsPerBatchMetrics(int updatedPartitions)
{
- Pair<CQL3CasRequest, Set<ColumnDefinition>> p = makeCasRequest(options, state);
- CQL3CasRequest casRequest = p.left;
- Set<ColumnDefinition> columnsWithConditions = p.right;
+ if (isLogged()) {
+ metrics.partitionsPerLoggedBatch.update(updatedPartitions);
+ } else if (isCounter()) {
+ metrics.partitionsPerCounterBatch.update(updatedPartitions);
+ } else {
+ metrics.partitionsPerUnloggedBatch.update(updatedPartitions);
+ }
+ }
- String ksName = casRequest.cfm.ksName;
- String tableName = casRequest.cfm.cfName;
+ private ResultMessage executeWithConditions(BatchQueryOptions options, QueryState state, long queryStartNanoTime)
+ {
+ Pair<CQL3CasRequest, Set<ColumnMetadata>> p = makeCasRequest(options, state);
+ CQL3CasRequest casRequest = p.left;
+ Set<ColumnMetadata> columnsWithConditions = p.right;
+
+ String ksName = casRequest.metadata.keyspace;
+ String tableName = casRequest.metadata.name;
try (RowIterator result = StorageProxy.cas(ksName,
tableName,
@@ -390,32 +442,39 @@
options.getSerialConsistency(),
options.getConsistency(),
state.getClientState(),
+ options.getNowInSeconds(state),
queryStartNanoTime))
{
- return new ResultMessage.Rows(ModificationStatement.buildCasResultSet(ksName, tableName, result, columnsWithConditions, true, options.forStatement(0)));
+ return new ResultMessage.Rows(ModificationStatement.buildCasResultSet(ksName,
+ tableName,
+ result,
+ columnsWithConditions,
+ true,
+ state,
+ options.forStatement(0)));
}
}
-
- private Pair<CQL3CasRequest,Set<ColumnDefinition>> makeCasRequest(BatchQueryOptions options, QueryState state)
+ private Pair<CQL3CasRequest,Set<ColumnMetadata>> makeCasRequest(BatchQueryOptions options, QueryState state)
{
- long now = state.getTimestamp();
+ long batchTimestamp = options.getTimestamp(state);
+ int nowInSeconds = options.getNowInSeconds(state);
DecoratedKey key = null;
CQL3CasRequest casRequest = null;
- Set<ColumnDefinition> columnsWithConditions = new LinkedHashSet<>();
+ Set<ColumnMetadata> columnsWithConditions = new LinkedHashSet<>();
for (int i = 0; i < statements.size(); i++)
{
ModificationStatement statement = statements.get(i);
QueryOptions statementOptions = options.forStatement(i);
- long timestamp = attrs.getTimestamp(now, statementOptions);
+ long timestamp = attrs.getTimestamp(batchTimestamp, statementOptions);
List<ByteBuffer> pks = statement.buildPartitionKeyNames(statementOptions);
if (statement.getRestrictions().keyIsInRelation())
throw new IllegalArgumentException("Batch with conditions cannot span multiple partitions (you cannot use IN on the partition key)");
if (key == null)
{
- key = statement.cfm.decorateKey(pks.get(0));
- casRequest = new CQL3CasRequest(statement.cfm, key, true, conditionColumns, updatesRegularRows, updatesStaticRow);
+ key = statement.metadata().partitioner.decorateKey(pks.get(0));
+ casRequest = new CQL3CasRequest(statement.metadata(), key, conditionColumns, updatesRegularRows, updatesStaticRow);
}
else if (!key.getKey().equals(pks.get(0)))
{
@@ -438,7 +497,7 @@
for (Slice slice : slices)
{
- casRequest.addRangeDeletion(slice, statement, statementOptions, timestamp);
+ casRequest.addRangeDeletion(slice, statement, statementOptions, timestamp, nowInSeconds);
}
}
@@ -454,41 +513,62 @@
else if (columnsWithConditions != null)
Iterables.addAll(columnsWithConditions, statement.getColumnsWithConditions());
}
- casRequest.addRowUpdate(clustering, statement, statementOptions, timestamp);
+ casRequest.addRowUpdate(clustering, statement, statementOptions, timestamp, nowInSeconds);
}
}
return Pair.create(casRequest, columnsWithConditions);
}
- public ResultMessage executeInternal(QueryState queryState, QueryOptions options) throws RequestValidationException, RequestExecutionException
+ public boolean hasConditions()
{
- if (hasConditions)
- return executeInternalWithConditions(BatchQueryOptions.withoutPerStatementVariables(options), queryState);
+ return hasConditions;
+ }
- executeInternalWithoutCondition(queryState, options, System.nanoTime());
+ public ResultMessage executeLocally(QueryState queryState, QueryOptions options) throws RequestValidationException, RequestExecutionException
+ {
+ BatchQueryOptions batchOptions = BatchQueryOptions.withoutPerStatementVariables(options);
+
+ if (hasConditions)
+ return executeInternalWithConditions(batchOptions, queryState);
+
+ executeInternalWithoutCondition(queryState, batchOptions, System.nanoTime());
return new ResultMessage.Void();
}
- private ResultMessage executeInternalWithoutCondition(QueryState queryState, QueryOptions options, long queryStartNanoTime) throws RequestValidationException, RequestExecutionException
+ private ResultMessage executeInternalWithoutCondition(QueryState queryState, BatchQueryOptions batchOptions, long queryStartNanoTime)
{
- for (IMutation mutation : getMutations(BatchQueryOptions.withoutPerStatementVariables(options), true, queryState.getTimestamp(), queryStartNanoTime))
+ long timestamp = batchOptions.getTimestamp(queryState);
+ int nowInSeconds = batchOptions.getNowInSeconds(queryState);
+
+ for (IMutation mutation : getMutations(batchOptions, true, timestamp, nowInSeconds, queryStartNanoTime))
mutation.apply();
return null;
}
- private ResultMessage executeInternalWithConditions(BatchQueryOptions options, QueryState state) throws RequestExecutionException, RequestValidationException
+ private ResultMessage executeInternalWithConditions(BatchQueryOptions options, QueryState state)
{
- Pair<CQL3CasRequest, Set<ColumnDefinition>> p = makeCasRequest(options, state);
+ Pair<CQL3CasRequest, Set<ColumnMetadata>> p = makeCasRequest(options, state);
CQL3CasRequest request = p.left;
- Set<ColumnDefinition> columnsWithConditions = p.right;
+ Set<ColumnMetadata> columnsWithConditions = p.right;
- String ksName = request.cfm.ksName;
- String tableName = request.cfm.cfName;
+ String ksName = request.metadata.keyspace;
+ String tableName = request.metadata.name;
- try (RowIterator result = ModificationStatement.casInternal(request, state))
+ long timestamp = options.getTimestamp(state);
+ int nowInSeconds = options.getNowInSeconds(state);
+
+ try (RowIterator result = ModificationStatement.casInternal(request, timestamp, nowInSeconds))
{
- return new ResultMessage.Rows(ModificationStatement.buildCasResultSet(ksName, tableName, result, columnsWithConditions, true, options.forStatement(0)));
+ ResultSet resultSet =
+ ModificationStatement.buildCasResultSet(ksName,
+ tableName,
+ result,
+ columnsWithConditions,
+ true,
+ state,
+ options.forStatement(0));
+ return new ResultMessage.Rows(resultSet);
}
}
@@ -497,7 +577,7 @@
return String.format("BatchStatement(type=%s, statements=%s)", type, statements);
}
- public static class Parsed extends CFStatement
+ public static class Parsed extends QualifiedStatement
{
private final Type type;
private final Attributes.Raw attrs;
@@ -512,72 +592,54 @@
}
@Override
- public void prepareKeyspace(ClientState state) throws InvalidRequestException
+ public void setKeyspace(ClientState state) throws InvalidRequestException
{
for (ModificationStatement.Parsed statement : parsedStatements)
- statement.prepareKeyspace(state);
+ statement.setKeyspace(state);
}
- public ParsedStatement.Prepared prepare(ClientState clientState) throws InvalidRequestException
+ public BatchStatement prepare(ClientState state)
{
- VariableSpecifications boundNames = getBoundVariables();
-
- String firstKS = null;
- String firstCF = null;
- boolean haveMultipleCFs = false;
-
List<ModificationStatement> statements = new ArrayList<>(parsedStatements.size());
- for (ModificationStatement.Parsed parsed : parsedStatements)
- {
- if (firstKS == null)
- {
- firstKS = parsed.keyspace();
- firstCF = parsed.columnFamily();
- }
- else if (!haveMultipleCFs)
- {
- haveMultipleCFs = !firstKS.equals(parsed.keyspace()) || !firstCF.equals(parsed.columnFamily());
- }
-
- statements.add(parsed.prepare(boundNames, clientState));
- }
+ parsedStatements.forEach(s -> statements.add(s.prepare(bindVariables)));
Attributes prepAttrs = attrs.prepare("[batch]", "[batch]");
- prepAttrs.collectMarkerSpecification(boundNames);
+ prepAttrs.collectMarkerSpecification(bindVariables);
- BatchStatement batchStatement = new BatchStatement(boundNames.size(), type, statements, prepAttrs);
+ BatchStatement batchStatement = new BatchStatement(type, bindVariables, statements, prepAttrs);
batchStatement.validate();
- // Use the CFMetadata of the first statement for partition key bind indexes. If the statements affect
- // multiple tables, we won't send partition key bind indexes.
- short[] partitionKeyBindIndexes = (haveMultipleCFs || batchStatement.statements.isEmpty())? null
- : boundNames.getPartitionKeyBindIndexes(batchStatement.statements.get(0).cfm);
-
- return new ParsedStatement.Prepared(batchStatement, boundNames, partitionKeyBindIndexes);
+ return batchStatement;
}
}
private static class MultiTableColumnsBuilder
{
- private final Map<UUID, PartitionColumns.Builder> perTableBuilders = new HashMap<>();
+ private final Map<TableId, RegularAndStaticColumns.Builder> perTableBuilders = new HashMap<>();
- public void addAll(CFMetaData table, PartitionColumns columns)
+ public void addAll(TableMetadata table, RegularAndStaticColumns columns)
{
- PartitionColumns.Builder builder = perTableBuilders.get(table.cfId);
+ RegularAndStaticColumns.Builder builder = perTableBuilders.get(table.id);
if (builder == null)
{
- builder = PartitionColumns.builder();
- perTableBuilders.put(table.cfId, builder);
+ builder = RegularAndStaticColumns.builder();
+ perTableBuilders.put(table.id, builder);
}
builder.addAll(columns);
}
- public Map<UUID, PartitionColumns> build()
+ public Map<TableId, RegularAndStaticColumns> build()
{
- Map<UUID, PartitionColumns> m = Maps.newHashMapWithExpectedSize(perTableBuilders.size());
- for (Map.Entry<UUID, PartitionColumns.Builder> p : perTableBuilders.entrySet())
+ Map<TableId, RegularAndStaticColumns> m = Maps.newHashMapWithExpectedSize(perTableBuilders.size());
+ for (Map.Entry<TableId, RegularAndStaticColumns.Builder> p : perTableBuilders.entrySet())
m.put(p.getKey(), p.getValue().build());
return m;
}
}
+
+ @Override
+ public AuditLogContext getAuditLogContext()
+ {
+ return new AuditLogContext(AuditLogEntryType.BATCH);
+ }
}
diff --git a/src/java/org/apache/cassandra/cql3/statements/BatchUpdatesCollector.java b/src/java/org/apache/cassandra/cql3/statements/BatchUpdatesCollector.java
new file mode 100644
index 0000000..8f70ffc
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/statements/BatchUpdatesCollector.java
@@ -0,0 +1,273 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.statements;
+
+import java.nio.ByteBuffer;
+import java.util.*;
+
+import com.google.common.collect.ImmutableMap;
+
+import org.apache.cassandra.db.virtual.VirtualMutation;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.db.*;
+import org.apache.cassandra.db.partitions.PartitionUpdate;
+
+/**
+ * Utility class to collect updates.
+ *
+ * <p>In a batch statement we don't want to recreate mutations every time as this is particularly inefficient when
+ * applying multiple batch to the same partition (see #6737). </p>
+ *
+ */
+final class BatchUpdatesCollector implements UpdatesCollector
+{
+ /**
+ * The columns that will be updated for each table (keyed by the table ID).
+ */
+ private final Map<TableId, RegularAndStaticColumns> updatedColumns;
+
+ /**
+ * The estimated number of updated row.
+ */
+ private final int updatedRows;
+
+ /**
+ * The mutations per keyspace.
+ */
+ private final Map<String, Map<ByteBuffer, IMutationBuilder>> mutationBuilders = new HashMap<>();
+
+ BatchUpdatesCollector(Map<TableId, RegularAndStaticColumns> updatedColumns, int updatedRows)
+ {
+ super();
+ this.updatedColumns = updatedColumns;
+ this.updatedRows = updatedRows;
+ }
+
+ /**
+ * Gets the <code>PartitionUpdate.Builder</code> for the specified column family and key. If the builder does not
+ * exist it will be created.
+ *
+ * @param metadata the column family meta data
+ * @param dk the partition key
+ * @param consistency the consistency level
+ * @return the <code>PartitionUpdate.Builder</code> for the specified column family and key
+ */
+ public PartitionUpdate.Builder getPartitionUpdateBuilder(TableMetadata metadata, DecoratedKey dk, ConsistencyLevel consistency)
+ {
+ IMutationBuilder mut = getMutationBuilder(metadata, dk, consistency);
+ PartitionUpdate.Builder upd = mut.get(metadata.id);
+ if (upd == null)
+ {
+ RegularAndStaticColumns columns = updatedColumns.get(metadata.id);
+ assert columns != null;
+ upd = new PartitionUpdate.Builder(metadata, dk, columns, updatedRows);
+ mut.add(upd);
+ }
+ return upd;
+ }
+
+ private IMutationBuilder getMutationBuilder(TableMetadata metadata, DecoratedKey dk, ConsistencyLevel consistency)
+ {
+ return keyspaceMap(metadata.keyspace).computeIfAbsent(dk.getKey(), k -> makeMutationBuilder(metadata, dk, consistency));
+ }
+
+ private IMutationBuilder makeMutationBuilder(TableMetadata metadata, DecoratedKey partitionKey, ConsistencyLevel cl)
+ {
+ if (metadata.isVirtual())
+ {
+ return new VirtualMutationBuilder(metadata.keyspace, partitionKey);
+ }
+ else
+ {
+ MutationBuilder builder = new MutationBuilder(metadata.keyspace, partitionKey);
+ return metadata.isCounter() ? new CounterMutationBuilder(builder, cl) : builder;
+ }
+ }
+
+ /**
+ * Returns a collection containing all the mutations.
+ * @return a collection containing all the mutations.
+ */
+ public List<IMutation> toMutations()
+ {
+ //TODO: The case where all statement where on the same keyspace is pretty common, optimize for that?
+ List<IMutation> ms = new ArrayList<>();
+ for (Map<ByteBuffer, IMutationBuilder> ksMap : mutationBuilders.values())
+ {
+ for (IMutationBuilder builder : ksMap.values())
+ {
+ IMutation mutation = builder.build();
+ mutation.validateIndexedColumns();
+ ms.add(mutation);
+ }
+ }
+ return ms;
+ }
+
+ /**
+ * Returns the key-mutation mappings for the specified keyspace.
+ *
+ * @param ksName the keyspace name
+ * @return the key-mutation mappings for the specified keyspace.
+ */
+ private Map<ByteBuffer, IMutationBuilder> keyspaceMap(String ksName)
+ {
+ return mutationBuilders.computeIfAbsent(ksName, k -> new HashMap<>());
+ }
+
+ private interface IMutationBuilder
+ {
+ /**
+ * Add a new PartitionUpdate builder to this mutation builder
+ * @param builder the builder to add
+ * @return this
+ */
+ IMutationBuilder add(PartitionUpdate.Builder builder);
+
+ /**
+ * Build the immutable mutation
+ */
+ IMutation build();
+
+ /**
+ * Get the builder for the given tableId
+ */
+ PartitionUpdate.Builder get(TableId tableId);
+ }
+
+ private static class MutationBuilder implements IMutationBuilder
+ {
+ private final HashMap<TableId, PartitionUpdate.Builder> modifications = new HashMap<>();
+ private final DecoratedKey key;
+ private final String keyspaceName;
+ private final long createdAt = System.currentTimeMillis();
+
+ private MutationBuilder(String keyspaceName, DecoratedKey key)
+ {
+ this.keyspaceName = keyspaceName;
+ this.key = key;
+ }
+
+ public MutationBuilder add(PartitionUpdate.Builder updateBuilder)
+ {
+ assert updateBuilder != null;
+ assert updateBuilder.partitionKey().getPartitioner() == key.getPartitioner();
+ PartitionUpdate.Builder prev = modifications.put(updateBuilder.metadata().id, updateBuilder);
+ if (prev != null)
+ // developer error
+ throw new IllegalArgumentException("Table " + updateBuilder.metadata().name + " already has modifications in this mutation: " + prev);
+ return this;
+ }
+
+ public Mutation build()
+ {
+ ImmutableMap.Builder<TableId, PartitionUpdate> updates = new ImmutableMap.Builder<>();
+ for (Map.Entry<TableId, PartitionUpdate.Builder> updateEntry : modifications.entrySet())
+ {
+ PartitionUpdate update = updateEntry.getValue().build();
+ updates.put(updateEntry.getKey(), update);
+ }
+ return new Mutation(keyspaceName, key, updates.build(), createdAt);
+ }
+
+ public PartitionUpdate.Builder get(TableId tableId)
+ {
+ return modifications.get(tableId);
+ }
+
+ public DecoratedKey key()
+ {
+ return key;
+ }
+
+ public boolean isEmpty()
+ {
+ return modifications.isEmpty();
+ }
+
+ public String getKeyspaceName()
+ {
+ return keyspaceName;
+ }
+ }
+
+ private static class CounterMutationBuilder implements IMutationBuilder
+ {
+ private final MutationBuilder mutationBuilder;
+ private final ConsistencyLevel cl;
+
+ private CounterMutationBuilder(MutationBuilder mutationBuilder, ConsistencyLevel cl)
+ {
+ this.mutationBuilder = mutationBuilder;
+ this.cl = cl;
+ }
+
+ public IMutationBuilder add(PartitionUpdate.Builder builder)
+ {
+ return mutationBuilder.add(builder);
+ }
+
+ public IMutation build()
+ {
+ return new CounterMutation(mutationBuilder.build(), cl);
+ }
+
+ public PartitionUpdate.Builder get(TableId id)
+ {
+ return mutationBuilder.get(id);
+ }
+ }
+
+ private static class VirtualMutationBuilder implements IMutationBuilder
+ {
+ private final String keyspaceName;
+ private final DecoratedKey partitionKey;
+
+ private final HashMap<TableId, PartitionUpdate.Builder> modifications = new HashMap<>();
+
+ private VirtualMutationBuilder(String keyspaceName, DecoratedKey partitionKey)
+ {
+ this.keyspaceName = keyspaceName;
+ this.partitionKey = partitionKey;
+ }
+
+ @Override
+ public VirtualMutationBuilder add(PartitionUpdate.Builder builder)
+ {
+ PartitionUpdate.Builder prev = modifications.put(builder.metadata().id, builder);
+ if (null != prev)
+ throw new IllegalStateException();
+ return this;
+ }
+
+ @Override
+ public VirtualMutation build()
+ {
+ ImmutableMap.Builder<TableId, PartitionUpdate> updates = new ImmutableMap.Builder<>();
+ modifications.forEach((tableId, updateBuilder) -> updates.put(tableId, updateBuilder.build()));
+ return new VirtualMutation(keyspaceName, partitionKey, updates.build());
+ }
+
+ @Override
+ public PartitionUpdate.Builder get(TableId tableId)
+ {
+ return modifications.get(tableId);
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/cql3/statements/Bound.java b/src/java/org/apache/cassandra/cql3/statements/Bound.java
index 824743c..7682ac5 100644
--- a/src/java/org/apache/cassandra/cql3/statements/Bound.java
+++ b/src/java/org/apache/cassandra/cql3/statements/Bound.java
@@ -17,7 +17,7 @@
*/
package org.apache.cassandra.cql3.statements;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.schema.ColumnMetadata;
public enum Bound
{
@@ -33,12 +33,12 @@
/**
* Reverses the bound if the column type is a reversed one.
*
- * @param columnDefinition the column definition
+ * @param columnMetadata the column definition
* @return the bound reversed if the column type was a reversed one or the original bound
*/
- public Bound reverseIfNeeded(ColumnDefinition columnDefinition)
+ public Bound reverseIfNeeded(ColumnMetadata columnMetadata)
{
- return columnDefinition.isReversedType() ? reverse() : this;
+ return columnMetadata.isReversedType() ? reverse() : this;
}
public Bound reverse()
diff --git a/src/java/org/apache/cassandra/cql3/statements/CFProperties.java b/src/java/org/apache/cassandra/cql3/statements/CFProperties.java
deleted file mode 100644
index 92dd994..0000000
--- a/src/java/org/apache/cassandra/cql3/statements/CFProperties.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.cql3.statements;
-
-import java.util.LinkedHashMap;
-import java.util.Map;
-
-import org.apache.cassandra.cql3.ColumnIdentifier;
-import org.apache.cassandra.db.marshal.AbstractType;
-import org.apache.cassandra.db.marshal.ReversedType;
-
-public class CFProperties
-{
- public final TableAttributes properties = new TableAttributes();
- final Map<ColumnIdentifier, Boolean> definedOrdering = new LinkedHashMap<>(); // Insertion ordering is important
- boolean useCompactStorage = false;
-
- public void validate()
- {
- properties.validate();
- }
-
- public void setOrdering(ColumnIdentifier alias, boolean reversed)
- {
- definedOrdering.put(alias, reversed);
- }
-
- public void setCompactStorage()
- {
- useCompactStorage = true;
- }
-
- public AbstractType getReversableType(ColumnIdentifier targetIdentifier, AbstractType<?> type)
- {
- if (!definedOrdering.containsKey(targetIdentifier))
- {
- return type;
- }
- return definedOrdering.get(targetIdentifier) ? ReversedType.getInstance(type) : type;
- }
-}
diff --git a/src/java/org/apache/cassandra/cql3/statements/CFStatement.java b/src/java/org/apache/cassandra/cql3/statements/CFStatement.java
deleted file mode 100644
index 9b2987c..0000000
--- a/src/java/org/apache/cassandra/cql3/statements/CFStatement.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.cql3.statements;
-
-import org.apache.cassandra.cql3.CFName;
-import org.apache.cassandra.service.ClientState;
-import org.apache.cassandra.exceptions.InvalidRequestException;
-
-/**
- * Abstract class for statements that apply on a given column family.
- */
-public abstract class CFStatement extends ParsedStatement
-{
- protected final CFName cfName;
-
- protected CFStatement(CFName cfName)
- {
- this.cfName = cfName;
- }
-
- public void prepareKeyspace(ClientState state) throws InvalidRequestException
- {
- if (!cfName.hasKeyspace())
- {
- // XXX: We explicitely only want to call state.getKeyspace() in this case, as we don't want to throw
- // if not logged in any keyspace but a keyspace is explicitely set on the statement. So don't move
- // the call outside the 'if' or replace the method by 'prepareKeyspace(state.getKeyspace())'
- cfName.setKeyspace(state.getKeyspace(), true);
- }
- }
-
- // Only for internal calls, use the version with ClientState for user queries
- public void prepareKeyspace(String keyspace)
- {
- if (!cfName.hasKeyspace())
- cfName.setKeyspace(keyspace, true);
- }
-
- public String keyspace()
- {
- assert cfName.hasKeyspace() : "The statement hasn't be prepared correctly";
- return cfName.getKeyspace();
- }
-
- public String columnFamily()
- {
- return cfName.getColumnFamily();
- }
-}
diff --git a/src/java/org/apache/cassandra/cql3/statements/CQL3CasRequest.java b/src/java/org/apache/cassandra/cql3/statements/CQL3CasRequest.java
index 47920a4..ed985db 100644
--- a/src/java/org/apache/cassandra/cql3/statements/CQL3CasRequest.java
+++ b/src/java/org/apache/cassandra/cql3/statements/CQL3CasRequest.java
@@ -22,8 +22,10 @@
import com.google.common.collect.*;
-import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.index.IndexRegistry;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.cql3.*;
+import org.apache.cassandra.cql3.conditions.ColumnCondition;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.filter.*;
import org.apache.cassandra.db.rows.Row;
@@ -33,16 +35,17 @@
import org.apache.cassandra.exceptions.InvalidRequestException;
import org.apache.cassandra.service.CASRequest;
import org.apache.cassandra.utils.Pair;
+import org.apache.commons.lang3.builder.ToStringBuilder;
+import org.apache.commons.lang3.builder.ToStringStyle;
/**
* Processed CAS conditions and update on potentially multiple rows of the same partition.
*/
public class CQL3CasRequest implements CASRequest
{
- public final CFMetaData cfm;
+ public final TableMetadata metadata;
public final DecoratedKey key;
- public final boolean isBatch;
- private final PartitionColumns conditionColumns;
+ private final RegularAndStaticColumns conditionColumns;
private final boolean updatesRegularRows;
private final boolean updatesStaticRow;
private boolean hasExists; // whether we have an exist or if not exist condition
@@ -58,30 +61,28 @@
private final List<RowUpdate> updates = new ArrayList<>();
private final List<RangeDeletion> rangeDeletions = new ArrayList<>();
- public CQL3CasRequest(CFMetaData cfm,
+ public CQL3CasRequest(TableMetadata metadata,
DecoratedKey key,
- boolean isBatch,
- PartitionColumns conditionColumns,
+ RegularAndStaticColumns conditionColumns,
boolean updatesRegularRows,
boolean updatesStaticRow)
{
- this.cfm = cfm;
+ this.metadata = metadata;
this.key = key;
- this.conditions = new TreeMap<>(cfm.comparator);
- this.isBatch = isBatch;
+ this.conditions = new TreeMap<>(metadata.comparator);
this.conditionColumns = conditionColumns;
this.updatesRegularRows = updatesRegularRows;
this.updatesStaticRow = updatesStaticRow;
}
- public void addRowUpdate(Clustering clustering, ModificationStatement stmt, QueryOptions options, long timestamp)
+ void addRowUpdate(Clustering clustering, ModificationStatement stmt, QueryOptions options, long timestamp, int nowInSeconds)
{
- updates.add(new RowUpdate(clustering, stmt, options, timestamp));
+ updates.add(new RowUpdate(clustering, stmt, options, timestamp, nowInSeconds));
}
- public void addRangeDeletion(Slice slice, ModificationStatement stmt, QueryOptions options, long timestamp)
+ void addRangeDeletion(Slice slice, ModificationStatement stmt, QueryOptions options, long timestamp, int nowInSeconds)
{
- rangeDeletions.add(new RangeDeletion(slice, stmt, options, timestamp));
+ rangeDeletions.add(new RangeDeletion(slice, stmt, options, timestamp, nowInSeconds));
}
public void addNotExist(Clustering clustering) throws InvalidRequestException
@@ -161,40 +162,45 @@
}
}
- private PartitionColumns columnsToRead()
+ private RegularAndStaticColumns columnsToRead()
{
- PartitionColumns allColumns = cfm.partitionColumns();
-
- // If we update static row, we won't have any conditions on regular rows.
- // If we update regular row, we have to fetch all regular rows (which would satisfy column condition) and
- // static rows that take part in column condition.
- // In both cases, we're fetching enough rows to distinguish between "all conditions are nulls" and "row does not exist".
- // We have to do this as we can't rely on row marker for that (see #6623)
- Columns statics = updatesStaticRow ? allColumns.statics : conditionColumns.statics;
- Columns regulars = updatesRegularRows ? allColumns.regulars : conditionColumns.regulars;
- return new PartitionColumns(statics, regulars);
+ // If all our conditions are columns conditions (IF x = ?), then it's enough to query
+ // the columns from the conditions. If we have a IF EXISTS or IF NOT EXISTS however,
+ // we need to query all columns for the row since if the condition fails, we want to
+ // return everything to the user. Static columns make this a bit more complex, in that
+ // if an insert only static columns, then the existence condition applies only to the
+ // static columns themselves, and so we don't want to include regular columns in that
+ // case.
+ if (hasExists)
+ {
+ RegularAndStaticColumns allColumns = metadata.regularAndStaticColumns();
+ Columns statics = updatesStaticRow ? allColumns.statics : Columns.NONE;
+ Columns regulars = updatesRegularRows ? allColumns.regulars : Columns.NONE;
+ return new RegularAndStaticColumns(statics, regulars);
+ }
+ return conditionColumns;
}
- public SinglePartitionReadCommand readCommand(int nowInSec)
+ public SinglePartitionReadQuery readCommand(int nowInSec)
{
assert staticConditions != null || !conditions.isEmpty();
// Fetch all columns, but query only the selected ones
- ColumnFilter columnFilter = ColumnFilter.selection(columnsToRead());
+ ColumnFilter columnFilter = ColumnFilter.selection(metadata, columnsToRead());
// With only a static condition, we still want to make the distinction between a non-existing partition and one
// that exists (has some live data) but has not static content. So we query the first live row of the partition.
if (conditions.isEmpty())
- return SinglePartitionReadCommand.create(cfm,
- nowInSec,
- columnFilter,
- RowFilter.NONE,
- DataLimits.cqlLimits(1),
- key,
- new ClusteringIndexSliceFilter(Slices.ALL, false));
+ return SinglePartitionReadQuery.create(metadata,
+ nowInSec,
+ columnFilter,
+ RowFilter.NONE,
+ DataLimits.cqlLimits(1),
+ key,
+ new ClusteringIndexSliceFilter(Slices.ALL, false));
ClusteringIndexNamesFilter filter = new ClusteringIndexNamesFilter(conditions.navigableKeySet(), false);
- return SinglePartitionReadCommand.create(cfm, nowInSec, key, columnFilter, filter);
+ return SinglePartitionReadQuery.create(metadata, nowInSec, key, columnFilter, filter);
}
/**
@@ -219,9 +225,9 @@
return true;
}
- private PartitionColumns updatedColumns()
+ private RegularAndStaticColumns updatedColumns()
{
- PartitionColumns.Builder builder = PartitionColumns.builder();
+ RegularAndStaticColumns.Builder builder = RegularAndStaticColumns.builder();
for (RowUpdate upd : updates)
builder.addAll(upd.stmt.updatedColumns());
return builder.build();
@@ -229,14 +235,16 @@
public PartitionUpdate makeUpdates(FilteredPartition current) throws InvalidRequestException
{
- PartitionUpdate update = new PartitionUpdate(cfm, key, updatedColumns(), conditions.size());
+ PartitionUpdate.Builder updateBuilder = new PartitionUpdate.Builder(metadata, key, updatedColumns(), conditions.size());
for (RowUpdate upd : updates)
- upd.applyUpdates(current, update);
+ upd.applyUpdates(current, updateBuilder);
for (RangeDeletion upd : rangeDeletions)
- upd.applyUpdates(current, update);
+ upd.applyUpdates(current, updateBuilder);
- Keyspace.openAndGetStore(cfm).indexManager.validate(update);
- return update;
+ PartitionUpdate partitionUpdate = updateBuilder.build();
+ IndexRegistry.obtain(metadata).validate(partitionUpdate);
+
+ return partitionUpdate;
}
/**
@@ -251,20 +259,29 @@
private final ModificationStatement stmt;
private final QueryOptions options;
private final long timestamp;
+ private final int nowInSeconds;
- private RowUpdate(Clustering clustering, ModificationStatement stmt, QueryOptions options, long timestamp)
+ private RowUpdate(Clustering clustering, ModificationStatement stmt, QueryOptions options, long timestamp, int nowInSeconds)
{
this.clustering = clustering;
this.stmt = stmt;
this.options = options;
this.timestamp = timestamp;
+ this.nowInSeconds = nowInSeconds;
}
- public void applyUpdates(FilteredPartition current, PartitionUpdate updates) throws InvalidRequestException
+ void applyUpdates(FilteredPartition current, PartitionUpdate.Builder updateBuilder)
{
- Map<DecoratedKey, Partition> map = stmt.requiresRead() ? Collections.<DecoratedKey, Partition>singletonMap(key, current) : null;
- UpdateParameters params = new UpdateParameters(cfm, updates.columns(), options, timestamp, stmt.getTimeToLive(options), map);
- stmt.addUpdateForKey(updates, clustering, params);
+ Map<DecoratedKey, Partition> map = stmt.requiresRead() ? Collections.singletonMap(key, current) : null;
+ UpdateParameters params =
+ new UpdateParameters(metadata,
+ updateBuilder.columns(),
+ options,
+ timestamp,
+ nowInSeconds,
+ stmt.getTimeToLive(options),
+ map);
+ stmt.addUpdateForKey(updateBuilder, clustering, params);
}
}
@@ -274,21 +291,30 @@
private final ModificationStatement stmt;
private final QueryOptions options;
private final long timestamp;
+ private final int nowInSeconds;
- private RangeDeletion(Slice slice, ModificationStatement stmt, QueryOptions options, long timestamp)
+ private RangeDeletion(Slice slice, ModificationStatement stmt, QueryOptions options, long timestamp, int nowInSeconds)
{
this.slice = slice;
this.stmt = stmt;
this.options = options;
this.timestamp = timestamp;
+ this.nowInSeconds = nowInSeconds;
}
- public void applyUpdates(FilteredPartition current, PartitionUpdate updates) throws InvalidRequestException
+ void applyUpdates(FilteredPartition current, PartitionUpdate.Builder updateBuilder)
{
// No slice statements currently require a read, but this maintains consistency with RowUpdate, and future proofs us
- Map<DecoratedKey, Partition> map = stmt.requiresRead() ? Collections.<DecoratedKey, Partition>singletonMap(key, current) : null;
- UpdateParameters params = new UpdateParameters(cfm, updates.columns(), options, timestamp, stmt.getTimeToLive(options), map);
- stmt.addUpdateForKey(updates, slice, params);
+ Map<DecoratedKey, Partition> map = stmt.requiresRead() ? Collections.singletonMap(key, current) : null;
+ UpdateParameters params =
+ new UpdateParameters(metadata,
+ updateBuilder.columns(),
+ options,
+ timestamp,
+ nowInSeconds,
+ stmt.getTimeToLive(options),
+ map);
+ stmt.addUpdateForKey(updateBuilder, slice, params);
}
}
@@ -359,4 +385,10 @@
return true;
}
}
+
+ @Override
+ public String toString()
+ {
+ return ToStringBuilder.reflectionToString(this, ToStringStyle.SHORT_PREFIX_STYLE);
+ }
}
diff --git a/src/java/org/apache/cassandra/cql3/statements/CreateAggregateStatement.java b/src/java/org/apache/cassandra/cql3/statements/CreateAggregateStatement.java
deleted file mode 100644
index e8a5e06..0000000
--- a/src/java/org/apache/cassandra/cql3/statements/CreateAggregateStatement.java
+++ /dev/null
@@ -1,262 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.cql3.statements;
-
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Objects;
-import java.util.List;
-
-import org.apache.cassandra.auth.*;
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.config.Schema;
-import org.apache.cassandra.cql3.*;
-import org.apache.cassandra.cql3.functions.*;
-import org.apache.cassandra.db.marshal.AbstractType;
-import org.apache.cassandra.exceptions.*;
-import org.apache.cassandra.serializers.MarshalException;
-import org.apache.cassandra.service.ClientState;
-import org.apache.cassandra.service.MigrationManager;
-import org.apache.cassandra.service.QueryState;
-import org.apache.cassandra.thrift.ThriftValidation;
-import org.apache.cassandra.transport.Event;
-import org.apache.cassandra.transport.ProtocolVersion;
-
-/**
- * A {@code CREATE AGGREGATE} statement parsed from a CQL query.
- */
-public final class CreateAggregateStatement extends SchemaAlteringStatement
-{
- private final boolean orReplace;
- private final boolean ifNotExists;
- private FunctionName functionName;
- private FunctionName stateFunc;
- private FunctionName finalFunc;
- private final CQL3Type.Raw stateTypeRaw;
-
- private final List<CQL3Type.Raw> argRawTypes;
- private final Term.Raw ival;
-
- private List<AbstractType<?>> argTypes;
- private AbstractType<?> returnType;
- private ScalarFunction stateFunction;
- private ScalarFunction finalFunction;
- private ByteBuffer initcond;
-
- public CreateAggregateStatement(FunctionName functionName,
- List<CQL3Type.Raw> argRawTypes,
- String stateFunc,
- CQL3Type.Raw stateType,
- String finalFunc,
- Term.Raw ival,
- boolean orReplace,
- boolean ifNotExists)
- {
- this.functionName = functionName;
- this.argRawTypes = argRawTypes;
- this.stateFunc = new FunctionName(functionName.keyspace, stateFunc);
- this.finalFunc = finalFunc != null ? new FunctionName(functionName.keyspace, finalFunc) : null;
- this.stateTypeRaw = stateType;
- this.ival = ival;
- this.orReplace = orReplace;
- this.ifNotExists = ifNotExists;
- }
-
- public Prepared prepare(ClientState clientState)
- {
- argTypes = new ArrayList<>(argRawTypes.size());
- for (CQL3Type.Raw rawType : argRawTypes)
- argTypes.add(prepareType("arguments", rawType));
-
- AbstractType<?> stateType = prepareType("state type", stateTypeRaw);
-
- List<AbstractType<?>> stateArgs = stateArguments(stateType, argTypes);
-
- Function f = Schema.instance.findFunction(stateFunc, stateArgs).orElse(null);
- if (!(f instanceof ScalarFunction))
- throw new InvalidRequestException("State function " + stateFuncSig(stateFunc, stateTypeRaw, argRawTypes) + " does not exist or is not a scalar function");
- stateFunction = (ScalarFunction)f;
-
- AbstractType<?> stateReturnType = stateFunction.returnType();
- if (!stateReturnType.equals(stateType))
- throw new InvalidRequestException("State function " + stateFuncSig(stateFunction.name(), stateTypeRaw, argRawTypes) + " return type must be the same as the first argument type - check STYPE, argument and return types");
-
- if (finalFunc != null)
- {
- List<AbstractType<?>> finalArgs = Collections.<AbstractType<?>>singletonList(stateType);
- f = Schema.instance.findFunction(finalFunc, finalArgs).orElse(null);
- if (!(f instanceof ScalarFunction))
- throw new InvalidRequestException("Final function " + finalFunc + '(' + stateTypeRaw + ") does not exist or is not a scalar function");
- finalFunction = (ScalarFunction) f;
- returnType = finalFunction.returnType();
- }
- else
- {
- returnType = stateReturnType;
- }
-
- if (ival != null)
- {
- initcond = Terms.asBytes(functionName.keyspace, ival.toString(), stateType);
-
- if (initcond != null)
- {
- try
- {
- stateType.validate(initcond);
- }
- catch (MarshalException e)
- {
- throw new InvalidRequestException(String.format("Invalid value for INITCOND of type %s%s", stateType.asCQL3Type(),
- e.getMessage() == null ? "" : String.format(" (%s)", e.getMessage())));
- }
- }
-
- // Sanity check that converts the initcond to a CQL literal and parse it back to avoid getting in CASSANDRA-11064.
- String initcondAsCql = stateType.asCQL3Type().toCQLLiteral(initcond, ProtocolVersion.CURRENT);
- assert Objects.equals(initcond, Terms.asBytes(functionName.keyspace, initcondAsCql, stateType));
-
- if (Constants.NULL_LITERAL != ival && UDHelper.isNullOrEmpty(stateType, initcond))
- throw new InvalidRequestException("INITCOND must not be empty for all types except TEXT, ASCII, BLOB");
- }
-
- return super.prepare(clientState);
- }
-
- private AbstractType<?> prepareType(String typeName, CQL3Type.Raw rawType)
- {
- if (rawType.isFrozen())
- throw new InvalidRequestException(String.format("The function %s should not be frozen; remove the frozen<> modifier", typeName));
-
- // UDT are not supported non frozen but we do not allow the frozen keyword for argument. So for the moment we
- // freeze them here
- if (!rawType.canBeNonFrozen())
- rawType.freeze();
-
- AbstractType<?> type = rawType.prepare(functionName.keyspace).getType();
- return type;
- }
-
- public void prepareKeyspace(ClientState state) throws InvalidRequestException
- {
- if (!functionName.hasKeyspace() && state.getRawKeyspace() != null)
- functionName = new FunctionName(state.getKeyspace(), functionName.name);
-
- if (!functionName.hasKeyspace())
- throw new InvalidRequestException("Functions must be fully qualified with a keyspace name if a keyspace is not set for the session");
-
- ThriftValidation.validateKeyspaceNotSystem(functionName.keyspace);
-
- stateFunc = new FunctionName(functionName.keyspace, stateFunc.name);
- if (finalFunc != null)
- finalFunc = new FunctionName(functionName.keyspace, finalFunc.name);
- }
-
- protected void grantPermissionsToCreator(QueryState state)
- {
- try
- {
- IResource resource = FunctionResource.function(functionName.keyspace, functionName.name, argTypes);
- DatabaseDescriptor.getAuthorizer().grant(AuthenticatedUser.SYSTEM_USER,
- resource.applicablePermissions(),
- resource,
- RoleResource.role(state.getClientState().getUser().getName()));
- }
- catch (RequestExecutionException e)
- {
- throw new RuntimeException(e);
- }
- }
-
- public void checkAccess(ClientState state) throws UnauthorizedException, InvalidRequestException
- {
- if (Schema.instance.findFunction(functionName, argTypes).isPresent() && orReplace)
- state.ensureHasPermission(Permission.ALTER, FunctionResource.function(functionName.keyspace,
- functionName.name,
- argTypes));
- else
- state.ensureHasPermission(Permission.CREATE, FunctionResource.keyspace(functionName.keyspace));
-
- state.ensureHasPermission(Permission.EXECUTE, stateFunction);
-
- if (finalFunction != null)
- state.ensureHasPermission(Permission.EXECUTE, finalFunction);
- }
-
- public void validate(ClientState state) throws InvalidRequestException
- {
- if (ifNotExists && orReplace)
- throw new InvalidRequestException("Cannot use both 'OR REPLACE' and 'IF NOT EXISTS' directives");
-
- if (Schema.instance.getKSMetaData(functionName.keyspace) == null)
- throw new InvalidRequestException(String.format("Cannot add aggregate '%s' to non existing keyspace '%s'.", functionName.name, functionName.keyspace));
- }
-
- public Event.SchemaChange announceMigration(QueryState queryState, boolean isLocalOnly) throws RequestValidationException
- {
- Function old = Schema.instance.findFunction(functionName, argTypes).orElse(null);
- boolean replaced = old != null;
- if (replaced)
- {
- if (ifNotExists)
- return null;
- if (!orReplace)
- throw new InvalidRequestException(String.format("Function %s already exists", old));
- if (!(old instanceof AggregateFunction))
- throw new InvalidRequestException(String.format("Aggregate %s can only replace an aggregate", old));
-
- // Means we're replacing the function. We still need to validate that 1) it's not a native function and 2) that the return type
- // matches (or that could break existing code badly)
- if (old.isNative())
- throw new InvalidRequestException(String.format("Cannot replace native aggregate %s", old));
- if (!old.returnType().isValueCompatibleWith(returnType))
- throw new InvalidRequestException(String.format("Cannot replace aggregate %s, the new return type %s is not compatible with the return type %s of existing function",
- functionName, returnType.asCQL3Type(), old.returnType().asCQL3Type()));
- }
-
- if (!stateFunction.isCalledOnNullInput() && initcond == null)
- throw new InvalidRequestException(String.format("Cannot create aggregate %s without INITCOND because state function %s does not accept 'null' arguments", functionName, stateFunc));
-
- UDAggregate udAggregate = new UDAggregate(functionName, argTypes, returnType, stateFunction, finalFunction, initcond);
-
- MigrationManager.announceNewAggregate(udAggregate, isLocalOnly);
-
- return new Event.SchemaChange(replaced ? Event.SchemaChange.Change.UPDATED : Event.SchemaChange.Change.CREATED,
- Event.SchemaChange.Target.AGGREGATE,
- udAggregate.name().keyspace, udAggregate.name().name, AbstractType.asCQLTypeStringList(udAggregate.argTypes()));
- }
-
- private static String stateFuncSig(FunctionName stateFuncName, CQL3Type.Raw stateTypeRaw, List<CQL3Type.Raw> argRawTypes)
- {
- StringBuilder sb = new StringBuilder();
- sb.append(stateFuncName.toString()).append('(').append(stateTypeRaw);
- for (CQL3Type.Raw argRawType : argRawTypes)
- sb.append(", ").append(argRawType);
- sb.append(')');
- return sb.toString();
- }
-
- private static List<AbstractType<?>> stateArguments(AbstractType<?> stateType, List<AbstractType<?>> argTypes)
- {
- List<AbstractType<?>> r = new ArrayList<>(argTypes.size() + 1);
- r.add(stateType);
- r.addAll(argTypes);
- return r;
- }
-}
diff --git a/src/java/org/apache/cassandra/cql3/statements/CreateFunctionStatement.java b/src/java/org/apache/cassandra/cql3/statements/CreateFunctionStatement.java
deleted file mode 100644
index dfe522b..0000000
--- a/src/java/org/apache/cassandra/cql3/statements/CreateFunctionStatement.java
+++ /dev/null
@@ -1,184 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.cql3.statements;
-
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-
-import org.apache.cassandra.auth.*;
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.config.Schema;
-import org.apache.cassandra.cql3.CQL3Type;
-import org.apache.cassandra.cql3.ColumnIdentifier;
-import org.apache.cassandra.cql3.functions.*;
-import org.apache.cassandra.db.marshal.AbstractType;
-import org.apache.cassandra.exceptions.*;
-import org.apache.cassandra.schema.Functions;
-import org.apache.cassandra.service.ClientState;
-import org.apache.cassandra.service.MigrationManager;
-import org.apache.cassandra.service.QueryState;
-import org.apache.cassandra.thrift.ThriftValidation;
-import org.apache.cassandra.transport.Event;
-
-/**
- * A {@code CREATE FUNCTION} statement parsed from a CQL query.
- */
-public final class CreateFunctionStatement extends SchemaAlteringStatement
-{
- private final boolean orReplace;
- private final boolean ifNotExists;
- private FunctionName functionName;
- private final String language;
- private final String body;
-
- private final List<ColumnIdentifier> argNames;
- private final List<CQL3Type.Raw> argRawTypes;
- private final CQL3Type.Raw rawReturnType;
- private final boolean calledOnNullInput;
-
- private List<AbstractType<?>> argTypes;
- private AbstractType<?> returnType;
-
- public CreateFunctionStatement(FunctionName functionName,
- String language,
- String body,
- List<ColumnIdentifier> argNames,
- List<CQL3Type.Raw> argRawTypes,
- CQL3Type.Raw rawReturnType,
- boolean calledOnNullInput,
- boolean orReplace,
- boolean ifNotExists)
- {
- this.functionName = functionName;
- this.language = language;
- this.body = body;
- this.argNames = argNames;
- this.argRawTypes = argRawTypes;
- this.rawReturnType = rawReturnType;
- this.calledOnNullInput = calledOnNullInput;
- this.orReplace = orReplace;
- this.ifNotExists = ifNotExists;
- }
-
- public Prepared prepare(ClientState clientState) throws InvalidRequestException
- {
- if (new HashSet<>(argNames).size() != argNames.size())
- throw new InvalidRequestException(String.format("duplicate argument names for given function %s with argument names %s",
- functionName, argNames));
-
- argTypes = new ArrayList<>(argRawTypes.size());
- for (CQL3Type.Raw rawType : argRawTypes)
- argTypes.add(prepareType("arguments", rawType));
-
- returnType = prepareType("return type", rawReturnType);
- return super.prepare(clientState);
- }
-
- public void prepareKeyspace(ClientState state) throws InvalidRequestException
- {
- if (!functionName.hasKeyspace() && state.getRawKeyspace() != null)
- functionName = new FunctionName(state.getRawKeyspace(), functionName.name);
-
- if (!functionName.hasKeyspace())
- throw new InvalidRequestException("Functions must be fully qualified with a keyspace name if a keyspace is not set for the session");
-
- ThriftValidation.validateKeyspaceNotSystem(functionName.keyspace);
- }
-
- protected void grantPermissionsToCreator(QueryState state)
- {
- try
- {
- IResource resource = FunctionResource.function(functionName.keyspace, functionName.name, argTypes);
- DatabaseDescriptor.getAuthorizer().grant(AuthenticatedUser.SYSTEM_USER,
- resource.applicablePermissions(),
- resource,
- RoleResource.role(state.getClientState().getUser().getName()));
- }
- catch (RequestExecutionException e)
- {
- throw new RuntimeException(e);
- }
- }
-
- public void checkAccess(ClientState state) throws UnauthorizedException, InvalidRequestException
- {
- if (Schema.instance.findFunction(functionName, argTypes).isPresent() && orReplace)
- state.ensureHasPermission(Permission.ALTER, FunctionResource.function(functionName.keyspace,
- functionName.name,
- argTypes));
- else
- state.ensureHasPermission(Permission.CREATE, FunctionResource.keyspace(functionName.keyspace));
- }
-
- public void validate(ClientState state) throws InvalidRequestException
- {
- UDFunction.assertUdfsEnabled(language);
-
- if (ifNotExists && orReplace)
- throw new InvalidRequestException("Cannot use both 'OR REPLACE' and 'IF NOT EXISTS' directives");
-
- if (Schema.instance.getKSMetaData(functionName.keyspace) == null)
- throw new InvalidRequestException(String.format("Cannot add function '%s' to non existing keyspace '%s'.", functionName.name, functionName.keyspace));
- }
-
- public Event.SchemaChange announceMigration(QueryState queryState, boolean isLocalOnly) throws RequestValidationException
- {
- Function old = Schema.instance.findFunction(functionName, argTypes).orElse(null);
- boolean replaced = old != null;
- if (replaced)
- {
- if (ifNotExists)
- return null;
- if (!orReplace)
- throw new InvalidRequestException(String.format("Function %s already exists", old));
- if (!(old instanceof ScalarFunction))
- throw new InvalidRequestException(String.format("Function %s can only replace a function", old));
- if (calledOnNullInput != ((ScalarFunction) old).isCalledOnNullInput())
- throw new InvalidRequestException(String.format("Function %s can only be replaced with %s", old,
- calledOnNullInput ? "CALLED ON NULL INPUT" : "RETURNS NULL ON NULL INPUT"));
-
- if (!Functions.typesMatch(old.returnType(), returnType))
- throw new InvalidRequestException(String.format("Cannot replace function %s, the new return type %s is not compatible with the return type %s of existing function",
- functionName, returnType.asCQL3Type(), old.returnType().asCQL3Type()));
- }
-
- UDFunction udFunction = UDFunction.create(functionName, argNames, argTypes, returnType, calledOnNullInput, language, body);
-
- MigrationManager.announceNewFunction(udFunction, isLocalOnly);
-
- return new Event.SchemaChange(replaced ? Event.SchemaChange.Change.UPDATED : Event.SchemaChange.Change.CREATED,
- Event.SchemaChange.Target.FUNCTION,
- udFunction.name().keyspace, udFunction.name().name, AbstractType.asCQLTypeStringList(udFunction.argTypes()));
- }
-
- private AbstractType<?> prepareType(String typeName, CQL3Type.Raw rawType)
- {
- if (rawType.isFrozen())
- throw new InvalidRequestException(String.format("The function %s should not be frozen; remove the frozen<> modifier", typeName));
-
- // UDT are not supported non frozen but we do not allow the frozen keyword for argument. So for the moment we
- // freeze them here
- if (!rawType.canBeNonFrozen())
- rawType.freeze();
-
- AbstractType<?> type = rawType.prepare(functionName.keyspace).getType();
- return type;
- }
-}
diff --git a/src/java/org/apache/cassandra/cql3/statements/CreateIndexStatement.java b/src/java/org/apache/cassandra/cql3/statements/CreateIndexStatement.java
deleted file mode 100644
index f51ad44..0000000
--- a/src/java/org/apache/cassandra/cql3/statements/CreateIndexStatement.java
+++ /dev/null
@@ -1,274 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.cql3.statements;
-
-import java.util.*;
-
-import com.google.common.base.Optional;
-import com.google.common.base.Strings;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Sets;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.cassandra.auth.Permission;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.config.Schema;
-import org.apache.cassandra.cql3.CFName;
-import org.apache.cassandra.cql3.ColumnIdentifier;
-import org.apache.cassandra.cql3.IndexName;
-import org.apache.cassandra.db.marshal.MapType;
-import org.apache.cassandra.exceptions.InvalidRequestException;
-import org.apache.cassandra.exceptions.RequestValidationException;
-import org.apache.cassandra.exceptions.UnauthorizedException;
-import org.apache.cassandra.index.sasi.SASIIndex;
-import org.apache.cassandra.schema.IndexMetadata;
-import org.apache.cassandra.schema.Indexes;
-import org.apache.cassandra.service.ClientState;
-import org.apache.cassandra.service.ClientWarn;
-import org.apache.cassandra.service.MigrationManager;
-import org.apache.cassandra.service.QueryState;
-import org.apache.cassandra.thrift.ThriftValidation;
-import org.apache.cassandra.transport.Event;
-
-import static org.apache.cassandra.cql3.statements.RequestValidations.checkFalse;
-import static org.apache.cassandra.cql3.statements.RequestValidations.invalidRequest;
-
-/** A <code>CREATE INDEX</code> statement parsed from a CQL query. */
-public class CreateIndexStatement extends SchemaAlteringStatement
-{
- private static final Logger logger = LoggerFactory.getLogger(CreateIndexStatement.class);
-
- private final String indexName;
- private final List<IndexTarget.Raw> rawTargets;
- private final IndexPropDefs properties;
- private final boolean ifNotExists;
-
- public CreateIndexStatement(CFName name,
- IndexName indexName,
- List<IndexTarget.Raw> targets,
- IndexPropDefs properties,
- boolean ifNotExists)
- {
- super(name);
- this.indexName = indexName.getIdx();
- this.rawTargets = targets;
- this.properties = properties;
- this.ifNotExists = ifNotExists;
- }
-
- public void checkAccess(ClientState state) throws UnauthorizedException, InvalidRequestException
- {
- state.hasColumnFamilyAccess(keyspace(), columnFamily(), Permission.ALTER);
- }
-
- public void validate(ClientState state) throws RequestValidationException
- {
- CFMetaData cfm = ThriftValidation.validateColumnFamily(keyspace(), columnFamily());
-
- if (cfm.isCounter())
- throw new InvalidRequestException("Secondary indexes are not supported on counter tables");
-
- if (cfm.isView())
- throw new InvalidRequestException("Secondary indexes are not supported on materialized views");
-
- if (cfm.isCompactTable() && !cfm.isStaticCompactTable())
- throw new InvalidRequestException("Secondary indexes are not supported on COMPACT STORAGE tables that have clustering columns");
-
- List<IndexTarget> targets = new ArrayList<>(rawTargets.size());
- for (IndexTarget.Raw rawTarget : rawTargets)
- targets.add(rawTarget.prepare(cfm));
-
- if (targets.isEmpty() && !properties.isCustom)
- throw new InvalidRequestException("Only CUSTOM indexes can be created without specifying a target column");
-
- if (targets.size() > 1)
- validateTargetsForMultiColumnIndex(targets);
-
- for (IndexTarget target : targets)
- {
- ColumnDefinition cd = cfm.getColumnDefinitionForCQL(target.column);
-
- if (cd == null)
- throw new InvalidRequestException("No column definition found for column " + target.column);
-
- if (cd.type.referencesDuration())
- {
- checkFalse(cd.type.isCollection(), "Secondary indexes are not supported on collections containing durations");
- checkFalse(cd.type.isTuple(), "Secondary indexes are not supported on tuples containing durations");
- checkFalse(cd.type.isUDT(), "Secondary indexes are not supported on UDTs containing durations");
- throw invalidRequest("Secondary indexes are not supported on duration columns");
- }
-
- // TODO: we could lift that limitation
- if (cfm.isCompactTable())
- {
- if (cd.isPrimaryKeyColumn())
- throw new InvalidRequestException("Secondary indexes are not supported on PRIMARY KEY columns in COMPACT STORAGE tables");
- if (cfm.compactValueColumn().equals(cd))
- throw new InvalidRequestException("Secondary indexes are not supported on compact value column of COMPACT STORAGE tables");
- }
-
- if (cd.kind == ColumnDefinition.Kind.PARTITION_KEY && cfm.getKeyValidatorAsClusteringComparator().size() == 1)
- throw new InvalidRequestException(String.format("Cannot create secondary index on partition key column %s", target.column));
-
- boolean isMap = cd.type instanceof MapType;
- boolean isFrozenCollection = cd.type.isCollection() && !cd.type.isMultiCell();
- if (isFrozenCollection)
- {
- validateForFrozenCollection(target);
- }
- else
- {
- validateNotFullIndex(target);
- validateIsSimpleIndexIfTargetColumnNotCollection(cd, target);
- validateTargetColumnIsMapIfIndexInvolvesKeys(isMap, target);
- }
-
- checkFalse(cd.type.isUDT() && cd.type.isMultiCell(), "Secondary indexes are not supported on non-frozen UDTs");
- }
-
- if (!Strings.isNullOrEmpty(indexName))
- {
- if (Schema.instance.getKSMetaData(keyspace()).existingIndexNames(null).contains(indexName))
- {
- if (ifNotExists)
- return;
- else
- throw new InvalidRequestException(String.format("Index %s already exists", indexName));
- }
- }
-
- properties.validate();
- }
-
- private void validateForFrozenCollection(IndexTarget target) throws InvalidRequestException
- {
- if (target.type != IndexTarget.Type.FULL)
- throw new InvalidRequestException(String.format("Cannot create %s() index on frozen column %s. " +
- "Frozen collections only support full() indexes",
- target.type, target.column));
- }
-
- private void validateNotFullIndex(IndexTarget target) throws InvalidRequestException
- {
- if (target.type == IndexTarget.Type.FULL)
- throw new InvalidRequestException("full() indexes can only be created on frozen collections");
- }
-
- private void validateIsSimpleIndexIfTargetColumnNotCollection(ColumnDefinition cd, IndexTarget target) throws InvalidRequestException
- {
- if (!cd.type.isCollection() && target.type != IndexTarget.Type.SIMPLE)
- throw new InvalidRequestException(String.format("Cannot create %s() index on %s. " +
- "Non-collection columns support only simple indexes",
- target.type.toString(), target.column));
- }
-
- private void validateTargetColumnIsMapIfIndexInvolvesKeys(boolean isMap, IndexTarget target) throws InvalidRequestException
- {
- if (target.type == IndexTarget.Type.KEYS || target.type == IndexTarget.Type.KEYS_AND_VALUES)
- {
- if (!isMap)
- throw new InvalidRequestException(String.format("Cannot create index on %s of column %s with non-map type",
- target.type, target.column));
- }
- }
-
- private void validateTargetsForMultiColumnIndex(List<IndexTarget> targets)
- {
- if (!properties.isCustom)
- throw new InvalidRequestException("Only CUSTOM indexes support multiple columns");
-
- Set<ColumnIdentifier> columns = Sets.newHashSetWithExpectedSize(targets.size());
- for (IndexTarget target : targets)
- if (!columns.add(target.column))
- throw new InvalidRequestException("Duplicate column " + target.column + " in index target list");
- }
-
- public Event.SchemaChange announceMigration(QueryState queryState, boolean isLocalOnly) throws RequestValidationException
- {
- CFMetaData cfm = Schema.instance.getCFMetaData(keyspace(), columnFamily()).copy();
- List<IndexTarget> targets = new ArrayList<>(rawTargets.size());
- for (IndexTarget.Raw rawTarget : rawTargets)
- targets.add(rawTarget.prepare(cfm));
-
- String acceptedName = indexName;
- if (Strings.isNullOrEmpty(acceptedName))
- {
- acceptedName = Indexes.getAvailableIndexName(keyspace(),
- columnFamily(),
- targets.size() == 1 ? targets.get(0).column.toString() : null);
- }
-
- if (Schema.instance.getKSMetaData(keyspace()).existingIndexNames(null).contains(acceptedName))
- {
- if (ifNotExists)
- return null;
- else
- throw new InvalidRequestException(String.format("Index %s already exists", acceptedName));
- }
-
- IndexMetadata.Kind kind;
- Map<String, String> indexOptions;
- if (properties.isCustom)
- {
- kind = IndexMetadata.Kind.CUSTOM;
- indexOptions = properties.getOptions();
-
- if (properties.customClass.equals(SASIIndex.class.getName()))
- {
- if (!DatabaseDescriptor.getEnableSASIIndexes())
- throw new InvalidRequestException("SASI indexes are disabled. Enable in cassandra.yaml to use.");
-
- logger.warn("Creating SASI index {} for {}.{}. {}",
- acceptedName, cfm.ksName, cfm.cfName, SASIIndex.USAGE_WARNING);
-
- ClientWarn.instance.warn(SASIIndex.USAGE_WARNING);
- }
- }
- else
- {
- indexOptions = Collections.emptyMap();
- kind = cfm.isCompound() ? IndexMetadata.Kind.COMPOSITES : IndexMetadata.Kind.KEYS;
- }
-
- IndexMetadata index = IndexMetadata.fromIndexTargets(cfm, targets, acceptedName, kind, indexOptions);
-
- // check to disallow creation of an index which duplicates an existing one in all but name
- Optional<IndexMetadata> existingIndex = Iterables.tryFind(cfm.getIndexes(), existing -> existing.equalsWithoutName(index));
- if (existingIndex.isPresent())
- {
- if (ifNotExists)
- return null;
- else
- throw new InvalidRequestException(String.format("Index %s is a duplicate of existing index %s",
- index.name,
- existingIndex.get().name));
- }
-
- logger.trace("Updating index definition for {}", indexName);
- cfm.indexes(cfm.getIndexes().with(index));
-
- MigrationManager.announceColumnFamilyUpdate(cfm, isLocalOnly);
-
- // Creating an index is akin to updating the CF
- return new Event.SchemaChange(Event.SchemaChange.Change.UPDATED, Event.SchemaChange.Target.TABLE, keyspace(), columnFamily());
- }
-}
diff --git a/src/java/org/apache/cassandra/cql3/statements/CreateKeyspaceStatement.java b/src/java/org/apache/cassandra/cql3/statements/CreateKeyspaceStatement.java
deleted file mode 100644
index b34ae26..0000000
--- a/src/java/org/apache/cassandra/cql3/statements/CreateKeyspaceStatement.java
+++ /dev/null
@@ -1,136 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.cql3.statements;
-
-import java.util.regex.Pattern;
-
-import org.apache.cassandra.auth.*;
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.config.SchemaConstants;
-import org.apache.cassandra.exceptions.*;
-import org.apache.cassandra.locator.LocalStrategy;
-import org.apache.cassandra.schema.KeyspaceMetadata;
-import org.apache.cassandra.schema.KeyspaceParams;
-import org.apache.cassandra.service.*;
-import org.apache.cassandra.thrift.ThriftValidation;
-import org.apache.cassandra.transport.Event;
-
-/** A <code>CREATE KEYSPACE</code> statement parsed from a CQL query. */
-public class CreateKeyspaceStatement extends SchemaAlteringStatement
-{
- private static final Pattern PATTERN_WORD_CHARS = Pattern.compile("\\w+");
-
- private final String name;
- private final KeyspaceAttributes attrs;
- private final boolean ifNotExists;
-
- /**
- * Creates a new <code>CreateKeyspaceStatement</code> instance for a given
- * keyspace name and keyword arguments.
- *
- * @param name the name of the keyspace to create
- * @param attrs map of the raw keyword arguments that followed the <code>WITH</code> keyword.
- */
- public CreateKeyspaceStatement(String name, KeyspaceAttributes attrs, boolean ifNotExists)
- {
- super();
- this.name = name;
- this.attrs = attrs;
- this.ifNotExists = ifNotExists;
- }
-
- @Override
- public String keyspace()
- {
- return name;
- }
-
- public void checkAccess(ClientState state) throws UnauthorizedException
- {
- state.hasAllKeyspacesAccess(Permission.CREATE);
- }
-
- /**
- * The <code>CqlParser</code> only goes as far as extracting the keyword arguments
- * from these statements, so this method is responsible for processing and
- * validating.
- *
- * @throws InvalidRequestException if arguments are missing or unacceptable
- */
- public void validate(ClientState state) throws RequestValidationException
- {
- ThriftValidation.validateKeyspaceNotSystem(name);
-
- // keyspace name
- if (!PATTERN_WORD_CHARS.matcher(name).matches())
- throw new InvalidRequestException(String.format("\"%s\" is not a valid keyspace name", name));
- if (name.length() > SchemaConstants.NAME_LENGTH)
- throw new InvalidRequestException(String.format("Keyspace names shouldn't be more than %s characters long (got \"%s\")", SchemaConstants.NAME_LENGTH, name));
-
- attrs.validate();
-
- if (attrs.getReplicationStrategyClass() == null)
- throw new ConfigurationException("Missing mandatory replication strategy class");
-
- // The strategy is validated through KSMetaData.validate() in announceNewKeyspace below.
- // However, for backward compatibility with thrift, this doesn't validate unexpected options yet,
- // so doing proper validation here.
- KeyspaceParams params = attrs.asNewKeyspaceParams();
- params.validate(name);
- if (params.replication.klass.equals(LocalStrategy.class))
- throw new ConfigurationException("Unable to use given strategy class: LocalStrategy is reserved for internal use.");
- }
-
- public Event.SchemaChange announceMigration(QueryState queryState, boolean isLocalOnly) throws RequestValidationException
- {
- KeyspaceMetadata ksm = KeyspaceMetadata.create(name, attrs.asNewKeyspaceParams());
- try
- {
- MigrationManager.announceNewKeyspace(ksm, isLocalOnly);
- return new Event.SchemaChange(Event.SchemaChange.Change.CREATED, keyspace());
- }
- catch (AlreadyExistsException e)
- {
- if (ifNotExists)
- return null;
- throw e;
- }
- }
-
- protected void grantPermissionsToCreator(QueryState state)
- {
- try
- {
- RoleResource role = RoleResource.role(state.getClientState().getUser().getName());
- DataResource keyspace = DataResource.keyspace(keyspace());
- DatabaseDescriptor.getAuthorizer().grant(AuthenticatedUser.SYSTEM_USER,
- keyspace.applicablePermissions(),
- keyspace,
- role);
- FunctionResource functions = FunctionResource.keyspace(keyspace());
- DatabaseDescriptor.getAuthorizer().grant(AuthenticatedUser.SYSTEM_USER,
- functions.applicablePermissions(),
- functions,
- role);
- }
- catch (RequestExecutionException e)
- {
- throw new RuntimeException(e);
- }
- }
-}
diff --git a/src/java/org/apache/cassandra/cql3/statements/CreateRoleStatement.java b/src/java/org/apache/cassandra/cql3/statements/CreateRoleStatement.java
index 9be4c89..574d661 100644
--- a/src/java/org/apache/cassandra/cql3/statements/CreateRoleStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/CreateRoleStatement.java
@@ -17,27 +17,33 @@
*/
package org.apache.cassandra.cql3.statements;
+import org.apache.cassandra.audit.AuditLogContext;
+import org.apache.cassandra.audit.AuditLogEntryType;
import org.apache.cassandra.auth.*;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.RoleName;
import org.apache.cassandra.exceptions.*;
import org.apache.cassandra.service.ClientState;
import org.apache.cassandra.transport.messages.ResultMessage;
+import org.apache.commons.lang3.builder.ToStringBuilder;
+import org.apache.commons.lang3.builder.ToStringStyle;
public class CreateRoleStatement extends AuthenticationStatement
{
private final RoleResource role;
private final RoleOptions opts;
+ final DCPermissions dcPermissions;
private final boolean ifNotExists;
- public CreateRoleStatement(RoleName name, RoleOptions options, boolean ifNotExists)
+ public CreateRoleStatement(RoleName name, RoleOptions options, DCPermissions dcPermissions, boolean ifNotExists)
{
this.role = RoleResource.role(name.getName());
this.opts = options;
+ this.dcPermissions = dcPermissions;
this.ifNotExists = ifNotExists;
}
- public void checkAccess(ClientState state) throws UnauthorizedException
+ public void authorize(ClientState state) throws UnauthorizedException
{
super.checkPermission(state, Permission.CREATE, RoleResource.root());
if (opts.getSuperuser().isPresent())
@@ -51,10 +57,15 @@
{
opts.validate();
+ if (dcPermissions != null)
+ {
+ dcPermissions.validate();
+ }
+
if (role.getRoleName().isEmpty())
throw new InvalidRequestException("Role name can't be an empty string");
- // validate login here before checkAccess to avoid leaking role existence to anonymous users.
+ // validate login here before authorize to avoid leaking role existence to anonymous users.
state.ensureNotAnonymous();
if (!ifNotExists && DatabaseDescriptor.getRoleManager().isExistingRole(role))
@@ -68,13 +79,17 @@
return null;
DatabaseDescriptor.getRoleManager().createRole(state.getUser(), role, opts);
+ if (DatabaseDescriptor.getNetworkAuthorizer().requireAuthorization())
+ {
+ DatabaseDescriptor.getNetworkAuthorizer().setRoleDatacenters(role, dcPermissions);
+ }
grantPermissionsToCreator(state);
return null;
}
/**
* Grant all applicable permissions on the newly created role to the user performing the request
- * see also: SchemaAlteringStatement#grantPermissionsToCreator and the overridden implementations
+ * see also: AlterTableStatement#createdResources() and the overridden implementations
* of it in subclasses CreateKeyspaceStatement & CreateTableStatement.
* @param state
*/
@@ -99,4 +114,15 @@
}
}
}
+
+ @Override
+ public String toString()
+ {
+ return ToStringBuilder.reflectionToString(this, ToStringStyle.SHORT_PREFIX_STYLE);
+ }
+ @Override
+ public AuditLogContext getAuditLogContext()
+ {
+ return new AuditLogContext(AuditLogEntryType.CREATE_ROLE);
+ }
}
diff --git a/src/java/org/apache/cassandra/cql3/statements/CreateTableStatement.java b/src/java/org/apache/cassandra/cql3/statements/CreateTableStatement.java
deleted file mode 100644
index 90ed288..0000000
--- a/src/java/org/apache/cassandra/cql3/statements/CreateTableStatement.java
+++ /dev/null
@@ -1,402 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.cql3.statements;
-
-import java.nio.ByteBuffer;
-import java.util.*;
-import java.util.regex.Pattern;
-import com.google.common.collect.HashMultiset;
-import com.google.common.collect.Multiset;
-import org.apache.commons.lang3.StringUtils;
-
-import org.apache.cassandra.auth.*;
-import org.apache.cassandra.config.*;
-import org.apache.cassandra.cql3.*;
-import org.apache.cassandra.db.*;
-import org.apache.cassandra.db.marshal.*;
-import org.apache.cassandra.exceptions.*;
-import org.apache.cassandra.schema.KeyspaceMetadata;
-import org.apache.cassandra.schema.TableParams;
-import org.apache.cassandra.schema.Types;
-import org.apache.cassandra.service.ClientState;
-import org.apache.cassandra.service.MigrationManager;
-import org.apache.cassandra.service.QueryState;
-import org.apache.cassandra.transport.Event;
-
-/** A {@code CREATE TABLE} parsed from a CQL query statement. */
-public class CreateTableStatement extends SchemaAlteringStatement
-{
- private static final Pattern PATTERN_WORD_CHARS = Pattern.compile("\\w+");
-
- private List<AbstractType<?>> keyTypes;
- private List<AbstractType<?>> clusteringTypes;
-
- private final Map<ByteBuffer, AbstractType> multicellColumns = new HashMap<>();
-
- private final List<ColumnIdentifier> keyAliases = new ArrayList<>();
- private final List<ColumnIdentifier> columnAliases = new ArrayList<>();
-
- private boolean isDense;
- private boolean isCompound;
- private boolean hasCounters;
-
- // use a TreeMap to preserve ordering across JDK versions (see CASSANDRA-9492)
- private final Map<ColumnIdentifier, AbstractType> columns = new TreeMap<>((o1, o2) -> o1.bytes.compareTo(o2.bytes));
-
- private final Set<ColumnIdentifier> staticColumns;
- private final TableParams params;
- private final boolean ifNotExists;
- private final UUID id;
-
- public CreateTableStatement(CFName name, TableParams params, boolean ifNotExists, Set<ColumnIdentifier> staticColumns, UUID id)
- {
- super(name);
- this.params = params;
- this.ifNotExists = ifNotExists;
- this.staticColumns = staticColumns;
- this.id = id;
- }
-
- public void checkAccess(ClientState state) throws UnauthorizedException, InvalidRequestException
- {
- state.hasKeyspaceAccess(keyspace(), Permission.CREATE);
- }
-
- public void validate(ClientState state)
- {
- // validated in announceMigration()
- }
-
- public Event.SchemaChange announceMigration(QueryState queryState, boolean isLocalOnly) throws RequestValidationException
- {
- try
- {
- MigrationManager.announceNewColumnFamily(getCFMetaData(), isLocalOnly);
- return new Event.SchemaChange(Event.SchemaChange.Change.CREATED, Event.SchemaChange.Target.TABLE, keyspace(), columnFamily());
- }
- catch (AlreadyExistsException e)
- {
- if (ifNotExists)
- return null;
- throw e;
- }
- }
-
- protected void grantPermissionsToCreator(QueryState state)
- {
- try
- {
- IResource resource = DataResource.table(keyspace(), columnFamily());
- DatabaseDescriptor.getAuthorizer().grant(AuthenticatedUser.SYSTEM_USER,
- resource.applicablePermissions(),
- resource,
- RoleResource.role(state.getClientState().getUser().getName()));
- }
- catch (RequestExecutionException e)
- {
- throw new RuntimeException(e);
- }
- }
-
- public CFMetaData.Builder metadataBuilder()
- {
- CFMetaData.Builder builder = CFMetaData.Builder.create(keyspace(), columnFamily(), isDense, isCompound, hasCounters);
- builder.withId(id);
- for (int i = 0; i < keyAliases.size(); i++)
- builder.addPartitionKey(keyAliases.get(i), keyTypes.get(i));
- for (int i = 0; i < columnAliases.size(); i++)
- builder.addClusteringColumn(columnAliases.get(i), clusteringTypes.get(i));
-
- boolean isStaticCompact = !isDense && !isCompound;
- for (Map.Entry<ColumnIdentifier, AbstractType> entry : columns.entrySet())
- {
- ColumnIdentifier name = entry.getKey();
- // Note that for "static" no-clustering compact storage we use static for the defined columns
- if (staticColumns.contains(name) || isStaticCompact)
- builder.addStaticColumn(name, entry.getValue());
- else
- builder.addRegularColumn(name, entry.getValue());
- }
-
- boolean isCompactTable = isDense || !isCompound;
- if (isCompactTable)
- {
- CompactTables.DefaultNames names = CompactTables.defaultNameGenerator(builder.usedColumnNames());
- // Compact tables always have a clustering and a single regular value.
- if (isStaticCompact)
- {
- builder.addClusteringColumn(names.defaultClusteringName(), UTF8Type.instance);
- builder.addRegularColumn(names.defaultCompactValueName(), hasCounters ? CounterColumnType.instance : BytesType.instance);
- }
- else if (isDense && !builder.hasRegulars())
- {
- // Even for dense, we might not have our regular column if it wasn't part of the declaration. If
- // that's the case, add it but with a specific EmptyType so we can recognize that case later
- builder.addRegularColumn(names.defaultCompactValueName(), EmptyType.instance);
- }
- }
-
- return builder;
- }
-
- /**
- * Returns a CFMetaData instance based on the parameters parsed from this
- * {@code CREATE} statement, or defaults where applicable.
- *
- * @return a CFMetaData instance corresponding to the values parsed from this statement
- * @throws InvalidRequestException on failure to validate parsed parameters
- */
- public CFMetaData getCFMetaData()
- {
- return metadataBuilder().build().params(params);
- }
-
- public TableParams params()
- {
- return params;
- }
-
- public static class RawStatement extends CFStatement
- {
- private final Map<ColumnIdentifier, CQL3Type.Raw> definitions = new HashMap<>();
- public final CFProperties properties = new CFProperties();
-
- private final List<List<ColumnIdentifier>> keyAliases = new ArrayList<>();
- private final List<ColumnIdentifier> columnAliases = new ArrayList<>();
- private final Set<ColumnIdentifier> staticColumns = new HashSet<>();
-
- private final Multiset<ColumnIdentifier> definedNames = HashMultiset.create(1);
-
- private final boolean ifNotExists;
-
- public RawStatement(CFName name, boolean ifNotExists)
- {
- super(name);
- this.ifNotExists = ifNotExists;
- }
-
- /**
- * Transform this raw statement into a CreateTableStatement.
- */
- public ParsedStatement.Prepared prepare(ClientState clientState) throws RequestValidationException
- {
- KeyspaceMetadata ksm = Schema.instance.getKSMetaData(keyspace());
- if (ksm == null)
- throw new ConfigurationException(String.format("Keyspace %s doesn't exist", keyspace()));
- return prepare(ksm.types);
- }
-
- public ParsedStatement.Prepared prepare(Types udts) throws RequestValidationException
- {
- // Column family name
- if (!PATTERN_WORD_CHARS.matcher(columnFamily()).matches())
- throw new InvalidRequestException(String.format("\"%s\" is not a valid table name (must be alphanumeric character or underscore only: [a-zA-Z_0-9]+)", columnFamily()));
- if (columnFamily().length() > SchemaConstants.NAME_LENGTH)
- throw new InvalidRequestException(String.format("Table names shouldn't be more than %s characters long (got \"%s\")", SchemaConstants.NAME_LENGTH, columnFamily()));
-
- for (Multiset.Entry<ColumnIdentifier> entry : definedNames.entrySet())
- if (entry.getCount() > 1)
- throw new InvalidRequestException(String.format("Multiple definition of identifier %s", entry.getElement()));
-
- properties.validate();
-
- TableParams params = properties.properties.asNewTableParams();
-
- CreateTableStatement stmt = new CreateTableStatement(cfName, params, ifNotExists, staticColumns, properties.properties.getId());
-
- for (Map.Entry<ColumnIdentifier, CQL3Type.Raw> entry : definitions.entrySet())
- {
- ColumnIdentifier id = entry.getKey();
- CQL3Type pt = entry.getValue().prepare(keyspace(), udts);
- if (pt.getType().isMultiCell())
- stmt.multicellColumns.put(id.bytes, pt.getType());
- if (entry.getValue().isCounter())
- stmt.hasCounters = true;
-
- // check for non-frozen UDTs or collections in a non-frozen UDT
- if (pt.getType().isUDT() && pt.getType().isMultiCell())
- {
- for (AbstractType<?> innerType : ((UserType) pt.getType()).fieldTypes())
- {
- if (innerType.isMultiCell())
- {
- assert innerType.isCollection(); // shouldn't get this far with a nested non-frozen UDT
- throw new InvalidRequestException("Non-frozen UDTs with nested non-frozen collections are not supported");
- }
- }
- }
-
- stmt.columns.put(id, pt.getType()); // we'll remove what is not a column below
- }
-
- if (keyAliases.isEmpty())
- throw new InvalidRequestException("No PRIMARY KEY specifed (exactly one required)");
- if (keyAliases.size() > 1)
- throw new InvalidRequestException("Multiple PRIMARY KEYs specifed (exactly one required)");
- if (stmt.hasCounters && params.defaultTimeToLive > 0)
- throw new InvalidRequestException("Cannot set default_time_to_live on a table with counters");
-
- List<ColumnIdentifier> kAliases = keyAliases.get(0);
- stmt.keyTypes = new ArrayList<>(kAliases.size());
- for (ColumnIdentifier alias : kAliases)
- {
- stmt.keyAliases.add(alias);
- AbstractType<?> t = getTypeAndRemove(stmt.columns, alias);
- if (t.asCQL3Type().getType() instanceof CounterColumnType)
- throw new InvalidRequestException(String.format("counter type is not supported for PRIMARY KEY part %s", alias));
- if (t.asCQL3Type().getType().referencesDuration())
- throw new InvalidRequestException(String.format("duration type is not supported for PRIMARY KEY part %s", alias));
- if (staticColumns.contains(alias))
- throw new InvalidRequestException(String.format("Static column %s cannot be part of the PRIMARY KEY", alias));
- stmt.keyTypes.add(t);
- }
-
- stmt.clusteringTypes = new ArrayList<>(columnAliases.size());
- // Handle column aliases
- for (ColumnIdentifier t : columnAliases)
- {
- stmt.columnAliases.add(t);
-
- AbstractType<?> type = getTypeAndRemove(stmt.columns, t);
- if (type.asCQL3Type().getType() instanceof CounterColumnType)
- throw new InvalidRequestException(String.format("counter type is not supported for PRIMARY KEY part %s", t));
- if (type.asCQL3Type().getType().referencesDuration())
- throw new InvalidRequestException(String.format("duration type is not supported for PRIMARY KEY part %s", t));
- if (staticColumns.contains(t))
- throw new InvalidRequestException(String.format("Static column %s cannot be part of the PRIMARY KEY", t));
- stmt.clusteringTypes.add(type);
- }
-
- // We've handled anything that is not a rpimary key so stmt.columns only contains NON-PK columns. So
- // if it's a counter table, make sure we don't have non-counter types
- if (stmt.hasCounters)
- {
- for (AbstractType<?> type : stmt.columns.values())
- if (!type.isCounter())
- throw new InvalidRequestException("Cannot mix counter and non counter columns in the same table");
- }
-
- boolean useCompactStorage = properties.useCompactStorage;
- // Dense means that on the thrift side, no part of the "thrift column name" stores a "CQL/metadata column name".
- // This means COMPACT STORAGE with at least one clustering type (otherwise it's a thrift "static" CF).
- stmt.isDense = useCompactStorage && !stmt.clusteringTypes.isEmpty();
- // Compound means that on the thrift side, the "thrift column name" is a composite one. It's the case unless
- // we use compact storage COMPACT STORAGE and we have either no clustering columns (thrift "static" CF) or
- // only one of them (if more than one, it's a "dense composite").
- stmt.isCompound = !(useCompactStorage && stmt.clusteringTypes.size() <= 1);
-
- // For COMPACT STORAGE, we reject any "feature" that we wouldn't be able to translate back to thrift.
- if (useCompactStorage)
- {
- if (!stmt.multicellColumns.isEmpty())
- throw new InvalidRequestException("Non-frozen collections and UDTs are not supported with COMPACT STORAGE");
- if (!staticColumns.isEmpty())
- throw new InvalidRequestException("Static columns are not supported in COMPACT STORAGE tables");
-
- if (stmt.clusteringTypes.isEmpty())
- {
- // It's a thrift "static CF" so there should be some columns definition
- if (stmt.columns.isEmpty())
- throw new InvalidRequestException("No definition found that is not part of the PRIMARY KEY");
- }
-
- if (stmt.isDense)
- {
- // We can have no columns (only the PK), but we can't have more than one.
- if (stmt.columns.size() > 1)
- throw new InvalidRequestException(String.format("COMPACT STORAGE with composite PRIMARY KEY allows no more than one column not part of the PRIMARY KEY (got: %s)", StringUtils.join(stmt.columns.keySet(), ", ")));
- }
- else
- {
- // we are in the "static" case, so we need at least one column defined. For non-compact however, having
- // just the PK is fine.
- if (stmt.columns.isEmpty())
- throw new InvalidRequestException("COMPACT STORAGE with non-composite PRIMARY KEY require one column not part of the PRIMARY KEY, none given");
- }
- }
- else
- {
- if (stmt.clusteringTypes.isEmpty() && !staticColumns.isEmpty())
- {
- // Static columns only make sense if we have at least one clustering column. Otherwise everything is static anyway
- if (columnAliases.isEmpty())
- throw new InvalidRequestException("Static columns are only useful (and thus allowed) if the table has at least one clustering column");
- }
- }
-
- // If we give a clustering order, we must explicitly do so for all aliases and in the order of the PK
- if (!properties.definedOrdering.isEmpty())
- {
- if (properties.definedOrdering.size() > columnAliases.size())
- throw new InvalidRequestException("Only clustering key columns can be defined in CLUSTERING ORDER directive");
-
- int i = 0;
- for (ColumnIdentifier id : properties.definedOrdering.keySet())
- {
- ColumnIdentifier c = columnAliases.get(i);
- if (!id.equals(c))
- {
- if (properties.definedOrdering.containsKey(c))
- throw new InvalidRequestException(String.format("The order of columns in the CLUSTERING ORDER directive must be the one of the clustering key (%s must appear before %s)", c, id));
- else
- throw new InvalidRequestException(String.format("Missing CLUSTERING ORDER for column %s", c));
- }
- ++i;
- }
- }
-
- return new ParsedStatement.Prepared(stmt);
- }
-
- private AbstractType<?> getTypeAndRemove(Map<ColumnIdentifier, AbstractType> columns, ColumnIdentifier t) throws InvalidRequestException
- {
- AbstractType type = columns.get(t);
- if (type == null)
- throw new InvalidRequestException(String.format("Unknown definition %s referenced in PRIMARY KEY", t));
- if (type.isMultiCell())
- {
- if (type.isCollection())
- throw new InvalidRequestException(String.format("Invalid non-frozen collection type for PRIMARY KEY component %s", t));
- else
- throw new InvalidRequestException(String.format("Invalid non-frozen user-defined type for PRIMARY KEY component %s", t));
- }
-
- columns.remove(t);
- Boolean isReversed = properties.definedOrdering.get(t);
- return isReversed != null && isReversed ? ReversedType.getInstance(type) : type;
- }
-
- public void addDefinition(ColumnIdentifier def, CQL3Type.Raw type, boolean isStatic)
- {
- definedNames.add(def);
- definitions.put(def, type);
- if (isStatic)
- staticColumns.add(def);
- }
-
- public void addKeyAliases(List<ColumnIdentifier> aliases)
- {
- keyAliases.add(aliases);
- }
-
- public void addColumnAlias(ColumnIdentifier alias)
- {
- columnAliases.add(alias);
- }
- }
-}
diff --git a/src/java/org/apache/cassandra/cql3/statements/CreateTriggerStatement.java b/src/java/org/apache/cassandra/cql3/statements/CreateTriggerStatement.java
deleted file mode 100644
index 5d29996..0000000
--- a/src/java/org/apache/cassandra/cql3/statements/CreateTriggerStatement.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.cql3.statements;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.Schema;
-import org.apache.cassandra.cql3.CFName;
-import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.exceptions.InvalidRequestException;
-import org.apache.cassandra.exceptions.RequestValidationException;
-import org.apache.cassandra.exceptions.UnauthorizedException;
-import org.apache.cassandra.schema.TriggerMetadata;
-import org.apache.cassandra.schema.Triggers;
-import org.apache.cassandra.service.ClientState;
-import org.apache.cassandra.service.MigrationManager;
-import org.apache.cassandra.service.QueryState;
-import org.apache.cassandra.thrift.ThriftValidation;
-import org.apache.cassandra.transport.Event;
-import org.apache.cassandra.triggers.TriggerExecutor;
-
-public class CreateTriggerStatement extends SchemaAlteringStatement
-{
- private static final Logger logger = LoggerFactory.getLogger(CreateTriggerStatement.class);
-
- private final String triggerName;
- private final String triggerClass;
- private final boolean ifNotExists;
-
- public CreateTriggerStatement(CFName name, String triggerName, String clazz, boolean ifNotExists)
- {
- super(name);
- this.triggerName = triggerName;
- this.triggerClass = clazz;
- this.ifNotExists = ifNotExists;
- }
-
- public void checkAccess(ClientState state) throws UnauthorizedException
- {
- state.ensureIsSuper("Only superusers are allowed to perform CREATE TRIGGER queries");
- }
-
- public void validate(ClientState state) throws RequestValidationException
- {
- CFMetaData cfm = ThriftValidation.validateColumnFamily(keyspace(), columnFamily());
- if (cfm.isView())
- throw new InvalidRequestException("Cannot CREATE TRIGGER against a materialized view");
-
- try
- {
- TriggerExecutor.instance.loadTriggerInstance(triggerClass);
- }
- catch (Exception e)
- {
- throw new ConfigurationException(String.format("Trigger class '%s' doesn't exist", triggerClass));
- }
- }
-
- public Event.SchemaChange announceMigration(QueryState queryState, boolean isLocalOnly) throws ConfigurationException, InvalidRequestException
- {
- CFMetaData cfm = Schema.instance.getCFMetaData(keyspace(), columnFamily()).copy();
- Triggers triggers = cfm.getTriggers();
-
- if (triggers.get(triggerName).isPresent())
- {
- if (ifNotExists)
- return null;
- else
- throw new InvalidRequestException(String.format("Trigger %s already exists", triggerName));
- }
-
- cfm.triggers(triggers.with(TriggerMetadata.create(triggerName, triggerClass)));
- logger.info("Adding trigger with name {} and class {}", triggerName, triggerClass);
- MigrationManager.announceColumnFamilyUpdate(cfm, isLocalOnly);
- return new Event.SchemaChange(Event.SchemaChange.Change.UPDATED, Event.SchemaChange.Target.TABLE, keyspace(), columnFamily());
- }
-}
diff --git a/src/java/org/apache/cassandra/cql3/statements/CreateTypeStatement.java b/src/java/org/apache/cassandra/cql3/statements/CreateTypeStatement.java
deleted file mode 100644
index ff9af75..0000000
--- a/src/java/org/apache/cassandra/cql3/statements/CreateTypeStatement.java
+++ /dev/null
@@ -1,135 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.cql3.statements;
-
-import java.util.*;
-import java.util.stream.Collectors;
-
-import org.apache.cassandra.auth.Permission;
-import org.apache.cassandra.config.*;
-import org.apache.cassandra.cql3.*;
-import org.apache.cassandra.db.marshal.AbstractType;
-import org.apache.cassandra.db.marshal.UserType;
-import org.apache.cassandra.exceptions.*;
-import org.apache.cassandra.schema.KeyspaceMetadata;
-import org.apache.cassandra.schema.Types;
-import org.apache.cassandra.service.ClientState;
-import org.apache.cassandra.service.MigrationManager;
-import org.apache.cassandra.service.QueryState;
-import org.apache.cassandra.transport.Event;
-
-public class CreateTypeStatement extends SchemaAlteringStatement
-{
- private final UTName name;
- private final List<FieldIdentifier> columnNames = new ArrayList<>();
- private final List<CQL3Type.Raw> columnTypes = new ArrayList<>();
- private final boolean ifNotExists;
-
- public CreateTypeStatement(UTName name, boolean ifNotExists)
- {
- super();
- this.name = name;
- this.ifNotExists = ifNotExists;
- }
-
- @Override
- public void prepareKeyspace(ClientState state) throws InvalidRequestException
- {
- if (!name.hasKeyspace())
- name.setKeyspace(state.getKeyspace());
- }
-
- public void addDefinition(FieldIdentifier name, CQL3Type.Raw type)
- {
- columnNames.add(name);
- columnTypes.add(type);
- }
-
- public void checkAccess(ClientState state) throws UnauthorizedException, InvalidRequestException
- {
- state.hasKeyspaceAccess(keyspace(), Permission.CREATE);
- }
-
- public void validate(ClientState state) throws RequestValidationException
- {
- KeyspaceMetadata ksm = Schema.instance.getKSMetaData(name.getKeyspace());
- if (ksm == null)
- throw new InvalidRequestException(String.format("Cannot add type in unknown keyspace %s", name.getKeyspace()));
-
- if (ksm.types.get(name.getUserTypeName()).isPresent() && !ifNotExists)
- throw new InvalidRequestException(String.format("A user type of name %s already exists", name));
-
- for (CQL3Type.Raw type : columnTypes)
- {
- if (type.isCounter())
- throw new InvalidRequestException("A user type cannot contain counters");
- if (type.isUDT() && !type.isFrozen())
- throw new InvalidRequestException("A user type cannot contain non-frozen UDTs");
- }
- }
-
- public static void checkForDuplicateNames(UserType type) throws InvalidRequestException
- {
- for (int i = 0; i < type.size() - 1; i++)
- {
- FieldIdentifier fieldName = type.fieldName(i);
- for (int j = i+1; j < type.size(); j++)
- {
- if (fieldName.equals(type.fieldName(j)))
- throw new InvalidRequestException(String.format("Duplicate field name %s in type %s", fieldName, type.name));
- }
- }
- }
-
- public void addToRawBuilder(Types.RawBuilder builder) throws InvalidRequestException
- {
- builder.add(name.getStringTypeName(),
- columnNames.stream().map(FieldIdentifier::toString).collect(Collectors.toList()),
- columnTypes.stream().map(CQL3Type.Raw::toString).collect(Collectors.toList()));
- }
-
- @Override
- public String keyspace()
- {
- return name.getKeyspace();
- }
-
- public UserType createType() throws InvalidRequestException
- {
- List<AbstractType<?>> types = new ArrayList<>(columnTypes.size());
- for (CQL3Type.Raw type : columnTypes)
- types.add(type.prepare(keyspace()).getType());
-
- return new UserType(name.getKeyspace(), name.getUserTypeName(), columnNames, types, true);
- }
-
- public Event.SchemaChange announceMigration(QueryState queryState, boolean isLocalOnly) throws InvalidRequestException, ConfigurationException
- {
- KeyspaceMetadata ksm = Schema.instance.getKSMetaData(name.getKeyspace());
- assert ksm != null; // should haven't validate otherwise
-
- // Can happen with ifNotExists
- if (ksm.types.get(name.getUserTypeName()).isPresent())
- return null;
-
- UserType type = createType();
- checkForDuplicateNames(type);
- MigrationManager.announceNewType(type, isLocalOnly);
- return new Event.SchemaChange(Event.SchemaChange.Change.CREATED, Event.SchemaChange.Target.TYPE, keyspace(), name.getStringTypeName());
- }
-}
diff --git a/src/java/org/apache/cassandra/cql3/statements/CreateViewStatement.java b/src/java/org/apache/cassandra/cql3/statements/CreateViewStatement.java
deleted file mode 100644
index 51e0aaf..0000000
--- a/src/java/org/apache/cassandra/cql3/statements/CreateViewStatement.java
+++ /dev/null
@@ -1,370 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.cql3.statements;
-
-import java.util.*;
-import java.util.stream.Collectors;
-
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Sets;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.cassandra.auth.Permission;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.config.Schema;
-import org.apache.cassandra.config.ViewDefinition;
-import org.apache.cassandra.cql3.*;
-import org.apache.cassandra.cql3.restrictions.StatementRestrictions;
-import org.apache.cassandra.cql3.selection.RawSelector;
-import org.apache.cassandra.cql3.selection.Selectable;
-import org.apache.cassandra.db.marshal.AbstractType;
-import org.apache.cassandra.db.marshal.DurationType;
-import org.apache.cassandra.db.marshal.ReversedType;
-import org.apache.cassandra.db.view.View;
-import org.apache.cassandra.exceptions.AlreadyExistsException;
-import org.apache.cassandra.exceptions.InvalidRequestException;
-import org.apache.cassandra.exceptions.RequestValidationException;
-import org.apache.cassandra.exceptions.UnauthorizedException;
-import org.apache.cassandra.schema.TableParams;
-import org.apache.cassandra.service.ClientState;
-import org.apache.cassandra.service.ClientWarn;
-import org.apache.cassandra.service.MigrationManager;
-import org.apache.cassandra.service.QueryState;
-import org.apache.cassandra.thrift.ThriftValidation;
-import org.apache.cassandra.transport.Event;
-
-public class CreateViewStatement extends SchemaAlteringStatement
-{
- private static final Logger logger = LoggerFactory.getLogger(CreateViewStatement.class);
-
- private final CFName baseName;
- private final List<RawSelector> selectClause;
- private final WhereClause whereClause;
- private final List<ColumnDefinition.Raw> partitionKeys;
- private final List<ColumnDefinition.Raw> clusteringKeys;
- public final CFProperties properties = new CFProperties();
- private final boolean ifNotExists;
-
- public CreateViewStatement(CFName viewName,
- CFName baseName,
- List<RawSelector> selectClause,
- WhereClause whereClause,
- List<ColumnDefinition.Raw> partitionKeys,
- List<ColumnDefinition.Raw> clusteringKeys,
- boolean ifNotExists)
- {
- super(viewName);
- this.baseName = baseName;
- this.selectClause = selectClause;
- this.whereClause = whereClause;
- this.partitionKeys = partitionKeys;
- this.clusteringKeys = clusteringKeys;
- this.ifNotExists = ifNotExists;
- }
-
-
- public void checkAccess(ClientState state) throws UnauthorizedException, InvalidRequestException
- {
- if (!baseName.hasKeyspace())
- baseName.setKeyspace(keyspace(), true);
- state.hasColumnFamilyAccess(keyspace(), baseName.getColumnFamily(), Permission.ALTER);
- }
-
- public void validate(ClientState state) throws RequestValidationException
- {
- // We do validation in announceMigration to reduce doubling up of work
- }
-
- private interface AddColumn
- {
- void add(ColumnIdentifier identifier, AbstractType<?> type);
- }
-
- private void add(CFMetaData baseCfm, Iterable<ColumnIdentifier> columns, AddColumn adder)
- {
- for (ColumnIdentifier column : columns)
- {
- AbstractType<?> type = baseCfm.getColumnDefinition(column).type;
- if (properties.definedOrdering.containsKey(column))
- {
- boolean desc = properties.definedOrdering.get(column);
- if (!desc && type.isReversed())
- {
- type = ((ReversedType)type).baseType;
- }
- else if (desc && !type.isReversed())
- {
- type = ReversedType.getInstance(type);
- }
- }
- adder.add(column, type);
- }
- }
-
- public Event.SchemaChange announceMigration(QueryState queryState, boolean isLocalOnly) throws RequestValidationException
- {
- if (!DatabaseDescriptor.getEnableMaterializedViews())
- {
- throw new InvalidRequestException("Materialized views are disabled. Enable in cassandra.yaml to use.");
- }
-
- // We need to make sure that:
- // - primary key includes all columns in base table's primary key
- // - make sure that the select statement does not have anything other than columns
- // and their names match the base table's names
- // - make sure that primary key does not include any collections
- // - make sure there is no where clause in the select statement
- // - make sure there is not currently a table or view
- // - make sure baseTable gcGraceSeconds > 0
-
- properties.validate();
-
- if (properties.useCompactStorage)
- throw new InvalidRequestException("Cannot use 'COMPACT STORAGE' when defining a materialized view");
-
- // We enforce the keyspace because if the RF is different, the logic to wait for a
- // specific replica would break
- if (!baseName.getKeyspace().equals(keyspace()))
- throw new InvalidRequestException("Cannot create a materialized view on a table in a separate keyspace");
-
- CFMetaData cfm = ThriftValidation.validateColumnFamily(baseName.getKeyspace(), baseName.getColumnFamily());
-
- if (cfm.isCounter())
- throw new InvalidRequestException("Materialized views are not supported on counter tables");
- if (cfm.isSuper())
- throw new InvalidRequestException("Materialized views are not supported on SuperColumn tables");
- if (cfm.isView())
- throw new InvalidRequestException("Materialized views cannot be created against other materialized views");
-
- if (cfm.params.gcGraceSeconds == 0)
- {
- throw new InvalidRequestException(String.format("Cannot create materialized view '%s' for base table " +
- "'%s' with gc_grace_seconds of 0, since this value is " +
- "used to TTL undelivered updates. Setting gc_grace_seconds" +
- " too low might cause undelivered updates to expire " +
- "before being replayed.", cfName.getColumnFamily(),
- baseName.getColumnFamily()));
- }
-
- Set<ColumnIdentifier> included = Sets.newHashSetWithExpectedSize(selectClause.size());
- for (RawSelector selector : selectClause)
- {
- Selectable.Raw selectable = selector.selectable;
- if (selectable instanceof Selectable.WithFieldSelection.Raw)
- throw new InvalidRequestException("Cannot select out a part of type when defining a materialized view");
- if (selectable instanceof Selectable.WithFunction.Raw)
- throw new InvalidRequestException("Cannot use function when defining a materialized view");
- if (selectable instanceof Selectable.WritetimeOrTTL.Raw)
- throw new InvalidRequestException("Cannot use function when defining a materialized view");
- if (selector.alias != null)
- throw new InvalidRequestException("Cannot use alias when defining a materialized view");
-
- Selectable s = selectable.prepare(cfm);
- if (s instanceof Term.Raw)
- throw new InvalidRequestException("Cannot use terms in selection when defining a materialized view");
-
- ColumnDefinition cdef = (ColumnDefinition)s;
- included.add(cdef.name);
- }
-
- Set<ColumnDefinition.Raw> targetPrimaryKeys = new HashSet<>();
- for (ColumnDefinition.Raw identifier : Iterables.concat(partitionKeys, clusteringKeys))
- {
- if (!targetPrimaryKeys.add(identifier))
- throw new InvalidRequestException("Duplicate entry found in PRIMARY KEY: "+identifier);
-
- ColumnDefinition cdef = identifier.prepare(cfm);
-
- if (cdef.type.isMultiCell())
- throw new InvalidRequestException(String.format("Cannot use MultiCell column '%s' in PRIMARY KEY of materialized view", identifier));
-
- if (cdef.isStatic())
- throw new InvalidRequestException(String.format("Cannot use Static column '%s' in PRIMARY KEY of materialized view", identifier));
-
- if (cdef.type instanceof DurationType)
- throw new InvalidRequestException(String.format("Cannot use Duration column '%s' in PRIMARY KEY of materialized view", identifier));
- }
-
- // build the select statement
- Map<ColumnDefinition.Raw, Boolean> orderings = Collections.emptyMap();
- List<ColumnDefinition.Raw> groups = Collections.emptyList();
- SelectStatement.Parameters parameters = new SelectStatement.Parameters(orderings, groups, false, true, false);
-
- SelectStatement.RawStatement rawSelect = new SelectStatement.RawStatement(baseName, parameters, selectClause, whereClause, null, null);
-
- ClientState state = ClientState.forInternalCalls();
- state.setKeyspace(keyspace());
-
- rawSelect.prepareKeyspace(state);
- rawSelect.setBoundVariables(getBoundVariables());
-
- ParsedStatement.Prepared prepared = rawSelect.prepare(true, queryState.getClientState());
- SelectStatement select = (SelectStatement) prepared.statement;
- StatementRestrictions restrictions = select.getRestrictions();
-
- if (!prepared.boundNames.isEmpty())
- throw new InvalidRequestException("Cannot use query parameters in CREATE MATERIALIZED VIEW statements");
-
- // SEE CASSANDRA-13798, use it if the use case is append-only.
- final boolean allowFilteringNonKeyColumns = Boolean.parseBoolean(System.getProperty("cassandra.mv.allow_filtering_nonkey_columns_unsafe",
- "false"));
- if (!restrictions.nonPKRestrictedColumns(false).isEmpty() && !allowFilteringNonKeyColumns)
- {
- throw new InvalidRequestException(
- String.format("Non-primary key columns cannot be restricted in the SELECT statement used"
- + " for materialized view creation (got restrictions on: %s)",
- restrictions.nonPKRestrictedColumns(false)
- .stream()
- .map(def -> def.name.toString())
- .collect(Collectors.joining(", "))));
- }
-
- String whereClauseText = View.relationsToWhereClause(whereClause.relations);
-
- Set<ColumnIdentifier> basePrimaryKeyCols = new HashSet<>();
- for (ColumnDefinition definition : Iterables.concat(cfm.partitionKeyColumns(), cfm.clusteringColumns()))
- basePrimaryKeyCols.add(definition.name);
-
- List<ColumnIdentifier> targetClusteringColumns = new ArrayList<>();
- List<ColumnIdentifier> targetPartitionKeys = new ArrayList<>();
-
- // This is only used as an intermediate state; this is to catch whether multiple non-PK columns are used
- boolean hasNonPKColumn = false;
- for (ColumnDefinition.Raw raw : partitionKeys)
- hasNonPKColumn |= getColumnIdentifier(cfm, basePrimaryKeyCols, hasNonPKColumn, raw, targetPartitionKeys, restrictions);
-
- for (ColumnDefinition.Raw raw : clusteringKeys)
- hasNonPKColumn |= getColumnIdentifier(cfm, basePrimaryKeyCols, hasNonPKColumn, raw, targetClusteringColumns, restrictions);
-
- // We need to include all of the primary key columns from the base table in order to make sure that we do not
- // overwrite values in the view. We cannot support "collapsing" the base table into a smaller number of rows in
- // the view because if we need to generate a tombstone, we have no way of knowing which value is currently being
- // used in the view and whether or not to generate a tombstone. In order to not surprise our users, we require
- // that they include all of the columns. We provide them with a list of all of the columns left to include.
- boolean missingClusteringColumns = false;
- StringBuilder columnNames = new StringBuilder();
- List<ColumnIdentifier> includedColumns = new ArrayList<>();
- for (ColumnDefinition def : cfm.allColumns())
- {
- ColumnIdentifier identifier = def.name;
- boolean includeDef = included.isEmpty() || included.contains(identifier);
-
- if (includeDef && def.isStatic())
- {
- throw new InvalidRequestException(String.format("Unable to include static column '%s' which would be included by Materialized View SELECT * statement", identifier));
- }
-
- boolean defInTargetPrimaryKey = targetClusteringColumns.contains(identifier)
- || targetPartitionKeys.contains(identifier);
-
- if (includeDef && !defInTargetPrimaryKey)
- {
- includedColumns.add(identifier);
- }
- if (!def.isPrimaryKeyColumn()) continue;
-
- if (!defInTargetPrimaryKey)
- {
- if (missingClusteringColumns)
- columnNames.append(',');
- else
- missingClusteringColumns = true;
- columnNames.append(identifier);
- }
- }
- if (missingClusteringColumns)
- throw new InvalidRequestException(String.format("Cannot create Materialized View %s without primary key columns from base %s (%s)",
- columnFamily(), baseName.getColumnFamily(), columnNames.toString()));
-
- if (targetPartitionKeys.isEmpty())
- throw new InvalidRequestException("Must select at least a column for a Materialized View");
-
- if (targetClusteringColumns.isEmpty())
- throw new InvalidRequestException("No columns are defined for Materialized View other than primary key");
-
- TableParams params = properties.properties.asNewTableParams();
-
- if (params.defaultTimeToLive > 0)
- {
- throw new InvalidRequestException("Cannot set default_time_to_live for a materialized view. " +
- "Data in a materialized view always expire at the same time than " +
- "the corresponding data in the parent table.");
- }
-
- CFMetaData.Builder cfmBuilder = CFMetaData.Builder.createView(keyspace(), columnFamily());
- add(cfm, targetPartitionKeys, cfmBuilder::addPartitionKey);
- add(cfm, targetClusteringColumns, cfmBuilder::addClusteringColumn);
- add(cfm, includedColumns, cfmBuilder::addRegularColumn);
- cfmBuilder.withId(properties.properties.getId());
-
- CFMetaData viewCfm = cfmBuilder.build().params(params);
- ViewDefinition definition = new ViewDefinition(keyspace(),
- columnFamily(),
- Schema.instance.getId(keyspace(), baseName.getColumnFamily()),
- baseName.getColumnFamily(),
- included.isEmpty(),
- rawSelect,
- whereClauseText,
- viewCfm);
-
- logger.warn("Creating materialized view {} for {}.{}. {}",
- definition.viewName, cfm.ksName, cfm.cfName, View.USAGE_WARNING);
-
- try
- {
- ClientWarn.instance.warn(View.USAGE_WARNING);
- MigrationManager.announceNewView(definition, isLocalOnly);
- return new Event.SchemaChange(Event.SchemaChange.Change.CREATED, Event.SchemaChange.Target.TABLE, keyspace(), columnFamily());
- }
- catch (AlreadyExistsException e)
- {
- if (ifNotExists)
- return null;
- throw e;
- }
- }
-
- private static boolean getColumnIdentifier(CFMetaData cfm,
- Set<ColumnIdentifier> basePK,
- boolean hasNonPKColumn,
- ColumnDefinition.Raw raw,
- List<ColumnIdentifier> columns,
- StatementRestrictions restrictions)
- {
- ColumnDefinition def = raw.prepare(cfm);
-
- boolean isPk = basePK.contains(def.name);
- if (!isPk && hasNonPKColumn)
- throw new InvalidRequestException(String.format("Cannot include more than one non-primary key column '%s' in materialized view primary key", def.name));
-
- // We don't need to include the "IS NOT NULL" filter on a non-composite partition key
- // because we will never allow a single partition key to be NULL
- boolean isSinglePartitionKey = def.isPartitionKey()
- && cfm.partitionKeyColumns().size() == 1;
- if (!isSinglePartitionKey && !restrictions.isRestricted(def))
- throw new InvalidRequestException(String.format("Primary key column '%s' is required to be filtered by 'IS NOT NULL'", def.name));
-
- columns.add(def.name);
- return !isPk;
- }
-}
diff --git a/src/java/org/apache/cassandra/cql3/statements/DeleteStatement.java b/src/java/org/apache/cassandra/cql3/statements/DeleteStatement.java
index a343fb4..129bf87 100644
--- a/src/java/org/apache/cassandra/cql3/statements/DeleteStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/DeleteStatement.java
@@ -19,15 +19,21 @@
import java.util.List;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.audit.AuditLogContext;
+import org.apache.cassandra.audit.AuditLogEntryType;
import org.apache.cassandra.cql3.*;
+import org.apache.cassandra.cql3.conditions.ColumnCondition;
+import org.apache.cassandra.cql3.conditions.Conditions;
import org.apache.cassandra.cql3.restrictions.StatementRestrictions;
import org.apache.cassandra.db.Clustering;
import org.apache.cassandra.db.Slice;
import org.apache.cassandra.db.partitions.PartitionUpdate;
import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.utils.Pair;
+import org.apache.commons.lang3.builder.ToStringBuilder;
+import org.apache.commons.lang3.builder.ToStringStyle;
import static org.apache.cassandra.cql3.statements.RequestValidations.checkFalse;
import static org.apache.cassandra.cql3.statements.RequestValidations.checkTrue;
@@ -37,20 +43,22 @@
*/
public class DeleteStatement extends ModificationStatement
{
- private DeleteStatement(int boundTerms,
- CFMetaData cfm,
+ private DeleteStatement(VariableSpecifications bindVariables,
+ TableMetadata cfm,
Operations operations,
StatementRestrictions restrictions,
Conditions conditions,
Attributes attrs)
{
- super(StatementType.DELETE, boundTerms, cfm, operations, restrictions, conditions, attrs);
+ super(StatementType.DELETE, bindVariables, cfm, operations, restrictions, conditions, attrs);
}
@Override
- public void addUpdateForKey(PartitionUpdate update, Clustering clustering, UpdateParameters params)
+ public void addUpdateForKey(PartitionUpdate.Builder updateBuilder, Clustering clustering, UpdateParameters params)
throws InvalidRequestException
{
+ TableMetadata metadata = metadata();
+
List<Operation> regularDeletions = getRegularOperations();
List<Operation> staticDeletions = getStaticOperations();
@@ -59,19 +67,19 @@
// We're not deleting any specific columns so it's either a full partition deletion ....
if (clustering.size() == 0)
{
- update.addPartitionDeletion(params.deletionTime());
+ updateBuilder.addPartitionDeletion(params.deletionTime());
}
// ... or a row deletion ...
- else if (clustering.size() == cfm.clusteringColumns().size())
+ else if (clustering.size() == metadata.clusteringColumns().size())
{
params.newRow(clustering);
params.addRowDeletion();
- update.add(params.buildRow());
+ updateBuilder.add(params.buildRow());
}
// ... or a range of rows deletion.
else
{
- update.add(params.makeRangeTombstone(cfm.comparator, clustering));
+ updateBuilder.add(params.makeRangeTombstone(metadata.comparator, clustering));
}
}
else
@@ -81,28 +89,28 @@
// if the clustering size is zero but there are some clustering columns, it means that it's a
// range deletion (the full partition) in which case we need to throw an error as range deletion
// do not support specific columns
- checkFalse(clustering.size() == 0 && cfm.clusteringColumns().size() != 0,
+ checkFalse(clustering.size() == 0 && metadata.clusteringColumns().size() != 0,
"Range deletions are not supported for specific columns");
params.newRow(clustering);
for (Operation op : regularDeletions)
- op.execute(update.partitionKey(), params);
- update.add(params.buildRow());
+ op.execute(updateBuilder.partitionKey(), params);
+ updateBuilder.add(params.buildRow());
}
if (!staticDeletions.isEmpty())
{
params.newRow(Clustering.STATIC_CLUSTERING);
for (Operation op : staticDeletions)
- op.execute(update.partitionKey(), params);
- update.add(params.buildRow());
+ op.execute(updateBuilder.partitionKey(), params);
+ updateBuilder.add(params.buildRow());
}
}
}
@Override
- public void addUpdateForKey(PartitionUpdate update, Slice slice, UpdateParameters params)
+ public void addUpdateForKey(PartitionUpdate.Builder update, Slice slice, UpdateParameters params)
{
List<Operation> regularDeletions = getRegularOperations();
List<Operation> staticDeletions = getStaticOperations();
@@ -116,13 +124,13 @@
public static class Parsed extends ModificationStatement.Parsed
{
private final List<Operation.RawDeletion> deletions;
- private WhereClause whereClause;
+ private final WhereClause whereClause;
- public Parsed(CFName name,
+ public Parsed(QualifiedName name,
Attributes.Raw attrs,
List<Operation.RawDeletion> deletions,
WhereClause whereClause,
- List<Pair<ColumnDefinition.Raw, ColumnCondition.Raw>> conditions,
+ List<Pair<ColumnMetadata.Raw, ColumnCondition.Raw>> conditions,
boolean ifExists)
{
super(name, StatementType.DELETE, attrs, conditions, false, ifExists);
@@ -132,42 +140,36 @@
@Override
- protected ModificationStatement prepareInternal(CFMetaData cfm,
- VariableSpecifications boundNames,
+ protected ModificationStatement prepareInternal(TableMetadata metadata,
+ VariableSpecifications bindVariables,
Conditions conditions,
Attributes attrs)
{
+ checkFalse(metadata.isVirtual(), "Virtual tables don't support DELETE statements");
+
Operations operations = new Operations(type);
- if (cfm.isSuper() && cfm.isDense())
+ for (Operation.RawDeletion deletion : deletions)
{
- conditions = SuperColumnCompatibility.rebuildLWTColumnConditions(conditions, cfm, whereClause);
- whereClause = SuperColumnCompatibility.prepareDeleteOperations(cfm, whereClause, boundNames, operations);
- }
- else
- {
- for (Operation.RawDeletion deletion : deletions)
- {
- ColumnDefinition def = getColumnDefinition(cfm, deletion.affectedColumn());
+ ColumnMetadata def = getColumnDefinition(metadata, deletion.affectedColumn());
- // For compact, we only have one value except the key, so the only form of DELETE that make sense is without a column
- // list. However, we support having the value name for coherence with the static/sparse case
- checkFalse(def.isPrimaryKeyColumn(), "Invalid identifier %s for deletion (should not be a PRIMARY KEY part)", def.name);
+ // For compact, we only have one value except the key, so the only form of DELETE that make sense is without a column
+ // list. However, we support having the value name for coherence with the static/sparse case
+ checkFalse(def.isPrimaryKeyColumn(), "Invalid identifier %s for deletion (should not be a PRIMARY KEY part)", def.name);
- Operation op = deletion.prepare(cfm.ksName, def, cfm);
- op.collectMarkerSpecification(boundNames);
- operations.add(op);
- }
+ Operation op = deletion.prepare(metadata.keyspace, def, metadata);
+ op.collectMarkerSpecification(bindVariables);
+ operations.add(op);
}
- StatementRestrictions restrictions = newRestrictions(cfm,
- boundNames,
+ StatementRestrictions restrictions = newRestrictions(metadata,
+ bindVariables,
operations,
whereClause,
conditions);
- DeleteStatement stmt = new DeleteStatement(boundNames.size(),
- cfm,
+ DeleteStatement stmt = new DeleteStatement(bindVariables,
+ metadata,
operations,
restrictions,
conditions,
@@ -187,4 +189,15 @@
return stmt;
}
}
+
+ @Override
+ public String toString()
+ {
+ return ToStringBuilder.reflectionToString(this, ToStringStyle.SHORT_PREFIX_STYLE);
+ }
+ @Override
+ public AuditLogContext getAuditLogContext()
+ {
+ return new AuditLogContext(AuditLogEntryType.DELETE, keyspace(), columnFamily());
+ }
}
diff --git a/src/java/org/apache/cassandra/cql3/statements/DescribeStatement.java b/src/java/org/apache/cassandra/cql3/statements/DescribeStatement.java
new file mode 100644
index 0000000..48b4160
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/statements/DescribeStatement.java
@@ -0,0 +1,750 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.statements;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.*;
+import java.util.function.BiFunction;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import com.google.common.collect.ImmutableList;
+
+import org.apache.cassandra.audit.AuditLogContext;
+import org.apache.cassandra.audit.AuditLogEntryType;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.*;
+import org.apache.cassandra.cql3.functions.FunctionName;
+import org.apache.cassandra.db.KeyspaceNotDefinedException;
+import org.apache.cassandra.db.marshal.ListType;
+import org.apache.cassandra.db.marshal.MapType;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.db.virtual.VirtualKeyspaceRegistry;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.exceptions.RequestExecutionException;
+import org.apache.cassandra.exceptions.RequestValidationException;
+import org.apache.cassandra.io.util.DataInputBuffer;
+import org.apache.cassandra.io.util.DataOutputBuffer;
+import org.apache.cassandra.schema.*;
+import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.service.QueryState;
+import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.service.pager.PagingState;
+import org.apache.cassandra.transport.messages.ResultMessage;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.UUIDGen;
+
+import static java.lang.String.format;
+import static org.apache.cassandra.cql3.statements.RequestValidations.checkNotEmpty;
+import static org.apache.cassandra.cql3.statements.RequestValidations.checkNotNull;
+import static org.apache.cassandra.cql3.statements.RequestValidations.checkTrue;
+import static org.apache.cassandra.cql3.statements.RequestValidations.invalidRequest;
+import static org.apache.cassandra.utils.ByteBufferUtil.bytes;
+
+/**
+ * The differents <code>DESCRIBE</code> statements parsed from a CQL statement.
+ */
+public abstract class DescribeStatement<T> extends CQLStatement.Raw implements CQLStatement
+{
+ private static final String KS = "system";
+ private static final String CF = "describe";
+
+ /**
+ * The columns returned by the describe queries that only list elements names (e.g. DESCRIBE KEYSPACES, DESCRIBE TABLES...)
+ */
+ private static final List<ColumnSpecification> LIST_METADATA =
+ ImmutableList.of(new ColumnSpecification(KS, CF, new ColumnIdentifier("keyspace_name", true), UTF8Type.instance),
+ new ColumnSpecification(KS, CF, new ColumnIdentifier("type", true), UTF8Type.instance),
+ new ColumnSpecification(KS, CF, new ColumnIdentifier("name", true), UTF8Type.instance));
+
+ /**
+ * The columns returned by the describe queries that returns the CREATE STATEMENT for the different elements (e.g. DESCRIBE KEYSPACE, DESCRIBE TABLE ...)
+ */
+ private static final List<ColumnSpecification> ELEMENT_METADATA =
+ ImmutableList.<ColumnSpecification>builder().addAll(LIST_METADATA)
+ .add(new ColumnSpecification(KS, CF, new ColumnIdentifier("create_statement", true), UTF8Type.instance))
+ .build();
+
+ /**
+ * "Magic version" for the paging state.
+ */
+ private static final int PAGING_STATE_VERSION = 0x0001;
+
+ static final String SCHEMA_CHANGED_WHILE_PAGING_MESSAGE = "The schema has changed since the previous page of the DESCRIBE statement result. " +
+ "Please retry the DESCRIBE statement.";
+
+ private boolean includeInternalDetails;
+
+ public final void withInternalDetails()
+ {
+ this.includeInternalDetails = true;
+ }
+
+ @Override
+ public final CQLStatement prepare(ClientState clientState) throws RequestValidationException
+ {
+ return this;
+ }
+
+ public final List<ColumnSpecification> getBindVariables()
+ {
+ return Collections.emptyList();
+ }
+
+ @Override
+ public final void authorize(ClientState state)
+ {
+ }
+
+ @Override
+ public final void validate(ClientState state)
+ {
+ }
+
+ public final AuditLogContext getAuditLogContext()
+ {
+ return new AuditLogContext(AuditLogEntryType.DESCRIBE);
+ }
+
+ @Override
+ public final ResultMessage execute(QueryState state, QueryOptions options, long queryStartNanoTime) throws RequestValidationException, RequestExecutionException
+ {
+ return executeLocally(state, options);
+ }
+
+ @Override
+ public ResultMessage executeLocally(QueryState state, QueryOptions options)
+ {
+ Keyspaces keyspaces = Schema.instance.snapshot();
+ UUID schemaVersion = Schema.instance.getVersion();
+
+ keyspaces = Keyspaces.builder()
+ .add(keyspaces)
+ .add(VirtualKeyspaceRegistry.instance.virtualKeyspacesMetadata())
+ .build();
+
+ PagingState pagingState = options.getPagingState();
+
+ // The paging implemented here uses some arbitray row number as the partition-key for paging,
+ // which is used to skip/limit the result from the Java Stream. This works good enough for
+ // reasonably sized schemas. Even a 'DESCRIBE SCHEMA' for an abnormally schema with 10000 tables
+ // completes within a few seconds. This seems good enough for now. Once Cassandra actually supports
+ // more than a few hundred tables, the implementation here should be reconsidered.
+ //
+ // Paging is only supported on row-level.
+ //
+ // The "partition key" in the paging-state contains a serialized object:
+ // (short) version, currently 0x0001
+ // (long) row offset
+ // (vint bytes) serialized schema hash (currently the result of Keyspaces.hashCode())
+ //
+
+ long offset = getOffset(pagingState, schemaVersion);
+ int pageSize = options.getPageSize();
+
+ Stream<? extends T> stream = describe(state.getClientState(), keyspaces);
+
+ if (offset > 0L)
+ stream = stream.skip(offset);
+ if (pageSize > 0)
+ stream = stream.limit(pageSize);
+
+ List<List<ByteBuffer>> rows = stream.map(e -> toRow(e, includeInternalDetails))
+ .collect(Collectors.toList());
+
+ ResultSet.ResultMetadata resultMetadata = new ResultSet.ResultMetadata(metadata(state.getClientState()));
+ ResultSet result = new ResultSet(resultMetadata, rows);
+
+ if (pageSize > 0 && rows.size() == pageSize)
+ {
+ result.metadata.setHasMorePages(getPagingState(offset + pageSize, schemaVersion));
+ }
+
+ return new ResultMessage.Rows(result);
+ }
+
+ /**
+ * Returns the columns of the {@code ResultMetadata}
+ */
+ protected abstract List<ColumnSpecification> metadata(ClientState state);
+
+ private PagingState getPagingState(long nextPageOffset, UUID schemaVersion)
+ {
+ try (DataOutputBuffer out = new DataOutputBuffer())
+ {
+ out.writeShort(PAGING_STATE_VERSION);
+ out.writeUTF(FBUtilities.getReleaseVersionString());
+ out.write(UUIDGen.decompose(schemaVersion));
+ out.writeLong(nextPageOffset);
+
+ return new PagingState(out.asNewBuffer(),
+ null,
+ Integer.MAX_VALUE,
+ Integer.MAX_VALUE);
+ }
+ catch (IOException e)
+ {
+ throw new InvalidRequestException("Invalid paging state.", e);
+ }
+ }
+
+ private long getOffset(PagingState pagingState, UUID schemaVersion)
+ {
+ if (pagingState == null)
+ return 0L;
+
+ try (DataInputBuffer in = new DataInputBuffer(pagingState.partitionKey, false))
+ {
+ checkTrue(in.readShort() == PAGING_STATE_VERSION, "Incompatible paging state");
+
+ final String pagingStateServerVersion = in.readUTF();
+ final String releaseVersion = FBUtilities.getReleaseVersionString();
+ checkTrue(pagingStateServerVersion.equals(releaseVersion),
+ "The server version of the paging state %s is different from the one of the server %s",
+ pagingStateServerVersion,
+ releaseVersion);
+
+ byte[] bytes = new byte[UUIDGen.UUID_LEN];
+ in.read(bytes);
+ UUID version = UUIDGen.getUUID(ByteBuffer.wrap(bytes));
+ checkTrue(schemaVersion.equals(version), SCHEMA_CHANGED_WHILE_PAGING_MESSAGE);
+
+ return in.readLong();
+ }
+ catch (IOException e)
+ {
+ throw new InvalidRequestException("Invalid paging state.", e);
+ }
+ }
+
+ protected abstract List<ByteBuffer> toRow(T element, boolean withInternals);
+
+ /**
+ * Returns the schema elements that must be part of the output.
+ */
+ protected abstract Stream<? extends T> describe(ClientState state, Keyspaces keyspaces);
+
+ /**
+ * Returns the metadata for the given keyspace or throws a {@link KeyspaceNotDefinedException} exception.
+ */
+ private static KeyspaceMetadata validateKeyspace(String ks, Keyspaces keyspaces)
+ {
+ return keyspaces.get(ks)
+ .orElseThrow(() -> new KeyspaceNotDefinedException(format("'%s' not found in keyspaces", ks)));
+ }
+
+ /**
+ * {@code DescribeStatement} implementation used for describe queries that only list elements names.
+ */
+ public static final class Listing extends DescribeStatement<SchemaElement>
+ {
+ private final java.util.function.Function<KeyspaceMetadata, Stream<? extends SchemaElement>> elementsProvider;
+
+ public Listing(java.util.function.Function<KeyspaceMetadata, Stream<? extends SchemaElement>> elementsProvider)
+ {
+ this.elementsProvider = elementsProvider;
+ }
+
+ @Override
+ protected Stream<? extends SchemaElement> describe(ClientState state, Keyspaces keyspaces)
+ {
+ String keyspace = state.getRawKeyspace();
+ Stream<KeyspaceMetadata> stream = keyspace == null ? keyspaces.stream().sorted(SchemaElement.NAME_COMPARATOR)
+ : Stream.of(validateKeyspace(keyspace, keyspaces));
+
+ return stream.flatMap(k -> elementsProvider.apply(k).sorted(SchemaElement.NAME_COMPARATOR));
+ }
+
+ @Override
+ protected List<ColumnSpecification> metadata(ClientState state)
+ {
+ return LIST_METADATA;
+ }
+
+ @Override
+ protected List<ByteBuffer> toRow(SchemaElement element, boolean withInternals)
+ {
+ return ImmutableList.of(bytes(element.elementKeyspaceQuotedIfNeeded()),
+ bytes(element.elementType().toString()),
+ bytes(element.elementNameQuotedIfNeeded()));
+ }
+ }
+
+ /**
+ * Creates a {@link DescribeStatement} for {@code DESCRIBE TABLES}.
+ */
+ public static DescribeStatement<SchemaElement> tables()
+ {
+ return new Listing(ks -> ks.tables.stream());
+ }
+
+ /**
+ * Creates a {@link DescribeStatement} for {@code DESCRIBE TYPES}.
+ */
+ public static DescribeStatement<SchemaElement> types()
+ {
+ return new Listing(ks -> ks.types.stream());
+ }
+
+ /**
+ * Creates a {@link DescribeStatement} for {@code DESCRIBE FUNCTIONS}.
+ */
+ public static DescribeStatement<SchemaElement> functions()
+ {
+ return new Listing(ks -> ks.functions.udfs());
+ }
+
+ /**
+ * Creates a {@link DescribeStatement} for {@code DESCRIBE AGGREGATES}.
+ */
+ public static DescribeStatement<SchemaElement> aggregates()
+ {
+ return new Listing(ks -> ks.functions.udas());
+ }
+
+ /**
+ * Creates a {@link DescribeStatement} for {@code DESCRIBE KEYSPACES}.
+ */
+ public static DescribeStatement<SchemaElement> keyspaces()
+ {
+ return new DescribeStatement<SchemaElement>()
+ {
+ @Override
+ protected Stream<? extends SchemaElement> describe(ClientState state, Keyspaces keyspaces)
+ {
+ return keyspaces.stream().sorted(SchemaElement.NAME_COMPARATOR);
+ }
+
+ @Override
+ protected List<ColumnSpecification> metadata(ClientState state)
+ {
+ return LIST_METADATA;
+ }
+
+ @Override
+ protected List<ByteBuffer> toRow(SchemaElement element, boolean withInternals)
+ {
+ return ImmutableList.of(bytes(element.elementKeyspaceQuotedIfNeeded()),
+ bytes(element.elementType().toString()),
+ bytes(element.elementNameQuotedIfNeeded()));
+ }
+ };
+ }
+
+ /**
+ * Creates a {@link DescribeStatement} for {@code DESCRIBE [FULL] SCHEMA}.
+ */
+ public static DescribeStatement<SchemaElement> schema(boolean includeSystemKeyspaces)
+ {
+ return new DescribeStatement<SchemaElement>()
+ {
+ @Override
+ protected Stream<? extends SchemaElement> describe(ClientState state, Keyspaces keyspaces)
+ {
+ return keyspaces.stream()
+ .filter(ks -> includeSystemKeyspaces || !SchemaConstants.isSystemKeyspace(ks.name))
+ .sorted(SchemaElement.NAME_COMPARATOR)
+ .flatMap(ks -> getKeyspaceElements(ks, false));
+ }
+
+ @Override
+ protected List<ColumnSpecification> metadata(ClientState state)
+ {
+ return ELEMENT_METADATA;
+ }
+
+ @Override
+ protected List<ByteBuffer> toRow(SchemaElement element, boolean withInternals)
+ {
+ return ImmutableList.of(bytes(element.elementKeyspaceQuotedIfNeeded()),
+ bytes(element.elementType().toString()),
+ bytes(element.elementNameQuotedIfNeeded()),
+ bytes(element.toCqlString(withInternals)));
+ }
+ };
+ }
+
+ /**
+ * {@code DescribeStatement} implementation used for describe queries for a single schema element.
+ */
+ public static class Element extends DescribeStatement<SchemaElement>
+ {
+ /**
+ * The keyspace name
+ */
+ private final String keyspace;
+
+ /**
+ * The element name
+ */
+ private final String name;
+
+ private final BiFunction<KeyspaceMetadata, String, Stream<? extends SchemaElement>> elementsProvider;
+
+ public Element(String keyspace, String name, BiFunction<KeyspaceMetadata, String, Stream<? extends SchemaElement>> elementsProvider)
+ {
+ this.keyspace = keyspace;
+ this.name = name;
+ this.elementsProvider = elementsProvider;
+ }
+
+ @Override
+ protected Stream<? extends SchemaElement> describe(ClientState state, Keyspaces keyspaces)
+ {
+ String ks = keyspace == null ? checkNotNull(state.getRawKeyspace(), "No keyspace specified and no current keyspace")
+ : keyspace;
+
+ return elementsProvider.apply(validateKeyspace(ks, keyspaces), name);
+ }
+
+ @Override
+ protected List<ColumnSpecification> metadata(ClientState state)
+ {
+ return ELEMENT_METADATA;
+ }
+
+ @Override
+ protected List<ByteBuffer> toRow(SchemaElement element, boolean withInternals)
+ {
+ return ImmutableList.of(bytes(element.elementKeyspaceQuotedIfNeeded()),
+ bytes(element.elementType().toString()),
+ bytes(element.elementNameQuotedIfNeeded()),
+ bytes(element.toCqlString(withInternals)));
+ }
+ }
+
+ /**
+ * Creates a {@link DescribeStatement} for {@code DESCRIBE KEYSPACE}.
+ */
+ public static DescribeStatement<SchemaElement> keyspace(String keyspace, boolean onlyKeyspaceDefinition)
+ {
+ return new Element(keyspace, null, (ks, t) -> getKeyspaceElements(ks, onlyKeyspaceDefinition));
+ }
+
+ private static Stream<? extends SchemaElement> getKeyspaceElements(KeyspaceMetadata ks, boolean onlyKeyspace)
+ {
+ Stream<? extends SchemaElement> s = Stream.of(ks);
+
+ if (!onlyKeyspace)
+ {
+ s = Stream.concat(s, ks.types.sortedStream());
+ s = Stream.concat(s, ks.functions.udfs().sorted(SchemaElement.NAME_COMPARATOR));
+ s = Stream.concat(s, ks.functions.udas().sorted(SchemaElement.NAME_COMPARATOR));
+ s = Stream.concat(s, ks.tables.stream().sorted(SchemaElement.NAME_COMPARATOR)
+ .flatMap(tm -> getTableElements(ks, tm)));
+ }
+
+ return s;
+ }
+
+ private static Stream<? extends SchemaElement> getTableElements(KeyspaceMetadata ks, TableMetadata table)
+ {
+ Stream<? extends SchemaElement> s = Stream.of(table);
+ s = Stream.concat(s, table.indexes.stream()
+ .map(i -> toDescribable(table, i))
+ .sorted(SchemaElement.NAME_COMPARATOR));
+ s = Stream.concat(s, ks.views.stream(table.id)
+ .sorted(SchemaElement.NAME_COMPARATOR));
+ return s;
+ }
+
+ /**
+ * Creates a {@link DescribeStatement} for {@code DESCRIBE TABLE}.
+ */
+ public static DescribeStatement<SchemaElement> table(String keyspace, String name)
+ {
+ return new Element(keyspace, name, (ks, t) -> {
+
+ TableMetadata table = checkNotNull(ks.getTableOrViewNullable(t),
+ "Table '%s' not found in keyspace '%s'", t, ks.name);
+
+ return Stream.concat(Stream.of(table), table.indexes.stream()
+ .map(index -> toDescribable(table, index))
+ .sorted(SchemaElement.NAME_COMPARATOR));
+ });
+ }
+
+ /**
+ * Creates a {@link DescribeStatement} for {@code DESCRIBE INDEX}.
+ */
+ public static DescribeStatement<SchemaElement> index(String keyspace, String name)
+ {
+ return new Element(keyspace, name, (ks, index) -> {
+
+ TableMetadata tm = ks.findIndexedTable(index)
+ .orElseThrow(() -> invalidRequest("Table for existing index '%s' not found in '%s'",
+ index,
+ ks.name));
+ return tm.indexes.get(index)
+ .map(i -> toDescribable(tm, i))
+ .map(Stream::of)
+ .orElseThrow(() -> invalidRequest("Index '%s' not found in '%s'", index, ks.name));
+ });
+ }
+
+ /**
+ * Creates a {@link DescribeStatement} for {@code DESCRIBE MATERIALIZED VIEW}.
+ */
+ public static DescribeStatement<SchemaElement> view(String keyspace, String name)
+ {
+ return new Element(keyspace, name, (ks, view) -> {
+
+ return ks.views.get(view)
+ .map(Stream::of)
+ .orElseThrow(() -> invalidRequest("Materialized view '%s' not found in '%s'", view, ks.name));
+ });
+ }
+
+ /**
+ * Creates a {@link DescribeStatement} for {@code DESCRIBE TYPE}.
+ */
+ public static DescribeStatement<SchemaElement> type(String keyspace, String name)
+ {
+ return new Element(keyspace, name, (ks, type) -> {
+
+ return ks.types.get(ByteBufferUtil.bytes(type))
+ .map(Stream::of)
+ .orElseThrow(() -> invalidRequest("User defined type '%s' not found in '%s'",
+ type,
+ ks.name));
+ });
+ }
+
+ /**
+ * Creates a {@link DescribeStatement} for {@code DESCRIBE FUNCTION}.
+ */
+ public static DescribeStatement<SchemaElement> function(String keyspace, String name)
+ {
+ return new Element(keyspace, name, (ks, n) -> {
+
+ return checkNotEmpty(ks.functions.getUdfs(new FunctionName(ks.name, n)),
+ "User defined function '%s' not found in '%s'", n, ks.name).stream()
+ .sorted(SchemaElement.NAME_COMPARATOR);
+ });
+ }
+
+ /**
+ * Creates a {@link DescribeStatement} for {@code DESCRIBE FUNCTION}.
+ */
+ public static DescribeStatement<SchemaElement> aggregate(String keyspace, String name)
+ {
+ return new Element(keyspace, name, (ks, n) -> {
+
+ return checkNotEmpty(ks.functions.getUdas(new FunctionName(ks.name, n)),
+ "User defined aggregate '%s' not found in '%s'", n, ks.name).stream()
+ .sorted(SchemaElement.NAME_COMPARATOR);
+ });
+ }
+
+ private static SchemaElement toDescribable(TableMetadata table, IndexMetadata index)
+ {
+ return new SchemaElement()
+ {
+ @Override
+ public SchemaElementType elementType()
+ {
+ return SchemaElementType.INDEX;
+ }
+
+ @Override
+ public String elementKeyspace()
+ {
+ return table.keyspace;
+ }
+
+ @Override
+ public String elementName()
+ {
+ return index.name;
+ }
+
+ @Override
+ public String toCqlString(boolean withInternals)
+ {
+ return index.toCqlString(table);
+ }
+ };
+ }
+
+ /**
+ * Creates a {@link DescribeStatement} for the generic {@code DESCRIBE ...}.
+ */
+ public static DescribeStatement<SchemaElement> generic(String keyspace, String name)
+ {
+ return new DescribeStatement<SchemaElement>()
+ {
+ private DescribeStatement<SchemaElement> delegate;
+
+ private DescribeStatement<SchemaElement> resolve(ClientState state, Keyspaces keyspaces)
+ {
+ String ks = keyspace;
+
+ // from cqlsh help: "keyspace or a table or an index or a materialized view (in this order)."
+ if (keyspace == null)
+ {
+ if (keyspaces.containsKeyspace(name))
+ return keyspace(name, false);
+
+ String rawKeyspace = state.getRawKeyspace();
+ ks = rawKeyspace == null ? name : rawKeyspace;
+ }
+
+ KeyspaceMetadata keyspaceMetadata = validateKeyspace(ks, keyspaces);
+
+ if (keyspaceMetadata.tables.getNullable(name) != null)
+ return table(ks, name);
+
+ Optional<TableMetadata> indexed = keyspaceMetadata.findIndexedTable(name);
+ if (indexed.isPresent())
+ {
+ Optional<IndexMetadata> index = indexed.get().indexes.get(name);
+ if (index.isPresent())
+ return index(ks, name);
+ }
+
+ if (keyspaceMetadata.views.getNullable(name) != null)
+ return view(ks, name);
+
+ throw invalidRequest("'%s' not found in keyspace '%s'", name, ks);
+ }
+
+ @Override
+ protected Stream<? extends SchemaElement> describe(ClientState state, Keyspaces keyspaces)
+ {
+ delegate = resolve(state, keyspaces);
+ return delegate.describe(state, keyspaces);
+ }
+
+ @Override
+ protected List<ColumnSpecification> metadata(ClientState state)
+ {
+ return delegate.metadata(state);
+ }
+
+ @Override
+ protected List<ByteBuffer> toRow(SchemaElement element, boolean withInternals)
+ {
+ return delegate.toRow(element, withInternals);
+ }
+ };
+ }
+
+ /**
+ * Creates a {@link DescribeStatement} for {@code DESCRIBE CLUSTER}.
+ */
+ public static DescribeStatement<List<Object>> cluster()
+ {
+ return new DescribeStatement<List<Object>>()
+ {
+ /**
+ * The column index of the cluster name
+ */
+ private static final int CLUSTER_NAME_INDEX = 0;
+
+ /**
+ * The column index of the partitioner name
+ */
+ private static final int PARTITIONER_NAME_INDEX = 1;
+
+ /**
+ * The column index of the snitch class
+ */
+ private static final int SNITCH_CLASS_INDEX = 2;
+
+ /**
+ * The range ownerships index
+ */
+ private static final int RANGE_OWNERSHIPS_INDEX = 3;
+
+ @Override
+ protected Stream<List<Object>> describe(ClientState state, Keyspaces keyspaces)
+ {
+ List<Object> list = new ArrayList<Object>();
+ list.add(DatabaseDescriptor.getClusterName());
+ list.add(trimIfPresent(DatabaseDescriptor.getPartitionerName(), "org.apache.cassandra.dht."));
+ list.add(trimIfPresent(DatabaseDescriptor.getEndpointSnitch().getClass().getName(),
+ "org.apache.cassandra.locator."));
+
+ String useKs = state.getRawKeyspace();
+ if (mustReturnsRangeOwnerships(useKs))
+ {
+ list.add(StorageService.instance.getRangeToAddressMap(useKs)
+ .entrySet()
+ .stream()
+ .sorted(Comparator.comparing(Map.Entry::getKey))
+ .collect(Collectors.toMap(e -> e.getKey().right.toString(),
+ e -> e.getValue()
+ .stream()
+ .map(r -> r.endpoint().toString())
+ .collect(Collectors.toList()))));
+ }
+ return Stream.of(list);
+ }
+
+ private boolean mustReturnsRangeOwnerships(String useKs)
+ {
+ return useKs != null && !SchemaConstants.isLocalSystemKeyspace(useKs) && !SchemaConstants.isSystemKeyspace(useKs);
+ }
+
+ @Override
+ protected List<ColumnSpecification> metadata(ClientState state)
+ {
+ ImmutableList.Builder<ColumnSpecification> builder = ImmutableList.builder();
+ builder.add(new ColumnSpecification(KS, CF, new ColumnIdentifier("cluster", true), UTF8Type.instance),
+ new ColumnSpecification(KS, CF, new ColumnIdentifier("partitioner", true), UTF8Type.instance),
+ new ColumnSpecification(KS, CF, new ColumnIdentifier("snitch", true), UTF8Type.instance));
+
+ if (mustReturnsRangeOwnerships(state.getRawKeyspace()))
+ builder.add(new ColumnSpecification(KS, CF, new ColumnIdentifier("range_ownership", true), MapType.getInstance(UTF8Type.instance,
+ ListType.getInstance(UTF8Type.instance, false), false)));
+
+ return builder.build();
+ }
+
+ @Override
+ protected List<ByteBuffer> toRow(List<Object> elements, boolean withInternals)
+ {
+ ImmutableList.Builder<ByteBuffer> builder = ImmutableList.builder();
+
+ builder.add(UTF8Type.instance.decompose((String) elements.get(CLUSTER_NAME_INDEX)),
+ UTF8Type.instance.decompose((String) elements.get(PARTITIONER_NAME_INDEX)),
+ UTF8Type.instance.decompose((String) elements.get(SNITCH_CLASS_INDEX)));
+
+ if (elements.size() > 3)
+ {
+ MapType<String, List<String>> rangeOwnershipType = MapType.getInstance(UTF8Type.instance,
+ ListType.getInstance(UTF8Type.instance, false),
+ false);
+
+ builder.add(rangeOwnershipType.decompose((Map<String, List<String>>) elements.get(RANGE_OWNERSHIPS_INDEX)));
+ }
+
+ return builder.build();
+ }
+
+ private String trimIfPresent(String src, String begin)
+ {
+ if (src.startsWith(begin))
+ return src.substring(begin.length());
+ return src;
+ }
+ };
+ }
+}
diff --git a/src/java/org/apache/cassandra/cql3/statements/DropAggregateStatement.java b/src/java/org/apache/cassandra/cql3/statements/DropAggregateStatement.java
deleted file mode 100644
index ae8ad8c..0000000
--- a/src/java/org/apache/cassandra/cql3/statements/DropAggregateStatement.java
+++ /dev/null
@@ -1,153 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.cql3.statements;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-
-import org.apache.cassandra.auth.Permission;
-import org.apache.cassandra.config.Schema;
-import org.apache.cassandra.cql3.CQL3Type;
-import org.apache.cassandra.cql3.functions.*;
-import org.apache.cassandra.db.marshal.AbstractType;
-import org.apache.cassandra.exceptions.InvalidRequestException;
-import org.apache.cassandra.exceptions.RequestValidationException;
-import org.apache.cassandra.exceptions.UnauthorizedException;
-import org.apache.cassandra.service.ClientState;
-import org.apache.cassandra.service.MigrationManager;
-import org.apache.cassandra.service.QueryState;
-import org.apache.cassandra.thrift.ThriftValidation;
-import org.apache.cassandra.transport.Event;
-
-/**
- * A {@code DROP AGGREGATE} statement parsed from a CQL query.
- */
-public final class DropAggregateStatement extends SchemaAlteringStatement
-{
- private FunctionName functionName;
- private final boolean ifExists;
- private final List<CQL3Type.Raw> argRawTypes;
- private final boolean argsPresent;
-
- public DropAggregateStatement(FunctionName functionName,
- List<CQL3Type.Raw> argRawTypes,
- boolean argsPresent,
- boolean ifExists)
- {
- this.functionName = functionName;
- this.argRawTypes = argRawTypes;
- this.argsPresent = argsPresent;
- this.ifExists = ifExists;
- }
-
- public void prepareKeyspace(ClientState state) throws InvalidRequestException
- {
- if (!functionName.hasKeyspace() && state.getRawKeyspace() != null)
- functionName = new FunctionName(state.getKeyspace(), functionName.name);
-
- if (!functionName.hasKeyspace())
- throw new InvalidRequestException("Functions must be fully qualified with a keyspace name if a keyspace is not set for the session");
-
- ThriftValidation.validateKeyspaceNotSystem(functionName.keyspace);
- }
-
- public void checkAccess(ClientState state) throws UnauthorizedException, InvalidRequestException
- {
- // TODO CASSANDRA-7557 (function DDL permission)
-
- state.hasKeyspaceAccess(functionName.keyspace, Permission.DROP);
- }
-
- public void validate(ClientState state) throws RequestValidationException
- {
- }
-
- public Event.SchemaChange announceMigration(QueryState queryState, boolean isLocalOnly) throws RequestValidationException
- {
- Collection<Function> olds = Schema.instance.getFunctions(functionName);
-
- if (!argsPresent && olds != null && olds.size() > 1)
- throw new InvalidRequestException(String.format("'DROP AGGREGATE %s' matches multiple function definitions; " +
- "specify the argument types by issuing a statement like " +
- "'DROP AGGREGATE %s (type, type, ...)'. Hint: use cqlsh " +
- "'DESCRIBE AGGREGATE %s' command to find all overloads",
- functionName, functionName, functionName));
-
- Function old = null;
- if (argsPresent)
- {
- if (Schema.instance.getKSMetaData(functionName.keyspace) != null)
- {
- List<AbstractType<?>> argTypes = new ArrayList<>(argRawTypes.size());
- for (CQL3Type.Raw rawType : argRawTypes)
- argTypes.add(prepareType("arguments", rawType));
-
- old = Schema.instance.findFunction(functionName, argTypes).orElse(null);
- }
- if (old == null || !(old instanceof AggregateFunction))
- {
- if (ifExists)
- return null;
- // just build a nicer error message
- StringBuilder sb = new StringBuilder();
- for (CQL3Type.Raw rawType : argRawTypes)
- {
- if (sb.length() > 0)
- sb.append(", ");
- sb.append(rawType);
- }
- throw new InvalidRequestException(String.format("Cannot drop non existing aggregate '%s(%s)'",
- functionName, sb));
- }
- }
- else
- {
- if (olds == null || olds.isEmpty() || !(olds.iterator().next() instanceof AggregateFunction))
- {
- if (ifExists)
- return null;
- throw new InvalidRequestException(String.format("Cannot drop non existing aggregate '%s'", functionName));
- }
- old = olds.iterator().next();
- }
-
- if (old.isNative())
- throw new InvalidRequestException(String.format("Cannot drop aggregate '%s' because it is a " +
- "native (built-in) function", functionName));
-
- MigrationManager.announceAggregateDrop((UDAggregate)old, isLocalOnly);
- return new Event.SchemaChange(Event.SchemaChange.Change.DROPPED, Event.SchemaChange.Target.AGGREGATE,
- old.name().keyspace, old.name().name, AbstractType.asCQLTypeStringList(old.argTypes()));
-
- }
-
- private AbstractType<?> prepareType(String typeName, CQL3Type.Raw rawType)
- {
- if (rawType.isFrozen())
- throw new InvalidRequestException(String.format("The function %s should not be frozen; remove the frozen<> modifier", typeName));
-
- // UDT are not supported non frozen but we do not allow the frozen keyword for argument. So for the moment we
- // freeze them here
- if (!rawType.canBeNonFrozen())
- rawType.freeze();
-
- AbstractType<?> type = rawType.prepare(functionName.keyspace).getType();
- return type;
- }
-}
diff --git a/src/java/org/apache/cassandra/cql3/statements/DropFunctionStatement.java b/src/java/org/apache/cassandra/cql3/statements/DropFunctionStatement.java
deleted file mode 100644
index 8845a82..0000000
--- a/src/java/org/apache/cassandra/cql3/statements/DropFunctionStatement.java
+++ /dev/null
@@ -1,190 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.cql3.statements;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-
-import com.google.common.base.Joiner;
-
-import org.apache.cassandra.auth.FunctionResource;
-import org.apache.cassandra.auth.Permission;
-import org.apache.cassandra.config.Schema;
-import org.apache.cassandra.cql3.CQL3Type;
-import org.apache.cassandra.cql3.functions.*;
-import org.apache.cassandra.db.marshal.AbstractType;
-import org.apache.cassandra.exceptions.InvalidRequestException;
-import org.apache.cassandra.exceptions.RequestValidationException;
-import org.apache.cassandra.exceptions.UnauthorizedException;
-import org.apache.cassandra.schema.KeyspaceMetadata;
-import org.apache.cassandra.service.ClientState;
-import org.apache.cassandra.service.MigrationManager;
-import org.apache.cassandra.service.QueryState;
-import org.apache.cassandra.thrift.ThriftValidation;
-import org.apache.cassandra.transport.Event;
-
-/**
- * A {@code DROP FUNCTION} statement parsed from a CQL query.
- */
-public final class DropFunctionStatement extends SchemaAlteringStatement
-{
- private FunctionName functionName;
- private final boolean ifExists;
- private final List<CQL3Type.Raw> argRawTypes;
- private final boolean argsPresent;
-
- private List<AbstractType<?>> argTypes;
-
- public DropFunctionStatement(FunctionName functionName,
- List<CQL3Type.Raw> argRawTypes,
- boolean argsPresent,
- boolean ifExists)
- {
- this.functionName = functionName;
- this.argRawTypes = argRawTypes;
- this.argsPresent = argsPresent;
- this.ifExists = ifExists;
- }
-
- @Override
- public Prepared prepare(ClientState clientState) throws InvalidRequestException
- {
- if (Schema.instance.getKSMetaData(functionName.keyspace) != null)
- {
- argTypes = new ArrayList<>(argRawTypes.size());
- for (CQL3Type.Raw rawType : argRawTypes)
- {
- if (rawType.isFrozen())
- throw new InvalidRequestException("The function arguments should not be frozen; remove the frozen<> modifier");
-
- // UDT are not supported non frozen but we do not allow the frozen keyword for argument. So for the moment we
- // freeze them here
- if (!rawType.canBeNonFrozen())
- rawType.freeze();
-
- argTypes.add(rawType.prepare(functionName.keyspace).getType());
- }
- }
-
- return super.prepare(clientState);
- }
-
- @Override
- public void prepareKeyspace(ClientState state) throws InvalidRequestException
- {
- if (!functionName.hasKeyspace() && state.getRawKeyspace() != null)
- functionName = new FunctionName(state.getKeyspace(), functionName.name);
-
- if (!functionName.hasKeyspace())
- throw new InvalidRequestException("Functions must be fully qualified with a keyspace name if a keyspace is not set for the session");
-
- ThriftValidation.validateKeyspaceNotSystem(functionName.keyspace);
- }
-
- public void checkAccess(ClientState state) throws UnauthorizedException, InvalidRequestException
- {
- Function function = findFunction();
- if (function == null)
- {
- if (!ifExists)
- throw new InvalidRequestException(String.format("Unconfigured function %s.%s(%s)",
- functionName.keyspace,
- functionName.name,
- Joiner.on(",").join(argRawTypes)));
- }
- else
- {
- state.ensureHasPermission(Permission.DROP, FunctionResource.function(function.name().keyspace,
- function.name().name,
- function.argTypes()));
- }
- }
-
- public void validate(ClientState state)
- {
- Collection<Function> olds = Schema.instance.getFunctions(functionName);
-
- if (!argsPresent && olds != null && olds.size() > 1)
- throw new InvalidRequestException(String.format("'DROP FUNCTION %s' matches multiple function definitions; " +
- "specify the argument types by issuing a statement like " +
- "'DROP FUNCTION %s (type, type, ...)'. Hint: use cqlsh " +
- "'DESCRIBE FUNCTION %s' command to find all overloads",
- functionName, functionName, functionName));
- }
-
- public Event.SchemaChange announceMigration(QueryState queryState, boolean isLocalOnly) throws RequestValidationException
- {
- Function old = findFunction();
- if (old == null)
- {
- if (ifExists)
- return null;
- else
- throw new InvalidRequestException(getMissingFunctionError());
- }
-
- KeyspaceMetadata ksm = Schema.instance.getKSMetaData(old.name().keyspace);
- Collection<UDAggregate> referrers = ksm.functions.aggregatesUsingFunction(old);
- if (!referrers.isEmpty())
- throw new InvalidRequestException(String.format("Function '%s' still referenced by %s", old, referrers));
-
- MigrationManager.announceFunctionDrop((UDFunction) old, isLocalOnly);
-
- return new Event.SchemaChange(Event.SchemaChange.Change.DROPPED, Event.SchemaChange.Target.FUNCTION,
- old.name().keyspace, old.name().name, AbstractType.asCQLTypeStringList(old.argTypes()));
- }
-
- private String getMissingFunctionError()
- {
- // just build a nicer error message
- StringBuilder sb = new StringBuilder("Cannot drop non existing function '");
- sb.append(functionName);
- if (argsPresent)
- sb.append(Joiner.on(", ").join(argRawTypes));
- sb.append('\'');
- return sb.toString();
- }
-
- private Function findFunction()
- {
- Function old;
- if (argsPresent)
- {
- if (argTypes == null)
- {
- return null;
- }
-
- old = Schema.instance.findFunction(functionName, argTypes).orElse(null);
- if (old == null || !(old instanceof ScalarFunction))
- {
- return null;
- }
- }
- else
- {
- Collection<Function> olds = Schema.instance.getFunctions(functionName);
- if (olds == null || olds.isEmpty() || !(olds.iterator().next() instanceof ScalarFunction))
- return null;
-
- old = olds.iterator().next();
- }
- return old;
- }
-}
diff --git a/src/java/org/apache/cassandra/cql3/statements/DropIndexStatement.java b/src/java/org/apache/cassandra/cql3/statements/DropIndexStatement.java
deleted file mode 100644
index fcd06d4..0000000
--- a/src/java/org/apache/cassandra/cql3/statements/DropIndexStatement.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.cql3.statements;
-
-import org.apache.cassandra.auth.Permission;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.Schema;
-import org.apache.cassandra.cql3.IndexName;
-import org.apache.cassandra.cql3.QueryOptions;
-import org.apache.cassandra.db.KeyspaceNotDefinedException;
-import org.apache.cassandra.exceptions.*;
-import org.apache.cassandra.schema.KeyspaceMetadata;
-import org.apache.cassandra.service.ClientState;
-import org.apache.cassandra.service.MigrationManager;
-import org.apache.cassandra.service.QueryState;
-import org.apache.cassandra.transport.Event;
-import org.apache.cassandra.transport.messages.ResultMessage;
-
-public class DropIndexStatement extends SchemaAlteringStatement
-{
- public final String indexName;
- public final boolean ifExists;
-
- public DropIndexStatement(IndexName indexName, boolean ifExists)
- {
- super(indexName.getCfName());
- this.indexName = indexName.getIdx();
- this.ifExists = ifExists;
- }
-
- public String columnFamily()
- {
- CFMetaData cfm = lookupIndexedTable();
- return cfm == null ? null : cfm.cfName;
- }
-
- public void checkAccess(ClientState state) throws UnauthorizedException, InvalidRequestException
- {
- CFMetaData cfm = lookupIndexedTable();
- if (cfm == null)
- return;
-
- state.hasColumnFamilyAccess(cfm.ksName, cfm.cfName, Permission.ALTER);
- }
-
- public void validate(ClientState state)
- {
- // validated in lookupIndexedTable()
- }
-
- @Override
- public ResultMessage execute(QueryState state, QueryOptions options, long queryStartNanoTime) throws RequestValidationException
- {
- Event.SchemaChange ce = announceMigration(state, false);
- return ce == null ? null : new ResultMessage.SchemaChange(ce);
- }
-
- public Event.SchemaChange announceMigration(QueryState queryState, boolean isLocalOnly) throws InvalidRequestException, ConfigurationException
- {
- CFMetaData cfm = lookupIndexedTable();
- if (cfm == null)
- return null;
-
- CFMetaData updatedCfm = cfm.copy();
- updatedCfm.indexes(updatedCfm.getIndexes().without(indexName));
- MigrationManager.announceColumnFamilyUpdate(updatedCfm, isLocalOnly);
- // Dropping an index is akin to updating the CF
- // Note that we shouldn't call columnFamily() at this point because the index has been dropped and the call to lookupIndexedTable()
- // in that method would now throw.
- return new Event.SchemaChange(Event.SchemaChange.Change.UPDATED, Event.SchemaChange.Target.TABLE, cfm.ksName, cfm.cfName);
- }
-
- /**
- * The table for which the index should be dropped, or null if the index doesn't exist
- *
- * @return the metadata for the table containing the dropped index, or {@code null}
- * if the index to drop cannot be found but "IF EXISTS" is set on the statement.
- *
- * @throws InvalidRequestException if the index cannot be found and "IF EXISTS" is not
- * set on the statement.
- */
- private CFMetaData lookupIndexedTable()
- {
- KeyspaceMetadata ksm = Schema.instance.getKSMetaData(keyspace());
- if (ksm == null)
- throw new KeyspaceNotDefinedException("Keyspace " + keyspace() + " does not exist");
-
- return ksm.findIndexedTable(indexName)
- .orElseGet(() -> {
- if (ifExists)
- return null;
- else
- throw new InvalidRequestException(String.format("Index '%s' could not be found in any " +
- "of the tables of keyspace '%s'",
- indexName, keyspace()));
- });
- }
-}
diff --git a/src/java/org/apache/cassandra/cql3/statements/DropKeyspaceStatement.java b/src/java/org/apache/cassandra/cql3/statements/DropKeyspaceStatement.java
deleted file mode 100644
index 7144f39..0000000
--- a/src/java/org/apache/cassandra/cql3/statements/DropKeyspaceStatement.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.cql3.statements;
-
-import org.apache.cassandra.auth.Permission;
-import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.exceptions.InvalidRequestException;
-import org.apache.cassandra.exceptions.RequestValidationException;
-import org.apache.cassandra.exceptions.UnauthorizedException;
-import org.apache.cassandra.service.ClientState;
-import org.apache.cassandra.service.MigrationManager;
-import org.apache.cassandra.service.QueryState;
-import org.apache.cassandra.thrift.ThriftValidation;
-import org.apache.cassandra.transport.Event;
-
-public class DropKeyspaceStatement extends SchemaAlteringStatement
-{
- private final String keyspace;
- private final boolean ifExists;
-
- public DropKeyspaceStatement(String keyspace, boolean ifExists)
- {
- super();
- this.keyspace = keyspace;
- this.ifExists = ifExists;
- }
-
- public void checkAccess(ClientState state) throws UnauthorizedException, InvalidRequestException
- {
- state.hasKeyspaceAccess(keyspace, Permission.DROP);
- }
-
- public void validate(ClientState state) throws RequestValidationException
- {
- ThriftValidation.validateKeyspaceNotSystem(keyspace);
- }
-
- @Override
- public String keyspace()
- {
- return keyspace;
- }
-
- public Event.SchemaChange announceMigration(QueryState queryState, boolean isLocalOnly) throws ConfigurationException
- {
- try
- {
- MigrationManager.announceKeyspaceDrop(keyspace, isLocalOnly);
- return new Event.SchemaChange(Event.SchemaChange.Change.DROPPED, keyspace());
- }
- catch(ConfigurationException e)
- {
- if (ifExists)
- return null;
- throw e;
- }
- }
-}
diff --git a/src/java/org/apache/cassandra/cql3/statements/DropRoleStatement.java b/src/java/org/apache/cassandra/cql3/statements/DropRoleStatement.java
index 55fa83a..058ab01 100644
--- a/src/java/org/apache/cassandra/cql3/statements/DropRoleStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/DropRoleStatement.java
@@ -17,12 +17,16 @@
*/
package org.apache.cassandra.cql3.statements;
+import org.apache.cassandra.audit.AuditLogContext;
+import org.apache.cassandra.audit.AuditLogEntryType;
import org.apache.cassandra.auth.*;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.RoleName;
import org.apache.cassandra.exceptions.*;
import org.apache.cassandra.service.ClientState;
import org.apache.cassandra.transport.messages.ResultMessage;
+import org.apache.commons.lang3.builder.ToStringBuilder;
+import org.apache.commons.lang3.builder.ToStringStyle;
public class DropRoleStatement extends AuthenticationStatement
{
@@ -35,7 +39,7 @@
this.ifExists = ifExists;
}
- public void checkAccess(ClientState state) throws UnauthorizedException
+ public void authorize(ClientState state) throws UnauthorizedException
{
super.checkPermission(state, Permission.DROP, role);
@@ -49,7 +53,7 @@
public void validate(ClientState state) throws RequestValidationException
{
- // validate login here before checkAccess to avoid leaking user existence to anonymous users.
+ // validate login here before authorize to avoid leaking user existence to anonymous users.
state.ensureNotAnonymous();
if (!ifExists && !DatabaseDescriptor.getRoleManager().isExistingRole(role))
@@ -70,6 +74,19 @@
DatabaseDescriptor.getRoleManager().dropRole(state.getUser(), role);
DatabaseDescriptor.getAuthorizer().revokeAllFrom(role);
DatabaseDescriptor.getAuthorizer().revokeAllOn(role);
+ DatabaseDescriptor.getNetworkAuthorizer().drop(role);
return null;
}
+
+ @Override
+ public String toString()
+ {
+ return ToStringBuilder.reflectionToString(this, ToStringStyle.SHORT_PREFIX_STYLE);
+ }
+
+ @Override
+ public AuditLogContext getAuditLogContext()
+ {
+ return new AuditLogContext(AuditLogEntryType.DROP_ROLE);
+ }
}
diff --git a/src/java/org/apache/cassandra/cql3/statements/DropTableStatement.java b/src/java/org/apache/cassandra/cql3/statements/DropTableStatement.java
deleted file mode 100644
index 5641185..0000000
--- a/src/java/org/apache/cassandra/cql3/statements/DropTableStatement.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.cql3.statements;
-
-import org.apache.cassandra.auth.Permission;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ViewDefinition;
-import org.apache.cassandra.config.Schema;
-import org.apache.cassandra.cql3.CFName;
-import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.exceptions.InvalidRequestException;
-import org.apache.cassandra.exceptions.UnauthorizedException;
-import org.apache.cassandra.schema.KeyspaceMetadata;
-import org.apache.cassandra.service.ClientState;
-import org.apache.cassandra.service.MigrationManager;
-import org.apache.cassandra.service.QueryState;
-import org.apache.cassandra.transport.Event;
-
-public class DropTableStatement extends SchemaAlteringStatement
-{
- private final boolean ifExists;
-
- public DropTableStatement(CFName name, boolean ifExists)
- {
- super(name);
- this.ifExists = ifExists;
- }
-
- public void checkAccess(ClientState state) throws UnauthorizedException, InvalidRequestException
- {
- try
- {
- state.hasColumnFamilyAccess(keyspace(), columnFamily(), Permission.DROP);
- }
- catch (InvalidRequestException e)
- {
- if (!ifExists)
- throw e;
- }
- }
-
- public void validate(ClientState state)
- {
- // validated in announceMigration()
- }
-
- public Event.SchemaChange announceMigration(QueryState queryState, boolean isLocalOnly) throws ConfigurationException
- {
- try
- {
- KeyspaceMetadata ksm = Schema.instance.getKSMetaData(keyspace());
- if (ksm == null)
- throw new ConfigurationException(String.format("Cannot drop table in unknown keyspace '%s'", keyspace()));
- CFMetaData cfm = ksm.getTableOrViewNullable(columnFamily());
- if (cfm != null)
- {
- if (cfm.isView())
- throw new InvalidRequestException("Cannot use DROP TABLE on Materialized View");
-
- boolean rejectDrop = false;
- StringBuilder messageBuilder = new StringBuilder();
- for (ViewDefinition def : ksm.views)
- {
- if (def.baseTableId.equals(cfm.cfId))
- {
- if (rejectDrop)
- messageBuilder.append(',');
- rejectDrop = true;
- messageBuilder.append(def.viewName);
- }
- }
- if (rejectDrop)
- {
- throw new InvalidRequestException(String.format("Cannot drop table when materialized views still depend on it (%s.{%s})",
- keyspace(),
- messageBuilder.toString()));
- }
- }
- MigrationManager.announceColumnFamilyDrop(keyspace(), columnFamily(), isLocalOnly);
- return new Event.SchemaChange(Event.SchemaChange.Change.DROPPED, Event.SchemaChange.Target.TABLE, keyspace(), columnFamily());
- }
- catch (ConfigurationException e)
- {
- if (ifExists)
- return null;
- throw e;
- }
- }
-}
diff --git a/src/java/org/apache/cassandra/cql3/statements/DropTriggerStatement.java b/src/java/org/apache/cassandra/cql3/statements/DropTriggerStatement.java
deleted file mode 100644
index 162c736..0000000
--- a/src/java/org/apache/cassandra/cql3/statements/DropTriggerStatement.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.cql3.statements;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.Schema;
-import org.apache.cassandra.cql3.CFName;
-import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.exceptions.InvalidRequestException;
-import org.apache.cassandra.exceptions.RequestValidationException;
-import org.apache.cassandra.exceptions.UnauthorizedException;
-import org.apache.cassandra.schema.Triggers;
-import org.apache.cassandra.service.ClientState;
-import org.apache.cassandra.service.MigrationManager;
-import org.apache.cassandra.service.QueryState;
-import org.apache.cassandra.thrift.ThriftValidation;
-import org.apache.cassandra.transport.Event;
-
-public class DropTriggerStatement extends SchemaAlteringStatement
-{
- private static final Logger logger = LoggerFactory.getLogger(DropTriggerStatement.class);
-
- private final String triggerName;
-
- private final boolean ifExists;
-
- public DropTriggerStatement(CFName name, String triggerName, boolean ifExists)
- {
- super(name);
- this.triggerName = triggerName;
- this.ifExists = ifExists;
- }
-
- public void checkAccess(ClientState state) throws UnauthorizedException
- {
- state.ensureIsSuper("Only superusers are allowed to perfrom DROP TRIGGER queries");
- }
-
- public void validate(ClientState state) throws RequestValidationException
- {
- ThriftValidation.validateColumnFamily(keyspace(), columnFamily());
- }
-
- public Event.SchemaChange announceMigration(QueryState queryState, boolean isLocalOnly) throws ConfigurationException, InvalidRequestException
- {
- CFMetaData cfm = Schema.instance.getCFMetaData(keyspace(), columnFamily()).copy();
- Triggers triggers = cfm.getTriggers();
-
- if (!triggers.get(triggerName).isPresent())
- {
- if (ifExists)
- return null;
- else
- throw new InvalidRequestException(String.format("Trigger %s was not found", triggerName));
- }
-
- logger.info("Dropping trigger with name {}", triggerName);
- cfm.triggers(triggers.without(triggerName));
- MigrationManager.announceColumnFamilyUpdate(cfm, isLocalOnly);
- return new Event.SchemaChange(Event.SchemaChange.Change.UPDATED, Event.SchemaChange.Target.TABLE, keyspace(), columnFamily());
- }
-}
diff --git a/src/java/org/apache/cassandra/cql3/statements/DropTypeStatement.java b/src/java/org/apache/cassandra/cql3/statements/DropTypeStatement.java
deleted file mode 100644
index cd6daae..0000000
--- a/src/java/org/apache/cassandra/cql3/statements/DropTypeStatement.java
+++ /dev/null
@@ -1,121 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.cql3.statements;
-
-import org.apache.cassandra.auth.Permission;
-import org.apache.cassandra.config.*;
-import org.apache.cassandra.cql3.*;
-import org.apache.cassandra.cql3.functions.Function;
-import org.apache.cassandra.db.marshal.*;
-import org.apache.cassandra.exceptions.*;
-import org.apache.cassandra.schema.KeyspaceMetadata;
-import org.apache.cassandra.service.ClientState;
-import org.apache.cassandra.service.MigrationManager;
-import org.apache.cassandra.service.QueryState;
-import org.apache.cassandra.transport.Event;
-
-public class DropTypeStatement extends SchemaAlteringStatement
-{
- private final UTName name;
- private final boolean ifExists;
-
- public DropTypeStatement(UTName name, boolean ifExists)
- {
- this.name = name;
- this.ifExists = ifExists;
- }
-
- @Override
- public void prepareKeyspace(ClientState state) throws InvalidRequestException
- {
- if (!name.hasKeyspace())
- name.setKeyspace(state.getKeyspace());
- }
-
- public void checkAccess(ClientState state) throws UnauthorizedException, InvalidRequestException
- {
- state.hasKeyspaceAccess(keyspace(), Permission.DROP);
- }
-
- public void validate(ClientState state) throws RequestValidationException
- {
- KeyspaceMetadata ksm = Schema.instance.getKSMetaData(name.getKeyspace());
- if (ksm == null)
- {
- if (ifExists)
- return;
- else
- throw new InvalidRequestException(String.format("Cannot drop type in unknown keyspace %s", name.getKeyspace()));
- }
-
- if (!ksm.types.get(name.getUserTypeName()).isPresent())
- {
- if (ifExists)
- return;
- else
- throw new InvalidRequestException(String.format("No user type named %s exists.", name));
- }
-
- // We don't want to drop a type unless it's not used anymore (mainly because
- // if someone drops a type and recreates one with the same name but different
- // definition with the previous name still in use, things can get messy).
- // We have two places to check: 1) other user type that can nest the one
- // we drop and 2) existing tables referencing the type (maybe in a nested
- // way).
-
- for (Function function : ksm.functions)
- {
- if (function.returnType().referencesUserType(name.getStringTypeName()))
- throw new InvalidRequestException(String.format("Cannot drop user type %s as it is still used by function %s", name, function));
-
- for (AbstractType<?> argType : function.argTypes())
- if (argType.referencesUserType(name.getStringTypeName()))
- throw new InvalidRequestException(String.format("Cannot drop user type %s as it is still used by function %s", name, function));
- }
-
- for (UserType ut : ksm.types)
- if (!ut.name.equals(name.getUserTypeName()) && ut.referencesUserType(name.getStringTypeName()))
- throw new InvalidRequestException(String.format("Cannot drop user type %s as it is still used by user type %s", name, ut.getNameAsString()));
-
- for (CFMetaData cfm : ksm.tablesAndViews())
- for (ColumnDefinition def : cfm.allColumns())
- if (def.type.referencesUserType(name.getStringTypeName()))
- throw new InvalidRequestException(String.format("Cannot drop user type %s as it is still used by table %s.%s", name, cfm.ksName, cfm.cfName));
- }
-
- @Override
- public String keyspace()
- {
- return name.getKeyspace();
- }
-
- public Event.SchemaChange announceMigration(QueryState queryState, boolean isLocalOnly) throws InvalidRequestException, ConfigurationException
- {
- KeyspaceMetadata ksm = Schema.instance.getKSMetaData(name.getKeyspace());
- if (ksm == null)
- return null; // do not assert (otherwise IF EXISTS case fails)
-
- UserType toDrop = ksm.types.getNullable(name.getUserTypeName());
- // Can be null with ifExists
- if (toDrop == null)
- return null;
-
- MigrationManager.announceTypeDrop(toDrop, isLocalOnly);
- return new Event.SchemaChange(Event.SchemaChange.Change.DROPPED, Event.SchemaChange.Target.TYPE, keyspace(), name.getStringTypeName());
- }
-}
diff --git a/src/java/org/apache/cassandra/cql3/statements/DropViewStatement.java b/src/java/org/apache/cassandra/cql3/statements/DropViewStatement.java
deleted file mode 100644
index 2f393d3..0000000
--- a/src/java/org/apache/cassandra/cql3/statements/DropViewStatement.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.cql3.statements;
-
-import org.apache.cassandra.auth.Permission;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.cql3.CFName;
-import org.apache.cassandra.db.view.View;
-import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.exceptions.InvalidRequestException;
-import org.apache.cassandra.exceptions.UnauthorizedException;
-import org.apache.cassandra.service.ClientState;
-import org.apache.cassandra.service.MigrationManager;
-import org.apache.cassandra.service.QueryState;
-import org.apache.cassandra.transport.Event;
-
-public class DropViewStatement extends SchemaAlteringStatement
-{
- public final boolean ifExists;
-
- public DropViewStatement(CFName cf, boolean ifExists)
- {
- super(cf);
- this.ifExists = ifExists;
- }
-
- public void checkAccess(ClientState state) throws UnauthorizedException, InvalidRequestException
- {
- CFMetaData baseTable = View.findBaseTable(keyspace(), columnFamily());
- if (baseTable != null)
- state.hasColumnFamilyAccess(keyspace(), baseTable.cfName, Permission.ALTER);
- }
-
- public void validate(ClientState state)
- {
- // validated in findIndexedCf()
- }
-
- public Event.SchemaChange announceMigration(QueryState queryState, boolean isLocalOnly) throws InvalidRequestException, ConfigurationException
- {
- try
- {
-// ViewDefinition view = Schema.instance.getViewDefinition(keyspace(), columnFamily());
-// if (view == null)
-// {
-// if (Schema.instance.getCFMetaData(keyspace(), columnFamily()) != null)
-// throw new ConfigurationException(String.format("Cannot drop table '%s' in keyspace '%s'.", columnFamily(), keyspace()));
-//
-// throw new ConfigurationException(String.format("Cannot drop non existing materialized view '%s' in keyspace '%s'.", columnFamily(), keyspace()));
-// }
-//
-// CFMetaData baseCfm = Schema.instance.getCFMetaData(view.baseTableId);
-// if (baseCfm == null)
-// {
-// if (ifExists)
-// throw new ConfigurationException(String.format("Cannot drop materialized view '%s' in keyspace '%s' without base CF.", columnFamily(), keyspace()));
-// else
-// throw new InvalidRequestException(String.format("View '%s' could not be found in any of the tables of keyspace '%s'", cfName, keyspace()));
-// }
-
- MigrationManager.announceViewDrop(keyspace(), columnFamily(), isLocalOnly);
- return new Event.SchemaChange(Event.SchemaChange.Change.DROPPED, Event.SchemaChange.Target.TABLE, keyspace(), columnFamily());
- }
- catch (ConfigurationException e)
- {
- if (ifExists)
- return null;
- throw e;
- }
- }
-}
diff --git a/src/java/org/apache/cassandra/cql3/statements/GrantPermissionsStatement.java b/src/java/org/apache/cassandra/cql3/statements/GrantPermissionsStatement.java
index 06a53e2..3db20e3 100644
--- a/src/java/org/apache/cassandra/cql3/statements/GrantPermissionsStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/GrantPermissionsStatement.java
@@ -19,6 +19,8 @@
import java.util.Set;
+import org.apache.cassandra.audit.AuditLogContext;
+import org.apache.cassandra.audit.AuditLogEntryType;
import org.apache.cassandra.auth.IResource;
import org.apache.cassandra.auth.Permission;
import org.apache.cassandra.config.DatabaseDescriptor;
@@ -40,4 +42,12 @@
DatabaseDescriptor.getAuthorizer().grant(state.getUser(), permissions, resource, grantee);
return null;
}
+
+ @Override
+ public AuditLogContext getAuditLogContext()
+ {
+ String keyspace = resource.hasParent() ? resource.getParent().getName() : resource.getName();
+ return new AuditLogContext(AuditLogEntryType.GRANT, keyspace, resource.getName());
+ }
+
}
diff --git a/src/java/org/apache/cassandra/cql3/statements/GrantRoleStatement.java b/src/java/org/apache/cassandra/cql3/statements/GrantRoleStatement.java
index a22a99a..d6240c5 100644
--- a/src/java/org/apache/cassandra/cql3/statements/GrantRoleStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/GrantRoleStatement.java
@@ -17,12 +17,16 @@
*/
package org.apache.cassandra.cql3.statements;
+import org.apache.cassandra.audit.AuditLogContext;
+import org.apache.cassandra.audit.AuditLogEntryType;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.RoleName;
import org.apache.cassandra.exceptions.RequestExecutionException;
import org.apache.cassandra.exceptions.RequestValidationException;
import org.apache.cassandra.service.ClientState;
import org.apache.cassandra.transport.messages.ResultMessage;
+import org.apache.commons.lang3.builder.ToStringBuilder;
+import org.apache.commons.lang3.builder.ToStringStyle;
public class GrantRoleStatement extends RoleManagementStatement
{
@@ -36,4 +40,16 @@
DatabaseDescriptor.getRoleManager().grantRole(state.getUser(), role, grantee);
return null;
}
+
+ @Override
+ public String toString()
+ {
+ return ToStringBuilder.reflectionToString(this, ToStringStyle.SHORT_PREFIX_STYLE);
+ }
+
+ @Override
+ public AuditLogContext getAuditLogContext()
+ {
+ return new AuditLogContext(AuditLogEntryType.GRANT);
+ }
}
diff --git a/src/java/org/apache/cassandra/cql3/statements/IndexPropDefs.java b/src/java/org/apache/cassandra/cql3/statements/IndexPropDefs.java
deleted file mode 100644
index b8ce7ec..0000000
--- a/src/java/org/apache/cassandra/cql3/statements/IndexPropDefs.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.cql3.statements;
-
-import java.util.*;
-
-import org.apache.cassandra.exceptions.InvalidRequestException;
-import org.apache.cassandra.exceptions.RequestValidationException;
-import org.apache.cassandra.exceptions.SyntaxException;
-
-public class IndexPropDefs extends PropertyDefinitions
-{
- public static final String KW_OPTIONS = "options";
-
- public static final Set<String> keywords = new HashSet<>();
- public static final Set<String> obsoleteKeywords = new HashSet<>();
-
- public boolean isCustom;
- public String customClass;
-
- static
- {
- keywords.add(KW_OPTIONS);
- }
-
- public void validate() throws RequestValidationException
- {
- validate(keywords, obsoleteKeywords);
-
- if (isCustom && customClass == null)
- throw new InvalidRequestException("CUSTOM index requires specifiying the index class");
-
- if (!isCustom && customClass != null)
- throw new InvalidRequestException("Cannot specify index class for a non-CUSTOM index");
-
- if (!isCustom && !properties.isEmpty())
- throw new InvalidRequestException("Cannot specify options for a non-CUSTOM index");
-
- if (getRawOptions().containsKey(IndexTarget.CUSTOM_INDEX_OPTION_NAME))
- throw new InvalidRequestException(String.format("Cannot specify %s as a CUSTOM option",
- IndexTarget.CUSTOM_INDEX_OPTION_NAME));
-
- if (getRawOptions().containsKey(IndexTarget.TARGET_OPTION_NAME))
- throw new InvalidRequestException(String.format("Cannot specify %s as a CUSTOM option",
- IndexTarget.TARGET_OPTION_NAME));
-
- }
-
- public Map<String, String> getRawOptions() throws SyntaxException
- {
- Map<String, String> options = getMap(KW_OPTIONS);
- return options == null ? Collections.<String, String>emptyMap() : options;
- }
-
- public Map<String, String> getOptions() throws SyntaxException
- {
- Map<String, String> options = new HashMap<>(getRawOptions());
- options.put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, customClass);
- return options;
- }
-}
diff --git a/src/java/org/apache/cassandra/cql3/statements/IndexTarget.java b/src/java/org/apache/cassandra/cql3/statements/IndexTarget.java
deleted file mode 100644
index 84af273..0000000
--- a/src/java/org/apache/cassandra/cql3/statements/IndexTarget.java
+++ /dev/null
@@ -1,150 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.cql3.statements;
-
-import java.util.regex.Pattern;
-
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
-import org.apache.cassandra.cql3.ColumnIdentifier;
-
-public class IndexTarget
-{
- public static final String TARGET_OPTION_NAME = "target";
- public static final String CUSTOM_INDEX_OPTION_NAME = "class_name";
-
- /**
- * The name of the option used to specify that the index is on the collection keys.
- */
- public static final String INDEX_KEYS_OPTION_NAME = "index_keys";
-
- /**
- * The name of the option used to specify that the index is on the collection (map) entries.
- */
- public static final String INDEX_ENTRIES_OPTION_NAME = "index_keys_and_values";
-
- /**
- * Regex for *unquoted* column names, anything which does not match this pattern must be a quoted name
- */
- private static final Pattern COLUMN_IDENTIFIER_PATTERN = Pattern.compile("[a-z_0-9]+");
-
- public final ColumnIdentifier column;
- public final boolean quoteName;
- public final Type type;
-
- public IndexTarget(ColumnIdentifier column, Type type)
- {
- this.column = column;
- this.type = type;
-
- // if the column name contains anything other than lower case alphanumerics
- // or underscores, then it must be quoted when included in the target string
- quoteName = !COLUMN_IDENTIFIER_PATTERN.matcher(column.toString()).matches();
- }
-
- public String asCqlString(CFMetaData cfm)
- {
- if (!cfm.getColumnDefinition(column).type.isCollection())
- return column.toCQLString();
-
- return String.format("%s(%s)", type.toString(), column.toCQLString());
- }
-
- public static class Raw
- {
- private final ColumnDefinition.Raw column;
- private final Type type;
-
- private Raw(ColumnDefinition.Raw column, Type type)
- {
- this.column = column;
- this.type = type;
- }
-
- public static Raw simpleIndexOn(ColumnDefinition.Raw c)
- {
- return new Raw(c, Type.SIMPLE);
- }
-
- public static Raw valuesOf(ColumnDefinition.Raw c)
- {
- return new Raw(c, Type.VALUES);
- }
-
- public static Raw keysOf(ColumnDefinition.Raw c)
- {
- return new Raw(c, Type.KEYS);
- }
-
- public static Raw keysAndValuesOf(ColumnDefinition.Raw c)
- {
- return new Raw(c, Type.KEYS_AND_VALUES);
- }
-
- public static Raw fullCollection(ColumnDefinition.Raw c)
- {
- return new Raw(c, Type.FULL);
- }
-
- public IndexTarget prepare(CFMetaData cfm)
- {
- // Until we've prepared the target column, we can't be certain about the target type
- // because (for backwards compatibility) an index on a collection's values uses the
- // same syntax as an index on a regular column (i.e. the 'values' in
- // 'CREATE INDEX on table(values(collection));' is optional). So we correct the target type
- // when the target column is a collection & the target type is SIMPLE.
- ColumnDefinition columnDef = column.prepare(cfm);
- Type actualType = (type == Type.SIMPLE && columnDef.type.isCollection()) ? Type.VALUES : type;
- return new IndexTarget(columnDef.name, actualType);
- }
- }
-
- public static enum Type
- {
- VALUES, KEYS, KEYS_AND_VALUES, FULL, SIMPLE;
-
- public String toString()
- {
- switch (this)
- {
- case KEYS: return "keys";
- case KEYS_AND_VALUES: return "entries";
- case FULL: return "full";
- case VALUES: return "values";
- case SIMPLE: return "";
- default: return "";
- }
- }
-
- public static Type fromString(String s)
- {
- if ("".equals(s))
- return SIMPLE;
- else if ("values".equals(s))
- return VALUES;
- else if ("keys".equals(s))
- return KEYS;
- else if ("entries".equals(s))
- return KEYS_AND_VALUES;
- else if ("full".equals(s))
- return FULL;
-
- throw new AssertionError("Unrecognized index target type " + s);
- }
- }
-}
diff --git a/src/java/org/apache/cassandra/cql3/statements/KeyspaceAttributes.java b/src/java/org/apache/cassandra/cql3/statements/KeyspaceAttributes.java
deleted file mode 100644
index db6b0d6..0000000
--- a/src/java/org/apache/cassandra/cql3/statements/KeyspaceAttributes.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.cql3.statements;
-
-import java.util.*;
-
-import com.google.common.collect.ImmutableSet;
-
-import org.apache.cassandra.schema.KeyspaceParams;
-import org.apache.cassandra.schema.KeyspaceParams.Option;
-import org.apache.cassandra.schema.ReplicationParams;
-
-public final class KeyspaceAttributes extends PropertyDefinitions
-{
- private static final Set<String> validKeywords;
- private static final Set<String> obsoleteKeywords;
-
- static
- {
- ImmutableSet.Builder<String> validBuilder = ImmutableSet.builder();
- for (Option option : Option.values())
- validBuilder.add(option.toString());
- validKeywords = validBuilder.build();
- obsoleteKeywords = ImmutableSet.of();
- }
-
- public void validate()
- {
- validate(validKeywords, obsoleteKeywords);
- }
-
- public String getReplicationStrategyClass()
- {
- return getAllReplicationOptions().get(ReplicationParams.CLASS);
- }
-
- public Map<String, String> getReplicationOptions()
- {
- Map<String, String> replication = new HashMap<>(getAllReplicationOptions());
- replication.remove(ReplicationParams.CLASS);
- return replication;
- }
-
- public Map<String, String> getAllReplicationOptions()
- {
- Map<String, String> replication = getMap(Option.REPLICATION.toString());
- return replication == null
- ? Collections.emptyMap()
- : replication;
- }
-
- public KeyspaceParams asNewKeyspaceParams()
- {
- boolean durableWrites = getBoolean(Option.DURABLE_WRITES.toString(), KeyspaceParams.DEFAULT_DURABLE_WRITES);
- return KeyspaceParams.create(durableWrites, getAllReplicationOptions());
- }
-
- public KeyspaceParams asAlteredKeyspaceParams(KeyspaceParams previous)
- {
- boolean durableWrites = getBoolean(Option.DURABLE_WRITES.toString(), previous.durableWrites);
- ReplicationParams replication = getReplicationStrategyClass() == null
- ? previous.replication
- : ReplicationParams.fromMap(getAllReplicationOptions());
- return new KeyspaceParams(durableWrites, replication);
- }
-}
diff --git a/src/java/org/apache/cassandra/cql3/statements/ListPermissionsStatement.java b/src/java/org/apache/cassandra/cql3/statements/ListPermissionsStatement.java
index b8f2f92..4b5aa60 100644
--- a/src/java/org/apache/cassandra/cql3/statements/ListPermissionsStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/ListPermissionsStatement.java
@@ -19,9 +19,11 @@
import java.util.*;
+import org.apache.cassandra.audit.AuditLogContext;
+import org.apache.cassandra.audit.AuditLogEntryType;
import org.apache.cassandra.auth.*;
import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.config.SchemaConstants;
+import org.apache.cassandra.schema.SchemaConstants;
import org.apache.cassandra.cql3.*;
import org.apache.cassandra.db.marshal.UTF8Type;
import org.apache.cassandra.exceptions.InvalidRequestException;
@@ -29,6 +31,8 @@
import org.apache.cassandra.exceptions.RequestValidationException;
import org.apache.cassandra.service.ClientState;
import org.apache.cassandra.transport.messages.ResultMessage;
+import org.apache.commons.lang3.builder.ToStringBuilder;
+import org.apache.commons.lang3.builder.ToStringStyle;
public class ListPermissionsStatement extends AuthorizationStatement
{
@@ -76,7 +80,7 @@
throw new InvalidRequestException(String.format("%s doesn't exist", grantee));
}
- public void checkAccess(ClientState state)
+ public void authorize(ClientState state)
{
// checked in validate
}
@@ -118,7 +122,8 @@
if (details.isEmpty())
return new ResultMessage.Void();
- ResultSet result = new ResultSet(metadata);
+ ResultSet.ResultMetadata resultMetadata = new ResultSet.ResultMetadata(metadata);
+ ResultSet result = new ResultSet(resultMetadata);
for (PermissionDetails pd : details)
{
result.addColumnValue(UTF8Type.instance.decompose(pd.grantee));
@@ -128,4 +133,16 @@
}
return new ResultMessage.Rows(result);
}
+
+ @Override
+ public String toString()
+ {
+ return ToStringBuilder.reflectionToString(this, ToStringStyle.SHORT_PREFIX_STYLE);
+ }
+
+ @Override
+ public AuditLogContext getAuditLogContext()
+ {
+ return new AuditLogContext(AuditLogEntryType.LIST_PERMISSIONS);
+ }
}
diff --git a/src/java/org/apache/cassandra/cql3/statements/ListRolesStatement.java b/src/java/org/apache/cassandra/cql3/statements/ListRolesStatement.java
index 3fee57a..8a75f8a 100644
--- a/src/java/org/apache/cassandra/cql3/statements/ListRolesStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/ListRolesStatement.java
@@ -24,9 +24,11 @@
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
+import org.apache.cassandra.audit.AuditLogContext;
+import org.apache.cassandra.audit.AuditLogEntryType;
import org.apache.cassandra.auth.*;
import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.config.SchemaConstants;
+import org.apache.cassandra.schema.SchemaConstants;
import org.apache.cassandra.cql3.*;
import org.apache.cassandra.db.marshal.BooleanType;
import org.apache.cassandra.db.marshal.MapType;
@@ -34,6 +36,8 @@
import org.apache.cassandra.exceptions.*;
import org.apache.cassandra.service.ClientState;
import org.apache.cassandra.transport.messages.ResultMessage;
+import org.apache.commons.lang3.builder.ToStringBuilder;
+import org.apache.commons.lang3.builder.ToStringStyle;
public class ListRolesStatement extends AuthorizationStatement
{
@@ -46,7 +50,8 @@
ImmutableList.of(new ColumnSpecification(KS, CF, new ColumnIdentifier("role", true), UTF8Type.instance),
new ColumnSpecification(KS, CF, new ColumnIdentifier("super", true), BooleanType.instance),
new ColumnSpecification(KS, CF, new ColumnIdentifier("login", true), BooleanType.instance),
- new ColumnSpecification(KS, CF, new ColumnIdentifier("options", true), optionsType));
+ new ColumnSpecification(KS, CF, new ColumnIdentifier("options", true), optionsType),
+ new ColumnSpecification(KS, CF, new ColumnIdentifier("datacenters", true), UTF8Type.instance));
private final RoleResource grantee;
private final boolean recursive;
@@ -70,7 +75,7 @@
throw new InvalidRequestException(String.format("%s doesn't exist", grantee));
}
- public void checkAccess(ClientState state) throws InvalidRequestException
+ public void authorize(ClientState state) throws InvalidRequestException
{
}
@@ -112,16 +117,31 @@
// overridden in ListUsersStatement to include legacy metadata
protected ResultMessage formatResults(List<RoleResource> sortedRoles)
{
- ResultSet result = new ResultSet(metadata);
+ ResultSet.ResultMetadata resultMetadata = new ResultSet.ResultMetadata(metadata);
+ ResultSet result = new ResultSet(resultMetadata);
IRoleManager roleManager = DatabaseDescriptor.getRoleManager();
+ INetworkAuthorizer networkAuthorizer = DatabaseDescriptor.getNetworkAuthorizer();
for (RoleResource role : sortedRoles)
{
result.addColumnValue(UTF8Type.instance.decompose(role.getRoleName()));
result.addColumnValue(BooleanType.instance.decompose(roleManager.isSuper(role)));
result.addColumnValue(BooleanType.instance.decompose(roleManager.canLogin(role)));
result.addColumnValue(optionsType.decompose(roleManager.getCustomOptions(role)));
+ result.addColumnValue(UTF8Type.instance.decompose(networkAuthorizer.authorize(role).toString()));
}
return new ResultMessage.Rows(result);
}
+
+ @Override
+ public String toString()
+ {
+ return ToStringBuilder.reflectionToString(this, ToStringStyle.SHORT_PREFIX_STYLE);
+ }
+
+ @Override
+ public AuditLogContext getAuditLogContext()
+ {
+ return new AuditLogContext(AuditLogEntryType.LIST_ROLES);
+ }
}
diff --git a/src/java/org/apache/cassandra/cql3/statements/ListUsersStatement.java b/src/java/org/apache/cassandra/cql3/statements/ListUsersStatement.java
index 0101363..be3e587 100644
--- a/src/java/org/apache/cassandra/cql3/statements/ListUsersStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/ListUsersStatement.java
@@ -23,13 +23,15 @@
import org.apache.cassandra.auth.*;
import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.config.SchemaConstants;
+import org.apache.cassandra.schema.SchemaConstants;
import org.apache.cassandra.cql3.ColumnIdentifier;
import org.apache.cassandra.cql3.ColumnSpecification;
import org.apache.cassandra.cql3.ResultSet;
import org.apache.cassandra.db.marshal.BooleanType;
import org.apache.cassandra.db.marshal.UTF8Type;
import org.apache.cassandra.transport.messages.ResultMessage;
+import org.apache.commons.lang3.builder.ToStringBuilder;
+import org.apache.commons.lang3.builder.ToStringStyle;
public class ListUsersStatement extends ListRolesStatement
{
@@ -39,21 +41,32 @@
private static final List<ColumnSpecification> metadata =
ImmutableList.of(new ColumnSpecification(KS, CF, new ColumnIdentifier("name", true), UTF8Type.instance),
- new ColumnSpecification(KS, CF, new ColumnIdentifier("super", true), BooleanType.instance));
+ new ColumnSpecification(KS, CF, new ColumnIdentifier("super", true), BooleanType.instance),
+ new ColumnSpecification(KS, CF, new ColumnIdentifier("datacenters", true), UTF8Type.instance));
@Override
protected ResultMessage formatResults(List<RoleResource> sortedRoles)
{
- ResultSet result = new ResultSet(metadata);
+ ResultSet.ResultMetadata resultMetadata = new ResultSet.ResultMetadata(metadata);
+ ResultSet result = new ResultSet(resultMetadata);
IRoleManager roleManager = DatabaseDescriptor.getRoleManager();
+ INetworkAuthorizer networkAuthorizer = DatabaseDescriptor.getNetworkAuthorizer();
for (RoleResource role : sortedRoles)
{
if (!roleManager.canLogin(role))
continue;
result.addColumnValue(UTF8Type.instance.decompose(role.getRoleName()));
result.addColumnValue(BooleanType.instance.decompose(Roles.hasSuperuserStatus(role)));
+ result.addColumnValue(UTF8Type.instance.decompose(networkAuthorizer.authorize(role).toString()));
}
+
return new ResultMessage.Rows(result);
}
+
+ @Override
+ public String toString()
+ {
+ return ToStringBuilder.reflectionToString(this, ToStringStyle.SHORT_PREFIX_STYLE);
+ }
}
diff --git a/src/java/org/apache/cassandra/cql3/statements/ModificationStatement.java b/src/java/org/apache/cassandra/cql3/statements/ModificationStatement.java
index 8a896e9..a8367f0 100644
--- a/src/java/org/apache/cassandra/cql3/statements/ModificationStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/ModificationStatement.java
@@ -20,20 +20,25 @@
import java.nio.ByteBuffer;
import java.util.*;
+import com.google.common.collect.ImmutableList;
import com.google.common.collect.Iterables;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.auth.Permission;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
-import org.apache.cassandra.config.Schema;
-import org.apache.cassandra.config.ColumnDefinition.Raw;
-import org.apache.cassandra.config.ViewDefinition;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.schema.ViewMetadata;
import org.apache.cassandra.cql3.*;
+import org.apache.cassandra.cql3.conditions.ColumnCondition;
+import org.apache.cassandra.cql3.conditions.ColumnConditions;
+import org.apache.cassandra.cql3.conditions.Conditions;
import org.apache.cassandra.cql3.functions.Function;
import org.apache.cassandra.cql3.restrictions.StatementRestrictions;
+import org.apache.cassandra.cql3.selection.ResultSetBuilder;
import org.apache.cassandra.cql3.selection.Selection;
+import org.apache.cassandra.cql3.selection.Selection.Selectors;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.filter.*;
import org.apache.cassandra.db.marshal.BooleanType;
@@ -45,10 +50,10 @@
import org.apache.cassandra.service.QueryState;
import org.apache.cassandra.service.StorageProxy;
import org.apache.cassandra.service.paxos.Commit;
-import org.apache.cassandra.thrift.ThriftValidation;
import org.apache.cassandra.transport.messages.ResultMessage;
import org.apache.cassandra.triggers.TriggerExecutor;
import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.MD5Digest;
import org.apache.cassandra.utils.Pair;
import org.apache.cassandra.utils.UUIDGen;
@@ -62,6 +67,8 @@
{
protected static final Logger logger = LoggerFactory.getLogger(ModificationStatement.class);
+ private final static MD5Digest EMPTY_HASH = MD5Digest.wrap(new byte[] {});
+
public static final String CUSTOM_EXPRESSIONS_NOT_ALLOWED =
"Custom index expressions cannot be used in WHERE clauses for UPDATE or DELETE statements";
@@ -69,33 +76,34 @@
protected final StatementType type;
- private final int boundTerms;
- public final CFMetaData cfm;
+ protected final VariableSpecifications bindVariables;
+
+ public final TableMetadata metadata;
private final Attributes attrs;
private final StatementRestrictions restrictions;
private final Operations operations;
- private final PartitionColumns updatedColumns;
+ private final RegularAndStaticColumns updatedColumns;
private final Conditions conditions;
- private final PartitionColumns conditionColumns;
+ private final RegularAndStaticColumns conditionColumns;
- private final PartitionColumns requiresRead;
+ private final RegularAndStaticColumns requiresRead;
public ModificationStatement(StatementType type,
- int boundTerms,
- CFMetaData cfm,
+ VariableSpecifications bindVariables,
+ TableMetadata metadata,
Operations operations,
StatementRestrictions restrictions,
Conditions conditions,
Attributes attrs)
{
this.type = type;
- this.boundTerms = boundTerms;
- this.cfm = cfm;
+ this.bindVariables = bindVariables;
+ this.metadata = metadata;
this.restrictions = restrictions;
this.operations = operations;
this.conditions = conditions;
@@ -103,17 +111,17 @@
if (!conditions.isEmpty())
{
- checkFalse(cfm.isCounter(), "Conditional updates are not supported on counter tables");
+ checkFalse(metadata.isCounter(), "Conditional updates are not supported on counter tables");
checkFalse(attrs.isTimestampSet(), "Cannot provide custom timestamp for conditional updates");
}
- PartitionColumns.Builder conditionColumnsBuilder = PartitionColumns.builder();
- Iterable<ColumnDefinition> columns = conditions.getColumns();
+ RegularAndStaticColumns.Builder conditionColumnsBuilder = RegularAndStaticColumns.builder();
+ Iterable<ColumnMetadata> columns = conditions.getColumns();
if (columns != null)
conditionColumnsBuilder.addAll(columns);
- PartitionColumns.Builder updatedColumnsBuilder = PartitionColumns.builder();
- PartitionColumns.Builder requiresReadBuilder = PartitionColumns.builder();
+ RegularAndStaticColumns.Builder updatedColumnsBuilder = RegularAndStaticColumns.builder();
+ RegularAndStaticColumns.Builder requiresReadBuilder = RegularAndStaticColumns.builder();
for (Operation operation : operations)
{
updatedColumnsBuilder.add(operation.column);
@@ -126,19 +134,32 @@
}
}
- PartitionColumns modifiedColumns = updatedColumnsBuilder.build();
+ RegularAndStaticColumns modifiedColumns = updatedColumnsBuilder.build();
// Compact tables have not row marker. So if we don't actually update any particular column,
// this means that we're only updating the PK, which we allow if only those were declared in
// the definition. In that case however, we do went to write the compactValueColumn (since again
// we can't use a "row marker") so add it automatically.
- if (cfm.isCompactTable() && modifiedColumns.isEmpty() && updatesRegularRows())
- modifiedColumns = cfm.partitionColumns();
+ if (metadata.isCompactTable() && modifiedColumns.isEmpty() && updatesRegularRows())
+ modifiedColumns = metadata.regularAndStaticColumns();
this.updatedColumns = modifiedColumns;
this.conditionColumns = conditionColumnsBuilder.build();
this.requiresRead = requiresReadBuilder.build();
}
+ @Override
+ public List<ColumnSpecification> getBindVariables()
+ {
+ return bindVariables.getBindVariables();
+ }
+
+ @Override
+ public short[] getPartitionKeyBindVariableIndexes()
+ {
+ return bindVariables.getPartitionKeyBindVariableIndexes(metadata);
+ }
+
+ @Override
public Iterable<Function> getFunctions()
{
List<Function> functions = new ArrayList<>();
@@ -154,6 +175,11 @@
conditions.addFunctionsTo(functions);
}
+ public TableMetadata metadata()
+ {
+ return metadata;
+ }
+
/*
* May be used by QueryHandler implementations
*/
@@ -162,33 +188,33 @@
return restrictions;
}
- public abstract void addUpdateForKey(PartitionUpdate update, Clustering clustering, UpdateParameters params);
+ public abstract void addUpdateForKey(PartitionUpdate.Builder updateBuilder, Clustering clustering, UpdateParameters params);
- public abstract void addUpdateForKey(PartitionUpdate update, Slice slice, UpdateParameters params);
-
- public int getBoundTerms()
- {
- return boundTerms;
- }
+ public abstract void addUpdateForKey(PartitionUpdate.Builder updateBuilder, Slice slice, UpdateParameters params);
public String keyspace()
{
- return cfm.ksName;
+ return metadata.keyspace;
}
public String columnFamily()
{
- return cfm.cfName;
+ return metadata.name;
}
public boolean isCounter()
{
- return cfm.isCounter();
+ return metadata().isCounter();
}
public boolean isView()
{
- return cfm.isView();
+ return metadata().isView();
+ }
+
+ public boolean isVirtual()
+ {
+ return metadata().isVirtual();
}
public long getTimestamp(long now, QueryOptions options) throws InvalidRequestException
@@ -203,31 +229,31 @@
public int getTimeToLive(QueryOptions options) throws InvalidRequestException
{
- return attrs.getTimeToLive(options, cfm);
+ return attrs.getTimeToLive(options, metadata);
}
- public void checkAccess(ClientState state) throws InvalidRequestException, UnauthorizedException
+ public void authorize(ClientState state) throws InvalidRequestException, UnauthorizedException
{
- state.hasColumnFamilyAccess(cfm, Permission.MODIFY);
+ state.ensureTablePermission(metadata, Permission.MODIFY);
// CAS updates can be used to simulate a SELECT query, so should require Permission.SELECT as well.
if (hasConditions())
- state.hasColumnFamilyAccess(cfm, Permission.SELECT);
+ state.ensureTablePermission(metadata, Permission.SELECT);
// MV updates need to get the current state from the table, and might update the views
// Require Permission.SELECT on the base table, and Permission.MODIFY on the views
- Iterator<ViewDefinition> views = View.findAll(keyspace(), columnFamily()).iterator();
+ Iterator<ViewMetadata> views = View.findAll(keyspace(), columnFamily()).iterator();
if (views.hasNext())
{
- state.hasColumnFamilyAccess(cfm, Permission.SELECT);
+ state.ensureTablePermission(metadata, Permission.SELECT);
do
{
- state.hasColumnFamilyAccess(views.next().metadata, Permission.MODIFY);
+ state.ensureTablePermission(views.next().metadata, Permission.MODIFY);
} while (views.hasNext());
}
for (Function function : getFunctions())
- state.ensureHasPermission(Permission.EXECUTE, function);
+ state.ensurePermission(Permission.EXECUTE, function);
}
public void validate(ClientState state) throws InvalidRequestException
@@ -236,14 +262,16 @@
checkFalse(isCounter() && attrs.isTimestampSet(), "Cannot provide custom timestamp for counter updates");
checkFalse(isCounter() && attrs.isTimeToLiveSet(), "Cannot provide custom TTL for counter updates");
checkFalse(isView(), "Cannot directly modify a materialized view");
+ checkFalse(isVirtual() && attrs.isTimeToLiveSet(), "Expiring columns are not supported by virtual tables");
+ checkFalse(isVirtual() && hasConditions(), "Conditional updates are not supported by virtual tables");
}
- public PartitionColumns updatedColumns()
+ public RegularAndStaticColumns updatedColumns()
{
return updatedColumns;
}
- public PartitionColumns conditionColumns()
+ public RegularAndStaticColumns conditionColumns()
{
return conditionColumns;
}
@@ -255,7 +283,7 @@
// columns is if we set some static columns, and in that case no clustering
// columns should be given. So in practice, it's enough to check if we have
// either the table has no clustering or if it has at least one of them set.
- return cfm.clusteringColumns().isEmpty() || restrictions.hasClusteringColumnsRestrictions();
+ return metadata().clusteringColumns().isEmpty() || restrictions.hasClusteringColumnsRestrictions();
}
public boolean updatesStaticRow()
@@ -278,7 +306,7 @@
return operations;
}
- public Iterable<ColumnDefinition> getColumnsWithConditions()
+ public Iterable<ColumnMetadata> getColumnsWithConditions()
{
return conditions.getColumns();
}
@@ -307,7 +335,7 @@
throws InvalidRequestException
{
if (appliesOnlyToStaticColumns() && !restrictions.hasClusteringColumnsRestrictions())
- return FBUtilities.singleton(CBuilder.STATIC_BUILDER.build(), cfm.comparator);
+ return FBUtilities.singleton(CBuilder.STATIC_BUILDER.build(), metadata().comparator);
return restrictions.getClusteringColumns(options);
}
@@ -347,6 +375,7 @@
DataLimits limits,
boolean local,
ConsistencyLevel cl,
+ int nowInSeconds,
long queryStartNanoTime)
{
if (!requiresRead())
@@ -362,14 +391,13 @@
}
List<SinglePartitionReadCommand> commands = new ArrayList<>(partitionKeys.size());
- int nowInSec = FBUtilities.nowInSeconds();
for (ByteBuffer key : partitionKeys)
- commands.add(SinglePartitionReadCommand.create(cfm,
- nowInSec,
+ commands.add(SinglePartitionReadCommand.create(metadata(),
+ nowInSeconds,
ColumnFilter.selection(this.requiresRead),
RowFilter.NONE,
limits,
- cfm.decorateKey(key),
+ metadata().partitioner.decorateKey(key),
filter));
SinglePartitionReadCommand.Group group = new SinglePartitionReadCommand.Group(commands, DataLimits.NONE);
@@ -428,21 +456,28 @@
private ResultMessage executeWithoutCondition(QueryState queryState, QueryOptions options, long queryStartNanoTime)
throws RequestExecutionException, RequestValidationException
{
+ if (isVirtual())
+ return executeInternalWithoutCondition(queryState, options, queryStartNanoTime);
+
ConsistencyLevel cl = options.getConsistency();
if (isCounter())
- cl.validateCounterForWrite(cfm);
+ cl.validateCounterForWrite(metadata());
else
- cl.validateForWrite(cfm.ksName);
+ cl.validateForWrite(metadata.keyspace);
- Collection<? extends IMutation> mutations = getMutations(options, false, options.getTimestamp(queryState), queryStartNanoTime);
+ List<? extends IMutation> mutations =
+ getMutations(options,
+ false,
+ options.getTimestamp(queryState),
+ options.getNowInSeconds(queryState),
+ queryStartNanoTime);
if (!mutations.isEmpty())
StorageProxy.mutateWithTriggers(mutations, cl, false, queryStartNanoTime);
return null;
}
- public ResultMessage executeWithCondition(QueryState queryState, QueryOptions options, long queryStartNanoTime)
- throws RequestExecutionException, RequestValidationException
+ private ResultMessage executeWithCondition(QueryState queryState, QueryOptions options, long queryStartNanoTime)
{
CQL3CasRequest request = makeCasRequest(queryState, options);
@@ -453,9 +488,10 @@
options.getSerialConsistency(),
options.getConsistency(),
queryState.getClientState(),
+ options.getNowInSeconds(queryState),
queryStartNanoTime))
{
- return new ResultMessage.Rows(buildCasResultSet(result, options));
+ return new ResultMessage.Rows(buildCasResultSet(result, queryState, options));
}
}
@@ -467,18 +503,19 @@
"IN on the partition key is not supported with conditional %s",
type.isUpdate()? "updates" : "deletions");
- DecoratedKey key = cfm.decorateKey(keys.get(0));
- long now = options.getTimestamp(queryState);
+ DecoratedKey key = metadata().partitioner.decorateKey(keys.get(0));
+ long timestamp = options.getTimestamp(queryState);
+ int nowInSeconds = options.getNowInSeconds(queryState);
checkFalse(restrictions.clusteringKeyRestrictionsHasIN(),
"IN on the clustering key columns is not supported with conditional %s",
type.isUpdate()? "updates" : "deletions");
Clustering clustering = Iterables.getOnlyElement(createClustering(options));
- CQL3CasRequest request = new CQL3CasRequest(cfm, key, false, conditionColumns(), updatesRegularRows(), updatesStaticRow());
+ CQL3CasRequest request = new CQL3CasRequest(metadata(), key, conditionColumns(), updatesRegularRows(), updatesStaticRow());
addConditions(clustering, request, options);
- request.addRowUpdate(clustering, this, options, now);
+ request.addRowUpdate(clustering, this, options, timestamp, nowInSeconds);
return request;
}
@@ -488,22 +525,39 @@
conditions.addConditionsTo(request, clustering, options);
}
- private ResultSet buildCasResultSet(RowIterator partition, QueryOptions options) throws InvalidRequestException
+ private static ResultSet.ResultMetadata buildCASSuccessMetadata(String ksName, String cfName)
{
- return buildCasResultSet(keyspace(), columnFamily(), partition, getColumnsWithConditions(), false, options);
+ List<ColumnSpecification> specs = new ArrayList<>();
+ specs.add(casResultColumnSpecification(ksName, cfName));
+
+ return new ResultSet.ResultMetadata(EMPTY_HASH, specs);
}
- public static ResultSet buildCasResultSet(String ksName, String tableName, RowIterator partition, Iterable<ColumnDefinition> columnsWithConditions, boolean isBatch, QueryOptions options)
- throws InvalidRequestException
+ private static ColumnSpecification casResultColumnSpecification(String ksName, String cfName)
+ {
+ return new ColumnSpecification(ksName, cfName, CAS_RESULT_COLUMN, BooleanType.instance);
+ }
+
+ private ResultSet buildCasResultSet(RowIterator partition, QueryState state, QueryOptions options)
+ {
+ return buildCasResultSet(keyspace(), columnFamily(), partition, getColumnsWithConditions(), false, state, options);
+ }
+
+ static ResultSet buildCasResultSet(String ksName,
+ String tableName,
+ RowIterator partition,
+ Iterable<ColumnMetadata> columnsWithConditions,
+ boolean isBatch,
+ QueryState state,
+ QueryOptions options)
{
boolean success = partition == null;
- ColumnSpecification spec = new ColumnSpecification(ksName, tableName, CAS_RESULT_COLUMN, BooleanType.instance);
- ResultSet.ResultMetadata metadata = new ResultSet.ResultMetadata(Collections.singletonList(spec));
+ ResultSet.ResultMetadata metadata = buildCASSuccessMetadata(ksName, tableName);
List<List<ByteBuffer>> rows = Collections.singletonList(Collections.singletonList(BooleanType.instance.decompose(success)));
ResultSet rs = new ResultSet(metadata, rows);
- return success ? rs : merge(rs, buildCasFailureResultSet(partition, columnsWithConditions, isBatch, options));
+ return success ? rs : merge(rs, buildCasFailureResultSet(partition, columnsWithConditions, isBatch, options, options.getNowInSeconds(state)));
}
private static ResultSet merge(ResultSet left, ResultSet right)
@@ -526,82 +580,75 @@
row.addAll(right.rows.get(i));
rows.add(row);
}
- return new ResultSet(new ResultSet.ResultMetadata(specs), rows);
+ return new ResultSet(new ResultSet.ResultMetadata(EMPTY_HASH, specs), rows);
}
- private static ResultSet buildCasFailureResultSet(RowIterator partition, Iterable<ColumnDefinition> columnsWithConditions, boolean isBatch, QueryOptions options)
- throws InvalidRequestException
+ private static ResultSet buildCasFailureResultSet(RowIterator partition,
+ Iterable<ColumnMetadata> columnsWithConditions,
+ boolean isBatch,
+ QueryOptions options,
+ int nowInSeconds)
{
- CFMetaData cfm = partition.metadata();
+ TableMetadata metadata = partition.metadata();
Selection selection;
if (columnsWithConditions == null)
{
- selection = Selection.wildcard(cfm);
+ selection = Selection.wildcard(metadata, false);
}
else
{
// We can have multiple conditions on the same columns (for collections) so use a set
// to avoid duplicate, but preserve the order just to it follows the order of IF in the query in general
- Set<ColumnDefinition> defs = new LinkedHashSet<>();
+ Set<ColumnMetadata> defs = new LinkedHashSet<>();
// Adding the partition key for batches to disambiguate if the conditions span multipe rows (we don't add them outside
// of batches for compatibility sakes).
if (isBatch)
- {
- defs.addAll(cfm.partitionKeyColumns());
- defs.addAll(cfm.clusteringColumns());
- }
+ Iterables.addAll(defs, metadata.primaryKeyColumns());
+ Iterables.addAll(defs, columnsWithConditions);
+ selection = Selection.forColumns(metadata, new ArrayList<>(defs));
-
- if (cfm.isSuper() && cfm.isDense())
- {
- defs.add(cfm.superColumnValueColumn());
- }
- else
- {
- for (ColumnDefinition def : columnsWithConditions)
- defs.add(def);
- }
-
- selection = Selection.forColumns(cfm, new ArrayList<>(defs));
}
- Selection.ResultSetBuilder builder = selection.resultSetBuilder(options, false);
- SelectStatement.forSelection(cfm, selection).processPartition(partition,
- options,
- builder,
- FBUtilities.nowInSeconds());
+ Selectors selectors = selection.newSelectors(options);
+ ResultSetBuilder builder = new ResultSetBuilder(selection.getResultMetadata(), selectors);
+ SelectStatement.forSelection(metadata, selection)
+ .processPartition(partition, options, builder, nowInSeconds);
return builder.build();
}
- public ResultMessage executeInternal(QueryState queryState, QueryOptions options) throws RequestValidationException, RequestExecutionException
+ public ResultMessage executeLocally(QueryState queryState, QueryOptions options) throws RequestValidationException, RequestExecutionException
{
return hasConditions()
? executeInternalWithCondition(queryState, options)
: executeInternalWithoutCondition(queryState, options, System.nanoTime());
}
- public ResultMessage executeInternalWithoutCondition(QueryState queryState, QueryOptions options, long queryStartNanoTime) throws RequestValidationException, RequestExecutionException
+ public ResultMessage executeInternalWithoutCondition(QueryState queryState, QueryOptions options, long queryStartNanoTime)
+ throws RequestValidationException, RequestExecutionException
{
- for (IMutation mutation : getMutations(options, true, queryState.getTimestamp(), queryStartNanoTime))
+ long timestamp = options.getTimestamp(queryState);
+ int nowInSeconds = options.getNowInSeconds(queryState);
+ for (IMutation mutation : getMutations(options, true, timestamp, nowInSeconds, queryStartNanoTime))
mutation.apply();
return null;
}
- public ResultMessage executeInternalWithCondition(QueryState state, QueryOptions options) throws RequestValidationException, RequestExecutionException
+ public ResultMessage executeInternalWithCondition(QueryState state, QueryOptions options)
{
CQL3CasRequest request = makeCasRequest(state, options);
- try (RowIterator result = casInternal(request, state))
+
+ try (RowIterator result = casInternal(request, options.getTimestamp(state), options.getNowInSeconds(state)))
{
- return new ResultMessage.Rows(buildCasResultSet(result, options));
+ return new ResultMessage.Rows(buildCasResultSet(result, state, options));
}
}
- static RowIterator casInternal(CQL3CasRequest request, QueryState state)
+ static RowIterator casInternal(CQL3CasRequest request, long timestamp, int nowInSeconds)
{
- UUID ballot = UUIDGen.getTimeUUIDFromMicros(state.getTimestamp());
+ UUID ballot = UUIDGen.getTimeUUIDFromMicros(timestamp);
- SinglePartitionReadCommand readCommand = request.readCommand(FBUtilities.nowInSeconds());
+ SinglePartitionReadQuery readCommand = request.readCommand(nowInSeconds);
FilteredPartition current;
try (ReadExecutionController executionController = readCommand.executionController();
PartitionIterator iter = readCommand.executeInternal(executionController))
@@ -625,23 +672,26 @@
*
* @param options value for prepared statement markers
* @param local if true, any requests (for collections) performed by getMutation should be done locally only.
- * @param now the current timestamp in microseconds to use if no timestamp is user provided.
+ * @param timestamp the current timestamp in microseconds to use if no timestamp is user provided.
*
* @return list of the mutations
*/
- private Collection<? extends IMutation> getMutations(QueryOptions options, boolean local, long now, long queryStartNanoTime)
+ private List<? extends IMutation> getMutations(QueryOptions options,
+ boolean local,
+ long timestamp,
+ int nowInSeconds,
+ long queryStartNanoTime)
{
- UpdatesCollector collector = new UpdatesCollector(Collections.singletonMap(cfm.cfId, updatedColumns), 1);
- addUpdates(collector, options, local, now, queryStartNanoTime);
- collector.validateIndexedColumns();
-
+ UpdatesCollector collector = new SingleTableUpdatesCollector(metadata, updatedColumns, 1);
+ addUpdates(collector, options, local, timestamp, nowInSeconds, queryStartNanoTime);
return collector.toMutations();
}
final void addUpdates(UpdatesCollector collector,
QueryOptions options,
boolean local,
- long now,
+ long timestamp,
+ int nowInSeconds,
long queryStartNanoTime)
{
List<ByteBuffer> keys = buildPartitionKeyNames(options);
@@ -659,17 +709,18 @@
options,
DataLimits.NONE,
local,
- now,
+ timestamp,
+ nowInSeconds,
queryStartNanoTime);
for (ByteBuffer key : keys)
{
- ThriftValidation.validateKey(cfm, key);
- DecoratedKey dk = cfm.decorateKey(key);
+ Validation.validateKey(metadata(), key);
+ DecoratedKey dk = metadata().partitioner.decorateKey(key);
- PartitionUpdate upd = collector.getPartitionUpdate(cfm, dk, options.getConsistency());
+ PartitionUpdate.Builder updateBuilder = collector.getPartitionUpdateBuilder(metadata(), dk, options.getConsistency());
for (Slice slice : slices)
- addUpdateForKey(upd, slice, params);
+ addUpdateForKey(updateBuilder, slice, params);
}
}
else
@@ -680,32 +731,31 @@
if (restrictions.hasClusteringColumnsRestrictions() && clusterings.isEmpty())
return;
- UpdateParameters params = makeUpdateParameters(keys, clusterings, options, local, now, queryStartNanoTime);
+ UpdateParameters params = makeUpdateParameters(keys, clusterings, options, local, timestamp, nowInSeconds, queryStartNanoTime);
for (ByteBuffer key : keys)
{
- ThriftValidation.validateKey(cfm, key);
- DecoratedKey dk = cfm.decorateKey(key);
+ Validation.validateKey(metadata(), key);
+ DecoratedKey dk = metadata().partitioner.decorateKey(key);
- PartitionUpdate upd = collector.getPartitionUpdate(cfm, dk, options.getConsistency());
+ PartitionUpdate.Builder updateBuilder = collector.getPartitionUpdateBuilder(metadata(), dk, options.getConsistency());
if (!restrictions.hasClusteringColumnsRestrictions())
{
- addUpdateForKey(upd, Clustering.EMPTY, params);
+ addUpdateForKey(updateBuilder, Clustering.EMPTY, params);
}
else
{
for (Clustering clustering : clusterings)
{
- for (ByteBuffer c : clustering.getRawValues())
- {
- if (c != null && c.remaining() > FBUtilities.MAX_UNSIGNED_SHORT)
- throw new InvalidRequestException(String.format("Key length of %d is longer than maximum of %d",
- clustering.dataSize(),
- FBUtilities.MAX_UNSIGNED_SHORT));
- }
-
- addUpdateForKey(upd, clustering, params);
+ for (ByteBuffer c : clustering.getRawValues())
+ {
+ if (c != null && c.remaining() > FBUtilities.MAX_UNSIGNED_SHORT)
+ throw new InvalidRequestException(String.format("Key length of %d is longer than maximum of %d",
+ clustering.dataSize(),
+ FBUtilities.MAX_UNSIGNED_SHORT));
+ }
+ addUpdateForKey(updateBuilder, clustering, params);
}
}
}
@@ -724,7 +774,8 @@
NavigableSet<Clustering> clusterings,
QueryOptions options,
boolean local,
- long now,
+ long timestamp,
+ int nowInSeconds,
long queryStartNanoTime)
{
if (clusterings.contains(Clustering.STATIC_CLUSTERING))
@@ -733,7 +784,8 @@
options,
DataLimits.cqlLimits(1),
local,
- now,
+ timestamp,
+ nowInSeconds,
queryStartNanoTime);
return makeUpdateParameters(keys,
@@ -741,7 +793,8 @@
options,
DataLimits.NONE,
local,
- now,
+ timestamp,
+ nowInSeconds,
queryStartNanoTime);
}
@@ -750,19 +803,34 @@
QueryOptions options,
DataLimits limits,
boolean local,
- long now,
+ long timestamp,
+ int nowInSeconds,
long queryStartNanoTime)
{
// Some lists operation requires reading
- Map<DecoratedKey, Partition> lists = readRequiredLists(keys, filter, limits, local, options.getConsistency(), queryStartNanoTime);
- return new UpdateParameters(cfm, updatedColumns(), options, getTimestamp(now, options), getTimeToLive(options), lists);
+ Map<DecoratedKey, Partition> lists =
+ readRequiredLists(keys,
+ filter,
+ limits,
+ local,
+ options.getConsistency(),
+ nowInSeconds,
+ queryStartNanoTime);
+
+ return new UpdateParameters(metadata(),
+ updatedColumns(),
+ options,
+ getTimestamp(timestamp, options),
+ nowInSeconds,
+ getTimeToLive(options),
+ lists);
}
private Slices toSlices(SortedSet<ClusteringBound> startBounds, SortedSet<ClusteringBound> endBounds)
{
assert startBounds.size() == endBounds.size();
- Slices.Builder builder = new Slices.Builder(cfm.comparator);
+ Slices.Builder builder = new Slices.Builder(metadata().comparator);
Iterator<ClusteringBound> starts = startBounds.iterator();
Iterator<ClusteringBound> ends = endBounds.iterator();
@@ -770,7 +838,7 @@
while (starts.hasNext())
{
Slice slice = Slice.make(starts.next(), ends.next());
- if (!slice.isEmpty(cfm.comparator))
+ if (!slice.isEmpty(metadata().comparator))
{
builder.add(slice);
}
@@ -779,59 +847,54 @@
return builder.build();
}
- public static abstract class Parsed extends CFStatement
+ public static abstract class Parsed extends QualifiedStatement
{
protected final StatementType type;
private final Attributes.Raw attrs;
- private final List<Pair<ColumnDefinition.Raw, ColumnCondition.Raw>> conditions;
+ private final List<Pair<ColumnMetadata.Raw, ColumnCondition.Raw>> conditions;
private final boolean ifNotExists;
private final boolean ifExists;
- protected Parsed(CFName name,
+ protected Parsed(QualifiedName name,
StatementType type,
Attributes.Raw attrs,
- List<Pair<ColumnDefinition.Raw, ColumnCondition.Raw>> conditions,
+ List<Pair<ColumnMetadata.Raw, ColumnCondition.Raw>> conditions,
boolean ifNotExists,
boolean ifExists)
{
super(name);
this.type = type;
this.attrs = attrs;
- this.conditions = conditions == null ? Collections.<Pair<ColumnDefinition.Raw, ColumnCondition.Raw>>emptyList() : conditions;
+ this.conditions = conditions == null ? Collections.emptyList() : conditions;
this.ifNotExists = ifNotExists;
this.ifExists = ifExists;
}
- public ParsedStatement.Prepared prepare(ClientState clientState)
+ public ModificationStatement prepare(ClientState state)
{
- VariableSpecifications boundNames = getBoundVariables();
- ModificationStatement statement = prepare(boundNames, clientState);
- return new ParsedStatement.Prepared(statement, boundNames, boundNames.getPartitionKeyBindIndexes(statement.cfm));
+ return prepare(bindVariables);
}
- public ModificationStatement prepare(VariableSpecifications boundNames, ClientState clientState)
+ public ModificationStatement prepare(VariableSpecifications bindVariables)
{
- CFMetaData metadata = ThriftValidation.validateColumnFamilyWithCompactMode(keyspace(), columnFamily(), clientState.isNoCompactMode());
+ TableMetadata metadata = Schema.instance.validateTable(keyspace(), name());
- Attributes preparedAttributes = attrs.prepare(keyspace(), columnFamily());
- preparedAttributes.collectMarkerSpecification(boundNames);
+ Attributes preparedAttributes = attrs.prepare(keyspace(), name());
+ preparedAttributes.collectMarkerSpecification(bindVariables);
- Conditions preparedConditions = prepareConditions(metadata, boundNames);
+ Conditions preparedConditions = prepareConditions(metadata, bindVariables);
- return prepareInternal(metadata,
- boundNames,
- preparedConditions,
- preparedAttributes);
+ return prepareInternal(metadata, bindVariables, preparedConditions, preparedAttributes);
}
/**
* Returns the column conditions.
*
* @param metadata the column family meta data
- * @param boundNames the bound names
+ * @param bindVariables the bound names
* @return the column conditions.
*/
- private Conditions prepareConditions(CFMetaData metadata, VariableSpecifications boundNames)
+ private Conditions prepareConditions(TableMetadata metadata, VariableSpecifications bindVariables)
{
// To have both 'IF EXISTS'/'IF NOT EXISTS' and some other conditions doesn't make sense.
// So far this is enforced by the parser, but let's assert it for sanity if ever the parse changes.
@@ -852,27 +915,27 @@
if (conditions.isEmpty())
return Conditions.EMPTY_CONDITION;
- return prepareColumnConditions(metadata, boundNames);
+ return prepareColumnConditions(metadata, bindVariables);
}
/**
* Returns the column conditions.
*
* @param metadata the column family meta data
- * @param boundNames the bound names
+ * @param bindVariables the bound names
* @return the column conditions.
*/
- private ColumnConditions prepareColumnConditions(CFMetaData metadata, VariableSpecifications boundNames)
+ private ColumnConditions prepareColumnConditions(TableMetadata metadata, VariableSpecifications bindVariables)
{
checkNull(attrs.timestamp, "Cannot provide custom timestamp for conditional updates");
ColumnConditions.Builder builder = ColumnConditions.newBuilder();
- for (Pair<ColumnDefinition.Raw, ColumnCondition.Raw> entry : conditions)
+ for (Pair<ColumnMetadata.Raw, ColumnCondition.Raw> entry : conditions)
{
- ColumnDefinition def = entry.left.prepare(metadata);
+ ColumnMetadata def = entry.left.prepare(metadata);
ColumnCondition condition = entry.right.prepare(keyspace(), def, metadata);
- condition.collectMarkerSpecification(boundNames);
+ condition.collectMarkerSpecification(bindVariables);
checkFalse(def.isPrimaryKeyColumn(), "PRIMARY KEY column '%s' cannot have IF conditions", def.name);
builder.add(condition);
@@ -880,22 +943,22 @@
return builder.build();
}
- protected abstract ModificationStatement prepareInternal(CFMetaData cfm,
- VariableSpecifications boundNames,
+ protected abstract ModificationStatement prepareInternal(TableMetadata metadata,
+ VariableSpecifications bindVariables,
Conditions conditions,
Attributes attrs);
/**
* Creates the restrictions.
*
- * @param cfm the column family meta data
+ * @param metadata the column family meta data
* @param boundNames the bound names
* @param operations the column operations
* @param where the where clause
* @param conditions the conditions
* @return the restrictions
*/
- protected StatementRestrictions newRestrictions(CFMetaData cfm,
+ protected StatementRestrictions newRestrictions(TableMetadata metadata,
VariableSpecifications boundNames,
Operations operations,
WhereClause where,
@@ -905,19 +968,29 @@
throw new InvalidRequestException(CUSTOM_EXPRESSIONS_NOT_ALLOWED);
boolean applyOnlyToStaticColumns = appliesOnlyToStaticColumns(operations, conditions);
- return new StatementRestrictions(type, cfm, where, boundNames, applyOnlyToStaticColumns, false, false, false);
+ return new StatementRestrictions(type, metadata, where, boundNames, applyOnlyToStaticColumns, false, false);
}
/**
- * Retrieves the <code>ColumnDefinition</code> corresponding to the specified raw <code>ColumnIdentifier</code>.
+ * Retrieves the <code>ColumnMetadata</code> corresponding to the specified raw <code>ColumnIdentifier</code>.
*
- * @param cfm the column family meta data
+ * @param metadata the column family meta data
* @param rawId the raw <code>ColumnIdentifier</code>
- * @return the <code>ColumnDefinition</code> corresponding to the specified raw <code>ColumnIdentifier</code>
+ * @return the <code>ColumnMetadata</code> corresponding to the specified raw <code>ColumnIdentifier</code>
*/
- protected static ColumnDefinition getColumnDefinition(CFMetaData cfm, Raw rawId)
+ protected static ColumnMetadata getColumnDefinition(TableMetadata metadata, ColumnMetadata.Raw rawId)
{
- return rawId.prepare(cfm);
+ return rawId.prepare(metadata);
+ }
+
+ public List<Pair<ColumnMetadata.Raw, ColumnCondition.Raw>> getConditions()
+ {
+ ImmutableList.Builder<Pair<ColumnMetadata.Raw, ColumnCondition.Raw>> builder = ImmutableList.builderWithExpectedSize(conditions.size());
+
+ for (Pair<ColumnMetadata.Raw, ColumnCondition.Raw> condition : conditions)
+ builder.add(Pair.create(condition.left, condition.right));
+
+ return builder.build();
}
}
}
diff --git a/src/java/org/apache/cassandra/cql3/statements/ParsedStatement.java b/src/java/org/apache/cassandra/cql3/statements/ParsedStatement.java
deleted file mode 100644
index aa292f5..0000000
--- a/src/java/org/apache/cassandra/cql3/statements/ParsedStatement.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.cql3.statements;
-
-import java.util.Collections;
-import java.util.List;
-
-import org.apache.cassandra.cql3.*;
-import org.apache.cassandra.cql3.functions.Function;
-import org.apache.cassandra.exceptions.RequestValidationException;
-import org.apache.cassandra.service.ClientState;
-
-public abstract class ParsedStatement
-{
- private VariableSpecifications variables;
-
- public VariableSpecifications getBoundVariables()
- {
- return variables;
- }
-
- // Used by the parser and preparable statement
- public void setBoundVariables(List<ColumnIdentifier> boundNames)
- {
- this.variables = new VariableSpecifications(boundNames);
- }
-
- public void setBoundVariables(VariableSpecifications variables)
- {
- this.variables = variables;
- }
-
- public abstract Prepared prepare(ClientState clientState) throws RequestValidationException;
-
- public static class Prepared
- {
- /**
- * Contains the CQL statement source if the statement has been "regularly" perpared via
- * {@link org.apache.cassandra.cql3.QueryProcessor#prepare(java.lang.String, org.apache.cassandra.service.ClientState, boolean)} /
- * {@link QueryHandler#prepare(java.lang.String, org.apache.cassandra.service.QueryState, java.util.Map)}.
- * Other usages of this class may or may not contain the CQL statement source.
- */
- public String rawCQLStatement;
-
- public final CQLStatement statement;
- public final List<ColumnSpecification> boundNames;
- public final short[] partitionKeyBindIndexes;
-
- protected Prepared(CQLStatement statement, List<ColumnSpecification> boundNames, short[] partitionKeyBindIndexes)
- {
- this.statement = statement;
- this.boundNames = boundNames;
- this.partitionKeyBindIndexes = partitionKeyBindIndexes;
- this.rawCQLStatement = "";
- }
-
- public Prepared(CQLStatement statement, VariableSpecifications names, short[] partitionKeyBindIndexes)
- {
- this(statement, names.getSpecifications(), partitionKeyBindIndexes);
- }
-
- public Prepared(CQLStatement statement)
- {
- this(statement, Collections.<ColumnSpecification>emptyList(), null);
- }
- }
-
- public Iterable<Function> getFunctions()
- {
- return Collections.emptyList();
- }
-}
diff --git a/src/java/org/apache/cassandra/cql3/statements/PermissionsManagementStatement.java b/src/java/org/apache/cassandra/cql3/statements/PermissionsManagementStatement.java
index 0d07e12..aa7e85b 100644
--- a/src/java/org/apache/cassandra/cql3/statements/PermissionsManagementStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/PermissionsManagementStatement.java
@@ -21,12 +21,14 @@
import org.apache.cassandra.auth.*;
import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.config.SchemaConstants;
+import org.apache.cassandra.schema.SchemaConstants;
import org.apache.cassandra.cql3.RoleName;
import org.apache.cassandra.exceptions.InvalidRequestException;
import org.apache.cassandra.exceptions.RequestValidationException;
import org.apache.cassandra.exceptions.UnauthorizedException;
import org.apache.cassandra.service.ClientState;
+import org.apache.commons.lang3.builder.ToStringBuilder;
+import org.apache.commons.lang3.builder.ToStringStyle;
public abstract class PermissionsManagementStatement extends AuthorizationStatement
{
@@ -43,14 +45,14 @@
public void validate(ClientState state) throws RequestValidationException
{
- // validate login here before checkAccess to avoid leaking user existence to anonymous users.
+ // validate login here before authorize to avoid leaking user existence to anonymous users.
state.ensureNotAnonymous();
if (!DatabaseDescriptor.getRoleManager().isExistingRole(grantee))
throw new InvalidRequestException(String.format("Role %s doesn't exist", grantee.getRoleName()));
// if a keyspace is omitted when GRANT/REVOKE ON TABLE <table>, we need to correct the resource.
- // called both here and in checkAccess(), as in some cases we do not call the latter.
+ // called both here and in authorize(), as in some cases we do not call the latter.
resource = maybeCorrectResource(resource, state);
// altering permissions on builtin functions is not supported
@@ -64,16 +66,22 @@
throw new InvalidRequestException(String.format("Resource %s doesn't exist", resource));
}
- public void checkAccess(ClientState state) throws UnauthorizedException
+ public void authorize(ClientState state) throws UnauthorizedException
{
// if a keyspace is omitted when GRANT/REVOKE ON TABLE <table>, we need to correct the resource.
resource = maybeCorrectResource(resource, state);
// check that the user has AUTHORIZE permission on the resource or its parents, otherwise reject GRANT/REVOKE.
- state.ensureHasPermission(Permission.AUTHORIZE, resource);
+ state.ensurePermission(Permission.AUTHORIZE, resource);
// check that the user has [a single permission or all in case of ALL] on the resource or its parents.
for (Permission p : permissions)
- state.ensureHasPermission(p, resource);
+ state.ensurePermission(p, resource);
+ }
+
+ @Override
+ public String toString()
+ {
+ return ToStringBuilder.reflectionToString(this, ToStringStyle.SHORT_PREFIX_STYLE);
}
}
diff --git a/src/java/org/apache/cassandra/cql3/statements/QualifiedStatement.java b/src/java/org/apache/cassandra/cql3/statements/QualifiedStatement.java
new file mode 100644
index 0000000..a9c1f19
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/statements/QualifiedStatement.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.statements;
+
+import org.apache.commons.lang.builder.ToStringBuilder;
+import org.apache.commons.lang.builder.ToStringStyle;
+
+import org.apache.cassandra.cql3.CQLStatement;
+import org.apache.cassandra.cql3.QualifiedName;
+import org.apache.cassandra.service.ClientState;
+
+/**
+ * Abstract class for statements that work on sub-keyspace level (tables, views, indexes, functions, etc.)
+ */
+public abstract class QualifiedStatement extends CQLStatement.Raw
+{
+ final QualifiedName qualifiedName;
+
+ QualifiedStatement(QualifiedName qualifiedName)
+ {
+ this.qualifiedName = qualifiedName;
+ }
+
+ public void setKeyspace(ClientState state)
+ {
+ if (!qualifiedName.hasKeyspace())
+ {
+ // XXX: We explicitly only want to call state.getKeyspace() in this case, as we don't want to throw
+ // if not logged in any keyspace but a keyspace is explicitly set on the statement. So don't move
+ // the call outside the 'if' or replace the method by 'setKeyspace(state.getKeyspace())'
+ qualifiedName.setKeyspace(state.getKeyspace(), true);
+ }
+ }
+
+ // Only for internal calls, use the version with ClientState for user queries. In particular, the
+ // version with ClientState throws an exception if the statement does not have keyspace set *and*
+ // ClientState has no keyspace
+ public void setKeyspace(String keyspace)
+ {
+ qualifiedName.setKeyspace(keyspace, true);
+ }
+
+ public String keyspace()
+ {
+ if (!qualifiedName.hasKeyspace())
+ throw new IllegalStateException("Statement must have keyspace set");
+
+ return qualifiedName.getKeyspace();
+ }
+
+ public String name()
+ {
+ return qualifiedName.getName();
+ }
+
+ @Override
+ public String toString()
+ {
+ return ToStringBuilder.reflectionToString(this, ToStringStyle.SHORT_PREFIX_STYLE);
+ }
+}
diff --git a/src/java/org/apache/cassandra/cql3/statements/RequestValidations.java b/src/java/org/apache/cassandra/cql3/statements/RequestValidations.java
index fc07878..f351788 100644
--- a/src/java/org/apache/cassandra/cql3/statements/RequestValidations.java
+++ b/src/java/org/apache/cassandra/cql3/statements/RequestValidations.java
@@ -19,6 +19,7 @@
import java.nio.ByteBuffer;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.HashSet;
import java.util.List;
@@ -142,6 +143,23 @@
}
/**
+ * Checks that the specified collections is NOT <code>empty</code>.
+ * If it is an <code>InvalidRequestException</code> will be throws.
+ *
+ * @param collection the collection to test
+ * @param messageTemplate the template used to build the error message
+ * @param messageArgs the message arguments
+ * @return the collection
+ * @throws InvalidRequestException if the specified collection is <code>empty</code>.
+ */
+ public static <T extends Collection<E>, E> T checkNotEmpty(T collection, String messageTemplate, Object... messageArgs)
+ throws InvalidRequestException
+ {
+ checkTrue(!collection.isEmpty(), messageTemplate, messageArgs);
+ return collection;
+ }
+
+ /**
* Checks that the specified bind marker value is set to a meaningful value.
* If it is not a <code>InvalidRequestException</code> will be thrown.
*
diff --git a/src/java/org/apache/cassandra/cql3/statements/RevokePermissionsStatement.java b/src/java/org/apache/cassandra/cql3/statements/RevokePermissionsStatement.java
index 9acc685..57d0631 100644
--- a/src/java/org/apache/cassandra/cql3/statements/RevokePermissionsStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/RevokePermissionsStatement.java
@@ -19,6 +19,8 @@
import java.util.Set;
+import org.apache.cassandra.audit.AuditLogContext;
+import org.apache.cassandra.audit.AuditLogEntryType;
import org.apache.cassandra.auth.IResource;
import org.apache.cassandra.auth.Permission;
import org.apache.cassandra.config.DatabaseDescriptor;
@@ -27,6 +29,8 @@
import org.apache.cassandra.exceptions.RequestValidationException;
import org.apache.cassandra.service.ClientState;
import org.apache.cassandra.transport.messages.ResultMessage;
+import org.apache.commons.lang3.builder.ToStringBuilder;
+import org.apache.commons.lang3.builder.ToStringStyle;
public class RevokePermissionsStatement extends PermissionsManagementStatement
{
@@ -40,4 +44,17 @@
DatabaseDescriptor.getAuthorizer().revoke(state.getUser(), permissions, resource, grantee);
return null;
}
+
+ @Override
+ public String toString()
+ {
+ return ToStringBuilder.reflectionToString(this, ToStringStyle.SHORT_PREFIX_STYLE);
+ }
+
+ @Override
+ public AuditLogContext getAuditLogContext()
+ {
+ String keyspace = resource.hasParent() ? resource.getParent().getName() : resource.getName();
+ return new AuditLogContext(AuditLogEntryType.REVOKE, keyspace, resource.getName());
+ }
}
diff --git a/src/java/org/apache/cassandra/cql3/statements/RevokeRoleStatement.java b/src/java/org/apache/cassandra/cql3/statements/RevokeRoleStatement.java
index 4de905f..651743f 100644
--- a/src/java/org/apache/cassandra/cql3/statements/RevokeRoleStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/RevokeRoleStatement.java
@@ -17,12 +17,16 @@
*/
package org.apache.cassandra.cql3.statements;
+import org.apache.cassandra.audit.AuditLogContext;
+import org.apache.cassandra.audit.AuditLogEntryType;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.RoleName;
import org.apache.cassandra.exceptions.RequestExecutionException;
import org.apache.cassandra.exceptions.RequestValidationException;
import org.apache.cassandra.service.ClientState;
import org.apache.cassandra.transport.messages.ResultMessage;
+import org.apache.commons.lang3.builder.ToStringBuilder;
+import org.apache.commons.lang3.builder.ToStringStyle;
public class RevokeRoleStatement extends RoleManagementStatement
{
@@ -36,4 +40,16 @@
DatabaseDescriptor.getRoleManager().revokeRole(state.getUser(), role, grantee);
return null;
}
+
+ @Override
+ public String toString()
+ {
+ return ToStringBuilder.reflectionToString(this, ToStringStyle.SHORT_PREFIX_STYLE);
+ }
+
+ @Override
+ public AuditLogContext getAuditLogContext()
+ {
+ return new AuditLogContext(AuditLogEntryType.REVOKE);
+ }
}
diff --git a/src/java/org/apache/cassandra/cql3/statements/RoleManagementStatement.java b/src/java/org/apache/cassandra/cql3/statements/RoleManagementStatement.java
index e12b626..a5274dd 100644
--- a/src/java/org/apache/cassandra/cql3/statements/RoleManagementStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/RoleManagementStatement.java
@@ -25,6 +25,8 @@
import org.apache.cassandra.exceptions.RequestValidationException;
import org.apache.cassandra.exceptions.UnauthorizedException;
import org.apache.cassandra.service.ClientState;
+import org.apache.commons.lang3.builder.ToStringBuilder;
+import org.apache.commons.lang3.builder.ToStringStyle;
public abstract class RoleManagementStatement extends AuthenticationStatement
{
@@ -37,7 +39,7 @@
this.grantee = RoleResource.role(grantee.getName());
}
- public void checkAccess(ClientState state) throws UnauthorizedException
+ public void authorize(ClientState state) throws UnauthorizedException
{
super.checkPermission(state, Permission.AUTHORIZE, role);
}
@@ -52,4 +54,10 @@
if (!DatabaseDescriptor.getRoleManager().isExistingRole(grantee))
throw new InvalidRequestException(String.format("%s doesn't exist", grantee.getRoleName()));
}
+
+ @Override
+ public String toString()
+ {
+ return ToStringBuilder.reflectionToString(this, ToStringStyle.SHORT_PREFIX_STYLE);
+ }
}
diff --git a/src/java/org/apache/cassandra/cql3/statements/SchemaAlteringStatement.java b/src/java/org/apache/cassandra/cql3/statements/SchemaAlteringStatement.java
deleted file mode 100644
index 5079603..0000000
--- a/src/java/org/apache/cassandra/cql3/statements/SchemaAlteringStatement.java
+++ /dev/null
@@ -1,153 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.cql3.statements;
-
-import org.apache.cassandra.auth.AuthenticatedUser;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.Schema;
-import org.apache.cassandra.cql3.CFName;
-import org.apache.cassandra.cql3.CQLStatement;
-import org.apache.cassandra.cql3.QueryOptions;
-import org.apache.cassandra.exceptions.InvalidRequestException;
-import org.apache.cassandra.exceptions.RequestValidationException;
-import org.apache.cassandra.service.ClientState;
-import org.apache.cassandra.service.QueryState;
-import org.apache.cassandra.thrift.ThriftValidation;
-import org.apache.cassandra.transport.Event;
-import org.apache.cassandra.transport.messages.ResultMessage;
-
-import static org.apache.cassandra.cql3.statements.RequestValidations.invalidRequest;
-
-/**
- * Abstract class for statements that alter the schema.
- */
-public abstract class SchemaAlteringStatement extends CFStatement implements CQLStatement
-{
- private final boolean isColumnFamilyLevel;
-
- protected SchemaAlteringStatement()
- {
- super(null);
- this.isColumnFamilyLevel = false;
- }
-
- protected SchemaAlteringStatement(CFName name)
- {
- super(name);
- this.isColumnFamilyLevel = true;
- }
-
- public int getBoundTerms()
- {
- return 0;
- }
-
- @Override
- public void prepareKeyspace(ClientState state) throws InvalidRequestException
- {
- if (isColumnFamilyLevel)
- super.prepareKeyspace(state);
- }
-
- @Override
- public Prepared prepare(ClientState clientState)
- {
- // We don't allow schema changes in no-compact mode on compact tables because it feels like unnecessary
- // complication: applying the change on the non compact version of the table might be unsafe (the table is
- // still compact in general), and applying it to the compact version in a no-compact connection feels
- // confusing/unintuitive. If user want to alter the compact version, they can simply do so in a normal
- // connection; if they want to alter the non-compact version, they should finish their transition and properly
- // DROP COMPACT STORAGE on the table before doing so.
- if (isColumnFamilyLevel && clientState.isNoCompactMode())
- {
- CFMetaData table = ThriftValidation.validateColumnFamily(keyspace(), columnFamily());
- if (table.isCompactTable())
- {
- throw invalidRequest("Cannot alter schema of compact table %s.%s from a connection in NO-COMPACT mode",
- table.ksName, table.cfName);
- }
- else if (table.isView())
- {
- CFMetaData baseTable = Schema.instance.getView(table.ksName, table.cfName).baseTableMetadata();
- if (baseTable.isCompactTable())
- throw new InvalidRequestException(String.format("Cannot ALTER schema of view %s.%s on compact table %s from "
- + "a connection in NO-COMPACT mode",
- table.ksName, table.cfName,
- baseTable.ksName, baseTable.cfName));
- }
- }
-
- return new Prepared(this);
- }
-
- /**
- * Schema alteration may result in a new database object (keyspace, table, role, function) being created capable of
- * having permissions GRANTed on it. The creator of the object (the primary role assigned to the AuthenticatedUser
- * performing the operation) is automatically granted ALL applicable permissions on the object. This is a hook for
- * subclasses to override in order to perform that grant when the statement is executed.
- */
- protected void grantPermissionsToCreator(QueryState state)
- {
- // no-op by default
- }
-
- /**
- * Announces the migration to other nodes in the cluster.
- *
- * @return the schema change event corresponding to the execution of this statement, or {@code null} if no schema change
- * has occurred (when IF NOT EXISTS is used, for example)
- *
- * @throws RequestValidationException
- */
- protected abstract Event.SchemaChange announceMigration(QueryState queryState, boolean isLocalOnly) throws RequestValidationException;
-
- public ResultMessage execute(QueryState state, QueryOptions options, long queryStartNanoTime) throws RequestValidationException
- {
- // If an IF [NOT] EXISTS clause was used, this may not result in an actual schema change. To avoid doing
- // extra work in the drivers to handle schema changes, we return an empty message in this case. (CASSANDRA-7600)
- Event.SchemaChange ce = announceMigration(state, false);
- if (ce == null)
- return new ResultMessage.Void();
-
- // when a schema alteration results in a new db object being created, we grant permissions on the new
- // object to the user performing the request if:
- // * the user is not anonymous
- // * the configured IAuthorizer supports granting of permissions (not all do, AllowAllAuthorizer doesn't and
- // custom external implementations may not)
- AuthenticatedUser user = state.getClientState().getUser();
- if (user != null && !user.isAnonymous() && ce.change == Event.SchemaChange.Change.CREATED)
- {
- try
- {
- grantPermissionsToCreator(state);
- }
- catch (UnsupportedOperationException e)
- {
- // not a problem, grant is an optional method on IAuthorizer
- }
- }
-
- return new ResultMessage.SchemaChange(ce);
- }
-
- public ResultMessage executeInternal(QueryState state, QueryOptions options)
- {
- Event.SchemaChange ce = announceMigration(state, true);
- return ce == null ? new ResultMessage.Void() : new ResultMessage.SchemaChange(ce);
- }
-}
diff --git a/src/java/org/apache/cassandra/cql3/statements/SelectStatement.java b/src/java/org/apache/cassandra/cql3/statements/SelectStatement.java
index 348fa52..6e52ab1 100644
--- a/src/java/org/apache/cassandra/cql3/statements/SelectStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/SelectStatement.java
@@ -18,59 +18,33 @@
package org.apache.cassandra.cql3.statements;
import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.NavigableSet;
-import java.util.SortedSet;
+import java.util.*;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.MoreObjects;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.cassandra.audit.AuditLogContext;
+import org.apache.cassandra.audit.AuditLogEntryType;
import org.apache.cassandra.auth.Permission;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
-import org.apache.cassandra.cql3.CFName;
-import org.apache.cassandra.cql3.CQLStatement;
-import org.apache.cassandra.cql3.ColumnIdentifier;
-import org.apache.cassandra.cql3.ColumnSpecification;
-import org.apache.cassandra.cql3.QueryOptions;
-import org.apache.cassandra.cql3.QueryProcessor;
-import org.apache.cassandra.cql3.ResultSet;
-import org.apache.cassandra.cql3.SuperColumnCompatibility;
-import org.apache.cassandra.cql3.Term;
-import org.apache.cassandra.cql3.VariableSpecifications;
-import org.apache.cassandra.cql3.WhereClause;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.schema.TableMetadataRef;
+import org.apache.cassandra.cql3.*;
import org.apache.cassandra.cql3.functions.Function;
import org.apache.cassandra.cql3.restrictions.StatementRestrictions;
import org.apache.cassandra.cql3.selection.RawSelector;
+import org.apache.cassandra.cql3.selection.ResultSetBuilder;
+import org.apache.cassandra.cql3.selection.Selectable;
import org.apache.cassandra.cql3.selection.Selection;
+import org.apache.cassandra.cql3.selection.Selection.Selectors;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.aggregation.AggregationSpecification;
-import org.apache.cassandra.db.Clustering;
-import org.apache.cassandra.db.ColumnFamilyStore;
-import org.apache.cassandra.db.ConsistencyLevel;
-import org.apache.cassandra.db.DataRange;
-import org.apache.cassandra.db.DecoratedKey;
-import org.apache.cassandra.db.Keyspace;
-import org.apache.cassandra.db.PartitionPosition;
-import org.apache.cassandra.db.PartitionRangeReadCommand;
-import org.apache.cassandra.db.ReadQuery;
-import org.apache.cassandra.db.SinglePartitionReadCommand;
-import org.apache.cassandra.db.Slice;
-import org.apache.cassandra.db.Slices;
-import org.apache.cassandra.db.filter.ClusteringIndexFilter;
-import org.apache.cassandra.db.filter.ClusteringIndexNamesFilter;
-import org.apache.cassandra.db.filter.ClusteringIndexSliceFilter;
-import org.apache.cassandra.db.filter.ColumnFilter;
-import org.apache.cassandra.db.filter.DataLimits;
-import org.apache.cassandra.db.filter.RowFilter;
+import org.apache.cassandra.db.aggregation.GroupMaker;
+import org.apache.cassandra.db.filter.*;
import org.apache.cassandra.db.marshal.CollectionType;
import org.apache.cassandra.db.marshal.CompositeType;
import org.apache.cassandra.db.marshal.Int32Type;
@@ -81,11 +55,8 @@
import org.apache.cassandra.db.rows.RowIterator;
import org.apache.cassandra.db.view.View;
import org.apache.cassandra.dht.AbstractBounds;
-import org.apache.cassandra.exceptions.InvalidRequestException;
-import org.apache.cassandra.exceptions.RequestExecutionException;
-import org.apache.cassandra.exceptions.RequestValidationException;
-import org.apache.cassandra.exceptions.UnauthorizedException;
-import org.apache.cassandra.index.SecondaryIndexManager;
+import org.apache.cassandra.exceptions.*;
+import org.apache.cassandra.index.IndexRegistry;
import org.apache.cassandra.serializers.MarshalException;
import org.apache.cassandra.service.ClientState;
import org.apache.cassandra.service.ClientWarn;
@@ -93,11 +64,11 @@
import org.apache.cassandra.service.pager.AggregationQueryPager;
import org.apache.cassandra.service.pager.PagingState;
import org.apache.cassandra.service.pager.QueryPager;
-import org.apache.cassandra.thrift.ThriftValidation;
import org.apache.cassandra.transport.ProtocolVersion;
import org.apache.cassandra.transport.messages.ResultMessage;
import org.apache.cassandra.utils.ByteBufferUtil;
-import org.apache.cassandra.utils.FBUtilities;
+import org.apache.commons.lang3.builder.ToStringBuilder;
+import org.apache.commons.lang3.builder.ToStringStyle;
import static org.apache.cassandra.cql3.statements.RequestValidations.checkFalse;
import static org.apache.cassandra.cql3.statements.RequestValidations.checkNotNull;
@@ -120,8 +91,8 @@
public static final int DEFAULT_PAGE_SIZE = 10000;
- private final int boundTerms;
- public final CFMetaData cfm;
+ public final VariableSpecifications bindVariables;
+ public final TableMetadata table;
public final Parameters parameters;
private final Selection selection;
private final Term limit;
@@ -141,8 +112,6 @@
*/
private final Comparator<List<ByteBuffer>> orderingComparator;
- private final ColumnFilter queriedColumns;
-
// Used by forSelection below
private static final Parameters defaultParameters = new Parameters(Collections.emptyMap(),
Collections.emptyList(),
@@ -150,8 +119,8 @@
false,
false);
- public SelectStatement(CFMetaData cfm,
- int boundTerms,
+ public SelectStatement(TableMetadata table,
+ VariableSpecifications bindVariables,
Parameters parameters,
Selection selection,
StatementRestrictions restrictions,
@@ -161,8 +130,8 @@
Term limit,
Term perPartitionLimit)
{
- this.cfm = cfm;
- this.boundTerms = boundTerms;
+ this.table = table;
+ this.bindVariables = bindVariables;
this.selection = selection;
this.restrictions = restrictions;
this.isReversed = isReversed;
@@ -171,9 +140,21 @@
this.parameters = parameters;
this.limit = limit;
this.perPartitionLimit = perPartitionLimit;
- this.queriedColumns = gatherQueriedColumns();
}
+ @Override
+ public List<ColumnSpecification> getBindVariables()
+ {
+ return bindVariables.getBindVariables();
+ }
+
+ @Override
+ public short[] getPartitionKeyBindVariableIndexes()
+ {
+ return bindVariables.getPartitionKeyBindVariableIndexes(table);
+ }
+
+ @Override
public Iterable<Function> getFunctions()
{
List<Function> functions = new ArrayList<>();
@@ -193,42 +174,25 @@
perPartitionLimit.addFunctionsTo(functions);
}
- // Note that the queried columns internally is different from the one selected by the
- // user as it also include any column for which we have a restriction on.
- private ColumnFilter gatherQueriedColumns()
- {
- if (selection.isWildcard())
- return ColumnFilter.all(cfm);
-
- ColumnFilter.Builder builder = ColumnFilter.allColumnsBuilder(cfm);
- // Adds all selected columns
- for (ColumnDefinition def : selection.getColumns())
- if (!def.isPrimaryKeyColumn())
- builder.add(def);
- // as well as any restricted column (so we can actually apply the restriction)
- builder.addAll(restrictions.nonPKRestrictedColumns(true));
- return builder.build();
- }
-
/**
* The columns to fetch internally for this SELECT statement (which can be more than the one selected by the
* user as it also include any restricted column in particular).
*/
public ColumnFilter queriedColumns()
{
- return queriedColumns;
+ return selection.newSelectors(QueryOptions.DEFAULT).getColumnFilter();
}
// Creates a simple select based on the given selection.
// Note that the results select statement should not be used for actual queries, but only for processing already
// queried data through processColumnFamily.
- static SelectStatement forSelection(CFMetaData cfm, Selection selection)
+ static SelectStatement forSelection(TableMetadata table, Selection selection)
{
- return new SelectStatement(cfm,
- 0,
+ return new SelectStatement(table,
+ VariableSpecifications.empty(),
defaultParameters,
selection,
- StatementRestrictions.empty(StatementType.SELECT, cfm),
+ StatementRestrictions.empty(StatementType.SELECT, table),
false,
null,
null,
@@ -238,29 +202,24 @@
public ResultSet.ResultMetadata getResultMetadata()
{
- return selection.getResultMetadata(parameters.isJson);
+ return selection.getResultMetadata();
}
- public int getBoundTerms()
+ public void authorize(ClientState state) throws InvalidRequestException, UnauthorizedException
{
- return boundTerms;
- }
-
- public void checkAccess(ClientState state) throws InvalidRequestException, UnauthorizedException
- {
- if (cfm.isView())
+ if (table.isView())
{
- CFMetaData baseTable = View.findBaseTable(keyspace(), columnFamily());
+ TableMetadataRef baseTable = View.findBaseTable(keyspace(), columnFamily());
if (baseTable != null)
- state.hasColumnFamilyAccess(baseTable, Permission.SELECT);
+ state.ensureTablePermission(baseTable, Permission.SELECT);
}
else
{
- state.hasColumnFamilyAccess(cfm, Permission.SELECT);
+ state.ensureTablePermission(table, Permission.SELECT);
}
for (Function function : getFunctions())
- state.ensureHasPermission(Permission.EXECUTE, function);
+ state.ensurePermission(Permission.EXECUTE, function);
}
public void validate(ClientState state) throws InvalidRequestException
@@ -268,56 +227,82 @@
// Nothing to do, all validation has been done by RawStatement.prepare()
}
- public ResultMessage.Rows execute(QueryState state, QueryOptions options, long queryStartNanoTime) throws RequestExecutionException, RequestValidationException
+ public ResultMessage.Rows execute(QueryState state, QueryOptions options, long queryStartNanoTime)
{
ConsistencyLevel cl = options.getConsistency();
checkNotNull(cl, "Invalid empty consistency level");
cl.validateForRead(keyspace());
- int nowInSec = FBUtilities.nowInSeconds();
+ int nowInSec = options.getNowInSeconds(state);
int userLimit = getLimit(options);
int userPerPartitionLimit = getPerPartitionLimit(options);
int pageSize = options.getPageSize();
- ReadQuery query = getQuery(options, nowInSec, userLimit, userPerPartitionLimit, pageSize);
+
+ Selectors selectors = selection.newSelectors(options);
+ ReadQuery query = getQuery(options, selectors.getColumnFilter(), nowInSec, userLimit, userPerPartitionLimit, pageSize);
if (aggregationSpec == null && (pageSize <= 0 || (query.limits().count() <= pageSize)))
- return execute(query, options, state, nowInSec, userLimit, queryStartNanoTime);
+ return execute(query, options, state, selectors, nowInSec, userLimit, queryStartNanoTime);
QueryPager pager = getPager(query, options);
- return execute(Pager.forDistributedQuery(pager, cl, state.getClientState()), options, pageSize, nowInSec, userLimit, queryStartNanoTime);
+ return execute(Pager.forDistributedQuery(pager, cl, state.getClientState()),
+ options,
+ selectors,
+ pageSize,
+ nowInSec,
+ userLimit,
+ queryStartNanoTime);
}
public ReadQuery getQuery(QueryOptions options, int nowInSec) throws RequestValidationException
{
- return getQuery(options, nowInSec, getLimit(options), getPerPartitionLimit(options), options.getPageSize());
+ Selectors selectors = selection.newSelectors(options);
+ return getQuery(options,
+ selectors.getColumnFilter(),
+ nowInSec,
+ getLimit(options),
+ getPerPartitionLimit(options),
+ options.getPageSize());
}
- public ReadQuery getQuery(QueryOptions options, int nowInSec, int userLimit, int perPartitionLimit, int pageSize)
+ public ReadQuery getQuery(QueryOptions options,
+ ColumnFilter columnFilter,
+ int nowInSec,
+ int userLimit,
+ int perPartitionLimit,
+ int pageSize)
{
boolean isPartitionRangeQuery = restrictions.isKeyRange() || restrictions.usesSecondaryIndexing();
DataLimits limit = getDataLimits(userLimit, perPartitionLimit, pageSize);
if (isPartitionRangeQuery)
- return getRangeCommand(options, limit, nowInSec);
+ return getRangeCommand(options, columnFilter, limit, nowInSec);
- return getSliceCommands(options, limit, nowInSec);
+ return getSliceCommands(options, columnFilter, limit, nowInSec);
}
private ResultMessage.Rows execute(ReadQuery query,
QueryOptions options,
QueryState state,
+ Selectors selectors,
int nowInSec,
int userLimit, long queryStartNanoTime) throws RequestValidationException, RequestExecutionException
{
try (PartitionIterator data = query.execute(options.getConsistency(), state.getClientState(), queryStartNanoTime))
{
- return processResults(data, options, nowInSec, userLimit);
+ return processResults(data, options, selectors, nowInSec, userLimit);
}
}
+ @Override
+ public AuditLogContext getAuditLogContext()
+ {
+ return new AuditLogContext(AuditLogEntryType.SELECT, keyspace(), table.name);
+ }
+
// Simple wrapper class to avoid some code duplication
private static abstract class Pager
{
@@ -387,6 +372,7 @@
private ResultMessage.Rows execute(Pager pager,
QueryOptions options,
+ Selectors selectors,
int pageSize,
int nowInSec,
int userLimit,
@@ -413,7 +399,7 @@
ResultMessage.Rows msg;
try (PartitionIterator page = pager.fetchPage(pageSize, queryStartNanoTime))
{
- msg = processResults(page, options, nowInSec, userLimit);
+ msg = processResults(page, options, selectors, nowInSec, userLimit);
}
// Please note that the isExhausted state of the pager only gets updated when we've closed the page, so this
@@ -432,16 +418,17 @@
private ResultMessage.Rows processResults(PartitionIterator partitions,
QueryOptions options,
+ Selectors selectors,
int nowInSec,
int userLimit) throws RequestValidationException
{
- ResultSet rset = process(partitions, options, nowInSec, userLimit);
+ ResultSet rset = process(partitions, options, selectors, nowInSec, userLimit);
return new ResultMessage.Rows(rset);
}
- public ResultMessage.Rows executeInternal(QueryState state, QueryOptions options) throws RequestExecutionException, RequestValidationException
+ public ResultMessage.Rows executeLocally(QueryState state, QueryOptions options) throws RequestExecutionException, RequestValidationException
{
- return executeInternal(state, options, FBUtilities.nowInSeconds(), System.nanoTime());
+ return executeInternal(state, options, options.getNowInSeconds(state), System.nanoTime());
}
public ResultMessage.Rows executeInternal(QueryState state, QueryOptions options, int nowInSec, long queryStartNanoTime) throws RequestExecutionException, RequestValidationException
@@ -449,7 +436,9 @@
int userLimit = getLimit(options);
int userPerPartitionLimit = getPerPartitionLimit(options);
int pageSize = options.getPageSize();
- ReadQuery query = getQuery(options, nowInSec, userLimit, userPerPartitionLimit, pageSize);
+
+ Selectors selectors = selection.newSelectors(options);
+ ReadQuery query = getQuery(options, selectors.getColumnFilter(), nowInSec, userLimit, userPerPartitionLimit, pageSize);
try (ReadExecutionController executionController = query.executionController())
{
@@ -457,15 +446,19 @@
{
try (PartitionIterator data = query.executeInternal(executionController))
{
- return processResults(data, options, nowInSec, userLimit);
+ return processResults(data, options, selectors, nowInSec, userLimit);
}
}
- else
- {
- QueryPager pager = getPager(query, options);
- return execute(Pager.forInternalQuery(pager, executionController), options, pageSize, nowInSec, userLimit, queryStartNanoTime);
- }
+ QueryPager pager = getPager(query, options);
+
+ return execute(Pager.forInternalQuery(pager, executionController),
+ options,
+ selectors,
+ pageSize,
+ nowInSec,
+ userLimit,
+ queryStartNanoTime);
}
}
@@ -473,7 +466,7 @@
{
QueryPager pager = query.getPager(options.getPagingState(), options.getProtocolVersion());
- if (aggregationSpec == null || query == ReadQuery.EMPTY)
+ if (aggregationSpec == null || query.isEmpty())
return pager;
return new AggregationQueryPager(pager, query.limits());
@@ -481,17 +474,19 @@
public ResultSet process(PartitionIterator partitions, int nowInSec) throws InvalidRequestException
{
- return process(partitions, QueryOptions.DEFAULT, nowInSec, getLimit(QueryOptions.DEFAULT));
+ QueryOptions options = QueryOptions.DEFAULT;
+ Selectors selectors = selection.newSelectors(options);
+ return process(partitions, options, selectors, nowInSec, getLimit(options));
}
public String keyspace()
{
- return cfm.ksName;
+ return table.keyspace;
}
public String columnFamily()
{
- return cfm.cfName;
+ return table.name;
}
/**
@@ -510,30 +505,26 @@
return restrictions;
}
- private ReadQuery getSliceCommands(QueryOptions options, DataLimits limit, int nowInSec) throws RequestValidationException
+ private ReadQuery getSliceCommands(QueryOptions options, ColumnFilter columnFilter, DataLimits limit, int nowInSec)
{
Collection<ByteBuffer> keys = restrictions.getPartitionKeys(options);
if (keys.isEmpty())
- return ReadQuery.EMPTY;
+ return ReadQuery.empty(table);
- ClusteringIndexFilter filter = makeClusteringIndexFilter(options);
+ ClusteringIndexFilter filter = makeClusteringIndexFilter(options, columnFilter);
if (filter == null)
- return ReadQuery.EMPTY;
+ return ReadQuery.empty(table);
RowFilter rowFilter = getRowFilter(options);
- // Note that we use the total limit for every key, which is potentially inefficient.
- // However, IN + LIMIT is not a very sensible choice.
- List<SinglePartitionReadCommand> commands = new ArrayList<>(keys.size());
+ List<DecoratedKey> decoratedKeys = new ArrayList<>(keys.size());
for (ByteBuffer key : keys)
{
QueryProcessor.validateKey(key);
- DecoratedKey dk = cfm.decorateKey(ByteBufferUtil.clone(key));
- ColumnFilter cf = (cfm.isSuper() && cfm.isDense()) ? SuperColumnCompatibility.getColumnFilter(cfm, options, restrictions.getSuperColumnRestrictions()) : queriedColumns;
- commands.add(SinglePartitionReadCommand.create(cfm, nowInSec, cf, rowFilter, limit, dk, filter));
+ decoratedKeys.add(table.partitioner.decorateKey(ByteBufferUtil.clone(key)));
}
- return new SinglePartitionReadCommand.Group(commands, limit);
+ return SinglePartitionReadQuery.createGroup(table, nowInSec, columnFilter, rowFilter, limit, decoratedKeys, filter);
}
/**
@@ -547,11 +538,12 @@
public Slices clusteringIndexFilterAsSlices()
{
QueryOptions options = QueryOptions.forInternalCalls(Collections.emptyList());
- ClusteringIndexFilter filter = makeClusteringIndexFilter(options);
+ ColumnFilter columnFilter = selection.newSelectors(options).getColumnFilter();
+ ClusteringIndexFilter filter = makeClusteringIndexFilter(options, columnFilter);
if (filter instanceof ClusteringIndexSliceFilter)
return ((ClusteringIndexSliceFilter)filter).requestedSlices();
- Slices.Builder builder = new Slices.Builder(cfm.comparator);
+ Slices.Builder builder = new Slices.Builder(table.comparator);
for (Clustering clustering: ((ClusteringIndexNamesFilter)filter).requestedRows())
builder.add(Slice.make(clustering));
return builder.build();
@@ -564,9 +556,10 @@
public SinglePartitionReadCommand internalReadForView(DecoratedKey key, int nowInSec)
{
QueryOptions options = QueryOptions.forInternalCalls(Collections.emptyList());
- ClusteringIndexFilter filter = makeClusteringIndexFilter(options);
+ ColumnFilter columnFilter = selection.newSelectors(options).getColumnFilter();
+ ClusteringIndexFilter filter = makeClusteringIndexFilter(options, columnFilter);
RowFilter rowFilter = getRowFilter(options);
- return SinglePartitionReadCommand.create(cfm, nowInSec, queriedColumns, rowFilter, DataLimits.NONE, key, filter);
+ return SinglePartitionReadCommand.create(table, nowInSec, columnFilter, rowFilter, DataLimits.NONE, key, filter);
}
/**
@@ -577,11 +570,11 @@
return getRowFilter(QueryOptions.forInternalCalls(Collections.emptyList()));
}
- private ReadQuery getRangeCommand(QueryOptions options, DataLimits limit, int nowInSec) throws RequestValidationException
+ private ReadQuery getRangeCommand(QueryOptions options, ColumnFilter columnFilter, DataLimits limit, int nowInSec)
{
- ClusteringIndexFilter clusteringIndexFilter = makeClusteringIndexFilter(options);
+ ClusteringIndexFilter clusteringIndexFilter = makeClusteringIndexFilter(options, columnFilter);
if (clusteringIndexFilter == null)
- return ReadQuery.EMPTY;
+ return ReadQuery.empty(table);
RowFilter rowFilter = getRowFilter(options);
@@ -589,10 +582,10 @@
// We want to have getRangeSlice to count the number of columns, not the number of keys.
AbstractBounds<PartitionPosition> keyBounds = restrictions.getPartitionKeyBounds(options);
if (keyBounds == null)
- return ReadQuery.EMPTY;
+ return ReadQuery.empty(table);
- PartitionRangeReadCommand command =
- PartitionRangeReadCommand.create(false, cfm, nowInSec, queriedColumns, rowFilter, limit, new DataRange(keyBounds, clusteringIndexFilter));
+ ReadQuery command =
+ PartitionRangeReadQuery.create(table, nowInSec, columnFilter, rowFilter, limit, new DataRange(keyBounds, clusteringIndexFilter));
// If there's a secondary index that the command can use, have it validate the request parameters.
command.maybeValidateIndex();
@@ -600,8 +593,7 @@
return command;
}
- private ClusteringIndexFilter makeClusteringIndexFilter(QueryOptions options)
- throws InvalidRequestException
+ private ClusteringIndexFilter makeClusteringIndexFilter(QueryOptions options, ColumnFilter columnFilter)
{
if (parameters.isDistinct)
{
@@ -623,20 +615,19 @@
return new ClusteringIndexSliceFilter(slices, isReversed);
}
- else
- {
- NavigableSet<Clustering> clusterings = getRequestedRows(options);
- // We can have no clusterings if either we're only selecting the static columns, or if we have
- // a 'IN ()' for clusterings. In that case, we still want to query if some static columns are
- // queried. But we're fine otherwise.
- if (clusterings.isEmpty() && queriedColumns.fetchedColumns().statics.isEmpty())
- return null;
- return new ClusteringIndexNamesFilter(clusterings, isReversed);
- }
+ NavigableSet<Clustering> clusterings = getRequestedRows(options);
+ // We can have no clusterings if either we're only selecting the static columns, or if we have
+ // a 'IN ()' for clusterings. In that case, we still want to query if some static columns are
+ // queried. But we're fine otherwise.
+ if (clusterings.isEmpty() && columnFilter.fetchedColumns().statics.isEmpty())
+ return null;
+
+ return new ClusteringIndexNamesFilter(clusterings, isReversed);
}
- private Slices makeSlices(QueryOptions options)
+ @VisibleForTesting
+ public Slices makeSlices(QueryOptions options)
throws InvalidRequestException
{
SortedSet<ClusteringBound> startBounds = restrictions.getClusteringColumnsBounds(Bound.START, options);
@@ -648,12 +639,12 @@
{
ClusteringBound start = startBounds.first();
ClusteringBound end = endBounds.first();
- return cfm.comparator.compare(start, end) > 0
+ return Slice.isEmpty(table.comparator, start, end)
? Slices.NONE
- : Slices.with(cfm.comparator, Slice.make(start, end));
+ : Slices.with(table.comparator, Slice.make(start, end));
}
- Slices.Builder builder = new Slices.Builder(cfm.comparator, startBounds.size());
+ Slices.Builder builder = new Slices.Builder(table.comparator, startBounds.size());
Iterator<ClusteringBound> startIter = startBounds.iterator();
Iterator<ClusteringBound> endIter = endBounds.iterator();
while (startIter.hasNext() && endIter.hasNext())
@@ -662,7 +653,7 @@
ClusteringBound end = endIter.next();
// Ignore slices that are nonsensical
- if (cfm.comparator.compare(start, end) > 0)
+ if (Slice.isEmpty(table.comparator, start, end))
continue;
builder.add(start, end);
@@ -770,18 +761,18 @@
*/
public RowFilter getRowFilter(QueryOptions options) throws InvalidRequestException
{
- ColumnFamilyStore cfs = Keyspace.open(keyspace()).getColumnFamilyStore(columnFamily());
- SecondaryIndexManager secondaryIndexManager = cfs.indexManager;
- RowFilter filter = restrictions.getRowFilter(secondaryIndexManager, options);
- return filter;
+ IndexRegistry indexRegistry = IndexRegistry.obtain(table);
+ return restrictions.getRowFilter(indexRegistry, options);
}
private ResultSet process(PartitionIterator partitions,
QueryOptions options,
+ Selectors selectors,
int nowInSec,
int userLimit) throws InvalidRequestException
{
- Selection.ResultSetBuilder result = selection.resultSetBuilder(options, parameters.isJson, aggregationSpec);
+ GroupMaker groupMaker = aggregationSpec == null ? null : aggregationSpec.newGroupMaker();
+ ResultSetBuilder result = new ResultSetBuilder(getResultMetadata(), selectors, groupMaker);
while (partitions.hasNext())
{
@@ -800,12 +791,12 @@
return cqlRows;
}
- public static ByteBuffer[] getComponents(CFMetaData cfm, DecoratedKey dk)
+ public static ByteBuffer[] getComponents(TableMetadata metadata, DecoratedKey dk)
{
ByteBuffer key = dk.getKey();
- if (cfm.getKeyValidator() instanceof CompositeType)
+ if (metadata.partitionKeyType instanceof CompositeType)
{
- return ((CompositeType)cfm.getKeyValidator()).split(key);
+ return ((CompositeType)metadata.partitionKeyType).split(key);
}
else
{
@@ -813,30 +804,34 @@
}
}
+ // Determines whether, when we have a partition result with not rows, we still return the static content (as a
+ // result set row with null for all other regular columns.)
+ private boolean returnStaticContentOnPartitionWithNoRows()
+ {
+ // The general rational is that if some rows are specifically selected by the query (have clustering or
+ // regular columns restrictions), we ignore partitions that are empty outside of static content, but if it's a full partition
+ // query, then we include that content.
+ // We make an exception for "static compact" table are from a CQL standpoint we always want to show their static
+ // content for backward compatiblity.
+ return queriesFullPartitions() || table.isStaticCompactTable();
+ }
+
// Used by ModificationStatement for CAS operations
- void processPartition(RowIterator partition, QueryOptions options, Selection.ResultSetBuilder result, int nowInSec)
+ void processPartition(RowIterator partition, QueryOptions options, ResultSetBuilder result, int nowInSec)
throws InvalidRequestException
{
- if (cfm.isSuper() && cfm.isDense())
- {
- SuperColumnCompatibility.processPartition(cfm, selection, partition, result, options.getProtocolVersion(), restrictions.getSuperColumnRestrictions(), options);
- return;
- }
-
ProtocolVersion protocolVersion = options.getProtocolVersion();
- ByteBuffer[] keyComponents = getComponents(cfm, partition.partitionKey());
+ ByteBuffer[] keyComponents = getComponents(table, partition.partitionKey());
Row staticRow = partition.staticRow();
- // If there is no rows, and there's no restriction on clustering/regular columns,
- // then provided the select was a full partition selection (either by partition key and/or by static column),
- // we want to include static columns and we're done.
+ // If there is no rows, we include the static content if we should and we're done.
if (!partition.hasNext())
{
- if (!staticRow.isEmpty() && (queriesFullPartitions() || cfm.isStaticCompactTable()))
+ if (!staticRow.isEmpty() && returnStaticContentOnPartitionWithNoRows())
{
result.newRow(partition.partitionKey(), staticRow.clustering());
- for (ColumnDefinition def : selection.getColumns())
+ for (ColumnMetadata def : selection.getColumns())
{
switch (def.kind)
{
@@ -859,7 +854,7 @@
Row row = partition.next();
result.newRow( partition.partitionKey(), row.clustering());
// Respect selection order
- for (ColumnDefinition def : selection.getColumns())
+ for (ColumnMetadata def : selection.getColumns())
{
switch (def.kind)
{
@@ -889,7 +884,7 @@
return !restrictions.hasClusteringColumnsRestrictions() && !restrictions.hasRegularColumnsRestrictions();
}
- private static void addValue(Selection.ResultSetBuilder result, ColumnDefinition def, Row row, int nowInSec, ProtocolVersion protocolVersion)
+ private static void addValue(ResultSetBuilder result, ColumnMetadata def, Row row, int nowInSec, ProtocolVersion protocolVersion)
{
if (def.isComplex())
{
@@ -925,7 +920,7 @@
Collections.sort(cqlRows.rows, orderingComparator);
}
- public static class RawStatement extends CFStatement
+ public static class RawStatement extends QualifiedStatement
{
public final Parameters parameters;
public final List<RawSelector> selectClause;
@@ -933,7 +928,8 @@
public final Term.Raw limit;
public final Term.Raw perPartitionLimit;
- public RawStatement(CFName cfName, Parameters parameters,
+ public RawStatement(QualifiedName cfName,
+ Parameters parameters,
List<RawSelector> selectClause,
WhereClause whereClause,
Term.Raw limit,
@@ -947,27 +943,39 @@
this.perPartitionLimit = perPartitionLimit;
}
- public ParsedStatement.Prepared prepare(ClientState clientState) throws InvalidRequestException
+ public SelectStatement prepare(ClientState state)
{
- return prepare(false, clientState);
+ return prepare(false);
}
- public ParsedStatement.Prepared prepare(boolean forView, ClientState clientState) throws InvalidRequestException
+ public SelectStatement prepare(boolean forView) throws InvalidRequestException
{
- CFMetaData cfm = ThriftValidation.validateColumnFamilyWithCompactMode(keyspace(), columnFamily(), clientState.isNoCompactMode());
- VariableSpecifications boundNames = getBoundVariables();
+ TableMetadata table = Schema.instance.validateTable(keyspace(), name());
- Selection selection = prepareSelection(cfm, boundNames);
+ List<Selectable> selectables = RawSelector.toSelectables(selectClause, table);
+ boolean containsOnlyStaticColumns = selectOnlyStaticColumns(table, selectables);
- StatementRestrictions restrictions = prepareRestrictions(cfm, boundNames, selection, forView);
+ StatementRestrictions restrictions = prepareRestrictions(table, bindVariables, containsOnlyStaticColumns, forView);
+
+ // If we order post-query, the sorted column needs to be in the ResultSet for sorting,
+ // even if we don't ultimately ship them to the client (CASSANDRA-4911).
+ Map<ColumnMetadata, Boolean> orderingColumns = getOrderingColumns(table);
+ Set<ColumnMetadata> resultSetOrderingColumns = restrictions.keyIsInRelation() ? orderingColumns.keySet()
+ : Collections.emptySet();
+
+ Selection selection = prepareSelection(table,
+ selectables,
+ bindVariables,
+ resultSetOrderingColumns,
+ restrictions);
if (parameters.isDistinct)
{
checkNull(perPartitionLimit, "PER PARTITION LIMIT is not allowed with SELECT DISTINCT queries");
- validateDistinctSelection(cfm, selection, restrictions);
+ validateDistinctSelection(table, selection, restrictions);
}
- AggregationSpecification aggregationSpec = getAggregationSpecification(cfm,
+ AggregationSpecification aggregationSpec = getAggregationSpecification(table,
selection,
restrictions,
parameters.isDistinct);
@@ -978,69 +986,106 @@
Comparator<List<ByteBuffer>> orderingComparator = null;
boolean isReversed = false;
- if (!parameters.orderings.isEmpty())
+ if (!orderingColumns.isEmpty())
{
assert !forView;
verifyOrderingIsAllowed(restrictions);
- orderingComparator = getOrderingComparator(cfm, selection, restrictions, parameters.isJson);
- isReversed = isReversed(cfm);
+ orderingComparator = getOrderingComparator(selection, restrictions, orderingColumns);
+ isReversed = isReversed(table, orderingColumns, restrictions);
if (isReversed)
orderingComparator = Collections.reverseOrder(orderingComparator);
}
checkNeedsFiltering(restrictions);
- SelectStatement stmt = new SelectStatement(cfm,
- boundNames.size(),
- parameters,
- selection,
- restrictions,
- isReversed,
- aggregationSpec,
- orderingComparator,
- prepareLimit(boundNames, limit, keyspace(), limitReceiver()),
- prepareLimit(boundNames, perPartitionLimit, keyspace(), perPartitionLimitReceiver()));
-
- return new ParsedStatement.Prepared(stmt, boundNames, boundNames.getPartitionKeyBindIndexes(cfm));
+ return new SelectStatement(table,
+ bindVariables,
+ parameters,
+ selection,
+ restrictions,
+ isReversed,
+ aggregationSpec,
+ orderingComparator,
+ prepareLimit(bindVariables, limit, keyspace(), limitReceiver()),
+ prepareLimit(bindVariables, perPartitionLimit, keyspace(), perPartitionLimitReceiver()));
}
- /**
- * Prepares the selection to use for the statement.
- *
- * @param cfm the table metadata
- * @param boundNames the bound names
- * @return the selection to use for the statement
- */
- private Selection prepareSelection(CFMetaData cfm, VariableSpecifications boundNames)
+ private Selection prepareSelection(TableMetadata table,
+ List<Selectable> selectables,
+ VariableSpecifications boundNames,
+ Set<ColumnMetadata> resultSetOrderingColumns,
+ StatementRestrictions restrictions)
{
boolean hasGroupBy = !parameters.groups.isEmpty();
- if (selectClause.isEmpty())
- return hasGroupBy ? Selection.wildcardWithGroupBy(cfm, boundNames) : Selection.wildcard(cfm);
+ if (selectables.isEmpty()) // wildcard query
+ {
+ return hasGroupBy ? Selection.wildcardWithGroupBy(table, boundNames, parameters.isJson)
+ : Selection.wildcard(table, parameters.isJson);
+ }
- return Selection.fromSelectors(cfm, selectClause, boundNames, hasGroupBy);
+ return Selection.fromSelectors(table,
+ selectables,
+ boundNames,
+ resultSetOrderingColumns,
+ restrictions.nonPKRestrictedColumns(false),
+ hasGroupBy,
+ parameters.isJson);
+ }
+
+ /**
+ * Checks if the specified selectables select only partition key columns or static columns
+ *
+ * @param table the table metadata
+ * @param selectables the selectables to check
+ * @return {@code true} if the specified selectables select only partition key columns or static columns,
+ * {@code false} otherwise.
+ */
+ private boolean selectOnlyStaticColumns(TableMetadata table, List<Selectable> selectables)
+ {
+ if (table.isStaticCompactTable() || !table.hasStaticColumns() || selectables.isEmpty())
+ return false;
+
+ return Selectable.selectColumns(selectables, (column) -> column.isStatic())
+ && !Selectable.selectColumns(selectables, (column) -> !column.isPartitionKey() && !column.isStatic());
+ }
+
+ /**
+ * Returns the columns used to order the data.
+ * @return the columns used to order the data.
+ */
+ private Map<ColumnMetadata, Boolean> getOrderingColumns(TableMetadata table)
+ {
+ if (parameters.orderings.isEmpty())
+ return Collections.emptyMap();
+
+ Map<ColumnMetadata, Boolean> orderingColumns = new LinkedHashMap<>();
+ for (Map.Entry<ColumnMetadata.Raw, Boolean> entry : parameters.orderings.entrySet())
+ {
+ orderingColumns.put(entry.getKey().prepare(table), entry.getValue());
+ }
+ return orderingColumns;
}
/**
* Prepares the restrictions.
*
- * @param cfm the column family meta data
+ * @param metadata the column family meta data
* @param boundNames the variable specifications
- * @param selection the selection
+ * @param selectsOnlyStaticColumns {@code true} if the query select only static columns, {@code false} otherwise.
* @return the restrictions
* @throws InvalidRequestException if a problem occurs while building the restrictions
*/
- private StatementRestrictions prepareRestrictions(CFMetaData cfm,
+ private StatementRestrictions prepareRestrictions(TableMetadata metadata,
VariableSpecifications boundNames,
- Selection selection,
+ boolean selectsOnlyStaticColumns,
boolean forView) throws InvalidRequestException
{
return new StatementRestrictions(StatementType.SELECT,
- cfm,
+ metadata,
whereClause,
boundNames,
- selection.containsOnlyStaticColumns(),
- selection.containsAComplexColumn(),
+ selectsOnlyStaticColumns,
parameters.allowFiltering,
forView);
}
@@ -1063,17 +1108,17 @@
checkFalse(restrictions.isKeyRange(), "ORDER BY is only supported when the partition key is restricted by an EQ or an IN.");
}
- private static void validateDistinctSelection(CFMetaData cfm,
+ private static void validateDistinctSelection(TableMetadata metadata,
Selection selection,
StatementRestrictions restrictions)
throws InvalidRequestException
{
checkFalse(restrictions.hasClusteringColumnsRestrictions() ||
- (restrictions.hasNonPrimaryKeyRestrictions() && !restrictions.nonPKRestrictedColumns(true).stream().allMatch(ColumnDefinition::isStatic)),
+ (restrictions.hasNonPrimaryKeyRestrictions() && !restrictions.nonPKRestrictedColumns(true).stream().allMatch(ColumnMetadata::isStatic)),
"SELECT DISTINCT with WHERE clause only supports restriction by partition key and/or static columns.");
- Collection<ColumnDefinition> requestedColumns = selection.getColumns();
- for (ColumnDefinition def : requestedColumns)
+ Collection<ColumnMetadata> requestedColumns = selection.getColumns();
+ for (ColumnMetadata def : requestedColumns)
checkFalse(!def.isPartitionKey() && !def.isStatic(),
"SELECT DISTINCT queries must only request partition key columns and/or static columns (not %s)",
def.name);
@@ -1083,7 +1128,7 @@
if (!restrictions.isKeyRange())
return;
- for (ColumnDefinition def : cfm.partitionKeyColumns())
+ for (ColumnMetadata def : metadata.partitionKeyColumns())
checkTrue(requestedColumns.contains(def),
"SELECT DISTINCT queries must request all the partition key columns (missing %s)", def.name);
}
@@ -1091,13 +1136,13 @@
/**
* Creates the <code>AggregationSpecification</code>s used to make the aggregates.
*
- * @param cfm the column family metadata
+ * @param metadata the table metadata
* @param selection the selection
* @param restrictions the restrictions
- * @param isDistinct <code>true</code> if the query is a DISTINCT one.
+ * @param isDistinct <code>true</code> if the query is a DISTINCT one.
* @return the <code>AggregationSpecification</code>s used to make the aggregates
*/
- private AggregationSpecification getAggregationSpecification(CFMetaData cfm,
+ private AggregationSpecification getAggregationSpecification(TableMetadata metadata,
Selection selection,
StatementRestrictions restrictions,
boolean isDistinct)
@@ -1108,10 +1153,10 @@
int clusteringPrefixSize = 0;
- Iterator<ColumnDefinition> pkColumns = cfm.primaryKeyColumns().iterator();
- for (ColumnDefinition.Raw raw : parameters.groups)
+ Iterator<ColumnMetadata> pkColumns = metadata.primaryKeyColumns().iterator();
+ for (ColumnMetadata.Raw raw : parameters.groups)
{
- ColumnDefinition def = raw.prepare(cfm);
+ ColumnMetadata def = raw.prepare(metadata);
checkTrue(def.isPartitionKey() || def.isClusteringColumn(),
"Group by is currently only supported on the columns of the PRIMARY KEY, got %s", def.name);
@@ -1121,7 +1166,7 @@
checkTrue(pkColumns.hasNext(),
"Group by currently only support groups of columns following their declared order in the PRIMARY KEY");
- ColumnDefinition pkColumn = pkColumns.next();
+ ColumnMetadata pkColumn = pkColumns.next();
if (pkColumn.isClusteringColumn())
clusteringPrefixSize++;
@@ -1142,62 +1187,47 @@
checkFalse(clusteringPrefixSize > 0 && isDistinct,
"Grouping on clustering columns is not allowed for SELECT DISTINCT queries");
- return AggregationSpecification.aggregatePkPrefix(cfm.comparator, clusteringPrefixSize);
+ return AggregationSpecification.aggregatePkPrefix(metadata.comparator, clusteringPrefixSize);
}
- private Comparator<List<ByteBuffer>> getOrderingComparator(CFMetaData cfm,
- Selection selection,
+ private Comparator<List<ByteBuffer>> getOrderingComparator(Selection selection,
StatementRestrictions restrictions,
- boolean isJson)
+ Map<ColumnMetadata, Boolean> orderingColumns)
throws InvalidRequestException
{
if (!restrictions.keyIsInRelation())
return null;
- Map<ColumnDefinition, Integer> orderingIndexes = getOrderingIndex(cfm, selection, isJson);
+ List<Integer> idToSort = new ArrayList<Integer>(orderingColumns.size());
+ List<Comparator<ByteBuffer>> sorters = new ArrayList<Comparator<ByteBuffer>>(orderingColumns.size());
- List<Integer> idToSort = new ArrayList<Integer>();
- List<Comparator<ByteBuffer>> sorters = new ArrayList<Comparator<ByteBuffer>>();
-
- for (ColumnDefinition.Raw raw : parameters.orderings.keySet())
+ for (ColumnMetadata orderingColumn : orderingColumns.keySet())
{
- ColumnDefinition orderingColumn = raw.prepare(cfm);
- idToSort.add(orderingIndexes.get(orderingColumn));
+ idToSort.add(selection.getOrderingIndex(orderingColumn));
sorters.add(orderingColumn.type);
}
return idToSort.size() == 1 ? new SingleColumnComparator(idToSort.get(0), sorters.get(0))
: new CompositeComparator(sorters, idToSort);
}
- private Map<ColumnDefinition, Integer> getOrderingIndex(CFMetaData cfm, Selection selection, boolean isJson)
- throws InvalidRequestException
+ private boolean isReversed(TableMetadata table, Map<ColumnMetadata, Boolean> orderingColumns, StatementRestrictions restrictions) throws InvalidRequestException
{
- // If we order post-query (see orderResults), the sorted column needs to be in the ResultSet for sorting,
- // even if we don't
- // ultimately ship them to the client (CASSANDRA-4911).
- for (ColumnDefinition.Raw raw : parameters.orderings.keySet())
- {
- final ColumnDefinition def = raw.prepare(cfm);
- selection.addColumnForOrdering(def);
- }
- return selection.getOrderingIndex(isJson);
- }
-
- private boolean isReversed(CFMetaData cfm) throws InvalidRequestException
- {
- Boolean[] reversedMap = new Boolean[cfm.clusteringColumns().size()];
+ Boolean[] reversedMap = new Boolean[table.clusteringColumns().size()];
int i = 0;
- for (Map.Entry<ColumnDefinition.Raw, Boolean> entry : parameters.orderings.entrySet())
+ for (Map.Entry<ColumnMetadata, Boolean> entry : orderingColumns.entrySet())
{
- ColumnDefinition def = entry.getKey().prepare(cfm);
+ ColumnMetadata def = entry.getKey();
boolean reversed = entry.getValue();
checkTrue(def.isClusteringColumn(),
"Order by is currently only supported on the clustered columns of the PRIMARY KEY, got %s", def.name);
- checkTrue(i++ == def.position(),
- "Order by currently only support the ordering of columns following their declared order in the PRIMARY KEY");
-
+ while (i != def.position())
+ {
+ checkTrue(restrictions.isColumnRestrictedByEq(table.clusteringColumns().get(i++)),
+ "Order by currently only supports the ordering of columns following their declared order in the PRIMARY KEY");
+ }
+ i++;
reversedMap[def.position()] = (reversed != def.isReversedType());
}
@@ -1235,19 +1265,19 @@
private ColumnSpecification limitReceiver()
{
- return new ColumnSpecification(keyspace(), columnFamily(), new ColumnIdentifier("[limit]", true), Int32Type.instance);
+ return new ColumnSpecification(keyspace(), name(), new ColumnIdentifier("[limit]", true), Int32Type.instance);
}
private ColumnSpecification perPartitionLimitReceiver()
{
- return new ColumnSpecification(keyspace(), columnFamily(), new ColumnIdentifier("[per_partition_limit]", true), Int32Type.instance);
+ return new ColumnSpecification(keyspace(), name(), new ColumnIdentifier("[per_partition_limit]", true), Int32Type.instance);
}
@Override
public String toString()
{
return MoreObjects.toStringHelper(this)
- .add("name", cfName)
+ .add("name", qualifiedName)
.add("selectClause", selectClause)
.add("whereClause", whereClause)
.add("isDistinct", parameters.isDistinct)
@@ -1258,14 +1288,14 @@
public static class Parameters
{
// Public because CASSANDRA-9858
- public final Map<ColumnDefinition.Raw, Boolean> orderings;
- public final List<ColumnDefinition.Raw> groups;
+ public final Map<ColumnMetadata.Raw, Boolean> orderings;
+ public final List<ColumnMetadata.Raw> groups;
public final boolean isDistinct;
public final boolean allowFiltering;
public final boolean isJson;
- public Parameters(Map<ColumnDefinition.Raw, Boolean> orderings,
- List<ColumnDefinition.Raw> groups,
+ public Parameters(Map<ColumnMetadata.Raw, Boolean> orderings,
+ List<ColumnMetadata.Raw> groups,
boolean isDistinct,
boolean allowFiltering,
boolean isJson)
@@ -1339,4 +1369,10 @@
return 0;
}
}
+
+ @Override
+ public String toString()
+ {
+ return ToStringBuilder.reflectionToString(this, ToStringStyle.SHORT_PREFIX_STYLE);
+ }
}
diff --git a/src/java/org/apache/cassandra/cql3/statements/SingleTableUpdatesCollector.java b/src/java/org/apache/cassandra/cql3/statements/SingleTableUpdatesCollector.java
new file mode 100644
index 0000000..6ef551d
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/statements/SingleTableUpdatesCollector.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.statements;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.cassandra.db.ConsistencyLevel;
+import org.apache.cassandra.db.CounterMutation;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.IMutation;
+import org.apache.cassandra.db.Mutation;
+import org.apache.cassandra.db.RegularAndStaticColumns;
+import org.apache.cassandra.db.partitions.PartitionUpdate;
+import org.apache.cassandra.db.virtual.VirtualMutation;
+import org.apache.cassandra.schema.TableMetadata;
+
+/**
+ * Utility class to collect updates.
+ */
+final class SingleTableUpdatesCollector implements UpdatesCollector
+{
+ /**
+ * the table to be updated
+ */
+ private final TableMetadata metadata;
+
+ /**
+ * the columns to update
+ */
+ private final RegularAndStaticColumns updatedColumns;
+
+ /**
+ * The estimated number of updated row.
+ */
+ private final int updatedRows;
+
+ /**
+ * the partition update builders per key
+ */
+ private final Map<ByteBuffer, PartitionUpdate.Builder> puBuilders = new HashMap<>();
+
+ /**
+ * if it is a counter table, we will set this
+ */
+ private ConsistencyLevel counterConsistencyLevel = null;
+
+ SingleTableUpdatesCollector(TableMetadata metadata, RegularAndStaticColumns updatedColumns, int updatedRows)
+ {
+ this.metadata = metadata;
+ this.updatedColumns = updatedColumns;
+ this.updatedRows = updatedRows;
+ }
+
+ public PartitionUpdate.Builder getPartitionUpdateBuilder(TableMetadata metadata, DecoratedKey dk, ConsistencyLevel consistency)
+ {
+ if (metadata.isCounter())
+ counterConsistencyLevel = consistency;
+ return puBuilders.computeIfAbsent(dk.getKey(), (k) -> new PartitionUpdate.Builder(metadata, dk, updatedColumns, updatedRows));
+ }
+
+ /**
+ * Returns a collection containing all the mutations.
+ * @return a collection containing all the mutations.
+ */
+ public List<IMutation> toMutations()
+ {
+ List<IMutation> ms = new ArrayList<>();
+ for (PartitionUpdate.Builder builder : puBuilders.values())
+ {
+ IMutation mutation;
+
+ if (metadata.isVirtual())
+ mutation = new VirtualMutation(builder.build());
+ else if (metadata.isCounter())
+ mutation = new CounterMutation(new Mutation(builder.build()), counterConsistencyLevel);
+ else
+ mutation = new Mutation(builder.build());
+
+ mutation.validateIndexedColumns();
+ ms.add(mutation);
+ }
+
+ return ms;
+ }
+}
diff --git a/src/java/org/apache/cassandra/cql3/statements/TableAttributes.java b/src/java/org/apache/cassandra/cql3/statements/TableAttributes.java
deleted file mode 100644
index 04b9532..0000000
--- a/src/java/org/apache/cassandra/cql3/statements/TableAttributes.java
+++ /dev/null
@@ -1,233 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.cql3.statements;
-
-import java.util.Map;
-import java.util.Set;
-import java.util.UUID;
-
-import com.google.common.collect.ImmutableSet;
-
-import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.exceptions.SyntaxException;
-import org.apache.cassandra.schema.*;
-import org.apache.cassandra.schema.TableParams.Option;
-import org.apache.cassandra.service.ClientWarn;
-
-import static java.lang.String.format;
-
-public final class TableAttributes extends PropertyDefinitions
-{
- private static final String KW_ID = "id";
- private static final Set<String> validKeywords;
- private static final Set<String> obsoleteKeywords;
-
- private static boolean loggedReadRepairChanceDeprecationWarnings;
-
- static
- {
- ImmutableSet.Builder<String> validBuilder = ImmutableSet.builder();
- for (Option option : Option.values())
- validBuilder.add(option.toString());
- validBuilder.add(KW_ID);
- validKeywords = validBuilder.build();
- obsoleteKeywords = ImmutableSet.of();
- }
-
- public void validate()
- {
- validate(validKeywords, obsoleteKeywords);
- build(TableParams.builder()).validate();
- }
-
- public TableParams asNewTableParams()
- {
- return build(TableParams.builder());
- }
-
- public TableParams asAlteredTableParams(TableParams previous)
- {
- if (getId() != null)
- throw new ConfigurationException("Cannot alter table id.");
- return build(TableParams.builder(previous));
- }
-
- public UUID getId() throws ConfigurationException
- {
- String id = getSimple(KW_ID);
- try
- {
- return id != null ? UUID.fromString(id) : null;
- }
- catch (IllegalArgumentException e)
- {
- throw new ConfigurationException("Invalid table id", e);
- }
- }
-
- private TableParams build(TableParams.Builder builder)
- {
- if (hasOption(Option.BLOOM_FILTER_FP_CHANCE))
- builder.bloomFilterFpChance(getDouble(Option.BLOOM_FILTER_FP_CHANCE));
-
- if (hasOption(Option.CACHING))
- builder.caching(CachingParams.fromMap(getMap(Option.CACHING)));
-
- if (hasOption(Option.COMMENT))
- builder.comment(getString(Option.COMMENT));
-
- if (hasOption(Option.COMPACTION))
- builder.compaction(CompactionParams.fromMap(getMap(Option.COMPACTION)));
-
- if (hasOption(Option.COMPRESSION))
- {
- //crc_check_chance was "promoted" from a compression property to a top-level-property after #9839
- //so we temporarily accept it to be defined as a compression option, to maintain backwards compatibility
- Map<String, String> compressionOpts = getMap(Option.COMPRESSION);
- if (compressionOpts.containsKey(Option.CRC_CHECK_CHANCE.toString().toLowerCase()))
- {
- Double crcCheckChance = getDeprecatedCrcCheckChance(compressionOpts);
- builder.crcCheckChance(crcCheckChance);
- }
- builder.compression(CompressionParams.fromMap(getMap(Option.COMPRESSION)));
- }
-
- if (hasOption(Option.DCLOCAL_READ_REPAIR_CHANCE))
- {
- double chance = getDouble(Option.DCLOCAL_READ_REPAIR_CHANCE);
-
- if (chance != 0.0)
- {
- ClientWarn.instance.warn("dclocal_read_repair_chance table option has been deprecated and will be removed in version 4.0");
- maybeLogReadRepairChanceDeprecationWarning();
- }
-
- builder.dcLocalReadRepairChance(chance);
- }
-
- if (hasOption(Option.DEFAULT_TIME_TO_LIVE))
- builder.defaultTimeToLive(getInt(Option.DEFAULT_TIME_TO_LIVE));
-
- if (hasOption(Option.GC_GRACE_SECONDS))
- builder.gcGraceSeconds(getInt(Option.GC_GRACE_SECONDS));
-
- if (hasOption(Option.MAX_INDEX_INTERVAL))
- builder.maxIndexInterval(getInt(Option.MAX_INDEX_INTERVAL));
-
- if (hasOption(Option.MEMTABLE_FLUSH_PERIOD_IN_MS))
- builder.memtableFlushPeriodInMs(getInt(Option.MEMTABLE_FLUSH_PERIOD_IN_MS));
-
- if (hasOption(Option.MIN_INDEX_INTERVAL))
- builder.minIndexInterval(getInt(Option.MIN_INDEX_INTERVAL));
-
- if (hasOption(Option.READ_REPAIR_CHANCE))
- {
- double chance = getDouble(Option.READ_REPAIR_CHANCE);
-
- if (chance != 0.0)
- {
- ClientWarn.instance.warn("read_repair_chance table option has been deprecated and will be removed in version 4.0");
- maybeLogReadRepairChanceDeprecationWarning();
- }
-
- builder.readRepairChance(chance);
- }
-
- if (hasOption(Option.SPECULATIVE_RETRY))
- builder.speculativeRetry(SpeculativeRetryParam.fromString(getString(Option.SPECULATIVE_RETRY)));
-
- if (hasOption(Option.CRC_CHECK_CHANCE))
- builder.crcCheckChance(getDouble(Option.CRC_CHECK_CHANCE));
-
- if (hasOption(Option.CDC))
- builder.cdc(getBoolean(Option.CDC.toString(), false));
-
- return builder.build();
- }
-
- private void maybeLogReadRepairChanceDeprecationWarning()
- {
- if (!loggedReadRepairChanceDeprecationWarnings)
- {
- logger.warn("dclocal_read_repair_chance and read_repair_chance table options have been deprecated and will be removed in version 4.0");
- loggedReadRepairChanceDeprecationWarnings = true;
- }
- }
-
- private Double getDeprecatedCrcCheckChance(Map<String, String> compressionOpts)
- {
- String value = compressionOpts.get(Option.CRC_CHECK_CHANCE.toString().toLowerCase());
- try
- {
- return Double.valueOf(value);
- }
- catch (NumberFormatException e)
- {
- throw new SyntaxException(String.format("Invalid double value %s for crc_check_chance.'", value));
- }
- }
-
- private double getDouble(Option option)
- {
- String value = getString(option);
-
- try
- {
- return Double.parseDouble(value);
- }
- catch (NumberFormatException e)
- {
- throw new SyntaxException(format("Invalid double value %s for '%s'", value, option));
- }
- }
-
- private int getInt(Option option)
- {
- String value = getString(option);
-
- try
- {
- return Integer.parseInt(value);
- }
- catch (NumberFormatException e)
- {
- throw new SyntaxException(String.format("Invalid integer value %s for '%s'", value, option));
- }
- }
-
- private String getString(Option option)
- {
- String value = getSimple(option.toString());
- if (value == null)
- throw new IllegalStateException(format("Option '%s' is absent", option));
- return value;
- }
-
- private Map<String, String> getMap(Option option)
- {
- Map<String, String> value = getMap(option.toString());
- if (value == null)
- throw new IllegalStateException(format("Option '%s' is absent", option));
- return value;
- }
-
- private boolean hasOption(Option option)
- {
- return hasProperty(option.toString());
- }
-}
diff --git a/src/java/org/apache/cassandra/cql3/statements/TruncateStatement.java b/src/java/org/apache/cassandra/cql3/statements/TruncateStatement.java
index 302d2e2..206d116 100644
--- a/src/java/org/apache/cassandra/cql3/statements/TruncateStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/TruncateStatement.java
@@ -19,55 +19,56 @@
import java.util.concurrent.TimeoutException;
+import org.apache.cassandra.audit.AuditLogContext;
+import org.apache.cassandra.audit.AuditLogEntryType;
import org.apache.cassandra.auth.Permission;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.Schema;
import org.apache.cassandra.cql3.*;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.exceptions.*;
-import org.apache.cassandra.transport.messages.ResultMessage;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.service.ClientState;
import org.apache.cassandra.service.QueryState;
import org.apache.cassandra.service.StorageProxy;
-import org.apache.cassandra.thrift.ThriftValidation;
+import org.apache.cassandra.transport.messages.ResultMessage;
+import org.apache.commons.lang3.builder.ToStringBuilder;
+import org.apache.commons.lang3.builder.ToStringStyle;
-public class TruncateStatement extends CFStatement implements CQLStatement
+public class TruncateStatement extends QualifiedStatement implements CQLStatement
{
- public TruncateStatement(CFName name)
+ public TruncateStatement(QualifiedName name)
{
super(name);
}
- public int getBoundTerms()
+ public TruncateStatement prepare(ClientState state)
{
- return 0;
+ return this;
}
- public Prepared prepare(ClientState clientState) throws InvalidRequestException
+ public void authorize(ClientState state) throws InvalidRequestException, UnauthorizedException
{
- return new Prepared(this);
- }
-
- public void checkAccess(ClientState state) throws InvalidRequestException, UnauthorizedException
- {
- state.hasColumnFamilyAccess(keyspace(), columnFamily(), Permission.MODIFY);
+ state.ensureTablePermission(keyspace(), name(), Permission.MODIFY);
}
public void validate(ClientState state) throws InvalidRequestException
{
- ThriftValidation.validateColumnFamily(keyspace(), columnFamily());
+ Schema.instance.validateTable(keyspace(), name());
}
public ResultMessage execute(QueryState state, QueryOptions options, long queryStartNanoTime) throws InvalidRequestException, TruncateException
{
try
{
- CFMetaData metaData = Schema.instance.getCFMetaData(keyspace(), columnFamily());
+ TableMetadata metaData = Schema.instance.getTableMetadata(keyspace(), name());
if (metaData.isView())
throw new InvalidRequestException("Cannot TRUNCATE materialized view directly; must truncate base table instead");
- StorageProxy.truncateBlocking(keyspace(), columnFamily());
+ if (metaData.isVirtual())
+ throw new InvalidRequestException("Cannot truncate virtual tables");
+
+ StorageProxy.truncateBlocking(keyspace(), name());
}
catch (UnavailableException | TimeoutException e)
{
@@ -76,11 +77,18 @@
return null;
}
- public ResultMessage executeInternal(QueryState state, QueryOptions options)
+ public ResultMessage executeLocally(QueryState state, QueryOptions options)
{
try
{
- ColumnFamilyStore cfs = Keyspace.open(keyspace()).getColumnFamilyStore(columnFamily());
+ TableMetadata metaData = Schema.instance.getTableMetadata(keyspace(), name());
+ if (metaData.isView())
+ throw new InvalidRequestException("Cannot TRUNCATE materialized view directly; must truncate base table instead");
+
+ if (metaData.isVirtual())
+ throw new InvalidRequestException("Cannot truncate virtual tables");
+
+ ColumnFamilyStore cfs = Keyspace.open(keyspace()).getColumnFamilyStore(name());
cfs.truncateBlocking();
}
catch (Exception e)
@@ -89,4 +97,16 @@
}
return null;
}
+
+ @Override
+ public String toString()
+ {
+ return ToStringBuilder.reflectionToString(this, ToStringStyle.SHORT_PREFIX_STYLE);
+ }
+
+ @Override
+ public AuditLogContext getAuditLogContext()
+ {
+ return new AuditLogContext(AuditLogEntryType.TRUNCATE, keyspace(), name());
+ }
}
diff --git a/src/java/org/apache/cassandra/cql3/statements/UpdateStatement.java b/src/java/org/apache/cassandra/cql3/statements/UpdateStatement.java
index 6638752..21323d2 100644
--- a/src/java/org/apache/cassandra/cql3/statements/UpdateStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/UpdateStatement.java
@@ -17,21 +17,27 @@
*/
package org.apache.cassandra.cql3.statements;
+import java.util.Collection;
import java.util.Collections;
import java.util.List;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.audit.AuditLogContext;
+import org.apache.cassandra.audit.AuditLogEntryType;
import org.apache.cassandra.cql3.*;
+import org.apache.cassandra.cql3.conditions.ColumnCondition;
+import org.apache.cassandra.cql3.conditions.Conditions;
import org.apache.cassandra.cql3.restrictions.StatementRestrictions;
import org.apache.cassandra.db.Clustering;
import org.apache.cassandra.db.CompactTables;
import org.apache.cassandra.db.Slice;
import org.apache.cassandra.db.partitions.PartitionUpdate;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.Pair;
+import org.apache.commons.lang3.builder.ToStringBuilder;
+import org.apache.commons.lang3.builder.ToStringStyle;
-import static com.google.common.collect.Lists.newArrayList;
import static org.apache.cassandra.cql3.statements.RequestValidations.checkContainsNoDuplicates;
import static org.apache.cassandra.cql3.statements.RequestValidations.checkFalse;
import static org.apache.cassandra.cql3.statements.RequestValidations.checkTrue;
@@ -45,23 +51,18 @@
private static final Constants.Value EMPTY = new Constants.Value(ByteBufferUtil.EMPTY_BYTE_BUFFER);
private UpdateStatement(StatementType type,
- int boundTerms,
- CFMetaData cfm,
+ VariableSpecifications bindVariables,
+ TableMetadata metadata,
Operations operations,
StatementRestrictions restrictions,
Conditions conditions,
Attributes attrs)
{
- super(type, boundTerms, cfm, operations, restrictions, conditions, attrs);
- }
-
- public boolean requireFullClusteringKey()
- {
- return true;
+ super(type, bindVariables, metadata, operations, restrictions, conditions, attrs);
}
@Override
- public void addUpdateForKey(PartitionUpdate update, Clustering clustering, UpdateParameters params)
+ public void addUpdateForKey(PartitionUpdate.Builder updateBuilder, Clustering clustering, UpdateParameters params)
{
if (updatesRegularRows())
{
@@ -70,50 +71,47 @@
// We update the row timestamp (ex-row marker) only on INSERT (#6782)
// Further, COMPACT tables semantic differs from "CQL3" ones in that a row exists only if it has
// a non-null column, so we don't want to set the row timestamp for them.
- if (type.isInsert() && cfm.isCQLTable())
+ if (type.isInsert() && metadata().isCQLTable())
params.addPrimaryKeyLivenessInfo();
List<Operation> updates = getRegularOperations();
- // For compact table, when we translate it to thrift, we don't have a row marker. So we don't accept an insert/update
- // that only sets the PK unless the is no declared non-PK columns (in the latter we just set the value empty).
-
- // For a dense layout, when we translate it to thrift, we don't have a row marker. So we don't accept an insert/update
- // that only sets the PK unless the is no declared non-PK columns (which we recognize because in that case the compact
- // value is of type "EmptyType").
- if ((cfm.isCompactTable() && !cfm.isSuper()) && updates.isEmpty())
+ // For compact table, we don't accept an insert/update that only sets the PK unless the is no
+ // declared non-PK columns (which we recognize because in that case
+ // the compact value is of type "EmptyType").
+ if (metadata().isCompactTable() && updates.isEmpty())
{
- checkTrue(CompactTables.hasEmptyCompactValue(cfm),
+ checkTrue(CompactTables.hasEmptyCompactValue(metadata),
"Column %s is mandatory for this COMPACT STORAGE table",
- cfm.compactValueColumn().name);
+ metadata().compactValueColumn.name);
- updates = Collections.<Operation>singletonList(new Constants.Setter(cfm.compactValueColumn(), EMPTY));
+ updates = Collections.<Operation>singletonList(new Constants.Setter(metadata().compactValueColumn, EMPTY));
}
for (Operation op : updates)
- op.execute(update.partitionKey(), params);
+ op.execute(updateBuilder.partitionKey(), params);
- update.add(params.buildRow());
+ updateBuilder.add(params.buildRow());
}
if (updatesStaticRow())
{
params.newRow(Clustering.STATIC_CLUSTERING);
for (Operation op : getStaticOperations())
- op.execute(update.partitionKey(), params);
- update.add(params.buildRow());
+ op.execute(updateBuilder.partitionKey(), params);
+ updateBuilder.add(params.buildRow());
}
}
@Override
- public void addUpdateForKey(PartitionUpdate update, Slice slice, UpdateParameters params)
+ public void addUpdateForKey(PartitionUpdate.Builder update, Slice slice, UpdateParameters params)
{
throw new UnsupportedOperationException();
}
public static class ParsedInsert extends ModificationStatement.Parsed
{
- private final List<ColumnDefinition.Raw> columnNames;
+ private final List<ColumnMetadata.Raw> columnNames;
private final List<Term.Raw> columnValues;
/**
@@ -125,9 +123,9 @@
* @param columnValues list of column values (corresponds to names)
* @param ifNotExists true if an IF NOT EXISTS condition was specified, false otherwise
*/
- public ParsedInsert(CFName name,
+ public ParsedInsert(QualifiedName name,
Attributes.Raw attrs,
- List<ColumnDefinition.Raw> columnNames,
+ List<ColumnMetadata.Raw> columnNames,
List<Term.Raw> columnValues,
boolean ifNotExists)
{
@@ -137,14 +135,14 @@
}
@Override
- protected ModificationStatement prepareInternal(CFMetaData cfm,
- VariableSpecifications boundNames,
+ protected ModificationStatement prepareInternal(TableMetadata metadata,
+ VariableSpecifications bindVariables,
Conditions conditions,
Attributes attrs)
{
// Created from an INSERT
- checkFalse(cfm.isCounter(), "INSERT statements are not allowed on counter tables, use UPDATE instead");
+ checkFalse(metadata.isCounter(), "INSERT statements are not allowed on counter tables, use UPDATE instead");
checkFalse(columnNames == null, "Column names for INSERT must be provided when using VALUES");
checkFalse(columnNames.isEmpty(), "No columns provided to INSERT");
@@ -155,50 +153,40 @@
Operations operations = new Operations(type);
boolean hasClusteringColumnsSet = false;
- if (cfm.isSuper() && cfm.isDense())
+ for (int i = 0; i < columnNames.size(); i++)
{
- // SuperColumn familiy updates are always row-level
- hasClusteringColumnsSet = true;
- SuperColumnCompatibility.prepareInsertOperations(cfm, columnNames, whereClause, columnValues, boundNames, operations);
- }
- else
- {
- for (int i = 0; i < columnNames.size(); i++)
+ ColumnMetadata def = getColumnDefinition(metadata, columnNames.get(i));
+
+ if (def.isClusteringColumn())
+ hasClusteringColumnsSet = true;
+
+ Term.Raw value = columnValues.get(i);
+
+ if (def.isPrimaryKeyColumn())
{
- ColumnDefinition def = getColumnDefinition(cfm, columnNames.get(i));
-
- if (def.isClusteringColumn())
- hasClusteringColumnsSet = true;
-
- Term.Raw value = columnValues.get(i);
-
- if (def.isPrimaryKeyColumn())
- {
- whereClause.add(new SingleColumnRelation(columnNames.get(i), Operator.EQ, value));
- }
- else
- {
- Operation operation = new Operation.SetValue(value).prepare(cfm, def);
- operation.collectMarkerSpecification(boundNames);
- operations.add(operation);
- }
+ whereClause.add(new SingleColumnRelation(columnNames.get(i), Operator.EQ, value));
+ }
+ else
+ {
+ Operation operation = new Operation.SetValue(value).prepare(metadata, def);
+ operation.collectMarkerSpecification(bindVariables);
+ operations.add(operation);
}
}
boolean applyOnlyToStaticColumns = !hasClusteringColumnsSet && appliesOnlyToStaticColumns(operations, conditions);
StatementRestrictions restrictions = new StatementRestrictions(type,
- cfm,
+ metadata,
whereClause.build(),
- boundNames,
+ bindVariables,
applyOnlyToStaticColumns,
false,
- false,
false);
return new UpdateStatement(type,
- boundNames.size(),
- cfm,
+ bindVariables,
+ metadata,
operations,
restrictions,
conditions,
@@ -214,7 +202,7 @@
private final Json.Raw jsonValue;
private final boolean defaultUnset;
- public ParsedInsertJson(CFName name, Attributes.Raw attrs, Json.Raw jsonValue, boolean defaultUnset, boolean ifNotExists)
+ public ParsedInsertJson(QualifiedName name, Attributes.Raw attrs, Json.Raw jsonValue, boolean defaultUnset, boolean ifNotExists)
{
super(name, StatementType.INSERT, attrs, null, ifNotExists, false);
this.jsonValue = jsonValue;
@@ -222,33 +210,21 @@
}
@Override
- protected ModificationStatement prepareInternal(CFMetaData cfm,
- VariableSpecifications boundNames,
+ protected ModificationStatement prepareInternal(TableMetadata metadata,
+ VariableSpecifications bindVariables,
Conditions conditions,
Attributes attrs)
{
- checkFalse(cfm.isCounter(), "INSERT statements are not allowed on counter tables, use UPDATE instead");
+ checkFalse(metadata.isCounter(), "INSERT statements are not allowed on counter tables, use UPDATE instead");
- List<ColumnDefinition> defs = newArrayList(cfm.allColumnsInSelectOrder());
- Json.Prepared prepared = jsonValue.prepareAndCollectMarkers(cfm, defs, boundNames);
+ Collection<ColumnMetadata> defs = metadata.columns();
+ Json.Prepared prepared = jsonValue.prepareAndCollectMarkers(metadata, defs, bindVariables);
WhereClause.Builder whereClause = new WhereClause.Builder();
Operations operations = new Operations(type);
boolean hasClusteringColumnsSet = false;
- if (cfm.isSuper() && cfm.isDense())
- {
- hasClusteringColumnsSet = true;
- SuperColumnCompatibility.prepareInsertJSONOperations(cfm, defs, boundNames, prepared, whereClause, operations);
- }
- else
- {
-
-
-
-// TODO: indent
-
- for (ColumnDefinition def : defs)
+ for (ColumnMetadata def : defs)
{
if (def.isClusteringColumn())
hasClusteringColumnsSet = true;
@@ -256,35 +232,29 @@
Term.Raw raw = prepared.getRawTermForColumn(def, defaultUnset);
if (def.isPrimaryKeyColumn())
{
- whereClause.add(new SingleColumnRelation(ColumnDefinition.Raw.forColumn(def), Operator.EQ, raw));
+ whereClause.add(new SingleColumnRelation(ColumnMetadata.Raw.forColumn(def), Operator.EQ, raw));
}
else
{
- Operation operation = new Operation.SetValue(raw).prepare(cfm, def);
- operation.collectMarkerSpecification(boundNames);
+ Operation operation = new Operation.SetValue(raw).prepare(metadata, def);
+ operation.collectMarkerSpecification(bindVariables);
operations.add(operation);
}
}
-
-
-
- }
-
boolean applyOnlyToStaticColumns = !hasClusteringColumnsSet && appliesOnlyToStaticColumns(operations, conditions);
StatementRestrictions restrictions = new StatementRestrictions(type,
- cfm,
+ metadata,
whereClause.build(),
- boundNames,
+ bindVariables,
applyOnlyToStaticColumns,
false,
- false,
false);
return new UpdateStatement(type,
- boundNames.size(),
- cfm,
+ bindVariables,
+ metadata,
operations,
restrictions,
conditions,
@@ -295,8 +265,8 @@
public static class ParsedUpdate extends ModificationStatement.Parsed
{
// Provided for an UPDATE
- private final List<Pair<ColumnDefinition.Raw, Operation.RawUpdate>> updates;
- private WhereClause whereClause;
+ private final List<Pair<ColumnMetadata.Raw, Operation.RawUpdate>> updates;
+ private final WhereClause whereClause;
/**
* Creates a new UpdateStatement from a column family name, columns map, consistency
@@ -308,11 +278,11 @@
* @param whereClause the where clause
* @param ifExists flag to check if row exists
* */
- public ParsedUpdate(CFName name,
+ public ParsedUpdate(QualifiedName name,
Attributes.Raw attrs,
- List<Pair<ColumnDefinition.Raw, Operation.RawUpdate>> updates,
+ List<Pair<ColumnMetadata.Raw, Operation.RawUpdate>> updates,
WhereClause whereClause,
- List<Pair<ColumnDefinition.Raw, ColumnCondition.Raw>> conditions,
+ List<Pair<ColumnMetadata.Raw, ColumnCondition.Raw>> conditions,
boolean ifExists)
{
super(name, StatementType.UPDATE, attrs, conditions, false, ifExists);
@@ -321,45 +291,49 @@
}
@Override
- protected ModificationStatement prepareInternal(CFMetaData cfm,
- VariableSpecifications boundNames,
+ protected ModificationStatement prepareInternal(TableMetadata metadata,
+ VariableSpecifications bindVariables,
Conditions conditions,
Attributes attrs)
{
Operations operations = new Operations(type);
- if (cfm.isSuper() && cfm.isDense())
+ for (Pair<ColumnMetadata.Raw, Operation.RawUpdate> entry : updates)
{
- conditions = SuperColumnCompatibility.rebuildLWTColumnConditions(conditions, cfm, whereClause);
- whereClause = SuperColumnCompatibility.prepareUpdateOperations(cfm, whereClause, updates, boundNames, operations);
- }
- else
- {
- for (Pair<ColumnDefinition.Raw, Operation.RawUpdate> entry : updates)
- {
- ColumnDefinition def = getColumnDefinition(cfm, entry.left);
+ ColumnMetadata def = getColumnDefinition(metadata, entry.left);
- checkFalse(def.isPrimaryKeyColumn(), "PRIMARY KEY part %s found in SET part", def.name);
+ checkFalse(def.isPrimaryKeyColumn(), "PRIMARY KEY part %s found in SET part", def.name);
- Operation operation = entry.right.prepare(cfm, def);
- operation.collectMarkerSpecification(boundNames);
- operations.add(operation);
- }
+ Operation operation = entry.right.prepare(metadata, def);
+ operation.collectMarkerSpecification(bindVariables);
+ operations.add(operation);
}
-
- StatementRestrictions restrictions = newRestrictions(cfm,
- boundNames,
+
+ StatementRestrictions restrictions = newRestrictions(metadata,
+ bindVariables,
operations,
whereClause,
conditions);
return new UpdateStatement(type,
- boundNames.size(),
- cfm,
+ bindVariables,
+ metadata,
operations,
restrictions,
conditions,
attrs);
}
}
+
+ @Override
+ public String toString()
+ {
+ return ToStringBuilder.reflectionToString(this, ToStringStyle.SHORT_PREFIX_STYLE);
+ }
+
+ @Override
+ public AuditLogContext getAuditLogContext()
+ {
+ return new AuditLogContext(AuditLogEntryType.UPDATE, keyspace(), columnFamily());
+ }
}
diff --git a/src/java/org/apache/cassandra/cql3/statements/UpdatesCollector.java b/src/java/org/apache/cassandra/cql3/statements/UpdatesCollector.java
index 1d65a78..c3dd334 100644
--- a/src/java/org/apache/cassandra/cql3/statements/UpdatesCollector.java
+++ b/src/java/org/apache/cassandra/cql3/statements/UpdatesCollector.java
@@ -15,126 +15,19 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.cassandra.cql3.statements;
-import java.nio.ByteBuffer;
-import java.util.*;
+import java.util.List;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.db.*;
+import org.apache.cassandra.db.ConsistencyLevel;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.IMutation;
import org.apache.cassandra.db.partitions.PartitionUpdate;
+import org.apache.cassandra.schema.TableMetadata;
-/**
- * Utility class to collect updates.
- *
- * <p>In a batch statement we don't want to recreate mutations every time as this is particularly inefficient when
- * applying multiple batch to the same partition (see #6737). </p>
- *
- */
-final class UpdatesCollector
+public interface UpdatesCollector
{
- /**
- * The columns that will be updated for each table (keyed by the table ID).
- */
- private final Map<UUID, PartitionColumns> updatedColumns;
-
- /**
- * The estimated number of updated row.
- */
- private final int updatedRows;
-
- /**
- * The mutations per keyspace.
- */
- private final Map<String, Map<ByteBuffer, IMutation>> mutations = new HashMap<>();
-
- public UpdatesCollector(Map<UUID, PartitionColumns> updatedColumns, int updatedRows)
- {
- super();
- this.updatedColumns = updatedColumns;
- this.updatedRows = updatedRows;
- }
-
- /**
- * Gets the <code>PartitionUpdate</code> for the specified column family and key. If the update does not
- * exist it will be created.
- *
- * @param cfm the column family meta data
- * @param dk the partition key
- * @param consistency the consistency level
- * @return the <code>PartitionUpdate</code> for the specified column family and key
- */
- public PartitionUpdate getPartitionUpdate(CFMetaData cfm, DecoratedKey dk, ConsistencyLevel consistency)
- {
- Mutation mut = getMutation(cfm, dk, consistency);
- PartitionUpdate upd = mut.get(cfm);
- if (upd == null)
- {
- PartitionColumns columns = updatedColumns.get(cfm.cfId);
- assert columns != null;
- upd = new PartitionUpdate(cfm, dk, columns, updatedRows);
- mut.add(upd);
- }
- return upd;
- }
-
- /**
- * Check all partition updates contain only valid values for any
- * indexed columns.
- */
- public void validateIndexedColumns()
- {
- for (Map<ByteBuffer, IMutation> perKsMutations : mutations.values())
- for (IMutation mutation : perKsMutations.values())
- for (PartitionUpdate update : mutation.getPartitionUpdates())
- Keyspace.openAndGetStore(update.metadata()).indexManager.validate(update);
- }
-
- private Mutation getMutation(CFMetaData cfm, DecoratedKey dk, ConsistencyLevel consistency)
- {
- String ksName = cfm.ksName;
- IMutation mutation = keyspaceMap(ksName).get(dk.getKey());
- if (mutation == null)
- {
- Mutation mut = new Mutation(ksName, dk);
- mutation = cfm.isCounter() ? new CounterMutation(mut, consistency) : mut;
- keyspaceMap(ksName).put(dk.getKey(), mutation);
- return mut;
- }
- return cfm.isCounter() ? ((CounterMutation) mutation).getMutation() : (Mutation) mutation;
- }
-
- /**
- * Returns a collection containing all the mutations.
- * @return a collection containing all the mutations.
- */
- public Collection<IMutation> toMutations()
- {
- // The case where all statement where on the same keyspace is pretty common
- if (mutations.size() == 1)
- return mutations.values().iterator().next().values();
-
- List<IMutation> ms = new ArrayList<>();
- for (Map<ByteBuffer, IMutation> ksMap : mutations.values())
- ms.addAll(ksMap.values());
-
- return ms;
- }
-
- /**
- * Returns the key-mutation mappings for the specified keyspace.
- *
- * @param ksName the keyspace name
- * @return the key-mutation mappings for the specified keyspace.
- */
- private Map<ByteBuffer, IMutation> keyspaceMap(String ksName)
- {
- Map<ByteBuffer, IMutation> ksMap = mutations.get(ksName);
- if (ksMap == null)
- {
- ksMap = new HashMap<>();
- mutations.put(ksName, ksMap);
- }
- return ksMap;
- }
+ PartitionUpdate.Builder getPartitionUpdateBuilder(TableMetadata metadata, DecoratedKey dk, ConsistencyLevel consistency);
+ List<IMutation> toMutations();
}
diff --git a/src/java/org/apache/cassandra/cql3/statements/UseStatement.java b/src/java/org/apache/cassandra/cql3/statements/UseStatement.java
index 0242f09..3013d9f 100644
--- a/src/java/org/apache/cassandra/cql3/statements/UseStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/UseStatement.java
@@ -17,6 +17,8 @@
*/
package org.apache.cassandra.cql3.statements;
+import org.apache.cassandra.audit.AuditLogContext;
+import org.apache.cassandra.audit.AuditLogEntryType;
import org.apache.cassandra.cql3.CQLStatement;
import org.apache.cassandra.cql3.QueryOptions;
import org.apache.cassandra.exceptions.InvalidRequestException;
@@ -24,8 +26,10 @@
import org.apache.cassandra.transport.messages.ResultMessage;
import org.apache.cassandra.service.ClientState;
import org.apache.cassandra.service.QueryState;
+import org.apache.commons.lang3.builder.ToStringBuilder;
+import org.apache.commons.lang3.builder.ToStringStyle;
-public class UseStatement extends ParsedStatement implements CQLStatement
+public class UseStatement extends CQLStatement.Raw implements CQLStatement
{
private final String keyspace;
@@ -34,17 +38,12 @@
this.keyspace = keyspace;
}
- public int getBoundTerms()
+ public UseStatement prepare(ClientState state)
{
- return 0;
+ return this;
}
- public Prepared prepare(ClientState clientState) throws InvalidRequestException
- {
- return new Prepared(this);
- }
-
- public void checkAccess(ClientState state) throws UnauthorizedException
+ public void authorize(ClientState state) throws UnauthorizedException
{
state.validateLogin();
}
@@ -59,10 +58,22 @@
return new ResultMessage.SetKeyspace(keyspace);
}
- public ResultMessage executeInternal(QueryState state, QueryOptions options) throws InvalidRequestException
+ public ResultMessage executeLocally(QueryState state, QueryOptions options) throws InvalidRequestException
{
// In production, internal queries are exclusively on the system keyspace and 'use' is thus useless
// but for some unit tests we need to set the keyspace (e.g. for tests with DROP INDEX)
return execute(state, options, System.nanoTime());
}
+
+ @Override
+ public String toString()
+ {
+ return ToStringBuilder.reflectionToString(this, ToStringStyle.SHORT_PREFIX_STYLE);
+ }
+
+ @Override
+ public AuditLogContext getAuditLogContext()
+ {
+ return new AuditLogContext(AuditLogEntryType.USE_KEYSPACE, keyspace);
+ }
}
diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/AlterKeyspaceStatement.java b/src/java/org/apache/cassandra/cql3/statements/schema/AlterKeyspaceStatement.java
new file mode 100644
index 0000000..2f0c188
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/statements/schema/AlterKeyspaceStatement.java
@@ -0,0 +1,203 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.statements.schema;
+
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import com.google.common.collect.ImmutableSet;
+
+import org.apache.cassandra.audit.AuditLogContext;
+import org.apache.cassandra.audit.AuditLogEntryType;
+import org.apache.cassandra.auth.Permission;
+import org.apache.cassandra.config.Config;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.CQLStatement;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.gms.Gossiper;
+import org.apache.cassandra.locator.AbstractReplicationStrategy;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.LocalStrategy;
+import org.apache.cassandra.locator.ReplicationFactor;
+import org.apache.cassandra.schema.KeyspaceMetadata;
+import org.apache.cassandra.schema.KeyspaceMetadata.KeyspaceDiff;
+import org.apache.cassandra.schema.Keyspaces;
+import org.apache.cassandra.schema.Keyspaces.KeyspacesDiff;
+import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.transport.Event.SchemaChange;
+import org.apache.cassandra.transport.Event.SchemaChange.Change;
+import org.apache.cassandra.utils.FBUtilities;
+
+public final class AlterKeyspaceStatement extends AlterSchemaStatement
+{
+ private static final boolean allow_alter_rf_during_range_movement = Boolean.getBoolean(Config.PROPERTY_PREFIX + "allow_alter_rf_during_range_movement");
+ private static final boolean allow_unsafe_transient_changes = Boolean.getBoolean(Config.PROPERTY_PREFIX + "allow_unsafe_transient_changes");
+
+ private final KeyspaceAttributes attrs;
+
+ public AlterKeyspaceStatement(String keyspaceName, KeyspaceAttributes attrs)
+ {
+ super(keyspaceName);
+ this.attrs = attrs;
+ }
+
+ public Keyspaces apply(Keyspaces schema)
+ {
+ attrs.validate();
+
+ KeyspaceMetadata keyspace = schema.getNullable(keyspaceName);
+ if (null == keyspace)
+ throw ire("Keyspace '%s' doesn't exist", keyspaceName);
+
+ KeyspaceMetadata newKeyspace = keyspace.withSwapped(attrs.asAlteredKeyspaceParams(keyspace.params));
+
+ if (newKeyspace.params.replication.klass.equals(LocalStrategy.class))
+ throw ire("Unable to use given strategy class: LocalStrategy is reserved for internal use.");
+
+ newKeyspace.params.validate(keyspaceName);
+
+ validateNoRangeMovements();
+ validateTransientReplication(keyspace.createReplicationStrategy(), newKeyspace.createReplicationStrategy());
+
+ return schema.withAddedOrUpdated(newKeyspace);
+ }
+
+ SchemaChange schemaChangeEvent(KeyspacesDiff diff)
+ {
+ return new SchemaChange(Change.UPDATED, keyspaceName);
+ }
+
+ public void authorize(ClientState client)
+ {
+ client.ensureKeyspacePermission(keyspaceName, Permission.ALTER);
+ }
+
+ @Override
+ Set<String> clientWarnings(KeyspacesDiff diff)
+ {
+ if (diff.isEmpty())
+ return ImmutableSet.of();
+
+ KeyspaceDiff keyspaceDiff = diff.altered.get(0);
+
+ AbstractReplicationStrategy before = keyspaceDiff.before.createReplicationStrategy();
+ AbstractReplicationStrategy after = keyspaceDiff.after.createReplicationStrategy();
+
+ return before.getReplicationFactor().fullReplicas < after.getReplicationFactor().fullReplicas
+ ? ImmutableSet.of("When increasing replication factor you need to run a full (-full) repair to distribute the data.")
+ : ImmutableSet.of();
+ }
+
+ private void validateNoRangeMovements()
+ {
+ if (allow_alter_rf_during_range_movement)
+ return;
+
+ Stream<InetAddressAndPort> endpoints = Stream.concat(Gossiper.instance.getLiveMembers().stream(), Gossiper.instance.getUnreachableMembers().stream());
+ List<InetAddressAndPort> notNormalEndpoints = endpoints.filter(endpoint -> !FBUtilities.getBroadcastAddressAndPort().equals(endpoint) &&
+ !Gossiper.instance.getEndpointStateForEndpoint(endpoint).isNormalState())
+ .collect(Collectors.toList());
+ if (!notNormalEndpoints.isEmpty())
+ {
+ throw new ConfigurationException("Cannot alter RF while some endpoints are not in normal state (no range movements): " + notNormalEndpoints);
+ }
+ }
+
+ private void validateTransientReplication(AbstractReplicationStrategy oldStrategy, AbstractReplicationStrategy newStrategy)
+ {
+ //If there is no read traffic there are some extra alterations you can safely make, but this is so atypical
+ //that a good default is to not allow unsafe changes
+ if (allow_unsafe_transient_changes)
+ return;
+
+ ReplicationFactor oldRF = oldStrategy.getReplicationFactor();
+ ReplicationFactor newRF = newStrategy.getReplicationFactor();
+
+ int oldTrans = oldRF.transientReplicas();
+ int oldFull = oldRF.fullReplicas;
+ int newTrans = newRF.transientReplicas();
+ int newFull = newRF.fullReplicas;
+
+ if (newTrans > 0)
+ {
+ if (DatabaseDescriptor.getNumTokens() > 1)
+ throw new ConfigurationException(String.format("Transient replication is not supported with vnodes yet"));
+
+ Keyspace ks = Keyspace.open(keyspaceName);
+ for (ColumnFamilyStore cfs : ks.getColumnFamilyStores())
+ {
+ if (cfs.viewManager.hasViews())
+ {
+ throw new ConfigurationException("Cannot use transient replication on keyspaces using materialized views");
+ }
+
+ if (cfs.indexManager.hasIndexes())
+ {
+ throw new ConfigurationException("Cannot use transient replication on keyspaces using secondary indexes");
+ }
+ }
+ }
+
+ //This is true right now because the transition from transient -> full lacks the pending state
+ //necessary for correctness. What would happen if we allowed this is that we would attempt
+ //to read from a transient replica as if it were a full replica.
+ if (oldFull > newFull && oldTrans > 0)
+ throw new ConfigurationException("Can't add full replicas if there are any transient replicas. You must first remove all transient replicas, then change the # of full replicas, then add back the transient replicas");
+
+ //Don't increase transient replication factor by more than one at a time if changing number of replicas
+ //Just like with changing full replicas it's not safe to do this as you could read from too many replicas
+ //that don't have the necessary data. W/O transient replication this alteration was allowed and it's not clear
+ //if it should be.
+ //This is structured so you can convert as many full replicas to transient replicas as you want.
+ boolean numReplicasChanged = oldTrans + oldFull != newTrans + newFull;
+ if (numReplicasChanged && (newTrans > oldTrans && newTrans != oldTrans + 1))
+ throw new ConfigurationException("Can only safely increase number of transients one at a time with incremental repair run in between each time");
+ }
+
+ @Override
+ public AuditLogContext getAuditLogContext()
+ {
+ return new AuditLogContext(AuditLogEntryType.ALTER_KEYSPACE, keyspaceName);
+ }
+
+ public String toString()
+ {
+ return String.format("%s (%s)", getClass().getSimpleName(), keyspaceName);
+ }
+
+ public static final class Raw extends CQLStatement.Raw
+ {
+ private final String keyspaceName;
+ private final KeyspaceAttributes attrs;
+
+ public Raw(String keyspaceName, KeyspaceAttributes attrs)
+ {
+ this.keyspaceName = keyspaceName;
+ this.attrs = attrs;
+ }
+
+ public AlterKeyspaceStatement prepare(ClientState state)
+ {
+ return new AlterKeyspaceStatement(keyspaceName, attrs);
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/AlterSchemaStatement.java b/src/java/org/apache/cassandra/cql3/statements/schema/AlterSchemaStatement.java
new file mode 100644
index 0000000..161c9c4
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/statements/schema/AlterSchemaStatement.java
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.statements.schema;
+
+import java.util.Set;
+
+import com.google.common.collect.ImmutableSet;
+
+import org.apache.cassandra.auth.AuthenticatedUser;
+import org.apache.cassandra.auth.IResource;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.CQLStatement;
+import org.apache.cassandra.cql3.QueryOptions;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.schema.*;
+import org.apache.cassandra.schema.Keyspaces.KeyspacesDiff;
+import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.service.ClientWarn;
+import org.apache.cassandra.service.QueryState;
+import org.apache.cassandra.transport.Event.SchemaChange;
+import org.apache.cassandra.transport.messages.ResultMessage;
+
+abstract class AlterSchemaStatement implements CQLStatement, SchemaTransformation
+{
+ protected final String keyspaceName; // name of the keyspace affected by the statement
+
+ protected AlterSchemaStatement(String keyspaceName)
+ {
+ this.keyspaceName = keyspaceName;
+ }
+
+ public final void validate(ClientState state)
+ {
+ // no-op; validation is performed while executing the statement, in apply()
+ }
+
+ public ResultMessage execute(QueryState state, QueryOptions options, long queryStartNanoTime)
+ {
+ return execute(state, false);
+ }
+
+ public ResultMessage executeLocally(QueryState state, QueryOptions options)
+ {
+ return execute(state, true);
+ }
+
+ /**
+ * TODO: document
+ */
+ abstract SchemaChange schemaChangeEvent(KeyspacesDiff diff);
+
+ /**
+ * Schema alteration may result in a new database object (keyspace, table, role, function) being created capable of
+ * having permissions GRANTed on it. The creator of the object (the primary role assigned to the AuthenticatedUser
+ * performing the operation) is automatically granted ALL applicable permissions on the object. This is a hook for
+ * subclasses to override in order indicate which resources to to perform that grant on when the statement is executed.
+ *
+ * Only called if the transformation resulted in a non-empty diff.
+ */
+ Set<IResource> createdResources(KeyspacesDiff diff)
+ {
+ return ImmutableSet.of();
+ }
+
+ /**
+ * Schema alteration might produce a client warning (e.g. a warning to run full repair when increading RF of a keyspace).
+ * This method should be used to generate them instead of calling warn() in transformation code.
+ *
+ * Only called if the transformation resulted in a non-empty diff.
+ */
+ Set<String> clientWarnings(KeyspacesDiff diff)
+ {
+ return ImmutableSet.of();
+ }
+
+ public ResultMessage execute(QueryState state, boolean locally)
+ {
+ if (SchemaConstants.isLocalSystemKeyspace(keyspaceName))
+ throw ire("System keyspace '%s' is not user-modifiable", keyspaceName);
+
+ KeyspaceMetadata keyspace = Schema.instance.getKeyspaceMetadata(keyspaceName);
+ if (null != keyspace && keyspace.isVirtual())
+ throw ire("Virtual keyspace '%s' is not user-modifiable", keyspaceName);
+
+ validateKeyspaceName();
+
+ KeyspacesDiff diff = MigrationManager.announce(this, locally);
+
+ clientWarnings(diff).forEach(ClientWarn.instance::warn);
+
+ if (diff.isEmpty())
+ return new ResultMessage.Void();
+
+ /*
+ * When a schema alteration results in a new db object being created, we grant permissions on the new
+ * object to the user performing the request if:
+ * - the user is not anonymous
+ * - the configured IAuthorizer supports granting of permissions (not all do, AllowAllAuthorizer doesn't and
+ * custom external implementations may not)
+ */
+ AuthenticatedUser user = state.getClientState().getUser();
+ if (null != user && !user.isAnonymous())
+ createdResources(diff).forEach(r -> grantPermissionsOnResource(r, user));
+
+ return new ResultMessage.SchemaChange(schemaChangeEvent(diff));
+ }
+
+ private void validateKeyspaceName()
+ {
+ if (!SchemaConstants.isValidName(keyspaceName))
+ {
+ throw ire("Keyspace name must not be empty, more than %d characters long, " +
+ "or contain non-alphanumeric-underscore characters (got '%s')",
+ SchemaConstants.NAME_LENGTH, keyspaceName);
+ }
+ }
+
+ private void grantPermissionsOnResource(IResource resource, AuthenticatedUser user)
+ {
+ try
+ {
+ DatabaseDescriptor.getAuthorizer()
+ .grant(AuthenticatedUser.SYSTEM_USER,
+ resource.applicablePermissions(),
+ resource,
+ user.getPrimaryRole());
+ }
+ catch (UnsupportedOperationException e)
+ {
+ // not a problem - grant is an optional method on IAuthorizer
+ }
+ }
+
+ static InvalidRequestException ire(String format, Object... args)
+ {
+ return new InvalidRequestException(String.format(format, args));
+ }
+}
diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/AlterTableStatement.java b/src/java/org/apache/cassandra/cql3/statements/schema/AlterTableStatement.java
new file mode 100644
index 0000000..b269762
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/statements/schema/AlterTableStatement.java
@@ -0,0 +1,469 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.statements.schema;
+
+import java.util.*;
+
+import org.apache.cassandra.audit.AuditLogContext;
+import org.apache.cassandra.audit.AuditLogEntryType;
+import org.apache.cassandra.auth.Permission;
+import org.apache.cassandra.cql3.*;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.schema.*;
+import org.apache.cassandra.schema.Keyspaces.KeyspacesDiff;
+import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.service.reads.repair.ReadRepairStrategy;
+import org.apache.cassandra.transport.Event.SchemaChange;
+import org.apache.cassandra.transport.Event.SchemaChange.Change;
+import org.apache.cassandra.transport.Event.SchemaChange.Target;
+import org.apache.cassandra.utils.FBUtilities;
+
+import static java.lang.String.join;
+
+import static com.google.common.collect.Iterables.isEmpty;
+import static com.google.common.collect.Iterables.transform;
+
+public abstract class AlterTableStatement extends AlterSchemaStatement
+{
+ protected final String tableName;
+
+ public AlterTableStatement(String keyspaceName, String tableName)
+ {
+ super(keyspaceName);
+ this.tableName = tableName;
+ }
+
+ public Keyspaces apply(Keyspaces schema)
+ {
+ KeyspaceMetadata keyspace = schema.getNullable(keyspaceName);
+
+ TableMetadata table = null == keyspace
+ ? null
+ : keyspace.getTableOrViewNullable(tableName);
+
+ if (null == table)
+ throw ire("Table '%s.%s' doesn't exist", keyspaceName, tableName);
+
+ if (table.isView())
+ throw ire("Cannot use ALTER TABLE on a materialized view; use ALTER MATERIALIZED VIEW instead");
+
+ return schema.withAddedOrUpdated(apply(keyspace, table));
+ }
+
+ SchemaChange schemaChangeEvent(KeyspacesDiff diff)
+ {
+ return new SchemaChange(Change.UPDATED, Target.TABLE, keyspaceName, tableName);
+ }
+
+ public void authorize(ClientState client)
+ {
+ client.ensureTablePermission(keyspaceName, tableName, Permission.ALTER);
+ }
+
+ @Override
+ public AuditLogContext getAuditLogContext()
+ {
+ return new AuditLogContext(AuditLogEntryType.ALTER_TABLE, keyspaceName, tableName);
+ }
+
+ public String toString()
+ {
+ return String.format("%s (%s, %s)", getClass().getSimpleName(), keyspaceName, tableName);
+ }
+
+ abstract KeyspaceMetadata apply(KeyspaceMetadata keyspace, TableMetadata table);
+
+ /**
+ * ALTER TABLE <table> ALTER <column> TYPE <newtype>;
+ *
+ * No longer supported.
+ */
+ public static class AlterColumn extends AlterTableStatement
+ {
+ AlterColumn(String keyspaceName, String tableName)
+ {
+ super(keyspaceName, tableName);
+ }
+
+ public KeyspaceMetadata apply(KeyspaceMetadata keyspace, TableMetadata table)
+ {
+ throw ire("Altering column types is no longer supported");
+ }
+ }
+
+ /**
+ * ALTER TABLE <table> ADD <column> <newtype>
+ * ALTER TABLE <table> ADD (<column> <newtype>, <column1> <newtype1>, ... <columnn> <newtypen>)
+ */
+ private static class AddColumns extends AlterTableStatement
+ {
+ private static class Column
+ {
+ private final ColumnMetadata.Raw name;
+ private final CQL3Type.Raw type;
+ private final boolean isStatic;
+
+ Column(ColumnMetadata.Raw name, CQL3Type.Raw type, boolean isStatic)
+ {
+ this.name = name;
+ this.type = type;
+ this.isStatic = isStatic;
+ }
+ }
+
+ private final Collection<Column> newColumns;
+
+ private AddColumns(String keyspaceName, String tableName, Collection<Column> newColumns)
+ {
+ super(keyspaceName, tableName);
+ this.newColumns = newColumns;
+ }
+
+ public KeyspaceMetadata apply(KeyspaceMetadata keyspace, TableMetadata table)
+ {
+ TableMetadata.Builder tableBuilder = table.unbuild();
+ Views.Builder viewsBuilder = keyspace.views.unbuild();
+ newColumns.forEach(c -> addColumn(keyspace, table, c, tableBuilder, viewsBuilder));
+
+ return keyspace.withSwapped(keyspace.tables.withSwapped(tableBuilder.build()))
+ .withSwapped(viewsBuilder.build());
+ }
+
+ private void addColumn(KeyspaceMetadata keyspace,
+ TableMetadata table,
+ Column column,
+ TableMetadata.Builder tableBuilder,
+ Views.Builder viewsBuilder)
+ {
+ ColumnIdentifier name = column.name.getIdentifier(table);
+ AbstractType<?> type = column.type.prepare(keyspaceName, keyspace.types).getType();
+ boolean isStatic = column.isStatic;
+
+ if (null != tableBuilder.getColumn(name))
+ throw ire("Column with name '%s' already exists", name);
+
+ if (isStatic && table.clusteringColumns().isEmpty())
+ throw ire("Static columns are only useful (and thus allowed) if the table has at least one clustering column");
+
+ ColumnMetadata droppedColumn = table.getDroppedColumn(name.bytes);
+ if (null != droppedColumn)
+ {
+ // After #8099, not safe to re-add columns of incompatible types - until *maybe* deser logic with dropped
+ // columns is pushed deeper down the line. The latter would still be problematic in cases of schema races.
+ if (!type.isValueCompatibleWith(droppedColumn.type))
+ {
+ throw ire("Cannot re-add previously dropped column '%s' of type %s, incompatible with previous type %s",
+ name,
+ type.asCQL3Type(),
+ droppedColumn.type.asCQL3Type());
+ }
+
+ if (droppedColumn.isStatic() != isStatic)
+ {
+ throw ire("Cannot re-add previously dropped column '%s' of kind %s, incompatible with previous kind %s",
+ name,
+ isStatic ? ColumnMetadata.Kind.STATIC : ColumnMetadata.Kind.REGULAR,
+ droppedColumn.kind);
+ }
+
+ // Cannot re-add a dropped counter column. See #7831.
+ if (table.isCounter())
+ throw ire("Cannot re-add previously dropped counter column %s", name);
+ }
+
+ if (isStatic)
+ tableBuilder.addStaticColumn(name, type);
+ else
+ tableBuilder.addRegularColumn(name, type);
+
+ if (!isStatic)
+ {
+ for (ViewMetadata view : keyspace.views.forTable(table.id))
+ {
+ if (view.includeAllColumns)
+ {
+ ColumnMetadata viewColumn = ColumnMetadata.regularColumn(view.metadata, name.bytes, type);
+ viewsBuilder.put(viewsBuilder.get(view.name()).withAddedRegularColumn(viewColumn));
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * ALTER TABLE <table> DROP <column>
+ * ALTER TABLE <table> DROP ( <column>, <column1>, ... <columnn>)
+ */
+ // TODO: swap UDT refs with expanded tuples on drop
+ private static class DropColumns extends AlterTableStatement
+ {
+ private final Collection<ColumnMetadata.Raw> removedColumns;
+ private final Long timestamp;
+
+ private DropColumns(String keyspaceName, String tableName, Collection<ColumnMetadata.Raw> removedColumns, Long timestamp)
+ {
+ super(keyspaceName, tableName);
+ this.removedColumns = removedColumns;
+ this.timestamp = timestamp;
+ }
+
+ public KeyspaceMetadata apply(KeyspaceMetadata keyspace, TableMetadata table)
+ {
+ TableMetadata.Builder builder = table.unbuild();
+ removedColumns.forEach(c -> dropColumn(keyspace, table, c, builder));
+ return keyspace.withSwapped(keyspace.tables.withSwapped(builder.build()));
+ }
+
+ private void dropColumn(KeyspaceMetadata keyspace, TableMetadata table, ColumnMetadata.Raw column, TableMetadata.Builder builder)
+ {
+ ColumnIdentifier name = column.getIdentifier(table);
+
+ ColumnMetadata currentColumn = table.getColumn(name);
+ if (null == currentColumn)
+ throw ire("Column %s was not found in table '%s'", name, table);
+
+ if (currentColumn.isPrimaryKeyColumn())
+ throw ire("Cannot drop PRIMARY KEY column %s", name);
+
+ /*
+ * Cannot allow dropping top-level columns of user defined types that aren't frozen because we cannot convert
+ * the type into an equivalent tuple: we only support frozen tuples currently. And as such we cannot persist
+ * the correct type in system_schema.dropped_columns.
+ */
+ if (currentColumn.type.isUDT() && currentColumn.type.isMultiCell())
+ throw ire("Cannot drop non-frozen column %s of user type %s", name, currentColumn.type.asCQL3Type());
+
+ // TODO: some day try and find a way to not rely on Keyspace/IndexManager/Index to find dependent indexes
+ Set<IndexMetadata> dependentIndexes = Keyspace.openAndGetStore(table).indexManager.getDependentIndexes(currentColumn);
+ if (!dependentIndexes.isEmpty())
+ {
+ throw ire("Cannot drop column %s because it has dependent secondary indexes (%s)",
+ currentColumn,
+ join(", ", transform(dependentIndexes, i -> i.name)));
+ }
+
+ if (!isEmpty(keyspace.views.forTable(table.id)))
+ throw ire("Cannot drop column %s on base table %s with materialized views", currentColumn, table.name);
+
+ builder.removeRegularOrStaticColumn(name);
+ builder.recordColumnDrop(currentColumn, getTimestamp());
+ }
+
+ /**
+ * @return timestamp from query, otherwise return current time in micros
+ */
+ private long getTimestamp()
+ {
+ return timestamp == null ? ClientState.getTimestamp() : timestamp;
+ }
+ }
+
+ /**
+ * ALTER TABLE <table> RENAME <column> TO <column>;
+ */
+ private static class RenameColumns extends AlterTableStatement
+ {
+ private final Map<ColumnMetadata.Raw, ColumnMetadata.Raw> renamedColumns;
+
+ private RenameColumns(String keyspaceName, String tableName, Map<ColumnMetadata.Raw, ColumnMetadata.Raw> renamedColumns)
+ {
+ super(keyspaceName, tableName);
+ this.renamedColumns = renamedColumns;
+ }
+
+ public KeyspaceMetadata apply(KeyspaceMetadata keyspace, TableMetadata table)
+ {
+ TableMetadata.Builder tableBuilder = table.unbuild();
+ Views.Builder viewsBuilder = keyspace.views.unbuild();
+ renamedColumns.forEach((o, n) -> renameColumn(keyspace, table, o, n, tableBuilder, viewsBuilder));
+
+ return keyspace.withSwapped(keyspace.tables.withSwapped(tableBuilder.build()))
+ .withSwapped(viewsBuilder.build());
+ }
+
+ private void renameColumn(KeyspaceMetadata keyspace,
+ TableMetadata table,
+ ColumnMetadata.Raw oldName,
+ ColumnMetadata.Raw newName,
+ TableMetadata.Builder tableBuilder,
+ Views.Builder viewsBuilder)
+ {
+ ColumnIdentifier oldColumnName = oldName.getIdentifier(table);
+ ColumnIdentifier newColumnName = newName.getIdentifier(table);
+
+ ColumnMetadata column = table.getColumn(oldColumnName);
+ if (null == column)
+ throw ire("Column %s was not found in table %s", oldColumnName, table);
+
+ if (!column.isPrimaryKeyColumn())
+ throw ire("Cannot rename non PRIMARY KEY column %s", oldColumnName);
+
+ if (null != table.getColumn(newColumnName))
+ {
+ throw ire("Cannot rename column %s to %s in table '%s'; another column with that name already exists",
+ oldColumnName,
+ newColumnName,
+ table);
+ }
+
+ // TODO: some day try and find a way to not rely on Keyspace/IndexManager/Index to find dependent indexes
+ Set<IndexMetadata> dependentIndexes = Keyspace.openAndGetStore(table).indexManager.getDependentIndexes(column);
+ if (!dependentIndexes.isEmpty())
+ {
+ throw ire("Can't rename column %s because it has dependent secondary indexes (%s)",
+ oldColumnName,
+ join(", ", transform(dependentIndexes, i -> i.name)));
+ }
+
+ for (ViewMetadata view : keyspace.views.forTable(table.id))
+ {
+ if (view.includes(oldColumnName))
+ {
+ ColumnIdentifier oldViewColumn = oldName.getIdentifier(view.metadata);
+ ColumnIdentifier newViewColumn = newName.getIdentifier(view.metadata);
+
+ viewsBuilder.put(viewsBuilder.get(view.name()).withRenamedPrimaryKeyColumn(oldViewColumn, newViewColumn));
+ }
+ }
+
+ tableBuilder.renamePrimaryKeyColumn(oldColumnName, newColumnName);
+ }
+ }
+
+ /**
+ * ALTER TABLE <table> WITH <property> = <value>
+ */
+ private static class AlterOptions extends AlterTableStatement
+ {
+ private final TableAttributes attrs;
+
+ private AlterOptions(String keyspaceName, String tableName, TableAttributes attrs)
+ {
+ super(keyspaceName, tableName);
+ this.attrs = attrs;
+ }
+
+ public KeyspaceMetadata apply(KeyspaceMetadata keyspace, TableMetadata table)
+ {
+ attrs.validate();
+
+ TableParams params = attrs.asAlteredTableParams(table.params);
+
+ if (table.isCounter() && params.defaultTimeToLive > 0)
+ throw ire("Cannot set default_time_to_live on a table with counters");
+
+ if (!isEmpty(keyspace.views.forTable(table.id)) && params.gcGraceSeconds == 0)
+ {
+ throw ire("Cannot alter gc_grace_seconds of the base table of a " +
+ "materialized view to 0, since this value is used to TTL " +
+ "undelivered updates. Setting gc_grace_seconds too low might " +
+ "cause undelivered updates to expire " +
+ "before being replayed.");
+ }
+
+ if (keyspace.createReplicationStrategy().hasTransientReplicas()
+ && params.readRepair != ReadRepairStrategy.NONE)
+ {
+ throw ire("read_repair must be set to 'NONE' for transiently replicated keyspaces");
+ }
+
+ return keyspace.withSwapped(keyspace.tables.withSwapped(table.withSwapped(params)));
+ }
+ }
+
+ public static final class Raw extends CQLStatement.Raw
+ {
+ private enum Kind
+ {
+ ALTER_COLUMN, ADD_COLUMNS, DROP_COLUMNS, RENAME_COLUMNS, ALTER_OPTIONS
+ }
+
+ private final QualifiedName name;
+
+ private Kind kind;
+
+ // ADD
+ private final List<AddColumns.Column> addedColumns = new ArrayList<>();
+
+ // DROP
+ private final List<ColumnMetadata.Raw> droppedColumns = new ArrayList<>();
+ private Long timestamp = null; // will use execution timestamp if not provided by query
+
+ // RENAME
+ private final Map<ColumnMetadata.Raw, ColumnMetadata.Raw> renamedColumns = new HashMap<>();
+
+ // OPTIONS
+ public final TableAttributes attrs = new TableAttributes();
+
+ public Raw(QualifiedName name)
+ {
+ this.name = name;
+ }
+
+ public AlterTableStatement prepare(ClientState state)
+ {
+ String keyspaceName = name.hasKeyspace() ? name.getKeyspace() : state.getKeyspace();
+ String tableName = name.getName();
+
+ switch (kind)
+ {
+ case ALTER_COLUMN: return new AlterColumn(keyspaceName, tableName);
+ case ADD_COLUMNS: return new AddColumns(keyspaceName, tableName, addedColumns);
+ case DROP_COLUMNS: return new DropColumns(keyspaceName, tableName, droppedColumns, timestamp);
+ case RENAME_COLUMNS: return new RenameColumns(keyspaceName, tableName, renamedColumns);
+ case ALTER_OPTIONS: return new AlterOptions(keyspaceName, tableName, attrs);
+ }
+
+ throw new AssertionError();
+ }
+
+ public void alter(ColumnMetadata.Raw name, CQL3Type.Raw type)
+ {
+ kind = Kind.ALTER_COLUMN;
+ }
+
+ public void add(ColumnMetadata.Raw name, CQL3Type.Raw type, boolean isStatic)
+ {
+ kind = Kind.ADD_COLUMNS;
+ addedColumns.add(new AddColumns.Column(name, type, isStatic));
+ }
+
+ public void drop(ColumnMetadata.Raw name)
+ {
+ kind = Kind.DROP_COLUMNS;
+ droppedColumns.add(name);
+ }
+
+ public void timestamp(long timestamp)
+ {
+ this.timestamp = timestamp;
+ }
+
+ public void rename(ColumnMetadata.Raw from, ColumnMetadata.Raw to)
+ {
+ kind = Kind.RENAME_COLUMNS;
+ renamedColumns.put(from, to);
+ }
+
+ public void attrs()
+ {
+ this.kind = Kind.ALTER_OPTIONS;
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/AlterTypeStatement.java b/src/java/org/apache/cassandra/cql3/statements/schema/AlterTypeStatement.java
new file mode 100644
index 0000000..f883038
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/statements/schema/AlterTypeStatement.java
@@ -0,0 +1,234 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.statements.schema;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.cassandra.audit.AuditLogContext;
+import org.apache.cassandra.audit.AuditLogEntryType;
+import org.apache.cassandra.auth.Permission;
+import org.apache.cassandra.cql3.*;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.UserType;
+import org.apache.cassandra.schema.KeyspaceMetadata;
+import org.apache.cassandra.schema.Keyspaces;
+import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.transport.Event.SchemaChange;
+import org.apache.cassandra.transport.Event.SchemaChange.Change;
+import org.apache.cassandra.transport.Event.SchemaChange.Target;
+
+import static java.lang.String.join;
+import static java.util.function.Predicate.isEqual;
+import static java.util.stream.Collectors.toList;
+
+import static org.apache.cassandra.utils.ByteBufferUtil.bytes;
+
+public abstract class AlterTypeStatement extends AlterSchemaStatement
+{
+ protected final String typeName;
+
+ public AlterTypeStatement(String keyspaceName, String typeName)
+ {
+ super(keyspaceName);
+ this.typeName = typeName;
+ }
+
+ public void authorize(ClientState client)
+ {
+ client.ensureKeyspacePermission(keyspaceName, Permission.ALTER);
+ }
+
+ SchemaChange schemaChangeEvent(Keyspaces.KeyspacesDiff diff)
+ {
+ return new SchemaChange(Change.UPDATED, Target.TYPE, keyspaceName, typeName);
+ }
+
+ public Keyspaces apply(Keyspaces schema)
+ {
+ KeyspaceMetadata keyspace = schema.getNullable(keyspaceName);
+
+ UserType type = null == keyspace
+ ? null
+ : keyspace.types.getNullable(bytes(typeName));
+
+ if (null == type)
+ throw ire("Type %s.%s doesn't exist", keyspaceName, typeName);
+
+ return schema.withAddedOrUpdated(keyspace.withUpdatedUserType(apply(keyspace, type)));
+ }
+
+ abstract UserType apply(KeyspaceMetadata keyspace, UserType type);
+
+ @Override
+ public AuditLogContext getAuditLogContext()
+ {
+ return new AuditLogContext(AuditLogEntryType.ALTER_TYPE, keyspaceName, typeName);
+ }
+
+ public String toString()
+ {
+ return String.format("%s (%s, %s)", getClass().getSimpleName(), keyspaceName, typeName);
+ }
+
+ private static final class AddField extends AlterTypeStatement
+ {
+ private final FieldIdentifier fieldName;
+ private final CQL3Type.Raw type;
+
+ private AddField(String keyspaceName, String typeName, FieldIdentifier fieldName, CQL3Type.Raw type)
+ {
+ super(keyspaceName, typeName);
+ this.fieldName = fieldName;
+ this.type = type;
+ }
+
+ UserType apply(KeyspaceMetadata keyspace, UserType userType)
+ {
+ if (userType.fieldPosition(fieldName) >= 0)
+ throw ire("Cannot add field %s to type %s: a field with name %s already exists", fieldName, userType.getCqlTypeName(), fieldName);
+
+ AbstractType<?> fieldType = type.prepare(keyspaceName, keyspace.types).getType();
+ if (fieldType.referencesUserType(userType.name))
+ throw ire("Cannot add new field %s of type %s to user type %s as it would create a circular reference", fieldName, type, userType.getCqlTypeName());
+
+ List<FieldIdentifier> fieldNames = new ArrayList<>(userType.fieldNames()); fieldNames.add(fieldName);
+ List<AbstractType<?>> fieldTypes = new ArrayList<>(userType.fieldTypes()); fieldTypes.add(fieldType);
+
+ return new UserType(keyspaceName, userType.name, fieldNames, fieldTypes, true);
+ }
+ }
+
+ private static final class RenameFields extends AlterTypeStatement
+ {
+ private final Map<FieldIdentifier, FieldIdentifier> renamedFields;
+
+ private RenameFields(String keyspaceName, String typeName, Map<FieldIdentifier, FieldIdentifier> renamedFields)
+ {
+ super(keyspaceName, typeName);
+ this.renamedFields = renamedFields;
+ }
+
+ UserType apply(KeyspaceMetadata keyspace, UserType userType)
+ {
+ List<String> dependentAggregates =
+ keyspace.functions
+ .udas()
+ .filter(uda -> null != uda.initialCondition() && uda.stateType().referencesUserType(userType.name))
+ .map(uda -> uda.name().toString())
+ .collect(toList());
+
+ if (!dependentAggregates.isEmpty())
+ {
+ throw ire("Cannot alter user type %s as it is still used in INITCOND by aggregates %s",
+ userType.getCqlTypeName(),
+ join(", ", dependentAggregates));
+ }
+
+ List<FieldIdentifier> fieldNames = new ArrayList<>(userType.fieldNames());
+
+ renamedFields.forEach((oldName, newName) ->
+ {
+ int idx = userType.fieldPosition(oldName);
+ if (idx < 0)
+ throw ire("Unkown field %s in user type %s", oldName, keyspaceName, userType.getCqlTypeName());
+ fieldNames.set(idx, newName);
+ });
+
+ fieldNames.forEach(name ->
+ {
+ if (fieldNames.stream().filter(isEqual(name)).count() > 1)
+ throw ire("Duplicate field name %s in type %s", name, keyspaceName, userType.getCqlTypeName());
+ });
+
+ return new UserType(keyspaceName, userType.name, fieldNames, userType.fieldTypes(), true);
+ }
+ }
+
+ private static final class AlterField extends AlterTypeStatement
+ {
+ private AlterField(String keyspaceName, String typeName)
+ {
+ super(keyspaceName, typeName);
+ }
+
+ UserType apply(KeyspaceMetadata keyspace, UserType userType)
+ {
+ throw ire("Alterting field types is no longer supported");
+ }
+ }
+
+ public static final class Raw extends CQLStatement.Raw
+ {
+ private enum Kind
+ {
+ ADD_FIELD, RENAME_FIELDS, ALTER_FIELD
+ }
+
+ private final UTName name;
+
+ private Kind kind;
+
+ // ADD
+ private FieldIdentifier newFieldName;
+ private CQL3Type.Raw newFieldType;
+
+ // RENAME
+ private final Map<FieldIdentifier, FieldIdentifier> renamedFields = new HashMap<>();
+
+ public Raw(UTName name)
+ {
+ this.name = name;
+ }
+
+ public AlterTypeStatement prepare(ClientState state)
+ {
+ String keyspaceName = name.hasKeyspace() ? name.getKeyspace() : state.getKeyspace();
+ String typeName = name.getStringTypeName();
+
+ switch (kind)
+ {
+ case ADD_FIELD: return new AddField(keyspaceName, typeName, newFieldName, newFieldType);
+ case RENAME_FIELDS: return new RenameFields(keyspaceName, typeName, renamedFields);
+ case ALTER_FIELD: return new AlterField(keyspaceName, typeName);
+ }
+
+ throw new AssertionError();
+ }
+
+ public void add(FieldIdentifier name, CQL3Type.Raw type)
+ {
+ kind = Kind.ADD_FIELD;
+ newFieldName = name;
+ newFieldType = type;
+ }
+
+ public void rename(FieldIdentifier from, FieldIdentifier to)
+ {
+ kind = Kind.RENAME_FIELDS;
+ renamedFields.put(from, to);
+ }
+
+ public void alter(FieldIdentifier name, CQL3Type.Raw type)
+ {
+ kind = Kind.ALTER_FIELD;
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/AlterViewStatement.java b/src/java/org/apache/cassandra/cql3/statements/schema/AlterViewStatement.java
new file mode 100644
index 0000000..1931bb4
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/statements/schema/AlterViewStatement.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.statements.schema;
+
+import org.apache.cassandra.audit.AuditLogContext;
+import org.apache.cassandra.audit.AuditLogEntryType;
+import org.apache.cassandra.auth.Permission;
+import org.apache.cassandra.cql3.CQLStatement;
+import org.apache.cassandra.cql3.QualifiedName;
+import org.apache.cassandra.schema.*;
+import org.apache.cassandra.schema.Keyspaces.KeyspacesDiff;
+import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.transport.Event.SchemaChange;
+import org.apache.cassandra.transport.Event.SchemaChange.Change;
+import org.apache.cassandra.transport.Event.SchemaChange.Target;
+
+public final class AlterViewStatement extends AlterSchemaStatement
+{
+ private final String viewName;
+ private final TableAttributes attrs;
+
+ public AlterViewStatement(String keyspaceName, String viewName, TableAttributes attrs)
+ {
+ super(keyspaceName);
+ this.viewName = viewName;
+ this.attrs = attrs;
+ }
+
+ public Keyspaces apply(Keyspaces schema)
+ {
+ KeyspaceMetadata keyspace = schema.getNullable(keyspaceName);
+
+ ViewMetadata view = null == keyspace
+ ? null
+ : keyspace.views.getNullable(viewName);
+
+ if (null == view)
+ throw ire("Materialized view '%s.%s' doesn't exist", keyspaceName, viewName);
+
+ attrs.validate();
+
+ TableParams params = attrs.asAlteredTableParams(view.metadata.params);
+
+ if (params.gcGraceSeconds == 0)
+ {
+ throw ire("Cannot alter gc_grace_seconds of a materialized view to 0, since this " +
+ "value is used to TTL undelivered updates. Setting gc_grace_seconds too " +
+ "low might cause undelivered updates to expire before being replayed.");
+ }
+
+ if (params.defaultTimeToLive > 0)
+ {
+ throw ire("Cannot set or alter default_time_to_live for a materialized view. " +
+ "Data in a materialized view always expire at the same time than " +
+ "the corresponding data in the parent table.");
+ }
+
+ ViewMetadata newView = view.copy(view.metadata.withSwapped(params));
+ return schema.withAddedOrUpdated(keyspace.withSwapped(keyspace.views.withSwapped(newView)));
+ }
+
+ SchemaChange schemaChangeEvent(KeyspacesDiff diff)
+ {
+ return new SchemaChange(Change.UPDATED, Target.TABLE, keyspaceName, viewName);
+ }
+
+ public void authorize(ClientState client)
+ {
+ ViewMetadata view = Schema.instance.getView(keyspaceName, viewName);
+ if (null != view)
+ client.ensureTablePermission(keyspaceName, view.baseTableName, Permission.ALTER);
+ }
+
+ @Override
+ public AuditLogContext getAuditLogContext()
+ {
+ return new AuditLogContext(AuditLogEntryType.ALTER_VIEW, keyspaceName, viewName);
+ }
+
+ public String toString()
+ {
+ return String.format("%s (%s, %s)", getClass().getSimpleName(), keyspaceName, viewName);
+ }
+
+ public static final class Raw extends CQLStatement.Raw
+ {
+ private final QualifiedName name;
+ private final TableAttributes attrs;
+
+ public Raw(QualifiedName name, TableAttributes attrs)
+ {
+ this.name = name;
+ this.attrs = attrs;
+ }
+
+ public AlterViewStatement prepare(ClientState state)
+ {
+ String keyspaceName = name.hasKeyspace() ? name.getKeyspace() : state.getKeyspace();
+ return new AlterViewStatement(keyspaceName, name.getName(), attrs);
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/CreateAggregateStatement.java b/src/java/org/apache/cassandra/cql3/statements/schema/CreateAggregateStatement.java
new file mode 100644
index 0000000..462623d
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/statements/schema/CreateAggregateStatement.java
@@ -0,0 +1,334 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.statements.schema;
+
+import java.nio.ByteBuffer;
+import java.util.List;
+import java.util.Set;
+
+import com.google.common.base.Objects;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Lists;
+
+import org.apache.cassandra.audit.AuditLogContext;
+import org.apache.cassandra.audit.AuditLogEntryType;
+import org.apache.cassandra.auth.FunctionResource;
+import org.apache.cassandra.auth.IResource;
+import org.apache.cassandra.auth.Permission;
+import org.apache.cassandra.cql3.*;
+import org.apache.cassandra.cql3.functions.*;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.schema.Functions.FunctionsDiff;
+import org.apache.cassandra.schema.KeyspaceMetadata;
+import org.apache.cassandra.schema.Keyspaces;
+import org.apache.cassandra.schema.Keyspaces.KeyspacesDiff;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.serializers.MarshalException;
+import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.transport.Event.SchemaChange;
+import org.apache.cassandra.transport.Event.SchemaChange.Change;
+import org.apache.cassandra.transport.Event.SchemaChange.Target;
+import org.apache.cassandra.transport.ProtocolVersion;
+
+import static java.lang.String.format;
+import static java.lang.String.join;
+import static java.util.Collections.singleton;
+import static java.util.Collections.singletonList;
+import static java.util.stream.Collectors.toList;
+
+import static com.google.common.collect.Iterables.concat;
+import static com.google.common.collect.Iterables.transform;
+
+public final class CreateAggregateStatement extends AlterSchemaStatement
+{
+ private final String aggregateName;
+ private final List<CQL3Type.Raw> rawArgumentTypes;
+ private final CQL3Type.Raw rawStateType;
+ private final FunctionName stateFunctionName;
+ private final FunctionName finalFunctionName;
+ private final Term.Raw rawInitialValue;
+ private final boolean orReplace;
+ private final boolean ifNotExists;
+
+ public CreateAggregateStatement(String keyspaceName,
+ String aggregateName,
+ List<CQL3Type.Raw> rawArgumentTypes,
+ CQL3Type.Raw rawStateType,
+ FunctionName stateFunctionName,
+ FunctionName finalFunctionName,
+ Term.Raw rawInitialValue,
+ boolean orReplace,
+ boolean ifNotExists)
+ {
+ super(keyspaceName);
+ this.aggregateName = aggregateName;
+ this.rawArgumentTypes = rawArgumentTypes;
+ this.rawStateType = rawStateType;
+ this.stateFunctionName = stateFunctionName;
+ this.finalFunctionName = finalFunctionName;
+ this.rawInitialValue = rawInitialValue;
+ this.orReplace = orReplace;
+ this.ifNotExists = ifNotExists;
+ }
+
+ public Keyspaces apply(Keyspaces schema)
+ {
+ if (ifNotExists && orReplace)
+ throw ire("Cannot use both 'OR REPLACE' and 'IF NOT EXISTS' directives");
+
+ rawArgumentTypes.stream()
+ .filter(CQL3Type.Raw::isFrozen)
+ .findFirst()
+ .ifPresent(t -> { throw ire("Argument '%s' cannot be frozen; remove frozen<> modifier from '%s'", t, t); });
+
+ if (rawStateType.isFrozen())
+ throw ire("State type '%s' cannot be frozen; remove frozen<> modifier from '%s'", rawStateType, rawStateType);
+
+ KeyspaceMetadata keyspace = schema.getNullable(keyspaceName);
+ if (null == keyspace)
+ throw ire("Keyspace '%s' doesn't exist", keyspaceName);
+
+ /*
+ * Resolve the state function
+ */
+
+ List<AbstractType<?>> argumentTypes =
+ rawArgumentTypes.stream()
+ .map(t -> t.prepare(keyspaceName, keyspace.types).getType())
+ .collect(toList());
+ AbstractType<?> stateType = rawStateType.prepare(keyspaceName, keyspace.types).getType();
+ List<AbstractType<?>> stateFunctionArguments = Lists.newArrayList(concat(singleton(stateType), argumentTypes));
+
+ Function stateFunction =
+ keyspace.functions
+ .find(stateFunctionName, stateFunctionArguments)
+ .orElseThrow(() -> ire("State function %s doesn't exist", stateFunctionString()));
+
+ if (stateFunction.isAggregate())
+ throw ire("State function %s isn't a scalar function", stateFunctionString());
+
+ if (!stateFunction.returnType().equals(stateType))
+ {
+ throw ire("State function %s return type must be the same as the first argument type - check STYPE, argument and return types",
+ stateFunctionString());
+ }
+
+ /*
+ * Resolve the final function and return type
+ */
+
+ Function finalFunction = null;
+ AbstractType<?> returnType = stateFunction.returnType();
+
+ if (null != finalFunctionName)
+ {
+ finalFunction = keyspace.functions.find(finalFunctionName, singletonList(stateType)).orElse(null);
+ if (null == finalFunction)
+ throw ire("Final function %s doesn't exist", finalFunctionString());
+
+ if (finalFunction.isAggregate())
+ throw ire("Final function %s isn't a scalar function", finalFunctionString());
+
+ // override return type with that of the final function
+ returnType = finalFunction.returnType();
+ }
+
+ /*
+ * Validate initial condition
+ */
+
+ ByteBuffer initialValue = null;
+ if (null != rawInitialValue)
+ {
+ initialValue = Terms.asBytes(keyspaceName, rawInitialValue.toString(), stateType);
+
+ if (null != initialValue)
+ {
+ try
+ {
+ stateType.validate(initialValue);
+ }
+ catch (MarshalException e)
+ {
+ throw ire("Invalid value for INITCOND of type %s", stateType.asCQL3Type());
+ }
+ }
+
+ // Converts initcond to a CQL literal and parse it back to avoid another CASSANDRA-11064
+ String initialValueString = stateType.asCQL3Type().toCQLLiteral(initialValue, ProtocolVersion.CURRENT);
+ assert Objects.equal(initialValue, Terms.asBytes(keyspaceName, initialValueString, stateType));
+
+ if (Constants.NULL_LITERAL != rawInitialValue && UDHelper.isNullOrEmpty(stateType, initialValue))
+ throw ire("INITCOND must not be empty for all types except TEXT, ASCII, BLOB");
+ }
+
+ if (!((UDFunction) stateFunction).isCalledOnNullInput() && null == initialValue)
+ {
+ throw ire("Cannot create aggregate '%s' without INITCOND because state function %s does not accept 'null' arguments",
+ aggregateName,
+ stateFunctionName);
+ }
+
+ /*
+ * Create or replace
+ */
+
+ UDAggregate aggregate =
+ new UDAggregate(new FunctionName(keyspaceName, aggregateName),
+ argumentTypes,
+ returnType,
+ (ScalarFunction) stateFunction,
+ (ScalarFunction) finalFunction,
+ initialValue);
+
+ Function existingAggregate = keyspace.functions.find(aggregate.name(), argumentTypes).orElse(null);
+ if (null != existingAggregate)
+ {
+ if (!existingAggregate.isAggregate())
+ throw ire("Aggregate '%s' cannot replace a function", aggregateName);
+
+ if (ifNotExists)
+ return schema;
+
+ if (!orReplace)
+ throw ire("Aggregate '%s' already exists", aggregateName);
+
+ if (!returnType.isCompatibleWith(existingAggregate.returnType()))
+ {
+ throw ire("Cannot replace aggregate '%s', the new return type %s isn't compatible with the return type %s of existing function",
+ aggregateName,
+ returnType.asCQL3Type(),
+ existingAggregate.returnType().asCQL3Type());
+ }
+ }
+
+ return schema.withAddedOrUpdated(keyspace.withSwapped(keyspace.functions.withAddedOrUpdated(aggregate)));
+ }
+
+ SchemaChange schemaChangeEvent(KeyspacesDiff diff)
+ {
+ assert diff.altered.size() == 1;
+ FunctionsDiff<UDAggregate> udasDiff = diff.altered.get(0).udas;
+
+ assert udasDiff.created.size() + udasDiff.altered.size() == 1;
+ boolean created = !udasDiff.created.isEmpty();
+
+ return new SchemaChange(created ? Change.CREATED : Change.UPDATED,
+ Target.AGGREGATE,
+ keyspaceName,
+ aggregateName,
+ rawArgumentTypes.stream().map(CQL3Type.Raw::toString).collect(toList()));
+ }
+
+ public void authorize(ClientState client)
+ {
+ FunctionName name = new FunctionName(keyspaceName, aggregateName);
+
+ if (Schema.instance.findFunction(name, Lists.transform(rawArgumentTypes, t -> t.prepare(keyspaceName).getType())).isPresent() && orReplace)
+ client.ensurePermission(Permission.ALTER, FunctionResource.functionFromCql(keyspaceName, aggregateName, rawArgumentTypes));
+ else
+ client.ensurePermission(Permission.CREATE, FunctionResource.keyspace(keyspaceName));
+
+ FunctionResource stateFunction =
+ FunctionResource.functionFromCql(stateFunctionName, Lists.newArrayList(concat(singleton(rawStateType), rawArgumentTypes)));
+ client.ensurePermission(Permission.EXECUTE, stateFunction);
+
+ if (null != finalFunctionName)
+ client.ensurePermission(Permission.EXECUTE, FunctionResource.functionFromCql(finalFunctionName, singletonList(rawStateType)));
+ }
+
+ @Override
+ Set<IResource> createdResources(KeyspacesDiff diff)
+ {
+ assert diff.altered.size() == 1;
+ FunctionsDiff<UDAggregate> udasDiff = diff.altered.get(0).udas;
+
+ assert udasDiff.created.size() + udasDiff.altered.size() == 1;
+
+ return udasDiff.created.isEmpty()
+ ? ImmutableSet.of()
+ : ImmutableSet.of(FunctionResource.functionFromCql(keyspaceName, aggregateName, rawArgumentTypes));
+ }
+
+ @Override
+ public AuditLogContext getAuditLogContext()
+ {
+ return new AuditLogContext(AuditLogEntryType.CREATE_AGGREGATE, keyspaceName, aggregateName);
+ }
+
+ public String toString()
+ {
+ return String.format("%s (%s, %s)", getClass().getSimpleName(), keyspaceName, aggregateName);
+ }
+
+ private String stateFunctionString()
+ {
+ return format("%s(%s)", stateFunctionName, join(", ", transform(concat(singleton(rawStateType), rawArgumentTypes), Object::toString)));
+ }
+
+ private String finalFunctionString()
+ {
+ return format("%s(%s)", finalFunctionName, rawStateType);
+ }
+
+ public static final class Raw extends CQLStatement.Raw
+ {
+ private final FunctionName aggregateName;
+ private final List<CQL3Type.Raw> rawArgumentTypes;
+ private final CQL3Type.Raw rawStateType;
+ private final String stateFunctionName;
+ private final String finalFunctionName;
+ private final Term.Raw rawInitialValue;
+ private final boolean orReplace;
+ private final boolean ifNotExists;
+
+ public Raw(FunctionName aggregateName,
+ List<CQL3Type.Raw> rawArgumentTypes,
+ CQL3Type.Raw rawStateType,
+ String stateFunctionName,
+ String finalFunctionName,
+ Term.Raw rawInitialValue,
+ boolean orReplace,
+ boolean ifNotExists)
+ {
+ this.aggregateName = aggregateName;
+ this.rawArgumentTypes = rawArgumentTypes;
+ this.rawStateType = rawStateType;
+ this.stateFunctionName = stateFunctionName;
+ this.finalFunctionName = finalFunctionName;
+ this.rawInitialValue = rawInitialValue;
+ this.orReplace = orReplace;
+ this.ifNotExists = ifNotExists;
+ }
+
+ public CreateAggregateStatement prepare(ClientState state)
+ {
+ String keyspaceName = aggregateName.hasKeyspace() ? aggregateName.keyspace : state.getKeyspace();
+
+ return new CreateAggregateStatement(keyspaceName,
+ aggregateName.name,
+ rawArgumentTypes,
+ rawStateType,
+ new FunctionName(keyspaceName, stateFunctionName),
+ null != finalFunctionName ? new FunctionName(keyspaceName, finalFunctionName) : null,
+ rawInitialValue,
+ orReplace,
+ ifNotExists);
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/CreateFunctionStatement.java b/src/java/org/apache/cassandra/cql3/statements/schema/CreateFunctionStatement.java
new file mode 100644
index 0000000..20c4ad9
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/statements/schema/CreateFunctionStatement.java
@@ -0,0 +1,255 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.statements.schema;
+
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Lists;
+
+import org.apache.cassandra.audit.AuditLogContext;
+import org.apache.cassandra.audit.AuditLogEntryType;
+import org.apache.cassandra.auth.FunctionResource;
+import org.apache.cassandra.auth.IResource;
+import org.apache.cassandra.auth.*;
+import org.apache.cassandra.cql3.CQL3Type;
+import org.apache.cassandra.cql3.CQLStatement;
+import org.apache.cassandra.cql3.ColumnIdentifier;
+import org.apache.cassandra.cql3.functions.Function;
+import org.apache.cassandra.cql3.functions.FunctionName;
+import org.apache.cassandra.cql3.functions.UDFunction;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.schema.Functions.FunctionsDiff;
+import org.apache.cassandra.schema.KeyspaceMetadata;
+import org.apache.cassandra.schema.Keyspaces;
+import org.apache.cassandra.schema.Keyspaces.KeyspacesDiff;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.transport.Event.SchemaChange;
+import org.apache.cassandra.transport.Event.SchemaChange.Change;
+import org.apache.cassandra.transport.Event.SchemaChange.Target;
+
+import static java.util.stream.Collectors.toList;
+
+public final class CreateFunctionStatement extends AlterSchemaStatement
+{
+ private final String functionName;
+ private final List<ColumnIdentifier> argumentNames;
+ private final List<CQL3Type.Raw> rawArgumentTypes;
+ private final CQL3Type.Raw rawReturnType;
+ private final boolean calledOnNullInput;
+ private final String language;
+ private final String body;
+ private final boolean orReplace;
+ private final boolean ifNotExists;
+
+ public CreateFunctionStatement(String keyspaceName,
+ String functionName,
+ List<ColumnIdentifier> argumentNames,
+ List<CQL3Type.Raw> rawArgumentTypes,
+ CQL3Type.Raw rawReturnType,
+ boolean calledOnNullInput,
+ String language,
+ String body,
+ boolean orReplace,
+ boolean ifNotExists)
+ {
+ super(keyspaceName);
+ this.functionName = functionName;
+ this.argumentNames = argumentNames;
+ this.rawArgumentTypes = rawArgumentTypes;
+ this.rawReturnType = rawReturnType;
+ this.calledOnNullInput = calledOnNullInput;
+ this.language = language;
+ this.body = body;
+ this.orReplace = orReplace;
+ this.ifNotExists = ifNotExists;
+ }
+
+ // TODO: replace affected aggregates !!
+ public Keyspaces apply(Keyspaces schema)
+ {
+ if (ifNotExists && orReplace)
+ throw ire("Cannot use both 'OR REPLACE' and 'IF NOT EXISTS' directives");
+
+ UDFunction.assertUdfsEnabled(language);
+
+ if (new HashSet<>(argumentNames).size() != argumentNames.size())
+ throw ire("Duplicate argument names for given function %s with argument names %s", functionName, argumentNames);
+
+ rawArgumentTypes.stream()
+ .filter(CQL3Type.Raw::isFrozen)
+ .findFirst()
+ .ifPresent(t -> { throw ire("Argument '%s' cannot be frozen; remove frozen<> modifier from '%s'", t, t); });
+
+ if (rawReturnType.isFrozen())
+ throw ire("Return type '%s' cannot be frozen; remove frozen<> modifier from '%s'", rawReturnType, rawReturnType);
+
+ KeyspaceMetadata keyspace = schema.getNullable(keyspaceName);
+ if (null == keyspace)
+ throw ire("Keyspace '%s' doesn't exist", keyspaceName);
+
+ List<AbstractType<?>> argumentTypes =
+ rawArgumentTypes.stream()
+ .map(t -> t.prepare(keyspaceName, keyspace.types).getType())
+ .collect(toList());
+ AbstractType<?> returnType = rawReturnType.prepare(keyspaceName, keyspace.types).getType();
+
+ UDFunction function =
+ UDFunction.create(new FunctionName(keyspaceName, functionName),
+ argumentNames,
+ argumentTypes,
+ returnType,
+ calledOnNullInput,
+ language,
+ body);
+
+ Function existingFunction = keyspace.functions.find(function.name(), argumentTypes).orElse(null);
+ if (null != existingFunction)
+ {
+ if (existingFunction.isAggregate())
+ throw ire("Function '%s' cannot replace an aggregate", functionName);
+
+ if (ifNotExists)
+ return schema;
+
+ if (!orReplace)
+ throw ire("Function '%s' already exists", functionName);
+
+ if (calledOnNullInput != ((UDFunction) existingFunction).isCalledOnNullInput())
+ {
+ throw ire("Function '%s' must have %s directive",
+ functionName,
+ calledOnNullInput ? "CALLED ON NULL INPUT" : "RETURNS NULL ON NULL INPUT");
+ }
+
+ if (!returnType.isCompatibleWith(existingFunction.returnType()))
+ {
+ throw ire("Cannot replace function '%s', the new return type %s is not compatible with the return type %s of existing function",
+ functionName,
+ returnType.asCQL3Type(),
+ existingFunction.returnType().asCQL3Type());
+ }
+
+ // TODO: update dependent aggregates
+ }
+
+ return schema.withAddedOrUpdated(keyspace.withSwapped(keyspace.functions.withAddedOrUpdated(function)));
+ }
+
+ SchemaChange schemaChangeEvent(KeyspacesDiff diff)
+ {
+ assert diff.altered.size() == 1;
+ FunctionsDiff<UDFunction> udfsDiff = diff.altered.get(0).udfs;
+
+ assert udfsDiff.created.size() + udfsDiff.altered.size() == 1;
+ boolean created = !udfsDiff.created.isEmpty();
+
+ return new SchemaChange(created ? Change.CREATED : Change.UPDATED,
+ Target.FUNCTION,
+ keyspaceName,
+ functionName,
+ rawArgumentTypes.stream().map(CQL3Type.Raw::toString).collect(toList()));
+ }
+
+ public void authorize(ClientState client)
+ {
+ FunctionName name = new FunctionName(keyspaceName, functionName);
+
+ if (Schema.instance.findFunction(name, Lists.transform(rawArgumentTypes, t -> t.prepare(keyspaceName).getType())).isPresent() && orReplace)
+ client.ensurePermission(Permission.ALTER, FunctionResource.functionFromCql(keyspaceName, functionName, rawArgumentTypes));
+ else
+ client.ensurePermission(Permission.CREATE, FunctionResource.keyspace(keyspaceName));
+ }
+
+ @Override
+ Set<IResource> createdResources(KeyspacesDiff diff)
+ {
+ assert diff.altered.size() == 1;
+ FunctionsDiff<UDFunction> udfsDiff = diff.altered.get(0).udfs;
+
+ assert udfsDiff.created.size() + udfsDiff.altered.size() == 1;
+
+ return udfsDiff.created.isEmpty()
+ ? ImmutableSet.of()
+ : ImmutableSet.of(FunctionResource.functionFromCql(keyspaceName, functionName, rawArgumentTypes));
+ }
+
+ @Override
+ public AuditLogContext getAuditLogContext()
+ {
+ return new AuditLogContext(AuditLogEntryType.CREATE_FUNCTION, keyspaceName, functionName);
+ }
+
+ public String toString()
+ {
+ return String.format("%s (%s, %s)", getClass().getSimpleName(), keyspaceName, functionName);
+ }
+
+ public static final class Raw extends CQLStatement.Raw
+ {
+ private final FunctionName name;
+ private final List<ColumnIdentifier> argumentNames;
+ private final List<CQL3Type.Raw> rawArgumentTypes;
+ private final CQL3Type.Raw rawReturnType;
+ private final boolean calledOnNullInput;
+ private final String language;
+ private final String body;
+ private final boolean orReplace;
+ private final boolean ifNotExists;
+
+ public Raw(FunctionName name,
+ List<ColumnIdentifier> argumentNames,
+ List<CQL3Type.Raw> rawArgumentTypes,
+ CQL3Type.Raw rawReturnType,
+ boolean calledOnNullInput,
+ String language,
+ String body,
+ boolean orReplace,
+ boolean ifNotExists)
+ {
+ this.name = name;
+ this.argumentNames = argumentNames;
+ this.rawArgumentTypes = rawArgumentTypes;
+ this.rawReturnType = rawReturnType;
+ this.calledOnNullInput = calledOnNullInput;
+ this.language = language;
+ this.body = body;
+ this.orReplace = orReplace;
+ this.ifNotExists = ifNotExists;
+ }
+
+ public CreateFunctionStatement prepare(ClientState state)
+ {
+ String keyspaceName = name.hasKeyspace() ? name.keyspace : state.getKeyspace();
+
+ return new CreateFunctionStatement(keyspaceName,
+ name.name,
+ argumentNames,
+ rawArgumentTypes,
+ rawReturnType,
+ calledOnNullInput,
+ language,
+ body,
+ orReplace,
+ ifNotExists);
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/CreateIndexStatement.java b/src/java/org/apache/cassandra/cql3/statements/schema/CreateIndexStatement.java
new file mode 100644
index 0000000..e011c81
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/statements/schema/CreateIndexStatement.java
@@ -0,0 +1,259 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.statements.schema;
+
+import java.util.*;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Lists;
+
+import org.apache.cassandra.audit.AuditLogContext;
+import org.apache.cassandra.audit.AuditLogEntryType;
+import org.apache.cassandra.auth.Permission;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.CQLStatement;
+import org.apache.cassandra.cql3.ColumnIdentifier;
+import org.apache.cassandra.cql3.QualifiedName;
+import org.apache.cassandra.cql3.statements.schema.IndexTarget.Type;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.marshal.MapType;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.index.sasi.SASIIndex;
+import org.apache.cassandra.schema.*;
+import org.apache.cassandra.schema.Keyspaces.KeyspacesDiff;
+import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.transport.Event.SchemaChange;
+import org.apache.cassandra.transport.Event.SchemaChange.Change;
+import org.apache.cassandra.transport.Event.SchemaChange.Target;
+
+import static com.google.common.collect.Iterables.transform;
+import static com.google.common.collect.Iterables.tryFind;
+
+public final class CreateIndexStatement extends AlterSchemaStatement
+{
+ private final String indexName;
+ private final String tableName;
+ private final List<IndexTarget.Raw> rawIndexTargets;
+ private final IndexAttributes attrs;
+ private final boolean ifNotExists;
+
+ public CreateIndexStatement(String keyspaceName,
+ String tableName,
+ String indexName,
+ List<IndexTarget.Raw> rawIndexTargets,
+ IndexAttributes attrs,
+ boolean ifNotExists)
+ {
+ super(keyspaceName);
+ this.tableName = tableName;
+ this.indexName = indexName;
+ this.rawIndexTargets = rawIndexTargets;
+ this.attrs = attrs;
+ this.ifNotExists = ifNotExists;
+ }
+
+ public Keyspaces apply(Keyspaces schema)
+ {
+ attrs.validate();
+
+ if (attrs.isCustom && attrs.customClass.equals(SASIIndex.class.getName()) && !DatabaseDescriptor.getEnableSASIIndexes())
+ throw new InvalidRequestException("SASI indexes are disabled. Enable in cassandra.yaml to use.");
+
+ KeyspaceMetadata keyspace = schema.getNullable(keyspaceName);
+ if (null == keyspace)
+ throw ire("Keyspace '%s' doesn't exist", keyspaceName);
+
+ TableMetadata table = keyspace.getTableOrViewNullable(tableName);
+ if (null == table)
+ throw ire("Table '%s' doesn't exist", tableName);
+
+ if (null != indexName && keyspace.hasIndex(indexName))
+ {
+ if (ifNotExists)
+ return schema;
+
+ throw ire("Index '%s' already exists", indexName);
+ }
+
+ if (table.isCounter())
+ throw ire("Secondary indexes on counter tables aren't supported");
+
+ if (table.isView())
+ throw ire("Secondary indexes on materialized views aren't supported");
+
+ if (Keyspace.open(table.keyspace).getReplicationStrategy().hasTransientReplicas())
+ throw new InvalidRequestException("Secondary indexes are not supported on transiently replicated keyspaces");
+
+ List<IndexTarget> indexTargets = Lists.newArrayList(transform(rawIndexTargets, t -> t.prepare(table)));
+
+ if (indexTargets.isEmpty() && !attrs.isCustom)
+ throw ire("Only CUSTOM indexes can be created without specifying a target column");
+
+ if (indexTargets.size() > 1)
+ {
+ if (!attrs.isCustom)
+ throw ire("Only CUSTOM indexes support multiple columns");
+
+ Set<ColumnIdentifier> columns = new HashSet<>();
+ for (IndexTarget target : indexTargets)
+ if (!columns.add(target.column))
+ throw ire("Duplicate column '%s' in index target list", target.column);
+ }
+
+ indexTargets.forEach(t -> validateIndexTarget(table, t));
+
+ String name = null == indexName ? generateIndexName(keyspace, indexTargets) : indexName;
+
+ IndexMetadata.Kind kind = attrs.isCustom ? IndexMetadata.Kind.CUSTOM : IndexMetadata.Kind.COMPOSITES;
+
+ Map<String, String> options = attrs.isCustom ? attrs.getOptions() : Collections.emptyMap();
+
+ IndexMetadata index = IndexMetadata.fromIndexTargets(indexTargets, name, kind, options);
+
+ // check to disallow creation of an index which duplicates an existing one in all but name
+ IndexMetadata equalIndex = tryFind(table.indexes, i -> i.equalsWithoutName(index)).orNull();
+ if (null != equalIndex)
+ {
+ if (ifNotExists)
+ return schema;
+
+ throw ire("Index %s is a duplicate of existing index %s", index.name, equalIndex.name);
+ }
+
+ TableMetadata newTable = table.withSwapped(table.indexes.with(index));
+ newTable.validate();
+
+ return schema.withAddedOrUpdated(keyspace.withSwapped(keyspace.tables.withSwapped(newTable)));
+ }
+
+ @Override
+ Set<String> clientWarnings(KeyspacesDiff diff)
+ {
+ if (attrs.isCustom && attrs.customClass.equals(SASIIndex.class.getName()))
+ return ImmutableSet.of(SASIIndex.USAGE_WARNING);
+
+ return ImmutableSet.of();
+ }
+
+ private void validateIndexTarget(TableMetadata table, IndexTarget target)
+ {
+ ColumnMetadata column = table.getColumn(target.column);
+
+ if (null == column)
+ throw ire("Column '%s' doesn't exist", target.column);
+
+ if (column.type.referencesDuration())
+ {
+ if (column.type.isCollection())
+ throw ire("Secondary indexes are not supported on collections containing durations");
+
+ if (column.type.isTuple())
+ throw ire("Secondary indexes are not supported on tuples containing durations");
+
+ if (column.type.isUDT())
+ throw ire("Secondary indexes are not supported on UDTs containing durations");
+
+ throw ire("Secondary indexes are not supported on duration columns");
+ }
+
+ if (column.isPartitionKey() && table.partitionKeyColumns().size() == 1)
+ throw ire("Cannot create secondary index on the only partition key column %s", column);
+
+ if (column.type.isFrozenCollection() && target.type != Type.FULL)
+ throw ire("Cannot create %s() index on frozen column %s. Frozen collections are immutable and must be fully " +
+ "indexed by using the 'full(%s)' modifier", target.type, column, column);
+
+ if (!column.type.isFrozenCollection() && target.type == Type.FULL)
+ throw ire("full() indexes can only be created on frozen collections");
+
+ if (!column.type.isCollection() && target.type != Type.SIMPLE)
+ throw ire("Cannot create %s() index on %s. Non-collection columns only support simple indexes", target.type, column);
+
+ if (!(column.type instanceof MapType && column.type.isMultiCell()) && (target.type == Type.KEYS || target.type == Type.KEYS_AND_VALUES))
+ throw ire("Cannot create index on %s of column %s with non-map type", target.type, column);
+
+ if (column.type.isUDT() && column.type.isMultiCell())
+ throw ire("Cannot create index on non-frozen UDT column %s", column);
+ }
+
+ private String generateIndexName(KeyspaceMetadata keyspace, List<IndexTarget> targets)
+ {
+ String baseName = targets.size() == 1
+ ? IndexMetadata.generateDefaultIndexName(tableName, targets.get(0).column)
+ : IndexMetadata.generateDefaultIndexName(tableName);
+ return keyspace.findAvailableIndexName(baseName);
+ }
+
+ SchemaChange schemaChangeEvent(KeyspacesDiff diff)
+ {
+ return new SchemaChange(Change.UPDATED, Target.TABLE, keyspaceName, tableName);
+ }
+
+ public void authorize(ClientState client)
+ {
+ client.ensureTablePermission(keyspaceName, tableName, Permission.ALTER);
+ }
+
+ @Override
+ public AuditLogContext getAuditLogContext()
+ {
+ return new AuditLogContext(AuditLogEntryType.CREATE_INDEX, keyspaceName, indexName);
+ }
+
+ public String toString()
+ {
+ return String.format("%s (%s, %s)", getClass().getSimpleName(), keyspaceName, indexName);
+ }
+
+ public static final class Raw extends CQLStatement.Raw
+ {
+ private final QualifiedName tableName;
+ private final QualifiedName indexName;
+ private final List<IndexTarget.Raw> rawIndexTargets;
+ private final IndexAttributes attrs;
+ private final boolean ifNotExists;
+
+ public Raw(QualifiedName tableName,
+ QualifiedName indexName,
+ List<IndexTarget.Raw> rawIndexTargets,
+ IndexAttributes attrs,
+ boolean ifNotExists)
+ {
+ this.tableName = tableName;
+ this.indexName = indexName;
+ this.rawIndexTargets = rawIndexTargets;
+ this.attrs = attrs;
+ this.ifNotExists = ifNotExists;
+ }
+
+ public CreateIndexStatement prepare(ClientState state)
+ {
+ String keyspaceName = tableName.hasKeyspace()
+ ? tableName.getKeyspace()
+ : indexName.hasKeyspace() ? indexName.getKeyspace() : state.getKeyspace();
+
+ if (tableName.hasKeyspace() && !keyspaceName.equals(tableName.getKeyspace()))
+ throw ire("Keyspace name '%s' doesn't match table name '%s'", keyspaceName, tableName);
+
+ if (indexName.hasKeyspace() && !keyspaceName.equals(indexName.getKeyspace()))
+ throw ire("Keyspace name '%s' doesn't match index name '%s'", keyspaceName, tableName);
+
+ return new CreateIndexStatement(keyspaceName, tableName.getName(), indexName.getName(), rawIndexTargets, attrs, ifNotExists);
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/CreateKeyspaceStatement.java b/src/java/org/apache/cassandra/cql3/statements/schema/CreateKeyspaceStatement.java
new file mode 100644
index 0000000..f85a4e9
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/statements/schema/CreateKeyspaceStatement.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.statements.schema;
+
+import java.util.Set;
+
+import com.google.common.collect.ImmutableSet;
+
+import org.apache.cassandra.audit.AuditLogContext;
+import org.apache.cassandra.audit.AuditLogEntryType;
+import org.apache.cassandra.auth.*;
+import org.apache.cassandra.cql3.CQLStatement;
+import org.apache.cassandra.exceptions.AlreadyExistsException;
+import org.apache.cassandra.locator.LocalStrategy;
+import org.apache.cassandra.schema.KeyspaceMetadata;
+import org.apache.cassandra.schema.KeyspaceParams.Option;
+import org.apache.cassandra.schema.Keyspaces;
+import org.apache.cassandra.schema.Keyspaces.KeyspacesDiff;
+import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.transport.Event.SchemaChange;
+import org.apache.cassandra.transport.Event.SchemaChange.Change;
+
+public final class CreateKeyspaceStatement extends AlterSchemaStatement
+{
+ private final KeyspaceAttributes attrs;
+ private final boolean ifNotExists;
+
+ public CreateKeyspaceStatement(String keyspaceName, KeyspaceAttributes attrs, boolean ifNotExists)
+ {
+ super(keyspaceName);
+ this.attrs = attrs;
+ this.ifNotExists = ifNotExists;
+ }
+
+ public Keyspaces apply(Keyspaces schema)
+ {
+ attrs.validate();
+
+ if (!attrs.hasOption(Option.REPLICATION))
+ throw ire("Missing mandatory option '%s'", Option.REPLICATION);
+
+ if (schema.containsKeyspace(keyspaceName))
+ {
+ if (ifNotExists)
+ return schema;
+
+ throw new AlreadyExistsException(keyspaceName);
+ }
+
+ KeyspaceMetadata keyspace = KeyspaceMetadata.create(keyspaceName, attrs.asNewKeyspaceParams());
+
+ if (keyspace.params.replication.klass.equals(LocalStrategy.class))
+ throw ire("Unable to use given strategy class: LocalStrategy is reserved for internal use.");
+
+ keyspace.params.validate(keyspaceName);
+
+ return schema.withAddedOrUpdated(keyspace);
+ }
+
+ SchemaChange schemaChangeEvent(KeyspacesDiff diff)
+ {
+ return new SchemaChange(Change.CREATED, keyspaceName);
+ }
+
+ public void authorize(ClientState client)
+ {
+ client.ensureAllKeyspacesPermission(Permission.CREATE);
+ }
+
+ @Override
+ Set<IResource> createdResources(KeyspacesDiff diff)
+ {
+ return ImmutableSet.of(DataResource.keyspace(keyspaceName), FunctionResource.keyspace(keyspaceName));
+ }
+
+ @Override
+ public AuditLogContext getAuditLogContext()
+ {
+ return new AuditLogContext(AuditLogEntryType.CREATE_KEYSPACE, keyspaceName);
+ }
+
+ public String toString()
+ {
+ return String.format("%s (%s)", getClass().getSimpleName(), keyspaceName);
+ }
+
+ public static final class Raw extends CQLStatement.Raw
+ {
+ public final String keyspaceName;
+ private final KeyspaceAttributes attrs;
+ private final boolean ifNotExists;
+
+ public Raw(String keyspaceName, KeyspaceAttributes attrs, boolean ifNotExists)
+ {
+ this.keyspaceName = keyspaceName;
+ this.attrs = attrs;
+ this.ifNotExists = ifNotExists;
+ }
+
+ public CreateKeyspaceStatement prepare(ClientState state)
+ {
+ return new CreateKeyspaceStatement(keyspaceName, attrs, ifNotExists);
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/CreateTableStatement.java b/src/java/org/apache/cassandra/cql3/statements/schema/CreateTableStatement.java
new file mode 100644
index 0000000..0b3e14d
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/statements/schema/CreateTableStatement.java
@@ -0,0 +1,372 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.statements.schema;
+
+import java.util.*;
+
+import com.google.common.collect.ImmutableSet;
+
+import org.apache.cassandra.audit.AuditLogContext;
+import org.apache.cassandra.audit.AuditLogEntryType;
+import org.apache.cassandra.auth.DataResource;
+import org.apache.cassandra.auth.IResource;
+import org.apache.cassandra.auth.Permission;
+import org.apache.cassandra.cql3.*;
+import org.apache.cassandra.db.marshal.*;
+import org.apache.cassandra.exceptions.AlreadyExistsException;
+import org.apache.cassandra.schema.*;
+import org.apache.cassandra.schema.Keyspaces.KeyspacesDiff;
+import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.service.reads.repair.ReadRepairStrategy;
+import org.apache.cassandra.transport.Event.SchemaChange;
+import org.apache.cassandra.transport.Event.SchemaChange.Change;
+import org.apache.cassandra.transport.Event.SchemaChange.Target;
+
+import static java.util.Comparator.comparing;
+
+import static com.google.common.collect.Iterables.concat;
+
+public final class CreateTableStatement extends AlterSchemaStatement
+{
+ private final String tableName;
+
+ private final Map<ColumnIdentifier, CQL3Type.Raw> rawColumns;
+ private final Set<ColumnIdentifier> staticColumns;
+ private final List<ColumnIdentifier> partitionKeyColumns;
+ private final List<ColumnIdentifier> clusteringColumns;
+
+ private final LinkedHashMap<ColumnIdentifier, Boolean> clusteringOrder;
+ private final TableAttributes attrs;
+
+ private final boolean ifNotExists;
+
+ public CreateTableStatement(String keyspaceName,
+ String tableName,
+
+ Map<ColumnIdentifier, CQL3Type.Raw> rawColumns,
+ Set<ColumnIdentifier> staticColumns,
+ List<ColumnIdentifier> partitionKeyColumns,
+ List<ColumnIdentifier> clusteringColumns,
+
+ LinkedHashMap<ColumnIdentifier, Boolean> clusteringOrder,
+ TableAttributes attrs,
+
+ boolean ifNotExists)
+ {
+ super(keyspaceName);
+ this.tableName = tableName;
+
+ this.rawColumns = rawColumns;
+ this.staticColumns = staticColumns;
+ this.partitionKeyColumns = partitionKeyColumns;
+ this.clusteringColumns = clusteringColumns;
+
+ this.clusteringOrder = clusteringOrder;
+ this.attrs = attrs;
+
+ this.ifNotExists = ifNotExists;
+ }
+
+ public Keyspaces apply(Keyspaces schema)
+ {
+ KeyspaceMetadata keyspace = schema.getNullable(keyspaceName);
+ if (null == keyspace)
+ throw ire("Keyspace '%s' doesn't exist", keyspaceName);
+
+ if (keyspace.hasTable(tableName))
+ {
+ if (ifNotExists)
+ return schema;
+
+ throw new AlreadyExistsException(keyspaceName, tableName);
+ }
+
+ TableMetadata table = builder(keyspace.types).build();
+ table.validate();
+
+ if (keyspace.createReplicationStrategy().hasTransientReplicas()
+ && table.params.readRepair != ReadRepairStrategy.NONE)
+ {
+ throw ire("read_repair must be set to 'NONE' for transiently replicated keyspaces");
+ }
+
+ return schema.withAddedOrUpdated(keyspace.withSwapped(keyspace.tables.with(table)));
+ }
+
+ SchemaChange schemaChangeEvent(KeyspacesDiff diff)
+ {
+ return new SchemaChange(Change.CREATED, Target.TABLE, keyspaceName, tableName);
+ }
+
+ public void authorize(ClientState client)
+ {
+ client.ensureKeyspacePermission(keyspaceName, Permission.CREATE);
+ }
+
+ @Override
+ Set<IResource> createdResources(KeyspacesDiff diff)
+ {
+ return ImmutableSet.of(DataResource.table(keyspaceName, tableName));
+ }
+
+ @Override
+ public AuditLogContext getAuditLogContext()
+ {
+ return new AuditLogContext(AuditLogEntryType.CREATE_TABLE, keyspaceName, tableName);
+ }
+
+ public String toString()
+ {
+ return String.format("%s (%s, %s)", getClass().getSimpleName(), keyspaceName, tableName);
+ }
+
+ public TableMetadata.Builder builder(Types types)
+ {
+ attrs.validate();
+ TableParams params = attrs.asNewTableParams();
+
+ // use a TreeMap to preserve ordering across JDK versions (see CASSANDRA-9492) - important for stable unit tests
+ Map<ColumnIdentifier, CQL3Type> columns = new TreeMap<>(comparing(o -> o.bytes));
+ rawColumns.forEach((column, type) -> columns.put(column, type.prepare(keyspaceName, types)));
+
+ // check for nested non-frozen UDTs or collections in a non-frozen UDT
+ columns.forEach((column, type) ->
+ {
+ if (type.isUDT() && type.getType().isMultiCell())
+ {
+ ((UserType) type.getType()).fieldTypes().forEach(field ->
+ {
+ if (field.isMultiCell())
+ throw ire("Non-frozen UDTs with nested non-frozen collections are not supported");
+ });
+ }
+ });
+
+ /*
+ * Deal with PRIMARY KEY columns
+ */
+
+ HashSet<ColumnIdentifier> primaryKeyColumns = new HashSet<>();
+ concat(partitionKeyColumns, clusteringColumns).forEach(column ->
+ {
+ CQL3Type type = columns.get(column);
+ if (null == type)
+ throw ire("Unknown column '%s' referenced in PRIMARY KEY for table '%s'", column, tableName);
+
+ if (!primaryKeyColumns.add(column))
+ throw ire("Duplicate column '%s' in PRIMARY KEY clause for table '%s'", column, tableName);
+
+ if (type.getType().isMultiCell())
+ {
+ if (type.isCollection())
+ throw ire("Invalid non-frozen collection type %s for PRIMARY KEY column '%s'", type, column);
+ else
+ throw ire("Invalid non-frozen user-defined type %s for PRIMARY KEY column '%s'", type, column);
+ }
+
+ if (type.getType().isCounter())
+ throw ire("counter type is not supported for PRIMARY KEY column '%s'", column);
+
+ if (type.getType().referencesDuration())
+ throw ire("duration type is not supported for PRIMARY KEY column '%s'", column);
+
+ if (staticColumns.contains(column))
+ throw ire("Static column '%s' cannot be part of the PRIMARY KEY", column);
+ });
+
+ List<AbstractType<?>> partitionKeyTypes = new ArrayList<>();
+ List<AbstractType<?>> clusteringTypes = new ArrayList<>();
+
+ partitionKeyColumns.forEach(column ->
+ {
+ CQL3Type type = columns.remove(column);
+ partitionKeyTypes.add(type.getType());
+ });
+
+ clusteringColumns.forEach(column ->
+ {
+ CQL3Type type = columns.remove(column);
+ boolean reverse = !clusteringOrder.getOrDefault(column, true);
+ clusteringTypes.add(reverse ? ReversedType.getInstance(type.getType()) : type.getType());
+ });
+
+ if (clusteringOrder.size() > clusteringColumns.size())
+ throw ire("Only clustering columns can be defined in CLUSTERING ORDER directive");
+
+ int n = 0;
+ for (ColumnIdentifier id : clusteringOrder.keySet())
+ {
+ ColumnIdentifier c = clusteringColumns.get(n);
+ if (!id.equals(c))
+ {
+ if (clusteringOrder.containsKey(c))
+ throw ire("The order of columns in the CLUSTERING ORDER directive must match that of the clustering columns (%s must appear before %s)", c, id);
+ else
+ throw ire("Missing CLUSTERING ORDER for column %s", c);
+ }
+ ++n;
+ }
+
+ // Static columns only make sense if we have at least one clustering column. Otherwise everything is static anyway
+ if (clusteringColumns.isEmpty() && !staticColumns.isEmpty())
+ throw ire("Static columns are only useful (and thus allowed) if the table has at least one clustering column");
+
+ /*
+ * Counter table validation
+ */
+
+ boolean hasCounters = rawColumns.values().stream().anyMatch(CQL3Type.Raw::isCounter);
+ if (hasCounters)
+ {
+ // We've handled anything that is not a PRIMARY KEY so columns only contains NON-PK columns. So
+ // if it's a counter table, make sure we don't have non-counter types
+ if (columns.values().stream().anyMatch(t -> !t.getType().isCounter()))
+ throw ire("Cannot mix counter and non counter columns in the same table");
+
+ if (params.defaultTimeToLive > 0)
+ throw ire("Cannot set %s on a table with counters", TableParams.Option.DEFAULT_TIME_TO_LIVE);
+ }
+
+ /*
+ * Create the builder
+ */
+
+ TableMetadata.Builder builder = TableMetadata.builder(keyspaceName, tableName);
+
+ if (attrs.hasProperty(TableAttributes.ID))
+ builder.id(attrs.getId());
+
+ builder.isCounter(hasCounters)
+ .params(params);
+
+ for (int i = 0; i < partitionKeyColumns.size(); i++)
+ builder.addPartitionKeyColumn(partitionKeyColumns.get(i), partitionKeyTypes.get(i));
+
+ for (int i = 0; i < clusteringColumns.size(); i++)
+ builder.addClusteringColumn(clusteringColumns.get(i), clusteringTypes.get(i));
+
+ columns.forEach((column, type) ->
+ {
+ if (staticColumns.contains(column))
+ builder.addStaticColumn(column, type.getType());
+ else
+ builder.addRegularColumn(column, type.getType());
+ });
+
+ return builder;
+ }
+
+ public static TableMetadata.Builder parse(String cql, String keyspace)
+ {
+ return CQLFragmentParser.parseAny(CqlParser::createTableStatement, cql, "CREATE TABLE")
+ .keyspace(keyspace)
+ .prepare(null) // works around a messy ClientState/QueryProcessor class init deadlock
+ .builder(Types.none());
+ }
+
+ public final static class Raw extends CQLStatement.Raw
+ {
+ private final QualifiedName name;
+ private final boolean ifNotExists;
+
+ private final Map<ColumnIdentifier, CQL3Type.Raw> rawColumns = new HashMap<>();
+ private final Set<ColumnIdentifier> staticColumns = new HashSet<>();
+ private final List<ColumnIdentifier> clusteringColumns = new ArrayList<>();
+
+ private List<ColumnIdentifier> partitionKeyColumns;
+
+ private final LinkedHashMap<ColumnIdentifier, Boolean> clusteringOrder = new LinkedHashMap<>();
+ public final TableAttributes attrs = new TableAttributes();
+
+ public Raw(QualifiedName name, boolean ifNotExists)
+ {
+ this.name = name;
+ this.ifNotExists = ifNotExists;
+ }
+
+ public CreateTableStatement prepare(ClientState state)
+ {
+ String keyspaceName = name.hasKeyspace() ? name.getKeyspace() : state.getKeyspace();
+
+ if (null == partitionKeyColumns)
+ throw ire("No PRIMARY KEY specifed for table '%s' (exactly one required)", name);
+
+ return new CreateTableStatement(keyspaceName,
+ name.getName(),
+
+ rawColumns,
+ staticColumns,
+ partitionKeyColumns,
+ clusteringColumns,
+
+ clusteringOrder,
+ attrs,
+
+ ifNotExists);
+ }
+
+ public String keyspace()
+ {
+ return name.getKeyspace();
+ }
+
+ public Raw keyspace(String keyspace)
+ {
+ name.setKeyspace(keyspace, true);
+ return this;
+ }
+
+ public String table()
+ {
+ return name.getName();
+ }
+
+ public void addColumn(ColumnIdentifier column, CQL3Type.Raw type, boolean isStatic)
+ {
+ if (null != rawColumns.put(column, type))
+ throw ire("Duplicate column '%s' declaration for table '%s'", column, name);
+
+ if (isStatic)
+ staticColumns.add(column);
+ }
+
+ public void setPartitionKeyColumn(ColumnIdentifier column)
+ {
+ setPartitionKeyColumns(Collections.singletonList(column));
+ }
+
+ public void setPartitionKeyColumns(List<ColumnIdentifier> columns)
+ {
+ if (null != partitionKeyColumns)
+ throw ire("Multiple PRIMARY KEY specified for table '%s' (exactly one required)", name);
+
+ partitionKeyColumns = columns;
+ }
+
+ public void markClusteringColumn(ColumnIdentifier column)
+ {
+ clusteringColumns.add(column);
+ }
+
+ public void extendClusteringOrder(ColumnIdentifier column, boolean ascending)
+ {
+ if (null != clusteringOrder.put(column, ascending))
+ throw ire("Duplicate column '%s' in CLUSTERING ORDER BY clause for table '%s'", column, name);
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/CreateTriggerStatement.java b/src/java/org/apache/cassandra/cql3/statements/schema/CreateTriggerStatement.java
new file mode 100644
index 0000000..e85ffd8
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/statements/schema/CreateTriggerStatement.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.statements.schema;
+
+import org.apache.cassandra.audit.AuditLogContext;
+import org.apache.cassandra.audit.AuditLogEntryType;
+import org.apache.cassandra.cql3.CQLStatement;
+import org.apache.cassandra.cql3.QualifiedName;
+import org.apache.cassandra.schema.*;
+import org.apache.cassandra.schema.Keyspaces.KeyspacesDiff;
+import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.triggers.TriggerExecutor;
+import org.apache.cassandra.transport.Event.SchemaChange;
+import org.apache.cassandra.transport.Event.SchemaChange.Change;
+import org.apache.cassandra.transport.Event.SchemaChange.Target;
+
+public final class CreateTriggerStatement extends AlterSchemaStatement
+{
+ private final String tableName;
+ private final String triggerName;
+ private final String triggerClass;
+ private final boolean ifNotExists;
+
+ public CreateTriggerStatement(String keyspaceName, String tableName, String triggerName, String triggerClass, boolean ifNotExists)
+ {
+ super(keyspaceName);
+ this.tableName = tableName;
+ this.triggerName = triggerName;
+ this.triggerClass = triggerClass;
+ this.ifNotExists = ifNotExists;
+ }
+
+ public Keyspaces apply(Keyspaces schema)
+ {
+ KeyspaceMetadata keyspace = schema.getNullable(keyspaceName);
+ if (null == keyspace)
+ throw ire("Keyspace '%s' doesn't exist", keyspaceName);
+
+ TableMetadata table = keyspace.getTableOrViewNullable(tableName);
+ if (null == table)
+ throw ire("Table '%s' doesn't exist", tableName);
+
+ if (table.isView())
+ throw ire("Cannot CREATE TRIGGER for a materialized view");
+
+ TriggerMetadata existingTrigger = table.triggers.get(triggerName).orElse(null);
+ if (null != existingTrigger)
+ {
+ if (ifNotExists)
+ return schema;
+
+ throw ire("Trigger '%s' already exists", triggerName);
+ }
+
+ try
+ {
+ TriggerExecutor.instance.loadTriggerInstance(triggerClass);
+ }
+ catch (Exception e)
+ {
+ throw ire("Trigger class '%s' couldn't be loaded", triggerClass);
+ }
+
+ TableMetadata newTable = table.withSwapped(table.triggers.with(TriggerMetadata.create(triggerName, triggerClass)));
+ return schema.withAddedOrUpdated(keyspace.withSwapped(keyspace.tables.withSwapped(newTable)));
+ }
+
+ SchemaChange schemaChangeEvent(KeyspacesDiff diff)
+ {
+ return new SchemaChange(Change.UPDATED, Target.TABLE, keyspaceName, tableName);
+ }
+
+ public void authorize(ClientState client)
+ {
+ client.ensureIsSuperuser("Only superusers are allowed to perform CREATE TRIGGER queries");
+ }
+
+ @Override
+ public AuditLogContext getAuditLogContext()
+ {
+ return new AuditLogContext(AuditLogEntryType.CREATE_TRIGGER, keyspaceName, triggerName);
+ }
+
+ public String toString()
+ {
+ return String.format("%s (%s, %s)", getClass().getSimpleName(), keyspaceName, triggerName);
+ }
+
+ public static final class Raw extends CQLStatement.Raw
+ {
+ private final QualifiedName tableName;
+ private final String triggerName;
+ private final String triggerClass;
+ private final boolean ifNotExists;
+
+ public Raw(QualifiedName tableName, String triggerName, String triggerClass, boolean ifNotExists)
+ {
+ this.tableName = tableName;
+ this.triggerName = triggerName;
+ this.triggerClass = triggerClass;
+ this.ifNotExists = ifNotExists;
+ }
+
+ public CreateTriggerStatement prepare(ClientState state)
+ {
+ String keyspaceName = tableName.hasKeyspace() ? tableName.getKeyspace() : state.getKeyspace();
+ return new CreateTriggerStatement(keyspaceName, tableName.getName(), triggerName, triggerClass, ifNotExists);
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/CreateTypeStatement.java b/src/java/org/apache/cassandra/cql3/statements/schema/CreateTypeStatement.java
new file mode 100644
index 0000000..7c1717e
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/statements/schema/CreateTypeStatement.java
@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.statements.schema;
+
+import java.util.*;
+
+import org.apache.cassandra.audit.AuditLogContext;
+import org.apache.cassandra.audit.AuditLogEntryType;
+import org.apache.cassandra.auth.Permission;
+import org.apache.cassandra.cql3.CQL3Type;
+import org.apache.cassandra.cql3.CQLStatement;
+import org.apache.cassandra.cql3.FieldIdentifier;
+import org.apache.cassandra.cql3.UTName;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.UserType;
+import org.apache.cassandra.schema.KeyspaceMetadata;
+import org.apache.cassandra.schema.Keyspaces;
+import org.apache.cassandra.schema.Keyspaces.KeyspacesDiff;
+import org.apache.cassandra.schema.Types;
+import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.transport.Event.SchemaChange;
+import org.apache.cassandra.transport.Event.SchemaChange.Change;
+import org.apache.cassandra.transport.Event.SchemaChange.Target;
+
+import static org.apache.cassandra.utils.ByteBufferUtil.bytes;
+
+import static java.util.stream.Collectors.toList;
+
+public final class CreateTypeStatement extends AlterSchemaStatement
+{
+ private final String typeName;
+ private final List<FieldIdentifier> fieldNames;
+ private final List<CQL3Type.Raw> rawFieldTypes;
+ private final boolean ifNotExists;
+
+ public CreateTypeStatement(String keyspaceName,
+ String typeName,
+ List<FieldIdentifier> fieldNames,
+ List<CQL3Type.Raw> rawFieldTypes,
+ boolean ifNotExists)
+ {
+ super(keyspaceName);
+ this.typeName = typeName;
+ this.fieldNames = fieldNames;
+ this.rawFieldTypes = rawFieldTypes;
+ this.ifNotExists = ifNotExists;
+ }
+
+ public Keyspaces apply(Keyspaces schema)
+ {
+ KeyspaceMetadata keyspace = schema.getNullable(keyspaceName);
+ if (null == keyspace)
+ throw ire("Keyspace '%s' doesn't exist", keyspaceName);
+
+ UserType existingType = keyspace.types.getNullable(bytes(typeName));
+ if (null != existingType)
+ {
+ if (ifNotExists)
+ return schema;
+
+ throw ire("A user type with name '%s' already exists", typeName);
+ }
+
+ Set<FieldIdentifier> usedNames = new HashSet<>();
+ for (FieldIdentifier name : fieldNames)
+ if (!usedNames.add(name))
+ throw ire("Duplicate field name '%s' in type '%s'", name, typeName);
+
+ for (CQL3Type.Raw type : rawFieldTypes)
+ {
+ if (type.isCounter())
+ throw ire("A user type cannot contain counters");
+
+ if (type.isUDT() && !type.isFrozen())
+ throw ire("A user type cannot contain non-frozen UDTs");
+ }
+
+ List<AbstractType<?>> fieldTypes =
+ rawFieldTypes.stream()
+ .map(t -> t.prepare(keyspaceName, keyspace.types).getType())
+ .collect(toList());
+
+ UserType udt = new UserType(keyspaceName, bytes(typeName), fieldNames, fieldTypes, true);
+ return schema.withAddedOrUpdated(keyspace.withSwapped(keyspace.types.with(udt)));
+ }
+
+ SchemaChange schemaChangeEvent(KeyspacesDiff diff)
+ {
+ return new SchemaChange(Change.CREATED, Target.TYPE, keyspaceName, typeName);
+ }
+
+ public void authorize(ClientState client)
+ {
+ client.ensureKeyspacePermission(keyspaceName, Permission.CREATE);
+ }
+
+ @Override
+ public AuditLogContext getAuditLogContext()
+ {
+ return new AuditLogContext(AuditLogEntryType.CREATE_TYPE, keyspaceName, typeName);
+ }
+
+ public String toString()
+ {
+ return String.format("%s (%s, %s)", getClass().getSimpleName(), keyspaceName, typeName);
+ }
+
+ public static final class Raw extends CQLStatement.Raw
+ {
+ private final UTName name;
+ private final boolean ifNotExists;
+
+ private final List<FieldIdentifier> fieldNames = new ArrayList<>();
+ private final List<CQL3Type.Raw> rawFieldTypes = new ArrayList<>();
+
+ public Raw(UTName name, boolean ifNotExists)
+ {
+ this.name = name;
+ this.ifNotExists = ifNotExists;
+ }
+
+ public CreateTypeStatement prepare(ClientState state)
+ {
+ String keyspaceName = name.hasKeyspace() ? name.getKeyspace() : state.getKeyspace();
+ return new CreateTypeStatement(keyspaceName, name.getStringTypeName(), fieldNames, rawFieldTypes, ifNotExists);
+ }
+
+ public void addField(FieldIdentifier name, CQL3Type.Raw type)
+ {
+ fieldNames.add(name);
+ rawFieldTypes.add(type);
+ }
+
+ public void addToRawBuilder(Types.RawBuilder builder)
+ {
+ builder.add(name.getStringTypeName(),
+ fieldNames.stream().map(FieldIdentifier::toString).collect(toList()),
+ rawFieldTypes.stream().map(CQL3Type.Raw::toString).collect(toList()));
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/CreateViewStatement.java b/src/java/org/apache/cassandra/cql3/statements/schema/CreateViewStatement.java
new file mode 100644
index 0000000..7e51eb2
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/statements/schema/CreateViewStatement.java
@@ -0,0 +1,423 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.statements.schema;
+
+import java.util.*;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Lists;
+
+import org.apache.cassandra.audit.AuditLogContext;
+import org.apache.cassandra.audit.AuditLogEntryType;
+import org.apache.cassandra.auth.Permission;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.*;
+import org.apache.cassandra.cql3.restrictions.StatementRestrictions;
+import org.apache.cassandra.cql3.selection.RawSelector;
+import org.apache.cassandra.cql3.selection.Selectable;
+import org.apache.cassandra.cql3.statements.StatementType;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.ReversedType;
+import org.apache.cassandra.db.view.View;
+import org.apache.cassandra.exceptions.AlreadyExistsException;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.schema.*;
+import org.apache.cassandra.schema.Keyspaces.KeyspacesDiff;
+import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.transport.Event.SchemaChange;
+import org.apache.cassandra.transport.Event.SchemaChange.Change;
+import org.apache.cassandra.transport.Event.SchemaChange.Target;
+
+import static java.lang.String.join;
+
+import static com.google.common.collect.Iterables.concat;
+import static com.google.common.collect.Iterables.filter;
+import static com.google.common.collect.Iterables.transform;
+
+public final class CreateViewStatement extends AlterSchemaStatement
+{
+ private final String tableName;
+ private final String viewName;
+
+ private final List<RawSelector> rawColumns;
+ private final List<ColumnIdentifier> partitionKeyColumns;
+ private final List<ColumnIdentifier> clusteringColumns;
+
+ private final WhereClause whereClause;
+
+ private final LinkedHashMap<ColumnIdentifier, Boolean> clusteringOrder;
+ private final TableAttributes attrs;
+
+ private final boolean ifNotExists;
+
+ public CreateViewStatement(String keyspaceName,
+ String tableName,
+ String viewName,
+
+ List<RawSelector> rawColumns,
+ List<ColumnIdentifier> partitionKeyColumns,
+ List<ColumnIdentifier> clusteringColumns,
+
+ WhereClause whereClause,
+
+ LinkedHashMap<ColumnIdentifier, Boolean> clusteringOrder,
+ TableAttributes attrs,
+
+ boolean ifNotExists)
+ {
+ super(keyspaceName);
+ this.tableName = tableName;
+ this.viewName = viewName;
+
+ this.rawColumns = rawColumns;
+ this.partitionKeyColumns = partitionKeyColumns;
+ this.clusteringColumns = clusteringColumns;
+
+ this.whereClause = whereClause;
+
+ this.clusteringOrder = clusteringOrder;
+ this.attrs = attrs;
+
+ this.ifNotExists = ifNotExists;
+ }
+
+ public Keyspaces apply(Keyspaces schema)
+ {
+ if (!DatabaseDescriptor.getEnableMaterializedViews())
+ throw ire("Materialized views are disabled. Enable in cassandra.yaml to use.");
+
+ /*
+ * Basic dependency validations
+ */
+
+ KeyspaceMetadata keyspace = schema.getNullable(keyspaceName);
+ if (null == keyspace)
+ throw ire("Keyspace '%s' doesn't exist", keyspaceName);
+
+ if (keyspace.createReplicationStrategy().hasTransientReplicas())
+ throw new InvalidRequestException("Materialized views are not supported on transiently replicated keyspaces");
+
+ TableMetadata table = keyspace.tables.getNullable(tableName);
+ if (null == table)
+ throw ire("Base table '%s' doesn't exist", tableName);
+
+ if (keyspace.hasTable(viewName))
+ throw ire("Cannot create materialized view '%s' - a table with the same name already exists", viewName);
+
+ if (keyspace.hasView(viewName))
+ {
+ if (ifNotExists)
+ return schema;
+
+ throw new AlreadyExistsException(keyspaceName, viewName);
+ }
+
+ /*
+ * Base table validation
+ */
+
+ if (table.isCounter())
+ throw ire("Materialized views are not supported on counter tables");
+
+ if (table.isView())
+ throw ire("Materialized views cannot be created against other materialized views");
+
+ if (table.params.gcGraceSeconds == 0)
+ {
+ throw ire("Cannot create materialized view '%s' for base table " +
+ "'%s' with gc_grace_seconds of 0, since this value is " +
+ "used to TTL undelivered updates. Setting gc_grace_seconds" +
+ " too low might cause undelivered updates to expire " +
+ "before being replayed.",
+ viewName, tableName);
+ }
+
+ /*
+ * Process SELECT clause
+ */
+
+ Set<ColumnIdentifier> selectedColumns = new HashSet<>();
+
+ if (rawColumns.isEmpty()) // SELECT *
+ table.columns().forEach(c -> selectedColumns.add(c.name));
+
+ rawColumns.forEach(selector ->
+ {
+ if (null != selector.alias)
+ throw ire("Cannot use aliases when defining a materialized view (got %s)", selector);
+
+ if (!(selector.selectable instanceof Selectable.RawIdentifier))
+ throw ire("Can only select columns by name when defining a materialized view (got %s)", selector.selectable);
+
+ // will throw IRE if the column doesn't exist in the base table
+ ColumnMetadata column = (ColumnMetadata) selector.selectable.prepare(table);
+
+ selectedColumns.add(column.name);
+ });
+
+ selectedColumns.stream()
+ .map(table::getColumn)
+ .filter(ColumnMetadata::isStatic)
+ .findAny()
+ .ifPresent(c -> { throw ire("Cannot include static column '%s' in materialized view '%s'", c, viewName); });
+
+ /*
+ * Process PRIMARY KEY columns and CLUSTERING ORDER BY clause
+ */
+
+ if (partitionKeyColumns.isEmpty())
+ throw ire("Must provide at least one partition key column for materialized view '%s'", viewName);
+
+ HashSet<ColumnIdentifier> primaryKeyColumns = new HashSet<>();
+
+ concat(partitionKeyColumns, clusteringColumns).forEach(name ->
+ {
+ ColumnMetadata column = table.getColumn(name);
+ if (null == column || !selectedColumns.contains(name))
+ throw ire("Unknown column '%s' referenced in PRIMARY KEY for materialized view '%s'", name, viewName);
+
+ if (!primaryKeyColumns.add(name))
+ throw ire("Duplicate column '%s' in PRIMARY KEY clause for materialized view '%s'", name, viewName);
+
+ AbstractType<?> type = column.type;
+
+ if (type.isMultiCell())
+ {
+ if (type.isCollection())
+ throw ire("Invalid non-frozen collection type '%s' for PRIMARY KEY column '%s'", type, name);
+ else
+ throw ire("Invalid non-frozen user-defined type '%s' for PRIMARY KEY column '%s'", type, name);
+ }
+
+ if (type.isCounter())
+ throw ire("counter type is not supported for PRIMARY KEY column '%s'", name);
+
+ if (type.referencesDuration())
+ throw ire("duration type is not supported for PRIMARY KEY column '%s'", name);
+ });
+
+ // If we give a clustering order, we must explicitly do so for all aliases and in the order of the PK
+ if (!clusteringOrder.isEmpty() && !clusteringColumns.equals(new ArrayList<>(clusteringOrder.keySet())))
+ throw ire("Clustering key columns must exactly match columns in CLUSTERING ORDER BY directive");
+
+ /*
+ * We need to include all of the primary key columns from the base table in order to make sure that we do not
+ * overwrite values in the view. We cannot support "collapsing" the base table into a smaller number of rows in
+ * the view because if we need to generate a tombstone, we have no way of knowing which value is currently being
+ * used in the view and whether or not to generate a tombstone. In order to not surprise our users, we require
+ * that they include all of the columns. We provide them with a list of all of the columns left to include.
+ */
+ List<ColumnIdentifier> missingPrimaryKeyColumns =
+ Lists.newArrayList(filter(transform(table.primaryKeyColumns(), c -> c.name), c -> !primaryKeyColumns.contains(c)));
+
+ if (!missingPrimaryKeyColumns.isEmpty())
+ {
+ throw ire("Cannot create materialized view '%s' without primary key columns %s from base table '%s'",
+ viewName, join(", ", transform(missingPrimaryKeyColumns, ColumnIdentifier::toString)), tableName);
+ }
+
+ Set<ColumnIdentifier> regularBaseTableColumnsInViewPrimaryKey = new HashSet<>(primaryKeyColumns);
+ transform(table.primaryKeyColumns(), c -> c.name).forEach(regularBaseTableColumnsInViewPrimaryKey::remove);
+ if (regularBaseTableColumnsInViewPrimaryKey.size() > 1)
+ {
+ throw ire("Cannot include more than one non-primary key column in materialized view primary key (got %s)",
+ join(", ", transform(regularBaseTableColumnsInViewPrimaryKey, ColumnIdentifier::toString)));
+ }
+
+ /*
+ * Process WHERE clause
+ */
+
+ if (whereClause.containsCustomExpressions())
+ throw ire("WHERE clause for materialized view '%s' cannot contain custom index expressions", viewName);
+
+ StatementRestrictions restrictions =
+ new StatementRestrictions(StatementType.SELECT,
+ table,
+ whereClause,
+ VariableSpecifications.empty(),
+ false,
+ false,
+ true,
+ true);
+
+ List<ColumnIdentifier> nonRestrictedPrimaryKeyColumns =
+ Lists.newArrayList(filter(primaryKeyColumns, name -> !restrictions.isRestricted(table.getColumn(name))));
+
+ if (!nonRestrictedPrimaryKeyColumns.isEmpty())
+ {
+ throw ire("Primary key columns %s must be restricted with 'IS NOT NULL' or otherwise",
+ join(", ", transform(nonRestrictedPrimaryKeyColumns, ColumnIdentifier::toString)));
+ }
+
+ // See CASSANDRA-13798
+ Set<ColumnMetadata> restrictedNonPrimaryKeyColumns = restrictions.nonPKRestrictedColumns(false);
+ if (!restrictedNonPrimaryKeyColumns.isEmpty() && !Boolean.getBoolean("cassandra.mv.allow_filtering_nonkey_columns_unsafe"))
+ {
+ throw ire("Non-primary key columns can only be restricted with 'IS NOT NULL' (got: %s restricted illegally)",
+ join(",", transform(restrictedNonPrimaryKeyColumns, ColumnMetadata::toString)));
+ }
+
+ /*
+ * Validate WITH params
+ */
+
+ attrs.validate();
+
+ if (attrs.hasOption(TableParams.Option.DEFAULT_TIME_TO_LIVE))
+ {
+ throw ire("Cannot set default_time_to_live for a materialized view. " +
+ "Data in a materialized view always expire at the same time than " +
+ "the corresponding data in the parent table.");
+ }
+
+ /*
+ * Build the thing
+ */
+
+ TableMetadata.Builder builder = TableMetadata.builder(keyspaceName, viewName);
+
+ if (attrs.hasProperty(TableAttributes.ID))
+ builder.id(attrs.getId());
+
+ builder.params(attrs.asNewTableParams())
+ .kind(TableMetadata.Kind.VIEW);
+
+ partitionKeyColumns.forEach(name -> builder.addPartitionKeyColumn(name, getType(table, name)));
+ clusteringColumns.forEach(name -> builder.addClusteringColumn(name, getType(table, name)));
+
+ selectedColumns.stream()
+ .filter(name -> !primaryKeyColumns.contains(name))
+ .forEach(name -> builder.addRegularColumn(name, getType(table, name)));
+
+ ViewMetadata view = new ViewMetadata(table.id, table.name, rawColumns.isEmpty(), whereClause, builder.build());
+ view.metadata.validate();
+
+ return schema.withAddedOrUpdated(keyspace.withSwapped(keyspace.views.with(view)));
+ }
+
+ SchemaChange schemaChangeEvent(KeyspacesDiff diff)
+ {
+ return new SchemaChange(Change.CREATED, Target.TABLE, keyspaceName, viewName);
+ }
+
+ public void authorize(ClientState client)
+ {
+ client.ensureTablePermission(keyspaceName, tableName, Permission.ALTER);
+ }
+
+ private AbstractType<?> getType(TableMetadata table, ColumnIdentifier name)
+ {
+ AbstractType<?> type = table.getColumn(name).type;
+ boolean reverse = !clusteringOrder.getOrDefault(name, true);
+
+ if (type.isReversed() && !reverse)
+ return ((ReversedType) type).baseType;
+ else if (!type.isReversed() && reverse)
+ return ReversedType.getInstance(type);
+ else
+ return type;
+ }
+
+ @Override
+ Set<String> clientWarnings(KeyspacesDiff diff)
+ {
+ return ImmutableSet.of(View.USAGE_WARNING);
+ }
+
+ @Override
+ public AuditLogContext getAuditLogContext()
+ {
+ return new AuditLogContext(AuditLogEntryType.CREATE_VIEW, keyspaceName, viewName);
+ }
+
+ public String toString()
+ {
+ return String.format("%s (%s, %s)", getClass().getSimpleName(), keyspaceName, viewName);
+ }
+
+ public final static class Raw extends CQLStatement.Raw
+ {
+ private final QualifiedName tableName;
+ private final QualifiedName viewName;
+ private final boolean ifNotExists;
+
+ private final List<RawSelector> rawColumns;
+ private final List<ColumnIdentifier> clusteringColumns = new ArrayList<>();
+ private List<ColumnIdentifier> partitionKeyColumns;
+
+ private final WhereClause whereClause;
+
+ private final LinkedHashMap<ColumnIdentifier, Boolean> clusteringOrder = new LinkedHashMap<>();
+ public final TableAttributes attrs = new TableAttributes();
+
+ public Raw(QualifiedName tableName, QualifiedName viewName, List<RawSelector> rawColumns, WhereClause whereClause, boolean ifNotExists)
+ {
+ this.tableName = tableName;
+ this.viewName = viewName;
+ this.rawColumns = rawColumns;
+ this.whereClause = whereClause;
+ this.ifNotExists = ifNotExists;
+ }
+
+ public CreateViewStatement prepare(ClientState state)
+ {
+ String keyspaceName = viewName.hasKeyspace() ? viewName.getKeyspace() : state.getKeyspace();
+
+ if (tableName.hasKeyspace() && !keyspaceName.equals(tableName.getKeyspace()))
+ throw ire("Cannot create a materialized view on a table in a different keyspace");
+
+ if (!bindVariables.isEmpty())
+ throw ire("Bind variables are not allowed in CREATE MATERIALIZED VIEW statements");
+
+ if (null == partitionKeyColumns)
+ throw ire("No PRIMARY KEY specifed for view '%s' (exactly one required)", viewName);
+
+ return new CreateViewStatement(keyspaceName,
+ tableName.getName(),
+ viewName.getName(),
+
+ rawColumns,
+ partitionKeyColumns,
+ clusteringColumns,
+
+ whereClause,
+
+ clusteringOrder,
+ attrs,
+
+ ifNotExists);
+ }
+
+ public void setPartitionKeyColumns(List<ColumnIdentifier> columns)
+ {
+ partitionKeyColumns = columns;
+ }
+
+ public void markClusteringColumn(ColumnIdentifier column)
+ {
+ clusteringColumns.add(column);
+ }
+
+ public void extendClusteringOrder(ColumnIdentifier column, boolean ascending)
+ {
+ if (null != clusteringOrder.put(column, ascending))
+ throw ire("Duplicate column '%s' in CLUSTERING ORDER BY clause for view '%s'", column, viewName);
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/DropAggregateStatement.java b/src/java/org/apache/cassandra/cql3/statements/schema/DropAggregateStatement.java
new file mode 100644
index 0000000..d24f77e
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/statements/schema/DropAggregateStatement.java
@@ -0,0 +1,179 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.statements.schema;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.function.Predicate;
+import java.util.stream.Stream;
+
+import org.apache.cassandra.audit.AuditLogContext;
+import org.apache.cassandra.audit.AuditLogEntryType;
+import org.apache.cassandra.auth.FunctionResource;
+import org.apache.cassandra.auth.Permission;
+import org.apache.cassandra.cql3.CQL3Type;
+import org.apache.cassandra.cql3.CQLStatement;
+import org.apache.cassandra.cql3.functions.Function;
+import org.apache.cassandra.cql3.functions.FunctionName;
+import org.apache.cassandra.cql3.functions.UDAggregate;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.schema.*;
+import org.apache.cassandra.schema.Keyspaces.KeyspacesDiff;
+import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.transport.Event.SchemaChange;
+import org.apache.cassandra.transport.Event.SchemaChange.Change;
+
+import static java.lang.String.format;
+import static java.lang.String.join;
+import static java.util.stream.Collectors.toList;
+
+import static com.google.common.collect.Iterables.transform;
+
+public final class DropAggregateStatement extends AlterSchemaStatement
+{
+ private final String aggregateName;
+ private final List<CQL3Type.Raw> arguments;
+ private final boolean argumentsSpeficied;
+ private final boolean ifExists;
+
+ public DropAggregateStatement(String keyspaceName,
+ String aggregateName,
+ List<CQL3Type.Raw> arguments,
+ boolean argumentsSpeficied,
+ boolean ifExists)
+ {
+ super(keyspaceName);
+ this.aggregateName = aggregateName;
+ this.arguments = arguments;
+ this.argumentsSpeficied = argumentsSpeficied;
+ this.ifExists = ifExists;
+ }
+
+ public Keyspaces apply(Keyspaces schema)
+ {
+ String name =
+ argumentsSpeficied
+ ? format("%s.%s(%s)", keyspaceName, aggregateName, join(", ", transform(arguments, CQL3Type.Raw::toString)))
+ : format("%s.%s", keyspaceName, aggregateName);
+
+ KeyspaceMetadata keyspace = schema.getNullable(keyspaceName);
+ if (null == keyspace)
+ {
+ if (ifExists)
+ return schema;
+
+ throw ire("Aggregate '%s' doesn't exist", name);
+ }
+
+ Collection<Function> aggregates = keyspace.functions.get(new FunctionName(keyspaceName, aggregateName));
+ if (aggregates.size() > 1 && !argumentsSpeficied)
+ {
+ throw ire("'DROP AGGREGATE %s' matches multiple function definitions; " +
+ "specify the argument types by issuing a statement like " +
+ "'DROP AGGREGATE %s (type, type, ...)'. You can use cqlsh " +
+ "'DESCRIBE AGGREGATE %s' command to find all overloads",
+ aggregateName, aggregateName, aggregateName);
+ }
+
+ arguments.stream()
+ .filter(CQL3Type.Raw::isFrozen)
+ .findFirst()
+ .ifPresent(t -> { throw ire("Argument '%s' cannot be frozen; remove frozen<> modifier from '%s'", t, t); });
+
+ List<AbstractType<?>> argumentTypes = prepareArgumentTypes(keyspace.types);
+
+ Predicate<Function> filter = Functions.Filter.UDA;
+ if (argumentsSpeficied)
+ filter = filter.and(f -> Functions.typesMatch(f.argTypes(), argumentTypes));
+
+ Function aggregate = aggregates.stream().filter(filter).findAny().orElse(null);
+ if (null == aggregate)
+ {
+ if (ifExists)
+ return schema;
+
+ throw ire("Aggregate '%s' doesn't exist", name);
+ }
+
+ return schema.withAddedOrUpdated(keyspace.withSwapped(keyspace.functions.without(aggregate)));
+ }
+
+ SchemaChange schemaChangeEvent(KeyspacesDiff diff)
+ {
+ Functions dropped = diff.altered.get(0).udas.dropped;
+ assert dropped.size() == 1;
+ return SchemaChange.forAggregate(Change.DROPPED, (UDAggregate) dropped.iterator().next());
+ }
+
+ public void authorize(ClientState client)
+ {
+ KeyspaceMetadata keyspace = Schema.instance.getKeyspaceMetadata(keyspaceName);
+ if (null == keyspace)
+ return;
+
+ Stream<Function> functions = keyspace.functions.get(new FunctionName(keyspaceName, aggregateName)).stream();
+ if (argumentsSpeficied)
+ functions = functions.filter(f -> Functions.typesMatch(f.argTypes(), prepareArgumentTypes(keyspace.types)));
+
+ functions.forEach(f -> client.ensurePermission(Permission.DROP, FunctionResource.function(f)));
+ }
+
+ @Override
+ public AuditLogContext getAuditLogContext()
+ {
+ return new AuditLogContext(AuditLogEntryType.DROP_AGGREGATE, keyspaceName, aggregateName);
+ }
+
+ public String toString()
+ {
+ return String.format("%s (%s, %s)", getClass().getSimpleName(), keyspaceName, aggregateName);
+ }
+
+ private List<AbstractType<?>> prepareArgumentTypes(Types types)
+ {
+ return arguments.stream()
+ .map(t -> t.prepare(keyspaceName, types))
+ .map(CQL3Type::getType)
+ .collect(toList());
+ }
+
+ public static final class Raw extends CQLStatement.Raw
+ {
+ private final FunctionName name;
+ private final List<CQL3Type.Raw> arguments;
+ private final boolean argumentsSpecified;
+ private final boolean ifExists;
+
+ public Raw(FunctionName name,
+ List<CQL3Type.Raw> arguments,
+ boolean argumentsSpecified,
+ boolean ifExists)
+ {
+ this.name = name;
+ this.arguments = arguments;
+ this.argumentsSpecified = argumentsSpecified;
+ this.ifExists = ifExists;
+ }
+
+ public DropAggregateStatement prepare(ClientState state)
+ {
+ String keyspaceName = name.hasKeyspace() ? name.keyspace : state.getKeyspace();
+ return new DropAggregateStatement(keyspaceName, name.name, arguments, argumentsSpecified, ifExists);
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/DropFunctionStatement.java b/src/java/org/apache/cassandra/cql3/statements/schema/DropFunctionStatement.java
new file mode 100644
index 0000000..f7d7d4a
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/statements/schema/DropFunctionStatement.java
@@ -0,0 +1,187 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.statements.schema;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.function.Predicate;
+import java.util.stream.Stream;
+
+import org.apache.cassandra.audit.AuditLogContext;
+import org.apache.cassandra.audit.AuditLogEntryType;
+import org.apache.cassandra.auth.FunctionResource;
+import org.apache.cassandra.auth.Permission;
+import org.apache.cassandra.cql3.CQL3Type;
+import org.apache.cassandra.cql3.CQLStatement;
+import org.apache.cassandra.cql3.functions.*;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.schema.*;
+import org.apache.cassandra.schema.Keyspaces.KeyspacesDiff;
+import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.transport.Event.SchemaChange;
+import org.apache.cassandra.transport.Event.SchemaChange.Change;
+
+import static java.lang.String.format;
+import static java.lang.String.join;
+import static java.util.stream.Collectors.joining;
+import static java.util.stream.Collectors.toList;
+
+import static com.google.common.collect.Iterables.transform;
+
+public final class DropFunctionStatement extends AlterSchemaStatement
+{
+ private final String functionName;
+ private final Collection<CQL3Type.Raw> arguments;
+ private final boolean argumentsSpeficied;
+ private final boolean ifExists;
+
+ public DropFunctionStatement(String keyspaceName,
+ String functionName,
+ Collection<CQL3Type.Raw> arguments,
+ boolean argumentsSpeficied,
+ boolean ifExists)
+ {
+ super(keyspaceName);
+ this.functionName = functionName;
+ this.arguments = arguments;
+ this.argumentsSpeficied = argumentsSpeficied;
+ this.ifExists = ifExists;
+ }
+
+ public Keyspaces apply(Keyspaces schema)
+ {
+ String name =
+ argumentsSpeficied
+ ? format("%s.%s(%s)", keyspaceName, functionName, join(", ", transform(arguments, CQL3Type.Raw::toString)))
+ : format("%s.%s", keyspaceName, functionName);
+
+ KeyspaceMetadata keyspace = schema.getNullable(keyspaceName);
+ if (null == keyspace)
+ {
+ if (ifExists)
+ return schema;
+
+ throw ire("Function '%s' doesn't exist", name);
+ }
+
+ Collection<Function> functions = keyspace.functions.get(new FunctionName(keyspaceName, functionName));
+ if (functions.size() > 1 && !argumentsSpeficied)
+ {
+ throw ire("'DROP FUNCTION %s' matches multiple function definitions; " +
+ "specify the argument types by issuing a statement like " +
+ "'DROP FUNCTION %s (type, type, ...)'. You can use cqlsh " +
+ "'DESCRIBE FUNCTION %s' command to find all overloads",
+ functionName, functionName, functionName);
+ }
+
+ arguments.stream()
+ .filter(CQL3Type.Raw::isFrozen)
+ .findFirst()
+ .ifPresent(t -> { throw ire("Argument '%s' cannot be frozen; remove frozen<> modifier from '%s'", t, t); });
+
+ List<AbstractType<?>> argumentTypes = prepareArgumentTypes(keyspace.types);
+
+ Predicate<Function> filter = Functions.Filter.UDF;
+ if (argumentsSpeficied)
+ filter = filter.and(f -> Functions.typesMatch(f.argTypes(), argumentTypes));
+
+ Function function = functions.stream().filter(filter).findAny().orElse(null);
+ if (null == function)
+ {
+ if (ifExists)
+ return schema;
+
+ throw ire("Function '%s' doesn't exist", name);
+ }
+
+ String dependentAggregates =
+ keyspace.functions
+ .aggregatesUsingFunction(function)
+ .map(a -> a.name().toString())
+ .collect(joining(", "));
+
+ if (!dependentAggregates.isEmpty())
+ throw ire("Function '%s' is still referenced by aggregates %s", name, dependentAggregates);
+
+ return schema.withAddedOrUpdated(keyspace.withSwapped(keyspace.functions.without(function)));
+ }
+
+ SchemaChange schemaChangeEvent(KeyspacesDiff diff)
+ {
+ Functions dropped = diff.altered.get(0).udfs.dropped;
+ assert dropped.size() == 1;
+ return SchemaChange.forFunction(Change.DROPPED, (UDFunction) dropped.iterator().next());
+ }
+
+ public void authorize(ClientState client)
+ {
+ KeyspaceMetadata keyspace = Schema.instance.getKeyspaceMetadata(keyspaceName);
+ if (null == keyspace)
+ return;
+
+ Stream<Function> functions = keyspace.functions.get(new FunctionName(keyspaceName, functionName)).stream();
+ if (argumentsSpeficied)
+ functions = functions.filter(f -> Functions.typesMatch(f.argTypes(), prepareArgumentTypes(keyspace.types)));
+
+ functions.forEach(f -> client.ensurePermission(Permission.DROP, FunctionResource.function(f)));
+ }
+
+ @Override
+ public AuditLogContext getAuditLogContext()
+ {
+ return new AuditLogContext(AuditLogEntryType.DROP_FUNCTION, keyspaceName, functionName);
+ }
+
+ public String toString()
+ {
+ return String.format("%s (%s, %s)", getClass().getSimpleName(), keyspaceName, functionName);
+ }
+
+ private List<AbstractType<?>> prepareArgumentTypes(Types types)
+ {
+ return arguments.stream()
+ .map(t -> t.prepare(keyspaceName, types))
+ .map(CQL3Type::getType)
+ .collect(toList());
+ }
+
+ public static final class Raw extends CQLStatement.Raw
+ {
+ private final FunctionName name;
+ private final List<CQL3Type.Raw> arguments;
+ private final boolean argumentsSpecified;
+ private final boolean ifExists;
+
+ public Raw(FunctionName name,
+ List<CQL3Type.Raw> arguments,
+ boolean argumentsSpecified,
+ boolean ifExists)
+ {
+ this.name = name;
+ this.arguments = arguments;
+ this.argumentsSpecified = argumentsSpecified;
+ this.ifExists = ifExists;
+ }
+
+ public DropFunctionStatement prepare(ClientState state)
+ {
+ String keyspaceName = name.hasKeyspace() ? name.keyspace : state.getKeyspace();
+ return new DropFunctionStatement(keyspaceName, name.name, arguments, argumentsSpecified, ifExists);
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/DropIndexStatement.java b/src/java/org/apache/cassandra/cql3/statements/schema/DropIndexStatement.java
new file mode 100644
index 0000000..2186470
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/statements/schema/DropIndexStatement.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.statements.schema;
+
+import org.apache.cassandra.audit.AuditLogContext;
+import org.apache.cassandra.audit.AuditLogEntryType;
+import org.apache.cassandra.auth.Permission;
+import org.apache.cassandra.cql3.CQLStatement;
+import org.apache.cassandra.cql3.QualifiedName;
+import org.apache.cassandra.schema.Diff;
+import org.apache.cassandra.schema.*;
+import org.apache.cassandra.schema.KeyspaceMetadata.KeyspaceDiff;
+import org.apache.cassandra.schema.Keyspaces.KeyspacesDiff;
+import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.transport.Event.SchemaChange;
+import org.apache.cassandra.transport.Event.SchemaChange.Change;
+import org.apache.cassandra.transport.Event.SchemaChange.Target;
+
+public final class DropIndexStatement extends AlterSchemaStatement
+{
+ private final String indexName;
+ private final boolean ifExists;
+
+ public DropIndexStatement(String keyspaceName, String indexName, boolean ifExists)
+ {
+ super(keyspaceName);
+ this.indexName = indexName;
+ this.ifExists = ifExists;
+ }
+
+ public Keyspaces apply(Keyspaces schema)
+ {
+ KeyspaceMetadata keyspace = schema.getNullable(keyspaceName);
+
+ TableMetadata table = null == keyspace
+ ? null
+ : keyspace.findIndexedTable(indexName).orElse(null);
+
+ if (null == table)
+ {
+ if (ifExists)
+ return schema;
+
+ throw ire("Index '%s.%s' doesn't exist'", keyspaceName, indexName);
+ }
+
+ TableMetadata newTable = table.withSwapped(table.indexes.without(indexName));
+ return schema.withAddedOrUpdated(keyspace.withSwapped(keyspace.tables.withSwapped(newTable)));
+ }
+
+ SchemaChange schemaChangeEvent(KeyspacesDiff diff)
+ {
+ assert diff.altered.size() == 1;
+ KeyspaceDiff ksDiff = diff.altered.get(0);
+
+ assert ksDiff.tables.altered.size() == 1;
+ Diff.Altered<TableMetadata> tableDiff = ksDiff.tables.altered.iterator().next();
+
+ return new SchemaChange(Change.UPDATED, Target.TABLE, keyspaceName, tableDiff.after.name);
+ }
+
+ public void authorize(ClientState client)
+ {
+ KeyspaceMetadata keyspace = Schema.instance.getKeyspaceMetadata(keyspaceName);
+ if (null == keyspace)
+ return;
+
+ keyspace.findIndexedTable(indexName)
+ .ifPresent(t -> client.ensureTablePermission(keyspaceName, t.name, Permission.ALTER));
+ }
+
+ @Override
+ public AuditLogContext getAuditLogContext()
+ {
+ return new AuditLogContext(AuditLogEntryType.DROP_INDEX, keyspaceName, indexName);
+ }
+
+ public String toString()
+ {
+ return String.format("%s (%s, %s)", getClass().getSimpleName(), keyspaceName, indexName);
+ }
+
+ public static final class Raw extends CQLStatement.Raw
+ {
+ private final QualifiedName name;
+ private final boolean ifExists;
+
+ public Raw(QualifiedName name, boolean ifExists)
+ {
+ this.name = name;
+ this.ifExists = ifExists;
+ }
+
+ public DropIndexStatement prepare(ClientState state)
+ {
+ String keyspaceName = name.hasKeyspace() ? name.getKeyspace() : state.getKeyspace();
+ return new DropIndexStatement(keyspaceName, name.getName(), ifExists);
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/DropKeyspaceStatement.java b/src/java/org/apache/cassandra/cql3/statements/schema/DropKeyspaceStatement.java
new file mode 100644
index 0000000..f2bd30b
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/statements/schema/DropKeyspaceStatement.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.statements.schema;
+
+import org.apache.cassandra.audit.AuditLogContext;
+import org.apache.cassandra.audit.AuditLogEntryType;
+import org.apache.cassandra.auth.Permission;
+import org.apache.cassandra.cql3.CQLStatement;
+import org.apache.cassandra.schema.Keyspaces;
+import org.apache.cassandra.schema.Keyspaces.KeyspacesDiff;
+import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.transport.Event.SchemaChange;
+import org.apache.cassandra.transport.Event.SchemaChange.Change;
+
+public final class DropKeyspaceStatement extends AlterSchemaStatement
+{
+ private final boolean ifExists;
+
+ public DropKeyspaceStatement(String keyspaceName, boolean ifExists)
+ {
+ super(keyspaceName);
+ this.ifExists = ifExists;
+ }
+
+ public Keyspaces apply(Keyspaces schema)
+ {
+ if (schema.containsKeyspace(keyspaceName))
+ return schema.without(keyspaceName);
+
+ if (ifExists)
+ return schema;
+
+ throw ire("Keyspace '%s' doesn't exist", keyspaceName);
+ }
+
+ SchemaChange schemaChangeEvent(KeyspacesDiff diff)
+ {
+ return new SchemaChange(Change.DROPPED, keyspaceName);
+ }
+
+ public void authorize(ClientState client)
+ {
+ client.ensureKeyspacePermission(keyspaceName, Permission.DROP);
+ }
+
+ @Override
+ public AuditLogContext getAuditLogContext()
+ {
+ return new AuditLogContext(AuditLogEntryType.DROP_KEYSPACE, keyspaceName);
+ }
+
+ public String toString()
+ {
+ return String.format("%s (%s)", getClass().getSimpleName(), keyspaceName);
+ }
+
+ public static final class Raw extends CQLStatement.Raw
+ {
+ private final String keyspaceName;
+ private final boolean ifExists;
+
+ public Raw(String keyspaceName, boolean ifExists)
+ {
+ this.keyspaceName = keyspaceName;
+ this.ifExists = ifExists;
+ }
+
+ public DropKeyspaceStatement prepare(ClientState state)
+ {
+ return new DropKeyspaceStatement(keyspaceName, ifExists);
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/DropTableStatement.java b/src/java/org/apache/cassandra/cql3/statements/schema/DropTableStatement.java
new file mode 100644
index 0000000..15c2a03
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/statements/schema/DropTableStatement.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.statements.schema;
+
+import org.apache.cassandra.audit.AuditLogContext;
+import org.apache.cassandra.audit.AuditLogEntryType;
+import org.apache.cassandra.auth.Permission;
+import org.apache.cassandra.cql3.CQLStatement;
+import org.apache.cassandra.cql3.QualifiedName;
+import org.apache.cassandra.schema.*;
+import org.apache.cassandra.schema.Keyspaces.KeyspacesDiff;
+import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.transport.Event.SchemaChange;
+import org.apache.cassandra.transport.Event.SchemaChange.Change;
+import org.apache.cassandra.transport.Event.SchemaChange.Target;
+
+import static java.lang.String.join;
+
+import static com.google.common.collect.Iterables.isEmpty;
+import static com.google.common.collect.Iterables.transform;
+
+public final class DropTableStatement extends AlterSchemaStatement
+{
+ private final String tableName;
+ private final boolean ifExists;
+
+ public DropTableStatement(String keyspaceName, String tableName, boolean ifExists)
+ {
+ super(keyspaceName);
+ this.tableName = tableName;
+ this.ifExists = ifExists;
+ }
+
+ public Keyspaces apply(Keyspaces schema)
+ {
+ KeyspaceMetadata keyspace = schema.getNullable(keyspaceName);
+
+ TableMetadata table = null == keyspace
+ ? null
+ : keyspace.getTableOrViewNullable(tableName);
+
+ if (null == table)
+ {
+ if (ifExists)
+ return schema;
+
+ throw ire("Table '%s.%s' doesn't exist", keyspaceName, tableName);
+ }
+
+ if (table.isView())
+ throw ire("Cannot use DROP TABLE on a materialized view. Please use DROP MATERIALIZED VIEW instead.");
+
+ Iterable<ViewMetadata> views = keyspace.views.forTable(table.id);
+ if (!isEmpty(views))
+ {
+ throw ire("Cannot drop a table when materialized views still depend on it (%s)",
+ keyspaceName,
+ join(", ", transform(views, ViewMetadata::name)));
+ }
+
+ return schema.withAddedOrUpdated(keyspace.withSwapped(keyspace.tables.without(table)));
+ }
+
+ SchemaChange schemaChangeEvent(KeyspacesDiff diff)
+ {
+ return new SchemaChange(Change.DROPPED, Target.TABLE, keyspaceName, tableName);
+ }
+
+ public void authorize(ClientState client)
+ {
+ client.ensureTablePermission(keyspaceName, tableName, Permission.DROP);
+ }
+
+ @Override
+ public AuditLogContext getAuditLogContext()
+ {
+ return new AuditLogContext(AuditLogEntryType.DROP_TABLE, keyspaceName, tableName);
+ }
+
+ public String toString()
+ {
+ return String.format("%s (%s, %s)", getClass().getSimpleName(), keyspaceName, tableName);
+ }
+
+ public static final class Raw extends CQLStatement.Raw
+ {
+ private final QualifiedName name;
+ private final boolean ifExists;
+
+ public Raw(QualifiedName name, boolean ifExists)
+ {
+ this.name = name;
+ this.ifExists = ifExists;
+ }
+
+ public DropTableStatement prepare(ClientState state)
+ {
+ String keyspaceName = name.hasKeyspace() ? name.getKeyspace() : state.getKeyspace();
+ return new DropTableStatement(keyspaceName, name.getName(), ifExists);
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/DropTriggerStatement.java b/src/java/org/apache/cassandra/cql3/statements/schema/DropTriggerStatement.java
new file mode 100644
index 0000000..967e568
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/statements/schema/DropTriggerStatement.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.statements.schema;
+
+import org.apache.cassandra.audit.AuditLogContext;
+import org.apache.cassandra.audit.AuditLogEntryType;
+import org.apache.cassandra.cql3.CQLStatement;
+import org.apache.cassandra.cql3.QualifiedName;
+import org.apache.cassandra.schema.*;
+import org.apache.cassandra.schema.Keyspaces.KeyspacesDiff;
+import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.transport.Event.SchemaChange;
+import org.apache.cassandra.transport.Event.SchemaChange.Change;
+import org.apache.cassandra.transport.Event.SchemaChange.Target;
+
+public final class DropTriggerStatement extends AlterSchemaStatement
+{
+ private final String tableName;
+ private final String triggerName;
+ private final boolean ifExists;
+
+ public DropTriggerStatement(String keyspaceName, String tableName, String triggerName, boolean ifExists)
+ {
+ super(keyspaceName);
+ this.tableName = tableName;
+ this.triggerName = triggerName;
+ this.ifExists = ifExists;
+ }
+
+ public Keyspaces apply(Keyspaces schema)
+ {
+ KeyspaceMetadata keyspace = schema.getNullable(keyspaceName);
+
+ TableMetadata table = null == keyspace
+ ? null
+ : keyspace.tables.getNullable(tableName);
+
+ TriggerMetadata trigger = null == table
+ ? null
+ : table.triggers.get(triggerName).orElse(null);
+
+ if (null == trigger)
+ {
+ if (ifExists)
+ return schema;
+
+ throw ire("Trigger '%s' on '%s.%s' doesn't exist", triggerName, keyspaceName, tableName);
+ }
+
+ TableMetadata newTable = table.withSwapped(table.triggers.without(triggerName));
+ return schema.withAddedOrUpdated(keyspace.withSwapped(keyspace.tables.withSwapped(newTable)));
+ }
+
+ SchemaChange schemaChangeEvent(KeyspacesDiff diff)
+ {
+ return new SchemaChange(Change.UPDATED, Target.TABLE, keyspaceName, tableName);
+ }
+
+ public void authorize(ClientState client)
+ {
+ client.ensureIsSuperuser("Only superusers are allowed to perfrom DROP TRIGGER queries");
+ }
+
+ @Override
+ public AuditLogContext getAuditLogContext()
+ {
+ return new AuditLogContext(AuditLogEntryType.DROP_TRIGGER, keyspaceName, triggerName);
+ }
+
+ public String toString()
+ {
+ return String.format("%s (%s, %s)", getClass().getSimpleName(), keyspaceName, triggerName);
+ }
+
+ public static final class Raw extends CQLStatement.Raw
+ {
+ private final QualifiedName tableName;
+ private final String triggerName;
+ private final boolean ifExists;
+
+ public Raw(QualifiedName tableName, String triggerName, boolean ifExists)
+ {
+ this.tableName = tableName;
+ this.triggerName = triggerName;
+ this.ifExists = ifExists;
+ }
+
+ public DropTriggerStatement prepare(ClientState state)
+ {
+ String keyspaceName = tableName.hasKeyspace() ? tableName.getKeyspace() : state.getKeyspace();
+ return new DropTriggerStatement(keyspaceName, tableName.getName(), triggerName, ifExists);
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/DropTypeStatement.java b/src/java/org/apache/cassandra/cql3/statements/schema/DropTypeStatement.java
new file mode 100644
index 0000000..6cda7ba
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/statements/schema/DropTypeStatement.java
@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.statements.schema;
+
+import java.nio.ByteBuffer;
+
+import org.apache.cassandra.audit.AuditLogContext;
+import org.apache.cassandra.audit.AuditLogEntryType;
+import org.apache.cassandra.auth.Permission;
+import org.apache.cassandra.cql3.CQLStatement;
+import org.apache.cassandra.cql3.UTName;
+import org.apache.cassandra.cql3.functions.Function;
+import org.apache.cassandra.db.marshal.UserType;
+import org.apache.cassandra.schema.KeyspaceMetadata;
+import org.apache.cassandra.schema.Keyspaces.KeyspacesDiff;
+import org.apache.cassandra.schema.Keyspaces;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.transport.Event.SchemaChange.Change;
+import org.apache.cassandra.transport.Event.SchemaChange.Target;
+import org.apache.cassandra.transport.Event.SchemaChange;
+
+import static java.lang.String.join;
+
+import static com.google.common.collect.Iterables.isEmpty;
+import static com.google.common.collect.Iterables.transform;
+
+import static org.apache.cassandra.utils.ByteBufferUtil.bytes;
+
+public final class DropTypeStatement extends AlterSchemaStatement
+{
+ private final String typeName;
+ private final boolean ifExists;
+
+ public DropTypeStatement(String keyspaceName, String typeName, boolean ifExists)
+ {
+ super(keyspaceName);
+ this.typeName = typeName;
+ this.ifExists = ifExists;
+ }
+
+ // TODO: expand types into tuples in all dropped columns of all tables
+ public Keyspaces apply(Keyspaces schema)
+ {
+ ByteBuffer name = bytes(typeName);
+
+ KeyspaceMetadata keyspace = schema.getNullable(keyspaceName);
+
+ UserType type = null == keyspace
+ ? null
+ : keyspace.types.getNullable(name);
+
+ if (null == type)
+ {
+ if (ifExists)
+ return schema;
+
+ throw ire("Type '%s.%s' doesn't exist", keyspaceName, typeName);
+ }
+
+ /*
+ * We don't want to drop a type unless it's not used anymore (mainly because
+ * if someone drops a type and recreates one with the same name but different
+ * definition with the previous name still in use, things can get messy).
+ * We have three places to check:
+ * 1) UDFs and UDAs using the type
+ * 2) other user type that can nest the one we drop and
+ * 3) existing tables referencing the type (maybe in a nested way).
+ */
+ Iterable<Function> functions = keyspace.functions.referencingUserType(name);
+ if (!isEmpty(functions))
+ {
+ throw ire("Cannot drop user type '%s.%s' as it is still used by functions %s",
+ keyspaceName,
+ typeName,
+ join(", ", transform(functions, f -> f.name().toString())));
+ }
+
+ Iterable<UserType> types = keyspace.types.referencingUserType(name);
+ if (!isEmpty(types))
+ {
+ throw ire("Cannot drop user type '%s.%s' as it is still used by user types %s",
+ keyspaceName,
+ typeName,
+ join(", ", transform(types, UserType::getNameAsString)));
+
+ }
+
+ Iterable<TableMetadata> tables = keyspace.tables.referencingUserType(name);
+ if (!isEmpty(tables))
+ {
+ throw ire("Cannot drop user type '%s.%s' as it is still used by tables %s",
+ keyspaceName,
+ typeName,
+ join(", ", transform(tables, t -> t.name)));
+ }
+
+ return schema.withAddedOrUpdated(keyspace.withSwapped(keyspace.types.without(type)));
+ }
+
+ SchemaChange schemaChangeEvent(KeyspacesDiff diff)
+ {
+ return new SchemaChange(Change.DROPPED, Target.TYPE, keyspaceName, typeName);
+ }
+
+ public void authorize(ClientState client)
+ {
+ client.ensureKeyspacePermission(keyspaceName, Permission.DROP);
+ }
+
+ @Override
+ public AuditLogContext getAuditLogContext()
+ {
+ return new AuditLogContext(AuditLogEntryType.DROP_TYPE, keyspaceName, typeName);
+ }
+
+ public String toString()
+ {
+ return String.format("%s (%s, %s)", getClass().getSimpleName(), keyspaceName, typeName);
+ }
+
+ public static final class Raw extends CQLStatement.Raw
+ {
+ private final UTName name;
+ private final boolean ifExists;
+
+ public Raw(UTName name, boolean ifExists)
+ {
+ this.name = name;
+ this.ifExists = ifExists;
+ }
+
+ public DropTypeStatement prepare(ClientState state)
+ {
+ String keyspaceName = name.hasKeyspace() ? name.getKeyspace() : state.getKeyspace();
+ return new DropTypeStatement(keyspaceName, name.getStringTypeName(), ifExists);
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/DropViewStatement.java b/src/java/org/apache/cassandra/cql3/statements/schema/DropViewStatement.java
new file mode 100644
index 0000000..2c73717
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/statements/schema/DropViewStatement.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.statements.schema;
+
+import org.apache.cassandra.audit.AuditLogContext;
+import org.apache.cassandra.audit.AuditLogEntryType;
+import org.apache.cassandra.auth.Permission;
+import org.apache.cassandra.cql3.CQLStatement;
+import org.apache.cassandra.cql3.QualifiedName;
+import org.apache.cassandra.schema.*;
+import org.apache.cassandra.schema.Keyspaces.KeyspacesDiff;
+import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.transport.Event.SchemaChange;
+import org.apache.cassandra.transport.Event.SchemaChange.Change;
+import org.apache.cassandra.transport.Event.SchemaChange.Target;
+
+public final class DropViewStatement extends AlterSchemaStatement
+{
+ private final String viewName;
+ private final boolean ifExists;
+
+ public DropViewStatement(String keyspaceName, String viewName, boolean ifExists)
+ {
+ super(keyspaceName);
+ this.viewName = viewName;
+ this.ifExists = ifExists;
+ }
+
+ public Keyspaces apply(Keyspaces schema)
+ {
+ KeyspaceMetadata keyspace = schema.getNullable(keyspaceName);
+
+ ViewMetadata view = null == keyspace
+ ? null
+ : keyspace.views.getNullable(viewName);
+
+ if (null == view)
+ {
+ if (ifExists)
+ return schema;
+
+ throw ire("Materialized view '%s.%s' doesn't exist", keyspaceName, viewName);
+ }
+
+ return schema.withAddedOrUpdated(keyspace.withSwapped(keyspace.views.without(viewName)));
+ }
+
+ SchemaChange schemaChangeEvent(KeyspacesDiff diff)
+ {
+ return new SchemaChange(Change.DROPPED, Target.TABLE, keyspaceName, viewName);
+ }
+
+ public void authorize(ClientState client)
+ {
+ ViewMetadata view = Schema.instance.getView(keyspaceName, viewName);
+ if (null != view)
+ client.ensureTablePermission(keyspaceName, view.baseTableName, Permission.ALTER);
+ }
+
+ @Override
+ public AuditLogContext getAuditLogContext()
+ {
+ return new AuditLogContext(AuditLogEntryType.DROP_VIEW, keyspaceName, viewName);
+ }
+
+ public String toString()
+ {
+ return String.format("%s (%s, %s)", getClass().getSimpleName(), keyspaceName, viewName);
+ }
+
+ public static final class Raw extends CQLStatement.Raw
+ {
+ private final QualifiedName name;
+ private final boolean ifExists;
+
+ public Raw(QualifiedName name, boolean ifExists)
+ {
+ this.name = name;
+ this.ifExists = ifExists;
+ }
+
+ public DropViewStatement prepare(ClientState state)
+ {
+ String keyspaceName = name.hasKeyspace() ? name.getKeyspace() : state.getKeyspace();
+ return new DropViewStatement(keyspaceName, name.getName(), ifExists);
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/IndexAttributes.java b/src/java/org/apache/cassandra/cql3/statements/schema/IndexAttributes.java
new file mode 100644
index 0000000..f30c502
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/statements/schema/IndexAttributes.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.statements.schema;
+
+import java.util.*;
+
+import org.apache.cassandra.cql3.statements.PropertyDefinitions;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.exceptions.RequestValidationException;
+import org.apache.cassandra.exceptions.SyntaxException;
+
+public class IndexAttributes extends PropertyDefinitions
+{
+ private static final String KW_OPTIONS = "options";
+
+ private static final Set<String> keywords = new HashSet<>();
+ private static final Set<String> obsoleteKeywords = new HashSet<>();
+
+ public boolean isCustom;
+ public String customClass;
+
+ static
+ {
+ keywords.add(KW_OPTIONS);
+ }
+
+ public void validate() throws RequestValidationException
+ {
+ validate(keywords, obsoleteKeywords);
+
+ if (isCustom && customClass == null)
+ throw new InvalidRequestException("CUSTOM index requires specifiying the index class");
+
+ if (!isCustom && customClass != null)
+ throw new InvalidRequestException("Cannot specify index class for a non-CUSTOM index");
+
+ if (!isCustom && !properties.isEmpty())
+ throw new InvalidRequestException("Cannot specify options for a non-CUSTOM index");
+
+ if (getRawOptions().containsKey(IndexTarget.CUSTOM_INDEX_OPTION_NAME))
+ throw new InvalidRequestException(String.format("Cannot specify %s as a CUSTOM option",
+ IndexTarget.CUSTOM_INDEX_OPTION_NAME));
+
+ if (getRawOptions().containsKey(IndexTarget.TARGET_OPTION_NAME))
+ throw new InvalidRequestException(String.format("Cannot specify %s as a CUSTOM option",
+ IndexTarget.TARGET_OPTION_NAME));
+
+ }
+
+ private Map<String, String> getRawOptions() throws SyntaxException
+ {
+ Map<String, String> options = getMap(KW_OPTIONS);
+ return options == null ? Collections.emptyMap() : options;
+ }
+
+ public Map<String, String> getOptions() throws SyntaxException
+ {
+ Map<String, String> options = new HashMap<>(getRawOptions());
+ options.put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, customClass);
+ return options;
+ }
+}
diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/IndexTarget.java b/src/java/org/apache/cassandra/cql3/statements/schema/IndexTarget.java
new file mode 100644
index 0000000..dff933d
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/statements/schema/IndexTarget.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.statements.schema;
+
+import org.apache.cassandra.cql3.ColumnIdentifier;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableMetadata;
+
+public class IndexTarget
+{
+ public static final String TARGET_OPTION_NAME = "target";
+ public static final String CUSTOM_INDEX_OPTION_NAME = "class_name";
+
+ public final ColumnIdentifier column;
+ public final Type type;
+
+ public IndexTarget(ColumnIdentifier column, Type type)
+ {
+ this.column = column;
+ this.type = type;
+ }
+
+ public String asCqlString()
+ {
+ return type == Type.SIMPLE
+ ? column.toCQLString()
+ : String.format("%s(%s)", type.toString(), column.toCQLString());
+ }
+
+ public static class Raw
+ {
+ private final ColumnMetadata.Raw column;
+ private final Type type;
+
+ private Raw(ColumnMetadata.Raw column, Type type)
+ {
+ this.column = column;
+ this.type = type;
+ }
+
+ public static Raw simpleIndexOn(ColumnMetadata.Raw c)
+ {
+ return new Raw(c, Type.SIMPLE);
+ }
+
+ public static Raw valuesOf(ColumnMetadata.Raw c)
+ {
+ return new Raw(c, Type.VALUES);
+ }
+
+ public static Raw keysOf(ColumnMetadata.Raw c)
+ {
+ return new Raw(c, Type.KEYS);
+ }
+
+ public static Raw keysAndValuesOf(ColumnMetadata.Raw c)
+ {
+ return new Raw(c, Type.KEYS_AND_VALUES);
+ }
+
+ public static Raw fullCollection(ColumnMetadata.Raw c)
+ {
+ return new Raw(c, Type.FULL);
+ }
+
+ public IndexTarget prepare(TableMetadata table)
+ {
+ // Until we've prepared the target column, we can't be certain about the target type
+ // because (for backwards compatibility) an index on a collection's values uses the
+ // same syntax as an index on a regular column (i.e. the 'values' in
+ // 'CREATE INDEX on table(values(collection));' is optional). So we correct the target type
+ // when the target column is a collection & the target type is SIMPLE.
+ ColumnMetadata columnDef = column.prepare(table);
+ Type actualType = (type == Type.SIMPLE && columnDef.type.isCollection()) ? Type.VALUES : type;
+ return new IndexTarget(columnDef.name, actualType);
+ }
+ }
+
+ public enum Type
+ {
+ VALUES, KEYS, KEYS_AND_VALUES, FULL, SIMPLE;
+
+ public String toString()
+ {
+ switch (this)
+ {
+ case KEYS: return "keys";
+ case KEYS_AND_VALUES: return "entries";
+ case FULL: return "full";
+ case VALUES: return "values";
+ case SIMPLE: return "";
+ default: return "";
+ }
+ }
+
+ public static Type fromString(String s)
+ {
+ if ("".equals(s))
+ return SIMPLE;
+ else if ("values".equals(s))
+ return VALUES;
+ else if ("keys".equals(s))
+ return KEYS;
+ else if ("entries".equals(s))
+ return KEYS_AND_VALUES;
+ else if ("full".equals(s))
+ return FULL;
+
+ throw new AssertionError("Unrecognized index target type " + s);
+ }
+ }
+
+ @Override
+ public String toString()
+ {
+ return asCqlString();
+ }
+}
diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/KeyspaceAttributes.java b/src/java/org/apache/cassandra/cql3/statements/schema/KeyspaceAttributes.java
new file mode 100644
index 0000000..42fcaf4
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/statements/schema/KeyspaceAttributes.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.statements.schema;
+
+import java.util.*;
+
+import com.google.common.collect.ImmutableSet;
+
+import org.apache.cassandra.cql3.statements.PropertyDefinitions;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.KeyspaceParams.Option;
+import org.apache.cassandra.schema.ReplicationParams;
+
+public final class KeyspaceAttributes extends PropertyDefinitions
+{
+ private static final Set<String> validKeywords;
+ private static final Set<String> obsoleteKeywords;
+
+ static
+ {
+ ImmutableSet.Builder<String> validBuilder = ImmutableSet.builder();
+ for (Option option : Option.values())
+ validBuilder.add(option.toString());
+ validKeywords = validBuilder.build();
+ obsoleteKeywords = ImmutableSet.of();
+ }
+
+ public void validate()
+ {
+ validate(validKeywords, obsoleteKeywords);
+
+ Map<String, String> replicationOptions = getAllReplicationOptions();
+ if (!replicationOptions.isEmpty() && !replicationOptions.containsKey(ReplicationParams.CLASS))
+ throw new ConfigurationException("Missing replication strategy class");
+ }
+
+ private String getReplicationStrategyClass()
+ {
+ return getAllReplicationOptions().get(ReplicationParams.CLASS);
+ }
+
+ private Map<String, String> getAllReplicationOptions()
+ {
+ Map<String, String> replication = getMap(Option.REPLICATION.toString());
+ return replication == null
+ ? Collections.emptyMap()
+ : replication;
+ }
+
+ KeyspaceParams asNewKeyspaceParams()
+ {
+ boolean durableWrites = getBoolean(Option.DURABLE_WRITES.toString(), KeyspaceParams.DEFAULT_DURABLE_WRITES);
+ return KeyspaceParams.create(durableWrites, getAllReplicationOptions());
+ }
+
+ KeyspaceParams asAlteredKeyspaceParams(KeyspaceParams previous)
+ {
+ boolean durableWrites = getBoolean(Option.DURABLE_WRITES.toString(), previous.durableWrites);
+ Map<String, String> previousOptions = previous.replication.options;
+ ReplicationParams replication = getReplicationStrategyClass() == null
+ ? previous.replication
+ : ReplicationParams.fromMapWithDefaults(getAllReplicationOptions(), previousOptions);
+ return new KeyspaceParams(durableWrites, replication);
+ }
+
+ public boolean hasOption(Option option)
+ {
+ return hasProperty(option.toString());
+ }
+}
diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/TableAttributes.java b/src/java/org/apache/cassandra/cql3/statements/schema/TableAttributes.java
new file mode 100644
index 0000000..126e6d7
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/statements/schema/TableAttributes.java
@@ -0,0 +1,207 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.statements.schema;
+
+import java.util.Map;
+import java.util.Set;
+
+import com.google.common.collect.ImmutableSet;
+
+import org.apache.cassandra.cql3.statements.PropertyDefinitions;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.exceptions.SyntaxException;
+import org.apache.cassandra.schema.CachingParams;
+import org.apache.cassandra.schema.CompactionParams;
+import org.apache.cassandra.schema.CompressionParams;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.schema.TableParams;
+import org.apache.cassandra.schema.TableParams.Option;
+import org.apache.cassandra.service.reads.SpeculativeRetryPolicy;
+import org.apache.cassandra.service.reads.repair.ReadRepairStrategy;
+
+import static java.lang.String.format;
+
+public final class TableAttributes extends PropertyDefinitions
+{
+ public static final String ID = "id";
+ private static final Set<String> validKeywords;
+ private static final Set<String> obsoleteKeywords;
+
+ static
+ {
+ ImmutableSet.Builder<String> validBuilder = ImmutableSet.builder();
+ for (Option option : Option.values())
+ validBuilder.add(option.toString());
+ validBuilder.add(ID);
+ validKeywords = validBuilder.build();
+ obsoleteKeywords = ImmutableSet.of();
+ }
+
+ public void validate()
+ {
+ validate(validKeywords, obsoleteKeywords);
+ build(TableParams.builder()).validate();
+ }
+
+ TableParams asNewTableParams()
+ {
+ return build(TableParams.builder());
+ }
+
+ TableParams asAlteredTableParams(TableParams previous)
+ {
+ if (getId() != null)
+ throw new ConfigurationException("Cannot alter table id.");
+ return build(previous.unbuild());
+ }
+
+ public TableId getId() throws ConfigurationException
+ {
+ String id = getSimple(ID);
+ try
+ {
+ return id != null ? TableId.fromString(id) : null;
+ }
+ catch (IllegalArgumentException e)
+ {
+ throw new ConfigurationException("Invalid table id", e);
+ }
+ }
+
+ private TableParams build(TableParams.Builder builder)
+ {
+ if (hasOption(Option.BLOOM_FILTER_FP_CHANCE))
+ builder.bloomFilterFpChance(getDouble(Option.BLOOM_FILTER_FP_CHANCE));
+
+ if (hasOption(Option.CACHING))
+ builder.caching(CachingParams.fromMap(getMap(Option.CACHING)));
+
+ if (hasOption(Option.COMMENT))
+ builder.comment(getString(Option.COMMENT));
+
+ if (hasOption(Option.COMPACTION))
+ builder.compaction(CompactionParams.fromMap(getMap(Option.COMPACTION)));
+
+ if (hasOption(Option.COMPRESSION))
+ {
+ //crc_check_chance was "promoted" from a compression property to a top-level-property after #9839
+ //so we temporarily accept it to be defined as a compression option, to maintain backwards compatibility
+ Map<String, String> compressionOpts = getMap(Option.COMPRESSION);
+ if (compressionOpts.containsKey(Option.CRC_CHECK_CHANCE.toString().toLowerCase()))
+ {
+ Double crcCheckChance = getDeprecatedCrcCheckChance(compressionOpts);
+ builder.crcCheckChance(crcCheckChance);
+ }
+ builder.compression(CompressionParams.fromMap(getMap(Option.COMPRESSION)));
+ }
+
+ if (hasOption(Option.DEFAULT_TIME_TO_LIVE))
+ builder.defaultTimeToLive(getInt(Option.DEFAULT_TIME_TO_LIVE));
+
+ if (hasOption(Option.GC_GRACE_SECONDS))
+ builder.gcGraceSeconds(getInt(Option.GC_GRACE_SECONDS));
+
+ if (hasOption(Option.MAX_INDEX_INTERVAL))
+ builder.maxIndexInterval(getInt(Option.MAX_INDEX_INTERVAL));
+
+ if (hasOption(Option.MEMTABLE_FLUSH_PERIOD_IN_MS))
+ builder.memtableFlushPeriodInMs(getInt(Option.MEMTABLE_FLUSH_PERIOD_IN_MS));
+
+ if (hasOption(Option.MIN_INDEX_INTERVAL))
+ builder.minIndexInterval(getInt(Option.MIN_INDEX_INTERVAL));
+
+ if (hasOption(Option.SPECULATIVE_RETRY))
+ builder.speculativeRetry(SpeculativeRetryPolicy.fromString(getString(Option.SPECULATIVE_RETRY)));
+
+ if (hasOption(Option.ADDITIONAL_WRITE_POLICY))
+ builder.additionalWritePolicy(SpeculativeRetryPolicy.fromString(getString(Option.ADDITIONAL_WRITE_POLICY)));
+
+ if (hasOption(Option.CRC_CHECK_CHANCE))
+ builder.crcCheckChance(getDouble(Option.CRC_CHECK_CHANCE));
+
+ if (hasOption(Option.CDC))
+ builder.cdc(getBoolean(Option.CDC.toString(), false));
+
+ if (hasOption(Option.READ_REPAIR))
+ builder.readRepair(ReadRepairStrategy.fromString(getString(Option.READ_REPAIR)));
+
+ return builder.build();
+ }
+
+ private Double getDeprecatedCrcCheckChance(Map<String, String> compressionOpts)
+ {
+ String value = compressionOpts.get(Option.CRC_CHECK_CHANCE.toString().toLowerCase());
+ try
+ {
+ return Double.valueOf(value);
+ }
+ catch (NumberFormatException e)
+ {
+ throw new SyntaxException(String.format("Invalid double value %s for crc_check_chance.'", value));
+ }
+ }
+
+ private double getDouble(Option option)
+ {
+ String value = getString(option);
+
+ try
+ {
+ return Double.parseDouble(value);
+ }
+ catch (NumberFormatException e)
+ {
+ throw new SyntaxException(format("Invalid double value %s for '%s'", value, option));
+ }
+ }
+
+ private int getInt(Option option)
+ {
+ String value = getString(option);
+
+ try
+ {
+ return Integer.parseInt(value);
+ }
+ catch (NumberFormatException e)
+ {
+ throw new SyntaxException(String.format("Invalid integer value %s for '%s'", value, option));
+ }
+ }
+
+ private String getString(Option option)
+ {
+ String value = getSimple(option.toString());
+ if (value == null)
+ throw new IllegalStateException(format("Option '%s' is absent", option));
+ return value;
+ }
+
+ private Map<String, String> getMap(Option option)
+ {
+ Map<String, String> value = getMap(option.toString());
+ if (value == null)
+ throw new IllegalStateException(format("Option '%s' is absent", option));
+ return value;
+ }
+
+ public boolean hasOption(Option option)
+ {
+ return hasProperty(option.toString());
+ }
+}
diff --git a/src/java/org/apache/cassandra/db/AbstractBufferClusteringPrefix.java b/src/java/org/apache/cassandra/db/AbstractBufferClusteringPrefix.java
index 95bc777..dd4a095 100644
--- a/src/java/org/apache/cassandra/db/AbstractBufferClusteringPrefix.java
+++ b/src/java/org/apache/cassandra/db/AbstractBufferClusteringPrefix.java
@@ -24,7 +24,6 @@
public abstract class AbstractBufferClusteringPrefix extends AbstractClusteringPrefix
{
public static final ByteBuffer[] EMPTY_VALUES_ARRAY = new ByteBuffer[0];
- private static final long EMPTY_SIZE = ObjectSizes.measure(Clustering.make(EMPTY_VALUES_ARRAY));
protected final Kind kind;
protected final ByteBuffer[] values;
@@ -62,11 +61,11 @@
public long unsharedHeapSize()
{
- return EMPTY_SIZE + ObjectSizes.sizeOnHeapOf(values);
+ return Clustering.EMPTY_SIZE + ObjectSizes.sizeOnHeapOf(values);
}
public long unsharedHeapSizeExcludingData()
{
- return EMPTY_SIZE + ObjectSizes.sizeOnHeapExcludingData(values);
+ return Clustering.EMPTY_SIZE + ObjectSizes.sizeOnHeapExcludingData(values);
}
}
diff --git a/src/java/org/apache/cassandra/db/AbstractClusteringPrefix.java b/src/java/org/apache/cassandra/db/AbstractClusteringPrefix.java
index 0b1daf7..8714936 100644
--- a/src/java/org/apache/cassandra/db/AbstractClusteringPrefix.java
+++ b/src/java/org/apache/cassandra/db/AbstractClusteringPrefix.java
@@ -18,11 +18,8 @@
package org.apache.cassandra.db;
import java.nio.ByteBuffer;
-import java.security.MessageDigest;
import java.util.Objects;
-import org.apache.cassandra.utils.FBUtilities;
-
public abstract class AbstractClusteringPrefix implements ClusteringPrefix
{
public ClusteringPrefix clustering()
@@ -41,15 +38,15 @@
return size;
}
- public void digest(MessageDigest digest)
+ public void digest(Digest digest)
{
for (int i = 0; i < size(); i++)
{
ByteBuffer bb = get(i);
if (bb != null)
- digest.update(bb.duplicate());
+ digest.update(bb);
}
- FBUtilities.updateWithByte(digest, kind().ordinal());
+ digest.updateWithByte(kind().ordinal());
}
@Override
diff --git a/src/java/org/apache/cassandra/db/AbstractCompactionController.java b/src/java/org/apache/cassandra/db/AbstractCompactionController.java
new file mode 100644
index 0000000..99193f8
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/AbstractCompactionController.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db;
+
+import java.util.function.LongPredicate;
+
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.schema.CompactionParams;
+
+/**
+ * AbstractCompactionController allows custom implementations of the CompactionController for use in tooling, without being tied to the SSTableReader and local filesystem
+ */
+public abstract class AbstractCompactionController implements AutoCloseable
+{
+ public final ColumnFamilyStore cfs;
+ public final int gcBefore;
+ public final CompactionParams.TombstoneOption tombstoneOption;
+
+ public AbstractCompactionController(final ColumnFamilyStore cfs, final int gcBefore, CompactionParams.TombstoneOption tombstoneOption)
+ {
+ assert cfs != null;
+ this.cfs = cfs;
+ this.gcBefore = gcBefore;
+ this.tombstoneOption = tombstoneOption;
+ }
+
+ public abstract boolean compactingRepaired();
+
+ public String getKeyspace()
+ {
+ return cfs.keyspace.getName();
+ }
+
+ public String getColumnFamily()
+ {
+ return cfs.name;
+ }
+
+ public Iterable<UnfilteredRowIterator> shadowSources(DecoratedKey key, boolean tombstoneOnly)
+ {
+ return null;
+ }
+
+ public abstract LongPredicate getPurgeEvaluator(DecoratedKey key);
+}
diff --git a/src/java/org/apache/cassandra/db/AbstractReadCommandBuilder.java b/src/java/org/apache/cassandra/db/AbstractReadCommandBuilder.java
index ba91db8..4df1bd3 100644
--- a/src/java/org/apache/cassandra/db/AbstractReadCommandBuilder.java
+++ b/src/java/org/apache/cassandra/db/AbstractReadCommandBuilder.java
@@ -21,10 +21,10 @@
import java.nio.ByteBuffer;
import java.util.*;
-import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Sets;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.cql3.ColumnIdentifier;
import org.apache.cassandra.cql3.Operator;
import org.apache.cassandra.db.filter.*;
@@ -66,28 +66,28 @@
public AbstractReadCommandBuilder fromIncl(Object... values)
{
assert lowerClusteringBound == null && clusterings == null;
- this.lowerClusteringBound = ClusteringBound.create(cfs.metadata.comparator, true, true, values);
+ this.lowerClusteringBound = ClusteringBound.create(cfs.metadata().comparator, true, true, values);
return this;
}
public AbstractReadCommandBuilder fromExcl(Object... values)
{
assert lowerClusteringBound == null && clusterings == null;
- this.lowerClusteringBound = ClusteringBound.create(cfs.metadata.comparator, true, false, values);
+ this.lowerClusteringBound = ClusteringBound.create(cfs.metadata().comparator, true, false, values);
return this;
}
public AbstractReadCommandBuilder toIncl(Object... values)
{
assert upperClusteringBound == null && clusterings == null;
- this.upperClusteringBound = ClusteringBound.create(cfs.metadata.comparator, false, true, values);
+ this.upperClusteringBound = ClusteringBound.create(cfs.metadata().comparator, false, true, values);
return this;
}
public AbstractReadCommandBuilder toExcl(Object... values)
{
assert upperClusteringBound == null && clusterings == null;
- this.upperClusteringBound = ClusteringBound.create(cfs.metadata.comparator, false, false, values);
+ this.upperClusteringBound = ClusteringBound.create(cfs.metadata().comparator, false, false, values);
return this;
}
@@ -96,9 +96,9 @@
assert lowerClusteringBound == null && upperClusteringBound == null;
if (this.clusterings == null)
- this.clusterings = new TreeSet<>(cfs.metadata.comparator);
+ this.clusterings = new TreeSet<>(cfs.metadata().comparator);
- this.clusterings.add(cfs.metadata.comparator.make(values));
+ this.clusterings.add(cfs.metadata().comparator.make(values));
return this;
}
@@ -123,7 +123,7 @@
public AbstractReadCommandBuilder columns(String... columns)
{
if (this.columns == null)
- this.columns = new HashSet<>();
+ this.columns = Sets.newHashSetWithExpectedSize(columns.length);
for (String column : columns)
this.columns.add(ColumnIdentifier.getInterned(column, true));
@@ -163,10 +163,9 @@
throw new AssertionError();
}
- @VisibleForTesting
public AbstractReadCommandBuilder filterOn(String column, Operator op, Object value)
{
- ColumnDefinition def = cfs.metadata.getColumnDefinitionForCQL(ColumnIdentifier.getInterned(column, true));
+ ColumnMetadata def = cfs.metadata().getColumn(ColumnIdentifier.getInterned(column, true));
assert def != null;
AbstractType<?> type = def.type;
@@ -182,11 +181,11 @@
protected ColumnFilter makeColumnFilter()
{
if (columns == null || columns.isEmpty())
- return ColumnFilter.all(cfs.metadata);
+ return ColumnFilter.all(cfs.metadata());
ColumnFilter.Builder filter = ColumnFilter.selectionBuilder();
for (ColumnIdentifier column : columns)
- filter.add(cfs.metadata.getColumnDefinition(column));
+ filter.add(cfs.metadata().getColumn(column));
return filter.build();
}
@@ -196,8 +195,8 @@
// SelectStatement.makeClusteringIndexFilter uses a names filter with no clusterings for static
// compact tables, here we reproduce this behavior (CASSANDRA-11223). Note that this code is only
// called by tests.
- if (cfs.metadata.isStaticCompactTable())
- return new ClusteringIndexNamesFilter(new TreeSet<>(cfs.metadata.comparator), reversed);
+ if (cfs.metadata().isStaticCompactTable())
+ return new ClusteringIndexNamesFilter(new TreeSet<>(cfs.metadata().comparator), reversed);
if (clusterings != null)
{
@@ -207,7 +206,7 @@
{
Slice slice = Slice.make(lowerClusteringBound == null ? ClusteringBound.BOTTOM : lowerClusteringBound,
upperClusteringBound == null ? ClusteringBound.TOP : upperClusteringBound);
- return new ClusteringIndexSliceFilter(Slices.with(cfs.metadata.comparator, slice), reversed);
+ return new ClusteringIndexSliceFilter(Slices.with(cfs.metadata().comparator, slice), reversed);
}
}
@@ -234,7 +233,7 @@
@Override
public ReadCommand build()
{
- return SinglePartitionReadCommand.create(cfs.metadata, nowInSeconds, makeColumnFilter(), filter, makeLimits(), partitionKey, makeFilter());
+ return SinglePartitionReadCommand.create(cfs.metadata(), nowInSeconds, makeColumnFilter(), filter, makeLimits(), partitionKey, makeFilter());
}
}
@@ -265,7 +264,7 @@
@Override
public ReadCommand build()
{
- return SinglePartitionReadCommand.create(cfs.metadata, nowInSeconds, makeColumnFilter(), filter, makeLimits(), partitionKey, makeFilter());
+ return SinglePartitionReadCommand.create(cfs.metadata(), nowInSeconds, makeColumnFilter(), filter, makeLimits(), partitionKey, makeFilter());
}
}
@@ -285,7 +284,7 @@
{
assert startKey == null;
this.startInclusive = true;
- this.startKey = makeKey(cfs.metadata, values);
+ this.startKey = makeKey(cfs.metadata(), values);
return this;
}
@@ -293,7 +292,7 @@
{
assert startKey == null;
this.startInclusive = false;
- this.startKey = makeKey(cfs.metadata, values);
+ this.startKey = makeKey(cfs.metadata(), values);
return this;
}
@@ -301,7 +300,7 @@
{
assert endKey == null;
this.endInclusive = true;
- this.endKey = makeKey(cfs.metadata, values);
+ this.endKey = makeKey(cfs.metadata(), values);
return this;
}
@@ -309,7 +308,7 @@
{
assert endKey == null;
this.endInclusive = false;
- this.endKey = makeKey(cfs.metadata, values);
+ this.endKey = makeKey(cfs.metadata(), values);
return this;
}
@@ -339,16 +338,16 @@
else
bounds = new ExcludingBounds<>(start, end);
- return PartitionRangeReadCommand.create(false, cfs.metadata, nowInSeconds, makeColumnFilter(), filter, makeLimits(), new DataRange(bounds, makeFilter()));
+ return PartitionRangeReadCommand.create(cfs.metadata(), nowInSeconds, makeColumnFilter(), filter, makeLimits(), new DataRange(bounds, makeFilter()));
}
- static DecoratedKey makeKey(CFMetaData metadata, Object... partitionKey)
+ static DecoratedKey makeKey(TableMetadata metadata, Object... partitionKey)
{
if (partitionKey.length == 1 && partitionKey[0] instanceof DecoratedKey)
return (DecoratedKey)partitionKey[0];
- ByteBuffer key = CFMetaData.serializePartitionKey(metadata.getKeyValidatorAsClusteringComparator().make(partitionKey));
- return metadata.decorateKey(key);
+ ByteBuffer key = metadata.partitionKeyAsClusteringComparator().make(partitionKey).serializeAsPartitionKey();
+ return metadata.partitioner.decorateKey(key);
}
}
}
diff --git a/src/java/org/apache/cassandra/db/AbstractReadQuery.java b/src/java/org/apache/cassandra/db/AbstractReadQuery.java
new file mode 100644
index 0000000..c6ec329
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/AbstractReadQuery.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db;
+
+import org.apache.cassandra.db.filter.ColumnFilter;
+import org.apache.cassandra.db.filter.DataLimits;
+import org.apache.cassandra.db.filter.RowFilter;
+import org.apache.cassandra.db.monitoring.MonitorableImpl;
+import org.apache.cassandra.db.partitions.PartitionIterator;
+import org.apache.cassandra.db.partitions.UnfilteredPartitionIterators;
+import org.apache.cassandra.schema.TableMetadata;
+
+/**
+ * Base class for {@code ReadQuery} implementations.
+ */
+abstract class AbstractReadQuery extends MonitorableImpl implements ReadQuery
+{
+ private final TableMetadata metadata;
+ private final int nowInSec;
+
+ private final ColumnFilter columnFilter;
+ private final RowFilter rowFilter;
+ private final DataLimits limits;
+
+ protected AbstractReadQuery(TableMetadata metadata, int nowInSec, ColumnFilter columnFilter, RowFilter rowFilter, DataLimits limits)
+ {
+ this.metadata = metadata;
+ this.nowInSec = nowInSec;
+ this.columnFilter = columnFilter;
+ this.rowFilter = rowFilter;
+ this.limits = limits;
+ }
+
+ @Override
+ public TableMetadata metadata()
+ {
+ return metadata;
+ }
+
+ // Monitorable interface
+ public String name()
+ {
+ return toCQLString();
+ }
+
+ @Override
+ public PartitionIterator executeInternal(ReadExecutionController controller)
+ {
+ return UnfilteredPartitionIterators.filter(executeLocally(controller), nowInSec());
+ }
+
+ @Override
+ public DataLimits limits()
+ {
+ return limits;
+ }
+
+ @Override
+ public int nowInSec()
+ {
+ return nowInSec;
+ }
+
+ @Override
+ public RowFilter rowFilter()
+ {
+ return rowFilter;
+ }
+
+ @Override
+ public ColumnFilter columnFilter()
+ {
+ return columnFilter;
+ }
+
+ /**
+ * Recreate the CQL string corresponding to this query.
+ * <p>
+ * Note that in general the returned string will not be exactly the original user string, first
+ * because there isn't always a single syntax for a given query, but also because we don't have
+ * all the information needed (we know the non-PK columns queried but not the PK ones as internally
+ * we query them all). So this shouldn't be relied too strongly, but this should be good enough for
+ * debugging purpose which is what this is for.
+ */
+ public String toCQLString()
+ {
+ StringBuilder sb = new StringBuilder().append("SELECT ")
+ .append(columnFilter())
+ .append(" FROM ")
+ .append(metadata().keyspace)
+ .append('.')
+ .append(metadata().name);
+ appendCQLWhereClause(sb);
+
+ if (limits() != DataLimits.NONE)
+ sb.append(' ').append(limits());
+ return sb.toString();
+ }
+
+ protected abstract void appendCQLWhereClause(StringBuilder sb);
+}
diff --git a/src/java/org/apache/cassandra/db/BufferClustering.java b/src/java/org/apache/cassandra/db/BufferClustering.java
index 7ca9132..fc635ab 100644
--- a/src/java/org/apache/cassandra/db/BufferClustering.java
+++ b/src/java/org/apache/cassandra/db/BufferClustering.java
@@ -29,9 +29,8 @@
* prefix used by rows.
* <p>
* Note however that while it's size must be equal to the table clustering size, a clustering can have
- * {@code null} values, and this mostly for thrift backward compatibility (in practice, if a value is null,
- * all of the following ones will be too because that's what thrift allows, but it's never assumed by the
- * code so we could start generally allowing nulls for clustering columns if we wanted to).
+ * {@code null} values (this is currently only allowed in COMPACT table for historical reasons, but we
+ * could imagine lifting that limitation if we decide it make sense from a CQL point of view).
*/
public class BufferClustering extends AbstractBufferClusteringPrefix implements Clustering
{
@@ -44,5 +43,6 @@
{
if (!ByteBufferUtil.canMinimize(values))
return this;
- return new BufferClustering(ByteBufferUtil.minimizeBuffers(values)); }
+ return new BufferClustering(ByteBufferUtil.minimizeBuffers(values));
+ }
}
diff --git a/src/java/org/apache/cassandra/db/CassandraKeyspaceWriteHandler.java b/src/java/org/apache/cassandra/db/CassandraKeyspaceWriteHandler.java
new file mode 100644
index 0000000..efba11f
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/CassandraKeyspaceWriteHandler.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db;
+
+import org.apache.cassandra.db.commitlog.CommitLog;
+import org.apache.cassandra.db.commitlog.CommitLogPosition;
+import org.apache.cassandra.exceptions.RequestExecutionException;
+import org.apache.cassandra.tracing.Tracing;
+import org.apache.cassandra.utils.concurrent.OpOrder;
+
+public class CassandraKeyspaceWriteHandler implements KeyspaceWriteHandler
+{
+ private final Keyspace keyspace;
+
+ public CassandraKeyspaceWriteHandler(Keyspace keyspace)
+ {
+ this.keyspace = keyspace;
+ }
+
+ @Override
+ @SuppressWarnings("resource") // group is closed when CassandraWriteContext is closed
+ public WriteContext beginWrite(Mutation mutation, boolean makeDurable) throws RequestExecutionException
+ {
+ OpOrder.Group group = null;
+ try
+ {
+ group = Keyspace.writeOrder.start();
+
+ // write the mutation to the commitlog and memtables
+ CommitLogPosition position = null;
+ if (makeDurable)
+ {
+ Tracing.trace("Appending to commitlog");
+ position = CommitLog.instance.add(mutation);
+ }
+ return new CassandraWriteContext(group, position);
+ }
+ catch (Throwable t)
+ {
+ if (group != null)
+ {
+ group.close();
+ }
+ throw t;
+ }
+ }
+
+ @SuppressWarnings("resource") // group is closed when CassandraWriteContext is closed
+ private WriteContext createEmptyContext()
+ {
+ OpOrder.Group group = null;
+ try
+ {
+ group = Keyspace.writeOrder.start();
+ return new CassandraWriteContext(group, null);
+ }
+ catch (Throwable t)
+ {
+ if (group != null)
+ {
+ group.close();
+ }
+ throw t;
+ }
+ }
+
+ @Override
+ public WriteContext createContextForIndexing()
+ {
+ return createEmptyContext();
+ }
+
+ @Override
+ public WriteContext createContextForRead()
+ {
+ return createEmptyContext();
+ }
+}
diff --git a/src/java/org/apache/cassandra/db/CassandraTableWriteHandler.java b/src/java/org/apache/cassandra/db/CassandraTableWriteHandler.java
new file mode 100644
index 0000000..146539c
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/CassandraTableWriteHandler.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db;
+
+import org.apache.cassandra.db.partitions.PartitionUpdate;
+import org.apache.cassandra.index.transactions.UpdateTransaction;
+import org.apache.cassandra.tracing.Tracing;
+
+public class CassandraTableWriteHandler implements TableWriteHandler
+{
+ private final ColumnFamilyStore cfs;
+
+ public CassandraTableWriteHandler(ColumnFamilyStore cfs)
+ {
+ this.cfs = cfs;
+ }
+
+ @Override
+ @SuppressWarnings("resource")
+ public void write(PartitionUpdate update, WriteContext context, UpdateTransaction updateTransaction)
+ {
+ CassandraWriteContext ctx = CassandraWriteContext.fromContext(context);
+ Tracing.trace("Adding to {} memtable", update.metadata().name);
+ cfs.apply(update, updateTransaction, ctx.getGroup(), ctx.getPosition());
+ }
+}
diff --git a/src/java/org/apache/cassandra/db/CassandraWriteContext.java b/src/java/org/apache/cassandra/db/CassandraWriteContext.java
new file mode 100644
index 0000000..bac1351
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/CassandraWriteContext.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db;
+
+import com.google.common.base.Preconditions;
+
+import org.apache.cassandra.db.commitlog.CommitLogPosition;
+import org.apache.cassandra.utils.concurrent.OpOrder;
+
+public class CassandraWriteContext implements WriteContext
+{
+ private final OpOrder.Group opGroup;
+ private final CommitLogPosition position;
+
+ public CassandraWriteContext(OpOrder.Group opGroup, CommitLogPosition position)
+ {
+ Preconditions.checkArgument(opGroup != null);
+ this.opGroup = opGroup;
+ this.position = position;
+ }
+
+ public static CassandraWriteContext fromContext(WriteContext context)
+ {
+ Preconditions.checkArgument(context instanceof CassandraWriteContext);
+ return (CassandraWriteContext) context;
+ }
+
+ public OpOrder.Group getGroup()
+ {
+ return opGroup;
+ }
+
+ public CommitLogPosition getPosition()
+ {
+ return position;
+ }
+
+ @Override
+ public void close()
+ {
+ opGroup.close();
+ }
+}
diff --git a/src/java/org/apache/cassandra/db/Clustering.java b/src/java/org/apache/cassandra/db/Clustering.java
index fa38ce1..451a087 100644
--- a/src/java/org/apache/cassandra/db/Clustering.java
+++ b/src/java/org/apache/cassandra/db/Clustering.java
@@ -22,19 +22,22 @@
import java.nio.ByteBuffer;
import java.util.List;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.io.util.DataInputBuffer;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputBuffer;
import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.utils.ObjectSizes;
import org.apache.cassandra.utils.memory.AbstractAllocator;
import static org.apache.cassandra.db.AbstractBufferClusteringPrefix.EMPTY_VALUES_ARRAY;
public interface Clustering extends ClusteringPrefix
{
+ static final long EMPTY_SIZE = ObjectSizes.measure(new BufferClustering(EMPTY_VALUES_ARRAY));
+
public static final Serializer serializer = new Serializer();
public long unsharedHeapSizeExcludingData();
@@ -43,7 +46,7 @@
{
// Important for STATIC_CLUSTERING (but must copy empty native clustering types).
if (size() == 0)
- return kind() == Kind.STATIC_CLUSTERING ? this : new BufferClustering(EMPTY_VALUES_ARRAY);
+ return kind() == Kind.STATIC_CLUSTERING ? this : EMPTY;
ByteBuffer[] newValues = new ByteBuffer[size()];
for (int i = 0; i < size(); i++)
@@ -54,23 +57,23 @@
return new BufferClustering(newValues);
}
- public default String toString(CFMetaData metadata)
+ public default String toString(TableMetadata metadata)
{
StringBuilder sb = new StringBuilder();
for (int i = 0; i < size(); i++)
{
- ColumnDefinition c = metadata.clusteringColumns().get(i);
+ ColumnMetadata c = metadata.clusteringColumns().get(i);
sb.append(i == 0 ? "" : ", ").append(c.name).append('=').append(get(i) == null ? "null" : c.type.getString(get(i)));
}
return sb.toString();
}
- public default String toCQLString(CFMetaData metadata)
+ public default String toCQLString(TableMetadata metadata)
{
StringBuilder sb = new StringBuilder();
for (int i = 0; i < size(); i++)
{
- ColumnDefinition c = metadata.clusteringColumns().get(i);
+ ColumnMetadata c = metadata.clusteringColumns().get(i);
sb.append(i == 0 ? "" : ", ").append(c.type.getString(get(i)));
}
return sb.toString();
@@ -78,7 +81,7 @@
public static Clustering make(ByteBuffer... values)
{
- return new BufferClustering(values);
+ return values.length == 0 ? EMPTY : new BufferClustering(values);
}
/**
@@ -100,7 +103,7 @@
}
@Override
- public String toString(CFMetaData metadata)
+ public String toString(TableMetadata metadata)
{
return toString();
}
@@ -110,7 +113,7 @@
public static final Clustering EMPTY = new BufferClustering(EMPTY_VALUES_ARRAY)
{
@Override
- public String toString(CFMetaData metadata)
+ public String toString(TableMetadata metadata)
{
return "EMPTY";
}
@@ -140,7 +143,7 @@
}
catch (IOException e)
{
- throw new RuntimeException("Writting to an in-memory buffer shouldn't trigger an IOException", e);
+ throw new RuntimeException("Writing to an in-memory buffer shouldn't trigger an IOException", e);
}
}
diff --git a/src/java/org/apache/cassandra/db/ClusteringBound.java b/src/java/org/apache/cassandra/db/ClusteringBound.java
index 8bfeb32..6ae0816 100644
--- a/src/java/org/apache/cassandra/db/ClusteringBound.java
+++ b/src/java/org/apache/cassandra/db/ClusteringBound.java
@@ -41,6 +41,13 @@
super(kind, values);
}
+ public ClusteringPrefix minimize()
+ {
+ if (!ByteBufferUtil.canMinimize(values))
+ return this;
+ return new ClusteringBound(kind, ByteBufferUtil.minimizeBuffers(values));
+ }
+
public static ClusteringBound create(Kind kind, ByteBuffer[] values)
{
assert !kind.isBoundary();
@@ -122,13 +129,6 @@
return (ClusteringBound) super.copy(allocator);
}
- public ClusteringPrefix minimize()
- {
- if (!ByteBufferUtil.canMinimize(values))
- return this;
- return new ClusteringBound(kind, ByteBufferUtil.minimizeBuffers(values));
- }
-
public boolean isStart()
{
return kind().isStart();
diff --git a/src/java/org/apache/cassandra/db/ClusteringBoundOrBoundary.java b/src/java/org/apache/cassandra/db/ClusteringBoundOrBoundary.java
index 7a2cce1..84a9e30 100644
--- a/src/java/org/apache/cassandra/db/ClusteringBoundOrBoundary.java
+++ b/src/java/org/apache/cassandra/db/ClusteringBoundOrBoundary.java
@@ -24,7 +24,7 @@
import java.nio.ByteBuffer;
import java.util.List;
-import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
@@ -111,7 +111,7 @@
return create(kind(), newValues);
}
- public String toString(CFMetaData metadata)
+ public String toString(TableMetadata metadata)
{
return toString(metadata.comparator);
}
diff --git a/src/java/org/apache/cassandra/db/ClusteringBoundary.java b/src/java/org/apache/cassandra/db/ClusteringBoundary.java
index 7a6842b..b1dcdb6 100644
--- a/src/java/org/apache/cassandra/db/ClusteringBoundary.java
+++ b/src/java/org/apache/cassandra/db/ClusteringBoundary.java
@@ -36,6 +36,13 @@
super(kind, values);
}
+ public ClusteringPrefix minimize()
+ {
+ if (!ByteBufferUtil.canMinimize(values))
+ return this;
+ return new ClusteringBoundary(kind, ByteBufferUtil.minimizeBuffers(values));
+ }
+
public static ClusteringBoundary create(Kind kind, ByteBuffer[] values)
{
assert kind.isBoundary();
@@ -54,13 +61,6 @@
return (ClusteringBoundary) super.copy(allocator);
}
- public ClusteringPrefix minimize()
- {
- if (!ByteBufferUtil.canMinimize(values))
- return this;
- return new ClusteringBoundary(kind, ByteBufferUtil.minimizeBuffers(values));
- }
-
public ClusteringBound openBound(boolean reversed)
{
return ClusteringBound.create(kind.openBoundOfBoundary(reversed), values);
diff --git a/src/java/org/apache/cassandra/db/ClusteringComparator.java b/src/java/org/apache/cassandra/db/ClusteringComparator.java
index 4374a46..50cf5bf 100644
--- a/src/java/org/apache/cassandra/db/ClusteringComparator.java
+++ b/src/java/org/apache/cassandra/db/ClusteringComparator.java
@@ -53,7 +53,7 @@
this(ImmutableList.copyOf(clusteringTypes));
}
- public ClusteringComparator(List<AbstractType<?>> clusteringTypes)
+ public ClusteringComparator(Iterable<AbstractType<?>> clusteringTypes)
{
// copy the list to ensure despatch is monomorphic
this.clusteringTypes = ImmutableList.copyOf(clusteringTypes);
@@ -62,7 +62,7 @@
this.indexReverseComparator = (o1, o2) -> ClusteringComparator.this.compare(o1.firstName, o2.firstName);
this.reverseComparator = (c1, c2) -> ClusteringComparator.this.compare(c2, c1);
for (AbstractType<?> type : clusteringTypes)
- type.checkComparable(); // this should already be enforced by CFMetaData.rebuild, but we check again for other constructors
+ type.checkComparable(); // this should already be enforced by TableMetadata.Builder.addColumn, but we check again for other constructors
}
/**
diff --git a/src/java/org/apache/cassandra/db/ClusteringPrefix.java b/src/java/org/apache/cassandra/db/ClusteringPrefix.java
index 1f93ca8..357d746 100644
--- a/src/java/org/apache/cassandra/db/ClusteringPrefix.java
+++ b/src/java/org/apache/cassandra/db/ClusteringPrefix.java
@@ -19,15 +19,16 @@
import java.io.IOException;
import java.nio.ByteBuffer;
-import java.security.MessageDigest;
import java.util.*;
import org.apache.cassandra.cache.IMeasurableMemory;
import org.apache.cassandra.config.*;
+import org.apache.cassandra.db.marshal.CompositeType;
import org.apache.cassandra.db.rows.*;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.utils.ByteBufferUtil;
/**
@@ -215,11 +216,11 @@
public ByteBuffer get(int i);
/**
- * Adds the data of this clustering prefix to the provided digest.
+ * Adds the data of this clustering prefix to the provided Digest instance.
*
- * @param digest the digest to which to add this prefix.
+ * @param digest the Digest instance to which to add this prefix.
*/
- public void digest(MessageDigest digest);
+ public void digest(Digest digest);
/**
* The size of the data hold by this prefix.
@@ -235,8 +236,24 @@
* @param metadata the metadata for the table the clustering prefix is of.
* @return a human-readable string representation fo this prefix.
*/
- public String toString(CFMetaData metadata);
+ public String toString(TableMetadata metadata);
+ /*
+ * TODO: we should stop using Clustering for partition keys. Maybe we can add
+ * a few methods to DecoratedKey so we don't have to (note that while using a Clustering
+ * allows to use buildBound(), it's actually used for partition keys only when every restriction
+ * is an equal, so we could easily create a specific method for keys for that.
+ */
+ default ByteBuffer serializeAsPartitionKey()
+ {
+ if (size() == 1)
+ return get(0);
+
+ ByteBuffer[] values = new ByteBuffer[size()];
+ for (int i = 0; i < size(); i++)
+ values[i] = get(i);
+ return CompositeType.build(values);
+ }
/**
* The values of this prefix as an array.
* <p>
diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
index 99cac7c..f7411b7 100644
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
@@ -20,6 +20,8 @@
import java.io.File;
import java.io.IOException;
import java.io.PrintStream;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
import java.nio.ByteBuffer;
import java.nio.file.Files;
import java.util.*;
@@ -38,7 +40,6 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.clearspring.analytics.stream.Counter;
import org.apache.cassandra.cache.*;
import org.apache.cassandra.concurrent.*;
import org.apache.cassandra.config.*;
@@ -47,6 +48,8 @@
import org.apache.cassandra.db.compaction.*;
import org.apache.cassandra.db.filter.ClusteringIndexFilter;
import org.apache.cassandra.db.filter.DataLimits;
+import org.apache.cassandra.db.streaming.CassandraStreamManager;
+import org.apache.cassandra.db.repair.CassandraTableRepairManager;
import org.apache.cassandra.db.view.TableViews;
import org.apache.cassandra.db.lifecycle.*;
import org.apache.cassandra.db.partitions.CachedPartition;
@@ -59,30 +62,32 @@
import org.apache.cassandra.index.SecondaryIndexManager;
import org.apache.cassandra.index.internal.CassandraIndex;
import org.apache.cassandra.index.transactions.UpdateTransaction;
-import org.apache.cassandra.io.FSError;
+import org.apache.cassandra.io.FSReadError;
import org.apache.cassandra.io.FSWriteError;
import org.apache.cassandra.io.sstable.Component;
-import org.apache.cassandra.io.sstable.CorruptSSTableException;
import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.io.sstable.SSTableMultiWriter;
import org.apache.cassandra.io.sstable.format.*;
-import org.apache.cassandra.io.sstable.format.big.BigFormat;
import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.metrics.Sampler;
+import org.apache.cassandra.metrics.Sampler.Sample;
+import org.apache.cassandra.metrics.Sampler.SamplerType;
import org.apache.cassandra.metrics.TableMetrics;
-import org.apache.cassandra.metrics.TableMetrics.Sampler;
+import org.apache.cassandra.repair.TableRepairManager;
import org.apache.cassandra.schema.*;
import org.apache.cassandra.schema.CompactionParams.TombstoneOption;
import org.apache.cassandra.service.CacheService;
import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.streaming.TableStreamManager;
import org.apache.cassandra.utils.*;
-import org.apache.cassandra.utils.TopKSampler.SamplerResult;
import org.apache.cassandra.utils.concurrent.OpOrder;
import org.apache.cassandra.utils.concurrent.Refs;
import org.apache.cassandra.utils.memory.MemtableAllocator;
import org.json.simple.JSONArray;
import org.json.simple.JSONObject;
+import static java.util.concurrent.TimeUnit.NANOSECONDS;
import static org.apache.cassandra.utils.ExecutorUtils.awaitTermination;
import static org.apache.cassandra.utils.ExecutorUtils.shutdown;
import static org.apache.cassandra.utils.Throwables.maybeFail;
@@ -90,44 +95,6 @@
public class ColumnFamilyStore implements ColumnFamilyStoreMBean
{
- // The directories which will be searched for sstables on cfs instantiation.
- private static volatile Directories.DataDirectory[] initialDirectories = Directories.dataDirectories;
-
- /**
- * A hook to add additional directories to initialDirectories.
- * Any additional directories should be added prior to ColumnFamilyStore instantiation on startup
- *
- * Since the directories used by a given table are determined by the compaction strategy,
- * it's possible for sstables to be written to directories specified outside of cassandra.yaml.
- * By adding additional directories to initialDirectories, sstables in these extra locations are
- * made discoverable on sstable instantiation.
- */
- public static synchronized void addInitialDirectories(Directories.DataDirectory[] newDirectories)
- {
- assert newDirectories != null;
-
- Set<Directories.DataDirectory> existing = Sets.newHashSet(initialDirectories);
-
- List<Directories.DataDirectory> replacementList = Lists.newArrayList(initialDirectories);
- for (Directories.DataDirectory directory: newDirectories)
- {
- if (!existing.contains(directory))
- {
- replacementList.add(directory);
- }
- }
-
- Directories.DataDirectory[] replacementArray = new Directories.DataDirectory[replacementList.size()];
- replacementList.toArray(replacementArray);
- initialDirectories = replacementArray;
- }
-
- public static Directories.DataDirectory[] getInitialDirectories()
- {
- Directories.DataDirectory[] src = initialDirectories;
- return Arrays.copyOf(src, src.length);
- }
-
private static final Logger logger = LoggerFactory.getLogger(ColumnFamilyStore.class);
/*
@@ -139,19 +106,20 @@
have that many flushes going at the same time.
*/
private static final ExecutorService flushExecutor = new JMXEnabledThreadPoolExecutor(DatabaseDescriptor.getFlushWriters(),
- StageManager.KEEPALIVE,
+ Stage.KEEP_ALIVE_SECONDS,
TimeUnit.SECONDS,
new LinkedBlockingQueue<Runnable>(),
new NamedThreadFactory("MemtableFlushWriter"),
"internal");
private static final ExecutorService [] perDiskflushExecutors = new ExecutorService[DatabaseDescriptor.getAllDataFileLocations().length];
+
static
{
for (int i = 0; i < DatabaseDescriptor.getAllDataFileLocations().length; i++)
{
perDiskflushExecutors[i] = new JMXEnabledThreadPoolExecutor(DatabaseDescriptor.getFlushWriters(),
- StageManager.KEEPALIVE,
+ Stage.KEEP_ALIVE_SECONDS,
TimeUnit.SECONDS,
new LinkedBlockingQueue<Runnable>(),
new NamedThreadFactory("PerDiskMemtableFlushWriter_"+i),
@@ -161,35 +129,28 @@
// post-flush executor is single threaded to provide guarantee that any flush Future on a CF will never return until prior flushes have completed
private static final ExecutorService postFlushExecutor = new JMXEnabledThreadPoolExecutor(1,
- StageManager.KEEPALIVE,
+ Stage.KEEP_ALIVE_SECONDS,
TimeUnit.SECONDS,
new LinkedBlockingQueue<Runnable>(),
new NamedThreadFactory("MemtablePostFlush"),
"internal");
private static final ExecutorService reclaimExecutor = new JMXEnabledThreadPoolExecutor(1,
- StageManager.KEEPALIVE,
+ Stage.KEEP_ALIVE_SECONDS,
TimeUnit.SECONDS,
new LinkedBlockingQueue<Runnable>(),
new NamedThreadFactory("MemtableReclaimMemory"),
"internal");
- private static final String[] COUNTER_NAMES = new String[]{"raw", "count", "error", "string"};
+ private static final String[] COUNTER_NAMES = new String[]{"table", "count", "error", "value"};
private static final String[] COUNTER_DESCS = new String[]
- { "partition key in raw hex bytes",
- "value of this partition for given sampler",
- "value is within the error bounds plus or minus of this",
- "the partition key turned into a human readable format" };
+ { "keyspace.tablename",
+ "number of occurances",
+ "error bounds",
+ "value" };
private static final CompositeType COUNTER_COMPOSITE_TYPE;
- private static final TabularType COUNTER_TYPE;
-
- private static final String[] SAMPLER_NAMES = new String[]{"cardinality", "partitions"};
- private static final String[] SAMPLER_DESCS = new String[]
- { "cardinality of partitions",
- "list of counter results" };
private static final String SAMPLING_RESULTS_NAME = "SAMPLING_RESULTS";
- private static final CompositeType SAMPLING_RESULT;
public static final String SNAPSHOT_TRUNCATE_PREFIX = "truncated";
public static final String SNAPSHOT_DROP_PREFIX = "dropped";
@@ -200,19 +161,15 @@
{
OpenType<?>[] counterTypes = new OpenType[] { SimpleType.STRING, SimpleType.LONG, SimpleType.LONG, SimpleType.STRING };
COUNTER_COMPOSITE_TYPE = new CompositeType(SAMPLING_RESULTS_NAME, SAMPLING_RESULTS_NAME, COUNTER_NAMES, COUNTER_DESCS, counterTypes);
- COUNTER_TYPE = new TabularType(SAMPLING_RESULTS_NAME, SAMPLING_RESULTS_NAME, COUNTER_COMPOSITE_TYPE, COUNTER_NAMES);
-
- OpenType<?>[] samplerTypes = new OpenType[] { SimpleType.LONG, COUNTER_TYPE };
- SAMPLING_RESULT = new CompositeType(SAMPLING_RESULTS_NAME, SAMPLING_RESULTS_NAME, SAMPLER_NAMES, SAMPLER_DESCS, samplerTypes);
} catch (OpenDataException e)
{
- throw Throwables.propagate(e);
+ throw new RuntimeException(e);
}
}
public final Keyspace keyspace;
public final String name;
- public final CFMetaData metadata;
+ public final TableMetadataRef metadata;
private final String mbeanName;
@Deprecated
private final String oldMBeanName;
@@ -244,22 +201,25 @@
private final CompactionStrategyManager compactionStrategyManager;
- private volatile Directories directories;
+ private final Directories directories;
public final TableMetrics metric;
- public volatile long sampleLatencyNanos;
- private final ScheduledFuture<?> latencyCalculator;
+ public volatile long sampleReadLatencyNanos;
+ public volatile long additionalWriteLatencyNanos;
+
+ private final CassandraTableWriteHandler writeHandler;
+ private final CassandraStreamManager streamManager;
+
+ private final TableRepairManager repairManager;
+
+ private final SSTableImporter sstableImporter;
private volatile boolean compactionSpaceCheck = true;
@VisibleForTesting
final DiskBoundaryManager diskBoundaryManager = new DiskBoundaryManager();
- public static void shutdownFlushExecutor() throws InterruptedException
- {
- flushExecutor.shutdown();
- flushExecutor.awaitTermination(60, TimeUnit.SECONDS);
- }
+ private volatile boolean neverPurgeTombstones = false;
public static void shutdownPostFlushExecutor() throws InterruptedException
{
@@ -282,16 +242,15 @@
// only update these runtime-modifiable settings if they have not been modified.
if (!minCompactionThreshold.isModified())
for (ColumnFamilyStore cfs : concatWithIndexes())
- cfs.minCompactionThreshold = new DefaultValue(metadata.params.compaction.minCompactionThreshold());
+ cfs.minCompactionThreshold = new DefaultValue(metadata().params.compaction.minCompactionThreshold());
if (!maxCompactionThreshold.isModified())
for (ColumnFamilyStore cfs : concatWithIndexes())
- cfs.maxCompactionThreshold = new DefaultValue(metadata.params.compaction.maxCompactionThreshold());
+ cfs.maxCompactionThreshold = new DefaultValue(metadata().params.compaction.maxCompactionThreshold());
if (!crcCheckChance.isModified())
for (ColumnFamilyStore cfs : concatWithIndexes())
- cfs.crcCheckChance = new DefaultValue(metadata.params.crcCheckChance);
+ cfs.crcCheckChance = new DefaultValue(metadata().params.crcCheckChance);
- compactionStrategyManager.maybeReload(metadata);
- directories = compactionStrategyManager.getDirectories();
+ compactionStrategyManager.maybeReload(metadata());
scheduleFlush();
@@ -299,13 +258,13 @@
// If the CF comparator has changed, we need to change the memtable,
// because the old one still aliases the previous comparator.
- if (data.getView().getCurrentMemtable().initialComparator != metadata.comparator)
+ if (data.getView().getCurrentMemtable().initialComparator != metadata().comparator)
switchMemtable();
}
void scheduleFlush()
{
- int period = metadata.params.memtableFlushPeriodInMs;
+ int period = metadata().params.memtableFlushPeriodInMs;
if (period > 0)
{
logger.trace("scheduling flush in {} ms", period);
@@ -350,9 +309,9 @@
};
}
- public void setCompactionParametersJson(String options)
+ public Map<String, String> getCompactionParameters()
{
- setCompactionParameters(FBUtilities.fromJsonMap(options));
+ return compactionStrategyManager.getCompactionParams().asMap();
}
public String getCompactionParametersJson()
@@ -376,22 +335,28 @@
}
}
- public Map<String, String> getCompactionParameters()
+ public void setCompactionParametersJson(String options)
{
- return compactionStrategyManager.getCompactionParams().asMap();
+ setCompactionParameters(FBUtilities.fromJsonMap(options));
}
public Map<String,String> getCompressionParameters()
{
- return metadata.params.compression.asMap();
+ return metadata().params.compression.asMap();
+ }
+
+ public String getCompressionParametersJson()
+ {
+ return FBUtilities.json(getCompressionParameters());
}
public void setCompressionParameters(Map<String,String> opts)
{
try
{
- metadata.compression(CompressionParams.fromMap(opts));
- metadata.params.compression.validate();
+ CompressionParams params = CompressionParams.fromMap(opts);
+ params.validate();
+ throw new UnsupportedOperationException(); // TODO FIXME CASSANDRA-12949
}
catch (ConfigurationException e)
{
@@ -399,30 +364,36 @@
}
}
+ public void setCompressionParametersJson(String options)
+ {
+ setCompressionParameters(FBUtilities.fromJsonMap(options));
+ }
+
@VisibleForTesting
public ColumnFamilyStore(Keyspace keyspace,
- String columnFamilyName,
- int generation,
- CFMetaData metadata,
- Directories directories,
- boolean loadSSTables,
- boolean registerBookeeping,
- boolean offline)
+ String columnFamilyName,
+ int generation,
+ TableMetadataRef metadata,
+ Directories directories,
+ boolean loadSSTables,
+ boolean registerBookeeping,
+ boolean offline)
{
assert directories != null;
assert metadata != null : "null metadata for " + keyspace + ":" + columnFamilyName;
this.keyspace = keyspace;
this.metadata = metadata;
+ this.directories = directories;
name = columnFamilyName;
- minCompactionThreshold = new DefaultValue<>(metadata.params.compaction.minCompactionThreshold());
- maxCompactionThreshold = new DefaultValue<>(metadata.params.compaction.maxCompactionThreshold());
- crcCheckChance = new DefaultValue<>(metadata.params.crcCheckChance);
- indexManager = new SecondaryIndexManager(this);
- viewManager = keyspace.viewManager.forTable(metadata);
+ minCompactionThreshold = new DefaultValue<>(metadata.get().params.compaction.minCompactionThreshold());
+ maxCompactionThreshold = new DefaultValue<>(metadata.get().params.compaction.maxCompactionThreshold());
+ crcCheckChance = new DefaultValue<>(metadata.get().params.crcCheckChance);
+ viewManager = keyspace.viewManager.forTable(metadata.id);
metric = new TableMetrics(this);
fileIndexGenerator.set(generation);
- sampleLatencyNanos = TimeUnit.MILLISECONDS.toNanos(DatabaseDescriptor.getReadRpcTimeout() / 2);
+ sampleReadLatencyNanos = DatabaseDescriptor.getReadRpcTimeout(NANOSECONDS) / 2;
+ additionalWriteLatencyNanos = DatabaseDescriptor.getWriteRpcTimeout(NANOSECONDS) / 2;
logger.info("Initializing {}.{}", keyspace.getName(), name);
@@ -440,22 +411,9 @@
data.addInitialSSTables(sstables);
}
- /**
- * When creating a CFS offline we change the default logic needed by CASSANDRA-8671
- * and link the passed directories to be picked up by the compaction strategy
- */
- if (offline)
- this.directories = directories;
- else
- this.directories = new Directories(metadata, Directories.dataDirectories);
-
-
// compaction strategy should be created after the CFS has been prepared
compactionStrategyManager = new CompactionStrategyManager(this);
- // Since compaction can re-define data dir we need to reinit directories
- this.directories = compactionStrategyManager.getDirectories();
-
if (maxCompactionThreshold.value() <= 0 || minCompactionThreshold.value() <=0)
{
logger.warn("Disabling compaction strategy by setting compaction thresholds to 0 is deprecated, set the compaction option 'enabled' to 'false' instead.");
@@ -463,8 +421,9 @@
}
// create the private ColumnFamilyStores for the secondary column indexes
- for (IndexMetadata info : metadata.getIndexes())
- indexManager.addIndex(info);
+ indexManager = new SecondaryIndexManager(this);
+ for (IndexMetadata info : metadata.get().indexes)
+ indexManager.addIndex(info, true);
if (registerBookeeping)
{
@@ -475,46 +434,53 @@
oldMBeanName = String.format("org.apache.cassandra.db:type=%s,keyspace=%s,columnfamily=%s",
isIndex() ? "IndexColumnFamilies" : "ColumnFamilies",
keyspace.getName(), name);
- try
- {
- ObjectName[] objectNames = {new ObjectName(mbeanName), new ObjectName(oldMBeanName)};
- for (ObjectName objectName : objectNames)
- {
- MBeanWrapper.instance.registerMBean(this, objectName);
- }
- }
- catch (Exception e)
- {
- throw new RuntimeException(e);
- }
- logger.trace("retryPolicy for {} is {}", name, this.metadata.params.speculativeRetry);
- latencyCalculator = ScheduledExecutors.optionalTasks.scheduleWithFixedDelay(new Runnable()
- {
- public void run()
- {
- SpeculativeRetryParam retryPolicy = ColumnFamilyStore.this.metadata.params.speculativeRetry;
- switch (retryPolicy.kind())
- {
- case PERCENTILE:
- // get percentile in nanos
- sampleLatencyNanos = (long) (metric.coordinatorReadLatency.getSnapshot().getValue(retryPolicy.threshold()));
- break;
- case CUSTOM:
- sampleLatencyNanos = (long) retryPolicy.threshold();
- break;
- default:
- sampleLatencyNanos = Long.MAX_VALUE;
- break;
- }
- }
- }, DatabaseDescriptor.getReadRpcTimeout(), DatabaseDescriptor.getReadRpcTimeout(), TimeUnit.MILLISECONDS);
+
+ String[] objectNames = {mbeanName, oldMBeanName};
+ for (String objectName : objectNames)
+ MBeanWrapper.instance.registerMBean(this, objectName);
}
else
{
- latencyCalculator = ScheduledExecutors.optionalTasks.schedule(Runnables.doNothing(), 0, TimeUnit.NANOSECONDS);
mbeanName = null;
oldMBeanName= null;
}
+ writeHandler = new CassandraTableWriteHandler(this);
+ streamManager = new CassandraStreamManager(this);
+ repairManager = new CassandraTableRepairManager(this);
+ sstableImporter = new SSTableImporter(this);
+ }
+
+ public void updateSpeculationThreshold()
+ {
+ try
+ {
+ sampleReadLatencyNanos = metadata().params.speculativeRetry.calculateThreshold(metric.coordinatorReadLatency.getSnapshot(), sampleReadLatencyNanos);
+ additionalWriteLatencyNanos = metadata().params.additionalWritePolicy.calculateThreshold(metric.coordinatorWriteLatency.getSnapshot(), additionalWriteLatencyNanos);
+ }
+ catch (Throwable e)
+ {
+ logger.error("Exception caught while calculating speculative retry threshold for {}: {}", metadata(), e);
+ }
+ }
+
+ public TableWriteHandler getWriteHandler()
+ {
+ return writeHandler;
+ }
+
+ public TableStreamManager getStreamManager()
+ {
+ return streamManager;
+ }
+
+ public TableRepairManager getRepairManager()
+ {
+ return repairManager;
+ }
+
+ public TableMetadata metadata()
+ {
+ return metadata.get();
}
public Directories getDirectories()
@@ -522,15 +488,15 @@
return directories;
}
- public SSTableMultiWriter createSSTableMultiWriter(Descriptor descriptor, long keyCount, long repairedAt, int sstableLevel, SerializationHeader header, LifecycleNewTracker lifecycleNewTracker)
+ public SSTableMultiWriter createSSTableMultiWriter(Descriptor descriptor, long keyCount, long repairedAt, UUID pendingRepair, boolean isTransient, int sstableLevel, SerializationHeader header, LifecycleNewTracker lifecycleNewTracker)
{
- MetadataCollector collector = new MetadataCollector(metadata.comparator).sstableLevel(sstableLevel);
- return createSSTableMultiWriter(descriptor, keyCount, repairedAt, collector, header, lifecycleNewTracker);
+ MetadataCollector collector = new MetadataCollector(metadata().comparator).sstableLevel(sstableLevel);
+ return createSSTableMultiWriter(descriptor, keyCount, repairedAt, pendingRepair, isTransient, collector, header, lifecycleNewTracker);
}
- public SSTableMultiWriter createSSTableMultiWriter(Descriptor descriptor, long keyCount, long repairedAt, MetadataCollector metadataCollector, SerializationHeader header, LifecycleNewTracker lifecycleNewTracker)
+ public SSTableMultiWriter createSSTableMultiWriter(Descriptor descriptor, long keyCount, long repairedAt, UUID pendingRepair, boolean isTransient, MetadataCollector metadataCollector, SerializationHeader header, LifecycleNewTracker lifecycleNewTracker)
{
- return getCompactionStrategyManager().createSSTableMultiWriter(descriptor, keyCount, repairedAt, metadataCollector, header, indexManager.listIndexes(), lifecycleNewTracker);
+ return getCompactionStrategyManager().createSSTableMultiWriter(descriptor, keyCount, repairedAt, pendingRepair, isTransient, metadataCollector, header, indexManager.listIndexes(), lifecycleNewTracker);
}
public boolean supportsEarlyOpen()
@@ -563,13 +529,12 @@
}
}
- latencyCalculator.cancel(false);
compactionStrategyManager.shutdown();
- SystemKeyspace.removeTruncationRecord(metadata.cfId);
+ SystemKeyspace.removeTruncationRecord(metadata.id);
data.dropSSTables();
LifecycleTransaction.waitForDeletions();
- indexManager.invalidateAllIndexesBlocking();
+ indexManager.dropAllIndexes();
invalidateCaches();
}
@@ -597,24 +562,24 @@
}
- public static ColumnFamilyStore createColumnFamilyStore(Keyspace keyspace, CFMetaData metadata, boolean loadSSTables)
+ public static ColumnFamilyStore createColumnFamilyStore(Keyspace keyspace, TableMetadataRef metadata, boolean loadSSTables)
{
- return createColumnFamilyStore(keyspace, metadata.cfName, metadata, loadSSTables);
+ return createColumnFamilyStore(keyspace, metadata.name, metadata, loadSSTables);
}
public static synchronized ColumnFamilyStore createColumnFamilyStore(Keyspace keyspace,
String columnFamily,
- CFMetaData metadata,
+ TableMetadataRef metadata,
boolean loadSSTables)
{
- Directories directories = new Directories(metadata, initialDirectories);
+ Directories directories = new Directories(metadata.get());
return createColumnFamilyStore(keyspace, columnFamily, metadata, directories, loadSSTables, true, false);
}
/** This is only directly used by offline tools */
public static synchronized ColumnFamilyStore createColumnFamilyStore(Keyspace keyspace,
String columnFamily,
- CFMetaData metadata,
+ TableMetadataRef metadata,
Directories directories,
boolean loadSSTables,
boolean registerBookkeeping,
@@ -641,9 +606,9 @@
* Removes unnecessary files from the cf directory at startup: these include temp files, orphans, zero-length files
* and compacted sstables. Files that cannot be recognized will be ignored.
*/
- public static void scrubDataDirectories(CFMetaData metadata) throws StartupException
+ public static void scrubDataDirectories(TableMetadata metadata) throws StartupException
{
- Directories directories = new Directories(metadata, initialDirectories);
+ Directories directories = new Directories(metadata);
Set<File> cleanedDirectories = new HashSet<>();
// clear ephemeral snapshots that were not properly cleared last session (CASSANDRA-7357)
@@ -651,15 +616,15 @@
directories.removeTemporaryDirectories();
- logger.trace("Removing temporary or obsoleted files from unfinished operations for table {}", metadata.cfName);
+ logger.trace("Removing temporary or obsoleted files from unfinished operations for table {}", metadata.name);
if (!LifecycleTransaction.removeUnfinishedLeftovers(metadata))
throw new StartupException(StartupException.ERR_WRONG_DISK_STATE,
- String.format("Cannot remove temporary or obsoleted files for %s.%s due to a problem with transaction " +
+ String.format("Cannot remove temporary or obsoleted files for %s due to a problem with transaction " +
"log files. Please check records with problems in the log messages above and fix them. " +
"Refer to the 3.0 upgrading instructions in NEWS.txt " +
- "for a description of transaction log files.", metadata.ksName, metadata.cfName));
+ "for a description of transaction log files.", metadata.toString()));
- logger.trace("Further extra check for orphan sstable files for {}", metadata.cfName);
+ logger.trace("Further extra check for orphan sstable files for {}", metadata.name);
for (Map.Entry<Descriptor,Set<Component>> sstableFiles : directories.sstableLister(Directories.OnTxnErr.IGNORE).list().entrySet())
{
Descriptor desc = sstableFiles.getKey();
@@ -670,7 +635,10 @@
{
cleanedDirectories.add(directory);
for (File tmpFile : desc.getTemporaryFiles())
+ {
+ logger.info("Removing unfinished temporary file {}", tmpFile);
tmpFile.delete();
+ }
}
File dataFile = new File(desc.filenameFor(Component.DATA));
@@ -689,7 +657,7 @@
}
// cleanup incomplete saved caches
- Pattern tmpCacheFilePattern = Pattern.compile(metadata.ksName + "-" + metadata.cfName + "-(Key|Row)Cache.*\\.tmp$");
+ Pattern tmpCacheFilePattern = Pattern.compile(metadata.keyspace + "-" + metadata.name + "-(Key|Row)Cache.*\\.tmp$");
File dir = new File(DatabaseDescriptor.getSavedCachesLocation());
if (dir.exists())
@@ -702,16 +670,16 @@
}
// also clean out any index leftovers.
- for (IndexMetadata index : metadata.getIndexes())
+ for (IndexMetadata index : metadata.indexes)
if (!index.isCustom())
{
- CFMetaData indexMetadata = CassandraIndex.indexCfsMetadata(metadata, index);
+ TableMetadata indexMetadata = CassandraIndex.indexCfsMetadata(metadata, index);
scrubDataDirectories(indexMetadata);
}
}
/**
- * See #{@code StorageService.loadNewSSTables(String, String)} for more info
+ * See #{@code StorageService.importNewSSTables} for more info
*
* @param ksName The keyspace name
* @param cfName The columnFamily name
@@ -723,121 +691,72 @@
keyspace.getColumnFamilyStore(cfName).loadNewSSTables();
}
+ @Deprecated
+ public void loadNewSSTables()
+ {
+
+ SSTableImporter.Options options = SSTableImporter.Options.options().resetLevel(true).build();
+ sstableImporter.importNewSSTables(options);
+ }
+
/**
* #{@inheritDoc}
*/
- public synchronized void loadNewSSTables()
+ public synchronized List<String> importNewSSTables(Set<String> srcPaths, boolean resetLevel, boolean clearRepaired, boolean verifySSTables, boolean verifyTokens, boolean invalidateCaches, boolean extendedVerify)
{
- logger.info("Loading new SSTables for {}/{}...", keyspace.getName(), name);
+ SSTableImporter.Options options = SSTableImporter.Options.options(srcPaths)
+ .resetLevel(resetLevel)
+ .clearRepaired(clearRepaired)
+ .verifySSTables(verifySSTables)
+ .verifyTokens(verifyTokens)
+ .invalidateCaches(invalidateCaches)
+ .extendedVerify(extendedVerify).build();
- Set<Descriptor> currentDescriptors = new HashSet<>();
- for (SSTableReader sstable : getSSTables(SSTableSet.CANONICAL))
- currentDescriptors.add(sstable.descriptor);
- Set<SSTableReader> newSSTables = new HashSet<>();
+ return sstableImporter.importNewSSTables(options);
+ }
- Directories.SSTableLister lister = getDirectories().sstableLister(Directories.OnTxnErr.IGNORE).skipTemporary(true);
- for (Map.Entry<Descriptor, Set<Component>> entry : lister.list().entrySet())
+ Descriptor getUniqueDescriptorFor(Descriptor descriptor, File targetDirectory)
+ {
+ Descriptor newDescriptor;
+ do
{
- Descriptor descriptor = entry.getKey();
-
- if (currentDescriptors.contains(descriptor))
- continue; // old (initialized) SSTable found, skipping
-
- if (!descriptor.isCompatible())
- throw new RuntimeException(String.format("Can't open incompatible SSTable! Current version %s, found file: %s",
- descriptor.getFormat().getLatestVersion(),
- descriptor));
-
- // force foreign sstables to level 0
- try
- {
- if (new File(descriptor.filenameFor(Component.STATS)).exists())
- descriptor.getMetadataSerializer().mutateLevel(descriptor, 0);
- }
- catch (IOException e)
- {
- FileUtils.handleCorruptSSTable(new CorruptSSTableException(e, entry.getKey().filenameFor(Component.STATS)));
- logger.error("Cannot read sstable {}; other IO error, skipping table", entry, e);
- continue;
- }
-
- // Increment the generation until we find a filename that doesn't exist. This is needed because the new
- // SSTables that are being loaded might already use these generation numbers.
- Descriptor newDescriptor;
- do
- {
- newDescriptor = new Descriptor(descriptor.version,
- descriptor.directory,
- descriptor.ksname,
- descriptor.cfname,
- fileIndexGenerator.incrementAndGet(),
- descriptor.formatType,
- descriptor.digestComponent);
- }
- while (new File(newDescriptor.filenameFor(Component.DATA)).exists());
-
- logger.info("Renaming new SSTable {} to {}", descriptor, newDescriptor);
- SSTableWriter.rename(descriptor, newDescriptor, entry.getValue());
-
- SSTableReader reader;
- try
- {
- reader = SSTableReader.open(newDescriptor, entry.getValue(), metadata);
- }
- catch (CorruptSSTableException ex)
- {
- FileUtils.handleCorruptSSTable(ex);
- logger.error("Corrupt sstable {}; skipping table", entry, ex);
- continue;
- }
- catch (FSError ex)
- {
- FileUtils.handleFSError(ex);
- logger.error("Cannot read sstable {}; file system error, skipping table", entry, ex);
- continue;
- }
- catch (IOException ex)
- {
- FileUtils.handleCorruptSSTable(new CorruptSSTableException(ex, entry.getKey().filenameFor(Component.DATA)));
- logger.error("Cannot read sstable {}; other IO error, skipping table", entry, ex);
- continue;
- }
- newSSTables.add(reader);
+ newDescriptor = new Descriptor(descriptor.version,
+ targetDirectory,
+ descriptor.ksname,
+ descriptor.cfname,
+ // Increment the generation until we find a filename that doesn't exist. This is needed because the new
+ // SSTables that are being loaded might already use these generation numbers.
+ fileIndexGenerator.incrementAndGet(),
+ descriptor.formatType);
}
-
- if (newSSTables.isEmpty())
- {
- logger.info("No new SSTables were found for {}/{}", keyspace.getName(), name);
- return;
- }
-
- logger.info("Loading new SSTables and building secondary indexes for {}/{}: {}", keyspace.getName(), name, newSSTables);
-
- try (Refs<SSTableReader> refs = Refs.ref(newSSTables))
- {
- data.addSSTables(newSSTables);
- indexManager.buildAllIndexesBlocking(newSSTables);
- }
-
- logger.info("Done loading load new SSTables for {}/{}", keyspace.getName(), name);
+ while (new File(newDescriptor.filenameFor(Component.DATA)).exists());
+ return newDescriptor;
}
public void rebuildSecondaryIndex(String idxName)
{
- rebuildSecondaryIndex(keyspace.getName(), metadata.cfName, idxName);
+ rebuildSecondaryIndex(keyspace.getName(), metadata.name, idxName);
}
public static void rebuildSecondaryIndex(String ksName, String cfName, String... idxNames)
{
ColumnFamilyStore cfs = Keyspace.open(ksName).getColumnFamilyStore(cfName);
- Set<String> indexes = new HashSet<String>(Arrays.asList(idxNames));
+ logger.info("User Requested secondary index re-build for {}/{} indexes: {}", ksName, cfName, Joiner.on(',').join(idxNames));
+ cfs.indexManager.rebuildIndexesBlocking(Sets.newHashSet(Arrays.asList(idxNames)));
+ }
- Iterable<SSTableReader> sstables = cfs.getSSTables(SSTableSet.CANONICAL);
- try (Refs<SSTableReader> refs = Refs.ref(sstables))
+ public AbstractCompactionStrategy createCompactionStrategyInstance(CompactionParams compactionParams)
+ {
+ try
{
- logger.info("User Requested secondary index re-build for {}/{} indexes: {}", ksName, cfName, Joiner.on(',').join(idxNames));
- cfs.indexManager.rebuildIndexesBlocking(refs, indexes);
+ Constructor<? extends AbstractCompactionStrategy> constructor =
+ compactionParams.klass().getConstructor(ColumnFamilyStore.class, Map.class);
+ return constructor.newInstance(this, compactionParams.options());
+ }
+ catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException | InstantiationException e)
+ {
+ throw new RuntimeException(e);
}
}
@@ -852,26 +771,24 @@
return name;
}
- public String getSSTablePath(File directory)
+ public Descriptor newSSTableDescriptor(File directory)
{
- return getSSTablePath(directory, SSTableFormat.Type.current().info.getLatestVersion(), SSTableFormat.Type.current());
+ return newSSTableDescriptor(directory, SSTableFormat.Type.current().info.getLatestVersion(), SSTableFormat.Type.current());
}
- public String getSSTablePath(File directory, SSTableFormat.Type format)
+ public Descriptor newSSTableDescriptor(File directory, SSTableFormat.Type format)
{
- return getSSTablePath(directory, format.info.getLatestVersion(), format);
+ return newSSTableDescriptor(directory, format.info.getLatestVersion(), format);
}
- private String getSSTablePath(File directory, Version version, SSTableFormat.Type format)
+ public Descriptor newSSTableDescriptor(File directory, Version version, SSTableFormat.Type format)
{
- Descriptor desc = new Descriptor(version,
- directory,
- keyspace.getName(),
- name,
- fileIndexGenerator.incrementAndGet(),
- format,
- Component.digestFor(BigFormat.latestVersion.uncompressedChecksumType()));
- return desc.filenameFor(Component.DATA);
+ return new Descriptor(version,
+ directory,
+ keyspace.getName(),
+ name,
+ fileIndexGenerator.incrementAndGet(),
+ format);
}
/**
@@ -929,7 +846,7 @@
offHeapTotal += allocator.offHeap().owns();
}
- logger.debug("Enqueuing flush of {}: {}",
+ logger.info("Enqueuing flush of {}: {}",
name,
String.format("%s (%.0f%%) on-heap, %s (%.0f%%) off-heap",
FBUtilities.prettyPrintMemory(onHeapTotal),
@@ -1030,7 +947,7 @@
{
Memtable memtable = memtables.get(0);
commitLogUpperBound = memtable.getCommitLogUpperBound();
- CommitLog.instance.discardCompletedSegments(metadata.cfId, memtable.getCommitLogLowerBound(), commitLogUpperBound);
+ CommitLog.instance.discardCompletedSegments(metadata.id, memtable.getCommitLogLowerBound(), commitLogUpperBound);
}
metric.pendingFlushes.dec();
@@ -1319,7 +1236,7 @@
float flushingOffHeap = Memtable.MEMORY_POOL.offHeap.reclaimingRatio();
float thisOnHeap = largest.getAllocator().onHeap().ownershipRatio();
float thisOffHeap = largest.getAllocator().offHeap().ownershipRatio();
- logger.debug("Flushing largest {} to free up room. Used total: {}, live: {}, flushing: {}, this: {}",
+ logger.info("Flushing largest {} to free up room. Used total: {}, live: {}, flushing: {}, this: {}",
largest.cfs, ratio(usedOnHeap, usedOffHeap), ratio(liveOnHeap, liveOffHeap),
ratio(flushingOnHeap, flushingOffHeap), ratio(thisOnHeap, thisOffHeap));
largest.cfs.switchMemtableIfCurrent(largest);
@@ -1349,8 +1266,10 @@
long timeDelta = mt.put(update, indexer, opGroup);
DecoratedKey key = update.partitionKey();
invalidateCachedPartition(key);
- metric.samplers.get(Sampler.WRITES).addSample(key.getKey(), key.hashCode(), 1);
- StorageHook.instance.reportWrite(metadata.cfId, update);
+ metric.topWritePartitionFrequency.addSample(key.getKey(), 1);
+ if (metric.topWritePartitionSize.isEnabled()) // dont compute datasize if not needed
+ metric.topWritePartitionSize.addSample(key.getKey(), update.dataSize());
+ StorageHook.instance.reportWrite(metadata.id, update);
metric.writeLatency.addNano(System.nanoTime() - start);
// CASSANDRA-11117 - certain resolution paths on memtable put can result in very
// large time deltas, either through a variety of sentinel timestamps (used for empty values, ensuring
@@ -1455,7 +1374,7 @@
public void addSSTable(SSTableReader sstable)
{
assert sstable.getColumnFamilyName().equals(name);
- addSSTables(Arrays.asList(sstable));
+ addSSTables(Collections.singletonList(sstable));
}
public void addSSTables(Collection<SSTableReader> sstables)
@@ -1486,12 +1405,12 @@
// cleanup size estimation only counts bytes for keys local to this node
long expectedFileSize = 0;
- Collection<Range<Token>> ranges = StorageService.instance.getLocalRanges(keyspace.getName());
+ Collection<Range<Token>> ranges = StorageService.instance.getLocalReplicas(keyspace.getName()).ranges();
for (SSTableReader sstable : sstables)
{
- List<Pair<Long, Long>> positions = sstable.getPositionsForRanges(ranges);
- for (Pair<Long, Long> position : positions)
- expectedFileSize += position.right - position.left;
+ List<SSTableReader.PartitionPositionBounds> positions = sstable.getPositionsForRanges(ranges);
+ for (SSTableReader.PartitionPositionBounds position : positions)
+ expectedFileSize += position.upperPosition - position.lowerPosition;
}
double compressionRatio = metric.compressionRatio.getValue();
@@ -1573,9 +1492,9 @@
return true;
}
- public CompactionManager.AllSSTableOpStatus verify(boolean extendedVerify) throws ExecutionException, InterruptedException
+ public CompactionManager.AllSSTableOpStatus verify(Verifier.Options options) throws ExecutionException, InterruptedException
{
- return CompactionManager.instance.performVerify(ColumnFamilyStore.this, extendedVerify);
+ return CompactionManager.instance.performVerify(ColumnFamilyStore.this, options);
}
public CompactionManager.AllSSTableOpStatus sstablesRewrite(boolean excludeCurrentVersion, int jobs) throws ExecutionException, InterruptedException
@@ -1601,7 +1520,9 @@
void replaceFlushed(Memtable memtable, Collection<SSTableReader> sstables)
{
- compactionStrategyManager.replaceFlushed(memtable, sstables);
+ data.replaceFlushed(memtable, sstables);
+ if (sstables != null && !sstables.isEmpty())
+ CompactionManager.instance.submitBackground(this);
}
public boolean isValid()
@@ -1632,7 +1553,11 @@
return data.getUncompacting();
}
- public boolean isFilterFullyCoveredBy(ClusteringIndexFilter filter, DataLimits limits, CachedPartition cached, int nowInSec)
+ public boolean isFilterFullyCoveredBy(ClusteringIndexFilter filter,
+ DataLimits limits,
+ CachedPartition cached,
+ int nowInSec,
+ boolean enforceStrictLiveness)
{
// We can use the cached value only if we know that no data it doesn't contain could be covered
// by the query filter, that is if:
@@ -1643,7 +1568,7 @@
// what we're caching. Wen doing that, we should be careful about expiring cells: we should count
// something expired that wasn't when the partition was cached, or we could decide that the whole
// partition is cached when it's not. This is why we use CachedPartition#cachedLiveRows.
- if (cached.cachedLiveRows() < metadata.params.caching.rowsPerPartitionToCache())
+ if (cached.cachedLiveRows() < metadata().params.caching.rowsPerPartitionToCache())
return true;
// If the whole partition isn't cached, then we must guarantee that the filter cannot select data that
@@ -1653,13 +1578,13 @@
return (filter.isHeadFilter() && limits.hasEnoughLiveData(cached,
nowInSec,
filter.selectsAllPartition(),
- metadata.enforceStrictLiveness()))
+ enforceStrictLiveness))
|| filter.isFullyCoveredBy(cached);
}
public int gcBefore(int nowInSec)
{
- return nowInSec - metadata.params.gcGraceSeconds;
+ return nowInSec - metadata().params.gcGraceSeconds;
}
@SuppressWarnings("resource")
@@ -1704,7 +1629,7 @@
public List<String> getSSTablesForKey(String key, boolean hexFormat)
{
- ByteBuffer keyBuffer = hexFormat ? ByteBufferUtil.hexToBytes(key) : metadata.getKeyValidator().fromString(key);
+ ByteBuffer keyBuffer = hexFormat ? ByteBufferUtil.hexToBytes(key) : metadata().partitionKeyType.fromString(key);
DecoratedKey dk = decorateKey(keyBuffer);
try (OpOrder.Group op = readOrdering.start())
{
@@ -1719,30 +1644,28 @@
}
}
-
- public void beginLocalSampling(String sampler, int capacity)
+ public void beginLocalSampling(String sampler, int capacity, int durationMillis)
{
- metric.samplers.get(Sampler.valueOf(sampler)).beginSampling(capacity);
+ metric.samplers.get(SamplerType.valueOf(sampler)).beginSampling(capacity, durationMillis);
}
- public CompositeData finishLocalSampling(String sampler, int count) throws OpenDataException
+ @SuppressWarnings({ "rawtypes", "unchecked" })
+ public List<CompositeData> finishLocalSampling(String sampler, int count) throws OpenDataException
{
- SamplerResult<ByteBuffer> samplerResults = metric.samplers.get(Sampler.valueOf(sampler))
- .finishSampling(count);
- TabularDataSupport result = new TabularDataSupport(COUNTER_TYPE);
- for (Counter<ByteBuffer> counter : samplerResults.topK)
+ Sampler samplerImpl = metric.samplers.get(SamplerType.valueOf(sampler));
+ List<Sample> samplerResults = samplerImpl.finishSampling(count);
+ List<CompositeData> result = new ArrayList<>(count);
+ for (Sample counter : samplerResults)
{
//Not duplicating the buffer for safety because AbstractSerializer and ByteBufferUtil.bytesToHex
//don't modify position or limit
- ByteBuffer key = counter.getItem();
- result.put(new CompositeDataSupport(COUNTER_COMPOSITE_TYPE, COUNTER_NAMES, new Object[] {
- ByteBufferUtil.bytesToHex(key), // raw
- counter.getCount(), // count
- counter.getError(), // error
- metadata.getKeyValidator().getString(key) })); // string
+ result.add(new CompositeDataSupport(COUNTER_COMPOSITE_TYPE, COUNTER_NAMES, new Object[] {
+ keyspace.getName() + "." + name,
+ counter.count,
+ counter.error,
+ samplerImpl.toString(counter.value) })); // string
}
- return new CompositeDataSupport(SAMPLING_RESULT, SAMPLER_NAMES, new Object[]{
- samplerResults.cardinality, result});
+ return result;
}
public boolean isCompactionDiskSpaceCheckEnabled()
@@ -1757,25 +1680,25 @@
public void cleanupCache()
{
- Collection<Range<Token>> ranges = StorageService.instance.getLocalRanges(keyspace.getName());
+ Collection<Range<Token>> ranges = StorageService.instance.getLocalReplicas(keyspace.getName()).ranges();
for (Iterator<RowCacheKey> keyIter = CacheService.instance.rowCache.keyIterator();
keyIter.hasNext(); )
{
RowCacheKey key = keyIter.next();
DecoratedKey dk = decorateKey(ByteBuffer.wrap(key.key));
- if (key.ksAndCFName.equals(metadata.ksAndCFName) && !Range.isInRanges(dk.getToken(), ranges))
+ if (key.sameTable(metadata()) && !Range.isInRanges(dk.getToken(), ranges))
invalidateCachedPartition(dk);
}
- if (metadata.isCounter())
+ if (metadata().isCounter())
{
for (Iterator<CounterCacheKey> keyIter = CacheService.instance.counterCache.keyIterator();
keyIter.hasNext(); )
{
CounterCacheKey key = keyIter.next();
- DecoratedKey dk = decorateKey(ByteBuffer.wrap(key.partitionKey));
- if (key.ksAndCFName.equals(metadata.ksAndCFName) && !Range.isInRanges(dk.getToken(), ranges))
+ DecoratedKey dk = decorateKey(key.partitionKey());
+ if (key.sameTable(metadata()) && !Range.isInRanges(dk.getToken(), ranges))
CacheService.instance.counterCache.remove(key);
}
}
@@ -1783,7 +1706,7 @@
public ClusteringComparator getComparator()
{
- return metadata.comparator;
+ return metadata().comparator;
}
public void snapshotWithoutFlush(String snapshotName)
@@ -1816,7 +1739,7 @@
}
writeSnapshotManifest(filesJSONArr, snapshotName);
- if (!SchemaConstants.isLocalSystemKeyspace(metadata.ksName) && !SchemaConstants.isReplicatedSystemKeyspace(metadata.ksName))
+ if (!SchemaConstants.isLocalSystemKeyspace(metadata.keyspace) && !SchemaConstants.isReplicatedSystemKeyspace(metadata.keyspace))
writeSnapshotSchema(snapshotName);
if (ephemeral)
@@ -1857,8 +1780,9 @@
try (PrintStream out = new PrintStream(schemaFile))
{
- for (String s: ColumnFamilyStoreCQLHelper.dumpReCreateStatements(metadata))
- out.println(s);
+ SchemaCQLHelper.reCreateStatementsForSchemaCql(metadata(),
+ keyspace.getMetadata().types)
+ .forEach(out::println);
}
}
catch (IOException e)
@@ -1877,7 +1801,8 @@
ephemeralSnapshotMarker.getParentFile().mkdirs();
Files.createFile(ephemeralSnapshotMarker.toPath());
- logger.trace("Created ephemeral snapshot marker file on {}.", ephemeralSnapshotMarker.getAbsolutePath());
+ if (logger.isTraceEnabled())
+ logger.trace("Created ephemeral snapshot marker file on {}.", ephemeralSnapshotMarker.getAbsolutePath());
}
catch (IOException e)
{
@@ -1897,7 +1822,7 @@
}
}
- public Refs<SSTableReader> getSnapshotSSTableReader(String tag) throws IOException
+ public Refs<SSTableReader> getSnapshotSSTableReaders(String tag) throws IOException
{
Map<Integer, SSTableReader> active = new HashMap<>();
for (SSTableReader sstable : getSSTables(SSTableSet.CANONICAL))
@@ -1927,7 +1852,7 @@
}
}
}
- catch (IOException | RuntimeException e)
+ catch (FSReadError | RuntimeException e)
{
// In case one of the snapshot sstables fails to open,
// we must release the references to the ones we opened so far
@@ -1998,7 +1923,7 @@
* @return Return a map of all snapshots to space being used
* The pair for a snapshot has true size and size on disk.
*/
- public Map<String, Pair<Long,Long>> getSnapshotDetails()
+ public Map<String, Directories.SnapshotSizeDetails> getSnapshotDetails()
{
return getDirectories().getSnapshotDetails();
}
@@ -2015,16 +1940,16 @@
{
if (!isRowCacheEnabled())
return null;
- IRowCacheEntry cached = CacheService.instance.rowCache.getInternal(new RowCacheKey(metadata.ksAndCFName, key));
+ IRowCacheEntry cached = CacheService.instance.rowCache.getInternal(new RowCacheKey(metadata(), key));
return cached == null || cached instanceof RowCacheSentinel ? null : (CachedPartition)cached;
}
private void invalidateCaches()
{
- CacheService.instance.invalidateKeyCacheForCf(metadata.ksAndCFName);
- CacheService.instance.invalidateRowCacheForCf(metadata.ksAndCFName);
- if (metadata.isCounter())
- CacheService.instance.invalidateCounterCacheForCf(metadata.ksAndCFName);
+ CacheService.instance.invalidateKeyCacheForCf(metadata());
+ CacheService.instance.invalidateRowCacheForCf(metadata());
+ if (metadata().isCounter())
+ CacheService.instance.invalidateCounterCacheForCf(metadata());
}
public int invalidateRowCache(Collection<Bounds<Token>> boundsToInvalidate)
@@ -2035,7 +1960,7 @@
{
RowCacheKey key = keyIter.next();
DecoratedKey dk = decorateKey(ByteBuffer.wrap(key.key));
- if (key.ksAndCFName.equals(metadata.ksAndCFName) && Bounds.isInBounds(dk.getToken(), boundsToInvalidate))
+ if (key.sameTable(metadata()) && Bounds.isInBounds(dk.getToken(), boundsToInvalidate))
{
invalidateCachedPartition(dk);
invalidatedKeys++;
@@ -2051,8 +1976,8 @@
keyIter.hasNext(); )
{
CounterCacheKey key = keyIter.next();
- DecoratedKey dk = decorateKey(ByteBuffer.wrap(key.partitionKey));
- if (key.ksAndCFName.equals(metadata.ksAndCFName) && Bounds.isInBounds(dk.getToken(), boundsToInvalidate))
+ DecoratedKey dk = decorateKey(key.partitionKey());
+ if (key.sameTable(metadata()) && Bounds.isInBounds(dk.getToken(), boundsToInvalidate))
{
CacheService.instance.counterCache.remove(key);
invalidatedKeys++;
@@ -2066,7 +1991,7 @@
*/
public boolean containsCachedParition(DecoratedKey key)
{
- return CacheService.instance.rowCache.getCapacity() != 0 && CacheService.instance.rowCache.containsKey(new RowCacheKey(metadata.ksAndCFName, key));
+ return CacheService.instance.rowCache.getCapacity() != 0 && CacheService.instance.rowCache.containsKey(new RowCacheKey(metadata(), key));
}
public void invalidateCachedPartition(RowCacheKey key)
@@ -2079,21 +2004,21 @@
if (!isRowCacheEnabled())
return;
- invalidateCachedPartition(new RowCacheKey(metadata.ksAndCFName, key));
+ invalidateCachedPartition(new RowCacheKey(metadata(), key));
}
- public ClockAndCount getCachedCounter(ByteBuffer partitionKey, Clustering clustering, ColumnDefinition column, CellPath path)
+ public ClockAndCount getCachedCounter(ByteBuffer partitionKey, Clustering clustering, ColumnMetadata column, CellPath path)
{
if (CacheService.instance.counterCache.getCapacity() == 0L) // counter cache disabled.
return null;
- return CacheService.instance.counterCache.get(CounterCacheKey.create(metadata.ksAndCFName, partitionKey, clustering, column, path));
+ return CacheService.instance.counterCache.get(CounterCacheKey.create(metadata(), partitionKey, clustering, column, path));
}
- public void putCachedCounter(ByteBuffer partitionKey, Clustering clustering, ColumnDefinition column, CellPath path, ClockAndCount clockAndCount)
+ public void putCachedCounter(ByteBuffer partitionKey, Clustering clustering, ColumnMetadata column, CellPath path, ClockAndCount clockAndCount)
{
if (CacheService.instance.counterCache.getCapacity() == 0L) // counter cache disabled.
return;
- CacheService.instance.counterCache.put(CounterCacheKey.create(metadata.ksAndCFName, partitionKey, clustering, column, path), clockAndCount);
+ CacheService.instance.counterCache.put(CounterCacheKey.create(metadata(), partitionKey, clustering, column, path), clockAndCount);
}
public void forceMajorCompaction() throws InterruptedException, ExecutionException
@@ -2256,29 +2181,47 @@
public <V> V runWithCompactionsDisabled(Callable<V> callable, boolean interruptValidation, boolean interruptViews)
{
+ return runWithCompactionsDisabled(callable, (sstable) -> true, interruptValidation, interruptViews, true);
+ }
+
+ /**
+ * Runs callable with compactions paused and compactions including sstables matching sstablePredicate stopped
+ *
+ * @param callable what to do when compactions are paused
+ * @param sstablesPredicate which sstables should we cancel compactions for
+ * @param interruptValidation if we should interrupt validation compactions
+ * @param interruptViews if we should interrupt view compactions
+ * @param interruptIndexes if we should interrupt compactions on indexes. NOTE: if you set this to true your sstablePredicate
+ * must be able to handle LocalPartitioner sstables!
+ */
+ public <V> V runWithCompactionsDisabled(Callable<V> callable, Predicate<SSTableReader> sstablesPredicate, boolean interruptValidation, boolean interruptViews, boolean interruptIndexes)
+ {
// synchronize so that concurrent invocations don't re-enable compactions partway through unexpectedly,
// and so we only run one major compaction at a time
synchronized (this)
{
- logger.trace("Cancelling in-progress compactions for {}", metadata.cfName);
+ logger.trace("Cancelling in-progress compactions for {}", metadata.name);
+ Iterable<ColumnFamilyStore> toInterruptFor = interruptIndexes
+ ? concatWithIndexes()
+ : Collections.singleton(this);
- Iterable<ColumnFamilyStore> selfWithAuxiliaryCfs = interruptViews
- ? Iterables.concat(concatWithIndexes(), viewManager.allViewsCfs())
- : concatWithIndexes();
+ toInterruptFor = interruptViews
+ ? Iterables.concat(toInterruptFor, viewManager.allViewsCfs())
+ : toInterruptFor;
try (CompactionManager.CompactionPauser pause = CompactionManager.instance.pauseGlobalCompaction();
- CompactionManager.CompactionPauser pausedStrategies = pauseCompactionStrategies(selfWithAuxiliaryCfs))
+ CompactionManager.CompactionPauser pausedStrategies = pauseCompactionStrategies(toInterruptFor))
{
// interrupt in-progress compactions
- CompactionManager.instance.interruptCompactionForCFs(selfWithAuxiliaryCfs, interruptValidation);
- CompactionManager.instance.waitForCessation(selfWithAuxiliaryCfs);
+ CompactionManager.instance.interruptCompactionForCFs(toInterruptFor, sstablesPredicate, interruptValidation);
+ CompactionManager.instance.waitForCessation(toInterruptFor, sstablesPredicate);
// doublecheck that we finished, instead of timing out
- for (ColumnFamilyStore cfs : selfWithAuxiliaryCfs)
+ for (ColumnFamilyStore cfs : toInterruptFor)
{
- if (!cfs.getTracker().getCompacting().isEmpty())
+ if (cfs.getTracker().getCompacting().stream().anyMatch(sstablesPredicate))
{
- logger.warn("Unable to cancel in-progress compactions for {}. Perhaps there is an unusually large row in progress somewhere, or the system is simply overloaded.", metadata.cfName);
+ logger.warn("Unable to cancel in-progress compactions for {}. Perhaps there is an unusually large row in progress somewhere, or the system is simply overloaded.", metadata.name);
return null;
}
}
@@ -2473,14 +2416,14 @@
// End JMX get/set.
- public int getMeanColumns()
+ public int getMeanEstimatedCellPerPartitionCount()
{
long sum = 0;
long count = 0;
for (SSTableReader sstable : getSSTables(SSTableSet.CANONICAL))
{
- long n = sstable.getEstimatedColumnCount().count();
- sum += sstable.getEstimatedColumnCount().mean() * n;
+ long n = sstable.getEstimatedCellPerPartitionCount().count();
+ sum += sstable.getEstimatedCellPerPartitionCount().mean() * n;
count += n;
}
return count > 0 ? (int) (sum / count) : 0;
@@ -2499,6 +2442,19 @@
return count > 0 ? sum * 1.0 / count : 0;
}
+ public int getMeanRowCount()
+ {
+ long totalRows = 0;
+ long totalPartitions = 0;
+ for (SSTableReader sstable : getSSTables(SSTableSet.CANONICAL))
+ {
+ totalPartitions += sstable.getEstimatedPartitionSize().count();
+ totalRows += sstable.getTotalRows();
+ }
+
+ return totalPartitions > 0 ? (int) (totalRows / totalPartitions) : 0;
+ }
+
public long estimateKeys()
{
long n = 0;
@@ -2509,18 +2465,18 @@
public IPartitioner getPartitioner()
{
- return metadata.partitioner;
+ return metadata().partitioner;
}
public DecoratedKey decorateKey(ByteBuffer key)
{
- return metadata.decorateKey(key);
+ return getPartitioner().decorateKey(key);
}
/** true if this CFS contains secondary index data */
public boolean isIndex()
{
- return metadata.isIndex();
+ return metadata().isIndex();
}
public Iterable<ColumnFamilyStore> concatWithIndexes()
@@ -2591,19 +2547,19 @@
public boolean isRowCacheEnabled()
{
- boolean retval = metadata.params.caching.cacheRows() && CacheService.instance.rowCache.getCapacity() > 0;
+ boolean retval = metadata().params.caching.cacheRows() && CacheService.instance.rowCache.getCapacity() > 0;
assert(!retval || !isIndex());
return retval;
}
public boolean isCounterCacheEnabled()
{
- return metadata.isCounter() && CacheService.instance.counterCache.getCapacity() > 0;
+ return metadata().isCounter() && CacheService.instance.counterCache.getCapacity() > 0;
}
public boolean isKeyCacheEnabled()
{
- return metadata.params.caching.cacheKeys() && CacheService.instance.keyCache.getCapacity() > 0;
+ return metadata().params.caching.cacheKeys() && CacheService.instance.keyCache.getCapacity() > 0;
}
/**
@@ -2636,10 +2592,10 @@
long allColumns = 0;
int localTime = (int)(System.currentTimeMillis()/1000);
- for (SSTableReader sstable : getSSTables(SSTableSet.CANONICAL))
+ for (SSTableReader sstable : getSSTables(SSTableSet.LIVE))
{
- allDroppable += sstable.getDroppableTombstonesBefore(localTime - sstable.metadata.params.gcGraceSeconds);
- allColumns += sstable.getEstimatedColumnCount().mean() * sstable.getEstimatedColumnCount().count();
+ allDroppable += sstable.getDroppableTombstonesBefore(localTime - metadata().params.gcGraceSeconds);
+ allColumns += sstable.getEstimatedCellPerPartitionCount().mean() * sstable.getEstimatedCellPerPartitionCount().count();
}
return allColumns > 0 ? allDroppable / allColumns : 0;
}
@@ -2649,27 +2605,23 @@
return getDirectories().trueSnapshotsSize();
}
- @VisibleForTesting
- void resetFileIndexGenerator()
- {
- fileIndexGenerator.set(0);
- }
-
/**
- * Returns a ColumnFamilyStore by cfId if it exists, null otherwise
+ * Returns a ColumnFamilyStore by id if it exists, null otherwise
* Differently from others, this method does not throw exception if the table does not exist.
*/
- public static ColumnFamilyStore getIfExists(UUID cfId)
+ public static ColumnFamilyStore getIfExists(TableId id)
{
- Pair<String, String> kscf = Schema.instance.getCF(cfId);
- if (kscf == null)
+ TableMetadata metadata = Schema.instance.getTableMetadata(id);
+ if (metadata == null)
return null;
- Keyspace keyspace = Keyspace.open(kscf.left);
+ Keyspace keyspace = Keyspace.open(metadata.keyspace);
if (keyspace == null)
return null;
- return keyspace.getColumnFamilyStore(cfId);
+ return keyspace.hasColumnFamilyStore(id)
+ ? keyspace.getColumnFamilyStore(id)
+ : null;
}
/**
@@ -2685,14 +2637,14 @@
if (keyspace == null)
return null;
- UUID id = Schema.instance.getId(ksName, cfName);
- if (id == null)
+ TableMetadata table = Schema.instance.getTableMetadata(ksName, cfName);
+ if (table == null)
return null;
- return keyspace.getColumnFamilyStore(id);
+ return keyspace.getColumnFamilyStore(table.id);
}
- public static TableMetrics metricsFor(UUID tableId)
+ public static TableMetrics metricsFor(TableId tableId)
{
return getIfExists(tableId).metric;
}
@@ -2706,4 +2658,21 @@
{
diskBoundaryManager.invalidate();
}
+
+ @Override
+ public void setNeverPurgeTombstones(boolean value)
+ {
+ if (neverPurgeTombstones != value)
+ logger.info("Changing neverPurgeTombstones for {}.{} from {} to {}", keyspace.getName(), getTableName(), neverPurgeTombstones, value);
+ else
+ logger.info("Not changing neverPurgeTombstones for {}.{}, it is {}", keyspace.getName(), getTableName(), neverPurgeTombstones);
+
+ neverPurgeTombstones = value;
+ }
+
+ @Override
+ public boolean getNeverPurgeTombstones()
+ {
+ return neverPurgeTombstones;
+ }
}
diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStoreCQLHelper.java b/src/java/org/apache/cassandra/db/ColumnFamilyStoreCQLHelper.java
deleted file mode 100644
index 54c8117..0000000
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStoreCQLHelper.java
+++ /dev/null
@@ -1,442 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.db;
-
-import java.nio.ByteBuffer;
-import java.util.*;
-import java.util.concurrent.atomic.*;
-import java.util.function.*;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.Iterables;
-
-import org.apache.cassandra.config.*;
-import org.apache.cassandra.cql3.*;
-import org.apache.cassandra.cql3.statements.*;
-import org.apache.cassandra.db.marshal.*;
-import org.apache.cassandra.schema.*;
-import org.apache.cassandra.utils.*;
-
-/**
- * Helper methods to represent CFMetadata and related objects in CQL format
- */
-public class ColumnFamilyStoreCQLHelper
-{
- public static List<String> dumpReCreateStatements(CFMetaData metadata)
- {
- List<String> l = new ArrayList<>();
- // Types come first, as table can't be created without them
- l.addAll(ColumnFamilyStoreCQLHelper.getUserTypesAsCQL(metadata));
- // Record re-create schema statements
- l.add(ColumnFamilyStoreCQLHelper.getCFMetadataAsCQL(metadata, true));
- // Dropped columns (and re-additions)
- l.addAll(ColumnFamilyStoreCQLHelper.getDroppedColumnsAsCQL(metadata));
- // Indexes applied as last, since otherwise they may interfere with column drops / re-additions
- l.addAll(ColumnFamilyStoreCQLHelper.getIndexesAsCQL(metadata));
- return l;
- }
-
- private static List<ColumnDefinition> getClusteringColumns(CFMetaData metadata)
- {
- List<ColumnDefinition> cds = new ArrayList<>(metadata.clusteringColumns().size());
-
- if (!metadata.isStaticCompactTable())
- for (ColumnDefinition cd : metadata.clusteringColumns())
- cds.add(cd);
-
- return cds;
- }
-
- private static List<ColumnDefinition> getPartitionColumns(CFMetaData metadata)
- {
- List<ColumnDefinition> cds = new ArrayList<>(metadata.partitionColumns().size());
-
- for (ColumnDefinition cd : metadata.partitionColumns().statics)
- cds.add(cd);
-
- if (metadata.isDense())
- {
- // remove an empty type
- for (ColumnDefinition cd : metadata.partitionColumns().withoutStatics())
- if (!cd.type.equals(EmptyType.instance))
- cds.add(cd);
- }
- // "regular" columns are not exposed for static compact tables
- else if (!metadata.isStaticCompactTable())
- {
- for (ColumnDefinition cd : metadata.partitionColumns().withoutStatics())
- cds.add(cd);
- }
-
- return cds;
- }
-
- /**
- * Build a CQL String representation of Column Family Metadata
- */
- @VisibleForTesting
- public static String getCFMetadataAsCQL(CFMetaData metadata, boolean includeDroppedColumns)
- {
- StringBuilder sb = new StringBuilder();
- if (!isCqlCompatible(metadata))
- {
- sb.append(String.format("/*\nWarning: Table %s.%s omitted because it has constructs not compatible with CQL (was created via legacy API).\n",
- metadata.ksName,
- metadata.cfName));
- sb.append("\nApproximate structure, for reference:");
- sb.append("\n(this should not be used to reproduce this schema)\n\n");
- }
-
- sb.append("CREATE TABLE IF NOT EXISTS ");
- sb.append(quoteIdentifier(metadata.ksName)).append('.').append(quoteIdentifier(metadata.cfName)).append(" (");
-
- List<ColumnDefinition> partitionKeyColumns = metadata.partitionKeyColumns();
- List<ColumnDefinition> clusteringColumns = getClusteringColumns(metadata);
- List<ColumnDefinition> partitionColumns = getPartitionColumns(metadata);
-
- Consumer<StringBuilder> cdCommaAppender = commaAppender("\n\t");
- sb.append("\n\t");
- for (ColumnDefinition cfd: partitionKeyColumns)
- {
- cdCommaAppender.accept(sb);
- sb.append(toCQL(cfd));
- if (partitionKeyColumns.size() == 1 && clusteringColumns.size() == 0)
- sb.append(" PRIMARY KEY");
- }
-
- for (ColumnDefinition cfd: clusteringColumns)
- {
- cdCommaAppender.accept(sb);
- sb.append(toCQL(cfd));
- }
-
- for (ColumnDefinition cfd: partitionColumns)
- {
- cdCommaAppender.accept(sb);
- sb.append(toCQL(cfd, metadata.isStaticCompactTable()));
- }
-
- if (includeDroppedColumns)
- {
- for (Map.Entry<ByteBuffer, CFMetaData.DroppedColumn> entry: metadata.getDroppedColumns().entrySet())
- {
- if (metadata.getColumnDefinition(entry.getKey()) != null)
- continue;
-
- CFMetaData.DroppedColumn droppedColumn = entry.getValue();
- cdCommaAppender.accept(sb);
- sb.append(quoteIdentifier(droppedColumn.name));
- sb.append(' ');
- sb.append(droppedColumn.type.asCQL3Type().toString());
- }
- }
-
- if (clusteringColumns.size() > 0 || partitionKeyColumns.size() > 1)
- {
- sb.append(",\n\tPRIMARY KEY (");
- if (partitionKeyColumns.size() > 1)
- {
- sb.append("(");
- Consumer<StringBuilder> pkCommaAppender = commaAppender(" ");
- for (ColumnDefinition cfd : partitionKeyColumns)
- {
- pkCommaAppender.accept(sb);
- sb.append(quoteIdentifier(cfd.name.toString()));
- }
- sb.append(")");
- }
- else
- {
- sb.append(quoteIdentifier(partitionKeyColumns.get(0).name.toString()));
- }
-
- for (ColumnDefinition cfd : metadata.clusteringColumns())
- sb.append(", ").append(quoteIdentifier(cfd.name.toString()));
-
- sb.append(')');
- }
- sb.append(")\n\t");
- sb.append("WITH ");
-
- sb.append("ID = ").append(metadata.cfId).append("\n\tAND ");
-
- if (metadata.isCompactTable())
- sb.append("COMPACT STORAGE\n\tAND ");
-
- if (clusteringColumns.size() > 0)
- {
- sb.append("CLUSTERING ORDER BY (");
-
- Consumer<StringBuilder> cOrderCommaAppender = commaAppender(" ");
- for (ColumnDefinition cd : clusteringColumns)
- {
- cOrderCommaAppender.accept(sb);
- sb.append(quoteIdentifier(cd.name.toString())).append(' ').append(cd.clusteringOrder().toString());
- }
- sb.append(")\n\tAND ");
- }
-
- sb.append(toCQL(metadata.params));
- sb.append(";");
-
- if (!isCqlCompatible(metadata))
- {
- sb.append("\n*/");
- }
- return sb.toString();
- }
-
- /**
- * Build a CQL String representation of User Types used in the given Column Family.
- *
- * Type order is ensured as types are built incrementally: from the innermost (most nested)
- * to the outermost.
- */
- @VisibleForTesting
- public static List<String> getUserTypesAsCQL(CFMetaData metadata)
- {
- List<AbstractType> types = new ArrayList<>();
- Set<AbstractType> typeSet = new HashSet<>();
- for (ColumnDefinition cd: Iterables.concat(metadata.partitionKeyColumns(), metadata.clusteringColumns(), metadata.partitionColumns()))
- {
- AbstractType type = cd.type;
- if (type.isUDT())
- resolveUserType((UserType) type, typeSet, types);
- }
-
- List<String> typeStrings = new ArrayList<>();
- for (AbstractType type: types)
- typeStrings.add(toCQL((UserType) type));
- return typeStrings;
- }
-
- /**
- * Build a CQL String representation of Dropped Columns in the given Column Family.
- *
- * If the column was dropped once, but is now re-created `ADD` will be appended accordingly.
- */
- @VisibleForTesting
- public static List<String> getDroppedColumnsAsCQL(CFMetaData metadata)
- {
- List<String> droppedColumns = new ArrayList<>();
-
- for (Map.Entry<ByteBuffer, CFMetaData.DroppedColumn> entry: metadata.getDroppedColumns().entrySet())
- {
- CFMetaData.DroppedColumn column = entry.getValue();
- droppedColumns.add(toCQLDrop(metadata.ksName, metadata.cfName, column));
- if (metadata.getColumnDefinition(entry.getKey()) != null)
- droppedColumns.add(toCQLAdd(metadata.ksName, metadata.cfName, metadata.getColumnDefinition(entry.getKey())));
- }
-
- return droppedColumns;
- }
-
- /**
- * Build a CQL String representation of Indexes on columns in the given Column Family
- */
- @VisibleForTesting
- public static List<String> getIndexesAsCQL(CFMetaData metadata)
- {
- List<String> indexes = new ArrayList<>();
- for (IndexMetadata indexMetadata: metadata.getIndexes())
- indexes.add(toCQL(metadata.ksName, metadata.cfName, indexMetadata));
- return indexes;
- }
-
- private static String toCQL(String keyspace, String cf, IndexMetadata indexMetadata)
- {
- if (indexMetadata.isCustom())
- {
- Map<String, String> options = new HashMap<>();
- indexMetadata.options.forEach((k, v) -> {
- if (!k.equals(IndexTarget.TARGET_OPTION_NAME) && !k.equals(IndexTarget.CUSTOM_INDEX_OPTION_NAME))
- options.put(k, v);
- });
-
- return String.format("CREATE CUSTOM INDEX %s ON %s.%s (%s) USING '%s'%s;",
- quoteIdentifier(indexMetadata.name),
- quoteIdentifier(keyspace),
- quoteIdentifier(cf),
- indexMetadata.options.get(IndexTarget.TARGET_OPTION_NAME),
- indexMetadata.options.get(IndexTarget.CUSTOM_INDEX_OPTION_NAME),
- options.isEmpty() ? "" : " WITH OPTIONS " + toCQL(options));
- }
- else
- {
- return String.format("CREATE INDEX %s ON %s.%s (%s);",
- quoteIdentifier(indexMetadata.name),
- quoteIdentifier(keyspace),
- quoteIdentifier(cf),
- indexMetadata.options.get(IndexTarget.TARGET_OPTION_NAME));
- }
- }
- private static String toCQL(UserType userType)
- {
- StringBuilder sb = new StringBuilder();
- sb.append(String.format("CREATE TYPE %s.%s(",
- quoteIdentifier(userType.keyspace),
- quoteIdentifier(userType.getNameAsString())));
-
- Consumer<StringBuilder> commaAppender = commaAppender(" ");
- for (int i = 0; i < userType.size(); i++)
- {
- commaAppender.accept(sb);
- sb.append(String.format("%s %s",
- userType.fieldNameAsString(i),
- userType.fieldType(i).asCQL3Type()));
- }
- sb.append(");");
- return sb.toString();
- }
-
- private static String toCQL(TableParams tableParams)
- {
- StringBuilder builder = new StringBuilder();
-
- builder.append("bloom_filter_fp_chance = ").append(tableParams.bloomFilterFpChance);
- builder.append("\n\tAND dclocal_read_repair_chance = ").append(tableParams.dcLocalReadRepairChance);
- builder.append("\n\tAND crc_check_chance = ").append(tableParams.crcCheckChance);
- builder.append("\n\tAND default_time_to_live = ").append(tableParams.defaultTimeToLive);
- builder.append("\n\tAND gc_grace_seconds = ").append(tableParams.gcGraceSeconds);
- builder.append("\n\tAND min_index_interval = ").append(tableParams.minIndexInterval);
- builder.append("\n\tAND max_index_interval = ").append(tableParams.maxIndexInterval);
- builder.append("\n\tAND memtable_flush_period_in_ms = ").append(tableParams.memtableFlushPeriodInMs);
- builder.append("\n\tAND read_repair_chance = ").append(tableParams.readRepairChance);
- builder.append("\n\tAND speculative_retry = '").append(tableParams.speculativeRetry).append("'");
- builder.append("\n\tAND comment = ").append(singleQuote(tableParams.comment));
- builder.append("\n\tAND caching = ").append(toCQL(tableParams.caching.asMap()));
- builder.append("\n\tAND compaction = ").append(toCQL(tableParams.compaction.asMap()));
- builder.append("\n\tAND compression = ").append(toCQL(tableParams.compression.asMap()));
- builder.append("\n\tAND cdc = ").append(tableParams.cdc);
-
- builder.append("\n\tAND extensions = { ");
- for (Map.Entry<String, ByteBuffer> entry : tableParams.extensions.entrySet())
- {
- builder.append(singleQuote(entry.getKey()));
- builder.append(": ");
- builder.append("0x").append(ByteBufferUtil.bytesToHex(entry.getValue()));
- }
- builder.append(" }");
- return builder.toString();
- }
-
- private static String toCQL(Map<?, ?> map)
- {
- StringBuilder builder = new StringBuilder("{ ");
-
- boolean isFirst = true;
- for (Map.Entry entry: map.entrySet())
- {
- if (isFirst)
- isFirst = false;
- else
- builder.append(", ");
- builder.append(singleQuote(entry.getKey().toString()));
- builder.append(": ");
- builder.append(singleQuote(entry.getValue().toString()));
- }
-
- builder.append(" }");
- return builder.toString();
- }
-
- private static String toCQL(ColumnDefinition cd)
- {
- return toCQL(cd, false);
- }
-
- private static String toCQL(ColumnDefinition cd, boolean isStaticCompactTable)
- {
- return String.format("%s %s%s",
- quoteIdentifier(cd.name.toString()),
- cd.type.asCQL3Type().toString(),
- cd.isStatic() && !isStaticCompactTable ? " static" : "");
- }
-
- private static String toCQLAdd(String keyspace, String cf, ColumnDefinition cd)
- {
- return String.format("ALTER TABLE %s.%s ADD %s %s%s;",
- quoteIdentifier(keyspace),
- quoteIdentifier(cf),
- quoteIdentifier(cd.name.toString()),
- cd.type.asCQL3Type().toString(),
- cd.isStatic() ? " static" : "");
- }
-
- private static String toCQLDrop(String keyspace, String cf, CFMetaData.DroppedColumn droppedColumn)
- {
- return String.format("ALTER TABLE %s.%s DROP %s USING TIMESTAMP %s;",
- quoteIdentifier(keyspace),
- quoteIdentifier(cf),
- quoteIdentifier(droppedColumn.name),
- droppedColumn.droppedTime);
- }
-
- private static void resolveUserType(UserType type, Set<AbstractType> typeSet, List<AbstractType> types)
- {
- for (AbstractType subType: type.fieldTypes())
- if (!typeSet.contains(subType) && subType.isUDT())
- resolveUserType((UserType) subType, typeSet, types);
-
- if (!typeSet.contains(type))
- {
- typeSet.add(type);
- types.add(type);
- }
- }
-
- private static String singleQuote(String s)
- {
- return String.format("'%s'", s.replaceAll("'", "''"));
- }
-
- private static Consumer<StringBuilder> commaAppender(String afterComma)
- {
- AtomicBoolean isFirst = new AtomicBoolean(true);
- return new Consumer<StringBuilder>()
- {
- public void accept(StringBuilder stringBuilder)
- {
- if (!isFirst.getAndSet(false))
- stringBuilder.append(',').append(afterComma);
- }
- };
- }
-
- private static String quoteIdentifier(String id)
- {
- return ColumnIdentifier.maybeQuote(id);
- }
-
- /**
- * Whether or not the given metadata is compatible / representable with CQL Language
- */
- public static boolean isCqlCompatible(CFMetaData metaData)
- {
- if (metaData.isSuper())
- return false;
-
- if (metaData.isCompactTable()
- && metaData.partitionColumns().withoutStatics().size() > 1
- && metaData.clusteringColumns().size() >= 1)
- return false;
-
- return true;
- }
-}
diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java b/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java
index d788e2e..fb0d611 100644
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java
@@ -20,6 +20,7 @@
import java.util.Collection;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.concurrent.ExecutionException;
import javax.management.openmbean.CompositeData;
@@ -102,11 +103,14 @@
*/
public Map<String,String> getCompressionParameters();
+ public String getCompressionParametersJson();
+
/**
- * Set the compression parameters
+ * Set the compression parameters locally for this node
* @param opts map of string names to values
*/
public void setCompressionParameters(Map<String,String> opts);
+ public void setCompressionParametersJson(String options);
/**
* Set new crc check chance
@@ -140,11 +144,29 @@
public List<String> getSSTablesForKey(String key, boolean hexFormat);
/**
- * Scan through Keyspace/ColumnFamily's data directory
- * determine which SSTables should be loaded and load them
+ * Load new sstables from the given directory
+ *
+ * @param srcPaths the path to the new sstables - if it is an empty set, the data directories will be scanned
+ * @param resetLevel if the level should be reset to 0 on the new sstables
+ * @param clearRepaired if repaired info should be wiped from the new sstables
+ * @param verifySSTables if the new sstables should be verified that they are not corrupt
+ * @param verifyTokens if the tokens in the new sstables should be verified that they are owned by the current node
+ * @param invalidateCaches if row cache should be invalidated for the keys in the new sstables
+ * @param jbodCheck if the new sstables should be placed 'optimally' - count tokens and move the sstable to the directory where it has the most keys
+ * @param extendedVerify if we should run an extended verify checking all values in the new sstables
+ *
+ * @return list of failed import directories
*/
- public void loadNewSSTables();
+ public List<String> importNewSSTables(Set<String> srcPaths,
+ boolean resetLevel,
+ boolean clearRepaired,
+ boolean verifySSTables,
+ boolean verifyTokens,
+ boolean invalidateCaches,
+ boolean extendedVerify);
+ @Deprecated
+ public void loadNewSSTables();
/**
* @return the number of SSTables in L0. Always return 0 if Leveled compaction is not enabled.
*/
@@ -176,12 +198,12 @@
* begin sampling for a specific sampler with a given capacity. The cardinality may
* be larger than the capacity, but depending on the use case it may affect its accuracy
*/
- public void beginLocalSampling(String sampler, int capacity);
+ public void beginLocalSampling(String sampler, int capacity, int durationMillis);
/**
* @return top <i>count</i> items for the sampler since beginLocalSampling was called
*/
- public CompositeData finishLocalSampling(String sampler, int count) throws OpenDataException;
+ public List<CompositeData> finishLocalSampling(String sampler, int count) throws OpenDataException;
/*
Is Compaction space check enabled
@@ -192,4 +214,8 @@
Enable/Disable compaction space check
*/
public void compactionDiskSpaceCheck(boolean enable);
+
+ public void setNeverPurgeTombstones(boolean value);
+
+ public boolean getNeverPurgeTombstones();
}
diff --git a/src/java/org/apache/cassandra/db/ColumnIndex.java b/src/java/org/apache/cassandra/db/ColumnIndex.java
index de1b1df..e11f784 100644
--- a/src/java/org/apache/cassandra/db/ColumnIndex.java
+++ b/src/java/org/apache/cassandra/db/ColumnIndex.java
@@ -53,6 +53,7 @@
public int columnIndexCount;
private int[] indexOffsets;
+ private final SerializationHelper helper;
private final SerializationHeader header;
private final int version;
private final SequentialWriter writer;
@@ -69,6 +70,8 @@
private DeletionTime openMarker;
+ private int cacheSizeThreshold;
+
private final Collection<SSTableFlushObserver> observers;
public ColumnIndex(SerializationHeader header,
@@ -77,6 +80,7 @@
Collection<SSTableFlushObserver> observers,
ISerializer<IndexInfo> indexInfoSerializer)
{
+ this.helper = new SerializationHelper(header);
this.header = header;
this.writer = writer;
this.version = version.correspondingMessagingVersion();
@@ -97,9 +101,12 @@
this.firstClustering = null;
this.lastClustering = null;
this.openMarker = null;
- if (this.buffer != null)
+
+ int newCacheSizeThreshold = DatabaseDescriptor.getColumnIndexCacheSize();
+ if (this.buffer != null && this.cacheSizeThreshold == newCacheSizeThreshold)
this.reusableBuffer = this.buffer;
this.buffer = null;
+ this.cacheSizeThreshold = newCacheSizeThreshold;
}
public void buildRowIndex(UnfilteredRowIterator iterator) throws IOException
@@ -121,7 +128,7 @@
{
Row staticRow = iterator.staticRow();
- UnfilteredSerializer.serializer.serializeStaticRow(staticRow, header, writer, version);
+ UnfilteredSerializer.serializer.serializeStaticRow(staticRow, helper, writer, version);
if (!observers.isEmpty())
observers.forEach((o) -> o.nextUnfilteredCluster(staticRow));
}
@@ -139,7 +146,7 @@
public List<IndexInfo> indexSamples()
{
- if (indexSamplesSerializedSize + columnIndexCount * TypeSizes.sizeof(0) <= DatabaseDescriptor.getColumnIndexCacheSize())
+ if (indexSamplesSerializedSize + columnIndexCount * TypeSizes.sizeof(0) <= cacheSizeThreshold)
{
return indexSamples;
}
@@ -197,7 +204,7 @@
if (buffer == null)
{
indexSamplesSerializedSize += idxSerializer.serializedSize(cIndexInfo);
- if (indexSamplesSerializedSize + columnIndexCount * TypeSizes.sizeof(0) > DatabaseDescriptor.getColumnIndexCacheSize())
+ if (indexSamplesSerializedSize + columnIndexCount * TypeSizes.sizeof(0) > cacheSizeThreshold)
{
buffer = reuseOrAllocateBuffer();
for (IndexInfo indexSample : indexSamples)
@@ -210,7 +217,7 @@
indexSamples.add(cIndexInfo);
}
}
- // don't put an else here...
+ // don't put an else here since buffer may be allocated in preceding if block
if (buffer != null)
{
idxSerializer.serialize(cIndexInfo, buffer);
@@ -229,7 +236,7 @@
return buffer;
}
// don't use the standard RECYCLER as that only recycles up to 1MB and requires proper cleanup
- return new DataOutputBuffer(DatabaseDescriptor.getColumnIndexCacheSize() * 2);
+ return new DataOutputBuffer(cacheSizeThreshold * 2);
}
private void add(Unfiltered unfiltered) throws IOException
@@ -243,7 +250,7 @@
startPosition = pos;
}
- UnfilteredSerializer.serializer.serialize(unfiltered, header, writer, pos - previousRowStart, version);
+ UnfilteredSerializer.serializer.serialize(unfiltered, helper, writer, pos - previousRowStart, version);
// notify observers about each new row
if (!observers.isEmpty())
diff --git a/src/java/org/apache/cassandra/db/Columns.java b/src/java/org/apache/cassandra/db/Columns.java
index 5a0ad29..fe13919 100644
--- a/src/java/org/apache/cassandra/db/Columns.java
+++ b/src/java/org/apache/cassandra/db/Columns.java
@@ -22,14 +22,14 @@
import java.util.function.Consumer;
import java.util.function.Predicate;
import java.nio.ByteBuffer;
-import java.security.MessageDigest;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Iterators;
import net.nicoulaj.compilecommand.annotations.DontInline;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.exceptions.UnknownColumnException;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.cql3.ColumnIdentifier;
import org.apache.cassandra.db.marshal.SetType;
import org.apache.cassandra.db.marshal.UTF8Type;
@@ -46,28 +46,28 @@
* An immutable and sorted list of (non-PK) columns for a given table.
* <p>
* Note that in practice, it will either store only static columns, or only regular ones. When
- * we need both type of columns, we use a {@link PartitionColumns} object.
+ * we need both type of columns, we use a {@link RegularAndStaticColumns} object.
*/
-public class Columns extends AbstractCollection<ColumnDefinition> implements Collection<ColumnDefinition>
+public class Columns extends AbstractCollection<ColumnMetadata> implements Collection<ColumnMetadata>
{
public static final Serializer serializer = new Serializer();
public static final Columns NONE = new Columns(BTree.empty(), 0);
- private static final ColumnDefinition FIRST_COMPLEX_STATIC =
- new ColumnDefinition("",
- "",
- ColumnIdentifier.getInterned(ByteBufferUtil.EMPTY_BYTE_BUFFER, UTF8Type.instance),
- SetType.getInstance(UTF8Type.instance, true),
- ColumnDefinition.NO_POSITION,
- ColumnDefinition.Kind.STATIC);
+ public static final ColumnMetadata FIRST_COMPLEX_STATIC =
+ new ColumnMetadata("",
+ "",
+ ColumnIdentifier.getInterned(ByteBufferUtil.EMPTY_BYTE_BUFFER, UTF8Type.instance),
+ SetType.getInstance(UTF8Type.instance, true),
+ ColumnMetadata.NO_POSITION,
+ ColumnMetadata.Kind.STATIC);
- private static final ColumnDefinition FIRST_COMPLEX_REGULAR =
- new ColumnDefinition("",
- "",
- ColumnIdentifier.getInterned(ByteBufferUtil.EMPTY_BYTE_BUFFER, UTF8Type.instance),
- SetType.getInstance(UTF8Type.instance, true),
- ColumnDefinition.NO_POSITION,
- ColumnDefinition.Kind.REGULAR);
+ public static final ColumnMetadata FIRST_COMPLEX_REGULAR =
+ new ColumnMetadata("",
+ "",
+ ColumnIdentifier.getInterned(ByteBufferUtil.EMPTY_BYTE_BUFFER, UTF8Type.instance),
+ SetType.getInstance(UTF8Type.instance, true),
+ ColumnMetadata.NO_POSITION,
+ ColumnMetadata.Kind.REGULAR);
private final Object[] columns;
private final int complexIdx; // Index of the first complex column
@@ -91,7 +91,7 @@
*
* @return the newly created {@code Columns} containing only {@code c}.
*/
- public static Columns of(ColumnDefinition c)
+ public static Columns of(ColumnMetadata c)
{
return new Columns(BTree.singleton(c), c.isComplex() ? 0 : 1);
}
@@ -102,9 +102,9 @@
* @param s the set from which to create the new {@code Columns}.
* @return the newly created {@code Columns} containing the columns from {@code s}.
*/
- public static Columns from(Collection<ColumnDefinition> s)
+ public static Columns from(Collection<ColumnMetadata> s)
{
- Object[] tree = BTree.<ColumnDefinition>builder(Comparator.naturalOrder()).addAll(s).build();
+ Object[] tree = BTree.<ColumnMetadata>builder(Comparator.naturalOrder()).addAll(s).build();
return new Columns(tree, findFirstComplexIdx(tree));
}
@@ -114,7 +114,7 @@
return 0;
int size = BTree.size(tree);
- ColumnDefinition last = BTree.findByIndex(tree, size - 1);
+ ColumnMetadata last = BTree.findByIndex(tree, size - 1);
return last.isSimple()
? size
: BTree.ceilIndex(tree, Comparator.naturalOrder(), last.isStatic() ? FIRST_COMPLEX_STATIC : FIRST_COMPLEX_REGULAR);
@@ -188,7 +188,7 @@
*
* @return the {@code i}th simple column in this object.
*/
- public ColumnDefinition getSimple(int i)
+ public ColumnMetadata getSimple(int i)
{
return BTree.findByIndex(columns, i);
}
@@ -201,7 +201,7 @@
*
* @return the {@code i}th complex column in this object.
*/
- public ColumnDefinition getComplex(int i)
+ public ColumnMetadata getComplex(int i)
{
return BTree.findByIndex(columns, complexIdx + i);
}
@@ -215,7 +215,7 @@
* @return the index for simple column {@code c} if it is contains in this
* object
*/
- public int simpleIdx(ColumnDefinition c)
+ public int simpleIdx(ColumnMetadata c)
{
return BTree.findIndex(columns, Comparator.naturalOrder(), c);
}
@@ -229,7 +229,7 @@
* @return the index for complex column {@code c} if it is contains in this
* object
*/
- public int complexIdx(ColumnDefinition c)
+ public int complexIdx(ColumnMetadata c)
{
return BTree.findIndex(columns, Comparator.naturalOrder(), c) - complexIdx;
}
@@ -241,7 +241,7 @@
*
* @return whether {@code c} is contained by this object.
*/
- public boolean contains(ColumnDefinition c)
+ public boolean contains(ColumnMetadata c)
{
return BTree.findIndex(columns, Comparator.naturalOrder(), c) >= 0;
}
@@ -263,8 +263,8 @@
if (this == NONE)
return other;
- Object[] tree = BTree.<ColumnDefinition>merge(this.columns, other.columns, Comparator.naturalOrder(),
- UpdateFunction.noOp());
+ Object[] tree = BTree.<ColumnMetadata>merge(this.columns, other.columns, Comparator.naturalOrder(),
+ UpdateFunction.noOp());
if (tree == this.columns)
return this;
if (tree == other.columns)
@@ -287,9 +287,9 @@
if (other.size() > this.size())
return false;
- BTreeSearchIterator<ColumnDefinition, ColumnDefinition> iter = BTree.slice(columns, Comparator.naturalOrder(), BTree.Dir.ASC);
+ BTreeSearchIterator<ColumnMetadata, ColumnMetadata> iter = BTree.slice(columns, Comparator.naturalOrder(), BTree.Dir.ASC);
for (Object def : other)
- if (iter.next((ColumnDefinition) def) == null)
+ if (iter.next((ColumnMetadata) def) == null)
return false;
return true;
}
@@ -299,7 +299,7 @@
*
* @return an iterator over the simple columns of this object.
*/
- public Iterator<ColumnDefinition> simpleColumns()
+ public Iterator<ColumnMetadata> simpleColumns()
{
return BTree.iterator(columns, 0, complexIdx - 1, BTree.Dir.ASC);
}
@@ -309,7 +309,7 @@
*
* @return an iterator over the complex columns of this object.
*/
- public Iterator<ColumnDefinition> complexColumns()
+ public Iterator<ColumnMetadata> complexColumns()
{
return BTree.iterator(columns, complexIdx, BTree.size(columns) - 1, BTree.Dir.ASC);
}
@@ -319,9 +319,9 @@
*
* @return an iterator over all the columns of this object.
*/
- public BTreeSearchIterator<ColumnDefinition, ColumnDefinition> iterator()
+ public BTreeSearchIterator<ColumnMetadata, ColumnMetadata> iterator()
{
- return BTree.<ColumnDefinition, ColumnDefinition>slice(columns, Comparator.naturalOrder(), BTree.Dir.ASC);
+ return BTree.<ColumnMetadata, ColumnMetadata>slice(columns, Comparator.naturalOrder(), BTree.Dir.ASC);
}
/**
@@ -331,11 +331,11 @@
*
* @return an iterator returning columns in alphabetical order.
*/
- public Iterator<ColumnDefinition> selectOrderIterator()
+ public Iterator<ColumnMetadata> selectOrderIterator()
{
// In wildcard selection, we want to return all columns in alphabetical order,
// irregarding of whether they are complex or not
- return Iterators.<ColumnDefinition>
+ return Iterators.<ColumnMetadata>
mergeSorted(ImmutableList.of(simpleColumns(), complexColumns()),
(s, c) ->
{
@@ -352,12 +352,12 @@
* @return newly allocated columns containing all the columns of {@code this} expect
* for {@code column}.
*/
- public Columns without(ColumnDefinition column)
+ public Columns without(ColumnMetadata column)
{
if (!contains(column))
return this;
- Object[] newColumns = BTreeRemoval.<ColumnDefinition>remove(columns, Comparator.naturalOrder(), column);
+ Object[] newColumns = BTreeRemoval.<ColumnMetadata>remove(columns, Comparator.naturalOrder(), column);
return new Columns(newColumns);
}
@@ -367,33 +367,25 @@
*
* @return a predicate to test the inclusion of sorted columns in this object.
*/
- public Predicate<ColumnDefinition> inOrderInclusionTester()
+ public Predicate<ColumnMetadata> inOrderInclusionTester()
{
- SearchIterator<ColumnDefinition, ColumnDefinition> iter = BTree.slice(columns, Comparator.naturalOrder(), BTree.Dir.ASC);
+ SearchIterator<ColumnMetadata, ColumnMetadata> iter = BTree.slice(columns, Comparator.naturalOrder(), BTree.Dir.ASC);
return column -> iter.next(column) != null;
}
- public void digest(MessageDigest digest)
+ public void digest(Digest digest)
{
- for (ColumnDefinition c : this)
- digest.update(c.name.bytes.duplicate());
- }
-
- public void digest(MessageDigest digest, Set<ByteBuffer> columnsToExclude)
- {
- for (ColumnDefinition c : this)
- if (!columnsToExclude.contains(c.name.bytes))
- digest.update(c.name.bytes.duplicate());
+ for (ColumnMetadata c : this)
+ digest.update(c.name.bytes);
}
/**
* Apply a function to each column definition in forwards or reversed order.
* @param function
- * @param reversed
*/
- public void apply(Consumer<ColumnDefinition> function, boolean reversed)
+ public void apply(Consumer<ColumnMetadata> function)
{
- BTree.apply(columns, function, reversed);
+ BTree.apply(columns, function);
}
@Override
@@ -419,7 +411,7 @@
{
StringBuilder sb = new StringBuilder("[");
boolean first = true;
- for (ColumnDefinition def : this)
+ for (ColumnMetadata def : this)
{
if (first) first = false; else sb.append(" ");
sb.append(def.name);
@@ -432,36 +424,35 @@
public void serialize(Columns columns, DataOutputPlus out) throws IOException
{
out.writeUnsignedVInt(columns.size());
- for (ColumnDefinition column : columns)
+ for (ColumnMetadata column : columns)
ByteBufferUtil.writeWithVIntLength(column.name.bytes, out);
}
public long serializedSize(Columns columns)
{
long size = TypeSizes.sizeofUnsignedVInt(columns.size());
- for (ColumnDefinition column : columns)
+ for (ColumnMetadata column : columns)
size += ByteBufferUtil.serializedSizeWithVIntLength(column.name.bytes);
return size;
}
- public Columns deserialize(DataInputPlus in, CFMetaData metadata) throws IOException
+ public Columns deserialize(DataInputPlus in, TableMetadata metadata) throws IOException
{
int length = (int)in.readUnsignedVInt();
- BTree.Builder<ColumnDefinition> builder = BTree.builder(Comparator.naturalOrder());
+ BTree.Builder<ColumnMetadata> builder = BTree.builder(Comparator.naturalOrder());
builder.auto(false);
for (int i = 0; i < length; i++)
{
ByteBuffer name = ByteBufferUtil.readWithVIntLength(in);
- ColumnDefinition column = metadata.getColumnDefinition(name);
-
+ ColumnMetadata column = metadata.getColumn(name);
if (column == null)
{
// If we don't find the definition, it could be we have data for a dropped column, and we shouldn't
- // fail deserialization because of that. So we grab a "fake" ColumnDefinition that ensure proper
+ // fail deserialization because of that. So we grab a "fake" ColumnMetadata that ensure proper
// deserialization. The column will be ignore later on anyway.
- column = metadata.getDroppedColumnDefinition(name);
+ column = metadata.getDroppedColumn(name);
if (column == null)
- throw new RuntimeException("Unknown column " + UTF8Type.instance.getString(name) + " during deserialization");
+ throw new UnknownColumnException("Unknown column " + UTF8Type.instance.getString(name) + " during deserialization");
}
builder.add(column);
}
@@ -472,7 +463,7 @@
* If both ends have a pre-shared superset of the columns we are serializing, we can send them much
* more efficiently. Both ends must provide the identically same set of columns.
*/
- public void serializeSubset(Collection<ColumnDefinition> columns, Columns superset, DataOutputPlus out) throws IOException
+ public void serializeSubset(Collection<ColumnMetadata> columns, Columns superset, DataOutputPlus out) throws IOException
{
/**
* We weight this towards small sets, and sets where the majority of items are present, since
@@ -502,7 +493,7 @@
}
}
- public long serializedSubsetSize(Collection<ColumnDefinition> columns, Columns superset)
+ public long serializedSubsetSize(Collection<ColumnMetadata> columns, Columns superset)
{
int columnCount = columns.size();
int supersetCount = superset.size();
@@ -533,9 +524,9 @@
}
else
{
- BTree.Builder<ColumnDefinition> builder = BTree.builder(Comparator.naturalOrder());
+ BTree.Builder<ColumnMetadata> builder = BTree.builder(Comparator.naturalOrder());
int firstComplexIdx = 0;
- for (ColumnDefinition column : superset)
+ for (ColumnMetadata column : superset)
{
if ((encoded & 1) == 0)
{
@@ -553,13 +544,13 @@
// encodes a 1 bit for every *missing* column, on the assumption presence is more common,
// and because this is consistent with encoding 0 to represent all present
- private static long encodeBitmap(Collection<ColumnDefinition> columns, Columns superset, int supersetCount)
+ private static long encodeBitmap(Collection<ColumnMetadata> columns, Columns superset, int supersetCount)
{
long bitmap = 0L;
- BTreeSearchIterator<ColumnDefinition, ColumnDefinition> iter = superset.iterator();
+ BTreeSearchIterator<ColumnMetadata, ColumnMetadata> iter = superset.iterator();
// the index we would encounter next if all columns are present
int expectIndex = 0;
- for (ColumnDefinition column : columns)
+ for (ColumnMetadata column : columns)
{
if (iter.next(column) == null)
throw new IllegalStateException(columns + " is not a subset of " + superset);
@@ -578,15 +569,15 @@
}
@DontInline
- private void serializeLargeSubset(Collection<ColumnDefinition> columns, int columnCount, Columns superset, int supersetCount, DataOutputPlus out) throws IOException
+ private void serializeLargeSubset(Collection<ColumnMetadata> columns, int columnCount, Columns superset, int supersetCount, DataOutputPlus out) throws IOException
{
// write flag indicating we're in lengthy mode
out.writeUnsignedVInt(supersetCount - columnCount);
- BTreeSearchIterator<ColumnDefinition, ColumnDefinition> iter = superset.iterator();
+ BTreeSearchIterator<ColumnMetadata, ColumnMetadata> iter = superset.iterator();
if (columnCount < supersetCount / 2)
{
// write present columns
- for (ColumnDefinition column : columns)
+ for (ColumnMetadata column : columns)
{
if (iter.next(column) == null)
throw new IllegalStateException();
@@ -597,7 +588,7 @@
{
// write missing columns
int prev = -1;
- for (ColumnDefinition column : columns)
+ for (ColumnMetadata column : columns)
{
if (iter.next(column) == null)
throw new IllegalStateException();
@@ -616,7 +607,7 @@
int supersetCount = superset.size();
int columnCount = supersetCount - delta;
- BTree.Builder<ColumnDefinition> builder = BTree.builder(Comparator.naturalOrder());
+ BTree.Builder<ColumnMetadata> builder = BTree.builder(Comparator.naturalOrder());
if (columnCount < supersetCount / 2)
{
for (int i = 0 ; i < columnCount ; i++)
@@ -627,7 +618,7 @@
}
else
{
- Iterator<ColumnDefinition> iter = superset.iterator();
+ Iterator<ColumnMetadata> iter = superset.iterator();
int idx = 0;
int skipped = 0;
while (true)
@@ -635,7 +626,7 @@
int nextMissingIndex = skipped < delta ? (int)in.readUnsignedVInt() : supersetCount;
while (idx < nextMissingIndex)
{
- ColumnDefinition def = iter.next();
+ ColumnMetadata def = iter.next();
builder.add(def);
idx++;
}
@@ -650,15 +641,15 @@
}
@DontInline
- private int serializeLargeSubsetSize(Collection<ColumnDefinition> columns, int columnCount, Columns superset, int supersetCount)
+ private int serializeLargeSubsetSize(Collection<ColumnMetadata> columns, int columnCount, Columns superset, int supersetCount)
{
// write flag indicating we're in lengthy mode
int size = TypeSizes.sizeofUnsignedVInt(supersetCount - columnCount);
- BTreeSearchIterator<ColumnDefinition, ColumnDefinition> iter = superset.iterator();
+ BTreeSearchIterator<ColumnMetadata, ColumnMetadata> iter = superset.iterator();
if (columnCount < supersetCount / 2)
{
// write present columns
- for (ColumnDefinition column : columns)
+ for (ColumnMetadata column : columns)
{
if (iter.next(column) == null)
throw new IllegalStateException();
@@ -669,7 +660,7 @@
{
// write missing columns
int prev = -1;
- for (ColumnDefinition column : columns)
+ for (ColumnMetadata column : columns)
{
if (iter.next(column) == null)
throw new IllegalStateException();
diff --git a/src/java/org/apache/cassandra/db/CompactTables.java b/src/java/org/apache/cassandra/db/CompactTables.java
index 9da4d94..29993a2 100644
--- a/src/java/org/apache/cassandra/db/CompactTables.java
+++ b/src/java/org/apache/cassandra/db/CompactTables.java
@@ -17,15 +17,13 @@
*/
package org.apache.cassandra.db;
-import java.util.HashSet;
-import java.util.Set;
+import java.nio.ByteBuffer;
+import java.util.*;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
-import org.apache.cassandra.cql3.SuperColumnCompatibility;
-import org.apache.cassandra.db.marshal.AbstractType;
-import org.apache.cassandra.db.marshal.EmptyType;
-import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.db.marshal.*;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.utils.ByteBufferUtil;
/**
* Small utility methods pertaining to the encoding of COMPACT STORAGE tables.
@@ -56,52 +54,65 @@
* On variation is that if the table comparator is a CompositeType, then the underlying table will have one clustering column by
* element of the CompositeType, but the rest of the layout is as above.
*
- * SuperColumn families handling and detailed format description can be found in {@code SuperColumnCompatibility}.
+ * As far as thrift is concerned, one exception to this is super column families, which have a different layout. Namely, a super
+ * column families is encoded with:
+ * {@code
+ * CREATE TABLE super (
+ * key [key_validation_class],
+ * super_column_name [comparator],
+ * [column_metadata_1] [type1],
+ * ...,
+ * [column_metadata_n] [type1],
+ * "" map<[sub_comparator], [default_validation_class]>
+ * PRIMARY KEY (key, super_column_name)
+ * )
+ * }
+ * In other words, every super column is encoded by a row. That row has one column for each defined "column_metadata", but it also
+ * has a special map column (whose name is the empty string as this is guaranteed to never conflict with a user-defined
+ * "column_metadata") which stores the super column "dynamic" sub-columns.
*/
public abstract class CompactTables
{
+ // We use an empty value for the 1) this can't conflict with a user-defined column and 2) this actually
+ // validate with any comparator.
+ public static final ByteBuffer SUPER_COLUMN_MAP_COLUMN = ByteBufferUtil.EMPTY_BYTE_BUFFER;
+
private CompactTables() {}
- public static ColumnDefinition getCompactValueColumn(PartitionColumns columns)
+ public static ColumnMetadata getCompactValueColumn(RegularAndStaticColumns columns, boolean isSuper)
{
+ if (isSuper)
+ {
+ for (ColumnMetadata column : columns.regulars)
+ if (column.name.bytes.equals(SUPER_COLUMN_MAP_COLUMN))
+ return column;
+ throw new AssertionError("Invalid super column table definition, no 'dynamic' map column");
+ }
assert columns.regulars.simpleColumnCount() == 1 && columns.regulars.complexColumnCount() == 0;
return columns.regulars.getSimple(0);
}
- public static AbstractType<?> columnDefinitionComparator(String kind, boolean isSuper, AbstractType<?> rawComparator, AbstractType<?> rawSubComparator)
+ public static boolean hasEmptyCompactValue(TableMetadata metadata)
{
- if (!"regular".equals(kind))
- return UTF8Type.instance;
-
- return isSuper ? rawSubComparator : rawComparator;
+ return metadata.compactValueColumn.type instanceof EmptyType;
}
- public static boolean hasEmptyCompactValue(CFMetaData metadata)
+ public static boolean isSuperColumnMapColumn(ColumnMetadata column)
{
- return metadata.compactValueColumn().type instanceof EmptyType;
+ return column.kind == ColumnMetadata.Kind.REGULAR && column.name.bytes.equals(SUPER_COLUMN_MAP_COLUMN);
}
public static DefaultNames defaultNameGenerator(Set<String> usedNames)
{
- return new DefaultNames(new HashSet<String>(usedNames));
- }
-
- public static DefaultNames defaultNameGenerator(Iterable<ColumnDefinition> defs)
- {
- Set<String> usedNames = new HashSet<>();
- for (ColumnDefinition def : defs)
- usedNames.add(def.name.toString());
- return new DefaultNames(usedNames);
+ return new DefaultNames(new HashSet<>(usedNames));
}
public static class DefaultNames
{
- private static final String DEFAULT_PARTITION_KEY_NAME = "key";
private static final String DEFAULT_CLUSTERING_NAME = "column";
private static final String DEFAULT_COMPACT_VALUE_NAME = "value";
private final Set<String> usedNames;
- private int partitionIndex = 0;
private int clusteringIndex = 1;
private int compactIndex = 0;
@@ -110,19 +121,6 @@
this.usedNames = usedNames;
}
- public String defaultPartitionKeyName()
- {
- while (true)
- {
- // For compatibility sake, we call the first alias 'key' rather than 'key1'. This
- // is inconsistent with column alias, but it's probably not worth risking breaking compatibility now.
- String candidate = partitionIndex == 0 ? DEFAULT_PARTITION_KEY_NAME : DEFAULT_PARTITION_KEY_NAME + (partitionIndex + 1);
- ++partitionIndex;
- if (usedNames.add(candidate))
- return candidate;
- }
- }
-
public String defaultClusteringName()
{
while (true)
diff --git a/src/java/org/apache/cassandra/db/Conflicts.java b/src/java/org/apache/cassandra/db/Conflicts.java
deleted file mode 100644
index 9e8bd9a..0000000
--- a/src/java/org/apache/cassandra/db/Conflicts.java
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.db;
-
-import java.nio.ByteBuffer;
-
-import org.apache.cassandra.db.context.CounterContext;
-
-public abstract class Conflicts
-{
- private Conflicts() {}
-
- public enum Resolution { LEFT_WINS, MERGE, RIGHT_WINS };
-
- public static Resolution resolveRegular(long leftTimestamp,
- boolean leftLive,
- int leftLocalDeletionTime,
- ByteBuffer leftValue,
- long rightTimestamp,
- boolean rightLive,
- int rightLocalDeletionTime,
- ByteBuffer rightValue)
- {
- if (leftTimestamp != rightTimestamp)
- return leftTimestamp < rightTimestamp ? Resolution.RIGHT_WINS : Resolution.LEFT_WINS;
-
- if (leftLive != rightLive)
- return leftLive ? Resolution.RIGHT_WINS : Resolution.LEFT_WINS;
-
- int c = leftValue.compareTo(rightValue);
- if (c < 0)
- return Resolution.RIGHT_WINS;
- else if (c > 0)
- return Resolution.LEFT_WINS;
-
- // Prefer the longest ttl if relevant
- return leftLocalDeletionTime < rightLocalDeletionTime ? Resolution.RIGHT_WINS : Resolution.LEFT_WINS;
- }
-
- public static Resolution resolveCounter(long leftTimestamp,
- boolean leftLive,
- ByteBuffer leftValue,
- long rightTimestamp,
- boolean rightLive,
- ByteBuffer rightValue)
- {
- // No matter what the counter cell's timestamp is, a tombstone always takes precedence. See CASSANDRA-7346.
- if (!leftLive)
- // left is a tombstone: it has precedence over right if either right is not a tombstone, or left has a greater timestamp
- return rightLive || leftTimestamp > rightTimestamp ? Resolution.LEFT_WINS : Resolution.RIGHT_WINS;
-
- // If right is a tombstone, since left isn't one, it has precedence
- if (!rightLive)
- return Resolution.RIGHT_WINS;
-
- // Handle empty values. Counters can't truly have empty values, but we can have a counter cell that temporarily
- // has one on read if the column for the cell is not queried by the user due to the optimization of #10657. We
- // thus need to handle this (see #11726 too).
- if (!leftValue.hasRemaining())
- return rightValue.hasRemaining() || leftTimestamp > rightTimestamp ? Resolution.LEFT_WINS : Resolution.RIGHT_WINS;
-
- if (!rightValue.hasRemaining())
- return Resolution.RIGHT_WINS;
-
- return Resolution.MERGE;
- }
-
- public static ByteBuffer mergeCounterValues(ByteBuffer left, ByteBuffer right)
- {
- return CounterContext.instance().merge(left, right);
- }
-
-}
diff --git a/src/java/org/apache/cassandra/db/ConsistencyLevel.java b/src/java/org/apache/cassandra/db/ConsistencyLevel.java
index ab4243f..e3da5b3 100644
--- a/src/java/org/apache/cassandra/db/ConsistencyLevel.java
+++ b/src/java/org/apache/cassandra/db/ConsistencyLevel.java
@@ -17,26 +17,19 @@
*/
package org.apache.cassandra.db;
-import java.net.InetAddress;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import com.google.common.collect.Iterables;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.cassandra.config.CFMetaData;
+import com.carrotsearch.hppc.ObjectIntHashMap;
+import org.apache.cassandra.locator.Endpoints;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.config.ReadRepairDecision;
import org.apache.cassandra.exceptions.InvalidRequestException;
-import org.apache.cassandra.exceptions.UnavailableException;
import org.apache.cassandra.locator.AbstractReplicationStrategy;
import org.apache.cassandra.locator.NetworkTopologyStrategy;
import org.apache.cassandra.transport.ProtocolException;
+import static org.apache.cassandra.locator.Replicas.addToCountPerDc;
+import static org.apache.cassandra.locator.Replicas.countInOurDc;
+
public enum ConsistencyLevel
{
ANY (0),
@@ -49,9 +42,8 @@
EACH_QUORUM (7),
SERIAL (8),
LOCAL_SERIAL(9),
- LOCAL_ONE (10, true);
-
- private static final Logger logger = LoggerFactory.getLogger(ConsistencyLevel.class);
+ LOCAL_ONE (10, true),
+ NODE_LOCAL (11, true);
// Used by the binary protocol
public final int code;
@@ -89,18 +81,49 @@
return codeIdx[code];
}
- private int quorumFor(Keyspace keyspace)
+ public static int quorumFor(Keyspace keyspace)
{
- return (keyspace.getReplicationStrategy().getReplicationFactor() / 2) + 1;
+ return (keyspace.getReplicationStrategy().getReplicationFactor().allReplicas / 2) + 1;
}
- private int localQuorumFor(Keyspace keyspace, String dc)
+ public static int localQuorumFor(Keyspace keyspace, String dc)
{
return (keyspace.getReplicationStrategy() instanceof NetworkTopologyStrategy)
- ? (((NetworkTopologyStrategy) keyspace.getReplicationStrategy()).getReplicationFactor(dc) / 2) + 1
+ ? (((NetworkTopologyStrategy) keyspace.getReplicationStrategy()).getReplicationFactor(dc).allReplicas / 2) + 1
: quorumFor(keyspace);
}
+ public static int localQuorumForOurDc(Keyspace keyspace)
+ {
+ return localQuorumFor(keyspace, DatabaseDescriptor.getLocalDataCenter());
+ }
+
+ public static ObjectIntHashMap<String> eachQuorumForRead(Keyspace keyspace)
+ {
+ AbstractReplicationStrategy strategy = keyspace.getReplicationStrategy();
+ if (strategy instanceof NetworkTopologyStrategy)
+ {
+ NetworkTopologyStrategy npStrategy = (NetworkTopologyStrategy) strategy;
+ ObjectIntHashMap<String> perDc = new ObjectIntHashMap<>(((npStrategy.getDatacenters().size() + 1) * 4) / 3);
+ for (String dc : npStrategy.getDatacenters())
+ perDc.put(dc, ConsistencyLevel.localQuorumFor(keyspace, dc));
+ return perDc;
+ }
+ else
+ {
+ ObjectIntHashMap<String> perDc = new ObjectIntHashMap<>(1);
+ perDc.put(DatabaseDescriptor.getLocalDataCenter(), quorumFor(keyspace));
+ return perDc;
+ }
+ }
+
+ public static ObjectIntHashMap<String> eachQuorumForWrite(Keyspace keyspace, Endpoints<?> pendingWithDown)
+ {
+ ObjectIntHashMap<String> perDc = eachQuorumForRead(keyspace);
+ addToCountPerDc(perDc, pendingWithDown, 1);
+ return perDc;
+ }
+
public int blockFor(Keyspace keyspace)
{
switch (this)
@@ -118,10 +141,10 @@
case SERIAL:
return quorumFor(keyspace);
case ALL:
- return keyspace.getReplicationStrategy().getReplicationFactor();
+ return keyspace.getReplicationStrategy().getReplicationFactor().allReplicas;
case LOCAL_QUORUM:
case LOCAL_SERIAL:
- return localQuorumFor(keyspace, DatabaseDescriptor.getLocalDataCenter());
+ return localQuorumForOurDc(keyspace);
case EACH_QUORUM:
if (keyspace.getReplicationStrategy() instanceof NetworkTopologyStrategy)
{
@@ -140,201 +163,40 @@
}
}
- public boolean isDatacenterLocal()
+ public int blockForWrite(Keyspace keyspace, Endpoints<?> pending)
{
- return isDCLocal;
- }
+ assert pending != null;
- public boolean isLocal(InetAddress endpoint)
- {
- return DatabaseDescriptor.getLocalDataCenter().equals(DatabaseDescriptor.getEndpointSnitch().getDatacenter(endpoint));
- }
-
- public int countLocalEndpoints(Iterable<InetAddress> liveEndpoints)
- {
- int count = 0;
- for (InetAddress endpoint : liveEndpoints)
- if (isLocal(endpoint))
- count++;
- return count;
- }
-
- private Map<String, Integer> countPerDCEndpoints(Keyspace keyspace, Iterable<InetAddress> liveEndpoints)
- {
- NetworkTopologyStrategy strategy = (NetworkTopologyStrategy) keyspace.getReplicationStrategy();
-
- Map<String, Integer> dcEndpoints = new HashMap<String, Integer>();
- for (String dc: strategy.getDatacenters())
- dcEndpoints.put(dc, 0);
-
- for (InetAddress endpoint : liveEndpoints)
- {
- String dc = DatabaseDescriptor.getEndpointSnitch().getDatacenter(endpoint);
- dcEndpoints.put(dc, dcEndpoints.get(dc) + 1);
- }
- return dcEndpoints;
- }
-
- public List<InetAddress> filterForQuery(Keyspace keyspace, List<InetAddress> liveEndpoints)
- {
- return filterForQuery(keyspace, liveEndpoints, ReadRepairDecision.NONE);
- }
-
- public List<InetAddress> filterForQuery(Keyspace keyspace, List<InetAddress> liveEndpoints, ReadRepairDecision readRepair)
- {
- /*
- * If we are doing an each quorum query, we have to make sure that the endpoints we select
- * provide a quorum for each data center. If we are not using a NetworkTopologyStrategy,
- * we should fall through and grab a quorum in the replication strategy.
- */
- if (this == EACH_QUORUM && keyspace.getReplicationStrategy() instanceof NetworkTopologyStrategy)
- return filterForEachQuorum(keyspace, liveEndpoints, readRepair);
-
- /*
- * Endpoints are expected to be restricted to live replicas, sorted by snitch preference.
- * For LOCAL_QUORUM, move local-DC replicas in front first as we need them there whether
- * we do read repair (since the first replica gets the data read) or not (since we'll take
- * the blockFor first ones).
- */
- if (isDCLocal)
- Collections.sort(liveEndpoints, DatabaseDescriptor.getLocalComparator());
-
- switch (readRepair)
- {
- case NONE:
- return liveEndpoints.subList(0, Math.min(liveEndpoints.size(), blockFor(keyspace)));
- case GLOBAL:
- return liveEndpoints;
- case DC_LOCAL:
- List<InetAddress> local = new ArrayList<InetAddress>();
- List<InetAddress> other = new ArrayList<InetAddress>();
- for (InetAddress add : liveEndpoints)
- {
- if (isLocal(add))
- local.add(add);
- else
- other.add(add);
- }
- // check if blockfor more than we have localep's
- int blockFor = blockFor(keyspace);
- if (local.size() < blockFor)
- local.addAll(other.subList(0, Math.min(blockFor - local.size(), other.size())));
- return local;
- default:
- throw new AssertionError();
- }
- }
-
- private List<InetAddress> filterForEachQuorum(Keyspace keyspace, List<InetAddress> liveEndpoints, ReadRepairDecision readRepair)
- {
- NetworkTopologyStrategy strategy = (NetworkTopologyStrategy) keyspace.getReplicationStrategy();
-
- // quickly drop out if read repair is GLOBAL, since we just use all of the live endpoints
- if (readRepair == ReadRepairDecision.GLOBAL)
- return liveEndpoints;
-
- Map<String, List<InetAddress>> dcsEndpoints = new HashMap<>();
- for (String dc: strategy.getDatacenters())
- dcsEndpoints.put(dc, new ArrayList<>());
-
- for (InetAddress add : liveEndpoints)
- {
- String dc = DatabaseDescriptor.getEndpointSnitch().getDatacenter(add);
- dcsEndpoints.get(dc).add(add);
- }
-
- List<InetAddress> waitSet = new ArrayList<>();
- for (Map.Entry<String, List<InetAddress>> dcEndpoints : dcsEndpoints.entrySet())
- {
- List<InetAddress> dcEndpoint = dcEndpoints.getValue();
- if (readRepair == ReadRepairDecision.DC_LOCAL && dcEndpoints.getKey().equals(DatabaseDescriptor.getLocalDataCenter()))
- waitSet.addAll(dcEndpoint);
- else
- waitSet.addAll(dcEndpoint.subList(0, Math.min(localQuorumFor(keyspace, dcEndpoints.getKey()), dcEndpoint.size())));
- }
-
- return waitSet;
- }
-
- public boolean isSufficientLiveNodes(Keyspace keyspace, Iterable<InetAddress> liveEndpoints)
- {
- switch (this)
- {
- case ANY:
- // local hint is acceptable, and local node is always live
- return true;
- case LOCAL_ONE:
- return countLocalEndpoints(liveEndpoints) >= 1;
- case LOCAL_QUORUM:
- return countLocalEndpoints(liveEndpoints) >= blockFor(keyspace);
- case EACH_QUORUM:
- if (keyspace.getReplicationStrategy() instanceof NetworkTopologyStrategy)
- {
- for (Map.Entry<String, Integer> entry : countPerDCEndpoints(keyspace, liveEndpoints).entrySet())
- {
- if (entry.getValue() < localQuorumFor(keyspace, entry.getKey()))
- return false;
- }
- return true;
- }
- // Fallthough on purpose for SimpleStrategy
- default:
- return Iterables.size(liveEndpoints) >= blockFor(keyspace);
- }
- }
-
- public void assureSufficientLiveNodes(Keyspace keyspace, Iterable<InetAddress> liveEndpoints) throws UnavailableException
- {
int blockFor = blockFor(keyspace);
switch (this)
{
case ANY:
- // local hint is acceptable, and local node is always live
break;
- case LOCAL_ONE:
- if (countLocalEndpoints(liveEndpoints) == 0)
- throw new UnavailableException(this, 1, 0);
+ case LOCAL_ONE: case LOCAL_QUORUM: case LOCAL_SERIAL:
+ // we will only count local replicas towards our response count, as these queries only care about local guarantees
+ blockFor += countInOurDc(pending).allReplicas();
break;
- case LOCAL_QUORUM:
- int localLive = countLocalEndpoints(liveEndpoints);
- if (localLive < blockFor)
- {
- if (logger.isTraceEnabled())
- {
- StringBuilder builder = new StringBuilder("Local replicas [");
- for (InetAddress endpoint : liveEndpoints)
- {
- if (isLocal(endpoint))
- builder.append(endpoint).append(",");
- }
- builder.append("] are insufficient to satisfy LOCAL_QUORUM requirement of ").append(blockFor).append(" live nodes in '").append(DatabaseDescriptor.getLocalDataCenter()).append("'");
- logger.trace(builder.toString());
- }
- throw new UnavailableException(this, blockFor, localLive);
- }
- break;
- case EACH_QUORUM:
- if (keyspace.getReplicationStrategy() instanceof NetworkTopologyStrategy)
- {
- for (Map.Entry<String, Integer> entry : countPerDCEndpoints(keyspace, liveEndpoints).entrySet())
- {
- int dcBlockFor = localQuorumFor(keyspace, entry.getKey());
- int dcLive = entry.getValue();
- if (dcLive < dcBlockFor)
- throw new UnavailableException(this, entry.getKey(), dcBlockFor, dcLive);
- }
- break;
- }
- // Fallthough on purpose for SimpleStrategy
- default:
- int live = Iterables.size(liveEndpoints);
- if (live < blockFor)
- {
- logger.trace("Live nodes {} do not satisfy ConsistencyLevel ({} required)", Iterables.toString(liveEndpoints), blockFor);
- throw new UnavailableException(this, blockFor, live);
- }
- break;
+ case ONE: case TWO: case THREE:
+ case QUORUM: case EACH_QUORUM:
+ case SERIAL:
+ case ALL:
+ blockFor += pending.size();
}
+ return blockFor;
+ }
+
+ /**
+ * Determine if this consistency level meets or exceeds the consistency requirements of the given cl for the given keyspace
+ * WARNING: this is not locality aware; you cannot safely use this with mixed locality consistency levels (e.g. LOCAL_QUORUM and QUORUM)
+ */
+ public boolean satisfies(ConsistencyLevel other, Keyspace keyspace)
+ {
+ return blockFor(keyspace) >= other.blockFor(keyspace);
+ }
+
+ public boolean isDatacenterLocal()
+ {
+ return isDCLocal;
}
public void validateForRead(String keyspaceName) throws InvalidRequestException
@@ -381,10 +243,10 @@
return this == SERIAL || this == LOCAL_SERIAL;
}
- public void validateCounterForWrite(CFMetaData metadata) throws InvalidRequestException
+ public void validateCounterForWrite(TableMetadata metadata) throws InvalidRequestException
{
if (this == ConsistencyLevel.ANY)
- throw new InvalidRequestException("Consistency level ANY is not yet supported for counter table " + metadata.cfName);
+ throw new InvalidRequestException("Consistency level ANY is not yet supported for counter table " + metadata.name);
if (isSerialConsistency())
throw new InvalidRequestException("Counter operations are inherently non-serializable");
diff --git a/src/java/org/apache/cassandra/db/CounterMutation.java b/src/java/org/apache/cassandra/db/CounterMutation.java
index 4e4a30d..722ad73 100644
--- a/src/java/org/apache/cassandra/db/CounterMutation.java
+++ b/src/java/org/apache/cassandra/db/CounterMutation.java
@@ -38,13 +38,17 @@
import org.apache.cassandra.io.IVersionedSerializer;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
-import org.apache.cassandra.net.MessageOut;
-import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.service.CacheService;
import org.apache.cassandra.tracing.Tracing;
import org.apache.cassandra.utils.*;
import org.apache.cassandra.utils.btree.BTreeSet;
+import static java.util.concurrent.TimeUnit.*;
+import static org.apache.cassandra.net.MessagingService.VERSION_30;
+import static org.apache.cassandra.net.MessagingService.VERSION_3014;
+import static org.apache.cassandra.net.MessagingService.VERSION_40;
+
public class CounterMutation implements IMutation
{
public static final CounterMutationSerializer serializer = new CounterMutationSerializer();
@@ -65,9 +69,9 @@
return mutation.getKeyspaceName();
}
- public Collection<UUID> getColumnFamilyIds()
+ public Collection<TableId> getTableIds()
{
- return mutation.getColumnFamilyIds();
+ return mutation.getTableIds();
}
public Collection<PartitionUpdate> getPartitionUpdates()
@@ -75,6 +79,15 @@
return mutation.getPartitionUpdates();
}
+ public void validateSize(int version, int overhead)
+ {
+ long totalSize = serializedSize(version) + overhead;
+ if(totalSize > MAX_MUTATION_SIZE)
+ {
+ throw new MutationExceededMaxSizeException(this, version, totalSize);
+ }
+ }
+
public Mutation getMutation()
{
return mutation;
@@ -90,11 +103,6 @@
return consistency;
}
- public MessageOut<CounterMutation> makeMutationMessage()
- {
- return new MessageOut<>(MessagingService.Verb.COUNTER_MUTATION, this, serializer);
- }
-
/**
* Applies the counter mutation, returns the result Mutation (for replication to other nodes).
*
@@ -111,7 +119,7 @@
*/
public Mutation applyCounterMutation() throws WriteTimeoutException
{
- Mutation result = new Mutation(getKeyspaceName(), key());
+ Mutation.PartitionUpdateCollector resultBuilder = new Mutation.PartitionUpdateCollector(getKeyspaceName(), key());
Keyspace keyspace = Keyspace.open(getKeyspaceName());
List<Lock> locks = new ArrayList<>();
@@ -120,7 +128,9 @@
{
grabCounterLocks(keyspace, locks);
for (PartitionUpdate upd : getPartitionUpdates())
- result.add(processModifications(upd));
+ resultBuilder.add(processModifications(upd));
+
+ Mutation result = resultBuilder.build();
result.apply();
return result;
}
@@ -142,10 +152,10 @@
for (Lock lock : LOCKS.bulkGet(getCounterLockKeys()))
{
- long timeout = TimeUnit.MILLISECONDS.toNanos(getTimeout()) - (System.nanoTime() - startTime);
+ long timeout = getTimeout(NANOSECONDS) - (System.nanoTime() - startTime);
try
{
- if (!lock.tryLock(timeout, TimeUnit.NANOSECONDS))
+ if (!lock.tryLock(timeout, NANOSECONDS))
throw new WriteTimeoutException(WriteType.COUNTER, consistency(), 0, consistency().blockFor(keyspace));
locks.add(lock);
}
@@ -175,7 +185,7 @@
{
public Object apply(final ColumnData data)
{
- return Objects.hashCode(update.metadata().cfId, key(), row.clustering(), data.column());
+ return Objects.hashCode(update.metadata().id, key(), row.clustering(), data.column());
}
}));
}
@@ -186,7 +196,7 @@
private PartitionUpdate processModifications(PartitionUpdate changes)
{
- ColumnFamilyStore cfs = Keyspace.open(getKeyspaceName()).getColumnFamilyStore(changes.metadata().cfId);
+ ColumnFamilyStore cfs = Keyspace.open(getKeyspaceName()).getColumnFamilyStore(changes.metadata().id);
List<PartitionUpdate.CounterMark> marks = changes.collectCounterMarks();
@@ -239,7 +249,7 @@
private void updateWithCurrentValuesFromCFS(List<PartitionUpdate.CounterMark> marks, ColumnFamilyStore cfs)
{
ColumnFilter.Builder builder = ColumnFilter.selectionBuilder();
- BTreeSet.Builder<Clustering> names = BTreeSet.builder(cfs.metadata.comparator);
+ BTreeSet.Builder<Clustering> names = BTreeSet.builder(cfs.metadata().comparator);
for (PartitionUpdate.CounterMark mark : marks)
{
if (mark.clustering() != Clustering.STATIC_CLUSTERING)
@@ -252,7 +262,7 @@
int nowInSec = FBUtilities.nowInSeconds();
ClusteringIndexNamesFilter filter = new ClusteringIndexNamesFilter(names.build(), false);
- SinglePartitionReadCommand cmd = SinglePartitionReadCommand.create(cfs.metadata, nowInSec, key(), builder.build(), filter);
+ SinglePartitionReadCommand cmd = SinglePartitionReadCommand.create(cfs.metadata(), nowInSec, key(), builder.build(), filter);
PeekingIterator<PartitionUpdate.CounterMark> markIter = Iterators.peekingIterator(marks.iterator());
try (ReadExecutionController controller = cmd.executionController();
RowIterator partition = UnfilteredRowIterators.filter(cmd.queryMemtableAndDisk(cfs, controller), nowInSec))
@@ -305,9 +315,34 @@
}
}
- public long getTimeout()
+ public long getTimeout(TimeUnit unit)
{
- return DatabaseDescriptor.getCounterWriteRpcTimeout();
+ return DatabaseDescriptor.getCounterWriteRpcTimeout(unit);
+ }
+
+ private int serializedSize30;
+ private int serializedSize3014;
+ private int serializedSize40;
+
+ public int serializedSize(int version)
+ {
+ switch (version)
+ {
+ case VERSION_30:
+ if (serializedSize30 == 0)
+ serializedSize30 = (int) serializer.serializedSize(this, VERSION_30);
+ return serializedSize30;
+ case VERSION_3014:
+ if (serializedSize3014 == 0)
+ serializedSize3014 = (int) serializer.serializedSize(this, VERSION_3014);
+ return serializedSize3014;
+ case VERSION_40:
+ if (serializedSize40 == 0)
+ serializedSize40 = (int) serializer.serializedSize(this, VERSION_40);
+ return serializedSize40;
+ default:
+ throw new IllegalStateException("Unknown serialization version: " + version);
+ }
}
@Override
@@ -338,7 +373,7 @@
public long serializedSize(CounterMutation cm, int version)
{
- return Mutation.serializer.serializedSize(cm.mutation, version)
+ return cm.mutation.serializedSize(version)
+ TypeSizes.sizeof(cm.consistency.name());
}
}
diff --git a/src/java/org/apache/cassandra/db/CounterMutationVerbHandler.java b/src/java/org/apache/cassandra/db/CounterMutationVerbHandler.java
index bd273e4..a30ce66 100644
--- a/src/java/org/apache/cassandra/db/CounterMutationVerbHandler.java
+++ b/src/java/org/apache/cassandra/db/CounterMutationVerbHandler.java
@@ -22,22 +22,23 @@
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.net.IVerbHandler;
-import org.apache.cassandra.net.MessageIn;
+import org.apache.cassandra.net.Message;
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.service.StorageProxy;
-import org.apache.cassandra.utils.FBUtilities;
public class CounterMutationVerbHandler implements IVerbHandler<CounterMutation>
{
+ public static final CounterMutationVerbHandler instance = new CounterMutationVerbHandler();
+
private static final Logger logger = LoggerFactory.getLogger(CounterMutationVerbHandler.class);
- public void doVerb(final MessageIn<CounterMutation> message, final int id)
+ public void doVerb(final Message<CounterMutation> message)
{
long queryStartNanoTime = System.nanoTime();
final CounterMutation cm = message.payload;
logger.trace("Applying forwarded {}", cm);
- String localDataCenter = DatabaseDescriptor.getEndpointSnitch().getDatacenter(FBUtilities.getBroadcastAddress());
+ String localDataCenter = DatabaseDescriptor.getEndpointSnitch().getLocalDatacenter();
// We should not wait for the result of the write in this thread,
// otherwise we could have a distributed deadlock between replicas
// running this VerbHandler (see #4578).
@@ -45,12 +46,9 @@
// will not be called if the request timeout, but this is ok
// because the coordinator of the counter mutation will timeout on
// it's own in that case.
- StorageProxy.applyCounterMutationOnLeader(cm, localDataCenter, new Runnable()
- {
- public void run()
- {
- MessagingService.instance().sendReply(WriteResponse.createMessage(), id, message.from);
- }
- }, queryStartNanoTime);
+ StorageProxy.applyCounterMutationOnLeader(cm,
+ localDataCenter,
+ () -> MessagingService.instance().send(message.emptyResponse(), message.from()),
+ queryStartNanoTime);
}
}
diff --git a/src/java/org/apache/cassandra/db/DataRange.java b/src/java/org/apache/cassandra/db/DataRange.java
index a78d3b6..aa23f3d 100644
--- a/src/java/org/apache/cassandra/db/DataRange.java
+++ b/src/java/org/apache/cassandra/db/DataRange.java
@@ -19,8 +19,8 @@
import java.io.IOException;
import java.nio.ByteBuffer;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.db.filter.*;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.db.marshal.CompositeType;
@@ -251,12 +251,12 @@
return new DataRange(range, clusteringIndexFilter);
}
- public String toString(CFMetaData metadata)
+ public String toString(TableMetadata metadata)
{
- return String.format("range=%s pfilter=%s", keyRange.getString(metadata.getKeyValidator()), clusteringIndexFilter.toString(metadata));
+ return String.format("range=%s pfilter=%s", keyRange.getString(metadata.partitionKeyType), clusteringIndexFilter.toString(metadata));
}
- public String toCQLString(CFMetaData metadata)
+ public String toCQLString(TableMetadata metadata)
{
if (isUnrestricted())
return "UNRESTRICTED";
@@ -284,16 +284,16 @@
return sb.toString();
}
- private void appendClause(PartitionPosition pos, StringBuilder sb, CFMetaData metadata, boolean isStart, boolean isInclusive)
+ private void appendClause(PartitionPosition pos, StringBuilder sb, TableMetadata metadata, boolean isStart, boolean isInclusive)
{
sb.append("token(");
- sb.append(ColumnDefinition.toCQLString(metadata.partitionKeyColumns()));
+ sb.append(ColumnMetadata.toCQLString(metadata.partitionKeyColumns()));
sb.append(") ");
if (pos instanceof DecoratedKey)
{
sb.append(getOperator(isStart, isInclusive)).append(" ");
sb.append("token(");
- appendKeyString(sb, metadata.getKeyValidator(), ((DecoratedKey)pos).getKey());
+ appendKeyString(sb, metadata.partitionKeyType, ((DecoratedKey)pos).getKey());
sb.append(")");
}
else
@@ -398,10 +398,10 @@
}
@Override
- public String toString(CFMetaData metadata)
+ public String toString(TableMetadata metadata)
{
return String.format("range=%s (paging) pfilter=%s lastReturned=%s (%s)",
- keyRange.getString(metadata.getKeyValidator()),
+ keyRange.getString(metadata.partitionKeyType),
clusteringIndexFilter.toString(metadata),
lastReturned.toString(metadata),
inclusive ? "included" : "excluded");
@@ -410,7 +410,7 @@
public static class Serializer
{
- public void serialize(DataRange range, DataOutputPlus out, int version, CFMetaData metadata) throws IOException
+ public void serialize(DataRange range, DataOutputPlus out, int version, TableMetadata metadata) throws IOException
{
AbstractBounds.rowPositionSerializer.serialize(range.keyRange, out, version);
ClusteringIndexFilter.serializer.serialize(range.clusteringIndexFilter, out, version);
@@ -423,7 +423,7 @@
}
}
- public DataRange deserialize(DataInputPlus in, int version, CFMetaData metadata) throws IOException
+ public DataRange deserialize(DataInputPlus in, int version, TableMetadata metadata) throws IOException
{
AbstractBounds<PartitionPosition> range = AbstractBounds.rowPositionSerializer.deserialize(in, metadata.partitioner, version);
ClusteringIndexFilter filter = ClusteringIndexFilter.serializer.deserialize(in, version, metadata);
@@ -440,7 +440,7 @@
}
}
- public long serializedSize(DataRange range, int version, CFMetaData metadata)
+ public long serializedSize(DataRange range, int version, TableMetadata metadata)
{
long size = AbstractBounds.rowPositionSerializer.serializedSize(range.keyRange, version)
+ ClusteringIndexFilter.serializer.serializedSize(range.clusteringIndexFilter, version)
diff --git a/src/java/org/apache/cassandra/db/DefinitionsUpdateVerbHandler.java b/src/java/org/apache/cassandra/db/DefinitionsUpdateVerbHandler.java
deleted file mode 100644
index 8b3e121..0000000
--- a/src/java/org/apache/cassandra/db/DefinitionsUpdateVerbHandler.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.db;
-
-import java.util.Collection;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.cassandra.concurrent.Stage;
-import org.apache.cassandra.concurrent.StageManager;
-import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.net.IVerbHandler;
-import org.apache.cassandra.net.MessageIn;
-import org.apache.cassandra.schema.SchemaKeyspace;
-import org.apache.cassandra.utils.WrappedRunnable;
-
-/**
- * Called when node receives updated schema state from the schema migration coordinator node.
- * Such happens when user makes local schema migration on one of the nodes in the ring
- * (which is going to act as coordinator) and that node sends (pushes) it's updated schema state
- * (in form of mutations) to all the alive nodes in the cluster.
- */
-public class DefinitionsUpdateVerbHandler implements IVerbHandler<Collection<Mutation>>
-{
- private static final Logger logger = LoggerFactory.getLogger(DefinitionsUpdateVerbHandler.class);
-
- public void doVerb(final MessageIn<Collection<Mutation>> message, int id)
- {
- logger.trace("Received schema mutation push from {}", message.from);
-
- StageManager.getStage(Stage.MIGRATION).submit(new WrappedRunnable()
- {
- public void runMayThrow() throws ConfigurationException
- {
- SchemaKeyspace.mergeSchemaAndAnnounceVersion(message.payload);
- }
- });
- }
-}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/db/DeletionTime.java b/src/java/org/apache/cassandra/db/DeletionTime.java
index 652689c..b2d9343 100644
--- a/src/java/org/apache/cassandra/db/DeletionTime.java
+++ b/src/java/org/apache/cassandra/db/DeletionTime.java
@@ -18,7 +18,6 @@
package org.apache.cassandra.db;
import java.io.IOException;
-import java.security.MessageDigest;
import com.google.common.base.Objects;
@@ -27,7 +26,6 @@
import org.apache.cassandra.io.ISerializer;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
-import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.ObjectSizes;
/**
@@ -80,12 +78,21 @@
return markedForDeleteAt() == Long.MIN_VALUE && localDeletionTime() == Integer.MAX_VALUE;
}
- public void digest(MessageDigest digest)
+ public void digest(Digest digest)
{
// localDeletionTime is basically a metadata of the deletion time that tells us when it's ok to purge it.
// It's thus intrinsically a local information and shouldn't be part of the digest (which exists for
// cross-nodes comparisons).
- FBUtilities.updateWithLong(digest, markedForDeleteAt());
+ digest.updateWithLong(markedForDeleteAt());
+ }
+
+ /**
+ * check if this deletion time is valid - localDeletionTime can never be negative
+ * @return true if it is valid
+ */
+ public boolean validate()
+ {
+ return localDeletionTime >= 0;
}
@Override
diff --git a/src/java/org/apache/cassandra/db/Digest.java b/src/java/org/apache/cassandra/db/Digest.java
new file mode 100644
index 0000000..bac6386
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/Digest.java
@@ -0,0 +1,200 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db;
+
+import java.nio.ByteBuffer;
+
+import com.google.common.hash.Hasher;
+import com.google.common.hash.Hashing;
+
+import org.apache.cassandra.db.context.CounterContext;
+import org.apache.cassandra.utils.FastByteOperations;
+
+public class Digest
+{
+ private static final ThreadLocal<byte[]> localBuffer = ThreadLocal.withInitial(() -> new byte[4096]);
+
+ private final Hasher hasher;
+ private long inputBytes = 0;
+
+ @SuppressWarnings("deprecation")
+ private static Hasher md5()
+ {
+ return Hashing.md5().newHasher();
+ }
+
+ public static Digest forReadResponse()
+ {
+ return new Digest(md5());
+ }
+
+ public static Digest forSchema()
+ {
+ return new Digest(md5());
+ }
+
+ public static Digest forValidator()
+ {
+ // Uses a Hasher that concatenates the hash code from 2 hash functions
+ // (murmur3_128) with different seeds to produce a 256 bit hashcode
+ return new Digest(Hashing.concatenating(Hashing.murmur3_128(1000),
+ Hashing.murmur3_128(2000))
+ .newHasher());
+ }
+
+ public static Digest forRepairedDataTracking()
+ {
+ return new Digest(Hashing.crc32c().newHasher())
+ {
+ @Override
+ public Digest updateWithCounterContext(ByteBuffer context)
+ {
+ // for the purposes of repaired data tracking on the read path, exclude
+ // contexts with legacy shards as these may be irrevocably different on
+ // different replicas
+ if (CounterContext.instance().hasLegacyShards(context))
+ return this;
+
+ return super.updateWithCounterContext(context);
+ }
+ };
+ }
+
+ Digest(Hasher hasher)
+ {
+ this.hasher = hasher;
+ }
+
+ public Digest update(byte[] input, int offset, int len)
+ {
+ hasher.putBytes(input, offset, len);
+ inputBytes += len;
+ return this;
+ }
+
+ /**
+ * Update the digest with the bytes from the supplied buffer. This does
+ * not modify the position of the supplied buffer, so callers are not
+ * required to duplicate() the source buffer before calling
+ */
+ public Digest update(ByteBuffer input)
+ {
+ return update(input, input.position(), input.remaining());
+ }
+
+ /**
+ * Update the digest with the bytes sliced from the supplied buffer. This does
+ * not modify the position of the supplied buffer, so callers are not
+ * required to duplicate() the source buffer before calling
+ */
+ private Digest update(ByteBuffer input, int pos, int len)
+ {
+ if (len <= 0)
+ return this;
+
+ if (input.hasArray())
+ {
+ byte[] b = input.array();
+ int ofs = input.arrayOffset();
+ hasher.putBytes(b, ofs + pos, len);
+ inputBytes += len;
+ }
+ else
+ {
+ byte[] tempArray = localBuffer.get();
+ while (len > 0)
+ {
+ int chunk = Math.min(len, tempArray.length);
+ FastByteOperations.copy(input, pos, tempArray, 0, chunk);
+ hasher.putBytes(tempArray, 0, chunk);
+ len -= chunk;
+ pos += chunk;
+ inputBytes += chunk;
+ }
+ }
+ return this;
+ }
+
+ /**
+ * Update the digest with the content of a counter context.
+ * Note that this skips the header entirely since the header information
+ * has local meaning only, while digests are meant for comparison across
+ * nodes. This means in particular that we always have:
+ * updateDigest(ctx) == updateDigest(clearAllLocal(ctx))
+ */
+ public Digest updateWithCounterContext(ByteBuffer context)
+ {
+ // context can be empty due to the optimization from CASSANDRA-10657
+ if (!context.hasRemaining())
+ return this;
+
+ int pos = context.position() + CounterContext.headerLength(context);
+ int len = context.limit() - pos;
+ update(context, pos, len);
+ return this;
+ }
+
+ public Digest updateWithByte(int val)
+ {
+ hasher.putByte((byte) (val & 0xFF));
+ inputBytes++;
+ return this;
+ }
+
+ public Digest updateWithInt(int val)
+ {
+ hasher.putByte((byte) ((val >>> 24) & 0xFF));
+ hasher.putByte((byte) ((val >>> 16) & 0xFF));
+ hasher.putByte((byte) ((val >>> 8) & 0xFF));
+ hasher.putByte((byte) ((val >>> 0) & 0xFF));
+ inputBytes += 4;
+ return this;
+ }
+
+ public Digest updateWithLong(long val)
+ {
+ hasher.putByte((byte) ((val >>> 56) & 0xFF));
+ hasher.putByte((byte) ((val >>> 48) & 0xFF));
+ hasher.putByte((byte) ((val >>> 40) & 0xFF));
+ hasher.putByte((byte) ((val >>> 32) & 0xFF));
+ hasher.putByte((byte) ((val >>> 24) & 0xFF));
+ hasher.putByte((byte) ((val >>> 16) & 0xFF));
+ hasher.putByte((byte) ((val >>> 8) & 0xFF));
+ hasher.putByte((byte) ((val >>> 0) & 0xFF));
+ inputBytes += 8;
+ return this;
+ }
+
+ public Digest updateWithBoolean(boolean val)
+ {
+ updateWithByte(val ? 0 : 1);
+ return this;
+ }
+
+ public byte[] digest()
+ {
+ return hasher.hash().asBytes();
+ }
+
+ public long inputBytes()
+ {
+ return inputBytes;
+ }
+}
+
diff --git a/src/java/org/apache/cassandra/db/Directories.java b/src/java/org/apache/cassandra/db/Directories.java
index 8665c8d..2ddeb64 100644
--- a/src/java/org/apache/cassandra/db/Directories.java
+++ b/src/java/org/apache/cassandra/db/Directories.java
@@ -21,17 +21,17 @@
import java.io.FileFilter;
import java.io.IOError;
import java.io.IOException;
+import java.nio.file.FileStore;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.*;
import java.util.concurrent.ThreadLocalRandom;
-import java.util.function.BiFunction;
-
+import java.util.function.BiPredicate;
import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Predicate;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterables;
+import com.google.common.collect.Maps;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
@@ -44,8 +44,8 @@
import org.apache.cassandra.io.FSWriteError;
import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.io.sstable.*;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.utils.DirectorySizeCalculator;
-import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.Pair;
@@ -61,13 +61,13 @@
* } </pre>
*
* Until v2.0, {@code <cf dir>} is just column family name.
- * Since v2.1, {@code <cf dir>} has column family ID(cfId) added to its end.
+ * Since v2.1, {@code <cf dir>} has column family ID(tableId) added to its end.
*
* SSTables from secondary indexes were put in the same directory as their parent.
* Since v2.2, they have their own directory under the parent directory whose name is index name.
* Upon startup, those secondary index files are moved to new directory when upgrading.
*
- * For backward compatibility, Directories can use directory without cfId if exists.
+ * For backward compatibility, Directories can use directory without tableId if exists.
*
* In addition, more that one 'root' data directory can be specified so that
* {@code <path_to_data_dir>} potentially represents multiple locations.
@@ -175,17 +175,17 @@
}
}
- private final CFMetaData metadata;
+ private final TableMetadata metadata;
private final DataDirectory[] paths;
private final File[] dataPaths;
private final ImmutableMap<Path, DataDirectory> canonicalPathToDD;
- public Directories(final CFMetaData metadata)
+ public Directories(final TableMetadata metadata)
{
this(metadata, dataDirectories);
}
- public Directories(final CFMetaData metadata, Collection<DataDirectory> paths)
+ public Directories(final TableMetadata metadata, Collection<DataDirectory> paths)
{
this(metadata, paths.toArray(new DataDirectory[paths.size()]));
}
@@ -196,21 +196,19 @@
*
* @param metadata metadata of ColumnFamily
*/
- public Directories(final CFMetaData metadata, DataDirectory[] paths)
+ public Directories(final TableMetadata metadata, DataDirectory[] paths)
{
this.metadata = metadata;
this.paths = paths;
-
ImmutableMap.Builder<Path, DataDirectory> canonicalPathsBuilder = ImmutableMap.builder();
-
- String cfId = ByteBufferUtil.bytesToHex(ByteBufferUtil.bytes(metadata.cfId));
- int idx = metadata.cfName.indexOf(SECONDARY_INDEX_NAME_SEPARATOR);
- String cfName = idx >= 0 ? metadata.cfName.substring(0, idx) : metadata.cfName;
- String indexNameWithDot = idx >= 0 ? metadata.cfName.substring(idx) : null;
+ String tableId = metadata.id.toHexString();
+ int idx = metadata.name.indexOf(SECONDARY_INDEX_NAME_SEPARATOR);
+ String cfName = idx >= 0 ? metadata.name.substring(0, idx) : metadata.name;
+ String indexNameWithDot = idx >= 0 ? metadata.name.substring(idx) : null;
this.dataPaths = new File[paths.length];
// If upgraded from version less than 2.1, use existing directories
- String oldSSTableRelativePath = join(metadata.ksName, cfName);
+ String oldSSTableRelativePath = join(metadata.keyspace, cfName);
for (int i = 0; i < paths.length; ++i)
{
// check if old SSTable directory exists
@@ -223,10 +221,10 @@
{
canonicalPathsBuilder = ImmutableMap.builder();
// use 2.1+ style
- String newSSTableRelativePath = join(metadata.ksName, cfName + '-' + cfId);
+ String newSSTableRelativePath = join(metadata.keyspace, cfName + '-' + tableId);
for (int i = 0; i < paths.length; ++i)
{
- File dataPath = new File(paths[i].location, newSSTableRelativePath);;
+ File dataPath = new File(paths[i].location, newSSTableRelativePath);
dataPaths[i] = dataPath;
canonicalPathsBuilder.put(Paths.get(FileUtils.getCanonicalPath(dataPath)), paths[i]);
}
@@ -270,9 +268,8 @@
if (file.isDirectory())
return false;
- Pair<Descriptor, Component> pair = SSTable.tryComponentFromFilename(file.getParentFile(),
- file.getName());
- return pair != null && pair.left.ksname.equals(metadata.ksName) && pair.left.cfname.equals(metadata.cfName);
+ Descriptor desc = SSTable.tryDescriptorFromFilename(file);
+ return desc != null && desc.ksname.equals(metadata.keyspace) && desc.cfname.equals(metadata.name);
}
});
@@ -318,8 +315,9 @@
{
for (File dir : dataPaths)
{
- if (new File(dir, filename).exists())
- return Descriptor.fromFilename(dir, filename).left;
+ File file = new File(dir, filename);
+ if (file.exists())
+ return Descriptor.fromFilename(file);
}
return null;
}
@@ -350,6 +348,41 @@
}
/**
+ * Returns a data directory to load the file {@code sourceFile}. If the sourceFile is on same disk partition as any
+ * data directory then use that one as data directory otherwise use {@link #getWriteableLocationAsFile(long)} to
+ * find suitable data directory.
+ *
+ * Also makes sure returned directory is not disallowed.
+ *
+ * @throws FSWriteError if all directories are disallowed.
+ */
+ public File getWriteableLocationToLoadFile(final File sourceFile)
+ {
+ try
+ {
+ final FileStore srcFileStore = Files.getFileStore(sourceFile.toPath());
+ for (final File dataPath : dataPaths)
+ {
+ if (DisallowedDirectories.isUnwritable(dataPath))
+ {
+ continue;
+ }
+
+ if (Files.getFileStore(dataPath.toPath()).equals(srcFileStore))
+ {
+ return dataPath;
+ }
+ }
+ }
+ catch (final IOException e)
+ {
+ // pass exceptions in finding filestore. This is best effort anyway. Fall back on getWriteableLocationAsFile()
+ }
+
+ return getWriteableLocationAsFile(sourceFile.length());
+ }
+
+ /**
* Returns a temporary subdirectory on allowed data directory
* that _currently_ has {@code writeSize} bytes as usable space.
* This method does not create the temporary directory.
@@ -658,10 +691,15 @@
public SSTableLister sstableLister(OnTxnErr onTxnErr)
{
- return new SSTableLister(onTxnErr);
+ return new SSTableLister(this.dataPaths, this.metadata, onTxnErr);
}
- public class SSTableLister
+ public SSTableLister sstableLister(File directory, OnTxnErr onTxnErr)
+ {
+ return new SSTableLister(new File[]{directory}, metadata, onTxnErr);
+ }
+
+ public static class SSTableLister
{
private final OnTxnErr onTxnErr;
private boolean skipTemporary;
@@ -671,9 +709,13 @@
private final Map<Descriptor, Set<Component>> components = new HashMap<>();
private boolean filtered;
private String snapshotName;
+ private final File[] dataPaths;
+ private final TableMetadata metadata;
- private SSTableLister(OnTxnErr onTxnErr)
+ private SSTableLister(File[] dataPaths, TableMetadata metadata, OnTxnErr onTxnErr)
{
+ this.dataPaths = dataPaths;
+ this.metadata = metadata;
this.onTxnErr = onTxnErr;
}
@@ -756,7 +798,7 @@
filtered = true;
}
- private BiFunction<File, FileType, Boolean> getFilter()
+ private BiPredicate<File, FileType> getFilter()
{
// This function always return false since it adds to the components map
return (file, type) ->
@@ -770,12 +812,12 @@
return false;
case FINAL:
- Pair<Descriptor, Component> pair = SSTable.tryComponentFromFilename(file.getParentFile(), file.getName());
+ Pair<Descriptor, Component> pair = SSTable.tryComponentFromFilename(file);
if (pair == null)
return false;
// we are only interested in the SSTable files that belong to the specific ColumnFamily
- if (!pair.left.ksname.equals(metadata.ksName) || !pair.left.cfname.equals(metadata.cfName))
+ if (!pair.left.ksname.equals(metadata.keyspace) || !pair.left.cfname.equals(metadata.name))
return false;
Set<Component> previous = components.get(pair.left);
@@ -784,24 +826,6 @@
previous = new HashSet<>();
components.put(pair.left, previous);
}
- else if (pair.right.type == Component.Type.DIGEST)
- {
- if (pair.right != pair.left.digestComponent)
- {
- // Need to update the DIGEST component as it might be set to another
- // digest type as a guess. This may happen if the first component is
- // not the DIGEST (but the Data component for example), so the digest
- // type is _guessed_ from the Version.
- // Although the Version explicitly defines the digest type, it doesn't
- // seem to be true under all circumstances. Generated sstables from a
- // post 2.1.8 snapshot produced Digest.sha1 files although Version
- // defines Adler32.
- // TL;DR this piece of code updates the digest component to be "correct".
- components.remove(pair.left);
- Descriptor updated = pair.left.withDigestComponent(pair.right);
- components.put(updated, previous);
- }
- }
previous.add(pair.right);
nbFiles++;
return false;
@@ -818,18 +842,19 @@
* @return Return a map of all snapshots to space being used
* The pair for a snapshot has size on disk and true size.
*/
- public Map<String, Pair<Long, Long>> getSnapshotDetails()
+ public Map<String, SnapshotSizeDetails> getSnapshotDetails()
{
- final Map<String, Pair<Long, Long>> snapshotSpaceMap = new HashMap<>();
- for (File snapshot : listSnapshots())
+ List<File> snapshots = listSnapshots();
+ final Map<String, SnapshotSizeDetails> snapshotSpaceMap = Maps.newHashMapWithExpectedSize(snapshots.size());
+ for (File snapshot : snapshots)
{
final long sizeOnDisk = FileUtils.folderSize(snapshot);
final long trueSize = getTrueAllocatedSizeIn(snapshot);
- Pair<Long, Long> spaceUsed = snapshotSpaceMap.get(snapshot.getName());
+ SnapshotSizeDetails spaceUsed = snapshotSpaceMap.get(snapshot.getName());
if (spaceUsed == null)
- spaceUsed = Pair.create(sizeOnDisk,trueSize);
+ spaceUsed = new SnapshotSizeDetails(sizeOnDisk,trueSize);
else
- spaceUsed = Pair.create(spaceUsed.left + sizeOnDisk, spaceUsed.right + trueSize);
+ spaceUsed = new SnapshotSizeDetails(spaceUsed.sizeOnDiskBytes + sizeOnDisk, spaceUsed.dataSizeBytes + trueSize);
snapshotSpaceMap.put(snapshot.getName(), spaceUsed);
}
return snapshotSpaceMap;
@@ -968,7 +993,7 @@
}
catch (IOException e)
{
- logger.error("Could not calculate the size of {}. {}", input, e);
+ logger.error("Could not calculate the size of {}. {}", input, e.getMessage());
}
return visitor.getAllocatedSize();
@@ -1063,11 +1088,39 @@
public boolean isAcceptable(Path path)
{
File file = path.toFile();
- Pair<Descriptor, Component> pair = SSTable.tryComponentFromFilename(path.getParent().toFile(), file.getName());
- return pair != null
- && pair.left.ksname.equals(metadata.ksName)
- && pair.left.cfname.equals(metadata.cfName)
- && !toSkip.contains(file);
+ Descriptor desc = SSTable.tryDescriptorFromFilename(file);
+ return desc != null
+ && desc.ksname.equals(metadata.keyspace)
+ && desc.cfname.equals(metadata.name)
+ && !toSkip.contains(file);
+ }
+ }
+
+ public static class SnapshotSizeDetails
+ {
+ final long sizeOnDiskBytes;
+ final long dataSizeBytes;
+
+ private SnapshotSizeDetails(long sizeOnDiskBytes, long dataSizeBytes)
+ {
+ this.sizeOnDiskBytes = sizeOnDiskBytes;
+ this.dataSizeBytes = dataSizeBytes;
+ }
+
+ @Override
+ public final int hashCode()
+ {
+ int hashCode = (int) sizeOnDiskBytes ^ (int) (sizeOnDiskBytes >>> 32);
+ return 31 * (hashCode ^ (int) ((int) dataSizeBytes ^ (dataSizeBytes >>> 32)));
+ }
+
+ @Override
+ public final boolean equals(Object o)
+ {
+ if(!(o instanceof SnapshotSizeDetails))
+ return false;
+ SnapshotSizeDetails that = (SnapshotSizeDetails)o;
+ return sizeOnDiskBytes == that.sizeOnDiskBytes && dataSizeBytes == that.dataSizeBytes;
}
}
}
diff --git a/src/java/org/apache/cassandra/db/DiskBoundaries.java b/src/java/org/apache/cassandra/db/DiskBoundaries.java
index cb046eb..5b377e2 100644
--- a/src/java/org/apache/cassandra/db/DiskBoundaries.java
+++ b/src/java/org/apache/cassandra/db/DiskBoundaries.java
@@ -24,6 +24,7 @@
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableList;
+import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.service.StorageService;
@@ -104,7 +105,7 @@
{
if (positions == null)
{
- return getBoundariesFromSSTableDirectory(sstable);
+ return getBoundariesFromSSTableDirectory(sstable.descriptor);
}
int pos = Collections.binarySearch(positions, sstable.first);
@@ -115,9 +116,9 @@
/**
* Try to figure out location based on sstable directory
*/
- private int getBoundariesFromSSTableDirectory(SSTableReader sstable)
+ public int getBoundariesFromSSTableDirectory(Descriptor descriptor)
{
- Directories.DataDirectory actualDirectory = cfs.getDirectories().getDataDirectoryForFile(sstable.descriptor);
+ Directories.DataDirectory actualDirectory = cfs.getDirectories().getDataDirectoryForFile(descriptor);
for (int i = 0; i < directories.size(); i++)
{
Directories.DataDirectory directory = directories.get(i);
@@ -131,4 +132,19 @@
{
return directories.get(getDiskIndex(sstable));
}
-}
\ No newline at end of file
+
+ public Directories.DataDirectory getCorrectDiskForKey(DecoratedKey key)
+ {
+ if (positions == null)
+ return null;
+
+ return directories.get(getDiskIndex(key));
+ }
+
+ private int getDiskIndex(DecoratedKey key)
+ {
+ int pos = Collections.binarySearch(positions, key);
+ assert pos < 0;
+ return -pos - 1;
+ }
+}
diff --git a/src/java/org/apache/cassandra/db/DiskBoundaryManager.java b/src/java/org/apache/cassandra/db/DiskBoundaryManager.java
index 51343ad..bbb6dbb 100644
--- a/src/java/org/apache/cassandra/db/DiskBoundaryManager.java
+++ b/src/java/org/apache/cassandra/db/DiskBoundaryManager.java
@@ -19,8 +19,9 @@
package org.apache.cassandra.db;
import java.util.ArrayList;
-import java.util.Collection;
+import java.util.Comparator;
import java.util.List;
+import java.util.stream.Collectors;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -30,6 +31,8 @@
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Splitter;
import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.locator.RangesAtEndpoint;
+import org.apache.cassandra.locator.Replica;
import org.apache.cassandra.locator.TokenMetadata;
import org.apache.cassandra.service.PendingRangeCalculatorService;
import org.apache.cassandra.service.StorageService;
@@ -68,7 +71,7 @@
private static DiskBoundaries getDiskBoundaryValue(ColumnFamilyStore cfs)
{
- Collection<Range<Token>> localRanges;
+ RangesAtEndpoint localRanges;
long ringVersion;
TokenMetadata tmd;
@@ -80,14 +83,14 @@
&& !StorageService.isReplacingSameAddress()) // When replacing same address, the node marks itself as UN locally
{
PendingRangeCalculatorService.instance.blockUntilFinished();
- localRanges = tmd.getPendingRanges(cfs.keyspace.getName(), FBUtilities.getBroadcastAddress());
+ localRanges = tmd.getPendingRanges(cfs.keyspace.getName(), FBUtilities.getBroadcastAddressAndPort());
}
else
{
// Reason we use use the future settled TMD is that if we decommission a node, we want to stream
// from that node to the correct location on disk, if we didn't, we would put new files in the wrong places.
// We do this to minimize the amount of data we need to move in rebalancedisks once everything settled
- localRanges = cfs.keyspace.getReplicationStrategy().getAddressRanges(tmd.cloneAfterAllSettled()).get(FBUtilities.getBroadcastAddress());
+ localRanges = cfs.keyspace.getReplicationStrategy().getAddressReplicas(tmd.cloneAfterAllSettled(), FBUtilities.getBroadcastAddressAndPort());
}
logger.debug("Got local ranges {} (ringVersion = {})", localRanges, ringVersion);
}
@@ -106,9 +109,8 @@
if (localRanges == null || localRanges.isEmpty())
return new DiskBoundaries(cfs, dirs, null, ringVersion, directoriesVersion);
- List<Range<Token>> sortedLocalRanges = Range.sort(localRanges);
+ List<PartitionPosition> positions = getDiskBoundaries(localRanges, cfs.getPartitioner(), dirs);
- List<PartitionPosition> positions = getDiskBoundaries(sortedLocalRanges, cfs.getPartitioner(), dirs);
return new DiskBoundaries(cfs, dirs, positions, ringVersion, directoriesVersion);
}
@@ -121,15 +123,27 @@
*
* The final entry in the returned list will always be the partitioner maximum tokens upper key bound
*/
- private static List<PartitionPosition> getDiskBoundaries(List<Range<Token>> sortedLocalRanges, IPartitioner partitioner, Directories.DataDirectory[] dataDirectories)
+ private static List<PartitionPosition> getDiskBoundaries(RangesAtEndpoint replicas, IPartitioner partitioner, Directories.DataDirectory[] dataDirectories)
{
assert partitioner.splitter().isPresent();
+
Splitter splitter = partitioner.splitter().get();
boolean dontSplitRanges = DatabaseDescriptor.getNumTokens() > 1;
- List<Token> boundaries = splitter.splitOwnedRanges(dataDirectories.length, sortedLocalRanges, dontSplitRanges);
+
+ List<Splitter.WeightedRange> weightedRanges = new ArrayList<>(replicas.size());
+ // note that Range.sort unwraps any wraparound ranges, so we need to sort them here
+ for (Range<Token> r : Range.sort(replicas.onlyFull().ranges()))
+ weightedRanges.add(new Splitter.WeightedRange(1.0, r));
+
+ for (Range<Token> r : Range.sort(replicas.onlyTransient().ranges()))
+ weightedRanges.add(new Splitter.WeightedRange(0.1, r));
+
+ weightedRanges.sort(Comparator.comparing(Splitter.WeightedRange::left));
+
+ List<Token> boundaries = splitter.splitOwnedRanges(dataDirectories.length, weightedRanges, dontSplitRanges);
// If we can't split by ranges, split evenly to ensure utilisation of all disks
if (dontSplitRanges && boundaries.size() < dataDirectories.length)
- boundaries = splitter.splitOwnedRanges(dataDirectories.length, sortedLocalRanges, false);
+ boundaries = splitter.splitOwnedRanges(dataDirectories.length, weightedRanges, false);
List<PartitionPosition> diskBoundaries = new ArrayList<>();
for (int i = 0; i < boundaries.size() - 1; i++)
diff --git a/src/java/org/apache/cassandra/db/EmptyIterators.java b/src/java/org/apache/cassandra/db/EmptyIterators.java
index 6bf8fff..04ff31b 100644
--- a/src/java/org/apache/cassandra/db/EmptyIterators.java
+++ b/src/java/org/apache/cassandra/db/EmptyIterators.java
@@ -20,7 +20,7 @@
import java.util.NoSuchElementException;
-import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.db.partitions.BasePartitionIterator;
import org.apache.cassandra.db.partitions.PartitionIterator;
import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
@@ -52,21 +52,14 @@
private static class EmptyUnfilteredPartitionIterator extends EmptyBasePartitionIterator<UnfilteredRowIterator> implements UnfilteredPartitionIterator
{
- final CFMetaData metadata;
- final boolean isForThrift;
+ final TableMetadata metadata;
- public EmptyUnfilteredPartitionIterator(CFMetaData metadata, boolean isForThrift)
+ public EmptyUnfilteredPartitionIterator(TableMetadata metadata)
{
this.metadata = metadata;
- this.isForThrift = isForThrift;
}
- public boolean isForThrift()
- {
- return isForThrift;
- }
-
- public CFMetaData metadata()
+ public TableMetadata metadata()
{
return metadata;
}
@@ -83,13 +76,13 @@
private static class EmptyBaseRowIterator<U extends Unfiltered> implements BaseRowIterator<U>
{
- final PartitionColumns columns;
- final CFMetaData metadata;
+ final RegularAndStaticColumns columns;
+ final TableMetadata metadata;
final DecoratedKey partitionKey;
final boolean isReverseOrder;
final Row staticRow;
- EmptyBaseRowIterator(PartitionColumns columns, CFMetaData metadata, DecoratedKey partitionKey, boolean isReverseOrder, Row staticRow)
+ EmptyBaseRowIterator(RegularAndStaticColumns columns, TableMetadata metadata, DecoratedKey partitionKey, boolean isReverseOrder, Row staticRow)
{
this.columns = columns;
this.metadata = metadata;
@@ -98,7 +91,7 @@
this.staticRow = staticRow;
}
- public CFMetaData metadata()
+ public TableMetadata metadata()
{
return metadata;
}
@@ -108,7 +101,7 @@
return isReverseOrder;
}
- public PartitionColumns columns()
+ public RegularAndStaticColumns columns()
{
return columns;
}
@@ -146,7 +139,7 @@
private static class EmptyUnfilteredRowIterator extends EmptyBaseRowIterator<Unfiltered> implements UnfilteredRowIterator
{
final DeletionTime partitionLevelDeletion;
- public EmptyUnfilteredRowIterator(PartitionColumns columns, CFMetaData metadata, DecoratedKey partitionKey,
+ public EmptyUnfilteredRowIterator(RegularAndStaticColumns columns, TableMetadata metadata, DecoratedKey partitionKey,
boolean isReverseOrder, Row staticRow, DeletionTime partitionLevelDeletion)
{
super(columns, metadata, partitionKey, isReverseOrder, staticRow);
@@ -171,15 +164,15 @@
private static class EmptyRowIterator extends EmptyBaseRowIterator<Row> implements RowIterator
{
- public EmptyRowIterator(CFMetaData metadata, DecoratedKey partitionKey, boolean isReverseOrder, Row staticRow)
+ public EmptyRowIterator(TableMetadata metadata, DecoratedKey partitionKey, boolean isReverseOrder, Row staticRow)
{
- super(PartitionColumns.NONE, metadata, partitionKey, isReverseOrder, staticRow);
+ super(RegularAndStaticColumns.NONE, metadata, partitionKey, isReverseOrder, staticRow);
}
}
- public static UnfilteredPartitionIterator unfilteredPartition(CFMetaData metadata, boolean isForThrift)
+ public static UnfilteredPartitionIterator unfilteredPartition(TableMetadata metadata)
{
- return new EmptyUnfilteredPartitionIterator(metadata, isForThrift);
+ return new EmptyUnfilteredPartitionIterator(metadata);
}
public static PartitionIterator partition()
@@ -188,11 +181,11 @@
}
// this method is the only one that can return a non-empty iterator, but it still has no rows, so it seems cleanest to keep it here
- public static UnfilteredRowIterator unfilteredRow(CFMetaData metadata, DecoratedKey partitionKey, boolean isReverseOrder, Row staticRow, DeletionTime partitionDeletion)
+ public static UnfilteredRowIterator unfilteredRow(TableMetadata metadata, DecoratedKey partitionKey, boolean isReverseOrder, Row staticRow, DeletionTime partitionDeletion)
{
- PartitionColumns columns = PartitionColumns.NONE;
+ RegularAndStaticColumns columns = RegularAndStaticColumns.NONE;
if (!staticRow.isEmpty())
- columns = new PartitionColumns(Columns.from(staticRow.columns()), Columns.NONE);
+ columns = new RegularAndStaticColumns(Columns.from(staticRow.columns()), Columns.NONE);
else
staticRow = Rows.EMPTY_STATIC_ROW;
@@ -202,12 +195,12 @@
return new EmptyUnfilteredRowIterator(columns, metadata, partitionKey, isReverseOrder, staticRow, partitionDeletion);
}
- public static UnfilteredRowIterator unfilteredRow(CFMetaData metadata, DecoratedKey partitionKey, boolean isReverseOrder)
+ public static UnfilteredRowIterator unfilteredRow(TableMetadata metadata, DecoratedKey partitionKey, boolean isReverseOrder)
{
- return new EmptyUnfilteredRowIterator(PartitionColumns.NONE, metadata, partitionKey, isReverseOrder, Rows.EMPTY_STATIC_ROW, DeletionTime.LIVE);
+ return new EmptyUnfilteredRowIterator(RegularAndStaticColumns.NONE, metadata, partitionKey, isReverseOrder, Rows.EMPTY_STATIC_ROW, DeletionTime.LIVE);
}
- public static RowIterator row(CFMetaData metadata, DecoratedKey partitionKey, boolean isReverseOrder)
+ public static RowIterator row(TableMetadata metadata, DecoratedKey partitionKey, boolean isReverseOrder)
{
return new EmptyRowIterator(metadata, partitionKey, isReverseOrder, Rows.EMPTY_STATIC_ROW);
}
diff --git a/src/java/org/apache/cassandra/db/ExpirationDateOverflowHandling.java b/src/java/org/apache/cassandra/db/ExpirationDateOverflowHandling.java
index 852dcb1..81e3d1e 100644
--- a/src/java/org/apache/cassandra/db/ExpirationDateOverflowHandling.java
+++ b/src/java/org/apache/cassandra/db/ExpirationDateOverflowHandling.java
@@ -25,17 +25,16 @@
import org.slf4j.LoggerFactory;
import org.slf4j.helpers.MessageFormatter;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.cql3.Attributes;
import org.apache.cassandra.db.rows.BufferCell;
import org.apache.cassandra.db.rows.Cell;
import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.service.ClientWarn;
import org.apache.cassandra.utils.NoSpamLogger;
public class ExpirationDateOverflowHandling
{
- private static final Logger logger = LoggerFactory.getLogger(Attributes.class);
+ private static final Logger logger = LoggerFactory.getLogger(ExpirationDateOverflowHandling.class);
private static final int EXPIRATION_OVERFLOW_WARNING_INTERVAL_MINUTES = Integer.getInteger("cassandra.expiration_overflow_warning_interval_minutes", 5);
@@ -70,7 +69,7 @@
"the expiration date overflow policy or upgrade to a version where this limitation " +
"is fixed. See CASSANDRA-14092 for more details.";
- public static void maybeApplyExpirationDateOverflowPolicy(CFMetaData metadata, int ttl, boolean isDefaultTTL) throws InvalidRequestException
+ public static void maybeApplyExpirationDateOverflowPolicy(TableMetadata metadata, int ttl, boolean isDefaultTTL) throws InvalidRequestException
{
if (ttl == BufferCell.NO_TTL)
return;
@@ -82,8 +81,8 @@
switch (policy)
{
case CAP:
- ClientWarn.instance.warn(MessageFormatter.arrayFormat(MAXIMUM_EXPIRATION_DATE_EXCEEDED_WARNING, new Object[] { metadata.ksName,
- metadata.cfName,
+ ClientWarn.instance.warn(MessageFormatter.arrayFormat(MAXIMUM_EXPIRATION_DATE_EXCEEDED_WARNING, new Object[] { metadata.keyspace,
+ metadata.name,
isDefaultTTL? "default " : "", ttl })
.getMessage());
case CAP_NOWARN:
@@ -93,11 +92,11 @@
* to {@link org.apache.cassandra.db.BufferExpiringCell#MAX_DELETION_TIME}
*/
NoSpamLogger.log(logger, NoSpamLogger.Level.WARN, EXPIRATION_OVERFLOW_WARNING_INTERVAL_MINUTES, TimeUnit.MINUTES, MAXIMUM_EXPIRATION_DATE_EXCEEDED_WARNING,
- metadata.ksName, metadata.cfName, isDefaultTTL? "default " : "", ttl);
+ metadata.keyspace, metadata.name, isDefaultTTL? "default " : "", ttl);
return;
default:
- throw new InvalidRequestException(String.format(MAXIMUM_EXPIRATION_DATE_EXCEEDED_REJECT_MESSAGE, metadata.ksName, metadata.cfName,
+ throw new InvalidRequestException(String.format(MAXIMUM_EXPIRATION_DATE_EXCEEDED_REJECT_MESSAGE, metadata.keyspace, metadata.name,
isDefaultTTL? "default " : "", ttl));
}
}
@@ -108,7 +107,7 @@
* which is {@link Cell#MAX_DELETION_TIME}.
*
* Please note that the {@link ExpirationDateOverflowHandling.ExpirationDateOverflowPolicy} is applied
- * during {@link ExpirationDateOverflowHandling#maybeApplyExpirationDateOverflowPolicy(CFMetaData, int, boolean)},
+ * during {@link ExpirationDateOverflowHandling#maybeApplyExpirationDateOverflowPolicy(org.apache.cassandra.schema.TableMetadata, int, boolean)},
* so if the request was not denied it means its expiration date should be capped.
*
* See CASSANDRA-14092
diff --git a/src/java/org/apache/cassandra/db/IMutation.java b/src/java/org/apache/cassandra/db/IMutation.java
index c734e16..10472c1 100644
--- a/src/java/org/apache/cassandra/db/IMutation.java
+++ b/src/java/org/apache/cassandra/db/IMutation.java
@@ -18,17 +18,53 @@
package org.apache.cassandra.db;
import java.util.Collection;
-import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.partitions.PartitionUpdate;
+import org.apache.cassandra.schema.TableId;
public interface IMutation
{
+ public long MAX_MUTATION_SIZE = DatabaseDescriptor.getMaxMutationSize();
+
public void apply();
public String getKeyspaceName();
- public Collection<UUID> getColumnFamilyIds();
+ public Collection<TableId> getTableIds();
public DecoratedKey key();
- public long getTimeout();
+ public long getTimeout(TimeUnit unit);
public String toString(boolean shallow);
public Collection<PartitionUpdate> getPartitionUpdates();
+
+ public default void validateIndexedColumns()
+ {
+ for (PartitionUpdate pu : getPartitionUpdates())
+ pu.validateIndexedColumns();
+ }
+
+ /**
+ * Validates size of mutation does not exceed {@link DatabaseDescriptor#getMaxMutationSize()}.
+ *
+ * @param version the MessagingService version the mutation is being serialized for.
+ * see {@link org.apache.cassandra.net.MessagingService#current_version}
+ * @param overhead overhadd to add for mutation size to validate. Pass zero if not required but not a negative value.
+ * @throws {@link MutationExceededMaxSizeException} if {@link DatabaseDescriptor#getMaxMutationSize()} is exceeded
+ */
+ public void validateSize(int version, int overhead);
+
+ /**
+ * Computes the total data size of the specified mutations.
+ * @param mutations the mutations
+ * @return the total data size of the specified mutations
+ */
+ public static long dataSize(Collection<? extends IMutation> mutations)
+ {
+ long size = 0;
+ for (IMutation mutation : mutations)
+ {
+ for (PartitionUpdate update : mutation.getPartitionUpdates())
+ size += update.dataSize();
+ }
+ return size;
+ }
}
diff --git a/src/java/org/apache/cassandra/db/Keyspace.java b/src/java/org/apache/cassandra/db/Keyspace.java
index 5e39823..fc4f56f 100644
--- a/src/java/org/apache/cassandra/db/Keyspace.java
+++ b/src/java/org/apache/cassandra/db/Keyspace.java
@@ -30,13 +30,11 @@
import org.slf4j.LoggerFactory;
import org.apache.cassandra.concurrent.Stage;
-import org.apache.cassandra.concurrent.StageManager;
import org.apache.cassandra.config.*;
-import org.apache.cassandra.db.commitlog.CommitLog;
-import org.apache.cassandra.db.commitlog.CommitLogPosition;
import org.apache.cassandra.db.compaction.CompactionManager;
import org.apache.cassandra.db.lifecycle.SSTableSet;
import org.apache.cassandra.db.partitions.PartitionUpdate;
+import org.apache.cassandra.db.repair.CassandraKeyspaceRepairManager;
import org.apache.cassandra.db.view.ViewManager;
import org.apache.cassandra.exceptions.WriteTimeoutException;
import org.apache.cassandra.index.Index;
@@ -45,13 +43,21 @@
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.locator.AbstractReplicationStrategy;
import org.apache.cassandra.metrics.KeyspaceMetrics;
+import org.apache.cassandra.repair.KeyspaceRepairManager;
import org.apache.cassandra.schema.KeyspaceMetadata;
import org.apache.cassandra.schema.ReplicationParams;
-import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.SchemaConstants;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.schema.TableMetadataRef;
import org.apache.cassandra.tracing.Tracing;
import org.apache.cassandra.utils.*;
import org.apache.cassandra.utils.concurrent.OpOrder;
+import static java.util.concurrent.TimeUnit.*;
+import static org.apache.cassandra.utils.MonotonicClock.approxTime;
+
/**
* It represents a Keyspace.
*/
@@ -80,10 +86,13 @@
public static final OpOrder writeOrder = new OpOrder();
/* ColumnFamilyStore per column family */
- private final ConcurrentMap<UUID, ColumnFamilyStore> columnFamilyStores = new ConcurrentHashMap<>();
+ private final ConcurrentMap<TableId, ColumnFamilyStore> columnFamilyStores = new ConcurrentHashMap<>();
+
private volatile AbstractReplicationStrategy replicationStrategy;
public final ViewManager viewManager;
+ private final KeyspaceWriteHandler writeHandler;
private volatile ReplicationParams replicationParams;
+ private final KeyspaceRepairManager repairManager;
public static final Function<String,Keyspace> keyspaceTransformer = new Function<String, Keyspace>()
{
@@ -154,9 +163,14 @@
}
}
- public static ColumnFamilyStore openAndGetStore(CFMetaData cfm)
+ public static ColumnFamilyStore openAndGetStore(TableMetadataRef tableRef)
{
- return open(cfm.ksName).getColumnFamilyStore(cfm.cfId);
+ return open(tableRef.keyspace).getColumnFamilyStore(tableRef.id);
+ }
+
+ public static ColumnFamilyStore openAndGetStore(TableMetadata table)
+ {
+ return open(table.keyspace).getColumnFamilyStore(table.id);
}
/**
@@ -193,13 +207,13 @@
public ColumnFamilyStore getColumnFamilyStore(String cfName)
{
- UUID id = Schema.instance.getId(getName(), cfName);
- if (id == null)
+ TableMetadata table = Schema.instance.getTableMetadata(getName(), cfName);
+ if (table == null)
throw new IllegalArgumentException(String.format("Unknown keyspace/cf pair (%s.%s)", getName(), cfName));
- return getColumnFamilyStore(id);
+ return getColumnFamilyStore(table.id);
}
- public ColumnFamilyStore getColumnFamilyStore(UUID id)
+ public ColumnFamilyStore getColumnFamilyStore(TableId id)
{
ColumnFamilyStore cfs = columnFamilyStores.get(id);
if (cfs == null)
@@ -207,7 +221,7 @@
return cfs;
}
- public boolean hasColumnFamilyStore(UUID id)
+ public boolean hasColumnFamilyStore(TableId id)
{
return columnFamilyStores.containsKey(id);
}
@@ -295,7 +309,7 @@
*/
public static void clearSnapshot(String snapshotName, String keyspace)
{
- List<File> snapshotDirs = Directories.getKSChildDirectories(keyspace, ColumnFamilyStore.getInitialDirectories());
+ List<File> snapshotDirs = Directories.getKSChildDirectories(keyspace);
Directories.clearSnapshot(snapshotName, snapshotDirs);
}
@@ -312,18 +326,23 @@
private Keyspace(String keyspaceName, boolean loadSSTables)
{
- metadata = Schema.instance.getKSMetaData(keyspaceName);
+ metadata = Schema.instance.getKeyspaceMetadata(keyspaceName);
assert metadata != null : "Unknown keyspace " + keyspaceName;
+ if (metadata.isVirtual())
+ throw new IllegalStateException("Cannot initialize Keyspace with virtual metadata " + keyspaceName);
createReplicationStrategy(metadata);
this.metric = new KeyspaceMetrics(this);
this.viewManager = new ViewManager(this);
- for (CFMetaData cfm : metadata.tablesAndViews())
+ for (TableMetadata cfm : metadata.tablesAndViews())
{
- logger.trace("Initializing {}.{}", getName(), cfm.cfName);
- initCf(cfm, loadSSTables);
+ logger.trace("Initializing {}.{}", getName(), cfm.name);
+ initCf(Schema.instance.getTableMetadataRef(cfm.id), loadSSTables);
}
- this.viewManager.reload();
+ this.viewManager.reload(false);
+
+ this.repairManager = new CassandraKeyspaceRepairManager(this);
+ this.writeHandler = new CassandraKeyspaceWriteHandler(this);
}
private Keyspace(KeyspaceMetadata metadata)
@@ -332,6 +351,13 @@
createReplicationStrategy(metadata);
this.metric = new KeyspaceMetrics(this);
this.viewManager = new ViewManager(this);
+ this.repairManager = new CassandraKeyspaceRepairManager(this);
+ this.writeHandler = new CassandraKeyspaceWriteHandler(this);
+ }
+
+ public KeyspaceRepairManager getRepairManager()
+ {
+ return repairManager;
}
public static Keyspace mockKS(KeyspaceMetadata metadata)
@@ -341,11 +367,8 @@
private void createReplicationStrategy(KeyspaceMetadata ksm)
{
- replicationStrategy = AbstractReplicationStrategy.createReplicationStrategy(ksm.name,
- ksm.params.replication.klass,
- StorageService.instance.getTokenMetadata(),
- DatabaseDescriptor.getEndpointSnitch(),
- ksm.params.replication.options);
+ logger.info("Creating replication strategy " + ksm.name + " params " + ksm.params);
+ replicationStrategy = ksm.createReplicationStrategy();
if (!ksm.params.replication.equals(replicationParams))
{
logger.debug("New replication settings for keyspace {} - invalidating disk boundary caches", ksm.name);
@@ -355,15 +378,15 @@
}
// best invoked on the compaction mananger.
- public void dropCf(UUID cfId)
+ public void dropCf(TableId tableId)
{
- assert columnFamilyStores.containsKey(cfId);
- ColumnFamilyStore cfs = columnFamilyStores.remove(cfId);
+ assert columnFamilyStores.containsKey(tableId);
+ ColumnFamilyStore cfs = columnFamilyStores.remove(tableId);
if (cfs == null)
return;
cfs.getCompactionStrategyManager().shutdown();
- CompactionManager.instance.interruptCompactionForCFs(cfs.concatWithIndexes(), true);
+ CompactionManager.instance.interruptCompactionForCFs(cfs.concatWithIndexes(), (sstable) -> true, true);
// wait for any outstanding reads/writes that might affect the CFS
cfs.keyspace.writeOrder.awaitNewBarrier();
cfs.readOrdering.awaitNewBarrier();
@@ -384,17 +407,17 @@
*/
public void initCfCustom(ColumnFamilyStore newCfs)
{
- ColumnFamilyStore cfs = columnFamilyStores.get(newCfs.metadata.cfId);
+ ColumnFamilyStore cfs = columnFamilyStores.get(newCfs.metadata.id);
if (cfs == null)
{
// CFS being created for the first time, either on server startup or new CF being added.
// We don't worry about races here; startup is safe, and adding multiple idential CFs
// simultaneously is a "don't do that" scenario.
- ColumnFamilyStore oldCfs = columnFamilyStores.putIfAbsent(newCfs.metadata.cfId, newCfs);
+ ColumnFamilyStore oldCfs = columnFamilyStores.putIfAbsent(newCfs.metadata.id, newCfs);
// CFS mbean instantiation will error out before we hit this, but in case that changes...
if (oldCfs != null)
- throw new IllegalStateException("added multiple mappings for cf id " + newCfs.metadata.cfId);
+ throw new IllegalStateException("added multiple mappings for cf id " + newCfs.metadata.id);
}
else
{
@@ -402,28 +425,33 @@
}
}
+ public KeyspaceWriteHandler getWriteHandler()
+ {
+ return writeHandler;
+ }
+
/**
* adds a cf to internal structures, ends up creating disk files).
*/
- public void initCf(CFMetaData metadata, boolean loadSSTables)
+ public void initCf(TableMetadataRef metadata, boolean loadSSTables)
{
- ColumnFamilyStore cfs = columnFamilyStores.get(metadata.cfId);
+ ColumnFamilyStore cfs = columnFamilyStores.get(metadata.id);
if (cfs == null)
{
// CFS being created for the first time, either on server startup or new CF being added.
// We don't worry about races here; startup is safe, and adding multiple idential CFs
// simultaneously is a "don't do that" scenario.
- ColumnFamilyStore oldCfs = columnFamilyStores.putIfAbsent(metadata.cfId, ColumnFamilyStore.createColumnFamilyStore(this, metadata, loadSSTables));
+ ColumnFamilyStore oldCfs = columnFamilyStores.putIfAbsent(metadata.id, ColumnFamilyStore.createColumnFamilyStore(this, metadata, loadSSTables));
// CFS mbean instantiation will error out before we hit this, but in case that changes...
if (oldCfs != null)
- throw new IllegalStateException("added multiple mappings for cf id " + metadata.cfId);
+ throw new IllegalStateException("added multiple mappings for cf id " + metadata.id);
}
else
{
// re-initializing an existing CF. This will happen if you cleared the schema
// on this node and it's getting repopulated from the rest of the cluster.
- assert cfs.name.equals(metadata.cfName);
+ assert cfs.name.equals(metadata.name);
cfs.reload();
}
}
@@ -457,17 +485,16 @@
*
* @param mutation the row to write. Must not be modified after calling apply, since commitlog append
* may happen concurrently, depending on the CL Executor type.
- * @param writeCommitLog false to disable commitlog append entirely
+ * @param makeDurable if true, don't return unless write has been made durable
* @param updateIndexes false to disable index updates (used by CollationController "defragmenting")
* @param isDroppable true if this should throw WriteTimeoutException if it does not acquire lock within write_request_timeout_in_ms
- * @throws ExecutionException
*/
public void apply(final Mutation mutation,
- final boolean writeCommitLog,
+ final boolean makeDurable,
boolean updateIndexes,
boolean isDroppable)
{
- applyInternal(mutation, writeCommitLog, updateIndexes, isDroppable, false, null);
+ applyInternal(mutation, makeDurable, updateIndexes, isDroppable, false, null);
}
/**
@@ -475,13 +502,13 @@
*
* @param mutation the row to write. Must not be modified after calling apply, since commitlog append
* may happen concurrently, depending on the CL Executor type.
- * @param writeCommitLog false to disable commitlog append entirely
+ * @param makeDurable if true, don't return unless write has been made durable
* @param updateIndexes false to disable index updates (used by CollationController "defragmenting")
* @param isDroppable true if this should throw WriteTimeoutException if it does not acquire lock within write_request_timeout_in_ms
* @param isDeferrable true if caller is not waiting for future to complete, so that future may be deferred
*/
private CompletableFuture<?> applyInternal(final Mutation mutation,
- final boolean writeCommitLog,
+ final boolean makeDurable,
boolean updateIndexes,
boolean isDroppable,
boolean isDeferrable,
@@ -499,14 +526,14 @@
mutation.viewLockAcquireStart.compareAndSet(0L, System.currentTimeMillis());
// the order of lock acquisition doesn't matter (from a deadlock perspective) because we only use tryLock()
- Collection<UUID> columnFamilyIds = mutation.getColumnFamilyIds();
- Iterator<UUID> idIterator = columnFamilyIds.iterator();
+ Collection<TableId> tableIds = mutation.getTableIds();
+ Iterator<TableId> idIterator = tableIds.iterator();
- locks = new Lock[columnFamilyIds.size()];
- for (int i = 0; i < columnFamilyIds.size(); i++)
+ locks = new Lock[tableIds.size()];
+ for (int i = 0; i < tableIds.size(); i++)
{
- UUID cfid = idIterator.next();
- int lockKey = Objects.hash(mutation.key().getKey(), cfid);
+ TableId tableId = idIterator.next();
+ int lockKey = Objects.hash(mutation.key().getKey(), tableId);
while (true)
{
Lock lock = null;
@@ -519,12 +546,13 @@
if (lock == null)
{
//throw WTE only if request is droppable
- if (isDroppable && (System.currentTimeMillis() - mutation.createdAt) > DatabaseDescriptor.getWriteRpcTimeout())
+ if (isDroppable && (approxTime.isAfter(mutation.approxCreatedAtNanos + DatabaseDescriptor.getWriteRpcTimeout(NANOSECONDS))))
{
for (int j = 0; j < i; j++)
locks[j].unlock();
- logger.trace("Could not acquire lock for {} and table {}", ByteBufferUtil.bytesToHex(mutation.key().getKey()), columnFamilyStores.get(cfid).name);
+ if (logger.isTraceEnabled())
+ logger.trace("Could not acquire lock for {} and table {}", ByteBufferUtil.bytesToHex(mutation.key().getKey()), columnFamilyStores.get(tableId).name);
Tracing.trace("Could not acquire MV lock");
if (future != null)
{
@@ -542,8 +570,8 @@
// This view update can't happen right now. so rather than keep this thread busy
// we will re-apply ourself to the queue and try again later
final CompletableFuture<?> mark = future;
- StageManager.getStage(Stage.MUTATION).execute(() ->
- applyInternal(mutation, writeCommitLog, true, isDroppable, true, mark)
+ Stage.MUTATION.execute(() ->
+ applyInternal(mutation, makeDurable, true, isDroppable, true, mark)
);
return future;
}
@@ -578,27 +606,19 @@
// Bulk non-droppable operations (e.g. commitlog replay, hint delivery) are not measured
if (isDroppable)
{
- for(UUID cfid : columnFamilyIds)
- columnFamilyStores.get(cfid).metric.viewLockAcquireTime.update(acquireTime, TimeUnit.MILLISECONDS);
+ for(TableId tableId : tableIds)
+ columnFamilyStores.get(tableId).metric.viewLockAcquireTime.update(acquireTime, MILLISECONDS);
}
}
int nowInSec = FBUtilities.nowInSeconds();
- try (OpOrder.Group opGroup = writeOrder.start())
+ try (WriteContext ctx = getWriteHandler().beginWrite(mutation, makeDurable))
{
- // write the mutation to the commitlog and memtables
- CommitLogPosition commitLogPosition = null;
- if (writeCommitLog)
- {
- Tracing.trace("Appending to commitlog");
- commitLogPosition = CommitLog.instance.add(mutation);
- }
-
for (PartitionUpdate upd : mutation.getPartitionUpdates())
{
- ColumnFamilyStore cfs = columnFamilyStores.get(upd.metadata().cfId);
+ ColumnFamilyStore cfs = columnFamilyStores.get(upd.metadata().id);
if (cfs == null)
{
- logger.error("Attempting to mutate non-existant table {} ({}.{})", upd.metadata().cfId, upd.metadata().ksName, upd.metadata().cfName);
+ logger.error("Attempting to mutate non-existant table {} ({}.{})", upd.metadata().id, upd.metadata().keyspace, upd.metadata().name);
continue;
}
AtomicLong baseComplete = new AtomicLong(Long.MAX_VALUE);
@@ -608,22 +628,22 @@
try
{
Tracing.trace("Creating materialized view mutations from base table replica");
- viewManager.forTable(upd.metadata()).pushViewReplicaUpdates(upd, writeCommitLog, baseComplete);
+ viewManager.forTable(upd.metadata().id).pushViewReplicaUpdates(upd, makeDurable, baseComplete);
}
catch (Throwable t)
{
JVMStabilityInspector.inspectThrowable(t);
- logger.error(String.format("Unknown exception caught while attempting to update MaterializedView! %s.%s",
- upd.metadata().ksName, upd.metadata().cfName), t);
+ logger.error(String.format("Unknown exception caught while attempting to update MaterializedView! %s",
+ upd.metadata().toString()), t);
throw t;
}
}
- Tracing.trace("Adding to {} memtable", upd.metadata().cfName);
UpdateTransaction indexTransaction = updateIndexes
- ? cfs.indexManager.newUpdateTransaction(upd, opGroup, nowInSec)
+ ? cfs.indexManager.newUpdateTransaction(upd, ctx, nowInSec)
: UpdateTransaction.NO_OP;
- cfs.apply(upd, indexTransaction, opGroup, commitLogPosition);
+ cfs.getWriteHandler().write(upd, ctx, indexTransaction);
+
if (requiresViewUpdate)
baseComplete.set(System.currentTimeMillis());
}
@@ -692,7 +712,7 @@
Index index = baseCfs.indexManager.getIndexByName(indexName);
if (index == null)
throw new IllegalArgumentException(String.format("Invalid index specified: %s/%s.",
- baseCfs.metadata.cfName,
+ baseCfs.metadata.name,
indexName));
if (index.getBackingTable().isPresent())
@@ -715,7 +735,7 @@
Set<ColumnFamilyStore> stores = new HashSet<>();
for (ColumnFamilyStore indexCfs : baseCfs.indexManager.getAllIndexColumnFamilyStores())
{
- logger.info("adding secondary index table {} to operation", indexCfs.metadata.cfName);
+ logger.info("adding secondary index table {} to operation", indexCfs.metadata.name);
stores.add(indexCfs);
}
return stores;
diff --git a/src/java/org/apache/cassandra/db/KeyspaceWriteHandler.java b/src/java/org/apache/cassandra/db/KeyspaceWriteHandler.java
new file mode 100644
index 0000000..19cca72
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/KeyspaceWriteHandler.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db;
+
+import org.apache.cassandra.exceptions.RequestExecutionException;
+
+public interface KeyspaceWriteHandler
+{
+ // mutation can be null if makeDurable is false
+ WriteContext beginWrite(Mutation mutation, boolean makeDurable) throws RequestExecutionException;
+ WriteContext createContextForIndexing();
+ WriteContext createContextForRead();
+}
diff --git a/src/java/org/apache/cassandra/db/LegacyLayout.java b/src/java/org/apache/cassandra/db/LegacyLayout.java
deleted file mode 100644
index b28c72a..0000000
--- a/src/java/org/apache/cassandra/db/LegacyLayout.java
+++ /dev/null
@@ -1,2795 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.db;
-
-import java.io.DataInput;
-import java.io.IOException;
-import java.io.IOError;
-import java.nio.ByteBuffer;
-import java.security.MessageDigest;
-import java.util.*;
-import java.util.concurrent.TimeUnit;
-import java.util.stream.Collectors;
-
-import org.apache.cassandra.cql3.ColumnIdentifier;
-import org.apache.cassandra.cql3.SuperColumnCompatibility;
-import org.apache.cassandra.config.SchemaConstants;
-import org.apache.cassandra.utils.AbstractIterator;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.Iterators;
-import com.google.common.collect.Lists;
-import com.google.common.collect.PeekingIterator;
-
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
-import org.apache.cassandra.db.filter.ColumnFilter;
-import org.apache.cassandra.db.filter.DataLimits;
-import org.apache.cassandra.db.rows.*;
-import org.apache.cassandra.db.partitions.*;
-import org.apache.cassandra.db.context.CounterContext;
-import org.apache.cassandra.db.marshal.*;
-import org.apache.cassandra.io.util.DataInputPlus;
-import org.apache.cassandra.io.util.DataOutputPlus;
-import org.apache.cassandra.net.MessagingService;
-import org.apache.cassandra.utils.*;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import static com.google.common.collect.Iterables.all;
-import static org.apache.cassandra.utils.ByteBufferUtil.bytes;
-
-/**
- * Functions to deal with the old format.
- */
-public abstract class LegacyLayout
-{
- private static final Logger logger = LoggerFactory.getLogger(LegacyLayout.class);
- private static final NoSpamLogger noSpamLogger = NoSpamLogger.getLogger(logger, 1L, TimeUnit.MINUTES);
-
- public final static int MAX_CELL_NAME_LENGTH = FBUtilities.MAX_UNSIGNED_SHORT;
-
- public final static int STATIC_PREFIX = 0xFFFF;
-
- public final static int DELETION_MASK = 0x01;
- public final static int EXPIRATION_MASK = 0x02;
- public final static int COUNTER_MASK = 0x04;
- public final static int COUNTER_UPDATE_MASK = 0x08;
- private final static int RANGE_TOMBSTONE_MASK = 0x10;
-
- // Used in decodeBound if the number of components in the legacy bound is greater than the clustering size,
- // indicating a complex column deletion (i.e. a collection tombstone), but the referenced column is either
- // not present in the current table metadata, or is not currently a complex column. In that case, we'll
- // check the dropped columns for the table which should contain the previous column definition. If that
- // previous definition is also not complex (indicating that the column may have been dropped and re-added
- // with different types multiple times), we use this fake definition to ensure that the complex deletion
- // can be safely processed. This resulting deletion should be filtered out of any row created by a
- // CellGrouper by the dropped column check, but this gives us an extra level of confidence as that check
- // is timestamp based and so is fallible in the face of clock drift.
- private static final ColumnDefinition INVALID_DROPPED_COMPLEX_SUBSTITUTE_COLUMN =
- new ColumnDefinition("",
- "",
- ColumnIdentifier.getInterned(ByteBufferUtil.EMPTY_BYTE_BUFFER, UTF8Type.instance),
- SetType.getInstance(UTF8Type.instance, true),
- ColumnDefinition.NO_POSITION,
- ColumnDefinition.Kind.REGULAR);
-
- private LegacyLayout() {}
-
- public static AbstractType<?> makeLegacyComparator(CFMetaData metadata)
- {
- ClusteringComparator comparator = metadata.comparator;
- if (!metadata.isCompound())
- {
- assert comparator.size() == 1;
- return comparator.subtype(0);
- }
-
- boolean hasCollections = metadata.hasCollectionColumns() || metadata.hasDroppedCollectionColumns();
- List<AbstractType<?>> types = new ArrayList<>(comparator.size() + (metadata.isDense() ? 0 : 1) + (hasCollections ? 1 : 0));
-
- types.addAll(comparator.subtypes());
-
- if (!metadata.isDense())
- {
- types.add(UTF8Type.instance);
-
- if (hasCollections)
- {
- Map<ByteBuffer, CollectionType> defined = new HashMap<>();
-
- for (CFMetaData.DroppedColumn def : metadata.getDroppedColumns().values())
- if (def.type instanceof CollectionType && def.type.isMultiCell())
- defined.put(bytes(def.name), (CollectionType) def.type);
-
- for (ColumnDefinition def : metadata.partitionColumns())
- if (def.type instanceof CollectionType && def.type.isMultiCell())
- defined.put(def.name.bytes, (CollectionType) def.type);
-
- types.add(ColumnToCollectionType.getInstance(defined));
- }
- }
- return CompositeType.getInstance(types);
- }
-
- public static LegacyCellName decodeCellName(CFMetaData metadata, ByteBuffer superColumnName, ByteBuffer cellname)
- throws UnknownColumnException
- {
- assert cellname != null;
- if (metadata.isSuper())
- {
- assert superColumnName != null;
- return decodeForSuperColumn(metadata, Clustering.make(superColumnName), cellname);
- }
-
- assert superColumnName == null;
- return decodeCellName(metadata, cellname);
- }
-
- private static LegacyCellName decodeForSuperColumn(CFMetaData metadata, Clustering clustering, ByteBuffer subcol)
- {
- ColumnDefinition def = metadata.getColumnDefinition(subcol);
- if (def != null)
- {
- // it's a statically defined subcolumn
- return new LegacyCellName(clustering, def, null);
- }
-
- def = metadata.compactValueColumn();
- assert def != null && def.type instanceof MapType;
- return new LegacyCellName(clustering, def, subcol);
- }
-
- public static LegacyCellName decodeCellName(CFMetaData metadata, ByteBuffer cellname) throws UnknownColumnException
- {
- return decodeCellName(metadata, cellname, false);
- }
-
- public static LegacyCellName decodeCellName(CFMetaData metadata, ByteBuffer cellname, boolean readAllAsDynamic) throws UnknownColumnException
- {
- Clustering clustering = decodeClustering(metadata, cellname);
-
- if (metadata.isSuper())
- return decodeForSuperColumn(metadata, clustering, CompositeType.extractComponent(cellname, 1));
-
- if (metadata.isDense() || (metadata.isCompactTable() && readAllAsDynamic))
- return new LegacyCellName(clustering, metadata.compactValueColumn(), null);
-
- ByteBuffer column = metadata.isCompound() ? CompositeType.extractComponent(cellname, metadata.comparator.size()) : cellname;
- if (column == null)
- {
- // Tables for composite 2ndary indexes used to be compound but dense, but we've transformed them into regular tables
- // (non compact ones) but with no regular column (i.e. we only care about the clustering). So we'll get here
- // in that case, and what we want to return is basically a row marker.
- if (metadata.partitionColumns().isEmpty())
- return new LegacyCellName(clustering, null, null);
-
- // Otherwise, we shouldn't get there
- throw new IllegalArgumentException("No column name component found in cell name");
- }
-
- // Row marker, this is ok
- if (!column.hasRemaining())
- return new LegacyCellName(clustering, null, null);
-
- ColumnDefinition def = metadata.getColumnDefinition(column);
-
- if (metadata.isCompactTable())
- {
- if (def == null || def.isPrimaryKeyColumn())
- // If it's a compact table, it means the column is in fact a "dynamic" one
- return new LegacyCellName(Clustering.make(column), metadata.compactValueColumn(), null);
- }
- else if (def == null)
- {
- throw new UnknownColumnException(metadata, column);
- }
-
- ByteBuffer collectionElement = metadata.isCompound() ? CompositeType.extractComponent(cellname, metadata.comparator.size() + 1) : null;
- if (collectionElement != null && def.type instanceof CollectionType)
- {
- ((CollectionType)def.type).nameComparator().validateIfFixedSize(collectionElement);
- }
-
- // Note that because static compact columns are translated to static defs in the new world order, we need to force a static
- // clustering if the definition is static (as it might not be in this case).
- return new LegacyCellName(def.isStatic() ? Clustering.STATIC_CLUSTERING : clustering, def, collectionElement);
- }
-
- public static LegacyBound decodeSliceBound(CFMetaData metadata, ByteBuffer bound, boolean isStart)
- {
- return decodeBound(metadata, bound, isStart, false);
- }
-
- public static LegacyBound decodeTombstoneBound(CFMetaData metadata, ByteBuffer bound, boolean isStart)
- {
- return decodeBound(metadata, bound, isStart, true);
- }
-
- private static LegacyBound decodeBound(CFMetaData metadata, ByteBuffer bound, boolean isStart, boolean isDeletion)
- {
- if (!bound.hasRemaining())
- return isStart ? LegacyBound.BOTTOM : LegacyBound.TOP;
-
- if (!metadata.isCompound())
- {
- // The non compound case is a lot easier, in that there is no EOC nor collection to worry about, so dealing
- // with that first.
- metadata.comparator.subtype(0).validateIfFixedSize(bound);
- return new LegacyBound(isStart ? ClusteringBound.inclusiveStartOf(bound) : ClusteringBound.inclusiveEndOf(bound), false, null);
- }
-
- int clusteringSize = metadata.comparator.size();
-
- boolean isStatic = metadata.isCompound() && CompositeType.isStaticName(bound);
- List<ByteBuffer> components = CompositeType.splitName(bound);
- byte eoc = CompositeType.lastEOC(bound);
- for (int i=0; i<Math.min(clusteringSize, components.size()); i++)
- {
- metadata.comparator.subtype(i).validateIfFixedSize(components.get(i));
- }
-
- // if the bound we have decoded is static, 2.2 format requires there to be N empty clusterings
- assert !isStatic ||
- (components.size() >= clusteringSize
- && all(components.subList(0, clusteringSize), ByteBufferUtil.EMPTY_BYTE_BUFFER::equals));
-
- ColumnDefinition collectionName = null;
- if (components.size() > clusteringSize)
- {
- // For a deletion, there can be more components than the clustering size only in the case this is the
- // bound of a collection range tombstone. In such a case, there is exactly one more component, and that
- // component is the name of the collection being deleted, since we do not support collection range deletions.
- // If the bound is not part of a deletion, it is from slice query filter. The column name may be:
- // - a valid, non-collection column; in this case we expect a single extra component
- // - an empty buffer, representing a row marker; in this case we also expect a single extra empty component
- // - a valid collection column and the first part of a cell path; in this case we expect exactly two extra components
- // In any of these slice cases, these items are unnecessary for the bound we construct,
- // so we can simply remove them, after corroborating we have encountered one of these scenario.
- assert !metadata.isCompactTable() : toDebugHex(components);
-
- // In all cases, the element straight after the clusterings should contain the name of a column.
- if (components.size() > clusteringSize + 1)
- {
- // we accept bounds from paging state that occur inside a complex column - in this case, we expect
- // two excess components, the first of which is a column name, the second a key into the collection
- if (isDeletion)
- throw new IllegalArgumentException("Invalid bound " + toDebugHex(components) + ": deletion can have at most one extra component");
-
- if (clusteringSize + 2 != components.size())
- throw new IllegalArgumentException("Invalid bound " + toDebugHex(components) + ": complex slices require exactly two extra components");
-
- // decode simply to verify that we have (or may have had) a complex column; we assume the collection key is valid
- decodeBoundLookupComplexColumn(metadata, components, clusteringSize, isStatic);
- components.remove(clusteringSize + 1);
- }
- else if (isDeletion)
- {
- collectionName = decodeBoundLookupComplexColumn(metadata, components, clusteringSize, isStatic);
- }
- else if (components.get(clusteringSize).hasRemaining())
- {
- decodeBoundVerifySimpleColumn(metadata, components, clusteringSize, isStatic);
- }
- components.remove(clusteringSize);
- }
-
- boolean isInclusive;
- if (isStart)
- {
- isInclusive = eoc <= 0;
- }
- else
- {
- isInclusive = eoc >= 0;
-
- // for an end bound, if we only have a prefix of all the components and the final EOC is zero,
- // then it should only match up to the prefix but no further, that is, it is an inclusive bound
- // of the exact prefix but an exclusive bound of anything beyond it, so adding an empty
- // composite value ensures this behavior, see CASSANDRA-12423 for more details
- if (eoc == 0 && components.size() < clusteringSize)
- {
- components.add(ByteBufferUtil.EMPTY_BYTE_BUFFER);
- isInclusive = false;
- }
- }
-
- ClusteringPrefix.Kind boundKind = ClusteringBound.boundKind(isStart, isInclusive);
- ClusteringBound cb = ClusteringBound.create(boundKind, components.toArray(new ByteBuffer[components.size()]));
- return new LegacyBound(cb, isStatic, collectionName);
- }
-
- // finds the simple column definition associated with components.get(clusteringSize)
- // if no such columns exists, or ever existed, we throw an exception; if we do not know, we return a dummy column definition
- private static ColumnDefinition decodeBoundLookupComplexColumn(CFMetaData metadata, List<ByteBuffer> components, int clusteringSize, boolean isStatic)
- {
- ByteBuffer columnNameBytes = components.get(clusteringSize);
- ColumnDefinition columnName = metadata.getColumnDefinition(columnNameBytes);
- if (columnName == null || !columnName.isComplex())
- {
- columnName = metadata.getDroppedColumnDefinition(columnNameBytes, isStatic);
- // if no record of the column having ever existed is found, something is badly wrong
- if (columnName == null)
- throw new IllegalArgumentException("Invalid bound " + toDebugHex(components) + ": expected complex column at position " + clusteringSize);
-
- // if we do have a record of dropping this column but it wasn't previously complex, use a fake
- // column definition for safety (see the comment on the constant declaration for details)
- if (!columnName.isComplex())
- columnName = INVALID_DROPPED_COMPLEX_SUBSTITUTE_COLUMN;
- }
-
- return columnName;
- }
-
- // finds the simple column definition associated with components.get(clusteringSize)
- // if no such columns exists, and definitely never existed, we throw an exception
- private static void decodeBoundVerifySimpleColumn(CFMetaData metadata, List<ByteBuffer> components, int clusteringSize, boolean isStatic)
- {
- ByteBuffer columnNameBytes = components.get(clusteringSize);
- ColumnDefinition columnName = metadata.getColumnDefinition(columnNameBytes);
- if (columnName == null || !columnName.isSimple())
- {
- columnName = metadata.getDroppedColumnDefinition(columnNameBytes, isStatic);
- // if no record of the column having ever existed is found, something is badly wrong
- if (columnName == null)
- throw new IllegalArgumentException("Invalid bound " + toDebugHex(components) + ": expected simple column at position " + clusteringSize);
- }
- }
-
- private static String toDebugHex(Collection<ByteBuffer> buffers)
- {
- return buffers.stream().map(ByteBufferUtil::bytesToHex).collect(Collectors.joining());
- }
-
- public static ByteBuffer encodeBound(CFMetaData metadata, ClusteringBound bound, boolean isStart)
- {
- if (bound == ClusteringBound.BOTTOM || bound == ClusteringBound.TOP || metadata.comparator.size() == 0)
- return ByteBufferUtil.EMPTY_BYTE_BUFFER;
-
- ClusteringPrefix clustering = bound.clustering();
-
- if (!metadata.isCompound())
- {
- assert clustering.size() == 1;
- return clustering.get(0);
- }
-
- CompositeType ctype = CompositeType.getInstance(metadata.comparator.subtypes());
- CompositeType.Builder builder = ctype.builder();
- for (int i = 0; i < clustering.size(); i++)
- builder.add(clustering.get(i));
-
- if (isStart)
- return bound.isInclusive() ? builder.build() : builder.buildAsEndOfRange();
- else
- return bound.isInclusive() ? builder.buildAsEndOfRange() : builder.build();
- }
-
- public static ByteBuffer encodeCellName(CFMetaData metadata, ClusteringPrefix clustering, ByteBuffer columnName, ByteBuffer collectionElement)
- {
- boolean isStatic = clustering == Clustering.STATIC_CLUSTERING;
-
- if (!metadata.isCompound())
- {
- if (isStatic)
- return columnName;
-
- assert clustering.size() == 1 : "Expected clustering size to be 1, but was " + clustering.size();
- return clustering.get(0);
- }
-
- // We use comparator.size() rather than clustering.size() because of static clusterings
- int clusteringSize = metadata.comparator.size();
- int size = clusteringSize + (metadata.isDense() ? 0 : 1) + (collectionElement == null ? 0 : 1);
- if (metadata.isSuper())
- size = clusteringSize + 1;
- ByteBuffer[] values = new ByteBuffer[size];
- for (int i = 0; i < clusteringSize; i++)
- {
- if (isStatic)
- {
- values[i] = ByteBufferUtil.EMPTY_BYTE_BUFFER;
- continue;
- }
-
- ByteBuffer v = clustering.get(i);
- // we can have null (only for dense compound tables for backward compatibility reasons) but that
- // means we're done and should stop there as far as building the composite is concerned.
- if (v == null)
- return CompositeType.build(Arrays.copyOfRange(values, 0, i));
-
- values[i] = v;
- }
-
- if (metadata.isSuper())
- {
- // We need to set the "column" (in thrift terms) name, i.e. the value corresponding to the subcomparator.
- // What it is depends if this a cell for a declared "static" column or a "dynamic" column part of the
- // super-column internal map.
- assert columnName != null; // This should never be null for supercolumns, see decodeForSuperColumn() above
- values[clusteringSize] = columnName.equals(SuperColumnCompatibility.SUPER_COLUMN_MAP_COLUMN)
- ? collectionElement
- : columnName;
- }
- else
- {
- if (!metadata.isDense())
- values[clusteringSize] = columnName;
- if (collectionElement != null)
- values[clusteringSize + 1] = collectionElement;
- }
-
- return CompositeType.build(isStatic, values);
- }
-
- public static Clustering decodeClustering(CFMetaData metadata, ByteBuffer value)
- {
- int csize = metadata.comparator.size();
- if (csize == 0)
- return Clustering.EMPTY;
-
- if (metadata.isCompound() && CompositeType.isStaticName(value))
- return Clustering.STATIC_CLUSTERING;
-
- List<ByteBuffer> components = metadata.isCompound()
- ? CompositeType.splitName(value)
- : Collections.singletonList(value);
-
- for (int i=0; i<Math.min(csize, components.size()); i++)
- {
- AbstractType<?> type = metadata.comparator.subtype(i);
- type.validateIfFixedSize(components.get(i));
- }
- return Clustering.make(components.subList(0, Math.min(csize, components.size())).toArray(new ByteBuffer[csize]));
- }
-
- public static ByteBuffer encodeClustering(CFMetaData metadata, ClusteringPrefix clustering)
- {
- if (clustering.size() == 0)
- return ByteBufferUtil.EMPTY_BYTE_BUFFER;
-
- if (!metadata.isCompound())
- {
- assert clustering.size() == 1;
- return clustering.get(0);
- }
-
- ByteBuffer[] values = new ByteBuffer[clustering.size()];
- for (int i = 0; i < clustering.size(); i++)
- values[i] = clustering.get(i);
- return CompositeType.build(values);
- }
-
- /**
- * The maximum number of cells to include per partition when converting to the old format.
- * <p>
- * We already apply the limit during the actual query, but for queries that counts cells and not rows (thrift queries
- * and distinct queries as far as old nodes are concerned), we may still include a little bit more than requested
- * because {@link DataLimits} always include full rows. So if the limit ends in the middle of a queried row, the
- * full row will be part of our result. This would confuse old nodes however so we make sure to truncate it to
- * what's expected before writting it on the wire.
- *
- * @param command the read commmand for which to determine the maximum cells per partition. This can be {@code null}
- * in which case {@code Integer.MAX_VALUE} is returned.
- * @return the maximum number of cells per partition that should be enforced according to the read command if
- * post-query limitation are in order (see above). This will be {@code Integer.MAX_VALUE} if no such limits are
- * necessary.
- */
- private static int maxLiveCellsPerPartition(ReadCommand command)
- {
- if (command == null)
- return Integer.MAX_VALUE;
-
- DataLimits limits = command.limits();
-
- // There is 2 types of DISTINCT queries: those that includes only the partition key, and those that include static columns.
- // On old nodes, the latter expects the first row in term of CQL count, which is what we already have and there is no additional
- // limit to apply. The former however expect only one cell per partition and rely on it (See CASSANDRA-10762).
- if (limits.isDistinct())
- return command.columnFilter().fetchedColumns().statics.isEmpty() ? 1 : Integer.MAX_VALUE;
-
- switch (limits.kind())
- {
- case THRIFT_LIMIT:
- case SUPER_COLUMN_COUNTING_LIMIT:
- return limits.perPartitionCount();
- default:
- return Integer.MAX_VALUE;
- }
- }
-
- // For serializing to old wire format
- public static LegacyUnfilteredPartition fromUnfilteredRowIterator(ReadCommand command, UnfilteredRowIterator iterator)
- {
- // we need to extract the range tombstone so materialize the partition. Since this is
- // used for the on-wire format, this is not worst than it used to be.
- final ImmutableBTreePartition partition = ImmutableBTreePartition.create(iterator);
- DeletionInfo info = partition.deletionInfo();
- Pair<LegacyRangeTombstoneList, Iterator<LegacyCell>> pair = fromRowIterator(partition.metadata(), partition.iterator(), partition.staticRow());
-
- LegacyLayout.LegacyRangeTombstoneList rtl = pair.left;
-
- // Processing the cell iterator results in the LegacyRangeTombstoneList being populated, so we do this
- // before we use the LegacyRangeTombstoneList at all
- List<LegacyLayout.LegacyCell> cells = Lists.newArrayList(pair.right);
-
- int maxCellsPerPartition = maxLiveCellsPerPartition(command);
- cells = maybeTrimLiveCells(cells, maxCellsPerPartition, command);
-
- // The LegacyRangeTombstoneList already has range tombstones for the single-row deletions and complex
- // deletions. Go through our normal range tombstones and add then to the LegacyRTL so that the range
- // tombstones all get merged and sorted properly.
- if (info.hasRanges())
- {
- Iterator<RangeTombstone> rangeTombstoneIterator = info.rangeIterator(false);
- while (rangeTombstoneIterator.hasNext())
- {
- RangeTombstone rt = rangeTombstoneIterator.next();
- Slice slice = rt.deletedSlice();
- LegacyLayout.LegacyBound start = new LegacyLayout.LegacyBound(slice.start(), false, null);
- LegacyLayout.LegacyBound end = new LegacyLayout.LegacyBound(slice.end(), false, null);
- rtl.add(start, end, rt.deletionTime().markedForDeleteAt(), rt.deletionTime().localDeletionTime());
- }
- }
-
- return new LegacyUnfilteredPartition(info.getPartitionDeletion(), rtl, cells);
- }
-
- private static List<LegacyCell> maybeTrimLiveCells(List<LegacyCell> cells, int maxLiveCells, ReadCommand command)
- {
- if (null == command || maxLiveCells >= cells.size())
- return cells;
-
- int nowInSec = command.nowInSec();
- int live = 0;
- int dead = 0;
-
- for (int i = 0; i < cells.size() && live < maxLiveCells; i++)
- {
- if (cells.get(i).isLive(nowInSec))
- live++;
- else
- dead++;
- }
-
- return cells.subList(0, live + dead);
- }
-
- public static void serializeAsLegacyPartition(ReadCommand command, UnfilteredRowIterator partition, DataOutputPlus out, int version) throws IOException
- {
- assert version < MessagingService.VERSION_30;
-
- out.writeBoolean(true);
-
- LegacyLayout.LegacyUnfilteredPartition legacyPartition = LegacyLayout.fromUnfilteredRowIterator(command, partition);
-
- UUIDSerializer.serializer.serialize(partition.metadata().cfId, out, version);
- DeletionTime.serializer.serialize(legacyPartition.partitionDeletion, out);
-
- legacyPartition.rangeTombstones.serialize(out, partition.metadata());
-
- // begin cell serialization
- out.writeInt(legacyPartition.cells.size());
- for (LegacyLayout.LegacyCell cell : legacyPartition.cells)
- {
- ByteBufferUtil.writeWithShortLength(cell.name.encode(partition.metadata()), out);
- out.writeByte(cell.serializationFlags());
- if (cell.isExpiring())
- {
- out.writeInt(cell.ttl);
- out.writeInt(cell.localDeletionTime);
- }
- else if (cell.isTombstone())
- {
- out.writeLong(cell.timestamp);
- out.writeInt(TypeSizes.sizeof(cell.localDeletionTime));
- out.writeInt(cell.localDeletionTime);
- continue;
- }
- else if (cell.isCounterUpdate())
- {
- out.writeLong(cell.timestamp);
- long count = CounterContext.instance().getUpdateCount(cell.value);
- ByteBufferUtil.writeWithLength(ByteBufferUtil.bytes(count), out);
- continue;
- }
- else if (cell.isCounter())
- {
- out.writeLong(Long.MIN_VALUE); // timestampOfLastDelete (not used, and MIN_VALUE is the default)
- }
-
- out.writeLong(cell.timestamp);
- ByteBufferUtil.writeWithLength(cell.value, out);
- }
- }
-
- // For the old wire format
- // Note: this can return null if an empty partition is serialized!
- public static UnfilteredRowIterator deserializeLegacyPartition(DataInputPlus in, int version, SerializationHelper.Flag flag, ByteBuffer key) throws IOException
- {
- assert version < MessagingService.VERSION_30;
-
- // This is only used in mutation, and mutation have never allowed "null" column families
- boolean present = in.readBoolean();
- if (!present)
- return null;
-
- CFMetaData metadata = CFMetaData.serializer.deserialize(in, version);
- LegacyDeletionInfo info = LegacyDeletionInfo.deserialize(metadata, in);
- int size = in.readInt();
- Iterator<LegacyCell> cells = deserializeCells(metadata, in, flag, size);
- SerializationHelper helper = new SerializationHelper(metadata, version, flag);
- return onWireCellstoUnfilteredRowIterator(metadata, metadata.partitioner.decorateKey(key), info, cells, false, helper);
- }
-
- // For the old wire format
- public static long serializedSizeAsLegacyPartition(ReadCommand command, UnfilteredRowIterator partition, int version)
- {
- assert version < MessagingService.VERSION_30;
-
- if (partition.isEmpty())
- return TypeSizes.sizeof(false);
-
- long size = TypeSizes.sizeof(true);
-
- LegacyLayout.LegacyUnfilteredPartition legacyPartition = LegacyLayout.fromUnfilteredRowIterator(command, partition);
-
- size += UUIDSerializer.serializer.serializedSize(partition.metadata().cfId, version);
- size += DeletionTime.serializer.serializedSize(legacyPartition.partitionDeletion);
- size += legacyPartition.rangeTombstones.serializedSize(partition.metadata());
-
- // begin cell serialization
- size += TypeSizes.sizeof(legacyPartition.cells.size());
- for (LegacyLayout.LegacyCell cell : legacyPartition.cells)
- {
- size += ByteBufferUtil.serializedSizeWithShortLength(cell.name.encode(partition.metadata()));
- size += 1; // serialization flags
- if (cell.isExpiring())
- {
- size += TypeSizes.sizeof(cell.ttl);
- size += TypeSizes.sizeof(cell.localDeletionTime);
- }
- else if (cell.isTombstone())
- {
- size += TypeSizes.sizeof(cell.timestamp);
- // localDeletionTime replaces cell.value as the body
- size += TypeSizes.sizeof(TypeSizes.sizeof(cell.localDeletionTime));
- size += TypeSizes.sizeof(cell.localDeletionTime);
- continue;
- }
- else if (cell.isCounterUpdate())
- {
- size += TypeSizes.sizeof(cell.timestamp);
- long count = CounterContext.instance().getUpdateCount(cell.value);
- size += ByteBufferUtil.serializedSizeWithLength(ByteBufferUtil.bytes(count));
- continue;
- }
- else if (cell.isCounter())
- {
- size += TypeSizes.sizeof(Long.MIN_VALUE); // timestampOfLastDelete
- }
-
- size += TypeSizes.sizeof(cell.timestamp);
- size += ByteBufferUtil.serializedSizeWithLength(cell.value);
- }
-
- return size;
- }
-
- // For thrift sake
- public static UnfilteredRowIterator toUnfilteredRowIterator(CFMetaData metadata,
- DecoratedKey key,
- LegacyDeletionInfo delInfo,
- Iterator<LegacyCell> cells)
- {
- SerializationHelper helper = new SerializationHelper(metadata, 0, SerializationHelper.Flag.LOCAL);
- return toUnfilteredRowIterator(metadata, key, delInfo, cells, false, helper);
- }
-
- // For deserializing old wire format
- public static UnfilteredRowIterator onWireCellstoUnfilteredRowIterator(CFMetaData metadata,
- DecoratedKey key,
- LegacyDeletionInfo delInfo,
- Iterator<LegacyCell> cells,
- boolean reversed,
- SerializationHelper helper)
- {
-
- // If the table is a static compact, the "column_metadata" are now internally encoded as
- // static. This has already been recognized by decodeCellName, but it means the cells
- // provided are not in the expected order (the "static" cells are not necessarily at the front).
- // So sort them to make sure toUnfilteredRowIterator works as expected.
- // Further, if the query is reversed, then the on-wire format still has cells in non-reversed
- // order, but we need to have them reverse in the final UnfilteredRowIterator. So reverse them.
- if (metadata.isStaticCompactTable() || reversed)
- {
- List<LegacyCell> l = new ArrayList<>();
- Iterators.addAll(l, cells);
- Collections.sort(l, legacyCellComparator(metadata, reversed));
- cells = l.iterator();
- }
-
- return toUnfilteredRowIterator(metadata, key, delInfo, cells, reversed, helper);
- }
-
- private static UnfilteredRowIterator toUnfilteredRowIterator(CFMetaData metadata,
- DecoratedKey key,
- LegacyDeletionInfo delInfo,
- Iterator<LegacyCell> cells,
- boolean reversed,
- SerializationHelper helper)
- {
- // A reducer that basically does nothing, we know the 2 merged iterators can't have conflicting atoms (since we merge cells with range tombstones).
- MergeIterator.Reducer<LegacyAtom, LegacyAtom> reducer = new MergeIterator.Reducer<LegacyAtom, LegacyAtom>()
- {
- private LegacyAtom atom;
-
- public void reduce(int idx, LegacyAtom current)
- {
- // We're merging cell with range tombstones, so we should always only have a single atom to reduce.
- assert atom == null;
- atom = current;
- }
-
- protected LegacyAtom getReduced()
- {
- return atom;
- }
-
- protected void onKeyChange()
- {
- atom = null;
- }
- };
- List<Iterator<LegacyAtom>> iterators = Arrays.asList(asLegacyAtomIterator(cells), asLegacyAtomIterator(delInfo.inRowRangeTombstones()));
- PeekingIterator<LegacyAtom> atoms = Iterators.peekingIterator(MergeIterator.get(iterators, legacyAtomComparator(metadata), reducer));
-
- // Check if we have some static
- Row staticRow = atoms.hasNext() && atoms.peek().isStatic()
- ? getNextRow(CellGrouper.staticGrouper(metadata, helper), atoms)
- : Rows.EMPTY_STATIC_ROW;
-
- Iterator<Row> rows = convertToRows(new CellGrouper(metadata, helper), atoms);
- Iterator<RangeTombstone> ranges = delInfo.deletionInfo.rangeIterator(reversed);
- return new RowAndDeletionMergeIterator(metadata,
- key,
- delInfo.deletionInfo.getPartitionDeletion(),
- ColumnFilter.all(metadata),
- staticRow,
- reversed,
- EncodingStats.NO_STATS,
- rows,
- ranges,
- true);
- }
-
- public static Row extractStaticColumns(CFMetaData metadata, DataInputPlus in, Columns statics) throws IOException
- {
- assert !statics.isEmpty();
- assert metadata.isCompactTable();
-
- if (metadata.isSuper())
- // TODO: there is in practice nothing to do here, but we need to handle the column_metadata for super columns somewhere else
- throw new UnsupportedOperationException();
-
- Set<ByteBuffer> columnsToFetch = new HashSet<>(statics.size());
- for (ColumnDefinition column : statics)
- columnsToFetch.add(column.name.bytes);
-
- Row.Builder builder = BTreeRow.unsortedBuilder(FBUtilities.nowInSeconds());
- builder.newRow(Clustering.STATIC_CLUSTERING);
-
- boolean foundOne = false;
- LegacyAtom atom;
- while ((atom = readLegacyAtomSkippingUnknownColumn(metadata,in)) != null)
- {
- if (atom.isCell())
- {
- LegacyCell cell = atom.asCell();
- if (!columnsToFetch.contains(cell.name.encode(metadata)))
- continue;
-
- foundOne = true;
- cell.name.column.type.validateIfFixedSize(cell.value);
- builder.addCell(new BufferCell(cell.name.column, cell.timestamp, cell.ttl, cell.localDeletionTime, cell.value, null));
- }
- else
- {
- LegacyRangeTombstone tombstone = atom.asRangeTombstone();
- // TODO: we need to track tombstones and potentially ignore cells that are
- // shadowed (or even better, replace them by tombstones).
- throw new UnsupportedOperationException();
- }
- }
-
- return foundOne ? builder.build() : Rows.EMPTY_STATIC_ROW;
- }
-
- private static LegacyAtom readLegacyAtomSkippingUnknownColumn(CFMetaData metadata, DataInputPlus in)
- throws IOException
- {
- while (true)
- {
- try
- {
- return readLegacyAtom(metadata, in, false);
- }
- catch (UnknownColumnException e)
- {
- // Simply skip, as the method name implies.
- }
- }
-
- }
-
- private static Row getNextRow(CellGrouper grouper, PeekingIterator<? extends LegacyAtom> cells)
- {
- if (!cells.hasNext())
- return null;
-
- grouper.reset();
- while (cells.hasNext() && grouper.addAtom(cells.peek()))
- {
- // We've added the cell already in the grouper, so just skip it
- cells.next();
- }
- return grouper.getRow();
- }
-
- @SuppressWarnings("unchecked")
- private static Iterator<LegacyAtom> asLegacyAtomIterator(Iterator<? extends LegacyAtom> iter)
- {
- return (Iterator<LegacyAtom>)iter;
- }
-
- private static Iterator<Row> convertToRows(final CellGrouper grouper, final PeekingIterator<LegacyAtom> atoms)
- {
- return new AbstractIterator<Row>()
- {
- protected Row computeNext()
- {
- if (!atoms.hasNext())
- return endOfData();
-
- return getNextRow(grouper, atoms);
- }
- };
- }
-
- public static Pair<LegacyRangeTombstoneList, Iterator<LegacyCell>> fromRowIterator(final RowIterator iterator)
- {
- return fromRowIterator(iterator.metadata(), iterator, iterator.staticRow());
- }
-
- private static Pair<LegacyRangeTombstoneList, Iterator<LegacyCell>> fromRowIterator(final CFMetaData metadata, final Iterator<Row> iterator, final Row staticRow)
- {
- LegacyRangeTombstoneList deletions = new LegacyRangeTombstoneList(new LegacyBoundComparator(metadata.comparator), 10);
- Iterator<LegacyCell> cells = new AbstractIterator<LegacyCell>()
- {
- private Iterator<LegacyCell> currentRow = initializeRow();
-
- private Iterator<LegacyCell> initializeRow()
- {
- if (staticRow == null || staticRow.isEmpty())
- return Collections.<LegacyLayout.LegacyCell>emptyIterator();
-
- Pair<LegacyRangeTombstoneList, Iterator<LegacyCell>> row = fromRow(metadata, staticRow);
- deletions.addAll(row.left);
- return row.right;
- }
-
- protected LegacyCell computeNext()
- {
- while (true)
- {
- if (currentRow.hasNext())
- return currentRow.next();
-
- if (!iterator.hasNext())
- return endOfData();
-
- Pair<LegacyRangeTombstoneList, Iterator<LegacyCell>> row = fromRow(metadata, iterator.next());
- deletions.addAll(row.left);
- currentRow = row.right;
- }
- }
- };
-
- return Pair.create(deletions, cells);
- }
-
- private static Pair<LegacyRangeTombstoneList, Iterator<LegacyCell>> fromRow(final CFMetaData metadata, final Row row)
- {
- // convert any complex deletions or row deletion into normal range tombstones so that we can build and send a proper RangeTombstoneList
- // to legacy nodes
- LegacyRangeTombstoneList deletions = new LegacyRangeTombstoneList(new LegacyBoundComparator(metadata.comparator), 10);
-
- if (!row.deletion().isLive())
- {
- Clustering clustering = row.clustering();
- ClusteringBound startBound = ClusteringBound.inclusiveStartOf(clustering);
- ClusteringBound endBound = ClusteringBound.inclusiveEndOf(clustering);
-
- LegacyBound start = new LegacyLayout.LegacyBound(startBound, false, null);
- LegacyBound end = new LegacyLayout.LegacyBound(endBound, false, null);
-
- deletions.add(start, end, row.deletion().time().markedForDeleteAt(), row.deletion().time().localDeletionTime());
- }
-
- for (ColumnData cd : row)
- {
- ColumnDefinition col = cd.column();
- if (col.isSimple())
- continue;
-
- DeletionTime delTime = ((ComplexColumnData)cd).complexDeletion();
- if (!delTime.isLive())
- {
- Clustering clustering = row.clustering();
- boolean isStatic = clustering == Clustering.STATIC_CLUSTERING;
- assert isStatic == col.isStatic();
-
- ClusteringBound startBound = isStatic
- ? LegacyDeletionInfo.staticBound(metadata, true)
- : ClusteringBound.inclusiveStartOf(clustering);
- ClusteringBound endBound = isStatic
- ? LegacyDeletionInfo.staticBound(metadata, false)
- : ClusteringBound.inclusiveEndOf(clustering);
-
- LegacyLayout.LegacyBound start = new LegacyLayout.LegacyBound(startBound, isStatic, col);
- LegacyLayout.LegacyBound end = new LegacyLayout.LegacyBound(endBound, isStatic, col);
-
- deletions.add(start, end, delTime.markedForDeleteAt(), delTime.localDeletionTime());
- }
- }
-
- Iterator<LegacyCell> cells = new AbstractIterator<LegacyCell>()
- {
- private final Iterator<Cell> cells = row.cellsInLegacyOrder(metadata, false).iterator();
- // we don't have (and shouldn't have) row markers for compact tables.
- private boolean hasReturnedRowMarker = metadata.isCompactTable();
-
- protected LegacyCell computeNext()
- {
- if (!hasReturnedRowMarker)
- {
- hasReturnedRowMarker = true;
-
- // don't include a row marker if there's no timestamp on the primary key; this is the 3.0+ equivalent
- // of a row marker
- if (!row.primaryKeyLivenessInfo().isEmpty())
- {
- LegacyCellName cellName = new LegacyCellName(row.clustering(), null, null);
- LivenessInfo info = row.primaryKeyLivenessInfo();
- return new LegacyCell(info.isExpiring() ? LegacyCell.Kind.EXPIRING : LegacyCell.Kind.REGULAR, cellName, ByteBufferUtil.EMPTY_BYTE_BUFFER, info.timestamp(), info.localExpirationTime(), info.ttl());
- }
- }
-
- if (!cells.hasNext())
- return endOfData();
-
- return makeLegacyCell(row.clustering(), cells.next());
- }
- };
- return Pair.create(deletions, cells);
- }
-
- private static LegacyCell makeLegacyCell(Clustering clustering, Cell cell)
- {
- LegacyCell.Kind kind;
- if (cell.isCounterCell())
- kind = LegacyCell.Kind.COUNTER;
- else if (cell.isTombstone())
- kind = LegacyCell.Kind.DELETED;
- else if (cell.isExpiring())
- kind = LegacyCell.Kind.EXPIRING;
- else
- kind = LegacyCell.Kind.REGULAR;
-
- CellPath path = cell.path();
- assert path == null || path.size() == 1;
- LegacyCellName name = new LegacyCellName(clustering, cell.column(), path == null ? null : path.get(0));
- return new LegacyCell(kind, name, cell.value(), cell.timestamp(), cell.localDeletionTime(), cell.ttl());
- }
-
- public static RowIterator toRowIterator(final CFMetaData metadata,
- final DecoratedKey key,
- final Iterator<LegacyCell> cells,
- final int nowInSec)
- {
- SerializationHelper helper = new SerializationHelper(metadata, 0, SerializationHelper.Flag.LOCAL);
- return UnfilteredRowIterators.filter(toUnfilteredRowIterator(metadata, key, LegacyDeletionInfo.live(), cells, false, helper), nowInSec);
- }
-
- public static Comparator<LegacyCell> legacyCellComparator(CFMetaData metadata)
- {
- return legacyCellComparator(metadata, false);
- }
-
- public static Comparator<LegacyCell> legacyCellComparator(final CFMetaData metadata, final boolean reversed)
- {
- final Comparator<LegacyCellName> cellNameComparator = legacyCellNameComparator(metadata, reversed);
- return new Comparator<LegacyCell>()
- {
- public int compare(LegacyCell cell1, LegacyCell cell2)
- {
- LegacyCellName c1 = cell1.name;
- LegacyCellName c2 = cell2.name;
-
- int c = cellNameComparator.compare(c1, c2);
- if (c != 0)
- return c;
-
- // The actual sorting when the cellname is equal doesn't matter, we just want to make
- // sure the cells are not considered equal.
- if (cell1.timestamp != cell2.timestamp)
- return cell1.timestamp < cell2.timestamp ? -1 : 1;
-
- if (cell1.localDeletionTime != cell2.localDeletionTime)
- return cell1.localDeletionTime < cell2.localDeletionTime ? -1 : 1;
-
- return cell1.value.compareTo(cell2.value);
- }
- };
- }
-
- // Note that this doesn't exactly compare cells as they were pre-3.0 because within a row they sort columns like
- // in 3.0, that is, with simple columns before complex columns. In other words, this comparator makes sure cells
- // are in the proper order to convert them to actual 3.0 rows.
- public static Comparator<LegacyCellName> legacyCellNameComparator(final CFMetaData metadata, final boolean reversed)
- {
- return new Comparator<LegacyCellName>()
- {
- public int compare(LegacyCellName c1, LegacyCellName c2)
- {
- // Compare clustering first
- if (c1.clustering == Clustering.STATIC_CLUSTERING)
- {
- if (c2.clustering != Clustering.STATIC_CLUSTERING)
- return -1;
- }
- else if (c2.clustering == Clustering.STATIC_CLUSTERING)
- {
- return 1;
- }
- else
- {
- int c = metadata.comparator.compare(c1.clustering, c2.clustering);
- if (c != 0)
- return reversed ? -c : c;
- }
-
- // Note that when reversed, we only care about the clustering being reversed, so it's ok
- // not to take reversed into account below.
-
- // Then check the column name
- if (c1.column != c2.column)
- {
- // A null for the column means it's a row marker
- if (c1.column == null)
- return -1;
- if (c2.column == null)
- return 1;
-
- assert c1.column.isRegular() || c1.column.isStatic();
- assert c2.column.isRegular() || c2.column.isStatic();
- int cmp = c1.column.compareTo(c2.column);
- if (cmp != 0)
- return cmp;
- }
-
- assert (c1.collectionElement == null) == (c2.collectionElement == null);
-
- if (c1.collectionElement != null)
- {
- AbstractType<?> colCmp = ((CollectionType)c1.column.type).nameComparator();
- return colCmp.compare(c1.collectionElement, c2.collectionElement);
- }
- return 0;
- }
- };
- }
-
- private static boolean equalValues(ClusteringPrefix c1, ClusteringPrefix c2, ClusteringComparator comparator)
- {
- assert c1.size() == c2.size();
- for (int i = 0; i < c1.size(); i++)
- {
- if (comparator.compareComponent(i, c1.get(i), c2.get(i)) != 0)
- return false;
- }
- return true;
- }
-
- static Comparator<LegacyAtom> legacyAtomComparator(CFMetaData metadata)
- {
- return (o1, o2) ->
- {
- // First we want to compare by clustering, but we have to be careful with range tombstone, because
- // we can have collection deletion and we want those to sort properly just before the column they
- // delete, not before the whole row.
- // We also want to special case static so they sort before any non-static. Note in particular that
- // this special casing is important in the case of one of the Atom being Bound.BOTTOM: we want
- // it to sort after the static as we deal with static first in toUnfilteredAtomIterator and having
- // Bound.BOTTOM first would mess that up (note that static deletion is handled through a specific
- // static tombstone, see LegacyDeletionInfo.add()).
- if (o1.isStatic() != o2.isStatic())
- return o1.isStatic() ? -1 : 1;
-
- ClusteringPrefix c1 = o1.clustering();
- ClusteringPrefix c2 = o2.clustering();
-
- int clusteringComparison;
- if (c1.size() != c2.size() || (o1.isCell() == o2.isCell()) || !equalValues(c1, c2, metadata.comparator))
- {
- clusteringComparison = metadata.comparator.compare(c1, c2);
- }
- else
- {
- // one is a cell and one is a range tombstone, and both have the same prefix size (that is, the
- // range tombstone is either a row deletion or a collection deletion).
- LegacyRangeTombstone rt = o1.isCell() ? o2.asRangeTombstone() : o1.asRangeTombstone();
- clusteringComparison = rt.isCollectionTombstone()
- ? 0
- : metadata.comparator.compare(c1, c2);
- }
-
- // Note that if both are range tombstones and have the same clustering, then they are equal.
- if (clusteringComparison != 0)
- return clusteringComparison;
-
- if (o1.isCell())
- {
- LegacyCell cell1 = o1.asCell();
- if (o2.isCell())
- {
- LegacyCell cell2 = o2.asCell();
- // Check for row marker cells
- if (cell1.name.column == null)
- return cell2.name.column == null ? 0 : -1;
- return cell2.name.column == null ? 1 : cell1.name.column.compareTo(cell2.name.column);
- }
-
- LegacyRangeTombstone rt2 = o2.asRangeTombstone();
- assert rt2.isCollectionTombstone(); // otherwise, we shouldn't have got a clustering equality
- if (cell1.name.column == null)
- return -1;
- int cmp = cell1.name.column.compareTo(rt2.start.collectionName);
- // If both are for the same column, then the RT should come first
- return cmp == 0 ? 1 : cmp;
- }
- else
- {
- assert o2.isCell();
- LegacyCell cell2 = o2.asCell();
-
- LegacyRangeTombstone rt1 = o1.asRangeTombstone();
- assert rt1.isCollectionTombstone(); // otherwise, we shouldn't have got a clustering equality
-
- if (cell2.name.column == null)
- return 1;
-
- int cmp = rt1.start.collectionName.compareTo(cell2.name.column);
- // If both are for the same column, then the RT should come first
- return cmp == 0 ? -1 : cmp;
- }
- };
- }
-
- public static LegacyAtom readLegacyAtom(CFMetaData metadata, DataInputPlus in, boolean readAllAsDynamic)
- throws IOException, UnknownColumnException
- {
- ByteBuffer cellname = ByteBufferUtil.readWithShortLength(in);
- if (!cellname.hasRemaining())
- return null; // END_OF_ROW
-
- try
- {
- int b = in.readUnsignedByte();
- return (b & RANGE_TOMBSTONE_MASK) != 0
- ? readLegacyRangeTombstoneBody(metadata, in, cellname)
- : readLegacyCellBody(metadata, in, cellname, b, SerializationHelper.Flag.LOCAL, readAllAsDynamic);
- }
- catch (UnknownColumnException e)
- {
- // We legitimately can get here in 2 cases:
- // 1) for system tables, because we've unceremoniously removed columns (without registering them as dropped)
- // 2) for dropped columns.
- // In any other case, there is a mismatch between the schema and the data, and we complain loudly in
- // that case. Note that if we are in a legit case of an unknown column, we want to simply skip that cell,
- // but we don't do this here and re-throw the exception because the calling code sometimes has to know
- // about this happening. This does mean code calling this method should handle this case properly.
- if (!metadata.ksName.equals(SchemaConstants.SYSTEM_KEYSPACE_NAME) && metadata.getDroppedColumnDefinition(e.columnName) == null)
- logger.warn(String.format("Got cell for unknown column %s in sstable of %s.%s: " +
- "This suggest a problem with the schema which doesn't list " +
- "this column. Even if that column was dropped, it should have " +
- "been listed as such", metadata.ksName, metadata.cfName, UTF8Type.instance.compose(e.columnName)), e);
-
- throw e;
- }
- }
-
- public static LegacyCell readLegacyCell(CFMetaData metadata, DataInput in, SerializationHelper.Flag flag) throws IOException, UnknownColumnException
- {
- ByteBuffer cellname = ByteBufferUtil.readWithShortLength(in);
- int b = in.readUnsignedByte();
- return readLegacyCellBody(metadata, in, cellname, b, flag, false);
- }
-
- public static LegacyCell readLegacyCellBody(CFMetaData metadata, DataInput in, ByteBuffer cellname, int mask, SerializationHelper.Flag flag, boolean readAllAsDynamic)
- throws IOException, UnknownColumnException
- {
- // Note that we want to call decodeCellName only after we've deserialized other parts, since it can throw
- // and we want to throw only after having deserialized the full cell.
- if ((mask & COUNTER_MASK) != 0)
- {
- in.readLong(); // timestampOfLastDelete: this has been unused for a long time so we ignore it
- long ts = in.readLong();
- ByteBuffer value = ByteBufferUtil.readWithLength(in);
- if (flag == SerializationHelper.Flag.FROM_REMOTE || (flag == SerializationHelper.Flag.LOCAL && CounterContext.instance().shouldClearLocal(value)))
- value = CounterContext.instance().clearAllLocal(value);
- return new LegacyCell(LegacyCell.Kind.COUNTER, decodeCellName(metadata, cellname, readAllAsDynamic), value, ts, Cell.NO_DELETION_TIME, Cell.NO_TTL);
- }
- else if ((mask & EXPIRATION_MASK) != 0)
- {
- int ttl = in.readInt();
- int expiration = in.readInt();
- long ts = in.readLong();
- ByteBuffer value = ByteBufferUtil.readWithLength(in);
- return new LegacyCell(LegacyCell.Kind.EXPIRING, decodeCellName(metadata, cellname, readAllAsDynamic), value, ts, expiration, ttl);
- }
- else
- {
- long ts = in.readLong();
- ByteBuffer value = ByteBufferUtil.readWithLength(in);
- LegacyCellName name = decodeCellName(metadata, cellname, readAllAsDynamic);
- return (mask & COUNTER_UPDATE_MASK) != 0
- ? new LegacyCell(LegacyCell.Kind.COUNTER, name, CounterContext.instance().createUpdate(ByteBufferUtil.toLong(value)), ts, Cell.NO_DELETION_TIME, Cell.NO_TTL)
- : ((mask & DELETION_MASK) == 0
- ? new LegacyCell(LegacyCell.Kind.REGULAR, name, value, ts, Cell.NO_DELETION_TIME, Cell.NO_TTL)
- : new LegacyCell(LegacyCell.Kind.DELETED, name, ByteBufferUtil.EMPTY_BYTE_BUFFER, ts, ByteBufferUtil.toInt(value), Cell.NO_TTL));
- }
- }
-
- public static LegacyRangeTombstone readLegacyRangeTombstoneBody(CFMetaData metadata, DataInputPlus in, ByteBuffer boundname) throws IOException
- {
- LegacyBound min = decodeTombstoneBound(metadata, boundname, true);
- LegacyBound max = decodeTombstoneBound(metadata, ByteBufferUtil.readWithShortLength(in), false);
- DeletionTime dt = DeletionTime.serializer.deserialize(in);
- return new LegacyRangeTombstone(min, max, dt);
- }
-
- public static Iterator<LegacyCell> deserializeCells(final CFMetaData metadata,
- final DataInput in,
- final SerializationHelper.Flag flag,
- final int size)
- {
- return new AbstractIterator<LegacyCell>()
- {
- private int i = 0;
-
- protected LegacyCell computeNext()
- {
- if (i >= size)
- return endOfData();
-
- ++i;
- try
- {
- return readLegacyCell(metadata, in, flag);
- }
- catch (UnknownColumnException e)
- {
- // We can get there if we read a cell for a dropped column, and if that is the case,
- // then simply ignore the cell is fine. But also not that we ignore if it's the
- // system keyspace because for those table we actually remove columns without registering
- // them in the dropped columns
- if (metadata.ksName.equals(SchemaConstants.SYSTEM_KEYSPACE_NAME) || metadata.getDroppedColumnDefinition(e.columnName) != null)
- return computeNext();
- else
- throw new IOError(e);
- }
- catch (IOException e)
- {
- throw new IOError(e);
- }
- }
- };
- }
-
- public static class CellGrouper
- {
- /**
- * The fake TTL used for expired rows that have been compacted.
- */
- private static final int FAKE_TTL = 1;
-
- public final CFMetaData metadata;
- private final boolean isStatic;
- private final SerializationHelper helper;
- private final Row.Builder builder;
- private Clustering clustering;
-
- private LegacyRangeTombstone rowDeletion;
- private LegacyRangeTombstone collectionDeletion;
-
- /**
- * Used to track if we need to add pk liveness info (row marker) when removing invalid legacy cells.
- *
- * In 2.1 these invalid cells existed but were not queryable, in this case specifically because they
- * represented values for clustering key columns that were written as data cells.
- *
- * However, the presence (or not) of such cells on an otherwise empty CQL row (or partition) would decide
- * if an empty result row were returned for the CQL row (or partition). To maintain this behaviour we
- * insert a row marker containing the liveness info of these invalid cells iff we have no other data
- * on the row.
- *
- * See also CASSANDRA-15365
- */
- private boolean hasValidCells = false;
- private LivenessInfo invalidLivenessInfo = null;
-
- public CellGrouper(CFMetaData metadata, SerializationHelper helper)
- {
- this(metadata, helper, false);
- }
-
- private CellGrouper(CFMetaData metadata, SerializationHelper helper, boolean isStatic)
- {
- this.metadata = metadata;
- this.isStatic = isStatic;
- this.helper = helper;
- // We cannot use a sorted builder because we don't have exactly the same ordering in 3.0 and pre-3.0. More precisely, within a row, we
- // store all simple columns before the complex ones in 3.0, which we use to sort everything sorted by the column name before. Note however
- // that the unsorted builder won't have to reconcile cells, so the exact value we pass for nowInSec doesn't matter.
- this.builder = BTreeRow.unsortedBuilder(FBUtilities.nowInSeconds());
- }
-
- public static CellGrouper staticGrouper(CFMetaData metadata, SerializationHelper helper)
- {
- return new CellGrouper(metadata, helper, true);
- }
-
- public void reset()
- {
- this.clustering = null;
- this.rowDeletion = null;
- this.collectionDeletion = null;
- this.invalidLivenessInfo = null;
- this.hasValidCells = false;
- }
-
- /**
- * Try adding the provided atom to the currently grouped row.
- *
- * @param atom the new atom to try to add. This <b>must</b> be a "row" atom, that is either a cell or a legacy
- * range tombstone that covers only one row (row deletion) or a subset of it (collection
- * deletion). Meaning that legacy range tombstone covering multiple rows (that should be handled as
- * legit range tombstone in the new storage engine) should be handled separately. Atoms should also
- * be provided in proper clustering order.
- * @return {@code true} if the provided atom has been "consumed" by this grouper (this does _not_ mean the
- * atom has been "used" by the grouper as the grouper will skip some shadowed atoms for instance, just
- * that {@link #getRow()} shouldn't be called just yet if there is more atom in the atom iterator we're
- * grouping). {@code false} otherwise, that is if the row currently built by this grouper is done
- * _without_ the provided atom being "consumed" (and so {@link #getRow()} should be called and the
- * grouper resetted, after which the provided atom should be provided again).
- */
- public boolean addAtom(LegacyAtom atom)
- {
- assert atom.isRowAtom(metadata) : "Unexpected non in-row legacy range tombstone " + atom;
- return atom.isCell()
- ? addCell(atom.asCell())
- : addRangeTombstone(atom.asRangeTombstone());
- }
-
- private boolean addCell(LegacyCell cell)
- {
- if (clustering == null)
- {
- clustering = cell.name.clustering;
- assert !isStatic || clustering == Clustering.STATIC_CLUSTERING;
- builder.newRow(clustering);
- }
- else if (!clustering.equals(cell.name.clustering))
- {
- return false;
- }
-
- // Ignore shadowed cells
- if (rowDeletion != null && rowDeletion.deletionTime.deletes(cell.timestamp))
- return true;
-
- ColumnDefinition column = cell.name.column;
- if (column == null)
- {
- // It's the row marker
- assert !cell.value.hasRemaining();
-
- // In 2.1, the row marker expired cell might have been converted into a deleted one by compaction.
- // If we do not set the primary key liveness info for this row and it does not contains any regular columns
- // the row will be empty. To avoid that, we reuse the localDeletionTime but use a fake TTL.
- // The only time in 2.x that we actually delete a row marker is in 2i tables, so in that case we do
- // want to actually propagate the row deletion. (CASSANDRA-13320)
- if (!cell.isTombstone())
- builder.addPrimaryKeyLivenessInfo(LivenessInfo.withExpirationTime(cell.timestamp, cell.ttl, cell.localDeletionTime));
- else if (metadata.isIndex())
- builder.addRowDeletion(Row.Deletion.regular(new DeletionTime(cell.timestamp, cell.localDeletionTime)));
- else
- builder.addPrimaryKeyLivenessInfo(LivenessInfo.create(cell.timestamp, FAKE_TTL, cell.localDeletionTime));
- hasValidCells = true;
- }
- else if (column.isPrimaryKeyColumn() && metadata.isCQLTable())
- {
- // SSTables generated offline and side-loaded may include invalid cells which have the column name
- // of a primary key column. So that we don't fail when encountering these cells, we treat them the
- // same way as 2.1 did, namely we include their clusterings in the new CQL row, but drop the invalid
- // column part of the cell
- noSpamLogger.warn("Illegal cell name for CQL3 table {}.{}. {} is defined as a primary key column",
- metadata.ksName, metadata.cfName, column.name);
-
- if (invalidLivenessInfo != null)
- {
- // when we have several invalid cells we follow the logic in LivenessInfo#supersedes when picking the PKLI to keep:
- LivenessInfo newInvalidLiveness = LivenessInfo.create(cell.timestamp, cell.isTombstone() ? FAKE_TTL : cell.ttl, cell.localDeletionTime);
- if (newInvalidLiveness.supersedes(invalidLivenessInfo))
- invalidLivenessInfo = newInvalidLiveness;
- }
- else
- {
- invalidLivenessInfo = LivenessInfo.create(cell.timestamp, cell.isTombstone() ? FAKE_TTL : cell.ttl, cell.localDeletionTime);
- }
- return true;
- }
- else
- {
- if (collectionDeletion != null && collectionDeletion.start.collectionName.name.equals(column.name) && collectionDeletion.deletionTime.deletes(cell.timestamp))
- return true;
-
- if (helper.includes(column))
- {
- hasValidCells = true;
- CellPath path = null;
- if (column.isComplex())
- {
- // Recalling startOfComplexColumn for every cell is a big inefficient, but it's ok in practice
- // and it's simpler. And since 1) this only matter for super column selection in thrift in
- // practice and 2) is only used during upgrade, it's probably worth keeping things simple.
- helper.startOfComplexColumn(column);
- path = cell.name.collectionElement == null ? null : CellPath.create(cell.name.collectionElement);
- if (!helper.includes(path))
- return true;
- }
- column.type.validateIfFixedSize(cell.value);
- Cell c = new BufferCell(column, cell.timestamp, cell.ttl, cell.localDeletionTime, cell.value, path);
- if (!helper.isDropped(c, column.isComplex()))
- builder.addCell(c);
- if (column.isComplex())
- {
- helper.endOfComplexColumn();
- }
- }
- }
- return true;
- }
-
- private boolean addRangeTombstone(LegacyRangeTombstone tombstone)
- {
- if (tombstone.isRowDeletion(metadata))
- {
- return addRowTombstone(tombstone);
- }
- else
- {
- // The isRowAtom() assertion back in addAtom would have already triggered otherwise, but spelling it
- // out nonetheless.
- assert tombstone.isCollectionTombstone();
- return addCollectionTombstone(tombstone);
- }
- }
-
- private boolean addRowTombstone(LegacyRangeTombstone tombstone)
- {
- if (clustering != null)
- {
- // If we're already in the row, there might be a chance that there were two range tombstones
- // written, as 2.x storage format does not guarantee just one range tombstone, unlike 3.x.
- // We have to make sure that clustering matches, which would mean that tombstone is for the
- // same row.
- if (clustering.equals(tombstone.start.getAsClustering(metadata)))
- {
- // If the tombstone superceeds the previous delete, we discard the previous one.
- // This assumes that we are building the row from a sane source (ie, this row deletion
- // does not delete anything already added to the builder). See CASSANDRA-15789 for details
- if (rowDeletion == null || tombstone.deletionTime.supersedes(rowDeletion.deletionTime))
- {
- builder.addRowDeletion(Row.Deletion.regular(tombstone.deletionTime));
- rowDeletion = tombstone;
- hasValidCells = true;
- }
- return true;
- }
-
- // different clustering -> new row
- return false;
- }
-
- clustering = tombstone.start.getAsClustering(metadata);
- builder.newRow(clustering);
- builder.addRowDeletion(Row.Deletion.regular(tombstone.deletionTime));
- rowDeletion = tombstone;
- hasValidCells = true;
-
- return true;
- }
-
- private boolean addCollectionTombstone(LegacyRangeTombstone tombstone)
- {
- // If the collection tombstone is not included in the query (which technically would only apply to thrift
- // queries since CQL one "fetch" everything), we can skip it (so return), but we're problably still within
- // the current row so we return `true`. Technically, it is possible that tombstone belongs to another row
- // that the row currently grouped, but as we ignore it, returning `true` is ok in that case too.
- if (!helper.includes(tombstone.start.collectionName))
- return true; // see CASSANDRA-13109
-
- // The helper needs to be informed about the current complex column identifier before
- // it can perform the comparison between the recorded drop time and the RT deletion time.
- // If the RT has been superceded by a drop, we still return true as we don't want the
- // grouper to terminate yet.
- helper.startOfComplexColumn(tombstone.start.collectionName);
- if (helper.isDroppedComplexDeletion(tombstone.deletionTime))
- return true;
-
- if (clustering == null)
- {
- clustering = tombstone.start.getAsClustering(metadata);
- builder.newRow(clustering);
- }
- else if (!clustering.equals(tombstone.start.getAsClustering(metadata)))
- {
- return false;
- }
-
- builder.addComplexDeletion(tombstone.start.collectionName, tombstone.deletionTime);
- if (rowDeletion == null || tombstone.deletionTime.supersedes(rowDeletion.deletionTime))
- collectionDeletion = tombstone;
- hasValidCells = true;
-
- return true;
- }
-
- /**
- * Whether the provided range tombstone starts strictly after the current row of the cell grouper (if no row is
- * currently started, this return false).
- */
- public boolean startsAfterCurrentRow(LegacyRangeTombstone rangeTombstone)
- {
- return clustering != null && metadata.comparator.compare(rangeTombstone.start.bound, clustering) > 0;
- }
-
- /**
- * The clustering of the current row of the cell grouper, or {@code null} if no row is currently started.
- */
- public Clustering currentRowClustering()
- {
- return clustering;
- }
-
- /**
- * Generates the row currently grouped by this grouper and reset it for the following row.
- * <p>
- * Note that the only correct way to call this is when either all the atom we're trying to group has been
- * consumed, or when {@link #addAtom(LegacyAtom)} returns {@code false}.
- *
- * @return the current row that has been grouped, or {@code null} in the rare case where all the atoms
- * "consumed" by {@link #addAtom(LegacyAtom)} for this row were skipped (we skip atoms under a few conditions).
- */
- public Row getRow()
- {
- if (!hasValidCells && invalidLivenessInfo != null)
- builder.addPrimaryKeyLivenessInfo(invalidLivenessInfo);
- return builder.build();
- }
- }
-
- public static class LegacyUnfilteredPartition
- {
- public final DeletionTime partitionDeletion;
- public final LegacyRangeTombstoneList rangeTombstones;
- public final List<LegacyCell> cells;
-
- private LegacyUnfilteredPartition(DeletionTime partitionDeletion, LegacyRangeTombstoneList rangeTombstones, List<LegacyCell> cells)
- {
- this.partitionDeletion = partitionDeletion;
- this.rangeTombstones = rangeTombstones;
- this.cells = cells;
- }
-
- public void digest(CFMetaData metadata, MessageDigest digest)
- {
- for (LegacyCell cell : cells)
- {
- digest.update(cell.name.encode(metadata).duplicate());
-
- if (cell.isCounter())
- CounterContext.instance().updateDigest(digest, cell.value);
- else
- digest.update(cell.value.duplicate());
-
- FBUtilities.updateWithLong(digest, cell.timestamp);
- FBUtilities.updateWithByte(digest, cell.serializationFlags());
-
- if (cell.isExpiring())
- FBUtilities.updateWithInt(digest, cell.ttl);
-
- if (cell.isCounter())
- {
- // Counters used to have the timestampOfLastDelete field, which we stopped using long ago and has been hard-coded
- // to Long.MIN_VALUE but was still taken into account in 2.2 counter digests (to maintain backward compatibility
- // in the first place).
- FBUtilities.updateWithLong(digest, Long.MIN_VALUE);
- }
- }
-
- if (partitionDeletion.markedForDeleteAt() != Long.MIN_VALUE)
- digest.update(ByteBufferUtil.bytes(partitionDeletion.markedForDeleteAt()));
-
- if (!rangeTombstones.isEmpty())
- rangeTombstones.updateDigest(digest);
- }
- }
-
- public static class LegacyCellName
- {
- public final Clustering clustering;
- public final ColumnDefinition column;
- public final ByteBuffer collectionElement;
-
- @VisibleForTesting
- public LegacyCellName(Clustering clustering, ColumnDefinition column, ByteBuffer collectionElement)
- {
- this.clustering = clustering;
- this.column = column;
- this.collectionElement = collectionElement;
- }
-
- public static LegacyCellName create(Clustering clustering, ColumnDefinition column)
- {
- return new LegacyCellName(clustering, column, null);
- }
-
- public ByteBuffer encode(CFMetaData metadata)
- {
- return encodeCellName(metadata, clustering, column == null ? ByteBufferUtil.EMPTY_BYTE_BUFFER : column.name.bytes, collectionElement);
- }
-
- public ByteBuffer superColumnSubName()
- {
- assert collectionElement != null;
- return collectionElement;
- }
-
- public ByteBuffer superColumnName()
- {
- return clustering.get(0);
- }
-
- @Override
- public String toString()
- {
- StringBuilder sb = new StringBuilder();
- for (int i = 0; i < clustering.size(); i++)
- sb.append(i > 0 ? ":" : "").append(clustering.get(i) == null ? "null" : ByteBufferUtil.bytesToHex(clustering.get(i)));
- return String.format("Cellname(clustering=%s, column=%s, collElt=%s)", sb.toString(), column == null ? "null" : column.name, collectionElement == null ? "null" : ByteBufferUtil.bytesToHex(collectionElement));
- }
- }
-
- public static class LegacyBound
- {
- public static final LegacyBound BOTTOM = new LegacyBound(ClusteringBound.BOTTOM, false, null);
- public static final LegacyBound TOP = new LegacyBound(ClusteringBound.TOP, false, null);
-
- public final ClusteringBound bound;
- public final boolean isStatic;
- public final ColumnDefinition collectionName;
-
- public LegacyBound(ClusteringBound bound, boolean isStatic, ColumnDefinition collectionName)
- {
- this.bound = bound;
- this.isStatic = isStatic;
- this.collectionName = collectionName;
- }
-
- public Clustering getAsClustering(CFMetaData metadata)
- {
- if (isStatic)
- return Clustering.STATIC_CLUSTERING;
-
- assert bound.size() == metadata.comparator.size();
- ByteBuffer[] values = new ByteBuffer[bound.size()];
- for (int i = 0; i < bound.size(); i++)
- values[i] = bound.get(i);
- return Clustering.make(values);
- }
-
- @Override
- public String toString()
- {
- StringBuilder sb = new StringBuilder();
- sb.append(bound.kind()).append('(');
- for (int i = 0; i < bound.size(); i++)
- sb.append(i > 0 ? ":" : "").append(bound.get(i) == null ? "null" : ByteBufferUtil.bytesToHex(bound.get(i)));
- sb.append(')');
- return String.format("Bound(%s, collection=%s)", sb.toString(), collectionName == null ? "null" : collectionName.name);
- }
- }
-
- public interface LegacyAtom
- {
- public boolean isCell();
-
- // note that for static atoms, LegacyCell and LegacyRangeTombstone behave differently here:
- // - LegacyCell returns the modern Clustering.STATIC_CLUSTERING
- // - LegacyRangeTombstone returns the 2.2 bound (i.e. N empty ByteBuffer, where N is number of clusterings)
- // in LegacyDeletionInfo.add(), we split any LRT with a static bound out into the inRowRangeTombstones collection
- // these are merged with regular row cells, in the CellGrouper, and their clustering is obtained via start.bound.getAsClustering
- // (also, it should be impossibly to issue raw static row deletions anyway)
- public ClusteringPrefix clustering();
- public boolean isStatic();
-
- public LegacyCell asCell();
- public LegacyRangeTombstone asRangeTombstone();
-
- /**
- * Whether the atom is one that becomes part of a {@link Row} in the new storage engine, meaning it is either
- * as cell or a legacy range tombstone that covers a single row, or parts of one.
- */
- public boolean isRowAtom(CFMetaData metadata);
- }
-
- /**
- * A legacy cell.
- * <p>
- * This is used as a temporary object to facilitate dealing with the legacy format, this
- * is not meant to be optimal.
- */
- public static class LegacyCell implements LegacyAtom
- {
- private final static int DELETION_MASK = 0x01;
- private final static int EXPIRATION_MASK = 0x02;
- private final static int COUNTER_MASK = 0x04;
- private final static int COUNTER_UPDATE_MASK = 0x08;
- private final static int RANGE_TOMBSTONE_MASK = 0x10;
-
- public enum Kind { REGULAR, EXPIRING, DELETED, COUNTER }
-
- public final Kind kind;
-
- public final LegacyCellName name;
- public final ByteBuffer value;
-
- public final long timestamp;
- public final int localDeletionTime;
- public final int ttl;
-
- @VisibleForTesting
- public LegacyCell(Kind kind, LegacyCellName name, ByteBuffer value, long timestamp, int localDeletionTime, int ttl)
- {
- this.kind = kind;
- this.name = name;
- this.value = value;
- this.timestamp = timestamp;
- this.localDeletionTime = localDeletionTime;
- this.ttl = ttl;
- }
-
- public static LegacyCell regular(CFMetaData metadata, ByteBuffer superColumnName, ByteBuffer name, ByteBuffer value, long timestamp)
- throws UnknownColumnException
- {
- return new LegacyCell(Kind.REGULAR, decodeCellName(metadata, superColumnName, name), value, timestamp, Cell.NO_DELETION_TIME, Cell.NO_TTL);
- }
-
- public static LegacyCell expiring(CFMetaData metadata, ByteBuffer superColumnName, ByteBuffer name, ByteBuffer value, long timestamp, int ttl, int nowInSec)
- throws UnknownColumnException
- {
- /*
- * CASSANDRA-14092: Max expiration date capping is maybe performed here, expiration overflow policy application
- * is done at {@link org.apache.cassandra.thrift.ThriftValidation#validateTtl(CFMetaData, Column)}
- */
- return new LegacyCell(Kind.EXPIRING, decodeCellName(metadata, superColumnName, name), value, timestamp, ExpirationDateOverflowHandling.computeLocalExpirationTime(nowInSec, ttl), ttl);
- }
-
- public static LegacyCell tombstone(CFMetaData metadata, ByteBuffer superColumnName, ByteBuffer name, long timestamp, int nowInSec)
- throws UnknownColumnException
- {
- return new LegacyCell(Kind.DELETED, decodeCellName(metadata, superColumnName, name), ByteBufferUtil.EMPTY_BYTE_BUFFER, timestamp, nowInSec, LivenessInfo.NO_TTL);
- }
-
- public static LegacyCell counterUpdate(CFMetaData metadata, ByteBuffer superColumnName, ByteBuffer name, long value)
- throws UnknownColumnException
- {
- // See UpdateParameters.addCounter() for more details on this
- ByteBuffer counterValue = CounterContext.instance().createUpdate(value);
- return counter(decodeCellName(metadata, superColumnName, name), counterValue);
- }
-
- public static LegacyCell counter(LegacyCellName name, ByteBuffer value)
- {
- return new LegacyCell(Kind.COUNTER, name, value, FBUtilities.timestampMicros(), Cell.NO_DELETION_TIME, Cell.NO_TTL);
- }
-
- public byte serializationFlags()
- {
- if (isExpiring())
- return EXPIRATION_MASK;
- if (isTombstone())
- return DELETION_MASK;
- if (isCounterUpdate())
- return COUNTER_UPDATE_MASK;
- if (isCounter())
- return COUNTER_MASK;
- return 0;
- }
-
- public boolean isCounterUpdate()
- {
- // See UpdateParameters.addCounter() for more details on this
- return isCounter() && CounterContext.instance().isUpdate(value);
- }
-
- public ClusteringPrefix clustering()
- {
- return name.clustering;
- }
-
- public boolean isStatic()
- {
- return name.clustering == Clustering.STATIC_CLUSTERING;
- }
-
- public boolean isCell()
- {
- return true;
- }
-
- public LegacyCell asCell()
- {
- return this;
- }
-
- public LegacyRangeTombstone asRangeTombstone()
- {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public boolean isRowAtom(CFMetaData metaData)
- {
- return true;
- }
-
- public boolean isCounter()
- {
- return kind == Kind.COUNTER;
- }
-
- public boolean isExpiring()
- {
- return kind == Kind.EXPIRING;
- }
-
- public boolean isTombstone()
- {
- return kind == Kind.DELETED;
- }
-
- public boolean isLive(int nowInSec)
- {
- if (isTombstone())
- return false;
-
- return !isExpiring() || nowInSec < localDeletionTime;
- }
-
- @Override
- public String toString()
- {
- return String.format("LegacyCell(%s, name=%s, v=%s, ts=%s, ldt=%s, ttl=%s)", kind, name, ByteBufferUtil.bytesToHex(value), timestamp, localDeletionTime, ttl);
- }
- }
-
- /**
- * A legacy range tombstone.
- * <p>
- * This is used as a temporary object to facilitate dealing with the legacy format, this
- * is not meant to be optimal.
- */
- public static class LegacyRangeTombstone implements LegacyAtom
- {
- public final LegacyBound start;
- public final LegacyBound stop;
- public final DeletionTime deletionTime;
-
- public LegacyRangeTombstone(LegacyBound start, LegacyBound stop, DeletionTime deletionTime)
- {
- // Because of the way RangeTombstoneList work, we can have a tombstone where only one of
- // the bound has a collectionName. That happens if we have a big tombstone A (spanning one
- // or multiple rows) and a collection tombstone B. In that case, RangeTombstoneList will
- // split this into 3 RTs: the first one from the beginning of A to the beginning of B,
- // then B, then a third one from the end of B to the end of A. To make this simpler, if
- // we detect that case we transform the 1st and 3rd tombstone so they don't end in the middle
- // of a row (which is still correct).
- if ((start.collectionName == null) != (stop.collectionName == null))
- {
- if (start.collectionName == null)
- stop = new LegacyBound(ClusteringBound.inclusiveEndOf(stop.bound.values), stop.isStatic, null);
- else
- start = new LegacyBound(ClusteringBound.inclusiveStartOf(start.bound.values), start.isStatic, null);
- }
- else if (!Objects.equals(start.collectionName, stop.collectionName))
- {
- // We're in the similar but slightly more complex case where on top of the big tombstone
- // A, we have 2 (or more) collection tombstones B and C within A. So we also end up with
- // a tombstone that goes between the end of B and the start of C.
- start = new LegacyBound(start.bound, start.isStatic, null);
- stop = new LegacyBound(stop.bound, stop.isStatic, null);
- }
-
- this.start = start;
- this.stop = stop;
- this.deletionTime = deletionTime;
- }
-
- /** @see LegacyAtom#clustering for static inconsistencies explained */
- public ClusteringPrefix clustering()
- {
- return start.bound;
- }
-
- public LegacyRangeTombstone withNewStart(LegacyBound newStart)
- {
- return new LegacyRangeTombstone(newStart, stop, deletionTime);
- }
-
- public LegacyRangeTombstone withNewStart(ClusteringBound newStart)
- {
- return withNewStart(new LegacyBound(newStart, start.isStatic, null));
- }
-
- public LegacyRangeTombstone withNewEnd(LegacyBound newStop)
- {
- return new LegacyRangeTombstone(start, newStop, deletionTime);
- }
-
- public LegacyRangeTombstone withNewEnd(ClusteringBound newEnd)
- {
- return withNewEnd(new LegacyBound(newEnd, stop.isStatic, null));
- }
-
- public boolean isCell()
- {
- return false;
- }
-
- public boolean isStatic()
- {
- return start.isStatic || stop.isStatic;
- }
-
- public LegacyCell asCell()
- {
- throw new UnsupportedOperationException();
- }
-
- public LegacyRangeTombstone asRangeTombstone()
- {
- return this;
- }
-
- @Override
- public boolean isRowAtom(CFMetaData metadata)
- {
- return isCollectionTombstone() || isRowDeletion(metadata);
- }
-
- public boolean isCollectionTombstone()
- {
- return start.collectionName != null;
- }
-
- public boolean isRowDeletion(CFMetaData metadata)
- {
- if (start.collectionName != null
- || stop.collectionName != null
- || start.bound.size() != metadata.comparator.size()
- || stop.bound.size() != metadata.comparator.size())
- return false;
-
- for (int i = 0; i < start.bound.size(); i++)
- if (!Objects.equals(start.bound.get(i), stop.bound.get(i)))
- return false;
- return true;
- }
-
- @Override
- public String toString()
- {
- return String.format("RT(%s-%s, %s)", start, stop, deletionTime);
- }
- }
-
- public static class LegacyDeletionInfo
- {
- public final MutableDeletionInfo deletionInfo;
- public final List<LegacyRangeTombstone> inRowTombstones = new ArrayList<>();
-
- private LegacyDeletionInfo(MutableDeletionInfo deletionInfo)
- {
- this.deletionInfo = deletionInfo;
- }
-
- public static LegacyDeletionInfo live()
- {
- return new LegacyDeletionInfo(MutableDeletionInfo.live());
- }
-
- public void add(DeletionTime topLevel)
- {
- deletionInfo.add(topLevel);
- }
-
- private static ClusteringBound staticBound(CFMetaData metadata, boolean isStart)
- {
- // In pre-3.0 nodes, static row started by a clustering with all empty values so we
- // preserve that here. Note that in practice, it doesn't really matter since the rest
- // of the code will ignore the bound for RT that have their static flag set.
- ByteBuffer[] values = new ByteBuffer[metadata.comparator.size()];
- for (int i = 0; i < values.length; i++)
- values[i] = ByteBufferUtil.EMPTY_BYTE_BUFFER;
- return isStart
- ? ClusteringBound.inclusiveStartOf(values)
- : ClusteringBound.inclusiveEndOf(values);
- }
-
- public void add(CFMetaData metadata, LegacyRangeTombstone tombstone)
- {
- if (metadata.hasStaticColumns())
- {
- /*
- * For table having static columns we have to deal with the following cases:
- * 1. the end of the tombstone is static (in which case either the start is static or is BOTTOM, which is the same
- * for our consideration). This mean that either the range only delete the static row, or that it's a collection
- * tombstone of a static collection. In both case, we just add the tombstone to the inRowTombstones.
- * 2. only the start is static. There is then 2 subcase: either the start is inclusive, and that mean we include the
- * static row and more (so we add an inRowTombstone for the static and deal with the rest normally). Or the start
- * is exclusive, and that means we explicitely exclude the static (in which case we can just add the tombstone
- * as if it started at BOTTOM).
- * 3. none of the bound are static but the start is BOTTOM. This means we intended to delete the static row so we
- * need to add it to the inRowTombstones (and otherwise handle the range normally).
- */
- if (tombstone.stop.isStatic)
- {
- // If the start is BOTTOM, we replace it by the beginning of the starting row so as to not confuse the
- // RangeTombstone.isRowDeletion() method
- if (tombstone.start == LegacyBound.BOTTOM)
- tombstone = tombstone.withNewStart(new LegacyBound(staticBound(metadata, true), true, null));
- inRowTombstones.add(tombstone);
- return;
- }
-
- if (tombstone.start.isStatic)
- {
- if (tombstone.start.bound.isInclusive())
- inRowTombstones.add(tombstone.withNewEnd(new LegacyBound(staticBound(metadata, false), true, null)));
-
- tombstone = tombstone.withNewStart(LegacyBound.BOTTOM);
- }
- else if (tombstone.start == LegacyBound.BOTTOM)
- {
- inRowTombstones.add(new LegacyRangeTombstone(new LegacyBound(staticBound(metadata, true), true, null),
- new LegacyBound(staticBound(metadata, false), true, null),
- tombstone.deletionTime));
- }
- }
-
- if (tombstone.isCollectionTombstone() || tombstone.isRowDeletion(metadata))
- inRowTombstones.add(tombstone);
- else
- add(metadata, new RangeTombstone(Slice.make(tombstone.start.bound, tombstone.stop.bound), tombstone.deletionTime));
- }
-
- public void add(CFMetaData metadata, RangeTombstone tombstone)
- {
- deletionInfo.add(tombstone, metadata.comparator);
- }
-
- public Iterator<LegacyRangeTombstone> inRowRangeTombstones()
- {
- return inRowTombstones.iterator();
- }
-
- public static LegacyDeletionInfo deserialize(CFMetaData metadata, DataInputPlus in) throws IOException
- {
- DeletionTime topLevel = DeletionTime.serializer.deserialize(in);
-
- int rangeCount = in.readInt();
- if (rangeCount == 0)
- return new LegacyDeletionInfo(new MutableDeletionInfo(topLevel));
-
- LegacyDeletionInfo delInfo = new LegacyDeletionInfo(new MutableDeletionInfo(topLevel));
- for (int i = 0; i < rangeCount; i++)
- {
- LegacyBound start = decodeTombstoneBound(metadata, ByteBufferUtil.readWithShortLength(in), true);
- LegacyBound end = decodeTombstoneBound(metadata, ByteBufferUtil.readWithShortLength(in), false);
- int delTime = in.readInt();
- long markedAt = in.readLong();
-
- delInfo.add(metadata, new LegacyRangeTombstone(start, end, new DeletionTime(markedAt, delTime)));
- }
- return delInfo;
- }
- }
-
- /**
- * A helper class for LegacyRangeTombstoneList. This replaces the Comparator<Composite> that RTL used before 3.0.
- */
- private static class LegacyBoundComparator implements Comparator<LegacyBound>
- {
- ClusteringComparator clusteringComparator;
-
- public LegacyBoundComparator(ClusteringComparator clusteringComparator)
- {
- this.clusteringComparator = clusteringComparator;
- }
-
- public int compare(LegacyBound a, LegacyBound b)
- {
- // In the legacy sorting, BOTTOM comes before anything else
- if (a == LegacyBound.BOTTOM)
- return b == LegacyBound.BOTTOM ? 0 : -1;
- if (b == LegacyBound.BOTTOM)
- return 1;
-
- // Excluding BOTTOM, statics are always before anything else.
- if (a.isStatic != b.isStatic)
- return a.isStatic ? -1 : 1;
-
- // We have to be careful with bound comparison because of collections. Namely, if the 2 bounds represent the
- // same prefix, then we should take the collectionName into account before taking the bounds kind
- // (ClusteringPrefix.Kind). This means we can't really call ClusteringComparator.compare() directly.
- // For instance, if
- // a is (bound=INCL_START_BOUND('x'), collectionName='d')
- // b is (bound=INCL_END_BOUND('x'), collectionName='c')
- // Ten b < a since the element 'c' of collection 'x' comes before element 'd', but calling
- // clusteringComparator.compare(a.bound, b.bound) returns -1.
- // See CASSANDRA-13125 for details.
- int sa = a.bound.size();
- int sb = b.bound.size();
- for (int i = 0; i < Math.min(sa, sb); i++)
- {
- int cmp = clusteringComparator.compareComponent(i, a.bound.get(i), b.bound.get(i));
- if (cmp != 0)
- return cmp;
- }
-
- if (sa != sb)
- return sa < sb ? a.bound.kind().comparedToClustering : -b.bound.kind().comparedToClustering;
-
- // Both bound represent the same prefix, compare the collection names
- // If one has a collection name and the other doesn't, the other comes before as it points to the beginning of the row.
- if ((a.collectionName == null) != (b.collectionName == null))
- return a.collectionName == null ? -1 : 1;
-
- // If they both have a collection, compare that first
- if (a.collectionName != null)
- {
- int cmp = UTF8Type.instance.compare(a.collectionName.name.bytes, b.collectionName.name.bytes);
- if (cmp != 0)
- return cmp;
- }
-
- // Lastly, if everything so far is equal, compare their clustering kind
- return ClusteringPrefix.Kind.compare(a.bound.kind(), b.bound.kind());
- }
- }
-
- /**
- * Almost an entire copy of RangeTombstoneList from C* 2.1. The main difference is that LegacyBoundComparator
- * is used in place of {@code Comparator<Composite>} (because Composite doesn't exist any more).
- *
- * This class is needed to allow us to convert single-row deletions and complex deletions into range tombstones
- * and properly merge them into the normal set of range tombstones.
- */
- public static class LegacyRangeTombstoneList
- {
- private final LegacyBoundComparator comparator;
-
- // Note: we don't want to use a List for the markedAts and delTimes to avoid boxing. We could
- // use a List for starts and ends, but having arrays everywhere is almost simpler.
- LegacyBound[] starts;
- LegacyBound[] ends;
- private long[] markedAts;
- private int[] delTimes;
-
- private int size;
-
- private LegacyRangeTombstoneList(LegacyBoundComparator comparator, LegacyBound[] starts, LegacyBound[] ends, long[] markedAts, int[] delTimes, int size)
- {
- assert starts.length == ends.length && starts.length == markedAts.length && starts.length == delTimes.length;
- this.comparator = comparator;
- this.starts = starts;
- this.ends = ends;
- this.markedAts = markedAts;
- this.delTimes = delTimes;
- this.size = size;
- }
-
- public LegacyRangeTombstoneList(LegacyBoundComparator comparator, int capacity)
- {
- this(comparator, new LegacyBound[capacity], new LegacyBound[capacity], new long[capacity], new int[capacity], 0);
- }
-
- @Override
- public String toString()
- {
- StringBuilder sb = new StringBuilder();
- sb.append('[');
- for (int i = 0; i < size; i++)
- {
- if (i > 0)
- sb.append(',');
- sb.append('(').append(starts[i]).append(", ").append(ends[i]).append(')');
- }
- return sb.append(']').toString();
- }
-
- public boolean isEmpty()
- {
- return size == 0;
- }
-
- public int size()
- {
- return size;
- }
-
- /**
- * Adds a new range tombstone.
- *
- * This method will be faster if the new tombstone sort after all the currently existing ones (this is a common use case),
- * but it doesn't assume it.
- */
- public void add(LegacyBound start, LegacyBound end, long markedAt, int delTime)
- {
- if (isEmpty())
- {
- addInternal(0, start, end, markedAt, delTime);
- return;
- }
-
- int c = comparator.compare(ends[size-1], start);
-
- // Fast path if we add in sorted order
- if (c <= 0)
- {
- addInternal(size, start, end, markedAt, delTime);
- }
- else
- {
- // Note: insertFrom expect i to be the insertion point in term of interval ends
- int pos = Arrays.binarySearch(ends, 0, size, start, comparator);
- insertFrom((pos >= 0 ? pos : -pos-1), start, end, markedAt, delTime);
- }
- }
-
- /*
- * Inserts a new element starting at index i. This method assumes that:
- * ends[i-1] <= start <= ends[i]
- *
- * A RangeTombstoneList is a list of range [s_0, e_0]...[s_n, e_n] such that:
- * - s_i <= e_i
- * - e_i <= s_i+1
- * - if s_i == e_i and e_i == s_i+1 then s_i+1 < e_i+1
- * Basically, range are non overlapping except for their bound and in order. And while
- * we allow ranges with the same value for the start and end, we don't allow repeating
- * such range (so we can't have [0, 0][0, 0] even though it would respect the first 2
- * conditions).
- *
- */
-
- /**
- * Adds all the range tombstones of {@code tombstones} to this RangeTombstoneList.
- */
- public void addAll(LegacyRangeTombstoneList tombstones)
- {
- if (tombstones.isEmpty())
- return;
-
- if (isEmpty())
- {
- copyArrays(tombstones, this);
- return;
- }
-
- /*
- * We basically have 2 techniques we can use here: either we repeatedly call add() on tombstones values,
- * or we do a merge of both (sorted) lists. If this lists is bigger enough than the one we add, then
- * calling add() will be faster, otherwise it's merging that will be faster.
- *
- * Let's note that during memtables updates, it might not be uncommon that a new update has only a few range
- * tombstones, while the CF we're adding it to (the one in the memtable) has many. In that case, using add() is
- * likely going to be faster.
- *
- * In other cases however, like when diffing responses from multiple nodes, the tombstone lists we "merge" will
- * be likely sized, so using add() might be a bit inefficient.
- *
- * Roughly speaking (this ignore the fact that updating an element is not exactly constant but that's not a big
- * deal), if n is the size of this list and m is tombstones size, merging is O(n+m) while using add() is O(m*log(n)).
- *
- * But let's not crank up a logarithm computation for that. Long story short, merging will be a bad choice only
- * if this list size is lot bigger that the other one, so let's keep it simple.
- */
- if (size > 10 * tombstones.size)
- {
- for (int i = 0; i < tombstones.size; i++)
- add(tombstones.starts[i], tombstones.ends[i], tombstones.markedAts[i], tombstones.delTimes[i]);
- }
- else
- {
- int i = 0;
- int j = 0;
- while (i < size && j < tombstones.size)
- {
- if (comparator.compare(tombstones.starts[j], ends[i]) <= 0)
- {
- insertFrom(i, tombstones.starts[j], tombstones.ends[j], tombstones.markedAts[j], tombstones.delTimes[j]);
- j++;
- }
- else
- {
- i++;
- }
- }
- // Addds the remaining ones from tombstones if any (note that addInternal will increment size if relevant).
- for (; j < tombstones.size; j++)
- addInternal(size, tombstones.starts[j], tombstones.ends[j], tombstones.markedAts[j], tombstones.delTimes[j]);
- }
- }
-
- private static void copyArrays(LegacyRangeTombstoneList src, LegacyRangeTombstoneList dst)
- {
- dst.grow(src.size);
- System.arraycopy(src.starts, 0, dst.starts, 0, src.size);
- System.arraycopy(src.ends, 0, dst.ends, 0, src.size);
- System.arraycopy(src.markedAts, 0, dst.markedAts, 0, src.size);
- System.arraycopy(src.delTimes, 0, dst.delTimes, 0, src.size);
- dst.size = src.size;
- }
-
- private void insertFrom(int i, LegacyBound start, LegacyBound end, long markedAt, int delTime)
- {
- while (i < size)
- {
- assert i == 0 || comparator.compare(ends[i-1], start) <= 0;
-
- int c = comparator.compare(start, ends[i]);
- assert c <= 0;
- if (c == 0)
- {
- // If start == ends[i], then we can insert from the next one (basically the new element
- // really start at the next element), except for the case where starts[i] == ends[i].
- // In this latter case, if we were to move to next element, we could end up with ...[x, x][x, x]...
- if (comparator.compare(starts[i], ends[i]) == 0)
- {
- // The current element cover a single value which is equal to the start of the inserted
- // element. If the inserted element overwrites the current one, just remove the current
- // (it's included in what we insert) and proceed with the insert.
- if (markedAt > markedAts[i])
- {
- removeInternal(i);
- continue;
- }
-
- // Otherwise (the current singleton interval override the new one), we want to leave the
- // current element and move to the next, unless start == end since that means the new element
- // is in fact fully covered by the current one (so we're done)
- if (comparator.compare(start, end) == 0)
- return;
- }
- i++;
- continue;
- }
-
- // Do we overwrite the current element?
- if (markedAt > markedAts[i])
- {
- // We do overwrite.
-
- // First deal with what might come before the newly added one.
- if (comparator.compare(starts[i], start) < 0)
- {
- addInternal(i, starts[i], start, markedAts[i], delTimes[i]);
- i++;
- // We don't need to do the following line, but in spirit that's what we want to do
- // setInternal(i, start, ends[i], markedAts, delTime])
- }
-
- // now, start <= starts[i]
-
- // Does the new element stops before/at the current one,
- int endCmp = comparator.compare(end, starts[i]);
- if (endCmp <= 0)
- {
- // Here start <= starts[i] and end <= starts[i]
- // This means the current element is before the current one. However, one special
- // case is if end == starts[i] and starts[i] == ends[i]. In that case,
- // the new element entirely overwrite the current one and we can just overwrite
- if (endCmp == 0 && comparator.compare(starts[i], ends[i]) == 0)
- setInternal(i, start, end, markedAt, delTime);
- else
- addInternal(i, start, end, markedAt, delTime);
- return;
- }
-
- // Do we overwrite the current element fully?
- int cmp = comparator.compare(ends[i], end);
- if (cmp <= 0)
- {
- // We do overwrite fully:
- // update the current element until it's end and continue
- // on with the next element (with the new inserted start == current end).
-
- // If we're on the last element, we can optimize
- if (i == size-1)
- {
- setInternal(i, start, end, markedAt, delTime);
- return;
- }
-
- setInternal(i, start, ends[i], markedAt, delTime);
- if (cmp == 0)
- return;
-
- start = ends[i];
- i++;
- }
- else
- {
- // We don't ovewrite fully. Insert the new interval, and then update the now next
- // one to reflect the not overwritten parts. We're then done.
- addInternal(i, start, end, markedAt, delTime);
- i++;
- setInternal(i, end, ends[i], markedAts[i], delTimes[i]);
- return;
- }
- }
- else
- {
- // we don't overwrite the current element
-
- // If the new interval starts before the current one, insert that new interval
- if (comparator.compare(start, starts[i]) < 0)
- {
- // If we stop before the start of the current element, just insert the new
- // interval and we're done; otherwise insert until the beginning of the
- // current element
- if (comparator.compare(end, starts[i]) <= 0)
- {
- addInternal(i, start, end, markedAt, delTime);
- return;
- }
- addInternal(i, start, starts[i], markedAt, delTime);
- i++;
- }
-
- // After that, we're overwritten on the current element but might have
- // some residual parts after ...
-
- // ... unless we don't extend beyond it.
- if (comparator.compare(end, ends[i]) <= 0)
- return;
-
- start = ends[i];
- i++;
- }
- }
-
- // If we got there, then just insert the remainder at the end
- addInternal(i, start, end, markedAt, delTime);
- }
-
- private int capacity()
- {
- return starts.length;
- }
-
- private void addInternal(int i, LegacyBound start, LegacyBound end, long markedAt, int delTime)
- {
- assert i >= 0;
-
- if (size == capacity())
- growToFree(i);
- else if (i < size)
- moveElements(i);
-
- setInternal(i, start, end, markedAt, delTime);
- size++;
- }
-
- private void removeInternal(int i)
- {
- assert i >= 0;
-
- System.arraycopy(starts, i+1, starts, i, size - i - 1);
- System.arraycopy(ends, i+1, ends, i, size - i - 1);
- System.arraycopy(markedAts, i+1, markedAts, i, size - i - 1);
- System.arraycopy(delTimes, i+1, delTimes, i, size - i - 1);
-
- --size;
- starts[size] = null;
- ends[size] = null;
- }
-
- /*
- * Grow the arrays, leaving index i "free" in the process.
- */
- private void growToFree(int i)
- {
- int newLength = (capacity() * 3) / 2 + 1;
- grow(i, newLength);
- }
-
- /*
- * Grow the arrays to match newLength capacity.
- */
- private void grow(int newLength)
- {
- if (capacity() < newLength)
- grow(-1, newLength);
- }
-
- private void grow(int i, int newLength)
- {
- starts = grow(starts, size, newLength, i);
- ends = grow(ends, size, newLength, i);
- markedAts = grow(markedAts, size, newLength, i);
- delTimes = grow(delTimes, size, newLength, i);
- }
-
- private static LegacyBound[] grow(LegacyBound[] a, int size, int newLength, int i)
- {
- if (i < 0 || i >= size)
- return Arrays.copyOf(a, newLength);
-
- LegacyBound[] newA = new LegacyBound[newLength];
- System.arraycopy(a, 0, newA, 0, i);
- System.arraycopy(a, i, newA, i+1, size - i);
- return newA;
- }
-
- private static long[] grow(long[] a, int size, int newLength, int i)
- {
- if (i < 0 || i >= size)
- return Arrays.copyOf(a, newLength);
-
- long[] newA = new long[newLength];
- System.arraycopy(a, 0, newA, 0, i);
- System.arraycopy(a, i, newA, i+1, size - i);
- return newA;
- }
-
- private static int[] grow(int[] a, int size, int newLength, int i)
- {
- if (i < 0 || i >= size)
- return Arrays.copyOf(a, newLength);
-
- int[] newA = new int[newLength];
- System.arraycopy(a, 0, newA, 0, i);
- System.arraycopy(a, i, newA, i+1, size - i);
- return newA;
- }
-
- /*
- * Move elements so that index i is "free", assuming the arrays have at least one free slot at the end.
- */
- private void moveElements(int i)
- {
- if (i >= size)
- return;
-
- System.arraycopy(starts, i, starts, i+1, size - i);
- System.arraycopy(ends, i, ends, i+1, size - i);
- System.arraycopy(markedAts, i, markedAts, i+1, size - i);
- System.arraycopy(delTimes, i, delTimes, i+1, size - i);
- // we set starts[i] to null to indicate the position is now empty, so that we update boundaryHeapSize
- // when we set it
- starts[i] = null;
- }
-
- private void setInternal(int i, LegacyBound start, LegacyBound end, long markedAt, int delTime)
- {
- starts[i] = start;
- ends[i] = end;
- markedAts[i] = markedAt;
- delTimes[i] = delTime;
- }
-
- public void updateDigest(MessageDigest digest)
- {
- ByteBuffer longBuffer = ByteBuffer.allocate(8);
- for (int i = 0; i < size; i++)
- {
- for (int j = 0; j < starts[i].bound.size(); j++)
- digest.update(starts[i].bound.get(j).duplicate());
- if (starts[i].collectionName != null)
- digest.update(starts[i].collectionName.name.bytes.duplicate());
- for (int j = 0; j < ends[i].bound.size(); j++)
- digest.update(ends[i].bound.get(j).duplicate());
- if (ends[i].collectionName != null)
- digest.update(ends[i].collectionName.name.bytes.duplicate());
-
- longBuffer.putLong(0, markedAts[i]);
- digest.update(longBuffer.array(), 0, 8);
- }
- }
-
- public void serialize(DataOutputPlus out, CFMetaData metadata) throws IOException
- {
- out.writeInt(size);
- if (size == 0)
- return;
-
- if (metadata.isCompound())
- serializeCompound(out, metadata.isDense());
- else
- serializeSimple(out);
- }
-
- private void serializeCompound(DataOutputPlus out, boolean isDense) throws IOException
- {
- List<AbstractType<?>> types = new ArrayList<>(comparator.clusteringComparator.subtypes());
-
- if (!isDense)
- types.add(UTF8Type.instance);
-
- CompositeType type = CompositeType.getInstance(types);
-
- for (int i = 0; i < size; i++)
- {
- LegacyBound start = starts[i];
- LegacyBound end = ends[i];
-
- CompositeType.Builder startBuilder = type.builder(start.isStatic);
- CompositeType.Builder endBuilder = type.builder(end.isStatic);
- for (int j = 0; j < start.bound.clustering().size(); j++)
- {
- startBuilder.add(start.bound.get(j));
- endBuilder.add(end.bound.get(j));
- }
-
- if (start.collectionName != null)
- startBuilder.add(start.collectionName.name.bytes);
- if (end.collectionName != null)
- endBuilder.add(end.collectionName.name.bytes);
-
- ByteBufferUtil.writeWithShortLength(startBuilder.build(), out);
- ByteBufferUtil.writeWithShortLength(endBuilder.buildAsEndOfRange(), out);
-
- out.writeInt(delTimes[i]);
- out.writeLong(markedAts[i]);
- }
- }
-
- private void serializeSimple(DataOutputPlus out) throws IOException
- {
- List<AbstractType<?>> types = new ArrayList<>(comparator.clusteringComparator.subtypes());
- assert types.size() == 1 : types;
-
- for (int i = 0; i < size; i++)
- {
- LegacyBound start = starts[i];
- LegacyBound end = ends[i];
-
- ClusteringPrefix startClustering = start.bound.clustering();
- ClusteringPrefix endClustering = end.bound.clustering();
-
- assert startClustering.size() == 1;
- assert endClustering.size() == 1;
-
- ByteBufferUtil.writeWithShortLength(startClustering.get(0), out);
- ByteBufferUtil.writeWithShortLength(endClustering.get(0), out);
-
- out.writeInt(delTimes[i]);
- out.writeLong(markedAts[i]);
- }
- }
-
- public long serializedSize(CFMetaData metadata)
- {
- long size = 0;
- size += TypeSizes.sizeof(this.size);
-
- if (this.size == 0)
- return size;
-
- if (metadata.isCompound())
- return size + serializedSizeCompound(metadata.isDense());
- else
- return size + serializedSizeSimple();
- }
-
- private long serializedSizeCompound(boolean isDense)
- {
- long size = 0;
- List<AbstractType<?>> types = new ArrayList<>(comparator.clusteringComparator.subtypes());
- if (!isDense)
- types.add(UTF8Type.instance);
- CompositeType type = CompositeType.getInstance(types);
-
- for (int i = 0; i < this.size; i++)
- {
- LegacyBound start = starts[i];
- LegacyBound end = ends[i];
-
- CompositeType.Builder startBuilder = type.builder();
- CompositeType.Builder endBuilder = type.builder();
- for (int j = 0; j < start.bound.size(); j++)
- startBuilder.add(start.bound.get(j));
- for (int j = 0; j < end.bound.size(); j++)
- endBuilder.add(end.bound.get(j));
-
- if (start.collectionName != null)
- startBuilder.add(start.collectionName.name.bytes);
- if (end.collectionName != null)
- endBuilder.add(end.collectionName.name.bytes);
-
- size += ByteBufferUtil.serializedSizeWithShortLength(startBuilder.build());
- size += ByteBufferUtil.serializedSizeWithShortLength(endBuilder.buildAsEndOfRange());
-
- size += TypeSizes.sizeof(delTimes[i]);
- size += TypeSizes.sizeof(markedAts[i]);
- }
- return size;
- }
-
- private long serializedSizeSimple()
- {
- long size = 0;
- List<AbstractType<?>> types = new ArrayList<>(comparator.clusteringComparator.subtypes());
- assert types.size() == 1 : types;
-
- for (int i = 0; i < this.size; i++)
- {
- LegacyBound start = starts[i];
- LegacyBound end = ends[i];
-
- ClusteringPrefix startClustering = start.bound.clustering();
- ClusteringPrefix endClustering = end.bound.clustering();
-
- assert startClustering.size() == 1;
- assert endClustering.size() == 1;
-
- size += ByteBufferUtil.serializedSizeWithShortLength(startClustering.get(0));
- size += ByteBufferUtil.serializedSizeWithShortLength(endClustering.get(0));
-
- size += TypeSizes.sizeof(delTimes[i]);
- size += TypeSizes.sizeof(markedAts[i]);
- }
- return size;
- }
- }
-}
diff --git a/src/java/org/apache/cassandra/db/LivenessInfo.java b/src/java/org/apache/cassandra/db/LivenessInfo.java
index c2a2291..b1ea3f6 100644
--- a/src/java/org/apache/cassandra/db/LivenessInfo.java
+++ b/src/java/org/apache/cassandra/db/LivenessInfo.java
@@ -18,12 +18,9 @@
package org.apache.cassandra.db;
import java.util.Objects;
-import java.security.MessageDigest;
-import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.db.rows.Cell;
import org.apache.cassandra.serializers.MarshalException;
-import org.apache.cassandra.utils.FBUtilities;
/**
* Stores the information relating to the liveness of the primary key columns of a row.
@@ -155,9 +152,9 @@
*
* @param digest the digest to add this liveness information to.
*/
- public void digest(MessageDigest digest)
+ public void digest(Digest digest)
{
- FBUtilities.updateWithLong(digest, timestamp());
+ digest.updateWithLong(timestamp());
}
/**
@@ -331,11 +328,11 @@
}
@Override
- public void digest(MessageDigest digest)
+ public void digest(Digest digest)
{
super.digest(digest);
- FBUtilities.updateWithInt(digest, localExpirationTime);
- FBUtilities.updateWithInt(digest, ttl);
+ digest.updateWithInt(localExpirationTime)
+ .updateWithInt(ttl);
}
@Override
diff --git a/src/java/org/apache/cassandra/db/Memtable.java b/src/java/org/apache/cassandra/db/Memtable.java
index 5af789e..c969616 100644
--- a/src/java/org/apache/cassandra/db/Memtable.java
+++ b/src/java/org/apache/cassandra/db/Memtable.java
@@ -29,10 +29,10 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.config.SchemaConstants;
+import org.apache.cassandra.schema.SchemaConstants;
import org.apache.cassandra.db.commitlog.CommitLog;
import org.apache.cassandra.db.commitlog.CommitLogPosition;
import org.apache.cassandra.db.commitlog.IntervalSet;
@@ -77,10 +77,6 @@
case heap_buffers:
return new SlabPool(heapLimit, 0, DatabaseDescriptor.getMemtableCleanupThreshold(), new ColumnFamilyStore.FlushLargestColumnFamily());
case offheap_buffers:
- if (!FileUtils.isCleanerAvailable)
- {
- throw new IllegalStateException("Could not free direct byte buffer: offheap_buffers is not a safe memtable_allocation_type without this ability, please adjust your config. This feature is only guaranteed to work on an Oracle JVM. Refusing to start.");
- }
return new SlabPool(heapLimit, offHeapLimit, DatabaseDescriptor.getMemtableCleanupThreshold(), new ColumnFamilyStore.FlushLargestColumnFamily());
case offheap_objects:
return new NativePool(heapLimit, offHeapLimit, DatabaseDescriptor.getMemtableCleanupThreshold(), new ColumnFamilyStore.FlushLargestColumnFamily());
@@ -143,19 +139,19 @@
this.cfs = cfs;
this.commitLogLowerBound = commitLogLowerBound;
this.allocator = MEMORY_POOL.newAllocator();
- this.initialComparator = cfs.metadata.comparator;
+ this.initialComparator = cfs.metadata().comparator;
this.cfs.scheduleFlush();
- this.columnsCollector = new ColumnsCollector(cfs.metadata.partitionColumns());
+ this.columnsCollector = new ColumnsCollector(cfs.metadata().regularAndStaticColumns());
}
// ONLY to be used for testing, to create a mock Memtable
@VisibleForTesting
- public Memtable(CFMetaData metadata)
+ public Memtable(TableMetadata metadata)
{
this.initialComparator = metadata.comparator;
this.cfs = null;
this.allocator = null;
- this.columnsCollector = new ColumnsCollector(metadata.partitionColumns());
+ this.columnsCollector = new ColumnsCollector(metadata.regularAndStaticColumns());
}
public MemtableAllocator getAllocator()
@@ -247,7 +243,7 @@
*/
public boolean isExpired()
{
- int period = cfs.metadata.params.memtableFlushPeriodInMs;
+ int period = cfs.metadata().params.memtableFlushPeriodInMs;
return period > 0 && (System.nanoTime() - creationNano >= TimeUnit.MILLISECONDS.toNanos(period));
}
@@ -339,7 +335,7 @@
100 * allocator.onHeap().ownershipRatio(), 100 * allocator.offHeap().ownershipRatio());
}
- public MemtableUnfilteredPartitionIterator makePartitionIterator(final ColumnFilter columnFilter, final DataRange dataRange, final boolean isForThrift)
+ public MemtableUnfilteredPartitionIterator makePartitionIterator(final ColumnFilter columnFilter, final DataRange dataRange)
{
AbstractBounds<PartitionPosition> keyRange = dataRange.keyRange();
@@ -365,7 +361,7 @@
final Iterator<Map.Entry<PartitionPosition, AtomicBTreePartition>> iter = subMap.entrySet().iterator();
- return new MemtableUnfilteredPartitionIterator(cfs, iter, isForThrift, minLocalDeletionTime, columnFilter, dataRange);
+ return new MemtableUnfilteredPartitionIterator(cfs, iter, minLocalDeletionTime, columnFilter, dataRange);
}
private int findMinLocalDeletionTime(Iterator<Map.Entry<PartitionPosition, AtomicBTreePartition>> iterator)
@@ -440,9 +436,9 @@
this.isBatchLogTable = cfs.name.equals(SystemKeyspace.BATCHES) && cfs.keyspace.getName().equals(SchemaConstants.SYSTEM_KEYSPACE_NAME);
if (flushLocation == null)
- writer = createFlushWriter(txn, cfs.getSSTablePath(getDirectories().getWriteableLocationAsFile(estimatedSize)), columnsCollector.get(), statsCollector.get());
+ writer = createFlushWriter(txn, cfs.newSSTableDescriptor(getDirectories().getWriteableLocationAsFile(estimatedSize)), columnsCollector.get(), statsCollector.get());
else
- writer = createFlushWriter(txn, cfs.getSSTablePath(getDirectories().getLocationForDisk(flushLocation)), columnsCollector.get(), statsCollector.get());
+ writer = createFlushWriter(txn, cfs.newSSTableDescriptor(getDirectories().getLocationForDisk(flushLocation)), columnsCollector.get(), statsCollector.get());
}
@@ -453,7 +449,7 @@
private void writeSortedContents()
{
- logger.debug("Writing {}, flushed range = ({}, {}]", Memtable.this.toString(), from, to);
+ logger.info("Writing {}, flushed range = ({}, {}]", Memtable.this.toString(), from, to);
boolean trackContention = logger.isTraceEnabled();
int heavilyContendedRowCount = 0;
@@ -482,10 +478,10 @@
}
long bytesFlushed = writer.getFilePointer();
- logger.debug("Completed flushing {} ({}) for commitlog position {}",
- writer.getFilename(),
- FBUtilities.prettyPrintMemory(bytesFlushed),
- commitLogUpperBound);
+ logger.info("Completed flushing {} ({}) for commitlog position {}",
+ writer.getFilename(),
+ FBUtilities.prettyPrintMemory(bytesFlushed),
+ commitLogUpperBound);
// Update the metrics
cfs.metric.bytesFlushed.inc(bytesFlushed);
@@ -494,18 +490,20 @@
}
public SSTableMultiWriter createFlushWriter(LifecycleTransaction txn,
- String filename,
- PartitionColumns columns,
- EncodingStats stats)
+ Descriptor descriptor,
+ RegularAndStaticColumns columns,
+ EncodingStats stats)
{
- MetadataCollector sstableMetadataCollector = new MetadataCollector(cfs.metadata.comparator)
+ MetadataCollector sstableMetadataCollector = new MetadataCollector(cfs.metadata().comparator)
.commitLogIntervals(new IntervalSet<>(commitLogLowerBound.get(), commitLogUpperBound.get()));
- return cfs.createSSTableMultiWriter(Descriptor.fromFilename(filename),
+ return cfs.createSSTableMultiWriter(descriptor,
toFlush.size(),
ActiveRepairService.UNREPAIRED_SSTABLE,
+ ActiveRepairService.NO_PENDING_REPAIR,
+ false,
sstableMetadataCollector,
- new SerializationHeader(true, cfs.metadata, columns, stats), txn);
+ new SerializationHeader(true, cfs.metadata(), columns, stats), txn);
}
@Override
@@ -541,34 +539,27 @@
{
private final ColumnFamilyStore cfs;
private final Iterator<Map.Entry<PartitionPosition, AtomicBTreePartition>> iter;
- private final boolean isForThrift;
private final int minLocalDeletionTime;
private final ColumnFilter columnFilter;
private final DataRange dataRange;
- public MemtableUnfilteredPartitionIterator(ColumnFamilyStore cfs, Iterator<Map.Entry<PartitionPosition, AtomicBTreePartition>> iter, boolean isForThrift, int minLocalDeletionTime, ColumnFilter columnFilter, DataRange dataRange)
+ public MemtableUnfilteredPartitionIterator(ColumnFamilyStore cfs, Iterator<Map.Entry<PartitionPosition, AtomicBTreePartition>> iter, int minLocalDeletionTime, ColumnFilter columnFilter, DataRange dataRange)
{
this.cfs = cfs;
this.iter = iter;
- this.isForThrift = isForThrift;
this.minLocalDeletionTime = minLocalDeletionTime;
this.columnFilter = columnFilter;
this.dataRange = dataRange;
}
- public boolean isForThrift()
- {
- return isForThrift;
- }
-
public int getMinLocalDeletionTime()
{
return minLocalDeletionTime;
}
- public CFMetaData metadata()
+ public TableMetadata metadata()
{
- return cfs.metadata;
+ return cfs.metadata();
}
public boolean hasNext()
@@ -590,25 +581,25 @@
private static class ColumnsCollector
{
- private final HashMap<ColumnDefinition, AtomicBoolean> predefined = new HashMap<>();
- private final ConcurrentSkipListSet<ColumnDefinition> extra = new ConcurrentSkipListSet<>();
- ColumnsCollector(PartitionColumns columns)
+ private final HashMap<ColumnMetadata, AtomicBoolean> predefined = new HashMap<>();
+ private final ConcurrentSkipListSet<ColumnMetadata> extra = new ConcurrentSkipListSet<>();
+ ColumnsCollector(RegularAndStaticColumns columns)
{
- for (ColumnDefinition def : columns.statics)
+ for (ColumnMetadata def : columns.statics)
predefined.put(def, new AtomicBoolean());
- for (ColumnDefinition def : columns.regulars)
+ for (ColumnMetadata def : columns.regulars)
predefined.put(def, new AtomicBoolean());
}
- public void update(PartitionColumns columns)
+ public void update(RegularAndStaticColumns columns)
{
- for (ColumnDefinition s : columns.statics)
+ for (ColumnMetadata s : columns.statics)
update(s);
- for (ColumnDefinition r : columns.regulars)
+ for (ColumnMetadata r : columns.regulars)
update(r);
}
- private void update(ColumnDefinition definition)
+ private void update(ColumnMetadata definition)
{
AtomicBoolean present = predefined.get(definition);
if (present != null)
@@ -622,10 +613,10 @@
}
}
- public PartitionColumns get()
+ public RegularAndStaticColumns get()
{
- PartitionColumns.Builder builder = PartitionColumns.builder();
- for (Map.Entry<ColumnDefinition, AtomicBoolean> e : predefined.entrySet())
+ RegularAndStaticColumns.Builder builder = RegularAndStaticColumns.builder();
+ for (Map.Entry<ColumnMetadata, AtomicBoolean> e : predefined.entrySet())
if (e.getValue().get())
builder.add(e.getKey());
return builder.addAll(extra).build();
diff --git a/src/java/org/apache/cassandra/db/MigrationRequestVerbHandler.java b/src/java/org/apache/cassandra/db/MigrationRequestVerbHandler.java
deleted file mode 100644
index 3666b27..0000000
--- a/src/java/org/apache/cassandra/db/MigrationRequestVerbHandler.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.db;
-
-import java.util.Collection;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.cassandra.net.IVerbHandler;
-import org.apache.cassandra.net.MessageIn;
-import org.apache.cassandra.net.MessageOut;
-import org.apache.cassandra.net.MessagingService;
-import org.apache.cassandra.schema.SchemaKeyspace;
-import org.apache.cassandra.service.MigrationManager;
-
-/**
- * Sends it's current schema state in form of mutations in reply to the remote node's request.
- * Such a request is made when one of the nodes, by means of Gossip, detects schema disagreement in the ring.
- */
-public class MigrationRequestVerbHandler implements IVerbHandler
-{
- private static final Logger logger = LoggerFactory.getLogger(MigrationRequestVerbHandler.class);
-
- public void doVerb(MessageIn message, int id)
- {
- logger.trace("Received migration request from {}.", message.from);
- MessageOut<Collection<Mutation>> response = new MessageOut<>(MessagingService.Verb.INTERNAL_RESPONSE,
- SchemaKeyspace.convertSchemaToMutations(),
- MigrationManager.MigrationsSerializer.instance);
- MessagingService.instance().sendReply(response, id, message.from);
- }
-}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/db/MultiCBuilder.java b/src/java/org/apache/cassandra/db/MultiCBuilder.java
index ae8c26c..c4cff02 100644
--- a/src/java/org/apache/cassandra/db/MultiCBuilder.java
+++ b/src/java/org/apache/cassandra/db/MultiCBuilder.java
@@ -23,7 +23,7 @@
import java.util.List;
import java.util.NavigableSet;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.btree.BTreeSet;
@@ -177,7 +177,7 @@
public abstract NavigableSet<ClusteringBound> buildBoundForSlice(boolean isStart,
boolean isInclusive,
boolean isOtherBoundInclusive,
- List<ColumnDefinition> columnDefs);
+ List<ColumnMetadata> columnDefs);
/**
* Builds the <code>ClusteringBound</code>s
@@ -266,7 +266,7 @@
public NavigableSet<ClusteringBound> buildBoundForSlice(boolean isStart,
boolean isInclusive,
boolean isOtherBoundInclusive,
- List<ColumnDefinition> columnDefs)
+ List<ColumnMetadata> columnDefs)
{
return buildBound(isStart, columnDefs.get(0).isReversedType() ? isOtherBoundInclusive : isInclusive);
}
@@ -421,7 +421,7 @@
public NavigableSet<ClusteringBound> buildBoundForSlice(boolean isStart,
boolean isInclusive,
boolean isOtherBoundInclusive,
- List<ColumnDefinition> columnDefs)
+ List<ColumnMetadata> columnDefs)
{
built = true;
@@ -454,7 +454,7 @@
// For example: if we have clustering_0 DESC and clustering_1 ASC a slice like (clustering_0, clustering_1) > (1, 2)
// will produce 2 slices: [BOTTOM, 1) and (1.2, 1]
// So, the END bound will return 2 bounds with the same values 1
- ColumnDefinition lastColumn = columnDefs.get(columnDefs.size() - 1);
+ ColumnMetadata lastColumn = columnDefs.get(columnDefs.size() - 1);
if (elements.size() <= lastColumn.position() && i < m - 1 && elements.equals(elementsList.get(i + 1)))
{
set.add(builder.buildBoundWith(elements, isStart, false));
@@ -463,7 +463,7 @@
}
// Handle the normal bounds
- ColumnDefinition column = columnDefs.get(elements.size() - 1 - offset);
+ ColumnMetadata column = columnDefs.get(elements.size() - 1 - offset);
set.add(builder.buildBoundWith(elements, isStart, column.isReversedType() ? isOtherBoundInclusive : isInclusive));
}
return set.build();
diff --git a/src/java/org/apache/cassandra/db/MutableDeletionInfo.java b/src/java/org/apache/cassandra/db/MutableDeletionInfo.java
index 39728c5..356d763 100644
--- a/src/java/org/apache/cassandra/db/MutableDeletionInfo.java
+++ b/src/java/org/apache/cassandra/db/MutableDeletionInfo.java
@@ -22,6 +22,7 @@
import com.google.common.base.Objects;
+import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.rows.*;
import org.apache.cassandra.db.rows.EncodingStats;
import org.apache.cassandra.utils.ObjectSizes;
@@ -113,8 +114,8 @@
public void add(RangeTombstone tombstone, ClusteringComparator comparator)
{
- if (ranges == null)
- ranges = new RangeTombstoneList(comparator, 1);
+ if (ranges == null) // Introduce getInitialRangeTombstoneAllocationSize
+ ranges = new RangeTombstoneList(comparator, DatabaseDescriptor.getInitialRangeTombstoneListAllocationSize());
ranges.add(tombstone);
}
diff --git a/src/java/org/apache/cassandra/db/Mutation.java b/src/java/org/apache/cassandra/db/Mutation.java
index df6deba..16d20db 100644
--- a/src/java/org/apache/cassandra/db/Mutation.java
+++ b/src/java/org/apache/cassandra/db/Mutation.java
@@ -18,93 +18,88 @@
package org.apache.cassandra.db;
import java.io.IOException;
-import java.nio.ByteBuffer;
import java.util.*;
import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
-import com.google.common.base.Throwables;
-import com.google.common.util.concurrent.Uninterruptibles;
+import com.google.common.collect.ImmutableCollection;
+import com.google.common.collect.ImmutableMap;
import org.apache.commons.lang3.StringUtils;
-import org.apache.cassandra.config.CFMetaData;
+
import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.config.Schema;
import org.apache.cassandra.db.partitions.PartitionUpdate;
-import org.apache.cassandra.db.rows.SerializationHelper;
+import org.apache.cassandra.db.rows.DeserializationHelper;
import org.apache.cassandra.io.IVersionedSerializer;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
-import org.apache.cassandra.net.MessageOut;
-import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.utils.ByteBufferUtil;
-// TODO convert this to a Builder pattern instead of encouraging M.add directly,
-// which is less-efficient since we have to keep a mutable HashMap around
+import static org.apache.cassandra.net.MessagingService.VERSION_30;
+import static org.apache.cassandra.net.MessagingService.VERSION_3014;
+import static org.apache.cassandra.net.MessagingService.VERSION_40;
+import static org.apache.cassandra.utils.MonotonicClock.approxTime;
+
public class Mutation implements IMutation
{
public static final MutationSerializer serializer = new MutationSerializer();
- public static final String FORWARD_TO = "FWD_TO";
- public static final String FORWARD_FROM = "FWD_FRM";
-
// todo this is redundant
// when we remove it, also restore SerializationsTest.testMutationRead to not regenerate new Mutations each test
private final String keyspaceName;
private final DecoratedKey key;
// map of column family id to mutations for that column family.
- private final Map<UUID, PartitionUpdate> modifications;
+ private final ImmutableMap<TableId, PartitionUpdate> modifications;
- // Time at which this mutation was instantiated
- public final long createdAt = System.currentTimeMillis();
+ // Time at which this mutation or the builder that built it was instantiated
+ final long approxCreatedAtNanos;
// keep track of when mutation has started waiting for a MV partition lock
- public final AtomicLong viewLockAcquireStart = new AtomicLong(0);
+ final AtomicLong viewLockAcquireStart = new AtomicLong(0);
- private boolean cdcEnabled = false;
-
- public Mutation(String keyspaceName, DecoratedKey key)
- {
- this(keyspaceName, key, new HashMap<>());
- }
+ private final boolean cdcEnabled;
public Mutation(PartitionUpdate update)
{
- this(update.metadata().ksName, update.partitionKey(), Collections.singletonMap(update.metadata().cfId, update));
+ this(update.metadata().keyspace, update.partitionKey(), ImmutableMap.of(update.metadata().id, update), approxTime.now());
}
- protected Mutation(String keyspaceName, DecoratedKey key, Map<UUID, PartitionUpdate> modifications)
+ public Mutation(String keyspaceName, DecoratedKey key, ImmutableMap<TableId, PartitionUpdate> modifications, long approxCreatedAtNanos)
{
this.keyspaceName = keyspaceName;
this.key = key;
this.modifications = modifications;
+
+ boolean cdc = false;
for (PartitionUpdate pu : modifications.values())
- cdcEnabled |= pu.metadata().params.cdc;
+ cdc |= pu.metadata().params.cdc;
+ this.cdcEnabled = cdc;
+ this.approxCreatedAtNanos = approxCreatedAtNanos;
}
- public Mutation copy()
+ public Mutation without(Set<TableId> tableIds)
{
- return new Mutation(keyspaceName, key, new HashMap<>(modifications));
- }
-
- public Mutation without(Set<UUID> cfIds)
- {
- if (cfIds.isEmpty())
+ if (tableIds.isEmpty())
return this;
- Mutation copy = copy();
- copy.modifications.keySet().removeAll(cfIds);
+ ImmutableMap.Builder<TableId, PartitionUpdate> builder = new ImmutableMap.Builder<>();
+ for (Map.Entry<TableId, PartitionUpdate> update : modifications.entrySet())
+ {
+ if (!tableIds.contains(update.getKey()))
+ {
+ builder.put(update);
+ }
+ }
- copy.cdcEnabled = false;
- for (PartitionUpdate pu : modifications.values())
- copy.cdcEnabled |= pu.metadata().params.cdc;
-
- return copy;
+ return new Mutation(keyspaceName, key, builder.build(), approxCreatedAtNanos);
}
- public Mutation without(UUID cfId)
+ public Mutation without(TableId tableId)
{
- return without(Collections.singleton(cfId));
+ return without(Collections.singleton(tableId));
}
public String getKeyspaceName()
@@ -112,7 +107,7 @@
return keyspaceName;
}
- public Collection<UUID> getColumnFamilyIds()
+ public Collection<TableId> getTableIds()
{
return modifications.keySet();
}
@@ -122,41 +117,23 @@
return key;
}
- public Collection<PartitionUpdate> getPartitionUpdates()
+ public ImmutableCollection<PartitionUpdate> getPartitionUpdates()
{
return modifications.values();
}
- public PartitionUpdate getPartitionUpdate(UUID cfId)
+ public void validateSize(int version, int overhead)
{
- return modifications.get(cfId);
+ long totalSize = serializedSize(version) + overhead;
+ if(totalSize > MAX_MUTATION_SIZE)
+ {
+ throw new MutationExceededMaxSizeException(this, version, totalSize);
+ }
}
- /**
- * Adds PartitionUpdate to the local set of modifications.
- * Assumes no updates for the Table this PartitionUpdate impacts.
- *
- * @param update PartitionUpdate to append to Modifications list
- * @return Mutation this mutation
- * @throws IllegalArgumentException If PartitionUpdate for duplicate table is passed as argument
- */
- public Mutation add(PartitionUpdate update)
+ public PartitionUpdate getPartitionUpdate(TableMetadata table)
{
- assert update != null;
- assert update.partitionKey().getPartitioner() == key.getPartitioner();
-
- cdcEnabled |= update.metadata().params.cdc;
-
- PartitionUpdate prev = modifications.put(update.metadata().cfId, update);
- if (prev != null)
- // developer error
- throw new IllegalArgumentException("Table " + update.metadata().cfName + " already has modifications in this mutation: " + prev);
- return this;
- }
-
- public PartitionUpdate get(CFMetaData cfm)
- {
- return modifications.get(cfm.cfId);
+ return table == null ? null : modifications.get(table.id);
}
public boolean isEmpty()
@@ -182,7 +159,7 @@
if (mutations.size() == 1)
return mutations.get(0);
- Set<UUID> updatedTables = new HashSet<>();
+ Set<TableId> updatedTables = new HashSet<>();
String ks = null;
DecoratedKey key = null;
for (Mutation mutation : mutations)
@@ -197,8 +174,8 @@
}
List<PartitionUpdate> updates = new ArrayList<>(mutations.size());
- Map<UUID, PartitionUpdate> modifications = new HashMap<>(updatedTables.size());
- for (UUID table : updatedTables)
+ ImmutableMap.Builder<TableId, PartitionUpdate> modifications = new ImmutableMap.Builder<>();
+ for (TableId table : updatedTables)
{
for (Mutation mutation : mutations)
{
@@ -213,7 +190,7 @@
modifications.put(table, updates.size() == 1 ? updates.get(0) : PartitionUpdate.merge(updates));
updates.clear();
}
- return new Mutation(ks, key, modifications);
+ return new Mutation(ks, key, modifications.build(), approxTime.now());
}
public CompletableFuture<?> applyFuture()
@@ -246,19 +223,9 @@
apply(false);
}
- public MessageOut<Mutation> createMessage()
+ public long getTimeout(TimeUnit unit)
{
- return createMessage(MessagingService.Verb.MUTATION);
- }
-
- public MessageOut<Mutation> createMessage(MessagingService.Verb verb)
- {
- return new MessageOut<>(verb, this, serializer);
- }
-
- public long getTimeout()
- {
- return DatabaseDescriptor.getWriteRpcTimeout();
+ return DatabaseDescriptor.getWriteRpcTimeout(unit);
}
public int smallestGCGS()
@@ -288,10 +255,10 @@
if (shallow)
{
List<String> cfnames = new ArrayList<>(modifications.size());
- for (UUID cfid : modifications.keySet())
+ for (TableId tableId : modifications.keySet())
{
- CFMetaData cfm = Schema.instance.getCFMetaData(cfid);
- cfnames.add(cfm == null ? "-dropped-" : cfm.cfName);
+ TableMetadata cfm = Schema.instance.getTableMetadata(tableId);
+ cfnames.add(cfm == null ? "-dropped-" : cfm.name);
}
buff.append(StringUtils.join(cfnames, ", "));
}
@@ -301,6 +268,30 @@
}
return buff.append("])").toString();
}
+ private int serializedSize30;
+ private int serializedSize3014;
+ private int serializedSize40;
+
+ public int serializedSize(int version)
+ {
+ switch (version)
+ {
+ case VERSION_30:
+ if (serializedSize30 == 0)
+ serializedSize30 = (int) serializer.serializedSize(this, VERSION_30);
+ return serializedSize30;
+ case VERSION_3014:
+ if (serializedSize3014 == 0)
+ serializedSize3014 = (int) serializer.serializedSize(this, VERSION_3014);
+ return serializedSize3014;
+ case VERSION_40:
+ if (serializedSize40 == 0)
+ serializedSize40 = (int) serializer.serializedSize(this, VERSION_40);
+ return serializedSize40;
+ default:
+ throw new IllegalStateException("Unknown serialization version: " + version);
+ }
+ }
/**
* Creates a new simple mutuation builder.
@@ -349,7 +340,7 @@
* @return a builder for the partition identified by {@code metadata} (and the partition key for which this is a
* mutation of).
*/
- public PartitionUpdate.SimpleBuilder update(CFMetaData metadata);
+ public PartitionUpdate.SimpleBuilder update(TableMetadata metadata);
/**
* Adds an update for table identified by the provided name and return a builder for that partition.
@@ -372,90 +363,96 @@
{
public void serialize(Mutation mutation, DataOutputPlus out, int version) throws IOException
{
- if (version < MessagingService.VERSION_20)
- out.writeUTF(mutation.getKeyspaceName());
-
/* serialize the modifications in the mutation */
int size = mutation.modifications.size();
-
- if (version < MessagingService.VERSION_30)
- {
- ByteBufferUtil.writeWithShortLength(mutation.key().getKey(), out);
- out.writeInt(size);
- }
- else
- {
- out.writeUnsignedVInt(size);
- }
+ out.writeUnsignedVInt(size);
assert size > 0;
- for (Map.Entry<UUID, PartitionUpdate> entry : mutation.modifications.entrySet())
+ for (Map.Entry<TableId, PartitionUpdate> entry : mutation.modifications.entrySet())
PartitionUpdate.serializer.serialize(entry.getValue(), out, version);
}
- public Mutation deserialize(DataInputPlus in, int version, SerializationHelper.Flag flag) throws IOException
+ public Mutation deserialize(DataInputPlus in, int version, DeserializationHelper.Flag flag) throws IOException
{
- if (version < MessagingService.VERSION_20)
- in.readUTF(); // read pre-2.0 keyspace name
-
- ByteBuffer key = null;
- int size;
- if (version < MessagingService.VERSION_30)
- {
- key = ByteBufferUtil.readWithShortLength(in);
- size = in.readInt();
- }
- else
- {
- size = (int)in.readUnsignedVInt();
- }
-
+ int size = (int)in.readUnsignedVInt();
assert size > 0;
- PartitionUpdate update = PartitionUpdate.serializer.deserialize(in, version, flag, key);
+ PartitionUpdate update = PartitionUpdate.serializer.deserialize(in, version, flag);
if (size == 1)
return new Mutation(update);
- Map<UUID, PartitionUpdate> modifications = new HashMap<>(size);
+ ImmutableMap.Builder<TableId, PartitionUpdate> modifications = new ImmutableMap.Builder<>();
DecoratedKey dk = update.partitionKey();
- modifications.put(update.metadata().cfId, update);
+ modifications.put(update.metadata().id, update);
for (int i = 1; i < size; ++i)
{
- update = PartitionUpdate.serializer.deserialize(in, version, flag, dk);
- modifications.put(update.metadata().cfId, update);
+ update = PartitionUpdate.serializer.deserialize(in, version, flag);
+ modifications.put(update.metadata().id, update);
}
-
- return new Mutation(update.metadata().ksName, dk, modifications);
+ return new Mutation(update.metadata().keyspace, dk, modifications.build(), approxTime.now());
}
public Mutation deserialize(DataInputPlus in, int version) throws IOException
{
- return deserialize(in, version, SerializationHelper.Flag.FROM_REMOTE);
+ return deserialize(in, version, DeserializationHelper.Flag.FROM_REMOTE);
}
public long serializedSize(Mutation mutation, int version)
{
- int size = 0;
-
- if (version < MessagingService.VERSION_20)
- size += TypeSizes.sizeof(mutation.getKeyspaceName());
-
- if (version < MessagingService.VERSION_30)
- {
- int keySize = mutation.key().getKey().remaining();
- size += TypeSizes.sizeof((short) keySize) + keySize;
- size += TypeSizes.sizeof(mutation.modifications.size());
- }
- else
- {
- size += TypeSizes.sizeofUnsignedVInt(mutation.modifications.size());
- }
-
- for (Map.Entry<UUID, PartitionUpdate> entry : mutation.modifications.entrySet())
+ int size = TypeSizes.sizeofUnsignedVInt(mutation.modifications.size());
+ for (Map.Entry<TableId, PartitionUpdate> entry : mutation.modifications.entrySet())
size += PartitionUpdate.serializer.serializedSize(entry.getValue(), version);
return size;
}
}
+
+ /**
+ * Collects finalized partition updates
+ */
+ public static class PartitionUpdateCollector
+ {
+ private final ImmutableMap.Builder<TableId, PartitionUpdate> modifications = new ImmutableMap.Builder<>();
+ private final String keyspaceName;
+ private final DecoratedKey key;
+ private final long approxCreatedAtNanos = approxTime.now();
+ private boolean empty = true;
+
+ public PartitionUpdateCollector(String keyspaceName, DecoratedKey key)
+ {
+ this.keyspaceName = keyspaceName;
+ this.key = key;
+ }
+
+ public PartitionUpdateCollector add(PartitionUpdate partitionUpdate)
+ {
+ assert partitionUpdate != null;
+ assert partitionUpdate.partitionKey().getPartitioner() == key.getPartitioner();
+ // note that ImmutableMap.Builder only allows put:ing the same key once, it will fail during build() below otherwise
+ modifications.put(partitionUpdate.metadata().id, partitionUpdate);
+ empty = false;
+ return this;
+ }
+
+ public DecoratedKey key()
+ {
+ return key;
+ }
+
+ public String getKeyspaceName()
+ {
+ return keyspaceName;
+ }
+
+ public boolean isEmpty()
+ {
+ return empty;
+ }
+
+ public Mutation build()
+ {
+ return new Mutation(keyspaceName, key, modifications.build(), approxCreatedAtNanos);
+ }
+ }
}
diff --git a/src/java/org/apache/cassandra/db/MutationExceededMaxSizeException.java b/src/java/org/apache/cassandra/db/MutationExceededMaxSizeException.java
new file mode 100644
index 0000000..084c21e
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/MutationExceededMaxSizeException.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db;
+
+import java.util.Iterator;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import org.apache.cassandra.db.partitions.PartitionUpdate;
+
+import static org.apache.cassandra.db.IMutation.MAX_MUTATION_SIZE;
+
+public class MutationExceededMaxSizeException extends RuntimeException
+{
+ public static final int PARTITION_MESSAGE_LIMIT = 1024;
+
+ public final long mutationSize;
+
+ MutationExceededMaxSizeException(IMutation mutation, int serializationVersion, long totalSize)
+ {
+ super(prepareMessage(mutation, serializationVersion, totalSize));
+ this.mutationSize = totalSize;
+ }
+
+ private static String prepareMessage(IMutation mutation, int version, long totalSize)
+ {
+ List<String> topPartitions = mutation.getPartitionUpdates().stream()
+ .sorted((upd1, upd2) ->
+ Long.compare(PartitionUpdate.serializer.serializedSize(upd2, version),
+ PartitionUpdate.serializer.serializedSize(upd1, version)))
+ .map(upd -> String.format("%s.%s",
+ upd.metadata().name,
+ upd.metadata().partitionKeyType.getString(upd.partitionKey().getKey())))
+ .collect(Collectors.toList());
+
+ String topKeys = makeTopKeysString(topPartitions, PARTITION_MESSAGE_LIMIT);
+ return String.format("Encountered an oversized mutation (%d/%d) for keyspace: %s. Top keys are: %s",
+ totalSize,
+ MAX_MUTATION_SIZE,
+ mutation.getKeyspaceName(),
+ topKeys);
+ }
+
+ @VisibleForTesting
+ static String makeTopKeysString(List<String> keys, int maxLength) {
+ Iterator<String> iterator = keys.listIterator();
+ StringBuilder stringBuilder = new StringBuilder();
+ while (iterator.hasNext())
+ {
+ String key = iterator.next();
+
+ if (stringBuilder.length() == 0)
+ {
+ stringBuilder.append(key); //ensures atleast one key is added
+ iterator.remove();
+ }
+ else if (stringBuilder.length() + key.length() + 2 <= maxLength) // 2 for ", "
+ {
+ stringBuilder.append(", ").append(key);
+ iterator.remove();
+ }
+ else
+ break;
+ }
+
+ if (keys.size() > 0)
+ stringBuilder.append(" and ").append(keys.size()).append(" more.");
+
+ return stringBuilder.toString();
+ }
+}
diff --git a/src/java/org/apache/cassandra/db/MutationVerbHandler.java b/src/java/org/apache/cassandra/db/MutationVerbHandler.java
index 5888438..bcb9cc7 100644
--- a/src/java/org/apache/cassandra/db/MutationVerbHandler.java
+++ b/src/java/org/apache/cassandra/db/MutationVerbHandler.java
@@ -17,22 +17,19 @@
*/
package org.apache.cassandra.db;
-import java.io.DataInputStream;
-import java.io.IOException;
-import java.net.InetAddress;
-
-import org.apache.cassandra.batchlog.LegacyBatchlogMigrator;
import org.apache.cassandra.exceptions.WriteTimeoutException;
-import org.apache.cassandra.io.util.FastByteArrayInputStream;
+import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.net.*;
import org.apache.cassandra.tracing.Tracing;
public class MutationVerbHandler implements IVerbHandler<Mutation>
{
- private void reply(int id, InetAddress replyTo)
+ public static final MutationVerbHandler instance = new MutationVerbHandler();
+
+ private void respond(Message<?> respondTo, InetAddressAndPort respondToAddress)
{
- Tracing.trace("Enqueuing response to {}", replyTo);
- MessagingService.instance().sendReply(WriteResponse.createMessage(), id, replyTo);
+ Tracing.trace("Enqueuing response to {}", respondToAddress);
+ MessagingService.instance().send(respondTo.emptyResponse(), respondToAddress);
}
private void failed()
@@ -40,35 +37,28 @@
Tracing.trace("Payload application resulted in WriteTimeout, not replying");
}
- public void doVerb(MessageIn<Mutation> message, int id) throws IOException
+ public void doVerb(Message<Mutation> message)
{
// Check if there were any forwarding headers in this message
- byte[] from = message.parameters.get(Mutation.FORWARD_FROM);
- InetAddress replyTo;
+ InetAddressAndPort from = message.respondTo();
+ InetAddressAndPort respondToAddress;
if (from == null)
{
- replyTo = message.from;
- byte[] forwardBytes = message.parameters.get(Mutation.FORWARD_TO);
- if (forwardBytes != null)
- forwardToLocalNodes(message.payload, message.verb, forwardBytes, message.from);
+ respondToAddress = message.from();
+ ForwardingInfo forwardTo = message.forwardTo();
+ if (forwardTo != null) forwardToLocalNodes(message, forwardTo);
}
else
{
- replyTo = InetAddress.getByAddress(from);
+ respondToAddress = from;
}
try
{
- if (message.version < MessagingService.VERSION_30 && LegacyBatchlogMigrator.isLegacyBatchlogMutation(message.payload))
- {
- LegacyBatchlogMigrator.handleLegacyMutation(message.payload);
- reply(id, replyTo);
- }
- else
- message.payload.applyFuture().thenAccept(o -> reply(id, replyTo)).exceptionally(wto -> {
- failed();
- return null;
- });
+ message.payload.applyFuture().thenAccept(o -> respond(message, respondToAddress)).exceptionally(wto -> {
+ failed();
+ return null;
+ });
}
catch (WriteTimeoutException wto)
{
@@ -76,26 +66,21 @@
}
}
- /**
- * Older version (< 1.0) will not send this message at all, hence we don't
- * need to check the version of the data.
- */
- private static void forwardToLocalNodes(Mutation mutation, MessagingService.Verb verb, byte[] forwardBytes, InetAddress from) throws IOException
+ private static void forwardToLocalNodes(Message<Mutation> originalMessage, ForwardingInfo forwardTo)
{
- try (DataInputStream in = new DataInputStream(new FastByteArrayInputStream(forwardBytes)))
- {
- int size = in.readInt();
+ Message.Builder<Mutation> builder =
+ Message.builder(originalMessage)
+ .withParam(ParamType.RESPOND_TO, originalMessage.from())
+ .withoutParam(ParamType.FORWARD_TO);
- // tell the recipients who to send their ack to
- MessageOut<Mutation> message = new MessageOut<>(verb, mutation, Mutation.serializer).withParameter(Mutation.FORWARD_FROM, from.getAddress());
- // Send a message to each of the addresses on our Forward List
- for (int i = 0; i < size; i++)
- {
- InetAddress address = CompactEndpointSerializationHelper.deserialize(in);
- int id = in.readInt();
- Tracing.trace("Enqueuing forwarded write to {}", address);
- MessagingService.instance().sendOneWay(message, id, address);
- }
- }
+ boolean useSameMessageID = forwardTo.useSameMessageID();
+ // reuse the same Message if all ids are identical (as they will be for 4.0+ node originated messages)
+ Message<Mutation> message = useSameMessageID ? builder.build() : null;
+
+ forwardTo.forEach((id, target) ->
+ {
+ Tracing.trace("Enqueuing forwarded write to {}", target);
+ MessagingService.instance().send(useSameMessageID ? message : builder.withId(id).build(), target);
+ });
}
}
diff --git a/src/java/org/apache/cassandra/db/NativeClustering.java b/src/java/org/apache/cassandra/db/NativeClustering.java
index e96435b..780f6a4 100644
--- a/src/java/org/apache/cassandra/db/NativeClustering.java
+++ b/src/java/org/apache/cassandra/db/NativeClustering.java
@@ -34,6 +34,11 @@
private NativeClustering() { peer = 0; }
+ public ClusteringPrefix minimize()
+ {
+ return this;
+ }
+
public NativeClustering(NativeAllocator allocator, OpOrder.Group writeOp, Clustering clustering)
{
int count = clustering.size();
@@ -122,9 +127,4 @@
{
return EMPTY_SIZE;
}
-
- public ClusteringPrefix minimize()
- {
- return this;
- }
}
diff --git a/src/java/org/apache/cassandra/db/PartitionColumns.java b/src/java/org/apache/cassandra/db/PartitionColumns.java
deleted file mode 100644
index bf4ac43..0000000
--- a/src/java/org/apache/cassandra/db/PartitionColumns.java
+++ /dev/null
@@ -1,199 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.db;
-
-import java.util.*;
-
-import com.google.common.collect.Iterators;
-
-import org.apache.cassandra.config.ColumnDefinition;
-import org.apache.cassandra.utils.btree.BTreeSet;
-
-import static java.util.Comparator.naturalOrder;
-
-/**
- * Columns (or a subset of the columns) that a partition contains.
- * This mainly groups both static and regular columns for convenience.
- */
-public class PartitionColumns implements Iterable<ColumnDefinition>
-{
- public static PartitionColumns NONE = new PartitionColumns(Columns.NONE, Columns.NONE);
-
- public final Columns statics;
- public final Columns regulars;
-
- public PartitionColumns(Columns statics, Columns regulars)
- {
- assert statics != null && regulars != null;
- this.statics = statics;
- this.regulars = regulars;
- }
-
- public static PartitionColumns of(ColumnDefinition column)
- {
- return new PartitionColumns(column.isStatic() ? Columns.of(column) : Columns.NONE,
- column.isStatic() ? Columns.NONE : Columns.of(column));
- }
-
- public PartitionColumns without(ColumnDefinition column)
- {
- return new PartitionColumns(column.isStatic() ? statics.without(column) : statics,
- column.isStatic() ? regulars : regulars.without(column));
- }
-
- public PartitionColumns withoutStatics()
- {
- return statics.isEmpty() ? this : new PartitionColumns(Columns.NONE, regulars);
- }
-
- public PartitionColumns mergeTo(PartitionColumns that)
- {
- if (this == that)
- return this;
- Columns statics = this.statics.mergeTo(that.statics);
- Columns regulars = this.regulars.mergeTo(that.regulars);
- if (statics == this.statics && regulars == this.regulars)
- return this;
- if (statics == that.statics && regulars == that.regulars)
- return that;
- return new PartitionColumns(statics, regulars);
- }
-
- public boolean isEmpty()
- {
- return statics.isEmpty() && regulars.isEmpty();
- }
-
- public Columns columns(boolean isStatic)
- {
- return isStatic ? statics : regulars;
- }
-
- public boolean contains(ColumnDefinition column)
- {
- return column.isStatic() ? statics.contains(column) : regulars.contains(column);
- }
-
- public boolean includes(PartitionColumns columns)
- {
- return statics.containsAll(columns.statics) && regulars.containsAll(columns.regulars);
- }
-
- public Iterator<ColumnDefinition> iterator()
- {
- return Iterators.concat(statics.iterator(), regulars.iterator());
- }
-
- public Iterator<ColumnDefinition> selectOrderIterator()
- {
- return Iterators.concat(statics.selectOrderIterator(), regulars.selectOrderIterator());
- }
-
- /** * Returns the total number of static and regular columns. */
- public int size()
- {
- return regulars.size() + statics.size();
- }
-
- @Override
- public String toString()
- {
- StringBuilder sb = new StringBuilder();
- sb.append("[").append(statics).append(" | ").append(regulars).append("]");
- return sb.toString();
- }
-
- @Override
- public boolean equals(Object other)
- {
- if (!(other instanceof PartitionColumns))
- return false;
-
- PartitionColumns that = (PartitionColumns)other;
- return this.statics.equals(that.statics)
- && this.regulars.equals(that.regulars);
- }
-
- @Override
- public int hashCode()
- {
- return Objects.hash(statics, regulars);
- }
-
- public static Builder builder()
- {
- return new Builder();
- }
-
- public static class Builder
- {
- // Note that we do want to use sorted sets because we want the column definitions to be compared
- // through compareTo, not equals. The former basically check it's the same column name, while the latter
- // check it's the same object, including the same type.
- private BTreeSet.Builder<ColumnDefinition> regularColumns;
- private BTreeSet.Builder<ColumnDefinition> staticColumns;
-
- public Builder add(ColumnDefinition c)
- {
- if (c.isStatic())
- {
- if (staticColumns == null)
- staticColumns = BTreeSet.builder(naturalOrder());
- staticColumns.add(c);
- }
- else
- {
- assert c.isRegular();
- if (regularColumns == null)
- regularColumns = BTreeSet.builder(naturalOrder());
- regularColumns.add(c);
- }
- return this;
- }
-
- public Builder addAll(Iterable<ColumnDefinition> columns)
- {
- for (ColumnDefinition c : columns)
- add(c);
- return this;
- }
-
- public Builder addAll(PartitionColumns columns)
- {
- if (regularColumns == null && !columns.regulars.isEmpty())
- regularColumns = BTreeSet.builder(naturalOrder());
-
- for (ColumnDefinition c : columns.regulars)
- regularColumns.add(c);
-
- if (staticColumns == null && !columns.statics.isEmpty())
- staticColumns = BTreeSet.builder(naturalOrder());
-
- for (ColumnDefinition c : columns.statics)
- staticColumns.add(c);
-
- return this;
- }
-
- public PartitionColumns build()
- {
- return new PartitionColumns(staticColumns == null ? Columns.NONE : Columns.from(staticColumns.build()),
- regularColumns == null ? Columns.NONE : Columns.from(regularColumns.build()));
- }
- }
-}
diff --git a/src/java/org/apache/cassandra/db/PartitionRangeReadCommand.java b/src/java/org/apache/cassandra/db/PartitionRangeReadCommand.java
index 30c3034..82b6e8a 100644
--- a/src/java/org/apache/cassandra/db/PartitionRangeReadCommand.java
+++ b/src/java/org/apache/cassandra/db/PartitionRangeReadCommand.java
@@ -18,13 +18,13 @@
package org.apache.cassandra.db;
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
+import java.util.concurrent.TimeUnit;
import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.Iterables;
-import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.net.MessageFlag;
+import org.apache.cassandra.net.Verb;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.filter.*;
import org.apache.cassandra.db.lifecycle.View;
@@ -41,44 +41,37 @@
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.metrics.TableMetrics;
-import org.apache.cassandra.net.MessageOut;
-import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.net.Message;
import org.apache.cassandra.schema.IndexMetadata;
import org.apache.cassandra.service.ClientState;
import org.apache.cassandra.service.StorageProxy;
-import org.apache.cassandra.service.pager.*;
-import org.apache.cassandra.thrift.ThriftResultsMerger;
import org.apache.cassandra.tracing.Tracing;
-import org.apache.cassandra.transport.ProtocolVersion;
-import org.apache.cassandra.utils.FBUtilities;
/**
* A read command that selects a (part of a) range of partitions.
*/
-public class PartitionRangeReadCommand extends ReadCommand
+public class PartitionRangeReadCommand extends ReadCommand implements PartitionRangeReadQuery
{
protected static final SelectionDeserializer selectionDeserializer = new Deserializer();
private final DataRange dataRange;
- private int oldestUnrepairedTombstone = Integer.MAX_VALUE;
private PartitionRangeReadCommand(boolean isDigest,
- int digestVersion,
- boolean isForThrift,
- CFMetaData metadata,
- int nowInSec,
- ColumnFilter columnFilter,
- RowFilter rowFilter,
- DataLimits limits,
- DataRange dataRange,
- IndexMetadata index)
+ int digestVersion,
+ boolean acceptsTransient,
+ TableMetadata metadata,
+ int nowInSec,
+ ColumnFilter columnFilter,
+ RowFilter rowFilter,
+ DataLimits limits,
+ DataRange dataRange,
+ IndexMetadata index)
{
- super(Kind.PARTITION_RANGE, isDigest, digestVersion, isForThrift, metadata, nowInSec, columnFilter, rowFilter, limits, index);
+ super(Kind.PARTITION_RANGE, isDigest, digestVersion, acceptsTransient, metadata, nowInSec, columnFilter, rowFilter, limits, index);
this.dataRange = dataRange;
}
- public static PartitionRangeReadCommand create(boolean isForThrift,
- CFMetaData metadata,
+ public static PartitionRangeReadCommand create(TableMetadata metadata,
int nowInSec,
ColumnFilter columnFilter,
RowFilter rowFilter,
@@ -87,7 +80,7 @@
{
return new PartitionRangeReadCommand(false,
0,
- isForThrift,
+ false,
metadata,
nowInSec,
columnFilter,
@@ -105,9 +98,11 @@
*
* @return a newly created read command that queries everything in the table.
*/
- public static PartitionRangeReadCommand allDataRead(CFMetaData metadata, int nowInSec)
+ public static PartitionRangeReadCommand allDataRead(TableMetadata metadata, int nowInSec)
{
- return new PartitionRangeReadCommand(false, 0, false,
+ return new PartitionRangeReadCommand(false,
+ 0,
+ false,
metadata,
nowInSec,
ColumnFilter.all(metadata),
@@ -156,7 +151,7 @@
// on the ring.
return new PartitionRangeReadCommand(isDigestQuery(),
digestVersion(),
- isForThrift(),
+ acceptsTransient(),
metadata(),
nowInSec(),
columnFilter(),
@@ -170,7 +165,7 @@
{
return new PartitionRangeReadCommand(isDigestQuery(),
digestVersion(),
- isForThrift(),
+ acceptsTransient(),
metadata(),
nowInSec(),
columnFilter(),
@@ -180,11 +175,12 @@
indexMetadata());
}
- public PartitionRangeReadCommand copyAsDigestQuery()
+ @Override
+ protected PartitionRangeReadCommand copyAsDigestQuery()
{
return new PartitionRangeReadCommand(true,
digestVersion(),
- isForThrift(),
+ false,
metadata(),
nowInSec(),
columnFilter(),
@@ -194,11 +190,27 @@
indexMetadata());
}
- public ReadCommand withUpdatedLimit(DataLimits newLimits)
+ @Override
+ protected PartitionRangeReadCommand copyAsTransientQuery()
+ {
+ return new PartitionRangeReadCommand(false,
+ 0,
+ true,
+ metadata(),
+ nowInSec(),
+ columnFilter(),
+ rowFilter(),
+ limits(),
+ dataRange(),
+ indexMetadata());
+ }
+
+ @Override
+ public PartitionRangeReadCommand withUpdatedLimit(DataLimits newLimits)
{
return new PartitionRangeReadCommand(isDigestQuery(),
digestVersion(),
- isForThrift(),
+ acceptsTransient(),
metadata(),
nowInSec(),
columnFilter(),
@@ -208,25 +220,12 @@
indexMetadata());
}
- public PartitionRangeReadCommand withUpdatedDataRange(DataRange newDataRange)
- {
- return new PartitionRangeReadCommand(isDigestQuery(),
- digestVersion(),
- isForThrift(),
- metadata(),
- nowInSec(),
- columnFilter(),
- rowFilter(),
- limits(),
- newDataRange,
- indexMetadata());
- }
-
+ @Override
public PartitionRangeReadCommand withUpdatedLimitsAndDataRange(DataLimits newLimits, DataRange newDataRange)
{
return new PartitionRangeReadCommand(isDigestQuery(),
digestVersion(),
- isForThrift(),
+ acceptsTransient(),
metadata(),
nowInSec(),
columnFilter(),
@@ -236,9 +235,9 @@
indexMetadata());
}
- public long getTimeout()
+ public long getTimeout(TimeUnit unit)
{
- return DatabaseDescriptor.getRangeRpcTimeout();
+ return DatabaseDescriptor.getRangeRpcTimeout(unit);
}
public boolean isReversed()
@@ -246,34 +245,11 @@
return dataRange.isReversed();
}
- public boolean selectsKey(DecoratedKey key)
- {
- if (!dataRange().contains(key))
- return false;
-
- return rowFilter().partitionKeyRestrictionsAreSatisfiedBy(key, metadata().getKeyValidator());
- }
-
- public boolean selectsClustering(DecoratedKey key, Clustering clustering)
- {
- if (clustering == Clustering.STATIC_CLUSTERING)
- return !columnFilter().fetchedColumns().statics.isEmpty();
-
- if (!dataRange().clusteringIndexFilter(key).selects(clustering))
- return false;
- return rowFilter().clusteringKeyRestrictionsAreSatisfiedBy(clustering);
- }
-
public PartitionIterator execute(ConsistencyLevel consistency, ClientState clientState, long queryStartNanoTime) throws RequestExecutionException
{
return StorageProxy.getRangeSlice(this, consistency, queryStartNanoTime);
}
- public QueryPager getPager(PagingState pagingState, ProtocolVersion protocolVersion)
- {
- return new PartitionRangeQueryPager(this, pagingState, protocolVersion);
- }
-
protected void recordLatency(TableMetrics metric, long latencyNanos)
{
metric.rangeLatency.addNano(latencyNanos);
@@ -283,54 +259,46 @@
public UnfilteredPartitionIterator queryStorage(final ColumnFamilyStore cfs, ReadExecutionController executionController)
{
ColumnFamilyStore.ViewFragment view = cfs.select(View.selectLive(dataRange().keyRange()));
- Tracing.trace("Executing seq scan across {} sstables for {}", view.sstables.size(), dataRange().keyRange().getString(metadata().getKeyValidator()));
+ Tracing.trace("Executing seq scan across {} sstables for {}", view.sstables.size(), dataRange().keyRange().getString(metadata().partitionKeyType));
// fetch data from current memtable, historical memtables, and SSTables in the correct order.
- final List<UnfilteredPartitionIterator> iterators = new ArrayList<>(Iterables.size(view.memtables) + view.sstables.size());
-
+ InputCollector<UnfilteredPartitionIterator> inputCollector = iteratorsForRange(view);
try
{
for (Memtable memtable : view.memtables)
{
@SuppressWarnings("resource") // We close on exception and on closing the result returned by this method
- Memtable.MemtableUnfilteredPartitionIterator iter = memtable.makePartitionIterator(columnFilter(), dataRange(), isForThrift());
-
- @SuppressWarnings("resource") // We close on exception and on closing the result returned by this method
- UnfilteredPartitionIterator iterator = isForThrift() ? ThriftResultsMerger.maybeWrap(iter, metadata(), nowInSec()) : iter;
- iterators.add(RTBoundValidator.validate(iterator, RTBoundValidator.Stage.MEMTABLE, false));
-
+ Memtable.MemtableUnfilteredPartitionIterator iter = memtable.makePartitionIterator(columnFilter(), dataRange());
oldestUnrepairedTombstone = Math.min(oldestUnrepairedTombstone, iter.getMinLocalDeletionTime());
+ inputCollector.addMemtableIterator(RTBoundValidator.validate(iter, RTBoundValidator.Stage.MEMTABLE, false));
}
SSTableReadsListener readCountUpdater = newReadCountUpdater();
for (SSTableReader sstable : view.sstables)
{
@SuppressWarnings("resource") // We close on exception and on closing the result returned by this method
- UnfilteredPartitionIterator iter = sstable.getScanner(columnFilter(), dataRange(), isForThrift(), readCountUpdater);
-
- if (isForThrift())
- iter = ThriftResultsMerger.maybeWrap(iter, metadata(), nowInSec());
-
- iterators.add(RTBoundValidator.validate(iter, RTBoundValidator.Stage.SSTABLE, false));
+ UnfilteredPartitionIterator iter = sstable.getScanner(columnFilter(), dataRange(), readCountUpdater);
+ inputCollector.addSSTableIterator(sstable, RTBoundValidator.validate(iter, RTBoundValidator.Stage.SSTABLE, false));
if (!sstable.isRepaired())
oldestUnrepairedTombstone = Math.min(oldestUnrepairedTombstone, sstable.getMinLocalDeletionTime());
}
// iterators can be empty for offline tools
- return iterators.isEmpty() ? EmptyIterators.unfilteredPartition(metadata(), isForThrift())
- : checkCacheFilter(UnfilteredPartitionIterators.mergeLazily(iterators, nowInSec()), cfs);
+ if (inputCollector.isEmpty())
+ return EmptyIterators.unfilteredPartition(metadata());
+
+ return checkCacheFilter(UnfilteredPartitionIterators.mergeLazily(inputCollector.finalizeIterators(cfs, nowInSec(), oldestUnrepairedTombstone)), cfs);
}
catch (RuntimeException | Error e)
{
try
{
- FBUtilities.closeAll(iterators);
+ inputCollector.close();
}
- catch (Exception suppressed)
+ catch (Exception e1)
{
- e.addSuppressed(suppressed);
+ e.addSuppressed(e1);
}
-
throw e;
}
}
@@ -351,12 +319,6 @@
};
}
- @Override
- protected int oldestUnrepairedTombstone()
- {
- return oldestUnrepairedTombstone;
- }
-
private UnfilteredPartitionIterator checkCacheFilter(UnfilteredPartitionIterator iter, final ColumnFamilyStore cfs)
{
class CacheFilter extends Transformation
@@ -372,7 +334,11 @@
CachedPartition cached = cfs.getRawCachedPartition(dk);
ClusteringIndexFilter filter = dataRange().clusteringIndexFilter(dk);
- if (cached != null && cfs.isFilterFullyCoveredBy(filter, limits(), cached, nowInSec()))
+ if (cached != null && cfs.isFilterFullyCoveredBy(filter,
+ limits(),
+ cached,
+ nowInSec(),
+ iter.metadata().enforceStrictLiveness()))
{
// We won't use 'iter' so close it now.
iter.close();
@@ -386,11 +352,10 @@
return Transformation.apply(iter, new CacheFilter());
}
- public MessageOut<ReadCommand> createMessage(int version)
+ @Override
+ public Verb verb()
{
- return dataRange().isPaging()
- ? new MessageOut<>(MessagingService.Verb.PAGED_RANGE, this, pagedRangeSerializer)
- : new MessageOut<>(MessagingService.Verb.RANGE_SLICE, this, rangeSliceSerializer);
+ return Verb.RANGE_REQ;
}
protected void appendCQLWhereClause(StringBuilder sb)
@@ -418,24 +383,16 @@
*/
public PartitionIterator postReconciliationProcessing(PartitionIterator result)
{
- ColumnFamilyStore cfs = Keyspace.open(metadata().ksName).getColumnFamilyStore(metadata().cfName);
+ ColumnFamilyStore cfs = Keyspace.open(metadata().keyspace).getColumnFamilyStore(metadata().name);
Index index = getIndex(cfs);
return index == null ? result : index.postProcessorFor(this).apply(result, this);
}
@Override
- public boolean selectsFullPartition()
- {
- return metadata().isStaticCompactTable() ||
- (dataRange.selectsAllPartition() && !rowFilter().hasExpressionOnClusteringOrRegularColumns());
- }
-
- @Override
public String toString()
{
- return String.format("Read(%s.%s columns=%s rowfilter=%s limits=%s %s)",
- metadata().ksName,
- metadata().cfName,
+ return String.format("Read(%s columns=%s rowfilter=%s limits=%s %s)",
+ metadata().toString(),
columnFilter(),
rowFilter(),
limits(),
@@ -465,14 +422,19 @@
&& dataRange.startKey().equals(dataRange.stopKey());
}
+ public boolean isRangeRequest()
+ {
+ return true;
+ }
+
private static class Deserializer extends SelectionDeserializer
{
public ReadCommand deserialize(DataInputPlus in,
int version,
boolean isDigest,
int digestVersion,
- boolean isForThrift,
- CFMetaData metadata,
+ boolean acceptsTransient,
+ TableMetadata metadata,
int nowInSec,
ColumnFilter columnFilter,
RowFilter rowFilter,
@@ -481,7 +443,7 @@
throws IOException
{
DataRange range = DataRange.serializer.deserialize(in, version, metadata);
- return new PartitionRangeReadCommand(isDigest, digestVersion, isForThrift, metadata, nowInSec, columnFilter, rowFilter, limits, range, index);
+ return new PartitionRangeReadCommand(isDigest, digestVersion, acceptsTransient, metadata, nowInSec, columnFilter, rowFilter, limits, range, index);
}
}
}
diff --git a/src/java/org/apache/cassandra/db/PartitionRangeReadQuery.java b/src/java/org/apache/cassandra/db/PartitionRangeReadQuery.java
new file mode 100644
index 0000000..560fbe9
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/PartitionRangeReadQuery.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db;
+
+import org.apache.cassandra.db.filter.ColumnFilter;
+import org.apache.cassandra.db.filter.DataLimits;
+import org.apache.cassandra.db.filter.RowFilter;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.service.pager.PagingState;
+import org.apache.cassandra.service.pager.PartitionRangeQueryPager;
+import org.apache.cassandra.service.pager.QueryPager;
+import org.apache.cassandra.transport.ProtocolVersion;
+
+/**
+ * A {@code ReadQuery} for a range of partitions.
+ */
+public interface PartitionRangeReadQuery extends ReadQuery
+{
+ static ReadQuery create(TableMetadata table,
+ int nowInSec,
+ ColumnFilter columnFilter,
+ RowFilter rowFilter,
+ DataLimits limits,
+ DataRange dataRange)
+ {
+ if (table.isVirtual())
+ return VirtualTablePartitionRangeReadQuery.create(table, nowInSec, columnFilter, rowFilter, limits, dataRange);
+
+ return PartitionRangeReadCommand.create(table, nowInSec, columnFilter, rowFilter, limits, dataRange);
+ }
+
+ DataRange dataRange();
+
+ /**
+ * Creates a new {@code PartitionRangeReadQuery} with the updated limits.
+ *
+ * @param newLimits the new limits
+ * @return the new {@code PartitionRangeReadQuery}
+ */
+ PartitionRangeReadQuery withUpdatedLimit(DataLimits newLimits);
+
+ /**
+ * Creates a new {@code PartitionRangeReadQuery} with the updated limits and data range.
+ *
+ * @param newLimits the new limits
+ * @return the new {@code PartitionRangeReadQuery}
+ */
+ PartitionRangeReadQuery withUpdatedLimitsAndDataRange(DataLimits newLimits, DataRange newDataRange);
+
+ default QueryPager getPager(PagingState pagingState, ProtocolVersion protocolVersion)
+ {
+ return new PartitionRangeQueryPager(this, pagingState, protocolVersion);
+ }
+
+ default boolean selectsKey(DecoratedKey key)
+ {
+ if (!dataRange().contains(key))
+ return false;
+
+ return rowFilter().partitionKeyRestrictionsAreSatisfiedBy(key, metadata().partitionKeyType);
+ }
+
+ default boolean selectsClustering(DecoratedKey key, Clustering clustering)
+ {
+ if (clustering == Clustering.STATIC_CLUSTERING)
+ return !columnFilter().fetchedColumns().statics.isEmpty();
+
+ if (!dataRange().clusteringIndexFilter(key).selects(clustering))
+ return false;
+ return rowFilter().clusteringKeyRestrictionsAreSatisfiedBy(clustering);
+ }
+
+ default boolean selectsFullPartition()
+ {
+ return metadata().isStaticCompactTable() ||
+ (dataRange().selectsAllPartition() && !rowFilter().hasExpressionOnClusteringOrRegularColumns());
+ }
+}
diff --git a/src/java/org/apache/cassandra/db/RangeSliceVerbHandler.java b/src/java/org/apache/cassandra/db/RangeSliceVerbHandler.java
deleted file mode 100644
index 55826f5..0000000
--- a/src/java/org/apache/cassandra/db/RangeSliceVerbHandler.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.db;
-
-import org.apache.cassandra.io.IVersionedSerializer;
-
-public class RangeSliceVerbHandler extends ReadCommandVerbHandler
-{
- @Override
- protected IVersionedSerializer<ReadResponse> serializer()
- {
- return ReadResponse.rangeSliceSerializer;
- }
-}
diff --git a/src/java/org/apache/cassandra/db/RangeTombstoneList.java b/src/java/org/apache/cassandra/db/RangeTombstoneList.java
index 1aa20c1..acc5f17 100644
--- a/src/java/org/apache/cassandra/db/RangeTombstoneList.java
+++ b/src/java/org/apache/cassandra/db/RangeTombstoneList.java
@@ -22,6 +22,7 @@
import java.util.Collections;
import java.util.Iterator;
+import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.utils.AbstractIterator;
import com.google.common.collect.Iterators;
@@ -344,6 +345,7 @@
return iterator(false);
}
+ @SuppressWarnings("resource")
public Iterator<RangeTombstone> iterator(boolean reversed)
{
return reversed
@@ -670,7 +672,12 @@
*/
private void growToFree(int i)
{
- int newLength = (capacity() * 3) / 2 + 1;
+ // Introduce getRangeTombstoneResizeFactor
+ int newLength = (int) Math.ceil(capacity() * DatabaseDescriptor.getRangeTombstoneListGrowthFactor());
+ // Fallback to the original calculation if the newLength calculated from the resize factor is not valid.
+ if (newLength <= capacity())
+ newLength = ((capacity() * 3) / 2) + 1;
+
grow(i, newLength);
}
diff --git a/src/java/org/apache/cassandra/db/ReadCommand.java b/src/java/org/apache/cassandra/db/ReadCommand.java
index d4a7737..ffdfc7c 100644
--- a/src/java/org/apache/cassandra/db/ReadCommand.java
+++ b/src/java/org/apache/cassandra/db/ReadCommand.java
@@ -20,19 +20,23 @@
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.*;
-import java.util.function.Predicate;
+import java.util.concurrent.TimeUnit;
+import java.util.function.BiFunction;
+import java.util.function.LongPredicate;
+import java.util.function.Function;
import javax.annotation.Nullable;
-import com.google.common.collect.Lists;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Sets;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.config.*;
-import org.apache.cassandra.cql3.Operator;
import org.apache.cassandra.db.filter.*;
-import org.apache.cassandra.db.monitoring.ApproximateTime;
-import org.apache.cassandra.db.monitoring.MonitorableImpl;
+import org.apache.cassandra.net.MessageFlag;
+import org.apache.cassandra.net.Verb;
import org.apache.cassandra.db.partitions.*;
import org.apache.cassandra.db.rows.*;
import org.apache.cassandra.db.transform.RTBoundCloser;
@@ -40,23 +44,31 @@
import org.apache.cassandra.db.transform.RTBoundValidator.Stage;
import org.apache.cassandra.db.transform.StoppingTransformation;
import org.apache.cassandra.db.transform.Transformation;
-import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.exceptions.UnknownIndexException;
import org.apache.cassandra.index.Index;
import org.apache.cassandra.index.IndexNotAvailableException;
-import org.apache.cassandra.io.ForwardingVersionedSerializer;
+import org.apache.cassandra.index.IndexRegistry;
import org.apache.cassandra.io.IVersionedSerializer;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.locator.Replica;
import org.apache.cassandra.metrics.TableMetrics;
-import org.apache.cassandra.net.MessageOut;
-import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.net.Message;
import org.apache.cassandra.schema.IndexMetadata;
-import org.apache.cassandra.schema.UnknownIndexException;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.SchemaConstants;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.service.ActiveRepairService;
import org.apache.cassandra.service.ClientWarn;
import org.apache.cassandra.tracing.Tracing;
-import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FBUtilities;
-import org.apache.cassandra.utils.Pair;
+
+import static com.google.common.collect.Iterables.any;
+import static com.google.common.collect.Iterables.filter;
+import static org.apache.cassandra.utils.MonotonicClock.approxTime;
+import static org.apache.cassandra.db.partitions.UnfilteredPartitionIterators.MergeListener.NOOP;
/**
* General interface for storage-engine read commands (common to both range and
@@ -64,61 +76,25 @@
* <p>
* This contains all the informations needed to do a local read.
*/
-public abstract class ReadCommand extends MonitorableImpl implements ReadQuery
+public abstract class ReadCommand extends AbstractReadQuery
{
private static final int TEST_ITERATION_DELAY_MILLIS = Integer.parseInt(System.getProperty("cassandra.test.read_iteration_delay_ms", "0"));
protected static final Logger logger = LoggerFactory.getLogger(ReadCommand.class);
public static final IVersionedSerializer<ReadCommand> serializer = new Serializer();
- // For READ verb: will either dispatch on 'serializer' for 3.0 or 'legacyReadCommandSerializer' for earlier version.
- // Can be removed (and replaced by 'serializer') once we drop pre-3.0 backward compatibility.
- public static final IVersionedSerializer<ReadCommand> readSerializer = new ForwardingVersionedSerializer<ReadCommand>()
- {
- protected IVersionedSerializer<ReadCommand> delegate(int version)
- {
- return version < MessagingService.VERSION_30
- ? legacyReadCommandSerializer : serializer;
- }
- };
-
- // For RANGE_SLICE verb: will either dispatch on 'serializer' for 3.0 or 'legacyRangeSliceCommandSerializer' for earlier version.
- // Can be removed (and replaced by 'serializer') once we drop pre-3.0 backward compatibility.
- public static final IVersionedSerializer<ReadCommand> rangeSliceSerializer = new ForwardingVersionedSerializer<ReadCommand>()
- {
- protected IVersionedSerializer<ReadCommand> delegate(int version)
- {
- return version < MessagingService.VERSION_30
- ? legacyRangeSliceCommandSerializer : serializer;
- }
- };
-
- // For PAGED_RANGE verb: will either dispatch on 'serializer' for 3.0 or 'legacyPagedRangeCommandSerializer' for earlier version.
- // Can be removed (and replaced by 'serializer') once we drop pre-3.0 backward compatibility.
- public static final IVersionedSerializer<ReadCommand> pagedRangeSerializer = new ForwardingVersionedSerializer<ReadCommand>()
- {
- protected IVersionedSerializer<ReadCommand> delegate(int version)
- {
- return version < MessagingService.VERSION_30
- ? legacyPagedRangeCommandSerializer : serializer;
- }
- };
-
- public static final IVersionedSerializer<ReadCommand> legacyRangeSliceCommandSerializer = new LegacyRangeSliceCommandSerializer();
- public static final IVersionedSerializer<ReadCommand> legacyPagedRangeCommandSerializer = new LegacyPagedRangeCommandSerializer();
- public static final IVersionedSerializer<ReadCommand> legacyReadCommandSerializer = new LegacyReadCommandSerializer();
-
private final Kind kind;
- private final CFMetaData metadata;
- private final int nowInSec;
-
- private final ColumnFilter columnFilter;
- private final RowFilter rowFilter;
- private final DataLimits limits;
private final boolean isDigestQuery;
+ private final boolean acceptsTransient;
// if a digest query, the version for which the digest is expected. Ignored if not a digest.
private int digestVersion;
- private final boolean isForThrift;
+
+ // for data queries, coordinators may request information on the repaired data used in constructing the response
+ private boolean trackRepairedStatus = false;
+ // tracker for repaired data, initialized to singleton null object
+ private RepairedDataInfo repairedDataInfo = RepairedDataInfo.NULL_REPAIRED_DATA_INFO;
+
+ int oldestUnrepairedTombstone = Integer.MAX_VALUE;
@Nullable
private final IndexMetadata index;
@@ -129,8 +105,8 @@
int version,
boolean isDigest,
int digestVersion,
- boolean isForThrift,
- CFMetaData metadata,
+ boolean acceptsTransient,
+ TableMetadata metadata,
int nowInSec,
ColumnFilter columnFilter,
RowFilter rowFilter,
@@ -154,23 +130,22 @@
protected ReadCommand(Kind kind,
boolean isDigestQuery,
int digestVersion,
- boolean isForThrift,
- CFMetaData metadata,
+ boolean acceptsTransient,
+ TableMetadata metadata,
int nowInSec,
ColumnFilter columnFilter,
RowFilter rowFilter,
DataLimits limits,
IndexMetadata index)
{
+ super(metadata, nowInSec, columnFilter, rowFilter, limits);
+ if (acceptsTransient && isDigestQuery)
+ throw new IllegalArgumentException("Attempted to issue a digest response to transient replica");
+
this.kind = kind;
this.isDigestQuery = isDigestQuery;
this.digestVersion = digestVersion;
- this.isForThrift = isForThrift;
- this.metadata = metadata;
- this.nowInSec = nowInSec;
- this.columnFilter = columnFilter;
- this.rowFilter = rowFilter;
- this.limits = limits;
+ this.acceptsTransient = acceptsTransient;
this.index = index;
}
@@ -179,6 +154,8 @@
public abstract boolean isLimitedToOnePartition();
+ public abstract boolean isRangeRequest();
+
/**
* Creates a new <code>ReadCommand</code> instance with new limits.
*
@@ -188,72 +165,11 @@
public abstract ReadCommand withUpdatedLimit(DataLimits newLimits);
/**
- * The metadata for the table queried.
- *
- * @return the metadata for the table queried.
- */
- public CFMetaData metadata()
- {
- return metadata;
- }
-
- /**
- * The time in seconds to use as "now" for this query.
- * <p>
- * We use the same time as "now" for the whole query to avoid considering different
- * values as expired during the query, which would be buggy (would throw of counting amongst other
- * things).
- *
- * @return the time (in seconds) to use as "now".
- */
- public int nowInSec()
- {
- return nowInSec;
- }
-
- /**
* The configured timeout for this command.
*
* @return the configured timeout for this command.
*/
- public abstract long getTimeout();
-
- /**
- * A filter on which (non-PK) columns must be returned by the query.
- *
- * @return which columns must be fetched by this query.
- */
- public ColumnFilter columnFilter()
- {
- return columnFilter;
- }
-
- /**
- * Filters/Resrictions on CQL rows.
- * <p>
- * This contains the restrictions that are not directly handled by the
- * {@code ClusteringIndexFilter}. More specifically, this includes any non-PK column
- * restrictions and can include some PK columns restrictions when those can't be
- * satisfied entirely by the clustering index filter (because not all clustering columns
- * have been restricted for instance). If there is 2ndary indexes on the table,
- * one of this restriction might be handled by a 2ndary index.
- *
- * @return the filter holding the expression that rows must satisfy.
- */
- public RowFilter rowFilter()
- {
- return rowFilter;
- }
-
- /**
- * The limits set on this query.
- *
- * @return the limits set on this query.
- */
- public DataLimits limits()
- {
- return limits;
- }
+ public abstract long getTimeout(TimeUnit unit);
/**
* Whether this query is a digest one or not.
@@ -293,13 +209,73 @@
}
/**
- * Whether this query is for thrift or not.
- *
- * @return whether this query is for thrift.
+ * @return Whether this query expects only a transient data response, or a full response
*/
- public boolean isForThrift()
+ public boolean acceptsTransient()
{
- return isForThrift;
+ return acceptsTransient;
+ }
+
+ /**
+ * Activates repaired data tracking for this command.
+ *
+ * When active, a digest will be created from data read from repaired SSTables. The digests
+ * from each replica can then be compared on the coordinator to detect any divergence in their
+ * repaired datasets. In this context, an sstable is considered repaired if it is marked
+ * repaired or has a pending repair session which has been committed.
+ * In addition to the digest, a set of ids for any pending but as yet uncommitted repair sessions
+ * is recorded and returned to the coordinator. This is to help reduce false positives caused
+ * by compaction lagging which can leave sstables from committed sessions in the pending state
+ * for a time.
+ */
+ public void trackRepairedStatus()
+ {
+ trackRepairedStatus = true;
+ }
+
+ /**
+ * Whether or not repaired status of any data read is being tracked or not
+ *
+ * @return Whether repaired status tracking is active for this command
+ */
+ public boolean isTrackingRepairedStatus()
+ {
+ return trackRepairedStatus;
+ }
+
+ /**
+ * Returns a digest of the repaired data read in the execution of this command.
+ *
+ * If either repaired status tracking is not active or the command has not yet been
+ * executed, then this digest will be an empty buffer.
+ * Otherwise, it will contain a digest* of the repaired data read, or empty buffer
+ * if no repaired data was read.
+ * @return digest of the repaired data read in the execution of the command
+ */
+ public ByteBuffer getRepairedDataDigest()
+ {
+ return repairedDataInfo.getDigest();
+ }
+
+ /**
+ * Returns a boolean indicating whether any relevant sstables were skipped during the read
+ * that produced the repaired data digest.
+ *
+ * If true, then no pending repair sessions or partition deletes have influenced the extent
+ * of the repaired sstables that went into generating the digest.
+ * This indicates whether or not the digest can reliably be used to infer consistency
+ * issues between the repaired sets across replicas.
+ *
+ * If either repaired status tracking is not active or the command has not yet been
+ * executed, then this will always return true.
+ *
+ * @return boolean to indicate confidence in the dwhether or not the digest of the repaired data can be
+ * reliably be used to infer inconsistency issues between the repaired sets across
+ * replicas.
+ */
+ public boolean isRepairedDataDigestConclusive()
+ {
+ return repairedDataInfo.isConclusive();
}
/**
@@ -334,13 +310,56 @@
public abstract ReadCommand copy();
/**
+ * Returns a copy of this command with acceptsTransient set to true.
+ */
+ public ReadCommand copyAsTransientQuery(Replica replica)
+ {
+ Preconditions.checkArgument(replica.isTransient(),
+ "Can't make a transient request on a full replica: " + replica);
+ return copyAsTransientQuery();
+ }
+
+ /**
+ * Returns a copy of this command with acceptsTransient set to true.
+ */
+ public ReadCommand copyAsTransientQuery(Iterable<Replica> replicas)
+ {
+ if (any(replicas, Replica::isFull))
+ throw new IllegalArgumentException("Can't make a transient request on full replicas: " + Iterables.toString(filter(replicas, Replica::isFull)));
+ return copyAsTransientQuery();
+ }
+
+ protected abstract ReadCommand copyAsTransientQuery();
+
+ /**
* Returns a copy of this command with isDigestQuery set to true.
*/
- public abstract ReadCommand copyAsDigestQuery();
+ public ReadCommand copyAsDigestQuery(Replica replica)
+ {
+ Preconditions.checkArgument(replica.isFull(),
+ "Can't make a digest request on a transient replica " + replica);
+ return copyAsDigestQuery();
+ }
+
+ /**
+ * Returns a copy of this command with isDigestQuery set to true.
+ */
+ public ReadCommand copyAsDigestQuery(Iterable<Replica> replicas)
+ {
+ if (any(replicas, Replica::isTransient))
+ throw new IllegalArgumentException("Can't make a digest request on a transient replica " + Iterables.toString(filter(replicas, Replica::isTransient)));
+
+ return copyAsDigestQuery();
+ }
+
+ protected abstract ReadCommand copyAsDigestQuery();
protected abstract UnfilteredPartitionIterator queryStorage(ColumnFamilyStore cfs, ReadExecutionController executionController);
- protected abstract int oldestUnrepairedTombstone();
+ protected int oldestUnrepairedTombstone()
+ {
+ return oldestUnrepairedTombstone;
+ }
/**
* Whether the underlying {@code ClusteringIndexFilter} is reversed or not.
@@ -349,6 +368,7 @@
*/
public abstract boolean isReversed();
+ @SuppressWarnings("resource")
public ReadResponse createResponse(UnfilteredPartitionIterator iterator)
{
// validate that the sequence of RT markers is correct: open is followed by close, deletion times for both
@@ -374,9 +394,9 @@
: null;
}
- static IndexMetadata findIndex(CFMetaData table, RowFilter rowFilter)
+ static IndexMetadata findIndex(TableMetadata table, RowFilter rowFilter)
{
- if (table.getIndexes().isEmpty() || rowFilter.isEmpty())
+ if (table.indexes.isEmpty() || rowFilter.isEmpty())
return null;
ColumnFamilyStore cfs = Keyspace.openAndGetStore(table);
@@ -396,9 +416,8 @@
*/
public void maybeValidateIndex()
{
- Index index = getIndex(Keyspace.openAndGetStore(metadata));
if (null != index)
- index.validate(this);
+ IndexRegistry.obtain(metadata()).getIndex(index).validate(this);
}
/**
@@ -424,7 +443,16 @@
throw new IndexNotAvailableException(index);
searcher = index.searcherFor(this);
- Tracing.trace("Executing read on {}.{} using index {}", cfs.metadata.ksName, cfs.metadata.cfName, index.getIndexMetadata().name);
+ Tracing.trace("Executing read on {}.{} using index {}", cfs.metadata.keyspace, cfs.metadata.name, index.getIndexMetadata().name);
+ }
+
+ if (isTrackingRepairedStatus())
+ {
+ final DataLimits.Counter repairedReadCount = limits().newCounter(nowInSec(),
+ false,
+ selectsFullPartition(),
+ metadata().enforceStrictLiveness()).onlyCount();
+ repairedDataInfo = new RepairedDataInfo(repairedReadCount);
}
UnfilteredPartitionIterator iterator = (null == searcher) ? queryStorage(cfs, executionController) : searcher.search(executionController);
@@ -450,7 +478,22 @@
// apply the limits/row counter; this transformation is stopping and would close the iterator as soon
// as the count is observed; if that happens in the middle of an open RT, its end bound will not be included.
- iterator = limits().filter(iterator, nowInSec(), selectsFullPartition());
+ // If tracking repaired data, the counter is needed for overreading repaired data, otherwise we can
+ // optimise the case where this.limit = DataLimits.NONE which skips an unnecessary transform
+ if (isTrackingRepairedStatus())
+ {
+ DataLimits.Counter limit =
+ limits().newCounter(nowInSec(), false, selectsFullPartition(), metadata().enforceStrictLiveness());
+ iterator = limit.applyTo(iterator);
+ // ensure that a consistent amount of repaired data is read on each replica. This causes silent
+ // overreading from the repaired data set, up to limits(). The extra data is not visible to
+ // the caller, only iterated to produce the repaired data digest.
+ iterator = repairedDataInfo.extend(iterator, limit);
+ }
+ else
+ {
+ iterator = limits().filter(iterator, nowInSec(), selectsFullPartition());
+ }
// because of the above, we need to append an aritifical end bound if the source iterator was stopped short by a counter.
return RTBoundCloser.close(iterator);
@@ -464,11 +507,6 @@
protected abstract void recordLatency(TableMetrics metric, long latencyNanos);
- public PartitionIterator executeInternal(ReadExecutionController controller)
- {
- return UnfilteredPartitionIterators.filter(executeLocally(controller), nowInSec());
- }
-
public ReadExecutionController executionController()
{
return ReadExecutionController.forCommand(this);
@@ -485,8 +523,8 @@
private final int failureThreshold = DatabaseDescriptor.getTombstoneFailureThreshold();
private final int warningThreshold = DatabaseDescriptor.getTombstoneWarnThreshold();
- private final boolean respectTombstoneThresholds = !SchemaConstants.isLocalSystemKeyspace(ReadCommand.this.metadata().ksName);
- private final boolean enforceStrictLiveness = metadata.enforceStrictLiveness();
+ private final boolean respectTombstoneThresholds = !SchemaConstants.isLocalSystemKeyspace(ReadCommand.this.metadata().keyspace);
+ private final boolean enforceStrictLiveness = metadata().enforceStrictLiveness();
private int liveRows = 0;
private int tombstones = 0;
@@ -506,14 +544,6 @@
return applyToRow(row);
}
- /**
- * Count the number of live rows returned by the read command and the number of tombstones.
- *
- * Tombstones come in two forms on rows :
- * - cells that aren't live anymore (either expired through TTL or deleted) : 1 tombstone per cell
- * - Rows that aren't live and have no cell (DELETEs performed on the primary key) : 1 tombstone per row
- * We avoid counting rows as tombstones if they contain nothing but expired cells.
- */
@Override
public Row applyToRow(Row row)
{
@@ -554,6 +584,7 @@
{
String query = ReadCommand.this.toCQLString();
Tracing.trace("Scanned over {} tombstones for query {}; query aborted (see tombstone_failure_threshold)", failureThreshold, query);
+ metric.tombstoneFailures.inc();
throw new TombstoneOverwhelmingException(tombstones, query, ReadCommand.this.metadata(), currentKey, clustering);
}
}
@@ -573,6 +604,11 @@
"Read %d live rows and %d tombstone cells for query %1.512s; token %s (see tombstone_warn_threshold)",
liveRows, tombstones, ReadCommand.this.toCQLString(), currentKey.getToken());
ClientWarn.instance.warn(msg);
+ if (tombstones < failureThreshold)
+ {
+ metric.tombstoneWarnings.inc();
+ }
+
logger.warn(msg);
}
@@ -611,14 +647,15 @@
private boolean maybeAbort()
{
/**
- * The value returned by ApproximateTime.currentTimeMillis() is updated only every
- * {@link ApproximateTime.CHECK_INTERVAL_MS}, by default 10 millis. Since MonitorableImpl
- * relies on ApproximateTime, we don't need to check unless the approximate time has elapsed.
+ * TODO: this is not a great way to abort early; why not expressly limit checks to 10ms intervals?
+ * The value returned by approxTime.now() is updated only every
+ * {@link org.apache.cassandra.utils.MonotonicClock.SampledClock.CHECK_INTERVAL_MS}, by default 2 millis. Since MonitorableImpl
+ * relies on approxTime, we don't need to check unless the approximate time has elapsed.
*/
- if (lastChecked == ApproximateTime.currentTimeMillis())
+ if (lastChecked == approxTime.now())
return false;
- lastChecked = ApproximateTime.currentTimeMillis();
+ lastChecked = approxTime.now();
if (isAborted())
{
@@ -631,7 +668,7 @@
private void maybeDelayForTesting()
{
- if (!metadata.ksName.startsWith("system"))
+ if (!metadata().keyspace.startsWith("system"))
FBUtilities.sleepQuietly(TEST_ITERATION_DELAY_MILLIS);
}
}
@@ -644,7 +681,14 @@
/**
* Creates a message for this command.
*/
- public abstract MessageOut<ReadCommand> createMessage(int version);
+ public Message<ReadCommand> createMessage(boolean trackRepairedData)
+ {
+ return trackRepairedData
+ ? Message.outWithFlags(verb(), this, MessageFlag.CALL_BACK_ON_FAILURE, MessageFlag.TRACK_REPAIRED_DATA)
+ : Message.outWithFlag (verb(), this, MessageFlag.CALL_BACK_ON_FAILURE);
+ }
+
+ public abstract Verb verb();
protected abstract void appendCQLWhereClause(StringBuilder sb);
@@ -653,20 +697,16 @@
// are to some extend an artefact of compaction lagging behind and hence counting them is somewhat unintuitive).
protected UnfilteredPartitionIterator withoutPurgeableTombstones(UnfilteredPartitionIterator iterator, ColumnFamilyStore cfs)
{
- final boolean isForThrift = iterator.isForThrift();
class WithoutPurgeableTombstones extends PurgeFunction
{
public WithoutPurgeableTombstones()
{
- super(isForThrift,
- nowInSec(),
- cfs.gcBefore(nowInSec()),
- oldestUnrepairedTombstone(),
+ super(nowInSec(), cfs.gcBefore(nowInSec()), oldestUnrepairedTombstone(),
cfs.getCompactionStrategyManager().onlyPurgeRepairedTombstones(),
- cfs.metadata.enforceStrictLiveness());
+ iterator.metadata().enforceStrictLiveness());
}
- protected Predicate<Long> getPurgeEvaluator()
+ protected LongPredicate getPurgeEvaluator()
{
return time -> true;
}
@@ -687,7 +727,7 @@
{
StringBuilder sb = new StringBuilder();
sb.append("SELECT ").append(columnFilter());
- sb.append(" FROM ").append(metadata().ksName).append('.').append(metadata.cfName);
+ sb.append(" FROM ").append(metadata().keyspace).append('.').append(metadata().name);
appendCQLWhereClause(sb);
if (limits() != DataLimits.NONE)
@@ -701,6 +741,169 @@
return toCQLString();
}
+ @SuppressWarnings("resource") // resultant iterators are closed by their callers
+ InputCollector<UnfilteredRowIterator> iteratorsForPartition(ColumnFamilyStore.ViewFragment view)
+ {
+ final BiFunction<List<UnfilteredRowIterator>, RepairedDataInfo, UnfilteredRowIterator> merge =
+ (unfilteredRowIterators, repairedDataInfo) -> {
+ UnfilteredRowIterator repaired = UnfilteredRowIterators.merge(unfilteredRowIterators);
+ return repairedDataInfo.withRepairedDataInfo(repaired);
+ };
+
+ // For single partition reads, after reading up to the command's DataLimit nothing extra is required.
+ // The merged & repaired row iterator will be consumed until it's exhausted or the RepairedDataInfo's
+ // internal counter is satisfied
+ final Function<UnfilteredRowIterator, UnfilteredPartitionIterator> postLimitPartitions =
+ (rows) -> EmptyIterators.unfilteredPartition(metadata());
+ return new InputCollector<>(view, repairedDataInfo, merge, postLimitPartitions, isTrackingRepairedStatus());
+ }
+
+ @SuppressWarnings("resource") // resultant iterators are closed by their callers
+ InputCollector<UnfilteredPartitionIterator> iteratorsForRange(ColumnFamilyStore.ViewFragment view)
+ {
+ final BiFunction<List<UnfilteredPartitionIterator>, RepairedDataInfo, UnfilteredPartitionIterator> merge =
+ (unfilteredPartitionIterators, repairedDataInfo) -> {
+ UnfilteredPartitionIterator repaired = UnfilteredPartitionIterators.merge(unfilteredPartitionIterators,
+ NOOP);
+ return repairedDataInfo.withRepairedDataInfo(repaired);
+ };
+
+ // Uses identity function to provide additional partitions to be consumed after the command's
+ // DataLimits are satisfied. The input to the function will be the iterator of merged, repaired partitions
+ // which we'll keep reading until the RepairedDataInfo's internal counter is satisfied.
+ return new InputCollector<>(view, repairedDataInfo, merge, Function.identity(), isTrackingRepairedStatus());
+ }
+
+ /**
+ * Handles the collation of unfiltered row or partition iterators that comprise the
+ * input for a query. Separates them according to repaired status and of repaired
+ * status is being tracked, handles the merge and wrapping in a digest generator of
+ * the repaired iterators.
+ *
+ * Intentionally not AutoCloseable so we don't mistakenly use this in ARM blocks
+ * as this prematurely closes the underlying iterators
+ */
+ static class InputCollector<T extends AutoCloseable>
+ {
+ final RepairedDataInfo repairedDataInfo;
+ private final boolean isTrackingRepairedStatus;
+ Set<SSTableReader> repairedSSTables;
+ BiFunction<List<T>, RepairedDataInfo, T> repairedMerger;
+ Function<T, UnfilteredPartitionIterator> postLimitAdditionalPartitions;
+ List<T> repairedIters;
+ List<T> unrepairedIters;
+
+ InputCollector(ColumnFamilyStore.ViewFragment view,
+ RepairedDataInfo repairedDataInfo,
+ BiFunction<List<T>, RepairedDataInfo, T> repairedMerger,
+ Function<T, UnfilteredPartitionIterator> postLimitAdditionalPartitions,
+ boolean isTrackingRepairedStatus)
+ {
+ this.repairedDataInfo = repairedDataInfo;
+ this.isTrackingRepairedStatus = isTrackingRepairedStatus;
+ if (isTrackingRepairedStatus)
+ {
+ for (SSTableReader sstable : view.sstables)
+ {
+ if (considerRepairedForTracking(sstable))
+ {
+ if (repairedSSTables == null)
+ repairedSSTables = Sets.newHashSetWithExpectedSize(view.sstables.size());
+ repairedSSTables.add(sstable);
+ }
+ }
+ }
+ if (repairedSSTables == null)
+ {
+ repairedIters = Collections.emptyList();
+ unrepairedIters = new ArrayList<>(view.sstables.size());
+ }
+ else
+ {
+ repairedIters = new ArrayList<>(repairedSSTables.size());
+ // when we're done collating, we'll merge the repaired iters and add the
+ // result to the unrepaired list, so size that list accordingly
+ unrepairedIters = new ArrayList<>((view.sstables.size() - repairedSSTables.size()) + Iterables.size(view.memtables) + 1);
+ }
+ this.repairedMerger = repairedMerger;
+ this.postLimitAdditionalPartitions = postLimitAdditionalPartitions;
+ }
+
+ void addMemtableIterator(T iter)
+ {
+ unrepairedIters.add(iter);
+ }
+
+ void addSSTableIterator(SSTableReader sstable, T iter)
+ {
+ if (repairedSSTables != null && repairedSSTables.contains(sstable))
+ repairedIters.add(iter);
+ else
+ unrepairedIters.add(iter);
+ }
+
+ @SuppressWarnings("resource") // the returned iterators are closed by the caller
+ List<T> finalizeIterators(ColumnFamilyStore cfs, int nowInSec, int oldestUnrepairedTombstone)
+ {
+ if (repairedIters.isEmpty())
+ return unrepairedIters;
+
+ // merge the repaired data before returning, wrapping in a digest generator
+ repairedDataInfo.prepare(cfs, nowInSec, oldestUnrepairedTombstone);
+ T repairedIter = repairedMerger.apply(repairedIters, repairedDataInfo);
+ repairedDataInfo.finalize(postLimitAdditionalPartitions.apply(repairedIter));
+ unrepairedIters.add(repairedIter);
+ return unrepairedIters;
+ }
+
+ boolean isEmpty()
+ {
+ return repairedIters.isEmpty() && unrepairedIters.isEmpty();
+ }
+
+ // For tracking purposes we consider data repaired if the sstable is either:
+ // * marked repaired
+ // * marked pending, but the local session has been committed. This reduces the window
+ // whereby the tracking is affected by compaction backlog causing repaired sstables to
+ // remain in the pending state
+ // If an sstable is involved in a pending repair which is not yet committed, we mark the
+ // repaired data info inconclusive, as the same data on other replicas may be in a
+ // slightly different state.
+ private boolean considerRepairedForTracking(SSTableReader sstable)
+ {
+ if (!isTrackingRepairedStatus)
+ return false;
+
+ UUID pendingRepair = sstable.getPendingRepair();
+ if (pendingRepair != ActiveRepairService.NO_PENDING_REPAIR)
+ {
+ if (ActiveRepairService.instance.consistent.local.isSessionFinalized(pendingRepair))
+ return true;
+
+ // In the edge case where compaction is backed up long enough for the session to
+ // timeout and be purged by LocalSessions::cleanup, consider the sstable unrepaired
+ // as it will be marked unrepaired when compaction catches up
+ if (!ActiveRepairService.instance.consistent.local.sessionExists(pendingRepair))
+ return false;
+
+ repairedDataInfo.markInconclusive();
+ }
+
+ return sstable.isRepaired();
+ }
+
+ void markInconclusive()
+ {
+ repairedDataInfo.markInconclusive();
+ }
+
+ public void close() throws Exception
+ {
+ FBUtilities.closeAll(unrepairedIters);
+ FBUtilities.closeAll(repairedIters);
+ }
+ }
+
private static class Serializer implements IVersionedSerializer<ReadCommand>
{
private static int digestFlag(boolean isDigest)
@@ -713,11 +916,21 @@
return (flags & 0x01) != 0;
}
- private static int thriftFlag(boolean isForThrift)
+ private static boolean acceptsTransient(int flags)
{
- return isForThrift ? 0x02 : 0;
+ return (flags & 0x08) != 0;
}
+ private static int acceptsTransientFlag(boolean acceptsTransient)
+ {
+ return acceptsTransient ? 0x08 : 0;
+ }
+
+ // We don't set this flag anymore, but still look if we receive a
+ // command with it set in case someone is using thrift a mixed 3.0/4.0+
+ // cluster (which is unsupported). This is also a reminder for not
+ // re-using this flag until we drop 3.0/3.X compatibility (since it's
+ // used by these release for thrift and would thus confuse things)
private static boolean isForThrift(int flags)
{
return (flags & 0x02) != 0;
@@ -735,17 +948,19 @@
public void serialize(ReadCommand command, DataOutputPlus out, int version) throws IOException
{
- assert version >= MessagingService.VERSION_30;
-
out.writeByte(command.kind.ordinal());
- out.writeByte(digestFlag(command.isDigestQuery()) | thriftFlag(command.isForThrift()) | indexFlag(null != command.index));
+ out.writeByte(
+ digestFlag(command.isDigestQuery())
+ | indexFlag(null != command.indexMetadata())
+ | acceptsTransientFlag(command.acceptsTransient())
+ );
if (command.isDigestQuery())
out.writeUnsignedVInt(command.digestVersion());
- CFMetaData.serializer.serialize(command.metadata(), out, version);
+ command.metadata().id.serialize(out);
out.writeInt(command.nowInSec());
ColumnFilter.serializer.serialize(command.columnFilter(), out, version);
RowFilter.serializer.serialize(command.rowFilter(), out, version);
- DataLimits.serializer.serialize(command.limits(), out, version, command.metadata.comparator);
+ DataLimits.serializer.serialize(command.limits(), out, version, command.metadata().comparator);
if (null != command.index)
IndexMetadata.serializer.serialize(command.index, out, version);
@@ -754,29 +969,35 @@
public ReadCommand deserialize(DataInputPlus in, int version) throws IOException
{
- assert version >= MessagingService.VERSION_30;
-
Kind kind = Kind.values()[in.readByte()];
int flags = in.readByte();
boolean isDigest = isDigest(flags);
- boolean isForThrift = isForThrift(flags);
+ boolean acceptsTransient = acceptsTransient(flags);
+ // Shouldn't happen or it's a user error (see comment above) but
+ // better complain loudly than doing the wrong thing.
+ if (isForThrift(flags))
+ throw new IllegalStateException("Received a command with the thrift flag set. "
+ + "This means thrift is in use in a mixed 3.0/3.X and 4.0+ cluster, "
+ + "which is unsupported. Make sure to stop using thrift before "
+ + "upgrading to 4.0");
+
boolean hasIndex = hasIndex(flags);
int digestVersion = isDigest ? (int)in.readUnsignedVInt() : 0;
- CFMetaData metadata = CFMetaData.serializer.deserialize(in, version);
+ TableMetadata metadata = Schema.instance.getExistingTableMetadata(TableId.deserialize(in));
int nowInSec = in.readInt();
ColumnFilter columnFilter = ColumnFilter.serializer.deserialize(in, version, metadata);
RowFilter rowFilter = RowFilter.serializer.deserialize(in, version, metadata);
DataLimits limits = DataLimits.serializer.deserialize(in, version, metadata.comparator);
IndexMetadata index = hasIndex ? deserializeIndexMetadata(in, version, metadata) : null;
- return kind.selectionDeserializer.deserialize(in, version, isDigest, digestVersion, isForThrift, metadata, nowInSec, columnFilter, rowFilter, limits, index);
+ return kind.selectionDeserializer.deserialize(in, version, isDigest, digestVersion, acceptsTransient, metadata, nowInSec, columnFilter, rowFilter, limits, index);
}
- private IndexMetadata deserializeIndexMetadata(DataInputPlus in, int version, CFMetaData cfm) throws IOException
+ private IndexMetadata deserializeIndexMetadata(DataInputPlus in, int version, TableMetadata metadata) throws IOException
{
try
{
- return IndexMetadata.serializer.deserialize(in, version, cfm);
+ return IndexMetadata.serializer.deserialize(in, version, metadata);
}
catch (UnknownIndexException e)
{
@@ -784,1031 +1005,22 @@
"If an index was just created, this is likely due to the schema not " +
"being fully propagated. Local read will proceed without using the " +
"index. Please wait for schema agreement after index creation.",
- cfm.ksName, cfm.cfName, e.indexId);
+ metadata.keyspace, metadata.name, e.indexId);
return null;
}
}
public long serializedSize(ReadCommand command, int version)
{
- assert version >= MessagingService.VERSION_30;
-
return 2 // kind + flags
- + (command.isDigestQuery() ? TypeSizes.sizeofUnsignedVInt(command.digestVersion()) : 0)
- + CFMetaData.serializer.serializedSize(command.metadata(), version)
- + TypeSizes.sizeof(command.nowInSec())
- + ColumnFilter.serializer.serializedSize(command.columnFilter(), version)
- + RowFilter.serializer.serializedSize(command.rowFilter(), version)
- + DataLimits.serializer.serializedSize(command.limits(), version, command.metadata.comparator)
- + command.selectionSerializedSize(version)
- + command.indexSerializedSize(version);
- }
- }
-
- private enum LegacyType
- {
- GET_BY_NAMES((byte)1),
- GET_SLICES((byte)2);
-
- public final byte serializedValue;
-
- LegacyType(byte b)
- {
- this.serializedValue = b;
- }
-
- public static LegacyType fromPartitionFilterKind(ClusteringIndexFilter.Kind kind)
- {
- return kind == ClusteringIndexFilter.Kind.SLICE
- ? GET_SLICES
- : GET_BY_NAMES;
- }
-
- public static LegacyType fromSerializedValue(byte b)
- {
- return b == 1 ? GET_BY_NAMES : GET_SLICES;
- }
- }
-
- /**
- * Serializer for pre-3.0 RangeSliceCommands.
- */
- private static class LegacyRangeSliceCommandSerializer implements IVersionedSerializer<ReadCommand>
- {
- public void serialize(ReadCommand command, DataOutputPlus out, int version) throws IOException
- {
- assert version < MessagingService.VERSION_30;
-
- PartitionRangeReadCommand rangeCommand = (PartitionRangeReadCommand) command;
- assert !rangeCommand.dataRange().isPaging();
-
- // convert pre-3.0 incompatible names filters to slice filters
- rangeCommand = maybeConvertNamesToSlice(rangeCommand);
-
- CFMetaData metadata = rangeCommand.metadata();
-
- out.writeUTF(metadata.ksName);
- out.writeUTF(metadata.cfName);
- out.writeLong(rangeCommand.nowInSec() * 1000L); // convert from seconds to millis
-
- // begin DiskAtomFilterSerializer.serialize()
- if (rangeCommand.isNamesQuery())
- {
- out.writeByte(1); // 0 for slices, 1 for names
- ClusteringIndexNamesFilter filter = (ClusteringIndexNamesFilter) rangeCommand.dataRange().clusteringIndexFilter;
- LegacyReadCommandSerializer.serializeNamesFilter(rangeCommand, filter, out);
- }
- else
- {
- out.writeByte(0); // 0 for slices, 1 for names
-
- // slice filter serialization
- ClusteringIndexSliceFilter filter = (ClusteringIndexSliceFilter) rangeCommand.dataRange().clusteringIndexFilter;
-
- boolean makeStaticSlice = !rangeCommand.columnFilter().fetchedColumns().statics.isEmpty() && !filter.requestedSlices().selects(Clustering.STATIC_CLUSTERING);
- LegacyReadCommandSerializer.serializeSlices(out, filter.requestedSlices(), filter.isReversed(), makeStaticSlice, metadata);
-
- out.writeBoolean(filter.isReversed());
-
- // limit
- DataLimits limits = rangeCommand.limits();
- if (limits.isDistinct())
- out.writeInt(1);
- else
- out.writeInt(LegacyReadCommandSerializer.updateLimitForQuery(rangeCommand.limits().count(), filter.requestedSlices()));
-
- int compositesToGroup;
- boolean selectsStatics = !rangeCommand.columnFilter().fetchedColumns().statics.isEmpty() && filter.requestedSlices().selects(Clustering.STATIC_CLUSTERING);
- if (limits.kind() == DataLimits.Kind.THRIFT_LIMIT)
- compositesToGroup = -1;
- else if (limits.isDistinct() && !selectsStatics)
- compositesToGroup = -2; // for DISTINCT queries (CASSANDRA-8490)
- else
- compositesToGroup = metadata.isDense() ? -1 : metadata.clusteringColumns().size();
-
- out.writeInt(compositesToGroup);
- }
-
- serializeRowFilter(out, rangeCommand.rowFilter());
- AbstractBounds.rowPositionSerializer.serialize(rangeCommand.dataRange().keyRange(), out, version);
-
- // maxResults
- out.writeInt(rangeCommand.limits().count());
-
- // countCQL3Rows
- if (rangeCommand.isForThrift() || rangeCommand.limits().perPartitionCount() == 1) // if for Thrift or DISTINCT
- out.writeBoolean(false);
- else
- out.writeBoolean(true);
-
- // isPaging
- out.writeBoolean(false);
- }
-
- public ReadCommand deserialize(DataInputPlus in, int version) throws IOException
- {
- assert version < MessagingService.VERSION_30;
-
- String keyspace = in.readUTF();
- String columnFamily = in.readUTF();
-
- CFMetaData metadata = Schema.instance.getCFMetaData(keyspace, columnFamily);
- if (metadata == null)
- {
- String message = String.format("Got legacy range command for nonexistent table %s.%s.", keyspace, columnFamily);
- throw new UnknownColumnFamilyException(message, null);
- }
-
- int nowInSec = (int) (in.readLong() / 1000); // convert from millis to seconds
-
- ClusteringIndexFilter filter;
- ColumnFilter selection;
- int compositesToGroup = 0;
- int perPartitionLimit = -1;
- byte readType = in.readByte(); // 0 for slices, 1 for names
- if (readType == 1)
- {
- Pair<ColumnFilter, ClusteringIndexNamesFilter> selectionAndFilter = LegacyReadCommandSerializer.deserializeNamesSelectionAndFilter(in, metadata);
- selection = selectionAndFilter.left;
- filter = selectionAndFilter.right;
- }
- else
- {
- Pair<ClusteringIndexSliceFilter, Boolean> p = LegacyReadCommandSerializer.deserializeSlicePartitionFilter(in, metadata);
- filter = p.left;
- perPartitionLimit = in.readInt();
- compositesToGroup = in.readInt();
- selection = getColumnSelectionForSlice(p.right, compositesToGroup, metadata);
- }
-
- RowFilter rowFilter = deserializeRowFilter(in, metadata);
-
- AbstractBounds<PartitionPosition> keyRange = AbstractBounds.rowPositionSerializer.deserialize(in, metadata.partitioner, version);
- int maxResults = in.readInt();
-
- boolean countCQL3Rows = in.readBoolean(); // countCQL3Rows (not needed)
- in.readBoolean(); // isPaging (not needed)
-
- boolean selectsStatics = (!selection.fetchedColumns().statics.isEmpty() || filter.selects(Clustering.STATIC_CLUSTERING));
- // We have 2 types of DISTINCT queries: ones on only the partition key, and ones on the partition key and static columns. For the former,
- // we can easily detect the case because compositeToGroup is -2 and that's the only case it can be that. The latter one is slightly less
- // direct, but we know that on 2.1/2.2 queries, DISTINCT queries are the only CQL queries that have countCQL3Rows to false so we use
- // that fact.
- boolean isDistinct = compositesToGroup == -2 || (compositesToGroup != -1 && !countCQL3Rows);
- DataLimits limits;
- if (isDistinct)
- limits = DataLimits.distinctLimits(maxResults);
- else if (compositesToGroup == -1)
- limits = DataLimits.thriftLimits(maxResults, perPartitionLimit);
- else if (metadata.isStaticCompactTable())
- limits = DataLimits.legacyCompactStaticCqlLimits(maxResults);
- else
- limits = DataLimits.cqlLimits(maxResults);
-
- return PartitionRangeReadCommand.create(true, metadata, nowInSec, selection, rowFilter, limits, new DataRange(keyRange, filter));
- }
-
- static void serializeRowFilter(DataOutputPlus out, RowFilter rowFilter) throws IOException
- {
- ArrayList<RowFilter.Expression> indexExpressions = Lists.newArrayList(rowFilter.iterator());
- out.writeInt(indexExpressions.size());
- for (RowFilter.Expression expression : indexExpressions)
- {
- ByteBufferUtil.writeWithShortLength(expression.column().name.bytes, out);
- expression.operator().writeTo(out);
- ByteBufferUtil.writeWithShortLength(expression.getIndexValue(), out);
- }
- }
-
- static RowFilter deserializeRowFilter(DataInputPlus in, CFMetaData metadata) throws IOException
- {
- int numRowFilters = in.readInt();
- if (numRowFilters == 0)
- return RowFilter.NONE;
-
- RowFilter rowFilter = RowFilter.create(numRowFilters);
- for (int i = 0; i < numRowFilters; i++)
- {
- ByteBuffer columnName = ByteBufferUtil.readWithShortLength(in);
- ColumnDefinition column = metadata.getColumnDefinition(columnName);
- Operator op = Operator.readFrom(in);
- ByteBuffer indexValue = ByteBufferUtil.readWithShortLength(in);
- rowFilter.add(column, op, indexValue);
- }
- return rowFilter;
- }
-
- static long serializedRowFilterSize(RowFilter rowFilter)
- {
- long size = TypeSizes.sizeof(0); // rowFilterCount
- for (RowFilter.Expression expression : rowFilter)
- {
- size += ByteBufferUtil.serializedSizeWithShortLength(expression.column().name.bytes);
- size += TypeSizes.sizeof(0); // operator int value
- size += ByteBufferUtil.serializedSizeWithShortLength(expression.getIndexValue());
- }
- return size;
- }
-
- public long serializedSize(ReadCommand command, int version)
- {
- assert version < MessagingService.VERSION_30;
- assert command.kind == Kind.PARTITION_RANGE;
-
- PartitionRangeReadCommand rangeCommand = (PartitionRangeReadCommand) command;
- rangeCommand = maybeConvertNamesToSlice(rangeCommand);
- CFMetaData metadata = rangeCommand.metadata();
-
- long size = TypeSizes.sizeof(metadata.ksName);
- size += TypeSizes.sizeof(metadata.cfName);
- size += TypeSizes.sizeof((long) rangeCommand.nowInSec());
-
- size += 1; // single byte flag: 0 for slices, 1 for names
- if (rangeCommand.isNamesQuery())
- {
- PartitionColumns columns = rangeCommand.columnFilter().fetchedColumns();
- ClusteringIndexNamesFilter filter = (ClusteringIndexNamesFilter) rangeCommand.dataRange().clusteringIndexFilter;
- size += LegacyReadCommandSerializer.serializedNamesFilterSize(filter, metadata, columns);
- }
- else
- {
- ClusteringIndexSliceFilter filter = (ClusteringIndexSliceFilter) rangeCommand.dataRange().clusteringIndexFilter;
- boolean makeStaticSlice = !rangeCommand.columnFilter().fetchedColumns().statics.isEmpty() && !filter.requestedSlices().selects(Clustering.STATIC_CLUSTERING);
- size += LegacyReadCommandSerializer.serializedSlicesSize(filter.requestedSlices(), makeStaticSlice, metadata);
- size += TypeSizes.sizeof(filter.isReversed());
- size += TypeSizes.sizeof(rangeCommand.limits().perPartitionCount());
- size += TypeSizes.sizeof(0); // compositesToGroup
- }
-
- if (rangeCommand.rowFilter().equals(RowFilter.NONE))
- {
- size += TypeSizes.sizeof(0);
- }
- else
- {
- ArrayList<RowFilter.Expression> indexExpressions = Lists.newArrayList(rangeCommand.rowFilter().iterator());
- size += TypeSizes.sizeof(indexExpressions.size());
- for (RowFilter.Expression expression : indexExpressions)
- {
- size += ByteBufferUtil.serializedSizeWithShortLength(expression.column().name.bytes);
- size += TypeSizes.sizeof(expression.operator().ordinal());
- size += ByteBufferUtil.serializedSizeWithShortLength(expression.getIndexValue());
- }
- }
-
- size += AbstractBounds.rowPositionSerializer.serializedSize(rangeCommand.dataRange().keyRange(), version);
- size += TypeSizes.sizeof(rangeCommand.limits().count());
- size += TypeSizes.sizeof(!rangeCommand.isForThrift());
- return size + TypeSizes.sizeof(rangeCommand.dataRange().isPaging());
- }
-
- static PartitionRangeReadCommand maybeConvertNamesToSlice(PartitionRangeReadCommand command)
- {
- if (!command.dataRange().isNamesQuery())
- return command;
-
- CFMetaData metadata = command.metadata();
- if (!LegacyReadCommandSerializer.shouldConvertNamesToSlice(metadata, command.columnFilter().fetchedColumns()))
- return command;
-
- ClusteringIndexNamesFilter filter = (ClusteringIndexNamesFilter) command.dataRange().clusteringIndexFilter;
- ClusteringIndexSliceFilter sliceFilter = LegacyReadCommandSerializer.convertNamesFilterToSliceFilter(filter, metadata);
- DataRange newRange = new DataRange(command.dataRange().keyRange(), sliceFilter);
-
- return command.withUpdatedDataRange(newRange);
- }
-
- static ColumnFilter getColumnSelectionForSlice(boolean selectsStatics, int compositesToGroup, CFMetaData metadata)
- {
- // A value of -2 indicates this is a DISTINCT query that doesn't select static columns, only partition keys.
- // In that case, we'll basically be querying the first row of the partition, but we must make sure we include
- // all columns so we get at least one cell if there is a live row as it would confuse pre-3.0 nodes otherwise.
- if (compositesToGroup == -2)
- return ColumnFilter.all(metadata);
-
- // if a slice query from a pre-3.0 node doesn't cover statics, we shouldn't select them at all
- PartitionColumns columns = selectsStatics
- ? metadata.partitionColumns()
- : metadata.partitionColumns().withoutStatics();
- return ColumnFilter.selectionBuilder().addAll(columns).build();
- }
- }
-
- /**
- * Serializer for pre-3.0 PagedRangeCommands.
- */
- private static class LegacyPagedRangeCommandSerializer implements IVersionedSerializer<ReadCommand>
- {
- public void serialize(ReadCommand command, DataOutputPlus out, int version) throws IOException
- {
- assert version < MessagingService.VERSION_30;
-
- PartitionRangeReadCommand rangeCommand = (PartitionRangeReadCommand) command;
- assert rangeCommand.dataRange().isPaging();
-
- CFMetaData metadata = rangeCommand.metadata();
-
- out.writeUTF(metadata.ksName);
- out.writeUTF(metadata.cfName);
- out.writeLong(rangeCommand.nowInSec() * 1000L); // convert from seconds to millis
-
- AbstractBounds.rowPositionSerializer.serialize(rangeCommand.dataRange().keyRange(), out, version);
-
- // pre-3.0 nodes don't accept names filters for paged range commands
- ClusteringIndexSliceFilter filter;
- if (rangeCommand.dataRange().clusteringIndexFilter.kind() == ClusteringIndexFilter.Kind.NAMES)
- filter = LegacyReadCommandSerializer.convertNamesFilterToSliceFilter((ClusteringIndexNamesFilter) rangeCommand.dataRange().clusteringIndexFilter, metadata);
- else
- filter = (ClusteringIndexSliceFilter) rangeCommand.dataRange().clusteringIndexFilter;
-
- // slice filter
- boolean makeStaticSlice = !rangeCommand.columnFilter().fetchedColumns().statics.isEmpty() && !filter.requestedSlices().selects(Clustering.STATIC_CLUSTERING);
- LegacyReadCommandSerializer.serializeSlices(out, filter.requestedSlices(), filter.isReversed(), makeStaticSlice, metadata);
- out.writeBoolean(filter.isReversed());
-
- // slice filter's count
- DataLimits.Kind kind = rangeCommand.limits().kind();
- boolean isDistinct = (kind == DataLimits.Kind.CQL_LIMIT || kind == DataLimits.Kind.CQL_PAGING_LIMIT) && rangeCommand.limits().perPartitionCount() == 1;
- if (isDistinct)
- out.writeInt(1);
- else
- out.writeInt(LegacyReadCommandSerializer.updateLimitForQuery(rangeCommand.limits().perPartitionCount(), filter.requestedSlices()));
-
- // compositesToGroup
- boolean selectsStatics = !rangeCommand.columnFilter().fetchedColumns().statics.isEmpty() || filter.requestedSlices().selects(Clustering.STATIC_CLUSTERING);
- int compositesToGroup;
- if (kind == DataLimits.Kind.THRIFT_LIMIT)
- compositesToGroup = -1;
- else if (isDistinct && !selectsStatics)
- compositesToGroup = -2; // for DISTINCT queries (CASSANDRA-8490)
- else
- compositesToGroup = metadata.isDense() ? -1 : metadata.clusteringColumns().size();
-
- out.writeInt(compositesToGroup);
-
- // command-level "start" and "stop" composites. The start is the last-returned cell name if there is one,
- // otherwise it's the same as the slice filter's start. The stop appears to always be the same as the
- // slice filter's stop.
- DataRange.Paging pagingRange = (DataRange.Paging) rangeCommand.dataRange();
- Clustering lastReturned = pagingRange.getLastReturned();
- ClusteringBound newStart = ClusteringBound.inclusiveStartOf(lastReturned);
- Slice lastSlice = filter.requestedSlices().get(filter.requestedSlices().size() - 1);
- ByteBufferUtil.writeWithShortLength(LegacyLayout.encodeBound(metadata, newStart, true), out);
- ByteBufferUtil.writeWithShortLength(LegacyLayout.encodeClustering(metadata, lastSlice.end().clustering()), out);
-
- LegacyRangeSliceCommandSerializer.serializeRowFilter(out, rangeCommand.rowFilter());
-
- // command-level limit
- // Pre-3.0 we would always request one more row than we actually needed and the command-level "start" would
- // be the last-returned cell name, so the response would always include it.
- int maxResults = rangeCommand.limits().count() + 1;
- out.writeInt(maxResults);
-
- // countCQL3Rows
- if (rangeCommand.isForThrift() || rangeCommand.limits().perPartitionCount() == 1) // for Thrift or DISTINCT
- out.writeBoolean(false);
- else
- out.writeBoolean(true);
- }
-
- public ReadCommand deserialize(DataInputPlus in, int version) throws IOException
- {
- assert version < MessagingService.VERSION_30;
-
- String keyspace = in.readUTF();
- String columnFamily = in.readUTF();
-
- CFMetaData metadata = Schema.instance.getCFMetaData(keyspace, columnFamily);
- if (metadata == null)
- {
- String message = String.format("Got legacy paged range command for nonexistent table %s.%s.", keyspace, columnFamily);
- throw new UnknownColumnFamilyException(message, null);
- }
-
- int nowInSec = (int) (in.readLong() / 1000); // convert from millis to seconds
- AbstractBounds<PartitionPosition> keyRange = AbstractBounds.rowPositionSerializer.deserialize(in, metadata.partitioner, version);
-
- Pair<ClusteringIndexSliceFilter, Boolean> p = LegacyReadCommandSerializer.deserializeSlicePartitionFilter(in, metadata);
- ClusteringIndexSliceFilter filter = p.left;
- boolean selectsStatics = p.right;
-
- int perPartitionLimit = in.readInt();
- int compositesToGroup = in.readInt();
-
- // command-level Composite "start" and "stop"
- LegacyLayout.LegacyBound startBound = LegacyLayout.decodeSliceBound(metadata, ByteBufferUtil.readWithShortLength(in), true);
-
- ByteBufferUtil.readWithShortLength(in); // the composite "stop", which isn't actually needed
-
- ColumnFilter selection = LegacyRangeSliceCommandSerializer.getColumnSelectionForSlice(selectsStatics, compositesToGroup, metadata);
-
- RowFilter rowFilter = LegacyRangeSliceCommandSerializer.deserializeRowFilter(in, metadata);
- int maxResults = in.readInt();
- boolean countCQL3Rows = in.readBoolean();
-
- // We have 2 types of DISTINCT queries: ones on only the partition key, and ones on the partition key and static columns. For the former,
- // we can easily detect the case because compositeToGroup is -2 and that's the only case it can be that. The latter one is slightly less
- // direct, but we know that on 2.1/2.2 queries, DISTINCT queries are the only CQL queries that have countCQL3Rows to false so we use
- // that fact.
- boolean isDistinct = compositesToGroup == -2 || (compositesToGroup != -1 && !countCQL3Rows);
- DataLimits limits;
- if (isDistinct)
- limits = DataLimits.distinctLimits(maxResults);
- else
- limits = DataLimits.cqlLimits(maxResults);
-
- limits = limits.forPaging(maxResults);
-
- // The pagedRangeCommand is used in pre-3.0 for both the first page and the following ones. On the first page, the startBound will be
- // the start of the overall slice and will not be a proper Clustering. So detect that case and just return a non-paging DataRange, which
- // is what 3.0 does.
- DataRange dataRange = new DataRange(keyRange, filter);
- Slices slices = filter.requestedSlices();
- if (!isDistinct && startBound != LegacyLayout.LegacyBound.BOTTOM && !startBound.bound.equals(slices.get(0).start()))
- {
- // pre-3.0 nodes normally expect pages to include the last cell from the previous page, but they handle it
- // missing without any problems, so we can safely always set "inclusive" to false in the data range
- dataRange = dataRange.forPaging(keyRange, metadata.comparator, startBound.getAsClustering(metadata), false);
- }
- return PartitionRangeReadCommand.create(true, metadata, nowInSec, selection, rowFilter, limits, dataRange);
- }
-
- public long serializedSize(ReadCommand command, int version)
- {
- assert version < MessagingService.VERSION_30;
- assert command.kind == Kind.PARTITION_RANGE;
-
- PartitionRangeReadCommand rangeCommand = (PartitionRangeReadCommand) command;
- CFMetaData metadata = rangeCommand.metadata();
- assert rangeCommand.dataRange().isPaging();
-
- long size = TypeSizes.sizeof(metadata.ksName);
- size += TypeSizes.sizeof(metadata.cfName);
- size += TypeSizes.sizeof((long) rangeCommand.nowInSec());
-
- size += AbstractBounds.rowPositionSerializer.serializedSize(rangeCommand.dataRange().keyRange(), version);
-
- // pre-3.0 nodes only accept slice filters for paged range commands
- ClusteringIndexSliceFilter filter;
- if (rangeCommand.dataRange().clusteringIndexFilter.kind() == ClusteringIndexFilter.Kind.NAMES)
- filter = LegacyReadCommandSerializer.convertNamesFilterToSliceFilter((ClusteringIndexNamesFilter) rangeCommand.dataRange().clusteringIndexFilter, metadata);
- else
- filter = (ClusteringIndexSliceFilter) rangeCommand.dataRange().clusteringIndexFilter;
-
- // slice filter
- boolean makeStaticSlice = !rangeCommand.columnFilter().fetchedColumns().statics.isEmpty() && !filter.requestedSlices().selects(Clustering.STATIC_CLUSTERING);
- size += LegacyReadCommandSerializer.serializedSlicesSize(filter.requestedSlices(), makeStaticSlice, metadata);
- size += TypeSizes.sizeof(filter.isReversed());
-
- // slice filter's count
- size += TypeSizes.sizeof(rangeCommand.limits().perPartitionCount());
-
- // compositesToGroup
- size += TypeSizes.sizeof(0);
-
- // command-level Composite "start" and "stop"
- DataRange.Paging pagingRange = (DataRange.Paging) rangeCommand.dataRange();
- Clustering lastReturned = pagingRange.getLastReturned();
- Slice lastSlice = filter.requestedSlices().get(filter.requestedSlices().size() - 1);
- size += ByteBufferUtil.serializedSizeWithShortLength(LegacyLayout.encodeClustering(metadata, lastReturned));
- size += ByteBufferUtil.serializedSizeWithShortLength(LegacyLayout.encodeClustering(metadata, lastSlice.end().clustering()));
-
- size += LegacyRangeSliceCommandSerializer.serializedRowFilterSize(rangeCommand.rowFilter());
-
- // command-level limit
- size += TypeSizes.sizeof(rangeCommand.limits().count());
-
- // countCQL3Rows
- return size + TypeSizes.sizeof(true);
- }
- }
-
- /**
- * Serializer for pre-3.0 ReadCommands.
- */
- static class LegacyReadCommandSerializer implements IVersionedSerializer<ReadCommand>
- {
- public void serialize(ReadCommand command, DataOutputPlus out, int version) throws IOException
- {
- assert version < MessagingService.VERSION_30;
- assert command.kind == Kind.SINGLE_PARTITION;
-
- SinglePartitionReadCommand singleReadCommand = (SinglePartitionReadCommand) command;
- singleReadCommand = maybeConvertNamesToSlice(singleReadCommand);
-
- CFMetaData metadata = singleReadCommand.metadata();
-
- out.writeByte(LegacyType.fromPartitionFilterKind(singleReadCommand.clusteringIndexFilter().kind()).serializedValue);
-
- out.writeBoolean(singleReadCommand.isDigestQuery());
- out.writeUTF(metadata.ksName);
- ByteBufferUtil.writeWithShortLength(singleReadCommand.partitionKey().getKey(), out);
- out.writeUTF(metadata.cfName);
- out.writeLong(singleReadCommand.nowInSec() * 1000L); // convert from seconds to millis
-
- if (singleReadCommand.clusteringIndexFilter().kind() == ClusteringIndexFilter.Kind.SLICE)
- serializeSliceCommand(singleReadCommand, out);
- else
- serializeNamesCommand(singleReadCommand, out);
- }
-
- public ReadCommand deserialize(DataInputPlus in, int version) throws IOException
- {
- assert version < MessagingService.VERSION_30;
- LegacyType msgType = LegacyType.fromSerializedValue(in.readByte());
-
- boolean isDigest = in.readBoolean();
- String keyspaceName = in.readUTF();
- ByteBuffer key = ByteBufferUtil.readWithShortLength(in);
- String cfName = in.readUTF();
- long nowInMillis = in.readLong();
- int nowInSeconds = (int) (nowInMillis / 1000); // convert from millis to seconds
- CFMetaData metadata = Schema.instance.getCFMetaData(keyspaceName, cfName);
- DecoratedKey dk = metadata.partitioner.decorateKey(key);
-
- switch (msgType)
- {
- case GET_BY_NAMES:
- return deserializeNamesCommand(in, isDigest, metadata, dk, nowInSeconds, version);
- case GET_SLICES:
- return deserializeSliceCommand(in, isDigest, metadata, dk, nowInSeconds, version);
- default:
- throw new AssertionError();
- }
- }
-
- public long serializedSize(ReadCommand command, int version)
- {
- assert version < MessagingService.VERSION_30;
- assert command.kind == Kind.SINGLE_PARTITION;
- SinglePartitionReadCommand singleReadCommand = (SinglePartitionReadCommand) command;
- singleReadCommand = maybeConvertNamesToSlice(singleReadCommand);
-
- int keySize = singleReadCommand.partitionKey().getKey().remaining();
-
- CFMetaData metadata = singleReadCommand.metadata();
-
- long size = 1; // message type (single byte)
- size += TypeSizes.sizeof(command.isDigestQuery());
- size += TypeSizes.sizeof(metadata.ksName);
- size += TypeSizes.sizeof((short) keySize) + keySize;
- size += TypeSizes.sizeof((long) command.nowInSec());
-
- if (singleReadCommand.clusteringIndexFilter().kind() == ClusteringIndexFilter.Kind.SLICE)
- return size + serializedSliceCommandSize(singleReadCommand);
- else
- return size + serializedNamesCommandSize(singleReadCommand);
- }
-
- private void serializeNamesCommand(SinglePartitionReadCommand command, DataOutputPlus out) throws IOException
- {
- serializeNamesFilter(command, (ClusteringIndexNamesFilter)command.clusteringIndexFilter(), out);
- }
-
- private static void serializeNamesFilter(ReadCommand command, ClusteringIndexNamesFilter filter, DataOutputPlus out) throws IOException
- {
- PartitionColumns columns = command.columnFilter().fetchedColumns();
- CFMetaData metadata = command.metadata();
- SortedSet<Clustering> requestedRows = filter.requestedRows();
-
- if (requestedRows.isEmpty())
- {
- // only static columns are requested
- out.writeInt(columns.size());
- for (ColumnDefinition column : columns)
- ByteBufferUtil.writeWithShortLength(column.name.bytes, out);
- }
- else
- {
- out.writeInt(requestedRows.size() * columns.size());
- for (Clustering clustering : requestedRows)
- {
- for (ColumnDefinition column : columns)
- ByteBufferUtil.writeWithShortLength(LegacyLayout.encodeCellName(metadata, clustering, column.name.bytes, null), out);
- }
- }
-
- // countCql3Rows should be true if it's not for Thrift or a DISTINCT query
- if (command.isForThrift() || (command.limits().kind() == DataLimits.Kind.CQL_LIMIT && command.limits().perPartitionCount() == 1))
- out.writeBoolean(false); // it's compact and not a DISTINCT query
- else
- out.writeBoolean(true);
- }
-
- static long serializedNamesFilterSize(ClusteringIndexNamesFilter filter, CFMetaData metadata, PartitionColumns fetchedColumns)
- {
- SortedSet<Clustering> requestedRows = filter.requestedRows();
-
- long size = 0;
- if (requestedRows.isEmpty())
- {
- // only static columns are requested
- size += TypeSizes.sizeof(fetchedColumns.size());
- for (ColumnDefinition column : fetchedColumns)
- size += ByteBufferUtil.serializedSizeWithShortLength(column.name.bytes);
- }
- else
- {
- size += TypeSizes.sizeof(requestedRows.size() * fetchedColumns.size());
- for (Clustering clustering : requestedRows)
- {
- for (ColumnDefinition column : fetchedColumns)
- size += ByteBufferUtil.serializedSizeWithShortLength(LegacyLayout.encodeCellName(metadata, clustering, column.name.bytes, null));
- }
- }
-
- return size + TypeSizes.sizeof(true); // countCql3Rows
- }
-
- private SinglePartitionReadCommand deserializeNamesCommand(DataInputPlus in, boolean isDigest, CFMetaData metadata, DecoratedKey key, int nowInSeconds, int version) throws IOException
- {
- Pair<ColumnFilter, ClusteringIndexNamesFilter> selectionAndFilter = deserializeNamesSelectionAndFilter(in, metadata);
-
- return SinglePartitionReadCommand.legacyNamesCommand(isDigest, version, metadata, nowInSeconds, selectionAndFilter.left, key, selectionAndFilter.right);
- }
-
- static Pair<ColumnFilter, ClusteringIndexNamesFilter> deserializeNamesSelectionAndFilter(DataInputPlus in, CFMetaData metadata) throws IOException
- {
- int numCellNames = in.readInt();
-
- // The names filter could include either a) static columns or b) normal columns with the clustering columns
- // fully specified. We need to handle those cases differently in 3.0.
- NavigableSet<Clustering> clusterings = new TreeSet<>(metadata.comparator);
-
- ColumnFilter.Builder selectionBuilder = ColumnFilter.selectionBuilder();
- for (int i = 0; i < numCellNames; i++)
- {
- ByteBuffer buffer = ByteBufferUtil.readWithShortLength(in);
- LegacyLayout.LegacyCellName cellName;
- try
- {
- cellName = LegacyLayout.decodeCellName(metadata, buffer);
- }
- catch (UnknownColumnException exc)
- {
- // TODO this probably needs a new exception class that shares a parent with UnknownColumnFamilyException
- throw new UnknownColumnFamilyException(
- "Received legacy range read command with names filter for unrecognized column name. " +
- "Fill name in filter (hex): " + ByteBufferUtil.bytesToHex(buffer), metadata.cfId);
- }
-
- // If we're querying for a static column, we may also need to read it
- // as if it were a thrift dynamic column (because the column metadata,
- // which makes it a static column in 3.0+, may have been added *after*
- // some values were written). Note that all cql queries on non-compact
- // tables used slice & not name filters prior to 3.0 so this path is
- // not taken for non-compact tables. It is theoretically possible to
- // get here via thrift, hence the check on metadata.isStaticCompactTable.
- // See CASSANDRA-11087.
- if (metadata.isStaticCompactTable() && cellName.clustering.equals(Clustering.STATIC_CLUSTERING))
- {
- clusterings.add(Clustering.make(cellName.column.name.bytes));
- selectionBuilder.add(metadata.compactValueColumn());
- }
- else
- {
- clusterings.add(cellName.clustering);
- }
-
- selectionBuilder.add(cellName.column);
- }
-
- // for compact storage tables without clustering keys, the column holding the selected value is named
- // 'value' internally we add it to the selection here to prevent errors due to unexpected column names
- // when serializing the initial local data response
- if (metadata.isStaticCompactTable() && clusterings.isEmpty())
- selectionBuilder.addAll(metadata.partitionColumns());
-
- in.readBoolean(); // countCql3Rows
-
- // clusterings cannot include STATIC_CLUSTERING, so if the names filter is for static columns, clusterings
- // will be empty. However, by requesting the static columns in our ColumnFilter, this will still work.
- ClusteringIndexNamesFilter filter = new ClusteringIndexNamesFilter(clusterings, false);
- return Pair.create(selectionBuilder.build(), filter);
- }
-
- private long serializedNamesCommandSize(SinglePartitionReadCommand command)
- {
- ClusteringIndexNamesFilter filter = (ClusteringIndexNamesFilter)command.clusteringIndexFilter();
- PartitionColumns columns = command.columnFilter().fetchedColumns();
- return serializedNamesFilterSize(filter, command.metadata(), columns);
- }
-
- private void serializeSliceCommand(SinglePartitionReadCommand command, DataOutputPlus out) throws IOException
- {
- CFMetaData metadata = command.metadata();
- ClusteringIndexSliceFilter filter = (ClusteringIndexSliceFilter)command.clusteringIndexFilter();
-
- Slices slices = filter.requestedSlices();
- boolean makeStaticSlice = !command.columnFilter().fetchedColumns().statics.isEmpty() && !slices.selects(Clustering.STATIC_CLUSTERING);
- serializeSlices(out, slices, filter.isReversed(), makeStaticSlice, metadata);
-
- out.writeBoolean(filter.isReversed());
-
- boolean selectsStatics = !command.columnFilter().fetchedColumns().statics.isEmpty() || slices.selects(Clustering.STATIC_CLUSTERING);
- DataLimits limits = command.limits();
- if (limits.isDistinct())
- out.writeInt(1); // the limit is always 1 for DISTINCT queries
- else
- out.writeInt(updateLimitForQuery(command.limits().count(), filter.requestedSlices()));
-
- int compositesToGroup;
- if (limits.kind() == DataLimits.Kind.THRIFT_LIMIT || metadata.isDense())
- compositesToGroup = -1;
- else if (limits.isDistinct() && !selectsStatics)
- compositesToGroup = -2; // for DISTINCT queries (CASSANDRA-8490)
- else
- compositesToGroup = metadata.clusteringColumns().size();
-
- out.writeInt(compositesToGroup);
- }
-
- private SinglePartitionReadCommand deserializeSliceCommand(DataInputPlus in, boolean isDigest, CFMetaData metadata, DecoratedKey key, int nowInSeconds, int version) throws IOException
- {
- Pair<ClusteringIndexSliceFilter, Boolean> p = deserializeSlicePartitionFilter(in, metadata);
- ClusteringIndexSliceFilter filter = p.left;
- boolean selectsStatics = p.right;
- int count = in.readInt();
- int compositesToGroup = in.readInt();
-
- // if a slice query from a pre-3.0 node doesn't cover statics, we shouldn't select them at all
- ColumnFilter columnFilter = LegacyRangeSliceCommandSerializer.getColumnSelectionForSlice(selectsStatics, compositesToGroup, metadata);
-
- // We have 2 types of DISTINCT queries: ones on only the partition key, and ones on the partition key and static columns. For the former,
- // we can easily detect the case because compositeToGroup is -2 and that's the only case it can be that. The latter is probablematic
- // however as we have no way to distinguish it from a normal select with a limit of 1 (and this, contrarily to the range query case
- // were the countCQL3Rows boolean allows us to decide).
- // So we consider this case not distinct here. This is ok because even if it is a distinct (with static), the count will be 1 and
- // we'll still just query one row (a distinct DataLimits currently behave exactly like a CQL limit with a count of 1). The only
- // drawback is that we'll send back the first row entirely while a 2.1/2.2 node would return only the first cell in that same
- // situation. This isn't a problem for 2.1/2.2 code however (it would be for a range query, as it would throw off the count for
- // reasons similar to CASSANDRA-10762, but it's ok for single partition queries).
- // We do _not_ want to do the reverse however and consider a 'SELECT * FROM foo LIMIT 1' as a DISTINCT query as that would make
- // us only return the 1st cell rather then 1st row.
- DataLimits limits;
- if (compositesToGroup == -2)
- limits = DataLimits.distinctLimits(count); // See CASSANDRA-8490 for the explanation of this value
- else if (compositesToGroup == -1)
- limits = DataLimits.thriftLimits(1, count);
- else
- limits = DataLimits.cqlLimits(count);
-
- return SinglePartitionReadCommand.legacySliceCommand(isDigest, version, metadata, nowInSeconds, columnFilter, limits, key, filter);
- }
-
- private long serializedSliceCommandSize(SinglePartitionReadCommand command)
- {
- CFMetaData metadata = command.metadata();
- ClusteringIndexSliceFilter filter = (ClusteringIndexSliceFilter)command.clusteringIndexFilter();
-
- Slices slices = filter.requestedSlices();
- boolean makeStaticSlice = !command.columnFilter().fetchedColumns().statics.isEmpty() && !slices.selects(Clustering.STATIC_CLUSTERING);
-
- long size = serializedSlicesSize(slices, makeStaticSlice, metadata);
- size += TypeSizes.sizeof(command.clusteringIndexFilter().isReversed());
- size += TypeSizes.sizeof(command.limits().count());
- return size + TypeSizes.sizeof(0); // compositesToGroup
- }
-
- static void serializeSlices(DataOutputPlus out, Slices slices, boolean isReversed, boolean makeStaticSlice, CFMetaData metadata) throws IOException
- {
- out.writeInt(slices.size() + (makeStaticSlice ? 1 : 0));
-
- // In 3.0 we always store the slices in normal comparator order. Pre-3.0 nodes expect the slices to
- // be in reversed order if the query is reversed, so we handle that here.
- if (isReversed)
- {
- for (int i = slices.size() - 1; i >= 0; i--)
- serializeSlice(out, slices.get(i), true, metadata);
- if (makeStaticSlice)
- serializeStaticSlice(out, true, metadata);
- }
- else
- {
- if (makeStaticSlice)
- serializeStaticSlice(out, false, metadata);
- for (Slice slice : slices)
- serializeSlice(out, slice, false, metadata);
- }
- }
-
- static long serializedSlicesSize(Slices slices, boolean makeStaticSlice, CFMetaData metadata)
- {
- long size = TypeSizes.sizeof(slices.size());
-
- for (Slice slice : slices)
- {
- ByteBuffer sliceStart = LegacyLayout.encodeBound(metadata, slice.start(), true);
- size += ByteBufferUtil.serializedSizeWithShortLength(sliceStart);
- ByteBuffer sliceEnd = LegacyLayout.encodeBound(metadata, slice.end(), false);
- size += ByteBufferUtil.serializedSizeWithShortLength(sliceEnd);
- }
-
- if (makeStaticSlice)
- size += serializedStaticSliceSize(metadata);
-
- return size;
- }
-
- static long serializedStaticSliceSize(CFMetaData metadata)
- {
- // unlike serializeStaticSlice(), but we don't care about reversal for size calculations
- ByteBuffer sliceStart = LegacyLayout.encodeBound(metadata, ClusteringBound.BOTTOM, false);
- long size = ByteBufferUtil.serializedSizeWithShortLength(sliceStart);
-
- size += TypeSizes.sizeof((short) (metadata.comparator.size() * 3 + 2));
- size += TypeSizes.sizeof((short) LegacyLayout.STATIC_PREFIX);
- for (int i = 0; i < metadata.comparator.size(); i++)
- {
- size += ByteBufferUtil.serializedSizeWithShortLength(ByteBufferUtil.EMPTY_BYTE_BUFFER);
- size += 1; // EOC
- }
- return size;
- }
-
- private static void serializeSlice(DataOutputPlus out, Slice slice, boolean isReversed, CFMetaData metadata) throws IOException
- {
- ByteBuffer sliceStart = LegacyLayout.encodeBound(metadata, isReversed ? slice.end() : slice.start(), !isReversed);
- ByteBufferUtil.writeWithShortLength(sliceStart, out);
-
- ByteBuffer sliceEnd = LegacyLayout.encodeBound(metadata, isReversed ? slice.start() : slice.end(), isReversed);
- ByteBufferUtil.writeWithShortLength(sliceEnd, out);
- }
-
- private static void serializeStaticSlice(DataOutputPlus out, boolean isReversed, CFMetaData metadata) throws IOException
- {
- // if reversed, write an empty bound for the slice start; if reversed, write out an empty bound for the
- // slice finish after we've written the static slice start
- if (!isReversed)
- {
- ByteBuffer sliceStart = LegacyLayout.encodeBound(metadata, ClusteringBound.BOTTOM, false);
- ByteBufferUtil.writeWithShortLength(sliceStart, out);
- }
-
- // write out the length of the composite
- out.writeShort(2 + metadata.comparator.size() * 3); // two bytes + EOC for each component, plus static prefix
- out.writeShort(LegacyLayout.STATIC_PREFIX);
- for (int i = 0; i < metadata.comparator.size(); i++)
- {
- ByteBufferUtil.writeWithShortLength(ByteBufferUtil.EMPTY_BYTE_BUFFER, out);
- // write the EOC, using an inclusive end if we're on the final component
- out.writeByte(i == metadata.comparator.size() - 1 ? 1 : 0);
- }
-
- if (isReversed)
- {
- ByteBuffer sliceStart = LegacyLayout.encodeBound(metadata, ClusteringBound.BOTTOM, false);
- ByteBufferUtil.writeWithShortLength(sliceStart, out);
- }
- }
-
- // Returns the deserialized filter, and whether static columns are queried (in pre-3.0, both info are determined by the slices,
- // but in 3.0 they are separated: whether static columns are queried or not depends on the ColumnFilter).
- static Pair<ClusteringIndexSliceFilter, Boolean> deserializeSlicePartitionFilter(DataInputPlus in, CFMetaData metadata) throws IOException
- {
- int numSlices = in.readInt();
- ByteBuffer[] startBuffers = new ByteBuffer[numSlices];
- ByteBuffer[] finishBuffers = new ByteBuffer[numSlices];
- for (int i = 0; i < numSlices; i++)
- {
- startBuffers[i] = ByteBufferUtil.readWithShortLength(in);
- finishBuffers[i] = ByteBufferUtil.readWithShortLength(in);
- }
-
- boolean reversed = in.readBoolean();
-
- if (reversed)
- {
- // pre-3.0, reversed query slices put the greater element at the start of the slice
- ByteBuffer[] tmp = finishBuffers;
- finishBuffers = startBuffers;
- startBuffers = tmp;
- }
-
- boolean selectsStatics = false;
- Slices.Builder slicesBuilder = new Slices.Builder(metadata.comparator);
- for (int i = 0; i < numSlices; i++)
- {
- LegacyLayout.LegacyBound start = LegacyLayout.decodeSliceBound(metadata, startBuffers[i], true);
- LegacyLayout.LegacyBound finish = LegacyLayout.decodeSliceBound(metadata, finishBuffers[i], false);
-
- if (start.isStatic)
- {
- // If we start at the static block, this means we start at the beginning of the partition in 3.0
- // terms (since 3.0 handles static outside of the slice).
- start = LegacyLayout.LegacyBound.BOTTOM;
-
- // Then if we include the static, records it
- if (start.bound.isInclusive())
- selectsStatics = true;
- }
- else if (start == LegacyLayout.LegacyBound.BOTTOM)
- {
- selectsStatics = true;
- }
-
- // If the end of the slice is the end of the statics, then that mean this slice was just selecting static
- // columns. We have already recorded that in selectsStatics, so we can ignore the slice (which doesn't make
- // sense for 3.0).
- if (finish.isStatic)
- {
- assert finish.bound.isInclusive(); // it would make no sense for a pre-3.0 node to have a slice that stops
- // before the static columns (since there is nothing before that)
- continue;
- }
-
- slicesBuilder.add(Slice.make(start.bound, finish.bound));
- }
-
- return Pair.create(new ClusteringIndexSliceFilter(slicesBuilder.build(), reversed), selectsStatics);
- }
-
- private static SinglePartitionReadCommand maybeConvertNamesToSlice(SinglePartitionReadCommand command)
- {
- if (command.clusteringIndexFilter().kind() != ClusteringIndexFilter.Kind.NAMES)
- return command;
-
- CFMetaData metadata = command.metadata();
-
- if (!shouldConvertNamesToSlice(metadata, command.columnFilter().fetchedColumns()))
- return command;
-
- ClusteringIndexNamesFilter filter = (ClusteringIndexNamesFilter)command.clusteringIndexFilter();
- ClusteringIndexSliceFilter sliceFilter = convertNamesFilterToSliceFilter(filter, metadata);
-
- return command.withUpdatedClusteringIndexFilter(sliceFilter);
- }
-
- /**
- * Returns true if a names filter on the given table and column selection should be converted to a slice
- * filter for compatibility with pre-3.0 nodes, false otherwise.
- */
- static boolean shouldConvertNamesToSlice(CFMetaData metadata, PartitionColumns columns)
- {
- // On pre-3.0 nodes, due to CASSANDRA-5762, we always do a slice for CQL3 tables (not dense, composite).
- if (!metadata.isDense() && metadata.isCompound())
- return true;
-
- // pre-3.0 nodes don't support names filters for reading collections, so if we're requesting any of those,
- // we need to convert this to a slice filter
- for (ColumnDefinition column : columns)
- {
- if (column.type.isMultiCell())
- return true;
- }
- return false;
- }
-
- /**
- * Converts a names filter that is incompatible with pre-3.0 nodes to a slice filter that is compatible.
- */
- private static ClusteringIndexSliceFilter convertNamesFilterToSliceFilter(ClusteringIndexNamesFilter filter, CFMetaData metadata)
- {
- SortedSet<Clustering> requestedRows = filter.requestedRows();
- Slices slices;
- if (requestedRows.isEmpty())
- {
- slices = Slices.NONE;
- }
- else if (requestedRows.size() == 1 && requestedRows.first().size() == 0)
- {
- slices = Slices.ALL;
- }
- else
- {
- Slices.Builder slicesBuilder = new Slices.Builder(metadata.comparator);
- for (Clustering clustering : requestedRows)
- slicesBuilder.add(ClusteringBound.inclusiveStartOf(clustering), ClusteringBound.inclusiveEndOf(clustering));
- slices = slicesBuilder.build();
- }
-
- return new ClusteringIndexSliceFilter(slices, filter.isReversed());
- }
-
- /**
- * Potentially increases the existing query limit to account for the lack of exclusive bounds in pre-3.0 nodes.
- * @param limit the existing query limit
- * @param slices the requested slices
- * @return the updated limit
- */
- static int updateLimitForQuery(int limit, Slices slices)
- {
- // Pre-3.0 nodes don't support exclusive bounds for slices. Instead, we query one more element if necessary
- // and filter it later (in LegacyRemoteDataResponse)
- if (!slices.hasLowerBound() && ! slices.hasUpperBound())
- return limit;
-
- for (Slice slice : slices)
- {
- if (limit == Integer.MAX_VALUE)
- return limit;
-
- if (!slice.start().isInclusive())
- limit++;
- if (!slice.end().isInclusive())
- limit++;
- }
- return limit;
+ + (command.isDigestQuery() ? TypeSizes.sizeofUnsignedVInt(command.digestVersion()) : 0)
+ + command.metadata().id.serializedSize()
+ + TypeSizes.sizeof(command.nowInSec())
+ + ColumnFilter.serializer.serializedSize(command.columnFilter(), version)
+ + RowFilter.serializer.serializedSize(command.rowFilter(), version)
+ + DataLimits.serializer.serializedSize(command.limits(), version, command.metadata().comparator)
+ + command.selectionSerializedSize(version)
+ + command.indexSerializedSize(version);
}
}
}
diff --git a/src/java/org/apache/cassandra/db/ReadCommandVerbHandler.java b/src/java/org/apache/cassandra/db/ReadCommandVerbHandler.java
index a71e92d..2c28ed9 100644
--- a/src/java/org/apache/cassandra/db/ReadCommandVerbHandler.java
+++ b/src/java/org/apache/cassandra/db/ReadCommandVerbHandler.java
@@ -17,23 +17,29 @@
*/
package org.apache.cassandra.db;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
-import org.apache.cassandra.io.IVersionedSerializer;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.locator.Replica;
import org.apache.cassandra.net.IVerbHandler;
-import org.apache.cassandra.net.MessageIn;
-import org.apache.cassandra.net.MessageOut;
+import org.apache.cassandra.net.Message;
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.tracing.Tracing;
+import static java.util.concurrent.TimeUnit.NANOSECONDS;
+
public class ReadCommandVerbHandler implements IVerbHandler<ReadCommand>
{
- protected IVersionedSerializer<ReadResponse> serializer()
- {
- return ReadResponse.serializer;
- }
+ public static final ReadCommandVerbHandler instance = new ReadCommandVerbHandler();
- public void doVerb(MessageIn<ReadCommand> message, int id)
+ private static final Logger logger = LoggerFactory.getLogger(ReadCommandVerbHandler.class);
+
+ public void doVerb(Message<ReadCommand> message)
{
if (StorageService.instance.isBootstrapMode())
{
@@ -41,7 +47,13 @@
}
ReadCommand command = message.payload;
- command.setMonitoringTime(message.constructionTime, message.isCrossNode(), message.getTimeout(), message.getSlowQueryTimeout());
+ validateTransientStatus(message);
+
+ long timeout = message.expiresAtNanos() - message.createdAtNanos();
+ command.setMonitoringTime(message.createdAtNanos(), message.isCrossNode(), timeout, DatabaseDescriptor.getSlowQueryTimeout(NANOSECONDS));
+
+ if (message.trackRepairedData())
+ command.trackRepairedStatus();
ReadResponse response;
try (ReadExecutionController executionController = command.executionController();
@@ -52,13 +64,45 @@
if (!command.complete())
{
- Tracing.trace("Discarding partial response to {} (timed out)", message.from);
- MessagingService.instance().incrementDroppedMessages(message, message.getLifetimeInMS());
+ Tracing.trace("Discarding partial response to {} (timed out)", message.from());
+ MessagingService.instance().metrics.recordDroppedMessage(message, message.elapsedSinceCreated(NANOSECONDS), NANOSECONDS);
return;
}
- Tracing.trace("Enqueuing response to {}", message.from);
- MessageOut<ReadResponse> reply = new MessageOut<>(MessagingService.Verb.REQUEST_RESPONSE, response, serializer());
- MessagingService.instance().sendReply(reply, id, message.from);
+ Tracing.trace("Enqueuing response to {}", message.from());
+ Message<ReadResponse> reply = message.responseWith(response);
+ MessagingService.instance().send(reply, message.from());
+ }
+
+ private void validateTransientStatus(Message<ReadCommand> message)
+ {
+ ReadCommand command = message.payload;
+ Token token;
+
+ if (command instanceof SinglePartitionReadCommand)
+ token = ((SinglePartitionReadCommand) command).partitionKey().getToken();
+ else
+ token = ((PartitionRangeReadCommand) command).dataRange().keyRange().right.getToken();
+
+ Replica replica = Keyspace.open(command.metadata().keyspace)
+ .getReplicationStrategy()
+ .getLocalReplicaFor(token);
+
+ if (replica == null)
+ {
+ logger.warn("Received a read request from {} for a range that is not owned by the current replica {}.",
+ message.from(),
+ command);
+ return;
+ }
+
+ if (!command.acceptsTransient() && replica.isTransient())
+ {
+ MessagingService.instance().metrics.recordDroppedMessage(message, message.elapsedSinceCreated(NANOSECONDS), NANOSECONDS);
+ throw new InvalidRequestException(String.format("Attempted to serve %s data request from %s node in %s",
+ command.acceptsTransient() ? "transient" : "full",
+ replica.isTransient() ? "transient" : "full",
+ this));
+ }
}
}
diff --git a/src/java/org/apache/cassandra/db/ReadExecutionController.java b/src/java/org/apache/cassandra/db/ReadExecutionController.java
index 56bb0d3..73ddad8 100644
--- a/src/java/org/apache/cassandra/db/ReadExecutionController.java
+++ b/src/java/org/apache/cassandra/db/ReadExecutionController.java
@@ -17,21 +17,37 @@
*/
package org.apache.cassandra.db;
-import org.apache.cassandra.config.CFMetaData;
+import java.util.concurrent.TimeUnit;
+
import org.apache.cassandra.index.Index;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.utils.MonotonicClock;
import org.apache.cassandra.utils.concurrent.OpOrder;
+import static org.apache.cassandra.utils.MonotonicClock.preciseTime;
+
public class ReadExecutionController implements AutoCloseable
{
+ private static final long NO_SAMPLING = Long.MIN_VALUE;
+
// For every reads
private final OpOrder.Group baseOp;
- private final CFMetaData baseMetadata; // kept to sanity check that we have take the op order on the right table
+ private final TableMetadata baseMetadata; // kept to sanity check that we have take the op order on the right table
// For index reads
private final ReadExecutionController indexController;
- private final OpOrder.Group writeOp;
+ private final WriteContext writeContext;
+ private final ReadCommand command;
+ static MonotonicClock clock = preciseTime;
- private ReadExecutionController(OpOrder.Group baseOp, CFMetaData baseMetadata, ReadExecutionController indexController, OpOrder.Group writeOp)
+ private final long createdAtNanos; // Only used while sampling
+
+ private ReadExecutionController(ReadCommand command,
+ OpOrder.Group baseOp,
+ TableMetadata baseMetadata,
+ ReadExecutionController indexController,
+ WriteContext writeContext,
+ long createdAtNanos)
{
// We can have baseOp == null, but only when empty() is called, in which case the controller will never really be used
// (which validForReadOn should ensure). But if it's not null, we should have the proper metadata too.
@@ -39,7 +55,9 @@
this.baseOp = baseOp;
this.baseMetadata = baseMetadata;
this.indexController = indexController;
- this.writeOp = writeOp;
+ this.writeContext = writeContext;
+ this.command = command;
+ this.createdAtNanos = createdAtNanos;
}
public ReadExecutionController indexReadController()
@@ -47,19 +65,19 @@
return indexController;
}
- public OpOrder.Group writeOpOrderGroup()
+ public WriteContext getWriteContext()
{
- return writeOp;
+ return writeContext;
}
- public boolean validForReadOn(ColumnFamilyStore cfs)
+ boolean validForReadOn(ColumnFamilyStore cfs)
{
- return baseOp != null && cfs.metadata.cfId.equals(baseMetadata.cfId);
+ return baseOp != null && cfs.metadata.id.equals(baseMetadata.id);
}
public static ReadExecutionController empty()
{
- return new ReadExecutionController(null, null, null, null);
+ return new ReadExecutionController(null, null, null, null, null, NO_SAMPLING);
}
/**
@@ -77,40 +95,42 @@
ColumnFamilyStore baseCfs = Keyspace.openAndGetStore(command.metadata());
ColumnFamilyStore indexCfs = maybeGetIndexCfs(baseCfs, command);
+ long createdAtNanos = baseCfs.metric.topLocalReadQueryTime.isEnabled() ? clock.now() : NO_SAMPLING;
+
if (indexCfs == null)
+ return new ReadExecutionController(command, baseCfs.readOrdering.start(), baseCfs.metadata(), null, null, createdAtNanos);
+
+ OpOrder.Group baseOp = null;
+ WriteContext writeContext = null;
+ ReadExecutionController indexController = null;
+ // OpOrder.start() shouldn't fail, but better safe than sorry.
+ try
{
- return new ReadExecutionController(baseCfs.readOrdering.start(), baseCfs.metadata, null, null);
+ baseOp = baseCfs.readOrdering.start();
+ indexController = new ReadExecutionController(command, indexCfs.readOrdering.start(), indexCfs.metadata(), null, null, NO_SAMPLING);
+ /*
+ * TODO: this should perhaps not open and maintain a writeOp for the full duration, but instead only *try*
+ * to delete stale entries, without blocking if there's no room
+ * as it stands, we open a writeOp and keep it open for the duration to ensure that should this CF get flushed to make room we don't block the reclamation of any room being made
+ */
+ writeContext = baseCfs.keyspace.getWriteHandler().createContextForRead();
+ return new ReadExecutionController(command, baseOp, baseCfs.metadata(), indexController, writeContext, createdAtNanos);
}
- else
+ catch (RuntimeException e)
{
- OpOrder.Group baseOp = null, writeOp = null;
- ReadExecutionController indexController = null;
- // OpOrder.start() shouldn't fail, but better safe than sorry.
+ // Note that must have writeContext == null since ReadOrderGroup ctor can't fail
+ assert writeContext == null;
try
{
- baseOp = baseCfs.readOrdering.start();
- indexController = new ReadExecutionController(indexCfs.readOrdering.start(), indexCfs.metadata, null, null);
- // TODO: this should perhaps not open and maintain a writeOp for the full duration, but instead only *try* to delete stale entries, without blocking if there's no room
- // as it stands, we open a writeOp and keep it open for the duration to ensure that should this CF get flushed to make room we don't block the reclamation of any room being made
- writeOp = Keyspace.writeOrder.start();
- return new ReadExecutionController(baseOp, baseCfs.metadata, indexController, writeOp);
+ if (baseOp != null)
+ baseOp.close();
}
- catch (RuntimeException e)
+ finally
{
- // Note that must have writeOp == null since ReadOrderGroup ctor can't fail
- assert writeOp == null;
- try
- {
- if (baseOp != null)
- baseOp.close();
- }
- finally
- {
- if (indexController != null)
- indexController.close();
- }
- throw e;
+ if (indexController != null)
+ indexController.close();
}
+ throw e;
}
}
@@ -120,7 +140,7 @@
return index == null ? null : index.getBackingTable().orElse(null);
}
- public CFMetaData metaData()
+ public TableMetadata metadata()
{
return baseMetadata;
}
@@ -142,9 +162,21 @@
}
finally
{
- writeOp.close();
+ writeContext.close();
}
}
}
+
+ if (createdAtNanos != NO_SAMPLING)
+ addSample();
+ }
+
+ private void addSample()
+ {
+ String cql = command.toCQLString();
+ int timeMicros = (int) Math.min(TimeUnit.NANOSECONDS.toMicros(clock.now() - createdAtNanos), Integer.MAX_VALUE);
+ ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(baseMetadata.id);
+ if (cfs != null)
+ cfs.metric.topLocalReadQueryTime.addSample(cql, timeMicros);
}
}
diff --git a/src/java/org/apache/cassandra/db/ReadQuery.java b/src/java/org/apache/cassandra/db/ReadQuery.java
index 64aeb2a..fd94aa1 100644
--- a/src/java/org/apache/cassandra/db/ReadQuery.java
+++ b/src/java/org/apache/cassandra/db/ReadQuery.java
@@ -17,74 +17,113 @@
*/
package org.apache.cassandra.db;
+import org.apache.cassandra.db.filter.ColumnFilter;
import org.apache.cassandra.db.filter.DataLimits;
+import org.apache.cassandra.db.filter.RowFilter;
import org.apache.cassandra.db.partitions.*;
import org.apache.cassandra.exceptions.RequestExecutionException;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.service.ClientState;
import org.apache.cassandra.service.pager.QueryPager;
import org.apache.cassandra.service.pager.PagingState;
import org.apache.cassandra.transport.ProtocolVersion;
+import org.apache.cassandra.utils.FBUtilities;
/**
* Generic abstraction for read queries.
- * <p>
- * The main implementation of this is {@link ReadCommand} but we have this interface because
- * {@link SinglePartitionReadCommand.Group} is also consider as a "read query" but is not a
- * {@code ReadCommand}.
*/
public interface ReadQuery
{
- ReadQuery EMPTY = new ReadQuery()
+ public static ReadQuery empty(final TableMetadata metadata)
{
- public ReadExecutionController executionController()
+ return new ReadQuery()
{
- return ReadExecutionController.empty();
- }
+ public TableMetadata metadata()
+ {
+ return metadata;
+ }
- public PartitionIterator execute(ConsistencyLevel consistency, ClientState clientState, long queryStartNanoTime) throws RequestExecutionException
- {
- return EmptyIterators.partition();
- }
+ public ReadExecutionController executionController()
+ {
+ return ReadExecutionController.empty();
+ }
- public PartitionIterator executeInternal(ReadExecutionController controller)
- {
- return EmptyIterators.partition();
- }
+ public PartitionIterator execute(ConsistencyLevel consistency, ClientState clientState, long queryStartNanoTime) throws RequestExecutionException
+ {
+ return EmptyIterators.partition();
+ }
- public UnfilteredPartitionIterator executeLocally(ReadExecutionController executionController)
- {
- return EmptyIterators.unfilteredPartition(executionController.metaData(), false);
- }
+ public PartitionIterator executeInternal(ReadExecutionController controller)
+ {
+ return EmptyIterators.partition();
+ }
- public DataLimits limits()
- {
- // What we return here doesn't matter much in practice. However, returning DataLimits.NONE means
- // "no particular limit", which makes SelectStatement.execute() take the slightly more complex "paging"
- // path. Not a big deal but it's easy enough to return a limit of 0 rows which avoids this.
- return DataLimits.cqlLimits(0);
- }
+ public UnfilteredPartitionIterator executeLocally(ReadExecutionController executionController)
+ {
+ return EmptyIterators.unfilteredPartition(executionController.metadata());
+ }
- public QueryPager getPager(PagingState state, ProtocolVersion protocolVersion)
- {
- return QueryPager.EMPTY;
- }
+ public DataLimits limits()
+ {
+ // What we return here doesn't matter much in practice. However, returning DataLimits.NONE means
+ // "no particular limit", which makes SelectStatement.execute() take the slightly more complex "paging"
+ // path. Not a big deal but it's easy enough to return a limit of 0 rows which avoids this.
+ return DataLimits.cqlLimits(0);
+ }
- public boolean selectsKey(DecoratedKey key)
- {
- return false;
- }
+ public QueryPager getPager(PagingState state, ProtocolVersion protocolVersion)
+ {
+ return QueryPager.EMPTY;
+ }
- public boolean selectsClustering(DecoratedKey key, Clustering clustering)
- {
- return false;
- }
+ public boolean selectsKey(DecoratedKey key)
+ {
+ return false;
+ }
- @Override
- public boolean selectsFullPartition()
- {
- return false;
- }
- };
+ public boolean selectsClustering(DecoratedKey key, Clustering clustering)
+ {
+ return false;
+ }
+
+ @Override
+ public int nowInSec()
+ {
+ return FBUtilities.nowInSeconds();
+ }
+
+ @Override
+ public boolean selectsFullPartition()
+ {
+ return false;
+ }
+
+ @Override
+ public boolean isEmpty()
+ {
+ return true;
+ }
+
+ @Override
+ public RowFilter rowFilter()
+ {
+ return RowFilter.NONE;
+ }
+
+ @Override
+ public ColumnFilter columnFilter()
+ {
+ return ColumnFilter.NONE;
+ }
+ };
+ }
+
+ /**
+ * The metadata for the table this is a query on.
+ *
+ * @return the metadata for the table this is a query on.
+ */
+ public TableMetadata metadata();
/**
* Starts a new read operation.
@@ -120,7 +159,7 @@
* Execute the query locally. This is similar to {@link ReadQuery#executeInternal(ReadExecutionController)}
* but it returns an unfiltered partition iterator that can be merged later on.
*
- * @param controller the {@code ReadExecutionController} protecting the read.
+ * @param executionController the {@code ReadExecutionController} protecting the read.
* @return the result of the read query.
*/
public UnfilteredPartitionIterator executeLocally(ReadExecutionController executionController);
@@ -156,8 +195,63 @@
public boolean selectsClustering(DecoratedKey key, Clustering clustering);
/**
+ * The time in seconds to use as "now" for this query.
+ * <p>
+ * We use the same time as "now" for the whole query to avoid considering different
+ * values as expired during the query, which would be buggy (would throw of counting amongst other
+ * things).
+ *
+ * @return the time (in seconds) to use as "now".
+ */
+ public int nowInSec();
+
+ /**
* Checks if this {@code ReadQuery} selects full partitions, that is it has no filtering on clustering or regular columns.
* @return {@code true} if this {@code ReadQuery} selects full partitions, {@code false} otherwise.
*/
public boolean selectsFullPartition();
+
+ /**
+ * Filters/Resrictions on CQL rows.
+ * <p>
+ * This contains the restrictions that are not directly handled by the
+ * {@code ClusteringIndexFilter}. More specifically, this includes any non-PK column
+ * restrictions and can include some PK columns restrictions when those can't be
+ * satisfied entirely by the clustering index filter (because not all clustering columns
+ * have been restricted for instance). If there is 2ndary indexes on the table,
+ * one of this restriction might be handled by a 2ndary index.
+ *
+ * @return the filter holding the expression that rows must satisfy.
+ */
+ public RowFilter rowFilter();
+
+ /**
+ * A filter on which (non-PK) columns must be returned by the query.
+ *
+ * @return which columns must be fetched by this query.
+ */
+ public ColumnFilter columnFilter();
+
+ /**
+ * Whether this query is known to return nothing upfront.
+ * <p>
+ * This is overridden by the {@code ReadQuery} created through {@link #empty(TableMetadata)}, and that's probably the
+ * only place that should override it.
+ *
+ * @return if this method is guaranteed to return no results whatsoever.
+ */
+ public default boolean isEmpty()
+ {
+ return false;
+ }
+
+ /**
+ * If the index manager for the table determines that there's an applicable
+ * 2i that can be used to execute this query, call its (optional)
+ * validation method to check that nothing in this query's parameters
+ * violates the implementation specific validation rules.
+ */
+ default void maybeValidateIndex()
+ {
+ }
}
diff --git a/src/java/org/apache/cassandra/db/ReadRepairVerbHandler.java b/src/java/org/apache/cassandra/db/ReadRepairVerbHandler.java
index 2e499e7..903b3d4 100644
--- a/src/java/org/apache/cassandra/db/ReadRepairVerbHandler.java
+++ b/src/java/org/apache/cassandra/db/ReadRepairVerbHandler.java
@@ -18,14 +18,16 @@
package org.apache.cassandra.db;
import org.apache.cassandra.net.IVerbHandler;
-import org.apache.cassandra.net.MessageIn;
+import org.apache.cassandra.net.Message;
import org.apache.cassandra.net.MessagingService;
public class ReadRepairVerbHandler implements IVerbHandler<Mutation>
{
- public void doVerb(MessageIn<Mutation> message, int id)
+ public static final ReadRepairVerbHandler instance = new ReadRepairVerbHandler();
+
+ public void doVerb(Message<Mutation> message)
{
message.payload.apply();
- MessagingService.instance().sendReply(WriteResponse.createMessage(), id, message.from);
+ MessagingService.instance().send(message.emptyResponse(), message.from());
}
}
diff --git a/src/java/org/apache/cassandra/db/ReadResponse.java b/src/java/org/apache/cassandra/db/ReadResponse.java
index 7aa915e..3f6481d 100644
--- a/src/java/org/apache/cassandra/db/ReadResponse.java
+++ b/src/java/org/apache/cassandra/db/ReadResponse.java
@@ -19,56 +19,28 @@
import java.io.*;
import java.nio.ByteBuffer;
-import java.security.MessageDigest;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
import com.google.common.annotations.VisibleForTesting;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.db.filter.ClusteringIndexFilter;
import org.apache.cassandra.db.filter.ColumnFilter;
-import org.apache.cassandra.db.rows.*;
import org.apache.cassandra.db.partitions.*;
-import org.apache.cassandra.dht.*;
-import org.apache.cassandra.io.ForwardingVersionedSerializer;
+import org.apache.cassandra.db.rows.*;
import org.apache.cassandra.io.IVersionedSerializer;
import org.apache.cassandra.io.util.DataInputBuffer;
import org.apache.cassandra.io.util.DataInputPlus;
-import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.io.util.DataOutputBuffer;
+import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.net.MessagingService;
-import org.apache.cassandra.thrift.ThriftResultsMerger;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.utils.ByteBufferUtil;
-import org.apache.cassandra.utils.FBUtilities;
public abstract class ReadResponse
{
// Serializer for single partition read response
public static final IVersionedSerializer<ReadResponse> serializer = new Serializer();
- // Serializer for the pre-3.0 rang slice responses.
- public static final IVersionedSerializer<ReadResponse> legacyRangeSliceReplySerializer = new LegacyRangeSliceReplySerializer();
- // Serializer for partition range read response (this actually delegate to 'serializer' in 3.0 and to
- // 'legacyRangeSliceReplySerializer' in older version.
- public static final IVersionedSerializer<ReadResponse> rangeSliceSerializer = new ForwardingVersionedSerializer<ReadResponse>()
- {
- @Override
- protected IVersionedSerializer<ReadResponse> delegate(int version)
- {
- return version < MessagingService.VERSION_30
- ? legacyRangeSliceReplySerializer
- : serializer;
- }
- };
- // This is used only when serializing data responses and we can't it easily in other cases. So this can be null, which is slighly
- // hacky, but as this hack doesn't escape this class, and it's easy enough to validate that it's not null when we need, it's "good enough".
- private final ReadCommand command;
-
- protected ReadResponse(ReadCommand command)
+ protected ReadResponse()
{
- this.command = command;
}
public static ReadResponse createDataResponse(UnfilteredPartitionIterator data, ReadCommand command)
@@ -77,11 +49,19 @@
}
@VisibleForTesting
- public static ReadResponse createRemoteDataResponse(UnfilteredPartitionIterator data, ReadCommand command)
+ public static ReadResponse createRemoteDataResponse(UnfilteredPartitionIterator data,
+ ByteBuffer repairedDataDigest,
+ boolean isRepairedDigestConclusive,
+ ReadCommand command,
+ int version)
{
- return new RemoteDataResponse(LocalDataResponse.build(data, command.columnFilter()));
+ return new RemoteDataResponse(LocalDataResponse.build(data, command.columnFilter()),
+ repairedDataDigest,
+ isRepairedDigestConclusive,
+ version);
}
+
public static ReadResponse createDigestResponse(UnfilteredPartitionIterator data, ReadCommand command)
{
return new DigestResponse(makeDigest(data, command));
@@ -89,6 +69,9 @@
public abstract UnfilteredPartitionIterator makeIterator(ReadCommand command);
public abstract ByteBuffer digest(ReadCommand command);
+ public abstract ByteBuffer repairedDataDigest();
+ public abstract boolean isRepairedDigestConclusive();
+ public abstract boolean mayIncludeRepairedDigest();
public abstract boolean isDigestResponse();
@@ -111,19 +94,22 @@
}
}
}
- return "<key " + key + " not found>";
+ return String.format("<key %s not found (repaired_digest=%s repaired_digest_conclusive=%s)>",
+ key, ByteBufferUtil.bytesToHex(repairedDataDigest()), isRepairedDigestConclusive());
}
- private String toDebugString(UnfilteredRowIterator partition, CFMetaData metadata)
+ private String toDebugString(UnfilteredRowIterator partition, TableMetadata metadata)
{
StringBuilder sb = new StringBuilder();
- sb.append(String.format("[%s.%s] key=%s partition_deletion=%s columns=%s",
- metadata.ksName,
- metadata.cfName,
- metadata.getKeyValidator().getString(partition.partitionKey().getKey()),
+ sb.append(String.format("[%s] key=%s partition_deletion=%s columns=%s repaired_digest=%s repaired_digest_conclusive==%s",
+ metadata,
+ metadata.partitionKeyType.getString(partition.partitionKey().getKey()),
partition.partitionLevelDeletion(),
- partition.columns()));
+ partition.columns(),
+ ByteBufferUtil.bytesToHex(repairedDataDigest()),
+ isRepairedDigestConclusive()
+ ));
if (partition.staticRow() != Rows.EMPTY_STATIC_ROW)
sb.append("\n ").append(partition.staticRow().toString(metadata, true));
@@ -136,8 +122,8 @@
protected static ByteBuffer makeDigest(UnfilteredPartitionIterator iterator, ReadCommand command)
{
- MessageDigest digest = FBUtilities.threadLocalMD5Digest();
- UnfilteredPartitionIterators.digest(command, iterator, digest, command.digestVersion());
+ Digest digest = Digest.forReadResponse();
+ UnfilteredPartitionIterators.digest(iterator, digest, command.digestVersion());
return ByteBuffer.wrap(digest.digest());
}
@@ -147,7 +133,7 @@
private DigestResponse(ByteBuffer digest)
{
- super(null);
+ super();
assert digest.hasRemaining();
this.digest = digest;
}
@@ -157,6 +143,21 @@
throw new UnsupportedOperationException();
}
+ public boolean mayIncludeRepairedDigest()
+ {
+ return false;
+ }
+
+ public ByteBuffer repairedDataDigest()
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ public boolean isRepairedDigestConclusive()
+ {
+ throw new UnsupportedOperationException();
+ }
+
public ByteBuffer digest(ReadCommand command)
{
// We assume that the digest is in the proper version, which bug excluded should be true since this is called with
@@ -177,7 +178,11 @@
{
private LocalDataResponse(UnfilteredPartitionIterator iter, ReadCommand command)
{
- super(command, build(iter, command.columnFilter()), SerializationHelper.Flag.LOCAL);
+ super(build(iter, command.columnFilter()),
+ command.getRepairedDataDigest(),
+ command.isRepairedDataDigestConclusive(),
+ MessagingService.current_version,
+ DeserializationHelper.Flag.LOCAL);
}
private static ByteBuffer build(UnfilteredPartitionIterator iter, ColumnFilter selection)
@@ -198,9 +203,12 @@
// built on the coordinator node receiving a response
private static class RemoteDataResponse extends DataResponse
{
- protected RemoteDataResponse(ByteBuffer data)
+ protected RemoteDataResponse(ByteBuffer data,
+ ByteBuffer repairedDataDigest,
+ boolean isRepairedDigestConclusive,
+ int version)
{
- super(null, data, SerializationHelper.Flag.FROM_REMOTE);
+ super(data, repairedDataDigest, isRepairedDigestConclusive, version, DeserializationHelper.Flag.FROM_REMOTE);
}
}
@@ -209,12 +217,22 @@
// TODO: can the digest be calculated over the raw bytes now?
// The response, serialized in the current messaging version
private final ByteBuffer data;
- private final SerializationHelper.Flag flag;
+ private final ByteBuffer repairedDataDigest;
+ private final boolean isRepairedDigestConclusive;
+ private final int dataSerializationVersion;
+ private final DeserializationHelper.Flag flag;
- protected DataResponse(ReadCommand command, ByteBuffer data, SerializationHelper.Flag flag)
+ protected DataResponse(ByteBuffer data,
+ ByteBuffer repairedDataDigest,
+ boolean isRepairedDigestConclusive,
+ int dataSerializationVersion,
+ DeserializationHelper.Flag flag)
{
- super(command);
+ super();
this.data = data;
+ this.repairedDataDigest = repairedDataDigest;
+ this.isRepairedDigestConclusive = isRepairedDigestConclusive;
+ this.dataSerializationVersion = dataSerializationVersion;
this.flag = flag;
}
@@ -226,7 +244,7 @@
// the later can be null (for RemoteDataResponse as those are created in the serializers and
// those don't have easy access to the command). This is also why we need the command as parameter here.
return UnfilteredPartitionIterators.serializerForIntraNode().deserialize(in,
- MessagingService.current_version,
+ dataSerializationVersion,
command.metadata(),
command.columnFilter(),
flag);
@@ -238,101 +256,19 @@
}
}
- public ByteBuffer digest(ReadCommand command)
+ public boolean mayIncludeRepairedDigest()
{
- try (UnfilteredPartitionIterator iterator = makeIterator(command))
- {
- return makeDigest(iterator, command);
- }
+ return dataSerializationVersion >= MessagingService.VERSION_40;
}
- public boolean isDigestResponse()
+ public ByteBuffer repairedDataDigest()
{
- return false;
- }
- }
-
- /**
- * A remote response from a pre-3.0 node. This needs a separate class in order to cleanly handle trimming and
- * reversal of results when the read command calls for it. Pre-3.0 nodes always return results in the normal
- * sorted order, even if the query asks for reversed results. Additionally, pre-3.0 nodes do not have a notion of
- * exclusive slices on non-composite tables, so extra rows may need to be trimmed.
- */
- @VisibleForTesting
- static class LegacyRemoteDataResponse extends ReadResponse
- {
- private final List<ImmutableBTreePartition> partitions;
-
- @VisibleForTesting
- LegacyRemoteDataResponse(List<ImmutableBTreePartition> partitions)
- {
- super(null); // we never serialize LegacyRemoteDataResponses, so we don't care about the command
- this.partitions = partitions;
+ return repairedDataDigest;
}
- public UnfilteredPartitionIterator makeIterator(final ReadCommand command)
+ public boolean isRepairedDigestConclusive()
{
- // Due to a bug in the serialization of AbstractBounds, anything that isn't a Range is understood by pre-3.0 nodes
- // as a Bound, which means IncludingExcludingBounds and ExcludingBounds responses may include keys they shouldn't.
- // So filter partitions that shouldn't be included here.
- boolean skipFirst = false;
- boolean skipLast = false;
- if (!partitions.isEmpty() && command instanceof PartitionRangeReadCommand)
- {
- AbstractBounds<PartitionPosition> keyRange = ((PartitionRangeReadCommand)command).dataRange().keyRange();
- boolean isExcludingBounds = keyRange instanceof ExcludingBounds;
- skipFirst = isExcludingBounds && !keyRange.contains(partitions.get(0).partitionKey());
- skipLast = (isExcludingBounds || keyRange instanceof IncludingExcludingBounds) && !keyRange.contains(partitions.get(partitions.size() - 1).partitionKey());
- }
-
- final List<ImmutableBTreePartition> toReturn;
- if (skipFirst || skipLast)
- {
- toReturn = partitions.size() == 1
- ? Collections.emptyList()
- : partitions.subList(skipFirst ? 1 : 0, skipLast ? partitions.size() - 1 : partitions.size());
- }
- else
- {
- toReturn = partitions;
- }
-
- return new AbstractUnfilteredPartitionIterator()
- {
- private int idx;
-
- public boolean isForThrift()
- {
- return true;
- }
-
- public CFMetaData metadata()
- {
- return command.metadata();
- }
-
- public boolean hasNext()
- {
- return idx < toReturn.size();
- }
-
- public UnfilteredRowIterator next()
- {
- ImmutableBTreePartition partition = toReturn.get(idx++);
-
- ClusteringIndexFilter filter = command.clusteringIndexFilter(partition.partitionKey());
-
- // Pre-3.0, we would always request one more row than we actually needed and the command-level "start" would
- // be the last-returned cell name, so the response would always include it.
- UnfilteredRowIterator iterator = partition.unfilteredIterator(command.columnFilter(), filter.getSlices(command.metadata()), filter.isReversed());
-
- // Wrap results with a ThriftResultMerger only if they're intended for the thrift command.
- if (command.isForThrift())
- return ThriftResultsMerger.maybeWrap(iterator, command.nowInSec());
- else
- return iterator;
- }
- };
+ return isRepairedDigestConclusive;
}
public ByteBuffer digest(ReadCommand command)
@@ -355,31 +291,25 @@
{
boolean isDigest = response instanceof DigestResponse;
ByteBuffer digest = isDigest ? ((DigestResponse)response).digest : ByteBufferUtil.EMPTY_BYTE_BUFFER;
- if (version < MessagingService.VERSION_30)
- {
- out.writeInt(digest.remaining());
- out.write(digest);
- out.writeBoolean(isDigest);
- if (!isDigest)
- {
- assert response.command != null; // we only serialize LocalDataResponse, which always has the command set
- try (UnfilteredPartitionIterator iter = response.makeIterator(response.command))
- {
- assert iter.hasNext();
- try (UnfilteredRowIterator partition = iter.next())
- {
- ByteBufferUtil.writeWithShortLength(partition.partitionKey().getKey(), out);
- LegacyLayout.serializeAsLegacyPartition(response.command, partition, out, version);
- }
- assert !iter.hasNext();
- }
- }
- return;
- }
-
ByteBufferUtil.writeWithVIntLength(digest, out);
if (!isDigest)
{
+ // From 4.0, a coordinator may request additional info about the repaired data that
+ // makes up the response, namely a digest generated from the repaired data and a
+ // flag indicating our level of confidence in that digest. The digest may be considered
+ // inconclusive if it may have been affected by some unrepaired data during read.
+ // e.g. some sstables read during this read were involved in pending but not yet
+ // committed repair sessions or an unrepaired partition tombstone meant that not all
+ // repaired sstables were read (but they might be on other replicas).
+ // If the coordinator did not request this info, the response contains an empty digest
+ // and a true for the isConclusive flag.
+ // If the messaging version is < 4.0, these are omitted altogether.
+ if (version >= MessagingService.VERSION_40)
+ {
+ ByteBufferUtil.writeWithVIntLength(response.repairedDataDigest(), out);
+ out.writeBoolean(response.isRepairedDigestConclusive());
+ }
+
ByteBuffer data = ((DataResponse)response).data;
ByteBufferUtil.writeWithVIntLength(data, out);
}
@@ -387,73 +317,46 @@
public ReadResponse deserialize(DataInputPlus in, int version) throws IOException
{
- if (version < MessagingService.VERSION_30)
- {
- byte[] digest = null;
- int digestSize = in.readInt();
- if (digestSize > 0)
- {
- digest = new byte[digestSize];
- in.readFully(digest, 0, digestSize);
- }
- boolean isDigest = in.readBoolean();
- assert isDigest == digestSize > 0;
- if (isDigest)
- {
- assert digest != null;
- return new DigestResponse(ByteBuffer.wrap(digest));
- }
-
- // ReadResponses from older versions are always single-partition (ranges are handled by RangeSliceReply)
- ByteBuffer key = ByteBufferUtil.readWithShortLength(in);
- try (UnfilteredRowIterator rowIterator = LegacyLayout.deserializeLegacyPartition(in, version, SerializationHelper.Flag.FROM_REMOTE, key))
- {
- if (rowIterator == null)
- return new LegacyRemoteDataResponse(Collections.emptyList());
-
- return new LegacyRemoteDataResponse(Collections.singletonList(ImmutableBTreePartition.create(rowIterator)));
- }
- }
-
ByteBuffer digest = ByteBufferUtil.readWithVIntLength(in);
if (digest.hasRemaining())
return new DigestResponse(digest);
- assert version >= MessagingService.VERSION_30;
+ // A data response may also contain a digest of the portion of its payload
+ // that comes from the replica's repaired set, along with a flag indicating
+ // whether or not the digest may be influenced by unrepaired/pending
+ // repaired data
+ boolean repairedDigestConclusive;
+ if (version >= MessagingService.VERSION_40)
+ {
+ digest = ByteBufferUtil.readWithVIntLength(in);
+ repairedDigestConclusive = in.readBoolean();
+ }
+ else
+ {
+ digest = ByteBufferUtil.EMPTY_BYTE_BUFFER;
+ repairedDigestConclusive = true;
+ }
+
ByteBuffer data = ByteBufferUtil.readWithVIntLength(in);
- return new RemoteDataResponse(data);
+ return new RemoteDataResponse(data, digest, repairedDigestConclusive, version);
}
public long serializedSize(ReadResponse response, int version)
{
boolean isDigest = response instanceof DigestResponse;
ByteBuffer digest = isDigest ? ((DigestResponse)response).digest : ByteBufferUtil.EMPTY_BYTE_BUFFER;
-
- if (version < MessagingService.VERSION_30)
- {
- long size = TypeSizes.sizeof(digest.remaining())
- + digest.remaining()
- + TypeSizes.sizeof(isDigest);
- if (!isDigest)
- {
- assert response.command != null; // we only serialize LocalDataResponse, which always has the command set
- try (UnfilteredPartitionIterator iter = response.makeIterator(response.command))
- {
- assert iter.hasNext();
- try (UnfilteredRowIterator partition = iter.next())
- {
- size += ByteBufferUtil.serializedSizeWithShortLength(partition.partitionKey().getKey());
- size += LegacyLayout.serializedSizeAsLegacyPartition(response.command, partition, version);
- }
- assert !iter.hasNext();
- }
- }
- return size;
- }
-
long size = ByteBufferUtil.serializedSizeWithVIntLength(digest);
+
if (!isDigest)
{
+ // From 4.0, a coordinator may request an additional info about the repaired data
+ // that makes up the response.
+ if (version >= MessagingService.VERSION_40)
+ {
+ size += ByteBufferUtil.serializedSizeWithVIntLength(response.repairedDataDigest());
+ size += 1;
+ }
+
// In theory, we should deserialize/re-serialize if the version asked is different from the current
// version as the content could have a different serialization format. So far though, we haven't made
// change to partition iterators serialization since 3.0 so we skip this.
@@ -464,81 +367,4 @@
return size;
}
}
-
- private static class LegacyRangeSliceReplySerializer implements IVersionedSerializer<ReadResponse>
- {
- public void serialize(ReadResponse response, DataOutputPlus out, int version) throws IOException
- {
- assert version < MessagingService.VERSION_30;
-
- // determine the number of partitions upfront for serialization
- int numPartitions = 0;
- assert response.command != null; // we only serialize LocalDataResponse, which always has the command set
- try (UnfilteredPartitionIterator iterator = response.makeIterator(response.command))
- {
- while (iterator.hasNext())
- {
- try (UnfilteredRowIterator atomIterator = iterator.next())
- {
- numPartitions++;
-
- // we have to fully exhaust the subiterator
- while (atomIterator.hasNext())
- atomIterator.next();
- }
- }
- }
-
- out.writeInt(numPartitions);
-
- try (UnfilteredPartitionIterator iterator = response.makeIterator(response.command))
- {
- while (iterator.hasNext())
- {
- try (UnfilteredRowIterator partition = iterator.next())
- {
- ByteBufferUtil.writeWithShortLength(partition.partitionKey().getKey(), out);
- LegacyLayout.serializeAsLegacyPartition(response.command, partition, out, version);
- }
- }
- }
- }
-
- public ReadResponse deserialize(DataInputPlus in, int version) throws IOException
- {
- assert version < MessagingService.VERSION_30;
-
- int partitionCount = in.readInt();
- ArrayList<ImmutableBTreePartition> partitions = new ArrayList<>(partitionCount);
- for (int i = 0; i < partitionCount; i++)
- {
- ByteBuffer key = ByteBufferUtil.readWithShortLength(in);
- try (UnfilteredRowIterator partition = LegacyLayout.deserializeLegacyPartition(in, version, SerializationHelper.Flag.FROM_REMOTE, key))
- {
- partitions.add(ImmutableBTreePartition.create(partition));
- }
- }
- return new LegacyRemoteDataResponse(partitions);
- }
-
- public long serializedSize(ReadResponse response, int version)
- {
- assert version < MessagingService.VERSION_30;
- long size = TypeSizes.sizeof(0); // number of partitions
-
- assert response.command != null; // we only serialize LocalDataResponse, which always has the command set
- try (UnfilteredPartitionIterator iterator = response.makeIterator(response.command))
- {
- while (iterator.hasNext())
- {
- try (UnfilteredRowIterator partition = iterator.next())
- {
- size += ByteBufferUtil.serializedSizeWithShortLength(partition.partitionKey().getKey());
- size += LegacyLayout.serializedSizeAsLegacyPartition(response.command, partition, version);
- }
- }
- }
- return size;
- }
- }
}
diff --git a/src/java/org/apache/cassandra/db/RegularAndStaticColumns.java b/src/java/org/apache/cassandra/db/RegularAndStaticColumns.java
new file mode 100644
index 0000000..fab7730
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/RegularAndStaticColumns.java
@@ -0,0 +1,194 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db;
+
+import java.util.*;
+
+import com.google.common.collect.Iterators;
+
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.utils.btree.BTreeSet;
+
+import static java.util.Comparator.naturalOrder;
+
+/**
+ * Columns (or a subset of the columns) that a partition contains.
+ * This mainly groups both static and regular columns for convenience.
+ */
+public class RegularAndStaticColumns implements Iterable<ColumnMetadata>
+{
+ public static RegularAndStaticColumns NONE = new RegularAndStaticColumns(Columns.NONE, Columns.NONE);
+
+ public final Columns statics;
+ public final Columns regulars;
+
+ public RegularAndStaticColumns(Columns statics, Columns regulars)
+ {
+ assert statics != null && regulars != null;
+ this.statics = statics;
+ this.regulars = regulars;
+ }
+
+ public static RegularAndStaticColumns of(ColumnMetadata column)
+ {
+ return new RegularAndStaticColumns(column.isStatic() ? Columns.of(column) : Columns.NONE,
+ column.isStatic() ? Columns.NONE : Columns.of(column));
+ }
+
+ public RegularAndStaticColumns without(ColumnMetadata column)
+ {
+ return new RegularAndStaticColumns(column.isStatic() ? statics.without(column) : statics,
+ column.isStatic() ? regulars : regulars.without(column));
+ }
+
+ public RegularAndStaticColumns mergeTo(RegularAndStaticColumns that)
+ {
+ if (this == that)
+ return this;
+ Columns statics = this.statics.mergeTo(that.statics);
+ Columns regulars = this.regulars.mergeTo(that.regulars);
+ if (statics == this.statics && regulars == this.regulars)
+ return this;
+ if (statics == that.statics && regulars == that.regulars)
+ return that;
+ return new RegularAndStaticColumns(statics, regulars);
+ }
+
+ public boolean isEmpty()
+ {
+ return statics.isEmpty() && regulars.isEmpty();
+ }
+
+ public Columns columns(boolean isStatic)
+ {
+ return isStatic ? statics : regulars;
+ }
+
+ public boolean contains(ColumnMetadata column)
+ {
+ return column.isStatic() ? statics.contains(column) : regulars.contains(column);
+ }
+
+ public boolean includes(RegularAndStaticColumns columns)
+ {
+ return statics.containsAll(columns.statics) && regulars.containsAll(columns.regulars);
+ }
+
+ public Iterator<ColumnMetadata> iterator()
+ {
+ return Iterators.concat(statics.iterator(), regulars.iterator());
+ }
+
+ public Iterator<ColumnMetadata> selectOrderIterator()
+ {
+ return Iterators.concat(statics.selectOrderIterator(), regulars.selectOrderIterator());
+ }
+
+ /** * Returns the total number of static and regular columns. */
+ public int size()
+ {
+ return regulars.size() + statics.size();
+ }
+
+ @Override
+ public String toString()
+ {
+ StringBuilder sb = new StringBuilder();
+ sb.append("[").append(statics).append(" | ").append(regulars).append("]");
+ return sb.toString();
+ }
+
+ @Override
+ public boolean equals(Object other)
+ {
+ if (!(other instanceof RegularAndStaticColumns))
+ return false;
+
+ RegularAndStaticColumns that = (RegularAndStaticColumns)other;
+ return this.statics.equals(that.statics)
+ && this.regulars.equals(that.regulars);
+ }
+
+ @Override
+ public int hashCode()
+ {
+ return Objects.hash(statics, regulars);
+ }
+
+ public static Builder builder()
+ {
+ return new Builder();
+ }
+
+ public static class Builder
+ {
+ // Note that we do want to use sorted sets because we want the column definitions to be compared
+ // through compareTo, not equals. The former basically check it's the same column name, while the latter
+ // check it's the same object, including the same type.
+ private BTreeSet.Builder<ColumnMetadata> regularColumns;
+ private BTreeSet.Builder<ColumnMetadata> staticColumns;
+
+ public Builder add(ColumnMetadata c)
+ {
+ if (c.isStatic())
+ {
+ if (staticColumns == null)
+ staticColumns = BTreeSet.builder(naturalOrder());
+ staticColumns.add(c);
+ }
+ else
+ {
+ assert c.isRegular();
+ if (regularColumns == null)
+ regularColumns = BTreeSet.builder(naturalOrder());
+ regularColumns.add(c);
+ }
+ return this;
+ }
+
+ public Builder addAll(Iterable<ColumnMetadata> columns)
+ {
+ for (ColumnMetadata c : columns)
+ add(c);
+ return this;
+ }
+
+ public Builder addAll(RegularAndStaticColumns columns)
+ {
+ if (regularColumns == null && !columns.regulars.isEmpty())
+ regularColumns = BTreeSet.builder(naturalOrder());
+
+ for (ColumnMetadata c : columns.regulars)
+ regularColumns.add(c);
+
+ if (staticColumns == null && !columns.statics.isEmpty())
+ staticColumns = BTreeSet.builder(naturalOrder());
+
+ for (ColumnMetadata c : columns.statics)
+ staticColumns.add(c);
+
+ return this;
+ }
+
+ public RegularAndStaticColumns build()
+ {
+ return new RegularAndStaticColumns(staticColumns == null ? Columns.NONE : Columns.from(staticColumns.build()),
+ regularColumns == null ? Columns.NONE : Columns.from(regularColumns.build()));
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/db/RepairedDataInfo.java b/src/java/org/apache/cassandra/db/RepairedDataInfo.java
new file mode 100644
index 0000000..c136f26
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/RepairedDataInfo.java
@@ -0,0 +1,326 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db;
+
+import java.nio.ByteBuffer;
+import java.util.function.LongPredicate;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.cassandra.db.filter.DataLimits;
+import org.apache.cassandra.db.partitions.PurgeFunction;
+import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
+import org.apache.cassandra.db.rows.RangeTombstoneMarker;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.db.transform.MoreRows;
+import org.apache.cassandra.db.transform.Transformation;
+import org.apache.cassandra.metrics.TableMetrics;
+import org.apache.cassandra.tracing.Tracing;
+import org.apache.cassandra.utils.ByteBufferUtil;
+
+class RepairedDataInfo
+{
+ public static final RepairedDataInfo NULL_REPAIRED_DATA_INFO = new RepairedDataInfo(null)
+ {
+ boolean isConclusive(){ return true; }
+ ByteBuffer getDigest(){ return ByteBufferUtil.EMPTY_BYTE_BUFFER; }
+ };
+
+ // Keeps a digest of the partition currently being processed. Since we won't know
+ // whether a partition will be fully purged from a read result until it's been
+ // consumed, we buffer this per-partition digest and add it to the final digest
+ // when the partition is closed (if it wasn't fully purged).
+ private Digest perPartitionDigest;
+ private Digest perCommandDigest;
+ private boolean isConclusive = true;
+ private ByteBuffer calculatedDigest = null;
+
+ // Doesn't actually purge from the underlying iterators, but excludes from the digest
+ // the purger can't be initialized until we've iterated all the sstables for the query
+ // as it requires the oldest repaired tombstone
+ private RepairedDataPurger purger;
+ private boolean isFullyPurged = true;
+
+ // Supplies additional partitions from the repaired data set to be consumed when the limit of
+ // executing ReadCommand has been reached. This is to ensure that each replica attempts to
+ // read the same amount of repaired data, otherwise comparisons of the repaired data digests
+ // may be invalidated by varying amounts of repaired data being present on each replica.
+ // This can't be initialized until after the underlying repaired iterators have been merged.
+ private UnfilteredPartitionIterator postLimitPartitions = null;
+ private final DataLimits.Counter repairedCounter;
+ private UnfilteredRowIterator currentPartition;
+ private TableMetrics metrics;
+
+ public RepairedDataInfo(DataLimits.Counter repairedCounter)
+ {
+ this.repairedCounter = repairedCounter;
+ }
+
+ ByteBuffer getDigest()
+ {
+ if (calculatedDigest != null)
+ return calculatedDigest;
+
+ calculatedDigest = perCommandDigest == null
+ ? ByteBufferUtil.EMPTY_BYTE_BUFFER
+ : ByteBuffer.wrap(perCommandDigest.digest());
+
+ return calculatedDigest;
+ }
+
+ void prepare(ColumnFamilyStore cfs, int nowInSec, int oldestUnrepairedTombstone)
+ {
+ this.purger = new RepairedDataPurger(cfs, nowInSec, oldestUnrepairedTombstone);
+ this.metrics = cfs.metric;
+ }
+
+ void finalize(UnfilteredPartitionIterator postLimitPartitions)
+ {
+ this.postLimitPartitions = postLimitPartitions;
+ }
+
+ boolean isConclusive()
+ {
+ return isConclusive;
+ }
+
+ void markInconclusive()
+ {
+ isConclusive = false;
+ }
+
+ private void onNewPartition(UnfilteredRowIterator partition)
+ {
+ assert purger != null;
+ purger.setCurrentKey(partition.partitionKey());
+ purger.setIsReverseOrder(partition.isReverseOrder());
+ this.currentPartition = partition;
+ }
+
+ private Digest getPerPartitionDigest()
+ {
+ if (perPartitionDigest == null)
+ perPartitionDigest = Digest.forRepairedDataTracking();
+
+ return perPartitionDigest;
+ }
+
+ public UnfilteredPartitionIterator withRepairedDataInfo(final UnfilteredPartitionIterator iterator)
+ {
+ class WithTracking extends Transformation<UnfilteredRowIterator>
+ {
+ protected UnfilteredRowIterator applyToPartition(UnfilteredRowIterator partition)
+ {
+ return withRepairedDataInfo(partition);
+ }
+ }
+ return Transformation.apply(iterator, new WithTracking());
+ }
+
+ public UnfilteredRowIterator withRepairedDataInfo(final UnfilteredRowIterator iterator)
+ {
+ class WithTracking extends Transformation<UnfilteredRowIterator>
+ {
+ protected DecoratedKey applyToPartitionKey(DecoratedKey key)
+ {
+ getPerPartitionDigest().update(key.getKey());
+ return key;
+ }
+
+ protected DeletionTime applyToDeletion(DeletionTime deletionTime)
+ {
+ if (repairedCounter.isDone())
+ return deletionTime;
+
+ assert purger != null;
+ DeletionTime purged = purger.applyToDeletion(deletionTime);
+ if (!purged.isLive())
+ isFullyPurged = false;
+ purged.digest(getPerPartitionDigest());
+ return deletionTime;
+ }
+
+ protected RangeTombstoneMarker applyToMarker(RangeTombstoneMarker marker)
+ {
+ if (repairedCounter.isDone())
+ return marker;
+
+ assert purger != null;
+ RangeTombstoneMarker purged = purger.applyToMarker(marker);
+ if (purged != null)
+ {
+ isFullyPurged = false;
+ purged.digest(getPerPartitionDigest());
+ }
+ return marker;
+ }
+
+ protected Row applyToStatic(Row row)
+ {
+ return applyToRow(row);
+ }
+
+ protected Row applyToRow(Row row)
+ {
+ if (repairedCounter.isDone())
+ return row;
+
+ assert purger != null;
+ Row purged = purger.applyToRow(row);
+ if (purged != null && !purged.isEmpty())
+ {
+ isFullyPurged = false;
+ purged.digest(getPerPartitionDigest());
+ }
+ return row;
+ }
+
+ protected void onPartitionClose()
+ {
+ if (perPartitionDigest != null)
+ {
+ // If the partition wasn't completely emptied by the purger,
+ // calculate the digest for the partition and use it to
+ // update the overall digest
+ if (!isFullyPurged)
+ {
+ if (perCommandDigest == null)
+ perCommandDigest = Digest.forRepairedDataTracking();
+
+ byte[] partitionDigest = perPartitionDigest.digest();
+ perCommandDigest.update(partitionDigest, 0, partitionDigest.length);
+ }
+
+ perPartitionDigest = null;
+ }
+ isFullyPurged = true;
+ }
+ }
+
+ if (repairedCounter.isDone())
+ return iterator;
+
+ UnfilteredRowIterator tracked = repairedCounter.applyTo(Transformation.apply(iterator, new WithTracking()));
+ onNewPartition(tracked);
+ return tracked;
+ }
+
+ public UnfilteredPartitionIterator extend(final UnfilteredPartitionIterator partitions,
+ final DataLimits.Counter limit)
+ {
+ class OverreadRepairedData extends Transformation<UnfilteredRowIterator> implements MoreRows<UnfilteredRowIterator>
+ {
+
+ protected UnfilteredRowIterator applyToPartition(UnfilteredRowIterator partition)
+ {
+ return MoreRows.extend(partition, this, partition.columns());
+ }
+
+ public UnfilteredRowIterator moreContents()
+ {
+ // We don't need to do anything until the DataLimits of the
+ // of the read have been reached
+ if (!limit.isDone() || repairedCounter.isDone())
+ return null;
+
+ long countBeforeOverreads = repairedCounter.counted();
+ long overreadStartTime = System.nanoTime();
+ if (currentPartition != null)
+ consumePartition(currentPartition, repairedCounter);
+
+ if (postLimitPartitions != null)
+ while (postLimitPartitions.hasNext() && !repairedCounter.isDone())
+ consumePartition(postLimitPartitions.next(), repairedCounter);
+
+ // we're not actually providing any more rows, just consuming the repaired data
+ long rows = repairedCounter.counted() - countBeforeOverreads;
+ long nanos = System.nanoTime() - overreadStartTime;
+ metrics.repairedDataTrackingOverreadRows.update(rows);
+ metrics.repairedDataTrackingOverreadTime.update(nanos, TimeUnit.NANOSECONDS);
+ Tracing.trace("Read {} additional rows of repaired data for tracking in {}ps", rows, TimeUnit.NANOSECONDS.toMicros(nanos));
+ return null;
+ }
+
+ private void consumePartition(UnfilteredRowIterator partition, DataLimits.Counter counter)
+ {
+ if (partition == null)
+ return;
+
+ while (!counter.isDone() && partition.hasNext())
+ partition.next();
+
+ partition.close();
+ }
+ }
+ // If the read didn't touch any sstables prepare() hasn't been called and
+ // we can skip this transformation
+ if (metrics == null || repairedCounter.isDone())
+ return partitions;
+ return Transformation.apply(partitions, new OverreadRepairedData());
+ }
+
+ /**
+ * Although PurgeFunction extends Transformation, this is never applied to an iterator.
+ * Instead, it is used by RepairedDataInfo during the generation of a repaired data
+ * digest to exclude data which will actually be purged later on in the read pipeline.
+ */
+ private static class RepairedDataPurger extends PurgeFunction
+ {
+ RepairedDataPurger(ColumnFamilyStore cfs,
+ int nowInSec,
+ int oldestUnrepairedTombstone)
+ {
+ super(nowInSec,
+ cfs.gcBefore(nowInSec),
+ oldestUnrepairedTombstone,
+ cfs.getCompactionStrategyManager().onlyPurgeRepairedTombstones(),
+ cfs.metadata.get().enforceStrictLiveness());
+ }
+
+ protected LongPredicate getPurgeEvaluator()
+ {
+ return (time) -> true;
+ }
+
+ void setCurrentKey(DecoratedKey key)
+ {
+ super.onNewPartition(key);
+ }
+
+ void setIsReverseOrder(boolean isReverseOrder)
+ {
+ super.setReverseOrder(isReverseOrder);
+ }
+
+ public DeletionTime applyToDeletion(DeletionTime deletionTime)
+ {
+ return super.applyToDeletion(deletionTime);
+ }
+
+ public Row applyToRow(Row row)
+ {
+ return super.applyToRow(row);
+ }
+
+ public RangeTombstoneMarker applyToMarker(RangeTombstoneMarker marker)
+ {
+ return super.applyToMarker(marker);
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/db/RowIndexEntry.java b/src/java/org/apache/cassandra/db/RowIndexEntry.java
index e620dc0..546c0c8 100644
--- a/src/java/org/apache/cassandra/db/RowIndexEntry.java
+++ b/src/java/org/apache/cassandra/db/RowIndexEntry.java
@@ -22,7 +22,6 @@
import java.util.List;
import com.codahale.metrics.Histogram;
-import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.cache.IMeasurableMemory;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.io.ISerializer;
@@ -33,6 +32,7 @@
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.io.util.FileDataInput;
import org.apache.cassandra.io.util.FileHandle;
+import org.apache.cassandra.io.util.RandomAccessReader;
import org.apache.cassandra.io.util.TrackedDataInputPlus;
import org.apache.cassandra.metrics.DefaultNameFactory;
import org.apache.cassandra.metrics.MetricNameFactory;
@@ -110,9 +110,6 @@
* <li>{@link ShallowIndexedEntry} is for index entries with index samples
* that exceed {@link org.apache.cassandra.config.Config#column_index_cache_size_in_kb}
* for sstables with an offset table to the index samples.</li>
- * <li>{@link LegacyShallowIndexedEntry} is for index entries with index samples
- * that exceed {@link org.apache.cassandra.config.Config#column_index_cache_size_in_kb}
- * but for legacy sstables.</li>
* </ul>
* <p>
* Since access to index samples on disk (obviously) requires some file
@@ -139,12 +136,14 @@
static final Histogram indexEntrySizeHistogram;
static final Histogram indexInfoCountHistogram;
static final Histogram indexInfoGetsHistogram;
- static
+ static final Histogram indexInfoReadsHistogram;
+ static
{
MetricNameFactory factory = new DefaultNameFactory("Index", "RowIndexEntry");
indexEntrySizeHistogram = Metrics.histogram(factory.createMetricName("IndexedEntrySize"), false);
indexInfoCountHistogram = Metrics.histogram(factory.createMetricName("IndexInfoCount"), false);
indexInfoGetsHistogram = Metrics.histogram(factory.createMetricName("IndexInfoGets"), false);
+ indexInfoReadsHistogram = Metrics.histogram(factory.createMetricName("IndexInfoReads"), false);
}
public final long position;
@@ -173,16 +172,6 @@
throw new UnsupportedOperationException();
}
- /**
- * The length of the row header (partition key, partition deletion and static row).
- * This value is only provided for indexed entries and this method will throw
- * {@code UnsupportedOperationException} if {@code !isIndexed()}.
- */
- public long headerLength()
- {
- throw new UnsupportedOperationException();
- }
-
public int columnsIndexCount()
{
return 0;
@@ -205,10 +194,10 @@
* @param idxInfoSerializer the {@link IndexInfo} serializer
*/
public static RowIndexEntry<IndexInfo> create(long dataFilePosition, long indexFilePosition,
- DeletionTime deletionTime, long headerLength, int columnIndexCount,
- int indexedPartSize,
- List<IndexInfo> indexSamples, int[] offsets,
- ISerializer<IndexInfo> idxInfoSerializer)
+ DeletionTime deletionTime, long headerLength, int columnIndexCount,
+ int indexedPartSize,
+ List<IndexInfo> indexSamples, int[] offsets,
+ ISerializer<IndexInfo> idxInfoSerializer)
{
// If the "partition building code" in BigTableWriter.append() via ColumnIndex returns a list
// of IndexInfo objects, which is the case if the serialized size is less than
@@ -237,7 +226,20 @@
public interface IndexSerializer<T>
{
void serialize(RowIndexEntry<T> rie, DataOutputPlus out, ByteBuffer indexInfo) throws IOException;
+
RowIndexEntry<T> deserialize(DataInputPlus in, long indexFilePosition) throws IOException;
+ default RowIndexEntry<T> deserialize(RandomAccessReader reader) throws IOException
+ {
+ return deserialize(reader, reader.getFilePointer());
+
+ }
+
+ default RowIndexEntry<T> deserialize(FileDataInput input) throws IOException
+ {
+ return deserialize(input, input.getFilePointer());
+
+ }
+
void serializeForCache(RowIndexEntry<T> rie, DataOutputPlus out) throws IOException;
RowIndexEntry<T> deserializeForCache(DataInputPlus in) throws IOException;
@@ -251,9 +253,9 @@
private final IndexInfo.Serializer idxInfoSerializer;
private final Version version;
- public Serializer(CFMetaData metadata, Version version, SerializationHeader header)
+ public Serializer(Version version, SerializationHeader header)
{
- this.idxInfoSerializer = metadata.serializers().indexInfoSerializer(version, header);
+ this.idxInfoSerializer = IndexInfo.serializer(version, header);
this.version = version;
}
@@ -264,22 +266,16 @@
public void serialize(RowIndexEntry<IndexInfo> rie, DataOutputPlus out, ByteBuffer indexInfo) throws IOException
{
- assert version.storeRows() : "We read old index files but we should never write them";
-
- rie.serialize(out, idxInfoSerializer, indexInfo);
+ rie.serialize(out, indexInfo);
}
public void serializeForCache(RowIndexEntry<IndexInfo> rie, DataOutputPlus out) throws IOException
{
- assert version.storeRows();
-
rie.serializeForCache(out);
}
public RowIndexEntry<IndexInfo> deserializeForCache(DataInputPlus in) throws IOException
{
- assert version.storeRows();
-
long position = in.readUnsignedVInt();
switch (in.readByte())
@@ -287,7 +283,7 @@
case CACHE_NOT_INDEXED:
return new RowIndexEntry<>(position);
case CACHE_INDEXED:
- return new IndexedEntry(position, in, idxInfoSerializer, version);
+ return new IndexedEntry(position, in, idxInfoSerializer);
case CACHE_INDEXED_SHALLOW:
return new ShallowIndexedEntry(position, in, idxInfoSerializer);
default:
@@ -295,11 +291,9 @@
}
}
- public static void skipForCache(DataInputPlus in, Version version) throws IOException
+ public static void skipForCache(DataInputPlus in) throws IOException
{
- assert version.storeRows();
-
- /* long position = */in.readUnsignedVInt();
+ in.readUnsignedVInt();
switch (in.readByte())
{
case CACHE_NOT_INDEXED:
@@ -317,9 +311,6 @@
public RowIndexEntry<IndexInfo> deserialize(DataInputPlus in, long indexFilePosition) throws IOException
{
- if (!version.storeRows())
- return LegacyShallowIndexedEntry.deserialize(in, indexFilePosition, idxInfoSerializer);
-
long position = in.readUnsignedVInt();
int size = (int)in.readUnsignedVInt();
@@ -338,7 +329,7 @@
if (size <= DatabaseDescriptor.getColumnIndexCacheSize())
{
return new IndexedEntry(position, in, deletionTime, headerLength, columnsIndexCount,
- idxInfoSerializer, version, indexedPartSize);
+ idxInfoSerializer, indexedPartSize);
}
else
{
@@ -354,10 +345,13 @@
public long deserializePositionAndSkip(DataInputPlus in) throws IOException
{
- if (!version.storeRows())
- return LegacyShallowIndexedEntry.deserializePositionAndSkip(in);
+ long position = in.readUnsignedVInt();
- return ShallowIndexedEntry.deserializePositionAndSkip(in);
+ int size = (int) in.readUnsignedVInt();
+ if (size > 0)
+ in.skipBytesFully(size);
+
+ return position;
}
/**
@@ -365,20 +359,20 @@
* of reading an entry, so this is only useful if you know what you are doing and in most case 'deserialize'
* should be used instead.
*/
- public static long readPosition(DataInputPlus in, Version version) throws IOException
+ public static long readPosition(DataInputPlus in) throws IOException
{
- return version.storeRows() ? in.readUnsignedVInt() : in.readLong();
+ return in.readUnsignedVInt();
}
public static void skip(DataInputPlus in, Version version) throws IOException
{
- readPosition(in, version);
- skipPromotedIndex(in, version);
+ readPosition(in);
+ skipPromotedIndex(in);
}
- private static void skipPromotedIndex(DataInputPlus in, Version version) throws IOException
+ private static void skipPromotedIndex(DataInputPlus in) throws IOException
{
- int size = version.storeRows() ? (int)in.readUnsignedVInt() : in.readInt();
+ int size = (int)in.readUnsignedVInt();
if (size <= 0)
return;
@@ -399,7 +393,7 @@
+ TypeSizes.sizeofUnsignedVInt(columnIndexCount);
}
- public void serialize(DataOutputPlus out, IndexInfo.Serializer idxInfoSerializer, ByteBuffer indexInfo) throws IOException
+ public void serialize(DataOutputPlus out, ByteBuffer indexInfo) throws IOException
{
out.writeUnsignedVInt(position);
@@ -413,164 +407,6 @@
out.writeByte(CACHE_NOT_INDEXED);
}
- private static final class LegacyShallowIndexedEntry extends RowIndexEntry<IndexInfo>
- {
- private static final long BASE_SIZE;
- static
- {
- BASE_SIZE = ObjectSizes.measure(new LegacyShallowIndexedEntry(0, 0, DeletionTime.LIVE, 0, new int[0], null, 0));
- }
-
- private final long indexFilePosition;
- private final int[] offsets;
- @Unmetered
- private final IndexInfo.Serializer idxInfoSerializer;
- private final DeletionTime deletionTime;
- private final long headerLength;
- private final int serializedSize;
-
- private LegacyShallowIndexedEntry(long dataFilePosition, long indexFilePosition,
- DeletionTime deletionTime, long headerLength,
- int[] offsets, IndexInfo.Serializer idxInfoSerializer,
- int serializedSize)
- {
- super(dataFilePosition);
- this.deletionTime = deletionTime;
- this.headerLength = headerLength;
- this.indexFilePosition = indexFilePosition;
- this.offsets = offsets;
- this.idxInfoSerializer = idxInfoSerializer;
- this.serializedSize = serializedSize;
- }
-
- @Override
- public DeletionTime deletionTime()
- {
- return deletionTime;
- }
-
- @Override
- public long headerLength()
- {
- return headerLength;
- }
-
- @Override
- public long unsharedHeapSize()
- {
- return BASE_SIZE + offsets.length * TypeSizes.sizeof(0);
- }
-
- @Override
- public int columnsIndexCount()
- {
- return offsets.length;
- }
-
- @Override
- public void serialize(DataOutputPlus out, IndexInfo.Serializer idxInfoSerializer, ByteBuffer indexInfo)
- {
- throw new UnsupportedOperationException("serializing legacy index entries is not supported");
- }
-
- @Override
- public void serializeForCache(DataOutputPlus out)
- {
- throw new UnsupportedOperationException("serializing legacy index entries is not supported");
- }
-
- @Override
- public IndexInfoRetriever openWithIndex(FileHandle indexFile)
- {
- int fieldsSize = (int) DeletionTime.serializer.serializedSize(deletionTime)
- + TypeSizes.sizeof(0); // columnIndexCount
- indexEntrySizeHistogram.update(serializedSize);
- indexInfoCountHistogram.update(offsets.length);
- return new LegacyIndexInfoRetriever(indexFilePosition +
- TypeSizes.sizeof(0L) + // position
- TypeSizes.sizeof(0) + // indexInfoSize
- fieldsSize,
- offsets, indexFile.createReader(), idxInfoSerializer);
- }
-
- public static RowIndexEntry<IndexInfo> deserialize(DataInputPlus in, long indexFilePosition,
- IndexInfo.Serializer idxInfoSerializer) throws IOException
- {
- long dataFilePosition = in.readLong();
-
- int size = in.readInt();
- if (size == 0)
- {
- return new RowIndexEntry<>(dataFilePosition);
- }
- else if (size <= DatabaseDescriptor.getColumnIndexCacheSize())
- {
- return new IndexedEntry(dataFilePosition, in, idxInfoSerializer);
- }
- else
- {
- DeletionTime deletionTime = DeletionTime.serializer.deserialize(in);
-
- // For legacy sstables (i.e. sstables pre-"ma", pre-3.0) we have to scan all serialized IndexInfo
- // objects to calculate the offsets array. However, it might be possible to deserialize all
- // IndexInfo objects here - but to just skip feels more gentle to the heap/GC.
-
- int entries = in.readInt();
- int[] offsets = new int[entries];
-
- TrackedDataInputPlus tracked = new TrackedDataInputPlus(in);
- long start = tracked.getBytesRead();
- long headerLength = 0L;
- for (int i = 0; i < entries; i++)
- {
- offsets[i] = (int) (tracked.getBytesRead() - start);
- if (i == 0)
- {
- IndexInfo info = idxInfoSerializer.deserialize(tracked);
- headerLength = info.offset;
- }
- else
- idxInfoSerializer.skip(tracked);
- }
-
- return new LegacyShallowIndexedEntry(dataFilePosition, indexFilePosition, deletionTime, headerLength, offsets, idxInfoSerializer, size);
- }
- }
-
- static long deserializePositionAndSkip(DataInputPlus in) throws IOException
- {
- long position = in.readLong();
-
- int size = in.readInt();
- if (size > 0)
- in.skipBytesFully(size);
-
- return position;
- }
- }
-
- private static final class LegacyIndexInfoRetriever extends FileIndexInfoRetriever
- {
- private final int[] offsets;
-
- private LegacyIndexInfoRetriever(long indexFilePosition, int[] offsets, FileDataInput reader, IndexInfo.Serializer idxInfoSerializer)
- {
- super(indexFilePosition, reader, idxInfoSerializer);
- this.offsets = offsets;
- }
-
- IndexInfo fetchIndex(int index) throws IOException
- {
- retrievals++;
-
- // seek to posision of IndexInfo
- indexReader.seek(indexInfoFilePosition + offsets[index]);
-
- // deserialize IndexInfo
- return idxInfoSerializer.deserialize(indexReader);
- }
- }
-
/**
* An entry in the row index for a row whose columns are indexed - used for both legacy and current formats.
*/
@@ -609,8 +445,7 @@
private IndexedEntry(long dataFilePosition, DataInputPlus in,
DeletionTime deletionTime, long headerLength, int columnIndexCount,
- IndexInfo.Serializer idxInfoSerializer,
- Version version, int indexedPartSize) throws IOException
+ IndexInfo.Serializer idxInfoSerializer, int indexedPartSize) throws IOException
{
super(dataFilePosition);
@@ -622,14 +457,9 @@
for (int i = 0; i < columnsIndexCount; i++)
this.columnsIndex[i] = idxInfoSerializer.deserialize(in);
- int[] offsets = null;
- if (version.storeRows())
- {
- offsets = new int[this.columnsIndex.length];
- for (int i = 0; i < offsets.length; i++)
- offsets[i] = in.readInt();
- }
- this.offsets = offsets;
+ this.offsets = new int[this.columnsIndex.length];
+ for (int i = 0; i < offsets.length; i++)
+ offsets[i] = in.readInt();
this.indexedPartSize = indexedPartSize;
@@ -639,7 +469,7 @@
/**
* Constructor called from {@link Serializer#deserializeForCache(org.apache.cassandra.io.util.DataInputPlus)}.
*/
- private IndexedEntry(long dataFilePosition, DataInputPlus in, IndexInfo.Serializer idxInfoSerializer, Version version) throws IOException
+ private IndexedEntry(long dataFilePosition, DataInputPlus in, ISerializer<IndexInfo> idxInfoSerializer) throws IOException
{
super(dataFilePosition);
@@ -660,36 +490,6 @@
this.idxInfoSerializer = idxInfoSerializer;
}
- /**
- * Constructor called from {@link LegacyShallowIndexedEntry#deserialize(org.apache.cassandra.io.util.DataInputPlus, long, org.apache.cassandra.io.sstable.IndexInfo.Serializer)}.
- * Only for legacy sstables.
- */
- private IndexedEntry(long dataFilePosition, DataInputPlus in, IndexInfo.Serializer idxInfoSerializer) throws IOException
- {
- super(dataFilePosition);
-
- long headerLength = 0;
- this.deletionTime = DeletionTime.serializer.deserialize(in);
- int columnsIndexCount = in.readInt();
-
- TrackedDataInputPlus trackedIn = new TrackedDataInputPlus(in);
-
- this.columnsIndex = new IndexInfo[columnsIndexCount];
- for (int i = 0; i < columnsIndexCount; i++)
- {
- this.columnsIndex[i] = idxInfoSerializer.deserialize(trackedIn);
- if (i == 0)
- headerLength = this.columnsIndex[i].offset;
- }
- this.headerLength = headerLength;
-
- this.offsets = null;
-
- this.indexedPartSize = (int) trackedIn.getBytesRead();
-
- this.idxInfoSerializer = idxInfoSerializer;
- }
-
@Override
public boolean indexOnHeap()
{
@@ -709,12 +509,6 @@
}
@Override
- public long headerLength()
- {
- return headerLength;
- }
-
- @Override
public IndexInfoRetriever openWithIndex(FileHandle indexFile)
{
indexEntrySizeHistogram.update(serializedSize(deletionTime, headerLength, columnsIndex.length) + indexedPartSize);
@@ -748,8 +542,7 @@
+ ObjectSizes.sizeOfReferenceArray(columnsIndex.length);
}
- @Override
- public void serialize(DataOutputPlus out, IndexInfo.Serializer idxInfoSerializer, ByteBuffer indexInfo) throws IOException
+ public void serialize(DataOutputPlus out, ByteBuffer indexInfo) throws IOException
{
assert indexedPartSize != Integer.MIN_VALUE;
@@ -782,11 +575,11 @@
static void skipForCache(DataInputPlus in) throws IOException
{
- /*long headerLength =*/in.readUnsignedVInt();
- /*DeletionTime deletionTime = */DeletionTime.serializer.skip(in);
- /*int columnsIndexCount = (int)*/in.readUnsignedVInt();
+ in.readUnsignedVInt();
+ DeletionTime.serializer.skip(in);
+ in.readUnsignedVInt();
- /*int indexedPartSize = (int)*/in.readUnsignedVInt();
+ in.readUnsignedVInt();
}
}
@@ -873,12 +666,6 @@
}
@Override
- public long headerLength()
- {
- return headerLength;
- }
-
- @Override
public IndexInfoRetriever openWithIndex(FileHandle indexFile)
{
indexEntrySizeHistogram.update(indexedPartSize + fieldsSerializedSize);
@@ -898,7 +685,7 @@
}
@Override
- public void serialize(DataOutputPlus out, IndexInfo.Serializer idxInfoSerializer, ByteBuffer indexInfo) throws IOException
+ public void serialize(DataOutputPlus out, ByteBuffer indexInfo) throws IOException
{
out.writeUnsignedVInt(position);
@@ -911,17 +698,6 @@
out.write(indexInfo);
}
- static long deserializePositionAndSkip(DataInputPlus in) throws IOException
- {
- long position = in.readUnsignedVInt();
-
- int size = (int) in.readUnsignedVInt();
- if (size > 0)
- in.skipBytesFully(size);
-
- return position;
- }
-
@Override
public void serializeForCache(DataOutputPlus out) throws IOException
{
@@ -939,13 +715,13 @@
static void skipForCache(DataInputPlus in) throws IOException
{
- /*long indexFilePosition =*/in.readUnsignedVInt();
+ in.readUnsignedVInt();
- /*long headerLength =*/in.readUnsignedVInt();
- /*DeletionTime deletionTime = */DeletionTime.serializer.skip(in);
- /*int columnsIndexCount = (int)*/in.readUnsignedVInt();
+ in.readUnsignedVInt();
+ DeletionTime.serializer.skip(in);
+ in.readUnsignedVInt();
- /*int indexedPartSize = (int)*/in.readUnsignedVInt();
+ in.readUnsignedVInt();
}
}
@@ -962,8 +738,6 @@
IndexInfo fetchIndex(int index) throws IOException
{
- retrievals++;
-
// seek to position in "offsets to IndexInfo" table
indexReader.seek(indexInfoFilePosition + offsetsOffset + index * TypeSizes.sizeof(0));
@@ -1014,6 +788,7 @@
public final IndexInfo columnsIndex(int index) throws IOException
{
+ retrievals++;
return fetchIndex(index);
}
@@ -1024,6 +799,7 @@
indexReader.close();
indexInfoGetsHistogram.update(retrievals);
+ indexInfoReadsHistogram.update(retrievals);
}
}
}
diff --git a/src/java/org/apache/cassandra/db/SSTableImporter.java b/src/java/org/apache/cassandra/db/SSTableImporter.java
new file mode 100644
index 0000000..7597f82
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/SSTableImporter.java
@@ -0,0 +1,468 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.db.compaction.Verifier;
+import org.apache.cassandra.db.lifecycle.SSTableSet;
+import org.apache.cassandra.io.sstable.Component;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.KeyIterator;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.format.SSTableWriter;
+import org.apache.cassandra.io.util.RandomAccessReader;
+import org.apache.cassandra.service.ActiveRepairService;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.Pair;
+import org.apache.cassandra.utils.concurrent.Refs;
+
+public class SSTableImporter
+{
+ private static final Logger logger = LoggerFactory.getLogger(ColumnFamilyStore.class);
+
+ private final ColumnFamilyStore cfs;
+
+ public SSTableImporter(ColumnFamilyStore cfs)
+ {
+ this.cfs = cfs;
+ }
+
+ /**
+ * Imports sstables from the directories given in options.srcPaths
+ *
+ * If import fails in any of the directories, that directory is skipped and the failed directories
+ * are returned so that the user can re-upload files or remove corrupt files.
+ *
+ * If one of the directories in srcPaths is not readable/does not exist, we exit immediately to let
+ * the user change permissions or similar on the directory.
+ *
+ * @param options
+ * @return list of failed directories
+ */
+ @VisibleForTesting
+ synchronized List<String> importNewSSTables(Options options)
+ {
+ logger.info("Loading new SSTables for {}/{}: {}",
+ cfs.keyspace.getName(), cfs.getTableName(), options);
+
+ List<Pair<Directories.SSTableLister, String>> listers = getSSTableListers(options.srcPaths);
+
+ Set<Descriptor> currentDescriptors = new HashSet<>();
+ for (SSTableReader sstable : cfs.getSSTables(SSTableSet.CANONICAL))
+ currentDescriptors.add(sstable.descriptor);
+ List<String> failedDirectories = new ArrayList<>();
+
+ // verify first to avoid starting to copy sstables to the data directories and then have to abort.
+ if (options.verifySSTables || options.verifyTokens)
+ {
+ for (Pair<Directories.SSTableLister, String> listerPair : listers)
+ {
+ Directories.SSTableLister lister = listerPair.left;
+ String dir = listerPair.right;
+ for (Map.Entry<Descriptor, Set<Component>> entry : lister.list().entrySet())
+ {
+ Descriptor descriptor = entry.getKey();
+ if (!currentDescriptors.contains(entry.getKey()))
+ {
+ try
+ {
+ verifySSTableForImport(descriptor, entry.getValue(), options.verifyTokens, options.verifySSTables, options.extendedVerify);
+ }
+ catch (Throwable t)
+ {
+ if (dir != null)
+ {
+ logger.error("Failed verifying sstable {} in directory {}", descriptor, dir, t);
+ failedDirectories.add(dir);
+ }
+ else
+ {
+ logger.error("Failed verifying sstable {}", descriptor, t);
+ throw new RuntimeException("Failed verifying sstable "+descriptor, t);
+ }
+ break;
+ }
+ }
+ }
+ }
+ }
+
+ Set<SSTableReader> newSSTables = new HashSet<>();
+ for (Pair<Directories.SSTableLister, String> listerPair : listers)
+ {
+ Directories.SSTableLister lister = listerPair.left;
+ String dir = listerPair.right;
+ if (failedDirectories.contains(dir))
+ continue;
+
+ Set<MovedSSTable> movedSSTables = new HashSet<>();
+ Set<SSTableReader> newSSTablesPerDirectory = new HashSet<>();
+ for (Map.Entry<Descriptor, Set<Component>> entry : lister.list().entrySet())
+ {
+ try
+ {
+ Descriptor oldDescriptor = entry.getKey();
+ if (currentDescriptors.contains(oldDescriptor))
+ continue;
+
+ File targetDir = getTargetDirectory(dir, oldDescriptor, entry.getValue());
+ Descriptor newDescriptor = cfs.getUniqueDescriptorFor(entry.getKey(), targetDir);
+ maybeMutateMetadata(entry.getKey(), options);
+ movedSSTables.add(new MovedSSTable(newDescriptor, entry.getKey(), entry.getValue()));
+ SSTableReader sstable = SSTableReader.moveAndOpenSSTable(cfs, entry.getKey(), newDescriptor, entry.getValue());
+ newSSTablesPerDirectory.add(sstable);
+ }
+ catch (Throwable t)
+ {
+ newSSTablesPerDirectory.forEach(s -> s.selfRef().release());
+ if (dir != null)
+ {
+ logger.error("Failed importing sstables in directory {}", dir, t);
+ failedDirectories.add(dir);
+ moveSSTablesBack(movedSSTables);
+ movedSSTables.clear();
+ newSSTablesPerDirectory.clear();
+ break;
+ }
+ else
+ {
+ logger.error("Failed importing sstables from data directory - renamed sstables are: {}", movedSSTables);
+ throw new RuntimeException("Failed importing sstables", t);
+ }
+ }
+ }
+ newSSTables.addAll(newSSTablesPerDirectory);
+ }
+
+ if (newSSTables.isEmpty())
+ {
+ logger.info("No new SSTables were found for {}/{}", cfs.keyspace.getName(), cfs.getTableName());
+ return failedDirectories;
+ }
+
+ logger.info("Loading new SSTables and building secondary indexes for {}/{}: {}", cfs.keyspace.getName(), cfs.getTableName(), newSSTables);
+
+ try (Refs<SSTableReader> refs = Refs.ref(newSSTables))
+ {
+ cfs.getTracker().addSSTables(newSSTables);
+ for (SSTableReader reader : newSSTables)
+ {
+ if (options.invalidateCaches && cfs.isRowCacheEnabled())
+ invalidateCachesForSSTable(reader.descriptor);
+ }
+
+ }
+
+ logger.info("Done loading load new SSTables for {}/{}", cfs.keyspace.getName(), cfs.getTableName());
+ return failedDirectories;
+ }
+
+ /**
+ * Opens the sstablereader described by descriptor and figures out the correct directory for it based
+ * on the first token
+ *
+ * srcPath == null means that the sstable is in a data directory and we can use that directly.
+ *
+ * If we fail figuring out the directory we will pick the one with the most available disk space.
+ */
+ private File getTargetDirectory(String srcPath, Descriptor descriptor, Set<Component> components)
+ {
+ if (srcPath == null)
+ return descriptor.directory;
+
+ File targetDirectory = null;
+ SSTableReader sstable = null;
+ try
+ {
+ sstable = SSTableReader.open(descriptor, components, cfs.metadata);
+ targetDirectory = cfs.getDirectories().getLocationForDisk(cfs.diskBoundaryManager.getDiskBoundaries(cfs).getCorrectDiskForSSTable(sstable));
+ }
+ finally
+ {
+ if (sstable != null)
+ sstable.selfRef().release();
+ }
+ return targetDirectory == null ? cfs.getDirectories().getWriteableLocationToLoadFile(new File(descriptor.baseFilename())) : targetDirectory;
+ }
+
+ /**
+ * Create SSTableListers based on srcPaths
+ *
+ * If srcPaths is empty, we create a lister that lists sstables in the data directories (deprecated use)
+ */
+ private List<Pair<Directories.SSTableLister, String>> getSSTableListers(Set<String> srcPaths)
+ {
+ List<Pair<Directories.SSTableLister, String>> listers = new ArrayList<>();
+
+ if (!srcPaths.isEmpty())
+ {
+ for (String path : srcPaths)
+ {
+ File dir = new File(path);
+ if (!dir.exists())
+ {
+ throw new RuntimeException(String.format("Directory %s does not exist", path));
+ }
+ if (!Directories.verifyFullPermissions(dir, path))
+ {
+ throw new RuntimeException("Insufficient permissions on directory " + path);
+ }
+ listers.add(Pair.create(cfs.getDirectories().sstableLister(dir, Directories.OnTxnErr.IGNORE).skipTemporary(true), path));
+ }
+ }
+ else
+ {
+ listers.add(Pair.create(cfs.getDirectories().sstableLister(Directories.OnTxnErr.IGNORE).skipTemporary(true), null));
+ }
+
+ return listers;
+ }
+
+ private static class MovedSSTable
+ {
+ private final Descriptor newDescriptor;
+ private final Descriptor oldDescriptor;
+ private final Set<Component> components;
+
+ private MovedSSTable(Descriptor newDescriptor, Descriptor oldDescriptor, Set<Component> components)
+ {
+ this.newDescriptor = newDescriptor;
+ this.oldDescriptor = oldDescriptor;
+ this.components = components;
+ }
+
+ public String toString()
+ {
+ return String.format("%s moved to %s with components %s", oldDescriptor, newDescriptor, components);
+ }
+ }
+
+ /**
+ * If we fail when opening the sstable (if for example the user passes in --no-verify and there are corrupt sstables)
+ * we might have started copying sstables to the data directory, these need to be moved back to the original name/directory
+ */
+ private void moveSSTablesBack(Set<MovedSSTable> movedSSTables)
+ {
+ for (MovedSSTable movedSSTable : movedSSTables)
+ {
+ if (new File(movedSSTable.newDescriptor.filenameFor(Component.DATA)).exists())
+ {
+ logger.debug("Moving sstable {} back to {}", movedSSTable.newDescriptor.filenameFor(Component.DATA)
+ , movedSSTable.oldDescriptor.filenameFor(Component.DATA));
+ SSTableWriter.rename(movedSSTable.newDescriptor, movedSSTable.oldDescriptor, movedSSTable.components);
+ }
+ }
+ }
+
+ /**
+ * Iterates over all keys in the sstable index and invalidates the row cache
+ */
+ @VisibleForTesting
+ void invalidateCachesForSSTable(Descriptor desc)
+ {
+ try (KeyIterator iter = new KeyIterator(desc, cfs.metadata()))
+ {
+ while (iter.hasNext())
+ {
+ DecoratedKey decoratedKey = iter.next();
+ cfs.invalidateCachedPartition(decoratedKey);
+ }
+ }
+ }
+
+ /**
+ * Verify an sstable for import, throws exception if there is a failure verifying.
+ *
+ * @param verifyTokens to verify that the tokens are owned by the current node
+ * @param verifySSTables to verify the sstables given. If this is false a "quick" verification will be run, just deserializing metadata
+ * @param extendedVerify to validate the values in the sstables
+ */
+ private void verifySSTableForImport(Descriptor descriptor, Set<Component> components, boolean verifyTokens, boolean verifySSTables, boolean extendedVerify)
+ {
+ SSTableReader reader = null;
+ try
+ {
+ reader = SSTableReader.open(descriptor, components, cfs.metadata);
+ Verifier.Options verifierOptions = Verifier.options()
+ .extendedVerification(extendedVerify)
+ .checkOwnsTokens(verifyTokens)
+ .quick(!verifySSTables)
+ .invokeDiskFailurePolicy(false)
+ .mutateRepairStatus(false).build();
+ try (Verifier verifier = new Verifier(cfs, reader, false, verifierOptions))
+ {
+ verifier.verify();
+ }
+ }
+ catch (Throwable t)
+ {
+ throw new RuntimeException("Can't import sstable " + descriptor, t);
+ }
+ finally
+ {
+ if (reader != null)
+ reader.selfRef().release();
+ }
+ }
+
+ /**
+ * Depending on the options passed in, this might reset level on the sstable to 0 and/or remove the repair information
+ * from the sstable
+ */
+ private void maybeMutateMetadata(Descriptor descriptor, Options options) throws IOException
+ {
+ if (new File(descriptor.filenameFor(Component.STATS)).exists())
+ {
+ if (options.resetLevel)
+ {
+ descriptor.getMetadataSerializer().mutateLevel(descriptor, 0);
+ }
+ if (options.clearRepaired)
+ {
+ descriptor.getMetadataSerializer().mutateRepairMetadata(descriptor, ActiveRepairService.UNREPAIRED_SSTABLE,
+ null,
+ false);
+ }
+ }
+ }
+
+ public static class Options
+ {
+ private final Set<String> srcPaths;
+ private final boolean resetLevel;
+ private final boolean clearRepaired;
+ private final boolean verifySSTables;
+ private final boolean verifyTokens;
+ private final boolean invalidateCaches;
+ private final boolean extendedVerify;
+
+ public Options(Set<String> srcPaths, boolean resetLevel, boolean clearRepaired, boolean verifySSTables, boolean verifyTokens, boolean invalidateCaches, boolean extendedVerify)
+ {
+ this.srcPaths = srcPaths;
+ this.resetLevel = resetLevel;
+ this.clearRepaired = clearRepaired;
+ this.verifySSTables = verifySSTables;
+ this.verifyTokens = verifyTokens;
+ this.invalidateCaches = invalidateCaches;
+ this.extendedVerify = extendedVerify;
+ }
+
+ public static Builder options(String srcDir)
+ {
+ return new Builder(Collections.singleton(srcDir));
+ }
+
+ public static Builder options(Set<String> srcDirs)
+ {
+ return new Builder(srcDirs);
+ }
+
+ public static Builder options()
+ {
+ return options(Collections.emptySet());
+ }
+
+ @Override
+ public String toString()
+ {
+ return "Options{" +
+ "srcPaths='" + srcPaths + '\'' +
+ ", resetLevel=" + resetLevel +
+ ", clearRepaired=" + clearRepaired +
+ ", verifySSTables=" + verifySSTables +
+ ", verifyTokens=" + verifyTokens +
+ ", invalidateCaches=" + invalidateCaches +
+ ", extendedVerify=" + extendedVerify +
+ '}';
+ }
+
+ static class Builder
+ {
+ private final Set<String> srcPaths;
+ private boolean resetLevel = false;
+ private boolean clearRepaired = false;
+ private boolean verifySSTables = false;
+ private boolean verifyTokens = false;
+ private boolean invalidateCaches = false;
+ private boolean extendedVerify = false;
+
+ private Builder(Set<String> srcPath)
+ {
+ assert srcPath != null;
+ this.srcPaths = srcPath;
+ }
+
+ public Builder resetLevel(boolean value)
+ {
+ resetLevel = value;
+ return this;
+ }
+
+ public Builder clearRepaired(boolean value)
+ {
+ clearRepaired = value;
+ return this;
+ }
+
+ public Builder verifySSTables(boolean value)
+ {
+ verifySSTables = value;
+ return this;
+ }
+
+ public Builder verifyTokens(boolean value)
+ {
+ verifyTokens = value;
+ return this;
+ }
+
+ public Builder invalidateCaches(boolean value)
+ {
+ invalidateCaches = value;
+ return this;
+ }
+
+ public Builder extendedVerify(boolean value)
+ {
+ extendedVerify = value;
+ return this;
+ }
+
+ public Options build()
+ {
+ return new Options(srcPaths, resetLevel, clearRepaired, verifySSTables, verifyTokens, invalidateCaches, extendedVerify);
+ }
+ }
+ }
+
+}
diff --git a/src/java/org/apache/cassandra/db/SchemaCQLHelper.java b/src/java/org/apache/cassandra/db/SchemaCQLHelper.java
new file mode 100644
index 0000000..6f9e526
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/SchemaCQLHelper.java
@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db;
+
+import java.nio.ByteBuffer;
+import java.util.stream.Stream;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.db.marshal.UserType;
+import org.apache.cassandra.schema.*;
+
+/**
+ * Helper methods to represent TableMetadata and related objects in CQL format
+ */
+public class SchemaCQLHelper
+{
+ /**
+ * Generates the DDL statement for a {@code schema.cql} snapshot file.
+ */
+ public static Stream<String> reCreateStatementsForSchemaCql(TableMetadata metadata, Types types)
+ {
+ // Types come first, as table can't be created without them
+ Stream<String> udts = SchemaCQLHelper.getUserTypesAsCQL(metadata, types);
+
+ return Stream.concat(udts,
+ reCreateStatements(metadata,
+ true,
+ true,
+ true,
+ true));
+ }
+
+ public static Stream<String> reCreateStatements(TableMetadata metadata,
+ boolean includeDroppedColumns,
+ boolean internals,
+ boolean ifNotExists,
+ boolean includeIndexes)
+ {
+ // Record re-create schema statements
+ Stream<String> r = Stream.of(metadata)
+ .map((tm) -> SchemaCQLHelper.getTableMetadataAsCQL(tm,
+ includeDroppedColumns,
+ internals,
+ ifNotExists));
+
+ if (includeIndexes)
+ {
+ // Indexes applied as last, since otherwise they may interfere with column drops / re-additions
+ r = Stream.concat(r, SchemaCQLHelper.getIndexesAsCQL(metadata));
+ }
+
+ return r;
+ }
+
+ /**
+ * Build a CQL String representation of Column Family Metadata.
+ *
+ * *Note*: this is _only_ visible for testing; you generally shouldn't re-create a single table in isolation as
+ * that will not contain everything needed for user types.
+ */
+ @VisibleForTesting
+ public static String getTableMetadataAsCQL(TableMetadata metadata,
+ boolean includeDroppedColumns,
+ boolean internals,
+ boolean ifNotExists)
+ {
+ if (metadata.isView())
+ {
+ KeyspaceMetadata keyspaceMetadata = Schema.instance.getKeyspaceMetadata(metadata.keyspace);
+ ViewMetadata viewMetadata = keyspaceMetadata.views.get(metadata.name).orElse(null);
+ assert viewMetadata != null;
+ return viewMetadata.toCqlString(internals, ifNotExists);
+ }
+
+ return metadata.toCqlString(includeDroppedColumns, internals, ifNotExists);
+ }
+
+ /**
+ * Build a CQL String representation of User Types used in the given table.
+ *
+ * Type order is ensured as types are built incrementally: from the innermost (most nested)
+ * to the outermost.
+ *
+ * @param metadata the table for which to extract the user types CQL statements.
+ * @param types the user types defined in the keyspace of the dumped table (which will thus contain any user type
+ * used by {@code metadata}).
+ * @return a list of {@code CREATE TYPE} statements corresponding to all the types used in {@code metadata}.
+ */
+ @VisibleForTesting
+ public static Stream<String> getUserTypesAsCQL(TableMetadata metadata, Types types)
+ {
+ /*
+ * Implementation note: at first approximation, it may seem like we don't need the Types argument and instead
+ * directly extract the user types from the provided TableMetadata. Indeed, full user types definitions are
+ * contained in UserType instances.
+ *
+ * However, the UserType instance found within the TableMetadata may have been frozen in such a way that makes
+ * it challenging.
+ *
+ * Consider the user has created:
+ * CREATE TYPE inner (a set<int>);
+ * CREATE TYPE outer (b inner);
+ * CREATE TABLE t (k int PRIMARY KEY, c1 frozen<outer>, c2 set<frozen<inner>>)
+ * The corresponding TableMetadata would have, as types (where 'mc=true' means that the type has his isMultiCell
+ * set to true):
+ * c1: UserType(mc=false, "outer", b->UserType(mc=false, "inner", a->SetType(mc=fase, Int32Type)))
+ * c2: SetType(mc=true, UserType(mc=false, "inner", a->SetType(mc=fase, Int32Type)))
+ * From which, it's impossible to decide if we should dump the types above, or instead:
+ * CREATE TYPE inner (a frozen<set<int>>);
+ * CREATE TYPE outer (b frozen<inner>);
+ * or anything in-between.
+ *
+ * And while, as of the current limitation around multi-cell types (that are only support non-frozen at
+ * top-level), any of the generated definition would kind of "work", 1) this could confuse users and 2) this
+ * would break if we do lift the limitation, which wouldn't be future proof.
+ */
+ return metadata.getReferencedUserTypes()
+ .stream()
+ .map(name -> getType(metadata, types, name).toCqlString(false));
+ }
+
+ /**
+ * Build a CQL String representation of Indexes on columns in the given Column Family
+ */
+ @VisibleForTesting
+ public static Stream<String> getIndexesAsCQL(TableMetadata metadata)
+ {
+ return metadata.indexes
+ .stream()
+ .map(indexMetadata -> indexMetadata.toCqlString(metadata));
+ }
+
+ private static UserType getType(TableMetadata metadata, Types types, ByteBuffer name)
+ {
+ return types.get(name)
+ .orElseThrow(() -> new IllegalStateException(String.format("user type %s is part of table %s definition but its definition was missing",
+ UTF8Type.instance.getString(name),
+ metadata)));
+ }
+}
diff --git a/src/java/org/apache/cassandra/db/SchemaCheckVerbHandler.java b/src/java/org/apache/cassandra/db/SchemaCheckVerbHandler.java
deleted file mode 100644
index be501de..0000000
--- a/src/java/org/apache/cassandra/db/SchemaCheckVerbHandler.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.db;
-
-import java.util.UUID;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.cassandra.config.Schema;
-import org.apache.cassandra.net.IVerbHandler;
-import org.apache.cassandra.net.MessageIn;
-import org.apache.cassandra.net.MessageOut;
-import org.apache.cassandra.net.MessagingService;
-import org.apache.cassandra.utils.UUIDSerializer;
-
-public class SchemaCheckVerbHandler implements IVerbHandler
-{
- private final Logger logger = LoggerFactory.getLogger(SchemaCheckVerbHandler.class);
-
- public void doVerb(MessageIn message, int id)
- {
- logger.trace("Received schema check request.");
-
- /*
- 3.11 is special here: We return the 3.0 compatible version, if the requesting node
- is running 3.0. Otherwise the "real" schema version.
- */
- MessageOut<UUID> response = new MessageOut<>(MessagingService.Verb.INTERNAL_RESPONSE,
- Schema.instance.getVersion(),
- UUIDSerializer.serializer);
- MessagingService.instance().sendReply(response, id, message.from);
- }
-}
diff --git a/src/java/org/apache/cassandra/db/SerializationHeader.java b/src/java/org/apache/cassandra/db/SerializationHeader.java
index 5b62b0a..1c22feb 100644
--- a/src/java/org/apache/cassandra/db/SerializationHeader.java
+++ b/src/java/org/apache/cassandra/db/SerializationHeader.java
@@ -22,20 +22,22 @@
import java.util.*;
import com.google.common.collect.ImmutableList;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
+
import org.apache.cassandra.db.filter.ColumnFilter;
-import org.apache.cassandra.db.rows.*;
import org.apache.cassandra.db.marshal.AbstractType;
-import org.apache.cassandra.db.marshal.UTF8Type;
import org.apache.cassandra.db.marshal.TypeParser;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.db.rows.*;
+import org.apache.cassandra.exceptions.UnknownColumnException;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.sstable.format.Version;
-import org.apache.cassandra.io.sstable.metadata.MetadataType;
-import org.apache.cassandra.io.sstable.metadata.MetadataComponent;
import org.apache.cassandra.io.sstable.metadata.IMetadataComponentSerializer;
+import org.apache.cassandra.io.sstable.metadata.MetadataComponent;
+import org.apache.cassandra.io.sstable.metadata.MetadataType;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.utils.ByteBufferUtil;
public class SerializationHeader
@@ -47,7 +49,7 @@
private final AbstractType<?> keyType;
private final List<AbstractType<?>> clusteringTypes;
- private final PartitionColumns columns;
+ private final RegularAndStaticColumns columns;
private final EncodingStats stats;
private final Map<ByteBuffer, AbstractType<?>> typeMap;
@@ -55,7 +57,7 @@
private SerializationHeader(boolean isForSSTable,
AbstractType<?> keyType,
List<AbstractType<?>> clusteringTypes,
- PartitionColumns columns,
+ RegularAndStaticColumns columns,
EncodingStats stats,
Map<ByteBuffer, AbstractType<?>> typeMap)
{
@@ -67,12 +69,12 @@
this.typeMap = typeMap;
}
- public static SerializationHeader makeWithoutStats(CFMetaData metadata)
+ public static SerializationHeader makeWithoutStats(TableMetadata metadata)
{
- return new SerializationHeader(true, metadata, metadata.partitionColumns(), EncodingStats.NO_STATS);
+ return new SerializationHeader(true, metadata, metadata.regularAndStaticColumns(), EncodingStats.NO_STATS);
}
- public static SerializationHeader make(CFMetaData metadata, Collection<SSTableReader> sstables)
+ public static SerializationHeader make(TableMetadata metadata, Collection<SSTableReader> sstables)
{
// The serialization header has to be computed before the start of compaction (since it's used to write)
// the result. This means that when compacting multiple sources, we won't have perfectly accurate stats
@@ -85,17 +87,14 @@
// our stats merging on the compacted files headers, which as we just said can be somewhat inaccurate,
// but rather on their stats stored in StatsMetadata that are fully accurate.
EncodingStats.Collector stats = new EncodingStats.Collector();
- PartitionColumns.Builder columns = PartitionColumns.builder();
- // We need to order the SSTables by descending generation to be sure that we use latest column definitions.
+ RegularAndStaticColumns.Builder columns = RegularAndStaticColumns.builder();
+ // We need to order the SSTables by descending generation to be sure that we use latest column metadata.
for (SSTableReader sstable : orderByDescendingGeneration(sstables))
{
stats.updateTimestamp(sstable.getMinTimestamp());
stats.updateLocalDeletionTime(sstable.getMinLocalDeletionTime());
stats.updateTTL(sstable.getMinTTL());
- if (sstable.header == null)
- columns.addAll(metadata.partitionColumns());
- else
- columns.addAll(sstable.header.columns());
+ columns.addAll(sstable.header.columns());
}
return new SerializationHeader(true, metadata, columns.build(), stats.get());
}
@@ -111,19 +110,19 @@
}
public SerializationHeader(boolean isForSSTable,
- CFMetaData metadata,
- PartitionColumns columns,
+ TableMetadata metadata,
+ RegularAndStaticColumns columns,
EncodingStats stats)
{
this(isForSSTable,
- metadata.getKeyValidator(),
+ metadata.partitionKeyType,
metadata.comparator.subtypes(),
columns,
stats,
null);
}
- public PartitionColumns columns()
+ public RegularAndStaticColumns columns()
{
return columns;
}
@@ -158,7 +157,7 @@
return isStatic ? columns.statics : columns.regulars;
}
- public AbstractType<?> getType(ColumnDefinition column)
+ public AbstractType<?> getType(ColumnMetadata column)
{
return typeMap == null ? column.type : typeMap.get(column.name.bytes);
}
@@ -252,9 +251,9 @@
{
Map<ByteBuffer, AbstractType<?>> staticColumns = new LinkedHashMap<>();
Map<ByteBuffer, AbstractType<?>> regularColumns = new LinkedHashMap<>();
- for (ColumnDefinition column : columns.statics)
+ for (ColumnMetadata column : columns.statics)
staticColumns.put(column.name.bytes, column.type);
- for (ColumnDefinition column : columns.regulars)
+ for (ColumnMetadata column : columns.regulars)
regularColumns.put(column.name.bytes, column.type);
return new Component(keyType, clusteringTypes, staticColumns, regularColumns, stats);
}
@@ -266,7 +265,7 @@
}
/**
- * We need the CFMetadata to properly deserialize a SerializationHeader but it's clunky to pass that to
+ * We need the TableMetadata to properly deserialize a SerializationHeader but it's clunky to pass that to
* a SSTable component, so we use this temporary object to delay the actual need for the metadata.
*/
public static class Component extends MetadataComponent
@@ -307,11 +306,11 @@
return MetadataType.HEADER;
}
- public SerializationHeader toHeader(CFMetaData metadata)
+ public SerializationHeader toHeader(TableMetadata metadata) throws UnknownColumnException
{
Map<ByteBuffer, AbstractType<?>> typeMap = new HashMap<>(staticColumns.size() + regularColumns.size());
- PartitionColumns.Builder builder = PartitionColumns.builder();
+ RegularAndStaticColumns.Builder builder = RegularAndStaticColumns.builder();
for (Map<ByteBuffer, AbstractType<?>> map : ImmutableList.of(staticColumns, regularColumns))
{
boolean isStatic = map == staticColumns;
@@ -322,7 +321,7 @@
if (other != null && !other.equals(e.getValue()))
throw new IllegalStateException("Column " + name + " occurs as both regular and static with types " + other + "and " + e.getValue());
- ColumnDefinition column = metadata.getColumnDefinition(name);
+ ColumnMetadata column = metadata.getColumn(name);
if (column == null || column.isStatic() != isStatic)
{
// TODO: this imply we don't read data for a column we don't yet know about, which imply this is theoretically
@@ -333,9 +332,9 @@
// If we don't find the definition, it could be we have data for a dropped column, and we shouldn't
// fail deserialization because of that. So we grab a "fake" ColumnDefinition that ensure proper
// deserialization. The column will be ignore later on anyway.
- column = metadata.getDroppedColumnDefinition(name, isStatic);
+ column = metadata.getDroppedColumn(name, isStatic);
if (column == null)
- throw new RuntimeException("Unknown column " + UTF8Type.instance.getString(name) + " during deserialization");
+ throw new UnknownColumnException("Unknown column " + UTF8Type.instance.getString(name) + " during deserialization");
}
builder.add(column);
}
@@ -417,11 +416,11 @@
}
}
- public SerializationHeader deserializeForMessaging(DataInputPlus in, CFMetaData metadata, ColumnFilter selection, boolean hasStatic) throws IOException
+ public SerializationHeader deserializeForMessaging(DataInputPlus in, TableMetadata metadata, ColumnFilter selection, boolean hasStatic) throws IOException
{
EncodingStats stats = EncodingStats.serializer.deserialize(in);
- AbstractType<?> keyType = metadata.getKeyValidator();
+ AbstractType<?> keyType = metadata.partitionKeyType;
List<AbstractType<?>> clusteringTypes = metadata.comparator.subtypes();
Columns statics, regulars;
@@ -436,7 +435,7 @@
regulars = Columns.serializer.deserializeSubset(selection.fetchedColumns().regulars, in);
}
- return new SerializationHeader(false, keyType, clusteringTypes, new PartitionColumns(statics, regulars), stats, null);
+ return new SerializationHeader(false, keyType, clusteringTypes, new RegularAndStaticColumns(statics, regulars), stats, null);
}
public long serializedSizeForMessaging(SerializationHeader header, ColumnFilter selection, boolean hasStatic)
diff --git a/src/java/org/apache/cassandra/db/Serializers.java b/src/java/org/apache/cassandra/db/Serializers.java
deleted file mode 100644
index 02c9995..0000000
--- a/src/java/org/apache/cassandra/db/Serializers.java
+++ /dev/null
@@ -1,186 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.db;
-
-import java.io.*;
-import java.nio.ByteBuffer;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.db.marshal.AbstractType;
-import org.apache.cassandra.db.marshal.CompositeType;
-import org.apache.cassandra.io.ISerializer;
-import org.apache.cassandra.io.sstable.IndexInfo;
-import org.apache.cassandra.io.sstable.format.big.BigFormat;
-import org.apache.cassandra.io.util.DataInputPlus;
-import org.apache.cassandra.io.util.DataOutputPlus;
-import org.apache.cassandra.io.sstable.format.Version;
-import org.apache.cassandra.utils.ByteBufferUtil;
-
-/**
- * Holds references on serializers that depend on the table definition.
- */
-public class Serializers
-{
- private final CFMetaData metadata;
-
- private Map<Version, IndexInfo.Serializer> otherVersionClusteringSerializers;
-
- private final IndexInfo.Serializer latestVersionIndexSerializer;
-
- public Serializers(CFMetaData metadata)
- {
- this.metadata = metadata;
- this.latestVersionIndexSerializer = new IndexInfo.Serializer(BigFormat.latestVersion,
- indexEntryClusteringPrefixSerializer(BigFormat.latestVersion, SerializationHeader.makeWithoutStats(metadata)));
- }
-
- IndexInfo.Serializer indexInfoSerializer(Version version, SerializationHeader header)
- {
- // null header indicates streaming from pre-3.0 sstables
- if (version.equals(BigFormat.latestVersion) && header != null)
- return latestVersionIndexSerializer;
-
- if (otherVersionClusteringSerializers == null)
- otherVersionClusteringSerializers = new ConcurrentHashMap<>();
- IndexInfo.Serializer serializer = otherVersionClusteringSerializers.get(version);
- if (serializer == null)
- {
- serializer = new IndexInfo.Serializer(version,
- indexEntryClusteringPrefixSerializer(version, header));
- otherVersionClusteringSerializers.put(version, serializer);
- }
- return serializer;
- }
-
- // TODO: Once we drop support for old (pre-3.0) sstables, we can drop this method and inline the calls to
- // ClusteringPrefix.serializer directly. At which point this whole class probably becomes
- // unecessary (since IndexInfo.Serializer won't depend on the metadata either).
- private ISerializer<ClusteringPrefix> indexEntryClusteringPrefixSerializer(Version version, SerializationHeader header)
- {
- if (!version.storeRows() || header == null) //null header indicates streaming from pre-3.0 sstables
- {
- return oldFormatSerializer(version);
- }
-
- return new NewFormatSerializer(version, header.clusteringTypes());
- }
-
- private ISerializer<ClusteringPrefix> oldFormatSerializer(Version version)
- {
- return new ISerializer<ClusteringPrefix>()
- {
- List<AbstractType<?>> clusteringTypes = SerializationHeader.makeWithoutStats(metadata).clusteringTypes();
-
- public void serialize(ClusteringPrefix clustering, DataOutputPlus out) throws IOException
- {
- //we deserialize in the old format and serialize in the new format
- ClusteringPrefix.serializer.serialize(clustering, out,
- version.correspondingMessagingVersion(),
- clusteringTypes);
- }
-
- @Override
- public void skip(DataInputPlus in) throws IOException
- {
- ByteBufferUtil.skipShortLength(in);
- }
-
- public ClusteringPrefix deserialize(DataInputPlus in) throws IOException
- {
- // We're reading the old cellname/composite
- ByteBuffer bb = ByteBufferUtil.readWithShortLength(in);
- assert bb.hasRemaining(); // empty cellnames were invalid
-
- int clusteringSize = metadata.clusteringColumns().size();
- // If the table has no clustering column, then the cellname will just be the "column" name, which we ignore here.
- if (clusteringSize == 0)
- return Clustering.EMPTY;
-
- if (metadata.isCompound() && CompositeType.isStaticName(bb))
- return Clustering.STATIC_CLUSTERING;
-
- if (!metadata.isCompound())
- return Clustering.make(bb);
-
- List<ByteBuffer> components = CompositeType.splitName(bb);
- byte eoc = CompositeType.lastEOC(bb);
-
- if (eoc == 0 || components.size() >= clusteringSize)
- {
- // That's a clustering.
- if (components.size() > clusteringSize)
- components = components.subList(0, clusteringSize);
-
- return Clustering.make(components.toArray(new ByteBuffer[clusteringSize]));
- }
- else
- {
- // It's a range tombstone bound. It is a start since that's the only part we've ever included
- // in the index entries.
- ClusteringPrefix.Kind boundKind = eoc > 0
- ? ClusteringPrefix.Kind.EXCL_START_BOUND
- : ClusteringPrefix.Kind.INCL_START_BOUND;
-
- return ClusteringBound.create(boundKind, components.toArray(new ByteBuffer[components.size()]));
- }
- }
-
- public long serializedSize(ClusteringPrefix clustering)
- {
- return ClusteringPrefix.serializer.serializedSize(clustering, version.correspondingMessagingVersion(),
- clusteringTypes);
- }
- };
- }
-
- private static class NewFormatSerializer implements ISerializer<ClusteringPrefix>
- {
- private final Version version;
- private final List<AbstractType<?>> clusteringTypes;
-
- NewFormatSerializer(Version version, List<AbstractType<?>> clusteringTypes)
- {
- this.version = version;
- this.clusteringTypes = clusteringTypes;
- }
-
- public void serialize(ClusteringPrefix clustering, DataOutputPlus out) throws IOException
- {
- ClusteringPrefix.serializer.serialize(clustering, out, version.correspondingMessagingVersion(), clusteringTypes);
- }
-
- @Override
- public void skip(DataInputPlus in) throws IOException
- {
- ClusteringPrefix.serializer.skip(in, version.correspondingMessagingVersion(), clusteringTypes);
- }
-
- public ClusteringPrefix deserialize(DataInputPlus in) throws IOException
- {
- return ClusteringPrefix.serializer.deserialize(in, version.correspondingMessagingVersion(), clusteringTypes);
- }
-
- public long serializedSize(ClusteringPrefix clustering)
- {
- return ClusteringPrefix.serializer.serializedSize(clustering, version.correspondingMessagingVersion(), clusteringTypes);
- }
- }
-}
diff --git a/src/java/org/apache/cassandra/db/SimpleBuilders.java b/src/java/org/apache/cassandra/db/SimpleBuilders.java
index 6e65743..0fb40a7 100644
--- a/src/java/org/apache/cassandra/db/SimpleBuilders.java
+++ b/src/java/org/apache/cassandra/db/SimpleBuilders.java
@@ -20,9 +20,10 @@
import java.nio.ByteBuffer;
import java.util.*;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
-import org.apache.cassandra.config.Schema;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.cql3.ColumnIdentifier;
import org.apache.cassandra.db.context.CounterContext;
import org.apache.cassandra.db.partitions.PartitionUpdate;
@@ -43,16 +44,16 @@
{
}
- private static DecoratedKey makePartitonKey(CFMetaData metadata, Object... partitionKey)
+ private static DecoratedKey makePartitonKey(TableMetadata metadata, Object... partitionKey)
{
if (partitionKey.length == 1 && partitionKey[0] instanceof DecoratedKey)
return (DecoratedKey)partitionKey[0];
- ByteBuffer key = CFMetaData.serializePartitionKey(metadata.getKeyValidatorAsClusteringComparator().make(partitionKey));
- return metadata.decorateKey(key);
+ ByteBuffer key = metadata.partitionKeyAsClusteringComparator().make(partitionKey).serializeAsPartitionKey();
+ return metadata.partitioner.decorateKey(key);
}
- private static Clustering makeClustering(CFMetaData metadata, Object... clusteringColumns)
+ private static Clustering makeClustering(TableMetadata metadata, Object... clusteringColumns)
{
if (clusteringColumns.length == 1 && clusteringColumns[0] instanceof Clustering)
return (Clustering)clusteringColumns[0];
@@ -61,7 +62,7 @@
{
// If the table has clustering columns, passing no values is for updating the static values, so check we
// do have some static columns defined.
- assert metadata.comparator.size() == 0 || !metadata.partitionColumns().statics.isEmpty();
+ assert metadata.comparator.size() == 0 || !metadata.staticColumns().isEmpty();
return metadata.comparator.size() == 0 ? Clustering.EMPTY : Clustering.STATIC_CLUSTERING;
}
else
@@ -107,7 +108,7 @@
private final String keyspaceName;
private final DecoratedKey key;
- private final Map<UUID, PartitionUpdateBuilder> updateBuilders = new HashMap<>();
+ private final Map<TableId, PartitionUpdateBuilder> updateBuilders = new HashMap<>();
public MutationBuilder(String keyspaceName, DecoratedKey key)
{
@@ -115,15 +116,15 @@
this.key = key;
}
- public PartitionUpdate.SimpleBuilder update(CFMetaData metadata)
+ public PartitionUpdate.SimpleBuilder update(TableMetadata metadata)
{
- assert metadata.ksName.equals(keyspaceName);
+ assert metadata.keyspace.equals(keyspaceName);
- PartitionUpdateBuilder builder = updateBuilders.get(metadata.cfId);
+ PartitionUpdateBuilder builder = updateBuilders.get(metadata.id);
if (builder == null)
{
builder = new PartitionUpdateBuilder(metadata, key);
- updateBuilders.put(metadata.cfId, builder);
+ updateBuilders.put(metadata.id, builder);
}
copyParams(builder);
@@ -133,7 +134,7 @@
public PartitionUpdate.SimpleBuilder update(String tableName)
{
- CFMetaData metadata = Schema.instance.getCFMetaData(keyspaceName, tableName);
+ TableMetadata metadata = Schema.instance.getTableMetadata(keyspaceName, tableName);
assert metadata != null : "Unknown table " + tableName + " in keyspace " + keyspaceName;
return update(metadata);
}
@@ -145,29 +146,30 @@
if (updateBuilders.size() == 1)
return new Mutation(updateBuilders.values().iterator().next().build());
- Mutation mutation = new Mutation(keyspaceName, key);
+ Mutation.PartitionUpdateCollector mutationBuilder = new Mutation.PartitionUpdateCollector(keyspaceName, key);
for (PartitionUpdateBuilder builder : updateBuilders.values())
- mutation.add(builder.build());
- return mutation;
+ mutationBuilder.add(builder.build());
+ return mutationBuilder.build();
}
}
public static class PartitionUpdateBuilder extends AbstractBuilder<PartitionUpdate.SimpleBuilder> implements PartitionUpdate.SimpleBuilder
{
- private final CFMetaData metadata;
+ private final TableMetadata metadata;
private final DecoratedKey key;
private final Map<Clustering, RowBuilder> rowBuilders = new HashMap<>();
private List<RTBuilder> rangeBuilders = null; // We use that rarely, so create lazily
+ private List<RangeTombstone> rangeTombstones = null;
private DeletionTime partitionDeletion = DeletionTime.LIVE;
- public PartitionUpdateBuilder(CFMetaData metadata, Object... partitionKeyValues)
+ public PartitionUpdateBuilder(TableMetadata metadata, Object... partitionKeyValues)
{
this.metadata = metadata;
this.key = makePartitonKey(metadata, partitionKeyValues);
}
- public CFMetaData metadata()
+ public TableMetadata metadata()
{
return metadata;
}
@@ -203,16 +205,24 @@
return builder;
}
+ public PartitionUpdate.SimpleBuilder addRangeTombstone(RangeTombstone rt)
+ {
+ if (rangeTombstones == null)
+ rangeTombstones = new ArrayList<>();
+ rangeTombstones.add(rt);
+ return this;
+ }
+
public PartitionUpdate build()
{
// Collect all updated columns
- PartitionColumns.Builder columns = PartitionColumns.builder();
+ RegularAndStaticColumns.Builder columns = RegularAndStaticColumns.builder();
for (RowBuilder builder : rowBuilders.values())
columns.addAll(builder.columns());
// Note that rowBuilders.size() could include the static column so could be 1 off the really need capacity
// of the final PartitionUpdate, but as that's just a sizing hint, we'll live.
- PartitionUpdate update = new PartitionUpdate(metadata, key, columns.build(), rowBuilders.size());
+ PartitionUpdate.Builder update = new PartitionUpdate.Builder(metadata, key, columns.build(), rowBuilders.size());
update.addPartitionDeletion(partitionDeletion);
if (rangeBuilders != null)
@@ -221,10 +231,16 @@
update.add(builder.build());
}
+ if (rangeTombstones != null)
+ {
+ for (RangeTombstone rt : rangeTombstones)
+ update.add(rt);
+ }
+
for (RowBuilder builder : rowBuilders.values())
update.add(builder.build());
- return update;
+ return update.build();
}
public Mutation buildAsMutation()
@@ -296,23 +312,23 @@
public static class RowBuilder extends AbstractBuilder<Row.SimpleBuilder> implements Row.SimpleBuilder
{
- private final CFMetaData metadata;
+ private final TableMetadata metadata;
- private final Set<ColumnDefinition> columns = new HashSet<>();
+ private final Set<ColumnMetadata> columns = new HashSet<>();
private final Row.Builder builder;
private boolean initiated;
private boolean noPrimaryKeyLivenessInfo;
- public RowBuilder(CFMetaData metadata, Object... clusteringColumns)
+ public RowBuilder(TableMetadata metadata, Object... clusteringColumns)
{
this.metadata = metadata;
- this.builder = BTreeRow.unsortedBuilder(FBUtilities.nowInSeconds());
+ this.builder = BTreeRow.unsortedBuilder();
this.builder.newRow(makeClustering(metadata, clusteringColumns));
}
- Set<ColumnDefinition> columns()
+ Set<ColumnMetadata> columns()
{
return columns;
}
@@ -345,7 +361,7 @@
private Row.SimpleBuilder add(String columnName, Object value, boolean overwriteForCollection)
{
maybeInit();
- ColumnDefinition column = getColumn(columnName);
+ ColumnMetadata column = getColumn(columnName);
if (!overwriteForCollection && !(column.type.isMultiCell() && column.type.isCollection()))
throw new IllegalArgumentException("appendAll() can only be called on non-frozen colletions");
@@ -421,16 +437,16 @@
return builder.build();
}
- private ColumnDefinition getColumn(String columnName)
+ private ColumnMetadata getColumn(String columnName)
{
- ColumnDefinition column = metadata.getColumnDefinition(new ColumnIdentifier(columnName, true));
+ ColumnMetadata column = metadata.getColumn(new ColumnIdentifier(columnName, true));
assert column != null : "Cannot find column " + columnName;
assert !column.isPrimaryKeyColumn();
assert !column.isStatic() || builder.clustering() == Clustering.STATIC_CLUSTERING : "Cannot add non-static column to static-row";
return column;
}
- private Cell cell(ColumnDefinition column, ByteBuffer value, CellPath path)
+ private Cell cell(ColumnMetadata column, ByteBuffer value, CellPath path)
{
if (value == null)
return BufferCell.tombstone(column, timestamp, nowInSec, path);
diff --git a/src/java/org/apache/cassandra/db/SinglePartitionReadCommand.java b/src/java/org/apache/cassandra/db/SinglePartitionReadCommand.java
index a820a89..6fbe523 100644
--- a/src/java/org/apache/cassandra/db/SinglePartitionReadCommand.java
+++ b/src/java/org/apache/cassandra/db/SinglePartitionReadCommand.java
@@ -20,23 +20,18 @@
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.*;
-import java.util.stream.Collectors;
+import java.util.concurrent.TimeUnit;
-import com.google.common.collect.Iterables;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Sets;
-import org.apache.commons.lang3.tuple.Pair;
-
import org.apache.cassandra.cache.IRowCacheEntry;
import org.apache.cassandra.cache.RowCacheKey;
import org.apache.cassandra.cache.RowCacheSentinel;
import org.apache.cassandra.concurrent.Stage;
-import org.apache.cassandra.concurrent.StageManager;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.db.lifecycle.*;
import org.apache.cassandra.db.filter.*;
+import org.apache.cassandra.db.lifecycle.*;
import org.apache.cassandra.db.partitions.*;
import org.apache.cassandra.db.rows.*;
import org.apache.cassandra.db.transform.RTBoundValidator;
@@ -47,46 +42,40 @@
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.metrics.TableMetrics;
-import org.apache.cassandra.net.MessageOut;
-import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.net.Verb;
+import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.schema.IndexMetadata;
-import org.apache.cassandra.service.CacheService;
-import org.apache.cassandra.service.ClientState;
-import org.apache.cassandra.service.StorageProxy;
-import org.apache.cassandra.service.pager.*;
-import org.apache.cassandra.thrift.ThriftResultsMerger;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.service.*;
import org.apache.cassandra.tracing.Tracing;
-import org.apache.cassandra.transport.ProtocolVersion;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.SearchIterator;
import org.apache.cassandra.utils.btree.BTreeSet;
-
/**
* A read command that selects a (part of a) single partition.
*/
-public class SinglePartitionReadCommand extends ReadCommand
+public class SinglePartitionReadCommand extends ReadCommand implements SinglePartitionReadQuery
{
protected static final SelectionDeserializer selectionDeserializer = new Deserializer();
private final DecoratedKey partitionKey;
private final ClusteringIndexFilter clusteringIndexFilter;
- private int oldestUnrepairedTombstone = Integer.MAX_VALUE;
-
- private SinglePartitionReadCommand(boolean isDigest,
- int digestVersion,
- boolean isForThrift,
- CFMetaData metadata,
- int nowInSec,
- ColumnFilter columnFilter,
- RowFilter rowFilter,
- DataLimits limits,
- DecoratedKey partitionKey,
- ClusteringIndexFilter clusteringIndexFilter,
- IndexMetadata index)
+ @VisibleForTesting
+ protected SinglePartitionReadCommand(boolean isDigest,
+ int digestVersion,
+ boolean acceptsTransient,
+ TableMetadata metadata,
+ int nowInSec,
+ ColumnFilter columnFilter,
+ RowFilter rowFilter,
+ DataLimits limits,
+ DecoratedKey partitionKey,
+ ClusteringIndexFilter clusteringIndexFilter,
+ IndexMetadata index)
{
- super(Kind.SINGLE_PARTITION, isDigest, digestVersion, isForThrift, metadata, nowInSec, columnFilter, rowFilter, limits, index);
+ super(Kind.SINGLE_PARTITION, isDigest, digestVersion, acceptsTransient, metadata, nowInSec, columnFilter, rowFilter, limits, index);
assert partitionKey.getPartitioner() == metadata.partitioner;
this.partitionKey = partitionKey;
this.clusteringIndexFilter = clusteringIndexFilter;
@@ -95,7 +84,6 @@
/**
* Creates a new read command on a single partition.
*
- * @param isForThrift whether the query is for thrift or not.
* @param metadata the table to query.
* @param nowInSec the time in seconds to use are "now" for this query.
* @param columnFilter the column filter to use for the query.
@@ -107,8 +95,7 @@
*
* @return a newly created read command.
*/
- public static SinglePartitionReadCommand create(boolean isForThrift,
- CFMetaData metadata,
+ public static SinglePartitionReadCommand create(TableMetadata metadata,
int nowInSec,
ColumnFilter columnFilter,
RowFilter rowFilter,
@@ -119,7 +106,7 @@
{
return new SinglePartitionReadCommand(false,
0,
- isForThrift,
+ false,
metadata,
nowInSec,
columnFilter,
@@ -143,7 +130,7 @@
*
* @return a newly created read command.
*/
- public static SinglePartitionReadCommand create(CFMetaData metadata,
+ public static SinglePartitionReadCommand create(TableMetadata metadata,
int nowInSec,
ColumnFilter columnFilter,
RowFilter rowFilter,
@@ -151,34 +138,7 @@
DecoratedKey partitionKey,
ClusteringIndexFilter clusteringIndexFilter)
{
- return create(false, metadata, nowInSec, columnFilter, rowFilter, limits, partitionKey, clusteringIndexFilter);
- }
-
- /**
- * Creates a new read command on a single partition.
- *
- * @param isForThrift whether the query is for thrift or not.
- * @param metadata the table to query.
- * @param nowInSec the time in seconds to use are "now" for this query.
- * @param columnFilter the column filter to use for the query.
- * @param rowFilter the row filter to use for the query.
- * @param limits the limits to use for the query.
- * @param partitionKey the partition key for the partition to query.
- * @param clusteringIndexFilter the clustering index filter to use for the query.
- *
- * @return a newly created read command.
- */
- public static SinglePartitionReadCommand create(boolean isForThrift,
- CFMetaData metadata,
- int nowInSec,
- ColumnFilter columnFilter,
- RowFilter rowFilter,
- DataLimits limits,
- DecoratedKey partitionKey,
- ClusteringIndexFilter clusteringIndexFilter)
- {
- return create(isForThrift,
- metadata,
+ return create(metadata,
nowInSec,
columnFilter,
rowFilter,
@@ -199,7 +159,7 @@
*
* @return a newly created read command. The returned command will use no row filter and have no limits.
*/
- public static SinglePartitionReadCommand create(CFMetaData metadata,
+ public static SinglePartitionReadCommand create(TableMetadata metadata,
int nowInSec,
DecoratedKey key,
ColumnFilter columnFilter,
@@ -217,7 +177,7 @@
*
* @return a newly created read command that queries all the rows of {@code key}.
*/
- public static SinglePartitionReadCommand fullPartitionRead(CFMetaData metadata, int nowInSec, DecoratedKey key)
+ public static SinglePartitionReadCommand fullPartitionRead(TableMetadata metadata, int nowInSec, DecoratedKey key)
{
return create(metadata, nowInSec, key, Slices.ALL);
}
@@ -231,9 +191,9 @@
*
* @return a newly created read command that queries all the rows of {@code key}.
*/
- public static SinglePartitionReadCommand fullPartitionRead(CFMetaData metadata, int nowInSec, ByteBuffer key)
+ public static SinglePartitionReadCommand fullPartitionRead(TableMetadata metadata, int nowInSec, ByteBuffer key)
{
- return create(metadata, nowInSec, metadata.decorateKey(key), Slices.ALL);
+ return create(metadata, nowInSec, metadata.partitioner.decorateKey(key), Slices.ALL);
}
/**
@@ -247,7 +207,7 @@
* @return a newly created read command that queries {@code slice} in {@code key}. The returned query will
* query every columns for the table (without limit or row filtering) and be in forward order.
*/
- public static SinglePartitionReadCommand create(CFMetaData metadata, int nowInSec, DecoratedKey key, Slice slice)
+ public static SinglePartitionReadCommand create(TableMetadata metadata, int nowInSec, DecoratedKey key, Slice slice)
{
return create(metadata, nowInSec, key, Slices.with(metadata.comparator, slice));
}
@@ -263,7 +223,7 @@
* @return a newly created read command that queries the {@code slices} in {@code key}. The returned query will
* query every columns for the table (without limit or row filtering) and be in forward order.
*/
- public static SinglePartitionReadCommand create(CFMetaData metadata, int nowInSec, DecoratedKey key, Slices slices)
+ public static SinglePartitionReadCommand create(TableMetadata metadata, int nowInSec, DecoratedKey key, Slices slices)
{
ClusteringIndexSliceFilter filter = new ClusteringIndexSliceFilter(slices, false);
return create(metadata, nowInSec, ColumnFilter.all(metadata), RowFilter.NONE, DataLimits.NONE, key, filter);
@@ -280,9 +240,9 @@
* @return a newly created read command that queries the {@code slices} in {@code key}. The returned query will
* query every columns for the table (without limit or row filtering) and be in forward order.
*/
- public static SinglePartitionReadCommand create(CFMetaData metadata, int nowInSec, ByteBuffer key, Slices slices)
+ public static SinglePartitionReadCommand create(TableMetadata metadata, int nowInSec, ByteBuffer key, Slices slices)
{
- return create(metadata, nowInSec, metadata.decorateKey(key), slices);
+ return create(metadata, nowInSec, metadata.partitioner.decorateKey(key), slices);
}
/**
@@ -296,7 +256,7 @@
* @return a newly created read command that queries the {@code names} in {@code key}. The returned query will
* query every columns (without limit or row filtering) and be in forward order.
*/
- public static SinglePartitionReadCommand create(CFMetaData metadata, int nowInSec, DecoratedKey key, NavigableSet<Clustering> names)
+ public static SinglePartitionReadCommand create(TableMetadata metadata, int nowInSec, DecoratedKey key, NavigableSet<Clustering> names)
{
ClusteringIndexNamesFilter filter = new ClusteringIndexNamesFilter(names, false);
return create(metadata, nowInSec, ColumnFilter.all(metadata), RowFilter.NONE, DataLimits.NONE, key, filter);
@@ -313,7 +273,7 @@
* @return a newly created read command that queries {@code name} in {@code key}. The returned query will
* query every columns (without limit or row filtering).
*/
- public static SinglePartitionReadCommand create(CFMetaData metadata, int nowInSec, DecoratedKey key, Clustering name)
+ public static SinglePartitionReadCommand create(TableMetadata metadata, int nowInSec, DecoratedKey key, Clustering name)
{
return create(metadata, nowInSec, key, FBUtilities.singleton(name, metadata.comparator));
}
@@ -322,7 +282,7 @@
{
return new SinglePartitionReadCommand(isDigestQuery(),
digestVersion(),
- isForThrift(),
+ acceptsTransient(),
metadata(),
nowInSec(),
columnFilter(),
@@ -333,11 +293,12 @@
indexMetadata());
}
- public SinglePartitionReadCommand copyAsDigestQuery()
+ @Override
+ protected SinglePartitionReadCommand copyAsDigestQuery()
{
return new SinglePartitionReadCommand(true,
digestVersion(),
- isForThrift(),
+ acceptsTransient(),
metadata(),
nowInSec(),
columnFilter(),
@@ -348,11 +309,28 @@
indexMetadata());
}
+ @Override
+ protected SinglePartitionReadCommand copyAsTransientQuery()
+ {
+ return new SinglePartitionReadCommand(false,
+ 0,
+ true,
+ metadata(),
+ nowInSec(),
+ columnFilter(),
+ rowFilter(),
+ limits(),
+ partitionKey(),
+ clusteringIndexFilter(),
+ indexMetadata());
+ }
+
+ @Override
public SinglePartitionReadCommand withUpdatedLimit(DataLimits newLimits)
{
return new SinglePartitionReadCommand(isDigestQuery(),
digestVersion(),
- isForThrift(),
+ acceptsTransient(),
metadata(),
nowInSec(),
columnFilter(),
@@ -363,61 +341,13 @@
indexMetadata());
}
- public SinglePartitionReadCommand withUpdatedClusteringIndexFilter(ClusteringIndexFilter filter)
- {
- return new SinglePartitionReadCommand(isDigestQuery(),
- digestVersion(),
- isForThrift(),
- metadata(),
- nowInSec(),
- columnFilter(),
- rowFilter(),
- limits(),
- partitionKey(),
- filter,
- indexMetadata());
- }
-
- static SinglePartitionReadCommand legacySliceCommand(boolean isDigest,
- int digestVersion,
- CFMetaData metadata,
- int nowInSec,
- ColumnFilter columnFilter,
- DataLimits limits,
- DecoratedKey partitionKey,
- ClusteringIndexSliceFilter filter)
- {
- // messages from old nodes will expect the thrift format, so always use 'true' for isForThrift
- return new SinglePartitionReadCommand(isDigest,
- digestVersion,
- true,
- metadata,
- nowInSec,
- columnFilter,
- RowFilter.NONE,
- limits,
- partitionKey,
- filter,
- null);
- }
-
- static SinglePartitionReadCommand legacyNamesCommand(boolean isDigest,
- int digestVersion,
- CFMetaData metadata,
- int nowInSec,
- ColumnFilter columnFilter,
- DecoratedKey partitionKey,
- ClusteringIndexNamesFilter filter)
- {
- // messages from old nodes will expect the thrift format, so always use 'true' for isForThrift
- return new SinglePartitionReadCommand(isDigest, digestVersion, true, metadata, nowInSec, columnFilter, RowFilter.NONE, DataLimits.NONE, partitionKey, filter,null);
- }
-
+ @Override
public DecoratedKey partitionKey()
{
return partitionKey;
}
+ @Override
public ClusteringIndexFilter clusteringIndexFilter()
{
return clusteringIndexFilter;
@@ -428,9 +358,9 @@
return clusteringIndexFilter;
}
- public long getTimeout()
+ public long getTimeout(TimeUnit unit)
{
- return DatabaseDescriptor.getReadRpcTimeout();
+ return DatabaseDescriptor.getReadRpcTimeout(unit);
}
public boolean isReversed()
@@ -438,41 +368,12 @@
return clusteringIndexFilter.isReversed();
}
- public boolean selectsKey(DecoratedKey key)
- {
- if (!this.partitionKey().equals(key))
- return false;
-
- return rowFilter().partitionKeyRestrictionsAreSatisfiedBy(key, metadata().getKeyValidator());
- }
-
- public boolean selectsClustering(DecoratedKey key, Clustering clustering)
- {
- if (clustering == Clustering.STATIC_CLUSTERING)
- return !columnFilter().fetchedColumns().statics.isEmpty();
-
- if (!clusteringIndexFilter().selects(clustering))
- return false;
-
- return rowFilter().clusteringKeyRestrictionsAreSatisfiedBy(clustering);
- }
-
- /**
- * Returns a new command suitable to paging from the last returned row.
- *
- * @param lastReturned the last row returned by the previous page. The newly created command
- * will only query row that comes after this (in query order). This can be {@code null} if this
- * is the first page.
- * @param limits the limits to use for the page to query.
- *
- * @return the newly create command.
- */
+ @Override
public SinglePartitionReadCommand forPaging(Clustering lastReturned, DataLimits limits)
{
// We shouldn't have set digest yet when reaching that point
assert !isDigestQuery();
- return create(isForThrift(),
- metadata(),
+ return create(metadata(),
nowInSec(),
columnFilter(),
rowFilter(),
@@ -486,16 +387,6 @@
return StorageProxy.read(Group.one(this), consistency, clientState, queryStartNanoTime);
}
- public SinglePartitionPager getPager(PagingState pagingState, ProtocolVersion protocolVersion)
- {
- return getPager(this, pagingState, protocolVersion);
- }
-
- private static SinglePartitionPager getPager(SinglePartitionReadCommand command, PagingState pagingState, ProtocolVersion protocolVersion)
- {
- return new SinglePartitionPager(command, pagingState, protocolVersion);
- }
-
protected void recordLatency(TableMetrics metric, long latencyNanos)
{
metric.readLatency.addNano(latencyNanos);
@@ -504,10 +395,12 @@
@SuppressWarnings("resource") // we close the created iterator through closing the result of this method (and SingletonUnfilteredPartitionIterator ctor cannot fail)
protected UnfilteredPartitionIterator queryStorage(final ColumnFamilyStore cfs, ReadExecutionController executionController)
{
- UnfilteredRowIterator partition = cfs.isRowCacheEnabled()
+ // skip the row cache and go directly to sstables/memtable if repaired status of
+ // data is being tracked. This is only requested after an initial digest mismatch
+ UnfilteredRowIterator partition = cfs.isRowCacheEnabled() && !isTrackingRepairedStatus()
? getThroughCache(cfs, executionController)
: queryMemtableAndDisk(cfs, executionController);
- return new SingletonUnfilteredPartitionIterator(partition, isForThrift());
+ return new SingletonUnfilteredPartitionIterator(partition);
}
/**
@@ -519,12 +412,13 @@
* If the partition is is not cached, we figure out what filter is "biggest", read
* that from disk, then filter the result and either cache that or return it.
*/
+ @SuppressWarnings("resource")
private UnfilteredRowIterator getThroughCache(ColumnFamilyStore cfs, ReadExecutionController executionController)
{
assert !cfs.isIndex(); // CASSANDRA-5732
assert cfs.isRowCacheEnabled() : String.format("Row cache is not enabled on table [%s]", cfs.name);
- RowCacheKey key = new RowCacheKey(metadata().ksAndCFName, partitionKey());
+ RowCacheKey key = new RowCacheKey(metadata(), partitionKey());
// Attempt a sentinel-read-cache sequence. if a write invalidates our sentinel, we'll return our
// (now potentially obsolete) data, but won't cache it. see CASSANDRA-3862
@@ -541,7 +435,7 @@
}
CachedPartition cachedPartition = (CachedPartition)cached;
- if (cfs.isFilterFullyCoveredBy(clusteringIndexFilter(), limits(), cachedPartition, nowInSec()))
+ if (cfs.isFilterFullyCoveredBy(clusteringIndexFilter(), limits(), cachedPartition, nowInSec(), metadata().enforceStrictLiveness()))
{
cfs.metric.rowCacheHit.inc();
Tracing.trace("Row cache hit");
@@ -613,7 +507,6 @@
// We want to cache only rowsToCache rows
CachedPartition toCache = CachedBTreePartition.create(toCacheIterator, nowInSec());
-
if (sentinelSuccess && !toCache.isEmpty())
{
Tracing.trace("Caching {} rows", toCache.rowCount());
@@ -675,12 +568,6 @@
return queryMemtableAndDiskInternal(cfs);
}
- @Override
- protected int oldestUnrepairedTombstone()
- {
- return oldestUnrepairedTombstone;
- }
-
private UnfilteredRowIterator queryMemtableAndDiskInternal(ColumnFamilyStore cfs)
{
/*
@@ -693,16 +580,19 @@
* and if we have neither non-frozen collections/UDTs nor counters (indeed, for a non-frozen collection or UDT,
* we can't guarantee an older sstable won't have some elements that weren't in the most recent sstables,
* and counters are intrinsically a collection of shards and so have the same problem).
+ * Also, if tracking repaired data then we skip this optimization so we can collate the repaired sstables
+ * and generate a digest over their merge, which procludes an early return.
*/
- if (clusteringIndexFilter() instanceof ClusteringIndexNamesFilter && !queriesMulticellType())
+ if (clusteringIndexFilter() instanceof ClusteringIndexNamesFilter && !queriesMulticellType() && !isTrackingRepairedStatus())
return queryMemtableAndSSTablesInTimestampOrder(cfs, (ClusteringIndexNamesFilter)clusteringIndexFilter());
Tracing.trace("Acquiring sstable references");
ColumnFamilyStore.ViewFragment view = cfs.select(View.select(SSTableSet.LIVE, partitionKey()));
- List<UnfilteredRowIterator> iterators = new ArrayList<>(Iterables.size(view.memtables) + view.sstables.size());
+ Collections.sort(view.sstables, SSTableReader.maxTimestampDescending);
ClusteringIndexFilter filter = clusteringIndexFilter();
long minTimestamp = Long.MAX_VALUE;
-
+ long mostRecentPartitionTombstone = Long.MIN_VALUE;
+ InputCollector<UnfilteredRowIterator> inputCollector = iteratorsForPartition(view);
try
{
for (Memtable memtable : view.memtables)
@@ -715,10 +605,13 @@
@SuppressWarnings("resource") // 'iter' is added to iterators which is closed on exception, or through the closing of the final merged iterator
UnfilteredRowIterator iter = filter.getUnfilteredRowIterator(columnFilter(), partition);
- if (isForThrift())
- iter = ThriftResultsMerger.maybeWrap(iter, nowInSec());
+
+ // Memtable data is always considered unrepaired
oldestUnrepairedTombstone = Math.min(oldestUnrepairedTombstone, partition.stats().minLocalDeletionTime);
- iterators.add(RTBoundValidator.validate(iter, RTBoundValidator.Stage.MEMTABLE, false));
+ inputCollector.addMemtableIterator(RTBoundValidator.validate(iter, RTBoundValidator.Stage.MEMTABLE, false));
+
+ mostRecentPartitionTombstone = Math.max(mostRecentPartitionTombstone,
+ iter.partitionLevelDeletion().markedForDeleteAt());
}
/*
@@ -734,18 +627,26 @@
* elimination in one pass, and minimize the number of sstables for which we read a partition tombstone.
*/
Collections.sort(view.sstables, SSTableReader.maxTimestampDescending);
- long mostRecentPartitionTombstone = Long.MIN_VALUE;
int nonIntersectingSSTables = 0;
int includedDueToTombstones = 0;
SSTableReadMetricsCollector metricsCollector = new SSTableReadMetricsCollector();
+ if (isTrackingRepairedStatus())
+ Tracing.trace("Collecting data from sstables and tracking repaired status");
+
for (SSTableReader sstable : view.sstables)
{
// if we've already seen a partition tombstone with a timestamp greater
// than the most recent update to this sstable, we can skip it
+ // if we're tracking repaired status, we mark the repaired digest inconclusive
+ // as other replicas may not have seen this partition delete and so could include
+ // data from this sstable (or others) in their digests
if (sstable.getMaxTimestamp() < mostRecentPartitionTombstone)
+ {
+ inputCollector.markInconclusive();
break;
+ }
if (shouldInclude(sstable))
{
@@ -754,28 +655,27 @@
// 'iter' is added to iterators which is closed on exception, or through the closing of the final merged iterator
@SuppressWarnings("resource")
- UnfilteredRowIterator iter = makeIterator(cfs, sstable, true, metricsCollector);
- iterators.add(iter);
+ UnfilteredRowIteratorWithLowerBound iter = makeIterator(cfs, sstable, metricsCollector);
+ inputCollector.addSSTableIterator(sstable, iter);
mostRecentPartitionTombstone = Math.max(mostRecentPartitionTombstone,
iter.partitionLevelDeletion().markedForDeleteAt());
}
else
{
-
nonIntersectingSSTables++;
// sstable contains no tombstone if maxLocalDeletionTime == Integer.MAX_VALUE, so we can safely skip those entirely
if (sstable.mayHaveTombstones())
{
// 'iter' is added to iterators which is closed on exception, or through the closing of the final merged iterator
@SuppressWarnings("resource")
- UnfilteredRowIterator iter = makeIterator(cfs, sstable, true, metricsCollector);
+ UnfilteredRowIteratorWithLowerBound iter = makeIterator(cfs, sstable, metricsCollector);
// if the sstable contains a partition delete, then we must include it regardless of whether it
// shadows any other data seen locally as we can't guarantee that other replicas have seen it
if (!iter.partitionLevelDeletion().isLive())
{
if (!sstable.isRepaired())
oldestUnrepairedTombstone = Math.min(oldestUnrepairedTombstone, sstable.getMinLocalDeletionTime());
- iterators.add(iter);
+ inputCollector.addSSTableIterator(sstable, iter);
includedDueToTombstones++;
mostRecentPartitionTombstone = Math.max(mostRecentPartitionTombstone,
iter.partitionLevelDeletion().markedForDeleteAt());
@@ -792,21 +692,22 @@
Tracing.trace("Skipped {}/{} non-slice-intersecting sstables, included {} due to tombstones",
nonIntersectingSSTables, view.sstables.size(), includedDueToTombstones);
- if (iterators.isEmpty())
- return EmptyIterators.unfilteredRow(cfs.metadata, partitionKey(), filter.isReversed());
+ if (inputCollector.isEmpty())
+ return EmptyIterators.unfilteredRow(cfs.metadata(), partitionKey(), filter.isReversed());
- StorageHook.instance.reportRead(cfs.metadata.cfId, partitionKey());
- return withSSTablesIterated(iterators, cfs.metric, metricsCollector);
+ StorageHook.instance.reportRead(cfs.metadata().id, partitionKey());
+
+ return withSSTablesIterated(inputCollector.finalizeIterators(cfs, nowInSec(), oldestUnrepairedTombstone), cfs.metric, metricsCollector);
}
catch (RuntimeException | Error e)
{
try
{
- FBUtilities.closeAll(iterators);
+ inputCollector.close();
}
- catch (Exception suppressed)
+ catch (Exception e1)
{
- e.addSuppressed(suppressed);
+ e.addSuppressed(e1);
}
throw e;
}
@@ -824,8 +725,7 @@
}
private UnfilteredRowIteratorWithLowerBound makeIterator(ColumnFamilyStore cfs,
- final SSTableReader sstable,
- boolean applyThriftTransformation,
+ SSTableReader sstable,
SSTableReadsListener listener)
{
return StorageHook.instance.makeRowIteratorWithLowerBound(cfs,
@@ -833,9 +733,6 @@
sstable,
clusteringIndexFilter(),
columnFilter(),
- isForThrift(),
- nowInSec(),
- applyThriftTransformation,
listener);
}
@@ -845,17 +742,18 @@
* Note that we cannot use the Transformations framework because they greedily get the static row, which
* would cause all iterators to be initialized and hence all sstables to be accessed.
*/
+ @SuppressWarnings("resource")
private UnfilteredRowIterator withSSTablesIterated(List<UnfilteredRowIterator> iterators,
TableMetrics metrics,
SSTableReadMetricsCollector metricsCollector)
{
@SuppressWarnings("resource") // Closed through the closing of the result of the caller method.
- UnfilteredRowIterator merged = UnfilteredRowIterators.merge(iterators, nowInSec());
+ UnfilteredRowIterator merged = UnfilteredRowIterators.merge(iterators);
if (!merged.isEmpty())
{
DecoratedKey key = merged.partitionKey();
- metrics.samplers.get(TableMetrics.Sampler.READS).addSample(key.getKey(), key.hashCode(), 1);
+ metrics.topReadPartitionFrequency.addSample(key.getKey(), 1);
}
class UpdateSstablesIterated extends Transformation
@@ -872,7 +770,7 @@
private boolean queriesMulticellType()
{
- for (ColumnDefinition column : columnFilter().fetchedColumns())
+ for (ColumnMetadata column : columnFilter().fetchedColumns())
{
if (column.type.isMultiCell() || column.type.isCounter())
return true;
@@ -909,7 +807,7 @@
continue;
result = add(
- RTBoundValidator.validate(isForThrift() ? ThriftResultsMerger.maybeWrap(iter, nowInSec()) : iter, RTBoundValidator.Stage.MEMTABLE, false),
+ RTBoundValidator.validate(iter, RTBoundValidator.Stage.MEMTABLE, false),
result,
filter,
false
@@ -951,7 +849,6 @@
filter.getSlices(metadata()),
columnFilter(),
filter.isReversed(),
- isForThrift(),
metricsCollector))
{
if (!iter.partitionLevelDeletion().isLive())
@@ -987,7 +884,6 @@
filter.getSlices(metadata()),
columnFilter(),
filter.isReversed(),
- isForThrift(),
metricsCollector))
{
if (iter.isEmpty())
@@ -997,7 +893,7 @@
onlyUnrepaired = false;
result = add(
- RTBoundValidator.validate(isForThrift() ? ThriftResultsMerger.maybeWrap(iter, nowInSec()) : iter, RTBoundValidator.Stage.SSTABLE, false),
+ RTBoundValidator.validate(iter, RTBoundValidator.Stage.SSTABLE, false),
result,
filter,
sstable.isRepaired()
@@ -1011,8 +907,8 @@
return EmptyIterators.unfilteredRow(metadata(), partitionKey(), false);
DecoratedKey key = result.partitionKey();
- cfs.metric.samplers.get(TableMetrics.Sampler.READS).addSample(key.getKey(), key.hashCode(), 1);
- StorageHook.instance.reportRead(cfs.metadata.cfId, partitionKey());
+ cfs.metric.topReadPartitionFrequency.addSample(key.getKey(), 1);
+ StorageHook.instance.reportRead(cfs.metadata.id, partitionKey());
// "hoist up" the requested data into a more recent sstable
if (metricsCollector.getMergedSSTables() > cfs.getMinimumCompactionThreshold()
@@ -1027,7 +923,7 @@
try (UnfilteredRowIterator iter = result.unfilteredIterator(columnFilter(), Slices.ALL, false))
{
final Mutation mutation = new Mutation(PartitionUpdate.fromIterator(iter, columnFilter()));
- StageManager.getStage(Stage.MUTATION).execute(() -> {
+ Stage.MUTATION.execute(() -> {
// skipping commitlog and index updates is fine since we're just de-fragmenting existing data
Keyspace.open(mutation.getKeyspaceName()).apply(mutation, false, false);
});
@@ -1046,7 +942,7 @@
if (result == null)
return ImmutableBTreePartition.create(iter, maxRows);
- try (UnfilteredRowIterator merged = UnfilteredRowIterators.merge(Arrays.asList(iter, result.unfilteredIterator(columnFilter(), Slices.ALL, filter.isReversed())), nowInSec()))
+ try (UnfilteredRowIterator merged = UnfilteredRowIterators.merge(Arrays.asList(iter, result.unfilteredIterator(columnFilter(), Slices.ALL, filter.isReversed()))))
{
return ImmutableBTreePartition.create(merged, maxRows);
}
@@ -1059,7 +955,7 @@
SearchIterator<Clustering, Row> searchIter = result.searchIterator(columnFilter(), false);
- PartitionColumns columns = columnFilter().fetchedColumns();
+ RegularAndStaticColumns columns = columnFilter().fetchedColumns();
NavigableSet<Clustering> clusterings = filter.requestedRows();
// We want to remove rows for which we have values for all requested columns. We have to deal with both static and regular rows.
@@ -1111,7 +1007,7 @@
if (row.primaryKeyLivenessInfo().isEmpty() || row.primaryKeyLivenessInfo().timestamp() <= sstableTimestamp)
return false;
- for (ColumnDefinition column : requestedColumns)
+ for (ColumnMetadata column : requestedColumns)
{
Cell cell = row.getCell(column);
if (cell == null || cell.timestamp() <= sstableTimestamp)
@@ -1130,28 +1026,28 @@
@Override
public String toString()
{
- return String.format("Read(%s.%s columns=%s rowFilter=%s limits=%s key=%s filter=%s, nowInSec=%d)",
- metadata().ksName,
- metadata().cfName,
+ return String.format("Read(%s columns=%s rowFilter=%s limits=%s key=%s filter=%s, nowInSec=%d)",
+ metadata().toString(),
columnFilter(),
rowFilter(),
limits(),
- metadata().getKeyValidator().getString(partitionKey().getKey()),
+ metadata().partitionKeyType.getString(partitionKey().getKey()),
clusteringIndexFilter.toString(metadata()),
nowInSec());
}
- public MessageOut<ReadCommand> createMessage(int version)
+ @Override
+ public Verb verb()
{
- return new MessageOut<>(MessagingService.Verb.READ, this, readSerializer);
+ return Verb.READ_REQ;
}
protected void appendCQLWhereClause(StringBuilder sb)
{
sb.append(" WHERE ");
- sb.append(ColumnDefinition.toCQLString(metadata().partitionKeyColumns())).append(" = ");
- DataRange.appendKeyString(sb, metadata().getKeyValidator(), partitionKey().getKey());
+ sb.append(ColumnMetadata.toCQLString(metadata().partitionKeyColumns())).append(" = ");
+ DataRange.appendKeyString(sb, metadata().partitionKeyType, partitionKey().getKey());
// We put the row filter first because the clustering index filter can end by "ORDER BY"
if (!rowFilter().isEmpty())
@@ -1164,13 +1060,13 @@
protected void serializeSelection(DataOutputPlus out, int version) throws IOException
{
- metadata().getKeyValidator().writeValue(partitionKey().getKey(), out);
+ metadata().partitionKeyType.writeValue(partitionKey().getKey(), out);
ClusteringIndexFilter.serializer.serialize(clusteringIndexFilter(), out, version);
}
protected long selectionSerializedSize(int version)
{
- return metadata().getKeyValidator().writtenLength(partitionKey().getKey())
+ return metadata().partitionKeyType.writtenLength(partitionKey().getKey())
+ ClusteringIndexFilter.serializer.serializedSize(clusteringIndexFilter(), version);
}
@@ -1179,26 +1075,42 @@
return true;
}
+ public boolean isRangeRequest()
+ {
+ return false;
+ }
+
/**
* Groups multiple single partition read commands.
*/
- public static class Group implements ReadQuery
+ public static class Group extends SinglePartitionReadQuery.Group<SinglePartitionReadCommand>
{
- public final List<SinglePartitionReadCommand> commands;
- private final DataLimits limits;
- private final int nowInSec;
- private final boolean selectsFullPartitions;
+ public static Group create(TableMetadata metadata,
+ int nowInSec,
+ ColumnFilter columnFilter,
+ RowFilter rowFilter,
+ DataLimits limits,
+ List<DecoratedKey> partitionKeys,
+ ClusteringIndexFilter clusteringIndexFilter)
+ {
+ List<SinglePartitionReadCommand> commands = new ArrayList<>(partitionKeys.size());
+ for (DecoratedKey partitionKey : partitionKeys)
+ {
+ commands.add(SinglePartitionReadCommand.create(metadata,
+ nowInSec,
+ columnFilter,
+ rowFilter,
+ limits,
+ partitionKey,
+ clusteringIndexFilter));
+ }
+
+ return new Group(commands, limits);
+ }
public Group(List<SinglePartitionReadCommand> commands, DataLimits limits)
{
- assert !commands.isEmpty();
- this.commands = commands;
- this.limits = limits;
- SinglePartitionReadCommand firstCommand = commands.get(0);
- this.nowInSec = firstCommand.nowInSec();
- this.selectsFullPartitions = firstCommand.selectsFullPartition();
- for (int i = 1; i < commands.size(); i++)
- assert commands.get(i).nowInSec() == nowInSec;
+ super(commands, limits);
}
public static Group one(SinglePartitionReadCommand command)
@@ -1210,97 +1122,6 @@
{
return StorageProxy.read(this, consistency, clientState, queryStartNanoTime);
}
-
- public int nowInSec()
- {
- return nowInSec;
- }
-
- public DataLimits limits()
- {
- return limits;
- }
-
- public CFMetaData metadata()
- {
- return commands.get(0).metadata();
- }
-
- @Override
- public boolean selectsFullPartition()
- {
- return selectsFullPartitions;
- }
-
- public ReadExecutionController executionController()
- {
- // Note that the only difference between the command in a group must be the partition key on which
- // they applied. So as far as ReadOrderGroup is concerned, we can use any of the commands to start one.
- return commands.get(0).executionController();
- }
-
- public PartitionIterator executeInternal(ReadExecutionController controller)
- {
- // Note that the only difference between the command in a group must be the partition key on which
- // they applied.
- boolean enforceStrictLiveness = commands.get(0).metadata().enforceStrictLiveness();
- return limits.filter(UnfilteredPartitionIterators.filter(executeLocally(controller, false), nowInSec),
- nowInSec,
- selectsFullPartitions,
- enforceStrictLiveness);
- }
-
- public UnfilteredPartitionIterator executeLocally(ReadExecutionController executionController)
- {
- return executeLocally(executionController, true);
- }
-
- /**
- * Implementation of {@link ReadQuery#executeLocally(ReadExecutionController)}.
- *
- * @param executionController - the {@code ReadExecutionController} protecting the read.
- * @param sort - whether to sort the inner commands by partition key, required for merging the iterator
- * later on. This will be false when called by {@link ReadQuery#executeInternal(ReadExecutionController)}
- * because in this case it is safe to do so as there is no merging involved and we don't want to
- * change the old behavior which was to not sort by partition.
- *
- * @return - the iterator that can be used to retrieve the query result.
- */
- private UnfilteredPartitionIterator executeLocally(ReadExecutionController executionController, boolean sort)
- {
- List<Pair<DecoratedKey, UnfilteredPartitionIterator>> partitions = new ArrayList<>(commands.size());
- for (SinglePartitionReadCommand cmd : commands)
- partitions.add(Pair.of(cmd.partitionKey, cmd.executeLocally(executionController)));
-
- if (sort)
- Collections.sort(partitions, (p1, p2) -> p1.getLeft().compareTo(p2.getLeft()));
-
- return UnfilteredPartitionIterators.concat(partitions.stream().map(p -> p.getRight()).collect(Collectors.toList()));
- }
-
- public QueryPager getPager(PagingState pagingState, ProtocolVersion protocolVersion)
- {
- if (commands.size() == 1)
- return SinglePartitionReadCommand.getPager(commands.get(0), pagingState, protocolVersion);
-
- return new MultiPartitionPager(this, pagingState, protocolVersion);
- }
-
- public boolean selectsKey(DecoratedKey key)
- {
- return Iterables.any(commands, c -> c.selectsKey(key));
- }
-
- public boolean selectsClustering(DecoratedKey key, Clustering clustering)
- {
- return Iterables.any(commands, c -> c.selectsClustering(key, clustering));
- }
-
- @Override
- public String toString()
- {
- return commands.toString();
- }
}
private static class Deserializer extends SelectionDeserializer
@@ -1309,8 +1130,8 @@
int version,
boolean isDigest,
int digestVersion,
- boolean isForThrift,
- CFMetaData metadata,
+ boolean acceptsTransient,
+ TableMetadata metadata,
int nowInSec,
ColumnFilter columnFilter,
RowFilter rowFilter,
@@ -1318,9 +1139,9 @@
IndexMetadata index)
throws IOException
{
- DecoratedKey key = metadata.decorateKey(metadata.getKeyValidator().readValue(in, DatabaseDescriptor.getMaxValueSize()));
+ DecoratedKey key = metadata.partitioner.decorateKey(metadata.partitionKeyType.readValue(in, DatabaseDescriptor.getMaxValueSize()));
ClusteringIndexFilter filter = ClusteringIndexFilter.serializer.deserialize(in, version, metadata);
- return new SinglePartitionReadCommand(isDigest, digestVersion, isForThrift, metadata, nowInSec, columnFilter, rowFilter, limits, key, filter, index);
+ return new SinglePartitionReadCommand(isDigest, digestVersion, acceptsTransient, metadata, nowInSec, columnFilter, rowFilter, limits, key, filter, index);
}
}
diff --git a/src/java/org/apache/cassandra/db/SinglePartitionReadQuery.java b/src/java/org/apache/cassandra/db/SinglePartitionReadQuery.java
new file mode 100644
index 0000000..f9f0014
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/SinglePartitionReadQuery.java
@@ -0,0 +1,290 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import com.google.common.collect.Iterables;
+
+import org.apache.commons.lang3.tuple.Pair;
+
+import org.apache.cassandra.db.filter.ClusteringIndexFilter;
+import org.apache.cassandra.db.filter.ColumnFilter;
+import org.apache.cassandra.db.filter.DataLimits;
+import org.apache.cassandra.db.filter.RowFilter;
+import org.apache.cassandra.db.partitions.PartitionIterator;
+import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
+import org.apache.cassandra.db.partitions.UnfilteredPartitionIterators;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.service.pager.MultiPartitionPager;
+import org.apache.cassandra.service.pager.PagingState;
+import org.apache.cassandra.service.pager.QueryPager;
+import org.apache.cassandra.service.pager.SinglePartitionPager;
+import org.apache.cassandra.transport.ProtocolVersion;
+
+/**
+ * A {@code ReadQuery} for a single partition.
+ */
+public interface SinglePartitionReadQuery extends ReadQuery
+{
+ public static Group<? extends SinglePartitionReadQuery> createGroup(TableMetadata metadata,
+ int nowInSec,
+ ColumnFilter columnFilter,
+ RowFilter rowFilter,
+ DataLimits limits,
+ List<DecoratedKey> partitionKeys,
+ ClusteringIndexFilter clusteringIndexFilter)
+ {
+ return metadata.isVirtual()
+ ? VirtualTableSinglePartitionReadQuery.Group.create(metadata, nowInSec, columnFilter, rowFilter, limits, partitionKeys, clusteringIndexFilter)
+ : SinglePartitionReadCommand.Group.create(metadata, nowInSec, columnFilter, rowFilter, limits, partitionKeys, clusteringIndexFilter);
+ }
+
+
+ /**
+ * Creates a new read query on a single partition.
+ *
+ * @param metadata the table to query.
+ * @param nowInSec the time in seconds to use are "now" for this query.
+ * @param key the partition key for the partition to query.
+ * @param columnFilter the column filter to use for the query.
+ * @param filter the clustering index filter to use for the query.
+ *
+ * @return a newly created read query. The returned query will use no row filter and have no limits.
+ */
+ public static SinglePartitionReadQuery create(TableMetadata metadata,
+ int nowInSec,
+ DecoratedKey key,
+ ColumnFilter columnFilter,
+ ClusteringIndexFilter filter)
+ {
+ return create(metadata, nowInSec, columnFilter, RowFilter.NONE, DataLimits.NONE, key, filter);
+ }
+
+ /**
+ * Creates a new read query on a single partition.
+ *
+ * @param metadata the table to query.
+ * @param nowInSec the time in seconds to use are "now" for this query.
+ * @param columnFilter the column filter to use for the query.
+ * @param rowFilter the row filter to use for the query.
+ * @param limits the limits to use for the query.
+ * @param partitionKey the partition key for the partition to query.
+ * @param clusteringIndexFilter the clustering index filter to use for the query.
+ *
+ * @return a newly created read query.
+ */
+ public static SinglePartitionReadQuery create(TableMetadata metadata,
+ int nowInSec,
+ ColumnFilter columnFilter,
+ RowFilter rowFilter,
+ DataLimits limits,
+ DecoratedKey partitionKey,
+ ClusteringIndexFilter clusteringIndexFilter)
+ {
+ return metadata.isVirtual()
+ ? VirtualTableSinglePartitionReadQuery.create(metadata, nowInSec, columnFilter, rowFilter, limits, partitionKey, clusteringIndexFilter)
+ : SinglePartitionReadCommand.create(metadata, nowInSec, columnFilter, rowFilter, limits, partitionKey, clusteringIndexFilter);
+ }
+
+ /**
+ * Returns the key of the partition queried by this {@code ReadQuery}
+ * @return the key of the partition queried
+ */
+ DecoratedKey partitionKey();
+
+ /**
+ * Creates a new {@code SinglePartitionReadQuery} with the specified limits.
+ *
+ * @param newLimits the new limits
+ * @return the new {@code SinglePartitionReadQuery}
+ */
+ SinglePartitionReadQuery withUpdatedLimit(DataLimits newLimits);
+
+ /**
+ * Returns a new {@code SinglePartitionReadQuery} suitable to paging from the last returned row.
+ *
+ * @param lastReturned the last row returned by the previous page. The newly created query
+ * will only query row that comes after this (in query order). This can be {@code null} if this
+ * is the first page.
+ * @param limits the limits to use for the page to query.
+ *
+ * @return the newly create query.
+ */
+ SinglePartitionReadQuery forPaging(Clustering lastReturned, DataLimits limits);
+
+ @Override
+ default SinglePartitionPager getPager(PagingState pagingState, ProtocolVersion protocolVersion)
+ {
+ return new SinglePartitionPager(this, pagingState, protocolVersion);
+ }
+
+ ClusteringIndexFilter clusteringIndexFilter();
+
+ default boolean selectsKey(DecoratedKey key)
+ {
+ if (!this.partitionKey().equals(key))
+ return false;
+
+ return rowFilter().partitionKeyRestrictionsAreSatisfiedBy(key, metadata().partitionKeyType);
+ }
+
+ default boolean selectsClustering(DecoratedKey key, Clustering clustering)
+ {
+ if (clustering == Clustering.STATIC_CLUSTERING)
+ return !columnFilter().fetchedColumns().statics.isEmpty();
+
+ if (!clusteringIndexFilter().selects(clustering))
+ return false;
+
+ return rowFilter().clusteringKeyRestrictionsAreSatisfiedBy(clustering);
+ }
+
+ /**
+ * Groups multiple single partition read queries.
+ */
+ abstract class Group<T extends SinglePartitionReadQuery> implements ReadQuery
+ {
+ public final List<T> queries;
+ private final DataLimits limits;
+ private final int nowInSec;
+ private final boolean selectsFullPartitions;
+
+ public Group(List<T> queries, DataLimits limits)
+ {
+ assert !queries.isEmpty();
+ this.queries = queries;
+ this.limits = limits;
+ T firstQuery = queries.get(0);
+ this.nowInSec = firstQuery.nowInSec();
+ this.selectsFullPartitions = firstQuery.selectsFullPartition();
+ for (int i = 1; i < queries.size(); i++)
+ assert queries.get(i).nowInSec() == nowInSec;
+ }
+
+ public int nowInSec()
+ {
+ return nowInSec;
+ }
+
+ public DataLimits limits()
+ {
+ return limits;
+ }
+
+ public TableMetadata metadata()
+ {
+ return queries.get(0).metadata();
+ }
+
+ @Override
+ public boolean selectsFullPartition()
+ {
+ return selectsFullPartitions;
+ }
+
+ public ReadExecutionController executionController()
+ {
+ // Note that the only difference between the queries in a group must be the partition key on which
+ // they applied. So as far as ReadOrderGroup is concerned, we can use any of the queries to start one.
+ return queries.get(0).executionController();
+ }
+
+ public PartitionIterator executeInternal(ReadExecutionController controller)
+ {
+ // Note that the only difference between the queries in a group must be the partition key on which
+ // they applied.
+ boolean enforceStrictLiveness = queries.get(0).metadata().enforceStrictLiveness();
+ return limits.filter(UnfilteredPartitionIterators.filter(executeLocally(controller, false), nowInSec),
+ nowInSec,
+ selectsFullPartitions,
+ enforceStrictLiveness);
+ }
+
+ public UnfilteredPartitionIterator executeLocally(ReadExecutionController executionController)
+ {
+ return executeLocally(executionController, true);
+ }
+
+ /**
+ * Implementation of {@link ReadQuery#executeLocally(ReadExecutionController)}.
+ *
+ * @param executionController - the {@code ReadExecutionController} protecting the read.
+ * @param sort - whether to sort the inner queries by partition key, required for merging the iterator
+ * later on. This will be false when called by {@link ReadQuery#executeInternal(ReadExecutionController)}
+ * because in this case it is safe to do so as there is no merging involved and we don't want to
+ * change the old behavior which was to not sort by partition.
+ *
+ * @return - the iterator that can be used to retrieve the query result.
+ */
+ private UnfilteredPartitionIterator executeLocally(ReadExecutionController executionController, boolean sort)
+ {
+ List<Pair<DecoratedKey, UnfilteredPartitionIterator>> partitions = new ArrayList<>(queries.size());
+ for (T query : queries)
+ partitions.add(Pair.of(query.partitionKey(), query.executeLocally(executionController)));
+
+ if (sort)
+ Collections.sort(partitions, (p1, p2) -> p1.getLeft().compareTo(p2.getLeft()));
+
+ return UnfilteredPartitionIterators.concat(partitions.stream().map(p -> p.getRight()).collect(Collectors.toList()));
+ }
+
+ public QueryPager getPager(PagingState pagingState, ProtocolVersion protocolVersion)
+ {
+ if (queries.size() == 1)
+ return new SinglePartitionPager(queries.get(0), pagingState, protocolVersion);
+
+ return new MultiPartitionPager<T>(this, pagingState, protocolVersion);
+ }
+
+ public boolean selectsKey(DecoratedKey key)
+ {
+ return Iterables.any(queries, c -> c.selectsKey(key));
+ }
+
+ public boolean selectsClustering(DecoratedKey key, Clustering clustering)
+ {
+ return Iterables.any(queries, c -> c.selectsClustering(key, clustering));
+ }
+
+ @Override
+ public RowFilter rowFilter()
+ {
+ // Note that the only difference between the query in a group must be the partition key on which
+ // they applied.
+ return queries.get(0).rowFilter();
+ }
+
+ @Override
+ public ColumnFilter columnFilter()
+ {
+ // Note that the only difference between the query in a group must be the partition key on which
+ // they applied.
+ return queries.get(0).columnFilter();
+ }
+
+ @Override
+ public String toString()
+ {
+ return queries.toString();
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/db/SizeEstimatesRecorder.java b/src/java/org/apache/cassandra/db/SizeEstimatesRecorder.java
index ebe3f9a..fe38d64 100644
--- a/src/java/org/apache/cassandra/db/SizeEstimatesRecorder.java
+++ b/src/java/org/apache/cassandra/db/SizeEstimatesRecorder.java
@@ -30,8 +30,8 @@
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.locator.TokenMetadata;
-import org.apache.cassandra.service.MigrationListener;
-import org.apache.cassandra.service.MigrationManager;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.SchemaChangeListener;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.Pair;
@@ -40,14 +40,13 @@
/**
* A very simplistic/crude partition count/size estimator.
*
- * Exposing per-primary-range estimated partitions count and size in CQL form,
- * as a direct CQL alternative to Thrift's describe_splits_ex().
+ * Exposing per-primary-range estimated partitions count and size in CQL form.
*
* Estimates (per primary range) are calculated and dumped into a system table (system.size_estimates) every 5 minutes.
*
* See CASSANDRA-7688.
*/
-public class SizeEstimatesRecorder extends MigrationListener implements Runnable
+public class SizeEstimatesRecorder extends SchemaChangeListener implements Runnable
{
private static final Logger logger = LoggerFactory.getLogger(SizeEstimatesRecorder.class);
@@ -55,13 +54,13 @@
private SizeEstimatesRecorder()
{
- MigrationManager.instance.register(this);
+ Schema.instance.registerListener(this);
}
public void run()
{
TokenMetadata metadata = StorageService.instance.getTokenMetadata().cloneOnlyTokenMap();
- if (!metadata.isMember(FBUtilities.getBroadcastAddress()))
+ if (!metadata.isMember(FBUtilities.getBroadcastAddressAndPort()))
{
logger.debug("Node is not part of the ring; not recording size estimates");
return;
@@ -71,27 +70,55 @@
for (Keyspace keyspace : Keyspace.nonLocalStrategy())
{
- Collection<Range<Token>> localRanges = StorageService.instance.getPrimaryRangesForEndpoint(keyspace.getName(),
- FBUtilities.getBroadcastAddress());
+ // In tools the call to describe_splits_ex() used to be coupled with the call to describe_local_ring() so
+ // most access was for the local primary range; after creating the size_estimates table this was changed
+ // to be the primary range.
+ // In a multi-dc setup its not uncommon for the local ring to be offset by 1 for the next DC; example:
+ // DC1: [0, 10, 20, 30]
+ // DC2: [1, 11, 21, 31]
+ // DC3: [2, 12, 22, 32]
+ // When working with the primary ring we have:
+ // [0, 1, 2, 10, 11, 12, 20, 21, 22, 30, 31, 32]
+ // this then leads to primrary ranges with one token in it, which cause the estimates to be less useful.
+ // Since only one range was published some tools make this assumption; for this reason we can't publish
+ // all ranges (including the replica ranges) nor can we keep backwards compatability and publish primary
+ // range. If we publish multiple ranges downstream integrations may start to see duplicate data.
+ // See CASSANDRA-15637
+ Collection<Range<Token>> primaryRanges = StorageService.instance.getPrimaryRanges(keyspace.getName());
+ Collection<Range<Token>> localPrimaryRanges = StorageService.instance.getLocalPrimaryRange();
+ boolean rangesAreEqual = primaryRanges.equals(localPrimaryRanges);
for (ColumnFamilyStore table : keyspace.getColumnFamilyStores())
{
long start = System.nanoTime();
- recordSizeEstimates(table, localRanges);
+
+ // compute estimates for primary ranges for backwards compatability
+ Map<Range<Token>, Pair<Long, Long>> estimates = computeSizeEstimates(table, primaryRanges);
+ SystemKeyspace.updateSizeEstimates(table.metadata.keyspace, table.metadata.name, estimates);
+ SystemKeyspace.updateTableEstimates(table.metadata.keyspace, table.metadata.name, SystemKeyspace.TABLE_ESTIMATES_TYPE_PRIMARY, estimates);
+
+ if (!rangesAreEqual)
+ {
+ // compute estimate for local primary range
+ estimates = computeSizeEstimates(table, localPrimaryRanges);
+ }
+ SystemKeyspace.updateTableEstimates(table.metadata.keyspace, table.metadata.name, SystemKeyspace.TABLE_ESTIMATES_TYPE_LOCAL_PRIMARY, estimates);
+
long passed = System.nanoTime() - start;
- logger.trace("Spent {} milliseconds on estimating {}.{} size",
- TimeUnit.NANOSECONDS.toMillis(passed),
- table.metadata.ksName,
- table.metadata.cfName);
+ if (logger.isTraceEnabled())
+ logger.trace("Spent {} milliseconds on estimating {}.{} size",
+ TimeUnit.NANOSECONDS.toMillis(passed),
+ table.metadata.keyspace,
+ table.metadata.name);
}
}
}
@SuppressWarnings("resource")
- private void recordSizeEstimates(ColumnFamilyStore table, Collection<Range<Token>> localRanges)
+ private static Map<Range<Token>, Pair<Long, Long>> computeSizeEstimates(ColumnFamilyStore table, Collection<Range<Token>> ranges)
{
// for each local primary range, estimate (crudely) mean partition size and partitions count.
- Map<Range<Token>, Pair<Long, Long>> estimates = new HashMap<>(localRanges.size());
- for (Range<Token> localRange : localRanges)
+ Map<Range<Token>, Pair<Long, Long>> estimates = new HashMap<>(ranges.size());
+ for (Range<Token> localRange : ranges)
{
for (Range<Token> unwrappedRange : localRange.unwrap())
{
@@ -124,11 +151,10 @@
}
}
- // atomically update the estimates.
- SystemKeyspace.updateSizeEstimates(table.metadata.ksName, table.metadata.cfName, estimates);
+ return estimates;
}
- private long estimatePartitionsCount(Collection<SSTableReader> sstables, Range<Token> range)
+ private static long estimatePartitionsCount(Collection<SSTableReader> sstables, Range<Token> range)
{
long count = 0;
for (SSTableReader sstable : sstables)
@@ -136,7 +162,7 @@
return count;
}
- private long estimateMeanPartitionSize(Collection<SSTableReader> sstables)
+ private static long estimateMeanPartitionSize(Collection<SSTableReader> sstables)
{
long sum = 0, count = 0;
for (SSTableReader sstable : sstables)
@@ -149,8 +175,8 @@
}
@Override
- public void onDropColumnFamily(String keyspace, String table)
+ public void onDropTable(String keyspace, String table)
{
- SystemKeyspace.clearSizeEstimates(keyspace, table);
+ SystemKeyspace.clearEstimates(keyspace, table);
}
}
diff --git a/src/java/org/apache/cassandra/db/Slice.java b/src/java/org/apache/cassandra/db/Slice.java
index 5f58e3b..384158f 100644
--- a/src/java/org/apache/cassandra/db/Slice.java
+++ b/src/java/org/apache/cassandra/db/Slice.java
@@ -21,7 +21,6 @@
import java.nio.ByteBuffer;
import java.util.*;
-import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
@@ -161,7 +160,15 @@
public static boolean isEmpty(ClusteringComparator comparator, ClusteringBound start, ClusteringBound end)
{
assert start.isStart() && end.isEnd();
- return comparator.compare(end, start) <= 0;
+
+ int cmp = comparator.compare(start, end);
+
+ if (cmp < 0)
+ return false;
+ else if (cmp > 0)
+ return true;
+ else
+ return start.isExclusive() || end.isExclusive();
}
/**
@@ -240,11 +247,6 @@
return start.compareTo(comparator, maxClusteringValues) <= 0 && end.compareTo(comparator, minClusteringValues) >= 0;
}
- public String toString(CFMetaData metadata)
- {
- return toString(metadata.comparator);
- }
-
public String toString(ClusteringComparator comparator)
{
StringBuilder sb = new StringBuilder();
diff --git a/src/java/org/apache/cassandra/db/Slices.java b/src/java/org/apache/cassandra/db/Slices.java
index 93dcab9..3d19fe9 100644
--- a/src/java/org/apache/cassandra/db/Slices.java
+++ b/src/java/org/apache/cassandra/db/Slices.java
@@ -21,10 +21,11 @@
import java.nio.ByteBuffer;
import java.util.*;
+import com.google.common.base.Preconditions;
import com.google.common.collect.Iterators;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
@@ -61,7 +62,7 @@
if (slice.start() == ClusteringBound.BOTTOM && slice.end() == ClusteringBound.TOP)
return Slices.ALL;
- assert comparator.compare(slice.start(), slice.end()) <= 0;
+ Preconditions.checkArgument(!slice.isEmpty(comparator));
return new ArrayBackedSlices(comparator, new Slice[]{ slice });
}
@@ -140,7 +141,7 @@
*/
public abstract boolean intersects(List<ByteBuffer> minClusteringValues, List<ByteBuffer> maxClusteringValues);
- public abstract String toCQLString(CFMetaData metadata);
+ public abstract String toCQLString(TableMetadata metadata);
/**
* Checks if this <code>Slices</code> is empty.
@@ -192,7 +193,7 @@
public Builder add(Slice slice)
{
- assert comparator.compare(slice.start(), slice.end()) <= 0;
+ Preconditions.checkArgument(!slice.isEmpty(comparator));
if (slices.size() > 0 && comparator.compare(slices.get(slices.size()-1).end(), slice.start()) > 0)
needsNormalizing = true;
slices.add(slice);
@@ -323,7 +324,7 @@
return size;
}
- public Slices deserialize(DataInputPlus in, int version, CFMetaData metadata) throws IOException
+ public Slices deserialize(DataInputPlus in, int version, TableMetadata metadata) throws IOException
{
int size = (int)in.readUnsignedVInt();
@@ -548,7 +549,7 @@
return sb.append("}").toString();
}
- public String toCQLString(CFMetaData metadata)
+ public String toCQLString(TableMetadata metadata)
{
StringBuilder sb = new StringBuilder();
@@ -572,7 +573,7 @@
boolean needAnd = false;
for (int i = 0; i < clusteringSize; i++)
{
- ColumnDefinition column = metadata.clusteringColumns().get(i);
+ ColumnMetadata column = metadata.clusteringColumns().get(i);
List<ComponentOfSlice> componentInfo = columnComponents.get(i);
if (componentInfo.isEmpty())
break;
@@ -646,7 +647,7 @@
return sb.toString();
}
- // An somewhat adhoc utility class only used by toCQLString
+ // An somewhat adhoc utility class only used by nameAsCQLString
private static class ComponentOfSlice
{
public final boolean startInclusive;
@@ -763,7 +764,7 @@
return "ALL";
}
- public String toCQLString(CFMetaData metadata)
+ public String toCQLString(TableMetadata metadata)
{
return "";
}
@@ -838,7 +839,7 @@
return "NONE";
}
- public String toCQLString(CFMetaData metadata)
+ public String toCQLString(TableMetadata metadata)
{
return "";
}
diff --git a/src/java/org/apache/cassandra/db/SnapshotCommand.java b/src/java/org/apache/cassandra/db/SnapshotCommand.java
index eb6f67a..484db2f 100644
--- a/src/java/org/apache/cassandra/db/SnapshotCommand.java
+++ b/src/java/org/apache/cassandra/db/SnapshotCommand.java
@@ -22,8 +22,8 @@
import org.apache.cassandra.io.IVersionedSerializer;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
-import org.apache.cassandra.net.MessageOut;
-import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.net.Message;
+import org.apache.cassandra.net.Verb;
public class SnapshotCommand
{
@@ -42,11 +42,6 @@
this.clear_snapshot = clearSnapshot;
}
- public MessageOut createMessage()
- {
- return new MessageOut<SnapshotCommand>(MessagingService.Verb.SNAPSHOT, this, serializer);
- }
-
@Override
public String toString()
{
diff --git a/src/java/org/apache/cassandra/db/SnapshotDetailsTabularData.java b/src/java/org/apache/cassandra/db/SnapshotDetailsTabularData.java
index 97caea1..5ef729a 100644
--- a/src/java/org/apache/cassandra/db/SnapshotDetailsTabularData.java
+++ b/src/java/org/apache/cassandra/db/SnapshotDetailsTabularData.java
@@ -69,12 +69,12 @@
}
- public static void from(final String snapshot, final String ks, final String cf, Map.Entry<String, Pair<Long,Long>> snapshotDetail, TabularDataSupport result)
+ public static void from(final String snapshot, final String ks, final String cf, Map.Entry<String, Directories.SnapshotSizeDetails> snapshotDetail, TabularDataSupport result)
{
try
{
- final String totalSize = FileUtils.stringifyFileSize(snapshotDetail.getValue().left);
- final String liveSize = FileUtils.stringifyFileSize(snapshotDetail.getValue().right);
+ final String totalSize = FileUtils.stringifyFileSize(snapshotDetail.getValue().sizeOnDiskBytes);
+ final String liveSize = FileUtils.stringifyFileSize(snapshotDetail.getValue().dataSizeBytes);
result.put(new CompositeDataSupport(COMPOSITE_TYPE, ITEM_NAMES,
new Object[]{ snapshot, ks, cf, liveSize, totalSize }));
}
diff --git a/src/java/org/apache/cassandra/db/StorageHook.java b/src/java/org/apache/cassandra/db/StorageHook.java
index 48d7ede..be1d0bf 100644
--- a/src/java/org/apache/cassandra/db/StorageHook.java
+++ b/src/java/org/apache/cassandra/db/StorageHook.java
@@ -18,8 +18,6 @@
package org.apache.cassandra.db;
-import java.util.UUID;
-
import org.apache.cassandra.db.filter.ClusteringIndexFilter;
import org.apache.cassandra.db.filter.ColumnFilter;
import org.apache.cassandra.db.partitions.PartitionUpdate;
@@ -27,31 +25,27 @@
import org.apache.cassandra.db.rows.UnfilteredRowIteratorWithLowerBound;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.sstable.format.SSTableReadsListener;
+import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.utils.FBUtilities;
public interface StorageHook
{
public static final StorageHook instance = createHook();
- public void reportWrite(UUID cfid, PartitionUpdate partitionUpdate);
- public void reportRead(UUID cfid, DecoratedKey key);
+ public void reportWrite(TableId tableId, PartitionUpdate partitionUpdate);
+ public void reportRead(TableId tableId, DecoratedKey key);
public UnfilteredRowIteratorWithLowerBound makeRowIteratorWithLowerBound(ColumnFamilyStore cfs,
DecoratedKey partitionKey,
SSTableReader sstable,
ClusteringIndexFilter filter,
ColumnFilter selectedColumns,
- boolean isForThrift,
- int nowInSec,
- boolean applyThriftTransformation,
SSTableReadsListener listener);
-
public UnfilteredRowIterator makeRowIterator(ColumnFamilyStore cfs,
SSTableReader sstable,
DecoratedKey key,
Slices slices,
ColumnFilter selectedColumns,
boolean reversed,
- boolean isForThrift,
SSTableReadsListener listener);
static StorageHook createHook()
@@ -64,27 +58,21 @@
return new StorageHook()
{
- public void reportWrite(UUID cfid, PartitionUpdate partitionUpdate) {}
+ public void reportWrite(TableId tableId, PartitionUpdate partitionUpdate) {}
- public void reportRead(UUID cfid, DecoratedKey key) {}
+ public void reportRead(TableId tableId, DecoratedKey key) {}
public UnfilteredRowIteratorWithLowerBound makeRowIteratorWithLowerBound(ColumnFamilyStore cfs,
DecoratedKey partitionKey,
SSTableReader sstable,
ClusteringIndexFilter filter,
ColumnFilter selectedColumns,
- boolean isForThrift,
- int nowInSec,
- boolean applyThriftTransformation,
SSTableReadsListener listener)
{
return new UnfilteredRowIteratorWithLowerBound(partitionKey,
sstable,
filter,
selectedColumns,
- isForThrift,
- nowInSec,
- applyThriftTransformation,
listener);
}
@@ -94,10 +82,9 @@
Slices slices,
ColumnFilter selectedColumns,
boolean reversed,
- boolean isForThrift,
SSTableReadsListener listener)
{
- return sstable.iterator(key, slices, selectedColumns, reversed, isForThrift, listener);
+ return sstable.iterator(key, slices, selectedColumns, reversed, listener);
}
};
}
diff --git a/src/java/org/apache/cassandra/db/SystemKeyspace.java b/src/java/org/apache/cassandra/db/SystemKeyspace.java
index 196face..655c7a0 100644
--- a/src/java/org/apache/cassandra/db/SystemKeyspace.java
+++ b/src/java/org/apache/cassandra/db/SystemKeyspace.java
@@ -23,56 +23,57 @@
import java.net.InetAddress;
import java.nio.ByteBuffer;
import java.util.*;
-import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import java.util.stream.StreamSupport;
import javax.management.openmbean.OpenDataException;
import javax.management.openmbean.TabularData;
-import java.util.concurrent.Future;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.HashMultimap;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.SetMultimap;
import com.google.common.collect.Sets;
import com.google.common.io.ByteStreams;
+import com.google.common.util.concurrent.ListenableFuture;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.util.concurrent.Futures;
-
-import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.config.SchemaConstants;
import org.apache.cassandra.cql3.QueryProcessor;
import org.apache.cassandra.cql3.UntypedResultSet;
import org.apache.cassandra.cql3.functions.*;
+import org.apache.cassandra.cql3.statements.schema.CreateTableStatement;
import org.apache.cassandra.db.commitlog.CommitLogPosition;
import org.apache.cassandra.db.compaction.CompactionHistoryTabularData;
import org.apache.cassandra.db.marshal.*;
-import org.apache.cassandra.db.rows.Rows;
import org.apache.cassandra.db.partitions.PartitionUpdate;
+import org.apache.cassandra.db.rows.Rows;
import org.apache.cassandra.dht.*;
import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.io.util.*;
import org.apache.cassandra.locator.IEndpointSnitch;
+import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.metrics.RestorableMeter;
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.schema.*;
+import org.apache.cassandra.schema.SchemaConstants;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.service.paxos.Commit;
import org.apache.cassandra.service.paxos.PaxosState;
-import org.apache.cassandra.thrift.cassandraConstants;
+import org.apache.cassandra.streaming.StreamOperation;
import org.apache.cassandra.transport.ProtocolVersion;
import org.apache.cassandra.utils.*;
+import static java.lang.String.format;
import static java.util.Collections.emptyMap;
import static java.util.Collections.singletonMap;
+
import static org.apache.cassandra.cql3.QueryProcessor.executeInternal;
import static org.apache.cassandra.cql3.QueryProcessor.executeOnceInternal;
-import static org.apache.cassandra.io.util.FileUtils.visitDirectory;
+import static org.apache.cassandra.locator.Replica.fullReplica;
+import static org.apache.cassandra.locator.Replica.transientReplica;
public final class SystemKeyspace
{
@@ -95,42 +96,41 @@
public static final String PAXOS = "paxos";
public static final String BUILT_INDEXES = "IndexInfo";
public static final String LOCAL = "local";
- public static final String PEERS = "peers";
- public static final String PEER_EVENTS = "peer_events";
- public static final String RANGE_XFERS = "range_xfers";
+ public static final String PEERS_V2 = "peers_v2";
+ public static final String PEER_EVENTS_V2 = "peer_events_v2";
public static final String COMPACTION_HISTORY = "compaction_history";
public static final String SSTABLE_ACTIVITY = "sstable_activity";
- public static final String SIZE_ESTIMATES = "size_estimates";
- public static final String AVAILABLE_RANGES = "available_ranges";
- public static final String TRANSFERRED_RANGES = "transferred_ranges";
- public static final String VIEWS_BUILDS_IN_PROGRESS = "views_builds_in_progress";
+ public static final String TABLE_ESTIMATES = "table_estimates";
+ public static final String TABLE_ESTIMATES_TYPE_PRIMARY = "primary";
+ public static final String TABLE_ESTIMATES_TYPE_LOCAL_PRIMARY = "local_primary";
+ public static final String AVAILABLE_RANGES_V2 = "available_ranges_v2";
+ public static final String TRANSFERRED_RANGES_V2 = "transferred_ranges_v2";
+ public static final String VIEW_BUILDS_IN_PROGRESS = "view_builds_in_progress";
public static final String BUILT_VIEWS = "built_views";
public static final String PREPARED_STATEMENTS = "prepared_statements";
+ public static final String REPAIRS = "repairs";
- @Deprecated public static final String LEGACY_HINTS = "hints";
- @Deprecated public static final String LEGACY_BATCHLOG = "batchlog";
- @Deprecated public static final String LEGACY_KEYSPACES = "schema_keyspaces";
- @Deprecated public static final String LEGACY_COLUMNFAMILIES = "schema_columnfamilies";
- @Deprecated public static final String LEGACY_COLUMNS = "schema_columns";
- @Deprecated public static final String LEGACY_TRIGGERS = "schema_triggers";
- @Deprecated public static final String LEGACY_USERTYPES = "schema_usertypes";
- @Deprecated public static final String LEGACY_FUNCTIONS = "schema_functions";
- @Deprecated public static final String LEGACY_AGGREGATES = "schema_aggregates";
+ @Deprecated public static final String LEGACY_PEERS = "peers";
+ @Deprecated public static final String LEGACY_PEER_EVENTS = "peer_events";
+ @Deprecated public static final String LEGACY_TRANSFERRED_RANGES = "transferred_ranges";
+ @Deprecated public static final String LEGACY_AVAILABLE_RANGES = "available_ranges";
+ @Deprecated public static final String LEGACY_SIZE_ESTIMATES = "size_estimates";
- public static final CFMetaData Batches =
- compile(BATCHES,
- "batches awaiting replay",
- "CREATE TABLE %s ("
- + "id timeuuid,"
- + "mutations list<blob>,"
- + "version int,"
- + "PRIMARY KEY ((id)))")
- .copy(new LocalPartitioner(TimeUUIDType.instance))
- .compaction(CompactionParams.scts(singletonMap("min_threshold", "2")))
- .gcGraceSeconds(0);
- private static final CFMetaData Paxos =
- compile(PAXOS,
+ public static final TableMetadata Batches =
+ parse(BATCHES,
+ "batches awaiting replay",
+ "CREATE TABLE %s ("
+ + "id timeuuid,"
+ + "mutations list<blob>,"
+ + "version int,"
+ + "PRIMARY KEY ((id)))")
+ .partitioner(new LocalPartitioner(TimeUUIDType.instance))
+ .compaction(CompactionParams.stcs(singletonMap("min_threshold", "2")))
+ .build();
+
+ private static final TableMetadata Paxos =
+ parse(PAXOS,
"in-progress paxos proposals",
"CREATE TABLE %s ("
+ "row_key blob,"
@@ -143,74 +143,78 @@
+ "proposal_ballot timeuuid,"
+ "proposal_version int,"
+ "PRIMARY KEY ((row_key), cf_id))")
- .compaction(CompactionParams.lcs(emptyMap()));
+ .compaction(CompactionParams.lcs(emptyMap()))
+ .build();
- private static final CFMetaData BuiltIndexes =
- compile(BUILT_INDEXES,
- "built column indexes",
- "CREATE TABLE \"%s\" ("
- + "table_name text," // table_name here is the name of the keyspace - don't be fooled
- + "index_name text,"
- + "PRIMARY KEY ((table_name), index_name)) "
- + "WITH COMPACT STORAGE");
+ private static final TableMetadata BuiltIndexes =
+ parse(BUILT_INDEXES,
+ "built column indexes",
+ "CREATE TABLE \"%s\" ("
+ + "table_name text," // table_name here is the name of the keyspace - don't be fooled
+ + "index_name text,"
+ + "value blob," // Table used to be compact in previous versions
+ + "PRIMARY KEY ((table_name), index_name)) ")
+ .build();
- private static final CFMetaData Local =
- compile(LOCAL,
+ private static final TableMetadata Local =
+ parse(LOCAL,
"information about the local node",
"CREATE TABLE %s ("
+ "key text,"
+ "bootstrapped text,"
+ "broadcast_address inet,"
+ + "broadcast_port int,"
+ "cluster_name text,"
+ "cql_version text,"
+ "data_center text,"
+ "gossip_generation int,"
+ "host_id uuid,"
+ "listen_address inet,"
+ + "listen_port int,"
+ "native_protocol_version text,"
+ "partitioner text,"
+ "rack text,"
+ "release_version text,"
+ "rpc_address inet,"
+ + "rpc_port int,"
+ "schema_version uuid,"
- + "thrift_version text,"
+ "tokens set<varchar>,"
+ "truncated_at map<uuid, blob>,"
- + "PRIMARY KEY ((key)))");
+ + "PRIMARY KEY ((key)))"
+ ).recordDeprecatedSystemColumn("thrift_version", UTF8Type.instance)
+ .build();
- private static final CFMetaData Peers =
- compile(PEERS,
+ private static final TableMetadata PeersV2 =
+ parse(PEERS_V2,
"information about known peers in the cluster",
"CREATE TABLE %s ("
+ "peer inet,"
+ + "peer_port int,"
+ "data_center text,"
+ "host_id uuid,"
+ "preferred_ip inet,"
+ + "preferred_port int,"
+ "rack text,"
+ "release_version text,"
- + "rpc_address inet,"
+ + "native_address inet,"
+ + "native_port int,"
+ "schema_version uuid,"
+ "tokens set<varchar>,"
- + "PRIMARY KEY ((peer)))");
+ + "PRIMARY KEY ((peer), peer_port))")
+ .build();
- private static final CFMetaData PeerEvents =
- compile(PEER_EVENTS,
+ private static final TableMetadata PeerEventsV2 =
+ parse(PEER_EVENTS_V2,
"events related to peers",
"CREATE TABLE %s ("
+ "peer inet,"
+ + "peer_port int,"
+ "hints_dropped map<uuid, int>,"
- + "PRIMARY KEY ((peer)))");
+ + "PRIMARY KEY ((peer), peer_port))")
+ .build();
- private static final CFMetaData RangeXfers =
- compile(RANGE_XFERS,
- "ranges requested for transfer",
- "CREATE TABLE %s ("
- + "token_bytes blob,"
- + "requested_at timestamp,"
- + "PRIMARY KEY ((token_bytes)))");
-
- private static final CFMetaData CompactionHistory =
- compile(COMPACTION_HISTORY,
+ private static final TableMetadata CompactionHistory =
+ parse(COMPACTION_HISTORY,
"week-long compaction history",
"CREATE TABLE %s ("
+ "id uuid,"
@@ -221,10 +225,11 @@
+ "keyspace_name text,"
+ "rows_merged map<int, bigint>,"
+ "PRIMARY KEY ((id)))")
- .defaultTimeToLive((int) TimeUnit.DAYS.toSeconds(7));
+ .defaultTimeToLive((int) TimeUnit.DAYS.toSeconds(7))
+ .build();
- private static final CFMetaData SSTableActivity =
- compile(SSTABLE_ACTIVITY,
+ private static final TableMetadata SSTableActivity =
+ parse(SSTABLE_ACTIVITY,
"historic sstable read rates",
"CREATE TABLE %s ("
+ "keyspace_name text,"
@@ -232,11 +237,13 @@
+ "generation int,"
+ "rate_120m double,"
+ "rate_15m double,"
- + "PRIMARY KEY ((keyspace_name, columnfamily_name, generation)))");
+ + "PRIMARY KEY ((keyspace_name, columnfamily_name, generation)))")
+ .build();
- private static final CFMetaData SizeEstimates =
- compile(SIZE_ESTIMATES,
- "per-table primary range size estimates",
+ @Deprecated
+ private static final TableMetadata LegacySizeEstimates =
+ parse(LEGACY_SIZE_ESTIMATES,
+ "per-table primary range size estimates, table is deprecated in favor of " + TABLE_ESTIMATES,
"CREATE TABLE %s ("
+ "keyspace_name text,"
+ "table_name text,"
@@ -245,200 +252,150 @@
+ "mean_partition_size bigint,"
+ "partitions_count bigint,"
+ "PRIMARY KEY ((keyspace_name), table_name, range_start, range_end))")
- .gcGraceSeconds(0);
+ .build();
- private static final CFMetaData AvailableRanges =
- compile(AVAILABLE_RANGES,
- "available keyspace/ranges during bootstrap/replace that are ready to be served",
- "CREATE TABLE %s ("
- + "keyspace_name text,"
- + "ranges set<blob>,"
- + "PRIMARY KEY ((keyspace_name)))");
+ private static final TableMetadata TableEstimates =
+ parse(TABLE_ESTIMATES,
+ "per-table range size estimates",
+ "CREATE TABLE %s ("
+ + "keyspace_name text,"
+ + "table_name text,"
+ + "range_type text,"
+ + "range_start text,"
+ + "range_end text,"
+ + "mean_partition_size bigint,"
+ + "partitions_count bigint,"
+ + "PRIMARY KEY ((keyspace_name), table_name, range_type, range_start, range_end))")
+ .build();
- private static final CFMetaData TransferredRanges =
- compile(TRANSFERRED_RANGES,
+ private static final TableMetadata AvailableRangesV2 =
+ parse(AVAILABLE_RANGES_V2,
+ "available keyspace/ranges during bootstrap/replace that are ready to be served",
+ "CREATE TABLE %s ("
+ + "keyspace_name text,"
+ + "full_ranges set<blob>,"
+ + "transient_ranges set<blob>,"
+ + "PRIMARY KEY ((keyspace_name)))")
+ .build();
+
+ private static final TableMetadata TransferredRangesV2 =
+ parse(TRANSFERRED_RANGES_V2,
"record of transferred ranges for streaming operation",
"CREATE TABLE %s ("
+ "operation text,"
+ "peer inet,"
+ + "peer_port int,"
+ "keyspace_name text,"
+ "ranges set<blob>,"
- + "PRIMARY KEY ((operation, keyspace_name), peer))");
+ + "PRIMARY KEY ((operation, keyspace_name), peer, peer_port))")
+ .build();
- private static final CFMetaData ViewsBuildsInProgress =
- compile(VIEWS_BUILDS_IN_PROGRESS,
- "views builds current progress",
- "CREATE TABLE %s ("
- + "keyspace_name text,"
- + "view_name text,"
- + "last_token varchar,"
- + "generation_number int,"
- + "PRIMARY KEY ((keyspace_name), view_name))");
+ private static final TableMetadata ViewBuildsInProgress =
+ parse(VIEW_BUILDS_IN_PROGRESS,
+ "views builds current progress",
+ "CREATE TABLE %s ("
+ + "keyspace_name text,"
+ + "view_name text,"
+ + "start_token varchar,"
+ + "end_token varchar,"
+ + "last_token varchar,"
+ + "keys_built bigint,"
+ + "PRIMARY KEY ((keyspace_name), view_name, start_token, end_token))")
+ .build();
- private static final CFMetaData BuiltViews =
- compile(BUILT_VIEWS,
+ private static final TableMetadata BuiltViews =
+ parse(BUILT_VIEWS,
"built views",
"CREATE TABLE %s ("
+ "keyspace_name text,"
+ "view_name text,"
+ "status_replicated boolean,"
- + "PRIMARY KEY ((keyspace_name), view_name))");
+ + "PRIMARY KEY ((keyspace_name), view_name))")
+ .build();
- private static final CFMetaData PreparedStatements =
- compile(PREPARED_STATEMENTS,
+ private static final TableMetadata PreparedStatements =
+ parse(PREPARED_STATEMENTS,
"prepared statements",
"CREATE TABLE %s ("
+ "prepared_id blob,"
+ "logged_keyspace text,"
+ "query_string text,"
- + "PRIMARY KEY ((prepared_id)))");
+ + "PRIMARY KEY ((prepared_id)))")
+ .build();
+
+ private static final TableMetadata Repairs =
+ parse(REPAIRS,
+ "repairs",
+ "CREATE TABLE %s ("
+ + "parent_id timeuuid, "
+ + "started_at timestamp, "
+ + "last_update timestamp, "
+ + "repaired_at timestamp, "
+ + "state int, "
+ + "coordinator inet, "
+ + "coordinator_port int,"
+ + "participants set<inet>,"
+ + "participants_wp set<text>,"
+ + "ranges set<blob>, "
+ + "cfids set<uuid>, "
+ + "PRIMARY KEY (parent_id))").build();
@Deprecated
- public static final CFMetaData LegacyHints =
- compile(LEGACY_HINTS,
- "*DEPRECATED* hints awaiting delivery",
- "CREATE TABLE %s ("
- + "target_id uuid,"
- + "hint_id timeuuid,"
- + "message_version int,"
- + "mutation blob,"
- + "PRIMARY KEY ((target_id), hint_id, message_version)) "
- + "WITH COMPACT STORAGE")
- .compaction(CompactionParams.scts(singletonMap("enabled", "false")))
- .gcGraceSeconds(0);
+ private static final TableMetadata LegacyPeers =
+ parse(LEGACY_PEERS,
+ "information about known peers in the cluster",
+ "CREATE TABLE %s ("
+ + "peer inet,"
+ + "data_center text,"
+ + "host_id uuid,"
+ + "preferred_ip inet,"
+ + "rack text,"
+ + "release_version text,"
+ + "rpc_address inet,"
+ + "schema_version uuid,"
+ + "tokens set<varchar>,"
+ + "PRIMARY KEY ((peer)))")
+ .build();
@Deprecated
- public static final CFMetaData LegacyBatchlog =
- compile(LEGACY_BATCHLOG,
- "*DEPRECATED* batchlog entries",
- "CREATE TABLE %s ("
- + "id uuid,"
- + "data blob,"
- + "version int,"
- + "written_at timestamp,"
- + "PRIMARY KEY ((id)))")
- .compaction(CompactionParams.scts(singletonMap("min_threshold", "2")))
- .gcGraceSeconds(0);
+ private static final TableMetadata LegacyPeerEvents =
+ parse(LEGACY_PEER_EVENTS,
+ "events related to peers",
+ "CREATE TABLE %s ("
+ + "peer inet,"
+ + "hints_dropped map<uuid, int>,"
+ + "PRIMARY KEY ((peer)))")
+ .build();
@Deprecated
- public static final CFMetaData LegacyKeyspaces =
- compile(LEGACY_KEYSPACES,
- "*DEPRECATED* keyspace definitions",
- "CREATE TABLE %s ("
- + "keyspace_name text,"
- + "durable_writes boolean,"
- + "strategy_class text,"
- + "strategy_options text,"
- + "PRIMARY KEY ((keyspace_name))) "
- + "WITH COMPACT STORAGE");
+ private static final TableMetadata LegacyTransferredRanges =
+ parse(LEGACY_TRANSFERRED_RANGES,
+ "record of transferred ranges for streaming operation",
+ "CREATE TABLE %s ("
+ + "operation text,"
+ + "peer inet,"
+ + "keyspace_name text,"
+ + "ranges set<blob>,"
+ + "PRIMARY KEY ((operation, keyspace_name), peer))")
+ .build();
@Deprecated
- public static final CFMetaData LegacyColumnfamilies =
- compile(LEGACY_COLUMNFAMILIES,
- "*DEPRECATED* table definitions",
- "CREATE TABLE %s ("
- + "keyspace_name text,"
- + "columnfamily_name text,"
- + "bloom_filter_fp_chance double,"
- + "caching text,"
- + "cf_id uuid," // post-2.1 UUID cfid
- + "comment text,"
- + "compaction_strategy_class text,"
- + "compaction_strategy_options text,"
- + "comparator text,"
- + "compression_parameters text,"
- + "default_time_to_live int,"
- + "default_validator text,"
- + "dropped_columns map<text, bigint>,"
- + "gc_grace_seconds int,"
- + "is_dense boolean,"
- + "key_validator text,"
- + "local_read_repair_chance double,"
- + "max_compaction_threshold int,"
- + "max_index_interval int,"
- + "memtable_flush_period_in_ms int,"
- + "min_compaction_threshold int,"
- + "min_index_interval int,"
- + "read_repair_chance double,"
- + "speculative_retry text,"
- + "subcomparator text,"
- + "type text,"
- + "PRIMARY KEY ((keyspace_name), columnfamily_name))");
+ private static final TableMetadata LegacyAvailableRanges =
+ parse(LEGACY_AVAILABLE_RANGES,
+ "available keyspace/ranges during bootstrap/replace that are ready to be served",
+ "CREATE TABLE %s ("
+ + "keyspace_name text,"
+ + "ranges set<blob>,"
+ + "PRIMARY KEY ((keyspace_name)))")
+ .build();
- @Deprecated
- public static final CFMetaData LegacyColumns =
- compile(LEGACY_COLUMNS,
- "*DEPRECATED* column definitions",
- "CREATE TABLE %s ("
- + "keyspace_name text,"
- + "columnfamily_name text,"
- + "column_name text,"
- + "component_index int,"
- + "index_name text,"
- + "index_options text,"
- + "index_type text,"
- + "type text,"
- + "validator text,"
- + "PRIMARY KEY ((keyspace_name), columnfamily_name, column_name))");
-
- @Deprecated
- public static final CFMetaData LegacyTriggers =
- compile(LEGACY_TRIGGERS,
- "*DEPRECATED* trigger definitions",
- "CREATE TABLE %s ("
- + "keyspace_name text,"
- + "columnfamily_name text,"
- + "trigger_name text,"
- + "trigger_options map<text, text>,"
- + "PRIMARY KEY ((keyspace_name), columnfamily_name, trigger_name))");
-
- @Deprecated
- public static final CFMetaData LegacyUsertypes =
- compile(LEGACY_USERTYPES,
- "*DEPRECATED* user defined type definitions",
- "CREATE TABLE %s ("
- + "keyspace_name text,"
- + "type_name text,"
- + "field_names list<text>,"
- + "field_types list<text>,"
- + "PRIMARY KEY ((keyspace_name), type_name))");
-
- @Deprecated
- public static final CFMetaData LegacyFunctions =
- compile(LEGACY_FUNCTIONS,
- "*DEPRECATED* user defined function definitions",
- "CREATE TABLE %s ("
- + "keyspace_name text,"
- + "function_name text,"
- + "signature frozen<list<text>>,"
- + "argument_names list<text>,"
- + "argument_types list<text>,"
- + "body text,"
- + "language text,"
- + "return_type text,"
- + "called_on_null_input boolean,"
- + "PRIMARY KEY ((keyspace_name), function_name, signature))");
-
- @Deprecated
- public static final CFMetaData LegacyAggregates =
- compile(LEGACY_AGGREGATES,
- "*DEPRECATED* user defined aggregate definitions",
- "CREATE TABLE %s ("
- + "keyspace_name text,"
- + "aggregate_name text,"
- + "signature frozen<list<text>>,"
- + "argument_types list<text>,"
- + "final_func text,"
- + "initcond blob,"
- + "return_type text,"
- + "state_func text,"
- + "state_type text,"
- + "PRIMARY KEY ((keyspace_name), aggregate_name, signature))");
-
- private static CFMetaData compile(String name, String description, String schema)
+ private static TableMetadata.Builder parse(String table, String description, String cql)
{
- return CFMetaData.compile(String.format(schema, name), SchemaConstants.SYSTEM_KEYSPACE_NAME)
- .comment(description);
+ return CreateTableStatement.parse(format(cql, table), SchemaConstants.SYSTEM_KEYSPACE_NAME)
+ .id(TableId.forSystemTable(SchemaConstants.SYSTEM_KEYSPACE_NAME, table))
+ .gcGraceSeconds(0)
+ .memtableFlushPeriod((int) TimeUnit.HOURS.toMillis(1))
+ .comment(description);
}
public static KeyspaceMetadata metadata()
@@ -452,26 +409,22 @@
Batches,
Paxos,
Local,
- Peers,
- PeerEvents,
- RangeXfers,
+ PeersV2,
+ LegacyPeers,
+ PeerEventsV2,
+ LegacyPeerEvents,
CompactionHistory,
SSTableActivity,
- SizeEstimates,
- AvailableRanges,
- TransferredRanges,
- ViewsBuildsInProgress,
+ LegacySizeEstimates,
+ TableEstimates,
+ AvailableRangesV2,
+ LegacyAvailableRanges,
+ TransferredRangesV2,
+ LegacyTransferredRanges,
+ ViewBuildsInProgress,
BuiltViews,
- LegacyHints,
- LegacyBatchlog,
PreparedStatements,
- LegacyKeyspaces,
- LegacyColumnfamilies,
- LegacyColumns,
- LegacyTriggers,
- LegacyUsertypes,
- LegacyFunctions,
- LegacyAggregates);
+ Repairs);
}
private static Functions functions()
@@ -482,10 +435,11 @@
.add(BytesConversionFcts.all())
.add(AggregateFcts.all())
.add(CastFcts.all())
+ .add(OperationFcts.all())
.build();
}
- private static volatile Map<UUID, Pair<CommitLogPosition, Long>> truncationRecords;
+ private static volatile Map<TableId, Pair<CommitLogPosition, Long>> truncationRecords;
public enum BootstrapState
{
@@ -507,29 +461,33 @@
"cluster_name," +
"release_version," +
"cql_version," +
- "thrift_version," +
"native_protocol_version," +
"data_center," +
"rack," +
"partitioner," +
"rpc_address," +
+ "rpc_port," +
"broadcast_address," +
- "listen_address" +
- ") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)";
+ "broadcast_port," +
+ "listen_address," +
+ "listen_port" +
+ ") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)";
IEndpointSnitch snitch = DatabaseDescriptor.getEndpointSnitch();
- executeOnceInternal(String.format(req, LOCAL),
+ executeOnceInternal(format(req, LOCAL),
LOCAL,
DatabaseDescriptor.getClusterName(),
FBUtilities.getReleaseVersionString(),
QueryProcessor.CQL_VERSION.toString(),
- cassandraConstants.VERSION,
String.valueOf(ProtocolVersion.CURRENT.asInt()),
- snitch.getDatacenter(FBUtilities.getBroadcastAddress()),
- snitch.getRack(FBUtilities.getBroadcastAddress()),
+ snitch.getLocalDatacenter(),
+ snitch.getLocalRack(),
DatabaseDescriptor.getPartitioner().getClass().getName(),
DatabaseDescriptor.getRpcAddress(),
- FBUtilities.getBroadcastAddress(),
- FBUtilities.getLocalAddress());
+ DatabaseDescriptor.getNativeTransportPort(),
+ FBUtilities.getJustBroadcastAddress(),
+ DatabaseDescriptor.getStoragePort(),
+ FBUtilities.getJustLocalAddress(),
+ DatabaseDescriptor.getStoragePort());
}
public static void updateCompactionHistory(String ksname,
@@ -543,7 +501,7 @@
if (ksname.equals("system") && cfname.equals(COMPACTION_HISTORY))
return;
String req = "INSERT INTO system.%s (id, keyspace_name, columnfamily_name, compacted_at, bytes_in, bytes_out, rows_merged) VALUES (?, ?, ?, ?, ?, ?, ?)";
- executeInternal(String.format(req, COMPACTION_HISTORY),
+ executeInternal(format(req, COMPACTION_HISTORY),
UUIDGen.getTimeUUID(),
ksname,
cfname,
@@ -555,21 +513,21 @@
public static TabularData getCompactionHistory() throws OpenDataException
{
- UntypedResultSet queryResultSet = executeInternal(String.format("SELECT * from system.%s", COMPACTION_HISTORY));
+ UntypedResultSet queryResultSet = executeInternal(format("SELECT * from system.%s", COMPACTION_HISTORY));
return CompactionHistoryTabularData.from(queryResultSet);
}
public static boolean isViewBuilt(String keyspaceName, String viewName)
{
String req = "SELECT view_name FROM %s.\"%s\" WHERE keyspace_name=? AND view_name=?";
- UntypedResultSet result = executeInternal(String.format(req, SchemaConstants.SYSTEM_KEYSPACE_NAME, BUILT_VIEWS), keyspaceName, viewName);
+ UntypedResultSet result = executeInternal(format(req, SchemaConstants.SYSTEM_KEYSPACE_NAME, BUILT_VIEWS), keyspaceName, viewName);
return !result.isEmpty();
}
public static boolean isViewStatusReplicated(String keyspaceName, String viewName)
{
String req = "SELECT status_replicated FROM %s.\"%s\" WHERE keyspace_name=? AND view_name=?";
- UntypedResultSet result = executeInternal(String.format(req, SchemaConstants.SYSTEM_KEYSPACE_NAME, BUILT_VIEWS), keyspaceName, viewName);
+ UntypedResultSet result = executeInternal(format(req, SchemaConstants.SYSTEM_KEYSPACE_NAME, BUILT_VIEWS), keyspaceName, viewName);
if (result.isEmpty())
return false;
@@ -579,28 +537,22 @@
public static void setViewBuilt(String keyspaceName, String viewName, boolean replicated)
{
+ if (isViewBuilt(keyspaceName, viewName) && isViewStatusReplicated(keyspaceName, viewName) == replicated)
+ return;
+
String req = "INSERT INTO %s.\"%s\" (keyspace_name, view_name, status_replicated) VALUES (?, ?, ?)";
- executeInternal(String.format(req, SchemaConstants.SYSTEM_KEYSPACE_NAME, BUILT_VIEWS), keyspaceName, viewName, replicated);
+ executeInternal(format(req, SchemaConstants.SYSTEM_KEYSPACE_NAME, BUILT_VIEWS), keyspaceName, viewName, replicated);
forceBlockingFlush(BUILT_VIEWS);
}
public static void setViewRemoved(String keyspaceName, String viewName)
{
- String buildReq = "DELETE FROM %S.%s WHERE keyspace_name = ? AND view_name = ?";
- executeInternal(String.format(buildReq, SchemaConstants.SYSTEM_KEYSPACE_NAME, VIEWS_BUILDS_IN_PROGRESS), keyspaceName, viewName);
- forceBlockingFlush(VIEWS_BUILDS_IN_PROGRESS);
+ String buildReq = "DELETE FROM %s.%s WHERE keyspace_name = ? AND view_name = ?";
+ executeInternal(String.format(buildReq, SchemaConstants.SYSTEM_KEYSPACE_NAME, VIEW_BUILDS_IN_PROGRESS), keyspaceName, viewName);
- String builtReq = "DELETE FROM %s.\"%s\" WHERE keyspace_name = ? AND view_name = ?";
+ String builtReq = "DELETE FROM %s.\"%s\" WHERE keyspace_name = ? AND view_name = ? IF EXISTS";
executeInternal(String.format(builtReq, SchemaConstants.SYSTEM_KEYSPACE_NAME, BUILT_VIEWS), keyspaceName, viewName);
- forceBlockingFlush(BUILT_VIEWS);
- }
-
- public static void beginViewBuild(String ksname, String viewName, int generationNumber)
- {
- executeInternal(String.format("INSERT INTO system.%s (keyspace_name, view_name, generation_number) VALUES (?, ?, ?)", VIEWS_BUILDS_IN_PROGRESS),
- ksname,
- viewName,
- generationNumber);
+ forceBlockingFlush(VIEW_BUILDS_IN_PROGRESS, BUILT_VIEWS);
}
public static void finishViewBuildStatus(String ksname, String viewName)
@@ -611,8 +563,8 @@
// Also, if writing to the built_view succeeds, but the view_builds_in_progress deletion fails, we will be able
// to skip the view build next boot.
setViewBuilt(ksname, viewName, false);
- executeInternal(String.format("DELETE FROM system.%s WHERE keyspace_name = ? AND view_name = ?", VIEWS_BUILDS_IN_PROGRESS), ksname, viewName);
- forceBlockingFlush(VIEWS_BUILDS_IN_PROGRESS);
+ executeInternal(String.format("DELETE FROM system.%s WHERE keyspace_name = ? AND view_name = ?", VIEW_BUILDS_IN_PROGRESS), ksname, viewName);
+ forceBlockingFlush(VIEW_BUILDS_IN_PROGRESS);
}
public static void setViewBuiltReplicated(String ksname, String viewName)
@@ -620,39 +572,47 @@
setViewBuilt(ksname, viewName, true);
}
- public static void updateViewBuildStatus(String ksname, String viewName, Token token)
+ public static void updateViewBuildStatus(String ksname, String viewName, Range<Token> range, Token lastToken, long keysBuilt)
{
- String req = "INSERT INTO system.%s (keyspace_name, view_name, last_token) VALUES (?, ?, ?)";
- Token.TokenFactory factory = ViewsBuildsInProgress.partitioner.getTokenFactory();
- executeInternal(String.format(req, VIEWS_BUILDS_IN_PROGRESS), ksname, viewName, factory.toString(token));
+ String req = "INSERT INTO system.%s (keyspace_name, view_name, start_token, end_token, last_token, keys_built) VALUES (?, ?, ?, ?, ?, ?)";
+ Token.TokenFactory factory = ViewBuildsInProgress.partitioner.getTokenFactory();
+ executeInternal(format(req, VIEW_BUILDS_IN_PROGRESS),
+ ksname,
+ viewName,
+ factory.toString(range.left),
+ factory.toString(range.right),
+ factory.toString(lastToken),
+ keysBuilt);
}
- public static Pair<Integer, Token> getViewBuildStatus(String ksname, String viewName)
+ public static Map<Range<Token>, Pair<Token, Long>> getViewBuildStatus(String ksname, String viewName)
{
- String req = "SELECT generation_number, last_token FROM system.%s WHERE keyspace_name = ? AND view_name = ?";
- UntypedResultSet queryResultSet = executeInternal(String.format(req, VIEWS_BUILDS_IN_PROGRESS), ksname, viewName);
- if (queryResultSet == null || queryResultSet.isEmpty())
- return null;
+ String req = "SELECT start_token, end_token, last_token, keys_built FROM system.%s WHERE keyspace_name = ? AND view_name = ?";
+ Token.TokenFactory factory = ViewBuildsInProgress.partitioner.getTokenFactory();
+ UntypedResultSet rs = executeInternal(format(req, VIEW_BUILDS_IN_PROGRESS), ksname, viewName);
- UntypedResultSet.Row row = queryResultSet.one();
+ if (rs == null || rs.isEmpty())
+ return Collections.emptyMap();
- Integer generation = null;
- Token lastKey = null;
- if (row.has("generation_number"))
- generation = row.getInt("generation_number");
- if (row.has("last_key"))
+ Map<Range<Token>, Pair<Token, Long>> status = new HashMap<>();
+ for (UntypedResultSet.Row row : rs)
{
- Token.TokenFactory factory = ViewsBuildsInProgress.partitioner.getTokenFactory();
- lastKey = factory.fromString(row.getString("last_key"));
- }
+ Token start = factory.fromString(row.getString("start_token"));
+ Token end = factory.fromString(row.getString("end_token"));
+ Range<Token> range = new Range<>(start, end);
- return Pair.create(generation, lastKey);
+ Token lastToken = row.has("last_token") ? factory.fromString(row.getString("last_token")) : null;
+ long keysBuilt = row.has("keys_built") ? row.getLong("keys_built") : 0;
+
+ status.put(range, Pair.create(lastToken, keysBuilt));
+ }
+ return status;
}
public static synchronized void saveTruncationRecord(ColumnFamilyStore cfs, long truncatedAt, CommitLogPosition position)
{
String req = "UPDATE system.%s SET truncated_at = truncated_at + ? WHERE key = '%s'";
- executeInternal(String.format(req, LOCAL, LOCAL), truncationAsMapEntry(cfs, truncatedAt, position));
+ executeInternal(format(req, LOCAL, LOCAL), truncationAsMapEntry(cfs, truncatedAt, position));
truncationRecords = null;
forceBlockingFlush(LOCAL);
}
@@ -660,10 +620,14 @@
/**
* This method is used to remove information about truncation time for specified column family
*/
- public static synchronized void removeTruncationRecord(UUID cfId)
+ public static synchronized void removeTruncationRecord(TableId id)
{
+ Pair<CommitLogPosition, Long> truncationRecord = getTruncationRecord(id);
+ if (truncationRecord == null)
+ return;
+
String req = "DELETE truncated_at[?] from system.%s WHERE key = '%s'";
- executeInternal(String.format(req, LOCAL, LOCAL), cfId);
+ executeInternal(format(req, LOCAL, LOCAL), id.asUUID());
truncationRecords = null;
forceBlockingFlush(LOCAL);
}
@@ -674,7 +638,7 @@
{
CommitLogPosition.serializer.serialize(position, out);
out.writeLong(truncatedAt);
- return singletonMap(cfs.metadata.cfId, out.asNewBuffer());
+ return singletonMap(cfs.metadata.id.asUUID(), out.asNewBuffer());
}
catch (IOException e)
{
@@ -682,36 +646,36 @@
}
}
- public static CommitLogPosition getTruncatedPosition(UUID cfId)
+ public static CommitLogPosition getTruncatedPosition(TableId id)
{
- Pair<CommitLogPosition, Long> record = getTruncationRecord(cfId);
+ Pair<CommitLogPosition, Long> record = getTruncationRecord(id);
return record == null ? null : record.left;
}
- public static long getTruncatedAt(UUID cfId)
+ public static long getTruncatedAt(TableId id)
{
- Pair<CommitLogPosition, Long> record = getTruncationRecord(cfId);
+ Pair<CommitLogPosition, Long> record = getTruncationRecord(id);
return record == null ? Long.MIN_VALUE : record.right;
}
- private static synchronized Pair<CommitLogPosition, Long> getTruncationRecord(UUID cfId)
+ private static synchronized Pair<CommitLogPosition, Long> getTruncationRecord(TableId id)
{
if (truncationRecords == null)
truncationRecords = readTruncationRecords();
- return truncationRecords.get(cfId);
+ return truncationRecords.get(id);
}
- private static Map<UUID, Pair<CommitLogPosition, Long>> readTruncationRecords()
+ private static Map<TableId, Pair<CommitLogPosition, Long>> readTruncationRecords()
{
- UntypedResultSet rows = executeInternal(String.format("SELECT truncated_at FROM system.%s WHERE key = '%s'", LOCAL, LOCAL));
+ UntypedResultSet rows = executeInternal(format("SELECT truncated_at FROM system.%s WHERE key = '%s'", LOCAL, LOCAL));
- Map<UUID, Pair<CommitLogPosition, Long>> records = new HashMap<>();
+ Map<TableId, Pair<CommitLogPosition, Long>> records = new HashMap<>();
if (!rows.isEmpty() && rows.one().has("truncated_at"))
{
Map<UUID, ByteBuffer> map = rows.one().getMap("truncated_at", UUIDType.instance, BytesType.instance);
for (Map.Entry<UUID, ByteBuffer> entry : map.entrySet())
- records.put(entry.getKey(), truncationRecordFromBlob(entry.getValue()));
+ records.put(TableId.fromUUID(entry.getKey()), truncationRecordFromBlob(entry.getValue()));
}
return records;
@@ -732,54 +696,71 @@
/**
* Record tokens being used by another node
*/
- public static Future<?> updateTokens(final InetAddress ep, final Collection<Token> tokens, ExecutorService executorService)
+ public static synchronized void updateTokens(InetAddressAndPort ep, Collection<Token> tokens)
{
- if (ep.equals(FBUtilities.getBroadcastAddress()))
- return Futures.immediateFuture(null);
-
- String req = "INSERT INTO system.%s (peer, tokens) VALUES (?, ?)";
- return executorService.submit((Runnable) () -> executeInternal(String.format(req, PEERS), ep, tokensAsSet(tokens)));
- }
-
- public static void updatePreferredIP(InetAddress ep, InetAddress preferred_ip)
- {
- String req = "INSERT INTO system.%s (peer, preferred_ip) VALUES (?, ?)";
- executeInternal(String.format(req, PEERS), ep, preferred_ip);
- forceBlockingFlush(PEERS);
- }
-
- public static Future<?> updatePeerInfo(final InetAddress ep, final String columnName, final Object value, ExecutorService executorService)
- {
- if (ep.equals(FBUtilities.getBroadcastAddress()))
- return Futures.immediateFuture(null);
-
- String req = "INSERT INTO system.%s (peer, %s) VALUES (?, ?)";
- return executorService.submit((Runnable) () -> executeInternal(String.format(req, PEERS, columnName), ep, value));
- }
-
- public static void updatePeerReleaseVersion(final InetAddress ep, final Object value, Runnable postUpdateTask, ExecutorService executorService)
- {
- if (ep.equals(FBUtilities.getBroadcastAddress()))
+ if (ep.equals(FBUtilities.getBroadcastAddressAndPort()))
return;
- String req = "INSERT INTO system.%s (peer, release_version) VALUES (?, ?)";
- executorService.execute(() -> {
- executeInternal(String.format(req, PEERS), ep, value);
- postUpdateTask.run();
- });
+ String req = "INSERT INTO system.%s (peer, tokens) VALUES (?, ?)";
+ executeInternal(String.format(req, LEGACY_PEERS), ep.address, tokensAsSet(tokens));
+ req = "INSERT INTO system.%s (peer, peer_port, tokens) VALUES (?, ?, ?)";
+ executeInternal(String.format(req, PEERS_V2), ep.address, ep.port, tokensAsSet(tokens));
}
- public static synchronized void updateHintsDropped(InetAddress ep, UUID timePeriod, int value)
+ public static synchronized boolean updatePreferredIP(InetAddressAndPort ep, InetAddressAndPort preferred_ip)
+ {
+ if (preferred_ip.equals(getPreferredIP(ep)))
+ return false;
+
+ String req = "INSERT INTO system.%s (peer, preferred_ip) VALUES (?, ?)";
+ executeInternal(String.format(req, LEGACY_PEERS), ep.address, preferred_ip.address);
+ req = "INSERT INTO system.%s (peer, peer_port, preferred_ip, preferred_port) VALUES (?, ?, ?, ?)";
+ executeInternal(String.format(req, PEERS_V2), ep.address, ep.port, preferred_ip.address, preferred_ip.port);
+ forceBlockingFlush(LEGACY_PEERS, PEERS_V2);
+ return true;
+ }
+
+ public static synchronized void updatePeerInfo(InetAddressAndPort ep, String columnName, Object value)
+ {
+ if (ep.equals(FBUtilities.getBroadcastAddressAndPort()))
+ return;
+
+ String req = "INSERT INTO system.%s (peer, %s) VALUES (?, ?)";
+ executeInternal(String.format(req, LEGACY_PEERS, columnName), ep.address, value);
+ //This column doesn't match across the two tables
+ if (columnName.equals("rpc_address"))
+ {
+ columnName = "native_address";
+ }
+ req = "INSERT INTO system.%s (peer, peer_port, %s) VALUES (?, ?, ?)";
+ executeInternal(String.format(req, PEERS_V2, columnName), ep.address, ep.port, value);
+ }
+
+ public static synchronized void updatePeerNativeAddress(InetAddressAndPort ep, InetAddressAndPort address)
+ {
+ if (ep.equals(FBUtilities.getBroadcastAddressAndPort()))
+ return;
+
+ String req = "INSERT INTO system.%s (peer, rpc_address) VALUES (?, ?)";
+ executeInternal(String.format(req, LEGACY_PEERS), ep.address, address.address);
+ req = "INSERT INTO system.%s (peer, peer_port, native_address, native_port) VALUES (?, ?, ?, ?)";
+ executeInternal(String.format(req, PEERS_V2), ep.address, ep.port, address.address, address.port);
+ }
+
+
+ public static synchronized void updateHintsDropped(InetAddressAndPort ep, UUID timePeriod, int value)
{
// with 30 day TTL
String req = "UPDATE system.%s USING TTL 2592000 SET hints_dropped[ ? ] = ? WHERE peer = ?";
- executeInternal(String.format(req, PEER_EVENTS), timePeriod, value, ep);
+ executeInternal(String.format(req, LEGACY_PEER_EVENTS), timePeriod, value, ep.address);
+ req = "UPDATE system.%s USING TTL 2592000 SET hints_dropped[ ? ] = ? WHERE peer = ? AND peer_port = ?";
+ executeInternal(String.format(req, PEER_EVENTS_V2), timePeriod, value, ep.address, ep.port);
}
public static synchronized void updateSchemaVersion(UUID version)
{
String req = "INSERT INTO system.%s (key, schema_version) VALUES ('%s', ?)";
- executeInternal(String.format(req, LOCAL, LOCAL), version);
+ executeInternal(format(req, LOCAL, LOCAL), version);
}
private static Set<String> tokensAsSet(Collection<Token> tokens)
@@ -805,40 +786,57 @@
/**
* Remove stored tokens being used by another node
*/
- public static void removeEndpoint(InetAddress ep)
+ public static synchronized void removeEndpoint(InetAddressAndPort ep)
{
String req = "DELETE FROM system.%s WHERE peer = ?";
- executeInternal(String.format(req, PEERS), ep);
- forceBlockingFlush(PEERS);
+ executeInternal(String.format(req, LEGACY_PEERS), ep.address);
+ req = String.format("DELETE FROM system.%s WHERE peer = ? AND peer_port = ?", PEERS_V2);
+ executeInternal(req, ep.address, ep.port);
+ forceBlockingFlush(LEGACY_PEERS, PEERS_V2);
}
/**
* This method is used to update the System Keyspace with the new tokens for this node
- */
+ */
public static synchronized void updateTokens(Collection<Token> tokens)
{
assert !tokens.isEmpty() : "removeEndpoint should be used instead";
+
+ Collection<Token> savedTokens = getSavedTokens();
+ if (tokens.containsAll(savedTokens) && tokens.size() == savedTokens.size())
+ return;
+
String req = "INSERT INTO system.%s (key, tokens) VALUES ('%s', ?)";
- executeInternal(String.format(req, LOCAL, LOCAL), tokensAsSet(tokens));
+ executeInternal(format(req, LOCAL, LOCAL), tokensAsSet(tokens));
forceBlockingFlush(LOCAL);
}
- public static void forceBlockingFlush(String cfname)
+ public static void forceBlockingFlush(String ...cfnames)
{
if (!DatabaseDescriptor.isUnsafeSystem())
- FBUtilities.waitOnFuture(Keyspace.open(SchemaConstants.SYSTEM_KEYSPACE_NAME).getColumnFamilyStore(cfname).forceFlush());
+ {
+ List<ListenableFuture<CommitLogPosition>> futures = new ArrayList<>();
+
+ for (String cfname : cfnames)
+ {
+ futures.add(Keyspace.open(SchemaConstants.SYSTEM_KEYSPACE_NAME).getColumnFamilyStore(cfname).forceFlush());
+ }
+ FBUtilities.waitOnFutures(futures);
+ }
}
/**
* Return a map of stored tokens to IP addresses
*
*/
- public static SetMultimap<InetAddress, Token> loadTokens()
+ public static SetMultimap<InetAddressAndPort, Token> loadTokens()
{
- SetMultimap<InetAddress, Token> tokenMap = HashMultimap.create();
- for (UntypedResultSet.Row row : executeInternal("SELECT peer, tokens FROM system." + PEERS))
+ SetMultimap<InetAddressAndPort, Token> tokenMap = HashMultimap.create();
+ for (UntypedResultSet.Row row : executeInternal("SELECT peer, peer_port, tokens FROM system." + PEERS_V2))
{
- InetAddress peer = row.getInetAddress("peer");
+ InetAddress address = row.getInetAddress("peer");
+ Integer port = row.getInt("peer_port");
+ InetAddressAndPort peer = InetAddressAndPort.getByAddressOverrideDefaults(address, port);
if (row.has("tokens"))
tokenMap.putAll(peer, deserializeTokens(row.getSet("tokens", UTF8Type.instance)));
}
@@ -850,12 +848,14 @@
* Return a map of store host_ids to IP addresses
*
*/
- public static Map<InetAddress, UUID> loadHostIds()
+ public static Map<InetAddressAndPort, UUID> loadHostIds()
{
- Map<InetAddress, UUID> hostIdMap = new HashMap<>();
- for (UntypedResultSet.Row row : executeInternal("SELECT peer, host_id FROM system." + PEERS))
+ Map<InetAddressAndPort, UUID> hostIdMap = new HashMap<>();
+ for (UntypedResultSet.Row row : executeInternal("SELECT peer, peer_port, host_id FROM system." + PEERS_V2))
{
- InetAddress peer = row.getInetAddress("peer");
+ InetAddress address = row.getInetAddress("peer");
+ Integer port = row.getInt("peer_port");
+ InetAddressAndPort peer = InetAddressAndPort.getByAddressOverrideDefaults(address, port);
if (row.has("host_id"))
{
hostIdMap.put(peer, row.getUUID("host_id"));
@@ -865,60 +865,34 @@
}
/**
- * Return a map of IP address to C* version. If an invalid version string, or no version
- * at all is stored for a given peer IP, then NULL_VERSION will be reported for that peer
- */
- public static Map<InetAddress, CassandraVersion> loadPeerVersions()
- {
- Map<InetAddress, CassandraVersion> releaseVersionMap = new HashMap<>();
- for (UntypedResultSet.Row row : executeInternal("SELECT peer, release_version FROM system." + PEERS))
- {
- InetAddress peer = row.getInetAddress("peer");
- if (row.has("release_version"))
- {
- try
- {
- releaseVersionMap.put(peer, new CassandraVersion(row.getString("release_version")));
- }
- catch (IllegalArgumentException e)
- {
- logger.info("Invalid version string found for {}", peer);
- releaseVersionMap.put(peer, NULL_VERSION);
- }
- }
- else
- {
- logger.info("No version string found for {}", peer);
- releaseVersionMap.put(peer, NULL_VERSION);
- }
- }
- return releaseVersionMap;
- }
-
- /**
* Get preferred IP for given endpoint if it is known. Otherwise this returns given endpoint itself.
*
* @param ep endpoint address to check
* @return Preferred IP for given endpoint if present, otherwise returns given ep
*/
- public static InetAddress getPreferredIP(InetAddress ep)
+ public static InetAddressAndPort getPreferredIP(InetAddressAndPort ep)
{
- String req = "SELECT preferred_ip FROM system.%s WHERE peer=?";
- UntypedResultSet result = executeInternal(String.format(req, PEERS), ep);
+ String req = "SELECT preferred_ip, preferred_port FROM system.%s WHERE peer=? AND peer_port = ?";
+ UntypedResultSet result = executeInternal(String.format(req, PEERS_V2), ep.address, ep.port);
if (!result.isEmpty() && result.one().has("preferred_ip"))
- return result.one().getInetAddress("preferred_ip");
+ {
+ UntypedResultSet.Row row = result.one();
+ return InetAddressAndPort.getByAddressOverrideDefaults(row.getInetAddress("preferred_ip"), row.getInt("preferred_port"));
+ }
return ep;
}
/**
* Return a map of IP addresses containing a map of dc and rack info
*/
- public static Map<InetAddress, Map<String,String>> loadDcRackInfo()
+ public static Map<InetAddressAndPort, Map<String,String>> loadDcRackInfo()
{
- Map<InetAddress, Map<String, String>> result = new HashMap<>();
- for (UntypedResultSet.Row row : executeInternal("SELECT peer, data_center, rack from system." + PEERS))
+ Map<InetAddressAndPort, Map<String, String>> result = new HashMap<>();
+ for (UntypedResultSet.Row row : executeInternal("SELECT peer, peer_port, data_center, rack from system." + PEERS_V2))
{
- InetAddress peer = row.getInetAddress("peer");
+ InetAddress address = row.getInetAddress("peer");
+ Integer port = row.getInt("peer_port");
+ InetAddressAndPort peer = InetAddressAndPort.getByAddressOverrideDefaults(address, port);
if (row.has("data_center") && row.has("rack"))
{
Map<String, String> dcRack = new HashMap<>();
@@ -937,16 +911,16 @@
* @param ep endpoint address to check
* @return Release version or null if version is unknown.
*/
- public static CassandraVersion getReleaseVersion(InetAddress ep)
+ public static CassandraVersion getReleaseVersion(InetAddressAndPort ep)
{
try
{
- if (FBUtilities.getBroadcastAddress().equals(ep))
+ if (FBUtilities.getBroadcastAddressAndPort().equals(ep))
{
return new CassandraVersion(FBUtilities.getReleaseVersionString());
}
- String req = "SELECT release_version FROM system.%s WHERE peer=?";
- UntypedResultSet result = executeInternal(String.format(req, PEERS), ep);
+ String req = "SELECT release_version FROM system.%s WHERE peer=? AND peer_port=?";
+ UntypedResultSet result = executeInternal(String.format(req, PEERS_V2), ep.address, ep.port);
if (result != null && result.one().has("release_version"))
{
return new CassandraVersion(result.one().getString("release_version"));
@@ -985,7 +959,7 @@
ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(LOCAL);
String req = "SELECT cluster_name FROM system.%s WHERE key='%s'";
- UntypedResultSet result = executeInternal(String.format(req, LOCAL, LOCAL));
+ UntypedResultSet result = executeInternal(format(req, LOCAL, LOCAL));
if (result.isEmpty() || !result.one().has("cluster_name"))
{
@@ -1005,7 +979,7 @@
public static Collection<Token> getSavedTokens()
{
String req = "SELECT tokens FROM system.%s WHERE key='%s'";
- UntypedResultSet result = executeInternal(String.format(req, LOCAL, LOCAL));
+ UntypedResultSet result = executeInternal(format(req, LOCAL, LOCAL));
return result.isEmpty() || !result.one().has("tokens")
? Collections.<Token>emptyList()
: deserializeTokens(result.one().getSet("tokens", UTF8Type.instance));
@@ -1014,7 +988,7 @@
public static int incrementAndGetGeneration()
{
String req = "SELECT gossip_generation FROM system.%s WHERE key='%s'";
- UntypedResultSet result = executeInternal(String.format(req, LOCAL, LOCAL));
+ UntypedResultSet result = executeInternal(format(req, LOCAL, LOCAL));
int generation;
if (result.isEmpty() || !result.one().has("gossip_generation"))
@@ -1042,7 +1016,7 @@
}
req = "INSERT INTO system.%s (key, gossip_generation) VALUES ('%s', ?)";
- executeInternal(String.format(req, LOCAL, LOCAL), generation);
+ executeInternal(format(req, LOCAL, LOCAL), generation);
forceBlockingFlush(LOCAL);
return generation;
@@ -1051,7 +1025,7 @@
public static BootstrapState getBootstrapState()
{
String req = "SELECT bootstrapped FROM system.%s WHERE key='%s'";
- UntypedResultSet result = executeInternal(String.format(req, LOCAL, LOCAL));
+ UntypedResultSet result = executeInternal(format(req, LOCAL, LOCAL));
if (result.isEmpty() || !result.one().has("bootstrapped"))
return BootstrapState.NEEDS_BOOTSTRAP;
@@ -1076,28 +1050,31 @@
public static void setBootstrapState(BootstrapState state)
{
+ if (getBootstrapState() == state)
+ return;
+
String req = "INSERT INTO system.%s (key, bootstrapped) VALUES ('%s', ?)";
- executeInternal(String.format(req, LOCAL, LOCAL), state.name());
+ executeInternal(format(req, LOCAL, LOCAL), state.name());
forceBlockingFlush(LOCAL);
}
public static boolean isIndexBuilt(String keyspaceName, String indexName)
{
String req = "SELECT index_name FROM %s.\"%s\" WHERE table_name=? AND index_name=?";
- UntypedResultSet result = executeInternal(String.format(req, SchemaConstants.SYSTEM_KEYSPACE_NAME, BUILT_INDEXES), keyspaceName, indexName);
+ UntypedResultSet result = executeInternal(format(req, SchemaConstants.SYSTEM_KEYSPACE_NAME, BUILT_INDEXES), keyspaceName, indexName);
return !result.isEmpty();
}
public static void setIndexBuilt(String keyspaceName, String indexName)
{
- String req = "INSERT INTO %s.\"%s\" (table_name, index_name) VALUES (?, ?)";
+ String req = "INSERT INTO %s.\"%s\" (table_name, index_name) VALUES (?, ?) IF NOT EXISTS;";
executeInternal(String.format(req, SchemaConstants.SYSTEM_KEYSPACE_NAME, BUILT_INDEXES), keyspaceName, indexName);
forceBlockingFlush(BUILT_INDEXES);
}
public static void setIndexRemoved(String keyspaceName, String indexName)
{
- String req = "DELETE FROM %s.\"%s\" WHERE table_name = ? AND index_name = ?";
+ String req = "DELETE FROM %s.\"%s\" WHERE table_name = ? AND index_name = ? IF EXISTS";
executeInternal(String.format(req, SchemaConstants.SYSTEM_KEYSPACE_NAME, BUILT_INDEXES), keyspaceName, indexName);
forceBlockingFlush(BUILT_INDEXES);
}
@@ -1106,7 +1083,7 @@
{
List<String> names = new ArrayList<>(indexNames);
String req = "SELECT index_name from %s.\"%s\" WHERE table_name=? AND index_name IN ?";
- UntypedResultSet results = executeInternal(String.format(req, SchemaConstants.SYSTEM_KEYSPACE_NAME, BUILT_INDEXES), keyspaceName, names);
+ UntypedResultSet results = executeInternal(format(req, SchemaConstants.SYSTEM_KEYSPACE_NAME, BUILT_INDEXES), keyspaceName, names);
return StreamSupport.stream(results.spliterator(), false)
.map(r -> r.getString("index_name"))
.collect(Collectors.toList());
@@ -1119,7 +1096,7 @@
public static UUID getLocalHostId()
{
String req = "SELECT host_id FROM system.%s WHERE key='%s'";
- UntypedResultSet result = executeInternal(String.format(req, LOCAL, LOCAL));
+ UntypedResultSet result = executeInternal(format(req, LOCAL, LOCAL));
// Look up the Host UUID (return it if found)
if (!result.isEmpty() && result.one().has("host_id"))
@@ -1137,7 +1114,7 @@
public static UUID setLocalHostId(UUID hostId)
{
String req = "INSERT INTO system.%s (key, host_id) VALUES ('%s', ?)";
- executeInternal(String.format(req, LOCAL, LOCAL), hostId);
+ executeInternal(format(req, LOCAL, LOCAL), hostId);
return hostId;
}
@@ -1147,7 +1124,7 @@
public static String getRack()
{
String req = "SELECT rack FROM system.%s WHERE key='%s'";
- UntypedResultSet result = executeInternal(String.format(req, LOCAL, LOCAL));
+ UntypedResultSet result = executeInternal(format(req, LOCAL, LOCAL));
// Look up the Rack (return it if found)
if (!result.isEmpty() && result.one().has("rack"))
@@ -1162,7 +1139,7 @@
public static String getDatacenter()
{
String req = "SELECT data_center FROM system.%s WHERE key='%s'";
- UntypedResultSet result = executeInternal(String.format(req, LOCAL, LOCAL));
+ UntypedResultSet result = executeInternal(format(req, LOCAL, LOCAL));
// Look up the Data center (return it if found)
if (!result.isEmpty() && result.one().has("data_center"))
@@ -1171,25 +1148,26 @@
return null;
}
- public static PaxosState loadPaxosState(DecoratedKey key, CFMetaData metadata, int nowInSec)
+ public static PaxosState loadPaxosState(DecoratedKey key, TableMetadata metadata, int nowInSec)
{
String req = "SELECT * FROM system.%s WHERE row_key = ? AND cf_id = ?";
- UntypedResultSet results = QueryProcessor.executeInternalWithNow(nowInSec, System.nanoTime(), String.format(req, PAXOS), key.getKey(), metadata.cfId);
+ UntypedResultSet results = QueryProcessor.executeInternalWithNow(nowInSec, System.nanoTime(), format(req, PAXOS), key.getKey(), metadata.id.asUUID());
if (results.isEmpty())
return new PaxosState(key, metadata);
UntypedResultSet.Row row = results.one();
+
Commit promised = row.has("in_progress_ballot")
- ? new Commit(row.getUUID("in_progress_ballot"), new PartitionUpdate(metadata, key, metadata.partitionColumns(), 1))
+ ? new Commit(row.getUUID("in_progress_ballot"), new PartitionUpdate.Builder(metadata, key, metadata.regularAndStaticColumns(), 1).build())
: Commit.emptyCommit(key, metadata);
// either we have both a recently accepted ballot and update or we have neither
- int proposalVersion = row.has("proposal_version") ? row.getInt("proposal_version") : MessagingService.VERSION_21;
- Commit accepted = row.has("proposal")
- ? new Commit(row.getUUID("proposal_ballot"), PartitionUpdate.fromBytes(row.getBytes("proposal"), proposalVersion, key))
+ Commit accepted = row.has("proposal_version") && row.has("proposal")
+ ? new Commit(row.getUUID("proposal_ballot"),
+ PartitionUpdate.fromBytes(row.getBytes("proposal"), row.getInt("proposal_version")))
: Commit.emptyCommit(key, metadata);
// either most_recent_commit and most_recent_commit_at will both be set, or neither
- int mostRecentVersion = row.has("most_recent_commit_version") ? row.getInt("most_recent_commit_version") : MessagingService.VERSION_21;
- Commit mostRecent = row.has("most_recent_commit")
- ? new Commit(row.getUUID("most_recent_commit_at"), PartitionUpdate.fromBytes(row.getBytes("most_recent_commit"), mostRecentVersion, key))
+ Commit mostRecent = row.has("most_recent_commit_version") && row.has("most_recent_commit")
+ ? new Commit(row.getUUID("most_recent_commit_at"),
+ PartitionUpdate.fromBytes(row.getBytes("most_recent_commit"), row.getInt("most_recent_commit_version")))
: Commit.emptyCommit(key, metadata);
return new PaxosState(promised, accepted, mostRecent);
}
@@ -1197,27 +1175,27 @@
public static void savePaxosPromise(Commit promise)
{
String req = "UPDATE system.%s USING TIMESTAMP ? AND TTL ? SET in_progress_ballot = ? WHERE row_key = ? AND cf_id = ?";
- executeInternal(String.format(req, PAXOS),
+ executeInternal(format(req, PAXOS),
UUIDGen.microsTimestamp(promise.ballot),
paxosTtlSec(promise.update.metadata()),
promise.ballot,
promise.update.partitionKey().getKey(),
- promise.update.metadata().cfId);
+ promise.update.metadata().id.asUUID());
}
public static void savePaxosProposal(Commit proposal)
{
- executeInternal(String.format("UPDATE system.%s USING TIMESTAMP ? AND TTL ? SET proposal_ballot = ?, proposal = ?, proposal_version = ? WHERE row_key = ? AND cf_id = ?", PAXOS),
+ executeInternal(format("UPDATE system.%s USING TIMESTAMP ? AND TTL ? SET proposal_ballot = ?, proposal = ?, proposal_version = ? WHERE row_key = ? AND cf_id = ?", PAXOS),
UUIDGen.microsTimestamp(proposal.ballot),
paxosTtlSec(proposal.update.metadata()),
proposal.ballot,
PartitionUpdate.toBytes(proposal.update, MessagingService.current_version),
MessagingService.current_version,
proposal.update.partitionKey().getKey(),
- proposal.update.metadata().cfId);
+ proposal.update.metadata().id.asUUID());
}
- public static int paxosTtlSec(CFMetaData metadata)
+ public static int paxosTtlSec(TableMetadata metadata)
{
// keep paxos state around for at least 3h
return Math.max(3 * 3600, metadata.params.gcGraceSeconds);
@@ -1228,14 +1206,14 @@
// We always erase the last proposal (with the commit timestamp to no erase more recent proposal in case the commit is old)
// even though that's really just an optimization since SP.beginAndRepairPaxos will exclude accepted proposal older than the mrc.
String cql = "UPDATE system.%s USING TIMESTAMP ? AND TTL ? SET proposal_ballot = null, proposal = null, most_recent_commit_at = ?, most_recent_commit = ?, most_recent_commit_version = ? WHERE row_key = ? AND cf_id = ?";
- executeInternal(String.format(cql, PAXOS),
+ executeInternal(format(cql, PAXOS),
UUIDGen.microsTimestamp(commit.ballot),
paxosTtlSec(commit.update.metadata()),
commit.ballot,
PartitionUpdate.toBytes(commit.update, MessagingService.current_version),
MessagingService.current_version,
commit.update.partitionKey().getKey(),
- commit.update.metadata().cfId);
+ commit.update.metadata().id.asUUID());
}
/**
@@ -1248,7 +1226,7 @@
public static RestorableMeter getSSTableReadMeter(String keyspace, String table, int generation)
{
String cql = "SELECT * FROM system.%s WHERE keyspace_name=? and columnfamily_name=? and generation=?";
- UntypedResultSet results = executeInternal(String.format(cql, SSTABLE_ACTIVITY), keyspace, table, generation);
+ UntypedResultSet results = executeInternal(format(cql, SSTABLE_ACTIVITY), keyspace, table, generation);
if (results.isEmpty())
return new RestorableMeter();
@@ -1266,7 +1244,7 @@
{
// Store values with a one-day TTL to handle corner cases where cleanup might not occur
String cql = "INSERT INTO system.%s (keyspace_name, columnfamily_name, generation, rate_15m, rate_120m) VALUES (?, ?, ?, ?, ?) USING TTL 864000";
- executeInternal(String.format(cql, SSTABLE_ACTIVITY),
+ executeInternal(format(cql, SSTABLE_ACTIVITY),
keyspace,
table,
generation,
@@ -1280,7 +1258,7 @@
public static void clearSSTableReadMeter(String keyspace, String table, int generation)
{
String cql = "DELETE FROM system.%s WHERE keyspace_name=? AND columnfamily_name=? and generation=?";
- executeInternal(String.format(cql, SSTABLE_ACTIVITY), keyspace, table, generation);
+ executeInternal(format(cql, SSTABLE_ACTIVITY), keyspace, table, generation);
}
/**
@@ -1289,98 +1267,130 @@
public static void updateSizeEstimates(String keyspace, String table, Map<Range<Token>, Pair<Long, Long>> estimates)
{
long timestamp = FBUtilities.timestampMicros();
- PartitionUpdate update = new PartitionUpdate(SizeEstimates, UTF8Type.instance.decompose(keyspace), SizeEstimates.partitionColumns(), estimates.size());
- Mutation mutation = new Mutation(update);
-
- // delete all previous values with a single range tombstone.
int nowInSec = FBUtilities.nowInSeconds();
- update.add(new RangeTombstone(Slice.make(SizeEstimates.comparator, table), new DeletionTime(timestamp - 1, nowInSec)));
+ PartitionUpdate.Builder update = new PartitionUpdate.Builder(LegacySizeEstimates, UTF8Type.instance.decompose(keyspace), LegacySizeEstimates.regularAndStaticColumns(), estimates.size());
+ // delete all previous values with a single range tombstone.
+ update.add(new RangeTombstone(Slice.make(LegacySizeEstimates.comparator, table), new DeletionTime(timestamp - 1, nowInSec)));
// add a CQL row for each primary token range.
for (Map.Entry<Range<Token>, Pair<Long, Long>> entry : estimates.entrySet())
{
Range<Token> range = entry.getKey();
Pair<Long, Long> values = entry.getValue();
- update.add(Rows.simpleBuilder(SizeEstimates, table, range.left.toString(), range.right.toString())
+ update.add(Rows.simpleBuilder(LegacySizeEstimates, table, range.left.toString(), range.right.toString())
+ .timestamp(timestamp)
+ .add("partitions_count", values.left)
+ .add("mean_partition_size", values.right)
+ .build());
+ }
+ new Mutation(update.build()).apply();
+ }
+
+ /**
+ * Writes the current partition count and size estimates into table_estimates
+ */
+ public static void updateTableEstimates(String keyspace, String table, String type, Map<Range<Token>, Pair<Long, Long>> estimates)
+ {
+ long timestamp = FBUtilities.timestampMicros();
+ int nowInSec = FBUtilities.nowInSeconds();
+ PartitionUpdate.Builder update = new PartitionUpdate.Builder(TableEstimates, UTF8Type.instance.decompose(keyspace), TableEstimates.regularAndStaticColumns(), estimates.size());
+
+ // delete all previous values with a single range tombstone.
+ update.add(new RangeTombstone(Slice.make(TableEstimates.comparator, table, type), new DeletionTime(timestamp - 1, nowInSec)));
+
+ // add a CQL row for each primary token range.
+ for (Map.Entry<Range<Token>, Pair<Long, Long>> entry : estimates.entrySet())
+ {
+ Range<Token> range = entry.getKey();
+ Pair<Long, Long> values = entry.getValue();
+ update.add(Rows.simpleBuilder(TableEstimates, table, type, range.left.toString(), range.right.toString())
.timestamp(timestamp)
.add("partitions_count", values.left)
.add("mean_partition_size", values.right)
.build());
}
- mutation.apply();
+ new Mutation(update.build()).apply();
}
+
/**
* Clears size estimates for a table (on table drop)
*/
- public static void clearSizeEstimates(String keyspace, String table)
+ public static void clearEstimates(String keyspace, String table)
{
- String cql = String.format("DELETE FROM %s.%s WHERE keyspace_name = ? AND table_name = ?", SchemaConstants.SYSTEM_KEYSPACE_NAME, SIZE_ESTIMATES);
+ String cqlFormat = "DELETE FROM %s WHERE keyspace_name = ? AND table_name = ?";
+ String cql = format(cqlFormat, LegacySizeEstimates.toString());
+ executeInternal(cql, keyspace, table);
+ cql = String.format(cqlFormat, TableEstimates.toString());
executeInternal(cql, keyspace, table);
}
/**
- * Clears size estimates for a keyspace (used to manually clean when we miss a keyspace drop)
+ * truncates size_estimates and table_estimates tables
*/
- public static void clearSizeEstimates(String keyspace)
+ public static void clearAllEstimates()
{
- String cql = String.format("DELETE FROM %s.%s WHERE keyspace_name = ?", SchemaConstants.SYSTEM_KEYSPACE_NAME, SIZE_ESTIMATES);
- executeInternal(cql, keyspace);
+ for (TableMetadata table : Arrays.asList(LegacySizeEstimates, TableEstimates))
+ {
+ String cql = String.format("TRUNCATE TABLE " + table.toString());
+ executeInternal(cql);
+ }
+ }
+
+ public static synchronized void updateAvailableRanges(String keyspace, Collection<Range<Token>> completedFullRanges, Collection<Range<Token>> completedTransientRanges)
+ {
+ String cql = "UPDATE system.%s SET full_ranges = full_ranges + ?, transient_ranges = transient_ranges + ? WHERE keyspace_name = ?";
+ executeInternal(format(cql, AVAILABLE_RANGES_V2),
+ completedFullRanges.stream().map(SystemKeyspace::rangeToBytes).collect(Collectors.toSet()),
+ completedTransientRanges.stream().map(SystemKeyspace::rangeToBytes).collect(Collectors.toSet()),
+ keyspace);
}
/**
- * @return A multimap from keyspace to table for all tables with entries in size estimates
+ * List of the streamed ranges, where transientness is encoded based on the source, where range was streamed from.
*/
-
- public static synchronized SetMultimap<String, String> getTablesWithSizeEstimates()
+ public static synchronized AvailableRanges getAvailableRanges(String keyspace, IPartitioner partitioner)
{
- SetMultimap<String, String> keyspaceTableMap = HashMultimap.create();
- String cql = String.format("SELECT keyspace_name, table_name FROM %s.%s", SchemaConstants.SYSTEM_KEYSPACE_NAME, SIZE_ESTIMATES);
- UntypedResultSet rs = executeInternal(cql);
- for (UntypedResultSet.Row row : rs)
- {
- keyspaceTableMap.put(row.getString("keyspace_name"), row.getString("table_name"));
- }
-
- return keyspaceTableMap;
- }
-
- public static synchronized void updateAvailableRanges(String keyspace, Collection<Range<Token>> completedRanges)
- {
- String cql = "UPDATE system.%s SET ranges = ranges + ? WHERE keyspace_name = ?";
- Set<ByteBuffer> rangesToUpdate = new HashSet<>(completedRanges.size());
- for (Range<Token> range : completedRanges)
- {
- rangesToUpdate.add(rangeToBytes(range));
- }
- executeInternal(String.format(cql, AVAILABLE_RANGES), rangesToUpdate, keyspace);
- }
-
- public static synchronized Set<Range<Token>> getAvailableRanges(String keyspace, IPartitioner partitioner)
- {
- Set<Range<Token>> result = new HashSet<>();
String query = "SELECT * FROM system.%s WHERE keyspace_name=?";
- UntypedResultSet rs = executeInternal(String.format(query, AVAILABLE_RANGES), keyspace);
+ UntypedResultSet rs = executeInternal(format(query, AVAILABLE_RANGES_V2), keyspace);
+
+ ImmutableSet.Builder<Range<Token>> full = new ImmutableSet.Builder<>();
+ ImmutableSet.Builder<Range<Token>> trans = new ImmutableSet.Builder<>();
for (UntypedResultSet.Row row : rs)
{
- Set<ByteBuffer> rawRanges = row.getSet("ranges", BytesType.instance);
- for (ByteBuffer rawRange : rawRanges)
- {
- result.add(byteBufferToRange(rawRange, partitioner));
- }
+ Optional.ofNullable(row.getSet("full_ranges", BytesType.instance))
+ .ifPresent(full_ranges -> full_ranges.stream()
+ .map(buf -> byteBufferToRange(buf, partitioner))
+ .forEach(full::add));
+ Optional.ofNullable(row.getSet("transient_ranges", BytesType.instance))
+ .ifPresent(transient_ranges -> transient_ranges.stream()
+ .map(buf -> byteBufferToRange(buf, partitioner))
+ .forEach(trans::add));
}
- return ImmutableSet.copyOf(result);
+ return new AvailableRanges(full.build(), trans.build());
+ }
+
+ public static class AvailableRanges
+ {
+ public Set<Range<Token>> full;
+ public Set<Range<Token>> trans;
+
+ private AvailableRanges(Set<Range<Token>> full, Set<Range<Token>> trans)
+ {
+ this.full = full;
+ this.trans = trans;
+ }
}
public static void resetAvailableRanges()
{
- ColumnFamilyStore availableRanges = Keyspace.open(SchemaConstants.SYSTEM_KEYSPACE_NAME).getColumnFamilyStore(AVAILABLE_RANGES);
+ ColumnFamilyStore availableRanges = Keyspace.open(SchemaConstants.SYSTEM_KEYSPACE_NAME).getColumnFamilyStore(AVAILABLE_RANGES_V2);
availableRanges.truncateBlocking();
}
- public static synchronized void updateTransferredRanges(String description,
- InetAddress peer,
+ public static synchronized void updateTransferredRanges(StreamOperation streamOperation,
+ InetAddressAndPort peer,
String keyspace,
Collection<Range<Token>> streamedRanges)
{
@@ -1390,17 +1400,21 @@
{
rangesToUpdate.add(rangeToBytes(range));
}
- executeInternal(String.format(cql, TRANSFERRED_RANGES), rangesToUpdate, description, peer, keyspace);
+ executeInternal(format(cql, LEGACY_TRANSFERRED_RANGES), rangesToUpdate, streamOperation.getDescription(), peer.address, keyspace);
+ cql = "UPDATE system.%s SET ranges = ranges + ? WHERE operation = ? AND peer = ? AND peer_port = ? AND keyspace_name = ?";
+ executeInternal(String.format(cql, TRANSFERRED_RANGES_V2), rangesToUpdate, streamOperation.getDescription(), peer.address, peer.port, keyspace);
}
- public static synchronized Map<InetAddress, Set<Range<Token>>> getTransferredRanges(String description, String keyspace, IPartitioner partitioner)
+ public static synchronized Map<InetAddressAndPort, Set<Range<Token>>> getTransferredRanges(String description, String keyspace, IPartitioner partitioner)
{
- Map<InetAddress, Set<Range<Token>>> result = new HashMap<>();
+ Map<InetAddressAndPort, Set<Range<Token>>> result = new HashMap<>();
String query = "SELECT * FROM system.%s WHERE operation = ? AND keyspace_name = ?";
- UntypedResultSet rs = executeInternal(String.format(query, TRANSFERRED_RANGES), description, keyspace);
+ UntypedResultSet rs = executeInternal(String.format(query, TRANSFERRED_RANGES_V2), description, keyspace);
for (UntypedResultSet.Row row : rs)
{
- InetAddress peer = row.getInetAddress("peer");
+ InetAddress peerAddress = row.getInetAddress("peer");
+ int port = row.getInt("peer_port");
+ InetAddressAndPort peer = InetAddressAndPort.getByAddressOverrideDefaults(peerAddress, port);
Set<ByteBuffer> rawRanges = row.getSet("ranges", BytesType.instance);
Set<Range<Token>> ranges = Sets.newHashSetWithExpectedSize(rawRanges.size());
for (ByteBuffer rawRange : rawRanges)
@@ -1414,32 +1428,29 @@
/**
* Compare the release version in the system.local table with the one included in the distro.
- * If they don't match, snapshot all tables in the system keyspace. This is intended to be
- * called at startup to create a backup of the system tables during an upgrade
+ * If they don't match, snapshot all tables in the system and schema keyspaces. This is intended
+ * to be called at startup to create a backup of the system tables during an upgrade
*
* @throws IOException
*/
- public static boolean snapshotOnVersionChange() throws IOException
+ public static void snapshotOnVersionChange() throws IOException
{
String previous = getPreviousVersionString();
String next = FBUtilities.getReleaseVersionString();
FBUtilities.setPreviousReleaseVersionString(previous);
- // if we're restarting after an upgrade, snapshot the system keyspace
+ // if we're restarting after an upgrade, snapshot the system and schema keyspaces
if (!previous.equals(NULL_VERSION.toString()) && !previous.equals(next))
{
- logger.info("Detected version upgrade from {} to {}, snapshotting system keyspace", previous, next);
- String snapshotName = Keyspace.getTimestampedSnapshotName(String.format("upgrade-%s-%s",
- previous,
- next));
- Keyspace systemKs = Keyspace.open(SchemaConstants.SYSTEM_KEYSPACE_NAME);
- systemKs.snapshot(snapshotName, null);
- return true;
+ logger.info("Detected version upgrade from {} to {}, snapshotting system keyspaces", previous, next);
+ String snapshotName = Keyspace.getTimestampedSnapshotName(format("upgrade-%s-%s",
+ previous,
+ next));
+ for (String keyspace : SchemaConstants.LOCAL_SYSTEM_KEYSPACE_NAMES)
+ Keyspace.open(keyspace).snapshot(snapshotName, null);
}
-
- return false;
}
/**
@@ -1456,7 +1467,7 @@
private static String getPreviousVersionString()
{
String req = "SELECT release_version FROM system.%s WHERE key='%s'";
- UntypedResultSet result = executeInternal(String.format(req, SystemKeyspace.LOCAL, SystemKeyspace.LOCAL));
+ UntypedResultSet result = executeInternal(format(req, SystemKeyspace.LOCAL, SystemKeyspace.LOCAL));
if (result.isEmpty() || !result.one().has("release_version"))
{
// it isn't inconceivable that one might try to upgrade a node straight from <= 1.1 to whatever
@@ -1479,50 +1490,23 @@
return result.one().getString("release_version");
}
- /**
- * Check data directories for old files that can be removed when migrating from 2.1 or 2.2 to 3.0,
- * these checks can be removed in 4.0, see CASSANDRA-7066
- */
- public static void migrateDataDirs()
+ @VisibleForTesting
+ public static Set<Range<Token>> rawRangesToRangeSet(Set<ByteBuffer> rawRanges, IPartitioner partitioner)
{
- Iterable<String> dirs = Arrays.asList(DatabaseDescriptor.getAllDataFileLocations());
- for (String dataDir : dirs)
- {
- logger.debug("Checking {} for legacy files", dataDir);
- File dir = new File(dataDir);
- assert dir.exists() : dir + " should have been created by startup checks";
-
- visitDirectory(dir.toPath(),
- File::isDirectory,
- ksdir ->
- {
- logger.trace("Checking {} for legacy files", ksdir);
- visitDirectory(ksdir.toPath(),
- File::isDirectory,
- cfdir ->
- {
- logger.trace("Checking {} for legacy files", cfdir);
-
- if (Descriptor.isLegacyFile(cfdir))
- {
- FileUtils.deleteRecursive(cfdir);
- }
- else
- {
- visitDirectory(cfdir.toPath(),
- Descriptor::isLegacyFile,
- FileUtils::delete);
- }
- });
- });
- }
+ return rawRanges.stream().map(buf -> byteBufferToRange(buf, partitioner)).collect(Collectors.toSet());
}
- private static ByteBuffer rangeToBytes(Range<Token> range)
+ static ByteBuffer rangeToBytes(Range<Token> range)
{
try (DataOutputBuffer out = new DataOutputBuffer())
{
- Range.tokenSerializer.serialize(range, out, MessagingService.VERSION_22);
+ // The format with which token ranges are serialized in the system tables is the pre-3.0 serialization
+ // formot for ranges, so we should maintain that for now. And while we don't really support pre-3.0
+ // messaging versions, we know AbstractBounds.Serializer still support it _exactly_ for this use case, so we
+ // pass 0 as the version to trigger that legacy code.
+ // In the future, it might be worth switching to a stable text format for the ranges to 1) save that and 2)
+ // be more user friendly (the serialization format we currently use is pretty custom).
+ Range.tokenSerializer.serialize(range, out, 0);
return out.buffer();
}
catch (IOException e)
@@ -1536,9 +1520,10 @@
{
try
{
+ // See rangeToBytes above for why version is 0.
return (Range<Token>) Range.tokenSerializer.deserialize(ByteStreams.newDataInput(ByteBufferUtil.getArray(rawRange)),
partitioner,
- MessagingService.VERSION_22);
+ 0);
}
catch (IOException e)
{
@@ -1548,18 +1533,15 @@
public static void writePreparedStatement(String loggedKeyspace, MD5Digest key, String cql)
{
- executeInternal(String.format("INSERT INTO %s.%s"
- + " (logged_keyspace, prepared_id, query_string) VALUES (?, ?, ?)",
- SchemaConstants.SYSTEM_KEYSPACE_NAME, PREPARED_STATEMENTS),
+ executeInternal(format("INSERT INTO %s (logged_keyspace, prepared_id, query_string) VALUES (?, ?, ?)",
+ PreparedStatements.toString()),
loggedKeyspace, key.byteBuffer(), cql);
logger.debug("stored prepared statement for logged keyspace '{}': '{}'", loggedKeyspace, cql);
}
public static void removePreparedStatement(MD5Digest key)
{
- executeInternal(String.format("DELETE FROM %s.%s"
- + " WHERE prepared_id = ?",
- SchemaConstants.SYSTEM_KEYSPACE_NAME, PREPARED_STATEMENTS),
+ executeInternal(format("DELETE FROM %s WHERE prepared_id = ?", PreparedStatements.toString()),
key.byteBuffer());
}
@@ -1571,7 +1553,7 @@
public static List<Pair<String, String>> loadPreparedStatements()
{
- String query = String.format("SELECT logged_keyspace, query_string FROM %s.%s", SchemaConstants.SYSTEM_KEYSPACE_NAME, PREPARED_STATEMENTS);
+ String query = format("SELECT logged_keyspace, query_string FROM %s", PreparedStatements.toString());
UntypedResultSet resultSet = executeOnceInternal(query);
List<Pair<String, String>> r = new ArrayList<>();
for (UntypedResultSet.Row row : resultSet)
diff --git a/src/java/org/apache/cassandra/db/SystemKeyspaceMigrator40.java b/src/java/org/apache/cassandra/db/SystemKeyspaceMigrator40.java
new file mode 100644
index 0000000..e0a58ba
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/SystemKeyspaceMigrator40.java
@@ -0,0 +1,229 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db;
+
+import java.nio.ByteBuffer;
+import java.util.Collections;
+import java.util.Optional;
+import java.util.Set;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.schema.SchemaConstants;
+import org.apache.cassandra.cql3.QueryProcessor;
+import org.apache.cassandra.cql3.UntypedResultSet;
+import org.apache.cassandra.db.marshal.BytesType;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.db.marshal.UUIDType;
+
+/**
+ * Migrate 3.0 versions of some tables to 4.0. In this case it's just extra columns and some keys
+ * that are changed.
+ *
+ * Can't just add the additional columns because they are primary key columns and C* doesn't support changing
+ * key columns even if it's just clustering columns.
+ */
+public class SystemKeyspaceMigrator40
+{
+ static final String legacyPeersName = String.format("%s.%s", SchemaConstants.SYSTEM_KEYSPACE_NAME, SystemKeyspace.LEGACY_PEERS);
+ static final String peersName = String.format("%s.%s", SchemaConstants.SYSTEM_KEYSPACE_NAME, SystemKeyspace.PEERS_V2);
+ static final String legacyPeerEventsName = String.format("%s.%s", SchemaConstants.SYSTEM_KEYSPACE_NAME, SystemKeyspace.LEGACY_PEER_EVENTS);
+ static final String peerEventsName = String.format("%s.%s", SchemaConstants.SYSTEM_KEYSPACE_NAME, SystemKeyspace.PEER_EVENTS_V2);
+ static final String legacyTransferredRangesName = String.format("%s.%s", SchemaConstants.SYSTEM_KEYSPACE_NAME, SystemKeyspace.LEGACY_TRANSFERRED_RANGES);
+ static final String transferredRangesName = String.format("%s.%s", SchemaConstants.SYSTEM_KEYSPACE_NAME, SystemKeyspace.TRANSFERRED_RANGES_V2);
+ static final String legacyAvailableRangesName = String.format("%s.%s", SchemaConstants.SYSTEM_KEYSPACE_NAME, SystemKeyspace.LEGACY_AVAILABLE_RANGES);
+ static final String availableRangesName = String.format("%s.%s", SchemaConstants.SYSTEM_KEYSPACE_NAME, SystemKeyspace.AVAILABLE_RANGES_V2);
+
+
+ private static final Logger logger = LoggerFactory.getLogger(SystemKeyspaceMigrator40.class);
+
+ private SystemKeyspaceMigrator40() {}
+
+ public static void migrate()
+ {
+ migratePeers();
+ migratePeerEvents();
+ migrateTransferredRanges();
+ migrateAvailableRanges();
+ }
+
+ private static void migratePeers()
+ {
+ ColumnFamilyStore newPeers = Keyspace.open(SchemaConstants.SYSTEM_KEYSPACE_NAME).getColumnFamilyStore(SystemKeyspace.PEERS_V2);
+
+ if (!newPeers.isEmpty())
+ return;
+
+ logger.info("{} table was empty, migrating legacy {}, if this fails you should fix the issue and then truncate {} to have it try again.",
+ peersName, legacyPeersName, peersName);
+
+ String query = String.format("SELECT * FROM %s",
+ legacyPeersName);
+
+ String insert = String.format("INSERT INTO %s ( "
+ + "peer, "
+ + "peer_port, "
+ + "data_center, "
+ + "host_id, "
+ + "preferred_ip, "
+ + "preferred_port, "
+ + "rack, "
+ + "release_version, "
+ + "native_address, "
+ + "native_port, "
+ + "schema_version, "
+ + "tokens) "
+ + " values ( ?, ?, ? , ? , ?, ?, ?, ?, ?, ?, ?, ?)",
+ peersName);
+
+ UntypedResultSet rows = QueryProcessor.executeInternalWithPaging(query, 1000);
+ int transferred = 0;
+ logger.info("Migrating rows from legacy {} to {}", legacyPeersName, peersName);
+ for (UntypedResultSet.Row row : rows)
+ {
+ logger.debug("Transferring row {}", transferred);
+ QueryProcessor.executeInternal(insert,
+ row.has("peer") ? row.getInetAddress("peer") : null,
+ DatabaseDescriptor.getStoragePort(),
+ row.has("data_center") ? row.getString("data_center") : null,
+ row.has("host_id") ? row.getUUID("host_id") : null,
+ row.has("preferred_ip") ? row.getInetAddress("preferred_ip") : null,
+ DatabaseDescriptor.getStoragePort(),
+ row.has("rack") ? row.getString("rack") : null,
+ row.has("release_version") ? row.getString("release_version") : null,
+ row.has("rpc_address") ? row.getInetAddress("rpc_address") : null,
+ DatabaseDescriptor.getNativeTransportPort(),
+ row.has("schema_version") ? row.getUUID("schema_version") : null,
+ row.has("tokens") ? row.getSet("tokens", UTF8Type.instance) : null);
+ transferred++;
+ }
+ logger.info("Migrated {} rows from legacy {} to {}", transferred, legacyPeersName, peersName);
+ }
+
+ private static void migratePeerEvents()
+ {
+ ColumnFamilyStore newPeerEvents = Keyspace.open(SchemaConstants.SYSTEM_KEYSPACE_NAME).getColumnFamilyStore(SystemKeyspace.PEER_EVENTS_V2);
+
+ if (!newPeerEvents.isEmpty())
+ return;
+
+ logger.info("{} table was empty, migrating legacy {} to {}", peerEventsName, legacyPeerEventsName, peerEventsName);
+
+ String query = String.format("SELECT * FROM %s",
+ legacyPeerEventsName);
+
+ String insert = String.format("INSERT INTO %s ( "
+ + "peer, "
+ + "peer_port, "
+ + "hints_dropped) "
+ + " values ( ?, ?, ? )",
+ peerEventsName);
+
+ UntypedResultSet rows = QueryProcessor.executeInternalWithPaging(query, 1000);
+ int transferred = 0;
+ for (UntypedResultSet.Row row : rows)
+ {
+ logger.debug("Transferring row {}", transferred);
+ QueryProcessor.executeInternal(insert,
+ row.has("peer") ? row.getInetAddress("peer") : null,
+ DatabaseDescriptor.getStoragePort(),
+ row.has("hints_dropped") ? row.getMap("hints_dropped", UUIDType.instance, Int32Type.instance) : null);
+ transferred++;
+ }
+ logger.info("Migrated {} rows from legacy {} to {}", transferred, legacyPeerEventsName, peerEventsName);
+ }
+
+ static void migrateTransferredRanges()
+ {
+ ColumnFamilyStore newTransferredRanges = Keyspace.open(SchemaConstants.SYSTEM_KEYSPACE_NAME).getColumnFamilyStore(SystemKeyspace.TRANSFERRED_RANGES_V2);
+
+ if (!newTransferredRanges.isEmpty())
+ return;
+
+ logger.info("{} table was empty, migrating legacy {} to {}", transferredRangesName, legacyTransferredRangesName, transferredRangesName);
+
+ String query = String.format("SELECT * FROM %s",
+ legacyTransferredRangesName);
+
+ String insert = String.format("INSERT INTO %s ("
+ + "operation, "
+ + "peer, "
+ + "peer_port, "
+ + "keyspace_name, "
+ + "ranges) "
+ + " values ( ?, ?, ? , ?, ?)",
+ transferredRangesName);
+
+ UntypedResultSet rows = QueryProcessor.executeInternalWithPaging(query, 1000);
+ int transferred = 0;
+ for (UntypedResultSet.Row row : rows)
+ {
+ logger.debug("Transferring row {}", transferred);
+ QueryProcessor.executeInternal(insert,
+ row.has("operation") ? row.getString("operation") : null,
+ row.has("peer") ? row.getInetAddress("peer") : null,
+ DatabaseDescriptor.getStoragePort(),
+ row.has("keyspace_name") ? row.getString("keyspace_name") : null,
+ row.has("ranges") ? row.getSet("ranges", BytesType.instance) : null);
+ transferred++;
+ }
+
+ logger.info("Migrated {} rows from legacy {} to {}", transferred, legacyTransferredRangesName, transferredRangesName);
+ }
+
+ static void migrateAvailableRanges()
+ {
+ ColumnFamilyStore newAvailableRanges = Keyspace.open(SchemaConstants.SYSTEM_KEYSPACE_NAME).getColumnFamilyStore(SystemKeyspace.AVAILABLE_RANGES_V2);
+
+ if (!newAvailableRanges.isEmpty())
+ return;
+
+ logger.info("{} table was empty, migrating legacy {} to {}", availableRangesName, legacyAvailableRangesName, availableRangesName);
+
+ String query = String.format("SELECT * FROM %s",
+ legacyAvailableRangesName);
+
+ String insert = String.format("INSERT INTO %s ("
+ + "keyspace_name, "
+ + "full_ranges, "
+ + "transient_ranges) "
+ + " values ( ?, ?, ? )",
+ availableRangesName);
+
+ UntypedResultSet rows = QueryProcessor.executeInternalWithPaging(query, 1000);
+ int transferred = 0;
+ for (UntypedResultSet.Row row : rows)
+ {
+ logger.debug("Transferring row {}", transferred);
+ String keyspace = row.getString("keyspace_name");
+ Set<ByteBuffer> ranges = Optional.ofNullable(row.getSet("ranges", BytesType.instance)).orElse(Collections.emptySet());
+ QueryProcessor.executeInternal(insert,
+ keyspace,
+ ranges,
+ Collections.emptySet());
+ transferred++;
+ }
+
+ logger.info("Migrated {} rows from legacy {} to {}", transferred, legacyAvailableRangesName, availableRangesName);
+ }
+
+}
diff --git a/src/java/org/apache/cassandra/db/TableWriteHandler.java b/src/java/org/apache/cassandra/db/TableWriteHandler.java
new file mode 100644
index 0000000..4e47221
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/TableWriteHandler.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db;
+
+import org.apache.cassandra.db.partitions.PartitionUpdate;
+import org.apache.cassandra.index.transactions.UpdateTransaction;
+
+public interface TableWriteHandler
+{
+ void write(PartitionUpdate update, WriteContext context, UpdateTransaction updateTransaction);
+}
diff --git a/src/java/org/apache/cassandra/db/TruncateRequest.java b/src/java/org/apache/cassandra/db/TruncateRequest.java
new file mode 100644
index 0000000..64950b1
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/TruncateRequest.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db;
+
+import java.io.IOException;
+
+import org.apache.cassandra.io.IVersionedSerializer;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+
+/**
+ * A truncate operation descriptor
+ */
+public class TruncateRequest
+{
+ public static final IVersionedSerializer<TruncateRequest> serializer = new Serializer();
+
+ public final String keyspace;
+ public final String table;
+
+ public TruncateRequest(String keyspace, String table)
+ {
+ this.keyspace = keyspace;
+ this.table = table;
+ }
+
+ @Override
+ public String toString()
+ {
+ return String.format("TruncateRequest(keyspace='%s', table='%s')'", keyspace, table);
+ }
+
+ private static class Serializer implements IVersionedSerializer<TruncateRequest>
+ {
+ public void serialize(TruncateRequest request, DataOutputPlus out, int version) throws IOException
+ {
+ out.writeUTF(request.keyspace);
+ out.writeUTF(request.table);
+ }
+
+ public TruncateRequest deserialize(DataInputPlus in, int version) throws IOException
+ {
+ String keyspace = in.readUTF();
+ String table = in.readUTF();
+ return new TruncateRequest(keyspace, table);
+ }
+
+ public long serializedSize(TruncateRequest request, int version)
+ {
+ return TypeSizes.sizeof(request.keyspace) + TypeSizes.sizeof(request.table);
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/db/TruncateResponse.java b/src/java/org/apache/cassandra/db/TruncateResponse.java
index af4ed8f..822c9cc 100644
--- a/src/java/org/apache/cassandra/db/TruncateResponse.java
+++ b/src/java/org/apache/cassandra/db/TruncateResponse.java
@@ -22,8 +22,6 @@
import org.apache.cassandra.io.IVersionedSerializer;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
-import org.apache.cassandra.net.MessageOut;
-import org.apache.cassandra.net.MessagingService;
/**
* This message is sent back the truncate operation and basically specifies if
@@ -44,11 +42,6 @@
this.success = success;
}
- public MessageOut<TruncateResponse> createMessage()
- {
- return new MessageOut<TruncateResponse>(MessagingService.Verb.REQUEST_RESPONSE, this, serializer);
- }
-
public static class TruncateResponseSerializer implements IVersionedSerializer<TruncateResponse>
{
public void serialize(TruncateResponse tr, DataOutputPlus out, int version) throws IOException
diff --git a/src/java/org/apache/cassandra/db/TruncateVerbHandler.java b/src/java/org/apache/cassandra/db/TruncateVerbHandler.java
index 226262c..c605d1f 100644
--- a/src/java/org/apache/cassandra/db/TruncateVerbHandler.java
+++ b/src/java/org/apache/cassandra/db/TruncateVerbHandler.java
@@ -22,21 +22,23 @@
import org.apache.cassandra.io.FSError;
import org.apache.cassandra.net.IVerbHandler;
-import org.apache.cassandra.net.MessageIn;
+import org.apache.cassandra.net.Message;
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.tracing.Tracing;
-public class TruncateVerbHandler implements IVerbHandler<Truncation>
+public class TruncateVerbHandler implements IVerbHandler<TruncateRequest>
{
+ public static final TruncateVerbHandler instance = new TruncateVerbHandler();
+
private static final Logger logger = LoggerFactory.getLogger(TruncateVerbHandler.class);
- public void doVerb(MessageIn<Truncation> message, int id)
+ public void doVerb(Message<TruncateRequest> message)
{
- Truncation t = message.payload;
- Tracing.trace("Applying truncation of {}.{}", t.keyspace, t.columnFamily);
+ TruncateRequest t = message.payload;
+ Tracing.trace("Applying truncation of {}.{}", t.keyspace, t.table);
try
{
- ColumnFamilyStore cfs = Keyspace.open(t.keyspace).getColumnFamilyStore(t.columnFamily);
+ ColumnFamilyStore cfs = Keyspace.open(t.keyspace).getColumnFamilyStore(t.table);
cfs.truncateBlocking();
}
catch (Exception e)
@@ -47,16 +49,16 @@
if (FSError.findNested(e) != null)
throw FSError.findNested(e);
}
- Tracing.trace("Enqueuing response to truncate operation to {}", message.from);
+ Tracing.trace("Enqueuing response to truncate operation to {}", message.from());
- TruncateResponse response = new TruncateResponse(t.keyspace, t.columnFamily, true);
- logger.trace("{} applied. Enqueuing response to {}@{} ", new Object[]{ t, id, message.from });
- MessagingService.instance().sendReply(response.createMessage(), id, message.from);
+ TruncateResponse response = new TruncateResponse(t.keyspace, t.table, true);
+ logger.trace("{} applied. Enqueuing response to {}@{} ", t, message.id(), message.from());
+ MessagingService.instance().send(message.responseWith(response), message.from());
}
- private static void respondError(Truncation t, MessageIn truncateRequestMessage)
+ private static void respondError(TruncateRequest t, Message truncateRequestMessage)
{
- TruncateResponse response = new TruncateResponse(t.keyspace, t.columnFamily, false);
- MessagingService.instance().sendOneWay(response.createMessage(), truncateRequestMessage.from);
+ TruncateResponse response = new TruncateResponse(t.keyspace, t.table, false);
+ MessagingService.instance().send(truncateRequestMessage.responseWith(response), truncateRequestMessage.from());
}
}
diff --git a/src/java/org/apache/cassandra/db/Truncation.java b/src/java/org/apache/cassandra/db/Truncation.java
deleted file mode 100644
index 39a2ec6..0000000
--- a/src/java/org/apache/cassandra/db/Truncation.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.db;
-
-import java.io.IOException;
-
-import org.apache.cassandra.io.IVersionedSerializer;
-import org.apache.cassandra.io.util.DataInputPlus;
-import org.apache.cassandra.io.util.DataOutputPlus;
-import org.apache.cassandra.net.MessageOut;
-import org.apache.cassandra.net.MessagingService;
-
-/**
- * A truncate operation descriptor
- */
-public class Truncation
-{
- public static final IVersionedSerializer<Truncation> serializer = new TruncationSerializer();
-
- public final String keyspace;
- public final String columnFamily;
-
- public Truncation(String keyspace, String columnFamily)
- {
- this.keyspace = keyspace;
- this.columnFamily = columnFamily;
- }
-
- public MessageOut<Truncation> createMessage()
- {
- return new MessageOut<Truncation>(MessagingService.Verb.TRUNCATE, this, serializer);
- }
-
- public String toString()
- {
- return "Truncation(" + "keyspace='" + keyspace + '\'' + ", cf='" + columnFamily + "\')";
- }
-}
-
-class TruncationSerializer implements IVersionedSerializer<Truncation>
-{
- public void serialize(Truncation t, DataOutputPlus out, int version) throws IOException
- {
- out.writeUTF(t.keyspace);
- out.writeUTF(t.columnFamily);
- }
-
- public Truncation deserialize(DataInputPlus in, int version) throws IOException
- {
- String keyspace = in.readUTF();
- String columnFamily = in.readUTF();
- return new Truncation(keyspace, columnFamily);
- }
-
- public long serializedSize(Truncation truncation, int version)
- {
- return TypeSizes.sizeof(truncation.keyspace) + TypeSizes.sizeof(truncation.columnFamily);
- }
-}
diff --git a/src/java/org/apache/cassandra/db/UnfilteredDeserializer.java b/src/java/org/apache/cassandra/db/UnfilteredDeserializer.java
index 262b333..f9ff1d7 100644
--- a/src/java/org/apache/cassandra/db/UnfilteredDeserializer.java
+++ b/src/java/org/apache/cassandra/db/UnfilteredDeserializer.java
@@ -18,19 +18,10 @@
package org.apache.cassandra.db;
import java.io.IOException;
-import java.io.IOError;
-import java.util.*;
-import java.util.function.Supplier;
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.PeekingIterator;
-
-import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.db.rows.*;
import org.apache.cassandra.io.util.DataInputPlus;
-import org.apache.cassandra.io.util.FileDataInput;
-import org.apache.cassandra.net.MessagingService;
/**
* Helper class to deserialize Unfiltered object from disk efficiently.
@@ -39,38 +30,73 @@
* we don't do more work than necessary (i.e. we don't allocate/deserialize
* objects for things we don't care about).
*/
-public abstract class UnfilteredDeserializer
+public class UnfilteredDeserializer
{
- protected final CFMetaData metadata;
+ protected final TableMetadata metadata;
protected final DataInputPlus in;
- protected final SerializationHelper helper;
+ protected final DeserializationHelper helper;
- protected UnfilteredDeserializer(CFMetaData metadata,
- DataInputPlus in,
- SerializationHelper helper)
+ private final ClusteringPrefix.Deserializer clusteringDeserializer;
+ private final SerializationHeader header;
+
+ private int nextFlags;
+ private int nextExtendedFlags;
+ private boolean isReady;
+ private boolean isDone;
+
+ private final Row.Builder builder;
+
+ private UnfilteredDeserializer(TableMetadata metadata,
+ DataInputPlus in,
+ SerializationHeader header,
+ DeserializationHelper helper)
{
this.metadata = metadata;
this.in = in;
this.helper = helper;
+ this.header = header;
+ this.clusteringDeserializer = new ClusteringPrefix.Deserializer(metadata.comparator, in, header);
+ this.builder = BTreeRow.sortedBuilder();
}
- public static UnfilteredDeserializer create(CFMetaData metadata,
+ public static UnfilteredDeserializer create(TableMetadata metadata,
DataInputPlus in,
SerializationHeader header,
- SerializationHelper helper,
- DeletionTime partitionDeletion,
- boolean readAllAsDynamic)
+ DeserializationHelper helper)
{
- if (helper.version >= MessagingService.VERSION_30)
- return new CurrentDeserializer(metadata, in, header, helper);
- else
- return new OldFormatDeserializer(metadata, in, helper, partitionDeletion, readAllAsDynamic);
+ return new UnfilteredDeserializer(metadata, in, header, helper);
}
/**
* Whether or not there is more atom to read.
*/
- public abstract boolean hasNext() throws IOException;
+ public boolean hasNext() throws IOException
+ {
+ if (isReady)
+ return true;
+
+ prepareNext();
+ return !isDone;
+ }
+
+ private void prepareNext() throws IOException
+ {
+ if (isDone)
+ return;
+
+ nextFlags = in.readUnsignedByte();
+ if (UnfilteredSerializer.isEndOfPartition(nextFlags))
+ {
+ isDone = true;
+ isReady = false;
+ return;
+ }
+
+ nextExtendedFlags = UnfilteredSerializer.readExtendedFlags(in, nextFlags);
+
+ clusteringDeserializer.prepare(nextFlags, nextExtendedFlags);
+ isReady = true;
+ }
/**
* Compare the provided bound to the next atom to read on disk.
@@ -79,822 +105,68 @@
* comparison. Whenever we know what to do with this atom (read it or skip it),
* readNext or skipNext should be called.
*/
- public abstract int compareNextTo(ClusteringBound bound) throws IOException;
+ public int compareNextTo(ClusteringBound bound) throws IOException
+ {
+ if (!isReady)
+ prepareNext();
+
+ assert !isDone;
+
+ return clusteringDeserializer.compareNextTo(bound);
+ }
/**
* Returns whether the next atom is a row or not.
*/
- public abstract boolean nextIsRow() throws IOException;
+ public boolean nextIsRow() throws IOException
+ {
+ if (!isReady)
+ prepareNext();
- /**
- * Returns whether the next atom is the static row or not.
- */
- public abstract boolean nextIsStatic() throws IOException;
+ return UnfilteredSerializer.kind(nextFlags) == Unfiltered.Kind.ROW;
+ }
/**
* Returns the next atom.
*/
- public abstract Unfiltered readNext() throws IOException;
+ public Unfiltered readNext() throws IOException
+ {
+ isReady = false;
+ if (UnfilteredSerializer.kind(nextFlags) == Unfiltered.Kind.RANGE_TOMBSTONE_MARKER)
+ {
+ ClusteringBoundOrBoundary bound = clusteringDeserializer.deserializeNextBound();
+ return UnfilteredSerializer.serializer.deserializeMarkerBody(in, header, bound);
+ }
+ else
+ {
+ builder.newRow(clusteringDeserializer.deserializeNextClustering());
+ return UnfilteredSerializer.serializer.deserializeRowBody(in, header, helper, nextFlags, nextExtendedFlags, builder);
+ }
+ }
/**
* Clears any state in this deserializer.
*/
- public abstract void clearState() throws IOException;
+ public void clearState()
+ {
+ isReady = false;
+ isDone = false;
+ }
/**
* Skips the next atom.
*/
- public abstract void skipNext() throws IOException;
-
-
- /**
- * For the legacy layout deserializer, we have to deal with the fact that a row can span multiple index blocks and that
- * the call to hasNext() reads the next element upfront. We must take that into account when we check in AbstractSSTableIterator if
- * we're past the end of an index block boundary as that check expect to account for only consumed data (that is, if hasNext has
- * been called and made us cross an index boundary but neither readNext() or skipNext() as yet been called, we shouldn't consider
- * the index block boundary crossed yet).
- *
- * TODO: we don't care about this for the current file format because a row can never span multiple index blocks (further, hasNext()
- * only just basically read 2 bytes from disk in that case). So once we drop backward compatibility with pre-3.0 sstable, we should
- * remove this.
- */
- public abstract long bytesReadForUnconsumedData();
-
- private static class CurrentDeserializer extends UnfilteredDeserializer
+ public void skipNext() throws IOException
{
- private final ClusteringPrefix.Deserializer clusteringDeserializer;
- private final SerializationHeader header;
-
- private int nextFlags;
- private int nextExtendedFlags;
- private boolean isReady;
- private boolean isDone;
-
- private final Row.Builder builder;
-
- private CurrentDeserializer(CFMetaData metadata,
- DataInputPlus in,
- SerializationHeader header,
- SerializationHelper helper)
+ isReady = false;
+ clusteringDeserializer.skipNext();
+ if (UnfilteredSerializer.kind(nextFlags) == Unfiltered.Kind.RANGE_TOMBSTONE_MARKER)
{
- super(metadata, in, helper);
- this.header = header;
- this.clusteringDeserializer = new ClusteringPrefix.Deserializer(metadata.comparator, in, header);
- this.builder = BTreeRow.sortedBuilder();
+ UnfilteredSerializer.serializer.skipMarkerBody(in);
}
-
- public boolean hasNext() throws IOException
+ else
{
- if (isReady)
- return true;
-
- prepareNext();
- return !isDone;
- }
-
- private void prepareNext() throws IOException
- {
- if (isDone)
- return;
-
- nextFlags = in.readUnsignedByte();
- if (UnfilteredSerializer.isEndOfPartition(nextFlags))
- {
- isDone = true;
- isReady = false;
- return;
- }
-
- nextExtendedFlags = UnfilteredSerializer.readExtendedFlags(in, nextFlags);
-
- clusteringDeserializer.prepare(nextFlags, nextExtendedFlags);
- isReady = true;
- }
-
- public int compareNextTo(ClusteringBound bound) throws IOException
- {
- if (!isReady)
- prepareNext();
-
- assert !isDone;
-
- return clusteringDeserializer.compareNextTo(bound);
- }
-
- public boolean nextIsRow() throws IOException
- {
- if (!isReady)
- prepareNext();
-
- return UnfilteredSerializer.kind(nextFlags) == Unfiltered.Kind.ROW;
- }
-
- public boolean nextIsStatic() throws IOException
- {
- // This exists only for the sake of the OldFormatDeserializer
- throw new UnsupportedOperationException();
- }
-
- public Unfiltered readNext() throws IOException
- {
- isReady = false;
- if (UnfilteredSerializer.kind(nextFlags) == Unfiltered.Kind.RANGE_TOMBSTONE_MARKER)
- {
- ClusteringBoundOrBoundary bound = clusteringDeserializer.deserializeNextBound();
- return UnfilteredSerializer.serializer.deserializeMarkerBody(in, header, bound);
- }
- else
- {
- builder.newRow(clusteringDeserializer.deserializeNextClustering());
- return UnfilteredSerializer.serializer.deserializeRowBody(in, header, helper, nextFlags, nextExtendedFlags, builder);
- }
- }
-
- public void skipNext() throws IOException
- {
- isReady = false;
- clusteringDeserializer.skipNext();
- if (UnfilteredSerializer.kind(nextFlags) == Unfiltered.Kind.RANGE_TOMBSTONE_MARKER)
- {
- UnfilteredSerializer.serializer.skipMarkerBody(in);
- }
- else
- {
- UnfilteredSerializer.serializer.skipRowBody(in);
- }
- }
-
- public void clearState()
- {
- isReady = false;
- isDone = false;
- }
-
- public long bytesReadForUnconsumedData()
- {
- // In theory, hasNext() does consume 2-3 bytes, but we don't care about this for the current file format so returning
- // 0 to mean "do nothing".
- return 0;
- }
- }
-
- public static class OldFormatDeserializer extends UnfilteredDeserializer
- {
- private final boolean readAllAsDynamic;
- private boolean skipStatic;
-
- // The next Unfiltered to return, computed by hasNext()
- private Unfiltered next;
-
- // Saved position in the input after the next Unfiltered that will be consumed
- private long nextConsumedPosition;
-
- // A temporary storage for an Unfiltered that isn't returned next but should be looked at just afterwards
- private Stash stash;
-
- private boolean couldBeStartOfPartition = true;
-
- // The Unfiltered as read from the old format input
- private final UnfilteredIterator iterator;
-
- // The position in the input after the last data consumption (readNext/skipNext).
- private long lastConsumedPosition;
-
- // Tracks the size of the last LegacyAtom read from disk, because this needs to be accounted
- // for when marking lastConsumedPosition after readNext/skipNext
- // Reading/skipping an Unfiltered consumes LegacyAtoms from the underlying legacy atom iterator
- // e.g. hasNext() -> iterator.hasNext() -> iterator.readRow() -> atoms.next()
- // The stop condition of the loop which groups legacy atoms into rows causes that AtomIterator
- // to read in the first atom which doesn't belong in the row. So by that point, our position
- // is actually past the end of the next Unfiltered. To compensate, we record the size of
- // the last LegacyAtom read and subtract it from the current position when we calculate lastConsumedPosition.
- // If we don't, then when reading an indexed block, we can over correct and may think that we've
- // exhausted the block before we actually have.
- private long bytesReadForNextAtom = 0L;
-
- private OldFormatDeserializer(CFMetaData metadata,
- DataInputPlus in,
- SerializationHelper helper,
- DeletionTime partitionDeletion,
- boolean readAllAsDynamic)
- {
- super(metadata, in, helper);
- this.iterator = new UnfilteredIterator(metadata, partitionDeletion, helper, this::readAtom);
- this.readAllAsDynamic = readAllAsDynamic;
- this.lastConsumedPosition = currentPosition();
- }
-
- private LegacyLayout.LegacyAtom readAtom()
- {
- while (true)
- {
- try
- {
- long pos = currentPosition();
- LegacyLayout.LegacyAtom atom = LegacyLayout.readLegacyAtom(metadata, in, readAllAsDynamic);
- bytesReadForNextAtom = currentPosition() - pos;
- return atom;
- }
- catch (UnknownColumnException e)
- {
- // This is ok, see LegacyLayout.readLegacyAtom() for why this only happens in case were we're ok
- // skipping the cell. We do want to catch this at this level however because when that happen,
- // we should *not* count the byte of that discarded cell as part of the bytes for the atom
- // we will eventually return, as doing so could throw the logic bytesReadForNextAtom participates in.
- }
- catch (IOException e)
- {
- throw new IOError(e);
- }
- }
- }
-
- public void setSkipStatic()
- {
- this.skipStatic = true;
- }
-
- private boolean isStatic(Unfiltered unfiltered)
- {
- return unfiltered.isRow() && ((Row)unfiltered).isStatic();
- }
-
- public boolean hasNext() throws IOException
- {
- try
- {
- while (next == null)
- {
- if (null != stash)
- {
- next = stash.unfiltered;
- nextConsumedPosition = stash.consumedPosition;
- stash = null;
- }
- else
- {
- if (!iterator.hasNext())
- return false;
- next = iterator.next();
- nextConsumedPosition = currentPosition() - bytesReadForNextAtom;
- }
-
- /*
- * The sstable iterators assume that if there is one, the static row is the first thing this deserializer will return.
- * However, in the old format, a range tombstone with an empty start would sort before any static cell. So we should
- * detect that case and return the static parts first if necessary.
- */
- if (couldBeStartOfPartition && next.isRangeTombstoneMarker() && next.clustering().size() == 0 && iterator.hasNext())
- {
- Unfiltered unfiltered = iterator.next();
- long consumedPosition = currentPosition() - bytesReadForNextAtom;
-
- stash = new Stash(unfiltered, consumedPosition);
-
- /*
- * reorder next and stash (see the comment above that explains why), but retain their positions
- * it's ok to do so since consumedPosition value is only used to determine if we have gone past
- * the end of the index ‘block’; since the edge case requires that the first value be the ‘bottom’
- * RT bound (i.e. with no byte buffers), this has a small and well-defined size, and it must be
- * the case that both unfiltered are in the same index ‘block’ if we began at the beginning of it.
- * if we don't do this, however, we risk aborting early and not returning the BOTTOM rt bound,
- * if the static row is large enough to cross block boundaries.
- */
- if (isStatic(unfiltered))
- {
- stash.unfiltered = next;
- next = unfiltered;
- }
- }
- couldBeStartOfPartition = false;
-
- // When reading old tables, we sometimes want to skip static data (due to how staticly defined column of compact
- // tables are handled).
- if (skipStatic && isStatic(next))
- next = null;
- }
-
- return true;
- }
- catch (IOError e)
- {
- if (e.getCause() != null && e.getCause() instanceof IOException)
- throw (IOException)e.getCause();
- throw e;
- }
- }
-
- public int compareNextTo(ClusteringBound bound) throws IOException
- {
- if (!hasNext())
- throw new IllegalStateException();
- return metadata.comparator.compare(next.clustering(), bound);
- }
-
- public boolean nextIsRow() throws IOException
- {
- if (!hasNext())
- throw new IllegalStateException();
- return next.isRow();
- }
-
- public boolean nextIsStatic() throws IOException
- {
- return nextIsRow() && ((Row)next).isStatic();
- }
-
- private long currentPosition()
- {
- // We return a bogus value if the input is not file based, but check we never rely
- // on that value in that case in bytesReadForUnconsumedData
- return in instanceof FileDataInput ? ((FileDataInput)in).getFilePointer() : 0;
- }
-
- public Unfiltered readNext() throws IOException
- {
- if (!hasNext())
- throw new IllegalStateException();
- Unfiltered toReturn = next;
- next = null;
- lastConsumedPosition = nextConsumedPosition;
- return toReturn;
- }
-
- public void skipNext() throws IOException
- {
- readNext();
- }
-
- // in case we had to reorder an empty RT bound with a static row, this won't be returning the precise unconsumed size,
- // that corresponds to the last returned Unfiltered, but use the natural order in the sstable instead
- public long bytesReadForUnconsumedData()
- {
- if (!(in instanceof FileDataInput))
- throw new AssertionError();
-
- return currentPosition() - lastConsumedPosition;
- }
-
- public void clearState()
- {
- next = null;
- stash = null;
- couldBeStartOfPartition = true;
- iterator.clearState();
- lastConsumedPosition = currentPosition();
- bytesReadForNextAtom = 0L;
- }
-
- private static final class Stash
- {
- private Unfiltered unfiltered;
- long consumedPosition;
-
- private Stash(Unfiltered unfiltered, long consumedPosition)
- {
- this.unfiltered = unfiltered;
- this.consumedPosition = consumedPosition;
- }
- }
-
- // Groups atoms from the input into proper Unfiltered.
- // Note: this could use guava AbstractIterator except that we want to be able to clear
- // the internal state of the iterator so it's cleaner to do it ourselves.
- @VisibleForTesting
- static class UnfilteredIterator implements PeekingIterator<Unfiltered>
- {
- private final AtomIterator atoms;
- private final LegacyLayout.CellGrouper grouper;
- private final TombstoneTracker tombstoneTracker;
- private final CFMetaData metadata;
- private final SerializationHelper helper;
-
- private Unfiltered next;
-
- UnfilteredIterator(CFMetaData metadata,
- DeletionTime partitionDeletion,
- SerializationHelper helper,
- Supplier<LegacyLayout.LegacyAtom> atomReader)
- {
- this.metadata = metadata;
- this.helper = helper;
- this.grouper = new LegacyLayout.CellGrouper(metadata, helper);
- this.tombstoneTracker = new TombstoneTracker(partitionDeletion);
- this.atoms = new AtomIterator(atomReader, metadata);
- }
-
-
- public boolean hasNext()
- {
- // Note that we loop on next == null because TombstoneTracker.openNew() could return null below or the atom might be shadowed.
- while (next == null)
- {
- if (atoms.hasNext())
- {
- // If there is a range tombstone to open strictly before the next row/RT, we need to return that open (or boundary) marker first.
- if (tombstoneTracker.hasOpeningMarkerBefore(atoms.peek()))
- {
- next = tombstoneTracker.popOpeningMarker();
- }
- // If a range tombstone closes strictly before the next row/RT, we need to return that close (or boundary) marker first.
- else if (tombstoneTracker.hasClosingMarkerBefore(atoms.peek()))
- {
- next = tombstoneTracker.popClosingMarker();
- }
- else
- {
- LegacyLayout.LegacyAtom atom = atoms.next();
- if (tombstoneTracker.isShadowed(atom))
- continue;
-
- if (atom.isRowAtom(metadata))
- next = readRow(atom);
- else
- tombstoneTracker.openNew(atom.asRangeTombstone());
- }
- }
- else if (tombstoneTracker.hasOpenTombstones())
- {
- next = tombstoneTracker.popMarker();
- }
- else
- {
- return false;
- }
- }
- return true;
- }
-
- private Unfiltered readRow(LegacyLayout.LegacyAtom first)
- {
- LegacyLayout.CellGrouper grouper = first.isStatic()
- ? LegacyLayout.CellGrouper.staticGrouper(metadata, helper)
- : this.grouper;
- grouper.reset();
- // We know the first atom is not shadowed and is a "row" atom, so can be added blindly.
- grouper.addAtom(first);
-
- // We're less sure about the next atoms. In particular, CellGrouper want to make sure we only pass it
- // "row" atoms (it's the only type it knows how to handle) so we should handle anything else.
- while (atoms.hasNext())
- {
- // Peek, but don't consume the next atom just yet
- LegacyLayout.LegacyAtom atom = atoms.peek();
- // First, that atom may be shadowed in which case we can simply ignore it. Note that this handles
- // the case of repeated RT start marker after we've crossed an index boundary, which could well
- // appear in the middle of a row (CASSANDRA-14008).
- if (!tombstoneTracker.hasClosingMarkerBefore(atom) && tombstoneTracker.isShadowed(atom))
- {
- atoms.next(); // consume the atom since we only peeked it so far
- continue;
- }
-
- // Second, we should only pass "row" atoms to the cell grouper
- if (atom.isRowAtom(metadata))
- {
- if (!grouper.addAtom(atom))
- break; // done with the row; don't consume the atom
- atoms.next(); // the grouper "accepted" the atom, consume it since we only peeked above
- }
- else
- {
- LegacyLayout.LegacyRangeTombstone rt = (LegacyLayout.LegacyRangeTombstone) atom;
- // This means we have a non-row range tombstone. Unfortunately, that does not guarantee the
- // current row is finished (though it may), because due to the logic within LegacyRangeTombstone
- // constructor, we can get an out-of-order RT that includes on the current row (even if it is
- // already started) and extends past it.
-
- // So first, evacuate the easy case of the range tombstone simply starting after the current
- // row, in which case we're done with the current row (but don't consume the new RT yet so it
- // gets handled as any other non-row RT).
- if (grouper.startsAfterCurrentRow(rt))
- break;
-
- // Otherwise, we "split" the RT in 2: the part covering the current row, which is now an
- // inRowAtom and can be passed to the grouper, and the part after that, which we push back into
- // the iterator for later processing.
- Clustering currentRow = grouper.currentRowClustering();
- atoms.next(); // consume since we had only just peeked it so far and we're using it
- atoms.pushOutOfOrder(rt.withNewStart(ClusteringBound.exclusiveStartOf(currentRow)));
- // Note: in theory the withNewStart is a no-op here, but not taking any risk
- grouper.addAtom(rt.withNewStart(ClusteringBound.inclusiveStartOf(currentRow))
- .withNewEnd(ClusteringBound.inclusiveEndOf(currentRow)));
- }
- }
-
- return grouper.getRow();
- }
-
- public Unfiltered next()
- {
- if (!hasNext())
- throw new UnsupportedOperationException();
- Unfiltered toReturn = next;
- next = null;
- return toReturn;
- }
-
- public Unfiltered peek()
- {
- if (!hasNext())
- throw new UnsupportedOperationException();
- return next;
- }
-
- public void clearState()
- {
- atoms.clearState();
- tombstoneTracker.clearState();
- next = null;
- }
-
- public void remove()
- {
- throw new UnsupportedOperationException();
- }
-
- // Wraps the input of the deserializer to provide an iterator (and skip shadowed atoms).
- // Note: this could use guava AbstractIterator except that we want to be able to clear
- // the internal state of the iterator so it's cleaner to do it ourselves.
- private static class AtomIterator implements PeekingIterator<LegacyLayout.LegacyAtom>
- {
- private final Supplier<LegacyLayout.LegacyAtom> atomReader;
- private boolean readerExhausted;
- private LegacyLayout.LegacyAtom next;
-
- private final Comparator<LegacyLayout.LegacyAtom> atomComparator;
- // May temporarily store atoms that needs to be handler later than when they were deserialized.
- // Lazily initialized since it is used infrequently.
- private Queue<LegacyLayout.LegacyAtom> outOfOrderAtoms;
-
- private AtomIterator(Supplier<LegacyLayout.LegacyAtom> atomReader, CFMetaData metadata)
- {
- this.atomReader = atomReader;
- this.atomComparator = LegacyLayout.legacyAtomComparator(metadata);
- }
-
- public boolean hasNext()
- {
- if (readerExhausted)
- return hasOutOfOrderAtoms(); // We have to return out of order atoms when reader exhausts
-
- // Note that next() and peek() assumes that next has been set by this method, so we do it even if
- // we have some outOfOrderAtoms stacked up.
- if (next == null)
- next = atomReader.get();
-
- readerExhausted = next == null;
- return !readerExhausted || hasOutOfOrderAtoms();
- }
-
- public LegacyLayout.LegacyAtom next()
- {
- if (!hasNext())
- throw new UnsupportedOperationException();
-
- if (hasOutOrderAtomBeforeNext())
- return outOfOrderAtoms.poll();
-
- LegacyLayout.LegacyAtom toReturn = next;
- next = null;
- return toReturn;
- }
-
- private boolean hasOutOfOrderAtoms()
- {
- return outOfOrderAtoms != null && !outOfOrderAtoms.isEmpty();
- }
-
- private boolean hasOutOrderAtomBeforeNext()
- {
- // Note that if outOfOrderAtoms is null, the first condition will be false, so we can save a null
- // check on calling `outOfOrderAtoms.peek()` in the right branch.
- return hasOutOfOrderAtoms()
- && (next == null || atomComparator.compare(outOfOrderAtoms.peek(), next) <= 0);
- }
-
- public LegacyLayout.LegacyAtom peek()
- {
- if (!hasNext())
- throw new UnsupportedOperationException();
- if (hasOutOrderAtomBeforeNext())
- return outOfOrderAtoms.peek();
- return next;
- }
-
- /**
- * Push back an atom in the iterator assuming said atom sorts strictly _after_ the atom returned by
- * the last next() call (meaning the pushed atom fall in the part of the iterator that has not been
- * returned yet, not before). The atom will then be returned by the iterator in proper order.
- */
- public void pushOutOfOrder(LegacyLayout.LegacyAtom atom)
- {
- if (outOfOrderAtoms == null)
- outOfOrderAtoms = new PriorityQueue<>(atomComparator);
- outOfOrderAtoms.offer(atom);
- }
-
- public void clearState()
- {
- this.next = null;
- this.readerExhausted = false;
- if (outOfOrderAtoms != null)
- outOfOrderAtoms.clear();
- }
-
- public void remove()
- {
- throw new UnsupportedOperationException();
- }
- }
-
- /**
- * Tracks which range tombstones are open when deserializing the old format.
- * <p>
- * This is a bit tricky because in the old of format we could have duplicated tombstones, overlapping ones,
- * shadowed ones, etc.., but we should generate from that a "flat" output where at most one non-shadoowed
- * range is open at any given time and without empty range.
- * <p>
- * One consequence of that is that we have to be careful to not generate markers too soon. For instance,
- * we might get a range tombstone [1, 1]@3 followed by [1, 10]@5. So if we generate an opening marker on
- * the first tombstone (so INCL_START(1)@3), we're screwed when we get to the 2nd range tombstone: we really
- * should ignore the first tombstone in that that and generate INCL_START(1)@5 (assuming obviously we don't
- * have one more range tombstone starting at 1 in the stream). This is why we have the
- * {@link #hasOpeningMarkerBefore} method: in practice, we remember when a marker should be opened, but only
- * generate that opening marker when we're sure that we won't get anything shadowing that marker.
- * <p>
- * For closing marker, we also have a {@link #hasClosingMarkerBefore} because in the old format the closing
- * markers comes with the opening one, but we should generate them "in order" in the new format.
- */
- private class TombstoneTracker
- {
- private final DeletionTime partitionDeletion;
-
- // As explained in the javadoc, we need to wait to generate an opening marker until we're sure we have
- // seen anything that could shadow it. So this remember a marker that needs to be opened but hasn't
- // been yet. This is truly returned when hasOpeningMarkerBefore tells us it's safe to.
- private RangeTombstoneMarker openMarkerToReturn;
-
- // Open tombstones sorted by their closing bound (i.e. first tombstone is the first to close).
- // As we only track non-fully-shadowed ranges, the first range is necessarily the currently
- // open tombstone (the one with the higher timestamp).
- private final SortedSet<LegacyLayout.LegacyRangeTombstone> openTombstones;
-
- public TombstoneTracker(DeletionTime partitionDeletion)
- {
- this.partitionDeletion = partitionDeletion;
- this.openTombstones = new TreeSet<>((rt1, rt2) -> metadata.comparator.compare(rt1.stop.bound, rt2.stop.bound));
- }
-
- /**
- * Checks if the provided atom is fully shadowed by the open tombstones of this tracker (or the partition deletion).
- */
- public boolean isShadowed(LegacyLayout.LegacyAtom atom)
- {
- assert !hasClosingMarkerBefore(atom);
- long timestamp = atom.isCell() ? atom.asCell().timestamp : atom.asRangeTombstone().deletionTime.markedForDeleteAt();
-
- if (partitionDeletion.deletes(timestamp))
- return true;
-
- SortedSet<LegacyLayout.LegacyRangeTombstone> coveringTombstones = atom.isRowAtom(metadata) ? openTombstones : openTombstones.tailSet(atom.asRangeTombstone());
- return Iterables.any(coveringTombstones, tombstone -> tombstone.deletionTime.deletes(timestamp));
- }
-
- /**
- * Whether there is an outstanding opening marker that should be returned before we process the provided row/RT.
- */
- public boolean hasOpeningMarkerBefore(LegacyLayout.LegacyAtom atom)
- {
- return openMarkerToReturn != null
- && metadata.comparator.compare(openMarkerToReturn.openBound(false), atom.clustering()) < 0;
- }
-
- public Unfiltered popOpeningMarker()
- {
- assert openMarkerToReturn != null;
- Unfiltered toReturn = openMarkerToReturn;
- openMarkerToReturn = null;
- return toReturn;
- }
-
- /**
- * Whether the currently open marker closes stricly before the provided row/RT.
- */
- public boolean hasClosingMarkerBefore(LegacyLayout.LegacyAtom atom)
- {
- return !openTombstones.isEmpty()
- && metadata.comparator.compare(openTombstones.first().stop.bound, atom.clustering()) < 0;
- }
-
- /**
- * Returns the unfiltered corresponding to closing the currently open marker (and update the tracker accordingly).
- */
- public Unfiltered popClosingMarker()
- {
- assert !openTombstones.isEmpty();
-
- Iterator<LegacyLayout.LegacyRangeTombstone> iter = openTombstones.iterator();
- LegacyLayout.LegacyRangeTombstone first = iter.next();
- iter.remove();
-
- // If that was the last open tombstone, we just want to close it. Otherwise, we have a boundary with the
- // next tombstone
- if (!iter.hasNext())
- return new RangeTombstoneBoundMarker(first.stop.bound, first.deletionTime);
-
- LegacyLayout.LegacyRangeTombstone next = iter.next();
- return RangeTombstoneBoundaryMarker.makeBoundary(false, first.stop.bound, first.stop.bound.invert(), first.deletionTime, next.deletionTime);
- }
-
- /**
- * Pop whatever next marker needs to be popped. This should be called as many time as necessary (until
- * {@link #hasOpenTombstones} returns {@false}) when all atoms have been consumed to "empty" the tracker.
- */
- public Unfiltered popMarker()
- {
- assert hasOpenTombstones();
- return openMarkerToReturn == null ? popClosingMarker() : popOpeningMarker();
- }
-
- /**
- * Update the tracker given the provided newly open tombstone. This potentially update openMarkerToReturn
- * to account for th new opening.
- *
- * Note that this method assumes that:
- + 1) the added tombstone is not fully shadowed: !isShadowed(tombstone).
- + 2) there is no marker to open that open strictly before this new tombstone: !hasOpeningMarkerBefore(tombstone).
- + 3) no opened tombstone closes before that tombstone: !hasClosingMarkerBefore(tombstone).
- + One can check that this is only called after the condition above have been checked in UnfilteredIterator.hasNext above.
- */
- public void openNew(LegacyLayout.LegacyRangeTombstone tombstone)
- {
- if (openTombstones.isEmpty())
- {
- // If we have an openMarkerToReturn, the corresponding RT must be in openTombstones (or we wouldn't know when to close it)
- assert openMarkerToReturn == null;
- openTombstones.add(tombstone);
- openMarkerToReturn = new RangeTombstoneBoundMarker(tombstone.start.bound, tombstone.deletionTime);
- return;
- }
-
- if (openMarkerToReturn != null)
- {
- // If the new opening supersedes the one we're about to return, we need to update the one to return.
- if (tombstone.deletionTime.supersedes(openMarkerToReturn.openDeletionTime(false)))
- openMarkerToReturn = openMarkerToReturn.withNewOpeningDeletionTime(false, tombstone.deletionTime);
- }
- else
- {
- // We have no openMarkerToReturn set yet so set it now if needs be.
- // Since openTombstones isn't empty, it means we have a currently ongoing deletion. And if the new tombstone
- // supersedes that ongoing deletion, we need to close the opening deletion and open with the new one.
- DeletionTime currentOpenDeletion = openTombstones.first().deletionTime;
- if (tombstone.deletionTime.supersedes(currentOpenDeletion))
- openMarkerToReturn = RangeTombstoneBoundaryMarker.makeBoundary(false, tombstone.start.bound.invert(), tombstone.start.bound, currentOpenDeletion, tombstone.deletionTime);
- }
-
- // In all cases, we know !isShadowed(tombstone) so we need to add the tombstone (note however that we may not have set openMarkerToReturn if the
- // new tombstone doesn't supersedes the current deletion _but_ extend past the marker currently open)
- add(tombstone);
- }
-
- /**
- * Adds a new tombstone to openTombstones, removing anything that would be shadowed by this new tombstone.
- */
- private void add(LegacyLayout.LegacyRangeTombstone tombstone)
- {
- // First, remove existing tombstone that is shadowed by this tombstone.
- Iterator<LegacyLayout.LegacyRangeTombstone> iter = openTombstones.iterator();
- while (iter.hasNext())
- {
-
- LegacyLayout.LegacyRangeTombstone existing = iter.next();
- // openTombstones is ordered by stop bound and the new tombstone can't be shadowing anything that
- // stop after it.
- if (metadata.comparator.compare(tombstone.stop.bound, existing.stop.bound) < 0)
- break;
-
- // Note that we remove an existing tombstone even if it is equal to the new one because in that case,
- // either the existing strictly stops before the new one and we don't want it, or it stops exactly
- // like the new one but we're going to inconditionally add the new one anyway.
- if (!existing.deletionTime.supersedes(tombstone.deletionTime))
- iter.remove();
- }
- openTombstones.add(tombstone);
- }
-
- public boolean hasOpenTombstones()
- {
- return openMarkerToReturn != null || !openTombstones.isEmpty();
- }
-
- public void clearState()
- {
- openMarkerToReturn = null;
- openTombstones.clear();
- }
- }
+ UnfilteredSerializer.serializer.skipRowBody(in);
}
}
}
diff --git a/src/java/org/apache/cassandra/db/UnfilteredValidation.java b/src/java/org/apache/cassandra/db/UnfilteredValidation.java
new file mode 100644
index 0000000..6d8bbfd
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/UnfilteredValidation.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db;
+
+import java.util.concurrent.TimeUnit;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.config.Config;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.rows.Unfiltered;
+
+import org.apache.cassandra.io.sstable.CorruptSSTableException;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.serializers.MarshalException;
+import org.apache.cassandra.utils.NoSpamLogger;
+
+
+/**
+ * Handles unfiltered validation - if configured, it checks if the provided unfiltered has
+ * invalid deletions (if the local deletion time is negative or if the ttl is negative) and
+ * then either logs or throws an exception if so.
+ */
+public class UnfilteredValidation
+{
+ private static final Logger logger = LoggerFactory.getLogger(UnfilteredValidation.class);
+ private static final NoSpamLogger nospam1m = NoSpamLogger.getLogger(logger, 1, TimeUnit.MINUTES);
+
+ public static void maybeValidateUnfiltered(Unfiltered unfiltered, TableMetadata metadata, DecoratedKey key, SSTableReader sstable)
+ {
+ Config.CorruptedTombstoneStrategy strat = DatabaseDescriptor.getCorruptedTombstoneStrategy();
+ if (strat != Config.CorruptedTombstoneStrategy.disabled && unfiltered != null && !unfiltered.isEmpty())
+ {
+ boolean hasInvalidDeletions = false;
+ try
+ {
+ hasInvalidDeletions = unfiltered.hasInvalidDeletions();
+ }
+ catch (Throwable t) // make sure no unknown exceptions fail the read/compaction
+ {
+ nospam1m.error("Could not check if Unfiltered in {} had any invalid deletions", sstable, t);
+ }
+
+ if (hasInvalidDeletions)
+ {
+ String content;
+ try
+ {
+ content = unfiltered.toString(metadata, true);
+ }
+ catch (Throwable t)
+ {
+ content = "Could not get string representation: " + t.getMessage();
+ }
+ handleInvalid(metadata, key, sstable, content);
+ }
+ }
+ }
+
+ public static void handleInvalid(TableMetadata metadata, DecoratedKey key, SSTableReader sstable, String invalidContent)
+ {
+ Config.CorruptedTombstoneStrategy strat = DatabaseDescriptor.getCorruptedTombstoneStrategy();
+ String keyString;
+ try
+ {
+ keyString = metadata.partitionKeyType.getString(key.getKey());
+ }
+ catch (Throwable t)
+ {
+ keyString = "[corrupt token="+key.getToken()+"]";
+ }
+
+ if (strat == Config.CorruptedTombstoneStrategy.exception)
+ {
+ String msg = String.format("Key %s in %s.%s is invalid in %s: %s",
+ keyString,
+ metadata.keyspace,
+ metadata.name,
+ sstable,
+ invalidContent);
+ // we mark suspect to make sure this sstable is not included in future compactions - it would just keep
+ // throwing exceptions
+ sstable.markSuspect();
+ throw new CorruptSSTableException(new MarshalException(msg), sstable.getFilename());
+ }
+ else if (strat == Config.CorruptedTombstoneStrategy.warn)
+ {
+ String msgTemplate = String.format("Key {} in %s.%s is invalid in %s: {}",
+ metadata.keyspace,
+ metadata.name,
+ sstable);
+ nospam1m.warn(msgTemplate, keyString, invalidContent);
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/db/UnknownColumnException.java b/src/java/org/apache/cassandra/db/UnknownColumnException.java
deleted file mode 100644
index 55dc453..0000000
--- a/src/java/org/apache/cassandra/db/UnknownColumnException.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.db;
-
-import java.nio.ByteBuffer;
-
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.db.marshal.UTF8Type;
-import org.apache.cassandra.utils.ByteBufferUtil;
-
-/**
- * Exception thrown when we read a column internally that is unknown. Note that
- * this is an internal exception and is not meant to be user facing.
- */
-public class UnknownColumnException extends Exception
-{
- public final ByteBuffer columnName;
-
- public UnknownColumnException(CFMetaData metadata, ByteBuffer columnName)
- {
- super(String.format("Unknown column %s in table %s.%s", stringify(columnName), metadata.ksName, metadata.cfName));
- this.columnName = columnName;
- }
-
- private static String stringify(ByteBuffer name)
- {
- try
- {
- return UTF8Type.instance.getString(name);
- }
- catch (Exception e)
- {
- return ByteBufferUtil.bytesToHex(name);
- }
- }
-}
diff --git a/src/java/org/apache/cassandra/db/UnknownColumnFamilyException.java b/src/java/org/apache/cassandra/db/UnknownColumnFamilyException.java
deleted file mode 100644
index c43b50a..0000000
--- a/src/java/org/apache/cassandra/db/UnknownColumnFamilyException.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.db;
-
-import java.io.IOException;
-import java.util.UUID;
-
-
-public class UnknownColumnFamilyException extends IOException
-{
- public final UUID cfId;
-
- public UnknownColumnFamilyException(String msg, UUID cfId)
- {
- super(msg);
- this.cfId = cfId;
- }
-}
diff --git a/src/java/org/apache/cassandra/db/VirtualTablePartitionRangeReadQuery.java b/src/java/org/apache/cassandra/db/VirtualTablePartitionRangeReadQuery.java
new file mode 100644
index 0000000..48cafa1
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/VirtualTablePartitionRangeReadQuery.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db;
+
+import org.apache.cassandra.db.filter.ColumnFilter;
+import org.apache.cassandra.db.filter.DataLimits;
+import org.apache.cassandra.db.filter.RowFilter;
+import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
+import org.apache.cassandra.db.virtual.VirtualKeyspaceRegistry;
+import org.apache.cassandra.db.virtual.VirtualTable;
+import org.apache.cassandra.schema.TableMetadata;
+
+/**
+ * A read query that selects a (part of a) range of partitions of a virtual table.
+ */
+public class VirtualTablePartitionRangeReadQuery extends VirtualTableReadQuery implements PartitionRangeReadQuery
+{
+ private final DataRange dataRange;
+
+ public static VirtualTablePartitionRangeReadQuery create(TableMetadata metadata,
+ int nowInSec,
+ ColumnFilter columnFilter,
+ RowFilter rowFilter,
+ DataLimits limits,
+ DataRange dataRange)
+ {
+ return new VirtualTablePartitionRangeReadQuery(metadata,
+ nowInSec,
+ columnFilter,
+ rowFilter,
+ limits,
+ dataRange);
+ }
+
+ private VirtualTablePartitionRangeReadQuery(TableMetadata metadata,
+ int nowInSec,
+ ColumnFilter columnFilter,
+ RowFilter rowFilter,
+ DataLimits limits,
+ DataRange dataRange)
+ {
+ super(metadata, nowInSec, columnFilter, rowFilter, limits);
+ this.dataRange = dataRange;
+ }
+
+ @Override
+ public DataRange dataRange()
+ {
+ return dataRange;
+ }
+
+ @Override
+ public PartitionRangeReadQuery withUpdatedLimit(DataLimits newLimits)
+ {
+ return new VirtualTablePartitionRangeReadQuery(metadata(),
+ nowInSec(),
+ columnFilter(),
+ rowFilter(),
+ newLimits,
+ dataRange());
+ }
+
+ @Override
+ public PartitionRangeReadQuery withUpdatedLimitsAndDataRange(DataLimits newLimits, DataRange newDataRange)
+ {
+ return new VirtualTablePartitionRangeReadQuery(metadata(),
+ nowInSec(),
+ columnFilter(),
+ rowFilter(),
+ newLimits,
+ newDataRange);
+ }
+
+ @Override
+ protected UnfilteredPartitionIterator queryVirtualTable()
+ {
+ VirtualTable view = VirtualKeyspaceRegistry.instance.getTableNullable(metadata().id);
+ return view.select(dataRange, columnFilter());
+ }
+
+ @Override
+ protected void appendCQLWhereClause(StringBuilder sb)
+ {
+ if (dataRange.isUnrestricted() && rowFilter().isEmpty())
+ return;
+
+ sb.append(" WHERE ");
+ // We put the row filter first because the data range can end by "ORDER BY"
+ if (!rowFilter().isEmpty())
+ {
+ sb.append(rowFilter());
+ if (!dataRange.isUnrestricted())
+ sb.append(" AND ");
+ }
+ if (!dataRange.isUnrestricted())
+ sb.append(dataRange.toCQLString(metadata()));
+ }
+}
diff --git a/src/java/org/apache/cassandra/db/VirtualTableReadQuery.java b/src/java/org/apache/cassandra/db/VirtualTableReadQuery.java
new file mode 100644
index 0000000..ad22a58
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/VirtualTableReadQuery.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db;
+
+import org.apache.cassandra.db.filter.ColumnFilter;
+import org.apache.cassandra.db.filter.DataLimits;
+import org.apache.cassandra.db.filter.RowFilter;
+import org.apache.cassandra.db.partitions.PartitionIterator;
+import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
+import org.apache.cassandra.exceptions.RequestExecutionException;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.service.ClientState;
+
+/**
+ * Base class for the {@code ReadQuery} implementations use to query virtual tables.
+ */
+public abstract class VirtualTableReadQuery extends AbstractReadQuery
+{
+ protected VirtualTableReadQuery(TableMetadata metadata,
+ int nowInSec,
+ ColumnFilter columnFilter,
+ RowFilter rowFilter,
+ DataLimits limits)
+ {
+ super(metadata, nowInSec, columnFilter, rowFilter, limits);
+ }
+
+ @Override
+ public ReadExecutionController executionController()
+ {
+ return ReadExecutionController.empty();
+ }
+
+ @Override
+ public PartitionIterator execute(ConsistencyLevel consistency,
+ ClientState clientState,
+ long queryStartNanoTime) throws RequestExecutionException
+ {
+ return executeInternal(executionController());
+ }
+
+ @Override
+ @SuppressWarnings("resource")
+ public UnfilteredPartitionIterator executeLocally(ReadExecutionController executionController)
+ {
+ UnfilteredPartitionIterator resultIterator = queryVirtualTable();
+ return limits().filter(rowFilter().filter(resultIterator, nowInSec()), nowInSec(), selectsFullPartition());
+ }
+
+ protected abstract UnfilteredPartitionIterator queryVirtualTable();
+}
diff --git a/src/java/org/apache/cassandra/db/VirtualTableSinglePartitionReadQuery.java b/src/java/org/apache/cassandra/db/VirtualTableSinglePartitionReadQuery.java
new file mode 100644
index 0000000..11f1f77
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/VirtualTableSinglePartitionReadQuery.java
@@ -0,0 +1,194 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import org.apache.cassandra.db.filter.ClusteringIndexFilter;
+import org.apache.cassandra.db.filter.ColumnFilter;
+import org.apache.cassandra.db.filter.DataLimits;
+import org.apache.cassandra.db.filter.RowFilter;
+import org.apache.cassandra.db.partitions.PartitionIterator;
+import org.apache.cassandra.db.partitions.PartitionIterators;
+import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
+import org.apache.cassandra.db.virtual.VirtualKeyspaceRegistry;
+import org.apache.cassandra.db.virtual.VirtualTable;
+import org.apache.cassandra.exceptions.RequestExecutionException;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.service.ClientState;
+
+/**
+ * A read query that selects a (part of a) single partition of a virtual table.
+ */
+public class VirtualTableSinglePartitionReadQuery extends VirtualTableReadQuery implements SinglePartitionReadQuery
+{
+ private final DecoratedKey partitionKey;
+ private final ClusteringIndexFilter clusteringIndexFilter;
+
+ public static VirtualTableSinglePartitionReadQuery create(TableMetadata metadata,
+ int nowInSec,
+ ColumnFilter columnFilter,
+ RowFilter rowFilter,
+ DataLimits limits,
+ DecoratedKey partitionKey,
+ ClusteringIndexFilter clusteringIndexFilter)
+ {
+ return new VirtualTableSinglePartitionReadQuery(metadata,
+ nowInSec,
+ columnFilter,
+ rowFilter,
+ limits,
+ partitionKey,
+ clusteringIndexFilter);
+ }
+
+ private VirtualTableSinglePartitionReadQuery(TableMetadata metadata,
+ int nowInSec,
+ ColumnFilter columnFilter,
+ RowFilter rowFilter,
+ DataLimits limits,
+ DecoratedKey partitionKey,
+ ClusteringIndexFilter clusteringIndexFilter)
+ {
+ super(metadata, nowInSec, columnFilter, rowFilter, limits);
+ this.partitionKey = partitionKey;
+ this.clusteringIndexFilter = clusteringIndexFilter;
+ }
+
+ @Override
+ protected void appendCQLWhereClause(StringBuilder sb)
+ {
+ sb.append(" WHERE ");
+
+ sb.append(ColumnMetadata.toCQLString(metadata().partitionKeyColumns())).append(" = ");
+ DataRange.appendKeyString(sb, metadata().partitionKeyType, partitionKey().getKey());
+
+ // We put the row filter first because the clustering index filter can end by "ORDER BY"
+ if (!rowFilter().isEmpty())
+ sb.append(" AND ").append(rowFilter());
+
+ String filterString = clusteringIndexFilter().toCQLString(metadata());
+ if (!filterString.isEmpty())
+ sb.append(" AND ").append(filterString);
+ }
+
+ @Override
+ public ClusteringIndexFilter clusteringIndexFilter()
+ {
+ return clusteringIndexFilter;
+ }
+
+ @Override
+ public boolean selectsFullPartition()
+ {
+ return clusteringIndexFilter.selectsAllPartition() && !rowFilter().hasExpressionOnClusteringOrRegularColumns();
+ }
+
+ @Override
+ public DecoratedKey partitionKey()
+ {
+ return partitionKey;
+ }
+
+ @Override
+ public SinglePartitionReadQuery withUpdatedLimit(DataLimits newLimits)
+ {
+ return new VirtualTableSinglePartitionReadQuery(metadata(),
+ nowInSec(),
+ columnFilter(),
+ rowFilter(),
+ newLimits,
+ partitionKey(),
+ clusteringIndexFilter);
+ }
+
+ @Override
+ public SinglePartitionReadQuery forPaging(Clustering lastReturned, DataLimits limits)
+ {
+ return new VirtualTableSinglePartitionReadQuery(metadata(),
+ nowInSec(),
+ columnFilter(),
+ rowFilter(),
+ limits,
+ partitionKey(),
+ lastReturned == null ? clusteringIndexFilter
+ : clusteringIndexFilter.forPaging(metadata().comparator,
+ lastReturned,
+ false));
+ }
+
+ @Override
+ protected UnfilteredPartitionIterator queryVirtualTable()
+ {
+ VirtualTable view = VirtualKeyspaceRegistry.instance.getTableNullable(metadata().id);
+ return view.select(partitionKey, clusteringIndexFilter, columnFilter());
+ }
+
+ /**
+ * Groups multiple single partition read queries.
+ */
+ public static class Group extends SinglePartitionReadQuery.Group<VirtualTableSinglePartitionReadQuery>
+ {
+ public static Group create(TableMetadata metadata,
+ int nowInSec,
+ ColumnFilter columnFilter,
+ RowFilter rowFilter,
+ DataLimits limits,
+ List<DecoratedKey> partitionKeys,
+ ClusteringIndexFilter clusteringIndexFilter)
+ {
+ List<VirtualTableSinglePartitionReadQuery> queries = new ArrayList<>(partitionKeys.size());
+ for (DecoratedKey partitionKey : partitionKeys)
+ {
+ queries.add(VirtualTableSinglePartitionReadQuery.create(metadata,
+ nowInSec,
+ columnFilter,
+ rowFilter,
+ limits,
+ partitionKey,
+ clusteringIndexFilter));
+ }
+
+ return new Group(queries, limits);
+ }
+
+ public Group(List<VirtualTableSinglePartitionReadQuery> queries, DataLimits limits)
+ {
+ super(queries, limits);
+ }
+
+ public static Group one(VirtualTableSinglePartitionReadQuery query)
+ {
+ return new Group(Collections.singletonList(query), query.limits());
+ }
+
+ public PartitionIterator execute(ConsistencyLevel consistency, ClientState clientState, long queryStartNanoTime) throws RequestExecutionException
+ {
+ if (queries.size() == 1)
+ return queries.get(0).execute(consistency, clientState, queryStartNanoTime);
+
+ return PartitionIterators.concat(queries.stream()
+ .map(q -> q.execute(consistency, clientState, queryStartNanoTime))
+ .collect(Collectors.toList()));
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/db/WriteContext.java b/src/java/org/apache/cassandra/db/WriteContext.java
new file mode 100644
index 0000000..102ab50
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/WriteContext.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db;
+
+/**
+ * Issued by the keyspace write handler and used in the write path (as expected), as well as the read path
+ * and some async index building code. In the read and index paths, the write context is intended to be used
+ * as a marker for ordering operations. Reads can also end up performing writes in some cases, particularly
+ * when correcting secondary indexes.
+ */
+public interface WriteContext extends AutoCloseable
+{
+ @Override
+ void close();
+}
diff --git a/src/java/org/apache/cassandra/db/WriteResponse.java b/src/java/org/apache/cassandra/db/WriteResponse.java
deleted file mode 100644
index 0dddaab..0000000
--- a/src/java/org/apache/cassandra/db/WriteResponse.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.db;
-
-import java.io.IOException;
-
-import org.apache.cassandra.io.IVersionedSerializer;
-import org.apache.cassandra.io.util.DataInputPlus;
-import org.apache.cassandra.io.util.DataOutputPlus;
-import org.apache.cassandra.net.MessageOut;
-import org.apache.cassandra.net.MessagingService;
-
-/*
- * This empty response is sent by a replica to inform the coordinator that the write succeeded
- */
-public final class WriteResponse
-{
- public static final Serializer serializer = new Serializer();
-
- private static final WriteResponse instance = new WriteResponse();
-
- private WriteResponse()
- {
- }
-
- public static MessageOut<WriteResponse> createMessage()
- {
- return new MessageOut<>(MessagingService.Verb.REQUEST_RESPONSE, instance, serializer);
- }
-
- public static class Serializer implements IVersionedSerializer<WriteResponse>
- {
- public void serialize(WriteResponse wm, DataOutputPlus out, int version) throws IOException
- {
- }
-
- public WriteResponse deserialize(DataInputPlus in, int version) throws IOException
- {
- return instance;
- }
-
- public long serializedSize(WriteResponse response, int version)
- {
- return 0;
- }
- }
-}
diff --git a/src/java/org/apache/cassandra/db/aggregation/GroupMaker.java b/src/java/org/apache/cassandra/db/aggregation/GroupMaker.java
index f6ad433..ff0ecaf 100644
--- a/src/java/org/apache/cassandra/db/aggregation/GroupMaker.java
+++ b/src/java/org/apache/cassandra/db/aggregation/GroupMaker.java
@@ -18,12 +18,10 @@
package org.apache.cassandra.db.aggregation;
import java.nio.ByteBuffer;
-import java.util.Arrays;
import org.apache.cassandra.db.Clustering;
import org.apache.cassandra.db.ClusteringComparator;
import org.apache.cassandra.db.DecoratedKey;
-import org.apache.cassandra.utils.ByteBufferUtil;
/**
* A <code>GroupMaker</code> can be used to determine if some sorted rows belongs to the same group or not.
diff --git a/src/java/org/apache/cassandra/db/columniterator/AbstractSSTableIterator.java b/src/java/org/apache/cassandra/db/columniterator/AbstractSSTableIterator.java
index 4eaf8f6..c631f1c 100644
--- a/src/java/org/apache/cassandra/db/columniterator/AbstractSSTableIterator.java
+++ b/src/java/org/apache/cassandra/db/columniterator/AbstractSSTableIterator.java
@@ -22,14 +22,13 @@
import java.util.Iterator;
import java.util.NoSuchElementException;
-import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.filter.ColumnFilter;
import org.apache.cassandra.db.rows.*;
import org.apache.cassandra.io.sstable.IndexInfo;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.sstable.CorruptSSTableException;
-import org.apache.cassandra.io.sstable.format.Version;
import org.apache.cassandra.io.util.FileDataInput;
import org.apache.cassandra.io.util.DataPosition;
import org.apache.cassandra.io.util.FileHandle;
@@ -38,16 +37,17 @@
public abstract class AbstractSSTableIterator implements UnfilteredRowIterator
{
protected final SSTableReader sstable;
+ // We could use sstable.metadata(), but that can change during execution so it's good hygiene to grab an immutable instance
+ protected final TableMetadata metadata;
+
protected final DecoratedKey key;
protected final DeletionTime partitionLevelDeletion;
protected final ColumnFilter columns;
- protected final SerializationHelper helper;
+ protected final DeserializationHelper helper;
protected final Row staticRow;
protected final Reader reader;
- private final boolean isForThrift;
-
protected final FileHandle ifile;
private boolean isClosed;
@@ -62,16 +62,15 @@
RowIndexEntry indexEntry,
Slices slices,
ColumnFilter columnFilter,
- boolean isForThrift,
FileHandle ifile)
{
this.sstable = sstable;
+ this.metadata = sstable.metadata();
this.ifile = ifile;
this.key = key;
this.columns = columnFilter;
this.slices = slices;
- this.helper = new SerializationHelper(sstable.metadata, sstable.descriptor.version.correspondingMessagingVersion(), SerializationHelper.Flag.LOCAL, columnFilter);
- this.isForThrift = isForThrift;
+ this.helper = new DeserializationHelper(metadata, sstable.descriptor.version.correspondingMessagingVersion(), DeserializationHelper.Flag.LOCAL, columnFilter);
if (indexEntry == null)
{
@@ -90,12 +89,6 @@
// - we're querying static columns.
boolean needSeekAtPartitionStart = !indexEntry.isIndexed() || !columns.fetchedColumns().statics.isEmpty();
- // For CQL queries on static compact tables, we only want to consider static value (only those are exposed),
- // but readStaticRow have already read them and might in fact have consumed the whole partition (when reading
- // the legacy file format), so set the reader to null so we don't try to read anything more. We can remove this
- // once we drop support for the legacy file format
- boolean needsReader = sstable.descriptor.version.storeRows() || isForThrift || !sstable.metadata.isStaticCompactTable();
-
if (needSeekAtPartitionStart)
{
// Not indexed (or is reading static), set to the beginning of the partition and read partition level deletion there
@@ -109,15 +102,17 @@
// Note that this needs to be called after file != null and after the partitionDeletion has been set, but before readStaticRow
// (since it uses it) so we can't move that up (but we'll be able to simplify as soon as we drop support for the old file format).
- this.reader = needsReader ? createReader(indexEntry, file, shouldCloseFile) : null;
- this.staticRow = readStaticRow(sstable, file, helper, columns.fetchedColumns().statics, isForThrift, reader == null ? null : reader.deserializer);
+ this.reader = createReader(indexEntry, file, shouldCloseFile);
+ this.staticRow = readStaticRow(sstable, file, helper, columns.fetchedColumns().statics);
}
else
{
this.partitionLevelDeletion = indexEntry.deletionTime();
this.staticRow = Rows.EMPTY_STATIC_ROW;
- this.reader = needsReader ? createReader(indexEntry, file, shouldCloseFile) : null;
+ this.reader = createReader(indexEntry, file, shouldCloseFile);
}
+ if (!partitionLevelDeletion.validate())
+ UnfilteredValidation.handleInvalid(metadata(), key, sstable, "partitionLevelDeletion="+partitionLevelDeletion.toString());
if (reader != null && !slices.isEmpty())
reader.setForSlice(nextSlice());
@@ -164,38 +159,9 @@
private static Row readStaticRow(SSTableReader sstable,
FileDataInput file,
- SerializationHelper helper,
- Columns statics,
- boolean isForThrift,
- UnfilteredDeserializer deserializer) throws IOException
+ DeserializationHelper helper,
+ Columns statics) throws IOException
{
- if (!sstable.descriptor.version.storeRows())
- {
- if (!sstable.metadata.isCompactTable())
- {
- assert deserializer != null;
- return deserializer.hasNext() && deserializer.nextIsStatic()
- ? (Row)deserializer.readNext()
- : Rows.EMPTY_STATIC_ROW;
- }
-
- // For compact tables, we use statics for the "column_metadata" definition. However, in the old format, those
- // "column_metadata" are intermingled as any other "cell". In theory, this means that we'd have to do a first
- // pass to extract the static values. However, for thrift, we'll use the ThriftResultsMerger right away which
- // will re-merge static values with dynamic ones, so we can just ignore static and read every cell as a
- // "dynamic" one. For CQL, if the table is a "static compact", then is has only static columns exposed and no
- // dynamic ones. So we do a pass to extract static columns here, but will have no more work to do. Otherwise,
- // the table won't have static columns.
- if (statics.isEmpty() || isForThrift)
- return Rows.EMPTY_STATIC_ROW;
-
- assert sstable.metadata.isStaticCompactTable();
-
- // As said above, if it's a CQL query and the table is a "static compact", the only exposed columns are the
- // static ones. So we don't have to mark the position to seek back later.
- return LegacyLayout.extractStaticColumns(sstable.metadata, file, statics);
- }
-
if (!sstable.header.hasStatic())
return Rows.EMPTY_STATIC_ROW;
@@ -218,12 +184,12 @@
: createReaderInternal(indexEntry, file, shouldCloseFile);
};
- public CFMetaData metadata()
+ public TableMetadata metadata()
{
- return sstable.metadata;
+ return metadata;
}
- public PartitionColumns columns()
+ public RegularAndStaticColumns columns()
{
return columns.fetchedColumns();
}
@@ -328,7 +294,6 @@
{
private final boolean shouldCloseFile;
public FileDataInput file;
- public final Version version;
protected UnfilteredDeserializer deserializer;
@@ -339,7 +304,6 @@
{
this.file = file;
this.shouldCloseFile = shouldCloseFile;
- this.version = sstable.descriptor.version;
if (file != null)
createDeserializer();
@@ -348,7 +312,7 @@
private void createDeserializer()
{
assert file != null && deserializer == null;
- deserializer = UnfilteredDeserializer.create(sstable.metadata, file, sstable.header, helper, partitionLevelDeletion, isForThrift);
+ deserializer = UnfilteredDeserializer.create(metadata, file, sstable.header, helper);
}
protected void seekToPosition(long position) throws IOException
@@ -492,19 +456,6 @@
currentIndexIdx = blockIdx;
reader.openMarker = blockIdx > 0 ? index(blockIdx - 1).endOpenMarker : null;
-
- // If we're reading an old format file and we move to the first block in the index (i.e. the
- // head of the partition), we skip the static row as it's already been read when we first opened
- // the iterator. If we don't do this and a static row is present, we'll re-read it but treat it
- // as a regular row, causing deserialization to blow up later as that row's flags will be invalid
- // see CASSANDRA-12088 & CASSANDRA-13236
- if (!reader.version.storeRows()
- && blockIdx == 0
- && reader.deserializer.hasNext()
- && reader.deserializer.nextIsStatic())
- {
- reader.deserializer.skipNext();
- }
}
private long columnOffset(int i) throws IOException
@@ -559,8 +510,7 @@
public boolean isPastCurrentBlock() throws IOException
{
assert reader.deserializer != null;
- long correction = reader.deserializer.bytesReadForUnconsumedData();
- return reader.file.bytesPastMark(mark) - correction >= currentIndex().width;
+ return reader.file.bytesPastMark(mark) >= currentIndex().width;
}
public int currentBlockIdx()
diff --git a/src/java/org/apache/cassandra/db/columniterator/SSTableIterator.java b/src/java/org/apache/cassandra/db/columniterator/SSTableIterator.java
index e33c748..9346345 100644
--- a/src/java/org/apache/cassandra/db/columniterator/SSTableIterator.java
+++ b/src/java/org/apache/cassandra/db/columniterator/SSTableIterator.java
@@ -43,10 +43,9 @@
RowIndexEntry indexEntry,
Slices slices,
ColumnFilter columns,
- boolean isForThrift,
FileHandle ifile)
{
- super(sstable, file, key, indexEntry, slices, columns, isForThrift, ifile);
+ super(sstable, file, key, indexEntry, slices, columns, ifile);
}
protected Reader createReaderInternal(RowIndexEntry indexEntry, FileDataInput file, boolean shouldCloseFile)
@@ -151,6 +150,7 @@
return null;
Unfiltered next = deserializer.readNext();
+ UnfilteredValidation.maybeValidateUnfiltered(next, metadata(), key, sstable);
// We may get empty row for the same reason expressed on UnfilteredSerializer.deserializeOne.
if (next.isEmpty())
continue;
@@ -214,7 +214,7 @@
private ForwardIndexedReader(RowIndexEntry indexEntry, FileDataInput file, boolean shouldCloseFile)
{
super(file, shouldCloseFile);
- this.indexState = new IndexState(this, sstable.metadata.comparator, indexEntry, false, ifile);
+ this.indexState = new IndexState(this, metadata.comparator, indexEntry, false, ifile);
this.lastBlockIdx = indexState.blocksCount(); // if we never call setForSlice, that's where we want to stop
}
@@ -271,12 +271,10 @@
// so if currentIdx == lastBlockIdx and slice.end < indexes[currentIdx].firstName, we're guaranteed that the
// whole slice is between the previous block end and this block start, and thus has no corresponding
// data. One exception is if the previous block ends with an openMarker as it will cover our slice
- // and we need to return it (we also don't skip the slice for the old format because we didn't have the openMarker
- // info in that case and can't rely on this optimization).
+ // and we need to return it.
if (indexState.currentBlockIdx() == lastBlockIdx
&& metadata().comparator.compare(slice.end(), indexState.currentIndex().firstName) < 0
- && openMarker == null
- && sstable.descriptor.version.storeRows())
+ && openMarker == null)
{
sliceDone = true;
}
@@ -302,6 +300,7 @@
Unfiltered next = deserializer.readNext();
+ UnfilteredValidation.maybeValidateUnfiltered(next, metadata(), key, sstable);
// We may get empty row for the same reason expressed on UnfilteredSerializer.deserializeOne.
if (next.isEmpty())
continue;
diff --git a/src/java/org/apache/cassandra/db/columniterator/SSTableReversedIterator.java b/src/java/org/apache/cassandra/db/columniterator/SSTableReversedIterator.java
index 23835ee..1e1030c 100644
--- a/src/java/org/apache/cassandra/db/columniterator/SSTableReversedIterator.java
+++ b/src/java/org/apache/cassandra/db/columniterator/SSTableReversedIterator.java
@@ -20,9 +20,6 @@
import java.io.IOException;
import java.util.*;
-import com.google.common.base.Verify;
-
-import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.filter.ColumnFilter;
import org.apache.cassandra.db.partitions.ImmutableBTreePartition;
@@ -30,6 +27,7 @@
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.util.FileDataInput;
import org.apache.cassandra.io.util.FileHandle;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.utils.AbstractIterator;
import org.apache.cassandra.utils.btree.BTree;
@@ -49,10 +47,9 @@
RowIndexEntry indexEntry,
Slices slices,
ColumnFilter columns,
- boolean isForThrift,
FileHandle ifile)
{
- super(sstable, file, key, indexEntry, slices, columns, isForThrift, ifile);
+ super(sstable, file, key, indexEntry, slices, columns, ifile);
}
protected Reader createReaderInternal(RowIndexEntry indexEntry, FileDataInput file, boolean shouldCloseFile)
@@ -89,8 +86,6 @@
protected boolean skipFirstIteratedItem;
protected boolean skipLastIteratedItem;
- protected Unfiltered mostRecentlyEmitted = null;
-
private ReverseReader(FileDataInput file, boolean shouldCloseFile)
{
super(file, shouldCloseFile);
@@ -99,7 +94,7 @@
protected ReusablePartitionData createBuffer(int blocksCount)
{
int estimatedRowCount = 16;
- int columnCount = metadata().partitionColumns().regulars.size();
+ int columnCount = metadata().regularColumns().size();
if (columnCount == 0 || metadata().clusteringColumns().isEmpty())
{
estimatedRowCount = 1;
@@ -113,7 +108,7 @@
// FIXME: so far we only keep stats on cells, so to get a rough estimate on the number of rows,
// we divide by the number of regular columns the table has. We should fix once we collect the
// stats on rows
- int estimatedRowsPerPartition = (int)(sstable.getEstimatedColumnCount().percentile(0.75) / columnCount);
+ int estimatedRowsPerPartition = (int)(sstable.getEstimatedCellPerPartitionCount().percentile(0.75) / columnCount);
estimatedRowCount = Math.max(estimatedRowsPerPartition / blocksCount, 1);
}
catch (IllegalStateException e)
@@ -134,7 +129,7 @@
// Note that we can reuse that buffer between slices (we could alternatively re-read from disk
// every time, but that feels more wasteful) so we want to include everything from the beginning.
// We can stop at the slice end however since any following slice will be before that.
- loadFromDisk(null, slice.end(), false, false, null, null);
+ loadFromDisk(null, slice.end(), false, false);
}
setIterator(slice);
}
@@ -167,9 +162,7 @@
{
if (!hasNext())
throw new NoSuchElementException();
- Unfiltered next = iterator.next();
- mostRecentlyEmitted = next;
- return next;
+ return iterator.next();
}
protected boolean stopReadingDisk() throws IOException
@@ -177,20 +170,12 @@
return false;
}
- // checks if left prefix precedes right prefix
- private boolean precedes(ClusteringPrefix left, ClusteringPrefix right)
- {
- return metadata().comparator.compare(left, right) < 0;
- }
-
// Reads the unfiltered from disk and load them into the reader buffer. It stops reading when either the partition
// is fully read, or when stopReadingDisk() returns true.
protected void loadFromDisk(ClusteringBound start,
ClusteringBound end,
boolean hasPreviousBlock,
- boolean hasNextBlock,
- ClusteringPrefix currentFirstName,
- ClusteringPrefix nextLastName) throws IOException
+ boolean hasNextBlock) throws IOException
{
// start != null means it's the block covering the beginning of the slice, so it has to be the last block for this slice.
assert start == null || !hasNextBlock;
@@ -199,14 +184,11 @@
skipFirstIteratedItem = false;
skipLastIteratedItem = false;
- boolean isFirst = true;
-
// If the start might be in this block, skip everything that comes before it.
if (start != null)
{
while (deserializer.hasNext() && deserializer.compareNextTo(start) <= 0 && !stopReadingDisk())
{
- isFirst = false;
if (deserializer.nextIsRow())
deserializer.skipNext();
else
@@ -241,82 +223,15 @@
&& !stopReadingDisk())
{
Unfiltered unfiltered = deserializer.readNext();
-
- if (isFirst && openMarker == null
- && currentFirstName != null && nextLastName != null
- && (precedes(currentFirstName, nextLastName) || precedes(unfiltered.clustering(), currentFirstName)))
- {
- // Range tombstones spanning multiple index blocks when reading legacy sstables need special handling.
- // Pre-3.0, the column index didn't encode open markers. Instead, open range tombstones were rewritten
- // at the start of index blocks they at least partially covered. These rewritten RTs found at the
- // beginning of index blocks need to be handled as though they were an open marker, otherwise iterator
- // validation will fail and/or some rows will be excluded from the result. These rewritten RTs can be
- // detected based on their relation to the current index block and the next one depending on what wrote
- // the sstable. For sstables coming from a memtable flush, a rewritten RT will have a clustering value
- // less than the first name of its index block. For sstables coming from compaction, the index block
- // first name will be the RT open bound, which will be less than the last name of the next block. So,
- // here we compare the first name of this block to the last name of the next block to detect the
- // compaction case, and clustering value of the unfiltered we just read to the index block's first name
- // to detect the flush case.
- Verify.verify(!sstable.descriptor.version.storeRows());
- Verify.verify(openMarker == null);
- Verify.verify(!skipLastIteratedItem);
- Verify.verify(unfiltered.isRangeTombstoneMarker());
+ UnfilteredValidation.maybeValidateUnfiltered(unfiltered, metadata(), key, sstable);
+ // We may get empty row for the same reason expressed on UnfilteredSerializer.deserializeOne.
+ if (!unfiltered.isEmpty())
buffer.add(unfiltered);
- if (hasNextBlock)
- skipLastIteratedItem = true;
- }
- else if (isFirst && nextLastName != null && !precedes(nextLastName, unfiltered.clustering()))
- {
- // When dealing with old format sstable, we have the problem that a row can span 2 index block, i.e. it can
- // start at the end of a block and end at the beginning of the next one. That's not a problem per se for
- // UnfilteredDeserializer.OldFormatSerializer, since it always read rows entirely, even if they span index
- // blocks, but as we reading index block in reverse we must be careful to not read the end of the row at
- // beginning of a block before we're reading the beginning of that row. So what we do is that if we detect
- // that the row starting this block is also the row ending the next one we're read (previous on disk), then
- // we'll skip that first result and let it be read with the next block.
- Verify.verify(!sstable.descriptor.version.storeRows());
- isFirst = false;
- }
- else if (unfiltered.isEmpty())
- {
- isFirst = false;
- }
- else
- {
- buffer.add(unfiltered);
- isFirst = false;
- }
if (unfiltered.isRangeTombstoneMarker())
updateOpenMarker((RangeTombstoneMarker)unfiltered);
}
- if (!sstable.descriptor.version.storeRows()
- && deserializer.hasNext()
- && (end == null || deserializer.compareNextTo(end) < 0))
- {
- // Range tombstone start and end bounds are stored together in legacy sstables. When we read one, we
- // stash the closing bound until we reach the appropriate place to emit it, which is immediately before
- // the next unfiltered with a greater clustering.
- // If SSTRI considers the block exhausted before encountering such a clustering though, this end marker
- // will never be emitted. So here we just check if there's a closing bound left in the deserializer.
- // If there is, we compare it against the most recently emitted unfiltered (i.e.: the last unfiltered
- // that this RT would enclose. And we have to do THAT comparison because the last name field on the
- // current index block will be whatever was written at the end of the index block (i.e. the last name
- // physically in the block), not the closing bound of the range tombstone (i.e. the last name logically
- // in the block). If all this indicates that there is indeed a range tombstone we're missing, we add it
- // to the buffer and update the open marker field.
- Unfiltered unfiltered = deserializer.readNext();
- RangeTombstoneMarker marker = unfiltered.isRangeTombstoneMarker() ? (RangeTombstoneMarker) unfiltered : null;
- if (marker != null && marker.isClose(false)
- && (mostRecentlyEmitted == null || precedes(marker.clustering(), mostRecentlyEmitted.clustering())))
- {
- buffer.add(marker);
- updateOpenMarker(marker);
- }
- }
-
// If we have an open marker, we should close it before finishing
if (openMarker != null)
{
@@ -350,7 +265,7 @@
private ReverseIndexedReader(RowIndexEntry indexEntry, FileDataInput file, boolean shouldCloseFile)
{
super(file, shouldCloseFile);
- this.indexState = new IndexState(this, sstable.metadata.comparator, indexEntry, true, ifile);
+ this.indexState = new IndexState(this, metadata.comparator, indexEntry, true, ifile);
}
@Override
@@ -448,28 +363,15 @@
if (buffer == null)
buffer = createBuffer(indexState.blocksCount());
- int currentBlock = indexState.currentBlockIdx();
-
// The slice start (resp. slice end) is only meaningful on the last (resp. first) block read (since again,
// we read blocks in reverse order).
boolean canIncludeSliceStart = !hasNextBlock;
boolean canIncludeSliceEnd = !hasPreviousBlock;
- ClusteringPrefix currentFirstName = null;
- ClusteringPrefix nextLastName = null;
- if (!sstable.descriptor.version.storeRows() && currentBlock > 0)
- {
- currentFirstName = indexState.index(currentBlock).firstName;
- nextLastName = indexState.index(currentBlock - 1).lastName;
- }
-
loadFromDisk(canIncludeSliceStart ? slice.start() : null,
canIncludeSliceEnd ? slice.end() : null,
hasPreviousBlock,
- hasNextBlock,
- currentFirstName,
- nextLastName
- );
+ hasNextBlock);
setIterator(slice);
}
@@ -482,18 +384,18 @@
private class ReusablePartitionData
{
- private final CFMetaData metadata;
+ private final TableMetadata metadata;
private final DecoratedKey partitionKey;
- private final PartitionColumns columns;
+ private final RegularAndStaticColumns columns;
private MutableDeletionInfo.Builder deletionBuilder;
private MutableDeletionInfo deletionInfo;
private BTree.Builder<Row> rowBuilder;
private ImmutableBTreePartition built;
- private ReusablePartitionData(CFMetaData metadata,
+ private ReusablePartitionData(TableMetadata metadata,
DecoratedKey partitionKey,
- PartitionColumns columns,
+ RegularAndStaticColumns columns,
int initialRowCapacity)
{
this.metadata = metadata;
diff --git a/src/java/org/apache/cassandra/db/commitlog/AbstractCommitLogSegmentManager.java b/src/java/org/apache/cassandra/db/commitlog/AbstractCommitLogSegmentManager.java
index 20c59a4..dccca88 100755
--- a/src/java/org/apache/cassandra/db/commitlog/AbstractCommitLogSegmentManager.java
+++ b/src/java/org/apache/cassandra/db/commitlog/AbstractCommitLogSegmentManager.java
@@ -32,8 +32,11 @@
import net.nicoulaj.compilecommand.annotations.DontInline;
import org.apache.cassandra.concurrent.NamedThreadFactory;
import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.config.Schema;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.db.*;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.utils.*;
import org.apache.cassandra.utils.concurrent.WaitQueue;
@@ -79,7 +82,8 @@
*/
private final AtomicLong size = new AtomicLong();
- private Thread managerThread;
+ @VisibleForTesting
+ Thread managerThread;
protected final CommitLog commitLog;
private volatile boolean shutdown;
private final BooleanSupplier managerThreadWaitCondition = () -> (availableSegment == null && !atSegmentBufferLimit()) || shutdown;
@@ -178,19 +182,12 @@
}
}
-
/**
* Allocate a segment within this CLSM. Should either succeed or throw.
*/
public abstract Allocation allocate(Mutation mutation, int size);
/**
- * The recovery and replay process replays mutations into memtables and flushes them to disk. Individual CLSM
- * decide what to do with those segments on disk after they've been replayed.
- */
- abstract void handleReplayedSegment(final File file);
-
- /**
* Hook to allow segment managers to track state surrounding creation of new segments. Onl perform as task submit
* to segment manager so it's performed on segment management thread.
*/
@@ -271,7 +268,7 @@
*
* Flushes any dirty CFs for this segment and any older segments, and then discards the segments
*/
- void forceRecycleAll(Iterable<UUID> droppedCfs)
+ void forceRecycleAll(Iterable<TableId> droppedTables)
{
List<CommitLogSegment> segmentsToRecycle = new ArrayList<>(activeSegments);
CommitLogSegment last = segmentsToRecycle.get(segmentsToRecycle.size() - 1);
@@ -291,8 +288,8 @@
future.get();
for (CommitLogSegment segment : activeSegments)
- for (UUID cfId : droppedCfs)
- segment.markClean(cfId, CommitLogPosition.NONE, segment.getCurrentCommitLogPosition());
+ for (TableId tableId : droppedTables)
+ segment.markClean(tableId, CommitLogPosition.NONE, segment.getCurrentCommitLogPosition());
// now recycle segments that are unused, as we may not have triggered a discardCompletedSegments()
// if the previous active segment was the only one to recycle (since an active segment isn't
@@ -330,6 +327,18 @@
}
/**
+ * Delete untracked segment files after replay
+ *
+ * @param file segment file that is no longer in use.
+ */
+ void handleReplayedSegment(final File file)
+ {
+ // (don't decrease managed size, since this was never a "live" segment)
+ logger.trace("(Unopened) segment {} is no longer needed and will be deleted now", file);
+ FileUtils.deleteWithConfirm(file);
+ }
+
+ /**
* Adjust the tracked on-disk size. Called by individual segments to reflect writes, allocations and discards.
* @param addedSize
*/
@@ -366,27 +375,26 @@
final CommitLogPosition maxCommitLogPosition = segments.get(segments.size() - 1).getCurrentCommitLogPosition();
// a map of CfId -> forceFlush() to ensure we only queue one flush per cf
- final Map<UUID, ListenableFuture<?>> flushes = new LinkedHashMap<>();
+ final Map<TableId, ListenableFuture<?>> flushes = new LinkedHashMap<>();
for (CommitLogSegment segment : segments)
{
- for (UUID dirtyCFId : segment.getDirtyCFIDs())
+ for (TableId dirtyTableId : segment.getDirtyTableIds())
{
- Pair<String,String> pair = Schema.instance.getCF(dirtyCFId);
- if (pair == null)
+ TableMetadata metadata = Schema.instance.getTableMetadata(dirtyTableId);
+ if (metadata == null)
{
// even though we remove the schema entry before a final flush when dropping a CF,
// it's still possible for a writer to race and finish his append after the flush.
- logger.trace("Marking clean CF {} that doesn't exist anymore", dirtyCFId);
- segment.markClean(dirtyCFId, CommitLogPosition.NONE, segment.getCurrentCommitLogPosition());
+ logger.trace("Marking clean CF {} that doesn't exist anymore", dirtyTableId);
+ segment.markClean(dirtyTableId, CommitLogPosition.NONE, segment.getCurrentCommitLogPosition());
}
- else if (!flushes.containsKey(dirtyCFId))
+ else if (!flushes.containsKey(dirtyTableId))
{
- String keyspace = pair.left;
- final ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(dirtyCFId);
+ final ColumnFamilyStore cfs = Keyspace.open(metadata.keyspace).getColumnFamilyStore(dirtyTableId);
// can safely call forceFlush here as we will only ever block (briefly) for other attempts to flush,
// no deadlock possibility since switchLock removal
- flushes.put(dirtyCFId, force ? cfs.forceFlush() : cfs.forceFlush(maxCommitLogPosition));
+ flushes.put(dirtyTableId, force ? cfs.forceFlush() : cfs.forceFlush(maxCommitLogPosition));
}
}
}
@@ -478,8 +486,11 @@
*/
public void awaitTermination() throws InterruptedException
{
- managerThread.join();
- managerThread = null;
+ if (managerThread != null)
+ {
+ managerThread.join();
+ managerThread = null;
+ }
for (CommitLogSegment segment : activeSegments)
segment.close();
diff --git a/src/java/org/apache/cassandra/db/commitlog/AbstractCommitLogService.java b/src/java/org/apache/cassandra/db/commitlog/AbstractCommitLogService.java
index b7ab705..a65ef00 100644
--- a/src/java/org/apache/cassandra/db/commitlog/AbstractCommitLogService.java
+++ b/src/java/org/apache/cassandra/db/commitlog/AbstractCommitLogService.java
@@ -26,11 +26,10 @@
import org.slf4j.LoggerFactory;
import com.codahale.metrics.Timer.Context;
-
import org.apache.cassandra.concurrent.NamedThreadFactory;
import org.apache.cassandra.config.Config;
import org.apache.cassandra.db.commitlog.CommitLogSegment.Allocation;
-import org.apache.cassandra.utils.Clock;
+import org.apache.cassandra.utils.MonotonicClock;
import org.apache.cassandra.utils.NoSpamLogger;
import org.apache.cassandra.utils.concurrent.WaitQueue;
@@ -133,21 +132,20 @@
throw new IllegalArgumentException(String.format("Commit log flush interval must be positive: %fms",
syncIntervalNanos * 1e-6));
shutdown = false;
- Runnable runnable = new SyncRunnable(new Clock());
- thread = NamedThreadFactory.createThread(runnable, name);
+ thread = NamedThreadFactory.createThread(new SyncRunnable(MonotonicClock.preciseTime), name);
thread.start();
}
class SyncRunnable implements Runnable
{
- private final Clock clock;
+ private final MonotonicClock clock;
private long firstLagAt = 0;
private long totalSyncDuration = 0; // total time spent syncing since firstLagAt
private long syncExceededIntervalBy = 0; // time that syncs exceeded pollInterval since firstLagAt
private int lagCount = 0;
private int syncCount = 0;
- SyncRunnable(Clock clock)
+ SyncRunnable(MonotonicClock clock)
{
this.clock = clock;
}
@@ -169,7 +167,7 @@
try
{
// sync and signal
- long pollStarted = clock.nanoTime();
+ long pollStarted = clock.now();
boolean flushToDisk = lastSyncedAt + syncIntervalNanos <= pollStarted || shutdownRequested || syncRequested;
if (flushToDisk)
{
@@ -186,7 +184,7 @@
commitLog.sync(false);
}
- long now = clock.nanoTime();
+ long now = clock.now();
if (flushToDisk)
maybeLogFlushLag(pollStarted, now);
@@ -314,7 +312,8 @@
public void awaitTermination() throws InterruptedException
{
- thread.join();
+ if (thread != null)
+ thread.join();
}
public long getCompletedTasks()
diff --git a/src/java/org/apache/cassandra/db/commitlog/BatchCommitLogService.java b/src/java/org/apache/cassandra/db/commitlog/BatchCommitLogService.java
index 4edfa34..78bf30c 100644
--- a/src/java/org/apache/cassandra/db/commitlog/BatchCommitLogService.java
+++ b/src/java/org/apache/cassandra/db/commitlog/BatchCommitLogService.java
@@ -17,13 +17,18 @@
*/
package org.apache.cassandra.db.commitlog;
-import org.apache.cassandra.config.DatabaseDescriptor;
-
class BatchCommitLogService extends AbstractCommitLogService
{
+ /**
+ * Batch mode does not rely on the sync thread in {@link AbstractCommitLogService} to wake up for triggering
+ * the disk sync. Instead we trigger it explicitly in {@link #maybeWaitForSync(CommitLogSegment.Allocation)}.
+ * This value here is largely irrelevant, but should high enough so the sync thread is not continually waking up.
+ */
+ private static final int POLL_TIME_MILLIS = 1000;
+
public BatchCommitLogService(CommitLog commitLog)
{
- super(commitLog, "COMMIT-LOG-WRITER", (int) DatabaseDescriptor.getCommitLogSyncBatchWindow());
+ super(commitLog, "COMMIT-LOG-WRITER", POLL_TIME_MILLIS);
}
protected void maybeWaitForSync(CommitLogSegment.Allocation alloc)
diff --git a/src/java/org/apache/cassandra/db/commitlog/CommitLog.java b/src/java/org/apache/cassandra/db/commitlog/CommitLog.java
index a9ac968..e7f8743 100644
--- a/src/java/org/apache/cassandra/db/commitlog/CommitLog.java
+++ b/src/java/org/apache/cassandra/db/commitlog/CommitLog.java
@@ -20,6 +20,7 @@
import java.io.*;
import java.nio.ByteBuffer;
import java.util.*;
+import java.util.function.Function;
import java.util.zip.CRC32;
import com.google.common.annotations.VisibleForTesting;
@@ -27,11 +28,10 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.cassandra.config.Config;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.config.ParameterizedClass;
import org.apache.cassandra.db.*;
-import org.apache.cassandra.exceptions.WriteTimeoutException;
+import org.apache.cassandra.exceptions.CDCWriteException;
import org.apache.cassandra.io.FSWriteError;
import org.apache.cassandra.io.compress.ICompressor;
import org.apache.cassandra.io.util.BufferedDataOutputStreamPlus;
@@ -41,6 +41,7 @@
import org.apache.cassandra.metrics.CommitLogMetrics;
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.schema.CompressionParams;
+import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.security.EncryptionContext;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.FBUtilities;
@@ -63,10 +64,6 @@
public static final CommitLog instance = CommitLog.construct();
- // we only permit records HALF the size of a commit log, to ensure we don't spin allocating many mostly
- // empty segments when writing large records
- final long MAX_MUTATION_SIZE = DatabaseDescriptor.getMaxMutationSize();
-
final public AbstractCommitLogSegmentManager segmentManager;
public final CommitLogArchiver archiver;
@@ -74,18 +71,25 @@
final AbstractCommitLogService executor;
volatile Configuration configuration;
+ private boolean started = false;
private static CommitLog construct()
{
- CommitLog log = new CommitLog(CommitLogArchiver.construct());
+ CommitLog log = new CommitLog(CommitLogArchiver.construct(), DatabaseDescriptor.getCommitLogSegmentMgrProvider());
MBeanWrapper.instance.registerMBean(log, "org.apache.cassandra.db:type=Commitlog");
- return log.start();
+ return log;
}
@VisibleForTesting
CommitLog(CommitLogArchiver archiver)
{
+ this(archiver, DatabaseDescriptor.getCommitLogSegmentMgrProvider());
+ }
+
+ @VisibleForTesting
+ CommitLog(CommitLogArchiver archiver, Function<CommitLog, AbstractCommitLogSegmentManager> segmentManagerProvider)
+ {
this.configuration = new Configuration(DatabaseDescriptor.getCommitLogCompression(),
DatabaseDescriptor.getEncryptionContext());
DatabaseDescriptor.createAllDirectories();
@@ -93,22 +97,45 @@
this.archiver = archiver;
metrics = new CommitLogMetrics();
- executor = DatabaseDescriptor.getCommitLogSync() == Config.CommitLogSync.batch
- ? new BatchCommitLogService(this)
- : new PeriodicCommitLogService(this);
+ switch (DatabaseDescriptor.getCommitLogSync())
+ {
+ case periodic:
+ executor = new PeriodicCommitLogService(this);
+ break;
+ case batch:
+ executor = new BatchCommitLogService(this);
+ break;
+ case group:
+ executor = new GroupCommitLogService(this);
+ break;
+ default:
+ throw new IllegalArgumentException("Unknown commitlog service type: " + DatabaseDescriptor.getCommitLogSync());
+ }
- segmentManager = DatabaseDescriptor.isCDCEnabled()
- ? new CommitLogSegmentManagerCDC(this, DatabaseDescriptor.getCommitLogLocation())
- : new CommitLogSegmentManagerStandard(this, DatabaseDescriptor.getCommitLogLocation());
+ segmentManager = segmentManagerProvider.apply(this);
// register metrics
metrics.attach(executor, segmentManager);
}
- CommitLog start()
+ /**
+ * Tries to start the CommitLog if not already started.
+ */
+ synchronized public CommitLog start()
{
- segmentManager.start();
- executor.start();
+ if (started)
+ return this;
+
+ try
+ {
+ segmentManager.start();
+ executor.start();
+ started = true;
+ } catch (Throwable t)
+ {
+ started = false;
+ throw t;
+ }
return this;
}
@@ -124,7 +151,7 @@
// submit all files for this segment manager for archiving prior to recovery - CASSANDRA-6904
// The files may have already been archived by normal CommitLog operation. This may cause errors in this
- // archiving pass, which we should not treat as serious.
+ // archiving pass, which we should not treat as serious.
for (File file : new File(segmentManager.storageDirectory).listFiles(unmanagedFilesFilter))
{
archiver.maybeArchive(file.getPath(), file.getName());
@@ -195,9 +222,9 @@
/**
* Flushes all dirty CFs, waiting for them to free and recycle any segments they were retaining
*/
- public void forceRecycleAllSegments(Iterable<UUID> droppedCfs)
+ public void forceRecycleAllSegments(Iterable<TableId> droppedTables)
{
- segmentManager.forceRecycleAll(droppedCfs);
+ segmentManager.forceRecycleAll(droppedTables);
}
/**
@@ -205,7 +232,7 @@
*/
public void forceRecycleAllSegments()
{
- segmentManager.forceRecycleAll(Collections.<UUID>emptyList());
+ segmentManager.forceRecycleAll(Collections.emptyList());
}
/**
@@ -228,25 +255,19 @@
* Add a Mutation to the commit log. If CDC is enabled, this can fail.
*
* @param mutation the Mutation to add to the log
- * @throws WriteTimeoutException
+ * @throws CDCWriteException
*/
- public CommitLogPosition add(Mutation mutation) throws WriteTimeoutException
+ public CommitLogPosition add(Mutation mutation) throws CDCWriteException
{
assert mutation != null;
+ mutation.validateSize(MessagingService.current_version, ENTRY_OVERHEAD_SIZE);
+
try (DataOutputBuffer dob = DataOutputBuffer.scratchBuffer.get())
{
Mutation.serializer.serialize(mutation, dob, MessagingService.current_version);
int size = dob.getLength();
-
int totalSize = size + ENTRY_OVERHEAD_SIZE;
- if (totalSize > MAX_MUTATION_SIZE)
- {
- throw new IllegalArgumentException(String.format("Mutation of %s is too large for the maximum size of %s",
- FBUtilities.prettyPrintMemory(totalSize),
- FBUtilities.prettyPrintMemory(MAX_MUTATION_SIZE)));
- }
-
Allocation alloc = segmentManager.allocate(mutation, totalSize);
CRC32 checksum = new CRC32();
@@ -285,13 +306,13 @@
* Modifies the per-CF dirty cursors of any commit log segments for the column family according to the position
* given. Discards any commit log segments that are no longer used.
*
- * @param cfId the column family ID that was flushed
+ * @param id the table that was flushed
* @param lowerBound the lowest covered replay position of the flush
* @param lowerBound the highest covered replay position of the flush
*/
- public void discardCompletedSegments(final UUID cfId, final CommitLogPosition lowerBound, final CommitLogPosition upperBound)
+ public void discardCompletedSegments(final TableId id, final CommitLogPosition lowerBound, final CommitLogPosition upperBound)
{
- logger.trace("discard completed log segments for {}-{}, table {}", lowerBound, upperBound, cfId);
+ logger.trace("discard completed log segments for {}-{}, table {}", lowerBound, upperBound, id);
// Go thru the active segment files, which are ordered oldest to newest, marking the
// flushed CF as clean, until we reach the segment file containing the CommitLogPosition passed
@@ -300,7 +321,7 @@
for (Iterator<CommitLogSegment> iter = segmentManager.getActiveSegments().iterator(); iter.hasNext();)
{
CommitLogSegment segment = iter.next();
- segment.markClean(cfId, lowerBound, upperBound);
+ segment.markClean(id, lowerBound, upperBound);
if (segment.isUnused())
{
@@ -353,8 +374,9 @@
public List<String> getActiveSegmentNames()
{
- List<String> segmentNames = new ArrayList<>();
- for (CommitLogSegment seg : segmentManager.getActiveSegments())
+ Collection<CommitLogSegment> segments = segmentManager.getActiveSegments();
+ List<String> segmentNames = new ArrayList<>(segments.size());
+ for (CommitLogSegment seg : segments)
segmentNames.add(seg.getName());
return segmentNames;
}
@@ -392,8 +414,12 @@
* Shuts down the threads used by the commit log, blocking until completion.
* TODO this should accept a timeout, and throw TimeoutException
*/
- public void shutdownBlocking() throws InterruptedException
+ synchronized public void shutdownBlocking() throws InterruptedException
{
+ if (!started)
+ return;
+
+ started = false;
executor.shutdown();
executor.awaitTermination();
segmentManager.shutdown();
@@ -404,7 +430,8 @@
* FOR TESTING PURPOSES
* @return the number of files recovered
*/
- public int resetUnsafe(boolean deleteSegments) throws IOException
+ @VisibleForTesting
+ synchronized public int resetUnsafe(boolean deleteSegments) throws IOException
{
stopUnsafe(deleteSegments);
resetConfiguration();
@@ -414,16 +441,20 @@
/**
* FOR TESTING PURPOSES.
*/
- public void resetConfiguration()
+ @VisibleForTesting
+ synchronized public void resetConfiguration()
{
configuration = new Configuration(DatabaseDescriptor.getCommitLogCompression(),
DatabaseDescriptor.getEncryptionContext());
}
/**
+ * FOR TESTING PURPOSES
*/
- public void stopUnsafe(boolean deleteSegments)
+ @VisibleForTesting
+ synchronized public void stopUnsafe(boolean deleteSegments)
{
+ started = false;
executor.shutdown();
try
{
@@ -438,14 +469,15 @@
if (DatabaseDescriptor.isCDCEnabled() && deleteSegments)
for (File f : new File(DatabaseDescriptor.getCDCLogLocation()).listFiles())
FileUtils.deleteWithConfirm(f);
-
}
/**
* FOR TESTING PURPOSES
*/
- public int restartUnsafe() throws IOException
+ @VisibleForTesting
+ synchronized public int restartUnsafe() throws IOException
{
+ started = false;
return start().recoverSegmentsOnDisk();
}
diff --git a/src/java/org/apache/cassandra/db/commitlog/CommitLogArchiver.java b/src/java/org/apache/cassandra/db/commitlog/CommitLogArchiver.java
index a30ca0e..b58a316 100644
--- a/src/java/org/apache/cassandra/db/commitlog/CommitLogArchiver.java
+++ b/src/java/org/apache/cassandra/db/commitlog/CommitLogArchiver.java
@@ -42,7 +42,6 @@
import org.slf4j.LoggerFactory;
import com.google.common.base.Strings;
-import com.google.common.base.Throwables;
public class CommitLogArchiver
{
@@ -233,7 +232,7 @@
throw new IllegalStateException("Cannot safely construct descriptor for segment, either from its name or its header: " + fromFile.getPath());
else if (fromHeader != null && fromName != null && !fromHeader.equalsIgnoringCompression(fromName))
throw new IllegalStateException(String.format("Cannot safely construct descriptor for segment, as name and header descriptors do not match (%s vs %s): %s", fromHeader, fromName, fromFile.getPath()));
- else if (fromName != null && fromHeader == null && fromName.version >= CommitLogDescriptor.VERSION_21)
+ else if (fromName != null && fromHeader == null)
throw new IllegalStateException("Cannot safely construct descriptor for segment, as name descriptor implies a version that should contain a header descriptor, but that descriptor could not be read: " + fromFile.getPath());
else if (fromHeader != null)
descriptor = fromHeader;
diff --git a/src/java/org/apache/cassandra/db/commitlog/CommitLogDescriptor.java b/src/java/org/apache/cassandra/db/commitlog/CommitLogDescriptor.java
index a74bfe7..700f12a 100644
--- a/src/java/org/apache/cassandra/db/commitlog/CommitLogDescriptor.java
+++ b/src/java/org/apache/cassandra/db/commitlog/CommitLogDescriptor.java
@@ -57,18 +57,16 @@
static final String COMPRESSION_PARAMETERS_KEY = "compressionParameters";
static final String COMPRESSION_CLASS_KEY = "compressionClass";
- public static final int VERSION_12 = 2;
- public static final int VERSION_20 = 3;
- public static final int VERSION_21 = 4;
- public static final int VERSION_22 = 5;
+ // We don't support anything pre-3.0
public static final int VERSION_30 = 6;
+ public static final int VERSION_40 = 7;
/**
* Increment this number if there is a changes in the commit log disc layout or MessagingVersion changes.
* Note: make sure to handle {@link #getMessagingVersion()}
*/
@VisibleForTesting
- public static final int current_version = VERSION_30;
+ public static final int current_version = VERSION_40;
final int version;
public final long id;
@@ -104,20 +102,15 @@
out.putLong(descriptor.id);
updateChecksumInt(crc, (int) (descriptor.id & 0xFFFFFFFFL));
updateChecksumInt(crc, (int) (descriptor.id >>> 32));
- if (descriptor.version >= VERSION_22)
- {
- String parametersString = constructParametersString(descriptor.compression, descriptor.encryptionContext, additionalHeaders);
- byte[] parametersBytes = parametersString.getBytes(StandardCharsets.UTF_8);
- if (parametersBytes.length != (((short) parametersBytes.length) & 0xFFFF))
- throw new ConfigurationException(String.format("Compression parameters too long, length %d cannot be above 65535.",
- parametersBytes.length));
- out.putShort((short) parametersBytes.length);
- updateChecksumInt(crc, parametersBytes.length);
- out.put(parametersBytes);
- crc.update(parametersBytes, 0, parametersBytes.length);
- }
- else
- assert descriptor.compression == null;
+ String parametersString = constructParametersString(descriptor.compression, descriptor.encryptionContext, additionalHeaders);
+ byte[] parametersBytes = parametersString.getBytes(StandardCharsets.UTF_8);
+ if (parametersBytes.length != (((short) parametersBytes.length) & 0xFFFF))
+ throw new ConfigurationException(String.format("Compression parameters too long, length %d cannot be above 65535.",
+ parametersBytes.length));
+ out.putShort((short) parametersBytes.length);
+ updateChecksumInt(crc, parametersBytes.length);
+ out.put(parametersBytes);
+ crc.update(parametersBytes, 0, parametersBytes.length);
out.putInt((int) crc.getValue());
}
@@ -157,16 +150,15 @@
{
CRC32 checkcrc = new CRC32();
int version = input.readInt();
+ if (version < VERSION_30)
+ throw new IllegalArgumentException("Unsupported pre-3.0 commit log found; cannot read.");
+
updateChecksumInt(checkcrc, version);
long id = input.readLong();
updateChecksumInt(checkcrc, (int) (id & 0xFFFFFFFFL));
updateChecksumInt(checkcrc, (int) (id >>> 32));
- int parametersLength = 0;
- if (version >= VERSION_22)
- {
- parametersLength = input.readShort() & 0xFFFF;
- updateChecksumInt(checkcrc, parametersLength);
- }
+ int parametersLength = input.readShort() & 0xFFFF;
+ updateChecksumInt(checkcrc, parametersLength);
// This should always succeed as parametersLength cannot be too long even for a
// corrupt segment file.
byte[] parametersBytes = new byte[parametersLength];
@@ -213,16 +205,10 @@
{
switch (version)
{
- case VERSION_12:
- return MessagingService.VERSION_12;
- case VERSION_20:
- return MessagingService.VERSION_20;
- case VERSION_21:
- return MessagingService.VERSION_21;
- case VERSION_22:
- return MessagingService.VERSION_22;
case VERSION_30:
- return MessagingService.FORCE_3_0_PROTOCOL_VERSION ? MessagingService.VERSION_30 : MessagingService.VERSION_3014;
+ return MessagingService.VERSION_30;
+ case VERSION_40:
+ return MessagingService.VERSION_40;
default:
throw new IllegalStateException("Unknown commitlog version " + version);
}
@@ -233,6 +219,11 @@
return FILENAME_PREFIX + version + SEPARATOR + id + FILENAME_EXTENSION;
}
+ public String cdcIndexFileName()
+ {
+ return FILENAME_PREFIX + version + SEPARATOR + id + "_cdc.idx";
+ }
+
/**
* @param filename the filename to check
* @return true if filename could be a commit log based on it's filename
diff --git a/src/java/org/apache/cassandra/db/commitlog/CommitLogPosition.java b/src/java/org/apache/cassandra/db/commitlog/CommitLogPosition.java
index 84054a4..3ffb04c 100644
--- a/src/java/org/apache/cassandra/db/commitlog/CommitLogPosition.java
+++ b/src/java/org/apache/cassandra/db/commitlog/CommitLogPosition.java
@@ -40,6 +40,7 @@
public static final CommitLogPosition NONE = new CommitLogPosition(-1, 0);
public final long segmentId;
+ // Indicates the end position of the mutation in the CommitLog
public final int position;
public static final Comparator<CommitLogPosition> comparator = new Comparator<CommitLogPosition>()
diff --git a/src/java/org/apache/cassandra/db/commitlog/CommitLogReadHandler.java b/src/java/org/apache/cassandra/db/commitlog/CommitLogReadHandler.java
index 0602147..ee05235 100644
--- a/src/java/org/apache/cassandra/db/commitlog/CommitLogReadHandler.java
+++ b/src/java/org/apache/cassandra/db/commitlog/CommitLogReadHandler.java
@@ -69,7 +69,7 @@
*
* @param m deserialized mutation
* @param size serialized size of the mutation
- * @param entryLocation filePointer offset inside the CommitLogSegment for the record
+ * @param entryLocation filePointer offset inside the CommitLogSegment for the end of the record
* @param desc CommitLogDescriptor for mutation being processed
*/
void handleMutation(Mutation m, int size, int entryLocation, CommitLogDescriptor desc);
diff --git a/src/java/org/apache/cassandra/db/commitlog/CommitLogReader.java b/src/java/org/apache/cassandra/db/commitlog/CommitLogReader.java
index 4d74557..5123580 100644
--- a/src/java/org/apache/cassandra/db/commitlog/CommitLogReader.java
+++ b/src/java/org/apache/cassandra/db/commitlog/CommitLogReader.java
@@ -17,6 +17,8 @@
*/
package org.apache.cassandra.db.commitlog;
+import java.nio.file.Files;
+import java.nio.file.Path;
import java.io.*;
import java.util.*;
import java.util.concurrent.atomic.AtomicInteger;
@@ -29,17 +31,17 @@
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.Mutation;
-import org.apache.cassandra.db.UnknownColumnFamilyException;
import org.apache.cassandra.db.commitlog.CommitLogReadHandler.CommitLogReadErrorReason;
import org.apache.cassandra.db.commitlog.CommitLogReadHandler.CommitLogReadException;
import org.apache.cassandra.db.partitions.PartitionUpdate;
-import org.apache.cassandra.db.rows.SerializationHelper;
+import org.apache.cassandra.db.rows.DeserializationHelper;
+import org.apache.cassandra.exceptions.UnknownTableException;
import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.io.util.ChannelProxy;
import org.apache.cassandra.io.util.DataInputBuffer;
import org.apache.cassandra.io.util.FileDataInput;
import org.apache.cassandra.io.util.RandomAccessReader;
import org.apache.cassandra.io.util.RebufferingInputStream;
+import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.utils.JVMStabilityInspector;
import static org.apache.cassandra.utils.FBUtilities.updateChecksumInt;
@@ -53,7 +55,7 @@
@VisibleForTesting
public static final int ALL_MUTATIONS = -1;
private final CRC32 checksum;
- private final Map<UUID, AtomicInteger> invalidMutations;
+ private final Map<TableId, AtomicInteger> invalidMutations;
private byte[] buffer;
@@ -64,7 +66,7 @@
buffer = new byte[4096];
}
- public Set<Map.Entry<UUID, AtomicInteger>> getInvalidMutations()
+ public Set<Map.Entry<TableId, AtomicInteger>> getInvalidMutations()
{
return invalidMutations.entrySet();
}
@@ -79,11 +81,6 @@
private static boolean shouldSkip(File file) throws IOException, ConfigurationException
{
- CommitLogDescriptor desc = CommitLogDescriptor.fromFileName(file.getName());
- if (desc.version < CommitLogDescriptor.VERSION_21)
- {
- return false;
- }
try(RandomAccessReader reader = RandomAccessReader.open(file))
{
CommitLogDescriptor.readHeader(reader, DatabaseDescriptor.getEncryptionContext());
@@ -93,7 +90,7 @@
}
}
- private static List<File> filterCommitLogFiles(File[] toFilter)
+ static List<File> filterCommitLogFiles(File[] toFilter)
{
List<File> filtered = new ArrayList<>(toFilter.length);
for (File file: toFilter)
@@ -142,6 +139,14 @@
}
/**
+ * Reads all mutations from passed in file from minPosition
+ */
+ public void readCommitLogSegment(CommitLogReadHandler handler, File file, CommitLogPosition minPosition, boolean tolerateTruncation) throws IOException
+ {
+ readCommitLogSegment(handler, file, minPosition, ALL_MUTATIONS, tolerateTruncation);
+ }
+
+ /**
* Reads passed in file fully, up to mutationLimit count
*/
@VisibleForTesting
@@ -154,7 +159,7 @@
* Reads mutations from file, handing them off to handler
* @param handler Handler that will take action based on deserialized Mutations
* @param file CommitLogSegment file to read
- * @param minPosition Optional minimum CommitLogPosition - all segments with id > or matching w/greater position will be read
+ * @param minPosition Optional minimum CommitLogPosition - all segments with id larger or matching w/greater position will be read
* @param mutationLimit Optional limit on # of mutations to replay. Local ALL_MUTATIONS serves as marker to play all.
* @param tolerateTruncation Whether or not we should allow truncation of this file or throw if EOF found
*
@@ -171,19 +176,6 @@
try(RandomAccessReader reader = RandomAccessReader.open(file))
{
- if (desc.version < CommitLogDescriptor.VERSION_21)
- {
- if (!shouldSkipSegmentId(file, desc, minPosition))
- {
- if (minPosition.segmentId == desc.id)
- reader.seek(minPosition.position);
- ReadStatusTracker statusTracker = new ReadStatusTracker(mutationLimit, tolerateTruncation);
- statusTracker.errorContext = desc.fileName();
- readSection(handler, reader, minPosition, (int) reader.length(), statusTracker, desc);
- }
- return;
- }
-
final long segmentIdFromFilename = desc.id;
try
{
@@ -198,7 +190,7 @@
if (desc == null)
{
// don't care about whether or not the handler thinks we can continue. We can't w/out descriptor.
- // whether or not we continue with startup will depend on whether this is the last segment
+ // whether or not we can continue depends on whether this is the last segment
handler.handleUnrecoverableError(new CommitLogReadException(
String.format("Could not read commit log descriptor in file %s", file),
CommitLogReadErrorReason.UNRECOVERABLE_DESCRIPTOR_ERROR,
@@ -261,7 +253,7 @@
throw (IOException) re.getCause();
throw re;
}
- logger.debug("Finished reading {}", file);
+ logger.info("Finished reading {}", file);
}
}
@@ -322,7 +314,7 @@
// read an EOF here
if(end - reader.getFilePointer() < 4)
{
- logger.trace("Not enough bytes left for another mutation in this CommitLog segment, continuing");
+ logger.trace("Not enough bytes left for another mutation in this CommitLog section, continuing");
statusTracker.requestTermination();
return;
}
@@ -417,7 +409,7 @@
* @param inputBuffer raw byte array w/Mutation data
* @param size deserialized size of mutation
* @param minPosition We need to suppress replay of mutations that are before the required minPosition
- * @param entryLocation filePointer offset of mutation within CommitLogSegment
+ * @param entryLocation filePointer offset of end of mutation within CommitLogSegment
* @param desc CommitLogDescriptor being worked on
*/
@VisibleForTesting
@@ -437,20 +429,20 @@
{
mutation = Mutation.serializer.deserialize(bufIn,
desc.getMessagingVersion(),
- SerializationHelper.Flag.LOCAL);
+ DeserializationHelper.Flag.LOCAL);
// doublecheck that what we read is still] valid for the current schema
for (PartitionUpdate upd : mutation.getPartitionUpdates())
upd.validate();
}
- catch (UnknownColumnFamilyException ex)
+ catch (UnknownTableException ex)
{
- if (ex.cfId == null)
+ if (ex.id == null)
return;
- AtomicInteger i = invalidMutations.get(ex.cfId);
+ AtomicInteger i = invalidMutations.get(ex.id);
if (i == null)
{
i = new AtomicInteger(1);
- invalidMutations.put(ex.cfId, i);
+ invalidMutations.put(ex.id, i);
}
else
i.incrementAndGet();
@@ -459,9 +451,9 @@
catch (Throwable t)
{
JVMStabilityInspector.inspectThrowable(t);
- File f = File.createTempFile("mutation", "dat");
+ Path p = Files.createTempFile("mutation", "dat");
- try (DataOutputStream out = new DataOutputStream(new FileOutputStream(f)))
+ try (DataOutputStream out = new DataOutputStream(Files.newOutputStream(p)))
{
out.write(inputBuffer, 0, size);
}
@@ -471,7 +463,7 @@
String.format(
"Unexpected error deserializing mutation; saved to %s. " +
"This may be caused by replaying a mutation against a table with the same name but incompatible schema. " +
- "Exception follows: %s", f.getAbsolutePath(), t),
+ "Exception follows: %s", p.toString(), t),
CommitLogReadErrorReason.MUTATION_ERROR,
false));
return;
@@ -492,42 +484,17 @@
{
public static long calculateClaimedChecksum(FileDataInput input, int commitLogVersion) throws IOException
{
- switch (commitLogVersion)
- {
- case CommitLogDescriptor.VERSION_12:
- case CommitLogDescriptor.VERSION_20:
- return input.readLong();
- // Changed format in 2.1
- default:
- return input.readInt() & 0xffffffffL;
- }
+ return input.readInt() & 0xffffffffL;
}
public static void updateChecksum(CRC32 checksum, int serializedSize, int commitLogVersion)
{
- switch (commitLogVersion)
- {
- case CommitLogDescriptor.VERSION_12:
- checksum.update(serializedSize);
- break;
- // Changed format in 2.0
- default:
- updateChecksumInt(checksum, serializedSize);
- break;
- }
+ updateChecksumInt(checksum, serializedSize);
}
public static long calculateClaimedCRC32(FileDataInput input, int commitLogVersion) throws IOException
{
- switch (commitLogVersion)
- {
- case CommitLogDescriptor.VERSION_12:
- case CommitLogDescriptor.VERSION_20:
- return input.readLong();
- // Changed format in 2.1
- default:
- return input.readInt() & 0xffffffffL;
- }
+ return input.readInt() & 0xffffffffL;
}
}
diff --git a/src/java/org/apache/cassandra/db/commitlog/CommitLogReplayer.java b/src/java/org/apache/cassandra/db/commitlog/CommitLogReplayer.java
index ea62fd8..aabdb9a 100644
--- a/src/java/org/apache/cassandra/db/commitlog/CommitLogReplayer.java
+++ b/src/java/org/apache/cassandra/db/commitlog/CommitLogReplayer.java
@@ -26,24 +26,31 @@
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Predicate;
-import com.google.common.collect.*;
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Multimap;
+import com.google.common.collect.Ordering;
+
import org.apache.commons.lang3.StringUtils;
import org.cliffc.high_scale_lib.NonBlockingHashSet;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.concurrent.Stage;
-import org.apache.cassandra.concurrent.StageManager;
-import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.config.Config;
-import org.apache.cassandra.config.Schema;
-import org.apache.cassandra.config.SchemaConstants;
+import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.db.Mutation;
import org.apache.cassandra.db.SystemKeyspace;
import org.apache.cassandra.db.partitions.PartitionUpdate;
import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.io.util.RandomAccessReader;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.SchemaConstants;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.schema.TableMetadataRef;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.WrappedRunnable;
@@ -61,7 +68,7 @@
private final Queue<Future<Integer>> futures;
private final AtomicInteger replayedCount;
- private final Map<UUID, IntervalSet<CommitLogPosition>> cfPersisted;
+ private final Map<TableId, IntervalSet<CommitLogPosition>> cfPersisted;
private final CommitLogPosition globalPosition;
// Used to throttle speed of replay of mutations if we pass the max outstanding count
@@ -71,15 +78,18 @@
private final CommitLogArchiver archiver;
@VisibleForTesting
+ protected boolean sawCDCMutation;
+
+ @VisibleForTesting
protected CommitLogReader commitLogReader;
CommitLogReplayer(CommitLog commitLog,
CommitLogPosition globalPosition,
- Map<UUID, IntervalSet<CommitLogPosition>> cfPersisted,
+ Map<TableId, IntervalSet<CommitLogPosition>> cfPersisted,
ReplayFilter replayFilter)
{
- this.keyspacesReplayed = new NonBlockingHashSet<Keyspace>();
- this.futures = new ArrayDeque<Future<Integer>>();
+ this.keyspacesReplayed = new NonBlockingHashSet<>();
+ this.futures = new ArrayDeque<>();
// count the number of replayed mutation. We don't really care about atomicity, but we need it to be a reference.
this.replayedCount = new AtomicInteger();
this.cfPersisted = cfPersisted;
@@ -92,35 +102,35 @@
public static CommitLogReplayer construct(CommitLog commitLog)
{
// compute per-CF and global replay intervals
- Map<UUID, IntervalSet<CommitLogPosition>> cfPersisted = new HashMap<>();
+ Map<TableId, IntervalSet<CommitLogPosition>> cfPersisted = new HashMap<>();
ReplayFilter replayFilter = ReplayFilter.create();
for (ColumnFamilyStore cfs : ColumnFamilyStore.all())
{
// but, if we've truncated the cf in question, then we need to need to start replay after the truncation
- CommitLogPosition truncatedAt = SystemKeyspace.getTruncatedPosition(cfs.metadata.cfId);
+ CommitLogPosition truncatedAt = SystemKeyspace.getTruncatedPosition(cfs.metadata.id);
if (truncatedAt != null)
{
// Point in time restore is taken to mean that the tables need to be replayed even if they were
// deleted at a later point in time. Any truncation record after that point must thus be cleared prior
// to replay (CASSANDRA-9195).
long restoreTime = commitLog.archiver.restorePointInTime;
- long truncatedTime = SystemKeyspace.getTruncatedAt(cfs.metadata.cfId);
+ long truncatedTime = SystemKeyspace.getTruncatedAt(cfs.metadata.id);
if (truncatedTime > restoreTime)
{
if (replayFilter.includes(cfs.metadata))
{
logger.info("Restore point in time is before latest truncation of table {}.{}. Clearing truncation record.",
- cfs.metadata.ksName,
- cfs.metadata.cfName);
- SystemKeyspace.removeTruncationRecord(cfs.metadata.cfId);
+ cfs.metadata.keyspace,
+ cfs.metadata.name);
+ SystemKeyspace.removeTruncationRecord(cfs.metadata.id);
truncatedAt = null;
}
}
}
IntervalSet<CommitLogPosition> filter = persistedIntervals(cfs.getLiveSSTables(), truncatedAt);
- cfPersisted.put(cfs.metadata.cfId, filter);
+ cfPersisted.put(cfs.metadata.id, filter);
}
CommitLogPosition globalPosition = firstNotCovered(cfPersisted.values());
logger.debug("Global replay position is {} from columnfamilies {}", globalPosition, FBUtilities.toString(cfPersisted));
@@ -129,21 +139,62 @@
public void replayPath(File file, boolean tolerateTruncation) throws IOException
{
+ sawCDCMutation = false;
commitLogReader.readCommitLogSegment(this, file, globalPosition, CommitLogReader.ALL_MUTATIONS, tolerateTruncation);
+ if (sawCDCMutation)
+ handleCDCReplayCompletion(file);
}
public void replayFiles(File[] clogs) throws IOException
{
- commitLogReader.readAllFiles(this, clogs, globalPosition);
+ List<File> filteredLogs = CommitLogReader.filterCommitLogFiles(clogs);
+ int i = 0;
+ for (File file: filteredLogs)
+ {
+ i++;
+ sawCDCMutation = false;
+ commitLogReader.readCommitLogSegment(this, file, globalPosition, i == filteredLogs.size());
+ if (sawCDCMutation)
+ handleCDCReplayCompletion(file);
+ }
}
+
+ /**
+ * Upon replay completion, CDC needs to hard-link files in the CDC folder and calculate index files so consumers can
+ * begin their work.
+ */
+ private void handleCDCReplayCompletion(File f) throws IOException
+ {
+ // Can only reach this point if CDC is enabled, thus we have a CDCSegmentManager
+ ((CommitLogSegmentManagerCDC)CommitLog.instance.segmentManager).addCDCSize(f.length());
+
+ File dest = new File(DatabaseDescriptor.getCDCLogLocation(), f.getName());
+
+ // If hard link already exists, assume it's from a previous node run. If people are mucking around in the cdc_raw
+ // directory that's on them.
+ if (!dest.exists())
+ FileUtils.createHardLink(f, dest);
+
+ // The reader has already verified we can deserialize the descriptor.
+ CommitLogDescriptor desc;
+ try(RandomAccessReader reader = RandomAccessReader.open(f))
+ {
+ desc = CommitLogDescriptor.readHeader(reader, DatabaseDescriptor.getEncryptionContext());
+ assert desc != null;
+ assert f.length() < Integer.MAX_VALUE;
+ CommitLogSegment.writeCDCIndexFile(desc, (int)f.length(), true);
+ }
+ }
+
+
/**
* Flushes all keyspaces associated with this replayer in parallel, blocking until their flushes are complete.
* @return the number of mutations replayed
*/
public int blockForWrites()
{
- for (Map.Entry<UUID, AtomicInteger> entry : commitLogReader.getInvalidMutations())
+ for (Map.Entry<TableId, AtomicInteger> entry : commitLogReader.getInvalidMutations())
logger.warn("Skipped {} mutations from unknown (probably removed) CF with id {}", entry.getValue(), entry.getKey());
// wait for all the writes to finish on the mutation stage
@@ -189,7 +240,7 @@
{
public void runMayThrow()
{
- if (Schema.instance.getKSMetaData(mutation.getKeyspaceName()) == null)
+ if (Schema.instance.getKeyspaceMetadata(mutation.getKeyspaceName()) == null)
return;
if (commitLogReplayer.pointInTimeExceeded(mutation))
return;
@@ -201,32 +252,32 @@
// b) have already been flushed,
// or c) are part of a cf that was dropped.
// Keep in mind that the cf.name() is suspect. do every thing based on the cfid instead.
- Mutation newMutation = null;
+ Mutation.PartitionUpdateCollector newPUCollector = null;
for (PartitionUpdate update : commitLogReplayer.replayFilter.filter(mutation))
{
- if (Schema.instance.getCF(update.metadata().cfId) == null)
+ if (Schema.instance.getTableMetadata(update.metadata().id) == null)
continue; // dropped
// replay if current segment is newer than last flushed one or,
// if it is the last known segment, if we are after the commit log segment position
- if (commitLogReplayer.shouldReplay(update.metadata().cfId, new CommitLogPosition(segmentId, entryLocation)))
+ if (commitLogReplayer.shouldReplay(update.metadata().id, new CommitLogPosition(segmentId, entryLocation)))
{
- if (newMutation == null)
- newMutation = new Mutation(mutation.getKeyspaceName(), mutation.key());
- newMutation.add(update);
+ if (newPUCollector == null)
+ newPUCollector = new Mutation.PartitionUpdateCollector(mutation.getKeyspaceName(), mutation.key());
+ newPUCollector.add(update);
commitLogReplayer.replayedCount.incrementAndGet();
}
}
- if (newMutation != null)
+ if (newPUCollector != null)
{
- assert !newMutation.isEmpty();
+ assert !newPUCollector.isEmpty();
- Keyspace.open(newMutation.getKeyspaceName()).apply(newMutation, false, true, false);
+ Keyspace.open(newPUCollector.getKeyspaceName()).apply(newPUCollector.build(), false, true, false);
commitLogReplayer.keyspacesReplayed.add(keyspace);
}
}
};
- return StageManager.getStage(Stage.MUTATION).submit(runnable, serializedSize);
+ return Stage.MUTATION.submit(runnable, serializedSize);
}
}
@@ -269,7 +320,7 @@
{
public abstract Iterable<PartitionUpdate> filter(Mutation mutation);
- public abstract boolean includes(CFMetaData metadata);
+ public abstract boolean includes(TableMetadataRef metadata);
public static ReplayFilter create()
{
@@ -304,7 +355,7 @@
return mutation.getPartitionUpdates();
}
- public boolean includes(CFMetaData metadata)
+ public boolean includes(TableMetadataRef metadata)
{
return true;
}
@@ -329,14 +380,14 @@
{
public boolean apply(PartitionUpdate upd)
{
- return cfNames.contains(upd.metadata().cfName);
+ return cfNames.contains(upd.metadata().name);
}
});
}
- public boolean includes(CFMetaData metadata)
+ public boolean includes(TableMetadataRef metadata)
{
- return toReplay.containsEntry(metadata.ksName, metadata.cfName);
+ return toReplay.containsEntry(metadata.keyspace, metadata.name);
}
}
@@ -346,9 +397,9 @@
*
* @return true iff replay is necessary
*/
- private boolean shouldReplay(UUID cfId, CommitLogPosition position)
+ private boolean shouldReplay(TableId tableId, CommitLogPosition position)
{
- return !cfPersisted.get(cfId).contains(position);
+ return !cfPersisted.get(tableId).contains(position);
}
protected boolean pointInTimeExceeded(Mutation fm)
@@ -365,6 +416,9 @@
public void handleMutation(Mutation m, int size, int entryLocation, CommitLogDescriptor desc)
{
+ if (DatabaseDescriptor.isCDCEnabled() && m.trackedByCDC())
+ sawCDCMutation = true;
+
pendingMutationBytes += size;
futures.offer(mutationInitiator.initiateMutation(m,
desc.id,
diff --git a/src/java/org/apache/cassandra/db/commitlog/CommitLogSegment.java b/src/java/org/apache/cassandra/db/commitlog/CommitLogSegment.java
index 193be91..5303de9 100644
--- a/src/java/org/apache/cassandra/db/commitlog/CommitLogSegment.java
+++ b/src/java/org/apache/cassandra/db/commitlog/CommitLogSegment.java
@@ -18,6 +18,7 @@
package org.apache.cassandra.db.commitlog;
import java.io.File;
+import java.io.FileWriter;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
@@ -28,6 +29,7 @@
import java.util.concurrent.atomic.AtomicInteger;
import java.util.zip.CRC32;
+import com.google.common.annotations.VisibleForTesting;
import org.cliffc.high_scale_lib.NonBlockingHashMap;
import com.codahale.metrics.Timer;
@@ -37,6 +39,9 @@
import org.apache.cassandra.db.partitions.PartitionUpdate;
import org.apache.cassandra.io.FSWriteError;
import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.utils.NativeLibrary;
import org.apache.cassandra.utils.IntegerInterval;
import org.apache.cassandra.utils.concurrent.OpOrder;
@@ -89,7 +94,8 @@
// Everything before this offset has been synced and written. The SYNC_MARKER_SIZE bytes after
// each sync are reserved, and point forwards to the next such offset. The final
// sync marker in a segment will be zeroed out, or point to a position too close to the EOF to fit a marker.
- private volatile int lastSyncedOffset;
+ @VisibleForTesting
+ volatile int lastSyncedOffset;
/**
* Everything before this offset has it's markers written into the {@link #buffer}, but has not necessarily
@@ -106,10 +112,10 @@
private final WaitQueue syncComplete = new WaitQueue();
// a map of Cf->dirty interval in this segment; if interval is not covered by the clean set, the log contains unflushed data
- private final NonBlockingHashMap<UUID, IntegerInterval> cfDirty = new NonBlockingHashMap<>(1024);
+ private final NonBlockingHashMap<TableId, IntegerInterval> tableDirty = new NonBlockingHashMap<>(1024);
// a map of Cf->clean intervals; separate map from above to permit marking Cfs clean whilst the log is still in use
- private final ConcurrentHashMap<UUID, IntegerInterval.Set> cfClean = new ConcurrentHashMap<>();
+ private final ConcurrentHashMap<TableId, IntegerInterval.Set> tableClean = new ConcurrentHashMap<>();
public final long id;
@@ -216,7 +222,10 @@
opGroup.close();
return null;
}
- markDirty(mutation, position);
+
+ for (PartitionUpdate update : mutation.getPartitionUpdates())
+ coverInMap(tableDirty, update.metadata().id, position);
+
return new Allocation(this, opGroup, position, (ByteBuffer) buffer.duplicate().position(position).limit(position + size));
}
catch (Throwable t)
@@ -358,6 +367,8 @@
if (flush || close)
{
flush(startMarker, sectionEnd);
+ if (cdcState == CDCState.CONTAINS)
+ writeCDCIndexFile(descriptor, sectionEnd, close);
lastSyncedOffset = lastMarkerOffset = nextMarker;
if (close)
@@ -368,6 +379,27 @@
}
/**
+ * We persist the offset of the last data synced to disk so clients can parse only durable data if they choose. Data
+ * in shared / memory-mapped buffers reflects un-synced data so we need an external sentinel for clients to read to
+ * determine actual durable data persisted.
+ */
+ public static void writeCDCIndexFile(CommitLogDescriptor desc, int offset, boolean complete)
+ {
+ try(FileWriter writer = new FileWriter(new File(DatabaseDescriptor.getCDCLogLocation(), desc.cdcIndexFileName())))
+ {
+ writer.write(String.valueOf(offset));
+ if (complete)
+ writer.write("\nCOMPLETED");
+ writer.flush();
+ }
+ catch (IOException e)
+ {
+ if (!CommitLog.instance.handleCommitError("Failed to sync CDC Index: " + desc.cdcIndexFileName(), e))
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
* Create a sync marker to delineate sections of the commit log, typically created on each sync of the file.
* The sync marker consists of a file pointer to where the next sync marker should be (effectively declaring the length
* of this section), as well as a CRC value.
@@ -434,6 +466,22 @@
return logFile.getName();
}
+ /**
+ * @return a File object representing the CDC directory and this file name for hard-linking
+ */
+ public File getCDCFile()
+ {
+ return new File(DatabaseDescriptor.getCDCLogLocation(), logFile.getName());
+ }
+
+ /**
+ * @return a File object representing the CDC Index file holding the offset and completion status of this segment
+ */
+ public File getCDCIndexFile()
+ {
+ return new File(DatabaseDescriptor.getCDCLogLocation(), descriptor.cdcIndexFileName());
+ }
+
void waitForFinalSync()
{
while (true)
@@ -504,30 +552,24 @@
i.expandToCover(value);
}
- void markDirty(Mutation mutation, int allocatedPosition)
- {
- for (PartitionUpdate update : mutation.getPartitionUpdates())
- coverInMap(cfDirty, update.metadata().cfId, allocatedPosition);
- }
-
/**
- * Marks the ColumnFamily specified by cfId as clean for this log segment. If the
+ * Marks the ColumnFamily specified by id as clean for this log segment. If the
* given context argument is contained in this file, it will only mark the CF as
* clean if no newer writes have taken place.
*
- * @param cfId the column family ID that is now clean
+ * @param tableId the table that is now clean
* @param startPosition the start of the range that is clean
* @param endPosition the end of the range that is clean
*/
- public synchronized void markClean(UUID cfId, CommitLogPosition startPosition, CommitLogPosition endPosition)
+ public synchronized void markClean(TableId tableId, CommitLogPosition startPosition, CommitLogPosition endPosition)
{
if (startPosition.segmentId > id || endPosition.segmentId < id)
return;
- if (!cfDirty.containsKey(cfId))
+ if (!tableDirty.containsKey(tableId))
return;
int start = startPosition.segmentId == id ? startPosition.position : 0;
int end = endPosition.segmentId == id ? endPosition.position : Integer.MAX_VALUE;
- cfClean.computeIfAbsent(cfId, k -> new IntegerInterval.Set()).add(start, end);
+ tableClean.computeIfAbsent(tableId, k -> new IntegerInterval.Set()).add(start, end);
removeCleanFromDirty();
}
@@ -537,16 +579,16 @@
if (isStillAllocating())
return;
- Iterator<Map.Entry<UUID, IntegerInterval.Set>> iter = cfClean.entrySet().iterator();
+ Iterator<Map.Entry<TableId, IntegerInterval.Set>> iter = tableClean.entrySet().iterator();
while (iter.hasNext())
{
- Map.Entry<UUID, IntegerInterval.Set> clean = iter.next();
- UUID cfId = clean.getKey();
+ Map.Entry<TableId, IntegerInterval.Set> clean = iter.next();
+ TableId tableId = clean.getKey();
IntegerInterval.Set cleanSet = clean.getValue();
- IntegerInterval dirtyInterval = cfDirty.get(cfId);
+ IntegerInterval dirtyInterval = tableDirty.get(tableId);
if (dirtyInterval != null && cleanSet.covers(dirtyInterval))
{
- cfDirty.remove(cfId);
+ tableDirty.remove(tableId);
iter.remove();
}
}
@@ -555,17 +597,17 @@
/**
* @return a collection of dirty CFIDs for this segment file.
*/
- public synchronized Collection<UUID> getDirtyCFIDs()
+ public synchronized Collection<TableId> getDirtyTableIds()
{
- if (cfClean.isEmpty() || cfDirty.isEmpty())
- return cfDirty.keySet();
+ if (tableClean.isEmpty() || tableDirty.isEmpty())
+ return tableDirty.keySet();
- List<UUID> r = new ArrayList<>(cfDirty.size());
- for (Map.Entry<UUID, IntegerInterval> dirty : cfDirty.entrySet())
+ List<TableId> r = new ArrayList<>(tableDirty.size());
+ for (Map.Entry<TableId, IntegerInterval> dirty : tableDirty.entrySet())
{
- UUID cfId = dirty.getKey();
+ TableId tableId = dirty.getKey();
IntegerInterval dirtyInterval = dirty.getValue();
- IntegerInterval.Set cleanSet = cfClean.get(cfId);
+ IntegerInterval.Set cleanSet = tableClean.get(tableId);
if (cleanSet == null || !cleanSet.covers(dirtyInterval))
r.add(dirty.getKey());
}
@@ -578,12 +620,12 @@
public synchronized boolean isUnused()
{
// if room to allocate, we're still in use as the active allocatingFrom,
- // so we don't want to race with updates to cfClean with removeCleanFromDirty
+ // so we don't want to race with updates to tableClean with removeCleanFromDirty
if (isStillAllocating())
return false;
removeCleanFromDirty();
- return cfDirty.isEmpty();
+ return tableDirty.isEmpty();
}
/**
@@ -601,12 +643,12 @@
public String dirtyString()
{
StringBuilder sb = new StringBuilder();
- for (UUID cfId : getDirtyCFIDs())
+ for (TableId tableId : getDirtyTableIds())
{
- CFMetaData m = Schema.instance.getCFMetaData(cfId);
- sb.append(m == null ? "<deleted>" : m.cfName).append(" (").append(cfId)
- .append(", dirty: ").append(cfDirty.get(cfId))
- .append(", clean: ").append(cfClean.get(cfId))
+ TableMetadata m = Schema.instance.getTableMetadata(tableId);
+ sb.append(m == null ? "<deleted>" : m.name).append(" (").append(tableId)
+ .append(", dirty: ").append(tableDirty.get(tableId))
+ .append(", clean: ").append(tableClean.get(tableId))
.append("), ");
}
return sb.toString();
@@ -652,6 +694,7 @@
// Also synchronized in CDCSizeTracker.processNewSegment and .processDiscardedSegment
synchronized(cdcStateLock)
{
+ // Need duplicate CONTAINS to be idempotent since 2 threads can race on this lock
if (cdcState == CDCState.CONTAINS && newState != CDCState.CONTAINS)
throw new IllegalArgumentException("Cannot transition from CONTAINS to any other state.");
@@ -702,6 +745,9 @@
segment.waitForSync(position, waitingOnCommit);
}
+ /**
+ * Returns the position in the CommitLogSegment at the end of this allocation.
+ */
public CommitLogPosition getCommitLogPosition()
{
return new CommitLogPosition(segment.id, buffer.limit());
diff --git a/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManagerCDC.java b/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManagerCDC.java
index a91384f..66c8a39 100644
--- a/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManagerCDC.java
+++ b/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManagerCDC.java
@@ -32,9 +32,9 @@
import org.slf4j.LoggerFactory;
import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.db.*;
+import org.apache.cassandra.db.Mutation;
import org.apache.cassandra.db.commitlog.CommitLogSegment.CDCState;
-import org.apache.cassandra.exceptions.WriteTimeoutException;
+import org.apache.cassandra.exceptions.CDCWriteException;
import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.utils.DirectorySizeCalculator;
import org.apache.cassandra.utils.NoSpamLogger;
@@ -64,12 +64,20 @@
cdcSizeTracker.processDiscardedSegment(segment);
- if (segment.getCDCState() == CDCState.CONTAINS)
- FileUtils.renameWithConfirm(segment.logFile.getAbsolutePath(), DatabaseDescriptor.getCDCLogLocation() + File.separator + segment.logFile.getName());
- else
+ if (delete)
+ FileUtils.deleteWithConfirm(segment.logFile);
+
+ if (segment.getCDCState() != CDCState.CONTAINS)
{
- if (delete)
- FileUtils.deleteWithConfirm(segment.logFile);
+ // Always delete hard-link from cdc folder if this segment didn't contain CDC data. Note: File may not exist
+ // if processing discard during startup.
+ File cdcLink = segment.getCDCFile();
+ if (cdcLink.exists())
+ FileUtils.deleteWithConfirm(cdcLink);
+
+ File cdcIndexFile = segment.getCDCIndexFile();
+ if (cdcIndexFile.exists())
+ FileUtils.deleteWithConfirm(cdcIndexFile);
}
}
@@ -89,10 +97,10 @@
* @param mutation Mutation to allocate in segment manager
* @param size total size (overhead + serialized) of mutation
* @return the created Allocation object
- * @throws WriteTimeoutException If segment disallows CDC mutations, we throw WTE
+ * @throws CDCWriteException If segment disallows CDC mutations, we throw
*/
@Override
- public CommitLogSegment.Allocation allocate(Mutation mutation, int size) throws WriteTimeoutException
+ public CommitLogSegment.Allocation allocate(Mutation mutation, int size) throws CDCWriteException
{
CommitLogSegment segment = allocatingFrom();
CommitLogSegment.Allocation alloc;
@@ -113,44 +121,66 @@
return alloc;
}
- private void throwIfForbidden(Mutation mutation, CommitLogSegment segment) throws WriteTimeoutException
+ private void throwIfForbidden(Mutation mutation, CommitLogSegment segment) throws CDCWriteException
{
if (mutation.trackedByCDC() && segment.getCDCState() == CDCState.FORBIDDEN)
{
cdcSizeTracker.submitOverflowSizeRecalculation();
+ String logMsg = String.format("Rejecting mutation to keyspace %s. Free up space in %s by processing CDC logs.",
+ mutation.getKeyspaceName(), DatabaseDescriptor.getCDCLogLocation());
NoSpamLogger.log(logger,
NoSpamLogger.Level.WARN,
10,
TimeUnit.SECONDS,
- "Rejecting Mutation containing CDC-enabled table. Free up space in {}.",
- DatabaseDescriptor.getCDCLogLocation());
- throw new WriteTimeoutException(WriteType.CDC, ConsistencyLevel.LOCAL_ONE, 0, 1);
+ logMsg);
+ throw new CDCWriteException(logMsg);
}
}
/**
- * Move files to cdc_raw after replay, since recovery will flush to SSTable and these mutations won't be available
- * in the CL subsystem otherwise.
- */
- void handleReplayedSegment(final File file)
- {
- logger.trace("Moving (Unopened) segment {} to cdc_raw directory after replay", file);
- FileUtils.renameWithConfirm(file.getAbsolutePath(), DatabaseDescriptor.getCDCLogLocation() + File.separator + file.getName());
- cdcSizeTracker.addFlushedSize(file.length());
- }
-
- /**
* On segment creation, flag whether the segment should accept CDC mutations or not based on the total currently
* allocated unflushed CDC segments and the contents of cdc_raw
*/
public CommitLogSegment createSegment()
{
CommitLogSegment segment = CommitLogSegment.createSegment(commitLog, this);
+
+ // Hard link file in cdc folder for realtime tracking
+ FileUtils.createHardLink(segment.logFile, segment.getCDCFile());
+
cdcSizeTracker.processNewSegment(segment);
return segment;
}
/**
+ * Delete untracked segment files after replay
+ *
+ * @param file segment file that is no longer in use.
+ */
+ @Override
+ void handleReplayedSegment(final File file)
+ {
+ super.handleReplayedSegment(file);
+
+ // delete untracked cdc segment hard link files if their index files do not exist
+ File cdcFile = new File(DatabaseDescriptor.getCDCLogLocation(), file.getName());
+ File cdcIndexFile = new File(DatabaseDescriptor.getCDCLogLocation(), CommitLogDescriptor.fromFileName(file.getName()).cdcIndexFileName());
+ if (cdcFile.exists() && !cdcIndexFile.exists())
+ {
+ logger.trace("(Unopened) CDC segment {} is no longer needed and will be deleted now", cdcFile);
+ FileUtils.deleteWithConfirm(cdcFile);
+ }
+ }
+
+ /**
+ * For use after replay when replayer hard-links / adds tracking of replayed segments
+ */
+ public void addCDCSize(long size)
+ {
+ cdcSizeTracker.addSize(size);
+ }
+
+ /**
* Tracks total disk usage of CDC subsystem, defined by the summation of all unflushed CommitLogSegments with CDC
* data in them and all segments archived into cdc_raw.
*
@@ -162,7 +192,6 @@
private final RateLimiter rateLimiter = RateLimiter.create(1000.0 / DatabaseDescriptor.getCDCDiskCheckInterval());
private ExecutorService cdcSizeCalculationExecutor;
private CommitLogSegmentManagerCDC segmentManager;
- private volatile long unflushedCDCSize;
// Used instead of size during walk to remove chance of over-allocation
private volatile long sizeInProgress = 0;
@@ -179,7 +208,6 @@
public void start()
{
size = 0;
- unflushedCDCSize = 0;
cdcSizeCalculationExecutor = new ThreadPoolExecutor(1, 1, 1000, TimeUnit.SECONDS, new SynchronousQueue<>(), new ThreadPoolExecutor.DiscardPolicy());
}
@@ -202,7 +230,7 @@
? CDCState.FORBIDDEN
: CDCState.PERMITTED);
if (segment.getCDCState() == CDCState.PERMITTED)
- unflushedCDCSize += defaultSegmentSize();
+ size += defaultSegmentSize();
}
// Take this opportunity to kick off a recalc to pick up any consumer file deletion.
@@ -218,7 +246,7 @@
if (segment.getCDCState() == CDCState.CONTAINS)
size += segment.onDiskSize();
if (segment.getCDCState() != CDCState.FORBIDDEN)
- unflushedCDCSize -= defaultSegmentSize();
+ size -= defaultSegmentSize();
}
// Take this opportunity to kick off a recalc to pick up any consumer file deletion.
@@ -278,19 +306,23 @@
return FileVisitResult.CONTINUE;
}
- private void addFlushedSize(long toAdd)
+
+ public void shutdown()
+ {
+ if (cdcSizeCalculationExecutor != null && !cdcSizeCalculationExecutor.isShutdown())
+ {
+ cdcSizeCalculationExecutor.shutdown();
+ }
+ }
+
+ private void addSize(long toAdd)
{
size += toAdd;
}
private long totalCDCSizeOnDisk()
{
- return unflushedCDCSize + size;
- }
-
- public void shutdown()
- {
- cdcSizeCalculationExecutor.shutdown();
+ return size;
}
}
diff --git a/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManagerStandard.java b/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManagerStandard.java
index 86e886b..b9bd744 100644
--- a/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManagerStandard.java
+++ b/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManagerStandard.java
@@ -61,19 +61,7 @@
return alloc;
}
- /**
- * Simply delete untracked segment files w/standard, as it'll be flushed to sstables during recovery
- *
- * @param file segment file that is no longer in use.
- */
- void handleReplayedSegment(final File file)
- {
- // (don't decrease managed size, since this was never a "live" segment)
- logger.trace("(Unopened) segment {} is no longer needed and will be deleted now", file);
- FileUtils.deleteWithConfirm(file);
- }
-
- public CommitLogSegment createSegment()
+ public CommitLogSegment createSegment()
{
return CommitLogSegment.createSegment(commitLog, this);
}
diff --git a/src/java/org/apache/cassandra/db/commitlog/GroupCommitLogService.java b/src/java/org/apache/cassandra/db/commitlog/GroupCommitLogService.java
new file mode 100644
index 0000000..a76923e
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/commitlog/GroupCommitLogService.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.commitlog;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+
+/**
+ * A commitlog service that will block returning an ACK back to the a coordinator/client
+ * for a minimum amount of time as we wait until the the commit log segment is flushed.
+ */
+public class GroupCommitLogService extends AbstractCommitLogService
+{
+ public GroupCommitLogService(CommitLog commitLog)
+ {
+ super(commitLog, "GROUP-COMMIT-LOG-WRITER", (int) DatabaseDescriptor.getCommitLogSyncGroupWindow());
+ }
+
+ protected void maybeWaitForSync(CommitLogSegment.Allocation alloc)
+ {
+ // wait until record has been safely persisted to disk
+ pending.incrementAndGet();
+ // wait for commitlog_sync_group_window_in_ms
+ alloc.awaitDiskSync(commitLog.metrics.waitingOnCommit);
+ pending.decrementAndGet();
+ }
+}
+
diff --git a/src/java/org/apache/cassandra/db/commitlog/IntervalSet.java b/src/java/org/apache/cassandra/db/commitlog/IntervalSet.java
index 371c646..45db2f6 100644
--- a/src/java/org/apache/cassandra/db/commitlog/IntervalSet.java
+++ b/src/java/org/apache/cassandra/db/commitlog/IntervalSet.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.cassandra.db.commitlog;
import java.io.IOException;
diff --git a/src/java/org/apache/cassandra/db/commitlog/MemoryMappedSegment.java b/src/java/org/apache/cassandra/db/commitlog/MemoryMappedSegment.java
index e79278d..6ecdbd3 100644
--- a/src/java/org/apache/cassandra/db/commitlog/MemoryMappedSegment.java
+++ b/src/java/org/apache/cassandra/db/commitlog/MemoryMappedSegment.java
@@ -102,8 +102,7 @@
@Override
protected void internalClose()
{
- if (FileUtils.isCleanerAvailable)
- FileUtils.clean(buffer);
+ FileUtils.clean(buffer);
super.internalClose();
}
}
diff --git a/src/java/org/apache/cassandra/db/commitlog/PeriodicCommitLogService.java b/src/java/org/apache/cassandra/db/commitlog/PeriodicCommitLogService.java
index efd3394..e94c616 100644
--- a/src/java/org/apache/cassandra/db/commitlog/PeriodicCommitLogService.java
+++ b/src/java/org/apache/cassandra/db/commitlog/PeriodicCommitLogService.java
@@ -17,11 +17,13 @@
*/
package org.apache.cassandra.db.commitlog;
+import java.util.concurrent.TimeUnit;
+
import org.apache.cassandra.config.DatabaseDescriptor;
class PeriodicCommitLogService extends AbstractCommitLogService
{
- private static final long blockWhenSyncLagsNanos = (long) (DatabaseDescriptor.getCommitLogSyncPeriod() * 1.5e6);
+ private static final long blockWhenSyncLagsNanos = TimeUnit.MILLISECONDS.toNanos(DatabaseDescriptor.getPeriodicCommitLogSyncBlock());
public PeriodicCommitLogService(final CommitLog commitLog)
{
diff --git a/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java b/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java
index 5d43143..1e72ae5 100644
--- a/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java
+++ b/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java
@@ -19,10 +19,8 @@
import java.util.*;
-import com.google.common.base.Throwables;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableMap;
-import com.google.common.base.Predicate;
-import com.google.common.collect.Iterables;
import org.apache.cassandra.db.Directories;
import org.apache.cassandra.db.SerializationHeader;
@@ -36,7 +34,6 @@
import org.slf4j.LoggerFactory;
import org.apache.cassandra.db.ColumnFamilyStore;
-import org.apache.cassandra.db.Memtable;
import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
@@ -44,8 +41,8 @@
import org.apache.cassandra.io.sstable.Component;
import org.apache.cassandra.io.sstable.ISSTableScanner;
import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
+import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
import org.apache.cassandra.schema.CompactionParams;
-import org.apache.cassandra.utils.JVMStabilityInspector;
/**
* Pluggable compaction strategy determines how SSTables get merged.
@@ -115,8 +112,6 @@
uncheckedTombstoneCompaction = optionValue == null ? DEFAULT_UNCHECKED_TOMBSTONE_COMPACTION_OPTION : Boolean.parseBoolean(optionValue);
optionValue = options.get(LOG_ALL_OPTION);
logAll = optionValue == null ? DEFAULT_LOG_ALL_OPTION : Boolean.parseBoolean(optionValue);
- if (!shouldBeEnabled())
- this.disable();
}
catch (ConfigurationException e)
{
@@ -213,45 +208,6 @@
*/
public abstract long getMaxSSTableBytes();
- public void enable()
- {
- }
-
- public void disable()
- {
- }
-
- /**
- * @return whether or not MeteredFlusher should be able to trigger memtable flushes for this CF.
- */
- public boolean isAffectedByMeteredFlusher()
- {
- return true;
- }
-
- /**
- * If not affected by MeteredFlusher (and handling flushing on its own), override to tell MF how much
- * space to reserve for this CF, i.e., how much space to subtract from `memtable_total_space_in_mb` when deciding
- * if other memtables should be flushed or not.
- */
- public long getMemtableReservedSize()
- {
- return 0;
- }
-
- /**
- * Handle a flushed memtable.
- *
- * @param memtable the flushed memtable
- * @param sstables the written sstables. can be null or empty if the memtable was clean.
- */
- public void replaceFlushed(Memtable memtable, Collection<SSTableReader> sstables)
- {
- cfs.getTracker().replaceFlushed(memtable, sstables);
- if (sstables != null && !sstables.isEmpty())
- CompactionManager.instance.submitBackground(cfs);
- }
-
/**
* Filters SSTables that are to be excluded from the given collection
*
@@ -287,19 +243,11 @@
try
{
for (SSTableReader sstable : sstables)
- scanners.add(sstable.getScanner(ranges, null));
+ scanners.add(sstable.getScanner(ranges));
}
catch (Throwable t)
{
- try
- {
- new ScannerList(scanners).close();
- }
- catch (Throwable t2)
- {
- t.addSuppressed(t2);
- }
- throw t;
+ ISSTableScanner.closeAllAndPropagate(scanners, t);
}
return new ScannerList(scanners);
}
@@ -332,6 +280,31 @@
public abstract void removeSSTable(SSTableReader sstable);
+ public void removeSSTables(Iterable<SSTableReader> removed)
+ {
+ for (SSTableReader sstable : removed)
+ removeSSTable(sstable);
+ }
+
+ /**
+ * Returns the sstables managed by this strategy instance
+ */
+ @VisibleForTesting
+ protected abstract Set<SSTableReader> getSSTables();
+
+ /**
+ * Called when the metadata has changed for an sstable - for example if the level changed
+ *
+ * Not called when repair status changes (which is also metadata), because this results in the
+ * sstable getting removed from the compaction strategy instance.
+ *
+ * @param oldMetadata
+ * @param sstable
+ */
+ public void metadataChanged(StatsMetadata oldMetadata, SSTableReader sstable)
+ {
+ }
+
public static class ScannerList implements AutoCloseable
{
public final List<ISSTableScanner> scanners;
@@ -343,8 +316,8 @@
public long getTotalBytesScanned()
{
long bytesScanned = 0L;
- for (ISSTableScanner scanner : scanners)
- bytesScanned += scanner.getBytesScanned();
+ for (int i=0, isize=scanners.size(); i<isize; i++)
+ bytesScanned += scanners.get(i).getBytesScanned();
return bytesScanned;
}
@@ -352,8 +325,8 @@
public long getTotalCompressedSize()
{
long compressedSize = 0;
- for (ISSTableScanner scanner : scanners)
- compressedSize += scanner.getCompressedLengthInBytes();
+ for (int i=0, isize=scanners.size(); i<isize; i++)
+ compressedSize += scanners.get(i).getCompressedLengthInBytes();
return compressedSize;
}
@@ -363,8 +336,10 @@
double compressed = 0.0;
double uncompressed = 0.0;
- for (ISSTableScanner scanner : scanners)
+ for (int i=0, isize=scanners.size(); i<isize; i++)
{
+ @SuppressWarnings("resource")
+ ISSTableScanner scanner = scanners.get(i);
compressed += scanner.getCompressedLengthInBytes();
uncompressed += scanner.getLengthInBytes();
}
@@ -377,24 +352,7 @@
public void close()
{
- Throwable t = null;
- for (ISSTableScanner scanner : scanners)
- {
- try
- {
- scanner.close();
- }
- catch (Throwable t2)
- {
- JVMStabilityInspector.inspectThrowable(t2);
- if (t == null)
- t = t2;
- else
- t.addSuppressed(t2);
- }
- }
- if (t != null)
- throw Throwables.propagate(t);
+ ISSTableScanner.closeAllAndPropagate(scanners, null);
}
}
@@ -413,7 +371,7 @@
*/
protected boolean worthDroppingTombstones(SSTableReader sstable, int gcBefore)
{
- if (disableTombstoneCompactions || CompactionController.NEVER_PURGE_TOMBSTONES)
+ if (disableTombstoneCompactions || CompactionController.NEVER_PURGE_TOMBSTONES || cfs.getNeverPurgeTombstones())
return false;
// since we use estimations to calculate, there is a chance that compaction will not drop tombstones actually.
// if that happens we will end up in infinite compaction loop, so first we check enough if enough time has
@@ -454,8 +412,8 @@
ranges.add(new Range<>(overlap.first.getToken(), overlap.last.getToken()));
long remainingKeys = keys - sstable.estimatedKeysForRanges(ranges);
// next, calculate what percentage of columns we have within those keys
- long columns = sstable.getEstimatedColumnCount().mean() * remainingKeys;
- double remainingColumnsRatio = ((double) columns) / (sstable.getEstimatedColumnCount().count() * sstable.getEstimatedColumnCount().mean());
+ long columns = sstable.getEstimatedCellPerPartitionCount().mean() * remainingKeys;
+ double remainingColumnsRatio = ((double) columns) / (sstable.getEstimatedCellPerPartitionCount().count() * sstable.getEstimatedCellPerPartitionCount().mean());
// return if we still expect to have droppable tombstones in rest of columns
return remainingColumnsRatio * droppableRatio > tombstoneThreshold;
@@ -534,14 +492,6 @@
return uncheckedOptions;
}
- public boolean shouldBeEnabled()
- {
- String optionValue = options.get(COMPACTION_ENABLED);
-
- return optionValue == null || Boolean.parseBoolean(optionValue);
- }
-
-
/**
* Method for grouping similar SSTables together, This will be used by
* anti-compaction to determine which SSTables should be anitcompacted
@@ -555,7 +505,7 @@
Collections.sort(sortedSSTablesToGroup, SSTableReader.sstableComparator);
Collection<Collection<SSTableReader>> groupedSSTables = new ArrayList<>();
- Collection<SSTableReader> currGroup = new ArrayList<>();
+ Collection<SSTableReader> currGroup = new ArrayList<>(groupSize);
for (SSTableReader sstable : sortedSSTablesToGroup)
{
@@ -563,7 +513,7 @@
if (currGroup.size() == groupSize)
{
groupedSSTables.add(currGroup);
- currGroup = new ArrayList<>();
+ currGroup = new ArrayList<>(groupSize);
}
}
@@ -580,12 +530,14 @@
public SSTableMultiWriter createSSTableMultiWriter(Descriptor descriptor,
long keyCount,
long repairedAt,
+ UUID pendingRepair,
+ boolean isTransient,
MetadataCollector meta,
SerializationHeader header,
Collection<Index> indexes,
LifecycleNewTracker lifecycleNewTracker)
{
- return SimpleSSTableMultiWriter.create(descriptor, keyCount, repairedAt, cfs.metadata, meta, header, indexes, lifecycleNewTracker);
+ return SimpleSSTableMultiWriter.create(descriptor, keyCount, repairedAt, pendingRepair, isTransient, cfs.metadata, meta, header, indexes, lifecycleNewTracker);
}
public boolean supportsEarlyOpen()
diff --git a/src/java/org/apache/cassandra/db/compaction/AbstractCompactionTask.java b/src/java/org/apache/cassandra/db/compaction/AbstractCompactionTask.java
index 430c916..989c21c 100644
--- a/src/java/org/apache/cassandra/db/compaction/AbstractCompactionTask.java
+++ b/src/java/org/apache/cassandra/db/compaction/AbstractCompactionTask.java
@@ -17,11 +17,14 @@
*/
package org.apache.cassandra.db.compaction;
+import java.util.Iterator;
import java.util.Set;
+import java.util.UUID;
+
+import com.google.common.base.Preconditions;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Directories;
-import org.apache.cassandra.db.compaction.CompactionManager.CompactionExecutorStatsCollector;
import org.apache.cassandra.db.compaction.writers.CompactionAwareWriter;
import org.apache.cassandra.io.FSDiskFullWriteError;
import org.apache.cassandra.io.sstable.format.SSTableReader;
@@ -49,16 +52,52 @@
Set<SSTableReader> compacting = transaction.tracker.getCompacting();
for (SSTableReader sstable : transaction.originals())
assert compacting.contains(sstable) : sstable.getFilename() + " is not correctly marked compacting";
+
+ validateSSTables(transaction.originals());
+ }
+
+ /**
+ * Confirm that we're not attempting to compact repaired/unrepaired/pending repair sstables together
+ */
+ private void validateSSTables(Set<SSTableReader> sstables)
+ {
+ // do not allow to be compacted together
+ if (!sstables.isEmpty())
+ {
+ Iterator<SSTableReader> iter = sstables.iterator();
+ SSTableReader first = iter.next();
+ boolean isRepaired = first.isRepaired();
+ UUID pendingRepair = first.getPendingRepair();
+ while (iter.hasNext())
+ {
+ SSTableReader next = iter.next();
+ Preconditions.checkArgument(isRepaired == next.isRepaired(),
+ "Cannot compact repaired and unrepaired sstables");
+
+ if (pendingRepair == null)
+ {
+ Preconditions.checkArgument(!next.isPendingRepair(),
+ "Cannot compact pending repair and non-pending repair sstables");
+ }
+ else
+ {
+ Preconditions.checkArgument(next.isPendingRepair(),
+ "Cannot compact pending repair and non-pending repair sstables");
+ Preconditions.checkArgument(pendingRepair.equals(next.getPendingRepair()),
+ "Cannot compact sstables from different pending repairs");
+ }
+ }
+ }
}
/**
* executes the task and unmarks sstables compacting
*/
- public int execute(CompactionExecutorStatsCollector collector)
+ public int execute(ActiveCompactionsTracker activeCompactions)
{
try
{
- return executeInternal(collector);
+ return executeInternal(activeCompactions);
}
catch(FSDiskFullWriteError e)
{
@@ -73,7 +112,7 @@
}
public abstract CompactionAwareWriter getCompactionAwareWriter(ColumnFamilyStore cfs, Directories directories, LifecycleTransaction txn, Set<SSTableReader> nonExpiredSSTables);
- protected abstract int executeInternal(CompactionExecutorStatsCollector collector);
+ protected abstract int executeInternal(ActiveCompactionsTracker activeCompactions);
public AbstractCompactionTask setUserDefined(boolean isUserDefined)
{
diff --git a/src/java/org/apache/cassandra/db/compaction/AbstractStrategyHolder.java b/src/java/org/apache/cassandra/db/compaction/AbstractStrategyHolder.java
new file mode 100644
index 0000000..63b7909
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/compaction/AbstractStrategyHolder.java
@@ -0,0 +1,210 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.UUID;
+import java.util.function.Supplier;
+
+import com.google.common.base.Preconditions;
+
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.SerializationHeader;
+import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.index.Index;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.ISSTableScanner;
+import org.apache.cassandra.io.sstable.SSTableMultiWriter;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
+import org.apache.cassandra.schema.CompactionParams;
+
+/**
+ * Wrapper that's aware of how sstables are divided between separate strategies,
+ * and provides a standard interface to them
+ *
+ * not threadsafe, calls must be synchronized by caller
+ */
+public abstract class AbstractStrategyHolder
+{
+ public static class TaskSupplier implements Comparable<TaskSupplier>
+ {
+ private final int numRemaining;
+ private final Supplier<AbstractCompactionTask> supplier;
+
+ TaskSupplier(int numRemaining, Supplier<AbstractCompactionTask> supplier)
+ {
+ this.numRemaining = numRemaining;
+ this.supplier = supplier;
+ }
+
+ public AbstractCompactionTask getTask()
+ {
+ return supplier.get();
+ }
+
+ public int compareTo(TaskSupplier o)
+ {
+ return o.numRemaining - numRemaining;
+ }
+ }
+
+ public static interface DestinationRouter
+ {
+ int getIndexForSSTable(SSTableReader sstable);
+ int getIndexForSSTableDirectory(Descriptor descriptor);
+ }
+
+ /**
+ * Maps sstables to their token partition bucket
+ */
+ static class GroupedSSTableContainer
+ {
+ private final AbstractStrategyHolder holder;
+ private final Set<SSTableReader>[] groups;
+
+ private GroupedSSTableContainer(AbstractStrategyHolder holder)
+ {
+ this.holder = holder;
+ Preconditions.checkArgument(holder.numTokenPartitions > 0, "numTokenPartitions not set");
+ groups = new Set[holder.numTokenPartitions];
+ }
+
+ void add(SSTableReader sstable)
+ {
+ Preconditions.checkArgument(holder.managesSSTable(sstable), "this strategy holder doesn't manage %s", sstable);
+ int idx = holder.router.getIndexForSSTable(sstable);
+ Preconditions.checkState(idx >= 0 && idx < holder.numTokenPartitions, "Invalid sstable index (%s) for %s", idx, sstable);
+ if (groups[idx] == null)
+ groups[idx] = new HashSet<>();
+ groups[idx].add(sstable);
+ }
+
+ int numGroups()
+ {
+ return groups.length;
+ }
+
+ Set<SSTableReader> getGroup(int i)
+ {
+ Preconditions.checkArgument(i >= 0 && i < groups.length);
+ Set<SSTableReader> group = groups[i];
+ return group != null ? group : Collections.emptySet();
+ }
+
+ boolean isGroupEmpty(int i)
+ {
+ return getGroup(i).isEmpty();
+ }
+
+ boolean isEmpty()
+ {
+ for (int i = 0; i < groups.length; i++)
+ if (!isGroupEmpty(i))
+ return false;
+ return true;
+ }
+ }
+
+ protected final ColumnFamilyStore cfs;
+ final DestinationRouter router;
+ private int numTokenPartitions = -1;
+
+ AbstractStrategyHolder(ColumnFamilyStore cfs, DestinationRouter router)
+ {
+ this.cfs = cfs;
+ this.router = router;
+ }
+
+ public abstract void startup();
+
+ public abstract void shutdown();
+
+ final void setStrategy(CompactionParams params, int numTokenPartitions)
+ {
+ Preconditions.checkArgument(numTokenPartitions > 0, "at least one token partition required");
+ shutdown();
+ this.numTokenPartitions = numTokenPartitions;
+ setStrategyInternal(params, numTokenPartitions);
+ }
+
+ protected abstract void setStrategyInternal(CompactionParams params, int numTokenPartitions);
+
+ /**
+ * SSTables are grouped by their repaired and pending repair status. This method determines if this holder
+ * holds the sstable for the given repaired/grouped statuses. Holders should be mutually exclusive in the
+ * groups they deal with. IOW, if one holder returns true for a given isRepaired/isPendingRepair combo,
+ * none of the others should.
+ */
+ public abstract boolean managesRepairedGroup(boolean isRepaired, boolean isPendingRepair, boolean isTransient);
+
+ public boolean managesSSTable(SSTableReader sstable)
+ {
+ return managesRepairedGroup(sstable.isRepaired(), sstable.isPendingRepair(), sstable.isTransient());
+ }
+
+ public abstract AbstractCompactionStrategy getStrategyFor(SSTableReader sstable);
+
+ public abstract Iterable<AbstractCompactionStrategy> allStrategies();
+
+ public abstract Collection<TaskSupplier> getBackgroundTaskSuppliers(int gcBefore);
+
+ public abstract Collection<AbstractCompactionTask> getMaximalTasks(int gcBefore, boolean splitOutput);
+
+ public abstract Collection<AbstractCompactionTask> getUserDefinedTasks(GroupedSSTableContainer sstables, int gcBefore);
+
+ public GroupedSSTableContainer createGroupedSSTableContainer()
+ {
+ return new GroupedSSTableContainer(this);
+ }
+
+ public abstract void addSSTables(GroupedSSTableContainer sstables);
+
+ public abstract void removeSSTables(GroupedSSTableContainer sstables);
+
+ public abstract void replaceSSTables(GroupedSSTableContainer removed, GroupedSSTableContainer added);
+
+ public abstract List<ISSTableScanner> getScanners(GroupedSSTableContainer sstables, Collection<Range<Token>> ranges);
+
+
+ public abstract SSTableMultiWriter createSSTableMultiWriter(Descriptor descriptor,
+ long keyCount,
+ long repairedAt,
+ UUID pendingRepair,
+ boolean isTransient,
+ MetadataCollector collector,
+ SerializationHeader header,
+ Collection<Index> indexes,
+ LifecycleNewTracker lifecycleNewTracker);
+
+ /**
+ * Return the directory index the given compaction strategy belongs to, or -1
+ * if it's not held by this holder
+ */
+ public abstract int getStrategyIndex(AbstractCompactionStrategy strategy);
+
+ public abstract boolean containsSSTable(SSTableReader sstable);
+}
diff --git a/src/java/org/apache/cassandra/db/compaction/ActiveCompactions.java b/src/java/org/apache/cassandra/db/compaction/ActiveCompactions.java
new file mode 100644
index 0000000..5bcb06f
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/compaction/ActiveCompactions.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.IdentityHashMap;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+
+public class ActiveCompactions implements ActiveCompactionsTracker
+{
+ // a synchronized identity set of running tasks to their compaction info
+ private final Set<CompactionInfo.Holder> compactions = Collections.synchronizedSet(Collections.newSetFromMap(new IdentityHashMap<>()));
+
+ public List<CompactionInfo.Holder> getCompactions()
+ {
+ return new ArrayList<>(compactions);
+ }
+
+ public void beginCompaction(CompactionInfo.Holder ci)
+ {
+ compactions.add(ci);
+ }
+
+ public void finishCompaction(CompactionInfo.Holder ci)
+ {
+ compactions.remove(ci);
+ CompactionManager.instance.getMetrics().bytesCompacted.inc(ci.getCompactionInfo().getTotal());
+ CompactionManager.instance.getMetrics().totalCompactionsCompleted.mark();
+ }
+
+ /**
+ * Iterates over the active compactions and tries to find the CompactionInfo for the given sstable
+ *
+ * Number of entries in compactions should be small (< 10) but avoid calling in any time-sensitive context
+ */
+ public CompactionInfo getCompactionForSSTable(SSTableReader sstable)
+ {
+ CompactionInfo toReturn = null;
+ synchronized (compactions)
+ {
+ for (CompactionInfo.Holder holder : compactions)
+ {
+ if (holder.getCompactionInfo().getSSTables().contains(sstable))
+ {
+ if (toReturn != null)
+ throw new IllegalStateException("SSTable " + sstable + " involved in several compactions");
+ toReturn = holder.getCompactionInfo();
+ }
+ }
+ }
+ return toReturn;
+ }
+}
diff --git a/src/java/org/apache/cassandra/db/compaction/ActiveCompactionsTracker.java b/src/java/org/apache/cassandra/db/compaction/ActiveCompactionsTracker.java
new file mode 100644
index 0000000..c1bbbd8
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/compaction/ActiveCompactionsTracker.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+public interface ActiveCompactionsTracker
+{
+ public void beginCompaction(CompactionInfo.Holder ci);
+ public void finishCompaction(CompactionInfo.Holder ci);
+
+ public static final ActiveCompactionsTracker NOOP = new ActiveCompactionsTracker()
+ {
+ public void beginCompaction(CompactionInfo.Holder ci)
+ {}
+
+ public void finishCompaction(CompactionInfo.Holder ci)
+ {}
+ };
+}
diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionController.java b/src/java/org/apache/cassandra/db/compaction/CompactionController.java
index 84aac09..e1b0f32 100644
--- a/src/java/org/apache/cassandra/db/compaction/CompactionController.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionController.java
@@ -18,26 +18,22 @@
package org.apache.cassandra.db.compaction;
import java.util.*;
-import java.util.function.Predicate;
-
-import org.apache.cassandra.config.Config;
-import org.apache.cassandra.db.Memtable;
-import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import java.util.function.LongPredicate;
import com.google.common.base.Predicates;
import com.google.common.collect.Iterables;
import com.google.common.util.concurrent.RateLimiter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.apache.cassandra.config.Config;
+import org.apache.cassandra.db.*;
import org.apache.cassandra.db.partitions.Partition;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.util.FileDataInput;
import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.schema.CompactionParams.TombstoneOption;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.cassandra.db.*;
import org.apache.cassandra.utils.AlwaysPresentFilter;
import org.apache.cassandra.utils.OverlapIterator;
import org.apache.cassandra.utils.concurrent.Refs;
@@ -47,13 +43,12 @@
/**
* Manage compaction options.
*/
-public class CompactionController implements AutoCloseable
+public class CompactionController extends AbstractCompactionController
{
private static final Logger logger = LoggerFactory.getLogger(CompactionController.class);
private static final String NEVER_PURGE_TOMBSTONES_PROPERTY = Config.PROPERTY_PREFIX + "never_purge_tombstones";
static final boolean NEVER_PURGE_TOMBSTONES = Boolean.getBoolean(NEVER_PURGE_TOMBSTONES_PROPERTY);
- public final ColumnFamilyStore cfs;
private final boolean compactingRepaired;
// note that overlapIterator and overlappingSSTables will be null if NEVER_PURGE_TOMBSTONES is set - this is a
// good thing so that noone starts using them and thinks that if overlappingSSTables is empty, there
@@ -63,11 +58,8 @@
private final Iterable<SSTableReader> compacting;
private final RateLimiter limiter;
private final long minTimestamp;
- final TombstoneOption tombstoneOption;
final Map<SSTableReader, FileDataInput> openDataFiles = new HashMap<>();
- public final int gcBefore;
-
protected CompactionController(ColumnFamilyStore cfs, int maxValue)
{
this(cfs, null, maxValue);
@@ -81,13 +73,10 @@
public CompactionController(ColumnFamilyStore cfs, Set<SSTableReader> compacting, int gcBefore, RateLimiter limiter, TombstoneOption tombstoneOption)
{
- assert cfs != null;
- this.cfs = cfs;
- this.gcBefore = gcBefore;
+ super(cfs, gcBefore, tombstoneOption);
this.compacting = compacting;
this.limiter = limiter;
compactingRepaired = compacting != null && compacting.stream().allMatch(SSTableReader::isRepaired);
- this.tombstoneOption = tombstoneOption;
this.minTimestamp = compacting != null && !compacting.isEmpty() // check needed for test
? compacting.stream().mapToLong(SSTableReader::getMinTimestamp).min().getAsLong()
: 0;
@@ -111,6 +100,12 @@
return;
}
+ if (cfs.getNeverPurgeTombstones())
+ {
+ logger.debug("not refreshing overlaps for {}.{} - neverPurgeTombstones is enabled", cfs.keyspace.getName(), cfs.getTableName());
+ return;
+ }
+
for (SSTableReader reader : overlappingSSTables)
{
if (reader.isMarkedCompacted())
@@ -123,7 +118,7 @@
private void refreshOverlaps()
{
- if (NEVER_PURGE_TOMBSTONES)
+ if (NEVER_PURGE_TOMBSTONES || cfs.getNeverPurgeTombstones())
return;
if (this.overlappingSSTables != null)
@@ -166,8 +161,8 @@
{
logger.trace("Checking droppable sstables in {}", cfStore);
- if (NEVER_PURGE_TOMBSTONES || compacting == null)
- return Collections.emptySet();
+ if (NEVER_PURGE_TOMBSTONES || compacting == null || cfStore.getNeverPurgeTombstones())
+ return Collections.<SSTableReader>emptySet();
if (cfStore.getCompactionStrategyManager().onlyPurgeRepairedTombstones() && !Iterables.all(compacting, SSTableReader::isRepaired))
return Collections.emptySet();
@@ -239,16 +234,6 @@
return getFullyExpiredSSTables(cfStore, compacting, overlapping, gcBefore, false);
}
- public String getKeyspace()
- {
- return cfs.keyspace.getName();
- }
-
- public String getColumnFamily()
- {
- return cfs.name;
- }
-
/**
* @param key
* @return a predicate for whether tombstones marked for deletion at the given time for the given partition are
@@ -256,9 +241,10 @@
* containing his partition and not participating in the compaction. This means there isn't any data in those
* sstables that might still need to be suppressed by a tombstone at this timestamp.
*/
- public Predicate<Long> getPurgeEvaluator(DecoratedKey key)
+ @Override
+ public LongPredicate getPurgeEvaluator(DecoratedKey key)
{
- if (NEVER_PURGE_TOMBSTONES || !compactingRepaired())
+ if (NEVER_PURGE_TOMBSTONES || !compactingRepaired() || cfs.getNeverPurgeTombstones())
return time -> false;
overlapIterator.update(key);
@@ -320,7 +306,7 @@
// caller must close iterators
public Iterable<UnfilteredRowIterator> shadowSources(DecoratedKey key, boolean tombstoneOnly)
{
- if (!provideTombstoneSources() || !compactingRepaired() || NEVER_PURGE_TOMBSTONES)
+ if (!provideTombstoneSources() || !compactingRepaired() || NEVER_PURGE_TOMBSTONES || cfs.getNeverPurgeTombstones())
return null;
overlapIterator.update(key);
return Iterables.filter(Iterables.transform(overlapIterator.overlaps(),
diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionInfo.java b/src/java/org/apache/cassandra/db/compaction/CompactionInfo.java
index a6dbd9d..275ce70 100644
--- a/src/java/org/apache/cassandra/db/compaction/CompactionInfo.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionInfo.java
@@ -17,91 +17,84 @@
*/
package org.apache.cassandra.db.compaction;
-import java.io.Serializable;
+import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
import java.util.UUID;
+import java.util.function.Predicate;
-import org.apache.cassandra.config.CFMetaData;
+import com.google.common.base.Joiner;
+import com.google.common.collect.ImmutableSet;
-/** Implements serializable to allow structured info to be returned via JMX. */
-public final class CompactionInfo implements Serializable
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.schema.TableMetadata;
+
+public final class CompactionInfo
{
- private static final long serialVersionUID = 3695381572726744816L;
- private final CFMetaData cfm;
+ public static final String ID = "id";
+ public static final String KEYSPACE = "keyspace";
+ public static final String COLUMNFAMILY = "columnfamily";
+ public static final String COMPLETED = "completed";
+ public static final String TOTAL = "total";
+ public static final String TASK_TYPE = "taskType";
+ public static final String UNIT = "unit";
+ public static final String COMPACTION_ID = "compactionId";
+ public static final String SSTABLES = "sstables";
+
+ private final TableMetadata metadata;
private final OperationType tasktype;
private final long completed;
private final long total;
private final Unit unit;
private final UUID compactionId;
+ private final ImmutableSet<SSTableReader> sstables;
- public static enum Unit
+ public CompactionInfo(TableMetadata metadata, OperationType tasktype, long bytesComplete, long totalBytes, UUID compactionId, Collection<SSTableReader> sstables)
{
- BYTES("bytes"), RANGES("ranges"), KEYS("keys");
-
- private final String name;
-
- private Unit(String name)
- {
- this.name = name;
- }
-
- @Override
- public String toString()
- {
- return name;
- }
-
- public static boolean isFileSize(String unit)
- {
- return BYTES.toString().equals(unit);
- }
+ this(metadata, tasktype, bytesComplete, totalBytes, Unit.BYTES, compactionId, sstables);
}
- public CompactionInfo(CFMetaData cfm, OperationType tasktype, long bytesComplete, long totalBytes, UUID compactionId)
- {
- this(cfm, tasktype, bytesComplete, totalBytes, Unit.BYTES, compactionId);
- }
-
- public CompactionInfo(OperationType tasktype, long completed, long total, Unit unit, UUID compactionId)
- {
- this(null, tasktype, completed, total, unit, compactionId);
- }
-
- public CompactionInfo(CFMetaData cfm, OperationType tasktype, long completed, long total, Unit unit, UUID compactionId)
+ private CompactionInfo(TableMetadata metadata, OperationType tasktype, long completed, long total, Unit unit, UUID compactionId, Collection<SSTableReader> sstables)
{
this.tasktype = tasktype;
this.completed = completed;
this.total = total;
- this.cfm = cfm;
+ this.metadata = metadata;
this.unit = unit;
this.compactionId = compactionId;
+ this.sstables = ImmutableSet.copyOf(sstables);
+ }
+
+ /**
+ * Special compaction info where we always need to cancel the compaction - for example ViewBuilderTask and AutoSavingCache where we don't know
+ * the sstables at construction
+ */
+ public static CompactionInfo withoutSSTables(TableMetadata metadata, OperationType tasktype, long completed, long total, Unit unit, UUID compactionId)
+ {
+ return new CompactionInfo(metadata, tasktype, completed, total, unit, compactionId, ImmutableSet.of());
}
/** @return A copy of this CompactionInfo with updated progress. */
public CompactionInfo forProgress(long complete, long total)
{
- return new CompactionInfo(cfm, tasktype, complete, total, unit, compactionId);
+ return new CompactionInfo(metadata, tasktype, complete, total, unit, compactionId, sstables);
}
- public UUID getId()
+ public Optional<String> getKeyspace()
{
- return cfm != null ? cfm.cfId : null;
+ return Optional.ofNullable(metadata != null ? metadata.keyspace : null);
}
- public String getKeyspace()
+ public Optional<String> getTable()
{
- return cfm != null ? cfm.ksName : null;
+ return Optional.ofNullable(metadata != null ? metadata.name : null);
}
- public String getColumnFamily()
+ public TableMetadata getTableMetadata()
{
- return cfm != null ? cfm.cfName : null;
- }
-
- public CFMetaData getCFMetaData()
- {
- return cfm;
+ return metadata;
}
public long getCompleted()
@@ -119,42 +112,68 @@
return tasktype;
}
- public UUID compactionId()
+ public UUID getTaskId()
{
return compactionId;
}
+ public Unit getUnit()
+ {
+ return unit;
+ }
+
+ public Set<SSTableReader> getSSTables()
+ {
+ return sstables;
+ }
+
public String toString()
{
StringBuilder buff = new StringBuilder();
buff.append(getTaskType());
- if (cfm != null)
+
+ if (metadata != null)
{
- buff.append('@').append(getId()).append('(');
- buff.append(getKeyspace()).append(", ").append(getColumnFamily()).append(", ");
+ buff.append('@').append(metadata.id).append('(');
+ buff.append(metadata.keyspace).append(", ").append(metadata.name).append(", ");
}
else
{
buff.append('(');
}
- buff.append(getCompleted()).append('/').append(getTotal());
- return buff.append(')').append(unit).toString();
+ buff.append(getCompleted())
+ .append('/')
+ .append(getTotal())
+ .append(')')
+ .append(unit);
+
+ return buff.toString();
}
public Map<String, String> asMap()
{
Map<String, String> ret = new HashMap<String, String>();
- ret.put("id", getId() == null ? "" : getId().toString());
- ret.put("keyspace", getKeyspace());
- ret.put("columnfamily", getColumnFamily());
- ret.put("completed", Long.toString(completed));
- ret.put("total", Long.toString(total));
- ret.put("taskType", tasktype.toString());
- ret.put("unit", unit.toString());
- ret.put("compactionId", compactionId == null ? "" : compactionId.toString());
+ ret.put(ID, metadata != null ? metadata.id.toString() : "");
+ ret.put(KEYSPACE, getKeyspace().orElse(null));
+ ret.put(COLUMNFAMILY, getTable().orElse(null));
+ ret.put(COMPLETED, Long.toString(completed));
+ ret.put(TOTAL, Long.toString(total));
+ ret.put(TASK_TYPE, tasktype.toString());
+ ret.put(UNIT, unit.toString());
+ ret.put(COMPACTION_ID, compactionId == null ? "" : compactionId.toString());
+ ret.put(SSTABLES, Joiner.on(',').join(sstables));
return ret;
}
+ boolean shouldStop(Predicate<SSTableReader> sstablePredicate)
+ {
+ if (sstables.isEmpty())
+ {
+ return true;
+ }
+ return sstables.stream().anyMatch(sstablePredicate);
+ }
+
public static abstract class Holder
{
private volatile boolean stopRequested = false;
@@ -176,4 +195,27 @@
return stopRequested || (isGlobal() && CompactionManager.instance.isGlobalCompactionPaused());
}
}
+
+ public enum Unit
+ {
+ BYTES("bytes"), RANGES("token range parts"), KEYS("keys");
+
+ private final String name;
+
+ Unit(String name)
+ {
+ this.name = name;
+ }
+
+ @Override
+ public String toString()
+ {
+ return this.name;
+ }
+
+ public static boolean isFileSize(String unit)
+ {
+ return BYTES.toString().equals(unit);
+ }
+ }
}
diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionIterator.java b/src/java/org/apache/cassandra/db/compaction/CompactionIterator.java
index 4b35e9d..78bdfb0 100644
--- a/src/java/org/apache/cassandra/db/compaction/CompactionIterator.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionIterator.java
@@ -18,11 +18,13 @@
package org.apache.cassandra.db.compaction;
import java.util.*;
-import java.util.function.Predicate;
+import java.util.function.LongPredicate;
+import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Ordering;
-import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.db.transform.DuplicateRowChecker;
import org.apache.cassandra.db.*;
@@ -34,7 +36,6 @@
import org.apache.cassandra.db.transform.Transformation;
import org.apache.cassandra.index.transactions.CompactionTransaction;
import org.apache.cassandra.io.sstable.ISSTableScanner;
-import org.apache.cassandra.metrics.CompactionMetrics;
import org.apache.cassandra.schema.CompactionParams.TombstoneOption;
/**
@@ -58,8 +59,9 @@
private static final long UNFILTERED_TO_UPDATE_PROGRESS = 100;
private final OperationType type;
- private final CompactionController controller;
+ private final AbstractCompactionController controller;
private final List<ISSTableScanner> scanners;
+ private final ImmutableSet<SSTableReader> sstables;
private final int nowInSec;
private final UUID compactionId;
@@ -75,15 +77,15 @@
private final long[] mergeCounters;
private final UnfilteredPartitionIterator compacted;
- private final CompactionMetrics metrics;
+ private final ActiveCompactionsTracker activeCompactions;
- public CompactionIterator(OperationType type, List<ISSTableScanner> scanners, CompactionController controller, int nowInSec, UUID compactionId)
+ public CompactionIterator(OperationType type, List<ISSTableScanner> scanners, AbstractCompactionController controller, int nowInSec, UUID compactionId)
{
- this(type, scanners, controller, nowInSec, compactionId, null);
+ this(type, scanners, controller, nowInSec, compactionId, ActiveCompactionsTracker.NOOP);
}
@SuppressWarnings("resource") // We make sure to close mergedIterator in close() and CompactionIterator is itself an AutoCloseable
- public CompactionIterator(OperationType type, List<ISSTableScanner> scanners, CompactionController controller, int nowInSec, UUID compactionId, CompactionMetrics metrics)
+ public CompactionIterator(OperationType type, List<ISSTableScanner> scanners, AbstractCompactionController controller, int nowInSec, UUID compactionId, ActiveCompactionsTracker activeCompactions)
{
this.controller = controller;
this.type = type;
@@ -97,37 +99,32 @@
bytes += scanner.getLengthInBytes();
this.totalBytes = bytes;
this.mergeCounters = new long[scanners.size()];
- this.metrics = metrics;
-
- if (metrics != null)
- metrics.beginCompaction(this);
+ this.activeCompactions = activeCompactions == null ? ActiveCompactionsTracker.NOOP : activeCompactions;
+ this.activeCompactions.beginCompaction(this); // note that CompactionTask also calls this, but CT only creates CompactionIterator with a NOOP ActiveCompactions
UnfilteredPartitionIterator merged = scanners.isEmpty()
- ? EmptyIterators.unfilteredPartition(controller.cfs.metadata, false)
- : UnfilteredPartitionIterators.merge(scanners, nowInSec, listener());
- boolean isForThrift = merged.isForThrift(); // to stop capture of iterator in Purger, which is confusing for debug
- merged = Transformation.apply(merged, new GarbageSkipper(controller, nowInSec));
- merged = Transformation.apply(merged, new Purger(isForThrift, controller, nowInSec));
- this.compacted = DuplicateRowChecker.duringCompaction(merged, type);
+ ? EmptyIterators.unfilteredPartition(controller.cfs.metadata())
+ : UnfilteredPartitionIterators.merge(scanners, listener());
+ merged = Transformation.apply(merged, new GarbageSkipper(controller));
+ merged = Transformation.apply(merged, new Purger(controller, nowInSec));
+ merged = DuplicateRowChecker.duringCompaction(merged, type);
+ compacted = Transformation.apply(merged, new AbortableUnfilteredPartitionTransformation(this));
+ sstables = scanners.stream().map(ISSTableScanner::getBackingSSTables).flatMap(Collection::stream).collect(ImmutableSet.toImmutableSet());
}
- public boolean isForThrift()
+ public TableMetadata metadata()
{
- return false;
- }
-
- public CFMetaData metadata()
- {
- return controller.cfs.metadata;
+ return controller.cfs.metadata();
}
public CompactionInfo getCompactionInfo()
{
- return new CompactionInfo(controller.cfs.metadata,
+ return new CompactionInfo(controller.cfs.metadata(),
type,
bytesRead,
totalBytes,
- compactionId);
+ compactionId,
+ sstables);
}
public boolean isGlobal()
@@ -158,8 +155,10 @@
public UnfilteredRowIterators.MergeListener getRowMergeListener(DecoratedKey partitionKey, List<UnfilteredRowIterator> versions)
{
int merged = 0;
- for (UnfilteredRowIterator iter : versions)
+ for (int i=0, isize=versions.size(); i<isize; i++)
{
+ @SuppressWarnings("resource")
+ UnfilteredRowIterator iter = versions.get(i);
if (iter != null)
merged++;
}
@@ -173,15 +172,17 @@
Columns statics = Columns.NONE;
Columns regulars = Columns.NONE;
- for (UnfilteredRowIterator iter : versions)
+ for (int i=0, isize=versions.size(); i<isize; i++)
{
+ @SuppressWarnings("resource")
+ UnfilteredRowIterator iter = versions.get(i);
if (iter != null)
{
statics = statics.mergeTo(iter.columns().statics);
regulars = regulars.mergeTo(iter.columns().regulars);
}
}
- final PartitionColumns partitionColumns = new PartitionColumns(statics, regulars);
+ final RegularAndStaticColumns regularAndStaticColumns = new RegularAndStaticColumns(statics, regulars);
// If we have a 2ndary index, we must update it with deleted/shadowed cells.
// we can reuse a single CleanupTransaction for the duration of a partition.
@@ -195,7 +196,7 @@
// TODO: this should probably be done asynchronously and batched.
final CompactionTransaction indexTransaction =
controller.cfs.indexManager.newCompactionTransaction(partitionKey,
- partitionColumns,
+ regularAndStaticColumns,
versions.size(),
nowInSec);
@@ -260,8 +261,7 @@
}
finally
{
- if (metrics != null)
- metrics.finishCompaction(this);
+ activeCompactions.finishCompaction(this);
}
}
@@ -272,21 +272,18 @@
private class Purger extends PurgeFunction
{
- private final CompactionController controller;
+ private final AbstractCompactionController controller;
private DecoratedKey currentKey;
- private Predicate<Long> purgeEvaluator;
+ private LongPredicate purgeEvaluator;
private long compactedUnfiltered;
- private Purger(boolean isForThrift, CompactionController controller, int nowInSec)
+ private Purger(AbstractCompactionController controller, int nowInSec)
{
- super(isForThrift,
- nowInSec,
- controller.gcBefore,
- controller.compactingRepaired() ? Integer.MAX_VALUE : Integer.MIN_VALUE,
+ super(nowInSec, controller.gcBefore, controller.compactingRepaired() ? Integer.MAX_VALUE : Integer.MIN_VALUE,
controller.cfs.getCompactionStrategyManager().onlyPurgeRepairedTombstones(),
- controller.cfs.metadata.enforceStrictLiveness());
+ controller.cfs.metadata.get().enforceStrictLiveness());
this.controller = controller;
}
@@ -318,7 +315,7 @@
* This is computed lazily on demand as we only need this if there is tombstones and this a bit expensive
* (see #8914).
*/
- protected Predicate<Long> getPurgeEvaluator()
+ protected LongPredicate getPurgeEvaluator()
{
if (purgeEvaluator == null)
{
@@ -339,8 +336,7 @@
final DeletionTime partitionLevelDeletion;
final Row staticRow;
final ColumnFilter cf;
- final int nowInSec;
- final CFMetaData metadata;
+ final TableMetadata metadata;
final boolean cellLevelGC;
DeletionTime tombOpenDeletionTime = DeletionTime.LIVE;
@@ -357,15 +353,13 @@
*
* @param dataSource The input row. The result is a filtered version of this.
* @param tombSource Tombstone source, i.e. iterator used to identify deleted data in the input row.
- * @param nowInSec Current time, used in choosing the winner when cell expiration is involved.
* @param cellLevelGC If false, the iterator will only look at row-level deletion times and tombstones.
* If true, deleted or overwritten cells within a surviving row will also be removed.
*/
- protected GarbageSkippingUnfilteredRowIterator(UnfilteredRowIterator dataSource, UnfilteredRowIterator tombSource, int nowInSec, boolean cellLevelGC)
+ protected GarbageSkippingUnfilteredRowIterator(UnfilteredRowIterator dataSource, UnfilteredRowIterator tombSource, boolean cellLevelGC)
{
super(dataSource);
this.tombSource = tombSource;
- this.nowInSec = nowInSec;
this.cellLevelGC = cellLevelGC;
metadata = dataSource.metadata();
cf = ColumnFilter.all(metadata);
@@ -470,7 +464,7 @@
{
if (cellLevelGC)
{
- return Rows.removeShadowedCells(dataRow, tombRow, activeDeletionTime, nowInSec);
+ return Rows.removeShadowedCells(dataRow, tombRow, activeDeletionTime);
}
else
{
@@ -529,14 +523,12 @@
*/
private static class GarbageSkipper extends Transformation<UnfilteredRowIterator>
{
- final int nowInSec;
- final CompactionController controller;
+ final AbstractCompactionController controller;
final boolean cellLevelGC;
- private GarbageSkipper(CompactionController controller, int nowInSec)
+ private GarbageSkipper(AbstractCompactionController controller)
{
this.controller = controller;
- this.nowInSec = nowInSec;
cellLevelGC = controller.tombstoneOption == TombstoneOption.CELL;
}
@@ -557,7 +549,42 @@
if (iters.isEmpty())
return partition;
- return new GarbageSkippingUnfilteredRowIterator(partition, UnfilteredRowIterators.merge(iters, nowInSec), nowInSec, cellLevelGC);
+ return new GarbageSkippingUnfilteredRowIterator(partition, UnfilteredRowIterators.merge(iters), cellLevelGC);
+ }
+ }
+
+ private static class AbortableUnfilteredPartitionTransformation extends Transformation<UnfilteredRowIterator>
+ {
+ private final AbortableUnfilteredRowTransformation abortableIter;
+
+ private AbortableUnfilteredPartitionTransformation(CompactionIterator iter)
+ {
+ this.abortableIter = new AbortableUnfilteredRowTransformation(iter);
+ }
+
+ @Override
+ protected UnfilteredRowIterator applyToPartition(UnfilteredRowIterator partition)
+ {
+ if (abortableIter.iter.isStopRequested())
+ throw new CompactionInterruptedException(abortableIter.iter.getCompactionInfo());
+ return Transformation.apply(partition, abortableIter);
+ }
+ }
+
+ private static class AbortableUnfilteredRowTransformation extends Transformation
+ {
+ private final CompactionIterator iter;
+
+ private AbortableUnfilteredRowTransformation(CompactionIterator iter)
+ {
+ this.iter = iter;
+ }
+
+ public Row applyToRow(Row row)
+ {
+ if (iter.isStopRequested())
+ throw new CompactionInterruptedException(iter.getCompactionInfo());
+ return row;
}
}
}
diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionLogger.java b/src/java/org/apache/cassandra/db/compaction/CompactionLogger.java
index 1a4abfe..f473be7 100644
--- a/src/java/org/apache/cassandra/db/compaction/CompactionLogger.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionLogger.java
@@ -170,7 +170,7 @@
node.put("size", sstable.onDiskLength());
JsonNode logResult = strategy.strategyLogger().sstable(sstable);
if (logResult != null)
- node.put("details", logResult);
+ node.set("details", logResult);
return node;
}
@@ -182,7 +182,7 @@
return node;
node.put("strategyId", getId(strategy));
node.put("type", strategy.getName());
- node.put("tables", formatSSTables(strategy));
+ node.set("tables", formatSSTables(strategy));
node.put("repaired", csm.isRepaired(strategy));
List<String> folders = csm.getStrategyFolders(strategy);
ArrayNode folderNode = json.arrayNode();
@@ -190,11 +190,11 @@
{
folderNode.add(folder);
}
- node.put("folders", folderNode);
+ node.set("folders", folderNode);
JsonNode logResult = strategy.strategyLogger().options();
if (logResult != null)
- node.put("options", logResult);
+ node.set("options", logResult);
return node;
}
@@ -209,7 +209,7 @@
{
ObjectNode node = json.objectNode();
node.put("strategyId", getId(strategy));
- node.put("table", formatSSTable(strategy, sstable));
+ node.set("table", formatSSTable(strategy, sstable));
return node;
}
@@ -228,7 +228,7 @@
ObjectNode node = json.objectNode();
node.put("type", "enable");
describeStrategy(node);
- node.put("strategies", compactionStrategyMap(this::startStrategy));
+ node.set("strategies", compactionStrategyMap(this::startStrategy));
return node;
}
@@ -247,7 +247,7 @@
ObjectNode node = json.objectNode();
node.put("type", "disable");
describeStrategy(node);
- node.put("strategies", compactionStrategyMap(this::shutdownStrategy));
+ node.set("strategies", compactionStrategyMap(this::shutdownStrategy));
serializer.write(node, this::startStrategies, this);
}
}
@@ -259,7 +259,7 @@
ObjectNode node = json.objectNode();
node.put("type", "flush");
describeStrategy(node);
- node.put("tables", sstableMap(sstables, this::describeSSTable));
+ node.set("tables", sstableMap(sstables, this::describeSSTable));
serializer.write(node, this::startStrategies, this);
}
}
@@ -273,8 +273,8 @@
describeStrategy(node);
node.put("start", String.valueOf(startTime));
node.put("end", String.valueOf(endTime));
- node.put("input", sstableMap(input, this::describeSSTable));
- node.put("output", sstableMap(output, this::describeSSTable));
+ node.set("input", sstableMap(input, this::describeSSTable));
+ node.set("output", sstableMap(output, this::describeSSTable));
serializer.write(node, this::startStrategies, this);
}
}
diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
index 56d2d29..7df56fc 100644
--- a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
@@ -22,15 +22,19 @@
import java.util.*;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.BooleanSupplier;
import java.util.function.Predicate;
import java.util.stream.Collectors;
import javax.management.openmbean.OpenDataException;
import javax.management.openmbean.TabularData;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
import com.google.common.collect.*;
import com.google.common.util.concurrent.*;
+import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.locator.RangesAtEndpoint;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -39,9 +43,10 @@
import org.apache.cassandra.concurrent.DebuggableThreadPoolExecutor;
import org.apache.cassandra.concurrent.JMXEnabledThreadPoolExecutor;
import org.apache.cassandra.concurrent.NamedThreadFactory;
-import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.config.Schema;
+import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.compaction.CompactionInfo.Holder;
import org.apache.cassandra.db.lifecycle.ILifecycleTransaction;
@@ -51,7 +56,7 @@
import org.apache.cassandra.db.lifecycle.View;
import org.apache.cassandra.db.lifecycle.WrappedLifecycleTransaction;
import org.apache.cassandra.db.rows.UnfilteredRowIterator;
-import org.apache.cassandra.db.view.ViewBuilder;
+import org.apache.cassandra.db.view.ViewBuilderTask;
import org.apache.cassandra.dht.Bounds;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
@@ -64,16 +69,21 @@
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.sstable.format.SSTableWriter;
import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
+import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.metrics.CompactionMetrics;
-import org.apache.cassandra.repair.Validator;
+import org.apache.cassandra.metrics.TableMetrics;
import org.apache.cassandra.schema.CompactionParams.TombstoneOption;
import org.apache.cassandra.service.ActiveRepairService;
import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.streaming.PreviewKind;
import org.apache.cassandra.utils.*;
+import org.apache.cassandra.utils.Throwables;
import org.apache.cassandra.utils.concurrent.Refs;
import static java.util.Collections.singleton;
+import static org.apache.cassandra.service.ActiveRepairService.NO_PENDING_REPAIR;
+import static org.apache.cassandra.service.ActiveRepairService.UNREPAIRED_SSTABLE;
/**
* <p>
@@ -89,6 +99,9 @@
private static final Logger logger = LoggerFactory.getLogger(CompactionManager.class);
public static final CompactionManager instance;
+ @VisibleForTesting
+ public final AtomicInteger currentlyBackgroundUpgrading = new AtomicInteger(0);
+
public static final int NO_GC = Integer.MIN_VALUE;
public static final int GC_ALL = Integer.MAX_VALUE;
@@ -111,18 +124,26 @@
}
private final CompactionExecutor executor = new CompactionExecutor();
- private final CompactionExecutor validationExecutor = new ValidationExecutor();
+ private final ValidationExecutor validationExecutor = new ValidationExecutor();
private final CompactionExecutor cacheCleanupExecutor = new CacheCleanupExecutor();
+ private final CompactionExecutor viewBuildExecutor = new ViewBuildExecutor();
- private final CompactionMetrics metrics = new CompactionMetrics(executor, validationExecutor);
+ private final CompactionMetrics metrics = new CompactionMetrics(executor, validationExecutor, viewBuildExecutor);
@VisibleForTesting
final Multiset<ColumnFamilyStore> compactingCF = ConcurrentHashMultiset.create();
+ public final ActiveCompactions active = new ActiveCompactions();
+
// used to temporarily pause non-strategy managed compactions (like index summary redistribution)
private final AtomicInteger globalCompactionPauseCount = new AtomicInteger(0);
private final RateLimiter compactionRateLimiter = RateLimiter.create(Double.MAX_VALUE);
+ public CompactionMetrics getMetrics()
+ {
+ return metrics;
+ }
+
/**
* Gets compaction rate limiter.
* Rate unit is bytes per sec.
@@ -191,10 +212,10 @@
return futures;
}
- public boolean isCompacting(Iterable<ColumnFamilyStore> cfses)
+ public boolean isCompacting(Iterable<ColumnFamilyStore> cfses, Predicate<SSTableReader> sstablePredicate)
{
for (ColumnFamilyStore cfs : cfses)
- if (!cfs.getTracker().getCompacting().isEmpty())
+ if (cfs.getTracker().getCompacting().stream().anyMatch(sstablePredicate))
return true;
return false;
}
@@ -208,10 +229,11 @@
// shutdown executors to prevent further submission
executor.shutdown();
validationExecutor.shutdown();
+ viewBuildExecutor.shutdown();
cacheCleanupExecutor.shutdown();
// interrupt compactions and validations
- for (Holder compactionHolder : CompactionMetrics.getCompactions())
+ for (Holder compactionHolder : active.getCompactions())
{
compactionHolder.stop();
}
@@ -219,7 +241,7 @@
// wait for tasks to terminate
// compaction tasks are interrupted above, so it shuold be fairy quick
// until not interrupted tasks to complete.
- for (ExecutorService exec : Arrays.asList(executor, validationExecutor, cacheCleanupExecutor))
+ for (ExecutorService exec : Arrays.asList(executor, validationExecutor, viewBuildExecutor, cacheCleanupExecutor))
{
try
{
@@ -241,6 +263,7 @@
// the actual sstables to compact are not determined until we run the BCT; that way, if new sstables
// are created between task submission and execution, we execute against the most up-to-date information
+ @VisibleForTesting
class BackgroundCompactionCandidate implements Runnable
{
private final ColumnFamilyStore cfs;
@@ -253,6 +276,7 @@
public void run()
{
+ boolean ranCompaction = false;
try
{
logger.trace("Checking {}.{}", cfs.keyspace.getName(), cfs.name);
@@ -266,17 +290,51 @@
AbstractCompactionTask task = strategy.getNextBackgroundTask(getDefaultGcBefore(cfs, FBUtilities.nowInSeconds()));
if (task == null)
{
- logger.trace("No tasks available");
- return;
+ if (DatabaseDescriptor.automaticSSTableUpgrade())
+ ranCompaction = maybeRunUpgradeTask(strategy);
}
- task.execute(metrics);
+ else
+ {
+ task.execute(active);
+ ranCompaction = true;
+ }
}
finally
{
compactingCF.remove(cfs);
}
- submitBackground(cfs);
+ if (ranCompaction) // only submit background if we actually ran a compaction - otherwise we end up in an infinite loop submitting noop background tasks
+ submitBackground(cfs);
}
+
+ boolean maybeRunUpgradeTask(CompactionStrategyManager strategy)
+ {
+ logger.debug("Checking for upgrade tasks {}.{}", cfs.keyspace.getName(), cfs.getTableName());
+ try
+ {
+ if (currentlyBackgroundUpgrading.incrementAndGet() <= DatabaseDescriptor.maxConcurrentAutoUpgradeTasks())
+ {
+ AbstractCompactionTask upgradeTask = strategy.findUpgradeSSTableTask();
+ if (upgradeTask != null)
+ {
+ upgradeTask.execute(active);
+ return true;
+ }
+ }
+ }
+ finally
+ {
+ currentlyBackgroundUpgrading.decrementAndGet();
+ }
+ logger.trace("No tasks available");
+ return false;
+ }
+ }
+
+ @VisibleForTesting
+ public BackgroundCompactionCandidate getBackgroundCompactionCandidate(ColumnFamilyStore cfs)
+ {
+ return new BackgroundCompactionCandidate(cfs);
}
/**
@@ -394,14 +452,14 @@
}
@Override
- public void execute(LifecycleTransaction input) throws IOException
+ public void execute(LifecycleTransaction input)
{
- scrubOne(cfs, input, skipCorrupted, checkData, reinsertOverflowedTTL);
+ scrubOne(cfs, input, skipCorrupted, checkData, reinsertOverflowedTTL, active);
}
}, jobs, OperationType.SCRUB);
}
- public AllSSTableOpStatus performVerify(final ColumnFamilyStore cfs, final boolean extendedVerify) throws InterruptedException, ExecutionException
+ public AllSSTableOpStatus performVerify(ColumnFamilyStore cfs, Verifier.Options options) throws InterruptedException, ExecutionException
{
assert !cfs.isIndex();
return parallelAllSSTableOperation(cfs, new OneSSTableOperation()
@@ -413,9 +471,9 @@
}
@Override
- public void execute(LifecycleTransaction input) throws IOException
+ public void execute(LifecycleTransaction input)
{
- verifyOne(cfs, input.onlyOne(), extendedVerify);
+ verifyOne(cfs, input.onlyOne(), options, active);
}
}, 0, OperationType.VERIFY);
}
@@ -448,7 +506,7 @@
AbstractCompactionTask task = cfs.getCompactionStrategyManager().getCompactionTask(txn, NO_GC, Long.MAX_VALUE);
task.setUserDefined(true);
task.setCompactionType(OperationType.UPGRADE_SSTABLES);
- task.execute(metrics);
+ task.execute(active);
}
}, jobs, OperationType.UPGRADE_SSTABLES);
}
@@ -463,7 +521,10 @@
return AllSSTableOpStatus.ABORTED;
}
// if local ranges is empty, it means no data should remain
- final Collection<Range<Token>> ranges = StorageService.instance.getLocalRanges(keyspace.getName());
+ final RangesAtEndpoint replicas = StorageService.instance.getLocalReplicas(keyspace.getName());
+ final Set<Range<Token>> allRanges = replicas.ranges();
+ final Set<Range<Token>> transientRanges = replicas.onlyTransient().ranges();
+ final Set<Range<Token>> fullRanges = replicas.onlyFull().ranges();
final boolean hasIndexes = cfStore.indexManager.hasIndexes();
return parallelAllSSTableOperation(cfStore, new OneSSTableOperation()
@@ -478,18 +539,27 @@
while (sstableIter.hasNext())
{
SSTableReader sstable = sstableIter.next();
+ boolean needsCleanupFull = needsCleanup(sstable, fullRanges);
+ boolean needsCleanupTransient = needsCleanup(sstable, transientRanges);
+ //If there are no ranges for which the table needs cleanup either due to lack of intersection or lack
+ //of the table being repaired.
totalSSTables++;
- if (!needsCleanup(sstable, ranges))
+ if (!needsCleanupFull && (!needsCleanupTransient || !sstable.isRepaired()))
{
- logger.debug("Not cleaning up {} ([{}, {}]) - no tokens outside owned ranges {}",
- sstable, sstable.first.getToken(), sstable.last.getToken(), ranges);
+ logger.debug("Skipping {} ([{}, {}]) for cleanup; all rows should be kept. Needs cleanup full ranges: {} Needs cleanup transient ranges: {} Repaired: {}",
+ sstable,
+ sstable.first.getToken(),
+ sstable.last.getToken(),
+ needsCleanupFull,
+ needsCleanupTransient,
+ sstable.isRepaired());
sstableIter.remove();
transaction.cancel(sstable);
skippedSStables++;
}
}
- logger.info("Skipping cleanup for {}/{} sstables for {}.{} since they are fully contained in owned ranges ({})",
- skippedSStables, totalSSTables, cfStore.keyspace.getName(), cfStore.getTableName(), ranges);
+ logger.info("Skipping cleanup for {}/{} sstables for {}.{} since they are fully contained in owned ranges (full ranges: {}, transient ranges: {})",
+ skippedSStables, totalSSTables, cfStore.keyspace.getName(), cfStore.getTableName(), fullRanges, transientRanges);
sortedSSTables.sort(SSTableReader.sizeComparator);
return sortedSSTables;
}
@@ -497,8 +567,8 @@
@Override
public void execute(LifecycleTransaction txn) throws IOException
{
- CleanupStrategy cleanupStrategy = CleanupStrategy.get(cfStore, ranges, FBUtilities.nowInSeconds());
- doCleanupOne(cfStore, txn, cleanupStrategy, ranges, hasIndexes);
+ CleanupStrategy cleanupStrategy = CleanupStrategy.get(cfStore, allRanges, transientRanges, txn.onlyOne().isRepaired(), FBUtilities.nowInSeconds());
+ doCleanupOne(cfStore, txn, cleanupStrategy, replicas.ranges(), fullRanges, transientRanges, hasIndexes);
}
}, jobs, OperationType.CLEANUP);
}
@@ -534,7 +604,7 @@
};
task.setUserDefined(true);
task.setCompactionType(OperationType.GARBAGE_COLLECT);
- task.execute(metrics);
+ task.execute(active);
}
}, jobs, OperationType.GARBAGE_COLLECT);
}
@@ -546,9 +616,8 @@
logger.info("Partitioner does not support splitting");
return AllSSTableOpStatus.ABORTED;
}
- final Collection<Range<Token>> r = StorageService.instance.getLocalRanges(cfs.keyspace.getName());
- if (r.isEmpty())
+ if (StorageService.instance.getLocalReplicas(cfs.keyspace.getName()).isEmpty())
{
logger.info("Relocate cannot run before a node has joined the ring");
return AllSSTableOpStatus.ABORTED;
@@ -607,152 +676,114 @@
AbstractCompactionTask task = cfs.getCompactionStrategyManager().getCompactionTask(txn, NO_GC, Long.MAX_VALUE);
task.setUserDefined(true);
task.setCompactionType(OperationType.RELOCATE);
- task.execute(metrics);
+ task.execute(active);
}
}, jobs, OperationType.RELOCATE);
}
/**
- * Submit anti-compactions for a collection of SSTables over a set of repaired ranges and marks corresponding SSTables
- * as repaired.
- *
- * @param cfs Column family for anti-compaction
- * @param ranges Repaired ranges to be anti-compacted into separate SSTables.
- * @param sstables {@link Refs} of SSTables within CF to anti-compact.
- * @param repairedAt Unix timestamp of when repair was completed.
- * @param parentRepairSession Corresponding repair session
- * @return Futures executing anti-compaction.
+ * Splits the given token ranges of the given sstables into a pending repair silo
*/
- public ListenableFuture<?> submitAntiCompaction(final ColumnFamilyStore cfs,
- final Collection<Range<Token>> ranges,
- final Refs<SSTableReader> sstables,
- final long repairedAt,
- final UUID parentRepairSession)
+ public ListenableFuture<?> submitPendingAntiCompaction(ColumnFamilyStore cfs,
+ RangesAtEndpoint tokenRanges,
+ Refs<SSTableReader> sstables,
+ LifecycleTransaction txn,
+ UUID sessionId,
+ BooleanSupplier isCancelled)
{
Runnable runnable = new WrappedRunnable()
{
- @Override
- @SuppressWarnings("resource")
- public void runMayThrow() throws Exception
+ protected void runMayThrow() throws Exception
{
- LifecycleTransaction modifier = null;
- while (modifier == null)
+ try (TableMetrics.TableTimer.Context ctx = cfs.metric.anticompactionTime.time())
{
- for (SSTableReader compactingSSTable : cfs.getTracker().getCompacting())
- sstables.releaseIfHolds(compactingSSTable);
- // We don't anti-compact any SSTable that has been compacted during repair as it may have been compacted
- // with unrepaired data.
- Set<SSTableReader> compactedSSTables = new HashSet<>();
- for (SSTableReader sstable : sstables)
- if (sstable.isMarkedCompacted())
- compactedSSTables.add(sstable);
- sstables.release(compactedSSTables);
- modifier = cfs.getTracker().tryModify(sstables, OperationType.ANTICOMPACTION);
+ performAnticompaction(cfs, tokenRanges, sstables, txn, sessionId, isCancelled);
}
- performAnticompaction(cfs, ranges, sstables, modifier, repairedAt, parentRepairSession);
}
};
- ListenableFuture<?> ret = null;
+ ListenableFuture<?> task = null;
try
{
- ret = executor.submitIfRunning(runnable, "anticompaction");
- return ret;
+ task = executor.submitIfRunning(runnable, "pending anticompaction");
+ return task;
}
finally
{
- if (ret == null || ret.isCancelled())
+ if (task == null || task.isCancelled())
+ {
sstables.release();
+ txn.abort();
+ }
}
}
/**
+ * for sstables that are fully contained in the given ranges, just rewrite their metadata with
+ * the pending repair id and remove them from the transaction
+ */
+ private static void mutateFullyContainedSSTables(ColumnFamilyStore cfs,
+ Refs<SSTableReader> refs,
+ Iterator<SSTableReader> sstableIterator,
+ Collection<Range<Token>> ranges,
+ LifecycleTransaction txn,
+ UUID sessionID,
+ boolean isTransient) throws IOException
+ {
+ if (ranges.isEmpty())
+ return;
+
+ List<Range<Token>> normalizedRanges = Range.normalize(ranges);
+
+ Set<SSTableReader> fullyContainedSSTables = findSSTablesToAnticompact(sstableIterator, normalizedRanges, sessionID);
+
+ cfs.metric.bytesMutatedAnticompaction.inc(SSTableReader.getTotalBytes(fullyContainedSSTables));
+ cfs.getCompactionStrategyManager().mutateRepaired(fullyContainedSSTables, UNREPAIRED_SSTABLE, sessionID, isTransient);
+ // since we're just re-writing the sstable metdata for the fully contained sstables, we don't want
+ // them obsoleted when the anti-compaction is complete. So they're removed from the transaction here
+ txn.cancel(fullyContainedSSTables);
+ refs.release(fullyContainedSSTables);
+ }
+
+ /**
* Make sure the {validatedForRepair} are marked for compaction before calling this.
*
* Caller must reference the validatedForRepair sstables (via ParentRepairSession.getActiveRepairedSSTableRefs(..)).
*
- * NOTE: Repairs can take place on both unrepaired (incremental + full) and repaired (full) data.
- * Although anti-compaction could work on repaired sstables as well and would result in having more accurate
- * repairedAt values for these, we avoid anti-compacting already repaired sstables, as we currently don't
- * make use of any actual repairedAt value and splitting up sstables just for that is not worth it. However, we will
- * still update repairedAt if the SSTable is fully contained within the repaired ranges, as this does not require
- * anticompaction.
- *
* @param cfs
- * @param ranges Ranges that the repair was carried out on
+ * @param replicas token ranges to be repaired
* @param validatedForRepair SSTables containing the repaired ranges. Should be referenced before passing them.
- * @param txn Transaction across all SSTables that were repaired.
- * @param parentRepairSession parent repair session ID
+ * @param sessionID the repair session we're anti-compacting for
+ * @param isCancelled function that indicates if active anti-compaction should be canceled
* @throws InterruptedException
* @throws IOException
*/
public void performAnticompaction(ColumnFamilyStore cfs,
- Collection<Range<Token>> ranges,
+ RangesAtEndpoint replicas,
Refs<SSTableReader> validatedForRepair,
LifecycleTransaction txn,
- long repairedAt,
- UUID parentRepairSession) throws InterruptedException, IOException
+ UUID sessionID,
+ BooleanSupplier isCancelled) throws IOException
{
- logger.info("[repair #{}] Starting anticompaction for {}.{} on {}/{} sstables", parentRepairSession, cfs.keyspace.getName(), cfs.getTableName(), validatedForRepair.size(), cfs.getLiveSSTables());
- logger.trace("[repair #{}] Starting anticompaction for ranges {}", parentRepairSession, ranges);
- Set<SSTableReader> sstables = new HashSet<>(validatedForRepair);
- Set<SSTableReader> mutatedRepairStatuses = new HashSet<>(); // SSTables that were completely repaired only
- Set<SSTableReader> nonAnticompacting = new HashSet<>();
-
- Iterator<SSTableReader> sstableIterator = sstables.iterator();
try
{
- List<Range<Token>> normalizedRanges = Range.normalize(ranges);
+ ActiveRepairService.ParentRepairSession prs = ActiveRepairService.instance.getParentRepairSession(sessionID);
+ Preconditions.checkArgument(!prs.isPreview(), "Cannot anticompact for previews");
+ Preconditions.checkArgument(!replicas.isEmpty(), "No ranges to anti-compact");
- while (sstableIterator.hasNext())
- {
- SSTableReader sstable = sstableIterator.next();
- List<String> anticompactRanges = new ArrayList<>();
- // We don't anti-compact SSTables already marked repaired. See CASSANDRA-13153
- // and CASSANDRA-14423.
- if (sstable.isRepaired()) // We never anti-compact already repaired SSTables
- nonAnticompacting.add(sstable);
+ if (logger.isInfoEnabled())
+ logger.info("{} Starting anticompaction for {}.{} on {}/{} sstables", PreviewKind.NONE.logPrefix(sessionID), cfs.keyspace.getName(), cfs.getTableName(), validatedForRepair.size(), cfs.getLiveSSTables().size());
+ if (logger.isTraceEnabled())
+ logger.trace("{} Starting anticompaction for ranges {}", PreviewKind.NONE.logPrefix(sessionID), replicas);
- Bounds<Token> sstableBounds = new Bounds<>(sstable.first.getToken(), sstable.last.getToken());
+ Set<SSTableReader> sstables = new HashSet<>(validatedForRepair);
+ validateSSTableBoundsForAnticompaction(sessionID, sstables, replicas);
+ mutateFullyContainedSSTables(cfs, validatedForRepair, sstables.iterator(), replicas.onlyFull().ranges(), txn, sessionID, false);
+ mutateFullyContainedSSTables(cfs, validatedForRepair, sstables.iterator(), replicas.onlyTransient().ranges(), txn, sessionID, true);
- boolean shouldAnticompact = false;
-
- for (Range<Token> r : normalizedRanges)
- {
- if (r.contains(sstableBounds.left) && r.contains(sstableBounds.right))
- {
- logger.info("[repair #{}] SSTable {} fully contained in range {}, mutating repairedAt instead of anticompacting", parentRepairSession, sstable, r);
- sstable.descriptor.getMetadataSerializer().mutateRepairedAt(sstable.descriptor, repairedAt);
- sstable.reloadSSTableMetadata();
- if (!nonAnticompacting.contains(sstable)) // don't notify if the SSTable was already repaired
- mutatedRepairStatuses.add(sstable);
- sstableIterator.remove();
- shouldAnticompact = true;
- break;
- }
- else if (r.intersects(sstableBounds) && !nonAnticompacting.contains(sstable))
- {
- anticompactRanges.add(r.toString());
- shouldAnticompact = true;
- }
- }
-
- if (!anticompactRanges.isEmpty())
- logger.info("[repair #{}] SSTable {} ({}) will be anticompacted on range {}", parentRepairSession, sstable, sstableBounds, String.join(", ", anticompactRanges));
-
- if (!shouldAnticompact)
- {
- logger.info("[repair #{}] SSTable {} ({}) not subject to anticompaction of repaired ranges {}, not touching repairedAt.", parentRepairSession, sstable, sstableBounds, normalizedRanges);
- nonAnticompacting.add(sstable);
- sstableIterator.remove();
- }
- }
- cfs.getTracker().notifySSTableRepairedStatusChanged(mutatedRepairStatuses);
- txn.cancel(Sets.union(nonAnticompacting, mutatedRepairStatuses));
- validatedForRepair.release(Sets.union(nonAnticompacting, mutatedRepairStatuses));
assert txn.originals().equals(sstables);
if (!sstables.isEmpty())
- doAntiCompaction(cfs, ranges, txn, repairedAt);
+ doAntiCompaction(cfs, replicas, txn, sessionID, isCancelled);
txn.finish();
}
finally
@@ -761,7 +792,57 @@
txn.close();
}
- logger.info("[repair #{}] Completed anticompaction successfully", parentRepairSession);
+ logger.info("{} Completed anticompaction successfully", PreviewKind.NONE.logPrefix(sessionID));
+ }
+
+ static void validateSSTableBoundsForAnticompaction(UUID sessionID,
+ Collection<SSTableReader> sstables,
+ RangesAtEndpoint ranges)
+ {
+ List<Range<Token>> normalizedRanges = Range.normalize(ranges.ranges());
+ for (SSTableReader sstable : sstables)
+ {
+ Bounds<Token> bounds = new Bounds<>(sstable.first.getToken(), sstable.last.getToken());
+
+ if (!Iterables.any(normalizedRanges, r -> (r.contains(bounds.left) && r.contains(bounds.right)) || r.intersects(bounds)))
+ {
+ // this should never happen - in PendingAntiCompaction#getSSTables we select all sstables that intersect the repaired ranges, that can't have changed here
+ String message = String.format("%s SSTable %s (%s) does not intersect repaired ranges %s, this sstable should not have been included.",
+ PreviewKind.NONE.logPrefix(sessionID), sstable, bounds, normalizedRanges);
+ logger.error(message);
+ throw new IllegalStateException(message);
+ }
+ }
+
+ }
+
+ @VisibleForTesting
+ static Set<SSTableReader> findSSTablesToAnticompact(Iterator<SSTableReader> sstableIterator, List<Range<Token>> normalizedRanges, UUID parentRepairSession)
+ {
+ Set<SSTableReader> fullyContainedSSTables = new HashSet<>();
+ while (sstableIterator.hasNext())
+ {
+ SSTableReader sstable = sstableIterator.next();
+
+ Bounds<Token> sstableBounds = new Bounds<>(sstable.first.getToken(), sstable.last.getToken());
+
+ for (Range<Token> r : normalizedRanges)
+ {
+ // ranges are normalized - no wrap around - if first and last are contained we know that all tokens are contained in the range
+ if (r.contains(sstable.first.getToken()) && r.contains(sstable.last.getToken()))
+ {
+ logger.info("{} SSTable {} fully contained in range {}, mutating repairedAt instead of anticompacting", PreviewKind.NONE.logPrefix(parentRepairSession), sstable, r);
+ fullyContainedSSTables.add(sstable);
+ sstableIterator.remove();
+ break;
+ }
+ else if (r.intersects(sstableBounds))
+ {
+ logger.info("{} SSTable {} ({}) will be anticompacted on range {}", PreviewKind.NONE.logPrefix(parentRepairSession), sstable, sstableBounds, r);
+ }
+ }
+ }
+ return fullyContainedSSTables;
}
public void performMaximal(final ColumnFamilyStore cfStore, boolean splitOutput)
@@ -769,14 +850,15 @@
FBUtilities.waitOnFutures(submitMaximal(cfStore, getDefaultGcBefore(cfStore, FBUtilities.nowInSeconds()), splitOutput));
}
+ @SuppressWarnings("resource") // the tasks are executed in parallel on the executor, making sure that they get closed
public List<Future<?>> submitMaximal(final ColumnFamilyStore cfStore, final int gcBefore, boolean splitOutput)
{
// here we compute the task off the compaction executor, so having that present doesn't
// confuse runWithCompactionsDisabled -- i.e., we don't want to deadlock ourselves, waiting
// for ourselves to finish/acknowledge cancellation before continuing.
- final Collection<AbstractCompactionTask> tasks = cfStore.getCompactionStrategyManager().getMaximalTasks(gcBefore, splitOutput);
+ CompactionTasks tasks = cfStore.getCompactionStrategyManager().getMaximalTasks(gcBefore, splitOutput);
- if (tasks == null)
+ if (tasks.isEmpty())
return Collections.emptyList();
List<Future<?>> futures = new ArrayList<>();
@@ -791,7 +873,7 @@
{
protected void runMayThrow()
{
- task.execute(metrics);
+ task.execute(active);
}
};
@@ -802,45 +884,42 @@
if (nonEmptyTasks > 1)
logger.info("Major compaction will not result in a single sstable - repaired and unrepaired data is kept separate and compaction runs per data_file_directory.");
-
return futures;
}
public void forceCompactionForTokenRange(ColumnFamilyStore cfStore, Collection<Range<Token>> ranges)
{
- Callable<Collection<AbstractCompactionTask>> taskCreator = () -> {
+ Callable<CompactionTasks> taskCreator = () -> {
Collection<SSTableReader> sstables = sstablesInBounds(cfStore, ranges);
if (sstables == null || sstables.isEmpty())
{
logger.debug("No sstables found for the provided token range");
- return null;
+ return CompactionTasks.empty();
}
return cfStore.getCompactionStrategyManager().getUserDefinedTasks(sstables, getDefaultGcBefore(cfStore, FBUtilities.nowInSeconds()));
};
- final Collection<AbstractCompactionTask> tasks = cfStore.runWithCompactionsDisabled(taskCreator, false, false);
-
- if (tasks == null)
- return;
-
- Runnable runnable = new WrappedRunnable()
+ try (CompactionTasks tasks = cfStore.runWithCompactionsDisabled(taskCreator,
+ (sstable) -> new Bounds<>(sstable.first.getToken(), sstable.last.getToken()).intersects(ranges),
+ false,
+ false,
+ false))
{
- protected void runMayThrow() throws Exception
+ if (tasks.isEmpty())
+ return;
+
+ Runnable runnable = new WrappedRunnable()
{
- try
+ protected void runMayThrow()
{
for (AbstractCompactionTask task : tasks)
if (task != null)
- task.execute(metrics);
+ task.execute(active);
}
- finally
- {
- FBUtilities.closeAll(tasks.stream().map(task -> task.transaction).collect(Collectors.toList()));
- }
- }
- };
+ };
- FBUtilities.waitOnFuture(executor.submitIfRunning(runnable, "force compaction for token range"));
+ FBUtilities.waitOnFuture(executor.submitIfRunning(runnable, "force compaction for token range"));
+ }
}
private static Collection<SSTableReader> sstablesInBounds(ColumnFamilyStore cfs, Collection<Range<Token>> tokenRangeCollection)
@@ -851,8 +930,21 @@
for (Range<Token> tokenRange : tokenRangeCollection)
{
- Iterable<SSTableReader> ssTableReaders = View.sstablesInBounds(tokenRange.left.minKeyBound(), tokenRange.right.maxKeyBound(), tree);
- Iterables.addAll(sstables, ssTableReaders);
+ if (!AbstractBounds.strictlyWrapsAround(tokenRange.left, tokenRange.right))
+ {
+ Iterable<SSTableReader> ssTableReaders = View.sstablesInBounds(tokenRange.left.minKeyBound(), tokenRange.right.maxKeyBound(), tree);
+ Iterables.addAll(sstables, ssTableReaders);
+ }
+ else
+ {
+ // Searching an interval tree will not return the correct results for a wrapping range
+ // so we have to unwrap it first
+ for (Range<Token> unwrappedRange : tokenRange.unwrap())
+ {
+ Iterable<SSTableReader> ssTableReaders = View.sstablesInBounds(unwrappedRange.left.minKeyBound(), unwrappedRange.right.maxKeyBound(), tree);
+ Iterables.addAll(sstables, ssTableReaders);
+ }
+ }
}
return sstables;
}
@@ -866,7 +958,7 @@
{
// extract keyspace and columnfamily name from filename
Descriptor desc = Descriptor.fromFilename(filename.trim());
- if (Schema.instance.getCFMetaData(desc) == null)
+ if (Schema.instance.getTableMetadataRef(desc) == null)
{
logger.warn("Schema does not exist for file {}. Skipping.", filename);
continue;
@@ -876,7 +968,7 @@
descriptors.put(cfs, cfs.getDirectories().find(new File(filename.trim()).getName()));
}
- List<Future<?>> futures = new ArrayList<>();
+ List<Future<?>> futures = new ArrayList<>(descriptors.size());
int nowInSec = FBUtilities.nowInSeconds();
for (ColumnFamilyStore cfs : descriptors.keySet())
futures.add(submitUserDefined(cfs, descriptors.get(cfs), getDefaultGcBefore(cfs, nowInSec)));
@@ -892,7 +984,7 @@
{
// extract keyspace and columnfamily name from filename
Descriptor desc = Descriptor.fromFilename(filename.trim());
- if (Schema.instance.getCFMetaData(desc) == null)
+ if (Schema.instance.getTableMetadataRef(desc) == null)
{
logger.warn("Schema does not exist for file {}. Skipping.", filename);
continue;
@@ -914,7 +1006,10 @@
{
ColumnFamilyStore cfs = entry.getKey();
Keyspace keyspace = cfs.keyspace;
- Collection<Range<Token>> ranges = StorageService.instance.getLocalRanges(keyspace.getName());
+ final RangesAtEndpoint replicas = StorageService.instance.getLocalReplicas(keyspace.getName());
+ final Set<Range<Token>> allRanges = replicas.ranges();
+ final Set<Range<Token>> transientRanges = replicas.onlyTransient().ranges();
+ final Set<Range<Token>> fullRanges = replicas.onlyFull().ranges();
boolean hasIndexes = cfs.indexManager.hasIndexes();
SSTableReader sstable = lookupSSTable(cfs, entry.getValue());
@@ -924,10 +1019,10 @@
}
else
{
- CleanupStrategy cleanupStrategy = CleanupStrategy.get(cfs, ranges, FBUtilities.nowInSeconds());
+ CleanupStrategy cleanupStrategy = CleanupStrategy.get(cfs, allRanges, transientRanges, sstable.isRepaired(), FBUtilities.nowInSeconds());
try (LifecycleTransaction txn = cfs.getTracker().tryModify(sstable, OperationType.CLEANUP))
{
- doCleanupOne(cfs, txn, cleanupStrategy, ranges, hasIndexes);
+ doCleanupOne(cfs, txn, cleanupStrategy, allRanges, fullRanges, transientRanges, hasIndexes);
}
catch (IOException e)
{
@@ -967,19 +1062,14 @@
}
else
{
- List<AbstractCompactionTask> tasks = cfs.getCompactionStrategyManager().getUserDefinedTasks(sstables, gcBefore);
- try
+ try (CompactionTasks tasks = cfs.getCompactionStrategyManager().getUserDefinedTasks(sstables, gcBefore))
{
for (AbstractCompactionTask task : tasks)
{
if (task != null)
- task.execute(metrics);
+ task.execute(active);
}
}
- finally
- {
- FBUtilities.closeAll(tasks.stream().map(task -> task.transaction).collect(Collectors.toList()));
- }
}
}
};
@@ -999,30 +1089,9 @@
return null;
}
- /**
- * Does not mutate data, so is not scheduled.
- */
- public Future<?> submitValidation(final ColumnFamilyStore cfStore, final Validator validator)
+ public Future<?> submitValidation(Callable<Object> validation)
{
- Callable<Object> callable = new Callable<Object>()
- {
- public Object call() throws IOException
- {
- try
- {
- doValidationCompaction(cfStore, validator);
- }
- catch (Throwable e)
- {
- // we need to inform the remote end of our failure, otherwise it will hang on repair forever
- validator.fail();
- throw e;
- }
- return this;
- }
- };
-
- return validationExecutor.submitIfRunning(callable, "validation");
+ return validationExecutor.submitIfRunning(validation, "validation");
}
/* Used in tests. */
@@ -1035,37 +1104,39 @@
}
}
- private void scrubOne(ColumnFamilyStore cfs, LifecycleTransaction modifier, boolean skipCorrupted, boolean checkData, boolean reinsertOverflowedTTL) throws IOException
+ @VisibleForTesting
+ void scrubOne(ColumnFamilyStore cfs, LifecycleTransaction modifier, boolean skipCorrupted, boolean checkData, boolean reinsertOverflowedTTL, ActiveCompactionsTracker activeCompactions)
{
CompactionInfo.Holder scrubInfo = null;
try (Scrubber scrubber = new Scrubber(cfs, modifier, skipCorrupted, checkData, reinsertOverflowedTTL))
{
scrubInfo = scrubber.getScrubInfo();
- metrics.beginCompaction(scrubInfo);
+ activeCompactions.beginCompaction(scrubInfo);
scrubber.scrub();
}
finally
{
if (scrubInfo != null)
- metrics.finishCompaction(scrubInfo);
+ activeCompactions.finishCompaction(scrubInfo);
}
}
- private void verifyOne(ColumnFamilyStore cfs, SSTableReader sstable, boolean extendedVerify) throws IOException
+ @VisibleForTesting
+ void verifyOne(ColumnFamilyStore cfs, SSTableReader sstable, Verifier.Options options, ActiveCompactionsTracker activeCompactions)
{
CompactionInfo.Holder verifyInfo = null;
- try (Verifier verifier = new Verifier(cfs, sstable, false))
+ try (Verifier verifier = new Verifier(cfs, sstable, false, options))
{
verifyInfo = verifier.getVerifyInfo();
- metrics.beginCompaction(verifyInfo);
- verifier.verify(extendedVerify);
+ activeCompactions.beginCompaction(verifyInfo);
+ verifier.verify();
}
finally
{
if (verifyInfo != null)
- metrics.finishCompaction(verifyInfo);
+ activeCompactions.finishCompaction(verifyInfo);
}
}
@@ -1132,18 +1203,24 @@
*
* @throws IOException
*/
- private void doCleanupOne(final ColumnFamilyStore cfs, LifecycleTransaction txn, CleanupStrategy cleanupStrategy, Collection<Range<Token>> ranges, boolean hasIndexes) throws IOException
+ private void doCleanupOne(final ColumnFamilyStore cfs,
+ LifecycleTransaction txn,
+ CleanupStrategy cleanupStrategy,
+ Collection<Range<Token>> allRanges,
+ Collection<Range<Token>> fullRanges,
+ Collection<Range<Token>> transientRanges,
+ boolean hasIndexes) throws IOException
{
assert !cfs.isIndex();
SSTableReader sstable = txn.onlyOne();
// if ranges is empty and no index, entire sstable is discarded
- if (!hasIndexes && !new Bounds<>(sstable.first.getToken(), sstable.last.getToken()).intersects(ranges))
+ if (!hasIndexes && !new Bounds<>(sstable.first.getToken(), sstable.last.getToken()).intersects(allRanges))
{
txn.obsoleteOriginals();
txn.finish();
- logger.info("SSTable {} ([{}, {}]) does not intersect the owned ranges ({}), dropping it", sstable, sstable.first.getToken(), sstable.last.getToken(), ranges);
+ logger.info("SSTable {} ([{}, {}]) does not intersect the owned ranges ({}), dropping it", sstable, sstable.first.getToken(), sstable.last.getToken(), allRanges);
return;
}
@@ -1151,7 +1228,7 @@
long totalkeysWritten = 0;
- long expectedBloomFilterSize = Math.max(cfs.metadata.params.minIndexInterval,
+ long expectedBloomFilterSize = Math.max(cfs.metadata().params.minIndexInterval,
SSTableReader.getApproximateKeyCount(txn.originals()));
if (logger.isTraceEnabled())
logger.trace("Expected bloom filter size : {}", expectedBloomFilterSize);
@@ -1168,20 +1245,17 @@
int nowInSec = FBUtilities.nowInSeconds();
try (SSTableRewriter writer = SSTableRewriter.construct(cfs, txn, false, sstable.maxDataAge);
- ISSTableScanner scanner = cleanupStrategy.getScanner(sstable, null);
+ ISSTableScanner scanner = cleanupStrategy.getScanner(sstable);
CompactionController controller = new CompactionController(cfs, txn.originals(), getDefaultGcBefore(cfs, nowInSec));
Refs<SSTableReader> refs = Refs.ref(Collections.singleton(sstable));
- CompactionIterator ci = new CompactionIterator(OperationType.CLEANUP, Collections.singletonList(scanner), controller, nowInSec, UUIDGen.getTimeUUID(), metrics))
+ CompactionIterator ci = new CompactionIterator(OperationType.CLEANUP, Collections.singletonList(scanner), controller, nowInSec, UUIDGen.getTimeUUID(), active))
{
- writer.switchWriter(createWriter(cfs, compactionFileLocation, expectedBloomFilterSize, sstable.getSSTableMetadata().repairedAt, sstable, txn));
+ StatsMetadata metadata = sstable.getSSTableMetadata();
+ writer.switchWriter(createWriter(cfs, compactionFileLocation, expectedBloomFilterSize, metadata.repairedAt, metadata.pendingRepair, metadata.isTransient, sstable, txn));
long lastBytesScanned = 0;
-
while (ci.hasNext())
{
- if (ci.isStopRequested())
- throw new CompactionInterruptedException(ci.getCompactionInfo());
-
try (UnfilteredRowIterator partition = ci.next();
UnfilteredRowIterator notCleaned = cleanupStrategy.cleanup(partition))
{
@@ -1245,19 +1319,29 @@
this.nowInSec = nowInSec;
}
- public static CleanupStrategy get(ColumnFamilyStore cfs, Collection<Range<Token>> ranges, int nowInSec)
+ public static CleanupStrategy get(ColumnFamilyStore cfs, Collection<Range<Token>> ranges, Collection<Range<Token>> transientRanges, boolean isRepaired, int nowInSec)
{
- return cfs.indexManager.hasIndexes()
- ? new Full(cfs, ranges, nowInSec)
- : new Bounded(cfs, ranges, nowInSec);
+ if (cfs.indexManager.hasIndexes())
+ {
+ if (!transientRanges.isEmpty())
+ {
+ //Shouldn't have been possible to create this situation
+ throw new AssertionError("Can't have indexes and transient ranges");
+ }
+ return new Full(cfs, ranges, nowInSec);
+ }
+ return new Bounded(cfs, ranges, transientRanges, isRepaired, nowInSec);
}
- public abstract ISSTableScanner getScanner(SSTableReader sstable, RateLimiter limiter);
+ public abstract ISSTableScanner getScanner(SSTableReader sstable);
public abstract UnfilteredRowIterator cleanup(UnfilteredRowIterator partition);
private static final class Bounded extends CleanupStrategy
{
- public Bounded(final ColumnFamilyStore cfs, Collection<Range<Token>> ranges, int nowInSec)
+ private final Collection<Range<Token>> transientRanges;
+ private final boolean isRepaired;
+
+ public Bounded(final ColumnFamilyStore cfs, Collection<Range<Token>> ranges, Collection<Range<Token>> transientRanges, boolean isRepaired, int nowInSec)
{
super(ranges, nowInSec);
instance.cacheCleanupExecutor.submit(new Runnable()
@@ -1268,12 +1352,23 @@
cfs.cleanupCache();
}
});
+ this.transientRanges = transientRanges;
+ this.isRepaired = isRepaired;
}
@Override
- public ISSTableScanner getScanner(SSTableReader sstable, RateLimiter limiter)
+ public ISSTableScanner getScanner(SSTableReader sstable)
{
- return sstable.getScanner(ranges, limiter);
+ //If transient replication is enabled and there are transient ranges
+ //then cleanup should remove any partitions that are repaired and in the transient range
+ //as they should already be synchronized at other full replicas.
+ //So just don't scan the portion of the table containing the repaired transient ranges
+ Collection<Range<Token>> rangesToScan = ranges;
+ if (isRepaired)
+ {
+ rangesToScan = Collections2.filter(ranges, range -> !transientRanges.contains(range));
+ }
+ return sstable.getScanner(rangesToScan);
}
@Override
@@ -1294,9 +1389,9 @@
}
@Override
- public ISSTableScanner getScanner(SSTableReader sstable, RateLimiter limiter)
+ public ISSTableScanner getScanner(SSTableReader sstable)
{
- return sstable.getScanner(limiter);
+ return sstable.getScanner();
}
@Override
@@ -1317,20 +1412,21 @@
File compactionFileLocation,
long expectedBloomFilterSize,
long repairedAt,
+ UUID pendingRepair,
+ boolean isTransient,
SSTableReader sstable,
LifecycleTransaction txn)
{
FileUtils.createDirectory(compactionFileLocation);
- SerializationHeader header = sstable.header;
- if (header == null)
- header = SerializationHeader.make(sstable.metadata, Collections.singleton(sstable));
return SSTableWriter.create(cfs.metadata,
- Descriptor.fromFilename(cfs.getSSTablePath(compactionFileLocation)),
+ cfs.newSSTableDescriptor(compactionFileLocation),
expectedBloomFilterSize,
repairedAt,
+ pendingRepair,
+ isTransient,
sstable.getSSTableLevel(),
- header,
+ sstable.header,
cfs.indexManager.listIndexes(),
txn);
}
@@ -1339,6 +1435,8 @@
File compactionFileLocation,
int expectedBloomFilterSize,
long repairedAt,
+ UUID pendingRepair,
+ boolean isTransient,
Collection<SSTableReader> sstables,
ILifecycleTransaction txn)
{
@@ -1358,249 +1456,92 @@
break;
}
}
- return SSTableWriter.create(Descriptor.fromFilename(cfs.getSSTablePath(compactionFileLocation)),
+ return SSTableWriter.create(cfs.newSSTableDescriptor(compactionFileLocation),
(long) expectedBloomFilterSize,
repairedAt,
+ pendingRepair,
+ isTransient,
cfs.metadata,
- new MetadataCollector(sstables, cfs.metadata.comparator, minLevel),
- SerializationHeader.make(cfs.metadata, sstables),
+ new MetadataCollector(sstables, cfs.metadata().comparator, minLevel),
+ SerializationHeader.make(cfs.metadata(), sstables),
cfs.indexManager.listIndexes(),
txn);
}
-
- /**
- * Performs a readonly "compaction" of all sstables in order to validate complete rows,
- * but without writing the merge result
- */
- @SuppressWarnings("resource")
- private void doValidationCompaction(ColumnFamilyStore cfs, Validator validator) throws IOException
- {
- // this isn't meant to be race-proof, because it's not -- it won't cause bugs for a CFS to be dropped
- // mid-validation, or to attempt to validate a droped CFS. this is just a best effort to avoid useless work,
- // particularly in the scenario where a validation is submitted before the drop, and there are compactions
- // started prior to the drop keeping some sstables alive. Since validationCompaction can run
- // concurrently with other compactions, it would otherwise go ahead and scan those again.
- if (!cfs.isValid())
- return;
-
- Refs<SSTableReader> sstables = null;
- try
- {
-
- int gcBefore;
- int nowInSec = FBUtilities.nowInSeconds();
- UUID parentRepairSessionId = validator.desc.parentSessionId;
- String snapshotName;
- boolean isGlobalSnapshotValidation = cfs.snapshotExists(parentRepairSessionId.toString());
- if (isGlobalSnapshotValidation)
- snapshotName = parentRepairSessionId.toString();
- else
- snapshotName = validator.desc.sessionId.toString();
- boolean isSnapshotValidation = cfs.snapshotExists(snapshotName);
-
- if (isSnapshotValidation)
- {
- // If there is a snapshot created for the session then read from there.
- // note that we populate the parent repair session when creating the snapshot, meaning the sstables in the snapshot are the ones we
- // are supposed to validate.
- sstables = cfs.getSnapshotSSTableReader(snapshotName);
-
-
- // Computing gcbefore based on the current time wouldn't be very good because we know each replica will execute
- // this at a different time (that's the whole purpose of repair with snaphsot). So instead we take the creation
- // time of the snapshot, which should give us roughtly the same time on each replica (roughtly being in that case
- // 'as good as in the non-snapshot' case)
- gcBefore = cfs.gcBefore((int)(cfs.getSnapshotCreationTime(snapshotName) / 1000));
- }
- else
- {
- // flush first so everyone is validating data that is as similar as possible
- StorageService.instance.forceKeyspaceFlush(cfs.keyspace.getName(), cfs.name);
- sstables = getSSTablesToValidate(cfs, validator);
- if (sstables == null)
- return; // this means the parent repair session was removed - the repair session failed on another node and we removed it
- if (validator.gcBefore > 0)
- gcBefore = validator.gcBefore;
- else
- gcBefore = getDefaultGcBefore(cfs, nowInSec);
- }
-
- // Create Merkle trees suitable to hold estimated partitions for the given ranges.
- // We blindly assume that a partition is evenly distributed on all sstables for now.
- MerkleTrees tree = createMerkleTrees(sstables, validator.desc.ranges, cfs);
- long start = System.nanoTime();
- try (AbstractCompactionStrategy.ScannerList scanners = cfs.getCompactionStrategyManager().getScanners(sstables, validator.desc.ranges);
- ValidationCompactionController controller = new ValidationCompactionController(cfs, gcBefore);
- CompactionIterator ci = new ValidationCompactionIterator(scanners.scanners, controller, nowInSec, metrics))
- {
- // validate the CF as we iterate over it
- validator.prepare(cfs, tree);
- while (ci.hasNext())
- {
- if (ci.isStopRequested())
- throw new CompactionInterruptedException(ci.getCompactionInfo());
- try (UnfilteredRowIterator partition = ci.next())
- {
- validator.add(partition);
- }
- }
- validator.complete();
- }
- finally
- {
- if (isSnapshotValidation && !isGlobalSnapshotValidation)
- {
- // we can only clear the snapshot if we are not doing a global snapshot validation (we then clear it once anticompaction
- // is done).
- cfs.clearSnapshot(snapshotName);
- }
- }
-
- if (logger.isDebugEnabled())
- {
- long duration = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start);
- logger.debug("Validation finished in {} msec, for {}",
- duration,
- validator.desc);
- }
- }
- finally
- {
- if (sstables != null)
- sstables.release();
- }
- }
-
- private static MerkleTrees createMerkleTrees(Iterable<SSTableReader> sstables, Collection<Range<Token>> ranges, ColumnFamilyStore cfs)
- {
- MerkleTrees tree = new MerkleTrees(cfs.getPartitioner());
- long allPartitions = 0;
- Map<Range<Token>, Long> rangePartitionCounts = Maps.newHashMapWithExpectedSize(ranges.size());
- for (Range<Token> range : ranges)
- {
- long numPartitions = 0;
- for (SSTableReader sstable : sstables)
- numPartitions += sstable.estimatedKeysForRanges(Collections.singleton(range));
- rangePartitionCounts.put(range, numPartitions);
- allPartitions += numPartitions;
- }
-
- for (Range<Token> range : ranges)
- {
- long numPartitions = rangePartitionCounts.get(range);
- double rangeOwningRatio = allPartitions > 0 ? (double)numPartitions / allPartitions : 0;
- // determine max tree depth proportional to range size to avoid blowing up memory with multiple tress,
- // capping at a configurable depth (default 18) to prevent large tree (CASSANDRA-11390, CASSANDRA-14096)
- int maxDepth = rangeOwningRatio > 0
- ? (int) Math.floor(Math.max(0.0, DatabaseDescriptor.getRepairSessionMaxTreeDepth() -
- Math.log(1 / rangeOwningRatio) / Math.log(2)))
- : 0;
-
- // determine tree depth from number of partitions, capping at max tree depth (CASSANDRA-5263)
- int depth = numPartitions > 0 ? (int) Math.min(Math.ceil(Math.log(numPartitions) / Math.log(2)), maxDepth) : 0;
- tree.addMerkleTree((int) Math.pow(2, depth), range);
- }
- if (logger.isDebugEnabled())
- {
- // MT serialize may take time
- logger.debug("Created {} merkle trees with merkle trees size {}, {} partitions, {} bytes", tree.ranges().size(), tree.size(), allPartitions, MerkleTrees.serializer.serializedSize(tree, 0));
- }
-
- return tree;
- }
-
- private synchronized Refs<SSTableReader> getSSTablesToValidate(ColumnFamilyStore cfs, Validator validator)
- {
- Refs<SSTableReader> sstables;
-
- ActiveRepairService.ParentRepairSession prs = ActiveRepairService.instance.getParentRepairSession(validator.desc.parentSessionId);
- if (prs == null)
- return null;
- Set<SSTableReader> sstablesToValidate = new HashSet<>();
- if (prs.isGlobal)
- prs.markSSTablesRepairing(cfs.metadata.cfId, validator.desc.parentSessionId);
- // note that we always grab all existing sstables for this - if we were to just grab the ones that
- // were marked as repairing, we would miss any ranges that were compacted away and this would cause us to overstream
- try (ColumnFamilyStore.RefViewFragment sstableCandidates = cfs.selectAndReference(View.select(SSTableSet.CANONICAL, (s) -> !prs.isIncremental || !s.isRepaired())))
- {
- for (SSTableReader sstable : sstableCandidates.sstables)
- {
- if (new Bounds<>(sstable.first.getToken(), sstable.last.getToken()).intersects(validator.desc.ranges))
- {
- sstablesToValidate.add(sstable);
- }
- }
-
- sstables = Refs.tryRef(sstablesToValidate);
- if (sstables == null)
- {
- logger.error("Could not reference sstables");
- throw new RuntimeException("Could not reference sstables");
- }
- }
-
- return sstables;
- }
-
/**
* Splits up an sstable into two new sstables. The first of the new tables will store repaired ranges, the second
* will store the non-repaired ranges. Once anticompation is completed, the original sstable is marked as compacted
* and subsequently deleted.
* @param cfs
- * @param repaired a transaction over the repaired sstables to anticompacy
- * @param ranges Repaired ranges to be placed into one of the new sstables. The repaired table will be tracked via
- * the {@link org.apache.cassandra.io.sstable.metadata.StatsMetadata#repairedAt} field.
+ * @param txn a transaction over the repaired sstables to anticompact
+ * @param ranges full and transient ranges to be placed into one of the new sstables. The repaired table will be tracked via
+ * the {@link org.apache.cassandra.io.sstable.metadata.StatsMetadata#pendingRepair} field.
+ * @param sessionID the repair session we're anti-compacting for
+ * @param isCancelled function that indicates if active anti-compaction should be canceled
*/
- private void doAntiCompaction(ColumnFamilyStore cfs, Collection<Range<Token>> ranges, LifecycleTransaction repaired, long repairedAt)
+ private void doAntiCompaction(ColumnFamilyStore cfs,
+ RangesAtEndpoint ranges,
+ LifecycleTransaction txn,
+ UUID pendingRepair,
+ BooleanSupplier isCancelled)
{
- int numAnticompact = repaired.originals().size();
- logger.info("Performing anticompaction on {} sstables", numAnticompact);
+ int originalCount = txn.originals().size();
+ logger.info("Performing anticompaction on {} sstables for {}", originalCount, pendingRepair);
//Group SSTables
- Collection<Collection<SSTableReader>> groupedSSTables = cfs.getCompactionStrategyManager().groupSSTablesForAntiCompaction(repaired.originals());
- // iterate over sstables to check if the repaired / unrepaired ranges intersect them.
+ Set<SSTableReader> sstables = txn.originals();
+
+ // Repairs can take place on both unrepaired (incremental + full) and repaired (full) data.
+ // Although anti-compaction could work on repaired sstables as well and would result in having more accurate
+ // repairedAt values for these, we still avoid anti-compacting already repaired sstables, as we currently don't
+ // make use of any actual repairedAt value and splitting up sstables just for that is not worth it at this point.
+ Set<SSTableReader> unrepairedSSTables = sstables.stream().filter((s) -> !s.isRepaired()).collect(Collectors.toSet());
+ cfs.metric.bytesAnticompacted.inc(SSTableReader.getTotalBytes(unrepairedSSTables));
+ Collection<Collection<SSTableReader>> groupedSSTables = cfs.getCompactionStrategyManager().groupSSTablesForAntiCompaction(unrepairedSSTables);
+
+ // iterate over sstables to check if the full / transient / unrepaired ranges intersect them.
int antiCompactedSSTableCount = 0;
for (Collection<SSTableReader> sstableGroup : groupedSSTables)
{
- try (LifecycleTransaction txn = repaired.split(sstableGroup))
+ try (LifecycleTransaction groupTxn = txn.split(sstableGroup))
{
- int antiCompacted = antiCompactGroup(cfs, ranges, txn, repairedAt);
+ int antiCompacted = antiCompactGroup(cfs, ranges, groupTxn, pendingRepair, isCancelled);
antiCompactedSSTableCount += antiCompacted;
}
}
-
- String format = "Anticompaction completed successfully, anticompacted from {} to {} sstable(s).";
- logger.info(format, numAnticompact, antiCompactedSSTableCount);
+ String format = "Anticompaction completed successfully, anticompacted from {} to {} sstable(s) for {}.";
+ logger.info(format, originalCount, antiCompactedSSTableCount, pendingRepair);
}
-
@VisibleForTesting
- int antiCompactGroup(ColumnFamilyStore cfs, Collection<Range<Token>> ranges,
- LifecycleTransaction anticompactionGroup, long repairedAt)
+ int antiCompactGroup(ColumnFamilyStore cfs,
+ RangesAtEndpoint ranges,
+ LifecycleTransaction txn,
+ UUID pendingRepair,
+ BooleanSupplier isCancelled)
{
+ Preconditions.checkArgument(!ranges.isEmpty(), "need at least one full or transient range");
long groupMaxDataAge = -1;
- for (Iterator<SSTableReader> i = anticompactionGroup.originals().iterator(); i.hasNext();)
+ for (Iterator<SSTableReader> i = txn.originals().iterator(); i.hasNext();)
{
SSTableReader sstable = i.next();
if (groupMaxDataAge < sstable.maxDataAge)
groupMaxDataAge = sstable.maxDataAge;
}
- if (anticompactionGroup.originals().size() == 0)
+ if (txn.originals().size() == 0)
{
logger.info("No valid anticompactions for this group, All sstables were compacted and are no longer available");
return 0;
}
- logger.info("Anticompacting {}", anticompactionGroup);
- Set<SSTableReader> sstableAsSet = anticompactionGroup.originals();
+ logger.info("Anticompacting {} in {}.{} for {}", txn.originals(), cfs.keyspace.getName(), cfs.getTableName(), pendingRepair);
+ Set<SSTableReader> sstableAsSet = txn.originals();
File destination = cfs.getDirectories().getWriteableLocationAsFile(cfs.getExpectedCompactedFileSize(sstableAsSet, OperationType.ANTICOMPACTION));
- long repairedKeyCount = 0;
- long unrepairedKeyCount = 0;
int nowInSec = FBUtilities.nowInSeconds();
+ RateLimiter limiter = getRateLimiter();
/**
* HACK WARNING
@@ -1637,82 +1578,122 @@
}
CompactionStrategyManager strategy = cfs.getCompactionStrategyManager();
- try (SharedTxn sharedTxn = new SharedTxn(anticompactionGroup);
- SSTableRewriter repairedSSTableWriter = SSTableRewriter.constructWithoutEarlyOpening(sharedTxn, false, groupMaxDataAge);
- SSTableRewriter unRepairedSSTableWriter = SSTableRewriter.constructWithoutEarlyOpening(sharedTxn, false, groupMaxDataAge);
- AbstractCompactionStrategy.ScannerList scanners = strategy.getScanners(anticompactionGroup.originals());
- CompactionController controller = new CompactionController(cfs, sstableAsSet, getDefaultGcBefore(cfs, nowInSec));
- CompactionIterator ci = new CompactionIterator(OperationType.ANTICOMPACTION, scanners.scanners, controller, nowInSec, UUIDGen.getTimeUUID(), metrics))
- {
- int expectedBloomFilterSize = Math.max(cfs.metadata.params.minIndexInterval, (int)(SSTableReader.getApproximateKeyCount(sstableAsSet)));
+ try (SharedTxn sharedTxn = new SharedTxn(txn);
+ SSTableRewriter fullWriter = SSTableRewriter.constructWithoutEarlyOpening(sharedTxn, false, groupMaxDataAge);
+ SSTableRewriter transWriter = SSTableRewriter.constructWithoutEarlyOpening(sharedTxn, false, groupMaxDataAge);
+ SSTableRewriter unrepairedWriter = SSTableRewriter.constructWithoutEarlyOpening(sharedTxn, false, groupMaxDataAge);
- repairedSSTableWriter.switchWriter(CompactionManager.createWriterForAntiCompaction(cfs, destination, expectedBloomFilterSize, repairedAt, sstableAsSet, sharedTxn));
- unRepairedSSTableWriter.switchWriter(CompactionManager.createWriterForAntiCompaction(cfs, destination, expectedBloomFilterSize, ActiveRepairService.UNREPAIRED_SSTABLE, sstableAsSet, sharedTxn));
- Range.OrderedRangeContainmentChecker containmentChecker = new Range.OrderedRangeContainmentChecker(ranges);
+ AbstractCompactionStrategy.ScannerList scanners = strategy.getScanners(txn.originals());
+ CompactionController controller = new CompactionController(cfs, sstableAsSet, getDefaultGcBefore(cfs, nowInSec));
+ CompactionIterator ci = getAntiCompactionIterator(scanners.scanners, controller, nowInSec, UUIDGen.getTimeUUID(), active, isCancelled))
+ {
+ int expectedBloomFilterSize = Math.max(cfs.metadata().params.minIndexInterval, (int)(SSTableReader.getApproximateKeyCount(sstableAsSet)));
+
+ fullWriter.switchWriter(CompactionManager.createWriterForAntiCompaction(cfs, destination, expectedBloomFilterSize, UNREPAIRED_SSTABLE, pendingRepair, false, sstableAsSet, txn));
+ transWriter.switchWriter(CompactionManager.createWriterForAntiCompaction(cfs, destination, expectedBloomFilterSize, UNREPAIRED_SSTABLE, pendingRepair, true, sstableAsSet, txn));
+ unrepairedWriter.switchWriter(CompactionManager.createWriterForAntiCompaction(cfs, destination, expectedBloomFilterSize, UNREPAIRED_SSTABLE, NO_PENDING_REPAIR, false, sstableAsSet, txn));
+
+ Predicate<Token> fullChecker = !ranges.onlyFull().isEmpty() ? new Range.OrderedRangeContainmentChecker(ranges.onlyFull().ranges()) : t -> false;
+ Predicate<Token> transChecker = !ranges.onlyTransient().isEmpty() ? new Range.OrderedRangeContainmentChecker(ranges.onlyTransient().ranges()) : t -> false;
+ double compressionRatio = scanners.getCompressionRatio();
+ if (compressionRatio == MetadataCollector.NO_COMPRESSION_RATIO)
+ compressionRatio = 1.0;
+
+ long lastBytesScanned = 0;
+
while (ci.hasNext())
{
try (UnfilteredRowIterator partition = ci.next())
{
- // if current range from sstable is repaired, save it into the new repaired sstable
- if (containmentChecker.contains(partition.partitionKey().getToken()))
+ Token token = partition.partitionKey().getToken();
+ // if this row is contained in the full or transient ranges, append it to the appropriate sstable
+ if (fullChecker.test(token))
{
- repairedSSTableWriter.append(partition);
- repairedKeyCount++;
+ fullWriter.append(partition);
}
- // otherwise save into the new 'non-repaired' table
+ else if (transChecker.test(token))
+ {
+ transWriter.append(partition);
+ }
else
{
- unRepairedSSTableWriter.append(partition);
- unrepairedKeyCount++;
+ // otherwise, append it to the unrepaired sstable
+ unrepairedWriter.append(partition);
}
+ long bytesScanned = scanners.getTotalBytesScanned();
+ compactionRateLimiterAcquire(limiter, bytesScanned, lastBytesScanned, compressionRatio);
+ lastBytesScanned = bytesScanned;
}
}
- List<SSTableReader> anticompactedSSTables = new ArrayList<>();
+ fullWriter.prepareToCommit();
+ transWriter.prepareToCommit();
+ unrepairedWriter.prepareToCommit();
+ txn.checkpoint();
+ txn.obsoleteOriginals();
+ txn.prepareToCommit();
- repairedSSTableWriter.setRepairedAt(repairedAt).prepareToCommit();
- unRepairedSSTableWriter.prepareToCommit();
- anticompactionGroup.checkpoint();
- anticompactionGroup.obsoleteOriginals();
- anticompactionGroup.prepareToCommit();
- anticompactedSSTables.addAll(repairedSSTableWriter.finished());
- anticompactedSSTables.addAll(unRepairedSSTableWriter.finished());
- repairedSSTableWriter.commit();
- unRepairedSSTableWriter.commit();
- Throwables.maybeFail(anticompactionGroup.commit(null));
+ List<SSTableReader> fullSSTables = new ArrayList<>(fullWriter.finished());
+ List<SSTableReader> transSSTables = new ArrayList<>(transWriter.finished());
+ List<SSTableReader> unrepairedSSTables = new ArrayList<>(unrepairedWriter.finished());
- logger.trace("Repaired {} keys out of {} for {}/{} in {}", repairedKeyCount,
- repairedKeyCount + unrepairedKeyCount,
- cfs.keyspace.getName(),
- cfs.getColumnFamilyName(),
- anticompactionGroup);
- return anticompactedSSTables.size();
+ fullWriter.commit();
+ transWriter.commit();
+ unrepairedWriter.commit();
+ txn.commit();
+ logger.info("Anticompacted {} in {}.{} to full = {}, transient = {}, unrepaired = {} for {}",
+ sstableAsSet,
+ cfs.keyspace.getName(),
+ cfs.getTableName(),
+ fullSSTables,
+ transSSTables,
+ unrepairedSSTables,
+ pendingRepair);
+ return fullSSTables.size() + transSSTables.size() + unrepairedSSTables.size();
}
catch (Throwable e)
{
- JVMStabilityInspector.inspectThrowable(e);
- logger.error("Error anticompacting " + anticompactionGroup, e);
+ if (e instanceof CompactionInterruptedException && isCancelled.getAsBoolean())
+ {
+ logger.info("Anticompaction has been canceled for session {}", pendingRepair);
+ logger.trace(e.getMessage(), e);
+ }
+ else
+ {
+ JVMStabilityInspector.inspectThrowable(e);
+ logger.error("Error anticompacting " + txn + " for " + pendingRepair, e);
+ }
+ throw e;
}
- return 0;
}
- /**
- * Is not scheduled, because it is performing disjoint work from sstable compaction.
- */
- public Future<?> submitIndexBuild(final SecondaryIndexBuilder builder)
+ @VisibleForTesting
+ public static CompactionIterator getAntiCompactionIterator(List<ISSTableScanner> scanners, CompactionController controller, int nowInSec, UUID timeUUID, ActiveCompactionsTracker activeCompactions, BooleanSupplier isCancelled)
+ {
+ return new CompactionIterator(OperationType.ANTICOMPACTION, scanners, controller, nowInSec, timeUUID, activeCompactions) {
+
+ public boolean isStopRequested()
+ {
+ return super.isStopRequested() || isCancelled.getAsBoolean();
+ }
+ };
+ }
+
+ @VisibleForTesting
+ ListenableFuture<?> submitIndexBuild(final SecondaryIndexBuilder builder, ActiveCompactionsTracker activeCompactions)
{
Runnable runnable = new Runnable()
{
public void run()
{
- metrics.beginCompaction(builder);
+ activeCompactions.beginCompaction(builder);
try
{
builder.build();
}
finally
{
- metrics.finishCompaction(builder);
+ activeCompactions.finishCompaction(builder);
}
}
};
@@ -1720,8 +1701,21 @@
return executor.submitIfRunning(runnable, "index build");
}
+ /**
+ * Is not scheduled, because it is performing disjoint work from sstable compaction.
+ */
+ public ListenableFuture<?> submitIndexBuild(final SecondaryIndexBuilder builder)
+ {
+ return submitIndexBuild(builder, active);
+ }
+
public Future<?> submitCacheWrite(final AutoSavingCache.Writer writer)
{
+ return submitCacheWrite(writer, active);
+ }
+
+ Future<?> submitCacheWrite(final AutoSavingCache.Writer writer, ActiveCompactionsTracker activeCompactions)
+ {
Runnable runnable = new Runnable()
{
public void run()
@@ -1733,14 +1727,14 @@
}
try
{
- metrics.beginCompaction(writer);
+ activeCompactions.beginCompaction(writer);
try
{
writer.saveCache();
}
finally
{
- metrics.finishCompaction(writer);
+ activeCompactions.finishCompaction(writer);
}
}
finally
@@ -1755,15 +1749,20 @@
public List<SSTableReader> runIndexSummaryRedistribution(IndexSummaryRedistribution redistribution) throws IOException
{
- metrics.beginCompaction(redistribution);
+ return runIndexSummaryRedistribution(redistribution, active);
+ }
+ @VisibleForTesting
+ List<SSTableReader> runIndexSummaryRedistribution(IndexSummaryRedistribution redistribution, ActiveCompactionsTracker activeCompactions) throws IOException
+ {
+ activeCompactions.beginCompaction(redistribution);
try
{
return redistribution.redistributeSummaries();
}
finally
{
- metrics.finishCompaction(redistribution);
+ activeCompactions.finishCompaction(redistribution);
}
}
@@ -1774,73 +1773,30 @@
return cfs.isIndex() ? nowInSec : cfs.gcBefore(nowInSec);
}
- private static class ValidationCompactionIterator extends CompactionIterator
+ public ListenableFuture<Long> submitViewBuilder(final ViewBuilderTask task)
{
- public ValidationCompactionIterator(List<ISSTableScanner> scanners, ValidationCompactionController controller, int nowInSec, CompactionMetrics metrics)
- {
- super(OperationType.VALIDATION, scanners, controller, nowInSec, UUIDGen.getTimeUUID(), metrics);
- }
+ return submitViewBuilder(task, active);
}
- /*
- * Controller for validation compaction that always purges.
- * Note that we should not call cfs.getOverlappingSSTables on the provided
- * sstables because those sstables are not guaranteed to be active sstables
- * (since we can run repair on a snapshot).
- */
- private static class ValidationCompactionController extends CompactionController
+ @VisibleForTesting
+ ListenableFuture<Long> submitViewBuilder(final ViewBuilderTask task, ActiveCompactionsTracker activeCompactions)
{
- public ValidationCompactionController(ColumnFamilyStore cfs, int gcBefore)
- {
- super(cfs, gcBefore);
- }
-
- @Override
- public Predicate<Long> getPurgeEvaluator(DecoratedKey key)
- {
- /*
- * The main reason we always purge is that including gcable tombstone would mean that the
- * repair digest will depends on the scheduling of compaction on the different nodes. This
- * is still not perfect because gcbefore is currently dependend on the current time at which
- * the validation compaction start, which while not too bad for normal repair is broken for
- * repair on snapshots. A better solution would be to agree on a gcbefore that all node would
- * use, and we'll do that with CASSANDRA-4932.
- * Note validation compaction includes all sstables, so we don't have the problem of purging
- * a tombstone that could shadow a column in another sstable, but this is doubly not a concern
- * since validation compaction is read-only.
- */
- return time -> true;
- }
- }
-
- public Future<?> submitViewBuilder(final ViewBuilder builder)
- {
- Runnable runnable = new Runnable()
- {
- public void run()
+ return viewBuildExecutor.submitIfRunning(() -> {
+ activeCompactions.beginCompaction(task);
+ try
{
- metrics.beginCompaction(builder);
- try
- {
- builder.run();
- }
- finally
- {
- metrics.finishCompaction(builder);
- }
+ return task.call();
}
- };
- if (executor.isShutdown())
- {
- logger.info("Compaction executor has shut down, not submitting index build");
- return null;
- }
-
- return executor.submit(runnable);
+ finally
+ {
+ activeCompactions.finishCompaction(task);
+ }
+ }, "view build");
}
+
public int getActiveCompactions()
{
- return CompactionMetrics.getCompactions().size();
+ return active.getCompactions().size();
}
static class CompactionExecutor extends JMXEnabledThreadPoolExecutor
@@ -1912,7 +1868,7 @@
* @return the future that will deliver the task result, or a future that has already been
* cancelled if the task could not be submitted.
*/
- public ListenableFuture<?> submitIfRunning(Callable<?> task, String name)
+ public <T> ListenableFuture<T> submitIfRunning(Callable<T> task, String name)
{
if (isShutdown())
{
@@ -1922,7 +1878,7 @@
try
{
- ListenableFutureTask ret = ListenableFutureTask.create(task);
+ ListenableFutureTask<T> ret = ListenableFutureTask.create(task);
execute(ret);
return ret;
}
@@ -1938,11 +1894,43 @@
}
}
- private static class ValidationExecutor extends CompactionExecutor
+ // TODO: pull out relevant parts of CompactionExecutor and move to ValidationManager
+ public static class ValidationExecutor extends CompactionExecutor
{
+ // CompactionExecutor, and by extension ValidationExecutor, use DebuggableThreadPoolExecutor's
+ // default RejectedExecutionHandler which blocks the submitting thread when the work queue is
+ // full. The calling thread in this case is AntiEntropyStage, so in most cases we don't actually
+ // want to block when the ValidationExecutor is saturated as this prevents progress on all
+ // repair tasks and may cause repair sessions to time out. Also, it can lead to references to
+ // heavyweight validation responses containing merkle trees being held for extended periods which
+ // increases GC pressure. Using LinkedBlockingQueue instead of the default SynchronousQueue allows
+ // tasks to be submitted without blocking the caller, but will always prefer queueing to creating
+ // new threads if the pool already has at least `corePoolSize` threads already running. For this
+ // reason we set corePoolSize to the maximum desired concurrency, but allow idle core threads to
+ // be terminated.
+
public ValidationExecutor()
{
- super(1, Integer.MAX_VALUE, "ValidationExecutor", new SynchronousQueue<Runnable>());
+ super(DatabaseDescriptor.getConcurrentValidations(),
+ DatabaseDescriptor.getConcurrentValidations(),
+ "ValidationExecutor",
+ new LinkedBlockingQueue());
+
+ allowCoreThreadTimeOut(true);
+ }
+
+ public void adjustPoolSize()
+ {
+ setMaximumPoolSize(DatabaseDescriptor.getConcurrentValidations());
+ setCorePoolSize(DatabaseDescriptor.getConcurrentValidations());
+ }
+ }
+
+ private static class ViewBuildExecutor extends CompactionExecutor
+ {
+ public ViewBuildExecutor()
+ {
+ super(DatabaseDescriptor.getConcurrentViewBuilders(), "ViewBuildExecutor");
}
}
@@ -1954,16 +1942,25 @@
}
}
- public interface CompactionExecutorStatsCollector
+ public void incrementAborted()
{
- void beginCompaction(CompactionInfo.Holder ci);
-
- void finishCompaction(CompactionInfo.Holder ci);
+ metrics.compactionsAborted.inc();
}
+ public void incrementCompactionsReduced()
+ {
+ metrics.compactionsReduced.inc();
+ }
+
+ public void incrementSstablesDropppedFromCompactions(long num)
+ {
+ metrics.sstablesDropppedFromCompactions.inc(num);
+ }
+
+
public List<Map<String, String>> getCompactions()
{
- List<Holder> compactionHolders = CompactionMetrics.getCompactions();
+ List<Holder> compactionHolders = active.getCompactions();
List<Map<String, String>> out = new ArrayList<Map<String, String>>(compactionHolders.size());
for (CompactionInfo.Holder ci : compactionHolders)
out.add(ci.getCompactionInfo().asMap());
@@ -1972,7 +1969,7 @@
public List<String> getCompactionSummary()
{
- List<Holder> compactionHolders = CompactionMetrics.getCompactions();
+ List<Holder> compactionHolders = active.getCompactions();
List<String> out = new ArrayList<String>(compactionHolders.size());
for (CompactionInfo.Holder ci : compactionHolders)
out.add(ci.getCompactionInfo().toString());
@@ -2014,7 +2011,7 @@
public void stopCompaction(String type)
{
OperationType operation = OperationType.valueOf(type);
- for (Holder holder : CompactionMetrics.getCompactions())
+ for (Holder holder : active.getCompactions())
{
if (holder.getCompactionInfo().getTaskType() == operation)
holder.stop();
@@ -2023,9 +2020,9 @@
public void stopCompactionById(String compactionId)
{
- for (Holder holder : CompactionMetrics.getCompactions())
+ for (Holder holder : active.getCompactions())
{
- UUID holderId = holder.getCompactionInfo().compactionId();
+ UUID holderId = holder.getCompactionInfo().getTaskId();
if (holderId != null && holderId.equals(UUID.fromString(compactionId)))
holder.stop();
}
@@ -2047,6 +2044,27 @@
}
}
+ public void setConcurrentValidations()
+ {
+ validationExecutor.adjustPoolSize();
+ }
+
+ public void setConcurrentViewBuilders(int value)
+ {
+ if (value > viewBuildExecutor.getCorePoolSize())
+ {
+ // we are increasing the value
+ viewBuildExecutor.setMaximumPoolSize(value);
+ viewBuildExecutor.setCorePoolSize(value);
+ }
+ else if (value < viewBuildExecutor.getCorePoolSize())
+ {
+ // we are reducing the value
+ viewBuildExecutor.setCorePoolSize(value);
+ viewBuildExecutor.setMaximumPoolSize(value);
+ }
+ }
+
public int getCoreCompactorThreads()
{
return executor.getCorePoolSize();
@@ -2087,6 +2105,65 @@
validationExecutor.setMaximumPoolSize(number);
}
+ public boolean getDisableSTCSInL0()
+ {
+ return DatabaseDescriptor.getDisableSTCSInL0();
+ }
+
+ public void setDisableSTCSInL0(boolean disabled)
+ {
+ if (disabled != DatabaseDescriptor.getDisableSTCSInL0())
+ logger.info("Changing STCS in L0 disabled from {} to {}", DatabaseDescriptor.getDisableSTCSInL0(), disabled);
+ DatabaseDescriptor.setDisableSTCSInL0(disabled);
+ }
+
+ public int getCoreViewBuildThreads()
+ {
+ return viewBuildExecutor.getCorePoolSize();
+ }
+
+ public void setCoreViewBuildThreads(int number)
+ {
+ viewBuildExecutor.setCorePoolSize(number);
+ }
+
+ public int getMaximumViewBuildThreads()
+ {
+ return viewBuildExecutor.getMaximumPoolSize();
+ }
+
+ public void setMaximumViewBuildThreads(int number)
+ {
+ viewBuildExecutor.setMaximumPoolSize(number);
+ }
+
+ public boolean getAutomaticSSTableUpgradeEnabled()
+ {
+ return DatabaseDescriptor.automaticSSTableUpgrade();
+ }
+
+ public void setAutomaticSSTableUpgradeEnabled(boolean enabled)
+ {
+ DatabaseDescriptor.setAutomaticSSTableUpgradeEnabled(enabled);
+ }
+
+ public int getMaxConcurrentAutoUpgradeTasks()
+ {
+ return DatabaseDescriptor.maxConcurrentAutoUpgradeTasks();
+ }
+
+ public void setMaxConcurrentAutoUpgradeTasks(int value)
+ {
+ try
+ {
+ DatabaseDescriptor.setMaxConcurrentAutoUpgradeTasks(value);
+ }
+ catch (ConfigurationException e)
+ {
+ throw new RuntimeException(e.getMessage());
+ }
+ }
+
/**
* Try to stop all of the compactions for given ColumnFamilies.
*
@@ -2094,48 +2171,63 @@
* isCompacting if you want that behavior.
*
* @param columnFamilies The ColumnFamilies to try to stop compaction upon.
+ * @param sstablePredicate the sstable predicate to match on
* @param interruptValidation true if validation operations for repair should also be interrupted
- *
*/
- public void interruptCompactionFor(Iterable<CFMetaData> columnFamilies, boolean interruptValidation)
+ public void interruptCompactionFor(Iterable<TableMetadata> columnFamilies, Predicate<SSTableReader> sstablePredicate, boolean interruptValidation)
{
assert columnFamilies != null;
// interrupt in-progress compactions
- for (Holder compactionHolder : CompactionMetrics.getCompactions())
+ for (Holder compactionHolder : active.getCompactions())
{
CompactionInfo info = compactionHolder.getCompactionInfo();
if ((info.getTaskType() == OperationType.VALIDATION) && !interruptValidation)
continue;
- // cfmetadata is null for index summary redistributions which are 'global' - they involve all keyspaces/tables
- if (info.getCFMetaData() == null || Iterables.contains(columnFamilies, info.getCFMetaData()))
- compactionHolder.stop(); // signal compaction to stop
+ if (info.getTableMetadata() == null || Iterables.contains(columnFamilies, info.getTableMetadata()))
+ {
+ if (info.shouldStop(sstablePredicate))
+ compactionHolder.stop();
+ }
}
}
- public void interruptCompactionForCFs(Iterable<ColumnFamilyStore> cfss, boolean interruptValidation)
+ public void interruptCompactionForCFs(Iterable<ColumnFamilyStore> cfss, Predicate<SSTableReader> sstablePredicate, boolean interruptValidation)
{
- List<CFMetaData> metadata = new ArrayList<>();
+ List<TableMetadata> metadata = new ArrayList<>();
for (ColumnFamilyStore cfs : cfss)
- metadata.add(cfs.metadata);
+ metadata.add(cfs.metadata());
- interruptCompactionFor(metadata, interruptValidation);
+ interruptCompactionFor(metadata, sstablePredicate, interruptValidation);
}
- public void waitForCessation(Iterable<ColumnFamilyStore> cfss)
+ public void waitForCessation(Iterable<ColumnFamilyStore> cfss, Predicate<SSTableReader> sstablePredicate)
{
long start = System.nanoTime();
long delay = TimeUnit.MINUTES.toNanos(1);
+
while (System.nanoTime() - start < delay)
{
- if (CompactionManager.instance.isCompacting(cfss))
+ if (CompactionManager.instance.isCompacting(cfss, sstablePredicate))
Uninterruptibles.sleepUninterruptibly(1, TimeUnit.MILLISECONDS);
else
break;
}
}
+
+ public List<CompactionInfo> getSSTableTasks()
+ {
+ return active.getCompactions()
+ .stream()
+ .map(CompactionInfo.Holder::getCompactionInfo)
+ .filter(task -> task.getTaskType() != OperationType.COUNTER_CACHE_SAVE
+ && task.getTaskType() != OperationType.KEY_CACHE_SAVE
+ && task.getTaskType() != OperationType.ROW_CACHE_SAVE)
+ .collect(Collectors.toList());
+ }
+
/**
* Return whether "global" compactions should be paused, used by ColumnFamilyStore#runWithCompactionsDisabled
*
diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionManagerMBean.java b/src/java/org/apache/cassandra/db/compaction/CompactionManagerMBean.java
index 8785b41..d298c8b 100644
--- a/src/java/org/apache/cassandra/db/compaction/CompactionManagerMBean.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionManagerMBean.java
@@ -116,4 +116,53 @@
* @param number New maximum of validator threads
*/
public void setMaximumValidatorThreads(int number);
+
+ /**
+ * Returns core size of view build thread pool
+ */
+ public int getCoreViewBuildThreads();
+
+ /**
+ * Enable / disable STCS in L0
+ */
+ public boolean getDisableSTCSInL0();
+ public void setDisableSTCSInL0(boolean disabled);
+
+ /**
+ * Allows user to resize maximum size of the view build thread pool.
+ * @param number New maximum of view build threads
+ */
+ public void setCoreViewBuildThreads(int number);
+
+ /**
+ * Returns size of view build thread pool
+ */
+ public int getMaximumViewBuildThreads();
+
+ /**
+ * Allows user to resize maximum size of the view build thread pool.
+ * @param number New maximum of view build threads
+ */
+ public void setMaximumViewBuildThreads(int number);
+
+ /**
+ * Get automatic sstable upgrade enabled
+ */
+ public boolean getAutomaticSSTableUpgradeEnabled();
+ /**
+ * Set if automatic sstable upgrade should be enabled
+ */
+ public void setAutomaticSSTableUpgradeEnabled(boolean enabled);
+
+ /**
+ * Get the number of concurrent sstable upgrade tasks we should run
+ * when automatic sstable upgrades are enabled
+ */
+ public int getMaxConcurrentAutoUpgradeTasks();
+
+ /**
+ * Set the number of concurrent sstable upgrade tasks we should run
+ * when automatic sstable upgrades are enabled
+ */
+ public void setMaxConcurrentAutoUpgradeTasks(int value);
}
diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionStrategyHolder.java b/src/java/org/apache/cassandra/db/compaction/CompactionStrategyHolder.java
new file mode 100644
index 0000000..129ee79
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionStrategyHolder.java
@@ -0,0 +1,265 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.UUID;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Iterables;
+
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.SerializationHeader;
+import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.index.Index;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.ISSTableScanner;
+import org.apache.cassandra.io.sstable.SSTableMultiWriter;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
+import org.apache.cassandra.schema.CompactionParams;
+import org.apache.cassandra.service.ActiveRepairService;
+
+public class CompactionStrategyHolder extends AbstractStrategyHolder
+{
+ private final List<AbstractCompactionStrategy> strategies = new ArrayList<>();
+ private final boolean isRepaired;
+
+ public CompactionStrategyHolder(ColumnFamilyStore cfs, DestinationRouter router, boolean isRepaired)
+ {
+ super(cfs, router);
+ this.isRepaired = isRepaired;
+ }
+
+ @Override
+ public void startup()
+ {
+ strategies.forEach(AbstractCompactionStrategy::startup);
+ }
+
+ @Override
+ public void shutdown()
+ {
+ strategies.forEach(AbstractCompactionStrategy::shutdown);
+ }
+
+ @Override
+ public void setStrategyInternal(CompactionParams params, int numTokenPartitions)
+ {
+ strategies.clear();
+ for (int i = 0; i < numTokenPartitions; i++)
+ strategies.add(cfs.createCompactionStrategyInstance(params));
+ }
+
+ @Override
+ public boolean managesRepairedGroup(boolean isRepaired, boolean isPendingRepair, boolean isTransient)
+ {
+ if (!isPendingRepair)
+ {
+ Preconditions.checkArgument(!isTransient, "isTransient can only be true for sstables pending repairs");
+ return this.isRepaired == isRepaired;
+ }
+ else
+ {
+ Preconditions.checkArgument(!isRepaired, "SSTables cannot be both repaired and pending repair");
+ return false;
+
+ }
+ }
+
+ @Override
+ public AbstractCompactionStrategy getStrategyFor(SSTableReader sstable)
+ {
+ Preconditions.checkArgument(managesSSTable(sstable), "Attempting to get compaction strategy from wrong holder");
+ return strategies.get(router.getIndexForSSTable(sstable));
+ }
+
+ @Override
+ public Iterable<AbstractCompactionStrategy> allStrategies()
+ {
+ return strategies;
+ }
+
+ @Override
+ public Collection<TaskSupplier> getBackgroundTaskSuppliers(int gcBefore)
+ {
+ List<TaskSupplier> suppliers = new ArrayList<>(strategies.size());
+ for (AbstractCompactionStrategy strategy : strategies)
+ suppliers.add(new TaskSupplier(strategy.getEstimatedRemainingTasks(), () -> strategy.getNextBackgroundTask(gcBefore)));
+
+ return suppliers;
+ }
+
+ @Override
+ public Collection<AbstractCompactionTask> getMaximalTasks(int gcBefore, boolean splitOutput)
+ {
+ List<AbstractCompactionTask> tasks = new ArrayList<>(strategies.size());
+ for (AbstractCompactionStrategy strategy : strategies)
+ {
+ Collection<AbstractCompactionTask> task = strategy.getMaximalTask(gcBefore, splitOutput);
+ if (task != null)
+ tasks.addAll(task);
+ }
+ return tasks;
+ }
+
+ @Override
+ public Collection<AbstractCompactionTask> getUserDefinedTasks(GroupedSSTableContainer sstables, int gcBefore)
+ {
+ List<AbstractCompactionTask> tasks = new ArrayList<>(strategies.size());
+ for (int i = 0; i < strategies.size(); i++)
+ {
+ if (sstables.isGroupEmpty(i))
+ continue;
+
+ tasks.add(strategies.get(i).getUserDefinedTask(sstables.getGroup(i), gcBefore));
+ }
+ return tasks;
+ }
+
+ @Override
+ public void addSSTables(GroupedSSTableContainer sstables)
+ {
+ Preconditions.checkArgument(sstables.numGroups() == strategies.size());
+ for (int i = 0; i < strategies.size(); i++)
+ {
+ if (!sstables.isGroupEmpty(i))
+ strategies.get(i).addSSTables(sstables.getGroup(i));
+ }
+ }
+
+ @Override
+ public void removeSSTables(GroupedSSTableContainer sstables)
+ {
+ Preconditions.checkArgument(sstables.numGroups() == strategies.size());
+ for (int i = 0; i < strategies.size(); i++)
+ {
+ if (!sstables.isGroupEmpty(i))
+ strategies.get(i).removeSSTables(sstables.getGroup(i));
+ }
+ }
+
+ @Override
+ public void replaceSSTables(GroupedSSTableContainer removed, GroupedSSTableContainer added)
+ {
+ Preconditions.checkArgument(removed.numGroups() == strategies.size());
+ Preconditions.checkArgument(added.numGroups() == strategies.size());
+ for (int i = 0; i < strategies.size(); i++)
+ {
+ if (removed.isGroupEmpty(i) && added.isGroupEmpty(i))
+ continue;
+
+ if (removed.isGroupEmpty(i))
+ strategies.get(i).addSSTables(added.getGroup(i));
+ else
+ strategies.get(i).replaceSSTables(removed.getGroup(i), added.getGroup(i));
+ }
+ }
+
+ public AbstractCompactionStrategy first()
+ {
+ return strategies.get(0);
+ }
+
+ @Override
+ @SuppressWarnings("resource")
+ public List<ISSTableScanner> getScanners(GroupedSSTableContainer sstables, Collection<Range<Token>> ranges)
+ {
+ List<ISSTableScanner> scanners = new ArrayList<>(strategies.size());
+ for (int i = 0; i < strategies.size(); i++)
+ {
+ if (sstables.isGroupEmpty(i))
+ continue;
+
+ scanners.addAll(strategies.get(i).getScanners(sstables.getGroup(i), ranges).scanners);
+ }
+ return scanners;
+ }
+
+ Collection<Collection<SSTableReader>> groupForAnticompaction(Iterable<SSTableReader> sstables)
+ {
+ Preconditions.checkState(!isRepaired);
+ GroupedSSTableContainer group = createGroupedSSTableContainer();
+ sstables.forEach(group::add);
+
+ Collection<Collection<SSTableReader>> anticompactionGroups = new ArrayList<>();
+ for (int i = 0; i < strategies.size(); i++)
+ {
+ if (group.isGroupEmpty(i))
+ continue;
+
+ anticompactionGroups.addAll(strategies.get(i).groupSSTablesForAntiCompaction(group.getGroup(i)));
+ }
+
+ return anticompactionGroups;
+ }
+
+ @Override
+ public SSTableMultiWriter createSSTableMultiWriter(Descriptor descriptor,
+ long keyCount,
+ long repairedAt,
+ UUID pendingRepair,
+ boolean isTransient,
+ MetadataCollector collector,
+ SerializationHeader header,
+ Collection<Index> indexes,
+ LifecycleNewTracker lifecycleNewTracker)
+ {
+ if (isRepaired)
+ {
+ Preconditions.checkArgument(repairedAt != ActiveRepairService.UNREPAIRED_SSTABLE,
+ "Repaired CompactionStrategyHolder can't create unrepaired sstable writers");
+ }
+ else
+ {
+ Preconditions.checkArgument(repairedAt == ActiveRepairService.UNREPAIRED_SSTABLE,
+ "Unrepaired CompactionStrategyHolder can't create repaired sstable writers");
+ }
+ Preconditions.checkArgument(pendingRepair == null,
+ "CompactionStrategyHolder can't create sstable writer with pendingRepair id");
+ // to avoid creating a compaction strategy for the wrong pending repair manager, we get the index based on where the sstable is to be written
+ AbstractCompactionStrategy strategy = strategies.get(router.getIndexForSSTableDirectory(descriptor));
+ return strategy.createSSTableMultiWriter(descriptor,
+ keyCount,
+ repairedAt,
+ pendingRepair,
+ isTransient,
+ collector,
+ header,
+ indexes,
+ lifecycleNewTracker);
+ }
+
+ @Override
+ public int getStrategyIndex(AbstractCompactionStrategy strategy)
+ {
+ return strategies.indexOf(strategy);
+ }
+
+ @Override
+ public boolean containsSSTable(SSTableReader sstable)
+ {
+ return Iterables.any(strategies, acs -> acs.getSSTables().contains(sstable));
+ }
+}
diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionStrategyManager.java b/src/java/org/apache/cassandra/db/compaction/CompactionStrategyManager.java
index 86170a1..546f61b 100644
--- a/src/java/org/apache/cassandra/db/compaction/CompactionStrategyManager.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionStrategyManager.java
@@ -18,46 +18,71 @@
package org.apache.cassandra.db.compaction;
-import java.util.*;
-import java.util.concurrent.Callable;
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.ConcurrentModificationException;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Objects;
+import java.util.Set;
+import java.util.UUID;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.function.Supplier;
import java.util.stream.Collectors;
-import java.util.stream.Stream;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableList;
import com.google.common.collect.Iterables;
-import com.google.common.primitives.Ints;
-
+import com.google.common.collect.Lists;
+import com.google.common.primitives.Longs;
import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Directories;
import org.apache.cassandra.db.DiskBoundaries;
-import org.apache.cassandra.db.Memtable;
import org.apache.cassandra.db.SerializationHeader;
+import org.apache.cassandra.db.compaction.AbstractStrategyHolder.TaskSupplier;
import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
import org.apache.cassandra.db.lifecycle.SSTableSet;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.index.Index;
+import org.apache.cassandra.io.sstable.Component;
import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.ISSTableScanner;
+import org.apache.cassandra.io.sstable.SSTable;
import org.apache.cassandra.io.sstable.SSTableMultiWriter;
import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.io.sstable.ISSTableScanner;
import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
-import org.apache.cassandra.notifications.*;
+import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
+import org.apache.cassandra.notifications.INotification;
+import org.apache.cassandra.notifications.INotificationConsumer;
+import org.apache.cassandra.notifications.SSTableAddedNotification;
+import org.apache.cassandra.notifications.SSTableDeletingNotification;
+import org.apache.cassandra.notifications.SSTableListChangedNotification;
+import org.apache.cassandra.notifications.SSTableMetadataChanged;
+import org.apache.cassandra.notifications.SSTableRepairStatusChanged;
import org.apache.cassandra.schema.CompactionParams;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.service.ActiveRepairService;
+import static org.apache.cassandra.db.compaction.AbstractStrategyHolder.GroupedSSTableContainer;
+
/**
* Manages the compaction strategies.
*
- * Currently has two instances of actual compaction strategies per data directory - one for repaired data and one for
- * unrepaired data. This is done to be able to totally separate the different sets of sstables.
+ * SSTables are isolated from each other based on their incremental repair status (repaired, unrepaired, or pending repair)
+ * and directory (determined by their starting token). This class handles the routing between {@link AbstractStrategyHolder}
+ * instances based on repair status, and the {@link AbstractStrategyHolder} instances have separate compaction strategies
+ * for each directory, which it routes sstables to. Note that {@link PendingRepairHolder} also divides sstables on their
+ * pending repair id.
*
* Operations on this class are guarded by a {@link ReentrantReadWriteLock}. This lock performs mutual exclusion on
* reads and writes to the following variables: {@link this#repaired}, {@link this#unrepaired}, {@link this#isActive},
@@ -70,6 +95,7 @@
* before acquiring the read lock to acess the strategies.
*
*/
+
public class CompactionStrategyManager implements INotificationConsumer
{
private static final Logger logger = LoggerFactory.getLogger(CompactionStrategyManager.class);
@@ -88,21 +114,25 @@
/**
* Variables guarded by read and write lock above
*/
- //TODO check possibility of getting rid of these locks by encapsulating these in an immutable atomic object
- private final List<AbstractCompactionStrategy> repaired = new ArrayList<>();
- private final List<AbstractCompactionStrategy> unrepaired = new ArrayList<>();
+ private final PendingRepairHolder transientRepairs;
+ private final PendingRepairHolder pendingRepairs;
+ private final CompactionStrategyHolder repaired;
+ private final CompactionStrategyHolder unrepaired;
+
+ private final ImmutableList<AbstractStrategyHolder> holders;
+
private volatile CompactionParams params;
private DiskBoundaries currentBoundaries;
private volatile boolean enabled = true;
private volatile boolean isActive = true;
- /**
+ /*
We keep a copy of the schema compaction parameters here to be able to decide if we
- should update the compaction strategy in {@link this#maybeReload(CFMetaData)} due to an ALTER.
+ should update the compaction strategy in maybeReload() due to an ALTER.
If a user changes the local compaction strategy and then later ALTERs a compaction parameter,
we will use the new compaction parameters.
- **/
+ */
private volatile CompactionParams schemaCompactionParams;
private boolean shouldDefragment;
private boolean supportsEarlyOpen;
@@ -117,22 +147,39 @@
public CompactionStrategyManager(ColumnFamilyStore cfs, Supplier<DiskBoundaries> boundariesSupplier,
boolean partitionSSTablesByTokenRange)
{
+ AbstractStrategyHolder.DestinationRouter router = new AbstractStrategyHolder.DestinationRouter()
+ {
+ public int getIndexForSSTable(SSTableReader sstable)
+ {
+ return compactionStrategyIndexFor(sstable);
+ }
+
+ public int getIndexForSSTableDirectory(Descriptor descriptor)
+ {
+ return compactionStrategyIndexForDirectory(descriptor);
+ }
+ };
+ transientRepairs = new PendingRepairHolder(cfs, router, true);
+ pendingRepairs = new PendingRepairHolder(cfs, router, false);
+ repaired = new CompactionStrategyHolder(cfs, router, true);
+ unrepaired = new CompactionStrategyHolder(cfs, router, false);
+ holders = ImmutableList.of(transientRepairs, pendingRepairs, repaired, unrepaired);
+
cfs.getTracker().subscribe(this);
logger.trace("{} subscribed to the data tracker.", this);
this.cfs = cfs;
this.compactionLogger = new CompactionLogger(cfs, this);
this.boundariesSupplier = boundariesSupplier;
this.partitionSSTablesByTokenRange = partitionSSTablesByTokenRange;
- params = cfs.metadata.params.compaction;
+ params = cfs.metadata().params.compaction;
enabled = params.isEnabled();
- reload(cfs.metadata.params.compaction);
+ reload(cfs.metadata().params.compaction);
}
/**
* Return the next background task
*
* Returns a task for the compaction strategy that needs it the most (most estimated remaining tasks)
- *
*/
public AbstractCompactionTask getNextBackgroundTask(int gcBefore)
{
@@ -143,22 +190,69 @@
if (!isEnabled())
return null;
- List<AbstractCompactionStrategy> strategies = new ArrayList<>();
+ int numPartitions = getNumTokenPartitions();
- strategies.addAll(repaired);
- strategies.addAll(unrepaired);
- Collections.sort(strategies, (o1, o2) -> Ints.compare(o2.getEstimatedRemainingTasks(), o1.getEstimatedRemainingTasks()));
- for (AbstractCompactionStrategy strategy : strategies)
+ // first try to promote/demote sstables from completed repairs
+ AbstractCompactionTask repairFinishedTask;
+ repairFinishedTask = pendingRepairs.getNextRepairFinishedTask();
+ if (repairFinishedTask != null)
+ return repairFinishedTask;
+
+ repairFinishedTask = transientRepairs.getNextRepairFinishedTask();
+ if (repairFinishedTask != null)
+ return repairFinishedTask;
+
+ // sort compaction task suppliers by remaining tasks descending
+ List<TaskSupplier> suppliers = new ArrayList<>(numPartitions * holders.size());
+ for (AbstractStrategyHolder holder : holders)
+ suppliers.addAll(holder.getBackgroundTaskSuppliers(gcBefore));
+
+ Collections.sort(suppliers);
+
+ // return the first non-null task
+ for (TaskSupplier supplier : suppliers)
{
- AbstractCompactionTask task = strategy.getNextBackgroundTask(gcBefore);
+ AbstractCompactionTask task = supplier.getTask();
if (task != null)
return task;
}
+
+ return null;
}
finally
{
readLock.unlock();
}
+ }
+
+ /**
+ * finds the oldest (by modification date) non-latest-version sstable on disk and creates an upgrade task for it
+ * @return
+ */
+ @VisibleForTesting
+ @SuppressWarnings("resource") // transaction is closed by AbstractCompactionTask::execute
+ AbstractCompactionTask findUpgradeSSTableTask()
+ {
+ if (!isEnabled() || !DatabaseDescriptor.automaticSSTableUpgrade())
+ return null;
+ Set<SSTableReader> compacting = cfs.getTracker().getCompacting();
+ List<SSTableReader> potentialUpgrade = cfs.getLiveSSTables()
+ .stream()
+ .filter(s -> !compacting.contains(s) && !s.descriptor.version.isLatestVersion())
+ .sorted((o1, o2) -> {
+ File f1 = new File(o1.descriptor.filenameFor(Component.DATA));
+ File f2 = new File(o2.descriptor.filenameFor(Component.DATA));
+ return Longs.compare(f1.lastModified(), f2.lastModified());
+ }).collect(Collectors.toList());
+ for (SSTableReader sstable : potentialUpgrade)
+ {
+ LifecycleTransaction txn = cfs.getTracker().tryModify(sstable, OperationType.UPGRADE_SSTABLES);
+ if (txn != null)
+ {
+ logger.debug("Running automatic sstable upgrade for {}", sstable);
+ return getCompactionStrategyFor(sstable).getCompactionTask(txn, Integer.MIN_VALUE, Long.MAX_VALUE);
+ }
+ }
return null;
}
@@ -214,19 +308,17 @@
if (sstable.openReason != SSTableReader.OpenReason.EARLY)
compactionStrategyFor(sstable).addSSTable(sstable);
}
- repaired.forEach(AbstractCompactionStrategy::startup);
- unrepaired.forEach(AbstractCompactionStrategy::startup);
- shouldDefragment = repaired.get(0).shouldDefragment();
- supportsEarlyOpen = repaired.get(0).supportsEarlyOpen();
- fanout = (repaired.get(0) instanceof LeveledCompactionStrategy) ? ((LeveledCompactionStrategy) repaired.get(0)).getLevelFanoutSize() : LeveledCompactionStrategy.DEFAULT_LEVEL_FANOUT_SIZE;
+ holders.forEach(AbstractStrategyHolder::startup);
+ shouldDefragment = repaired.first().shouldDefragment();
+ supportsEarlyOpen = repaired.first().supportsEarlyOpen();
+ fanout = (repaired.first() instanceof LeveledCompactionStrategy) ? ((LeveledCompactionStrategy) repaired.first()).getLevelFanoutSize() : LeveledCompactionStrategy.DEFAULT_LEVEL_FANOUT_SIZE;
}
finally
{
writeLock.unlock();
}
- repaired.forEach(AbstractCompactionStrategy::startup);
- unrepaired.forEach(AbstractCompactionStrategy::startup);
- if (Stream.concat(repaired.stream(), unrepaired.stream()).anyMatch(cs -> cs.logAll))
+
+ if (repaired.first().logAll)
compactionLogger.enable();
}
@@ -237,24 +329,20 @@
* @param sstable
* @return
*/
- protected AbstractCompactionStrategy getCompactionStrategyFor(SSTableReader sstable)
+ public AbstractCompactionStrategy getCompactionStrategyFor(SSTableReader sstable)
{
maybeReloadDiskBoundaries();
return compactionStrategyFor(sstable);
}
@VisibleForTesting
- protected AbstractCompactionStrategy compactionStrategyFor(SSTableReader sstable)
+ AbstractCompactionStrategy compactionStrategyFor(SSTableReader sstable)
{
// should not call maybeReloadDiskBoundaries because it may be called from within lock
readLock.lock();
try
{
- int index = compactionStrategyIndexFor(sstable);
- if (sstable.isRepaired())
- return repaired.get(index);
- else
- return unrepaired.get(index);
+ return getHolder(sstable).getStrategyFor(sstable);
}
finally
{
@@ -274,9 +362,9 @@
* @return
*/
@VisibleForTesting
- protected int compactionStrategyIndexFor(SSTableReader sstable)
+ int compactionStrategyIndexFor(SSTableReader sstable)
{
- // should not call maybeReload because it may be called from within lock
+ // should not call maybeReloadDiskBoundaries because it may be called from within lock
readLock.lock();
try
{
@@ -293,14 +381,63 @@
}
}
+ private int compactionStrategyIndexForDirectory(Descriptor descriptor)
+ {
+ readLock.lock();
+ try
+ {
+ return partitionSSTablesByTokenRange ? currentBoundaries.getBoundariesFromSSTableDirectory(descriptor) : 0;
+ }
+ finally
+ {
+ readLock.unlock();
+ }
+ }
+
+ @VisibleForTesting
+ CompactionStrategyHolder getRepairedUnsafe()
+ {
+ return repaired;
+ }
+
+ @VisibleForTesting
+ CompactionStrategyHolder getUnrepairedUnsafe()
+ {
+ return unrepaired;
+ }
+
+ @VisibleForTesting
+ PendingRepairHolder getPendingRepairsUnsafe()
+ {
+ return pendingRepairs;
+ }
+
+ @VisibleForTesting
+ PendingRepairHolder getTransientRepairsUnsafe()
+ {
+ return transientRepairs;
+ }
+
+ public boolean hasDataForPendingRepair(UUID sessionID)
+ {
+ readLock.lock();
+ try
+ {
+ return pendingRepairs.hasDataForSession(sessionID) || transientRepairs.hasDataForSession(sessionID);
+ }
+ finally
+ {
+ readLock.unlock();
+ }
+ }
+
public void shutdown()
{
writeLock.lock();
try
{
isActive = false;
- repaired.forEach(AbstractCompactionStrategy::shutdown);
- unrepaired.forEach(AbstractCompactionStrategy::shutdown);
+ holders.forEach(AbstractStrategyHolder::shutdown);
compactionLogger.disable();
}
finally
@@ -309,7 +446,7 @@
}
}
- public void maybeReload(CFMetaData metadata)
+ public void maybeReload(TableMetadata metadata)
{
// compare the old schema configuration to the new one, ignore any locally set changes.
if (metadata.params.compaction.equals(schemaCompactionParams))
@@ -384,7 +521,7 @@
currentBoundaries = boundariesSupplier.get();
setStrategy(newCompactionParams);
- schemaCompactionParams = cfs.metadata.params.compaction;
+ schemaCompactionParams = cfs.metadata().params.compaction;
if (disabledWithJMX || !shouldBeEnabled() && !enabledWithJMX)
disable();
@@ -393,11 +530,9 @@
startup();
}
- public void replaceFlushed(Memtable memtable, Collection<SSTableReader> sstables)
+ private Iterable<AbstractCompactionStrategy> getAllStrategies()
{
- cfs.getTracker().replaceFlushed(memtable, sstables);
- if (sstables != null && !sstables.isEmpty())
- CompactionManager.instance.submitBackground(cfs);
+ return Iterables.concat(Iterables.transform(holders, AbstractStrategyHolder::allStrategies));
}
public int getUnleveledSSTables()
@@ -406,12 +541,10 @@
readLock.lock();
try
{
- if (repaired.get(0) instanceof LeveledCompactionStrategy && unrepaired.get(0) instanceof LeveledCompactionStrategy)
+ if (repaired.first() instanceof LeveledCompactionStrategy)
{
int count = 0;
- for (AbstractCompactionStrategy strategy : repaired)
- count += ((LeveledCompactionStrategy) strategy).getLevelSize(0);
- for (AbstractCompactionStrategy strategy : unrepaired)
+ for (AbstractCompactionStrategy strategy : getAllStrategies())
count += ((LeveledCompactionStrategy) strategy).getLevelSize(0);
return count;
}
@@ -434,19 +567,14 @@
readLock.lock();
try
{
- if (repaired.get(0) instanceof LeveledCompactionStrategy && unrepaired.get(0) instanceof LeveledCompactionStrategy)
+ if (repaired.first() instanceof LeveledCompactionStrategy)
{
int[] res = new int[LeveledManifest.MAX_LEVEL_COUNT];
- for (AbstractCompactionStrategy strategy : repaired)
+ for (AbstractCompactionStrategy strategy : getAllStrategies())
{
int[] repairedCountPerLevel = ((LeveledCompactionStrategy) strategy).getAllLevelSize();
res = sumArrays(res, repairedCountPerLevel);
}
- for (AbstractCompactionStrategy strategy : unrepaired)
- {
- int[] unrepairedCountPerLevel = ((LeveledCompactionStrategy) strategy).getAllLevelSize();
- res = sumArrays(res, unrepairedCountPerLevel);
- }
return res;
}
}
@@ -457,7 +585,7 @@
return null;
}
- private static int[] sumArrays(int[] a, int[] b)
+ static int[] sumArrays(int[] a, int[] b)
{
int[] res = new int[Math.max(a.length, b.length)];
for (int i = 0; i < res.length; i++)
@@ -477,21 +605,6 @@
return shouldDefragment;
}
- public Directories getDirectories()
- {
- maybeReloadDiskBoundaries();
- readLock.lock();
- try
- {
- assert repaired.get(0).getClass().equals(unrepaired.get(0).getClass());
- return repaired.get(0).getDirectories();
- }
- finally
- {
- readLock.unlock();
- }
- }
-
private void handleFlushNotification(Iterable<SSTableReader> added)
{
// If reloaded, SSTables will be placed in their correct locations
@@ -511,6 +624,76 @@
}
}
+ private int getHolderIndex(SSTableReader sstable)
+ {
+ for (int i = 0; i < holders.size(); i++)
+ {
+ if (holders.get(i).managesSSTable(sstable))
+ return i;
+ }
+
+ throw new IllegalStateException("No holder claimed " + sstable);
+ }
+
+ private AbstractStrategyHolder getHolder(SSTableReader sstable)
+ {
+ for (AbstractStrategyHolder holder : holders)
+ {
+ if (holder.managesSSTable(sstable))
+ return holder;
+ }
+
+ throw new IllegalStateException("No holder claimed " + sstable);
+ }
+
+ private AbstractStrategyHolder getHolder(long repairedAt, UUID pendingRepair, boolean isTransient)
+ {
+ return getHolder(repairedAt != ActiveRepairService.UNREPAIRED_SSTABLE,
+ pendingRepair != ActiveRepairService.NO_PENDING_REPAIR,
+ isTransient);
+ }
+
+ @VisibleForTesting
+ AbstractStrategyHolder getHolder(boolean isRepaired, boolean isPendingRepair, boolean isTransient)
+ {
+ for (AbstractStrategyHolder holder : holders)
+ {
+ if (holder.managesRepairedGroup(isRepaired, isPendingRepair, isTransient))
+ return holder;
+ }
+
+ throw new IllegalStateException(String.format("No holder claimed isPendingRepair: %s, isPendingRepair %s",
+ isRepaired, isPendingRepair));
+ }
+
+ @VisibleForTesting
+ ImmutableList<AbstractStrategyHolder> getHolders()
+ {
+ return holders;
+ }
+
+ /**
+ * Split sstables into a list of grouped sstable containers, the list index an sstable
+ *
+ * lives in matches the list index of the holder that's responsible for it
+ */
+ @VisibleForTesting
+ List<GroupedSSTableContainer> groupSSTables(Iterable<SSTableReader> sstables)
+ {
+ List<GroupedSSTableContainer> classified = new ArrayList<>(holders.size());
+ for (AbstractStrategyHolder holder : holders)
+ {
+ classified.add(holder.createGroupedSSTableContainer());
+ }
+
+ for (SSTableReader sstable : sstables)
+ {
+ classified.get(getHolderIndex(sstable)).add(sstable);
+ }
+
+ return classified;
+ }
+
private void handleListChangedNotification(Iterable<SSTableReader> added, Iterable<SSTableReader> removed)
{
// If reloaded, SSTables will be placed in their correct locations
@@ -521,50 +704,11 @@
readLock.lock();
try
{
- // a bit of gymnastics to be able to replace sstables in compaction strategies
- // we use this to know that a compaction finished and where to start the next compaction in LCS
- int locationSize = partitionSSTablesByTokenRange? currentBoundaries.directories.size() : 1;
-
- List<Set<SSTableReader>> repairedRemoved = new ArrayList<>(locationSize);
- List<Set<SSTableReader>> repairedAdded = new ArrayList<>(locationSize);
- List<Set<SSTableReader>> unrepairedRemoved = new ArrayList<>(locationSize);
- List<Set<SSTableReader>> unrepairedAdded = new ArrayList<>(locationSize);
-
- for (int i = 0; i < locationSize; i++)
+ List<GroupedSSTableContainer> addedGroups = groupSSTables(added);
+ List<GroupedSSTableContainer> removedGroups = groupSSTables(removed);
+ for (int i=0; i<holders.size(); i++)
{
- repairedRemoved.add(new HashSet<>());
- repairedAdded.add(new HashSet<>());
- unrepairedRemoved.add(new HashSet<>());
- unrepairedAdded.add(new HashSet<>());
- }
-
- for (SSTableReader sstable : removed)
- {
- int i = compactionStrategyIndexFor(sstable);
- if (sstable.isRepaired())
- repairedRemoved.get(i).add(sstable);
- else
- unrepairedRemoved.get(i).add(sstable);
- }
- for (SSTableReader sstable : added)
- {
- int i = compactionStrategyIndexFor(sstable);
- if (sstable.isRepaired())
- repairedAdded.get(i).add(sstable);
- else
- unrepairedAdded.get(i).add(sstable);
- }
- for (int i = 0; i < locationSize; i++)
- {
- if (!repairedRemoved.get(i).isEmpty())
- repaired.get(i).replaceSSTables(repairedRemoved.get(i), repairedAdded.get(i));
- else
- repaired.get(i).addSSTables(repairedAdded.get(i));
-
- if (!unrepairedRemoved.get(i).isEmpty())
- unrepaired.get(i).replaceSSTables(unrepairedRemoved.get(i), unrepairedAdded.get(i));
- else
- unrepaired.get(i).addSSTables(unrepairedAdded.get(i));
+ holders.get(i).replaceSSTables(removedGroups.get(i), addedGroups.get(i));
}
}
finally
@@ -583,19 +727,25 @@
readLock.lock();
try
{
- for (SSTableReader sstable : sstables)
+ List<GroupedSSTableContainer> groups = groupSSTables(sstables);
+ for (int i = 0; i < holders.size(); i++)
{
- int index = compactionStrategyIndexFor(sstable);
- if (sstable.isRepaired())
+ GroupedSSTableContainer group = groups.get(i);
+
+ if (group.isEmpty())
+ continue;
+
+ AbstractStrategyHolder dstHolder = holders.get(i);
+
+ for (AbstractStrategyHolder holder : holders)
{
- unrepaired.get(index).removeSSTable(sstable);
- repaired.get(index).addSSTable(sstable);
+ if (holder != dstHolder)
+ holder.removeSSTables(group);
}
- else
- {
- repaired.get(index).removeSSTable(sstable);
- unrepaired.get(index).addSSTable(sstable);
- }
+
+ // adding sstables into another strategy may change its level,
+ // thus it won't be removed from original LCS. We have to remove sstables first
+ dstHolder.addSSTables(group);
}
}
finally
@@ -604,6 +754,12 @@
}
}
+ private void handleMetadataChangedNotification(SSTableReader sstable, StatsMetadata oldMetadata)
+ {
+ AbstractCompactionStrategy acs = getCompactionStrategyFor(sstable);
+ acs.metadataChanged(oldMetadata, sstable);
+ }
+
private void handleDeletingNotification(SSTableReader deleted)
{
// If reloaded, SSTables will be placed in their correct locations
@@ -640,6 +796,11 @@
{
handleDeletingNotification(((SSTableDeletingNotification) notification).deleting);
}
+ else if (notification instanceof SSTableMetadataChanged)
+ {
+ SSTableMetadataChanged lcNotification = (SSTableMetadataChanged) notification;
+ handleMetadataChangedNotification(lcNotification.sstable, lcNotification.oldMetadata);
+ }
}
public void enable()
@@ -647,10 +808,6 @@
writeLock.lock();
try
{
- if (repaired != null)
- repaired.forEach(AbstractCompactionStrategy::enable);
- if (unrepaired != null)
- unrepaired.forEach(AbstractCompactionStrategy::enable);
// enable this last to make sure the strategies are ready to get calls.
enabled = true;
}
@@ -665,12 +822,7 @@
writeLock.lock();
try
{
- // disable this first avoid asking disabled strategies for compaction tasks
enabled = false;
- if (repaired != null)
- repaired.forEach(AbstractCompactionStrategy::disable);
- if (unrepaired != null)
- unrepaired.forEach(AbstractCompactionStrategy::disable);
}
finally
{
@@ -687,48 +839,46 @@
* @return
*/
@SuppressWarnings("resource")
- public AbstractCompactionStrategy.ScannerList getScanners(Collection<SSTableReader> sstables, Collection<Range<Token>> ranges)
+ public AbstractCompactionStrategy.ScannerList maybeGetScanners(Collection<SSTableReader> sstables, Collection<Range<Token>> ranges)
{
maybeReloadDiskBoundaries();
readLock.lock();
+ List<ISSTableScanner> scanners = new ArrayList<>(sstables.size());
try
{
- assert repaired.size() == unrepaired.size();
- List<Set<SSTableReader>> repairedSSTables = new ArrayList<>();
- List<Set<SSTableReader>> unrepairedSSTables = new ArrayList<>();
+ List<GroupedSSTableContainer> sstableGroups = groupSSTables(sstables);
- for (int i = 0; i < repaired.size(); i++)
+ for (int i = 0; i < holders.size(); i++)
{
- repairedSSTables.add(new HashSet<>());
- unrepairedSSTables.add(new HashSet<>());
+ AbstractStrategyHolder holder = holders.get(i);
+ GroupedSSTableContainer group = sstableGroups.get(i);
+ scanners.addAll(holder.getScanners(group, ranges));
}
-
- for (SSTableReader sstable : sstables)
- {
- if (sstable.isRepaired())
- repairedSSTables.get(compactionStrategyIndexFor(sstable)).add(sstable);
- else
- unrepairedSSTables.get(compactionStrategyIndexFor(sstable)).add(sstable);
- }
-
- List<ISSTableScanner> scanners = new ArrayList<>(sstables.size());
- for (int i = 0; i < repairedSSTables.size(); i++)
- {
- if (!repairedSSTables.get(i).isEmpty())
- scanners.addAll(repaired.get(i).getScanners(repairedSSTables.get(i), ranges).scanners);
- }
- for (int i = 0; i < unrepairedSSTables.size(); i++)
- {
- if (!unrepairedSSTables.get(i).isEmpty())
- scanners.addAll(unrepaired.get(i).getScanners(unrepairedSSTables.get(i), ranges).scanners);
- }
-
- return new AbstractCompactionStrategy.ScannerList(scanners);
+ }
+ catch (PendingRepairManager.IllegalSSTableArgumentException e)
+ {
+ ISSTableScanner.closeAllAndPropagate(scanners, new ConcurrentModificationException(e));
}
finally
{
readLock.unlock();
}
+ return new AbstractCompactionStrategy.ScannerList(scanners);
+ }
+
+ public AbstractCompactionStrategy.ScannerList getScanners(Collection<SSTableReader> sstables, Collection<Range<Token>> ranges)
+ {
+ while (true)
+ {
+ try
+ {
+ return maybeGetScanners(sstables, ranges);
+ }
+ catch (ConcurrentModificationException e)
+ {
+ logger.debug("SSTable repairedAt/pendingRepaired values changed while getting scanners");
+ }
+ }
}
public AbstractCompactionStrategy.ScannerList getScanners(Collection<SSTableReader> sstables)
@@ -742,12 +892,7 @@
readLock.lock();
try
{
- Map<Integer, List<SSTableReader>> groups = sstablesToGroup.stream().collect(Collectors.groupingBy((s) -> compactionStrategyIndexFor(s)));
- Collection<Collection<SSTableReader>> anticompactionGroups = new ArrayList<>();
-
- for (Map.Entry<Integer, List<SSTableReader>> group : groups.entrySet())
- anticompactionGroups.addAll(unrepaired.get(group.getKey()).groupSSTablesForAntiCompaction(group.getValue()));
- return anticompactionGroups;
+ return unrepaired.groupForAnticompaction(sstablesToGroup);
}
finally
{
@@ -760,7 +905,7 @@
readLock.lock();
try
{
- return unrepaired.get(0).getMaxSSTableBytes();
+ return unrepaired.first().getMaxSSTableBytes();
}
finally
{
@@ -793,57 +938,45 @@
assert firstSSTable != null;
boolean repaired = firstSSTable.isRepaired();
int firstIndex = compactionStrategyIndexFor(firstSSTable);
+ boolean isPending = firstSSTable.isPendingRepair();
+ UUID pendingRepair = firstSSTable.getSSTableMetadata().pendingRepair;
for (SSTableReader sstable : input)
{
if (sstable.isRepaired() != repaired)
throw new UnsupportedOperationException("You can't mix repaired and unrepaired data in a compaction");
if (firstIndex != compactionStrategyIndexFor(sstable))
throw new UnsupportedOperationException("You can't mix sstables from different directories in a compaction");
+ if (isPending && !pendingRepair.equals(sstable.getSSTableMetadata().pendingRepair))
+ throw new UnsupportedOperationException("You can't compact sstables from different pending repair sessions");
}
}
finally
{
readLock.unlock();
}
-
}
- public Collection<AbstractCompactionTask> getMaximalTasks(final int gcBefore, final boolean splitOutput)
+ public CompactionTasks getMaximalTasks(final int gcBefore, final boolean splitOutput)
{
maybeReloadDiskBoundaries();
// runWithCompactionsDisabled cancels active compactions and disables them, then we are able
// to make the repaired/unrepaired strategies mark their own sstables as compacting. Once the
// sstables are marked the compactions are re-enabled
- return cfs.runWithCompactionsDisabled(new Callable<Collection<AbstractCompactionTask>>()
- {
- @Override
- public Collection<AbstractCompactionTask> call()
+ return cfs.runWithCompactionsDisabled(() -> {
+ List<AbstractCompactionTask> tasks = new ArrayList<>();
+ readLock.lock();
+ try
{
- List<AbstractCompactionTask> tasks = new ArrayList<>();
- readLock.lock();
- try
+ for (AbstractStrategyHolder holder : holders)
{
- for (AbstractCompactionStrategy strategy : repaired)
- {
- Collection<AbstractCompactionTask> task = strategy.getMaximalTask(gcBefore, splitOutput);
- if (task != null)
- tasks.addAll(task);
- }
- for (AbstractCompactionStrategy strategy : unrepaired)
- {
- Collection<AbstractCompactionTask> task = strategy.getMaximalTask(gcBefore, splitOutput);
- if (task != null)
- tasks.addAll(task);
- }
+ tasks.addAll(holder.getMaximalTasks(gcBefore, splitOutput));
}
- finally
- {
- readLock.unlock();
- }
- if (tasks.isEmpty())
- return null;
- return tasks;
}
+ finally
+ {
+ readLock.unlock();
+ }
+ return CompactionTasks.create(tasks);
}, false, false);
}
@@ -856,37 +989,19 @@
* @param gcBefore gc grace period, throw away tombstones older than this
* @return a list of compaction tasks corresponding to the sstables requested
*/
- public List<AbstractCompactionTask> getUserDefinedTasks(Collection<SSTableReader> sstables, int gcBefore)
- {
- return getUserDefinedTasks(sstables, gcBefore, false);
- }
-
- public List<AbstractCompactionTask> getUserDefinedTasks(Collection<SSTableReader> sstables, int gcBefore, boolean validateForCompaction)
+ public CompactionTasks getUserDefinedTasks(Collection<SSTableReader> sstables, int gcBefore)
{
maybeReloadDiskBoundaries();
List<AbstractCompactionTask> ret = new ArrayList<>();
readLock.lock();
try
{
- if (validateForCompaction)
- validateForCompaction(sstables);
-
- Map<Integer, List<SSTableReader>> repairedSSTables = sstables.stream()
- .filter(s -> !s.isMarkedSuspect() && s.isRepaired())
- .collect(Collectors.groupingBy((s) -> compactionStrategyIndexFor(s)));
-
- Map<Integer, List<SSTableReader>> unrepairedSSTables = sstables.stream()
- .filter(s -> !s.isMarkedSuspect() && !s.isRepaired())
- .collect(Collectors.groupingBy((s) -> compactionStrategyIndexFor(s)));
-
-
- for (Map.Entry<Integer, List<SSTableReader>> group : repairedSSTables.entrySet())
- ret.add(repaired.get(group.getKey()).getUserDefinedTask(group.getValue(), gcBefore));
-
- for (Map.Entry<Integer, List<SSTableReader>> group : unrepairedSSTables.entrySet())
- ret.add(unrepaired.get(group.getKey()).getUserDefinedTask(group.getValue(), gcBefore));
-
- return ret;
+ List<GroupedSSTableContainer> groupedSSTables = groupSSTables(sstables);
+ for (int i = 0; i < holders.size(); i++)
+ {
+ ret.addAll(holders.get(i).getUserDefinedTasks(groupedSSTables.get(i), gcBefore));
+ }
+ return CompactionTasks.create(ret);
}
finally
{
@@ -894,17 +1009,6 @@
}
}
- /**
- * @deprecated use {@link #getUserDefinedTasks(Collection, int)} instead.
- */
- @Deprecated()
- public AbstractCompactionTask getUserDefinedTask(Collection<SSTableReader> sstables, int gcBefore)
- {
- List<AbstractCompactionTask> tasks = getUserDefinedTasks(sstables, gcBefore, true);
- assert tasks.size() == 1;
- return tasks.get(0);
- }
-
public int getEstimatedRemainingTasks()
{
maybeReloadDiskBoundaries();
@@ -912,10 +1016,7 @@
readLock.lock();
try
{
-
- for (AbstractCompactionStrategy strategy : repaired)
- tasks += strategy.getEstimatedRemainingTasks();
- for (AbstractCompactionStrategy strategy : unrepaired)
+ for (AbstractCompactionStrategy strategy : getAllStrategies())
tasks += strategy.getEstimatedRemainingTasks();
}
finally
@@ -936,7 +1037,7 @@
readLock.lock();
try
{
- return unrepaired.get(0).getName();
+ return unrepaired.first().getName();
}
finally
{
@@ -950,7 +1051,9 @@
readLock.lock();
try
{
- return Arrays.asList(repaired, unrepaired);
+ return Arrays.asList(Lists.newArrayList(repaired.allStrategies()),
+ Lists.newArrayList(unrepaired.allStrategies()),
+ Lists.newArrayList(pendingRepairs.allStrategies()));
}
finally
{
@@ -977,26 +1080,16 @@
}
}
+ private int getNumTokenPartitions()
+ {
+ return partitionSSTablesByTokenRange ? currentBoundaries.directories.size() : 1;
+ }
+
private void setStrategy(CompactionParams params)
{
- repaired.forEach(AbstractCompactionStrategy::shutdown);
- unrepaired.forEach(AbstractCompactionStrategy::shutdown);
- repaired.clear();
- unrepaired.clear();
-
- if (partitionSSTablesByTokenRange)
- {
- for (int i = 0; i < currentBoundaries.directories.size(); i++)
- {
- repaired.add(CFMetaData.createCompactionStrategyInstance(cfs, params));
- unrepaired.add(CFMetaData.createCompactionStrategyInstance(cfs, params));
- }
- }
- else
- {
- repaired.add(CFMetaData.createCompactionStrategyInstance(cfs, params));
- unrepaired.add(CFMetaData.createCompactionStrategyInstance(cfs, params));
- }
+ int numPartitions = getNumTokenPartitions();
+ for (AbstractStrategyHolder holder : holders)
+ holder.setStrategy(params, numPartitions);
this.params = params;
}
@@ -1013,23 +1106,27 @@
public SSTableMultiWriter createSSTableMultiWriter(Descriptor descriptor,
long keyCount,
long repairedAt,
+ UUID pendingRepair,
+ boolean isTransient,
MetadataCollector collector,
SerializationHeader header,
Collection<Index> indexes,
LifecycleNewTracker lifecycleNewTracker)
{
+ SSTable.validateRepairedMetadata(repairedAt, pendingRepair, isTransient);
maybeReloadDiskBoundaries();
readLock.lock();
try
{
- if (repairedAt == ActiveRepairService.UNREPAIRED_SSTABLE)
- {
- return unrepaired.get(0).createSSTableMultiWriter(descriptor, keyCount, repairedAt, collector, header, indexes, lifecycleNewTracker);
- }
- else
- {
- return repaired.get(0).createSSTableMultiWriter(descriptor, keyCount, repairedAt, collector, header, indexes, lifecycleNewTracker);
- }
+ return getHolder(repairedAt, pendingRepair, isTransient).createSSTableMultiWriter(descriptor,
+ keyCount,
+ repairedAt,
+ pendingRepair,
+ isTransient,
+ collector,
+ header,
+ indexes,
+ lifecycleNewTracker);
}
finally
{
@@ -1039,15 +1136,7 @@
public boolean isRepaired(AbstractCompactionStrategy strategy)
{
- readLock.lock();
- try
- {
- return repaired.contains(strategy);
- }
- finally
- {
- readLock.unlock();
- }
+ return repaired.getStrategyIndex(strategy) >= 0;
}
public List<String> getStrategyFolders(AbstractCompactionStrategy strategy)
@@ -1055,21 +1144,17 @@
readLock.lock();
try
{
- List<Directories.DataDirectory> locations = currentBoundaries.directories;
+ Directories.DataDirectory[] locations = cfs.getDirectories().getWriteableLocations();
if (partitionSSTablesByTokenRange)
{
- int unrepairedIndex = unrepaired.indexOf(strategy);
- if (unrepairedIndex > 0)
+ for (AbstractStrategyHolder holder : holders)
{
- return Collections.singletonList(locations.get(unrepairedIndex).location.getAbsolutePath());
- }
- int repairedIndex = repaired.indexOf(strategy);
- if (repairedIndex > 0)
- {
- return Collections.singletonList(locations.get(repairedIndex).location.getAbsolutePath());
+ int idx = holder.getStrategyIndex(strategy);
+ if (idx >= 0)
+ return Collections.singletonList(locations[idx].location.getAbsolutePath());
}
}
- List<String> folders = new ArrayList<>(locations.size());
+ List<String> folders = new ArrayList<>(locations.length);
for (Directories.DataDirectory location : locations)
{
folders.add(location.location.getAbsolutePath());
@@ -1080,11 +1165,69 @@
{
readLock.unlock();
}
-
}
public boolean supportsEarlyOpen()
{
return supportsEarlyOpen;
}
+
+ @VisibleForTesting
+ List<PendingRepairManager> getPendingRepairManagers()
+ {
+ maybeReloadDiskBoundaries();
+ readLock.lock();
+ try
+ {
+ return Lists.newArrayList(pendingRepairs.getManagers());
+ }
+ finally
+ {
+ readLock.unlock();
+ }
+ }
+
+ /**
+ * Mutates sstable repairedAt times and notifies listeners of the change with the writeLock held. Prevents races
+ * with other processes between when the metadata is changed and when sstables are moved between strategies.
+ */
+ public void mutateRepaired(Collection<SSTableReader> sstables, long repairedAt, UUID pendingRepair, boolean isTransient) throws IOException
+ {
+ Set<SSTableReader> changed = new HashSet<>();
+
+ writeLock.lock();
+ try
+ {
+ for (SSTableReader sstable: sstables)
+ {
+ sstable.descriptor.getMetadataSerializer().mutateRepairMetadata(sstable.descriptor, repairedAt, pendingRepair, isTransient);
+ sstable.reloadSSTableMetadata();
+ verifyMetadata(sstable, repairedAt, pendingRepair, isTransient);
+ changed.add(sstable);
+ }
+ }
+ finally
+ {
+ try
+ {
+ // if there was an exception mutating repairedAt, we should still notify for the
+ // sstables that we were able to modify successfully before releasing the lock
+ cfs.getTracker().notifySSTableRepairedStatusChanged(changed);
+ }
+ finally
+ {
+ writeLock.unlock();
+ }
+ }
+ }
+
+ private static void verifyMetadata(SSTableReader sstable, long repairedAt, UUID pendingRepair, boolean isTransient)
+ {
+ if (!Objects.equals(pendingRepair, sstable.getPendingRepair()))
+ throw new IllegalStateException(String.format("Failed setting pending repair to %s on %s (pending repair is %s)", pendingRepair, sstable, sstable.getPendingRepair()));
+ if (repairedAt != sstable.getRepairedAt())
+ throw new IllegalStateException(String.format("Failed setting repairedAt to %d on %s (repairedAt is %d)", repairedAt, sstable, sstable.getRepairedAt()));
+ if (isTransient != sstable.isTransient())
+ throw new IllegalStateException(String.format("Failed setting isTransient to %b on %s (isTransient is %b)", isTransient, sstable, sstable.isTransient()));
+ }
}
diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionTask.java b/src/java/org/apache/cassandra/db/compaction/CompactionTask.java
index 2efcd11..764ad5b 100644
--- a/src/java/org/apache/cassandra/db/compaction/CompactionTask.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionTask.java
@@ -19,6 +19,7 @@
import java.util.Collection;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
@@ -27,22 +28,19 @@
import com.google.common.base.Predicate;
import com.google.common.collect.Iterables;
import com.google.common.collect.Sets;
-import com.google.common.primitives.Ints;
import com.google.common.util.concurrent.RateLimiter;
-
-import org.apache.cassandra.db.Directories;
-import org.apache.cassandra.db.compaction.writers.CompactionAwareWriter;
-import org.apache.cassandra.db.compaction.writers.DefaultCompactionWriter;
-import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Directories;
import org.apache.cassandra.db.SystemKeyspace;
-import org.apache.cassandra.db.compaction.CompactionManager.CompactionExecutorStatsCollector;
+import org.apache.cassandra.db.compaction.writers.CompactionAwareWriter;
+import org.apache.cassandra.db.compaction.writers.DefaultCompactionWriter;
import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
import org.apache.cassandra.service.ActiveRepairService;
import org.apache.cassandra.utils.FBUtilities;
@@ -54,7 +52,7 @@
protected final int gcBefore;
protected final boolean keepOriginals;
protected static long totalBytesCompacted = 0;
- private CompactionExecutorStatsCollector collector;
+ private ActiveCompactionsTracker activeCompactions;
public CompactionTask(ColumnFamilyStore cfs, LifecycleTransaction txn, int gcBefore)
{
@@ -79,14 +77,14 @@
return totalBytesCompacted += bytesCompacted;
}
- protected int executeInternal(CompactionExecutorStatsCollector collector)
+ protected int executeInternal(ActiveCompactionsTracker activeCompactions)
{
- this.collector = collector;
+ this.activeCompactions = activeCompactions == null ? ActiveCompactionsTracker.NOOP : activeCompactions;
run();
return transaction.originals().size();
}
- public boolean reduceScopeForLimitedSpace(long expectedSize)
+ public boolean reduceScopeForLimitedSpace(Set<SSTableReader> nonExpiredSSTables, long expectedSize)
{
if (partialCompactionsAcceptable() && transaction.originals().size() > 1)
{
@@ -97,7 +95,7 @@
// Note that we have removed files that are still marked as compacting.
// This suboptimal but ok since the caller will unmark all the sstables at the end.
- SSTableReader removedSSTable = cfs.getMaxSizeFile(transaction.originals());
+ SSTableReader removedSSTable = cfs.getMaxSizeFile(nonExpiredSSTables);
transaction.cancel(removedSSTable);
return true;
}
@@ -125,44 +123,46 @@
if (DatabaseDescriptor.isSnapshotBeforeCompaction())
cfs.snapshotWithoutFlush(System.currentTimeMillis() + "-compact-" + cfs.name);
- // note that we need to do a rough estimate early if we can fit the compaction on disk - this is pessimistic, but
- // since we might remove sstables from the compaction in checkAvailableDiskSpace it needs to be done here
-
- checkAvailableDiskSpace();
-
- // sanity check: all sstables must belong to the same cfs
- assert !Iterables.any(transaction.originals(), new Predicate<SSTableReader>()
- {
- @Override
- public boolean apply(SSTableReader sstable)
- {
- return !sstable.descriptor.cfname.equals(cfs.name);
- }
- });
-
- UUID taskId = transaction.opId();
-
- // new sstables from flush can be added during a compaction, but only the compaction can remove them,
- // so in our single-threaded compaction world this is a valid way of determining if we're compacting
- // all the sstables (that existed when we started)
- StringBuilder ssTableLoggerMsg = new StringBuilder("[");
- for (SSTableReader sstr : transaction.originals())
- {
- ssTableLoggerMsg.append(String.format("%s:level=%d, ", sstr.getFilename(), sstr.getSSTableLevel()));
- }
- ssTableLoggerMsg.append("]");
-
- logger.debug("Compacting ({}) {}", taskId, ssTableLoggerMsg);
-
- RateLimiter limiter = CompactionManager.instance.getRateLimiter();
- long start = System.nanoTime();
- long startTime = System.currentTimeMillis();
- long totalKeysWritten = 0;
- long estimatedKeys = 0;
- long inputSizeBytes;
try (CompactionController controller = getCompactionController(transaction.originals()))
{
- Set<SSTableReader> actuallyCompact = Sets.difference(transaction.originals(), controller.getFullyExpiredSSTables());
+
+ final Set<SSTableReader> fullyExpiredSSTables = controller.getFullyExpiredSSTables();
+
+ // select SSTables to compact based on available disk space.
+ buildCompactionCandidatesForAvailableDiskSpace(fullyExpiredSSTables);
+
+ // sanity check: all sstables must belong to the same cfs
+ assert !Iterables.any(transaction.originals(), new Predicate<SSTableReader>()
+ {
+ @Override
+ public boolean apply(SSTableReader sstable)
+ {
+ return !sstable.descriptor.cfname.equals(cfs.name);
+ }
+ });
+
+ UUID taskId = transaction.opId();
+
+ // new sstables from flush can be added during a compaction, but only the compaction can remove them,
+ // so in our single-threaded compaction world this is a valid way of determining if we're compacting
+ // all the sstables (that existed when we started)
+ StringBuilder ssTableLoggerMsg = new StringBuilder("[");
+ for (SSTableReader sstr : transaction.originals())
+ {
+ ssTableLoggerMsg.append(String.format("%s:level=%d, ", sstr.getFilename(), sstr.getSSTableLevel()));
+ }
+ ssTableLoggerMsg.append("]");
+
+ logger.info("Compacting ({}) {}", taskId, ssTableLoggerMsg);
+
+ RateLimiter limiter = CompactionManager.instance.getRateLimiter();
+ long start = System.nanoTime();
+ long startTime = System.currentTimeMillis();
+ long totalKeysWritten = 0;
+ long estimatedKeys = 0;
+ long inputSizeBytes;
+
+ Set<SSTableReader> actuallyCompact = Sets.difference(transaction.originals(), fullyExpiredSSTables);
Collection<SSTableReader> newSStables;
long[] mergedRowCounts;
@@ -184,9 +184,7 @@
long lastBytesScanned = 0;
- if (collector != null)
- collector.beginCompaction(ci);
-
+ activeCompactions.beginCompaction(ci);
try (CompactionAwareWriter writer = getCompactionAwareWriter(cfs, getDirectories(), transaction, actuallyCompact))
{
// Note that we need to re-check this flag after calling beginCompaction above to avoid a window
@@ -198,9 +196,6 @@
estimatedKeys = writer.estimatedKeys();
while (ci.hasNext())
{
- if (ci.isStopRequested())
- throw new CompactionInterruptedException(ci.getCompactionInfo());
-
if (writer.append(ci.next()))
totalKeysWritten++;
@@ -224,11 +219,8 @@
}
finally
{
- if (collector != null)
- collector.finishCompaction(ci);
-
+ activeCompactions.finishCompaction(ci);
mergedRowCounts = ci.getMergedRowCounts();
-
totalSourceCQLRows = ci.getTotalSourceCQLRows();
}
}
@@ -255,7 +247,8 @@
totalSourceRows += mergedRowCounts[i] * (i + 1);
String mergeSummary = updateCompactionHistory(cfs.keyspace.getName(), cfs.getTableName(), mergedRowCounts, startsize, endsize);
- logger.debug(String.format("Compacted (%s) %d sstables to [%s] to level=%d. %s to %s (~%d%% of original) in %,dms. Read Throughput = %s, Write Throughput = %s, Row Throughput = ~%,d/s. %,d total partitions merged to %,d. Partition merge counts were {%s}",
+
+ logger.info(String.format("Compacted (%s) %d sstables to [%s] to level=%d. %s to %s (~%d%% of original) in %,dms. Read Throughput = %s, Write Throughput = %s, Row Throughput = ~%,d/s. %,d total partitions merged to %,d. Partition merge counts were {%s}",
taskId,
transaction.originals().size(),
newSSTableNames.toString(),
@@ -270,8 +263,11 @@
totalSourceRows,
totalKeysWritten,
mergeSummary));
- logger.trace("CF Total Bytes Compacted: {}", FBUtilities.prettyPrintMemory(CompactionTask.addToTotalBytesCompacted(endsize)));
- logger.trace("Actual #keys: {}, Estimated #keys:{}, Err%: {}", totalKeysWritten, estimatedKeys, ((double)(totalKeysWritten - estimatedKeys)/totalKeysWritten));
+ if (logger.isTraceEnabled())
+ {
+ logger.trace("CF Total Bytes Compacted: {}", FBUtilities.prettyPrintMemory(CompactionTask.addToTotalBytesCompacted(endsize)));
+ logger.trace("Actual #keys: {}, Estimated #keys:{}, Err%: {}", totalKeysWritten, estimatedKeys, ((double)(totalKeysWritten - estimatedKeys)/totalKeysWritten));
+ }
cfs.getCompactionStrategyManager().compactionLogger.compaction(startTime, transaction.originals(), System.currentTimeMillis(), newSStables);
// update the metrics
@@ -322,40 +318,96 @@
return minRepairedAt;
}
+ public static UUID getPendingRepair(Set<SSTableReader> sstables)
+ {
+ if (sstables.isEmpty())
+ {
+ return ActiveRepairService.NO_PENDING_REPAIR;
+ }
+ Set<UUID> ids = new HashSet<>();
+ for (SSTableReader sstable: sstables)
+ ids.add(sstable.getSSTableMetadata().pendingRepair);
+
+ if (ids.size() != 1)
+ throw new RuntimeException(String.format("Attempting to compact pending repair sstables with sstables from other repair, or sstables not pending repair: %s", ids));
+
+ return ids.iterator().next();
+ }
+
+ public static boolean getIsTransient(Set<SSTableReader> sstables)
+ {
+ if (sstables.isEmpty())
+ {
+ return false;
+ }
+
+ boolean isTransient = sstables.iterator().next().isTransient();
+
+ if (!Iterables.all(sstables, sstable -> sstable.isTransient() == isTransient))
+ {
+ throw new RuntimeException("Attempting to compact transient sstables with non transient sstables");
+ }
+
+ return isTransient;
+ }
+
+
/*
- Checks if we have enough disk space to execute the compaction. Drops the largest sstable out of the Task until
- there's enough space (in theory) to handle the compaction. Does not take into account space that will be taken by
- other compactions.
+ * Checks if we have enough disk space to execute the compaction. Drops the largest sstable out of the Task until
+ * there's enough space (in theory) to handle the compaction. Does not take into account space that will be taken by
+ * other compactions.
*/
- protected void checkAvailableDiskSpace()
+ protected void buildCompactionCandidatesForAvailableDiskSpace(final Set<SSTableReader> fullyExpiredSSTables)
{
if(!cfs.isCompactionDiskSpaceCheckEnabled() && compactionType == OperationType.COMPACTION)
{
logger.info("Compaction space check is disabled");
- return;
+ return; // try to compact all SSTables
}
+ final Set<SSTableReader> nonExpiredSSTables = Sets.difference(transaction.originals(), fullyExpiredSSTables);
CompactionStrategyManager strategy = cfs.getCompactionStrategyManager();
+ int sstablesRemoved = 0;
- while(true)
+ while(!nonExpiredSSTables.isEmpty())
{
- long expectedWriteSize = cfs.getExpectedCompactedFileSize(transaction.originals(), compactionType);
+ // Only consider write size of non expired SSTables
+ long expectedWriteSize = cfs.getExpectedCompactedFileSize(nonExpiredSSTables, compactionType);
long estimatedSSTables = Math.max(1, expectedWriteSize / strategy.getMaxSSTableBytes());
if(cfs.getDirectories().hasAvailableDiskSpace(estimatedSSTables, expectedWriteSize))
break;
- if (!reduceScopeForLimitedSpace(expectedWriteSize))
+ if (!reduceScopeForLimitedSpace(nonExpiredSSTables, expectedWriteSize))
{
// we end up here if we can't take any more sstables out of the compaction.
// usually means we've run out of disk space
+
+ // but we can still compact expired SSTables
+ if(partialCompactionsAcceptable() && fullyExpiredSSTables.size() > 0 )
+ {
+ // sanity check to make sure we compact only fully expired SSTables.
+ assert transaction.originals().equals(fullyExpiredSSTables);
+ break;
+ }
+
String msg = String.format("Not enough space for compaction, estimated sstables = %d, expected write size = %d", estimatedSSTables, expectedWriteSize);
logger.warn(msg);
+ CompactionManager.instance.incrementAborted();
throw new RuntimeException(msg);
}
+
+ sstablesRemoved++;
logger.warn("Not enough space for compaction, {}MB estimated. Reducing scope.",
- (float) expectedWriteSize / 1024 / 1024);
+ (float) expectedWriteSize / 1024 / 1024);
}
+
+ if(sstablesRemoved > 0)
+ {
+ CompactionManager.instance.incrementCompactionsReduced();
+ CompactionManager.instance.incrementSstablesDropppedFromCompactions(sstablesRemoved);
+ }
+
}
protected int getLevel()
diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionTasks.java b/src/java/org/apache/cassandra/db/compaction/CompactionTasks.java
new file mode 100644
index 0000000..af0dbd0
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionTasks.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+import java.util.AbstractCollection;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.stream.Collectors;
+
+import org.apache.cassandra.utils.FBUtilities;
+
+public class CompactionTasks extends AbstractCollection<AbstractCompactionTask> implements AutoCloseable
+{
+ @SuppressWarnings("resource")
+ private static final CompactionTasks EMPTY = new CompactionTasks(Collections.emptyList());
+
+ private final Collection<AbstractCompactionTask> tasks;
+
+ private CompactionTasks(Collection<AbstractCompactionTask> tasks)
+ {
+ this.tasks = tasks;
+ }
+
+ public static CompactionTasks create(Collection<AbstractCompactionTask> tasks)
+ {
+ if (tasks == null || tasks.isEmpty())
+ return EMPTY;
+ return new CompactionTasks(tasks);
+ }
+
+ public static CompactionTasks empty()
+ {
+ return EMPTY;
+ }
+
+ public Iterator<AbstractCompactionTask> iterator()
+ {
+ return tasks.iterator();
+ }
+
+ public int size()
+ {
+ return tasks.size();
+ }
+
+ public void close()
+ {
+ try
+ {
+ FBUtilities.closeAll(tasks.stream().map(task -> task.transaction).collect(Collectors.toList()));
+ }
+ catch (Exception e)
+ {
+ throw new RuntimeException(e);
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java b/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
index ed3b172..ab2b6ae 100644
--- a/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
+++ b/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
@@ -26,6 +26,9 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.JsonNodeFactory;
+import com.fasterxml.jackson.databind.node.ObjectNode;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
import org.apache.cassandra.db.lifecycle.SSTableSet;
@@ -33,9 +36,6 @@
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.schema.CompactionParams;
import org.apache.cassandra.utils.Pair;
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.node.JsonNodeFactory;
-import com.fasterxml.jackson.databind.node.ObjectNode;
import static com.google.common.collect.Iterables.filter;
@@ -235,6 +235,12 @@
sstables.remove(sstable);
}
+ @Override
+ protected Set<SSTableReader> getSSTables()
+ {
+ return ImmutableSet.copyOf(sstables);
+ }
+
/**
* A target time span used for bucketing SSTables based on timestamps.
*/
@@ -455,7 +461,7 @@
@Override
public Collection<Collection<SSTableReader>> groupSSTablesForAntiCompaction(Collection<SSTableReader> sstablesToGroup)
{
- Collection<Collection<SSTableReader>> groups = new ArrayList<>();
+ Collection<Collection<SSTableReader>> groups = new ArrayList<>(sstablesToGroup.size());
for (SSTableReader sstable : sstablesToGroup)
{
groups.add(Collections.singleton(sstable));
diff --git a/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyOptions.java b/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyOptions.java
index 9362bde..7604bbc 100644
--- a/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyOptions.java
+++ b/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyOptions.java
@@ -54,7 +54,7 @@
String optionValue = options.get(TIMESTAMP_RESOLUTION_KEY);
timestampResolution = optionValue == null ? DEFAULT_TIMESTAMP_RESOLUTION : TimeUnit.valueOf(optionValue);
if (timestampResolution != DEFAULT_TIMESTAMP_RESOLUTION)
- logger.warn("Using a non-default timestamp_resolution {} - are you really doing inserts with USING TIMESTAMP <non_microsecond_timestamp> (or driver equivalent)?", timestampResolution.toString());
+ logger.warn("Using a non-default timestamp_resolution {} - are you really doing inserts with USING TIMESTAMP <non_microsecond_timestamp> (or driver equivalent)?", timestampResolution);
optionValue = options.get(MAX_SSTABLE_AGE_KEY);
double fractionalDays = optionValue == null ? DEFAULT_MAX_SSTABLE_AGE_DAYS : Double.parseDouble(optionValue);
maxSSTableAge = Math.round(fractionalDays * timestampResolution.convert(1, TimeUnit.DAYS));
diff --git a/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java b/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java
index 8c37bb4..74ffccb 100644
--- a/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java
+++ b/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java
@@ -28,7 +28,12 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.cassandra.config.CFMetaData;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.JsonNodeFactory;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
+import org.apache.cassandra.schema.CompactionParams;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.config.Config;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
@@ -38,9 +43,6 @@
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.io.sstable.ISSTableScanner;
import org.apache.cassandra.io.sstable.format.SSTableReader;
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.node.JsonNodeFactory;
-import com.fasterxml.jackson.databind.node.ObjectNode;
public class LeveledCompactionStrategy extends AbstractCompactionStrategy
{
@@ -48,18 +50,21 @@
private static final String SSTABLE_SIZE_OPTION = "sstable_size_in_mb";
private static final boolean tolerateSstableSize = Boolean.getBoolean(Config.PROPERTY_PREFIX + "tolerate_sstable_size");
private static final String LEVEL_FANOUT_SIZE_OPTION = "fanout_size";
+ private static final String SINGLE_SSTABLE_UPLEVEL_OPTION = "single_sstable_uplevel";
public static final int DEFAULT_LEVEL_FANOUT_SIZE = 10;
@VisibleForTesting
final LeveledManifest manifest;
private final int maxSSTableSizeInMB;
private final int levelFanoutSize;
+ private final boolean singleSSTableUplevel;
public LeveledCompactionStrategy(ColumnFamilyStore cfs, Map<String, String> options)
{
super(cfs, options);
int configuredMaxSSTableSize = 160;
int configuredLevelFanoutSize = DEFAULT_LEVEL_FANOUT_SIZE;
+ boolean configuredSingleSSTableUplevel = false;
SizeTieredCompactionStrategyOptions localOptions = new SizeTieredCompactionStrategyOptions(options);
if (options != null)
{
@@ -70,10 +75,10 @@
{
if (configuredMaxSSTableSize >= 1000)
logger.warn("Max sstable size of {}MB is configured for {}.{}; having a unit of compaction this large is probably a bad idea",
- configuredMaxSSTableSize, cfs.name, cfs.getColumnFamilyName());
+ configuredMaxSSTableSize, cfs.name, cfs.getTableName());
if (configuredMaxSSTableSize < 50)
logger.warn("Max sstable size of {}MB is configured for {}.{}. Testing done for CASSANDRA-5727 indicates that performance improves up to 160MB",
- configuredMaxSSTableSize, cfs.name, cfs.getColumnFamilyName());
+ configuredMaxSSTableSize, cfs.name, cfs.getTableName());
}
}
@@ -81,9 +86,15 @@
{
configuredLevelFanoutSize = Integer.parseInt(options.get(LEVEL_FANOUT_SIZE_OPTION));
}
+
+ if (options.containsKey(SINGLE_SSTABLE_UPLEVEL_OPTION))
+ {
+ configuredSingleSSTableUplevel = Boolean.parseBoolean(options.get(SINGLE_SSTABLE_UPLEVEL_OPTION));
+ }
}
maxSSTableSizeInMB = configuredMaxSSTableSize;
levelFanoutSize = configuredLevelFanoutSize;
+ singleSSTableUplevel = configuredSingleSSTableUplevel;
manifest = new LeveledManifest(cfs, this.maxSSTableSizeInMB, this.levelFanoutSize, localOptions);
logger.trace("Created {}", manifest);
@@ -150,7 +161,12 @@
LifecycleTransaction txn = cfs.getTracker().tryModify(candidate.sstables, OperationType.COMPACTION);
if (txn != null)
{
- LeveledCompactionTask newTask = new LeveledCompactionTask(cfs, txn, candidate.level, gcBefore, candidate.maxSSTableBytes, false);
+ AbstractCompactionTask newTask;
+ if (!singleSSTableUplevel || op == OperationType.TOMBSTONE_COMPACTION || txn.originals().size() > 1)
+ newTask = new LeveledCompactionTask(cfs, txn, candidate.level, gcBefore, candidate.maxSSTableBytes, false);
+ else
+ newTask = new SingleSSTableLCSTask(cfs, txn, candidate.level);
+
newTask.setCompactionType(op);
return newTask;
}
@@ -235,14 +251,14 @@
for (Collection<SSTableReader> levelOfSSTables : sstablesByLevel.values())
{
- Collection<SSTableReader> currGroup = new ArrayList<>();
+ Collection<SSTableReader> currGroup = new ArrayList<>(groupSize);
for (SSTableReader sstable : levelOfSSTables)
{
currGroup.add(sstable);
if (currGroup.size() == groupSize)
{
groupedSSTables.add(currGroup);
- currGroup = new ArrayList<>();
+ currGroup = new ArrayList<>(groupSize);
}
}
@@ -302,7 +318,7 @@
{
// L0 makes no guarantees about overlapping-ness. Just create a direct scanner for each
for (SSTableReader sstable : byLevel.get(level))
- scanners.add(sstable.getScanner(ranges, null));
+ scanners.add(sstable.getScanner(ranges));
}
else
{
@@ -311,7 +327,7 @@
if (!intersecting.isEmpty())
{
@SuppressWarnings("resource") // The ScannerList will be in charge of closing (and we close properly on errors)
- ISSTableScanner scanner = new LeveledScanner(intersecting, ranges);
+ ISSTableScanner scanner = new LeveledScanner(cfs.metadata(), intersecting, ranges);
scanners.add(scanner);
}
}
@@ -319,15 +335,7 @@
}
catch (Throwable t)
{
- try
- {
- new ScannerList(scanners).close();
- }
- catch (Throwable t2)
- {
- t.addSuppressed(t2);
- }
- throw t;
+ ISSTableScanner.closeAllAndPropagate(scanners, t);
}
return new ScannerList(scanners);
@@ -340,6 +348,13 @@
}
@Override
+ public void metadataChanged(StatsMetadata oldMetadata, SSTableReader sstable)
+ {
+ if (sstable.getSSTableLevel() != oldMetadata.sstableLevel)
+ manifest.newLevel(sstable, oldMetadata.sstableLevel);
+ }
+
+ @Override
public void addSSTable(SSTableReader added)
{
manifest.add(added);
@@ -351,10 +366,17 @@
manifest.remove(sstable);
}
+ @Override
+ protected Set<SSTableReader> getSSTables()
+ {
+ return manifest.getSSTables();
+ }
+
// Lazily creates SSTableBoundedScanner for sstable that are assumed to be from the
// same level (e.g. non overlapping) - see #4142
private static class LeveledScanner extends AbstractIterator<UnfilteredRowIterator> implements ISSTableScanner
{
+ private final TableMetadata metadata;
private final Collection<Range<Token>> ranges;
private final List<SSTableReader> sstables;
private final Iterator<SSTableReader> sstableIterator;
@@ -365,8 +387,9 @@
private long positionOffset;
private long totalBytesScanned = 0;
- public LeveledScanner(Collection<SSTableReader> sstables, Collection<Range<Token>> ranges)
+ public LeveledScanner(TableMetadata metadata, Collection<SSTableReader> sstables, Collection<Range<Token>> ranges)
{
+ this.metadata = metadata;
this.ranges = ranges;
// add only sstables that intersect our range, and estimate how much data that involves
@@ -392,7 +415,7 @@
sstableIterator = this.sstables.iterator();
assert sstableIterator.hasNext(); // caller should check intersecting first
SSTableReader currentSSTable = sstableIterator.next();
- currentScanner = currentSSTable.getScanner(ranges, null);
+ currentScanner = currentSSTable.getScanner(ranges);
}
@@ -414,15 +437,9 @@
return filtered;
}
-
- public boolean isForThrift()
+ public TableMetadata metadata()
{
- return false;
- }
-
- public CFMetaData metadata()
- {
- return sstables.get(0).metadata; // The ctor checks we have at least one sstable
+ return metadata;
}
protected UnfilteredRowIterator computeNext()
@@ -446,7 +463,7 @@
return endOfData();
}
SSTableReader currentSSTable = sstableIterator.next();
- currentScanner = currentSSTable.getScanner(ranges, null);
+ currentScanner = currentSSTable.getScanner(ranges);
}
}
@@ -476,9 +493,9 @@
return currentScanner == null ? totalBytesScanned : totalBytesScanned + currentScanner.getBytesScanned();
}
- public String getBackingFiles()
+ public Set<SSTableReader> getBackingSSTables()
{
- return Joiner.on(", ").join(sstables);
+ return ImmutableSet.copyOf(sstables);
}
}
@@ -574,6 +591,10 @@
}
uncheckedOptions.remove(LEVEL_FANOUT_SIZE_OPTION);
+ uncheckedOptions.remove(SINGLE_SSTABLE_UPLEVEL_OPTION);
+
+ uncheckedOptions.remove(CompactionParams.Option.MIN_THRESHOLD.toString());
+ uncheckedOptions.remove(CompactionParams.Option.MAX_THRESHOLD.toString());
uncheckedOptions = SizeTieredCompactionStrategyOptions.validateOptions(options, uncheckedOptions);
diff --git a/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java b/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java
index 8a8362f..2c32361 100644
--- a/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java
+++ b/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java
@@ -23,6 +23,7 @@
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Predicate;
import com.google.common.base.Predicates;
+import com.google.common.collect.ImmutableSet;
import com.google.common.collect.ImmutableSortedSet;
import com.google.common.collect.Iterables;
import com.google.common.collect.Sets;
@@ -48,11 +49,11 @@
private static final Logger logger = LoggerFactory.getLogger(LeveledManifest.class);
/**
- * limit the number of L0 sstables we do at once, because compaction bloom filter creation
- * uses a pessimistic estimate of how many keys overlap (none), so we risk wasting memory
- * or even OOMing when compacting highly overlapping sstables
+ * if we have more than MAX_COMPACTING_L0 sstables in L0, we will run a round of STCS with at most
+ * cfs.getMaxCompactionThreshold() sstables.
*/
private static final int MAX_COMPACTING_L0 = 32;
+
/**
* If we go this many rounds without compacting
* in the highest level, we start bringing in sstables from
@@ -162,6 +163,7 @@
// The add(..):ed sstable will be sent to level 0
try
{
+ logger.debug("Could not add sstable {} in level {} - dropping to 0", reader, reader.getSSTableLevel());
reader.descriptor.getMetadataSerializer().mutateLevel(reader.descriptor, 0);
reader.reloadSSTableMetadata();
}
@@ -345,12 +347,12 @@
// L2: 12 [ideal: 100]
//
// The problem is that L0 has a much higher score (almost 250) than L1 (11), so what we'll
- // do is compact a batch of MAX_COMPACTING_L0 sstables with all 117 L1 sstables, and put the
- // result (say, 120 sstables) in L1. Then we'll compact the next batch of MAX_COMPACTING_L0,
+ // do is compact a batch of cfs.getMaximumCompactionThreshold() sstables with all 117 L1 sstables, and put the
+ // result (say, 120 sstables) in L1. Then we'll compact the next batch of cfs.getMaxCompactionThreshold(),
// and so forth. So we spend most of our i/o rewriting the L1 data with each batch.
//
// If we could just do *all* L0 a single time with L1, that would be ideal. But we can't
- // -- see the javadoc for MAX_COMPACTING_L0.
+ // since we might run out of memory
//
// LevelDB's way around this is to simply block writes if L0 compaction falls behind.
// We don't have that luxury.
@@ -364,6 +366,11 @@
// This isn't a magic wand -- if you are consistently writing too fast for LCS to keep
// up, you're still screwed. But if instead you have intermittent bursts of activity,
// it can help a lot.
+
+ // Let's check that L0 is far enough behind to warrant STCS.
+ // If it is, it will be used before proceeding any of higher level
+ CompactionCandidate l0Compaction = getSTCSInL0CompactionCandidate();
+
for (int i = generations.length - 1; i > 0; i--)
{
List<SSTableReader> sstables = getLevel(i);
@@ -378,7 +385,6 @@
if (score > 1.001)
{
// before proceeding with a higher level, let's see if L0 is far enough behind to warrant STCS
- CompactionCandidate l0Compaction = getSTCSInL0CompactionCandidate();
if (l0Compaction != null)
return l0Compaction;
@@ -408,7 +414,7 @@
// Since we don't have any other compactions to do, see if there is a STCS compaction to perform in L0; if
// there is a long running compaction, we want to make sure that we continue to keep the number of SSTables
// small in L0.
- return getSTCSInL0CompactionCandidate();
+ return l0Compaction;
}
return new CompactionCandidate(candidates, getNextLevel(candidates), maxSSTableSizeInBytes);
}
@@ -436,7 +442,8 @@
options.bucketHigh,
options.bucketLow,
options.minSSTableSize);
- return SizeTieredCompactionStrategy.mostInterestingBucket(buckets, 4, 32);
+ return SizeTieredCompactionStrategy.mostInterestingBucket(buckets,
+ cfs.getMinimumCompactionThreshold(), cfs.getMaximumCompactionThreshold());
}
/**
@@ -546,6 +553,16 @@
return level;
}
+ public synchronized Set<SSTableReader> getSSTables()
+ {
+ ImmutableSet.Builder<SSTableReader> builder = ImmutableSet.builder();
+ for (List<SSTableReader> sstables : generations)
+ {
+ builder.addAll(sstables);
+ }
+ return builder.build();
+ }
+
private static Set<SSTableReader> overlapping(Collection<SSTableReader> candidates, Iterable<SSTableReader> others)
{
assert !candidates.isEmpty();
@@ -651,7 +668,7 @@
// 1a. add sstables to the candidate set until we have at least maxSSTableSizeInMB
// 1b. prefer choosing older sstables as candidates, to newer ones
// 1c. any L0 sstables that overlap a candidate, will also become candidates
- // 2. At most MAX_COMPACTING_L0 sstables from L0 will be compacted at once
+ // 2. At most max_threshold sstables from L0 will be compacted at once
// 3. If total candidate size is less than maxSSTableSizeInMB, we won't bother compacting with L1,
// and the result of the compaction will stay in L0 instead of being promoted (see promote())
//
@@ -677,10 +694,10 @@
remaining.remove(newCandidate);
}
- if (candidates.size() > MAX_COMPACTING_L0)
+ if (candidates.size() > cfs.getMaximumCompactionThreshold())
{
- // limit to only the MAX_COMPACTING_L0 oldest candidates
- candidates = new HashSet<>(ageSortedSSTables(candidates).subList(0, MAX_COMPACTING_L0));
+ // limit to only the cfs.getMaximumCompactionThreshold() oldest candidates
+ candidates = new HashSet<>(ageSortedSSTables(candidates).subList(0, cfs.getMaximumCompactionThreshold()));
break;
}
}
@@ -803,6 +820,13 @@
tasks += estimated[i];
}
+ if (!DatabaseDescriptor.getDisableSTCSInL0() && getLevel(0).size() > cfs.getMaximumCompactionThreshold())
+ {
+ int l0compactions = getLevel(0).size() / cfs.getMaximumCompactionThreshold();
+ tasks += l0compactions;
+ estimated[0] += l0compactions;
+ }
+
logger.trace("Estimating {} compactions to do for {}.{}",
Arrays.toString(estimated), cfs.keyspace.getName(), cfs.name);
return Ints.checkedCast(tasks);
@@ -842,6 +866,14 @@
return sstables;
}
+ public synchronized void newLevel(SSTableReader sstable, int oldLevel)
+ {
+ boolean removed = generations[oldLevel].remove(sstable);
+ assert removed : "Could not remove " + sstable +" from " + oldLevel;
+ add(sstable);
+ lastCompactedKeys[oldLevel] = sstable.last;
+ }
+
public static class CompactionCandidate
{
public final Collection<SSTableReader> sstables;
diff --git a/src/java/org/apache/cassandra/db/compaction/PendingRepairHolder.java b/src/java/org/apache/cassandra/db/compaction/PendingRepairHolder.java
new file mode 100644
index 0000000..03d4111
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/compaction/PendingRepairHolder.java
@@ -0,0 +1,285 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.UUID;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Iterables;
+
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.SerializationHeader;
+import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.index.Index;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.ISSTableScanner;
+import org.apache.cassandra.io.sstable.SSTableMultiWriter;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
+import org.apache.cassandra.schema.CompactionParams;
+import org.apache.cassandra.service.ActiveRepairService;
+
+public class PendingRepairHolder extends AbstractStrategyHolder
+{
+ private final List<PendingRepairManager> managers = new ArrayList<>();
+ private final boolean isTransient;
+
+ public PendingRepairHolder(ColumnFamilyStore cfs, DestinationRouter router, boolean isTransient)
+ {
+ super(cfs, router);
+ this.isTransient = isTransient;
+ }
+
+ @Override
+ public void startup()
+ {
+ managers.forEach(PendingRepairManager::startup);
+ }
+
+ @Override
+ public void shutdown()
+ {
+ managers.forEach(PendingRepairManager::shutdown);
+ }
+
+ @Override
+ public void setStrategyInternal(CompactionParams params, int numTokenPartitions)
+ {
+ managers.clear();
+ for (int i = 0; i < numTokenPartitions; i++)
+ managers.add(new PendingRepairManager(cfs, params, isTransient));
+ }
+
+ @Override
+ public boolean managesRepairedGroup(boolean isRepaired, boolean isPendingRepair, boolean isTransient)
+ {
+ Preconditions.checkArgument(!isPendingRepair || !isRepaired,
+ "SSTables cannot be both repaired and pending repair");
+ return isPendingRepair && (this.isTransient == isTransient);
+ }
+
+ @Override
+ public AbstractCompactionStrategy getStrategyFor(SSTableReader sstable)
+ {
+ Preconditions.checkArgument(managesSSTable(sstable), "Attempting to get compaction strategy from wrong holder");
+ return managers.get(router.getIndexForSSTable(sstable)).getOrCreate(sstable);
+ }
+
+ @Override
+ public Iterable<AbstractCompactionStrategy> allStrategies()
+ {
+ return Iterables.concat(Iterables.transform(managers, PendingRepairManager::getStrategies));
+ }
+
+ Iterable<AbstractCompactionStrategy> getStrategiesFor(UUID session)
+ {
+ List<AbstractCompactionStrategy> strategies = new ArrayList<>(managers.size());
+ for (PendingRepairManager manager : managers)
+ {
+ AbstractCompactionStrategy strategy = manager.get(session);
+ if (strategy != null)
+ strategies.add(strategy);
+ }
+ return strategies;
+ }
+
+ public Iterable<PendingRepairManager> getManagers()
+ {
+ return managers;
+ }
+
+ @Override
+ public Collection<TaskSupplier> getBackgroundTaskSuppliers(int gcBefore)
+ {
+ List<TaskSupplier> suppliers = new ArrayList<>(managers.size());
+ for (PendingRepairManager manager : managers)
+ suppliers.add(new TaskSupplier(manager.getMaxEstimatedRemainingTasks(), () -> manager.getNextBackgroundTask(gcBefore)));
+
+ return suppliers;
+ }
+
+ @Override
+ public Collection<AbstractCompactionTask> getMaximalTasks(int gcBefore, boolean splitOutput)
+ {
+ List<AbstractCompactionTask> tasks = new ArrayList<>(managers.size());
+ for (PendingRepairManager manager : managers)
+ {
+ Collection<AbstractCompactionTask> task = manager.getMaximalTasks(gcBefore, splitOutput);
+ if (task != null)
+ tasks.addAll(task);
+ }
+ return tasks;
+ }
+
+ @Override
+ public Collection<AbstractCompactionTask> getUserDefinedTasks(GroupedSSTableContainer sstables, int gcBefore)
+ {
+ List<AbstractCompactionTask> tasks = new ArrayList<>(managers.size());
+
+ for (int i = 0; i < managers.size(); i++)
+ {
+ if (sstables.isGroupEmpty(i))
+ continue;
+
+ tasks.addAll(managers.get(i).createUserDefinedTasks(sstables.getGroup(i), gcBefore));
+ }
+ return tasks;
+ }
+
+ AbstractCompactionTask getNextRepairFinishedTask()
+ {
+ List<TaskSupplier> repairFinishedSuppliers = getRepairFinishedTaskSuppliers();
+ if (!repairFinishedSuppliers.isEmpty())
+ {
+ Collections.sort(repairFinishedSuppliers);
+ for (TaskSupplier supplier : repairFinishedSuppliers)
+ {
+ AbstractCompactionTask task = supplier.getTask();
+ if (task != null)
+ return task;
+ }
+ }
+ return null;
+ }
+
+ private ArrayList<TaskSupplier> getRepairFinishedTaskSuppliers()
+ {
+ ArrayList<TaskSupplier> suppliers = new ArrayList<>(managers.size());
+ for (PendingRepairManager manager : managers)
+ {
+ int numPending = manager.getNumPendingRepairFinishedTasks();
+ if (numPending > 0)
+ {
+ suppliers.add(new TaskSupplier(numPending, manager::getNextRepairFinishedTask));
+ }
+ }
+
+ return suppliers;
+ }
+
+ @Override
+ public void addSSTables(GroupedSSTableContainer sstables)
+ {
+ Preconditions.checkArgument(sstables.numGroups() == managers.size());
+ for (int i = 0; i < managers.size(); i++)
+ {
+ if (!sstables.isGroupEmpty(i))
+ managers.get(i).addSSTables(sstables.getGroup(i));
+ }
+ }
+
+ @Override
+ public void removeSSTables(GroupedSSTableContainer sstables)
+ {
+ Preconditions.checkArgument(sstables.numGroups() == managers.size());
+ for (int i = 0; i < managers.size(); i++)
+ {
+ if (!sstables.isGroupEmpty(i))
+ managers.get(i).removeSSTables(sstables.getGroup(i));
+ }
+ }
+
+ @Override
+ public void replaceSSTables(GroupedSSTableContainer removed, GroupedSSTableContainer added)
+ {
+ Preconditions.checkArgument(removed.numGroups() == managers.size());
+ Preconditions.checkArgument(added.numGroups() == managers.size());
+ for (int i = 0; i < managers.size(); i++)
+ {
+ if (removed.isGroupEmpty(i) && added.isGroupEmpty(i))
+ continue;
+
+ if (removed.isGroupEmpty(i))
+ managers.get(i).addSSTables(added.getGroup(i));
+ else
+ managers.get(i).replaceSSTables(removed.getGroup(i), added.getGroup(i));
+ }
+ }
+
+ @Override
+ public List<ISSTableScanner> getScanners(GroupedSSTableContainer sstables, Collection<Range<Token>> ranges)
+ {
+ List<ISSTableScanner> scanners = new ArrayList<>(managers.size());
+ for (int i = 0; i < managers.size(); i++)
+ {
+ if (sstables.isGroupEmpty(i))
+ continue;
+
+ scanners.addAll(managers.get(i).getScanners(sstables.getGroup(i), ranges));
+ }
+ return scanners;
+ }
+
+ @Override
+ public SSTableMultiWriter createSSTableMultiWriter(Descriptor descriptor,
+ long keyCount,
+ long repairedAt,
+ UUID pendingRepair,
+ boolean isTransient,
+ MetadataCollector collector,
+ SerializationHeader header,
+ Collection<Index> indexes,
+ LifecycleNewTracker lifecycleNewTracker)
+ {
+ Preconditions.checkArgument(repairedAt == ActiveRepairService.UNREPAIRED_SSTABLE,
+ "PendingRepairHolder can't create sstablewriter with repaired at set");
+ Preconditions.checkArgument(pendingRepair != null,
+ "PendingRepairHolder can't create sstable writer without pendingRepair id");
+ // to avoid creating a compaction strategy for the wrong pending repair manager, we get the index based on where the sstable is to be written
+ AbstractCompactionStrategy strategy = managers.get(router.getIndexForSSTableDirectory(descriptor)).getOrCreate(pendingRepair);
+ return strategy.createSSTableMultiWriter(descriptor,
+ keyCount,
+ repairedAt,
+ pendingRepair,
+ isTransient,
+ collector,
+ header,
+ indexes,
+ lifecycleNewTracker);
+ }
+
+ @Override
+ public int getStrategyIndex(AbstractCompactionStrategy strategy)
+ {
+ for (int i = 0; i < managers.size(); i++)
+ {
+ if (managers.get(i).hasStrategy(strategy))
+ return i;
+ }
+ return -1;
+ }
+
+ public boolean hasDataForSession(UUID sessionID)
+ {
+ return Iterables.any(managers, prm -> prm.hasDataForSession(sessionID));
+ }
+
+ @Override
+ public boolean containsSSTable(SSTableReader sstable)
+ {
+ return Iterables.any(managers, prm -> prm.containsSSTable(sstable));
+ }
+}
diff --git a/src/java/org/apache/cassandra/db/compaction/PendingRepairManager.java b/src/java/org/apache/cassandra/db/compaction/PendingRepairManager.java
new file mode 100644
index 0000000..764a4dc
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/compaction/PendingRepairManager.java
@@ -0,0 +1,489 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.stream.Collectors;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Maps;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Directories;
+import org.apache.cassandra.db.compaction.writers.CompactionAwareWriter;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.io.sstable.ISSTableScanner;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.schema.CompactionParams;
+import org.apache.cassandra.service.ActiveRepairService;
+import org.apache.cassandra.utils.Pair;
+
+/**
+ * Companion to CompactionStrategyManager which manages the sstables marked pending repair.
+ *
+ * SSTables are classified as pending repair by the anti-compaction performed at the beginning
+ * of an incremental repair, or when they're streamed in with a pending repair id. This prevents
+ * unrepaired / pending repaired sstables from being compacted together. Once the repair session
+ * has completed, or failed, sstables will be re-classified as part of the compaction process.
+ */
+class PendingRepairManager
+{
+ private static final Logger logger = LoggerFactory.getLogger(PendingRepairManager.class);
+
+ private final ColumnFamilyStore cfs;
+ private final CompactionParams params;
+ private final boolean isTransient;
+ private volatile ImmutableMap<UUID, AbstractCompactionStrategy> strategies = ImmutableMap.of();
+
+ /**
+ * Indicates we're being asked to do something with an sstable that isn't marked pending repair
+ */
+ public static class IllegalSSTableArgumentException extends IllegalArgumentException
+ {
+ public IllegalSSTableArgumentException(String s)
+ {
+ super(s);
+ }
+ }
+
+ PendingRepairManager(ColumnFamilyStore cfs, CompactionParams params, boolean isTransient)
+ {
+ this.cfs = cfs;
+ this.params = params;
+ this.isTransient = isTransient;
+ }
+
+ private ImmutableMap.Builder<UUID, AbstractCompactionStrategy> mapBuilder()
+ {
+ return ImmutableMap.builder();
+ }
+
+ AbstractCompactionStrategy get(UUID id)
+ {
+ return strategies.get(id);
+ }
+
+ AbstractCompactionStrategy get(SSTableReader sstable)
+ {
+ assert sstable.isPendingRepair();
+ return get(sstable.getSSTableMetadata().pendingRepair);
+ }
+
+ AbstractCompactionStrategy getOrCreate(UUID id)
+ {
+ checkPendingID(id);
+ assert id != null;
+ AbstractCompactionStrategy strategy = get(id);
+ if (strategy == null)
+ {
+ synchronized (this)
+ {
+ strategy = get(id);
+
+ if (strategy == null)
+ {
+ logger.debug("Creating {}.{} compaction strategy for pending repair: {}", cfs.metadata.keyspace, cfs.metadata.name, id);
+ strategy = cfs.createCompactionStrategyInstance(params);
+ strategies = mapBuilder().putAll(strategies).put(id, strategy).build();
+ }
+ }
+ }
+ return strategy;
+ }
+
+ private static void checkPendingID(UUID pendingID)
+ {
+ if (pendingID == null)
+ {
+ throw new IllegalSSTableArgumentException("sstable is not pending repair");
+ }
+ }
+
+ AbstractCompactionStrategy getOrCreate(SSTableReader sstable)
+ {
+ return getOrCreate(sstable.getSSTableMetadata().pendingRepair);
+ }
+
+ private synchronized void removeSessionIfEmpty(UUID sessionID)
+ {
+ if (!strategies.containsKey(sessionID) || !strategies.get(sessionID).getSSTables().isEmpty())
+ return;
+
+ logger.debug("Removing compaction strategy for pending repair {} on {}.{}", sessionID, cfs.metadata.keyspace, cfs.metadata.name);
+ strategies = ImmutableMap.copyOf(Maps.filterKeys(strategies, k -> !k.equals(sessionID)));
+ }
+
+ synchronized void removeSSTable(SSTableReader sstable)
+ {
+ for (Map.Entry<UUID, AbstractCompactionStrategy> entry : strategies.entrySet())
+ {
+ entry.getValue().removeSSTable(sstable);
+ removeSessionIfEmpty(entry.getKey());
+ }
+ }
+
+
+ void removeSSTables(Iterable<SSTableReader> removed)
+ {
+ for (SSTableReader sstable : removed)
+ removeSSTable(sstable);
+ }
+
+ synchronized void addSSTable(SSTableReader sstable)
+ {
+ Preconditions.checkArgument(sstable.isTransient() == isTransient);
+ getOrCreate(sstable).addSSTable(sstable);
+ }
+
+ void addSSTables(Iterable<SSTableReader> added)
+ {
+ for (SSTableReader sstable : added)
+ addSSTable(sstable);
+ }
+
+ synchronized void replaceSSTables(Set<SSTableReader> removed, Set<SSTableReader> added)
+ {
+ if (removed.isEmpty() && added.isEmpty())
+ return;
+
+ // left=removed, right=added
+ Map<UUID, Pair<Set<SSTableReader>, Set<SSTableReader>>> groups = new HashMap<>();
+ for (SSTableReader sstable : removed)
+ {
+ UUID sessionID = sstable.getSSTableMetadata().pendingRepair;
+ if (!groups.containsKey(sessionID))
+ {
+ groups.put(sessionID, Pair.create(new HashSet<>(), new HashSet<>()));
+ }
+ groups.get(sessionID).left.add(sstable);
+ }
+
+ for (SSTableReader sstable : added)
+ {
+ UUID sessionID = sstable.getSSTableMetadata().pendingRepair;
+ if (!groups.containsKey(sessionID))
+ {
+ groups.put(sessionID, Pair.create(new HashSet<>(), new HashSet<>()));
+ }
+ groups.get(sessionID).right.add(sstable);
+ }
+
+ for (Map.Entry<UUID, Pair<Set<SSTableReader>, Set<SSTableReader>>> entry : groups.entrySet())
+ {
+ AbstractCompactionStrategy strategy = getOrCreate(entry.getKey());
+ Set<SSTableReader> groupRemoved = entry.getValue().left;
+ Set<SSTableReader> groupAdded = entry.getValue().right;
+
+ if (!groupRemoved.isEmpty())
+ strategy.replaceSSTables(groupRemoved, groupAdded);
+ else
+ strategy.addSSTables(groupAdded);
+
+ removeSessionIfEmpty(entry.getKey());
+ }
+ }
+
+ synchronized void startup()
+ {
+ strategies.values().forEach(AbstractCompactionStrategy::startup);
+ }
+
+ synchronized void shutdown()
+ {
+ strategies.values().forEach(AbstractCompactionStrategy::shutdown);
+ }
+
+ private int getEstimatedRemainingTasks(UUID sessionID, AbstractCompactionStrategy strategy)
+ {
+ if (canCleanup(sessionID))
+ {
+ return 0;
+ }
+ else
+ {
+ return strategy.getEstimatedRemainingTasks();
+ }
+ }
+
+ int getEstimatedRemainingTasks()
+ {
+ int tasks = 0;
+ for (Map.Entry<UUID, AbstractCompactionStrategy> entry : strategies.entrySet())
+ {
+ tasks += getEstimatedRemainingTasks(entry.getKey(), entry.getValue());
+ }
+ return tasks;
+ }
+
+ /**
+ * @return the highest max remaining tasks of all contained compaction strategies
+ */
+ int getMaxEstimatedRemainingTasks()
+ {
+ int tasks = 0;
+ for (Map.Entry<UUID, AbstractCompactionStrategy> entry : strategies.entrySet())
+ {
+ tasks = Math.max(tasks, getEstimatedRemainingTasks(entry.getKey(), entry.getValue()));
+ }
+ return tasks;
+ }
+
+ @SuppressWarnings("resource")
+ private RepairFinishedCompactionTask getRepairFinishedCompactionTask(UUID sessionID)
+ {
+ Set<SSTableReader> sstables = get(sessionID).getSSTables();
+ long repairedAt = ActiveRepairService.instance.consistent.local.getFinalSessionRepairedAt(sessionID);
+ LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.COMPACTION);
+ return txn == null ? null : new RepairFinishedCompactionTask(cfs, txn, sessionID, repairedAt);
+ }
+
+ synchronized int getNumPendingRepairFinishedTasks()
+ {
+ int count = 0;
+ for (UUID sessionID : strategies.keySet())
+ {
+ if (canCleanup(sessionID))
+ {
+ count++;
+ }
+ }
+ return count;
+ }
+
+ synchronized AbstractCompactionTask getNextRepairFinishedTask()
+ {
+ for (UUID sessionID : strategies.keySet())
+ {
+ if (canCleanup(sessionID))
+ {
+ return getRepairFinishedCompactionTask(sessionID);
+ }
+ }
+ return null;
+ }
+
+ synchronized AbstractCompactionTask getNextBackgroundTask(int gcBefore)
+ {
+ if (strategies.isEmpty())
+ return null;
+
+ Map<UUID, Integer> numTasks = new HashMap<>(strategies.size());
+ ArrayList<UUID> sessions = new ArrayList<>(strategies.size());
+ for (Map.Entry<UUID, AbstractCompactionStrategy> entry : strategies.entrySet())
+ {
+ if (canCleanup(entry.getKey()))
+ {
+ continue;
+ }
+ numTasks.put(entry.getKey(), getEstimatedRemainingTasks(entry.getKey(), entry.getValue()));
+ sessions.add(entry.getKey());
+ }
+
+ if (sessions.isEmpty())
+ return null;
+
+ // we want the session with the most compactions at the head of the list
+ sessions.sort((o1, o2) -> numTasks.get(o2) - numTasks.get(o1));
+
+ UUID sessionID = sessions.get(0);
+ return get(sessionID).getNextBackgroundTask(gcBefore);
+ }
+
+ synchronized Collection<AbstractCompactionTask> getMaximalTasks(int gcBefore, boolean splitOutput)
+ {
+ if (strategies.isEmpty())
+ return null;
+
+ List<AbstractCompactionTask> maximalTasks = new ArrayList<>(strategies.size());
+ for (Map.Entry<UUID, AbstractCompactionStrategy> entry : strategies.entrySet())
+ {
+ if (canCleanup(entry.getKey()))
+ {
+ maximalTasks.add(getRepairFinishedCompactionTask(entry.getKey()));
+ }
+ else
+ {
+ Collection<AbstractCompactionTask> tasks = entry.getValue().getMaximalTask(gcBefore, splitOutput);
+ if (tasks != null)
+ maximalTasks.addAll(tasks);
+ }
+ }
+ return !maximalTasks.isEmpty() ? maximalTasks : null;
+ }
+
+ Collection<AbstractCompactionStrategy> getStrategies()
+ {
+ return strategies.values();
+ }
+
+ Set<UUID> getSessions()
+ {
+ return strategies.keySet();
+ }
+
+ boolean canCleanup(UUID sessionID)
+ {
+ return !ActiveRepairService.instance.consistent.local.isSessionInProgress(sessionID);
+ }
+
+ @SuppressWarnings("resource")
+ synchronized Set<ISSTableScanner> getScanners(Collection<SSTableReader> sstables, Collection<Range<Token>> ranges)
+ {
+ if (sstables.isEmpty())
+ {
+ return Collections.emptySet();
+ }
+
+ Map<UUID, Set<SSTableReader>> sessionSSTables = new HashMap<>();
+ for (SSTableReader sstable : sstables)
+ {
+ UUID sessionID = sstable.getSSTableMetadata().pendingRepair;
+ checkPendingID(sessionID);
+ sessionSSTables.computeIfAbsent(sessionID, k -> new HashSet<>()).add(sstable);
+ }
+
+ Set<ISSTableScanner> scanners = new HashSet<>(sessionSSTables.size());
+ try
+ {
+ for (Map.Entry<UUID, Set<SSTableReader>> entry : sessionSSTables.entrySet())
+ {
+ scanners.addAll(getOrCreate(entry.getKey()).getScanners(entry.getValue(), ranges).scanners);
+ }
+ }
+ catch (Throwable t)
+ {
+ ISSTableScanner.closeAllAndPropagate(scanners, t);
+ }
+ return scanners;
+ }
+
+ public boolean hasStrategy(AbstractCompactionStrategy strategy)
+ {
+ return strategies.values().contains(strategy);
+ }
+
+ public synchronized boolean hasDataForSession(UUID sessionID)
+ {
+ return strategies.keySet().contains(sessionID);
+ }
+
+ boolean containsSSTable(SSTableReader sstable)
+ {
+ if (!sstable.isPendingRepair())
+ return false;
+
+ AbstractCompactionStrategy strategy = strategies.get(sstable.getPendingRepair());
+ return strategy != null && strategy.getSSTables().contains(sstable);
+ }
+
+ public Collection<AbstractCompactionTask> createUserDefinedTasks(Collection<SSTableReader> sstables, int gcBefore)
+ {
+ Map<UUID, List<SSTableReader>> group = sstables.stream().collect(Collectors.groupingBy(s -> s.getSSTableMetadata().pendingRepair));
+ return group.entrySet().stream().map(g -> strategies.get(g.getKey()).getUserDefinedTask(g.getValue(), gcBefore)).collect(Collectors.toList());
+ }
+
+ /**
+ * promotes/demotes sstables involved in a consistent repair that has been finalized, or failed
+ */
+ class RepairFinishedCompactionTask extends AbstractCompactionTask
+ {
+ private final UUID sessionID;
+ private final long repairedAt;
+
+ RepairFinishedCompactionTask(ColumnFamilyStore cfs, LifecycleTransaction transaction, UUID sessionID, long repairedAt)
+ {
+ super(cfs, transaction);
+ this.sessionID = sessionID;
+ this.repairedAt = repairedAt;
+ }
+
+ @VisibleForTesting
+ UUID getSessionID()
+ {
+ return sessionID;
+ }
+
+ protected void runMayThrow() throws Exception
+ {
+ boolean completed = false;
+ boolean obsoleteSSTables = isTransient && repairedAt > 0;
+ try
+ {
+ if (obsoleteSSTables)
+ {
+ logger.info("Obsoleting transient repaired sstables for {}", sessionID);
+ Preconditions.checkState(Iterables.all(transaction.originals(), SSTableReader::isTransient));
+ transaction.obsoleteOriginals();
+ }
+ else
+ {
+ logger.info("Moving {} from pending to repaired with repaired at = {} and session id = {}", transaction.originals(), repairedAt, sessionID);
+ cfs.getCompactionStrategyManager().mutateRepaired(transaction.originals(), repairedAt, ActiveRepairService.NO_PENDING_REPAIR, false);
+ }
+ completed = true;
+ }
+ finally
+ {
+ if (obsoleteSSTables)
+ {
+ transaction.finish();
+ }
+ else
+ {
+ // we abort here because mutating metadata isn't guarded by LifecycleTransaction, so this won't roll
+ // anything back. Also, we don't want to obsolete the originals. We're only using it to prevent other
+ // compactions from marking these sstables compacting, and unmarking them when we're done
+ transaction.abort();
+ }
+ if (completed)
+ {
+ removeSessionIfEmpty(sessionID);
+ }
+ }
+ }
+
+ public CompactionAwareWriter getCompactionAwareWriter(ColumnFamilyStore cfs, Directories directories, LifecycleTransaction txn, Set<SSTableReader> nonExpiredSSTables)
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ protected int executeInternal(ActiveCompactionsTracker activeCompactions)
+ {
+ run();
+ return transaction.originals().size();
+ }
+ }
+
+}
diff --git a/src/java/org/apache/cassandra/db/compaction/SSTableSplitter.java b/src/java/org/apache/cassandra/db/compaction/SSTableSplitter.java
index 924e29c..1746d7c 100644
--- a/src/java/org/apache/cassandra/db/compaction/SSTableSplitter.java
+++ b/src/java/org/apache/cassandra/db/compaction/SSTableSplitter.java
@@ -18,6 +18,7 @@
package org.apache.cassandra.db.compaction;
import java.util.*;
+import java.util.function.LongPredicate;
import java.util.function.Predicate;
import org.apache.cassandra.db.*;
@@ -30,8 +31,6 @@
{
private final SplittingCompactionTask task;
- private CompactionInfo.Holder info;
-
public SSTableSplitter(ColumnFamilyStore cfs, LifecycleTransaction transaction, int sstableSizeInMB)
{
this.task = new SplittingCompactionTask(cfs, transaction, sstableSizeInMB);
@@ -39,20 +38,7 @@
public void split()
{
- task.execute(new StatsCollector());
- }
-
- public class StatsCollector implements CompactionManager.CompactionExecutorStatsCollector
- {
- public void beginCompaction(CompactionInfo.Holder ci)
- {
- SSTableSplitter.this.info = ci;
- }
-
- public void finishCompaction(CompactionInfo.Holder ci)
- {
- // no-op
- }
+ task.execute(ActiveCompactionsTracker.NOOP);
}
public static class SplittingCompactionTask extends CompactionTask
@@ -98,7 +84,7 @@
}
@Override
- public Predicate<Long> getPurgeEvaluator(DecoratedKey key)
+ public LongPredicate getPurgeEvaluator(DecoratedKey key)
{
return time -> false;
}
diff --git a/src/java/org/apache/cassandra/db/compaction/Scrubber.java b/src/java/org/apache/cassandra/db/compaction/Scrubber.java
index 622e793..92cff54 100644
--- a/src/java/org/apache/cassandra/db/compaction/Scrubber.java
+++ b/src/java/org/apache/cassandra/db/compaction/Scrubber.java
@@ -23,8 +23,9 @@
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Throwables;
+import com.google.common.collect.ImmutableSet;
-import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
import org.apache.cassandra.db.rows.*;
@@ -32,6 +33,7 @@
import org.apache.cassandra.io.sstable.*;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.sstable.format.SSTableWriter;
+import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.io.util.RandomAccessReader;
import org.apache.cassandra.service.ActiveRepairService;
@@ -73,20 +75,20 @@
private static final Comparator<Partition> partitionComparator = new Comparator<Partition>()
{
- public int compare(Partition r1, Partition r2)
- {
- return r1.partitionKey().compareTo(r2.partitionKey());
- }
+ public int compare(Partition r1, Partition r2)
+ {
+ return r1.partitionKey().compareTo(r2.partitionKey());
+ }
};
private final SortedSet<Partition> outOfOrder = new TreeSet<>(partitionComparator);
- public Scrubber(ColumnFamilyStore cfs, LifecycleTransaction transaction, boolean skipCorrupted, boolean checkData) throws IOException
+ public Scrubber(ColumnFamilyStore cfs, LifecycleTransaction transaction, boolean skipCorrupted, boolean checkData)
{
this(cfs, transaction, skipCorrupted, checkData, false);
}
public Scrubber(ColumnFamilyStore cfs, LifecycleTransaction transaction, boolean skipCorrupted, boolean checkData,
- boolean reinsertOverflowedTTLRows) throws IOException
+ boolean reinsertOverflowedTTLRows)
{
this(cfs, transaction, skipCorrupted, new OutputHandler.LogOutput(), checkData, reinsertOverflowedTTLRows);
}
@@ -97,7 +99,7 @@
boolean skipCorrupted,
OutputHandler outputHandler,
boolean checkData,
- boolean reinsertOverflowedTTLRows) throws IOException
+ boolean reinsertOverflowedTTLRows)
{
this.cfs = cfs;
this.transaction = transaction;
@@ -105,15 +107,14 @@
this.outputHandler = outputHandler;
this.skipCorrupted = skipCorrupted;
this.reinsertOverflowedTTLRows = reinsertOverflowedTTLRows;
- this.rowIndexEntrySerializer = sstable.descriptor.version.getSSTableFormat().getIndexSerializer(sstable.metadata,
+ this.rowIndexEntrySerializer = sstable.descriptor.version.getSSTableFormat().getIndexSerializer(cfs.metadata(),
sstable.descriptor.version,
sstable.header);
List<SSTableReader> toScrub = Collections.singletonList(sstable);
-
this.destination = cfs.getDirectories().getLocationForDisk(cfs.getDiskBoundaries().getCorrectDiskForSSTable(sstable));
- this.isCommutative = cfs.metadata.isCounter();
+ this.isCommutative = cfs.metadata().isCounter();
boolean hasIndexFile = (new File(sstable.descriptor.filenameFor(Component.PRIMARY_INDEX))).exists();
this.isIndex = cfs.isIndex();
@@ -124,8 +125,8 @@
}
this.checkData = checkData && !this.isIndex; //LocalByPartitionerType does not support validation
this.expectedBloomFilterSize = Math.max(
- cfs.metadata.params.minIndexInterval,
- hasIndexFile ? SSTableReader.getApproximateKeyCount(toScrub) : 0);
+ cfs.metadata().params.minIndexInterval,
+ hasIndexFile ? SSTableReader.getApproximateKeyCount(toScrub) : 0);
// loop through each row, deserializing to check for damage.
// we'll also loop through the index at the same time, using the position from the index to recover if the
@@ -136,8 +137,8 @@
: sstable.openDataReader(CompactionManager.instance.getRateLimiter());
this.indexFile = hasIndexFile
- ? RandomAccessReader.open(new File(sstable.descriptor.filenameFor(Component.PRIMARY_INDEX)))
- : null;
+ ? RandomAccessReader.open(new File(sstable.descriptor.filenameFor(Component.PRIMARY_INDEX)))
+ : null;
this.scrubInfo = new ScrubInfo(dataFile, sstable);
@@ -169,7 +170,8 @@
assert firstRowPositionFromIndex == 0 : firstRowPositionFromIndex;
}
- writer.switchWriter(CompactionManager.createWriter(cfs, destination, expectedBloomFilterSize, sstable.getSSTableMetadata().repairedAt, sstable, transaction));
+ StatsMetadata metadata = sstable.getSSTableMetadata();
+ writer.switchWriter(CompactionManager.createWriter(cfs, destination, expectedBloomFilterSize, metadata.repairedAt, metadata.pendingRepair, metadata.isTransient, sstable, transaction));
DecoratedKey prevKey = null;
@@ -218,8 +220,8 @@
if (currentIndexKey != null && !key.getKey().equals(currentIndexKey))
{
throw new IOError(new IOException(String.format("Key from data file (%s) does not match key from index file (%s)",
- //ByteBufferUtil.bytesToHex(key.getKey()), ByteBufferUtil.bytesToHex(currentIndexKey))));
- "_too big_", ByteBufferUtil.bytesToHex(currentIndexKey))));
+ //ByteBufferUtil.bytesToHex(key.getKey()), ByteBufferUtil.bytesToHex(currentIndexKey))));
+ "_too big_", ByteBufferUtil.bytesToHex(currentIndexKey))));
}
if (indexFile != null && dataSizeFromIndex > dataFile.length())
@@ -240,7 +242,7 @@
&& (key == null || !key.getKey().equals(currentIndexKey) || dataStart != dataStartFromIndex))
{
outputHandler.output(String.format("Retrying from row index; data is %s bytes starting at %s",
- dataSizeFromIndex, dataStartFromIndex));
+ dataSizeFromIndex, dataStartFromIndex));
key = sstable.decorateKey(currentIndexKey);
try
{
@@ -274,9 +276,9 @@
if (!outOfOrder.isEmpty())
{
// out of order rows, but no bad rows found - we can keep our repairedAt time
- long repairedAt = badRows > 0 ? ActiveRepairService.UNREPAIRED_SSTABLE : sstable.getSSTableMetadata().repairedAt;
+ long repairedAt = badRows > 0 ? ActiveRepairService.UNREPAIRED_SSTABLE : metadata.repairedAt;
SSTableReader newInOrderSstable;
- try (SSTableWriter inOrderWriter = CompactionManager.createWriter(cfs, destination, expectedBloomFilterSize, repairedAt, sstable, transaction))
+ try (SSTableWriter inOrderWriter = CompactionManager.createWriter(cfs, destination, expectedBloomFilterSize, repairedAt, metadata.pendingRepair, metadata.isTransient, sstable, transaction))
{
for (Partition partition : outOfOrder)
inOrderWriter.append(partition.unfilteredIterator());
@@ -325,7 +327,7 @@
// that one row is out of order, it will stop returning them. The remaining rows will be sorted and added
// to the outOfOrder set that will be later written to a new SSTable.
OrderCheckerIterator sstableIterator = new OrderCheckerIterator(getIterator(key),
- cfs.metadata.comparator);
+ cfs.metadata().comparator);
try (UnfilteredRowIterator iterator = withValidation(sstableIterator, dataFile.getPath()))
{
@@ -354,6 +356,7 @@
* Only wrap with {@link FixNegativeLocalDeletionTimeIterator} if {@link #reinsertOverflowedTTLRows} option
* is specified
*/
+ @SuppressWarnings("resource")
private UnfilteredRowIterator getIterator(DecoratedKey key)
{
RowMergingSSTableIterator rowMergingIterator = new RowMergingSSTableIterator(SSTableIdentityIterator.create(sstable, dataFile, key));
@@ -371,8 +374,8 @@
nextIndexKey = !indexAvailable() ? null : ByteBufferUtil.readWithShortLength(indexFile);
nextRowPositionFromIndex = !indexAvailable()
- ? dataFile.length()
- : rowIndexEntrySerializer.deserializePositionAndSkip(indexFile);
+ ? dataFile.length()
+ : rowIndexEntrySerializer.deserializePositionAndSkip(indexFile);
}
catch (Throwable th)
{
@@ -469,11 +472,12 @@
{
try
{
- return new CompactionInfo(sstable.metadata,
+ return new CompactionInfo(sstable.metadata(),
OperationType.SCRUB,
dataFile.getFilePointer(),
dataFile.length(),
- scrubCompactionId);
+ scrubCompactionId,
+ ImmutableSet.of(sstable));
}
catch (Exception e)
{
@@ -551,7 +555,7 @@
}
// Duplicate row, merge it.
- next = Rows.merge((Row) next, (Row) peek, FBUtilities.nowInSeconds());
+ next = Rows.merge((Row) next, (Row) peek);
}
}
@@ -586,7 +590,7 @@
this.comparator = comparator;
}
- public CFMetaData metadata()
+ public TableMetadata metadata()
{
return iterator.metadata();
}
@@ -596,7 +600,7 @@
return iterator.isReverseOrder();
}
- public PartitionColumns columns()
+ public RegularAndStaticColumns columns()
{
return iterator.columns();
}
@@ -684,7 +688,7 @@
this.negativeLocalExpirationTimeMetrics = negativeLocalDeletionInfoMetrics;
}
- public CFMetaData metadata()
+ public TableMetadata metadata()
{
return iterator.metadata();
}
@@ -694,7 +698,7 @@
return iterator.isReverseOrder();
}
- public PartitionColumns columns()
+ public RegularAndStaticColumns columns()
{
return iterator.columns();
}
@@ -807,5 +811,4 @@
return builder.build();
}
}
-
}
diff --git a/src/java/org/apache/cassandra/db/compaction/SingleSSTableLCSTask.java b/src/java/org/apache/cassandra/db/compaction/SingleSSTableLCSTask.java
new file mode 100644
index 0000000..02a1c49
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/compaction/SingleSSTableLCSTask.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+import java.util.Set;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Directories;
+import org.apache.cassandra.db.compaction.writers.CompactionAwareWriter;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.io.sstable.Component;
+import org.apache.cassandra.io.sstable.CorruptSSTableException;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
+
+/**
+ * Special compaction task that does not do any compaction, instead it
+ * just mutates the level metadata on the sstable and notifies the compaction
+ * strategy.
+ */
+public class SingleSSTableLCSTask extends AbstractCompactionTask
+{
+ private static final Logger logger = LoggerFactory.getLogger(SingleSSTableLCSTask.class);
+
+ private final int level;
+
+ public SingleSSTableLCSTask(ColumnFamilyStore cfs, LifecycleTransaction txn, int level)
+ {
+ super(cfs, txn);
+ assert txn.originals().size() == 1;
+ this.level = level;
+ }
+
+ @Override
+ public CompactionAwareWriter getCompactionAwareWriter(ColumnFamilyStore cfs, Directories directories, LifecycleTransaction txn, Set<SSTableReader> nonExpiredSSTables)
+ {
+ throw new UnsupportedOperationException("This method should never be called on SingleSSTableLCSTask");
+ }
+
+ @Override
+ protected int executeInternal(ActiveCompactionsTracker activeCompactions)
+ {
+ run();
+ return 1;
+ }
+
+ @Override
+ protected void runMayThrow()
+ {
+ SSTableReader sstable = transaction.onlyOne();
+ StatsMetadata metadataBefore = sstable.getSSTableMetadata();
+ if (level == metadataBefore.sstableLevel)
+ {
+ logger.info("Not compacting {}, level is already {}", sstable, level);
+ }
+ else
+ {
+ try
+ {
+ logger.info("Changing level on {} from {} to {}", sstable, metadataBefore.sstableLevel, level);
+ sstable.descriptor.getMetadataSerializer().mutateLevel(sstable.descriptor, level);
+ sstable.reloadSSTableMetadata();
+ }
+ catch (Throwable t)
+ {
+ transaction.abort();
+ throw new CorruptSSTableException(t, sstable.descriptor.filenameFor(Component.DATA));
+ }
+ cfs.getTracker().notifySSTableMetadataChanged(sstable, metadataBefore);
+ }
+ finishTransaction(sstable);
+ }
+
+ private void finishTransaction(SSTableReader sstable)
+ {
+ // we simply cancel the transaction since no sstables are added or removed - we just
+ // write a new sstable metadata above and then atomically move the new file on top of the old
+ transaction.cancel(sstable);
+ transaction.prepareToCommit();
+ transaction.commit();
+ }
+}
diff --git a/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategy.java b/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategy.java
index 01b9181..a79299e 100644
--- a/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategy.java
+++ b/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategy.java
@@ -21,6 +21,7 @@
import java.util.Map.Entry;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -340,6 +341,12 @@
sstables.remove(sstable);
}
+ @Override
+ protected Set<SSTableReader> getSSTables()
+ {
+ return ImmutableSet.copyOf(sstables);
+ }
+
public String toString()
{
return String.format("SizeTieredCompactionStrategy[%s/%s]",
diff --git a/src/java/org/apache/cassandra/db/compaction/TimeWindowCompactionStrategy.java b/src/java/org/apache/cassandra/db/compaction/TimeWindowCompactionStrategy.java
index 6186826..4166805 100644
--- a/src/java/org/apache/cassandra/db/compaction/TimeWindowCompactionStrategy.java
+++ b/src/java/org/apache/cassandra/db/compaction/TimeWindowCompactionStrategy.java
@@ -190,6 +190,12 @@
sstables.remove(sstable);
}
+ @Override
+ protected Set<SSTableReader> getSSTables()
+ {
+ return ImmutableSet.copyOf(sstables);
+ }
+
/**
* Find the lowest and highest timestamps in a given timestamp/unit pair
* Returns milliseconds, caller should adjust accordingly
diff --git a/src/java/org/apache/cassandra/db/compaction/TimeWindowCompactionStrategyOptions.java b/src/java/org/apache/cassandra/db/compaction/TimeWindowCompactionStrategyOptions.java
index 24b4fe0..8b2ba23 100644
--- a/src/java/org/apache/cassandra/db/compaction/TimeWindowCompactionStrategyOptions.java
+++ b/src/java/org/apache/cassandra/db/compaction/TimeWindowCompactionStrategyOptions.java
@@ -63,7 +63,7 @@
String optionValue = options.get(TIMESTAMP_RESOLUTION_KEY);
timestampResolution = optionValue == null ? DEFAULT_TIMESTAMP_RESOLUTION : TimeUnit.valueOf(optionValue);
if (timestampResolution != DEFAULT_TIMESTAMP_RESOLUTION)
- logger.warn("Using a non-default timestamp_resolution {} - are you really doing inserts with USING TIMESTAMP <non_microsecond_timestamp> (or driver equivalent)?", timestampResolution.toString());
+ logger.warn("Using a non-default timestamp_resolution {} - are you really doing inserts with USING TIMESTAMP <non_microsecond_timestamp> (or driver equivalent)?", timestampResolution);
optionValue = options.get(COMPACTION_WINDOW_UNIT_KEY);
sstableWindowUnit = optionValue == null ? DEFAULT_COMPACTION_WINDOW_UNIT : TimeUnit.valueOf(optionValue);
diff --git a/src/java/org/apache/cassandra/db/compaction/Upgrader.java b/src/java/org/apache/cassandra/db/compaction/Upgrader.java
index 7a5b719..e1406aa 100644
--- a/src/java/org/apache/cassandra/db/compaction/Upgrader.java
+++ b/src/java/org/apache/cassandra/db/compaction/Upgrader.java
@@ -19,6 +19,7 @@
import java.io.File;
import java.util.*;
+import java.util.function.LongPredicate;
import java.util.function.Predicate;
import com.google.common.base.Throwables;
@@ -32,6 +33,7 @@
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.sstable.format.SSTableWriter;
import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
+import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.OutputHandler;
import org.apache.cassandra.utils.UUIDGen;
@@ -61,21 +63,23 @@
this.controller = new UpgradeController(cfs);
this.strategyManager = cfs.getCompactionStrategyManager();
- long estimatedTotalKeys = Math.max(cfs.metadata.params.minIndexInterval, SSTableReader.getApproximateKeyCount(Arrays.asList(this.sstable)));
+ long estimatedTotalKeys = Math.max(cfs.metadata().params.minIndexInterval, SSTableReader.getApproximateKeyCount(Arrays.asList(this.sstable)));
long estimatedSSTables = Math.max(1, SSTableReader.getTotalBytes(Arrays.asList(this.sstable)) / strategyManager.getMaxSSTableBytes());
this.estimatedRows = (long) Math.ceil((double) estimatedTotalKeys / estimatedSSTables);
}
- private SSTableWriter createCompactionWriter(long repairedAt)
+ private SSTableWriter createCompactionWriter(StatsMetadata metadata)
{
MetadataCollector sstableMetadataCollector = new MetadataCollector(cfs.getComparator());
sstableMetadataCollector.sstableLevel(sstable.getSSTableLevel());
- return SSTableWriter.create(Descriptor.fromFilename(cfs.getSSTablePath(directory)),
+ return SSTableWriter.create(cfs.newSSTableDescriptor(directory),
estimatedRows,
- repairedAt,
+ metadata.repairedAt,
+ metadata.pendingRepair,
+ metadata.isTransient,
cfs.metadata,
sstableMetadataCollector,
- SerializationHeader.make(cfs.metadata, Sets.newHashSet(sstable)),
+ SerializationHeader.make(cfs.metadata(), Sets.newHashSet(sstable)),
cfs.indexManager.listIndexes(),
transaction);
}
@@ -88,7 +92,7 @@
AbstractCompactionStrategy.ScannerList scanners = strategyManager.getScanners(transaction.originals());
CompactionIterator iter = new CompactionIterator(transaction.opType(), scanners.scanners, controller, nowInSec, UUIDGen.getTimeUUID()))
{
- writer.switchWriter(createCompactionWriter(sstable.getSSTableMetadata().repairedAt));
+ writer.switchWriter(createCompactionWriter(sstable.getSSTableMetadata()));
while (iter.hasNext())
writer.append(iter.next());
@@ -113,7 +117,7 @@
}
@Override
- public Predicate<Long> getPurgeEvaluator(DecoratedKey key)
+ public LongPredicate getPurgeEvaluator(DecoratedKey key)
{
return time -> false;
}
diff --git a/src/java/org/apache/cassandra/db/compaction/Verifier.java b/src/java/org/apache/cassandra/db/compaction/Verifier.java
index 8c5e8bb..2500a24 100644
--- a/src/java/org/apache/cassandra/db/compaction/Verifier.java
+++ b/src/java/org/apache/cassandra/db/compaction/Verifier.java
@@ -17,13 +17,19 @@
*/
package org.apache.cassandra.db.compaction;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Throwables;
+import com.google.common.collect.ImmutableSet;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
import org.apache.cassandra.io.sstable.Component;
import org.apache.cassandra.io.sstable.CorruptSSTableException;
+import org.apache.cassandra.io.sstable.IndexSummary;
+import org.apache.cassandra.io.sstable.KeyIterator;
import org.apache.cassandra.io.sstable.SSTableIdentityIterator;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.sstable.metadata.MetadataComponent;
@@ -33,19 +39,28 @@
import org.apache.cassandra.io.util.DataIntegrityMetadata.FileDigestValidator;
import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.io.util.RandomAccessReader;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.service.ActiveRepairService;
+import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.utils.BloomFilterSerializer;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.IFilter;
import org.apache.cassandra.utils.OutputHandler;
import org.apache.cassandra.utils.UUIDGen;
+import java.io.BufferedInputStream;
import java.io.Closeable;
+import java.io.DataInputStream;
import java.io.File;
import java.io.IOError;
import java.io.IOException;
import java.nio.ByteBuffer;
+import java.nio.file.Files;
+import java.nio.file.Paths;
import java.util.*;
-import java.util.function.Predicate;
+import java.util.function.Function;
+import java.util.function.LongPredicate;
public class Verifier implements Closeable
{
@@ -59,23 +74,31 @@
private final RandomAccessReader indexFile;
private final VerifyInfo verifyInfo;
private final RowIndexEntry.IndexSerializer rowIndexEntrySerializer;
+ private final Options options;
+ private final boolean isOffline;
+ /**
+ * Given a keyspace, return the set of local and pending token ranges. By default {@link StorageService#getLocalAndPendingRanges(String)}
+ * is expected, but for the standalone verifier case we can't use that, so this is here to allow the CLI to provide
+ * the token ranges.
+ */
+ private final Function<String, ? extends Collection<Range<Token>>> tokenLookup;
private int goodRows;
private final OutputHandler outputHandler;
private FileDigestValidator validator;
- public Verifier(ColumnFamilyStore cfs, SSTableReader sstable, boolean isOffline)
+ public Verifier(ColumnFamilyStore cfs, SSTableReader sstable, boolean isOffline, Options options)
{
- this(cfs, sstable, new OutputHandler.LogOutput(), isOffline);
+ this(cfs, sstable, new OutputHandler.LogOutput(), isOffline, options);
}
- public Verifier(ColumnFamilyStore cfs, SSTableReader sstable, OutputHandler outputHandler, boolean isOffline)
+ public Verifier(ColumnFamilyStore cfs, SSTableReader sstable, OutputHandler outputHandler, boolean isOffline, Options options)
{
this.cfs = cfs;
this.sstable = sstable;
this.outputHandler = outputHandler;
- this.rowIndexEntrySerializer = sstable.descriptor.version.getSSTableFormat().getIndexSerializer(sstable.metadata, sstable.descriptor.version, sstable.header);
+ this.rowIndexEntrySerializer = sstable.descriptor.version.getSSTableFormat().getIndexSerializer(cfs.metadata(), sstable.descriptor.version, sstable.header);
this.controller = new VerifyController(cfs);
@@ -84,13 +107,25 @@
: sstable.openDataReader(CompactionManager.instance.getRateLimiter());
this.indexFile = RandomAccessReader.open(new File(sstable.descriptor.filenameFor(Component.PRIMARY_INDEX)));
this.verifyInfo = new VerifyInfo(dataFile, sstable);
+ this.options = options;
+ this.isOffline = isOffline;
+ this.tokenLookup = options.tokenLookup;
}
- public void verify(boolean extended) throws IOException
+ public void verify()
{
+ boolean extended = options.extendedVerification;
long rowStart = 0;
outputHandler.output(String.format("Verifying %s (%s)", sstable, FBUtilities.prettyPrintMemory(dataFile.length())));
+ if (options.checkVersion && !sstable.descriptor.version.isLatestVersion())
+ {
+ String msg = String.format("%s is not the latest version, run upgradesstables", sstable);
+ outputHandler.output(msg);
+ // don't use markAndThrow here because we don't want a CorruptSSTableException for this.
+ throw new RuntimeException(msg);
+ }
+
outputHandler.output(String.format("Deserializing sstable metadata for %s ", sstable));
try
{
@@ -102,19 +137,77 @@
}
catch (Throwable t)
{
- outputHandler.debug(t.getMessage());
+ outputHandler.warn(t.getMessage());
markAndThrow(false);
}
- outputHandler.output(String.format("Checking computed hash of %s ", sstable));
+ try
+ {
+ outputHandler.debug("Deserializing index for "+sstable);
+ deserializeIndex(sstable);
+ }
+ catch (Throwable t)
+ {
+ outputHandler.warn(t.getMessage());
+ markAndThrow();
+ }
+
+ try
+ {
+ outputHandler.debug("Deserializing index summary for "+sstable);
+ deserializeIndexSummary(sstable);
+ }
+ catch (Throwable t)
+ {
+ outputHandler.output("Index summary is corrupt - if it is removed it will get rebuilt on startup "+sstable.descriptor.filenameFor(Component.SUMMARY));
+ outputHandler.warn(t.getMessage());
+ markAndThrow(false);
+ }
+
+ try
+ {
+ outputHandler.debug("Deserializing bloom filter for "+sstable);
+ deserializeBloomFilter(sstable);
+
+ }
+ catch (Throwable t)
+ {
+ outputHandler.warn(t.getMessage());
+ markAndThrow();
+ }
+
+ if (options.checkOwnsTokens && !isOffline)
+ {
+ outputHandler.debug("Checking that all tokens are owned by the current node");
+ try (KeyIterator iter = new KeyIterator(sstable.descriptor, sstable.metadata()))
+ {
+ List<Range<Token>> ownedRanges = Range.normalize(tokenLookup.apply(cfs.metadata.keyspace));
+ if (ownedRanges.isEmpty())
+ return;
+ RangeOwnHelper rangeOwnHelper = new RangeOwnHelper(ownedRanges);
+ while (iter.hasNext())
+ {
+ DecoratedKey key = iter.next();
+ rangeOwnHelper.validate(key);
+ }
+ }
+ catch (Throwable t)
+ {
+ outputHandler.warn(t.getMessage());
+ markAndThrow();
+ }
+ }
+
+ if (options.quick)
+ return;
// Verify will use the Digest files, which works for both compressed and uncompressed sstables
+ outputHandler.output(String.format("Checking computed hash of %s ", sstable));
try
{
validator = null;
- if (sstable.descriptor.digestComponent != null &&
- new File(sstable.descriptor.filenameFor(sstable.descriptor.digestComponent)).exists())
+ if (new File(sstable.descriptor.filenameFor(Component.DIGEST)).exists())
{
validator = DataIntegrityMetadata.fileDigestValidator(sstable.descriptor);
validator.validate();
@@ -127,7 +220,7 @@
}
catch (IOException e)
{
- outputHandler.debug(e.getMessage());
+ outputHandler.warn(e.getMessage());
markAndThrow();
}
finally
@@ -135,12 +228,11 @@
FileUtils.closeQuietly(validator);
}
- if ( !extended )
+ if (!extended)
return;
outputHandler.output("Extended Verify requested, proceeding to inspect values");
-
try
{
ByteBuffer nextIndexKey = ByteBufferUtil.readWithShortLength(indexFile);
@@ -150,6 +242,8 @@
markAndThrow();
}
+ List<Range<Token>> ownedRanges = isOffline ? Collections.emptyList() : Range.normalize(tokenLookup.apply(cfs.metadata().keyspace));
+ RangeOwnHelper rangeOwnHelper = new RangeOwnHelper(ownedRanges);
DecoratedKey prevKey = null;
while (!dataFile.isEOF())
@@ -172,6 +266,19 @@
// check for null key below
}
+ if (options.checkOwnsTokens && ownedRanges.size() > 0)
+ {
+ try
+ {
+ rangeOwnHelper.validate(key);
+ }
+ catch (Throwable t)
+ {
+ outputHandler.warn(String.format("Key %s in sstable %s not owned by local ranges %s", key, sstable, ownedRanges), t);
+ markAndThrow();
+ }
+ }
+
ByteBuffer currentIndexKey = nextIndexKey;
long nextRowPositionFromIndex = 0;
try
@@ -236,6 +343,106 @@
outputHandler.output("Verify of " + sstable + " succeeded. All " + goodRows + " rows read successfully");
}
+ /**
+ * Use the fact that check(..) is called with sorted tokens - we keep a pointer in to the normalized ranges
+ * and only bump the pointer if the key given is out of range. This is done to avoid calling .contains(..) many
+ * times for each key (with vnodes for example)
+ */
+ @VisibleForTesting
+ public static class RangeOwnHelper
+ {
+ private final List<Range<Token>> normalizedRanges;
+ private int rangeIndex = 0;
+ private DecoratedKey lastKey;
+
+ public RangeOwnHelper(List<Range<Token>> normalizedRanges)
+ {
+ this.normalizedRanges = normalizedRanges;
+ Range.assertNormalized(normalizedRanges);
+ }
+
+ /**
+ * check if the given key is contained in any of the given ranges
+ *
+ * Must be called in sorted order - key should be increasing
+ *
+ * @param key the key
+ * @throws RuntimeException if the key is not contained
+ */
+ public void validate(DecoratedKey key)
+ {
+ if (!check(key))
+ throw new RuntimeException("Key " + key + " is not contained in the given ranges");
+ }
+
+ /**
+ * check if the given key is contained in any of the given ranges
+ *
+ * Must be called in sorted order - key should be increasing
+ *
+ * @param key the key
+ * @return boolean
+ */
+ public boolean check(DecoratedKey key)
+ {
+ assert lastKey == null || key.compareTo(lastKey) > 0;
+ lastKey = key;
+
+ if (normalizedRanges.isEmpty()) // handle tests etc where we don't have any ranges
+ return true;
+
+ if (rangeIndex > normalizedRanges.size() - 1)
+ throw new IllegalStateException("RangeOwnHelper can only be used to find the first out-of-range-token");
+
+ while (!normalizedRanges.get(rangeIndex).contains(key.getToken()))
+ {
+ rangeIndex++;
+ if (rangeIndex > normalizedRanges.size() - 1)
+ return false;
+ }
+
+ return true;
+ }
+ }
+
+ private void deserializeIndex(SSTableReader sstable) throws IOException
+ {
+ try (RandomAccessReader primaryIndex = RandomAccessReader.open(new File(sstable.descriptor.filenameFor(Component.PRIMARY_INDEX))))
+ {
+ long indexSize = primaryIndex.length();
+
+ while ((primaryIndex.getFilePointer()) != indexSize)
+ {
+ ByteBuffer key = ByteBufferUtil.readWithShortLength(primaryIndex);
+ RowIndexEntry.Serializer.skip(primaryIndex, sstable.descriptor.version);
+ }
+ }
+ }
+
+ private void deserializeIndexSummary(SSTableReader sstable) throws IOException
+ {
+ File file = new File(sstable.descriptor.filenameFor(Component.SUMMARY));
+ TableMetadata metadata = cfs.metadata();
+ try (DataInputStream iStream = new DataInputStream(Files.newInputStream(file.toPath())))
+ {
+ try (IndexSummary indexSummary = IndexSummary.serializer.deserialize(iStream,
+ cfs.getPartitioner(),
+ metadata.params.minIndexInterval,
+ metadata.params.maxIndexInterval))
+ {
+ ByteBufferUtil.readWithLength(iStream);
+ ByteBufferUtil.readWithLength(iStream);
+ }
+ }
+ }
+
+ private void deserializeBloomFilter(SSTableReader sstable) throws IOException
+ {
+ try (DataInputStream stream = new DataInputStream(new BufferedInputStream(Files.newInputStream(Paths.get(sstable.descriptor.filenameFor(Component.FILTER)))));
+ IFilter bf = BloomFilterSerializer.deserialize(stream, sstable.descriptor.version.hasOldBfFormat()))
+ {}
+ }
+
public void close()
{
FileUtils.closeQuietly(dataFile);
@@ -248,18 +455,18 @@
throw (Error) th;
}
- private void markAndThrow() throws IOException
+ private void markAndThrow()
{
markAndThrow(true);
}
- private void markAndThrow(boolean mutateRepaired) throws IOException
+ private void markAndThrow(boolean mutateRepaired)
{
- if (mutateRepaired) // if we are able to mutate repaired flag, an incremental repair should be enough
+ if (mutateRepaired && options.mutateRepairStatus) // if we are able to mutate repaired flag, an incremental repair should be enough
{
try
{
- sstable.descriptor.getMetadataSerializer().mutateRepairedAt(sstable.descriptor, ActiveRepairService.UNREPAIRED_SSTABLE);
+ sstable.descriptor.getMetadataSerializer().mutateRepairMetadata(sstable.descriptor, ActiveRepairService.UNREPAIRED_SSTABLE, sstable.getPendingRepair(), sstable.isTransient());
sstable.reloadSSTableMetadata();
cfs.getTracker().notifySSTableRepairedStatusChanged(Collections.singleton(sstable));
}
@@ -268,7 +475,11 @@
outputHandler.output("Error mutating repairedAt for SSTable " + sstable.getFilename() + ", as part of markAndThrow");
}
}
- throw new CorruptSSTableException(new Exception(String.format("Invalid SSTable %s, please force %srepair", sstable.getFilename(), mutateRepaired ? "" : "a full ")), sstable.getFilename());
+ Exception e = new Exception(String.format("Invalid SSTable %s, please force %srepair", sstable.getFilename(), (mutateRepaired && options.mutateRepairStatus) ? "" : "a full "));
+ if (options.invokeDiskFailurePolicy)
+ throw new CorruptSSTableException(e, sstable.getFilename());
+ else
+ throw new RuntimeException(e);
}
public CompactionInfo.Holder getVerifyInfo()
@@ -293,11 +504,12 @@
{
try
{
- return new CompactionInfo(sstable.metadata,
+ return new CompactionInfo(sstable.metadata(),
OperationType.VERIFY,
dataFile.getFilePointer(),
dataFile.length(),
- verificationCompactionId);
+ verificationCompactionId,
+ ImmutableSet.of(sstable));
}
catch (Exception e)
{
@@ -319,9 +531,108 @@
}
@Override
- public Predicate<Long> getPurgeEvaluator(DecoratedKey key)
+ public LongPredicate getPurgeEvaluator(DecoratedKey key)
{
return time -> false;
}
}
+
+ public static Options.Builder options()
+ {
+ return new Options.Builder();
+ }
+
+ public static class Options
+ {
+ public final boolean invokeDiskFailurePolicy;
+ public final boolean extendedVerification;
+ public final boolean checkVersion;
+ public final boolean mutateRepairStatus;
+ public final boolean checkOwnsTokens;
+ public final boolean quick;
+ public final Function<String, ? extends Collection<Range<Token>>> tokenLookup;
+
+ private Options(boolean invokeDiskFailurePolicy, boolean extendedVerification, boolean checkVersion, boolean mutateRepairStatus, boolean checkOwnsTokens, boolean quick, Function<String, ? extends Collection<Range<Token>>> tokenLookup)
+ {
+ this.invokeDiskFailurePolicy = invokeDiskFailurePolicy;
+ this.extendedVerification = extendedVerification;
+ this.checkVersion = checkVersion;
+ this.mutateRepairStatus = mutateRepairStatus;
+ this.checkOwnsTokens = checkOwnsTokens;
+ this.quick = quick;
+ this.tokenLookup = tokenLookup;
+ }
+
+ @Override
+ public String toString()
+ {
+ return "Options{" +
+ "invokeDiskFailurePolicy=" + invokeDiskFailurePolicy +
+ ", extendedVerification=" + extendedVerification +
+ ", checkVersion=" + checkVersion +
+ ", mutateRepairStatus=" + mutateRepairStatus +
+ ", checkOwnsTokens=" + checkOwnsTokens +
+ ", quick=" + quick +
+ '}';
+ }
+
+ public static class Builder
+ {
+ private boolean invokeDiskFailurePolicy = false; // invoking disk failure policy can stop the node if we find a corrupt stable
+ private boolean extendedVerification = false;
+ private boolean checkVersion = false;
+ private boolean mutateRepairStatus = false; // mutating repair status can be dangerous
+ private boolean checkOwnsTokens = false;
+ private boolean quick = false;
+ private Function<String, ? extends Collection<Range<Token>>> tokenLookup = StorageService.instance::getLocalAndPendingRanges;
+
+ public Builder invokeDiskFailurePolicy(boolean param)
+ {
+ this.invokeDiskFailurePolicy = param;
+ return this;
+ }
+
+ public Builder extendedVerification(boolean param)
+ {
+ this.extendedVerification = param;
+ return this;
+ }
+
+ public Builder checkVersion(boolean param)
+ {
+ this.checkVersion = param;
+ return this;
+ }
+
+ public Builder mutateRepairStatus(boolean param)
+ {
+ this.mutateRepairStatus = param;
+ return this;
+ }
+
+ public Builder checkOwnsTokens(boolean param)
+ {
+ this.checkOwnsTokens = param;
+ return this;
+ }
+
+ public Builder quick(boolean param)
+ {
+ this.quick = param;
+ return this;
+ }
+
+ public Builder tokenLookup(Function<String, ? extends Collection<Range<Token>>> tokenLookup)
+ {
+ this.tokenLookup = tokenLookup;
+ return this;
+ }
+
+ public Options build()
+ {
+ return new Options(invokeDiskFailurePolicy, extendedVerification, checkVersion, mutateRepairStatus, checkOwnsTokens, quick, tokenLookup);
+ }
+
+ }
+ }
}
diff --git a/src/java/org/apache/cassandra/db/compaction/writers/CompactionAwareWriter.java b/src/java/org/apache/cassandra/db/compaction/writers/CompactionAwareWriter.java
index bc6115e..c1ae9ec 100644
--- a/src/java/org/apache/cassandra/db/compaction/writers/CompactionAwareWriter.java
+++ b/src/java/org/apache/cassandra/db/compaction/writers/CompactionAwareWriter.java
@@ -22,6 +22,7 @@
import java.util.Collection;
import java.util.List;
import java.util.Set;
+import java.util.UUID;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -56,6 +57,8 @@
protected final long estimatedTotalKeys;
protected final long maxAge;
protected final long minRepairedAt;
+ protected final UUID pendingRepair;
+ protected final boolean isTransient;
protected final SSTableRewriter sstableWriter;
protected final LifecycleTransaction txn;
@@ -89,6 +92,8 @@
maxAge = CompactionTask.getMaxDataAge(nonExpiredSSTables);
sstableWriter = SSTableRewriter.construct(cfs, txn, keepOriginals, maxAge);
minRepairedAt = CompactionTask.getMinRepairedAt(nonExpiredSSTables);
+ pendingRepair = CompactionTask.getPendingRepair(nonExpiredSSTables);
+ isTransient = CompactionTask.getIsTransient(nonExpiredSSTables);
DiskBoundaries db = cfs.getDiskBoundaries();
diskBoundaries = db.positions;
locations = db.directories;
diff --git a/src/java/org/apache/cassandra/db/compaction/writers/DefaultCompactionWriter.java b/src/java/org/apache/cassandra/db/compaction/writers/DefaultCompactionWriter.java
index f8ecd87..6180f96 100644
--- a/src/java/org/apache/cassandra/db/compaction/writers/DefaultCompactionWriter.java
+++ b/src/java/org/apache/cassandra/db/compaction/writers/DefaultCompactionWriter.java
@@ -28,7 +28,6 @@
import org.apache.cassandra.db.SerializationHeader;
import org.apache.cassandra.db.rows.UnfilteredRowIterator;
import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
-import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.sstable.format.SSTableWriter;
import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
@@ -69,12 +68,14 @@
public void switchCompactionLocation(Directories.DataDirectory directory)
{
@SuppressWarnings("resource")
- SSTableWriter writer = SSTableWriter.create(Descriptor.fromFilename(cfs.getSSTablePath(getDirectories().getLocationForDisk(directory))),
+ SSTableWriter writer = SSTableWriter.create(cfs.newSSTableDescriptor(getDirectories().getLocationForDisk(directory)),
estimatedTotalKeys,
minRepairedAt,
+ pendingRepair,
+ isTransient,
cfs.metadata,
- new MetadataCollector(txn.originals(), cfs.metadata.comparator, sstableLevel),
- SerializationHeader.make(cfs.metadata, nonExpiredSSTables),
+ new MetadataCollector(txn.originals(), cfs.metadata().comparator, sstableLevel),
+ SerializationHeader.make(cfs.metadata(), nonExpiredSSTables),
cfs.indexManager.listIndexes(),
txn);
sstableWriter.switchWriter(writer);
diff --git a/src/java/org/apache/cassandra/db/compaction/writers/MajorLeveledCompactionWriter.java b/src/java/org/apache/cassandra/db/compaction/writers/MajorLeveledCompactionWriter.java
index 0beb505..2b93eb4 100644
--- a/src/java/org/apache/cassandra/db/compaction/writers/MajorLeveledCompactionWriter.java
+++ b/src/java/org/apache/cassandra/db/compaction/writers/MajorLeveledCompactionWriter.java
@@ -26,7 +26,6 @@
import org.apache.cassandra.db.rows.UnfilteredRowIterator;
import org.apache.cassandra.db.compaction.LeveledManifest;
import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
-import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.sstable.format.SSTableWriter;
import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
@@ -105,16 +104,17 @@
{
this.sstableDirectory = location;
averageEstimatedKeysPerSSTable = Math.round(((double) averageEstimatedKeysPerSSTable * sstablesWritten + partitionsWritten) / (sstablesWritten + 1));
- sstableWriter.switchWriter(SSTableWriter.create(Descriptor.fromFilename(cfs.getSSTablePath(getDirectories().getLocationForDisk(sstableDirectory))),
+ sstableWriter.switchWriter(SSTableWriter.create(cfs.newSSTableDescriptor(getDirectories().getLocationForDisk(sstableDirectory)),
keysPerSSTable,
minRepairedAt,
+ pendingRepair,
+ isTransient,
cfs.metadata,
- new MetadataCollector(txn.originals(), cfs.metadata.comparator, currentLevel),
- SerializationHeader.make(cfs.metadata, txn.originals()),
+ new MetadataCollector(txn.originals(), cfs.metadata().comparator, currentLevel),
+ SerializationHeader.make(cfs.metadata(), txn.originals()),
cfs.indexManager.listIndexes(),
txn));
partitionsWritten = 0;
sstablesWritten = 0;
-
}
}
diff --git a/src/java/org/apache/cassandra/db/compaction/writers/MaxSSTableSizeWriter.java b/src/java/org/apache/cassandra/db/compaction/writers/MaxSSTableSizeWriter.java
index 864185e..df7eeaf 100644
--- a/src/java/org/apache/cassandra/db/compaction/writers/MaxSSTableSizeWriter.java
+++ b/src/java/org/apache/cassandra/db/compaction/writers/MaxSSTableSizeWriter.java
@@ -26,7 +26,6 @@
import org.apache.cassandra.db.compaction.OperationType;
import org.apache.cassandra.db.rows.UnfilteredRowIterator;
import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
-import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.sstable.format.SSTableWriter;
import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
@@ -108,12 +107,14 @@
{
sstableDirectory = location;
@SuppressWarnings("resource")
- SSTableWriter writer = SSTableWriter.create(Descriptor.fromFilename(cfs.getSSTablePath(getDirectories().getLocationForDisk(sstableDirectory))),
+ SSTableWriter writer = SSTableWriter.create(cfs.newSSTableDescriptor(getDirectories().getLocationForDisk(sstableDirectory)),
estimatedTotalKeys / estimatedSSTables,
minRepairedAt,
+ pendingRepair,
+ isTransient,
cfs.metadata,
- new MetadataCollector(allSSTables, cfs.metadata.comparator, level),
- SerializationHeader.make(cfs.metadata, nonExpiredSSTables),
+ new MetadataCollector(allSSTables, cfs.metadata().comparator, level),
+ SerializationHeader.make(cfs.metadata(), nonExpiredSSTables),
cfs.indexManager.listIndexes(),
txn);
diff --git a/src/java/org/apache/cassandra/db/compaction/writers/SplittingSizeTieredCompactionWriter.java b/src/java/org/apache/cassandra/db/compaction/writers/SplittingSizeTieredCompactionWriter.java
index 46cb891..7533f1d 100644
--- a/src/java/org/apache/cassandra/db/compaction/writers/SplittingSizeTieredCompactionWriter.java
+++ b/src/java/org/apache/cassandra/db/compaction/writers/SplittingSizeTieredCompactionWriter.java
@@ -29,7 +29,6 @@
import org.apache.cassandra.db.SerializationHeader;
import org.apache.cassandra.db.rows.UnfilteredRowIterator;
import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
-import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.sstable.format.SSTableWriter;
import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
@@ -104,12 +103,14 @@
this.location = location;
long currentPartitionsToWrite = Math.round(ratios[currentRatioIndex] * estimatedTotalKeys);
@SuppressWarnings("resource")
- SSTableWriter writer = SSTableWriter.create(Descriptor.fromFilename(cfs.getSSTablePath(getDirectories().getLocationForDisk(location))),
+ SSTableWriter writer = SSTableWriter.create(cfs.newSSTableDescriptor(getDirectories().getLocationForDisk(location)),
currentPartitionsToWrite,
minRepairedAt,
+ pendingRepair,
+ isTransient,
cfs.metadata,
- new MetadataCollector(allSSTables, cfs.metadata.comparator, 0),
- SerializationHeader.make(cfs.metadata, nonExpiredSSTables),
+ new MetadataCollector(allSSTables, cfs.metadata().comparator, 0),
+ SerializationHeader.make(cfs.metadata(), nonExpiredSSTables),
cfs.indexManager.listIndexes(),
txn);
logger.trace("Switching writer, currentPartitionsToWrite = {}", currentPartitionsToWrite);
diff --git a/src/java/org/apache/cassandra/db/context/CounterContext.java b/src/java/org/apache/cassandra/db/context/CounterContext.java
index d0952d0..6a618ca 100644
--- a/src/java/org/apache/cassandra/db/context/CounterContext.java
+++ b/src/java/org/apache/cassandra/db/context/CounterContext.java
@@ -18,7 +18,6 @@
package org.apache.cassandra.db.context;
import java.nio.ByteBuffer;
-import java.security.MessageDigest;
import java.util.ArrayList;
import java.util.List;
@@ -137,14 +136,6 @@
}
/**
- * Returns the count associated with the update counter id, or 0 if no such shard is present.
- */
- public long getUpdateCount(ByteBuffer context)
- {
- return getClockAndCountOf(context, UPDATE_CLOCK_ID).count;
- }
-
- /**
* Creates a counter context with a single global, 2.1+ shard (a result of increment).
*/
public ByteBuffer createGlobal(CounterId id, long clock, long count)
@@ -176,7 +167,7 @@
return state.context;
}
- private static int headerLength(ByteBuffer context)
+ public static int headerLength(ByteBuffer context)
{
return HEADER_SIZE_LENGTH + Math.abs(context.getShort(context.position())) * HEADER_ELT_LENGTH;
}
@@ -637,7 +628,7 @@
ByteBuffer marked = ByteBuffer.allocate(context.remaining());
marked.putShort(marked.position(), (short) (count * -1));
- ByteBufferUtil.arrayCopy(context,
+ ByteBufferUtil.copyBytes(context,
context.position() + HEADER_SIZE_LENGTH,
marked,
marked.position() + HEADER_SIZE_LENGTH,
@@ -676,7 +667,7 @@
cleared.putShort(cleared.position() + HEADER_SIZE_LENGTH + i * HEADER_ELT_LENGTH, globalShardIndexes.get(i));
int origHeaderLength = headerLength(context);
- ByteBufferUtil.arrayCopy(context,
+ ByteBufferUtil.copyBytes(context,
context.position() + origHeaderLength,
cleared,
cleared.position() + headerLength(cleared),
@@ -692,23 +683,6 @@
}
/**
- * Update a MessageDigest with the content of a context.
- * Note that this skips the header entirely since the header information
- * has local meaning only, while digests are meant for comparison across
- * nodes. This means in particular that we always have:
- * updateDigest(ctx) == updateDigest(clearAllLocal(ctx))
- */
- public void updateDigest(MessageDigest message, ByteBuffer context)
- {
- // context can be empty due to the optimization from CASSANDRA-10657
- if (!context.hasRemaining())
- return;
- ByteBuffer dup = context.duplicate();
- dup.position(context.position() + headerLength(context));
- message.update(dup);
- }
-
- /**
* Returns the clock and the count associated with the local counter id, or (0, 0) if no such shard is present.
*/
public ClockAndCount getLocalClockAndCount(ByteBuffer context)
@@ -717,6 +691,14 @@
}
/**
+ * Returns the count associated with the local counter id, or 0 if no such shard is present.
+ */
+ public long getLocalCount(ByteBuffer context)
+ {
+ return getLocalClockAndCount(context).count;
+ }
+
+ /**
* Returns the clock and the count associated with the given counter id, or (0, 0) if no such shard is present.
*/
@VisibleForTesting
diff --git a/src/java/org/apache/cassandra/db/filter/AbstractClusteringIndexFilter.java b/src/java/org/apache/cassandra/db/filter/AbstractClusteringIndexFilter.java
index 51e9d8e..c28117c 100644
--- a/src/java/org/apache/cassandra/db/filter/AbstractClusteringIndexFilter.java
+++ b/src/java/org/apache/cassandra/db/filter/AbstractClusteringIndexFilter.java
@@ -19,8 +19,8 @@
import java.io.IOException;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.marshal.ReversedType;
import org.apache.cassandra.io.util.DataInputPlus;
@@ -45,13 +45,13 @@
protected abstract void serializeInternal(DataOutputPlus out, int version) throws IOException;
protected abstract long serializedSizeInternal(int version);
- protected void appendOrderByToCQLString(CFMetaData metadata, StringBuilder sb)
+ protected void appendOrderByToCQLString(TableMetadata metadata, StringBuilder sb)
{
if (reversed)
{
sb.append(" ORDER BY (");
int i = 0;
- for (ColumnDefinition column : metadata.clusteringColumns())
+ for (ColumnMetadata column : metadata.clusteringColumns())
sb.append(i++ == 0 ? "" : ", ").append(column.name).append(column.type instanceof ReversedType ? " ASC" : " DESC");
sb.append(')');
}
@@ -69,7 +69,7 @@
filter.serializeInternal(out, version);
}
- public ClusteringIndexFilter deserialize(DataInputPlus in, int version, CFMetaData metadata) throws IOException
+ public ClusteringIndexFilter deserialize(DataInputPlus in, int version, TableMetadata metadata) throws IOException
{
Kind kind = Kind.values()[in.readUnsignedByte()];
boolean reversed = in.readBoolean();
diff --git a/src/java/org/apache/cassandra/db/filter/ClusteringIndexFilter.java b/src/java/org/apache/cassandra/db/filter/ClusteringIndexFilter.java
index f184035..cdb61c9 100644
--- a/src/java/org/apache/cassandra/db/filter/ClusteringIndexFilter.java
+++ b/src/java/org/apache/cassandra/db/filter/ClusteringIndexFilter.java
@@ -19,14 +19,14 @@
import java.io.IOException;
-import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.db.*;
-import org.apache.cassandra.db.rows.*;
import org.apache.cassandra.db.partitions.CachedPartition;
import org.apache.cassandra.db.partitions.Partition;
+import org.apache.cassandra.db.rows.*;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.schema.TableMetadata;
/**
* A filter that selects a subset of the rows of a given partition by using the "clustering index".
@@ -54,7 +54,7 @@
static interface InternalDeserializer
{
- public ClusteringIndexFilter deserialize(DataInputPlus in, int version, CFMetaData metadata, boolean reversed) throws IOException;
+ public ClusteringIndexFilter deserialize(DataInputPlus in, int version, TableMetadata metadata, boolean reversed) throws IOException;
}
/**
@@ -114,10 +114,10 @@
/**
* Returns an iterator that only returns the rows of the provided iterator that this filter selects.
* <p>
- * This method is the "dumb" counterpart to {@link #getSlices(CFMetaData)} in that it has no way to quickly get
+ * This method is the "dumb" counterpart to {@link #getSlices(TableMetadata)} in that it has no way to quickly get
* to what is actually selected, so it simply iterate over it all and filters out what shouldn't be returned. This should
* be avoided in general.
- * Another difference with {@link #getSlices(CFMetaData)} is that this method also filter the queried
+ * Another difference with {@link #getSlices(TableMetadata)} is that this method also filter the queried
* columns in the returned result, while the former assumes that the provided iterator has already done it.
*
* @param columnFilter the columns to include in the rows of the result iterator.
@@ -127,7 +127,7 @@
*/
public UnfilteredRowIterator filterNotIndexed(ColumnFilter columnFilter, UnfilteredRowIterator iterator);
- public Slices getSlices(CFMetaData metadata);
+ public Slices getSlices(TableMetadata metadata);
/**
* Given a partition, returns a row iterator for the rows of this partition that are selected by this filter.
@@ -150,13 +150,13 @@
public Kind kind();
- public String toString(CFMetaData metadata);
- public String toCQLString(CFMetaData metadata);
+ public String toString(TableMetadata metadata);
+ public String toCQLString(TableMetadata metadata);
public interface Serializer
{
public void serialize(ClusteringIndexFilter filter, DataOutputPlus out, int version) throws IOException;
- public ClusteringIndexFilter deserialize(DataInputPlus in, int version, CFMetaData metadata) throws IOException;
+ public ClusteringIndexFilter deserialize(DataInputPlus in, int version, TableMetadata metadata) throws IOException;
public long serializedSize(ClusteringIndexFilter filter, int version);
}
}
diff --git a/src/java/org/apache/cassandra/db/filter/ClusteringIndexNamesFilter.java b/src/java/org/apache/cassandra/db/filter/ClusteringIndexNamesFilter.java
index 6c7e14b..63815a1 100644
--- a/src/java/org/apache/cassandra/db/filter/ClusteringIndexNamesFilter.java
+++ b/src/java/org/apache/cassandra/db/filter/ClusteringIndexNamesFilter.java
@@ -21,15 +21,15 @@
import java.nio.ByteBuffer;
import java.util.*;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
import org.apache.cassandra.db.*;
-import org.apache.cassandra.db.rows.*;
import org.apache.cassandra.db.partitions.*;
+import org.apache.cassandra.db.rows.*;
import org.apache.cassandra.db.transform.Transformation;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.utils.SearchIterator;
import org.apache.cassandra.utils.btree.BTreeSet;
@@ -128,7 +128,7 @@
return Transformation.apply(iterator, new FilterNotIndexed());
}
- public Slices getSlices(CFMetaData metadata)
+ public Slices getSlices(TableMetadata metadata)
{
Slices.Builder builder = new Slices.Builder(metadata.comparator, clusteringsInQueryOrder.size());
for (Clustering clustering : clusteringsInQueryOrder)
@@ -142,12 +142,12 @@
final SearchIterator<Clustering, Row> searcher = partition.searchIterator(columnFilter, reversed);
return new AbstractUnfilteredRowIterator(partition.metadata(),
- partition.partitionKey(),
- partition.partitionLevelDeletion(),
- columnFilter.fetchedColumns(),
- searcher.next(Clustering.STATIC_CLUSTERING),
- reversed,
- partition.stats())
+ partition.partitionKey(),
+ partition.partitionLevelDeletion(),
+ columnFilter.fetchedColumns(),
+ searcher.next(Clustering.STATIC_CLUSTERING),
+ reversed,
+ partition.stats())
{
protected Unfiltered computeNext()
{
@@ -164,7 +164,7 @@
public boolean shouldInclude(SSTableReader sstable)
{
- ClusteringComparator comparator = sstable.metadata.comparator;
+ ClusteringComparator comparator = sstable.metadata().comparator;
List<ByteBuffer> minClusteringValues = sstable.getSSTableMetadata().minClusteringValues;
List<ByteBuffer> maxClusteringValues = sstable.getSSTableMetadata().maxClusteringValues;
@@ -177,7 +177,7 @@
return false;
}
- public String toString(CFMetaData metadata)
+ public String toString(TableMetadata metadata)
{
StringBuilder sb = new StringBuilder();
sb.append("names(");
@@ -189,13 +189,13 @@
return sb.append(')').toString();
}
- public String toCQLString(CFMetaData metadata)
+ public String toCQLString(TableMetadata metadata)
{
if (metadata.clusteringColumns().isEmpty() || clusterings.isEmpty())
return "";
StringBuilder sb = new StringBuilder();
- sb.append('(').append(ColumnDefinition.toCQLString(metadata.clusteringColumns())).append(')');
+ sb.append('(').append(ColumnMetadata.toCQLString(metadata.clusteringColumns())).append(')');
sb.append(clusterings.size() == 1 ? " = " : " IN (");
int i = 0;
for (Clustering clustering : clusterings)
@@ -206,6 +206,20 @@
return sb.toString();
}
+ public boolean equals(Object o)
+ {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ ClusteringIndexNamesFilter that = (ClusteringIndexNamesFilter) o;
+ return Objects.equals(clusterings, that.clusterings) &&
+ Objects.equals(reversed, that.reversed);
+ }
+
+ public int hashCode()
+ {
+ return Objects.hash(clusterings, reversed);
+ }
+
public Kind kind()
{
return Kind.NAMES;
@@ -230,7 +244,7 @@
private static class NamesDeserializer implements InternalDeserializer
{
- public ClusteringIndexFilter deserialize(DataInputPlus in, int version, CFMetaData metadata, boolean reversed) throws IOException
+ public ClusteringIndexFilter deserialize(DataInputPlus in, int version, TableMetadata metadata, boolean reversed) throws IOException
{
ClusteringComparator comparator = metadata.comparator;
BTreeSet.Builder<Clustering> clusterings = BTreeSet.builder(comparator);
diff --git a/src/java/org/apache/cassandra/db/filter/ClusteringIndexSliceFilter.java b/src/java/org/apache/cassandra/db/filter/ClusteringIndexSliceFilter.java
index 02a44d7..9490adf 100644
--- a/src/java/org/apache/cassandra/db/filter/ClusteringIndexSliceFilter.java
+++ b/src/java/org/apache/cassandra/db/filter/ClusteringIndexSliceFilter.java
@@ -21,7 +21,7 @@
import java.util.List;
import java.nio.ByteBuffer;
-import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.rows.*;
import org.apache.cassandra.db.partitions.CachedPartition;
@@ -109,7 +109,7 @@
return Transformation.apply(iterator, new FilterNotIndexed());
}
- public Slices getSlices(CFMetaData metadata)
+ public Slices getSlices(TableMetadata metadata)
{
return slices;
}
@@ -130,12 +130,12 @@
return slices.intersects(minClusteringValues, maxClusteringValues);
}
- public String toString(CFMetaData metadata)
+ public String toString(TableMetadata metadata)
{
return String.format("slice(slices=%s, reversed=%b)", slices, reversed);
}
- public String toCQLString(CFMetaData metadata)
+ public String toCQLString(TableMetadata metadata)
{
StringBuilder sb = new StringBuilder();
@@ -164,7 +164,7 @@
private static class SliceDeserializer implements InternalDeserializer
{
- public ClusteringIndexFilter deserialize(DataInputPlus in, int version, CFMetaData metadata, boolean reversed) throws IOException
+ public ClusteringIndexFilter deserialize(DataInputPlus in, int version, TableMetadata metadata, boolean reversed) throws IOException
{
Slices slices = Slices.serializer.deserialize(in, version, metadata);
return new ClusteringIndexSliceFilter(slices, reversed);
diff --git a/src/java/org/apache/cassandra/db/filter/ColumnFilter.java b/src/java/org/apache/cassandra/db/filter/ColumnFilter.java
index 37da86a..20a1656 100644
--- a/src/java/org/apache/cassandra/db/filter/ColumnFilter.java
+++ b/src/java/org/apache/cassandra/db/filter/ColumnFilter.java
@@ -20,17 +20,21 @@
import java.io.IOException;
import java.util.*;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Iterators;
import com.google.common.collect.SortedSetMultimap;
import com.google.common.collect.TreeMultimap;
-import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.cql3.ColumnIdentifier;
import org.apache.cassandra.db.*;
+import org.apache.cassandra.db.rows.Cell;
import org.apache.cassandra.db.rows.CellPath;
-import org.apache.cassandra.config.ColumnDefinition;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableMetadata;
/**
* Represents which (non-PK) columns (and optionally which sub-part of a column for complex columns) are selected
@@ -41,14 +45,14 @@
* in its request.
*
* The reason for distinguishing those 2 sets is that due to the CQL semantic (see #6588 for more details), we
- * often need to internally fetch all columns for the queried table, but can still do some optimizations for those
- * columns that are not directly queried by the user (see #10657 for more details).
+ * often need to internally fetch all regular columns for the queried table, but can still do some optimizations for
+ * those columns that are not directly queried by the user (see #10657 for more details).
*
* Note that in practice:
* - the _queried_ columns set is always included in the _fetched_ one.
- * - whenever those sets are different, we know the _fetched_ set contains all columns for the table, so we
- * don't have to record this set, we just keep a pointer to the table metadata. The only set we concretely
- * store is thus the _queried_ one.
+ * - whenever those sets are different, we know 1) the _fetched_ set contains all regular columns for the table and 2)
+ * _fetched_ == _queried_ for static columns, so we don't have to record this set, we just keep a pointer to the
+ * table metadata. The only set we concretely store is thus the _queried_ one.
* - in the special case of a {@code SELECT *} query, we want to query all columns, and _fetched_ == _queried.
* As this is a common case, we special case it by keeping the _queried_ set {@code null} (and we retrieve
* the columns through the metadata pointer).
@@ -61,25 +65,57 @@
*/
public class ColumnFilter
{
+ public static final ColumnFilter NONE = selection(RegularAndStaticColumns.NONE);
+
public static final Serializer serializer = new Serializer();
- // True if _fetched_ is all the columns, in which case metadata must not be null. If false,
- // then _fetched_ == _queried_ and we only store _queried_.
- private final boolean isFetchAll;
+ // True if _fetched_ includes all regular columns (and any static in _queried_), in which case metadata must not be
+ // null. If false, then _fetched_ == _queried_ and we only store _queried_.
+ final boolean fetchAllRegulars;
- private final PartitionColumns fetched;
- private final PartitionColumns queried; // can be null if isFetchAll and _fetched_ == _queried_
+ final RegularAndStaticColumns fetched;
+ final RegularAndStaticColumns queried; // can be null if fetchAllRegulars, to represent a wildcard query (all
+ // static and regular columns are both _fetched_ and _queried_).
private final SortedSetMultimap<ColumnIdentifier, ColumnSubselection> subSelections; // can be null
- private ColumnFilter(boolean isFetchAll,
- PartitionColumns fetched,
- PartitionColumns queried,
+ private ColumnFilter(boolean fetchAllRegulars,
+ TableMetadata metadata,
+ RegularAndStaticColumns queried,
SortedSetMultimap<ColumnIdentifier, ColumnSubselection> subSelections)
{
- assert !isFetchAll || fetched != null;
- assert isFetchAll || queried != null;
- this.isFetchAll = isFetchAll;
- this.fetched = isFetchAll ? fetched : queried;
+ assert !fetchAllRegulars || metadata != null;
+ assert fetchAllRegulars || queried != null;
+ this.fetchAllRegulars = fetchAllRegulars;
+
+ if (fetchAllRegulars)
+ {
+ RegularAndStaticColumns all = metadata.regularAndStaticColumns();
+
+ this.fetched = (all.statics.isEmpty() || queried == null)
+ ? all
+ : new RegularAndStaticColumns(queried.statics, all.regulars);
+ }
+ else
+ {
+ this.fetched = queried;
+ }
+
+ this.queried = queried;
+ this.subSelections = subSelections;
+ }
+
+ /**
+ * Used on replica for deserialisation
+ */
+ private ColumnFilter(boolean fetchAllRegulars,
+ RegularAndStaticColumns fetched,
+ RegularAndStaticColumns queried,
+ SortedSetMultimap<ColumnIdentifier, ColumnSubselection> subSelections)
+ {
+ assert !fetchAllRegulars || fetched != null;
+ assert fetchAllRegulars || queried != null;
+ this.fetchAllRegulars = fetchAllRegulars;
+ this.fetched = fetchAllRegulars ? fetched : queried;
this.queried = queried;
this.subSelections = subSelections;
}
@@ -87,9 +123,9 @@
/**
* A filter that includes all columns for the provided table.
*/
- public static ColumnFilter all(CFMetaData metadata)
+ public static ColumnFilter all(TableMetadata metadata)
{
- return new ColumnFilter(true, metadata.partitionColumns(), null, null);
+ return new ColumnFilter(true, metadata, null, null);
}
/**
@@ -99,18 +135,18 @@
* preserve CQL semantic (see class javadoc). This is ok for some internal queries however (and
* for #6588 if/when we implement it).
*/
- public static ColumnFilter selection(PartitionColumns columns)
+ public static ColumnFilter selection(RegularAndStaticColumns columns)
{
- return new ColumnFilter(false, null, columns, null);
+ return new ColumnFilter(false, (TableMetadata) null, columns, null);
}
/**
* A filter that fetches all columns for the provided table, but returns
* only the queried ones.
*/
- public static ColumnFilter selection(CFMetaData metadata, PartitionColumns queried)
+ public static ColumnFilter selection(TableMetadata metadata, RegularAndStaticColumns queried)
{
- return new ColumnFilter(true, metadata.partitionColumns(), queried, null);
+ return new ColumnFilter(true, metadata, queried, null);
}
/**
@@ -118,7 +154,7 @@
*
* @return the columns to fetch for this filter.
*/
- public PartitionColumns fetchedColumns()
+ public RegularAndStaticColumns fetchedColumns()
{
return fetched;
}
@@ -128,14 +164,27 @@
* <p>
* Note that this is in general not all the columns that are fetched internally (see {@link #fetchedColumns}).
*/
- public PartitionColumns queriedColumns()
+ public RegularAndStaticColumns queriedColumns()
{
return queried == null ? fetched : queried;
}
- public boolean fetchesAllColumns()
+ /**
+ * Wether all the (regular or static) columns are fetched by this filter.
+ * <p>
+ * Note that this method is meant as an optimization but a negative return
+ * shouldn't be relied upon strongly: this can return {@code false} but
+ * still have all the columns fetches if those were manually selected by the
+ * user. The goal here is to cheaply avoid filtering things on wildcard
+ * queries, as those are common.
+ *
+ * @param isStatic whether to check for static columns or not. If {@code true},
+ * the method returns if all static columns are fetched, otherwise it checks
+ * regular columns.
+ */
+ public boolean fetchesAllColumns(boolean isStatic)
{
- return isFetchAll;
+ return isStatic ? queried == null : fetchAllRegulars;
}
/**
@@ -144,15 +193,20 @@
*/
public boolean allFetchedColumnsAreQueried()
{
- return !isFetchAll || (queried == null && subSelections == null);
+ return !fetchAllRegulars || queried == null;
}
/**
* Whether the provided column is fetched by this filter.
*/
- public boolean fetches(ColumnDefinition column)
+ public boolean fetches(ColumnMetadata column)
{
- return isFetchAll || queried.contains(column);
+ // For statics, it is included only if it's part of _queried_, or if _queried_ is null (wildcard query).
+ if (column.isStatic())
+ return queried == null || queried.contains(column);
+
+ // For regulars, if 'fetchAllRegulars', then it's included automatically. Otherwise, it depends on _queried_.
+ return fetchAllRegulars || queried.contains(column);
}
/**
@@ -163,9 +217,9 @@
* columns that this class made before using this method. If unsure, you probably want
* to use the {@link #fetches} method.
*/
- public boolean fetchedColumnIsQueried(ColumnDefinition column)
+ public boolean fetchedColumnIsQueried(ColumnMetadata column)
{
- return !isFetchAll || queried == null || queried.contains(column);
+ return !fetchAllRegulars || queried == null || queried.contains(column);
}
/**
@@ -176,10 +230,10 @@
* columns that this class made before using this method. If unsure, you probably want
* to use the {@link #fetches} method.
*/
- public boolean fetchedCellIsQueried(ColumnDefinition column, CellPath path)
+ public boolean fetchedCellIsQueried(ColumnMetadata column, CellPath path)
{
assert path != null;
- if (!isFetchAll || subSelections == null)
+ if (!fetchAllRegulars || subSelections == null)
return true;
SortedSet<ColumnSubselection> s = subSelections.get(column.name);
@@ -198,10 +252,11 @@
* Creates a new {@code Tester} to efficiently test the inclusion of cells of complex column
* {@code column}.
*
+ * @param column for complex column for which to create a tester.
* @return the created tester or {@code null} if all the cells from the provided column
* are queried.
*/
- public Tester newTester(ColumnDefinition column)
+ public Tester newTester(ColumnMetadata column)
{
if (subSelections == null || !column.isComplex())
return null;
@@ -210,14 +265,31 @@
if (s.isEmpty())
return null;
- return new Tester(isFetchAll, s.iterator());
+ return new Tester(!column.isStatic() && fetchAllRegulars, s.iterator());
}
/**
- * Returns a {@code ColumnFilter}} builder that fetches all columns (and queries the columns
+ * Given an iterator on the cell of a complex column, returns an iterator that only include the cells selected by
+ * this filter.
+ *
+ * @param column the (complex) column for which the cells are.
+ * @param cells the cells to filter.
+ * @return a filtered iterator that only include the cells from {@code cells} that are included by this filter.
+ */
+ public Iterator<Cell> filterComplexCells(ColumnMetadata column, Iterator<Cell> cells)
+ {
+ Tester tester = newTester(column);
+ if (tester == null)
+ return cells;
+
+ return Iterators.filter(cells, cell -> tester.fetchedCellIsQueried(cell.path()));
+ }
+
+ /**
+ * Returns a {@code ColumnFilter}} builder that fetches all regular columns (and queries the columns
* added to the builder, or everything if no column is added).
*/
- public static Builder allColumnsBuilder(CFMetaData metadata)
+ public static Builder allRegularColumnsBuilder(TableMetadata metadata)
{
return new Builder(metadata);
}
@@ -232,19 +304,19 @@
public static class Tester
{
- private final boolean isFetchAll;
+ private final boolean isFetched;
private ColumnSubselection current;
private final Iterator<ColumnSubselection> iterator;
- private Tester(boolean isFetchAll, Iterator<ColumnSubselection> iterator)
+ private Tester(boolean isFetched, Iterator<ColumnSubselection> iterator)
{
- this.isFetchAll = isFetchAll;
+ this.isFetched = isFetched;
this.iterator = iterator;
}
public boolean fetches(CellPath path)
{
- return isFetchAll || hasSubselection(path);
+ return isFetched || hasSubselection(path);
}
/**
@@ -252,7 +324,7 @@
*/
public boolean fetchedCellIsQueried(CellPath path)
{
- return !isFetchAll || hasSubselection(path);
+ return !isFetched || hasSubselection(path);
}
private boolean hasSubselection(CellPath path)
@@ -284,51 +356,74 @@
*
* Note that for a allColumnsBuilder, if no queried columns are added, this is interpreted as querying
* all columns, not querying none (but if you know you want to query all columns, prefer
- * {@link ColumnFilter#all(CFMetaData)}. For selectionBuilder, adding no queried columns means no column will be
+ * {@link ColumnFilter#all(TableMetadata)}. For selectionBuilder, adding no queried columns means no column will be
* fetched (so the builder will return {@code PartitionColumns.NONE}).
+ *
+ * Also, if only a subselection of a complex column should be queried, then only the corresponding
+ * subselection method of the builder ({@link #slice} or {@link #select}) should be called for the
+ * column, but {@link #add} shouldn't. if {@link #add} is also called, the whole column will be
+ * queried and the subselection(s) will be ignored. This is done for correctness of CQL where
+ * if you do "SELECT m, m[2..5]", you are really querying the whole collection.
*/
public static class Builder
{
- private final CFMetaData metadata; // null if we don't fetch all columns
- private PartitionColumns.Builder queriedBuilder;
+ private final TableMetadata metadata; // null if we don't fetch all columns
+ private RegularAndStaticColumns.Builder queriedBuilder;
private List<ColumnSubselection> subSelections;
- private Builder(CFMetaData metadata)
+ private Set<ColumnMetadata> fullySelectedComplexColumns;
+
+ private Builder(TableMetadata metadata)
{
this.metadata = metadata;
}
- public Builder add(ColumnDefinition c)
+ public Builder add(ColumnMetadata c)
{
- if (queriedBuilder == null)
- queriedBuilder = PartitionColumns.builder();
- queriedBuilder.add(c);
+ if (c.isComplex() && c.type.isMultiCell())
+ {
+ if (fullySelectedComplexColumns == null)
+ fullySelectedComplexColumns = new HashSet<>();
+ fullySelectedComplexColumns.add(c);
+ }
+ return addInternal(c);
+ }
+
+ public Builder addAll(Iterable<ColumnMetadata> columns)
+ {
+ for (ColumnMetadata column : columns)
+ add(column);
return this;
}
- public Builder addAll(Iterable<ColumnDefinition> columns)
+ private Builder addInternal(ColumnMetadata c)
{
+ if (c.isPrimaryKeyColumn())
+ return this;
+
if (queriedBuilder == null)
- queriedBuilder = PartitionColumns.builder();
- queriedBuilder.addAll(columns);
+ queriedBuilder = RegularAndStaticColumns.builder();
+ queriedBuilder.add(c);
return this;
}
private Builder addSubSelection(ColumnSubselection subSelection)
{
- add(subSelection.column());
+ ColumnMetadata column = subSelection.column();
+ assert column.isComplex() && column.type.isMultiCell();
+ addInternal(column);
if (subSelections == null)
subSelections = new ArrayList<>();
subSelections.add(subSelection);
return this;
}
- public Builder slice(ColumnDefinition c, CellPath from, CellPath to)
+ public Builder slice(ColumnMetadata c, CellPath from, CellPath to)
{
return addSubSelection(ColumnSubselection.slice(c, from, to));
}
- public Builder select(ColumnDefinition c, CellPath elt)
+ public Builder select(ColumnMetadata c, CellPath elt)
{
return addSubSelection(ColumnSubselection.element(c, elt));
}
@@ -337,21 +432,24 @@
{
boolean isFetchAll = metadata != null;
- PartitionColumns queried = queriedBuilder == null ? null : queriedBuilder.build();
+ RegularAndStaticColumns queried = queriedBuilder == null ? null : queriedBuilder.build();
// It's only ok to have queried == null in ColumnFilter if isFetchAll. So deal with the case of a selectionBuilder
// with nothing selected (we can at least happen on some backward compatible queries - CASSANDRA-10471).
if (!isFetchAll && queried == null)
- queried = PartitionColumns.NONE;
+ queried = RegularAndStaticColumns.NONE;
SortedSetMultimap<ColumnIdentifier, ColumnSubselection> s = null;
if (subSelections != null)
{
s = TreeMultimap.create(Comparator.<ColumnIdentifier>naturalOrder(), Comparator.<ColumnSubselection>naturalOrder());
for (ColumnSubselection subSelection : subSelections)
- s.put(subSelection.column().name, subSelection);
+ {
+ if (fullySelectedComplexColumns == null || !fullySelectedComplexColumns.contains(subSelection.column()))
+ s.put(subSelection.column().name, subSelection);
+ }
}
- return new ColumnFilter(isFetchAll, isFetchAll ? metadata.partitionColumns() : null, queried, s);
+ return new ColumnFilter(isFetchAll, metadata, queried, s);
}
}
@@ -366,7 +464,7 @@
ColumnFilter otherCf = (ColumnFilter) other;
- return otherCf.isFetchAll == this.isFetchAll &&
+ return otherCf.fetchAllRegulars == this.fetchAllRegulars &&
Objects.equals(otherCf.fetched, this.fetched) &&
Objects.equals(otherCf.queried, this.queried) &&
Objects.equals(otherCf.subSelections, this.subSelections);
@@ -375,13 +473,13 @@
@Override
public String toString()
{
- if (isFetchAll)
+ if (fetchAllRegulars && queried == null)
return "*";
if (queried.isEmpty())
return "";
- Iterator<ColumnDefinition> defs = queried.selectOrderIterator();
+ Iterator<ColumnMetadata> defs = queried.selectOrderIterator();
if (!defs.hasNext())
return "<none>";
@@ -395,7 +493,7 @@
return sb.toString();
}
- private void appendColumnDef(StringBuilder sb, ColumnDefinition column)
+ private void appendColumnDef(StringBuilder sb, ColumnMetadata column)
{
if (subSelections == null)
{
@@ -417,22 +515,45 @@
public static class Serializer
{
- private static final int IS_FETCH_ALL_MASK = 0x01;
- private static final int HAS_QUERIED_MASK = 0x02;
+ private static final int FETCH_ALL_MASK = 0x01;
+ private static final int HAS_QUERIED_MASK = 0x02;
private static final int HAS_SUB_SELECTIONS_MASK = 0x04;
private static int makeHeaderByte(ColumnFilter selection)
{
- return (selection.isFetchAll ? IS_FETCH_ALL_MASK : 0)
+ return (selection.fetchAllRegulars ? FETCH_ALL_MASK : 0)
| (selection.queried != null ? HAS_QUERIED_MASK : 0)
| (selection.subSelections != null ? HAS_SUB_SELECTIONS_MASK : 0);
}
+ @VisibleForTesting
+ public static ColumnFilter maybeUpdateForBackwardCompatility(ColumnFilter selection, int version)
+ {
+ if (version > MessagingService.VERSION_3014 || !selection.fetchAllRegulars || selection.queried == null)
+ return selection;
+
+ // The meaning of fetchAllRegulars changed (at least when queried != null) due to CASSANDRA-12768: in
+ // pre-4.0 it means that *all* columns are fetched, not just the regular ones, and so 3.0/3.X nodes
+ // would send us more than we'd like. So instead recreating a filter that correspond to what we
+ // actually want (it's a tiny bit less efficient as we include all columns manually and will mark as
+ // queried some columns that are actually only fetched, but it's fine during upgrade).
+ // More concretely, we replace our filter by a non-fetch-all one that queries every columns that our
+ // current filter fetches.
+ Set<ColumnMetadata> queriedStatic = new HashSet<>();
+ Iterables.addAll(queriedStatic, Iterables.filter(selection.queried, ColumnMetadata::isStatic));
+ return new ColumnFilter(false,
+ (TableMetadata) null,
+ new RegularAndStaticColumns(Columns.from(queriedStatic), selection.fetched.regulars),
+ selection.subSelections);
+ }
+
public void serialize(ColumnFilter selection, DataOutputPlus out, int version) throws IOException
{
+ selection = maybeUpdateForBackwardCompatility(selection, version);
+
out.writeByte(makeHeaderByte(selection));
- if (version >= MessagingService.VERSION_3014 && selection.isFetchAll)
+ if (version >= MessagingService.VERSION_3014 && selection.fetchAllRegulars)
{
Columns.serializer.serialize(selection.fetched.statics, out);
Columns.serializer.serialize(selection.fetched.regulars, out);
@@ -452,15 +573,15 @@
}
}
- public ColumnFilter deserialize(DataInputPlus in, int version, CFMetaData metadata) throws IOException
+ public ColumnFilter deserialize(DataInputPlus in, int version, TableMetadata metadata) throws IOException
{
int header = in.readUnsignedByte();
- boolean isFetchAll = (header & IS_FETCH_ALL_MASK) != 0;
+ boolean isFetchAll = (header & FETCH_ALL_MASK) != 0;
boolean hasQueried = (header & HAS_QUERIED_MASK) != 0;
boolean hasSubSelections = (header & HAS_SUB_SELECTIONS_MASK) != 0;
- PartitionColumns fetched = null;
- PartitionColumns queried = null;
+ RegularAndStaticColumns fetched = null;
+ RegularAndStaticColumns queried = null;
if (isFetchAll)
{
@@ -468,11 +589,11 @@
{
Columns statics = Columns.serializer.deserialize(in, metadata);
Columns regulars = Columns.serializer.deserialize(in, metadata);
- fetched = new PartitionColumns(statics, regulars);
+ fetched = new RegularAndStaticColumns(statics, regulars);
}
else
{
- fetched = metadata.partitionColumns();
+ fetched = metadata.regularAndStaticColumns();
}
}
@@ -480,7 +601,7 @@
{
Columns statics = Columns.serializer.deserialize(in, metadata);
Columns regulars = Columns.serializer.deserialize(in, metadata);
- queried = new PartitionColumns(statics, regulars);
+ queried = new RegularAndStaticColumns(statics, regulars);
}
SortedSetMultimap<ColumnIdentifier, ColumnSubselection> subSelections = null;
@@ -495,14 +616,25 @@
}
}
+ // Same concern than in serialize/serializedSize: we should be wary of the change in meaning for isFetchAll.
+ // If we get a filter with isFetchAll from 3.0/3.x, it actually expects all static columns to be fetched,
+ // make sure we do that (note that if queried == null, that's already what we do).
+ // Note that here again this will make us do a bit more work that necessary, namely we'll _query_ all
+ // statics even though we only care about _fetching_ them all, but that's a minor inefficiency, so fine
+ // during upgrade.
+ if (version <= MessagingService.VERSION_30 && isFetchAll && queried != null)
+ queried = new RegularAndStaticColumns(metadata.staticColumns(), queried.regulars);
+
return new ColumnFilter(isFetchAll, fetched, queried, subSelections);
}
public long serializedSize(ColumnFilter selection, int version)
{
+ selection = maybeUpdateForBackwardCompatility(selection, version);
+
long size = 1; // header byte
- if (version >= MessagingService.VERSION_3014 && selection.isFetchAll)
+ if (version >= MessagingService.VERSION_3014 && selection.fetchAllRegulars)
{
size += Columns.serializer.serializedSize(selection.fetched.statics);
size += Columns.serializer.serializedSize(selection.fetched.regulars);
diff --git a/src/java/org/apache/cassandra/db/filter/ColumnSubselection.java b/src/java/org/apache/cassandra/db/filter/ColumnSubselection.java
index b762fa5..d0cc514 100644
--- a/src/java/org/apache/cassandra/db/filter/ColumnSubselection.java
+++ b/src/java/org/apache/cassandra/db/filter/ColumnSubselection.java
@@ -21,15 +21,16 @@
import java.nio.ByteBuffer;
import java.util.Comparator;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
import org.apache.cassandra.db.TypeSizes;
-import org.apache.cassandra.db.rows.CellPath;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.db.marshal.CollectionType;
import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.db.rows.CellPath;
+import org.apache.cassandra.exceptions.UnknownColumnException;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.utils.ByteBufferUtil;
/**
@@ -44,28 +45,28 @@
private enum Kind { SLICE, ELEMENT }
- protected final ColumnDefinition column;
+ protected final ColumnMetadata column;
- protected ColumnSubselection(ColumnDefinition column)
+ protected ColumnSubselection(ColumnMetadata column)
{
this.column = column;
}
- public static ColumnSubselection slice(ColumnDefinition column, CellPath from, CellPath to)
+ public static ColumnSubselection slice(ColumnMetadata column, CellPath from, CellPath to)
{
assert column.isComplex() && column.type instanceof CollectionType;
assert from.size() <= 1 && to.size() <= 1;
return new Slice(column, from, to);
}
- public static ColumnSubselection element(ColumnDefinition column, CellPath elt)
+ public static ColumnSubselection element(ColumnMetadata column, CellPath elt)
{
assert column.isComplex() && column.type instanceof CollectionType;
assert elt.size() == 1;
return new Element(column, elt);
}
- public ColumnDefinition column()
+ public ColumnMetadata column()
{
return column;
}
@@ -91,7 +92,7 @@
private final CellPath from;
private final CellPath to;
- private Slice(ColumnDefinition column, CellPath from, CellPath to)
+ private Slice(ColumnMetadata column, CellPath from, CellPath to)
{
super(column);
this.from = from;
@@ -132,7 +133,7 @@
{
private final CellPath element;
- private Element(ColumnDefinition column, CellPath elt)
+ private Element(ColumnMetadata column, CellPath elt)
{
super(column);
this.element = elt;
@@ -166,7 +167,7 @@
{
public void serialize(ColumnSubselection subSel, DataOutputPlus out, int version) throws IOException
{
- ColumnDefinition column = subSel.column();
+ ColumnMetadata column = subSel.column();
ByteBufferUtil.writeWithShortLength(column.name.bytes, out);
out.writeByte(subSel.kind().ordinal());
switch (subSel.kind())
@@ -185,18 +186,18 @@
}
}
- public ColumnSubselection deserialize(DataInputPlus in, int version, CFMetaData metadata) throws IOException
+ public ColumnSubselection deserialize(DataInputPlus in, int version, TableMetadata metadata) throws IOException
{
ByteBuffer name = ByteBufferUtil.readWithShortLength(in);
- ColumnDefinition column = metadata.getColumnDefinition(name);
+ ColumnMetadata column = metadata.getColumn(name);
if (column == null)
{
// If we don't find the definition, it could be we have data for a dropped column, and we shouldn't
- // fail deserialization because of that. So we grab a "fake" ColumnDefinition that ensure proper
+ // fail deserialization because of that. So we grab a "fake" ColumnMetadata that ensure proper
// deserialization. The column will be ignore later on anyway.
- column = metadata.getDroppedColumnDefinition(name);
+ column = metadata.getDroppedColumn(name);
if (column == null)
- throw new RuntimeException("Unknown column " + UTF8Type.instance.getString(name) + " during deserialization");
+ throw new UnknownColumnException("Unknown column " + UTF8Type.instance.getString(name) + " during deserialization");
}
Kind kind = Kind.values()[in.readUnsignedByte()];
@@ -217,7 +218,7 @@
{
long size = 0;
- ColumnDefinition column = subSel.column();
+ ColumnMetadata column = subSel.column();
size += TypeSizes.sizeofWithShortLength(column.name.bytes);
size += 1; // kind
switch (subSel.kind())
diff --git a/src/java/org/apache/cassandra/db/filter/DataLimits.java b/src/java/org/apache/cassandra/db/filter/DataLimits.java
index cf2bc13..3a766e0 100644
--- a/src/java/org/apache/cassandra/db/filter/DataLimits.java
+++ b/src/java/org/apache/cassandra/db/filter/DataLimits.java
@@ -20,7 +20,6 @@
import java.io.IOException;
import java.nio.ByteBuffer;
-import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.aggregation.GroupMaker;
import org.apache.cassandra.db.aggregation.GroupingState;
@@ -38,8 +37,8 @@
/**
* Object in charge of tracking if we have fetch enough data for a given query.
*
- * The reason this is not just a simple integer is that Thrift and CQL3 count
- * stuffs in different ways. This is what abstract those differences.
+ * This is more complicated than a single count because we support PER PARTITION
+ * limits, but also due to GROUP BY and paging.
*/
public abstract class DataLimits
{
@@ -82,32 +81,13 @@
// partition (see SelectStatement.makeFilter). So an "unbounded" distinct is still actually doing some filtering.
public static final DataLimits DISTINCT_NONE = new CQLLimits(NO_LIMIT, 1, true);
- public enum Kind { CQL_LIMIT, CQL_PAGING_LIMIT, THRIFT_LIMIT, SUPER_COLUMN_COUNTING_LIMIT, CQL_GROUP_BY_LIMIT, CQL_GROUP_BY_PAGING_LIMIT }
+ public enum Kind { CQL_LIMIT, CQL_PAGING_LIMIT, CQL_GROUP_BY_LIMIT, CQL_GROUP_BY_PAGING_LIMIT }
public static DataLimits cqlLimits(int cqlRowLimit)
{
return cqlRowLimit == NO_LIMIT ? NONE : new CQLLimits(cqlRowLimit);
}
- // mixed mode partition range scans on compact storage tables without clustering columns coordinated by 2.x are
- // returned as one (cql) row per cell, but we need to count each partition as a single row. So we just return a
- // CQLLimits instance that doesn't count rows towards it's limit. See CASSANDRA-15072
- public static DataLimits legacyCompactStaticCqlLimits(int cqlRowLimits)
- {
- return new CQLLimits(cqlRowLimits) {
- public Counter newCounter(int nowInSec, boolean assumeLiveData, boolean countPartitionsWithOnlyStaticData, boolean enforceStrictLiveness)
- {
- return new CQLCounter(nowInSec, assumeLiveData, countPartitionsWithOnlyStaticData, enforceStrictLiveness) {
- public Row applyToRow(Row row)
- {
- // noop: only count full partitions
- return row;
- }
- };
- }
- };
- }
-
public static DataLimits cqlLimits(int cqlRowLimit, int perPartitionLimit)
{
return cqlRowLimit == NO_LIMIT && perPartitionLimit == NO_LIMIT
@@ -135,16 +115,6 @@
return CQLLimits.distinct(cqlRowLimit);
}
- public static DataLimits thriftLimits(int partitionLimit, int cellPerPartitionLimit)
- {
- return new ThriftLimits(partitionLimit, cellPerPartitionLimit);
- }
-
- public static DataLimits superColumnCountingLimits(int partitionLimit, int cellPerPartitionLimit)
- {
- return new SuperColumnCountingLimits(partitionLimit, cellPerPartitionLimit);
- }
-
public abstract Kind kind();
public abstract boolean isUnlimited();
@@ -202,8 +172,8 @@
/**
* The max number of results this limits enforces.
* <p>
- * Note that the actual definition of "results" depends a bit: for CQL, it's always rows, but for
- * thrift, it means cells.
+ * Note that the actual definition of "results" depends a bit: for "normal" queries it's a number of rows,
+ * but for GROUP BY queries it's a number of groups.
*
* @return the maximum number of results this limits enforces.
*/
@@ -245,8 +215,7 @@
}
/**
- * Estimate the number of results (the definition of "results" will be rows for CQL queries
- * and partitions for thrift ones) that a full scan of the provided cfs would yield.
+ * Estimate the number of results that a full scan of the provided cfs would yield.
*/
public abstract float estimateTotalResults(ColumnFamilyStore cfs);
@@ -477,7 +446,7 @@
{
// TODO: we should start storing stats on the number of rows (instead of the number of cells, which
// is what getMeanColumns returns)
- float rowsPerPartition = ((float) cfs.getMeanColumns()) / cfs.metadata.partitionColumns().regulars.size();
+ float rowsPerPartition = ((float) cfs.getMeanEstimatedCellPerPartitionCount()) / cfs.metadata().regularColumns().size();
return rowsPerPartition * (cfs.estimateKeys());
}
@@ -1156,253 +1125,6 @@
}
}
- /**
- * Limits used by thrift; this count partition and cells.
- */
- private static class ThriftLimits extends DataLimits
- {
- protected final int partitionLimit;
- protected final int cellPerPartitionLimit;
-
- private ThriftLimits(int partitionLimit, int cellPerPartitionLimit)
- {
- this.partitionLimit = partitionLimit;
- this.cellPerPartitionLimit = cellPerPartitionLimit;
- }
-
- public Kind kind()
- {
- return Kind.THRIFT_LIMIT;
- }
-
- public boolean isUnlimited()
- {
- return partitionLimit == NO_LIMIT && cellPerPartitionLimit == NO_LIMIT;
- }
-
- public boolean isDistinct()
- {
- return false;
- }
-
- public DataLimits forPaging(int pageSize)
- {
- // We don't support paging on thrift in general but do use paging under the hood for get_count. For
- // that case, we only care about limiting cellPerPartitionLimit (since it's paging over a single
- // partition). We do check that the partition limit is 1 however to make sure this is not misused
- // (as this wouldn't work properly for range queries).
- assert partitionLimit == 1;
- return new ThriftLimits(partitionLimit, pageSize);
- }
-
- public DataLimits forPaging(int pageSize, ByteBuffer lastReturnedKey, int lastReturnedKeyRemaining)
- {
- throw new UnsupportedOperationException();
- }
-
- public DataLimits forShortReadRetry(int toFetch)
- {
- // Short read retries are always done for a single partition at a time, so it's ok to ignore the
- // partition limit for those
- return new ThriftLimits(1, toFetch);
- }
-
- public boolean hasEnoughLiveData(CachedPartition cached, int nowInSec, boolean countPartitionsWithOnlyStaticData, boolean enforceStrictLiveness)
- {
- // We want the number of cells that are currently live. Getting that precise number forces
- // us to iterate the cached partition in general, but we can avoid that if:
- // - The number of non-expiring live cells is greater than the number of cells asked (we then
- // know we have enough live cells).
- // - The number of cells cached is less than requested, in which case we know we won't have enough.
- if (cached.nonExpiringLiveCells() >= cellPerPartitionLimit)
- return true;
-
- if (cached.nonTombstoneCellCount() < cellPerPartitionLimit)
- return false;
-
- // Otherwise, we need to re-count
- DataLimits.Counter counter = newCounter(nowInSec, false, countPartitionsWithOnlyStaticData, enforceStrictLiveness);
- try (UnfilteredRowIterator cacheIter = cached.unfilteredIterator(ColumnFilter.selection(cached.columns()), Slices.ALL, false);
- UnfilteredRowIterator iter = counter.applyTo(cacheIter))
- {
- // Consume the iterator until we've counted enough
- while (iter.hasNext())
- iter.next();
- return counter.isDone();
- }
- }
-
- public Counter newCounter(int nowInSec, boolean assumeLiveData, boolean countPartitionsWithOnlyStaticData, boolean enforceStrictLiveness)
- {
- return new ThriftCounter(nowInSec, assumeLiveData, enforceStrictLiveness);
- }
-
- public int count()
- {
- return partitionLimit * cellPerPartitionLimit;
- }
-
- public int perPartitionCount()
- {
- return cellPerPartitionLimit;
- }
-
- public DataLimits withoutState()
- {
- return this;
- }
-
- public float estimateTotalResults(ColumnFamilyStore cfs)
- {
- // remember that getMeansColumns returns a number of cells: we should clean nomenclature
- float cellsPerPartition = ((float) cfs.getMeanColumns()) / cfs.metadata.partitionColumns().regulars.size();
- return cellsPerPartition * cfs.estimateKeys();
- }
-
- protected class ThriftCounter extends Counter
- {
- protected int partitionsCounted;
- protected int cellsCounted;
- protected int cellsInCurrentPartition;
-
- public ThriftCounter(int nowInSec, boolean assumeLiveData, boolean enforceStrictLiveness)
- {
- super(nowInSec, assumeLiveData, enforceStrictLiveness);
- }
-
- @Override
- public void applyToPartition(DecoratedKey partitionKey, Row staticRow)
- {
- cellsInCurrentPartition = 0;
- if (!staticRow.isEmpty())
- applyToRow(staticRow);
- }
-
- @Override
- public Row applyToRow(Row row)
- {
- for (Cell cell : row.cells())
- {
- if (assumeLiveData || cell.isLive(nowInSec))
- {
- ++cellsCounted;
- if (++cellsInCurrentPartition >= cellPerPartitionLimit)
- stopInPartition();
- }
- }
- return row;
- }
-
- @Override
- public void onPartitionClose()
- {
- if (++partitionsCounted >= partitionLimit)
- stop();
- super.onPartitionClose();
- }
-
- public int counted()
- {
- return cellsCounted;
- }
-
- public int countedInCurrentPartition()
- {
- return cellsInCurrentPartition;
- }
-
- public int rowCounted()
- {
- throw new UnsupportedOperationException();
- }
-
- public int rowCountedInCurrentPartition()
- {
- throw new UnsupportedOperationException();
- }
-
- public boolean isDone()
- {
- return partitionsCounted >= partitionLimit;
- }
-
- public boolean isDoneForPartition()
- {
- return isDone() || cellsInCurrentPartition >= cellPerPartitionLimit;
- }
- }
-
- @Override
- public String toString()
- {
- // This is not valid CQL, but that's ok since it's not used for CQL queries.
- return String.format("THRIFT LIMIT (partitions=%d, cells_per_partition=%d)", partitionLimit, cellPerPartitionLimit);
- }
- }
-
- /**
- * Limits used for thrift get_count when we only want to count super columns.
- */
- private static class SuperColumnCountingLimits extends ThriftLimits
- {
- private SuperColumnCountingLimits(int partitionLimit, int cellPerPartitionLimit)
- {
- super(partitionLimit, cellPerPartitionLimit);
- }
-
- public Kind kind()
- {
- return Kind.SUPER_COLUMN_COUNTING_LIMIT;
- }
-
- public DataLimits forPaging(int pageSize)
- {
- // We don't support paging on thrift in general but do use paging under the hood for get_count. For
- // that case, we only care about limiting cellPerPartitionLimit (since it's paging over a single
- // partition). We do check that the partition limit is 1 however to make sure this is not misused
- // (as this wouldn't work properly for range queries).
- assert partitionLimit == 1;
- return new SuperColumnCountingLimits(partitionLimit, pageSize);
- }
-
- public DataLimits forShortReadRetry(int toFetch)
- {
- // Short read retries are always done for a single partition at a time, so it's ok to ignore the
- // partition limit for those
- return new SuperColumnCountingLimits(1, toFetch);
- }
-
- @Override
- public Counter newCounter(int nowInSec, boolean assumeLiveData, boolean countPartitionsWithOnlyStaticData, boolean enforceStrictLiveness)
- {
- return new SuperColumnCountingCounter(nowInSec, assumeLiveData, enforceStrictLiveness);
- }
-
- protected class SuperColumnCountingCounter extends ThriftCounter
- {
- private final boolean enforceStrictLiveness;
-
- public SuperColumnCountingCounter(int nowInSec, boolean assumeLiveData, boolean enforceStrictLiveness)
- {
- super(nowInSec, assumeLiveData, enforceStrictLiveness);
- this.enforceStrictLiveness = enforceStrictLiveness;
- }
-
- @Override
- public Row applyToRow(Row row)
- {
- // In the internal format, a row == a super column, so that's what we want to count.
- if (isLive(row))
- {
- ++cellsCounted;
- if (++cellsInCurrentPartition >= cellPerPartitionLimit)
- stopInPartition();
- }
- return row;
- }
- }
- }
-
public static class Serializer
{
public void serialize(DataLimits limits, DataOutputPlus out, int version, ClusteringComparator comparator) throws IOException
@@ -1442,12 +1164,6 @@
out.writeUnsignedVInt(pagingLimits.lastReturnedKeyRemaining);
}
break;
- case THRIFT_LIMIT:
- case SUPER_COLUMN_COUNTING_LIMIT:
- ThriftLimits thriftLimits = (ThriftLimits)limits;
- out.writeUnsignedVInt(thriftLimits.partitionLimit);
- out.writeUnsignedVInt(thriftLimits.cellPerPartitionLimit);
- break;
}
}
@@ -1496,13 +1212,6 @@
lastKey,
lastRemaining);
}
- case THRIFT_LIMIT:
- case SUPER_COLUMN_COUNTING_LIMIT:
- int partitionLimit = (int) in.readUnsignedVInt();
- int cellPerPartitionLimit = (int) in.readUnsignedVInt();
- return kind == Kind.THRIFT_LIMIT
- ? new ThriftLimits(partitionLimit, cellPerPartitionLimit)
- : new SuperColumnCountingLimits(partitionLimit, cellPerPartitionLimit);
}
throw new AssertionError();
}
@@ -1544,12 +1253,6 @@
size += TypeSizes.sizeofUnsignedVInt(pagingLimits.lastReturnedKeyRemaining);
}
break;
- case THRIFT_LIMIT:
- case SUPER_COLUMN_COUNTING_LIMIT:
- ThriftLimits thriftLimits = (ThriftLimits) limits;
- size += TypeSizes.sizeofUnsignedVInt(thriftLimits.partitionLimit);
- size += TypeSizes.sizeofUnsignedVInt(thriftLimits.cellPerPartitionLimit);
- break;
default:
throw new AssertionError();
}
diff --git a/src/java/org/apache/cassandra/db/filter/RowFilter.java b/src/java/org/apache/cassandra/db/filter/RowFilter.java
index b4f8a7f..71cbb9e 100644
--- a/src/java/org/apache/cassandra/db/filter/RowFilter.java
+++ b/src/java/org/apache/cassandra/db/filter/RowFilter.java
@@ -28,14 +28,10 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
import org.apache.cassandra.cql3.Operator;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.context.*;
import org.apache.cassandra.db.marshal.*;
-import org.apache.cassandra.db.partitions.FilteredPartition;
-import org.apache.cassandra.db.partitions.ImmutableBTreePartition;
import org.apache.cassandra.db.partitions.PartitionIterator;
import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
import org.apache.cassandra.db.rows.*;
@@ -43,8 +39,9 @@
import org.apache.cassandra.exceptions.InvalidRequestException;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
-import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.schema.IndexMetadata;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FBUtilities;
@@ -84,32 +81,21 @@
return new CQLFilter(new ArrayList<>(capacity));
}
- public static RowFilter forThrift(int capacity)
- {
- return new ThriftFilter(new ArrayList<>(capacity));
- }
-
- public SimpleExpression add(ColumnDefinition def, Operator op, ByteBuffer value)
+ public SimpleExpression add(ColumnMetadata def, Operator op, ByteBuffer value)
{
SimpleExpression expression = new SimpleExpression(def, op, value);
add(expression);
return expression;
}
- public void addMapEquality(ColumnDefinition def, ByteBuffer key, Operator op, ByteBuffer value)
+ public void addMapEquality(ColumnMetadata def, ByteBuffer key, Operator op, ByteBuffer value)
{
add(new MapEqualityExpression(def, key, op, value));
}
- public void addThriftExpression(CFMetaData metadata, ByteBuffer name, Operator op, ByteBuffer value)
+ public void addCustomIndexExpression(TableMetadata metadata, IndexMetadata targetIndex, ByteBuffer value)
{
- assert (this instanceof ThriftFilter);
- add(new ThriftExpression(metadata, name, op, value));
- }
-
- public void addCustomIndexExpression(CFMetaData cfm, IndexMetadata targetIndex, ByteBuffer value)
- {
- add(new CustomExpression(cfm, targetIndex, value));
+ add(new CustomExpression(metadata, targetIndex, value));
}
private void add(Expression expression)
@@ -136,14 +122,14 @@
{
for (Expression expression : expressions)
{
- ColumnDefinition column = expression.column();
+ ColumnMetadata column = expression.column();
if (column.isClusteringColumn() || column.isRegular())
return true;
}
return false;
}
- protected abstract Transformation<BaseRowIterator<?>> filter(CFMetaData metadata, int nowInSec);
+ protected abstract Transformation<BaseRowIterator<?>> filter(TableMetadata metadata, int nowInSec);
/**
* Filters the provided iterator so that only the row satisfying the expression of this filter
@@ -166,7 +152,7 @@
* @param nowInSec the time of query in seconds.
* @return the filtered iterator.
*/
- public PartitionIterator filter(PartitionIterator iter, CFMetaData metadata, int nowInSec)
+ public PartitionIterator filter(PartitionIterator iter, TableMetadata metadata, int nowInSec)
{
return expressions.isEmpty() ? iter : Transformation.apply(iter, filter(metadata, nowInSec));
}
@@ -180,7 +166,7 @@
* @param nowInSec the current time in seconds (to know what is live and what isn't).
* @return {@code true} if {@code row} in partition {@code partitionKey} satisfies this row filter.
*/
- public boolean isSatisfiedBy(CFMetaData metadata, DecoratedKey partitionKey, Row row, int nowInSec)
+ public boolean isSatisfiedBy(TableMetadata metadata, DecoratedKey partitionKey, Row row, int nowInSec)
{
// We purge all tombstones as the expressions isSatisfiedBy methods expects it
Row purged = row.purge(DeletionPurger.PURGE_ALL, nowInSec, metadata.enforceStrictLiveness());
@@ -269,20 +255,6 @@
return expressions.iterator();
}
- private static Clustering makeCompactClustering(CFMetaData metadata, ByteBuffer name)
- {
- assert metadata.isCompactTable();
- if (metadata.isCompound())
- {
- List<ByteBuffer> values = CompositeType.splitName(name);
- return Clustering.make(values.toArray(new ByteBuffer[metadata.comparator.size()]));
- }
- else
- {
- return Clustering.make(name);
- }
- }
-
@Override
public String toString()
{
@@ -303,7 +275,7 @@
super(expressions);
}
- protected Transformation<BaseRowIterator<?>> filter(CFMetaData metadata, int nowInSec)
+ protected Transformation<BaseRowIterator<?>> filter(TableMetadata metadata, int nowInSec)
{
List<Expression> partitionLevelExpressions = new ArrayList<>();
List<Expression> rowLevelExpressions = new ArrayList<>();
@@ -321,6 +293,8 @@
return new Transformation<BaseRowIterator<?>>()
{
DecoratedKey pk;
+
+ @SuppressWarnings("resource")
protected BaseRowIterator<?> applyToPartition(BaseRowIterator<?> partition)
{
pk = partition.partitionKey();
@@ -367,75 +341,22 @@
}
}
- private static class ThriftFilter extends RowFilter
- {
- private ThriftFilter(List<Expression> expressions)
- {
- super(expressions);
- }
-
- protected Transformation<BaseRowIterator<?>> filter(CFMetaData metadata, int nowInSec)
- {
- // Thrift does not filter rows, it filters entire partition if any of the expression is not
- // satisfied, which forces us to materialize the result (in theory we could materialize only
- // what we need which might or might not be everything, but we keep it simple since in practice
- // it's not worth that it has ever been).
- return new Transformation<BaseRowIterator<?>>()
- {
- protected BaseRowIterator<?> applyToPartition(BaseRowIterator<?> partition)
- {
- return partition instanceof UnfilteredRowIterator ? applyTo((UnfilteredRowIterator) partition)
- : applyTo((RowIterator) partition);
- }
-
- private UnfilteredRowIterator applyTo(UnfilteredRowIterator partition)
- {
- ImmutableBTreePartition result = ImmutableBTreePartition.create(partition);
- partition.close();
- return accepts(result) ? result.unfilteredIterator() : null;
- }
-
- private RowIterator applyTo(RowIterator partition)
- {
- FilteredPartition result = FilteredPartition.create(partition);
- return accepts(result) ? result.rowIterator() : null;
- }
-
- private boolean accepts(ImmutableBTreePartition result)
- {
- // The partition needs to have a row for every expression, and the expression needs to be valid.
- for (Expression expr : expressions)
- {
- assert expr instanceof ThriftExpression;
- Row row = result.getRow(makeCompactClustering(metadata, expr.column().name.bytes));
- if (row == null || !expr.isSatisfiedBy(metadata, result.partitionKey(), row))
- return false;
- }
- // If we get there, it means all expressions where satisfied, so return the original result
- return true;
- }
- };
- }
-
- protected RowFilter withNewExpressions(List<Expression> expressions)
- {
- return new ThriftFilter(expressions);
- }
- }
-
public static abstract class Expression
{
private static final Serializer serializer = new Serializer();
- // Note: the order of this enum matter, it's used for serialization
- protected enum Kind { SIMPLE, MAP_EQUALITY, THRIFT_DYN_EXPR, CUSTOM, USER }
+ // Note: the order of this enum matter, it's used for serialization,
+ // and this is why we have some UNUSEDX for values we don't use anymore
+ // (we could clean those on a major protocol update, but it's not worth
+ // the trouble for now)
+ protected enum Kind { SIMPLE, MAP_EQUALITY, UNUSED1, CUSTOM, USER }
protected abstract Kind kind();
- protected final ColumnDefinition column;
+ protected final ColumnMetadata column;
protected final Operator operator;
protected final ByteBuffer value;
- protected Expression(ColumnDefinition column, Operator operator, ByteBuffer value)
+ protected Expression(ColumnMetadata column, Operator operator, ByteBuffer value)
{
this.column = column;
this.operator = operator;
@@ -452,7 +373,7 @@
return kind() == Kind.USER;
}
- public ColumnDefinition column()
+ public ColumnMetadata column()
{
return column;
}
@@ -509,19 +430,21 @@
/**
* Returns whether the provided row satisfied this expression or not.
*
+ *
+ * @param metadata
* @param partitionKey the partition key for row to check.
* @param row the row to check. It should *not* contain deleted cells
* (i.e. it should come from a RowIterator).
* @return whether the row is satisfied by this expression.
*/
- public abstract boolean isSatisfiedBy(CFMetaData metadata, DecoratedKey partitionKey, Row row);
+ public abstract boolean isSatisfiedBy(TableMetadata metadata, DecoratedKey partitionKey, Row row);
- protected ByteBuffer getValue(CFMetaData metadata, DecoratedKey partitionKey, Row row)
+ protected ByteBuffer getValue(TableMetadata metadata, DecoratedKey partitionKey, Row row)
{
switch (column.kind)
{
case PARTITION_KEY:
- return metadata.getKeyValidator() instanceof CompositeType
+ return metadata.partitionKeyType instanceof CompositeType
? CompositeType.extractComponent(partitionKey.getKey(), column.position())
: partitionKey.getKey();
case CLUSTERING:
@@ -559,16 +482,12 @@
{
public void serialize(Expression expression, DataOutputPlus out, int version) throws IOException
{
- if (version >= MessagingService.VERSION_30)
- out.writeByte(expression.kind().ordinal());
+ out.writeByte(expression.kind().ordinal());
// Custom expressions include neither a column or operator, but all
- // other expressions do. Also, custom expressions are 3.0+ only, so
- // the column & operator will always be the first things written for
- // any pre-3.0 version
+ // other expressions do.
if (expression.kind() == Kind.CUSTOM)
{
- assert version >= MessagingService.VERSION_30;
IndexMetadata.serializer.serialize(((CustomExpression)expression).targetIndex, out, version);
ByteBufferUtil.writeWithShortLength(expression.value, out);
return;
@@ -576,7 +495,6 @@
if (expression.kind() == Kind.USER)
{
- assert version >= MessagingService.VERSION_30;
UserExpression.serialize((UserExpression)expression, out, version);
return;
}
@@ -591,97 +509,52 @@
break;
case MAP_EQUALITY:
MapEqualityExpression mexpr = (MapEqualityExpression)expression;
- if (version < MessagingService.VERSION_30)
- {
- ByteBufferUtil.writeWithShortLength(mexpr.getIndexValue(), out);
- }
- else
- {
- ByteBufferUtil.writeWithShortLength(mexpr.key, out);
- ByteBufferUtil.writeWithShortLength(mexpr.value, out);
- }
- break;
- case THRIFT_DYN_EXPR:
- ByteBufferUtil.writeWithShortLength(((ThriftExpression)expression).value, out);
+ ByteBufferUtil.writeWithShortLength(mexpr.key, out);
+ ByteBufferUtil.writeWithShortLength(mexpr.value, out);
break;
}
}
- public Expression deserialize(DataInputPlus in, int version, CFMetaData metadata) throws IOException
+ public Expression deserialize(DataInputPlus in, int version, TableMetadata metadata) throws IOException
{
- Kind kind = null;
- ByteBuffer name;
- Operator operator;
- ColumnDefinition column;
+ Kind kind = Kind.values()[in.readByte()];
- if (version >= MessagingService.VERSION_30)
+ // custom expressions (3.0+ only) do not contain a column or operator, only a value
+ if (kind == Kind.CUSTOM)
{
- kind = Kind.values()[in.readByte()];
- // custom expressions (3.0+ only) do not contain a column or operator, only a value
- if (kind == Kind.CUSTOM)
- {
- return new CustomExpression(metadata,
- IndexMetadata.serializer.deserialize(in, version, metadata),
- ByteBufferUtil.readWithShortLength(in));
- }
-
- if (kind == Kind.USER)
- {
- return UserExpression.deserialize(in, version, metadata);
- }
+ return new CustomExpression(metadata,
+ IndexMetadata.serializer.deserialize(in, version, metadata),
+ ByteBufferUtil.readWithShortLength(in));
}
- name = ByteBufferUtil.readWithShortLength(in);
- operator = Operator.readFrom(in);
- column = metadata.getColumnDefinition(name);
+ if (kind == Kind.USER)
+ return UserExpression.deserialize(in, version, metadata);
+
+ ByteBuffer name = ByteBufferUtil.readWithShortLength(in);
+ Operator operator = Operator.readFrom(in);
+ ColumnMetadata column = metadata.getColumn(name);
+
if (!metadata.isCompactTable() && column == null)
throw new RuntimeException("Unknown (or dropped) column " + UTF8Type.instance.getString(name) + " during deserialization");
- if (version < MessagingService.VERSION_30)
- {
- if (column == null)
- kind = Kind.THRIFT_DYN_EXPR;
- else if (column.type instanceof MapType && operator == Operator.EQ)
- kind = Kind.MAP_EQUALITY;
- else
- kind = Kind.SIMPLE;
- }
-
- assert kind != null;
switch (kind)
{
case SIMPLE:
return new SimpleExpression(column, operator, ByteBufferUtil.readWithShortLength(in));
case MAP_EQUALITY:
- ByteBuffer key, value;
- if (version < MessagingService.VERSION_30)
- {
- ByteBuffer composite = ByteBufferUtil.readWithShortLength(in);
- key = CompositeType.extractComponent(composite, 0);
- value = CompositeType.extractComponent(composite, 0);
- }
- else
- {
- key = ByteBufferUtil.readWithShortLength(in);
- value = ByteBufferUtil.readWithShortLength(in);
- }
+ ByteBuffer key = ByteBufferUtil.readWithShortLength(in);
+ ByteBuffer value = ByteBufferUtil.readWithShortLength(in);
return new MapEqualityExpression(column, key, operator, value);
- case THRIFT_DYN_EXPR:
- return new ThriftExpression(metadata, name, operator, ByteBufferUtil.readWithShortLength(in));
}
throw new AssertionError();
}
-
public long serializedSize(Expression expression, int version)
{
- // version 3.0+ includes a byte for Kind
- long size = version >= MessagingService.VERSION_30 ? 1 : 0;
+ long size = 1; // kind byte
// Custom expressions include neither a column or operator, but all
- // other expressions do. Also, custom expressions are 3.0+ only, so
- // the column & operator will always be the first things written for
- // any pre-3.0 version
+ // other expressions do.
if (expression.kind() != Kind.CUSTOM && expression.kind() != Kind.USER)
size += ByteBufferUtil.serializedSizeWithShortLength(expression.column().name.bytes)
+ expression.operator.serializedSize();
@@ -693,23 +566,16 @@
break;
case MAP_EQUALITY:
MapEqualityExpression mexpr = (MapEqualityExpression)expression;
- if (version < MessagingService.VERSION_30)
- size += ByteBufferUtil.serializedSizeWithShortLength(mexpr.getIndexValue());
- else
- size += ByteBufferUtil.serializedSizeWithShortLength(mexpr.key)
- + ByteBufferUtil.serializedSizeWithShortLength(mexpr.value);
- break;
- case THRIFT_DYN_EXPR:
- size += ByteBufferUtil.serializedSizeWithShortLength(((ThriftExpression)expression).value);
+ size += ByteBufferUtil.serializedSizeWithShortLength(mexpr.key)
+ + ByteBufferUtil.serializedSizeWithShortLength(mexpr.value);
break;
case CUSTOM:
- if (version >= MessagingService.VERSION_30)
- size += IndexMetadata.serializer.serializedSize(((CustomExpression)expression).targetIndex, version)
- + ByteBufferUtil.serializedSizeWithShortLength(expression.value);
+ size += IndexMetadata.serializer.serializedSize(((CustomExpression)expression).targetIndex, version)
+ + ByteBufferUtil.serializedSizeWithShortLength(expression.value);
break;
case USER:
- if (version >= MessagingService.VERSION_30)
- size += UserExpression.serializedSize((UserExpression)expression, version);
+ size += UserExpression.serializedSize((UserExpression)expression, version);
+ break;
}
return size;
}
@@ -721,12 +587,12 @@
*/
public static class SimpleExpression extends Expression
{
- SimpleExpression(ColumnDefinition column, Operator operator, ByteBuffer value)
+ SimpleExpression(ColumnMetadata column, Operator operator, ByteBuffer value)
{
super(column, operator, value);
}
- public boolean isSatisfiedBy(CFMetaData metadata, DecoratedKey partitionKey, Row row)
+ public boolean isSatisfiedBy(TableMetadata metadata, DecoratedKey partitionKey, Row row)
{
// We support null conditions for LWT (in ColumnCondition) but not for RowFilter.
// TODO: we should try to merge both code someday.
@@ -875,7 +741,7 @@
{
private final ByteBuffer key;
- public MapEqualityExpression(ColumnDefinition column, ByteBuffer key, Operator operator, ByteBuffer value)
+ public MapEqualityExpression(ColumnMetadata column, ByteBuffer key, Operator operator, ByteBuffer value)
{
super(column, operator, value);
assert column.type instanceof MapType && operator == Operator.EQ;
@@ -897,7 +763,7 @@
return CompositeType.build(key, value);
}
- public boolean isSatisfiedBy(CFMetaData metadata, DecoratedKey partitionKey, Row row)
+ public boolean isSatisfiedBy(TableMetadata metadata, DecoratedKey partitionKey, Row row)
{
assert key != null;
// We support null conditions for LWT (in ColumnCondition) but not for RowFilter.
@@ -962,75 +828,27 @@
}
/**
- * An expression of the form 'name' = 'value', but where 'name' is actually the
- * clustering value for a compact table. This is only for thrift.
- */
- private static class ThriftExpression extends Expression
- {
- public ThriftExpression(CFMetaData metadata, ByteBuffer name, Operator operator, ByteBuffer value)
- {
- super(makeDefinition(metadata, name), operator, value);
- assert metadata.isCompactTable();
- }
-
- private static ColumnDefinition makeDefinition(CFMetaData metadata, ByteBuffer name)
- {
- ColumnDefinition def = metadata.getColumnDefinition(name);
- if (def != null)
- return def;
-
- // In thrift, we actually allow expression on non-defined columns for the sake of filtering. To accomodate
- // this we create a "fake" definition. This is messy but it works so is probably good enough.
- return ColumnDefinition.regularDef(metadata, name, metadata.compactValueColumn().type);
- }
-
- public boolean isSatisfiedBy(CFMetaData metadata, DecoratedKey partitionKey, Row row)
- {
- assert value != null;
-
- // On thrift queries, even if the column expression is a "static" one, we'll have convert it as a "dynamic"
- // one in ThriftResultsMerger, so we always expect it to be a dynamic one. Further, we expect this is only
- // called when the row clustering does match the column (see ThriftFilter above).
- assert row.clustering().equals(makeCompactClustering(metadata, column.name.bytes));
- Cell cell = row.getCell(metadata.compactValueColumn());
- return cell != null && operator.isSatisfiedBy(column.type, cell.value(), value);
- }
-
- @Override
- public String toString()
- {
- return String.format("%s %s %s", column.name, operator, column.type.getString(value));
- }
-
- @Override
- protected Kind kind()
- {
- return Kind.THRIFT_DYN_EXPR;
- }
- }
-
- /**
* A custom index expression for use with 2i implementations which support custom syntax and which are not
* necessarily linked to a single column in the base table.
*/
public static final class CustomExpression extends Expression
{
private final IndexMetadata targetIndex;
- private final CFMetaData cfm;
+ private final TableMetadata table;
- public CustomExpression(CFMetaData cfm, IndexMetadata targetIndex, ByteBuffer value)
+ public CustomExpression(TableMetadata table, IndexMetadata targetIndex, ByteBuffer value)
{
// The operator is not relevant, but Expression requires it so for now we just hardcode EQ
- super(makeDefinition(cfm, targetIndex), Operator.EQ, value);
+ super(makeDefinition(table, targetIndex), Operator.EQ, value);
this.targetIndex = targetIndex;
- this.cfm = cfm;
+ this.table = table;
}
- private static ColumnDefinition makeDefinition(CFMetaData cfm, IndexMetadata index)
+ private static ColumnMetadata makeDefinition(TableMetadata table, IndexMetadata index)
{
// Similarly to how we handle non-defined columns in thift, we create a fake column definition to
// represent the target index. This is definitely something that can be improved though.
- return ColumnDefinition.regularDef(cfm, ByteBuffer.wrap(index.name.getBytes()), BytesType.instance);
+ return ColumnMetadata.regularColumn(table, ByteBuffer.wrap(index.name.getBytes()), BytesType.instance);
}
public IndexMetadata getTargetIndex()
@@ -1047,7 +865,7 @@
{
return String.format("expr(%s, %s)",
targetIndex.name,
- Keyspace.openAndGetStore(cfm)
+ Keyspace.openAndGetStore(table)
.indexManager
.getIndex(targetIndex)
.customExpressionValueType());
@@ -1059,7 +877,7 @@
}
// Filtering by custom expressions isn't supported yet, so just accept any row
- public boolean isSatisfiedBy(CFMetaData metadata, DecoratedKey partitionKey, Row row)
+ public boolean isSatisfiedBy(TableMetadata metadata, DecoratedKey partitionKey, Row row)
{
return true;
}
@@ -1116,7 +934,7 @@
{
protected abstract UserExpression deserialize(DataInputPlus in,
int version,
- CFMetaData metadata) throws IOException;
+ TableMetadata metadata) throws IOException;
}
public static void register(Class<? extends UserExpression> expressionClass, Deserializer deserializer)
@@ -1124,7 +942,7 @@
deserializers.registerUserExpressionClass(expressionClass, deserializer);
}
- private static UserExpression deserialize(DataInputPlus in, int version, CFMetaData metadata) throws IOException
+ private static UserExpression deserialize(DataInputPlus in, int version, TableMetadata metadata) throws IOException
{
int id = in.readInt();
Deserializer deserializer = deserializers.getDeserializer(id);
@@ -1145,7 +963,7 @@
return 4 + expression.serializedSize(version);
}
- protected UserExpression(ColumnDefinition column, Operator operator, ByteBuffer value)
+ protected UserExpression(ColumnMetadata column, Operator operator, ByteBuffer value)
{
super(column, operator, value);
}
@@ -1163,29 +981,27 @@
{
public void serialize(RowFilter filter, DataOutputPlus out, int version) throws IOException
{
- out.writeBoolean(filter instanceof ThriftFilter);
+ out.writeBoolean(false); // Old "is for thrift" boolean
out.writeUnsignedVInt(filter.expressions.size());
for (Expression expr : filter.expressions)
Expression.serializer.serialize(expr, out, version);
}
- public RowFilter deserialize(DataInputPlus in, int version, CFMetaData metadata) throws IOException
+ public RowFilter deserialize(DataInputPlus in, int version, TableMetadata metadata) throws IOException
{
- boolean forThrift = in.readBoolean();
+ in.readBoolean(); // Unused
int size = (int)in.readUnsignedVInt();
List<Expression> expressions = new ArrayList<>(size);
for (int i = 0; i < size; i++)
expressions.add(Expression.serializer.deserialize(in, version, metadata));
- return forThrift
- ? new ThriftFilter(expressions)
- : new CQLFilter(expressions);
+ return new CQLFilter(expressions);
}
public long serializedSize(RowFilter filter, int version)
{
- long size = 1 // forThrift
+ long size = 1 // unused boolean
+ TypeSizes.sizeofUnsignedVInt(filter.expressions.size());
for (Expression expr : filter.expressions)
size += Expression.serializer.serializedSize(expr, version);
diff --git a/src/java/org/apache/cassandra/db/filter/TombstoneOverwhelmingException.java b/src/java/org/apache/cassandra/db/filter/TombstoneOverwhelmingException.java
index 622edb4..f7371ec 100644
--- a/src/java/org/apache/cassandra/db/filter/TombstoneOverwhelmingException.java
+++ b/src/java/org/apache/cassandra/db/filter/TombstoneOverwhelmingException.java
@@ -20,19 +20,19 @@
import java.nio.ByteBuffer;
-import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.marshal.*;
public class TombstoneOverwhelmingException extends RuntimeException
{
- public TombstoneOverwhelmingException(int numTombstones, String query, CFMetaData metadata, DecoratedKey lastPartitionKey, ClusteringPrefix lastClustering)
+ public TombstoneOverwhelmingException(int numTombstones, String query, TableMetadata metadata, DecoratedKey lastPartitionKey, ClusteringPrefix lastClustering)
{
super(String.format("Scanned over %d tombstones during query '%s' (last scanned row token was %s and partion key was (%s)); query aborted",
numTombstones, query, lastPartitionKey.getToken(), makePKString(metadata, lastPartitionKey.getKey(), lastClustering)));
}
- private static String makePKString(CFMetaData metadata, ByteBuffer partitionKey, ClusteringPrefix clustering)
+ private static String makePKString(TableMetadata metadata, ByteBuffer partitionKey, ClusteringPrefix clustering)
{
StringBuilder sb = new StringBuilder();
@@ -40,7 +40,7 @@
sb.append("(");
// TODO: We should probably make that a lot easier/transparent for partition keys
- AbstractType<?> pkType = metadata.getKeyValidator();
+ AbstractType<?> pkType = metadata.partitionKeyType;
if (pkType instanceof CompositeType)
{
CompositeType ct = (CompositeType)pkType;
diff --git a/src/java/org/apache/cassandra/db/lifecycle/LifecycleTransaction.java b/src/java/org/apache/cassandra/db/lifecycle/LifecycleTransaction.java
index 5994707..574c6a4 100644
--- a/src/java/org/apache/cassandra/db/lifecycle/LifecycleTransaction.java
+++ b/src/java/org/apache/cassandra/db/lifecycle/LifecycleTransaction.java
@@ -20,8 +20,7 @@
import java.io.File;
import java.nio.file.Path;
import java.util.*;
-import java.util.function.BiFunction;
-
+import java.util.function.BiPredicate;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Predicate;
import com.google.common.collect.*;
@@ -29,7 +28,7 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Directories;
import org.apache.cassandra.db.compaction.OperationType;
@@ -583,9 +582,9 @@
return LogTransaction.removeUnfinishedLeftovers(cfs.getDirectories().getCFDirectories());
}
- public static boolean removeUnfinishedLeftovers(CFMetaData cfMetaData)
+ public static boolean removeUnfinishedLeftovers(TableMetadata metadata)
{
- return LogTransaction.removeUnfinishedLeftovers(cfMetaData);
+ return LogTransaction.removeUnfinishedLeftovers(metadata);
}
/**
@@ -600,7 +599,7 @@
* @param filter - A function that receives each file and its type, it should return true to have the file returned
* @return - the list of files that were scanned and for which the filter returned true
*/
- public static List<File> getFiles(Path folder, BiFunction<File, Directories.FileType, Boolean> filter, Directories.OnTxnErr onTxnErr)
+ public static List<File> getFiles(Path folder, BiPredicate<File, Directories.FileType> filter, Directories.OnTxnErr onTxnErr)
{
return new LogAwareFileLister(folder, filter, onTxnErr).list();
}
diff --git a/src/java/org/apache/cassandra/db/lifecycle/LogAwareFileLister.java b/src/java/org/apache/cassandra/db/lifecycle/LogAwareFileLister.java
index 212076d..254966e 100644
--- a/src/java/org/apache/cassandra/db/lifecycle/LogAwareFileLister.java
+++ b/src/java/org/apache/cassandra/db/lifecycle/LogAwareFileLister.java
@@ -26,7 +26,7 @@
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.*;
-import java.util.function.BiFunction;
+import java.util.function.BiPredicate;
import java.util.stream.Collectors;
import java.util.stream.StreamSupport;
@@ -50,7 +50,7 @@
private final Path folder;
// The filter determines which files the client wants returned
- private final BiFunction<File, FileType, Boolean> filter; //file, file type
+ private final BiPredicate<File, FileType> filter; //file, file type
// The behavior when we fail to list files
private final OnTxnErr onTxnErr;
@@ -59,7 +59,7 @@
NavigableMap<File, Directories.FileType> files = new TreeMap<>();
@VisibleForTesting
- LogAwareFileLister(Path folder, BiFunction<File, FileType, Boolean> filter, OnTxnErr onTxnErr)
+ LogAwareFileLister(Path folder, BiPredicate<File, FileType> filter, OnTxnErr onTxnErr)
{
this.folder = folder;
this.filter = filter;
@@ -96,7 +96,7 @@
// Finally we apply the user filter before returning our result
return files.entrySet().stream()
- .filter((e) -> filter.apply(e.getKey(), e.getValue()))
+ .filter((e) -> filter.test(e.getKey(), e.getValue()))
.map(Map.Entry::getKey)
.collect(Collectors.toList());
}
diff --git a/src/java/org/apache/cassandra/db/lifecycle/LogFile.java b/src/java/org/apache/cassandra/db/lifecycle/LogFile.java
index 0ff7cf1..24b3334 100644
--- a/src/java/org/apache/cassandra/db/lifecycle/LogFile.java
+++ b/src/java/org/apache/cassandra/db/lifecycle/LogFile.java
@@ -66,7 +66,7 @@
private final LogReplicaSet replicas = new LogReplicaSet();
// The transaction records, this set must be ORDER PRESERVING
- private final LinkedHashSet<LogRecord> records = new LinkedHashSet<>();
+ private final Set<LogRecord> records = Collections.synchronizedSet(new LinkedHashSet<>()); // TODO: Hack until we fix CASSANDRA-14554
// The type of the transaction
private final OperationType type;
diff --git a/src/java/org/apache/cassandra/db/lifecycle/LogRecord.java b/src/java/org/apache/cassandra/db/lifecycle/LogRecord.java
index a9b7433..513ad87 100644
--- a/src/java/org/apache/cassandra/db/lifecycle/LogRecord.java
+++ b/src/java/org/apache/cassandra/db/lifecycle/LogRecord.java
@@ -126,7 +126,7 @@
Type type = Type.fromPrefix(matcher.group(1));
return new LogRecord(type,
- matcher.group(2) + Component.separator, // see comment on CASSANDRA-13294 below
+ matcher.group(2),
Long.parseLong(matcher.group(3)),
Integer.parseInt(matcher.group(4)),
Long.parseLong(matcher.group(5)),
@@ -151,10 +151,6 @@
public static LogRecord make(Type type, SSTable table)
{
- // CASSANDRA-13294: add the sstable component separator because for legacy (2.1) files
- // there is no separator after the generation number, and this would cause files of sstables with
- // a higher generation number that starts with the same number, to be incorrectly classified as files
- // of this record sstable
String absoluteTablePath = absolutePath(table.descriptor.baseFilename());
return make(type, getExistingFiles(absoluteTablePath), table.getAllFilePaths().size(), absoluteTablePath);
}
@@ -222,7 +218,7 @@
assert !type.hasFile() || absolutePath != null : "Expected file path for file records";
this.type = type;
- this.absolutePath = type.hasFile() ? Optional.of(absolutePath) : Optional.empty();
+ this.absolutePath = type.hasFile() ? Optional.of(absolutePath) : Optional.<String>empty();
this.updateTime = type == Type.REMOVE ? updateTime : 0;
this.numFiles = type.hasFile() ? numFiles : 0;
this.status = new Status();
@@ -352,25 +348,9 @@
: false;
}
- /**
- * Return the absolute path, if present, except for the last character (the descriptor separator), or
- * the empty string if the record has no path. This method is only to be used internally for writing
- * the record to file or computing the checksum.
- *
- * CASSANDRA-13294: the last character of the absolute path is the descriptor separator, it is removed
- * from the absolute path for backward compatibility, to make sure that on upgrade from 3.0.x to 3.0.y
- * or to 3.y or to 4.0, the checksum of existing txn files still matches (in case of non clean shutdown
- * some txn files may be present). By removing the last character here, it means that
- * it will never be written to txn files, but it is added after reading a txn file in LogFile.make().
- */
- private String absolutePath()
+ String absolutePath()
{
- if (!absolutePath.isPresent())
- return "";
-
- String ret = absolutePath.get();
- assert ret.charAt(ret.length() -1) == Component.separator : "Invalid absolute path, should end with '-'";
- return ret.substring(0, ret.length() - 1);
+ return absolutePath.isPresent() ? absolutePath.get() : "";
}
@Override
diff --git a/src/java/org/apache/cassandra/db/lifecycle/LogReplicaSet.java b/src/java/org/apache/cassandra/db/lifecycle/LogReplicaSet.java
index 67d9dfd..0295357 100644
--- a/src/java/org/apache/cassandra/db/lifecycle/LogReplicaSet.java
+++ b/src/java/org/apache/cassandra/db/lifecycle/LogReplicaSet.java
@@ -19,6 +19,7 @@
import java.io.File;
import java.util.Collection;
+import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@@ -46,7 +47,7 @@
{
private static final Logger logger = LoggerFactory.getLogger(LogReplicaSet.class);
- private final Map<File, LogReplica> replicasByFile = new LinkedHashMap<>();
+ private final Map<File, LogReplica> replicasByFile = Collections.synchronizedMap(new LinkedHashMap<>()); // TODO: Hack until we fix CASSANDRA-14554
private Collection<LogReplica> replicas()
{
diff --git a/src/java/org/apache/cassandra/db/lifecycle/LogTransaction.java b/src/java/org/apache/cassandra/db/lifecycle/LogTransaction.java
index 92f5f4c..4039322 100644
--- a/src/java/org/apache/cassandra/db/lifecycle/LogTransaction.java
+++ b/src/java/org/apache/cassandra/db/lifecycle/LogTransaction.java
@@ -30,12 +30,12 @@
import com.google.common.annotations.VisibleForTesting;
import com.google.common.util.concurrent.Runnables;
+import org.apache.cassandra.service.StorageService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.concurrent.ScheduledExecutors;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.db.Directories;
import org.apache.cassandra.db.SystemKeyspace;
import org.apache.cassandra.db.compaction.OperationType;
@@ -215,7 +215,9 @@
{
try
{
- if (logger.isTraceEnabled())
+ if (!StorageService.instance.isDaemonSetupCompleted())
+ logger.info("Unfinished transaction log, deleting {} ", file);
+ else if (logger.isTraceEnabled())
logger.trace("Deleting {}", file);
Files.delete(file.toPath());
@@ -230,7 +232,7 @@
{
e.printStackTrace(ps);
}
- logger.debug("Unable to delete {} as it does not exist, stack trace:\n {}", file, baos.toString());
+ logger.debug("Unable to delete {} as it does not exist, stack trace:\n {}", file, baos);
}
}
catch (IOException e)
@@ -425,14 +427,14 @@
* for further details on transaction logs.
*
* This method is called on startup and by the standalone sstableutil tool when the cleanup option is specified,
- * @see StandaloneSSTableUtil
+ * @see org.apache.cassandra.tools.StandaloneSSTableUtil
*
* @return true if the leftovers of all transaction logs found were removed, false otherwise.
*
*/
- static boolean removeUnfinishedLeftovers(CFMetaData metadata)
+ static boolean removeUnfinishedLeftovers(TableMetadata metadata)
{
- return removeUnfinishedLeftovers(new Directories(metadata, ColumnFamilyStore.getInitialDirectories()).getCFDirectories());
+ return removeUnfinishedLeftovers(new Directories(metadata).getCFDirectories());
}
@VisibleForTesting
@@ -480,6 +482,7 @@
{
try(LogFile txn = LogFile.make(entry.getKey(), entry.getValue()))
{
+ logger.info("Verifying logfile transaction {}", txn);
if (txn.verify())
{
Throwable failure = txn.removeUnfinishedLeftovers(null);
diff --git a/src/java/org/apache/cassandra/db/lifecycle/Tracker.java b/src/java/org/apache/cassandra/db/lifecycle/Tracker.java
index 39ed353..3ae6eaf 100644
--- a/src/java/org/apache/cassandra/db/lifecycle/Tracker.java
+++ b/src/java/org/apache/cassandra/db/lifecycle/Tracker.java
@@ -38,6 +38,7 @@
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.compaction.OperationType;
import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.metrics.StorageMetrics;
import org.apache.cassandra.notifications.*;
@@ -359,7 +360,7 @@
notifyDiscarded(memtable);
// TODO: if we're invalidated, should we notifyadded AND removed, or just skip both?
- fail = notifyAdded(sstables, fail);
+ fail = notifyAdded(sstables, memtable, fail);
if (!isDummy() && !cfstore.isValid())
dropSSTables();
@@ -417,9 +418,9 @@
return accumulate;
}
- Throwable notifyAdded(Iterable<SSTableReader> added, Throwable accumulate)
+ Throwable notifyAdded(Iterable<SSTableReader> added, Memtable memtable, Throwable accumulate)
{
- INotification notification = new SSTableAddedNotification(added);
+ INotification notification = new SSTableAddedNotification(added, memtable);
for (INotificationConsumer subscriber : subscribers)
{
try
@@ -436,7 +437,7 @@
public void notifyAdded(Iterable<SSTableReader> added)
{
- maybeFail(notifyAdded(added, null));
+ maybeFail(notifyAdded(added, null, null));
}
public void notifySSTableRepairedStatusChanged(Collection<SSTableReader> repairStatusesChanged)
@@ -446,6 +447,14 @@
subscriber.handleNotification(notification, this);
}
+ public void notifySSTableMetadataChanged(SSTableReader levelChanged, StatsMetadata oldMetadata)
+ {
+ INotification notification = new SSTableMetadataChanged(levelChanged, oldMetadata);
+ for (INotificationConsumer subscriber : subscribers)
+ subscriber.handleNotification(notification, this);
+
+ }
+
public void notifyDeleting(SSTableReader deleting)
{
INotification notification = new SSTableDeletingNotification(deleting);
diff --git a/src/java/org/apache/cassandra/db/marshal/AbstractCompositeType.java b/src/java/org/apache/cassandra/db/marshal/AbstractCompositeType.java
index 9eb5d82..0248629 100644
--- a/src/java/org/apache/cassandra/db/marshal/AbstractCompositeType.java
+++ b/src/java/org/apache/cassandra/db/marshal/AbstractCompositeType.java
@@ -297,12 +297,6 @@
return BytesSerializer.instance;
}
- @Override
- public boolean referencesUserType(String name)
- {
- return getComponents().stream().anyMatch(f -> f.referencesUserType(name));
- }
-
/**
* @return the comparator for the given component. static CompositeType will consult
* @param i DynamicCompositeType will read the type information from @param bb
diff --git a/src/java/org/apache/cassandra/db/marshal/AbstractType.java b/src/java/org/apache/cassandra/db/marshal/AbstractType.java
index 28851a4..b65a1c1 100644
--- a/src/java/org/apache/cassandra/db/marshal/AbstractType.java
+++ b/src/java/org/apache/cassandra/db/marshal/AbstractType.java
@@ -20,12 +20,7 @@
import java.io.IOException;
import java.lang.reflect.Method;
import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -36,15 +31,14 @@
import org.apache.cassandra.cql3.Term;
import org.apache.cassandra.db.TypeSizes;
import org.apache.cassandra.exceptions.SyntaxException;
-import org.apache.cassandra.serializers.TypeSerializer;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.serializers.MarshalException;
-
+import org.apache.cassandra.serializers.TypeSerializer;
import org.apache.cassandra.transport.ProtocolVersion;
+import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FastByteOperations;
import org.github.jamm.Unmetered;
-import org.apache.cassandra.io.util.DataOutputPlus;
-import org.apache.cassandra.io.util.DataInputPlus;
-import org.apache.cassandra.utils.ByteBufferUtil;
import static org.apache.cassandra.db.marshal.AbstractType.ComparisonType.CUSTOM;
@@ -348,6 +342,11 @@
return this;
}
+ public List<AbstractType<?>> subTypes()
+ {
+ return Collections.emptyList();
+ }
+
/**
* Returns an AbstractType instance that is equivalent to this one, but with all nested UDTs and collections
* explicitly frozen.
@@ -398,28 +397,16 @@
/**
* The length of values for this type if all values are of fixed length, -1 otherwise.
*/
- protected int valueLengthIfFixed()
+ public int valueLengthIfFixed()
{
return -1;
}
- public void validateIfFixedSize(ByteBuffer value)
- {
- if (valueLengthIfFixed() < 0)
- return;
-
- validate(value);
- }
-
// This assumes that no empty values are passed
public void writeValue(ByteBuffer value, DataOutputPlus out) throws IOException
{
assert value.hasRemaining();
- int valueLengthIfFixed = valueLengthIfFixed();
- assert valueLengthIfFixed < 0 || value.remaining() == valueLengthIfFixed : String.format("Expected exactly %d bytes, but was %d",
- valueLengthIfFixed, value.remaining());
-
- if (valueLengthIfFixed >= 0)
+ if (valueLengthIfFixed() >= 0)
out.write(value);
else
ByteBufferUtil.writeWithVIntLength(value, out);
@@ -428,11 +415,7 @@
public long writtenLength(ByteBuffer value)
{
assert value.hasRemaining();
- int valueLengthIfFixed = valueLengthIfFixed();
- assert valueLengthIfFixed < 0 || value.remaining() == valueLengthIfFixed : String.format("Expected exactly %d bytes, but was %d",
- valueLengthIfFixed, value.remaining());
-
- return valueLengthIfFixed >= 0
+ return valueLengthIfFixed() >= 0
? value.remaining()
: TypeSizes.sizeofWithVIntLength(value);
}
@@ -472,17 +455,60 @@
ByteBufferUtil.skipWithVIntLength(in);
}
- public boolean referencesUserType(String userTypeName)
+ public boolean referencesUserType(ByteBuffer name)
{
return false;
}
+ /**
+ * Returns an instance of this type with all references to the provided user type recursively replaced with its new
+ * definition.
+ */
+ public AbstractType<?> withUpdatedUserType(UserType udt)
+ {
+ return this;
+ }
+
+ /**
+ * Replace any instances of UserType with equivalent TupleType-s.
+ *
+ * We need it for dropped_columns, to allow safely dropping unused user types later without retaining any references
+ * to them in system_schema.dropped_columns.
+ */
+ public AbstractType<?> expandUserTypes()
+ {
+ return this;
+ }
+
public boolean referencesDuration()
{
return false;
}
/**
+ * Tests whether a CQL value having this type can be assigned to the provided receiver.
+ */
+ public AssignmentTestable.TestResult testAssignment(AbstractType<?> receiverType)
+ {
+ // testAssignement is for CQL literals and native protocol values, none of which make a meaningful
+ // difference between frozen or not and reversed or not.
+
+ if (isFreezable() && !isMultiCell())
+ receiverType = receiverType.freeze();
+
+ if (isReversed() && !receiverType.isReversed())
+ receiverType = ReversedType.getInstance(receiverType);
+
+ if (equals(receiverType))
+ return AssignmentTestable.TestResult.EXACT_MATCH;
+
+ if (receiverType.isValueCompatibleWith(this))
+ return AssignmentTestable.TestResult.WEAKLY_ASSIGNABLE;
+
+ return AssignmentTestable.TestResult.NOT_ASSIGNABLE;
+ }
+
+ /**
* This must be overriden by subclasses if necessary so that for any
* AbstractType, this == TypeParser.parse(toString()).
*
@@ -495,17 +521,6 @@
return getClass().getName();
}
- /**
- * Checks to see if two types are equal when ignoring or not ignoring differences in being frozen, depending on
- * the value of the ignoreFreezing parameter.
- * @param other type to compare
- * @param ignoreFreezing if true, differences in the types being frozen will be ignored
- */
- public boolean equals(Object other, boolean ignoreFreezing)
- {
- return this.equals(other);
- }
-
public void checkComparable()
{
switch (comparisonType)
@@ -517,21 +532,6 @@
public final AssignmentTestable.TestResult testAssignment(String keyspace, ColumnSpecification receiver)
{
- // We should ignore the fact that the output type is frozen in our comparison as functions do not support
- // frozen types for arguments
- AbstractType<?> receiverType = receiver.type;
- if (isFreezable() && !isMultiCell())
- receiverType = receiverType.freeze();
-
- if (isReversed())
- receiverType = ReversedType.getInstance(receiverType);
-
- if (equals(receiverType))
- return AssignmentTestable.TestResult.EXACT_MATCH;
-
- if (receiverType.isValueCompatibleWith(this))
- return AssignmentTestable.TestResult.WEAKLY_ASSIGNABLE;
-
- return AssignmentTestable.TestResult.NOT_ASSIGNABLE;
+ return testAssignment(receiver.type);
}
}
diff --git a/src/java/org/apache/cassandra/db/marshal/AsciiType.java b/src/java/org/apache/cassandra/db/marshal/AsciiType.java
index 3cd45de..05077ee 100644
--- a/src/java/org/apache/cassandra/db/marshal/AsciiType.java
+++ b/src/java/org/apache/cassandra/db/marshal/AsciiType.java
@@ -19,9 +19,9 @@
import java.nio.ByteBuffer;
import java.nio.CharBuffer;
-import java.nio.charset.Charset;
import java.nio.charset.CharsetEncoder;
import java.nio.charset.CharacterCodingException;
+import java.nio.charset.StandardCharsets;
import io.netty.util.concurrent.FastThreadLocal;
import org.apache.cassandra.cql3.Constants;
@@ -46,7 +46,7 @@
@Override
protected CharsetEncoder initialValue()
{
- return Charset.forName("US-ASCII").newEncoder();
+ return StandardCharsets.US_ASCII.newEncoder();
}
};
@@ -85,7 +85,7 @@
{
try
{
- return '"' + Json.quoteAsJsonString(ByteBufferUtil.string(buffer, Charset.forName("US-ASCII"))) + '"';
+ return '"' + Json.quoteAsJsonString(ByteBufferUtil.string(buffer, StandardCharsets.US_ASCII)) + '"';
}
catch (CharacterCodingException exc)
{
diff --git a/src/java/org/apache/cassandra/db/marshal/BooleanType.java b/src/java/org/apache/cassandra/db/marshal/BooleanType.java
index 1dbd1af..475cae6 100644
--- a/src/java/org/apache/cassandra/db/marshal/BooleanType.java
+++ b/src/java/org/apache/cassandra/db/marshal/BooleanType.java
@@ -97,7 +97,7 @@
}
@Override
- protected int valueLengthIfFixed()
+ public int valueLengthIfFixed()
{
return 1;
}
diff --git a/src/java/org/apache/cassandra/db/marshal/ByteType.java b/src/java/org/apache/cassandra/db/marshal/ByteType.java
index 55aea8f..c19fdd9 100644
--- a/src/java/org/apache/cassandra/db/marshal/ByteType.java
+++ b/src/java/org/apache/cassandra/db/marshal/ByteType.java
@@ -28,7 +28,7 @@
import org.apache.cassandra.transport.ProtocolVersion;
import org.apache.cassandra.utils.ByteBufferUtil;
-public class ByteType extends AbstractType<Byte>
+public class ByteType extends NumberType<Byte>
{
public static final ByteType instance = new ByteType();
@@ -88,4 +88,52 @@
{
return ByteSerializer.instance;
}
+
+ @Override
+ public byte toByte(ByteBuffer value)
+ {
+ return ByteBufferUtil.toByte(value);
+ }
+
+ @Override
+ public short toShort(ByteBuffer value)
+ {
+ return toByte(value);
+ }
+
+ @Override
+ protected int toInt(ByteBuffer value)
+ {
+ return toByte(value);
+ }
+
+ public ByteBuffer add(NumberType<?> leftType, ByteBuffer left, NumberType<?> rightType, ByteBuffer right)
+ {
+ return ByteBufferUtil.bytes((byte) (leftType.toByte(left) + rightType.toByte(right)));
+ }
+
+ public ByteBuffer substract(NumberType<?> leftType, ByteBuffer left, NumberType<?> rightType, ByteBuffer right)
+ {
+ return ByteBufferUtil.bytes((byte) (leftType.toByte(left) - rightType.toByte(right)));
+ }
+
+ public ByteBuffer multiply(NumberType<?> leftType, ByteBuffer left, NumberType<?> rightType, ByteBuffer right)
+ {
+ return ByteBufferUtil.bytes((byte) (leftType.toByte(left) * rightType.toByte(right)));
+ }
+
+ public ByteBuffer divide(NumberType<?> leftType, ByteBuffer left, NumberType<?> rightType, ByteBuffer right)
+ {
+ return ByteBufferUtil.bytes((byte) (leftType.toByte(left) / rightType.toByte(right)));
+ }
+
+ public ByteBuffer mod(NumberType<?> leftType, ByteBuffer left, NumberType<?> rightType, ByteBuffer right)
+ {
+ return ByteBufferUtil.bytes((byte) (leftType.toByte(left) % rightType.toByte(right)));
+ }
+
+ public ByteBuffer negate(ByteBuffer input)
+ {
+ return ByteBufferUtil.bytes((byte) -toByte(input));
+ }
}
diff --git a/src/java/org/apache/cassandra/db/marshal/CollectionType.java b/src/java/org/apache/cassandra/db/marshal/CollectionType.java
index 6e7d5d7..b198e0c 100644
--- a/src/java/org/apache/cassandra/db/marshal/CollectionType.java
+++ b/src/java/org/apache/cassandra/db/marshal/CollectionType.java
@@ -39,7 +39,8 @@
/**
* The abstract validator that is the base for maps, sets and lists (both frozen and non-frozen).
*
- * Please note that this comparator shouldn't be used "manually" (through thrift for instance).
+ * Please note that this comparator shouldn't be used "manually" (as a custom
+ * type for instance).
*/
public abstract class CollectionType<T> extends AbstractType<T>
{
@@ -210,7 +211,7 @@
}
@Override
- public boolean equals(Object o, boolean ignoreFreezing)
+ public boolean equals(Object o)
{
if (this == o)
return true;
@@ -223,11 +224,10 @@
if (kind != other.kind)
return false;
- if (!ignoreFreezing && isMultiCell() != other.isMultiCell())
+ if (isMultiCell() != other.isMultiCell())
return false;
- return nameComparator().equals(other.nameComparator(), ignoreFreezing) &&
- valueComparator().equals(other.valueComparator(), ignoreFreezing);
+ return nameComparator().equals(other.nameComparator()) && valueComparator().equals(other.valueComparator());
}
@Override
diff --git a/src/java/org/apache/cassandra/db/marshal/ColumnToCollectionType.java b/src/java/org/apache/cassandra/db/marshal/ColumnToCollectionType.java
deleted file mode 100644
index de50446..0000000
--- a/src/java/org/apache/cassandra/db/marshal/ColumnToCollectionType.java
+++ /dev/null
@@ -1,154 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.db.marshal;
-
-import java.nio.ByteBuffer;
-import java.util.HashMap;
-import java.util.Map;
-
-import com.google.common.collect.ImmutableMap;
-
-import org.apache.cassandra.cql3.Term;
-import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.exceptions.SyntaxException;
-import org.apache.cassandra.serializers.TypeSerializer;
-import org.apache.cassandra.serializers.BytesSerializer;
-import org.apache.cassandra.serializers.MarshalException;
-import org.apache.cassandra.transport.ProtocolVersion;
-import org.apache.cassandra.utils.ByteBufferUtil;
-
-/*
- * This class is deprecated and only kept for backward compatibility.
- */
-public class ColumnToCollectionType extends AbstractType<ByteBuffer>
-{
- // interning instances
- private static final Map<Map<ByteBuffer, CollectionType>, ColumnToCollectionType> instances = new HashMap<>();
-
- public final Map<ByteBuffer, CollectionType> defined;
-
- public static ColumnToCollectionType getInstance(TypeParser parser) throws SyntaxException, ConfigurationException
- {
- return getInstance(parser.getCollectionsParameters());
- }
-
- public static synchronized ColumnToCollectionType getInstance(Map<ByteBuffer, CollectionType> defined)
- {
- assert defined != null;
-
- ColumnToCollectionType t = instances.get(defined);
- if (t == null)
- {
- t = new ColumnToCollectionType(defined);
- instances.put(defined, t);
- }
- return t;
- }
-
- private ColumnToCollectionType(Map<ByteBuffer, CollectionType> defined)
- {
- super(ComparisonType.CUSTOM);
- this.defined = ImmutableMap.copyOf(defined);
- }
-
- public int compareCustom(ByteBuffer o1, ByteBuffer o2)
- {
- throw new UnsupportedOperationException("ColumnToCollectionType should only be used in composite types, never alone");
- }
-
- public int compareCollectionMembers(ByteBuffer o1, ByteBuffer o2, ByteBuffer collectionName)
- {
- CollectionType t = defined.get(collectionName);
- if (t == null)
- throw new RuntimeException(ByteBufferUtil.bytesToHex(collectionName) + " is not defined as a collection");
-
- return t.nameComparator().compare(o1, o2);
- }
-
- public String getString(ByteBuffer bytes)
- {
- return BytesType.instance.getString(bytes);
- }
-
- public ByteBuffer fromString(String source)
- {
- try
- {
- return ByteBufferUtil.hexToBytes(source);
- }
- catch (NumberFormatException e)
- {
- throw new MarshalException(String.format("cannot parse '%s' as hex bytes", source), e);
- }
- }
-
- @Override
- public Term fromJSONObject(Object parsed) throws MarshalException
- {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public String toJSONString(ByteBuffer buffer, ProtocolVersion protocolVersion)
- {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public void validate(ByteBuffer bytes)
- {
- throw new UnsupportedOperationException("ColumnToCollectionType should only be used in composite types, never alone");
- }
-
- public TypeSerializer<ByteBuffer> getSerializer()
- {
- return BytesSerializer.instance;
- }
-
- public void validateCollectionMember(ByteBuffer bytes, ByteBuffer collectionName) throws MarshalException
- {
- CollectionType t = defined.get(collectionName);
- if (t == null)
- throw new MarshalException(ByteBufferUtil.bytesToHex(collectionName) + " is not defined as a collection");
-
- t.nameComparator().validate(bytes);
- }
-
- @Override
- public boolean isCompatibleWith(AbstractType<?> previous)
- {
- if (!(previous instanceof ColumnToCollectionType))
- return false;
-
- ColumnToCollectionType prev = (ColumnToCollectionType)previous;
- // We are compatible if we have all the definitions previous have (but we can have more).
- for (Map.Entry<ByteBuffer, CollectionType> entry : prev.defined.entrySet())
- {
- CollectionType newType = defined.get(entry.getKey());
- if (newType == null || !newType.isCompatibleWith(entry.getValue()))
- return false;
- }
- return true;
- }
-
- @Override
- public String toString()
- {
- return getClass().getName() + TypeParser.stringifyCollectionsParameters(defined);
- }
-}
diff --git a/src/java/org/apache/cassandra/db/marshal/CompositeType.java b/src/java/org/apache/cassandra/db/marshal/CompositeType.java
index 6358d71..e3423ff 100644
--- a/src/java/org/apache/cassandra/db/marshal/CompositeType.java
+++ b/src/java/org/apache/cassandra/db/marshal/CompositeType.java
@@ -17,7 +17,6 @@
*/
package org.apache.cassandra.db.marshal;
-import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Arrays;
@@ -26,16 +25,16 @@
import java.util.concurrent.ConcurrentMap;
import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
-import org.apache.cassandra.cql3.ColumnIdentifier;
-import org.apache.cassandra.cql3.Operator;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.exceptions.SyntaxException;
-import org.apache.cassandra.io.util.DataOutputBuffer;
-import org.apache.cassandra.io.util.DataOutputBufferFixed;
import org.apache.cassandra.serializers.MarshalException;
import org.apache.cassandra.utils.ByteBufferUtil;
+import static com.google.common.collect.Iterables.any;
+import static com.google.common.collect.Iterables.transform;
+
/*
* The encoding of a CompositeType column name should be:
* <component><component><component> ...
@@ -63,21 +62,26 @@
*/
public class CompositeType extends AbstractCompositeType
{
- public static final int STATIC_MARKER = 0xFFFF;
+ private static final int STATIC_MARKER = 0xFFFF;
public final List<AbstractType<?>> types;
// interning instances
- private static final ConcurrentMap<List<AbstractType<?>>, CompositeType> instances = new ConcurrentHashMap<List<AbstractType<?>>, CompositeType>();
+ private static final ConcurrentMap<List<AbstractType<?>>, CompositeType> instances = new ConcurrentHashMap<>();
public static CompositeType getInstance(TypeParser parser) throws ConfigurationException, SyntaxException
{
return getInstance(parser.getTypeParameters());
}
+ public static CompositeType getInstance(Iterable<AbstractType<?>> types)
+ {
+ return getInstance(Lists.newArrayList(types));
+ }
+
public static CompositeType getInstance(AbstractType... types)
{
- return getInstance(Arrays.<AbstractType<?>>asList(types));
+ return getInstance(Arrays.asList(types));
}
protected boolean readIsStatic(ByteBuffer bb)
@@ -101,18 +105,10 @@
public static CompositeType getInstance(List<AbstractType<?>> types)
{
assert types != null && !types.isEmpty();
-
- CompositeType ct = instances.get(types);
- if (ct == null)
- {
- ct = new CompositeType(types);
- CompositeType previous = instances.putIfAbsent(types, ct);
- if (previous != null)
- {
- ct = previous;
- }
- }
- return ct;
+ CompositeType t = instances.get(types);
+ return null == t
+ ? instances.computeIfAbsent(types, CompositeType::new)
+ : t;
}
protected CompositeType(List<AbstractType<?>> types)
@@ -128,9 +124,8 @@
}
catch (IndexOutOfBoundsException e)
{
- // We shouldn't get there in general because 1) we shouldn't construct broken composites
- // from CQL and 2) broken composites coming from thrift should be rejected by validate.
- // There is a few cases however where, if the schema has changed since we created/validated
+ // We shouldn't get there in general we shouldn't construct broken composites
+ // but there is a few cases where if the schema has changed since we created/validated
// the composite, this will be thrown (see #6262). Those cases are a user error but
// throwing a more meaningful error message to make understanding such error easier. .
throw new RuntimeException("Cannot get comparator " + i + " in " + this + ". "
@@ -204,11 +199,6 @@
return l;
}
- public static byte lastEOC(ByteBuffer name)
- {
- return name.get(name.limit() - 1);
- }
-
// Extract component idx from bb. Return null if there is not enough component.
public static ByteBuffer extractComponent(ByteBuffer bb, int idx)
{
@@ -227,13 +217,6 @@
return null;
}
- // Extract CQL3 column name from the full column name.
- public ByteBuffer extractLastComponent(ByteBuffer bb)
- {
- int idx = types.get(types.size() - 1) instanceof ColumnToCollectionType ? types.size() - 2 : types.size() - 1;
- return extractComponent(bb, idx);
- }
-
public static boolean isStaticName(ByteBuffer bb)
{
return bb.remaining() >= 2 && (ByteBufferUtil.getShortLength(bb, bb.position()) & 0xFFFF) == STATIC_MARKER;
@@ -299,6 +282,29 @@
return true;
}
+ @Override
+ public boolean referencesUserType(ByteBuffer name)
+ {
+ return any(types, t -> t.referencesUserType(name));
+ }
+
+ @Override
+ public CompositeType withUpdatedUserType(UserType udt)
+ {
+ if (!referencesUserType(udt.name))
+ return this;
+
+ instances.remove(types);
+
+ return getInstance(transform(types, t -> t.withUpdatedUserType(udt)));
+ }
+
+ @Override
+ public AbstractType<?> expandUserTypes()
+ {
+ return getInstance(transform(types, AbstractType::expandUserTypes));
+ }
+
private static class StaticParsedComparator implements ParsedComparator
{
final AbstractType<?> type;
@@ -334,16 +340,6 @@
return getClass().getName() + TypeParser.stringifyTypeParameters(types);
}
- public Builder builder()
- {
- return new Builder(this);
- }
-
- public Builder builder(boolean isStatic)
- {
- return new Builder(this, isStatic);
- }
-
public static ByteBuffer build(ByteBuffer... buffers)
{
return build(false, buffers);
@@ -364,150 +360,11 @@
{
ByteBufferUtil.writeShortLength(out, bb.remaining());
int toCopy = bb.remaining();
- ByteBufferUtil.arrayCopy(bb, bb.position(), out, out.position(), toCopy);
+ ByteBufferUtil.copyBytes(bb, bb.position(), out, out.position(), toCopy);
out.position(out.position() + toCopy);
out.put((byte) 0);
}
out.flip();
return out;
}
-
- public static class Builder
- {
- private final CompositeType composite;
-
- private final List<ByteBuffer> components;
- private final byte[] endOfComponents;
- private int serializedSize;
- private final boolean isStatic;
-
- public Builder(CompositeType composite)
- {
- this(composite, false);
- }
-
- public Builder(CompositeType composite, boolean isStatic)
- {
- this(composite, new ArrayList<>(composite.types.size()), new byte[composite.types.size()], isStatic);
- }
-
- private Builder(CompositeType composite, List<ByteBuffer> components, byte[] endOfComponents, boolean isStatic)
- {
- assert endOfComponents.length == composite.types.size();
-
- this.composite = composite;
- this.components = components;
- this.endOfComponents = endOfComponents;
- this.isStatic = isStatic;
- if (isStatic)
- serializedSize = 2;
- }
-
- private Builder(Builder b)
- {
- this(b.composite, new ArrayList<>(b.components), Arrays.copyOf(b.endOfComponents, b.endOfComponents.length), b.isStatic);
- this.serializedSize = b.serializedSize;
- }
-
- public Builder add(ByteBuffer bb)
- {
- if (components.size() >= composite.types.size())
- throw new IllegalStateException("Composite column is already fully constructed");
-
- components.add(bb);
- serializedSize += 3 + bb.remaining(); // 2 bytes lenght + 1 byte eoc
- return this;
- }
-
- public Builder add(ColumnIdentifier name)
- {
- return add(name.bytes);
- }
-
- public int componentCount()
- {
- return components.size();
- }
-
- public int remainingCount()
- {
- return composite.types.size() - components.size();
- }
-
- public ByteBuffer get(int i)
- {
- return components.get(i);
- }
-
- public ByteBuffer build()
- {
- try (DataOutputBuffer out = new DataOutputBufferFixed(serializedSize))
- {
- if (isStatic)
- out.writeShort(STATIC_MARKER);
-
- for (int i = 0; i < components.size(); i++)
- {
- ByteBufferUtil.writeWithShortLength(components.get(i), out);
- out.write(endOfComponents[i]);
- }
- return ByteBuffer.wrap(out.getData(), 0, out.getLength());
- }
- catch (IOException e)
- {
- throw new RuntimeException(e);
- }
- }
-
- public ByteBuffer buildAsEndOfRange()
- {
- if (components.isEmpty())
- return ByteBufferUtil.EMPTY_BYTE_BUFFER;
-
- ByteBuffer bb = build();
- bb.put(bb.remaining() - 1, (byte)1);
- return bb;
- }
-
- public ByteBuffer buildForRelation(Operator op)
- {
- /*
- * Given the rules for eoc (end-of-component, see AbstractCompositeType.compare()),
- * We can select:
- * - = 'a' by using <'a'><0>
- * - < 'a' by using <'a'><-1>
- * - <= 'a' by using <'a'><1>
- * - > 'a' by using <'a'><1>
- * - >= 'a' by using <'a'><0>
- */
- int current = components.size() - 1;
- switch (op)
- {
- case LT:
- endOfComponents[current] = (byte) -1;
- break;
- case GT:
- case LTE:
- endOfComponents[current] = (byte) 1;
- break;
- default:
- endOfComponents[current] = (byte) 0;
- break;
- }
- return build();
- }
-
- public Builder copy()
- {
- return new Builder(this);
- }
-
- public ByteBuffer getComponent(int i)
- {
- if (i >= components.size())
- throw new IllegalArgumentException();
-
- return components.get(i);
- }
- }
}
diff --git a/src/java/org/apache/cassandra/db/marshal/CounterColumnType.java b/src/java/org/apache/cassandra/db/marshal/CounterColumnType.java
index 8bb1a25..8777e0e 100644
--- a/src/java/org/apache/cassandra/db/marshal/CounterColumnType.java
+++ b/src/java/org/apache/cassandra/db/marshal/CounterColumnType.java
@@ -22,13 +22,13 @@
import org.apache.cassandra.cql3.CQL3Type;
import org.apache.cassandra.cql3.Term;
import org.apache.cassandra.db.context.CounterContext;
-import org.apache.cassandra.serializers.TypeSerializer;
import org.apache.cassandra.serializers.CounterSerializer;
import org.apache.cassandra.serializers.MarshalException;
import org.apache.cassandra.transport.ProtocolVersion;
+import org.apache.cassandra.serializers.TypeSerializer;
import org.apache.cassandra.utils.ByteBufferUtil;
-public class CounterColumnType extends AbstractType<Long>
+public class CounterColumnType extends NumberType<Long>
{
public static final CounterColumnType instance = new CounterColumnType();
@@ -93,4 +93,40 @@
{
return CounterSerializer.instance;
}
+
+ @Override
+ protected long toLong(ByteBuffer value)
+ {
+ return ByteBufferUtil.toLong(value);
+ }
+
+ public ByteBuffer add(NumberType<?> leftType, ByteBuffer left, NumberType<?> rightType, ByteBuffer right)
+ {
+ return ByteBufferUtil.bytes(leftType.toLong(left) + rightType.toLong(right));
+ }
+
+ public ByteBuffer substract(NumberType<?> leftType, ByteBuffer left, NumberType<?> rightType, ByteBuffer right)
+ {
+ return ByteBufferUtil.bytes(leftType.toLong(left) - rightType.toLong(right));
+ }
+
+ public ByteBuffer multiply(NumberType<?> leftType, ByteBuffer left, NumberType<?> rightType, ByteBuffer right)
+ {
+ return ByteBufferUtil.bytes(leftType.toLong(left) * rightType.toLong(right));
+ }
+
+ public ByteBuffer divide(NumberType<?> leftType, ByteBuffer left, NumberType<?> rightType, ByteBuffer right)
+ {
+ return ByteBufferUtil.bytes(leftType.toLong(left) / rightType.toLong(right));
+ }
+
+ public ByteBuffer mod(NumberType<?> leftType, ByteBuffer left, NumberType<?> rightType, ByteBuffer right)
+ {
+ return ByteBufferUtil.bytes(leftType.toLong(left) % rightType.toLong(right));
+ }
+
+ public ByteBuffer negate(ByteBuffer input)
+ {
+ return ByteBufferUtil.bytes(-toLong(input));
+ }
}
diff --git a/src/java/org/apache/cassandra/db/marshal/DateType.java b/src/java/org/apache/cassandra/db/marshal/DateType.java
index 87b2cad..473cedf 100644
--- a/src/java/org/apache/cassandra/db/marshal/DateType.java
+++ b/src/java/org/apache/cassandra/db/marshal/DateType.java
@@ -119,7 +119,7 @@
}
@Override
- protected int valueLengthIfFixed()
+ public int valueLengthIfFixed()
{
return 8;
}
diff --git a/src/java/org/apache/cassandra/db/marshal/DecimalType.java b/src/java/org/apache/cassandra/db/marshal/DecimalType.java
index f1586e0..110dc0e 100644
--- a/src/java/org/apache/cassandra/db/marshal/DecimalType.java
+++ b/src/java/org/apache/cassandra/db/marshal/DecimalType.java
@@ -18,6 +18,9 @@
package org.apache.cassandra.db.marshal;
import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.math.MathContext;
+import java.math.RoundingMode;
import java.nio.ByteBuffer;
import org.apache.cassandra.cql3.CQL3Type;
@@ -29,9 +32,13 @@
import org.apache.cassandra.transport.ProtocolVersion;
import org.apache.cassandra.utils.ByteBufferUtil;
-public class DecimalType extends AbstractType<BigDecimal>
+public class DecimalType extends NumberType<BigDecimal>
{
public static final DecimalType instance = new DecimalType();
+ private static final int MIN_SCALE = 32;
+ private static final int MIN_SIGNIFICANT_DIGITS = MIN_SCALE;
+ private static final int MAX_SCALE = 1000;
+ private static final MathContext MAX_PRECISION = new MathContext(10000);
DecimalType() {super(ComparisonType.CUSTOM);} // singleton
@@ -40,6 +47,12 @@
return true;
}
+ @Override
+ public boolean isFloatingPoint()
+ {
+ return true;
+ }
+
public int compareCustom(ByteBuffer o1, ByteBuffer o2)
{
if (!o1.hasRemaining() || !o2.hasRemaining())
@@ -95,4 +108,84 @@
{
return DecimalSerializer.instance;
}
+
+ @Override
+ protected int toInt(ByteBuffer value)
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ protected float toFloat(ByteBuffer value)
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ protected long toLong(ByteBuffer value)
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ protected double toDouble(ByteBuffer value)
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ protected BigInteger toBigInteger(ByteBuffer value)
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ protected BigDecimal toBigDecimal(ByteBuffer value)
+ {
+ return compose(value);
+ }
+
+ public ByteBuffer add(NumberType<?> leftType, ByteBuffer left, NumberType<?> rightType, ByteBuffer right)
+ {
+ return decompose(leftType.toBigDecimal(left).add(rightType.toBigDecimal(right), MAX_PRECISION));
+ }
+
+ public ByteBuffer substract(NumberType<?> leftType, ByteBuffer left, NumberType<?> rightType, ByteBuffer right)
+ {
+ return decompose(leftType.toBigDecimal(left).subtract(rightType.toBigDecimal(right), MAX_PRECISION));
+ }
+
+ public ByteBuffer multiply(NumberType<?> leftType, ByteBuffer left, NumberType<?> rightType, ByteBuffer right)
+ {
+ return decompose(leftType.toBigDecimal(left).multiply(rightType.toBigDecimal(right), MAX_PRECISION));
+ }
+
+ public ByteBuffer divide(NumberType<?> leftType, ByteBuffer left, NumberType<?> rightType, ByteBuffer right)
+ {
+ BigDecimal leftOperand = leftType.toBigDecimal(left);
+ BigDecimal rightOperand = rightType.toBigDecimal(right);
+
+ // Predict position of first significant digit in the quotient.
+ // Note: it is possible to improve prediction accuracy by comparing first significant digits in operands
+ // but it requires additional computations so this step is omitted
+ int quotientFirstDigitPos = (leftOperand.precision() - leftOperand.scale()) - (rightOperand.precision() - rightOperand.scale());
+
+ int scale = MIN_SIGNIFICANT_DIGITS - quotientFirstDigitPos;
+ scale = Math.max(scale, leftOperand.scale());
+ scale = Math.max(scale, rightOperand.scale());
+ scale = Math.max(scale, MIN_SCALE);
+ scale = Math.min(scale, MAX_SCALE);
+
+ return decompose(leftOperand.divide(rightOperand, scale, RoundingMode.HALF_UP).stripTrailingZeros());
+ }
+
+ public ByteBuffer mod(NumberType<?> leftType, ByteBuffer left, NumberType<?> rightType, ByteBuffer right)
+ {
+ return decompose(leftType.toBigDecimal(left).remainder(rightType.toBigDecimal(right)));
+ }
+
+ public ByteBuffer negate(ByteBuffer input)
+ {
+ return decompose(toBigDecimal(input).negate());
+ }
}
diff --git a/src/java/org/apache/cassandra/db/marshal/DoubleType.java b/src/java/org/apache/cassandra/db/marshal/DoubleType.java
index 4b997dd..bcc4440 100644
--- a/src/java/org/apache/cassandra/db/marshal/DoubleType.java
+++ b/src/java/org/apache/cassandra/db/marshal/DoubleType.java
@@ -28,7 +28,7 @@
import org.apache.cassandra.transport.ProtocolVersion;
import org.apache.cassandra.utils.ByteBufferUtil;
-public class DoubleType extends AbstractType<Double>
+public class DoubleType extends NumberType<Double>
{
public static final DoubleType instance = new DoubleType();
@@ -39,6 +39,12 @@
return true;
}
+ @Override
+ public boolean isFloatingPoint()
+ {
+ return true;
+ }
+
public int compareCustom(ByteBuffer o1, ByteBuffer o2)
{
if (!o1.hasRemaining() || !o2.hasRemaining())
@@ -53,17 +59,14 @@
if (source.isEmpty())
return ByteBufferUtil.EMPTY_BYTE_BUFFER;
- Double d;
try
{
- d = Double.valueOf(source);
+ return decompose(Double.valueOf(source));
}
catch (NumberFormatException e1)
{
throw new MarshalException(String.format("Unable to make double from '%s'", source), e1);
}
-
- return decompose(d);
}
@Override
@@ -104,8 +107,62 @@
}
@Override
- protected int valueLengthIfFixed()
+ public int valueLengthIfFixed()
{
return 8;
}
+
+ @Override
+ protected int toInt(ByteBuffer value)
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ protected float toFloat(ByteBuffer value)
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ protected long toLong(ByteBuffer value)
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ protected double toDouble(ByteBuffer value)
+ {
+ return ByteBufferUtil.toDouble(value);
+ }
+
+ public ByteBuffer add(NumberType<?> leftType, ByteBuffer left, NumberType<?> rightType, ByteBuffer right)
+ {
+ return ByteBufferUtil.bytes(leftType.toDouble(left) + rightType.toDouble(right));
+ }
+
+ public ByteBuffer substract(NumberType<?> leftType, ByteBuffer left, NumberType<?> rightType, ByteBuffer right)
+ {
+ return ByteBufferUtil.bytes(leftType.toDouble(left) - rightType.toDouble(right));
+ }
+
+ public ByteBuffer multiply(NumberType<?> leftType, ByteBuffer left, NumberType<?> rightType, ByteBuffer right)
+ {
+ return ByteBufferUtil.bytes(leftType.toDouble(left) * rightType.toDouble(right));
+ }
+
+ public ByteBuffer divide(NumberType<?> leftType, ByteBuffer left, NumberType<?> rightType, ByteBuffer right)
+ {
+ return ByteBufferUtil.bytes(leftType.toDouble(left) / rightType.toDouble(right));
+ }
+
+ public ByteBuffer mod(NumberType<?> leftType, ByteBuffer left, NumberType<?> rightType, ByteBuffer right)
+ {
+ return ByteBufferUtil.bytes(leftType.toDouble(left) % rightType.toDouble(right));
+ }
+
+ public ByteBuffer negate(ByteBuffer input)
+ {
+ return ByteBufferUtil.bytes(-toDouble(input));
+ }
}
diff --git a/src/java/org/apache/cassandra/db/marshal/DynamicCompositeType.java b/src/java/org/apache/cassandra/db/marshal/DynamicCompositeType.java
index d314bd9..0458dc8 100644
--- a/src/java/org/apache/cassandra/db/marshal/DynamicCompositeType.java
+++ b/src/java/org/apache/cassandra/db/marshal/DynamicCompositeType.java
@@ -17,22 +17,25 @@
*/
package org.apache.cassandra.db.marshal;
-import java.nio.charset.CharacterCodingException;
import java.nio.ByteBuffer;
-import java.util.HashMap;
+import java.nio.charset.CharacterCodingException;
import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
-import org.apache.cassandra.cql3.Term;
+import com.google.common.collect.Maps;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.cassandra.cql3.Term;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.exceptions.SyntaxException;
-import org.apache.cassandra.serializers.TypeSerializer;
import org.apache.cassandra.serializers.MarshalException;
+import org.apache.cassandra.serializers.TypeSerializer;
import org.apache.cassandra.transport.ProtocolVersion;
import org.apache.cassandra.utils.ByteBufferUtil;
+import static com.google.common.collect.Iterables.any;
+
/*
* The encoding of a DynamicCompositeType column name should be:
* <component><component><component> ...
@@ -59,22 +62,19 @@
private final Map<Byte, AbstractType<?>> aliases;
// interning instances
- private static final Map<Map<Byte, AbstractType<?>>, DynamicCompositeType> instances = new HashMap<Map<Byte, AbstractType<?>>, DynamicCompositeType>();
+ private static final ConcurrentHashMap<Map<Byte, AbstractType<?>>, DynamicCompositeType> instances = new ConcurrentHashMap<>();
- public static synchronized DynamicCompositeType getInstance(TypeParser parser) throws ConfigurationException, SyntaxException
+ public static DynamicCompositeType getInstance(TypeParser parser)
{
return getInstance(parser.getAliasParameters());
}
- public static synchronized DynamicCompositeType getInstance(Map<Byte, AbstractType<?>> aliases)
+ public static DynamicCompositeType getInstance(Map<Byte, AbstractType<?>> aliases)
{
DynamicCompositeType dct = instances.get(aliases);
- if (dct == null)
- {
- dct = new DynamicCompositeType(aliases);
- instances.put(aliases, dct);
- }
- return dct;
+ return null == dct
+ ? instances.computeIfAbsent(aliases, DynamicCompositeType::new)
+ : dct;
}
private DynamicCompositeType(Map<Byte, AbstractType<?>> aliases)
@@ -124,7 +124,7 @@
* If both types are ReversedType(Type), we need to compare on the wrapped type (which may differ between the two types) to avoid
* incompatible comparisons being made.
*/
- if ((comp1 instanceof ReversedType) && (comp2 instanceof ReversedType))
+ if ((comp1 instanceof ReversedType) && (comp2 instanceof ReversedType))
{
comp1 = ((ReversedType<?>) comp1).baseType;
comp2 = ((ReversedType<?>) comp2).baseType;
@@ -200,19 +200,17 @@
valueStr = ByteBufferUtil.string(value);
comparator = TypeParser.parse(valueStr);
}
- catch (CharacterCodingException ce)
+ catch (CharacterCodingException ce)
{
- // ByteBufferUtil.string failed.
+ // ByteBufferUtil.string failed.
// Log it here and we'll further throw an exception below since comparator == null
- logger.error("Failed with [{}] when decoding the byte buffer in ByteBufferUtil.string()",
- ce.toString());
+ logger.error("Failed when decoding the byte buffer in ByteBufferUtil.string()", ce);
}
catch (Exception e)
{
- // parse failed.
+ // parse failed.
// Log it here and we'll further throw an exception below since comparator == null
- logger.error("Failed to parse value string \"{}\" with exception: [{}]",
- valueStr, e.toString());
+ logger.error("Failed to parse value string \"{}\" with exception:", valueStr, e);
}
}
else
@@ -258,6 +256,29 @@
return true;
}
+ @Override
+ public boolean referencesUserType(ByteBuffer name)
+ {
+ return any(aliases.values(), t -> t.referencesUserType(name));
+ }
+
+ @Override
+ public DynamicCompositeType withUpdatedUserType(UserType udt)
+ {
+ if (!referencesUserType(udt.name))
+ return this;
+
+ instances.remove(aliases);
+
+ return getInstance(Maps.transformValues(aliases, v -> v.withUpdatedUserType(udt)));
+ }
+
+ @Override
+ public AbstractType<?> expandUserTypes()
+ {
+ return getInstance(Maps.transformValues(aliases, v -> v.expandUserTypes()));
+ }
+
private class DynamicParsedComparator implements ParsedComparator
{
final AbstractType<?> type;
diff --git a/src/java/org/apache/cassandra/db/marshal/EmptyType.java b/src/java/org/apache/cassandra/db/marshal/EmptyType.java
index 00c919d..88d62c4 100644
--- a/src/java/org/apache/cassandra/db/marshal/EmptyType.java
+++ b/src/java/org/apache/cassandra/db/marshal/EmptyType.java
@@ -28,10 +28,10 @@
import org.apache.cassandra.cql3.Term;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
-import org.apache.cassandra.serializers.TypeSerializer;
-import org.apache.cassandra.transport.ProtocolVersion;
import org.apache.cassandra.serializers.EmptySerializer;
import org.apache.cassandra.serializers.MarshalException;
+import org.apache.cassandra.serializers.TypeSerializer;
+import org.apache.cassandra.transport.ProtocolVersion;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.NoSpamLogger;
@@ -115,7 +115,7 @@
}
@Override
- protected int valueLengthIfFixed()
+ public int valueLengthIfFixed()
{
return 0;
}
diff --git a/src/java/org/apache/cassandra/db/marshal/FloatType.java b/src/java/org/apache/cassandra/db/marshal/FloatType.java
index ad71416..6752872 100644
--- a/src/java/org/apache/cassandra/db/marshal/FloatType.java
+++ b/src/java/org/apache/cassandra/db/marshal/FloatType.java
@@ -29,7 +29,7 @@
import org.apache.cassandra.utils.ByteBufferUtil;
-public class FloatType extends AbstractType<Float>
+public class FloatType extends NumberType<Float>
{
public static final FloatType instance = new FloatType();
@@ -40,6 +40,12 @@
return true;
}
+ @Override
+ public boolean isFloatingPoint()
+ {
+ return true;
+ }
+
public int compareCustom(ByteBuffer o1, ByteBuffer o2)
{
if (!o1.hasRemaining() || !o2.hasRemaining())
@@ -56,8 +62,7 @@
try
{
- float f = Float.parseFloat(source);
- return ByteBufferUtil.bytes(f);
+ return decompose(Float.parseFloat(source));
}
catch (NumberFormatException e1)
{
@@ -103,8 +108,56 @@
}
@Override
- protected int valueLengthIfFixed()
+ public int valueLengthIfFixed()
{
return 4;
}
+
+ @Override
+ protected int toInt(ByteBuffer value)
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ protected float toFloat(ByteBuffer value)
+ {
+ return ByteBufferUtil.toFloat(value);
+ }
+
+ @Override
+ protected double toDouble(ByteBuffer value)
+ {
+ return toFloat(value);
+ }
+
+ public ByteBuffer add(NumberType<?> leftType, ByteBuffer left, NumberType<?> rightType, ByteBuffer right)
+ {
+ return ByteBufferUtil.bytes(leftType.toFloat(left) + rightType.toFloat(right));
+ }
+
+ public ByteBuffer substract(NumberType<?> leftType, ByteBuffer left, NumberType<?> rightType, ByteBuffer right)
+ {
+ return ByteBufferUtil.bytes(leftType.toFloat(left) - rightType.toFloat(right));
+ }
+
+ public ByteBuffer multiply(NumberType<?> leftType, ByteBuffer left, NumberType<?> rightType, ByteBuffer right)
+ {
+ return ByteBufferUtil.bytes(leftType.toFloat(left) * rightType.toFloat(right));
+ }
+
+ public ByteBuffer divide(NumberType<?> leftType, ByteBuffer left, NumberType<?> rightType, ByteBuffer right)
+ {
+ return ByteBufferUtil.bytes(leftType.toFloat(left) / rightType.toFloat(right));
+ }
+
+ public ByteBuffer mod(NumberType<?> leftType, ByteBuffer left, NumberType<?> rightType, ByteBuffer right)
+ {
+ return ByteBufferUtil.bytes(leftType.toFloat(left) % rightType.toFloat(right));
+ }
+
+ public ByteBuffer negate(ByteBuffer input)
+ {
+ return ByteBufferUtil.bytes(-toFloat(input));
+ }
}
diff --git a/src/java/org/apache/cassandra/db/marshal/Int32Type.java b/src/java/org/apache/cassandra/db/marshal/Int32Type.java
index 1c8c93e..a66f9dc 100644
--- a/src/java/org/apache/cassandra/db/marshal/Int32Type.java
+++ b/src/java/org/apache/cassandra/db/marshal/Int32Type.java
@@ -22,13 +22,13 @@
import org.apache.cassandra.cql3.CQL3Type;
import org.apache.cassandra.cql3.Constants;
import org.apache.cassandra.cql3.Term;
-import org.apache.cassandra.serializers.TypeSerializer;
import org.apache.cassandra.serializers.Int32Serializer;
import org.apache.cassandra.serializers.MarshalException;
import org.apache.cassandra.transport.ProtocolVersion;
+import org.apache.cassandra.serializers.TypeSerializer;
import org.apache.cassandra.utils.ByteBufferUtil;
-public class Int32Type extends AbstractType<Integer>
+public class Int32Type extends NumberType<Integer>
{
public static final Int32Type instance = new Int32Type();
@@ -112,8 +112,50 @@
}
@Override
- protected int valueLengthIfFixed()
+ public int valueLengthIfFixed()
{
return 4;
}
+
+ @Override
+ protected int toInt(ByteBuffer value)
+ {
+ return ByteBufferUtil.toInt(value);
+ }
+
+ @Override
+ protected float toFloat(ByteBuffer value)
+ {
+ return toInt(value);
+ }
+
+ public ByteBuffer add(NumberType<?> leftType, ByteBuffer left, NumberType<?> rightType, ByteBuffer right)
+ {
+ return ByteBufferUtil.bytes(leftType.toInt(left) + rightType.toInt(right));
+ }
+
+ public ByteBuffer substract(NumberType<?> leftType, ByteBuffer left, NumberType<?> rightType, ByteBuffer right)
+ {
+ return ByteBufferUtil.bytes(leftType.toInt(left) - rightType.toInt(right));
+ }
+
+ public ByteBuffer multiply(NumberType<?> leftType, ByteBuffer left, NumberType<?> rightType, ByteBuffer right)
+ {
+ return ByteBufferUtil.bytes(leftType.toInt(left) * rightType.toInt(right));
+ }
+
+ public ByteBuffer divide(NumberType<?> leftType, ByteBuffer left, NumberType<?> rightType, ByteBuffer right)
+ {
+ return ByteBufferUtil.bytes(leftType.toInt(left) / rightType.toInt(right));
+ }
+
+ public ByteBuffer mod(NumberType<?> leftType, ByteBuffer left, NumberType<?> rightType, ByteBuffer right)
+ {
+ return ByteBufferUtil.bytes(leftType.toInt(left) % rightType.toInt(right));
+ }
+
+ public ByteBuffer negate(ByteBuffer input)
+ {
+ return ByteBufferUtil.bytes(-toInt(input));
+ }
}
diff --git a/src/java/org/apache/cassandra/db/marshal/IntegerType.java b/src/java/org/apache/cassandra/db/marshal/IntegerType.java
index 944a231..e2b8518 100644
--- a/src/java/org/apache/cassandra/db/marshal/IntegerType.java
+++ b/src/java/org/apache/cassandra/db/marshal/IntegerType.java
@@ -17,6 +17,7 @@
*/
package org.apache.cassandra.db.marshal;
+import java.math.BigDecimal;
import java.math.BigInteger;
import java.nio.ByteBuffer;
@@ -29,7 +30,7 @@
import org.apache.cassandra.transport.ProtocolVersion;
import org.apache.cassandra.utils.ByteBufferUtil;
-public final class IntegerType extends AbstractType<BigInteger>
+public final class IntegerType extends NumberType<BigInteger>
{
public static final IntegerType instance = new IntegerType();
@@ -184,4 +185,70 @@
{
return IntegerSerializer.instance;
}
+
+ @Override
+ protected int toInt(ByteBuffer value)
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ protected float toFloat(ByteBuffer value)
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ protected long toLong(ByteBuffer value)
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ protected double toDouble(ByteBuffer value)
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ protected BigInteger toBigInteger(ByteBuffer value)
+ {
+ return compose(value);
+ }
+
+ @Override
+ protected BigDecimal toBigDecimal(ByteBuffer value)
+ {
+ return new BigDecimal(compose(value));
+ }
+
+ public ByteBuffer add(NumberType<?> leftType, ByteBuffer left, NumberType<?> rightType, ByteBuffer right)
+ {
+ return decompose(leftType.toBigInteger(left).add(rightType.toBigInteger(right)));
+ }
+
+ public ByteBuffer substract(NumberType<?> leftType, ByteBuffer left, NumberType<?> rightType, ByteBuffer right)
+ {
+ return decompose(leftType.toBigInteger(left).subtract(rightType.toBigInteger(right)));
+ }
+
+ public ByteBuffer multiply(NumberType<?> leftType, ByteBuffer left, NumberType<?> rightType, ByteBuffer right)
+ {
+ return decompose(leftType.toBigInteger(left).multiply(rightType.toBigInteger(right)));
+ }
+
+ public ByteBuffer divide(NumberType<?> leftType, ByteBuffer left, NumberType<?> rightType, ByteBuffer right)
+ {
+ return decompose(leftType.toBigInteger(left).divide(rightType.toBigInteger(right)));
+ }
+
+ public ByteBuffer mod(NumberType<?> leftType, ByteBuffer left, NumberType<?> rightType, ByteBuffer right)
+ {
+ return decompose(leftType.toBigInteger(left).remainder(rightType.toBigInteger(right)));
+ }
+
+ public ByteBuffer negate(ByteBuffer input)
+ {
+ return decompose(toBigInteger(input).negate());
+ }
}
diff --git a/src/java/org/apache/cassandra/db/marshal/LexicalUUIDType.java b/src/java/org/apache/cassandra/db/marshal/LexicalUUIDType.java
index 70767d4..de32a56 100644
--- a/src/java/org/apache/cassandra/db/marshal/LexicalUUIDType.java
+++ b/src/java/org/apache/cassandra/db/marshal/LexicalUUIDType.java
@@ -86,7 +86,7 @@
}
@Override
- protected int valueLengthIfFixed()
+ public int valueLengthIfFixed()
{
return 16;
}
diff --git a/src/java/org/apache/cassandra/db/marshal/ListType.java b/src/java/org/apache/cassandra/db/marshal/ListType.java
index c03f866..c6e0262 100644
--- a/src/java/org/apache/cassandra/db/marshal/ListType.java
+++ b/src/java/org/apache/cassandra/db/marshal/ListType.java
@@ -19,6 +19,7 @@
import java.nio.ByteBuffer;
import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
import org.apache.cassandra.cql3.Json;
import org.apache.cassandra.cql3.Lists;
@@ -27,20 +28,15 @@
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.exceptions.SyntaxException;
import org.apache.cassandra.serializers.CollectionSerializer;
-import org.apache.cassandra.serializers.MarshalException;
import org.apache.cassandra.serializers.ListSerializer;
+import org.apache.cassandra.serializers.MarshalException;
import org.apache.cassandra.transport.ProtocolVersion;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
public class ListType<T> extends CollectionType<List<T>>
{
- private static final Logger logger = LoggerFactory.getLogger(ListType.class);
-
// interning instances
- private static final Map<AbstractType<?>, ListType> instances = new HashMap<>();
- private static final Map<AbstractType<?>, ListType> frozenInstances = new HashMap<>();
+ private static final ConcurrentHashMap<AbstractType<?>, ListType> instances = new ConcurrentHashMap<>();
+ private static final ConcurrentHashMap<AbstractType<?>, ListType> frozenInstances = new ConcurrentHashMap<>();
private final AbstractType<T> elements;
public final ListSerializer<T> serializer;
@@ -55,16 +51,13 @@
return getInstance(l.get(0), true);
}
- public static synchronized <T> ListType<T> getInstance(AbstractType<T> elements, boolean isMultiCell)
+ public static <T> ListType<T> getInstance(AbstractType<T> elements, boolean isMultiCell)
{
- Map<AbstractType<?>, ListType> internMap = isMultiCell ? instances : frozenInstances;
+ ConcurrentHashMap<AbstractType<?>, ListType> internMap = isMultiCell ? instances : frozenInstances;
ListType<T> t = internMap.get(elements);
- if (t == null)
- {
- t = new ListType<T>(elements, isMultiCell);
- internMap.put(elements, t);
- }
- return t;
+ return null == t
+ ? internMap.computeIfAbsent(elements, k -> new ListType<>(k, isMultiCell))
+ : t;
}
private ListType(AbstractType<T> elements, boolean isMultiCell)
@@ -76,9 +69,26 @@
}
@Override
- public boolean referencesUserType(String userTypeName)
+ public boolean referencesUserType(ByteBuffer name)
{
- return getElementsType().referencesUserType(userTypeName);
+ return elements.referencesUserType(name);
+ }
+
+ @Override
+ public ListType<?> withUpdatedUserType(UserType udt)
+ {
+ if (!referencesUserType(udt.name))
+ return this;
+
+ (isMultiCell ? instances : frozenInstances).remove(elements);
+
+ return getInstance(elements.withUpdatedUserType(udt), isMultiCell);
+ }
+
+ @Override
+ public AbstractType<?> expandUserTypes()
+ {
+ return getInstance(elements.expandUserTypes(), isMultiCell);
}
@Override
@@ -129,6 +139,12 @@
}
@Override
+ public List<AbstractType<?>> subTypes()
+ {
+ return Collections.singletonList(elements);
+ }
+
+ @Override
public boolean isMultiCell()
{
return isMultiCell;
@@ -238,6 +254,12 @@
return sb.append("]").toString();
}
+ public ByteBuffer getSliceFromSerialized(ByteBuffer collection, ByteBuffer from, ByteBuffer to)
+ {
+ // We don't support slicing on lists so we don't need that function
+ throw new UnsupportedOperationException();
+ }
+
@Override
public String toJSONString(ByteBuffer buffer, ProtocolVersion protocolVersion)
{
diff --git a/src/java/org/apache/cassandra/db/marshal/LongType.java b/src/java/org/apache/cassandra/db/marshal/LongType.java
index c852461..ef96e2e 100644
--- a/src/java/org/apache/cassandra/db/marshal/LongType.java
+++ b/src/java/org/apache/cassandra/db/marshal/LongType.java
@@ -28,7 +28,7 @@
import org.apache.cassandra.transport.ProtocolVersion;
import org.apache.cassandra.utils.ByteBufferUtil;
-public class LongType extends AbstractType<Long>
+public class LongType extends NumberType<Long>
{
public static final LongType instance = new LongType();
@@ -120,8 +120,56 @@
}
@Override
- protected int valueLengthIfFixed()
+ public int valueLengthIfFixed()
{
return 8;
}
+
+ @Override
+ protected int toInt(ByteBuffer value)
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ protected float toFloat(ByteBuffer value)
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ protected long toLong(ByteBuffer value)
+ {
+ return ByteBufferUtil.toLong(value);
+ }
+
+ public ByteBuffer add(NumberType<?> leftType, ByteBuffer left, NumberType<?> rightType, ByteBuffer right)
+ {
+ return ByteBufferUtil.bytes(leftType.toLong(left) + rightType.toLong(right));
+ }
+
+ public ByteBuffer substract(NumberType<?> leftType, ByteBuffer left, NumberType<?> rightType, ByteBuffer right)
+ {
+ return ByteBufferUtil.bytes(leftType.toLong(left) - rightType.toLong(right));
+ }
+
+ public ByteBuffer multiply(NumberType<?> leftType, ByteBuffer left, NumberType<?> rightType, ByteBuffer right)
+ {
+ return ByteBufferUtil.bytes(leftType.toLong(left) * rightType.toLong(right));
+ }
+
+ public ByteBuffer divide(NumberType<?> leftType, ByteBuffer left, NumberType<?> rightType, ByteBuffer right)
+ {
+ return ByteBufferUtil.bytes(leftType.toLong(left) / rightType.toLong(right));
+ }
+
+ public ByteBuffer mod(NumberType<?> leftType, ByteBuffer left, NumberType<?> rightType, ByteBuffer right)
+ {
+ return ByteBufferUtil.bytes(leftType.toLong(left) % rightType.toLong(right));
+ }
+
+ public ByteBuffer negate(ByteBuffer input)
+ {
+ return ByteBufferUtil.bytes(-toLong(input));
+ }
}
diff --git a/src/java/org/apache/cassandra/db/marshal/MapType.java b/src/java/org/apache/cassandra/db/marshal/MapType.java
index 1817f31..6abe388 100644
--- a/src/java/org/apache/cassandra/db/marshal/MapType.java
+++ b/src/java/org/apache/cassandra/db/marshal/MapType.java
@@ -19,6 +19,7 @@
import java.nio.ByteBuffer;
import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
import org.apache.cassandra.cql3.Json;
import org.apache.cassandra.cql3.Maps;
@@ -35,8 +36,8 @@
public class MapType<K, V> extends CollectionType<Map<K, V>>
{
// interning instances
- private static final Map<Pair<AbstractType<?>, AbstractType<?>>, MapType> instances = new HashMap<>();
- private static final Map<Pair<AbstractType<?>, AbstractType<?>>, MapType> frozenInstances = new HashMap<>();
+ private static final ConcurrentHashMap<Pair<AbstractType<?>, AbstractType<?>>, MapType> instances = new ConcurrentHashMap<>();
+ private static final ConcurrentHashMap<Pair<AbstractType<?>, AbstractType<?>>, MapType> frozenInstances = new ConcurrentHashMap<>();
private final AbstractType<K> keys;
private final AbstractType<V> values;
@@ -52,17 +53,14 @@
return getInstance(l.get(0), l.get(1), true);
}
- public static synchronized <K, V> MapType<K, V> getInstance(AbstractType<K> keys, AbstractType<V> values, boolean isMultiCell)
+ public static <K, V> MapType<K, V> getInstance(AbstractType<K> keys, AbstractType<V> values, boolean isMultiCell)
{
- Map<Pair<AbstractType<?>, AbstractType<?>>, MapType> internMap = isMultiCell ? instances : frozenInstances;
- Pair<AbstractType<?>, AbstractType<?>> p = Pair.<AbstractType<?>, AbstractType<?>>create(keys, values);
+ ConcurrentHashMap<Pair<AbstractType<?>, AbstractType<?>>, MapType> internMap = isMultiCell ? instances : frozenInstances;
+ Pair<AbstractType<?>, AbstractType<?>> p = Pair.create(keys, values);
MapType<K, V> t = internMap.get(p);
- if (t == null)
- {
- t = new MapType<>(keys, values, isMultiCell);
- internMap.put(p, t);
- }
- return t;
+ return null == t
+ ? internMap.computeIfAbsent(p, k -> new MapType<>(k.left, k.right, isMultiCell))
+ : t;
}
private MapType(AbstractType<K> keys, AbstractType<V> values, boolean isMultiCell)
@@ -75,10 +73,26 @@
}
@Override
- public boolean referencesUserType(String userTypeName)
+ public boolean referencesUserType(ByteBuffer name)
{
- return getKeysType().referencesUserType(userTypeName) ||
- getValuesType().referencesUserType(userTypeName);
+ return keys.referencesUserType(name) || values.referencesUserType(name);
+ }
+
+ @Override
+ public MapType<?,?> withUpdatedUserType(UserType udt)
+ {
+ if (!referencesUserType(udt.name))
+ return this;
+
+ (isMultiCell ? instances : frozenInstances).remove(Pair.create(keys, values));
+
+ return getInstance(keys.withUpdatedUserType(udt), values.withUpdatedUserType(udt), isMultiCell);
+ }
+
+ @Override
+ public AbstractType<?> expandUserTypes()
+ {
+ return getInstance(keys.expandUserTypes(), values.expandUserTypes(), isMultiCell);
}
@Override
@@ -115,6 +129,12 @@
}
@Override
+ public List<AbstractType<?>> subTypes()
+ {
+ return Arrays.asList(keys, values);
+ }
+
+ @Override
public AbstractType<?> freeze()
{
if (isMultiCell)
diff --git a/src/java/org/apache/cassandra/db/marshal/NumberType.java b/src/java/org/apache/cassandra/db/marshal/NumberType.java
new file mode 100644
index 0000000..9e7697f
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/marshal/NumberType.java
@@ -0,0 +1,223 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db.marshal;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.nio.ByteBuffer;
+
+/**
+ * Base type for the numeric types.
+ */
+public abstract class NumberType<T extends Number> extends AbstractType<T>
+{
+ protected NumberType(ComparisonType comparisonType)
+ {
+ super(comparisonType);
+ }
+
+ /**
+ * Checks if this type support floating point numbers.
+ * @return {@code true} if this type support floating point numbers, {@code false} otherwise.
+ */
+ public boolean isFloatingPoint()
+ {
+ return false;
+ }
+
+ /**
+ * Converts the specified value into a <code>BigInteger</code> if allowed.
+ *
+ * @param value the value to convert
+ * @return the converted value
+ * @throws UnsupportedOperationException if the value cannot be converted without losing precision
+ */
+ protected BigInteger toBigInteger(ByteBuffer value)
+ {
+ return BigInteger.valueOf(toLong(value));
+ }
+
+ /**
+ * Converts the specified value into a <code>BigDecimal</code>.
+ *
+ * @param value the value to convert
+ * @return the converted value
+ */
+ protected BigDecimal toBigDecimal(ByteBuffer value)
+ {
+ double d = toDouble(value);
+
+ if (Double.isNaN(d))
+ throw new NumberFormatException("A NaN cannot be converted into a decimal");
+
+ if (Double.isInfinite(d))
+ throw new NumberFormatException("An infinite number cannot be converted into a decimal");
+
+ return BigDecimal.valueOf(d);
+ }
+
+ /**
+ * Converts the specified value into a <code>byte</code> if allowed.
+ *
+ * @param value the value to convert
+ * @return the converted value
+ * @throws UnsupportedOperationException if the value cannot be converted without losing precision
+ */
+ protected byte toByte(ByteBuffer value)
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ /**
+ * Converts the specified value into a <code>short</code> if allowed.
+ *
+ * @param value the value to convert
+ * @return the converted value
+ * @throws UnsupportedOperationException if the value cannot be converted without losing precision
+ */
+ protected short toShort(ByteBuffer value)
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ /**
+ * Converts the specified value into an <code>int</code> if allowed.
+ *
+ * @param value the value to convert
+ * @return the converted value
+ * @throws UnsupportedOperationException if the value cannot be converted without losing precision
+ */
+ protected int toInt(ByteBuffer value)
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ /**
+ * Converts the specified value into a <code>long</code> if allowed.
+ *
+ * @param value the value to convert
+ * @return the converted value
+ * @throws UnsupportedOperationException if the value cannot be converted without losing precision
+ */
+ protected long toLong(ByteBuffer value)
+ {
+ return toInt(value);
+ }
+
+ /**
+ * Converts the specified value into a <code>float</code> if allowed.
+ *
+ * @param value the value to convert
+ * @return the converted value
+ * @throws UnsupportedOperationException if the value cannot be converted without losing precision
+ */
+ protected float toFloat(ByteBuffer value)
+ {
+ return toInt(value);
+ }
+
+ /**
+ * Converts the specified value into a <code>double</code> if allowed.
+ *
+ * @param value the value to convert
+ * @return the converted value
+ * @throws UnsupportedOperationException if the value cannot be converted without losing precision
+ */
+ protected double toDouble(ByteBuffer value)
+ {
+ return toLong(value);
+ }
+
+ /**
+ * Adds the left argument to the right one.
+ *
+ * @param leftType the type associated to the left argument
+ * @param left the left argument
+ * @param rightType the type associated to the right argument
+ * @param right the right argument
+ * @return the addition result
+ */
+ public abstract ByteBuffer add(NumberType<?> leftType,
+ ByteBuffer left,
+ NumberType<?> rightType,
+ ByteBuffer right);
+
+ /**
+ * Substracts the left argument from the right one.
+ *
+ * @param leftType the type associated to the left argument
+ * @param left the left argument
+ * @param rightType the type associated to the right argument
+ * @param right the right argument
+ * @return the substraction result
+ */
+ public abstract ByteBuffer substract(NumberType<?> leftType,
+ ByteBuffer left,
+ NumberType<?> rightType,
+ ByteBuffer right);
+
+ /**
+ * Multiplies the left argument with the right one.
+ *
+ * @param leftType the type associated to the left argument
+ * @param left the left argument
+ * @param rightType the type associated to the right argument
+ * @param right the right argument
+ * @return the multiplication result
+ */
+ public abstract ByteBuffer multiply(NumberType<?> leftType,
+ ByteBuffer left,
+ NumberType<?> rightType,
+ ByteBuffer right);
+
+ /**
+ * Divides the left argument by the right one.
+ *
+ * @param leftType the type associated to the left argument
+ * @param left the left argument
+ * @param rightType the type associated to the right argument
+ * @param right the right argument
+ * @return the division result
+ */
+ public abstract ByteBuffer divide(NumberType<?> leftType,
+ ByteBuffer left,
+ NumberType<?> rightType,
+ ByteBuffer right);
+
+ /**
+ * Return the remainder.
+ *
+ * @param leftType the type associated to the left argument
+ * @param left the left argument
+ * @param rightType the type associated to the right argument
+ * @param right the right argument
+ * @return the remainder
+ */
+ public abstract ByteBuffer mod(NumberType<?> leftType,
+ ByteBuffer left,
+ NumberType<?> rightType,
+ ByteBuffer right);
+
+ /**
+ * Negates the argument.
+ *
+ * @param input the argument to negate
+ * @return the negated argument
+ */
+ public abstract ByteBuffer negate(ByteBuffer input);
+}
diff --git a/src/java/org/apache/cassandra/db/marshal/ReversedType.java b/src/java/org/apache/cassandra/db/marshal/ReversedType.java
index 0eb0046..63a900a 100644
--- a/src/java/org/apache/cassandra/db/marshal/ReversedType.java
+++ b/src/java/org/apache/cassandra/db/marshal/ReversedType.java
@@ -18,14 +18,13 @@
package org.apache.cassandra.db.marshal;
import java.nio.ByteBuffer;
-import java.util.HashMap;
import java.util.Map;
import java.util.List;
+import java.util.concurrent.ConcurrentHashMap;
import org.apache.cassandra.cql3.CQL3Type;
import org.apache.cassandra.cql3.Term;
import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.exceptions.SyntaxException;
import org.apache.cassandra.serializers.MarshalException;
import org.apache.cassandra.serializers.TypeSerializer;
import org.apache.cassandra.transport.ProtocolVersion;
@@ -33,11 +32,11 @@
public class ReversedType<T> extends AbstractType<T>
{
// interning instances
- private static final Map<AbstractType<?>, ReversedType> instances = new HashMap<AbstractType<?>, ReversedType>();
+ private static final Map<AbstractType<?>, ReversedType> instances = new ConcurrentHashMap<>();
public final AbstractType<T> baseType;
- public static <T> ReversedType<T> getInstance(TypeParser parser) throws ConfigurationException, SyntaxException
+ public static <T> ReversedType<T> getInstance(TypeParser parser)
{
List<AbstractType<?>> types = parser.getTypeParameters();
if (types.size() != 1)
@@ -45,15 +44,12 @@
return getInstance((AbstractType<T>) types.get(0));
}
- public static synchronized <T> ReversedType<T> getInstance(AbstractType<T> baseType)
+ public static <T> ReversedType<T> getInstance(AbstractType<T> baseType)
{
- ReversedType<T> type = instances.get(baseType);
- if (type == null)
- {
- type = new ReversedType<T>(baseType);
- instances.put(baseType, type);
- }
- return type;
+ ReversedType<T> t = instances.get(baseType);
+ return null == t
+ ? instances.computeIfAbsent(baseType, ReversedType::new)
+ : t;
}
private ReversedType(AbstractType<T> baseType)
@@ -126,13 +122,31 @@
return baseType.getSerializer();
}
- public boolean referencesUserType(String userTypeName)
+ @Override
+ public boolean referencesUserType(ByteBuffer name)
{
- return baseType.referencesUserType(userTypeName);
+ return baseType.referencesUserType(name);
}
@Override
- protected int valueLengthIfFixed()
+ public AbstractType<?> expandUserTypes()
+ {
+ return getInstance(baseType.expandUserTypes());
+ }
+
+ @Override
+ public ReversedType<?> withUpdatedUserType(UserType udt)
+ {
+ if (!referencesUserType(udt.name))
+ return this;
+
+ instances.remove(baseType);
+
+ return getInstance(baseType.withUpdatedUserType(udt));
+ }
+
+ @Override
+ public int valueLengthIfFixed()
{
return baseType.valueLengthIfFixed();
}
diff --git a/src/java/org/apache/cassandra/db/marshal/SetType.java b/src/java/org/apache/cassandra/db/marshal/SetType.java
index fdd29ec..6d17b67 100644
--- a/src/java/org/apache/cassandra/db/marshal/SetType.java
+++ b/src/java/org/apache/cassandra/db/marshal/SetType.java
@@ -19,6 +19,7 @@
import java.nio.ByteBuffer;
import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
import org.apache.cassandra.cql3.Json;
import org.apache.cassandra.cql3.Sets;
@@ -33,8 +34,8 @@
public class SetType<T> extends CollectionType<Set<T>>
{
// interning instances
- private static final Map<AbstractType<?>, SetType> instances = new HashMap<>();
- private static final Map<AbstractType<?>, SetType> frozenInstances = new HashMap<>();
+ private static final ConcurrentHashMap<AbstractType<?>, SetType> instances = new ConcurrentHashMap<>();
+ private static final ConcurrentHashMap<AbstractType<?>, SetType> frozenInstances = new ConcurrentHashMap<>();
private final AbstractType<T> elements;
private final SetSerializer<T> serializer;
@@ -49,16 +50,13 @@
return getInstance(l.get(0), true);
}
- public static synchronized <T> SetType<T> getInstance(AbstractType<T> elements, boolean isMultiCell)
+ public static <T> SetType<T> getInstance(AbstractType<T> elements, boolean isMultiCell)
{
- Map<AbstractType<?>, SetType> internMap = isMultiCell ? instances : frozenInstances;
+ ConcurrentHashMap<AbstractType<?>, SetType> internMap = isMultiCell ? instances : frozenInstances;
SetType<T> t = internMap.get(elements);
- if (t == null)
- {
- t = new SetType<T>(elements, isMultiCell);
- internMap.put(elements, t);
- }
- return t;
+ return null == t
+ ? internMap.computeIfAbsent(elements, k -> new SetType<>(k, isMultiCell))
+ : t;
}
public SetType(AbstractType<T> elements, boolean isMultiCell)
@@ -70,9 +68,26 @@
}
@Override
- public boolean referencesUserType(String userTypeName)
+ public boolean referencesUserType(ByteBuffer name)
{
- return getElementsType().referencesUserType(userTypeName);
+ return elements.referencesUserType(name);
+ }
+
+ @Override
+ public SetType<?> withUpdatedUserType(UserType udt)
+ {
+ if (!referencesUserType(udt.name))
+ return this;
+
+ (isMultiCell ? instances : frozenInstances).remove(elements);
+
+ return getInstance(elements.withUpdatedUserType(udt), isMultiCell);
+ }
+
+ @Override
+ public AbstractType<?> expandUserTypes()
+ {
+ return getInstance(elements.expandUserTypes(), isMultiCell);
}
public AbstractType<T> getElementsType()
@@ -106,6 +121,12 @@
}
@Override
+ public List<AbstractType<?>> subTypes()
+ {
+ return Collections.singletonList(elements);
+ }
+
+ @Override
public AbstractType<?> freezeNestedMulticellTypes()
{
if (!isMultiCell())
diff --git a/src/java/org/apache/cassandra/db/marshal/ShortType.java b/src/java/org/apache/cassandra/db/marshal/ShortType.java
index 7645ec6..01dca4e 100644
--- a/src/java/org/apache/cassandra/db/marshal/ShortType.java
+++ b/src/java/org/apache/cassandra/db/marshal/ShortType.java
@@ -28,7 +28,7 @@
import org.apache.cassandra.transport.ProtocolVersion;
import org.apache.cassandra.utils.ByteBufferUtil;
-public class ShortType extends AbstractType<Short>
+public class ShortType extends NumberType<Short>
{
public static final ShortType instance = new ShortType();
@@ -91,4 +91,47 @@
{
return ShortSerializer.instance;
}
+
+ @Override
+ public short toShort(ByteBuffer value)
+ {
+ return ByteBufferUtil.toShort(value);
+ }
+
+ @Override
+ public int toInt(ByteBuffer value)
+ {
+ return toShort(value);
+ }
+
+ @Override
+ public ByteBuffer add(NumberType<?> leftType, ByteBuffer left, NumberType<?> rightType, ByteBuffer right)
+ {
+ return ByteBufferUtil.bytes((short) (leftType.toShort(left) + rightType.toShort(right)));
+ }
+
+ public ByteBuffer substract(NumberType<?> leftType, ByteBuffer left, NumberType<?> rightType, ByteBuffer right)
+ {
+ return ByteBufferUtil.bytes((short) (leftType.toShort(left) - rightType.toShort(right)));
+ }
+
+ public ByteBuffer multiply(NumberType<?> leftType, ByteBuffer left, NumberType<?> rightType, ByteBuffer right)
+ {
+ return ByteBufferUtil.bytes((short) (leftType.toShort(left) * rightType.toShort(right)));
+ }
+
+ public ByteBuffer divide(NumberType<?> leftType, ByteBuffer left, NumberType<?> rightType, ByteBuffer right)
+ {
+ return ByteBufferUtil.bytes((short) (leftType.toShort(left) / rightType.toShort(right)));
+ }
+
+ public ByteBuffer mod(NumberType<?> leftType, ByteBuffer left, NumberType<?> rightType, ByteBuffer right)
+ {
+ return ByteBufferUtil.bytes((short) (leftType.toShort(left) % rightType.toShort(right)));
+ }
+
+ public ByteBuffer negate(ByteBuffer input)
+ {
+ return ByteBufferUtil.bytes((short) -toShort(input));
+ }
}
diff --git a/src/java/org/apache/cassandra/db/marshal/SimpleDateType.java b/src/java/org/apache/cassandra/db/marshal/SimpleDateType.java
index 9db5e36..f883ccd 100644
--- a/src/java/org/apache/cassandra/db/marshal/SimpleDateType.java
+++ b/src/java/org/apache/cassandra/db/marshal/SimpleDateType.java
@@ -21,14 +21,18 @@
import org.apache.cassandra.cql3.CQL3Type;
import org.apache.cassandra.cql3.Constants;
+import org.apache.cassandra.cql3.Duration;
import org.apache.cassandra.cql3.Term;
+import org.apache.cassandra.cql3.statements.RequestValidations;
import org.apache.cassandra.serializers.MarshalException;
import org.apache.cassandra.serializers.SimpleDateSerializer;
import org.apache.cassandra.serializers.TypeSerializer;
import org.apache.cassandra.transport.ProtocolVersion;
import org.apache.cassandra.utils.ByteBufferUtil;
-public class SimpleDateType extends AbstractType<Integer>
+import static org.apache.cassandra.cql3.statements.RequestValidations.invalidRequest;
+
+public class SimpleDateType extends TemporalType<Integer>
{
public static final SimpleDateType instance = new SimpleDateType();
@@ -39,11 +43,13 @@
return ByteBufferUtil.bytes(SimpleDateSerializer.dateStringToDays(source));
}
+ @Override
public ByteBuffer fromTimeInMillis(long millis) throws MarshalException
{
return ByteBufferUtil.bytes(SimpleDateSerializer.timeInMillisToDay(millis));
}
+ @Override
public long toTimeInMillis(ByteBuffer buffer) throws MarshalException
{
return SimpleDateSerializer.dayToTimeInMillis(ByteBufferUtil.toInt(buffer));
@@ -85,4 +91,12 @@
{
return SimpleDateSerializer.instance;
}
+
+ @Override
+ protected void validateDuration(Duration duration)
+ {
+ // Checks that the duration has no data below days.
+ if (!duration.hasDayPrecision())
+ throw invalidRequest("The duration must have a day precision. Was: %s", duration);
+ }
}
diff --git a/src/java/org/apache/cassandra/db/marshal/TemporalType.java b/src/java/org/apache/cassandra/db/marshal/TemporalType.java
new file mode 100644
index 0000000..4e2ac5a
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/marshal/TemporalType.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db.marshal;
+
+import java.nio.ByteBuffer;
+
+import org.apache.cassandra.cql3.Duration;
+
+/**
+ * Base type for temporal types (timestamp, date ...).
+ *
+ */
+public abstract class TemporalType<T> extends AbstractType<T>
+{
+ protected TemporalType(ComparisonType comparisonType)
+ {
+ super(comparisonType);
+ }
+
+ /**
+ * Returns the current temporal value.
+ * @return the current temporal value.
+ */
+ public ByteBuffer now()
+ {
+ return fromTimeInMillis(System.currentTimeMillis());
+ }
+
+ /**
+ * Converts this temporal in UNIX timestamp.
+ * @param value the temporal value.
+ * @return the UNIX timestamp corresponding to this temporal.
+ */
+ public long toTimeInMillis(ByteBuffer value)
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ /**
+ * Returns the temporal value corresponding to the specified UNIX timestamp.
+ * @param timeInMillis the UNIX timestamp to convert
+ * @return the temporal value corresponding to the specified UNIX timestamp
+ */
+ public ByteBuffer fromTimeInMillis(long timeInMillis)
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ /**
+ * Adds the duration to the specified value.
+ *
+ * @param temporal the value to add to
+ * @param duration the duration to add
+ * @return the addition result
+ */
+ public ByteBuffer addDuration(ByteBuffer temporal,
+ ByteBuffer duration)
+ {
+ long timeInMillis = toTimeInMillis(temporal);
+ Duration d = DurationType.instance.compose(duration);
+ validateDuration(d);
+ return fromTimeInMillis(d.addTo(timeInMillis));
+ }
+
+ /**
+ * Substract the duration from the specified value.
+ *
+ * @param temporal the value to substract from
+ * @param duration the duration to substract
+ * @return the substracion result
+ */
+ public ByteBuffer substractDuration(ByteBuffer temporal,
+ ByteBuffer duration)
+ {
+ long timeInMillis = toTimeInMillis(temporal);
+ Duration d = DurationType.instance.compose(duration);
+ validateDuration(d);
+ return fromTimeInMillis(d.substractFrom(timeInMillis));
+ }
+
+ /**
+ * Validates that the duration has the correct precision.
+ * @param duration the duration to validate.
+ */
+ protected void validateDuration(Duration duration)
+ {
+ }
+}
diff --git a/src/java/org/apache/cassandra/db/marshal/TimeType.java b/src/java/org/apache/cassandra/db/marshal/TimeType.java
index 99f4f67..be20ba7 100644
--- a/src/java/org/apache/cassandra/db/marshal/TimeType.java
+++ b/src/java/org/apache/cassandra/db/marshal/TimeType.java
@@ -18,6 +18,9 @@
package org.apache.cassandra.db.marshal;
import java.nio.ByteBuffer;
+import java.time.LocalTime;
+import java.time.ZoneId;
+import java.time.ZoneOffset;
import org.apache.cassandra.cql3.Constants;
import org.apache.cassandra.cql3.Term;
@@ -30,7 +33,7 @@
/**
* Nanosecond resolution time values
*/
-public class TimeType extends AbstractType<Long>
+public class TimeType extends TemporalType<Long>
{
public static final TimeType instance = new TimeType();
private TimeType() {super(ComparisonType.BYTE_ORDER);} // singleton
@@ -75,4 +78,10 @@
{
return TimeSerializer.instance;
}
+
+ @Override
+ public ByteBuffer now()
+ {
+ return decompose(LocalTime.now(ZoneOffset.UTC).toNanoOfDay());
+ }
}
diff --git a/src/java/org/apache/cassandra/db/marshal/TimeUUIDType.java b/src/java/org/apache/cassandra/db/marshal/TimeUUIDType.java
index 36305a3..39d1513 100644
--- a/src/java/org/apache/cassandra/db/marshal/TimeUUIDType.java
+++ b/src/java/org/apache/cassandra/db/marshal/TimeUUIDType.java
@@ -21,13 +21,15 @@
import java.util.UUID;
import org.apache.cassandra.cql3.CQL3Type;
+import org.apache.cassandra.cql3.ColumnSpecification;
import org.apache.cassandra.cql3.Constants;
import org.apache.cassandra.cql3.Term;
import org.apache.cassandra.serializers.TypeSerializer;
+import org.apache.cassandra.utils.UUIDGen;
import org.apache.cassandra.serializers.MarshalException;
import org.apache.cassandra.serializers.TimeUUIDSerializer;
-public class TimeUUIDType extends AbstractType<UUID>
+public class TimeUUIDType extends TemporalType<UUID>
{
public static final TimeUUIDType instance = new TimeUUIDType();
@@ -130,8 +132,32 @@
}
@Override
- protected int valueLengthIfFixed()
+ public int valueLengthIfFixed()
{
return 16;
}
+
+ @Override
+ public long toTimeInMillis(ByteBuffer value)
+ {
+ return UUIDGen.unixTimestamp(UUIDGen.getUUID(value));
+ }
+
+ @Override
+ public ByteBuffer addDuration(ByteBuffer temporal, ByteBuffer duration)
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public ByteBuffer substractDuration(ByteBuffer temporal, ByteBuffer duration)
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public ByteBuffer now()
+ {
+ return ByteBuffer.wrap(UUIDGen.getTimeUUIDBytes());
+ }
}
diff --git a/src/java/org/apache/cassandra/db/marshal/TimestampType.java b/src/java/org/apache/cassandra/db/marshal/TimestampType.java
index 953ae1b..0699050 100644
--- a/src/java/org/apache/cassandra/db/marshal/TimestampType.java
+++ b/src/java/org/apache/cassandra/db/marshal/TimestampType.java
@@ -21,7 +21,10 @@
import java.util.Date;
import org.apache.cassandra.cql3.Constants;
+import org.apache.cassandra.cql3.Duration;
import org.apache.cassandra.cql3.Term;
+import org.apache.cassandra.cql3.statements.RequestValidations;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.cql3.CQL3Type;
@@ -31,6 +34,8 @@
import org.apache.cassandra.transport.ProtocolVersion;
import org.apache.cassandra.utils.ByteBufferUtil;
+import static org.apache.cassandra.cql3.statements.RequestValidations.invalidRequest;
+
/**
* Type for date-time values.
*
@@ -38,7 +43,7 @@
* pre-unix-epoch dates, sorting them *after* post-unix-epoch ones (due to it's
* use of unsigned bytes comparison).
*/
-public class TimestampType extends AbstractType<Date>
+public class TimestampType extends TemporalType<Date>
{
private static final Logger logger = LoggerFactory.getLogger(TimestampType.class);
@@ -65,12 +70,19 @@
return ByteBufferUtil.bytes(TimestampSerializer.dateStringToTimestamp(source));
}
+ @Override
public ByteBuffer fromTimeInMillis(long millis) throws MarshalException
{
return ByteBufferUtil.bytes(millis);
}
@Override
+ public long toTimeInMillis(ByteBuffer value)
+ {
+ return ByteBufferUtil.toLong(value);
+ }
+
+ @Override
public Term fromJSONObject(Object parsed) throws MarshalException
{
if (parsed instanceof Long)
@@ -128,8 +140,15 @@
}
@Override
- protected int valueLengthIfFixed()
+ public int valueLengthIfFixed()
{
return 8;
}
+
+ @Override
+ protected void validateDuration(Duration duration)
+ {
+ if (!duration.hasMillisecondPrecision())
+ throw invalidRequest("The duration must have a millisecond precision. Was: %s", duration);
+ }
}
diff --git a/src/java/org/apache/cassandra/db/marshal/TupleType.java b/src/java/org/apache/cassandra/db/marshal/TupleType.java
index 495338c..dfdb8c2 100644
--- a/src/java/org/apache/cassandra/db/marshal/TupleType.java
+++ b/src/java/org/apache/cassandra/db/marshal/TupleType.java
@@ -23,9 +23,9 @@
import java.util.Iterator;
import java.util.List;
import java.util.regex.Pattern;
-import java.util.stream.Collectors;
import com.google.common.base.Objects;
+import com.google.common.collect.Lists;
import org.apache.cassandra.cql3.*;
import org.apache.cassandra.exceptions.ConfigurationException;
@@ -35,6 +35,9 @@
import org.apache.cassandra.transport.ProtocolVersion;
import org.apache.cassandra.utils.ByteBufferUtil;
+import static com.google.common.collect.Iterables.any;
+import static com.google.common.collect.Iterables.transform;
+
/**
* This is essentially like a CompositeType, but it's not primarily meant for comparison, just
* to pack multiple values together so has a more friendly encoding.
@@ -62,8 +65,9 @@
protected TupleType(List<AbstractType<?>> types, boolean freezeInner)
{
super(ComparisonType.CUSTOM);
+
if (freezeInner)
- this.types = types.stream().map(AbstractType::freeze).collect(Collectors.toList());
+ this.types = Lists.newArrayList(transform(types, AbstractType::freeze));
else
this.types = types;
this.serializer = new TupleSerializer(fieldSerializers(types));
@@ -87,9 +91,23 @@
}
@Override
- public boolean referencesUserType(String name)
+ public boolean referencesUserType(ByteBuffer name)
{
- return allTypes().stream().anyMatch(f -> f.referencesUserType(name));
+ return any(types, t -> t.referencesUserType(name));
+ }
+
+ @Override
+ public TupleType withUpdatedUserType(UserType udt)
+ {
+ return referencesUserType(udt.name)
+ ? new TupleType(Lists.newArrayList(transform(types, t -> t.withUpdatedUserType(udt))))
+ : this;
+ }
+
+ @Override
+ public AbstractType<?> expandUserTypes()
+ {
+ return new TupleType(Lists.newArrayList(transform(types, AbstractType::expandUserTypes)));
}
@Override
@@ -108,11 +126,22 @@
return types.size();
}
+ @Override
+ public List<AbstractType<?>> subTypes()
+ {
+ return types;
+ }
+
public List<AbstractType<?>> allTypes()
{
return types;
}
+ public boolean isTuple()
+ {
+ return true;
+ }
+
public int compareCustom(ByteBuffer o1, ByteBuffer o2)
{
if (!o1.hasRemaining() || !o2.hasRemaining())
@@ -394,12 +423,6 @@
}
@Override
- public boolean isTuple()
- {
- return true;
- }
-
- @Override
public CQL3Type asCQL3Type()
{
return CQL3Type.Tuple.create(this);
diff --git a/src/java/org/apache/cassandra/db/marshal/UTF8Type.java b/src/java/org/apache/cassandra/db/marshal/UTF8Type.java
index 1da6629..46e0d90 100644
--- a/src/java/org/apache/cassandra/db/marshal/UTF8Type.java
+++ b/src/java/org/apache/cassandra/db/marshal/UTF8Type.java
@@ -19,7 +19,7 @@
import java.nio.ByteBuffer;
import java.nio.charset.CharacterCodingException;
-import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
import org.apache.cassandra.cql3.Constants;
import org.apache.cassandra.cql3.Json;
@@ -63,7 +63,7 @@
{
try
{
- return '"' + Json.quoteAsJsonString(ByteBufferUtil.string(buffer, Charset.forName("UTF-8"))) + '"';
+ return '"' + Json.quoteAsJsonString(ByteBufferUtil.string(buffer, StandardCharsets.UTF_8)) + '"';
}
catch (CharacterCodingException exc)
{
diff --git a/src/java/org/apache/cassandra/db/marshal/UUIDType.java b/src/java/org/apache/cassandra/db/marshal/UUIDType.java
index 9722a52..27e3360 100644
--- a/src/java/org/apache/cassandra/db/marshal/UUIDType.java
+++ b/src/java/org/apache/cassandra/db/marshal/UUIDType.java
@@ -171,7 +171,7 @@
}
@Override
- protected int valueLengthIfFixed()
+ public int valueLengthIfFixed()
{
return 16;
}
diff --git a/src/java/org/apache/cassandra/db/marshal/UserType.java b/src/java/org/apache/cassandra/db/marshal/UserType.java
index febd91c..3c023b7 100644
--- a/src/java/org/apache/cassandra/db/marshal/UserType.java
+++ b/src/java/org/apache/cassandra/db/marshal/UserType.java
@@ -22,30 +22,29 @@
import java.util.stream.Collectors;
import com.google.common.base.Objects;
+import com.google.common.collect.Lists;
import org.apache.cassandra.cql3.*;
import org.apache.cassandra.db.rows.Cell;
import org.apache.cassandra.db.rows.CellPath;
-import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.exceptions.SyntaxException;
+import org.apache.cassandra.schema.Difference;
import org.apache.cassandra.serializers.MarshalException;
-import org.apache.cassandra.transport.ProtocolVersion;
import org.apache.cassandra.serializers.TypeSerializer;
import org.apache.cassandra.serializers.UserTypeSerializer;
+import org.apache.cassandra.transport.ProtocolVersion;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.Pair;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+
+import static com.google.common.collect.Iterables.any;
+import static com.google.common.collect.Iterables.transform;
/**
* A user defined type.
*
* A user type is really just a tuple type on steroids.
*/
-public class UserType extends TupleType
+public class UserType extends TupleType implements SchemaElement
{
- private static final Logger logger = LoggerFactory.getLogger(UserType.class);
-
public final String keyspace;
public final ByteBuffer name;
private final List<FieldIdentifier> fieldNames;
@@ -73,7 +72,7 @@
this.serializer = new UserTypeSerializer(fieldSerializers);
}
- public static UserType getInstance(TypeParser parser) throws ConfigurationException, SyntaxException
+ public static UserType getInstance(TypeParser parser)
{
Pair<Pair<String, ByteBuffer>, List<Pair<ByteBuffer, AbstractType>>> params = parser.getUserTypeParameters();
String keyspace = params.left.left;
@@ -95,6 +94,11 @@
return true;
}
+ public boolean isTuple()
+ {
+ return false;
+ }
+
@Override
public boolean isMultiCell()
{
@@ -328,34 +332,44 @@
@Override
public boolean equals(Object o)
{
- return o instanceof UserType && equals(o, false);
- }
-
- @Override
- public boolean equals(Object o, boolean ignoreFreezing)
- {
if(!(o instanceof UserType))
return false;
UserType that = (UserType)o;
- if (!keyspace.equals(that.keyspace) || !name.equals(that.name) || !fieldNames.equals(that.fieldNames))
- return false;
+ return equalsWithoutTypes(that) && types.equals(that.types);
+ }
- if (!ignoreFreezing && isMultiCell != that.isMultiCell)
- return false;
+ private boolean equalsWithoutTypes(UserType other)
+ {
+ return name.equals(other.name)
+ && fieldNames.equals(other.fieldNames)
+ && keyspace.equals(other.keyspace)
+ && isMultiCell == other.isMultiCell;
+ }
- if (this.types.size() != that.types.size())
- return false;
+ public Optional<Difference> compare(UserType other)
+ {
+ if (!equalsWithoutTypes(other))
+ return Optional.of(Difference.SHALLOW);
- Iterator<AbstractType<?>> otherTypeIter = that.types.iterator();
- for (AbstractType<?> type : types)
+ boolean differsDeeply = false;
+
+ for (int i = 0; i < fieldTypes().size(); i++)
{
- if (!type.equals(otherTypeIter.next(), ignoreFreezing))
- return false;
+ AbstractType<?> thisType = fieldType(i);
+ AbstractType<?> thatType = other.fieldType(i);
+
+ if (!thisType.equals(thatType))
+ {
+ if (thisType.asCQL3Type().toString().equals(thatType.asCQL3Type().toString()))
+ differsDeeply = true;
+ else
+ return Optional.of(Difference.SHALLOW);
+ }
}
- return true;
+ return differsDeeply ? Optional.of(Difference.DEEP) : Optional.empty();
}
@Override
@@ -365,10 +379,30 @@
}
@Override
- public boolean referencesUserType(String userTypeName)
+ public boolean referencesUserType(ByteBuffer name)
{
- return getNameAsString().equals(userTypeName) ||
- fieldTypes().stream().anyMatch(f -> f.referencesUserType(userTypeName));
+ return this.name.equals(name) || any(fieldTypes(), t -> t.referencesUserType(name));
+ }
+
+ @Override
+ public UserType withUpdatedUserType(UserType udt)
+ {
+ if (!referencesUserType(udt.name))
+ return this;
+
+ // preserve frozen/non-frozen status of the updated UDT
+ if (name.equals(udt.name))
+ {
+ return isMultiCell == udt.isMultiCell
+ ? udt
+ : new UserType(keyspace, name, udt.fieldNames(), udt.fieldTypes(), isMultiCell);
+ }
+
+ return new UserType(keyspace,
+ name,
+ fieldNames,
+ Lists.newArrayList(transform(fieldTypes(), t -> t.withUpdatedUserType(udt))),
+ isMultiCell());
}
@Override
@@ -384,12 +418,6 @@
}
@Override
- public boolean isTuple()
- {
- return false;
- }
-
- @Override
public String toString(boolean ignoreFreezing)
{
boolean includeFrozenType = !ignoreFreezing && !isMultiCell();
@@ -404,9 +432,62 @@
return sb.toString();
}
+ public String getCqlTypeName()
+ {
+ return String.format("%s.%s", ColumnIdentifier.maybeQuote(keyspace), ColumnIdentifier.maybeQuote(getNameAsString()));
+ }
+
@Override
public TypeSerializer<ByteBuffer> getSerializer()
{
return serializer;
}
+
+ @Override
+ public SchemaElementType elementType()
+ {
+ return SchemaElementType.TYPE;
+ }
+
+ @Override
+ public String elementKeyspace()
+ {
+ return keyspace;
+ }
+
+ @Override
+ public String elementName()
+ {
+ return getNameAsString();
+ }
+
+ @Override
+ public String toCqlString(boolean withInternals)
+ {
+ CqlBuilder builder = new CqlBuilder();
+ builder.append("CREATE TYPE ")
+ .appendQuotingIfNeeded(keyspace)
+ .append('.')
+ .appendQuotingIfNeeded(getNameAsString())
+ .append(" (")
+ .newLine()
+ .increaseIndent();
+
+ for (int i = 0; i < size(); i++)
+ {
+ if (i > 0)
+ builder.append(",")
+ .newLine();
+
+ builder.append(fieldNameAsString(i))
+ .append(' ')
+ .append(fieldType(i));
+ }
+
+ builder.newLine()
+ .decreaseIndent()
+ .append(");");
+
+ return builder.toString();
+ }
}
diff --git a/src/java/org/apache/cassandra/db/monitoring/ApproximateTime.java b/src/java/org/apache/cassandra/db/monitoring/ApproximateTime.java
deleted file mode 100644
index cc4b410..0000000
--- a/src/java/org/apache/cassandra/db/monitoring/ApproximateTime.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.db.monitoring;
-
-import java.util.concurrent.TimeUnit;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.cassandra.concurrent.ScheduledExecutors;
-import org.apache.cassandra.config.Config;
-
-/**
- * This is an approximation of System.currentTimeInMillis(). It updates its
- * time value at periodic intervals of CHECK_INTERVAL_MS milliseconds
- * (currently 10 milliseconds by default). It can be used as a faster alternative
- * to System.currentTimeInMillis() every time an imprecision of a few milliseconds
- * can be accepted.
- */
-public class ApproximateTime
-{
- private static final Logger logger = LoggerFactory.getLogger(ApproximateTime.class);
- private static final int CHECK_INTERVAL_MS = Math.max(5, Integer.parseInt(System.getProperty(Config.PROPERTY_PREFIX + "approximate_time_precision_ms", "10")));
-
- private static volatile long time = System.currentTimeMillis();
- static
- {
- logger.info("Scheduling approximate time-check task with a precision of {} milliseconds", CHECK_INTERVAL_MS);
- ScheduledExecutors.scheduledFastTasks.scheduleWithFixedDelay(() -> time = System.currentTimeMillis(),
- CHECK_INTERVAL_MS,
- CHECK_INTERVAL_MS,
- TimeUnit.MILLISECONDS);
- }
-
- public static long currentTimeMillis()
- {
- return time;
- }
-
- public static long precision()
- {
- return 2 * CHECK_INTERVAL_MS;
- }
-
-}
diff --git a/src/java/org/apache/cassandra/db/monitoring/Monitorable.java b/src/java/org/apache/cassandra/db/monitoring/Monitorable.java
index c9bf94e..10bd104 100644
--- a/src/java/org/apache/cassandra/db/monitoring/Monitorable.java
+++ b/src/java/org/apache/cassandra/db/monitoring/Monitorable.java
@@ -21,9 +21,9 @@
public interface Monitorable
{
String name();
- long constructionTime();
- long timeout();
- long slowTimeout();
+ long creationTimeNanos();
+ long timeoutNanos();
+ long slowTimeoutNanos();
boolean isInProgress();
boolean isAborted();
diff --git a/src/java/org/apache/cassandra/db/monitoring/MonitorableImpl.java b/src/java/org/apache/cassandra/db/monitoring/MonitorableImpl.java
index 48c8152..a6e7947 100644
--- a/src/java/org/apache/cassandra/db/monitoring/MonitorableImpl.java
+++ b/src/java/org/apache/cassandra/db/monitoring/MonitorableImpl.java
@@ -18,13 +18,15 @@
package org.apache.cassandra.db.monitoring;
+import static org.apache.cassandra.utils.MonotonicClock.approxTime;
+
public abstract class MonitorableImpl implements Monitorable
{
private MonitoringState state;
private boolean isSlow;
- private long constructionTime = -1;
- private long timeout;
- private long slowTimeout;
+ private long approxCreationTimeNanos = -1;
+ private long timeoutNanos;
+ private long slowTimeoutNanos;
private boolean isCrossNode;
protected MonitorableImpl()
@@ -38,23 +40,23 @@
* is too complex, it would require passing new parameters to all serializers
* or specializing the serializers to accept these message properties.
*/
- public void setMonitoringTime(long constructionTime, boolean isCrossNode, long timeout, long slowTimeout)
+ public void setMonitoringTime(long approxCreationTimeNanos, boolean isCrossNode, long timeoutNanos, long slowTimeoutNanos)
{
- assert constructionTime >= 0;
- this.constructionTime = constructionTime;
+ assert approxCreationTimeNanos >= 0;
+ this.approxCreationTimeNanos = approxCreationTimeNanos;
this.isCrossNode = isCrossNode;
- this.timeout = timeout;
- this.slowTimeout = slowTimeout;
+ this.timeoutNanos = timeoutNanos;
+ this.slowTimeoutNanos = slowTimeoutNanos;
}
- public long constructionTime()
+ public long creationTimeNanos()
{
- return constructionTime;
+ return approxCreationTimeNanos;
}
- public long timeout()
+ public long timeoutNanos()
{
- return timeout;
+ return timeoutNanos;
}
public boolean isCrossNode()
@@ -62,9 +64,9 @@
return isCrossNode;
}
- public long slowTimeout()
+ public long slowTimeoutNanos()
{
- return slowTimeout;
+ return slowTimeoutNanos;
}
public boolean isInProgress()
@@ -95,8 +97,8 @@
{
if (state == MonitoringState.IN_PROGRESS)
{
- if (constructionTime >= 0)
- MonitoringTask.addFailedOperation(this, ApproximateTime.currentTimeMillis());
+ if (approxCreationTimeNanos >= 0)
+ MonitoringTask.addFailedOperation(this, approxTime.now());
state = MonitoringState.ABORTED;
return true;
@@ -109,8 +111,8 @@
{
if (state == MonitoringState.IN_PROGRESS)
{
- if (isSlow && slowTimeout > 0 && constructionTime >= 0)
- MonitoringTask.addSlowOperation(this, ApproximateTime.currentTimeMillis());
+ if (isSlow && slowTimeoutNanos > 0 && approxCreationTimeNanos >= 0)
+ MonitoringTask.addSlowOperation(this, approxTime.now());
state = MonitoringState.COMPLETED;
return true;
@@ -121,15 +123,15 @@
private void check()
{
- if (constructionTime < 0 || state != MonitoringState.IN_PROGRESS)
+ if (approxCreationTimeNanos < 0 || state != MonitoringState.IN_PROGRESS)
return;
- long elapsed = ApproximateTime.currentTimeMillis() - constructionTime;
+ long minElapsedNanos = (approxTime.now() - approxCreationTimeNanos) - approxTime.error();
- if (elapsed >= slowTimeout && !isSlow)
+ if (minElapsedNanos >= slowTimeoutNanos && !isSlow)
isSlow = true;
- if (elapsed >= timeout)
+ if (minElapsedNanos >= timeoutNanos)
abort();
}
}
diff --git a/src/java/org/apache/cassandra/db/monitoring/MonitoringTask.java b/src/java/org/apache/cassandra/db/monitoring/MonitoringTask.java
index 9426042..0f8555f 100644
--- a/src/java/org/apache/cassandra/db/monitoring/MonitoringTask.java
+++ b/src/java/org/apache/cassandra/db/monitoring/MonitoringTask.java
@@ -39,6 +39,8 @@
import org.apache.cassandra.utils.NoSpamLogger;
import static java.lang.System.getProperty;
+import static java.util.concurrent.TimeUnit.NANOSECONDS;
+import static org.apache.cassandra.utils.MonotonicClock.approxTime;
/**
* A task for monitoring in progress operations, currently only read queries, and aborting them if they time out.
@@ -68,7 +70,7 @@
private final ScheduledFuture<?> reportingTask;
private final OperationsQueue failedOperationsQueue;
private final OperationsQueue slowOperationsQueue;
- private long lastLogTime;
+ private long approxLastLogTimeNanos;
@VisibleForTesting
@@ -88,10 +90,10 @@
this.failedOperationsQueue = new OperationsQueue(maxOperations);
this.slowOperationsQueue = new OperationsQueue(maxOperations);
- this.lastLogTime = ApproximateTime.currentTimeMillis();
+ this.approxLastLogTimeNanos = approxTime.now();
logger.info("Scheduling monitoring task with report interval of {} ms, max operations {}", reportIntervalMillis, maxOperations);
- this.reportingTask = ScheduledExecutors.scheduledTasks.scheduleWithFixedDelay(() -> logOperations(ApproximateTime.currentTimeMillis()),
+ this.reportingTask = ScheduledExecutors.scheduledTasks.scheduleWithFixedDelay(() -> logOperations(approxTime.now()),
reportIntervalMillis,
reportIntervalMillis,
TimeUnit.MILLISECONDS);
@@ -102,14 +104,14 @@
reportingTask.cancel(false);
}
- static void addFailedOperation(Monitorable operation, long now)
+ static void addFailedOperation(Monitorable operation, long nowNanos)
{
- instance.failedOperationsQueue.offer(new FailedOperation(operation, now));
+ instance.failedOperationsQueue.offer(new FailedOperation(operation, nowNanos));
}
- static void addSlowOperation(Monitorable operation, long now)
+ static void addSlowOperation(Monitorable operation, long nowNanos)
{
- instance.slowOperationsQueue.offer(new SlowOperation(operation, now));
+ instance.slowOperationsQueue.offer(new SlowOperation(operation, nowNanos));
}
@VisibleForTesting
@@ -131,27 +133,27 @@
}
@VisibleForTesting
- private void logOperations(long now)
+ private void logOperations(long approxCurrentTimeNanos)
{
- logSlowOperations(now);
- logFailedOperations(now);
+ logSlowOperations(approxCurrentTimeNanos);
+ logFailedOperations(approxCurrentTimeNanos);
- lastLogTime = now;
+ approxLastLogTimeNanos = approxCurrentTimeNanos;
}
@VisibleForTesting
- boolean logFailedOperations(long now)
+ boolean logFailedOperations(long nowNanos)
{
AggregatedOperations failedOperations = failedOperationsQueue.popOperations();
if (!failedOperations.isEmpty())
{
- long elapsed = now - lastLogTime;
+ long elapsedNanos = nowNanos - approxLastLogTimeNanos;
noSpamLogger.warn("Some operations timed out, details available at debug level (debug.log)");
if (logger.isDebugEnabled())
logger.debug("{} operations timed out in the last {} msecs:{}{}",
failedOperations.num(),
- elapsed,
+ NANOSECONDS.toMillis(elapsedNanos),
LINE_SEPARATOR,
failedOperations.getLogMessage());
return true;
@@ -161,18 +163,18 @@
}
@VisibleForTesting
- boolean logSlowOperations(long now)
+ boolean logSlowOperations(long approxCurrentTimeNanos)
{
AggregatedOperations slowOperations = slowOperationsQueue.popOperations();
if (!slowOperations.isEmpty())
{
- long elapsed = now - lastLogTime;
+ long approxElapsedNanos = approxCurrentTimeNanos - approxLastLogTimeNanos;
noSpamLogger.info("Some operations were slow, details available at debug level (debug.log)");
if (logger.isDebugEnabled())
logger.debug("{} operations were slow in the last {} msecs:{}{}",
slowOperations.num(),
- elapsed,
+ NANOSECONDS.toMillis(approxElapsedNanos),
LINE_SEPARATOR,
slowOperations.getLogMessage());
return true;
@@ -314,7 +316,7 @@
int numTimesReported;
/** The total time spent by this operation */
- long totalTime;
+ long totalTimeNanos;
/** The maximum time spent by this operation */
long maxTime;
@@ -326,13 +328,13 @@
* this is set lazily as it takes time to build the query CQL */
private String name;
- Operation(Monitorable operation, long failedAt)
+ Operation(Monitorable operation, long failedAtNanos)
{
this.operation = operation;
numTimesReported = 1;
- totalTime = failedAt - operation.constructionTime();
- minTime = totalTime;
- maxTime = totalTime;
+ totalTimeNanos = failedAtNanos - operation.creationTimeNanos();
+ minTime = totalTimeNanos;
+ maxTime = totalTimeNanos;
}
public String name()
@@ -345,7 +347,7 @@
void add(Operation operation)
{
numTimesReported++;
- totalTime += operation.totalTime;
+ totalTimeNanos += operation.totalTimeNanos;
maxTime = Math.max(maxTime, operation.maxTime);
minTime = Math.min(minTime, operation.minTime);
}
@@ -358,9 +360,9 @@
*/
private final static class FailedOperation extends Operation
{
- FailedOperation(Monitorable operation, long failedAt)
+ FailedOperation(Monitorable operation, long failedAtNanos)
{
- super(operation, failedAt);
+ super(operation, failedAtNanos);
}
public String getLogMessage()
@@ -368,17 +370,17 @@
if (numTimesReported == 1)
return String.format("<%s>, total time %d msec, timeout %d %s",
name(),
- totalTime,
- operation.timeout(),
+ NANOSECONDS.toMillis(totalTimeNanos),
+ NANOSECONDS.toMillis(operation.timeoutNanos()),
operation.isCrossNode() ? "msec/cross-node" : "msec");
else
return String.format("<%s> timed out %d times, avg/min/max %d/%d/%d msec, timeout %d %s",
name(),
numTimesReported,
- totalTime / numTimesReported,
- minTime,
- maxTime,
- operation.timeout(),
+ NANOSECONDS.toMillis(totalTimeNanos / numTimesReported),
+ NANOSECONDS.toMillis(minTime),
+ NANOSECONDS.toMillis(maxTime),
+ NANOSECONDS.toMillis(operation.timeoutNanos()),
operation.isCrossNode() ? "msec/cross-node" : "msec");
}
}
@@ -398,17 +400,17 @@
if (numTimesReported == 1)
return String.format("<%s>, time %d msec - slow timeout %d %s",
name(),
- totalTime,
- operation.slowTimeout(),
+ NANOSECONDS.toMillis(totalTimeNanos),
+ NANOSECONDS.toMillis(operation.slowTimeoutNanos()),
operation.isCrossNode() ? "msec/cross-node" : "msec");
else
return String.format("<%s>, was slow %d times: avg/min/max %d/%d/%d msec - slow timeout %d %s",
name(),
numTimesReported,
- totalTime / numTimesReported,
- minTime,
- maxTime,
- operation.slowTimeout(),
+ NANOSECONDS.toMillis(totalTimeNanos/ numTimesReported),
+ NANOSECONDS.toMillis(minTime),
+ NANOSECONDS.toMillis(maxTime),
+ NANOSECONDS.toMillis(operation.slowTimeoutNanos()),
operation.isCrossNode() ? "msec/cross-node" : "msec");
}
}
diff --git a/src/java/org/apache/cassandra/db/partitions/AbstractBTreePartition.java b/src/java/org/apache/cassandra/db/partitions/AbstractBTreePartition.java
index 34d6d46..fc50de6 100644
--- a/src/java/org/apache/cassandra/db/partitions/AbstractBTreePartition.java
+++ b/src/java/org/apache/cassandra/db/partitions/AbstractBTreePartition.java
@@ -20,42 +20,39 @@
import java.util.Iterator;
-import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.filter.ColumnFilter;
import org.apache.cassandra.db.rows.*;
import org.apache.cassandra.utils.SearchIterator;
import org.apache.cassandra.utils.btree.BTree;
-import org.apache.cassandra.utils.btree.BTreeSearchIterator;
import static org.apache.cassandra.utils.btree.BTree.Dir.desc;
public abstract class AbstractBTreePartition implements Partition, Iterable<Row>
{
- protected static final Holder EMPTY = new Holder(PartitionColumns.NONE, BTree.empty(), DeletionInfo.LIVE, Rows.EMPTY_STATIC_ROW, EncodingStats.NO_STATS);
+ protected static final Holder EMPTY = new Holder(RegularAndStaticColumns.NONE, BTree.empty(), DeletionInfo.LIVE, Rows.EMPTY_STATIC_ROW, EncodingStats.NO_STATS);
- protected final CFMetaData metadata;
protected final DecoratedKey partitionKey;
protected abstract Holder holder();
protected abstract boolean canHaveShadowedData();
- protected AbstractBTreePartition(CFMetaData metadata, DecoratedKey partitionKey)
+ protected AbstractBTreePartition(DecoratedKey partitionKey)
{
- this.metadata = metadata;
this.partitionKey = partitionKey;
}
protected static final class Holder
{
- final PartitionColumns columns;
+ final RegularAndStaticColumns columns;
final DeletionInfo deletionInfo;
// the btree of rows
final Object[] tree;
final Row staticRow;
final EncodingStats stats;
- Holder(PartitionColumns columns, Object[] tree, DeletionInfo deletionInfo, Row staticRow, EncodingStats stats)
+ Holder(RegularAndStaticColumns columns, Object[] tree, DeletionInfo deletionInfo, Row staticRow, EncodingStats stats)
{
this.columns = columns;
this.tree = tree;
@@ -87,10 +84,7 @@
return !BTree.isEmpty(holder.tree);
}
- public CFMetaData metadata()
- {
- return metadata;
- }
+ public abstract TableMetadata metadata();
public DecoratedKey partitionKey()
{
@@ -102,7 +96,7 @@
return deletionInfo().getPartitionDeletion();
}
- public PartitionColumns columns()
+ public RegularAndStaticColumns columns()
{
return holder().columns;
}
@@ -126,7 +120,7 @@
if (columns.fetchedColumns().statics.isEmpty() || (current.staticRow.isEmpty() && partitionDeletion.isLive()))
return Rows.EMPTY_STATIC_ROW;
- Row row = current.staticRow.filter(columns, partitionDeletion, setActiveDeletionToRow, metadata);
+ Row row = current.staticRow.filter(columns, partitionDeletion, setActiveDeletionToRow, metadata());
return row == null ? Rows.EMPTY_STATIC_ROW : row;
}
@@ -136,7 +130,7 @@
final Holder current = holder();
return new SearchIterator<Clustering, Row>()
{
- private final SearchIterator<Clustering, Row> rawIter = new BTreeSearchIterator<>(current.tree, metadata.comparator, desc(reversed));
+ private final SearchIterator<Clustering, Row> rawIter = BTree.slice(current.tree, metadata().comparator, desc(reversed));
private final DeletionTime partitionDeletion = current.deletionInfo.getPartitionDeletion();
public Row next(Clustering clustering)
@@ -164,7 +158,7 @@
return BTreeRow.emptyDeletedRow(clustering, Row.Deletion.regular(activeDeletion));
}
- return row.filter(columns, activeDeletion, true, metadata);
+ return row.filter(columns, activeDeletion, true, metadata());
}
};
}
@@ -185,7 +179,7 @@
if (slices.size() == 0)
{
DeletionTime partitionDeletion = current.deletionInfo.getPartitionDeletion();
- return UnfilteredRowIterators.noRowsIterator(metadata, partitionKey(), staticRow, partitionDeletion, reversed);
+ return UnfilteredRowIterators.noRowsIterator(metadata(), partitionKey(), staticRow, partitionDeletion, reversed);
}
return slices.size() == 1
@@ -197,7 +191,7 @@
{
ClusteringBound start = slice.start() == ClusteringBound.BOTTOM ? null : slice.start();
ClusteringBound end = slice.end() == ClusteringBound.TOP ? null : slice.end();
- Iterator<Row> rowIter = BTree.slice(current.tree, metadata.comparator, start, true, end, true, desc(reversed));
+ Iterator<Row> rowIter = BTree.slice(current.tree, metadata().comparator, start, true, end, true, desc(reversed));
Iterator<RangeTombstone> deleteIter = current.deletionInfo.rangeIterator(slice, reversed);
return merge(rowIter, deleteIter, selection, reversed, current, staticRow);
}
@@ -205,7 +199,7 @@
private RowAndDeletionMergeIterator merge(Iterator<Row> rowIter, Iterator<RangeTombstone> deleteIter,
ColumnFilter selection, boolean reversed, Holder current, Row staticRow)
{
- return new RowAndDeletionMergeIterator(metadata, partitionKey(), current.deletionInfo.getPartitionDeletion(),
+ return new RowAndDeletionMergeIterator(metadata(), partitionKey(), current.deletionInfo.getPartitionDeletion(),
selection, staticRow, reversed, current.stats,
rowIter, deleteIter,
canHaveShadowedData());
@@ -218,7 +212,7 @@
private AbstractIterator(Holder current, Row staticRow, ColumnFilter selection, boolean isReversed)
{
- super(AbstractBTreePartition.this.metadata,
+ super(AbstractBTreePartition.this.metadata(),
AbstractBTreePartition.this.partitionKey(),
current.deletionInfo.getPartitionDeletion(),
selection.fetchedColumns(), // non-selected columns will be filtered in subclasses by RowAndDeletionMergeIterator
@@ -273,18 +267,17 @@
protected static Holder build(UnfilteredRowIterator iterator, int initialRowCapacity)
{
- return build(iterator, initialRowCapacity, true, null);
+ return build(iterator, initialRowCapacity, true);
}
- protected static Holder build(UnfilteredRowIterator iterator, int initialRowCapacity, boolean ordered, BTree.Builder.QuickResolver<Row> quickResolver)
+ protected static Holder build(UnfilteredRowIterator iterator, int initialRowCapacity, boolean ordered)
{
- CFMetaData metadata = iterator.metadata();
- PartitionColumns columns = iterator.columns();
+ TableMetadata metadata = iterator.metadata();
+ RegularAndStaticColumns columns = iterator.columns();
boolean reversed = iterator.isReverseOrder();
BTree.Builder<Row> builder = BTree.builder(metadata.comparator, initialRowCapacity);
builder.auto(!ordered);
- builder.setQuickResolver(quickResolver);
MutableDeletionInfo.Builder deletionBuilder = MutableDeletionInfo.builder(iterator.partitionLevelDeletion(), metadata.comparator, reversed);
while (iterator.hasNext())
@@ -306,8 +299,8 @@
// passes a MutableDeletionInfo that it mutates later.
protected static Holder build(RowIterator rows, DeletionInfo deletion, boolean buildEncodingStats, int initialRowCapacity)
{
- CFMetaData metadata = rows.metadata();
- PartitionColumns columns = rows.columns();
+ TableMetadata metadata = rows.metadata();
+ RegularAndStaticColumns columns = rows.columns();
boolean reversed = rows.isReverseOrder();
BTree.Builder<Row> builder = BTree.builder(metadata.comparator, initialRowCapacity);
@@ -330,20 +323,19 @@
{
StringBuilder sb = new StringBuilder();
- sb.append(String.format("[%s.%s] key=%s partition_deletion=%s columns=%s",
- metadata.ksName,
- metadata.cfName,
- metadata.getKeyValidator().getString(partitionKey().getKey()),
+ sb.append(String.format("[%s] key=%s partition_deletion=%s columns=%s",
+ metadata(),
+ metadata().partitionKeyType.getString(partitionKey().getKey()),
partitionLevelDeletion(),
columns()));
if (staticRow() != Rows.EMPTY_STATIC_ROW)
- sb.append("\n ").append(staticRow().toString(metadata, true));
+ sb.append("\n ").append(staticRow().toString(metadata(), true));
try (UnfilteredRowIterator iter = unfilteredIterator())
{
while (iter.hasNext())
- sb.append("\n ").append(iter.next().toString(metadata, true));
+ sb.append("\n ").append(iter.next().toString(metadata(), true));
}
return sb.toString();
diff --git a/src/java/org/apache/cassandra/db/partitions/AtomicBTreePartition.java b/src/java/org/apache/cassandra/db/partitions/AtomicBTreePartition.java
index a588303..486bec7 100644
--- a/src/java/org/apache/cassandra/db/partitions/AtomicBTreePartition.java
+++ b/src/java/org/apache/cassandra/db/partitions/AtomicBTreePartition.java
@@ -24,18 +24,17 @@
import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
import java.util.concurrent.atomic.AtomicReferenceFieldUpdater;
-import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.schema.TableMetadataRef;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.filter.ColumnFilter;
import org.apache.cassandra.db.rows.*;
import org.apache.cassandra.index.transactions.UpdateTransaction;
-import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.ObjectSizes;
import org.apache.cassandra.utils.SearchIterator;
import org.apache.cassandra.utils.btree.BTree;
import org.apache.cassandra.utils.btree.UpdateFunction;
-import org.apache.cassandra.utils.concurrent.Locks;
import org.apache.cassandra.utils.concurrent.OpOrder;
import org.apache.cassandra.utils.memory.HeapAllocator;
import org.apache.cassandra.utils.memory.MemtableAllocator;
@@ -48,9 +47,9 @@
* other thread can see the state where only parts but not all rows have
* been added.
*/
-public class AtomicBTreePartition extends AbstractBTreePartition
+public final class AtomicBTreePartition extends AbstractBTreePartition
{
- public static final long EMPTY_SIZE = ObjectSizes.measure(new AtomicBTreePartition(CFMetaData.createFake("keyspace", "table"),
+ public static final long EMPTY_SIZE = ObjectSizes.measure(new AtomicBTreePartition(null,
DatabaseDescriptor.getPartitioner().decorateKey(ByteBuffer.allocate(1)),
null));
@@ -83,10 +82,13 @@
private final MemtableAllocator allocator;
private volatile Holder ref;
- public AtomicBTreePartition(CFMetaData metadata, DecoratedKey partitionKey, MemtableAllocator allocator)
+ private final TableMetadataRef metadata;
+
+ public AtomicBTreePartition(TableMetadataRef metadata, DecoratedKey partitionKey, MemtableAllocator allocator)
{
// involved in potential bug? partition columns may be a subset if we alter columns while it's in memtable
- super(metadata, partitionKey);
+ super(partitionKey);
+ this.metadata = metadata;
this.allocator = allocator;
this.ref = EMPTY;
}
@@ -96,11 +98,60 @@
return ref;
}
+ public TableMetadata metadata()
+ {
+ return metadata.get();
+ }
+
protected boolean canHaveShadowedData()
{
return true;
}
+ private long[] addAllWithSizeDeltaInternal(RowUpdater updater, PartitionUpdate update, UpdateTransaction indexer)
+ {
+ Holder current = ref;
+ updater.ref = current;
+ updater.reset();
+
+ if (!update.deletionInfo().getPartitionDeletion().isLive())
+ indexer.onPartitionDeletion(update.deletionInfo().getPartitionDeletion());
+
+ if (update.deletionInfo().hasRanges())
+ update.deletionInfo().rangeIterator(false).forEachRemaining(indexer::onRangeTombstone);
+
+ DeletionInfo deletionInfo;
+ if (update.deletionInfo().mayModify(current.deletionInfo))
+ {
+ if (updater.inputDeletionInfoCopy == null)
+ updater.inputDeletionInfoCopy = update.deletionInfo().copy(HeapAllocator.instance);
+
+ deletionInfo = current.deletionInfo.mutableCopy().add(updater.inputDeletionInfoCopy);
+ updater.allocated(deletionInfo.unsharedHeapSize() - current.deletionInfo.unsharedHeapSize());
+ }
+ else
+ {
+ deletionInfo = current.deletionInfo;
+ }
+
+ RegularAndStaticColumns columns = update.columns().mergeTo(current.columns);
+ Row newStatic = update.staticRow();
+ Row staticRow = newStatic.isEmpty()
+ ? current.staticRow
+ : (current.staticRow.isEmpty() ? updater.apply(newStatic) : updater.apply(current.staticRow, newStatic));
+ Object[] tree = BTree.update(current.tree, update.metadata().comparator, update, update.rowCount(), updater);
+ EncodingStats newStats = current.stats.mergeWith(update.stats());
+
+ if (tree != null && refUpdater.compareAndSet(this, current, new Holder(columns, tree, deletionInfo, staticRow, newStats)))
+ {
+ updater.finish();
+ return new long[]{ updater.dataSize, updater.colUpdateTimeDelta };
+ }
+ else
+ {
+ return null;
+ }
+ }
/**
* Adds a given update to this in-memtable partition.
*
@@ -110,63 +161,35 @@
public long[] addAllWithSizeDelta(final PartitionUpdate update, OpOrder.Group writeOp, UpdateTransaction indexer)
{
RowUpdater updater = new RowUpdater(this, allocator, writeOp, indexer);
- DeletionInfo inputDeletionInfoCopy = null;
- boolean monitorOwned = false;
try
{
- monitorOwned = maybeLock(writeOp);
+ boolean shouldLock = shouldLock(writeOp);
indexer.start();
while (true)
{
- Holder current = ref;
- updater.ref = current;
- updater.reset();
-
- if (!update.deletionInfo().getPartitionDeletion().isLive())
- indexer.onPartitionDeletion(update.deletionInfo().getPartitionDeletion());
-
- if (update.deletionInfo().hasRanges())
- update.deletionInfo().rangeIterator(false).forEachRemaining(indexer::onRangeTombstone);
-
- DeletionInfo deletionInfo;
- if (update.deletionInfo().mayModify(current.deletionInfo))
+ if (shouldLock)
{
- if (inputDeletionInfoCopy == null)
- inputDeletionInfoCopy = update.deletionInfo().copy(HeapAllocator.instance);
-
- deletionInfo = current.deletionInfo.mutableCopy().add(inputDeletionInfoCopy);
- updater.allocated(deletionInfo.unsharedHeapSize() - current.deletionInfo.unsharedHeapSize());
+ synchronized (this)
+ {
+ long[] result = addAllWithSizeDeltaInternal(updater, update, indexer);
+ if (result != null)
+ return result;
+ }
}
else
{
- deletionInfo = current.deletionInfo;
- }
+ long[] result = addAllWithSizeDeltaInternal(updater, update, indexer);
+ if (result != null)
+ return result;
- PartitionColumns columns = update.columns().mergeTo(current.columns);
- Row newStatic = update.staticRow();
- Row staticRow = newStatic.isEmpty()
- ? current.staticRow
- : (current.staticRow.isEmpty() ? updater.apply(newStatic) : updater.apply(current.staticRow, newStatic));
- Object[] tree = BTree.update(current.tree, update.metadata().comparator, update, update.rowCount(), updater);
- EncodingStats newStats = current.stats.mergeWith(update.stats());
-
- if (tree != null && refUpdater.compareAndSet(this, current, new Holder(columns, tree, deletionInfo, staticRow, newStats)))
- {
- updater.finish();
- return new long[]{ updater.dataSize, updater.colUpdateTimeDelta };
- }
- else if (!monitorOwned)
- {
- monitorOwned = maybeLock(updater.heapSize, writeOp);
+ shouldLock = shouldLock(updater.heapSize, writeOp);
}
}
}
finally
{
indexer.commit();
- if (monitorOwned)
- Locks.monitorExitUnsafe(this);
}
}
@@ -230,7 +253,7 @@
return allocator.ensureOnHeap().applyToPartition(super.iterator());
}
- private boolean maybeLock(OpOrder.Group writeOp)
+ private boolean shouldLock(OpOrder.Group writeOp)
{
if (!useLock())
return false;
@@ -238,7 +261,7 @@
return lockIfOldest(writeOp);
}
- private boolean maybeLock(long addWaste, OpOrder.Group writeOp)
+ private boolean shouldLock(long addWaste, OpOrder.Group writeOp)
{
if (!updateWastedAllocationTracker(addWaste))
return false;
@@ -255,7 +278,6 @@
return false;
}
- Locks.monitorEnterUnsafe(this);
return true;
}
@@ -313,7 +335,6 @@
final MemtableAllocator allocator;
final OpOrder.Group writeOp;
final UpdateTransaction indexer;
- final int nowInSec;
Holder ref;
Row.Builder regularBuilder;
long dataSize;
@@ -321,13 +342,14 @@
long colUpdateTimeDelta = Long.MAX_VALUE;
List<Row> inserted; // TODO: replace with walk of aborted BTree
+ DeletionInfo inputDeletionInfoCopy = null;
+
private RowUpdater(AtomicBTreePartition updating, MemtableAllocator allocator, OpOrder.Group writeOp, UpdateTransaction indexer)
{
this.updating = updating;
this.allocator = allocator;
this.writeOp = writeOp;
this.indexer = indexer;
- this.nowInSec = FBUtilities.nowInSeconds();
}
private Row.Builder builder(Clustering clustering)
@@ -358,7 +380,7 @@
public Row apply(Row existing, Row update)
{
Row.Builder builder = builder(existing.clustering());
- colUpdateTimeDelta = Math.min(colUpdateTimeDelta, Rows.merge(existing, update, builder, nowInSec));
+ colUpdateTimeDelta = Math.min(colUpdateTimeDelta, Rows.merge(existing, update, builder));
Row reconciled = builder.build();
diff --git a/src/java/org/apache/cassandra/db/partitions/CachedBTreePartition.java b/src/java/org/apache/cassandra/db/partitions/CachedBTreePartition.java
index 9c6ab59..9a2b331 100644
--- a/src/java/org/apache/cassandra/db/partitions/CachedBTreePartition.java
+++ b/src/java/org/apache/cassandra/db/partitions/CachedBTreePartition.java
@@ -19,7 +19,6 @@
import java.io.IOException;
-import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.filter.DataLimits;
import org.apache.cassandra.db.rows.*;
@@ -27,6 +26,9 @@
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.utils.btree.BTree;
public class CachedBTreePartition extends ImmutableBTreePartition implements CachedPartition
@@ -36,24 +38,17 @@
private final int cachedLiveRows;
private final int rowsWithNonExpiringCells;
- private final int nonTombstoneCellCount;
- private final int nonExpiringLiveCells;
-
- private CachedBTreePartition(CFMetaData metadata,
+ private CachedBTreePartition(TableMetadata metadata,
DecoratedKey partitionKey,
Holder holder,
int createdAtInSec,
int cachedLiveRows,
- int rowsWithNonExpiringCells,
- int nonTombstoneCellCount,
- int nonExpiringLiveCells)
+ int rowsWithNonExpiringCells)
{
super(metadata, partitionKey, holder);
this.createdAtInSec = createdAtInSec;
this.cachedLiveRows = cachedLiveRows;
this.rowsWithNonExpiringCells = rowsWithNonExpiringCells;
- this.nonTombstoneCellCount = nonTombstoneCellCount;
- this.nonExpiringLiveCells = nonExpiringLiveCells;
}
/**
@@ -89,8 +84,6 @@
int cachedLiveRows = 0;
int rowsWithNonExpiringCells = 0;
- int nonTombstoneCellCount = 0;
- int nonExpiringLiveCells = 0;
boolean enforceStrictLiveness = iterator.metadata().enforceStrictLiveness();
for (Row row : BTree.<Row>iterable(holder.tree))
@@ -98,22 +91,18 @@
if (row.hasLiveData(nowInSec, enforceStrictLiveness))
++cachedLiveRows;
- int nonExpiringLiveCellsThisRow = 0;
+ boolean hasNonExpiringLiveCell = false;
for (Cell cell : row.cells())
{
- if (!cell.isTombstone())
+ if (!cell.isTombstone() && !cell.isExpiring())
{
- ++nonTombstoneCellCount;
- if (!cell.isExpiring())
- ++nonExpiringLiveCellsThisRow;
+ hasNonExpiringLiveCell = true;
+ break;
}
}
- if (nonExpiringLiveCellsThisRow > 0)
- {
+ if (hasNonExpiringLiveCell)
++rowsWithNonExpiringCells;
- nonExpiringLiveCells += nonExpiringLiveCellsThisRow;
- }
}
return new CachedBTreePartition(iterator.metadata(),
@@ -121,9 +110,7 @@
holder,
nowInSec,
cachedLiveRows,
- rowsWithNonExpiringCells,
- nonTombstoneCellCount,
- nonExpiringLiveCells);
+ rowsWithNonExpiringCells);
}
/**
@@ -154,16 +141,6 @@
return rowsWithNonExpiringCells;
}
- public int nonTombstoneCellCount()
- {
- return nonTombstoneCellCount;
- }
-
- public int nonExpiringLiveCells()
- {
- return nonExpiringLiveCells;
- }
-
static class Serializer implements ISerializer<CachedPartition>
{
public void serialize(CachedPartition partition, DataOutputPlus out) throws IOException
@@ -176,9 +153,7 @@
out.writeInt(p.createdAtInSec);
out.writeInt(p.cachedLiveRows);
out.writeInt(p.rowsWithNonExpiringCells);
- out.writeInt(p.nonTombstoneCellCount);
- out.writeInt(p.nonExpiringLiveCells);
- CFMetaData.serializer.serialize(partition.metadata(), out, version);
+ partition.metadata().id.serialize(out);
try (UnfilteredRowIterator iter = p.unfilteredIterator())
{
UnfilteredRowIteratorSerializer.serializer.serialize(iter, null, out, version, p.rowCount());
@@ -199,28 +174,24 @@
int createdAtInSec = in.readInt();
int cachedLiveRows = in.readInt();
int rowsWithNonExpiringCells = in.readInt();
- int nonTombstoneCellCount = in.readInt();
- int nonExpiringLiveCells = in.readInt();
- CFMetaData metadata = CFMetaData.serializer.deserialize(in, version);
- UnfilteredRowIteratorSerializer.Header header = UnfilteredRowIteratorSerializer.serializer.deserializeHeader(metadata, null, in, version, SerializationHelper.Flag.LOCAL);
+ TableMetadata metadata = Schema.instance.getExistingTableMetadata(TableId.deserialize(in));
+ UnfilteredRowIteratorSerializer.Header header = UnfilteredRowIteratorSerializer.serializer.deserializeHeader(metadata, null, in, version, DeserializationHelper.Flag.LOCAL);
assert !header.isReversed && header.rowEstimate >= 0;
Holder holder;
- try (UnfilteredRowIterator partition = UnfilteredRowIteratorSerializer.serializer.deserialize(in, version, metadata, SerializationHelper.Flag.LOCAL, header))
+ try (UnfilteredRowIterator partition = UnfilteredRowIteratorSerializer.serializer.deserialize(in, version, metadata, DeserializationHelper.Flag.LOCAL, header))
{
holder = ImmutableBTreePartition.build(partition, header.rowEstimate);
}
return new CachedBTreePartition(metadata,
- header.key,
- holder,
- createdAtInSec,
- cachedLiveRows,
- rowsWithNonExpiringCells,
- nonTombstoneCellCount,
- nonExpiringLiveCells);
+ header.key,
+ holder,
+ createdAtInSec,
+ cachedLiveRows,
+ rowsWithNonExpiringCells);
}
@@ -236,9 +207,7 @@
return TypeSizes.sizeof(p.createdAtInSec)
+ TypeSizes.sizeof(p.cachedLiveRows)
+ TypeSizes.sizeof(p.rowsWithNonExpiringCells)
- + TypeSizes.sizeof(p.nonTombstoneCellCount)
- + TypeSizes.sizeof(p.nonExpiringLiveCells)
- + CFMetaData.serializer.serializedSize(partition.metadata(), version)
+ + partition.metadata().id.serializedSize()
+ UnfilteredRowIteratorSerializer.serializer.serializedSize(iter, null, MessagingService.current_version, p.rowCount());
}
}
diff --git a/src/java/org/apache/cassandra/db/partitions/CachedPartition.java b/src/java/org/apache/cassandra/db/partitions/CachedPartition.java
index 0cbaba0..6c781f5 100644
--- a/src/java/org/apache/cassandra/db/partitions/CachedPartition.java
+++ b/src/java/org/apache/cassandra/db/partitions/CachedPartition.java
@@ -71,24 +71,4 @@
* @return the last row of the partition, or {@code null} if the partition is empty.
*/
public Row lastRow();
-
- /**
- * The number of {@code cell} objects that are not tombstone in this cached partition.
- *
- * Please note that this is <b>not</b> the number of <em>live</em> cells since
- * some of the cells might be expired.
- *
- * @return the number of non tombstone cells in the partition.
- */
- public int nonTombstoneCellCount();
-
- /**
- * The number of cells in this cached partition that are neither tombstone nor expiring.
- *
- * Note that this is generally not a very meaningful number, but this is used by
- * {@link org.apache.cassandra.db.filter.DataLimits#hasEnoughLiveData} as an optimization.
- *
- * @return the number of cells that are neither tombstones nor expiring.
- */
- public int nonExpiringLiveCells();
}
diff --git a/src/java/org/apache/cassandra/db/partitions/FilteredPartition.java b/src/java/org/apache/cassandra/db/partitions/FilteredPartition.java
index 70a4678..5730076 100644
--- a/src/java/org/apache/cassandra/db/partitions/FilteredPartition.java
+++ b/src/java/org/apache/cassandra/db/partitions/FilteredPartition.java
@@ -19,10 +19,10 @@
import java.util.Iterator;
-import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.DeletionInfo;
-import org.apache.cassandra.db.PartitionColumns;
+import org.apache.cassandra.db.RegularAndStaticColumns;
import org.apache.cassandra.db.rows.*;
public class FilteredPartition extends ImmutableBTreePartition
@@ -48,9 +48,9 @@
final Iterator<Row> iter = iterator();
return new RowIterator()
{
- public CFMetaData metadata()
+ public TableMetadata metadata()
{
- return metadata;
+ return FilteredPartition.this.metadata();
}
public boolean isReverseOrder()
@@ -58,7 +58,7 @@
return false;
}
- public PartitionColumns columns()
+ public RegularAndStaticColumns columns()
{
return FilteredPartition.this.columns();
}
diff --git a/src/java/org/apache/cassandra/db/partitions/ImmutableBTreePartition.java b/src/java/org/apache/cassandra/db/partitions/ImmutableBTreePartition.java
index 8db5ee4..5139d40 100644
--- a/src/java/org/apache/cassandra/db/partitions/ImmutableBTreePartition.java
+++ b/src/java/org/apache/cassandra/db/partitions/ImmutableBTreePartition.java
@@ -18,34 +18,37 @@
*/
package org.apache.cassandra.db.partitions;
-import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.DeletionInfo;
-import org.apache.cassandra.db.PartitionColumns;
+import org.apache.cassandra.db.RegularAndStaticColumns;
import org.apache.cassandra.db.rows.*;
public class ImmutableBTreePartition extends AbstractBTreePartition
{
protected final Holder holder;
+ protected final TableMetadata metadata;
- public ImmutableBTreePartition(CFMetaData metadata,
- DecoratedKey partitionKey,
- PartitionColumns columns,
- Row staticRow,
- Object[] tree,
- DeletionInfo deletionInfo,
- EncodingStats stats)
+ public ImmutableBTreePartition(TableMetadata metadata,
+ DecoratedKey partitionKey,
+ RegularAndStaticColumns columns,
+ Row staticRow,
+ Object[] tree,
+ DeletionInfo deletionInfo,
+ EncodingStats stats)
{
- super(metadata, partitionKey);
+ super(partitionKey);
+ this.metadata = metadata;
this.holder = new Holder(columns, tree, deletionInfo, staticRow, stats);
}
- protected ImmutableBTreePartition(CFMetaData metadata,
+ protected ImmutableBTreePartition(TableMetadata metadata,
DecoratedKey partitionKey,
Holder holder)
{
- super(metadata, partitionKey);
+ super(partitionKey);
+ this.metadata = metadata;
this.holder = holder;
}
@@ -108,7 +111,12 @@
*/
public static ImmutableBTreePartition create(UnfilteredRowIterator iterator, int initialRowCapacity, boolean ordered)
{
- return new ImmutableBTreePartition(iterator.metadata(), iterator.partitionKey(), build(iterator, initialRowCapacity, ordered, null));
+ return new ImmutableBTreePartition(iterator.metadata(), iterator.partitionKey(), build(iterator, initialRowCapacity, ordered));
+ }
+
+ public TableMetadata metadata()
+ {
+ return metadata;
}
protected Holder holder()
diff --git a/src/java/org/apache/cassandra/db/partitions/Partition.java b/src/java/org/apache/cassandra/db/partitions/Partition.java
index 04568e9..baeb6d5 100644
--- a/src/java/org/apache/cassandra/db/partitions/Partition.java
+++ b/src/java/org/apache/cassandra/db/partitions/Partition.java
@@ -17,7 +17,7 @@
*/
package org.apache.cassandra.db.partitions;
-import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.db.Slices;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.rows.*;
@@ -34,11 +34,11 @@
*/
public interface Partition
{
- public CFMetaData metadata();
+ public TableMetadata metadata();
public DecoratedKey partitionKey();
public DeletionTime partitionLevelDeletion();
- public PartitionColumns columns();
+ public RegularAndStaticColumns columns();
public EncodingStats stats();
diff --git a/src/java/org/apache/cassandra/db/partitions/PartitionIterators.java b/src/java/org/apache/cassandra/db/partitions/PartitionIterators.java
index f88cf76..0c4084f 100644
--- a/src/java/org/apache/cassandra/db/partitions/PartitionIterators.java
+++ b/src/java/org/apache/cassandra/db/partitions/PartitionIterators.java
@@ -20,12 +20,11 @@
import java.util.*;
import org.apache.cassandra.db.EmptyIterators;
-import org.apache.cassandra.db.transform.FilteredPartitions;
import org.apache.cassandra.db.transform.MorePartitions;
import org.apache.cassandra.db.transform.Transformation;
import org.apache.cassandra.utils.AbstractIterator;
-import org.apache.cassandra.db.SinglePartitionReadCommand;
+import org.apache.cassandra.db.SinglePartitionReadQuery;
import org.apache.cassandra.db.rows.*;
public abstract class PartitionIterators
@@ -33,15 +32,15 @@
private PartitionIterators() {}
@SuppressWarnings("resource") // The created resources are returned right away
- public static RowIterator getOnlyElement(final PartitionIterator iter, SinglePartitionReadCommand command)
+ public static RowIterator getOnlyElement(final PartitionIterator iter, SinglePartitionReadQuery query)
{
// If the query has no results, we'll get an empty iterator, but we still
// want a RowIterator out of this method, so we return an empty one.
RowIterator toReturn = iter.hasNext()
? iter.next()
- : EmptyIterators.row(command.metadata(),
- command.partitionKey(),
- command.clusteringIndexFilter().isReversed());
+ : EmptyIterators.row(query.metadata(),
+ query.partitionKey(),
+ query.clusteringIndexFilter().isReversed());
// Note that in general, we should wrap the result so that it's close method actually
// close the whole PartitionIterator.
diff --git a/src/java/org/apache/cassandra/db/partitions/PartitionUpdate.java b/src/java/org/apache/cassandra/db/partitions/PartitionUpdate.java
index 4aca6d2..076c975 100644
--- a/src/java/org/apache/cassandra/db/partitions/PartitionUpdate.java
+++ b/src/java/org/apache/cassandra/db/partitions/PartitionUpdate.java
@@ -20,24 +20,22 @@
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
-import java.util.Iterator;
import java.util.List;
-import java.util.concurrent.TimeUnit;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.filter.ColumnFilter;
import org.apache.cassandra.db.rows.*;
+import org.apache.cassandra.index.IndexRegistry;
import org.apache.cassandra.io.util.*;
-import org.apache.cassandra.net.MessagingService;
-import org.apache.cassandra.utils.FBUtilities;
-import org.apache.cassandra.utils.NoSpamLogger;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.utils.btree.BTree;
import org.apache.cassandra.utils.btree.UpdateFunction;
@@ -61,67 +59,25 @@
public static final PartitionUpdateSerializer serializer = new PartitionUpdateSerializer();
- private final int createdAtInSec = FBUtilities.nowInSeconds();
-
- // Records whether this update is "built", i.e. if the build() method has been called, which
- // happens when the update is read. Further writing is then rejected though a manual call
- // to allowNewUpdates() allow new writes. We could make that more implicit but only triggers
- // really requires that so we keep it simple for now).
- private volatile boolean isBuilt;
- private boolean canReOpen = true;
-
- private Holder holder;
- private BTree.Builder<Row> rowBuilder;
- private MutableDeletionInfo deletionInfo;
+ private final Holder holder;
+ private final DeletionInfo deletionInfo;
+ private final TableMetadata metadata;
private final boolean canHaveShadowedData;
- private PartitionUpdate(CFMetaData metadata,
- DecoratedKey key,
- PartitionColumns columns,
- MutableDeletionInfo deletionInfo,
- int initialRowCapacity,
- boolean canHaveShadowedData)
- {
- super(metadata, key);
- this.deletionInfo = deletionInfo;
- this.holder = new Holder(columns, BTree.empty(), deletionInfo, Rows.EMPTY_STATIC_ROW, EncodingStats.NO_STATS);
- this.canHaveShadowedData = canHaveShadowedData;
- rowBuilder = builder(initialRowCapacity);
- }
-
- private PartitionUpdate(CFMetaData metadata,
+ private PartitionUpdate(TableMetadata metadata,
DecoratedKey key,
Holder holder,
MutableDeletionInfo deletionInfo,
boolean canHaveShadowedData)
{
- super(metadata, key);
+ super(key);
+ this.metadata = metadata;
this.holder = holder;
this.deletionInfo = deletionInfo;
- this.isBuilt = true;
this.canHaveShadowedData = canHaveShadowedData;
}
- public PartitionUpdate(CFMetaData metadata,
- DecoratedKey key,
- PartitionColumns columns,
- int initialRowCapacity)
- {
- this(metadata, key, columns, MutableDeletionInfo.live(), initialRowCapacity, true);
- }
-
- public PartitionUpdate(CFMetaData metadata,
- ByteBuffer key,
- PartitionColumns columns,
- int initialRowCapacity)
- {
- this(metadata,
- metadata.decorateKey(key),
- columns,
- initialRowCapacity);
- }
-
/**
* Creates a empty immutable partition update.
*
@@ -130,10 +86,10 @@
*
* @return the newly created empty (and immutable) update.
*/
- public static PartitionUpdate emptyUpdate(CFMetaData metadata, DecoratedKey key)
+ public static PartitionUpdate emptyUpdate(TableMetadata metadata, DecoratedKey key)
{
MutableDeletionInfo deletionInfo = MutableDeletionInfo.live();
- Holder holder = new Holder(PartitionColumns.NONE, BTree.empty(), deletionInfo, Rows.EMPTY_STATIC_ROW, EncodingStats.NO_STATS);
+ Holder holder = new Holder(RegularAndStaticColumns.NONE, BTree.empty(), deletionInfo, Rows.EMPTY_STATIC_ROW, EncodingStats.NO_STATS);
return new PartitionUpdate(metadata, key, holder, deletionInfo, false);
}
@@ -147,10 +103,10 @@
*
* @return the newly created partition deletion update.
*/
- public static PartitionUpdate fullPartitionDelete(CFMetaData metadata, DecoratedKey key, long timestamp, int nowInSec)
+ public static PartitionUpdate fullPartitionDelete(TableMetadata metadata, DecoratedKey key, long timestamp, int nowInSec)
{
MutableDeletionInfo deletionInfo = new MutableDeletionInfo(timestamp, nowInSec);
- Holder holder = new Holder(PartitionColumns.NONE, BTree.empty(), deletionInfo, Rows.EMPTY_STATIC_ROW, EncodingStats.NO_STATS);
+ Holder holder = new Holder(RegularAndStaticColumns.NONE, BTree.empty(), deletionInfo, Rows.EMPTY_STATIC_ROW, EncodingStats.NO_STATS);
return new PartitionUpdate(metadata, key, holder, deletionInfo, false);
}
@@ -164,11 +120,11 @@
*
* @return the newly created partition update containing only {@code row}.
*/
- public static PartitionUpdate singleRowUpdate(CFMetaData metadata, DecoratedKey key, Row row, Row staticRow)
+ public static PartitionUpdate singleRowUpdate(TableMetadata metadata, DecoratedKey key, Row row, Row staticRow)
{
MutableDeletionInfo deletionInfo = MutableDeletionInfo.live();
Holder holder = new Holder(
- new PartitionColumns(
+ new RegularAndStaticColumns(
staticRow == null ? Columns.NONE : Columns.from(staticRow.columns()),
row == null ? Columns.NONE : Columns.from(row.columns())
),
@@ -189,7 +145,7 @@
*
* @return the newly created partition update containing only {@code row}.
*/
- public static PartitionUpdate singleRowUpdate(CFMetaData metadata, DecoratedKey key, Row row)
+ public static PartitionUpdate singleRowUpdate(TableMetadata metadata, DecoratedKey key, Row row)
{
return singleRowUpdate(metadata, key, row.isStatic() ? null : row, row.isStatic() ? row : null);
}
@@ -203,9 +159,9 @@
*
* @return the newly created partition update containing only {@code row}.
*/
- public static PartitionUpdate singleRowUpdate(CFMetaData metadata, ByteBuffer key, Row row)
+ public static PartitionUpdate singleRowUpdate(TableMetadata metadata, ByteBuffer key, Row row)
{
- return singleRowUpdate(metadata, metadata.decorateKey(key), row);
+ return singleRowUpdate(metadata, metadata.partitioner.decorateKey(key), row);
}
/**
@@ -219,30 +175,11 @@
* Warning: this method does not close the provided iterator, it is up to
* the caller to close it.
*/
+ @SuppressWarnings("resource")
public static PartitionUpdate fromIterator(UnfilteredRowIterator iterator, ColumnFilter filter)
{
-
- return fromIterator(iterator, filter, true, null);
- }
-
- private static final NoSpamLogger rowMergingLogger = NoSpamLogger.getLogger(logger, 1, TimeUnit.MINUTES);
- /**
- * Removes duplicate rows from incoming iterator, to be used when we can't trust the underlying iterator (like when reading legacy sstables)
- */
- public static PartitionUpdate fromPre30Iterator(UnfilteredRowIterator iterator, ColumnFilter filter)
- {
- return fromIterator(iterator, filter, false, (a, b) -> {
- CFMetaData cfm = iterator.metadata();
- rowMergingLogger.warn(String.format("Merging rows from pre 3.0 iterator for partition key: %s",
- cfm.getKeyValidator().getString(iterator.partitionKey().getKey())));
- return Rows.merge(a, b, FBUtilities.nowInSeconds());
- });
- }
-
- private static PartitionUpdate fromIterator(UnfilteredRowIterator iterator, ColumnFilter filter, boolean ordered, BTree.Builder.QuickResolver<Row> quickResolver)
- {
iterator = UnfilteredRowIterators.withOnlyQueriedData(iterator, filter);
- Holder holder = build(iterator, 16, ordered, quickResolver);
+ Holder holder = build(iterator, 16);
MutableDeletionInfo deletionInfo = (MutableDeletionInfo) holder.deletionInfo;
return new PartitionUpdate(iterator.metadata(), iterator.partitionKey(), holder, deletionInfo, false);
}
@@ -258,6 +195,7 @@
* Warning: this method does not close the provided iterator, it is up to
* the caller to close it.
*/
+ @SuppressWarnings("resource")
public static PartitionUpdate fromIterator(RowIterator iterator, ColumnFilter filter)
{
iterator = RowIterators.withOnlyQueriedData(iterator, filter);
@@ -276,12 +214,11 @@
*
* @param bytes the byte buffer that contains the serialized update.
* @param version the version with which the update is serialized.
- * @param key the partition key for the update. This is only used if {@code version < 3.0}
- * and can be {@code null} otherwise.
*
* @return the deserialized update or {@code null} if {@code bytes == null}.
*/
- public static PartitionUpdate fromBytes(ByteBuffer bytes, int version, DecoratedKey key)
+ @SuppressWarnings("resource")
+ public static PartitionUpdate fromBytes(ByteBuffer bytes, int version)
{
if (bytes == null)
return null;
@@ -290,8 +227,7 @@
{
return serializer.deserialize(new DataInputBuffer(bytes, true),
version,
- SerializationHelper.Flag.LOCAL,
- version < MessagingService.VERSION_30 ? key : null);
+ DeserializationHelper.Flag.LOCAL);
}
catch (IOException e)
{
@@ -330,9 +266,9 @@
*
* @return the newly created partition deletion update.
*/
- public static PartitionUpdate fullPartitionDelete(CFMetaData metadata, ByteBuffer key, long timestamp, int nowInSec)
+ public static PartitionUpdate fullPartitionDelete(TableMetadata metadata, ByteBuffer key, long timestamp, int nowInSec)
{
- return fullPartitionDelete(metadata, metadata.decorateKey(key), timestamp, nowInSec);
+ return fullPartitionDelete(metadata, metadata.partitioner.decorateKey(key), timestamp, nowInSec);
}
/**
@@ -350,9 +286,8 @@
if (size == 1)
return Iterables.getOnlyElement(updates);
- int nowInSecs = FBUtilities.nowInSeconds();
List<UnfilteredRowIterator> asIterators = Lists.transform(updates, AbstractBTreePartition::unfilteredIterator);
- return fromIterator(UnfilteredRowIterators.merge(asIterators, nowInSecs), ColumnFilter.all(updates.get(0).metadata()));
+ return fromIterator(UnfilteredRowIterators.merge(asIterators), ColumnFilter.all(updates.get(0).metadata()));
}
// We override this, because the version in the super-class calls holder(), which build the update preventing
@@ -365,30 +300,6 @@
}
/**
- * Modify this update to set every timestamp for live data to {@code newTimestamp} and
- * every deletion timestamp to {@code newTimestamp - 1}.
- *
- * There is no reason to use that expect on the Paxos code path, where we need ensure that
- * anything inserted use the ballot timestamp (to respect the order of update decided by
- * the Paxos algorithm). We use {@code newTimestamp - 1} for deletions because tombstones
- * always win on timestamp equality and we don't want to delete our own insertions
- * (typically, when we overwrite a collection, we first set a complex deletion to delete the
- * previous collection before adding new elements. If we were to set that complex deletion
- * to the same timestamp that the new elements, it would delete those elements). And since
- * tombstones always wins on timestamp equality, using -1 guarantees our deletion will still
- * delete anything from a previous update.
- */
- public void updateAllTimestamp(long newTimestamp)
- {
- Holder holder = holder();
- deletionInfo.updateAllTimestamp(newTimestamp - 1);
- Object[] tree = BTree.<Row>transformAndFilter(holder.tree, (x) -> x.updateAllTimestamp(newTimestamp));
- Row staticRow = holder.staticRow.updateAllTimestamp(newTimestamp);
- EncodingStats newStats = EncodingStats.Collector.collect(staticRow, BTree.<Row>iterator(tree), deletionInfo);
- this.holder = new Holder(holder.columns, tree, deletionInfo, staticRow, newStats);
- }
-
- /**
* The number of "operations" contained in the update.
* <p>
* This is used by {@code Memtable} to approximate how much work this update does. In practice, this
@@ -430,8 +341,13 @@
return size;
}
+ public TableMetadata metadata()
+ {
+ return metadata;
+ }
+
@Override
- public PartitionColumns columns()
+ public RegularAndStaticColumns columns()
{
// The superclass implementation calls holder(), but that triggers a build of the PartitionUpdate. But since
// the columns are passed to the ctor, we know the holder always has the proper columns even if it doesn't have
@@ -441,7 +357,6 @@
protected Holder holder()
{
- maybeBuild();
return holder;
}
@@ -451,50 +366,6 @@
}
/**
- * If a partition update has been read (and is thus unmodifiable), a call to this method
- * makes the update modifiable again.
- * <p>
- * Please note that calling this method won't result in optimal behavior in the sense that
- * even if very little is added to the update after this call, the whole update will be sorted
- * again on read. This should thus be used sparingly (and if it turns that we end up using
- * this often, we should consider optimizing the behavior).
- */
- public synchronized void allowNewUpdates()
- {
- if (!canReOpen)
- throw new IllegalStateException("You cannot do more updates on collectCounterMarks has been called");
-
- // This is synchronized to make extra sure things work properly even if this is
- // called concurrently with sort() (which should be avoided in the first place, but
- // better safe than sorry).
- isBuilt = false;
- if (rowBuilder == null)
- rowBuilder = builder(16);
- }
-
- private BTree.Builder<Row> builder(int initialCapacity)
- {
- return BTree.<Row>builder(metadata.comparator, initialCapacity)
- .setQuickResolver((a, b) ->
- Rows.merge(a, b, createdAtInSec));
- }
-
- /**
- * Returns an iterator that iterates over the rows of this update in clustering order.
- * <p>
- * Note that this might trigger a sorting of the update, and as such the update will not
- * be modifiable anymore after this call.
- *
- * @return an iterator over the rows of this update.
- */
- @Override
- public Iterator<Row> iterator()
- {
- maybeBuild();
- return super.iterator();
- }
-
- /**
* Validates the data contained in this update.
*
* @throws org.apache.cassandra.serializers.MarshalException if some of the data contained in this update is corrupted.
@@ -516,8 +387,6 @@
*/
public long maxTimestamp()
{
- maybeBuild();
-
long maxTimestamp = deletionInfo.maxTimestamp();
for (Row row : this)
{
@@ -538,9 +407,9 @@
}
}
- if (holder.staticRow != null)
+ if (this.holder.staticRow != null)
{
- for (ColumnData cd : holder.staticRow.columnData())
+ for (ColumnData cd : this.holder.staticRow.columnData())
{
if (cd.column().isSimple())
{
@@ -567,11 +436,8 @@
public List<CounterMark> collectCounterMarks()
{
assert metadata().isCounter();
- maybeBuild();
// We will take aliases on the rows of this update, and update them in-place. So we should be sure the
// update is now immutable for all intent and purposes.
- canReOpen = false;
-
List<CounterMark> marks = new ArrayList<>();
addMarksForRow(staticRow(), marks);
for (Row row : this)
@@ -579,7 +445,7 @@
return marks;
}
- private void addMarksForRow(Row row, List<CounterMark> marks)
+ private static void addMarksForRow(Row row, List<CounterMark> marks)
{
for (Cell cell : row.cells())
{
@@ -588,110 +454,6 @@
}
}
- private void assertNotBuilt()
- {
- if (isBuilt)
- throw new IllegalStateException("An update should not be written again once it has been read");
- }
-
- public void addPartitionDeletion(DeletionTime deletionTime)
- {
- assertNotBuilt();
- deletionInfo.add(deletionTime);
- }
-
- public void add(RangeTombstone range)
- {
- assertNotBuilt();
- deletionInfo.add(range, metadata.comparator);
- }
-
- /**
- * Adds a row to this update.
- *
- * There is no particular assumption made on the order of row added to a partition update. It is further
- * allowed to add the same row (more precisely, multiple row objects for the same clustering).
- *
- * Note however that the columns contained in the added row must be a subset of the columns used when
- * creating this update.
- *
- * @param row the row to add.
- */
- public void add(Row row)
- {
- if (row.isEmpty())
- return;
-
- assertNotBuilt();
-
- if (row.isStatic())
- {
- // this assert is expensive, and possibly of limited value; we should consider removing it
- // or introducing a new class of assertions for test purposes
- assert columns().statics.containsAll(row.columns()) : columns().statics + " is not superset of " + row.columns();
- Row staticRow = holder.staticRow.isEmpty()
- ? row
- : Rows.merge(holder.staticRow, row, createdAtInSec);
- holder = new Holder(holder.columns, holder.tree, holder.deletionInfo, staticRow, holder.stats);
- }
- else
- {
- // this assert is expensive, and possibly of limited value; we should consider removing it
- // or introducing a new class of assertions for test purposes
- assert columns().regulars.containsAll(row.columns()) : columns().regulars + " is not superset of " + row.columns();
- rowBuilder.add(row);
- }
- }
-
- private void maybeBuild()
- {
- if (isBuilt)
- return;
-
- build();
- }
-
- private synchronized void build()
- {
- if (isBuilt)
- return;
-
- Holder holder = this.holder;
- Object[] cur = holder.tree;
- Object[] add = rowBuilder.build();
- Object[] merged = BTree.<Row>merge(cur, add, metadata.comparator,
- UpdateFunction.Simple.of((a, b) -> Rows.merge(a, b, createdAtInSec)));
-
- assert deletionInfo == holder.deletionInfo;
- EncodingStats newStats = EncodingStats.Collector.collect(holder.staticRow, BTree.<Row>iterator(merged), deletionInfo);
-
- this.holder = new Holder(holder.columns, merged, holder.deletionInfo, holder.staticRow, newStats);
- rowBuilder = null;
- isBuilt = true;
- }
-
- @Override
- public String toString()
- {
- if (isBuilt)
- return super.toString();
-
- // We intentionally override AbstractBTreePartition#toString() to avoid iterating over the rows in the
- // partition, which can result in build() being triggered and lead to errors if the PartitionUpdate is later
- // modified.
-
- StringBuilder sb = new StringBuilder();
- sb.append(String.format("[%s.%s] key=%s columns=%s",
- metadata.ksName,
- metadata.cfName,
- metadata.getKeyValidator().getString(partitionKey().getKey()),
- columns()));
-
- sb.append("\n deletionInfo=").append(deletionInfo);
- sb.append(" (not built)");
- return sb.toString();
- }
-
/**
* Creates a new simple partition update builder.
*
@@ -701,11 +463,16 @@
* Int32Type, string for UTF8Type, ...). It is also allowed to pass a single {@code DecoratedKey} value directly.
* @return a newly created builder.
*/
- public static SimpleBuilder simpleBuilder(CFMetaData metadata, Object... partitionKeyValues)
+ public static SimpleBuilder simpleBuilder(TableMetadata metadata, Object... partitionKeyValues)
{
return new SimpleBuilders.PartitionUpdateBuilder(metadata, partitionKeyValues);
}
+ public void validateIndexedColumns()
+ {
+ IndexRegistry.obtain(metadata()).validate(this);
+ }
+
/**
* Interface for building partition updates geared towards human.
* <p>
@@ -717,7 +484,7 @@
/**
* The metadata of the table this is a builder on.
*/
- public CFMetaData metadata();
+ public TableMetadata metadata();
/**
* Sets the timestamp to use for the following additions to this builder or any derived (row) builder.
@@ -771,6 +538,13 @@
public RangeTombstoneBuilder addRangeTombstone();
/**
+ * Adds a new range tombstone to this update
+ *
+ * @return this builder
+ */
+ public SimpleBuilder addRangeTombstone(RangeTombstone rt);
+
+ /**
* Build the update represented by this builder.
*
* @return the built update.
@@ -854,49 +628,14 @@
{
assert !iter.isReverseOrder();
- if (version < MessagingService.VERSION_30)
- {
- LegacyLayout.serializeAsLegacyPartition(null, iter, out, version);
- }
- else
- {
- CFMetaData.serializer.serialize(update.metadata(), out, version);
- UnfilteredRowIteratorSerializer.serializer.serialize(iter, null, out, version, update.rowCount());
- }
+ update.metadata.id.serialize(out);
+ UnfilteredRowIteratorSerializer.serializer.serialize(iter, null, out, version, update.rowCount());
}
}
- public PartitionUpdate deserialize(DataInputPlus in, int version, SerializationHelper.Flag flag, ByteBuffer key) throws IOException
+ public PartitionUpdate deserialize(DataInputPlus in, int version, DeserializationHelper.Flag flag) throws IOException
{
- if (version >= MessagingService.VERSION_30)
- {
- assert key == null; // key is only there for the old format
- return deserialize30(in, version, flag);
- }
- else
- {
- assert key != null;
- return deserializePre30(in, version, flag, key);
- }
- }
-
- // Used to share same decorated key between updates.
- public PartitionUpdate deserialize(DataInputPlus in, int version, SerializationHelper.Flag flag, DecoratedKey key) throws IOException
- {
- if (version >= MessagingService.VERSION_30)
- {
- return deserialize30(in, version, flag);
- }
- else
- {
- assert key != null;
- return deserializePre30(in, version, flag, key.getKey());
- }
- }
-
- private static PartitionUpdate deserialize30(DataInputPlus in, int version, SerializationHelper.Flag flag) throws IOException
- {
- CFMetaData metadata = CFMetaData.serializer.deserialize(in, version);
+ TableMetadata metadata = Schema.instance.getExistingTableMetadata(TableId.deserialize(in));
UnfilteredRowIteratorSerializer.Header header = UnfilteredRowIteratorSerializer.serializer.deserializeHeader(metadata, null, in, version, flag);
if (header.isEmpty)
return emptyUpdate(metadata, header.key);
@@ -928,23 +667,11 @@
false);
}
- private static PartitionUpdate deserializePre30(DataInputPlus in, int version, SerializationHelper.Flag flag, ByteBuffer key) throws IOException
- {
- try (UnfilteredRowIterator iterator = LegacyLayout.deserializeLegacyPartition(in, version, flag, key))
- {
- assert iterator != null; // This is only used in mutation, and mutation have never allowed "null" column families
- return PartitionUpdate.fromPre30Iterator(iterator, ColumnFilter.all(iterator.metadata()));
- }
- }
-
public long serializedSize(PartitionUpdate update, int version)
{
try (UnfilteredRowIterator iter = update.unfilteredIterator())
{
- if (version < MessagingService.VERSION_30)
- return LegacyLayout.serializedSizeAsLegacyPartition(null, iter, version);
-
- return CFMetaData.serializer.serializedSize(update.metadata(), version)
+ return update.metadata.id.serializedSize()
+ UnfilteredRowIteratorSerializer.serializer.serializedSize(iter, null, version, update.rowCount());
}
}
@@ -958,10 +685,10 @@
public static class CounterMark
{
private final Row row;
- private final ColumnDefinition column;
+ private final ColumnMetadata column;
private final CellPath path;
- private CounterMark(Row row, ColumnDefinition column, CellPath path)
+ private CounterMark(Row row, ColumnMetadata column, CellPath path)
{
this.row = row;
this.column = column;
@@ -973,7 +700,7 @@
return row.clustering();
}
- public ColumnDefinition column()
+ public ColumnMetadata column()
{
return column;
}
@@ -998,4 +725,205 @@
((BTreeRow)row).setValue(column, path, value);
}
}
+
+ /**
+ * Builder for PartitionUpdates
+ *
+ * This class is not thread safe, but the PartitionUpdate it produces is (since it is immutable).
+ */
+ public static class Builder
+ {
+ private final TableMetadata metadata;
+ private final DecoratedKey key;
+ private final MutableDeletionInfo deletionInfo;
+ private final boolean canHaveShadowedData;
+ private Object[] tree = BTree.empty();
+ private final BTree.Builder<Row> rowBuilder;
+ private Row staticRow = Rows.EMPTY_STATIC_ROW;
+ private final RegularAndStaticColumns columns;
+ private boolean isBuilt = false;
+
+ public Builder(TableMetadata metadata,
+ DecoratedKey key,
+ RegularAndStaticColumns columns,
+ int initialRowCapacity,
+ boolean canHaveShadowedData)
+ {
+ this(metadata, key, columns, initialRowCapacity, canHaveShadowedData, Rows.EMPTY_STATIC_ROW, MutableDeletionInfo.live(), BTree.empty());
+ }
+
+ private Builder(TableMetadata metadata,
+ DecoratedKey key,
+ RegularAndStaticColumns columns,
+ int initialRowCapacity,
+ boolean canHaveShadowedData,
+ Holder holder)
+ {
+ this(metadata, key, columns, initialRowCapacity, canHaveShadowedData, holder.staticRow, holder.deletionInfo, holder.tree);
+ }
+
+ private Builder(TableMetadata metadata,
+ DecoratedKey key,
+ RegularAndStaticColumns columns,
+ int initialRowCapacity,
+ boolean canHaveShadowedData,
+ Row staticRow,
+ DeletionInfo deletionInfo,
+ Object[] tree)
+ {
+ this.metadata = metadata;
+ this.key = key;
+ this.columns = columns;
+ this.rowBuilder = rowBuilder(initialRowCapacity);
+ this.canHaveShadowedData = canHaveShadowedData;
+ this.deletionInfo = deletionInfo.mutableCopy();
+ this.staticRow = staticRow;
+ this.tree = tree;
+ }
+
+ public Builder(TableMetadata metadata, DecoratedKey key, RegularAndStaticColumns columnDefinitions, int size)
+ {
+ this(metadata, key, columnDefinitions, size, true);
+ }
+
+ public Builder(PartitionUpdate base, int initialRowCapacity)
+ {
+ this(base.metadata, base.partitionKey, base.columns(), initialRowCapacity, base.canHaveShadowedData, base.holder);
+ }
+
+ public Builder(TableMetadata metadata,
+ ByteBuffer key,
+ RegularAndStaticColumns columns,
+ int initialRowCapacity)
+ {
+ this(metadata, metadata.partitioner.decorateKey(key), columns, initialRowCapacity, true);
+ }
+
+ /**
+ * Adds a row to this update.
+ *
+ * There is no particular assumption made on the order of row added to a partition update. It is further
+ * allowed to add the same row (more precisely, multiple row objects for the same clustering).
+ *
+ * Note however that the columns contained in the added row must be a subset of the columns used when
+ * creating this update.
+ *
+ * @param row the row to add.
+ */
+ public void add(Row row)
+ {
+ if (row.isEmpty())
+ return;
+
+ if (row.isStatic())
+ {
+ // this assert is expensive, and possibly of limited value; we should consider removing it
+ // or introducing a new class of assertions for test purposes
+ assert columns().statics.containsAll(row.columns()) : columns().statics + " is not superset of " + row.columns();
+ staticRow = staticRow.isEmpty()
+ ? row
+ : Rows.merge(staticRow, row);
+ }
+ else
+ {
+ // this assert is expensive, and possibly of limited value; we should consider removing it
+ // or introducing a new class of assertions for test purposes
+ assert columns().regulars.containsAll(row.columns()) : columns().regulars + " is not superset of " + row.columns();
+ rowBuilder.add(row);
+ }
+ }
+
+ public void addPartitionDeletion(DeletionTime deletionTime)
+ {
+ deletionInfo.add(deletionTime);
+ }
+
+ public void add(RangeTombstone range)
+ {
+ deletionInfo.add(range, metadata.comparator);
+ }
+
+ public DecoratedKey partitionKey()
+ {
+ return key;
+ }
+
+ public TableMetadata metadata()
+ {
+ return metadata;
+ }
+
+ public PartitionUpdate build()
+ {
+ // assert that we are not calling build() several times
+ assert !isBuilt : "A PartitionUpdate.Builder should only get built once";
+ Object[] add = rowBuilder.build();
+ Object[] merged = BTree.<Row>merge(tree, add, metadata.comparator,
+ UpdateFunction.Simple.of(Rows::merge));
+
+ EncodingStats newStats = EncodingStats.Collector.collect(staticRow, BTree.iterator(merged), deletionInfo);
+
+ isBuilt = true;
+ return new PartitionUpdate(metadata,
+ partitionKey(),
+ new Holder(columns,
+ merged,
+ deletionInfo,
+ staticRow,
+ newStats),
+ deletionInfo,
+ canHaveShadowedData);
+ }
+
+ public RegularAndStaticColumns columns()
+ {
+ return columns;
+ }
+
+ public DeletionTime partitionLevelDeletion()
+ {
+ return deletionInfo.getPartitionDeletion();
+ }
+
+ private BTree.Builder<Row> rowBuilder(int initialCapacity)
+ {
+ return BTree.<Row>builder(metadata.comparator, initialCapacity)
+ .setQuickResolver(Rows::merge);
+ }
+ /**
+ * Modify this update to set every timestamp for live data to {@code newTimestamp} and
+ * every deletion timestamp to {@code newTimestamp - 1}.
+ *
+ * There is no reason to use that expect on the Paxos code path, where we need ensure that
+ * anything inserted use the ballot timestamp (to respect the order of update decided by
+ * the Paxos algorithm). We use {@code newTimestamp - 1} for deletions because tombstones
+ * always win on timestamp equality and we don't want to delete our own insertions
+ * (typically, when we overwrite a collection, we first set a complex deletion to delete the
+ * previous collection before adding new elements. If we were to set that complex deletion
+ * to the same timestamp that the new elements, it would delete those elements). And since
+ * tombstones always wins on timestamp equality, using -1 guarantees our deletion will still
+ * delete anything from a previous update.
+ */
+ public Builder updateAllTimestamp(long newTimestamp)
+ {
+ deletionInfo.updateAllTimestamp(newTimestamp - 1);
+ tree = BTree.<Row>transformAndFilter(tree, (x) -> x.updateAllTimestamp(newTimestamp));
+ staticRow = this.staticRow.updateAllTimestamp(newTimestamp);
+ return this;
+ }
+
+ @Override
+ public String toString()
+ {
+ return "Builder{" +
+ "metadata=" + metadata +
+ ", key=" + key +
+ ", deletionInfo=" + deletionInfo +
+ ", canHaveShadowedData=" + canHaveShadowedData +
+ ", staticRow=" + staticRow +
+ ", columns=" + columns +
+ ", isBuilt=" + isBuilt +
+ '}';
+ }
+ }
}
diff --git a/src/java/org/apache/cassandra/db/partitions/PurgeFunction.java b/src/java/org/apache/cassandra/db/partitions/PurgeFunction.java
index 5cc9145..d9e9036 100644
--- a/src/java/org/apache/cassandra/db/partitions/PurgeFunction.java
+++ b/src/java/org/apache/cassandra/db/partitions/PurgeFunction.java
@@ -17,6 +17,7 @@
*/
package org.apache.cassandra.db.partitions;
+import java.util.function.LongPredicate;
import java.util.function.Predicate;
import org.apache.cassandra.db.*;
@@ -25,21 +26,15 @@
public abstract class PurgeFunction extends Transformation<UnfilteredRowIterator>
{
- private final boolean isForThrift;
private final DeletionPurger purger;
private final int nowInSec;
private final boolean enforceStrictLiveness;
private boolean isReverseOrder;
- public PurgeFunction(boolean isForThrift,
- int nowInSec,
- int gcBefore,
- int oldestUnrepairedTombstone,
- boolean onlyPurgeRepairedTombstones,
+ public PurgeFunction(int nowInSec, int gcBefore, int oldestUnrepairedTombstone, boolean onlyPurgeRepairedTombstones,
boolean enforceStrictLiveness)
{
- this.isForThrift = isForThrift;
this.nowInSec = nowInSec;
this.purger = (timestamp, localDeletionTime) ->
!(onlyPurgeRepairedTombstones && localDeletionTime >= oldestUnrepairedTombstone)
@@ -48,7 +43,7 @@
this.enforceStrictLiveness = enforceStrictLiveness;
}
- protected abstract Predicate<Long> getPurgeEvaluator();
+ protected abstract LongPredicate getPurgeEvaluator();
// Called at the beginning of each new partition
protected void onNewPartition(DecoratedKey partitionKey)
@@ -65,14 +60,20 @@
{
}
+ protected void setReverseOrder(boolean isReverseOrder)
+ {
+ this.isReverseOrder = isReverseOrder;
+ }
+
@Override
+ @SuppressWarnings("resource")
protected UnfilteredRowIterator applyToPartition(UnfilteredRowIterator partition)
{
onNewPartition(partition.partitionKey());
- isReverseOrder = partition.isReverseOrder();
+ setReverseOrder(partition.isReverseOrder());
UnfilteredRowIterator purged = Transformation.apply(partition, this);
- if (!isForThrift && purged.isEmpty())
+ if (purged.isEmpty())
{
onEmptyPartitionPostPurge(purged.partitionKey());
purged.close();
diff --git a/src/java/org/apache/cassandra/db/partitions/SingletonUnfilteredPartitionIterator.java b/src/java/org/apache/cassandra/db/partitions/SingletonUnfilteredPartitionIterator.java
index 1f966db..b739e8b 100644
--- a/src/java/org/apache/cassandra/db/partitions/SingletonUnfilteredPartitionIterator.java
+++ b/src/java/org/apache/cassandra/db/partitions/SingletonUnfilteredPartitionIterator.java
@@ -19,27 +19,20 @@
import java.util.NoSuchElementException;
-import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.db.rows.UnfilteredRowIterator;
public class SingletonUnfilteredPartitionIterator implements UnfilteredPartitionIterator
{
private final UnfilteredRowIterator iter;
- private final boolean isForThrift;
private boolean returned;
- public SingletonUnfilteredPartitionIterator(UnfilteredRowIterator iter, boolean isForThrift)
+ public SingletonUnfilteredPartitionIterator(UnfilteredRowIterator iter)
{
this.iter = iter;
- this.isForThrift = isForThrift;
}
- public boolean isForThrift()
- {
- return isForThrift;
- }
-
- public CFMetaData metadata()
+ public TableMetadata metadata()
{
return iter.metadata();
}
diff --git a/src/java/org/apache/cassandra/db/partitions/UnfilteredPartitionIterator.java b/src/java/org/apache/cassandra/db/partitions/UnfilteredPartitionIterator.java
index 201c934..cd8e47f 100644
--- a/src/java/org/apache/cassandra/db/partitions/UnfilteredPartitionIterator.java
+++ b/src/java/org/apache/cassandra/db/partitions/UnfilteredPartitionIterator.java
@@ -17,7 +17,7 @@
*/
package org.apache.cassandra.db.partitions;
-import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.db.rows.UnfilteredRowIterator;
/**
@@ -30,16 +30,5 @@
*/
public interface UnfilteredPartitionIterator extends BasePartitionIterator<UnfilteredRowIterator>
{
- /**
- * Whether that partition iterator is for a thrift queries.
- * <p>
- * If this is true, the partition iterator may return some empty UnfilteredRowIterator and those
- * should be preserved as thrift include partitions that "exists" (have some cells even
- * if this are actually deleted) but have nothing matching the query.
- *
- * @return whether the iterator is for a thrift query.
- */
- public boolean isForThrift();
-
- public CFMetaData metadata();
+ public TableMetadata metadata();
}
diff --git a/src/java/org/apache/cassandra/db/partitions/UnfilteredPartitionIterators.java b/src/java/org/apache/cassandra/db/partitions/UnfilteredPartitionIterators.java
index 245ea35..a051ee1 100644
--- a/src/java/org/apache/cassandra/db/partitions/UnfilteredPartitionIterators.java
+++ b/src/java/org/apache/cassandra/db/partitions/UnfilteredPartitionIterators.java
@@ -19,10 +19,8 @@
import java.io.IOError;
import java.io.IOException;
-import java.security.MessageDigest;
import java.util.*;
-import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.filter.ColumnFilter;
import org.apache.cassandra.db.rows.*;
@@ -31,7 +29,7 @@
import org.apache.cassandra.db.transform.Transformation;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
-import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.utils.MergeIterator;
/**
@@ -49,6 +47,8 @@
{
public UnfilteredRowIterators.MergeListener getRowMergeListener(DecoratedKey partitionKey, List<UnfilteredRowIterator> versions);
public default void close() {}
+
+ public static MergeListener NOOP = (partitionKey, versions) -> UnfilteredRowIterators.MergeListener.NOOP;
}
@SuppressWarnings("resource") // The created resources are returned right away
@@ -101,12 +101,12 @@
return FilteredPartitions.filter(iterator, nowInSec);
}
- public static UnfilteredPartitionIterator merge(final List<? extends UnfilteredPartitionIterator> iterators, final int nowInSec, final MergeListener listener)
+ @SuppressWarnings("resource")
+ public static UnfilteredPartitionIterator merge(final List<? extends UnfilteredPartitionIterator> iterators, final MergeListener listener)
{
assert !iterators.isEmpty();
- final boolean isForThrift = iterators.get(0).isForThrift();
- final CFMetaData metadata = iterators.get(0).metadata();
+ final TableMetadata metadata = iterators.get(0).metadata();
final MergeIterator<UnfilteredRowIterator, UnfilteredRowIterator> merged = MergeIterator.get(iterators, partitionComparator, new MergeIterator.Reducer<UnfilteredRowIterator, UnfilteredRowIterator>()
{
@@ -125,18 +125,28 @@
toMerge.set(idx, current);
}
+ @SuppressWarnings("resource")
protected UnfilteredRowIterator getReduced()
{
UnfilteredRowIterators.MergeListener rowListener = listener == null
? null
: listener.getRowMergeListener(partitionKey, toMerge);
+ // Make a single empty iterator object to merge, we don't need toMerge.size() copiess
+ UnfilteredRowIterator empty = null;
+
// Replace nulls by empty iterators
for (int i = 0; i < toMerge.size(); i++)
+ {
if (toMerge.get(i) == null)
- toMerge.set(i, EmptyIterators.unfilteredRow(metadata, partitionKey, isReverseOrder));
+ {
+ if (null == empty)
+ empty = EmptyIterators.unfilteredRow(metadata, partitionKey, isReverseOrder);
+ toMerge.set(i, empty);
+ }
+ }
- return UnfilteredRowIterators.merge(toMerge, nowInSec, rowListener);
+ return UnfilteredRowIterators.merge(toMerge, rowListener);
}
protected void onKeyChange()
@@ -149,12 +159,7 @@
return new AbstractUnfilteredPartitionIterator()
{
- public boolean isForThrift()
- {
- return isForThrift;
- }
-
- public CFMetaData metadata()
+ public TableMetadata metadata()
{
return metadata;
}
@@ -180,15 +185,15 @@
};
}
- public static UnfilteredPartitionIterator mergeLazily(final List<? extends UnfilteredPartitionIterator> iterators, final int nowInSec)
+ @SuppressWarnings("resource")
+ public static UnfilteredPartitionIterator mergeLazily(final List<? extends UnfilteredPartitionIterator> iterators)
{
assert !iterators.isEmpty();
if (iterators.size() == 1)
return iterators.get(0);
- final boolean isForThrift = iterators.get(0).isForThrift();
- final CFMetaData metadata = iterators.get(0).metadata();
+ final TableMetadata metadata = iterators.get(0).metadata();
final MergeIterator<UnfilteredRowIterator, UnfilteredRowIterator> merged = MergeIterator.get(iterators, partitionComparator, new MergeIterator.Reducer<UnfilteredRowIterator, UnfilteredRowIterator>()
{
@@ -205,7 +210,7 @@
{
protected UnfilteredRowIterator initializeIterator()
{
- return UnfilteredRowIterators.merge(toMerge, nowInSec);
+ return UnfilteredRowIterators.merge(toMerge);
}
};
}
@@ -218,12 +223,7 @@
return new AbstractUnfilteredPartitionIterator()
{
- public boolean isForThrift()
- {
- return isForThrift;
- }
-
- public CFMetaData metadata()
+ public TableMetadata metadata()
{
return metadata;
}
@@ -251,19 +251,17 @@
*
* Caller must close the provided iterator.
*
- * @param command the command that has yield {@code iterator}. This can be null if {@code version >= MessagingService.VERSION_30}
- * as this is only used when producing digest to be sent to legacy nodes.
* @param iterator the iterator to digest.
- * @param digest the {@code MessageDigest} to use for the digest.
+ * @param digest the {@link Digest} to use.
* @param version the messaging protocol to use when producing the digest.
*/
- public static void digest(ReadCommand command, UnfilteredPartitionIterator iterator, MessageDigest digest, int version)
+ public static void digest(UnfilteredPartitionIterator iterator, Digest digest, int version)
{
while (iterator.hasNext())
{
try (UnfilteredRowIterator partition = iterator.next())
{
- UnfilteredRowIterators.digest(command, partition, digest, version);
+ UnfilteredRowIterators.digest(partition, digest, version);
}
}
}
@@ -299,9 +297,9 @@
{
public void serialize(UnfilteredPartitionIterator iter, ColumnFilter selection, DataOutputPlus out, int version) throws IOException
{
- assert version >= MessagingService.VERSION_30; // We handle backward compatibility directy in ReadResponse.LegacyRangeSliceReplySerializer
-
- out.writeBoolean(iter.isForThrift());
+ // Previously, a boolean indicating if this was for a thrift query.
+ // Unused since 4.0 but kept on wire for compatibility.
+ out.writeBoolean(false);
while (iter.hasNext())
{
out.writeBoolean(true);
@@ -313,10 +311,10 @@
out.writeBoolean(false);
}
- public UnfilteredPartitionIterator deserialize(final DataInputPlus in, final int version, final CFMetaData metadata, final ColumnFilter selection, final SerializationHelper.Flag flag) throws IOException
+ public UnfilteredPartitionIterator deserialize(final DataInputPlus in, final int version, final TableMetadata metadata, final ColumnFilter selection, final DeserializationHelper.Flag flag) throws IOException
{
- assert version >= MessagingService.VERSION_30; // We handle backward compatibility directy in ReadResponse.LegacyRangeSliceReplySerializer
- final boolean isForThrift = in.readBoolean();
+ // Skip now unused isForThrift boolean
+ in.readBoolean();
return new AbstractUnfilteredPartitionIterator()
{
@@ -324,12 +322,7 @@
private boolean hasNext;
private boolean nextReturned = true;
- public boolean isForThrift()
- {
- return isForThrift;
- }
-
- public CFMetaData metadata()
+ public TableMetadata metadata()
{
return metadata;
}
diff --git a/src/java/org/apache/cassandra/db/repair/CassandraKeyspaceRepairManager.java b/src/java/org/apache/cassandra/db/repair/CassandraKeyspaceRepairManager.java
new file mode 100644
index 0000000..4fa8650
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/repair/CassandraKeyspaceRepairManager.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.repair;
+
+import java.util.Collection;
+import java.util.UUID;
+import java.util.concurrent.ExecutorService;
+import java.util.function.BooleanSupplier;
+
+import com.google.common.util.concurrent.ListenableFuture;
+
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.locator.RangesAtEndpoint;
+import org.apache.cassandra.repair.KeyspaceRepairManager;
+
+public class CassandraKeyspaceRepairManager implements KeyspaceRepairManager
+{
+ private final Keyspace keyspace;
+
+ public CassandraKeyspaceRepairManager(Keyspace keyspace)
+ {
+ this.keyspace = keyspace;
+ }
+
+ @Override
+ public ListenableFuture prepareIncrementalRepair(UUID sessionID,
+ Collection<ColumnFamilyStore> tables,
+ RangesAtEndpoint tokenRanges,
+ ExecutorService executor,
+ BooleanSupplier isCancelled)
+ {
+ PendingAntiCompaction pac = new PendingAntiCompaction(sessionID, tables, tokenRanges, executor, isCancelled);
+ return pac.run();
+ }
+}
diff --git a/src/java/org/apache/cassandra/db/repair/CassandraTableRepairManager.java b/src/java/org/apache/cassandra/db/repair/CassandraTableRepairManager.java
new file mode 100644
index 0000000..983e30f
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/repair/CassandraTableRepairManager.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.repair;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.UUID;
+import java.util.concurrent.Callable;
+import java.util.concurrent.Future;
+
+import com.google.common.base.Predicate;
+
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.dht.Bounds;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.repair.TableRepairManager;
+import org.apache.cassandra.repair.ValidationPartitionIterator;
+import org.apache.cassandra.repair.Validator;
+
+public class CassandraTableRepairManager implements TableRepairManager
+{
+ private final ColumnFamilyStore cfs;
+
+ public CassandraTableRepairManager(ColumnFamilyStore cfs)
+ {
+ this.cfs = cfs;
+ }
+
+ @Override
+ public ValidationPartitionIterator getValidationIterator(Collection<Range<Token>> ranges, UUID parentId, UUID sessionID, boolean isIncremental, int nowInSec) throws IOException
+ {
+ return new CassandraValidationIterator(cfs, ranges, parentId, sessionID, isIncremental, nowInSec);
+ }
+
+ @Override
+ public Future<?> submitValidation(Callable<Object> validation)
+ {
+ return CompactionManager.instance.submitValidation(validation);
+ }
+
+ @Override
+ public void incrementalSessionCompleted(UUID sessionID)
+ {
+ CompactionManager.instance.submitBackground(cfs);
+ }
+
+ @Override
+ public synchronized void snapshot(String name, Collection<Range<Token>> ranges, boolean force)
+ {
+ if (force || !cfs.snapshotExists(name))
+ {
+ cfs.snapshot(name, new Predicate<SSTableReader>()
+ {
+ public boolean apply(SSTableReader sstable)
+ {
+ return sstable != null &&
+ !sstable.metadata().isIndex() && // exclude SSTables from 2i
+ new Bounds<>(sstable.first.getToken(), sstable.last.getToken()).intersects(ranges);
+ }
+ }, true, false); //ephemeral snapshot, if repair fails, it will be cleaned next startup
+ }
+
+ }
+}
diff --git a/src/java/org/apache/cassandra/db/repair/CassandraValidationIterator.java b/src/java/org/apache/cassandra/db/repair/CassandraValidationIterator.java
new file mode 100644
index 0000000..9bddd86
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/repair/CassandraValidationIterator.java
@@ -0,0 +1,304 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.repair;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.function.LongPredicate;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Maps;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.compaction.AbstractCompactionStrategy;
+import org.apache.cassandra.db.compaction.ActiveCompactionsTracker;
+import org.apache.cassandra.db.compaction.CompactionController;
+import org.apache.cassandra.db.compaction.CompactionIterator;
+import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.db.compaction.OperationType;
+import org.apache.cassandra.db.lifecycle.SSTableSet;
+import org.apache.cassandra.db.lifecycle.View;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.dht.Bounds;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.io.sstable.ISSTableScanner;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.repair.ValidationPartitionIterator;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.service.ActiveRepairService;
+import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.utils.UUIDGen;
+import org.apache.cassandra.utils.concurrent.Refs;
+
+public class CassandraValidationIterator extends ValidationPartitionIterator
+{
+ private static final Logger logger = LoggerFactory.getLogger(CassandraValidationIterator.class);
+
+ /*
+ * Controller for validation compaction that always purges.
+ * Note that we should not call cfs.getOverlappingSSTables on the provided
+ * sstables because those sstables are not guaranteed to be active sstables
+ * (since we can run repair on a snapshot).
+ */
+ private static class ValidationCompactionController extends CompactionController
+ {
+ public ValidationCompactionController(ColumnFamilyStore cfs, int gcBefore)
+ {
+ super(cfs, gcBefore);
+ }
+
+ @Override
+ public LongPredicate getPurgeEvaluator(DecoratedKey key)
+ {
+ /*
+ * The main reason we always purge is that including gcable tombstone would mean that the
+ * repair digest will depends on the scheduling of compaction on the different nodes. This
+ * is still not perfect because gcbefore is currently dependend on the current time at which
+ * the validation compaction start, which while not too bad for normal repair is broken for
+ * repair on snapshots. A better solution would be to agree on a gcbefore that all node would
+ * use, and we'll do that with CASSANDRA-4932.
+ * Note validation compaction includes all sstables, so we don't have the problem of purging
+ * a tombstone that could shadow a column in another sstable, but this is doubly not a concern
+ * since validation compaction is read-only.
+ */
+ return time -> true;
+ }
+ }
+
+ public static int getDefaultGcBefore(ColumnFamilyStore cfs, int nowInSec)
+ {
+ // 2ndary indexes have ExpiringColumns too, so we need to purge tombstones deleted before now. We do not need to
+ // add any GcGrace however since 2ndary indexes are local to a node.
+ return cfs.isIndex() ? nowInSec : cfs.gcBefore(nowInSec);
+ }
+
+ private static class ValidationCompactionIterator extends CompactionIterator
+ {
+ public ValidationCompactionIterator(List<ISSTableScanner> scanners, ValidationCompactionController controller, int nowInSec, ActiveCompactionsTracker activeCompactions)
+ {
+ super(OperationType.VALIDATION, scanners, controller, nowInSec, UUIDGen.getTimeUUID(), activeCompactions);
+ }
+ }
+
+ @VisibleForTesting
+ static synchronized Refs<SSTableReader> getSSTablesToValidate(ColumnFamilyStore cfs, Collection<Range<Token>> ranges, UUID parentId, boolean isIncremental)
+ {
+ Refs<SSTableReader> sstables;
+
+ ActiveRepairService.ParentRepairSession prs = ActiveRepairService.instance.getParentRepairSession(parentId);
+ if (prs == null)
+ {
+ // this means the parent repair session was removed - the repair session failed on another node and we removed it
+ return new Refs<>();
+ }
+
+ Set<SSTableReader> sstablesToValidate = new HashSet<>();
+
+ com.google.common.base.Predicate<SSTableReader> predicate;
+ if (prs.isPreview())
+ {
+ predicate = prs.previewKind.predicate();
+
+ }
+ else if (isIncremental)
+ {
+ predicate = s -> parentId.equals(s.getSSTableMetadata().pendingRepair);
+ }
+ else
+ {
+ // note that we always grab all existing sstables for this - if we were to just grab the ones that
+ // were marked as repairing, we would miss any ranges that were compacted away and this would cause us to overstream
+ predicate = (s) -> !prs.isIncremental || !s.isRepaired();
+ }
+
+ try (ColumnFamilyStore.RefViewFragment sstableCandidates = cfs.selectAndReference(View.select(SSTableSet.CANONICAL, predicate)))
+ {
+ for (SSTableReader sstable : sstableCandidates.sstables)
+ {
+ if (new Bounds<>(sstable.first.getToken(), sstable.last.getToken()).intersects(ranges))
+ {
+ sstablesToValidate.add(sstable);
+ }
+ }
+
+ sstables = Refs.tryRef(sstablesToValidate);
+ if (sstables == null)
+ {
+ logger.error("Could not reference sstables for {}", parentId);
+ throw new RuntimeException("Could not reference sstables");
+ }
+ }
+
+ return sstables;
+ }
+
+ private final ColumnFamilyStore cfs;
+ private final Refs<SSTableReader> sstables;
+ private final String snapshotName;
+ private final boolean isGlobalSnapshotValidation;
+
+ private final boolean isSnapshotValidation;
+ private final AbstractCompactionStrategy.ScannerList scanners;
+ private final ValidationCompactionController controller;
+
+ private final CompactionIterator ci;
+
+ private final long estimatedBytes;
+ private final long estimatedPartitions;
+ private final Map<Range<Token>, Long> rangePartitionCounts;
+
+ public CassandraValidationIterator(ColumnFamilyStore cfs, Collection<Range<Token>> ranges, UUID parentId, UUID sessionID, boolean isIncremental, int nowInSec) throws IOException
+ {
+ this.cfs = cfs;
+
+ isGlobalSnapshotValidation = cfs.snapshotExists(parentId.toString());
+ if (isGlobalSnapshotValidation)
+ snapshotName = parentId.toString();
+ else
+ snapshotName = sessionID.toString();
+ isSnapshotValidation = cfs.snapshotExists(snapshotName);
+
+ if (isSnapshotValidation)
+ {
+ // If there is a snapshot created for the session then read from there.
+ // note that we populate the parent repair session when creating the snapshot, meaning the sstables in the snapshot are the ones we
+ // are supposed to validate.
+ sstables = cfs.getSnapshotSSTableReaders(snapshotName);
+ }
+ else
+ {
+ if (!isIncremental)
+ {
+ // flush first so everyone is validating data that is as similar as possible
+ StorageService.instance.forceKeyspaceFlush(cfs.keyspace.getName(), cfs.name);
+ }
+ sstables = getSSTablesToValidate(cfs, ranges, parentId, isIncremental);
+ }
+
+ Preconditions.checkArgument(sstables != null);
+ ActiveRepairService.ParentRepairSession prs = ActiveRepairService.instance.getParentRepairSession(parentId);
+ if (prs != null)
+ {
+ logger.info("{}, parentSessionId={}: Performing validation compaction on {} sstables in {}.{}",
+ prs.previewKind.logPrefix(sessionID),
+ parentId,
+ sstables.size(),
+ cfs.keyspace.getName(),
+ cfs.getTableName());
+ }
+
+ controller = new ValidationCompactionController(cfs, getDefaultGcBefore(cfs, nowInSec));
+ scanners = cfs.getCompactionStrategyManager().getScanners(sstables, ranges);
+ ci = new ValidationCompactionIterator(scanners.scanners, controller, nowInSec, CompactionManager.instance.active);
+
+ long allPartitions = 0;
+ rangePartitionCounts = Maps.newHashMapWithExpectedSize(ranges.size());
+ for (Range<Token> range : ranges)
+ {
+ long numPartitions = 0;
+ for (SSTableReader sstable : sstables)
+ numPartitions += sstable.estimatedKeysForRanges(Collections.singleton(range));
+ rangePartitionCounts.put(range, numPartitions);
+ allPartitions += numPartitions;
+ }
+ estimatedPartitions = allPartitions;
+
+ long estimatedTotalBytes = 0;
+ for (SSTableReader sstable : sstables)
+ {
+ for (SSTableReader.PartitionPositionBounds positionsForRanges : sstable.getPositionsForRanges(ranges))
+ estimatedTotalBytes += positionsForRanges.upperPosition - positionsForRanges.lowerPosition;
+ }
+ estimatedBytes = estimatedTotalBytes;
+ }
+
+ @Override
+ public void close()
+ {
+ // TODO: can any of this fail and leave stuff unreleased?
+ super.close();
+
+ if (ci != null)
+ ci.close();
+
+ if (scanners != null)
+ scanners.close();
+
+ if (controller != null)
+ controller.close();
+
+ if (isSnapshotValidation && !isGlobalSnapshotValidation)
+ {
+ // we can only clear the snapshot if we are not doing a global snapshot validation (we then clear it once anticompaction
+ // is done).
+ cfs.clearSnapshot(snapshotName);
+ }
+
+ if (sstables != null)
+ sstables.release();
+ }
+
+ @Override
+ public TableMetadata metadata()
+ {
+ return cfs.metadata.get();
+ }
+
+ @Override
+ public boolean hasNext()
+ {
+ return ci.hasNext();
+ }
+
+ @Override
+ public UnfilteredRowIterator next()
+ {
+ return ci.next();
+ }
+
+ @Override
+ public long getEstimatedBytes()
+ {
+ return estimatedBytes;
+ }
+
+ @Override
+ public long estimatedPartitions()
+ {
+ return estimatedPartitions;
+ }
+
+ @Override
+ public Map<Range<Token>, Long> getRangePartitionCounts()
+ {
+ return rangePartitionCounts;
+ }
+}
diff --git a/src/java/org/apache/cassandra/db/repair/PendingAntiCompaction.java b/src/java/org/apache/cassandra/db/repair/PendingAntiCompaction.java
new file mode 100644
index 0000000..e49e76e
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/repair/PendingAntiCompaction.java
@@ -0,0 +1,383 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.repair;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.function.BooleanSupplier;
+import java.util.stream.Collectors;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Predicate;
+import com.google.common.collect.Iterables;
+import com.google.common.util.concurrent.AsyncFunction;
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.ListenableFutureTask;
+import com.google.common.util.concurrent.MoreExecutors;
+import com.google.common.util.concurrent.Uninterruptibles;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.compaction.CompactionInfo;
+import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.db.compaction.OperationType;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
+import org.apache.cassandra.locator.RangesAtEndpoint;
+import org.apache.cassandra.service.ActiveRepairService;
+import org.apache.cassandra.utils.concurrent.Refs;
+
+import static org.apache.cassandra.service.ActiveRepairService.NO_PENDING_REPAIR;
+import static org.apache.cassandra.service.ActiveRepairService.UNREPAIRED_SSTABLE;
+
+/**
+ * Performs an anti compaction on a set of tables and token ranges, isolating the unrepaired sstables
+ * for a give token range into a pending repair group so they can't be compacted with other sstables
+ * while they are being repaired.
+ */
+public class PendingAntiCompaction
+{
+ private static final Logger logger = LoggerFactory.getLogger(PendingAntiCompaction.class);
+ private static final int ACQUIRE_SLEEP_MS = Integer.getInteger("cassandra.acquire_sleep_ms", 1000);
+ private static final int ACQUIRE_RETRY_SECONDS = Integer.getInteger("cassandra.acquire_retry_seconds", 60);
+
+ public static class AcquireResult
+ {
+ final ColumnFamilyStore cfs;
+ final Refs<SSTableReader> refs;
+ final LifecycleTransaction txn;
+
+ AcquireResult(ColumnFamilyStore cfs, Refs<SSTableReader> refs, LifecycleTransaction txn)
+ {
+ this.cfs = cfs;
+ this.refs = refs;
+ this.txn = txn;
+ }
+
+ @VisibleForTesting
+ public void abort()
+ {
+ if (txn != null)
+ txn.abort();
+ if (refs != null)
+ refs.release();
+ }
+ }
+
+ static class SSTableAcquisitionException extends RuntimeException
+ {
+ SSTableAcquisitionException(String message)
+ {
+ super(message);
+ }
+ }
+
+ @VisibleForTesting
+ static class AntiCompactionPredicate implements Predicate<SSTableReader>
+ {
+ private final Collection<Range<Token>> ranges;
+ private final UUID prsid;
+
+ public AntiCompactionPredicate(Collection<Range<Token>> ranges, UUID prsid)
+ {
+ this.ranges = ranges;
+ this.prsid = prsid;
+ }
+
+ public boolean apply(SSTableReader sstable)
+ {
+ if (!sstable.intersects(ranges))
+ return false;
+
+ StatsMetadata metadata = sstable.getSSTableMetadata();
+
+ // exclude repaired sstables
+ if (metadata.repairedAt != UNREPAIRED_SSTABLE)
+ return false;
+
+ if (!sstable.descriptor.version.hasPendingRepair())
+ {
+ String message = String.format("Prepare phase failed because it encountered legacy sstables that don't " +
+ "support pending repair, run upgradesstables before starting incremental " +
+ "repairs, repair session (%s)", prsid);
+ throw new SSTableAcquisitionException(message);
+ }
+
+ // exclude sstables pending repair, but record session ids for
+ // non-finalized sessions for a later error message
+ if (metadata.pendingRepair != NO_PENDING_REPAIR)
+ {
+ if (!ActiveRepairService.instance.consistent.local.isSessionFinalized(metadata.pendingRepair))
+ {
+ String message = String.format("Prepare phase for incremental repair session %s has failed because it encountered " +
+ "intersecting sstables belonging to another incremental repair session (%s). This is " +
+ "caused by starting an incremental repair session before a previous one has completed. " +
+ "Check nodetool repair_admin for hung sessions and fix them.", prsid, metadata.pendingRepair);
+ throw new SSTableAcquisitionException(message);
+ }
+ return false;
+ }
+ CompactionInfo ci = CompactionManager.instance.active.getCompactionForSSTable(sstable);
+ if (ci != null && ci.getTaskType() == OperationType.ANTICOMPACTION)
+ {
+ // todo: start tracking the parent repair session id that created the anticompaction to be able to give a better error messsage here:
+ String message = String.format("Prepare phase for incremental repair session %s has failed because it encountered " +
+ "intersecting sstables (%s) belonging to another incremental repair session. This is " +
+ "caused by starting multiple conflicting incremental repairs at the same time", prsid, ci.getSSTables());
+ throw new SSTableAcquisitionException(message);
+ }
+ return true;
+ }
+ }
+
+ public static class AcquisitionCallable implements Callable<AcquireResult>
+ {
+ private final ColumnFamilyStore cfs;
+ private final UUID sessionID;
+ private final AntiCompactionPredicate predicate;
+ private final int acquireRetrySeconds;
+ private final int acquireSleepMillis;
+
+ @VisibleForTesting
+ public AcquisitionCallable(ColumnFamilyStore cfs, Collection<Range<Token>> ranges, UUID sessionID, int acquireRetrySeconds, int acquireSleepMillis)
+ {
+ this(cfs, sessionID, acquireRetrySeconds, acquireSleepMillis, new AntiCompactionPredicate(ranges, sessionID));
+ }
+
+ @VisibleForTesting
+ AcquisitionCallable(ColumnFamilyStore cfs, UUID sessionID, int acquireRetrySeconds, int acquireSleepMillis, AntiCompactionPredicate predicate)
+ {
+ this.cfs = cfs;
+ this.sessionID = sessionID;
+ this.predicate = predicate;
+ this.acquireRetrySeconds = acquireRetrySeconds;
+ this.acquireSleepMillis = acquireSleepMillis;
+ }
+
+ @SuppressWarnings("resource")
+ private AcquireResult acquireTuple()
+ {
+ // this method runs with compactions stopped & disabled
+ try
+ {
+ // using predicate might throw if there are conflicting ranges
+ Set<SSTableReader> sstables = cfs.getLiveSSTables().stream().filter(predicate).collect(Collectors.toSet());
+ if (sstables.isEmpty())
+ return new AcquireResult(cfs, null, null);
+
+ LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.ANTICOMPACTION);
+ if (txn != null)
+ return new AcquireResult(cfs, Refs.ref(sstables), txn);
+ else
+ logger.error("Could not mark compacting for {} (sstables = {}, compacting = {})", sessionID, sstables, cfs.getTracker().getCompacting());
+ }
+ catch (SSTableAcquisitionException e)
+ {
+ logger.warn(e.getMessage());
+ logger.debug("Got exception trying to acquire sstables", e);
+ }
+
+ return null;
+ }
+
+ public AcquireResult call()
+ {
+ logger.debug("acquiring sstables for pending anti compaction on session {}", sessionID);
+ // try to modify after cancelling running compactions. This will attempt to cancel in flight compactions including the given sstables for
+ // up to a minute, after which point, null will be returned
+ long start = System.currentTimeMillis();
+ long delay = TimeUnit.SECONDS.toMillis(acquireRetrySeconds);
+ // Note that it is `predicate` throwing SSTableAcquisitionException if it finds a conflicting sstable
+ // and we only retry when runWithCompactionsDisabled throws when uses the predicate, not when acquireTuple is.
+ // This avoids the case when we have an sstable [0, 100] and a user starts a repair on [0, 50] and then [51, 100] before
+ // anticompaction has finished but not when the second repair is [25, 75] for example - then we will fail it without retry.
+ do
+ {
+ try
+ {
+ // Note that anticompactions are not disabled when running this. This is safe since runWithCompactionsDisabled
+ // is synchronized - acquireTuple and predicate can only be run by a single thread (for the given cfs).
+ return cfs.runWithCompactionsDisabled(this::acquireTuple, predicate, false, false, false);
+ }
+ catch (SSTableAcquisitionException e)
+ {
+ logger.warn("Session {} failed acquiring sstables: {}, retrying every {}ms for another {}s",
+ sessionID,
+ e.getMessage(),
+ acquireSleepMillis,
+ TimeUnit.SECONDS.convert(delay + start - System.currentTimeMillis(), TimeUnit.MILLISECONDS));
+ Uninterruptibles.sleepUninterruptibly(acquireSleepMillis, TimeUnit.MILLISECONDS);
+
+ if (System.currentTimeMillis() - start > delay)
+ logger.warn("{} Timed out waiting to acquire sstables", sessionID, e);
+
+ }
+ catch (Throwable t)
+ {
+ logger.error("Got exception disabling compactions for session {}", sessionID, t);
+ throw t;
+ }
+ } while (System.currentTimeMillis() - start < delay);
+ return null;
+ }
+ }
+
+ static class AcquisitionCallback implements AsyncFunction<List<AcquireResult>, Object>
+ {
+ private final UUID parentRepairSession;
+ private final RangesAtEndpoint tokenRanges;
+ private final BooleanSupplier isCancelled;
+
+ public AcquisitionCallback(UUID parentRepairSession, RangesAtEndpoint tokenRanges, BooleanSupplier isCancelled)
+ {
+ this.parentRepairSession = parentRepairSession;
+ this.tokenRanges = tokenRanges;
+ this.isCancelled = isCancelled;
+ }
+
+ ListenableFuture<?> submitPendingAntiCompaction(AcquireResult result)
+ {
+ return CompactionManager.instance.submitPendingAntiCompaction(result.cfs, tokenRanges, result.refs, result.txn, parentRepairSession, isCancelled);
+ }
+
+ private static boolean shouldAbort(AcquireResult result)
+ {
+ if (result == null)
+ return true;
+
+ // sstables in the acquire result are now marked compacting and are locked to this anti compaction. If any
+ // of them are marked repaired or pending repair, acquisition raced with another pending anti-compaction, or
+ // possibly even a repair session, and we need to abort to prevent sstables from moving between sessions.
+ return result.refs != null && Iterables.any(result.refs, sstable -> {
+ StatsMetadata metadata = sstable.getSSTableMetadata();
+ return metadata.pendingRepair != NO_PENDING_REPAIR || metadata.repairedAt != UNREPAIRED_SSTABLE;
+ });
+ }
+
+ public ListenableFuture apply(List<AcquireResult> results) throws Exception
+ {
+ if (Iterables.any(results, AcquisitionCallback::shouldAbort))
+ {
+ // Release all sstables, and report failure back to coordinator
+ for (AcquireResult result : results)
+ {
+ if (result != null)
+ {
+ logger.info("Releasing acquired sstables for {}.{}", result.cfs.metadata.keyspace, result.cfs.metadata.name);
+ result.abort();
+ }
+ }
+ String message = String.format("Prepare phase for incremental repair session %s was unable to " +
+ "acquire exclusive access to the neccesary sstables. " +
+ "This is usually caused by running multiple incremental repairs on nodes that share token ranges",
+ parentRepairSession);
+ logger.warn(message);
+ return Futures.immediateFailedFuture(new SSTableAcquisitionException(message));
+ }
+ else
+ {
+ List<ListenableFuture<?>> pendingAntiCompactions = new ArrayList<>(results.size());
+ for (AcquireResult result : results)
+ {
+ if (result.txn != null)
+ {
+ ListenableFuture<?> future = submitPendingAntiCompaction(result);
+ pendingAntiCompactions.add(future);
+ }
+ }
+
+ return Futures.allAsList(pendingAntiCompactions);
+ }
+ }
+ }
+
+ private final UUID prsId;
+ private final Collection<ColumnFamilyStore> tables;
+ private final RangesAtEndpoint tokenRanges;
+ private final ExecutorService executor;
+ private final int acquireRetrySeconds;
+ private final int acquireSleepMillis;
+ private final BooleanSupplier isCancelled;
+
+ public PendingAntiCompaction(UUID prsId,
+ Collection<ColumnFamilyStore> tables,
+ RangesAtEndpoint tokenRanges,
+ ExecutorService executor,
+ BooleanSupplier isCancelled)
+ {
+ this(prsId, tables, tokenRanges, ACQUIRE_RETRY_SECONDS, ACQUIRE_SLEEP_MS, executor, isCancelled);
+ }
+
+ @VisibleForTesting
+ PendingAntiCompaction(UUID prsId,
+ Collection<ColumnFamilyStore> tables,
+ RangesAtEndpoint tokenRanges,
+ int acquireRetrySeconds,
+ int acquireSleepMillis,
+ ExecutorService executor,
+ BooleanSupplier isCancelled)
+ {
+ this.prsId = prsId;
+ this.tables = tables;
+ this.tokenRanges = tokenRanges;
+ this.executor = executor;
+ this.acquireRetrySeconds = acquireRetrySeconds;
+ this.acquireSleepMillis = acquireSleepMillis;
+ this.isCancelled = isCancelled;
+ }
+
+ public ListenableFuture run()
+ {
+ List<ListenableFutureTask<AcquireResult>> tasks = new ArrayList<>(tables.size());
+ for (ColumnFamilyStore cfs : tables)
+ {
+ cfs.forceBlockingFlush();
+ ListenableFutureTask<AcquireResult> task = ListenableFutureTask.create(getAcquisitionCallable(cfs, tokenRanges.ranges(), prsId, acquireRetrySeconds, acquireSleepMillis));
+ executor.submit(task);
+ tasks.add(task);
+ }
+ ListenableFuture<List<AcquireResult>> acquisitionResults = Futures.successfulAsList(tasks);
+ ListenableFuture compactionResult = Futures.transformAsync(acquisitionResults, getAcquisitionCallback(prsId, tokenRanges), MoreExecutors.directExecutor());
+ return compactionResult;
+ }
+
+ @VisibleForTesting
+ protected AcquisitionCallable getAcquisitionCallable(ColumnFamilyStore cfs, Set<Range<Token>> ranges, UUID prsId, int acquireRetrySeconds, int acquireSleepMillis)
+ {
+ return new AcquisitionCallable(cfs, ranges, prsId, acquireRetrySeconds, acquireSleepMillis);
+ }
+
+ @VisibleForTesting
+ protected AcquisitionCallback getAcquisitionCallback(UUID prsId, RangesAtEndpoint tokenRanges)
+ {
+ return new AcquisitionCallback(prsId, tokenRanges, isCancelled);
+ }
+}
diff --git a/src/java/org/apache/cassandra/db/rows/AbstractCell.java b/src/java/org/apache/cassandra/db/rows/AbstractCell.java
index a993f36..3f2da96 100644
--- a/src/java/org/apache/cassandra/db/rows/AbstractCell.java
+++ b/src/java/org/apache/cassandra/db/rows/AbstractCell.java
@@ -18,18 +18,17 @@
package org.apache.cassandra.db.rows;
import java.nio.ByteBuffer;
-import java.security.MessageDigest;
import java.util.Objects;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.db.Digest;
import org.apache.cassandra.db.DeletionPurger;
import org.apache.cassandra.db.TypeSizes;
import org.apache.cassandra.db.context.CounterContext;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.db.marshal.CollectionType;
+import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.serializers.MarshalException;
import org.apache.cassandra.utils.ByteBufferUtil;
-import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.memory.AbstractAllocator;
/**
@@ -40,7 +39,7 @@
*/
public abstract class AbstractCell extends Cell
{
- protected AbstractCell(ColumnDefinition column)
+ protected AbstractCell(ColumnMetadata column)
{
super(column);
}
@@ -120,20 +119,16 @@
+ (path == null ? 0 : path.dataSize());
}
- public void digest(MessageDigest digest)
+ public void digest(Digest digest)
{
if (isCounterCell())
- {
- CounterContext.instance().updateDigest(digest, value());
- }
+ digest.updateWithCounterContext(value());
else
- {
- digest.update(value().duplicate());
- }
+ digest.update(value());
- FBUtilities.updateWithLong(digest, timestamp());
- FBUtilities.updateWithInt(digest, ttl());
- FBUtilities.updateWithBoolean(digest, isCounterCell());
+ digest.updateWithLong(timestamp())
+ .updateWithInt(ttl())
+ .updateWithBoolean(isCounterCell());
if (path() != null)
path().digest(digest);
}
@@ -148,12 +143,19 @@
throw new MarshalException("Shoud not have a TTL without an associated local deletion time");
// non-frozen UDTs require both the cell path & value to validate,
- // so that logic is pushed down into ColumnDefinition. Tombstone
+ // so that logic is pushed down into ColumnMetadata. Tombstone
// validation is done there too as it also involves the cell path
// for complex columns
column().validateCell(this);
}
+ public boolean hasInvalidDeletions()
+ {
+ if (ttl() < 0 || localDeletionTime() < 0 || (isExpiring() && localDeletionTime() == NO_DELETION_TIME))
+ return true;
+ return false;
+ }
+
public long maxTimestamp()
{
return timestamp();
@@ -196,8 +198,8 @@
CollectionType ct = (CollectionType)type;
return String.format("[%s[%s]=%s %s]",
column().name,
- safeToString(ct.nameComparator(), path().get(0)),
- safeToString(ct.valueComparator(), value()),
+ ct.nameComparator().getString(path().get(0)),
+ ct.valueComparator().getString(value()),
livenessInfoString());
}
if (isTombstone())
diff --git a/src/java/org/apache/cassandra/db/rows/AbstractRangeTombstoneMarker.java b/src/java/org/apache/cassandra/db/rows/AbstractRangeTombstoneMarker.java
index 153243c..a7c48c1 100644
--- a/src/java/org/apache/cassandra/db/rows/AbstractRangeTombstoneMarker.java
+++ b/src/java/org/apache/cassandra/db/rows/AbstractRangeTombstoneMarker.java
@@ -19,7 +19,7 @@
import java.nio.ByteBuffer;
-import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.db.ClusteringBoundOrBoundary;
public abstract class AbstractRangeTombstoneMarker<B extends ClusteringBoundOrBoundary> implements RangeTombstoneMarker
@@ -56,7 +56,7 @@
return bound.isClose(reversed);
}
- public void validateData(CFMetaData metadata)
+ public void validateData(TableMetadata metadata)
{
ClusteringBoundOrBoundary bound = clustering();
for (int i = 0; i < bound.size(); i++)
@@ -67,11 +67,11 @@
}
}
- public String toString(CFMetaData metadata, boolean fullDetails)
+ public String toString(TableMetadata metadata, boolean fullDetails)
{
return toString(metadata);
}
- public String toString(CFMetaData metadata, boolean includeClusteringKeys, boolean fullDetails)
+ public String toString(TableMetadata metadata, boolean includeClusteringKeys, boolean fullDetails)
{
return toString(metadata);
}
diff --git a/src/java/org/apache/cassandra/db/rows/AbstractRow.java b/src/java/org/apache/cassandra/db/rows/AbstractRow.java
index 7cc864d..fc90e34 100644
--- a/src/java/org/apache/cassandra/db/rows/AbstractRow.java
+++ b/src/java/org/apache/cassandra/db/rows/AbstractRow.java
@@ -17,23 +17,19 @@
package org.apache.cassandra.db.rows;
import java.nio.ByteBuffer;
-import java.security.MessageDigest;
-import java.util.AbstractCollection;
-import java.util.Collections;
import java.util.Objects;
-import java.util.Set;
import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.StreamSupport;
import com.google.common.collect.Iterables;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.db.*;
+import org.apache.cassandra.db.Clustering;
+import org.apache.cassandra.db.Digest;
import org.apache.cassandra.db.marshal.CollectionType;
import org.apache.cassandra.db.marshal.UserType;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.serializers.MarshalException;
-import org.apache.cassandra.utils.FBUtilities;
/**
* Base abstract class for {@code Row} implementations.
@@ -63,25 +59,18 @@
return clustering() == Clustering.STATIC_CLUSTERING;
}
- public void digest(MessageDigest digest)
+ public void digest(Digest digest)
{
- digest(digest, Collections.emptySet());
- }
-
- public void digest(MessageDigest digest, Set<ByteBuffer> columnsToExclude)
- {
- FBUtilities.updateWithByte(digest, kind().ordinal());
+ digest.updateWithByte(kind().ordinal());
clustering().digest(digest);
deletion().digest(digest);
primaryKeyLivenessInfo().digest(digest);
- for (ColumnData cd : this)
- if (!columnsToExclude.contains(cd.column.name.bytes))
- cd.digest(digest);
+ apply(ColumnData::digest, digest);
}
- public void validateData(CFMetaData metadata)
+ public void validateData(TableMetadata metadata)
{
Clustering clustering = clustering();
for (int i = 0; i < clustering.size(); i++)
@@ -104,15 +93,19 @@
if (deletion().time().localDeletionTime() < 0)
throw new MarshalException("A local deletion time should not be negative in '" + metadata + "'");
+ apply(cd -> cd.validate());
+ }
+
+ public boolean hasInvalidDeletions()
+ {
+ if (primaryKeyLivenessInfo().isExpiring() && (primaryKeyLivenessInfo().ttl() < 0 || primaryKeyLivenessInfo().localExpirationTime() < 0))
+ return true;
+ if (!deletion().time().validate())
+ return true;
for (ColumnData cd : this)
- try
- {
- cd.validate();
- }
- catch (Exception e)
- {
- throw new MarshalException("data for '" + cd.column.debugString() + "', " + cd + " in '" + metadata + "' didn't validate", e);
- }
+ if (cd.hasInvalidDeletions())
+ return true;
+ return false;
}
public String toString()
@@ -120,17 +113,17 @@
return columnData().toString();
}
- public String toString(CFMetaData metadata)
+ public String toString(TableMetadata metadata)
{
return toString(metadata, false);
}
- public String toString(CFMetaData metadata, boolean fullDetails)
+ public String toString(TableMetadata metadata, boolean fullDetails)
{
return toString(metadata, true, fullDetails);
}
- public String toString(CFMetaData metadata, boolean includeClusterKeys, boolean fullDetails)
+ public String toString(TableMetadata metadata, boolean includeClusterKeys, boolean fullDetails)
{
StringBuilder sb = new StringBuilder();
sb.append("Row");
@@ -200,7 +193,10 @@
ut.fieldType(fId).getString(cell.value()));
};
}
- transform = transform != null ? transform : cell -> "";
+ else
+ {
+ transform = cell -> "";
+ }
sb.append(StreamSupport.stream(complexData.spliterator(), false)
.map(transform)
.collect(Collectors.joining(", ", "{", "}")));
diff --git a/src/java/org/apache/cassandra/db/rows/AbstractUnfilteredRowIterator.java b/src/java/org/apache/cassandra/db/rows/AbstractUnfilteredRowIterator.java
index f2389a7..2c3f78f 100644
--- a/src/java/org/apache/cassandra/db/rows/AbstractUnfilteredRowIterator.java
+++ b/src/java/org/apache/cassandra/db/rows/AbstractUnfilteredRowIterator.java
@@ -17,25 +17,25 @@
*/
package org.apache.cassandra.db.rows;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.utils.AbstractIterator;
-import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.db.*;
public abstract class AbstractUnfilteredRowIterator extends AbstractIterator<Unfiltered> implements UnfilteredRowIterator
{
- protected final CFMetaData metadata;
+ protected final TableMetadata metadata;
protected final DecoratedKey partitionKey;
protected final DeletionTime partitionLevelDeletion;
- protected final PartitionColumns columns;
+ protected final RegularAndStaticColumns columns;
protected final Row staticRow;
protected final boolean isReverseOrder;
protected final EncodingStats stats;
- protected AbstractUnfilteredRowIterator(CFMetaData metadata,
+ protected AbstractUnfilteredRowIterator(TableMetadata metadata,
DecoratedKey partitionKey,
DeletionTime partitionLevelDeletion,
- PartitionColumns columns,
+ RegularAndStaticColumns columns,
Row staticRow,
boolean isReverseOrder,
EncodingStats stats)
@@ -49,12 +49,12 @@
this.stats = stats;
}
- public CFMetaData metadata()
+ public TableMetadata metadata()
{
return metadata;
}
- public PartitionColumns columns()
+ public RegularAndStaticColumns columns()
{
return columns;
}
diff --git a/src/java/org/apache/cassandra/db/rows/BTreeRow.java b/src/java/org/apache/cassandra/db/rows/BTreeRow.java
index ba81a4e..6689c77 100644
--- a/src/java/org/apache/cassandra/db/rows/BTreeRow.java
+++ b/src/java/org/apache/cassandra/db/rows/BTreeRow.java
@@ -19,20 +19,22 @@
import java.nio.ByteBuffer;
import java.util.*;
+import java.util.function.BiConsumer;
import java.util.function.Consumer;
import java.util.function.Predicate;
import com.google.common.base.Function;
import com.google.common.collect.Collections2;
-import com.google.common.collect.Iterables;
import com.google.common.collect.Iterators;
+import com.google.common.primitives.Ints;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.filter.ColumnFilter;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.db.partitions.PartitionUpdate;
+import org.apache.cassandra.schema.DroppedColumn;
import org.apache.cassandra.utils.*;
import org.apache.cassandra.utils.btree.BTree;
import org.apache.cassandra.utils.btree.BTreeSearchIterator;
@@ -52,6 +54,11 @@
// The data for each columns present in this row in column sorted order.
private final Object[] btree;
+ private static final ColumnData FIRST_COMPLEX_STATIC = new ComplexColumnData(Columns.FIRST_COMPLEX_STATIC, new Object[0], new DeletionTime(0, 0));
+ private static final ColumnData FIRST_COMPLEX_REGULAR = new ComplexColumnData(Columns.FIRST_COMPLEX_REGULAR, new Object[0], new DeletionTime(0, 0));
+ private static final Comparator<ColumnData> COLUMN_COMPARATOR = (cd1, cd2) -> cd1.column.compareTo(cd2.column);
+
+
// We need to filter the tombstones of a row on every read (twice in fact: first to remove purgeable tombstone, and then after reconciliation to remove
// all tombstone since we don't return them to the client) as well as on compaction. But it's likely that many rows won't have any tombstone at all, so
// we want to speed up that case by not having to iterate/copy the row in this case. We could keep a single boolean telling us if we have tombstones,
@@ -89,8 +96,8 @@
int minDeletionTime = Math.min(minDeletionTime(primaryKeyLivenessInfo), minDeletionTime(deletion.time()));
if (minDeletionTime != Integer.MIN_VALUE)
{
- for (ColumnData cd : BTree.<ColumnData>iterable(btree))
- minDeletionTime = Math.min(minDeletionTime, minDeletionTime(cd));
+ long result = BTree.<ColumnData>accumulate(btree, (cd, l) -> Math.min(l, minDeletionTime(cd)) , minDeletionTime);
+ minDeletionTime = Ints.checkedCast(result);
}
return create(clustering, primaryKeyLivenessInfo, deletion, btree, minDeletionTime);
@@ -167,23 +174,49 @@
return cd.column().isSimple() ? minDeletionTime((Cell) cd) : minDeletionTime((ComplexColumnData)cd);
}
- public void apply(Consumer<ColumnData> function, boolean reversed)
+ public void apply(Consumer<ColumnData> function)
{
- BTree.apply(btree, function, reversed);
+ BTree.apply(btree, function);
}
- public void apply(Consumer<ColumnData> funtion, com.google.common.base.Predicate<ColumnData> stopCondition, boolean reversed)
+ public <A> void apply(BiConsumer<A, ColumnData> function, A arg)
{
- BTree.apply(btree, funtion, stopCondition, reversed);
+ BTree.apply(btree, function, arg);
+ }
+
+ public long accumulate(LongAccumulator<ColumnData> accumulator, long initialValue)
+ {
+ return BTree.accumulate(btree, accumulator, initialValue);
+ }
+
+ public long accumulate(LongAccumulator<ColumnData> accumulator, Comparator<ColumnData> comparator, ColumnData from, long initialValue)
+ {
+ return BTree.accumulate(btree, accumulator, comparator, from, initialValue);
+ }
+
+ public <A> long accumulate(BiLongAccumulator<A, ColumnData> accumulator, A arg, long initialValue)
+ {
+ return BTree.accumulate(btree, accumulator, arg, initialValue);
+ }
+
+ public <A> long accumulate(BiLongAccumulator<A, ColumnData> accumulator, A arg, Comparator<ColumnData> comparator, ColumnData from, long initialValue)
+ {
+ return BTree.accumulate(btree, accumulator, arg, comparator, from, initialValue);
}
private static int minDeletionTime(Object[] btree, LivenessInfo info, DeletionTime rowDeletion)
{
- //we have to wrap this for the lambda
- final WrappedInt min = new WrappedInt(Math.min(minDeletionTime(info), minDeletionTime(rowDeletion)));
+ long min = Math.min(minDeletionTime(info), minDeletionTime(rowDeletion));
- BTree.<ColumnData>apply(btree, cd -> min.set( Math.min(min.get(), minDeletionTime(cd)) ), cd -> min.get() == Integer.MIN_VALUE, false);
- return min.get();
+ min = BTree.<ColumnData>accumulate(btree, (cd, l) -> {
+ int m = Math.min((int) l, minDeletionTime(cd));
+ return m != Integer.MIN_VALUE ? m : Long.MAX_VALUE;
+ }, min);
+
+ if (min == Long.MAX_VALUE)
+ return Integer.MIN_VALUE;
+
+ return Ints.checkedCast(min);
}
public Clustering clustering()
@@ -191,7 +224,7 @@
return clustering;
}
- public Collection<ColumnDefinition> columns()
+ public Collection<ColumnMetadata> columns()
{
return Collections2.transform(columnData(), ColumnData::column);
}
@@ -218,13 +251,13 @@
return deletion;
}
- public Cell getCell(ColumnDefinition c)
+ public Cell getCell(ColumnMetadata c)
{
assert !c.isComplex();
- return (Cell) BTree.<Object>find(btree, ColumnDefinition.asymmetricColumnDataComparator, c);
+ return (Cell) BTree.<Object>find(btree, ColumnMetadata.asymmetricColumnDataComparator, c);
}
- public Cell getCell(ColumnDefinition c, CellPath path)
+ public Cell getCell(ColumnMetadata c, CellPath path)
{
assert c.isComplex();
ComplexColumnData cd = getComplexColumnData(c);
@@ -233,10 +266,10 @@
return cd.getCell(path);
}
- public ComplexColumnData getComplexColumnData(ColumnDefinition c)
+ public ComplexColumnData getComplexColumnData(ColumnMetadata c)
{
assert c.isComplex();
- return (ComplexColumnData) BTree.<Object>find(btree, ColumnDefinition.asymmetricColumnDataComparator, c);
+ return (ComplexColumnData) BTree.<Object>find(btree, ColumnMetadata.asymmetricColumnDataComparator, c);
}
@Override
@@ -259,21 +292,21 @@
return CellIterator::new;
}
- public BTreeSearchIterator<ColumnDefinition, ColumnData> searchIterator()
+ public BTreeSearchIterator<ColumnMetadata, ColumnData> searchIterator()
{
- return BTree.slice(btree, ColumnDefinition.asymmetricColumnDataComparator, BTree.Dir.ASC);
+ return BTree.slice(btree, ColumnMetadata.asymmetricColumnDataComparator, BTree.Dir.ASC);
}
- public Row filter(ColumnFilter filter, CFMetaData metadata)
+ public Row filter(ColumnFilter filter, TableMetadata metadata)
{
return filter(filter, DeletionTime.LIVE, false, metadata);
}
- public Row filter(ColumnFilter filter, DeletionTime activeDeletion, boolean setActiveDeletionToRow, CFMetaData metadata)
+ public Row filter(ColumnFilter filter, DeletionTime activeDeletion, boolean setActiveDeletionToRow, TableMetadata metadata)
{
- Map<ByteBuffer, CFMetaData.DroppedColumn> droppedColumns = metadata.getDroppedColumns();
+ Map<ByteBuffer, DroppedColumn> droppedColumns = metadata.droppedColumns;
- boolean mayFilterColumns = !filter.fetchesAllColumns() || !filter.allFetchedColumnsAreQueried();
+ boolean mayFilterColumns = !filter.fetchesAllColumns(isStatic());
boolean mayHaveShadowed = activeDeletion.supersedes(deletion.time());
if (!mayFilterColumns && !mayHaveShadowed && droppedColumns.isEmpty())
@@ -292,16 +325,16 @@
}
Columns columns = filter.fetchedColumns().columns(isStatic());
- Predicate<ColumnDefinition> inclusionTester = columns.inOrderInclusionTester();
- Predicate<ColumnDefinition> queriedByUserTester = filter.queriedColumns().columns(isStatic()).inOrderInclusionTester();
+ Predicate<ColumnMetadata> inclusionTester = columns.inOrderInclusionTester();
+ Predicate<ColumnMetadata> queriedByUserTester = filter.queriedColumns().columns(isStatic()).inOrderInclusionTester();
final LivenessInfo rowLiveness = newInfo;
return transformAndFilter(newInfo, newDeletion, (cd) -> {
- ColumnDefinition column = cd.column();
+ ColumnMetadata column = cd.column();
if (!inclusionTester.test(column))
return null;
- CFMetaData.DroppedColumn dropped = droppedColumns.get(column.name.bytes);
+ DroppedColumn dropped = droppedColumns.get(column.name.bytes);
if (column.isComplex())
return ((ComplexColumnData) cd).filter(filter, mayHaveShadowed ? activeDeletion : DeletionTime.LIVE, dropped, rowLiveness);
@@ -323,7 +356,7 @@
return transformAndFilter(primaryKeyLivenessInfo, deletion, (cd) -> {
- ColumnDefinition column = cd.column();
+ ColumnMetadata column = cd.column();
if (column.isComplex())
return ((ComplexColumnData)cd).withOnlyQueriedData(filter);
@@ -333,33 +366,19 @@
public boolean hasComplex()
{
- // We start by the end cause we know complex columns sort after the simple ones
- ColumnData cd = Iterables.getFirst(BTree.<ColumnData>iterable(btree, BTree.Dir.DESC), null);
- return cd != null && cd.column.isComplex();
+ if (BTree.isEmpty(btree))
+ return false;
+
+ int size = BTree.size(btree);
+ ColumnData last = BTree.findByIndex(btree, size - 1);
+ return last.column.isComplex();
}
public boolean hasComplexDeletion()
{
- final WrappedBoolean result = new WrappedBoolean(false);
-
- // We start by the end cause we know complex columns sort before simple ones
- apply(c -> {}, cd -> {
- if (cd.column.isSimple())
- {
- result.set(false);
- return true;
- }
-
- if (!((ComplexColumnData) cd).complexDeletion().isLive())
- {
- result.set(true);
- return true;
- }
-
- return false;
- }, true);
-
- return result.get();
+ long result = accumulate((cd, v) -> ((ComplexColumnData) cd).complexDeletion().isLive() ? 0 : Long.MAX_VALUE,
+ COLUMN_COMPARATOR, isStatic() ? FIRST_COMPLEX_STATIC : FIRST_COMPLEX_REGULAR, 0L);
+ return result == Long.MAX_VALUE;
}
public Row markCounterLocalToBeCleared()
@@ -374,6 +393,15 @@
return nowInSec >= minLocalDeletionTime;
}
+ public boolean hasInvalidDeletions()
+ {
+ if (primaryKeyLivenessInfo().isExpiring() && (primaryKeyLivenessInfo().ttl() < 0 || primaryKeyLivenessInfo().localExpirationTime() < 0))
+ return true;
+ if (!deletion().time().validate())
+ return true;
+ return accumulate((cd, v) -> cd.hasInvalidDeletions() ? Long.MAX_VALUE : v, 0) != 0;
+ }
+
/**
* Returns a copy of the row where all timestamps for live data have replaced by {@code newTimestamp} and
* all deletion timestamp by {@code newTimestamp - 1}.
@@ -439,9 +467,7 @@
+ primaryKeyLivenessInfo.dataSize()
+ deletion.dataSize();
- for (ColumnData cd : this)
- dataSize += cd.dataSize();
- return dataSize;
+ return Ints.checkedCast(accumulate((cd, v) -> v + cd.dataSize(), dataSize));
}
public long unsharedHeapSizeExcludingData()
@@ -450,9 +476,7 @@
+ clustering.unsharedHeapSizeExcludingData()
+ BTree.sizeOfStructureOnHeap(btree);
- for (ColumnData cd : this)
- heapSize += cd.unsharedHeapSizeExcludingData();
- return heapSize;
+ return accumulate((cd, v) -> v + cd.unsharedHeapSizeExcludingData(), heapSize);
}
public static Row.Builder sortedBuilder()
@@ -460,25 +484,25 @@
return new Builder(true);
}
- public static Row.Builder unsortedBuilder(int nowInSec)
+ public static Row.Builder unsortedBuilder()
{
- return new Builder(false, nowInSec);
+ return new Builder(false);
}
// This is only used by PartitionUpdate.CounterMark but other uses should be avoided as much as possible as it breaks our general
// assumption that Row objects are immutable. This method should go away post-#6506 in particular.
// This method is in particular not exposed by the Row API on purpose.
// This method also *assumes* that the cell we're setting already exists.
- public void setValue(ColumnDefinition column, CellPath path, ByteBuffer value)
+ public void setValue(ColumnMetadata column, CellPath path, ByteBuffer value)
{
- ColumnData current = (ColumnData) BTree.<Object>find(btree, ColumnDefinition.asymmetricColumnDataComparator, column);
+ ColumnData current = (ColumnData) BTree.<Object>find(btree, ColumnMetadata.asymmetricColumnDataComparator, column);
if (column.isSimple())
BTree.replaceInSitu(btree, ColumnData.comparator, current, ((Cell) current).withUpdatedValue(value));
else
((ComplexColumnData) current).setValue(path, value);
}
- public Iterable<Cell> cellsInLegacyOrder(CFMetaData metadata, boolean reversed)
+ public Iterable<Cell> cellsInLegacyOrder(TableMetadata metadata, boolean reversed)
{
return () -> new CellInLegacyOrderIterator(metadata, reversed);
}
@@ -522,9 +546,9 @@
private Iterator<Cell> complexCells;
private final Object[] data;
- private CellInLegacyOrderIterator(CFMetaData metadata, boolean reversed)
+ private CellInLegacyOrderIterator(TableMetadata metadata, boolean reversed)
{
- AbstractType<?> nameComparator = metadata.getColumnDefinitionNameComparator(isStatic() ? ColumnDefinition.Kind.STATIC : ColumnDefinition.Kind.REGULAR);
+ AbstractType<?> nameComparator = metadata.columnDefinitionNameComparator(isStatic() ? ColumnMetadata.Kind.STATIC : ColumnMetadata.Kind.REGULAR);
this.comparator = reversed ? Collections.reverseOrder(nameComparator) : nameComparator;
this.reversed = reversed;
@@ -605,7 +629,7 @@
// a simple marker class that will sort to the beginning of a run of complex cells to store the deletion time
private static class ComplexColumnDeletion extends BufferCell
{
- public ComplexColumnDeletion(ColumnDefinition column, DeletionTime deletionTime)
+ public ComplexColumnDeletion(ColumnMetadata column, DeletionTime deletionTime)
{
super(column, deletionTime.markedForDeleteAt(), 0, deletionTime.localDeletionTime(), ByteBufferUtil.EMPTY_BYTE_BUFFER, CellPath.BOTTOM);
}
@@ -614,21 +638,16 @@
// converts a run of Cell with equal column into a ColumnData
private static class CellResolver implements BTree.Builder.Resolver
{
- final int nowInSec;
- private CellResolver(int nowInSec)
- {
- this.nowInSec = nowInSec;
- }
+ static final CellResolver instance = new CellResolver();
public ColumnData resolve(Object[] cells, int lb, int ub)
{
Cell cell = (Cell) cells[lb];
- ColumnDefinition column = cell.column;
+ ColumnMetadata column = cell.column;
if (cell.column.isSimple())
{
- assert lb + 1 == ub || nowInSec != Integer.MIN_VALUE;
while (++lb < ub)
- cell = Cells.reconcile(cell, (Cell) cells[lb], nowInSec);
+ cell = Cells.reconcile(cell, (Cell) cells[lb]);
return cell;
}
@@ -661,7 +680,7 @@
{
if (previous != null && column.cellComparator().compare(previous, c) == 0)
{
- c = Cells.reconcile(previous, c, nowInSec);
+ c = Cells.reconcile(previous, c);
buildFrom.set(buildFrom.size() - 1, c);
}
else
@@ -675,28 +694,21 @@
Object[] btree = BTree.build(buildFrom, UpdateFunction.noOp());
return new ComplexColumnData(column, btree, deletion);
}
-
}
+
protected Clustering clustering;
protected LivenessInfo primaryKeyLivenessInfo = LivenessInfo.EMPTY;
protected Deletion deletion = Deletion.LIVE;
private final boolean isSorted;
private BTree.Builder<Cell> cells_;
- private final CellResolver resolver;
private boolean hasComplex = false;
// For complex column at index i of 'columns', we store at complexDeletions[i] its complex deletion.
protected Builder(boolean isSorted)
{
- this(isSorted, Integer.MIN_VALUE);
- }
-
- protected Builder(boolean isSorted, int nowInSecs)
- {
cells_ = null;
- resolver = new CellResolver(nowInSecs);
this.isSorted = isSorted;
}
@@ -716,7 +728,6 @@
primaryKeyLivenessInfo = builder.primaryKeyLivenessInfo;
deletion = builder.deletion;
cells_ = builder.cells_ == null ? null : builder.cells_.copy();
- resolver = builder.resolver;
isSorted = builder.isSorted;
hasComplex = builder.hasComplex;
}
@@ -779,7 +790,7 @@
hasComplex |= cell.column.isComplex();
}
- public void addComplexDeletion(ColumnDefinition column, DeletionTime complexDeletion)
+ public void addComplexDeletion(ColumnMetadata column, DeletionTime complexDeletion)
{
getCells().add(new ComplexColumnDeletion(column, complexDeletion));
hasComplex = true;
@@ -792,7 +803,7 @@
// we can avoid resolving if we're sorted and have no complex values
// (because we'll only have unique simple cells, which are already in their final condition)
if (!isSorted | hasComplex)
- getCells().resolve(resolver);
+ getCells().resolve(CellResolver.instance);
Object[] btree = getCells().build();
if (deletion.isShadowedBy(primaryKeyLivenessInfo))
diff --git a/src/java/org/apache/cassandra/db/rows/BaseRowIterator.java b/src/java/org/apache/cassandra/db/rows/BaseRowIterator.java
index ce37297..4033248 100644
--- a/src/java/org/apache/cassandra/db/rows/BaseRowIterator.java
+++ b/src/java/org/apache/cassandra/db/rows/BaseRowIterator.java
@@ -18,9 +18,9 @@
*/
package org.apache.cassandra.db.rows;
-import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.db.DecoratedKey;
-import org.apache.cassandra.db.PartitionColumns;
+import org.apache.cassandra.db.RegularAndStaticColumns;
import org.apache.cassandra.utils.CloseableIterator;
/**
@@ -32,7 +32,7 @@
/**
* The metadata for the table this iterator on.
*/
- public CFMetaData metadata();
+ public TableMetadata metadata();
/**
* Whether or not the rows returned by this iterator are in reversed
@@ -44,7 +44,7 @@
* A subset of the columns for the (static and regular) rows returned by this iterator.
* Every row returned by this iterator must guarantee that it has only those columns.
*/
- public PartitionColumns columns();
+ public RegularAndStaticColumns columns();
/**
* The partition key of the partition this in an iterator over.
diff --git a/src/java/org/apache/cassandra/db/rows/BufferCell.java b/src/java/org/apache/cassandra/db/rows/BufferCell.java
index b62d95a..8bf8f7d 100644
--- a/src/java/org/apache/cassandra/db/rows/BufferCell.java
+++ b/src/java/org/apache/cassandra/db/rows/BufferCell.java
@@ -19,8 +19,8 @@
import java.nio.ByteBuffer;
-import org.apache.cassandra.config.ColumnDefinition;
import org.apache.cassandra.db.ExpirationDateOverflowHandling;
+import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.db.marshal.ByteType;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.ObjectSizes;
@@ -28,7 +28,7 @@
public class BufferCell extends AbstractCell
{
- private static final long EMPTY_SIZE = ObjectSizes.measure(new BufferCell(ColumnDefinition.regularDef("", "", "", ByteType.instance), 0L, 0, 0, ByteBufferUtil.EMPTY_BYTE_BUFFER, null));
+ private static final long EMPTY_SIZE = ObjectSizes.measure(new BufferCell(ColumnMetadata.regularColumn("", "", "", ByteType.instance), 0L, 0, 0, ByteBufferUtil.EMPTY_BYTE_BUFFER, null));
private final long timestamp;
private final int ttl;
@@ -37,7 +37,7 @@
private final ByteBuffer value;
private final CellPath path;
- public BufferCell(ColumnDefinition column, long timestamp, int ttl, int localDeletionTime, ByteBuffer value, CellPath path)
+ public BufferCell(ColumnMetadata column, long timestamp, int ttl, int localDeletionTime, ByteBuffer value, CellPath path)
{
super(column);
assert !column.isPrimaryKeyColumn();
@@ -49,33 +49,33 @@
this.path = path;
}
- public static BufferCell live(ColumnDefinition column, long timestamp, ByteBuffer value)
+ public static BufferCell live(ColumnMetadata column, long timestamp, ByteBuffer value)
{
return live(column, timestamp, value, null);
}
- public static BufferCell live(ColumnDefinition column, long timestamp, ByteBuffer value, CellPath path)
+ public static BufferCell live(ColumnMetadata column, long timestamp, ByteBuffer value, CellPath path)
{
return new BufferCell(column, timestamp, NO_TTL, NO_DELETION_TIME, value, path);
}
- public static BufferCell expiring(ColumnDefinition column, long timestamp, int ttl, int nowInSec, ByteBuffer value)
+ public static BufferCell expiring(ColumnMetadata column, long timestamp, int ttl, int nowInSec, ByteBuffer value)
{
return expiring(column, timestamp, ttl, nowInSec, value, null);
}
- public static BufferCell expiring(ColumnDefinition column, long timestamp, int ttl, int nowInSec, ByteBuffer value, CellPath path)
+ public static BufferCell expiring(ColumnMetadata column, long timestamp, int ttl, int nowInSec, ByteBuffer value, CellPath path)
{
assert ttl != NO_TTL;
return new BufferCell(column, timestamp, ttl, ExpirationDateOverflowHandling.computeLocalExpirationTime(nowInSec, ttl), value, path);
}
- public static BufferCell tombstone(ColumnDefinition column, long timestamp, int nowInSec)
+ public static BufferCell tombstone(ColumnMetadata column, long timestamp, int nowInSec)
{
return tombstone(column, timestamp, nowInSec, null);
}
- public static BufferCell tombstone(ColumnDefinition column, long timestamp, int nowInSec, CellPath path)
+ public static BufferCell tombstone(ColumnMetadata column, long timestamp, int nowInSec, CellPath path)
{
return new BufferCell(column, timestamp, NO_TTL, nowInSec, ByteBufferUtil.EMPTY_BYTE_BUFFER, path);
}
@@ -105,7 +105,7 @@
return path;
}
- public Cell withUpdatedColumn(ColumnDefinition newColumn)
+ public Cell withUpdatedColumn(ColumnMetadata newColumn)
{
return new BufferCell(newColumn, timestamp, ttl, localDeletionTime, value, path);
}
diff --git a/src/java/org/apache/cassandra/db/rows/Cell.java b/src/java/org/apache/cassandra/db/rows/Cell.java
index 1205b7d..959676a 100644
--- a/src/java/org/apache/cassandra/db/rows/Cell.java
+++ b/src/java/org/apache/cassandra/db/rows/Cell.java
@@ -21,17 +21,11 @@
import java.nio.ByteBuffer;
import java.util.Comparator;
-import com.google.common.annotations.VisibleForTesting;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
-import org.apache.cassandra.cql3.Attributes;
import org.apache.cassandra.config.*;
import org.apache.cassandra.db.*;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.memory.AbstractAllocator;
@@ -62,7 +56,7 @@
public static final Serializer serializer = new BufferCell.Serializer();
- protected Cell(ColumnDefinition column)
+ protected Cell(ColumnMetadata column)
{
super(column);
}
@@ -138,7 +132,7 @@
*/
public abstract CellPath path();
- public abstract Cell withUpdatedColumn(ColumnDefinition newColumn);
+ public abstract Cell withUpdatedColumn(ColumnMetadata newColumn);
public abstract Cell withUpdatedValue(ByteBuffer newValue);
@@ -181,7 +175,7 @@
private final static int USE_ROW_TIMESTAMP_MASK = 0x08; // Wether the cell has the same timestamp than the row this is a cell of.
private final static int USE_ROW_TTL_MASK = 0x10; // Wether the cell has the same ttl than the row this is a cell of.
- public void serialize(Cell cell, ColumnDefinition column, DataOutputPlus out, LivenessInfo rowLiveness, SerializationHeader header) throws IOException
+ public void serialize(Cell cell, ColumnMetadata column, DataOutputPlus out, LivenessInfo rowLiveness, SerializationHeader header) throws IOException
{
assert cell != null;
boolean hasValue = cell.value().hasRemaining();
@@ -220,7 +214,7 @@
header.getType(column).writeValue(cell.value(), out);
}
- public Cell deserialize(DataInputPlus in, LivenessInfo rowLiveness, ColumnDefinition column, SerializationHeader header, SerializationHelper helper) throws IOException
+ public Cell deserialize(DataInputPlus in, LivenessInfo rowLiveness, ColumnMetadata column, SerializationHeader header, DeserializationHelper helper) throws IOException
{
int flags = in.readUnsignedByte();
boolean hasValue = (flags & HAS_EMPTY_VALUE_MASK) == 0;
@@ -261,7 +255,7 @@
return new BufferCell(column, timestamp, ttl, localDeletionTime, value, path);
}
- public long serializedSize(Cell cell, ColumnDefinition column, LivenessInfo rowLiveness, SerializationHeader header)
+ public long serializedSize(Cell cell, ColumnMetadata column, LivenessInfo rowLiveness, SerializationHeader header)
{
long size = 1; // flags
boolean hasValue = cell.value().hasRemaining();
@@ -288,7 +282,7 @@
}
// Returns if the skipped cell was an actual cell (i.e. it had its presence flag).
- public boolean skip(DataInputPlus in, ColumnDefinition column, SerializationHeader header) throws IOException
+ public boolean skip(DataInputPlus in, ColumnMetadata column, SerializationHeader header) throws IOException
{
int flags = in.readUnsignedByte();
boolean hasValue = (flags & HAS_EMPTY_VALUE_MASK) == 0;
diff --git a/src/java/org/apache/cassandra/db/rows/CellPath.java b/src/java/org/apache/cassandra/db/rows/CellPath.java
index 91a5217..1bf8b8f 100644
--- a/src/java/org/apache/cassandra/db/rows/CellPath.java
+++ b/src/java/org/apache/cassandra/db/rows/CellPath.java
@@ -19,9 +19,9 @@
import java.io.IOException;
import java.nio.ByteBuffer;
-import java.security.MessageDigest;
import java.util.Objects;
+import org.apache.cassandra.db.Digest;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.utils.ByteBufferUtil;
@@ -54,10 +54,10 @@
return size;
}
- public void digest(MessageDigest digest)
+ public void digest(Digest digest)
{
for (int i = 0; i < size(); i++)
- digest.update(get(i).duplicate());
+ digest.update(get(i));
}
public abstract CellPath copy(AbstractAllocator allocator);
diff --git a/src/java/org/apache/cassandra/db/rows/Cells.java b/src/java/org/apache/cassandra/db/rows/Cells.java
index 38bde16..45d69e8 100644
--- a/src/java/org/apache/cassandra/db/rows/Cells.java
+++ b/src/java/org/apache/cassandra/db/rows/Cells.java
@@ -21,8 +21,8 @@
import java.util.Comparator;
import java.util.Iterator;
-import org.apache.cassandra.config.ColumnDefinition;
-import org.apache.cassandra.db.Conflicts;
+import org.apache.cassandra.db.context.CounterContext;
+import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.db.DeletionTime;
import org.apache.cassandra.db.partitions.PartitionStatisticsCollector;
@@ -66,9 +66,6 @@
* @param deletion the deletion time that applies to the cells being considered.
* This deletion time may delete both {@code existing} or {@code update}.
* @param builder the row builder to which the result of the reconciliation is written.
- * @param nowInSec the current time in seconds (which plays a role during reconciliation
- * because deleted cells always have precedence on timestamp equality and deciding if a
- * cell is a live or not depends on the current time due to expiring cells).
*
* @return the timestamp delta between existing and update, or {@code Long.MAX_VALUE} if one
* of them is {@code null} or deleted by {@code deletion}).
@@ -76,8 +73,7 @@
public static long reconcile(Cell existing,
Cell update,
DeletionTime deletion,
- Row.Builder builder,
- int nowInSec)
+ Row.Builder builder)
{
existing = existing == null || deletion.deletes(existing) ? null : existing;
update = update == null || deletion.deletes(update) ? null : update;
@@ -94,7 +90,7 @@
return Long.MAX_VALUE;
}
- Cell reconciled = reconcile(existing, update, nowInSec);
+ Cell reconciled = reconcile(existing, update);
builder.addCell(reconciled);
return Math.abs(existing.timestamp() - update.timestamp());
@@ -111,59 +107,107 @@
*
* @param c1 the first cell participating in the reconciliation.
* @param c2 the second cell participating in the reconciliation.
- * @param nowInSec the current time in seconds (which plays a role during reconciliation
- * because deleted cells always have precedence on timestamp equality and deciding if a
- * cell is a live or not depends on the current time due to expiring cells).
*
* @return a cell corresponding to the reconciliation of {@code c1} and {@code c2}.
* For non-counter cells, this will always be either {@code c1} or {@code c2}, but for
* counter cells this can be a newly allocated cell.
*/
- public static Cell reconcile(Cell c1, Cell c2, int nowInSec)
+ public static Cell reconcile(Cell c1, Cell c2)
{
- if (c1 == null)
- return c2 == null ? null : c2;
- if (c2 == null)
- return c1;
+ if (c1 == null || c2 == null)
+ return c2 == null ? c1 : c2;
if (c1.isCounterCell() || c2.isCounterCell())
+ return resolveCounter(c1, c2);
+
+ return resolveRegular(c1, c2);
+ }
+
+ private static Cell resolveRegular(Cell left, Cell right)
+ {
+ long leftTimestamp = left.timestamp();
+ long rightTimestamp = right.timestamp();
+ if (leftTimestamp != rightTimestamp)
+ return leftTimestamp > rightTimestamp ? left : right;
+
+ int leftLocalDeletionTime = left.localDeletionTime();
+ int rightLocalDeletionTime = right.localDeletionTime();
+
+ boolean leftIsExpiringOrTombstone = leftLocalDeletionTime != Cell.NO_DELETION_TIME;
+ boolean rightIsExpiringOrTombstone = rightLocalDeletionTime != Cell.NO_DELETION_TIME;
+
+ if (leftIsExpiringOrTombstone | rightIsExpiringOrTombstone)
{
- Conflicts.Resolution res = Conflicts.resolveCounter(c1.timestamp(),
- c1.isLive(nowInSec),
- c1.value(),
- c2.timestamp(),
- c2.isLive(nowInSec),
- c2.value());
+ // Tombstones always win reconciliation with live cells of the same timstamp
+ // CASSANDRA-14592: for consistency of reconciliation, regardless of system clock at time of reconciliation
+ // this requires us to treat expiring cells (which will become tombstones at some future date) the same wrt regular cells
+ if (leftIsExpiringOrTombstone != rightIsExpiringOrTombstone)
+ return leftIsExpiringOrTombstone ? left : right;
- switch (res)
- {
- case LEFT_WINS: return c1;
- case RIGHT_WINS: return c2;
- default:
- ByteBuffer merged = Conflicts.mergeCounterValues(c1.value(), c2.value());
- long timestamp = Math.max(c1.timestamp(), c2.timestamp());
+ // for most historical consistency, we still prefer tombstones over expiring cells.
+ // While this leads to the an inconsistency over which is chosen
+ // (i.e. before expiry, the pure tombstone; after expiry, whichever is more recent)
+ // this inconsistency has no user-visible distinction, as at this point they are both logically tombstones
+ // (the only possible difference is the time at which the cells become purgeable)
+ boolean leftIsTombstone = !left.isExpiring(); // !isExpiring() == isTombstone(), but does not need to consider localDeletionTime()
+ boolean rightIsTombstone = !right.isExpiring();
+ if (leftIsTombstone != rightIsTombstone)
+ return leftIsTombstone ? left : right;
- // We save allocating a new cell object if it turns out that one cell was
- // a complete superset of the other
- if (merged == c1.value() && timestamp == c1.timestamp())
- return c1;
- else if (merged == c2.value() && timestamp == c2.timestamp())
- return c2;
- else // merge clocks and timestamps.
- return new BufferCell(c1.column(), timestamp, Cell.NO_TTL, Cell.NO_DELETION_TIME, merged, c1.path());
- }
+ // ==> (leftIsExpiring && rightIsExpiring) or (leftIsTombstone && rightIsTombstone)
+ // if both are expiring, we do not want to consult the value bytes if we can avoid it, as like with C-14592
+ // the value bytes implicitly depend on the system time at reconciliation, as a
+ // would otherwise always win (unless it had an empty value), until it expired and was translated to a tombstone
+ if (leftLocalDeletionTime != rightLocalDeletionTime)
+ return leftLocalDeletionTime > rightLocalDeletionTime ? left : right;
}
- Conflicts.Resolution res = Conflicts.resolveRegular(c1.timestamp(),
- c1.isLive(nowInSec),
- c1.localDeletionTime(),
- c1.value(),
- c2.timestamp(),
- c2.isLive(nowInSec),
- c2.localDeletionTime(),
- c2.value());
- assert res != Conflicts.Resolution.MERGE;
- return res == Conflicts.Resolution.LEFT_WINS ? c1 : c2;
+ ByteBuffer leftValue = left.value();
+ ByteBuffer rightValue = right.value();
+ return leftValue.compareTo(rightValue) >= 0 ? left : right;
+ }
+
+ private static Cell resolveCounter(Cell left, Cell right)
+ {
+ long leftTimestamp = left.timestamp();
+ long rightTimestamp = right.timestamp();
+
+ boolean leftIsTombstone = left.isTombstone();
+ boolean rightIsTombstone = right.isTombstone();
+
+ if (leftIsTombstone | rightIsTombstone)
+ {
+ // No matter what the counter cell's timestamp is, a tombstone always takes precedence. See CASSANDRA-7346.
+ assert leftIsTombstone != rightIsTombstone;
+ return leftIsTombstone ? left : right;
+ }
+
+ ByteBuffer leftValue = left.value();
+ ByteBuffer rightValue = right.value();
+
+ // Handle empty values. Counters can't truly have empty values, but we can have a counter cell that temporarily
+ // has one on read if the column for the cell is not queried by the user due to the optimization of #10657. We
+ // thus need to handle this (see #11726 too).
+ boolean leftIsEmpty = !leftValue.hasRemaining();
+ boolean rightIsEmpty = !rightValue.hasRemaining();
+ if (leftIsEmpty || rightIsEmpty)
+ {
+ if (leftIsEmpty != rightIsEmpty)
+ return leftIsEmpty ? left : right;
+ return leftTimestamp > rightTimestamp ? left : right;
+ }
+
+ ByteBuffer merged = CounterContext.instance().merge(leftValue, rightValue);
+ long timestamp = Math.max(leftTimestamp, rightTimestamp);
+
+ // We save allocating a new cell object if it turns out that one cell was
+ // a complete superset of the other
+ if (merged == leftValue && timestamp == leftTimestamp)
+ return left;
+ else if (merged == rightValue && timestamp == rightTimestamp)
+ return right;
+ else // merge clocks and timestamps.
+ return new BufferCell(left.column(), timestamp, Cell.NO_TTL, Cell.NO_DELETION_TIME, merged, left.path());
}
/**
@@ -187,9 +231,6 @@
* @param deletion the deletion time that applies to the cells being considered.
* This deletion time may delete cells in both {@code existing} and {@code update}.
* @param builder the row build to which the result of the reconciliation is written.
- * @param nowInSec the current time in seconds (which plays a role during reconciliation
- * because deleted cells always have precedence on timestamp equality and deciding if a
- * cell is a live or not depends on the current time due to expiring cells).
*
* @return the smallest timestamp delta between corresponding cells from existing and update. A
* timestamp delta being computed as the difference between a cell from {@code update} and the
@@ -197,12 +238,11 @@
* of cells from {@code existing} and {@code update} having the same cell path is empty, this
* returns {@code Long.MAX_VALUE}.
*/
- public static long reconcileComplex(ColumnDefinition column,
+ public static long reconcileComplex(ColumnMetadata column,
Iterator<Cell> existing,
Iterator<Cell> update,
DeletionTime deletion,
- Row.Builder builder,
- int nowInSec)
+ Row.Builder builder)
{
Comparator<CellPath> comparator = column.cellPathComparator();
Cell nextExisting = getNext(existing);
@@ -215,17 +255,17 @@
: comparator.compare(nextExisting.path(), nextUpdate.path()));
if (cmp < 0)
{
- reconcile(nextExisting, null, deletion, builder, nowInSec);
+ reconcile(nextExisting, null, deletion, builder);
nextExisting = getNext(existing);
}
else if (cmp > 0)
{
- reconcile(null, nextUpdate, deletion, builder, nowInSec);
+ reconcile(null, nextUpdate, deletion, builder);
nextUpdate = getNext(update);
}
else
{
- timeDelta = Math.min(timeDelta, reconcile(nextExisting, nextUpdate, deletion, builder, nowInSec));
+ timeDelta = Math.min(timeDelta, reconcile(nextExisting, nextUpdate, deletion, builder));
nextExisting = getNext(existing);
nextUpdate = getNext(update);
}
@@ -246,20 +286,16 @@
* @param deletion the deletion time that applies to the cells being considered.
* This deletion time may delete both {@code existing} or {@code update}.
* @param builder the row builder to which the result of the filtering is written.
- * @param nowInSec the current time in seconds (which plays a role during reconciliation
- * because deleted cells always have precedence on timestamp equality and deciding if a
- * cell is a live or not depends on the current time due to expiring cells).
*/
public static void addNonShadowed(Cell existing,
Cell update,
DeletionTime deletion,
- Row.Builder builder,
- int nowInSec)
+ Row.Builder builder)
{
if (deletion.deletes(existing))
return;
- Cell reconciled = reconcile(existing, update, nowInSec);
+ Cell reconciled = reconcile(existing, update);
if (reconciled != update)
builder.addCell(existing);
}
@@ -278,16 +314,12 @@
* @param deletion the deletion time that applies to the cells being considered.
* This deletion time may delete both {@code existing} or {@code update}.
* @param builder the row builder to which the result of the filtering is written.
- * @param nowInSec the current time in seconds (which plays a role during reconciliation
- * because deleted cells always have precedence on timestamp equality and deciding if a
- * cell is a live or not depends on the current time due to expiring cells).
*/
- public static void addNonShadowedComplex(ColumnDefinition column,
+ public static void addNonShadowedComplex(ColumnMetadata column,
Iterator<Cell> existing,
Iterator<Cell> update,
DeletionTime deletion,
- Row.Builder builder,
- int nowInSec)
+ Row.Builder builder)
{
Comparator<CellPath> comparator = column.cellPathComparator();
Cell nextExisting = getNext(existing);
@@ -297,12 +329,12 @@
int cmp = nextUpdate == null ? -1 : comparator.compare(nextExisting.path(), nextUpdate.path());
if (cmp < 0)
{
- addNonShadowed(nextExisting, null, deletion, builder, nowInSec);
+ addNonShadowed(nextExisting, null, deletion, builder);
nextExisting = getNext(existing);
}
else if (cmp == 0)
{
- addNonShadowed(nextExisting, nextUpdate, deletion, builder, nowInSec);
+ addNonShadowed(nextExisting, nextUpdate, deletion, builder);
nextExisting = getNext(existing);
nextUpdate = getNext(update);
}
diff --git a/src/java/org/apache/cassandra/db/rows/ColumnData.java b/src/java/org/apache/cassandra/db/rows/ColumnData.java
index 933da6a..36aad97 100644
--- a/src/java/org/apache/cassandra/db/rows/ColumnData.java
+++ b/src/java/org/apache/cassandra/db/rows/ColumnData.java
@@ -17,10 +17,10 @@
*/
package org.apache.cassandra.db.rows;
-import java.security.MessageDigest;
import java.util.Comparator;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.db.Digest;
+import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.db.DeletionPurger;
import org.apache.cassandra.db.partitions.PartitionUpdate;
import org.apache.cassandra.serializers.MarshalException;
@@ -35,8 +35,8 @@
{
public static final Comparator<ColumnData> comparator = (cd1, cd2) -> cd1.column().compareTo(cd2.column());
- protected final ColumnDefinition column;
- protected ColumnData(ColumnDefinition column)
+ protected final ColumnMetadata column;
+ protected ColumnData(ColumnMetadata column)
{
this.column = column;
}
@@ -46,7 +46,7 @@
*
* @return the column this is a data for.
*/
- public final ColumnDefinition column() { return column; }
+ public final ColumnMetadata column() { return column; }
/**
* The size of the data hold by this {@code ColumnData}.
@@ -65,11 +65,23 @@
public abstract void validate();
/**
+ * Validates the deletions (ttl and local deletion time) if any.
+ *
+ * @return true if it has any invalid deletions, false otherwise
+ */
+ public abstract boolean hasInvalidDeletions();
+
+ /**
* Adds the data to the provided digest.
*
- * @param digest the {@code MessageDigest} to add the data to.
+ * @param digest the {@link Digest} to add the data to.
*/
- public abstract void digest(MessageDigest digest);
+ public abstract void digest(Digest digest);
+
+ public static void digest(Digest digest, ColumnData cd)
+ {
+ cd.digest(digest);
+ }
/**
* Returns a copy of the data where all timestamps for live data have replaced by {@code newTimestamp} and
diff --git a/src/java/org/apache/cassandra/db/rows/ColumnDefinitionVersionComparator.java b/src/java/org/apache/cassandra/db/rows/ColumnDefinitionVersionComparator.java
deleted file mode 100644
index 9be24c8..0000000
--- a/src/java/org/apache/cassandra/db/rows/ColumnDefinitionVersionComparator.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.db.rows;
-
-import java.util.Comparator;
-
-import org.apache.cassandra.config.ColumnDefinition;
-import org.apache.cassandra.db.marshal.*;
-
-/**
- * A {@code Comparator} use to determine which version of a {@link ColumnDefinition} should be used.
- * <p>
- * We can sometimes get 2 different versions of the definition of a give column due to differing types. This can happen
- * in at least 2 cases:
- * <ul>
- * <li>for UDT, where new fields can be added (see CASSANDRA-13776).</li>
- * <li>pre-CASSANDRA-12443, when we allowed type altering. And while we don't allow it anymore, it is possible
- * to still have sstables with metadata mentioning an old pre-altering type (such old version of pre-altering
- * types will be eventually eliminated from the system by compaction and thanks to this comparator, but we
- * cannot guarantee when that's fully done).</li>
- * </ul>
- */
-final class ColumnDefinitionVersionComparator implements Comparator<ColumnDefinition>
-{
- public static final Comparator<ColumnDefinition> INSTANCE = new ColumnDefinitionVersionComparator();
-
- private ColumnDefinitionVersionComparator()
- {
- }
-
- @Override
- public int compare(ColumnDefinition v1, ColumnDefinition v2)
- {
- assert v1.ksName.equals(v2.ksName)
- && v1.cfName.equals(v2.cfName)
- && v1.name.equals(v2.name) : v1.debugString() + " != " + v2.debugString();
-
- AbstractType<?> v1Type = v1.type;
- AbstractType<?> v2Type = v2.type;
-
- // In most cases, this is used on equal types, and on most types, equality is cheap (most are singleton classes
- // and just use reference equality), so evacuating that case first.
- if (v1Type.equals(v2Type))
- return 0;
-
- // If those aren't the same type, one must be "more general" than the other, that is accept strictly more values.
- if (v1Type.isValueCompatibleWith(v2Type))
- {
- // Note: if both accept the same values, there is really no good way to prefer one over the other and so we
- // consider them equal here. In practice, this mean we have 2 types that accepts the same values but are
- // not equal. For internal types, TimestampType/DataType/LongType is, afaik, the only example, but as user
- // can write custom types, who knows when this can happen. But excluding any user custom type weirdness
- // (that would really be a bug of their type), such types should only differ in the way they sort, and as
- // this method is only used for regular/static columns in practice, where sorting has no impact whatsoever,
- // it shouldn't matter too much what we return here.
- return v2Type.isValueCompatibleWith(v1Type) ? 0 : 1;
- }
- else if (v2Type.isValueCompatibleWith(v1Type))
- {
- return -1;
- }
- else
- {
- // Neither is a super type of the other: something is pretty wrong and we probably shouldn't ignore it.
- throw new IllegalArgumentException(String.format("Found 2 incompatible versions of column %s in %s.%s: one " +
- "of type %s and one of type %s (but both types are incompatible)",
- v1.name, v1.ksName, v1.cfName, v1Type, v2Type));
- }
- }
-}
diff --git a/src/java/org/apache/cassandra/db/rows/ColumnMetadataVersionComparator.java b/src/java/org/apache/cassandra/db/rows/ColumnMetadataVersionComparator.java
new file mode 100644
index 0000000..6b2d97c
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/rows/ColumnMetadataVersionComparator.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db.rows;
+
+import java.util.Comparator;
+
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.schema.ColumnMetadata;
+
+/**
+ * A {@code Comparator} use to determine which version of a {@link ColumnMetadata} should be used.
+ * <p>
+ * We can sometimes get 2 different versions of the definition of a give column due to differing types. This can happen
+ * in at least 2 cases:
+ * <ul>
+ * <li>for UDT, where new fields can be added (see CASSANDRA-13776).</li>
+ * <li>pre-CASSANDRA-12443, when we allowed type altering. And while we don't allow it anymore, it is possible
+ * to still have sstables with metadata mentioning an old pre-altering type (such old version of pre-altering
+ * types will be eventually eliminated from the system by compaction and thanks to this comparator, but we
+ * cannot guarantee when that's fully done).</li>
+ * </ul>
+ */
+final class ColumnMetadataVersionComparator implements Comparator<ColumnMetadata>
+{
+ public static final Comparator<ColumnMetadata> INSTANCE = new ColumnMetadataVersionComparator();
+
+ private ColumnMetadataVersionComparator()
+ {
+ }
+
+ @Override
+ public int compare(ColumnMetadata v1, ColumnMetadata v2)
+ {
+ assert v1.ksName.equals(v2.ksName)
+ && v1.cfName.equals(v2.cfName)
+ && v1.name.equals(v2.name) : v1.debugString() + " != " + v2.debugString();
+
+ AbstractType<?> v1Type = v1.type;
+ AbstractType<?> v2Type = v2.type;
+
+ // In most cases, this is used on equal types, and on most types, equality is cheap (most are singleton classes
+ // and just use reference equality), so evacuating that case first.
+ if (v1Type.equals(v2Type))
+ return 0;
+
+ // If those aren't the same type, one must be "more general" than the other, that is accept strictly more values.
+ if (v1Type.isValueCompatibleWith(v2Type))
+ {
+ // Note: if both accept the same values, there is really no good way to prefer one over the other and so we
+ // consider them equal here. In practice, this mean we have 2 types that accepts the same values but are
+ // not equal. For internal types, TimestampType/DataType/LongType is, afaik, the only example, but as user
+ // can write custom types, who knows when this can happen. But excluding any user custom type weirdness
+ // (that would really be a bug of their type), such types should only differ in the way they sort, and as
+ // this method is only used for regular/static columns in practice, where sorting has no impact whatsoever,
+ // it shouldn't matter too much what we return here.
+ return v2Type.isValueCompatibleWith(v1Type) ? 0 : 1;
+ }
+ else if (v2Type.isValueCompatibleWith(v1Type))
+ {
+ return -1;
+ }
+ else
+ {
+ // Neither is a super type of the other: something is pretty wrong and we probably shouldn't ignore it.
+ throw new IllegalArgumentException(String.format("Found 2 incompatible versions of column %s in %s.%s: one " +
+ "of type %s and one of type %s (but both types are incompatible)",
+ v1.name, v1.ksName, v1.cfName, v1Type, v2Type));
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/db/rows/ComplexColumnData.java b/src/java/org/apache/cassandra/db/rows/ComplexColumnData.java
index 1395782..5b03504 100644
--- a/src/java/org/apache/cassandra/db/rows/ComplexColumnData.java
+++ b/src/java/org/apache/cassandra/db/rows/ComplexColumnData.java
@@ -18,20 +18,23 @@
package org.apache.cassandra.db.rows;
import java.nio.ByteBuffer;
-import java.security.MessageDigest;
import java.util.Iterator;
import java.util.Objects;
import com.google.common.base.Function;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
import org.apache.cassandra.db.DeletionPurger;
import org.apache.cassandra.db.DeletionTime;
+import org.apache.cassandra.db.Digest;
import org.apache.cassandra.db.LivenessInfo;
import org.apache.cassandra.db.filter.ColumnFilter;
import org.apache.cassandra.db.marshal.ByteType;
import org.apache.cassandra.db.marshal.SetType;
+import org.apache.cassandra.db.partitions.PartitionStatisticsCollector;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.DroppedColumn;
+import org.apache.cassandra.utils.BiLongAccumulator;
+import org.apache.cassandra.utils.LongAccumulator;
import org.apache.cassandra.utils.ObjectSizes;
import org.apache.cassandra.utils.btree.BTree;
@@ -43,7 +46,7 @@
{
static final Cell[] NO_CELLS = new Cell[0];
- private static final long EMPTY_SIZE = ObjectSizes.measure(new ComplexColumnData(ColumnDefinition.regularDef("", "", "", SetType.getInstance(ByteType.instance, true)), NO_CELLS, new DeletionTime(0, 0)));
+ private static final long EMPTY_SIZE = ObjectSizes.measure(new ComplexColumnData(ColumnMetadata.regularColumn("", "", "", SetType.getInstance(ByteType.instance, true)), NO_CELLS, new DeletionTime(0, 0)));
// The cells for 'column' sorted by cell path.
private final Object[] cells;
@@ -51,7 +54,7 @@
private final DeletionTime complexDeletion;
// Only ArrayBackedRow should call this.
- ComplexColumnData(ColumnDefinition column, Object[] cells, DeletionTime complexDeletion)
+ ComplexColumnData(ColumnMetadata column, Object[] cells, DeletionTime complexDeletion)
{
super(column);
assert column.isComplex();
@@ -60,11 +63,6 @@
this.complexDeletion = complexDeletion;
}
- public boolean hasCells()
- {
- return !BTree.isEmpty(cells);
- }
-
public int cellsCount()
{
return BTree.size(cells);
@@ -106,6 +104,16 @@
return BTree.iterator(cells, BTree.Dir.DESC);
}
+ public long accumulate(LongAccumulator<Cell> accumulator, long initialValue)
+ {
+ return BTree.accumulate(cells, accumulator, initialValue);
+ }
+
+ public <A> long accumulate(BiLongAccumulator<A, Cell> accumulator, A arg, long initialValue)
+ {
+ return BTree.accumulate(cells, accumulator, arg, initialValue);
+ }
+
public int dataSize()
{
int size = complexDeletion.dataSize();
@@ -129,7 +137,7 @@
cell.validate();
}
- public void digest(MessageDigest digest)
+ public void digest(Digest digest)
{
if (!complexDeletion.isLive())
complexDeletion.digest(digest);
@@ -138,12 +146,22 @@
cell.digest(digest);
}
+ public boolean hasInvalidDeletions()
+ {
+ if (!complexDeletion.validate())
+ return true;
+ for (Cell cell : this)
+ if (cell.hasInvalidDeletions())
+ return true;
+ return false;
+ }
+
public ComplexColumnData markCounterLocalToBeCleared()
{
return transformAndFilter(complexDeletion, Cell::markCounterLocalToBeCleared);
}
- public ComplexColumnData filter(ColumnFilter filter, DeletionTime activeDeletion, CFMetaData.DroppedColumn dropped, LivenessInfo rowLiveness)
+ public ComplexColumnData filter(ColumnFilter filter, DeletionTime activeDeletion, DroppedColumn dropped, LivenessInfo rowLiveness)
{
ColumnFilter.Tester cellTester = filter.newTester(column);
if (cellTester == null && activeDeletion.isLive() && dropped == null)
@@ -235,10 +253,10 @@
public static class Builder
{
private DeletionTime complexDeletion;
- private ColumnDefinition column;
+ private ColumnMetadata column;
private BTree.Builder<Cell> builder;
- public void newColumn(ColumnDefinition column)
+ public void newColumn(ColumnMetadata column)
{
this.column = column;
this.complexDeletion = DeletionTime.LIVE; // default if writeComplexDeletion is not called
diff --git a/src/java/org/apache/cassandra/db/rows/DeserializationHelper.java b/src/java/org/apache/cassandra/db/rows/DeserializationHelper.java
new file mode 100644
index 0000000..386e6ef
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/rows/DeserializationHelper.java
@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db.rows;
+
+import java.nio.ByteBuffer;
+import java.util.*;
+
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.db.*;
+import org.apache.cassandra.db.context.CounterContext;
+import org.apache.cassandra.db.filter.ColumnFilter;
+import org.apache.cassandra.schema.DroppedColumn;
+
+public class DeserializationHelper
+{
+ /**
+ * Flag affecting deserialization behavior (this only affect counters in practice).
+ * - LOCAL: for deserialization of local data (Expired columns are
+ * converted to tombstones (to gain disk space)).
+ * - FROM_REMOTE: for deserialization of data received from remote hosts
+ * (Expired columns are converted to tombstone and counters have
+ * their delta cleared)
+ * - PRESERVE_SIZE: used when no transformation must be performed, i.e,
+ * when we must ensure that deserializing and reserializing the
+ * result yield the exact same bytes. Streaming uses this.
+ */
+ public enum Flag
+ {
+ LOCAL, FROM_REMOTE, PRESERVE_SIZE
+ }
+
+ private final Flag flag;
+ public final int version;
+
+ private final ColumnFilter columnsToFetch;
+ private ColumnFilter.Tester tester;
+
+ private final boolean hasDroppedColumns;
+ private final Map<ByteBuffer, DroppedColumn> droppedColumns;
+ private DroppedColumn currentDroppedComplex;
+
+
+ public DeserializationHelper(TableMetadata metadata, int version, Flag flag, ColumnFilter columnsToFetch)
+ {
+ this.flag = flag;
+ this.version = version;
+ this.columnsToFetch = columnsToFetch;
+ this.droppedColumns = metadata.droppedColumns;
+ this.hasDroppedColumns = droppedColumns.size() > 0;
+ }
+
+ public DeserializationHelper(TableMetadata metadata, int version, Flag flag)
+ {
+ this(metadata, version, flag, null);
+ }
+
+ public boolean includes(ColumnMetadata column)
+ {
+ return columnsToFetch == null || columnsToFetch.fetches(column);
+ }
+
+ public boolean includes(Cell cell, LivenessInfo rowLiveness)
+ {
+ if (columnsToFetch == null)
+ return true;
+
+ // During queries, some columns are included even though they are not queried by the user because
+ // we always need to distinguish between having a row (with potentially only null values) and not
+ // having a row at all (see #CASSANDRA-7085 for background). In the case where the column is not
+ // actually requested by the user however (canSkipValue), we can skip the full cell if the cell
+ // timestamp is lower than the row one, because in that case, the row timestamp is enough proof
+ // of the liveness of the row. Otherwise, we'll only be able to skip the values of those cells.
+ ColumnMetadata column = cell.column();
+ if (column.isComplex())
+ {
+ if (!includes(cell.path()))
+ return false;
+
+ return !canSkipValue(cell.path()) || cell.timestamp() >= rowLiveness.timestamp();
+ }
+ else
+ {
+ return columnsToFetch.fetchedColumnIsQueried(column) || cell.timestamp() >= rowLiveness.timestamp();
+ }
+ }
+
+ public boolean includes(CellPath path)
+ {
+ return path == null || tester == null || tester.fetches(path);
+ }
+
+ public boolean canSkipValue(ColumnMetadata column)
+ {
+ return columnsToFetch != null && !columnsToFetch.fetchedColumnIsQueried(column);
+ }
+
+ public boolean canSkipValue(CellPath path)
+ {
+ return path != null && tester != null && !tester.fetchedCellIsQueried(path);
+ }
+
+ public void startOfComplexColumn(ColumnMetadata column)
+ {
+ this.tester = columnsToFetch == null ? null : columnsToFetch.newTester(column);
+ this.currentDroppedComplex = droppedColumns.get(column.name.bytes);
+ }
+
+ public void endOfComplexColumn()
+ {
+ this.tester = null;
+ }
+
+ public boolean isDropped(Cell cell, boolean isComplex)
+ {
+ if (!hasDroppedColumns)
+ return false;
+
+ DroppedColumn dropped = isComplex ? currentDroppedComplex : droppedColumns.get(cell.column().name.bytes);
+ return dropped != null && cell.timestamp() <= dropped.droppedTime;
+ }
+
+ public boolean isDroppedComplexDeletion(DeletionTime complexDeletion)
+ {
+ return currentDroppedComplex != null && complexDeletion.markedForDeleteAt() <= currentDroppedComplex.droppedTime;
+ }
+
+ public ByteBuffer maybeClearCounterValue(ByteBuffer value)
+ {
+ return flag == Flag.FROM_REMOTE || (flag == Flag.LOCAL && CounterContext.instance().shouldClearLocal(value))
+ ? CounterContext.instance().clearAllLocal(value)
+ : value;
+ }
+}
diff --git a/src/java/org/apache/cassandra/db/rows/EncodingStats.java b/src/java/org/apache/cassandra/db/rows/EncodingStats.java
index 955ffc7..c73728d 100644
--- a/src/java/org/apache/cassandra/db/rows/EncodingStats.java
+++ b/src/java/org/apache/cassandra/db/rows/EncodingStats.java
@@ -19,6 +19,9 @@
import java.io.IOException;
import java.util.*;
+import java.util.function.Function;
+
+import com.google.common.collect.Iterables;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.partitions.PartitionStatisticsCollector;
@@ -41,7 +44,7 @@
{
// Default values for the timestamp, deletion time and ttl. We use this both for NO_STATS, but also to serialize
// an EncodingStats. Basically, we encode the diff of each value of to these epoch, which give values with better vint encoding.
- private static final long TIMESTAMP_EPOCH;
+ public static final long TIMESTAMP_EPOCH;
private static final int DELETION_TIME_EPOCH;
private static final int TTL_EPOCH = 0;
static
@@ -93,20 +96,43 @@
public EncodingStats mergeWith(EncodingStats that)
{
long minTimestamp = this.minTimestamp == TIMESTAMP_EPOCH
- ? that.minTimestamp
- : (that.minTimestamp == TIMESTAMP_EPOCH ? this.minTimestamp : Math.min(this.minTimestamp, that.minTimestamp));
+ ? that.minTimestamp
+ : (that.minTimestamp == TIMESTAMP_EPOCH ? this.minTimestamp : Math.min(this.minTimestamp, that.minTimestamp));
int minDelTime = this.minLocalDeletionTime == DELETION_TIME_EPOCH
- ? that.minLocalDeletionTime
- : (that.minLocalDeletionTime == DELETION_TIME_EPOCH ? this.minLocalDeletionTime : Math.min(this.minLocalDeletionTime, that.minLocalDeletionTime));
+ ? that.minLocalDeletionTime
+ : (that.minLocalDeletionTime == DELETION_TIME_EPOCH ? this.minLocalDeletionTime : Math.min(this.minLocalDeletionTime, that.minLocalDeletionTime));
int minTTL = this.minTTL == TTL_EPOCH
- ? that.minTTL
- : (that.minTTL == TTL_EPOCH ? this.minTTL : Math.min(this.minTTL, that.minTTL));
+ ? that.minTTL
+ : (that.minTTL == TTL_EPOCH ? this.minTTL : Math.min(this.minTTL, that.minTTL));
return new EncodingStats(minTimestamp, minDelTime, minTTL);
}
+ /**
+ * Merge one or more EncodingStats, that are lazily materialized from some list of arbitrary type by the provided function
+ */
+ public static <V, F extends Function<V, EncodingStats>> EncodingStats merge(List<V> values, F function)
+ {
+ if (values.size() == 1)
+ return function.apply(values.get(0));
+
+ Collector collector = new Collector();
+ for (int i=0, isize=values.size(); i<isize; i++)
+ {
+ V v = values.get(i);
+ EncodingStats stats = function.apply(v);
+ if (stats.minTimestamp != TIMESTAMP_EPOCH)
+ collector.updateTimestamp(stats.minTimestamp);
+ if(stats.minLocalDeletionTime != DELETION_TIME_EPOCH)
+ collector.updateLocalDeletionTime(stats.minLocalDeletionTime);
+ if(stats.minTTL != TTL_EPOCH)
+ collector.updateTTL(stats.minTTL);
+ }
+ return collector.get();
+ }
+
@Override
public boolean equals(Object o)
{
diff --git a/src/java/org/apache/cassandra/db/rows/LazilyInitializedUnfilteredRowIterator.java b/src/java/org/apache/cassandra/db/rows/LazilyInitializedUnfilteredRowIterator.java
index 504d60e..d8bd36f 100644
--- a/src/java/org/apache/cassandra/db/rows/LazilyInitializedUnfilteredRowIterator.java
+++ b/src/java/org/apache/cassandra/db/rows/LazilyInitializedUnfilteredRowIterator.java
@@ -17,9 +17,9 @@
*/
package org.apache.cassandra.db.rows;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.utils.AbstractIterator;
-import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.db.*;
/**
@@ -53,13 +53,13 @@
return iterator != null;
}
- public CFMetaData metadata()
+ public TableMetadata metadata()
{
maybeInit();
return iterator.metadata();
}
- public PartitionColumns columns()
+ public RegularAndStaticColumns columns()
{
maybeInit();
return iterator.columns();
diff --git a/src/java/org/apache/cassandra/db/rows/NativeCell.java b/src/java/org/apache/cassandra/db/rows/NativeCell.java
index 31ce0b7..c4cb6c1 100644
--- a/src/java/org/apache/cassandra/db/rows/NativeCell.java
+++ b/src/java/org/apache/cassandra/db/rows/NativeCell.java
@@ -20,7 +20,7 @@
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.utils.ObjectSizes;
import org.apache.cassandra.utils.concurrent.OpOrder;
import org.apache.cassandra.utils.memory.MemoryUtil;
@@ -61,7 +61,7 @@
public NativeCell(NativeAllocator allocator,
OpOrder.Group writeOp,
- ColumnDefinition column,
+ ColumnMetadata column,
long timestamp,
int ttl,
int localDeletionTime,
@@ -148,7 +148,7 @@
return new BufferCell(column, newTimestamp, ttl(), newLocalDeletionTime, value(), path());
}
- public Cell withUpdatedColumn(ColumnDefinition column)
+ public Cell withUpdatedColumn(ColumnMetadata column)
{
return new BufferCell(column, timestamp(), ttl(), localDeletionTime(), value(), path());
}
diff --git a/src/java/org/apache/cassandra/db/rows/RangeTombstoneBoundMarker.java b/src/java/org/apache/cassandra/db/rows/RangeTombstoneBoundMarker.java
index f6ba149..51d8264 100644
--- a/src/java/org/apache/cassandra/db/rows/RangeTombstoneBoundMarker.java
+++ b/src/java/org/apache/cassandra/db/rows/RangeTombstoneBoundMarker.java
@@ -18,11 +18,9 @@
package org.apache.cassandra.db.rows;
import java.nio.ByteBuffer;
-import java.security.MessageDigest;
import java.util.Objects;
-import java.util.Set;
-import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.db.*;
import org.apache.cassandra.utils.memory.AbstractAllocator;
@@ -68,6 +66,11 @@
return false;
}
+ public boolean hasInvalidDeletions()
+ {
+ return !deletionTime().validate();
+ }
+
/**
* The deletion time for the range tombstone this is a bound of.
*/
@@ -127,19 +130,13 @@
return new RangeTombstoneBoundMarker(clustering(), newDeletionTime);
}
- public void digest(MessageDigest digest)
+ public void digest(Digest digest)
{
bound.digest(digest);
deletion.digest(digest);
}
- @Override
- public void digest(MessageDigest digest, Set<ByteBuffer> columnsToExclude)
- {
- digest(digest);
- }
-
- public String toString(CFMetaData metadata)
+ public String toString(TableMetadata metadata)
{
return String.format("Marker %s@%d/%d", bound.toString(metadata), deletion.markedForDeleteAt(), deletion.localDeletionTime());
}
diff --git a/src/java/org/apache/cassandra/db/rows/RangeTombstoneBoundaryMarker.java b/src/java/org/apache/cassandra/db/rows/RangeTombstoneBoundaryMarker.java
index ad71784..6a931c9 100644
--- a/src/java/org/apache/cassandra/db/rows/RangeTombstoneBoundaryMarker.java
+++ b/src/java/org/apache/cassandra/db/rows/RangeTombstoneBoundaryMarker.java
@@ -18,11 +18,9 @@
package org.apache.cassandra.db.rows;
import java.nio.ByteBuffer;
-import java.security.MessageDigest;
import java.util.Objects;
-import java.util.Set;
-import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.db.*;
import org.apache.cassandra.utils.memory.AbstractAllocator;
@@ -116,6 +114,11 @@
return true;
}
+ public boolean hasInvalidDeletions()
+ {
+ return !startDeletion.validate() || !endDeletion.validate();
+ }
+
public RangeTombstoneBoundaryMarker copy(AbstractAllocator allocator)
{
return new RangeTombstoneBoundaryMarker(clustering().copy(allocator), endDeletion, startDeletion);
@@ -145,20 +148,14 @@
return new RangeTombstoneBoundMarker(openBound(reversed), openDeletionTime(reversed));
}
- public void digest(MessageDigest digest)
+ public void digest(Digest digest)
{
bound.digest(digest);
endDeletion.digest(digest);
startDeletion.digest(digest);
}
- @Override
- public void digest(MessageDigest digest, Set<ByteBuffer> columnsToExclude)
- {
- digest(digest);
- }
-
- public String toString(CFMetaData metadata)
+ public String toString(TableMetadata metadata)
{
return String.format("Marker %s@%d/%d-%d/%d",
bound.toString(metadata),
diff --git a/src/java/org/apache/cassandra/db/rows/Row.java b/src/java/org/apache/cassandra/db/rows/Row.java
index dd8e303..3703d549 100644
--- a/src/java/org/apache/cassandra/db/rows/Row.java
+++ b/src/java/org/apache/cassandra/db/rows/Row.java
@@ -18,17 +18,16 @@
package org.apache.cassandra.db.rows;
import java.util.*;
-import java.security.MessageDigest;
+import java.util.function.BiConsumer;
import java.util.function.Consumer;
-import com.google.common.base.Predicate;
-
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.filter.ColumnFilter;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.service.paxos.Commit;
-import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.BiLongAccumulator;
+import org.apache.cassandra.utils.LongAccumulator;
import org.apache.cassandra.utils.MergeIterator;
import org.apache.cassandra.utils.SearchIterator;
import org.apache.cassandra.utils.btree.BTree;
@@ -60,7 +59,7 @@
* An in-natural-order collection of the columns for which data (incl. simple tombstones)
* is present in this row.
*/
- public Collection<ColumnDefinition> columns();
+ public Collection<ColumnMetadata> columns();
/**
@@ -126,7 +125,7 @@
* @param c the simple column for which to fetch the cell.
* @return the corresponding cell or {@code null} if the row has no such cell.
*/
- public Cell getCell(ColumnDefinition c);
+ public Cell getCell(ColumnMetadata c);
/**
* Return a cell for a given complex column and cell path.
@@ -135,7 +134,7 @@
* @param path the cell path for which to fetch the cell.
* @return the corresponding cell or {@code null} if the row has no such cell.
*/
- public Cell getCell(ColumnDefinition c, CellPath path);
+ public Cell getCell(ColumnMetadata c, CellPath path);
/**
* The data for a complex column.
@@ -145,7 +144,7 @@
* @param c the complex column for which to return the complex data.
* @return the data for {@code c} or {@code null} if the row has no data for this column.
*/
- public ComplexColumnData getComplexColumnData(ColumnDefinition c);
+ public ComplexColumnData getComplexColumnData(ColumnMetadata c);
/**
* An iterable over the cells of this row.
@@ -176,7 +175,7 @@
* @param reversed if cells should returned in reverse order.
* @return an iterable over the cells of this row in "legacy order".
*/
- public Iterable<Cell> cellsInLegacyOrder(CFMetaData metadata, boolean reversed);
+ public Iterable<Cell> cellsInLegacyOrder(TableMetadata metadata, boolean reversed);
/**
* Whether the row stores any (non-live) complex deletion for any complex column.
@@ -200,14 +199,14 @@
*
* @return a search iterator for the cells of this row.
*/
- public SearchIterator<ColumnDefinition, ColumnData> searchIterator();
+ public SearchIterator<ColumnMetadata, ColumnData> searchIterator();
/**
* Returns a copy of this row that:
* 1) only includes the data for the column included by {@code filter}.
* 2) doesn't include any data that belongs to a dropped column (recorded in {@code metadata}).
*/
- public Row filter(ColumnFilter filter, CFMetaData metadata);
+ public Row filter(ColumnFilter filter, TableMetadata metadata);
/**
* Returns a copy of this row that:
@@ -216,7 +215,7 @@
* 3) doesn't include any data that is shadowed/deleted by {@code activeDeletion}.
* 4) uses {@code activeDeletion} as row deletion iff {@code setActiveDeletionToRow} and {@code activeDeletion} supersedes the row deletion.
*/
- public Row filter(ColumnFilter filter, DeletionTime activeDeletion, boolean setActiveDeletionToRow, CFMetaData metadata);
+ public Row filter(ColumnFilter filter, DeletionTime activeDeletion, boolean setActiveDeletionToRow, TableMetadata metadata);
/**
* Returns a copy of this row without any deletion info that should be purged according to {@code purger}.
@@ -224,7 +223,7 @@
* @param purger the {@code DeletionPurger} to use to decide what can be purged.
* @param nowInSec the current time to decide what is deleted and what isn't (in the case of expired cells).
* @param enforceStrictLiveness whether the row should be purged if there is no PK liveness info,
- * normally retrieved from {@link CFMetaData#enforceStrictLiveness()}
+ * normally retrieved from {@link TableMetadata#enforceStrictLiveness()}
*
* When enforceStrictLiveness is set, rows with empty PK liveness info
* and no row deletion are purged.
@@ -234,7 +233,7 @@
* is not live. See CASSANDRA-11500.
*
* @return this row but without any deletion info purged by {@code purger}. If the purged row is empty, returns
- * {@code null}.
+ * {@code null}.
*/
public Row purge(DeletionPurger purger, int nowInSec, boolean enforceStrictLiveness);
@@ -279,17 +278,29 @@
public long unsharedHeapSizeExcludingData();
- public String toString(CFMetaData metadata, boolean fullDetails);
+ public String toString(TableMetadata metadata, boolean fullDetails);
/**
* Apply a function to every column in a row
*/
- public void apply(Consumer<ColumnData> function, boolean reverse);
+ public void apply(Consumer<ColumnData> function);
/**
- * Apply a funtion to every column in a row until a stop condition is reached
+ * Apply a function to every column in a row
*/
- public void apply(Consumer<ColumnData> function, Predicate<ColumnData> stopCondition, boolean reverse);
+ public <A> void apply(BiConsumer<A, ColumnData> function, A arg);
+
+ /**
+ * Apply an accumulation funtion to every column in a row
+ */
+
+ public long accumulate(LongAccumulator<ColumnData> accumulator, long initialValue);
+
+ public long accumulate(LongAccumulator<ColumnData> accumulator, Comparator<ColumnData> comparator, ColumnData from, long initialValue);
+
+ public <A> long accumulate(BiLongAccumulator<A, ColumnData> accumulator, A arg, long initialValue);
+
+ public <A> long accumulate(BiLongAccumulator<A, ColumnData> accumulator, A arg, Comparator<ColumnData> comparator, ColumnData from, long initialValue);
/**
* A row deletion/tombstone.
@@ -391,10 +402,10 @@
return time.deletes(cell);
}
- public void digest(MessageDigest digest)
+ public void digest(Digest digest)
{
time.digest(digest);
- FBUtilities.updateWithBoolean(digest, isShadowable);
+ digest.updateWithBoolean(isShadowable);
}
public int dataSize()
@@ -512,7 +523,7 @@
* @param column the column for which to add the {@code complexDeletion}.
* @param complexDeletion the complex deletion time to add.
*/
- public void addComplexDeletion(ColumnDefinition column, DeletionTime complexDeletion);
+ public void addComplexDeletion(ColumnMetadata column, DeletionTime complexDeletion);
/**
* Builds and return built row.
@@ -636,11 +647,11 @@
private final List<ColumnData> dataBuffer = new ArrayList<>();
private final ColumnDataReducer columnDataReducer;
- public Merger(int size, int nowInSec, boolean hasComplex)
+ public Merger(int size, boolean hasComplex)
{
this.rows = new Row[size];
this.columnDataIterators = new ArrayList<>(size);
- this.columnDataReducer = new ColumnDataReducer(size, nowInSec, hasComplex);
+ this.columnDataReducer = new ColumnDataReducer(size, hasComplex);
}
public void clear()
@@ -660,6 +671,7 @@
lastRowSet = i;
}
+ @SuppressWarnings("resource")
public Row merge(DeletionTime activeDeletion)
{
// If for this clustering we have only one row version and have no activeDeletion (i.e. nothing to filter out),
@@ -725,9 +737,7 @@
private static class ColumnDataReducer extends MergeIterator.Reducer<ColumnData, ColumnData>
{
- private final int nowInSec;
-
- private ColumnDefinition column;
+ private ColumnMetadata column;
private final List<ColumnData> versions;
private DeletionTime activeDeletion;
@@ -736,13 +746,12 @@
private final List<Iterator<Cell>> complexCells;
private final CellReducer cellReducer;
- public ColumnDataReducer(int size, int nowInSec, boolean hasComplex)
+ public ColumnDataReducer(int size, boolean hasComplex)
{
- this.nowInSec = nowInSec;
this.versions = new ArrayList<>(size);
this.complexBuilder = hasComplex ? ComplexColumnData.builder() : null;
this.complexCells = hasComplex ? new ArrayList<>(size) : null;
- this.cellReducer = new CellReducer(nowInSec);
+ this.cellReducer = new CellReducer();
}
public void setActiveDeletion(DeletionTime activeDeletion)
@@ -752,35 +761,36 @@
public void reduce(int idx, ColumnData data)
{
- if (useColumnDefinition(data.column()))
+ if (useColumnMetadata(data.column()))
column = data.column();
versions.add(data);
}
/**
- * Determines it the {@code ColumnDefinition} is the one that should be used.
- * @param dataColumn the {@code ColumnDefinition} to use.
- * @return {@code true} if the {@code ColumnDefinition} is the one that should be used, {@code false} otherwise.
+ * Determines it the {@code ColumnMetadata} is the one that should be used.
+ * @param dataColumn the {@code ColumnMetadata} to use.
+ * @return {@code true} if the {@code ColumnMetadata} is the one that should be used, {@code false} otherwise.
*/
- private boolean useColumnDefinition(ColumnDefinition dataColumn)
+ private boolean useColumnMetadata(ColumnMetadata dataColumn)
{
if (column == null)
return true;
- return ColumnDefinitionVersionComparator.INSTANCE.compare(column, dataColumn) < 0;
+ return ColumnMetadataVersionComparator.INSTANCE.compare(column, dataColumn) < 0;
}
+ @SuppressWarnings("resource")
protected ColumnData getReduced()
{
if (column.isSimple())
{
Cell merged = null;
- for (ColumnData data : versions)
+ for (int i=0, isize=versions.size(); i<isize; i++)
{
- Cell cell = (Cell)data;
+ Cell cell = (Cell) versions.get(i);
if (!activeDeletion.deletes(cell))
- merged = merged == null ? cell : Cells.reconcile(merged, cell, nowInSec);
+ merged = merged == null ? cell : Cells.reconcile(merged, cell);
}
return merged;
}
@@ -789,8 +799,9 @@
complexBuilder.newColumn(column);
complexCells.clear();
DeletionTime complexDeletion = DeletionTime.LIVE;
- for (ColumnData data : versions)
+ for (int i=0, isize=versions.size(); i<isize; i++)
{
+ ColumnData data = versions.get(i);
ComplexColumnData cd = (ComplexColumnData)data;
if (cd.complexDeletion().supersedes(complexDeletion))
complexDeletion = cd.complexDeletion();
@@ -827,16 +838,9 @@
private static class CellReducer extends MergeIterator.Reducer<Cell, Cell>
{
- private final int nowInSec;
-
private DeletionTime activeDeletion;
private Cell merged;
- public CellReducer(int nowInSec)
- {
- this.nowInSec = nowInSec;
- }
-
public void setActiveDeletion(DeletionTime activeDeletion)
{
this.activeDeletion = activeDeletion;
@@ -846,7 +850,7 @@
public void reduce(int idx, Cell cell)
{
if (!activeDeletion.deletes(cell))
- merged = merged == null ? cell : Cells.reconcile(merged, cell, nowInSec);
+ merged = merged == null ? cell : Cells.reconcile(merged, cell);
}
protected Cell getReduced()
diff --git a/src/java/org/apache/cassandra/db/rows/RowAndDeletionMergeIterator.java b/src/java/org/apache/cassandra/db/rows/RowAndDeletionMergeIterator.java
index 5af6c4b..7552d60 100644
--- a/src/java/org/apache/cassandra/db/rows/RowAndDeletionMergeIterator.java
+++ b/src/java/org/apache/cassandra/db/rows/RowAndDeletionMergeIterator.java
@@ -20,7 +20,7 @@
import java.util.Comparator;
import java.util.Iterator;
-import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.filter.ColumnFilter;
@@ -51,7 +51,7 @@
// The currently open tombstone. Note that unless this is null, there is no point in checking nextRange.
private RangeTombstone openRange;
- public RowAndDeletionMergeIterator(CFMetaData metadata,
+ public RowAndDeletionMergeIterator(TableMetadata metadata,
DecoratedKey partitionKey,
DeletionTime partitionLevelDeletion,
ColumnFilter selection,
diff --git a/src/java/org/apache/cassandra/db/rows/RowDiffListener.java b/src/java/org/apache/cassandra/db/rows/RowDiffListener.java
index 0c7e32b..88a9bae 100644
--- a/src/java/org/apache/cassandra/db/rows/RowDiffListener.java
+++ b/src/java/org/apache/cassandra/db/rows/RowDiffListener.java
@@ -17,7 +17,7 @@
*/
package org.apache.cassandra.db.rows;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.db.*;
/**
@@ -63,7 +63,7 @@
* @param original the complex deletion of input {@code i} for column {@code column}. May be {@code null} if input {@code i}
* had no complex deletion but the merged row has.
*/
- public void onComplexDeletion(int i, Clustering clustering, ColumnDefinition column, DeletionTime merged, DeletionTime original);
+ public void onComplexDeletion(int i, Clustering clustering, ColumnMetadata column, DeletionTime merged, DeletionTime original);
/**
* Called for any cell that is either in the merged row or in input {@code i}.
diff --git a/src/java/org/apache/cassandra/db/rows/RowIterators.java b/src/java/org/apache/cassandra/db/rows/RowIterators.java
index 1463bf5..640cbc8 100644
--- a/src/java/org/apache/cassandra/db/rows/RowIterators.java
+++ b/src/java/org/apache/cassandra/db/rows/RowIterators.java
@@ -17,17 +17,13 @@
*/
package org.apache.cassandra.db.rows;
-import java.nio.ByteBuffer;
-import java.security.MessageDigest;
-import java.util.Set;
-
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.db.Digest;
import org.apache.cassandra.db.filter.ColumnFilter;
import org.apache.cassandra.db.transform.Transformation;
-import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.schema.TableMetadata;
/**
* Static methods to work with row iterators.
@@ -38,35 +34,20 @@
private RowIterators() {}
- public static void digest(RowIterator iterator, MessageDigest digest, MessageDigest altDigest, Set<ByteBuffer> columnsToExclude)
+ public static void digest(RowIterator iterator, Digest digest)
{
// TODO: we're not computing digest the same way that old nodes. This is
// currently ok as this is only used for schema digest and the is no exchange
// of schema digest between different versions. If this changes however,
// we'll need to agree on a version.
- digest.update(iterator.partitionKey().getKey().duplicate());
+ digest.update(iterator.partitionKey().getKey());
iterator.columns().regulars.digest(digest);
iterator.columns().statics.digest(digest);
- FBUtilities.updateWithBoolean(digest, iterator.isReverseOrder());
+ digest.updateWithBoolean(iterator.isReverseOrder());
iterator.staticRow().digest(digest);
- if (altDigest != null)
- {
- // Compute the "alternative digest" here.
- altDigest.update(iterator.partitionKey().getKey().duplicate());
- iterator.columns().regulars.digest(altDigest, columnsToExclude);
- iterator.columns().statics.digest(altDigest, columnsToExclude);
- FBUtilities.updateWithBoolean(altDigest, iterator.isReverseOrder());
- iterator.staticRow().digest(altDigest, columnsToExclude);
- }
-
while (iterator.hasNext())
- {
- Row row = iterator.next();
- row.digest(digest);
- if (altDigest != null)
- row.digest(altDigest, columnsToExclude);
- }
+ iterator.next().digest(digest);
}
/**
@@ -93,12 +74,12 @@
*/
public static RowIterator loggingIterator(RowIterator iterator, final String id)
{
- CFMetaData metadata = iterator.metadata();
+ TableMetadata metadata = iterator.metadata();
logger.info("[{}] Logging iterator on {}.{}, partition key={}, reversed={}",
id,
- metadata.ksName,
- metadata.cfName,
- metadata.getKeyValidator().getString(iterator.partitionKey().getKey()),
+ metadata.keyspace,
+ metadata.name,
+ metadata.partitionKeyType.getString(iterator.partitionKey().getKey()),
iterator.isReverseOrder());
class Log extends Transformation
diff --git a/src/java/org/apache/cassandra/db/rows/Rows.java b/src/java/org/apache/cassandra/db/rows/Rows.java
index 2701682..58284ac 100644
--- a/src/java/org/apache/cassandra/db/rows/Rows.java
+++ b/src/java/org/apache/cassandra/db/rows/Rows.java
@@ -22,12 +22,11 @@
import com.google.common.collect.Iterators;
import com.google.common.collect.PeekingIterator;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.partitions.PartitionStatisticsCollector;
import org.apache.cassandra.utils.MergeIterator;
-import org.apache.cassandra.utils.WrappedInt;
/**
* Static utilities to work on Row objects.
@@ -70,11 +69,51 @@
* only argument.
* @return a newly created builder.
*/
- public static Row.SimpleBuilder simpleBuilder(CFMetaData metadata, Object... clusteringValues)
+ public static Row.SimpleBuilder simpleBuilder(TableMetadata metadata, Object... clusteringValues)
{
return new SimpleBuilders.RowBuilder(metadata, clusteringValues);
}
+ private static class StatsAccumulation
+ {
+ private static final long COLUMN_INCR = 1L << 32;
+ private static final long CELL_INCR = 1L;
+
+ private static long accumulateOnCell(PartitionStatisticsCollector collector, Cell cell, long l)
+ {
+ Cells.collectStats(cell, collector);
+ return l + CELL_INCR;
+ }
+
+ private static long accumulateOnColumnData(PartitionStatisticsCollector collector, ColumnData cd, long l)
+ {
+ if (cd.column().isSimple())
+ {
+ l = accumulateOnCell(collector, (Cell) cd, l) + COLUMN_INCR;
+ }
+ else
+ {
+ ComplexColumnData complexData = (ComplexColumnData)cd;
+ collector.update(complexData.complexDeletion());
+ int startingCells = unpackCellCount(l);
+ l = complexData.accumulate(StatsAccumulation::accumulateOnCell, collector, l);
+ if (unpackCellCount(l) > startingCells)
+ l += COLUMN_INCR;
+ }
+ return l;
+ }
+
+ private static int unpackCellCount(long v)
+ {
+ return (int) (v & 0xFFFFFFFFL);
+ }
+
+ private static int unpackColumnCount(long v)
+ {
+ return (int) (v >>> 32);
+ }
+ }
+
/**
* Collect statistics on a given row.
*
@@ -89,35 +128,10 @@
collector.update(row.primaryKeyLivenessInfo());
collector.update(row.deletion().time());
- //we have to wrap these for the lambda
- final WrappedInt columnCount = new WrappedInt(0);
- final WrappedInt cellCount = new WrappedInt(0);
+ long result = row.accumulate(StatsAccumulation::accumulateOnColumnData, collector, 0);
- row.apply(cd -> {
- if (cd.column().isSimple())
- {
- columnCount.increment();
- cellCount.increment();
- Cells.collectStats((Cell) cd, collector);
- }
- else
- {
- ComplexColumnData complexData = (ComplexColumnData)cd;
- collector.update(complexData.complexDeletion());
- if (complexData.hasCells())
- {
- columnCount.increment();
- for (Cell cell : complexData)
- {
- cellCount.increment();
- Cells.collectStats(cell, collector);
- }
- }
- }
- }, false);
-
- collector.updateColumnSetPerRow(columnCount.get());
- return cellCount.get();
+ collector.updateColumnSetPerRow(StatsAccumulation.unpackColumnCount(result));
+ return StatsAccumulation.unpackCellCount(result);
}
/**
@@ -131,6 +145,7 @@
* @param merged the result of merging {@code inputs}.
* @param inputs the inputs whose merge yielded {@code merged}.
*/
+ @SuppressWarnings("resource")
public static void diff(RowDiffListener diffListener, Row merged, Row...inputs)
{
Clustering clustering = merged.clustering();
@@ -172,7 +187,7 @@
ColumnData input = inputDatas[i];
if (mergedData != null || input != null)
{
- ColumnDefinition column = (mergedData != null ? mergedData : input).column;
+ ColumnMetadata column = (mergedData != null ? mergedData : input).column;
if (column.isSimple())
{
diffListener.onCell(i, clustering, (Cell) mergedData, (Cell) input);
@@ -238,10 +253,10 @@
iter.next();
}
- public static Row merge(Row row1, Row row2, int nowInSec)
+ public static Row merge(Row row1, Row row2)
{
Row.Builder builder = BTreeRow.sortedBuilder();
- merge(row1, row2, builder, nowInSec);
+ merge(row1, row2, builder);
return builder.build();
}
@@ -255,9 +270,6 @@
* @param existing
* @param update
* @param builder the row build to which the result of the reconciliation is written.
- * @param nowInSec the current time in seconds (which plays a role during reconciliation
- * because deleted cells always have precedence on timestamp equality and deciding if a
- * cell is a live or not depends on the current time due to expiring cells).
*
* @return the smallest timestamp delta between corresponding rows from existing and update. A
* timestamp delta being computed as the difference between the cells and DeletionTimes from {@code existing}
@@ -265,8 +277,7 @@
*/
public static long merge(Row existing,
Row update,
- Row.Builder builder,
- int nowInSec)
+ Row.Builder builder)
{
Clustering clustering = existing.clustering();
builder.newRow(clustering);
@@ -297,11 +308,10 @@
int comparison = nexta == null ? 1 : nextb == null ? -1 : nexta.column.compareTo(nextb.column);
ColumnData cura = comparison <= 0 ? nexta : null;
ColumnData curb = comparison >= 0 ? nextb : null;
- ColumnDefinition column = getColumnDefinition(cura, curb);
-
+ ColumnMetadata column = getColumnMetadata(cura, curb);
if (column.isSimple())
{
- timeDelta = Math.min(timeDelta, Cells.reconcile((Cell) cura, (Cell) curb, deletion, builder, nowInSec));
+ timeDelta = Math.min(timeDelta, Cells.reconcile((Cell) cura, (Cell) curb, deletion, builder));
}
else
{
@@ -318,7 +328,7 @@
Iterator<Cell> existingCells = existingData == null ? null : existingData.iterator();
Iterator<Cell> updateCells = updateData == null ? null : updateData.iterator();
- timeDelta = Math.min(timeDelta, Cells.reconcileComplex(column, existingCells, updateCells, maxDt, builder, nowInSec));
+ timeDelta = Math.min(timeDelta, Cells.reconcileComplex(column, existingCells, updateCells, maxDt, builder));
}
if (cura != null)
@@ -337,11 +347,8 @@
* @param existing source row
* @param update shadowing row
* @param rangeDeletion extra {@code DeletionTime} from covering tombstone
- * @param nowInSec the current time in seconds (which plays a role during reconciliation
- * because deleted cells always have precedence on timestamp equality and deciding if a
- * cell is a live or not depends on the current time due to expiring cells).
*/
- public static Row removeShadowedCells(Row existing, Row update, DeletionTime rangeDeletion, int nowInSec)
+ public static Row removeShadowedCells(Row existing, Row update, DeletionTime rangeDeletion)
{
Row.Builder builder = BTreeRow.sortedBuilder();
Clustering clustering = existing.clustering();
@@ -367,11 +374,11 @@
if (comparison <= 0)
{
ColumnData cura = nexta;
- ColumnDefinition column = cura.column;
+ ColumnMetadata column = cura.column;
ColumnData curb = comparison == 0 ? nextb : null;
if (column.isSimple())
{
- Cells.addNonShadowed((Cell) cura, (Cell) curb, deletion, builder, nowInSec);
+ Cells.addNonShadowed((Cell) cura, (Cell) curb, deletion, builder);
}
else
{
@@ -390,7 +397,7 @@
Iterator<Cell> existingCells = existingData.iterator();
Iterator<Cell> updateCells = updateData == null ? null : updateData.iterator();
- Cells.addNonShadowedComplex(column, existingCells, updateCells, maxDt, builder, nowInSec);
+ Cells.addNonShadowedComplex(column, existingCells, updateCells, maxDt, builder);
}
nexta = a.hasNext() ? a.next() : null;
if (curb != null)
@@ -406,10 +413,10 @@
}
/**
- * Returns the {@code ColumnDefinition} to use for merging the columns.
- * If the 2 column definitions are different the latest one will be returned.
+ * Returns the {@code ColumnMetadata} to use for merging the columns.
+ * If the 2 column metadata are different the latest one will be returned.
*/
- private static ColumnDefinition getColumnDefinition(ColumnData cura, ColumnData curb)
+ private static ColumnMetadata getColumnMetadata(ColumnData cura, ColumnData curb)
{
if (cura == null)
return curb.column;
@@ -417,7 +424,7 @@
if (curb == null)
return cura.column;
- if (ColumnDefinitionVersionComparator.INSTANCE.compare(cura.column, curb.column) >= 0)
+ if (ColumnMetadataVersionComparator.INSTANCE.compare(cura.column, curb.column) >= 0)
return cura.column;
return curb.column;
diff --git a/src/java/org/apache/cassandra/db/rows/SerializationHelper.java b/src/java/org/apache/cassandra/db/rows/SerializationHelper.java
index e40a1e1..dca4240 100644
--- a/src/java/org/apache/cassandra/db/rows/SerializationHelper.java
+++ b/src/java/org/apache/cassandra/db/rows/SerializationHelper.java
@@ -15,129 +15,43 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.cassandra.db.rows;
-import java.nio.ByteBuffer;
-import java.util.*;
-
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
-import org.apache.cassandra.db.*;
-import org.apache.cassandra.db.context.CounterContext;
-import org.apache.cassandra.db.filter.ColumnFilter;
+import org.apache.cassandra.db.SerializationHeader;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.utils.SearchIterator;
+import org.apache.cassandra.utils.btree.BTreeSearchIterator;
public class SerializationHelper
{
- /**
- * Flag affecting deserialization behavior (this only affect counters in practice).
- * - LOCAL: for deserialization of local data (Expired columns are
- * converted to tombstones (to gain disk space)).
- * - FROM_REMOTE: for deserialization of data received from remote hosts
- * (Expired columns are converted to tombstone and counters have
- * their delta cleared)
- * - PRESERVE_SIZE: used when no transformation must be performed, i.e,
- * when we must ensure that deserializing and reserializing the
- * result yield the exact same bytes. Streaming uses this.
- */
- public enum Flag
+ public final SerializationHeader header;
+ private BTreeSearchIterator<ColumnMetadata, ColumnMetadata> statics = null;
+ private BTreeSearchIterator<ColumnMetadata, ColumnMetadata> regulars = null;
+
+ public SerializationHelper(SerializationHeader header)
{
- LOCAL, FROM_REMOTE, PRESERVE_SIZE
+ this.header = header;
}
- private final Flag flag;
- public final int version;
-
- private final ColumnFilter columnsToFetch;
- private ColumnFilter.Tester tester;
-
- private final Map<ByteBuffer, CFMetaData.DroppedColumn> droppedColumns;
- private CFMetaData.DroppedColumn currentDroppedComplex;
-
-
- public SerializationHelper(CFMetaData metadata, int version, Flag flag, ColumnFilter columnsToFetch)
+ private BTreeSearchIterator<ColumnMetadata, ColumnMetadata> statics()
{
- this.flag = flag;
- this.version = version;
- this.columnsToFetch = columnsToFetch;
- this.droppedColumns = metadata.getDroppedColumns();
+ if (statics == null)
+ statics = header.columns().statics.iterator();
+ return statics;
}
- public SerializationHelper(CFMetaData metadata, int version, Flag flag)
+ private BTreeSearchIterator<ColumnMetadata, ColumnMetadata> regulars()
{
- this(metadata, version, flag, null);
+ if (regulars == null)
+ regulars = header.columns().regulars.iterator();
+ return regulars;
}
- public boolean includes(ColumnDefinition column)
+ public SearchIterator<ColumnMetadata, ColumnMetadata> iterator(boolean isStatic)
{
- return columnsToFetch == null || columnsToFetch.fetches(column);
- }
-
- public boolean includes(Cell cell, LivenessInfo rowLiveness)
- {
- if (columnsToFetch == null)
- return true;
-
- // During queries, some columns are included even though they are not queried by the user because
- // we always need to distinguish between having a row (with potentially only null values) and not
- // having a row at all (see #CASSANDRA-7085 for background). In the case where the column is not
- // actually requested by the user however (canSkipValue), we can skip the full cell if the cell
- // timestamp is lower than the row one, because in that case, the row timestamp is enough proof
- // of the liveness of the row. Otherwise, we'll only be able to skip the values of those cells.
- ColumnDefinition column = cell.column();
- if (column.isComplex())
- {
- if (!includes(cell.path()))
- return false;
-
- return !canSkipValue(cell.path()) || cell.timestamp() >= rowLiveness.timestamp();
- }
- else
- {
- return columnsToFetch.fetchedColumnIsQueried(column) || cell.timestamp() >= rowLiveness.timestamp();
- }
- }
-
- public boolean includes(CellPath path)
- {
- return path == null || tester == null || tester.fetches(path);
- }
-
- public boolean canSkipValue(ColumnDefinition column)
- {
- return columnsToFetch != null && !columnsToFetch.fetchedColumnIsQueried(column);
- }
-
- public boolean canSkipValue(CellPath path)
- {
- return path != null && tester != null && !tester.fetchedCellIsQueried(path);
- }
-
- public void startOfComplexColumn(ColumnDefinition column)
- {
- this.tester = columnsToFetch == null ? null : columnsToFetch.newTester(column);
- this.currentDroppedComplex = droppedColumns.get(column.name.bytes);
- }
-
- public void endOfComplexColumn()
- {
- this.tester = null;
- }
-
- public boolean isDropped(Cell cell, boolean isComplex)
- {
- CFMetaData.DroppedColumn dropped = isComplex ? currentDroppedComplex : droppedColumns.get(cell.column().name.bytes);
- return dropped != null && cell.timestamp() <= dropped.droppedTime;
- }
-
- public boolean isDroppedComplexDeletion(DeletionTime complexDeletion)
- {
- return currentDroppedComplex != null && complexDeletion.markedForDeleteAt() <= currentDroppedComplex.droppedTime;
- }
-
- public ByteBuffer maybeClearCounterValue(ByteBuffer value)
- {
- return flag == Flag.FROM_REMOTE || (flag == Flag.LOCAL && CounterContext.instance().shouldClearLocal(value))
- ? CounterContext.instance().clearAllLocal(value)
- : value;
+ BTreeSearchIterator<ColumnMetadata, ColumnMetadata> iterator = isStatic ? statics() : regulars();
+ iterator.rewind();
+ return iterator;
}
}
diff --git a/src/java/org/apache/cassandra/db/rows/ThrottledUnfilteredIterator.java b/src/java/org/apache/cassandra/db/rows/ThrottledUnfilteredIterator.java
new file mode 100644
index 0000000..a2e8425
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/rows/ThrottledUnfilteredIterator.java
@@ -0,0 +1,263 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db.rows;
+
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Iterator;
+
+import org.apache.cassandra.db.DeletionTime;
+import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
+import org.apache.cassandra.utils.AbstractIterator;
+import org.apache.cassandra.utils.CloseableIterator;
+
+import com.google.common.annotations.VisibleForTesting;
+
+/**
+ * A utility class to split the given {@link#UnfilteredRowIterator} into smaller chunks each
+ * having at most {@link #throttle} + 1 unfiltereds.
+ *
+ * Only the first output contains partition level info: {@link UnfilteredRowIterator#partitionLevelDeletion}
+ * and {@link UnfilteredRowIterator#staticRow}.
+ *
+ * Besides splitting, this iterator will also ensure each chunk does not finish with an open tombstone marker,
+ * by closing any opened tombstone markers and re-opening on the next chunk.
+ *
+ * The lifecycle of outputed {{@link UnfilteredRowIterator} only last till next call to {@link #next()}.
+ *
+ * A subsequent {@link #next} call will exhaust the previously returned iterator before computing the next,
+ * effectively skipping unfiltereds up to the throttle size.
+ *
+ * Closing this iterator will close the underlying iterator.
+ *
+ */
+public class ThrottledUnfilteredIterator extends AbstractIterator<UnfilteredRowIterator> implements CloseableIterator<UnfilteredRowIterator>
+{
+ private final UnfilteredRowIterator origin;
+ private final int throttle;
+
+ // internal mutable state
+ private UnfilteredRowIterator throttledItr;
+
+ // extra unfiltereds from previous iteration
+ private Iterator<Unfiltered> overflowed = Collections.emptyIterator();
+
+ @VisibleForTesting
+ ThrottledUnfilteredIterator(UnfilteredRowIterator origin, int throttle)
+ {
+ assert origin != null;
+ assert throttle > 1 : "Throttle size must be higher than 1 to properly support open and close tombstone boundaries.";
+ this.origin = origin;
+ this.throttle = throttle;
+ this.throttledItr = null;
+ }
+
+ @Override
+ protected UnfilteredRowIterator computeNext()
+ {
+ // exhaust previous throttled iterator
+ while (throttledItr != null && throttledItr.hasNext())
+ throttledItr.next();
+
+ // The original UnfilteredRowIterator may have only partition deletion or static column but without unfiltereds.
+ // Return the original UnfilteredRowIterator
+ if (!origin.hasNext())
+ {
+ if (throttledItr != null)
+ return endOfData();
+ return throttledItr = origin;
+ }
+
+ throttledItr = new WrappingUnfilteredRowIterator(origin)
+ {
+ private int count = 0;
+ private boolean isFirst = throttledItr == null;
+
+ // current batch's openMarker. if it's generated in previous batch,
+ // it must be consumed as first element of current batch
+ private RangeTombstoneMarker openMarker;
+
+ // current batch's closeMarker.
+ // it must be consumed as last element of current batch
+ private RangeTombstoneMarker closeMarker = null;
+
+ @Override
+ public boolean hasNext()
+ {
+ return (withinLimit() && wrapped.hasNext()) || closeMarker != null;
+ }
+
+ @Override
+ public Unfiltered next()
+ {
+ if (closeMarker != null)
+ {
+ assert count == throttle;
+ Unfiltered toReturn = closeMarker;
+ closeMarker = null;
+ return toReturn;
+ }
+
+ Unfiltered next;
+ assert withinLimit();
+ // in the beginning of the batch, there might be remaining unfiltereds from previous iteration
+ if (overflowed.hasNext())
+ next = overflowed.next();
+ else
+ next = wrapped.next();
+ recordNext(next);
+ return next;
+ }
+
+ private void recordNext(Unfiltered unfiltered)
+ {
+ count++;
+ if (unfiltered.isRangeTombstoneMarker())
+ updateMarker((RangeTombstoneMarker) unfiltered);
+ // when reach throttle with a remaining openMarker, we need to create corresponding closeMarker.
+ if (count == throttle && openMarker != null)
+ {
+ assert wrapped.hasNext();
+ closeOpenMarker(wrapped.next());
+ }
+ }
+
+ private boolean withinLimit()
+ {
+ return count < throttle;
+ }
+
+ private void updateMarker(RangeTombstoneMarker marker)
+ {
+ openMarker = marker.isOpen(isReverseOrder()) ? marker : null;
+ }
+
+ /**
+ * There 3 cases for next, 1. if it's boundaryMarker, we split it as closeMarker for current batch, next
+ * openMarker for next batch 2. if it's boundMakrer, it must be closeMarker. 3. if it's Row, create
+ * corresponding closeMarker for current batch, and create next openMarker for next batch including current
+ * Row.
+ */
+ private void closeOpenMarker(Unfiltered next)
+ {
+ assert openMarker != null;
+
+ if (next.isRangeTombstoneMarker())
+ {
+ RangeTombstoneMarker marker = (RangeTombstoneMarker) next;
+ // if it's boundary, create closeMarker for current batch and openMarker for next batch
+ if (marker.isBoundary())
+ {
+ RangeTombstoneBoundaryMarker boundary = (RangeTombstoneBoundaryMarker) marker;
+ closeMarker = boundary.createCorrespondingCloseMarker(isReverseOrder());
+ overflowed = Collections.singleton((Unfiltered)boundary.createCorrespondingOpenMarker(isReverseOrder())).iterator();
+ }
+ else
+ {
+ // if it's bound, it must be closeMarker.
+ assert marker.isClose(isReverseOrder());
+ updateMarker(marker);
+ closeMarker = marker;
+ }
+ }
+ else
+ {
+ // it's Row, need to create closeMarker for current batch and openMarker for next batch
+ DeletionTime openDeletion = openMarker.openDeletionTime(isReverseOrder());
+ ByteBuffer[] buffers = next.clustering().getRawValues();
+ closeMarker = RangeTombstoneBoundMarker.exclusiveClose(isReverseOrder(), buffers, openDeletion);
+
+ // for next batch
+ overflowed = Arrays.asList(RangeTombstoneBoundMarker.inclusiveOpen(isReverseOrder(),
+ buffers,
+ openDeletion), next).iterator();
+ }
+ }
+
+ @Override
+ public DeletionTime partitionLevelDeletion()
+ {
+ return isFirst ? wrapped.partitionLevelDeletion() : DeletionTime.LIVE;
+ }
+
+ @Override
+ public Row staticRow()
+ {
+ return isFirst ? wrapped.staticRow() : Rows.EMPTY_STATIC_ROW;
+ }
+
+ @Override
+ public void close()
+ {
+ // no op
+ }
+ };
+ return throttledItr;
+ }
+
+ public void close()
+ {
+ if (origin != null)
+ origin.close();
+ }
+
+ /**
+ * Splits a {@link UnfilteredPartitionIterator} in {@link UnfilteredRowIterator} batches with size no higher than
+ * <b>maxBatchSize</b>
+ *
+ * @param partitionIterator
+ * @param maxBatchSize max number of unfiltereds in the UnfilteredRowIterator. if 0 is given, it means no throttle.
+ * @return
+ */
+ public static CloseableIterator<UnfilteredRowIterator> throttle(UnfilteredPartitionIterator partitionIterator, int maxBatchSize)
+ {
+ if (maxBatchSize == 0) // opt out
+ return partitionIterator;
+
+ return new AbstractIterator<UnfilteredRowIterator>()
+ {
+ ThrottledUnfilteredIterator current = null;
+
+ protected UnfilteredRowIterator computeNext()
+ {
+ if (current != null && !current.hasNext())
+ {
+ current.close();
+ current = null;
+ }
+
+ if (current == null && partitionIterator.hasNext())
+ {
+ current = new ThrottledUnfilteredIterator(partitionIterator.next(), maxBatchSize);
+ }
+
+ if (current != null && current.hasNext())
+ return current.next();
+
+ return endOfData();
+ }
+
+ public void close()
+ {
+ if (current != null)
+ current.close();
+ }
+ };
+ }
+}
diff --git a/src/java/org/apache/cassandra/db/rows/Unfiltered.java b/src/java/org/apache/cassandra/db/rows/Unfiltered.java
index 3d8a9b1..f5c5ed0 100644
--- a/src/java/org/apache/cassandra/db/rows/Unfiltered.java
+++ b/src/java/org/apache/cassandra/db/rows/Unfiltered.java
@@ -17,11 +17,8 @@
*/
package org.apache.cassandra.db.rows;
-import java.nio.ByteBuffer;
-import java.security.MessageDigest;
-import java.util.Set;
-
-import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.db.Digest;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.db.Clusterable;
/**
@@ -41,21 +38,11 @@
public Kind kind();
/**
- * Digest the atom using the provided {@code MessageDigest}.
+ * Digest the atom using the provided {@link Digest}.
*
- * @param digest the {@code MessageDigest} to use.
+ * @param digest the {@see Digest} to use.
*/
- public void digest(MessageDigest digest);
-
- /**
- * Digest the atom using the provided {@code MessageDigest}.
- * This method only exists in 3.11.
- * Same like {@link #digest(MessageDigest)}, but excludes the given columns from digest calculation.
- */
- public default void digest(MessageDigest digest, Set<ByteBuffer> columnsToExclude)
- {
- throw new UnsupportedOperationException("no no no - don't use this one - use digest(MessageDigest) instead");
- }
+ public void digest(Digest digest);
/**
* Validate the data of this atom.
@@ -65,13 +52,19 @@
* invalid (some value is invalid for its column type, or some field
* is nonsensical).
*/
- public void validateData(CFMetaData metadata);
+ public void validateData(TableMetadata metadata);
+ /**
+ * Do a quick validation of the deletions of the unfiltered (if any)
+ *
+ * @return true if any deletion is invalid
+ */
+ public boolean hasInvalidDeletions();
public boolean isEmpty();
- public String toString(CFMetaData metadata);
- public String toString(CFMetaData metadata, boolean fullDetails);
- public String toString(CFMetaData metadata, boolean includeClusterKeys, boolean fullDetails);
+ public String toString(TableMetadata metadata);
+ public String toString(TableMetadata metadata, boolean fullDetails);
+ public String toString(TableMetadata metadata, boolean includeClusterKeys, boolean fullDetails);
default boolean isRow()
{
diff --git a/src/java/org/apache/cassandra/db/rows/UnfilteredRowIteratorSerializer.java b/src/java/org/apache/cassandra/db/rows/UnfilteredRowIteratorSerializer.java
index 45c026f..df67754 100644
--- a/src/java/org/apache/cassandra/db/rows/UnfilteredRowIteratorSerializer.java
+++ b/src/java/org/apache/cassandra/db/rows/UnfilteredRowIteratorSerializer.java
@@ -23,11 +23,11 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.filter.ColumnFilter;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.utils.ByteBufferUtil;
/**
@@ -123,18 +123,19 @@
out.writeByte((byte)flags);
SerializationHeader.serializer.serializeForMessaging(header, selection, out, hasStatic);
+ SerializationHelper helper = new SerializationHelper(header);
if (!partitionDeletion.isLive())
header.writeDeletionTime(partitionDeletion, out);
if (hasStatic)
- UnfilteredSerializer.serializer.serialize(staticRow, header, out, version);
+ UnfilteredSerializer.serializer.serialize(staticRow, helper, out, version);
if (rowEstimate >= 0)
out.writeUnsignedVInt(rowEstimate);
while (iterator.hasNext())
- UnfilteredSerializer.serializer.serialize(iterator.next(), header, out, version);
+ UnfilteredSerializer.serializer.serialize(iterator.next(), helper, out, version);
UnfilteredSerializer.serializer.writeEndOfPartition(out);
}
@@ -147,6 +148,8 @@
iterator.columns(),
iterator.stats());
+ SerializationHelper helper = new SerializationHelper(header);
+
assert rowEstimate >= 0;
long size = ByteBufferUtil.serializedSizeWithVIntLength(iterator.partitionKey().getKey())
@@ -165,26 +168,26 @@
size += header.deletionTimeSerializedSize(partitionDeletion);
if (hasStatic)
- size += UnfilteredSerializer.serializer.serializedSize(staticRow, header, version);
+ size += UnfilteredSerializer.serializer.serializedSize(staticRow, helper, version);
if (rowEstimate >= 0)
size += TypeSizes.sizeofUnsignedVInt(rowEstimate);
while (iterator.hasNext())
- size += UnfilteredSerializer.serializer.serializedSize(iterator.next(), header, version);
+ size += UnfilteredSerializer.serializer.serializedSize(iterator.next(), helper, version);
size += UnfilteredSerializer.serializer.serializedSizeEndOfPartition();
return size;
}
- public Header deserializeHeader(CFMetaData metadata, ColumnFilter selection, DataInputPlus in, int version, SerializationHelper.Flag flag) throws IOException
+ public Header deserializeHeader(TableMetadata metadata, ColumnFilter selection, DataInputPlus in, int version, DeserializationHelper.Flag flag) throws IOException
{
- DecoratedKey key = metadata.decorateKey(ByteBufferUtil.readWithVIntLength(in));
+ DecoratedKey key = metadata.partitioner.decorateKey(ByteBufferUtil.readWithVIntLength(in));
int flags = in.readUnsignedByte();
boolean isReversed = (flags & IS_REVERSED) != 0;
if ((flags & IS_EMPTY) != 0)
{
- SerializationHeader sh = new SerializationHeader(false, metadata, PartitionColumns.NONE, EncodingStats.NO_STATS);
+ SerializationHeader sh = new SerializationHeader(false, metadata, RegularAndStaticColumns.NONE, EncodingStats.NO_STATS);
return new Header(sh, key, isReversed, true, null, null, 0);
}
@@ -198,18 +201,18 @@
Row staticRow = Rows.EMPTY_STATIC_ROW;
if (hasStatic)
- staticRow = UnfilteredSerializer.serializer.deserializeStaticRow(in, header, new SerializationHelper(metadata, version, flag));
+ staticRow = UnfilteredSerializer.serializer.deserializeStaticRow(in, header, new DeserializationHelper(metadata, version, flag));
int rowEstimate = hasRowEstimate ? (int)in.readUnsignedVInt() : -1;
return new Header(header, key, isReversed, false, partitionDeletion, staticRow, rowEstimate);
}
- public UnfilteredRowIterator deserialize(DataInputPlus in, int version, CFMetaData metadata, SerializationHelper.Flag flag, Header header) throws IOException
+ public UnfilteredRowIterator deserialize(DataInputPlus in, int version, TableMetadata metadata, DeserializationHelper.Flag flag, Header header) throws IOException
{
if (header.isEmpty)
return EmptyIterators.unfilteredRow(metadata, header.key, header.isReversed);
- final SerializationHelper helper = new SerializationHelper(metadata, version, flag);
+ final DeserializationHelper helper = new DeserializationHelper(metadata, version, flag);
final SerializationHeader sHeader = header.sHeader;
return new AbstractUnfilteredRowIterator(metadata, header.key, header.partitionDeletion, sHeader.columns(), header.staticRow, header.isReversed, sHeader.stats())
{
@@ -230,7 +233,7 @@
};
}
- public UnfilteredRowIterator deserialize(DataInputPlus in, int version, CFMetaData metadata, ColumnFilter selection, SerializationHelper.Flag flag) throws IOException
+ public UnfilteredRowIterator deserialize(DataInputPlus in, int version, TableMetadata metadata, ColumnFilter selection, DeserializationHelper.Flag flag) throws IOException
{
return deserialize(in, version, metadata, flag, deserializeHeader(metadata, selection, in, version, flag));
}
diff --git a/src/java/org/apache/cassandra/db/rows/UnfilteredRowIteratorWithLowerBound.java b/src/java/org/apache/cassandra/db/rows/UnfilteredRowIteratorWithLowerBound.java
index e381436..4407999 100644
--- a/src/java/org/apache/cassandra/db/rows/UnfilteredRowIteratorWithLowerBound.java
+++ b/src/java/org/apache/cassandra/db/rows/UnfilteredRowIteratorWithLowerBound.java
@@ -25,7 +25,7 @@
import java.util.Comparator;
import java.util.List;
-import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.filter.ClusteringIndexFilter;
import org.apache.cassandra.db.filter.ColumnFilter;
@@ -34,7 +34,6 @@
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.sstable.format.SSTableReadsListener;
import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
-import org.apache.cassandra.thrift.ThriftResultsMerger;
import org.apache.cassandra.utils.IteratorWithLowerBound;
/**
@@ -50,9 +49,6 @@
private final SSTableReader sstable;
private final ClusteringIndexFilter filter;
private final ColumnFilter selectedColumns;
- private final boolean isForThrift;
- private final int nowInSec;
- private final boolean applyThriftTransformation;
private final SSTableReadsListener listener;
private ClusteringBound lowerBound;
private boolean firstItemRetrieved;
@@ -61,18 +57,12 @@
SSTableReader sstable,
ClusteringIndexFilter filter,
ColumnFilter selectedColumns,
- boolean isForThrift,
- int nowInSec,
- boolean applyThriftTransformation,
SSTableReadsListener listener)
{
super(partitionKey);
this.sstable = sstable;
this.filter = filter;
this.selectedColumns = selectedColumns;
- this.isForThrift = isForThrift;
- this.nowInSec = nowInSec;
- this.applyThriftTransformation = applyThriftTransformation;
this.listener = listener;
this.lowerBound = null;
this.firstItemRetrieved = false;
@@ -105,12 +95,12 @@
protected UnfilteredRowIterator initializeIterator()
{
@SuppressWarnings("resource") // 'iter' is added to iterators which is closed on exception, or through the closing of the final merged iterator
- UnfilteredRowIterator iter = sstable.iterator(partitionKey(), filter.getSlices(metadata()), selectedColumns, filter.isReversed(), isForThrift, listener);
-
- if (isForThrift && applyThriftTransformation)
- iter = ThriftResultsMerger.maybeWrap(iter, nowInSec);
-
- return RTBoundValidator.validate(iter, RTBoundValidator.Stage.SSTABLE, false);
+ UnfilteredRowIterator iter = RTBoundValidator.validate(
+ sstable.iterator(partitionKey(), filter.getSlices(metadata()), selectedColumns, filter.isReversed(), listener),
+ RTBoundValidator.Stage.SSTABLE,
+ false
+ );
+ return iter;
}
@Override
@@ -125,8 +115,8 @@
if (lowerBound != null && ret != null)
assert comparator().compare(lowerBound, ret.clustering()) <= 0
: String.format("Lower bound [%s ]is bigger than first returned value [%s] for sstable %s",
- lowerBound.toString(sstable.metadata),
- ret.toString(sstable.metadata),
+ lowerBound.toString(metadata()),
+ ret.toString(metadata()),
sstable.getFilename());
return ret;
@@ -134,13 +124,13 @@
private Comparator<Clusterable> comparator()
{
- return filter.isReversed() ? sstable.metadata.comparator.reversed() : sstable.metadata.comparator;
+ return filter.isReversed() ? metadata().comparator.reversed() : metadata().comparator;
}
@Override
- public CFMetaData metadata()
+ public TableMetadata metadata()
{
- return sstable.metadata;
+ return sstable.metadata();
}
@Override
@@ -150,7 +140,7 @@
}
@Override
- public PartitionColumns columns()
+ public RegularAndStaticColumns columns()
{
return selectedColumns.fetchedColumns();
}
@@ -201,10 +191,10 @@
{
IndexInfo column = onHeapRetriever.columnsIndex(filter.isReversed() ? rowIndexEntry.columnsIndexCount() - 1 : 0);
ClusteringPrefix lowerBoundPrefix = filter.isReversed() ? column.lastName : column.firstName;
- assert lowerBoundPrefix.getRawValues().length <= sstable.metadata.comparator.size() :
+ assert lowerBoundPrefix.getRawValues().length <= metadata().comparator.size() :
String.format("Unexpected number of clustering values %d, expected %d or fewer for %s",
lowerBoundPrefix.getRawValues().length,
- sstable.metadata.comparator.size(),
+ metadata().comparator.size(),
sstable.getFilename());
return ClusteringBound.inclusiveOpen(filter.isReversed(), lowerBoundPrefix.getRawValues());
}
@@ -242,7 +232,7 @@
// Side-note: pre-2.1 sstable stat file had clustering value arrays whose size may not match the comparator size
// and that would break getMetadataLowerBound. We don't support upgrade from 2.0 to 3.0 directly however so it's
// not a true concern. Besides, !sstable.mayHaveTombstones already ensure this is a 3.0 sstable anyway.
- return !sstable.mayHaveTombstones() && !sstable.metadata.isCompactTable();
+ return !sstable.mayHaveTombstones() && !sstable.metadata().isCompactTable();
}
/**
@@ -256,10 +246,10 @@
final StatsMetadata m = sstable.getSSTableMetadata();
List<ByteBuffer> vals = filter.isReversed() ? m.maxClusteringValues : m.minClusteringValues;
- assert vals.size() <= sstable.metadata.comparator.size() :
+ assert vals.size() <= metadata().comparator.size() :
String.format("Unexpected number of clustering values %d, expected %d or fewer for %s",
vals.size(),
- sstable.metadata.comparator.size(),
+ metadata().comparator.size(),
sstable.getFilename());
return ClusteringBound.inclusiveOpen(filter.isReversed(), vals.toArray(new ByteBuffer[vals.size()]));
}
diff --git a/src/java/org/apache/cassandra/db/rows/UnfilteredRowIterators.java b/src/java/org/apache/cassandra/db/rows/UnfilteredRowIterators.java
index 1fda52a..2eb5d8f 100644
--- a/src/java/org/apache/cassandra/db/rows/UnfilteredRowIterators.java
+++ b/src/java/org/apache/cassandra/db/rows/UnfilteredRowIterators.java
@@ -18,21 +18,19 @@
package org.apache.cassandra.db.rows;
import java.util.*;
-import java.security.MessageDigest;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.filter.ColumnFilter;
import org.apache.cassandra.db.transform.FilteredRows;
import org.apache.cassandra.db.transform.MoreRows;
import org.apache.cassandra.db.transform.Transformation;
-import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.io.sstable.CorruptSSTableException;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.serializers.MarshalException;
-import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.IMergeIterator;
import org.apache.cassandra.utils.MergeIterator;
@@ -100,6 +98,17 @@
public void onMergedRangeTombstoneMarkers(RangeTombstoneMarker merged, RangeTombstoneMarker[] versions);
public void close();
+
+ public static MergeListener NOOP = new MergeListener()
+ {
+ public void onMergedPartitionLevelDeletion(DeletionTime mergedDeletion, DeletionTime[] versions) {}
+
+ public Row onMergedRows(Row merged, Row[] versions) {return merged;}
+
+ public void onMergedRangeTombstoneMarkers(RangeTombstoneMarker merged, RangeTombstoneMarker[] versions) {}
+
+ public void close() {}
+ };
}
/**
@@ -117,13 +126,13 @@
/**
* Returns an iterator that is the result of merging other iterators.
*/
- public static UnfilteredRowIterator merge(List<UnfilteredRowIterator> iterators, int nowInSec)
+ public static UnfilteredRowIterator merge(List<UnfilteredRowIterator> iterators)
{
assert !iterators.isEmpty();
if (iterators.size() == 1)
return iterators.get(0);
- return UnfilteredRowMergeIterator.create(iterators, nowInSec, null);
+ return UnfilteredRowMergeIterator.create(iterators, null);
}
/**
@@ -132,24 +141,24 @@
*
* Note that this method assumes that there is at least 2 iterators to merge.
*/
- public static UnfilteredRowIterator merge(List<UnfilteredRowIterator> iterators, int nowInSec, MergeListener mergeListener)
+ public static UnfilteredRowIterator merge(List<UnfilteredRowIterator> iterators, MergeListener mergeListener)
{
- return UnfilteredRowMergeIterator.create(iterators, nowInSec, mergeListener);
+ return UnfilteredRowMergeIterator.create(iterators, mergeListener);
}
/**
* Returns an empty unfiltered iterator for a given partition.
*/
- public static UnfilteredRowIterator noRowsIterator(final CFMetaData cfm, final DecoratedKey partitionKey, final Row staticRow, final DeletionTime partitionDeletion, final boolean isReverseOrder)
+ public static UnfilteredRowIterator noRowsIterator(final TableMetadata metadata, final DecoratedKey partitionKey, final Row staticRow, final DeletionTime partitionDeletion, final boolean isReverseOrder)
{
- return EmptyIterators.unfilteredRow(cfm, partitionKey, isReverseOrder, staticRow, partitionDeletion);
+ return EmptyIterators.unfilteredRow(metadata, partitionKey, isReverseOrder, staticRow, partitionDeletion);
}
public static UnfilteredRowIterator singleton(Unfiltered unfiltered,
- CFMetaData metadata,
+ TableMetadata metadata,
DecoratedKey partitionKey,
DeletionTime partitionLevelDeletion,
- PartitionColumns columns,
+ RegularAndStaticColumns columns,
Row staticRow,
boolean isReverseOrder,
EncodingStats encodingStats)
@@ -174,21 +183,13 @@
/**
* Digests the partition represented by the provided iterator.
*
- * @param command the command that has yield {@code iterator}. This can be null if {@code version >= MessagingService.VERSION_30}
- * as this is only used when producing digest to be sent to legacy nodes.
* @param iterator the iterator to digest.
- * @param digest the {@code MessageDigest} to use for the digest.
+ * @param digest the {@link Digest} to use.
* @param version the messaging protocol to use when producing the digest.
*/
- public static void digest(ReadCommand command, UnfilteredRowIterator iterator, MessageDigest digest, int version)
+ public static void digest(UnfilteredRowIterator iterator, Digest digest, int version)
{
- if (version < MessagingService.VERSION_30)
- {
- LegacyLayout.fromUnfilteredRowIterator(command, iterator).digest(iterator.metadata(), digest);
- return;
- }
-
- digest.update(iterator.partitionKey().getKey().duplicate());
+ digest.update(iterator.partitionKey().getKey());
iterator.partitionLevelDeletion().digest(digest);
iterator.columns().regulars.digest(digest);
// When serializing an iterator, we skip the static columns if the iterator has not static row, even if the
@@ -203,7 +204,7 @@
// upgrade) so we can only do on the next protocol version bump.
if (iterator.staticRow() != Rows.EMPTY_STATIC_ROW)
iterator.columns().statics.digest(digest);
- FBUtilities.updateWithBoolean(digest, iterator.isReverseOrder());
+ digest.updateWithBoolean(iterator.isReverseOrder());
iterator.staticRow().digest(digest);
while (iterator.hasNext())
@@ -238,7 +239,7 @@
*/
public static UnfilteredRowIterator concat(final UnfilteredRowIterator iter1, final UnfilteredRowIterator iter2)
{
- assert iter1.metadata().cfId.equals(iter2.metadata().cfId)
+ assert iter1.metadata().id.equals(iter2.metadata().id)
&& iter1.partitionKey().equals(iter2.partitionKey())
&& iter1.partitionLevelDeletion().equals(iter2.partitionLevelDeletion())
&& iter1.isReverseOrder() == iter2.isReverseOrder()
@@ -348,12 +349,12 @@
*/
public static UnfilteredRowIterator loggingIterator(UnfilteredRowIterator iterator, final String id, final boolean fullDetails)
{
- CFMetaData metadata = iterator.metadata();
+ TableMetadata metadata = iterator.metadata();
logger.info("[{}] Logging iterator on {}.{}, partition key={}, reversed={}, deletion={}",
id,
- metadata.ksName,
- metadata.cfName,
- metadata.getKeyValidator().getString(iterator.partitionKey().getKey()),
+ metadata.keyspace,
+ metadata.name,
+ metadata.partitionKeyType.getString(iterator.partitionKey().getKey()),
iterator.isReverseOrder(),
iterator.partitionLevelDeletion().markedForDeleteAt());
@@ -392,11 +393,10 @@
private final IMergeIterator<Unfiltered, Unfiltered> mergeIterator;
private final MergeListener listener;
- private UnfilteredRowMergeIterator(CFMetaData metadata,
+ private UnfilteredRowMergeIterator(TableMetadata metadata,
List<UnfilteredRowIterator> iterators,
- PartitionColumns columns,
+ RegularAndStaticColumns columns,
DeletionTime partitionDeletion,
- int nowInSec,
boolean reversed,
MergeListener listener)
{
@@ -404,17 +404,17 @@
iterators.get(0).partitionKey(),
partitionDeletion,
columns,
- mergeStaticRows(iterators, columns.statics, nowInSec, listener, partitionDeletion),
+ mergeStaticRows(iterators, columns.statics, listener, partitionDeletion),
reversed,
- mergeStats(iterators));
+ EncodingStats.merge(iterators, UnfilteredRowIterator::stats));
this.mergeIterator = MergeIterator.get(iterators,
reversed ? metadata.comparator.reversed() : metadata.comparator,
- new MergeReducer(iterators.size(), reversed, nowInSec, listener));
+ new MergeReducer(iterators.size(), reversed, listener));
this.listener = listener;
}
- private static UnfilteredRowMergeIterator create(List<UnfilteredRowIterator> iterators, int nowInSec, MergeListener listener)
+ private static UnfilteredRowMergeIterator create(List<UnfilteredRowIterator> iterators, MergeListener listener)
{
try
{
@@ -423,7 +423,6 @@
iterators,
collectColumns(iterators),
collectPartitionLevelDeletion(iterators, listener),
- nowInSec,
iterators.get(0).isReverseOrder(),
listener);
}
@@ -451,7 +450,7 @@
for (int i = 1; i < iterators.size(); i++)
{
UnfilteredRowIterator iter = iterators.get(i);
- assert first.metadata().cfId.equals(iter.metadata().cfId);
+ assert first.metadata().id.equals(iter.metadata().id);
assert first.partitionKey().equals(iter.partitionKey());
assert first.isReverseOrder() == iter.isReverseOrder();
}
@@ -479,7 +478,6 @@
private static Row mergeStaticRows(List<UnfilteredRowIterator> iterators,
Columns columns,
- int nowInSec,
MergeListener listener,
DeletionTime partitionDeletion)
{
@@ -489,7 +487,7 @@
if (iterators.stream().allMatch(iter -> iter.staticRow().isEmpty()))
return Rows.EMPTY_STATIC_ROW;
- Row.Merger merger = new Row.Merger(iterators.size(), nowInSec, columns.hasComplex());
+ Row.Merger merger = new Row.Merger(iterators.size(), columns.hasComplex());
for (int i = 0; i < iterators.size(); i++)
merger.add(i, iterators.get(i).staticRow());
@@ -504,28 +502,20 @@
return merged == null ? Rows.EMPTY_STATIC_ROW : merged;
}
- private static PartitionColumns collectColumns(List<UnfilteredRowIterator> iterators)
+ private static RegularAndStaticColumns collectColumns(List<UnfilteredRowIterator> iterators)
{
- PartitionColumns first = iterators.get(0).columns();
+ RegularAndStaticColumns first = iterators.get(0).columns();
Columns statics = first.statics;
Columns regulars = first.regulars;
for (int i = 1; i < iterators.size(); i++)
{
- PartitionColumns cols = iterators.get(i).columns();
+ RegularAndStaticColumns cols = iterators.get(i).columns();
statics = statics.mergeTo(cols.statics);
regulars = regulars.mergeTo(cols.regulars);
}
return statics == first.statics && regulars == first.regulars
? first
- : new PartitionColumns(statics, regulars);
- }
-
- private static EncodingStats mergeStats(List<UnfilteredRowIterator> iterators)
- {
- EncodingStats stats = EncodingStats.NO_STATS;
- for (UnfilteredRowIterator iter : iterators)
- stats = stats.mergeWith(iter.stats());
- return stats;
+ : new RegularAndStaticColumns(statics, regulars);
}
protected Unfiltered computeNext()
@@ -557,9 +547,9 @@
private final Row.Merger rowMerger;
private final RangeTombstoneMarker.Merger markerMerger;
- private MergeReducer(int size, boolean reversed, int nowInSec, MergeListener listener)
+ private MergeReducer(int size, boolean reversed, MergeListener listener)
{
- this.rowMerger = new Row.Merger(size, nowInSec, columns().regulars.hasComplex());
+ this.rowMerger = new Row.Merger(size, columns().regulars.hasComplex());
this.markerMerger = new RangeTombstoneMarker.Merger(size, partitionLevelDeletion(), reversed);
this.listener = listener;
}
@@ -613,4 +603,5 @@
}
}
}
+
}
diff --git a/src/java/org/apache/cassandra/db/rows/UnfilteredSerializer.java b/src/java/org/apache/cassandra/db/rows/UnfilteredSerializer.java
index 926f3ef..a5fad14 100644
--- a/src/java/org/apache/cassandra/db/rows/UnfilteredSerializer.java
+++ b/src/java/org/apache/cassandra/db/rows/UnfilteredSerializer.java
@@ -19,10 +19,8 @@
import java.io.IOException;
-import com.google.common.collect.Collections2;
-
import net.nicoulaj.compilecommand.annotations.Inline;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.rows.Row.Deletion;
import org.apache.cassandra.io.util.DataInputPlus;
@@ -118,40 +116,41 @@
@Deprecated
private final static int HAS_SHADOWABLE_DELETION = 0x02; // Whether the row deletion is shadowable. If there is no extended flag (or no row deletion), the deletion is assumed not shadowable.
- public void serialize(Unfiltered unfiltered, SerializationHeader header, DataOutputPlus out, int version)
+ public void serialize(Unfiltered unfiltered, SerializationHelper helper, DataOutputPlus out, int version)
throws IOException
{
- assert !header.isForSSTable();
- serialize(unfiltered, header, out, 0, version);
+ assert !helper.header.isForSSTable();
+ serialize(unfiltered, helper, out, 0, version);
}
- public void serialize(Unfiltered unfiltered, SerializationHeader header, DataOutputPlus out, long previousUnfilteredSize, int version)
+ public void serialize(Unfiltered unfiltered, SerializationHelper helper, DataOutputPlus out, long previousUnfilteredSize, int version)
throws IOException
{
if (unfiltered.kind() == Unfiltered.Kind.RANGE_TOMBSTONE_MARKER)
{
- serialize((RangeTombstoneMarker) unfiltered, header, out, previousUnfilteredSize, version);
+ serialize((RangeTombstoneMarker) unfiltered, helper, out, previousUnfilteredSize, version);
}
else
{
- serialize((Row) unfiltered, header, out, previousUnfilteredSize, version);
+ serialize((Row) unfiltered, helper, out, previousUnfilteredSize, version);
}
}
- public void serializeStaticRow(Row row, SerializationHeader header, DataOutputPlus out, int version)
+ public void serializeStaticRow(Row row, SerializationHelper helper, DataOutputPlus out, int version)
throws IOException
{
assert row.isStatic();
- serialize(row, header, out, 0, version);
+ serialize(row, helper, out, 0, version);
}
- private void serialize(Row row, SerializationHeader header, DataOutputPlus out, long previousUnfilteredSize, int version)
+ private void serialize(Row row, SerializationHelper helper, DataOutputPlus out, long previousUnfilteredSize, int version)
throws IOException
{
int flags = 0;
int extendedFlags = 0;
boolean isStatic = row.isStatic();
+ SerializationHeader header = helper.header;
Columns headerColumns = header.columns(isStatic);
LivenessInfo pkLiveness = row.primaryKeyLivenessInfo();
Row.Deletion deletion = row.deletion();
@@ -191,7 +190,7 @@
{
try (DataOutputBuffer dob = DataOutputBuffer.scratchBuffer.get())
{
- serializeRowBody(row, flags, header, dob);
+ serializeRowBody(row, flags, helper, dob);
out.writeUnsignedVInt(dob.position() + TypeSizes.sizeofUnsignedVInt(previousUnfilteredSize));
// We write the size of the previous unfiltered to make reverse queries more efficient (and simpler).
@@ -202,16 +201,17 @@
}
else
{
- serializeRowBody(row, flags, header, out);
+ serializeRowBody(row, flags, helper, out);
}
}
@Inline
- private void serializeRowBody(Row row, int flags, SerializationHeader header, DataOutputPlus out)
+ private void serializeRowBody(Row row, int flags, SerializationHelper helper, DataOutputPlus out)
throws IOException
{
boolean isStatic = row.isStatic();
+ SerializationHeader header = helper.header;
Columns headerColumns = header.columns(isStatic);
LivenessInfo pkLiveness = row.primaryKeyLivenessInfo();
Row.Deletion deletion = row.deletion();
@@ -229,7 +229,7 @@
if ((flags & HAS_ALL_COLUMNS) == 0)
Columns.serializer.serializeSubset(row.columns(), headerColumns, out);
- SearchIterator<ColumnDefinition, ColumnDefinition> si = headerColumns.iterator();
+ SearchIterator<ColumnMetadata, ColumnMetadata> si = helper.iterator(isStatic);
try
{
@@ -237,9 +237,9 @@
// We can obtain the column for data directly from data.column(). However, if the cell/complex data
// originates from a sstable, the column we'll get will have the type used when the sstable was serialized,
// and if that type have been recently altered, that may not be the type we want to serialize the column
- // with. So we use the ColumnDefinition from the "header" which is "current". Also see #11810 for what
+ // with. So we use the ColumnMetadata from the "header" which is "current". Also see #11810 for what
// happens if we don't do that.
- ColumnDefinition column = si.next(cd.column());
+ ColumnMetadata column = si.next(cd.column());
assert column != null : cd.column.toString();
try
@@ -253,7 +253,7 @@
{
throw new WrappedException(e);
}
- }, false);
+ });
}
catch (WrappedException e)
{
@@ -264,7 +264,7 @@
}
}
- private void writeComplexColumn(ComplexColumnData data, ColumnDefinition column, boolean hasComplexDeletion, LivenessInfo rowLiveness, SerializationHeader header, DataOutputPlus out)
+ private void writeComplexColumn(ComplexColumnData data, ColumnMetadata column, boolean hasComplexDeletion, LivenessInfo rowLiveness, SerializationHeader header, DataOutputPlus out)
throws IOException
{
if (hasComplexDeletion)
@@ -275,9 +275,10 @@
Cell.serializer.serialize(cell, column, out, rowLiveness, header);
}
- private void serialize(RangeTombstoneMarker marker, SerializationHeader header, DataOutputPlus out, long previousUnfilteredSize, int version)
+ private void serialize(RangeTombstoneMarker marker, SerializationHelper helper, DataOutputPlus out, long previousUnfilteredSize, int version)
throws IOException
{
+ SerializationHeader header = helper.header;
out.writeByte((byte)IS_MARKER);
ClusteringBoundOrBoundary.serializer.serialize(marker.clustering(), out, version, header.clusteringTypes());
@@ -299,20 +300,20 @@
}
}
- public long serializedSize(Unfiltered unfiltered, SerializationHeader header, int version)
+ public long serializedSize(Unfiltered unfiltered, SerializationHelper helper, int version)
{
- assert !header.isForSSTable();
- return serializedSize(unfiltered, header, 0, version);
+ assert !helper.header.isForSSTable();
+ return serializedSize(unfiltered, helper, 0, version);
}
- public long serializedSize(Unfiltered unfiltered, SerializationHeader header, long previousUnfilteredSize,int version)
+ public long serializedSize(Unfiltered unfiltered, SerializationHelper helper, long previousUnfilteredSize,int version)
{
return unfiltered.kind() == Unfiltered.Kind.RANGE_TOMBSTONE_MARKER
- ? serializedSize((RangeTombstoneMarker) unfiltered, header, previousUnfilteredSize, version)
- : serializedSize((Row) unfiltered, header, previousUnfilteredSize, version);
+ ? serializedSize((RangeTombstoneMarker) unfiltered, helper, previousUnfilteredSize, version)
+ : serializedSize((Row) unfiltered, helper, previousUnfilteredSize, version);
}
- private long serializedSize(Row row, SerializationHeader header, long previousUnfilteredSize, int version)
+ private long serializedSize(Row row, SerializationHelper helper, long previousUnfilteredSize, int version)
{
long size = 1; // flags
@@ -320,15 +321,16 @@
size += 1; // extended flags
if (!row.isStatic())
- size += Clustering.serializer.serializedSize(row.clustering(), version, header.clusteringTypes());
+ size += Clustering.serializer.serializedSize(row.clustering(), version, helper.header.clusteringTypes());
- return size + serializedRowBodySize(row, header, previousUnfilteredSize, version);
+ return size + serializedRowBodySize(row, helper, previousUnfilteredSize, version);
}
- private long serializedRowBodySize(Row row, SerializationHeader header, long previousUnfilteredSize, int version)
+ private long serializedRowBodySize(Row row, SerializationHelper helper, long previousUnfilteredSize, int version)
{
long size = 0;
+ SerializationHeader header = helper.header;
if (header.isForSSTable())
size += TypeSizes.sizeofUnsignedVInt(previousUnfilteredSize);
@@ -352,22 +354,19 @@
if (!hasAllColumns)
size += Columns.serializer.serializedSubsetSize(row.columns(), header.columns(isStatic));
- SearchIterator<ColumnDefinition, ColumnDefinition> si = headerColumns.iterator();
- for (ColumnData data : row)
- {
- ColumnDefinition column = si.next(data.column());
+ SearchIterator<ColumnMetadata, ColumnMetadata> si = helper.iterator(isStatic);
+ return row.accumulate((data, v) -> {
+ ColumnMetadata column = si.next(data.column());
assert column != null;
if (data.column.isSimple())
- size += Cell.serializer.serializedSize((Cell) data, column, pkLiveness, header);
+ return v + Cell.serializer.serializedSize((Cell) data, column, pkLiveness, header);
else
- size += sizeOfComplexColumn((ComplexColumnData) data, column, hasComplexDeletion, pkLiveness, header);
- }
-
- return size;
+ return v + sizeOfComplexColumn((ComplexColumnData) data, column, hasComplexDeletion, pkLiveness, header);
+ }, size);
}
- private long sizeOfComplexColumn(ComplexColumnData data, ColumnDefinition column, boolean hasComplexDeletion, LivenessInfo rowLiveness, SerializationHeader header)
+ private long sizeOfComplexColumn(ComplexColumnData data, ColumnMetadata column, boolean hasComplexDeletion, LivenessInfo rowLiveness, SerializationHeader header)
{
long size = 0;
@@ -381,12 +380,12 @@
return size;
}
- private long serializedSize(RangeTombstoneMarker marker, SerializationHeader header, long previousUnfilteredSize, int version)
+ private long serializedSize(RangeTombstoneMarker marker, SerializationHelper helper, long previousUnfilteredSize, int version)
{
- assert !header.isForSSTable();
+ assert !helper.header.isForSSTable();
return 1 // flags
- + ClusteringBoundOrBoundary.serializer.serializedSize(marker.clustering(), version, header.clusteringTypes())
- + serializedMarkerBodySize(marker, header, previousUnfilteredSize, version);
+ + ClusteringBoundOrBoundary.serializer.serializedSize(marker.clustering(), version, helper.header.clusteringTypes())
+ + serializedMarkerBodySize(marker, helper.header, previousUnfilteredSize, version);
}
private long serializedMarkerBodySize(RangeTombstoneMarker marker, SerializationHeader header, long previousUnfilteredSize, int version)
@@ -428,7 +427,7 @@
* @return the deserialized {@link Unfiltered} or {@code null} if we've read the end of a partition. This method is
* guaranteed to never return empty rows.
*/
- public Unfiltered deserialize(DataInputPlus in, SerializationHeader header, SerializationHelper helper, Row.Builder builder)
+ public Unfiltered deserialize(DataInputPlus in, SerializationHeader header, DeserializationHelper helper, Row.Builder builder)
throws IOException
{
while (true)
@@ -453,7 +452,7 @@
* But as {@link UnfilteredRowIterator} should not return empty
* rows, this mean consumer of this method should make sure to skip said empty rows.
*/
- private Unfiltered deserializeOne(DataInputPlus in, SerializationHeader header, SerializationHelper helper, Row.Builder builder)
+ private Unfiltered deserializeOne(DataInputPlus in, SerializationHeader header, DeserializationHelper helper, Row.Builder builder)
throws IOException
{
// It wouldn't be wrong per-se to use an unsorted builder, but it would be inefficient so make sure we don't do it by mistake
@@ -481,7 +480,7 @@
}
}
- public Unfiltered deserializeTombstonesOnly(FileDataInput in, SerializationHeader header, SerializationHelper helper)
+ public Unfiltered deserializeTombstonesOnly(FileDataInput in, SerializationHeader header, DeserializationHelper helper)
throws IOException
{
while (true)
@@ -533,7 +532,7 @@
}
}
- public Row deserializeStaticRow(DataInputPlus in, SerializationHeader header, SerializationHelper helper)
+ public Row deserializeStaticRow(DataInputPlus in, SerializationHeader header, DeserializationHelper helper)
throws IOException
{
int flags = in.readUnsignedByte();
@@ -561,7 +560,7 @@
public Row deserializeRowBody(DataInputPlus in,
SerializationHeader header,
- SerializationHelper helper,
+ DeserializationHelper helper,
int flags,
int extendedFlags,
Row.Builder builder)
@@ -614,7 +613,7 @@
{
throw new WrappedException(e);
}
- }, false);
+ });
}
catch (WrappedException e)
{
@@ -636,7 +635,7 @@
}
}
- private void readSimpleColumn(ColumnDefinition column, DataInputPlus in, SerializationHeader header, SerializationHelper helper, Row.Builder builder, LivenessInfo rowLiveness)
+ private void readSimpleColumn(ColumnMetadata column, DataInputPlus in, SerializationHeader header, DeserializationHelper helper, Row.Builder builder, LivenessInfo rowLiveness)
throws IOException
{
if (helper.includes(column))
@@ -651,7 +650,7 @@
}
}
- private void readComplexColumn(ColumnDefinition column, DataInputPlus in, SerializationHeader header, SerializationHelper helper, boolean hasComplexDeletion, Row.Builder builder, LivenessInfo rowLiveness)
+ private void readComplexColumn(ColumnMetadata column, DataInputPlus in, SerializationHeader header, DeserializationHelper helper, boolean hasComplexDeletion, Row.Builder builder, LivenessInfo rowLiveness)
throws IOException
{
if (helper.includes(column))
@@ -686,7 +685,7 @@
in.skipBytesFully(rowSize);
}
- public void skipStaticRow(DataInputPlus in, SerializationHeader header, SerializationHelper helper) throws IOException
+ public void skipStaticRow(DataInputPlus in, SerializationHeader header, DeserializationHelper helper) throws IOException
{
int flags = in.readUnsignedByte();
assert !isEndOfPartition(flags) && kind(flags) == Unfiltered.Kind.ROW && isExtended(flags) : "Flags is " + flags;
@@ -701,7 +700,7 @@
in.skipBytesFully(markerSize);
}
- private void skipComplexColumn(DataInputPlus in, ColumnDefinition column, SerializationHeader header, boolean hasComplexDeletion)
+ private void skipComplexColumn(DataInputPlus in, ColumnMetadata column, SerializationHeader header, boolean hasComplexDeletion)
throws IOException
{
if (hasComplexDeletion)
diff --git a/src/java/org/apache/cassandra/db/rows/WithOnlyQueriedData.java b/src/java/org/apache/cassandra/db/rows/WithOnlyQueriedData.java
index dcf0891..ecae172 100644
--- a/src/java/org/apache/cassandra/db/rows/WithOnlyQueriedData.java
+++ b/src/java/org/apache/cassandra/db/rows/WithOnlyQueriedData.java
@@ -17,7 +17,7 @@
*/
package org.apache.cassandra.db.rows;
-import org.apache.cassandra.db.PartitionColumns;
+import org.apache.cassandra.db.RegularAndStaticColumns;
import org.apache.cassandra.db.filter.ColumnFilter;
import org.apache.cassandra.db.transform.Transformation;
@@ -36,7 +36,7 @@
}
@Override
- protected PartitionColumns applyToPartitionColumns(PartitionColumns columns)
+ protected RegularAndStaticColumns applyToPartitionColumns(RegularAndStaticColumns columns)
{
return filter.queriedColumns();
}
diff --git a/src/java/org/apache/cassandra/db/rows/WrappingUnfilteredRowIterator.java b/src/java/org/apache/cassandra/db/rows/WrappingUnfilteredRowIterator.java
index 411950e..e38be09 100644
--- a/src/java/org/apache/cassandra/db/rows/WrappingUnfilteredRowIterator.java
+++ b/src/java/org/apache/cassandra/db/rows/WrappingUnfilteredRowIterator.java
@@ -19,7 +19,7 @@
import com.google.common.collect.UnmodifiableIterator;
-import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.db.*;
/**
@@ -40,12 +40,12 @@
this.wrapped = wrapped;
}
- public CFMetaData metadata()
+ public TableMetadata metadata()
{
return wrapped.metadata();
}
- public PartitionColumns columns()
+ public RegularAndStaticColumns columns()
{
return wrapped.columns();
}
diff --git a/src/java/org/apache/cassandra/db/streaming/CassandraCompressedStreamReader.java b/src/java/org/apache/cassandra/db/streaming/CassandraCompressedStreamReader.java
new file mode 100644
index 0000000..37b1a01
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/streaming/CassandraCompressedStreamReader.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db.streaming;
+
+import java.io.IOException;
+
+import com.google.common.base.Throwables;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.io.compress.CompressionMetadata;
+import org.apache.cassandra.io.sstable.SSTableMultiWriter;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.TrackedDataInputPlus;
+import org.apache.cassandra.streaming.ProgressInfo;
+import org.apache.cassandra.streaming.StreamSession;
+import org.apache.cassandra.streaming.messages.StreamMessageHeader;
+import org.apache.cassandra.utils.ChecksumType;
+import org.apache.cassandra.utils.FBUtilities;
+
+import static org.apache.cassandra.utils.Throwables.extractIOExceptionCause;
+
+/**
+ * CassandraStreamReader that reads from streamed compressed SSTable
+ */
+public class CassandraCompressedStreamReader extends CassandraStreamReader
+{
+ private static final Logger logger = LoggerFactory.getLogger(CassandraCompressedStreamReader.class);
+
+ protected final CompressionInfo compressionInfo;
+
+ public CassandraCompressedStreamReader(StreamMessageHeader header, CassandraStreamHeader streamHeader, StreamSession session)
+ {
+ super(header, streamHeader, session);
+ this.compressionInfo = streamHeader.compressionInfo;
+ }
+
+ /**
+ * @return SSTable transferred
+ * @throws java.io.IOException if reading the remote sstable fails. Will throw an RTE if local write fails.
+ */
+ @Override
+ @SuppressWarnings("resource") // input needs to remain open, streams on top of it can't be closed
+ public SSTableMultiWriter read(DataInputPlus inputPlus) throws IOException
+ {
+ long totalSize = totalSize();
+
+ ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(tableId);
+
+ if (cfs == null)
+ {
+ // schema was dropped during streaming
+ throw new IOException("CF " + tableId + " was dropped during streaming");
+ }
+
+ logger.debug("[Stream #{}] Start receiving file #{} from {}, repairedAt = {}, size = {}, ks = '{}', pendingRepair = '{}', table = '{}'.",
+ session.planId(), fileSeqNum, session.peer, repairedAt, totalSize, cfs.keyspace.getName(), pendingRepair,
+ cfs.getTableName());
+
+ StreamDeserializer deserializer = null;
+ SSTableMultiWriter writer = null;
+ try (CompressedInputStream cis = new CompressedInputStream(inputPlus, compressionInfo, ChecksumType.CRC32, cfs::getCrcCheckChance))
+ {
+ TrackedDataInputPlus in = new TrackedDataInputPlus(cis);
+ deserializer = new StreamDeserializer(cfs.metadata(), in, inputVersion, getHeader(cfs.metadata()));
+ writer = createWriter(cfs, totalSize, repairedAt, pendingRepair, format);
+ String filename = writer.getFilename();
+ int sectionIdx = 0;
+ for (SSTableReader.PartitionPositionBounds section : sections)
+ {
+ assert cis.chunkBytesRead() <= totalSize;
+ long sectionLength = section.upperPosition - section.lowerPosition;
+
+ logger.trace("[Stream #{}] Reading section {} with length {} from stream.", session.planId(), sectionIdx++, sectionLength);
+ // skip to beginning of section inside chunk
+ cis.position(section.lowerPosition);
+ in.reset(0);
+
+ while (in.getBytesRead() < sectionLength)
+ {
+ writePartition(deserializer, writer);
+ // when compressed, report total bytes of compressed chunks read since remoteFile.size is the sum of chunks transferred
+ session.progress(filename, ProgressInfo.Direction.IN, cis.chunkBytesRead(), totalSize);
+ }
+ assert in.getBytesRead() == sectionLength;
+ }
+ logger.trace("[Stream #{}] Finished receiving file #{} from {} readBytes = {}, totalSize = {}", session.planId(), fileSeqNum,
+ session.peer, FBUtilities.prettyPrintMemory(cis.chunkBytesRead()), FBUtilities.prettyPrintMemory(totalSize));
+ return writer;
+ }
+ catch (Throwable e)
+ {
+ Object partitionKey = deserializer != null ? deserializer.partitionKey() : "";
+ logger.warn("[Stream {}] Error while reading partition {} from stream on ks='{}' and table='{}'.",
+ session.planId(), partitionKey, cfs.keyspace.getName(), cfs.getTableName());
+ if (writer != null)
+ {
+ writer.abort(e);
+ }
+ if (extractIOExceptionCause(e).isPresent())
+ throw e;
+ throw Throwables.propagate(e);
+ }
+ }
+
+ @Override
+ protected long totalSize()
+ {
+ long size = 0;
+ // calculate total length of transferring chunks
+ for (CompressionMetadata.Chunk chunk : compressionInfo.chunks)
+ size += chunk.length + 4; // 4 bytes for CRC
+ return size;
+ }
+}
diff --git a/src/java/org/apache/cassandra/db/streaming/CassandraCompressedStreamWriter.java b/src/java/org/apache/cassandra/db/streaming/CassandraCompressedStreamWriter.java
new file mode 100644
index 0000000..d92314b
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/streaming/CassandraCompressedStreamWriter.java
@@ -0,0 +1,161 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db.streaming;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.io.compress.CompressionMetadata;
+import org.apache.cassandra.io.sstable.Component;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.util.ChannelProxy;
+import org.apache.cassandra.io.util.DataOutputStreamPlus;
+import org.apache.cassandra.net.AsyncStreamingOutputPlus;
+import org.apache.cassandra.streaming.ProgressInfo;
+import org.apache.cassandra.streaming.StreamSession;
+import org.apache.cassandra.utils.FBUtilities;
+
+/**
+ * CassandraStreamWriter for compressed SSTable.
+ */
+public class CassandraCompressedStreamWriter extends CassandraStreamWriter
+{
+ private static final int CHUNK_SIZE = 1 << 16;
+ private static final int CRC_LENGTH = 4;
+
+ private static final Logger logger = LoggerFactory.getLogger(CassandraCompressedStreamWriter.class);
+
+ private final CompressionInfo compressionInfo;
+
+ public CassandraCompressedStreamWriter(SSTableReader sstable, Collection<SSTableReader.PartitionPositionBounds> sections, CompressionInfo compressionInfo, StreamSession session)
+ {
+ super(sstable, sections, session);
+ this.compressionInfo = compressionInfo;
+ }
+
+ @Override
+ public void write(DataOutputStreamPlus output) throws IOException
+ {
+ AsyncStreamingOutputPlus out = (AsyncStreamingOutputPlus) output;
+ long totalSize = totalSize();
+ logger.debug("[Stream #{}] Start streaming file {} to {}, repairedAt = {}, totalSize = {}", session.planId(),
+ sstable.getFilename(), session.peer, sstable.getSSTableMetadata().repairedAt, totalSize);
+ try (ChannelProxy fc = sstable.getDataChannel().newChannel())
+ {
+ long progress = 0L;
+
+ // we want to send continuous chunks together to minimise reads from disk and network writes
+ List<Section> sections = fuseAdjacentChunks(compressionInfo.chunks);
+
+ int sectionIdx = 0;
+
+ // stream each of the required sections of the file
+ for (Section section : sections)
+ {
+ // length of the section to stream
+ long length = section.end - section.start;
+
+ logger.debug("[Stream #{}] Writing section {} with length {} to stream.", session.planId(), sectionIdx++, length);
+
+ // tracks write progress
+ long bytesTransferred = 0;
+ while (bytesTransferred < length)
+ {
+ int toTransfer = (int) Math.min(CHUNK_SIZE, length - bytesTransferred);
+ long position = section.start + bytesTransferred;
+
+ out.writeToChannel(bufferSupplier -> {
+ ByteBuffer outBuffer = bufferSupplier.get(toTransfer);
+ long read = fc.read(outBuffer, position);
+ assert read == toTransfer : String.format("could not read required number of bytes from file to be streamed: read %d bytes, wanted %d bytes", read, toTransfer);
+ outBuffer.flip();
+ }, limiter);
+
+ bytesTransferred += toTransfer;
+ progress += toTransfer;
+ session.progress(sstable.descriptor.filenameFor(Component.DATA), ProgressInfo.Direction.OUT, progress, totalSize);
+ }
+ }
+ logger.debug("[Stream #{}] Finished streaming file {} to {}, bytesTransferred = {}, totalSize = {}",
+ session.planId(), sstable.getFilename(), session.peer, FBUtilities.prettyPrintMemory(progress), FBUtilities.prettyPrintMemory(totalSize));
+ }
+ }
+
+ @Override
+ protected long totalSize()
+ {
+ long size = 0;
+ // calculate total length of transferring chunks
+ for (CompressionMetadata.Chunk chunk : compressionInfo.chunks)
+ size += chunk.length + 4; // 4 bytes for CRC
+ return size;
+ }
+
+ // chunks are assumed to be sorted by offset
+ private List<Section> fuseAdjacentChunks(CompressionMetadata.Chunk[] chunks)
+ {
+ if (chunks.length == 0)
+ return Collections.emptyList();
+
+ long start = chunks[0].offset;
+ long end = start + chunks[0].length + CRC_LENGTH;
+
+ List<Section> sections = new ArrayList<>();
+
+ for (int i = 1; i < chunks.length; i++)
+ {
+ CompressionMetadata.Chunk chunk = chunks[i];
+
+ if (chunk.offset == end)
+ {
+ end += (chunk.length + CRC_LENGTH);
+ }
+ else
+ {
+ sections.add(new Section(start, end));
+
+ start = chunk.offset;
+ end = start + chunk.length + CRC_LENGTH;
+ }
+ }
+ sections.add(new Section(start, end));
+
+ return sections;
+ }
+
+ // [start, end) positions in the compressed sstable file that we want to stream;
+ // each section contains 1..n adjacent compressed chunks in it.
+ private static class Section
+ {
+ private final long start;
+ private final long end;
+
+ private Section(long start, long end)
+ {
+ this.start = start;
+ this.end = end;
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/db/streaming/CassandraEntireSSTableStreamReader.java b/src/java/org/apache/cassandra/db/streaming/CassandraEntireSSTableStreamReader.java
new file mode 100644
index 0000000..eac37d1
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/streaming/CassandraEntireSSTableStreamReader.java
@@ -0,0 +1,183 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.streaming;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Collection;
+
+import com.google.common.base.Function;
+import com.google.common.base.Throwables;
+import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Directories;
+import org.apache.cassandra.io.sstable.Component;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.SSTableMultiWriter;
+import org.apache.cassandra.io.sstable.format.SSTableFormat;
+import org.apache.cassandra.io.sstable.format.big.BigTableZeroCopyWriter;
+import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.streaming.ProgressInfo;
+import org.apache.cassandra.streaming.StreamReceiver;
+import org.apache.cassandra.streaming.StreamSession;
+import org.apache.cassandra.streaming.messages.StreamMessageHeader;
+
+import static java.lang.String.format;
+import static org.apache.cassandra.utils.FBUtilities.prettyPrintMemory;
+
+/**
+ * CassandraEntireSSTableStreamReader reads SSTable off the wire and writes it to disk.
+ */
+public class CassandraEntireSSTableStreamReader implements IStreamReader
+{
+ private static final Logger logger = LoggerFactory.getLogger(CassandraEntireSSTableStreamReader.class);
+
+ private final TableId tableId;
+ private final StreamSession session;
+ private final StreamMessageHeader messageHeader;
+ private final CassandraStreamHeader header;
+ private final int fileSequenceNumber;
+
+ public CassandraEntireSSTableStreamReader(StreamMessageHeader messageHeader, CassandraStreamHeader streamHeader, StreamSession session)
+ {
+ if (streamHeader.format != SSTableFormat.Type.BIG)
+ throw new AssertionError("Unsupported SSTable format " + streamHeader.format);
+
+ if (session.getPendingRepair() != null)
+ {
+ // we should only ever be streaming pending repair sstables if the session has a pending repair id
+ if (!session.getPendingRepair().equals(messageHeader.pendingRepair))
+ throw new IllegalStateException(format("Stream Session & SSTable (%s) pendingRepair UUID mismatch.", messageHeader.tableId));
+ }
+
+ this.header = streamHeader;
+ this.session = session;
+ this.messageHeader = messageHeader;
+ this.tableId = messageHeader.tableId;
+ this.fileSequenceNumber = messageHeader.sequenceNumber;
+ }
+
+ /**
+ * @param in where this reads data from
+ * @return SSTable transferred
+ * @throws IOException if reading the remote sstable fails. Will throw an RTE if local write fails.
+ */
+ @SuppressWarnings("resource") // input needs to remain open, streams on top of it can't be closed
+ @Override
+ public SSTableMultiWriter read(DataInputPlus in) throws IOException
+ {
+ ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(tableId);
+ if (cfs == null)
+ {
+ // schema was dropped during streaming
+ throw new IOException("Table " + tableId + " was dropped during streaming");
+ }
+
+ ComponentManifest manifest = header.componentManifest;
+ long totalSize = manifest.totalSize();
+
+ logger.debug("[Stream #{}] Started receiving sstable #{} from {}, size = {}, table = {}",
+ session.planId(),
+ fileSequenceNumber,
+ session.peer,
+ prettyPrintMemory(totalSize),
+ cfs.metadata());
+
+ BigTableZeroCopyWriter writer = null;
+
+ try
+ {
+ writer = createWriter(cfs, totalSize, manifest.components());
+ long bytesRead = 0;
+ for (Component component : manifest.components())
+ {
+ long length = manifest.sizeOf(component);
+
+ logger.debug("[Stream #{}] Started receiving {} component from {}, componentSize = {}, readBytes = {}, totalSize = {}",
+ session.planId(),
+ component,
+ session.peer,
+ prettyPrintMemory(length),
+ prettyPrintMemory(bytesRead),
+ prettyPrintMemory(totalSize));
+
+ writer.writeComponent(component.type, in, length);
+ session.progress(writer.descriptor.filenameFor(component), ProgressInfo.Direction.IN, length, length);
+ bytesRead += length;
+
+ logger.debug("[Stream #{}] Finished receiving {} component from {}, componentSize = {}, readBytes = {}, totalSize = {}",
+ session.planId(),
+ component,
+ session.peer,
+ prettyPrintMemory(length),
+ prettyPrintMemory(bytesRead),
+ prettyPrintMemory(totalSize));
+ }
+
+ Function<StatsMetadata, StatsMetadata> transform = stats -> stats.mutateLevel(header.sstableLevel)
+ .mutateRepairedMetadata(messageHeader.repairedAt, messageHeader.pendingRepair, false);
+ writer.descriptor.getMetadataSerializer().mutate(writer.descriptor, transform);
+ return writer;
+ }
+ catch (Throwable e)
+ {
+ logger.error("[Stream {}] Error while reading sstable from stream for table = {}", session.planId(), cfs.metadata(), e);
+ if (writer != null)
+ e = writer.abort(e);
+ Throwables.throwIfUnchecked(e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ private File getDataDir(ColumnFamilyStore cfs, long totalSize) throws IOException
+ {
+ Directories.DataDirectory localDir = cfs.getDirectories().getWriteableLocation(totalSize);
+ if (localDir == null)
+ throw new IOException(format("Insufficient disk space to store %s", prettyPrintMemory(totalSize)));
+
+ File dir = cfs.getDirectories().getLocationForDisk(cfs.getDiskBoundaries().getCorrectDiskForKey(header.firstKey));
+
+ if (dir == null)
+ return cfs.getDirectories().getDirectoryForNewSSTables();
+
+ return dir;
+ }
+
+ @SuppressWarnings("resource")
+ protected BigTableZeroCopyWriter createWriter(ColumnFamilyStore cfs, long totalSize, Collection<Component> components) throws IOException
+ {
+ File dataDir = getDataDir(cfs, totalSize);
+
+ StreamReceiver streamReceiver = session.getAggregator(tableId);
+ assert streamReceiver instanceof CassandraStreamReceiver;
+
+ LifecycleNewTracker lifecycleNewTracker = CassandraStreamReceiver.fromReceiver(session.getAggregator(tableId)).createLifecycleNewTracker();
+
+ Descriptor desc = cfs.newSSTableDescriptor(dataDir, header.version, header.format);
+
+ logger.debug("[Table #{}] {} Components to write: {}", cfs.metadata(), desc.filenameFor(Component.DATA), components);
+
+ return new BigTableZeroCopyWriter(desc, cfs.metadata, lifecycleNewTracker, components);
+ }
+}
diff --git a/src/java/org/apache/cassandra/db/streaming/CassandraEntireSSTableStreamWriter.java b/src/java/org/apache/cassandra/db/streaming/CassandraEntireSSTableStreamWriter.java
new file mode 100644
index 0000000..401b20e
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/streaming/CassandraEntireSSTableStreamWriter.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.streaming;
+
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.nio.channels.FileChannel;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.io.sstable.Component;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.net.AsyncStreamingOutputPlus;
+import org.apache.cassandra.streaming.ProgressInfo;
+import org.apache.cassandra.streaming.StreamManager;
+import org.apache.cassandra.streaming.StreamSession;
+
+import static org.apache.cassandra.streaming.StreamManager.StreamRateLimiter;
+import static org.apache.cassandra.utils.FBUtilities.prettyPrintMemory;
+
+/**
+ * CassandraEntireSSTableStreamWriter streams the entire SSTable to given channel.
+ */
+public class CassandraEntireSSTableStreamWriter
+{
+ private static final Logger logger = LoggerFactory.getLogger(CassandraEntireSSTableStreamWriter.class);
+
+ private final SSTableReader sstable;
+ private final ComponentManifest manifest;
+ private final StreamSession session;
+ private final StreamRateLimiter limiter;
+
+ public CassandraEntireSSTableStreamWriter(SSTableReader sstable, StreamSession session, ComponentManifest manifest)
+ {
+ this.session = session;
+ this.sstable = sstable;
+ this.manifest = manifest;
+ this.limiter = StreamManager.getRateLimiter(session.peer);
+ }
+
+ /**
+ * Stream the entire file to given channel.
+ * <p>
+ * TODO: this currently requires a companion thread, but could be performed entirely asynchronously
+ * @param out where this writes data to
+ * @throws IOException on any I/O error
+ */
+ public void write(AsyncStreamingOutputPlus out) throws IOException
+ {
+ long totalSize = manifest.totalSize();
+ logger.debug("[Stream #{}] Start streaming sstable {} to {}, repairedAt = {}, totalSize = {}",
+ session.planId(),
+ sstable.getFilename(),
+ session.peer,
+ sstable.getSSTableMetadata().repairedAt,
+ prettyPrintMemory(totalSize));
+
+ long progress = 0L;
+
+ for (Component component : manifest.components())
+ {
+ @SuppressWarnings("resource") // this is closed after the file is transferred by AsyncChannelOutputPlus
+ FileChannel in = new RandomAccessFile(sstable.descriptor.filenameFor(component), "r").getChannel();
+
+ // Total Length to transmit for this file
+ long length = in.size();
+
+ // tracks write progress
+ logger.debug("[Stream #{}] Streaming {}.{} gen {} component {} size {}", session.planId(),
+ sstable.getKeyspaceName(),
+ sstable.getColumnFamilyName(),
+ sstable.descriptor.generation,
+ component,
+ prettyPrintMemory(length));
+
+ long bytesWritten = out.writeFileToChannel(in, limiter);
+ progress += bytesWritten;
+
+ session.progress(sstable.descriptor.filenameFor(component), ProgressInfo.Direction.OUT, bytesWritten, length);
+
+ logger.debug("[Stream #{}] Finished streaming {}.{} gen {} component {} to {}, xfered = {}, length = {}, totalSize = {}",
+ session.planId(),
+ sstable.getKeyspaceName(),
+ sstable.getColumnFamilyName(),
+ sstable.descriptor.generation,
+ component,
+ session.peer,
+ prettyPrintMemory(bytesWritten),
+ prettyPrintMemory(length),
+ prettyPrintMemory(totalSize));
+ }
+
+ out.flush();
+
+ logger.debug("[Stream #{}] Finished streaming sstable {} to {}, xfered = {}, totalSize = {}",
+ session.planId(),
+ sstable.getFilename(),
+ session.peer,
+ prettyPrintMemory(progress),
+ prettyPrintMemory(totalSize));
+
+ }
+}
diff --git a/src/java/org/apache/cassandra/db/streaming/CassandraIncomingFile.java b/src/java/org/apache/cassandra/db/streaming/CassandraIncomingFile.java
new file mode 100644
index 0000000..11a18a0
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/streaming/CassandraIncomingFile.java
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.streaming;
+
+import java.io.IOException;
+import java.util.Objects;
+
+import com.google.common.base.Preconditions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.io.sstable.SSTableMultiWriter;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.streaming.IncomingStream;
+import org.apache.cassandra.streaming.StreamSession;
+import org.apache.cassandra.streaming.messages.StreamMessageHeader;
+
+/**
+ * used to receive the part(or whole) of a SSTable data file.
+ *
+ * This class deserializes the data stream into partitions and rows, and writes that out as an sstable
+ */
+public class CassandraIncomingFile implements IncomingStream
+{
+ private final ColumnFamilyStore cfs;
+ private final StreamSession session;
+ private final StreamMessageHeader header;
+
+ private volatile SSTableMultiWriter sstable;
+ private volatile long size = -1;
+ private volatile int numFiles = 1;
+
+ private static final Logger logger = LoggerFactory.getLogger(CassandraIncomingFile.class);
+
+ public CassandraIncomingFile(ColumnFamilyStore cfs, StreamSession session, StreamMessageHeader header)
+ {
+ this.cfs = cfs;
+ this.session = session;
+ this.header = header;
+ }
+
+ @Override
+ public StreamSession session()
+ {
+ return session;
+ }
+
+ @Override
+ public synchronized void read(DataInputPlus in, int version) throws IOException
+ {
+ CassandraStreamHeader streamHeader = CassandraStreamHeader.serializer.deserialize(in, version);
+ logger.debug("Incoming stream entireSSTable={} components={}", streamHeader.isEntireSSTable, streamHeader.componentManifest);
+
+ IStreamReader reader;
+ if (streamHeader.isEntireSSTable)
+ {
+ reader = new CassandraEntireSSTableStreamReader(header, streamHeader, session);
+ numFiles = streamHeader.componentManifest.components().size();
+ }
+ else if (streamHeader.isCompressed())
+ reader = new CassandraCompressedStreamReader(header, streamHeader, session);
+ else
+ reader = new CassandraStreamReader(header, streamHeader, session);
+
+ size = streamHeader.size();
+ sstable = reader.read(in);
+ }
+
+ @Override
+ public synchronized String getName()
+ {
+ return sstable == null ? "null" : sstable.getFilename();
+ }
+
+ @Override
+ public synchronized long getSize()
+ {
+ Preconditions.checkState(size > 0, "Stream hasn't been read yet");
+ return size;
+ }
+
+ @Override
+ public int getNumFiles()
+ {
+ return numFiles;
+ }
+
+ @Override
+ public TableId getTableId()
+ {
+ Preconditions.checkState(sstable != null, "Stream hasn't been read yet");
+ return sstable.getTableId();
+ }
+
+ @Override
+ public String toString()
+ {
+ SSTableMultiWriter sst = sstable;
+ return "CassandraIncomingFile{" +
+ "sstable=" + (sst == null ? "null" : sst.getFilename()) +
+ '}';
+ }
+
+ public SSTableMultiWriter getSSTable()
+ {
+ Preconditions.checkState(sstable != null, "Stream hasn't been read yet");
+ return sstable;
+ }
+
+ public boolean equals(Object o)
+ {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ CassandraIncomingFile that = (CassandraIncomingFile) o;
+ return Objects.equals(cfs, that.cfs) &&
+ Objects.equals(session, that.session) &&
+ Objects.equals(header, that.header) &&
+ Objects.equals(sstable, that.sstable);
+ }
+
+ public int hashCode()
+ {
+
+ return Objects.hash(cfs, session, header, sstable);
+ }
+}
diff --git a/src/java/org/apache/cassandra/db/streaming/CassandraOutgoingFile.java b/src/java/org/apache/cassandra/db/streaming/CassandraOutgoingFile.java
new file mode 100644
index 0000000..0917fba
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/streaming/CassandraOutgoingFile.java
@@ -0,0 +1,231 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.streaming;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Objects;
+import java.util.UUID;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.io.sstable.Component;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.util.DataOutputStreamPlus;
+import org.apache.cassandra.net.AsyncStreamingOutputPlus;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.streaming.OutgoingStream;
+import org.apache.cassandra.streaming.StreamOperation;
+import org.apache.cassandra.streaming.StreamSession;
+import org.apache.cassandra.utils.concurrent.Ref;
+
+/**
+ * used to transfer the part(or whole) of a SSTable data file
+ */
+public class CassandraOutgoingFile implements OutgoingStream
+{
+ public static final List<Component> STREAM_COMPONENTS = ImmutableList.of(Component.DATA, Component.PRIMARY_INDEX, Component.STATS,
+ Component.COMPRESSION_INFO, Component.FILTER, Component.SUMMARY,
+ Component.DIGEST, Component.CRC);
+
+ private final Ref<SSTableReader> ref;
+ private final long estimatedKeys;
+ private final List<SSTableReader.PartitionPositionBounds> sections;
+ private final String filename;
+ private final CassandraStreamHeader header;
+ private final boolean keepSSTableLevel;
+ private final ComponentManifest manifest;
+
+ private final boolean shouldStreamEntireSSTable;
+
+ public CassandraOutgoingFile(StreamOperation operation, Ref<SSTableReader> ref,
+ List<SSTableReader.PartitionPositionBounds> sections, List<Range<Token>> normalizedRanges,
+ long estimatedKeys)
+ {
+ Preconditions.checkNotNull(ref.get());
+ Range.assertNormalized(normalizedRanges);
+ this.ref = ref;
+ this.estimatedKeys = estimatedKeys;
+ this.sections = sections;
+ this.filename = ref.get().getFilename();
+ this.manifest = getComponentManifest(ref.get());
+ this.shouldStreamEntireSSTable = computeShouldStreamEntireSSTables();
+
+ SSTableReader sstable = ref.get();
+ keepSSTableLevel = operation == StreamOperation.BOOTSTRAP || operation == StreamOperation.REBUILD;
+ this.header =
+ CassandraStreamHeader.builder()
+ .withSSTableFormat(sstable.descriptor.formatType)
+ .withSSTableVersion(sstable.descriptor.version)
+ .withSSTableLevel(keepSSTableLevel ? sstable.getSSTableLevel() : 0)
+ .withEstimatedKeys(estimatedKeys)
+ .withSections(sections)
+ .withCompressionMetadata(sstable.compression ? sstable.getCompressionMetadata() : null)
+ .withSerializationHeader(sstable.header.toComponent())
+ .isEntireSSTable(shouldStreamEntireSSTable)
+ .withComponentManifest(manifest)
+ .withFirstKey(sstable.first)
+ .withTableId(sstable.metadata().id)
+ .build();
+ }
+
+ @VisibleForTesting
+ public static ComponentManifest getComponentManifest(SSTableReader sstable)
+ {
+ LinkedHashMap<Component, Long> components = new LinkedHashMap<>(STREAM_COMPONENTS.size());
+ for (Component component : STREAM_COMPONENTS)
+ {
+ File file = new File(sstable.descriptor.filenameFor(component));
+ if (file.exists())
+ components.put(component, file.length());
+ }
+
+ return new ComponentManifest(components);
+ }
+
+ public static CassandraOutgoingFile fromStream(OutgoingStream stream)
+ {
+ Preconditions.checkArgument(stream instanceof CassandraOutgoingFile);
+ return (CassandraOutgoingFile) stream;
+ }
+
+ @VisibleForTesting
+ public Ref<SSTableReader> getRef()
+ {
+ return ref;
+ }
+
+ @Override
+ public String getName()
+ {
+ return filename;
+ }
+
+ @Override
+ public long getSize()
+ {
+ return header.size();
+ }
+
+ @Override
+ public TableId getTableId()
+ {
+ return ref.get().metadata().id;
+ }
+
+ @Override
+ public int getNumFiles()
+ {
+ return shouldStreamEntireSSTable ? getManifestSize() : 1;
+ }
+
+ @Override
+ public long getRepairedAt()
+ {
+ return ref.get().getRepairedAt();
+ }
+
+ @Override
+ public UUID getPendingRepair()
+ {
+ return ref.get().getPendingRepair();
+ }
+
+ public int getManifestSize()
+ {
+ return manifest.components().size();
+ }
+
+ @Override
+ public void write(StreamSession session, DataOutputStreamPlus out, int version) throws IOException
+ {
+ SSTableReader sstable = ref.get();
+ CassandraStreamHeader.serializer.serialize(header, out, version);
+ out.flush();
+
+ if (shouldStreamEntireSSTable && out instanceof AsyncStreamingOutputPlus)
+ {
+ CassandraEntireSSTableStreamWriter writer = new CassandraEntireSSTableStreamWriter(sstable, session, manifest);
+ writer.write((AsyncStreamingOutputPlus) out);
+ }
+ else
+ {
+ CassandraStreamWriter writer = (header.compressionInfo == null) ?
+ new CassandraStreamWriter(sstable, header.sections, session) :
+ new CassandraCompressedStreamWriter(sstable, header.sections,
+ header.compressionInfo, session);
+ writer.write(out);
+ }
+ }
+
+ @VisibleForTesting
+ public boolean computeShouldStreamEntireSSTables()
+ {
+ // don't stream if full sstable transfers are disabled or legacy counter shards are present
+ if (!DatabaseDescriptor.streamEntireSSTables() || ref.get().getSSTableMetadata().hasLegacyCounterShards)
+ return false;
+
+ return contained(sections, ref.get());
+ }
+
+ @VisibleForTesting
+ public boolean contained(List<SSTableReader.PartitionPositionBounds> sections, SSTableReader sstable)
+ {
+ if (sections == null || sections.isEmpty())
+ return false;
+
+ // if transfer sections contain entire sstable
+ long transferLength = sections.stream().mapToLong(p -> p.upperPosition - p.lowerPosition).sum();
+ return transferLength == sstable.uncompressedLength();
+ }
+
+ @Override
+ public void finish()
+ {
+ ref.release();
+ }
+
+ public boolean equals(Object o)
+ {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ CassandraOutgoingFile that = (CassandraOutgoingFile) o;
+ return estimatedKeys == that.estimatedKeys &&
+ Objects.equals(ref, that.ref) &&
+ Objects.equals(sections, that.sections);
+ }
+
+ public int hashCode()
+ {
+ return Objects.hash(ref, estimatedKeys, sections);
+ }
+
+ @Override
+ public String toString()
+ {
+ return "CassandraOutgoingFile{" + filename + '}';
+ }
+}
diff --git a/src/java/org/apache/cassandra/db/streaming/CassandraStreamHeader.java b/src/java/org/apache/cassandra/db/streaming/CassandraStreamHeader.java
new file mode 100644
index 0000000..2af56de
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/streaming/CassandraStreamHeader.java
@@ -0,0 +1,400 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db.streaming;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+import java.util.function.Function;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.SerializationHeader;
+import org.apache.cassandra.db.TypeSizes;
+import org.apache.cassandra.dht.IPartitioner;
+import org.apache.cassandra.io.IVersionedSerializer;
+import org.apache.cassandra.io.compress.CompressionMetadata;
+import org.apache.cassandra.io.sstable.format.SSTableFormat;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.format.Version;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.utils.ByteBufferUtil;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+public class CassandraStreamHeader
+{
+ /** SSTable version */
+ public final Version version;
+
+ /** SSTable format **/
+ public final SSTableFormat.Type format;
+ public final long estimatedKeys;
+ public final List<SSTableReader.PartitionPositionBounds> sections;
+ /**
+ * Compression info for SSTable to send. Can be null if SSTable is not compressed.
+ * On sender, this field is always null to avoid holding large number of Chunks.
+ * Use compressionMetadata instead.
+ */
+ private final CompressionMetadata compressionMetadata;
+ public volatile CompressionInfo compressionInfo;
+ public final int sstableLevel;
+ public final SerializationHeader.Component serializationHeader;
+
+ /* flag indicating whether this is a partial or entire sstable transfer */
+ public final boolean isEntireSSTable;
+ /* first token of the sstable required for faster streaming */
+ public final DecoratedKey firstKey;
+ public final TableId tableId;
+ public final ComponentManifest componentManifest;
+
+ /* cached size value */
+ private transient final long size;
+
+ private CassandraStreamHeader(Builder builder)
+ {
+ version = builder.version;
+ format = builder.format;
+ estimatedKeys = builder.estimatedKeys;
+ sections = builder.sections;
+ compressionMetadata = builder.compressionMetadata;
+ compressionInfo = builder.compressionInfo;
+ sstableLevel = builder.sstableLevel;
+ serializationHeader = builder.serializationHeader;
+ tableId = builder.tableId;
+ isEntireSSTable = builder.isEntireSSTable;
+ componentManifest = builder.componentManifest;
+ firstKey = builder.firstKey;
+ size = calculateSize();
+ }
+
+ public static Builder builder()
+ {
+ return new Builder();
+ }
+
+ public boolean isCompressed()
+ {
+ return compressionInfo != null;
+ }
+
+ /**
+ * @return total file size to transfer in bytes
+ */
+ public long size()
+ {
+ return size;
+ }
+
+ private long calculateSize()
+ {
+ if (isEntireSSTable)
+ return componentManifest.totalSize();
+
+ long transferSize = 0;
+ if (compressionInfo != null)
+ {
+ // calculate total length of transferring chunks
+ for (CompressionMetadata.Chunk chunk : compressionInfo.chunks)
+ transferSize += chunk.length + 4; // 4 bytes for CRC
+ }
+ else
+ {
+ for (SSTableReader.PartitionPositionBounds section : sections)
+ transferSize += section.upperPosition - section.lowerPosition;
+ }
+ return transferSize;
+ }
+
+ public synchronized void calculateCompressionInfo()
+ {
+ if (compressionMetadata != null && compressionInfo == null)
+ compressionInfo = CompressionInfo.fromCompressionMetadata(compressionMetadata, sections);
+ }
+
+ @Override
+ public String toString()
+ {
+ return "CassandraStreamHeader{" +
+ "version=" + version +
+ ", format=" + format +
+ ", estimatedKeys=" + estimatedKeys +
+ ", sections=" + sections +
+ ", sstableLevel=" + sstableLevel +
+ ", header=" + serializationHeader +
+ ", isEntireSSTable=" + isEntireSSTable +
+ ", firstKey=" + firstKey +
+ ", tableId=" + tableId +
+ '}';
+ }
+
+ public boolean equals(Object o)
+ {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ CassandraStreamHeader that = (CassandraStreamHeader) o;
+ return estimatedKeys == that.estimatedKeys &&
+ sstableLevel == that.sstableLevel &&
+ isEntireSSTable == that.isEntireSSTable &&
+ Objects.equals(version, that.version) &&
+ format == that.format &&
+ Objects.equals(sections, that.sections) &&
+ Objects.equals(compressionInfo, that.compressionInfo) &&
+ Objects.equals(serializationHeader, that.serializationHeader) &&
+ Objects.equals(componentManifest, that.componentManifest) &&
+ Objects.equals(firstKey, that.firstKey) &&
+ Objects.equals(tableId, that.tableId);
+ }
+
+ public int hashCode()
+ {
+ return Objects.hash(version, format, estimatedKeys, sections, compressionInfo, sstableLevel, serializationHeader, componentManifest,
+ isEntireSSTable, firstKey, tableId);
+ }
+
+ public static final IVersionedSerializer<CassandraStreamHeader> serializer = new CassandraStreamHeaderSerializer();
+
+ public static class CassandraStreamHeaderSerializer implements IVersionedSerializer<CassandraStreamHeader>
+ {
+ public void serialize(CassandraStreamHeader header, DataOutputPlus out, int version) throws IOException
+ {
+ out.writeUTF(header.version.toString());
+ out.writeUTF(header.format.name);
+
+ out.writeLong(header.estimatedKeys);
+ out.writeInt(header.sections.size());
+ for (SSTableReader.PartitionPositionBounds section : header.sections)
+ {
+ out.writeLong(section.lowerPosition);
+ out.writeLong(section.upperPosition);
+ }
+ header.calculateCompressionInfo();
+ CompressionInfo.serializer.serialize(header.compressionInfo, out, version);
+ out.writeInt(header.sstableLevel);
+
+ SerializationHeader.serializer.serialize(header.version, header.serializationHeader, out);
+
+ header.tableId.serialize(out);
+ out.writeBoolean(header.isEntireSSTable);
+
+ if (header.isEntireSSTable)
+ {
+ ComponentManifest.serializer.serialize(header.componentManifest, out, version);
+ ByteBufferUtil.writeWithVIntLength(header.firstKey.getKey(), out);
+ }
+ }
+
+ public CassandraStreamHeader deserialize(DataInputPlus in, int version) throws IOException
+ {
+ return deserialize(in, version, tableId -> {
+ ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(tableId);
+ if (cfs != null)
+ return cfs.getPartitioner();
+
+ return null;
+ });
+ }
+
+ @VisibleForTesting
+ public CassandraStreamHeader deserialize(DataInputPlus in, int version, Function<TableId, IPartitioner> partitionerMapper) throws IOException
+ {
+ Version sstableVersion = SSTableFormat.Type.current().info.getVersion(in.readUTF());
+ SSTableFormat.Type format = SSTableFormat.Type.validate(in.readUTF());
+
+ long estimatedKeys = in.readLong();
+ int count = in.readInt();
+ List<SSTableReader.PartitionPositionBounds> sections = new ArrayList<>(count);
+ for (int k = 0; k < count; k++)
+ sections.add(new SSTableReader.PartitionPositionBounds(in.readLong(), in.readLong()));
+ CompressionInfo compressionInfo = CompressionInfo.serializer.deserialize(in, version);
+ int sstableLevel = in.readInt();
+
+ SerializationHeader.Component header = SerializationHeader.serializer.deserialize(sstableVersion, in);
+
+ TableId tableId = TableId.deserialize(in);
+ boolean isEntireSSTable = in.readBoolean();
+ ComponentManifest manifest = null;
+ DecoratedKey firstKey = null;
+
+ if (isEntireSSTable)
+ {
+ manifest = ComponentManifest.serializer.deserialize(in, version);
+ ByteBuffer keyBuf = ByteBufferUtil.readWithVIntLength(in);
+ IPartitioner partitioner = partitionerMapper.apply(tableId);
+ if (partitioner == null)
+ throw new IllegalArgumentException(String.format("Could not determine partitioner for tableId %s", tableId));
+ firstKey = partitioner.decorateKey(keyBuf);
+ }
+
+ return builder().withSSTableFormat(format)
+ .withSSTableVersion(sstableVersion)
+ .withSSTableLevel(sstableLevel)
+ .withEstimatedKeys(estimatedKeys)
+ .withSections(sections)
+ .withCompressionInfo(compressionInfo)
+ .withSerializationHeader(header)
+ .withComponentManifest(manifest)
+ .isEntireSSTable(isEntireSSTable)
+ .withFirstKey(firstKey)
+ .withTableId(tableId)
+ .build();
+ }
+
+ public long serializedSize(CassandraStreamHeader header, int version)
+ {
+ long size = 0;
+ size += TypeSizes.sizeof(header.version.toString());
+ size += TypeSizes.sizeof(header.format.name);
+ size += TypeSizes.sizeof(header.estimatedKeys);
+
+ size += TypeSizes.sizeof(header.sections.size());
+ for (SSTableReader.PartitionPositionBounds section : header.sections)
+ {
+ size += TypeSizes.sizeof(section.lowerPosition);
+ size += TypeSizes.sizeof(section.upperPosition);
+ }
+
+ header.calculateCompressionInfo();
+ size += CompressionInfo.serializer.serializedSize(header.compressionInfo, version);
+ size += TypeSizes.sizeof(header.sstableLevel);
+
+ size += SerializationHeader.serializer.serializedSize(header.version, header.serializationHeader);
+
+ size += header.tableId.serializedSize();
+ size += TypeSizes.sizeof(header.isEntireSSTable);
+
+ if (header.isEntireSSTable)
+ {
+ size += ComponentManifest.serializer.serializedSize(header.componentManifest, version);
+ size += ByteBufferUtil.serializedSizeWithVIntLength(header.firstKey.getKey());
+ }
+ return size;
+ }
+ }
+
+ public static final class Builder
+ {
+ private Version version;
+ private SSTableFormat.Type format;
+ private long estimatedKeys;
+ private List<SSTableReader.PartitionPositionBounds> sections;
+ private CompressionMetadata compressionMetadata;
+ private CompressionInfo compressionInfo;
+ private int sstableLevel;
+ private SerializationHeader.Component serializationHeader;
+ private ComponentManifest componentManifest;
+ private boolean isEntireSSTable;
+ private DecoratedKey firstKey;
+ private TableId tableId;
+
+ public Builder withSSTableFormat(SSTableFormat.Type format)
+ {
+ this.format = format;
+ return this;
+ }
+
+ public Builder withSSTableVersion(Version version)
+ {
+ this.version = version;
+ return this;
+ }
+
+ public Builder withSSTableLevel(int sstableLevel)
+ {
+ this.sstableLevel = sstableLevel;
+ return this;
+ }
+
+ public Builder withEstimatedKeys(long estimatedKeys)
+ {
+ this.estimatedKeys = estimatedKeys;
+ return this;
+ }
+
+ public Builder withSections(List<SSTableReader.PartitionPositionBounds> sections)
+ {
+ this.sections = sections;
+ return this;
+ }
+
+ public Builder withCompressionMetadata(CompressionMetadata compressionMetadata)
+ {
+ this.compressionMetadata = compressionMetadata;
+ return this;
+ }
+
+ public Builder withCompressionInfo(CompressionInfo compressionInfo)
+ {
+ this.compressionInfo = compressionInfo;
+ return this;
+ }
+
+ public Builder withSerializationHeader(SerializationHeader.Component header)
+ {
+ this.serializationHeader = header;
+ return this;
+ }
+
+ public Builder withTableId(TableId tableId)
+ {
+ this.tableId = tableId;
+ return this;
+ }
+
+ public Builder isEntireSSTable(boolean isEntireSSTable)
+ {
+ this.isEntireSSTable = isEntireSSTable;
+ return this;
+ }
+
+ public Builder withComponentManifest(ComponentManifest componentManifest)
+ {
+ this.componentManifest = componentManifest;
+ return this;
+ }
+
+ public Builder withFirstKey(DecoratedKey firstKey)
+ {
+ this.firstKey = firstKey;
+ return this;
+ }
+
+ public CassandraStreamHeader build()
+ {
+ checkNotNull(version);
+ checkNotNull(format);
+ checkNotNull(sections);
+ checkNotNull(serializationHeader);
+ checkNotNull(tableId);
+
+ if (isEntireSSTable)
+ {
+ checkNotNull(componentManifest);
+ checkNotNull(firstKey);
+ }
+
+ return new CassandraStreamHeader(this);
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/db/streaming/CassandraStreamManager.java b/src/java/org/apache/cassandra/db/streaming/CassandraStreamManager.java
new file mode 100644
index 0000000..a84fd27
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/streaming/CassandraStreamManager.java
@@ -0,0 +1,157 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.streaming;
+
+import com.google.common.base.Predicate;
+import com.google.common.base.Predicates;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Sets;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.db.lifecycle.SSTableIntervalTree;
+import org.apache.cassandra.db.lifecycle.SSTableSet;
+import org.apache.cassandra.db.lifecycle.View;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.locator.RangesAtEndpoint;
+import org.apache.cassandra.locator.Replica;
+import org.apache.cassandra.service.ActiveRepairService;
+import org.apache.cassandra.streaming.IncomingStream;
+import org.apache.cassandra.streaming.OutgoingStream;
+import org.apache.cassandra.streaming.PreviewKind;
+import org.apache.cassandra.streaming.StreamReceiver;
+import org.apache.cassandra.streaming.StreamSession;
+import org.apache.cassandra.streaming.TableStreamManager;
+import org.apache.cassandra.streaming.messages.StreamMessageHeader;
+import org.apache.cassandra.utils.concurrent.Ref;
+import org.apache.cassandra.utils.concurrent.Refs;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Set;
+import java.util.UUID;
+
+/**
+ * Implements the streaming interface for the native cassandra storage engine.
+ *
+ * Handles the streaming a one or more section of one of more sstables to and from a specific
+ * remote node. The sending side performs a block-level transfer of the source stream, while the receiver
+ * must deserilaize that data stream into an partitions and rows, and then write that out as an sstable.
+ */
+public class CassandraStreamManager implements TableStreamManager
+{
+ private static final Logger logger = LoggerFactory.getLogger(CassandraStreamManager.class);
+
+ private final ColumnFamilyStore cfs;
+
+ public CassandraStreamManager(ColumnFamilyStore cfs)
+ {
+ this.cfs = cfs;
+ }
+
+ @Override
+ public IncomingStream prepareIncomingStream(StreamSession session, StreamMessageHeader header)
+ {
+ return new CassandraIncomingFile(cfs, session, header);
+ }
+
+ @Override
+ public StreamReceiver createStreamReceiver(StreamSession session, int totalStreams)
+ {
+ return new CassandraStreamReceiver(cfs, session, totalStreams);
+ }
+
+ @Override
+ public Collection<OutgoingStream> createOutgoingStreams(StreamSession session, RangesAtEndpoint replicas, UUID pendingRepair, PreviewKind previewKind)
+ {
+ Refs<SSTableReader> refs = new Refs<>();
+ try
+ {
+ final List<Range<PartitionPosition>> keyRanges = new ArrayList<>(replicas.size());
+ for (Replica replica : replicas)
+ keyRanges.add(Range.makeRowRange(replica.range()));
+ refs.addAll(cfs.selectAndReference(view -> {
+ Set<SSTableReader> sstables = Sets.newHashSet();
+ SSTableIntervalTree intervalTree = SSTableIntervalTree.build(view.select(SSTableSet.CANONICAL));
+ Predicate<SSTableReader> predicate;
+ if (previewKind.isPreview())
+ {
+ predicate = previewKind.predicate();
+ }
+ else if (pendingRepair == ActiveRepairService.NO_PENDING_REPAIR)
+ {
+ predicate = Predicates.alwaysTrue();
+ }
+ else
+ {
+ predicate = s -> s.isPendingRepair() && s.getSSTableMetadata().pendingRepair.equals(pendingRepair);
+ }
+
+ for (Range<PartitionPosition> keyRange : keyRanges)
+ {
+ // keyRange excludes its start, while sstableInBounds is inclusive (of both start and end).
+ // This is fine however, because keyRange has been created from a token range through Range.makeRowRange (see above).
+ // And that later method uses the Token.maxKeyBound() method to creates the range, which return a "fake" key that
+ // sort after all keys having the token. That "fake" key cannot however be equal to any real key, so that even
+ // including keyRange.left will still exclude any key having the token of the original token range, and so we're
+ // still actually selecting what we wanted.
+ for (SSTableReader sstable : Iterables.filter(View.sstablesInBounds(keyRange.left, keyRange.right, intervalTree), predicate))
+ {
+ sstables.add(sstable);
+ }
+ }
+
+ if (logger.isDebugEnabled())
+ logger.debug("ViewFilter for {}/{} sstables", sstables.size(), Iterables.size(view.select(SSTableSet.CANONICAL)));
+ return sstables;
+ }).refs);
+
+
+ List<Range<Token>> normalizedFullRanges = Range.normalize(replicas.onlyFull().ranges());
+ List<Range<Token>> normalizedAllRanges = Range.normalize(replicas.ranges());
+ //Create outgoing file streams for ranges possibly skipping repaired ranges in sstables
+ List<OutgoingStream> streams = new ArrayList<>(refs.size());
+ for (SSTableReader sstable : refs)
+ {
+ List<Range<Token>> ranges = sstable.isRepaired() ? normalizedFullRanges : normalizedAllRanges;
+ List<SSTableReader.PartitionPositionBounds> sections = sstable.getPositionsForRanges(ranges);
+
+ Ref<SSTableReader> ref = refs.get(sstable);
+ if (sections.isEmpty())
+ {
+ ref.release();
+ continue;
+ }
+ streams.add(new CassandraOutgoingFile(session.getStreamOperation(), ref, sections, ranges,
+ sstable.estimatedKeysForRanges(ranges)));
+ }
+
+ return streams;
+ }
+ catch (Throwable t)
+ {
+ refs.release();
+ throw t;
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/db/streaming/CassandraStreamReader.java b/src/java/org/apache/cassandra/db/streaming/CassandraStreamReader.java
new file mode 100644
index 0000000..686d874
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/streaming/CassandraStreamReader.java
@@ -0,0 +1,288 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db.streaming;
+
+import java.io.*;
+import java.util.Collection;
+import java.util.UUID;
+
+import com.google.common.base.Preconditions;
+import com.google.common.base.Throwables;
+import com.google.common.collect.UnmodifiableIterator;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.exceptions.UnknownColumnException;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.util.TrackedDataInputPlus;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.db.*;
+import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
+import org.apache.cassandra.db.rows.*;
+import org.apache.cassandra.io.sstable.SSTableMultiWriter;
+import org.apache.cassandra.io.sstable.SSTableSimpleIterator;
+import org.apache.cassandra.io.sstable.format.RangeAwareSSTableWriter;
+import org.apache.cassandra.io.sstable.format.SSTableFormat;
+import org.apache.cassandra.io.sstable.format.Version;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.streaming.ProgressInfo;
+import org.apache.cassandra.streaming.StreamReceiver;
+import org.apache.cassandra.streaming.StreamSession;
+import org.apache.cassandra.streaming.compress.StreamCompressionInputStream;
+import org.apache.cassandra.streaming.messages.StreamMessageHeader;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.FBUtilities;
+
+import static org.apache.cassandra.net.MessagingService.current_version;
+
+/**
+ * CassandraStreamReader reads from stream and writes to SSTable.
+ */
+public class CassandraStreamReader implements IStreamReader
+{
+ private static final Logger logger = LoggerFactory.getLogger(CassandraStreamReader.class);
+ protected final TableId tableId;
+ protected final long estimatedKeys;
+ protected final Collection<SSTableReader.PartitionPositionBounds> sections;
+ protected final StreamSession session;
+ protected final Version inputVersion;
+ protected final long repairedAt;
+ protected final UUID pendingRepair;
+ protected final SSTableFormat.Type format;
+ protected final int sstableLevel;
+ protected final SerializationHeader.Component header;
+ protected final int fileSeqNum;
+
+ public CassandraStreamReader(StreamMessageHeader header, CassandraStreamHeader streamHeader, StreamSession session)
+ {
+ if (session.getPendingRepair() != null)
+ {
+ // we should only ever be streaming pending repair
+ // sstables if the session has a pending repair id
+ assert session.getPendingRepair().equals(header.pendingRepair);
+ }
+ this.session = session;
+ this.tableId = header.tableId;
+ this.estimatedKeys = streamHeader.estimatedKeys;
+ this.sections = streamHeader.sections;
+ this.inputVersion = streamHeader.version;
+ this.repairedAt = header.repairedAt;
+ this.pendingRepair = header.pendingRepair;
+ this.format = streamHeader.format;
+ this.sstableLevel = streamHeader.sstableLevel;
+ this.header = streamHeader.serializationHeader;
+ this.fileSeqNum = header.sequenceNumber;
+ }
+
+ /**
+ * @param inputPlus where this reads data from
+ * @return SSTable transferred
+ * @throws IOException if reading the remote sstable fails. Will throw an RTE if local write fails.
+ */
+ @SuppressWarnings("resource") // input needs to remain open, streams on top of it can't be closed
+ @Override
+ public SSTableMultiWriter read(DataInputPlus inputPlus) throws IOException
+ {
+ long totalSize = totalSize();
+
+ ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(tableId);
+ if (cfs == null)
+ {
+ // schema was dropped during streaming
+ throw new IOException("CF " + tableId + " was dropped during streaming");
+ }
+
+ logger.debug("[Stream #{}] Start receiving file #{} from {}, repairedAt = {}, size = {}, ks = '{}', table = '{}', pendingRepair = '{}'.",
+ session.planId(), fileSeqNum, session.peer, repairedAt, totalSize, cfs.keyspace.getName(),
+ cfs.getTableName(), pendingRepair);
+
+ StreamDeserializer deserializer = null;
+ SSTableMultiWriter writer = null;
+ try (StreamCompressionInputStream streamCompressionInputStream = new StreamCompressionInputStream(inputPlus, current_version))
+ {
+ TrackedDataInputPlus in = new TrackedDataInputPlus(streamCompressionInputStream);
+ deserializer = new StreamDeserializer(cfs.metadata(), in, inputVersion, getHeader(cfs.metadata()));
+ writer = createWriter(cfs, totalSize, repairedAt, pendingRepair, format);
+ while (in.getBytesRead() < totalSize)
+ {
+ writePartition(deserializer, writer);
+ // TODO move this to BytesReadTracker
+ session.progress(writer.getFilename(), ProgressInfo.Direction.IN, in.getBytesRead(), totalSize);
+ }
+ logger.debug("[Stream #{}] Finished receiving file #{} from {} readBytes = {}, totalSize = {}",
+ session.planId(), fileSeqNum, session.peer, FBUtilities.prettyPrintMemory(in.getBytesRead()), FBUtilities.prettyPrintMemory(totalSize));
+ return writer;
+ }
+ catch (Throwable e)
+ {
+ Object partitionKey = deserializer != null ? deserializer.partitionKey() : "";
+ logger.warn("[Stream {}] Error while reading partition {} from stream on ks='{}' and table='{}'.",
+ session.planId(), partitionKey, cfs.keyspace.getName(), cfs.getTableName(), e);
+ if (writer != null)
+ {
+ writer.abort(e);
+ }
+ throw Throwables.propagate(e);
+ }
+ }
+
+ protected SerializationHeader getHeader(TableMetadata metadata) throws UnknownColumnException
+ {
+ return header != null? header.toHeader(metadata) : null; //pre-3.0 sstable have no SerializationHeader
+ }
+ @SuppressWarnings("resource")
+ protected SSTableMultiWriter createWriter(ColumnFamilyStore cfs, long totalSize, long repairedAt, UUID pendingRepair, SSTableFormat.Type format) throws IOException
+ {
+ Directories.DataDirectory localDir = cfs.getDirectories().getWriteableLocation(totalSize);
+ if (localDir == null)
+ throw new IOException(String.format("Insufficient disk space to store %s", FBUtilities.prettyPrintMemory(totalSize)));
+
+ StreamReceiver streamReceiver = session.getAggregator(tableId);
+ Preconditions.checkState(streamReceiver instanceof CassandraStreamReceiver);
+ LifecycleNewTracker lifecycleNewTracker = CassandraStreamReceiver.fromReceiver(session.getAggregator(tableId)).createLifecycleNewTracker();
+
+ RangeAwareSSTableWriter writer = new RangeAwareSSTableWriter(cfs, estimatedKeys, repairedAt, pendingRepair, false, format, sstableLevel, totalSize, lifecycleNewTracker, getHeader(cfs.metadata()));
+ return writer;
+ }
+
+ protected long totalSize()
+ {
+ long size = 0;
+ for (SSTableReader.PartitionPositionBounds section : sections)
+ size += section.upperPosition - section.lowerPosition;
+ return size;
+ }
+
+ protected void writePartition(StreamDeserializer deserializer, SSTableMultiWriter writer) throws IOException
+ {
+ writer.append(deserializer.newPartition());
+ deserializer.checkForExceptions();
+ }
+
+ public static class StreamDeserializer extends UnmodifiableIterator<Unfiltered> implements UnfilteredRowIterator
+ {
+ private final TableMetadata metadata;
+ private final DataInputPlus in;
+ private final SerializationHeader header;
+ private final DeserializationHelper helper;
+
+ private DecoratedKey key;
+ private DeletionTime partitionLevelDeletion;
+ private SSTableSimpleIterator iterator;
+ private Row staticRow;
+ private IOException exception;
+
+ public StreamDeserializer(TableMetadata metadata, DataInputPlus in, Version version, SerializationHeader header) throws IOException
+ {
+ this.metadata = metadata;
+ this.in = in;
+ this.helper = new DeserializationHelper(metadata, version.correspondingMessagingVersion(), DeserializationHelper.Flag.PRESERVE_SIZE);
+ this.header = header;
+ }
+
+ public StreamDeserializer newPartition() throws IOException
+ {
+ key = metadata.partitioner.decorateKey(ByteBufferUtil.readWithShortLength(in));
+ partitionLevelDeletion = DeletionTime.serializer.deserialize(in);
+ iterator = SSTableSimpleIterator.create(metadata, in, header, helper, partitionLevelDeletion);
+ staticRow = iterator.readStaticRow();
+ return this;
+ }
+
+ public TableMetadata metadata()
+ {
+ return metadata;
+ }
+
+ public RegularAndStaticColumns columns()
+ {
+ // We don't know which columns we'll get so assume it can be all of them
+ return metadata.regularAndStaticColumns();
+ }
+
+ public boolean isReverseOrder()
+ {
+ return false;
+ }
+
+ public DecoratedKey partitionKey()
+ {
+ return key;
+ }
+
+ public DeletionTime partitionLevelDeletion()
+ {
+ return partitionLevelDeletion;
+ }
+
+ public Row staticRow()
+ {
+ return staticRow;
+ }
+
+ public EncodingStats stats()
+ {
+ return header.stats();
+ }
+
+ public boolean hasNext()
+ {
+ try
+ {
+ return iterator.hasNext();
+ }
+ catch (IOError e)
+ {
+ if (e.getCause() != null && e.getCause() instanceof IOException)
+ {
+ exception = (IOException)e.getCause();
+ return false;
+ }
+ throw e;
+ }
+ }
+
+ public Unfiltered next()
+ {
+ // Note that in practice we know that IOException will be thrown by hasNext(), because that's
+ // where the actual reading happens, so we don't bother catching RuntimeException here (contrarily
+ // to what we do in hasNext)
+ Unfiltered unfiltered = iterator.next();
+ return metadata.isCounter() && unfiltered.kind() == Unfiltered.Kind.ROW
+ ? maybeMarkLocalToBeCleared((Row) unfiltered)
+ : unfiltered;
+ }
+
+ private Row maybeMarkLocalToBeCleared(Row row)
+ {
+ return metadata.isCounter() ? row.markCounterLocalToBeCleared() : row;
+ }
+
+ public void checkForExceptions() throws IOException
+ {
+ if (exception != null)
+ throw exception;
+ }
+
+ public void close()
+ {
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/db/streaming/CassandraStreamReceiver.java b/src/java/org/apache/cassandra/db/streaming/CassandraStreamReceiver.java
new file mode 100644
index 0000000..b2b2ce5
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/streaming/CassandraStreamReceiver.java
@@ -0,0 +1,280 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.streaming;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Set;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Iterables;
+
+import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
+import org.apache.cassandra.io.sstable.SSTable;
+import org.apache.cassandra.streaming.StreamReceiveTask;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.Mutation;
+import org.apache.cassandra.db.compaction.OperationType;
+import org.apache.cassandra.db.filter.ColumnFilter;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.db.partitions.PartitionUpdate;
+import org.apache.cassandra.db.rows.ThrottledUnfilteredIterator;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.db.view.View;
+import org.apache.cassandra.dht.Bounds;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.io.sstable.ISSTableScanner;
+import org.apache.cassandra.io.sstable.SSTableMultiWriter;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.streaming.IncomingStream;
+import org.apache.cassandra.streaming.StreamReceiver;
+import org.apache.cassandra.streaming.StreamSession;
+import org.apache.cassandra.utils.CloseableIterator;
+import org.apache.cassandra.utils.Throwables;
+import org.apache.cassandra.utils.concurrent.Refs;
+
+public class CassandraStreamReceiver implements StreamReceiver
+{
+ private static final Logger logger = LoggerFactory.getLogger(CassandraStreamReceiver.class);
+
+ private static final int MAX_ROWS_PER_BATCH = Integer.getInteger("cassandra.repair.mutation_repair_rows_per_batch", 100);
+
+ private final ColumnFamilyStore cfs;
+ private final StreamSession session;
+
+ // Transaction tracking new files received
+ private final LifecycleTransaction txn;
+
+ // holds references to SSTables received
+ protected Collection<SSTableReader> sstables;
+
+ private final boolean requiresWritePath;
+
+
+ public CassandraStreamReceiver(ColumnFamilyStore cfs, StreamSession session, int totalFiles)
+ {
+ this.cfs = cfs;
+ this.session = session;
+ // this is an "offline" transaction, as we currently manually expose the sstables once done;
+ // this should be revisited at a later date, so that LifecycleTransaction manages all sstable state changes
+ this.txn = LifecycleTransaction.offline(OperationType.STREAM);
+ this.sstables = new ArrayList<>(totalFiles);
+ this.requiresWritePath = requiresWritePath(cfs);
+ }
+
+ public static CassandraStreamReceiver fromReceiver(StreamReceiver receiver)
+ {
+ Preconditions.checkArgument(receiver instanceof CassandraStreamReceiver);
+ return (CassandraStreamReceiver) receiver;
+ }
+
+ private static CassandraIncomingFile getFile(IncomingStream stream)
+ {
+ Preconditions.checkArgument(stream instanceof CassandraIncomingFile, "Wrong stream type: {}", stream);
+ return (CassandraIncomingFile) stream;
+ }
+
+ @Override
+ @SuppressWarnings("resource")
+ public synchronized void received(IncomingStream stream)
+ {
+ CassandraIncomingFile file = getFile(stream);
+
+ Collection<SSTableReader> finished = null;
+ SSTableMultiWriter sstable = file.getSSTable();
+ try
+ {
+ finished = sstable.finish(true);
+ }
+ catch (Throwable t)
+ {
+ Throwables.maybeFail(sstable.abort(t));
+ }
+ txn.update(finished, false);
+ sstables.addAll(finished);
+ }
+
+ @Override
+ public void discardStream(IncomingStream stream)
+ {
+ CassandraIncomingFile file = getFile(stream);
+ Throwables.maybeFail(file.getSSTable().abort(null));
+ }
+
+ /**
+ * @return a LifecycleNewTracker whose operations are synchronised on this StreamReceiveTask.
+ */
+ public synchronized LifecycleNewTracker createLifecycleNewTracker()
+ {
+ return new LifecycleNewTracker()
+ {
+ @Override
+ public void trackNew(SSTable table)
+ {
+ synchronized (CassandraStreamReceiver.this)
+ {
+ txn.trackNew(table);
+ }
+ }
+
+ @Override
+ public void untrackNew(SSTable table)
+ {
+ synchronized (CassandraStreamReceiver.this)
+ {
+ txn.untrackNew(table);
+ }
+ }
+
+ public OperationType opType()
+ {
+ return txn.opType();
+ }
+ };
+ }
+
+
+ @Override
+ public synchronized void abort()
+ {
+ sstables.clear();
+ txn.abort();
+ }
+
+ private boolean hasViews(ColumnFamilyStore cfs)
+ {
+ return !Iterables.isEmpty(View.findAll(cfs.metadata.keyspace, cfs.getTableName()));
+ }
+
+ private boolean hasCDC(ColumnFamilyStore cfs)
+ {
+ return cfs.metadata().params.cdc;
+ }
+
+ /*
+ * We have a special path for views and for CDC.
+ *
+ * For views, since the view requires cleaning up any pre-existing state, we must put all partitions
+ * through the same write path as normal mutations. This also ensures any 2is are also updated.
+ *
+ * For CDC-enabled tables, we want to ensure that the mutations are run through the CommitLog so they
+ * can be archived by the CDC process on discard.
+ */
+ private boolean requiresWritePath(ColumnFamilyStore cfs) {
+ return hasCDC(cfs) || (session.streamOperation().requiresViewBuild() && hasViews(cfs));
+ }
+
+ private void sendThroughWritePath(ColumnFamilyStore cfs, Collection<SSTableReader> readers) {
+ boolean hasCdc = hasCDC(cfs);
+ ColumnFilter filter = ColumnFilter.all(cfs.metadata());
+ for (SSTableReader reader : readers)
+ {
+ Keyspace ks = Keyspace.open(reader.getKeyspaceName());
+ // When doing mutation-based repair we split each partition into smaller batches
+ // ({@link Stream MAX_ROWS_PER_BATCH}) to avoid OOMing and generating heap pressure
+ try (ISSTableScanner scanner = reader.getScanner();
+ CloseableIterator<UnfilteredRowIterator> throttledPartitions = ThrottledUnfilteredIterator.throttle(scanner, MAX_ROWS_PER_BATCH))
+ {
+ while (throttledPartitions.hasNext())
+ {
+ // MV *can* be applied unsafe if there's no CDC on the CFS as we flush
+ // before transaction is done.
+ //
+ // If the CFS has CDC, however, these updates need to be written to the CommitLog
+ // so they get archived into the cdc_raw folder
+ ks.apply(new Mutation(PartitionUpdate.fromIterator(throttledPartitions.next(), filter)),
+ hasCdc,
+ true,
+ false);
+ }
+ }
+ }
+ }
+
+ public synchronized void finishTransaction()
+ {
+ txn.finish();
+ }
+
+ @Override
+ public void finished()
+ {
+ boolean requiresWritePath = requiresWritePath(cfs);
+ Collection<SSTableReader> readers = sstables;
+
+ try (Refs<SSTableReader> refs = Refs.ref(readers))
+ {
+ if (requiresWritePath)
+ {
+ sendThroughWritePath(cfs, readers);
+ }
+ else
+ {
+ finishTransaction();
+
+ // add sstables (this will build secondary indexes too, see CASSANDRA-10130)
+ logger.debug("[Stream #{}] Received {} sstables from {} ({})", session.planId(), readers.size(), session.peer, readers);
+ cfs.addSSTables(readers);
+
+ //invalidate row and counter cache
+ if (cfs.isRowCacheEnabled() || cfs.metadata().isCounter())
+ {
+ List<Bounds<Token>> boundsToInvalidate = new ArrayList<>(readers.size());
+ readers.forEach(sstable -> boundsToInvalidate.add(new Bounds<Token>(sstable.first.getToken(), sstable.last.getToken())));
+ Set<Bounds<Token>> nonOverlappingBounds = Bounds.getNonOverlappingBounds(boundsToInvalidate);
+
+ if (cfs.isRowCacheEnabled())
+ {
+ int invalidatedKeys = cfs.invalidateRowCache(nonOverlappingBounds);
+ if (invalidatedKeys > 0)
+ logger.debug("[Stream #{}] Invalidated {} row cache entries on table {}.{} after stream " +
+ "receive task completed.", session.planId(), invalidatedKeys,
+ cfs.keyspace.getName(), cfs.getTableName());
+ }
+
+ if (cfs.metadata().isCounter())
+ {
+ int invalidatedKeys = cfs.invalidateCounterCache(nonOverlappingBounds);
+ if (invalidatedKeys > 0)
+ logger.debug("[Stream #{}] Invalidated {} counter cache entries on table {}.{} after stream " +
+ "receive task completed.", session.planId(), invalidatedKeys,
+ cfs.keyspace.getName(), cfs.getTableName());
+ }
+ }
+ }
+ }
+ }
+
+ @Override
+ public void cleanup()
+ {
+ // We don't keep the streamed sstables since we've applied them manually so we abort the txn and delete
+ // the streamed sstables.
+ if (requiresWritePath)
+ {
+ cfs.forceBlockingFlush();
+ abort();
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/db/streaming/CassandraStreamWriter.java b/src/java/org/apache/cassandra/db/streaming/CassandraStreamWriter.java
new file mode 100644
index 0000000..8382f0a
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/streaming/CassandraStreamWriter.java
@@ -0,0 +1,180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db.streaming;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Collection;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import net.jpountz.lz4.LZ4Compressor;
+import net.jpountz.lz4.LZ4Factory;
+import org.apache.cassandra.io.compress.BufferType;
+import org.apache.cassandra.io.sstable.Component;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.util.ChannelProxy;
+import org.apache.cassandra.io.util.DataIntegrityMetadata;
+import org.apache.cassandra.io.util.DataIntegrityMetadata.ChecksumValidator;
+import org.apache.cassandra.io.util.DataOutputStreamPlus;
+import org.apache.cassandra.net.AsyncStreamingOutputPlus;
+import org.apache.cassandra.streaming.ProgressInfo;
+import org.apache.cassandra.streaming.StreamManager;
+import org.apache.cassandra.streaming.StreamManager.StreamRateLimiter;
+import org.apache.cassandra.streaming.StreamSession;
+import org.apache.cassandra.streaming.async.StreamCompressionSerializer;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.memory.BufferPool;
+
+import static org.apache.cassandra.net.MessagingService.current_version;
+
+/**
+ * CassandraStreamWriter writes given section of the SSTable to given channel.
+ */
+public class CassandraStreamWriter
+{
+ private static final int DEFAULT_CHUNK_SIZE = 64 * 1024;
+
+ private static final Logger logger = LoggerFactory.getLogger(CassandraStreamWriter.class);
+
+ protected final SSTableReader sstable;
+ private final LZ4Compressor compressor = LZ4Factory.fastestInstance().fastCompressor();
+ protected final Collection<SSTableReader.PartitionPositionBounds> sections;
+ protected final StreamRateLimiter limiter;
+ protected final StreamSession session;
+
+ public CassandraStreamWriter(SSTableReader sstable, Collection<SSTableReader.PartitionPositionBounds> sections, StreamSession session)
+ {
+ this.session = session;
+ this.sstable = sstable;
+ this.sections = sections;
+ this.limiter = StreamManager.getRateLimiter(session.peer);
+ }
+
+ /**
+ * Stream file of specified sections to given channel.
+ *
+ * CassandraStreamWriter uses LZF compression on wire to decrease size to transfer.
+ *
+ * @param output where this writes data to
+ * @throws IOException on any I/O error
+ */
+ public void write(DataOutputStreamPlus output) throws IOException
+ {
+ long totalSize = totalSize();
+ logger.debug("[Stream #{}] Start streaming file {} to {}, repairedAt = {}, totalSize = {}", session.planId(),
+ sstable.getFilename(), session.peer, sstable.getSSTableMetadata().repairedAt, totalSize);
+
+ AsyncStreamingOutputPlus out = (AsyncStreamingOutputPlus) output;
+ try(ChannelProxy proxy = sstable.getDataChannel().newChannel();
+ ChecksumValidator validator = new File(sstable.descriptor.filenameFor(Component.CRC)).exists()
+ ? DataIntegrityMetadata.checksumValidator(sstable.descriptor)
+ : null)
+ {
+ int bufferSize = validator == null ? DEFAULT_CHUNK_SIZE: validator.chunkSize;
+
+ // setting up data compression stream
+ long progress = 0L;
+
+ // stream each of the required sections of the file
+ for (SSTableReader.PartitionPositionBounds section : sections)
+ {
+ long start = validator == null ? section.lowerPosition : validator.chunkStart(section.lowerPosition);
+ // if the transfer does not start on the valididator's chunk boundary, this is the number of bytes to offset by
+ int transferOffset = (int) (section.lowerPosition - start);
+ if (validator != null)
+ validator.seek(start);
+
+ // length of the section to read
+ long length = section.upperPosition - start;
+ // tracks write progress
+ long bytesRead = 0;
+ while (bytesRead < length)
+ {
+ int toTransfer = (int) Math.min(bufferSize, length - bytesRead);
+ long lastBytesRead = write(proxy, validator, out, start, transferOffset, toTransfer, bufferSize);
+ start += lastBytesRead;
+ bytesRead += lastBytesRead;
+ progress += (lastBytesRead - transferOffset);
+ session.progress(sstable.descriptor.filenameFor(Component.DATA), ProgressInfo.Direction.OUT, progress, totalSize);
+ transferOffset = 0;
+ }
+
+ // make sure that current section is sent
+ out.flush();
+ }
+ logger.debug("[Stream #{}] Finished streaming file {} to {}, bytesTransferred = {}, totalSize = {}",
+ session.planId(), sstable.getFilename(), session.peer, FBUtilities.prettyPrintMemory(progress), FBUtilities.prettyPrintMemory(totalSize));
+ }
+ }
+
+ protected long totalSize()
+ {
+ long size = 0;
+ for (SSTableReader.PartitionPositionBounds section : sections)
+ size += section.upperPosition - section.lowerPosition;
+ return size;
+ }
+
+ /**
+ * Sequentially read bytes from the file and write them to the output stream
+ *
+ * @param proxy The file reader to read from
+ * @param validator validator to verify data integrity
+ * @param start The readd offset from the beginning of the {@code proxy} file.
+ * @param transferOffset number of bytes to skip transfer, but include for validation.
+ * @param toTransfer The number of bytes to be transferred.
+ *
+ * @return Number of bytes transferred.
+ *
+ * @throws java.io.IOException on any I/O error
+ */
+ protected long write(ChannelProxy proxy, ChecksumValidator validator, AsyncStreamingOutputPlus output, long start, int transferOffset, int toTransfer, int bufferSize) throws IOException
+ {
+ // the count of bytes to read off disk
+ int minReadable = (int) Math.min(bufferSize, proxy.size() - start);
+
+ // this buffer will hold the data from disk. as it will be compressed on the fly by
+ // AsyncChannelCompressedStreamWriter.write(ByteBuffer), we can release this buffer as soon as we can.
+ ByteBuffer buffer = BufferPool.get(minReadable, BufferType.OFF_HEAP);
+ try
+ {
+ int readCount = proxy.read(buffer, start);
+ assert readCount == minReadable : String.format("could not read required number of bytes from file to be streamed: read %d bytes, wanted %d bytes", readCount, minReadable);
+ buffer.flip();
+
+ if (validator != null)
+ {
+ validator.validate(buffer);
+ buffer.flip();
+ }
+
+ buffer.position(transferOffset);
+ buffer.limit(transferOffset + (toTransfer - transferOffset));
+ output.writeToChannel(StreamCompressionSerializer.serialize(compressor, buffer, current_version), limiter);
+ }
+ finally
+ {
+ BufferPool.put(buffer);
+ }
+
+ return toTransfer;
+ }
+}
diff --git a/src/java/org/apache/cassandra/db/streaming/ComponentManifest.java b/src/java/org/apache/cassandra/db/streaming/ComponentManifest.java
new file mode 100644
index 0000000..90e3dbd
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/streaming/ComponentManifest.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.streaming;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import com.google.common.collect.Iterators;
+
+import org.apache.cassandra.db.TypeSizes;
+import org.apache.cassandra.io.IVersionedSerializer;
+import org.apache.cassandra.io.sstable.Component;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+
+public final class ComponentManifest implements Iterable<Component>
+{
+ private final LinkedHashMap<Component, Long> components;
+
+ public ComponentManifest(Map<Component, Long> components)
+ {
+ this.components = new LinkedHashMap<>(components);
+ }
+
+ public long sizeOf(Component component)
+ {
+ Long size = components.get(component);
+ if (size == null)
+ throw new IllegalArgumentException("Component " + component + " is not present in the manifest");
+ return size;
+ }
+
+ public long totalSize()
+ {
+ long totalSize = 0;
+ for (Long size : components.values())
+ totalSize += size;
+ return totalSize;
+ }
+
+ public List<Component> components()
+ {
+ return new ArrayList<>(components.keySet());
+ }
+
+ @Override
+ public boolean equals(Object o)
+ {
+ if (this == o)
+ return true;
+
+ if (!(o instanceof ComponentManifest))
+ return false;
+
+ ComponentManifest that = (ComponentManifest) o;
+ return components.equals(that.components);
+ }
+
+ @Override
+ public int hashCode()
+ {
+ return components.hashCode();
+ }
+
+ public static final IVersionedSerializer<ComponentManifest> serializer = new IVersionedSerializer<ComponentManifest>()
+ {
+ public void serialize(ComponentManifest manifest, DataOutputPlus out, int version) throws IOException
+ {
+ out.writeUnsignedVInt(manifest.components.size());
+ for (Map.Entry<Component, Long> entry : manifest.components.entrySet())
+ {
+ out.writeUTF(entry.getKey().name);
+ out.writeUnsignedVInt(entry.getValue());
+ }
+ }
+
+ public ComponentManifest deserialize(DataInputPlus in, int version) throws IOException
+ {
+ int size = (int) in.readUnsignedVInt();
+
+ LinkedHashMap<Component, Long> components = new LinkedHashMap<>(size);
+
+ for (int i = 0; i < size; i++)
+ {
+ Component component = Component.parse(in.readUTF());
+ long length = in.readUnsignedVInt();
+ components.put(component, length);
+ }
+
+ return new ComponentManifest(components);
+ }
+
+ public long serializedSize(ComponentManifest manifest, int version)
+ {
+ long size = TypeSizes.sizeofUnsignedVInt(manifest.components.size());
+ for (Map.Entry<Component, Long> entry : manifest.components.entrySet())
+ {
+ size += TypeSizes.sizeof(entry.getKey().name);
+ size += TypeSizes.sizeofUnsignedVInt(entry.getValue());
+ }
+ return size;
+ }
+ };
+
+ @Override
+ public Iterator<Component> iterator()
+ {
+ return Iterators.unmodifiableIterator(components.keySet().iterator());
+ }
+}
diff --git a/src/java/org/apache/cassandra/db/streaming/CompressedInputStream.java b/src/java/org/apache/cassandra/db/streaming/CompressedInputStream.java
new file mode 100644
index 0000000..b8626ff
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/streaming/CompressedInputStream.java
@@ -0,0 +1,234 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db.streaming;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Iterator;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.function.DoubleSupplier;
+
+import com.google.common.collect.Iterators;
+import com.google.common.primitives.Ints;
+
+import org.apache.cassandra.io.compress.CompressionMetadata;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.io.util.RebufferingInputStream;
+import org.apache.cassandra.schema.CompressionParams;
+import org.apache.cassandra.utils.ChecksumType;
+
+import static java.lang.Math.max;
+import static java.lang.String.format;
+
+/**
+ * InputStream which reads compressed chunks from the underlying input stream and deals with decompression
+ * and position tracking.
+ *
+ * The underlying input will be an instance of {@link RebufferingInputStream} except in some unit tests.
+ *
+ * Compressed chunks transferred will be a subset of all chunks in the source streamed sstable - just enough to
+ * deserialize the requested partition position ranges. Correctness of the entire operation depends on provided
+ * partition position ranges and compressed chunks properly matching, and there is no way on the receiving side to
+ * verify if that's the case, which arguably makes this a little brittle.
+ */
+public class CompressedInputStream extends RebufferingInputStream implements AutoCloseable
+{
+ private static final double GROWTH_FACTOR = 1.5;
+
+ private final DataInputPlus input;
+
+ private final Iterator<CompressionMetadata.Chunk> compressedChunks;
+ private final CompressionParams compressionParams;
+
+ private final ChecksumType checksumType;
+ private final DoubleSupplier validateChecksumChance;
+
+ /**
+ * The base offset of the current {@link #buffer} into the original sstable as if it were uncompressed.
+ */
+ private long uncompressedChunkPosition = Long.MIN_VALUE;
+
+ /**
+ * @param input Input input to read compressed data from
+ * @param compressionInfo Compression info
+ */
+ public CompressedInputStream(DataInputPlus input,
+ CompressionInfo compressionInfo,
+ ChecksumType checksumType,
+ DoubleSupplier validateChecksumChance)
+ {
+ super(ByteBuffer.allocateDirect(compressionInfo.parameters.chunkLength()));
+ buffer.limit(0);
+
+ this.input = input;
+ this.checksumType = checksumType;
+ this.validateChecksumChance = validateChecksumChance;
+
+ compressionParams = compressionInfo.parameters;
+ compressedChunks = Iterators.forArray(compressionInfo.chunks);
+ compressedChunk = ByteBuffer.allocateDirect(compressionParams.chunkLength());
+ }
+
+ /**
+ * Invoked when crossing into the next {@link SSTableReader.PartitionPositionBounds} section
+ * in {@link CassandraCompressedStreamReader#read(DataInputPlus)}.
+ * Will skip 1..n compressed chunks of the original sstable.
+ */
+ public void position(long position) throws IOException
+ {
+ if (position < uncompressedChunkPosition + buffer.position())
+ throw new IllegalStateException("stream can only move forward");
+
+ if (position >= uncompressedChunkPosition + buffer.limit())
+ {
+ loadNextChunk();
+ // uncompressedChunkPosition = position - (position % compressionParams.chunkLength())
+ uncompressedChunkPosition = position & -compressionParams.chunkLength();
+ }
+
+ buffer.position(Ints.checkedCast(position - uncompressedChunkPosition));
+ }
+
+ @Override
+ protected void reBuffer() throws IOException
+ {
+ if (uncompressedChunkPosition < 0)
+ throw new IllegalStateException("position(long position) wasn't called first");
+
+ /*
+ * reBuffer() will only be called if a partition range spanning multiple (adjacent) compressed chunks
+ * has consumed the current uncompressed buffer, and needs to move to the next adjacent chunk;
+ * uncompressedChunkPosition in this scenario *always* increases by the fixed chunk length.
+ */
+ loadNextChunk();
+ uncompressedChunkPosition += compressionParams.chunkLength();
+ }
+
+ /**
+ * Reads the next chunk, decompresses if necessary, and probabilistically verifies the checksum/CRC.
+ *
+ * Doesn't adjust uncompressedChunkPosition - it's up to the caller to do so.
+ */
+ private void loadNextChunk() throws IOException
+ {
+ if (!compressedChunks.hasNext())
+ throw new EOFException();
+
+ int chunkLength = compressedChunks.next().length;
+ chunkBytesRead += (chunkLength + 4); // chunk length + checksum or CRC length
+
+ /*
+ * uncompress if the buffer size is less than the max chunk size; else, if the buffer size is greater than
+ * or equal to the maxCompressedLength, we assume the buffer is not compressed (see CASSANDRA-10520)
+ */
+ if (chunkLength < compressionParams.maxCompressedLength())
+ {
+ if (compressedChunk.capacity() < chunkLength)
+ {
+ // with poorly compressible data, it's possible for a compressed chunk to be larger than
+ // configured uncompressed chunk size - depending on data, min_compress_ratio, and compressor;
+ // we may need to resize the compressed buffer.
+ FileUtils.clean(compressedChunk);
+ compressedChunk = ByteBuffer.allocateDirect(max((int) (compressedChunk.capacity() * GROWTH_FACTOR), chunkLength));
+ }
+
+ compressedChunk.position(0).limit(chunkLength);
+ readChunk(compressedChunk);
+ compressedChunk.position(0);
+
+ maybeValidateChecksum(compressedChunk, input.readInt());
+
+ buffer.clear();
+ compressionParams.getSstableCompressor().uncompress(compressedChunk, buffer);
+ buffer.flip();
+ }
+ else
+ {
+ buffer.position(0).limit(chunkLength);
+ readChunk(buffer);
+ buffer.position(0);
+
+ maybeValidateChecksum(buffer, input.readInt());
+ }
+ }
+ private ByteBuffer compressedChunk;
+
+ private void readChunk(ByteBuffer dst) throws IOException
+ {
+ if (input instanceof RebufferingInputStream)
+ ((RebufferingInputStream) input).readFully(dst);
+ else
+ readChunkSlow(dst);
+ }
+
+ // slow path that involves an intermediate copy into a byte array; only used by some of the unit tests
+ private void readChunkSlow(ByteBuffer dst) throws IOException
+ {
+ if (copyArray == null)
+ copyArray = new byte[dst.remaining()];
+ else if (copyArray.length < dst.remaining())
+ copyArray = new byte[max((int)(copyArray.length * GROWTH_FACTOR), dst.remaining())];
+
+ input.readFully(copyArray, 0, dst.remaining());
+ dst.put(copyArray, 0, dst.remaining());
+ }
+ private byte[] copyArray;
+
+ private void maybeValidateChecksum(ByteBuffer buffer, int expectedChecksum) throws IOException
+ {
+ double validateChance = validateChecksumChance.getAsDouble();
+
+ if (validateChance >= 1.0d || (validateChance > 0.0d && validateChance > ThreadLocalRandom.current().nextDouble()))
+ {
+ int position = buffer.position();
+ int actualChecksum = (int) checksumType.of(buffer);
+ buffer.position(position); // checksum calculation consumes the buffer, so we must reset its position afterwards
+
+ if (expectedChecksum != actualChecksum)
+ throw new IOException(format("Checksum didn't match (expected: %d, actual: %d)", expectedChecksum, actualChecksum));
+ }
+ }
+
+ @Override
+ public void close()
+ {
+ if (null != buffer)
+ {
+ FileUtils.clean(buffer);
+ buffer = null;
+ }
+
+ if (null != compressedChunk)
+ {
+ FileUtils.clean(compressedChunk);
+ compressedChunk = null;
+ }
+ }
+
+ /**
+ * @return accumulated size of all chunks read so far - including checksums
+ */
+ long chunkBytesRead()
+ {
+ return chunkBytesRead;
+ }
+ private long chunkBytesRead = 0;
+}
diff --git a/src/java/org/apache/cassandra/db/streaming/CompressionInfo.java b/src/java/org/apache/cassandra/db/streaming/CompressionInfo.java
new file mode 100644
index 0000000..aef57e3
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/streaming/CompressionInfo.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db.streaming;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.cassandra.db.TypeSizes;
+import org.apache.cassandra.io.IVersionedSerializer;
+import org.apache.cassandra.io.compress.CompressionMetadata;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.schema.CompressionParams;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+
+/**
+ * Container that carries compression parameters and chunks to decompress data from stream.
+ */
+public class CompressionInfo
+{
+ public static final IVersionedSerializer<CompressionInfo> serializer = new CompressionInfoSerializer();
+
+ public final CompressionMetadata.Chunk[] chunks;
+ public final CompressionParams parameters;
+
+ public CompressionInfo(CompressionMetadata.Chunk[] chunks, CompressionParams parameters)
+ {
+ assert chunks != null && parameters != null;
+ this.chunks = chunks;
+ this.parameters = parameters;
+ }
+
+ static CompressionInfo fromCompressionMetadata(CompressionMetadata metadata, List<SSTableReader.PartitionPositionBounds> sections)
+ {
+ if (metadata == null)
+ {
+ return null;
+ }
+ else
+ {
+ return new CompressionInfo(metadata.getChunksForSections(sections), metadata.parameters);
+ }
+
+ }
+
+ static class CompressionInfoSerializer implements IVersionedSerializer<CompressionInfo>
+ {
+ public void serialize(CompressionInfo info, DataOutputPlus out, int version) throws IOException
+ {
+ if (info == null)
+ {
+ out.writeInt(-1);
+ return;
+ }
+
+ int chunkCount = info.chunks.length;
+ out.writeInt(chunkCount);
+ for (int i = 0; i < chunkCount; i++)
+ CompressionMetadata.Chunk.serializer.serialize(info.chunks[i], out, version);
+ // compression params
+ CompressionParams.serializer.serialize(info.parameters, out, version);
+ }
+
+ public CompressionInfo deserialize(DataInputPlus in, int version) throws IOException
+ {
+ // chunks
+ int chunkCount = in.readInt();
+ if (chunkCount < 0)
+ return null;
+
+ CompressionMetadata.Chunk[] chunks = new CompressionMetadata.Chunk[chunkCount];
+ for (int i = 0; i < chunkCount; i++)
+ chunks[i] = CompressionMetadata.Chunk.serializer.deserialize(in, version);
+
+ // compression params
+ CompressionParams parameters = CompressionParams.serializer.deserialize(in, version);
+ return new CompressionInfo(chunks, parameters);
+ }
+
+ public long serializedSize(CompressionInfo info, int version)
+ {
+ if (info == null)
+ return TypeSizes.sizeof(-1);
+
+ // chunks
+ int chunkCount = info.chunks.length;
+ long size = TypeSizes.sizeof(chunkCount);
+ for (int i = 0; i < chunkCount; i++)
+ size += CompressionMetadata.Chunk.serializer.serializedSize(info.chunks[i], version);
+ // compression params
+ size += CompressionParams.serializer.serializedSize(info.parameters, version);
+ return size;
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/db/streaming/IStreamReader.java b/src/java/org/apache/cassandra/db/streaming/IStreamReader.java
new file mode 100644
index 0000000..cf93bc2
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/streaming/IStreamReader.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.streaming;
+
+import java.io.IOException;
+
+import org.apache.cassandra.io.sstable.SSTableMultiWriter;
+import org.apache.cassandra.io.util.DataInputPlus;
+
+/**
+ * This is the interface is used by the streaming code read a SSTable stream off a channel.
+ */
+public interface IStreamReader
+{
+ public SSTableMultiWriter read(DataInputPlus inputPlus) throws IOException;
+}
diff --git a/src/java/org/apache/cassandra/db/streaming/package-info.java b/src/java/org/apache/cassandra/db/streaming/package-info.java
new file mode 100644
index 0000000..1e117aa
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/streaming/package-info.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * <h2>File transfer</h2>
+ *
+ * When tranferring whole or subsections of an sstable, only the DATA component is shipped. To that end,
+ * there are three "modes" of an sstable transfer that need to be handled somewhat differently:
+ *
+ * 1) uncompressed sstable - data needs to be read into user space so it can be manipulated: checksum validation,
+ * apply stream compression (see next section), and/or TLS encryption.
+ *
+ * 2) compressed sstable, transferred with SSL/TLS - data needs to be read into user space as that is where the TLS encryption
+ * needs to happen. Netty does not allow the pretense of doing zero-copy transfers when TLS is in the pipeline;
+ * data must explicitly be pulled into user-space memory for TLS encryption to work.
+ *
+ * 3) compressed sstable, transferred without SSL/TLS - data can be streamed via zero-copy transfer as the data does not
+ * need to be manipulated (it can be sent "as-is").
+ *
+ * <h3>Compressing the data</h3>
+ * We always want to transfer as few bytes as possible of the wire when streaming a file. If the
+ * sstable is not already compressed via table compression options, we apply an on-the-fly stream compression
+ * to the data. The stream compression format is documented in
+ * {@link org.apache.cassandra.streaming.async.StreamCompressionSerializer}
+ *
+ * You may be wondering: why implement your own compression scheme? why not use netty's built-in compression codecs,
+ * like {@link io.netty.handler.codec.compression.Lz4FrameEncoder}? That makes complete sense if all the sstables
+ * to be streamed are non using sstable compression (and obviously you wouldn't use stream compression when the sstables
+ * are using sstable compression). The problem is when you have a mix of files, some using sstable compression
+ * and some not. You can either:
+ *
+ * - send the files of one type over one kind of socket, and the others over another socket
+ * - send them both over the same socket, but then auto-adjust per each file type.
+ *
+ * I've opted for the latter to keep socket/channel management simpler and cleaner.
+ *
+ */
+package org.apache.cassandra.db.streaming;
diff --git a/src/java/org/apache/cassandra/db/transform/BasePartitions.java b/src/java/org/apache/cassandra/db/transform/BasePartitions.java
index f6c486d..464ae6f 100644
--- a/src/java/org/apache/cassandra/db/transform/BasePartitions.java
+++ b/src/java/org/apache/cassandra/db/transform/BasePartitions.java
@@ -77,6 +77,7 @@
return fail;
}
+ @SuppressWarnings("resource")
public final boolean hasNext()
{
BaseRowIterator<?> next = null;
diff --git a/src/java/org/apache/cassandra/db/transform/BaseRows.java b/src/java/org/apache/cassandra/db/transform/BaseRows.java
index 42d272c..d5ad3f7 100644
--- a/src/java/org/apache/cassandra/db/transform/BaseRows.java
+++ b/src/java/org/apache/cassandra/db/transform/BaseRows.java
@@ -20,9 +20,9 @@
*/
package org.apache.cassandra.db.transform;
-import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.db.DecoratedKey;
-import org.apache.cassandra.db.PartitionColumns;
+import org.apache.cassandra.db.RegularAndStaticColumns;
import org.apache.cassandra.db.rows.*;
import static org.apache.cassandra.utils.Throwables.merge;
@@ -50,7 +50,7 @@
partitionKey = copyFrom.partitionKey();
}
- public CFMetaData metadata()
+ public TableMetadata metadata()
{
return input.metadata();
}
@@ -60,7 +60,7 @@
return input.isReverseOrder();
}
- public PartitionColumns columns()
+ public RegularAndStaticColumns columns()
{
return input.columns();
}
diff --git a/src/java/org/apache/cassandra/db/transform/DuplicateRowChecker.java b/src/java/org/apache/cassandra/db/transform/DuplicateRowChecker.java
index 7a6f7f9..aa1305a 100644
--- a/src/java/org/apache/cassandra/db/transform/DuplicateRowChecker.java
+++ b/src/java/org/apache/cassandra/db/transform/DuplicateRowChecker.java
@@ -18,20 +18,20 @@
package org.apache.cassandra.db.transform;
-import java.net.InetAddress;
import java.util.Collections;
import java.util.List;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.compaction.OperationType;
import org.apache.cassandra.db.partitions.PartitionIterator;
import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
import org.apache.cassandra.db.rows.*;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.utils.DiagnosticSnapshotService;
import org.apache.cassandra.utils.FBUtilities;
@@ -43,16 +43,16 @@
int duplicatesDetected = 0;
final String stage;
- final List<InetAddress> replicas;
- final CFMetaData metadata;
+ final List<InetAddressAndPort> replicas;
+ final TableMetadata metadata;
final DecoratedKey key;
final boolean snapshotOnDuplicate;
DuplicateRowChecker(final DecoratedKey key,
- final CFMetaData metadata,
+ final TableMetadata metadata,
final String stage,
final boolean snapshotOnDuplicate,
- final List<InetAddress> replicas)
+ final List<InetAddressAndPort> replicas)
{
this.key = key;
this.metadata = metadata;
@@ -90,7 +90,7 @@
{
logger.warn("Detected {} duplicate rows for {} during {}",
duplicatesDetected,
- metadata.getKeyValidator().getString(key.getKey()),
+ metadata.partitionKeyType.getString(key.getKey()),
stage);
if (snapshotOnDuplicate)
DiagnosticSnapshotService.duplicateRows(metadata, replicas);
@@ -104,7 +104,7 @@
{
if (!DatabaseDescriptor.checkForDuplicateRowsDuringCompaction())
return iterator;
- final List<InetAddress> address = Collections.singletonList(FBUtilities.getBroadcastAddress());
+ final List<InetAddressAndPort> address = Collections.singletonList(FBUtilities.getBroadcastAddressAndPort());
final boolean snapshot = DatabaseDescriptor.snapshotOnDuplicateRowDetection();
return Transformation.apply(iterator, new Transformation<UnfilteredRowIterator>()
{
@@ -119,7 +119,7 @@
});
}
- public static PartitionIterator duringRead(final PartitionIterator iterator, final List<InetAddress> replicas)
+ public static PartitionIterator duringRead(final PartitionIterator iterator, final List<InetAddressAndPort> replicas)
{
if (!DatabaseDescriptor.checkForDuplicateRowsDuringReads())
return iterator;
diff --git a/src/java/org/apache/cassandra/db/transform/Filter.java b/src/java/org/apache/cassandra/db/transform/Filter.java
index 48a1634..0bd3eab 100644
--- a/src/java/org/apache/cassandra/db/transform/Filter.java
+++ b/src/java/org/apache/cassandra/db/transform/Filter.java
@@ -35,6 +35,7 @@
}
@Override
+ @SuppressWarnings("resource")
protected RowIterator applyToPartition(BaseRowIterator iterator)
{
return iterator instanceof UnfilteredRows
diff --git a/src/java/org/apache/cassandra/db/transform/FilteredPartitions.java b/src/java/org/apache/cassandra/db/transform/FilteredPartitions.java
index b835a6b..a1b8571 100644
--- a/src/java/org/apache/cassandra/db/transform/FilteredPartitions.java
+++ b/src/java/org/apache/cassandra/db/transform/FilteredPartitions.java
@@ -50,17 +50,14 @@
/**
* Filter any RangeTombstoneMarker from the iterator's iterators, transforming it into a PartitionIterator.
*/
+ @SuppressWarnings("resource")
public static FilteredPartitions filter(UnfilteredPartitionIterator iterator, int nowInSecs)
{
- FilteredPartitions filtered = filter(iterator,
- new Filter(nowInSecs,
- iterator.metadata().enforceStrictLiveness()));
-
- return iterator.isForThrift()
- ? filtered
- : (FilteredPartitions) Transformation.apply(filtered, new EmptyPartitionsDiscarder());
+ FilteredPartitions filtered = filter(iterator, new Filter(nowInSecs, iterator.metadata().enforceStrictLiveness()));
+ return (FilteredPartitions) Transformation.apply(filtered, new EmptyPartitionsDiscarder());
}
+ @SuppressWarnings("resource")
public static FilteredPartitions filter(UnfilteredPartitionIterator iterator, Filter filter)
{
return iterator instanceof UnfilteredPartitions
diff --git a/src/java/org/apache/cassandra/db/transform/MoreRows.java b/src/java/org/apache/cassandra/db/transform/MoreRows.java
index 118739b..f3856c4 100644
--- a/src/java/org/apache/cassandra/db/transform/MoreRows.java
+++ b/src/java/org/apache/cassandra/db/transform/MoreRows.java
@@ -20,7 +20,7 @@
*/
package org.apache.cassandra.db.transform;
-import org.apache.cassandra.db.PartitionColumns;
+import org.apache.cassandra.db.RegularAndStaticColumns;
import org.apache.cassandra.db.rows.BaseRowIterator;
import org.apache.cassandra.db.rows.RowIterator;
import org.apache.cassandra.db.rows.UnfilteredRowIterator;
@@ -48,7 +48,7 @@
return add(mutable(iterator), more);
}
- public static UnfilteredRowIterator extend(UnfilteredRowIterator iterator, MoreRows<? super UnfilteredRowIterator> more, PartitionColumns columns)
+ public static UnfilteredRowIterator extend(UnfilteredRowIterator iterator, MoreRows<? super UnfilteredRowIterator> more, RegularAndStaticColumns columns)
{
return add(Transformation.wrapIterator(iterator, columns), more);
}
diff --git a/src/java/org/apache/cassandra/db/transform/RTBoundCloser.java b/src/java/org/apache/cassandra/db/transform/RTBoundCloser.java
index 192b2fe..5dea6e7 100644
--- a/src/java/org/apache/cassandra/db/transform/RTBoundCloser.java
+++ b/src/java/org/apache/cassandra/db/transform/RTBoundCloser.java
@@ -17,7 +17,6 @@
*/
package org.apache.cassandra.db.transform;
-import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.db.Clustering;
import org.apache.cassandra.db.DeletionTime;
import org.apache.cassandra.db.ReadExecutionController;
@@ -103,9 +102,7 @@
*/
if (null == lastRowClustering)
{
- CFMetaData metadata = partition.metadata();
- String message =
- String.format("UnfilteredRowIterator for %s.%s has an open RT bound as its last item", metadata.ksName, metadata.cfName);
+ String message = String.format("UnfilteredRowIterator for %s has an open RT bound as its last item", partition.metadata());
throw new IllegalStateException(message);
}
diff --git a/src/java/org/apache/cassandra/db/transform/RTBoundValidator.java b/src/java/org/apache/cassandra/db/transform/RTBoundValidator.java
index 1f675cf..eb37f4b 100644
--- a/src/java/org/apache/cassandra/db/transform/RTBoundValidator.java
+++ b/src/java/org/apache/cassandra/db/transform/RTBoundValidator.java
@@ -17,11 +17,11 @@
*/
package org.apache.cassandra.db.transform;
-import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.db.DeletionTime;
import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
import org.apache.cassandra.db.rows.RangeTombstoneMarker;
import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.schema.TableMetadata;
/**
* A validating transformation that sanity-checks the sequence of RT bounds and boundaries in every partition.
@@ -63,13 +63,13 @@
private final static class RowsTransformation extends Transformation
{
private final Stage stage;
- private final CFMetaData metadata;
+ private final TableMetadata metadata;
private final boolean isReverseOrder;
private final boolean enforceIsClosed;
private DeletionTime openMarkerDeletionTime;
- private RowsTransformation(Stage stage, CFMetaData metadata, boolean isReverseOrder, boolean enforceIsClosed)
+ private RowsTransformation(Stage stage, TableMetadata metadata, boolean isReverseOrder, boolean enforceIsClosed)
{
this.stage = stage;
this.metadata = metadata;
@@ -115,8 +115,8 @@
private IllegalStateException ise(String why)
{
- String message = String.format("%s UnfilteredRowIterator for %s.%s has an illegal RT bounds sequence: %s",
- stage, metadata.ksName, metadata.cfName, why);
+ String message =
+ String.format("%s UnfilteredRowIterator for %s has an illegal RT bounds sequence: %s", stage, metadata, why);
throw new IllegalStateException(message);
}
}
diff --git a/src/java/org/apache/cassandra/db/transform/Transformation.java b/src/java/org/apache/cassandra/db/transform/Transformation.java
index 77f91e4..41c76df 100644
--- a/src/java/org/apache/cassandra/db/transform/Transformation.java
+++ b/src/java/org/apache/cassandra/db/transform/Transformation.java
@@ -22,7 +22,7 @@
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.DeletionTime;
-import org.apache.cassandra.db.PartitionColumns;
+import org.apache.cassandra.db.RegularAndStaticColumns;
import org.apache.cassandra.db.partitions.PartitionIterator;
import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
import org.apache.cassandra.db.rows.*;
@@ -116,7 +116,7 @@
* NOTE: same remark than for applyToDeletion: it is only applied to the first iterator in a sequence of iterators
* filled by MoreContents.
*/
- protected PartitionColumns applyToPartitionColumns(PartitionColumns columns)
+ protected RegularAndStaticColumns applyToPartitionColumns(RegularAndStaticColumns columns)
{
return columns;
}
@@ -162,6 +162,7 @@
? (UnfilteredRows) iterator
: new UnfilteredRows(iterator);
}
+
static FilteredRows mutable(RowIterator iterator)
{
return iterator instanceof FilteredRows
@@ -187,7 +188,7 @@
* Using stacked transformations instead of wrapping would result into returning a single row, since the first
* iterator will signal the iterator is stopped.
*/
- static UnfilteredRows wrapIterator(UnfilteredRowIterator iterator, PartitionColumns columns)
+ static UnfilteredRows wrapIterator(UnfilteredRowIterator iterator, RegularAndStaticColumns columns)
{
return new UnfilteredRows(iterator, columns);
}
diff --git a/src/java/org/apache/cassandra/db/transform/UnfilteredPartitions.java b/src/java/org/apache/cassandra/db/transform/UnfilteredPartitions.java
index bad14ad..f0f295f 100644
--- a/src/java/org/apache/cassandra/db/transform/UnfilteredPartitions.java
+++ b/src/java/org/apache/cassandra/db/transform/UnfilteredPartitions.java
@@ -20,27 +20,19 @@
*/
package org.apache.cassandra.db.transform;
-import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
import org.apache.cassandra.db.rows.UnfilteredRowIterator;
final class UnfilteredPartitions extends BasePartitions<UnfilteredRowIterator, UnfilteredPartitionIterator> implements UnfilteredPartitionIterator
{
- final boolean isForThrift;
-
// wrap an iterator for transformation
public UnfilteredPartitions(UnfilteredPartitionIterator input)
{
super(input);
- this.isForThrift = input.isForThrift();
}
- public boolean isForThrift()
- {
- return isForThrift;
- }
-
- public CFMetaData metadata()
+ public TableMetadata metadata()
{
return input.metadata();
}
diff --git a/src/java/org/apache/cassandra/db/transform/UnfilteredRows.java b/src/java/org/apache/cassandra/db/transform/UnfilteredRows.java
index 2dccad7..b8720fc 100644
--- a/src/java/org/apache/cassandra/db/transform/UnfilteredRows.java
+++ b/src/java/org/apache/cassandra/db/transform/UnfilteredRows.java
@@ -21,14 +21,14 @@
package org.apache.cassandra.db.transform;
import org.apache.cassandra.db.DeletionTime;
-import org.apache.cassandra.db.PartitionColumns;
+import org.apache.cassandra.db.RegularAndStaticColumns;
import org.apache.cassandra.db.rows.EncodingStats;
import org.apache.cassandra.db.rows.Unfiltered;
import org.apache.cassandra.db.rows.UnfilteredRowIterator;
final class UnfilteredRows extends BaseRows<Unfiltered, UnfilteredRowIterator> implements UnfilteredRowIterator
{
- private PartitionColumns partitionColumns;
+ private RegularAndStaticColumns regularAndStaticColumns;
private DeletionTime partitionLevelDeletion;
public UnfilteredRows(UnfilteredRowIterator input)
@@ -36,10 +36,10 @@
this(input, input.columns());
}
- public UnfilteredRows(UnfilteredRowIterator input, PartitionColumns columns)
+ public UnfilteredRows(UnfilteredRowIterator input, RegularAndStaticColumns columns)
{
super(input);
- partitionColumns = columns;
+ regularAndStaticColumns = columns;
partitionLevelDeletion = input.partitionLevelDeletion();
}
@@ -47,14 +47,14 @@
void add(Transformation add)
{
super.add(add);
- partitionColumns = add.applyToPartitionColumns(partitionColumns);
+ regularAndStaticColumns = add.applyToPartitionColumns(regularAndStaticColumns);
partitionLevelDeletion = add.applyToDeletion(partitionLevelDeletion);
}
@Override
- public PartitionColumns columns()
+ public RegularAndStaticColumns columns()
{
- return partitionColumns;
+ return regularAndStaticColumns;
}
@Override
diff --git a/src/java/org/apache/cassandra/db/view/TableViews.java b/src/java/org/apache/cassandra/db/view/TableViews.java
index f1f48f6..09490e8 100644
--- a/src/java/org/apache/cassandra/db/view/TableViews.java
+++ b/src/java/org/apache/cassandra/db/view/TableViews.java
@@ -21,18 +21,23 @@
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
+import java.util.stream.Collectors;
+import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterables;
import com.google.common.collect.Iterators;
import com.google.common.collect.PeekingIterator;
-import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.commitlog.CommitLogPosition;
import org.apache.cassandra.db.filter.*;
-import org.apache.cassandra.db.rows.*;
import org.apache.cassandra.db.partitions.*;
+import org.apache.cassandra.db.rows.*;
import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.schema.TableMetadataRef;
import org.apache.cassandra.service.StorageProxy;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.btree.BTreeSet;
@@ -43,16 +48,21 @@
*/
public class TableViews extends AbstractCollection<View>
{
- private final CFMetaData baseTableMetadata;
+ private final TableMetadataRef baseTableMetadata;
// We need this to be thread-safe, but the number of times this is changed (when a view is created in the keyspace)
// is massively exceeded by the number of times it's read (for every mutation on the keyspace), so a copy-on-write
// list is the best option.
private final List<View> views = new CopyOnWriteArrayList();
- public TableViews(CFMetaData baseTableMetadata)
+ public TableViews(TableId id)
{
- this.baseTableMetadata = baseTableMetadata;
+ baseTableMetadata = Schema.instance.getTableMetadataRef(id);
+ }
+
+ public boolean hasViews()
+ {
+ return !views.isEmpty();
}
public int size()
@@ -79,8 +89,8 @@
public Iterable<ColumnFamilyStore> allViewsCfs()
{
- Keyspace keyspace = Keyspace.open(baseTableMetadata.ksName);
- return Iterables.transform(views, view -> keyspace.getColumnFamilyStore(view.getDefinition().viewName));
+ Keyspace keyspace = Keyspace.open(baseTableMetadata.keyspace);
+ return Iterables.transform(views, view -> keyspace.getColumnFamilyStore(view.getDefinition().name()));
}
public void forceBlockingFlush()
@@ -119,7 +129,7 @@
*/
public void pushViewReplicaUpdates(PartitionUpdate update, boolean writeCommitLog, AtomicLong baseComplete)
{
- assert update.metadata().cfId.equals(baseTableMetadata.cfId);
+ assert update.metadata().id.equals(baseTableMetadata.id);
Collection<View> views = updatedViews(update);
if (views.isEmpty())
@@ -169,7 +179,7 @@
int nowInSec,
boolean separateUpdates)
{
- assert updates.metadata().cfId.equals(baseTableMetadata.cfId);
+ assert updates.metadata().id.equals(baseTableMetadata.id);
List<ViewUpdateGenerator> generators = new ArrayList<>(views.size());
for (View view : views)
@@ -192,7 +202,7 @@
Row existingRow;
Row updateRow;
- int cmp = baseTableMetadata.comparator.compare(update, existing);
+ int cmp = baseTableMetadata.get().comparator.compare(update, existing);
if (cmp < 0)
{
// We have an update where there was nothing before
@@ -241,7 +251,7 @@
updateRow = ((Row)updatesIter.next()).withRowDeletion(updatesDeletion.currentDeletion());
}
- addToViewUpdateGenerators(existingRow, updateRow, generators, nowInSec);
+ addToViewUpdateGenerators(existingRow, updateRow, generators);
}
// We only care about more existing rows if the update deletion isn't live, i.e. if we had a partition deletion
@@ -256,13 +266,13 @@
continue;
Row existingRow = (Row)existing;
- addToViewUpdateGenerators(existingRow, emptyRow(existingRow.clustering(), updatesDeletion.currentDeletion()), generators, nowInSec);
+ addToViewUpdateGenerators(existingRow, emptyRow(existingRow.clustering(), updatesDeletion.currentDeletion()), generators);
}
}
if (separateUpdates)
{
- final Collection<Mutation> firstBuild = buildMutations(baseTableMetadata, generators);
+ final Collection<Mutation> firstBuild = buildMutations(baseTableMetadata.get(), generators);
return new Iterator<Collection<Mutation>>()
{
@@ -286,13 +296,12 @@
Row updateRow = (Row) update;
addToViewUpdateGenerators(emptyRow(updateRow.clustering(), existingsDeletion.currentDeletion()),
updateRow,
- generators,
- nowInSec);
+ generators);
// If the updates have been filtered, then we won't have any mutations; we need to make sure that we
// only return if the mutations are empty. Otherwise, we continue to search for an update which is
// not filtered
- Collection<Mutation> mutations = buildMutations(baseTableMetadata, generators);
+ Collection<Mutation> mutations = buildMutations(baseTableMetadata.get(), generators);
if (!mutations.isEmpty())
return mutations;
}
@@ -328,11 +337,10 @@
Row updateRow = (Row) update;
addToViewUpdateGenerators(emptyRow(updateRow.clustering(), existingsDeletion.currentDeletion()),
updateRow,
- generators,
- nowInSec);
+ generators);
}
- return Iterators.singletonIterator(buildMutations(baseTableMetadata, generators));
+ return Iterators.singletonIterator(buildMutations(baseTableMetadata.get(), generators));
}
}
@@ -370,7 +378,7 @@
{
Slices.Builder sliceBuilder = null;
DeletionInfo deletionInfo = updates.deletionInfo();
- CFMetaData metadata = updates.metadata();
+ TableMetadata metadata = updates.metadata();
DecoratedKey key = updates.partitionKey();
// TODO: This is subtle: we need to gather all the slices that we have to fetch between partition del, range tombstones and rows.
if (!deletionInfo.isLive())
@@ -431,8 +439,8 @@
// If we have more than one view, we should merge the queried columns by each views but to keep it simple we just
// include everything. We could change that in the future.
ColumnFilter queriedColumns = views.size() == 1 && metadata.enforceStrictLiveness()
- ? Iterables.getOnlyElement(views).getSelectStatement().queriedColumns()
- : ColumnFilter.all(metadata);
+ ? Iterables.getOnlyElement(views).getSelectStatement().queriedColumns()
+ : ColumnFilter.all(metadata);
// Note that the views could have restrictions on regular columns, but even if that's the case we shouldn't apply those
// when we read, because even if an existing row doesn't match the view filter, the update can change that in which
// case we'll need to know the existing content. There is also no easy way to merge those RowFilter when we have multiple views.
@@ -460,9 +468,8 @@
* @param existingBaseRow the base table row as it is before an update.
* @param updateBaseRow the newly updates made to {@code existingBaseRow}.
* @param generators the view update generators to add the new changes to.
- * @param nowInSec the current time in seconds. Used to decide if data is live or not.
*/
- private static void addToViewUpdateGenerators(Row existingBaseRow, Row updateBaseRow, Collection<ViewUpdateGenerator> generators, int nowInSec)
+ private static void addToViewUpdateGenerators(Row existingBaseRow, Row updateBaseRow, Collection<ViewUpdateGenerator> generators)
{
// Having existing empty is useful, it just means we'll insert a brand new entry for updateBaseRow,
// but if we have no update at all, we shouldn't get there.
@@ -470,7 +477,7 @@
// We allow existingBaseRow to be null, which we treat the same as being empty as an small optimization
// to avoid allocating empty row objects when we know there was nothing existing.
- Row mergedBaseRow = existingBaseRow == null ? updateBaseRow : Rows.merge(existingBaseRow, updateBaseRow, nowInSec);
+ Row mergedBaseRow = existingBaseRow == null ? updateBaseRow : Rows.merge(existingBaseRow, updateBaseRow);
for (ViewUpdateGenerator generator : generators)
generator.addBaseTableUpdate(existingBaseRow, mergedBaseRow);
}
@@ -492,7 +499,7 @@
* @param generators the generators from which to extract the view mutations from.
* @return the mutations created by all the generators in {@code generators}.
*/
- private Collection<Mutation> buildMutations(CFMetaData baseTableMetadata, List<ViewUpdateGenerator> generators)
+ private Collection<Mutation> buildMutations(TableMetadata baseTableMetadata, List<ViewUpdateGenerator> generators)
{
// One view is probably common enough and we can optimize a bit easily
if (generators.size() == 1)
@@ -507,23 +514,23 @@
return mutations;
}
- Map<DecoratedKey, Mutation> mutations = new HashMap<>();
+ Map<DecoratedKey, Mutation.PartitionUpdateCollector> mutations = new HashMap<>();
for (ViewUpdateGenerator generator : generators)
{
for (PartitionUpdate update : generator.generateViewUpdates())
{
DecoratedKey key = update.partitionKey();
- Mutation mutation = mutations.get(key);
- if (mutation == null)
+ Mutation.PartitionUpdateCollector collector = mutations.get(key);
+ if (collector == null)
{
- mutation = new Mutation(baseTableMetadata.ksName, key);
- mutations.put(key, mutation);
+ collector = new Mutation.PartitionUpdateCollector(baseTableMetadata.keyspace, key);
+ mutations.put(key, collector);
}
- mutation.add(update);
+ collector.add(update);
}
generator.clear();
}
- return mutations.values();
+ return mutations.values().stream().map(Mutation.PartitionUpdateCollector::build).collect(Collectors.toList());
}
/**
diff --git a/src/java/org/apache/cassandra/db/view/View.java b/src/java/org/apache/cassandra/db/view/View.java
index d7cd827..af470ff 100644
--- a/src/java/org/apache/cassandra/db/view/View.java
+++ b/src/java/org/apache/cassandra/db/view/View.java
@@ -17,38 +17,27 @@
*/
package org.apache.cassandra.db.view;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-import java.util.UUID;
+import java.util.*;
import java.util.stream.Collectors;
+
import javax.annotation.Nullable;
import com.google.common.collect.Iterables;
+
+import org.apache.cassandra.cql3.*;
+import org.apache.cassandra.cql3.selection.RawSelector;
+import org.apache.cassandra.cql3.statements.SelectStatement;
+import org.apache.cassandra.db.*;
+import org.apache.cassandra.db.rows.*;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.KeyspaceMetadata;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.TableMetadataRef;
+import org.apache.cassandra.schema.ViewMetadata;
+import org.apache.cassandra.utils.FBUtilities;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
-import org.apache.cassandra.config.Schema;
-import org.apache.cassandra.config.ViewDefinition;
-import org.apache.cassandra.cql3.MultiColumnRelation;
-import org.apache.cassandra.cql3.QueryOptions;
-import org.apache.cassandra.cql3.Relation;
-import org.apache.cassandra.cql3.SingleColumnRelation;
-import org.apache.cassandra.cql3.Term;
-import org.apache.cassandra.cql3.statements.ParsedStatement;
-import org.apache.cassandra.cql3.statements.SelectStatement;
-import org.apache.cassandra.db.ColumnFamilyStore;
-import org.apache.cassandra.db.DecoratedKey;
-import org.apache.cassandra.db.ReadQuery;
-import org.apache.cassandra.db.compaction.CompactionManager;
-import org.apache.cassandra.db.rows.Row;
-import org.apache.cassandra.schema.KeyspaceMetadata;
-import org.apache.cassandra.service.ClientState;
-import org.apache.cassandra.utils.FBUtilities;
-
/**
* A View copies data from a base table into a view table which can be queried independently from the
* base. Every update which targets the base table must be fed through the {@link ViewManager} to ensure
@@ -61,47 +50,39 @@
private static final Logger logger = LoggerFactory.getLogger(View.class);
public final String name;
- private volatile ViewDefinition definition;
+ private volatile ViewMetadata definition;
private final ColumnFamilyStore baseCfs;
- public volatile List<ColumnDefinition> baseNonPKColumnsInViewPK;
-
+ public volatile List<ColumnMetadata> baseNonPKColumnsInViewPK;
private ViewBuilder builder;
- // Only the raw statement can be final, because the statement cannot always be prepared when the MV is initialized.
- // For example, during startup, this view will be initialized as part of the Keyspace.open() work; preparing a statement
- // also requires the keyspace to be open, so this results in double-initialization problems.
- private final SelectStatement.RawStatement rawSelect;
private SelectStatement select;
private ReadQuery query;
- public View(ViewDefinition definition,
- ColumnFamilyStore baseCfs)
+ public View(ViewMetadata definition, ColumnFamilyStore baseCfs)
{
this.baseCfs = baseCfs;
- this.name = definition.viewName;
- this.rawSelect = definition.select;
+ this.name = definition.name();
updateDefinition(definition);
}
- public ViewDefinition getDefinition()
+ public ViewMetadata getDefinition()
{
return definition;
}
/**
- * This updates the columns stored which are dependent on the base CFMetaData.
+ * This updates the columns stored which are dependent on the base TableMetadata.
*/
- public void updateDefinition(ViewDefinition definition)
+ public void updateDefinition(ViewMetadata definition)
{
this.definition = definition;
-
- List<ColumnDefinition> nonPKDefPartOfViewPK = new ArrayList<>();
- for (ColumnDefinition baseColumn : baseCfs.metadata.allColumns())
+ List<ColumnMetadata> nonPKDefPartOfViewPK = new ArrayList<>();
+ for (ColumnMetadata baseColumn : baseCfs.metadata.get().columns())
{
- ColumnDefinition viewColumn = getViewColumn(baseColumn);
+ ColumnMetadata viewColumn = getViewColumn(baseColumn);
if (viewColumn != null && !baseColumn.isPrimaryKeyColumn() && viewColumn.isPrimaryKeyColumn())
nonPKDefPartOfViewPK.add(baseColumn);
}
@@ -112,18 +93,18 @@
* The view column corresponding to the provided base column. This <b>can</b>
* return {@code null} if the column is denormalized in the view.
*/
- public ColumnDefinition getViewColumn(ColumnDefinition baseColumn)
+ public ColumnMetadata getViewColumn(ColumnMetadata baseColumn)
{
- return definition.metadata.getColumnDefinition(baseColumn.name);
+ return definition.metadata.getColumn(baseColumn.name);
}
/**
* The base column corresponding to the provided view column. This should
* never return {@code null} since a view can't have its "own" columns.
*/
- public ColumnDefinition getBaseColumn(ColumnDefinition viewColumn)
+ public ColumnMetadata getBaseColumn(ColumnMetadata viewColumn)
{
- ColumnDefinition baseColumn = baseCfs.metadata.getColumnDefinition(viewColumn.name);
+ ColumnMetadata baseColumn = baseCfs.metadata().getColumn(viewColumn.name);
assert baseColumn != null;
return baseColumn;
}
@@ -167,118 +148,95 @@
public boolean matchesViewFilter(DecoratedKey partitionKey, Row baseRow, int nowInSec)
{
return getReadQuery().selectsClustering(partitionKey, baseRow.clustering())
- && getSelectStatement().rowFilterForInternalCalls().isSatisfiedBy(baseCfs.metadata, partitionKey, baseRow, nowInSec);
+ && getSelectStatement().rowFilterForInternalCalls().isSatisfiedBy(baseCfs.metadata(), partitionKey, baseRow, nowInSec);
}
/**
* Returns the SelectStatement used to populate and filter this view. Internal users should access the select
* statement this way to ensure it has been prepared.
*/
- public SelectStatement getSelectStatement()
+ SelectStatement getSelectStatement()
{
- if (select == null)
+ if (null == select)
{
- ClientState state = ClientState.forInternalCalls();
- state.setKeyspace(baseCfs.keyspace.getName());
- rawSelect.prepareKeyspace(state);
- ParsedStatement.Prepared prepared = rawSelect.prepare(true, ClientState.forInternalCalls());
- select = (SelectStatement) prepared.statement;
+ SelectStatement.Parameters parameters =
+ new SelectStatement.Parameters(Collections.emptyMap(),
+ Collections.emptyList(),
+ false,
+ true,
+ false);
+
+ SelectStatement.RawStatement rawSelect =
+ new SelectStatement.RawStatement(new QualifiedName(baseCfs.keyspace.getName(), baseCfs.name),
+ parameters,
+ selectClause(),
+ definition.whereClause,
+ null,
+ null);
+
+ rawSelect.setBindVariables(Collections.emptyList());
+
+ select = rawSelect.prepare(true);
}
return select;
}
+ private List<RawSelector> selectClause()
+ {
+ return definition.metadata
+ .columns()
+ .stream()
+ .map(c -> c.name.toString())
+ .map(ColumnMetadata.Raw::forQuoted)
+ .map(c -> new RawSelector(c, null))
+ .collect(Collectors.toList());
+ }
+
/**
* Returns the ReadQuery used to filter this view. Internal users should access the query this way to ensure it
* has been prepared.
*/
- public ReadQuery getReadQuery()
+ ReadQuery getReadQuery()
{
if (query == null)
- {
query = getSelectStatement().getQuery(QueryOptions.forInternalCalls(Collections.emptyList()), FBUtilities.nowInSeconds());
- logger.trace("View query: {}", rawSelect);
- }
return query;
}
public synchronized void build()
{
- if (this.builder != null)
- {
- logger.debug("Stopping current view builder due to schema change");
- this.builder.stop();
- this.builder = null;
- }
-
- this.builder = new ViewBuilder(baseCfs, this);
- CompactionManager.instance.submitViewBuilder(builder);
- }
-
- @Nullable
- public static CFMetaData findBaseTable(String keyspace, String viewName)
- {
- ViewDefinition view = Schema.instance.getView(keyspace, viewName);
- return (view == null) ? null : Schema.instance.getCFMetaData(view.baseTableId);
- }
-
- public static Iterable<ViewDefinition> findAll(String keyspace, String baseTable)
- {
- KeyspaceMetadata ksm = Schema.instance.getKSMetaData(keyspace);
- final UUID baseId = Schema.instance.getId(keyspace, baseTable);
- return Iterables.filter(ksm.views, view -> view.baseTableId.equals(baseId));
+ stopBuild();
+ builder = new ViewBuilder(baseCfs, this);
+ builder.start();
}
/**
- * Builds the string text for a materialized view's SELECT statement.
+ * Stops the building of this view, no-op if it isn't building.
*/
- public static String buildSelectStatement(String cfName, Collection<ColumnDefinition> includedColumns, String whereClause)
+ synchronized void stopBuild()
{
- StringBuilder rawSelect = new StringBuilder("SELECT ");
- if (includedColumns == null || includedColumns.isEmpty())
- rawSelect.append("*");
- else
- rawSelect.append(includedColumns.stream().map(id -> id.name.toCQLString()).collect(Collectors.joining(", ")));
- rawSelect.append(" FROM \"").append(cfName).append("\" WHERE ") .append(whereClause).append(" ALLOW FILTERING");
- return rawSelect.toString();
+ if (builder != null)
+ {
+ logger.debug("Stopping current view builder due to schema change");
+ builder.stop();
+ builder = null;
+ }
}
- public static String relationsToWhereClause(List<Relation> whereClause)
+ @Nullable
+ public static TableMetadataRef findBaseTable(String keyspace, String viewName)
{
- List<String> expressions = new ArrayList<>(whereClause.size());
- for (Relation rel : whereClause)
- {
- StringBuilder sb = new StringBuilder();
+ ViewMetadata view = Schema.instance.getView(keyspace, viewName);
+ return (view == null) ? null : Schema.instance.getTableMetadataRef(view.baseTableId);
+ }
- if (rel.isMultiColumn())
- {
- sb.append(((MultiColumnRelation) rel).getEntities().stream()
- .map(ColumnDefinition.Raw::toString)
- .collect(Collectors.joining(", ", "(", ")")));
- }
- else
- {
- sb.append(((SingleColumnRelation) rel).getEntity());
- }
-
- sb.append(" ").append(rel.operator()).append(" ");
-
- if (rel.isIN())
- {
- sb.append(rel.getInValues().stream()
- .map(Term.Raw::getText)
- .collect(Collectors.joining(", ", "(", ")")));
- }
- else
- {
- sb.append(rel.getValue().getText());
- }
-
- expressions.add(sb.toString());
- }
-
- return expressions.stream().collect(Collectors.joining(" AND "));
+ // TODO: REMOVE
+ public static Iterable<ViewMetadata> findAll(String keyspace, String baseTable)
+ {
+ KeyspaceMetadata ksm = Schema.instance.getKeyspaceMetadata(keyspace);
+ return Iterables.filter(ksm.views, view -> view.baseTableName.equals(baseTable));
}
public boolean hasSamePrimaryKeyColumnsAsBaseTable()
diff --git a/src/java/org/apache/cassandra/db/view/ViewBuilder.java b/src/java/org/apache/cassandra/db/view/ViewBuilder.java
index c4314f2..6717297 100644
--- a/src/java/org/apache/cassandra/db/view/ViewBuilder.java
+++ b/src/java/org/apache/cassandra/db/view/ViewBuilder.java
@@ -18,216 +18,227 @@
package org.apache.cassandra.db.view;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
import java.util.UUID;
+import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicLong;
import java.util.stream.Collectors;
-import javax.annotation.Nullable;
-
-import com.google.common.base.Function;
-import com.google.common.collect.Iterables;
-
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+import com.google.common.util.concurrent.FutureCallback;
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.MoreExecutors;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+
import org.apache.cassandra.concurrent.ScheduledExecutors;
-import org.apache.cassandra.db.*;
-import org.apache.cassandra.db.compaction.CompactionInfo;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.SystemKeyspace;
+import org.apache.cassandra.db.compaction.CompactionInterruptedException;
import org.apache.cassandra.db.compaction.CompactionManager;
-import org.apache.cassandra.db.compaction.OperationType;
-import org.apache.cassandra.db.compaction.CompactionInfo.Unit;
-import org.apache.cassandra.db.lifecycle.SSTableSet;
-import org.apache.cassandra.db.partitions.*;
-import org.apache.cassandra.db.rows.*;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
-import org.apache.cassandra.io.sstable.ReducingKeyIterator;
-import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.locator.RangesAtEndpoint;
+import org.apache.cassandra.locator.Replicas;
import org.apache.cassandra.repair.SystemDistributedKeyspace;
-import org.apache.cassandra.service.StorageProxy;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.Pair;
-import org.apache.cassandra.utils.UUIDGen;
-import org.apache.cassandra.utils.concurrent.Refs;
-public class ViewBuilder extends CompactionInfo.Holder
+import static java.util.stream.Collectors.toList;
+
+/**
+ * Builds a materialized view for the local token ranges.
+ * <p>
+ * The build is split in at least {@link #NUM_TASKS} {@link ViewBuilderTask tasks}, suitable of being parallelized by
+ * the {@link CompactionManager} which will execute them.
+ */
+class ViewBuilder
{
- private final ColumnFamilyStore baseCfs;
- private final View view;
- private final UUID compactionId;
- private volatile Token prevToken = null;
-
private static final Logger logger = LoggerFactory.getLogger(ViewBuilder.class);
- public ViewBuilder(ColumnFamilyStore baseCfs, View view)
+ private static final int NUM_TASKS = Runtime.getRuntime().availableProcessors() * 4;
+
+ private final ColumnFamilyStore baseCfs;
+ private final View view;
+ private final String ksName;
+ private final UUID localHostId = SystemKeyspace.getLocalHostId();
+ private final Set<Range<Token>> builtRanges = Sets.newConcurrentHashSet();
+ private final Map<Range<Token>, Pair<Token, Long>> pendingRanges = Maps.newConcurrentMap();
+ private final Set<ViewBuilderTask> tasks = Sets.newConcurrentHashSet();
+ private volatile long keysBuilt = 0;
+ private volatile boolean isStopped = false;
+ private volatile Future<?> future = Futures.immediateFuture(null);
+
+ ViewBuilder(ColumnFamilyStore baseCfs, View view)
{
this.baseCfs = baseCfs;
this.view = view;
- compactionId = UUIDGen.getTimeUUID();
+ ksName = baseCfs.metadata.keyspace;
}
- private void buildKey(DecoratedKey key)
+ public void start()
{
- ReadQuery selectQuery = view.getReadQuery();
-
- if (!selectQuery.selectsKey(key))
+ if (SystemKeyspace.isViewBuilt(ksName, view.name))
{
- logger.trace("Skipping {}, view query filters", key);
- return;
- }
-
- int nowInSec = FBUtilities.nowInSeconds();
- SinglePartitionReadCommand command = view.getSelectStatement().internalReadForView(key, nowInSec);
-
- // We're rebuilding everything from what's on disk, so we read everything, consider that as new updates
- // and pretend that there is nothing pre-existing.
- UnfilteredRowIterator empty = UnfilteredRowIterators.noRowsIterator(baseCfs.metadata, key, Rows.EMPTY_STATIC_ROW, DeletionTime.LIVE, false);
-
- try (ReadExecutionController orderGroup = command.executionController();
- UnfilteredRowIterator data = UnfilteredPartitionIterators.getOnlyElement(command.executeLocally(orderGroup), command))
- {
- Iterator<Collection<Mutation>> mutations = baseCfs.keyspace.viewManager
- .forTable(baseCfs.metadata)
- .generateViewUpdates(Collections.singleton(view), data, empty, nowInSec, true);
-
- AtomicLong noBase = new AtomicLong(Long.MAX_VALUE);
- mutations.forEachRemaining(m -> StorageProxy.mutateMV(key.getKey(), m, true, noBase, System.nanoTime()));
- }
- }
-
- public void run()
- {
- logger.debug("Starting view builder for {}.{}", baseCfs.metadata.ksName, view.name);
- logger.trace("Running view builder for {}.{}", baseCfs.metadata.ksName, view.name);
- UUID localHostId = SystemKeyspace.getLocalHostId();
- String ksname = baseCfs.metadata.ksName, viewName = view.name;
-
- if (SystemKeyspace.isViewBuilt(ksname, viewName))
- {
- logger.debug("View already marked built for {}.{}", baseCfs.metadata.ksName, view.name);
- if (!SystemKeyspace.isViewStatusReplicated(ksname, viewName))
- updateDistributed(ksname, viewName, localHostId);
- return;
- }
-
- Iterable<Range<Token>> ranges = StorageService.instance.getLocalRanges(baseCfs.metadata.ksName);
-
- final Pair<Integer, Token> buildStatus = SystemKeyspace.getViewBuildStatus(ksname, viewName);
- Token lastToken;
- Function<org.apache.cassandra.db.lifecycle.View, Iterable<SSTableReader>> function;
- if (buildStatus == null)
- {
- logger.debug("Starting new view build. flushing base table {}.{}", baseCfs.metadata.ksName, baseCfs.name);
- lastToken = null;
-
- //We don't track the generation number anymore since if a rebuild is stopped and
- //restarted the max generation filter may yield no sstables due to compactions.
- //We only care about max generation *during* a build, not across builds.
- //see CASSANDRA-13405
- SystemKeyspace.beginViewBuild(ksname, viewName, 0);
+ logger.debug("View already marked built for {}.{}", ksName, view.name);
+ if (!SystemKeyspace.isViewStatusReplicated(ksName, view.name))
+ updateDistributed();
}
else
{
- lastToken = buildStatus.right;
- logger.debug("Resuming view build from token {}. flushing base table {}.{}", lastToken, baseCfs.metadata.ksName, baseCfs.name);
- }
+ SystemDistributedKeyspace.startViewBuild(ksName, view.name, localHostId);
- baseCfs.forceBlockingFlush();
- function = org.apache.cassandra.db.lifecycle.View.selectFunction(SSTableSet.CANONICAL);
+ logger.debug("Starting build of view({}.{}). Flushing base table {}.{}",
+ ksName, view.name, ksName, baseCfs.name);
+ baseCfs.forceBlockingFlush();
- prevToken = lastToken;
- long keysBuilt = 0;
- try (Refs<SSTableReader> sstables = baseCfs.selectAndReference(function).refs;
- ReducingKeyIterator iter = new ReducingKeyIterator(sstables))
- {
- SystemDistributedKeyspace.startViewBuild(ksname, viewName, localHostId);
- while (!isStopRequested() && iter.hasNext())
- {
- DecoratedKey key = iter.next();
- Token token = key.getToken();
- if (lastToken == null || lastToken.compareTo(token) < 0)
- {
- for (Range<Token> range : ranges)
- {
- if (range.contains(token))
- {
- buildKey(key);
- ++keysBuilt;
-
- if (prevToken == null || prevToken.compareTo(token) != 0)
- {
- SystemKeyspace.updateViewBuildStatus(ksname, viewName, key.getToken());
- prevToken = token;
- }
- }
- }
-
- lastToken = null;
- }
- }
-
- if (!isStopRequested())
- {
- logger.debug("Marking view({}.{}) as built covered {} keys ", ksname, viewName, keysBuilt);
- SystemKeyspace.finishViewBuildStatus(ksname, viewName);
- updateDistributed(ksname, viewName, localHostId);
- }
- else
- {
- logger.debug("Stopped build for view({}.{}) after covering {} keys", ksname, viewName, keysBuilt);
- }
- }
- catch (Exception e)
- {
- ScheduledExecutors.nonPeriodicTasks.schedule(() -> CompactionManager.instance.submitViewBuilder(this),
- 5,
- TimeUnit.MINUTES);
- logger.warn("Materialized View failed to complete, sleeping 5 minutes before restarting", e);
+ loadStatusAndBuild();
}
}
- private void updateDistributed(String ksname, String viewName, UUID localHostId)
+ private void loadStatusAndBuild()
+ {
+ loadStatus();
+ build();
+ }
+
+ private void loadStatus()
+ {
+ builtRanges.clear();
+ pendingRanges.clear();
+ SystemKeyspace.getViewBuildStatus(ksName, view.name)
+ .forEach((range, pair) ->
+ {
+ Token lastToken = pair.left;
+ if (lastToken != null && lastToken.equals(range.right))
+ {
+ builtRanges.add(range);
+ keysBuilt += pair.right;
+ }
+ else
+ {
+ pendingRanges.put(range, pair);
+ }
+ });
+ }
+
+ private synchronized void build()
+ {
+ if (isStopped)
+ {
+ logger.debug("Stopped build for view({}.{}) after covering {} keys", ksName, view.name, keysBuilt);
+ return;
+ }
+
+ // Get the local ranges for which the view hasn't already been built nor it's building
+ RangesAtEndpoint replicatedRanges = StorageService.instance.getLocalReplicas(ksName);
+ Replicas.temporaryAssertFull(replicatedRanges);
+ Set<Range<Token>> newRanges = replicatedRanges.ranges()
+ .stream()
+ .map(r -> r.subtractAll(builtRanges))
+ .flatMap(Set::stream)
+ .map(r -> r.subtractAll(pendingRanges.keySet()))
+ .flatMap(Set::stream)
+ .collect(Collectors.toSet());
+ // If there are no new nor pending ranges we should finish the build
+ if (newRanges.isEmpty() && pendingRanges.isEmpty())
+ {
+ finish();
+ return;
+ }
+
+ // Split the new local ranges and add them to the pending set
+ DatabaseDescriptor.getPartitioner()
+ .splitter()
+ .map(s -> s.split(newRanges, NUM_TASKS))
+ .orElse(newRanges)
+ .forEach(r -> pendingRanges.put(r, Pair.<Token, Long>create(null, 0L)));
+
+ // Submit a new view build task for each building range.
+ // We keep record of all the submitted tasks to be able of stopping them.
+ List<ListenableFuture<Long>> futures = pendingRanges.entrySet()
+ .stream()
+ .map(e -> new ViewBuilderTask(baseCfs,
+ view,
+ e.getKey(),
+ e.getValue().left,
+ e.getValue().right))
+ .peek(tasks::add)
+ .map(CompactionManager.instance::submitViewBuilder)
+ .collect(toList());
+
+ // Add a callback to process any eventual new local range and mark the view as built, doing a delayed retry if
+ // the tasks don't succeed
+ ListenableFuture<List<Long>> future = Futures.allAsList(futures);
+ Futures.addCallback(future, new FutureCallback<List<Long>>()
+ {
+ public void onSuccess(List<Long> result)
+ {
+ keysBuilt += result.stream().mapToLong(x -> x).sum();
+ builtRanges.addAll(pendingRanges.keySet());
+ pendingRanges.clear();
+ build();
+ }
+
+ public void onFailure(Throwable t)
+ {
+ if (t instanceof CompactionInterruptedException)
+ {
+ internalStop(true);
+ keysBuilt = tasks.stream().mapToLong(ViewBuilderTask::keysBuilt).sum();
+ logger.info("Interrupted build for view({}.{}) after covering {} keys", ksName, view.name, keysBuilt);
+ }
+ else
+ {
+ ScheduledExecutors.nonPeriodicTasks.schedule(() -> loadStatusAndBuild(), 5, TimeUnit.MINUTES);
+ logger.warn("Materialized View failed to complete, sleeping 5 minutes before restarting", t);
+ }
+ }
+ }, MoreExecutors.directExecutor());
+ this.future = future;
+ }
+
+ private void finish()
+ {
+ logger.debug("Marking view({}.{}) as built after covering {} keys ", ksName, view.name, keysBuilt);
+ SystemKeyspace.finishViewBuildStatus(ksName, view.name);
+ updateDistributed();
+ }
+
+ private void updateDistributed()
{
try
{
- SystemDistributedKeyspace.successfulViewBuild(ksname, viewName, localHostId);
- SystemKeyspace.setViewBuiltReplicated(ksname, viewName);
+ SystemDistributedKeyspace.successfulViewBuild(ksName, view.name, localHostId);
+ SystemKeyspace.setViewBuiltReplicated(ksName, view.name);
}
catch (Exception e)
{
- ScheduledExecutors.nonPeriodicTasks.schedule(() -> CompactionManager.instance.submitViewBuilder(this),
- 5,
- TimeUnit.MINUTES);
- logger.warn("Failed to updated the distributed status of view, sleeping 5 minutes before retrying", e);
+ ScheduledExecutors.nonPeriodicTasks.schedule(this::updateDistributed, 5, TimeUnit.MINUTES);
+ logger.warn("Failed to update the distributed status of view, sleeping 5 minutes before retrying", e);
}
}
- public CompactionInfo getCompactionInfo()
+ /**
+ * Stops the view building.
+ */
+ synchronized void stop()
{
- long rangesCompleted = 0, rangesTotal = 0;
- Token lastToken = prevToken;
-
- // This approximation is not very accurate, but since we do not have a method which allows us to calculate the
- // percentage of a range covered by a second range, this is the best approximation that we can calculate.
- // Instead, we just count the total number of ranges that haven't been seen by the node (we use the order of
- // the tokens to determine whether they have been seen yet or not), and the total number of ranges that a node
- // has.
- for (Range<Token> range : StorageService.instance.getLocalRanges(baseCfs.keyspace.getName()))
- {
- rangesTotal++;
- if ((lastToken != null) && lastToken.compareTo(range.right) > 0)
- rangesCompleted++;
- }
- return new CompactionInfo(baseCfs.metadata, OperationType.VIEW_BUILD, rangesCompleted, rangesTotal, Unit.RANGES, compactionId);
+ boolean wasStopped = isStopped;
+ internalStop(false);
+ if (!wasStopped)
+ FBUtilities.waitOnFuture(future);
}
- public boolean isGlobal()
+ private void internalStop(boolean isCompactionInterrupted)
{
- return false;
+ isStopped = true;
+ tasks.forEach(task -> task.stop(isCompactionInterrupted));
}
}
diff --git a/src/java/org/apache/cassandra/db/view/ViewBuilderTask.java b/src/java/org/apache/cassandra/db/view/ViewBuilderTask.java
new file mode 100644
index 0000000..c84c697
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/view/ViewBuilderTask.java
@@ -0,0 +1,274 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.view;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.UUID;
+import java.util.concurrent.Callable;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Function;
+import com.google.common.base.Objects;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterators;
+import com.google.common.collect.PeekingIterator;
+import com.google.common.util.concurrent.Futures;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.DeletionTime;
+import org.apache.cassandra.db.Mutation;
+import org.apache.cassandra.db.ReadExecutionController;
+import org.apache.cassandra.db.ReadQuery;
+import org.apache.cassandra.db.SinglePartitionReadCommand;
+import org.apache.cassandra.db.SystemKeyspace;
+import org.apache.cassandra.db.compaction.CompactionInfo;
+import org.apache.cassandra.db.compaction.CompactionInfo.Unit;
+import org.apache.cassandra.db.compaction.CompactionInterruptedException;
+import org.apache.cassandra.db.compaction.OperationType;
+import org.apache.cassandra.db.lifecycle.SSTableSet;
+import org.apache.cassandra.db.partitions.UnfilteredPartitionIterators;
+import org.apache.cassandra.db.rows.Rows;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.db.rows.UnfilteredRowIterators;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.gms.Gossiper;
+import org.apache.cassandra.io.sstable.ReducingKeyIterator;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.service.StorageProxy;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.UUIDGen;
+import org.apache.cassandra.utils.concurrent.Refs;
+
+public class ViewBuilderTask extends CompactionInfo.Holder implements Callable<Long>
+{
+ private static final Logger logger = LoggerFactory.getLogger(ViewBuilderTask.class);
+
+ private static final int ROWS_BETWEEN_CHECKPOINTS = 1000;
+
+ private final ColumnFamilyStore baseCfs;
+ private final View view;
+ private final Range<Token> range;
+ private final UUID compactionId;
+ private volatile Token prevToken;
+ private volatile long keysBuilt = 0;
+ private volatile boolean isStopped = false;
+ private volatile boolean isCompactionInterrupted = false;
+
+ @VisibleForTesting
+ public ViewBuilderTask(ColumnFamilyStore baseCfs, View view, Range<Token> range, Token lastToken, long keysBuilt)
+ {
+ this.baseCfs = baseCfs;
+ this.view = view;
+ this.range = range;
+ this.compactionId = UUIDGen.getTimeUUID();
+ this.prevToken = lastToken;
+ this.keysBuilt = keysBuilt;
+ }
+
+ @SuppressWarnings("resource")
+ private void buildKey(DecoratedKey key)
+ {
+ ReadQuery selectQuery = view.getReadQuery();
+
+ if (!selectQuery.selectsKey(key))
+ {
+ logger.trace("Skipping {}, view query filters", key);
+ return;
+ }
+
+ int nowInSec = FBUtilities.nowInSeconds();
+ SinglePartitionReadCommand command = view.getSelectStatement().internalReadForView(key, nowInSec);
+
+ // We're rebuilding everything from what's on disk, so we read everything, consider that as new updates
+ // and pretend that there is nothing pre-existing.
+ UnfilteredRowIterator empty = UnfilteredRowIterators.noRowsIterator(baseCfs.metadata(), key, Rows.EMPTY_STATIC_ROW, DeletionTime.LIVE, false);
+
+ try (ReadExecutionController orderGroup = command.executionController();
+ UnfilteredRowIterator data = UnfilteredPartitionIterators.getOnlyElement(command.executeLocally(orderGroup), command))
+ {
+ Iterator<Collection<Mutation>> mutations = baseCfs.keyspace.viewManager
+ .forTable(baseCfs.metadata.id)
+ .generateViewUpdates(Collections.singleton(view), data, empty, nowInSec, true);
+
+ AtomicLong noBase = new AtomicLong(Long.MAX_VALUE);
+ mutations.forEachRemaining(m -> StorageProxy.mutateMV(key.getKey(), m, true, noBase, System.nanoTime()));
+ }
+ }
+
+ public Long call()
+ {
+ String ksName = baseCfs.metadata.keyspace;
+
+ if (prevToken == null)
+ logger.debug("Starting new view build for range {}", range);
+ else
+ logger.debug("Resuming view build for range {} from token {} with {} covered keys", range, prevToken, keysBuilt);
+
+ /*
+ * It's possible for view building to start before MV creation got propagated to other nodes. For this reason
+ * we should wait for schema to converge before attempting to send any view mutations to other nodes, or else
+ * face UnknownTableException upon Mutation deserialization on the nodes that haven't processed the schema change.
+ */
+ boolean schemaConverged = Gossiper.instance.waitForSchemaAgreement(10, TimeUnit.SECONDS, () -> this.isStopped);
+ if (!schemaConverged)
+ logger.warn("Failed to get schema to converge before building view {}.{}", baseCfs.keyspace.getName(), view.name);
+
+ Function<org.apache.cassandra.db.lifecycle.View, Iterable<SSTableReader>> function;
+ function = org.apache.cassandra.db.lifecycle.View.select(SSTableSet.CANONICAL, s -> range.intersects(s.getBounds()));
+
+ try (ColumnFamilyStore.RefViewFragment viewFragment = baseCfs.selectAndReference(function);
+ Refs<SSTableReader> sstables = viewFragment.refs;
+ ReducingKeyIterator keyIter = new ReducingKeyIterator(sstables))
+ {
+ PeekingIterator<DecoratedKey> iter = Iterators.peekingIterator(keyIter);
+ while (!isStopped && iter.hasNext())
+ {
+ DecoratedKey key = iter.next();
+ Token token = key.getToken();
+ //skip tokens already built or not present in range
+ if (range.contains(token) && (prevToken == null || token.compareTo(prevToken) > 0))
+ {
+ buildKey(key);
+ ++keysBuilt;
+ //build other keys sharing the same token
+ while (iter.hasNext() && iter.peek().getToken().equals(token))
+ {
+ key = iter.next();
+ buildKey(key);
+ ++keysBuilt;
+ }
+ if (keysBuilt % ROWS_BETWEEN_CHECKPOINTS == 1)
+ SystemKeyspace.updateViewBuildStatus(ksName, view.name, range, token, keysBuilt);
+ prevToken = token;
+ }
+ }
+ }
+
+ finish();
+
+ return keysBuilt;
+ }
+
+ private void finish()
+ {
+ String ksName = baseCfs.keyspace.getName();
+ if (!isStopped)
+ {
+ // Save the completed status using the end of the range as last token. This way it will be possible for
+ // future view build attempts to don't even create a task for this range
+ SystemKeyspace.updateViewBuildStatus(ksName, view.name, range, range.right, keysBuilt);
+
+ logger.debug("Completed build of view({}.{}) for range {} after covering {} keys ", ksName, view.name, range, keysBuilt);
+ }
+ else
+ {
+ logger.debug("Stopped build for view({}.{}) for range {} after covering {} keys", ksName, view.name, range, keysBuilt);
+
+ // If it's stopped due to a compaction interruption we should throw that exception.
+ // Otherwise we assume that the task has been stopped due to a schema update and we can finish successfully.
+ if (isCompactionInterrupted)
+ throw new StoppedException(ksName, view.name, getCompactionInfo());
+ }
+ }
+
+ @Override
+ public CompactionInfo getCompactionInfo()
+ {
+ // we don't know the sstables at construction of ViewBuilderTask and we could change this to return once we know the
+ // but since we basically only cancel view builds on truncation where we cancel all compactions anyway, this seems reasonable
+
+ // If there's splitter, calculate progress based on last token position
+ if (range.left.getPartitioner().splitter().isPresent())
+ {
+ long progress = prevToken == null ? 0 : Math.round(prevToken.getPartitioner().splitter().get().positionInRange(prevToken, range) * 1000);
+ return CompactionInfo.withoutSSTables(baseCfs.metadata(), OperationType.VIEW_BUILD, progress, 1000, Unit.RANGES, compactionId);
+ }
+
+ // When there is no splitter, estimate based on number of total keys but
+ // take the max with keysBuilt + 1 to avoid having more completed than total
+ long keysTotal = Math.max(keysBuilt + 1, baseCfs.estimatedKeysForRange(range));
+ return CompactionInfo.withoutSSTables(baseCfs.metadata(), OperationType.VIEW_BUILD, keysBuilt, keysTotal, Unit.KEYS, compactionId);
+ }
+
+ @Override
+ public void stop()
+ {
+ stop(true);
+ }
+
+ public boolean isGlobal()
+ {
+ return false;
+ }
+
+ synchronized void stop(boolean isCompactionInterrupted)
+ {
+ isStopped = true;
+ this.isCompactionInterrupted = isCompactionInterrupted;
+ }
+
+ long keysBuilt()
+ {
+ return keysBuilt;
+ }
+
+ /**
+ * {@link CompactionInterruptedException} with {@link Object#equals(Object)} and {@link Object#hashCode()}
+ * implementations that consider equals all the exceptions produced by the same view build, independently of their
+ * token range.
+ * <p>
+ * This is used to avoid Guava's {@link Futures#allAsList(Iterable)} log spamming when multiple build tasks fail
+ * due to compaction interruption.
+ */
+ static class StoppedException extends CompactionInterruptedException
+ {
+ private final String ksName, viewName;
+
+ private StoppedException(String ksName, String viewName, CompactionInfo info)
+ {
+ super(info);
+ this.ksName = ksName;
+ this.viewName = viewName;
+ }
+
+ @Override
+ public boolean equals(Object o)
+ {
+ if (!(o instanceof StoppedException))
+ return false;
+
+ StoppedException that = (StoppedException) o;
+ return Objects.equal(this.ksName, that.ksName) && Objects.equal(this.viewName, that.viewName);
+ }
+
+ @Override
+ public int hashCode()
+ {
+ return 31 * ksName.hashCode() + viewName.hashCode();
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/db/view/ViewManager.java b/src/java/org/apache/cassandra/db/view/ViewManager.java
index cb1e02b..7e3ea1b 100644
--- a/src/java/org/apache/cassandra/db/view/ViewManager.java
+++ b/src/java/org/apache/cassandra/db/view/ViewManager.java
@@ -22,16 +22,18 @@
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.locks.Lock;
+import com.google.common.collect.Maps;
import com.google.common.util.concurrent.Striped;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.config.ViewDefinition;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.schema.ViewMetadata;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.partitions.*;
import org.apache.cassandra.repair.SystemDistributedKeyspace;
+import org.apache.cassandra.schema.Views;
import org.apache.cassandra.service.StorageService;
/**
@@ -58,7 +60,7 @@
private static final boolean enableCoordinatorBatchlog = Boolean.getBoolean("cassandra.mv_enable_coordinator_batchlog");
private final ConcurrentMap<String, View> viewsByName = new ConcurrentHashMap<>();
- private final ConcurrentMap<UUID, TableViews> viewsByBaseTable = new ConcurrentHashMap<>();
+ private final ConcurrentMap<TableId, TableViews> viewsByBaseTable = new ConcurrentHashMap<>();
private final Keyspace keyspace;
public ViewManager(Keyspace keyspace)
@@ -75,12 +77,12 @@
{
for (PartitionUpdate update : mutation.getPartitionUpdates())
{
- assert keyspace.getName().equals(update.metadata().ksName);
+ assert keyspace.getName().equals(update.metadata().keyspace);
- if (coordinatorBatchlog && keyspace.getReplicationStrategy().getReplicationFactor() == 1)
+ if (coordinatorBatchlog && keyspace.getReplicationStrategy().getReplicationFactor().allReplicas == 1)
continue;
- if (!forTable(update.metadata()).updatedViews(update).isEmpty())
+ if (!forTable(update.metadata().id).updatedViews(update).isEmpty())
return true;
}
}
@@ -93,38 +95,24 @@
return viewsByName.values();
}
- public void update(String viewName)
+ public void reload(boolean buildAllViews)
{
- View view = viewsByName.get(viewName);
- assert view != null : "When updating a view, it should already be in the ViewManager";
- view.build();
-
- // We provide the new definition from the base metadata
- Optional<ViewDefinition> viewDefinition = keyspace.getMetadata().views.get(viewName);
- assert viewDefinition.isPresent() : "When updating a view, it should still be in the Keyspaces views";
- view.updateDefinition(viewDefinition.get());
- }
-
- public void reload()
- {
- Map<String, ViewDefinition> newViewsByName = new HashMap<>();
- for (ViewDefinition definition : keyspace.getMetadata().views)
+ Views views = keyspace.getMetadata().views;
+ Map<String, ViewMetadata> newViewsByName = Maps.newHashMapWithExpectedSize(views.size());
+ for (ViewMetadata definition : views)
{
- newViewsByName.put(definition.viewName, definition);
+ newViewsByName.put(definition.name(), definition);
}
- for (String viewName : viewsByName.keySet())
- {
- if (!newViewsByName.containsKey(viewName))
- removeView(viewName);
- }
-
- for (Map.Entry<String, ViewDefinition> entry : newViewsByName.entrySet())
+ for (Map.Entry<String, ViewMetadata> entry : newViewsByName.entrySet())
{
if (!viewsByName.containsKey(entry.getKey()))
addView(entry.getValue());
}
+ if (!buildAllViews)
+ return;
+
// Building views involves updating view build status in the system_distributed
// keyspace and therefore it requires ring information. This check prevents builds
// being submitted when Keyspaces are initialized during CassandraDaemon::setup as
@@ -147,30 +135,36 @@
}
}
- public void addView(ViewDefinition definition)
+ public void addView(ViewMetadata definition)
{
// Skip if the base table doesn't exist due to schema propagation issues, see CASSANDRA-13737
if (!keyspace.hasColumnFamilyStore(definition.baseTableId))
{
logger.warn("Not adding view {} because the base table {} is unknown",
- definition.viewName,
+ definition.name(),
definition.baseTableId);
return;
}
View view = new View(definition, keyspace.getColumnFamilyStore(definition.baseTableId));
- forTable(view.getDefinition().baseTableMetadata()).add(view);
- viewsByName.put(definition.viewName, view);
+ forTable(view.getDefinition().baseTableId).add(view);
+ viewsByName.put(definition.name(), view);
}
- public void removeView(String name)
+ /**
+ * Stops the building of the specified view, no-op if it isn't building.
+ *
+ * @param name the name of the view
+ */
+ public void dropView(String name)
{
View view = viewsByName.remove(name);
if (view == null)
return;
- forTable(view.getDefinition().baseTableMetadata()).removeByName(name);
+ view.stopBuild();
+ forTable(view.getDefinition().baseTableId).removeByName(name);
SystemKeyspace.setViewRemoved(keyspace.getName(), view.name);
SystemDistributedKeyspace.setViewRemoved(keyspace.getName(), view.name);
}
@@ -186,14 +180,13 @@
view.build();
}
- public TableViews forTable(CFMetaData metadata)
+ public TableViews forTable(TableId id)
{
- UUID baseId = metadata.cfId;
- TableViews views = viewsByBaseTable.get(baseId);
+ TableViews views = viewsByBaseTable.get(id);
if (views == null)
{
- views = new TableViews(metadata);
- TableViews previous = viewsByBaseTable.putIfAbsent(baseId, views);
+ views = new TableViews(id);
+ TableViews previous = viewsByBaseTable.putIfAbsent(id, views);
if (previous != null)
views = previous;
}
diff --git a/src/java/org/apache/cassandra/db/view/ViewUpdateGenerator.java b/src/java/org/apache/cassandra/db/view/ViewUpdateGenerator.java
index 7937e05..73ca240 100644
--- a/src/java/org/apache/cassandra/db/view/ViewUpdateGenerator.java
+++ b/src/java/org/apache/cassandra/db/view/ViewUpdateGenerator.java
@@ -19,12 +19,14 @@
import java.nio.ByteBuffer;
import java.util.*;
+import java.util.stream.Collectors;
import com.google.common.collect.Iterators;
import com.google.common.collect.PeekingIterator;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.rows.*;
import org.apache.cassandra.db.partitions.*;
@@ -45,14 +47,14 @@
private final View view;
private final int nowInSec;
- private final CFMetaData baseMetadata;
+ private final TableMetadata baseMetadata;
private final DecoratedKey baseDecoratedKey;
private final ByteBuffer[] basePartitionKey;
- private final CFMetaData viewMetadata;
+ private final TableMetadata viewMetadata;
private final boolean baseEnforceStrictLiveness;
- private final Map<DecoratedKey, PartitionUpdate> updates = new HashMap<>();
+ private final Map<DecoratedKey, PartitionUpdate.Builder> updates = new HashMap<>();
// Reused internally to build a new entry
private final ByteBuffer[] currentViewEntryPartitionKey;
@@ -89,9 +91,9 @@
this.baseMetadata = view.getDefinition().baseTableMetadata();
this.baseEnforceStrictLiveness = baseMetadata.enforceStrictLiveness();
this.baseDecoratedKey = basePartitionKey;
- this.basePartitionKey = extractKeyComponents(basePartitionKey, baseMetadata.getKeyValidator());
+ this.basePartitionKey = extractKeyComponents(basePartitionKey, baseMetadata.partitionKeyType);
- this.viewMetadata = view.getDefinition().metadata;
+ this.viewMetadata = Schema.instance.getTableMetadata(view.getDefinition().metadata.id);
this.currentViewEntryPartitionKey = new ByteBuffer[viewMetadata.partitionKeyColumns().size()];
this.currentViewEntryBuilder = BTreeRow.sortedBuilder();
@@ -142,7 +144,7 @@
*/
public Collection<PartitionUpdate> generateViewUpdates()
{
- return updates.values();
+ return updates.values().stream().map(PartitionUpdate.Builder::build).collect(Collectors.toList());
}
/**
@@ -194,7 +196,7 @@
: (mergedHasLiveData ? UpdateAction.NEW_ENTRY : UpdateAction.NONE);
}
- ColumnDefinition baseColumn = view.baseNonPKColumnsInViewPK.get(0);
+ ColumnMetadata baseColumn = view.baseNonPKColumnsInViewPK.get(0);
assert !baseColumn.isComplex() : "A complex column couldn't be part of the view PK";
Cell before = existingBaseRow == null ? null : existingBaseRow.getCell(baseColumn);
Cell after = mergedBaseRow.getCell(baseColumn);
@@ -242,7 +244,7 @@
for (ColumnData data : baseRow)
{
- ColumnDefinition viewColumn = view.getViewColumn(data.column());
+ ColumnMetadata viewColumn = view.getViewColumn(data.column());
// If that base table column is not denormalized in the view, we had nothing to do.
// Alose, if it's part of the view PK it's already been taken into account in the clustering.
if (viewColumn == null || viewColumn.isPrimaryKeyColumn())
@@ -304,8 +306,8 @@
PeekingIterator<ColumnData> existingIter = Iterators.peekingIterator(existingBaseRow.iterator());
for (ColumnData mergedData : mergedBaseRow)
{
- ColumnDefinition baseColumn = mergedData.column();
- ColumnDefinition viewColumn = view.getViewColumn(baseColumn);
+ ColumnMetadata baseColumn = mergedData.column();
+ ColumnMetadata viewColumn = view.getViewColumn(baseColumn);
// If that base table column is not denormalized in the view, we had nothing to do.
// Alose, if it's part of the view PK it's already been taken into account in the clustering.
if (viewColumn == null || viewColumn.isPrimaryKeyColumn())
@@ -394,8 +396,8 @@
long timestamp = computeTimestampForEntryDeletion(existingBaseRow, mergedBaseRow);
long rowDeletion = mergedBaseRow.deletion().time().markedForDeleteAt();
assert timestamp >= rowDeletion;
-
- // If computed deletion timestamp greater than row deletion, it must be coming from
+
+ // If computed deletion timestamp greater than row deletion, it must be coming from
// 1. non-pk base column used in view pk, or
// 2. unselected base column
// any case, we need to use it as expired livenessInfo
@@ -428,9 +430,9 @@
private void startNewUpdate(Row baseRow)
{
ByteBuffer[] clusteringValues = new ByteBuffer[viewMetadata.clusteringColumns().size()];
- for (ColumnDefinition viewColumn : viewMetadata.primaryKeyColumns())
+ for (ColumnMetadata viewColumn : viewMetadata.primaryKeyColumns())
{
- ColumnDefinition baseColumn = view.getBaseColumn(viewColumn);
+ ColumnMetadata baseColumn = view.getBaseColumn(viewColumn);
ByteBuffer value = getValueForPK(baseColumn, baseRow);
if (viewColumn.isPartitionKey())
currentViewEntryPartitionKey[viewColumn.position()] = value;
@@ -461,7 +463,7 @@
LivenessInfo baseLiveness = baseRow.primaryKeyLivenessInfo();
- if (view.hasSamePrimaryKeyColumnsAsBaseTable())
+ if (view.baseNonPKColumnsInViewPK.isEmpty())
{
if (view.getDefinition().includeAllColumns)
return baseLiveness;
@@ -531,7 +533,7 @@
return deletion.deletes(before) ? deletion.markedForDeleteAt() : before.timestamp();
}
- private void addColumnData(ColumnDefinition viewColumn, ColumnData baseTableData)
+ private void addColumnData(ColumnMetadata viewColumn, ColumnData baseTableData)
{
assert viewColumn.isComplex() == baseTableData.column().isComplex();
if (!viewColumn.isComplex())
@@ -546,7 +548,7 @@
addCell(viewColumn, cell);
}
- private void addCell(ColumnDefinition viewColumn, Cell baseTableCell)
+ private void addCell(ColumnMetadata viewColumn, Cell baseTableCell)
{
assert !viewColumn.isPrimaryKeyColumn();
currentViewEntryBuilder.addCell(baseTableCell.withUpdatedColumn(viewColumn));
@@ -565,14 +567,13 @@
return;
DecoratedKey partitionKey = makeCurrentPartitionKey();
- PartitionUpdate update = updates.get(partitionKey);
- if (update == null)
- {
- // We can't really know which columns of the view will be updated nor how many row will be updated for this key
- // so we rely on hopefully sane defaults.
- update = new PartitionUpdate(viewMetadata, partitionKey, viewMetadata.partitionColumns(), 4);
- updates.put(partitionKey, update);
- }
+ // We can't really know which columns of the view will be updated nor how many row will be updated for this key
+ // so we rely on hopefully sane defaults.
+ PartitionUpdate.Builder update = updates.computeIfAbsent(partitionKey,
+ k -> new PartitionUpdate.Builder(viewMetadata,
+ partitionKey,
+ viewMetadata.regularAndStaticColumns(),
+ 4));
update.add(row);
}
@@ -582,10 +583,10 @@
? currentViewEntryPartitionKey[0]
: CompositeType.build(currentViewEntryPartitionKey);
- return viewMetadata.decorateKey(rawKey);
+ return viewMetadata.partitioner.decorateKey(rawKey);
}
- private ByteBuffer getValueForPK(ColumnDefinition column, Row row)
+ private ByteBuffer getValueForPK(ColumnMetadata column, Row row)
{
switch (column.kind)
{
diff --git a/src/java/org/apache/cassandra/db/view/ViewUtils.java b/src/java/org/apache/cassandra/db/view/ViewUtils.java
index 4dc1766..e824732 100644
--- a/src/java/org/apache/cassandra/db/view/ViewUtils.java
+++ b/src/java/org/apache/cassandra/db/view/ViewUtils.java
@@ -18,16 +18,17 @@
package org.apache.cassandra.db.view;
-import java.net.InetAddress;
-import java.util.ArrayList;
-import java.util.List;
import java.util.Optional;
+import java.util.function.Predicate;
+import com.google.common.collect.Iterables;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.locator.AbstractReplicationStrategy;
+import org.apache.cassandra.locator.EndpointsForToken;
import org.apache.cassandra.locator.NetworkTopologyStrategy;
+import org.apache.cassandra.locator.Replica;
import org.apache.cassandra.utils.FBUtilities;
public final class ViewUtils
@@ -58,46 +59,51 @@
*
* @return Optional.empty() if this method is called using a base token which does not belong to this replica
*/
- public static Optional<InetAddress> getViewNaturalEndpoint(String keyspaceName, Token baseToken, Token viewToken)
+ public static Optional<Replica> getViewNaturalEndpoint(String keyspaceName, Token baseToken, Token viewToken)
{
AbstractReplicationStrategy replicationStrategy = Keyspace.open(keyspaceName).getReplicationStrategy();
- String localDataCenter = DatabaseDescriptor.getEndpointSnitch().getDatacenter(FBUtilities.getBroadcastAddress());
- List<InetAddress> baseEndpoints = new ArrayList<>();
- List<InetAddress> viewEndpoints = new ArrayList<>();
- for (InetAddress baseEndpoint : replicationStrategy.getNaturalEndpoints(baseToken))
- {
- // An endpoint is local if we're not using Net
- if (!(replicationStrategy instanceof NetworkTopologyStrategy) ||
- DatabaseDescriptor.getEndpointSnitch().getDatacenter(baseEndpoint).equals(localDataCenter))
- baseEndpoints.add(baseEndpoint);
- }
+ String localDataCenter = DatabaseDescriptor.getEndpointSnitch().getLocalDatacenter();
+ EndpointsForToken naturalBaseReplicas = replicationStrategy.getNaturalReplicasForToken(baseToken);
+ EndpointsForToken naturalViewReplicas = replicationStrategy.getNaturalReplicasForToken(viewToken);
- for (InetAddress viewEndpoint : replicationStrategy.getNaturalEndpoints(viewToken))
- {
- // If we are a base endpoint which is also a view replica, we use ourselves as our view replica
- if (viewEndpoint.equals(FBUtilities.getBroadcastAddress()))
- return Optional.of(viewEndpoint);
+ Optional<Replica> localReplica = Iterables.tryFind(naturalViewReplicas, Replica::isSelf).toJavaUtil();
+ if (localReplica.isPresent())
+ return localReplica;
- // We have to remove any endpoint which is shared between the base and the view, as it will select itself
- // and throw off the counts otherwise.
- if (baseEndpoints.contains(viewEndpoint))
- baseEndpoints.remove(viewEndpoint);
- else if (!(replicationStrategy instanceof NetworkTopologyStrategy) ||
- DatabaseDescriptor.getEndpointSnitch().getDatacenter(viewEndpoint).equals(localDataCenter))
- viewEndpoints.add(viewEndpoint);
- }
+ // We only select replicas from our own DC
+ // TODO: this is poor encapsulation, leaking implementation details of replication strategy
+ Predicate<Replica> isLocalDC = r -> !(replicationStrategy instanceof NetworkTopologyStrategy)
+ || DatabaseDescriptor.getEndpointSnitch().getDatacenter(r).equals(localDataCenter);
+
+ // We have to remove any endpoint which is shared between the base and the view, as it will select itself
+ // and throw off the counts otherwise.
+ EndpointsForToken baseReplicas = naturalBaseReplicas.filter(
+ r -> !naturalViewReplicas.endpoints().contains(r.endpoint()) && isLocalDC.test(r)
+ );
+ EndpointsForToken viewReplicas = naturalViewReplicas.filter(
+ r -> !naturalBaseReplicas.endpoints().contains(r.endpoint()) && isLocalDC.test(r)
+ );
// The replication strategy will be the same for the base and the view, as they must belong to the same keyspace.
// Since the same replication strategy is used, the same placement should be used and we should get the same
// number of replicas for all of the tokens in the ring.
- assert baseEndpoints.size() == viewEndpoints.size() : "Replication strategy should have the same number of endpoints for the base and the view";
- int baseIdx = baseEndpoints.indexOf(FBUtilities.getBroadcastAddress());
+ assert baseReplicas.size() == viewReplicas.size() : "Replication strategy should have the same number of endpoints for the base and the view";
+
+ int baseIdx = -1;
+ for (int i=0; i<baseReplicas.size(); i++)
+ {
+ if (baseReplicas.get(i).isSelf())
+ {
+ baseIdx = i;
+ break;
+ }
+ }
if (baseIdx < 0)
//This node is not a base replica of this key, so we return empty
return Optional.empty();
- return Optional.of(viewEndpoints.get(baseIdx));
+ return Optional.of(viewReplicas.get(baseIdx));
}
}
diff --git a/src/java/org/apache/cassandra/db/virtual/AbstractVirtualTable.java b/src/java/org/apache/cassandra/db/virtual/AbstractVirtualTable.java
new file mode 100644
index 0000000..6c49b9a
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/virtual/AbstractVirtualTable.java
@@ -0,0 +1,222 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db.virtual;
+
+import java.util.Iterator;
+import java.util.NavigableMap;
+
+import com.google.common.collect.AbstractIterator;
+
+import org.apache.cassandra.db.DataRange;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.EmptyIterators;
+import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.db.filter.ClusteringIndexFilter;
+import org.apache.cassandra.db.filter.ColumnFilter;
+import org.apache.cassandra.db.partitions.AbstractUnfilteredPartitionIterator;
+import org.apache.cassandra.db.partitions.PartitionUpdate;
+import org.apache.cassandra.db.partitions.SingletonUnfilteredPartitionIterator;
+import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.schema.TableMetadata;
+
+/**
+ * An abstract virtual table implementation that builds the resultset on demand.
+ */
+public abstract class AbstractVirtualTable implements VirtualTable
+{
+ protected final TableMetadata metadata;
+
+ protected AbstractVirtualTable(TableMetadata metadata)
+ {
+ if (!metadata.isVirtual())
+ throw new IllegalArgumentException();
+
+ this.metadata = metadata;
+ }
+
+ public TableMetadata metadata()
+ {
+ return metadata;
+ }
+
+ /**
+ * Provide a {@link DataSet} that is contains all of the virtual table's data.
+ */
+ public abstract DataSet data();
+
+ /**
+ * Provide a {@link DataSet} that is potentially restricted to the provided partition - but is allowed to contain
+ * other partitions.
+ */
+ public DataSet data(DecoratedKey partitionKey)
+ {
+ return data();
+ }
+
+ @Override
+ @SuppressWarnings("resource")
+ public final UnfilteredPartitionIterator select(DecoratedKey partitionKey, ClusteringIndexFilter clusteringIndexFilter, ColumnFilter columnFilter)
+ {
+ Partition partition = data(partitionKey).getPartition(partitionKey);
+
+ if (null == partition)
+ return EmptyIterators.unfilteredPartition(metadata);
+
+ long now = System.currentTimeMillis();
+ UnfilteredRowIterator rowIterator = partition.toRowIterator(metadata(), clusteringIndexFilter, columnFilter, now);
+ return new SingletonUnfilteredPartitionIterator(rowIterator);
+ }
+
+ @Override
+ public final UnfilteredPartitionIterator select(DataRange dataRange, ColumnFilter columnFilter)
+ {
+ DataSet data = data();
+
+ if (data.isEmpty())
+ return EmptyIterators.unfilteredPartition(metadata);
+
+ Iterator<Partition> iterator = data.getPartitions(dataRange);
+
+ long now = System.currentTimeMillis();
+
+ return new AbstractUnfilteredPartitionIterator()
+ {
+ @Override
+ public UnfilteredRowIterator next()
+ {
+ Partition partition = iterator.next();
+ return partition.toRowIterator(metadata, dataRange.clusteringIndexFilter(partition.key()), columnFilter, now);
+ }
+
+ @Override
+ public boolean hasNext()
+ {
+ return iterator.hasNext();
+ }
+
+ @Override
+ public TableMetadata metadata()
+ {
+ return metadata;
+ }
+ };
+ }
+
+ @Override
+ public void apply(PartitionUpdate update)
+ {
+ throw new InvalidRequestException("Modification is not supported by table " + metadata);
+ }
+
+ public interface DataSet
+ {
+ boolean isEmpty();
+ Partition getPartition(DecoratedKey partitionKey);
+ Iterator<Partition> getPartitions(DataRange range);
+ }
+
+ public interface Partition
+ {
+ DecoratedKey key();
+ UnfilteredRowIterator toRowIterator(TableMetadata metadata, ClusteringIndexFilter clusteringIndexFilter, ColumnFilter columnFilter, long now);
+ }
+
+ /**
+ * An abstract, map-backed DataSet implementation. Can be backed by any {@link NavigableMap}, then either maintained
+ * persistently, or built on demand and thrown away after use, depending on the implementing class.
+ */
+ public static abstract class AbstractDataSet implements DataSet
+ {
+ protected final NavigableMap<DecoratedKey, Partition> partitions;
+
+ protected AbstractDataSet(NavigableMap<DecoratedKey, Partition> partitions)
+ {
+ this.partitions = partitions;
+ }
+
+ public boolean isEmpty()
+ {
+ return partitions.isEmpty();
+ }
+
+ public Partition getPartition(DecoratedKey key)
+ {
+ return partitions.get(key);
+ }
+
+ public Iterator<Partition> getPartitions(DataRange dataRange)
+ {
+ AbstractBounds<PartitionPosition> keyRange = dataRange.keyRange();
+ PartitionPosition startKey = keyRange.left;
+ PartitionPosition endKey = keyRange.right;
+
+ NavigableMap<DecoratedKey, Partition> selection = partitions;
+
+ if (startKey.isMinimum() && endKey.isMinimum())
+ return selection.values().iterator();
+
+ if (startKey.isMinimum() && endKey instanceof DecoratedKey)
+ return selection.headMap((DecoratedKey) endKey, keyRange.isEndInclusive()).values().iterator();
+
+ if (startKey instanceof DecoratedKey && endKey instanceof DecoratedKey)
+ {
+ return selection.subMap((DecoratedKey) startKey, keyRange.isStartInclusive(), (DecoratedKey) endKey, keyRange.isEndInclusive())
+ .values()
+ .iterator();
+ }
+
+ if (startKey instanceof DecoratedKey)
+ selection = selection.tailMap((DecoratedKey) startKey, keyRange.isStartInclusive());
+
+ if (endKey instanceof DecoratedKey)
+ selection = selection.headMap((DecoratedKey) endKey, keyRange.isEndInclusive());
+
+ // If we have reach this point it means that one of the PartitionPosition is a KeyBound and we have
+ // to use filtering for eliminating the unwanted partitions.
+ Iterator<Partition> iterator = selection.values().iterator();
+
+ return new AbstractIterator<Partition>()
+ {
+ private boolean encounteredPartitionsWithinRange;
+
+ @Override
+ protected Partition computeNext()
+ {
+ while (iterator.hasNext())
+ {
+ Partition partition = iterator.next();
+ if (dataRange.contains(partition.key()))
+ {
+ encounteredPartitionsWithinRange = true;
+ return partition;
+ }
+
+ // we encountered some partitions within the range, but the last one is outside of the range: we are done
+ if (encounteredPartitionsWithinRange)
+ return endOfData();
+ }
+
+ return endOfData();
+ }
+ };
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/db/virtual/CachesTable.java b/src/java/org/apache/cassandra/db/virtual/CachesTable.java
new file mode 100644
index 0000000..5a265e6
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/virtual/CachesTable.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db.virtual;
+
+import org.apache.cassandra.cache.ChunkCache;
+import org.apache.cassandra.db.marshal.*;
+import org.apache.cassandra.dht.LocalPartitioner;
+import org.apache.cassandra.metrics.CacheMetrics;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.service.CacheService;
+
+final class CachesTable extends AbstractVirtualTable
+{
+ private static final String NAME = "name";
+ private static final String CAPACITY_BYTES = "capacity_bytes";
+ private static final String SIZE_BYTES = "size_bytes";
+ private static final String ENTRY_COUNT = "entry_count";
+ private static final String REQUEST_COUNT = "request_count";
+ private static final String HIT_COUNT = "hit_count";
+ private static final String HIT_RATIO = "hit_ratio";
+ private static final String RECENT_REQUEST_RATE_PER_SECOND = "recent_request_rate_per_second";
+ private static final String RECENT_HIT_RATE_PER_SECOND = "recent_hit_rate_per_second";
+
+ CachesTable(String keyspace)
+ {
+ super(TableMetadata.builder(keyspace, "caches")
+ .comment("system caches")
+ .kind(TableMetadata.Kind.VIRTUAL)
+ .partitioner(new LocalPartitioner(UTF8Type.instance))
+ .addPartitionKeyColumn(NAME, UTF8Type.instance)
+ .addRegularColumn(CAPACITY_BYTES, LongType.instance)
+ .addRegularColumn(SIZE_BYTES, LongType.instance)
+ .addRegularColumn(ENTRY_COUNT, Int32Type.instance)
+ .addRegularColumn(REQUEST_COUNT, LongType.instance)
+ .addRegularColumn(HIT_COUNT, LongType.instance)
+ .addRegularColumn(HIT_RATIO, DoubleType.instance)
+ .addRegularColumn(RECENT_REQUEST_RATE_PER_SECOND, LongType.instance)
+ .addRegularColumn(RECENT_HIT_RATE_PER_SECOND, LongType.instance)
+ .build());
+ }
+
+ private void addRow(SimpleDataSet result, String name, CacheMetrics metrics)
+ {
+ result.row(name)
+ .column(CAPACITY_BYTES, metrics.capacity.getValue())
+ .column(SIZE_BYTES, metrics.size.getValue())
+ .column(ENTRY_COUNT, metrics.entries.getValue())
+ .column(REQUEST_COUNT, metrics.requests.getCount())
+ .column(HIT_COUNT, metrics.hits.getCount())
+ .column(HIT_RATIO, metrics.hitRate.getValue())
+ .column(RECENT_REQUEST_RATE_PER_SECOND, (long) metrics.requests.getFifteenMinuteRate())
+ .column(RECENT_HIT_RATE_PER_SECOND, (long) metrics.hits.getFifteenMinuteRate());
+ }
+
+ public DataSet data()
+ {
+ SimpleDataSet result = new SimpleDataSet(metadata());
+
+ if (null != ChunkCache.instance)
+ addRow(result, "chunks", ChunkCache.instance.metrics);
+ addRow(result, "counters", CacheService.instance.counterCache.getMetrics());
+ addRow(result, "keys", CacheService.instance.keyCache.getMetrics());
+ addRow(result, "rows", CacheService.instance.rowCache.getMetrics());
+
+ return result;
+ }
+}
diff --git a/src/java/org/apache/cassandra/db/virtual/ClientsTable.java b/src/java/org/apache/cassandra/db/virtual/ClientsTable.java
new file mode 100644
index 0000000..40e175b
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/virtual/ClientsTable.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db.virtual;
+
+import java.net.InetSocketAddress;
+
+import org.apache.cassandra.db.marshal.*;
+import org.apache.cassandra.dht.LocalPartitioner;
+import org.apache.cassandra.metrics.ClientMetrics;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.transport.ConnectedClient;
+
+final class ClientsTable extends AbstractVirtualTable
+{
+ private static final String ADDRESS = "address";
+ private static final String PORT = "port";
+ private static final String HOSTNAME = "hostname";
+ private static final String USERNAME = "username";
+ private static final String CONNECTION_STAGE = "connection_stage";
+ private static final String PROTOCOL_VERSION = "protocol_version";
+ private static final String DRIVER_NAME = "driver_name";
+ private static final String DRIVER_VERSION = "driver_version";
+ private static final String REQUEST_COUNT = "request_count";
+ private static final String SSL_ENABLED = "ssl_enabled";
+ private static final String SSL_PROTOCOL = "ssl_protocol";
+ private static final String SSL_CIPHER_SUITE = "ssl_cipher_suite";
+
+ ClientsTable(String keyspace)
+ {
+ super(TableMetadata.builder(keyspace, "clients")
+ .comment("currently connected clients")
+ .kind(TableMetadata.Kind.VIRTUAL)
+ .partitioner(new LocalPartitioner(InetAddressType.instance))
+ .addPartitionKeyColumn(ADDRESS, InetAddressType.instance)
+ .addClusteringColumn(PORT, Int32Type.instance)
+ .addRegularColumn(HOSTNAME, UTF8Type.instance)
+ .addRegularColumn(USERNAME, UTF8Type.instance)
+ .addRegularColumn(CONNECTION_STAGE, UTF8Type.instance)
+ .addRegularColumn(PROTOCOL_VERSION, Int32Type.instance)
+ .addRegularColumn(DRIVER_NAME, UTF8Type.instance)
+ .addRegularColumn(DRIVER_VERSION, UTF8Type.instance)
+ .addRegularColumn(REQUEST_COUNT, LongType.instance)
+ .addRegularColumn(SSL_ENABLED, BooleanType.instance)
+ .addRegularColumn(SSL_PROTOCOL, UTF8Type.instance)
+ .addRegularColumn(SSL_CIPHER_SUITE, UTF8Type.instance)
+ .build());
+ }
+
+ @Override
+ public DataSet data()
+ {
+ SimpleDataSet result = new SimpleDataSet(metadata());
+
+ for (ConnectedClient client : ClientMetrics.instance.allConnectedClients())
+ {
+ InetSocketAddress remoteAddress = client.remoteAddress();
+
+ result.row(remoteAddress.getAddress(), remoteAddress.getPort())
+ .column(HOSTNAME, remoteAddress.getHostName())
+ .column(USERNAME, client.username().orElse(null))
+ .column(CONNECTION_STAGE, client.stage().toString().toLowerCase())
+ .column(PROTOCOL_VERSION, client.protocolVersion())
+ .column(DRIVER_NAME, client.driverName().orElse(null))
+ .column(DRIVER_VERSION, client.driverVersion().orElse(null))
+ .column(REQUEST_COUNT, client.requestCount())
+ .column(SSL_ENABLED, client.sslEnabled())
+ .column(SSL_PROTOCOL, client.sslProtocol().orElse(null))
+ .column(SSL_CIPHER_SUITE, client.sslCipherSuite().orElse(null));
+ }
+
+ return result;
+ }
+}
diff --git a/src/java/org/apache/cassandra/db/virtual/InternodeInboundTable.java b/src/java/org/apache/cassandra/db/virtual/InternodeInboundTable.java
new file mode 100644
index 0000000..b0afe8f
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/virtual/InternodeInboundTable.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db.virtual;
+
+import java.net.InetAddress;
+import java.nio.ByteBuffer;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.marshal.CompositeType;
+import org.apache.cassandra.db.marshal.InetAddressType;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.marshal.LongType;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.dht.LocalPartitioner;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.net.InboundMessageHandlers;
+import org.apache.cassandra.schema.TableMetadata;
+
+public final class InternodeInboundTable extends AbstractVirtualTable
+{
+ private static final String ADDRESS = "address";
+ private static final String PORT = "port";
+ private static final String DC = "dc";
+ private static final String RACK = "rack";
+
+ private static final String USING_BYTES = "using_bytes";
+ private static final String USING_RESERVE_BYTES = "using_reserve_bytes";
+ private static final String CORRUPT_FRAMES_RECOVERED = "corrupt_frames_recovered";
+ private static final String CORRUPT_FRAMES_UNRECOVERED = "corrupt_frames_unrecovered";
+ private static final String ERROR_BYTES = "error_bytes";
+ private static final String ERROR_COUNT = "error_count";
+ private static final String EXPIRED_BYTES = "expired_bytes";
+ private static final String EXPIRED_COUNT = "expired_count";
+ private static final String SCHEDULED_BYTES = "scheduled_bytes";
+ private static final String SCHEDULED_COUNT = "scheduled_count";
+ private static final String PROCESSED_BYTES = "processed_bytes";
+ private static final String PROCESSED_COUNT = "processed_count";
+ private static final String RECEIVED_BYTES = "received_bytes";
+ private static final String RECEIVED_COUNT = "received_count";
+ private static final String THROTTLED_COUNT = "throttled_count";
+ private static final String THROTTLED_NANOS = "throttled_nanos";
+
+ InternodeInboundTable(String keyspace)
+ {
+ super(TableMetadata.builder(keyspace, "internode_inbound")
+ .kind(TableMetadata.Kind.VIRTUAL)
+ .partitioner(new LocalPartitioner(CompositeType.getInstance(InetAddressType.instance, Int32Type.instance)))
+ .addPartitionKeyColumn(ADDRESS, InetAddressType.instance)
+ .addPartitionKeyColumn(PORT, Int32Type.instance)
+ .addClusteringColumn(DC, UTF8Type.instance)
+ .addClusteringColumn(RACK, UTF8Type.instance)
+ .addRegularColumn(USING_BYTES, LongType.instance)
+ .addRegularColumn(USING_RESERVE_BYTES, LongType.instance)
+ .addRegularColumn(CORRUPT_FRAMES_RECOVERED, LongType.instance)
+ .addRegularColumn(CORRUPT_FRAMES_UNRECOVERED, LongType.instance)
+ .addRegularColumn(ERROR_BYTES, LongType.instance)
+ .addRegularColumn(ERROR_COUNT, LongType.instance)
+ .addRegularColumn(EXPIRED_BYTES, LongType.instance)
+ .addRegularColumn(EXPIRED_COUNT, LongType.instance)
+ .addRegularColumn(SCHEDULED_BYTES, LongType.instance)
+ .addRegularColumn(SCHEDULED_COUNT, LongType.instance)
+ .addRegularColumn(PROCESSED_BYTES, LongType.instance)
+ .addRegularColumn(PROCESSED_COUNT, LongType.instance)
+ .addRegularColumn(RECEIVED_BYTES, LongType.instance)
+ .addRegularColumn(RECEIVED_COUNT, LongType.instance)
+ .addRegularColumn(THROTTLED_COUNT, LongType.instance)
+ .addRegularColumn(THROTTLED_NANOS, LongType.instance)
+ .build());
+ }
+
+ @Override
+ public DataSet data(DecoratedKey partitionKey)
+ {
+ ByteBuffer[] addressAndPortBytes = ((CompositeType) metadata().partitionKeyType).split(partitionKey.getKey());
+ InetAddress address = InetAddressType.instance.compose(addressAndPortBytes[0]);
+ int port = Int32Type.instance.compose(addressAndPortBytes[1]);
+ InetAddressAndPort addressAndPort = InetAddressAndPort.getByAddressOverrideDefaults(address, port);
+
+ SimpleDataSet result = new SimpleDataSet(metadata());
+ InboundMessageHandlers handlers = MessagingService.instance().messageHandlers.get(addressAndPort);
+ if (null != handlers)
+ addRow(result, addressAndPort, handlers);
+ return result;
+ }
+
+ @Override
+ public DataSet data()
+ {
+ SimpleDataSet result = new SimpleDataSet(metadata());
+ MessagingService.instance()
+ .messageHandlers
+ .forEach((addressAndPort, handlers) -> addRow(result, addressAndPort, handlers));
+ return result;
+ }
+
+ private void addRow(SimpleDataSet dataSet, InetAddressAndPort addressAndPort, InboundMessageHandlers handlers)
+ {
+ String dc = DatabaseDescriptor.getEndpointSnitch().getDatacenter(addressAndPort);
+ String rack = DatabaseDescriptor.getEndpointSnitch().getRack(addressAndPort);
+ dataSet.row(addressAndPort.address, addressAndPort.port, dc, rack)
+ .column(USING_BYTES, handlers.usingCapacity())
+ .column(USING_RESERVE_BYTES, handlers.usingEndpointReserveCapacity())
+ .column(CORRUPT_FRAMES_RECOVERED, handlers.corruptFramesRecovered())
+ .column(CORRUPT_FRAMES_UNRECOVERED, handlers.corruptFramesUnrecovered())
+ .column(ERROR_BYTES, handlers.errorBytes())
+ .column(ERROR_COUNT, handlers.errorCount())
+ .column(EXPIRED_BYTES, handlers.expiredBytes())
+ .column(EXPIRED_COUNT, handlers.expiredCount())
+ .column(SCHEDULED_BYTES, handlers.scheduledBytes())
+ .column(SCHEDULED_COUNT, handlers.scheduledCount())
+ .column(PROCESSED_BYTES, handlers.processedBytes())
+ .column(PROCESSED_COUNT, handlers.processedCount())
+ .column(RECEIVED_BYTES, handlers.receivedBytes())
+ .column(RECEIVED_COUNT, handlers.receivedCount())
+ .column(THROTTLED_COUNT, handlers.throttledCount())
+ .column(THROTTLED_NANOS, handlers.throttledNanos());
+ }
+}
diff --git a/src/java/org/apache/cassandra/db/virtual/InternodeOutboundTable.java b/src/java/org/apache/cassandra/db/virtual/InternodeOutboundTable.java
new file mode 100644
index 0000000..87b3823
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/virtual/InternodeOutboundTable.java
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db.virtual;
+
+import java.net.InetAddress;
+import java.nio.ByteBuffer;
+import java.util.function.ToLongFunction;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.marshal.CompositeType;
+import org.apache.cassandra.db.marshal.InetAddressType;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.marshal.LongType;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.dht.LocalPartitioner;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.net.OutboundConnection;
+import org.apache.cassandra.net.OutboundConnections;
+import org.apache.cassandra.schema.TableMetadata;
+
+public final class InternodeOutboundTable extends AbstractVirtualTable
+{
+ private static final String ADDRESS = "address";
+ private static final String PORT = "port";
+ private static final String DC = "dc";
+ private static final String RACK = "rack";
+
+ private static final String USING_BYTES = "using_bytes";
+ private static final String USING_RESERVE_BYTES = "using_reserve_bytes";
+ private static final String PENDING_COUNT = "pending_count";
+ private static final String PENDING_BYTES = "pending_bytes";
+ private static final String SENT_COUNT = "sent_count";
+ private static final String SENT_BYTES = "sent_bytes";
+ private static final String EXPIRED_COUNT = "expired_count";
+ private static final String EXPIRED_BYTES = "expired_bytes";
+ private static final String ERROR_COUNT = "error_count";
+ private static final String ERROR_BYTES = "error_bytes";
+ private static final String OVERLOAD_COUNT = "overload_count";
+ private static final String OVERLOAD_BYTES = "overload_bytes";
+ private static final String ACTIVE_CONNECTION_COUNT = "active_connections";
+ private static final String CONNECTION_ATTEMPTS = "connection_attempts";
+ private static final String SUCCESSFUL_CONNECTION_ATTEMPTS = "successful_connection_attempts";
+
+ InternodeOutboundTable(String keyspace)
+ {
+ super(TableMetadata.builder(keyspace, "internode_outbound")
+ .kind(TableMetadata.Kind.VIRTUAL)
+ .partitioner(new LocalPartitioner(CompositeType.getInstance(InetAddressType.instance, Int32Type.instance)))
+ .addPartitionKeyColumn(ADDRESS, InetAddressType.instance)
+ .addPartitionKeyColumn(PORT, Int32Type.instance)
+ .addClusteringColumn(DC, UTF8Type.instance)
+ .addClusteringColumn(RACK, UTF8Type.instance)
+ .addRegularColumn(USING_BYTES, LongType.instance)
+ .addRegularColumn(USING_RESERVE_BYTES, LongType.instance)
+ .addRegularColumn(PENDING_COUNT, LongType.instance)
+ .addRegularColumn(PENDING_BYTES, LongType.instance)
+ .addRegularColumn(SENT_COUNT, LongType.instance)
+ .addRegularColumn(SENT_BYTES, LongType.instance)
+ .addRegularColumn(EXPIRED_COUNT, LongType.instance)
+ .addRegularColumn(EXPIRED_BYTES, LongType.instance)
+ .addRegularColumn(ERROR_COUNT, LongType.instance)
+ .addRegularColumn(ERROR_BYTES, LongType.instance)
+ .addRegularColumn(OVERLOAD_COUNT, LongType.instance)
+ .addRegularColumn(OVERLOAD_BYTES, LongType.instance)
+ .addRegularColumn(ACTIVE_CONNECTION_COUNT, LongType.instance)
+ .addRegularColumn(CONNECTION_ATTEMPTS, LongType.instance)
+ .addRegularColumn(SUCCESSFUL_CONNECTION_ATTEMPTS, LongType.instance)
+ .build());
+ }
+
+ @Override
+ public DataSet data(DecoratedKey partitionKey)
+ {
+ ByteBuffer[] addressAndPortBytes = ((CompositeType) metadata().partitionKeyType).split(partitionKey.getKey());
+ InetAddress address = InetAddressType.instance.compose(addressAndPortBytes[0]);
+ int port = Int32Type.instance.compose(addressAndPortBytes[1]);
+ InetAddressAndPort addressAndPort = InetAddressAndPort.getByAddressOverrideDefaults(address, port);
+
+ SimpleDataSet result = new SimpleDataSet(metadata());
+ OutboundConnections connections = MessagingService.instance().channelManagers.get(addressAndPort);
+ if (null != connections)
+ addRow(result, addressAndPort, connections);
+ return result;
+ }
+
+ @Override
+ public DataSet data()
+ {
+ SimpleDataSet result = new SimpleDataSet(metadata());
+ MessagingService.instance()
+ .channelManagers
+ .forEach((addressAndPort, connections) -> addRow(result, addressAndPort, connections));
+ return result;
+ }
+
+ private void addRow(SimpleDataSet dataSet, InetAddressAndPort addressAndPort, OutboundConnections connections)
+ {
+ String dc = DatabaseDescriptor.getEndpointSnitch().getDatacenter(addressAndPort);
+ String rack = DatabaseDescriptor.getEndpointSnitch().getRack(addressAndPort);
+ long pendingBytes = sum(connections, OutboundConnection::pendingBytes);
+ dataSet.row(addressAndPort.address, addressAndPort.port, dc, rack)
+ .column(USING_BYTES, pendingBytes)
+ .column(USING_RESERVE_BYTES, connections.usingReserveBytes())
+ .column(PENDING_COUNT, sum(connections, OutboundConnection::pendingCount))
+ .column(PENDING_BYTES, pendingBytes)
+ .column(SENT_COUNT, sum(connections, OutboundConnection::sentCount))
+ .column(SENT_BYTES, sum(connections, OutboundConnection::sentBytes))
+ .column(EXPIRED_COUNT, sum(connections, OutboundConnection::expiredCount))
+ .column(EXPIRED_BYTES, sum(connections, OutboundConnection::expiredBytes))
+ .column(ERROR_COUNT, sum(connections, OutboundConnection::errorCount))
+ .column(ERROR_BYTES, sum(connections, OutboundConnection::errorBytes))
+ .column(OVERLOAD_COUNT, sum(connections, OutboundConnection::overloadedCount))
+ .column(OVERLOAD_BYTES, sum(connections, OutboundConnection::overloadedBytes))
+ .column(ACTIVE_CONNECTION_COUNT, sum(connections, c -> c.isConnected() ? 1 : 0))
+ .column(CONNECTION_ATTEMPTS, sum(connections, OutboundConnection::connectionAttempts))
+ .column(SUCCESSFUL_CONNECTION_ATTEMPTS, sum(connections, OutboundConnection::successfulConnections));
+ }
+
+ private static long sum(OutboundConnections connections, ToLongFunction<OutboundConnection> f)
+ {
+ return f.applyAsLong(connections.small) + f.applyAsLong(connections.large) + f.applyAsLong(connections.urgent);
+ }
+}
diff --git a/src/java/org/apache/cassandra/db/virtual/SSTableTasksTable.java b/src/java/org/apache/cassandra/db/virtual/SSTableTasksTable.java
new file mode 100644
index 0000000..20033df
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/virtual/SSTableTasksTable.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db.virtual;
+
+import org.apache.cassandra.db.compaction.CompactionInfo;
+import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.db.marshal.DoubleType;
+import org.apache.cassandra.db.marshal.LongType;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.db.marshal.UUIDType;
+import org.apache.cassandra.dht.LocalPartitioner;
+import org.apache.cassandra.schema.TableMetadata;
+
+final class SSTableTasksTable extends AbstractVirtualTable
+{
+ private final static String KEYSPACE_NAME = "keyspace_name";
+ private final static String TABLE_NAME = "table_name";
+ private final static String TASK_ID = "task_id";
+ private final static String COMPLETION_RATIO = "completion_ratio";
+ private final static String KIND = "kind";
+ private final static String PROGRESS = "progress";
+ private final static String TOTAL = "total";
+ private final static String UNIT = "unit";
+
+ SSTableTasksTable(String keyspace)
+ {
+ super(TableMetadata.builder(keyspace, "sstable_tasks")
+ .comment("current sstable tasks")
+ .kind(TableMetadata.Kind.VIRTUAL)
+ .partitioner(new LocalPartitioner(UTF8Type.instance))
+ .addPartitionKeyColumn(KEYSPACE_NAME, UTF8Type.instance)
+ .addClusteringColumn(TABLE_NAME, UTF8Type.instance)
+ .addClusteringColumn(TASK_ID, UUIDType.instance)
+ .addRegularColumn(COMPLETION_RATIO, DoubleType.instance)
+ .addRegularColumn(KIND, UTF8Type.instance)
+ .addRegularColumn(PROGRESS, LongType.instance)
+ .addRegularColumn(TOTAL, LongType.instance)
+ .addRegularColumn(UNIT, UTF8Type.instance)
+ .build());
+ }
+
+ public DataSet data()
+ {
+ SimpleDataSet result = new SimpleDataSet(metadata());
+
+ for (CompactionInfo task : CompactionManager.instance.getSSTableTasks())
+ {
+ long completed = task.getCompleted();
+ long total = task.getTotal();
+
+ double completionRatio = total == 0L ? 1.0 : (((double) completed) / total);
+
+ result.row(task.getKeyspace().orElse("*"),
+ task.getTable().orElse("*"),
+ task.getTaskId())
+ .column(COMPLETION_RATIO, completionRatio)
+ .column(KIND, task.getTaskType().toString().toLowerCase())
+ .column(PROGRESS, completed)
+ .column(TOTAL, total)
+ .column(UNIT, task.getUnit().toString().toLowerCase());
+ }
+
+ return result;
+ }
+}
diff --git a/src/java/org/apache/cassandra/db/virtual/SettingsTable.java b/src/java/org/apache/cassandra/db/virtual/SettingsTable.java
new file mode 100644
index 0000000..d11f69a
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/virtual/SettingsTable.java
@@ -0,0 +1,189 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db.virtual;
+
+import java.lang.reflect.Field;
+import java.lang.reflect.Modifier;
+import java.util.Arrays;
+import java.util.Map;
+import java.util.function.BiConsumer;
+import java.util.stream.Collectors;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Functions;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
+
+import org.apache.cassandra.audit.AuditLogOptions;
+import org.apache.cassandra.config.*;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.dht.LocalPartitioner;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.transport.ServerError;
+
+final class SettingsTable extends AbstractVirtualTable
+{
+ private static final String NAME = "name";
+ private static final String VALUE = "value";
+
+ @VisibleForTesting
+ static final Map<String, Field> FIELDS =
+ Arrays.stream(Config.class.getFields())
+ .filter(f -> !Modifier.isStatic(f.getModifiers()))
+ .collect(Collectors.toMap(Field::getName, Functions.identity()));
+
+ @VisibleForTesting
+ final Map<String, BiConsumer<SimpleDataSet, Field>> overrides =
+ ImmutableMap.<String, BiConsumer<SimpleDataSet, Field>>builder()
+ .put("audit_logging_options", this::addAuditLoggingOptions)
+ .put("client_encryption_options", this::addEncryptionOptions)
+ .put("server_encryption_options", this::addEncryptionOptions)
+ .put("transparent_data_encryption_options", this::addTransparentEncryptionOptions)
+ .build();
+
+ private final Config config;
+
+ SettingsTable(String keyspace)
+ {
+ this(keyspace, DatabaseDescriptor.getRawConfig());
+ }
+
+ SettingsTable(String keyspace, Config config)
+ {
+ super(TableMetadata.builder(keyspace, "settings")
+ .comment("current settings")
+ .kind(TableMetadata.Kind.VIRTUAL)
+ .partitioner(new LocalPartitioner(UTF8Type.instance))
+ .addPartitionKeyColumn(NAME, UTF8Type.instance)
+ .addRegularColumn(VALUE, UTF8Type.instance)
+ .build());
+ this.config = config;
+ }
+
+ @VisibleForTesting
+ Object getValue(Field f)
+ {
+ Object value;
+ try
+ {
+ value = f.get(config);
+ }
+ catch (IllegalAccessException | IllegalArgumentException e)
+ {
+ throw new ServerError(e);
+ }
+ return value;
+ }
+
+ private void addValue(SimpleDataSet result, Field f)
+ {
+ Object value = getValue(f);
+ if (value == null)
+ {
+ result.row(f.getName());
+ }
+ else if (overrides.containsKey(f.getName()))
+ {
+ overrides.get(f.getName()).accept(result, f);
+ }
+ else
+ {
+ if (value.getClass().isArray())
+ value = Arrays.toString((Object[]) value);
+ result.row(f.getName()).column(VALUE, value.toString());
+ }
+ }
+
+ @Override
+ public DataSet data(DecoratedKey partitionKey)
+ {
+ SimpleDataSet result = new SimpleDataSet(metadata());
+ String name = UTF8Type.instance.compose(partitionKey.getKey());
+ Field field = FIELDS.get(name);
+ if (field != null)
+ {
+ addValue(result, field);
+ }
+ else
+ {
+ // rows created by overrides might be directly queried so include them in result to be possibly filtered
+ for (String override : overrides.keySet())
+ if (name.startsWith(override))
+ addValue(result, FIELDS.get(override));
+ }
+ return result;
+ }
+
+ @Override
+ public DataSet data()
+ {
+ SimpleDataSet result = new SimpleDataSet(metadata());
+ for (Field setting : FIELDS.values())
+ addValue(result, setting);
+ return result;
+ }
+
+ private void addAuditLoggingOptions(SimpleDataSet result, Field f)
+ {
+ Preconditions.checkArgument(AuditLogOptions.class.isAssignableFrom(f.getType()));
+
+ AuditLogOptions value = (AuditLogOptions) getValue(f);
+ result.row(f.getName() + "_enabled").column(VALUE, Boolean.toString(value.enabled));
+ result.row(f.getName() + "_logger").column(VALUE, value.logger.class_name);
+ result.row(f.getName() + "_audit_logs_dir").column(VALUE, value.audit_logs_dir);
+ result.row(f.getName() + "_included_keyspaces").column(VALUE, value.included_keyspaces);
+ result.row(f.getName() + "_excluded_keyspaces").column(VALUE, value.excluded_keyspaces);
+ result.row(f.getName() + "_included_categories").column(VALUE, value.included_categories);
+ result.row(f.getName() + "_excluded_categories").column(VALUE, value.excluded_categories);
+ result.row(f.getName() + "_included_users").column(VALUE, value.included_users);
+ result.row(f.getName() + "_excluded_users").column(VALUE, value.excluded_users);
+ }
+
+ private void addEncryptionOptions(SimpleDataSet result, Field f)
+ {
+ Preconditions.checkArgument(EncryptionOptions.class.isAssignableFrom(f.getType()));
+
+ EncryptionOptions value = (EncryptionOptions) getValue(f);
+ result.row(f.getName() + "_enabled").column(VALUE, Boolean.toString(value.isEnabled()));
+ result.row(f.getName() + "_algorithm").column(VALUE, value.algorithm);
+ result.row(f.getName() + "_protocol").column(VALUE, value.protocol);
+ result.row(f.getName() + "_cipher_suites").column(VALUE, value.cipher_suites.toString());
+ result.row(f.getName() + "_client_auth").column(VALUE, Boolean.toString(value.require_client_auth));
+ result.row(f.getName() + "_endpoint_verification").column(VALUE, Boolean.toString(value.require_endpoint_verification));
+ result.row(f.getName() + "_optional").column(VALUE, Boolean.toString(value.optional));
+
+ if (value instanceof EncryptionOptions.ServerEncryptionOptions)
+ {
+ EncryptionOptions.ServerEncryptionOptions server = (EncryptionOptions.ServerEncryptionOptions) value;
+ result.row(f.getName() + "_internode_encryption").column(VALUE, server.internode_encryption.toString());
+ result.row(f.getName() + "_legacy_ssl_storage_port").column(VALUE, Boolean.toString(server.enable_legacy_ssl_storage_port));
+ }
+ }
+
+ private void addTransparentEncryptionOptions(SimpleDataSet result, Field f)
+ {
+ Preconditions.checkArgument(TransparentDataEncryptionOptions.class.isAssignableFrom(f.getType()));
+
+ TransparentDataEncryptionOptions value = (TransparentDataEncryptionOptions) getValue(f);
+ result.row(f.getName() + "_enabled").column(VALUE, Boolean.toString(value.enabled));
+ result.row(f.getName() + "_cipher").column(VALUE, value.cipher);
+ result.row(f.getName() + "_chunk_length_kb").column(VALUE, Integer.toString(value.chunk_length_kb));
+ result.row(f.getName() + "_iv_length").column(VALUE, Integer.toString(value.iv_length));
+ }
+}
diff --git a/src/java/org/apache/cassandra/db/virtual/SimpleDataSet.java b/src/java/org/apache/cassandra/db/virtual/SimpleDataSet.java
new file mode 100644
index 0000000..00acaed
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/virtual/SimpleDataSet.java
@@ -0,0 +1,198 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db.virtual;
+
+import java.nio.ByteBuffer;
+import java.util.*;
+import java.util.concurrent.TimeUnit;
+
+import com.google.common.collect.Iterables;
+
+import org.apache.cassandra.db.*;
+import org.apache.cassandra.db.filter.ClusteringIndexFilter;
+import org.apache.cassandra.db.filter.ColumnFilter;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.CompositeType;
+import org.apache.cassandra.db.rows.*;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.utils.ByteBufferUtil;
+
+/**
+ * A DataSet implementation that is filled on demand and has an easy to use API for adding rows.
+ */
+public class SimpleDataSet extends AbstractVirtualTable.AbstractDataSet
+{
+ private final TableMetadata metadata;
+
+ private Row currentRow;
+
+ public SimpleDataSet(TableMetadata metadata)
+ {
+ super(new TreeMap<>(DecoratedKey.comparator));
+ this.metadata = metadata;
+ }
+
+ public SimpleDataSet row(Object... primaryKeyValues)
+ {
+ if (Iterables.size(metadata.primaryKeyColumns()) != primaryKeyValues.length)
+ throw new IllegalArgumentException();
+
+ Object[] partitionKeyValues = new Object[metadata.partitionKeyColumns().size()];
+ Object[] clusteringValues = new Object[metadata.clusteringColumns().size()];
+
+ System.arraycopy(primaryKeyValues, 0, partitionKeyValues, 0, partitionKeyValues.length);
+ System.arraycopy(primaryKeyValues, partitionKeyValues.length, clusteringValues, 0, clusteringValues.length);
+
+ DecoratedKey partitionKey = makeDecoratedKey(partitionKeyValues);
+ Clustering clustering = makeClustering(clusteringValues);
+
+ currentRow = new Row(metadata, clustering);
+ SimplePartition partition = (SimplePartition) partitions.computeIfAbsent(partitionKey, pk -> new SimplePartition(metadata, pk));
+ partition.add(currentRow);
+
+ return this;
+ }
+
+ public SimpleDataSet column(String columnName, Object value)
+ {
+ if (null == currentRow)
+ throw new IllegalStateException();
+ if (null == columnName)
+ throw new IllegalStateException(String.format("Invalid column: %s=%s for %s", columnName, value, currentRow));
+ currentRow.add(columnName, value);
+ return this;
+ }
+
+ private DecoratedKey makeDecoratedKey(Object... partitionKeyValues)
+ {
+ ByteBuffer partitionKey = partitionKeyValues.length == 1
+ ? decompose(metadata.partitionKeyType, partitionKeyValues[0])
+ : ((CompositeType) metadata.partitionKeyType).decompose(partitionKeyValues);
+ return metadata.partitioner.decorateKey(partitionKey);
+ }
+
+ private Clustering makeClustering(Object... clusteringValues)
+ {
+ if (clusteringValues.length == 0)
+ return Clustering.EMPTY;
+
+ ByteBuffer[] clusteringByteBuffers = new ByteBuffer[clusteringValues.length];
+ for (int i = 0; i < clusteringValues.length; i++)
+ clusteringByteBuffers[i] = decompose(metadata.clusteringColumns().get(i).type, clusteringValues[i]);
+ return Clustering.make(clusteringByteBuffers);
+ }
+
+ private static final class SimplePartition implements AbstractVirtualTable.Partition
+ {
+ private final DecoratedKey key;
+ private final NavigableMap<Clustering, Row> rows;
+
+ private SimplePartition(TableMetadata metadata, DecoratedKey key)
+ {
+ this.key = key;
+ this.rows = new TreeMap<>(metadata.comparator);
+ }
+
+ private void add(Row row)
+ {
+ rows.put(row.clustering, row);
+ }
+
+ public DecoratedKey key()
+ {
+ return key;
+ }
+
+ public UnfilteredRowIterator toRowIterator(TableMetadata metadata,
+ ClusteringIndexFilter clusteringIndexFilter,
+ ColumnFilter columnFilter,
+ long now)
+ {
+ Iterator<Row> iterator = (clusteringIndexFilter.isReversed() ? rows.descendingMap() : rows).values().iterator();
+
+ return new AbstractUnfilteredRowIterator(metadata,
+ key,
+ DeletionTime.LIVE,
+ columnFilter.queriedColumns(),
+ Rows.EMPTY_STATIC_ROW,
+ false,
+ EncodingStats.NO_STATS)
+ {
+ protected Unfiltered computeNext()
+ {
+ while (iterator.hasNext())
+ {
+ Row row = iterator.next();
+ if (clusteringIndexFilter.selects(row.clustering))
+ return row.toTableRow(columns, now);
+ }
+ return endOfData();
+ }
+ };
+ }
+ }
+
+ private static class Row
+ {
+ private final TableMetadata metadata;
+ private final Clustering clustering;
+
+ private final Map<ColumnMetadata, Object> values = new HashMap<>();
+
+ private Row(TableMetadata metadata, Clustering clustering)
+ {
+ this.metadata = metadata;
+ this.clustering = clustering;
+ }
+
+ private void add(String columnName, Object value)
+ {
+ ColumnMetadata column = metadata.getColumn(ByteBufferUtil.bytes(columnName));
+ if (null == column || !column.isRegular())
+ throw new IllegalArgumentException();
+ values.put(column, value);
+ }
+
+ private org.apache.cassandra.db.rows.Row toTableRow(RegularAndStaticColumns columns, long now)
+ {
+ org.apache.cassandra.db.rows.Row.Builder builder = BTreeRow.unsortedBuilder();
+ builder.newRow(clustering);
+
+ columns.forEach(c ->
+ {
+ Object value = values.get(c);
+ if (null != value)
+ builder.addCell(BufferCell.live(c, now, decompose(c.type, value)));
+ });
+
+ return builder.build();
+ }
+
+ public String toString()
+ {
+ return "Row[...:" + clustering.toString(metadata)+']';
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ private static <T> ByteBuffer decompose(AbstractType<?> type, T value)
+ {
+ return ((AbstractType<T>) type).decompose(value);
+ }
+}
diff --git a/src/java/org/apache/cassandra/db/virtual/SystemPropertiesTable.java b/src/java/org/apache/cassandra/db/virtual/SystemPropertiesTable.java
new file mode 100644
index 0000000..e8c13e7
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/virtual/SystemPropertiesTable.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db.virtual;
+
+import java.util.Set;
+
+import com.google.common.collect.Sets;
+
+import org.apache.cassandra.config.Config;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.marshal.*;
+import org.apache.cassandra.dht.LocalPartitioner;
+import org.apache.cassandra.schema.TableMetadata;
+
+final class SystemPropertiesTable extends AbstractVirtualTable
+{
+ private static final String NAME = "name";
+ private static final String VALUE = "value";
+
+ private static final Set<String> CASSANDRA_RELEVANT_PROPERTIES = Sets.newHashSet(
+ // base jvm properties
+ "java.home",
+ "java.io.tmpdir",
+ "java.library.path",
+ "java.security.egd",
+ "java.version",
+ "java.vm.name",
+ "line.separator",
+ "os.arch",
+ "os.name",
+ "user.home",
+ "sun.arch.data.model",
+ // jmx properties
+ "java.rmi.server.hostname",
+ "java.rmi.server.randomID",
+ "com.sun.management.jmxremote.authenticate",
+ "com.sun.management.jmxremote.rmi.port",
+ "com.sun.management.jmxremote.ssl",
+ "com.sun.management.jmxremote.ssl.need.client.auth",
+ "com.sun.management.jmxremote.access.file",
+ "com.sun.management.jmxremote.password.file",
+ "com.sun.management.jmxremote.port",
+ "com.sun.management.jmxremote.ssl.enabled.protocols",
+ "com.sun.management.jmxremote.ssl.enabled.cipher.suites",
+ "mx4jaddress",
+ "mx4jport",
+ // cassandra properties (without the "cassandra." prefix)
+ "cassandra-foreground",
+ "cassandra-pidfile",
+ "default.provide.overlapping.tombstones",
+ "org.apache.cassandra.disable_mbean_registration",
+ // only for testing
+ "org.apache.cassandra.db.virtual.SystemPropertiesTableTest"
+ );
+
+ private static final Set<String> CASSANDRA_RELEVANT_ENVS = Sets.newHashSet(
+ "JAVA_HOME"
+ );
+
+ SystemPropertiesTable(String keyspace)
+ {
+ super(TableMetadata.builder(keyspace, "system_properties")
+ .comment("Cassandra relevant system properties")
+ .kind(TableMetadata.Kind.VIRTUAL)
+ .partitioner(new LocalPartitioner(UTF8Type.instance))
+ .addPartitionKeyColumn(NAME, UTF8Type.instance)
+ .addRegularColumn(VALUE, UTF8Type.instance)
+ .build());
+ }
+
+ public DataSet data()
+ {
+ SimpleDataSet result = new SimpleDataSet(metadata());
+
+ System.getenv().keySet()
+ .stream()
+ .filter(SystemPropertiesTable::isCassandraRelevant)
+ .forEach(name -> addRow(result, name, System.getenv(name)));
+
+ System.getProperties().stringPropertyNames()
+ .stream()
+ .filter(SystemPropertiesTable::isCassandraRelevant)
+ .forEach(name -> addRow(result, name, System.getProperty(name)));
+
+ return result;
+ }
+
+ @Override
+ public DataSet data(DecoratedKey partitionKey)
+ {
+ SimpleDataSet result = new SimpleDataSet(metadata());
+ String name = UTF8Type.instance.compose(partitionKey.getKey());
+ if (isCassandraRelevant(name))
+ addRow(result, name, System.getProperty(name, System.getenv(name)));
+
+ return result;
+ }
+
+ static boolean isCassandraRelevant(String name)
+ {
+ return name.startsWith(Config.PROPERTY_PREFIX) || CASSANDRA_RELEVANT_PROPERTIES.contains(name)
+ || CASSANDRA_RELEVANT_ENVS.contains(name);
+ }
+
+ private static void addRow(SimpleDataSet result, String name, String value)
+ {
+ result.row(name).column(VALUE, value);
+ }
+}
diff --git a/src/java/org/apache/cassandra/db/virtual/SystemViewsKeyspace.java b/src/java/org/apache/cassandra/db/virtual/SystemViewsKeyspace.java
new file mode 100644
index 0000000..92da4af
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/virtual/SystemViewsKeyspace.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db.virtual;
+
+import com.google.common.collect.ImmutableList;
+
+import static org.apache.cassandra.schema.SchemaConstants.VIRTUAL_VIEWS;
+
+public final class SystemViewsKeyspace extends VirtualKeyspace
+{
+ public static SystemViewsKeyspace instance = new SystemViewsKeyspace();
+
+ private SystemViewsKeyspace()
+ {
+ super(VIRTUAL_VIEWS, new ImmutableList.Builder<VirtualTable>()
+ .add(new CachesTable(VIRTUAL_VIEWS))
+ .add(new ClientsTable(VIRTUAL_VIEWS))
+ .add(new SettingsTable(VIRTUAL_VIEWS))
+ .add(new SystemPropertiesTable(VIRTUAL_VIEWS))
+ .add(new SSTableTasksTable(VIRTUAL_VIEWS))
+ .add(new ThreadPoolsTable(VIRTUAL_VIEWS))
+ .add(new InternodeOutboundTable(VIRTUAL_VIEWS))
+ .add(new InternodeInboundTable(VIRTUAL_VIEWS))
+ .addAll(TableMetricTables.getAll(VIRTUAL_VIEWS))
+ .build());
+ }
+}
diff --git a/src/java/org/apache/cassandra/db/virtual/TableMetricTables.java b/src/java/org/apache/cassandra/db/virtual/TableMetricTables.java
new file mode 100644
index 0000000..9ff421c
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/virtual/TableMetricTables.java
@@ -0,0 +1,265 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.virtual;
+
+import java.math.BigDecimal;
+import java.util.Collection;
+import java.util.function.Function;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.commons.math3.util.Precision;
+
+import com.codahale.metrics.Counting;
+import com.codahale.metrics.Gauge;
+import com.codahale.metrics.Metered;
+import com.codahale.metrics.Metric;
+import com.codahale.metrics.Sampling;
+import com.codahale.metrics.Snapshot;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.CompositeType;
+import org.apache.cassandra.db.marshal.DoubleType;
+import org.apache.cassandra.db.marshal.LongType;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.dht.IPartitioner;
+import org.apache.cassandra.dht.LocalPartitioner;
+import org.apache.cassandra.metrics.TableMetrics;
+import org.apache.cassandra.schema.TableMetadata;
+
+/**
+ * Contains multiple the Table Metric virtual tables. This is not a direct wrapper over the Metrics like with JMX but a
+ * view to the metrics so that the underlying mechanism can change but still give same appearance (like nodetool).
+ */
+public class TableMetricTables
+{
+ private final static String KEYSPACE_NAME = "keyspace_name";
+ private final static String TABLE_NAME = "table_name";
+ private final static String P50 = "p50th";
+ private final static String P99 = "p99th";
+ private final static String MAX = "max";
+ private final static String RATE = "per_second";
+ private final static double BYTES_TO_MIB = 1.0 / (1024 * 1024);
+ private final static double NS_TO_MS = 0.000001;
+
+ private final static AbstractType<?> TYPE = CompositeType.getInstance(UTF8Type.instance,
+ UTF8Type.instance);
+ private final static IPartitioner PARTITIONER = new LocalPartitioner(TYPE);
+
+ /**
+ * Generates all table metric tables in a collection
+ */
+ public static Collection<VirtualTable> getAll(String name)
+ {
+ return ImmutableList.of(
+ new LatencyTableMetric(name, "local_read_latency", t -> t.readLatency.latency),
+ new LatencyTableMetric(name, "local_scan_latency", t -> t.rangeLatency.latency),
+ new LatencyTableMetric(name, "coordinator_read_latency", t -> t.coordinatorReadLatency),
+ new LatencyTableMetric(name, "coordinator_scan_latency", t -> t.coordinatorScanLatency),
+ new LatencyTableMetric(name, "local_write_latency", t -> t.writeLatency.latency),
+ new LatencyTableMetric(name, "coordinator_write_latency", t -> t.coordinatorWriteLatency),
+ new HistogramTableMetric(name, "tombstones_per_read", t -> t.tombstoneScannedHistogram.cf),
+ new HistogramTableMetric(name, "rows_per_read", t -> t.liveScannedHistogram.cf),
+ new StorageTableMetric(name, "disk_usage", (TableMetrics t) -> t.totalDiskSpaceUsed),
+ new StorageTableMetric(name, "max_partition_size", (TableMetrics t) -> t.maxPartitionSize));
+ }
+
+ /**
+ * A table that describes a some amount of disk on space in a Counter or Gauge
+ */
+ private static class StorageTableMetric extends TableMetricTable
+ {
+ interface GaugeFunction extends Function<TableMetrics, Gauge<Long>> {}
+ interface CountingFunction<M extends Metric & Counting> extends Function<TableMetrics, M> {}
+
+ <M extends Metric & Counting> StorageTableMetric(String keyspace, String table, CountingFunction<M> func)
+ {
+ super(keyspace, table, func, "mebibytes", LongType.instance, "");
+ }
+
+ StorageTableMetric(String keyspace, String table, GaugeFunction func)
+ {
+ super(keyspace, table, func, "mebibytes", LongType.instance, "");
+ }
+
+ /**
+ * Convert bytes to mebibytes, always round up to nearest MiB
+ */
+ public void add(SimpleDataSet result, String column, long value)
+ {
+ result.column(column, (long) Math.ceil(value * BYTES_TO_MIB));
+ }
+ }
+
+ /**
+ * A table that describes a Latency metric, specifically a Timer
+ */
+ private static class HistogramTableMetric extends TableMetricTable
+ {
+ <M extends Metric & Sampling> HistogramTableMetric(String keyspace, String table, Function<TableMetrics, M> func)
+ {
+ this(keyspace, table, func, "");
+ }
+
+ <M extends Metric & Sampling> HistogramTableMetric(String keyspace, String table, Function<TableMetrics, M> func, String suffix)
+ {
+ super(keyspace, table, func, "count", LongType.instance, suffix);
+ }
+
+ /**
+ * When displaying in cqlsh if we allow doubles to be too precise we get scientific notation which is hard to
+ * read so round off at 0.000.
+ */
+ public void add(SimpleDataSet result, String column, double value)
+ {
+ result.column(column, Precision.round(value, 3, BigDecimal.ROUND_HALF_UP));
+ }
+ }
+
+ /**
+ * A table that describes a Latency metric, specifically a Timer
+ */
+ private static class LatencyTableMetric extends HistogramTableMetric
+ {
+ <M extends Metric & Sampling> LatencyTableMetric(String keyspace, String table, Function<TableMetrics, M> func)
+ {
+ super(keyspace, table, func, "_ms");
+ }
+
+ /**
+ * For the metrics that are time based, convert to to milliseconds
+ */
+ public void add(SimpleDataSet result, String column, double value)
+ {
+ if (column.endsWith(suffix))
+ value *= NS_TO_MS;
+
+ super.add(result, column, value);
+ }
+ }
+
+ /**
+ * Abstraction over the Metrics Gauge, Counter, and Timer that will turn it into a (keyspace_name, table_name)
+ * table.
+ */
+ private static class TableMetricTable extends AbstractVirtualTable
+ {
+ final Function<TableMetrics, ? extends Metric> func;
+ final String columnName;
+ final String suffix;
+
+ TableMetricTable(String keyspace, String table, Function<TableMetrics, ? extends Metric> func,
+ String colName, AbstractType colType, String suffix)
+ {
+ super(buildMetadata(keyspace, table, func, colName, colType, suffix));
+ this.func = func;
+ this.columnName = colName;
+ this.suffix = suffix;
+ }
+
+ public void add(SimpleDataSet result, String column, double value)
+ {
+ result.column(column, value);
+ }
+
+ public void add(SimpleDataSet result, String column, long value)
+ {
+ result.column(column, value);
+ }
+
+ public DataSet data()
+ {
+ SimpleDataSet result = new SimpleDataSet(metadata());
+
+ // Iterate over all tables and get metric by function
+ for (ColumnFamilyStore cfs : ColumnFamilyStore.all())
+ {
+ Metric metric = func.apply(cfs.metric);
+
+ // set new partition for this table
+ result.row(cfs.keyspace.getName(), cfs.name);
+
+ // extract information by metric type and put it in row based on implementation of `add`
+ if (metric instanceof Counting)
+ {
+ add(result, columnName, ((Counting) metric).getCount());
+ if (metric instanceof Sampling)
+ {
+ Sampling histo = (Sampling) metric;
+ Snapshot snapshot = histo.getSnapshot();
+ // EstimatedHistogram keeping them in ns is hard to parse as a human so convert to ms
+ add(result, P50 + suffix, snapshot.getMedian());
+ add(result, P99 + suffix, snapshot.get99thPercentile());
+ add(result, MAX + suffix, (double) snapshot.getMax());
+ }
+ if (metric instanceof Metered)
+ {
+ Metered timer = (Metered) metric;
+ add(result, RATE, timer.getFiveMinuteRate());
+ }
+ }
+ else if (metric instanceof Gauge)
+ {
+ add(result, columnName, (long) ((Gauge) metric).getValue());
+ }
+ }
+ return result;
+ }
+ }
+
+ /**
+ * Identify the type of Metric it is (gauge, counter etc) abd create the TableMetadata. The column name
+ * and type for a counter/gauge is formatted differently based on the units (bytes/time) so allowed to
+ * be set.
+ */
+ private static TableMetadata buildMetadata(String keyspace, String table, Function<TableMetrics, ? extends Metric> func,
+ String colName, AbstractType colType, String suffix)
+ {
+ TableMetadata.Builder metadata = TableMetadata.builder(keyspace, table)
+ .kind(TableMetadata.Kind.VIRTUAL)
+ .addPartitionKeyColumn(KEYSPACE_NAME, UTF8Type.instance)
+ .addPartitionKeyColumn(TABLE_NAME, UTF8Type.instance)
+ .partitioner(PARTITIONER);
+
+ // get a table from system keyspace and get metric from it for determining type of metric
+ Keyspace system = Keyspace.system().iterator().next();
+ Metric test = func.apply(system.getColumnFamilyStores().iterator().next().metric);
+
+ if (test instanceof Counting)
+ {
+ metadata.addRegularColumn(colName, colType);
+ // if it has a Histogram include some information about distribution
+ if (test instanceof Sampling)
+ {
+ metadata.addRegularColumn(P50 + suffix, DoubleType.instance)
+ .addRegularColumn(P99 + suffix, DoubleType.instance)
+ .addRegularColumn(MAX + suffix, DoubleType.instance);
+ }
+ if (test instanceof Metered)
+ {
+ metadata.addRegularColumn(RATE, DoubleType.instance);
+ }
+ }
+ else if (test instanceof Gauge)
+ {
+ metadata.addRegularColumn(colName, colType);
+ }
+ return metadata.build();
+ }
+}
diff --git a/src/java/org/apache/cassandra/db/virtual/ThreadPoolsTable.java b/src/java/org/apache/cassandra/db/virtual/ThreadPoolsTable.java
new file mode 100644
index 0000000..778e46f
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/virtual/ThreadPoolsTable.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db.virtual;
+
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.marshal.LongType;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.dht.LocalPartitioner;
+import org.apache.cassandra.metrics.ThreadPoolMetrics;
+import org.apache.cassandra.schema.TableMetadata;
+
+import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics;
+
+final class ThreadPoolsTable extends AbstractVirtualTable
+{
+ private static final String NAME = "name";
+ private static final String ACTIVE_TASKS = "active_tasks";
+ private static final String ACTIVE_TASKS_LIMIT = "active_tasks_limit";
+ private static final String PENDING_TASKS = "pending_tasks";
+ private static final String COMPLETED_TASKS = "completed_tasks";
+ private static final String BLOCKED_TASKS = "blocked_tasks";
+ private static final String BLOCKED_TASKS_ALL_TIME = "blocked_tasks_all_time";
+
+ ThreadPoolsTable(String keyspace)
+ {
+ super(TableMetadata.builder(keyspace, "thread_pools")
+ .kind(TableMetadata.Kind.VIRTUAL)
+ .partitioner(new LocalPartitioner(UTF8Type.instance))
+ .addPartitionKeyColumn(NAME, UTF8Type.instance)
+ .addRegularColumn(ACTIVE_TASKS, Int32Type.instance)
+ .addRegularColumn(ACTIVE_TASKS_LIMIT, Int32Type.instance)
+ .addRegularColumn(PENDING_TASKS, Int32Type.instance)
+ .addRegularColumn(COMPLETED_TASKS, LongType.instance)
+ .addRegularColumn(BLOCKED_TASKS, LongType.instance)
+ .addRegularColumn(BLOCKED_TASKS_ALL_TIME, LongType.instance)
+ .build());
+ }
+
+ @Override
+ public DataSet data(DecoratedKey partitionKey)
+ {
+ String poolName = UTF8Type.instance.compose(partitionKey.getKey());
+
+ SimpleDataSet result = new SimpleDataSet(metadata());
+ Metrics.getThreadPoolMetrics(poolName)
+ .ifPresent(metrics -> addRow(result, metrics));
+ return result;
+ }
+
+ @Override
+ public DataSet data()
+ {
+ SimpleDataSet result = new SimpleDataSet(metadata());
+ Metrics.allThreadPoolMetrics()
+ .forEach(metrics -> addRow(result, metrics));
+ return result;
+ }
+
+ private void addRow(SimpleDataSet dataSet, ThreadPoolMetrics metrics)
+ {
+ dataSet.row(metrics.poolName)
+ .column(ACTIVE_TASKS, metrics.activeTasks.getValue())
+ .column(ACTIVE_TASKS_LIMIT, metrics.maxPoolSize.getValue())
+ .column(PENDING_TASKS, metrics.pendingTasks.getValue())
+ .column(COMPLETED_TASKS, metrics.completedTasks.getValue())
+ .column(BLOCKED_TASKS, metrics.currentBlocked.getCount())
+ .column(BLOCKED_TASKS_ALL_TIME, metrics.totalBlocked.getCount());
+ }
+}
diff --git a/src/java/org/apache/cassandra/db/virtual/VirtualKeyspace.java b/src/java/org/apache/cassandra/db/virtual/VirtualKeyspace.java
new file mode 100644
index 0000000..6750215
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/virtual/VirtualKeyspace.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db.virtual;
+
+import java.util.Collection;
+
+import com.google.common.collect.ImmutableCollection;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+
+import org.apache.cassandra.schema.KeyspaceMetadata;
+import org.apache.cassandra.schema.Tables;
+
+public class VirtualKeyspace
+{
+ private final String name;
+ private final KeyspaceMetadata metadata;
+
+ private final ImmutableCollection<VirtualTable> tables;
+
+ public VirtualKeyspace(String name, Collection<VirtualTable> tables)
+ {
+ this.name = name;
+ this.tables = ImmutableList.copyOf(tables);
+
+ metadata = KeyspaceMetadata.virtual(name, Tables.of(Iterables.transform(tables, VirtualTable::metadata)));
+ }
+
+ public String name()
+ {
+ return name;
+ }
+
+ public KeyspaceMetadata metadata()
+ {
+ return metadata;
+ }
+
+ public ImmutableCollection<VirtualTable> tables()
+ {
+ return tables;
+ }
+}
diff --git a/src/java/org/apache/cassandra/db/virtual/VirtualKeyspaceRegistry.java b/src/java/org/apache/cassandra/db/virtual/VirtualKeyspaceRegistry.java
new file mode 100644
index 0000000..5e0f90c
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/virtual/VirtualKeyspaceRegistry.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db.virtual;
+
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import javax.annotation.Nullable;
+
+import com.google.common.collect.Iterables;
+
+import org.apache.cassandra.schema.KeyspaceMetadata;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.schema.TableMetadata;
+
+public final class VirtualKeyspaceRegistry
+{
+ public static final VirtualKeyspaceRegistry instance = new VirtualKeyspaceRegistry();
+
+ private final Map<String, VirtualKeyspace> virtualKeyspaces = new ConcurrentHashMap<>();
+ private final Map<TableId, VirtualTable> virtualTables = new ConcurrentHashMap<>();
+
+ private VirtualKeyspaceRegistry()
+ {
+ }
+
+ public void register(VirtualKeyspace keyspace)
+ {
+ virtualKeyspaces.put(keyspace.name(), keyspace);
+ keyspace.tables().forEach(t -> virtualTables.put(t.metadata().id, t));
+ }
+
+ @Nullable
+ public VirtualKeyspace getKeyspaceNullable(String name)
+ {
+ return virtualKeyspaces.get(name);
+ }
+
+ @Nullable
+ public VirtualTable getTableNullable(TableId id)
+ {
+ return virtualTables.get(id);
+ }
+
+ @Nullable
+ public KeyspaceMetadata getKeyspaceMetadataNullable(String name)
+ {
+ VirtualKeyspace keyspace = virtualKeyspaces.get(name);
+ return null != keyspace ? keyspace.metadata() : null;
+ }
+
+ @Nullable
+ public TableMetadata getTableMetadataNullable(TableId id)
+ {
+ VirtualTable table = virtualTables.get(id);
+ return null != table ? table.metadata() : null;
+ }
+
+ public Iterable<KeyspaceMetadata> virtualKeyspacesMetadata()
+ {
+ return Iterables.transform(virtualKeyspaces.values(), VirtualKeyspace::metadata);
+ }
+}
diff --git a/src/java/org/apache/cassandra/db/virtual/VirtualMutation.java b/src/java/org/apache/cassandra/db/virtual/VirtualMutation.java
new file mode 100644
index 0000000..09ac4a6
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/virtual/VirtualMutation.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db.virtual;
+
+import java.util.Collection;
+import java.util.concurrent.TimeUnit;
+
+import com.google.common.base.MoreObjects;
+import com.google.common.collect.ImmutableMap;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.IMutation;
+import org.apache.cassandra.db.partitions.PartitionUpdate;
+import org.apache.cassandra.schema.TableId;
+
+/**
+ * A specialised IMutation implementation for virtual keyspaces.
+ *
+ * Mainly overrides {@link #apply()} to go straight to {@link VirtualTable#apply(PartitionUpdate)} for every table involved.
+ */
+public final class VirtualMutation implements IMutation
+{
+ private final String keyspaceName;
+ private final DecoratedKey partitionKey;
+ private final ImmutableMap<TableId, PartitionUpdate> modifications;
+
+ public VirtualMutation(PartitionUpdate update)
+ {
+ this(update.metadata().keyspace, update.partitionKey(), ImmutableMap.of(update.metadata().id, update));
+ }
+
+ public VirtualMutation(String keyspaceName, DecoratedKey partitionKey, ImmutableMap<TableId, PartitionUpdate> modifications)
+ {
+ this.keyspaceName = keyspaceName;
+ this.partitionKey = partitionKey;
+ this.modifications = modifications;
+ }
+
+ @Override
+ public void apply()
+ {
+ modifications.forEach((id, update) -> VirtualKeyspaceRegistry.instance.getTableNullable(id).apply(update));
+ }
+
+ @Override
+ public String getKeyspaceName()
+ {
+ return keyspaceName;
+ }
+
+ @Override
+ public Collection<TableId> getTableIds()
+ {
+ return modifications.keySet();
+ }
+
+ @Override
+ public DecoratedKey key()
+ {
+ return partitionKey;
+ }
+
+ @Override
+ public long getTimeout(TimeUnit unit)
+ {
+ return DatabaseDescriptor.getWriteRpcTimeout(unit);
+ }
+
+ @Override
+ public String toString(boolean shallow)
+ {
+ MoreObjects.ToStringHelper helper =
+ MoreObjects.toStringHelper(this)
+ .add("keyspace", keyspaceName)
+ .add("partition key", partitionKey);
+
+ if (shallow)
+ helper.add("tables", getTableIds());
+ else
+ helper.add("modifications", getPartitionUpdates());
+
+ return helper.toString();
+ }
+
+ @Override
+ public Collection<PartitionUpdate> getPartitionUpdates()
+ {
+ return modifications.values();
+ }
+
+ @Override
+ public void validateIndexedColumns()
+ {
+ // no-op
+ }
+
+ public void validateSize(int version, int overhead)
+ {
+ // no-op
+ }
+}
diff --git a/src/java/org/apache/cassandra/db/virtual/VirtualSchemaKeyspace.java b/src/java/org/apache/cassandra/db/virtual/VirtualSchemaKeyspace.java
new file mode 100644
index 0000000..bb5a430
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/virtual/VirtualSchemaKeyspace.java
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db.virtual;
+
+import com.google.common.collect.ImmutableList;
+
+import org.apache.cassandra.db.marshal.BytesType;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.dht.LocalPartitioner;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.KeyspaceMetadata;
+import org.apache.cassandra.schema.TableMetadata;
+
+import static org.apache.cassandra.schema.SchemaConstants.VIRTUAL_SCHEMA;
+import static org.apache.cassandra.schema.TableMetadata.builder;
+
+public final class VirtualSchemaKeyspace extends VirtualKeyspace
+{
+ public static final VirtualSchemaKeyspace instance = new VirtualSchemaKeyspace();
+
+ private VirtualSchemaKeyspace()
+ {
+ super(VIRTUAL_SCHEMA, ImmutableList.of(new VirtualKeyspaces(VIRTUAL_SCHEMA), new VirtualTables(VIRTUAL_SCHEMA), new VirtualColumns(VIRTUAL_SCHEMA)));
+ }
+
+ private static final class VirtualKeyspaces extends AbstractVirtualTable
+ {
+ private static final String KEYSPACE_NAME = "keyspace_name";
+
+ private VirtualKeyspaces(String keyspace)
+ {
+ super(builder(keyspace, "keyspaces")
+ .comment("virtual keyspace definitions")
+ .kind(TableMetadata.Kind.VIRTUAL)
+ .partitioner(new LocalPartitioner(UTF8Type.instance))
+ .addPartitionKeyColumn(KEYSPACE_NAME, UTF8Type.instance)
+ .build());
+ }
+
+ public DataSet data()
+ {
+ SimpleDataSet result = new SimpleDataSet(metadata());
+ for (KeyspaceMetadata keyspace : VirtualKeyspaceRegistry.instance.virtualKeyspacesMetadata())
+ result.row(keyspace.name);
+ return result;
+ }
+ }
+
+ private static final class VirtualTables extends AbstractVirtualTable
+ {
+ private static final String KEYSPACE_NAME = "keyspace_name";
+ private static final String TABLE_NAME = "table_name";
+ private static final String COMMENT = "comment";
+
+ private VirtualTables(String keyspace)
+ {
+ super(builder(keyspace, "tables")
+ .comment("virtual table definitions")
+ .kind(TableMetadata.Kind.VIRTUAL)
+ .partitioner(new LocalPartitioner(UTF8Type.instance))
+ .addPartitionKeyColumn(KEYSPACE_NAME, UTF8Type.instance)
+ .addClusteringColumn(TABLE_NAME, UTF8Type.instance)
+ .addRegularColumn(COMMENT, UTF8Type.instance)
+ .build());
+ }
+
+ public DataSet data()
+ {
+ SimpleDataSet result = new SimpleDataSet(metadata());
+
+ for (KeyspaceMetadata keyspace : VirtualKeyspaceRegistry.instance.virtualKeyspacesMetadata())
+ {
+ for (TableMetadata table : keyspace.tables)
+ {
+ result.row(table.keyspace, table.name)
+ .column(COMMENT, table.params.comment);
+ }
+ }
+
+ return result;
+ }
+ }
+
+ private static final class VirtualColumns extends AbstractVirtualTable
+ {
+ private static final String KEYSPACE_NAME = "keyspace_name";
+ private static final String TABLE_NAME = "table_name";
+ private static final String COLUMN_NAME = "column_name";
+ private static final String CLUSTERING_ORDER = "clustering_order";
+ private static final String COLUMN_NAME_BYTES = "column_name_bytes";
+ private static final String KIND = "kind";
+ private static final String POSITION = "position";
+ private static final String TYPE = "type";
+
+ private VirtualColumns(String keyspace)
+ {
+ super(builder(keyspace, "columns")
+ .comment("virtual column definitions")
+ .kind(TableMetadata.Kind.VIRTUAL)
+ .partitioner(new LocalPartitioner(UTF8Type.instance))
+ .addPartitionKeyColumn(KEYSPACE_NAME, UTF8Type.instance)
+ .addClusteringColumn(TABLE_NAME, UTF8Type.instance)
+ .addClusteringColumn(COLUMN_NAME, UTF8Type.instance)
+ .addRegularColumn(CLUSTERING_ORDER, UTF8Type.instance)
+ .addRegularColumn(COLUMN_NAME_BYTES, BytesType.instance)
+ .addRegularColumn(KIND, UTF8Type.instance)
+ .addRegularColumn(POSITION, Int32Type.instance)
+ .addRegularColumn(TYPE, UTF8Type.instance)
+ .build());
+ }
+
+ public DataSet data()
+ {
+ SimpleDataSet result = new SimpleDataSet(metadata());
+
+ for (KeyspaceMetadata keyspace : VirtualKeyspaceRegistry.instance.virtualKeyspacesMetadata())
+ {
+ for (TableMetadata table : keyspace.tables)
+ {
+ for (ColumnMetadata column : table.columns())
+ {
+ result.row(column.ksName, column.cfName, column.name.toString())
+ .column(CLUSTERING_ORDER, column.clusteringOrder().toString().toLowerCase())
+ .column(COLUMN_NAME_BYTES, column.name.bytes)
+ .column(KIND, column.kind.toString().toLowerCase())
+ .column(POSITION, column.position())
+ .column(TYPE, column.type.asCQL3Type().toString());
+ }
+ }
+ }
+
+ return result;
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/db/virtual/VirtualTable.java b/src/java/org/apache/cassandra/db/virtual/VirtualTable.java
new file mode 100644
index 0000000..ea196ca
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/virtual/VirtualTable.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db.virtual;
+
+import org.apache.cassandra.db.DataRange;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.filter.ClusteringIndexFilter;
+import org.apache.cassandra.db.filter.ColumnFilter;
+import org.apache.cassandra.db.partitions.PartitionUpdate;
+import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
+import org.apache.cassandra.schema.TableMetadata;
+
+/**
+ * A system view used to expose system information.
+ */
+public interface VirtualTable
+{
+ /**
+ * Returns the view name.
+ *
+ * @return the view name.
+ */
+ default String name()
+ {
+ return metadata().name;
+ }
+
+ /**
+ * Returns the view metadata.
+ *
+ * @return the view metadata.
+ */
+ TableMetadata metadata();
+
+ /**
+ * Applies the specified update.
+ * @param update the update to apply
+ */
+ void apply(PartitionUpdate update);
+
+ /**
+ * Selects the rows from a single partition.
+ *
+ * @param partitionKey the partition key
+ * @param clusteringIndexFilter the clustering columns to selected
+ * @param columnFilter the selected columns
+ * @return the rows corresponding to the requested data.
+ */
+ UnfilteredPartitionIterator select(DecoratedKey partitionKey, ClusteringIndexFilter clusteringIndexFilter, ColumnFilter columnFilter);
+
+ /**
+ * Selects the rows from a range of partitions.
+ *
+ * @param dataRange the range of data to retrieve
+ * @param columnFilter the selected columns
+ * @return the rows corresponding to the requested data.
+ */
+ UnfilteredPartitionIterator select(DataRange dataRange, ColumnFilter columnFilter);
+}
diff --git a/src/java/org/apache/cassandra/dht/AbstractBounds.java b/src/java/org/apache/cassandra/dht/AbstractBounds.java
index 298c316..7a603b0 100644
--- a/src/java/org/apache/cassandra/dht/AbstractBounds.java
+++ b/src/java/org/apache/cassandra/dht/AbstractBounds.java
@@ -184,6 +184,9 @@
* The first int tells us if it's a range or bounds (depending on the value) _and_ if it's tokens or keys (depending on the
* sign). We use negative kind for keys so as to preserve the serialization of token from older version.
*/
+ // !WARNING! While we don't support the pre-3.0 messaging protocol, we serialize the token range in the
+ // system table (see SystemKeypsace.rangeToBytes) using the old/pre-3.0 format and until we deal with that
+ // problem, we have to preserve this code.
if (version < MessagingService.VERSION_30)
out.writeInt(kindInt(range));
else
@@ -195,6 +198,7 @@
public AbstractBounds<T> deserialize(DataInput in, IPartitioner p, int version) throws IOException
{
boolean isToken, startInclusive, endInclusive;
+ // !WARNING! See serialize method above for why we still need to have that condition.
if (version < MessagingService.VERSION_30)
{
int kind = in.readInt();
@@ -226,6 +230,7 @@
public long serializedSize(AbstractBounds<T> ab, int version)
{
+ // !WARNING! See serialize method above for why we still need to have that condition.
int size = version < MessagingService.VERSION_30
? TypeSizes.sizeof(kindInt(ab))
: 1;
diff --git a/src/java/org/apache/cassandra/dht/BootStrapper.java b/src/java/org/apache/cassandra/dht/BootStrapper.java
index 1e00f48..94bf283 100644
--- a/src/java/org/apache/cassandra/dht/BootStrapper.java
+++ b/src/java/org/apache/cassandra/dht/BootStrapper.java
@@ -17,8 +17,6 @@
*/
package org.apache.cassandra.dht;
-import java.io.IOException;
-import java.net.InetAddress;
import java.util.*;
import java.util.concurrent.atomic.AtomicInteger;
@@ -26,18 +24,16 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+
import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.config.Schema;
+import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.db.Keyspace;
-import org.apache.cassandra.db.TypeSizes;
import org.apache.cassandra.dht.tokenallocator.TokenAllocation;
import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.gms.FailureDetector;
import org.apache.cassandra.gms.Gossiper;
-import org.apache.cassandra.io.IVersionedSerializer;
-import org.apache.cassandra.io.util.DataInputPlus;
-import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.locator.AbstractReplicationStrategy;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.NetworkTopologyStrategy;
import org.apache.cassandra.locator.TokenMetadata;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.streaming.*;
@@ -51,12 +47,12 @@
private static final Logger logger = LoggerFactory.getLogger(BootStrapper.class);
/* endpoint that needs to be bootstrapped */
- protected final InetAddress address;
+ protected final InetAddressAndPort address;
/* token of the node being bootstrapped. */
protected final Collection<Token> tokens;
protected final TokenMetadata tokenMetadata;
- public BootStrapper(InetAddress address, Collection<Token> tokens, TokenMetadata tmd)
+ public BootStrapper(InetAddressAndPort address, Collection<Token> tokens, TokenMetadata tmd)
{
assert address != null;
assert tokens != null && !tokens.isEmpty();
@@ -73,14 +69,12 @@
RangeStreamer streamer = new RangeStreamer(tokenMetadata,
tokens,
address,
- "Bootstrap",
+ StreamOperation.BOOTSTRAP,
useStrictConsistency,
DatabaseDescriptor.getEndpointSnitch(),
stateStore,
- true);
- streamer.addSourceFilter(new RangeStreamer.FailureDetectorSourceFilter(FailureDetector.instance));
- streamer.addSourceFilter(new RangeStreamer.ExcludeLocalNodeFilter());
-
+ true,
+ DatabaseDescriptor.getStreamingConnectionsPerHost());
for (String keyspaceName : Schema.instance.getNonLocalStrategyKeyspaces())
{
AbstractReplicationStrategy strategy = Keyspace.open(keyspaceName).getReplicationStrategy();
@@ -158,16 +152,21 @@
* otherwise, if allocationKeyspace is specified use the token allocation algorithm to generate suitable tokens
* else choose num_tokens tokens at random
*/
- public static Collection<Token> getBootstrapTokens(final TokenMetadata metadata, InetAddress address, int schemaWaitDelay) throws ConfigurationException
+ public static Collection<Token> getBootstrapTokens(final TokenMetadata metadata, InetAddressAndPort address, int schemaWaitDelay) throws ConfigurationException
{
String allocationKeyspace = DatabaseDescriptor.getAllocateTokensForKeyspace();
+ Integer allocationLocalRf = DatabaseDescriptor.getAllocateTokensForLocalRf();
Collection<String> initialTokens = DatabaseDescriptor.getInitialTokens();
if (initialTokens.size() > 0 && allocationKeyspace != null)
logger.warn("manually specified tokens override automatic allocation");
// if user specified tokens, use those
if (initialTokens.size() > 0)
- return getSpecifiedTokens(metadata, initialTokens);
+ {
+ Collection<Token> tokens = getSpecifiedTokens(metadata, initialTokens);
+ BootstrapDiagnostics.useSpecifiedTokens(address, allocationKeyspace, tokens, DatabaseDescriptor.getNumTokens());
+ return tokens;
+ }
int numTokens = DatabaseDescriptor.getNumTokens();
if (numTokens < 1)
@@ -176,10 +175,15 @@
if (allocationKeyspace != null)
return allocateTokens(metadata, address, allocationKeyspace, numTokens, schemaWaitDelay);
+ if (allocationLocalRf != null)
+ return allocateTokens(metadata, address, allocationLocalRf, numTokens, schemaWaitDelay);
+
if (numTokens == 1)
logger.warn("Picking random token for a single vnode. You should probably add more vnodes and/or use the automatic token allocation mechanism.");
- return getRandomTokens(metadata, numTokens);
+ Collection<Token> tokens = getRandomTokens(metadata, numTokens);
+ BootstrapDiagnostics.useRandomTokens(address, metadata, numTokens, tokens);
+ return tokens;
}
private static Collection<Token> getSpecifiedTokens(final TokenMetadata metadata,
@@ -198,13 +202,13 @@
}
static Collection<Token> allocateTokens(final TokenMetadata metadata,
- InetAddress address,
+ InetAddressAndPort address,
String allocationKeyspace,
int numTokens,
int schemaWaitDelay)
{
StorageService.instance.waitForSchema(schemaWaitDelay);
- if (!FBUtilities.getBroadcastAddress().equals(InetAddress.getLoopbackAddress()))
+ if (!FBUtilities.getBroadcastAddressAndPort().equals(InetAddressAndPort.getLoopbackAddress()))
Gossiper.waitToSettle();
Keyspace ks = Keyspace.open(allocationKeyspace);
@@ -212,7 +216,25 @@
throw new ConfigurationException("Problem opening token allocation keyspace " + allocationKeyspace);
AbstractReplicationStrategy rs = ks.getReplicationStrategy();
- return TokenAllocation.allocateTokens(metadata, rs, address, numTokens);
+ Collection<Token> tokens = TokenAllocation.allocateTokens(metadata, rs, address, numTokens);
+ BootstrapDiagnostics.tokensAllocated(address, metadata, allocationKeyspace, numTokens, tokens);
+ return tokens;
+ }
+
+
+ static Collection<Token> allocateTokens(final TokenMetadata metadata,
+ InetAddressAndPort address,
+ int rf,
+ int numTokens,
+ int schemaWaitDelay)
+ {
+ StorageService.instance.waitForSchema(schemaWaitDelay);
+ if (!FBUtilities.getBroadcastAddressAndPort().equals(InetAddressAndPort.getLoopbackAddress()))
+ Gossiper.waitToSettle();
+
+ Collection<Token> tokens = TokenAllocation.allocateTokens(metadata, rf, address, numTokens);
+ BootstrapDiagnostics.tokensAllocated(address, metadata, rf, numTokens, tokens);
+ return tokens;
}
public static Collection<Token> getRandomTokens(TokenMetadata metadata, int numTokens)
@@ -228,24 +250,4 @@
logger.info("Generated random tokens. tokens are {}", tokens);
return tokens;
}
-
- public static class StringSerializer implements IVersionedSerializer<String>
- {
- public static final StringSerializer instance = new StringSerializer();
-
- public void serialize(String s, DataOutputPlus out, int version) throws IOException
- {
- out.writeUTF(s);
- }
-
- public String deserialize(DataInputPlus in, int version) throws IOException
- {
- return in.readUTF();
- }
-
- public long serializedSize(String s, int version)
- {
- return TypeSizes.sizeof(s);
- }
- }
}
diff --git a/src/java/org/apache/cassandra/dht/BootstrapDiagnostics.java b/src/java/org/apache/cassandra/dht/BootstrapDiagnostics.java
new file mode 100644
index 0000000..5c2b46a
--- /dev/null
+++ b/src/java/org/apache/cassandra/dht/BootstrapDiagnostics.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.dht;
+
+import java.util.Collection;
+import com.google.common.collect.ImmutableList;
+
+import org.apache.cassandra.dht.BootstrapEvent.BootstrapEventType;
+import org.apache.cassandra.diag.DiagnosticEventService;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.TokenMetadata;
+
+/**
+ * Utility methods for bootstrap related activities.
+ */
+final class BootstrapDiagnostics
+{
+ private static final DiagnosticEventService service = DiagnosticEventService.instance();
+
+ private BootstrapDiagnostics()
+ {
+ }
+
+ static void useSpecifiedTokens(InetAddressAndPort address, String allocationKeyspace, Collection<Token> initialTokens,
+ int numTokens)
+ {
+ if (isEnabled(BootstrapEventType.BOOTSTRAP_USING_SPECIFIED_TOKENS))
+ service.publish(new BootstrapEvent(BootstrapEventType.BOOTSTRAP_USING_SPECIFIED_TOKENS,
+ address,
+ null,
+ allocationKeyspace,
+ null,
+ numTokens,
+ ImmutableList.copyOf(initialTokens)));
+ }
+
+ static void useRandomTokens(InetAddressAndPort address, TokenMetadata metadata, int numTokens, Collection<Token> tokens)
+ {
+ if (isEnabled(BootstrapEventType.BOOTSTRAP_USING_RANDOM_TOKENS))
+ service.publish(new BootstrapEvent(BootstrapEventType.BOOTSTRAP_USING_RANDOM_TOKENS,
+ address,
+ metadata.cloneOnlyTokenMap(),
+ null,
+ null,
+ numTokens,
+ ImmutableList.copyOf(tokens)));
+ }
+
+ static void tokensAllocated(InetAddressAndPort address, TokenMetadata metadata,
+ String allocationKeyspace, int numTokens, Collection<Token> tokens)
+ {
+ if (isEnabled(BootstrapEventType.TOKENS_ALLOCATED))
+ service.publish(new BootstrapEvent(BootstrapEventType.TOKENS_ALLOCATED,
+ address,
+ metadata.cloneOnlyTokenMap(),
+ allocationKeyspace,
+ null,
+ numTokens,
+ ImmutableList.copyOf(tokens)));
+ }
+
+ static void tokensAllocated(InetAddressAndPort address, TokenMetadata metadata,
+ int rf, int numTokens, Collection<Token> tokens)
+ {
+ if (isEnabled(BootstrapEventType.TOKENS_ALLOCATED))
+ service.publish(new BootstrapEvent(BootstrapEventType.TOKENS_ALLOCATED,
+ address,
+ metadata.cloneOnlyTokenMap(),
+ null,
+ rf,
+ numTokens,
+ ImmutableList.copyOf(tokens)));
+ }
+
+ private static boolean isEnabled(BootstrapEventType type)
+ {
+ return service.isEnabled(BootstrapEvent.class, type);
+ }
+
+}
diff --git a/src/java/org/apache/cassandra/dht/BootstrapEvent.java b/src/java/org/apache/cassandra/dht/BootstrapEvent.java
new file mode 100644
index 0000000..4936c29
--- /dev/null
+++ b/src/java/org/apache/cassandra/dht/BootstrapEvent.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.dht;
+
+import java.io.Serializable;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+import javax.annotation.Nullable;
+
+import com.google.common.collect.ImmutableCollection;
+
+import org.apache.cassandra.diag.DiagnosticEvent;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.TokenMetadata;
+
+/**
+ * DiagnosticEvent implementation for bootstrap related activities.
+ */
+final class BootstrapEvent extends DiagnosticEvent
+{
+
+ private final BootstrapEventType type;
+ @Nullable
+ private final TokenMetadata tokenMetadata;
+ private final InetAddressAndPort address;
+ @Nullable
+ private final String allocationKeyspace;
+ @Nullable
+ private final Integer rf;
+ private final Integer numTokens;
+ private final Collection<Token> tokens;
+
+ BootstrapEvent(BootstrapEventType type, InetAddressAndPort address, @Nullable TokenMetadata tokenMetadata,
+ @Nullable String allocationKeyspace, @Nullable Integer rf, int numTokens, ImmutableCollection<Token> tokens)
+ {
+ this.type = type;
+ this.address = address;
+ this.tokenMetadata = tokenMetadata;
+ this.allocationKeyspace = allocationKeyspace;
+ this.rf = rf;
+ this.numTokens = numTokens;
+ this.tokens = tokens;
+ }
+
+ enum BootstrapEventType
+ {
+ BOOTSTRAP_USING_SPECIFIED_TOKENS,
+ BOOTSTRAP_USING_RANDOM_TOKENS,
+ TOKENS_ALLOCATED
+ }
+
+
+ public BootstrapEventType getType()
+ {
+ return type;
+ }
+
+ public Map<String, Serializable> toMap()
+ {
+ // be extra defensive against nulls and bugs
+ HashMap<String, Serializable> ret = new HashMap<>();
+ ret.put("tokenMetadata", String.valueOf(tokenMetadata));
+ ret.put("allocationKeyspace", allocationKeyspace);
+ ret.put("rf", rf);
+ ret.put("numTokens", numTokens);
+ ret.put("tokens", String.valueOf(tokens));
+ return ret;
+ }
+}
diff --git a/src/java/org/apache/cassandra/dht/ByteOrderedPartitioner.java b/src/java/org/apache/cassandra/dht/ByteOrderedPartitioner.java
index 1271a5a..a6314dc 100644
--- a/src/java/org/apache/cassandra/dht/ByteOrderedPartitioner.java
+++ b/src/java/org/apache/cassandra/dht/ByteOrderedPartitioner.java
@@ -17,8 +17,8 @@
*/
package org.apache.cassandra.dht;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.Schema;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.db.BufferDecoratedKey;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.marshal.AbstractType;
@@ -43,6 +43,8 @@
import java.util.Random;
import java.util.concurrent.ThreadLocalRandom;
+import com.google.common.collect.Maps;
+
public class ByteOrderedPartitioner implements IPartitioner
{
public static final BytesToken MINIMUM = new BytesToken(ArrayUtils.EMPTY_BYTE_ARRAY);
@@ -219,7 +221,7 @@
return new BytesToken(buffer);
}
- private final Token.TokenFactory tokenFactory = new Token.TokenFactory()
+ private final Token.TokenFactory tokenFactory = new Token.TokenFactory()
{
public ByteBuffer toByteArray(Token token)
{
@@ -232,6 +234,12 @@
return new BytesToken(bytes);
}
+ @Override
+ public int byteSize(Token token)
+ {
+ return ((BytesToken) token).token.length;
+ }
+
public String toString(Token token)
{
BytesToken bytesToken = (BytesToken) token;
@@ -273,7 +281,7 @@
public Map<Token, Float> describeOwnership(List<Token> sortedTokens)
{
// allTokens will contain the count and be returned, sorted_ranges is shorthand for token<->token math.
- Map<Token, Float> allTokens = new HashMap<Token, Float>();
+ Map<Token, Float> allTokens = Maps.newHashMapWithExpectedSize(sortedTokens.size());
List<Range<Token>> sortedRanges = new ArrayList<Range<Token>>(sortedTokens.size());
// this initializes the counts to 0 and calcs the ranges in order.
@@ -287,12 +295,12 @@
for (String ks : Schema.instance.getKeyspaces())
{
- for (CFMetaData cfmd : Schema.instance.getTablesAndViews(ks))
+ for (TableMetadata cfmd : Schema.instance.getTablesAndViews(ks))
{
for (Range<Token> r : sortedRanges)
{
// Looping over every KS:CF:Range, get the splits size and add it to the count
- allTokens.put(r.right, allTokens.get(r.right) + StorageService.instance.getSplits(ks, cfmd.cfName, r, 1).size());
+ allTokens.put(r.right, allTokens.get(r.right) + StorageService.instance.getSplits(ks, cfmd.name, r, 1).size());
}
}
}
diff --git a/src/java/org/apache/cassandra/dht/Datacenters.java b/src/java/org/apache/cassandra/dht/Datacenters.java
new file mode 100644
index 0000000..9695a09
--- /dev/null
+++ b/src/java/org/apache/cassandra/dht/Datacenters.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.dht;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.locator.IEndpointSnitch;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.utils.FBUtilities;
+
+public class Datacenters
+{
+
+ private static class DCHandle
+ {
+ private static final String thisDc = DatabaseDescriptor.getEndpointSnitch().getLocalDatacenter();
+ }
+
+ public static String thisDatacenter()
+ {
+ return DCHandle.thisDc;
+ }
+
+ /*
+ * (non-javadoc) Method to generate list of valid data center names to be used to validate the replication parameters during CREATE / ALTER keyspace operations.
+ * All peers of current node are fetched from {@link TokenMetadata} and then a set is build by fetching DC name of each peer.
+ * @return a set of valid DC names
+ */
+ public static Set<String> getValidDatacenters()
+ {
+ final Set<String> validDataCenters = new HashSet<>();
+ final IEndpointSnitch snitch = DatabaseDescriptor.getEndpointSnitch();
+
+ // Add data center of localhost.
+ validDataCenters.add(thisDatacenter());
+ // Fetch and add DCs of all peers.
+ for (InetAddressAndPort peer : StorageService.instance.getTokenMetadata().getAllEndpoints())
+ {
+ validDataCenters.add(snitch.getDatacenter(peer));
+ }
+
+ return validDataCenters;
+ }
+}
diff --git a/src/java/org/apache/cassandra/dht/IPartitioner.java b/src/java/org/apache/cassandra/dht/IPartitioner.java
index e342bd0..ef8ced2 100644
--- a/src/java/org/apache/cassandra/dht/IPartitioner.java
+++ b/src/java/org/apache/cassandra/dht/IPartitioner.java
@@ -18,6 +18,7 @@
package org.apache.cassandra.dht;
import java.nio.ByteBuffer;
+import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Optional;
@@ -25,9 +26,29 @@
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.service.StorageService;
public interface IPartitioner
{
+ static IPartitioner global()
+ {
+ return StorageService.instance.getTokenMetadata().partitioner;
+ }
+
+ static void validate(Collection<? extends AbstractBounds<?>> allBounds)
+ {
+ for (AbstractBounds<?> bounds : allBounds)
+ validate(bounds);
+ }
+
+ static void validate(AbstractBounds<?> bounds)
+ {
+ if (global() != bounds.left.getPartitioner())
+ throw new AssertionError(String.format("Partitioner in bounds serialization. Expected %s, was %s.",
+ global().getClass().getName(),
+ bounds.left.getPartitioner().getClass().getName()));
+ }
+
/**
* Transform key to object representation of the on-disk format.
*
@@ -45,7 +66,7 @@
public Token midpoint(Token left, Token right);
/**
- * Calculate a Token which take approximate 0 <= ratioToLeft <= 1 ownership of the given range.
+ * Calculate a Token which take {@code approximate 0 <= ratioToLeft <= 1} ownership of the given range.
*/
public Token split(Token left, Token right, double ratioToLeft);
@@ -114,4 +135,9 @@
{
return Optional.empty();
}
+
+ default public int getMaxTokenSize()
+ {
+ return Integer.MIN_VALUE;
+ }
}
diff --git a/src/java/org/apache/cassandra/dht/Murmur3Partitioner.java b/src/java/org/apache/cassandra/dht/Murmur3Partitioner.java
index 0f922e3..52d0efb 100644
--- a/src/java/org/apache/cassandra/dht/Murmur3Partitioner.java
+++ b/src/java/org/apache/cassandra/dht/Murmur3Partitioner.java
@@ -17,6 +17,7 @@
*/
package org.apache.cassandra.dht;
+import java.io.IOException;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.nio.ByteBuffer;
@@ -25,10 +26,12 @@
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.PreHashedDecoratedKey;
+import org.apache.cassandra.db.TypeSizes;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.db.marshal.PartitionerDefinedOrder;
import org.apache.cassandra.db.marshal.LongType;
import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.MurmurHash;
import org.apache.cassandra.utils.ObjectSizes;
@@ -42,6 +45,7 @@
{
public static final LongToken MINIMUM = new LongToken(Long.MIN_VALUE);
public static final long MAXIMUM = Long.MAX_VALUE;
+ private static final int MAXIMUM_TOKEN_SIZE = TypeSizes.sizeof(MAXIMUM);
private static final int HEAP_SIZE = (int) ObjectSizes.measureDeep(MINIMUM);
@@ -224,6 +228,11 @@
return new LongToken(normalize(hash[0]));
}
+ public int getMaxTokenSize()
+ {
+ return MAXIMUM_TOKEN_SIZE;
+ }
+
private long[] getHash(ByteBuffer key)
{
long[] hash = new long[2];
@@ -300,11 +309,35 @@
return ByteBufferUtil.bytes(longToken.token);
}
+ @Override
+ public void serialize(Token token, DataOutputPlus out) throws IOException
+ {
+ out.writeLong(((LongToken) token).token);
+ }
+
+ @Override
+ public void serialize(Token token, ByteBuffer out)
+ {
+ out.putLong(((LongToken) token).token);
+ }
+
+ @Override
+ public int byteSize(Token token)
+ {
+ return 8;
+ }
+
public Token fromByteArray(ByteBuffer bytes)
{
return new LongToken(ByteBufferUtil.toLong(bytes));
}
+ @Override
+ public Token fromByteBuffer(ByteBuffer bytes, int position, int length)
+ {
+ return new LongToken(bytes.getLong(position));
+ }
+
public String toString(Token token)
{
return token.toString();
diff --git a/src/java/org/apache/cassandra/dht/OrderPreservingPartitioner.java b/src/java/org/apache/cassandra/dht/OrderPreservingPartitioner.java
index 954b0af..16c5db1 100644
--- a/src/java/org/apache/cassandra/dht/OrderPreservingPartitioner.java
+++ b/src/java/org/apache/cassandra/dht/OrderPreservingPartitioner.java
@@ -23,13 +23,14 @@
import java.util.*;
import java.util.concurrent.ThreadLocalRandom;
-import org.apache.cassandra.config.*;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.CachedHashDecoratedKey;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.db.marshal.UTF8Type;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.gms.VersionedValue;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FBUtilities;
@@ -226,12 +227,12 @@
for (String ks : Schema.instance.getKeyspaces())
{
- for (CFMetaData cfmd : Schema.instance.getTablesAndViews(ks))
+ for (TableMetadata cfmd : Schema.instance.getTablesAndViews(ks))
{
for (Range<Token> r : sortedRanges)
{
// Looping over every KS:CF:Range, get the splits size and add it to the count
- allTokens.put(r.right, allTokens.get(r.right) + StorageService.instance.getSplits(ks, cfmd.cfName, r, cfmd.params.minIndexInterval).size());
+ allTokens.put(r.right, allTokens.get(r.right) + StorageService.instance.getSplits(ks, cfmd.name, r, cfmd.params.minIndexInterval).size());
}
}
}
diff --git a/src/java/org/apache/cassandra/dht/RandomPartitioner.java b/src/java/org/apache/cassandra/dht/RandomPartitioner.java
index bdf8b85..241b785 100644
--- a/src/java/org/apache/cassandra/dht/RandomPartitioner.java
+++ b/src/java/org/apache/cassandra/dht/RandomPartitioner.java
@@ -17,6 +17,7 @@
*/
package org.apache.cassandra.dht;
+import java.io.IOException;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.nio.ByteBuffer;
@@ -31,6 +32,7 @@
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.db.marshal.IntegerType;
import org.apache.cassandra.db.marshal.PartitionerDefinedOrder;
+import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.GuidGenerator;
@@ -45,6 +47,7 @@
public static final BigInteger ZERO = new BigInteger("0");
public static final BigIntegerToken MINIMUM = new BigIntegerToken("-1");
public static final BigInteger MAXIMUM = new BigInteger("2").pow(127);
+ public static final int MAXIMUM_TOKEN_SIZE = MAXIMUM.bitLength() / 8 + 1;
/**
* Maintain a separate threadlocal message digest, exclusively for token hashing. This is necessary because
@@ -161,11 +164,35 @@
return ByteBuffer.wrap(bigIntegerToken.token.toByteArray());
}
+ @Override
+ public void serialize(Token token, DataOutputPlus out) throws IOException
+ {
+ out.write(((BigIntegerToken) token).token.toByteArray());
+ }
+
+ @Override
+ public void serialize(Token token, ByteBuffer out)
+ {
+ out.put(((BigIntegerToken) token).token.toByteArray());
+ }
+
+ @Override
+ public int byteSize(Token token)
+ {
+ return ((BigIntegerToken) token).token.bitLength() / 8 + 1;
+ }
+
public Token fromByteArray(ByteBuffer bytes)
{
return new BigIntegerToken(new BigInteger(ByteBufferUtil.getArray(bytes)));
}
+ @Override
+ public Token fromByteBuffer(ByteBuffer bytes, int position, int length)
+ {
+ return new BigIntegerToken(new BigInteger(ByteBufferUtil.getArray(bytes, position, length)));
+ }
+
public String toString(Token token)
{
BigIntegerToken bigIntegerToken = (BigIntegerToken) token;
@@ -251,6 +278,11 @@
return new BigIntegerToken(hashToBigInteger(key));
}
+ public int getMaxTokenSize()
+ {
+ return MAXIMUM_TOKEN_SIZE;
+ }
+
public Map<Token, Float> describeOwnership(List<Token> sortedTokens)
{
Map<Token, Float> ownerships = new HashMap<Token, Float>();
diff --git a/src/java/org/apache/cassandra/dht/Range.java b/src/java/org/apache/cassandra/dht/Range.java
index 72c61c7..80c9ef1 100644
--- a/src/java/org/apache/cassandra/dht/Range.java
+++ b/src/java/org/apache/cassandra/dht/Range.java
@@ -19,6 +19,7 @@
import java.io.Serializable;
import java.util.*;
+import java.util.function.Predicate;
import org.apache.commons.lang3.ObjectUtils;
@@ -548,7 +549,7 @@
/**
* Helper class to check if a token is contained within a given collection of ranges
*/
- public static class OrderedRangeContainmentChecker
+ public static class OrderedRangeContainmentChecker implements Predicate<Token>
{
private final Iterator<Range<Token>> normalizedRangesIterator;
private Token lastToken = null;
@@ -569,7 +570,8 @@
* @param t token to check, must be larger than or equal to the last token passed
* @return true if the token is contained within the ranges given to the constructor.
*/
- public boolean contains(Token t)
+ @Override
+ public boolean test(Token t)
{
assert lastToken == null || lastToken.compareTo(t) <= 0;
lastToken = t;
@@ -586,4 +588,25 @@
}
}
}
+
+ public static <T extends RingPosition<T>> void assertNormalized(List<Range<T>> ranges)
+ {
+ Range<T> lastRange = null;
+ for (Range<T> range : ranges)
+ {
+ if (lastRange == null)
+ {
+ lastRange = range;
+ }
+ else if (lastRange.left.compareTo(range.left) >= 0 || lastRange.intersects(range))
+ {
+ throw new AssertionError(String.format("Ranges aren't properly normalized. lastRange %s, range %s, compareTo %d, intersects %b, all ranges %s%n",
+ lastRange,
+ range,
+ lastRange.compareTo(range),
+ lastRange.intersects(range),
+ ranges));
+ }
+ }
+ }
}
diff --git a/src/java/org/apache/cassandra/dht/RangeFetchMapCalculator.java b/src/java/org/apache/cassandra/dht/RangeFetchMapCalculator.java
new file mode 100644
index 0000000..2a2de01
--- /dev/null
+++ b/src/java/org/apache/cassandra/dht/RangeFetchMapCalculator.java
@@ -0,0 +1,540 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.dht;
+
+import java.math.BigInteger;
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Predicate;
+import com.google.common.base.Predicates;
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.Multimap;
+
+import org.apache.cassandra.locator.EndpointsByRange;
+import org.apache.cassandra.locator.EndpointsForRange;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.Replica;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.locator.Replicas;
+import org.psjava.algo.graph.flownetwork.FordFulkersonAlgorithm;
+import org.psjava.algo.graph.flownetwork.MaximumFlowAlgorithm;
+import org.psjava.algo.graph.flownetwork.MaximumFlowAlgorithmResult;
+import org.psjava.algo.graph.pathfinder.DFSPathFinder;
+import org.psjava.ds.graph.CapacityEdge;
+import org.psjava.ds.graph.MutableCapacityGraph;
+import org.psjava.ds.numbersystrem.IntegerNumberSystem;
+import org.psjava.ds.math.Function;
+
+/**
+ * We model the graph like this:
+ * * Each range we are about to stream is a vertex in the graph
+ * * Each node that can provide a range is a vertex in the graph
+ * * We add an edge from each range to the node that can provide the range
+ * * Then, to be able to solve the maximum flow problem using Ford-Fulkerson we add a super source with edges to all range vertices
+ * and a super sink with incoming edges from all the node vertices.
+ * * The capacity on the edges between the super source and the range-vertices is 1
+ * * The capacity on the edges between the range-vertices and the node vertices is infinite
+ * * The capacity on the edges between the nodes-vertices and the super sink is ceil(#range-vertices/#node-vertices)
+ * - if we have more machines than ranges to stream the capacity will be 1 (each machine will stream at most 1 range)
+ * * Since the sum of the capacity on the edges from the super source to the range-vertices is less or equal to the sum
+ * of the capacities between the node-vertices and super sink we know that to get maximum flow we will use all the
+ * range-vertices. (Say we have x ranges, y machines to provide them, total supersource -> range-vertice capacity will be x,
+ * total node-vertice -> supersink capacity will be (y * ceil(x / y)) which worst case is x if x==y). The capacity between
+ * the range-vertices and node-vertices is infinite.
+ * * Then we try to solve the max-flow problem using psjava
+ * * If we can't find a solution where the total flow is = number of range-vertices, we bump the capacity between the node-vertices
+ * and the super source and try again.
+ *
+ *
+ */
+public class RangeFetchMapCalculator
+{
+ private static final Logger logger = LoggerFactory.getLogger(RangeFetchMapCalculator.class);
+ private static final long TRIVIAL_RANGE_LIMIT = 1000;
+ private final EndpointsByRange rangesWithSources;
+ private final Predicate<Replica> sourceFilters;
+ private final String keyspace;
+ //We need two Vertices to act as source and destination in the algorithm
+ private final Vertex sourceVertex = OuterVertex.getSourceVertex();
+ private final Vertex destinationVertex = OuterVertex.getDestinationVertex();
+ private final Set<Range<Token>> trivialRanges;
+
+ public RangeFetchMapCalculator(EndpointsByRange rangesWithSources,
+ Collection<RangeStreamer.SourceFilter> sourceFilters,
+ String keyspace)
+ {
+ this.rangesWithSources = rangesWithSources;
+ this.sourceFilters = Predicates.and(sourceFilters);
+ this.keyspace = keyspace;
+ this.trivialRanges = rangesWithSources.keySet()
+ .stream()
+ .filter(RangeFetchMapCalculator::isTrivial)
+ .collect(Collectors.toSet());
+ }
+
+ static boolean isTrivial(Range<Token> range)
+ {
+ IPartitioner partitioner = DatabaseDescriptor.getPartitioner();
+ if (partitioner.splitter().isPresent())
+ {
+ BigInteger l = partitioner.splitter().get().valueForToken(range.left);
+ BigInteger r = partitioner.splitter().get().valueForToken(range.right);
+ if (r.compareTo(l) <= 0)
+ return false;
+ if (r.subtract(l).compareTo(BigInteger.valueOf(TRIVIAL_RANGE_LIMIT)) < 0)
+ return true;
+ }
+ return false;
+ }
+
+ public Multimap<InetAddressAndPort, Range<Token>> getRangeFetchMap()
+ {
+ Multimap<InetAddressAndPort, Range<Token>> fetchMap = HashMultimap.create();
+ fetchMap.putAll(getRangeFetchMapForNonTrivialRanges());
+ fetchMap.putAll(getRangeFetchMapForTrivialRanges(fetchMap));
+ return fetchMap;
+ }
+
+ @VisibleForTesting
+ Multimap<InetAddressAndPort, Range<Token>> getRangeFetchMapForNonTrivialRanges()
+ {
+ //Get the graph with edges between ranges and their source endpoints
+ MutableCapacityGraph<Vertex, Integer> graph = getGraph();
+ //Add source and destination vertex and edges
+ addSourceAndDestination(graph, getDestinationLinkCapacity(graph));
+
+ int flow = 0;
+ MaximumFlowAlgorithmResult<Integer, CapacityEdge<Vertex, Integer>> result = null;
+
+ //We might not be working on all ranges
+ while (flow < getTotalRangeVertices(graph))
+ {
+ if (flow > 0)
+ {
+ //We could not find a path with previous graph. Bump the capacity b/w endpoint vertices and destination by 1
+ incrementCapacity(graph, 1);
+ }
+
+ MaximumFlowAlgorithm fordFulkerson = FordFulkersonAlgorithm.getInstance(DFSPathFinder.getInstance());
+ result = fordFulkerson.calc(graph, sourceVertex, destinationVertex, IntegerNumberSystem.getInstance());
+
+ int newFlow = result.calcTotalFlow();
+ assert newFlow > flow; //We are not making progress which should not happen
+ flow = newFlow;
+ }
+
+ return getRangeFetchMapFromGraphResult(graph, result);
+ }
+
+ @VisibleForTesting
+ Multimap<InetAddressAndPort, Range<Token>> getRangeFetchMapForTrivialRanges(Multimap<InetAddressAndPort, Range<Token>> optimisedMap)
+ {
+ Multimap<InetAddressAndPort, Range<Token>> fetchMap = HashMultimap.create();
+ for (Range<Token> trivialRange : trivialRanges)
+ {
+ boolean added = false;
+ boolean localDCCheck = true;
+ while (!added)
+ {
+ // sort with the endpoint having the least number of streams first:
+ EndpointsForRange replicas = rangesWithSources.get(trivialRange)
+ .sorted(Comparator.comparingInt(o -> optimisedMap.get(o.endpoint()).size()));
+ Replicas.temporaryAssertFull(replicas);
+ for (Replica replica : replicas)
+ {
+ if (passFilters(replica, localDCCheck))
+ {
+ fetchMap.put(replica.endpoint(), trivialRange);
+ added = true;
+ break;
+ }
+ }
+ if (!added && !localDCCheck)
+ throw new IllegalStateException("Unable to find sufficient sources for streaming range " + trivialRange + " in keyspace " + keyspace);
+ if (!added)
+ logger.info("Using other DC endpoints for streaming for range: {} and keyspace {}", trivialRange, keyspace);
+ localDCCheck = false;
+ }
+ }
+ return fetchMap;
+ }
+ /*
+ Return the total number of range vertices in the graph
+ */
+ private int getTotalRangeVertices(MutableCapacityGraph<Vertex, Integer> graph)
+ {
+ int count = 0;
+ for (Vertex vertex : graph.getVertices())
+ {
+ if (vertex.isRangeVertex())
+ {
+ count++;
+ }
+ }
+
+ return count;
+ }
+
+ /**
+ * Convert the max flow graph to Multimap<InetAddress, Range<Token>>
+ * We iterate over all range vertices and find an edge with flow of more than zero connecting to endpoint vertex.
+ * @param graph The graph to convert
+ * @param result Flow algorithm result
+ * @return Multi Map of Machine to Ranges
+ */
+ private Multimap<InetAddressAndPort, Range<Token>> getRangeFetchMapFromGraphResult(MutableCapacityGraph<Vertex, Integer> graph, MaximumFlowAlgorithmResult<Integer, CapacityEdge<Vertex, Integer>> result)
+ {
+ final Multimap<InetAddressAndPort, Range<Token>> rangeFetchMapMap = HashMultimap.create();
+ if(result == null)
+ return rangeFetchMapMap;
+ final Function<CapacityEdge<Vertex, Integer>, Integer> flowFunction = result.calcFlowFunction();
+
+ for (Vertex vertex : graph.getVertices())
+ {
+ if (vertex.isRangeVertex())
+ {
+ boolean sourceFound = false;
+ for (CapacityEdge<Vertex, Integer> e : graph.getEdges(vertex))
+ {
+ if(flowFunction.get(e) > 0)
+ {
+ assert !sourceFound;
+ sourceFound = true;
+ if(e.to().isEndpointVertex())
+ rangeFetchMapMap.put(((EndpointVertex)e.to()).getEndpoint(), ((RangeVertex)vertex).getRange());
+ else if(e.from().isEndpointVertex())
+ rangeFetchMapMap.put(((EndpointVertex)e.from()).getEndpoint(), ((RangeVertex)vertex).getRange());
+ }
+ }
+
+ assert sourceFound;
+
+ }
+ }
+
+ return rangeFetchMapMap;
+ }
+
+ /**
+ * This will increase the capacity from endpoint vertices to destination by incrementalCapacity
+ * @param graph The graph to work on
+ * @param incrementalCapacity Amount by which to increment capacity
+ */
+ private void incrementCapacity(MutableCapacityGraph<Vertex, Integer> graph, int incrementalCapacity)
+ {
+ for (Vertex vertex : graph.getVertices())
+ {
+ if (vertex.isEndpointVertex())
+ {
+ graph.addEdge(vertex, destinationVertex, incrementalCapacity);
+ }
+ }
+ }
+
+ /**
+ * Add source and destination vertices. Add edges of capacity 1 b/w source and range vertices.
+ * Also add edges b/w endpoint vertices and destination vertex with capacity of 'destinationCapacity'
+ * @param graph Graph to work on
+ * @param destinationCapacity The capacity for edges b/w endpoint vertices and destination
+ */
+ private void addSourceAndDestination(MutableCapacityGraph<Vertex, Integer> graph, int destinationCapacity)
+ {
+ graph.insertVertex(sourceVertex);
+ graph.insertVertex(destinationVertex);
+ for (Vertex vertex : graph.getVertices())
+ {
+ if (vertex.isRangeVertex())
+ {
+ graph.addEdge(sourceVertex, vertex, 1);
+ }
+ else if (vertex.isEndpointVertex())
+ {
+ graph.addEdge(vertex, destinationVertex, destinationCapacity);
+ }
+ }
+ }
+
+ /**
+ * Find the initial capacity which we want to use b/w machine vertices and destination to keep things optimal
+ * @param graph Graph to work on
+ * @return The initial capacity
+ */
+ private int getDestinationLinkCapacity(MutableCapacityGraph<Vertex, Integer> graph)
+ {
+ //Find total nodes which are endpoints and ranges
+ double endpointVertices = 0;
+ double rangeVertices = 0;
+ for (Vertex vertex : graph.getVertices())
+ {
+ if (vertex.isEndpointVertex())
+ {
+ endpointVertices++;
+ }
+ else if (vertex.isRangeVertex())
+ {
+ rangeVertices++;
+ }
+ }
+
+ return (int) Math.ceil(rangeVertices / endpointVertices);
+ }
+
+ /**
+ * Generate a graph with all ranges and endpoints as vertices. It will create edges b/w a range and its filtered source endpoints
+ * It will try to use sources from local DC if possible
+ * @return The generated graph
+ */
+ private MutableCapacityGraph<Vertex, Integer> getGraph()
+ {
+ MutableCapacityGraph<Vertex, Integer> capacityGraph = MutableCapacityGraph.create();
+
+ //Connect all ranges with all source endpoints
+ for (Range<Token> range : rangesWithSources.keySet())
+ {
+ if (trivialRanges.contains(range))
+ {
+ logger.debug("Not optimising trivial range {} for keyspace {}", range, keyspace);
+ continue;
+ }
+
+ final RangeVertex rangeVertex = new RangeVertex(range);
+
+ //Try to only add source endpoints from same DC
+ boolean sourceFound = addEndpoints(capacityGraph, rangeVertex, true);
+
+ if (!sourceFound)
+ {
+ logger.info("Using other DC endpoints for streaming for range: {} and keyspace {}", range, keyspace);
+ sourceFound = addEndpoints(capacityGraph, rangeVertex, false);
+ }
+
+ if (!sourceFound)
+ throw new IllegalStateException("Unable to find sufficient sources for streaming range " + range + " in keyspace " + keyspace);
+
+ }
+
+ return capacityGraph;
+ }
+
+ /**
+ * Create edges with infinite capacity b/w range vertex and all its source endpoints which clear the filters
+ * @param capacityGraph The Capacity graph on which changes are made
+ * @param rangeVertex The range for which we need to add all its source endpoints
+ * @param localDCCheck Should add source endpoints from local DC only
+ * @return If we were able to add atleast one source for this range after applying filters to endpoints
+ */
+ private boolean addEndpoints(MutableCapacityGraph<Vertex, Integer> capacityGraph, RangeVertex rangeVertex, boolean localDCCheck)
+ {
+ boolean sourceFound = false;
+ Replicas.temporaryAssertFull(rangesWithSources.get(rangeVertex.getRange()));
+ for (Replica replica : rangesWithSources.get(rangeVertex.getRange()))
+ {
+ if (passFilters(replica, localDCCheck))
+ {
+ sourceFound = true;
+ // if we pass filters, it means that we don't filter away localhost and we can count it as a source:
+ if (replica.isSelf())
+ continue; // but don't add localhost to the graph to avoid streaming locally
+ final Vertex endpointVertex = new EndpointVertex(replica.endpoint());
+ capacityGraph.insertVertex(rangeVertex);
+ capacityGraph.insertVertex(endpointVertex);
+ capacityGraph.addEdge(rangeVertex, endpointVertex, Integer.MAX_VALUE);
+ }
+ }
+ return sourceFound;
+ }
+
+ private boolean isInLocalDC(Replica replica)
+ {
+ return DatabaseDescriptor.getLocalDataCenter().equals(DatabaseDescriptor.getEndpointSnitch().getDatacenter(replica));
+ }
+
+ /**
+ *
+ * @param replica Replica to check
+ * @param localDCCheck Allow endpoints with local DC
+ * @return True if filters pass this endpoint
+ */
+ private boolean passFilters(final Replica replica, boolean localDCCheck)
+ {
+ return sourceFilters.apply(replica) && (!localDCCheck || isInLocalDC(replica));
+ }
+
+ private static abstract class Vertex
+ {
+ public enum VERTEX_TYPE
+ {
+ ENDPOINT, RANGE, SOURCE, DESTINATION
+ }
+
+ public abstract VERTEX_TYPE getVertexType();
+
+ public boolean isEndpointVertex()
+ {
+ return getVertexType() == VERTEX_TYPE.ENDPOINT;
+ }
+
+ public boolean isRangeVertex()
+ {
+ return getVertexType() == VERTEX_TYPE.RANGE;
+ }
+ }
+
+ /*
+ This Vertex will contain the endpoints.
+ */
+ private static class EndpointVertex extends Vertex
+ {
+ private final InetAddressAndPort endpoint;
+
+ public EndpointVertex(InetAddressAndPort endpoint)
+ {
+ assert endpoint != null;
+ this.endpoint = endpoint;
+ }
+
+ public InetAddressAndPort getEndpoint()
+ {
+ return endpoint;
+ }
+
+
+ @Override
+ public VERTEX_TYPE getVertexType()
+ {
+ return VERTEX_TYPE.ENDPOINT;
+ }
+
+ @Override
+ public boolean equals(Object o)
+ {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ EndpointVertex that = (EndpointVertex) o;
+
+ return endpoint.equals(that.endpoint);
+
+ }
+
+ @Override
+ public int hashCode()
+ {
+ return endpoint.hashCode();
+ }
+ }
+
+ /*
+ This Vertex will contain the Range
+ */
+ private static class RangeVertex extends Vertex
+ {
+ private final Range<Token> range;
+
+ public RangeVertex(Range<Token> range)
+ {
+ assert range != null;
+ this.range = range;
+ }
+
+ public Range<Token> getRange()
+ {
+ return range;
+ }
+
+ @Override
+ public VERTEX_TYPE getVertexType()
+ {
+ return VERTEX_TYPE.RANGE;
+ }
+
+ @Override
+ public boolean equals(Object o)
+ {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ RangeVertex that = (RangeVertex) o;
+
+ return range.equals(that.range);
+
+ }
+
+ @Override
+ public int hashCode()
+ {
+ return range.hashCode();
+ }
+ }
+
+ /*
+ This denotes the source and destination Vertex we need for the flow graph
+ */
+ private static class OuterVertex extends Vertex
+ {
+ private final boolean source;
+
+ private OuterVertex(boolean source)
+ {
+ this.source = source;
+ }
+
+ public static Vertex getSourceVertex()
+ {
+ return new OuterVertex(true);
+ }
+
+ public static Vertex getDestinationVertex()
+ {
+ return new OuterVertex(false);
+ }
+
+ @Override
+ public VERTEX_TYPE getVertexType()
+ {
+ return source? VERTEX_TYPE.SOURCE : VERTEX_TYPE.DESTINATION;
+ }
+
+ @Override
+ public boolean equals(Object o)
+ {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ OuterVertex that = (OuterVertex) o;
+
+ return source == that.source;
+
+ }
+
+ @Override
+ public int hashCode()
+ {
+ return (source ? 1 : 0);
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/dht/RangeStreamer.java b/src/java/org/apache/cassandra/dht/RangeStreamer.java
index a3cc996..75ccb4b 100644
--- a/src/java/org/apache/cassandra/dht/RangeStreamer.java
+++ b/src/java/org/apache/cassandra/dht/RangeStreamer.java
@@ -17,66 +17,136 @@
*/
package org.apache.cassandra.dht;
-import java.net.InetAddress;
import java.util.*;
+import java.util.function.BiFunction;
+import java.util.function.Function;
+import java.util.stream.Collectors;
import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.ArrayListMultimap;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicate;
import com.google.common.collect.HashMultimap;
+import com.google.common.collect.Iterables;
import com.google.common.collect.Multimap;
-import com.google.common.collect.Sets;
+
+import org.apache.cassandra.db.SystemKeyspace;
+import org.apache.cassandra.gms.FailureDetector;
+import org.apache.cassandra.locator.Endpoints;
+import org.apache.cassandra.locator.EndpointsByReplica;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.LocalStrategy;
+
+import org.apache.cassandra.locator.EndpointsByRange;
+import org.apache.cassandra.locator.EndpointsForRange;
+import org.apache.cassandra.locator.RangesAtEndpoint;
+import org.apache.cassandra.locator.ReplicaCollection.Builder.Conflict;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.db.Keyspace;
-import org.apache.cassandra.db.SystemKeyspace;
-import org.apache.cassandra.gms.EndpointState;
import org.apache.cassandra.gms.Gossiper;
import org.apache.cassandra.gms.IFailureDetector;
import org.apache.cassandra.locator.AbstractReplicationStrategy;
import org.apache.cassandra.locator.IEndpointSnitch;
+import org.apache.cassandra.locator.Replica;
+import org.apache.cassandra.locator.ReplicaCollection;
+import org.apache.cassandra.locator.Replicas;
import org.apache.cassandra.locator.TokenMetadata;
-import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.streaming.PreviewKind;
import org.apache.cassandra.streaming.StreamPlan;
import org.apache.cassandra.streaming.StreamResultFuture;
+import org.apache.cassandra.streaming.StreamOperation;
import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.Pair;
+
+import static com.google.common.base.Predicates.and;
+import static com.google.common.base.Predicates.not;
+import static com.google.common.collect.Iterables.all;
+import static com.google.common.collect.Iterables.any;
+import static org.apache.cassandra.locator.Replica.fullReplica;
/**
- * Assists in streaming ranges to a node.
+ * Assists in streaming ranges to this node.
*/
public class RangeStreamer
{
private static final Logger logger = LoggerFactory.getLogger(RangeStreamer.class);
+ public static Predicate<Replica> ALIVE_PREDICATE = replica ->
+ (!Gossiper.instance.isEnabled() ||
+ (Gossiper.instance.getEndpointStateForEndpoint(replica.endpoint()) == null ||
+ Gossiper.instance.getEndpointStateForEndpoint(replica.endpoint()).isAlive())) &&
+ FailureDetector.instance.isAlive(replica.endpoint());
+
/* bootstrap tokens. can be null if replacing the node. */
private final Collection<Token> tokens;
/* current token ring */
private final TokenMetadata metadata;
/* address of this node */
- private final InetAddress address;
+ private final InetAddressAndPort address;
/* streaming description */
private final String description;
- private final Multimap<String, Map.Entry<InetAddress, Collection<Range<Token>>>> toFetch = HashMultimap.create();
- private final Set<ISourceFilter> sourceFilters = new HashSet<>();
+ private final Map<String, Multimap<InetAddressAndPort, FetchReplica>> toFetch = new HashMap<>();
+ private final List<SourceFilter> sourceFilters = new ArrayList<>();
private final StreamPlan streamPlan;
private final boolean useStrictConsistency;
private final IEndpointSnitch snitch;
private final StreamStateStore stateStore;
- /**
- * A filter applied to sources to stream from when constructing a fetch map.
- */
- public static interface ISourceFilter
+ public static class FetchReplica
{
- public boolean shouldInclude(InetAddress endpoint);
+ public final Replica local;
+ // Source replica
+ public final Replica remote;
+
+ public FetchReplica(Replica local, Replica remote)
+ {
+ Preconditions.checkNotNull(local);
+ Preconditions.checkNotNull(remote);
+ assert local.isSelf() && !remote.isSelf();
+ this.local = local;
+ this.remote = remote;
+ }
+
+ public String toString()
+ {
+ return "FetchReplica{" +
+ "local=" + local +
+ ", remote=" + remote +
+ '}';
+ }
+
+ public boolean equals(Object o)
+ {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ FetchReplica that = (FetchReplica) o;
+
+ if (!local.equals(that.local)) return false;
+ return remote.equals(that.remote);
+ }
+
+ public int hashCode()
+ {
+ int result = local.hashCode();
+ result = 31 * result + remote.hashCode();
+ return result;
+ }
+ }
+
+ public interface SourceFilter extends Predicate<Replica>
+ {
+ public boolean apply(Replica replica);
+ public String message(Replica replica);
}
/**
* Source filter which excludes any endpoints that are not alive according to a
* failure detector.
*/
- public static class FailureDetectorSourceFilter implements ISourceFilter
+ public static class FailureDetectorSourceFilter implements SourceFilter
{
private final IFailureDetector fd;
@@ -85,16 +155,23 @@
this.fd = fd;
}
- public boolean shouldInclude(InetAddress endpoint)
+ @Override
+ public boolean apply(Replica replica)
{
- return fd.isAlive(endpoint);
+ return fd.isAlive(replica.endpoint());
+ }
+
+ @Override
+ public String message(Replica replica)
+ {
+ return "Filtered " + replica + " out because it was down";
}
}
/**
* Source filter which excludes any endpoints that are not in a specific data center.
*/
- public static class SingleDatacenterFilter implements ISourceFilter
+ public static class SingleDatacenterFilter implements SourceFilter
{
private final String sourceDc;
private final IEndpointSnitch snitch;
@@ -105,290 +182,518 @@
this.snitch = snitch;
}
- public boolean shouldInclude(InetAddress endpoint)
+ @Override
+ public boolean apply(Replica replica)
{
- return snitch.getDatacenter(endpoint).equals(sourceDc);
+ return snitch.getDatacenter(replica).equals(sourceDc);
+ }
+
+ @Override
+ public String message(Replica replica)
+ {
+ return "Filtered " + replica + " out because it does not belong to " + sourceDc + " datacenter";
}
}
/**
* Source filter which excludes the current node from source calculations
*/
- public static class ExcludeLocalNodeFilter implements ISourceFilter
+ public static class ExcludeLocalNodeFilter implements SourceFilter
{
- public boolean shouldInclude(InetAddress endpoint)
+ @Override
+ public boolean apply(Replica replica)
{
- return !FBUtilities.getBroadcastAddress().equals(endpoint);
+ return !replica.isSelf();
+ }
+
+ @Override
+ public String message(Replica replica)
+ {
+ return "Filtered " + replica + " out because it is local";
}
}
/**
* Source filter which only includes endpoints contained within a provided set.
*/
- public static class AllowedSourcesFilter implements ISourceFilter
+ public static class AllowedSourcesFilter implements SourceFilter
{
- private final Set<InetAddress> allowedSources;
+ private final Set<InetAddressAndPort> allowedSources;
- public AllowedSourcesFilter(Set<InetAddress> allowedSources)
+ public AllowedSourcesFilter(Set<InetAddressAndPort> allowedSources)
{
this.allowedSources = allowedSources;
}
- public boolean shouldInclude(InetAddress endpoint)
+ public boolean apply(Replica replica)
{
- return allowedSources.contains(endpoint);
+ return allowedSources.contains(replica.endpoint());
+ }
+
+ @Override
+ public String message(Replica replica)
+ {
+ return "Filtered " + replica + " out because it was not in the allowed set: " + allowedSources;
}
}
public RangeStreamer(TokenMetadata metadata,
Collection<Token> tokens,
- InetAddress address,
- String description,
+ InetAddressAndPort address,
+ StreamOperation streamOperation,
boolean useStrictConsistency,
IEndpointSnitch snitch,
StreamStateStore stateStore,
- boolean connectSequentially)
+ boolean connectSequentially,
+ int connectionsPerHost)
{
+ this(metadata, tokens, address, streamOperation, useStrictConsistency, snitch, stateStore,
+ FailureDetector.instance, connectSequentially, connectionsPerHost);
+ }
+
+ RangeStreamer(TokenMetadata metadata,
+ Collection<Token> tokens,
+ InetAddressAndPort address,
+ StreamOperation streamOperation,
+ boolean useStrictConsistency,
+ IEndpointSnitch snitch,
+ StreamStateStore stateStore,
+ IFailureDetector failureDetector,
+ boolean connectSequentially,
+ int connectionsPerHost)
+ {
+ Preconditions.checkArgument(streamOperation == StreamOperation.BOOTSTRAP || streamOperation == StreamOperation.REBUILD, streamOperation);
this.metadata = metadata;
this.tokens = tokens;
this.address = address;
- this.description = description;
- this.streamPlan = new StreamPlan(description, true, connectSequentially);
+ this.description = streamOperation.getDescription();
+ this.streamPlan = new StreamPlan(streamOperation, connectionsPerHost, connectSequentially, null, PreviewKind.NONE);
this.useStrictConsistency = useStrictConsistency;
this.snitch = snitch;
this.stateStore = stateStore;
streamPlan.listeners(this.stateStore);
+
+ // We're _always_ filtering out a local node and down sources
+ addSourceFilter(new RangeStreamer.FailureDetectorSourceFilter(failureDetector));
+ addSourceFilter(new RangeStreamer.ExcludeLocalNodeFilter());
}
- public void addSourceFilter(ISourceFilter filter)
+ public void addSourceFilter(SourceFilter filter)
{
sourceFilters.add(filter);
}
+ // Creates error message from source filters
+ private static String buildErrorMessage(Collection<SourceFilter> sourceFilters, ReplicaCollection<?> replicas)
+ {
+ StringBuilder failureMessage = new StringBuilder();
+ for (Replica r : replicas)
+ {
+ for (SourceFilter filter : sourceFilters)
+ {
+ if (!filter.apply(r))
+ {
+ failureMessage.append(filter.message(r));
+ break;
+ }
+ }
+ }
+ return failureMessage.toString();
+ }
/**
* Add ranges to be streamed for given keyspace.
*
* @param keyspaceName keyspace name
- * @param ranges ranges to be streamed
+ * @param replicas ranges to be streamed
*/
- public void addRanges(String keyspaceName, Collection<Range<Token>> ranges)
+ public void addRanges(String keyspaceName, ReplicaCollection<?> replicas)
{
- Multimap<Range<Token>, InetAddress> rangesForKeyspace = useStrictSourcesForRanges(keyspaceName)
- ? getAllRangesWithStrictSourcesFor(keyspaceName, ranges) : getAllRangesWithSourcesFor(keyspaceName, ranges);
+ Keyspace keyspace = Keyspace.open(keyspaceName);
+ AbstractReplicationStrategy strat = keyspace.getReplicationStrategy();
+ if(strat instanceof LocalStrategy)
+ {
+ logger.info("Not adding ranges for Local Strategy keyspace={}", keyspaceName);
+ return;
+ }
+
+ boolean useStrictSource = useStrictSourcesForRanges(strat);
+ EndpointsByReplica fetchMap = calculateRangesToFetchWithPreferredEndpoints(replicas, keyspace, useStrictSource);
+
+ for (Map.Entry<Replica, Replica> entry : fetchMap.flattenEntries())
+ logger.info("{}: range {} exists on {} for keyspace {}", description, entry.getKey(), entry.getValue(), keyspaceName);
+
+ Multimap<InetAddressAndPort, FetchReplica> workMap;
+ //Only use the optimized strategy if we don't care about strict sources, have a replication factor > 1, and no
+ //transient replicas.
+ if (useStrictSource || strat == null || strat.getReplicationFactor().allReplicas == 1 || strat.getReplicationFactor().hasTransientReplicas())
+ {
+ workMap = convertPreferredEndpointsToWorkMap(fetchMap);
+ }
+ else
+ {
+ workMap = getOptimizedWorkMap(fetchMap, sourceFilters, keyspaceName);
+ }
+
+ if (toFetch.put(keyspaceName, workMap) != null)
+ throw new IllegalArgumentException("Keyspace is already added to fetch map");
if (logger.isTraceEnabled())
{
- for (Map.Entry<Range<Token>, InetAddress> entry : rangesForKeyspace.entries())
- logger.trace("{}: range {} exists on {}", description, entry.getKey(), entry.getValue());
- }
-
- for (Map.Entry<InetAddress, Collection<Range<Token>>> entry : getRangeFetchMap(rangesForKeyspace, sourceFilters, keyspaceName, useStrictConsistency).asMap().entrySet())
- {
- if (logger.isTraceEnabled())
+ for (Map.Entry<InetAddressAndPort, Collection<FetchReplica>> entry : workMap.asMap().entrySet())
{
- for (Range<Token> r : entry.getValue())
- logger.trace("{}: range {} from source {} for keyspace {}", description, r, entry.getKey(), keyspaceName);
+ for (FetchReplica r : entry.getValue())
+ logger.trace("{}: range source {} local range {} for keyspace {}", description, r.remote, r.local, keyspaceName);
}
- toFetch.put(keyspaceName, entry);
}
}
/**
- * @param keyspaceName keyspace name to check
+ * @param strat AbstractReplicationStrategy of keyspace to check
* @return true when the node is bootstrapping, useStrictConsistency is true and # of nodes in the cluster is more than # of replica
*/
- private boolean useStrictSourcesForRanges(String keyspaceName)
+ private boolean useStrictSourcesForRanges(AbstractReplicationStrategy strat)
{
- AbstractReplicationStrategy strat = Keyspace.open(keyspaceName).getReplicationStrategy();
return useStrictConsistency
&& tokens != null
- && metadata.getAllEndpoints().size() != strat.getReplicationFactor();
+ && metadata.getSizeOfAllEndpoints() != strat.getReplicationFactor().allReplicas;
}
/**
- * Get a map of all ranges and their respective sources that are candidates for streaming the given ranges
- * to us. For each range, the list of sources is sorted by proximity relative to the given destAddress.
- *
- * @throws java.lang.IllegalStateException when there is no source to get data streamed
+ * Wrapper method to assemble the arguments for invoking the implementation with RangeStreamer's parameters
*/
- private Multimap<Range<Token>, InetAddress> getAllRangesWithSourcesFor(String keyspaceName, Collection<Range<Token>> desiredRanges)
+ private EndpointsByReplica calculateRangesToFetchWithPreferredEndpoints(ReplicaCollection<?> fetchRanges, Keyspace keyspace, boolean useStrictConsistency)
{
- AbstractReplicationStrategy strat = Keyspace.open(keyspaceName).getReplicationStrategy();
- Multimap<Range<Token>, InetAddress> rangeAddresses = strat.getRangeAddresses(metadata.cloneOnlyTokenMap());
+ AbstractReplicationStrategy strat = keyspace.getReplicationStrategy();
- Multimap<Range<Token>, InetAddress> rangeSources = ArrayListMultimap.create();
- for (Range<Token> desiredRange : desiredRanges)
+ TokenMetadata tmd = metadata.cloneOnlyTokenMap();
+
+ TokenMetadata tmdAfter = null;
+
+ if (tokens != null)
{
- for (Range<Token> range : rangeAddresses.keySet())
- {
- if (range.contains(desiredRange))
- {
- List<InetAddress> preferred = snitch.getSortedListByProximity(address, rangeAddresses.get(range));
- rangeSources.putAll(desiredRange, preferred);
- break;
- }
- }
-
- if (!rangeSources.keySet().contains(desiredRange))
- throw new IllegalStateException("No sources found for " + desiredRange);
+ // Pending ranges
+ tmdAfter = tmd.cloneOnlyTokenMap();
+ tmdAfter.updateNormalTokens(tokens, address);
+ }
+ else if (useStrictConsistency)
+ {
+ throw new IllegalArgumentException("Can't ask for strict consistency and not supply tokens");
}
- return rangeSources;
+ return calculateRangesToFetchWithPreferredEndpoints(snitch::sortedByProximity,
+ strat,
+ fetchRanges,
+ useStrictConsistency,
+ tmd,
+ tmdAfter,
+ keyspace.getName(),
+ sourceFilters);
+
}
/**
* Get a map of all ranges and the source that will be cleaned up once this bootstrapped node is added for the given ranges.
* For each range, the list should only contain a single source. This allows us to consistently migrate data without violating
* consistency.
- *
- * @throws java.lang.IllegalStateException when there is no source to get data streamed, or more than 1 source found.
+ **/
+ public static EndpointsByReplica
+ calculateRangesToFetchWithPreferredEndpoints(BiFunction<InetAddressAndPort, EndpointsForRange, EndpointsForRange> snitchGetSortedListByProximity,
+ AbstractReplicationStrategy strat,
+ ReplicaCollection<?> fetchRanges,
+ boolean useStrictConsistency,
+ TokenMetadata tmdBefore,
+ TokenMetadata tmdAfter,
+ String keyspace,
+ Collection<SourceFilter> sourceFilters)
+ {
+ EndpointsByRange rangeAddresses = strat.getRangeAddresses(tmdBefore);
+
+ InetAddressAndPort localAddress = FBUtilities.getBroadcastAddressAndPort();
+ logger.debug ("Keyspace: {}", keyspace);
+ logger.debug("To fetch RN: {}", fetchRanges);
+ logger.debug("Fetch ranges: {}", rangeAddresses);
+
+ Predicate<Replica> testSourceFilters = and(sourceFilters);
+ Function<EndpointsForRange, EndpointsForRange> sorted =
+ endpoints -> snitchGetSortedListByProximity.apply(localAddress, endpoints);
+
+ //This list of replicas is just candidates. With strict consistency it's going to be a narrow list.
+ EndpointsByReplica.Builder rangesToFetchWithPreferredEndpoints = new EndpointsByReplica.Builder();
+ for (Replica toFetch : fetchRanges)
+ {
+ //Replica that is sufficient to provide the data we need
+ //With strict consistency and transient replication we may end up with multiple types
+ //so this isn't used with strict consistency
+ Predicate<Replica> isSufficient = r -> toFetch.isTransient() || r.isFull();
+
+ logger.debug("To fetch {}", toFetch);
+ for (Range<Token> range : rangeAddresses.keySet())
+ {
+ if (!range.contains(toFetch.range()))
+ continue;
+
+ final EndpointsForRange oldEndpoints = sorted.apply(rangeAddresses.get(range));
+
+ //Ultimately we populate this with whatever is going to be fetched from to satisfy toFetch
+ //It could be multiple endpoints and we must fetch from all of them if they are there
+ //With transient replication and strict consistency this is to get the full data from a full replica and
+ //transient data from the transient replica losing data
+ EndpointsForRange sources;
+ if (useStrictConsistency)
+ {
+ EndpointsForRange strictEndpoints;
+ //Due to CASSANDRA-5953 we can have a higher RF then we have endpoints.
+ //So we need to be careful to only be strict when endpoints == RF
+ if (oldEndpoints.size() == strat.getReplicationFactor().allReplicas)
+ {
+ //Start with two sets of who replicates the range before and who replicates it after
+ EndpointsForRange newEndpoints = strat.calculateNaturalReplicas(toFetch.range().right, tmdAfter);
+ logger.debug("Old endpoints {}", oldEndpoints);
+ logger.debug("New endpoints {}", newEndpoints);
+
+ // Remove new endpoints from old endpoints based on address
+ strictEndpoints = oldEndpoints.without(newEndpoints.endpoints());
+
+ if (strictEndpoints.size() > 1)
+ throw new AssertionError("Expected <= 1 endpoint but found " + strictEndpoints);
+
+ //We have to check the source filters here to see if they will remove any replicas
+ //required for strict consistency
+ if (!all(strictEndpoints, testSourceFilters))
+ throw new IllegalStateException("Necessary replicas for strict consistency were removed by source filters: " + buildErrorMessage(sourceFilters, strictEndpoints));
+
+ //If we are transitioning from transient to full and and the set of replicas for the range is not changing
+ //we might end up with no endpoints to fetch from by address. In that case we can pick any full replica safely
+ //since we are already a transient replica and the existing replica remains.
+ //The old behavior where we might be asked to fetch ranges we don't need shouldn't occur anymore.
+ //So it's an error if we don't find what we need.
+ if (strictEndpoints.isEmpty() && toFetch.isTransient())
+ throw new AssertionError("If there are no endpoints to fetch from then we must be transitioning from transient to full for range " + toFetch);
+
+ if (!any(strictEndpoints, isSufficient))
+ {
+ // need an additional replica; include all our filters, to ensure we include a matching node
+ Optional<Replica> fullReplica = Iterables.<Replica>tryFind(oldEndpoints, and(isSufficient, testSourceFilters)).toJavaUtil();
+ if (fullReplica.isPresent())
+ strictEndpoints = Endpoints.concat(strictEndpoints, EndpointsForRange.of(fullReplica.get()));
+ else
+ throw new IllegalStateException("Couldn't find any matching sufficient replica out of " + buildErrorMessage(sourceFilters, oldEndpoints));
+ }
+ }
+ else
+ {
+ strictEndpoints = sorted.apply(oldEndpoints.filter(and(isSufficient, testSourceFilters)));
+ }
+
+ sources = strictEndpoints;
+ }
+ else
+ {
+ //Without strict consistency we have given up on correctness so no point in fetching from
+ //a random full + transient replica since it's also likely to lose data
+ //Also apply testSourceFilters that were given to us so we can safely select a single source
+ sources = sorted.apply(oldEndpoints.filter(and(isSufficient, testSourceFilters)));
+ //Limit it to just the first possible source, we don't need more than one and downstream
+ //will fetch from every source we supply
+ sources = sources.size() > 0 ? sources.subList(0, 1) : sources;
+ }
+
+ // storing range and preferred endpoint set
+ rangesToFetchWithPreferredEndpoints.putAll(toFetch, sources, Conflict.NONE);
+ logger.debug("Endpoints to fetch for {} are {}", toFetch, sources);
+ }
+
+ EndpointsForRange addressList = rangesToFetchWithPreferredEndpoints.getIfPresent(toFetch);
+ if (addressList == null)
+ throw new IllegalStateException("Failed to find endpoints to fetch " + toFetch);
+
+ /*
+ * When we move forwards (shrink our bucket) we are the one losing a range and no one else loses
+ * from that action (we also don't gain). When we move backwards there are two people losing a range. One is a full replica
+ * and the other is a transient replica. So we must need fetch from two places in that case for the full range we gain.
+ * For a transient range we only need to fetch from one.
+ */
+ if (useStrictConsistency && addressList.size() > 1 && (addressList.filter(Replica::isFull).size() > 1 || addressList.filter(Replica::isTransient).size() > 1))
+ throw new IllegalStateException(String.format("Multiple strict sources found for %s, sources: %s", toFetch, addressList));
+
+ //We must have enough stuff to fetch from
+ if (!any(addressList, isSufficient))
+ {
+ if (strat.getReplicationFactor().allReplicas == 1)
+ {
+ if (useStrictConsistency)
+ {
+ logger.warn("A node required to move the data consistently is down");
+ throw new IllegalStateException("Unable to find sufficient sources for streaming range " + toFetch + " in keyspace " + keyspace + " with RF=1. " +
+ "Ensure this keyspace contains replicas in the source datacenter.");
+ }
+ else
+ logger.warn("Unable to find sufficient sources for streaming range {} in keyspace {} with RF=1. " +
+ "Keyspace might be missing data.", toFetch, keyspace);
+ }
+ else
+ {
+ if (useStrictConsistency)
+ logger.warn("A node required to move the data consistently is down");
+ throw new IllegalStateException("Unable to find sufficient sources for streaming range " + toFetch + " in keyspace " + keyspace);
+ }
+ }
+ }
+ return rangesToFetchWithPreferredEndpoints.build();
+ }
+
+ /**
+ * The preferred endpoint list is the wrong format because it is keyed by Replica (this node) rather than the source
+ * endpoint we will fetch from which streaming wants.
*/
- private Multimap<Range<Token>, InetAddress> getAllRangesWithStrictSourcesFor(String keyspace, Collection<Range<Token>> desiredRanges)
+ public static Multimap<InetAddressAndPort, FetchReplica> convertPreferredEndpointsToWorkMap(EndpointsByReplica preferredEndpoints)
{
- assert tokens != null;
- AbstractReplicationStrategy strat = Keyspace.open(keyspace).getReplicationStrategy();
-
- // Active ranges
- TokenMetadata metadataClone = metadata.cloneOnlyTokenMap();
- Multimap<Range<Token>, InetAddress> addressRanges = strat.getRangeAddresses(metadataClone);
-
- // Pending ranges
- metadataClone.updateNormalTokens(tokens, address);
- Multimap<Range<Token>, InetAddress> pendingRangeAddresses = strat.getRangeAddresses(metadataClone);
-
- // Collects the source that will have its range moved to the new node
- Multimap<Range<Token>, InetAddress> rangeSources = ArrayListMultimap.create();
-
- for (Range<Token> desiredRange : desiredRanges)
+ Multimap<InetAddressAndPort, FetchReplica> workMap = HashMultimap.create();
+ for (Map.Entry<Replica, EndpointsForRange> e : preferredEndpoints.entrySet())
{
- for (Map.Entry<Range<Token>, Collection<InetAddress>> preEntry : addressRanges.asMap().entrySet())
+ for (Replica source : e.getValue())
{
- if (preEntry.getKey().contains(desiredRange))
- {
- Set<InetAddress> oldEndpoints = Sets.newHashSet(preEntry.getValue());
- Set<InetAddress> newEndpoints = Sets.newHashSet(pendingRangeAddresses.get(desiredRange));
-
- // Due to CASSANDRA-5953 we can have a higher RF then we have endpoints.
- // So we need to be careful to only be strict when endpoints == RF
- if (oldEndpoints.size() == strat.getReplicationFactor())
- {
- oldEndpoints.removeAll(newEndpoints);
- assert oldEndpoints.size() == 1 : "Expected 1 endpoint but found " + oldEndpoints.size();
- }
-
- rangeSources.put(desiredRange, oldEndpoints.iterator().next());
- }
+ assert (e.getKey()).isSelf();
+ assert !source.isSelf();
+ workMap.put(source.endpoint(), new FetchReplica(e.getKey(), source));
}
-
- // Validate
- Collection<InetAddress> addressList = rangeSources.get(desiredRange);
- if (addressList == null || addressList.isEmpty())
- throw new IllegalStateException("No sources found for " + desiredRange);
-
- if (addressList.size() > 1)
- throw new IllegalStateException("Multiple endpoints found for " + desiredRange);
-
- InetAddress sourceIp = addressList.iterator().next();
- EndpointState sourceState = Gossiper.instance.getEndpointStateForEndpoint(sourceIp);
- if (Gossiper.instance.isEnabled() && (sourceState == null || !sourceState.isAlive()))
- throw new RuntimeException("A node required to move the data consistently is down (" + sourceIp + "). " +
- "If you wish to move the data from a potentially inconsistent replica, restart the node with -Dcassandra.consistent.rangemovement=false");
}
-
- return rangeSources;
+ logger.debug("Work map {}", workMap);
+ return workMap;
}
/**
- * @param rangesWithSources The ranges we want to fetch (key) and their potential sources (value)
- * @param sourceFilters A (possibly empty) collection of source filters to apply. In addition to any filters given
- * here, we always exclude ourselves.
- * @param keyspace keyspace name
- * @return Map of source endpoint to collection of ranges
+ * Optimized version that also outputs the final work map
*/
- private static Multimap<InetAddress, Range<Token>> getRangeFetchMap(Multimap<Range<Token>, InetAddress> rangesWithSources,
- Collection<ISourceFilter> sourceFilters, String keyspace,
- boolean useStrictConsistency)
+ private static Multimap<InetAddressAndPort, FetchReplica> getOptimizedWorkMap(EndpointsByReplica rangesWithSources,
+ Collection<SourceFilter> sourceFilters,
+ String keyspace)
{
- Multimap<InetAddress, Range<Token>> rangeFetchMapMap = HashMultimap.create();
- for (Range<Token> range : rangesWithSources.keySet())
+ //For now we just aren't going to use the optimized range fetch map with transient replication to shrink
+ //the surface area to test and introduce bugs.
+ //In the future it's possible we could run it twice once for full ranges with only full replicas
+ //and once with transient ranges and all replicas. Then merge the result.
+ EndpointsByRange.Builder unwrapped = new EndpointsByRange.Builder();
+ for (Map.Entry<Replica, Replica> entry : rangesWithSources.flattenEntries())
{
- boolean foundSource = false;
-
- outer:
- for (InetAddress address : rangesWithSources.get(range))
- {
- for (ISourceFilter filter : sourceFilters)
- {
- if (!filter.shouldInclude(address))
- continue outer;
- }
-
- if (address.equals(FBUtilities.getBroadcastAddress()))
- {
- // If localhost is a source, we have found one, but we don't add it to the map to avoid streaming locally
- foundSource = true;
- continue;
- }
-
- rangeFetchMapMap.put(address, range);
- foundSource = true;
- break; // ensure we only stream from one other node for each range
- }
-
- if (!foundSource)
- {
- AbstractReplicationStrategy strat = Keyspace.open(keyspace).getReplicationStrategy();
- if (strat != null && strat.getReplicationFactor() == 1)
- {
- if (useStrictConsistency)
- throw new IllegalStateException("Unable to find sufficient sources for streaming range " + range + " in keyspace " + keyspace + " with RF=1. " +
- "Ensure this keyspace contains replicas in the source datacenter.");
- else
- logger.warn("Unable to find sufficient sources for streaming range {} in keyspace {} with RF=1. " +
- "Keyspace might be missing data.", range, keyspace);
- }
- else
- throw new IllegalStateException("Unable to find sufficient sources for streaming range " + range + " in keyspace " + keyspace);
- }
+ Replicas.temporaryAssertFull(entry.getValue());
+ unwrapped.put(entry.getKey().range(), entry.getValue());
}
- return rangeFetchMapMap;
+ EndpointsByRange unwrappedView = unwrapped.build();
+ RangeFetchMapCalculator calculator = new RangeFetchMapCalculator(unwrappedView, sourceFilters, keyspace);
+ Multimap<InetAddressAndPort, Range<Token>> rangeFetchMapMap = calculator.getRangeFetchMap();
+ logger.info("Output from RangeFetchMapCalculator for keyspace {}", keyspace);
+ validateRangeFetchMap(unwrappedView, rangeFetchMapMap, keyspace);
+
+ //Need to rewrap as Replicas
+ Multimap<InetAddressAndPort, FetchReplica> wrapped = HashMultimap.create();
+ for (Map.Entry<InetAddressAndPort, Range<Token>> entry : rangeFetchMapMap.entries())
+ {
+ Replica toFetch = null;
+ for (Replica r : rangesWithSources.keySet())
+ {
+ if (r.range().equals(entry.getValue()))
+ {
+ if (toFetch != null)
+ throw new AssertionError(String.format("There shouldn't be multiple replicas for range %s, replica %s and %s here", r.range(), r, toFetch));
+ toFetch = r;
+ }
+ }
+ if (toFetch == null)
+ throw new AssertionError("Shouldn't be possible for the Replica we fetch to be null here");
+ //Committing the cardinal sin of synthesizing a Replica, but it's ok because we assert earlier all of them
+ //are full and optimized range fetch map doesn't support transient replication yet.
+ wrapped.put(entry.getKey(), new FetchReplica(toFetch, fullReplica(entry.getKey(), entry.getValue())));
+ }
+
+ return wrapped;
}
- public static Multimap<InetAddress, Range<Token>> getWorkMap(Multimap<Range<Token>, InetAddress> rangesWithSourceTarget, String keyspace,
- IFailureDetector fd, boolean useStrictConsistency)
+ /**
+ * Verify that source returned for each range is correct
+ */
+ private static void validateRangeFetchMap(EndpointsByRange rangesWithSources, Multimap<InetAddressAndPort, Range<Token>> rangeFetchMapMap, String keyspace)
{
- return getRangeFetchMap(rangesWithSourceTarget, Collections.<ISourceFilter>singleton(new FailureDetectorSourceFilter(fd)), keyspace, useStrictConsistency);
+ for (Map.Entry<InetAddressAndPort, Range<Token>> entry : rangeFetchMapMap.entries())
+ {
+ if(entry.getKey().equals(FBUtilities.getBroadcastAddressAndPort()))
+ {
+ throw new IllegalStateException("Trying to stream locally. Range: " + entry.getValue()
+ + " in keyspace " + keyspace);
+ }
+
+ if (!rangesWithSources.get(entry.getValue()).endpoints().contains(entry.getKey()))
+ {
+ throw new IllegalStateException("Trying to stream from wrong endpoint. Range: " + entry.getValue()
+ + " in keyspace " + keyspace + " from endpoint: " + entry.getKey());
+ }
+
+ logger.info("Streaming range {} from endpoint {} for keyspace {}", entry.getValue(), entry.getKey(), keyspace);
+ }
}
// For testing purposes
@VisibleForTesting
- Multimap<String, Map.Entry<InetAddress, Collection<Range<Token>>>> toFetch()
+ Map<String, Multimap<InetAddressAndPort, FetchReplica>> toFetch()
{
return toFetch;
}
public StreamResultFuture fetchAsync()
{
- for (Map.Entry<String, Map.Entry<InetAddress, Collection<Range<Token>>>> entry : toFetch.entries())
- {
- String keyspace = entry.getKey();
- InetAddress source = entry.getValue().getKey();
- InetAddress preferred = SystemKeyspace.getPreferredIP(source);
- Collection<Range<Token>> ranges = entry.getValue().getValue();
+ toFetch.forEach((keyspace, sources) -> {
+ logger.debug("Keyspace {} Sources {}", keyspace, sources);
+ sources.asMap().forEach((source, fetchReplicas) -> {
- // filter out already streamed ranges
- Set<Range<Token>> availableRanges = stateStore.getAvailableRanges(keyspace, StorageService.instance.getTokenMetadata().partitioner);
- if (ranges.removeAll(availableRanges))
- {
- logger.info("Some ranges of {} are already available. Skipping streaming those ranges.", availableRanges);
- }
+ // filter out already streamed ranges
+ SystemKeyspace.AvailableRanges available = stateStore.getAvailableRanges(keyspace, metadata.partitioner);
- if (logger.isTraceEnabled())
- logger.trace("{}ing from {} ranges {}", description, source, StringUtils.join(ranges, ", "));
- /* Send messages to respective folks to stream data over to me */
- streamPlan.requestRanges(source, preferred, keyspace, ranges);
- }
+ Predicate<FetchReplica> isAvailable = fetch -> {
+ boolean isInFull = available.full.contains(fetch.local.range());
+ boolean isInTrans = available.trans.contains(fetch.local.range());
+
+ if (!isInFull && !isInTrans)
+ //Range is unavailable
+ return false;
+
+ if (fetch.local.isFull())
+ //For full, pick only replicas with matching transientness
+ return isInFull == fetch.remote.isFull();
+
+ // Any transient or full will do
+ return true;
+ };
+
+ List<FetchReplica> remaining = fetchReplicas.stream().filter(not(isAvailable)).collect(Collectors.toList());
+
+ if (remaining.size() < available.full.size() + available.trans.size())
+ {
+ List<FetchReplica> skipped = fetchReplicas.stream().filter(isAvailable).collect(Collectors.toList());
+ logger.info("Some ranges of {} are already available. Skipping streaming those ranges. Skipping {}. Fully available {} Transiently available {}",
+ fetchReplicas, skipped, available.full, available.trans);
+ }
+
+ if (logger.isTraceEnabled())
+ logger.trace("{}ing from {} ranges {}", description, source, StringUtils.join(remaining, ", "));
+
+ InetAddressAndPort self = FBUtilities.getBroadcastAddressAndPort();
+ RangesAtEndpoint full = remaining.stream()
+ .filter(pair -> pair.remote.isFull())
+ .map(pair -> pair.local)
+ .collect(RangesAtEndpoint.collector(self));
+ RangesAtEndpoint transientReplicas = remaining.stream()
+ .filter(pair -> pair.remote.isTransient())
+ .map(pair -> pair.local)
+ .collect(RangesAtEndpoint.collector(self));
+
+ logger.debug("Source and our replicas {}", fetchReplicas);
+ logger.debug("Source {} Keyspace {} streaming full {} transient {}", source, keyspace, full, transientReplicas);
+
+ /* Send messages to respective folks to stream data over to me */
+ streamPlan.requestRanges(source, keyspace, full, transientReplicas);
+ });
+ });
return streamPlan.execute();
}
diff --git a/src/java/org/apache/cassandra/dht/Splitter.java b/src/java/org/apache/cassandra/dht/Splitter.java
index 4433f97..8578448 100644
--- a/src/java/org/apache/cassandra/dht/Splitter.java
+++ b/src/java/org/apache/cassandra/dht/Splitter.java
@@ -18,10 +18,20 @@
package org.apache.cassandra.dht;
+import java.math.BigDecimal;
import java.math.BigInteger;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.Collections;
+import java.util.HashSet;
import java.util.List;
+import java.util.Objects;
+import java.util.Set;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Sets;
+
+import static java.util.stream.Collectors.toSet;
/**
* Partition splitter.
@@ -35,36 +45,104 @@
this.partitioner = partitioner;
}
+ @VisibleForTesting
protected abstract Token tokenForValue(BigInteger value);
+ @VisibleForTesting
protected abstract BigInteger valueForToken(Token token);
- public List<Token> splitOwnedRanges(int parts, List<Range<Token>> localRanges, boolean dontSplitRanges)
+ @VisibleForTesting
+ protected BigInteger tokensInRange(Range<Token> range)
{
- if (localRanges.isEmpty() || parts == 1)
+ //full range case
+ if (range.left.equals(range.right))
+ return tokensInRange(new Range(partitioner.getMinimumToken(), partitioner.getMaximumToken()));
+
+ BigInteger totalTokens = BigInteger.ZERO;
+ for (Range<Token> unwrapped : range.unwrap())
+ {
+ totalTokens = totalTokens.add(valueForToken(token(unwrapped.right)).subtract(valueForToken(unwrapped.left))).abs();
+ }
+ return totalTokens;
+ }
+
+ /**
+ * Computes the number of elapsed tokens from the range start until this token
+ * @return the number of tokens from the range start to the token
+ */
+ @VisibleForTesting
+ protected BigInteger elapsedTokens(Token token, Range<Token> range)
+ {
+ // No token elapsed since range does not contain token
+ if (!range.contains(token))
+ return BigInteger.ZERO;
+
+ BigInteger elapsedTokens = BigInteger.ZERO;
+ for (Range<Token> unwrapped : range.unwrap())
+ {
+ if (unwrapped.contains(token))
+ {
+ elapsedTokens = elapsedTokens.add(tokensInRange(new Range<>(unwrapped.left, token)));
+ }
+ else if (token.compareTo(unwrapped.left) < 0)
+ {
+ elapsedTokens = elapsedTokens.add(tokensInRange(unwrapped));
+ }
+ }
+ return elapsedTokens;
+ }
+
+ /**
+ * Computes the normalized position of this token relative to this range
+ * @return A number between 0.0 and 1.0 representing this token's position
+ * in this range or -1.0 if this range doesn't contain this token.
+ */
+ public double positionInRange(Token token, Range<Token> range)
+ {
+ //full range case
+ if (range.left.equals(range.right))
+ return positionInRange(token, new Range(partitioner.getMinimumToken(), partitioner.getMaximumToken()));
+
+ // leftmost token means we are on position 0.0
+ if (token.equals(range.left))
+ return 0.0;
+
+ // rightmost token means we are on position 1.0
+ if (token.equals(range.right))
+ return 1.0;
+
+ // Impossible to find position when token is not contained in range
+ if (!range.contains(token))
+ return -1.0;
+
+ return new BigDecimal(elapsedTokens(token, range)).divide(new BigDecimal(tokensInRange(range)), 3, BigDecimal.ROUND_HALF_EVEN).doubleValue();
+ }
+
+ public List<Token> splitOwnedRanges(int parts, List<WeightedRange> weightedRanges, boolean dontSplitRanges)
+ {
+ if (weightedRanges.isEmpty() || parts == 1)
return Collections.singletonList(partitioner.getMaximumToken());
BigInteger totalTokens = BigInteger.ZERO;
- for (Range<Token> r : localRanges)
+ for (WeightedRange weightedRange : weightedRanges)
{
- BigInteger right = valueForToken(token(r.right));
- totalTokens = totalTokens.add(right.subtract(valueForToken(r.left)));
+ totalTokens = totalTokens.add(weightedRange.totalTokens(this));
}
+
BigInteger perPart = totalTokens.divide(BigInteger.valueOf(parts));
// the range owned is so tiny we can't split it:
if (perPart.equals(BigInteger.ZERO))
return Collections.singletonList(partitioner.getMaximumToken());
if (dontSplitRanges)
- return splitOwnedRangesNoPartialRanges(localRanges, perPart, parts);
+ return splitOwnedRangesNoPartialRanges(weightedRanges, perPart, parts);
List<Token> boundaries = new ArrayList<>();
BigInteger sum = BigInteger.ZERO;
- for (Range<Token> r : localRanges)
+ for (WeightedRange weightedRange : weightedRanges)
{
- Token right = token(r.right);
- BigInteger currentRangeWidth = valueForToken(right).subtract(valueForToken(r.left)).abs();
- BigInteger left = valueForToken(r.left);
+ BigInteger currentRangeWidth = weightedRange.totalTokens(this);
+ BigInteger left = valueForToken(weightedRange.left());
while (sum.add(currentRangeWidth).compareTo(perPart) >= 0)
{
BigInteger withinRangeBoundary = perPart.subtract(sum);
@@ -77,26 +155,24 @@
}
boundaries.set(boundaries.size() - 1, partitioner.getMaximumToken());
- assert boundaries.size() == parts : boundaries.size() + "!=" + parts + " " + boundaries + ":" + localRanges;
+ assert boundaries.size() == parts : boundaries.size() + "!=" + parts + " " + boundaries + ":" + weightedRanges;
return boundaries;
}
- private List<Token> splitOwnedRangesNoPartialRanges(List<Range<Token>> localRanges, BigInteger perPart, int parts)
+ private List<Token> splitOwnedRangesNoPartialRanges(List<WeightedRange> weightedRanges, BigInteger perPart, int parts)
{
List<Token> boundaries = new ArrayList<>(parts);
BigInteger sum = BigInteger.ZERO;
int i = 0;
- final int rangesCount = localRanges.size();
+ final int rangesCount = weightedRanges.size();
while (boundaries.size() < parts - 1 && i < rangesCount - 1)
{
- Range<Token> r = localRanges.get(i);
- Range<Token> nextRange = localRanges.get(i + 1);
- Token right = token(r.right);
- Token nextRight = token(nextRange.right);
+ WeightedRange r = weightedRanges.get(i);
+ WeightedRange nextRange = weightedRanges.get(i + 1);
- BigInteger currentRangeWidth = valueForToken(right).subtract(valueForToken(r.left));
- BigInteger nextRangeWidth = valueForToken(nextRight).subtract(valueForToken(nextRange.left));
+ BigInteger currentRangeWidth = r.totalTokens(this);
+ BigInteger nextRangeWidth = nextRange.totalTokens(this);
sum = sum.add(currentRangeWidth);
// does this or next range take us beyond the per part limit?
@@ -109,7 +185,7 @@
if (diffNext.compareTo(diffCurrent) >= 0)
{
sum = BigInteger.ZERO;
- boundaries.add(right);
+ boundaries.add(token(r.right()));
}
}
i++;
@@ -127,4 +203,112 @@
return t.equals(partitioner.getMinimumToken()) ? partitioner.getMaximumToken() : t;
}
+ /**
+ * Splits the specified token ranges in at least {@code parts} subranges.
+ * <p>
+ * Each returned subrange will be contained in exactly one of the specified ranges.
+ *
+ * @param ranges a collection of token ranges to be split
+ * @param parts the minimum number of returned ranges
+ * @return at least {@code minParts} token ranges covering {@code ranges}
+ */
+ public Set<Range<Token>> split(Collection<Range<Token>> ranges, int parts)
+ {
+ int numRanges = ranges.size();
+ if (numRanges >= parts)
+ {
+ return Sets.newHashSet(ranges);
+ }
+ else
+ {
+ int partsPerRange = (int) Math.ceil((double) parts / numRanges);
+ return ranges.stream()
+ .map(range -> split(range, partsPerRange))
+ .flatMap(Collection::stream)
+ .collect(toSet());
+ }
+ }
+
+ /**
+ * Splits the specified token range in at least {@code minParts} subranges, unless the range has not enough tokens
+ * in which case the range will be returned without splitting.
+ *
+ * @param range a token range
+ * @param parts the number of subranges
+ * @return {@code parts} even subranges of {@code range}
+ */
+ private Set<Range<Token>> split(Range<Token> range, int parts)
+ {
+ // the range might not have enough tokens to split
+ BigInteger numTokens = tokensInRange(range);
+ if (BigInteger.valueOf(parts).compareTo(numTokens) > 0)
+ return Collections.singleton(range);
+
+ Token left = range.left;
+ Set<Range<Token>> subranges = new HashSet<>(parts);
+ for (double i = 1; i <= parts; i++)
+ {
+ Token right = partitioner.split(range.left, range.right, i / parts);
+ subranges.add(new Range<>(left, right));
+ left = right;
+ }
+ return subranges;
+ }
+
+ public static class WeightedRange
+ {
+ private final double weight;
+ private final Range<Token> range;
+
+ public WeightedRange(double weight, Range<Token> range)
+ {
+ this.weight = weight;
+ this.range = range;
+ }
+
+ public BigInteger totalTokens(Splitter splitter)
+ {
+ BigInteger right = splitter.valueForToken(splitter.token(range.right));
+ BigInteger left = splitter.valueForToken(range.left);
+ BigInteger factor = BigInteger.valueOf(Math.max(1, (long) (1 / weight)));
+ BigInteger size = right.subtract(left);
+ return size.abs().divide(factor);
+ }
+
+ public Token left()
+ {
+ return range.left;
+ }
+
+ public Token right()
+ {
+ return range.right;
+ }
+
+ public Range<Token> range()
+ {
+ return range;
+ }
+
+ public String toString()
+ {
+ return "WeightedRange{" +
+ "weight=" + weight +
+ ", range=" + range +
+ '}';
+ }
+
+ public boolean equals(Object o)
+ {
+ if (this == o) return true;
+ if (!(o instanceof WeightedRange)) return false;
+ WeightedRange that = (WeightedRange) o;
+ return Objects.equals(range, that.range);
+ }
+
+ public int hashCode()
+ {
+ return Objects.hash(weight, range);
+ }
+ }
}
diff --git a/src/java/org/apache/cassandra/dht/StreamStateStore.java b/src/java/org/apache/cassandra/dht/StreamStateStore.java
index 47b3072..e62bc04 100644
--- a/src/java/org/apache/cassandra/dht/StreamStateStore.java
+++ b/src/java/org/apache/cassandra/dht/StreamStateStore.java
@@ -19,38 +19,47 @@
import java.util.Set;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Streams;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import org.apache.cassandra.db.SystemKeyspace;
import org.apache.cassandra.streaming.StreamEvent;
import org.apache.cassandra.streaming.StreamEventHandler;
import org.apache.cassandra.streaming.StreamRequest;
import org.apache.cassandra.streaming.StreamState;
+import org.apache.cassandra.utils.Pair;
/**
* Store and update available ranges (data already received) to system keyspace.
*/
public class StreamStateStore implements StreamEventHandler
{
- public Set<Range<Token>> getAvailableRanges(String keyspace, IPartitioner partitioner)
+ private static final Logger logger = LoggerFactory.getLogger(StreamStateStore.class);
+
+ public SystemKeyspace.AvailableRanges getAvailableRanges(String keyspace, IPartitioner partitioner)
{
return SystemKeyspace.getAvailableRanges(keyspace, partitioner);
}
/**
- * Check if given token's data is available in this node.
+ * Check if given token's data is available in this node. This doesn't handle transientness in a useful way
+ * so it's only used by a legacy test
*
* @param keyspace keyspace name
* @param token token to check
* @return true if given token in the keyspace is already streamed and ready to be served.
*/
+ @VisibleForTesting
public boolean isDataAvailable(String keyspace, Token token)
{
- Set<Range<Token>> availableRanges = getAvailableRanges(keyspace, token.getPartitioner());
- for (Range<Token> range : availableRanges)
- {
- if (range.contains(token))
- return true;
- }
- return false;
+ SystemKeyspace.AvailableRanges availableRanges = getAvailableRanges(keyspace, token.getPartitioner());
+
+ return Streams.concat(availableRanges.full.stream(),
+ availableRanges.trans.stream())
+ .anyMatch(range -> range.contains(token));
}
/**
@@ -69,11 +78,11 @@
Set<String> keyspaces = se.transferredRangesPerKeyspace.keySet();
for (String keyspace : keyspaces)
{
- SystemKeyspace.updateTransferredRanges(se.description, se.peer, keyspace, se.transferredRangesPerKeyspace.get(keyspace));
+ SystemKeyspace.updateTransferredRanges(se.streamOperation, se.peer, keyspace, se.transferredRangesPerKeyspace.get(keyspace));
}
for (StreamRequest request : se.requests)
{
- SystemKeyspace.updateAvailableRanges(request.keyspace, request.ranges);
+ SystemKeyspace.updateAvailableRanges(request.keyspace, request.full.ranges(), request.transientReplicas.ranges());
}
}
}
diff --git a/src/java/org/apache/cassandra/dht/Token.java b/src/java/org/apache/cassandra/dht/Token.java
index 20b45ef..ccb66fd 100644
--- a/src/java/org/apache/cassandra/dht/Token.java
+++ b/src/java/org/apache/cassandra/dht/Token.java
@@ -26,7 +26,6 @@
import org.apache.cassandra.db.TypeSizes;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.io.util.DataOutputPlus;
-import org.apache.cassandra.utils.ByteBufferUtil;
public abstract class Token implements RingPosition<Token>, Serializable
{
@@ -40,8 +39,30 @@
public abstract Token fromByteArray(ByteBuffer bytes);
public abstract String toString(Token token); // serialize as string, not necessarily human-readable
public abstract Token fromString(String string); // deserialize
-
public abstract void validate(String token) throws ConfigurationException;
+
+ public void serialize(Token token, DataOutputPlus out) throws IOException
+ {
+ out.write(toByteArray(token));
+ }
+
+ public void serialize(Token token, ByteBuffer out) throws IOException
+ {
+ out.put(toByteArray(token));
+ }
+
+ public Token fromByteBuffer(ByteBuffer bytes, int position, int length)
+ {
+ bytes = bytes.duplicate();
+ bytes.position(position)
+ .limit(position + length);
+ return fromByteArray(bytes);
+ }
+
+ public int byteSize(Token token)
+ {
+ return toByteArray(token).remaining();
+ }
}
public static class TokenSerializer implements IPartitionerDependentSerializer<Token>
@@ -49,23 +70,28 @@
public void serialize(Token token, DataOutputPlus out, int version) throws IOException
{
IPartitioner p = token.getPartitioner();
- ByteBuffer b = p.getTokenFactory().toByteArray(token);
- ByteBufferUtil.writeWithLength(b, out);
+ out.writeInt(p.getTokenFactory().byteSize(token));
+ p.getTokenFactory().serialize(token, out);
}
public Token deserialize(DataInput in, IPartitioner p, int version) throws IOException
{
- int size = in.readInt();
+ int size = deserializeSize(in);
byte[] bytes = new byte[size];
in.readFully(bytes);
return p.getTokenFactory().fromByteArray(ByteBuffer.wrap(bytes));
}
+ public int deserializeSize(DataInput in) throws IOException
+ {
+ return in.readInt();
+ }
+
public long serializedSize(Token object, int version)
{
IPartitioner p = object.getPartitioner();
- ByteBuffer b = p.getTokenFactory().toByteArray(object);
- return TypeSizes.sizeof(b.remaining()) + b.remaining();
+ int byteSize = p.getTokenFactory().byteSize(object);
+ return TypeSizes.sizeof(byteSize) + byteSize;
}
}
diff --git a/src/java/org/apache/cassandra/dht/tokenallocator/NoReplicationTokenAllocator.java b/src/java/org/apache/cassandra/dht/tokenallocator/NoReplicationTokenAllocator.java
index 54d80dc..255a2c9 100644
--- a/src/java/org/apache/cassandra/dht/tokenallocator/NoReplicationTokenAllocator.java
+++ b/src/java/org/apache/cassandra/dht/tokenallocator/NoReplicationTokenAllocator.java
@@ -20,14 +20,12 @@
import java.util.ArrayList;
import java.util.Collection;
-import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.NavigableMap;
import java.util.PriorityQueue;
import java.util.Queue;
-import java.util.Set;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
@@ -86,6 +84,7 @@
sortedUnits.add(new Weighted<UnitInfo>(unitInfo.ownership, unitInfo));
}
+ TokenAllocatorDiagnostics.tokenInfosCreated(this, sortedUnits, sortedTokens, first);
return first;
}
@@ -113,23 +112,6 @@
unitTokens.add(new Weighted<TokenInfo>(token.replicatedOwnership, token));
}
- private Collection<Token> generateRandomTokens(UnitInfo<Unit> newUnit, int numTokens, Map<Unit, UnitInfo<Unit>> unitInfos)
- {
- Set<Token> tokens = new HashSet<>(numTokens);
- while (tokens.size() < numTokens)
- {
- Token token = partitioner.getRandomToken();
- if (!sortedTokens.containsKey(token))
- {
- tokens.add(token);
- sortedTokens.put(token, newUnit.unit);
- }
- }
- unitInfos.put(newUnit.unit, newUnit);
- createTokenInfos(unitInfos);
- return tokens;
- }
-
public Collection<Token> addUnit(Unit newUnit, int numTokens)
{
assert !tokensInUnits.containsKey(newUnit);
@@ -139,10 +121,10 @@
Map<Unit, UnitInfo<Unit>> unitInfos = createUnitInfos(groups);
if (unitInfos.isEmpty())
- return generateRandomTokens(newUnitInfo, numTokens, unitInfos);
+ return generateSplits(newUnit, numTokens);
if (numTokens > sortedTokens.size())
- return generateRandomTokens(newUnitInfo, numTokens, unitInfos);
+ return generateSplits(newUnit, numTokens);
TokenInfo<Unit> head = createTokenInfos(unitInfos);
@@ -170,7 +152,19 @@
}
List<Token> newTokens = Lists.newArrayListWithCapacity(numTokens);
+ // Generate different size nodes, at most at 2/(numTokens*2+1) difference,
+ // but tighten the spread as the number of nodes grows (since it increases the time until we need to use nodes
+ // we have just split).
+ double sizeCorrection = Math.min(1.0, (numTokens + 1.0) / (unitInfos.size() + 1.0));
+ double spread = targetAverage * sizeCorrection * 2.0 / (2 * numTokens + 1);
+ // The biggest target is assigned to the biggest existing node. This should result in better balance in
+ // the amount of data that needs to be streamed from the different sources to the new node.
+ double target = targetAverage + spread / 2;
+
+ // This step intentionally divides by the count (rather than count - 1) because we also need to count the new
+ // node. This leaves the last position in the spread (i.e. the smallest size, least data to stream) for it.
+ double step = spread / unitsToChange.size();
int nr = 0;
// calculate the tokens
for (Weighted<UnitInfo> unit : unitsToChange)
@@ -191,7 +185,7 @@
unit.value.ownership -= wt.weight;
}
- double toTakeOver = unit.weight - targetAverage;
+ double toTakeOver = unit.weight - target;
// Split toTakeOver proportionally between the vnodes.
for (Weighted<TokenInfo> wt : tokens)
{
@@ -228,13 +222,23 @@
// adjust the weight for current unit
sortedUnits.add(new Weighted<>(unit.value.ownership, unit.value));
+ target -= step;
++nr;
}
sortedUnits.add(new Weighted<>(newUnitInfo.ownership, newUnitInfo));
+ TokenAllocatorDiagnostics.unitedAdded(this, numTokens, sortedUnits, sortedTokens, newTokens, newUnit);
return newTokens;
}
+ @Override
+ Collection<Token> generateSplits(Unit newUnit, int numTokens)
+ {
+ Collection<Token> tokens = super.generateSplits(newUnit, numTokens);
+ TokenAllocatorDiagnostics.splitsGenerated(this, numTokens, sortedUnits, sortedTokens, newUnit, tokens);
+ return tokens;
+ }
+
/**
* For testing, remove the given unit preserving correct state of the allocator.
*/
@@ -257,10 +261,16 @@
tokens.add(tokenInfo.value.token);
}
sortedTokens.keySet().removeAll(tokens);
+ TokenAllocatorDiagnostics.unitRemoved(this, n, sortedUnits, sortedTokens);
}
public int getReplicas()
{
return 1;
}
+
+ public String toString()
+ {
+ return getClass().getSimpleName();
+ }
}
diff --git a/src/java/org/apache/cassandra/dht/tokenallocator/ReplicationAwareTokenAllocator.java b/src/java/org/apache/cassandra/dht/tokenallocator/ReplicationAwareTokenAllocator.java
index 87dba59..539b467 100644
--- a/src/java/org/apache/cassandra/dht/tokenallocator/ReplicationAwareTokenAllocator.java
+++ b/src/java/org/apache/cassandra/dht/tokenallocator/ReplicationAwareTokenAllocator.java
@@ -60,11 +60,14 @@
assert !unitToTokens.containsKey(newUnit);
if (unitCount() < replicas)
- // Allocation does not matter; everything replicates everywhere.
- return generateRandomTokens(newUnit, numTokens);
+ // Allocation does not matter for now; everything replicates everywhere. However, at this point it is
+ // important to start the cluster/datacenter with suitably varied token range sizes so that the algorithm
+ // can maintain good balance for any number of nodes.
+ return generateSplits(newUnit, numTokens);
if (numTokens > sortedTokens.size())
- // Some of the heuristics below can't deal with this case. Use random for now, later allocations can fix any problems this may cause.
- return generateRandomTokens(newUnit, numTokens);
+ // Some of the heuristics below can't deal with this very unlikely case. Use splits for now,
+ // later allocations can fix any problems this may cause.
+ return generateSplits(newUnit, numTokens);
// ============= construct our initial token ring state =============
@@ -73,11 +76,11 @@
Map<Unit, UnitInfo<Unit>> unitInfos = createUnitInfos(groups);
if (groups.size() < replicas)
{
- // We need at least replicas groups to do allocation correctly. If there aren't enough,
- // use random allocation.
+ // We need at least replicas groups to do allocation correctly. If there aren't enough,
+ // use splits as above.
// This part of the code should only be reached via the RATATest. StrategyAdapter should disallow
// token allocation in this case as the algorithm is not able to cover the behavior of NetworkTopologyStrategy.
- return generateRandomTokens(newUnit, numTokens);
+ return generateSplits(newUnit, numTokens);
}
// initialise our new unit's state (with an idealised ownership)
@@ -132,22 +135,24 @@
}
}
- return ImmutableList.copyOf(unitToTokens.get(newUnit));
+ ImmutableList<Token> newTokens = ImmutableList.copyOf(unitToTokens.get(newUnit));
+ TokenAllocatorDiagnostics.unitedAdded(this, numTokens, unitToTokens, sortedTokens, newTokens, newUnit);
+ return newTokens;
}
- private Collection<Token> generateRandomTokens(Unit newUnit, int numTokens)
+ /**
+ * Selects tokens by repeatedly splitting the largest range in the ring at the given ratio.
+ * This is used to choose tokens for the first nodes in the ring where the algorithm cannot be applied (e.g. when
+ * number of nodes < RF). It generates a reasonably chaotic initial token split, after which the algorithm behaves
+ * well for an unbounded number of nodes.
+ */
+
+ @Override
+ Collection<Token> generateSplits(Unit newUnit, int numTokens)
{
- Set<Token> tokens = new HashSet<>(numTokens);
- while (tokens.size() < numTokens)
- {
- Token token = partitioner.getRandomToken();
- if (!sortedTokens.containsKey(token))
- {
- tokens.add(token);
- sortedTokens.put(token, newUnit);
- unitToTokens.put(newUnit, token);
- }
- }
+ Collection<Token> tokens = super.generateSplits(newUnit, numTokens);
+ unitToTokens.putAll(newUnit, tokens);
+ TokenAllocatorDiagnostics.splitsGenerated(this, numTokens, unitToTokens, sortedTokens, newUnit, tokens);
return tokens;
}
@@ -176,6 +181,7 @@
curr = curr.next;
} while (curr != first);
+ TokenAllocatorDiagnostics.tokenInfosCreated(this, unitToTokens, first);
return first;
}
@@ -526,6 +532,7 @@
{
Collection<Token> tokens = unitToTokens.removeAll(n);
sortedTokens.keySet().removeAll(tokens);
+ TokenAllocatorDiagnostics.unitRemoved(this, n, unitToTokens, sortedTokens);
}
public int unitCount()
@@ -557,15 +564,5 @@
return split.prev;
}
}
-
- static void dumpTokens(String lead, BaseTokenInfo<?, ?> tokens)
- {
- BaseTokenInfo<?, ?> token = tokens;
- do
- {
- System.out.format("%s%s: rs %s rt %s size %.2e%n", lead, token, token.replicationStart, token.replicationThreshold, token.replicatedOwnership);
- token = token.next;
- } while (token != null && token != tokens);
- }
}
diff --git a/src/java/org/apache/cassandra/dht/tokenallocator/TokenAllocation.java b/src/java/org/apache/cassandra/dht/tokenallocator/TokenAllocation.java
index 8a3ede7..bd6d980 100644
--- a/src/java/org/apache/cassandra/dht/tokenallocator/TokenAllocation.java
+++ b/src/java/org/apache/cassandra/dht/tokenallocator/TokenAllocation.java
@@ -17,7 +17,6 @@
*/
package org.apache.cassandra.dht.tokenallocator;
-import java.net.InetAddress;
import java.util.Collection;
import java.util.Iterator;
import java.util.List;
@@ -33,16 +32,16 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.gms.Gossiper;
import org.apache.cassandra.locator.AbstractReplicationStrategy;
import org.apache.cassandra.locator.IEndpointSnitch;
+import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.locator.NetworkTopologyStrategy;
import org.apache.cassandra.locator.SimpleStrategy;
import org.apache.cassandra.locator.TokenMetadata;
import org.apache.cassandra.locator.TokenMetadata.Topology;
-import org.apache.cassandra.utils.FBUtilities;
public class TokenAllocation
{
@@ -50,7 +49,7 @@
public static Collection<Token> allocateTokens(final TokenMetadata tokenMetadata,
final AbstractReplicationStrategy rs,
- final InetAddress endpoint,
+ final InetAddressAndPort endpoint,
int numTokens)
{
TokenMetadata tokenMetadataCopy = tokenMetadata.cloneOnlyTokenMap();
@@ -64,8 +63,8 @@
SummaryStatistics os = replicatedOwnershipStats(tokenMetadataCopy, rs, endpoint);
tokenMetadataCopy.updateNormalTokens(tokens, endpoint);
SummaryStatistics ns = replicatedOwnershipStats(tokenMetadataCopy, rs, endpoint);
- logger.warn("Replicated node load in datacentre before allocation {}", statToString(os));
- logger.warn("Replicated node load in datacentre after allocation {}", statToString(ns));
+ logger.warn("Replicated node load in datacenter before allocation {}", statToString(os));
+ logger.warn("Replicated node load in datacenter after allocation {}", statToString(ns));
// TODO: Is it worth doing the replicated ownership calculation always to be able to raise this alarm?
if (ns.getStandardDeviation() > os.getStandardDeviation())
@@ -74,6 +73,20 @@
return tokens;
}
+ public static Collection<Token> allocateTokens(final TokenMetadata tokenMetadata,
+ final int replicas,
+ final InetAddressAndPort endpoint,
+ int numTokens)
+ {
+ TokenMetadata tokenMetadataCopy = tokenMetadata.cloneOnlyTokenMap();
+ StrategyAdapter strategy = getStrategy(tokenMetadataCopy, replicas, endpoint);
+ Collection<Token> tokens = create(tokenMetadata, strategy).addUnit(endpoint, numTokens);
+ tokens = adjustForCrossDatacenterClashes(tokenMetadata, strategy, tokens);
+ logger.warn("Selected tokens {}", tokens);
+ // SummaryStatistics is not implemented for `allocate_tokens_for_local_replication_factor`
+ return tokens;
+ }
+
private static Collection<Token> adjustForCrossDatacenterClashes(final TokenMetadata tokenMetadata,
StrategyAdapter strategy, Collection<Token> tokens)
{
@@ -83,7 +96,7 @@
{
while (tokenMetadata.getEndpoint(t) != null)
{
- InetAddress other = tokenMetadata.getEndpoint(t);
+ InetAddressAndPort other = tokenMetadata.getEndpoint(t);
if (strategy.inAllocationRing(other))
throw new ConfigurationException(String.format("Allocated token %s already assigned to node %s. Is another node also allocating tokens?", t, other));
t = t.increaseSlightly();
@@ -94,9 +107,9 @@
}
// return the ratio of ownership for each endpoint
- public static Map<InetAddress, Double> evaluateReplicatedOwnership(TokenMetadata tokenMetadata, AbstractReplicationStrategy rs)
+ public static Map<InetAddressAndPort, Double> evaluateReplicatedOwnership(TokenMetadata tokenMetadata, AbstractReplicationStrategy rs)
{
- Map<InetAddress, Double> ownership = Maps.newHashMap();
+ Map<InetAddressAndPort, Double> ownership = Maps.newHashMap();
List<Token> sortedTokens = tokenMetadata.sortedTokens();
Iterator<Token> it = sortedTokens.iterator();
Token current = it.next();
@@ -111,11 +124,11 @@
return ownership;
}
- static void addOwnership(final TokenMetadata tokenMetadata, final AbstractReplicationStrategy rs, Token current, Token next, Map<InetAddress, Double> ownership)
+ static void addOwnership(final TokenMetadata tokenMetadata, final AbstractReplicationStrategy rs, Token current, Token next, Map<InetAddressAndPort, Double> ownership)
{
double size = current.size(next);
Token representative = current.getPartitioner().midpoint(current, next);
- for (InetAddress n : rs.calculateNaturalEndpoints(representative, tokenMetadata))
+ for (InetAddressAndPort n : rs.calculateNaturalReplicas(representative, tokenMetadata).endpoints())
{
Double v = ownership.get(n);
ownership.put(n, v != null ? v + size : size);
@@ -128,11 +141,11 @@
}
public static SummaryStatistics replicatedOwnershipStats(TokenMetadata tokenMetadata,
- AbstractReplicationStrategy rs, InetAddress endpoint)
+ AbstractReplicationStrategy rs, InetAddressAndPort endpoint)
{
SummaryStatistics stat = new SummaryStatistics();
StrategyAdapter strategy = getStrategy(tokenMetadata, rs, endpoint);
- for (Map.Entry<InetAddress, Double> en : evaluateReplicatedOwnership(tokenMetadata, rs).entrySet())
+ for (Map.Entry<InetAddressAndPort, Double> en : evaluateReplicatedOwnership(tokenMetadata, rs).entrySet())
{
// Filter only in the same datacentre.
if (strategy.inAllocationRing(en.getKey()))
@@ -141,10 +154,10 @@
return stat;
}
- static TokenAllocator<InetAddress> create(TokenMetadata tokenMetadata, StrategyAdapter strategy)
+ static TokenAllocator<InetAddressAndPort> create(TokenMetadata tokenMetadata, StrategyAdapter strategy)
{
- NavigableMap<Token, InetAddress> sortedTokens = new TreeMap<>();
- for (Map.Entry<Token, InetAddress> en : tokenMetadata.getNormalAndBootstrappingTokenToEndpointMap().entrySet())
+ NavigableMap<Token, InetAddressAndPort> sortedTokens = new TreeMap<>();
+ for (Map.Entry<Token, InetAddressAndPort> en : tokenMetadata.getNormalAndBootstrappingTokenToEndpointMap().entrySet())
{
if (strategy.inAllocationRing(en.getValue()))
sortedTokens.put(en.getKey(), en.getValue());
@@ -152,15 +165,15 @@
return TokenAllocatorFactory.createTokenAllocator(sortedTokens, strategy, tokenMetadata.partitioner);
}
- interface StrategyAdapter extends ReplicationStrategy<InetAddress>
+ interface StrategyAdapter extends ReplicationStrategy<InetAddressAndPort>
{
// return true iff the provided endpoint occurs in the same virtual token-ring we are allocating for
// i.e. the set of the nodes that share ownership with the node we are allocating
// alternatively: return false if the endpoint's ownership is independent of the node we are allocating tokens for
- boolean inAllocationRing(InetAddress other);
+ boolean inAllocationRing(InetAddressAndPort other);
}
- static StrategyAdapter getStrategy(final TokenMetadata tokenMetadata, final AbstractReplicationStrategy rs, final InetAddress endpoint)
+ static StrategyAdapter getStrategy(final TokenMetadata tokenMetadata, final AbstractReplicationStrategy rs, final InetAddressAndPort endpoint)
{
if (rs instanceof NetworkTopologyStrategy)
return getStrategy(tokenMetadata, (NetworkTopologyStrategy) rs, rs.snitch, endpoint);
@@ -169,9 +182,9 @@
throw new ConfigurationException("Token allocation does not support replication strategy " + rs.getClass().getSimpleName());
}
- static StrategyAdapter getStrategy(final TokenMetadata tokenMetadata, final SimpleStrategy rs, final InetAddress endpoint)
+ static StrategyAdapter getStrategy(final TokenMetadata tokenMetadata, final SimpleStrategy rs, final InetAddressAndPort endpoint)
{
- final int replicas = rs.getReplicationFactor();
+ final int replicas = rs.getReplicationFactor().allReplicas;
return new StrategyAdapter()
{
@@ -182,24 +195,34 @@
}
@Override
- public Object getGroup(InetAddress unit)
+ public Object getGroup(InetAddressAndPort unit)
{
return unit;
}
@Override
- public boolean inAllocationRing(InetAddress other)
+ public boolean inAllocationRing(InetAddressAndPort other)
{
return true;
}
};
}
- static StrategyAdapter getStrategy(final TokenMetadata tokenMetadata, final NetworkTopologyStrategy rs, final IEndpointSnitch snitch, final InetAddress endpoint)
+ static StrategyAdapter getStrategy(final TokenMetadata tokenMetadata, final NetworkTopologyStrategy rs, final IEndpointSnitch snitch, final InetAddressAndPort endpoint)
{
final String dc = snitch.getDatacenter(endpoint);
- final int replicas = rs.getReplicationFactor(dc);
+ final int replicas = rs.getReplicationFactor(dc).allReplicas;
+ return getStrategy(tokenMetadata, replicas, snitch, endpoint);
+ }
+ static StrategyAdapter getStrategy(final TokenMetadata tokenMetadata, final int replicas, final InetAddressAndPort endpoint)
+ {
+ return getStrategy(tokenMetadata, replicas, DatabaseDescriptor.getEndpointSnitch(), endpoint);
+ }
+
+ static StrategyAdapter getStrategy(final TokenMetadata tokenMetadata, final int replicas, final IEndpointSnitch snitch, final InetAddressAndPort endpoint)
+ {
+ final String dc = snitch.getDatacenter(endpoint);
if (replicas == 0 || replicas == 1)
{
// No replication, each node is treated as separate.
@@ -212,13 +235,13 @@
}
@Override
- public Object getGroup(InetAddress unit)
+ public Object getGroup(InetAddressAndPort unit)
{
return unit;
}
@Override
- public boolean inAllocationRing(InetAddress other)
+ public boolean inAllocationRing(InetAddressAndPort other)
{
return dc.equals(snitch.getDatacenter(other));
}
@@ -232,7 +255,7 @@
? topology.getDatacenterRacks().get(dc).asMap().size()
: 1;
- if (racks >= replicas)
+ if (racks > replicas)
{
return new StrategyAdapter()
{
@@ -243,18 +266,44 @@
}
@Override
- public Object getGroup(InetAddress unit)
+ public Object getGroup(InetAddressAndPort unit)
{
return snitch.getRack(unit);
}
@Override
- public boolean inAllocationRing(InetAddress other)
+ public boolean inAllocationRing(InetAddressAndPort other)
{
return dc.equals(snitch.getDatacenter(other));
}
};
}
+ else if (racks == replicas)
+ {
+ // When the number of racks is the same as the replication factor, everything must replicate exactly once
+ // in each rack. This is the same as having independent rings from each rack.
+ final String rack = snitch.getRack(endpoint);
+ return new StrategyAdapter()
+ {
+ @Override
+ public int replicas()
+ {
+ return 1;
+ }
+
+ @Override
+ public Object getGroup(InetAddressAndPort unit)
+ {
+ return unit;
+ }
+
+ @Override
+ public boolean inAllocationRing(InetAddressAndPort other)
+ {
+ return dc.equals(snitch.getDatacenter(other)) && rack.equals(snitch.getRack(other));
+ }
+ };
+ }
else if (racks == 1)
{
// One rack, each node treated as separate.
@@ -267,13 +316,13 @@
}
@Override
- public Object getGroup(InetAddress unit)
+ public Object getGroup(InetAddressAndPort unit)
{
return unit;
}
@Override
- public boolean inAllocationRing(InetAddress other)
+ public boolean inAllocationRing(InetAddressAndPort other)
{
return dc.equals(snitch.getDatacenter(other));
}
diff --git a/src/java/org/apache/cassandra/dht/tokenallocator/TokenAllocatorBase.java b/src/java/org/apache/cassandra/dht/tokenallocator/TokenAllocatorBase.java
index f59bfd4..3d7e6b9 100644
--- a/src/java/org/apache/cassandra/dht/tokenallocator/TokenAllocatorBase.java
+++ b/src/java/org/apache/cassandra/dht/tokenallocator/TokenAllocatorBase.java
@@ -18,9 +18,13 @@
package org.apache.cassandra.dht.tokenallocator;
+import java.util.Collection;
+import java.util.List;
import java.util.Map;
import java.util.NavigableMap;
+import java.util.Random;
+import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import org.apache.cassandra.dht.IPartitioner;
@@ -28,6 +32,9 @@
public abstract class TokenAllocatorBase<Unit> implements TokenAllocator<Unit>
{
+ static final double MIN_INITIAL_SPLITS_RATIO = 1.0 - 1.0 / Math.sqrt(5.0);
+ static final double MAX_INITIAL_SPLITS_RATIO = MIN_INITIAL_SPLITS_RATIO + 0.075;
+
final NavigableMap<Token, Unit> sortedTokens;
final ReplicationStrategy<Unit> strategy;
final IPartitioner partitioner;
@@ -79,6 +86,59 @@
return group;
}
+ Collection<Token> generateSplits(Unit newUnit, int numTokens)
+ {
+ return generateSplits(newUnit, numTokens, MIN_INITIAL_SPLITS_RATIO, MAX_INITIAL_SPLITS_RATIO);
+ }
+ /**
+ * Selects tokens by repeatedly splitting the largest range in the ring at the given ratio.
+ *
+ * This is used to choose tokens for the first nodes in the ring where the algorithm cannot be applied (e.g. when
+ * number of nodes < RF). It generates a reasonably chaotic initial token split, after which the algorithm behaves
+ * well for an unbounded number of nodes.
+ */
+ Collection<Token> generateSplits(Unit newUnit, int numTokens, double minRatio, double maxRatio)
+ {
+ Random random = new Random(sortedTokens.size());
+
+ double potentialRatioGrowth = maxRatio - minRatio;
+
+ List<Token> tokens = Lists.newArrayListWithExpectedSize(numTokens);
+
+ if (sortedTokens.isEmpty())
+ {
+ // Select a random start token. This has no effect on distribution, only on where the local ring is "centered".
+ // Using a random start decreases the chances of clash with the tokens of other datacenters in the ring.
+ Token t = partitioner.getRandomToken();
+ tokens.add(t);
+ sortedTokens.put(t, newUnit);
+ }
+
+ while (tokens.size() < numTokens)
+ {
+ // split max span using given ratio
+ Token prev = sortedTokens.lastKey();
+ double maxsz = 0;
+ Token t1 = null;
+ Token t2 = null;
+ for (Token curr : sortedTokens.keySet())
+ {
+ double sz = prev.size(curr);
+ if (sz > maxsz)
+ {
+ maxsz = sz;
+ t1 = prev; t2 = curr;
+ }
+ prev = curr;
+ }
+ assert t1 != null;
+ Token t = partitioner.split(t1, t2, minRatio + potentialRatioGrowth * random.nextDouble());
+ tokens.add(t);
+ sortedTokens.put(t, newUnit);
+ }
+ return tokens;
+ }
+
/**
* Unique group object that one or more UnitInfo objects link to.
*/
diff --git a/src/java/org/apache/cassandra/dht/tokenallocator/TokenAllocatorDiagnostics.java b/src/java/org/apache/cassandra/dht/tokenallocator/TokenAllocatorDiagnostics.java
new file mode 100644
index 0000000..04d7455
--- /dev/null
+++ b/src/java/org/apache/cassandra/dht/tokenallocator/TokenAllocatorDiagnostics.java
@@ -0,0 +1,196 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.dht.tokenallocator;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.NavigableMap;
+import java.util.Queue;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Multimap;
+
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.dht.tokenallocator.TokenAllocatorBase.TokenInfo;
+import org.apache.cassandra.dht.tokenallocator.TokenAllocatorBase.UnitInfo;
+import org.apache.cassandra.dht.tokenallocator.TokenAllocatorBase.Weighted;
+import org.apache.cassandra.dht.tokenallocator.TokenAllocatorEvent.TokenAllocatorEventType;
+import org.apache.cassandra.diag.DiagnosticEventService;
+
+/**
+ * Utility methods for DiagnosticEvent around {@link TokenAllocator} activities.
+ */
+final class TokenAllocatorDiagnostics
+{
+ private static final DiagnosticEventService service = DiagnosticEventService.instance();
+
+ private TokenAllocatorDiagnostics()
+ {
+ }
+
+ static <Unit> void noReplicationTokenAllocatorInstanciated(NoReplicationTokenAllocator<Unit> allocator)
+ {
+ if (isEnabled(TokenAllocatorEventType.NO_REPLICATION_AWARE_TOKEN_ALLOCATOR_INSTANCIATED))
+ service.publish(new TokenAllocatorEvent<>(TokenAllocatorEventType.NO_REPLICATION_AWARE_TOKEN_ALLOCATOR_INSTANCIATED,
+ allocator, null, null, null, null, null, null, null));
+ }
+
+ static <Unit> void replicationTokenAllocatorInstanciated(ReplicationAwareTokenAllocator<Unit> allocator)
+ {
+ if (isEnabled(TokenAllocatorEventType.REPLICATION_AWARE_TOKEN_ALLOCATOR_INSTANCIATED))
+ service.publish(new TokenAllocatorEvent<>(TokenAllocatorEventType.REPLICATION_AWARE_TOKEN_ALLOCATOR_INSTANCIATED,
+ allocator, null, null, null,null, null, null, null));
+ }
+
+ static <Unit> void unitedAdded(TokenAllocatorBase<Unit> allocator, int numTokens,
+ Queue<Weighted<UnitInfo>> sortedUnits, NavigableMap<Token, Unit> sortedTokens,
+ List<Token> tokens, Unit unit)
+ {
+ if (isEnabled(TokenAllocatorEventType.UNIT_ADDED))
+ service.publish(new TokenAllocatorEvent<>(TokenAllocatorEventType.UNIT_ADDED,
+ allocator,
+ numTokens,
+ ImmutableList.copyOf(sortedUnits),
+ null,
+ ImmutableMap.copyOf(sortedTokens),
+ ImmutableList.copyOf(tokens),
+ unit,
+ null));
+ }
+
+ static <Unit> void unitedAdded(TokenAllocatorBase<Unit> allocator, int numTokens,
+ Multimap<Unit, Token> unitToTokens, NavigableMap<Token, Unit> sortedTokens,
+ List<Token> tokens, Unit unit)
+ {
+ if (isEnabled(TokenAllocatorEventType.UNIT_ADDED))
+ service.publish(new TokenAllocatorEvent<>(TokenAllocatorEventType.UNIT_ADDED,
+ allocator,
+ numTokens,
+ null,
+ ImmutableMap.copyOf(unitToTokens.asMap()),
+ ImmutableMap.copyOf(sortedTokens),
+ ImmutableList.copyOf(tokens),
+ unit,
+ null));
+ }
+
+
+ static <Unit> void unitRemoved(TokenAllocatorBase<Unit> allocator, Unit unit,
+ Queue<Weighted<UnitInfo>> sortedUnits, Map<Token, Unit> sortedTokens)
+ {
+ if (isEnabled(TokenAllocatorEventType.UNIT_REMOVED))
+ service.publish(new TokenAllocatorEvent<>(TokenAllocatorEventType.UNIT_REMOVED,
+ allocator,
+ null,
+ ImmutableList.copyOf(sortedUnits),
+ null,
+ ImmutableMap.copyOf(sortedTokens),
+ null,
+ unit,
+ null));
+ }
+
+ static <Unit> void unitRemoved(TokenAllocatorBase<Unit> allocator, Unit unit,
+ Multimap<Unit, Token> unitToTokens, Map<Token, Unit> sortedTokens)
+ {
+ if (isEnabled(TokenAllocatorEventType.UNIT_REMOVED))
+ service.publish(new TokenAllocatorEvent<>(TokenAllocatorEventType.UNIT_REMOVED,
+ allocator,
+ null,
+ null,
+ ImmutableMap.copyOf(unitToTokens.asMap()),
+ ImmutableMap.copyOf(sortedTokens),
+ null,
+ unit,
+ null));
+ }
+
+ static <Unit> void tokenInfosCreated(TokenAllocatorBase<Unit> allocator, Queue<Weighted<UnitInfo>> sortedUnits,
+ Map<Token, Unit> sortedTokens, TokenInfo<Unit> tokenInfo)
+ {
+ if (isEnabled(TokenAllocatorEventType.TOKEN_INFOS_CREATED))
+ service.publish(new TokenAllocatorEvent<>(TokenAllocatorEventType.TOKEN_INFOS_CREATED,
+ allocator,
+ null,
+ ImmutableList.copyOf(sortedUnits),
+ null,
+ ImmutableMap.copyOf(sortedTokens),
+ null,
+ null,
+ tokenInfo));
+ }
+
+ static <Unit> void tokenInfosCreated(TokenAllocatorBase<Unit> allocator, Multimap<Unit, Token> unitToTokens,
+ TokenInfo<Unit> tokenInfo)
+ {
+ if (isEnabled(TokenAllocatorEventType.TOKEN_INFOS_CREATED))
+ service.publish(new TokenAllocatorEvent<>(TokenAllocatorEventType.TOKEN_INFOS_CREATED,
+ allocator,
+ null,
+ null,
+ ImmutableMap.copyOf(unitToTokens.asMap()),
+ null,
+ null,
+ null,
+ tokenInfo));
+ }
+
+ static <Unit> void splitsGenerated(TokenAllocatorBase<Unit> allocator,
+ int numTokens, Queue<Weighted<UnitInfo>> sortedUnits,
+ NavigableMap<Token, Unit> sortedTokens,
+ Unit newUnit,
+ Collection<Token> tokens)
+ {
+ if (isEnabled(TokenAllocatorEventType.RANDOM_TOKENS_GENERATED))
+ service.publish(new TokenAllocatorEvent<>(TokenAllocatorEventType.RANDOM_TOKENS_GENERATED,
+ allocator,
+ numTokens,
+ ImmutableList.copyOf(sortedUnits),
+ null,
+ ImmutableMap.copyOf(sortedTokens),
+ ImmutableList.copyOf(tokens),
+ newUnit,
+ null));
+ }
+
+ static <Unit> void splitsGenerated(TokenAllocatorBase<Unit> allocator,
+ int numTokens, Multimap<Unit, Token> unitToTokens,
+ NavigableMap<Token, Unit> sortedTokens, Unit newUnit,
+ Collection<Token> tokens)
+ {
+ if (isEnabled(TokenAllocatorEventType.RANDOM_TOKENS_GENERATED))
+ service.publish(new TokenAllocatorEvent<>(TokenAllocatorEventType.RANDOM_TOKENS_GENERATED,
+ allocator,
+ numTokens,
+ null,
+ ImmutableMap.copyOf(unitToTokens.asMap()),
+ ImmutableMap.copyOf(sortedTokens),
+ ImmutableList.copyOf(tokens),
+ newUnit,
+ null));
+ }
+
+ private static boolean isEnabled(TokenAllocatorEventType type)
+ {
+ return service.isEnabled(TokenAllocatorEvent.class, type);
+ }
+
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/dht/tokenallocator/TokenAllocatorEvent.java b/src/java/org/apache/cassandra/dht/tokenallocator/TokenAllocatorEvent.java
new file mode 100644
index 0000000..ca59938
--- /dev/null
+++ b/src/java/org/apache/cassandra/dht/tokenallocator/TokenAllocatorEvent.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.dht.tokenallocator;
+
+import java.io.Serializable;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import javax.annotation.Nullable;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.dht.tokenallocator.TokenAllocatorBase.TokenInfo;
+import org.apache.cassandra.dht.tokenallocator.TokenAllocatorBase.UnitInfo;
+import org.apache.cassandra.dht.tokenallocator.TokenAllocatorBase.Weighted;
+import org.apache.cassandra.diag.DiagnosticEvent;
+
+/**
+ * DiagnosticEvent implementation for {@link TokenAllocator} activities.
+ */
+final class TokenAllocatorEvent<Unit> extends DiagnosticEvent
+{
+
+ private final TokenAllocatorEventType type;
+ private final TokenAllocatorBase<Unit> allocator;
+ private final int replicas;
+ @Nullable
+ private final Integer numTokens;
+ @Nullable
+ private final Collection<Weighted<UnitInfo>> sortedUnits;
+ @Nullable
+ private final Map<Unit, Collection<Token>> unitToTokens;
+ @Nullable
+ private final ImmutableMap<Token, Unit> sortedTokens;
+ @Nullable
+ private final List<Token> tokens;
+ @Nullable
+ private final Unit unit;
+ @Nullable
+ private final TokenInfo<Unit> tokenInfo;
+
+ TokenAllocatorEvent(TokenAllocatorEventType type, TokenAllocatorBase<Unit> allocator, @Nullable Integer numTokens,
+ @Nullable ImmutableList<Weighted<UnitInfo>> sortedUnits, @Nullable ImmutableMap<Unit, Collection<Token>> unitToTokens,
+ @Nullable ImmutableMap<Token, Unit> sortedTokens, @Nullable ImmutableList<Token> tokens, Unit unit,
+ @Nullable TokenInfo<Unit> tokenInfo)
+ {
+ this.type = type;
+ this.allocator = allocator;
+ this.replicas = allocator.getReplicas();
+ this.numTokens = numTokens;
+ this.sortedUnits = sortedUnits;
+ this.unitToTokens = unitToTokens;
+ this.sortedTokens = sortedTokens;
+ this.tokens = tokens;
+ this.unit = unit;
+ this.tokenInfo = tokenInfo;
+ }
+
+ enum TokenAllocatorEventType
+ {
+ REPLICATION_AWARE_TOKEN_ALLOCATOR_INSTANCIATED,
+ NO_REPLICATION_AWARE_TOKEN_ALLOCATOR_INSTANCIATED,
+ UNIT_ADDED,
+ UNIT_REMOVED,
+ TOKEN_INFOS_CREATED,
+ RANDOM_TOKENS_GENERATED,
+ TOKENS_ALLOCATED
+ }
+
+ public TokenAllocatorEventType getType()
+ {
+ return type;
+ }
+
+ public HashMap<String, Serializable> toMap()
+ {
+ // be extra defensive against nulls and bugs
+ HashMap<String, Serializable> ret = new HashMap<>();
+ if (allocator != null)
+ {
+ if (allocator.partitioner != null) ret.put("partitioner", allocator.partitioner.getClass().getSimpleName());
+ if (allocator.strategy != null) ret.put("strategy", allocator.strategy.getClass().getSimpleName());
+ }
+ ret.put("replicas", replicas);
+ ret.put("numTokens", this.numTokens);
+ ret.put("sortedUnits", String.valueOf(sortedUnits));
+ ret.put("sortedTokens", String.valueOf(sortedTokens));
+ ret.put("unitToTokens", String.valueOf(unitToTokens));
+ ret.put("tokens", String.valueOf(tokens));
+ ret.put("unit", String.valueOf(unit));
+ ret.put("tokenInfo", String.valueOf(tokenInfo));
+ return ret;
+ }
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/dht/tokenallocator/TokenAllocatorFactory.java b/src/java/org/apache/cassandra/dht/tokenallocator/TokenAllocatorFactory.java
index 58acb56..117fd09 100644
--- a/src/java/org/apache/cassandra/dht/tokenallocator/TokenAllocatorFactory.java
+++ b/src/java/org/apache/cassandra/dht/tokenallocator/TokenAllocatorFactory.java
@@ -18,7 +18,6 @@
package org.apache.cassandra.dht.tokenallocator;
-import java.net.InetAddress;
import java.util.NavigableMap;
import org.slf4j.Logger;
@@ -26,20 +25,25 @@
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.locator.InetAddressAndPort;
public class TokenAllocatorFactory
{
private static final Logger logger = LoggerFactory.getLogger(TokenAllocatorFactory.class);
- public static TokenAllocator<InetAddress> createTokenAllocator(NavigableMap<Token, InetAddress> sortedTokens,
- ReplicationStrategy<InetAddress> strategy,
- IPartitioner partitioner)
+ public static TokenAllocator<InetAddressAndPort> createTokenAllocator(NavigableMap<Token, InetAddressAndPort> sortedTokens,
+ ReplicationStrategy<InetAddressAndPort> strategy,
+ IPartitioner partitioner)
{
if(strategy.replicas() == 1)
{
logger.info("Using NoReplicationTokenAllocator.");
- return new NoReplicationTokenAllocator<>(sortedTokens, strategy, partitioner);
+ NoReplicationTokenAllocator<InetAddressAndPort> allocator = new NoReplicationTokenAllocator<>(sortedTokens, strategy, partitioner);
+ TokenAllocatorDiagnostics.noReplicationTokenAllocatorInstanciated(allocator);
+ return allocator;
}
logger.info("Using ReplicationAwareTokenAllocator.");
- return new ReplicationAwareTokenAllocator<>(sortedTokens, strategy, partitioner);
+ ReplicationAwareTokenAllocator<InetAddressAndPort> allocator = new ReplicationAwareTokenAllocator<>(sortedTokens, strategy, partitioner);
+ TokenAllocatorDiagnostics.replicationTokenAllocatorInstanciated(allocator);
+ return allocator;
}
}
diff --git a/src/java/org/apache/cassandra/diag/DiagnosticEvent.java b/src/java/org/apache/cassandra/diag/DiagnosticEvent.java
new file mode 100644
index 0000000..5de703b
--- /dev/null
+++ b/src/java/org/apache/cassandra/diag/DiagnosticEvent.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.diag;
+
+import java.io.Serializable;
+import java.util.Map;
+
+/**
+ * Base class for internally emitted events used for diagnostics and testing.
+ */
+public abstract class DiagnosticEvent
+{
+ /**
+ * Event creation time.
+ */
+ public final long timestamp = System.currentTimeMillis();
+
+ /**
+ * Name of allocating thread.
+ */
+ public final String threadName = Thread.currentThread().getName();
+
+ /**
+ * Returns event type discriminator. This will usually be a enum value.
+ */
+ public abstract Enum<?> getType();
+
+ /**
+ * Returns map of key-value pairs containing relevant event details. Values can be complex objects like other
+ * maps, but must be Serializable, as returned values may be consumed by external clients. It's strongly recommended
+ * to stick to standard Java classes to avoid distributing custom classes to clients and also prevent potential
+ * class versioning conflicts.
+ */
+ public abstract Map<String, Serializable> toMap();
+}
diff --git a/src/java/org/apache/cassandra/diag/DiagnosticEventPersistence.java b/src/java/org/apache/cassandra/diag/DiagnosticEventPersistence.java
new file mode 100644
index 0000000..7da335c
--- /dev/null
+++ b/src/java/org/apache/cassandra/diag/DiagnosticEventPersistence.java
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.diag;
+
+import java.io.InvalidClassException;
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.NavigableMap;
+import java.util.SortedMap;
+import java.util.TreeMap;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.function.Consumer;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.diag.store.DiagnosticEventMemoryStore;
+import org.apache.cassandra.diag.store.DiagnosticEventStore;
+
+
+/**
+ * Manages storing and retrieving events based on enabled {@link DiagnosticEventStore} implementation.
+ */
+public final class DiagnosticEventPersistence
+{
+ private static final Logger logger = LoggerFactory.getLogger(DiagnosticEventPersistence.class);
+
+ private static final DiagnosticEventPersistence instance = new DiagnosticEventPersistence();
+
+ private final Map<Class, DiagnosticEventStore<Long>> stores = new ConcurrentHashMap<>();
+
+ private final Consumer<DiagnosticEvent> eventConsumer = this::onEvent;
+
+ public static void start()
+ {
+ // make sure id broadcaster is initialized (registered as MBean)
+ LastEventIdBroadcaster.instance();
+ }
+
+ public static DiagnosticEventPersistence instance()
+ {
+ return instance;
+ }
+
+ public SortedMap<Long, Map<String, Serializable>> getEvents(String eventClazz, Long key, int limit, boolean includeKey)
+ {
+ assert eventClazz != null;
+ assert key != null;
+ assert limit >= 0;
+
+ Class cls;
+ try
+ {
+ cls = getEventClass(eventClazz);
+ }
+ catch (ClassNotFoundException | InvalidClassException e)
+ {
+ throw new RuntimeException(e);
+ }
+ DiagnosticEventStore<Long> store = getStore(cls);
+
+ NavigableMap<Long, DiagnosticEvent> events = store.scan(key, includeKey ? limit : limit + 1);
+ if (!includeKey && !events.isEmpty()) events = events.tailMap(key, false);
+ TreeMap<Long, Map<String, Serializable>> ret = new TreeMap<>();
+ for (Map.Entry<Long, DiagnosticEvent> entry : events.entrySet())
+ {
+ DiagnosticEvent event = entry.getValue();
+ HashMap<String, Serializable> val = new HashMap<>(event.toMap());
+ val.put("class", event.getClass().getName());
+ val.put("type", event.getType().name());
+ val.put("ts", event.timestamp);
+ val.put("thread", event.threadName);
+ ret.put(entry.getKey(), val);
+ }
+ logger.debug("Returning {} {} events for key {} (limit {}) (includeKey {})", ret.size(), eventClazz, key, limit, includeKey);
+ return ret;
+ }
+
+ public void enableEventPersistence(String eventClazz)
+ {
+ try
+ {
+ logger.debug("Enabling events: {}", eventClazz);
+ DiagnosticEventService.instance().subscribe(getEventClass(eventClazz), eventConsumer);
+ }
+ catch (ClassNotFoundException | InvalidClassException e)
+ {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public void disableEventPersistence(String eventClazz)
+ {
+ try
+ {
+ logger.debug("Disabling events: {}", eventClazz);
+ DiagnosticEventService.instance().unsubscribe(getEventClass(eventClazz), eventConsumer);
+ }
+ catch (ClassNotFoundException | InvalidClassException e)
+ {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private void onEvent(DiagnosticEvent event)
+ {
+ Class<? extends DiagnosticEvent> cls = event.getClass();
+ if (logger.isTraceEnabled())
+ logger.trace("Persisting received {} event", cls.getName());
+ DiagnosticEventStore<Long> store = getStore(cls);
+ store.store(event);
+ LastEventIdBroadcaster.instance().setLastEventId(event.getClass().getName(), store.getLastEventId());
+ }
+
+ private Class<DiagnosticEvent> getEventClass(String eventClazz) throws ClassNotFoundException, InvalidClassException
+ {
+ // get class by eventClazz argument name
+ // restrict class loading for security reasons
+ if (!eventClazz.startsWith("org.apache.cassandra."))
+ throw new RuntimeException("Not a Cassandra event class: " + eventClazz);
+
+ Class<DiagnosticEvent> clazz = (Class<DiagnosticEvent>) Class.forName(eventClazz);
+
+ if (!(DiagnosticEvent.class.isAssignableFrom(clazz)))
+ throw new InvalidClassException("Event class must be of type DiagnosticEvent");
+
+ return clazz;
+ }
+
+ private DiagnosticEventStore<Long> getStore(Class cls)
+ {
+ return stores.computeIfAbsent(cls, (storeKey) -> new DiagnosticEventMemoryStore());
+ }
+}
diff --git a/src/java/org/apache/cassandra/diag/DiagnosticEventService.java b/src/java/org/apache/cassandra/diag/DiagnosticEventService.java
new file mode 100644
index 0000000..5953a1d
--- /dev/null
+++ b/src/java/org/apache/cassandra/diag/DiagnosticEventService.java
@@ -0,0 +1,341 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.diag;
+
+import java.io.Serializable;
+import java.lang.management.ManagementFactory;
+import java.util.Collection;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.function.Consumer;
+
+import javax.annotation.Nullable;
+import javax.management.MBeanServer;
+import javax.management.ObjectName;
+
+import com.google.common.collect.ImmutableCollection;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableMultimap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.ImmutableSetMultimap;
+import com.google.common.collect.Iterables;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.utils.MBeanWrapper;
+
+/**
+ * Service for publishing and consuming {@link DiagnosticEvent}s.
+ */
+public final class DiagnosticEventService implements DiagnosticEventServiceMBean
+{
+ private static final Logger logger = LoggerFactory.getLogger(DiagnosticEventService.class);
+
+ // Subscribers interested in consuming all kind of events
+ private ImmutableSet<Consumer<DiagnosticEvent>> subscribersAll = ImmutableSet.of();
+
+ // Subscribers for particular event class, e.g. BootstrapEvent
+ private ImmutableSetMultimap<Class<? extends DiagnosticEvent>, Consumer<DiagnosticEvent>> subscribersByClass = ImmutableSetMultimap.of();
+
+ // Subscribers for event class and type, e.g. BootstrapEvent#TOKENS_ALLOCATED
+ private ImmutableMap<Class, ImmutableSetMultimap<Enum<?>, Consumer<DiagnosticEvent>>> subscribersByClassAndType = ImmutableMap.of();
+
+ private static final DiagnosticEventService instance = new DiagnosticEventService();
+
+ private DiagnosticEventService()
+ {
+ MBeanWrapper.instance.registerMBean(this,"org.apache.cassandra.diag:type=DiagnosticEventService");
+
+ // register broadcasters for JMX events
+ DiagnosticEventPersistence.start();
+ }
+
+ /**
+ * Makes provided event available to all subscribers.
+ */
+ public void publish(DiagnosticEvent event)
+ {
+ if (!DatabaseDescriptor.diagnosticEventsEnabled())
+ return;
+
+ logger.trace("Publishing: {}", event);
+
+ // event class + type
+ ImmutableMultimap<Enum<?>, Consumer<DiagnosticEvent>> consumersByType = subscribersByClassAndType.get(event.getClass());
+ if (consumersByType != null)
+ {
+ ImmutableCollection<Consumer<DiagnosticEvent>> consumers = consumersByType.get(event.getType());
+ if (consumers != null)
+ {
+ for (Consumer<DiagnosticEvent> consumer : consumers)
+ consumer.accept(event);
+ }
+ }
+
+ // event class
+ Set<Consumer<DiagnosticEvent>> consumersByEvents = subscribersByClass.get(event.getClass());
+ if (consumersByEvents != null)
+ {
+ for (Consumer<DiagnosticEvent> consumer : consumersByEvents)
+ consumer.accept(event);
+ }
+
+ // all events
+ for (Consumer<DiagnosticEvent> consumer : subscribersAll)
+ consumer.accept(event);
+ }
+
+ /**
+ * Registers event handler for specified class of events.
+ * @param event DiagnosticEvent class implementation
+ * @param consumer Consumer for received events
+ */
+ public synchronized <E extends DiagnosticEvent> void subscribe(Class<E> event, Consumer<E> consumer)
+ {
+ logger.debug("Adding subscriber: {}", consumer);
+ subscribersByClass = ImmutableSetMultimap.<Class<? extends DiagnosticEvent>, Consumer<DiagnosticEvent>>builder()
+ .putAll(subscribersByClass)
+ .put(event, new TypedConsumerWrapper<>(consumer))
+ .build();
+ logger.debug("Total subscribers: {}", subscribersByClass.values().size());
+ }
+
+ /**
+ * Registers event handler for specified class of events.
+ * @param event DiagnosticEvent class implementation
+ * @param consumer Consumer for received events
+ */
+ public synchronized <E extends DiagnosticEvent, T extends Enum<T>> void subscribe(Class<E> event,
+ T eventType,
+ Consumer<E> consumer)
+ {
+ ImmutableSetMultimap.Builder<Enum<?>, Consumer<DiagnosticEvent>> byTypeBuilder = ImmutableSetMultimap.builder();
+ if (subscribersByClassAndType.containsKey(event))
+ byTypeBuilder.putAll(subscribersByClassAndType.get(event));
+ byTypeBuilder.put(eventType, new TypedConsumerWrapper<>(consumer));
+
+ ImmutableMap.Builder<Class, ImmutableSetMultimap<Enum<?>, Consumer<DiagnosticEvent>>> byClassBuilder = ImmutableMap.builder();
+ for (Class clazz : subscribersByClassAndType.keySet())
+ {
+ if (!clazz.equals(event))
+ byClassBuilder.put(clazz, subscribersByClassAndType.get(clazz));
+ }
+
+ subscribersByClassAndType = byClassBuilder
+ .put(event, byTypeBuilder.build())
+ .build();
+ }
+
+ /**
+ * Registers event handler for all DiagnosticEvents published from this point.
+ * @param consumer Consumer for received events
+ */
+ public synchronized void subscribeAll(Consumer<DiagnosticEvent> consumer)
+ {
+ subscribersAll = ImmutableSet.<Consumer<DiagnosticEvent>>builder()
+ .addAll(subscribersAll)
+ .add(consumer)
+ .build();
+ }
+
+ /**
+ * De-registers event handler from receiving any further events.
+ * @param consumer Consumer registered for receiving events
+ */
+ public synchronized <E extends DiagnosticEvent> void unsubscribe(Consumer<E> consumer)
+ {
+ unsubscribe(null, consumer);
+ }
+
+ /**
+ * De-registers event handler from receiving any further events.
+ * @param event DiagnosticEvent class to unsubscribe from
+ * @param consumer Consumer registered for receiving events
+ */
+ public synchronized <E extends DiagnosticEvent> void unsubscribe(@Nullable Class<E> event, Consumer<E> consumer)
+ {
+ // all events
+ subscribersAll = ImmutableSet.copyOf(Iterables.filter(subscribersAll, (c) -> c != consumer));
+
+ // event class
+ ImmutableSetMultimap.Builder<Class<? extends DiagnosticEvent>, Consumer<DiagnosticEvent>> byClassBuilder = ImmutableSetMultimap.builder();
+ Collection<Map.Entry<Class<? extends DiagnosticEvent>, Consumer<DiagnosticEvent>>> entries = subscribersByClass.entries();
+ for (Map.Entry<Class<? extends DiagnosticEvent>, Consumer<DiagnosticEvent>> entry : entries)
+ {
+ Consumer<DiagnosticEvent> subscriber = entry.getValue();
+ if (subscriber instanceof TypedConsumerWrapper)
+ subscriber = ((TypedConsumerWrapper)subscriber).wrapped;
+
+ // other consumers or other events
+ if (subscriber != consumer || (event != null && !entry.getKey().equals(event)))
+ {
+ byClassBuilder = byClassBuilder.put(entry);
+ }
+ }
+ subscribersByClass = byClassBuilder.build();
+
+
+ // event class + type
+ ImmutableMap.Builder<Class, ImmutableSetMultimap<Enum<?>, Consumer<DiagnosticEvent>>> byClassAndTypeBuilder = ImmutableMap.builder();
+ for (Map.Entry<Class, ImmutableSetMultimap<Enum<?>, Consumer<DiagnosticEvent>>> byClassEntry : subscribersByClassAndType.entrySet())
+ {
+ ImmutableSetMultimap.Builder<Enum<?>, Consumer<DiagnosticEvent>> byTypeBuilder = ImmutableSetMultimap.builder();
+ ImmutableSetMultimap<Enum<?>, Consumer<DiagnosticEvent>> byTypeConsumers = byClassEntry.getValue();
+ Iterables.filter(byTypeConsumers.entries(), (e) ->
+ {
+ if (e == null || e.getValue() == null) return false;
+ Consumer<DiagnosticEvent> subscriber = e.getValue();
+ if (subscriber instanceof TypedConsumerWrapper)
+ subscriber = ((TypedConsumerWrapper) subscriber).wrapped;
+ return subscriber != consumer || (event != null && !byClassEntry.getKey().equals(event));
+ }).forEach(byTypeBuilder::put);
+
+ ImmutableSetMultimap<Enum<?>, Consumer<DiagnosticEvent>> byType = byTypeBuilder.build();
+ if (!byType.isEmpty())
+ byClassAndTypeBuilder.put(byClassEntry.getKey(), byType);
+ }
+
+ subscribersByClassAndType = byClassAndTypeBuilder.build();
+ }
+
+ /**
+ * Indicates if any {@link Consumer} has been registered for the specified class of events.
+ * @param event DiagnosticEvent class implementation
+ */
+ public <E extends DiagnosticEvent> boolean hasSubscribers(Class<E> event)
+ {
+ return !subscribersAll.isEmpty() || subscribersByClass.containsKey(event) || subscribersByClassAndType.containsKey(event);
+ }
+
+ /**
+ * Indicates if any {@link Consumer} has been registered for the specified class of events.
+ * @param event DiagnosticEvent class implementation
+ * @param eventType Subscribed event type matched against {@link DiagnosticEvent#getType()}
+ */
+ public <E extends DiagnosticEvent, T extends Enum<T>> boolean hasSubscribers(Class<E> event, T eventType)
+ {
+ if (!subscribersAll.isEmpty())
+ return true;
+
+ ImmutableSet<Consumer<DiagnosticEvent>> subscribers = subscribersByClass.get(event);
+ if (subscribers != null && !subscribers.isEmpty())
+ return true;
+
+ ImmutableSetMultimap<Enum<?>, Consumer<DiagnosticEvent>> byType = subscribersByClassAndType.get(event);
+ if (byType == null || byType.isEmpty()) return false;
+
+ Set<Consumer<DiagnosticEvent>> consumers = byType.get(eventType);
+ return consumers != null && !consumers.isEmpty();
+ }
+
+ /**
+ * Indicates if events are enabled for specified event class based on {@link DatabaseDescriptor#diagnosticEventsEnabled()}
+ * and {@link #hasSubscribers(Class)}.
+ * @param event DiagnosticEvent class implementation
+ */
+ public <E extends DiagnosticEvent> boolean isEnabled(Class<E> event)
+ {
+ return DatabaseDescriptor.diagnosticEventsEnabled() && hasSubscribers(event);
+ }
+
+ /**
+ * Indicates if events are enabled for specified event class based on {@link DatabaseDescriptor#diagnosticEventsEnabled()}
+ * and {@link #hasSubscribers(Class, Enum)}.
+ * @param event DiagnosticEvent class implementation
+ * @param eventType Subscribed event type matched against {@link DiagnosticEvent#getType()}
+ */
+ public <E extends DiagnosticEvent, T extends Enum<T>> boolean isEnabled(Class<E> event, T eventType)
+ {
+ return DatabaseDescriptor.diagnosticEventsEnabled() && hasSubscribers(event, eventType);
+ }
+
+ public static DiagnosticEventService instance()
+ {
+ return instance;
+ }
+
+ /**
+ * Removes all active subscribers. Should only be called from testing.
+ */
+ public synchronized void cleanup()
+ {
+ subscribersByClass = ImmutableSetMultimap.of();
+ subscribersAll = ImmutableSet.of();
+ subscribersByClassAndType = ImmutableMap.of();
+ }
+
+ public boolean isDiagnosticsEnabled()
+ {
+ return DatabaseDescriptor.diagnosticEventsEnabled();
+ }
+
+ public void disableDiagnostics()
+ {
+ DatabaseDescriptor.setDiagnosticEventsEnabled(false);
+ }
+
+ public SortedMap<Long, Map<String, Serializable>> readEvents(String eventClazz, Long lastKey, int limit)
+ {
+ return DiagnosticEventPersistence.instance().getEvents(eventClazz, lastKey, limit, false);
+ }
+
+ public void enableEventPersistence(String eventClazz)
+ {
+ DiagnosticEventPersistence.instance().enableEventPersistence(eventClazz);
+ }
+
+ public void disableEventPersistence(String eventClazz)
+ {
+ DiagnosticEventPersistence.instance().disableEventPersistence(eventClazz);
+ }
+
+ /**
+ * Wrapper class for supporting typed event handling for consumers.
+ */
+ private static class TypedConsumerWrapper<E> implements Consumer<DiagnosticEvent>
+ {
+ private final Consumer<E> wrapped;
+
+ private TypedConsumerWrapper(Consumer<E> wrapped)
+ {
+ this.wrapped = wrapped;
+ }
+
+ public void accept(DiagnosticEvent e)
+ {
+ wrapped.accept((E)e);
+ }
+
+ public boolean equals(Object o)
+ {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ TypedConsumerWrapper<?> that = (TypedConsumerWrapper<?>) o;
+ return Objects.equals(wrapped, that.wrapped);
+ }
+
+ public int hashCode()
+ {
+ return Objects.hash(wrapped);
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/diag/DiagnosticEventServiceMBean.java b/src/java/org/apache/cassandra/diag/DiagnosticEventServiceMBean.java
new file mode 100644
index 0000000..b3af8a7
--- /dev/null
+++ b/src/java/org/apache/cassandra/diag/DiagnosticEventServiceMBean.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.diag;
+
+import java.io.Serializable;
+import java.util.Map;
+import java.util.SortedMap;
+
+/**
+ * Provides JMX enabled attributes and operations implemented by {@link DiagnosticEventService}.
+ */
+public interface DiagnosticEventServiceMBean
+{
+ /*
+ * Indicates if any events will be published.
+ */
+ boolean isDiagnosticsEnabled();
+
+ /**
+ * Kill switch for disabling all events immediately, without restarting the node. Please edit cassandra.yaml for
+ * making this permanent.
+ */
+ void disableDiagnostics();
+
+ /**
+ * Retrieved all events of specified type starting with provided key. Result will be sorted chronologically.
+ *
+ * @param eventClazz fqn of event class
+ * @param lastKey ID of first event to retrieve
+ * @param limit number of results to return
+ */
+ SortedMap<Long, Map<String, Serializable>> readEvents(String eventClazz, Long lastKey, int limit);
+
+ /**
+ * Start storing events to make them available via {@link #readEvents(String, Long, int)}.
+ */
+ void enableEventPersistence(String eventClazz);
+
+ /**
+ * Stop storing events.
+ */
+ void disableEventPersistence(String eventClazz);
+}
diff --git a/src/java/org/apache/cassandra/diag/LastEventIdBroadcaster.java b/src/java/org/apache/cassandra/diag/LastEventIdBroadcaster.java
new file mode 100644
index 0000000..8e991e6
--- /dev/null
+++ b/src/java/org/apache/cassandra/diag/LastEventIdBroadcaster.java
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.diag;
+
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
+import javax.management.Notification;
+import javax.management.NotificationBroadcasterSupport;
+import javax.management.NotificationFilter;
+import javax.management.NotificationListener;
+
+import org.apache.cassandra.concurrent.ScheduledExecutors;
+import org.apache.cassandra.utils.MBeanWrapper;
+import org.apache.cassandra.utils.progress.jmx.JMXBroadcastExecutor;
+
+/**
+ * Broadcaster for notifying JMX clients on newly available data. Periodically sends {@link Notification}s
+ * containing a list of event types and greatest event IDs. Consumers may use this information to
+ * query or poll events based on this data.
+ */
+final class LastEventIdBroadcaster extends NotificationBroadcasterSupport implements LastEventIdBroadcasterMBean
+{
+
+ private final static LastEventIdBroadcaster instance = new LastEventIdBroadcaster();
+
+ private final static int PERIODIC_BROADCAST_INTERVAL_MILLIS = 30000;
+ private final static int SHORT_TERM_BROADCAST_DELAY_MILLIS = 1000;
+
+ private final AtomicLong notificationSerialNumber = new AtomicLong();
+ private final AtomicReference<ScheduledFuture<?>> scheduledPeriodicalBroadcast = new AtomicReference<>();
+ private final AtomicReference<ScheduledFuture<?>> scheduledShortTermBroadcast = new AtomicReference<>();
+
+ private final Map<String, Comparable> summary = new ConcurrentHashMap<>();
+
+
+ private LastEventIdBroadcaster()
+ {
+ // use dedicated executor for handling JMX notifications
+ super(JMXBroadcastExecutor.executor);
+
+ summary.put("last_updated_at", 0L);
+
+ MBeanWrapper.instance.registerMBean(this, "org.apache.cassandra.diag:type=LastEventIdBroadcaster");
+ }
+
+ public static LastEventIdBroadcaster instance()
+ {
+ return instance;
+ }
+
+ public Map<String, Comparable> getLastEventIds()
+ {
+ return summary;
+ }
+
+ public Map<String, Comparable> getLastEventIdsIfModified(long lastUpdate)
+ {
+ if (lastUpdate >= (long)summary.get("last_updated_at")) return summary;
+ else return getLastEventIds();
+ }
+
+ public synchronized void addNotificationListener(NotificationListener listener, NotificationFilter filter, Object handback)
+ {
+ super.addNotificationListener(listener, filter, handback);
+
+ // lazily schedule periodical broadcast once we got our first subscriber
+ if (scheduledPeriodicalBroadcast.get() == null)
+ {
+ ScheduledFuture<?> scheduledFuture = ScheduledExecutors.scheduledTasks
+ .scheduleAtFixedRate(this::broadcastEventIds,
+ PERIODIC_BROADCAST_INTERVAL_MILLIS,
+ PERIODIC_BROADCAST_INTERVAL_MILLIS,
+ TimeUnit.MILLISECONDS);
+ if (!this.scheduledPeriodicalBroadcast.compareAndSet(null, scheduledFuture))
+ scheduledFuture.cancel(false);
+ }
+ }
+
+ public void setLastEventId(String key, Comparable id)
+ {
+ // ensure monotonic properties of ids
+ if (summary.compute(key, (k, v) -> v == null ? id : id.compareTo(v) > 0 ? id : v) == id) {
+ summary.put("last_updated_at", System.currentTimeMillis());
+ scheduleBroadcast();
+ }
+ }
+
+ private void scheduleBroadcast()
+ {
+ // schedule broadcast for timely announcing new events before next periodical broadcast
+ // this should allow us to buffer new updates for a while, while keeping broadcasts near-time
+ ScheduledFuture<?> running = scheduledShortTermBroadcast.get();
+ if (running == null || running.isDone())
+ {
+ ScheduledFuture<?> scheduledFuture = ScheduledExecutors.scheduledTasks
+ .schedule((Runnable)this::broadcastEventIds,
+ SHORT_TERM_BROADCAST_DELAY_MILLIS,
+ TimeUnit.MILLISECONDS);
+ if (!this.scheduledShortTermBroadcast.compareAndSet(running, scheduledFuture))
+ scheduledFuture.cancel(false);
+ }
+ }
+
+ private void broadcastEventIds()
+ {
+ if (!summary.isEmpty())
+ broadcastEventIds(summary);
+ }
+
+ private void broadcastEventIds(Map<String, Comparable> summary)
+ {
+ Notification notification = new Notification("event_last_id_summary",
+ "LastEventIdBroadcaster",
+ notificationSerialNumber.incrementAndGet(),
+ System.currentTimeMillis(),
+ "Event last IDs summary");
+ notification.setUserData(summary);
+ sendNotification(notification);
+ }
+}
diff --git a/src/java/org/apache/cassandra/diag/LastEventIdBroadcasterMBean.java b/src/java/org/apache/cassandra/diag/LastEventIdBroadcasterMBean.java
new file mode 100644
index 0000000..03f05dc
--- /dev/null
+++ b/src/java/org/apache/cassandra/diag/LastEventIdBroadcasterMBean.java
@@ -0,0 +1,41 @@
+package org.apache.cassandra.diag;
+
+import java.util.Map;
+
+/**
+ * Provides a list of event types and the corresponding highest event IDs. Consumers may these IDs to determine
+ * if new data is available.
+ *
+ * <p>Example result</p>
+ *
+ * <table>
+ * <tr>
+ * <th>Event</th>
+ * <th>Last ID</th>
+ * </tr>
+ * <tr>
+ * <td>BootstrapEvent</td>
+ * <td>312</td>
+ * </tr>
+ * <tr>
+ * <td>CompactionEvent</td>
+ * <td>a53f9338-5f24-11e8-9c2d-fa7ae01bbebc</td>
+ * </tr>
+ * </table>
+ *
+ * <p>Clients may either retrieve the current list of all events IDs, or make conditional requests for event IDs
+ * based on the timestamp of the last update (much in the sense of e.g. HTTP's If-Modified-Since semantics).</p>
+ */
+public interface LastEventIdBroadcasterMBean
+{
+ /**
+ * Retrieves a list of all event types and their highest IDs.
+ */
+ Map<String, Comparable> getLastEventIds();
+
+ /**
+ * Retrieves a list of all event types and their highest IDs, if updated since specified timestamp, or null.
+ * @param lastUpdate timestamp to use to determine if IDs have been updated
+ */
+ Map<String, Comparable> getLastEventIdsIfModified(long lastUpdate);
+}
diff --git a/src/java/org/apache/cassandra/diag/store/DiagnosticEventMemoryStore.java b/src/java/org/apache/cassandra/diag/store/DiagnosticEventMemoryStore.java
new file mode 100644
index 0000000..92fd42b
--- /dev/null
+++ b/src/java/org/apache/cassandra/diag/store/DiagnosticEventMemoryStore.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.diag.store;
+
+import java.util.Comparator;
+import java.util.Map;
+import java.util.NavigableMap;
+import java.util.concurrent.ConcurrentNavigableMap;
+import java.util.concurrent.ConcurrentSkipListMap;
+import java.util.concurrent.atomic.AtomicLong;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import org.apache.cassandra.diag.DiagnosticEvent;
+
+/**
+ * Simple on-heap memory store that allows to buffer and retrieve a fixed number of events.
+ */
+public final class DiagnosticEventMemoryStore implements DiagnosticEventStore<Long>
+{
+ private final AtomicLong lastKey = new AtomicLong(0);
+
+ private int maxSize = 200;
+
+ // event access will mostly happen based on a recent event offset, so we add new events to the head of the list
+ // for optimized search times
+ private final ConcurrentSkipListMap<Long, DiagnosticEvent> events = new ConcurrentSkipListMap<>(Comparator.reverseOrder());
+
+ public void load()
+ {
+ // no-op
+ }
+
+ public void store(DiagnosticEvent event)
+ {
+ long keyHead = lastKey.incrementAndGet();
+ events.put(keyHead, event);
+
+ // remove elements starting exceeding max size
+ if (keyHead > maxSize) events.tailMap(keyHead - maxSize).clear();
+ }
+
+ public NavigableMap<Long, DiagnosticEvent> scan(Long id, int limit)
+ {
+ assert id != null && id >= 0;
+ assert limit >= 0;
+
+ // [10..1].headMap(2, false): [10..3]
+ ConcurrentNavigableMap<Long, DiagnosticEvent> newerEvents = events.headMap(id, true);
+ // [3..10]
+ ConcurrentNavigableMap<Long, DiagnosticEvent> ret = newerEvents.descendingMap();
+ if (limit == 0)
+ {
+ return ret;
+ }
+ else
+ {
+ Map.Entry<Long, DiagnosticEvent> first = ret.firstEntry();
+ if (first == null) return ret;
+ else return ret.headMap(first.getKey() + limit);
+ }
+ }
+
+ public Long getLastEventId()
+ {
+ return lastKey.get();
+ }
+
+ @VisibleForTesting
+ int size()
+ {
+ return events.size();
+ }
+
+ @VisibleForTesting
+ void setMaxSize(int maxSize)
+ {
+ this.maxSize = maxSize;
+ }
+
+}
diff --git a/src/java/org/apache/cassandra/diag/store/DiagnosticEventStore.java b/src/java/org/apache/cassandra/diag/store/DiagnosticEventStore.java
new file mode 100644
index 0000000..86b2df3
--- /dev/null
+++ b/src/java/org/apache/cassandra/diag/store/DiagnosticEventStore.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.diag.store;
+
+import java.util.NavigableMap;
+
+import org.apache.cassandra.diag.DiagnosticEvent;
+
+/**
+ * Enables storing and retrieving {@link DiagnosticEvent}s.
+ * @param <T> type of key that is used to reference an event
+ */
+public interface DiagnosticEventStore<T extends Comparable<T>>
+{
+ /**
+ * Initializes the store.
+ */
+ void load();
+
+ /**
+ * Stores provided event and returns the new associated store key for it.
+ */
+ void store(DiagnosticEvent event);
+
+ /**
+ * Returns a view on all events with a key greater than the provided value (inclusive) up to the specified
+ * number of results. Events may be added or become unavailable over time. Keys must be unique, sortable and
+ * monotonically incrementing. Returns an empty map in case no events could be found.
+ */
+ NavigableMap<T, DiagnosticEvent> scan(T key, int limit);
+
+ /**
+ * Returns the greatest event ID that can be used to fetch events via {@link #scan(Comparable, int)}.
+ */
+ T getLastEventId();
+}
diff --git a/src/java/org/apache/cassandra/exceptions/CDCWriteException.java b/src/java/org/apache/cassandra/exceptions/CDCWriteException.java
new file mode 100644
index 0000000..d60c1d3
--- /dev/null
+++ b/src/java/org/apache/cassandra/exceptions/CDCWriteException.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.exceptions;
+
+public class CDCWriteException extends RequestExecutionException
+{
+ public CDCWriteException(String msg)
+ {
+ super(ExceptionCode.CDC_WRITE_FAILURE, msg);
+ }
+}
diff --git a/src/java/org/apache/cassandra/exceptions/CasWriteTimeoutException.java b/src/java/org/apache/cassandra/exceptions/CasWriteTimeoutException.java
new file mode 100644
index 0000000..b134764
--- /dev/null
+++ b/src/java/org/apache/cassandra/exceptions/CasWriteTimeoutException.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.exceptions;
+
+import org.apache.cassandra.db.ConsistencyLevel;
+import org.apache.cassandra.db.WriteType;
+
+public class CasWriteTimeoutException extends WriteTimeoutException
+{
+ public final int contentions;
+
+ public CasWriteTimeoutException(WriteType writeType, ConsistencyLevel consistency, int received, int blockFor, int contentions)
+ {
+ super(writeType, consistency, received, blockFor, String.format("CAS operation timed out - encountered contentions: %d", contentions));
+ this.contentions = contentions;
+ }
+}
diff --git a/src/java/org/apache/cassandra/exceptions/CasWriteUnknownResultException.java b/src/java/org/apache/cassandra/exceptions/CasWriteUnknownResultException.java
new file mode 100644
index 0000000..d5dda84
--- /dev/null
+++ b/src/java/org/apache/cassandra/exceptions/CasWriteUnknownResultException.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.exceptions;
+
+import org.apache.cassandra.db.ConsistencyLevel;
+
+public class CasWriteUnknownResultException extends RequestExecutionException
+{
+ public final ConsistencyLevel consistency;
+ public final int received;
+ public final int blockFor;
+
+ public CasWriteUnknownResultException(ConsistencyLevel consistency, int received, int blockFor)
+ {
+ super(ExceptionCode.CAS_WRITE_UNKNOWN, String.format("CAS operation result is unknown - proposal accepted by %d but not a quorum.", received));
+ this.consistency = consistency;
+ this.received = received;
+ this.blockFor = blockFor;
+ }
+}
diff --git a/src/java/org/apache/cassandra/exceptions/ChecksumMismatchException.java b/src/java/org/apache/cassandra/exceptions/ChecksumMismatchException.java
new file mode 100644
index 0000000..a76c46c
--- /dev/null
+++ b/src/java/org/apache/cassandra/exceptions/ChecksumMismatchException.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.exceptions;
+
+import java.io.IOException;
+
+public class ChecksumMismatchException extends IOException
+{
+ public ChecksumMismatchException()
+ {
+ super();
+ }
+
+ public ChecksumMismatchException(String s)
+ {
+ super(s);
+ }
+}
diff --git a/src/java/org/apache/cassandra/exceptions/ExceptionCode.java b/src/java/org/apache/cassandra/exceptions/ExceptionCode.java
index 6ad0577..1766951 100644
--- a/src/java/org/apache/cassandra/exceptions/ExceptionCode.java
+++ b/src/java/org/apache/cassandra/exceptions/ExceptionCode.java
@@ -33,15 +33,17 @@
BAD_CREDENTIALS (0x0100),
// 1xx: problem during request execution
- UNAVAILABLE (0x1000),
- OVERLOADED (0x1001),
- IS_BOOTSTRAPPING(0x1002),
- TRUNCATE_ERROR (0x1003),
- WRITE_TIMEOUT (0x1100),
- READ_TIMEOUT (0x1200),
- READ_FAILURE (0x1300),
- FUNCTION_FAILURE(0x1400),
- WRITE_FAILURE (0x1500),
+ UNAVAILABLE (0x1000),
+ OVERLOADED (0x1001),
+ IS_BOOTSTRAPPING (0x1002),
+ TRUNCATE_ERROR (0x1003),
+ WRITE_TIMEOUT (0x1100),
+ READ_TIMEOUT (0x1200),
+ READ_FAILURE (0x1300),
+ FUNCTION_FAILURE (0x1400),
+ WRITE_FAILURE (0x1500),
+ CDC_WRITE_FAILURE (0x1600),
+ CAS_WRITE_UNKNOWN (0x1700),
// 2xx: problem validating the request
SYNTAX_ERROR (0x2000),
@@ -59,7 +61,7 @@
valueToCode.put(code.value, code);
}
- private ExceptionCode(int value)
+ ExceptionCode(int value)
{
this.value = value;
}
diff --git a/src/java/org/apache/cassandra/exceptions/IncompatibleSchemaException.java b/src/java/org/apache/cassandra/exceptions/IncompatibleSchemaException.java
new file mode 100644
index 0000000..fe3a167
--- /dev/null
+++ b/src/java/org/apache/cassandra/exceptions/IncompatibleSchemaException.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.exceptions;
+
+import java.io.IOException;
+
+public class IncompatibleSchemaException extends IOException
+{
+ public IncompatibleSchemaException(String msg)
+ {
+ super(msg);
+ }
+}
diff --git a/src/java/org/apache/cassandra/exceptions/InvalidRequestException.java b/src/java/org/apache/cassandra/exceptions/InvalidRequestException.java
index 4259d1a..846c38a 100644
--- a/src/java/org/apache/cassandra/exceptions/InvalidRequestException.java
+++ b/src/java/org/apache/cassandra/exceptions/InvalidRequestException.java
@@ -23,4 +23,9 @@
{
super(ExceptionCode.INVALID, msg);
}
+
+ public InvalidRequestException(String msg, Throwable t)
+ {
+ super(ExceptionCode.INVALID, msg, t);
+ }
}
diff --git a/src/java/org/apache/cassandra/exceptions/OperationExecutionException.java b/src/java/org/apache/cassandra/exceptions/OperationExecutionException.java
new file mode 100644
index 0000000..4f9ffa4
--- /dev/null
+++ b/src/java/org/apache/cassandra/exceptions/OperationExecutionException.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.exceptions;
+
+import java.util.List;
+
+import org.apache.cassandra.db.marshal.AbstractType;
+
+/**
+ * Thrown when an operation problem has occured (e.g. division by zero with integer).
+ */
+public final class OperationExecutionException extends RequestExecutionException
+{
+
+ /**
+ * Creates a new <code>OperationExecutionException</code> for the specified operation.
+ *
+ * @param operator the operator
+ * @param argTypes the argument types
+ * @param e the original Exception
+ * @return a new <code>OperationExecutionException</code> for the specified operation
+ */
+ public static OperationExecutionException create(char operator, List<AbstractType<?>> argTypes, Exception e)
+ {
+ List<String> cqlTypes = AbstractType.asCQLTypeStringList(argTypes);
+ return new OperationExecutionException(String.format("the operation '%s %s %s' failed: %s",
+ cqlTypes.get(0),
+ operator,
+ cqlTypes.get(1),
+ e.getMessage()));
+ }
+
+ /**
+ * Creates an <code>OperationExecutionException</code> with the specified message.
+ * @param msg the error message
+ */
+ public OperationExecutionException(String msg)
+ {
+ super(ExceptionCode.FUNCTION_FAILURE, msg);
+ }
+
+}
diff --git a/src/java/org/apache/cassandra/exceptions/ReadFailureException.java b/src/java/org/apache/cassandra/exceptions/ReadFailureException.java
index 82885e3..744cad4 100644
--- a/src/java/org/apache/cassandra/exceptions/ReadFailureException.java
+++ b/src/java/org/apache/cassandra/exceptions/ReadFailureException.java
@@ -17,18 +17,20 @@
*/
package org.apache.cassandra.exceptions;
-import java.net.InetAddress;
import java.util.Map;
+import com.google.common.collect.ImmutableMap;
+
import org.apache.cassandra.db.ConsistencyLevel;
+import org.apache.cassandra.locator.InetAddressAndPort;
public class ReadFailureException extends RequestFailureException
{
public final boolean dataPresent;
- public ReadFailureException(ConsistencyLevel consistency, int received, int blockFor, boolean dataPresent, Map<InetAddress, RequestFailureReason> failureReasonByEndpoint)
+ public ReadFailureException(ConsistencyLevel consistency, int received, int blockFor, boolean dataPresent, Map<InetAddressAndPort, RequestFailureReason> failureReasonByEndpoint)
{
- super(ExceptionCode.READ_FAILURE, consistency, received, blockFor, failureReasonByEndpoint);
+ super(ExceptionCode.READ_FAILURE, consistency, received, blockFor, ImmutableMap.copyOf(failureReasonByEndpoint));
this.dataPresent = dataPresent;
}
}
diff --git a/src/java/org/apache/cassandra/exceptions/RepairException.java b/src/java/org/apache/cassandra/exceptions/RepairException.java
index 2f5f2c1..db219a2 100644
--- a/src/java/org/apache/cassandra/exceptions/RepairException.java
+++ b/src/java/org/apache/cassandra/exceptions/RepairException.java
@@ -18,6 +18,7 @@
package org.apache.cassandra.exceptions;
import org.apache.cassandra.repair.RepairJobDesc;
+import org.apache.cassandra.streaming.PreviewKind;
/**
* Exception thrown during repair
@@ -25,22 +26,23 @@
public class RepairException extends Exception
{
public final RepairJobDesc desc;
+ public final PreviewKind previewKind;
public RepairException(RepairJobDesc desc, String message)
{
- super(message);
- this.desc = desc;
+ this(desc, null, message);
}
- public RepairException(RepairJobDesc desc, String message, Throwable cause)
+ public RepairException(RepairJobDesc desc, PreviewKind previewKind, String message)
{
- super(message, cause);
+ super(message);
this.desc = desc;
+ this.previewKind = previewKind != null ? previewKind : PreviewKind.NONE;
}
@Override
public String getMessage()
{
- return desc + " " + super.getMessage();
+ return desc.toString(previewKind) + ' ' + super.getMessage();
}
}
diff --git a/src/java/org/apache/cassandra/exceptions/RequestFailureException.java b/src/java/org/apache/cassandra/exceptions/RequestFailureException.java
index 1a5289c..56cee1a 100644
--- a/src/java/org/apache/cassandra/exceptions/RequestFailureException.java
+++ b/src/java/org/apache/cassandra/exceptions/RequestFailureException.java
@@ -17,32 +17,40 @@
*/
package org.apache.cassandra.exceptions;
-import java.net.InetAddress;
-import java.util.HashMap;
import java.util.Map;
+import java.util.stream.Collectors;
import org.apache.cassandra.db.ConsistencyLevel;
+import org.apache.cassandra.locator.InetAddressAndPort;
public class RequestFailureException extends RequestExecutionException
{
public final ConsistencyLevel consistency;
public final int received;
public final int blockFor;
- public final Map<InetAddress, RequestFailureReason> failureReasonByEndpoint;
+ public final Map<InetAddressAndPort, RequestFailureReason> failureReasonByEndpoint;
- protected RequestFailureException(ExceptionCode code, ConsistencyLevel consistency, int received, int blockFor, Map<InetAddress, RequestFailureReason> failureReasonByEndpoint)
+ protected RequestFailureException(ExceptionCode code, ConsistencyLevel consistency, int received, int blockFor, Map<InetAddressAndPort, RequestFailureReason> failureReasonByEndpoint)
{
- super(code, String.format("Operation failed - received %d responses and %d failures", received, failureReasonByEndpoint.size()));
+ super(code, buildErrorMessage(received, failureReasonByEndpoint));
this.consistency = consistency;
this.received = received;
this.blockFor = blockFor;
+ this.failureReasonByEndpoint = failureReasonByEndpoint;
+ }
- // It is possible for the passed in failureReasonByEndpoint map
- // to have new entries added after this exception is constructed
- // (e.g. a delayed failure response from a replica). So to be safe
- // we make a copy of the map at this point to ensure it will not be
- // modified any further. Otherwise, there could be implications when
- // we encode this map for transport.
- this.failureReasonByEndpoint = new HashMap<>(failureReasonByEndpoint);
+ private static String buildErrorMessage(int received, Map<InetAddressAndPort, RequestFailureReason> failures)
+ {
+ return String.format("Operation failed - received %d responses and %d failures: %s",
+ received,
+ failures.size(),
+ buildFailureString(failures));
+ }
+
+ private static String buildFailureString(Map<InetAddressAndPort, RequestFailureReason> failures)
+ {
+ return failures.entrySet().stream()
+ .map(e -> String.format("%s from %s", e.getValue(), e.getKey()))
+ .collect(Collectors.joining(", "));
}
}
diff --git a/src/java/org/apache/cassandra/exceptions/RequestFailureReason.java b/src/java/org/apache/cassandra/exceptions/RequestFailureReason.java
index 96ab7b5..1cdbdb5 100644
--- a/src/java/org/apache/cassandra/exceptions/RequestFailureReason.java
+++ b/src/java/org/apache/cassandra/exceptions/RequestFailureReason.java
@@ -15,37 +15,101 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.cassandra.exceptions;
+import java.io.IOException;
+
+import com.google.common.primitives.Ints;
+
+import org.apache.cassandra.db.filter.TombstoneOverwhelmingException;
+import org.apache.cassandra.io.IVersionedSerializer;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.utils.vint.VIntCoding;
+
+import static java.lang.Math.max;
+import static org.apache.cassandra.net.MessagingService.VERSION_40;
+
public enum RequestFailureReason
{
- /**
- * The reason for the failure was none of the below reasons or was not recorded by the data node.
- */
- UNKNOWN (0x0000),
+ UNKNOWN (0),
+ READ_TOO_MANY_TOMBSTONES (1),
+ TIMEOUT (2),
+ INCOMPATIBLE_SCHEMA (3);
- /**
- * The data node read too many tombstones when attempting to execute a read query (see tombstone_failure_threshold).
- */
- READ_TOO_MANY_TOMBSTONES (0x0001);
+ public static final Serializer serializer = new Serializer();
- /** The code to be serialized as an unsigned 16 bit integer */
public final int code;
- public static final RequestFailureReason[] VALUES = values();
- RequestFailureReason(final int code)
+ RequestFailureReason(int code)
{
this.code = code;
}
- public static RequestFailureReason fromCode(final int code)
+ private static final RequestFailureReason[] codeToReasonMap;
+
+ static
{
- for (RequestFailureReason reasonCode : VALUES)
+ RequestFailureReason[] reasons = values();
+
+ int max = -1;
+ for (RequestFailureReason r : reasons)
+ max = max(r.code, max);
+
+ RequestFailureReason[] codeMap = new RequestFailureReason[max + 1];
+
+ for (RequestFailureReason reason : reasons)
{
- if (reasonCode.code == code)
- return reasonCode;
+ if (codeMap[reason.code] != null)
+ throw new RuntimeException("Two RequestFailureReason-s that map to the same code: " + reason.code);
+ codeMap[reason.code] = reason;
}
- throw new IllegalArgumentException("Unknown request failure reason error code: " + code);
+
+ codeToReasonMap = codeMap;
+ }
+
+ public static RequestFailureReason fromCode(int code)
+ {
+ if (code < 0)
+ throw new IllegalArgumentException("RequestFailureReason code must be non-negative (got " + code + ')');
+
+ // be forgiving and return UNKNOWN if we aren't aware of the code - for forward compatibility
+ return code < codeToReasonMap.length ? codeToReasonMap[code] : UNKNOWN;
+ }
+
+ public static RequestFailureReason forException(Throwable t)
+ {
+ if (t instanceof TombstoneOverwhelmingException)
+ return READ_TOO_MANY_TOMBSTONES;
+
+ if (t instanceof IncompatibleSchemaException)
+ return INCOMPATIBLE_SCHEMA;
+
+ return UNKNOWN;
+ }
+
+ public static final class Serializer implements IVersionedSerializer<RequestFailureReason>
+ {
+ private Serializer()
+ {
+ }
+
+ public void serialize(RequestFailureReason reason, DataOutputPlus out, int version) throws IOException
+ {
+ if (version < VERSION_40)
+ out.writeShort(reason.code);
+ else
+ out.writeUnsignedVInt(reason.code);
+ }
+
+ public RequestFailureReason deserialize(DataInputPlus in, int version) throws IOException
+ {
+ return fromCode(version < VERSION_40 ? in.readUnsignedShort() : Ints.checkedCast(in.readUnsignedVInt()));
+ }
+
+ public long serializedSize(RequestFailureReason reason, int version)
+ {
+ return version < VERSION_40 ? 2 : VIntCoding.computeVIntSize(reason.code);
+ }
}
}
diff --git a/src/java/org/apache/cassandra/exceptions/RequestTimeoutException.java b/src/java/org/apache/cassandra/exceptions/RequestTimeoutException.java
index 156dce7..853ba2f 100644
--- a/src/java/org/apache/cassandra/exceptions/RequestTimeoutException.java
+++ b/src/java/org/apache/cassandra/exceptions/RequestTimeoutException.java
@@ -32,4 +32,12 @@
this.received = received;
this.blockFor = blockFor;
}
+
+ protected RequestTimeoutException(ExceptionCode code, ConsistencyLevel consistency, int received, int blockFor, String msg)
+ {
+ super(code, msg);
+ this.consistency = consistency;
+ this.received = received;
+ this.blockFor = blockFor;
+ }
}
diff --git a/src/java/org/apache/cassandra/exceptions/UnavailableException.java b/src/java/org/apache/cassandra/exceptions/UnavailableException.java
index 7b4edd8..d6e8488 100644
--- a/src/java/org/apache/cassandra/exceptions/UnavailableException.java
+++ b/src/java/org/apache/cassandra/exceptions/UnavailableException.java
@@ -25,14 +25,26 @@
public final int required;
public final int alive;
- public UnavailableException(ConsistencyLevel consistency, int required, int alive)
+ public static UnavailableException create(ConsistencyLevel consistency, int required, int alive)
{
- this("Cannot achieve consistency level " + consistency, consistency, required, alive);
+ assert alive < required;
+ return create(consistency, required, 0, alive, 0);
}
- public UnavailableException(ConsistencyLevel consistency, String dc, int required, int alive)
+ public static UnavailableException create(ConsistencyLevel consistency, int required, int requiredFull, int alive, int aliveFull)
{
- this("Cannot achieve consistency level " + consistency + " in DC " + dc, consistency, required, alive);
+ if (required > alive)
+ return new UnavailableException("Cannot achieve consistency level " + consistency, consistency, required, alive);
+ assert requiredFull < aliveFull;
+ return new UnavailableException("Insufficient full replicas", consistency, required, alive);
+ }
+
+ public static UnavailableException create(ConsistencyLevel consistency, String dc, int required, int requiredFull, int alive, int aliveFull)
+ {
+ if (required > alive)
+ return new UnavailableException("Cannot achieve consistency level " + consistency + " in DC " + dc, consistency, required, alive);
+ assert requiredFull < aliveFull;
+ return new UnavailableException("Insufficient full replicas in DC " + dc, consistency, required, alive);
}
public UnavailableException(String msg, ConsistencyLevel consistency, int required, int alive)
diff --git a/src/java/org/apache/cassandra/exceptions/UnknownColumnException.java b/src/java/org/apache/cassandra/exceptions/UnknownColumnException.java
new file mode 100644
index 0000000..93a464e
--- /dev/null
+++ b/src/java/org/apache/cassandra/exceptions/UnknownColumnException.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.exceptions;
+
+public final class UnknownColumnException extends IncompatibleSchemaException
+{
+ public UnknownColumnException(String msg)
+ {
+ super(msg);
+ }
+}
diff --git a/src/java/org/apache/cassandra/exceptions/UnknownIndexException.java b/src/java/org/apache/cassandra/exceptions/UnknownIndexException.java
new file mode 100644
index 0000000..fdc6840
--- /dev/null
+++ b/src/java/org/apache/cassandra/exceptions/UnknownIndexException.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.exceptions;
+
+import java.io.IOException;
+import java.util.UUID;
+
+import org.apache.cassandra.schema.TableMetadata;
+
+/**
+ * Exception thrown when we read an index id from a serialized ReadCommand and no corresponding IndexMetadata
+ * can be found in the TableMetadata#indexes collection. Note that this is an internal exception and is not meant
+ * to be user facing, the node reading the ReadCommand should proceed as if no index id were present.
+ */
+public final class UnknownIndexException extends IOException
+{
+ public final UUID indexId;
+ public UnknownIndexException(TableMetadata metadata, UUID id)
+ {
+ super(String.format("Unknown index %s for table %s", id.toString(), metadata.toString()));
+ indexId = id;
+ }
+}
diff --git a/src/java/org/apache/cassandra/exceptions/UnknownTableException.java b/src/java/org/apache/cassandra/exceptions/UnknownTableException.java
new file mode 100644
index 0000000..3e9c775
--- /dev/null
+++ b/src/java/org/apache/cassandra/exceptions/UnknownTableException.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.exceptions;
+
+import org.apache.cassandra.schema.TableId;
+
+public class UnknownTableException extends IncompatibleSchemaException
+{
+ public final TableId id;
+
+ public UnknownTableException(String msg, TableId id)
+ {
+ super(msg);
+ this.id = id;
+ }
+}
diff --git a/src/java/org/apache/cassandra/exceptions/WriteFailureException.java b/src/java/org/apache/cassandra/exceptions/WriteFailureException.java
index 1a857fe..bcda9e6 100644
--- a/src/java/org/apache/cassandra/exceptions/WriteFailureException.java
+++ b/src/java/org/apache/cassandra/exceptions/WriteFailureException.java
@@ -17,19 +17,21 @@
*/
package org.apache.cassandra.exceptions;
-import java.net.InetAddress;
import java.util.Map;
+import com.google.common.collect.ImmutableMap;
+
import org.apache.cassandra.db.ConsistencyLevel;
import org.apache.cassandra.db.WriteType;
+import org.apache.cassandra.locator.InetAddressAndPort;
public class WriteFailureException extends RequestFailureException
{
public final WriteType writeType;
- public WriteFailureException(ConsistencyLevel consistency, int received, int blockFor, WriteType writeType, Map<InetAddress, RequestFailureReason> failureReasonByEndpoint)
+ public WriteFailureException(ConsistencyLevel consistency, int received, int blockFor, WriteType writeType, Map<InetAddressAndPort, RequestFailureReason> failureReasonByEndpoint)
{
- super(ExceptionCode.WRITE_FAILURE, consistency, received, blockFor, failureReasonByEndpoint);
+ super(ExceptionCode.WRITE_FAILURE, consistency, received, blockFor, ImmutableMap.copyOf(failureReasonByEndpoint));
this.writeType = writeType;
}
}
diff --git a/src/java/org/apache/cassandra/exceptions/WriteTimeoutException.java b/src/java/org/apache/cassandra/exceptions/WriteTimeoutException.java
index af8d42b..4b4ce38 100644
--- a/src/java/org/apache/cassandra/exceptions/WriteTimeoutException.java
+++ b/src/java/org/apache/cassandra/exceptions/WriteTimeoutException.java
@@ -29,4 +29,10 @@
super(ExceptionCode.WRITE_TIMEOUT, consistency, received, blockFor);
this.writeType = writeType;
}
+
+ public WriteTimeoutException(WriteType writeType, ConsistencyLevel consistency, int received, int blockFor, String msg)
+ {
+ super(ExceptionCode.WRITE_TIMEOUT, consistency, received, blockFor, msg);
+ this.writeType = writeType;
+ }
}
diff --git a/src/java/org/apache/cassandra/fql/FullQueryLogger.java b/src/java/org/apache/cassandra/fql/FullQueryLogger.java
new file mode 100644
index 0000000..6c38166
--- /dev/null
+++ b/src/java/org/apache/cassandra/fql/FullQueryLogger.java
@@ -0,0 +1,482 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.fql;
+
+import java.io.File;
+import java.nio.ByteBuffer;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+
+import javax.annotation.Nullable;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Sets;
+import com.google.common.primitives.Ints;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import io.netty.buffer.ByteBuf;
+import net.openhft.chronicle.bytes.BytesStore;
+import net.openhft.chronicle.wire.ValueOut;
+import net.openhft.chronicle.wire.WireOut;
+import org.apache.cassandra.cql3.CQLStatement;
+import org.apache.cassandra.cql3.QueryEvents;
+import org.apache.cassandra.cql3.QueryOptions;
+import org.apache.cassandra.cql3.statements.BatchStatement;
+import org.apache.cassandra.service.QueryState;
+import org.apache.cassandra.transport.CBUtil;
+import org.apache.cassandra.transport.Message;
+import org.apache.cassandra.utils.ObjectSizes;
+import org.apache.cassandra.utils.binlog.BinLog;
+import org.apache.cassandra.utils.concurrent.WeightedQueue;
+import org.github.jamm.MemoryLayoutSpecification;
+
+/**
+ * A logger that logs entire query contents after the query finishes (or times out).
+ */
+public class FullQueryLogger implements QueryEvents.Listener
+{
+ protected static final Logger logger = LoggerFactory.getLogger(FullQueryLogger.class);
+
+ public static final long CURRENT_VERSION = 0; // encode a dummy version, to prevent pain in decoding in the future
+
+ public static final String VERSION = "version";
+ public static final String TYPE = "type";
+
+ public static final String PROTOCOL_VERSION = "protocol-version";
+ public static final String QUERY_OPTIONS = "query-options";
+ public static final String QUERY_START_TIME = "query-start-time";
+
+ public static final String GENERATED_TIMESTAMP = "generated-timestamp";
+ public static final String GENERATED_NOW_IN_SECONDS = "generated-now-in-seconds";
+ public static final String KEYSPACE = "keyspace";
+
+ public static final String BATCH = "batch";
+ public static final String SINGLE_QUERY = "single-query";
+
+ public static final String QUERY = "query";
+ public static final String BATCH_TYPE = "batch-type";
+ public static final String QUERIES = "queries";
+ public static final String VALUES = "values";
+
+ private static final int EMPTY_BYTEBUFFER_SIZE = Ints.checkedCast(ObjectSizes.sizeOnHeapExcludingData(ByteBuffer.allocate(0)));
+
+ private static final int EMPTY_LIST_SIZE = Ints.checkedCast(ObjectSizes.measureDeep(new ArrayList(0)));
+ private static final int EMPTY_BYTEBUF_SIZE;
+
+ private static final int OBJECT_HEADER_SIZE = MemoryLayoutSpecification.SPEC.getObjectHeaderSize();
+ private static final int OBJECT_REFERENCE_SIZE = MemoryLayoutSpecification.SPEC.getReferenceSize();
+
+ public static final FullQueryLogger instance = new FullQueryLogger();
+
+ volatile BinLog binLog;
+
+ public synchronized void enable(Path path, String rollCycle, boolean blocking, int maxQueueWeight, long maxLogSize, String archiveCommand, int maxArchiveRetries)
+ {
+ if (this.binLog != null)
+ throw new IllegalStateException("Binlog is already configured");
+ this.binLog = new BinLog.Builder().path(path)
+ .rollCycle(rollCycle)
+ .blocking(blocking)
+ .maxQueueWeight(maxQueueWeight)
+ .maxLogSize(maxLogSize)
+ .archiveCommand(archiveCommand)
+ .maxArchiveRetries(maxArchiveRetries)
+ .build(true);
+ QueryEvents.instance.registerListener(this);
+ }
+
+ static
+ {
+ ByteBuf buf = CBUtil.allocator.buffer(0, 0);
+ try
+ {
+ EMPTY_BYTEBUF_SIZE = Ints.checkedCast(ObjectSizes.measure(buf));
+ }
+ finally
+ {
+ buf.release();
+ }
+ }
+
+ public synchronized void stop()
+ {
+ try
+ {
+ BinLog binLog = this.binLog;
+ if (binLog != null)
+ {
+ logger.info("Stopping full query logging to {}", binLog.path);
+ binLog.stop();
+ }
+ else
+ {
+ logger.info("Full query log already stopped");
+ }
+ }
+ catch (InterruptedException e)
+ {
+ throw new RuntimeException(e);
+ }
+ finally
+ {
+ QueryEvents.instance.unregisterListener(this);
+ this.binLog = null;
+ }
+ }
+
+ /**
+ * Need the path as a parameter as well because if the process is restarted the config file might be the only
+ * location for retrieving the path to the full query log files, but JMX also allows you to specify a path
+ * that isn't persisted anywhere so we have to clean that one as well.
+ */
+ public synchronized void reset(String fullQueryLogPath)
+ {
+ try
+ {
+ Set<File> pathsToClean = Sets.newHashSet();
+
+ //First decide whether to clean the path configured in the YAML
+ if (fullQueryLogPath != null)
+ {
+ File fullQueryLogPathFile = new File(fullQueryLogPath);
+ if (fullQueryLogPathFile.exists())
+ {
+ pathsToClean.add(fullQueryLogPathFile);
+ }
+ }
+
+ //Then decide whether to clean the last used path, possibly configured by JMX
+ if (binLog != null && binLog.path != null)
+ {
+ File pathFile = binLog.path.toFile();
+ if (pathFile.exists())
+ {
+ pathsToClean.add(pathFile);
+ }
+ }
+
+ logger.info("Reset (and deactivation) of full query log requested.");
+ if (binLog != null)
+ {
+ logger.info("Stopping full query log. Cleaning {}.", pathsToClean);
+ binLog.stop();
+ binLog = null;
+ }
+ else
+ {
+ logger.info("Full query log already deactivated. Cleaning {}.", pathsToClean);
+ }
+
+ Throwable accumulate = null;
+ for (File f : pathsToClean)
+ {
+ accumulate = BinLog.cleanDirectory(f, accumulate);
+ }
+ if (accumulate != null)
+ {
+ throw new RuntimeException(accumulate);
+ }
+ }
+ catch (Exception e)
+ {
+ if (e instanceof RuntimeException)
+ {
+ throw (RuntimeException)e;
+ }
+ throw new RuntimeException(e);
+ }
+ finally
+ {
+ QueryEvents.instance.unregisterListener(this);
+ }
+ }
+
+ public boolean isEnabled()
+ {
+ return this.binLog != null;
+ }
+
+ /**
+ * Log an invocation of a batch of queries
+ * @param type The type of the batch
+ * @param statements the prepared cql statements (unused here)
+ * @param queries CQL text of the queries
+ * @param values Values to bind to as parameters for the queries
+ * @param queryOptions Options associated with the query invocation
+ * @param queryState Timestamp state associated with the query invocation
+ * @param batchTimeMillis Approximate time in milliseconds since the epoch since the batch was invoked
+ * @param response the response from the batch query
+ */
+ public void batchSuccess(BatchStatement.Type type,
+ List<? extends CQLStatement> statements,
+ List<String> queries,
+ List<List<ByteBuffer>> values,
+ QueryOptions queryOptions,
+ QueryState queryState,
+ long batchTimeMillis,
+ Message.Response response)
+ {
+ Preconditions.checkNotNull(type, "type was null");
+ Preconditions.checkNotNull(queries, "queries was null");
+ Preconditions.checkNotNull(values, "value was null");
+ Preconditions.checkNotNull(queryOptions, "queryOptions was null");
+ Preconditions.checkNotNull(queryState, "queryState was null");
+ Preconditions.checkArgument(batchTimeMillis > 0, "batchTimeMillis must be > 0");
+
+ //Don't construct the wrapper if the log is disabled
+ BinLog binLog = this.binLog;
+ if (binLog == null)
+ {
+ return;
+ }
+
+ Batch wrappedBatch = new Batch(type, queries, values, queryOptions, queryState, batchTimeMillis);
+ binLog.logRecord(wrappedBatch);
+ }
+
+ /**
+ * Log a single CQL query
+ * @param query CQL query text
+ * @param queryOptions Options associated with the query invocation
+ * @param queryState Timestamp state associated with the query invocation
+ * @param queryTimeMillis Approximate time in milliseconds since the epoch since the batch was invoked
+ * @param response the response from this query
+ */
+ public void querySuccess(CQLStatement statement,
+ String query,
+ QueryOptions queryOptions,
+ QueryState queryState,
+ long queryTimeMillis,
+ Message.Response response)
+ {
+ Preconditions.checkNotNull(query, "query was null");
+ Preconditions.checkNotNull(queryOptions, "queryOptions was null");
+ Preconditions.checkNotNull(queryState, "queryState was null");
+ Preconditions.checkArgument(queryTimeMillis > 0, "queryTimeMillis must be > 0");
+
+ //Don't construct the wrapper if the log is disabled
+ BinLog binLog = this.binLog;
+ if (binLog == null)
+ return;
+
+ Query wrappedQuery = new Query(query, queryOptions, queryState, queryTimeMillis);
+ binLog.logRecord(wrappedQuery);
+ }
+
+ public void executeSuccess(CQLStatement statement, String query, QueryOptions options, QueryState state, long queryTime, Message.Response response)
+ {
+ querySuccess(statement, query, options, state, queryTime, response);
+ }
+
+ public static class Query extends AbstractLogEntry
+ {
+ private final String query;
+
+ public Query(String query, QueryOptions queryOptions, QueryState queryState, long queryStartTime)
+ {
+ super(queryOptions, queryState, queryStartTime);
+ this.query = query;
+ }
+
+ @Override
+ protected String type()
+ {
+ return SINGLE_QUERY;
+ }
+
+ @Override
+ public void writeMarshallablePayload(WireOut wire)
+ {
+ super.writeMarshallablePayload(wire);
+ wire.write(QUERY).text(query);
+ }
+
+ @Override
+ public int weight()
+ {
+ return Ints.checkedCast(ObjectSizes.sizeOf(query)) + super.weight();
+ }
+ }
+
+ public static class Batch extends AbstractLogEntry
+ {
+ private final int weight;
+ private final BatchStatement.Type batchType;
+ private final List<String> queries;
+ private final List<List<ByteBuffer>> values;
+
+ public Batch(BatchStatement.Type batchType,
+ List<String> queries,
+ List<List<ByteBuffer>> values,
+ QueryOptions queryOptions,
+ QueryState queryState,
+ long batchTimeMillis)
+ {
+ super(queryOptions, queryState, batchTimeMillis);
+
+ this.queries = queries;
+ this.values = values;
+ this.batchType = batchType;
+
+ int weight = super.weight();
+
+ // weight, queries, values, batch type
+ weight += 4 + // cached weight
+ 2 * EMPTY_LIST_SIZE + // queries + values lists
+ OBJECT_REFERENCE_SIZE; // batchType reference, worst case
+
+ for (String query : queries)
+ weight += ObjectSizes.sizeOf(query);
+
+ for (List<ByteBuffer> subValues : values)
+ {
+ weight += EMPTY_LIST_SIZE;
+ for (ByteBuffer value : subValues)
+ weight += EMPTY_BYTEBUFFER_SIZE + value.capacity();
+ }
+
+ this.weight = weight;
+ }
+
+ @Override
+ protected String type()
+ {
+ return BATCH;
+ }
+
+ @Override
+ public void writeMarshallablePayload(WireOut wire)
+ {
+ super.writeMarshallablePayload(wire);
+ wire.write(BATCH_TYPE).text(batchType.name());
+ ValueOut valueOut = wire.write(QUERIES);
+ valueOut.int32(queries.size());
+ for (String query : queries)
+ {
+ valueOut.text(query);
+ }
+ valueOut = wire.write(VALUES);
+ valueOut.int32(values.size());
+ for (List<ByteBuffer> subValues : values)
+ {
+ valueOut.int32(subValues.size());
+ for (ByteBuffer value : subValues)
+ {
+ valueOut.bytes(BytesStore.wrap(value));
+ }
+ }
+ }
+
+ @Override
+ public int weight()
+ {
+ return weight;
+ }
+ }
+
+ private static abstract class AbstractLogEntry extends BinLog.ReleaseableWriteMarshallable implements WeightedQueue.Weighable
+ {
+ private final long queryStartTime;
+ private final int protocolVersion;
+ private final ByteBuf queryOptionsBuffer;
+
+ private final long generatedTimestamp;
+ private final int generatedNowInSeconds;
+ @Nullable
+ private final String keyspace;
+
+ AbstractLogEntry(QueryOptions queryOptions, QueryState queryState, long queryStartTime)
+ {
+ this.queryStartTime = queryStartTime;
+
+ this.protocolVersion = queryOptions.getProtocolVersion().asInt();
+ int optionsSize = QueryOptions.codec.encodedSize(queryOptions, queryOptions.getProtocolVersion());
+ queryOptionsBuffer = CBUtil.allocator.buffer(optionsSize, optionsSize);
+
+ this.generatedTimestamp = queryState.generatedTimestamp();
+ this.generatedNowInSeconds = queryState.generatedNowInSeconds();
+ this.keyspace = queryState.getClientState().getRawKeyspace();
+
+ /*
+ * Struggled with what tradeoff to make in terms of query options which is potentially large and complicated
+ * There is tension between low garbage production (or allocator overhead), small working set size, and CPU overhead reserializing the
+ * query options into binary format.
+ *
+ * I went with the lowest risk most predictable option which is allocator overhead and CPU overhead
+ * rather then keep the original query message around so I could just serialize that as a memcpy. It's more
+ * instructions when turned on, but it doesn't change memory footprint quite as much and it's more pay for what you use
+ * in terms of query volume. The CPU overhead is spread out across producers so we should at least get
+ * some scaling.
+ *
+ */
+ try
+ {
+ QueryOptions.codec.encode(queryOptions, queryOptionsBuffer, queryOptions.getProtocolVersion());
+ }
+ catch (Throwable e)
+ {
+ queryOptionsBuffer.release();
+ throw e;
+ }
+ }
+
+ @Override
+ protected long version()
+ {
+ return CURRENT_VERSION;
+ }
+
+ @Override
+ public void writeMarshallablePayload(WireOut wire)
+ {
+ wire.write(QUERY_START_TIME).int64(queryStartTime);
+ wire.write(PROTOCOL_VERSION).int32(protocolVersion);
+ wire.write(QUERY_OPTIONS).bytes(BytesStore.wrap(queryOptionsBuffer.nioBuffer()));
+
+ wire.write(GENERATED_TIMESTAMP).int64(generatedTimestamp);
+ wire.write(GENERATED_NOW_IN_SECONDS).int32(generatedNowInSeconds);
+
+ wire.write(KEYSPACE).text(keyspace);
+ }
+
+ @Override
+ public void release()
+ {
+ queryOptionsBuffer.release();
+ }
+
+ @Override
+ public int weight()
+ {
+ return OBJECT_HEADER_SIZE
+ + 8 // queryStartTime
+ + 4 // protocolVersion
+ + EMPTY_BYTEBUF_SIZE + queryOptionsBuffer.capacity() // queryOptionsBuffer
+ + 8 // generatedTimestamp
+ + 4 // generatedNowInSeconds
+ + (keyspace != null
+ ? Ints.checkedCast(ObjectSizes.sizeOf(keyspace)) // keyspace
+ : OBJECT_REFERENCE_SIZE); // null
+ }
+ }
+
+}
diff --git a/src/java/org/apache/cassandra/fql/FullQueryLoggerOptions.java b/src/java/org/apache/cassandra/fql/FullQueryLoggerOptions.java
new file mode 100644
index 0000000..c54169c
--- /dev/null
+++ b/src/java/org/apache/cassandra/fql/FullQueryLoggerOptions.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.fql;
+
+import org.apache.commons.lang3.StringUtils;
+
+import org.apache.cassandra.utils.binlog.BinLogOptions;
+
+public class FullQueryLoggerOptions extends BinLogOptions
+{
+ public String log_dir = StringUtils.EMPTY;
+
+ public String toString()
+ {
+ return "FullQueryLoggerOptions{" +
+ "log_dir='" + log_dir + '\'' +
+ ", archive_command='" + archive_command + '\'' +
+ ", roll_cycle='" + roll_cycle + '\'' +
+ ", block=" + block +
+ ", max_queue_weight=" + max_queue_weight +
+ ", max_log_size=" + max_log_size +
+ '}';
+ }
+}
diff --git a/src/java/org/apache/cassandra/gms/ApplicationState.java b/src/java/org/apache/cassandra/gms/ApplicationState.java
index ade9208..211387d 100644
--- a/src/java/org/apache/cassandra/gms/ApplicationState.java
+++ b/src/java/org/apache/cassandra/gms/ApplicationState.java
@@ -19,15 +19,15 @@
public enum ApplicationState
{
- STATUS,
+ @Deprecated STATUS, //Deprecated and unsued in 4.0, stop publishing in 5.0, reclaim in 6.0
LOAD,
SCHEMA,
DC,
RACK,
RELEASE_VERSION,
REMOVAL_COORDINATOR,
- INTERNAL_IP,
- RPC_ADDRESS,
+ @Deprecated INTERNAL_IP, //Deprecated and unused in 4.0, stop publishing in 5.0, reclaim in 6.0
+ @Deprecated RPC_ADDRESS, // ^ Same
X_11_PADDING, // padding specifically for 1.1
SEVERITY,
NET_VERSION,
@@ -35,8 +35,9 @@
TOKENS,
RPC_READY,
// pad to allow adding new states to existing cluster
- X1,
- X2,
+ INTERNAL_ADDRESS_AND_PORT, //Replacement for INTERNAL_IP with up to two ports
+ NATIVE_ADDRESS_AND_PORT, //Replacement for RPC_ADDRESS
+ STATUS_WITH_PORT, //Replacement for STATUS
X3,
X4,
X5,
diff --git a/src/java/org/apache/cassandra/gms/EchoMessage.java b/src/java/org/apache/cassandra/gms/EchoMessage.java
deleted file mode 100644
index 2fee889..0000000
--- a/src/java/org/apache/cassandra/gms/EchoMessage.java
+++ /dev/null
@@ -1,56 +0,0 @@
-package org.apache.cassandra.gms;
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
-
-import java.io.IOException;
-
-import org.apache.cassandra.io.IVersionedSerializer;
-import org.apache.cassandra.io.util.DataInputPlus;
-import org.apache.cassandra.io.util.DataOutputPlus;
-
-public final class EchoMessage
-{
- public static final EchoMessage instance = new EchoMessage();
-
- public static final IVersionedSerializer<EchoMessage> serializer = new EchoMessageSerializer();
-
- private EchoMessage()
- {
- }
-
- public static class EchoMessageSerializer implements IVersionedSerializer<EchoMessage>
- {
- public void serialize(EchoMessage t, DataOutputPlus out, int version) throws IOException
- {
- }
-
- public EchoMessage deserialize(DataInputPlus in, int version) throws IOException
- {
- return EchoMessage.instance;
- }
-
- public long serializedSize(EchoMessage t, int version)
- {
- return 0;
- }
- }
-}
diff --git a/src/java/org/apache/cassandra/gms/EndpointState.java b/src/java/org/apache/cassandra/gms/EndpointState.java
index 674b597..8546a70 100644
--- a/src/java/org/apache/cassandra/gms/EndpointState.java
+++ b/src/java/org/apache/cassandra/gms/EndpointState.java
@@ -18,11 +18,7 @@
package org.apache.cassandra.gms;
import java.io.*;
-import java.util.Collections;
-import java.util.EnumMap;
-import java.util.Map;
-import java.util.Set;
-import java.util.UUID;
+import java.util.*;
import java.util.concurrent.atomic.AtomicReference;
import javax.annotation.Nullable;
@@ -54,7 +50,7 @@
private volatile long updateTimestamp;
private volatile boolean isAlive;
- EndpointState(HeartBeatState initialHbState)
+ public EndpointState(HeartBeatState initialHbState)
{
this(initialHbState, new EnumMap<ApplicationState, VersionedValue>(ApplicationState.class));
}
@@ -148,11 +144,22 @@
return rpcState != null && Boolean.parseBoolean(rpcState.value);
}
+ public boolean isNormalState()
+ {
+ return getStatus().equals(VersionedValue.STATUS_NORMAL);
+ }
+
public String getStatus()
{
- VersionedValue status = getApplicationState(ApplicationState.STATUS);
+ VersionedValue status = getApplicationState(ApplicationState.STATUS_WITH_PORT);
if (status == null)
+ {
+ status = getApplicationState(ApplicationState.STATUS);
+ }
+ if (status == null)
+ {
return "";
+ }
String[] pieces = status.value.split(VersionedValue.DELIMITER_STR, -1);
assert (pieces.length > 0);
diff --git a/src/java/org/apache/cassandra/gms/FailureDetector.java b/src/java/org/apache/cassandra/gms/FailureDetector.java
index d8e1324..d3a5f34 100644
--- a/src/java/org/apache/cassandra/gms/FailureDetector.java
+++ b/src/java/org/apache/cassandra/gms/FailureDetector.java
@@ -17,26 +17,31 @@
*/
package org.apache.cassandra.gms;
+import java.nio.file.Files;
+import java.nio.file.StandardOpenOption;
+import java.nio.file.Path;
import java.io.*;
-import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.TimeUnit;
+import java.util.function.Predicate;
import javax.management.openmbean.CompositeData;
import javax.management.openmbean.*;
+import org.apache.cassandra.locator.Replica;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.io.FSWriteError;
-import org.apache.cassandra.io.util.FileUtils;
-import org.apache.cassandra.utils.Clock;
+import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.MBeanWrapper;
+import static org.apache.cassandra.utils.MonotonicClock.preciseTime;
+
/**
* This FailureDetector is an implementation of the paper titled
* "The Phi Accrual Failure Detector" by Hayashibara.
@@ -51,7 +56,7 @@
private static final int DEBUG_PERCENTAGE = 80; // if the phi is larger than this percentage of the max, log a debug message
private static final long DEFAULT_MAX_PAUSE = 5000L * 1000000L; // 5 seconds
private static final long MAX_LOCAL_PAUSE_IN_NANOS = getMaxLocalPause();
- private long lastInterpret = Clock.instance.nanoTime();
+ private long lastInterpret = preciseTime.now();
private long lastPause = 0L;
private static long getMaxLocalPause()
@@ -67,6 +72,8 @@
}
public static final IFailureDetector instance = new FailureDetector();
+ public static final Predicate<InetAddressAndPort> isEndpointAlive = instance::isAlive;
+ public static final Predicate<Replica> isReplicaAlive = r -> isEndpointAlive.test(r.endpoint());
// this is useless except to provide backwards compatibility in phi_convict_threshold,
// because everyone seems pretty accustomed to the default of 8, and users who have
@@ -74,7 +81,7 @@
// change.
private final double PHI_FACTOR = 1.0 / Math.log(10.0); // 0.434...
- private final ConcurrentHashMap<InetAddress, ArrivalWindow> arrivalSamples = new ConcurrentHashMap<>();
+ private final ConcurrentHashMap<InetAddressAndPort, ArrivalWindow> arrivalSamples = new ConcurrentHashMap<>();
private final List<IFailureDetectionEventListener> fdEvntListeners = new CopyOnWriteArrayList<>();
public FailureDetector()
@@ -99,10 +106,20 @@
public String getAllEndpointStates()
{
+ return getAllEndpointStates(false);
+ }
+
+ public String getAllEndpointStatesWithPort()
+ {
+ return getAllEndpointStates(true);
+ }
+
+ public String getAllEndpointStates(boolean withPort)
+ {
StringBuilder sb = new StringBuilder();
- for (Map.Entry<InetAddress, EndpointState> entry : Gossiper.instance.endpointStateMap.entrySet())
+ for (Map.Entry<InetAddressAndPort, EndpointState> entry : Gossiper.instance.endpointStateMap.entrySet())
{
- sb.append(entry.getKey()).append("\n");
+ sb.append(entry.getKey().toString(withPort)).append("\n");
appendEndpointState(sb, entry.getValue());
}
return sb.toString();
@@ -110,13 +127,23 @@
public Map<String, String> getSimpleStates()
{
+ return getSimpleStates(false);
+ }
+
+ public Map<String, String> getSimpleStatesWithPort()
+ {
+ return getSimpleStates(true);
+ }
+
+ private Map<String, String> getSimpleStates(boolean withPort)
+ {
Map<String, String> nodesStatus = new HashMap<String, String>(Gossiper.instance.endpointStateMap.size());
- for (Map.Entry<InetAddress, EndpointState> entry : Gossiper.instance.endpointStateMap.entrySet())
+ for (Map.Entry<InetAddressAndPort, EndpointState> entry : Gossiper.instance.endpointStateMap.entrySet())
{
if (entry.getValue().isAlive())
- nodesStatus.put(entry.getKey().toString(), "UP");
+ nodesStatus.put(entry.getKey().toString(withPort), "UP");
else
- nodesStatus.put(entry.getKey().toString(), "DOWN");
+ nodesStatus.put(entry.getKey().toString(withPort), "DOWN");
}
return nodesStatus;
}
@@ -124,7 +151,7 @@
public int getDownEndpointCount()
{
int count = 0;
- for (Map.Entry<InetAddress, EndpointState> entry : Gossiper.instance.endpointStateMap.entrySet())
+ for (Map.Entry<InetAddressAndPort, EndpointState> entry : Gossiper.instance.endpointStateMap.entrySet())
{
if (!entry.getValue().isAlive())
count++;
@@ -135,7 +162,7 @@
public int getUpEndpointCount()
{
int count = 0;
- for (Map.Entry<InetAddress, EndpointState> entry : Gossiper.instance.endpointStateMap.entrySet())
+ for (Map.Entry<InetAddressAndPort, EndpointState> entry : Gossiper.instance.endpointStateMap.entrySet())
{
if (entry.getValue().isAlive())
count++;
@@ -146,13 +173,24 @@
@Override
public TabularData getPhiValues() throws OpenDataException
{
+ return getPhiValues(false);
+ }
+
+ @Override
+ public TabularData getPhiValuesWithPort() throws OpenDataException
+ {
+ return getPhiValues(true);
+ }
+
+ private TabularData getPhiValues(boolean withPort) throws OpenDataException
+ {
final CompositeType ct = new CompositeType("Node", "Node",
new String[]{"Endpoint", "PHI"},
new String[]{"IP of the endpoint", "PHI value"},
new OpenType[]{SimpleType.STRING, SimpleType.DOUBLE});
final TabularDataSupport results = new TabularDataSupport(new TabularType("PhiList", "PhiList", ct, new String[]{"Endpoint"}));
- for (final Map.Entry<InetAddress, ArrivalWindow> entry : arrivalSamples.entrySet())
+ for (final Map.Entry<InetAddressAndPort, ArrivalWindow> entry : arrivalSamples.entrySet())
{
final ArrivalWindow window = entry.getValue();
if (window.mean() > 0)
@@ -163,7 +201,7 @@
// returned values are scaled by PHI_FACTOR so that the are on the same scale as PhiConvictThreshold
final CompositeData data = new CompositeDataSupport(ct,
new String[]{"Endpoint", "PHI"},
- new Object[]{entry.getKey().toString(), phi * PHI_FACTOR});
+ new Object[]{entry.getKey().toString(withPort), phi * PHI_FACTOR});
results.put(data);
}
}
@@ -174,7 +212,7 @@
public String getEndpointState(String address) throws UnknownHostException
{
StringBuilder sb = new StringBuilder();
- EndpointState endpointState = Gossiper.instance.getEndpointStateForEndpoint(InetAddress.getByName(address));
+ EndpointState endpointState = Gossiper.instance.getEndpointStateForEndpoint(InetAddressAndPort.getByName(address));
appendEndpointState(sb, endpointState);
return sb.toString();
}
@@ -205,15 +243,18 @@
*/
public void dumpInterArrivalTimes()
{
- File file = FileUtils.createTempFile("failuredetector-", ".dat");
+ Path path = null;
+ try {
+ path = Files.createTempFile("failuredetector-", ".dat");
- try (OutputStream os = new BufferedOutputStream(new FileOutputStream(file, true)))
- {
- os.write(toString().getBytes());
+ try (OutputStream os = new BufferedOutputStream(Files.newOutputStream(path, StandardOpenOption.APPEND)))
+ {
+ os.write(toString().getBytes());
+ }
}
catch (IOException e)
{
- throw new FSWriteError(e, file);
+ throw new FSWriteError(e, (path == null) ? null : path.toFile());
}
}
@@ -227,9 +268,9 @@
return DatabaseDescriptor.getPhiConvictThreshold();
}
- public boolean isAlive(InetAddress ep)
+ public boolean isAlive(InetAddressAndPort ep)
{
- if (ep.equals(FBUtilities.getBroadcastAddress()))
+ if (ep.equals(FBUtilities.getBroadcastAddressAndPort()))
return true;
EndpointState epState = Gossiper.instance.getEndpointStateForEndpoint(ep);
@@ -241,9 +282,9 @@
return epState != null && epState.isAlive();
}
- public void report(InetAddress ep)
+ public void report(InetAddressAndPort ep)
{
- long now = Clock.instance.nanoTime();
+ long now = preciseTime.now();
ArrivalWindow heartbeatWindow = arrivalSamples.get(ep);
if (heartbeatWindow == null)
{
@@ -260,26 +301,26 @@
}
if (logger.isTraceEnabled() && heartbeatWindow != null)
- logger.trace("Average for {} is {}", ep, heartbeatWindow.mean());
+ logger.trace("Average for {} is {}ns", ep, heartbeatWindow.mean());
}
- public void interpret(InetAddress ep)
+ public void interpret(InetAddressAndPort ep)
{
ArrivalWindow hbWnd = arrivalSamples.get(ep);
if (hbWnd == null)
{
return;
}
- long now = Clock.instance.nanoTime();
+ long now = preciseTime.now();
long diff = now - lastInterpret;
lastInterpret = now;
if (diff > MAX_LOCAL_PAUSE_IN_NANOS)
{
- logger.warn("Not marking nodes down due to local pause of {} > {}", diff, MAX_LOCAL_PAUSE_IN_NANOS);
+ logger.warn("Not marking nodes down due to local pause of {}ns > {}ns", diff, MAX_LOCAL_PAUSE_IN_NANOS);
lastPause = now;
return;
}
- if (Clock.instance.nanoTime() - lastPause < MAX_LOCAL_PAUSE_IN_NANOS)
+ if (preciseTime.now() - lastPause < MAX_LOCAL_PAUSE_IN_NANOS)
{
logger.debug("Still not marking nodes down due to local pause");
return;
@@ -291,7 +332,7 @@
if (PHI_FACTOR * phi > getPhiConvictThreshold())
{
if (logger.isTraceEnabled())
- logger.trace("Node {} phi {} > {}; intervals: {} mean: {}", new Object[]{ep, PHI_FACTOR * phi, getPhiConvictThreshold(), hbWnd, hbWnd.mean()});
+ logger.trace("Node {} phi {} > {}; intervals: {} mean: {}ns", new Object[]{ep, PHI_FACTOR * phi, getPhiConvictThreshold(), hbWnd, hbWnd.mean()});
for (IFailureDetectionEventListener listener : fdEvntListeners)
{
listener.convict(ep, phi);
@@ -304,11 +345,11 @@
else if (logger.isTraceEnabled())
{
logger.trace("PHI for {} : {}", ep, phi);
- logger.trace("mean for {} : {}", ep, hbWnd.mean());
+ logger.trace("mean for {} : {}ns", ep, hbWnd.mean());
}
}
- public void forceConviction(InetAddress ep)
+ public void forceConviction(InetAddressAndPort ep)
{
logger.debug("Forcing conviction of {}", ep);
for (IFailureDetectionEventListener listener : fdEvntListeners)
@@ -317,7 +358,7 @@
}
}
- public void remove(InetAddress ep)
+ public void remove(InetAddressAndPort ep)
{
arrivalSamples.remove(ep);
}
@@ -335,10 +376,10 @@
public String toString()
{
StringBuilder sb = new StringBuilder();
- Set<InetAddress> eps = arrivalSamples.keySet();
+ Set<InetAddressAndPort> eps = arrivalSamples.keySet();
sb.append("-----------------------------------------------------------------------");
- for (InetAddress ep : eps)
+ for (InetAddressAndPort ep : eps)
{
ArrivalWindow hWnd = arrivalSamples.get(ep);
sb.append(ep).append(" : ");
@@ -431,7 +472,7 @@
}
}
- synchronized void add(long value, InetAddress ep)
+ synchronized void add(long value, InetAddressAndPort ep)
{
assert tLast >= 0;
if (tLast > 0L)
@@ -440,11 +481,11 @@
if (interArrivalTime <= MAX_INTERVAL_IN_NANO)
{
arrivalIntervals.add(interArrivalTime);
- logger.trace("Reporting interval time of {} for {}", interArrivalTime, ep);
+ logger.trace("Reporting interval time of {}ns for {}", interArrivalTime, ep);
}
else
{
- logger.trace("Ignoring interval time of {} for {}", interArrivalTime, ep);
+ logger.trace("Ignoring interval time of {}ns for {}", interArrivalTime, ep);
}
}
else
diff --git a/src/java/org/apache/cassandra/gms/FailureDetectorMBean.java b/src/java/org/apache/cassandra/gms/FailureDetectorMBean.java
index 23fae3a..6be31b0 100644
--- a/src/java/org/apache/cassandra/gms/FailureDetectorMBean.java
+++ b/src/java/org/apache/cassandra/gms/FailureDetectorMBean.java
@@ -31,15 +31,18 @@
public double getPhiConvictThreshold();
- public String getAllEndpointStates();
+ @Deprecated public String getAllEndpointStates();
+ public String getAllEndpointStatesWithPort();
public String getEndpointState(String address) throws UnknownHostException;
- public Map<String, String> getSimpleStates();
+ @Deprecated public Map<String, String> getSimpleStates();
+ public Map<String, String> getSimpleStatesWithPort();
public int getDownEndpointCount();
public int getUpEndpointCount();
- public TabularData getPhiValues() throws OpenDataException;
+ @Deprecated public TabularData getPhiValues() throws OpenDataException;
+ public TabularData getPhiValuesWithPort() throws OpenDataException;
}
diff --git a/src/java/org/apache/cassandra/gms/GossipDigest.java b/src/java/org/apache/cassandra/gms/GossipDigest.java
index 9dfd486..53f6c5c 100644
--- a/src/java/org/apache/cassandra/gms/GossipDigest.java
+++ b/src/java/org/apache/cassandra/gms/GossipDigest.java
@@ -18,13 +18,14 @@
package org.apache.cassandra.gms;
import java.io.*;
-import java.net.InetAddress;
import org.apache.cassandra.db.TypeSizes;
import org.apache.cassandra.io.IVersionedSerializer;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
-import org.apache.cassandra.net.CompactEndpointSerializationHelper;
+import org.apache.cassandra.locator.InetAddressAndPort;
+
+import static org.apache.cassandra.locator.InetAddressAndPort.Serializer.inetAddressAndPortSerializer;
/**
* Contains information about a specified list of Endpoints and the largest version
@@ -34,18 +35,18 @@
{
public static final IVersionedSerializer<GossipDigest> serializer = new GossipDigestSerializer();
- final InetAddress endpoint;
+ final InetAddressAndPort endpoint;
final int generation;
final int maxVersion;
- GossipDigest(InetAddress ep, int gen, int version)
+ GossipDigest(InetAddressAndPort ep, int gen, int version)
{
endpoint = ep;
generation = gen;
maxVersion = version;
}
- InetAddress getEndpoint()
+ InetAddressAndPort getEndpoint()
{
return endpoint;
}
@@ -83,14 +84,14 @@
{
public void serialize(GossipDigest gDigest, DataOutputPlus out, int version) throws IOException
{
- CompactEndpointSerializationHelper.serialize(gDigest.endpoint, out);
+ inetAddressAndPortSerializer.serialize(gDigest.endpoint, out, version);
out.writeInt(gDigest.generation);
out.writeInt(gDigest.maxVersion);
}
public GossipDigest deserialize(DataInputPlus in, int version) throws IOException
{
- InetAddress endpoint = CompactEndpointSerializationHelper.deserialize(in);
+ InetAddressAndPort endpoint = inetAddressAndPortSerializer.deserialize(in, version);
int generation = in.readInt();
int maxVersion = in.readInt();
return new GossipDigest(endpoint, generation, maxVersion);
@@ -98,7 +99,7 @@
public long serializedSize(GossipDigest gDigest, int version)
{
- long size = CompactEndpointSerializationHelper.serializedSize(gDigest.endpoint);
+ long size = inetAddressAndPortSerializer.serializedSize(gDigest.endpoint, version);
size += TypeSizes.sizeof(gDigest.generation);
size += TypeSizes.sizeof(gDigest.maxVersion);
return size;
diff --git a/src/java/org/apache/cassandra/gms/GossipDigestAck.java b/src/java/org/apache/cassandra/gms/GossipDigestAck.java
index cf71ae6..26494ea 100644
--- a/src/java/org/apache/cassandra/gms/GossipDigestAck.java
+++ b/src/java/org/apache/cassandra/gms/GossipDigestAck.java
@@ -18,7 +18,6 @@
package org.apache.cassandra.gms;
import java.io.IOException;
-import java.net.InetAddress;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -27,7 +26,9 @@
import org.apache.cassandra.io.IVersionedSerializer;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
-import org.apache.cassandra.net.CompactEndpointSerializationHelper;
+import org.apache.cassandra.locator.InetAddressAndPort;
+
+import static org.apache.cassandra.locator.InetAddressAndPort.Serializer.inetAddressAndPortSerializer;
/**
* This ack gets sent out as a result of the receipt of a GossipDigestSynMessage by an
@@ -38,9 +39,9 @@
public static final IVersionedSerializer<GossipDigestAck> serializer = new GossipDigestAckSerializer();
final List<GossipDigest> gDigestList;
- final Map<InetAddress, EndpointState> epStateMap;
+ final Map<InetAddressAndPort, EndpointState> epStateMap;
- GossipDigestAck(List<GossipDigest> gDigestList, Map<InetAddress, EndpointState> epStateMap)
+ GossipDigestAck(List<GossipDigest> gDigestList, Map<InetAddressAndPort, EndpointState> epStateMap)
{
this.gDigestList = gDigestList;
this.epStateMap = epStateMap;
@@ -51,7 +52,7 @@
return gDigestList;
}
- Map<InetAddress, EndpointState> getEndpointStateMap()
+ Map<InetAddressAndPort, EndpointState> getEndpointStateMap()
{
return epStateMap;
}
@@ -63,10 +64,10 @@
{
GossipDigestSerializationHelper.serialize(gDigestAckMessage.gDigestList, out, version);
out.writeInt(gDigestAckMessage.epStateMap.size());
- for (Map.Entry<InetAddress, EndpointState> entry : gDigestAckMessage.epStateMap.entrySet())
+ for (Map.Entry<InetAddressAndPort, EndpointState> entry : gDigestAckMessage.epStateMap.entrySet())
{
- InetAddress ep = entry.getKey();
- CompactEndpointSerializationHelper.serialize(ep, out);
+ InetAddressAndPort ep = entry.getKey();
+ inetAddressAndPortSerializer.serialize(ep, out, version);
EndpointState.serializer.serialize(entry.getValue(), out, version);
}
}
@@ -75,11 +76,11 @@
{
List<GossipDigest> gDigestList = GossipDigestSerializationHelper.deserialize(in, version);
int size = in.readInt();
- Map<InetAddress, EndpointState> epStateMap = new HashMap<InetAddress, EndpointState>(size);
+ Map<InetAddressAndPort, EndpointState> epStateMap = new HashMap<InetAddressAndPort, EndpointState>(size);
for (int i = 0; i < size; ++i)
{
- InetAddress ep = CompactEndpointSerializationHelper.deserialize(in);
+ InetAddressAndPort ep = inetAddressAndPortSerializer.deserialize(in, version);
EndpointState epState = EndpointState.serializer.deserialize(in, version);
epStateMap.put(ep, epState);
}
@@ -90,8 +91,8 @@
{
int size = GossipDigestSerializationHelper.serializedSize(ack.gDigestList, version);
size += TypeSizes.sizeof(ack.epStateMap.size());
- for (Map.Entry<InetAddress, EndpointState> entry : ack.epStateMap.entrySet())
- size += CompactEndpointSerializationHelper.serializedSize(entry.getKey())
+ for (Map.Entry<InetAddressAndPort, EndpointState> entry : ack.epStateMap.entrySet())
+ size += inetAddressAndPortSerializer.serializedSize(entry.getKey(), version)
+ EndpointState.serializer.serializedSize(entry.getValue(), version);
return size;
}
diff --git a/src/java/org/apache/cassandra/gms/GossipDigestAck2.java b/src/java/org/apache/cassandra/gms/GossipDigestAck2.java
index 9d779fe..0e4062b 100644
--- a/src/java/org/apache/cassandra/gms/GossipDigestAck2.java
+++ b/src/java/org/apache/cassandra/gms/GossipDigestAck2.java
@@ -18,7 +18,6 @@
package org.apache.cassandra.gms;
import java.io.*;
-import java.net.InetAddress;
import java.util.HashMap;
import java.util.Map;
@@ -26,7 +25,9 @@
import org.apache.cassandra.io.IVersionedSerializer;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
-import org.apache.cassandra.net.CompactEndpointSerializationHelper;
+import org.apache.cassandra.locator.InetAddressAndPort;
+
+import static org.apache.cassandra.locator.InetAddressAndPort.Serializer.inetAddressAndPortSerializer;
/**
* This ack gets sent out as a result of the receipt of a GossipDigestAckMessage. This the
@@ -36,14 +37,14 @@
{
public static final IVersionedSerializer<GossipDigestAck2> serializer = new GossipDigestAck2Serializer();
- final Map<InetAddress, EndpointState> epStateMap;
+ final Map<InetAddressAndPort, EndpointState> epStateMap;
- GossipDigestAck2(Map<InetAddress, EndpointState> epStateMap)
+ GossipDigestAck2(Map<InetAddressAndPort, EndpointState> epStateMap)
{
this.epStateMap = epStateMap;
}
- Map<InetAddress, EndpointState> getEndpointStateMap()
+ Map<InetAddressAndPort, EndpointState> getEndpointStateMap()
{
return epStateMap;
}
@@ -54,10 +55,10 @@
public void serialize(GossipDigestAck2 ack2, DataOutputPlus out, int version) throws IOException
{
out.writeInt(ack2.epStateMap.size());
- for (Map.Entry<InetAddress, EndpointState> entry : ack2.epStateMap.entrySet())
+ for (Map.Entry<InetAddressAndPort, EndpointState> entry : ack2.epStateMap.entrySet())
{
- InetAddress ep = entry.getKey();
- CompactEndpointSerializationHelper.serialize(ep, out);
+ InetAddressAndPort ep = entry.getKey();
+ inetAddressAndPortSerializer.serialize(ep, out, version);
EndpointState.serializer.serialize(entry.getValue(), out, version);
}
}
@@ -65,11 +66,11 @@
public GossipDigestAck2 deserialize(DataInputPlus in, int version) throws IOException
{
int size = in.readInt();
- Map<InetAddress, EndpointState> epStateMap = new HashMap<InetAddress, EndpointState>(size);
+ Map<InetAddressAndPort, EndpointState> epStateMap = new HashMap<>(size);
for (int i = 0; i < size; ++i)
{
- InetAddress ep = CompactEndpointSerializationHelper.deserialize(in);
+ InetAddressAndPort ep = inetAddressAndPortSerializer.deserialize(in, version);
EndpointState epState = EndpointState.serializer.deserialize(in, version);
epStateMap.put(ep, epState);
}
@@ -79,8 +80,8 @@
public long serializedSize(GossipDigestAck2 ack2, int version)
{
long size = TypeSizes.sizeof(ack2.epStateMap.size());
- for (Map.Entry<InetAddress, EndpointState> entry : ack2.epStateMap.entrySet())
- size += CompactEndpointSerializationHelper.serializedSize(entry.getKey())
+ for (Map.Entry<InetAddressAndPort, EndpointState> entry : ack2.epStateMap.entrySet())
+ size += inetAddressAndPortSerializer.serializedSize(entry.getKey(), version)
+ EndpointState.serializer.serializedSize(entry.getValue(), version);
return size;
}
diff --git a/src/java/org/apache/cassandra/gms/GossipDigestAck2VerbHandler.java b/src/java/org/apache/cassandra/gms/GossipDigestAck2VerbHandler.java
index 240bb40..58c1589 100644
--- a/src/java/org/apache/cassandra/gms/GossipDigestAck2VerbHandler.java
+++ b/src/java/org/apache/cassandra/gms/GossipDigestAck2VerbHandler.java
@@ -17,24 +17,25 @@
*/
package org.apache.cassandra.gms;
-import java.net.InetAddress;
import java.util.Map;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.cassandra.net.IVerbHandler;
-import org.apache.cassandra.net.MessageIn;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.net.Message;
-public class GossipDigestAck2VerbHandler implements IVerbHandler<GossipDigestAck2>
+public class GossipDigestAck2VerbHandler extends GossipVerbHandler<GossipDigestAck2>
{
+ public static final GossipDigestAck2VerbHandler instance = new GossipDigestAck2VerbHandler();
+
private static final Logger logger = LoggerFactory.getLogger(GossipDigestAck2VerbHandler.class);
- public void doVerb(MessageIn<GossipDigestAck2> message, int id)
+ public void doVerb(Message<GossipDigestAck2> message)
{
if (logger.isTraceEnabled())
{
- InetAddress from = message.from;
+ InetAddressAndPort from = message.from();
logger.trace("Received a GossipDigestAck2Message from {}", from);
}
if (!Gossiper.instance.isEnabled())
@@ -43,9 +44,11 @@
logger.trace("Ignoring GossipDigestAck2Message because gossip is disabled");
return;
}
- Map<InetAddress, EndpointState> remoteEpStateMap = message.payload.getEndpointStateMap();
+ Map<InetAddressAndPort, EndpointState> remoteEpStateMap = message.payload.getEndpointStateMap();
/* Notify the Failure Detector */
Gossiper.instance.notifyFailureDetector(remoteEpStateMap);
Gossiper.instance.applyStateLocally(remoteEpStateMap);
+
+ super.doVerb(message);
}
}
diff --git a/src/java/org/apache/cassandra/gms/GossipDigestAckVerbHandler.java b/src/java/org/apache/cassandra/gms/GossipDigestAckVerbHandler.java
index d6d9dfb..1e8604b 100644
--- a/src/java/org/apache/cassandra/gms/GossipDigestAckVerbHandler.java
+++ b/src/java/org/apache/cassandra/gms/GossipDigestAckVerbHandler.java
@@ -17,7 +17,6 @@
*/
package org.apache.cassandra.gms;
-import java.net.InetAddress;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -25,18 +24,21 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.cassandra.net.IVerbHandler;
-import org.apache.cassandra.net.MessageIn;
-import org.apache.cassandra.net.MessageOut;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.net.Message;
import org.apache.cassandra.net.MessagingService;
-public class GossipDigestAckVerbHandler implements IVerbHandler<GossipDigestAck>
+import static org.apache.cassandra.net.Verb.GOSSIP_DIGEST_ACK2;
+
+public class GossipDigestAckVerbHandler extends GossipVerbHandler<GossipDigestAck>
{
+ public static final GossipDigestAckVerbHandler instance = new GossipDigestAckVerbHandler();
+
private static final Logger logger = LoggerFactory.getLogger(GossipDigestAckVerbHandler.class);
- public void doVerb(MessageIn<GossipDigestAck> message, int id)
+ public void doVerb(Message<GossipDigestAck> message)
{
- InetAddress from = message.from;
+ InetAddressAndPort from = message.from();
if (logger.isTraceEnabled())
logger.trace("Received a GossipDigestAckMessage from {}", from);
if (!Gossiper.instance.isEnabled() && !Gossiper.instance.isInShadowRound())
@@ -48,7 +50,7 @@
GossipDigestAck gDigestAckMessage = message.payload;
List<GossipDigest> gDigestList = gDigestAckMessage.getGossipDigestList();
- Map<InetAddress, EndpointState> epStateMap = gDigestAckMessage.getEndpointStateMap();
+ Map<InetAddressAndPort, EndpointState> epStateMap = gDigestAckMessage.getEndpointStateMap();
logger.trace("Received ack with {} digests and {} states", gDigestList.size(), epStateMap.size());
if (Gossiper.instance.isInShadowRound())
@@ -79,20 +81,20 @@
}
/* Get the state required to send to this gossipee - construct GossipDigestAck2Message */
- Map<InetAddress, EndpointState> deltaEpStateMap = new HashMap<InetAddress, EndpointState>();
+ Map<InetAddressAndPort, EndpointState> deltaEpStateMap = new HashMap<InetAddressAndPort, EndpointState>();
for (GossipDigest gDigest : gDigestList)
{
- InetAddress addr = gDigest.getEndpoint();
+ InetAddressAndPort addr = gDigest.getEndpoint();
EndpointState localEpStatePtr = Gossiper.instance.getStateForVersionBiggerThan(addr, gDigest.getMaxVersion());
if (localEpStatePtr != null)
deltaEpStateMap.put(addr, localEpStatePtr);
}
- MessageOut<GossipDigestAck2> gDigestAck2Message = new MessageOut<GossipDigestAck2>(MessagingService.Verb.GOSSIP_DIGEST_ACK2,
- new GossipDigestAck2(deltaEpStateMap),
- GossipDigestAck2.serializer);
+ Message<GossipDigestAck2> gDigestAck2Message = Message.out(GOSSIP_DIGEST_ACK2, new GossipDigestAck2(deltaEpStateMap));
if (logger.isTraceEnabled())
logger.trace("Sending a GossipDigestAck2Message to {}", from);
- MessagingService.instance().sendOneWay(gDigestAck2Message, from);
+ MessagingService.instance().send(gDigestAck2Message, from);
+
+ super.doVerb(message);
}
}
diff --git a/src/java/org/apache/cassandra/gms/GossipDigestSynVerbHandler.java b/src/java/org/apache/cassandra/gms/GossipDigestSynVerbHandler.java
index 6d0afa2..520dbec 100644
--- a/src/java/org/apache/cassandra/gms/GossipDigestSynVerbHandler.java
+++ b/src/java/org/apache/cassandra/gms/GossipDigestSynVerbHandler.java
@@ -17,25 +17,27 @@
*/
package org.apache.cassandra.gms;
-import java.net.InetAddress;
import java.util.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.net.IVerbHandler;
-import org.apache.cassandra.net.MessageIn;
-import org.apache.cassandra.net.MessageOut;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.net.Message;
import org.apache.cassandra.net.MessagingService;
-public class GossipDigestSynVerbHandler implements IVerbHandler<GossipDigestSyn>
+import static org.apache.cassandra.net.Verb.*;
+
+public class GossipDigestSynVerbHandler extends GossipVerbHandler<GossipDigestSyn>
{
+ public static final GossipDigestSynVerbHandler instance = new GossipDigestSynVerbHandler();
+
private static final Logger logger = LoggerFactory.getLogger(GossipDigestSynVerbHandler.class);
- public void doVerb(MessageIn<GossipDigestSyn> message, int id)
+ public void doVerb(Message<GossipDigestSyn> message)
{
- InetAddress from = message.from;
+ InetAddressAndPort from = message.from();
if (logger.isTraceEnabled())
logger.trace("Received a GossipDigestSynMessage from {}", from);
if (!Gossiper.instance.isEnabled() && !Gossiper.instance.isInShadowRound())
@@ -79,10 +81,8 @@
logger.debug("Received a shadow round syn from {}. Gossip is disabled but " +
"currently also in shadow round, responding with a minimal ack", from);
MessagingService.instance()
- .sendOneWay(new MessageOut<>(MessagingService.Verb.GOSSIP_DIGEST_ACK,
- new GossipDigestAck(new ArrayList<>(), new HashMap<>()),
- GossipDigestAck.serializer),
- from);
+ .send(Message.out(GOSSIP_DIGEST_ACK, new GossipDigestAck(Collections.emptyList(), Collections.emptyMap())),
+ from);
return;
}
@@ -97,60 +97,15 @@
logger.trace("Gossip syn digests are : {}", sb);
}
- doSort(gDigestList);
-
List<GossipDigest> deltaGossipDigestList = new ArrayList<GossipDigest>();
- Map<InetAddress, EndpointState> deltaEpStateMap = new HashMap<InetAddress, EndpointState>();
+ Map<InetAddressAndPort, EndpointState> deltaEpStateMap = new HashMap<InetAddressAndPort, EndpointState>();
Gossiper.instance.examineGossiper(gDigestList, deltaGossipDigestList, deltaEpStateMap);
logger.trace("sending {} digests and {} deltas", deltaGossipDigestList.size(), deltaEpStateMap.size());
- MessageOut<GossipDigestAck> gDigestAckMessage = new MessageOut<GossipDigestAck>(MessagingService.Verb.GOSSIP_DIGEST_ACK,
- new GossipDigestAck(deltaGossipDigestList, deltaEpStateMap),
- GossipDigestAck.serializer);
+ Message<GossipDigestAck> gDigestAckMessage = Message.out(GOSSIP_DIGEST_ACK, new GossipDigestAck(deltaGossipDigestList, deltaEpStateMap));
if (logger.isTraceEnabled())
logger.trace("Sending a GossipDigestAckMessage to {}", from);
- MessagingService.instance().sendOneWay(gDigestAckMessage, from);
- }
+ MessagingService.instance().send(gDigestAckMessage, from);
- /*
- * First construct a map whose key is the endpoint in the GossipDigest and the value is the
- * GossipDigest itself. Then build a list of version differences i.e difference between the
- * version in the GossipDigest and the version in the local state for a given InetAddress.
- * Sort this list. Now loop through the sorted list and retrieve the GossipDigest corresponding
- * to the endpoint from the map that was initially constructed.
- */
- private void doSort(List<GossipDigest> gDigestList)
- {
- /* Construct a map of endpoint to GossipDigest. */
- Map<InetAddress, GossipDigest> epToDigestMap = new HashMap<InetAddress, GossipDigest>();
- for (GossipDigest gDigest : gDigestList)
- {
- epToDigestMap.put(gDigest.getEndpoint(), gDigest);
- }
-
- /*
- * These digests have their maxVersion set to the difference of the version
- * of the local EndpointState and the version found in the GossipDigest.
- */
- List<GossipDigest> diffDigests = new ArrayList<GossipDigest>(gDigestList.size());
- for (GossipDigest gDigest : gDigestList)
- {
- InetAddress ep = gDigest.getEndpoint();
- EndpointState epState = Gossiper.instance.getEndpointStateForEndpoint(ep);
- int version = (epState != null) ? Gossiper.instance.getMaxEndpointStateVersion(epState) : 0;
- int diffVersion = Math.abs(version - gDigest.getMaxVersion());
- diffDigests.add(new GossipDigest(ep, gDigest.getGeneration(), diffVersion));
- }
-
- gDigestList.clear();
- Collections.sort(diffDigests);
- int size = diffDigests.size();
- /*
- * Report the digests in descending order. This takes care of the endpoints
- * that are far behind w.r.t this local endpoint
- */
- for (int i = size - 1; i >= 0; --i)
- {
- gDigestList.add(epToDigestMap.get(diffDigests.get(i).getEndpoint()));
- }
+ super.doVerb(message);
}
}
diff --git a/src/java/org/apache/cassandra/gms/GossipShutdownVerbHandler.java b/src/java/org/apache/cassandra/gms/GossipShutdownVerbHandler.java
index 1691107..83c8568 100644
--- a/src/java/org/apache/cassandra/gms/GossipShutdownVerbHandler.java
+++ b/src/java/org/apache/cassandra/gms/GossipShutdownVerbHandler.java
@@ -18,23 +18,25 @@
package org.apache.cassandra.gms;
import org.apache.cassandra.net.IVerbHandler;
-import org.apache.cassandra.net.MessageIn;
+import org.apache.cassandra.net.Message;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class GossipShutdownVerbHandler implements IVerbHandler
{
+ public static final GossipShutdownVerbHandler instance = new GossipShutdownVerbHandler();
+
private static final Logger logger = LoggerFactory.getLogger(GossipShutdownVerbHandler.class);
- public void doVerb(MessageIn message, int id)
+ public void doVerb(Message message)
{
if (!Gossiper.instance.isEnabled())
{
- logger.debug("Ignoring shutdown message from {} because gossip is disabled", message.from);
+ logger.debug("Ignoring shutdown message from {} because gossip is disabled", message.from());
return;
}
- Gossiper.instance.markAsShutdown(message.from);
+ Gossiper.instance.markAsShutdown(message.from());
}
}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/gms/GossipVerbHandler.java b/src/java/org/apache/cassandra/gms/GossipVerbHandler.java
new file mode 100644
index 0000000..02aeaf4
--- /dev/null
+++ b/src/java/org/apache/cassandra/gms/GossipVerbHandler.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.gms;
+
+import org.apache.cassandra.net.IVerbHandler;
+import org.apache.cassandra.net.Message;
+
+public class GossipVerbHandler<T> implements IVerbHandler<T>
+{
+ public void doVerb(Message<T> message)
+ {
+ Gossiper.instance.setLastProcessedMessageAt(message.creationTimeMillis());
+ }
+}
diff --git a/src/java/org/apache/cassandra/gms/Gossiper.java b/src/java/org/apache/cassandra/gms/Gossiper.java
index d227200..f276fbd 100644
--- a/src/java/org/apache/cassandra/gms/Gossiper.java
+++ b/src/java/org/apache/cassandra/gms/Gossiper.java
@@ -17,24 +17,32 @@
*/
package org.apache.cassandra.gms;
-import java.lang.management.ManagementFactory;
-import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.*;
import java.util.Map.Entry;
import java.util.concurrent.*;
import java.util.concurrent.locks.ReentrantLock;
-import javax.annotation.Nullable;
+import java.util.function.BooleanSupplier;
+import java.util.function.Supplier;
import java.util.stream.Collectors;
+import javax.annotation.Nullable;
+
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Suppliers;
import com.google.common.base.Throwables;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.ImmutableSet;
import com.google.common.util.concurrent.ListenableFutureTask;
import com.google.common.util.concurrent.Uninterruptibles;
-import io.netty.util.concurrent.FastThreadLocal;
+import org.apache.cassandra.concurrent.JMXEnabledSingleThreadExecutor;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.net.NoPayload;
+import org.apache.cassandra.net.Verb;
+import org.apache.cassandra.utils.CassandraVersion;
import org.apache.cassandra.utils.ExecutorUtils;
import org.apache.cassandra.utils.MBeanWrapper;
import org.apache.cassandra.utils.NoSpamLogger;
@@ -45,20 +53,18 @@
import org.apache.cassandra.concurrent.DebuggableScheduledThreadPoolExecutor;
import org.apache.cassandra.concurrent.JMXEnabledThreadPoolExecutor;
import org.apache.cassandra.concurrent.Stage;
-import org.apache.cassandra.concurrent.StageManager;
import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.config.Schema;
import org.apache.cassandra.dht.Token;
-import org.apache.cassandra.net.IAsyncCallback;
-import org.apache.cassandra.net.MessageIn;
-import org.apache.cassandra.net.MessageOut;
+import org.apache.cassandra.net.RequestCallback;
+import org.apache.cassandra.net.Message;
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.service.StorageService;
-import org.apache.cassandra.utils.CassandraVersion;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.JVMStabilityInspector;
-import static org.apache.cassandra.utils.ExecutorUtils.awaitTermination;
-import static org.apache.cassandra.utils.ExecutorUtils.shutdown;
+
+import static org.apache.cassandra.net.NoPayload.noPayload;
+import static org.apache.cassandra.net.Verb.ECHO_REQ;
+import static org.apache.cassandra.net.Verb.GOSSIP_DIGEST_SYN;
/**
* This module is responsible for Gossiping information for the local endpoint. This abstraction
@@ -89,7 +95,6 @@
static final List<String> DEAD_STATES = Arrays.asList(VersionedValue.REMOVING_TOKEN, VersionedValue.REMOVED_TOKEN,
VersionedValue.STATUS_LEFT, VersionedValue.HIBERNATE);
static ArrayList<String> SILENT_SHUTDOWN_STATES = new ArrayList<>();
-
static
{
SILENT_SHUTDOWN_STATES.addAll(DEAD_STATES);
@@ -103,71 +108,101 @@
public final static int QUARANTINE_DELAY = StorageService.RING_DELAY * 2;
private static final Logger logger = LoggerFactory.getLogger(Gossiper.class);
private static final NoSpamLogger noSpamLogger = NoSpamLogger.getLogger(logger, 15L, TimeUnit.MINUTES);
- public static final Gossiper instance = new Gossiper();
+
+ public static final Gossiper instance = new Gossiper(true);
// Timestamp to prevent processing any in-flight messages for we've not send any SYN yet, see CASSANDRA-12653.
volatile long firstSynSendAt = 0L;
- public static final long aVeryLongTime = 259200 * 1000; // 3 days
+ public static final long aVeryLongTime = getVeryLongTime();
// Maximimum difference between generation value and local time we are willing to accept about a peer
static final int MAX_GENERATION_DIFFERENCE = 86400 * 365;
private long fatClientTimeout;
private final Random random = new Random();
- private final Comparator<InetAddress> inetcomparator = new Comparator<InetAddress>()
- {
- public int compare(InetAddress addr1, InetAddress addr2)
- {
- return addr1.getHostAddress().compareTo(addr2.getHostAddress());
- }
- };
/* subscribers for interest in EndpointState change */
private final List<IEndpointStateChangeSubscriber> subscribers = new CopyOnWriteArrayList<IEndpointStateChangeSubscriber>();
/* live member set */
- private final Set<InetAddress> liveEndpoints = new ConcurrentSkipListSet<InetAddress>(inetcomparator);
+ @VisibleForTesting
+ final Set<InetAddressAndPort> liveEndpoints = new ConcurrentSkipListSet<>();
/* unreachable member set */
- private final Map<InetAddress, Long> unreachableEndpoints = new ConcurrentHashMap<InetAddress, Long>();
+ private final Map<InetAddressAndPort, Long> unreachableEndpoints = new ConcurrentHashMap<>();
/* initial seeds for joining the cluster */
@VisibleForTesting
- final Set<InetAddress> seeds = new ConcurrentSkipListSet<InetAddress>(inetcomparator);
+ final Set<InetAddressAndPort> seeds = new ConcurrentSkipListSet<>();
/* map where key is the endpoint and value is the state associated with the endpoint */
- final ConcurrentMap<InetAddress, EndpointState> endpointStateMap = new ConcurrentHashMap<InetAddress, EndpointState>();
+ final ConcurrentMap<InetAddressAndPort, EndpointState> endpointStateMap = new ConcurrentHashMap<>();
/* map where key is endpoint and value is timestamp when this endpoint was removed from
* gossip. We will ignore any gossip regarding these endpoints for QUARANTINE_DELAY time
* after removal to prevent nodes from falsely reincarnating during the time when removal
* gossip gets propagated to all nodes */
- private final Map<InetAddress, Long> justRemovedEndpoints = new ConcurrentHashMap<InetAddress, Long>();
+ private final Map<InetAddressAndPort, Long> justRemovedEndpoints = new ConcurrentHashMap<>();
- private final Map<InetAddress, Long> expireTimeEndpointMap = new ConcurrentHashMap<InetAddress, Long>();
+ private final Map<InetAddressAndPort, Long> expireTimeEndpointMap = new ConcurrentHashMap<>();
- private volatile boolean anyNodeOn30 = false; // we assume the regular case here - all nodes are on 3.11
private volatile boolean inShadowRound = false;
// seeds gathered during shadow round that indicated to be in the shadow round phase as well
- private final Set<InetAddress> seedsInShadowRound = new ConcurrentSkipListSet<>(inetcomparator);
+ private final Set<InetAddressAndPort> seedsInShadowRound = new ConcurrentSkipListSet<>();
// endpoint states as gathered during shadow round
- private final Map<InetAddress, EndpointState> endpointShadowStateMap = new ConcurrentHashMap<>();
+ private final Map<InetAddressAndPort, EndpointState> endpointShadowStateMap = new ConcurrentHashMap<>();
private volatile long lastProcessedMessageAt = System.currentTimeMillis();
- private static FastThreadLocal<Boolean> isGossipStage = new FastThreadLocal<>();
+ //This property and anything that checks it should be removed in 5.0
+ private boolean haveMajorVersion3Nodes = true;
+
+ final com.google.common.base.Supplier<Boolean> haveMajorVersion3NodesSupplier = () ->
+ {
+ //Once there are no prior version nodes we don't need to keep rechecking
+ if (!haveMajorVersion3Nodes)
+ return false;
+
+ Iterable<InetAddressAndPort> allHosts = Iterables.concat(Gossiper.instance.getLiveMembers(), Gossiper.instance.getUnreachableMembers());
+ CassandraVersion referenceVersion = null;
+
+ for (InetAddressAndPort host : allHosts)
+ {
+ CassandraVersion version = getReleaseVersion(host);
+
+ //Raced with changes to gossip state
+ if (version == null)
+ continue;
+
+ if (referenceVersion == null)
+ referenceVersion = version;
+
+ if (version.major < 4)
+ return true;
+ }
+
+ haveMajorVersion3Nodes = false;
+ return false;
+ };
+
+ private final Supplier<Boolean> haveMajorVersion3NodesMemoized = Suppliers.memoizeWithExpiration(haveMajorVersion3NodesSupplier, 1, TimeUnit.MINUTES);
private static final boolean disableThreadValidation = Boolean.getBoolean(Props.DISABLE_THREAD_VALIDATION);
+ private static long getVeryLongTime()
+ {
+ String newVLT = System.getProperty("cassandra.very_long_time_ms");
+ if (newVLT != null)
+ {
+ logger.info("Overriding aVeryLongTime to {}ms", newVLT);
+ return Long.parseLong(newVLT);
+ }
+ return 259200 * 1000; // 3 days
+ }
+
private static boolean isInGossipStage()
{
- Boolean isGossip = isGossipStage.get();
- if (isGossip == null)
- {
- isGossip = Thread.currentThread().getName().contains(Stage.GOSSIP.getJmxName());
- isGossipStage.set(isGossip);
- }
- return isGossip;
+ return ((JMXEnabledSingleThreadExecutor) Stage.GOSSIP.executor()).isExecutedBy(Thread.currentThread());
}
private static void checkProperThreadForStateMutation()
@@ -198,9 +233,9 @@
taskLock.lock();
/* Update the local heartbeat counter. */
- endpointStateMap.get(FBUtilities.getBroadcastAddress()).getHeartBeatState().updateHeartBeat();
+ endpointStateMap.get(FBUtilities.getBroadcastAddressAndPort()).getHeartBeatState().updateHeartBeat();
if (logger.isTraceEnabled())
- logger.trace("My heartbeat is now {}", endpointStateMap.get(FBUtilities.getBroadcastAddress()).getHeartBeatState().getHeartBeatVersion());
+ logger.trace("My heartbeat is now {}", endpointStateMap.get(FBUtilities.getBroadcastAddressAndPort()).getHeartBeatState().getHeartBeatVersion());
final List<GossipDigest> gDigests = new ArrayList<GossipDigest>();
Gossiper.instance.makeRandomGossipDigest(gDigests);
@@ -209,9 +244,7 @@
GossipDigestSyn digestSynMessage = new GossipDigestSyn(DatabaseDescriptor.getClusterName(),
DatabaseDescriptor.getPartitionerName(),
gDigests);
- MessageOut<GossipDigestSyn> message = new MessageOut<GossipDigestSyn>(MessagingService.Verb.GOSSIP_DIGEST_SYN,
- digestSynMessage,
- GossipDigestSyn.serializer);
+ Message<GossipDigestSyn> message = Message.out(GOSSIP_DIGEST_SYN, digestSynMessage);
/* Gossip to some random live member */
boolean gossipedToSeed = doGossipToLiveMember(message);
@@ -252,7 +285,7 @@
}
}
- private Gossiper()
+ Gossiper(boolean registerJmx)
{
// half of QUARATINE_DELAY, to ensure justRemovedEndpoints has enough leeway to prevent re-gossip
fatClientTimeout = (QUARANTINE_DELAY / 2);
@@ -260,7 +293,10 @@
FailureDetector.instance.registerFailureDetectionEventListener(this);
// Register this instance with JMX
- MBeanWrapper.instance.registerMBean(this, MBEAN_NAME);
+ if (registerJmx)
+ {
+ MBeanWrapper.instance.registerMBean(this, MBEAN_NAME);
+ }
}
public void setLastProcessedMessageAt(long timeInMillis)
@@ -270,14 +306,24 @@
public boolean seenAnySeed()
{
- for (Map.Entry<InetAddress, EndpointState> entry : endpointStateMap.entrySet())
+ for (Map.Entry<InetAddressAndPort, EndpointState> entry : endpointStateMap.entrySet())
{
if (seeds.contains(entry.getKey()))
return true;
try
{
VersionedValue internalIp = entry.getValue().getApplicationState(ApplicationState.INTERNAL_IP);
- if (internalIp != null && seeds.contains(InetAddress.getByName(internalIp.value)))
+ VersionedValue internalIpAndPort = entry.getValue().getApplicationState(ApplicationState.INTERNAL_ADDRESS_AND_PORT);
+ InetAddressAndPort endpoint = null;
+ if (internalIpAndPort != null)
+ {
+ endpoint = InetAddressAndPort.getByName(internalIpAndPort.value);
+ }
+ else if (internalIp != null)
+ {
+ endpoint = InetAddressAndPort.getByName(internalIp.value);
+ }
+ if (endpoint != null && seeds.contains(endpoint))
return true;
}
catch (UnknownHostException e)
@@ -311,18 +357,18 @@
/**
* @return a list of live gossip participants, including fat clients
*/
- public Set<InetAddress> getLiveMembers()
+ public Set<InetAddressAndPort> getLiveMembers()
{
- Set<InetAddress> liveMembers = new HashSet<>(liveEndpoints);
- if (!liveMembers.contains(FBUtilities.getBroadcastAddress()))
- liveMembers.add(FBUtilities.getBroadcastAddress());
+ Set<InetAddressAndPort> liveMembers = new HashSet<>(liveEndpoints);
+ if (!liveMembers.contains(FBUtilities.getBroadcastAddressAndPort()))
+ liveMembers.add(FBUtilities.getBroadcastAddressAndPort());
return liveMembers;
}
/**
* @return a list of live ring members.
*/
- public Set<InetAddress> getLiveTokenOwners()
+ public Set<InetAddressAndPort> getLiveTokenOwners()
{
return StorageService.instance.getLiveRingMembers(true);
}
@@ -330,7 +376,7 @@
/**
* @return a list of unreachable gossip participants, including fat clients
*/
- public Set<InetAddress> getUnreachableMembers()
+ public Set<InetAddressAndPort> getUnreachableMembers()
{
return unreachableEndpoints.keySet();
}
@@ -338,10 +384,10 @@
/**
* @return a list of unreachable token owners
*/
- public Set<InetAddress> getUnreachableTokenOwners()
+ public Set<InetAddressAndPort> getUnreachableTokenOwners()
{
- Set<InetAddress> tokenOwners = new HashSet<>();
- for (InetAddress endpoint : unreachableEndpoints.keySet())
+ Set<InetAddressAndPort> tokenOwners = new HashSet<>();
+ for (InetAddressAndPort endpoint : unreachableEndpoints.keySet())
{
if (StorageService.instance.getTokenMetadata().isMember(endpoint))
tokenOwners.add(endpoint);
@@ -350,7 +396,7 @@
return tokenOwners;
}
- public long getEndpointDowntime(InetAddress ep)
+ public long getEndpointDowntime(InetAddressAndPort ep)
{
Long downtime = unreachableEndpoints.get(ep);
if (downtime != null)
@@ -359,14 +405,25 @@
return 0L;
}
- private boolean isShutdown(InetAddress endpoint)
+ private boolean isShutdown(InetAddressAndPort endpoint)
{
EndpointState epState = endpointStateMap.get(endpoint);
if (epState == null)
+ {
return false;
- if (epState.getApplicationState(ApplicationState.STATUS) == null)
- return false;
- String value = epState.getApplicationState(ApplicationState.STATUS).value;
+ }
+
+ VersionedValue versionedValue = epState.getApplicationState(ApplicationState.STATUS_WITH_PORT);
+ if (versionedValue == null)
+ {
+ versionedValue = epState.getApplicationState(ApplicationState.STATUS);
+ if (versionedValue == null)
+ {
+ return false;
+ }
+ }
+
+ String value = versionedValue.value;
String[] pieces = value.split(VersionedValue.DELIMITER_STR, -1);
assert (pieces.length > 0);
String state = pieces[0];
@@ -383,7 +440,7 @@
}
ListenableFutureTask task = ListenableFutureTask.create(runnable, null);
- StageManager.getStage(Stage.GOSSIP).execute(task);
+ Stage.GOSSIP.execute(task);
try
{
task.get();
@@ -400,7 +457,7 @@
*
* @param endpoint end point that is convicted.
*/
- public void convict(InetAddress endpoint, double phi)
+ public void convict(InetAddressAndPort endpoint, double phi)
{
runInGossipStageBlocking(() -> {
EndpointState epState = endpointStateMap.get(endpoint);
@@ -412,7 +469,6 @@
logger.debug("Convicting {} with status {} - alive {}", endpoint, getGossipStatus(epState), epState.isAlive());
-
if (isShutdown(endpoint))
{
markAsShutdown(endpoint);
@@ -421,6 +477,7 @@
{
markDead(endpoint, epState);
}
+ GossiperDiagnostics.convicted(this, endpoint, phi);
});
}
@@ -428,17 +485,19 @@
* This method is used to mark a node as shutdown; that is it gracefully exited on its own and told us about it
* @param endpoint endpoint that has shut itself down
*/
- protected void markAsShutdown(InetAddress endpoint)
+ protected void markAsShutdown(InetAddressAndPort endpoint)
{
checkProperThreadForStateMutation();
EndpointState epState = endpointStateMap.get(endpoint);
if (epState == null)
return;
+ epState.addApplicationState(ApplicationState.STATUS_WITH_PORT, StorageService.instance.valueFactory.shutdown(true));
epState.addApplicationState(ApplicationState.STATUS, StorageService.instance.valueFactory.shutdown(true));
epState.addApplicationState(ApplicationState.RPC_READY, StorageService.instance.valueFactory.rpcReady(false));
epState.getHeartBeatState().forceHighestPossibleVersionUnsafe();
markDead(endpoint, epState);
FailureDetector.instance.forceConviction(endpoint);
+ GossiperDiagnostics.markedAsShutdown(this, endpoint);
}
/**
@@ -460,7 +519,7 @@
*
* @param endpoint endpoint to be removed from the current membership.
*/
- private void evictFromMembership(InetAddress endpoint)
+ private void evictFromMembership(InetAddressAndPort endpoint)
{
checkProperThreadForStateMutation();
unreachableEndpoints.remove(endpoint);
@@ -470,12 +529,13 @@
quarantineEndpoint(endpoint);
if (logger.isDebugEnabled())
logger.debug("evicting {} from gossip", endpoint);
+ GossiperDiagnostics.evictedFromMembership(this, endpoint);
}
/**
* Removes the endpoint from Gossip but retains endpoint state
*/
- public void removeEndpoint(InetAddress endpoint)
+ public void removeEndpoint(InetAddressAndPort endpoint)
{
checkProperThreadForStateMutation();
// do subscribers first so anything in the subscriber that depends on gossiper state won't get confused
@@ -491,11 +551,12 @@
liveEndpoints.remove(endpoint);
unreachableEndpoints.remove(endpoint);
- MessagingService.instance().resetVersion(endpoint);
+ MessagingService.instance().versions.reset(endpoint);
quarantineEndpoint(endpoint);
- MessagingService.instance().destroyConnectionPool(endpoint);
- if (logger.isDebugEnabled())
- logger.debug("removing endpoint {}", endpoint);
+ MessagingService.instance().closeOutbound(endpoint);
+ MessagingService.instance().removeInbound(endpoint);
+ logger.debug("removing endpoint {}", endpoint);
+ GossiperDiagnostics.removedEndpoint(this, endpoint);
}
/**
@@ -503,7 +564,7 @@
*
* @param endpoint
*/
- private void quarantineEndpoint(InetAddress endpoint)
+ private void quarantineEndpoint(InetAddressAndPort endpoint)
{
quarantineEndpoint(endpoint, System.currentTimeMillis());
}
@@ -514,20 +575,22 @@
* @param endpoint
* @param quarantineExpiration
*/
- private void quarantineEndpoint(InetAddress endpoint, long quarantineExpiration)
+ private void quarantineEndpoint(InetAddressAndPort endpoint, long quarantineExpiration)
{
justRemovedEndpoints.put(endpoint, quarantineExpiration);
+ GossiperDiagnostics.quarantinedEndpoint(this, endpoint, quarantineExpiration);
}
/**
* Quarantine endpoint specifically for replacement purposes.
* @param endpoint
*/
- public void replacementQuarantine(InetAddress endpoint)
+ public void replacementQuarantine(InetAddressAndPort endpoint)
{
// remember, quarantineEndpoint will effectively already add QUARANTINE_DELAY, so this is 2x
logger.debug("");
quarantineEndpoint(endpoint, System.currentTimeMillis() + QUARANTINE_DELAY);
+ GossiperDiagnostics.replacementQuarantine(this, endpoint);
}
/**
@@ -536,12 +599,13 @@
*
* @param endpoint The endpoint that has been replaced
*/
- public void replacedEndpoint(InetAddress endpoint)
+ public void replacedEndpoint(InetAddressAndPort endpoint)
{
checkProperThreadForStateMutation();
removeEndpoint(endpoint);
evictFromMembership(endpoint);
replacementQuarantine(endpoint);
+ GossiperDiagnostics.replacedEndpoint(this, endpoint);
}
/**
@@ -557,9 +621,9 @@
int maxVersion = 0;
// local epstate will be part of endpointStateMap
- List<InetAddress> endpoints = new ArrayList<InetAddress>(endpointStateMap.keySet());
+ List<InetAddressAndPort> endpoints = new ArrayList<>(endpointStateMap.keySet());
Collections.shuffle(endpoints, random);
- for (InetAddress endpoint : endpoints)
+ for (InetAddressAndPort endpoint : endpoints)
{
epState = endpointStateMap.get(endpoint);
if (epState != null)
@@ -590,7 +654,7 @@
* @param hostId - the ID of the host being removed
* @param localHostId - my own host ID for replication coordination
*/
- public void advertiseRemoving(InetAddress endpoint, UUID hostId, UUID localHostId)
+ public void advertiseRemoving(InetAddressAndPort endpoint, UUID hostId, UUID localHostId)
{
EndpointState epState = endpointStateMap.get(endpoint);
// remember this node's generation
@@ -607,6 +671,7 @@
epState.updateTimestamp(); // make sure we don't evict it too soon
epState.getHeartBeatState().forceNewerGenerationUnsafe();
Map<ApplicationState, VersionedValue> states = new EnumMap<>(ApplicationState.class);
+ states.put(ApplicationState.STATUS_WITH_PORT, StorageService.instance.valueFactory.removingNonlocal(hostId));
states.put(ApplicationState.STATUS, StorageService.instance.valueFactory.removingNonlocal(hostId));
states.put(ApplicationState.REMOVAL_COORDINATOR, StorageService.instance.valueFactory.removalCoordinator(localHostId));
epState.addApplicationStates(states);
@@ -620,12 +685,13 @@
* @param endpoint
* @param hostId
*/
- public void advertiseTokenRemoved(InetAddress endpoint, UUID hostId)
+ public void advertiseTokenRemoved(InetAddressAndPort endpoint, UUID hostId)
{
EndpointState epState = endpointStateMap.get(endpoint);
epState.updateTimestamp(); // make sure we don't evict it too soon
epState.getHeartBeatState().forceNewerGenerationUnsafe();
long expireTime = computeExpireTime();
+ epState.addApplicationState(ApplicationState.STATUS_WITH_PORT, StorageService.instance.valueFactory.removedNonlocal(hostId, expireTime));
epState.addApplicationState(ApplicationState.STATUS, StorageService.instance.valueFactory.removedNonlocal(hostId, expireTime));
logger.info("Completing removal of {}", endpoint);
addExpireTimeForEndpoint(endpoint, expireTime);
@@ -650,7 +716,7 @@
*/
public void assassinateEndpoint(String address) throws UnknownHostException
{
- InetAddress endpoint = InetAddress.getByName(address);
+ InetAddressAndPort endpoint = InetAddressAndPort.getByName(address);
runInGossipStageBlocking(() -> {
EndpointState epState = endpointStateMap.get(endpoint);
Collection<Token> tokens = null;
@@ -691,6 +757,8 @@
}
// do not pass go, do not collect 200 dollars, just gtfo
+ long expireTime = computeExpireTime();
+ epState.addApplicationState(ApplicationState.STATUS_WITH_PORT, StorageService.instance.valueFactory.left(tokens, expireTime));
epState.addApplicationState(ApplicationState.STATUS, StorageService.instance.valueFactory.left(tokens, computeExpireTime()));
handleMajorStateChange(endpoint, epState);
Uninterruptibles.sleepUninterruptibly(intervalInMillis * 4, TimeUnit.MILLISECONDS);
@@ -698,12 +766,12 @@
});
}
- public boolean isKnownEndpoint(InetAddress endpoint)
+ public boolean isKnownEndpoint(InetAddressAndPort endpoint)
{
return endpointStateMap.containsKey(endpoint);
}
- public int getCurrentGenerationNumber(InetAddress endpoint)
+ public int getCurrentGenerationNumber(InetAddressAndPort endpoint)
{
return endpointStateMap.get(endpoint).getHeartBeatState().getGeneration();
}
@@ -715,26 +783,29 @@
* @param epSet a set of endpoint from which a random endpoint is chosen.
* @return true if the chosen endpoint is also a seed.
*/
- private boolean sendGossip(MessageOut<GossipDigestSyn> message, Set<InetAddress> epSet)
+ private boolean sendGossip(Message<GossipDigestSyn> message, Set<InetAddressAndPort> epSet)
{
- List<InetAddress> liveEndpoints = ImmutableList.copyOf(epSet);
+ List<InetAddressAndPort> liveEndpoints = ImmutableList.copyOf(epSet);
int size = liveEndpoints.size();
if (size < 1)
return false;
/* Generate a random number from 0 -> size */
int index = (size == 1) ? 0 : random.nextInt(size);
- InetAddress to = liveEndpoints.get(index);
+ InetAddressAndPort to = liveEndpoints.get(index);
if (logger.isTraceEnabled())
logger.trace("Sending a GossipDigestSyn to {} ...", to);
if (firstSynSendAt == 0)
firstSynSendAt = System.nanoTime();
- MessagingService.instance().sendOneWay(message, to);
- return seeds.contains(to);
+ MessagingService.instance().send(message, to);
+
+ boolean isSeed = seeds.contains(to);
+ GossiperDiagnostics.sendGossipDigestSyn(this, to);
+ return isSeed;
}
/* Sends a Gossip message to a live member and returns true if the recipient was a seed */
- private boolean doGossipToLiveMember(MessageOut<GossipDigestSyn> message)
+ private boolean doGossipToLiveMember(Message<GossipDigestSyn> message)
{
int size = liveEndpoints.size();
if (size == 0)
@@ -743,7 +814,7 @@
}
/* Sends a Gossip message to an unreachable member */
- private void maybeGossipToUnreachableMember(MessageOut<GossipDigestSyn> message)
+ private void maybeGossipToUnreachableMember(Message<GossipDigestSyn> message)
{
double liveEndpointCount = liveEndpoints.size();
double unreachableEndpointCount = unreachableEndpoints.size();
@@ -758,12 +829,12 @@
}
/* Possibly gossip to a seed for facilitating partition healing */
- private void maybeGossipToSeed(MessageOut<GossipDigestSyn> prod)
+ private void maybeGossipToSeed(Message<GossipDigestSyn> prod)
{
int size = seeds.size();
if (size > 0)
{
- if (size == 1 && seeds.contains(FBUtilities.getBroadcastAddress()))
+ if (size == 1 && seeds.contains(FBUtilities.getBroadcastAddressAndPort()))
{
return;
}
@@ -783,7 +854,7 @@
}
}
- public boolean isGossipOnlyMember(InetAddress endpoint)
+ public boolean isGossipOnlyMember(InetAddressAndPort endpoint)
{
EndpointState epState = endpointStateMap.get(endpoint);
if (epState == null)
@@ -808,8 +879,8 @@
* @param epStates - endpoint states in the cluster
* @return true if it is safe to start the node, false otherwise
*/
- public boolean isSafeForStartup(InetAddress endpoint, UUID localHostUUID, boolean isBootstrapping,
- Map<InetAddress, EndpointState> epStates)
+ public boolean isSafeForStartup(InetAddressAndPort endpoint, UUID localHostUUID, boolean isBootstrapping,
+ Map<InetAddressAndPort, EndpointState> epStates)
{
EndpointState epState = epStates.get(endpoint);
// if there's no previous state, or the node was previously removed from the cluster, we're good
@@ -847,7 +918,7 @@
long now = System.currentTimeMillis();
long nowNano = System.nanoTime();
- long pending = ((JMXEnabledThreadPoolExecutor) StageManager.getStage(Stage.GOSSIP)).metrics.pendingTasks.getValue();
+ long pending = ((JMXEnabledThreadPoolExecutor) Stage.GOSSIP.executor()).metrics.pendingTasks.getValue();
if (pending > 0 && lastProcessedMessageAt < now - 1000)
{
// if some new messages just arrived, give the executor some time to work on them
@@ -861,10 +932,10 @@
}
}
- Set<InetAddress> eps = endpointStateMap.keySet();
- for (InetAddress endpoint : eps)
+ Set<InetAddressAndPort> eps = endpointStateMap.keySet();
+ for (InetAddressAndPort endpoint : eps)
{
- if (endpoint.equals(FBUtilities.getBroadcastAddress()))
+ if (endpoint.equals(FBUtilities.getBroadcastAddressAndPort()))
continue;
FailureDetector.instance.interpret(endpoint);
@@ -900,7 +971,7 @@
if (!justRemovedEndpoints.isEmpty())
{
- for (Entry<InetAddress, Long> entry : justRemovedEndpoints.entrySet())
+ for (Entry<InetAddressAndPort, Long> entry : justRemovedEndpoints.entrySet())
{
if ((now - entry.getValue()) > QUARANTINE_DELAY)
{
@@ -912,34 +983,64 @@
}
}
- protected long getExpireTimeForEndpoint(InetAddress endpoint)
+ protected long getExpireTimeForEndpoint(InetAddressAndPort endpoint)
{
/* default expireTime is aVeryLongTime */
Long storedTime = expireTimeEndpointMap.get(endpoint);
return storedTime == null ? computeExpireTime() : storedTime;
}
- public EndpointState getEndpointStateForEndpoint(InetAddress ep)
+ public EndpointState getEndpointStateForEndpoint(InetAddressAndPort ep)
{
return endpointStateMap.get(ep);
}
- public Set<Entry<InetAddress, EndpointState>> getEndpointStates()
+ public ImmutableSet<InetAddressAndPort> getEndpoints()
{
- return endpointStateMap.entrySet();
+ return ImmutableSet.copyOf(endpointStateMap.keySet());
}
- public UUID getHostId(InetAddress endpoint)
+ public int getEndpointCount()
+ {
+ return endpointStateMap.size();
+ }
+
+ Map<InetAddressAndPort, EndpointState> getEndpointStateMap()
+ {
+ return ImmutableMap.copyOf(endpointStateMap);
+ }
+
+ Map<InetAddressAndPort, Long> getJustRemovedEndpoints()
+ {
+ return ImmutableMap.copyOf(justRemovedEndpoints);
+ }
+
+ Map<InetAddressAndPort, Long> getUnreachableEndpoints()
+ {
+ return ImmutableMap.copyOf(unreachableEndpoints);
+ }
+
+ Set<InetAddressAndPort> getSeedsInShadowRound()
+ {
+ return ImmutableSet.copyOf(seedsInShadowRound);
+ }
+
+ long getLastProcessedMessageAt()
+ {
+ return lastProcessedMessageAt;
+ }
+
+ public UUID getHostId(InetAddressAndPort endpoint)
{
return getHostId(endpoint, endpointStateMap);
}
- public UUID getHostId(InetAddress endpoint, Map<InetAddress, EndpointState> epStates)
+ public UUID getHostId(InetAddressAndPort endpoint, Map<InetAddressAndPort, EndpointState> epStates)
{
return UUID.fromString(epStates.get(endpoint).getApplicationState(ApplicationState.HOST_ID).value);
}
- EndpointState getStateForVersionBiggerThan(InetAddress forEndpoint, int version)
+ EndpointState getStateForVersionBiggerThan(InetAddressAndPort forEndpoint, int version)
{
EndpointState epState = endpointStateMap.get(forEndpoint);
EndpointState reqdEndpointState = null;
@@ -990,7 +1091,7 @@
/**
* determine which endpoint started up earlier
*/
- public int compareEndpointStartup(InetAddress addr1, InetAddress addr2)
+ public int compareEndpointStartup(InetAddressAndPort addr1, InetAddressAndPort addr2)
{
EndpointState ep1 = getEndpointStateForEndpoint(addr1);
EndpointState ep2 = getEndpointStateForEndpoint(addr2);
@@ -998,15 +1099,15 @@
return ep1.getHeartBeatState().getGeneration() - ep2.getHeartBeatState().getGeneration();
}
- void notifyFailureDetector(Map<InetAddress, EndpointState> remoteEpStateMap)
+ void notifyFailureDetector(Map<InetAddressAndPort, EndpointState> remoteEpStateMap)
{
- for (Entry<InetAddress, EndpointState> entry : remoteEpStateMap.entrySet())
+ for (Entry<InetAddressAndPort, EndpointState> entry : remoteEpStateMap.entrySet())
{
notifyFailureDetector(entry.getKey(), entry.getValue());
}
}
- void notifyFailureDetector(InetAddress endpoint, EndpointState remoteEndpointState)
+ void notifyFailureDetector(InetAddressAndPort endpoint, EndpointState remoteEndpointState)
{
EndpointState localEndpointState = endpointStateMap.get(endpoint);
/*
@@ -1047,36 +1148,25 @@
}
- private void markAlive(final InetAddress addr, final EndpointState localState)
+ private void markAlive(final InetAddressAndPort addr, final EndpointState localState)
{
- if (MessagingService.instance().getVersion(addr) < MessagingService.VERSION_20)
- {
- realMarkAlive(addr, localState);
- return;
- }
-
localState.markDead();
- MessageOut<EchoMessage> echoMessage = new MessageOut<EchoMessage>(MessagingService.Verb.ECHO, EchoMessage.instance, EchoMessage.serializer);
- logger.trace("Sending a EchoMessage to {}", addr);
- IAsyncCallback echoHandler = new IAsyncCallback()
+ Message<NoPayload> echoMessage = Message.out(ECHO_REQ, noPayload);
+ logger.trace("Sending ECHO_REQ to {}", addr);
+ RequestCallback echoHandler = msg ->
{
- public boolean isLatencyForSnitch()
- {
- return false;
- }
-
- public void response(MessageIn msg)
- {
- runInGossipStageBlocking(() -> realMarkAlive(addr, localState));
- }
+ // force processing of the echo response onto the gossip stage, as it comes in on the REQUEST_RESPONSE stage
+ runInGossipStageBlocking(() -> realMarkAlive(addr, localState));
};
- MessagingService.instance().sendRR(echoMessage, addr, echoHandler);
+ MessagingService.instance().sendWithCallback(echoMessage, addr, echoHandler);
+
+ GossiperDiagnostics.markedAlive(this, addr, localState);
}
@VisibleForTesting
- public void realMarkAlive(final InetAddress addr, final EndpointState localState)
+ public void realMarkAlive(final InetAddressAndPort addr, final EndpointState localState)
{
checkProperThreadForStateMutation();
if (logger.isTraceEnabled())
@@ -1092,10 +1182,12 @@
subscriber.onAlive(addr, localState);
if (logger.isTraceEnabled())
logger.trace("Notified {}", subscribers);
+
+ GossiperDiagnostics.realMarkedAlive(this, addr, localState);
}
@VisibleForTesting
- public void markDead(InetAddress addr, EndpointState localState)
+ public void markDead(InetAddressAndPort addr, EndpointState localState)
{
checkProperThreadForStateMutation();
if (logger.isTraceEnabled())
@@ -1108,6 +1200,8 @@
subscriber.onDead(addr, localState);
if (logger.isTraceEnabled())
logger.trace("Notified {}", subscribers);
+
+ GossiperDiagnostics.markedDead(this, addr, localState);
}
/**
@@ -1116,7 +1210,7 @@
* @param ep endpoint
* @param epState EndpointState for the endpoint
*/
- private void handleMajorStateChange(InetAddress ep, EndpointState epState)
+ private void handleMajorStateChange(InetAddressAndPort ep, EndpointState epState)
{
checkProperThreadForStateMutation();
EndpointState localEpState = endpointStateMap.get(ep);
@@ -1149,9 +1243,11 @@
// check this at the end so nodes will learn about the endpoint
if (isShutdown(ep))
markAsShutdown(ep);
+
+ GossiperDiagnostics.majorStateChangeHandled(this, ep, epState);
}
- public boolean isAlive(InetAddress endpoint)
+ public boolean isAlive(InetAddressAndPort endpoint)
{
EndpointState epState = getEndpointStateForEndpoint(endpoint);
if (epState == null)
@@ -1179,22 +1275,34 @@
private static String getGossipStatus(EndpointState epState)
{
- if (epState == null || epState.getApplicationState(ApplicationState.STATUS) == null)
+ if (epState == null)
+ {
return "";
+ }
- String value = epState.getApplicationState(ApplicationState.STATUS).value;
+ VersionedValue versionedValue = epState.getApplicationState(ApplicationState.STATUS_WITH_PORT);
+ if (versionedValue == null)
+ {
+ versionedValue = epState.getApplicationState(ApplicationState.STATUS);
+ if (versionedValue == null)
+ {
+ return "";
+ }
+ }
+
+ String value = versionedValue.value;
String[] pieces = value.split(VersionedValue.DELIMITER_STR, -1);
assert (pieces.length > 0);
return pieces[0];
}
- void applyStateLocally(Map<InetAddress, EndpointState> epStateMap)
+ void applyStateLocally(Map<InetAddressAndPort, EndpointState> epStateMap)
{
checkProperThreadForStateMutation();
- for (Entry<InetAddress, EndpointState> entry : epStateMap.entrySet())
+ for (Entry<InetAddressAndPort, EndpointState> entry : epStateMap.entrySet())
{
- InetAddress ep = entry.getKey();
- if ( ep.equals(FBUtilities.getBroadcastAddress()) && !isInShadowRound())
+ InetAddressAndPort ep = entry.getKey();
+ if ( ep.equals(FBUtilities.getBroadcastAddressAndPort()) && !isInShadowRound())
continue;
if (justRemovedEndpoints.containsKey(ep))
{
@@ -1260,29 +1368,9 @@
handleMajorStateChange(ep, remoteState);
}
}
-
- boolean any30 = anyEndpointOn30();
- if (any30 != anyNodeOn30)
- {
- logger.info(any30
- ? "There is at least one 3.0 node in the cluster - will store and announce compatible schema version"
- : "There are no 3.0 nodes in the cluster - will store and announce real schema version");
-
- anyNodeOn30 = any30;
- executor.submit(Schema.instance::updateVersionAndAnnounce);
- }
}
- private boolean anyEndpointOn30()
- {
- return endpointStateMap.values()
- .stream()
- .map(EndpointState::getReleaseVersion)
- .filter(Objects::nonNull)
- .anyMatch(CassandraVersion::is30);
- }
-
- private void applyNewStates(InetAddress addr, EndpointState localState, EndpointState remoteState)
+ private void applyNewStates(InetAddressAndPort addr, EndpointState localState, EndpointState remoteState)
{
// don't assert here, since if the node restarts the version will go back to zero
int oldVersion = localState.getHeartBeatState().getHeartBeatVersion();
@@ -1294,11 +1382,28 @@
Set<Entry<ApplicationState, VersionedValue>> remoteStates = remoteState.states();
assert remoteState.getHeartBeatState().getGeneration() == localState.getHeartBeatState().getGeneration();
- // filter out the states that are already up to date (has the same or higher version)
+
Set<Entry<ApplicationState, VersionedValue>> updatedStates = remoteStates.stream().filter(entry -> {
+ // Filter out pre-4.0 versions of data for more complete 4.0 versions
+ switch (entry.getKey())
+ {
+ case INTERNAL_IP:
+ if (remoteState.getApplicationState(ApplicationState.INTERNAL_ADDRESS_AND_PORT) != null) return false;
+ break;
+ case STATUS:
+ if (remoteState.getApplicationState(ApplicationState.STATUS_WITH_PORT) != null) return false;
+ break;
+ case RPC_ADDRESS:
+ if (remoteState.getApplicationState(ApplicationState.NATIVE_ADDRESS_AND_PORT) != null) return false;
+ break;
+ default:
+ break;
+ }
+
+ // filter out the states that are already up to date (has the same or higher version)
VersionedValue local = localState.getApplicationState(entry.getKey());
return (local == null || local.version < entry.getValue().version);
- }).collect(Collectors.toSet());
+ }).collect(Collectors.toSet());
if (logger.isTraceEnabled() && updatedStates.size() > 0)
{
@@ -1314,7 +1419,7 @@
}
// notify that a local application state is going to change (doesn't get triggered for remote changes)
- private void doBeforeChangeNotifications(InetAddress addr, EndpointState epState, ApplicationState apState, VersionedValue newValue)
+ private void doBeforeChangeNotifications(InetAddressAndPort addr, EndpointState epState, ApplicationState apState, VersionedValue newValue)
{
for (IEndpointStateChangeSubscriber subscriber : subscribers)
{
@@ -1323,7 +1428,7 @@
}
// notify that an application state has changed
- private void doOnChangeNotifications(InetAddress addr, ApplicationState state, VersionedValue value)
+ private void doOnChangeNotifications(InetAddressAndPort addr, ApplicationState state, VersionedValue value)
{
for (IEndpointStateChangeSubscriber subscriber : subscribers)
{
@@ -1341,7 +1446,7 @@
}
/* Send all the data with version greater than maxRemoteVersion */
- private void sendAll(GossipDigest gDigest, Map<InetAddress, EndpointState> deltaEpStateMap, int maxRemoteVersion)
+ private void sendAll(GossipDigest gDigest, Map<InetAddressAndPort, EndpointState> deltaEpStateMap, int maxRemoteVersion)
{
EndpointState localEpStatePtr = getStateForVersionBiggerThan(gDigest.getEndpoint(), maxRemoteVersion);
if (localEpStatePtr != null)
@@ -1352,15 +1457,15 @@
This method is used to figure the state that the Gossiper has but Gossipee doesn't. The delta digests
and the delta state are built up.
*/
- void examineGossiper(List<GossipDigest> gDigestList, List<GossipDigest> deltaGossipDigestList, Map<InetAddress, EndpointState> deltaEpStateMap)
+ void examineGossiper(List<GossipDigest> gDigestList, List<GossipDigest> deltaGossipDigestList, Map<InetAddressAndPort, EndpointState> deltaEpStateMap)
{
if (gDigestList.size() == 0)
{
/* we've been sent a *completely* empty syn, which should normally never happen since an endpoint will at least send a syn with itself.
- If this is happening then the node is attempting shadow gossip, and we should reply with everything we know.
+ If this is happening then the node is attempting shadow gossip, and we should respond with everything we know.
*/
logger.debug("Shadow request received, adding all states");
- for (Map.Entry<InetAddress, EndpointState> entry : endpointStateMap.entrySet())
+ for (Map.Entry<InetAddressAndPort, EndpointState> entry : endpointStateMap.entrySet())
{
gDigestList.add(new GossipDigest(entry.getKey(), 0, 0));
}
@@ -1435,7 +1540,7 @@
buildSeedsList();
/* initialize the heartbeat state for this localEndpoint */
maybeInitializeLocalState(generationNbr);
- EndpointState localState = endpointStateMap.get(FBUtilities.getBroadcastAddress());
+ EndpointState localState = endpointStateMap.get(FBUtilities.getBroadcastAddressAndPort());
localState.addApplicationStates(preloadLocalStates);
//notify snitches that Gossiper is about to start
@@ -1449,7 +1554,7 @@
TimeUnit.MILLISECONDS);
}
- public synchronized Map<InetAddress, EndpointState> doShadowRound()
+ public synchronized Map<InetAddressAndPort, EndpointState> doShadowRound()
{
return doShadowRound(Collections.EMPTY_SET);
}
@@ -1458,13 +1563,14 @@
* Do a single 'shadow' round of gossip by retrieving endpoint states that will be stored exclusively in the
* map return value, instead of endpointStateMap.
*
+ * Used when preparing to join the ring:
* <ul>
* <li>when replacing a node, to get and assume its tokens</li>
* <li>when joining, to check that the local host id matches any previous id for the endpoint address</li>
* </ul>
*
* Method is synchronized, as we use an in-progress flag to indicate that shadow round must be cleared
- * again by calling {@link Gossiper#maybeFinishShadowRound(InetAddress, boolean, Map)}. This will update
+ * again by calling {@link Gossiper#maybeFinishShadowRound(InetAddressAndPort, boolean, Map)}. This will update
* {@link Gossiper#endpointShadowStateMap} with received values, in order to return an immutable copy to the
* caller of {@link Gossiper#doShadowRound()}. Therefor only a single shadow round execution is permitted at
* the same time.
@@ -1472,7 +1578,7 @@
* @param peers Additional peers to try gossiping with.
* @return endpoint states gathered during shadow round or empty map
*/
- public synchronized Map<InetAddress, EndpointState> doShadowRound(Set<InetAddress> peers)
+ public synchronized Map<InetAddressAndPort, EndpointState> doShadowRound(Set<InetAddressAndPort> peers)
{
buildSeedsList();
// it may be that the local address is the only entry in the seed + peers
@@ -1480,7 +1586,7 @@
if (seeds.isEmpty() && peers.isEmpty())
return endpointShadowStateMap;
- boolean isSeed = DatabaseDescriptor.getSeeds().contains(FBUtilities.getBroadcastAddress());
+ boolean isSeed = DatabaseDescriptor.getSeeds().contains(FBUtilities.getBroadcastAddressAndPort());
// We double RING_DELAY if we're not a seed to increase chance of successful startup during a full cluster bounce,
// giving the seeds a chance to startup before we fail the shadow round
int shadowRoundDelay = isSeed ? StorageService.RING_DELAY : StorageService.RING_DELAY * 2;
@@ -1491,9 +1597,7 @@
GossipDigestSyn digestSynMessage = new GossipDigestSyn(DatabaseDescriptor.getClusterName(),
DatabaseDescriptor.getPartitionerName(),
gDigests);
- MessageOut<GossipDigestSyn> message = new MessageOut<GossipDigestSyn>(MessagingService.Verb.GOSSIP_DIGEST_SYN,
- digestSynMessage,
- GossipDigestSyn.serializer);
+ Message<GossipDigestSyn> message = Message.out(GOSSIP_DIGEST_SYN, digestSynMessage);
inShadowRound = true;
boolean includePeers = false;
@@ -1506,15 +1610,15 @@
{ // CASSANDRA-8072, retry at the beginning and every 5 seconds
logger.trace("Sending shadow round GOSSIP DIGEST SYN to seeds {}", seeds);
- for (InetAddress seed : seeds)
- MessagingService.instance().sendOneWay(message, seed);
+ for (InetAddressAndPort seed : seeds)
+ MessagingService.instance().send(message, seed);
// Send to any peers we already know about, but only if a seed didn't respond.
if (includePeers)
{
logger.trace("Sending shadow round GOSSIP DIGEST SYN to known peers {}", peers);
- for (InetAddress peer : peers)
- MessagingService.instance().sendOneWay(message, peer);
+ for (InetAddressAndPort peer : peers)
+ MessagingService.instance().send(message, peer);
}
includePeers = true;
}
@@ -1546,26 +1650,87 @@
@VisibleForTesting
void buildSeedsList()
{
- for (InetAddress seed : DatabaseDescriptor.getSeeds())
+ for (InetAddressAndPort seed : DatabaseDescriptor.getSeeds())
{
- if (seed.equals(FBUtilities.getBroadcastAddress()))
+ if (seed.equals(FBUtilities.getBroadcastAddressAndPort()))
continue;
seeds.add(seed);
}
}
+ /**
+ * JMX interface for triggering an update of the seed node list.
+ */
+ public List<String> reloadSeeds()
+ {
+ logger.trace("Triggering reload of seed node list");
+
+ // Get the new set in the same that buildSeedsList does
+ Set<InetAddressAndPort> tmp = new HashSet<>();
+ try
+ {
+ for (InetAddressAndPort seed : DatabaseDescriptor.getSeeds())
+ {
+ if (seed.equals(FBUtilities.getBroadcastAddressAndPort()))
+ continue;
+ tmp.add(seed);
+ }
+ }
+ // If using the SimpleSeedProvider invalid yaml added to the config since startup could
+ // cause this to throw. Additionally, third party seed providers may throw exceptions.
+ // Handle the error and return a null to indicate that there was a problem.
+ catch (Throwable e)
+ {
+ JVMStabilityInspector.inspectThrowable(e);
+ logger.warn("Error while getting seed node list: {}", e.getLocalizedMessage());
+ return null;
+ }
+
+ if (tmp.size() == 0)
+ {
+ logger.trace("New seed node list is empty. Not updating seed list.");
+ return getSeeds();
+ }
+
+ if (tmp.equals(seeds))
+ {
+ logger.trace("New seed node list matches the existing list.");
+ return getSeeds();
+ }
+
+ // Add the new entries
+ seeds.addAll(tmp);
+ // Remove the old entries
+ seeds.retainAll(tmp);
+ logger.trace("New seed node list after reload {}", seeds);
+ return getSeeds();
+ }
+
+ /**
+ * JMX endpoint for getting the list of seeds from the node
+ */
+ public List<String> getSeeds()
+ {
+ List<String> seedList = new ArrayList<String>();
+ for (InetAddressAndPort seed : seeds)
+ {
+ seedList.add(seed.toString());
+ }
+ return seedList;
+ }
+
// initialize local HB state if needed, i.e., if gossiper has never been started before.
public void maybeInitializeLocalState(int generationNbr)
{
HeartBeatState hbState = new HeartBeatState(generationNbr);
EndpointState localState = new EndpointState(hbState);
localState.markAlive();
- endpointStateMap.putIfAbsent(FBUtilities.getBroadcastAddress(), localState);
+ endpointStateMap.putIfAbsent(FBUtilities.getBroadcastAddressAndPort(), localState);
}
public void forceNewerGeneration()
{
- EndpointState epstate = endpointStateMap.get(FBUtilities.getBroadcastAddress());
+ EndpointState epstate = endpointStateMap.get(FBUtilities.getBroadcastAddressAndPort());
epstate.getHeartBeatState().forceNewerGenerationUnsafe();
}
@@ -1573,10 +1738,10 @@
/**
* Add an endpoint we knew about previously, but whose state is unknown
*/
- public void addSavedEndpoint(InetAddress ep)
+ public void addSavedEndpoint(InetAddressAndPort ep)
{
checkProperThreadForStateMutation();
- if (ep.equals(FBUtilities.getBroadcastAddress()))
+ if (ep.equals(FBUtilities.getBroadcastAddressAndPort()))
{
logger.debug("Attempt to add self as saved endpoint");
return;
@@ -1604,8 +1769,8 @@
private void addLocalApplicationStateInternal(ApplicationState state, VersionedValue value)
{
assert taskLock.isHeldByCurrentThread();
- EndpointState epState = endpointStateMap.get(FBUtilities.getBroadcastAddress());
- InetAddress epAddr = FBUtilities.getBroadcastAddress();
+ EndpointState epState = endpointStateMap.get(FBUtilities.getBroadcastAddressAndPort());
+ InetAddressAndPort epAddr = FBUtilities.getBroadcastAddressAndPort();
assert epState != null;
// Fire "before change" notifications:
doBeforeChangeNotifications(epAddr, epState, state, value);
@@ -1642,14 +1807,15 @@
public void stop()
{
- EndpointState mystate = endpointStateMap.get(FBUtilities.getBroadcastAddress());
+ EndpointState mystate = endpointStateMap.get(FBUtilities.getBroadcastAddressAndPort());
if (mystate != null && !isSilentShutdownState(mystate) && StorageService.instance.isJoined())
{
logger.info("Announcing shutdown");
+ addLocalApplicationState(ApplicationState.STATUS_WITH_PORT, StorageService.instance.valueFactory.shutdown(true));
addLocalApplicationState(ApplicationState.STATUS, StorageService.instance.valueFactory.shutdown(true));
- MessageOut message = new MessageOut(MessagingService.Verb.GOSSIP_SHUTDOWN);
- for (InetAddress ep : liveEndpoints)
- MessagingService.instance().sendOneWay(message, ep);
+ Message message = Message.out(Verb.GOSSIP_SHUTDOWN, noPayload);
+ for (InetAddressAndPort ep : liveEndpoints)
+ MessagingService.instance().send(message, ep);
Uninterruptibles.sleepUninterruptibly(Integer.getInteger("cassandra.shutdown_announce_in_ms", 2000), TimeUnit.MILLISECONDS);
}
else
@@ -1663,12 +1829,7 @@
return (scheduledGossipTask != null) && (!scheduledGossipTask.isCancelled());
}
- public boolean isAnyNodeOn30()
- {
- return anyNodeOn30;
- }
-
- protected void maybeFinishShadowRound(InetAddress respondent, boolean isInShadowRound, Map<InetAddress, EndpointState> epStateMap)
+ protected void maybeFinishShadowRound(InetAddressAndPort respondent, boolean isInShadowRound, Map<InetAddressAndPort, EndpointState> epStateMap)
{
if (inShadowRound)
{
@@ -1707,7 +1868,7 @@
}
@VisibleForTesting
- public void initializeNodeUnsafe(InetAddress addr, UUID uuid, int generationNbr)
+ public void initializeNodeUnsafe(InetAddressAndPort addr, UUID uuid, int generationNbr)
{
HeartBeatState hbState = new HeartBeatState(generationNbr);
EndpointState newState = new EndpointState(hbState);
@@ -1723,7 +1884,7 @@
}
@VisibleForTesting
- public void injectApplicationState(InetAddress endpoint, ApplicationState state, VersionedValue value)
+ public void injectApplicationState(InetAddressAndPort endpoint, ApplicationState state, VersionedValue value)
{
EndpointState localState = endpointStateMap.get(endpoint);
localState.addApplicationState(state, value);
@@ -1731,15 +1892,15 @@
public long getEndpointDowntime(String address) throws UnknownHostException
{
- return getEndpointDowntime(InetAddress.getByName(address));
+ return getEndpointDowntime(InetAddressAndPort.getByName(address));
}
public int getCurrentGenerationNumber(String address) throws UnknownHostException
{
- return getCurrentGenerationNumber(InetAddress.getByName(address));
+ return getCurrentGenerationNumber(InetAddressAndPort.getByName(address));
}
- public void addExpireTimeForEndpoint(InetAddress endpoint, long expireTime)
+ public void addExpireTimeForEndpoint(InetAddressAndPort endpoint, long expireTime)
{
if (logger.isDebugEnabled())
{
@@ -1754,14 +1915,35 @@
}
@Nullable
- public CassandraVersion getReleaseVersion(InetAddress ep)
+ public CassandraVersion getReleaseVersion(InetAddressAndPort ep)
{
EndpointState state = getEndpointStateForEndpoint(ep);
return state != null ? state.getReleaseVersion() : null;
}
+ public Map<String, List<String>> getReleaseVersionsWithPort()
+ {
+ Map<String, List<String>> results = new HashMap<String, List<String>>();
+ Iterable<InetAddressAndPort> allHosts = Iterables.concat(Gossiper.instance.getLiveMembers(), Gossiper.instance.getUnreachableMembers());
+
+ for (InetAddressAndPort host : allHosts)
+ {
+ CassandraVersion version = getReleaseVersion(host);
+ String stringVersion = version == null ? "" : version.toString();
+ List<String> hosts = results.get(stringVersion);
+ if (hosts == null)
+ {
+ hosts = new ArrayList<>();
+ results.put(stringVersion, hosts);
+ }
+ hosts.add(host.getHostAddress(true));
+ }
+
+ return results;
+ }
+
@Nullable
- public UUID getSchemaVersion(InetAddress ep)
+ public UUID getSchemaVersion(InetAddressAndPort ep)
{
EndpointState state = getEndpointStateForEndpoint(ep);
return state != null ? state.getSchemaVersion() : null;
@@ -1782,11 +1964,11 @@
Uninterruptibles.sleepUninterruptibly(GOSSIP_SETTLE_MIN_WAIT_MS, TimeUnit.MILLISECONDS);
int totalPolls = 0;
int numOkay = 0;
- int epSize = Gossiper.instance.getEndpointStates().size();
+ int epSize = Gossiper.instance.getEndpointCount();
while (numOkay < GOSSIP_SETTLE_POLL_SUCCESSES_REQUIRED)
{
Uninterruptibles.sleepUninterruptibly(GOSSIP_SETTLE_POLL_INTERVAL_MS, TimeUnit.MILLISECONDS);
- int currentSize = Gossiper.instance.getEndpointStates().size();
+ int currentSize = Gossiper.instance.getEndpointCount();
totalPolls++;
if (currentSize == epSize)
{
@@ -1812,6 +1994,58 @@
logger.info("No gossip backlog; proceeding");
}
+ /**
+ * Blockingly wait for all live nodes to agree on the current schema version.
+ *
+ * @param maxWait maximum time to wait for schema agreement
+ * @param unit TimeUnit of maxWait
+ * @return true if agreement was reached, false if not
+ */
+ public boolean waitForSchemaAgreement(long maxWait, TimeUnit unit, BooleanSupplier abortCondition)
+ {
+ int waited = 0;
+ int toWait = 50;
+
+ Set<InetAddressAndPort> members = getLiveTokenOwners();
+
+ while (true)
+ {
+ if (nodesAgreeOnSchema(members))
+ return true;
+
+ if (waited >= unit.toMillis(maxWait) || abortCondition.getAsBoolean())
+ return false;
+
+ Uninterruptibles.sleepUninterruptibly(toWait, TimeUnit.MILLISECONDS);
+ waited += toWait;
+ toWait = Math.min(1000, toWait * 2);
+ }
+ }
+
+ public boolean haveMajorVersion3Nodes()
+ {
+ return haveMajorVersion3NodesMemoized.get();
+ }
+
+ private boolean nodesAgreeOnSchema(Collection<InetAddressAndPort> nodes)
+ {
+ UUID expectedVersion = null;
+
+ for (InetAddressAndPort node : nodes)
+ {
+ EndpointState state = getEndpointStateForEndpoint(node);
+ UUID remoteVersion = state.getSchemaVersion();
+
+ if (null == expectedVersion)
+ expectedVersion = remoteVersion;
+
+ if (null == expectedVersion || !expectedVersion.equals(remoteVersion))
+ return false;
+ }
+
+ return true;
+ }
+
@VisibleForTesting
public void stopShutdownAndWait(long timeout, TimeUnit unit) throws InterruptedException, TimeoutException
{
diff --git a/src/java/org/apache/cassandra/gms/GossiperDiagnostics.java b/src/java/org/apache/cassandra/gms/GossiperDiagnostics.java
new file mode 100644
index 0000000..57552cc
--- /dev/null
+++ b/src/java/org/apache/cassandra/gms/GossiperDiagnostics.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.gms;
+
+
+import org.apache.cassandra.diag.DiagnosticEventService;
+import org.apache.cassandra.gms.GossiperEvent.GossiperEventType;
+import org.apache.cassandra.locator.InetAddressAndPort;
+
+/**
+ * Utility methods for DiagnosticEvent activities.
+ */
+final class GossiperDiagnostics
+{
+ private static final DiagnosticEventService service = DiagnosticEventService.instance();
+
+ private GossiperDiagnostics()
+ {
+ }
+
+ static void markedAsShutdown(Gossiper gossiper, InetAddressAndPort endpoint)
+ {
+ if (isEnabled(GossiperEventType.MARKED_AS_SHUTDOWN))
+ service.publish(new GossiperEvent(GossiperEventType.MARKED_AS_SHUTDOWN, gossiper, endpoint, null, null));
+ }
+
+ static void convicted(Gossiper gossiper, InetAddressAndPort endpoint, double phi)
+ {
+ if (isEnabled(GossiperEventType.CONVICTED))
+ service.publish(new GossiperEvent(GossiperEventType.CONVICTED, gossiper, endpoint, null, null));
+ }
+
+ static void replacementQuarantine(Gossiper gossiper, InetAddressAndPort endpoint)
+ {
+ if (isEnabled(GossiperEventType.REPLACEMENT_QUARANTINE))
+ service.publish(new GossiperEvent(GossiperEventType.REPLACEMENT_QUARANTINE, gossiper, endpoint, null, null));
+ }
+
+ static void replacedEndpoint(Gossiper gossiper, InetAddressAndPort endpoint)
+ {
+ if (isEnabled(GossiperEventType.REPLACED_ENDPOINT))
+ service.publish(new GossiperEvent(GossiperEventType.REPLACED_ENDPOINT, gossiper, endpoint, null, null));
+ }
+
+ static void evictedFromMembership(Gossiper gossiper, InetAddressAndPort endpoint)
+ {
+ if (isEnabled(GossiperEventType.EVICTED_FROM_MEMBERSHIP))
+ service.publish(new GossiperEvent(GossiperEventType.EVICTED_FROM_MEMBERSHIP, gossiper, endpoint, null, null));
+ }
+
+ static void removedEndpoint(Gossiper gossiper, InetAddressAndPort endpoint)
+ {
+ if (isEnabled(GossiperEventType.REMOVED_ENDPOINT))
+ service.publish(new GossiperEvent(GossiperEventType.REMOVED_ENDPOINT, gossiper, endpoint, null, null));
+ }
+
+ static void quarantinedEndpoint(Gossiper gossiper, InetAddressAndPort endpoint, long quarantineExpiration)
+ {
+ if (isEnabled(GossiperEventType.QUARANTINED_ENDPOINT))
+ service.publish(new GossiperEvent(GossiperEventType.QUARANTINED_ENDPOINT, gossiper, endpoint, quarantineExpiration, null));
+ }
+
+ static void markedAlive(Gossiper gossiper, InetAddressAndPort addr, EndpointState localState)
+ {
+ if (isEnabled(GossiperEventType.MARKED_ALIVE))
+ service.publish(new GossiperEvent(GossiperEventType.MARKED_ALIVE, gossiper, addr, null, localState));
+ }
+
+ static void realMarkedAlive(Gossiper gossiper, InetAddressAndPort addr, EndpointState localState)
+ {
+ if (isEnabled(GossiperEventType.REAL_MARKED_ALIVE))
+ service.publish(new GossiperEvent(GossiperEventType.REAL_MARKED_ALIVE, gossiper, addr, null, localState));
+ }
+
+ static void markedDead(Gossiper gossiper, InetAddressAndPort addr, EndpointState localState)
+ {
+ if (isEnabled(GossiperEventType.MARKED_DEAD))
+ service.publish(new GossiperEvent(GossiperEventType.MARKED_DEAD, gossiper, addr, null, localState));
+ }
+
+ static void majorStateChangeHandled(Gossiper gossiper, InetAddressAndPort addr, EndpointState state)
+ {
+ if (isEnabled(GossiperEventType.MAJOR_STATE_CHANGE_HANDLED))
+ service.publish(new GossiperEvent(GossiperEventType.MAJOR_STATE_CHANGE_HANDLED, gossiper, addr, null, state));
+ }
+
+ static void sendGossipDigestSyn(Gossiper gossiper, InetAddressAndPort to)
+ {
+ if (isEnabled(GossiperEventType.SEND_GOSSIP_DIGEST_SYN))
+ service.publish(new GossiperEvent(GossiperEventType.SEND_GOSSIP_DIGEST_SYN, gossiper, to, null, null));
+ }
+
+ private static boolean isEnabled(GossiperEventType type)
+ {
+ return service.isEnabled(GossiperEvent.class, type);
+ }
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/gms/GossiperEvent.java b/src/java/org/apache/cassandra/gms/GossiperEvent.java
new file mode 100644
index 0000000..ef7bd8d
--- /dev/null
+++ b/src/java/org/apache/cassandra/gms/GossiperEvent.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.gms;
+
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import javax.annotation.Nullable;
+
+import org.apache.cassandra.diag.DiagnosticEvent;
+import org.apache.cassandra.locator.InetAddressAndPort;
+
+/**
+ * DiagnosticEvent implementation for {@link Gossiper} activities.
+ */
+public final class GossiperEvent extends DiagnosticEvent
+{
+ private final InetAddressAndPort endpoint;
+ @Nullable
+ private final Long quarantineExpiration;
+ @Nullable
+ private final EndpointState localState;
+
+ private final Map<InetAddressAndPort, EndpointState> endpointStateMap;
+ private final boolean inShadowRound;
+ private final Map<InetAddressAndPort, Long> justRemovedEndpoints;
+ private final long lastProcessedMessageAt;
+ private final Set<InetAddressAndPort> liveEndpoints;
+ private final List<String> seeds;
+ private final Set<InetAddressAndPort> seedsInShadowRound;
+ private final Map<InetAddressAndPort, Long> unreachableEndpoints;
+
+
+ public enum GossiperEventType
+ {
+ MARKED_AS_SHUTDOWN,
+ CONVICTED,
+ REPLACEMENT_QUARANTINE,
+ REPLACED_ENDPOINT,
+ EVICTED_FROM_MEMBERSHIP,
+ REMOVED_ENDPOINT,
+ QUARANTINED_ENDPOINT,
+ MARKED_ALIVE,
+ REAL_MARKED_ALIVE,
+ MARKED_DEAD,
+ MAJOR_STATE_CHANGE_HANDLED,
+ SEND_GOSSIP_DIGEST_SYN
+ }
+
+ public GossiperEventType type;
+
+
+ GossiperEvent(GossiperEventType type, Gossiper gossiper, InetAddressAndPort endpoint,
+ @Nullable Long quarantineExpiration, @Nullable EndpointState localState)
+ {
+ this.type = type;
+ this.endpoint = endpoint;
+ this.quarantineExpiration = quarantineExpiration;
+ this.localState = localState;
+
+ this.endpointStateMap = gossiper.getEndpointStateMap();
+ this.inShadowRound = gossiper.isInShadowRound();
+ this.justRemovedEndpoints = gossiper.getJustRemovedEndpoints();
+ this.lastProcessedMessageAt = gossiper.getLastProcessedMessageAt();
+ this.liveEndpoints = gossiper.getLiveMembers();
+ this.seeds = gossiper.getSeeds();
+ this.seedsInShadowRound = gossiper.getSeedsInShadowRound();
+ this.unreachableEndpoints = gossiper.getUnreachableEndpoints();
+ }
+
+ public Enum<GossiperEventType> getType()
+ {
+ return type;
+ }
+
+ public HashMap<String, Serializable> toMap()
+ {
+ // be extra defensive against nulls and bugs
+ HashMap<String, Serializable> ret = new HashMap<>();
+ if (endpoint != null) ret.put("endpoint", endpoint.getHostAddress(true));
+ ret.put("quarantineExpiration", quarantineExpiration);
+ ret.put("localState", String.valueOf(localState));
+ ret.put("endpointStateMap", String.valueOf(endpointStateMap));
+ ret.put("inShadowRound", inShadowRound);
+ ret.put("justRemovedEndpoints", String.valueOf(justRemovedEndpoints));
+ ret.put("lastProcessedMessageAt", lastProcessedMessageAt);
+ ret.put("liveEndpoints", String.valueOf(liveEndpoints));
+ ret.put("seeds", String.valueOf(seeds));
+ ret.put("seedsInShadowRound", String.valueOf(seedsInShadowRound));
+ ret.put("unreachableEndpoints", String.valueOf(unreachableEndpoints));
+ return ret;
+ }
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/gms/GossiperMBean.java b/src/java/org/apache/cassandra/gms/GossiperMBean.java
index c4b244c..92df2cd 100644
--- a/src/java/org/apache/cassandra/gms/GossiperMBean.java
+++ b/src/java/org/apache/cassandra/gms/GossiperMBean.java
@@ -18,6 +18,8 @@
package org.apache.cassandra.gms;
import java.net.UnknownHostException;
+import java.util.List;
+import java.util.Map;
public interface GossiperMBean
{
@@ -29,4 +31,11 @@
public void assassinateEndpoint(String address) throws UnknownHostException;
+ public List<String> reloadSeeds();
+
+ public List<String> getSeeds();
+
+ /** Returns each node's database release version */
+ public Map<String, List<String>> getReleaseVersionsWithPort();
+
}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/gms/HeartBeatState.java b/src/java/org/apache/cassandra/gms/HeartBeatState.java
index 13e1ace..2abd5d7 100644
--- a/src/java/org/apache/cassandra/gms/HeartBeatState.java
+++ b/src/java/org/apache/cassandra/gms/HeartBeatState.java
@@ -27,7 +27,7 @@
/**
* HeartBeat State associated with any given endpoint.
*/
-class HeartBeatState
+public class HeartBeatState
{
public static final IVersionedSerializer<HeartBeatState> serializer = new HeartBeatStateSerializer();
@@ -39,7 +39,7 @@
this(gen, 0);
}
- HeartBeatState(int gen, int ver)
+ public HeartBeatState(int gen, int ver)
{
generation = gen;
version = ver;
diff --git a/src/java/org/apache/cassandra/gms/IEndpointStateChangeSubscriber.java b/src/java/org/apache/cassandra/gms/IEndpointStateChangeSubscriber.java
index 1bfd678..dc81650 100644
--- a/src/java/org/apache/cassandra/gms/IEndpointStateChangeSubscriber.java
+++ b/src/java/org/apache/cassandra/gms/IEndpointStateChangeSubscriber.java
@@ -17,7 +17,7 @@
*/
package org.apache.cassandra.gms;
-import java.net.InetAddress;
+import org.apache.cassandra.locator.InetAddressAndPort;
/**
* This is called by an instance of the IEndpointStateChangePublisher to notify
@@ -36,17 +36,17 @@
* @param endpoint endpoint for which the state change occurred.
* @param epState state that actually changed for the above endpoint.
*/
- public void onJoin(InetAddress endpoint, EndpointState epState);
+ public void onJoin(InetAddressAndPort endpoint, EndpointState epState);
- public void beforeChange(InetAddress endpoint, EndpointState currentState, ApplicationState newStateKey, VersionedValue newValue);
+ public void beforeChange(InetAddressAndPort endpoint, EndpointState currentState, ApplicationState newStateKey, VersionedValue newValue);
- public void onChange(InetAddress endpoint, ApplicationState state, VersionedValue value);
+ public void onChange(InetAddressAndPort endpoint, ApplicationState state, VersionedValue value);
- public void onAlive(InetAddress endpoint, EndpointState state);
+ public void onAlive(InetAddressAndPort endpoint, EndpointState state);
- public void onDead(InetAddress endpoint, EndpointState state);
+ public void onDead(InetAddressAndPort endpoint, EndpointState state);
- public void onRemove(InetAddress endpoint);
+ public void onRemove(InetAddressAndPort endpoint);
/**
* Called whenever a node is restarted.
@@ -54,5 +54,5 @@
* previously marked down. It will have only if {@code state.isAlive() == false}
* as {@code state} is from before the restarted node is marked up.
*/
- public void onRestart(InetAddress endpoint, EndpointState state);
+ public void onRestart(InetAddressAndPort endpoint, EndpointState state);
}
diff --git a/src/java/org/apache/cassandra/gms/IFailureDetectionEventListener.java b/src/java/org/apache/cassandra/gms/IFailureDetectionEventListener.java
index 8b274b6..4e0c663 100644
--- a/src/java/org/apache/cassandra/gms/IFailureDetectionEventListener.java
+++ b/src/java/org/apache/cassandra/gms/IFailureDetectionEventListener.java
@@ -17,7 +17,7 @@
*/
package org.apache.cassandra.gms;
-import java.net.InetAddress;
+import org.apache.cassandra.locator.InetAddressAndPort;
/**
* Implemented by the Gossiper to convict an endpoint
@@ -33,5 +33,5 @@
* @param ep endpoint to be convicted
* @param phi the value of phi with with ep was convicted
*/
- public void convict(InetAddress ep, double phi);
+ public void convict(InetAddressAndPort ep, double phi);
}
diff --git a/src/java/org/apache/cassandra/gms/IFailureDetector.java b/src/java/org/apache/cassandra/gms/IFailureDetector.java
index a860c7c..62fc97d 100644
--- a/src/java/org/apache/cassandra/gms/IFailureDetector.java
+++ b/src/java/org/apache/cassandra/gms/IFailureDetector.java
@@ -17,7 +17,7 @@
*/
package org.apache.cassandra.gms;
-import java.net.InetAddress;
+import org.apache.cassandra.locator.InetAddressAndPort;
/**
* An interface that provides an application with the ability
@@ -35,7 +35,7 @@
* @param ep endpoint in question.
* @return true if UP and false if DOWN.
*/
- public boolean isAlive(InetAddress ep);
+ public boolean isAlive(InetAddressAndPort ep);
/**
* This method is invoked by any entity wanting to interrogate the status of an endpoint.
@@ -44,7 +44,7 @@
*
* param ep endpoint for which we interpret the inter arrival times.
*/
- public void interpret(InetAddress ep);
+ public void interpret(InetAddressAndPort ep);
/**
* This method is invoked by the receiver of the heartbeat. In our case it would be
@@ -53,17 +53,17 @@
*
* param ep endpoint being reported.
*/
- public void report(InetAddress ep);
+ public void report(InetAddressAndPort ep);
/**
* remove endpoint from failure detector
*/
- public void remove(InetAddress ep);
+ public void remove(InetAddressAndPort ep);
/**
* force conviction of endpoint in the failure detector
*/
- public void forceConviction(InetAddress ep);
+ public void forceConviction(InetAddressAndPort ep);
/**
* Register interest for Failure Detector events.
diff --git a/src/java/org/apache/cassandra/gms/TokenSerializer.java b/src/java/org/apache/cassandra/gms/TokenSerializer.java
index 41bd821..c371d64 100644
--- a/src/java/org/apache/cassandra/gms/TokenSerializer.java
+++ b/src/java/org/apache/cassandra/gms/TokenSerializer.java
@@ -54,7 +54,8 @@
int size = in.readInt();
if (size < 1)
break;
- logger.trace("Reading token of {}", FBUtilities.prettyPrintMemory(size));
+ if (logger.isTraceEnabled())
+ logger.trace("Reading token of {}", FBUtilities.prettyPrintMemory(size));
byte[] bintoken = new byte[size];
in.readFully(bintoken);
tokens.add(partitioner.getTokenFactory().fromByteArray(ByteBuffer.wrap(bintoken)));
diff --git a/src/java/org/apache/cassandra/gms/VersionedValue.java b/src/java/org/apache/cassandra/gms/VersionedValue.java
index 0ec1712..94b8cb8 100644
--- a/src/java/org/apache/cassandra/gms/VersionedValue.java
+++ b/src/java/org/apache/cassandra/gms/VersionedValue.java
@@ -33,6 +33,7 @@
import org.apache.cassandra.io.IVersionedSerializer;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.commons.lang3.StringUtils;
@@ -134,11 +135,17 @@
return new VersionedValue(value.value);
}
+ @Deprecated
public VersionedValue bootReplacing(InetAddress oldNode)
{
return new VersionedValue(versionString(VersionedValue.STATUS_BOOTSTRAPPING_REPLACE, oldNode.getHostAddress()));
}
+ public VersionedValue bootReplacingWithPort(InetAddressAndPort oldNode)
+ {
+ return new VersionedValue(versionString(VersionedValue.STATUS_BOOTSTRAPPING_REPLACE, oldNode.toString()));
+ }
+
public VersionedValue bootstrapping(Collection<Token> tokens)
{
return new VersionedValue(versionString(VersionedValue.STATUS_BOOTSTRAPPING,
@@ -249,9 +256,14 @@
return new VersionedValue(endpoint.getHostAddress());
}
+ public VersionedValue nativeaddressAndPort(InetAddressAndPort address)
+ {
+ return new VersionedValue(address.toString());
+ }
+
public VersionedValue releaseVersion()
{
- return releaseVersion(FBUtilities.getReleaseVersionString());
+ return new VersionedValue(FBUtilities.getReleaseVersionString());
}
@VisibleForTesting
@@ -270,6 +282,11 @@
return new VersionedValue(private_ip);
}
+ public VersionedValue internalAddressAndPort(InetAddressAndPort address)
+ {
+ return new VersionedValue(address.toString());
+ }
+
public VersionedValue severity(double value)
{
return new VersionedValue(String.valueOf(value));
diff --git a/src/java/org/apache/cassandra/hadoop/ConfigHelper.java b/src/java/org/apache/cassandra/hadoop/ConfigHelper.java
index a4deb4a..f01197d 100644
--- a/src/java/org/apache/cassandra/hadoop/ConfigHelper.java
+++ b/src/java/org/apache/cassandra/hadoop/ConfigHelper.java
@@ -20,24 +20,15 @@
*/
package org.apache.cassandra.hadoop;
-import java.io.IOException;
-import java.util.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.schema.CompressionParams;
-import org.apache.cassandra.thrift.*;
import org.apache.cassandra.utils.FBUtilities;
-import org.apache.cassandra.utils.Hex;
+import org.apache.cassandra.utils.Pair;
import org.apache.hadoop.conf.Configuration;
-import org.apache.thrift.TBase;
-import org.apache.thrift.TDeserializer;
-import org.apache.thrift.TException;
-import org.apache.thrift.TSerializer;
-import org.apache.thrift.protocol.TBinaryProtocol;
-import org.apache.thrift.transport.TTransport;
public class ConfigHelper
{
@@ -59,16 +50,15 @@
private static final int DEFAULT_SPLIT_SIZE = 64 * 1024;
private static final String RANGE_BATCH_SIZE_CONFIG = "cassandra.range.batch.size";
private static final int DEFAULT_RANGE_BATCH_SIZE = 4096;
- private static final String INPUT_THRIFT_PORT = "cassandra.input.thrift.port";
- private static final String OUTPUT_THRIFT_PORT = "cassandra.output.thrift.port";
- private static final String INPUT_INITIAL_THRIFT_ADDRESS = "cassandra.input.thrift.address";
- private static final String OUTPUT_INITIAL_THRIFT_ADDRESS = "cassandra.output.thrift.address";
+ private static final String INPUT_INITIAL_ADDRESS = "cassandra.input.address";
+ private static final String OUTPUT_INITIAL_ADDRESS = "cassandra.output.address";
+ private static final String OUTPUT_INITIAL_PORT = "cassandra.output.port";
private static final String READ_CONSISTENCY_LEVEL = "cassandra.consistencylevel.read";
private static final String WRITE_CONSISTENCY_LEVEL = "cassandra.consistencylevel.write";
private static final String OUTPUT_COMPRESSION_CLASS = "cassandra.output.compression.class";
private static final String OUTPUT_COMPRESSION_CHUNK_LENGTH = "cassandra.output.compression.length";
private static final String OUTPUT_LOCAL_DC_ONLY = "cassandra.output.local.dc.only";
- private static final String THRIFT_FRAMED_TRANSPORT_SIZE_IN_MB = "cassandra.thrift.framed.size_mb";
+ private static final String DEFAULT_CASSANDRA_NATIVE_PORT = "7000";
private static final Logger logger = LoggerFactory.getLogger(ConfigHelper.class);
@@ -213,104 +203,28 @@
}
/**
- * Set the predicate that determines what columns will be selected from each row.
- *
- * @param conf Job configuration you are about to run
- * @param predicate
- */
- public static void setInputSlicePredicate(Configuration conf, SlicePredicate predicate)
- {
- conf.set(INPUT_PREDICATE_CONFIG, thriftToString(predicate));
- }
-
- public static SlicePredicate getInputSlicePredicate(Configuration conf)
- {
- String s = conf.get(INPUT_PREDICATE_CONFIG);
- return s == null ? null : predicateFromString(s);
- }
-
- private static String thriftToString(TBase object)
- {
- assert object != null;
- // this is so awful it's kind of cool!
- TSerializer serializer = new TSerializer(new TBinaryProtocol.Factory());
- try
- {
- return Hex.bytesToHex(serializer.serialize(object));
- }
- catch (TException e)
- {
- throw new RuntimeException(e);
- }
- }
-
- private static SlicePredicate predicateFromString(String st)
- {
- assert st != null;
- TDeserializer deserializer = new TDeserializer(new TBinaryProtocol.Factory());
- SlicePredicate predicate = new SlicePredicate();
- try
- {
- deserializer.deserialize(predicate, Hex.hexToBytes(st));
- }
- catch (TException e)
- {
- throw new RuntimeException(e);
- }
- return predicate;
- }
-
- /**
* Set the KeyRange to limit the rows.
* @param conf Job configuration you are about to run
*/
public static void setInputRange(Configuration conf, String startToken, String endToken)
{
- KeyRange range = new KeyRange().setStart_token(startToken).setEnd_token(endToken);
- conf.set(INPUT_KEYRANGE_CONFIG, thriftToString(range));
+ conf.set(INPUT_KEYRANGE_CONFIG, startToken + "," + endToken);
}
/**
- * Set the KeyRange to limit the rows.
- * @param conf Job configuration you are about to run
+ * The start and end token of the input key range as a pair.
+ *
+ * may be null if unset.
*/
- public static void setInputRange(Configuration conf, String startToken, String endToken, List<IndexExpression> filter)
- {
- KeyRange range = new KeyRange().setStart_token(startToken).setEnd_token(endToken).setRow_filter(filter);
- conf.set(INPUT_KEYRANGE_CONFIG, thriftToString(range));
- }
-
- /**
- * Set the KeyRange to limit the rows.
- * @param conf Job configuration you are about to run
- */
- public static void setInputRange(Configuration conf, List<IndexExpression> filter)
- {
- KeyRange range = new KeyRange().setRow_filter(filter);
- conf.set(INPUT_KEYRANGE_CONFIG, thriftToString(range));
- }
-
- /** may be null if unset */
- public static KeyRange getInputKeyRange(Configuration conf)
+ public static Pair<String, String> getInputKeyRange(Configuration conf)
{
String str = conf.get(INPUT_KEYRANGE_CONFIG);
- return str == null ? null : keyRangeFromString(str);
- }
+ if (str == null)
+ return null;
- private static KeyRange keyRangeFromString(String st)
- {
- assert st != null;
- TDeserializer deserializer = new TDeserializer(new TBinaryProtocol.Factory());
- KeyRange keyRange = new KeyRange();
- try
- {
- deserializer.deserialize(keyRange, Hex.hexToBytes(st));
- }
- catch (TException e)
- {
- throw new RuntimeException(e);
- }
- return keyRange;
+ String[] parts = str.split(",");
+ assert parts.length == 2;
+ return Pair.create(parts[0], parts[1]);
}
public static String getInputKeyspace(Configuration conf)
@@ -413,26 +327,15 @@
conf.set(WRITE_CONSISTENCY_LEVEL, consistencyLevel);
}
- public static int getInputRpcPort(Configuration conf)
- {
- return Integer.parseInt(conf.get(INPUT_THRIFT_PORT, "9160"));
- }
-
- public static void setInputRpcPort(Configuration conf, String port)
- {
- conf.set(INPUT_THRIFT_PORT, port);
- }
-
public static String getInputInitialAddress(Configuration conf)
{
- return conf.get(INPUT_INITIAL_THRIFT_ADDRESS);
+ return conf.get(INPUT_INITIAL_ADDRESS);
}
public static void setInputInitialAddress(Configuration conf, String address)
{
- conf.set(INPUT_INITIAL_THRIFT_ADDRESS, address);
+ conf.set(INPUT_INITIAL_ADDRESS, address);
}
-
public static void setInputPartitioner(Configuration conf, String classname)
{
conf.set(INPUT_PARTITIONER_CONFIG, classname);
@@ -443,24 +346,24 @@
return FBUtilities.newPartitioner(conf.get(INPUT_PARTITIONER_CONFIG));
}
- public static int getOutputRpcPort(Configuration conf)
- {
- return Integer.parseInt(conf.get(OUTPUT_THRIFT_PORT, "9160"));
- }
-
- public static void setOutputRpcPort(Configuration conf, String port)
- {
- conf.set(OUTPUT_THRIFT_PORT, port);
- }
-
public static String getOutputInitialAddress(Configuration conf)
{
- return conf.get(OUTPUT_INITIAL_THRIFT_ADDRESS);
+ return conf.get(OUTPUT_INITIAL_ADDRESS);
+ }
+
+ public static void setOutputInitialPort(Configuration conf, Integer port)
+ {
+ conf.set(OUTPUT_INITIAL_PORT, port.toString());
+ }
+
+ public static Integer getOutputInitialPort(Configuration conf)
+ {
+ return Integer.valueOf(conf.get(OUTPUT_INITIAL_PORT, DEFAULT_CASSANDRA_NATIVE_PORT));
}
public static void setOutputInitialAddress(Configuration conf, String address)
{
- conf.set(OUTPUT_INITIAL_THRIFT_ADDRESS, address);
+ conf.set(OUTPUT_INITIAL_ADDRESS, address);
}
public static void setOutputPartitioner(Configuration conf, String classname)
@@ -493,20 +396,6 @@
conf.set(OUTPUT_COMPRESSION_CHUNK_LENGTH, length);
}
- public static void setThriftFramedTransportSizeInMb(Configuration conf, int frameSizeInMB)
- {
- conf.setInt(THRIFT_FRAMED_TRANSPORT_SIZE_IN_MB, frameSizeInMB);
- }
-
- /**
- * @param conf The configuration to use.
- * @return Value (converts MBs to Bytes) set by {@link #setThriftFramedTransportSizeInMb(Configuration, int)} or default of 15MB
- */
- public static int getThriftFramedTransportSize(Configuration conf)
- {
- return conf.getInt(THRIFT_FRAMED_TRANSPORT_SIZE_IN_MB, 15) * 1024 * 1024; // 15MB is default in Cassandra
- }
-
public static boolean getOutputLocalDCOnly(Configuration conf)
{
return Boolean.parseBoolean(conf.get(OUTPUT_LOCAL_DC_ONLY, "false"));
@@ -516,89 +405,4 @@
{
conf.set(OUTPUT_LOCAL_DC_ONLY, Boolean.toString(localDCOnly));
}
-
- public static Cassandra.Client getClientFromInputAddressList(Configuration conf) throws IOException
- {
- return getClientFromAddressList(conf, ConfigHelper.getInputInitialAddress(conf).split(","), ConfigHelper.getInputRpcPort(conf));
- }
-
- public static Cassandra.Client getClientFromOutputAddressList(Configuration conf) throws IOException
- {
- return getClientFromAddressList(conf, ConfigHelper.getOutputInitialAddress(conf).split(","), ConfigHelper.getOutputRpcPort(conf));
- }
-
- private static Cassandra.Client getClientFromAddressList(Configuration conf, String[] addresses, int port) throws IOException
- {
- Cassandra.Client client = null;
- List<IOException> exceptions = new ArrayList<IOException>();
- for (String address : addresses)
- {
- try
- {
- client = createConnection(conf, address, port);
- break;
- }
- catch (IOException ioe)
- {
- exceptions.add(ioe);
- }
- }
- if (client == null)
- {
- logger.error("failed to connect to any initial addresses");
- for (IOException ioe : exceptions)
- {
- logger.error("", ioe);
- }
- throw exceptions.get(exceptions.size() - 1);
- }
- return client;
- }
-
- @SuppressWarnings("resource")
- public static Cassandra.Client createConnection(Configuration conf, String host, Integer port) throws IOException
- {
- try
- {
- TTransport transport = getClientTransportFactory(conf).openTransport(host, port);
- return new Cassandra.Client(new TBinaryProtocol(transport, true, true));
- }
- catch (Exception e)
- {
- throw new IOException("Unable to connect to server " + host + ":" + port, e);
- }
- }
-
- public static ITransportFactory getClientTransportFactory(Configuration conf)
- {
- String factoryClassName = conf.get(ITransportFactory.PROPERTY_KEY, TFramedTransportFactory.class.getName());
- ITransportFactory factory = getClientTransportFactory(factoryClassName);
- Map<String, String> options = getOptions(conf, factory.supportedOptions());
- factory.setOptions(options);
- return factory;
- }
-
- private static ITransportFactory getClientTransportFactory(String factoryClassName)
- {
- try
- {
- return (ITransportFactory) Class.forName(factoryClassName).newInstance();
- }
- catch (Exception e)
- {
- throw new RuntimeException("Failed to instantiate transport factory:" + factoryClassName, e);
- }
- }
-
- private static Map<String, String> getOptions(Configuration conf, Set<String> supportedOptions)
- {
- Map<String, String> options = new HashMap<>();
- for (String optionKey : supportedOptions)
- {
- String optionValue = conf.get(optionKey);
- if (optionValue != null)
- options.put(optionKey, optionValue);
- }
- return options;
- }
}
diff --git a/src/java/org/apache/cassandra/hadoop/cql3/CqlBulkRecordWriter.java b/src/java/org/apache/cassandra/hadoop/cql3/CqlBulkRecordWriter.java
index fd9ed00..77ad95f 100644
--- a/src/java/org/apache/cassandra/hadoop/cql3/CqlBulkRecordWriter.java
+++ b/src/java/org/apache/cassandra/hadoop/cql3/CqlBulkRecordWriter.java
@@ -21,16 +21,18 @@
import java.io.File;
import java.io.IOException;
import java.net.InetAddress;
+import java.net.InetSocketAddress;
import java.net.UnknownHostException;
import java.nio.ByteBuffer;
import java.util.*;
import java.util.concurrent.*;
+import com.google.common.net.HostAndPort;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.Config;
+
import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.statements.schema.CreateTableStatement;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.Murmur3Partitioner;
import org.apache.cassandra.exceptions.InvalidRequestException;
@@ -39,6 +41,8 @@
import org.apache.cassandra.io.sstable.CQLSSTableWriter;
import org.apache.cassandra.io.sstable.SSTableLoader;
import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.schema.TableMetadataRef;
import org.apache.cassandra.streaming.StreamState;
import org.apache.cassandra.utils.NativeSSTableLoaderClient;
import org.apache.cassandra.utils.OutputHandler;
@@ -79,7 +83,7 @@
protected SSTableLoader loader;
protected Progressable progress;
protected TaskAttemptContext context;
- protected final Set<InetAddress> ignores = new HashSet<>();
+ protected final Set<InetAddressAndPort> ignores = new HashSet<>();
private String keyspace;
private String table;
@@ -138,7 +142,7 @@
try
{
for (String hostToIgnore : CqlBulkOutputFormat.getIgnoreHosts(conf))
- ignores.add(InetAddress.getByName(hostToIgnore));
+ ignores.add(InetAddressAndPort.getByName(hostToIgnore));
}
catch (UnknownHostException e)
{
@@ -171,7 +175,7 @@
if (loader == null)
{
ExternalClient externalClient = new ExternalClient(conf);
- externalClient.setTableMetadata(CFMetaData.compile(schema, keyspace));
+ externalClient.setTableMetadata(TableMetadataRef.forOfflineTools(CreateTableStatement.parse(schema, keyspace).build()));
loader = new SSTableLoader(outputDir, externalClient, new NullOutputHandler())
{
@@ -283,21 +287,22 @@
public ExternalClient(Configuration conf)
{
super(resolveHostAddresses(conf),
- CqlConfigHelper.getOutputNativePort(conf),
+ ConfigHelper.getOutputInitialPort(conf),
ConfigHelper.getOutputKeyspaceUserName(conf),
ConfigHelper.getOutputKeyspacePassword(conf),
CqlConfigHelper.getSSLOptions(conf).orNull());
}
- private static Collection<InetAddress> resolveHostAddresses(Configuration conf)
+ private static Collection<InetSocketAddress> resolveHostAddresses(Configuration conf)
{
- Set<InetAddress> addresses = new HashSet<>();
-
+ Set<InetSocketAddress> addresses = new HashSet<>();
+ int port = CqlConfigHelper.getOutputNativePort(conf);
for (String host : ConfigHelper.getOutputInitialAddress(conf).split(","))
{
try
{
- addresses.add(InetAddress.getByName(host));
+ HostAndPort hap = HostAndPort.fromString(host);
+ addresses.add(new InetSocketAddress(InetAddress.getByName(hap.getHost()), hap.getPortOrDefault(port)));
}
catch (UnknownHostException e)
{
diff --git a/src/java/org/apache/cassandra/hadoop/cql3/CqlClientHelper.java b/src/java/org/apache/cassandra/hadoop/cql3/CqlClientHelper.java
new file mode 100644
index 0000000..d154243
--- /dev/null
+++ b/src/java/org/apache/cassandra/hadoop/cql3/CqlClientHelper.java
@@ -0,0 +1,91 @@
+package org.apache.cassandra.hadoop.cql3;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+import com.datastax.driver.core.Host;
+import com.datastax.driver.core.Metadata;
+import com.datastax.driver.core.Token;
+import com.datastax.driver.core.TokenRange;
+
+public class CqlClientHelper
+{
+ private CqlClientHelper()
+ {
+ }
+
+ public static Map<TokenRange, List<Host>> getLocalPrimaryRangeForDC(String keyspace, Metadata metadata, String targetDC)
+ {
+ Objects.requireNonNull(keyspace, "keyspace");
+ Objects.requireNonNull(metadata, "metadata");
+ Objects.requireNonNull(targetDC, "targetDC");
+
+ // In 2.1 the logic was to have a set of nodes used as a seed, they were used to query
+ // client.describe_local_ring(keyspace) -> List<TokenRange>; this should include all nodes in the local dc.
+ // TokenRange contained the endpoints in order, so .endpoints.get(0) is the primary owner
+ // Client does not have a similar API, instead it returns Set<Host>. To replicate this we first need
+ // to compute the primary owners, then add in the replicas
+
+ List<Token> tokens = new ArrayList<>();
+ Map<Token, Host> tokenToHost = new HashMap<>();
+ for (Host host : metadata.getAllHosts())
+ {
+ if (!targetDC.equals(host.getDatacenter()))
+ continue;
+
+ for (Token token : host.getTokens())
+ {
+ Host previous = tokenToHost.putIfAbsent(token, host);
+ if (previous != null)
+ throw new IllegalStateException("Two hosts share the same token; hosts " + host.getHostId() + ":"
+ + host.getTokens() + ", " + previous.getHostId() + ":" + previous.getTokens());
+ tokens.add(token);
+ }
+ }
+ Collections.sort(tokens);
+
+ Map<TokenRange, List<Host>> rangeToReplicas = new HashMap<>();
+
+ // The first token in the ring uses the last token as its 'start', handle this here to simplify the loop
+ Token start = tokens.get(tokens.size() - 1);
+ Token end = tokens.get(0);
+
+ addRange(keyspace, metadata, tokenToHost, rangeToReplicas, start, end);
+ for (int i = 1; i < tokens.size(); i++)
+ {
+ start = tokens.get(i - 1);
+ end = tokens.get(i);
+
+ addRange(keyspace, metadata, tokenToHost, rangeToReplicas, start, end);
+ }
+
+ return rangeToReplicas;
+ }
+
+ private static void addRange(String keyspace,
+ Metadata metadata,
+ Map<Token, Host> tokenToHost,
+ Map<TokenRange, List<Host>> rangeToReplicas,
+ Token start, Token end)
+ {
+ Host host = tokenToHost.get(end);
+ String dc = host.getDatacenter();
+
+ TokenRange range = metadata.newTokenRange(start, end);
+ List<Host> replicas = new ArrayList<>();
+ replicas.add(host);
+ // get all the replicas for the specific DC
+ for (Host replica : metadata.getReplicas(keyspace, range))
+ {
+ if (dc.equals(replica.getDatacenter()) && !host.equals(replica))
+ replicas.add(replica);
+ }
+ List<Host> previous = rangeToReplicas.put(range, replicas);
+ if (previous != null)
+ throw new IllegalStateException("Two hosts (" + host + ", " + previous + ") map to the same token range: " + range);
+ }
+}
diff --git a/src/java/org/apache/cassandra/hadoop/cql3/CqlConfigHelper.java b/src/java/org/apache/cassandra/hadoop/cql3/CqlConfigHelper.java
index 4c71273..f9a6f3a 100644
--- a/src/java/org/apache/cassandra/hadoop/cql3/CqlConfigHelper.java
+++ b/src/java/org/apache/cassandra/hadoop/cql3/CqlConfigHelper.java
@@ -19,7 +19,9 @@
* under the License.
*
*/
-import java.io.FileInputStream;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.io.InputStream;
import java.io.IOException;
import java.security.KeyManagementException;
import java.security.KeyStore;
@@ -86,7 +88,7 @@
private static final String OUTPUT_CQL = "cassandra.output.cql";
private static final String OUTPUT_NATIVE_PORT = "cassandra.output.native.port";
-
+
/**
* Set the CQL columns for the input of this job.
*
@@ -97,10 +99,10 @@
{
if (columns == null || columns.isEmpty())
return;
-
+
conf.set(INPUT_CQL_COLUMNS_CONFIG, columns);
}
-
+
/**
* Set the CQL query Limit for the input of this job.
*
@@ -127,10 +129,10 @@
{
if (clauses == null || clauses.isEmpty())
return;
-
+
conf.set(INPUT_CQL_WHERE_CLAUSE_CONFIG, clauses);
}
-
+
/**
* Set the CQL prepared statement for the output of this job.
*
@@ -141,7 +143,7 @@
{
if (cql == null || cql.isEmpty())
return;
-
+
conf.set(OUTPUT_CQL, cql);
}
@@ -283,7 +285,7 @@
return conf.get(OUTPUT_CQL);
}
- private static Optional<Integer> getProtocolVersion(Configuration conf)
+ private static Optional<Integer> getProtocolVersion(Configuration conf)
{
return getIntSetting(INPUT_NATIVE_PROTOCOL_VERSION, conf);
}
@@ -331,7 +333,7 @@
if (sslOptions.isPresent())
builder.withSSL(sslOptions.get());
- if (protocolVersion.isPresent())
+ if (protocolVersion.isPresent())
{
builder.withProtocolVersion(ProtocolVersion.fromInt(protocolVersion.get()));
}
@@ -356,7 +358,7 @@
public static void setInputMaxSimultReqPerConnections(Configuration conf, String reqs)
{
conf.set(INPUT_NATIVE_MAX_SIMULT_REQ_PER_CONNECTION, reqs);
- }
+ }
public static void setInputNativeConnectionTimeout(Configuration conf, String timeout)
{
@@ -396,7 +398,7 @@
public static void setInputNativeSSLTruststorePath(Configuration conf, String path)
{
conf.set(INPUT_NATIVE_SSL_TRUST_STORE_PATH, path);
- }
+ }
public static void setInputNativeSSLKeystorePath(Configuration conf, String path)
{
@@ -452,7 +454,7 @@
}
return poolingOptions;
- }
+ }
private static QueryOptions getReadQueryOptions(Configuration conf)
{
@@ -476,7 +478,7 @@
Optional<Integer> sendBufferSize = getInputNativeSendBufferSize(conf);
Optional<Integer> soLinger = getInputNativeSolinger(conf);
Optional<Boolean> tcpNoDelay = getInputNativeTcpNodelay(conf);
- Optional<Boolean> reuseAddress = getInputNativeReuseAddress(conf);
+ Optional<Boolean> reuseAddress = getInputNativeReuseAddress(conf);
Optional<Boolean> keepAlive = getInputNativeKeepAlive(conf);
if (connectTimeoutMillis.isPresent())
@@ -494,7 +496,7 @@
if (reuseAddress.isPresent())
socketOptions.setReuseAddress(reuseAddress.get());
if (keepAlive.isPresent())
- socketOptions.setKeepAlive(keepAlive.get());
+ socketOptions.setKeepAlive(keepAlive.get());
return socketOptions;
}
@@ -565,7 +567,7 @@
String setting = conf.get(parameter);
if (setting == null)
return Optional.absent();
- return Optional.of(Integer.valueOf(setting));
+ return Optional.of(Integer.valueOf(setting));
}
private static Optional<Boolean> getBooleanSetting(String parameter, Configuration conf)
@@ -573,7 +575,7 @@
String setting = conf.get(parameter);
if (setting == null)
return Optional.absent();
- return Optional.of(Boolean.valueOf(setting));
+ return Optional.of(Boolean.valueOf(setting));
}
private static Optional<String> getStringSetting(String parameter, Configuration conf)
@@ -581,7 +583,7 @@
String setting = conf.get(parameter);
if (setting == null)
return Optional.absent();
- return Optional.of(setting);
+ return Optional.of(setting);
}
private static AuthProvider getClientAuthProvider(String factoryClassName, Configuration conf)
@@ -623,7 +625,7 @@
TrustManagerFactory tmf = null;
if (truststorePath.isPresent())
{
- try (FileInputStream tsf = new FileInputStream(truststorePath.get()))
+ try (InputStream tsf = Files.newInputStream(Paths.get(truststorePath.get())))
{
KeyStore ts = KeyStore.getInstance("JKS");
ts.load(tsf, truststorePassword.isPresent() ? truststorePassword.get().toCharArray() : null);
@@ -635,7 +637,7 @@
KeyManagerFactory kmf = null;
if (keystorePath.isPresent())
{
- try (FileInputStream ksf = new FileInputStream(keystorePath.get()))
+ try (InputStream ksf = Files.newInputStream(Paths.get(keystorePath.get())))
{
KeyStore ks = KeyStore.getInstance("JKS");
ks.load(ksf, keystorePassword.isPresent() ? keystorePassword.get().toCharArray() : null);
diff --git a/src/java/org/apache/cassandra/hadoop/cql3/CqlInputFormat.java b/src/java/org/apache/cassandra/hadoop/cql3/CqlInputFormat.java
index 5e47ed5..1ea8eda 100644
--- a/src/java/org/apache/cassandra/hadoop/cql3/CqlInputFormat.java
+++ b/src/java/org/apache/cassandra/hadoop/cql3/CqlInputFormat.java
@@ -18,6 +18,7 @@
package org.apache.cassandra.hadoop.cql3;
import java.io.IOException;
+import java.net.InetAddress;
import java.util.*;
import java.util.concurrent.*;
@@ -27,9 +28,17 @@
import com.datastax.driver.core.ResultSet;
import com.datastax.driver.core.Row;
import com.datastax.driver.core.Session;
+import com.datastax.driver.core.SimpleStatement;
+import com.datastax.driver.core.Statement;
import com.datastax.driver.core.TokenRange;
-import org.apache.cassandra.config.SchemaConstants;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Multimap;
+
+import com.datastax.driver.core.exceptions.InvalidQueryException;
+import org.apache.cassandra.schema.SchemaConstants;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapred.InputSplit;
import org.apache.hadoop.mapred.JobConf;
@@ -42,15 +51,15 @@
import org.slf4j.LoggerFactory;
import org.apache.cassandra.db.SystemKeyspace;
import org.apache.cassandra.dht.*;
-import org.apache.cassandra.thrift.KeyRange;
import org.apache.cassandra.hadoop.*;
+import org.apache.cassandra.utils.*;
import static java.util.stream.Collectors.toMap;
/**
* Hadoop InputFormat allowing map/reduce against Cassandra rows within one ColumnFamily.
*
- * At minimum, you need to set the KS and CF in your Hadoop job Configuration.
+ * At minimum, you need to set the KS and CF in your Hadoop job Configuration.
* The ConfigHelper class is provided to make this
* simple:
* ConfigHelper.setInputColumnFamily
@@ -62,10 +71,10 @@
* If no value is provided for InputSplitSizeInMb, we default to using InputSplitSize.
*
* CQLConfigHelper.setInputCQLPageRowSize. The default page row size is 1000. You
- * should set it to "as big as possible, but no bigger." It set the LIMIT for the CQL
+ * should set it to "as big as possible, but no bigger." It set the LIMIT for the CQL
* query, so you need set it big enough to minimize the network overhead, and also
* not too big to avoid out of memory issue.
- *
+ *
* other native protocol connection parameters in CqlConfigHelper
*/
public class CqlInputFormat extends org.apache.hadoop.mapreduce.InputFormat<Long, Row> implements org.apache.hadoop.mapred.InputFormat<Long, Row>
@@ -129,47 +138,38 @@
ExecutorService executor = new ThreadPoolExecutor(0, 128, 60L, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>());
List<org.apache.hadoop.mapreduce.InputSplit> splits = new ArrayList<>();
- try (Cluster cluster = CqlConfigHelper.getInputCluster(ConfigHelper.getInputInitialAddress(conf).split(","), conf);
+ String[] inputInitialAddress = ConfigHelper.getInputInitialAddress(conf).split(",");
+ try (Cluster cluster = CqlConfigHelper.getInputCluster(inputInitialAddress, conf);
Session session = cluster.connect())
{
- List<Future<List<org.apache.hadoop.mapreduce.InputSplit>>> splitfutures = new ArrayList<>();
- KeyRange jobKeyRange = ConfigHelper.getInputKeyRange(conf);
+ List<SplitFuture> splitfutures = new ArrayList<>();
+ //TODO if the job range is defined and does perfectly match tokens, then the logic will be unable to get estimates since they are pre-computed
+ // tokens: [0, 10, 20]
+ // job range: [0, 10) - able to get estimate
+ // job range: [5, 15) - unable to get estimate
+ Pair<String, String> jobKeyRange = ConfigHelper.getInputKeyRange(conf);
Range<Token> jobRange = null;
if (jobKeyRange != null)
{
- if (jobKeyRange.start_key != null)
- {
- if (!partitioner.preservesOrder())
- throw new UnsupportedOperationException("KeyRange based on keys can only be used with a order preserving partitioner");
- if (jobKeyRange.start_token != null)
- throw new IllegalArgumentException("only start_key supported");
- if (jobKeyRange.end_token != null)
- throw new IllegalArgumentException("only start_key supported");
- jobRange = new Range<>(partitioner.getToken(jobKeyRange.start_key),
- partitioner.getToken(jobKeyRange.end_key));
- }
- else if (jobKeyRange.start_token != null)
- {
- jobRange = new Range<>(partitioner.getTokenFactory().fromString(jobKeyRange.start_token),
- partitioner.getTokenFactory().fromString(jobKeyRange.end_token));
- }
- else
- {
- logger.warn("ignoring jobKeyRange specified without start_key or start_token");
- }
+ jobRange = new Range<>(partitioner.getTokenFactory().fromString(jobKeyRange.left),
+ partitioner.getTokenFactory().fromString(jobKeyRange.right));
}
Metadata metadata = cluster.getMetadata();
// canonical ranges and nodes holding replicas
- Map<TokenRange, Set<Host>> masterRangeNodes = getRangeMap(keyspace, metadata);
-
+ Map<TokenRange, List<Host>> masterRangeNodes = getRangeMap(keyspace, metadata, getTargetDC(metadata, inputInitialAddress));
for (TokenRange range : masterRangeNodes.keySet())
{
if (jobRange == null)
{
- // for each tokenRange, pick a live owner and ask it to compute bite-sized splits
- splitfutures.add(executor.submit(new SplitCallable(range, masterRangeNodes.get(range), conf, session)));
+ for (TokenRange unwrapped : range.unwrap())
+ {
+ // for each tokenRange, pick a live owner and ask it for the byte-sized splits
+ SplitFuture task = new SplitFuture(new SplitCallable(unwrapped, masterRangeNodes.get(range), conf, session));
+ executor.submit(task);
+ splitfutures.add(task);
+ }
}
else
{
@@ -178,25 +178,63 @@
{
for (TokenRange intersection: range.intersectWith(jobTokenRange))
{
- // for each tokenRange, pick a live owner and ask it to compute bite-sized splits
- splitfutures.add(executor.submit(new SplitCallable(intersection, masterRangeNodes.get(range), conf, session)));
+ for (TokenRange unwrapped : intersection.unwrap())
+ {
+ // for each tokenRange, pick a live owner and ask it for the byte-sized splits
+ SplitFuture task = new SplitFuture(new SplitCallable(unwrapped, masterRangeNodes.get(range), conf, session));
+ executor.submit(task);
+ splitfutures.add(task);
+ }
}
}
}
}
// wait until we have all the results back
- for (Future<List<org.apache.hadoop.mapreduce.InputSplit>> futureInputSplits : splitfutures)
+ List<SplitFuture> failedTasks = new ArrayList<>();
+ int maxSplits = 0;
+ long expectedPartionsForFailedRanges = 0;
+ for (SplitFuture task : splitfutures)
{
try
{
- splits.addAll(futureInputSplits.get());
+ List<ColumnFamilySplit> tokenRangeSplits = task.get();
+ if (tokenRangeSplits.size() > maxSplits)
+ {
+ maxSplits = tokenRangeSplits.size();
+ expectedPartionsForFailedRanges = tokenRangeSplits.get(0).getLength();
+ }
+ splits.addAll(tokenRangeSplits);
}
catch (Exception e)
{
- throw new IOException("Could not get input splits", e);
+ failedTasks.add(task);
}
}
+ // The estimate is only stored on a single host, if that host is down then can not get the estimate
+ // its more than likely that a single host could be "too large" for one split but there is no way of
+ // knowning!
+ // This logic attempts to guess the estimate from all the successful ranges
+ if (!failedTasks.isEmpty())
+ {
+ // if every split failed this will be 0
+ if (maxSplits == 0)
+ throwAllSplitsFailed(failedTasks);
+ for (SplitFuture task : failedTasks)
+ {
+ try
+ {
+ // the task failed, so this should throw
+ task.get();
+ }
+ catch (Exception cause)
+ {
+ logger.warn("Unable to get estimate for {}, the host {} had a exception; falling back to default estimate", task.splitCallable.tokenRange, task.splitCallable.hosts.get(0), cause);
+ }
+ }
+ for (SplitFuture task : failedTasks)
+ splits.addAll(toSplit(task.splitCallable.hosts, splitTokenRange(task.splitCallable.tokenRange, maxSplits, expectedPartionsForFailedRanges)));
+ }
}
finally
{
@@ -208,19 +246,73 @@
return splits;
}
- private TokenRange rangeToTokenRange(Metadata metadata, Range<Token> range)
+ private static IllegalStateException throwAllSplitsFailed(List<SplitFuture> failedTasks)
{
- return metadata.newTokenRange(metadata.newToken(partitioner.getTokenFactory().toString(range.left)),
- metadata.newToken(partitioner.getTokenFactory().toString(range.right)));
+ IllegalStateException exception = new IllegalStateException("No successful tasks found");
+ for (SplitFuture task : failedTasks)
+ {
+ try
+ {
+ // the task failed, so this should throw
+ task.get();
+ }
+ catch (Exception cause)
+ {
+ exception.addSuppressed(cause);
+ }
+ }
+ throw exception;
}
- private Map<TokenRange, Long> getSubSplits(String keyspace, String cfName, TokenRange range, Configuration conf, Session session)
+ private static String getTargetDC(Metadata metadata, String[] inputInitialAddress)
{
- int splitSize = ConfigHelper.getInputSplitSize(conf);
- int splitSizeMb = ConfigHelper.getInputSplitSizeInMb(conf);
+ BiMultiValMap<InetAddress, String> addressToDc = new BiMultiValMap<>();
+ Multimap<String, InetAddress> dcToAddresses = addressToDc.inverse();
+
+ // only way to match is off the broadcast addresses, so for all hosts do a existence check
+ Set<InetAddress> addresses = new HashSet<>(inputInitialAddress.length);
+ for (String inputAddress : inputInitialAddress)
+ addresses.addAll(parseAddress(inputAddress));
+
+ for (Host host : metadata.getAllHosts())
+ {
+ InetAddress address = host.getBroadcastAddress();
+ if (addresses.contains(address))
+ addressToDc.put(address, host.getDatacenter());
+ }
+
+ switch (dcToAddresses.keySet().size())
+ {
+ case 1:
+ return Iterables.getOnlyElement(dcToAddresses.keySet());
+ case 0:
+ throw new IllegalStateException("Input addresses could not be used to find DC; non match client metadata");
+ default:
+ // Mutliple DCs found, attempt to pick the first based off address list. This is to mimic the 2.1
+ // behavior which would connect in order and the first node successfully able to connect to was the
+ // local DC to use; since client abstracts this, we rely on existence as a proxy for connect.
+ for (String inputAddress : inputInitialAddress)
+ {
+ for (InetAddress add : parseAddress(inputAddress))
+ {
+ String dc = addressToDc.get(add);
+ // possible the address isn't in the cluster and the client dropped, so ignore null
+ if (dc != null)
+ return dc;
+ }
+ }
+ // some how we were able to connect to the cluster, find multiple DCs using matching, and yet couldn't
+ // match again...
+ throw new AssertionError("Unable to infer datacenter from initial addresses; multiple datacenters found "
+ + dcToAddresses.keySet() + ", should only use addresses from one datacenter");
+ }
+ }
+
+ private static List<InetAddress> parseAddress(String str)
+ {
try
{
- return describeSplits(keyspace, cfName, range, splitSize, splitSizeMb, session);
+ return Arrays.asList(InetAddress.getAllByName(str));
}
catch (Exception e)
{
@@ -228,22 +320,35 @@
}
}
- private Map<TokenRange, Set<Host>> getRangeMap(String keyspace, Metadata metadata)
+ private TokenRange rangeToTokenRange(Metadata metadata, Range<Token> range)
{
- return metadata.getTokenRanges()
- .stream()
- .collect(toMap(p -> p, p -> metadata.getReplicas('"' + keyspace + '"', p)));
+ return metadata.newTokenRange(metadata.newToken(partitioner.getTokenFactory().toString(range.left)),
+ metadata.newToken(partitioner.getTokenFactory().toString(range.right)));
}
- private Map<TokenRange, Long> describeSplits(String keyspace, String table, TokenRange tokenRange, int splitSize, int splitSizeMb, Session session)
+ private Map<TokenRange, Long> getSubSplits(String keyspace, String cfName, TokenRange range, Host host, Configuration conf, Session session)
{
- String query = String.format("SELECT mean_partition_size, partitions_count " +
- "FROM %s.%s " +
- "WHERE keyspace_name = ? AND table_name = ? AND range_start = ? AND range_end = ?",
- SchemaConstants.SYSTEM_KEYSPACE_NAME,
- SystemKeyspace.SIZE_ESTIMATES);
+ int splitSize = ConfigHelper.getInputSplitSize(conf);
+ int splitSizeMb = ConfigHelper.getInputSplitSizeInMb(conf);
+ return describeSplits(keyspace, cfName, range, host, splitSize, splitSizeMb, session);
+ }
- ResultSet resultSet = session.execute(query, keyspace, table, tokenRange.getStart().toString(), tokenRange.getEnd().toString());
+ private static Map<TokenRange, List<Host>> getRangeMap(String keyspace, Metadata metadata, String targetDC)
+ {
+ return CqlClientHelper.getLocalPrimaryRangeForDC(keyspace, metadata, targetDC);
+ }
+
+ private Map<TokenRange, Long> describeSplits(String keyspace, String table, TokenRange tokenRange, Host host, int splitSize, int splitSizeMb, Session session)
+ {
+ // In 2.1 the host list was walked in-order (only move to next if IOException) and calls
+ // org.apache.cassandra.service.StorageService.getSplits(java.lang.String, java.lang.String, org.apache.cassandra.dht.Range<org.apache.cassandra.dht.Token>, int)
+ // that call computes totalRowCountEstimate (used to compute #splits) then splits the ring based off those estimates
+ //
+ // The main difference is that the estimates in 2.1 were computed based off the data, so replicas could answer the estimates
+ // In 3.0 we rely on the below CQL query which is local and only computes estimates for the primary range; this
+ // puts us in a sticky spot to answer, if the node fails what do we do? 3.0 behavior only matches 2.1 IFF all
+ // nodes are up and healthy
+ ResultSet resultSet = queryTableEstimates(session, host, keyspace, table, tokenRange);
Row row = resultSet.one();
@@ -267,14 +372,47 @@
if (splitCount == 0)
{
Map<TokenRange, Long> wrappedTokenRange = new HashMap<>();
- wrappedTokenRange.put(tokenRange, (long) 128);
+ wrappedTokenRange.put(tokenRange, partitionCount == 0 ? 128L : partitionCount);
return wrappedTokenRange;
}
+ return splitTokenRange(tokenRange, splitCount, partitionCount / splitCount);
+ }
+
+ private static ResultSet queryTableEstimates(Session session, Host host, String keyspace, String table, TokenRange tokenRange)
+ {
+ try
+ {
+ String query = String.format("SELECT mean_partition_size, partitions_count " +
+ "FROM %s.%s " +
+ "WHERE keyspace_name = ? AND table_name = ? AND range_type = '%s' AND range_start = ? AND range_end = ?",
+ SchemaConstants.SYSTEM_KEYSPACE_NAME,
+ SystemKeyspace.TABLE_ESTIMATES,
+ SystemKeyspace.TABLE_ESTIMATES_TYPE_LOCAL_PRIMARY);
+ Statement stmt = new SimpleStatement(query, keyspace, table, tokenRange.getStart().toString(), tokenRange.getEnd().toString()).setHost(host);
+ return session.execute(stmt);
+ }
+ catch (InvalidQueryException e)
+ {
+ // if the table doesn't exist, fall back to old table. This is likely to return no records in a multi
+ // DC setup, but should work fine in a single DC setup.
+ String query = String.format("SELECT mean_partition_size, partitions_count " +
+ "FROM %s.%s " +
+ "WHERE keyspace_name = ? AND table_name = ? AND range_start = ? AND range_end = ?",
+ SchemaConstants.SYSTEM_KEYSPACE_NAME,
+ SystemKeyspace.LEGACY_SIZE_ESTIMATES);
+
+ Statement stmt = new SimpleStatement(query, keyspace, table, tokenRange.getStart().toString(), tokenRange.getEnd().toString()).setHost(host);
+ return session.execute(stmt);
+ }
+ }
+
+ private static Map<TokenRange, Long> splitTokenRange(TokenRange tokenRange, int splitCount, long partitionCount)
+ {
List<TokenRange> splitRanges = tokenRange.splitEvenly(splitCount);
- Map<TokenRange, Long> rangesWithLength = new HashMap<>();
+ Map<TokenRange, Long> rangesWithLength = Maps.newHashMapWithExpectedSize(splitRanges.size());
for (TokenRange range : splitRanges)
- rangesWithLength.put(range, partitionCount/splitCount);
+ rangesWithLength.put(range, partitionCount);
return rangesWithLength;
}
@@ -294,56 +432,70 @@
* Gets a token tokenRange and splits it up according to the suggested
* size into input splits that Hadoop can use.
*/
- class SplitCallable implements Callable<List<org.apache.hadoop.mapreduce.InputSplit>>
+ class SplitCallable implements Callable<List<ColumnFamilySplit>>
{
private final TokenRange tokenRange;
- private final Set<Host> hosts;
+ private final List<Host> hosts;
private final Configuration conf;
private final Session session;
- public SplitCallable(TokenRange tr, Set<Host> hosts, Configuration conf, Session session)
+ public SplitCallable(TokenRange tokenRange, List<Host> hosts, Configuration conf, Session session)
{
- this.tokenRange = tr;
+ Preconditions.checkArgument(!hosts.isEmpty(), "hosts list requires at least 1 host but was empty");
+ this.tokenRange = tokenRange;
this.hosts = hosts;
this.conf = conf;
this.session = session;
}
- public List<org.apache.hadoop.mapreduce.InputSplit> call() throws Exception
+ public List<ColumnFamilySplit> call() throws Exception
{
- ArrayList<org.apache.hadoop.mapreduce.InputSplit> splits = new ArrayList<>();
- Map<TokenRange, Long> subSplits;
- subSplits = getSubSplits(keyspace, cfName, tokenRange, conf, session);
- // turn the sub-ranges into InputSplits
- String[] endpoints = new String[hosts.size()];
-
- // hadoop needs hostname, not ip
- int endpointIndex = 0;
- for (Host endpoint : hosts)
- endpoints[endpointIndex++] = endpoint.getAddress().getHostName();
-
- boolean partitionerIsOpp = partitioner instanceof OrderPreservingPartitioner || partitioner instanceof ByteOrderedPartitioner;
-
- for (Map.Entry<TokenRange, Long> subSplitEntry : subSplits.entrySet())
- {
- List<TokenRange> ranges = subSplitEntry.getKey().unwrap();
- for (TokenRange subrange : ranges)
- {
- ColumnFamilySplit split =
- new ColumnFamilySplit(
- partitionerIsOpp ?
- subrange.getStart().toString().substring(2) : subrange.getStart().toString(),
- partitionerIsOpp ?
- subrange.getEnd().toString().substring(2) : subrange.getEnd().toString(),
- subSplitEntry.getValue(),
- endpoints);
-
- logger.trace("adding {}", split);
- splits.add(split);
- }
- }
- return splits;
+ Map<TokenRange, Long> subSplits = getSubSplits(keyspace, cfName, tokenRange, hosts.get(0), conf, session);
+ return toSplit(hosts, subSplits);
}
+
+ }
+
+ private static class SplitFuture extends FutureTask<List<ColumnFamilySplit>>
+ {
+ private final SplitCallable splitCallable;
+
+ SplitFuture(SplitCallable splitCallable)
+ {
+ super(splitCallable);
+ this.splitCallable = splitCallable;
+ }
+ }
+
+ private List<ColumnFamilySplit> toSplit(List<Host> hosts, Map<TokenRange, Long> subSplits)
+ {
+ // turn the sub-ranges into InputSplits
+ String[] endpoints = new String[hosts.size()];
+
+ // hadoop needs hostname, not ip
+ int endpointIndex = 0;
+ for (Host endpoint : hosts)
+ endpoints[endpointIndex++] = endpoint.getAddress().getHostName();
+
+ boolean partitionerIsOpp = partitioner instanceof OrderPreservingPartitioner || partitioner instanceof ByteOrderedPartitioner;
+
+ ArrayList<ColumnFamilySplit> splits = new ArrayList<>();
+ for (Map.Entry<TokenRange, Long> subSplitEntry : subSplits.entrySet())
+ {
+ TokenRange subrange = subSplitEntry.getKey();
+ ColumnFamilySplit split =
+ new ColumnFamilySplit(
+ partitionerIsOpp ?
+ subrange.getStart().toString().substring(2) : subrange.getStart().toString(),
+ partitionerIsOpp ?
+ subrange.getEnd().toString().substring(2) : subrange.getEnd().toString(),
+ subSplitEntry.getValue(),
+ endpoints);
+
+ logger.trace("adding {}", split);
+ splits.add(split);
+ }
+ return splits;
}
}
diff --git a/src/java/org/apache/cassandra/hadoop/cql3/CqlRecordWriter.java b/src/java/org/apache/cassandra/hadoop/cql3/CqlRecordWriter.java
index d2a0d86..989d154 100644
--- a/src/java/org/apache/cassandra/hadoop/cql3/CqlRecordWriter.java
+++ b/src/java/org/apache/cassandra/hadoop/cql3/CqlRecordWriter.java
@@ -438,7 +438,7 @@
return true;
if (e instanceof NoHostAvailableException)
{
- if (((NoHostAvailableException) e).getErrors().values().size() == 1)
+ if (((NoHostAvailableException) e).getErrors().size() == 1)
{
Throwable cause = ((NoHostAvailableException) e).getErrors().values().iterator().next();
if (cause != null && cause.getCause() instanceof java.nio.channels.ClosedByInterruptException)
diff --git a/src/java/org/apache/cassandra/hadoop/cql3/LimitedLocalNodeFirstLocalBalancingPolicy.java b/src/java/org/apache/cassandra/hadoop/cql3/LimitedLocalNodeFirstLocalBalancingPolicy.java
index 5c8d3c5..59b4eca 100644
--- a/src/java/org/apache/cassandra/hadoop/cql3/LimitedLocalNodeFirstLocalBalancingPolicy.java
+++ b/src/java/org/apache/cassandra/hadoop/cql3/LimitedLocalNodeFirstLocalBalancingPolicy.java
@@ -56,6 +56,7 @@
private final CopyOnWriteArraySet<Host> liveReplicaHosts = new CopyOnWriteArraySet<>();
private final Set<InetAddress> replicaAddresses = new HashSet<>();
+ private final Set<String> allowedDCs = new CopyOnWriteArraySet<>();
public LimitedLocalNodeFirstLocalBalancingPolicy(String[] replicas)
{
@@ -71,21 +72,29 @@
logger.warn("Invalid replica host name: {}, skipping it", replica);
}
}
- logger.trace("Created instance with the following replicas: {}", Arrays.asList(replicas));
+ if (logger.isTraceEnabled())
+ logger.trace("Created instance with the following replicas: {}", Arrays.asList(replicas));
}
@Override
public void init(Cluster cluster, Collection<Host> hosts)
{
- List<Host> replicaHosts = new ArrayList<>();
+ // first find which DCs the user defined
+ Set<String> dcs = new HashSet<>();
for (Host host : hosts)
{
if (replicaAddresses.contains(host.getAddress()))
- {
+ dcs.add(host.getDatacenter());
+ }
+ // filter to all nodes within the targeted DCs
+ List<Host> replicaHosts = new ArrayList<>();
+ for (Host host : hosts)
+ {
+ if (dcs.contains(host.getDatacenter()))
replicaHosts.add(host);
- }
}
liveReplicaHosts.addAll(replicaHosts);
+ allowedDCs.addAll(dcs);
logger.trace("Initialized with replica hosts: {}", replicaHosts);
}
@@ -135,7 +144,7 @@
@Override
public void onAdd(Host host)
{
- if (replicaAddresses.contains(host.getAddress()))
+ if (liveReplicaHosts.contains(host))
{
liveReplicaHosts.add(host);
logger.trace("Added a new host {}", host);
@@ -145,7 +154,7 @@
@Override
public void onUp(Host host)
{
- if (replicaAddresses.contains(host.getAddress()))
+ if (liveReplicaHosts.contains(host))
{
liveReplicaHosts.add(host);
logger.trace("The host {} is now up", host);
diff --git a/src/java/org/apache/cassandra/hints/ChecksummedDataInput.java b/src/java/org/apache/cassandra/hints/ChecksummedDataInput.java
index 6ebc830..30d18fa 100644
--- a/src/java/org/apache/cassandra/hints/ChecksummedDataInput.java
+++ b/src/java/org/apache/cassandra/hints/ChecksummedDataInput.java
@@ -26,6 +26,7 @@
import org.apache.cassandra.io.compress.BufferType;
import org.apache.cassandra.io.util.*;
+import org.apache.cassandra.utils.Throwables;
import org.apache.cassandra.utils.NativeLibrary;
import org.apache.cassandra.utils.memory.BufferPool;
@@ -74,7 +75,15 @@
@SuppressWarnings("resource")
public static ChecksummedDataInput open(File file)
{
- return new ChecksummedDataInput(new ChannelProxy(file));
+ ChannelProxy channel = new ChannelProxy(file);
+ try
+ {
+ return new ChecksummedDataInput(channel);
+ }
+ catch (Throwable t)
+ {
+ throw Throwables.cleaned(channel.close(t));
+ }
}
public boolean isEOF()
diff --git a/src/java/org/apache/cassandra/hints/CompressedChecksummedDataInput.java b/src/java/org/apache/cassandra/hints/CompressedChecksummedDataInput.java
index 0766fa5..0381b00 100644
--- a/src/java/org/apache/cassandra/hints/CompressedChecksummedDataInput.java
+++ b/src/java/org/apache/cassandra/hints/CompressedChecksummedDataInput.java
@@ -23,11 +23,11 @@
import com.google.common.annotations.VisibleForTesting;
-import org.apache.cassandra.hints.ChecksummedDataInput.Position;
import org.apache.cassandra.io.FSReadError;
import org.apache.cassandra.io.compress.ICompressor;
import org.apache.cassandra.io.util.ChannelProxy;
import org.apache.cassandra.utils.memory.BufferPool;
+import org.apache.cassandra.utils.Throwables;
public final class CompressedChecksummedDataInput extends ChecksummedDataInput
{
@@ -161,7 +161,15 @@
long position = input.getPosition();
input.close();
- return new CompressedChecksummedDataInput(new ChannelProxy(input.getPath()), compressor, position);
+ ChannelProxy channel = new ChannelProxy(input.getPath());
+ try
+ {
+ return new CompressedChecksummedDataInput(channel, compressor, position);
+ }
+ catch (Throwable t)
+ {
+ throw Throwables.cleaned(channel.close(t));
+ }
}
@VisibleForTesting
diff --git a/src/java/org/apache/cassandra/hints/EncodedHintMessage.java b/src/java/org/apache/cassandra/hints/EncodedHintMessage.java
deleted file mode 100644
index 4fe05ac..0000000
--- a/src/java/org/apache/cassandra/hints/EncodedHintMessage.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.hints;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.UUID;
-
-import org.apache.cassandra.db.TypeSizes;
-import org.apache.cassandra.io.IVersionedSerializer;
-import org.apache.cassandra.io.util.DataInputPlus;
-import org.apache.cassandra.io.util.DataOutputPlus;
-import org.apache.cassandra.net.MessageOut;
-import org.apache.cassandra.net.MessagingService;
-import org.apache.cassandra.utils.UUIDSerializer;
-
-/**
- * A specialized version of {@link HintMessage} that takes an already encoded in a bytebuffer hint and sends it verbatim.
- *
- * An optimization for when dispatching a hint file of the current messaging version to a node of the same messaging version,
- * which is the most common case. Saves on extra ByteBuffer allocations one redundant hint deserialization-serialization cycle.
- *
- * Never deserialized as an EncodedHintMessage - the receiving side will always deserialize the message as vanilla
- * {@link HintMessage}.
- */
-final class EncodedHintMessage
-{
- private static final IVersionedSerializer<EncodedHintMessage> serializer = new Serializer();
-
- private final UUID hostId;
- private final ByteBuffer hint;
- private final int version;
-
- EncodedHintMessage(UUID hostId, ByteBuffer hint, int version)
- {
- this.hostId = hostId;
- this.hint = hint;
- this.version = version;
- }
-
- MessageOut<EncodedHintMessage> createMessageOut()
- {
- return new MessageOut<>(MessagingService.Verb.HINT, this, serializer);
- }
-
- private static class Serializer implements IVersionedSerializer<EncodedHintMessage>
- {
- public long serializedSize(EncodedHintMessage message, int version)
- {
- if (version != message.version)
- throw new IllegalArgumentException("serializedSize() called with non-matching version " + version);
-
- long size = UUIDSerializer.serializer.serializedSize(message.hostId, version);
- size += TypeSizes.sizeofUnsignedVInt(message.hint.remaining());
- size += message.hint.remaining();
- return size;
- }
-
- public void serialize(EncodedHintMessage message, DataOutputPlus out, int version) throws IOException
- {
- if (version != message.version)
- throw new IllegalArgumentException("serialize() called with non-matching version " + version);
-
- UUIDSerializer.serializer.serialize(message.hostId, out, version);
- out.writeUnsignedVInt(message.hint.remaining());
- out.write(message.hint);
- }
-
- public EncodedHintMessage deserialize(DataInputPlus in, int version) throws IOException
- {
- throw new UnsupportedOperationException();
- }
- }
-}
diff --git a/src/java/org/apache/cassandra/hints/EncryptedChecksummedDataInput.java b/src/java/org/apache/cassandra/hints/EncryptedChecksummedDataInput.java
index b01161d..5edd8a8 100644
--- a/src/java/org/apache/cassandra/hints/EncryptedChecksummedDataInput.java
+++ b/src/java/org/apache/cassandra/hints/EncryptedChecksummedDataInput.java
@@ -25,10 +25,10 @@
import io.netty.util.concurrent.FastThreadLocal;
import org.apache.cassandra.security.EncryptionUtils;
-import org.apache.cassandra.hints.CompressedChecksummedDataInput.Position;
import org.apache.cassandra.io.FSReadError;
import org.apache.cassandra.io.compress.ICompressor;
import org.apache.cassandra.io.util.ChannelProxy;
+import org.apache.cassandra.utils.Throwables;
public class EncryptedChecksummedDataInput extends ChecksummedDataInput
{
@@ -138,7 +138,15 @@
long position = input.getPosition();
input.close();
- return new EncryptedChecksummedDataInput(new ChannelProxy(input.getPath()), cipher, compressor, position);
+ ChannelProxy channel = new ChannelProxy(input.getPath());
+ try
+ {
+ return new EncryptedChecksummedDataInput(channel, cipher, compressor, position);
+ }
+ catch (Throwable t)
+ {
+ throw Throwables.cleaned(channel.close(t));
+ }
}
@VisibleForTesting
diff --git a/src/java/org/apache/cassandra/hints/Hint.java b/src/java/org/apache/cassandra/hints/Hint.java
index 17fbf5d..6c7c5d4 100644
--- a/src/java/org/apache/cassandra/hints/Hint.java
+++ b/src/java/org/apache/cassandra/hints/Hint.java
@@ -18,17 +18,23 @@
package org.apache.cassandra.hints;
import java.io.IOException;
-import java.util.*;
+import java.nio.ByteBuffer;
import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import com.google.common.base.Throwables;
+import javax.annotation.Nullable;
+
+import com.google.common.primitives.Ints;
+
import org.apache.cassandra.db.*;
import org.apache.cassandra.io.IVersionedSerializer;
+import org.apache.cassandra.io.util.DataInputBuffer;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.utils.vint.VIntCoding;
import static org.apache.cassandra.db.TypeSizes.sizeof;
import static org.apache.cassandra.db.TypeSizes.sizeofUnsignedVInt;
@@ -51,10 +57,11 @@
public final class Hint
{
public static final Serializer serializer = new Serializer();
+ static final int maxHintTTL = Integer.getInteger("cassandra.maxHintTTL", Integer.MAX_VALUE);
final Mutation mutation;
final long creationTime; // time of hint creation (in milliseconds)
- final int gcgs; // the smallest gc gs of all involved tables
+ final int gcgs; // the smallest gc gs of all involved tables (in seconds)
private Hint(Mutation mutation, long creationTime, int gcgs)
{
@@ -89,9 +96,9 @@
{
if (isLive())
{
- // filter out partition update for table that have been truncated since hint's creation
+ // filter out partition update for tables that have been truncated since hint's creation
Mutation filtered = mutation;
- for (UUID id : mutation.getColumnFamilyIds())
+ for (TableId id : mutation.getTableIds())
if (creationTime <= SystemKeyspace.getTruncatedAt(id))
filtered = filtered.without(id);
@@ -115,13 +122,25 @@
}
/**
+ * @return the overall ttl of the hint - the minimum of all mutation's tables' gc gs now and at the time of creation
+ */
+ int ttl()
+ {
+ return Math.min(gcgs, mutation.smallestGCGS());
+ }
+
+ /**
* @return calculates whether or not it is safe to apply the hint without risking to resurrect any deleted data
*/
- boolean isLive()
+ public boolean isLive()
{
- int smallestGCGS = Math.min(gcgs, mutation.smallestGCGS());
- long expirationTime = creationTime + TimeUnit.SECONDS.toMillis(smallestGCGS);
- return expirationTime > System.currentTimeMillis();
+ return isLive(creationTime, System.currentTimeMillis(), ttl());
+ }
+
+ static boolean isLive(long creationTime, long now, int hintTTL)
+ {
+ long expirationTime = creationTime + TimeUnit.SECONDS.toMillis(Math.min(hintTTL, maxHintTTL));
+ return expirationTime > now;
}
static final class Serializer implements IVersionedSerializer<Hint>
@@ -130,7 +149,7 @@
{
long size = sizeof(hint.creationTime);
size += sizeofUnsignedVInt(hint.gcgs);
- size += Mutation.serializer.serializedSize(hint.mutation, version);
+ size += hint.mutation.serializedSize(version);
return size;
}
@@ -147,5 +166,65 @@
int gcgs = (int) in.readUnsignedVInt();
return new Hint(Mutation.serializer.deserialize(in, version), creationTime, gcgs);
}
+
+ public long getHintCreationTime(ByteBuffer hintBuffer, int version)
+ {
+ return hintBuffer.getLong(0);
+ }
+
+ /**
+ * Will short-circuit Mutation deserialization if the hint is definitely dead. If a Hint instance is
+ * returned, there is a chance it's live, if gcgs on one of the table involved got reduced between
+ * hint creation and deserialization, but this does not impact correctness - an extra liveness check will
+ * also be performed on the receiving end.
+ *
+ * @return null if the hint is definitely dead, a Hint instance if it's likely live
+ */
+ @Nullable
+ Hint deserializeIfLive(DataInputPlus in, long now, long size, int version) throws IOException
+ {
+ long creationTime = in.readLong();
+ int gcgs = (int) in.readUnsignedVInt();
+ int bytesRead = sizeof(creationTime) + sizeofUnsignedVInt(gcgs);
+
+ if (isLive(creationTime, now, gcgs))
+ return new Hint(Mutation.serializer.deserialize(in, version), creationTime, gcgs);
+
+ in.skipBytesFully(Ints.checkedCast(size) - bytesRead);
+ return null;
+ }
+
+ /**
+ * Will short-circuit ByteBuffer allocation if the hint is definitely dead. If a ByteBuffer instance is
+ * returned, there is a chance it's live, if gcgs on one of the table involved got reduced between
+ * hint creation and deserialization, but this does not impact correctness - an extra liveness check will
+ * also be performed on the receiving end.
+ *
+ * @return null if the hint is definitely dead, a ByteBuffer instance if it's likely live
+ */
+ @Nullable
+ ByteBuffer readBufferIfLive(DataInputPlus in, long now, int size, int version) throws IOException
+ {
+ int maxHeaderSize = Math.min(sizeof(Long.MAX_VALUE) + VIntCoding.MAX_SIZE, size);
+ byte[] header = new byte[maxHeaderSize];
+ in.readFully(header);
+
+ try (DataInputBuffer input = new DataInputBuffer(header))
+ {
+ long creationTime = input.readLong();
+ int gcgs = (int) input.readUnsignedVInt();
+
+ if (!isLive(creationTime, now, gcgs))
+ {
+ in.skipBytesFully(size - maxHeaderSize);
+ return null;
+ }
+ }
+
+ byte[] bytes = new byte[size];
+ System.arraycopy(header, 0, bytes, 0, header.length);
+ in.readFully(bytes, header.length, size - header.length);
+ return ByteBuffer.wrap(bytes);
+ }
}
}
diff --git a/src/java/org/apache/cassandra/hints/HintDiagnostics.java b/src/java/org/apache/cassandra/hints/HintDiagnostics.java
new file mode 100644
index 0000000..3ff0834
--- /dev/null
+++ b/src/java/org/apache/cassandra/hints/HintDiagnostics.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.hints;
+
+import org.apache.cassandra.diag.DiagnosticEventService;
+import org.apache.cassandra.hints.HintEvent.HintEventType;
+import org.apache.cassandra.hints.HintEvent.HintResult;
+
+/**
+ * Utility methods for DiagnosticEvents around hinted handoff.
+ */
+final class HintDiagnostics
+{
+ private static final DiagnosticEventService service = DiagnosticEventService.instance();
+
+ private HintDiagnostics()
+ {
+ }
+
+ static void dispatcherCreated(HintsDispatcher dispatcher)
+ {
+ if (isEnabled(HintEventType.DISPATCHER_CREATED))
+ service.publish(new HintEvent(HintEventType.DISPATCHER_CREATED, dispatcher,
+ dispatcher.hostId, dispatcher.address, null, null, null, null));
+ }
+
+ static void dispatcherClosed(HintsDispatcher dispatcher)
+ {
+ if (isEnabled(HintEventType.DISPATCHER_CLOSED))
+ service.publish(new HintEvent(HintEventType.DISPATCHER_CLOSED, dispatcher,
+ dispatcher.hostId, dispatcher.address, null, null, null, null));
+ }
+
+ static void dispatchPage(HintsDispatcher dispatcher)
+ {
+ if (isEnabled(HintEventType.DISPATCHER_PAGE))
+ service.publish(new HintEvent(HintEventType.DISPATCHER_PAGE, dispatcher,
+ dispatcher.hostId, dispatcher.address, null, null, null, null));
+ }
+
+ static void abortRequested(HintsDispatcher dispatcher)
+ {
+ if (isEnabled(HintEventType.ABORT_REQUESTED))
+ service.publish(new HintEvent(HintEventType.ABORT_REQUESTED, dispatcher,
+ dispatcher.hostId, dispatcher.address, null, null, null, null));
+ }
+
+ static void pageSuccessResult(HintsDispatcher dispatcher, long success, long failures, long timeouts)
+ {
+ if (isEnabled(HintEventType.DISPATCHER_HINT_RESULT))
+ service.publish(new HintEvent(HintEventType.DISPATCHER_HINT_RESULT, dispatcher,
+ dispatcher.hostId, dispatcher.address, HintResult.PAGE_SUCCESS,
+ success, failures, timeouts));
+ }
+
+ static void pageFailureResult(HintsDispatcher dispatcher, long success, long failures, long timeouts)
+ {
+ if (isEnabled(HintEventType.DISPATCHER_HINT_RESULT))
+ service.publish(new HintEvent(HintEventType.DISPATCHER_HINT_RESULT, dispatcher,
+ dispatcher.hostId, dispatcher.address, HintResult.PAGE_FAILURE,
+ success, failures, timeouts));
+ }
+
+ private static boolean isEnabled(HintEventType type)
+ {
+ return service.isEnabled(HintEvent.class, type);
+ }
+
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/hints/HintEvent.java b/src/java/org/apache/cassandra/hints/HintEvent.java
new file mode 100644
index 0000000..011f248
--- /dev/null
+++ b/src/java/org/apache/cassandra/hints/HintEvent.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.hints;
+
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.UUID;
+import javax.annotation.Nullable;
+
+import org.apache.cassandra.diag.DiagnosticEvent;
+import org.apache.cassandra.locator.InetAddressAndPort;
+
+/**
+ * DiagnosticEvent implementation for hinted handoff.
+ */
+final class HintEvent extends DiagnosticEvent
+{
+ enum HintEventType
+ {
+ DISPATCHING_STARTED,
+ DISPATCHING_PAUSED,
+ DISPATCHING_RESUMED,
+ DISPATCHING_SHUTDOWN,
+
+ DISPATCHER_CREATED,
+ DISPATCHER_CLOSED,
+
+ DISPATCHER_PAGE,
+ DISPATCHER_HINT_RESULT,
+
+ ABORT_REQUESTED
+ }
+
+ enum HintResult
+ {
+ PAGE_SUCCESS, PAGE_FAILURE
+ }
+
+ private final HintEventType type;
+ private final HintsDispatcher dispatcher;
+ private final UUID targetHostId;
+ private final InetAddressAndPort targetAddress;
+ @Nullable
+ private final HintResult dispatchResult;
+ @Nullable
+ private final Long pageHintsSuccessful;
+ @Nullable
+ private final Long pageHintsFailed;
+ @Nullable
+ private final Long pageHintsTimeout;
+
+ HintEvent(HintEventType type, HintsDispatcher dispatcher, UUID targetHostId, InetAddressAndPort targetAddress,
+ @Nullable HintResult dispatchResult, @Nullable Long pageHintsSuccessful,
+ @Nullable Long pageHintsFailed, @Nullable Long pageHintsTimeout)
+ {
+ this.type = type;
+ this.dispatcher = dispatcher;
+ this.targetHostId = targetHostId;
+ this.targetAddress = targetAddress;
+ this.dispatchResult = dispatchResult;
+ this.pageHintsSuccessful = pageHintsSuccessful;
+ this.pageHintsFailed = pageHintsFailed;
+ this.pageHintsTimeout = pageHintsTimeout;
+ }
+
+ public Enum<HintEventType> getType()
+ {
+ return type;
+ }
+
+ public HashMap<String, Serializable> toMap()
+ {
+ // be extra defensive against nulls and bugs
+ HashMap<String, Serializable> ret = new HashMap<>();
+ ret.put("targetHostId", targetHostId);
+ ret.put("targetAddress", targetAddress.getHostAddress(true));
+ if (dispatchResult != null) ret.put("dispatchResult", dispatchResult.name());
+ if (pageHintsSuccessful != null || pageHintsFailed != null || pageHintsTimeout != null)
+ {
+ ret.put("hint.page.hints_succeeded", pageHintsSuccessful);
+ ret.put("hint.page.hints_failed", pageHintsFailed);
+ ret.put("hint.page.hints_timed_out", pageHintsTimeout);
+ }
+ return ret;
+ }
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/hints/HintMessage.java b/src/java/org/apache/cassandra/hints/HintMessage.java
index 723ab6d..333af84 100644
--- a/src/java/org/apache/cassandra/hints/HintMessage.java
+++ b/src/java/org/apache/cassandra/hints/HintMessage.java
@@ -19,6 +19,7 @@
package org.apache.cassandra.hints;
import java.io.IOException;
+import java.nio.ByteBuffer;
import java.util.Objects;
import java.util.UUID;
@@ -27,13 +28,12 @@
import com.google.common.primitives.Ints;
import org.apache.cassandra.db.TypeSizes;
-import org.apache.cassandra.db.UnknownColumnFamilyException;
-import org.apache.cassandra.io.IVersionedSerializer;
+import org.apache.cassandra.exceptions.UnknownTableException;
+import org.apache.cassandra.io.IVersionedAsymmetricSerializer;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
-import org.apache.cassandra.net.MessageOut;
-import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.io.util.TrackedDataInputPlus;
+import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.utils.UUIDSerializer;
/**
@@ -48,9 +48,9 @@
* Scenario (2) means that we got a hint from a node that's going through decommissioning and is streaming its hints
* elsewhere first.
*/
-public final class HintMessage
+public final class HintMessage implements SerializableHintMessage
{
- public static final IVersionedSerializer<HintMessage> serializer = new Serializer();
+ public static final IVersionedAsymmetricSerializer<SerializableHintMessage, HintMessage> serializer = new Serializer();
final UUID hostId;
@@ -58,7 +58,7 @@
final Hint hint;
@Nullable // will usually be null, unless a hint deserialization fails due to an unknown table id
- final UUID unknownTableID;
+ final TableId unknownTableID;
HintMessage(UUID hostId, Hint hint)
{
@@ -67,44 +67,79 @@
this.unknownTableID = null;
}
- HintMessage(UUID hostId, UUID unknownTableID)
+ HintMessage(UUID hostId, TableId unknownTableID)
{
this.hostId = hostId;
this.hint = null;
this.unknownTableID = unknownTableID;
}
- public MessageOut<HintMessage> createMessageOut()
+ public static class Serializer implements IVersionedAsymmetricSerializer<SerializableHintMessage, HintMessage>
{
- return new MessageOut<>(MessagingService.Verb.HINT, this, serializer);
- }
-
- public static class Serializer implements IVersionedSerializer<HintMessage>
- {
- public long serializedSize(HintMessage message, int version)
+ public long serializedSize(SerializableHintMessage obj, int version)
{
- long size = UUIDSerializer.serializer.serializedSize(message.hostId, version);
+ if (obj instanceof HintMessage)
+ {
+ HintMessage message = (HintMessage) obj;
+ long size = UUIDSerializer.serializer.serializedSize(message.hostId, version);
- long hintSize = Hint.serializer.serializedSize(message.hint, version);
- size += TypeSizes.sizeofUnsignedVInt(hintSize);
- size += hintSize;
+ long hintSize = Hint.serializer.serializedSize(message.hint, version);
+ size += TypeSizes.sizeofUnsignedVInt(hintSize);
+ size += hintSize;
- return size;
+ return size;
+ }
+ else if (obj instanceof Encoded)
+ {
+ Encoded message = (Encoded) obj;
+
+ if (version != message.version)
+ throw new IllegalArgumentException("serializedSize() called with non-matching version " + version);
+
+ long size = UUIDSerializer.serializer.serializedSize(message.hostId, version);
+ size += TypeSizes.sizeofUnsignedVInt(message.hint.remaining());
+ size += message.hint.remaining();
+ return size;
+ }
+ else
+ {
+ throw new IllegalStateException("Unexpected type: " + obj);
+ }
}
- public void serialize(HintMessage message, DataOutputPlus out, int version) throws IOException
+ public void serialize(SerializableHintMessage obj, DataOutputPlus out, int version) throws IOException
{
- Objects.requireNonNull(message.hint); // we should never *send* a HintMessage with null hint
+ if (obj instanceof HintMessage)
+ {
+ HintMessage message = (HintMessage) obj;
- UUIDSerializer.serializer.serialize(message.hostId, out, version);
+ Objects.requireNonNull(message.hint); // we should never *send* a HintMessage with null hint
- /*
- * We are serializing the hint size so that the receiver of the message could gracefully handle
- * deserialize failure when a table had been dropped, by simply skipping the unread bytes.
- */
- out.writeUnsignedVInt(Hint.serializer.serializedSize(message.hint, version));
+ UUIDSerializer.serializer.serialize(message.hostId, out, version);
- Hint.serializer.serialize(message.hint, out, version);
+ /*
+ * We are serializing the hint size so that the receiver of the message could gracefully handle
+ * deserialize failure when a table had been dropped, by simply skipping the unread bytes.
+ */
+ out.writeUnsignedVInt(Hint.serializer.serializedSize(message.hint, version));
+
+ Hint.serializer.serialize(message.hint, out, version);
+ }
+ else if (obj instanceof Encoded)
+ {
+ Encoded message = (Encoded) obj;
+
+ if (version != message.version)
+ throw new IllegalArgumentException("serialize() called with non-matching version " + version);
+
+ UUIDSerializer.serializer.serialize(message.hostId, out, version);
+ out.writeUnsignedVInt(message.hint.remaining());
+ out.write(message.hint);
+ }
+ else
+ {
+ throw new IllegalStateException("Unexpected type: " + obj);
+ }
}
/*
@@ -122,11 +157,39 @@
{
return new HintMessage(hostId, Hint.serializer.deserialize(countingIn, version));
}
- catch (UnknownColumnFamilyException e)
+ catch (UnknownTableException e)
{
in.skipBytes(Ints.checkedCast(hintSize - countingIn.getBytesRead()));
- return new HintMessage(hostId, e.cfId);
+ return new HintMessage(hostId, e.id);
}
}
}
+
+ /**
+ * A specialized version of {@link HintMessage} that takes an already encoded in a bytebuffer hint and sends it verbatim.
+ *
+ * An optimization for when dispatching a hint file of the current messaging version to a node of the same messaging version,
+ * which is the most common case. Saves on extra ByteBuffer allocations one redundant hint deserialization-serialization cycle.
+ *
+ * Never deserialized as an HintMessage.Encoded - the receiving side will always deserialize the message as vanilla
+ * {@link HintMessage}.
+ */
+ static final class Encoded implements SerializableHintMessage
+ {
+ private final UUID hostId;
+ private final ByteBuffer hint;
+ private final int version;
+
+ Encoded(UUID hostId, ByteBuffer hint, int version)
+ {
+ this.hostId = hostId;
+ this.hint = hint;
+ this.version = version;
+ }
+
+ public long getHintCreationTime()
+ {
+ return Hint.serializer.getHintCreationTime(hint, version);
+ }
+ }
}
diff --git a/src/java/org/apache/cassandra/hints/HintResponse.java b/src/java/org/apache/cassandra/hints/HintResponse.java
deleted file mode 100644
index 8aa888f..0000000
--- a/src/java/org/apache/cassandra/hints/HintResponse.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.cassandra.hints;
-
-import org.apache.cassandra.io.IVersionedSerializer;
-import org.apache.cassandra.io.util.DataInputPlus;
-import org.apache.cassandra.io.util.DataOutputPlus;
-import org.apache.cassandra.net.MessageOut;
-import org.apache.cassandra.net.MessagingService;
-
-/**
- * An empty successful response to a HintMessage.
- */
-public final class HintResponse
-{
- public static final IVersionedSerializer<HintResponse> serializer = new Serializer();
-
- static final HintResponse instance = new HintResponse();
- static final MessageOut<HintResponse> message =
- new MessageOut<>(MessagingService.Verb.REQUEST_RESPONSE, instance, serializer);
-
- private HintResponse()
- {
- }
-
- private static final class Serializer implements IVersionedSerializer<HintResponse>
- {
- public long serializedSize(HintResponse response, int version)
- {
- return 0;
- }
-
- public void serialize(HintResponse response, DataOutputPlus out, int version)
- {
- }
-
- public HintResponse deserialize(DataInputPlus in, int version)
- {
- return instance;
- }
- }
-}
diff --git a/src/java/org/apache/cassandra/hints/HintVerbHandler.java b/src/java/org/apache/cassandra/hints/HintVerbHandler.java
index 2b92a42..2fbe475 100644
--- a/src/java/org/apache/cassandra/hints/HintVerbHandler.java
+++ b/src/java/org/apache/cassandra/hints/HintVerbHandler.java
@@ -18,15 +18,15 @@
*/
package org.apache.cassandra.hints;
-import java.net.InetAddress;
import java.util.UUID;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.db.partitions.PartitionUpdate;
+import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.net.IVerbHandler;
-import org.apache.cassandra.net.MessageIn;
+import org.apache.cassandra.net.Message;
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.serializers.MarshalException;
import org.apache.cassandra.service.StorageProxy;
@@ -41,13 +41,15 @@
*/
public final class HintVerbHandler implements IVerbHandler<HintMessage>
{
+ public static final HintVerbHandler instance = new HintVerbHandler();
+
private static final Logger logger = LoggerFactory.getLogger(HintVerbHandler.class);
- public void doVerb(MessageIn<HintMessage> message, int id)
+ public void doVerb(Message<HintMessage> message)
{
UUID hostId = message.payload.hostId;
Hint hint = message.payload.hint;
- InetAddress address = StorageService.instance.getEndpointForHostId(hostId);
+ InetAddressAndPort address = StorageService.instance.getEndpointForHostId(hostId);
// If we see an unknown table id, it means the table, or one of the tables in the mutation, had been dropped.
// In that case there is nothing we can really do, or should do, other than log it go on.
@@ -59,7 +61,7 @@
address,
hostId,
message.payload.unknownTableID);
- reply(id, message.from);
+ respond(message);
return;
}
@@ -71,7 +73,7 @@
catch (MarshalException e)
{
logger.warn("Failed to validate a hint for {}: {} - skipped", address, hostId);
- reply(id, message.from);
+ respond(message);
return;
}
@@ -80,24 +82,24 @@
// the node is not the final destination of the hint (must have gotten it from a decommissioning node),
// so just store it locally, to be delivered later.
HintsService.instance.write(hostId, hint);
- reply(id, message.from);
+ respond(message);
}
else if (!StorageProxy.instance.appliesLocally(hint.mutation))
{
// the topology has changed, and we are no longer a replica of the mutation - since we don't know which node(s)
// it has been handed over to, re-address the hint to all replicas; see CASSANDRA-5902.
HintsService.instance.writeForAllReplicas(hint);
- reply(id, message.from);
+ respond(message);
}
else
{
// the common path - the node is both the destination and a valid replica for the hint.
- hint.applyFuture().thenAccept(o -> reply(id, message.from)).exceptionally(e -> {logger.debug("Failed to apply hint", e); return null;});
+ hint.applyFuture().thenAccept(o -> respond(message)).exceptionally(e -> {logger.debug("Failed to apply hint", e); return null;});
}
}
- private static void reply(int id, InetAddress to)
+ private static void respond(Message<HintMessage> respondTo)
{
- MessagingService.instance().sendReply(HintResponse.message, id, to);
+ MessagingService.instance().send(respondTo.emptyResponse(), respondTo.from());
}
}
diff --git a/src/java/org/apache/cassandra/hints/HintsBufferPool.java b/src/java/org/apache/cassandra/hints/HintsBufferPool.java
index f705de1..7f66efd 100644
--- a/src/java/org/apache/cassandra/hints/HintsBufferPool.java
+++ b/src/java/org/apache/cassandra/hints/HintsBufferPool.java
@@ -18,14 +18,12 @@
package org.apache.cassandra.hints;
import java.io.Closeable;
-import java.io.IOException;
import java.util.UUID;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import org.apache.cassandra.config.Config;
import org.apache.cassandra.net.MessagingService;
-import sun.nio.ch.DirectBuffer;
/**
* A primitive pool of {@link HintsBuffer} buffers. Under normal conditions should only hold two buffers - the currently
diff --git a/src/java/org/apache/cassandra/hints/HintsDescriptor.java b/src/java/org/apache/cassandra/hints/HintsDescriptor.java
index 00224e2..12d9598 100644
--- a/src/java/org/apache/cassandra/hints/HintsDescriptor.java
+++ b/src/java/org/apache/cassandra/hints/HintsDescriptor.java
@@ -61,7 +61,8 @@
private static final Logger logger = LoggerFactory.getLogger(HintsDescriptor.class);
static final int VERSION_30 = 1;
- static final int CURRENT_VERSION = VERSION_30;
+ static final int VERSION_40 = 2;
+ static final int CURRENT_VERSION = VERSION_40;
static final String COMPRESSION = "compression";
static final String ENCRYPTION = "encryption";
@@ -214,7 +215,9 @@
switch (hintsVersion)
{
case VERSION_30:
- return MessagingService.FORCE_3_0_PROTOCOL_VERSION ? MessagingService.VERSION_30 : MessagingService.VERSION_3014;
+ return MessagingService.VERSION_30;
+ case VERSION_40:
+ return MessagingService.VERSION_40;
default:
throw new AssertionError();
}
diff --git a/src/java/org/apache/cassandra/hints/HintsDispatchExecutor.java b/src/java/org/apache/cassandra/hints/HintsDispatchExecutor.java
index eda4179..b5eb0b1 100644
--- a/src/java/org/apache/cassandra/hints/HintsDispatchExecutor.java
+++ b/src/java/org/apache/cassandra/hints/HintsDispatchExecutor.java
@@ -18,13 +18,12 @@
package org.apache.cassandra.hints;
import java.io.File;
-import java.net.InetAddress;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.BooleanSupplier;
-import java.util.function.Function;
+import java.util.function.Predicate;
import java.util.function.Supplier;
import com.google.common.util.concurrent.RateLimiter;
@@ -36,6 +35,7 @@
import org.apache.cassandra.concurrent.NamedThreadFactory;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.io.FSReadError;
+import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.service.StorageService;
/**
@@ -50,10 +50,10 @@
private final File hintsDirectory;
private final ExecutorService executor;
private final AtomicBoolean isPaused;
- private final Function<InetAddress, Boolean> isAlive;
+ private final Predicate<InetAddressAndPort> isAlive;
private final Map<UUID, Future> scheduledDispatches;
- HintsDispatchExecutor(File hintsDirectory, int maxThreads, AtomicBoolean isPaused, Function<InetAddress, Boolean> isAlive)
+ HintsDispatchExecutor(File hintsDirectory, int maxThreads, AtomicBoolean isPaused, Predicate<InetAddressAndPort> isAlive)
{
this.hintsDirectory = hintsDirectory;
this.isPaused = isPaused;
@@ -154,7 +154,7 @@
public void run()
{
UUID hostId = hostIdSupplier.get();
- InetAddress address = StorageService.instance.getEndpointForHostId(hostId);
+ InetAddressAndPort address = StorageService.instance.getEndpointForHostId(hostId);
logger.info("Transferring all hints to {}: {}", address, hostId);
if (transfer(hostId))
return;
@@ -240,7 +240,7 @@
}
catch (FSReadError e)
{
- logger.error("Failed to dispatch hints file {}: file is corrupted ({})", descriptor.fileName(), e);
+ logger.error(String.format("Failed to dispatch hints file %s: file is corrupted", descriptor.fileName()), e);
store.cleanUp(descriptor);
store.markCorrupted(descriptor);
throw e;
@@ -255,7 +255,7 @@
{
logger.trace("Dispatching hints file {}", descriptor.fileName());
- InetAddress address = StorageService.instance.getEndpointForHostId(hostId);
+ InetAddressAndPort address = StorageService.instance.getEndpointForHostId(hostId);
if (address != null)
return deliver(descriptor, address);
@@ -264,12 +264,12 @@
return true;
}
- private boolean deliver(HintsDescriptor descriptor, InetAddress address)
+ private boolean deliver(HintsDescriptor descriptor, InetAddressAndPort address)
{
File file = new File(hintsDirectory, descriptor.fileName());
InputPosition offset = store.getDispatchOffset(descriptor);
- BooleanSupplier shouldAbort = () -> !isAlive.apply(address) || isPaused.get();
+ BooleanSupplier shouldAbort = () -> !isAlive.test(address) || isPaused.get();
try (HintsDispatcher dispatcher = HintsDispatcher.create(file, rateLimiter, address, descriptor.hostId, shouldAbort))
{
if (offset != null)
@@ -306,4 +306,14 @@
}
}
}
+
+ public boolean isPaused()
+ {
+ return isPaused.get();
+ }
+
+ public boolean hasScheduledDispatches()
+ {
+ return !scheduledDispatches.isEmpty();
+ }
}
diff --git a/src/java/org/apache/cassandra/hints/HintsDispatchTrigger.java b/src/java/org/apache/cassandra/hints/HintsDispatchTrigger.java
index cc1c221..ca38c0c 100644
--- a/src/java/org/apache/cassandra/hints/HintsDispatchTrigger.java
+++ b/src/java/org/apache/cassandra/hints/HintsDispatchTrigger.java
@@ -19,11 +19,11 @@
import java.util.concurrent.atomic.AtomicBoolean;
-import org.apache.cassandra.config.Schema;
import org.apache.cassandra.gms.ApplicationState;
import org.apache.cassandra.gms.Gossiper;
+import org.apache.cassandra.schema.Schema;
-import static org.apache.cassandra.utils.FBUtilities.getBroadcastAddress;
+import static org.apache.cassandra.utils.FBUtilities.getBroadcastAddressAndPort;
/**
* A simple dispatch trigger that's being run every 10 seconds.
diff --git a/src/java/org/apache/cassandra/hints/HintsDispatcher.java b/src/java/org/apache/cassandra/hints/HintsDispatcher.java
index c432553..39e4b25 100644
--- a/src/java/org/apache/cassandra/hints/HintsDispatcher.java
+++ b/src/java/org/apache/cassandra/hints/HintsDispatcher.java
@@ -18,10 +18,8 @@
package org.apache.cassandra.hints;
import java.io.File;
-import java.net.InetAddress;
import java.nio.ByteBuffer;
import java.util.*;
-import java.util.concurrent.TimeUnit;
import java.util.function.BooleanSupplier;
import java.util.function.Function;
@@ -29,17 +27,21 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.cassandra.net.RequestCallback;
import org.apache.cassandra.exceptions.RequestFailureReason;
+import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.metrics.HintsServiceMetrics;
-import org.apache.cassandra.net.IAsyncCallbackWithFailure;
-import org.apache.cassandra.net.MessageIn;
+import org.apache.cassandra.net.Message;
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.utils.concurrent.SimpleCondition;
+import static org.apache.cassandra.net.Verb.HINT_REQ;
+import static org.apache.cassandra.utils.MonotonicClock.approxTime;
+
/**
* Dispatches a single hints file to a specified node in a batched manner.
*
- * Uses either {@link EncodedHintMessage} - when dispatching hints into a node with the same messaging version as the hints file,
+ * Uses either {@link HintMessage.Encoded} - when dispatching hints into a node with the same messaging version as the hints file,
* or {@link HintMessage}, when conversion is required.
*/
final class HintsDispatcher implements AutoCloseable
@@ -49,14 +51,14 @@
private enum Action { CONTINUE, ABORT }
private final HintsReader reader;
- private final UUID hostId;
- private final InetAddress address;
+ final UUID hostId;
+ final InetAddressAndPort address;
private final int messagingVersion;
private final BooleanSupplier abortRequested;
private InputPosition currentPagePosition;
- private HintsDispatcher(HintsReader reader, UUID hostId, InetAddress address, int messagingVersion, BooleanSupplier abortRequested)
+ private HintsDispatcher(HintsReader reader, UUID hostId, InetAddressAndPort address, int messagingVersion, BooleanSupplier abortRequested)
{
currentPagePosition = null;
@@ -67,14 +69,17 @@
this.abortRequested = abortRequested;
}
- static HintsDispatcher create(File file, RateLimiter rateLimiter, InetAddress address, UUID hostId, BooleanSupplier abortRequested)
+ static HintsDispatcher create(File file, RateLimiter rateLimiter, InetAddressAndPort address, UUID hostId, BooleanSupplier abortRequested)
{
- int messagingVersion = MessagingService.instance().getVersion(address);
- return new HintsDispatcher(HintsReader.open(file, rateLimiter), hostId, address, messagingVersion, abortRequested);
+ int messagingVersion = MessagingService.instance().versions.get(address);
+ HintsDispatcher dispatcher = new HintsDispatcher(HintsReader.open(file, rateLimiter), hostId, address, messagingVersion, abortRequested);
+ HintDiagnostics.dispatcherCreated(dispatcher);
+ return dispatcher;
}
public void close()
{
+ HintDiagnostics.dispatcherClosed(this);
reader.close();
}
@@ -110,6 +115,7 @@
// retry in case of a timeout; stop in case of a failure, host going down, or delivery paused
private Action dispatch(HintsReader.Page page)
{
+ HintDiagnostics.dispatchPage(this);
return sendHintsAndAwait(page);
}
@@ -131,33 +137,34 @@
if (action == Action.ABORT)
return action;
- boolean hadFailures = false;
+ long success = 0, failures = 0, timeouts = 0;
for (Callback cb : callbacks)
{
Callback.Outcome outcome = cb.await();
- updateMetrics(outcome);
-
- if (outcome != Callback.Outcome.SUCCESS)
- hadFailures = true;
+ if (outcome == Callback.Outcome.SUCCESS) success++;
+ else if (outcome == Callback.Outcome.FAILURE) failures++;
+ else if (outcome == Callback.Outcome.TIMEOUT) timeouts++;
}
- return hadFailures ? Action.ABORT : Action.CONTINUE;
+ updateMetrics(success, failures, timeouts);
+
+ if (failures > 0 || timeouts > 0)
+ {
+ HintDiagnostics.pageFailureResult(this, success, failures, timeouts);
+ return Action.ABORT;
+ }
+ else
+ {
+ HintDiagnostics.pageSuccessResult(this, success, failures, timeouts);
+ return Action.CONTINUE;
+ }
}
- private void updateMetrics(Callback.Outcome outcome)
+ private void updateMetrics(long success, long failures, long timeouts)
{
- switch (outcome)
- {
- case SUCCESS:
- HintsServiceMetrics.hintsSucceeded.mark();
- break;
- case FAILURE:
- HintsServiceMetrics.hintsFailed.mark();
- break;
- case TIMEOUT:
- HintsServiceMetrics.hintsTimedOut.mark();
- break;
- }
+ HintsServiceMetrics.hintsSucceeded.mark(success);
+ HintsServiceMetrics.hintsFailed.mark(failures);
+ HintsServiceMetrics.hintsTimedOut.mark(timeouts);
}
/*
@@ -169,7 +176,10 @@
while (hints.hasNext())
{
if (abortRequested.getAsBoolean())
+ {
+ HintDiagnostics.abortRequested(this);
return Action.ABORT;
+ }
callbacks.add(sendFunction.apply(hints.next()));
}
return Action.CONTINUE;
@@ -177,9 +187,9 @@
private Callback sendHint(Hint hint)
{
- Callback callback = new Callback();
- HintMessage message = new HintMessage(hostId, hint);
- MessagingService.instance().sendRRWithFailure(message.createMessageOut(), address, callback);
+ Callback callback = new Callback(hint.creationTime);
+ Message<?> message = Message.out(HINT_REQ, new HintMessage(hostId, hint));
+ MessagingService.instance().sendWithCallback(message, address, callback);
return callback;
}
@@ -189,28 +199,32 @@
private Callback sendEncodedHint(ByteBuffer hint)
{
- Callback callback = new Callback();
- EncodedHintMessage message = new EncodedHintMessage(hostId, hint, messagingVersion);
- MessagingService.instance().sendRRWithFailure(message.createMessageOut(), address, callback);
+ HintMessage.Encoded message = new HintMessage.Encoded(hostId, hint, messagingVersion);
+ Callback callback = new Callback(message.getHintCreationTime());
+ MessagingService.instance().sendWithCallback(Message.out(HINT_REQ, message), address, callback);
return callback;
}
- private static final class Callback implements IAsyncCallbackWithFailure
+ private static final class Callback implements RequestCallback
{
enum Outcome { SUCCESS, TIMEOUT, FAILURE, INTERRUPTED }
- private final long start = System.nanoTime();
+ private final long start = approxTime.now();
private final SimpleCondition condition = new SimpleCondition();
private volatile Outcome outcome;
+ private final long hintCreationNanoTime;
+
+ private Callback(long hintCreationTimeMillisSinceEpoch)
+ {
+ this.hintCreationNanoTime = approxTime.translate().fromMillisSinceEpoch(hintCreationTimeMillisSinceEpoch);
+ }
Outcome await()
{
- long timeout = TimeUnit.MILLISECONDS.toNanos(MessagingService.Verb.HINT.getTimeout()) - (System.nanoTime() - start);
boolean timedOut;
-
try
{
- timedOut = !condition.await(timeout, TimeUnit.NANOSECONDS);
+ timedOut = !condition.awaitUntil(HINT_REQ.expiresAtNanos(start));
}
catch (InterruptedException e)
{
@@ -221,23 +235,27 @@
return timedOut ? Outcome.TIMEOUT : outcome;
}
- public void onFailure(InetAddress from, RequestFailureReason failureReason)
+ @Override
+ public boolean invokeOnFailure()
+ {
+ return true;
+ }
+
+ @Override
+ public void onFailure(InetAddressAndPort from, RequestFailureReason failureReason)
{
outcome = Outcome.FAILURE;
condition.signalAll();
}
- public void response(MessageIn msg)
+ @Override
+ public void onResponse(Message msg)
{
+ HintsServiceMetrics.updateDelayMetrics(msg.from(), approxTime.now() - this.hintCreationNanoTime);
outcome = Outcome.SUCCESS;
condition.signalAll();
}
- public boolean isLatencyForSnitch()
- {
- return false;
- }
-
@Override
public boolean supportsBackPressure()
{
diff --git a/src/java/org/apache/cassandra/hints/HintsReader.java b/src/java/org/apache/cassandra/hints/HintsReader.java
index ecc4f27..8d7ad1d 100644
--- a/src/java/org/apache/cassandra/hints/HintsReader.java
+++ b/src/java/org/apache/cassandra/hints/HintsReader.java
@@ -31,12 +31,10 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.cassandra.db.UnknownColumnFamilyException;
+import org.apache.cassandra.exceptions.UnknownTableException;
import org.apache.cassandra.io.FSReadError;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.AbstractIterator;
-import org.apache.cassandra.utils.ByteBufferUtil;
-import org.apache.cassandra.utils.NativeLibrary;
/**
* A paged non-compressed hints reader that provides two iterators:
@@ -166,6 +164,7 @@
final class HintsIterator extends AbstractIterator<Hint>
{
private final InputPosition offset;
+ private final long now = System.currentTimeMillis();
HintsIterator(InputPosition offset)
{
@@ -229,15 +228,15 @@
Hint hint;
try
{
- hint = Hint.serializer.deserialize(input, descriptor.messagingVersion());
+ hint = Hint.serializer.deserializeIfLive(input, now, size, descriptor.messagingVersion());
input.checkLimit(0);
}
- catch (UnknownColumnFamilyException e)
+ catch (UnknownTableException e)
{
logger.warn("Failed to read a hint for {}: {} - table with id {} is unknown in file {}",
StorageService.instance.getEndpointForHostId(descriptor.hostId),
descriptor.hostId,
- e.cfId,
+ e.id,
descriptor.fileName());
input.skipBytes(Ints.checkedCast(size - input.bytesPastLimit()));
@@ -263,6 +262,7 @@
final class BuffersIterator extends AbstractIterator<ByteBuffer>
{
private final InputPosition offset;
+ private final long now = System.currentTimeMillis();
BuffersIterator(InputPosition offset)
{
@@ -323,7 +323,7 @@
rateLimiter.acquire(size);
input.limit(size);
- ByteBuffer buffer = ByteBufferUtil.read(input, size);
+ ByteBuffer buffer = Hint.serializer.readBufferIfLive(input, now, size, descriptor.messagingVersion());
if (input.checkCrc())
return buffer;
diff --git a/src/java/org/apache/cassandra/hints/HintsService.java b/src/java/org/apache/cassandra/hints/HintsService.java
index ae54d9e..1fd2d1a 100644
--- a/src/java/org/apache/cassandra/hints/HintsService.java
+++ b/src/java/org/apache/cassandra/hints/HintsService.java
@@ -18,16 +18,20 @@
package org.apache.cassandra.hints;
import java.io.File;
-import java.net.InetAddress;
import java.net.UnknownHostException;
+import java.util.Collection;
import java.util.Collections;
+import java.util.List;
import java.util.UUID;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.Supplier;
+import java.util.stream.Collectors;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableMap;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.locator.ReplicaLayout;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -36,6 +40,8 @@
import org.apache.cassandra.config.ParameterizedClass;
import org.apache.cassandra.gms.FailureDetector;
import org.apache.cassandra.gms.IFailureDetector;
+import org.apache.cassandra.locator.EndpointsForToken;
+import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.metrics.HintedHandoffMetrics;
import org.apache.cassandra.metrics.StorageMetrics;
import org.apache.cassandra.dht.Token;
@@ -43,9 +49,7 @@
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.MBeanWrapper;
-import static com.google.common.collect.Iterables.filter;
import static com.google.common.collect.Iterables.transform;
-import static com.google.common.collect.Iterables.size;
/**
* A singleton-ish wrapper over various hints components:
@@ -70,8 +74,8 @@
private final HintsCatalog catalog;
private final HintsWriteExecutor writeExecutor;
private final HintsBufferPool bufferPool;
- private final HintsDispatchExecutor dispatchExecutor;
- private final AtomicBoolean isDispatchPaused;
+ final HintsDispatchExecutor dispatchExecutor;
+ final AtomicBoolean isDispatchPaused;
private volatile boolean isShutDown = false;
@@ -140,7 +144,7 @@
* @param hostIds host ids of the hint's target nodes
* @param hint the hint to store
*/
- public void write(Iterable<UUID> hostIds, Hint hint)
+ public void write(Collection<UUID> hostIds, Hint hint)
{
if (isShutDown)
throw new IllegalStateException("HintsService is shut down and can't accept new hints");
@@ -150,7 +154,7 @@
bufferPool.write(hostIds, hint);
- StorageMetrics.totalHints.inc(size(hostIds));
+ StorageMetrics.totalHints.inc(hostIds.size());
}
/**
@@ -172,9 +176,14 @@
String keyspaceName = hint.mutation.getKeyspaceName();
Token token = hint.mutation.key().getToken();
- Iterable<UUID> hostIds =
- transform(filter(StorageService.instance.getNaturalAndPendingEndpoints(keyspaceName, token), StorageProxy::shouldHint),
- StorageService.instance::getHostIdForEndpoint);
+ EndpointsForToken replicas = ReplicaLayout.forTokenWriteLiveAndDown(Keyspace.open(keyspaceName), token).all();
+
+ // judicious use of streams: eagerly materializing probably cheaper
+ // than performing filters / translations 2x extra via Iterables.filter/transform
+ List<UUID> hostIds = replicas.stream()
+ .filter(StorageProxy::shouldHint)
+ .map(replica -> StorageService.instance.getHostIdForEndpoint(replica.endpoint()))
+ .collect(Collectors.toList());
write(hostIds, hint);
}
@@ -198,6 +207,8 @@
isDispatchPaused.set(false);
+ HintsServiceDiagnostics.dispatchingStarted(this);
+
HintsDispatchTrigger trigger = new HintsDispatchTrigger(catalog, writeExecutor, dispatchExecutor, isDispatchPaused);
// triggering hint dispatch is now very cheap, so we can do it more often - every 10 seconds vs. every 10 minutes,
// previously; this reduces mean time to delivery, and positively affects batchlog delivery latencies, too
@@ -208,12 +219,16 @@
{
logger.info("Paused hints dispatch");
isDispatchPaused.set(true);
+
+ HintsServiceDiagnostics.dispatchingPaused(this);
}
public void resumeDispatch()
{
logger.info("Resumed hints dispatch");
isDispatchPaused.set(false);
+
+ HintsServiceDiagnostics.dispatchingResumed(this);
}
/**
@@ -239,6 +254,8 @@
dispatchExecutor.shutdownBlocking();
writeExecutor.shutdownBlocking();
+
+ HintsServiceDiagnostics.dispatchingShutdown(this);
bufferPool.close();
}
@@ -257,10 +274,10 @@
*/
public void deleteAllHintsForEndpoint(String address)
{
- InetAddress target;
+ InetAddressAndPort target;
try
{
- target = InetAddress.getByName(address);
+ target = InetAddressAndPort.getByName(address);
}
catch (UnknownHostException e)
{
@@ -274,7 +291,7 @@
*
* @param target inet address of the target node
*/
- public void deleteAllHintsForEndpoint(InetAddress target)
+ public void deleteAllHintsForEndpoint(InetAddressAndPort target)
{
UUID hostId = StorageService.instance.getHostIdForEndpoint(target);
if (hostId == null)
diff --git a/src/java/org/apache/cassandra/hints/HintsServiceDiagnostics.java b/src/java/org/apache/cassandra/hints/HintsServiceDiagnostics.java
new file mode 100644
index 0000000..f4cf149
--- /dev/null
+++ b/src/java/org/apache/cassandra/hints/HintsServiceDiagnostics.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.hints;
+
+
+import org.apache.cassandra.diag.DiagnosticEventService;
+import org.apache.cassandra.hints.HintsServiceEvent.HintsServiceEventType;
+
+/**
+ * Utility methods for DiagnosticEvents around the HintService.
+ */
+final class HintsServiceDiagnostics
+{
+ private static final DiagnosticEventService service = DiagnosticEventService.instance();
+
+ private HintsServiceDiagnostics()
+ {
+ }
+
+ static void dispatchingStarted(HintsService hintsService)
+ {
+ if (isEnabled(HintsServiceEventType.DISPATCHING_STARTED))
+ service.publish(new HintsServiceEvent(HintsServiceEventType.DISPATCHING_STARTED, hintsService));
+ }
+
+ static void dispatchingShutdown(HintsService hintsService)
+ {
+ if (isEnabled(HintsServiceEventType.DISPATCHING_SHUTDOWN))
+ service.publish(new HintsServiceEvent(HintsServiceEventType.DISPATCHING_SHUTDOWN, hintsService));
+ }
+
+ static void dispatchingPaused(HintsService hintsService)
+ {
+ if (isEnabled(HintsServiceEventType.DISPATCHING_PAUSED))
+ service.publish(new HintsServiceEvent(HintsServiceEventType.DISPATCHING_PAUSED, hintsService));
+ }
+
+ static void dispatchingResumed(HintsService hintsService)
+ {
+ if (isEnabled(HintsServiceEventType.DISPATCHING_RESUMED))
+ service.publish(new HintsServiceEvent(HintsServiceEventType.DISPATCHING_RESUMED, hintsService));
+ }
+
+ private static boolean isEnabled(HintsServiceEventType type)
+ {
+ return service.isEnabled(HintsServiceEvent.class, type);
+ }
+
+}
diff --git a/src/java/org/apache/cassandra/hints/HintsServiceEvent.java b/src/java/org/apache/cassandra/hints/HintsServiceEvent.java
new file mode 100644
index 0000000..72497a0
--- /dev/null
+++ b/src/java/org/apache/cassandra/hints/HintsServiceEvent.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.hints;
+
+import java.io.Serializable;
+import java.util.HashMap;
+
+import org.apache.cassandra.diag.DiagnosticEvent;
+
+/**
+ * DiagnosticEvent implementation for HintService.
+ */
+final class HintsServiceEvent extends DiagnosticEvent
+{
+ enum HintsServiceEventType
+ {
+ DISPATCHING_STARTED,
+ DISPATCHING_PAUSED,
+ DISPATCHING_RESUMED,
+ DISPATCHING_SHUTDOWN
+ }
+
+ private final HintsServiceEventType type;
+ private final HintsService service;
+ private final boolean isDispatchPaused;
+ private final boolean isShutdown;
+ private final boolean dispatchExecutorIsPaused;
+ private final boolean dispatchExecutorHasScheduledDispatches;
+
+ HintsServiceEvent(HintsServiceEventType type, HintsService service)
+ {
+ this.type = type;
+ this.service = service;
+ this.isDispatchPaused = service.isDispatchPaused.get();
+ this.isShutdown = service.isShutDown();
+ this.dispatchExecutorIsPaused = service.dispatchExecutor.isPaused();
+ this.dispatchExecutorHasScheduledDispatches = service.dispatchExecutor.hasScheduledDispatches();
+ }
+
+ public Enum<HintsServiceEventType> getType()
+ {
+ return type;
+ }
+
+ public HashMap<String, Serializable> toMap()
+ {
+ // be extra defensive against nulls and bugs
+ HashMap<String, Serializable> ret = new HashMap<>();
+ ret.put("isDispatchPaused", isDispatchPaused);
+ ret.put("isShutdown", isShutdown);
+ ret.put("dispatchExecutorIsPaused", dispatchExecutorIsPaused);
+ ret.put("dispatchExecutorHasScheduledDispatches", dispatchExecutorHasScheduledDispatches);
+ return ret;
+ }
+}
diff --git a/src/java/org/apache/cassandra/hints/HintsStore.java b/src/java/org/apache/cassandra/hints/HintsStore.java
index b08fc72..aeefbd7 100644
--- a/src/java/org/apache/cassandra/hints/HintsStore.java
+++ b/src/java/org/apache/cassandra/hints/HintsStore.java
@@ -19,7 +19,6 @@
import java.io.File;
import java.io.IOException;
-import java.net.InetAddress;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentLinkedDeque;
@@ -32,6 +31,7 @@
import org.apache.cassandra.gms.FailureDetector;
import org.apache.cassandra.io.FSWriteError;
+import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.SyncUtil;
@@ -84,14 +84,14 @@
return dispatchDequeue.size();
}
- InetAddress address()
+ InetAddressAndPort address()
{
return StorageService.instance.getEndpointForHostId(hostId);
}
boolean isLive()
{
- InetAddress address = address();
+ InetAddressAndPort address = address();
return address != null && FailureDetector.instance.isAlive(address);
}
diff --git a/src/java/org/apache/cassandra/hints/HintsWriter.java b/src/java/org/apache/cassandra/hints/HintsWriter.java
index 48b8c7c..589802b 100644
--- a/src/java/org/apache/cassandra/hints/HintsWriter.java
+++ b/src/java/org/apache/cassandra/hints/HintsWriter.java
@@ -81,18 +81,18 @@
ByteBuffer descriptorBytes = dob.buffer();
updateChecksum(crc, descriptorBytes);
channel.write(descriptorBytes);
+
+ if (descriptor.isEncrypted())
+ return new EncryptedHintsWriter(directory, descriptor, file, channel, fd, crc);
+ if (descriptor.isCompressed())
+ return new CompressedHintsWriter(directory, descriptor, file, channel, fd, crc);
+ return new HintsWriter(directory, descriptor, file, channel, fd, crc);
}
catch (Throwable e)
{
channel.close();
throw e;
}
-
- if (descriptor.isEncrypted())
- return new EncryptedHintsWriter(directory, descriptor, file, channel, fd, crc);
- if (descriptor.isCompressed())
- return new CompressedHintsWriter(directory, descriptor, file, channel, fd, crc);
- return new HintsWriter(directory, descriptor, file, channel, fd, crc);
}
HintsDescriptor descriptor()
@@ -143,6 +143,12 @@
}
}
+ @VisibleForTesting
+ File getFile()
+ {
+ return file;
+ }
+
/**
* Writes byte buffer into the file channel. Buffer should be flipped before calling this
*/
diff --git a/src/java/org/apache/cassandra/hints/LegacyHintsMigrator.java b/src/java/org/apache/cassandra/hints/LegacyHintsMigrator.java
deleted file mode 100644
index 50d8b6e..0000000
--- a/src/java/org/apache/cassandra/hints/LegacyHintsMigrator.java
+++ /dev/null
@@ -1,244 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.hints;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.*;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.TimeUnit;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.cassandra.config.SchemaConstants;
-import org.apache.cassandra.cql3.QueryProcessor;
-import org.apache.cassandra.cql3.UntypedResultSet;
-import org.apache.cassandra.db.*;
-import org.apache.cassandra.db.compaction.CompactionManager;
-import org.apache.cassandra.db.marshal.UUIDType;
-import org.apache.cassandra.db.partitions.PartitionUpdate;
-import org.apache.cassandra.io.FSWriteError;
-import org.apache.cassandra.io.sstable.Descriptor;
-import org.apache.cassandra.io.util.DataInputBuffer;
-import org.apache.cassandra.io.util.FileUtils;
-import org.apache.cassandra.serializers.MarshalException;
-import org.apache.cassandra.utils.FBUtilities;
-
-/**
- * A migrator that goes through the legacy system.hints table and writes all the hints to the new hints storage format.
- */
-@SuppressWarnings("deprecation")
-public final class LegacyHintsMigrator
-{
- private static final Logger logger = LoggerFactory.getLogger(LegacyHintsMigrator.class);
-
- private final File hintsDirectory;
- private final long maxHintsFileSize;
-
- private final ColumnFamilyStore legacyHintsTable;
- private final int pageSize;
-
- public LegacyHintsMigrator(File hintsDirectory, long maxHintsFileSize)
- {
- this.hintsDirectory = hintsDirectory;
- this.maxHintsFileSize = maxHintsFileSize;
-
- legacyHintsTable = Keyspace.open(SchemaConstants.SYSTEM_KEYSPACE_NAME).getColumnFamilyStore(SystemKeyspace.LEGACY_HINTS);
- pageSize = calculatePageSize(legacyHintsTable);
- }
-
- // read fewer columns (mutations) per page if they are very large
- private static int calculatePageSize(ColumnFamilyStore legacyHintsTable)
- {
- int size = 128;
-
- int meanCellCount = legacyHintsTable.getMeanColumns();
- double meanPartitionSize = legacyHintsTable.getMeanPartitionSize();
-
- if (meanCellCount != 0 && meanPartitionSize != 0)
- {
- int avgHintSize = (int) meanPartitionSize / meanCellCount;
- size = Math.max(2, Math.min(size, (512 << 10) / avgHintSize));
- }
-
- return size;
- }
-
- public void migrate()
- {
- // nothing to migrate
- if (legacyHintsTable.isEmpty())
- return;
- logger.info("Migrating legacy hints to new storage");
-
- // major-compact all of the existing sstables to get rid of the tombstones + expired hints
- logger.info("Forcing a major compaction of {}.{} table", SchemaConstants.SYSTEM_KEYSPACE_NAME, SystemKeyspace.LEGACY_HINTS);
- compactLegacyHints();
-
- // paginate over legacy hints and write them to the new storage
- logger.info("Writing legacy hints to the new storage");
- migrateLegacyHints();
-
- // truncate the legacy hints table
- logger.info("Truncating {}.{} table", SchemaConstants.SYSTEM_KEYSPACE_NAME, SystemKeyspace.LEGACY_HINTS);
- legacyHintsTable.truncateBlocking();
- }
-
- private void compactLegacyHints()
- {
- Collection<Descriptor> descriptors = new ArrayList<>();
- legacyHintsTable.getTracker().getUncompacting().forEach(sstable -> descriptors.add(sstable.descriptor));
- if (!descriptors.isEmpty())
- forceCompaction(descriptors);
- }
-
- private void forceCompaction(Collection<Descriptor> descriptors)
- {
- try
- {
- CompactionManager.instance.submitUserDefined(legacyHintsTable, descriptors, FBUtilities.nowInSeconds()).get();
- }
- catch (InterruptedException | ExecutionException e)
- {
- throw new RuntimeException(e);
- }
- }
-
- private void migrateLegacyHints()
- {
- ByteBuffer buffer = ByteBuffer.allocateDirect(256 * 1024);
- String query = String.format("SELECT DISTINCT target_id FROM %s.%s", SchemaConstants.SYSTEM_KEYSPACE_NAME, SystemKeyspace.LEGACY_HINTS);
- //noinspection ConstantConditions
- QueryProcessor.executeInternal(query).forEach(row -> migrateLegacyHints(row.getUUID("target_id"), buffer));
- FileUtils.clean(buffer);
- }
-
- private void migrateLegacyHints(UUID hostId, ByteBuffer buffer)
- {
- String query = String.format("SELECT target_id, hint_id, message_version, mutation, ttl(mutation) AS ttl, writeTime(mutation) AS write_time " +
- "FROM %s.%s " +
- "WHERE target_id = ?",
- SchemaConstants.SYSTEM_KEYSPACE_NAME,
- SystemKeyspace.LEGACY_HINTS);
-
- // read all the old hints (paged iterator), write them in the new format
- UntypedResultSet rows = QueryProcessor.executeInternalWithPaging(query, pageSize, hostId);
- migrateLegacyHints(hostId, rows, buffer);
-
- // delete the whole partition in the legacy table; we would truncate the whole table afterwards, but this allows
- // to not lose progress in case of a terminated conversion
- deleteLegacyHintsPartition(hostId);
- }
-
- private void migrateLegacyHints(UUID hostId, UntypedResultSet rows, ByteBuffer buffer)
- {
- migrateLegacyHints(hostId, rows.iterator(), buffer);
- }
-
- private void migrateLegacyHints(UUID hostId, Iterator<UntypedResultSet.Row> iterator, ByteBuffer buffer)
- {
- do
- {
- migrateLegacyHintsInternal(hostId, iterator, buffer);
- // if there are hints that didn't fit in the previous file, keep calling the method to write to a new
- // file until we get everything written.
- }
- while (iterator.hasNext());
- }
-
- private void migrateLegacyHintsInternal(UUID hostId, Iterator<UntypedResultSet.Row> iterator, ByteBuffer buffer)
- {
- HintsDescriptor descriptor = new HintsDescriptor(hostId, System.currentTimeMillis());
-
- try (HintsWriter writer = HintsWriter.create(hintsDirectory, descriptor))
- {
- try (HintsWriter.Session session = writer.newSession(buffer))
- {
- while (iterator.hasNext())
- {
- Hint hint = convertLegacyHint(iterator.next());
- if (hint != null)
- session.append(hint);
-
- if (session.position() >= maxHintsFileSize)
- break;
- }
- }
- }
- catch (IOException e)
- {
- throw new FSWriteError(e, descriptor.fileName());
- }
- }
-
- private static Hint convertLegacyHint(UntypedResultSet.Row row)
- {
- Mutation mutation = deserializeLegacyMutation(row);
- if (mutation == null)
- return null;
-
- long creationTime = row.getLong("write_time"); // milliseconds, not micros, for the hints table
- int expirationTime = FBUtilities.nowInSeconds() + row.getInt("ttl");
- int originalGCGS = expirationTime - (int) TimeUnit.MILLISECONDS.toSeconds(creationTime);
-
- int gcgs = Math.min(originalGCGS, mutation.smallestGCGS());
-
- return Hint.create(mutation, creationTime, gcgs);
- }
-
- private static Mutation deserializeLegacyMutation(UntypedResultSet.Row row)
- {
- try (DataInputBuffer dib = new DataInputBuffer(row.getBlob("mutation"), true))
- {
- Mutation mutation = Mutation.serializer.deserialize(dib,
- row.getInt("message_version"));
- mutation.getPartitionUpdates().forEach(PartitionUpdate::validate);
- return mutation;
- }
- catch (IOException e)
- {
- logger.error("Failed to migrate a hint for {} from legacy {}.{} table",
- row.getUUID("target_id"),
- SchemaConstants.SYSTEM_KEYSPACE_NAME,
- SystemKeyspace.LEGACY_HINTS,
- e);
- return null;
- }
- catch (MarshalException e)
- {
- logger.warn("Failed to validate a hint for {} from legacy {}.{} table - skipping",
- row.getUUID("target_id"),
- SchemaConstants.SYSTEM_KEYSPACE_NAME,
- SystemKeyspace.LEGACY_HINTS,
- e);
- return null;
- }
- }
-
- private static void deleteLegacyHintsPartition(UUID hostId)
- {
- // intentionally use millis, like the rest of the legacy implementation did, just in case
- Mutation mutation = new Mutation(PartitionUpdate.fullPartitionDelete(SystemKeyspace.LegacyHints,
- UUIDType.instance.decompose(hostId),
- System.currentTimeMillis(),
- FBUtilities.nowInSeconds()));
- mutation.applyUnsafe();
- }
-}
diff --git a/src/java/org/apache/cassandra/hints/SerializableHintMessage.java b/src/java/org/apache/cassandra/hints/SerializableHintMessage.java
new file mode 100644
index 0000000..43c289c
--- /dev/null
+++ b/src/java/org/apache/cassandra/hints/SerializableHintMessage.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.hints;
+
+public interface SerializableHintMessage
+{
+}
diff --git a/src/java/org/apache/cassandra/index/Index.java b/src/java/org/apache/cassandra/index/Index.java
index e254555..6d716be 100644
--- a/src/java/org/apache/cassandra/index/Index.java
+++ b/src/java/org/apache/cassandra/index/Index.java
@@ -26,7 +26,7 @@
import java.util.concurrent.Callable;
import java.util.function.BiFunction;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.cql3.Operator;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.compaction.OperationType;
@@ -44,7 +44,6 @@
import org.apache.cassandra.io.sstable.format.SSTableFlushObserver;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.schema.IndexMetadata;
-import org.apache.cassandra.utils.concurrent.OpOrder;
/**
* Consisting of a top level Index interface and two sub-interfaces which handle read and write operations,
@@ -123,7 +122,7 @@
*
* The input is the map of index options supplied in the WITH clause of a CREATE INDEX statement.
*
- * <pre>{@code public static Map<String, String> validateOptions(Map<String, String> options, CFMetaData cfm);}</pre>
+ * <pre>{@code public static Map<String, String> validateOptions(Map<String, String> options, TableMetadata metadata);}</pre>
*
* In this version, the base table's metadata is also supplied as an argument.
* If both overloaded methods are provided, only the one including the base table's metadata will be invoked.
@@ -136,6 +135,23 @@
*/
public interface Index
{
+ /**
+ * Supported loads. An index could be badly initialized and support only reads i.e.
+ */
+ public enum LoadType
+ {
+ READ, WRITE, ALL, NOOP;
+
+ public boolean supportsWrites()
+ {
+ return this == ALL || this == WRITE;
+ }
+
+ public boolean supportsReads()
+ {
+ return this == ALL || this == READ;
+ }
+ }
/*
* Helpers for building indexes from SSTable data
@@ -156,9 +172,10 @@
*/
public static class CollatedViewIndexBuildingSupport implements IndexBuildingSupport
{
+ @SuppressWarnings("resource")
public SecondaryIndexBuilder getIndexBuildTask(ColumnFamilyStore cfs, Set<Index> indexes, Collection<SSTableReader> sstables)
{
- return new CollatedViewIndexBuilder(cfs, indexes, new ReducingKeyIterator(sstables));
+ return new CollatedViewIndexBuilder(cfs, indexes, new ReducingKeyIterator(sstables), sstables);
}
}
@@ -179,13 +196,32 @@
* single pass through the data. The singleton instance returned from the default method implementation builds
* indexes using a {@code ReducingKeyIterator} to provide a collated view of the SSTable data.
*
- * @return an instance of the index build taski helper. Index implementations which return <b>the same instance</b>
+ * @return an instance of the index build task helper. Index implementations which return <b>the same instance</b>
* will be built using a single task.
*/
default IndexBuildingSupport getBuildTaskSupport()
{
return INDEX_BUILDER_SUPPORT;
}
+
+ /**
+ * Same as {@code getBuildTaskSupport} but can be overloaded with a specific 'recover' logic different than the index building one
+ */
+ default IndexBuildingSupport getRecoveryTaskSupport()
+ {
+ return getBuildTaskSupport();
+ }
+
+ /**
+ * Returns the type of operations supported by the index in case its building has failed and it's needing recovery.
+ *
+ * @param isInitialBuild {@code true} if the failure is for the initial build task on index creation, {@code false}
+ * if the failure is for a full rebuild or recovery.
+ */
+ default LoadType getSupportedLoadTypeOnFailure(boolean isInitialBuild)
+ {
+ return isInitialBuild ? LoadType.WRITE : LoadType.ALL;
+ }
/**
* Return a task to perform any initialization work when a new index instance is created.
@@ -303,7 +339,7 @@
* @return true if the index depends on the supplied column being present; false if the column may be
* safely dropped or modified without adversely affecting the index
*/
- public boolean dependsOn(ColumnDefinition column);
+ public boolean dependsOn(ColumnMetadata column);
/**
* Called to determine whether this index can provide a searcher to execute a query on the
@@ -313,7 +349,7 @@
* @param operator the operator of a search query predicate
* @return true if this index is capable of supporting such expressions, false otherwise
*/
- public boolean supportsExpression(ColumnDefinition column, Operator operator);
+ public boolean supportsExpression(ColumnMetadata column, Operator operator);
/**
* If the index supports custom search expressions using the
@@ -377,7 +413,7 @@
* This can be empty as an update might only contain partition, range and row deletions, but
* the indexer is guaranteed to not get any cells for a column that is not part of {@code columns}.
* @param nowInSec current time of the update operation
- * @param opGroup operation group spanning the update operation
+ * @param ctx WriteContext spanning the update operation
* @param transactionType indicates what kind of update is being performed on the base data
* i.e. a write time insert/update/delete or the result of compaction
* @return the newly created indexer or {@code null} if the index is not interested by the update
@@ -385,9 +421,9 @@
* that type of transaction, ...).
*/
public Indexer indexerFor(DecoratedKey key,
- PartitionColumns columns,
+ RegularAndStaticColumns columns,
int nowInSec,
- OpOrder.Group opGroup,
+ WriteContext ctx,
IndexTransaction.Type transactionType);
/**
diff --git a/src/java/org/apache/cassandra/index/IndexRegistry.java b/src/java/org/apache/cassandra/index/IndexRegistry.java
index 9f5ed02..0cf1cbb 100644
--- a/src/java/org/apache/cassandra/index/IndexRegistry.java
+++ b/src/java/org/apache/cassandra/index/IndexRegistry.java
@@ -21,8 +21,24 @@
package org.apache.cassandra.index;
import java.util.Collection;
+import java.util.Collections;
+import java.util.Optional;
+import java.util.concurrent.Callable;
+import java.util.function.BiFunction;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.Operator;
+import org.apache.cassandra.db.*;
+import org.apache.cassandra.db.filter.RowFilter;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.BytesType;
+import org.apache.cassandra.db.partitions.PartitionIterator;
+import org.apache.cassandra.db.partitions.PartitionUpdate;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.index.transactions.IndexTransaction;
+import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.schema.IndexMetadata;
+import org.apache.cassandra.schema.TableMetadata;
/**
* The collection of all Index instances for a base table.
@@ -34,9 +50,204 @@
*/
public interface IndexRegistry
{
+ /**
+ * An empty {@code IndexRegistry}
+ */
+ public static final IndexRegistry EMPTY = new IndexRegistry()
+ {
+ @Override
+ public void unregisterIndex(Index index)
+ {
+ }
+
+ @Override
+ public void registerIndex(Index index)
+ {
+ }
+
+ @Override
+ public Collection<Index> listIndexes()
+ {
+ return Collections.emptyList();
+ }
+
+ @Override
+ public Index getIndex(IndexMetadata indexMetadata)
+ {
+ return null;
+ }
+
+ @Override
+ public Optional<Index> getBestIndexFor(RowFilter.Expression expression)
+ {
+ return Optional.empty();
+ }
+
+ @Override
+ public void validate(PartitionUpdate update)
+ {
+ }
+ };
+
+ /**
+ * An {@code IndexRegistry} intended for use when Cassandra is initialized in client or tool mode.
+ * Contains a single stub {@code Index} which possesses no actual indexing or searching capabilities
+ * but enables query validation and preparation to succeed. Useful for tools which need to prepare
+ * CQL statements without instantiating the whole ColumnFamilyStore infrastructure.
+ */
+ public static final IndexRegistry NON_DAEMON = new IndexRegistry()
+ {
+ Index index = new Index()
+ {
+ public Callable<?> getInitializationTask()
+ {
+ return null;
+ }
+
+ public IndexMetadata getIndexMetadata()
+ {
+ return null;
+ }
+
+ public Callable<?> getMetadataReloadTask(IndexMetadata indexMetadata)
+ {
+ return null;
+ }
+
+ public void register(IndexRegistry registry)
+ {
+
+ }
+
+ public Optional<ColumnFamilyStore> getBackingTable()
+ {
+ return Optional.empty();
+ }
+
+ public Callable<?> getBlockingFlushTask()
+ {
+ return null;
+ }
+
+ public Callable<?> getInvalidateTask()
+ {
+ return null;
+ }
+
+ public Callable<?> getTruncateTask(long truncatedAt)
+ {
+ return null;
+ }
+
+ public boolean shouldBuildBlocking()
+ {
+ return false;
+ }
+
+ public boolean dependsOn(ColumnMetadata column)
+ {
+ return false;
+ }
+
+ public boolean supportsExpression(ColumnMetadata column, Operator operator)
+ {
+ return true;
+ }
+
+ public AbstractType<?> customExpressionValueType()
+ {
+ return BytesType.instance;
+ }
+
+ public RowFilter getPostIndexQueryFilter(RowFilter filter)
+ {
+ return null;
+ }
+
+ public long getEstimatedResultRows()
+ {
+ return 0;
+ }
+
+ public void validate(PartitionUpdate update) throws InvalidRequestException
+ {
+ }
+
+ public Indexer indexerFor(DecoratedKey key, RegularAndStaticColumns columns, int nowInSec, WriteContext ctx, IndexTransaction.Type transactionType)
+ {
+ return null;
+ }
+
+ public BiFunction<PartitionIterator, ReadCommand, PartitionIterator> postProcessorFor(ReadCommand command)
+ {
+ return null;
+ }
+
+ public Searcher searcherFor(ReadCommand command)
+ {
+ return null;
+ }
+ };
+
+ public void registerIndex(Index index)
+ {
+ }
+
+ public void unregisterIndex(Index index)
+ {
+ }
+
+ public Index getIndex(IndexMetadata indexMetadata)
+ {
+ return index;
+ }
+
+ public Collection<Index> listIndexes()
+ {
+ return Collections.singletonList(index);
+ }
+
+ public Optional<Index> getBestIndexFor(RowFilter.Expression expression)
+ {
+ return Optional.empty();
+ }
+
+ public void validate(PartitionUpdate update)
+ {
+
+ }
+ };
+
void registerIndex(Index index);
void unregisterIndex(Index index);
Index getIndex(IndexMetadata indexMetadata);
Collection<Index> listIndexes();
+
+ Optional<Index> getBestIndexFor(RowFilter.Expression expression);
+
+ /**
+ * Called at write time to ensure that values present in the update
+ * are valid according to the rules of all registered indexes which
+ * will process it. The partition key as well as the clustering and
+ * cell values for each row in the update may be checked by index
+ * implementations
+ *
+ * @param update PartitionUpdate containing the values to be validated by registered Index implementations
+ */
+ void validate(PartitionUpdate update);
+
+ /**
+ * Returns the {@code IndexRegistry} associated to the specified table.
+ *
+ * @param table the table metadata
+ * @return the {@code IndexRegistry} associated to the specified table
+ */
+ public static IndexRegistry obtain(TableMetadata table)
+ {
+ if (!DatabaseDescriptor.isDaemonInitialized())
+ return NON_DAEMON;
+
+ return table.isVirtual() ? EMPTY : Keyspace.openAndGetStore(table).indexManager;
+ }
}
diff --git a/src/java/org/apache/cassandra/index/SecondaryIndexBuilder.java b/src/java/org/apache/cassandra/index/SecondaryIndexBuilder.java
index 8276626..73dc334 100644
--- a/src/java/org/apache/cassandra/index/SecondaryIndexBuilder.java
+++ b/src/java/org/apache/cassandra/index/SecondaryIndexBuilder.java
@@ -25,6 +25,7 @@
public abstract class SecondaryIndexBuilder extends CompactionInfo.Holder
{
public abstract void build();
+
public boolean isGlobal()
{
return false;
diff --git a/src/java/org/apache/cassandra/index/SecondaryIndexManager.java b/src/java/org/apache/cassandra/index/SecondaryIndexManager.java
index c603404..3822549 100644
--- a/src/java/org/apache/cassandra/index/SecondaryIndexManager.java
+++ b/src/java/org/apache/cassandra/index/SecondaryIndexManager.java
@@ -20,6 +20,7 @@
import java.lang.reflect.Constructor;
import java.util.*;
import java.util.concurrent.*;
+import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@@ -29,21 +30,27 @@
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import com.google.common.primitives.Longs;
+import com.google.common.util.concurrent.FutureCallback;
import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.common.util.concurrent.MoreExecutors;
+import com.google.common.util.concurrent.SettableFuture;
+
import org.apache.commons.lang3.StringUtils;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.concurrent.JMXEnabledThreadPoolExecutor;
import org.apache.cassandra.concurrent.NamedThreadFactory;
-import org.apache.cassandra.concurrent.StageManager;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.concurrent.Stage;
import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.cql3.statements.IndexTarget;
+import org.apache.cassandra.cql3.statements.schema.IndexTarget;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.compaction.CompactionManager;
import org.apache.cassandra.db.filter.RowFilter;
@@ -52,17 +59,21 @@
import org.apache.cassandra.db.partitions.*;
import org.apache.cassandra.db.rows.*;
import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.index.Index.IndexBuildingSupport;
import org.apache.cassandra.index.internal.CassandraIndex;
import org.apache.cassandra.index.transactions.*;
import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.notifications.INotification;
+import org.apache.cassandra.notifications.INotificationConsumer;
+import org.apache.cassandra.notifications.SSTableAddedNotification;
+import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.schema.IndexMetadata;
import org.apache.cassandra.schema.Indexes;
import org.apache.cassandra.service.pager.SinglePartitionPager;
import org.apache.cassandra.tracing.Tracing;
import org.apache.cassandra.transport.ProtocolVersion;
-import org.apache.cassandra.transport.Server;
import org.apache.cassandra.utils.FBUtilities;
-import org.apache.cassandra.utils.concurrent.OpOrder;
+import org.apache.cassandra.utils.JVMStabilityInspector;
import org.apache.cassandra.utils.concurrent.Refs;
import static org.apache.cassandra.utils.ExecutorUtils.awaitTermination;
@@ -72,7 +83,7 @@
* Handles the core maintenance functionality associated with indexes: adding/removing them to or from
* a table, (re)building during bootstrap or other streaming operations, flushing, reloading metadata
* and so on.
- *
+ * <br><br>
* The Index interface defines a number of methods which return {@code Callable<?>}. These are primarily the
* management tasks for an index implementation. Most of them are currently executed in a blocking
* fashion via submission to SIM's blockingExecutor. This provides the desired behaviour in pretty
@@ -81,70 +92,102 @@
* then be defined with as void and called directly from SIM (rather than being run via the executor service).
* Separating the task defintion from execution gives us greater flexibility though, so that in future, for example,
* if the flush process allows it we leave open the possibility of executing more of these tasks asynchronously.
- *
+ * <br><br>
* The primary exception to the above is the Callable returned from Index#addIndexedColumn. This may
* involve a significant effort, building a new index over any existing data. We perform this task asynchronously;
* as it is called as part of a schema update, which we do not want to block for a long period. Building non-custom
* indexes is performed on the CompactionManager.
- *
+ * <br><br>
* This class also provides instances of processors which listen to updates to the base table and forward to
* registered Indexes the info required to keep those indexes up to date.
* There are two variants of these processors, each with a factory method provided by SIM:
- * IndexTransaction: deals with updates generated on the regular write path.
- * CleanupTransaction: used when partitions are modified during compaction or cleanup operations.
+ * IndexTransaction: deals with updates generated on the regular write path.
+ * CleanupTransaction: used when partitions are modified during compaction or cleanup operations.
* Further details on their usage and lifecycles can be found in the interface definitions below.
- *
- * Finally, the bestIndexFor method is used at query time to identify the most selective index of those able
+ * <br><br>
+ * The bestIndexFor method is used at query time to identify the most selective index of those able
* to satisfy any search predicates defined by a ReadCommand's RowFilter. It returns a thin IndexAccessor object
* which enables the ReadCommand to access the appropriate functions of the Index at various stages in its lifecycle.
* e.g. the getEstimatedResultRows is required when StorageProxy calculates the initial concurrency factor for
* distributing requests to replicas, whereas a Searcher instance is needed when the ReadCommand is executed locally on
* a target replica.
+ * <br><br>
+ * Finally, this class provides a clear and safe lifecycle to manage index builds, either full rebuilds via
+ * {@link this#rebuildIndexesBlocking(Set)} or builds of new sstables
+ * added via {@link org.apache.cassandra.notifications.SSTableAddedNotification}s, guaranteeing
+ * the following:
+ * <ul>
+ * <li>The initialization task and any subsequent successful (re)build mark the index as built.</li>
+ * <li>If any (re)build operation fails, the index is not marked as built, and only another full rebuild can mark the
+ * index as built.</li>
+ * <li>Full rebuilds cannot be run concurrently with other full or sstable (re)builds.</li>
+ * <li>SSTable builds can always be run concurrently with any other builds.</li>
+ * </ul>
*/
-public class SecondaryIndexManager implements IndexRegistry
+public class SecondaryIndexManager implements IndexRegistry, INotificationConsumer
{
private static final Logger logger = LoggerFactory.getLogger(SecondaryIndexManager.class);
// default page size (in rows) when rebuilding the index for a whole partition
public static final int DEFAULT_PAGE_SIZE = 10000;
- private Map<String, Index> indexes = Maps.newConcurrentMap();
+ /**
+ * All registered indexes.
+ */
+ private final Map<String, Index> indexes = Maps.newConcurrentMap();
/**
- * The indexes that are ready to server requests.
+ * The indexes that had a build failure.
*/
- private Set<String> builtIndexes = Sets.newConcurrentHashSet();
+ private final Set<String> needsFullRebuild = Sets.newConcurrentHashSet();
+
+ /**
+ * The indexes that are available for querying.
+ */
+ private final Set<String> queryableIndexes = Sets.newConcurrentHashSet();
+
+ /**
+ * The indexes that are available for writing.
+ */
+ private final Map<String, Index> writableIndexes = Maps.newConcurrentMap();
+
+ /**
+ * The count of pending index builds for each index.
+ */
+ private final Map<String, AtomicInteger> inProgressBuilds = Maps.newConcurrentMap();
// executes tasks returned by Indexer#addIndexColumn which may require index(es) to be (re)built
- private static final ExecutorService asyncExecutor =
- new JMXEnabledThreadPoolExecutor(1,
- StageManager.KEEPALIVE,
- TimeUnit.SECONDS,
- new LinkedBlockingQueue<>(),
- new NamedThreadFactory("SecondaryIndexManagement"),
- "internal");
+ private static final ListeningExecutorService asyncExecutor = MoreExecutors.listeningDecorator(
+ new JMXEnabledThreadPoolExecutor(1,
+ Stage.KEEP_ALIVE_SECONDS,
+ TimeUnit.SECONDS,
+ new LinkedBlockingQueue<>(),
+ new NamedThreadFactory("SecondaryIndexManagement"),
+ "internal"));
// executes all blocking tasks produced by Indexers e.g. getFlushTask, getMetadataReloadTask etc
- private static final ExecutorService blockingExecutor = MoreExecutors.newDirectExecutorService();
+ private static final ListeningExecutorService blockingExecutor = MoreExecutors.newDirectExecutorService();
/**
* The underlying column family containing the source data for these indexes
*/
public final ColumnFamilyStore baseCfs;
+ private final Keyspace keyspace;
public SecondaryIndexManager(ColumnFamilyStore baseCfs)
{
this.baseCfs = baseCfs;
+ this.keyspace = baseCfs.keyspace;
+ baseCfs.getTracker().subscribe(this);
}
-
/**
* Drops and adds new indexes associated with the underlying CF
*/
public void reload()
{
// figure out what needs to be added and dropped.
- Indexes tableIndexes = baseCfs.metadata.getIndexes();
+ Indexes tableIndexes = baseCfs.metadata().indexes;
indexes.keySet()
.stream()
.filter(indexName -> !tableIndexes.has(indexName))
@@ -153,7 +196,7 @@
// we call add for every index definition in the collection as
// some may not have been created here yet, only added to schema
for (IndexMetadata tableIndex : tableIndexes)
- addIndex(tableIndex);
+ addIndex(tableIndex, false);
}
private Future<?> reloadIndex(IndexMetadata indexDef)
@@ -165,35 +208,73 @@
: blockingExecutor.submit(reloadTask);
}
- private Future<?> createIndex(IndexMetadata indexDef)
+ @SuppressWarnings("unchecked")
+ private synchronized Future<?> createIndex(IndexMetadata indexDef, boolean isNewCF)
{
- Index index = createInstance(indexDef);
+ final Index index = createInstance(indexDef);
index.register(this);
+ if (writableIndexes.put(index.getIndexMetadata().name, index) == null)
+ logger.info("Index [{}] registered and writable.", index.getIndexMetadata().name);
+ markIndexesBuilding(ImmutableSet.of(index), true, isNewCF);
+
+ Callable<?> initialBuildTask = null;
// if the index didn't register itself, we can probably assume that no initialization needs to happen
- final Callable<?> initialBuildTask = indexes.containsKey(indexDef.name)
- ? index.getInitializationTask()
- : null;
+ if (indexes.containsKey(indexDef.name))
+ {
+ try
+ {
+ initialBuildTask = index.getInitializationTask();
+ }
+ catch (Throwable t)
+ {
+ logAndMarkIndexesFailed(Collections.singleton(index), t, true);
+ throw t;
+ }
+ }
+
+ // if there's no initialization, just mark as built and return:
if (initialBuildTask == null)
{
- // We need to make sure that the index is marked as built in the case where the initialBuildTask
- // does not need to be run (if the index didn't register itself or if the base table was empty).
- markIndexBuilt(indexDef.name);
+ markIndexBuilt(index, true);
return Futures.immediateFuture(null);
}
- return asyncExecutor.submit(index.getInitializationTask());
+
+ // otherwise run the initialization task asynchronously with a callback to mark it built or failed
+ final SettableFuture initialization = SettableFuture.create();
+ Futures.addCallback(asyncExecutor.submit(initialBuildTask), new FutureCallback()
+ {
+ @Override
+ public void onFailure(Throwable t)
+ {
+ logAndMarkIndexesFailed(Collections.singleton(index), t, true);
+ initialization.setException(t);
+ }
+
+ @Override
+ public void onSuccess(Object o)
+ {
+ markIndexBuilt(index, true);
+ initialization.set(o);
+ }
+ }, MoreExecutors.directExecutor());
+
+ return initialization;
}
/**
* Adds and builds a index
+ *
* @param indexDef the IndexMetadata describing the index
+ * @param isNewCF true if the index is added as part of a new table/columnfamily (i.e. loading a CF at startup),
+ * false for all other cases (i.e. newly added index)
*/
- public synchronized Future<?> addIndex(IndexMetadata indexDef)
+ public synchronized Future<?> addIndex(IndexMetadata indexDef, boolean isNewCF)
{
if (indexes.containsKey(indexDef.name))
return reloadIndex(indexDef);
else
- return createIndex(indexDef);
+ return createIndex(indexDef, isNewCF);
}
/**
@@ -204,7 +285,31 @@
*/
public boolean isIndexQueryable(Index index)
{
- return builtIndexes.contains(index.getIndexMetadata().name);
+ return queryableIndexes.contains(index.getIndexMetadata().name);
+ }
+
+ /**
+ * Checks if the specified index is writable.
+ *
+ * @param index the index
+ * @return <code>true</code> if the specified index is writable, <code>false</code> otherwise
+ */
+ public boolean isIndexWritable(Index index)
+ {
+ return writableIndexes.containsKey(index.getIndexMetadata().name);
+ }
+
+ /**
+ * Checks if the specified index has any running build task.
+ *
+ * @param indexName the index name
+ * @return {@code true} if the index is building, {@code false} otherwise
+ */
+ @VisibleForTesting
+ public synchronized boolean isIndexBuilding(String indexName)
+ {
+ AtomicInteger counter = inProgressBuilds.get(indexName);
+ return counter != null && counter.get() > 0;
}
public synchronized void removeIndex(String indexName)
@@ -213,12 +318,12 @@
if (null != index)
{
markIndexRemoved(indexName);
- executeBlocking(index.getInvalidateTask());
+ executeBlocking(index.getInvalidateTask(), null);
}
}
- public Set<IndexMetadata> getDependentIndexes(ColumnDefinition column)
+ public Set<IndexMetadata> getDependentIndexes(ColumnMetadata column)
{
if (indexes.isEmpty())
return Collections.emptySet();
@@ -236,58 +341,57 @@
*/
public void markAllIndexesRemoved()
{
- getBuiltIndexNames().forEach(this::markIndexRemoved);
+ getBuiltIndexNames().forEach(this::markIndexRemoved);
}
/**
- * Does a full, blocking rebuild of the indexes specified by columns from the sstables.
- * Caller must acquire and release references to the sstables used here.
- * Note also that only this method of (re)building indexes:
- * a) takes a set of index *names* rather than Indexers
- * b) marks exsiting indexes removed prior to rebuilding
- *
- * @param sstables the data to build from
- * @param indexNames the list of indexes to be rebuilt
- */
- public void rebuildIndexesBlocking(Collection<SSTableReader> sstables, Set<String> indexNames)
+ * Does a blocking full rebuild/recovery of the specifed indexes from all the sstables in the base table.
+ * Note also that this method of (re)building/recovering indexes:
+ * a) takes a set of index *names* rather than Indexers
+ * b) marks existing indexes removed prior to rebuilding
+ * c) fails if such marking operation conflicts with any ongoing index builds, as full rebuilds cannot be run
+ * concurrently
+ *
+ * @param indexNames the list of indexes to be rebuilt
+ */
+ public void rebuildIndexesBlocking(Set<String> indexNames)
{
- Set<Index> toRebuild = indexes.values().stream()
- .filter(index -> indexNames.contains(index.getIndexMetadata().name))
- .filter(Index::shouldBuildBlocking)
- .collect(Collectors.toSet());
+ // Get the set of indexes that require blocking build
+ Set<Index> toRebuild = indexes.values()
+ .stream()
+ .filter(index -> indexNames.contains(index.getIndexMetadata().name))
+ .filter(Index::shouldBuildBlocking)
+ .collect(Collectors.toSet());
+
if (toRebuild.isEmpty())
{
logger.info("No defined indexes with the supplied names: {}", Joiner.on(',').join(indexNames));
return;
}
- toRebuild.forEach(indexer -> markIndexRemoved(indexer.getIndexMetadata().name));
-
- buildIndexesBlocking(sstables, toRebuild);
-
- toRebuild.forEach(indexer -> markIndexBuilt(indexer.getIndexMetadata().name));
- }
-
- public void buildAllIndexesBlocking(Collection<SSTableReader> sstables)
- {
- buildIndexesBlocking(sstables, indexes.values()
- .stream()
- .filter(Index::shouldBuildBlocking)
- .collect(Collectors.toSet()));
- }
-
- // For convenience, may be called directly from Index impls
- public void buildIndexBlocking(Index index)
- {
- if (index.shouldBuildBlocking())
+ // Optimistically mark the indexes as writable, so we don't miss incoming writes
+ boolean needsFlush = false;
+ for (Index index : toRebuild)
{
- try (ColumnFamilyStore.RefViewFragment viewFragment = baseCfs.selectAndReference(View.selectFunction(SSTableSet.CANONICAL));
- Refs<SSTableReader> sstables = viewFragment.refs)
+ String name = index.getIndexMetadata().name;
+ if (writableIndexes.put(name, index) == null)
{
- buildIndexesBlocking(sstables, Collections.singleton(index));
- markIndexBuilt(index.getIndexMetadata().name);
+ logger.info("Index [{}] became writable starting recovery.", name);
+ needsFlush = true;
}
}
+
+ // Once we are tracking new writes, flush any memtable contents to not miss them from the sstable-based rebuild
+ if (needsFlush)
+ baseCfs.forceBlockingFlush();
+
+ // Now that we are tracking new writes and we haven't left untracked contents on the memtables, we are ready to
+ // index the sstables
+ try (ColumnFamilyStore.RefViewFragment viewFragment = baseCfs.selectAndReference(View.selectFunction(SSTableSet.CANONICAL));
+ Refs<SSTableReader> allSSTables = viewFragment.refs)
+ {
+ buildIndexesBlocking(allSSTables, toRebuild, true);
+ }
}
/**
@@ -361,55 +465,282 @@
return StringUtils.substringAfter(cfName, Directories.SECONDARY_INDEX_NAME_SEPARATOR);
}
- private void buildIndexesBlocking(Collection<SSTableReader> sstables, Set<Index> indexes)
+ /**
+ * Performs a blocking (re)indexing/recovery of the specified SSTables for the specified indexes.
+ *
+ * If the index doesn't support ALL {@link Index.LoadType} it performs a recovery {@link Index#getRecoveryTaskSupport()}
+ * instead of a build {@link Index#getBuildTaskSupport()}
+ *
+ * @param sstables the SSTables to be (re)indexed
+ * @param indexes the indexes to be (re)built for the specifed SSTables
+ * @param isFullRebuild True if this method is invoked as a full index rebuild, false otherwise
+ */
+ @SuppressWarnings({ "unchecked" })
+ private void buildIndexesBlocking(Collection<SSTableReader> sstables, Set<Index> indexes, boolean isFullRebuild)
{
if (indexes.isEmpty())
return;
- logger.info("Submitting index build of {} for data in {}",
- indexes.stream().map(i -> i.getIndexMetadata().name).collect(Collectors.joining(",")),
- sstables.stream().map(SSTableReader::toString).collect(Collectors.joining(",")));
+ // Mark all indexes as building: this step must happen first, because if any index can't be marked, the whole
+ // process needs to abort
+ markIndexesBuilding(indexes, isFullRebuild, false);
- Map<Index.IndexBuildingSupport, Set<Index>> byType = new HashMap<>();
- for (Index index : indexes)
+ // Build indexes in a try/catch, so that any index not marked as either built or failed will be marked as failed:
+ final Set<Index> builtIndexes = Sets.newConcurrentHashSet();
+ final Set<Index> unbuiltIndexes = Sets.newConcurrentHashSet();
+
+ // Any exception thrown during index building that could be suppressed by the finally block
+ Exception accumulatedFail = null;
+
+ try
{
- Set<Index> stored = byType.computeIfAbsent(index.getBuildTaskSupport(), i -> new HashSet<>());
- stored.add(index);
+ logger.info("Submitting index {} of {} for data in {}",
+ isFullRebuild ? "recovery" : "build",
+ indexes.stream().map(i -> i.getIndexMetadata().name).collect(Collectors.joining(",")),
+ sstables.stream().map(SSTableReader::toString).collect(Collectors.joining(",")));
+
+ // Group all building tasks
+ Map<Index.IndexBuildingSupport, Set<Index>> byType = new HashMap<>();
+ for (Index index : indexes)
+ {
+ IndexBuildingSupport buildOrRecoveryTask = isFullRebuild
+ ? index.getBuildTaskSupport()
+ : index.getRecoveryTaskSupport();
+ Set<Index> stored = byType.computeIfAbsent(buildOrRecoveryTask, i -> new HashSet<>());
+ stored.add(index);
+ }
+
+ // Schedule all index building tasks with a callback to mark them as built or failed
+ List<Future<?>> futures = new ArrayList<>(byType.size());
+ byType.forEach((buildingSupport, groupedIndexes) ->
+ {
+ SecondaryIndexBuilder builder = buildingSupport.getIndexBuildTask(baseCfs, groupedIndexes, sstables);
+ final SettableFuture build = SettableFuture.create();
+ Futures.addCallback(CompactionManager.instance.submitIndexBuild(builder), new FutureCallback()
+ {
+ @Override
+ public void onFailure(Throwable t)
+ {
+ logAndMarkIndexesFailed(groupedIndexes, t, false);
+ unbuiltIndexes.addAll(groupedIndexes);
+ build.setException(t);
+ }
+
+ @Override
+ public void onSuccess(Object o)
+ {
+ groupedIndexes.forEach(i -> markIndexBuilt(i, isFullRebuild));
+ logger.info("Index build of {} completed", getIndexNames(groupedIndexes));
+ builtIndexes.addAll(groupedIndexes);
+ build.set(o);
+ }
+ }, MoreExecutors.directExecutor());
+ futures.add(build);
+ });
+
+ // Finally wait for the index builds to finish and flush the indexes that built successfully
+ FBUtilities.waitOnFutures(futures);
}
+ catch (Exception e)
+ {
+ accumulatedFail = e;
+ throw e;
+ }
+ finally
+ {
+ try
+ {
+ // Fail any indexes that couldn't be marked
+ Set<Index> failedIndexes = Sets.difference(indexes, Sets.union(builtIndexes, unbuiltIndexes));
+ if (!failedIndexes.isEmpty())
+ {
+ logAndMarkIndexesFailed(failedIndexes, accumulatedFail, false);
+ }
- List<Future<?>> futures = byType.entrySet()
- .stream()
- .map((e) -> e.getKey().getIndexBuildTask(baseCfs, e.getValue(), sstables))
- .map(CompactionManager.instance::submitIndexBuild)
- .collect(Collectors.toList());
+ // Flush all built indexes with an aynchronous callback to log the success or failure of the flush
+ flushIndexesBlocking(builtIndexes, new FutureCallback()
+ {
+ String indexNames = StringUtils.join(builtIndexes.stream()
+ .map(i -> i.getIndexMetadata().name)
+ .collect(Collectors.toList()), ',');
- FBUtilities.waitOnFutures(futures);
+ @Override
+ public void onFailure(Throwable ignored)
+ {
+ logger.info("Index flush of {} failed", indexNames);
+ }
- flushIndexesBlocking(indexes);
- logger.info("Index build of {} complete",
- indexes.stream().map(i -> i.getIndexMetadata().name).collect(Collectors.joining(",")));
+ @Override
+ public void onSuccess(Object ignored)
+ {
+ logger.info("Index flush of {} completed", indexNames);
+ }
+ });
+ }
+ catch (Exception e)
+ {
+ if (accumulatedFail != null)
+ {
+ accumulatedFail.addSuppressed(e);
+ }
+ else
+ {
+ throw e;
+ }
+ }
+ }
+ }
+
+ private String getIndexNames(Set<Index> indexes)
+ {
+ List<String> indexNames = indexes.stream()
+ .map(i -> i.getIndexMetadata().name)
+ .collect(Collectors.toList());
+ return StringUtils.join(indexNames, ',');
}
/**
- * Marks the specified index as build.
- * <p>This method is public as it need to be accessible from the {@link Index} implementations</p>
- * @param indexName the index name
+ * Marks the specified indexes as (re)building if:
+ * 1) There's no in progress rebuild of any of the given indexes.
+ * 2) There's an in progress rebuild but the caller is not a full rebuild.
+ * <p>
+ * Otherwise, this method invocation fails, as it is not possible to run full rebuilds while other concurrent rebuilds
+ * are in progress. Please note this is checked atomically against all given indexes; that is, no index will be marked
+ * if even a single one fails.
+ * <p>
+ * Marking an index as "building" practically means:
+ * 1) The index is removed from the "failed" set if this is a full rebuild.
+ * 2) The index is removed from the system keyspace built indexes; this only happens if this method is not invoked
+ * for a new table initialization, as in such case there's no need to remove it (it is either already not present,
+ * or already present because already built).
+ * <p>
+ * Thread safety is guaranteed by having all methods managing index builds synchronized: being synchronized on
+ * the SecondaryIndexManager instance, it means all invocations for all different indexes will go through the same
+ * lock, but this is fine as the work done while holding such lock is trivial.
+ * <p>
+ * {@link #markIndexBuilt(Index, boolean)} or {@link #markIndexFailed(Index, boolean)} should be always called after
+ * the rebuilding has finished, so that the index build state can be correctly managed and the index rebuilt.
+ *
+ * @param indexes the index to be marked as building
+ * @param isFullRebuild {@code true} if this method is invoked as a full index rebuild, {@code false} otherwise
+ * @param isNewCF {@code true} if this method is invoked when initializing a new table/columnfamily (i.e. loading a CF at startup),
+ * {@code false} for all other cases (i.e. newly added index)
*/
- public void markIndexBuilt(String indexName)
+ private synchronized void markIndexesBuilding(Set<Index> indexes, boolean isFullRebuild, boolean isNewCF)
{
- builtIndexes.add(indexName);
- if (DatabaseDescriptor.isDaemonInitialized())
- SystemKeyspace.setIndexBuilt(baseCfs.keyspace.getName(), indexName);
+ String keyspaceName = baseCfs.keyspace.getName();
+
+ // First step is to validate against concurrent rebuilds; it would be more optimized to do everything on a single
+ // step, but we're not really expecting a very high number of indexes, and this isn't on any hot path, so
+ // we're favouring readability over performance
+ indexes.forEach(index ->
+ {
+ String indexName = index.getIndexMetadata().name;
+ AtomicInteger counter = inProgressBuilds.computeIfAbsent(indexName, ignored -> new AtomicInteger(0));
+
+ if (counter.get() > 0 && isFullRebuild)
+ throw new IllegalStateException(String.format("Cannot rebuild index %s as another index build for the same index is currently in progress.", indexName));
+ });
+
+ // Second step is the actual marking:
+ indexes.forEach(index ->
+ {
+ String indexName = index.getIndexMetadata().name;
+ AtomicInteger counter = inProgressBuilds.computeIfAbsent(indexName, ignored -> new AtomicInteger(0));
+
+ if (isFullRebuild)
+ needsFullRebuild.remove(indexName);
+
+ if (counter.getAndIncrement() == 0 && DatabaseDescriptor.isDaemonInitialized() && !isNewCF)
+ SystemKeyspace.setIndexRemoved(keyspaceName, indexName);
+ });
+ }
+
+ /**
+ * Marks the specified index as built if there are no in progress index builds and the index is not failed.
+ * {@link #markIndexesBuilding(Set, boolean, boolean)} should always be invoked before this method.
+ *
+ * @param index the index to be marked as built
+ * @param isFullRebuild {@code true} if this method is invoked as a full index rebuild, {@code false} otherwise
+ */
+ private synchronized void markIndexBuilt(Index index, boolean isFullRebuild)
+ {
+ String indexName = index.getIndexMetadata().name;
+ if (isFullRebuild)
+ {
+ if (queryableIndexes.add(indexName))
+ logger.info("Index [{}] became queryable after successful build.", indexName);
+
+ if (writableIndexes.put(indexName, index) == null)
+ logger.info("Index [{}] became writable after successful build.", indexName);
+ }
+
+ AtomicInteger counter = inProgressBuilds.get(indexName);
+ if (counter != null)
+ {
+ assert counter.get() > 0;
+ if (counter.decrementAndGet() == 0)
+ {
+ inProgressBuilds.remove(indexName);
+ if (!needsFullRebuild.contains(indexName) && DatabaseDescriptor.isDaemonInitialized())
+ SystemKeyspace.setIndexBuilt(baseCfs.keyspace.getName(), indexName);
+ }
+ }
+ }
+
+ /**
+ * Marks the specified index as failed.
+ * {@link #markIndexesBuilding(Set, boolean, boolean)} should always be invoked before this method.
+ *
+ * @param index the index to be marked as built
+ * @param isInitialBuild {@code true} if the index failed during its initial build, {@code false} otherwise
+ */
+ private synchronized void markIndexFailed(Index index, boolean isInitialBuild)
+ {
+ String indexName = index.getIndexMetadata().name;
+
+ AtomicInteger counter = inProgressBuilds.get(indexName);
+ if (counter != null)
+ {
+ assert counter.get() > 0;
+
+ counter.decrementAndGet();
+
+ if (DatabaseDescriptor.isDaemonInitialized())
+ SystemKeyspace.setIndexRemoved(baseCfs.keyspace.getName(), indexName);
+
+ needsFullRebuild.add(indexName);
+
+ if (!index.getSupportedLoadTypeOnFailure(isInitialBuild).supportsWrites() && writableIndexes.remove(indexName) != null)
+ logger.info("Index [{}] became not-writable because of failed build.", indexName);
+
+ if (!index.getSupportedLoadTypeOnFailure(isInitialBuild).supportsReads() && queryableIndexes.remove(indexName))
+ logger.info("Index [{}] became not-queryable because of failed build.", indexName);
+ }
+ }
+
+ private void logAndMarkIndexesFailed(Set<Index> indexes, Throwable indexBuildFailure, boolean isInitialBuild)
+ {
+ JVMStabilityInspector.inspectThrowable(indexBuildFailure);
+ if (indexBuildFailure != null)
+ logger.warn("Index build of {} failed. Please run full index rebuild to fix it.", getIndexNames(indexes), indexBuildFailure);
+ else
+ logger.warn("Index build of {} failed. Please run full index rebuild to fix it.", getIndexNames(indexes));
+ indexes.forEach(i -> this.markIndexFailed(i, isInitialBuild));
}
/**
* Marks the specified index as removed.
- * <p>This method is public as it need to be accessible from the {@link Index} implementations</p>
+ *
* @param indexName the index name
*/
- public void markIndexRemoved(String indexName)
+ private synchronized void markIndexRemoved(String indexName)
{
SystemKeyspace.setIndexRemoved(baseCfs.keyspace.getName(), indexName);
+ queryableIndexes.remove(indexName);
+ writableIndexes.remove(indexName);
+ needsFullRebuild.remove(indexName);
+ inProgressBuilds.remove(indexName);
}
public Index getIndexByName(String indexName)
@@ -424,7 +755,7 @@
{
assert indexDef.options != null;
String className = indexDef.options.get(IndexTarget.CUSTOM_INDEX_OPTION_NAME);
- assert ! Strings.isNullOrEmpty(className);
+ assert !Strings.isNullOrEmpty(className);
try
{
Class<? extends Index> indexClass = FBUtilities.classForName(className, "Index");
@@ -448,16 +779,22 @@
*/
public void truncateAllIndexesBlocking(final long truncatedAt)
{
- executeAllBlocking(indexes.values().stream(), (index) -> index.getTruncateTask(truncatedAt));
+ executeAllBlocking(indexes.values().stream(), (index) -> index.getTruncateTask(truncatedAt), null);
}
/**
* Remove all indexes
*/
- public void invalidateAllIndexesBlocking()
+ public void dropAllIndexes()
{
markAllIndexesRemoved();
- executeAllBlocking(indexes.values().stream(), Index::getInvalidateTask);
+ invalidateAllIndexesBlocking();
+ }
+
+ @VisibleForTesting
+ public void invalidateAllIndexesBlocking()
+ {
+ executeAllBlocking(indexes.values().stream(), Index::getInvalidateTask, null);
}
/**
@@ -465,7 +802,7 @@
*/
public void flushAllIndexesBlocking()
{
- flushIndexesBlocking(ImmutableSet.copyOf(indexes.values()));
+ flushIndexesBlocking(ImmutableSet.copyOf(indexes.values()));
}
/**
@@ -473,6 +810,34 @@
*/
public void flushIndexesBlocking(Set<Index> indexes)
{
+ flushIndexesBlocking(indexes, null);
+ }
+
+ /**
+ * Performs a blocking flush of all custom indexes
+ */
+ public void flushAllNonCFSBackedIndexesBlocking()
+ {
+ executeAllBlocking(indexes.values()
+ .stream()
+ .filter(index -> !index.getBackingTable().isPresent()),
+ Index::getBlockingFlushTask, null);
+ }
+
+ /**
+ * Performs a blocking execution of pre-join tasks of all indexes
+ */
+ public void executePreJoinTasksBlocking(boolean hadBootstrap)
+ {
+ logger.info("Executing pre-join{} tasks for: {}", hadBootstrap ? " post-bootstrap" : "", this.baseCfs);
+ executeAllBlocking(indexes.values().stream(), (index) ->
+ {
+ return index.getPreJoinTask(hadBootstrap);
+ }, null);
+ }
+
+ private void flushIndexesBlocking(Set<Index> indexes, FutureCallback<Object> callback)
+ {
if (indexes.isEmpty())
return;
@@ -484,46 +849,24 @@
synchronized (baseCfs.getTracker())
{
indexes.forEach(index ->
- index.getBackingTable()
- .map(cfs -> wait.add(cfs.forceFlush()))
- .orElseGet(() -> nonCfsIndexes.add(index)));
+ index.getBackingTable()
+ .map(cfs -> wait.add(cfs.forceFlush()))
+ .orElseGet(() -> nonCfsIndexes.add(index)));
}
- executeAllBlocking(nonCfsIndexes.stream(), Index::getBlockingFlushTask);
+ executeAllBlocking(nonCfsIndexes.stream(), Index::getBlockingFlushTask, callback);
FBUtilities.waitOnFutures(wait);
}
/**
- * Performs a blocking flush of all custom indexes
- */
- public void flushAllNonCFSBackedIndexesBlocking()
- {
- executeAllBlocking(indexes.values()
- .stream()
- .filter(index -> !index.getBackingTable().isPresent()),
- Index::getBlockingFlushTask);
- }
-
- /**
- * Performs a blocking execution of pre-join tasks of all indexes
- */
- public void executePreJoinTasksBlocking(boolean hadBootstrap)
- {
- logger.info("Executing pre-join{} tasks for: {}", hadBootstrap ? " post-bootstrap" : "", this.baseCfs);
- executeAllBlocking(indexes.values().stream(), (index) -> {
- return index.getPreJoinTask(hadBootstrap);
- });
- }
-
- /**
* @return all indexes which are marked as built and ready to use
*/
public List<String> getBuiltIndexNames()
{
Set<String> allIndexNames = new HashSet<>();
indexes.values().stream()
- .map(i -> i.getIndexMetadata().name)
- .forEach(allIndexNames::add);
+ .map(i -> i.getIndexMetadata().name)
+ .forEach(allIndexNames::add);
return SystemKeyspace.getBuiltIndexes(baseCfs.keyspace.getName(), allIndexNames);
}
@@ -551,11 +894,11 @@
public void indexPartition(DecoratedKey key, Set<Index> indexes, int pageSize)
{
if (logger.isTraceEnabled())
- logger.trace("Indexing partition {}", baseCfs.metadata.getKeyValidator().getString(key.getKey()));
+ logger.trace("Indexing partition {}", baseCfs.metadata().partitionKeyType.getString(key.getKey()));
if (!indexes.isEmpty())
{
- SinglePartitionReadCommand cmd = SinglePartitionReadCommand.fullPartitionRead(baseCfs.metadata,
+ SinglePartitionReadCommand cmd = SinglePartitionReadCommand.fullPartitionRead(baseCfs.metadata(),
FBUtilities.nowInSeconds(),
key);
int nowInSec = cmd.nowInSec();
@@ -565,18 +908,19 @@
while (!pager.isExhausted())
{
try (ReadExecutionController controller = cmd.executionController();
- OpOrder.Group writeGroup = Keyspace.writeOrder.start();
- UnfilteredPartitionIterator page = pager.fetchPageUnfiltered(baseCfs.metadata, pageSize, controller))
+ WriteContext ctx = keyspace.getWriteHandler().createContextForIndexing();
+ UnfilteredPartitionIterator page = pager.fetchPageUnfiltered(baseCfs.metadata(), pageSize, controller))
{
if (!page.hasNext())
break;
- try (UnfilteredRowIterator partition = page.next()) {
+ try (UnfilteredRowIterator partition = page.next())
+ {
Set<Index.Indexer> indexers = indexes.stream()
.map(index -> index.indexerFor(key,
partition.columns(),
nowInSec,
- writeGroup,
+ ctx,
IndexTransaction.Type.UPDATE))
.filter(Objects::nonNull)
.collect(Collectors.toSet());
@@ -645,11 +989,11 @@
if (meanPartitionSize <= 0)
return DEFAULT_PAGE_SIZE;
- int meanCellsPerPartition = baseCfs.getMeanColumns();
+ int meanCellsPerPartition = baseCfs.getMeanEstimatedCellPerPartitionCount();
if (meanCellsPerPartition <= 0)
return DEFAULT_PAGE_SIZE;
- int columnsPerRow = baseCfs.metadata.partitionColumns().regulars.size();
+ int columnsPerRow = baseCfs.metadata().regularColumns().size();
if (columnsPerRow <= 0)
return DEFAULT_PAGE_SIZE;
@@ -660,8 +1004,8 @@
logger.trace("Calculated page size {} for indexing {}.{} ({}/{}/{}/{})",
pageSize,
- baseCfs.metadata.ksName,
- baseCfs.metadata.cfName,
+ baseCfs.metadata.keyspace,
+ baseCfs.metadata.name,
meanPartitionSize,
meanCellsPerPartition,
meanRowsPerPartition,
@@ -673,7 +1017,7 @@
/**
* Delete all data from all indexes for this partition.
* For when cleanup rips a partition out entirely.
- *
+ * <p>
* TODO : improve cleanup transaction to batch updates and perform them async
*/
public void deletePartition(UnfilteredRowIterator partition, int nowInSec)
@@ -698,28 +1042,28 @@
partition.columns(),
nowInSec);
indexTransaction.start();
- indexTransaction.onRowDelete((Row)unfiltered);
+ indexTransaction.onRowDelete((Row) unfiltered);
indexTransaction.commit();
}
}
/**
* Called at query time to choose which (if any) of the registered index implementations to use for a given query.
- *
+ * <p>
* This is a two step processes, firstly compiling the set of searchable indexes then choosing the one which reduces
* the search space the most.
- *
+ * <p>
* In the first phase, if the command's RowFilter contains any custom index expressions, the indexes that they
* specify are automatically included. Following that, the registered indexes are filtered to include only those
* which support the standard expressions in the RowFilter.
- *
+ * <p>
* The filtered set then sorted by selectivity, as reported by the Index implementations' getEstimatedResultRows
* method.
- *
+ * <p>
* Implementation specific validation of the target expression, either custom or standard, by the selected
* index should be performed in the searcherFor method to ensure that we pick the right index regardless of
* the validity of the expression.
- *
+ * <p>
* This method is only called once during the lifecycle of a ReadCommand and the result is
* cached for future use when obtaining a Searcher, getting the index's underlying CFS for
* ReadOrderGroup, or an estimate of the result size from an average index query.
@@ -740,7 +1084,7 @@
{
// Only a single custom expression is allowed per query and, if present,
// we want to always favour the index specified in such an expression
- RowFilter.CustomExpression customExpression = (RowFilter.CustomExpression)expression;
+ RowFilter.CustomExpression customExpression = (RowFilter.CustomExpression) expression;
logger.trace("Command contains a custom index expression, using target index {}", customExpression.getTargetIndex().name);
Tracing.trace("Command contains a custom index expression, using target index {}", customExpression.getTargetIndex().name);
return indexes.get(customExpression.getTargetIndex().name);
@@ -789,6 +1133,7 @@
* will process it. The partition key as well as the clustering and
* cell values for each row in the update may be checked by index
* implementations
+ *
* @param update PartitionUpdate containing the values to be validated by registered Index implementations
* @throws InvalidRequestException
*/
@@ -798,9 +1143,10 @@
index.validate(update);
}
- /**
+ /*
* IndexRegistry methods
*/
+
public void registerIndex(Index index)
{
String name = index.getIndexMetadata().name;
@@ -816,9 +1162,7 @@
private Index unregisterIndex(String name)
{
Index removed = indexes.remove(name);
- builtIndexes.remove(name);
- logger.trace(removed == null ? "Index {} was not registered" : "Removed index {} from registry",
- name);
+ logger.trace(removed == null ? "Index {} was not registered" : "Removed index {} from registry", name);
return removed;
}
@@ -832,7 +1176,7 @@
return ImmutableSet.copyOf(indexes.values());
}
- /**
+ /*
* Handling of index updates.
* Implementations of the various IndexTransaction interfaces, for keeping indexes in sync with base data
* during updates, compaction and cleanup. Plus factory methods for obtaining transaction instances.
@@ -841,46 +1185,48 @@
/**
* Transaction for updates on the write path.
*/
- public UpdateTransaction newUpdateTransaction(PartitionUpdate update, OpOrder.Group opGroup, int nowInSec)
+ public UpdateTransaction newUpdateTransaction(PartitionUpdate update, WriteContext ctx, int nowInSec)
{
if (!hasIndexes())
return UpdateTransaction.NO_OP;
-
- Index.Indexer[] indexers = indexes.values().stream()
- .map(i -> i.indexerFor(update.partitionKey(),
- update.columns(),
- nowInSec,
- opGroup,
- IndexTransaction.Type.UPDATE))
- .filter(Objects::nonNull)
- .toArray(Index.Indexer[]::new);
-
- return indexers.length == 0 ? UpdateTransaction.NO_OP : new WriteTimeTransaction(indexers);
+
+ ArrayList<Index.Indexer> idxrs = new ArrayList<>();
+ for (Index i : writableIndexes.values())
+ {
+ Index.Indexer idxr = i.indexerFor(update.partitionKey(), update.columns(), nowInSec, ctx, IndexTransaction.Type.UPDATE);
+ if (idxr != null)
+ idxrs.add(idxr);
+ }
+
+ if (idxrs.size() == 0)
+ return UpdateTransaction.NO_OP;
+ else
+ return new WriteTimeTransaction(idxrs.toArray(new Index.Indexer[idxrs.size()]));
}
/**
* Transaction for use when merging rows during compaction
*/
public CompactionTransaction newCompactionTransaction(DecoratedKey key,
- PartitionColumns partitionColumns,
+ RegularAndStaticColumns regularAndStaticColumns,
int versions,
int nowInSec)
{
// the check for whether there are any registered indexes is already done in CompactionIterator
- return new IndexGCTransaction(key, partitionColumns, versions, nowInSec, listIndexes());
+ return new IndexGCTransaction(key, regularAndStaticColumns, keyspace, versions, nowInSec, writableIndexes.values());
}
/**
* Transaction for use when removing partitions during cleanup
*/
public CleanupTransaction newCleanupTransaction(DecoratedKey key,
- PartitionColumns partitionColumns,
+ RegularAndStaticColumns regularAndStaticColumns,
int nowInSec)
{
if (!hasIndexes())
return CleanupTransaction.NO_OP;
- return new CleanupGCTransaction(key, partitionColumns, nowInSec, listIndexes());
+ return new CleanupGCTransaction(key, regularAndStaticColumns, keyspace, nowInSec, writableIndexes.values());
}
/**
@@ -890,7 +1236,7 @@
{
private final Index.Indexer[] indexers;
- private WriteTimeTransaction(Index.Indexer...indexers)
+ private WriteTimeTransaction(Index.Indexer... indexers)
{
// don't allow null indexers, if we don't need any use a NullUpdater object
for (Index.Indexer indexer : indexers) assert indexer != null;
@@ -942,7 +1288,7 @@
{
}
- public void onComplexDeletion(int i, Clustering clustering, ColumnDefinition column, DeletionTime merged, DeletionTime original)
+ public void onComplexDeletion(int i, Clustering clustering, ColumnMetadata column, DeletionTime merged, DeletionTime original)
{
}
@@ -953,7 +1299,6 @@
if (merged == null || (original != null && shouldCleanupOldValue(original, merged)))
toRemove.addCell(original);
-
}
};
Rows.diff(diffListener, updated, existing);
@@ -993,7 +1338,8 @@
private static final class IndexGCTransaction implements CompactionTransaction
{
private final DecoratedKey key;
- private final PartitionColumns columns;
+ private final RegularAndStaticColumns columns;
+ private final Keyspace keyspace;
private final int versions;
private final int nowInSec;
private final Collection<Index> indexes;
@@ -1001,13 +1347,15 @@
private Row[] rows;
private IndexGCTransaction(DecoratedKey key,
- PartitionColumns columns,
+ RegularAndStaticColumns columns,
+ Keyspace keyspace,
int versions,
int nowInSec,
Collection<Index> indexes)
{
this.key = key;
this.columns = columns;
+ this.keyspace = keyspace;
this.versions = versions;
this.indexes = indexes;
this.nowInSec = nowInSec;
@@ -1019,7 +1367,7 @@
rows = new Row[versions];
}
- public void onRowMerge(Row merged, Row...versions)
+ public void onRowMerge(Row merged, Row... versions)
{
// Diff listener constructs rows representing deltas between the merged and original versions
// These delta rows are then passed to registered indexes for removal processing
@@ -1036,7 +1384,7 @@
{
}
- public void onComplexDeletion(int i, Clustering clustering, ColumnDefinition column, DeletionTime merged, DeletionTime original)
+ public void onComplexDeletion(int i, Clustering clustering, ColumnMetadata column, DeletionTime merged, DeletionTime original)
{
}
@@ -1059,7 +1407,7 @@
Rows.diff(diffListener, merged, versions);
- for(int i = 0; i < builders.length; i++)
+ for (int i = 0; i < builders.length; i++)
if (builders[i] != null)
rows[i] = builders[i].build();
}
@@ -1069,11 +1417,11 @@
if (rows == null)
return;
- try (OpOrder.Group opGroup = Keyspace.writeOrder.start())
+ try (WriteContext ctx = keyspace.getWriteHandler().createContextForIndexing())
{
for (Index index : indexes)
{
- Index.Indexer indexer = index.indexerFor(key, columns, nowInSec, opGroup, Type.COMPACTION);
+ Index.Indexer indexer = index.indexerFor(key, columns, nowInSec, ctx, Type.COMPACTION);
if (indexer == null)
continue;
@@ -1096,7 +1444,8 @@
private static final class CleanupGCTransaction implements CleanupTransaction
{
private final DecoratedKey key;
- private final PartitionColumns columns;
+ private final RegularAndStaticColumns columns;
+ private final Keyspace keyspace;
private final int nowInSec;
private final Collection<Index> indexes;
@@ -1104,12 +1453,14 @@
private DeletionTime partitionDelete;
private CleanupGCTransaction(DecoratedKey key,
- PartitionColumns columns,
+ RegularAndStaticColumns columns,
+ Keyspace keyspace,
int nowInSec,
Collection<Index> indexes)
{
this.key = key;
this.columns = columns;
+ this.keyspace = keyspace;
this.indexes = indexes;
this.nowInSec = nowInSec;
}
@@ -1133,11 +1484,11 @@
if (row == null && partitionDelete == null)
return;
- try (OpOrder.Group opGroup = Keyspace.writeOrder.start())
+ try (WriteContext ctx = keyspace.getWriteHandler().createContextForIndexing())
{
for (Index index : indexes)
{
- Index.Indexer indexer = index.indexerFor(key, columns, nowInSec, opGroup, Type.CLEANUP);
+ Index.Indexer indexer = index.indexerFor(key, columns, nowInSec, ctx, Type.CLEANUP);
if (indexer == null)
continue;
@@ -1155,13 +1506,17 @@
}
}
- private static void executeBlocking(Callable<?> task)
+ private void executeBlocking(Callable<?> task, FutureCallback<Object> callback)
{
if (null != task)
- FBUtilities.waitOnFuture(blockingExecutor.submit(task));
+ {
+ ListenableFuture<?> f = blockingExecutor.submit(task);
+ if (callback != null) Futures.addCallback(f, callback, MoreExecutors.directExecutor());
+ FBUtilities.waitOnFuture(f);
+ }
}
- private static void executeAllBlocking(Stream<Index> indexers, Function<Index, Callable<?>> function)
+ private void executeAllBlocking(Stream<Index> indexers, Function<Index, Callable<?>> function, FutureCallback<Object> callback)
{
if (function == null)
{
@@ -1170,19 +1525,40 @@
}
List<Future<?>> waitFor = new ArrayList<>();
- indexers.forEach(indexer -> {
- Callable<?> task = function.apply(indexer);
- if (null != task)
- waitFor.add(blockingExecutor.submit(task));
- });
+ indexers.forEach(indexer ->
+ {
+ Callable<?> task = function.apply(indexer);
+ if (null != task)
+ {
+ ListenableFuture<?> f = blockingExecutor.submit(task);
+ if (callback != null) Futures.addCallback(f, callback, MoreExecutors.directExecutor());
+ waitFor.add(f);
+ }
+ });
FBUtilities.waitOnFutures(waitFor);
}
- @VisibleForTesting
- public static void shutdownAndWait(long timeout, TimeUnit unit) throws InterruptedException, TimeoutException
+ public void handleNotification(INotification notification, Object sender)
{
- ExecutorService[] executors = new ExecutorService[]{ asyncExecutor, blockingExecutor };
- shutdown(executors);
- awaitTermination(timeout, unit, executors);
+ if (!indexes.isEmpty() && notification instanceof SSTableAddedNotification)
+ {
+ SSTableAddedNotification notice = (SSTableAddedNotification) notification;
+
+ // SSTables asociated to a memtable come from a flush, so their contents have already been indexed
+ if (!notice.memtable().isPresent())
+ buildIndexesBlocking(Lists.newArrayList(notice.added),
+ indexes.values()
+ .stream()
+ .filter(Index::shouldBuildBlocking)
+ .collect(Collectors.toSet()),
+ false);
+ }
+ }
+
+ @VisibleForTesting
+ public static void shutdownAndWait(long timeout, TimeUnit units) throws InterruptedException, TimeoutException
+ {
+ shutdown(asyncExecutor, blockingExecutor);
+ awaitTermination(timeout, units, asyncExecutor, blockingExecutor);
}
}
diff --git a/src/java/org/apache/cassandra/index/TargetParser.java b/src/java/org/apache/cassandra/index/TargetParser.java
index 96d03af..9ada4c6 100644
--- a/src/java/org/apache/cassandra/index/TargetParser.java
+++ b/src/java/org/apache/cassandra/index/TargetParser.java
@@ -22,10 +22,10 @@
import org.apache.commons.lang3.StringUtils;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.cql3.ColumnIdentifier;
-import org.apache.cassandra.cql3.statements.IndexTarget;
+import org.apache.cassandra.cql3.statements.schema.IndexTarget;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.schema.IndexMetadata;
import org.apache.cassandra.utils.Pair;
@@ -36,17 +36,17 @@
private static final Pattern TWO_QUOTES = Pattern.compile("\"\"");
private static final String QUOTE = "\"";
- public static Pair<ColumnDefinition, IndexTarget.Type> parse(CFMetaData cfm, IndexMetadata indexDef)
+ public static Pair<ColumnMetadata, IndexTarget.Type> parse(TableMetadata metadata, IndexMetadata indexDef)
{
String target = indexDef.options.get("target");
assert target != null : String.format("No target definition found for index %s", indexDef.name);
- Pair<ColumnDefinition, IndexTarget.Type> result = parse(cfm, target);
+ Pair<ColumnMetadata, IndexTarget.Type> result = parse(metadata, target);
if (result == null)
throw new ConfigurationException(String.format("Unable to parse targets for index %s (%s)", indexDef.name, target));
return result;
}
- public static Pair<ColumnDefinition, IndexTarget.Type> parse(CFMetaData cfm, String target)
+ public static Pair<ColumnMetadata, IndexTarget.Type> parse(TableMetadata metadata, String target)
{
// if the regex matches then the target is in the form "keys(foo)", "entries(bar)" etc
// if not, then it must be a simple column name and implictly its type is VALUES
@@ -80,11 +80,11 @@
// in that case we have to do a linear scan of the cfm's columns to get the matching one.
// After dropping compact storage (see CASSANDRA-10857), we can't distinguish between the
// former compact/thrift table, so we have to fall back to linear scan in both cases.
- ColumnDefinition cd = cfm.getColumnDefinition(new ColumnIdentifier(columnName, true));
+ ColumnMetadata cd = metadata.getColumn(new ColumnIdentifier(columnName, true));
if (cd != null)
return Pair.create(cd, targetType);
- for (ColumnDefinition column : cfm.allColumns())
+ for (ColumnMetadata column : metadata.columns())
if (column.name.toString().equals(columnName))
return Pair.create(column, targetType);
diff --git a/src/java/org/apache/cassandra/index/internal/CassandraIndex.java b/src/java/org/apache/cassandra/index/internal/CassandraIndex.java
index e23882f..f74a656 100644
--- a/src/java/org/apache/cassandra/index/internal/CassandraIndex.java
+++ b/src/java/org/apache/cassandra/index/internal/CassandraIndex.java
@@ -33,10 +33,11 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.schema.TableMetadataRef;
+import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.cql3.Operator;
-import org.apache.cassandra.cql3.statements.IndexTarget;
+import org.apache.cassandra.cql3.statements.schema.IndexTarget;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.compaction.CompactionManager;
import org.apache.cassandra.db.filter.RowFilter;
@@ -60,7 +61,6 @@
import org.apache.cassandra.schema.IndexMetadata;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.Pair;
-import org.apache.cassandra.utils.concurrent.OpOrder;
import org.apache.cassandra.utils.concurrent.Refs;
import static org.apache.cassandra.cql3.statements.RequestValidations.checkFalse;
@@ -76,7 +76,7 @@
public final ColumnFamilyStore baseCfs;
protected IndexMetadata metadata;
protected ColumnFamilyStore indexCfs;
- protected ColumnDefinition indexedColumn;
+ protected ColumnMetadata indexedColumn;
protected CassandraIndexFunctions functions;
protected CassandraIndex(ColumnFamilyStore baseCfs, IndexMetadata indexDef)
@@ -91,7 +91,7 @@
* @param operator
* @return
*/
- protected boolean supportsOperator(ColumnDefinition indexedColumn, Operator operator)
+ protected boolean supportsOperator(ColumnMetadata indexedColumn, Operator operator)
{
return operator == Operator.EQ;
}
@@ -144,15 +144,15 @@
Clustering clustering,
CellPath path,
ByteBuffer cellValue);
-
- public ColumnDefinition getIndexedColumn()
+
+ public ColumnMetadata getIndexedColumn()
{
return indexedColumn;
}
public ClusteringComparator getIndexComparator()
{
- return indexCfs.metadata.comparator;
+ return indexCfs.metadata().comparator;
}
public ColumnFamilyStore getIndexCfs()
@@ -201,7 +201,6 @@
public Callable<?> getMetadataReloadTask(IndexMetadata indexDef)
{
return () -> {
- indexCfs.metadata.reloadIndexMetadataProperties(baseCfs.metadata);
indexCfs.reload();
return null;
};
@@ -223,12 +222,12 @@
private void setMetadata(IndexMetadata indexDef)
{
metadata = indexDef;
- Pair<ColumnDefinition, IndexTarget.Type> target = TargetParser.parse(baseCfs.metadata, indexDef);
+ Pair<ColumnMetadata, IndexTarget.Type> target = TargetParser.parse(baseCfs.metadata(), indexDef);
functions = getFunctions(indexDef, target);
- CFMetaData cfm = indexCfsMetadata(baseCfs.metadata, indexDef);
+ TableMetadataRef tableRef = TableMetadataRef.forOfflineTools(indexCfsMetadata(baseCfs.metadata(), indexDef));
indexCfs = ColumnFamilyStore.createColumnFamilyStore(baseCfs.keyspace,
- cfm.cfName,
- cfm,
+ tableRef.name,
+ tableRef,
baseCfs.getTracker().loadsstables);
indexedColumn = target.left;
}
@@ -247,12 +246,12 @@
return true;
}
- public boolean dependsOn(ColumnDefinition column)
+ public boolean dependsOn(ColumnMetadata column)
{
return indexedColumn.name.equals(column.name);
}
- public boolean supportsExpression(ColumnDefinition column, Operator operator)
+ public boolean supportsExpression(ColumnMetadata column, Operator operator)
{
return indexedColumn.name.equals(column.name)
&& supportsOperator(indexedColumn, operator);
@@ -270,26 +269,7 @@
public long getEstimatedResultRows()
{
- long totalRows = 0;
- long totalPartitions = 0;
- for (SSTableReader sstable : indexCfs.getSSTables(SSTableSet.CANONICAL))
- {
- if (sstable.descriptor.version.storeRows())
- {
- totalPartitions += sstable.getEstimatedPartitionSize().count();
- totalRows += sstable.getTotalRows();
- } else
- {
- // for legacy sstables we don't have a total row count so we approximate it
- // using estimated column count (which is the same logic as pre-3.0
- // see CASSANDRA-15259
- long colCount = sstable.getEstimatedColumnCount().count();
- totalPartitions += colCount;
- totalRows += sstable.getEstimatedColumnCount().mean() * colCount;
- }
- }
-
- return totalPartitions > 0 ? (int) (totalRows / totalPartitions) : 0;
+ return indexCfs.getMeanRowCount();
}
/**
@@ -357,9 +337,9 @@
}
public Indexer indexerFor(final DecoratedKey key,
- final PartitionColumns columns,
+ final RegularAndStaticColumns columns,
final int nowInSec,
- final OpOrder.Group opGroup,
+ final WriteContext ctx,
final IndexTransaction.Type transactionType)
{
/**
@@ -464,7 +444,7 @@
clustering,
cell,
LivenessInfo.withExpirationTime(cell.timestamp(), cell.ttl(), cell.localDeletionTime()),
- opGroup);
+ ctx);
}
private void removeCells(Clustering clustering, Iterable<Cell> cells)
@@ -481,7 +461,7 @@
if (cell == null || !cell.isLive(nowInSec))
return;
- delete(key.getKey(), clustering, cell, opGroup, nowInSec);
+ delete(key.getKey(), clustering, cell, ctx, nowInSec);
}
private void indexPrimaryKey(final Clustering clustering,
@@ -489,10 +469,10 @@
final Row.Deletion deletion)
{
if (liveness.timestamp() != LivenessInfo.NO_TIMESTAMP)
- insert(key.getKey(), clustering, null, liveness, opGroup);
+ insert(key.getKey(), clustering, null, liveness, ctx);
if (!deletion.isLive())
- delete(key.getKey(), clustering, deletion.time(), opGroup);
+ delete(key.getKey(), clustering, deletion.time(), ctx);
}
private LivenessInfo getPrimaryKeyIndexLiveness(Row row)
@@ -522,14 +502,14 @@
* @param indexKey the partition key in the index table
* @param indexClustering the clustering in the index table
* @param deletion deletion timestamp etc
- * @param opGroup the operation under which to perform the deletion
+ * @param ctx the write context under which to perform the deletion
*/
public void deleteStaleEntry(DecoratedKey indexKey,
Clustering indexClustering,
DeletionTime deletion,
- OpOrder.Group opGroup)
+ WriteContext ctx)
{
- doDelete(indexKey, indexClustering, deletion, opGroup);
+ doDelete(indexKey, indexClustering, deletion, ctx);
logger.trace("Removed index entry for stale value {}", indexKey);
}
@@ -540,14 +520,14 @@
Clustering clustering,
Cell cell,
LivenessInfo info,
- OpOrder.Group opGroup)
+ WriteContext ctx)
{
DecoratedKey valueKey = getIndexKeyFor(getIndexedValue(rowKey,
clustering,
cell));
Row row = BTreeRow.noCellLiveRow(buildIndexClustering(rowKey, clustering, cell), info);
PartitionUpdate upd = partitionUpdate(valueKey, row);
- indexCfs.apply(upd, UpdateTransaction.NO_OP, opGroup, null);
+ indexCfs.getWriteHandler().write(upd, ctx, UpdateTransaction.NO_OP);
logger.trace("Inserted entry into index for value {}", valueKey);
}
@@ -557,7 +537,7 @@
private void delete(ByteBuffer rowKey,
Clustering clustering,
Cell cell,
- OpOrder.Group opGroup,
+ WriteContext ctx,
int nowInSec)
{
DecoratedKey valueKey = getIndexKeyFor(getIndexedValue(rowKey,
@@ -566,7 +546,7 @@
doDelete(valueKey,
buildIndexClustering(rowKey, clustering, cell),
new DeletionTime(cell.timestamp(), nowInSec),
- opGroup);
+ ctx);
}
/**
@@ -575,7 +555,7 @@
private void delete(ByteBuffer rowKey,
Clustering clustering,
DeletionTime deletion,
- OpOrder.Group opGroup)
+ WriteContext ctx)
{
DecoratedKey valueKey = getIndexKeyFor(getIndexedValue(rowKey,
clustering,
@@ -583,17 +563,17 @@
doDelete(valueKey,
buildIndexClustering(rowKey, clustering, null),
deletion,
- opGroup);
+ ctx);
}
private void doDelete(DecoratedKey indexKey,
Clustering indexClustering,
DeletionTime deletion,
- OpOrder.Group opGroup)
+ WriteContext ctx)
{
Row row = BTreeRow.emptyDeletedRow(indexClustering, Row.Deletion.regular(deletion));
PartitionUpdate upd = partitionUpdate(indexKey, row);
- indexCfs.apply(upd, UpdateTransaction.NO_OP, opGroup, null);
+ indexCfs.getWriteHandler().write(upd, ctx, UpdateTransaction.NO_OP);
logger.trace("Removed index entry for value {}", indexKey);
}
@@ -637,11 +617,10 @@
{
if (value != null && value.remaining() >= FBUtilities.MAX_UNSIGNED_SHORT)
throw new InvalidRequestException(String.format(
- "Cannot index value of size %d for index %s on %s.%s(%s) (maximum allowed size=%d)",
+ "Cannot index value of size %d for index %s on %s(%s) (maximum allowed size=%d)",
value.remaining(),
metadata.name,
- baseCfs.metadata.ksName,
- baseCfs.metadata.cfName,
+ baseCfs.metadata,
indexedColumn.name.toString(),
FBUtilities.MAX_UNSIGNED_SHORT));
}
@@ -673,15 +652,15 @@
private PartitionUpdate partitionUpdate(DecoratedKey valueKey, Row row)
{
- return PartitionUpdate.singleRowUpdate(indexCfs.metadata, valueKey, row);
+ return PartitionUpdate.singleRowUpdate(indexCfs.metadata(), valueKey, row);
}
private void invalidate()
{
// interrupt in-progress compactions
Collection<ColumnFamilyStore> cfss = Collections.singleton(indexCfs);
- CompactionManager.instance.interruptCompactionForCFs(cfss, true);
- CompactionManager.instance.waitForCessation(cfss);
+ CompactionManager.instance.interruptCompactionForCFs(cfss, (sstable) -> true, true);
+ CompactionManager.instance.waitForCessation(cfss, (sstable) -> true);
Keyspace.writeOrder.awaitNewBarrier();
indexCfs.forceBlockingFlush();
indexCfs.readOrdering.awaitNewBarrier();
@@ -706,6 +685,7 @@
};
}
+ @SuppressWarnings("resource")
private void buildBlocking()
{
baseCfs.forceBlockingFlush();
@@ -716,10 +696,9 @@
if (sstables.isEmpty())
{
logger.info("No SSTable data for {}.{} to build index {} from, marking empty index as built",
- baseCfs.metadata.ksName,
- baseCfs.metadata.cfName,
+ baseCfs.metadata.keyspace,
+ baseCfs.metadata.name,
metadata.name);
- baseCfs.indexManager.markIndexBuilt(metadata.name);
return;
}
@@ -729,11 +708,11 @@
SecondaryIndexBuilder builder = new CollatedViewIndexBuilder(baseCfs,
Collections.singleton(this),
- new ReducingKeyIterator(sstables));
+ new ReducingKeyIterator(sstables),
+ ImmutableSet.copyOf(sstables));
Future<?> future = CompactionManager.instance.submitIndexBuild(builder);
FBUtilities.waitOnFuture(future);
indexCfs.forceBlockingFlush();
- baseCfs.indexManager.markIndexBuilt(metadata.name);
}
logger.info("Index build of {} complete", metadata.name);
}
@@ -746,31 +725,28 @@
}
/**
- * Construct the CFMetadata for an index table, the clustering columns in the index table
+ * Construct the TableMetadata for an index table, the clustering columns in the index table
* vary dependent on the kind of the indexed value.
* @param baseCfsMetadata
* @param indexMetadata
* @return
*/
- public static final CFMetaData indexCfsMetadata(CFMetaData baseCfsMetadata, IndexMetadata indexMetadata)
+ public static TableMetadata indexCfsMetadata(TableMetadata baseCfsMetadata, IndexMetadata indexMetadata)
{
- Pair<ColumnDefinition, IndexTarget.Type> target = TargetParser.parse(baseCfsMetadata, indexMetadata);
+ Pair<ColumnMetadata, IndexTarget.Type> target = TargetParser.parse(baseCfsMetadata, indexMetadata);
CassandraIndexFunctions utils = getFunctions(indexMetadata, target);
- ColumnDefinition indexedColumn = target.left;
+ ColumnMetadata indexedColumn = target.left;
AbstractType<?> indexedValueType = utils.getIndexedValueType(indexedColumn);
- // Tables for legacy KEYS indexes are non-compound and dense
- CFMetaData.Builder builder = indexMetadata.isKeys()
- ? CFMetaData.Builder.create(baseCfsMetadata.ksName,
- baseCfsMetadata.indexColumnFamilyName(indexMetadata),
- true, false, false)
- : CFMetaData.Builder.create(baseCfsMetadata.ksName,
- baseCfsMetadata.indexColumnFamilyName(indexMetadata));
-
- builder = builder.withId(baseCfsMetadata.cfId)
- .withPartitioner(new LocalPartitioner(indexedValueType))
- .addPartitionKey(indexedColumn.name, indexedColumn.type)
- .addClusteringColumn("partition_key", baseCfsMetadata.partitioner.partitionOrdering());
+ TableMetadata.Builder builder =
+ TableMetadata.builder(baseCfsMetadata.keyspace, baseCfsMetadata.indexTableName(indexMetadata), baseCfsMetadata.id)
+ .kind(TableMetadata.Kind.INDEX)
+ // tables for legacy KEYS indexes are non-compound and dense
+ .isDense(indexMetadata.isKeys())
+ .isCompound(!indexMetadata.isKeys())
+ .partitioner(new LocalPartitioner(indexedValueType))
+ .addPartitionKeyColumn(indexedColumn.name, indexedColumn.type)
+ .addClusteringColumn("partition_key", baseCfsMetadata.partitioner.partitionOrdering());
if (indexMetadata.isKeys())
{
@@ -778,16 +754,16 @@
// value column defined, even though it is never used
CompactTables.DefaultNames names =
CompactTables.defaultNameGenerator(ImmutableSet.of(indexedColumn.name.toString(), "partition_key"));
- builder = builder.addRegularColumn(names.defaultCompactValueName(), EmptyType.instance);
+ builder.addRegularColumn(names.defaultCompactValueName(), EmptyType.instance);
}
else
{
// The clustering columns for a table backing a COMPOSITES index are dependent
// on the specific type of index (there are specializations for indexes on collections)
- builder = utils.addIndexClusteringColumns(builder, baseCfsMetadata, indexedColumn);
+ utils.addIndexClusteringColumns(builder, baseCfsMetadata, indexedColumn);
}
- return builder.build().reloadIndexMetadataProperties(baseCfsMetadata);
+ return builder.build().updateIndexTableMetadata(baseCfsMetadata.params);
}
/**
@@ -798,16 +774,16 @@
*/
public static CassandraIndex newIndex(ColumnFamilyStore baseCfs, IndexMetadata indexMetadata)
{
- return getFunctions(indexMetadata, TargetParser.parse(baseCfs.metadata, indexMetadata)).newIndexInstance(baseCfs, indexMetadata);
+ return getFunctions(indexMetadata, TargetParser.parse(baseCfs.metadata(), indexMetadata)).newIndexInstance(baseCfs, indexMetadata);
}
static CassandraIndexFunctions getFunctions(IndexMetadata indexDef,
- Pair<ColumnDefinition, IndexTarget.Type> target)
+ Pair<ColumnMetadata, IndexTarget.Type> target)
{
if (indexDef.isKeys())
return CassandraIndexFunctions.KEYS_INDEX_FUNCTIONS;
- ColumnDefinition indexedColumn = target.left;
+ ColumnMetadata indexedColumn = target.left;
if (indexedColumn.type.isCollection() && indexedColumn.type.isMultiCell())
{
switch (((CollectionType)indexedColumn.type).kind)
diff --git a/src/java/org/apache/cassandra/index/internal/CassandraIndexFunctions.java b/src/java/org/apache/cassandra/index/internal/CassandraIndexFunctions.java
index 89eebdf..3d500a1 100644
--- a/src/java/org/apache/cassandra/index/internal/CassandraIndexFunctions.java
+++ b/src/java/org/apache/cassandra/index/internal/CassandraIndexFunctions.java
@@ -20,8 +20,8 @@
import java.util.List;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.db.marshal.CollectionType;
@@ -46,20 +46,20 @@
* @param indexedColumn
* @return
*/
- default AbstractType<?> getIndexedValueType(ColumnDefinition indexedColumn)
+ default AbstractType<?> getIndexedValueType(ColumnMetadata indexedColumn)
{
return indexedColumn.type;
}
/**
- * Add the clustering columns for a specific type of index table to the a CFMetaData.Builder (which is being
- * used to construct the index table's CFMetadata. In the default implementation, the clustering columns of the
+ * Add the clustering columns for a specific type of index table to the a TableMetadata.Builder (which is being
+ * used to construct the index table's TableMetadata. In the default implementation, the clustering columns of the
* index table hold the partition key and clustering columns of the base table. This is overridden in several cases:
* * When the indexed value is itself a clustering column, in which case, we only need store the base table's
* *other* clustering values in the index - the indexed value being the index table's partition key
* * When the indexed value is a collection value, in which case we also need to capture the cell path from the base
* table
- * * In a KEYS index (for thrift/compact storage/static column indexes), where only the base partition key is
+ * * In a KEYS index (for compact storage/static column indexes), where only the base partition key is
* held in the index table.
*
* Called from indexCfsMetadata
@@ -68,11 +68,11 @@
* @param cfDef
* @return
*/
- default CFMetaData.Builder addIndexClusteringColumns(CFMetaData.Builder builder,
- CFMetaData baseMetadata,
- ColumnDefinition cfDef)
+ default TableMetadata.Builder addIndexClusteringColumns(TableMetadata.Builder builder,
+ TableMetadata baseMetadata,
+ ColumnMetadata cfDef)
{
- for (ColumnDefinition def : baseMetadata.clusteringColumns())
+ for (ColumnMetadata def : baseMetadata.clusteringColumns())
builder.addClusteringColumn(def.name, def.type);
return builder;
}
@@ -104,21 +104,22 @@
return new ClusteringColumnIndex(baseCfs, indexMetadata);
}
- public CFMetaData.Builder addIndexClusteringColumns(CFMetaData.Builder builder,
- CFMetaData baseMetadata,
- ColumnDefinition columnDef)
+ public TableMetadata.Builder addIndexClusteringColumns(TableMetadata.Builder builder,
+ TableMetadata baseMetadata,
+ ColumnMetadata columnDef)
{
- List<ColumnDefinition> cks = baseMetadata.clusteringColumns();
+ List<ColumnMetadata> cks = baseMetadata.clusteringColumns();
for (int i = 0; i < columnDef.position(); i++)
{
- ColumnDefinition def = cks.get(i);
+ ColumnMetadata def = cks.get(i);
builder.addClusteringColumn(def.name, def.type);
}
for (int i = columnDef.position() + 1; i < cks.size(); i++)
{
- ColumnDefinition def = cks.get(i);
+ ColumnMetadata def = cks.get(i);
builder.addClusteringColumn(def.name, def.type);
}
+
return builder;
}
};
@@ -138,7 +139,7 @@
return new CollectionKeyIndex(baseCfs, indexMetadata);
}
- public AbstractType<?> getIndexedValueType(ColumnDefinition indexedColumn)
+ public AbstractType<?> getIndexedValueType(ColumnMetadata indexedColumn)
{
return ((CollectionType) indexedColumn.type).nameComparator();
}
@@ -152,16 +153,16 @@
return new CollectionValueIndex(baseCfs, indexMetadata);
}
- public AbstractType<?> getIndexedValueType(ColumnDefinition indexedColumn)
+ public AbstractType<?> getIndexedValueType(ColumnMetadata indexedColumn)
{
return ((CollectionType)indexedColumn.type).valueComparator();
}
- public CFMetaData.Builder addIndexClusteringColumns(CFMetaData.Builder builder,
- CFMetaData baseMetadata,
- ColumnDefinition columnDef)
+ public TableMetadata.Builder addIndexClusteringColumns(TableMetadata.Builder builder,
+ TableMetadata baseMetadata,
+ ColumnMetadata columnDef)
{
- for (ColumnDefinition def : baseMetadata.clusteringColumns())
+ for (ColumnMetadata def : baseMetadata.clusteringColumns())
builder.addClusteringColumn(def.name, def.type);
// collection key
@@ -177,7 +178,7 @@
return new CollectionEntryIndex(baseCfs, indexMetadata);
}
- public AbstractType<?> getIndexedValueType(ColumnDefinition indexedColumn)
+ public AbstractType<?> getIndexedValueType(ColumnMetadata indexedColumn)
{
CollectionType colType = (CollectionType)indexedColumn.type;
return CompositeType.getInstance(colType.nameComparator(), colType.valueComparator());
diff --git a/src/java/org/apache/cassandra/index/internal/CassandraIndexSearcher.java b/src/java/org/apache/cassandra/index/internal/CassandraIndexSearcher.java
index 7c23345..81c05d6 100644
--- a/src/java/org/apache/cassandra/index/internal/CassandraIndexSearcher.java
+++ b/src/java/org/apache/cassandra/index/internal/CassandraIndexSearcher.java
@@ -26,7 +26,7 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.filter.*;
import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
@@ -76,8 +76,8 @@
{
ClusteringIndexFilter filter = makeIndexFilter(command);
ColumnFamilyStore indexCfs = index.getBackingTable().get();
- CFMetaData indexCfm = indexCfs.metadata;
- return SinglePartitionReadCommand.create(indexCfm, command.nowInSec(), indexKey, ColumnFilter.all(indexCfm), filter)
+ TableMetadata indexMetadata = indexCfs.metadata();
+ return SinglePartitionReadCommand.create(indexMetadata, command.nowInSec(), indexKey, ColumnFilter.all(indexMetadata), filter)
.queryMemtableAndDisk(indexCfs, executionController.indexReadController());
}
diff --git a/src/java/org/apache/cassandra/index/internal/CollatedViewIndexBuilder.java b/src/java/org/apache/cassandra/index/internal/CollatedViewIndexBuilder.java
index 811d857..3c005c4 100644
--- a/src/java/org/apache/cassandra/index/internal/CollatedViewIndexBuilder.java
+++ b/src/java/org/apache/cassandra/index/internal/CollatedViewIndexBuilder.java
@@ -17,18 +17,19 @@
*/
package org.apache.cassandra.index.internal;
+import java.util.Collection;
import java.util.Set;
import java.util.UUID;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.DecoratedKey;
-import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.db.compaction.CompactionInfo;
import org.apache.cassandra.db.compaction.CompactionInterruptedException;
import org.apache.cassandra.db.compaction.OperationType;
import org.apache.cassandra.index.Index;
import org.apache.cassandra.index.SecondaryIndexBuilder;
import org.apache.cassandra.io.sstable.ReducingKeyIterator;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.utils.UUIDGen;
/**
@@ -40,22 +41,25 @@
private final Set<Index> indexers;
private final ReducingKeyIterator iter;
private final UUID compactionId;
+ private final Collection<SSTableReader> sstables;
- public CollatedViewIndexBuilder(ColumnFamilyStore cfs, Set<Index> indexers, ReducingKeyIterator iter)
+ public CollatedViewIndexBuilder(ColumnFamilyStore cfs, Set<Index> indexers, ReducingKeyIterator iter, Collection<SSTableReader> sstables)
{
this.cfs = cfs;
this.indexers = indexers;
this.iter = iter;
this.compactionId = UUIDGen.getTimeUUID();
+ this.sstables = sstables;
}
public CompactionInfo getCompactionInfo()
{
- return new CompactionInfo(cfs.metadata,
+ return new CompactionInfo(cfs.metadata(),
OperationType.INDEX_BUILD,
iter.getBytesRead(),
iter.getTotalBytes(),
- compactionId);
+ compactionId,
+ sstables);
}
public void build()
diff --git a/src/java/org/apache/cassandra/index/internal/IndexEntry.java b/src/java/org/apache/cassandra/index/internal/IndexEntry.java
index 97525d6..3e4e41b 100644
--- a/src/java/org/apache/cassandra/index/internal/IndexEntry.java
+++ b/src/java/org/apache/cassandra/index/internal/IndexEntry.java
@@ -28,7 +28,7 @@
/**
* Entries in indexes on non-compact tables (tables with composite comparators)
* can be encapsulated as IndexedEntry instances. These are not used when dealing
- * with indexes on static/compact/thrift tables (i.e. KEYS indexes).
+ * with indexes on static/compact tables (i.e. KEYS indexes).
*/
public final class IndexEntry
{
diff --git a/src/java/org/apache/cassandra/index/internal/composites/ClusteringColumnIndex.java b/src/java/org/apache/cassandra/index/internal/composites/ClusteringColumnIndex.java
index d1917f9..17ab814 100644
--- a/src/java/org/apache/cassandra/index/internal/composites/ClusteringColumnIndex.java
+++ b/src/java/org/apache/cassandra/index/internal/composites/ClusteringColumnIndex.java
@@ -52,7 +52,7 @@
public ClusteringColumnIndex(ColumnFamilyStore baseCfs, IndexMetadata indexDef)
{
super(baseCfs, indexDef);
- this.enforceStrictLiveness = baseCfs.metadata.enforceStrictLiveness();
+ this.enforceStrictLiveness = baseCfs.metadata.get().enforceStrictLiveness();
}
@@ -79,7 +79,7 @@
public IndexEntry decodeEntry(DecoratedKey indexedValue,
Row indexEntry)
{
- int ckCount = baseCfs.metadata.clusteringColumns().size();
+ int ckCount = baseCfs.metadata().clusteringColumns().size();
Clustering clustering = indexEntry.clustering();
CBuilder builder = CBuilder.create(baseCfs.getComparator());
diff --git a/src/java/org/apache/cassandra/index/internal/composites/CollectionEntryIndex.java b/src/java/org/apache/cassandra/index/internal/composites/CollectionEntryIndex.java
index 1113600..efe84b6 100644
--- a/src/java/org/apache/cassandra/index/internal/composites/CollectionEntryIndex.java
+++ b/src/java/org/apache/cassandra/index/internal/composites/CollectionEntryIndex.java
@@ -19,7 +19,7 @@
import java.nio.ByteBuffer;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.db.Clustering;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.marshal.AbstractType;
@@ -57,7 +57,7 @@
ByteBuffer mapKey = components[0];
ByteBuffer mapValue = components[1];
- ColumnDefinition columnDef = indexedColumn;
+ ColumnMetadata columnDef = indexedColumn;
Cell cell = data.getCell(columnDef, CellPath.create(mapKey));
if (cell == null || !cell.isLive(nowInSec))
return true;
diff --git a/src/java/org/apache/cassandra/index/internal/composites/CollectionKeyIndex.java b/src/java/org/apache/cassandra/index/internal/composites/CollectionKeyIndex.java
index 42c45e5..4fc20ae 100644
--- a/src/java/org/apache/cassandra/index/internal/composites/CollectionKeyIndex.java
+++ b/src/java/org/apache/cassandra/index/internal/composites/CollectionKeyIndex.java
@@ -19,7 +19,7 @@
import java.nio.ByteBuffer;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.cql3.Operator;
import org.apache.cassandra.db.Clustering;
import org.apache.cassandra.db.ColumnFamilyStore;
@@ -56,7 +56,7 @@
return cell == null || !cell.isLive(nowInSec);
}
- public boolean supportsOperator(ColumnDefinition indexedColumn, Operator operator)
+ public boolean supportsOperator(ColumnMetadata indexedColumn, Operator operator)
{
return operator == Operator.CONTAINS_KEY ||
operator == Operator.CONTAINS && indexedColumn.type instanceof SetType;
diff --git a/src/java/org/apache/cassandra/index/internal/composites/CollectionKeyIndexBase.java b/src/java/org/apache/cassandra/index/internal/composites/CollectionKeyIndexBase.java
index ef76870..fccf522 100644
--- a/src/java/org/apache/cassandra/index/internal/composites/CollectionKeyIndexBase.java
+++ b/src/java/org/apache/cassandra/index/internal/composites/CollectionKeyIndexBase.java
@@ -73,7 +73,7 @@
indexedEntryClustering = Clustering.STATIC_CLUSTERING;
else
{
- int count = 1 + baseCfs.metadata.clusteringColumns().size();
+ int count = 1 + baseCfs.metadata().clusteringColumns().size();
CBuilder builder = CBuilder.create(baseCfs.getComparator());
for (int i = 0; i < count - 1; i++)
builder.add(clustering.get(i + 1));
diff --git a/src/java/org/apache/cassandra/index/internal/composites/CollectionValueIndex.java b/src/java/org/apache/cassandra/index/internal/composites/CollectionValueIndex.java
index 5929e69..4f0f2df 100644
--- a/src/java/org/apache/cassandra/index/internal/composites/CollectionValueIndex.java
+++ b/src/java/org/apache/cassandra/index/internal/composites/CollectionValueIndex.java
@@ -19,7 +19,7 @@
import java.nio.ByteBuffer;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.cql3.Operator;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.marshal.CollectionType;
@@ -67,7 +67,7 @@
// partition key is needed at query time.
// In the non-static case, cell will be present during indexing but
// not when searching (CASSANDRA-7525).
- if (prefix.size() == baseCfs.metadata.clusteringColumns().size() && path != null)
+ if (prefix.size() == baseCfs.metadata().clusteringColumns().size() && path != null)
builder.add(path.get(0));
return builder;
@@ -94,14 +94,14 @@
indexedEntryClustering);
}
- public boolean supportsOperator(ColumnDefinition indexedColumn, Operator operator)
+ public boolean supportsOperator(ColumnMetadata indexedColumn, Operator operator)
{
return operator == Operator.CONTAINS && !(indexedColumn.type instanceof SetType);
}
public boolean isStale(Row data, ByteBuffer indexValue, int nowInSec)
{
- ColumnDefinition columnDef = indexedColumn;
+ ColumnMetadata columnDef = indexedColumn;
ComplexColumnData complexData = data.getComplexColumnData(columnDef);
if (complexData == null)
return true;
diff --git a/src/java/org/apache/cassandra/index/internal/composites/CompositesSearcher.java b/src/java/org/apache/cassandra/index/internal/composites/CompositesSearcher.java
index 4429027..ba747ec 100644
--- a/src/java/org/apache/cassandra/index/internal/composites/CompositesSearcher.java
+++ b/src/java/org/apache/cassandra/index/internal/composites/CompositesSearcher.java
@@ -21,7 +21,7 @@
import java.util.ArrayList;
import java.util.List;
-import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.filter.ClusteringIndexNamesFilter;
import org.apache.cassandra.db.filter.ClusteringIndexSliceFilter;
@@ -69,12 +69,7 @@
private UnfilteredRowIterator next;
- public boolean isForThrift()
- {
- return command.isForThrift();
- }
-
- public CFMetaData metadata()
+ public TableMetadata metadata()
{
return command.metadata();
}
@@ -122,7 +117,7 @@
// If the index is on a static column, we just need to do a full read on the partition.
// Note that we want to re-use the command.columnFilter() in case of future change.
- dataCmd = SinglePartitionReadCommand.create(index.baseCfs.metadata,
+ dataCmd = SinglePartitionReadCommand.create(index.baseCfs.metadata(),
command.nowInSec(),
command.columnFilter(),
RowFilter.NONE,
@@ -159,8 +154,7 @@
// Query the gathered index hits. We still need to filter stale hits from the resulting query.
ClusteringIndexNamesFilter filter = new ClusteringIndexNamesFilter(clusterings.build(), false);
- dataCmd = SinglePartitionReadCommand.create(isForThrift(),
- index.baseCfs.metadata,
+ dataCmd = SinglePartitionReadCommand.create(index.baseCfs.metadata(),
command.nowInSec(),
command.columnFilter(),
command.rowFilter(),
@@ -176,7 +170,7 @@
filterStaleEntries(dataCmd.queryMemtableAndDisk(index.baseCfs, executionController),
indexKey.getKey(),
entries,
- executionController.writeOpOrderGroup(),
+ executionController.getWriteContext(),
command.nowInSec());
if (dataIter.isEmpty())
@@ -204,20 +198,21 @@
};
}
- private void deleteAllEntries(final List<IndexEntry> entries, final OpOrder.Group writeOp, final int nowInSec)
+ private void deleteAllEntries(final List<IndexEntry> entries, final WriteContext ctx, final int nowInSec)
{
entries.forEach(entry ->
index.deleteStaleEntry(entry.indexValue,
entry.indexClustering,
new DeletionTime(entry.timestamp, nowInSec),
- writeOp));
+ ctx));
}
// We assume all rows in dataIter belong to the same partition.
+ @SuppressWarnings("resource")
private UnfilteredRowIterator filterStaleEntries(UnfilteredRowIterator dataIter,
final ByteBuffer indexValue,
final List<IndexEntry> entries,
- final OpOrder.Group writeOp,
+ final WriteContext ctx,
final int nowInSec)
{
// collect stale index entries and delete them when we close this iterator
@@ -251,7 +246,7 @@
dataIter.partitionLevelDeletion(),
dataIter.isReverseOrder());
}
- deleteAllEntries(staleEntries, writeOp, nowInSec);
+ deleteAllEntries(staleEntries, ctx, nowInSec);
}
else
{
@@ -311,7 +306,7 @@
@Override
public void onPartitionClose()
{
- deleteAllEntries(staleEntries, writeOp, nowInSec);
+ deleteAllEntries(staleEntries, ctx, nowInSec);
}
}
iteratorToReturn = Transformation.apply(dataIter, new Transform());
diff --git a/src/java/org/apache/cassandra/index/internal/composites/PartitionKeyIndex.java b/src/java/org/apache/cassandra/index/internal/composites/PartitionKeyIndex.java
index d854102..9b03213 100644
--- a/src/java/org/apache/cassandra/index/internal/composites/PartitionKeyIndex.java
+++ b/src/java/org/apache/cassandra/index/internal/composites/PartitionKeyIndex.java
@@ -51,7 +51,7 @@
public PartitionKeyIndex(ColumnFamilyStore baseCfs, IndexMetadata indexDef)
{
super(baseCfs, indexDef);
- this.enforceStrictLiveness = baseCfs.metadata.enforceStrictLiveness();
+ this.enforceStrictLiveness = baseCfs.metadata.get().enforceStrictLiveness();
}
public ByteBuffer getIndexedValue(ByteBuffer partitionKey,
@@ -59,7 +59,7 @@
CellPath path,
ByteBuffer cellValue)
{
- CompositeType keyComparator = (CompositeType)baseCfs.metadata.getKeyValidator();
+ CompositeType keyComparator = (CompositeType)baseCfs.metadata().partitionKeyType;
ByteBuffer[] components = keyComparator.split(partitionKey);
return components[indexedColumn.position()];
}
@@ -77,7 +77,7 @@
public IndexEntry decodeEntry(DecoratedKey indexedValue, Row indexEntry)
{
- int ckCount = baseCfs.metadata.clusteringColumns().size();
+ int ckCount = baseCfs.metadata().clusteringColumns().size();
Clustering clustering = indexEntry.clustering();
CBuilder builder = CBuilder.create(baseCfs.getComparator());
for (int i = 0; i < ckCount; i++)
diff --git a/src/java/org/apache/cassandra/index/internal/keys/KeysIndex.java b/src/java/org/apache/cassandra/index/internal/keys/KeysIndex.java
index d680253..20a1915 100644
--- a/src/java/org/apache/cassandra/index/internal/keys/KeysIndex.java
+++ b/src/java/org/apache/cassandra/index/internal/keys/KeysIndex.java
@@ -22,8 +22,9 @@
import java.nio.ByteBuffer;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.schema.TableMetadataRef;
+import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.rows.Cell;
import org.apache.cassandra.db.rows.CellPath;
@@ -39,17 +40,17 @@
super(baseCfs, indexDef);
}
- public CFMetaData.Builder addIndexClusteringColumns(CFMetaData.Builder builder,
- CFMetaData baseMetadata,
- ColumnDefinition cfDef)
+ public TableMetadata.Builder addIndexClusteringColumns(TableMetadata.Builder builder,
+ TableMetadataRef baseMetadata,
+ ColumnMetadata cfDef)
{
// no additional clustering columns required
return builder;
}
protected CBuilder buildIndexClusteringPrefix(ByteBuffer partitionKey,
- ClusteringPrefix prefix,
- CellPath path)
+ ClusteringPrefix prefix,
+ CellPath path)
{
CBuilder builder = CBuilder.create(getIndexComparator());
builder.add(partitionKey);
diff --git a/src/java/org/apache/cassandra/index/internal/keys/KeysSearcher.java b/src/java/org/apache/cassandra/index/internal/keys/KeysSearcher.java
index fa05420..8b3a3d2 100644
--- a/src/java/org/apache/cassandra/index/internal/keys/KeysSearcher.java
+++ b/src/java/org/apache/cassandra/index/internal/keys/KeysSearcher.java
@@ -22,18 +22,15 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.filter.ColumnFilter;
import org.apache.cassandra.db.filter.DataLimits;
import org.apache.cassandra.db.filter.RowFilter;
-import org.apache.cassandra.db.partitions.ImmutableBTreePartition;
import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
import org.apache.cassandra.db.rows.*;
import org.apache.cassandra.index.internal.CassandraIndex;
import org.apache.cassandra.index.internal.CassandraIndexSearcher;
-import org.apache.cassandra.utils.concurrent.OpOrder;
+import org.apache.cassandra.schema.TableMetadata;
public class KeysSearcher extends CassandraIndexSearcher
{
@@ -57,12 +54,7 @@
{
private UnfilteredRowIterator next;
- public boolean isForThrift()
- {
- return command.isForThrift();
- }
-
- public CFMetaData metadata()
+ public TableMetadata metadata()
{
return command.metadata();
}
@@ -92,8 +84,7 @@
continue;
ColumnFilter extendedFilter = getExtendedFilter(command.columnFilter());
- SinglePartitionReadCommand dataCmd = SinglePartitionReadCommand.create(isForThrift(),
- index.baseCfs.metadata,
+ SinglePartitionReadCommand dataCmd = SinglePartitionReadCommand.create(index.baseCfs.metadata(),
command.nowInSec(),
extendedFilter,
command.rowFilter(),
@@ -108,8 +99,7 @@
UnfilteredRowIterator dataIter = filterIfStale(dataCmd.queryMemtableAndDisk(index.baseCfs, executionController),
hit,
indexKey.getKey(),
- executionController.writeOpOrderGroup(),
- isForThrift(),
+ executionController.getWriteContext(),
command.nowInSec());
if (dataIter != null)
@@ -151,73 +141,23 @@
private UnfilteredRowIterator filterIfStale(UnfilteredRowIterator iterator,
Row indexHit,
ByteBuffer indexedValue,
- OpOrder.Group writeOp,
- boolean isForThrift,
+ WriteContext ctx,
int nowInSec)
{
- if (isForThrift)
+ Row data = iterator.staticRow();
+ if (index.isStale(data, indexedValue, nowInSec))
{
- // The data we got has gone though ThrifResultsMerger, so we're looking for the row whose clustering
- // is the indexed name and so we need to materialize the partition.
- ImmutableBTreePartition result = ImmutableBTreePartition.create(iterator);
+ // Index is stale, remove the index entry and ignore
+ index.deleteStaleEntry(index.getIndexCfs().decorateKey(indexedValue),
+ makeIndexClustering(iterator.partitionKey().getKey(), Clustering.EMPTY),
+ new DeletionTime(indexHit.primaryKeyLivenessInfo().timestamp(), nowInSec),
+ ctx);
iterator.close();
- Row data = result.getRow(Clustering.make(index.getIndexedColumn().name.bytes));
- if (data == null)
- return null;
-
- // for thrift tables, we need to compare the index entry against the compact value column,
- // not the column actually designated as the indexed column so we don't use the index function
- // lib for the staleness check like we do in every other case
- Cell baseData = data.getCell(index.baseCfs.metadata.compactValueColumn());
- if (baseData == null || !baseData.isLive(nowInSec) || index.getIndexedColumn().type.compare(indexedValue, baseData.value()) != 0)
- {
- // Index is stale, remove the index entry and ignore
- index.deleteStaleEntry(index.getIndexCfs().decorateKey(indexedValue),
- Clustering.make(index.getIndexedColumn().name.bytes),
- new DeletionTime(indexHit.primaryKeyLivenessInfo().timestamp(), nowInSec),
- writeOp);
- return null;
- }
- else
- {
- if (command.columnFilter().fetches(index.getIndexedColumn()))
- return result.unfilteredIterator();
-
- // The query on the base table used an extended column filter to ensure that the
- // indexed column was actually read for use in the staleness check, before
- // returning the results we must filter the base table partition so that it
- // contains only the originally requested columns. See CASSANDRA-11523
- ClusteringComparator comparator = result.metadata().comparator;
- Slices.Builder slices = new Slices.Builder(comparator);
- for (ColumnDefinition selected : command.columnFilter().fetchedColumns())
- slices.add(Slice.make(comparator, selected.name.bytes));
- return result.unfilteredIterator(ColumnFilter.all(command.metadata()), slices.build(), false);
- }
+ return null;
}
else
{
- if (!iterator.metadata().isCompactTable())
- {
- logger.warn("Non-composite index was used on the table '{}' during the query. Starting from Cassandra 4.0, only " +
- "composite indexes will be supported. If compact flags were dropped for this table, drop and re-create " +
- "the index.", iterator.metadata().cfName);
- }
-
- Row data = iterator.staticRow();
- if (index.isStale(data, indexedValue, nowInSec))
- {
- // Index is stale, remove the index entry and ignore
- index.deleteStaleEntry(index.getIndexCfs().decorateKey(indexedValue),
- makeIndexClustering(iterator.partitionKey().getKey(), Clustering.EMPTY),
- new DeletionTime(indexHit.primaryKeyLivenessInfo().timestamp(), nowInSec),
- writeOp);
- iterator.close();
- return null;
- }
- else
- {
- return iterator;
- }
+ return iterator;
}
}
}
diff --git a/src/java/org/apache/cassandra/index/sasi/SASIIndex.java b/src/java/org/apache/cassandra/index/sasi/SASIIndex.java
index 4bf94ef..592499e 100644
--- a/src/java/org/apache/cassandra/index/sasi/SASIIndex.java
+++ b/src/java/org/apache/cassandra/index/sasi/SASIIndex.java
@@ -25,7 +25,7 @@
import org.apache.cassandra.config.*;
import org.apache.cassandra.cql3.Operator;
-import org.apache.cassandra.cql3.statements.IndexTarget;
+import org.apache.cassandra.cql3.statements.schema.IndexTarget;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.compaction.CompactionManager;
import org.apache.cassandra.db.compaction.OperationType;
@@ -52,11 +52,16 @@
import org.apache.cassandra.io.sstable.format.SSTableFlushObserver;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.notifications.*;
+import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.schema.IndexMetadata;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.Pair;
import org.apache.cassandra.utils.concurrent.OpOrder;
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+
public class SASIIndex implements Index, INotificationConsumer
{
public final static String USAGE_WARNING = "SASI indexes are experimental and are not recommended for production use.";
@@ -67,7 +72,7 @@
Set<Index> indexes,
Collection<SSTableReader> sstablesToRebuild)
{
- NavigableMap<SSTableReader, Map<ColumnDefinition, ColumnIndex>> sstables = new TreeMap<>((a, b) -> {
+ NavigableMap<SSTableReader, Map<ColumnMetadata, ColumnIndex>> sstables = new TreeMap<>((a, b) -> {
return Integer.compare(a.descriptor.generation, b.descriptor.generation);
});
@@ -79,7 +84,7 @@
sstablesToRebuild.stream()
.filter((sstable) -> !sasi.index.hasSSTable(sstable))
.forEach((sstable) -> {
- Map<ColumnDefinition, ColumnIndex> toBuild = sstables.get(sstable);
+ Map<ColumnMetadata, ColumnIndex> toBuild = sstables.get(sstable);
if (toBuild == null)
sstables.put(sstable, (toBuild = new HashMap<>()));
@@ -102,18 +107,18 @@
this.baseCfs = baseCfs;
this.config = config;
- ColumnDefinition column = TargetParser.parse(baseCfs.metadata, config).left;
- this.index = new ColumnIndex(baseCfs.metadata.getKeyValidator(), column, config);
+ ColumnMetadata column = TargetParser.parse(baseCfs.metadata(), config).left;
+ this.index = new ColumnIndex(baseCfs.metadata().partitionKeyType, column, config);
Tracker tracker = baseCfs.getTracker();
tracker.subscribe(this);
- SortedMap<SSTableReader, Map<ColumnDefinition, ColumnIndex>> toRebuild = new TreeMap<>((a, b)
+ SortedMap<SSTableReader, Map<ColumnMetadata, ColumnIndex>> toRebuild = new TreeMap<>((a, b)
-> Integer.compare(a.descriptor.generation, b.descriptor.generation));
for (SSTableReader sstable : index.init(tracker.getView().liveSSTables()))
{
- Map<ColumnDefinition, ColumnIndex> perSSTable = toRebuild.get(sstable);
+ Map<ColumnMetadata, ColumnIndex> perSSTable = toRebuild.get(sstable);
if (perSSTable == null)
toRebuild.put(sstable, (perSSTable = new HashMap<>()));
@@ -126,16 +131,16 @@
/**
* Called via reflection at {@link IndexMetadata#validateCustomIndexOptions}
*/
- public static Map<String, String> validateOptions(Map<String, String> options, CFMetaData cfm)
+ public static Map<String, String> validateOptions(Map<String, String> options, TableMetadata metadata)
{
- if (!(cfm.partitioner instanceof Murmur3Partitioner))
+ if (!(metadata.partitioner instanceof Murmur3Partitioner))
throw new ConfigurationException("SASI only supports Murmur3Partitioner.");
String targetColumn = options.get("target");
if (targetColumn == null)
throw new ConfigurationException("unknown target column");
- Pair<ColumnDefinition, IndexTarget.Type> target = TargetParser.parse(cfm, targetColumn);
+ Pair<ColumnMetadata, IndexTarget.Type> target = TargetParser.parse(metadata, targetColumn);
if (target == null)
throw new ConfigurationException("failed to retrieve target column for: " + targetColumn);
@@ -208,17 +213,17 @@
return Optional.empty();
}
- public boolean indexes(PartitionColumns columns)
+ public boolean indexes(RegularAndStaticColumns columns)
{
return columns.contains(index.getDefinition());
}
- public boolean dependsOn(ColumnDefinition column)
+ public boolean dependsOn(ColumnMetadata column)
{
return index.getDefinition().compareTo(column) == 0;
}
- public boolean supportsExpression(ColumnDefinition column, Operator operator)
+ public boolean supportsExpression(ColumnMetadata column, Operator operator)
{
return dependsOn(column) && index.supports(operator);
}
@@ -244,7 +249,7 @@
public void validate(PartitionUpdate update) throws InvalidRequestException
{}
- public Indexer indexerFor(DecoratedKey key, PartitionColumns columns, int nowInSec, OpOrder.Group opGroup, IndexTransaction.Type transactionType)
+ public Indexer indexerFor(DecoratedKey key, RegularAndStaticColumns columns, int nowInSec, WriteContext context, IndexTransaction.Type transactionType)
{
return new Indexer()
{
@@ -260,7 +265,7 @@
public void insertRow(Row row)
{
if (isNewData())
- adjustMemtableSize(index.index(key, row), opGroup);
+ adjustMemtableSize(index.index(key, row), CassandraWriteContext.fromContext(context).getGroup());
}
public void updateRow(Row oldRow, Row newRow)
@@ -290,14 +295,14 @@
public Searcher searcherFor(ReadCommand command) throws InvalidRequestException
{
- CFMetaData config = command.metadata();
- ColumnFamilyStore cfs = Schema.instance.getColumnFamilyStoreInstance(config.cfId);
- return controller -> new QueryPlan(cfs, command, DatabaseDescriptor.getRangeRpcTimeout()).execute(controller);
+ TableMetadata config = command.metadata();
+ ColumnFamilyStore cfs = Schema.instance.getColumnFamilyStoreInstance(config.id);
+ return controller -> new QueryPlan(cfs, command, DatabaseDescriptor.getRangeRpcTimeout(MILLISECONDS)).execute(controller);
}
public SSTableFlushObserver getFlushObserver(Descriptor descriptor, OperationType opType)
{
- return newWriter(baseCfs.metadata.getKeyValidator(), descriptor, Collections.singletonMap(index.getDefinition(), index), opType);
+ return newWriter(baseCfs.metadata().partitionKeyType, descriptor, Collections.singletonMap(index.getDefinition(), index), opType);
}
public BiFunction<PartitionIterator, ReadCommand, PartitionIterator> postProcessorFor(ReadCommand command)
@@ -344,7 +349,7 @@
protected static PerSSTableIndexWriter newWriter(AbstractType<?> keyValidator,
Descriptor descriptor,
- Map<ColumnDefinition, ColumnIndex> indexes,
+ Map<ColumnMetadata, ColumnIndex> indexes,
OperationType opType)
{
return new PerSSTableIndexWriter(keyValidator, descriptor, opType, indexes);
diff --git a/src/java/org/apache/cassandra/index/sasi/SASIIndexBuilder.java b/src/java/org/apache/cassandra/index/sasi/SASIIndexBuilder.java
index d50875a..bb42dc2 100644
--- a/src/java/org/apache/cassandra/index/sasi/SASIIndexBuilder.java
+++ b/src/java/org/apache/cassandra/index/sasi/SASIIndexBuilder.java
@@ -24,7 +24,7 @@
import java.io.IOException;
import java.util.*;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.RowIndexEntry;
@@ -49,12 +49,12 @@
private final ColumnFamilyStore cfs;
private final UUID compactionId = UUIDGen.getTimeUUID();
- private final SortedMap<SSTableReader, Map<ColumnDefinition, ColumnIndex>> sstables;
+ private final SortedMap<SSTableReader, Map<ColumnMetadata, ColumnIndex>> sstables;
private long bytesProcessed = 0;
private final long totalSizeInBytes;
- public SASIIndexBuilder(ColumnFamilyStore cfs, SortedMap<SSTableReader, Map<ColumnDefinition, ColumnIndex>> sstables)
+ public SASIIndexBuilder(ColumnFamilyStore cfs, SortedMap<SSTableReader, Map<ColumnMetadata, ColumnIndex>> sstables)
{
long totalIndexBytes = 0;
for (SSTableReader sstable : sstables.keySet())
@@ -67,18 +67,18 @@
public void build()
{
- AbstractType<?> keyValidator = cfs.metadata.getKeyValidator();
- for (Map.Entry<SSTableReader, Map<ColumnDefinition, ColumnIndex>> e : sstables.entrySet())
+ AbstractType<?> keyValidator = cfs.metadata().partitionKeyType;
+ for (Map.Entry<SSTableReader, Map<ColumnMetadata, ColumnIndex>> e : sstables.entrySet())
{
SSTableReader sstable = e.getKey();
- Map<ColumnDefinition, ColumnIndex> indexes = e.getValue();
+ Map<ColumnMetadata, ColumnIndex> indexes = e.getValue();
try (RandomAccessReader dataFile = sstable.openDataReader())
{
PerSSTableIndexWriter indexWriter = SASIIndex.newWriter(keyValidator, sstable.descriptor, indexes, OperationType.COMPACTION);
long previousKeyPosition = 0;
- try (KeyIterator keys = new KeyIterator(sstable.descriptor, cfs.metadata))
+ try (KeyIterator keys = new KeyIterator(sstable.descriptor, cfs.metadata()))
{
while (keys.hasNext())
{
@@ -99,7 +99,7 @@
try (SSTableIdentityIterator partition = SSTableIdentityIterator.create(sstable, dataFile, key))
{
// if the row has statics attached, it has to be indexed separately
- if (cfs.metadata.hasStaticColumns())
+ if (cfs.metadata().hasStaticColumns())
indexWriter.nextUnfilteredCluster(partition.staticRow());
while (partition.hasNext())
@@ -123,11 +123,12 @@
public CompactionInfo getCompactionInfo()
{
- return new CompactionInfo(cfs.metadata,
+ return new CompactionInfo(cfs.metadata(),
OperationType.INDEX_BUILD,
bytesProcessed,
totalSizeInBytes,
- compactionId);
+ compactionId,
+ sstables.keySet());
}
private long getPrimaryIndexLength(SSTable sstable)
diff --git a/src/java/org/apache/cassandra/index/sasi/analyzer/StandardAnalyzer.java b/src/java/org/apache/cassandra/index/sasi/analyzer/StandardAnalyzer.java
index e1a4a44..cd8feda 100644
--- a/src/java/org/apache/cassandra/index/sasi/analyzer/StandardAnalyzer.java
+++ b/src/java/org/apache/cassandra/index/sasi/analyzer/StandardAnalyzer.java
@@ -37,7 +37,7 @@
import com.google.common.annotations.VisibleForTesting;
import com.carrotsearch.hppc.IntObjectMap;
-import com.carrotsearch.hppc.IntObjectOpenHashMap;
+import com.carrotsearch.hppc.IntObjectHashMap;
public class StandardAnalyzer extends AbstractAnalyzer
{
@@ -61,7 +61,7 @@
KATAKANA(12),
HANGUL(13);
- private static final IntObjectMap<TokenType> TOKENS = new IntObjectOpenHashMap<>();
+ private static final IntObjectMap<TokenType> TOKENS = new IntObjectHashMap<>();
static
{
diff --git a/src/java/org/apache/cassandra/index/sasi/analyzer/StandardTokenizerInterface.java b/src/java/org/apache/cassandra/index/sasi/analyzer/StandardTokenizerInterface.java
index 57e35d7..327a674 100644
--- a/src/java/org/apache/cassandra/index/sasi/analyzer/StandardTokenizerInterface.java
+++ b/src/java/org/apache/cassandra/index/sasi/analyzer/StandardTokenizerInterface.java
@@ -46,7 +46,7 @@
* Resumes scanning until the next regular expression is matched,
* the end of input is encountered or an I/O-Error occurs.
*
- * @return the next token, {@link #YYEOF} on end of stream
+ * @return the next token, {@link StandardTokenizerImpl#YYEOF} on end of stream
* @exception java.io.IOException if any I/O-Error occurs
*/
int getNextToken() throws IOException;
diff --git a/src/java/org/apache/cassandra/index/sasi/analyzer/filter/StemmerFactory.java b/src/java/org/apache/cassandra/index/sasi/analyzer/filter/StemmerFactory.java
index ae232db..d278c28 100644
--- a/src/java/org/apache/cassandra/index/sasi/analyzer/filter/StemmerFactory.java
+++ b/src/java/org/apache/cassandra/index/sasi/analyzer/filter/StemmerFactory.java
@@ -22,12 +22,14 @@
import java.util.Locale;
import java.util.Map;
+import com.google.common.util.concurrent.MoreExecutors;
+
import org.tartarus.snowball.SnowballStemmer;
import org.tartarus.snowball.ext.*;
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.CacheLoader;
-import com.google.common.cache.LoadingCache;
+import com.github.benmanes.caffeine.cache.CacheLoader;
+import com.github.benmanes.caffeine.cache.Caffeine;
+import com.github.benmanes.caffeine.cache.LoadingCache;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -39,7 +41,8 @@
public class StemmerFactory
{
private static final Logger logger = LoggerFactory.getLogger(StemmerFactory.class);
- private static final LoadingCache<Class, Constructor<?>> STEMMER_CONSTRUCTOR_CACHE = CacheBuilder.newBuilder()
+ private static final LoadingCache<Class, Constructor<?>> STEMMER_CONSTRUCTOR_CACHE = Caffeine.newBuilder()
+ .executor(MoreExecutors.directExecutor())
.build(new CacheLoader<Class, Constructor<?>>()
{
public Constructor<?> load(Class aClass) throws Exception
diff --git a/src/java/org/apache/cassandra/index/sasi/analyzer/filter/StopWordFactory.java b/src/java/org/apache/cassandra/index/sasi/analyzer/filter/StopWordFactory.java
index 8ec02e0..1548a6a 100644
--- a/src/java/org/apache/cassandra/index/sasi/analyzer/filter/StopWordFactory.java
+++ b/src/java/org/apache/cassandra/index/sasi/analyzer/filter/StopWordFactory.java
@@ -26,11 +26,12 @@
import java.util.HashSet;
import java.util.Locale;
import java.util.Set;
-import java.util.concurrent.ExecutionException;
+import java.util.concurrent.CompletionException;
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.CacheLoader;
-import com.google.common.cache.LoadingCache;
+import com.google.common.util.concurrent.MoreExecutors;
+
+import com.github.benmanes.caffeine.cache.Caffeine;
+import com.github.benmanes.caffeine.cache.LoadingCache;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -49,14 +50,9 @@
Arrays.asList("ar","bg","cs","de","en","es","fi","fr","hi","hu","it",
"pl","pt","ro","ru","sv"));
- private static final LoadingCache<String, Set<String>> STOP_WORDS_CACHE = CacheBuilder.newBuilder()
- .build(new CacheLoader<String, Set<String>>()
- {
- public Set<String> load(String s)
- {
- return getStopWordsFromResource(s);
- }
- });
+ private static final LoadingCache<String, Set<String>> STOP_WORDS_CACHE = Caffeine.newBuilder()
+ .executor(MoreExecutors.directExecutor())
+ .build(StopWordFactory::getStopWordsFromResource);
public static Set<String> getStopWordsForLanguage(Locale locale)
{
@@ -68,7 +64,7 @@
{
return (!SUPPORTED_LANGUAGES.contains(rootLang)) ? null : STOP_WORDS_CACHE.get(rootLang);
}
- catch (ExecutionException e)
+ catch (CompletionException e)
{
logger.error("Failed to populate Stop Words Cache for language [{}]", locale.getLanguage(), e);
return null;
diff --git a/src/java/org/apache/cassandra/index/sasi/conf/ColumnIndex.java b/src/java/org/apache/cassandra/index/sasi/conf/ColumnIndex.java
index 0958113..269fc95 100644
--- a/src/java/org/apache/cassandra/index/sasi/conf/ColumnIndex.java
+++ b/src/java/org/apache/cassandra/index/sasi/conf/ColumnIndex.java
@@ -28,7 +28,7 @@
import com.google.common.annotations.VisibleForTesting;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.cql3.Operator;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.Memtable;
@@ -57,7 +57,7 @@
private final AbstractType<?> keyValidator;
- private final ColumnDefinition column;
+ private final ColumnMetadata column;
private final Optional<IndexMetadata> config;
private final AtomicReference<IndexMemtable> memtable;
@@ -70,7 +70,7 @@
private final boolean isTokenized;
- public ColumnIndex(AbstractType<?> keyValidator, ColumnDefinition column, IndexMetadata metadata)
+ public ColumnIndex(AbstractType<?> keyValidator, ColumnMetadata column, IndexMetadata metadata)
{
this.keyValidator = keyValidator;
this.column = column;
@@ -147,7 +147,7 @@
tracker.update(oldSSTables, newSSTables);
}
- public ColumnDefinition getDefinition()
+ public ColumnMetadata getDefinition()
{
return column;
}
@@ -229,7 +229,7 @@
}
- public static ByteBuffer getValueOf(ColumnDefinition column, Row row, int nowInSecs)
+ public static ByteBuffer getValueOf(ColumnMetadata column, Row row, int nowInSecs)
{
if (row == null)
return null;
diff --git a/src/java/org/apache/cassandra/index/sasi/conf/IndexMode.java b/src/java/org/apache/cassandra/index/sasi/conf/IndexMode.java
index 5709a0f..26b18a1 100644
--- a/src/java/org/apache/cassandra/index/sasi/conf/IndexMode.java
+++ b/src/java/org/apache/cassandra/index/sasi/conf/IndexMode.java
@@ -22,7 +22,7 @@
import java.util.Optional;
import java.util.Set;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.index.sasi.analyzer.AbstractAnalyzer;
import org.apache.cassandra.index.sasi.analyzer.NoOpAnalyzer;
import org.apache.cassandra.index.sasi.analyzer.NonTokenizingAnalyzer;
@@ -93,7 +93,7 @@
return analyzer;
}
- public static void validateAnalyzer(Map<String, String> indexOptions, ColumnDefinition cd) throws ConfigurationException
+ public static void validateAnalyzer(Map<String, String> indexOptions, ColumnMetadata cd) throws ConfigurationException
{
// validate that a valid analyzer class was provided if specified
if (indexOptions.containsKey(INDEX_ANALYZER_CLASS_OPTION))
@@ -126,12 +126,12 @@
}
}
- public static IndexMode getMode(ColumnDefinition column, Optional<IndexMetadata> config) throws ConfigurationException
+ public static IndexMode getMode(ColumnMetadata column, Optional<IndexMetadata> config) throws ConfigurationException
{
return getMode(column, config.isPresent() ? config.get().options : null);
}
- public static IndexMode getMode(ColumnDefinition column, Map<String, String> indexOptions) throws ConfigurationException
+ public static IndexMode getMode(ColumnMetadata column, Map<String, String> indexOptions) throws ConfigurationException
{
if (indexOptions == null || indexOptions.isEmpty())
return IndexMode.NOT_INDEXED;
diff --git a/src/java/org/apache/cassandra/index/sasi/disk/DynamicTokenTreeBuilder.java b/src/java/org/apache/cassandra/index/sasi/disk/DynamicTokenTreeBuilder.java
index 2ddfd89..0e906e2 100644
--- a/src/java/org/apache/cassandra/index/sasi/disk/DynamicTokenTreeBuilder.java
+++ b/src/java/org/apache/cassandra/index/sasi/disk/DynamicTokenTreeBuilder.java
@@ -23,7 +23,7 @@
import org.apache.cassandra.utils.AbstractIterator;
import org.apache.cassandra.utils.Pair;
-import com.carrotsearch.hppc.LongOpenHashSet;
+import com.carrotsearch.hppc.LongHashSet;
import com.carrotsearch.hppc.LongSet;
import com.carrotsearch.hppc.cursors.LongCursor;
@@ -49,7 +49,7 @@
{
LongSet found = tokens.get(token);
if (found == null)
- tokens.put(token, (found = new LongOpenHashSet(2)));
+ tokens.put(token, (found = new LongHashSet(2)));
found.add(keyPosition);
}
@@ -70,7 +70,7 @@
{
LongSet found = tokens.get(newEntry.getKey());
if (found == null)
- tokens.put(newEntry.getKey(), (found = new LongOpenHashSet(4)));
+ tokens.put(newEntry.getKey(), (found = new LongHashSet(4)));
for (LongCursor offset : newEntry.getValue())
found.add(offset.value);
diff --git a/src/java/org/apache/cassandra/index/sasi/disk/PerSSTableIndexWriter.java b/src/java/org/apache/cassandra/index/sasi/disk/PerSSTableIndexWriter.java
index d5ecaf7..b1ce521 100644
--- a/src/java/org/apache/cassandra/index/sasi/disk/PerSSTableIndexWriter.java
+++ b/src/java/org/apache/cassandra/index/sasi/disk/PerSSTableIndexWriter.java
@@ -27,7 +27,7 @@
import org.apache.cassandra.concurrent.JMXEnabledThreadPoolExecutor;
import org.apache.cassandra.concurrent.NamedThreadFactory;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.compaction.OperationType;
import org.apache.cassandra.db.rows.Row;
@@ -45,6 +45,7 @@
import org.apache.cassandra.utils.Pair;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Maps;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.Uninterruptibles;
@@ -82,7 +83,7 @@
private final AbstractType<?> keyValidator;
@VisibleForTesting
- protected final Map<ColumnDefinition, Index> indexes;
+ protected final Map<ColumnMetadata, Index> indexes;
private DecoratedKey currentKey;
private long currentKeyPosition;
@@ -91,13 +92,13 @@
public PerSSTableIndexWriter(AbstractType<?> keyValidator,
Descriptor descriptor,
OperationType source,
- Map<ColumnDefinition, ColumnIndex> supportedIndexes)
+ Map<ColumnMetadata, ColumnIndex> supportedIndexes)
{
this.keyValidator = keyValidator;
this.descriptor = descriptor;
this.source = source;
- this.indexes = new HashMap<>();
- for (Map.Entry<ColumnDefinition, ColumnIndex> entry : supportedIndexes.entrySet())
+ this.indexes = Maps.newHashMapWithExpectedSize(supportedIndexes.size());
+ for (Map.Entry<ColumnMetadata, ColumnIndex> entry : supportedIndexes.entrySet())
indexes.put(entry.getKey(), newIndex(entry.getValue()));
}
@@ -151,7 +152,7 @@
}
}
- public Index getIndex(ColumnDefinition columnDef)
+ public Index getIndex(ColumnMetadata columnDef)
{
return indexes.get(columnDef);
}
diff --git a/src/java/org/apache/cassandra/index/sasi/disk/TokenTree.java b/src/java/org/apache/cassandra/index/sasi/disk/TokenTree.java
index c69ce00..e510cdd 100644
--- a/src/java/org/apache/cassandra/index/sasi/disk/TokenTree.java
+++ b/src/java/org/apache/cassandra/index/sasi/disk/TokenTree.java
@@ -27,7 +27,7 @@
import org.apache.cassandra.index.sasi.utils.RangeIterator;
import org.apache.cassandra.utils.MergeIterator;
-import com.carrotsearch.hppc.LongOpenHashSet;
+import com.carrotsearch.hppc.LongHashSet;
import com.carrotsearch.hppc.LongSet;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Function;
@@ -410,7 +410,7 @@
public LongSet getOffsets()
{
- LongSet offsets = new LongOpenHashSet(4);
+ LongSet offsets = new LongHashSet(4);
for (TokenInfo i : info)
{
for (long offset : i.fetchOffsets())
diff --git a/src/java/org/apache/cassandra/index/sasi/exceptions/TimeQuotaExceededException.java b/src/java/org/apache/cassandra/index/sasi/exceptions/TimeQuotaExceededException.java
index af577dc..e237614 100644
--- a/src/java/org/apache/cassandra/index/sasi/exceptions/TimeQuotaExceededException.java
+++ b/src/java/org/apache/cassandra/index/sasi/exceptions/TimeQuotaExceededException.java
@@ -18,4 +18,8 @@
package org.apache.cassandra.index.sasi.exceptions;
public class TimeQuotaExceededException extends RuntimeException
-{}
+{
+ public TimeQuotaExceededException(String message) {
+ super(message);
+ }
+}
diff --git a/src/java/org/apache/cassandra/index/sasi/memory/KeyRangeIterator.java b/src/java/org/apache/cassandra/index/sasi/memory/KeyRangeIterator.java
index a2f2c0e..0f681b7 100644
--- a/src/java/org/apache/cassandra/index/sasi/memory/KeyRangeIterator.java
+++ b/src/java/org/apache/cassandra/index/sasi/memory/KeyRangeIterator.java
@@ -29,7 +29,7 @@
import org.apache.cassandra.index.sasi.utils.CombinedValue;
import org.apache.cassandra.index.sasi.utils.RangeIterator;
-import com.carrotsearch.hppc.LongOpenHashSet;
+import com.carrotsearch.hppc.LongHashSet;
import com.carrotsearch.hppc.LongSet;
import com.google.common.collect.PeekingIterator;
@@ -37,9 +37,9 @@
{
private final DKIterator iterator;
- public KeyRangeIterator(ConcurrentSkipListSet<DecoratedKey> keys)
+ public KeyRangeIterator(ConcurrentSkipListSet<DecoratedKey> keys, int size)
{
- super((Long) keys.first().getToken().getTokenValue(), (Long) keys.last().getToken().getTokenValue(), keys.size());
+ super((Long) keys.first().getToken().getTokenValue(), (Long) keys.last().getToken().getTokenValue(), size);
this.iterator = new DKIterator(keys.iterator());
}
@@ -95,7 +95,7 @@
public LongSet getOffsets()
{
- LongSet offsets = new LongOpenHashSet(4);
+ LongSet offsets = new LongHashSet(4);
for (DecoratedKey key : keys)
offsets.add((long) key.getToken().getTokenValue());
diff --git a/src/java/org/apache/cassandra/index/sasi/memory/SkipListMemIndex.java b/src/java/org/apache/cassandra/index/sasi/memory/SkipListMemIndex.java
index 69b57d0..b2cb83f 100644
--- a/src/java/org/apache/cassandra/index/sasi/memory/SkipListMemIndex.java
+++ b/src/java/org/apache/cassandra/index/sasi/memory/SkipListMemIndex.java
@@ -63,6 +63,7 @@
return overhead;
}
+ @SuppressWarnings("resource")
public RangeIterator<Long, Token> search(Expression expression)
{
ByteBuffer min = expression.lower == null ? null : expression.lower.value;
@@ -88,9 +89,12 @@
}
RangeUnionIterator.Builder<Long, Token> builder = RangeUnionIterator.builder();
- search.values().stream()
- .filter(keys -> !keys.isEmpty())
- .forEach(keys -> builder.add(new KeyRangeIterator(keys)));
+
+ for (ConcurrentSkipListSet<DecoratedKey> keys : search.values()) {
+ int size;
+ if ((size = keys.size()) > 0)
+ builder.add(new KeyRangeIterator(keys, size));
+ }
return builder.build();
}
diff --git a/src/java/org/apache/cassandra/index/sasi/memory/TrieMemIndex.java b/src/java/org/apache/cassandra/index/sasi/memory/TrieMemIndex.java
index ca60ac5..cbdb6b7 100644
--- a/src/java/org/apache/cassandra/index/sasi/memory/TrieMemIndex.java
+++ b/src/java/org/apache/cassandra/index/sasi/memory/TrieMemIndex.java
@@ -22,8 +22,8 @@
import java.util.List;
import java.util.concurrent.ConcurrentSkipListSet;
-import org.apache.cassandra.config.ColumnDefinition;
import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.index.sasi.conf.ColumnIndex;
import org.apache.cassandra.index.sasi.disk.OnDiskIndexBuilder;
import org.apache.cassandra.index.sasi.disk.Token;
@@ -106,9 +106,9 @@
{
public static final SizeEstimatingNodeFactory NODE_FACTORY = new SizeEstimatingNodeFactory();
- protected final ColumnDefinition definition;
+ protected final ColumnMetadata definition;
- public ConcurrentTrie(ColumnDefinition column)
+ public ConcurrentTrie(ColumnMetadata column)
{
definition = column;
}
@@ -137,6 +137,7 @@
return overhead;
}
+ @SuppressWarnings("resource")
public RangeIterator<Long, Token> search(Expression expression)
{
ByteBuffer prefix = expression.lower == null ? null : expression.lower.value;
@@ -146,8 +147,9 @@
RangeUnionIterator.Builder<Long, Token> builder = RangeUnionIterator.builder();
for (ConcurrentSkipListSet<DecoratedKey> keys : search)
{
- if (!keys.isEmpty())
- builder.add(new KeyRangeIterator(keys));
+ int size;
+ if ((size = keys.size()) > 0)
+ builder.add(new KeyRangeIterator(keys, size));
}
return builder.build();
@@ -162,7 +164,7 @@
{
private final ConcurrentRadixTree<ConcurrentSkipListSet<DecoratedKey>> trie;
- private ConcurrentPrefixTrie(ColumnDefinition column)
+ private ConcurrentPrefixTrie(ColumnMetadata column)
{
super(column);
trie = new ConcurrentRadixTree<>(NODE_FACTORY);
@@ -200,7 +202,7 @@
{
private final ConcurrentSuffixTree<ConcurrentSkipListSet<DecoratedKey>> trie;
- private ConcurrentSuffixTrie(ColumnDefinition column)
+ private ConcurrentSuffixTrie(ColumnMetadata column)
{
super(column);
trie = new ConcurrentSuffixTree<>(NODE_FACTORY);
diff --git a/src/java/org/apache/cassandra/index/sasi/plan/Expression.java b/src/java/org/apache/cassandra/index/sasi/plan/Expression.java
index fba7f34..8de45e8 100644
--- a/src/java/org/apache/cassandra/index/sasi/plan/Expression.java
+++ b/src/java/org/apache/cassandra/index/sasi/plan/Expression.java
@@ -22,7 +22,7 @@
import java.util.List;
import java.util.Objects;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.cql3.Operator;
import org.apache.cassandra.index.sasi.analyzer.AbstractAnalyzer;
import org.apache.cassandra.index.sasi.conf.ColumnIndex;
@@ -115,7 +115,7 @@
@VisibleForTesting
public Expression(String name, AbstractType<?> validator)
{
- this(null, new ColumnIndex(UTF8Type.instance, ColumnDefinition.regularDef("sasi", "internal", name, validator), null));
+ this(null, new ColumnIndex(UTF8Type.instance, ColumnMetadata.regularColumn("sasi", "internal", name, validator), null));
}
public Expression setLower(Bound newLower)
@@ -411,5 +411,13 @@
Bound o = (Bound) other;
return value.equals(o.value) && inclusive == o.inclusive;
}
+
+ public int hashCode()
+ {
+ HashCodeBuilder builder = new HashCodeBuilder();
+ builder.append(value);
+ builder.append(inclusive);
+ return builder.toHashCode();
+ }
}
}
diff --git a/src/java/org/apache/cassandra/index/sasi/plan/Operation.java b/src/java/org/apache/cassandra/index/sasi/plan/Operation.java
index aaa3068..40c6afb 100644
--- a/src/java/org/apache/cassandra/index/sasi/plan/Operation.java
+++ b/src/java/org/apache/cassandra/index/sasi/plan/Operation.java
@@ -21,8 +21,8 @@
import java.nio.ByteBuffer;
import java.util.*;
-import org.apache.cassandra.config.ColumnDefinition;
-import org.apache.cassandra.config.ColumnDefinition.Kind;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.ColumnMetadata.Kind;
import org.apache.cassandra.cql3.Operator;
import org.apache.cassandra.db.filter.RowFilter;
import org.apache.cassandra.db.rows.Row;
@@ -37,6 +37,7 @@
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.*;
+
import org.apache.cassandra.utils.FBUtilities;
@SuppressWarnings("resource")
@@ -65,14 +66,14 @@
private final QueryController controller;
protected final OperationType op;
- protected final ListMultimap<ColumnDefinition, Expression> expressions;
+ protected final ListMultimap<ColumnMetadata, Expression> expressions;
protected final RangeIterator<Long, Token> range;
protected Operation left, right;
private Operation(OperationType operation,
QueryController controller,
- ListMultimap<ColumnDefinition, Expression> expressions,
+ ListMultimap<ColumnMetadata, Expression> expressions,
RangeIterator<Long, Token> range,
Operation left, Operation right)
{
@@ -207,7 +208,7 @@
boolean result = false;
int idx = 0;
- for (ColumnDefinition column : expressions.keySet())
+ for (ColumnMetadata column : expressions.keySet())
{
if (column.kind == Kind.PARTITION_KEY)
continue;
@@ -262,11 +263,11 @@
}
@VisibleForTesting
- protected static ListMultimap<ColumnDefinition, Expression> analyzeGroup(QueryController controller,
- OperationType op,
- List<RowFilter.Expression> expressions)
+ protected static ListMultimap<ColumnMetadata, Expression> analyzeGroup(QueryController controller,
+ OperationType op,
+ List<RowFilter.Expression> expressions)
{
- ListMultimap<ColumnDefinition, Expression> analyzed = ArrayListMultimap.create();
+ ListMultimap<ColumnMetadata, Expression> analyzed = ArrayListMultimap.create();
// sort all of the expressions in the operation by name and priority of the logical operator
// this gives us an efficient way to handle inequality and combining into ranges without extra processing
@@ -429,7 +430,7 @@
{
if (!expressions.isEmpty())
{
- ListMultimap<ColumnDefinition, Expression> analyzedExpressions = analyzeGroup(controller, op, expressions);
+ ListMultimap<ColumnMetadata, Expression> analyzedExpressions = analyzeGroup(controller, op, expressions);
RangeIterator.Builder<Long, Token> range = controller.getIndexes(op, analyzedExpressions.values());
Operation rightOp = null;
diff --git a/src/java/org/apache/cassandra/index/sasi/plan/QueryController.java b/src/java/org/apache/cassandra/index/sasi/plan/QueryController.java
index 22fca68..db16c52 100644
--- a/src/java/org/apache/cassandra/index/sasi/plan/QueryController.java
+++ b/src/java/org/apache/cassandra/index/sasi/plan/QueryController.java
@@ -22,10 +22,15 @@
import com.google.common.collect.Sets;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.db.*;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DataRange;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.PartitionRangeReadCommand;
+import org.apache.cassandra.db.ReadExecutionController;
+import org.apache.cassandra.db.SinglePartitionReadCommand;
import org.apache.cassandra.db.filter.DataLimits;
import org.apache.cassandra.db.filter.RowFilter;
+import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.db.rows.UnfilteredRowIterator;
import org.apache.cassandra.index.Index;
import org.apache.cassandra.index.sasi.SASIIndex;
@@ -39,9 +44,9 @@
import org.apache.cassandra.index.sasi.utils.RangeIntersectionIterator;
import org.apache.cassandra.index.sasi.utils.RangeIterator;
import org.apache.cassandra.index.sasi.utils.RangeUnionIterator;
-import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.utils.Pair;
public class QueryController
@@ -63,12 +68,7 @@
this.executionStart = System.nanoTime();
}
- public boolean isForThrift()
- {
- return command.isForThrift();
- }
-
- public CFMetaData metadata()
+ public TableMetadata metadata()
{
return command.metadata();
}
@@ -85,7 +85,7 @@
public AbstractType<?> getKeyValidator()
{
- return cfs.metadata.getKeyValidator();
+ return cfs.metadata().partitionKeyType;
}
public ColumnIndex getIndex(RowFilter.Expression expression)
@@ -101,8 +101,7 @@
throw new NullPointerException();
try
{
- SinglePartitionReadCommand partition = SinglePartitionReadCommand.create(command.isForThrift(),
- cfs.metadata,
+ SinglePartitionReadCommand partition = SinglePartitionReadCommand.create(cfs.metadata(),
command.nowInSec(),
command.columnFilter(),
command.rowFilter().withoutExpressions(),
@@ -137,9 +136,10 @@
? RangeUnionIterator.<Long, Token>builder()
: RangeIntersectionIterator.<Long, Token>builder();
- List<RangeIterator<Long, Token>> perIndexUnions = new ArrayList<>();
+ Set<Map.Entry<Expression, Set<SSTableIndex>>> view = getView(op, expressions).entrySet();
+ List<RangeIterator<Long, Token>> perIndexUnions = new ArrayList<>(view.size());
- for (Map.Entry<Expression, Set<SSTableIndex>> e : getView(op, expressions).entrySet())
+ for (Map.Entry<Expression, Set<SSTableIndex>> e : view)
{
@SuppressWarnings("resource") // RangeIterators are closed by releaseIndexes
RangeIterator<Long, Token> index = TermIterator.build(e.getKey(), e.getValue());
@@ -154,8 +154,13 @@
public void checkpoint()
{
- if ((System.nanoTime() - executionStart) >= executionQuota)
- throw new TimeQuotaExceededException();
+ long executionTime = (System.nanoTime() - executionStart);
+
+ if (executionTime >= executionQuota)
+ throw new TimeQuotaExceededException(
+ "Command '" + command + "' took too long " +
+ "(" + TimeUnit.NANOSECONDS.toMillis(executionTime) +
+ " >= " + TimeUnit.NANOSECONDS.toMillis(executionQuota) + "ms).");
}
public void releaseIndexes(Operation operation)
diff --git a/src/java/org/apache/cassandra/index/sasi/plan/QueryPlan.java b/src/java/org/apache/cassandra/index/sasi/plan/QueryPlan.java
index 326ea0d..a54dfc8 100644
--- a/src/java/org/apache/cassandra/index/sasi/plan/QueryPlan.java
+++ b/src/java/org/apache/cassandra/index/sasi/plan/QueryPlan.java
@@ -19,7 +19,6 @@
import java.util.*;
-import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
import org.apache.cassandra.db.rows.*;
@@ -28,6 +27,7 @@
import org.apache.cassandra.index.sasi.plan.Operation.OperationType;
import org.apache.cassandra.exceptions.RequestTimeoutException;
import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.utils.AbstractIterator;
public class QueryPlan
@@ -156,12 +156,7 @@
}
}
- public boolean isForThrift()
- {
- return controller.isForThrift();
- }
-
- public CFMetaData metadata()
+ public TableMetadata metadata()
{
return controller.metadata();
}
diff --git a/src/java/org/apache/cassandra/index/sasi/utils/MappedBuffer.java b/src/java/org/apache/cassandra/index/sasi/utils/MappedBuffer.java
index efabe7b..0899be6 100644
--- a/src/java/org/apache/cassandra/index/sasi/utils/MappedBuffer.java
+++ b/src/java/org/apache/cassandra/index/sasi/utils/MappedBuffer.java
@@ -217,9 +217,6 @@
public void close()
{
- if (!FileUtils.isCleanerAvailable)
- return;
-
/*
* Try forcing the unmapping of pages using undocumented unsafe sun APIs.
* If this fails (non Sun JVM), we'll have to wait for the GC to finalize the mapping.
diff --git a/src/java/org/apache/cassandra/index/sasi/utils/RangeIntersectionIterator.java b/src/java/org/apache/cassandra/index/sasi/utils/RangeIntersectionIterator.java
index bd8c725..4d751da 100644
--- a/src/java/org/apache/cassandra/index/sasi/utils/RangeIntersectionIterator.java
+++ b/src/java/org/apache/cassandra/index/sasi/utils/RangeIntersectionIterator.java
@@ -127,6 +127,8 @@
protected D computeNext()
{
+ List<RangeIterator<K, D>> processed = null;
+
while (!ranges.isEmpty())
{
RangeIterator<K, D> head = ranges.poll();
@@ -142,7 +144,8 @@
return endOfData();
}
- List<RangeIterator<K, D>> processed = new ArrayList<>();
+ if (processed == null)
+ processed = new ArrayList<>();
boolean intersectsAll = true, exhausted = false;
while (!ranges.isEmpty())
@@ -183,8 +186,8 @@
ranges.add(head);
- for (RangeIterator<K, D> range : processed)
- ranges.add(range);
+ ranges.addAll(processed);
+ processed.clear();
if (exhausted)
return endOfData();
diff --git a/src/java/org/apache/cassandra/io/ForwardingVersionedSerializer.java b/src/java/org/apache/cassandra/io/ForwardingVersionedSerializer.java
deleted file mode 100644
index 64f91d7..0000000
--- a/src/java/org/apache/cassandra/io/ForwardingVersionedSerializer.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.io;
-
-import java.io.IOException;
-
-import org.apache.cassandra.io.util.DataInputPlus;
-import org.apache.cassandra.io.util.DataOutputPlus;
-
-/**
- * A serializer which forwards all its method calls to another serializer. Subclasses should override one or more
- * methods to modify the behavior of the backing serializer as desired per the decorator pattern.
- */
-public abstract class ForwardingVersionedSerializer<T> implements IVersionedSerializer<T>
-{
- protected ForwardingVersionedSerializer()
- {
- }
-
- /**
- * Returns the backing delegate instance that methods are forwarded to.
- *
- * @param version the server version
- * @return the backing delegate instance that methods are forwarded to.
- */
- protected abstract IVersionedSerializer<T> delegate(int version);
-
- public void serialize(T t, DataOutputPlus out, int version) throws IOException
- {
- delegate(version).serialize(t, out, version);
- }
-
- public T deserialize(DataInputPlus in, int version) throws IOException
- {
- return delegate(version).deserialize(in, version);
- }
-
- public long serializedSize(T t, int version)
- {
- return delegate(version).serializedSize(t, version);
- }
-}
diff --git a/src/java/org/apache/cassandra/io/ISSTableSerializer.java b/src/java/org/apache/cassandra/io/ISSTableSerializer.java
deleted file mode 100644
index 96a38ac..0000000
--- a/src/java/org/apache/cassandra/io/ISSTableSerializer.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.io;
-
-import java.io.DataInput;
-import java.io.IOException;
-
-import org.apache.cassandra.io.sstable.format.Version;
-import org.apache.cassandra.io.util.DataOutputPlus;
-
-public interface ISSTableSerializer<T>
-{
- /**
- * Serialize the specified type into the specified DataOutputStream
- * instance in the format suited for SSTables.
- *
- * @param t type that needs to be serialized
- * @param out DataOutput into which serialization needs to happen.
- * @throws java.io.IOException
- */
- public void serializeForSSTable(T t, DataOutputPlus out) throws IOException;
-
- /**
- * Deserialize into the specified DataInputStream instance in the format
- * suited for SSTables.
- * @param in DataInput from which deserialization needs to happen.
- * @param version the version for the sstable we're reading from
- * @throws IOException
- * @return the type that was deserialized
- */
- public T deserializeFromSSTable(DataInput in, Version version) throws IOException;
-}
diff --git a/src/java/org/apache/cassandra/io/IVersionedAsymmetricSerializer.java b/src/java/org/apache/cassandra/io/IVersionedAsymmetricSerializer.java
new file mode 100644
index 0000000..8ad2c28
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/IVersionedAsymmetricSerializer.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.io;
+
+import java.io.IOException;
+
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+
+public interface IVersionedAsymmetricSerializer<In, Out>
+{
+ /**
+ * Serialize the specified type into the specified DataOutputStream instance.
+ *
+ * @param t type that needs to be serialized
+ * @param out DataOutput into which serialization needs to happen.
+ * @param version protocol version
+ * @throws IOException if serialization fails
+ */
+ public void serialize(In t, DataOutputPlus out, int version) throws IOException;
+
+ /**
+ * Deserialize into the specified DataInputStream instance.
+ * @param in DataInput from which deserialization needs to happen.
+ * @param version protocol version
+ * @return the type that was deserialized
+ * @throws IOException if deserialization fails
+ */
+ public Out deserialize(DataInputPlus in, int version) throws IOException;
+
+ /**
+ * Calculate serialized size of object without actually serializing.
+ * @param t object to calculate serialized size
+ * @param version protocol version
+ * @return serialized size of object t
+ */
+ public long serializedSize(In t, int version);
+}
diff --git a/src/java/org/apache/cassandra/io/IVersionedSerializer.java b/src/java/org/apache/cassandra/io/IVersionedSerializer.java
index e555573..6730ec0 100644
--- a/src/java/org/apache/cassandra/io/IVersionedSerializer.java
+++ b/src/java/org/apache/cassandra/io/IVersionedSerializer.java
@@ -17,37 +17,6 @@
*/
package org.apache.cassandra.io;
-import java.io.IOException;
-
-import org.apache.cassandra.io.util.DataInputPlus;
-import org.apache.cassandra.io.util.DataOutputPlus;
-
-public interface IVersionedSerializer<T>
+public interface IVersionedSerializer<T> extends IVersionedAsymmetricSerializer<T, T>
{
- /**
- * Serialize the specified type into the specified DataOutputStream instance.
- *
- * @param t type that needs to be serialized
- * @param out DataOutput into which serialization needs to happen.
- * @param version protocol version
- * @throws java.io.IOException if serialization fails
- */
- public void serialize(T t, DataOutputPlus out, int version) throws IOException;
-
- /**
- * Deserialize into the specified DataInputStream instance.
- * @param in DataInput from which deserialization needs to happen.
- * @param version protocol version
- * @return the type that was deserialized
- * @throws IOException if deserialization fails
- */
- public T deserialize(DataInputPlus in, int version) throws IOException;
-
- /**
- * Calculate serialized size of object without actually serializing.
- * @param t object to calculate serialized size
- * @param version protocol version
- * @return serialized size of object t
- */
- public long serializedSize(T t, int version);
}
diff --git a/src/java/org/apache/cassandra/io/compress/CompressedSequentialWriter.java b/src/java/org/apache/cassandra/io/compress/CompressedSequentialWriter.java
index 4068be7..2190824 100644
--- a/src/java/org/apache/cassandra/io/compress/CompressedSequentialWriter.java
+++ b/src/java/org/apache/cassandra/io/compress/CompressedSequentialWriter.java
@@ -32,6 +32,7 @@
import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
import org.apache.cassandra.io.util.*;
import org.apache.cassandra.schema.CompressionParams;
+import org.apache.cassandra.utils.ByteBufferUtil;
import static org.apache.cassandra.utils.Throwables.merge;
@@ -60,6 +61,8 @@
private final ByteBuffer crcCheckBuffer = ByteBuffer.allocate(4);
private final Optional<File> digestFile;
+ private final int maxCompressedLength;
+
/**
* Create CompressedSequentialWriter without digest file.
*
@@ -90,6 +93,8 @@
// buffer for compression should be the same size as buffer itself
compressed = compressor.preferredBufferType().allocate(compressor.initialCompressedBufferLength(buffer.capacity()));
+ maxCompressedLength = parameters.maxCompressedLength();
+
/* Index File (-CompressionInfo.db component) and it's header */
metadataWriter = CompressionMetadata.Writer.open(parameters, offsetsPath);
@@ -144,8 +149,28 @@
throw new RuntimeException("Compression exception", e); // shouldn't happen
}
+ int uncompressedLength = buffer.position();
int compressedLength = compressed.position();
- uncompressedSize += buffer.position();
+ uncompressedSize += uncompressedLength;
+ ByteBuffer toWrite = compressed;
+ if (compressedLength >= maxCompressedLength)
+ {
+ toWrite = buffer;
+ if (uncompressedLength >= maxCompressedLength)
+ {
+ compressedLength = uncompressedLength;
+ }
+ else
+ {
+ // Pad the uncompressed data so that it reaches the max compressed length.
+ // This could make the chunk appear longer, but this path is only reached at the end of the file, where
+ // we use the file size to limit the buffer on reading.
+ assert maxCompressedLength <= buffer.capacity(); // verified by CompressionParams.validate
+ buffer.limit(maxCompressedLength);
+ ByteBufferUtil.writeZeroes(buffer, maxCompressedLength - uncompressedLength);
+ compressedLength = maxCompressedLength;
+ }
+ }
compressedSize += compressedLength;
try
@@ -155,18 +180,20 @@
chunkCount++;
// write out the compressed data
- compressed.flip();
- channel.write(compressed);
+ toWrite.flip();
+ channel.write(toWrite);
// write corresponding checksum
- compressed.rewind();
- crcMetadata.appendDirect(compressed, true);
+ toWrite.rewind();
+ crcMetadata.appendDirect(toWrite, true);
lastFlushOffset = uncompressedSize;
}
catch (IOException e)
{
throw new FSWriteError(e, getPath());
}
+ if (toWrite == buffer)
+ buffer.position(uncompressedLength);
// next chunk should be written right after current + length of the checksum (int)
chunkOffset += compressedLength + 4;
@@ -231,7 +258,10 @@
// Repopulate buffer from compressed data
buffer.clear();
compressed.flip();
- compressor.uncompress(compressed, buffer);
+ if (chunkSize < maxCompressedLength)
+ compressor.uncompress(compressed, buffer);
+ else
+ buffer.put(compressed);
}
catch (IOException e)
{
diff --git a/src/java/org/apache/cassandra/io/compress/CompressionMetadata.java b/src/java/org/apache/cassandra/io/compress/CompressionMetadata.java
index 90ba749a..3f08fe2 100644
--- a/src/java/org/apache/cassandra/io/compress/CompressionMetadata.java
+++ b/src/java/org/apache/cassandra/io/compress/CompressionMetadata.java
@@ -17,6 +17,8 @@
*/
package org.apache.cassandra.io.compress;
+import java.nio.file.Files;
+import java.nio.file.Paths;
import java.io.BufferedOutputStream;
import java.io.DataInput;
import java.io.DataInputStream;
@@ -47,13 +49,13 @@
import org.apache.cassandra.io.sstable.Component;
import org.apache.cassandra.io.sstable.CorruptSSTableException;
import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.io.util.Memory;
import org.apache.cassandra.io.util.SafeMemory;
import org.apache.cassandra.schema.CompressionParams;
-import org.apache.cassandra.utils.ChecksumType;
-import org.apache.cassandra.utils.Pair;
+import org.apache.cassandra.utils.SyncUtil;
import org.apache.cassandra.utils.concurrent.Transactional;
import org.apache.cassandra.utils.concurrent.Ref;
@@ -71,7 +73,6 @@
private final long chunkOffsetsSize;
public final String indexFilePath;
public final CompressionParams parameters;
- public final ChecksumType checksumType;
/**
* Create metadata about given compressed file including uncompressed data length, chunk size
@@ -86,17 +87,26 @@
*/
public static CompressionMetadata create(String dataFilePath)
{
- Descriptor desc = Descriptor.fromFilename(dataFilePath);
- return new CompressionMetadata(desc.filenameFor(Component.COMPRESSION_INFO), new File(dataFilePath).length(), desc.version.compressedChecksumType());
+ return createWithLength(dataFilePath, new File(dataFilePath).length());
+ }
+
+ public static CompressionMetadata createWithLength(String dataFilePath, long compressedLength)
+ {
+ return new CompressionMetadata(Descriptor.fromFilename(dataFilePath), compressedLength);
}
@VisibleForTesting
- public CompressionMetadata(String indexFilePath, long compressedLength, ChecksumType checksumType)
+ public CompressionMetadata(Descriptor desc, long compressedLength)
+ {
+ this(desc.filenameFor(Component.COMPRESSION_INFO), compressedLength, desc.version.hasMaxCompressedLength());
+ }
+
+ @VisibleForTesting
+ public CompressionMetadata(String indexFilePath, long compressedLength, boolean hasMaxCompressedSize)
{
this.indexFilePath = indexFilePath;
- this.checksumType = checksumType;
- try (DataInputStream stream = new DataInputStream(new FileInputStream(indexFilePath)))
+ try (DataInputStream stream = new DataInputStream(Files.newInputStream(Paths.get(indexFilePath))))
{
String compressorName = stream.readUTF();
int optionCount = stream.readInt();
@@ -108,9 +118,12 @@
options.put(key, value);
}
int chunkLength = stream.readInt();
+ int maxCompressedSize = Integer.MAX_VALUE;
+ if (hasMaxCompressedSize)
+ maxCompressedSize = stream.readInt();
try
{
- parameters = new CompressionParams(compressorName, chunkLength, options);
+ parameters = new CompressionParams(compressorName, chunkLength, maxCompressedSize, options);
}
catch (ConfigurationException e)
{
@@ -133,7 +146,9 @@
this.chunkOffsetsSize = chunkOffsets.size();
}
- private CompressionMetadata(String filePath, CompressionParams parameters, SafeMemory offsets, long offsetsSize, long dataLength, long compressedLength, ChecksumType checksumType)
+ // do not call this constructor directly, unless used in testing
+ @VisibleForTesting
+ public CompressionMetadata(String filePath, CompressionParams parameters, Memory offsets, long offsetsSize, long dataLength, long compressedLength)
{
this.indexFilePath = filePath;
this.parameters = parameters;
@@ -141,7 +156,6 @@
this.compressedFileLength = compressedLength;
this.chunkOffsets = offsets;
this.chunkOffsetsSize = offsetsSize;
- this.checksumType = checksumType;
}
public ICompressor compressor()
@@ -154,6 +168,11 @@
return parameters.chunkLength();
}
+ public int maxCompressedLength()
+ {
+ return parameters.maxCompressedLength();
+ }
+
/**
* Returns the amount of memory in bytes used off heap.
* @return the amount of memory in bytes used off heap
@@ -247,15 +266,18 @@
* @param sections Collection of sections in uncompressed file. Should not contain sections that overlap each other.
* @return Total chunk size in bytes for given sections including checksum.
*/
- public long getTotalSizeForSections(Collection<Pair<Long, Long>> sections)
+ public long getTotalSizeForSections(Collection<SSTableReader.PartitionPositionBounds> sections)
{
long size = 0;
long lastOffset = -1;
- for (Pair<Long, Long> section : sections)
+ for (SSTableReader.PartitionPositionBounds section : sections)
{
- int startIndex = (int) (section.left / parameters.chunkLength());
- int endIndex = (int) (section.right / parameters.chunkLength());
- endIndex = section.right % parameters.chunkLength() == 0 ? endIndex - 1 : endIndex;
+ int startIndex = (int) (section.lowerPosition / parameters.chunkLength());
+
+ int endIndex = (int) (section.upperPosition / parameters.chunkLength());
+ if (section.upperPosition % parameters.chunkLength() == 0)
+ endIndex--;
+
for (int i = startIndex; i <= endIndex; i++)
{
long offset = i * 8L;
@@ -277,21 +299,19 @@
* @param sections Collection of sections in uncompressed file
* @return Array of chunks which corresponds to given sections of uncompressed file, sorted by chunk offset
*/
- public Chunk[] getChunksForSections(Collection<Pair<Long, Long>> sections)
+ public Chunk[] getChunksForSections(Collection<SSTableReader.PartitionPositionBounds> sections)
{
// use SortedSet to eliminate duplicates and sort by chunk offset
- SortedSet<Chunk> offsets = new TreeSet<Chunk>(new Comparator<Chunk>()
+ SortedSet<Chunk> offsets = new TreeSet<>((o1, o2) -> Longs.compare(o1.offset, o2.offset));
+
+ for (SSTableReader.PartitionPositionBounds section : sections)
{
- public int compare(Chunk o1, Chunk o2)
- {
- return Longs.compare(o1.offset, o2.offset);
- }
- });
- for (Pair<Long, Long> section : sections)
- {
- int startIndex = (int) (section.left / parameters.chunkLength());
- int endIndex = (int) (section.right / parameters.chunkLength());
- endIndex = section.right % parameters.chunkLength() == 0 ? endIndex - 1 : endIndex;
+ int startIndex = (int) (section.lowerPosition / parameters.chunkLength());
+
+ int endIndex = (int) (section.upperPosition / parameters.chunkLength());
+ if (section.upperPosition % parameters.chunkLength() == 0)
+ endIndex--;
+
for (int i = startIndex; i <= endIndex; i++)
{
long offset = i * 8L;
@@ -302,6 +322,7 @@
offsets.add(new Chunk(chunkOffset, (int) (nextChunkOffset - chunkOffset - 4))); // "4" bytes reserved for checksum
}
}
+
return offsets.toArray(new Chunk[offsets.size()]);
}
@@ -358,6 +379,7 @@
// store the length of the chunk
out.writeInt(parameters.chunkLength());
+ out.writeInt(parameters.maxCompressedLength());
// store position and reserve a place for uncompressed data length and chunks count
out.writeLong(dataLength);
out.writeInt(chunks);
@@ -398,7 +420,7 @@
out.writeLong(offsets.getLong(i * 8L));
out.flush();
- fos.getFD().sync();
+ SyncUtil.sync(fos);
}
catch (IOException e)
{
@@ -409,19 +431,19 @@
@SuppressWarnings("resource")
public CompressionMetadata open(long dataLength, long compressedLength)
{
- SafeMemory offsets = this.offsets.sharedCopy();
+ SafeMemory tOffsets = this.offsets.sharedCopy();
// calculate how many entries we need, if our dataLength is truncated
- int count = (int) (dataLength / parameters.chunkLength());
+ int tCount = (int) (dataLength / parameters.chunkLength());
if (dataLength % parameters.chunkLength() != 0)
- count++;
+ tCount++;
- assert count > 0;
+ assert tCount > 0;
// grab our actual compressed length from the next offset from our the position we're opened to
- if (count < this.count)
- compressedLength = offsets.getLong(count * 8L);
+ if (tCount < this.count)
+ compressedLength = tOffsets.getLong(tCount * 8L);
- return new CompressionMetadata(filePath, parameters, offsets, count * 8L, dataLength, compressedLength, ChecksumType.CRC32);
+ return new CompressionMetadata(filePath, parameters, tOffsets, tCount * 8L, dataLength, compressedLength);
}
/**
diff --git a/src/java/org/apache/cassandra/io/compress/DeflateCompressor.java b/src/java/org/apache/cassandra/io/compress/DeflateCompressor.java
index 8557f5f..d3d7090 100644
--- a/src/java/org/apache/cassandra/io/compress/DeflateCompressor.java
+++ b/src/java/org/apache/cassandra/io/compress/DeflateCompressor.java
@@ -29,6 +29,8 @@
import java.util.zip.Deflater;
import java.util.zip.Inflater;
+import com.google.common.collect.ImmutableSet;
+
public class DeflateCompressor implements ICompressor
{
public static final DeflateCompressor instance = new DeflateCompressor();
@@ -49,6 +51,7 @@
private final FastThreadLocal<Deflater> deflater;
private final FastThreadLocal<Inflater> inflater;
+ private final Set<Uses> recommendedUses;
public static DeflateCompressor create(Map<String, String> compressionOptions)
{
@@ -74,6 +77,7 @@
return new Inflater();
}
};
+ recommendedUses = ImmutableSet.of(Uses.GENERAL);
}
public Set<String> supportedOptions()
@@ -226,4 +230,10 @@
// Prefer array-backed buffers.
return BufferType.ON_HEAP;
}
+
+ @Override
+ public Set<Uses> recommendedUses()
+ {
+ return recommendedUses;
+ }
}
diff --git a/src/java/org/apache/cassandra/io/compress/ICompressor.java b/src/java/org/apache/cassandra/io/compress/ICompressor.java
index 40dc7c2..fd6a104 100644
--- a/src/java/org/apache/cassandra/io/compress/ICompressor.java
+++ b/src/java/org/apache/cassandra/io/compress/ICompressor.java
@@ -19,10 +19,24 @@
import java.io.IOException;
import java.nio.ByteBuffer;
+import java.util.EnumSet;
import java.util.Set;
+import com.google.common.collect.ImmutableSet;
+
public interface ICompressor
{
+ /**
+ * Ways that a particular instance of ICompressor should be used internally in Cassandra.
+ *
+ * GENERAL: Suitable for general use
+ * FAST_COMPRESSION: Suitable for use in particularly latency sensitive compression situations (flushes).
+ */
+ enum Uses {
+ GENERAL,
+ FAST_COMPRESSION
+ }
+
public int initialCompressedBufferLength(int chunkLength);
public int uncompress(byte[] input, int inputOffset, int inputLength, byte[] output, int outputOffset) throws IOException;
@@ -57,4 +71,16 @@
public boolean supports(BufferType bufferType);
public Set<String> supportedOptions();
+
+ /**
+ * Hints to Cassandra which uses this compressor is recommended for. For example a compression algorithm which gets
+ * good compression ratio may trade off too much compression speed to be useful in certain compression heavy use
+ * cases such as flushes or mutation hints.
+ *
+ * Note that Cassandra may ignore these recommendations, it is not a strict contract.
+ */
+ default Set<Uses> recommendedUses()
+ {
+ return ImmutableSet.copyOf(EnumSet.allOf(Uses.class));
+ }
}
diff --git a/src/java/org/apache/cassandra/io/compress/LZ4Compressor.java b/src/java/org/apache/cassandra/io/compress/LZ4Compressor.java
index 1b3844d..6c333b7 100644
--- a/src/java/org/apache/cassandra/io/compress/LZ4Compressor.java
+++ b/src/java/org/apache/cassandra/io/compress/LZ4Compressor.java
@@ -20,12 +20,14 @@
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.Arrays;
+import java.util.EnumSet;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableSet;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -63,6 +65,10 @@
{
if (compressorType.equals(LZ4_FAST_COMPRESSOR) && args.get(LZ4_HIGH_COMPRESSION_LEVEL) != null)
logger.warn("'{}' parameter is ignored when '{}' is '{}'", LZ4_HIGH_COMPRESSION_LEVEL, LZ4_COMPRESSOR_TYPE, LZ4_FAST_COMPRESSOR);
+ if (compressorType.equals(LZ4_HIGH_COMPRESSOR))
+ logger.info("The ZstdCompressor may be preferable to LZ4 in 'high' mode. Zstd will typically " +
+ "compress much faster while achieving better ratio, but it may decompress more slowly,");
+
instance = new LZ4Compressor(compressorType, compressionLevel);
LZ4Compressor instanceFromMap = instances.putIfAbsent(compressorTypeAndLevel, instance);
if(instanceFromMap != null)
@@ -72,11 +78,12 @@
}
private final net.jpountz.lz4.LZ4Compressor compressor;
- private final net.jpountz.lz4.LZ4FastDecompressor decompressor;
+ private final net.jpountz.lz4.LZ4SafeDecompressor decompressor;
@VisibleForTesting
final String compressorType;
@VisibleForTesting
final Integer compressionLevel;
+ private final Set<Uses> recommendedUses;
private LZ4Compressor(String type, Integer compressionLevel)
{
@@ -88,16 +95,19 @@
case LZ4_HIGH_COMPRESSOR:
{
compressor = lz4Factory.highCompressor(compressionLevel);
+ // LZ4HC can be _extremely_ slow to compress, up to 10x slower
+ this.recommendedUses = ImmutableSet.of(Uses.GENERAL);
break;
}
case LZ4_FAST_COMPRESSOR:
default:
{
compressor = lz4Factory.fastCompressor();
+ this.recommendedUses = ImmutableSet.copyOf(EnumSet.allOf(Uses.class));
}
}
- decompressor = lz4Factory.fastDecompressor();
+ decompressor = lz4Factory.safeDecompressor();
}
public int initialCompressedBufferLength(int chunkLength)
@@ -131,20 +141,24 @@
| ((input[inputOffset + 2] & 0xFF) << 16)
| ((input[inputOffset + 3] & 0xFF) << 24);
- final int compressedLength;
+ final int writtenLength;
try
{
- compressedLength = decompressor.decompress(input, inputOffset + INTEGER_BYTES,
- output, outputOffset, decompressedLength);
+ writtenLength = decompressor.decompress(input,
+ inputOffset + INTEGER_BYTES,
+ inputLength - INTEGER_BYTES,
+ output,
+ outputOffset,
+ decompressedLength);
}
catch (LZ4Exception e)
{
throw new IOException(e);
}
- if (compressedLength != inputLength - INTEGER_BYTES)
+ if (writtenLength != decompressedLength)
{
- throw new IOException("Compressed lengths mismatch");
+ throw new IOException("Decompressed lengths mismatch");
}
return decompressedLength;
@@ -159,7 +173,8 @@
try
{
- int compressedLength = decompressor.decompress(input, input.position(), output, output.position(), decompressedLength);
+ int compressedLength = input.remaining();
+ decompressor.decompress(input, input.position(), input.remaining(), output, output.position(), decompressedLength);
input.position(input.position() + compressedLength);
output.position(output.position() + decompressedLength);
}
@@ -231,4 +246,10 @@
{
return true;
}
+
+ @Override
+ public Set<Uses> recommendedUses()
+ {
+ return recommendedUses;
+ }
}
diff --git a/src/java/org/apache/cassandra/io/compress/NoopCompressor.java b/src/java/org/apache/cassandra/io/compress/NoopCompressor.java
new file mode 100644
index 0000000..b6307ea
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/compress/NoopCompressor.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.compress;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Collections;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * A Compressor which doesn't actually compress any data. This is useful for either non-compressible data
+ * (typically already compressed or encrypted) that you still want block checksums for or for fast writing.
+ * Some relevant tickets:
+ * <p>
+ * <ul>
+ * <li>CASSANDRA-12682: Non compressed SSTables can silently corrupt data</li>
+ * <li>CASSANDRA-9264: Non compressed SSTables are written without checksums</li>
+ * </ul>
+ * </p>
+ */
+public class NoopCompressor implements ICompressor
+{
+ public static NoopCompressor create(Map<String, String> ignored)
+ {
+ return new NoopCompressor();
+ }
+
+ public int initialCompressedBufferLength(int chunkLength)
+ {
+ return chunkLength;
+ }
+
+ public int uncompress(byte[] input, int inputOffset, int inputLength, byte[] output, int outputOffset) throws IOException
+ {
+ System.arraycopy(input, inputOffset, output, outputOffset, inputLength);
+ return inputLength;
+ }
+
+ public void compress(ByteBuffer input, ByteBuffer output) throws IOException
+ {
+ output.put(input);
+ }
+
+ public void uncompress(ByteBuffer input, ByteBuffer output) throws IOException
+ {
+ output.put(input);
+ }
+
+ public BufferType preferredBufferType()
+ {
+ return BufferType.ON_HEAP;
+ }
+
+ public boolean supports(BufferType bufferType)
+ {
+ return true;
+ }
+
+ public Set<String> supportedOptions()
+ {
+ return Collections.emptySet();
+ }
+}
diff --git a/src/java/org/apache/cassandra/io/compress/ZstdCompressor.java b/src/java/org/apache/cassandra/io/compress/ZstdCompressor.java
new file mode 100644
index 0000000..c86db26
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/compress/ZstdCompressor.java
@@ -0,0 +1,248 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.compress;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableSet;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.github.luben.zstd.Zstd;
+
+/**
+ * ZSTD Compressor
+ */
+public class ZstdCompressor implements ICompressor
+{
+ private static final Logger logger = LoggerFactory.getLogger(ZstdCompressor.class);
+
+ // These might change with the version of Zstd we're using
+ public static final int FAST_COMPRESSION_LEVEL = Zstd.minCompressionLevel();
+ public static final int BEST_COMPRESSION_LEVEL = Zstd.maxCompressionLevel();
+
+ // Compressor Defaults
+ public static final int DEFAULT_COMPRESSION_LEVEL = 3;
+ private static final boolean ENABLE_CHECKSUM_FLAG = true;
+
+ @VisibleForTesting
+ public static final String COMPRESSION_LEVEL_OPTION_NAME = "compression_level";
+
+ private static final ConcurrentHashMap<Integer, ZstdCompressor> instances = new ConcurrentHashMap<>();
+
+ private final int compressionLevel;
+ private final Set<Uses> recommendedUses;
+
+ /**
+ * Create a Zstd compressor with the given options
+ *
+ * @param options
+ * @return
+ */
+ public static ZstdCompressor create(Map<String, String> options)
+ {
+ int level = getOrDefaultCompressionLevel(options);
+
+ if (!isValid(level))
+ throw new IllegalArgumentException(String.format("%s=%d is invalid", COMPRESSION_LEVEL_OPTION_NAME, level));
+
+ return getOrCreate(level);
+ }
+
+ /**
+ * Private constructor
+ *
+ * @param compressionLevel
+ */
+ private ZstdCompressor(int compressionLevel)
+ {
+ this.compressionLevel = compressionLevel;
+ this.recommendedUses = ImmutableSet.of(Uses.GENERAL);
+ logger.trace("Creating Zstd Compressor with compression level={}", compressionLevel);
+ }
+
+ /**
+ * Get a cached instance or return a new one
+ *
+ * @param level
+ * @return
+ */
+ public static ZstdCompressor getOrCreate(int level)
+ {
+ return instances.computeIfAbsent(level, l -> new ZstdCompressor(level));
+ }
+
+ /**
+ * Get initial compressed buffer length
+ *
+ * @param chunkLength
+ * @return
+ */
+ @Override
+ public int initialCompressedBufferLength(int chunkLength)
+ {
+ return (int) Zstd.compressBound(chunkLength);
+ }
+
+ /**
+ * Decompress data using arrays
+ *
+ * @param input
+ * @param inputOffset
+ * @param inputLength
+ * @param output
+ * @param outputOffset
+ * @return
+ * @throws IOException
+ */
+ @Override
+ public int uncompress(byte[] input, int inputOffset, int inputLength, byte[] output, int outputOffset)
+ throws IOException
+ {
+ long dsz = Zstd.decompressByteArray(output, outputOffset, output.length - outputOffset,
+ input, inputOffset, inputLength);
+
+ if (Zstd.isError(dsz))
+ throw new IOException(String.format("Decompression failed due to %s", Zstd.getErrorName(dsz)));
+
+ return (int) dsz;
+ }
+
+ /**
+ * Decompress data via ByteBuffers
+ *
+ * @param input
+ * @param output
+ * @throws IOException
+ */
+ @Override
+ public void uncompress(ByteBuffer input, ByteBuffer output) throws IOException
+ {
+ try
+ {
+ Zstd.decompress(output, input);
+ } catch (Exception e)
+ {
+ throw new IOException("Decompression failed", e);
+ }
+ }
+
+ /**
+ * Compress using ByteBuffers
+ *
+ * @param input
+ * @param output
+ * @throws IOException
+ */
+ @Override
+ public void compress(ByteBuffer input, ByteBuffer output) throws IOException
+ {
+ try
+ {
+ Zstd.compress(output, input, compressionLevel, ENABLE_CHECKSUM_FLAG);
+ } catch (Exception e)
+ {
+ throw new IOException("Compression failed", e);
+ }
+ }
+
+ /**
+ * Check if the given compression level is valid. This can be a negative value as well.
+ *
+ * @param level
+ * @return
+ */
+ private static boolean isValid(int level)
+ {
+ return (level >= FAST_COMPRESSION_LEVEL && level <= BEST_COMPRESSION_LEVEL);
+ }
+
+ /**
+ * Parse the compression options
+ *
+ * @param options
+ * @return
+ */
+ private static int getOrDefaultCompressionLevel(Map<String, String> options)
+ {
+ if (options == null)
+ return DEFAULT_COMPRESSION_LEVEL;
+
+ String val = options.get(COMPRESSION_LEVEL_OPTION_NAME);
+
+ if (val == null)
+ return DEFAULT_COMPRESSION_LEVEL;
+
+ return Integer.valueOf(val);
+ }
+
+ /**
+ * Return the preferred BufferType
+ *
+ * @return
+ */
+ @Override
+ public BufferType preferredBufferType()
+ {
+ return BufferType.OFF_HEAP;
+ }
+
+ /**
+ * Check whether the given BufferType is supported
+ *
+ * @param bufferType
+ * @return
+ */
+ @Override
+ public boolean supports(BufferType bufferType)
+ {
+ return bufferType == BufferType.OFF_HEAP;
+ }
+
+ /**
+ * Lists the supported options by this compressor
+ *
+ * @return
+ */
+ @Override
+ public Set<String> supportedOptions()
+ {
+ return new HashSet<>(Collections.singletonList(COMPRESSION_LEVEL_OPTION_NAME));
+ }
+
+
+ @VisibleForTesting
+ public int getCompressionLevel()
+ {
+ return compressionLevel;
+ }
+
+ @Override
+ public Set<Uses> recommendedUses()
+ {
+ return recommendedUses;
+ }
+}
diff --git a/src/java/org/apache/cassandra/io/sstable/AbstractSSTableSimpleWriter.java b/src/java/org/apache/cassandra/io/sstable/AbstractSSTableSimpleWriter.java
index 9a8f968..4eaf1fe 100644
--- a/src/java/org/apache/cassandra/io/sstable/AbstractSSTableSimpleWriter.java
+++ b/src/java/org/apache/cassandra/io/sstable/AbstractSSTableSimpleWriter.java
@@ -18,7 +18,7 @@
package org.apache.cassandra.io.sstable;
import java.io.File;
-import java.io.FilenameFilter;
+import java.io.FileFilter;
import java.io.IOException;
import java.io.Closeable;
import java.nio.ByteBuffer;
@@ -27,13 +27,12 @@
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
-import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.db.*;
-import org.apache.cassandra.db.rows.EncodingStats;
import org.apache.cassandra.db.partitions.PartitionUpdate;
+import org.apache.cassandra.db.rows.EncodingStats;
import org.apache.cassandra.io.sstable.format.SSTableFormat;
+import org.apache.cassandra.schema.TableMetadataRef;
import org.apache.cassandra.service.ActiveRepairService;
-import org.apache.cassandra.utils.Pair;
/**
* Base class for the sstable writers used by CQLSSTableWriter.
@@ -41,13 +40,13 @@
abstract class AbstractSSTableSimpleWriter implements Closeable
{
protected final File directory;
- protected final CFMetaData metadata;
- protected final PartitionColumns columns;
+ protected final TableMetadataRef metadata;
+ protected final RegularAndStaticColumns columns;
protected SSTableFormat.Type formatType = SSTableFormat.Type.current();
protected static AtomicInteger generation = new AtomicInteger(0);
protected boolean makeRangeAware = false;
- protected AbstractSSTableSimpleWriter(File directory, CFMetaData metadata, PartitionColumns columns)
+ protected AbstractSSTableSimpleWriter(File directory, TableMetadataRef metadata, RegularAndStaticColumns columns)
{
this.metadata = metadata;
this.directory = directory;
@@ -67,15 +66,17 @@
protected SSTableTxnWriter createWriter()
{
- SerializationHeader header = new SerializationHeader(true, metadata, columns, EncodingStats.NO_STATS);
+ SerializationHeader header = new SerializationHeader(true, metadata.get(), columns, EncodingStats.NO_STATS);
if (makeRangeAware)
- return SSTableTxnWriter.createRangeAware(metadata, 0, ActiveRepairService.UNREPAIRED_SSTABLE, formatType, 0, header);
+ return SSTableTxnWriter.createRangeAware(metadata, 0, ActiveRepairService.UNREPAIRED_SSTABLE, ActiveRepairService.NO_PENDING_REPAIR, false, formatType, 0, header);
return SSTableTxnWriter.create(metadata,
- createDescriptor(directory, metadata.ksName, metadata.cfName, formatType),
+ createDescriptor(directory, metadata.keyspace, metadata.name, formatType),
0,
ActiveRepairService.UNREPAIRED_SSTABLE,
+ ActiveRepairService.NO_PENDING_REPAIR,
+ false,
0,
header,
Collections.emptySet());
@@ -90,12 +91,11 @@
private static int getNextGeneration(File directory, final String columnFamily)
{
final Set<Descriptor> existing = new HashSet<>();
- directory.list(new FilenameFilter()
+ directory.listFiles(new FileFilter()
{
- public boolean accept(File dir, String name)
+ public boolean accept(File file)
{
- Pair<Descriptor, Component> p = SSTable.tryComponentFromFilename(dir, name);
- Descriptor desc = p == null ? null : p.left;
+ Descriptor desc = SSTable.tryDescriptorFromFilename(file);
if (desc == null)
return false;
@@ -116,9 +116,9 @@
return maxGen;
}
- PartitionUpdate getUpdateFor(ByteBuffer key) throws IOException
+ PartitionUpdate.Builder getUpdateFor(ByteBuffer key) throws IOException
{
- return getUpdateFor(metadata.decorateKey(key));
+ return getUpdateFor(metadata.get().partitioner.decorateKey(key));
}
/**
@@ -127,6 +127,6 @@
* @param key they partition key for which the returned update will be.
* @return an update on partition {@code key} that is tied to this writer.
*/
- abstract PartitionUpdate getUpdateFor(DecoratedKey key) throws IOException;
+ abstract PartitionUpdate.Builder getUpdateFor(DecoratedKey key) throws IOException;
}
diff --git a/src/java/org/apache/cassandra/io/sstable/CQLSSTableWriter.java b/src/java/org/apache/cassandra/io/sstable/CQLSSTableWriter.java
index 694fe37..3601ab0 100644
--- a/src/java/org/apache/cassandra/io/sstable/CQLSSTableWriter.java
+++ b/src/java/org/apache/cassandra/io/sstable/CQLSSTableWriter.java
@@ -27,44 +27,32 @@
import java.util.List;
import java.util.Map;
import java.util.SortedSet;
+import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
-import com.datastax.driver.core.ProtocolVersion;
-import com.datastax.driver.core.TypeCodec;
-import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.config.Schema;
-import org.apache.cassandra.config.SchemaConstants;
+import org.apache.cassandra.cql3.statements.schema.CreateTableStatement;
+import org.apache.cassandra.cql3.statements.schema.CreateTypeStatement;
import org.apache.cassandra.cql3.ColumnSpecification;
import org.apache.cassandra.cql3.QueryOptions;
import org.apache.cassandra.cql3.QueryProcessor;
import org.apache.cassandra.cql3.UpdateParameters;
import org.apache.cassandra.cql3.functions.UDHelper;
-import org.apache.cassandra.cql3.statements.CreateTableStatement;
-import org.apache.cassandra.cql3.statements.CreateTypeStatement;
+import org.apache.cassandra.cql3.functions.types.TypeCodec;
+import org.apache.cassandra.cql3.functions.types.UserType;
import org.apache.cassandra.cql3.statements.ModificationStatement;
-import org.apache.cassandra.cql3.statements.ParsedStatement;
import org.apache.cassandra.cql3.statements.UpdateStatement;
import org.apache.cassandra.db.Clustering;
-import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.SystemKeyspace;
-import org.apache.cassandra.db.marshal.UserType;
-import org.apache.cassandra.db.partitions.Partition;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.Murmur3Partitioner;
import org.apache.cassandra.exceptions.InvalidRequestException;
import org.apache.cassandra.exceptions.SyntaxException;
import org.apache.cassandra.io.sstable.format.SSTableFormat;
-import org.apache.cassandra.schema.Functions;
-import org.apache.cassandra.schema.KeyspaceMetadata;
-import org.apache.cassandra.schema.KeyspaceParams;
-import org.apache.cassandra.schema.SchemaKeyspace;
-import org.apache.cassandra.schema.Tables;
-import org.apache.cassandra.schema.Types;
-import org.apache.cassandra.schema.Views;
+import org.apache.cassandra.schema.*;
import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.transport.ProtocolVersion;
import org.apache.cassandra.utils.ByteBufferUtil;
-import org.apache.cassandra.utils.Pair;
/**
* Utility to write SSTables.
@@ -251,15 +239,16 @@
List<ByteBuffer> keys = insert.buildPartitionKeyNames(options);
SortedSet<Clustering> clusterings = insert.createClustering(options);
- long now = System.currentTimeMillis() * 1000;
+ long now = System.currentTimeMillis();
// Note that we asks indexes to not validate values (the last 'false' arg below) because that triggers a 'Keyspace.open'
// and that forces a lot of initialization that we don't want.
- UpdateParameters params = new UpdateParameters(insert.cfm,
+ UpdateParameters params = new UpdateParameters(insert.metadata,
insert.updatedColumns(),
options,
- insert.getTimestamp(now, options),
+ insert.getTimestamp(TimeUnit.MILLISECONDS.toMicros(now), options),
+ (int) TimeUnit.MILLISECONDS.toSeconds(now),
insert.getTimeToLive(options),
- Collections.<DecoratedKey, Partition>emptyMap());
+ Collections.emptyMap());
try
{
@@ -313,11 +302,11 @@
* @param dataType name of the User Defined type
* @return user defined type
*/
- public com.datastax.driver.core.UserType getUDType(String dataType)
+ public UserType getUDType(String dataType)
{
- KeyspaceMetadata ksm = Schema.instance.getKSMetaData(insert.keyspace());
- UserType userType = ksm.types.getNullable(ByteBufferUtil.bytes(dataType));
- return (com.datastax.driver.core.UserType) UDHelper.driverType(userType);
+ KeyspaceMetadata ksm = Schema.instance.getKeyspaceMetadata(insert.keyspace());
+ org.apache.cassandra.db.marshal.UserType userType = ksm.types.getNullable(ByteBufferUtil.bytes(dataType));
+ return (UserType) UDHelper.driverType(userType);
}
/**
@@ -336,7 +325,7 @@
if (value == null || value == UNSET_VALUE)
return (ByteBuffer) value;
- return codec.serialize(value, ProtocolVersion.NEWEST_SUPPORTED);
+ return codec.serialize(value, ProtocolVersion.CURRENT);
}
/**
* A Builder for a CQLSSTableWriter object.
@@ -347,8 +336,8 @@
protected SSTableFormat.Type formatType = null;
- private CreateTableStatement.RawStatement schemaStatement;
- private final List<CreateTypeStatement> typeStatements;
+ private CreateTableStatement.Raw schemaStatement;
+ private final List<CreateTypeStatement.Raw> typeStatements;
private ModificationStatement.Parsed insertStatement;
private IPartitioner partitioner;
@@ -397,7 +386,7 @@
public Builder withType(String typeDefinition) throws SyntaxException
{
- typeStatements.add(QueryProcessor.parseStatement(typeDefinition, CreateTypeStatement.class, "CREATE TYPE"));
+ typeStatements.add(QueryProcessor.parseStatement(typeDefinition, CreateTypeStatement.Raw.class, "CREATE TYPE"));
return this;
}
@@ -417,7 +406,7 @@
*/
public Builder forTable(String schema)
{
- this.schemaStatement = QueryProcessor.parseStatement(schema, CreateTableStatement.RawStatement.class, "CREATE TABLE");
+ this.schemaStatement = QueryProcessor.parseStatement(schema, CreateTableStatement.Raw.class, "CREATE TABLE");
return this;
}
@@ -513,16 +502,16 @@
synchronized (CQLSSTableWriter.class)
{
- if (Schema.instance.getKSMetaData(SchemaConstants.SCHEMA_KEYSPACE_NAME) == null)
+ if (Schema.instance.getKeyspaceMetadata(SchemaConstants.SCHEMA_KEYSPACE_NAME) == null)
Schema.instance.load(SchemaKeyspace.metadata());
- if (Schema.instance.getKSMetaData(SchemaConstants.SYSTEM_KEYSPACE_NAME) == null)
+ if (Schema.instance.getKeyspaceMetadata(SchemaConstants.SYSTEM_KEYSPACE_NAME) == null)
Schema.instance.load(SystemKeyspace.metadata());
- String keyspace = schemaStatement.keyspace();
+ String keyspaceName = schemaStatement.keyspace();
- if (Schema.instance.getKSMetaData(keyspace) == null)
+ if (Schema.instance.getKeyspaceMetadata(keyspaceName) == null)
{
- Schema.instance.load(KeyspaceMetadata.create(keyspace,
+ Schema.instance.load(KeyspaceMetadata.create(keyspaceName,
KeyspaceParams.simple(1),
Tables.none(),
Views.none(),
@@ -530,35 +519,34 @@
Functions.none()));
}
+ KeyspaceMetadata ksm = Schema.instance.getKeyspaceMetadata(keyspaceName);
- KeyspaceMetadata ksm = Schema.instance.getKSMetaData(keyspace);
- CFMetaData cfMetaData = ksm.tables.getNullable(schemaStatement.columnFamily());
- if (cfMetaData == null)
+ TableMetadata tableMetadata = ksm.tables.getNullable(schemaStatement.table());
+ if (tableMetadata == null)
{
- Types types = createTypes(keyspace);
- cfMetaData = createTable(types);
-
- Schema.instance.load(cfMetaData);
- Schema.instance.setKeyspaceMetadata(ksm.withSwapped(ksm.tables.with(cfMetaData)).withSwapped(types));
+ Types types = createTypes(keyspaceName);
+ tableMetadata = createTable(types);
+ Schema.instance.load(ksm.withSwapped(ksm.tables.with(tableMetadata)).withSwapped(types));
}
- Pair<UpdateStatement, List<ColumnSpecification>> preparedInsert = prepareInsert();
+ UpdateStatement preparedInsert = prepareInsert();
+ TableMetadataRef ref = TableMetadataRef.forOfflineTools(tableMetadata);
AbstractSSTableSimpleWriter writer = sorted
- ? new SSTableSimpleWriter(directory, cfMetaData, preparedInsert.left.updatedColumns())
- : new SSTableSimpleUnsortedWriter(directory, cfMetaData, preparedInsert.left.updatedColumns(), bufferSizeInMB);
+ ? new SSTableSimpleWriter(directory, ref, preparedInsert.updatedColumns())
+ : new SSTableSimpleUnsortedWriter(directory, ref, preparedInsert.updatedColumns(), bufferSizeInMB);
if (formatType != null)
writer.setSSTableFormatType(formatType);
- return new CQLSSTableWriter(writer, preparedInsert.left, preparedInsert.right);
+ return new CQLSSTableWriter(writer, preparedInsert, preparedInsert.getBindVariables());
}
}
private Types createTypes(String keyspace)
{
Types.RawBuilder builder = Types.rawBuilder(keyspace);
- for (CreateTypeStatement st : typeStatements)
+ for (CreateTypeStatement.Raw st : typeStatements)
st.addToRawBuilder(builder);
return builder.build();
}
@@ -568,17 +556,17 @@
*
* @param types types this table should be created with
*/
- private CFMetaData createTable(Types types)
+ private TableMetadata createTable(Types types)
{
- CreateTableStatement statement = (CreateTableStatement) schemaStatement.prepare(types).statement;
+ ClientState state = ClientState.forInternalCalls();
+ CreateTableStatement statement = schemaStatement.prepare(state);
statement.validate(ClientState.forInternalCalls());
- CFMetaData cfMetaData = statement.getCFMetaData();
-
+ TableMetadata.Builder builder = statement.builder(types);
if (partitioner != null)
- return cfMetaData.copy(partitioner);
- else
- return cfMetaData;
+ builder.partitioner(partitioner);
+
+ return builder.build();
}
/**
@@ -586,20 +574,20 @@
*
* @return prepared Insert statement and it's bound names
*/
- private Pair<UpdateStatement, List<ColumnSpecification>> prepareInsert()
+ private UpdateStatement prepareInsert()
{
- ParsedStatement.Prepared cqlStatement = insertStatement.prepare(ClientState.forInternalCalls());
- UpdateStatement insert = (UpdateStatement) cqlStatement.statement;
- insert.validate(ClientState.forInternalCalls());
+ ClientState state = ClientState.forInternalCalls();
+ UpdateStatement insert = (UpdateStatement) insertStatement.prepare(state);
+ insert.validate(state);
if (insert.hasConditions())
throw new IllegalArgumentException("Conditional statements are not supported");
if (insert.isCounter())
throw new IllegalArgumentException("Counter update statements are not supported");
- if (cqlStatement.boundNames.isEmpty())
+ if (insert.getBindVariables().isEmpty())
throw new IllegalArgumentException("Provided insert statement has no bind variables");
- return Pair.create(insert, cqlStatement.boundNames);
+ return insert;
}
}
}
diff --git a/src/java/org/apache/cassandra/io/sstable/Component.java b/src/java/org/apache/cassandra/io/sstable/Component.java
index 38152af..a81db85 100644
--- a/src/java/org/apache/cassandra/io/sstable/Component.java
+++ b/src/java/org/apache/cassandra/io/sstable/Component.java
@@ -17,15 +17,11 @@
*/
package org.apache.cassandra.io.sstable;
-import java.io.File;
import java.util.EnumSet;
import java.util.regex.Pattern;
import com.google.common.base.Objects;
-import org.apache.cassandra.utils.ChecksumType;
-import org.apache.cassandra.utils.Pair;
-
/**
* SSTables are made up of multiple components in separate files. Components are
* identified by a type and an id, but required unique components (such as the Data
@@ -37,6 +33,10 @@
final static EnumSet<Type> TYPES = EnumSet.allOf(Type.class);
+ /**
+ * WARNING: Be careful while changing the names or string representation of the enum
+ * members. Streaming code depends on the names during streaming (Ref: CASSANDRA-14556).
+ */
public enum Type
{
// the base data for an sstable: the remaining components can be regenerated
@@ -50,8 +50,8 @@
COMPRESSION_INFO("CompressionInfo.db"),
// statistical metadata about the content of the sstable
STATS("Statistics.db"),
- // holds adler32 checksum of the data file
- DIGEST("Digest.crc32", "Digest.adler32", "Digest.sha1"),
+ // holds CRC32 checksum of the data file
+ DIGEST("Digest.crc32"),
// holds the CRC32 for chunks in an a uncompressed file.
CRC("CRC.db"),
// holds SSTable Index Summary (sampling of Index component)
@@ -61,15 +61,11 @@
// built-in secondary index (may be multiple per sstable)
SECONDARY_INDEX("SI_.*.db"),
// custom component, used by e.g. custom compaction strategy
- CUSTOM(new String[] { null });
-
- final String[] repr;
- Type(String repr)
- {
- this(new String[] { repr });
- }
+ CUSTOM(null);
- Type(String... repr)
+ final String repr;
+
+ Type(String repr)
{
this.repr = repr;
}
@@ -78,9 +74,7 @@
{
for (Type type : TYPES)
{
- if (type.repr == null || type.repr.length == 0 || type.repr[0] == null)
- continue;
- if (Pattern.matches(type.repr[0], repr))
+ if (type.repr != null && Pattern.matches(type.repr, repr))
return type;
}
return CUSTOM;
@@ -93,36 +87,18 @@
public final static Component FILTER = new Component(Type.FILTER);
public final static Component COMPRESSION_INFO = new Component(Type.COMPRESSION_INFO);
public final static Component STATS = new Component(Type.STATS);
- private static final String digestCrc32 = "Digest.crc32";
- private static final String digestAdler32 = "Digest.adler32";
- private static final String digestSha1 = "Digest.sha1";
- public final static Component DIGEST_CRC32 = new Component(Type.DIGEST, digestCrc32);
- public final static Component DIGEST_ADLER32 = new Component(Type.DIGEST, digestAdler32);
- public final static Component DIGEST_SHA1 = new Component(Type.DIGEST, digestSha1);
+ public final static Component DIGEST = new Component(Type.DIGEST);
public final static Component CRC = new Component(Type.CRC);
public final static Component SUMMARY = new Component(Type.SUMMARY);
public final static Component TOC = new Component(Type.TOC);
- public static Component digestFor(ChecksumType checksumType)
- {
- switch (checksumType)
- {
- case Adler32:
- return DIGEST_ADLER32;
- case CRC32:
- return DIGEST_CRC32;
- }
- throw new AssertionError();
- }
-
public final Type type;
public final String name;
public final int hashCode;
public Component(Type type)
{
- this(type, type.repr[0]);
- assert type.repr.length == 1;
+ this(type, type.repr);
assert type != Type.CUSTOM;
}
@@ -143,45 +119,32 @@
}
/**
- * {@code
- * Filename of the form "<ksname>/<cfname>-[tmp-][<version>-]<gen>-<component>",
- * }
- * @return A Descriptor for the SSTable, and a Component for this particular file.
- * TODO move descriptor into Component field
+ * Parse the component part of a sstable filename into a {@code Component} object.
+ *
+ * @param name a string representing a sstable component.
+ * @return the component corresponding to {@code name}. Note that this always return a component as an unrecognized
+ * name is parsed into a CUSTOM component.
*/
- public static Pair<Descriptor,Component> fromFilename(File directory, String name)
+ public static Component parse(String name)
{
- Pair<Descriptor,String> path = Descriptor.fromFilename(directory, name);
+ Type type = Type.fromRepresentation(name);
- // parse the component suffix
- Type type = Type.fromRepresentation(path.right);
- // build (or retrieve singleton for) the component object
- Component component;
- switch(type)
+ // Build (or retrieve singleton for) the component object
+ switch (type)
{
- case DATA: component = Component.DATA; break;
- case PRIMARY_INDEX: component = Component.PRIMARY_INDEX; break;
- case FILTER: component = Component.FILTER; break;
- case COMPRESSION_INFO: component = Component.COMPRESSION_INFO; break;
- case STATS: component = Component.STATS; break;
- case DIGEST: switch (path.right)
- {
- case digestCrc32: component = Component.DIGEST_CRC32; break;
- case digestAdler32: component = Component.DIGEST_ADLER32; break;
- case digestSha1: component = Component.DIGEST_SHA1; break;
- default: throw new IllegalArgumentException("Invalid digest component " + path.right);
- }
- break;
- case CRC: component = Component.CRC; break;
- case SUMMARY: component = Component.SUMMARY; break;
- case TOC: component = Component.TOC; break;
- case SECONDARY_INDEX: component = new Component(Type.SECONDARY_INDEX, path.right); break;
- case CUSTOM: component = new Component(Type.CUSTOM, path.right); break;
- default:
- throw new IllegalStateException();
+ case DATA: return Component.DATA;
+ case PRIMARY_INDEX: return Component.PRIMARY_INDEX;
+ case FILTER: return Component.FILTER;
+ case COMPRESSION_INFO: return Component.COMPRESSION_INFO;
+ case STATS: return Component.STATS;
+ case DIGEST: return Component.DIGEST;
+ case CRC: return Component.CRC;
+ case SUMMARY: return Component.SUMMARY;
+ case TOC: return Component.TOC;
+ case SECONDARY_INDEX: return new Component(Type.SECONDARY_INDEX, name);
+ case CUSTOM: return new Component(Type.CUSTOM, name);
+ default: throw new AssertionError();
}
-
- return Pair.create(path.left, component);
}
@Override
diff --git a/src/java/org/apache/cassandra/io/sstable/Descriptor.java b/src/java/org/apache/cassandra/io/sstable/Descriptor.java
index b7155ef..fca0f8d 100644
--- a/src/java/org/apache/cassandra/io/sstable/Descriptor.java
+++ b/src/java/org/apache/cassandra/io/sstable/Descriptor.java
@@ -24,14 +24,13 @@
import java.util.regex.Pattern;
import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.CharMatcher;
import com.google.common.base.Objects;
+import com.google.common.base.Splitter;
import org.apache.cassandra.db.Directories;
import org.apache.cassandra.io.sstable.format.SSTableFormat;
import org.apache.cassandra.io.sstable.format.Version;
import org.apache.cassandra.io.sstable.metadata.IMetadataSerializer;
-import org.apache.cassandra.io.sstable.metadata.LegacyMetadataSerializer;
import org.apache.cassandra.io.sstable.metadata.MetadataSerializer;
import org.apache.cassandra.utils.Pair;
@@ -46,8 +45,13 @@
*/
public class Descriptor
{
+ private final static String LEGACY_TMP_REGEX_STR = "^((.*)\\-(.*)\\-)?tmp(link)?\\-((?:l|k).)\\-(\\d)*\\-(.*)$";
+ private final static Pattern LEGACY_TMP_REGEX = Pattern.compile(LEGACY_TMP_REGEX_STR);
+
public static String TMP_EXT = ".tmp";
+ private static final Splitter filenameSplitter = Splitter.on('-');
+
/** canonicalized path to the directory where SSTable resides */
public final File directory;
/** version has the following format: <code>[a-z]+</code> */
@@ -56,8 +60,6 @@
public final String cfname;
public final int generation;
public final SSTableFormat.Type formatType;
- /** digest component - might be {@code null} for old, legacy sstables */
- public final Component digestComponent;
private final int hashCode;
/**
@@ -66,7 +68,7 @@
@VisibleForTesting
public Descriptor(File directory, String ksname, String cfname, int generation)
{
- this(SSTableFormat.Type.current().info.getLatestVersion(), directory, ksname, cfname, generation, SSTableFormat.Type.current(), null);
+ this(SSTableFormat.Type.current().info.getLatestVersion(), directory, ksname, cfname, generation, SSTableFormat.Type.current());
}
/**
@@ -74,16 +76,10 @@
*/
public Descriptor(File directory, String ksname, String cfname, int generation, SSTableFormat.Type formatType)
{
- this(formatType.info.getLatestVersion(), directory, ksname, cfname, generation, formatType, Component.digestFor(formatType.info.getLatestVersion().uncompressedChecksumType()));
+ this(formatType.info.getLatestVersion(), directory, ksname, cfname, generation, formatType);
}
- @VisibleForTesting
- public Descriptor(String version, File directory, String ksname, String cfname, int generation, SSTableFormat.Type formatType)
- {
- this(formatType.info.getVersion(version), directory, ksname, cfname, generation, formatType, Component.digestFor(formatType.info.getLatestVersion().uncompressedChecksumType()));
- }
-
- public Descriptor(Version version, File directory, String ksname, String cfname, int generation, SSTableFormat.Type formatType, Component digestComponent)
+ public Descriptor(Version version, File directory, String ksname, String cfname, int generation, SSTableFormat.Type formatType)
{
assert version != null && directory != null && ksname != null && cfname != null && formatType.info.getLatestVersion().getClass().equals(version.getClass());
this.version = version;
@@ -99,24 +95,18 @@
this.cfname = cfname;
this.generation = generation;
this.formatType = formatType;
- this.digestComponent = digestComponent;
hashCode = Objects.hashCode(version, this.directory, generation, ksname, cfname, formatType);
}
public Descriptor withGeneration(int newGeneration)
{
- return new Descriptor(version, directory, ksname, cfname, newGeneration, formatType, digestComponent);
+ return new Descriptor(version, directory, ksname, cfname, newGeneration, formatType);
}
public Descriptor withFormatType(SSTableFormat.Type newType)
{
- return new Descriptor(newType.info.getLatestVersion(), directory, ksname, cfname, generation, newType, digestComponent);
- }
-
- public Descriptor withDigestComponent(Component newDigestComponent)
- {
- return new Descriptor(version, directory, ksname, cfname, generation, formatType, newDigestComponent);
+ return new Descriptor(newType.info.getLatestVersion(), directory, ksname, cfname, generation, newType);
}
public String tmpFilenameFor(Component component)
@@ -139,15 +129,9 @@
private void appendFileName(StringBuilder buff)
{
- if (!version.hasNewFileName())
- {
- buff.append(ksname).append(separator);
- buff.append(cfname).append(separator);
- }
buff.append(version).append(separator);
buff.append(generation);
- if (formatType != SSTableFormat.Type.LEGACY)
- buff.append(separator).append(formatType.name);
+ buff.append(separator).append(formatType.name);
}
public String relativeFilenameFor(Component component)
@@ -171,165 +155,166 @@
/** Return any temporary files found in the directory */
public List<File> getTemporaryFiles()
{
- List<File> ret = new ArrayList<>();
File[] tmpFiles = directory.listFiles((dir, name) ->
name.endsWith(Descriptor.TMP_EXT));
+ List<File> ret = new ArrayList<>(tmpFiles.length);
for (File tmpFile : tmpFiles)
ret.add(tmpFile);
return ret;
}
- /**
- * Files obsoleted by CASSANDRA-7066 : temporary files and compactions_in_progress. We support
- * versions 2.1 (ka) and 2.2 (la).
- * Temporary files have tmp- or tmplink- at the beginning for 2.2 sstables or after ks-cf- for 2.1 sstables
- */
-
- private final static String LEGACY_COMP_IN_PROG_REGEX_STR = "^compactions_in_progress(\\-[\\d,a-f]{32})?$";
- private final static Pattern LEGACY_COMP_IN_PROG_REGEX = Pattern.compile(LEGACY_COMP_IN_PROG_REGEX_STR);
- private final static String LEGACY_TMP_REGEX_STR = "^((.*)\\-(.*)\\-)?tmp(link)?\\-((?:l|k).)\\-(\\d)*\\-(.*)$";
- private final static Pattern LEGACY_TMP_REGEX = Pattern.compile(LEGACY_TMP_REGEX_STR);
-
- public static boolean isLegacyFile(File file)
+ public static boolean isValidFile(File file)
{
- if (file.isDirectory())
- return file.getParentFile() != null &&
- file.getParentFile().getName().equalsIgnoreCase("system") &&
- LEGACY_COMP_IN_PROG_REGEX.matcher(file.getName()).matches();
- else
- return LEGACY_TMP_REGEX.matcher(file.getName()).matches();
- }
-
- public static boolean isValidFile(String fileName)
- {
- return fileName.endsWith(".db") && !LEGACY_TMP_REGEX.matcher(fileName).matches();
+ String filename = file.getName();
+ return filename.endsWith(".db") && !LEGACY_TMP_REGEX.matcher(filename).matches();
}
/**
- * @see #fromFilename(File directory, String name)
- * @param filename The SSTable filename
- * @return Descriptor of the SSTable initialized from filename
+ * Parse a sstable filename into a Descriptor.
+ * <p>
+ * This is a shortcut for {@code fromFilename(new File(filename))}.
+ *
+ * @param filename the filename to a sstable component.
+ * @return the descriptor for the parsed file.
+ *
+ * @throws IllegalArgumentException if the provided {@code file} does point to a valid sstable filename. This could
+ * mean either that the filename doesn't look like a sstable file, or that it is for an old and unsupported
+ * versions.
*/
public static Descriptor fromFilename(String filename)
{
- return fromFilename(filename, false);
- }
-
- public static Descriptor fromFilename(String filename, SSTableFormat.Type formatType)
- {
- return fromFilename(filename).withFormatType(formatType);
- }
-
- public static Descriptor fromFilename(String filename, boolean skipComponent)
- {
- File file = new File(filename).getAbsoluteFile();
- return fromFilename(file.getParentFile(), file.getName(), skipComponent).left;
- }
-
- public static Pair<Descriptor, String> fromFilename(File directory, String name)
- {
- return fromFilename(directory, name, false);
+ return fromFilename(new File(filename));
}
/**
- * Filename of the form is vary by version:
+ * Parse a sstable filename into a Descriptor.
+ * <p>
+ * SSTables files are all located within subdirectories of the form {@code <keyspace>/<table>/}. Normal sstables are
+ * are directly within that subdirectory structure while 2ndary index, backups and snapshot are each inside an
+ * additional subdirectory. The file themselves have the form:
+ * {@code <version>-<gen>-<format>-<component>}.
+ * <p>
+ * Note that this method will only sucessfully parse sstable files of supported versions.
*
- * <ul>
- * <li><ksname>-<cfname>-(tmp-)?<version>-<gen>-<component> for cassandra 2.0 and before</li>
- * <li>(<tmp marker>-)?<version>-<gen>-<component> for cassandra 3.0 and later</li>
- * </ul>
+ * @param file the {@code File} object for the filename to parse.
+ * @return the descriptor for the parsed file.
*
- * If this is for SSTable of secondary index, directory should ends with index name for 2.1+.
- *
- * @param directory The directory of the SSTable files
- * @param name The name of the SSTable file
- * @param skipComponent true if the name param should not be parsed for a component tag
- *
- * @return A Descriptor for the SSTable, and the Component remainder.
+ * @throws IllegalArgumentException if the provided {@code file} does point to a valid sstable filename. This could
+ * mean either that the filename doesn't look like a sstable file, or that it is for an old and unsupported
+ * versions.
*/
- public static Pair<Descriptor, String> fromFilename(File directory, String name, boolean skipComponent)
+ public static Descriptor fromFilename(File file)
{
- File parentDirectory = directory != null ? directory : new File(".");
+ return fromFilenameWithComponent(file).left;
+ }
- // tokenize the filename
- StringTokenizer st = new StringTokenizer(name, String.valueOf(separator));
- String nexttok;
+ /**
+ * Parse a sstable filename, extracting both the {@code Descriptor} and {@code Component} part.
+ *
+ * @param file the {@code File} object for the filename to parse.
+ * @return a pair of the descriptor and component corresponding to the provided {@code file}.
+ *
+ * @throws IllegalArgumentException if the provided {@code file} does point to a valid sstable filename. This could
+ * mean either that the filename doesn't look like a sstable file, or that it is for an old and unsupported
+ * versions.
+ */
+ public static Pair<Descriptor, Component> fromFilenameWithComponent(File file)
+ {
+ // We need to extract the keyspace and table names from the parent directories, so make sure we deal with the
+ // absolute path.
+ if (!file.isAbsolute())
+ file = file.getAbsoluteFile();
- // read tokens backwards to determine version
- Deque<String> tokenStack = new ArrayDeque<>();
- while (st.hasMoreTokens())
+ String name = file.getName();
+ List<String> tokens = filenameSplitter.splitToList(name);
+ int size = tokens.size();
+
+ if (size != 4)
{
- tokenStack.push(st.nextToken());
+ // This is an invalid sstable file for this version. But to provide a more helpful error message, we detect
+ // old format sstable, which had the format:
+ // <keyspace>-<table>-(tmp-)?<version>-<gen>-<component>
+ // Note that we assume it's an old format sstable if it has the right number of tokens: this is not perfect
+ // but we're just trying to be helpful, not perfect.
+ if (size == 5 || size == 6)
+ throw new IllegalArgumentException(String.format("%s is of version %s which is now unsupported and cannot be read.",
+ name,
+ tokens.get(size - 3)));
+ throw new IllegalArgumentException(String.format("Invalid sstable file %s: the name doesn't look like a supported sstable file name", name));
}
- // component suffix
- String component = skipComponent ? null : tokenStack.pop();
+ String versionString = tokens.get(0);
+ if (!Version.validate(versionString))
+ throw invalidSSTable(name, "invalid version %s", versionString);
- nexttok = tokenStack.pop();
- // generation OR format type
- SSTableFormat.Type fmt = SSTableFormat.Type.LEGACY;
- if (!CharMatcher.DIGIT.matchesAllOf(nexttok))
+ int generation;
+ try
{
- fmt = SSTableFormat.Type.validate(nexttok);
- nexttok = tokenStack.pop();
+ generation = Integer.parseInt(tokens.get(1));
+ }
+ catch (NumberFormatException e)
+ {
+ throw invalidSSTable(name, "the 'generation' part of the name doesn't parse as a number");
}
- // generation
- int generation = Integer.parseInt(nexttok);
-
- // version
- nexttok = tokenStack.pop();
-
- if (!Version.validate(nexttok))
- throw new UnsupportedOperationException("SSTable " + name + " is too old to open. Upgrade to 2.0 first, and run upgradesstables");
-
- Version version = fmt.info.getVersion(nexttok);
-
- // ks/cf names
- String ksname, cfname;
- if (version.hasNewFileName())
+ String formatString = tokens.get(2);
+ SSTableFormat.Type format;
+ try
{
- // for 2.1+ read ks and cf names from directory
- File cfDirectory = parentDirectory;
- // check if this is secondary index
- String indexName = "";
- if (Directories.isSecondaryIndexFolder(cfDirectory))
- {
- indexName = cfDirectory.getName();
- cfDirectory = cfDirectory.getParentFile();
- }
- if (cfDirectory.getName().equals(Directories.BACKUPS_SUBDIR))
- {
- cfDirectory = cfDirectory.getParentFile();
- }
- else if (cfDirectory.getParentFile().getName().equals(Directories.SNAPSHOT_SUBDIR))
- {
- cfDirectory = cfDirectory.getParentFile().getParentFile();
- }
- cfname = cfDirectory.getName().split("-")[0] + indexName;
- ksname = cfDirectory.getParentFile().getName();
+ format = SSTableFormat.Type.validate(formatString);
}
- else
+ catch (IllegalArgumentException e)
{
- cfname = tokenStack.pop();
- ksname = tokenStack.pop();
+ throw invalidSSTable(name, "unknown 'format' part (%s)", formatString);
}
- assert tokenStack.isEmpty() : "Invalid file name " + name + " in " + directory;
- return Pair.create(new Descriptor(version, parentDirectory, ksname, cfname, generation, fmt,
- // _assume_ version from version
- Component.digestFor(version.uncompressedChecksumType())),
- component);
+ Component component = Component.parse(tokens.get(3));
+
+ Version version = format.info.getVersion(versionString);
+ if (!version.isCompatible())
+ throw invalidSSTable(name, "incompatible sstable version (%s); you should have run upgradesstables before upgrading", versionString);
+
+ File directory = parentOf(name, file);
+ File tableDir = directory;
+
+ // Check if it's a 2ndary index directory (not that it doesn't exclude it to be also a backup or snapshot)
+ String indexName = "";
+ if (Directories.isSecondaryIndexFolder(tableDir))
+ {
+ indexName = tableDir.getName();
+ tableDir = parentOf(name, tableDir);
+ }
+
+ // Then it can be a backup or a snapshot
+ if (tableDir.getName().equals(Directories.BACKUPS_SUBDIR))
+ tableDir = tableDir.getParentFile();
+ else if (parentOf(name, tableDir).getName().equals(Directories.SNAPSHOT_SUBDIR))
+ tableDir = parentOf(name, parentOf(name, tableDir));
+
+ String table = tableDir.getName().split("-")[0] + indexName;
+ String keyspace = parentOf(name, tableDir).getName();
+
+ return Pair.create(new Descriptor(version, directory, keyspace, table, generation, format), component);
+ }
+
+ private static File parentOf(String name, File file)
+ {
+ File parent = file.getParentFile();
+ if (parent == null)
+ throw invalidSSTable(name, "cannot extract keyspace and table name; make sure the sstable is in the proper sub-directories");
+ return parent;
+ }
+
+ private static IllegalArgumentException invalidSSTable(String name, String msgFormat, Object... parameters)
+ {
+ throw new IllegalArgumentException(String.format("Invalid sstable file " + name + ": " + msgFormat, parameters));
}
public IMetadataSerializer getMetadataSerializer()
{
- if (version.hasNewStatsFile())
- return new MetadataSerializer();
- else
- return new LegacyMetadataSerializer();
+ return new MetadataSerializer();
}
/**
diff --git a/src/java/org/apache/cassandra/io/sstable/ISSTableScanner.java b/src/java/org/apache/cassandra/io/sstable/ISSTableScanner.java
index 2dff34e..af661b7 100644
--- a/src/java/org/apache/cassandra/io/sstable/ISSTableScanner.java
+++ b/src/java/org/apache/cassandra/io/sstable/ISSTableScanner.java
@@ -19,7 +19,14 @@
package org.apache.cassandra.io.sstable;
+import java.util.Collection;
+import java.util.Set;
+
+import com.google.common.base.Throwables;
+
import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.utils.JVMStabilityInspector;
/**
* An ISSTableScanner is an abstraction allowing multiple SSTableScanners to be
@@ -31,5 +38,34 @@
public long getCompressedLengthInBytes();
public long getCurrentPosition();
public long getBytesScanned();
- public String getBackingFiles();
+ public Set<SSTableReader> getBackingSSTables();
+
+ public static void closeAllAndPropagate(Collection<ISSTableScanner> scanners, Throwable throwable)
+ {
+ for (ISSTableScanner scanner: scanners)
+ {
+ try
+ {
+ scanner.close();
+ }
+ catch (Throwable t2)
+ {
+ JVMStabilityInspector.inspectThrowable(t2);
+ if (throwable == null)
+ {
+ throwable = t2;
+ }
+ else
+ {
+ throwable.addSuppressed(t2);
+ }
+ }
+ }
+
+ if (throwable != null)
+ {
+ Throwables.propagate(throwable);
+ }
+
+ }
}
diff --git a/src/java/org/apache/cassandra/io/sstable/IndexInfo.java b/src/java/org/apache/cassandra/io/sstable/IndexInfo.java
index 9ee1996..03246c5 100644
--- a/src/java/org/apache/cassandra/io/sstable/IndexInfo.java
+++ b/src/java/org/apache/cassandra/io/sstable/IndexInfo.java
@@ -19,11 +19,14 @@
package org.apache.cassandra.io.sstable;
import java.io.IOException;
+import java.util.List;
import org.apache.cassandra.db.ClusteringPrefix;
import org.apache.cassandra.db.DeletionTime;
import org.apache.cassandra.db.RowIndexEntry;
+import org.apache.cassandra.db.SerializationHeader;
import org.apache.cassandra.db.TypeSizes;
+import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.io.ISerializer;
import org.apache.cassandra.io.sstable.format.Version;
import org.apache.cassandra.io.util.DataInputPlus;
@@ -79,6 +82,11 @@
this.endOpenMarker = endOpenMarker;
}
+ public static IndexInfo.Serializer serializer(Version version, SerializationHeader header)
+ {
+ return new IndexInfo.Serializer(version, header.clusteringTypes());
+ }
+
public static class Serializer implements ISerializer<IndexInfo>
{
// This is the default index size that we use to delta-encode width when serializing so we get better vint-encoding.
@@ -87,21 +95,19 @@
// size so using the default is almost surely better than using no base at all.
public static final long WIDTH_BASE = 64 * 1024;
- private final ISerializer<ClusteringPrefix> clusteringSerializer;
- private final Version version;
+ private final int version;
+ private final List<AbstractType<?>> clusteringTypes;
- public Serializer(Version version, ISerializer<ClusteringPrefix> clusteringSerializer)
+ public Serializer(Version version, List<AbstractType<?>> clusteringTypes)
{
- this.clusteringSerializer = clusteringSerializer;
- this.version = version;
+ this.version = version.correspondingMessagingVersion();
+ this.clusteringTypes = clusteringTypes;
}
public void serialize(IndexInfo info, DataOutputPlus out) throws IOException
{
- assert version.storeRows() : "We read old index files but we should never write them";
-
- clusteringSerializer.serialize(info.firstName, out);
- clusteringSerializer.serialize(info.lastName, out);
+ ClusteringPrefix.serializer.serialize(info.firstName, out, version, clusteringTypes);
+ ClusteringPrefix.serializer.serialize(info.lastName, out, version, clusteringTypes);
out.writeUnsignedVInt(info.offset);
out.writeVInt(info.width - WIDTH_BASE);
@@ -112,53 +118,33 @@
public void skip(DataInputPlus in) throws IOException
{
- clusteringSerializer.skip(in);
- clusteringSerializer.skip(in);
- if (version.storeRows())
- {
- in.readUnsignedVInt();
- in.readVInt();
- if (in.readBoolean())
- DeletionTime.serializer.skip(in);
- }
- else
- {
- in.skipBytes(TypeSizes.sizeof(0L));
- in.skipBytes(TypeSizes.sizeof(0L));
- }
+ ClusteringPrefix.serializer.skip(in, version, clusteringTypes);
+ ClusteringPrefix.serializer.skip(in, version, clusteringTypes);
+ in.readUnsignedVInt();
+ in.readVInt();
+ if (in.readBoolean())
+ DeletionTime.serializer.skip(in);
}
public IndexInfo deserialize(DataInputPlus in) throws IOException
{
- ClusteringPrefix firstName = clusteringSerializer.deserialize(in);
- ClusteringPrefix lastName = clusteringSerializer.deserialize(in);
- long offset;
- long width;
+ ClusteringPrefix firstName = ClusteringPrefix.serializer.deserialize(in, version, clusteringTypes);
+ ClusteringPrefix lastName = ClusteringPrefix.serializer.deserialize(in, version, clusteringTypes);
+ long offset = in.readUnsignedVInt();
+ long width = in.readVInt() + WIDTH_BASE;
DeletionTime endOpenMarker = null;
- if (version.storeRows())
- {
- offset = in.readUnsignedVInt();
- width = in.readVInt() + WIDTH_BASE;
- if (in.readBoolean())
- endOpenMarker = DeletionTime.serializer.deserialize(in);
- }
- else
- {
- offset = in.readLong();
- width = in.readLong();
- }
+ if (in.readBoolean())
+ endOpenMarker = DeletionTime.serializer.deserialize(in);
return new IndexInfo(firstName, lastName, offset, width, endOpenMarker);
}
public long serializedSize(IndexInfo info)
{
- assert version.storeRows() : "We read old index files but we should never write them";
-
- long size = clusteringSerializer.serializedSize(info.firstName)
- + clusteringSerializer.serializedSize(info.lastName)
- + TypeSizes.sizeofUnsignedVInt(info.offset)
- + TypeSizes.sizeofVInt(info.width - WIDTH_BASE)
- + TypeSizes.sizeof(info.endOpenMarker != null);
+ long size = ClusteringPrefix.serializer.serializedSize(info.firstName, version, clusteringTypes)
+ + ClusteringPrefix.serializer.serializedSize(info.lastName, version, clusteringTypes)
+ + TypeSizes.sizeofUnsignedVInt(info.offset)
+ + TypeSizes.sizeofVInt(info.width - WIDTH_BASE)
+ + TypeSizes.sizeof(info.endOpenMarker != null);
if (info.endOpenMarker != null)
size += DeletionTime.serializer.serializedSize(info.endOpenMarker);
diff --git a/src/java/org/apache/cassandra/io/sstable/IndexSummary.java b/src/java/org/apache/cassandra/io/sstable/IndexSummary.java
index 6de3478..303adfd 100644
--- a/src/java/org/apache/cassandra/io/sstable/IndexSummary.java
+++ b/src/java/org/apache/cassandra/io/sstable/IndexSummary.java
@@ -268,16 +268,13 @@
public static class IndexSummarySerializer
{
- public void serialize(IndexSummary t, DataOutputPlus out, boolean withSamplingLevel) throws IOException
+ public void serialize(IndexSummary t, DataOutputPlus out) throws IOException
{
out.writeInt(t.minIndexInterval);
out.writeInt(t.offsetCount);
out.writeLong(t.getOffHeapSize());
- if (withSamplingLevel)
- {
- out.writeInt(t.samplingLevel);
- out.writeInt(t.sizeAtFullSampling);
- }
+ out.writeInt(t.samplingLevel);
+ out.writeInt(t.sizeAtFullSampling);
// our on-disk representation treats the offsets and the summary data as one contiguous structure,
// in which the offsets are based from the start of the structure. i.e., if the offsets occupy
// X bytes, the value of the first offset will be X. In memory we split the two regions up, so that
@@ -297,7 +294,7 @@
}
@SuppressWarnings("resource")
- public IndexSummary deserialize(DataInputStream in, IPartitioner partitioner, boolean haveSamplingLevel, int expectedMinIndexInterval, int maxIndexInterval) throws IOException
+ public IndexSummary deserialize(DataInputStream in, IPartitioner partitioner, int expectedMinIndexInterval, int maxIndexInterval) throws IOException
{
int minIndexInterval = in.readInt();
if (minIndexInterval != expectedMinIndexInterval)
@@ -308,17 +305,8 @@
int offsetCount = in.readInt();
long offheapSize = in.readLong();
- int samplingLevel, fullSamplingSummarySize;
- if (haveSamplingLevel)
- {
- samplingLevel = in.readInt();
- fullSamplingSummarySize = in.readInt();
- }
- else
- {
- samplingLevel = BASE_SAMPLING_LEVEL;
- fullSamplingSummarySize = offsetCount;
- }
+ int samplingLevel = in.readInt();
+ int fullSamplingSummarySize = in.readInt();
int effectiveIndexInterval = (int) Math.ceil((BASE_SAMPLING_LEVEL / (double) samplingLevel) * minIndexInterval);
if (effectiveIndexInterval > maxIndexInterval)
@@ -355,13 +343,12 @@
*
* Only for use by offline tools like SSTableMetadataViewer, otherwise SSTable.first/last should be used.
*/
- public Pair<DecoratedKey, DecoratedKey> deserializeFirstLastKey(DataInputStream in, IPartitioner partitioner, boolean haveSamplingLevel) throws IOException
+ public Pair<DecoratedKey, DecoratedKey> deserializeFirstLastKey(DataInputStream in, IPartitioner partitioner) throws IOException
{
in.skipBytes(4); // minIndexInterval
int offsetCount = in.readInt();
long offheapSize = in.readLong();
- if (haveSamplingLevel)
- in.skipBytes(8); // samplingLevel, fullSamplingSummarySize
+ in.skipBytes(8); // samplingLevel, fullSamplingSummarySize
in.skip(offsetCount * 4);
in.skip(offheapSize - offsetCount * 4);
diff --git a/src/java/org/apache/cassandra/io/sstable/IndexSummaryManager.java b/src/java/org/apache/cassandra/io/sstable/IndexSummaryManager.java
index dea1cd6..880b738 100644
--- a/src/java/org/apache/cassandra/io/sstable/IndexSummaryManager.java
+++ b/src/java/org/apache/cassandra/io/sstable/IndexSummaryManager.java
@@ -23,7 +23,6 @@
import java.util.List;
import java.util.Map;
import java.util.Set;
-import java.util.UUID;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
@@ -45,6 +44,7 @@
import org.apache.cassandra.db.lifecycle.SSTableSet;
import org.apache.cassandra.db.lifecycle.View;
import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.utils.ExecutorUtils;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.MBeanWrapper;
@@ -193,14 +193,17 @@
}
/**
- * Returns a Pair of all compacting and non-compacting sstables. Non-compacting sstables will be marked as
- * compacting.
+ * Marks the non-compacting sstables as compacting for index summary redistribution for all keyspaces/tables.
+ *
+ * @return Pair containing:
+ * left: total size of the off heap index summaries for the sstables we were unable to mark compacting (they were involved in other compactions)
+ * right: the transactions, keyed by table id.
*/
@SuppressWarnings("resource")
- private Pair<List<SSTableReader>, Map<UUID, LifecycleTransaction>> getCompactingAndNonCompactingSSTables()
+ private Pair<Long, Map<TableId, LifecycleTransaction>> getRestributionTransactions()
{
List<SSTableReader> allCompacting = new ArrayList<>();
- Map<UUID, LifecycleTransaction> allNonCompacting = new HashMap<>();
+ Map<TableId, LifecycleTransaction> allNonCompacting = new HashMap<>();
for (Keyspace ks : Keyspace.all())
{
for (ColumnFamilyStore cfStore: ks.getColumnFamilyStores())
@@ -215,22 +218,25 @@
}
while (null == (txn = cfStore.getTracker().tryModify(nonCompacting, OperationType.UNKNOWN)));
- allNonCompacting.put(cfStore.metadata.cfId, txn);
+ allNonCompacting.put(cfStore.metadata.id, txn);
allCompacting.addAll(Sets.difference(allSSTables, nonCompacting));
}
}
- return Pair.create(allCompacting, allNonCompacting);
+ long nonRedistributingOffHeapSize = allCompacting.stream().mapToLong(SSTableReader::getIndexSummaryOffHeapSize).sum();
+ return Pair.create(nonRedistributingOffHeapSize, allNonCompacting);
}
public void redistributeSummaries() throws IOException
{
if (CompactionManager.instance.isGlobalCompactionPaused())
return;
- Pair<List<SSTableReader>, Map<UUID, LifecycleTransaction>> compactingAndNonCompacting = getCompactingAndNonCompactingSSTables();
+ Pair<Long, Map<TableId, LifecycleTransaction>> redistributionTransactionInfo = getRestributionTransactions();
+ Map<TableId, LifecycleTransaction> transactions = redistributionTransactionInfo.right;
+ long nonRedistributingOffHeapSize = redistributionTransactionInfo.left;
try
{
- redistributeSummaries(new IndexSummaryRedistribution(compactingAndNonCompacting.left,
- compactingAndNonCompacting.right,
+ redistributeSummaries(new IndexSummaryRedistribution(transactions,
+ nonRedistributingOffHeapSize,
this.memoryPoolBytes));
}
catch (Exception e)
@@ -243,7 +249,7 @@
{
try
{
- FBUtilities.closeAll(compactingAndNonCompacting.right.values());
+ FBUtilities.closeAll(transactions.values());
}
catch (Exception e)
{
diff --git a/src/java/org/apache/cassandra/io/sstable/IndexSummaryRedistribution.java b/src/java/org/apache/cassandra/io/sstable/IndexSummaryRedistribution.java
index b914963..90a8621 100644
--- a/src/java/org/apache/cassandra/io/sstable/IndexSummaryRedistribution.java
+++ b/src/java/org/apache/cassandra/io/sstable/IndexSummaryRedistribution.java
@@ -28,8 +28,6 @@
import java.util.UUID;
import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Iterables;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -42,6 +40,7 @@
import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.metrics.StorageMetrics;
+import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.Pair;
import org.apache.cassandra.utils.concurrent.Refs;
@@ -59,16 +58,22 @@
static final double UPSAMPLE_THRESHOLD = 1.5;
static final double DOWNSAMPLE_THESHOLD = 0.75;
- private final List<SSTableReader> compacting;
- private final Map<UUID, LifecycleTransaction> transactions;
+ private final Map<TableId, LifecycleTransaction> transactions;
+ private final long nonRedistributingOffHeapSize;
private final long memoryPoolBytes;
private final UUID compactionId;
private volatile long remainingSpace;
- public IndexSummaryRedistribution(List<SSTableReader> compacting, Map<UUID, LifecycleTransaction> transactions, long memoryPoolBytes)
+ /**
+ *
+ * @param transactions the transactions for the different keyspaces/tables we are to redistribute
+ * @param nonRedistributingOffHeapSize the total index summary off heap size for all sstables we were not able to mark compacting (due to them being involved in other compactions)
+ * @param memoryPoolBytes size of the memory pool
+ */
+ public IndexSummaryRedistribution(Map<TableId, LifecycleTransaction> transactions, long nonRedistributingOffHeapSize, long memoryPoolBytes)
{
- this.compacting = compacting;
this.transactions = transactions;
+ this.nonRedistributingOffHeapSize = nonRedistributingOffHeapSize;
this.memoryPoolBytes = memoryPoolBytes;
this.compactionId = UUID.randomUUID();
}
@@ -76,26 +81,14 @@
public List<SSTableReader> redistributeSummaries() throws IOException
{
logger.info("Redistributing index summaries");
- List<SSTableReader> oldFormatSSTables = new ArrayList<>();
List<SSTableReader> redistribute = new ArrayList<>();
for (LifecycleTransaction txn : transactions.values())
{
- for (SSTableReader sstable : ImmutableList.copyOf(txn.originals()))
- {
- // We can't change the sampling level of sstables with the old format, because the serialization format
- // doesn't include the sampling level. Leave this one as it is. (See CASSANDRA-8993 for details.)
- logger.trace("SSTable {} cannot be re-sampled due to old sstable format", sstable);
- if (!sstable.descriptor.version.hasSamplingLevel())
- {
- oldFormatSSTables.add(sstable);
- txn.cancel(sstable);
- }
- }
redistribute.addAll(txn.originals());
}
- long total = 0;
- for (SSTableReader sstable : Iterables.concat(compacting, redistribute))
+ long total = nonRedistributingOffHeapSize;
+ for (SSTableReader sstable : redistribute)
total += sstable.getIndexSummaryOffHeapSize();
logger.trace("Beginning redistribution of index summaries for {} sstables with memory pool size {} MB; current spaced used is {} MB",
@@ -121,9 +114,7 @@
List<SSTableReader> sstablesByHotness = new ArrayList<>(redistribute);
Collections.sort(sstablesByHotness, new ReadRateComparator(readRates));
- long remainingBytes = memoryPoolBytes;
- for (SSTableReader sstable : Iterables.concat(compacting, oldFormatSSTables))
- remainingBytes -= sstable.getIndexSummaryOffHeapSize();
+ long remainingBytes = memoryPoolBytes - nonRedistributingOffHeapSize;
logger.trace("Index summaries for compacting SSTables are using {} MB of space",
(memoryPoolBytes - remainingBytes) / 1024.0 / 1024.0);
@@ -135,17 +126,18 @@
for (LifecycleTransaction txn : transactions.values())
txn.finish();
}
- total = 0;
- for (SSTableReader sstable : Iterables.concat(compacting, oldFormatSSTables, newSSTables))
+ total = nonRedistributingOffHeapSize;
+ for (SSTableReader sstable : newSSTables)
total += sstable.getIndexSummaryOffHeapSize();
- logger.trace("Completed resizing of index summaries; current approximate memory used: {}",
+ if (logger.isTraceEnabled())
+ logger.trace("Completed resizing of index summaries; current approximate memory used: {}",
FBUtilities.prettyPrintMemory(total));
return newSSTables;
}
private List<SSTableReader> adjustSamplingLevels(List<SSTableReader> sstables,
- Map<UUID, LifecycleTransaction> transactions,
+ Map<TableId, LifecycleTransaction> transactions,
double totalReadsPerSec, long memoryPoolCapacity) throws IOException
{
List<ResampleEntry> toDownsample = new ArrayList<>(sstables.size() / 4);
@@ -162,8 +154,8 @@
if (isStopRequested())
throw new CompactionInterruptedException(getCompactionInfo());
- int minIndexInterval = sstable.metadata.params.minIndexInterval;
- int maxIndexInterval = sstable.metadata.params.maxIndexInterval;
+ int minIndexInterval = sstable.metadata().params.minIndexInterval;
+ int maxIndexInterval = sstable.metadata().params.maxIndexInterval;
double readsPerSec = sstable.getReadMeter() == null ? 0.0 : sstable.getReadMeter().fifteenMinuteRate();
long idealSpace = Math.round(remainingSpace * (readsPerSec / totalReadsPerSec));
@@ -190,12 +182,13 @@
int numEntriesAtNewSamplingLevel = IndexSummaryBuilder.entriesAtSamplingLevel(newSamplingLevel, maxSummarySize);
double effectiveIndexInterval = sstable.getEffectiveIndexInterval();
- logger.trace("{} has {} reads/sec; ideal space for index summary: {} ({} entries); considering moving " +
- "from level {} ({} entries, {}) " +
- "to level {} ({} entries, {})",
- sstable.getFilename(), readsPerSec, FBUtilities.prettyPrintMemory(idealSpace), targetNumEntries,
- currentSamplingLevel, currentNumEntries, FBUtilities.prettyPrintMemory((long) (currentNumEntries * avgEntrySize)),
- newSamplingLevel, numEntriesAtNewSamplingLevel, FBUtilities.prettyPrintMemory((long) (numEntriesAtNewSamplingLevel * avgEntrySize)));
+ if (logger.isTraceEnabled())
+ logger.trace("{} has {} reads/sec; ideal space for index summary: {} ({} entries); considering moving " +
+ "from level {} ({} entries, {}) " +
+ "to level {} ({} entries, {})",
+ sstable.getFilename(), readsPerSec, FBUtilities.prettyPrintMemory(idealSpace), targetNumEntries,
+ currentSamplingLevel, currentNumEntries, FBUtilities.prettyPrintMemory((long) (currentNumEntries * avgEntrySize)),
+ newSamplingLevel, numEntriesAtNewSamplingLevel, FBUtilities.prettyPrintMemory((long) (numEntriesAtNewSamplingLevel * avgEntrySize)));
if (effectiveIndexInterval < minIndexInterval)
{
@@ -235,7 +228,7 @@
logger.trace("SSTable {} is within thresholds of ideal sampling", sstable);
remainingSpace -= sstable.getIndexSummaryOffHeapSize();
newSSTables.add(sstable);
- transactions.get(sstable.metadata.cfId).cancel(sstable);
+ transactions.get(sstable.metadata().id).cancel(sstable);
}
totalReadsPerSec -= readsPerSec;
}
@@ -246,7 +239,7 @@
toDownsample = result.right;
newSSTables.addAll(result.left);
for (SSTableReader sstable : result.left)
- transactions.get(sstable.metadata.cfId).cancel(sstable);
+ transactions.get(sstable.metadata().id).cancel(sstable);
}
// downsample first, then upsample
@@ -262,12 +255,12 @@
logger.trace("Re-sampling index summary for {} from {}/{} to {}/{} of the original number of entries",
sstable, sstable.getIndexSummarySamplingLevel(), Downsampling.BASE_SAMPLING_LEVEL,
entry.newSamplingLevel, Downsampling.BASE_SAMPLING_LEVEL);
- ColumnFamilyStore cfs = Keyspace.open(sstable.metadata.ksName).getColumnFamilyStore(sstable.metadata.cfId);
+ ColumnFamilyStore cfs = Keyspace.open(sstable.metadata().keyspace).getColumnFamilyStore(sstable.metadata().id);
long oldSize = sstable.bytesOnDisk();
SSTableReader replacement = sstable.cloneWithNewSummarySamplingLevel(cfs, entry.newSamplingLevel);
long newSize = replacement.bytesOnDisk();
newSSTables.add(replacement);
- transactions.get(sstable.metadata.cfId).update(replacement, true);
+ transactions.get(sstable.metadata().id).update(replacement, true);
addHooks(cfs, transactions, oldSize, newSize);
}
@@ -278,9 +271,9 @@
* Add hooks to correctly update the storage load metrics once the transaction is closed/aborted
*/
@SuppressWarnings("resource") // Transactions are closed in finally outside of this method
- private void addHooks(ColumnFamilyStore cfs, Map<UUID, LifecycleTransaction> transactions, long oldSize, long newSize)
+ private void addHooks(ColumnFamilyStore cfs, Map<TableId, LifecycleTransaction> transactions, long oldSize, long newSize)
{
- LifecycleTransaction txn = transactions.get(cfs.metadata.cfId);
+ LifecycleTransaction txn = transactions.get(cfs.metadata.id);
txn.runOnCommit(() -> {
// The new size will be added in Transactional.commit() as an updated SSTable, more details: CASSANDRA-13738
StorageMetrics.load.dec(oldSize);
@@ -337,7 +330,7 @@
public CompactionInfo getCompactionInfo()
{
- return new CompactionInfo(OperationType.INDEX_SUMMARY, (memoryPoolBytes - remainingSpace), memoryPoolBytes, Unit.BYTES, compactionId);
+ return CompactionInfo.withoutSSTables(null, OperationType.INDEX_SUMMARY, (memoryPoolBytes - remainingSpace), memoryPoolBytes, Unit.BYTES, compactionId);
}
public boolean isGlobal()
diff --git a/src/java/org/apache/cassandra/io/sstable/KeyIterator.java b/src/java/org/apache/cassandra/io/sstable/KeyIterator.java
index d51e97b..091e969 100644
--- a/src/java/org/apache/cassandra/io/sstable/KeyIterator.java
+++ b/src/java/org/apache/cassandra/io/sstable/KeyIterator.java
@@ -20,14 +20,13 @@
import java.io.File;
import java.io.IOException;
-import org.apache.cassandra.utils.AbstractIterator;
-
-import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.RowIndexEntry;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.RandomAccessReader;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.utils.AbstractIterator;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.CloseableIterator;
@@ -86,7 +85,7 @@
private long keyPosition;
- public KeyIterator(Descriptor desc, CFMetaData metadata)
+ public KeyIterator(Descriptor desc, TableMetadata metadata)
{
this.desc = desc;
in = new In(new File(desc.filenameFor(Component.PRIMARY_INDEX)));
diff --git a/src/java/org/apache/cassandra/io/sstable/ReducingKeyIterator.java b/src/java/org/apache/cassandra/io/sstable/ReducingKeyIterator.java
index 6f395f8..e64d95d 100644
--- a/src/java/org/apache/cassandra/io/sstable/ReducingKeyIterator.java
+++ b/src/java/org/apache/cassandra/io/sstable/ReducingKeyIterator.java
@@ -39,7 +39,7 @@
{
iters = new ArrayList<>(sstables.size());
for (SSTableReader sstable : sstables)
- iters.add(new KeyIterator(sstable.descriptor, sstable.metadata));
+ iters.add(new KeyIterator(sstable.descriptor, sstable.metadata()));
}
private void maybeInit()
diff --git a/src/java/org/apache/cassandra/io/sstable/SSTable.java b/src/java/org/apache/cassandra/io/sstable/SSTable.java
index 0382a40..348d7f5 100644
--- a/src/java/org/apache/cassandra/io/sstable/SSTable.java
+++ b/src/java/org/apache/cassandra/io/sstable/SSTable.java
@@ -23,6 +23,7 @@
import java.util.*;
import java.util.concurrent.CopyOnWriteArraySet;
+import com.google.common.base.Preconditions;
import com.google.common.base.Predicates;
import com.google.common.collect.Collections2;
import com.google.common.collect.Sets;
@@ -30,18 +31,24 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.db.BufferDecoratedKey;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.RowIndexEntry;
+import org.apache.cassandra.dht.AbstractBounds;
import org.apache.cassandra.dht.IPartitioner;
+import org.apache.cassandra.dht.Token;
import org.apache.cassandra.io.FSWriteError;
import org.apache.cassandra.io.util.DiskOptimizationStrategy;
import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.io.util.RandomAccessReader;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.schema.TableMetadataRef;
import org.apache.cassandra.utils.ByteBufferUtil;
-import org.apache.cassandra.utils.memory.HeapAllocator;
import org.apache.cassandra.utils.Pair;
+import org.apache.cassandra.utils.memory.HeapAllocator;
+
+import static org.apache.cassandra.service.ActiveRepairService.NO_PENDING_REPAIR;
+import static org.apache.cassandra.service.ActiveRepairService.UNREPAIRED_SSTABLE;
/**
* This class is built on top of the SequenceFile. It stores
@@ -65,21 +72,20 @@
public final Descriptor descriptor;
protected final Set<Component> components;
- public final CFMetaData metadata;
public final boolean compression;
public DecoratedKey first;
public DecoratedKey last;
protected final DiskOptimizationStrategy optimizationStrategy;
+ protected final TableMetadataRef metadata;
- protected SSTable(Descriptor descriptor, Set<Component> components, CFMetaData metadata, DiskOptimizationStrategy optimizationStrategy)
+ protected SSTable(Descriptor descriptor, Set<Component> components, TableMetadataRef metadata, DiskOptimizationStrategy optimizationStrategy)
{
// In almost all cases, metadata shouldn't be null, but allowing null allows to create a mostly functional SSTable without
// full schema definition. SSTableLoader use that ability
assert descriptor != null;
assert components != null;
- assert metadata != null;
this.descriptor = descriptor;
Set<Component> dataComponents = new HashSet<>(components);
@@ -102,7 +108,7 @@
*/
public static boolean delete(Descriptor desc, Set<Component> components)
{
- logger.debug("Deleting sstable: {}", desc);
+ logger.info("Deleting sstable: {}", desc);
// remove the DATA component first if it exists
if (components.contains(Component.DATA))
FileUtils.deleteWithConfirm(desc.filenameFor(Component.DATA));
@@ -120,9 +126,14 @@
return true;
}
+ public TableMetadata metadata()
+ {
+ return metadata.get();
+ }
+
public IPartitioner getPartitioner()
{
- return metadata.partitioner;
+ return metadata().partitioner;
}
public DecoratedKey decorateKey(ByteBuffer key)
@@ -163,21 +174,47 @@
public List<String> getAllFilePaths()
{
- List<String> ret = new ArrayList<>();
+ List<String> ret = new ArrayList<>(components.size());
for (Component component : components)
ret.add(descriptor.filenameFor(component));
return ret;
}
/**
- * @return Descriptor and Component pair. null if given file is not acceptable as SSTable component.
- * If component is of unknown type, returns CUSTOM component.
+ * Parse a sstable filename into both a {@link Descriptor} and {@code Component} object.
+ *
+ * @param file the filename to parse.
+ * @return a pair of the {@code Descriptor} and {@code Component} corresponding to {@code file} if it corresponds to
+ * a valid and supported sstable filename, {@code null} otherwise. Note that components of an unknown type will be
+ * returned as CUSTOM ones.
*/
- public static Pair<Descriptor, Component> tryComponentFromFilename(File dir, String name)
+ public static Pair<Descriptor, Component> tryComponentFromFilename(File file)
{
try
{
- return Component.fromFilename(dir, name);
+ return Descriptor.fromFilenameWithComponent(file);
+ }
+ catch (Throwable e)
+ {
+ return null;
+ }
+ }
+
+ /**
+ * Parse a sstable filename into a {@link Descriptor} object.
+ * <p>
+ * Note that this method ignores the component part of the filename; if this is not what you want, use
+ * {@link #tryComponentFromFilename} instead.
+ *
+ * @param file the filename to parse.
+ * @return the {@code Descriptor} corresponding to {@code file} if it corresponds to a valid and supported sstable
+ * filename, {@code null} otherwise.
+ */
+ public static Descriptor tryDescriptorFromFilename(File file)
+ {
+ try
+ {
+ return Descriptor.fromFilename(file);
}
catch (Throwable e)
{
@@ -220,17 +257,9 @@
Set<Component> components = Sets.newHashSetWithExpectedSize(knownTypes.size());
for (Component.Type componentType : knownTypes)
{
- if (componentType == Component.Type.DIGEST)
- {
- if (desc.digestComponent != null && new File(desc.filenameFor(desc.digestComponent)).exists())
- components.add(desc.digestComponent);
- }
- else
- {
- Component component = new Component(componentType);
- if (new File(desc.filenameFor(component)).exists())
- components.add(component);
- }
+ Component component = new Component(componentType);
+ if (new File(desc.filenameFor(component)).exists())
+ components.add(component);
}
return components;
}
@@ -320,4 +349,18 @@
appendTOC(descriptor, componentsToAdd);
components.addAll(componentsToAdd);
}
+
+ public AbstractBounds<Token> getBounds()
+ {
+ return AbstractBounds.bounds(first.getToken(), true, last.getToken(), true);
+ }
+
+ public static void validateRepairedMetadata(long repairedAt, UUID pendingRepair, boolean isTransient)
+ {
+ Preconditions.checkArgument((pendingRepair == NO_PENDING_REPAIR) || (repairedAt == UNREPAIRED_SSTABLE),
+ "pendingRepair cannot be set on a repaired sstable");
+ Preconditions.checkArgument(!isTransient || (pendingRepair != NO_PENDING_REPAIR),
+ "isTransient can only be true for sstables pending repair");
+
+ }
}
diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableHeaderFix.java b/src/java/org/apache/cassandra/io/sstable/SSTableHeaderFix.java
index 5dad2b9..3577259 100644
--- a/src/java/org/apache/cassandra/io/sstable/SSTableHeaderFix.java
+++ b/src/java/org/apache/cassandra/io/sstable/SSTableHeaderFix.java
@@ -39,12 +39,8 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
-import org.apache.cassandra.config.Schema;
-import org.apache.cassandra.config.SchemaConstants;
import org.apache.cassandra.cql3.CQL3Type;
-import org.apache.cassandra.cql3.statements.IndexTarget;
+import org.apache.cassandra.cql3.statements.schema.IndexTarget;
import org.apache.cassandra.db.Directories;
import org.apache.cassandra.db.SerializationHeader;
import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
@@ -60,7 +56,11 @@
import org.apache.cassandra.db.marshal.UserType;
import org.apache.cassandra.io.sstable.metadata.MetadataComponent;
import org.apache.cassandra.io.sstable.metadata.MetadataType;
+import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.schema.IndexMetadata;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.SchemaConstants;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.CassandraVersion;
import org.apache.cassandra.utils.FBUtilities;
@@ -103,7 +103,7 @@
FBUtilities.getReleaseVersionString());
SSTableHeaderFix instance = SSTableHeaderFix.builder()
- .schemaCallback(() -> Schema.instance::getCFMetaData)
+ .schemaCallback(() -> Schema.instance::getTableMetadata)
.build();
instance.execute();
}
@@ -116,7 +116,7 @@
protected final Consumer<String> warn;
protected final Consumer<String> error;
protected final boolean dryRun;
- protected final Function<Descriptor, CFMetaData> schemaCallback;
+ protected final Function<Descriptor, TableMetadata> schemaCallback;
private final List<Descriptor> descriptors;
@@ -161,7 +161,7 @@
private Consumer<String> warn = (ln) -> logger.warn("{}", ln);
private Consumer<String> error = (ln) -> logger.error("{}", ln);
private boolean dryRun;
- private Supplier<Function<Descriptor, CFMetaData>> schemaCallback = () -> null;
+ private Supplier<Function<Descriptor, TableMetadata>> schemaCallback = () -> null;
private Builder()
{}
@@ -218,7 +218,7 @@
* Schema callback to retrieve the schema of a table. Production code always delegates to the
* live schema ({@code Schema.instance}). Unit tests use this method to feed a custom schema.
*/
- public Builder schemaCallback(Supplier<Function<Descriptor, CFMetaData>> schemaCallback)
+ public Builder schemaCallback(Supplier<Function<Descriptor, TableMetadata>> schemaCallback)
{
this.schemaCallback = schemaCallback;
return this;
@@ -298,10 +298,7 @@
{
Stream.of(path)
.flatMap(SSTableHeaderFix::maybeExpandDirectory)
- .filter(p -> {
- File f = p.toFile();
- return Component.fromFilename(f.getParentFile(), f.getName()).right.type == Component.Type.DATA;
- })
+ .filter(p -> Descriptor.fromFilenameWithComponent(p.toFile()).right.type == Component.Type.DATA)
.map(Path::toString)
.map(Descriptor::fromFilename)
.forEach(descriptors::add);
@@ -328,7 +325,7 @@
return;
}
- CFMetaData tableMetadata = schemaCallback.apply(desc);
+ TableMetadata tableMetadata = schemaCallback.apply(desc);
if (tableMetadata == null)
{
error("Table %s.%s not found in the schema - NOT checking sstable %s", desc.ksname, desc.cfname, desc);
@@ -364,8 +361,8 @@
List<AbstractType<?>> clusteringTypes = validateClusteringColumns(desc, tableMetadata, header);
// check static and regular columns
- Map<ByteBuffer, AbstractType<?>> staticColumns = validateColumns(desc, tableMetadata, header.getStaticColumns(), ColumnDefinition.Kind.STATIC);
- Map<ByteBuffer, AbstractType<?>> regularColumns = validateColumns(desc, tableMetadata, header.getRegularColumns(), ColumnDefinition.Kind.REGULAR);
+ Map<ByteBuffer, AbstractType<?>> staticColumns = validateColumns(desc, tableMetadata, header.getStaticColumns(), ColumnMetadata.Kind.STATIC);
+ Map<ByteBuffer, AbstractType<?>> regularColumns = validateColumns(desc, tableMetadata, header.getRegularColumns(), ColumnMetadata.Kind.REGULAR);
SerializationHeader.Component newHeader = SerializationHeader.Component.buildComponentForTools(keyType,
clusteringTypes,
@@ -383,11 +380,11 @@
updates.add(Pair.create(desc, newMetadata));
}
- private AbstractType<?> validatePartitionKey(Descriptor desc, CFMetaData tableMetadata, SerializationHeader.Component header)
+ private AbstractType<?> validatePartitionKey(Descriptor desc, TableMetadata tableMetadata, SerializationHeader.Component header)
{
boolean keyMismatch = false;
AbstractType<?> headerKeyType = header.getKeyType();
- AbstractType<?> schemaKeyType = tableMetadata.getKeyValidator();
+ AbstractType<?> schemaKeyType = tableMetadata.partitionKeyType;
boolean headerKeyComposite = headerKeyType instanceof CompositeType;
boolean schemaKeyComposite = schemaKeyType instanceof CompositeType;
if (headerKeyComposite != schemaKeyComposite)
@@ -452,12 +449,12 @@
return headerKeyType;
}
- private List<AbstractType<?>> validateClusteringColumns(Descriptor desc, CFMetaData tableMetadata, SerializationHeader.Component header)
+ private List<AbstractType<?>> validateClusteringColumns(Descriptor desc, TableMetadata tableMetadata, SerializationHeader.Component header)
{
List<AbstractType<?>> headerClusteringTypes = header.getClusteringTypes();
List<AbstractType<?>> clusteringTypes = new ArrayList<>();
boolean clusteringMismatch = false;
- List<ColumnDefinition> schemaClustering = tableMetadata.clusteringColumns();
+ List<ColumnMetadata> schemaClustering = tableMetadata.clusteringColumns();
if (schemaClustering.size() != headerClusteringTypes.size())
{
clusteringMismatch = true;
@@ -470,7 +467,7 @@
for (int i = 0; i < headerClusteringTypes.size(); i++)
{
AbstractType<?> headerType = headerClusteringTypes.get(i);
- ColumnDefinition column = schemaClustering.get(i);
+ ColumnMetadata column = schemaClustering.get(i);
AbstractType<?> schemaType = column.type;
AbstractType<?> fixedType = fixType(desc, column.name.bytes, headerType, schemaType, false);
if (fixedType == null)
@@ -488,7 +485,7 @@
return clusteringTypes;
}
- private Map<ByteBuffer, AbstractType<?>> validateColumns(Descriptor desc, CFMetaData tableMetadata, Map<ByteBuffer, AbstractType<?>> columns, ColumnDefinition.Kind kind)
+ private Map<ByteBuffer, AbstractType<?>> validateColumns(Descriptor desc, TableMetadata tableMetadata, Map<ByteBuffer, AbstractType<?>> columns, ColumnMetadata.Kind kind)
{
Map<ByteBuffer, AbstractType<?>> target = new LinkedHashMap<>();
for (Map.Entry<ByteBuffer, AbstractType<?>> nameAndType : columns.entrySet())
@@ -512,29 +509,29 @@
return target;
}
- private AbstractType<?> validateColumn(Descriptor desc, CFMetaData tableMetadata, ColumnDefinition.Kind kind, ByteBuffer name, AbstractType<?> type)
+ private AbstractType<?> validateColumn(Descriptor desc, TableMetadata tableMetadata, ColumnMetadata.Kind kind, ByteBuffer name, AbstractType<?> type)
{
- ColumnDefinition cd = tableMetadata.getColumnDefinition(name);
+ ColumnMetadata cd = tableMetadata.getColumn(name);
if (cd == null)
{
// In case the column was dropped, there is not much that we can actually validate.
// The column could have been recreated using the same or a different kind or the same or
// a different type. Lottery...
- cd = tableMetadata.getDroppedColumnDefinition(name, kind == ColumnDefinition.Kind.STATIC);
+ cd = tableMetadata.getDroppedColumn(name, kind == ColumnMetadata.Kind.STATIC);
if (cd == null)
{
- for (IndexMetadata indexMetadata : tableMetadata.getIndexes())
+ for (IndexMetadata indexMetadata : tableMetadata.indexes)
{
String target = indexMetadata.options.get(IndexTarget.TARGET_OPTION_NAME);
if (target != null && ByteBufferUtil.bytes(target).equals(name))
{
warn.accept(String.format("sstable %s: contains column '%s', which is not a column in the table '%s.%s', but a target for that table's index '%s'",
- desc,
- logColumnName(name),
- tableMetadata.ksName,
- tableMetadata.cfName,
- indexMetadata.name));
+ desc,
+ logColumnName(name),
+ tableMetadata.keyspace,
+ tableMetadata.name,
+ indexMetadata.name));
return type;
}
}
diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableIdentityIterator.java b/src/java/org/apache/cassandra/io/sstable/SSTableIdentityIterator.java
index 2a79f88..76e12c8 100644
--- a/src/java/org/apache/cassandra/io/sstable/SSTableIdentityIterator.java
+++ b/src/java/org/apache/cassandra/io/sstable/SSTableIdentityIterator.java
@@ -19,7 +19,7 @@
import java.io.*;
-import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.rows.*;
import org.apache.cassandra.io.sstable.format.SSTableReader;
@@ -49,13 +49,16 @@
this.staticRow = iterator.readStaticRow();
}
+ @SuppressWarnings("resource")
public static SSTableIdentityIterator create(SSTableReader sstable, RandomAccessReader file, DecoratedKey key)
{
try
{
DeletionTime partitionLevelDeletion = DeletionTime.serializer.deserialize(file);
- SerializationHelper helper = new SerializationHelper(sstable.metadata, sstable.descriptor.version.correspondingMessagingVersion(), SerializationHelper.Flag.LOCAL);
- SSTableSimpleIterator iterator = SSTableSimpleIterator.create(sstable.metadata, file, sstable.header, helper, partitionLevelDeletion);
+ if (!partitionLevelDeletion.validate())
+ UnfilteredValidation.handleInvalid(sstable.metadata(), key, sstable, "partitionLevelDeletion="+partitionLevelDeletion.toString());
+ DeserializationHelper helper = new DeserializationHelper(sstable.metadata(), sstable.descriptor.version.correspondingMessagingVersion(), DeserializationHelper.Flag.LOCAL);
+ SSTableSimpleIterator iterator = SSTableSimpleIterator.create(sstable.metadata(), file, sstable.header, helper, partitionLevelDeletion);
return new SSTableIdentityIterator(sstable, key, partitionLevelDeletion, file.getPath(), iterator);
}
catch (IOException e)
@@ -65,6 +68,7 @@
}
}
+ @SuppressWarnings("resource")
public static SSTableIdentityIterator create(SSTableReader sstable, FileDataInput dfile, RowIndexEntry<?> indexEntry, DecoratedKey key, boolean tombstoneOnly)
{
try
@@ -72,10 +76,10 @@
dfile.seek(indexEntry.position);
ByteBufferUtil.skipShortLength(dfile); // Skip partition key
DeletionTime partitionLevelDeletion = DeletionTime.serializer.deserialize(dfile);
- SerializationHelper helper = new SerializationHelper(sstable.metadata, sstable.descriptor.version.correspondingMessagingVersion(), SerializationHelper.Flag.LOCAL);
+ DeserializationHelper helper = new DeserializationHelper(sstable.metadata(), sstable.descriptor.version.correspondingMessagingVersion(), DeserializationHelper.Flag.LOCAL);
SSTableSimpleIterator iterator = tombstoneOnly
- ? SSTableSimpleIterator.createTombstoneOnly(sstable.metadata, dfile, sstable.header, helper, partitionLevelDeletion)
- : SSTableSimpleIterator.create(sstable.metadata, dfile, sstable.header, helper, partitionLevelDeletion);
+ ? SSTableSimpleIterator.createTombstoneOnly(sstable.metadata(), dfile, sstable.header, helper, partitionLevelDeletion)
+ : SSTableSimpleIterator.create(sstable.metadata(), dfile, sstable.header, helper, partitionLevelDeletion);
return new SSTableIdentityIterator(sstable, key, partitionLevelDeletion, dfile.getPath(), iterator);
}
catch (IOException e)
@@ -85,14 +89,14 @@
}
}
- public CFMetaData metadata()
+ public TableMetadata metadata()
{
- return sstable.metadata;
+ return iterator.metadata;
}
- public PartitionColumns columns()
+ public RegularAndStaticColumns columns()
{
- return metadata().partitionColumns();
+ return metadata().regularAndStaticColumns();
}
public boolean isReverseOrder()
@@ -167,7 +171,9 @@
protected Unfiltered doCompute()
{
- return iterator.next();
+ Unfiltered unfiltered = iterator.next();
+ UnfilteredValidation.maybeValidateUnfiltered(unfiltered, metadata(), key, sstable);
+ return unfiltered;
}
public void close()
@@ -184,7 +190,7 @@
{
// We could return sstable.header.stats(), but this may not be as accurate than the actual sstable stats (see
// SerializationHeader.make() for details) so we use the latter instead.
- return new EncodingStats(sstable.getMinTimestamp(), sstable.getMinLocalDeletionTime(), sstable.getMinTTL());
+ return sstable.stats();
}
public int compareTo(SSTableIdentityIterator o)
diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableLoader.java b/src/java/org/apache/cassandra/io/sstable/SSTableLoader.java
index 334e0e0..ec2a700 100644
--- a/src/java/org/apache/cassandra/io/sstable/SSTableLoader.java
+++ b/src/java/org/apache/cassandra/io/sstable/SSTableLoader.java
@@ -18,20 +18,20 @@
package org.apache.cassandra.io.sstable;
import java.io.File;
-import java.io.IOException;
-import java.net.InetAddress;
import java.util.*;
import com.google.common.collect.HashMultimap;
import com.google.common.collect.Multimap;
-import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.db.streaming.CassandraOutgoingFile;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.io.FSError;
+import org.apache.cassandra.schema.TableMetadataRef;
import org.apache.cassandra.db.Directories;
import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.service.ActiveRepairService;
import org.apache.cassandra.streaming.*;
import org.apache.cassandra.utils.OutputHandler;
import org.apache.cassandra.utils.Pair;
@@ -49,27 +49,27 @@
private final Client client;
private final int connectionsPerHost;
private final OutputHandler outputHandler;
- private final Set<InetAddress> failedHosts = new HashSet<>();
+ private final Set<InetAddressAndPort> failedHosts = new HashSet<>();
private final List<SSTableReader> sstables = new ArrayList<>();
- private final Multimap<InetAddress, StreamSession.SSTableStreamingSections> streamingDetails = HashMultimap.create();
+ private final Multimap<InetAddressAndPort, OutgoingStream> streamingDetails = HashMultimap.create();
public SSTableLoader(File directory, Client client, OutputHandler outputHandler)
{
- this(directory, client, outputHandler, 1);
+ this(directory, client, outputHandler, 1, null);
}
- public SSTableLoader(File directory, Client client, OutputHandler outputHandler, int connectionsPerHost)
+ public SSTableLoader(File directory, Client client, OutputHandler outputHandler, int connectionsPerHost, String targetKeyspace)
{
this.directory = directory;
- this.keyspace = directory.getParentFile().getName();
+ this.keyspace = targetKeyspace != null ? targetKeyspace : directory.getParentFile().getName();
this.client = client;
this.outputHandler = outputHandler;
this.connectionsPerHost = connectionsPerHost;
}
@SuppressWarnings("resource")
- protected Collection<SSTableReader> openSSTables(final Map<InetAddress, Collection<Range<Token>>> ranges)
+ protected Collection<SSTableReader> openSSTables(final Map<InetAddressAndPort, Collection<Range<Token>>> ranges)
{
outputHandler.output("Opening sstables and calculating sections to stream");
@@ -85,7 +85,7 @@
return false;
}
- Pair<Descriptor, Component> p = SSTable.tryComponentFromFilename(dir, name);
+ Pair<Descriptor, Component> p = SSTable.tryComponentFromFilename(file);
Descriptor desc = p == null ? null : p.left;
if (p == null || !p.right.equals(Component.DATA))
return false;
@@ -96,7 +96,7 @@
return false;
}
- CFMetaData metadata = client.getTableMetadata(desc.cfname);
+ TableMetadataRef metadata = client.getTableMetadata(desc.cfname);
if (metadata == null)
{
outputHandler.output(String.format("Skipping file %s: table %s.%s doesn't exist", name, keyspace, desc.cfname));
@@ -123,23 +123,24 @@
// calculate the sstable sections to stream as well as the estimated number of
// keys per host
- for (Map.Entry<InetAddress, Collection<Range<Token>>> entry : ranges.entrySet())
+ for (Map.Entry<InetAddressAndPort, Collection<Range<Token>>> entry : ranges.entrySet())
{
- InetAddress endpoint = entry.getKey();
- Collection<Range<Token>> tokenRanges = entry.getValue();
+ InetAddressAndPort endpoint = entry.getKey();
+ List<Range<Token>> tokenRanges = Range.normalize(entry.getValue());
- List<Pair<Long, Long>> sstableSections = sstable.getPositionsForRanges(tokenRanges);
+ List<SSTableReader.PartitionPositionBounds> sstableSections = sstable.getPositionsForRanges(tokenRanges);
long estimatedKeys = sstable.estimatedKeysForRanges(tokenRanges);
Ref<SSTableReader> ref = sstable.ref();
- StreamSession.SSTableStreamingSections details = new StreamSession.SSTableStreamingSections(ref, sstableSections, estimatedKeys, ActiveRepairService.UNREPAIRED_SSTABLE);
- streamingDetails.put(endpoint, details);
+ OutgoingStream stream = new CassandraOutgoingFile(StreamOperation.BULK_LOAD, ref, sstableSections, tokenRanges, estimatedKeys);
+ streamingDetails.put(endpoint, stream);
}
// to conserve heap space when bulk loading
sstable.releaseSummary();
}
- catch (IOException e)
+ catch (FSError e)
{
+ // todo: should we really continue if we can't open all sstables?
outputHandler.output(String.format("Skipping file %s, error opening it: %s", name, e.getMessage()));
}
return false;
@@ -151,17 +152,17 @@
public StreamResultFuture stream()
{
- return stream(Collections.<InetAddress>emptySet());
+ return stream(Collections.<InetAddressAndPort>emptySet());
}
- public StreamResultFuture stream(Set<InetAddress> toIgnore, StreamEventHandler... listeners)
+ public StreamResultFuture stream(Set<InetAddressAndPort> toIgnore, StreamEventHandler... listeners)
{
client.init(keyspace);
outputHandler.output("Established connection to initial hosts");
- StreamPlan plan = new StreamPlan("Bulk Load", 0, connectionsPerHost, false, false, false).connectionFactory(client.getConnectionFactory());
+ StreamPlan plan = new StreamPlan(StreamOperation.BULK_LOAD, connectionsPerHost, false, null, PreviewKind.NONE).connectionFactory(client.getConnectionFactory());
- Map<InetAddress, Collection<Range<Token>>> endpointToRanges = client.getEndpointToRangesMap();
+ Map<InetAddressAndPort, Collection<Range<Token>>> endpointToRanges = client.getEndpointToRangesMap();
openSSTables(endpointToRanges);
if (sstables.isEmpty())
{
@@ -171,21 +172,21 @@
outputHandler.output(String.format("Streaming relevant part of %s to %s", names(sstables), endpointToRanges.keySet()));
- for (Map.Entry<InetAddress, Collection<Range<Token>>> entry : endpointToRanges.entrySet())
+ for (Map.Entry<InetAddressAndPort, Collection<Range<Token>>> entry : endpointToRanges.entrySet())
{
- InetAddress remote = entry.getKey();
+ InetAddressAndPort remote = entry.getKey();
if (toIgnore.contains(remote))
continue;
- List<StreamSession.SSTableStreamingSections> endpointDetails = new LinkedList<>();
+ List<OutgoingStream> streams = new LinkedList<>();
// references are acquired when constructing the SSTableStreamingSections above
- for (StreamSession.SSTableStreamingSections details : streamingDetails.get(remote))
+ for (OutgoingStream stream : streamingDetails.get(remote))
{
- endpointDetails.add(details);
+ streams.add(stream);
}
- plan.transferFiles(remote, endpointDetails);
+ plan.transferStreams(remote, streams);
}
plan.listeners(this, listeners);
return plan.execute();
@@ -208,7 +209,7 @@
for (SSTableReader sstable : sstables)
{
sstable.selfRef().release();
- assert sstable.selfRef().globalCount() == 0;
+ assert sstable.selfRef().globalCount() == 0 : String.format("for sstable = %s, ref count = %d", sstable, sstable.selfRef().globalCount());
}
}
@@ -230,14 +231,14 @@
return builder.toString();
}
- public Set<InetAddress> getFailedHosts()
+ public Set<InetAddressAndPort> getFailedHosts()
{
return failedHosts;
}
public static abstract class Client
{
- private final Map<InetAddress, Collection<Range<Token>>> endpointToRanges = new HashMap<>();
+ private final Map<InetAddressAndPort, Collection<Range<Token>>> endpointToRanges = new HashMap<>();
/**
* Initialize the client.
@@ -272,19 +273,19 @@
* Validate that {@code keyspace} is an existing keyspace and {@code
* cfName} one of its existing column family.
*/
- public abstract CFMetaData getTableMetadata(String tableName);
+ public abstract TableMetadataRef getTableMetadata(String tableName);
- public void setTableMetadata(CFMetaData cfm)
+ public void setTableMetadata(TableMetadataRef cfm)
{
throw new RuntimeException();
}
- public Map<InetAddress, Collection<Range<Token>>> getEndpointToRangesMap()
+ public Map<InetAddressAndPort, Collection<Range<Token>>> getEndpointToRangesMap()
{
return endpointToRanges;
}
- protected void addRangeForEndpoint(Range<Token> range, InetAddress endpoint)
+ protected void addRangeForEndpoint(Range<Token> range, InetAddressAndPort endpoint)
{
Collection<Range<Token>> ranges = endpointToRanges.get(endpoint);
if (ranges == null)
diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableMultiWriter.java b/src/java/org/apache/cassandra/io/sstable/SSTableMultiWriter.java
index b92bc78..1be79ab 100644
--- a/src/java/org/apache/cassandra/io/sstable/SSTableMultiWriter.java
+++ b/src/java/org/apache/cassandra/io/sstable/SSTableMultiWriter.java
@@ -19,10 +19,10 @@
package org.apache.cassandra.io.sstable;
import java.util.Collection;
-import java.util.UUID;
import org.apache.cassandra.db.rows.UnfilteredRowIterator;
import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.utils.Throwables;
import org.apache.cassandra.utils.concurrent.Transactional;
@@ -44,7 +44,7 @@
String getFilename();
long getFilePointer();
- UUID getCfId();
+ TableId getTableId();
static void abortOrDie(SSTableMultiWriter writer)
{
diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableRewriter.java b/src/java/org/apache/cassandra/io/sstable/SSTableRewriter.java
index a71d1af..fb3aa2d 100644
--- a/src/java/org/apache/cassandra/io/sstable/SSTableRewriter.java
+++ b/src/java/org/apache/cassandra/io/sstable/SSTableRewriter.java
@@ -32,7 +32,6 @@
import org.apache.cassandra.db.rows.UnfilteredRowIterator;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.sstable.format.SSTableWriter;
-import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
import org.apache.cassandra.utils.NativeLibrary;
import org.apache.cassandra.utils.concurrent.Transactional;
@@ -117,7 +116,7 @@
private static long calculateOpenInterval(boolean shouldOpenEarly)
{
- long interval = DatabaseDescriptor.getSSTablePreempiveOpenIntervalInMB() * (1L << 20);
+ long interval = DatabaseDescriptor.getSSTablePreemptiveOpenIntervalInMB() * (1L << 20);
if (disableEarlyOpeningForTests || !shouldOpenEarly || interval < 0)
interval = Long.MAX_VALUE;
return interval;
@@ -134,14 +133,17 @@
DecoratedKey key = partition.partitionKey();
maybeReopenEarly(key);
RowIndexEntry index = writer.append(partition);
- if (!transaction.isOffline() && index != null)
+ if (DatabaseDescriptor.shouldMigrateKeycacheOnCompaction())
{
- for (SSTableReader reader : transaction.originals())
+ if (!transaction.isOffline() && index != null)
{
- if (reader.getCachedPosition(key, false) != null)
+ for (SSTableReader reader : transaction.originals())
{
- cachedKeys.put(key, index);
- break;
+ if (reader.getCachedPosition(key, false) != null)
+ {
+ cachedKeys.put(key, index);
+ break;
+ }
}
}
}
@@ -223,9 +225,7 @@
*/
private void moveStarts(SSTableReader newReader, DecoratedKey lowerbound)
{
- if (transaction.isOffline())
- return;
- if (preemptiveOpenInterval == Long.MAX_VALUE)
+ if (transaction.isOffline() || preemptiveOpenInterval == Long.MAX_VALUE)
return;
newReader.setupOnline();
diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableSimpleIterator.java b/src/java/org/apache/cassandra/io/sstable/SSTableSimpleIterator.java
index ce42126..fd1b6a0 100644
--- a/src/java/org/apache/cassandra/io/sstable/SSTableSimpleIterator.java
+++ b/src/java/org/apache/cassandra/io/sstable/SSTableSimpleIterator.java
@@ -21,17 +21,12 @@
import java.io.IOError;
import java.util.Iterator;
-import org.apache.cassandra.io.util.RewindableDataInput;
-import org.apache.cassandra.utils.AbstractIterator;
-
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.rows.*;
import org.apache.cassandra.io.util.DataInputPlus;
-import org.apache.cassandra.io.util.DataPosition;
import org.apache.cassandra.io.util.FileDataInput;
-import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.utils.AbstractIterator;
/**
* Utility class to handle deserializing atom from sstables.
@@ -41,31 +36,25 @@
*/
public abstract class SSTableSimpleIterator extends AbstractIterator<Unfiltered> implements Iterator<Unfiltered>
{
- protected final CFMetaData metadata;
+ final TableMetadata metadata;
protected final DataInputPlus in;
- protected final SerializationHelper helper;
+ protected final DeserializationHelper helper;
- private SSTableSimpleIterator(CFMetaData metadata, DataInputPlus in, SerializationHelper helper)
+ private SSTableSimpleIterator(TableMetadata metadata, DataInputPlus in, DeserializationHelper helper)
{
this.metadata = metadata;
this.in = in;
this.helper = helper;
}
- public static SSTableSimpleIterator create(CFMetaData metadata, DataInputPlus in, SerializationHeader header, SerializationHelper helper, DeletionTime partitionDeletion)
+ public static SSTableSimpleIterator create(TableMetadata metadata, DataInputPlus in, SerializationHeader header, DeserializationHelper helper, DeletionTime partitionDeletion)
{
- if (helper.version < MessagingService.VERSION_30)
- return new OldFormatIterator(metadata, in, helper, partitionDeletion);
- else
- return new CurrentFormatIterator(metadata, in, header, helper);
+ return new CurrentFormatIterator(metadata, in, header, helper);
}
- public static SSTableSimpleIterator createTombstoneOnly(CFMetaData metadata, DataInputPlus in, SerializationHeader header, SerializationHelper helper, DeletionTime partitionDeletion)
+ public static SSTableSimpleIterator createTombstoneOnly(TableMetadata metadata, DataInputPlus in, SerializationHeader header, DeserializationHelper helper, DeletionTime partitionDeletion)
{
- if (helper.version < MessagingService.VERSION_30)
- return new OldFormatTombstoneIterator(metadata, in, helper, partitionDeletion);
- else
- return new CurrentFormatTombstoneIterator(metadata, in, header, helper);
+ return new CurrentFormatTombstoneIterator(metadata, in, header, helper);
}
public abstract Row readStaticRow() throws IOException;
@@ -76,7 +65,7 @@
private final Row.Builder builder;
- private CurrentFormatIterator(CFMetaData metadata, DataInputPlus in, SerializationHeader header, SerializationHelper helper)
+ private CurrentFormatIterator(TableMetadata metadata, DataInputPlus in, SerializationHeader header, DeserializationHelper helper)
{
super(metadata, in, helper);
this.header = header;
@@ -106,7 +95,7 @@
{
private final SerializationHeader header;
- private CurrentFormatTombstoneIterator(CFMetaData metadata, DataInputPlus in, SerializationHeader header, SerializationHelper helper)
+ private CurrentFormatTombstoneIterator(TableMetadata metadata, DataInputPlus in, SerializationHeader header, DeserializationHelper helper)
{
super(metadata, in, helper);
this.header = header;
@@ -136,106 +125,4 @@
}
}
}
-
- private static class OldFormatIterator extends SSTableSimpleIterator
- {
- private final UnfilteredDeserializer deserializer;
-
- private OldFormatIterator(CFMetaData metadata, DataInputPlus in, SerializationHelper helper, DeletionTime partitionDeletion)
- {
- super(metadata, in, helper);
- // We use an UnfilteredDeserializer because even though we don't need all it's fanciness, it happens to handle all
- // the details we need for reading the old format.
- this.deserializer = UnfilteredDeserializer.create(metadata, in, null, helper, partitionDeletion, false);
- }
-
- public Row readStaticRow() throws IOException
- {
- if (metadata.isCompactTable())
- {
- // For static compact tables, in the old format, static columns are intermingled with the other columns, so we
- // need to extract them. Which imply 2 passes (one to extract the static, then one for other value).
- if (metadata.isStaticCompactTable())
- {
- assert in instanceof RewindableDataInput;
- RewindableDataInput file = (RewindableDataInput)in;
- DataPosition mark = file.mark();
- Row staticRow = LegacyLayout.extractStaticColumns(metadata, file, metadata.partitionColumns().statics);
- file.reset(mark);
-
- // We've extracted the static columns, so we must ignore them on the 2nd pass
- ((UnfilteredDeserializer.OldFormatDeserializer)deserializer).setSkipStatic();
- return staticRow;
- }
- else
- {
- return Rows.EMPTY_STATIC_ROW;
- }
- }
-
- return deserializer.hasNext() && deserializer.nextIsStatic()
- ? (Row)deserializer.readNext()
- : Rows.EMPTY_STATIC_ROW;
-
- }
-
- protected Unfiltered computeNext()
- {
- while (true)
- {
- try
- {
- if (!deserializer.hasNext())
- return endOfData();
-
- Unfiltered unfiltered = deserializer.readNext();
- if (metadata.isStaticCompactTable() && unfiltered.kind() == Unfiltered.Kind.ROW)
- {
- Row row = (Row) unfiltered;
- ColumnDefinition def = metadata.getColumnDefinition(LegacyLayout.encodeClustering(metadata, row.clustering()));
- if (def != null && def.isStatic())
- continue;
- }
- return unfiltered;
- }
- catch (IOException e)
- {
- throw new IOError(e);
- }
- }
- }
-
- }
-
- private static class OldFormatTombstoneIterator extends OldFormatIterator
- {
- private OldFormatTombstoneIterator(CFMetaData metadata, DataInputPlus in, SerializationHelper helper, DeletionTime partitionDeletion)
- {
- super(metadata, in, helper, partitionDeletion);
- }
-
- public Row readStaticRow() throws IOException
- {
- Row row = super.readStaticRow();
- if (!row.deletion().isLive())
- return BTreeRow.emptyDeletedRow(row.clustering(), row.deletion());
- return Rows.EMPTY_STATIC_ROW;
- }
-
- protected Unfiltered computeNext()
- {
- while (true)
- {
- Unfiltered unfiltered = super.computeNext();
- if (unfiltered == null || unfiltered.isRangeTombstoneMarker())
- return unfiltered;
-
- Row row = (Row) unfiltered;
- if (!row.deletion().isLive())
- return BTreeRow.emptyDeletedRow(row.clustering(), row.deletion());
- // Otherwise read next.
- }
- }
-
- }
}
diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableSimpleUnsortedWriter.java b/src/java/org/apache/cassandra/io/sstable/SSTableSimpleUnsortedWriter.java
index 23e18b5..7ac2ebc 100644
--- a/src/java/org/apache/cassandra/io/sstable/SSTableSimpleUnsortedWriter.java
+++ b/src/java/org/apache/cassandra/io/sstable/SSTableSimpleUnsortedWriter.java
@@ -28,12 +28,13 @@
import com.google.common.base.Throwables;
import io.netty.util.concurrent.FastThreadLocalThread;
-import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.rows.Row;
import org.apache.cassandra.db.rows.EncodingStats;
+import org.apache.cassandra.db.rows.SerializationHelper;
import org.apache.cassandra.db.rows.UnfilteredSerializer;
import org.apache.cassandra.db.partitions.PartitionUpdate;
+import org.apache.cassandra.schema.TableMetadataRef;
import org.apache.cassandra.utils.JVMStabilityInspector;
/**
@@ -56,28 +57,30 @@
// Used to compute the row serialized size
private final SerializationHeader header;
+ private final SerializationHelper helper;
private final BlockingQueue<Buffer> writeQueue = new SynchronousQueue<Buffer>();
private final DiskWriter diskWriter = new DiskWriter();
- SSTableSimpleUnsortedWriter(File directory, CFMetaData metadata, PartitionColumns columns, long bufferSizeInMB)
+ SSTableSimpleUnsortedWriter(File directory, TableMetadataRef metadata, RegularAndStaticColumns columns, long bufferSizeInMB)
{
super(directory, metadata, columns);
this.bufferSize = bufferSizeInMB * 1024L * 1024L;
- this.header = new SerializationHeader(true, metadata, columns, EncodingStats.NO_STATS);
+ this.header = new SerializationHeader(true, metadata.get(), columns, EncodingStats.NO_STATS);
+ this.helper = new SerializationHelper(this.header);
diskWriter.start();
}
- PartitionUpdate getUpdateFor(DecoratedKey key)
+ PartitionUpdate.Builder getUpdateFor(DecoratedKey key)
{
assert key != null;
-
- PartitionUpdate previous = buffer.get(key);
+ PartitionUpdate.Builder previous = buffer.get(key);
if (previous == null)
{
- previous = createPartitionUpdate(key);
- currentSize += PartitionUpdate.serializer.serializedSize(previous, formatType.info.getLatestVersion().correspondingMessagingVersion());
- previous.allowNewUpdates();
+ // todo: inefficient - we create and serialize a PU just to get its size, then recreate it
+ // todo: either allow PartitionUpdateBuilder to have .build() called several times or pre-calculate the size
+ currentSize += PartitionUpdate.serializer.serializedSize(createPartitionUpdateBuilder(key).build(), formatType.info.getLatestVersion().correspondingMessagingVersion());
+ previous = createPartitionUpdateBuilder(key);
buffer.put(key, previous);
}
return previous;
@@ -90,7 +93,7 @@
// improve that. In particular, what we count is closer to the serialized value, but it's debatable that it's the right thing
// to count since it will take a lot more space in memory and the bufferSize if first and foremost used to avoid OOM when
// using this writer.
- currentSize += UnfilteredSerializer.serializer.serializedSize(row, header, 0, formatType.info.getLatestVersion().correspondingMessagingVersion());
+ currentSize += UnfilteredSerializer.serializer.serializedSize(row, helper, 0, formatType.info.getLatestVersion().correspondingMessagingVersion());
}
private void maybeSync() throws SyncException
@@ -108,9 +111,9 @@
}
}
- private PartitionUpdate createPartitionUpdate(DecoratedKey key)
+ private PartitionUpdate.Builder createPartitionUpdateBuilder(DecoratedKey key)
{
- return new PartitionUpdate(metadata, key, columns, 4)
+ return new PartitionUpdate.Builder(metadata.get(), key, columns, 4)
{
@Override
public void add(Row row)
@@ -188,7 +191,7 @@
}
//// typedef
- static class Buffer extends TreeMap<DecoratedKey, PartitionUpdate> {}
+ static class Buffer extends TreeMap<DecoratedKey, PartitionUpdate.Builder> {}
private class DiskWriter extends FastThreadLocalThread
{
@@ -206,8 +209,8 @@
try (SSTableTxnWriter writer = createWriter())
{
- for (Map.Entry<DecoratedKey, PartitionUpdate> entry : b.entrySet())
- writer.append(entry.getValue().unfilteredIterator());
+ for (Map.Entry<DecoratedKey, PartitionUpdate.Builder> entry : b.entrySet())
+ writer.append(entry.getValue().build().unfilteredIterator());
writer.finish(false);
}
}
diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableSimpleWriter.java b/src/java/org/apache/cassandra/io/sstable/SSTableSimpleWriter.java
index 7fbd79d..530a03b 100644
--- a/src/java/org/apache/cassandra/io/sstable/SSTableSimpleWriter.java
+++ b/src/java/org/apache/cassandra/io/sstable/SSTableSimpleWriter.java
@@ -22,9 +22,9 @@
import com.google.common.base.Throwables;
-import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.partitions.PartitionUpdate;
+import org.apache.cassandra.schema.TableMetadataRef;
/**
* A SSTable writer that assumes rows are in (partitioner) sorted order.
@@ -39,11 +39,11 @@
class SSTableSimpleWriter extends AbstractSSTableSimpleWriter
{
protected DecoratedKey currentKey;
- protected PartitionUpdate update;
+ protected PartitionUpdate.Builder update;
private SSTableTxnWriter writer;
- protected SSTableSimpleWriter(File directory, CFMetaData metadata, PartitionColumns columns)
+ protected SSTableSimpleWriter(File directory, TableMetadataRef metadata, RegularAndStaticColumns columns)
{
super(directory, metadata, columns);
}
@@ -56,7 +56,7 @@
return writer;
}
- PartitionUpdate getUpdateFor(DecoratedKey key) throws IOException
+ PartitionUpdate.Builder getUpdateFor(DecoratedKey key) throws IOException
{
assert key != null;
@@ -65,9 +65,9 @@
if (!key.equals(currentKey))
{
if (update != null)
- writePartition(update);
+ writePartition(update.build());
currentKey = key;
- update = new PartitionUpdate(metadata, currentKey, columns, 4);
+ update = new PartitionUpdate.Builder(metadata.get(), currentKey, columns, 4);
}
assert update != null;
@@ -79,7 +79,7 @@
try
{
if (update != null)
- writePartition(update);
+ writePartition(update.build());
if (writer != null)
writer.finish(false);
}
diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableTxnWriter.java b/src/java/org/apache/cassandra/io/sstable/SSTableTxnWriter.java
index 015c5bb..cfb1365 100644
--- a/src/java/org/apache/cassandra/io/sstable/SSTableTxnWriter.java
+++ b/src/java/org/apache/cassandra/io/sstable/SSTableTxnWriter.java
@@ -20,8 +20,8 @@
import java.io.IOException;
import java.util.Collection;
+import java.util.UUID;
-import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.db.SerializationHeader;
@@ -33,6 +33,7 @@
import org.apache.cassandra.io.sstable.format.SSTableFormat;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
+import org.apache.cassandra.schema.TableMetadataRef;
import org.apache.cassandra.utils.concurrent.Transactional;
/**
@@ -98,29 +99,31 @@
}
@SuppressWarnings("resource") // log and writer closed during doPostCleanup
- public static SSTableTxnWriter create(ColumnFamilyStore cfs, Descriptor descriptor, long keyCount, long repairedAt, int sstableLevel, SerializationHeader header)
+ public static SSTableTxnWriter create(ColumnFamilyStore cfs, Descriptor descriptor, long keyCount, long repairedAt, UUID pendingRepair, boolean isTransient, int sstableLevel, SerializationHeader header)
{
LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.WRITE);
- SSTableMultiWriter writer = cfs.createSSTableMultiWriter(descriptor, keyCount, repairedAt, sstableLevel, header, txn);
+ SSTableMultiWriter writer = cfs.createSSTableMultiWriter(descriptor, keyCount, repairedAt, pendingRepair, isTransient, sstableLevel, header, txn);
return new SSTableTxnWriter(txn, writer);
}
@SuppressWarnings("resource") // log and writer closed during doPostCleanup
- public static SSTableTxnWriter createRangeAware(CFMetaData cfm,
+ public static SSTableTxnWriter createRangeAware(TableMetadataRef metadata,
long keyCount,
long repairedAt,
+ UUID pendingRepair,
+ boolean isTransient,
SSTableFormat.Type type,
int sstableLevel,
SerializationHeader header)
{
- ColumnFamilyStore cfs = Keyspace.open(cfm.ksName).getColumnFamilyStore(cfm.cfName);
+ ColumnFamilyStore cfs = Keyspace.open(metadata.keyspace).getColumnFamilyStore(metadata.name);
LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.WRITE);
SSTableMultiWriter writer;
try
{
- writer = new RangeAwareSSTableWriter(cfs, keyCount, repairedAt, type, sstableLevel, 0, txn, header);
+ writer = new RangeAwareSSTableWriter(cfs, keyCount, repairedAt, pendingRepair, isTransient, type, sstableLevel, 0, txn, header);
}
catch (IOException e)
{
@@ -133,29 +136,25 @@
}
@SuppressWarnings("resource") // log and writer closed during doPostCleanup
- public static SSTableTxnWriter create(CFMetaData cfm,
+ public static SSTableTxnWriter create(TableMetadataRef metadata,
Descriptor descriptor,
long keyCount,
long repairedAt,
+ UUID pendingRepair,
+ boolean isTransient,
int sstableLevel,
SerializationHeader header,
Collection<Index> indexes)
{
// if the column family store does not exist, we create a new default SSTableMultiWriter to use:
LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.WRITE);
- MetadataCollector collector = new MetadataCollector(cfm.comparator).sstableLevel(sstableLevel);
- SSTableMultiWriter writer = SimpleSSTableMultiWriter.create(descriptor, keyCount, repairedAt, cfm, collector, header, indexes, txn);
+ MetadataCollector collector = new MetadataCollector(metadata.get().comparator).sstableLevel(sstableLevel);
+ SSTableMultiWriter writer = SimpleSSTableMultiWriter.create(descriptor, keyCount, repairedAt, pendingRepair, isTransient, metadata, collector, header, indexes, txn);
return new SSTableTxnWriter(txn, writer);
}
- public static SSTableTxnWriter create(ColumnFamilyStore cfs, String filename, long keyCount, long repairedAt, int sstableLevel, SerializationHeader header)
+ public static SSTableTxnWriter create(ColumnFamilyStore cfs, Descriptor desc, long keyCount, long repairedAt, UUID pendingRepair, boolean isTransient, SerializationHeader header)
{
- Descriptor desc = Descriptor.fromFilename(filename);
- return create(cfs, desc, keyCount, repairedAt, sstableLevel, header);
- }
-
- public static SSTableTxnWriter create(ColumnFamilyStore cfs, String filename, long keyCount, long repairedAt, SerializationHeader header)
- {
- return create(cfs, filename, keyCount, repairedAt, 0, header);
+ return create(cfs, desc, keyCount, repairedAt, pendingRepair, isTransient, 0, header);
}
}
diff --git a/src/java/org/apache/cassandra/io/sstable/SimpleSSTableMultiWriter.java b/src/java/org/apache/cassandra/io/sstable/SimpleSSTableMultiWriter.java
index 76e4dbb..a84f07e 100644
--- a/src/java/org/apache/cassandra/io/sstable/SimpleSSTableMultiWriter.java
+++ b/src/java/org/apache/cassandra/io/sstable/SimpleSSTableMultiWriter.java
@@ -15,14 +15,12 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.cassandra.io.sstable;
import java.util.Collection;
import java.util.Collections;
import java.util.UUID;
-import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.db.RowIndexEntry;
import org.apache.cassandra.db.SerializationHeader;
import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
@@ -31,6 +29,8 @@
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.sstable.format.SSTableWriter;
import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.schema.TableMetadataRef;
public class SimpleSSTableMultiWriter implements SSTableMultiWriter
{
@@ -80,9 +80,9 @@
return writer.getFilePointer();
}
- public UUID getCfId()
+ public TableId getTableId()
{
- return writer.metadata.cfId;
+ return writer.metadata().id;
}
public Throwable commit(Throwable accumulate)
@@ -110,13 +110,15 @@
public static SSTableMultiWriter create(Descriptor descriptor,
long keyCount,
long repairedAt,
- CFMetaData cfm,
+ UUID pendingRepair,
+ boolean isTransient,
+ TableMetadataRef metadata,
MetadataCollector metadataCollector,
SerializationHeader header,
Collection<Index> indexes,
LifecycleNewTracker lifecycleNewTracker)
{
- SSTableWriter writer = SSTableWriter.create(descriptor, keyCount, repairedAt, cfm, metadataCollector, header, indexes, lifecycleNewTracker);
+ SSTableWriter writer = SSTableWriter.create(descriptor, keyCount, repairedAt, pendingRepair, isTransient, metadata, metadataCollector, header, indexes, lifecycleNewTracker);
return new SimpleSSTableMultiWriter(writer, lifecycleNewTracker);
}
}
diff --git a/src/java/org/apache/cassandra/io/sstable/format/RangeAwareSSTableWriter.java b/src/java/org/apache/cassandra/io/sstable/format/RangeAwareSSTableWriter.java
index 3358225..ef4deb7 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/RangeAwareSSTableWriter.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/RangeAwareSSTableWriter.java
@@ -33,6 +33,7 @@
import org.apache.cassandra.db.rows.UnfilteredRowIterator;
import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.io.sstable.SSTableMultiWriter;
+import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.utils.FBUtilities;
public class RangeAwareSSTableWriter implements SSTableMultiWriter
@@ -42,6 +43,8 @@
private final int sstableLevel;
private final long estimatedKeys;
private final long repairedAt;
+ private final UUID pendingRepair;
+ private final boolean isTransient;
private final SSTableFormat.Type format;
private final SerializationHeader header;
private final LifecycleNewTracker lifecycleNewTracker;
@@ -51,7 +54,7 @@
private final List<SSTableReader> finishedReaders = new ArrayList<>();
private SSTableMultiWriter currentWriter = null;
- public RangeAwareSSTableWriter(ColumnFamilyStore cfs, long estimatedKeys, long repairedAt, SSTableFormat.Type format, int sstableLevel, long totalSize, LifecycleNewTracker lifecycleNewTracker, SerializationHeader header) throws IOException
+ public RangeAwareSSTableWriter(ColumnFamilyStore cfs, long estimatedKeys, long repairedAt, UUID pendingRepair, boolean isTransient, SSTableFormat.Type format, int sstableLevel, long totalSize, LifecycleNewTracker lifecycleNewTracker, SerializationHeader header) throws IOException
{
DiskBoundaries db = cfs.getDiskBoundaries();
directories = db.directories;
@@ -59,6 +62,8 @@
this.cfs = cfs;
this.estimatedKeys = estimatedKeys / directories.size();
this.repairedAt = repairedAt;
+ this.pendingRepair = pendingRepair;
+ this.isTransient = isTransient;
this.format = format;
this.lifecycleNewTracker = lifecycleNewTracker;
this.header = header;
@@ -69,8 +74,8 @@
if (localDir == null)
throw new IOException(String.format("Insufficient disk space to store %s",
FBUtilities.prettyPrintMemory(totalSize)));
- Descriptor desc = Descriptor.fromFilename(cfs.getSSTablePath(cfs.getDirectories().getLocationForDisk(localDir), format));
- currentWriter = cfs.createSSTableMultiWriter(desc, estimatedKeys, repairedAt, sstableLevel, header, lifecycleNewTracker);
+ Descriptor desc = cfs.newSSTableDescriptor(cfs.getDirectories().getLocationForDisk(localDir), format);
+ currentWriter = cfs.createSSTableMultiWriter(desc, estimatedKeys, repairedAt, pendingRepair, isTransient, sstableLevel, header, lifecycleNewTracker);
}
}
@@ -91,8 +96,8 @@
if (currentWriter != null)
finishedWriters.add(currentWriter);
- Descriptor desc = Descriptor.fromFilename(cfs.getSSTablePath(cfs.getDirectories().getLocationForDisk(directories.get(currentIndex))), format);
- currentWriter = cfs.createSSTableMultiWriter(desc, estimatedKeys, repairedAt, sstableLevel, header, lifecycleNewTracker);
+ Descriptor desc = cfs.newSSTableDescriptor(cfs.getDirectories().getLocationForDisk(directories.get(currentIndex)), format);
+ currentWriter = cfs.createSSTableMultiWriter(desc, estimatedKeys, repairedAt, pendingRepair, isTransient, sstableLevel, header, lifecycleNewTracker);
}
}
@@ -160,9 +165,9 @@
}
@Override
- public UUID getCfId()
+ public TableId getTableId()
{
- return currentWriter.getCfId();
+ return currentWriter.getTableId();
}
@Override
diff --git a/src/java/org/apache/cassandra/io/sstable/format/SSTableFormat.java b/src/java/org/apache/cassandra/io/sstable/format/SSTableFormat.java
index 4391946..14f6602 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/SSTableFormat.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/SSTableFormat.java
@@ -18,7 +18,8 @@
package org.apache.cassandra.io.sstable.format;
import com.google.common.base.CharMatcher;
-import org.apache.cassandra.config.CFMetaData;
+
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.db.RowIndexEntry;
import org.apache.cassandra.db.SerializationHeader;
import org.apache.cassandra.io.sstable.format.big.BigFormat;
@@ -37,14 +38,10 @@
SSTableWriter.Factory getWriterFactory();
SSTableReader.Factory getReaderFactory();
- RowIndexEntry.IndexSerializer<?> getIndexSerializer(CFMetaData cfm, Version version, SerializationHeader header);
+ RowIndexEntry.IndexSerializer<?> getIndexSerializer(TableMetadata metadata, Version version, SerializationHeader header);
public static enum Type
{
- //Used internally to refer to files with no
- //format flag in the filename
- LEGACY("big", BigFormat.instance),
-
//The original sstable format
BIG("big", BigFormat.instance);
@@ -60,7 +57,7 @@
{
//Since format comes right after generation
//we disallow formats with numeric names
- assert !CharMatcher.DIGIT.matchesAllOf(name);
+ assert !CharMatcher.digit().matchesAllOf(name);
this.name = name;
this.info = info;
@@ -70,10 +67,6 @@
{
for (Type valid : Type.values())
{
- //This is used internally for old sstables
- if (valid == LEGACY)
- continue;
-
if (valid.name.equalsIgnoreCase(name))
return valid;
}
diff --git a/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java b/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
index 9312ac6..04c4826 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
@@ -17,6 +17,8 @@
*/
package org.apache.cassandra.io.sstable.format;
+import java.nio.file.Files;
+import java.nio.file.Paths;
import java.io.*;
import java.lang.ref.WeakReference;
import java.nio.ByteBuffer;
@@ -26,7 +28,6 @@
import java.util.concurrent.atomic.AtomicLong;
import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Predicate;
import com.google.common.collect.Iterables;
import com.google.common.collect.Ordering;
import com.google.common.primitives.Longs;
@@ -44,20 +45,18 @@
import org.apache.cassandra.concurrent.DebuggableThreadPoolExecutor;
import org.apache.cassandra.concurrent.NamedThreadFactory;
import org.apache.cassandra.concurrent.ScheduledExecutors;
-import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.config.Config;
import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.config.Schema;
-import org.apache.cassandra.config.SchemaConstants;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.filter.ColumnFilter;
import org.apache.cassandra.db.rows.Cell;
import org.apache.cassandra.db.rows.EncodingStats;
import org.apache.cassandra.db.rows.UnfilteredRowIterator;
import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.dht.Bounds;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
-import org.apache.cassandra.index.internal.CassandraIndex;
+import org.apache.cassandra.exceptions.UnknownColumnException;
import org.apache.cassandra.io.FSError;
import org.apache.cassandra.io.compress.CompressionMetadata;
import org.apache.cassandra.io.sstable.*;
@@ -66,13 +65,18 @@
import org.apache.cassandra.metrics.RestorableMeter;
import org.apache.cassandra.metrics.StorageMetrics;
import org.apache.cassandra.schema.CachingParams;
-import org.apache.cassandra.schema.IndexMetadata;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.SchemaConstants;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.schema.TableMetadataRef;
import org.apache.cassandra.service.ActiveRepairService;
import org.apache.cassandra.service.CacheService;
import org.apache.cassandra.utils.*;
import org.apache.cassandra.utils.concurrent.OpOrder;
import org.apache.cassandra.utils.concurrent.Ref;
import org.apache.cassandra.utils.concurrent.SelfRefCounted;
+import org.apache.cassandra.utils.BloomFilterSerializer;
import static org.apache.cassandra.db.Directories.SECONDARY_INDEX_NAME_SEPARATOR;
@@ -223,8 +227,8 @@
protected final AtomicLong keyCacheHit = new AtomicLong(0);
protected final AtomicLong keyCacheRequest = new AtomicLong(0);
- private final InstanceTidier tidy = new InstanceTidier(descriptor, metadata);
- private final Ref<SSTableReader> selfRef = new Ref<>(this, tidy);
+ private final InstanceTidier tidy;
+ private final Ref<SSTableReader> selfRef;
private RestorableMeter readMeter;
@@ -243,62 +247,53 @@
{
long count = -1;
- // check if cardinality estimator is available for all SSTables
- boolean cardinalityAvailable = !Iterables.isEmpty(sstables) && Iterables.all(sstables, new Predicate<SSTableReader>()
+ if (Iterables.isEmpty(sstables))
+ return count;
+
+ boolean failed = false;
+ ICardinality cardinality = null;
+ for (SSTableReader sstable : sstables)
{
- public boolean apply(SSTableReader sstable)
+ if (sstable.openReason == OpenReason.EARLY)
+ continue;
+
+ try
{
- return sstable.descriptor.version.hasNewStatsFile();
- }
- });
-
- // if it is, load them to estimate key count
- if (cardinalityAvailable)
- {
- boolean failed = false;
- ICardinality cardinality = null;
- for (SSTableReader sstable : sstables)
- {
- if (sstable.openReason == OpenReason.EARLY)
- continue;
-
- try
+ CompactionMetadata metadata = (CompactionMetadata) sstable.descriptor.getMetadataSerializer().deserialize(sstable.descriptor, MetadataType.COMPACTION);
+ // If we can't load the CompactionMetadata, we are forced to estimate the keys using the index
+ // summary. (CASSANDRA-10676)
+ if (metadata == null)
{
- CompactionMetadata metadata = (CompactionMetadata) sstable.descriptor.getMetadataSerializer().deserialize(sstable.descriptor, MetadataType.COMPACTION);
- // If we can't load the CompactionMetadata, we are forced to estimate the keys using the index
- // summary. (CASSANDRA-10676)
- if (metadata == null)
- {
- logger.warn("Reading cardinality from Statistics.db failed for {}", sstable.getFilename());
- failed = true;
- break;
- }
-
- if (cardinality == null)
- cardinality = metadata.cardinalityEstimator;
- else
- cardinality = cardinality.merge(metadata.cardinalityEstimator);
- }
- catch (IOException e)
- {
- logger.warn("Reading cardinality from Statistics.db failed.", e);
+ logger.warn("Reading cardinality from Statistics.db failed for {}", sstable.getFilename());
failed = true;
break;
}
- catch (CardinalityMergeException e)
- {
- logger.warn("Cardinality merge failed.", e);
- failed = true;
- break;
- }
+
+ if (cardinality == null)
+ cardinality = metadata.cardinalityEstimator;
+ else
+ cardinality = cardinality.merge(metadata.cardinalityEstimator);
}
- if (cardinality != null && !failed)
- count = cardinality.cardinality();
+ catch (IOException e)
+ {
+ logger.warn("Reading cardinality from Statistics.db failed.", e);
+ failed = true;
+ break;
+ }
+ catch (CardinalityMergeException e)
+ {
+ logger.warn("Cardinality merge failed.", e);
+ failed = true;
+ break;
+ }
}
+ if (cardinality != null && !failed)
+ count = cardinality.cardinality();
// if something went wrong above or cardinality is not available, calculate using index summary
if (count < 0)
{
+ count = 0;
for (SSTableReader sstable : sstables)
count += sstable.estimatedKeys();
}
@@ -353,46 +348,42 @@
return base;
}
- public static SSTableReader open(Descriptor descriptor) throws IOException
+ public static SSTableReader open(Descriptor descriptor)
{
- CFMetaData metadata;
+ TableMetadataRef metadata;
if (descriptor.cfname.contains(SECONDARY_INDEX_NAME_SEPARATOR))
{
int i = descriptor.cfname.indexOf(SECONDARY_INDEX_NAME_SEPARATOR);
- String parentName = descriptor.cfname.substring(0, i);
String indexName = descriptor.cfname.substring(i + 1);
- CFMetaData parent = Schema.instance.getCFMetaData(descriptor.ksname, parentName);
- IndexMetadata def = parent.getIndexes()
- .get(indexName)
- .orElseThrow(() -> new AssertionError(
- "Could not find index metadata for index cf " + i));
- metadata = CassandraIndex.indexCfsMetadata(parent, def);
+ metadata = Schema.instance.getIndexTableMetadataRef(descriptor.ksname, indexName);
+ if (metadata == null)
+ throw new AssertionError("Could not find index metadata for index cf " + i);
}
else
{
- metadata = Schema.instance.getCFMetaData(descriptor.ksname, descriptor.cfname);
+ metadata = Schema.instance.getTableMetadataRef(descriptor.ksname, descriptor.cfname);
}
return open(descriptor, metadata);
}
- public static SSTableReader open(Descriptor desc, CFMetaData metadata) throws IOException
+ public static SSTableReader open(Descriptor desc, TableMetadataRef metadata)
{
return open(desc, componentsFor(desc), metadata);
}
- public static SSTableReader open(Descriptor descriptor, Set<Component> components, CFMetaData metadata) throws IOException
+ public static SSTableReader open(Descriptor descriptor, Set<Component> components, TableMetadataRef metadata)
{
return open(descriptor, components, metadata, true, false);
}
// use only for offline or "Standalone" operations
- public static SSTableReader openNoValidation(Descriptor descriptor, Set<Component> components, ColumnFamilyStore cfs) throws IOException
+ public static SSTableReader openNoValidation(Descriptor descriptor, Set<Component> components, ColumnFamilyStore cfs)
{
return open(descriptor, components, cfs.metadata, false, true);
}
// use only for offline or "Standalone" operations
- public static SSTableReader openNoValidation(Descriptor descriptor, CFMetaData metadata) throws IOException
+ public static SSTableReader openNoValidation(Descriptor descriptor, TableMetadataRef metadata)
{
return open(descriptor, componentsFor(descriptor), metadata, false, true);
}
@@ -406,14 +397,22 @@
* @return opened SSTableReader
* @throws IOException
*/
- public static SSTableReader openForBatch(Descriptor descriptor, Set<Component> components, CFMetaData metadata) throws IOException
+ public static SSTableReader openForBatch(Descriptor descriptor, Set<Component> components, TableMetadataRef metadata)
{
// Minimum components without which we can't do anything
assert components.contains(Component.DATA) : "Data component is missing for sstable " + descriptor;
assert components.contains(Component.PRIMARY_INDEX) : "Primary index component is missing for sstable " + descriptor;
EnumSet<MetadataType> types = EnumSet.of(MetadataType.VALIDATION, MetadataType.STATS, MetadataType.HEADER);
- Map<MetadataType, MetadataComponent> sstableMetadata = descriptor.getMetadataSerializer().deserialize(descriptor, types);
+ Map<MetadataType, MetadataComponent> sstableMetadata;
+ try
+ {
+ sstableMetadata = descriptor.getMetadataSerializer().deserialize(descriptor, types);
+ }
+ catch (IOException e)
+ {
+ throw new CorruptSSTableException(e, descriptor.filenameFor(Component.STATS));
+ }
ValidationMetadata validationMetadata = (ValidationMetadata) sstableMetadata.get(MetadataType.VALIDATION);
StatsMetadata statsMetadata = (StatsMetadata) sstableMetadata.get(MetadataType.STATS);
@@ -422,7 +421,7 @@
// Check if sstable is created using same partitioner.
// Partitioner can be null, which indicates older version of sstable or no stats available.
// In that case, we skip the check.
- String partitionerName = metadata.partitioner.getClass().getCanonicalName();
+ String partitionerName = metadata.get().partitioner.getClass().getCanonicalName();
if (validationMetadata != null && !partitionerName.equals(validationMetadata.partitioner))
{
logger.error("Cannot open {}; partitioner {} does not match system partitioner {}. Note that the default partitioner starting with Cassandra 1.2 is Murmur3Partitioner, so you will need to edit that to match your old partitioner if upgrading.",
@@ -431,14 +430,23 @@
}
long fileLength = new File(descriptor.filenameFor(Component.DATA)).length();
- logger.debug("Opening {} ({})", descriptor, FBUtilities.prettyPrintMemory(fileLength));
- SSTableReader sstable = internalOpen(descriptor,
- components,
- metadata,
- System.currentTimeMillis(),
- statsMetadata,
- OpenReason.NORMAL,
- header == null? null : header.toHeader(metadata));
+ logger.info("Opening {} ({})", descriptor, FBUtilities.prettyPrintMemory(fileLength));
+
+ final SSTableReader sstable;
+ try
+ {
+ sstable = internalOpen(descriptor,
+ components,
+ metadata,
+ System.currentTimeMillis(),
+ statsMetadata,
+ OpenReason.NORMAL,
+ header.toHeader(metadata.get()));
+ }
+ catch (UnknownColumnException e)
+ {
+ throw new IllegalStateException(e);
+ }
try(FileHandle.Builder ibuilder = new FileHandle.Builder(sstable.descriptor.filenameFor(Component.PRIMARY_INDEX))
.mmapped(DatabaseDescriptor.getIndexAccessMode() == Config.DiskAccessMode.mmap)
@@ -448,7 +456,16 @@
.withChunkCache(ChunkCache.instance))
{
if (!sstable.loadSummary())
- sstable.buildSummary(false, false, Downsampling.BASE_SAMPLING_LEVEL);
+ {
+ try
+ {
+ sstable.buildSummary(false, false, Downsampling.BASE_SAMPLING_LEVEL);
+ }
+ catch(IOException e)
+ {
+ throw new CorruptSSTableException(e, sstable.getFilename());
+ }
+ }
long indexFileLength = new File(descriptor.filenameFor(Component.PRIMARY_INDEX)).length();
int dataBufferSize = sstable.optimizationStrategy.bufferSize(statsMetadata.estimatedPartitionSize.percentile(DatabaseDescriptor.getDiskOptimizationEstimatePercentile()));
int indexBufferSize = sstable.optimizationStrategy.bufferSize(indexFileLength / sstable.indexSummary.size());
@@ -473,16 +490,16 @@
*/
public static SSTableReader open(Descriptor descriptor,
Set<Component> components,
- CFMetaData metadata,
+ TableMetadataRef metadata,
boolean validate,
- boolean isOffline) throws IOException
+ boolean isOffline)
{
// Minimum components without which we can't do anything
assert components.contains(Component.DATA) : "Data component is missing for sstable " + descriptor;
assert !validate || components.contains(Component.PRIMARY_INDEX) : "Primary index component is missing for sstable " + descriptor;
// For the 3.0+ sstable format, the (misnomed) stats component hold the serialization header which we need to deserialize the sstable content
- assert !descriptor.version.storeRows() || components.contains(Component.STATS) : "Stats component is missing for sstable " + descriptor;
+ assert components.contains(Component.STATS) : "Stats component is missing for sstable " + descriptor;
EnumSet<MetadataType> types = EnumSet.of(MetadataType.VALIDATION, MetadataType.STATS, MetadataType.HEADER);
@@ -498,12 +515,12 @@
ValidationMetadata validationMetadata = (ValidationMetadata) sstableMetadata.get(MetadataType.VALIDATION);
StatsMetadata statsMetadata = (StatsMetadata) sstableMetadata.get(MetadataType.STATS);
SerializationHeader.Component header = (SerializationHeader.Component) sstableMetadata.get(MetadataType.HEADER);
- assert !descriptor.version.storeRows() || header != null;
+ assert header != null;
// Check if sstable is created using same partitioner.
// Partitioner can be null, which indicates older version of sstable or no stats available.
// In that case, we skip the check.
- String partitionerName = metadata.partitioner.getClass().getCanonicalName();
+ String partitionerName = metadata.get().partitioner.getClass().getCanonicalName();
if (validationMetadata != null && !partitionerName.equals(validationMetadata.partitioner))
{
logger.error("Cannot open {}; partitioner {} does not match system partitioner {}. Note that the default partitioner starting with Cassandra 1.2 is Murmur3Partitioner, so you will need to edit that to match your old partitioner if upgrading.",
@@ -512,14 +529,23 @@
}
long fileLength = new File(descriptor.filenameFor(Component.DATA)).length();
- logger.debug("Opening {} ({})", descriptor, FBUtilities.prettyPrintMemory(fileLength));
- SSTableReader sstable = internalOpen(descriptor,
- components,
- metadata,
- System.currentTimeMillis(),
- statsMetadata,
- OpenReason.NORMAL,
- header == null ? null : header.toHeader(metadata));
+ logger.info("Opening {} ({})", descriptor, FBUtilities.prettyPrintMemory(fileLength));
+
+ final SSTableReader sstable;
+ try
+ {
+ sstable = internalOpen(descriptor,
+ components,
+ metadata,
+ System.currentTimeMillis(),
+ statsMetadata,
+ OpenReason.NORMAL,
+ header.toHeader(metadata.get()));
+ }
+ catch (UnknownColumnException e)
+ {
+ throw new IllegalStateException(e);
+ }
try
{
@@ -544,16 +570,8 @@
}
}
- public static void logOpenException(Descriptor descriptor, IOException e)
- {
- if (e instanceof FileNotFoundException)
- logger.error("Missing sstable component in {}; skipped because of {}", descriptor, e.getMessage());
- else
- logger.error("Corrupt sstable {}; skipped", descriptor, e);
- }
-
public static Collection<SSTableReader> openAll(Set<Map.Entry<Descriptor, Set<Component>>> entries,
- final CFMetaData metadata)
+ final TableMetadataRef metadata)
{
final Collection<SSTableReader> sstables = new LinkedBlockingQueue<>();
@@ -581,12 +599,6 @@
logger.error("Cannot read sstable {}; file system error, skipping table", entry, ex);
return;
}
- catch (IOException ex)
- {
- FileUtils.handleCorruptSSTable(new CorruptSSTableException(ex, entry.getKey().filenameFor(Component.DATA)));
- logger.error("Cannot read sstable {}; other IO error, skipping table", entry, ex);
- return;
- }
sstables.add(sstable);
}
};
@@ -612,7 +624,7 @@
*/
public static SSTableReader internalOpen(Descriptor desc,
Set<Component> components,
- CFMetaData metadata,
+ TableMetadataRef metadata,
FileHandle ifile,
FileHandle dfile,
IndexSummary isummary,
@@ -637,12 +649,12 @@
private static SSTableReader internalOpen(final Descriptor descriptor,
- Set<Component> components,
- CFMetaData metadata,
- Long maxDataAge,
- StatsMetadata sstableMetadata,
- OpenReason openReason,
- SerializationHeader header)
+ Set<Component> components,
+ TableMetadataRef metadata,
+ Long maxDataAge,
+ StatsMetadata sstableMetadata,
+ OpenReason openReason,
+ SerializationHeader header)
{
Factory readerFactory = descriptor.getFormat().getReaderFactory();
@@ -651,7 +663,7 @@
protected SSTableReader(final Descriptor desc,
Set<Component> components,
- CFMetaData metadata,
+ TableMetadataRef metadata,
long maxDataAge,
StatsMetadata sstableMetadata,
OpenReason openReason,
@@ -662,7 +674,9 @@
this.header = header;
this.maxDataAge = maxDataAge;
this.openReason = openReason;
- this.rowIndexEntrySerializer = descriptor.version.getSSTableFormat().getIndexSerializer(metadata, desc.version, header);
+ this.rowIndexEntrySerializer = descriptor.version.getSSTableFormat().getIndexSerializer(metadata.get(), desc.version, header);
+ tidy = new InstanceTidier(descriptor, metadata.id);
+ selfRef = new Ref<>(this, tidy);
}
public static long getTotalBytes(Iterable<SSTableReader> sstables)
@@ -702,17 +716,15 @@
// under normal operation we can do this at any time, but SSTR is also used outside C* proper,
// e.g. by BulkLoader, which does not initialize the cache. As a kludge, we set up the cache
// here when we know we're being wired into the rest of the server infrastructure.
- keyCache = CacheService.instance.keyCache;
- final ColumnFamilyStore cfs = Schema.instance.getColumnFamilyStoreInstance(metadata.cfId);
+ InstrumentingCache<KeyCacheKey, RowIndexEntry> maybeKeyCache = CacheService.instance.keyCache;
+ if (maybeKeyCache.getCapacity() > 0)
+ keyCache = maybeKeyCache;
+
+ final ColumnFamilyStore cfs = Schema.instance.getColumnFamilyStoreInstance(metadata().id);
if (cfs != null)
setCrcCheckChance(cfs.getCrcCheckChance());
}
- public boolean isKeyCacheSetup()
- {
- return keyCache != null;
- }
-
/**
* See {@link #load(boolean, boolean)}
* @param validation Metadata for SSTable being loaded
@@ -721,7 +733,7 @@
*/
private void load(ValidationMetadata validation, boolean isOffline) throws IOException
{
- if (metadata.params.bloomFilterFpChance == 1.0)
+ if (metadata().params.bloomFilterFpChance == 1.0)
{
// bf is disabled.
load(false, !isOffline);
@@ -744,7 +756,7 @@
{
// bf is enabled and fp chance matches the currently configured value.
load(false, !isOffline);
- loadBloomFilter(descriptor.version.hasOldBfHashOrder());
+ loadBloomFilter(descriptor.version.hasOldBfFormat());
}
}
@@ -752,12 +764,13 @@
* Load bloom filter from Filter.db file.
*
* @throws IOException
+ * @param oldBfFormat
*/
- private void loadBloomFilter(boolean oldBfHashOrder) throws IOException
+ private void loadBloomFilter(boolean oldBfFormat) throws IOException
{
- try (DataInputStream stream = new DataInputStream(new BufferedInputStream(new FileInputStream(descriptor.filenameFor(Component.FILTER)))))
+ try (DataInputStream stream = new DataInputStream(new BufferedInputStream(Files.newInputStream(Paths.get(descriptor.filenameFor(Component.FILTER))))))
{
- bf = FilterFactory.deserialize(stream, true, oldBfHashOrder);
+ bf = BloomFilterSerializer.deserialize(stream, oldBfFormat);
}
}
@@ -833,8 +846,11 @@
*/
private void buildSummary(boolean recreateBloomFilter, boolean summaryLoaded, int samplingLevel) throws IOException
{
- if (!components.contains(Component.PRIMARY_INDEX))
- return;
+ if (!components.contains(Component.PRIMARY_INDEX))
+ return;
+
+ if (logger.isDebugEnabled())
+ logger.debug("Attempting to build summary for {}", descriptor);
// we read the positions in a BRAF so we don't have to worry about an entry spanning a mmap boundary.
try (RandomAccessReader primaryIndex = RandomAccessReader.open(new File(descriptor.filenameFor(Component.PRIMARY_INDEX))))
@@ -846,9 +862,9 @@
: estimateRowsFromIndex(primaryIndex); // statistics is supposed to be optional
if (recreateBloomFilter)
- bf = FilterFactory.getFilter(estimatedKeys, metadata.params.bloomFilterFpChance, true, descriptor.version.hasOldBfHashOrder());
+ bf = FilterFactory.getFilter(estimatedKeys, metadata().params.bloomFilterFpChance);
- try (IndexSummaryBuilder summaryBuilder = summaryLoaded ? null : new IndexSummaryBuilder(estimatedKeys, metadata.params.minIndexInterval, samplingLevel))
+ try (IndexSummaryBuilder summaryBuilder = summaryLoaded ? null : new IndexSummaryBuilder(estimatedKeys, metadata().params.minIndexInterval, samplingLevel))
{
long indexPosition;
@@ -893,14 +909,19 @@
{
File summariesFile = new File(descriptor.filenameFor(Component.SUMMARY));
if (!summariesFile.exists())
+ {
+ if (logger.isDebugEnabled())
+ logger.debug("SSTable Summary File {} does not exist", summariesFile.getAbsolutePath());
return false;
+ }
DataInputStream iStream = null;
try
{
- iStream = new DataInputStream(new FileInputStream(summariesFile));
+ TableMetadata metadata = metadata();
+ iStream = new DataInputStream(Files.newInputStream(summariesFile.toPath()));
indexSummary = IndexSummary.serializer.deserialize(
- iStream, getPartitioner(), descriptor.version.hasSamplingLevel(),
+ iStream, getPartitioner(),
metadata.params.minIndexInterval, metadata.params.maxIndexInterval);
first = decorateKey(ByteBufferUtil.readWithLength(iStream));
last = decorateKey(ByteBufferUtil.readWithLength(iStream));
@@ -949,7 +970,7 @@
try (DataOutputStreamPlus oStream = new BufferedDataOutputStreamPlus(new FileOutputStream(summariesFile));)
{
- IndexSummary.serializer.serialize(summary, oStream, descriptor.version.hasSamplingLevel());
+ IndexSummary.serializer.serialize(summary, oStream);
ByteBufferUtil.writeWithLength(first.getKey(), oStream);
ByteBufferUtil.writeWithLength(last.getKey(), oStream);
}
@@ -973,7 +994,7 @@
File filterFile = new File(descriptor.filenameFor(Component.FILTER));
try (DataOutputStreamPlus stream = new BufferedDataOutputStreamPlus(new FileOutputStream(filterFile)))
{
- FilterFactory.serialize(filter, stream);
+ BloomFilterSerializer.serialize((BloomFilter) filter, stream);
stream.flush();
}
catch (IOException e)
@@ -1147,14 +1168,12 @@
@SuppressWarnings("resource")
public SSTableReader cloneWithNewSummarySamplingLevel(ColumnFamilyStore parent, int samplingLevel) throws IOException
{
- assert descriptor.version.hasSamplingLevel();
-
synchronized (tidy.global)
{
assert openReason != OpenReason.EARLY;
- int minIndexInterval = metadata.params.minIndexInterval;
- int maxIndexInterval = metadata.params.maxIndexInterval;
+ int minIndexInterval = metadata().params.minIndexInterval;
+ int maxIndexInterval = metadata().params.maxIndexInterval;
double effectiveInterval = indexSummary.getEffectiveIndexInterval();
IndexSummary newSummary;
@@ -1193,7 +1212,7 @@
try
{
long indexSize = primaryIndex.length();
- try (IndexSummaryBuilder summaryBuilder = new IndexSummaryBuilder(estimatedKeys(), metadata.params.minIndexInterval, newSamplingLevel))
+ try (IndexSummaryBuilder summaryBuilder = new IndexSummaryBuilder(estimatedKeys(), metadata().params.minIndexInterval, newSamplingLevel))
{
long indexPosition;
while ((indexPosition = primaryIndex.getFilePointer()) != indexSize)
@@ -1354,9 +1373,9 @@
public long estimatedKeysForRanges(Collection<Range<Token>> ranges)
{
long sampleKeyCount = 0;
- List<Pair<Integer, Integer>> sampleIndexes = getSampleIndexesForRanges(indexSummary, ranges);
- for (Pair<Integer, Integer> sampleIndexRange : sampleIndexes)
- sampleKeyCount += (sampleIndexRange.right - sampleIndexRange.left + 1);
+ List<IndexesBounds> sampleIndexes = getSampleIndexesForRanges(indexSummary, ranges);
+ for (IndexesBounds sampleIndexRange : sampleIndexes)
+ sampleKeyCount += (sampleIndexRange.upperPosition - sampleIndexRange.lowerPosition + 1);
// adjust for the current sampling level: (BSL / SL) * index_interval_at_full_sampling
long estimatedKeys = sampleKeyCount * ((long) Downsampling.BASE_SAMPLING_LEVEL * indexSummary.getMinIndexInterval()) / indexSummary.getSamplingLevel();
@@ -1388,10 +1407,10 @@
return indexSummary.getKey(index);
}
- private static List<Pair<Integer,Integer>> getSampleIndexesForRanges(IndexSummary summary, Collection<Range<Token>> ranges)
+ private static List<IndexesBounds> getSampleIndexesForRanges(IndexSummary summary, Collection<Range<Token>> ranges)
{
// use the index to determine a minimal section for each range
- List<Pair<Integer,Integer>> positions = new ArrayList<>();
+ List<IndexesBounds> positions = new ArrayList<>();
for (Range<Token> range : Range.normalize(ranges))
{
@@ -1425,14 +1444,14 @@
if (left > right)
// empty range
continue;
- positions.add(Pair.create(left, right));
+ positions.add(new IndexesBounds(left, right));
}
return positions;
}
public Iterable<DecoratedKey> getKeySamples(final Range<Token> range)
{
- final List<Pair<Integer, Integer>> indexRanges = getSampleIndexesForRanges(indexSummary, Collections.singletonList(range));
+ final List<IndexesBounds> indexRanges = getSampleIndexesForRanges(indexSummary, Collections.singletonList(range));
if (indexRanges.isEmpty())
return Collections.emptyList();
@@ -1443,18 +1462,18 @@
{
return new Iterator<DecoratedKey>()
{
- private Iterator<Pair<Integer, Integer>> rangeIter = indexRanges.iterator();
- private Pair<Integer, Integer> current;
+ private Iterator<IndexesBounds> rangeIter = indexRanges.iterator();
+ private IndexesBounds current;
private int idx;
public boolean hasNext()
{
- if (current == null || idx > current.right)
+ if (current == null || idx > current.upperPosition)
{
if (rangeIter.hasNext())
{
current = rangeIter.next();
- idx = current.left;
+ idx = current.lowerPosition;
return true;
}
return false;
@@ -1482,10 +1501,10 @@
* Determine the minimal set of sections that can be extracted from this SSTable to cover the given ranges.
* @return A sorted list of (offset,end) pairs that cover the given ranges in the datafile for this SSTable.
*/
- public List<Pair<Long,Long>> getPositionsForRanges(Collection<Range<Token>> ranges)
+ public List<PartitionPositionBounds> getPositionsForRanges(Collection<Range<Token>> ranges)
{
// use the index to determine a minimal section for each range
- List<Pair<Long,Long>> positions = new ArrayList<>();
+ List<PartitionPositionBounds> positions = new ArrayList<>();
for (Range<Token> range : Range.normalize(ranges))
{
assert !range.isWrapAround() || range.right.isMinimum();
@@ -1507,36 +1526,38 @@
continue;
assert left < right : String.format("Range=%s openReason=%s first=%s last=%s left=%d right=%d", range, openReason, first, last, left, right);
- positions.add(Pair.create(left, right));
+ positions.add(new PartitionPositionBounds(left, right));
}
return positions;
}
public KeyCacheKey getCacheKey(DecoratedKey key)
{
- return new KeyCacheKey(metadata.ksAndCFName, descriptor, key.getKey());
+ return new KeyCacheKey(metadata(), descriptor, key.getKey());
}
public void cacheKey(DecoratedKey key, RowIndexEntry info)
{
- CachingParams caching = metadata.params.caching;
+ CachingParams caching = metadata().params.caching;
if (!caching.cacheKeys() || keyCache == null || keyCache.getCapacity() == 0)
return;
- KeyCacheKey cacheKey = new KeyCacheKey(metadata.ksAndCFName, descriptor, key.getKey());
+ KeyCacheKey cacheKey = new KeyCacheKey(metadata(), descriptor, key.getKey());
logger.trace("Adding cache entry for {} -> {}", cacheKey, info);
keyCache.put(cacheKey, info);
}
public RowIndexEntry getCachedPosition(DecoratedKey key, boolean updateStats)
{
- return getCachedPosition(new KeyCacheKey(metadata.ksAndCFName, descriptor, key.getKey()), updateStats);
+ if (isKeyCacheEnabled())
+ return getCachedPosition(new KeyCacheKey(metadata(), descriptor, key.getKey()), updateStats);
+ return null;
}
protected RowIndexEntry getCachedPosition(KeyCacheKey unifiedKey, boolean updateStats)
{
- if (keyCache != null && keyCache.getCapacity() > 0 && metadata.params.caching.cacheKeys())
+ if (isKeyCacheEnabled())
{
if (updateStats)
{
@@ -1557,6 +1578,11 @@
return null;
}
+ public boolean isKeyCacheEnabled()
+ {
+ return keyCache != null && metadata().params.caching.cacheKeys();
+ }
+
/**
* Retrieves the position while updating the key cache and the stats.
* @param key The key to apply as the rhs to the given Operator. A 'fake' key is allowed to
@@ -1580,10 +1606,13 @@
return getPosition(key, op, true, false, listener);
}
- public final RowIndexEntry getPosition(PartitionPosition key, Operator op, boolean updateCacheAndStats)
+ public final RowIndexEntry getPosition(PartitionPosition key,
+ Operator op,
+ boolean updateCacheAndStats)
{
return getPosition(key, op, updateCacheAndStats, false, SSTableReadsListener.NOOP_LISTENER);
}
+
/**
* @param key The key to apply as the rhs to the given Operator. A 'fake' key is allowed to
* allow key selection by token bounds but only if op != * EQ
@@ -1602,10 +1631,9 @@
Slices slices,
ColumnFilter selectedColumns,
boolean reversed,
- boolean isForThrift,
SSTableReadsListener listener);
- public abstract UnfilteredRowIterator iterator(FileDataInput file, DecoratedKey key, RowIndexEntry indexEntry, Slices slices, ColumnFilter selectedColumns, boolean reversed, boolean isForThrift);
+ public abstract UnfilteredRowIterator iterator(FileDataInput file, DecoratedKey key, RowIndexEntry indexEntry, Slices slices, ColumnFilter selectedColumns, boolean reversed);
public abstract UnfilteredRowIterator simpleIterator(FileDataInput file, DecoratedKey key, RowIndexEntry indexEntry, boolean tombstoneOnly);
@@ -1719,46 +1747,28 @@
isSuspect.getAndSet(true);
}
+ @VisibleForTesting
+ public void unmarkSuspect()
+ {
+ isSuspect.getAndSet(false);
+ }
+
public boolean isMarkedSuspect()
{
return isSuspect.get();
}
-
- /**
- * I/O SSTableScanner
- * @return A Scanner for seeking over the rows of the SSTable.
- */
- public ISSTableScanner getScanner()
- {
- return getScanner((RateLimiter) null);
- }
-
- /**
- * @param columns the columns to return.
- * @param dataRange filter to use when reading the columns
- * @param listener a listener used to handle internal read events
- * @return A Scanner for seeking over the rows of the SSTable.
- */
- public ISSTableScanner getScanner(ColumnFilter columns,
- DataRange dataRange,
- boolean isForThrift,
- SSTableReadsListener listener)
- {
- return getScanner(columns, dataRange, null, isForThrift, listener);
- }
-
/**
* Direct I/O SSTableScanner over a defined range of tokens.
*
* @param range the range of keys to cover
* @return A Scanner for seeking over the rows of the SSTable.
*/
- public ISSTableScanner getScanner(Range<Token> range, RateLimiter limiter)
+ public ISSTableScanner getScanner(Range<Token> range)
{
if (range == null)
- return getScanner(limiter);
- return getScanner(Collections.singletonList(range), limiter);
+ return getScanner();
+ return getScanner(Collections.singletonList(range));
}
/**
@@ -1766,7 +1776,7 @@
*
* @return A Scanner over the full content of the SSTable.
*/
- public abstract ISSTableScanner getScanner(RateLimiter limiter);
+ public abstract ISSTableScanner getScanner();
/**
* Direct I/O SSTableScanner over a defined collection of ranges of tokens.
@@ -1774,7 +1784,7 @@
* @param ranges the range of keys to cover
* @return A Scanner for seeking over the rows of the SSTable.
*/
- public abstract ISSTableScanner getScanner(Collection<Range<Token>> ranges, RateLimiter limiter);
+ public abstract ISSTableScanner getScanner(Collection<Range<Token>> ranges);
/**
* Direct I/O SSTableScanner over an iterator of bounds.
@@ -1790,11 +1800,7 @@
* @param listener a listener used to handle internal read events
* @return A Scanner for seeking over the rows of the SSTable.
*/
- public abstract ISSTableScanner getScanner(ColumnFilter columns,
- DataRange dataRange,
- RateLimiter limiter,
- boolean isForThrift,
- SSTableReadsListener listener);
+ public abstract ISSTableScanner getScanner(ColumnFilter columns, DataRange dataRange, SSTableReadsListener listener);
public FileDataInput getFileDataInput(long position)
{
@@ -1847,13 +1853,39 @@
// hint read path about key location if caching is enabled
// this saves index summary lookup and index file iteration which whould be pretty costly
// especially in presence of promoted column indexes
- if (isKeyCacheSetup())
- cacheKey(key, rowIndexEntrySerializer.deserialize(in, in.getFilePointer()));
+ if (isKeyCacheEnabled())
+ cacheKey(key, rowIndexEntrySerializer.deserialize(in));
}
return key;
}
+ public boolean isPendingRepair()
+ {
+ return sstableMetadata.pendingRepair != ActiveRepairService.NO_PENDING_REPAIR;
+ }
+
+ public UUID getPendingRepair()
+ {
+ return sstableMetadata.pendingRepair;
+ }
+
+ public long getRepairedAt()
+ {
+ return sstableMetadata.repairedAt;
+ }
+
+ public boolean isTransient()
+ {
+ return sstableMetadata.isTransient;
+ }
+
+ public boolean intersects(Collection<Range<Token>> ranges)
+ {
+ Bounds<Token> range = new Bounds<>(first.getToken(), last.getToken());
+ return Iterables.any(ranges, r -> r.intersects(range));
+ }
+
/**
* TODO: Move someplace reusable
*/
@@ -1915,9 +1947,9 @@
return sstableMetadata.estimatedPartitionSize;
}
- public EstimatedHistogram getEstimatedColumnCount()
+ public EstimatedHistogram getEstimatedCellPerPartitionCount()
{
- return sstableMetadata.estimatedColumnCount;
+ return sstableMetadata.estimatedCellPerPartitionCount;
}
public double getEstimatedDroppableTombstoneRatio(int gcBefore)
@@ -1960,15 +1992,13 @@
* <p>
* Note that having that method return {@code false} guarantees the sstable has no tombstones whatsoever (so no
* cell tombstone, no range tombstone maker and no expiring columns), but having it return {@code true} doesn't
- * guarantee it contains any as 1) it may simply have non-expired cells and 2) old-format sstables didn't contain
- * enough information to decide this and so always return {@code true}.
+ * guarantee it contains any as it may simply have non-expired cells.
*/
public boolean mayHaveTombstones()
{
- // A sstable is guaranteed to have no tombstones if it properly tracked the minLocalDeletionTime (which we only
- // do since 3.0 - see CASSANDRA-13366) and that value is still set to its default, Cell.NO_DELETION_TIME, which
- // is bigger than any valid deletion times.
- return !descriptor.version.storeRows() || getMinLocalDeletionTime() != Cell.NO_DELETION_TIME;
+ // A sstable is guaranteed to have no tombstones if minLocalDeletionTime is still set to its default,
+ // Cell.NO_DELETION_TIME, which is bigger than any valid deletion times.
+ return getMinLocalDeletionTime() != Cell.NO_DELETION_TIME;
}
public int getMinTTL()
@@ -2094,7 +2124,7 @@
{
// We could return sstable.header.stats(), but this may not be as accurate than the actual sstable stats (see
// SerializationHeader.make() for details) so we use the latter instead.
- return new EncodingStats(getMinTimestamp(), getMinLocalDeletionTime(), getMinTTL());
+ return sstableMetadata.encodingStats;
}
public Ref<SSTableReader> tryRef()
@@ -2147,7 +2177,7 @@
private static final class InstanceTidier implements Tidy
{
private final Descriptor descriptor;
- private final CFMetaData metadata;
+ private final TableId tableId;
private IFilter bf;
private IndexSummary summary;
@@ -2177,10 +2207,10 @@
global.ensureReadMeter();
}
- InstanceTidier(Descriptor descriptor, CFMetaData metadata)
+ InstanceTidier(Descriptor descriptor, TableId tableId)
{
this.descriptor = descriptor;
- this.metadata = metadata;
+ this.tableId = tableId;
}
public void tidy()
@@ -2192,7 +2222,7 @@
if (!setup)
return;
- final ColumnFamilyStore cfs = Schema.instance.getColumnFamilyStoreInstance(metadata.cfId);
+ final ColumnFamilyStore cfs = Schema.instance.getColumnFamilyStoreInstance(tableId);
final OpOrder.Barrier barrier;
if (cfs != null)
{
@@ -2362,7 +2392,7 @@
{
public abstract SSTableReader open(final Descriptor descriptor,
Set<Component> components,
- CFMetaData metadata,
+ TableMetadataRef metadata,
Long maxDataAge,
StatsMetadata sstableMetadata,
OpenReason openReason,
@@ -2370,6 +2400,104 @@
}
+ public static class PartitionPositionBounds
+ {
+ public final long lowerPosition;
+ public final long upperPosition;
+
+ public PartitionPositionBounds(long lower, long upper)
+ {
+ this.lowerPosition = lower;
+ this.upperPosition = upper;
+ }
+
+ @Override
+ public final int hashCode()
+ {
+ int hashCode = (int) lowerPosition ^ (int) (lowerPosition >>> 32);
+ return 31 * (hashCode ^ (int) ((int) upperPosition ^ (upperPosition >>> 32)));
+ }
+
+ @Override
+ public final boolean equals(Object o)
+ {
+ if(!(o instanceof PartitionPositionBounds))
+ return false;
+ PartitionPositionBounds that = (PartitionPositionBounds)o;
+ return lowerPosition == that.lowerPosition && upperPosition == that.upperPosition;
+ }
+ }
+
+ public static class IndexesBounds
+ {
+ public final int lowerPosition;
+ public final int upperPosition;
+
+ public IndexesBounds(int lower, int upper)
+ {
+ this.lowerPosition = lower;
+ this.upperPosition = upper;
+ }
+
+ @Override
+ public final int hashCode()
+ {
+ return 31 * lowerPosition * upperPosition;
+ }
+
+ @Override
+ public final boolean equals(Object o)
+ {
+ if (!(o instanceof IndexesBounds))
+ return false;
+ IndexesBounds that = (IndexesBounds) o;
+ return lowerPosition == that.lowerPosition && upperPosition == that.upperPosition;
+ }
+ }
+
+ /**
+ * Moves the sstable in oldDescriptor to a new place (with generation etc) in newDescriptor.
+ *
+ * All components given will be moved/renamed
+ */
+ public static SSTableReader moveAndOpenSSTable(ColumnFamilyStore cfs, Descriptor oldDescriptor, Descriptor newDescriptor, Set<Component> components)
+ {
+ if (!oldDescriptor.isCompatible())
+ throw new RuntimeException(String.format("Can't open incompatible SSTable! Current version %s, found file: %s",
+ oldDescriptor.getFormat().getLatestVersion(),
+ oldDescriptor));
+
+ boolean isLive = cfs.getLiveSSTables().stream().anyMatch(r -> r.descriptor.equals(newDescriptor)
+ || r.descriptor.equals(oldDescriptor));
+ if (isLive)
+ {
+ String message = String.format("Can't move and open a file that is already in use in the table %s -> %s", oldDescriptor, newDescriptor);
+ logger.error(message);
+ throw new RuntimeException(message);
+ }
+ if (new File(newDescriptor.filenameFor(Component.DATA)).exists())
+ {
+ String msg = String.format("File %s already exists, can't move the file there", newDescriptor.filenameFor(Component.DATA));
+ logger.error(msg);
+ throw new RuntimeException(msg);
+ }
+
+ logger.info("Renaming new SSTable {} to {}", oldDescriptor, newDescriptor);
+ SSTableWriter.rename(oldDescriptor, newDescriptor, components);
+
+ SSTableReader reader;
+ try
+ {
+ reader = SSTableReader.open(newDescriptor, components, cfs.metadata);
+ }
+ catch (Throwable t)
+ {
+ logger.error("Aborting import of sstables. {} was corrupt", newDescriptor);
+ throw new RuntimeException(newDescriptor + " is corrupt, can't import", t);
+ }
+ return reader;
+ }
+
public static void shutdownBlocking(long timeout, TimeUnit unit) throws InterruptedException, TimeoutException
{
diff --git a/src/java/org/apache/cassandra/io/sstable/format/SSTableReadsListener.java b/src/java/org/apache/cassandra/io/sstable/format/SSTableReadsListener.java
index 8f6e3c0..6d384bf 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/SSTableReadsListener.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/SSTableReadsListener.java
@@ -18,7 +18,6 @@
package org.apache.cassandra.io.sstable.format;
import org.apache.cassandra.db.RowIndexEntry;
-import org.apache.cassandra.io.sstable.format.SSTableReadsListener.SelectionReason;
/**
* Listener for receiving notifications associated with reading SSTables.
diff --git a/src/java/org/apache/cassandra/io/sstable/format/SSTableWriter.java b/src/java/org/apache/cassandra/io/sstable/format/SSTableWriter.java
index e320f30..f54bc03 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/SSTableWriter.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/SSTableWriter.java
@@ -24,9 +24,7 @@
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Sets;
-import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.config.Schema;
import org.apache.cassandra.db.RowIndexEntry;
import org.apache.cassandra.db.SerializationHeader;
import org.apache.cassandra.db.compaction.OperationType;
@@ -37,12 +35,14 @@
import org.apache.cassandra.io.sstable.Component;
import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.io.sstable.SSTable;
-import org.apache.cassandra.io.sstable.format.big.BigFormat;
import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
import org.apache.cassandra.io.sstable.metadata.MetadataComponent;
import org.apache.cassandra.io.sstable.metadata.MetadataType;
import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.schema.TableMetadataRef;
import org.apache.cassandra.utils.concurrent.Transactional;
/**
@@ -54,6 +54,8 @@
public abstract class SSTableWriter extends SSTable implements Transactional
{
protected long repairedAt;
+ protected UUID pendingRepair;
+ protected boolean isTransient;
protected long maxDataAge = -1;
protected final long keyCount;
protected final MetadataCollector metadataCollector;
@@ -75,89 +77,89 @@
protected SSTableWriter(Descriptor descriptor,
long keyCount,
long repairedAt,
- CFMetaData metadata,
+ UUID pendingRepair,
+ boolean isTransient,
+ TableMetadataRef metadata,
MetadataCollector metadataCollector,
SerializationHeader header,
Collection<SSTableFlushObserver> observers)
{
- super(descriptor, components(metadata), metadata, DatabaseDescriptor.getDiskOptimizationStrategy());
+ super(descriptor, components(metadata.get()), metadata, DatabaseDescriptor.getDiskOptimizationStrategy());
this.keyCount = keyCount;
this.repairedAt = repairedAt;
+ this.pendingRepair = pendingRepair;
+ this.isTransient = isTransient;
this.metadataCollector = metadataCollector;
- this.header = header != null ? header : SerializationHeader.makeWithoutStats(metadata); //null header indicates streaming from pre-3.0 sstable
- this.rowIndexEntrySerializer = descriptor.version.getSSTableFormat().getIndexSerializer(metadata, descriptor.version, header);
+ this.header = header;
+ this.rowIndexEntrySerializer = descriptor.version.getSSTableFormat().getIndexSerializer(metadata.get(), descriptor.version, header);
this.observers = observers == null ? Collections.emptySet() : observers;
}
public static SSTableWriter create(Descriptor descriptor,
Long keyCount,
Long repairedAt,
- CFMetaData metadata,
+ UUID pendingRepair,
+ boolean isTransient,
+ TableMetadataRef metadata,
MetadataCollector metadataCollector,
SerializationHeader header,
Collection<Index> indexes,
LifecycleNewTracker lifecycleNewTracker)
{
Factory writerFactory = descriptor.getFormat().getWriterFactory();
- return writerFactory.open(descriptor, keyCount, repairedAt, metadata, metadataCollector, header, observers(descriptor, indexes, lifecycleNewTracker.opType()), lifecycleNewTracker);
+ return writerFactory.open(descriptor, keyCount, repairedAt, pendingRepair, isTransient, metadata, metadataCollector, header, observers(descriptor, indexes, lifecycleNewTracker.opType()), lifecycleNewTracker);
}
public static SSTableWriter create(Descriptor descriptor,
long keyCount,
long repairedAt,
+ UUID pendingRepair,
+ boolean isTransient,
int sstableLevel,
SerializationHeader header,
Collection<Index> indexes,
LifecycleNewTracker lifecycleNewTracker)
{
- CFMetaData metadata = Schema.instance.getCFMetaData(descriptor);
- return create(metadata, descriptor, keyCount, repairedAt, sstableLevel, header, indexes, lifecycleNewTracker);
+ TableMetadataRef metadata = Schema.instance.getTableMetadataRef(descriptor);
+ return create(metadata, descriptor, keyCount, repairedAt, pendingRepair, isTransient, sstableLevel, header, indexes, lifecycleNewTracker);
}
- public static SSTableWriter create(CFMetaData metadata,
+ public static SSTableWriter create(TableMetadataRef metadata,
Descriptor descriptor,
long keyCount,
long repairedAt,
+ UUID pendingRepair,
+ boolean isTransient,
int sstableLevel,
SerializationHeader header,
Collection<Index> indexes,
LifecycleNewTracker lifecycleNewTracker)
{
- MetadataCollector collector = new MetadataCollector(metadata.comparator).sstableLevel(sstableLevel);
- return create(descriptor, keyCount, repairedAt, metadata, collector, header, indexes, lifecycleNewTracker);
- }
-
- public static SSTableWriter create(String filename,
- long keyCount,
- long repairedAt,
- int sstableLevel,
- SerializationHeader header,
- Collection<Index> indexes,
- LifecycleNewTracker lifecycleNewTracker)
- {
- return create(Descriptor.fromFilename(filename), keyCount, repairedAt, sstableLevel, header, indexes, lifecycleNewTracker);
+ MetadataCollector collector = new MetadataCollector(metadata.get().comparator).sstableLevel(sstableLevel);
+ return create(descriptor, keyCount, repairedAt, pendingRepair, isTransient, metadata, collector, header, indexes, lifecycleNewTracker);
}
@VisibleForTesting
- public static SSTableWriter create(String filename,
+ public static SSTableWriter create(Descriptor descriptor,
long keyCount,
long repairedAt,
+ UUID pendingRepair,
+ boolean isTransient,
SerializationHeader header,
Collection<Index> indexes,
LifecycleNewTracker lifecycleNewTracker)
{
- Descriptor descriptor = Descriptor.fromFilename(filename);
- return create(descriptor, keyCount, repairedAt, 0, header, indexes, lifecycleNewTracker);
+ return create(descriptor, keyCount, repairedAt, pendingRepair, isTransient, 0, header, indexes, lifecycleNewTracker);
}
- private static Set<Component> components(CFMetaData metadata)
+ private static Set<Component> components(TableMetadata metadata)
{
Set<Component> components = new HashSet<Component>(Arrays.asList(Component.DATA,
Component.PRIMARY_INDEX,
Component.STATS,
Component.SUMMARY,
Component.TOC,
- Component.digestFor(BigFormat.latestVersion.uncompressedChecksumType())));
+ Component.DIGEST));
if (metadata.params.bloomFilterFpChance < 1.0)
components.add(Component.FILTER);
@@ -311,8 +313,10 @@
protected Map<MetadataType, MetadataComponent> finalizeMetadata()
{
return metadataCollector.finalizeMetadata(getPartitioner().getClass().getCanonicalName(),
- metadata.params.bloomFilterFpChance,
+ metadata().params.bloomFilterFpChance,
repairedAt,
+ pendingRepair,
+ isTransient,
header);
}
@@ -341,7 +345,9 @@
public abstract SSTableWriter open(Descriptor descriptor,
long keyCount,
long repairedAt,
- CFMetaData metadata,
+ UUID pendingRepair,
+ boolean isTransient,
+ TableMetadataRef metadata,
MetadataCollector metadataCollector,
SerializationHeader header,
Collection<SSTableFlushObserver> observers,
diff --git a/src/java/org/apache/cassandra/io/sstable/format/Version.java b/src/java/org/apache/cassandra/io/sstable/format/Version.java
index 2b9dcbd..0e9e303 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/Version.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/Version.java
@@ -19,7 +19,6 @@
import java.util.regex.Pattern;
-import org.apache.cassandra.utils.ChecksumType;
/**
* A set of feature flags associated with a SSTable format
@@ -46,34 +45,27 @@
public abstract boolean isLatestVersion();
- public abstract boolean hasSamplingLevel();
-
- public abstract boolean hasNewStatsFile();
-
- public abstract ChecksumType compressedChecksumType();
-
- public abstract ChecksumType uncompressedChecksumType();
-
- public abstract boolean hasRepairedAt();
-
- public abstract boolean tracksLegacyCounterShards();
-
- public abstract boolean hasNewFileName();
-
- public abstract boolean storeRows();
-
public abstract int correspondingMessagingVersion(); // Only use by storage that 'storeRows' so far
- public abstract boolean hasOldBfHashOrder();
-
- public abstract boolean hasCompactionAncestors();
-
- public abstract boolean hasBoundaries();
-
public abstract boolean hasCommitLogLowerBound();
public abstract boolean hasCommitLogIntervals();
+ public abstract boolean hasMaxCompressedLength();
+
+ public abstract boolean hasPendingRepair();
+
+ public abstract boolean hasIsTransient();
+
+ public abstract boolean hasMetadataChecksum();
+
+ /**
+ * The old bloomfilter format serializes the data as BIG_ENDIAN long's, the new one uses the
+ * same format as in memory (serializes as bytes).
+ * @return True if the bloomfilter file is old serialization format
+ */
+ public abstract boolean hasOldBfFormat();
+
public abstract boolean hasAccurateMinMax();
public String getVersion()
diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java b/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java
index 9af7dc0..448808c 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java
@@ -19,8 +19,13 @@
import java.util.Collection;
import java.util.Set;
+import java.util.UUID;
-import org.apache.cassandra.config.CFMetaData;
+import com.google.common.base.Preconditions;
+
+import org.apache.cassandra.io.sstable.SSTable;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.schema.TableMetadataRef;
import org.apache.cassandra.db.RowIndexEntry;
import org.apache.cassandra.db.SerializationHeader;
import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
@@ -30,7 +35,6 @@
import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
import org.apache.cassandra.net.MessagingService;
-import org.apache.cassandra.utils.ChecksumType;
/**
* Legacy bigtable format
@@ -72,9 +76,9 @@
}
@Override
- public RowIndexEntry.IndexSerializer getIndexSerializer(CFMetaData metadata, Version version, SerializationHeader header)
+ public RowIndexEntry.IndexSerializer getIndexSerializer(TableMetadata metadata, Version version, SerializationHeader header)
{
- return new RowIndexEntry.Serializer(metadata, version, header);
+ return new RowIndexEntry.Serializer(version, header);
}
static class WriterFactory extends SSTableWriter.Factory
@@ -83,20 +87,23 @@
public SSTableWriter open(Descriptor descriptor,
long keyCount,
long repairedAt,
- CFMetaData metadata,
+ UUID pendingRepair,
+ boolean isTransient,
+ TableMetadataRef metadata,
MetadataCollector metadataCollector,
SerializationHeader header,
Collection<SSTableFlushObserver> observers,
LifecycleNewTracker lifecycleNewTracker)
{
- return new BigTableWriter(descriptor, keyCount, repairedAt, metadata, metadataCollector, header, observers, lifecycleNewTracker);
+ SSTable.validateRepairedMetadata(repairedAt, pendingRepair, isTransient);
+ return new BigTableWriter(descriptor, keyCount, repairedAt, pendingRepair, isTransient, metadata, metadataCollector, header, observers, lifecycleNewTracker);
}
}
static class ReaderFactory extends SSTableReader.Factory
{
@Override
- public SSTableReader open(Descriptor descriptor, Set<Component> components, CFMetaData metadata, Long maxDataAge, StatsMetadata sstableMetadata, SSTableReader.OpenReason openReason, SerializationHeader header)
+ public SSTableReader open(Descriptor descriptor, Set<Component> components, TableMetadataRef metadata, Long maxDataAge, StatsMetadata sstableMetadata, SSTableReader.OpenReason openReason, SerializationHeader header)
{
return new BigTableReader(descriptor, components, metadata, maxDataAge, sstableMetadata, openReason, header);
}
@@ -110,87 +117,50 @@
// we always incremented the major version.
static class BigVersion extends Version
{
- public static final String current_version = "md";
- public static final String earliest_supported_version = "jb";
+ public static final String current_version = "na";
+ public static final String earliest_supported_version = "ma";
- // jb (2.0.1): switch from crc32 to adler32 for compression checksums
- // checksum the compressed data
- // ka (2.1.0): new Statistics.db file format
- // index summaries can be downsampled and the sampling level is persisted
- // switch uncompressed checksums to adler32
- // tracks presense of legacy (local and remote) counter shards
- // la (2.2.0): new file name format
- // lb (2.2.7): commit log lower bound included
// ma (3.0.0): swap bf hash order
// store rows natively
// mb (3.0.7, 3.7): commit log lower bound included
// mc (3.0.8, 3.9): commit log intervals included
// md (3.0.18, 3.11.4): corrected sstable min/max clustering
+
+ // na (4.0.0): uncompressed chunks, pending repair session, isTransient, checksummed sstable metadata file, new Bloomfilter format
//
// NOTE: when adding a new version, please add that to LegacySSTableTest, too.
private final boolean isLatestVersion;
- private final boolean hasSamplingLevel;
- private final boolean newStatsFile;
- private final ChecksumType compressedChecksumType;
- private final ChecksumType uncompressedChecksumType;
- private final boolean hasRepairedAt;
- private final boolean tracksLegacyCounterShards;
- private final boolean newFileName;
- public final boolean storeRows;
- public final int correspondingMessagingVersion; // Only use by storage that 'storeRows' so far
- public final boolean hasBoundaries;
- /**
- * CASSANDRA-8413: 3.0 bloom filter representation changed (two longs just swapped)
- * have no 'static' bits caused by using the same upper bits for both bloom filter and token distribution.
- */
- private final boolean hasOldBfHashOrder;
+ public final int correspondingMessagingVersion;
private final boolean hasCommitLogLowerBound;
private final boolean hasCommitLogIntervals;
private final boolean hasAccurateMinMax;
+ public final boolean hasMaxCompressedLength;
+ private final boolean hasPendingRepair;
+ private final boolean hasMetadataChecksum;
+ private final boolean hasIsTransient;
/**
- * CASSANDRA-7066: compaction ancerstors are no longer used and have been removed.
+ * CASSANDRA-9067: 4.0 bloom filter representation changed (two longs just swapped)
+ * have no 'static' bits caused by using the same upper bits for both bloom filter and token distribution.
*/
- private final boolean hasCompactionAncestors;
+ private final boolean hasOldBfFormat;
BigVersion(String version)
{
super(instance, version);
isLatestVersion = version.compareTo(current_version) == 0;
- hasSamplingLevel = version.compareTo("ka") >= 0;
- newStatsFile = version.compareTo("ka") >= 0;
+ correspondingMessagingVersion = MessagingService.VERSION_30;
- //For a while Adler32 was in use, now the CRC32 instrinsic is very good especially after Haswell
- //PureJavaCRC32 was always faster than Adler32. See CASSANDRA-8684
- ChecksumType checksumType = ChecksumType.CRC32;
- if (version.compareTo("ka") >= 0 && version.compareTo("ma") < 0)
- checksumType = ChecksumType.Adler32;
- this.uncompressedChecksumType = checksumType;
-
- checksumType = ChecksumType.CRC32;
- if (version.compareTo("jb") >= 0 && version.compareTo("ma") < 0)
- checksumType = ChecksumType.Adler32;
- this.compressedChecksumType = checksumType;
-
- hasRepairedAt = version.compareTo("ka") >= 0;
- tracksLegacyCounterShards = version.compareTo("ka") >= 0;
-
- newFileName = version.compareTo("la") >= 0;
-
- hasOldBfHashOrder = version.compareTo("ma") < 0;
- hasCompactionAncestors = version.compareTo("ma") < 0;
- storeRows = version.compareTo("ma") >= 0;
- correspondingMessagingVersion = storeRows
- ? MessagingService.VERSION_30
- : MessagingService.VERSION_21;
-
- hasBoundaries = version.compareTo("ma") < 0;
- hasCommitLogLowerBound = (version.compareTo("lb") >= 0 && version.compareTo("ma") < 0)
- || version.compareTo("mb") >= 0;
+ hasCommitLogLowerBound = version.compareTo("mb") >= 0;
hasCommitLogIntervals = version.compareTo("mc") >= 0;
hasAccurateMinMax = version.compareTo("md") >= 0;
+ hasMaxCompressedLength = version.compareTo("na") >= 0;
+ hasPendingRepair = version.compareTo("na") >= 0;
+ hasIsTransient = version.compareTo("na") >= 0;
+ hasMetadataChecksum = version.compareTo("na") >= 0;
+ hasOldBfFormat = version.compareTo("na") < 0;
}
@Override
@@ -200,60 +170,6 @@
}
@Override
- public boolean hasSamplingLevel()
- {
- return hasSamplingLevel;
- }
-
- @Override
- public boolean hasNewStatsFile()
- {
- return newStatsFile;
- }
-
- @Override
- public ChecksumType compressedChecksumType()
- {
- return compressedChecksumType;
- }
-
- @Override
- public ChecksumType uncompressedChecksumType()
- {
- return uncompressedChecksumType;
- }
-
- @Override
- public boolean hasRepairedAt()
- {
- return hasRepairedAt;
- }
-
- @Override
- public boolean tracksLegacyCounterShards()
- {
- return tracksLegacyCounterShards;
- }
-
- @Override
- public boolean hasOldBfHashOrder()
- {
- return hasOldBfHashOrder;
- }
-
- @Override
- public boolean hasCompactionAncestors()
- {
- return hasCompactionAncestors;
- }
-
- @Override
- public boolean hasNewFileName()
- {
- return newFileName;
- }
-
- @Override
public boolean hasCommitLogLowerBound()
{
return hasCommitLogLowerBound;
@@ -265,16 +181,15 @@
return hasCommitLogIntervals;
}
- @Override
- public boolean hasAccurateMinMax()
+ public boolean hasPendingRepair()
{
- return hasAccurateMinMax;
+ return hasPendingRepair;
}
@Override
- public boolean storeRows()
+ public boolean hasIsTransient()
{
- return storeRows;
+ return hasIsTransient;
}
@Override
@@ -284,12 +199,17 @@
}
@Override
- public boolean hasBoundaries()
+ public boolean hasMetadataChecksum()
{
- return hasBoundaries;
+ return hasMetadataChecksum;
}
@Override
+ public boolean hasAccurateMinMax()
+ {
+ return hasAccurateMinMax;
+ }
+
public boolean isCompatible()
{
return version.compareTo(earliest_supported_version) >= 0 && version.charAt(0) <= current_version.charAt(0);
@@ -300,5 +220,17 @@
{
return isCompatible() && version.charAt(0) == current_version.charAt(0);
}
+
+ @Override
+ public boolean hasMaxCompressedLength()
+ {
+ return hasMaxCompressedLength;
+ }
+
+ @Override
+ public boolean hasOldBfFormat()
+ {
+ return hasOldBfFormat;
+ }
}
}
diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableReader.java b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableReader.java
index 8551819..03d7562 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableReader.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableReader.java
@@ -17,13 +17,17 @@
*/
package org.apache.cassandra.io.sstable.format.big;
-import com.google.common.util.concurrent.RateLimiter;
-import org.apache.cassandra.cache.KeyCacheKey;
-import org.apache.cassandra.config.CFMetaData;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.*;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import org.apache.cassandra.db.*;
-import org.apache.cassandra.db.filter.ColumnFilter;
import org.apache.cassandra.db.columniterator.SSTableIterator;
import org.apache.cassandra.db.columniterator.SSTableReversedIterator;
+import org.apache.cassandra.db.filter.ColumnFilter;
import org.apache.cassandra.db.rows.Rows;
import org.apache.cassandra.db.rows.UnfilteredRowIterator;
import org.apache.cassandra.db.rows.UnfilteredRowIterators;
@@ -33,18 +37,13 @@
import org.apache.cassandra.io.sstable.*;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.sstable.format.SSTableReadsListener;
-import org.apache.cassandra.io.sstable.format.SSTableReadsListener.SkippingReason;
import org.apache.cassandra.io.sstable.format.SSTableReadsListener.SelectionReason;
+import org.apache.cassandra.io.sstable.format.SSTableReadsListener.SkippingReason;
import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
import org.apache.cassandra.io.util.FileDataInput;
+import org.apache.cassandra.schema.TableMetadataRef;
import org.apache.cassandra.tracing.Tracing;
import org.apache.cassandra.utils.ByteBufferUtil;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.*;
/**
* SSTableReaders are open()ed by Keyspace.onStart; after that they are created by SSTableWriter.renameAndOpen.
@@ -54,34 +53,35 @@
{
private static final Logger logger = LoggerFactory.getLogger(BigTableReader.class);
- BigTableReader(Descriptor desc, Set<Component> components, CFMetaData metadata, Long maxDataAge, StatsMetadata sstableMetadata, OpenReason openReason, SerializationHeader header)
+ BigTableReader(Descriptor desc, Set<Component> components, TableMetadataRef metadata, Long maxDataAge, StatsMetadata sstableMetadata, OpenReason openReason, SerializationHeader header)
{
super(desc, components, metadata, maxDataAge, sstableMetadata, openReason, header);
}
- public UnfilteredRowIterator iterator(DecoratedKey key, Slices slices, ColumnFilter selectedColumns, boolean reversed, boolean isForThrift, SSTableReadsListener listener)
+ public UnfilteredRowIterator iterator(DecoratedKey key,
+ Slices slices,
+ ColumnFilter selectedColumns,
+ boolean reversed,
+ SSTableReadsListener listener)
{
RowIndexEntry rie = getPosition(key, SSTableReader.Operator.EQ, listener);
- return iterator(null, key, rie, slices, selectedColumns, reversed, isForThrift);
+ return iterator(null, key, rie, slices, selectedColumns, reversed);
}
- public UnfilteredRowIterator iterator(FileDataInput file, DecoratedKey key, RowIndexEntry indexEntry, Slices slices, ColumnFilter selectedColumns, boolean reversed, boolean isForThrift)
+ @SuppressWarnings("resource")
+ public UnfilteredRowIterator iterator(FileDataInput file, DecoratedKey key, RowIndexEntry indexEntry, Slices slices, ColumnFilter selectedColumns, boolean reversed)
{
if (indexEntry == null)
- return UnfilteredRowIterators.noRowsIterator(metadata, key, Rows.EMPTY_STATIC_ROW, DeletionTime.LIVE, reversed);
+ return UnfilteredRowIterators.noRowsIterator(metadata(), key, Rows.EMPTY_STATIC_ROW, DeletionTime.LIVE, reversed);
return reversed
- ? new SSTableReversedIterator(this, file, key, indexEntry, slices, selectedColumns, isForThrift, ifile)
- : new SSTableIterator(this, file, key, indexEntry, slices, selectedColumns, isForThrift, ifile);
+ ? new SSTableReversedIterator(this, file, key, indexEntry, slices, selectedColumns, ifile)
+ : new SSTableIterator(this, file, key, indexEntry, slices, selectedColumns, ifile);
}
@Override
- public ISSTableScanner getScanner(ColumnFilter columns,
- DataRange dataRange,
- RateLimiter limiter,
- boolean isForThrift,
- SSTableReadsListener listener)
+ public ISSTableScanner getScanner(ColumnFilter columns, DataRange dataRange, SSTableReadsListener listener)
{
- return BigTableScanner.getScanner(this, columns, dataRange, limiter, isForThrift, listener);
+ return BigTableScanner.getScanner(this, columns, dataRange, listener);
}
/**
@@ -100,9 +100,9 @@
*
* @return A Scanner for reading the full SSTable.
*/
- public ISSTableScanner getScanner(RateLimiter limiter)
+ public ISSTableScanner getScanner()
{
- return BigTableScanner.getScanner(this, limiter);
+ return BigTableScanner.getScanner(this);
}
/**
@@ -111,12 +111,12 @@
* @param ranges the range of keys to cover
* @return A Scanner for seeking over the rows of the SSTable.
*/
- public ISSTableScanner getScanner(Collection<Range<Token>> ranges, RateLimiter limiter)
+ public ISSTableScanner getScanner(Collection<Range<Token>> ranges)
{
if (ranges != null)
- return BigTableScanner.getScanner(this, ranges, limiter);
+ return BigTableScanner.getScanner(this, ranges);
else
- return getScanner(limiter);
+ return getScanner();
}
@@ -127,7 +127,13 @@
return SSTableIdentityIterator.create(this, dfile, position, key, tombstoneOnly);
}
- @Override
+ /**
+ * @param key The key to apply as the rhs to the given Operator. A 'fake' key is allowed to
+ * allow key selection by token bounds but only if op != * EQ
+ * @param op The Operator defining matching keys: the nearest key to the target matching the operator wins.
+ * @param updateCacheAndStats true if updating stats and cache
+ * @return The index entry corresponding to the key, or null if the key is not present
+ */
protected RowIndexEntry getPosition(PartitionPosition key,
Operator op,
boolean updateCacheAndStats,
@@ -148,9 +154,8 @@
// next, the key cache (only make sense for valid row key)
if ((op == Operator.EQ || op == Operator.GE) && (key instanceof DecoratedKey))
{
- DecoratedKey decoratedKey = (DecoratedKey)key;
- KeyCacheKey cacheKey = new KeyCacheKey(metadata.ksAndCFName, descriptor, decoratedKey.getKey());
- RowIndexEntry cachedPosition = getCachedPosition(cacheKey, updateCacheAndStats);
+ DecoratedKey decoratedKey = (DecoratedKey) key;
+ RowIndexEntry cachedPosition = getCachedPosition(decoratedKey, updateCacheAndStats);
if (cachedPosition != null)
{
listener.onSSTableSelected(this, cachedPosition, SelectionReason.KEY_CACHE_HIT);
@@ -239,7 +244,7 @@
if (opSatisfied)
{
// read data position from index entry
- RowIndexEntry indexEntry = rowIndexEntrySerializer.deserialize(in, in.getFilePointer());
+ RowIndexEntry indexEntry = rowIndexEntrySerializer.deserialize(in);
if (exactMatch && updateCacheAndStats)
{
assert key instanceof DecoratedKey; // key can be == to the index key only if it's a true row key
diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableScanner.java b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableScanner.java
index f4bd1ea..20105cd 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableScanner.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableScanner.java
@@ -21,11 +21,12 @@
import java.util.*;
import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.utils.AbstractIterator;
-import com.google.common.collect.Iterators;
-import com.google.common.util.concurrent.RateLimiter;
-import org.apache.cassandra.config.CFMetaData;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterators;
+
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.rows.*;
import org.apache.cassandra.db.filter.*;
@@ -43,7 +44,6 @@
import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.io.util.RandomAccessReader;
import org.apache.cassandra.utils.ByteBufferUtil;
-import org.apache.cassandra.utils.Pair;
import static org.apache.cassandra.dht.AbstractBounds.isEmpty;
import static org.apache.cassandra.dht.AbstractBounds.maxLeft;
@@ -62,7 +62,6 @@
private final ColumnFilter columns;
private final DataRange dataRange;
private final RowIndexEntry.IndexSerializer rowIndexEntrySerializer;
- private final boolean isForThrift;
private final SSTableReadsListener listener;
private long startScan = -1;
private long bytesScanned = 0;
@@ -70,69 +69,50 @@
protected Iterator<UnfilteredRowIterator> iterator;
// Full scan of the sstables
- public static ISSTableScanner getScanner(SSTableReader sstable, RateLimiter limiter)
+ public static ISSTableScanner getScanner(SSTableReader sstable)
{
- return new BigTableScanner(sstable,
- ColumnFilter.all(sstable.metadata),
- limiter,
- Iterators.singletonIterator(fullRange(sstable)));
+ return getScanner(sstable, Iterators.singletonIterator(fullRange(sstable)));
}
public static ISSTableScanner getScanner(SSTableReader sstable,
ColumnFilter columns,
DataRange dataRange,
- RateLimiter limiter,
- boolean isForThrift,
SSTableReadsListener listener)
{
- return new BigTableScanner(sstable, columns, dataRange, limiter, isForThrift, makeBounds(sstable, dataRange).iterator(), listener);
+ return new BigTableScanner(sstable, columns, dataRange, makeBounds(sstable, dataRange).iterator(), listener);
}
- public static ISSTableScanner getScanner(SSTableReader sstable, Collection<Range<Token>> tokenRanges, RateLimiter limiter)
+ public static ISSTableScanner getScanner(SSTableReader sstable, Collection<Range<Token>> tokenRanges)
{
// We want to avoid allocating a SSTableScanner if the range don't overlap the sstable (#5249)
- List<Pair<Long, Long>> positions = sstable.getPositionsForRanges(tokenRanges);
+ List<SSTableReader.PartitionPositionBounds> positions = sstable.getPositionsForRanges(tokenRanges);
if (positions.isEmpty())
return new EmptySSTableScanner(sstable);
- return new BigTableScanner(sstable,
- ColumnFilter.all(sstable.metadata),
- limiter,
- makeBounds(sstable, tokenRanges).iterator());
+ return getScanner(sstable, makeBounds(sstable, tokenRanges).iterator());
}
public static ISSTableScanner getScanner(SSTableReader sstable, Iterator<AbstractBounds<PartitionPosition>> rangeIterator)
{
- return new BigTableScanner(sstable, ColumnFilter.all(sstable.metadata), null, rangeIterator);
- }
-
- private BigTableScanner(SSTableReader sstable,
- ColumnFilter columns,
- RateLimiter limiter,
- Iterator<AbstractBounds<PartitionPosition>> rangeIterator)
- {
- this(sstable, columns, null, limiter, false, rangeIterator, SSTableReadsListener.NOOP_LISTENER);
+ return new BigTableScanner(sstable, ColumnFilter.all(sstable.metadata()), null, rangeIterator, SSTableReadsListener.NOOP_LISTENER);
}
private BigTableScanner(SSTableReader sstable,
ColumnFilter columns,
DataRange dataRange,
- RateLimiter limiter,
- boolean isForThrift,
Iterator<AbstractBounds<PartitionPosition>> rangeIterator,
SSTableReadsListener listener)
{
assert sstable != null;
- this.dfile = limiter == null ? sstable.openDataReader() : sstable.openDataReader(limiter);
+ this.dfile = sstable.openDataReader();
this.ifile = sstable.openIndexReader();
this.sstable = sstable;
this.columns = columns;
this.dataRange = dataRange;
- this.rowIndexEntrySerializer = sstable.descriptor.version.getSSTableFormat().getIndexSerializer(sstable.metadata,
+ this.rowIndexEntrySerializer = sstable.descriptor.version.getSSTableFormat().getIndexSerializer(sstable.metadata(),
sstable.descriptor.version,
sstable.header);
- this.isForThrift = isForThrift;
this.rangeIterator = rangeIterator;
this.listener = listener;
}
@@ -211,7 +191,7 @@
if (indexDecoratedKey.compareTo(currentRange.left) > 0 || currentRange.contains(indexDecoratedKey))
{
// Found, just read the dataPosition and seek into index and data files
- long dataPosition = RowIndexEntry.Serializer.readPosition(ifile, sstable.descriptor.version);
+ long dataPosition = RowIndexEntry.Serializer.readPosition(ifile);
ifile.seek(indexPosition);
dfile.seek(dataPosition);
break;
@@ -263,19 +243,15 @@
return sstable.onDiskLength();
}
- public String getBackingFiles()
+ public Set<SSTableReader> getBackingSSTables()
{
- return sstable.toString();
+ return ImmutableSet.of(sstable);
}
- public boolean isForThrift()
- {
- return isForThrift;
- }
- public CFMetaData metadata()
+ public TableMetadata metadata()
{
- return sstable.metadata;
+ return sstable.metadata();
}
public boolean hasNext()
@@ -299,7 +275,7 @@
private Iterator<UnfilteredRowIterator> createIterator()
{
- listener.onScanningStarted(sstable);
+ this.listener.onScanningStarted(sstable);
return new KeyScanningIterator();
}
@@ -333,7 +309,7 @@
return endOfData();
currentKey = sstable.decorateKey(ByteBufferUtil.readWithShortLength(ifile));
- currentEntry = rowIndexEntrySerializer.deserialize(ifile, ifile.getFilePointer());
+ currentEntry = rowIndexEntrySerializer.deserialize(ifile);
} while (!currentRange.contains(currentKey));
}
else
@@ -352,7 +328,7 @@
{
// we need the position of the start of the next key, regardless of whether it falls in the current range
nextKey = sstable.decorateKey(ByteBufferUtil.readWithShortLength(ifile));
- nextEntry = rowIndexEntrySerializer.deserialize(ifile, ifile.getFilePointer());
+ nextEntry = rowIndexEntrySerializer.deserialize(ifile);
if (!currentRange.contains(nextKey))
{
@@ -389,7 +365,7 @@
}
ClusteringIndexFilter filter = dataRange.clusteringIndexFilter(partitionKey());
- return sstable.iterator(dfile, partitionKey(), currentEntry, filter.getSlices(BigTableScanner.this.metadata()), columns, filter.isReversed(), isForThrift);
+ return sstable.iterator(dfile, partitionKey(), currentEntry, filter.getSlices(BigTableScanner.this.metadata()), columns, filter.isReversed());
}
catch (CorruptSSTableException | IOException e)
{
@@ -446,19 +422,14 @@
return 0;
}
- public String getBackingFiles()
+ public Set<SSTableReader> getBackingSSTables()
{
- return sstable.getFilename();
+ return ImmutableSet.of(sstable);
}
- public boolean isForThrift()
+ public TableMetadata metadata()
{
- return false;
- }
-
- public CFMetaData metadata()
- {
- return sstable.metadata;
+ return sstable.metadata();
}
public boolean hasNext()
diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java
index 9083cd3..04ab08a 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java
@@ -17,20 +17,16 @@
*/
package org.apache.cassandra.io.sstable.format.big;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
+import java.io.*;
import java.nio.ByteBuffer;
-import java.util.Collection;
-import java.util.Map;
-import java.util.Optional;
+import java.util.*;
+import org.apache.cassandra.db.compaction.OperationType;
import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.cache.ChunkCache;
-import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.config.Config;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.*;
@@ -38,6 +34,7 @@
import org.apache.cassandra.db.transform.Transformation;
import org.apache.cassandra.io.FSWriteError;
import org.apache.cassandra.io.compress.CompressedSequentialWriter;
+import org.apache.cassandra.io.compress.ICompressor;
import org.apache.cassandra.io.sstable.*;
import org.apache.cassandra.io.sstable.format.SSTableFlushObserver;
import org.apache.cassandra.io.sstable.format.SSTableReader;
@@ -47,6 +44,8 @@
import org.apache.cassandra.io.sstable.metadata.MetadataType;
import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
import org.apache.cassandra.io.util.*;
+import org.apache.cassandra.schema.CompressionParams;
+import org.apache.cassandra.schema.TableMetadataRef;
import org.apache.cassandra.utils.*;
import org.apache.cassandra.utils.concurrent.Transactional;
@@ -71,29 +70,33 @@
public BigTableWriter(Descriptor descriptor,
long keyCount,
long repairedAt,
- CFMetaData metadata,
+ UUID pendingRepair,
+ boolean isTransient,
+ TableMetadataRef metadata,
MetadataCollector metadataCollector,
SerializationHeader header,
Collection<SSTableFlushObserver> observers,
LifecycleNewTracker lifecycleNewTracker)
{
- super(descriptor, keyCount, repairedAt, metadata, metadataCollector, header, observers);
+ super(descriptor, keyCount, repairedAt, pendingRepair, isTransient, metadata, metadataCollector, header, observers);
lifecycleNewTracker.trackNew(this); // must track before any files are created
if (compression)
{
+ final CompressionParams compressionParams = compressionFor(lifecycleNewTracker.opType());
+
dataFile = new CompressedSequentialWriter(new File(getFilename()),
descriptor.filenameFor(Component.COMPRESSION_INFO),
- new File(descriptor.filenameFor(descriptor.digestComponent)),
+ new File(descriptor.filenameFor(Component.DIGEST)),
writerOption,
- metadata.params.compression,
+ compressionParams,
metadataCollector);
}
else
{
dataFile = new ChecksummedSequentialWriter(new File(getFilename()),
new File(descriptor.filenameFor(Component.CRC)),
- new File(descriptor.filenameFor(descriptor.digestComponent)),
+ new File(descriptor.filenameFor(Component.DIGEST)),
writerOption);
}
dbuilder = new FileHandle.Builder(descriptor.filenameFor(Component.DATA)).compressed(compression)
@@ -104,6 +107,45 @@
columnIndexWriter = new ColumnIndex(this.header, dataFile, descriptor.version, this.observers, getRowIndexEntrySerializer().indexInfoSerializer());
}
+ /**
+ * Given an OpType, determine the correct Compression Parameters
+ * @param opType
+ * @return {@link org.apache.cassandra.schema.CompressionParams}
+ */
+ private CompressionParams compressionFor(final OperationType opType)
+ {
+ CompressionParams compressionParams = metadata().params.compression;
+ final ICompressor compressor = compressionParams.getSstableCompressor();
+
+ if (null != compressor && opType == OperationType.FLUSH)
+ {
+ // When we are flushing out of the memtable throughput of the compressor is critical as flushes,
+ // especially of large tables, can queue up and potentially block writes.
+ // This optimization allows us to fall back to a faster compressor if a particular
+ // compression algorithm indicates we should. See CASSANDRA-15379 for more details.
+ switch (DatabaseDescriptor.getFlushCompression())
+ {
+ // It is relatively easier to insert a Noop compressor than to disable compressed writing
+ // entirely as the "compression" member field is provided outside the scope of this class.
+ // It may make sense in the future to refactor the ownership of the compression flag so that
+ // We can bypass the CompressedSequentialWriter in this case entirely.
+ case none:
+ compressionParams = CompressionParams.NOOP;
+ break;
+ case fast:
+ if (!compressor.recommendedUses().contains(ICompressor.Uses.FAST_COMPRESSION))
+ {
+ // The default compressor is generally fast (LZ4 with 16KiB block size)
+ compressionParams = CompressionParams.DEFAULT;
+ break;
+ }
+ case table:
+ default:
+ }
+ }
+ return compressionParams;
+ }
+
public void mark()
{
dataMark = dataFile.mark();
@@ -207,8 +249,8 @@
{
if (rowSize > DatabaseDescriptor.getCompactionLargePartitionWarningThreshold())
{
- String keyString = metadata.getKeyValidator().getString(key.getKey());
- logger.warn("Writing large partition {}/{}:{} ({}) to sstable {}", metadata.ksName, metadata.cfName, keyString, FBUtilities.prettyPrintMemory(rowSize), getFilename());
+ String keyString = metadata().partitionKeyType.getString(key.getKey());
+ logger.warn("Writing large partition {}/{}:{} ({}) to sstable {}", metadata.keyspace, metadata.name, keyString, FBUtilities.prettyPrintMemory(rowSize), getFilename());
}
}
@@ -280,7 +322,7 @@
StatsMetadata stats = statsMetadata();
assert boundary.indexLength > 0 && boundary.dataLength > 0;
// open the reader early
- IndexSummary indexSummary = iwriter.summary.build(metadata.partitioner, boundary);
+ IndexSummary indexSummary = iwriter.summary.build(metadata().partitioner, boundary);
long indexFileLength = new File(descriptor.filenameFor(Component.PRIMARY_INDEX)).length();
int indexBufferSize = optimizationStrategy.bufferSize(indexFileLength / indexSummary.size());
FileHandle ifile = iwriter.builder.bufferSize(indexBufferSize).complete(boundary.indexLength);
@@ -326,7 +368,7 @@
StatsMetadata stats = statsMetadata();
// finalize in-memory state for the reader
- IndexSummary indexSummary = iwriter.summary.build(this.metadata.partitioner);
+ IndexSummary indexSummary = iwriter.summary.build(metadata().partitioner);
long indexFileLength = new File(descriptor.filenameFor(Component.PRIMARY_INDEX)).length();
int dataBufferSize = optimizationStrategy.bufferSize(stats.estimatedPartitionSize.percentile(DatabaseDescriptor.getDiskOptimizationEstimatePercentile()));
int indexBufferSize = optimizationStrategy.bufferSize(indexFileLength / indexSummary.size());
@@ -337,7 +379,7 @@
invalidateCacheAtBoundary(dfile);
SSTableReader sstable = SSTableReader.internalOpen(descriptor,
components,
- this.metadata,
+ metadata,
ifile,
dfile,
indexSummary,
@@ -441,8 +483,8 @@
indexFile = new SequentialWriter(new File(descriptor.filenameFor(Component.PRIMARY_INDEX)), writerOption);
builder = new FileHandle.Builder(descriptor.filenameFor(Component.PRIMARY_INDEX)).mmapped(DatabaseDescriptor.getIndexAccessMode() == Config.DiskAccessMode.mmap);
chunkCache.ifPresent(builder::withChunkCache);
- summary = new IndexSummaryBuilder(keyCount, metadata.params.minIndexInterval, Downsampling.BASE_SAMPLING_LEVEL);
- bf = FilterFactory.getFilter(keyCount, metadata.params.bloomFilterFpChance, true, descriptor.version.hasOldBfHashOrder());
+ summary = new IndexSummaryBuilder(keyCount, metadata().params.minIndexInterval, Downsampling.BASE_SAMPLING_LEVEL);
+ bf = FilterFactory.getFilter(keyCount, metadata().params.bloomFilterFpChance);
// register listeners to be alerted when the data files are flushed
indexFile.setPostFlushListener(() -> summary.markIndexSynced(indexFile.getLastFlushOffset()));
dataFile.setPostFlushListener(() -> summary.markDataSynced(dataFile.getLastFlushOffset()));
@@ -487,7 +529,7 @@
DataOutputStreamPlus stream = new BufferedDataOutputStreamPlus(fos))
{
// bloom filter
- FilterFactory.serialize(bf, stream);
+ BloomFilterSerializer.serialize((BloomFilter) bf, stream);
stream.flush();
SyncUtil.sync(fos);
}
diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableZeroCopyWriter.java b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableZeroCopyWriter.java
new file mode 100644
index 0000000..f05ea94
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableZeroCopyWriter.java
@@ -0,0 +1,230 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.io.sstable.format.big;
+
+import java.io.EOFException;
+import java.io.File;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.EnumMap;
+import java.util.Map;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Sets;
+import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.io.FSWriteError;
+import org.apache.cassandra.io.compress.BufferType;
+import org.apache.cassandra.io.sstable.Component;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.SSTable;
+import org.apache.cassandra.io.sstable.SSTableMultiWriter;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.SequentialWriter;
+import org.apache.cassandra.io.util.SequentialWriterOption;
+import org.apache.cassandra.net.AsyncStreamingInputPlus;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.schema.TableMetadataRef;
+
+import static java.lang.String.format;
+import static org.apache.cassandra.utils.FBUtilities.prettyPrintMemory;
+
+public class BigTableZeroCopyWriter extends SSTable implements SSTableMultiWriter
+{
+ private static final Logger logger = LoggerFactory.getLogger(BigTableZeroCopyWriter.class);
+
+ private final TableMetadataRef metadata;
+ private volatile SSTableReader finalReader;
+ private final Map<Component.Type, SequentialWriter> componentWriters;
+
+ private static final SequentialWriterOption WRITER_OPTION =
+ SequentialWriterOption.newBuilder()
+ .trickleFsync(false)
+ .bufferSize(2 << 20)
+ .bufferType(BufferType.OFF_HEAP)
+ .build();
+
+ private static final ImmutableSet<Component> SUPPORTED_COMPONENTS =
+ ImmutableSet.of(Component.DATA,
+ Component.PRIMARY_INDEX,
+ Component.SUMMARY,
+ Component.STATS,
+ Component.COMPRESSION_INFO,
+ Component.FILTER,
+ Component.DIGEST,
+ Component.CRC);
+
+ public BigTableZeroCopyWriter(Descriptor descriptor,
+ TableMetadataRef metadata,
+ LifecycleNewTracker lifecycleNewTracker,
+ final Collection<Component> components)
+ {
+ super(descriptor, ImmutableSet.copyOf(components), metadata, DatabaseDescriptor.getDiskOptimizationStrategy());
+
+ lifecycleNewTracker.trackNew(this);
+ this.metadata = metadata;
+ this.componentWriters = new EnumMap<>(Component.Type.class);
+
+ if (!SUPPORTED_COMPONENTS.containsAll(components))
+ throw new AssertionError(format("Unsupported streaming component detected %s",
+ Sets.difference(ImmutableSet.copyOf(components), SUPPORTED_COMPONENTS)));
+
+ for (Component c : components)
+ componentWriters.put(c.type, makeWriter(descriptor, c));
+ }
+
+ private static SequentialWriter makeWriter(Descriptor descriptor, Component component)
+ {
+ return new SequentialWriter(new File(descriptor.filenameFor(component)), WRITER_OPTION, false);
+ }
+
+ private void write(DataInputPlus in, long size, SequentialWriter out) throws FSWriteError
+ {
+ final int BUFFER_SIZE = 1 << 20;
+ long bytesRead = 0;
+ byte[] buff = new byte[BUFFER_SIZE];
+ try
+ {
+ while (bytesRead < size)
+ {
+ int toRead = (int) Math.min(size - bytesRead, BUFFER_SIZE);
+ in.readFully(buff, 0, toRead);
+ int count = Math.min(toRead, BUFFER_SIZE);
+ out.write(buff, 0, count);
+ bytesRead += count;
+ }
+ out.sync();
+ }
+ catch (IOException e)
+ {
+ throw new FSWriteError(e, out.getPath());
+ }
+ }
+
+ @Override
+ public boolean append(UnfilteredRowIterator partition)
+ {
+ throw new UnsupportedOperationException("Operation not supported by BigTableBlockWriter");
+ }
+
+ @Override
+ public Collection<SSTableReader> finish(long repairedAt, long maxDataAge, boolean openResult)
+ {
+ return finish(openResult);
+ }
+
+ @Override
+ public Collection<SSTableReader> finish(boolean openResult)
+ {
+ setOpenResult(openResult);
+ return finished();
+ }
+
+ @Override
+ public Collection<SSTableReader> finished()
+ {
+ if (finalReader == null)
+ finalReader = SSTableReader.open(descriptor, components, metadata);
+
+ return ImmutableList.of(finalReader);
+ }
+
+ @Override
+ public SSTableMultiWriter setOpenResult(boolean openResult)
+ {
+ return null;
+ }
+
+ @Override
+ public long getFilePointer()
+ {
+ return 0;
+ }
+
+ @Override
+ public TableId getTableId()
+ {
+ return metadata.id;
+ }
+
+ @Override
+ public Throwable commit(Throwable accumulate)
+ {
+ for (SequentialWriter writer : componentWriters.values())
+ accumulate = writer.commit(accumulate);
+ return accumulate;
+ }
+
+ @Override
+ public Throwable abort(Throwable accumulate)
+ {
+ for (SequentialWriter writer : componentWriters.values())
+ accumulate = writer.abort(accumulate);
+ return accumulate;
+ }
+
+ @Override
+ public void prepareToCommit()
+ {
+ for (SequentialWriter writer : componentWriters.values())
+ writer.prepareToCommit();
+ }
+
+ @Override
+ public void close()
+ {
+ for (SequentialWriter writer : componentWriters.values())
+ writer.close();
+ }
+
+ public void writeComponent(Component.Type type, DataInputPlus in, long size)
+ {
+ logger.info("Writing component {} to {} length {}", type, componentWriters.get(type).getPath(), prettyPrintMemory(size));
+
+ if (in instanceof AsyncStreamingInputPlus)
+ write((AsyncStreamingInputPlus) in, size, componentWriters.get(type));
+ else
+ write(in, size, componentWriters.get(type));
+ }
+
+ private void write(AsyncStreamingInputPlus in, long size, SequentialWriter writer)
+ {
+ logger.info("Block Writing component to {} length {}", writer.getPath(), prettyPrintMemory(size));
+
+ try
+ {
+ in.consume(writer::writeDirectlyToChannel, size);
+ writer.sync();
+ }
+ // FIXME: handle ACIP exceptions properly
+ catch (EOFException | AsyncStreamingInputPlus.InputTimeoutException e)
+ {
+ in.close();
+ }
+ catch (IOException e)
+ {
+ throw new FSWriteError(e, writer.getPath());
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/io/sstable/metadata/CompactionMetadata.java b/src/java/org/apache/cassandra/io/sstable/metadata/CompactionMetadata.java
index ef3453a..c9dfe39 100644
--- a/src/java/org/apache/cassandra/io/sstable/metadata/CompactionMetadata.java
+++ b/src/java/org/apache/cassandra/io/sstable/metadata/CompactionMetadata.java
@@ -75,30 +75,17 @@
public int serializedSize(Version version, CompactionMetadata component) throws IOException
{
int sz = 0;
- if (version.hasCompactionAncestors())
- { // write empty ancestor marker
- sz = 4;
- }
byte[] serializedCardinality = component.cardinalityEstimator.getBytes();
return TypeSizes.sizeof(serializedCardinality.length) + serializedCardinality.length + sz;
}
public void serialize(Version version, CompactionMetadata component, DataOutputPlus out) throws IOException
{
- if (version.hasCompactionAncestors())
- { // write empty ancestor marker
- out.writeInt(0);
- }
ByteBufferUtil.writeWithLength(component.cardinalityEstimator.getBytes(), out);
}
public CompactionMetadata deserialize(Version version, DataInputPlus in) throws IOException
{
- if (version.hasCompactionAncestors())
- { // skip ancestors
- int nbAncestors = in.readInt();
- in.skipBytes(nbAncestors * TypeSizes.sizeof(nbAncestors));
- }
ICardinality cardinality = HyperLogLogPlus.Builder.build(ByteBufferUtil.readBytes(in, in.readInt()));
return new CompactionMetadata(cardinality);
}
diff --git a/src/java/org/apache/cassandra/io/sstable/metadata/IMetadataSerializer.java b/src/java/org/apache/cassandra/io/sstable/metadata/IMetadataSerializer.java
index a86327d..db9f161 100644
--- a/src/java/org/apache/cassandra/io/sstable/metadata/IMetadataSerializer.java
+++ b/src/java/org/apache/cassandra/io/sstable/metadata/IMetadataSerializer.java
@@ -20,6 +20,8 @@
import java.io.IOException;
import java.util.EnumSet;
import java.util.Map;
+import java.util.UUID;
+import java.util.function.Function;
import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.io.sstable.format.Version;
@@ -61,6 +63,15 @@
MetadataComponent deserialize(Descriptor descriptor, MetadataType type) throws IOException;
/**
+ * Mutate SSTable Metadata
+ *
+ * @param descriptor SSTable descriptor
+ * @param transform function to mutate sstable metadata
+ * @throws IOException
+ */
+ public void mutate(Descriptor descriptor, Function<StatsMetadata, StatsMetadata> transform) throws IOException;
+
+ /**
* Mutate SSTable level
*
* @param descriptor SSTable descriptor
@@ -70,9 +81,9 @@
void mutateLevel(Descriptor descriptor, int newLevel) throws IOException;
/**
- * Mutate repairedAt time
+ * Mutate the repairedAt time, pendingRepair ID, and transient status
*/
- void mutateRepairedAt(Descriptor descriptor, long newRepairedAt) throws IOException;
+ public void mutateRepairMetadata(Descriptor descriptor, long newRepairedAt, UUID newPendingRepair, boolean isTransient) throws IOException;
/**
* Replace the sstable metadata file ({@code -Statistics.db}) with the given components.
diff --git a/src/java/org/apache/cassandra/io/sstable/metadata/LegacyMetadataSerializer.java b/src/java/org/apache/cassandra/io/sstable/metadata/LegacyMetadataSerializer.java
deleted file mode 100644
index 6cc33f5..0000000
--- a/src/java/org/apache/cassandra/io/sstable/metadata/LegacyMetadataSerializer.java
+++ /dev/null
@@ -1,163 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.io.sstable.metadata;
-
-import java.io.*;
-import java.nio.ByteBuffer;
-import java.util.*;
-
-import org.apache.cassandra.db.TypeSizes;
-import org.apache.cassandra.db.commitlog.CommitLogPosition;
-import org.apache.cassandra.db.commitlog.IntervalSet;
-import org.apache.cassandra.io.sstable.Component;
-import org.apache.cassandra.io.sstable.Descriptor;
-import org.apache.cassandra.io.sstable.format.Version;
-import org.apache.cassandra.io.util.DataInputPlus.DataInputStreamPlus;
-import org.apache.cassandra.io.util.DataOutputPlus;
-import org.apache.cassandra.service.ActiveRepairService;
-import org.apache.cassandra.utils.ByteBufferUtil;
-import org.apache.cassandra.utils.EstimatedHistogram;
-import org.apache.cassandra.utils.StreamingHistogram;
-
-import static org.apache.cassandra.io.sstable.metadata.StatsMetadata.commitLogPositionSetSerializer;
-
-/**
- * Serializer for SSTable from legacy versions
- */
-@Deprecated
-public class LegacyMetadataSerializer extends MetadataSerializer
-{
- /**
- * Legacy serialization is only used for SSTable level reset.
- */
- @Override
- public void serialize(Map<MetadataType, MetadataComponent> components, DataOutputPlus out, Version version) throws IOException
- {
- ValidationMetadata validation = (ValidationMetadata) components.get(MetadataType.VALIDATION);
- StatsMetadata stats = (StatsMetadata) components.get(MetadataType.STATS);
- CompactionMetadata compaction = (CompactionMetadata) components.get(MetadataType.COMPACTION);
-
- assert validation != null && stats != null && compaction != null && validation.partitioner != null;
-
- EstimatedHistogram.serializer.serialize(stats.estimatedPartitionSize, out);
- EstimatedHistogram.serializer.serialize(stats.estimatedColumnCount, out);
- CommitLogPosition.serializer.serialize(stats.commitLogIntervals.upperBound().orElse(CommitLogPosition.NONE), out);
- out.writeLong(stats.minTimestamp);
- out.writeLong(stats.maxTimestamp);
- out.writeInt(stats.maxLocalDeletionTime);
- out.writeDouble(validation.bloomFilterFPChance);
- out.writeDouble(stats.compressionRatio);
- out.writeUTF(validation.partitioner);
- out.writeInt(0); // compaction ancestors
- StreamingHistogram.serializer.serialize(stats.estimatedTombstoneDropTime, out);
- out.writeInt(stats.sstableLevel);
- out.writeInt(stats.minClusteringValues.size());
- for (ByteBuffer value : stats.minClusteringValues)
- ByteBufferUtil.writeWithShortLength(value, out);
- out.writeInt(stats.maxClusteringValues.size());
- for (ByteBuffer value : stats.maxClusteringValues)
- ByteBufferUtil.writeWithShortLength(value, out);
- if (version.hasCommitLogLowerBound())
- CommitLogPosition.serializer.serialize(stats.commitLogIntervals.lowerBound().orElse(CommitLogPosition.NONE), out);
- if (version.hasCommitLogIntervals())
- commitLogPositionSetSerializer.serialize(stats.commitLogIntervals, out);
- }
-
- /**
- * Legacy serializer deserialize all components no matter what types are specified.
- */
- @Override
- public Map<MetadataType, MetadataComponent> deserialize(Descriptor descriptor, EnumSet<MetadataType> types) throws IOException
- {
- Map<MetadataType, MetadataComponent> components = new EnumMap<>(MetadataType.class);
-
- File statsFile = new File(descriptor.filenameFor(Component.STATS));
- if (!statsFile.exists() && types.contains(MetadataType.STATS))
- {
- components.put(MetadataType.STATS, MetadataCollector.defaultStatsMetadata());
- }
- else
- {
- try (DataInputStreamPlus in = new DataInputStreamPlus(new BufferedInputStream(new FileInputStream(statsFile))))
- {
- EstimatedHistogram partitionSizes = EstimatedHistogram.serializer.deserialize(in);
- EstimatedHistogram columnCounts = EstimatedHistogram.serializer.deserialize(in);
- CommitLogPosition commitLogLowerBound = CommitLogPosition.NONE;
- CommitLogPosition commitLogUpperBound = CommitLogPosition.serializer.deserialize(in);
- long minTimestamp = in.readLong();
- long maxTimestamp = in.readLong();
- int maxLocalDeletionTime = in.readInt();
- double bloomFilterFPChance = in.readDouble();
- double compressionRatio = in.readDouble();
- String partitioner = in.readUTF();
- int nbAncestors = in.readInt(); //skip compaction ancestors
- in.skipBytes(nbAncestors * TypeSizes.sizeof(nbAncestors));
- StreamingHistogram tombstoneHistogram = StreamingHistogram.serializer.deserialize(in);
- int sstableLevel = 0;
- if (in.available() > 0)
- sstableLevel = in.readInt();
-
- int colCount = in.readInt();
- List<ByteBuffer> minColumnNames = new ArrayList<>(colCount);
- for (int i = 0; i < colCount; i++)
- minColumnNames.add(ByteBufferUtil.readWithShortLength(in));
-
- colCount = in.readInt();
- List<ByteBuffer> maxColumnNames = new ArrayList<>(colCount);
- for (int i = 0; i < colCount; i++)
- maxColumnNames.add(ByteBufferUtil.readWithShortLength(in));
-
- if (descriptor.version.hasCommitLogLowerBound())
- commitLogLowerBound = CommitLogPosition.serializer.deserialize(in);
- IntervalSet<CommitLogPosition> commitLogIntervals;
- if (descriptor.version.hasCommitLogIntervals())
- commitLogIntervals = commitLogPositionSetSerializer.deserialize(in);
- else
- commitLogIntervals = new IntervalSet<>(commitLogLowerBound, commitLogUpperBound);
-
- if (types.contains(MetadataType.VALIDATION))
- components.put(MetadataType.VALIDATION,
- new ValidationMetadata(partitioner, bloomFilterFPChance));
- if (types.contains(MetadataType.STATS))
- components.put(MetadataType.STATS,
- new StatsMetadata(partitionSizes,
- columnCounts,
- commitLogIntervals,
- minTimestamp,
- maxTimestamp,
- Integer.MAX_VALUE,
- maxLocalDeletionTime,
- 0,
- Integer.MAX_VALUE,
- compressionRatio,
- tombstoneHistogram,
- sstableLevel,
- minColumnNames,
- maxColumnNames,
- true,
- ActiveRepairService.UNREPAIRED_SSTABLE,
- -1,
- -1));
- if (types.contains(MetadataType.COMPACTION))
- components.put(MetadataType.COMPACTION,
- new CompactionMetadata(null));
- }
- }
- return components;
- }
-}
diff --git a/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java b/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java
old mode 100644
new mode 100755
index ea88a3f..92d955b
--- a/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java
+++ b/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java
@@ -24,6 +24,7 @@
import java.util.EnumMap;
import java.util.List;
import java.util.Map;
+import java.util.UUID;
import com.google.common.base.Preconditions;
import com.google.common.collect.Maps;
@@ -42,7 +43,8 @@
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.EstimatedHistogram;
import org.apache.cassandra.utils.MurmurHash;
-import org.apache.cassandra.utils.StreamingHistogram;
+import org.apache.cassandra.utils.streamhist.TombstoneHistogram;
+import org.apache.cassandra.utils.streamhist.StreamingTombstoneHistogramBuilder;
public class MetadataCollector implements PartitionStatisticsCollector
{
@@ -61,9 +63,9 @@
return new EstimatedHistogram(150);
}
- static StreamingHistogram.StreamingHistogramBuilder defaultTombstoneDropTimeHistogramBuilder()
+ static TombstoneHistogram defaultTombstoneDropTimeHistogram()
{
- return new StreamingHistogram.StreamingHistogramBuilder(SSTable.TOMBSTONE_HISTOGRAM_BIN_SIZE, SSTable.TOMBSTONE_HISTOGRAM_SPOOL_SIZE, SSTable.TOMBSTONE_HISTOGRAM_TTL_ROUND_SECONDS);
+ return TombstoneHistogram.createDefault();
}
public static StatsMetadata defaultStatsMetadata()
@@ -78,14 +80,16 @@
0,
Integer.MAX_VALUE,
NO_COMPRESSION_RATIO,
- defaultTombstoneDropTimeHistogramBuilder().build(),
+ defaultTombstoneDropTimeHistogram(),
0,
Collections.<ByteBuffer>emptyList(),
Collections.<ByteBuffer>emptyList(),
true,
ActiveRepairService.UNREPAIRED_SSTABLE,
-1,
- -1);
+ -1,
+ null,
+ false);
}
protected EstimatedHistogram estimatedPartitionSize = defaultPartitionSizeHistogram();
@@ -96,7 +100,7 @@
protected final MinMaxIntTracker localDeletionTimeTracker = new MinMaxIntTracker(Cell.NO_DELETION_TIME, Cell.NO_DELETION_TIME);
protected final MinMaxIntTracker ttlTracker = new MinMaxIntTracker(Cell.NO_TTL, Cell.NO_TTL);
protected double compressionRatio = NO_COMPRESSION_RATIO;
- protected StreamingHistogram.StreamingHistogramBuilder estimatedTombstoneDropTime = defaultTombstoneDropTimeHistogramBuilder();
+ protected StreamingTombstoneHistogramBuilder estimatedTombstoneDropTime = new StreamingTombstoneHistogramBuilder(SSTable.TOMBSTONE_HISTOGRAM_BIN_SIZE, SSTable.TOMBSTONE_HISTOGRAM_SPOOL_SIZE, SSTable.TOMBSTONE_HISTOGRAM_TTL_ROUND_SECONDS);
protected int sstableLevel;
private ClusteringPrefix minClustering = null;
private ClusteringPrefix maxClustering = null;
@@ -152,12 +156,6 @@
return this;
}
- public MetadataCollector mergeTombstoneHistogram(StreamingHistogram histogram)
- {
- estimatedTombstoneDropTime.merge(histogram);
- return this;
- }
-
/**
* Ratio is compressed/uncompressed and it is
* if you have 1.x then compression isn't helping
@@ -241,7 +239,7 @@
this.hasLegacyCounterShards = this.hasLegacyCounterShards || hasLegacyCounterShards;
}
- public Map<MetadataType, MetadataComponent> finalizeMetadata(String partitioner, double bloomFilterFPChance, long repairedAt, SerializationHeader header)
+ public Map<MetadataType, MetadataComponent> finalizeMetadata(String partitioner, double bloomFilterFPChance, long repairedAt, UUID pendingRepair, boolean isTransient, SerializationHeader header)
{
Preconditions.checkState((minClustering == null && maxClustering == null)
|| comparator.compare(maxClustering, minClustering) >= 0);
@@ -266,7 +264,9 @@
hasLegacyCounterShards,
repairedAt,
totalColumnsSet,
- totalRows));
+ totalRows,
+ pendingRepair,
+ isTransient));
components.put(MetadataType.COMPACTION, new CompactionMetadata(cardinality));
components.put(MetadataType.HEADER, header.toComponent());
return components;
diff --git a/src/java/org/apache/cassandra/io/sstable/metadata/MetadataSerializer.java b/src/java/org/apache/cassandra/io/sstable/metadata/MetadataSerializer.java
index 779fc07..0ae7c32 100644
--- a/src/java/org/apache/cassandra/io/sstable/metadata/MetadataSerializer.java
+++ b/src/java/org/apache/cassandra/io/sstable/metadata/MetadataSerializer.java
@@ -19,14 +19,19 @@
import java.io.*;
import java.util.*;
+import java.util.function.Function;
+import java.util.zip.CRC32;
import com.google.common.collect.Lists;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.io.sstable.Component;
+import org.apache.cassandra.io.sstable.CorruptSSTableException;
import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.io.sstable.format.Version;
+import org.apache.cassandra.io.util.DataInputBuffer;
+import org.apache.cassandra.io.util.DataOutputBuffer;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.io.util.DataOutputStreamPlus;
import org.apache.cassandra.io.util.FileDataInput;
@@ -35,11 +40,13 @@
import org.apache.cassandra.io.util.RandomAccessReader;
import org.apache.cassandra.utils.FBUtilities;
+import static org.apache.cassandra.utils.FBUtilities.updateChecksumInt;
+
/**
- * Metadata serializer for SSTables {@code version >= 'k'}.
+ * Metadata serializer for SSTables {@code version >= 'na'}.
*
* <pre>
- * File format := | number of components (4 bytes) | toc | component1 | component2 | ... |
+ * File format := | number of components (4 bytes) | crc | toc | crc | component1 | c1 crc | component2 | c2 crc | ... |
* toc := | component type (4 bytes) | position of component |
* </pre>
*
@@ -49,32 +56,61 @@
{
private static final Logger logger = LoggerFactory.getLogger(MetadataSerializer.class);
+ private static final int CHECKSUM_LENGTH = 4; // CRC32
+
public void serialize(Map<MetadataType, MetadataComponent> components, DataOutputPlus out, Version version) throws IOException
{
+ boolean checksum = version.hasMetadataChecksum();
+ CRC32 crc = new CRC32();
// sort components by type
List<MetadataComponent> sortedComponents = Lists.newArrayList(components.values());
Collections.sort(sortedComponents);
// write number of component
out.writeInt(components.size());
+ updateChecksumInt(crc, components.size());
+ maybeWriteChecksum(crc, out, version);
+
// build and write toc
- int lastPosition = 4 + (8 * sortedComponents.size());
+ int lastPosition = 4 + (8 * sortedComponents.size()) + (checksum ? 2 * CHECKSUM_LENGTH : 0);
+ Map<MetadataType, Integer> sizes = new EnumMap<>(MetadataType.class);
for (MetadataComponent component : sortedComponents)
{
MetadataType type = component.getType();
// serialize type
out.writeInt(type.ordinal());
+ updateChecksumInt(crc, type.ordinal());
// serialize position
out.writeInt(lastPosition);
- lastPosition += type.serializer.serializedSize(version, component);
+ updateChecksumInt(crc, lastPosition);
+ int size = type.serializer.serializedSize(version, component);
+ lastPosition += size + (checksum ? CHECKSUM_LENGTH : 0);
+ sizes.put(type, size);
}
+ maybeWriteChecksum(crc, out, version);
+
// serialize components
for (MetadataComponent component : sortedComponents)
{
- component.getType().serializer.serialize(version, component, out);
+ byte[] bytes;
+ try (DataOutputBuffer dob = new DataOutputBuffer(sizes.get(component.getType())))
+ {
+ component.getType().serializer.serialize(version, component, dob);
+ bytes = dob.getData();
+ }
+ out.write(bytes);
+
+ crc.reset(); crc.update(bytes);
+ maybeWriteChecksum(crc, out, version);
}
}
+ private static void maybeWriteChecksum(CRC32 crc, DataOutputPlus out, Version version) throws IOException
+ {
+ if (version.hasMetadataChecksum())
+ out.writeInt((int) crc.getValue());
+ }
+
public Map<MetadataType, MetadataComponent> deserialize( Descriptor descriptor, EnumSet<MetadataType> types) throws IOException
{
Map<MetadataType, MetadataComponent> components;
@@ -101,49 +137,114 @@
return deserialize(descriptor, EnumSet.of(type)).get(type);
}
- public Map<MetadataType, MetadataComponent> deserialize(Descriptor descriptor, FileDataInput in, EnumSet<MetadataType> types) throws IOException
+ public Map<MetadataType, MetadataComponent> deserialize(Descriptor descriptor,
+ FileDataInput in,
+ EnumSet<MetadataType> selectedTypes)
+ throws IOException
{
- Map<MetadataType, MetadataComponent> components = new EnumMap<>(MetadataType.class);
- // read number of components
- int numComponents = in.readInt();
- // read toc
- Map<MetadataType, Integer> toc = new EnumMap<>(MetadataType.class);
- MetadataType[] values = MetadataType.values();
- for (int i = 0; i < numComponents; i++)
+ boolean isChecksummed = descriptor.version.hasMetadataChecksum();
+ CRC32 crc = new CRC32();
+
+ /*
+ * Read TOC
+ */
+
+ int length = (int) in.bytesRemaining();
+
+ int count = in.readInt();
+ updateChecksumInt(crc, count);
+ maybeValidateChecksum(crc, in, descriptor);
+
+ int[] ordinals = new int[count];
+ int[] offsets = new int[count];
+ int[] lengths = new int[count];
+
+ for (int i = 0; i < count; i++)
{
- toc.put(values[in.readInt()], in.readInt());
+ ordinals[i] = in.readInt();
+ updateChecksumInt(crc, ordinals[i]);
+
+ offsets[i] = in.readInt();
+ updateChecksumInt(crc, offsets[i]);
}
- for (MetadataType type : types)
+ maybeValidateChecksum(crc, in, descriptor);
+
+ lengths[count - 1] = length - offsets[count - 1];
+ for (int i = 0; i < count - 1; i++)
+ lengths[i] = offsets[i + 1] - offsets[i];
+
+ /*
+ * Read components
+ */
+
+ MetadataType[] allMetadataTypes = MetadataType.values();
+
+ Map<MetadataType, MetadataComponent> components = new EnumMap<>(MetadataType.class);
+
+ for (int i = 0; i < count; i++)
{
- Integer offset = toc.get(type);
- if (offset != null)
+ MetadataType type = allMetadataTypes[ordinals[i]];
+
+ if (!selectedTypes.contains(type))
{
- in.seek(offset);
- MetadataComponent component = type.serializer.deserialize(descriptor.version, in);
- components.put(type, component);
+ in.skipBytes(lengths[i]);
+ continue;
+ }
+
+ byte[] buffer = new byte[isChecksummed ? lengths[i] - CHECKSUM_LENGTH : lengths[i]];
+ in.readFully(buffer);
+
+ crc.reset(); crc.update(buffer);
+ maybeValidateChecksum(crc, in, descriptor);
+ try (DataInputBuffer dataInputBuffer = new DataInputBuffer(buffer))
+ {
+ components.put(type, type.serializer.deserialize(descriptor.version, dataInputBuffer));
}
}
+
return components;
}
+ private static void maybeValidateChecksum(CRC32 crc, FileDataInput in, Descriptor descriptor) throws IOException
+ {
+ if (!descriptor.version.hasMetadataChecksum())
+ return;
+
+ int actualChecksum = (int) crc.getValue();
+ int expectedChecksum = in.readInt();
+
+ if (actualChecksum != expectedChecksum)
+ {
+ String filename = descriptor.filenameFor(Component.STATS);
+ throw new CorruptSSTableException(new IOException("Checksums do not match for " + filename), filename);
+ }
+ }
+
+ @Override
+ public void mutate(Descriptor descriptor, Function<StatsMetadata, StatsMetadata> transform) throws IOException
+ {
+ Map<MetadataType, MetadataComponent> currentComponents = deserialize(descriptor, EnumSet.allOf(MetadataType.class));
+ StatsMetadata stats = (StatsMetadata) currentComponents.remove(MetadataType.STATS);
+
+ currentComponents.put(MetadataType.STATS, transform.apply(stats));
+ rewriteSSTableMetadata(descriptor, currentComponents);
+ }
+
public void mutateLevel(Descriptor descriptor, int newLevel) throws IOException
{
- logger.trace("Mutating {} to level {}", descriptor.filenameFor(Component.STATS), newLevel);
- Map<MetadataType, MetadataComponent> currentComponents = deserialize(descriptor, EnumSet.allOf(MetadataType.class));
- StatsMetadata stats = (StatsMetadata) currentComponents.remove(MetadataType.STATS);
- // mutate level
- currentComponents.put(MetadataType.STATS, stats.mutateLevel(newLevel));
- rewriteSSTableMetadata(descriptor, currentComponents);
+ if (logger.isTraceEnabled())
+ logger.trace("Mutating {} to level {}", descriptor.filenameFor(Component.STATS), newLevel);
+
+ mutate(descriptor, stats -> stats.mutateLevel(newLevel));
}
- public void mutateRepairedAt(Descriptor descriptor, long newRepairedAt) throws IOException
+ public void mutateRepairMetadata(Descriptor descriptor, long newRepairedAt, UUID newPendingRepair, boolean isTransient) throws IOException
{
- logger.trace("Mutating {} to repairedAt time {}", descriptor.filenameFor(Component.STATS), newRepairedAt);
- Map<MetadataType, MetadataComponent> currentComponents = deserialize(descriptor, EnumSet.allOf(MetadataType.class));
- StatsMetadata stats = (StatsMetadata) currentComponents.remove(MetadataType.STATS);
- // mutate level
- currentComponents.put(MetadataType.STATS, stats.mutateRepairedAt(newRepairedAt));
- rewriteSSTableMetadata(descriptor, currentComponents);
+ if (logger.isTraceEnabled())
+ logger.trace("Mutating {} to repairedAt time {} and pendingRepair {}",
+ descriptor.filenameFor(Component.STATS), newRepairedAt, newPendingRepair);
+
+ mutate(descriptor, stats -> stats.mutateRepairedMetadata(newRepairedAt, newPendingRepair, isTransient));
}
public void rewriteSSTableMetadata(Descriptor descriptor, Map<MetadataType, MetadataComponent> currentComponents) throws IOException
diff --git a/src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java b/src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java
old mode 100644
new mode 100755
index 94e8d41..f4e5beb
--- a/src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java
+++ b/src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java
@@ -21,7 +21,9 @@
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
+import java.util.UUID;
+import org.apache.cassandra.db.rows.EncodingStats;
import org.apache.cassandra.io.ISerializer;
import org.apache.cassandra.io.sstable.format.Version;
import org.apache.commons.lang3.builder.EqualsBuilder;
@@ -33,7 +35,8 @@
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.EstimatedHistogram;
-import org.apache.cassandra.utils.StreamingHistogram;
+import org.apache.cassandra.utils.streamhist.TombstoneHistogram;
+import org.apache.cassandra.utils.UUIDSerializer;
/**
* SSTable metadata that always stay on heap.
@@ -44,7 +47,7 @@
public static final ISerializer<IntervalSet<CommitLogPosition>> commitLogPositionSetSerializer = IntervalSet.serializer(CommitLogPosition.serializer);
public final EstimatedHistogram estimatedPartitionSize;
- public final EstimatedHistogram estimatedColumnCount;
+ public final EstimatedHistogram estimatedCellPerPartitionCount;
public final IntervalSet<CommitLogPosition> commitLogIntervals;
public final long minTimestamp;
public final long maxTimestamp;
@@ -53,7 +56,7 @@
public final int minTTL;
public final int maxTTL;
public final double compressionRatio;
- public final StreamingHistogram estimatedTombstoneDropTime;
+ public final TombstoneHistogram estimatedTombstoneDropTime;
public final int sstableLevel;
public final List<ByteBuffer> minClusteringValues;
public final List<ByteBuffer> maxClusteringValues;
@@ -61,9 +64,13 @@
public final long repairedAt;
public final long totalColumnsSet;
public final long totalRows;
+ public final UUID pendingRepair;
+ public final boolean isTransient;
+ // just holds the current encoding stats to avoid allocating - it is not serialized
+ public final EncodingStats encodingStats;
public StatsMetadata(EstimatedHistogram estimatedPartitionSize,
- EstimatedHistogram estimatedColumnCount,
+ EstimatedHistogram estimatedCellPerPartitionCount,
IntervalSet<CommitLogPosition> commitLogIntervals,
long minTimestamp,
long maxTimestamp,
@@ -72,17 +79,19 @@
int minTTL,
int maxTTL,
double compressionRatio,
- StreamingHistogram estimatedTombstoneDropTime,
+ TombstoneHistogram estimatedTombstoneDropTime,
int sstableLevel,
List<ByteBuffer> minClusteringValues,
List<ByteBuffer> maxClusteringValues,
boolean hasLegacyCounterShards,
long repairedAt,
long totalColumnsSet,
- long totalRows)
+ long totalRows,
+ UUID pendingRepair,
+ boolean isTransient)
{
this.estimatedPartitionSize = estimatedPartitionSize;
- this.estimatedColumnCount = estimatedColumnCount;
+ this.estimatedCellPerPartitionCount = estimatedCellPerPartitionCount;
this.commitLogIntervals = commitLogIntervals;
this.minTimestamp = minTimestamp;
this.maxTimestamp = maxTimestamp;
@@ -99,6 +108,9 @@
this.repairedAt = repairedAt;
this.totalColumnsSet = totalColumnsSet;
this.totalRows = totalRows;
+ this.pendingRepair = pendingRepair;
+ this.isTransient = isTransient;
+ this.encodingStats = new EncodingStats(minTimestamp, minLocalDeletionTime, minTTL);
}
public MetadataType getType()
@@ -112,7 +124,7 @@
*/
public double getEstimatedDroppableTombstoneRatio(int gcBefore)
{
- long estimatedColumnCount = this.estimatedColumnCount.mean() * this.estimatedColumnCount.count();
+ long estimatedColumnCount = this.estimatedCellPerPartitionCount.mean() * this.estimatedCellPerPartitionCount.count();
if (estimatedColumnCount > 0)
{
double droppable = getDroppableTombstonesBefore(gcBefore);
@@ -133,7 +145,7 @@
public StatsMetadata mutateLevel(int newLevel)
{
return new StatsMetadata(estimatedPartitionSize,
- estimatedColumnCount,
+ estimatedCellPerPartitionCount,
commitLogIntervals,
minTimestamp,
maxTimestamp,
@@ -149,13 +161,15 @@
hasLegacyCounterShards,
repairedAt,
totalColumnsSet,
- totalRows);
+ totalRows,
+ pendingRepair,
+ isTransient);
}
- public StatsMetadata mutateRepairedAt(long newRepairedAt)
+ public StatsMetadata mutateRepairedMetadata(long newRepairedAt, UUID newPendingRepair, boolean newIsTransient)
{
return new StatsMetadata(estimatedPartitionSize,
- estimatedColumnCount,
+ estimatedCellPerPartitionCount,
commitLogIntervals,
minTimestamp,
maxTimestamp,
@@ -171,7 +185,9 @@
hasLegacyCounterShards,
newRepairedAt,
totalColumnsSet,
- totalRows);
+ totalRows,
+ newPendingRepair,
+ newIsTransient);
}
@Override
@@ -183,7 +199,7 @@
StatsMetadata that = (StatsMetadata) o;
return new EqualsBuilder()
.append(estimatedPartitionSize, that.estimatedPartitionSize)
- .append(estimatedColumnCount, that.estimatedColumnCount)
+ .append(estimatedCellPerPartitionCount, that.estimatedCellPerPartitionCount)
.append(commitLogIntervals, that.commitLogIntervals)
.append(minTimestamp, that.minTimestamp)
.append(maxTimestamp, that.maxTimestamp)
@@ -200,6 +216,7 @@
.append(hasLegacyCounterShards, that.hasLegacyCounterShards)
.append(totalColumnsSet, that.totalColumnsSet)
.append(totalRows, that.totalRows)
+ .append(pendingRepair, that.pendingRepair)
.build();
}
@@ -208,7 +225,7 @@
{
return new HashCodeBuilder()
.append(estimatedPartitionSize)
- .append(estimatedColumnCount)
+ .append(estimatedCellPerPartitionCount)
.append(commitLogIntervals)
.append(minTimestamp)
.append(maxTimestamp)
@@ -225,6 +242,7 @@
.append(hasLegacyCounterShards)
.append(totalColumnsSet)
.append(totalRows)
+ .append(pendingRepair)
.build();
}
@@ -234,13 +252,10 @@
{
int size = 0;
size += EstimatedHistogram.serializer.serializedSize(component.estimatedPartitionSize);
- size += EstimatedHistogram.serializer.serializedSize(component.estimatedColumnCount);
+ size += EstimatedHistogram.serializer.serializedSize(component.estimatedCellPerPartitionCount);
size += CommitLogPosition.serializer.serializedSize(component.commitLogIntervals.upperBound().orElse(CommitLogPosition.NONE));
- if (version.storeRows())
- size += 8 + 8 + 4 + 4 + 4 + 4 + 8 + 8; // mix/max timestamp(long), min/maxLocalDeletionTime(int), min/max TTL, compressionRatio(double), repairedAt (long)
- else
- size += 8 + 8 + 4 + 8 + 8; // mix/max timestamp(long), maxLocalDeletionTime(int), compressionRatio(double), repairedAt (long)
- size += StreamingHistogram.serializer.serializedSize(component.estimatedTombstoneDropTime);
+ size += 8 + 8 + 4 + 4 + 4 + 4 + 8 + 8; // mix/max timestamp(long), min/maxLocalDeletionTime(int), min/max TTL, compressionRatio(double), repairedAt (long)
+ size += TombstoneHistogram.serializer.serializedSize(component.estimatedTombstoneDropTime);
size += TypeSizes.sizeof(component.sstableLevel);
// min column names
size += 4;
@@ -251,32 +266,40 @@
for (ByteBuffer value : component.maxClusteringValues)
size += 2 + value.remaining(); // with short length
size += TypeSizes.sizeof(component.hasLegacyCounterShards);
- if (version.storeRows())
- size += 8 + 8; // totalColumnsSet, totalRows
+ size += 8 + 8; // totalColumnsSet, totalRows
if (version.hasCommitLogLowerBound())
size += CommitLogPosition.serializer.serializedSize(component.commitLogIntervals.lowerBound().orElse(CommitLogPosition.NONE));
if (version.hasCommitLogIntervals())
size += commitLogPositionSetSerializer.serializedSize(component.commitLogIntervals);
+
+ if (version.hasPendingRepair())
+ {
+ size += 1;
+ if (component.pendingRepair != null)
+ size += UUIDSerializer.serializer.serializedSize(component.pendingRepair, 0);
+ }
+
+ if (version.hasIsTransient())
+ {
+ size += TypeSizes.sizeof(component.isTransient);
+ }
+
return size;
}
public void serialize(Version version, StatsMetadata component, DataOutputPlus out) throws IOException
{
EstimatedHistogram.serializer.serialize(component.estimatedPartitionSize, out);
- EstimatedHistogram.serializer.serialize(component.estimatedColumnCount, out);
+ EstimatedHistogram.serializer.serialize(component.estimatedCellPerPartitionCount, out);
CommitLogPosition.serializer.serialize(component.commitLogIntervals.upperBound().orElse(CommitLogPosition.NONE), out);
out.writeLong(component.minTimestamp);
out.writeLong(component.maxTimestamp);
- if (version.storeRows())
- out.writeInt(component.minLocalDeletionTime);
+ out.writeInt(component.minLocalDeletionTime);
out.writeInt(component.maxLocalDeletionTime);
- if (version.storeRows())
- {
- out.writeInt(component.minTTL);
- out.writeInt(component.maxTTL);
- }
+ out.writeInt(component.minTTL);
+ out.writeInt(component.maxTTL);
out.writeDouble(component.compressionRatio);
- StreamingHistogram.serializer.serialize(component.estimatedTombstoneDropTime, out);
+ TombstoneHistogram.serializer.serialize(component.estimatedTombstoneDropTime, out);
out.writeInt(component.sstableLevel);
out.writeLong(component.repairedAt);
out.writeInt(component.minClusteringValues.size());
@@ -287,16 +310,31 @@
ByteBufferUtil.writeWithShortLength(value, out);
out.writeBoolean(component.hasLegacyCounterShards);
- if (version.storeRows())
- {
- out.writeLong(component.totalColumnsSet);
- out.writeLong(component.totalRows);
- }
+ out.writeLong(component.totalColumnsSet);
+ out.writeLong(component.totalRows);
if (version.hasCommitLogLowerBound())
CommitLogPosition.serializer.serialize(component.commitLogIntervals.lowerBound().orElse(CommitLogPosition.NONE), out);
if (version.hasCommitLogIntervals())
commitLogPositionSetSerializer.serialize(component.commitLogIntervals, out);
+
+ if (version.hasPendingRepair())
+ {
+ if (component.pendingRepair != null)
+ {
+ out.writeByte(1);
+ UUIDSerializer.serializer.serialize(component.pendingRepair, out, 0);
+ }
+ else
+ {
+ out.writeByte(0);
+ }
+ }
+
+ if (version.hasIsTransient())
+ {
+ out.writeBoolean(component.isTransient);
+ }
}
public StatsMetadata deserialize(Version version, DataInputPlus in) throws IOException
@@ -307,17 +345,14 @@
commitLogUpperBound = CommitLogPosition.serializer.deserialize(in);
long minTimestamp = in.readLong();
long maxTimestamp = in.readLong();
- // We use MAX_VALUE as that's the default value for "no deletion time"
- int minLocalDeletionTime = version.storeRows() ? in.readInt() : Integer.MAX_VALUE;
+ int minLocalDeletionTime = in.readInt();
int maxLocalDeletionTime = in.readInt();
- int minTTL = version.storeRows() ? in.readInt() : 0;
- int maxTTL = version.storeRows() ? in.readInt() : Integer.MAX_VALUE;
+ int minTTL = in.readInt();
+ int maxTTL = in.readInt();
double compressionRatio = in.readDouble();
- StreamingHistogram tombstoneHistogram = StreamingHistogram.serializer.deserialize(in);
+ TombstoneHistogram tombstoneHistogram = TombstoneHistogram.serializer.deserialize(in);
int sstableLevel = in.readInt();
- long repairedAt = 0;
- if (version.hasRepairedAt())
- repairedAt = in.readLong();
+ long repairedAt = in.readLong();
// for legacy sstables, we skip deserializing the min and max clustering value
// to prevent erroneously excluding sstables from reads (see CASSANDRA-14861)
@@ -339,12 +374,10 @@
maxClusteringValues.add(val);
}
- boolean hasLegacyCounterShards = true;
- if (version.tracksLegacyCounterShards())
- hasLegacyCounterShards = in.readBoolean();
+ boolean hasLegacyCounterShards = in.readBoolean();
- long totalColumnsSet = version.storeRows() ? in.readLong() : -1L;
- long totalRows = version.storeRows() ? in.readLong() : -1L;
+ long totalColumnsSet = in.readLong();
+ long totalRows = in.readLong();
if (version.hasCommitLogLowerBound())
commitLogLowerBound = CommitLogPosition.serializer.deserialize(in);
@@ -354,6 +387,14 @@
else
commitLogIntervals = new IntervalSet<CommitLogPosition>(commitLogLowerBound, commitLogUpperBound);
+ UUID pendingRepair = null;
+ if (version.hasPendingRepair() && in.readByte() != 0)
+ {
+ pendingRepair = UUIDSerializer.serializer.deserialize(in, 0);
+ }
+
+ boolean isTransient = version.hasIsTransient() && in.readBoolean();
+
return new StatsMetadata(partitionSizes,
columnCounts,
commitLogIntervals,
@@ -371,7 +412,9 @@
hasLegacyCounterShards,
repairedAt,
totalColumnsSet,
- totalRows);
+ totalRows,
+ pendingRepair,
+ isTransient);
}
}
}
diff --git a/src/java/org/apache/cassandra/io/util/BufferManagingRebufferer.java b/src/java/org/apache/cassandra/io/util/BufferManagingRebufferer.java
index 1648bcf..f3b9a88 100644
--- a/src/java/org/apache/cassandra/io/util/BufferManagingRebufferer.java
+++ b/src/java/org/apache/cassandra/io/util/BufferManagingRebufferer.java
@@ -89,7 +89,7 @@
@Override
public String toString()
{
- return "BufferManagingRebufferer." + getClass().getSimpleName() + ":" + source.toString();
+ return "BufferManagingRebufferer." + getClass().getSimpleName() + ":" + source;
}
// BufferHolder methods
diff --git a/src/java/org/apache/cassandra/io/util/BufferedDataOutputStreamPlus.java b/src/java/org/apache/cassandra/io/util/BufferedDataOutputStreamPlus.java
index 54122ee..7d1e91d 100644
--- a/src/java/org/apache/cassandra/io/util/BufferedDataOutputStreamPlus.java
+++ b/src/java/org/apache/cassandra/io/util/BufferedDataOutputStreamPlus.java
@@ -24,14 +24,12 @@
import java.nio.ByteOrder;
import java.nio.channels.WritableByteChannel;
-import com.google.common.base.Function;
import com.google.common.base.Preconditions;
import net.nicoulaj.compilecommand.annotations.DontInline;
-
import org.apache.cassandra.config.Config;
+import org.apache.cassandra.utils.FastByteOperations;
import org.apache.cassandra.utils.memory.MemoryUtil;
-import org.apache.cassandra.utils.vint.VIntCoding;
/**
* An implementation of the DataOutputStreamPlus interface using a ByteBuffer to stage writes
@@ -45,15 +43,6 @@
protected ByteBuffer buffer;
- //Allow derived classes to specify writing to the channel
- //directly shouldn't happen because they intercept via doFlush for things
- //like compression or checksumming
- //Another hack for this value is that it also indicates that flushing early
- //should not occur, flushes aligned with buffer size are desired
- //Unless... it's the last flush. Compression and checksum formats
- //expect block (same as buffer size) alignment for everything except the last block
- protected boolean strictFlushing = false;
-
public BufferedDataOutputStreamPlus(RandomAccessFile ras)
{
this(ras.getChannel());
@@ -134,9 +123,6 @@
}
}
- // ByteBuffer to use for defensive copies
- private final ByteBuffer hollowBuffer = MemoryUtil.getHollowDirectByteBuffer();
-
/*
* Makes a defensive copy of the incoming ByteBuffer and don't modify the position or limit
* even temporarily so it is thread-safe WRT to the incoming buffer
@@ -144,48 +130,20 @@
* @see org.apache.cassandra.io.util.DataOutputPlus#write(java.nio.ByteBuffer)
*/
@Override
- public void write(ByteBuffer toWrite) throws IOException
+ public void write(ByteBuffer src) throws IOException
{
- if (toWrite.hasArray())
+ int srcPos = src.position();
+ int srcCount;
+ int trgAvailable;
+ while ((srcCount = src.limit() - srcPos) > (trgAvailable = buffer.remaining()))
{
- write(toWrite.array(), toWrite.arrayOffset() + toWrite.position(), toWrite.remaining());
+ FastByteOperations.copy(src, srcPos, buffer, buffer.position(), trgAvailable);
+ buffer.position(buffer.position() + trgAvailable);
+ srcPos += trgAvailable;
+ doFlush(src.limit() - srcPos);
}
- else
- {
- assert toWrite.isDirect();
- MemoryUtil.duplicateDirectByteBuffer(toWrite, hollowBuffer);
- int toWriteRemaining = toWrite.remaining();
-
- if (toWriteRemaining > buffer.remaining())
- {
- if (strictFlushing)
- {
- writeExcessSlow();
- }
- else
- {
- doFlush(toWriteRemaining - buffer.remaining());
- while (hollowBuffer.remaining() > buffer.capacity())
- channel.write(hollowBuffer);
- }
- }
-
- buffer.put(hollowBuffer);
- }
- }
-
- // writes anything we can't fit into the buffer
- @DontInline
- private void writeExcessSlow() throws IOException
- {
- int originalLimit = hollowBuffer.limit();
- while (originalLimit - hollowBuffer.position() > buffer.remaining())
- {
- hollowBuffer.limit(hollowBuffer.position() + buffer.remaining());
- buffer.put(hollowBuffer);
- doFlush(originalLimit - hollowBuffer.position());
- }
- hollowBuffer.limit(originalLimit);
+ FastByteOperations.copy(src, srcPos, buffer, buffer.position(), srcCount);
+ buffer.position(buffer.position() + srcCount);
}
@Override
@@ -244,25 +202,6 @@
}
@Override
- public void writeVInt(long value) throws IOException
- {
- writeUnsignedVInt(VIntCoding.encodeZigZag64(value));
- }
-
- @Override
- public void writeUnsignedVInt(long value) throws IOException
- {
- int size = VIntCoding.computeUnsignedVIntSize(value);
- if (size == 1)
- {
- write((int) value);
- return;
- }
-
- write(VIntCoding.encodeVInt(value, size), 0, size);
- }
-
- @Override
public void writeFloat(float v) throws IOException
{
writeInt(Float.floatToRawIntBits(v));
@@ -304,13 +243,6 @@
UnbufferedDataOutputStreamPlus.writeUTF(s, this);
}
- @Override
- public void write(Memory memory, long offset, long length) throws IOException
- {
- for (ByteBuffer buffer : memory.asByteBuffers(offset, length))
- write(buffer);
- }
-
/*
* Count is the number of bytes remaining to write ignoring already remaining capacity
*/
@@ -340,16 +272,6 @@
buffer = null;
}
- @Override
- public <R> R applyToChannel(Function<WritableByteChannel, R> f) throws IOException
- {
- if (strictFlushing)
- throw new UnsupportedOperationException();
- //Don't allow writes to the underlying channel while data is buffered
- flush();
- return f.apply(channel);
- }
-
public BufferedDataOutputStreamPlus order(ByteOrder order)
{
this.buffer.order(order);
diff --git a/src/java/org/apache/cassandra/io/util/ChannelProxy.java b/src/java/org/apache/cassandra/io/util/ChannelProxy.java
index 91bb03b..9ff46b7 100644
--- a/src/java/org/apache/cassandra/io/util/ChannelProxy.java
+++ b/src/java/org/apache/cassandra/io/util/ChannelProxy.java
@@ -111,6 +111,16 @@
}
}
+ /**
+ * {@link #sharedCopy()} can not be used if thread will be interruped, as the backing channel will be closed.
+ *
+ * @return a new channel instance
+ */
+ public final ChannelProxy newChannel()
+ {
+ return new ChannelProxy(filePath);
+ }
+
public ChannelProxy sharedCopy()
{
return new ChannelProxy(this);
diff --git a/src/java/org/apache/cassandra/io/util/CheckedFunction.java b/src/java/org/apache/cassandra/io/util/CheckedFunction.java
new file mode 100644
index 0000000..ec1ce9f
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/util/CheckedFunction.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.util;
+
+@FunctionalInterface
+public interface CheckedFunction<T, R, E extends Exception>
+{
+ R apply(T t) throws E;
+}
diff --git a/src/java/org/apache/cassandra/io/util/CompressedChunkReader.java b/src/java/org/apache/cassandra/io/util/CompressedChunkReader.java
index 177afb0..daec6c4 100644
--- a/src/java/org/apache/cassandra/io/util/CompressedChunkReader.java
+++ b/src/java/org/apache/cassandra/io/util/CompressedChunkReader.java
@@ -20,7 +20,6 @@
import java.io.IOException;
import java.nio.ByteBuffer;
-import java.util.concurrent.ThreadLocalRandom;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.primitives.Ints;
@@ -29,15 +28,18 @@
import org.apache.cassandra.io.compress.CompressionMetadata;
import org.apache.cassandra.io.compress.CorruptBlockException;
import org.apache.cassandra.io.sstable.CorruptSSTableException;
+import org.apache.cassandra.utils.ChecksumType;
public abstract class CompressedChunkReader extends AbstractReaderFileProxy implements ChunkReader
{
final CompressionMetadata metadata;
+ final int maxCompressedLength;
protected CompressedChunkReader(ChannelProxy channel, CompressionMetadata metadata)
{
super(channel, metadata.dataLength);
this.metadata = metadata;
+ this.maxCompressedLength = metadata.maxCompressedLength();
assert Integer.bitCount(metadata.chunkLength()) == 1; //must be a power of two
}
@@ -47,9 +49,9 @@
return metadata.parameters.getCrcCheckChance();
}
- protected final boolean shouldCheckCrc()
+ boolean shouldCheckCrc()
{
- return getCrcCheckChance() >= 1d || getCrcCheckChance() > ThreadLocalRandom.current().nextDouble();
+ return metadata.parameters.shouldCheckCrc();
}
@Override
@@ -94,7 +96,12 @@
public ByteBuffer allocateBuffer()
{
- return allocateBuffer(metadata.compressor().initialCompressedBufferLength(metadata.chunkLength()));
+ int compressedLength = Math.min(maxCompressedLength,
+ metadata.compressor().initialCompressedBufferLength(metadata.chunkLength()));
+
+ int checksumLength = Integer.BYTES;
+
+ return allocateBuffer(compressedLength + checksumLength);
}
public ByteBuffer allocateBuffer(int size)
@@ -112,57 +119,68 @@
assert position <= fileLength;
CompressionMetadata.Chunk chunk = metadata.chunkFor(position);
- ByteBuffer compressed = compressedHolder.get();
-
boolean shouldCheckCrc = shouldCheckCrc();
+ int length = shouldCheckCrc ? chunk.length + Integer.BYTES // compressed length + checksum length
+ : chunk.length;
- int length = shouldCheckCrc ? chunk.length + Integer.BYTES : chunk.length;
-
- if (compressed.capacity() < length)
+ if (chunk.length < maxCompressedLength)
{
- compressed = allocateBuffer(length);
- compressedHolder.set(compressed);
+ ByteBuffer compressed = compressedHolder.get();
+
+ assert compressed.capacity() >= length;
+ compressed.clear().limit(length);
+ if (channel.read(compressed, chunk.offset) != length)
+ throw new CorruptBlockException(channel.filePath(), chunk);
+
+ compressed.flip();
+ compressed.limit(chunk.length);
+ uncompressed.clear();
+
+ if (shouldCheckCrc)
+ {
+ int checksum = (int) ChecksumType.CRC32.of(compressed);
+
+ compressed.limit(length);
+ if (compressed.getInt() != checksum)
+ throw new CorruptBlockException(channel.filePath(), chunk);
+
+ compressed.position(0).limit(chunk.length);
+ }
+
+ try
+ {
+ metadata.compressor().uncompress(compressed, uncompressed);
+ }
+ catch (IOException e)
+ {
+ throw new CorruptBlockException(channel.filePath(), chunk, e);
+ }
}
else
{
- compressed.clear();
- }
-
- compressed.limit(length);
- if (channel.read(compressed, chunk.offset) != length)
- throw new CorruptBlockException(channel.filePath(), chunk);
-
- compressed.flip();
- uncompressed.clear();
-
- compressed.position(0).limit(chunk.length);
-
- if (shouldCheckCrc)
- {
- int checksum = (int) metadata.checksumType.of(compressed);
-
- compressed.limit(length);
- if (compressed.getInt() != checksum)
+ uncompressed.position(0).limit(chunk.length);
+ if (channel.read(uncompressed, chunk.offset) != chunk.length)
throw new CorruptBlockException(channel.filePath(), chunk);
- compressed.position(0).limit(chunk.length);
- }
+ if (shouldCheckCrc)
+ {
+ uncompressed.flip();
+ int checksum = (int) ChecksumType.CRC32.of(uncompressed);
- try
- {
- metadata.compressor().uncompress(compressed, uncompressed);
+ ByteBuffer scratch = compressedHolder.get();
+ scratch.clear().limit(Integer.BYTES);
+
+ if (channel.read(scratch, chunk.offset + chunk.length) != Integer.BYTES
+ || scratch.getInt(0) != checksum)
+ throw new CorruptBlockException(channel.filePath(), chunk);
+ }
}
- catch (IOException e)
- {
- throw new CorruptBlockException(channel.filePath(), chunk, e);
- }
- finally
- {
- uncompressed.flip();
- }
+ uncompressed.flip();
}
catch (CorruptBlockException e)
{
+ // Make sure reader does not see stale data.
+ uncompressed.position(0).limit(0);
throw new CorruptSSTableException(e, channel.filePath());
}
}
@@ -198,32 +216,34 @@
uncompressed.clear();
- if (shouldCheckCrc())
- {
- int checksum = (int) metadata.checksumType.of(compressedChunk);
-
- compressedChunk.limit(compressedChunk.capacity());
- if (compressedChunk.getInt() != checksum)
- throw new CorruptBlockException(channel.filePath(), chunk);
-
- compressedChunk.position(chunkOffset).limit(chunkOffset + chunk.length);
- }
-
try
{
- metadata.compressor().uncompress(compressedChunk, uncompressed);
+ if (shouldCheckCrc())
+ {
+ int checksum = (int) ChecksumType.CRC32.of(compressedChunk);
+
+ compressedChunk.limit(compressedChunk.capacity());
+ if (compressedChunk.getInt() != checksum)
+ throw new CorruptBlockException(channel.filePath(), chunk);
+
+ compressedChunk.position(chunkOffset).limit(chunkOffset + chunk.length);
+ }
+
+ if (chunk.length < maxCompressedLength)
+ metadata.compressor().uncompress(compressedChunk, uncompressed);
+ else
+ uncompressed.put(compressedChunk);
}
catch (IOException e)
{
throw new CorruptBlockException(channel.filePath(), chunk, e);
}
- finally
- {
- uncompressed.flip();
- }
+ uncompressed.flip();
}
catch (CorruptBlockException e)
{
+ // Make sure reader does not see stale data.
+ uncompressed.position(0).limit(0);
throw new CorruptSSTableException(e, channel.filePath());
}
diff --git a/src/java/org/apache/cassandra/io/util/DataIntegrityMetadata.java b/src/java/org/apache/cassandra/io/util/DataIntegrityMetadata.java
index 9afd0c0..277b359 100644
--- a/src/java/org/apache/cassandra/io/util/DataIntegrityMetadata.java
+++ b/src/java/org/apache/cassandra/io/util/DataIntegrityMetadata.java
@@ -20,9 +20,12 @@
import java.io.Closeable;
import java.io.File;
import java.io.IOException;
+import java.nio.ByteBuffer;
import java.util.zip.CheckedInputStream;
import java.util.zip.Checksum;
+import com.google.common.annotations.VisibleForTesting;
+
import org.apache.cassandra.io.sstable.Component;
import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.utils.ChecksumType;
@@ -44,7 +47,7 @@
public ChecksumValidator(Descriptor descriptor) throws IOException
{
- this(descriptor.version.uncompressedChecksumType(),
+ this(ChecksumType.CRC32,
RandomAccessReader.open(new File(descriptor.filenameFor(Component.CRC))),
descriptor.filenameFor(Component.DATA));
}
@@ -57,6 +60,15 @@
chunkSize = reader.readInt();
}
+ @VisibleForTesting
+ protected ChecksumValidator(ChecksumType checksumType, RandomAccessReader reader, int chunkSize)
+ {
+ this.checksumType = checksumType;
+ this.reader = reader;
+ this.dataFilename = null;
+ this.chunkSize = chunkSize;
+ }
+
public void seek(long offset)
{
long start = chunkStart(offset);
@@ -77,6 +89,20 @@
throw new IOException("Corrupted File : " + dataFilename);
}
+ /**
+ * validates the checksum with the bytes from the specified buffer.
+ *
+ * Upon return, the buffer's position will
+ * be updated to its limit; its limit will not have been changed.
+ */
+ public void validate(ByteBuffer buffer) throws IOException
+ {
+ int current = (int) checksumType.of(buffer);
+ int actual = reader.readInt();
+ if (current != actual)
+ throw new IOException("Corrupted File : " + dataFilename);
+ }
+
public void close()
{
reader.close();
@@ -99,8 +125,8 @@
public FileDigestValidator(Descriptor descriptor) throws IOException
{
this.descriptor = descriptor;
- checksum = descriptor.version.uncompressedChecksumType().newInstance();
- digestReader = RandomAccessReader.open(new File(descriptor.filenameFor(descriptor.digestComponent)));
+ checksum = ChecksumType.CRC32.newInstance();
+ digestReader = RandomAccessReader.open(new File(descriptor.filenameFor(Component.DIGEST)));
dataReader = RandomAccessReader.open(new File(descriptor.filenameFor(Component.DATA)));
try
{
diff --git a/src/java/org/apache/cassandra/io/util/DataOutputPlus.java b/src/java/org/apache/cassandra/io/util/DataOutputPlus.java
index a9dbb68..b94d097 100644
--- a/src/java/org/apache/cassandra/io/util/DataOutputPlus.java
+++ b/src/java/org/apache/cassandra/io/util/DataOutputPlus.java
@@ -20,9 +20,6 @@
import java.io.DataOutput;
import java.io.IOException;
import java.nio.ByteBuffer;
-import java.nio.channels.WritableByteChannel;
-
-import com.google.common.base.Function;
import org.apache.cassandra.utils.vint.VIntCoding;
@@ -35,13 +32,11 @@
// write the buffer without modifying its position
void write(ByteBuffer buffer) throws IOException;
- void write(Memory memory, long offset, long length) throws IOException;
-
- /**
- * Safe way to operate against the underlying channel. Impossible to stash a reference to the channel
- * and forget to flush
- */
- <R> R applyToChannel(Function<WritableByteChannel, R> c) throws IOException;
+ default void write(Memory memory, long offset, long length) throws IOException
+ {
+ for (ByteBuffer buffer : memory.asByteBuffers(offset, length))
+ write(buffer);
+ }
default void writeVInt(long i) throws IOException
{
diff --git a/src/java/org/apache/cassandra/io/util/DataOutputStreamPlus.java b/src/java/org/apache/cassandra/io/util/DataOutputStreamPlus.java
index 4adb6d2..e931899 100644
--- a/src/java/org/apache/cassandra/io/util/DataOutputStreamPlus.java
+++ b/src/java/org/apache/cassandra/io/util/DataOutputStreamPlus.java
@@ -119,7 +119,7 @@
{
int toWriteThisTime = Math.min(buf.length, toWrite - totalWritten);
- ByteBufferUtil.arrayCopy(src, src.position() + totalWritten, buf, 0, toWriteThisTime);
+ ByteBufferUtil.copyBytes(src, src.position() + totalWritten, buf, 0, toWriteThisTime);
DataOutputStreamPlus.this.write(buf, 0, toWriteThisTime);
diff --git a/src/java/org/apache/cassandra/io/util/FastByteArrayInputStream.java b/src/java/org/apache/cassandra/io/util/FastByteArrayInputStream.java
deleted file mode 100644
index f61546c..0000000
--- a/src/java/org/apache/cassandra/io/util/FastByteArrayInputStream.java
+++ /dev/null
@@ -1,249 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.io.util;
-
-import java.io.ByteArrayInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-
-/*
- * This file has been modified from Apache Harmony's ByteArrayInputStream
- * implementation. The synchronized methods of the original have been
- * replaced by non-synchronized methods. This makes this certain operations
- * FASTer, but also *not thread-safe*.
- *
- * This file remains formatted the same as the Apache Harmony original to
- * make patching easier if any bug fixes are made to the Harmony version.
- */
-
-/**
- * A specialized {@link InputStream } for reading the contents of a byte array.
- *
- * @see ByteArrayInputStream
- */
-public class FastByteArrayInputStream extends InputStream
-{
- /**
- * The {@code byte} array containing the bytes to stream over.
- */
- protected byte[] buf;
-
- /**
- * The current position within the byte array.
- */
- protected int pos;
-
- /**
- * The current mark position. Initially set to 0 or the <code>offset</code>
- * parameter within the constructor.
- */
- protected int mark;
-
- /**
- * The total number of bytes initially available in the byte array
- * {@code buf}.
- */
- protected int count;
-
- /**
- * Constructs a new {@code ByteArrayInputStream} on the byte array
- * {@code buf}.
- *
- * @param buf
- * the byte array to stream over.
- */
- public FastByteArrayInputStream(byte buf[])
- {
- this.mark = 0;
- this.buf = buf;
- this.count = buf.length;
- }
-
- /**
- * Constructs a new {@code ByteArrayInputStream} on the byte array
- * {@code buf} with the initial position set to {@code offset} and the
- * number of bytes available set to {@code offset} + {@code length}.
- *
- * @param buf
- * the byte array to stream over.
- * @param offset
- * the initial position in {@code buf} to start streaming from.
- * @param length
- * the number of bytes available for streaming.
- */
- public FastByteArrayInputStream(byte buf[], int offset, int length)
- {
- this.buf = buf;
- pos = offset;
- mark = offset;
- count = offset + length > buf.length ? buf.length : offset + length;
- }
-
- /**
- * Returns the number of bytes that are available before this stream will
- * block. This method returns the number of bytes yet to be read from the
- * source byte array.
- *
- * @return the number of bytes available before blocking.
- */
- @Override
- public int available()
- {
- return count - pos;
- }
-
- /**
- * Closes this stream and frees resources associated with this stream.
- *
- * @throws IOException
- * if an I/O error occurs while closing this stream.
- */
- @Override
- public void close() throws IOException
- {
- // Do nothing on close, this matches JDK behaviour.
- }
-
- /**
- * Sets a mark position in this ByteArrayInputStream. The parameter
- * {@code readlimit} is ignored. Sending {@code reset()} will reposition the
- * stream back to the marked position.
- *
- * @param readlimit
- * ignored.
- * @see #markSupported()
- * @see #reset()
- */
- @Override
- public void mark(int readlimit)
- {
- mark = pos;
- }
-
- /**
- * Indicates whether this stream supports the {@code mark()} and
- * {@code reset()} methods. Returns {@code true} since this class supports
- * these methods.
- *
- * @return always {@code true}.
- * @see #mark(int)
- * @see #reset()
- */
- @Override
- public boolean markSupported()
- {
- return true;
- }
-
- /**
- * Reads a single byte from the source byte array and returns it as an
- * integer in the range from 0 to 255. Returns -1 if the end of the source
- * array has been reached.
- *
- * @return the byte read or -1 if the end of this stream has been reached.
- */
- @Override
- public int read()
- {
- return pos < count ? buf[pos++] & 0xFF : -1;
- }
-
- /**
- * Reads at most {@code len} bytes from this stream and stores
- * them in byte array {@code b} starting at {@code offset}. This
- * implementation reads bytes from the source byte array.
- *
- * @param b
- * the byte array in which to store the bytes read.
- * @param offset
- * the initial position in {@code b} to store the bytes read from
- * this stream.
- * @param length
- * the maximum number of bytes to store in {@code b}.
- * @return the number of bytes actually read or -1 if no bytes were read and
- * the end of the stream was encountered.
- * @throws IndexOutOfBoundsException
- * if {@code offset < 0} or {@code length < 0}, or if
- * {@code offset + length} is greater than the size of
- * {@code b}.
- * @throws NullPointerException
- * if {@code b} is {@code null}.
- */
- @Override
- public int read(byte b[], int offset, int length)
- {
- if (b == null) {
- throw new NullPointerException();
- }
- // avoid int overflow
- if (offset < 0 || offset > b.length || length < 0
- || length > b.length - offset)
- {
- throw new IndexOutOfBoundsException();
- }
- // Are there any bytes available?
- if (this.pos >= this.count)
- {
- return -1;
- }
- if (length == 0)
- {
- return 0;
- }
-
- int copylen = this.count - pos < length ? this.count - pos : length;
- System.arraycopy(buf, pos, b, offset, copylen);
- pos += copylen;
- return copylen;
- }
-
- /**
- * Resets this stream to the last marked location. This implementation
- * resets the position to either the marked position, the start position
- * supplied in the constructor or 0 if neither has been provided.
- *
- * @see #mark(int)
- */
- @Override
- public void reset()
- {
- pos = mark;
- }
-
- /**
- * Skips {@code count} number of bytes in this InputStream. Subsequent
- * {@code read()}s will not return these bytes unless {@code reset()} is
- * used. This implementation skips {@code count} number of bytes in the
- * target stream. It does nothing and returns 0 if {@code n} is negative.
- *
- * @param n
- * the number of bytes to skip.
- * @return the number of bytes actually skipped.
- */
- @Override
- public long skip(long n)
- {
- if (n <= 0)
- {
- return 0;
- }
- int temp = pos;
- pos = this.count - pos < n ? this.count : (int) (pos + n);
- return pos - temp;
- }
-}
diff --git a/src/java/org/apache/cassandra/io/util/FileHandle.java b/src/java/org/apache/cassandra/io/util/FileHandle.java
index a3afc2f..b705769 100644
--- a/src/java/org/apache/cassandra/io/util/FileHandle.java
+++ b/src/java/org/apache/cassandra/io/util/FileHandle.java
@@ -33,6 +33,7 @@
import org.apache.cassandra.utils.concurrent.SharedCloseableImpl;
import static org.apache.cassandra.utils.Throwables.maybeFail;
+import org.apache.cassandra.utils.Throwables;
/**
* {@link FileHandle} provides access to a file for reading, including the ones written by various {@link SequentialWriter}
@@ -341,9 +342,11 @@
@SuppressWarnings("resource")
public FileHandle complete(long overrideLength)
{
+ boolean channelOpened = false;
if (channel == null)
{
channel = new ChannelProxy(path);
+ channelOpened = true;
}
ChannelProxy channelCopy = channel.sharedCopy();
@@ -388,6 +391,12 @@
catch (Throwable t)
{
channelCopy.close();
+ if (channelOpened)
+ {
+ ChannelProxy c = channel;
+ channel = null;
+ throw Throwables.cleaned(c.close(t));
+ }
throw t;
}
}
diff --git a/src/java/org/apache/cassandra/io/util/FileUtils.java b/src/java/org/apache/cassandra/io/util/FileUtils.java
index 6467a07..2be6b5e 100644
--- a/src/java/org/apache/cassandra/io/util/FileUtils.java
+++ b/src/java/org/apache/cassandra/io/util/FileUtils.java
@@ -18,7 +18,11 @@
package org.apache.cassandra.io.util;
import java.io.*;
+import java.lang.invoke.MethodHandle;
+import java.lang.invoke.MethodHandles;
+import java.lang.reflect.Method;
import java.nio.ByteBuffer;
+import java.nio.channels.Channels;
import java.nio.channels.FileChannel;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
@@ -29,16 +33,15 @@
import java.text.DecimalFormat;
import java.util.Arrays;
import java.util.Collections;
+import java.util.HashSet;
import java.util.List;
import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;
-import java.util.function.Consumer;
-import java.util.function.Predicate;
-import java.util.stream.StreamSupport;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import sun.nio.ch.DirectBuffer;
import org.apache.cassandra.concurrent.ScheduledExecutors;
import org.apache.cassandra.io.FSError;
@@ -46,7 +49,9 @@
import org.apache.cassandra.io.FSReadError;
import org.apache.cassandra.io.FSWriteError;
import org.apache.cassandra.io.sstable.CorruptSSTableException;
+import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.JVMStabilityInspector;
+import org.apache.cassandra.utils.SyncUtil;
import static com.google.common.base.Throwables.propagate;
import static org.apache.cassandra.utils.Throwables.maybeFail;
@@ -63,24 +68,39 @@
public static final long ONE_TB = 1024 * ONE_GB;
private static final DecimalFormat df = new DecimalFormat("#.##");
- public static final boolean isCleanerAvailable;
private static final AtomicReference<Optional<FSErrorHandler>> fsErrorHandler = new AtomicReference<>(Optional.empty());
+ private static Class clsDirectBuffer;
+ private static MethodHandle mhDirectBufferCleaner;
+ private static MethodHandle mhCleanerClean;
+
static
{
- boolean canClean = false;
try
{
+ clsDirectBuffer = Class.forName("sun.nio.ch.DirectBuffer");
+ Method mDirectBufferCleaner = clsDirectBuffer.getMethod("cleaner");
+ mhDirectBufferCleaner = MethodHandles.lookup().unreflect(mDirectBufferCleaner);
+ Method mCleanerClean = mDirectBufferCleaner.getReturnType().getMethod("clean");
+ mhCleanerClean = MethodHandles.lookup().unreflect(mCleanerClean);
+
ByteBuffer buf = ByteBuffer.allocateDirect(1);
- ((DirectBuffer) buf).cleaner().clean();
- canClean = true;
+ clean(buf);
+ }
+ catch (IllegalAccessException e)
+ {
+ logger.error("FATAL: Cassandra is unable to access required classes. This usually means it has been " +
+ "run without the aid of the standard startup scripts or the scripts have been edited. If this was " +
+ "intentional, and you are attempting to use Java 11+ you may need to add the --add-exports and " +
+ "--add-opens jvm options from either jvm11-server.options or jvm11-client.options");
+ throw new RuntimeException(e); // causes ExceptionInInitializerError, will prevent startup
}
catch (Throwable t)
{
+ logger.error("FATAL: Cannot initialize optimized memory deallocator.");
JVMStabilityInspector.inspectThrowable(t);
- logger.info("Cannot initialize un-mmaper. (Are you using a non-Oracle JVM?) Compacted data files will not be removed promptly. Consider using an Oracle JVM or using standard disk access mode");
+ throw new RuntimeException(t); // causes ExceptionInInitializerError, will prevent startup
}
- isCleanerAvailable = canClean;
}
public static void createHardLink(String from, String to)
@@ -105,11 +125,44 @@
}
}
+ private static final File tempDir = new File(System.getProperty("java.io.tmpdir"));
+ private static final AtomicLong tempFileNum = new AtomicLong();
+
+ public static File getTempDir()
+ {
+ return tempDir;
+ }
+
+ /**
+ * Pretty much like {@link File#createTempFile(String, String, File)}, but with
+ * the guarantee that the "random" part of the generated file name between
+ * {@code prefix} and {@code suffix} is a positive, increasing {@code long} value.
+ */
public static File createTempFile(String prefix, String suffix, File directory)
{
try
{
- return File.createTempFile(prefix, suffix, directory);
+ // Do not use java.io.File.createTempFile(), because some tests rely on the
+ // behavior that the "random" part in the temp file name is a positive 'long'.
+ // However, at least since Java 9 the code to generate the "random" part
+ // uses an _unsigned_ random long generated like this:
+ // Long.toUnsignedString(new java.util.Random.nextLong())
+
+ while (true)
+ {
+ // The contract of File.createTempFile() says, that it must not return
+ // the same file name again. We do that here in a very simple way,
+ // that probably doesn't cover all edge cases. Just rely on system
+ // wall clock and return strictly increasing values from that.
+ long num = tempFileNum.getAndIncrement();
+
+ // We have a positive long here, which is safe to use for example
+ // for CommitLogTest.
+ String fileName = prefix + Long.toString(num) + suffix;
+ File candidate = new File(directory, fileName);
+ if (candidate.createNewFile())
+ return candidate;
+ }
}
catch (IOException e)
{
@@ -119,22 +172,29 @@
public static File createTempFile(String prefix, String suffix)
{
- return createTempFile(prefix, suffix, new File(System.getProperty("java.io.tmpdir")));
+ return createTempFile(prefix, suffix, tempDir);
}
- public static Throwable deleteWithConfirm(String filePath, boolean expect, Throwable accumulate)
+ public static File createDeletableTempFile(String prefix, String suffix)
{
- return deleteWithConfirm(new File(filePath), expect, accumulate);
+ File f = createTempFile(prefix, suffix, getTempDir());
+ f.deleteOnExit();
+ return f;
}
- public static Throwable deleteWithConfirm(File file, boolean expect, Throwable accumulate)
+ public static Throwable deleteWithConfirm(String filePath, Throwable accumulate)
{
- boolean exists = file.exists();
- assert exists || !expect : "attempted to delete non-existing file " + file.getName();
+ return deleteWithConfirm(new File(filePath), accumulate);
+ }
+
+ public static Throwable deleteWithConfirm(File file, Throwable accumulate)
+ {
try
{
- if (exists)
- Files.delete(file.toPath());
+ if (!StorageService.instance.isDaemonSetupCompleted())
+ logger.info("Deleting file during startup: {}", file);
+
+ Files.delete(file.toPath());
}
catch (Throwable t)
{
@@ -157,7 +217,7 @@
public static void deleteWithConfirm(File file)
{
- maybeFail(deleteWithConfirm(file, true, null));
+ maybeFail(deleteWithConfirm(file, null));
}
public static void renameWithOutConfirm(String from, String to)
@@ -342,13 +402,29 @@
public static void clean(ByteBuffer buffer)
{
- if (buffer == null)
+ if (buffer == null || !buffer.isDirect())
return;
- if (isCleanerAvailable && buffer.isDirect())
+
+ // TODO Once we can get rid of Java 8, it's simpler to call sun.misc.Unsafe.invokeCleaner(ByteBuffer),
+ // but need to take care of the attachment handling (i.e. whether 'buf' is a duplicate or slice) - that
+ // is different in sun.misc.Unsafe.invokeCleaner and this implementation.
+
+ try
{
- DirectBuffer db = (DirectBuffer) buffer;
- if (db.cleaner() != null)
- db.cleaner().clean();
+ Object cleaner = mhDirectBufferCleaner.bindTo(buffer).invoke();
+ if (cleaner != null)
+ {
+ // ((DirectBuffer) buf).cleaner().clean();
+ mhCleanerClean.bindTo(cleaner).invoke();
+ }
+ }
+ catch (RuntimeException e)
+ {
+ throw e;
+ }
+ catch (Throwable e)
+ {
+ throw new RuntimeException(e);
}
}
@@ -368,21 +444,20 @@
public static boolean delete(String file)
{
+ if (!StorageService.instance.isDaemonSetupCompleted())
+ logger.info("Deleting file during startup: {}", file);
+
File f = new File(file);
return f.delete();
}
public static void delete(File... files)
{
- if (files == null)
- {
- // CASSANDRA-13389: some callers use Files.listFiles() which, on error, silently returns null
- logger.debug("Received null list of files to delete");
- return;
- }
-
for ( File file : files )
{
+ if (!StorageService.instance.isDaemonSetupCompleted())
+ logger.info("Deleting file during startup: {}", file);
+
file.delete();
}
}
@@ -399,19 +474,42 @@
ScheduledExecutors.nonPeriodicTasks.execute(runnable);
}
- public static void visitDirectory(Path dir, Predicate<? super File> filter, Consumer<? super File> consumer)
+ public static long parseFileSize(String value)
{
- try (DirectoryStream<Path> stream = Files.newDirectoryStream(dir))
+ long result;
+ if (!value.matches("\\d+(\\.\\d+)? (GiB|KiB|MiB|TiB|bytes)"))
{
- StreamSupport.stream(stream.spliterator(), false)
- .map(Path::toFile)
- // stream directories are weakly consistent so we always check if the file still exists
- .filter(f -> f.exists() && (filter == null || filter.test(f)))
- .forEach(consumer);
+ throw new IllegalArgumentException(
+ String.format("value %s is not a valid human-readable file size", value));
}
- catch (IOException|DirectoryIteratorException ex)
+ if (value.endsWith(" TiB"))
{
- logger.error("Failed to list files in {} with exception: {}", dir, ex.getMessage(), ex);
+ result = Math.round(Double.valueOf(value.replace(" TiB", "")) * ONE_TB);
+ return result;
+ }
+ else if (value.endsWith(" GiB"))
+ {
+ result = Math.round(Double.valueOf(value.replace(" GiB", "")) * ONE_GB);
+ return result;
+ }
+ else if (value.endsWith(" KiB"))
+ {
+ result = Math.round(Double.valueOf(value.replace(" KiB", "")) * ONE_KB);
+ return result;
+ }
+ else if (value.endsWith(" MiB"))
+ {
+ result = Math.round(Double.valueOf(value.replace(" MiB", "")) * ONE_MB);
+ return result;
+ }
+ else if (value.endsWith(" bytes"))
+ {
+ result = Math.round(Double.valueOf(value.replace(" bytes", "")));
+ return result;
+ }
+ else
+ {
+ throw new IllegalStateException(String.format("FileUtils.parseFileSize() reached an illegal state parsing %s", value));
}
}
@@ -530,7 +628,7 @@
}
catch (IOException e)
{
- logger.error("Error while getting {} folder size. {}", folder, e);
+ logger.error("Error while getting {} folder size. {}", folder, e.getMessage());
}
return sizeArr[0];
}
@@ -591,14 +689,46 @@
write(file, Arrays.asList(lines), StandardOpenOption.TRUNCATE_EXISTING);
}
+ /**
+ * Write lines to a file adding a newline to the end of each supplied line using the provided open options.
+ *
+ * If open option sync or dsync is provided this will not open the file with sync or dsync since it might end up syncing
+ * many times for a lot of lines. Instead it will write all the lines and sync once at the end. Since the file is
+ * never returned there is not much difference from the perspective of the caller.
+ * @param file
+ * @param lines
+ * @param options
+ */
public static void write(File file, List<String> lines, StandardOpenOption ... options)
{
- try
+ Set<StandardOpenOption> optionsSet = new HashSet<>(Arrays.asList(options));
+ //Emulate the old FileSystemProvider.newOutputStream behavior for open options.
+ if (optionsSet.isEmpty())
{
- Files.write(file.toPath(),
- lines,
- CHARSET,
- options);
+ optionsSet.add(StandardOpenOption.CREATE);
+ optionsSet.add(StandardOpenOption.TRUNCATE_EXISTING);
+ }
+ boolean sync = optionsSet.remove(StandardOpenOption.SYNC);
+ boolean dsync = optionsSet.remove(StandardOpenOption.DSYNC);
+ optionsSet.add(StandardOpenOption.WRITE);
+
+ Path filePath = file.toPath();
+ try (FileChannel fc = filePath.getFileSystem().provider().newFileChannel(filePath, optionsSet);
+ BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(Channels.newOutputStream(fc), CHARSET.newEncoder())))
+ {
+ for (CharSequence line: lines) {
+ writer.append(line);
+ writer.newLine();
+ }
+
+ if (sync)
+ {
+ SyncUtil.force(fc, true);
+ }
+ else if (dsync)
+ {
+ SyncUtil.force(fc, false);
+ }
}
catch (IOException ex)
{
diff --git a/src/java/org/apache/cassandra/io/util/LimitingRebufferer.java b/src/java/org/apache/cassandra/io/util/LimitingRebufferer.java
index a1e9715..b5c7f34 100644
--- a/src/java/org/apache/cassandra/io/util/LimitingRebufferer.java
+++ b/src/java/org/apache/cassandra/io/util/LimitingRebufferer.java
@@ -101,7 +101,7 @@
@Override
public String toString()
{
- return "LimitingRebufferer[" + limiter.toString() + "]:" + wrapped.toString();
+ return "LimitingRebufferer[" + limiter + "]:" + wrapped;
}
// BufferHolder methods
diff --git a/src/java/org/apache/cassandra/io/util/Memory.java b/src/java/org/apache/cassandra/io/util/Memory.java
index 431f73d..eaa6e91 100644
--- a/src/java/org/apache/cassandra/io/util/Memory.java
+++ b/src/java/org/apache/cassandra/io/util/Memory.java
@@ -28,7 +28,6 @@
import org.apache.cassandra.utils.concurrent.Ref;
import org.apache.cassandra.utils.memory.MemoryUtil;
import sun.misc.Unsafe;
-import sun.nio.ch.DirectBuffer;
/**
* An off-heap region of memory that must be manually free'd when no longer needed.
@@ -213,9 +212,9 @@
{
setBytes(memoryOffset, buffer.array(), buffer.arrayOffset() + buffer.position(), buffer.remaining());
}
- else if (buffer instanceof DirectBuffer)
+ else if (buffer.isDirect())
{
- unsafe.copyMemory(((DirectBuffer) buffer).address() + buffer.position(), peer + memoryOffset, buffer.remaining());
+ unsafe.copyMemory(MemoryUtil.getAddress(buffer) + buffer.position(), peer + memoryOffset, buffer.remaining());
}
else
throw new IllegalStateException();
@@ -418,7 +417,7 @@
public void setByteBuffer(ByteBuffer buffer, long offset, int length)
{
checkBounds(offset, offset + length);
- MemoryUtil.setByteBuffer(buffer, peer + offset, length);
+ MemoryUtil.setDirectByteBuffer(buffer, peer + offset, length);
}
public String toString()
diff --git a/src/java/org/apache/cassandra/io/util/MemoryOutputStream.java b/src/java/org/apache/cassandra/io/util/MemoryOutputStream.java
index e6c869d..c984738 100644
--- a/src/java/org/apache/cassandra/io/util/MemoryOutputStream.java
+++ b/src/java/org/apache/cassandra/io/util/MemoryOutputStream.java
@@ -27,7 +27,7 @@
{
private final Memory mem;
- private int position = 0;
+ private long position = 0;
public MemoryOutputStream(Memory mem)
{
@@ -45,9 +45,4 @@
mem.setBytes(position, b, off, len);
position += len;
}
-
- public int position()
- {
- return position;
- }
}
diff --git a/src/java/org/apache/cassandra/io/util/MmappedRegions.java b/src/java/org/apache/cassandra/io/util/MmappedRegions.java
index 13b476a..0b7dd39 100644
--- a/src/java/org/apache/cassandra/io/util/MmappedRegions.java
+++ b/src/java/org/apache/cassandra/io/util/MmappedRegions.java
@@ -325,14 +325,6 @@
{
accumulate = channel.close(accumulate);
- /*
- * Try forcing the unmapping of segments using undocumented unsafe sun APIs.
- * If this fails (non Sun JVM), we'll have to wait for the GC to finalize the mapping.
- * If this works and a thread tries to access any segment, hell will unleash on earth.
- */
- if (!FileUtils.isCleanerAvailable)
- return accumulate;
-
return perform(accumulate, channel.filePath(), Throwables.FileOpType.READ,
of(buffers)
.map((buffer) ->
diff --git a/src/java/org/apache/cassandra/io/util/RandomAccessReader.java b/src/java/org/apache/cassandra/io/util/RandomAccessReader.java
index 5157eac..a0ea520 100644
--- a/src/java/org/apache/cassandra/io/util/RandomAccessReader.java
+++ b/src/java/org/apache/cassandra/io/util/RandomAccessReader.java
@@ -169,7 +169,7 @@
@Override
public String toString()
{
- return getClass().getSimpleName() + ':' + rebufferer.toString();
+ return getClass().getSimpleName() + ':' + rebufferer;
}
/**
diff --git a/src/java/org/apache/cassandra/io/util/RebufferingInputStream.java b/src/java/org/apache/cassandra/io/util/RebufferingInputStream.java
index 094115a..18cabd3 100644
--- a/src/java/org/apache/cassandra/io/util/RebufferingInputStream.java
+++ b/src/java/org/apache/cassandra/io/util/RebufferingInputStream.java
@@ -25,11 +25,13 @@
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
+import com.google.common.base.Preconditions;
+
import net.nicoulaj.compilecommand.annotations.DontInline;
import org.apache.cassandra.utils.FastByteOperations;
import org.apache.cassandra.utils.vint.VIntCoding;
-import com.google.common.base.Preconditions;
+import static java.lang.Math.min;
/**
* Rough equivalent of BufferedInputStream and DataInputStream wrapping a ByteBuffer that can be refilled
@@ -91,7 +93,7 @@
if (remaining == 0)
return copied == 0 ? -1 : copied;
}
- int toCopy = Math.min(len - copied, remaining);
+ int toCopy = min(len - copied, remaining);
FastByteOperations.copy(buffer, position, b, off + copied, toCopy);
buffer.position(position + toCopy);
copied += toCopy;
@@ -100,6 +102,38 @@
return copied;
}
+ /**
+ * Equivalent to {@link #read(byte[], int, int)}, where offset is {@code dst.position()} and length is {@code dst.remaining()}
+ */
+ public void readFully(ByteBuffer dst) throws IOException
+ {
+ int offset = dst.position();
+ int len = dst.limit() - offset;
+
+ int copied = 0;
+ while (copied < len)
+ {
+ int position = buffer.position();
+ int remaining = buffer.limit() - position;
+
+ if (remaining == 0)
+ {
+ reBuffer();
+
+ position = buffer.position();
+ remaining = buffer.limit() - position;
+
+ if (remaining == 0)
+ throw new EOFException("EOF after " + copied + " bytes out of " + len);
+ }
+
+ int toCopy = min(len - copied, remaining);
+ FastByteOperations.copy(buffer, position, dst, offset + copied, toCopy);
+ buffer.position(position + toCopy);
+ copied += toCopy;
+ }
+ }
+
@DontInline
protected long readPrimitiveSlowly(int bytes) throws IOException
{
@@ -112,7 +146,7 @@
@Override
public int skipBytes(int n) throws IOException
{
- if (n < 0)
+ if (n <= 0)
return 0;
int requested = n;
int position = buffer.position(), limit = buffer.limit(), remaining;
diff --git a/src/java/org/apache/cassandra/io/util/SequentialWriter.java b/src/java/org/apache/cassandra/io/util/SequentialWriter.java
index e71f2fa..9ad944b 100644
--- a/src/java/org/apache/cassandra/io/util/SequentialWriter.java
+++ b/src/java/org/apache/cassandra/io/util/SequentialWriter.java
@@ -19,6 +19,7 @@
import java.io.File;
import java.io.IOException;
+import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.file.StandardOpenOption;
@@ -43,6 +44,15 @@
protected final FileChannel fchannel;
+ //Allow derived classes to specify writing to the channel
+ //directly shouldn't happen because they intercept via doFlush for things
+ //like compression or checksumming
+ //Another hack for this value is that it also indicates that flushing early
+ //should not occur, flushes aligned with buffer size are desired
+ //Unless... it's the last flush. Compression and checksum formats
+ //expect block (same as buffer size) alignment for everything except the last block
+ private final boolean strictFlushing;
+
// whether to do trickling fsync() to avoid sudden bursts of dirty buffer flushing by kernel causing read
// latency spikes
private final SequentialWriterOption option;
@@ -138,11 +148,22 @@
*/
public SequentialWriter(File file, SequentialWriterOption option)
{
- super(openChannel(file), option.allocateBuffer());
- strictFlushing = true;
- fchannel = (FileChannel)channel;
+ this(file, option, true);
+ }
- filePath = file.getAbsolutePath();
+ /**
+ * Create SequentialWriter for given file with specific writer option.
+ * @param file
+ * @param option
+ * @param strictFlushing
+ */
+ public SequentialWriter(File file, SequentialWriterOption option, boolean strictFlushing)
+ {
+ super(openChannel(file), option.allocateBuffer());
+ this.strictFlushing = strictFlushing;
+ this.fchannel = (FileChannel)channel;
+
+ this.filePath = file.getAbsolutePath();
this.option = option;
}
@@ -377,6 +398,15 @@
txnProxy.close();
}
+ public int writeDirectlyToChannel(ByteBuffer buf) throws IOException
+ {
+ if (strictFlushing)
+ throw new UnsupportedOperationException();
+ // Don't allow writes to the underlying channel while data is buffered
+ flush();
+ return channel.write(buf);
+ }
+
public final void finish()
{
txnProxy.finish();
diff --git a/src/java/org/apache/cassandra/io/util/UnbufferedDataOutputStreamPlus.java b/src/java/org/apache/cassandra/io/util/UnbufferedDataOutputStreamPlus.java
index 54b4cb1..3d83212 100644
--- a/src/java/org/apache/cassandra/io/util/UnbufferedDataOutputStreamPlus.java
+++ b/src/java/org/apache/cassandra/io/util/UnbufferedDataOutputStreamPlus.java
@@ -371,15 +371,4 @@
}
}
- public void write(Memory memory, long offset, long length) throws IOException
- {
- for (ByteBuffer buffer : memory.asByteBuffers(offset, length))
- write(buffer);
- }
-
- @Override
- public <R> R applyToChannel(Function<WritableByteChannel, R> f) throws IOException
- {
- return f.apply(channel);
- }
}
diff --git a/src/java/org/apache/cassandra/locator/AbstractEndpointSnitch.java b/src/java/org/apache/cassandra/locator/AbstractEndpointSnitch.java
index 546d15e..2e7408b 100644
--- a/src/java/org/apache/cassandra/locator/AbstractEndpointSnitch.java
+++ b/src/java/org/apache/cassandra/locator/AbstractEndpointSnitch.java
@@ -17,14 +17,12 @@
*/
package org.apache.cassandra.locator;
-import java.net.InetAddress;
-import java.util.*;
-
+import com.google.common.collect.Iterables;
import org.apache.cassandra.config.DatabaseDescriptor;
public abstract class AbstractEndpointSnitch implements IEndpointSnitch
{
- public abstract int compareEndpoints(InetAddress target, InetAddress a1, InetAddress a2);
+ public abstract int compareEndpoints(InetAddressAndPort target, Replica r1, Replica r2);
/**
* Sorts the <tt>Collection</tt> of node addresses by proximity to the given address
@@ -32,27 +30,9 @@
* @param unsortedAddress the nodes to sort
* @return a new sorted <tt>List</tt>
*/
- public List<InetAddress> getSortedListByProximity(InetAddress address, Collection<InetAddress> unsortedAddress)
+ public <C extends ReplicaCollection<? extends C>> C sortedByProximity(final InetAddressAndPort address, C unsortedAddress)
{
- List<InetAddress> preferred = new ArrayList<InetAddress>(unsortedAddress);
- sortByProximity(address, preferred);
- return preferred;
- }
-
- /**
- * Sorts the <tt>List</tt> of node addresses, in-place, by proximity to the given address
- * @param address the address to sort the proximity by
- * @param addresses the nodes to sort
- */
- public void sortByProximity(final InetAddress address, List<InetAddress> addresses)
- {
- Collections.sort(addresses, new Comparator<InetAddress>()
- {
- public int compare(InetAddress a1, InetAddress a2)
- {
- return compareEndpoints(address, a1, a2);
- }
- });
+ return unsortedAddress.sorted((r1, r2) -> compareEndpoints(address, r1, r2));
}
public void gossiperStarting()
@@ -60,7 +40,7 @@
// noop by default
}
- public boolean isWorthMergingForRangeQuery(List<InetAddress> merged, List<InetAddress> l1, List<InetAddress> l2)
+ public boolean isWorthMergingForRangeQuery(ReplicaCollection<?> merged, ReplicaCollection<?> l1, ReplicaCollection<?> l2)
{
// Querying remote DC is likely to be an order of magnitude slower than
// querying locally, so 2 queries to local nodes is likely to still be
@@ -71,14 +51,9 @@
: true;
}
- private boolean hasRemoteNode(List<InetAddress> l)
+ private boolean hasRemoteNode(ReplicaCollection<?> l)
{
String localDc = DatabaseDescriptor.getLocalDataCenter();
- for (InetAddress ep : l)
- {
- if (!localDc.equals(getDatacenter(ep)))
- return true;
- }
- return false;
+ return Iterables.any(l, replica -> !localDc.equals(getDatacenter(replica)));
}
}
diff --git a/src/java/org/apache/cassandra/locator/AbstractNetworkTopologySnitch.java b/src/java/org/apache/cassandra/locator/AbstractNetworkTopologySnitch.java
index b5606d6..08c41f0 100644
--- a/src/java/org/apache/cassandra/locator/AbstractNetworkTopologySnitch.java
+++ b/src/java/org/apache/cassandra/locator/AbstractNetworkTopologySnitch.java
@@ -17,8 +17,6 @@
*/
package org.apache.cassandra.locator;
-import java.net.InetAddress;
-
/**
* An endpoint snitch tells Cassandra information about network topology that it can use to route
* requests more efficiently.
@@ -30,17 +28,20 @@
* @param endpoint a specified endpoint
* @return string of rack
*/
- abstract public String getRack(InetAddress endpoint);
+ abstract public String getRack(InetAddressAndPort endpoint);
/**
* Return the data center for which an endpoint resides in
* @param endpoint a specified endpoint
* @return string of data center
*/
- abstract public String getDatacenter(InetAddress endpoint);
+ abstract public String getDatacenter(InetAddressAndPort endpoint);
- public int compareEndpoints(InetAddress address, InetAddress a1, InetAddress a2)
+ @Override
+ public int compareEndpoints(InetAddressAndPort address, Replica r1, Replica r2)
{
+ InetAddressAndPort a1 = r1.endpoint();
+ InetAddressAndPort a2 = r2.endpoint();
if (address.equals(a1) && !address.equals(a2))
return -1;
if (address.equals(a2) && !address.equals(a1))
diff --git a/src/java/org/apache/cassandra/locator/AbstractReplicaCollection.java b/src/java/org/apache/cassandra/locator/AbstractReplicaCollection.java
new file mode 100644
index 0000000..2ec555c
--- /dev/null
+++ b/src/java/org/apache/cassandra/locator/AbstractReplicaCollection.java
@@ -0,0 +1,548 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.locator;
+
+import com.carrotsearch.hppc.ObjectIntHashMap;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Iterables;
+
+import java.util.AbstractMap;
+import java.util.AbstractSet;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.Iterator;
+import java.util.Objects;
+import java.util.Set;
+import java.util.function.BiConsumer;
+import java.util.function.BinaryOperator;
+import java.util.function.Function;
+import java.util.function.Predicate;
+import java.util.function.Supplier;
+import java.util.stream.Collector;
+import java.util.stream.Stream;
+
+/**
+ * A collection like class for Replica objects. Since the Replica class contains inetaddress, range, and
+ * transient replication status, basic contains and remove methods can be ambiguous. Replicas forces you
+ * to be explicit about what you're checking the container for, or removing from it.
+ */
+public abstract class AbstractReplicaCollection<C extends AbstractReplicaCollection<C>> implements ReplicaCollection<C>
+{
+ protected static final ReplicaList EMPTY_LIST = new ReplicaList(); // since immutable, can safely return this to avoid megamorphic callsites
+
+ public static <C extends ReplicaCollection<C>, B extends Builder<C>> Collector<Replica, B, C> collector(Set<Collector.Characteristics> characteristics, Supplier<B> supplier)
+ {
+ return new Collector<Replica, B, C>()
+ {
+ private final BiConsumer<B, Replica> accumulator = Builder::add;
+ private final BinaryOperator<B> combiner = (a, b) -> { a.addAll(b); return a; };
+ private final Function<B, C> finisher = Builder::build;
+ public Supplier<B> supplier() { return supplier; }
+ public BiConsumer<B, Replica> accumulator() { return accumulator; }
+ public BinaryOperator<B> combiner() { return combiner; }
+ public Function<B, C> finisher() { return finisher; }
+ public Set<Characteristics> characteristics() { return characteristics; }
+ };
+ }
+
+ /**
+ * A simple list with no comodification checks and immutability by default (only append permitted, and only one initial copy)
+ * this permits us to reduce the amount of garbage generated, by not wrapping iterators or unnecessarily copying
+ * and reduces the amount of indirection necessary, as well as ensuring monomorphic callsites
+ */
+ protected static class ReplicaList implements Iterable<Replica>
+ {
+ private static final Replica[] EMPTY = new Replica[0];
+ Replica[] contents;
+ int begin, size;
+
+ public ReplicaList() { this(0); }
+ public ReplicaList(int capacity) { contents = capacity == 0 ? EMPTY : new Replica[capacity]; }
+ public ReplicaList(Replica[] contents, int begin, int size) { this.contents = contents; this.begin = begin; this.size = size; }
+
+ public boolean isSubList(ReplicaList subList)
+ {
+ return subList.contents == contents;
+ }
+
+ public Replica get(int index)
+ {
+ if (index > size)
+ throw new IndexOutOfBoundsException();
+ return contents[begin + index];
+ }
+
+ public void add(Replica replica)
+ {
+ // can only add to full array - if we have sliced it, we must be a snapshot
+ if (begin != 0)
+ throw new IllegalStateException();
+
+ if (size == contents.length)
+ {
+ int newSize;
+ if (size < 3) newSize = 3;
+ else if (size < 9) newSize = 9;
+ else newSize = size * 2;
+ contents = Arrays.copyOf(contents, newSize);
+ }
+ contents[size++] = replica;
+ }
+
+ public int size()
+ {
+ return size;
+ }
+
+ public boolean isEmpty()
+ {
+ return size == 0;
+ }
+
+ public ReplicaList subList(int begin, int end)
+ {
+ if (end > size || begin > end) throw new IndexOutOfBoundsException();
+ return new ReplicaList(contents, this.begin + begin, end - begin);
+ }
+
+ public ReplicaList sorted(Comparator<Replica> comparator)
+ {
+ Replica[] copy = Arrays.copyOfRange(contents, begin, begin + size);
+ Arrays.sort(copy, comparator);
+ return new ReplicaList(copy, 0, copy.length);
+ }
+
+ public Stream<Replica> stream()
+ {
+ return Arrays.stream(contents, begin, begin + size);
+ }
+
+ // we implement our own iterator, because it is trivial to do so, and in monomorphic call sites
+ // will compile down to almost optimal indexed for loop
+ @Override
+ public Iterator<Replica> iterator()
+ {
+ return new Iterator<Replica>()
+ {
+ final int end = begin + size;
+ int i = begin;
+ @Override
+ public boolean hasNext()
+ {
+ return i < end;
+ }
+
+ @Override
+ public Replica next()
+ {
+ if (!hasNext()) throw new IllegalStateException();
+ return contents[i++];
+ }
+ };
+ }
+
+ // we implement our own iterator, because it is trivial to do so, and in monomorphic call sites
+ // will compile down to almost optimal indexed for loop
+ public <K> Iterator<K> transformIterator(Function<Replica, K> function)
+ {
+ return new Iterator<K>()
+ {
+ final int end = begin + size;
+ int i = begin;
+ @Override
+ public boolean hasNext()
+ {
+ return i < end;
+ }
+
+ @Override
+ public K next()
+ {
+ return function.apply(contents[i++]);
+ }
+ };
+ }
+
+ // we implement our own iterator, because it is trivial to do so, and in monomorphic call sites
+ // will compile down to almost optimal indexed for loop
+ // in this case, especially, it is impactful versus Iterables.limit(Iterables.filter())
+ private Iterator<Replica> filterIterator(Predicate<Replica> predicate, int limit)
+ {
+ return new Iterator<Replica>()
+ {
+ final int end = begin + size;
+ int next = begin;
+ int count = 0;
+ { updateNext(); }
+ void updateNext()
+ {
+ if (count == limit) next = end;
+ while (next < end && !predicate.test(contents[next]))
+ ++next;
+ ++count;
+ }
+ @Override
+ public boolean hasNext()
+ {
+ return next < end;
+ }
+
+ @Override
+ public Replica next()
+ {
+ if (!hasNext()) throw new IllegalStateException();
+ Replica result = contents[next++];
+ updateNext();
+ return result;
+ }
+ };
+ }
+
+ @VisibleForTesting
+ public boolean equals(Object to)
+ {
+ if (to == null || to.getClass() != ReplicaList.class)
+ return false;
+ ReplicaList that = (ReplicaList) to;
+ if (this.size != that.size) return false;
+ return Iterables.elementsEqual(this, that);
+ }
+ }
+
+ /**
+ * A simple map that ensures the underlying list's iteration order is maintained, and can be shared with
+ * subLists (either produced via subList, or via filter that naturally produced a subList).
+ * This permits us to reduce the amount of garbage generated, by not unnecessarily copying,
+ * reduces the amount of indirection necessary, as well as ensuring monomorphic callsites.
+ * The underlying map is also more efficient, particularly for such small collections as we typically produce.
+ */
+ protected static class ReplicaMap<K> extends AbstractMap<K, Replica>
+ {
+ private final Function<Replica, K> toKey;
+ private final ReplicaList list;
+ // we maintain a map of key -> index in our list; this lets us share with subLists (or between Builder and snapshots)
+ // since we only need to corroborate that the list index we find is within the bounds of our list
+ // (if not, it's a shared map, and the key only occurs in one of our ancestors)
+ private final ObjectIntHashMap<K> map;
+ private Set<K> keySet;
+ private Set<Entry<K, Replica>> entrySet;
+
+ abstract class AbstractImmutableSet<T> extends AbstractSet<T>
+ {
+ @Override
+ public boolean removeAll(Collection<?> c) { throw new UnsupportedOperationException(); }
+ @Override
+ public boolean remove(Object o) { throw new UnsupportedOperationException(); }
+ @Override
+ public int size() { return list.size(); }
+ }
+
+ class KeySet extends AbstractImmutableSet<K>
+ {
+ @Override
+ public boolean contains(Object o) { return containsKey(o); }
+ @Override
+ public Iterator<K> iterator() { return list.transformIterator(toKey); }
+ }
+
+ class EntrySet extends AbstractImmutableSet<Entry<K, Replica>>
+ {
+ @Override
+ public boolean contains(Object o)
+ {
+ Preconditions.checkNotNull(o);
+ if (!(o instanceof Entry<?, ?>)) return false;
+ return Objects.equals(get(((Entry) o).getKey()), ((Entry) o).getValue());
+ }
+
+ @Override
+ public Iterator<Entry<K, Replica>> iterator()
+ {
+ return list.transformIterator(r -> new SimpleImmutableEntry<>(toKey.apply(r), r));
+ }
+ }
+
+ public ReplicaMap(ReplicaList list, Function<Replica, K> toKey)
+ {
+ // 8*0.65 => RF=5; 16*0.65 ==> RF=10
+ // use list capacity if empty, otherwise use actual list size
+ this.toKey = toKey;
+ this.map = new ObjectIntHashMap<>(list.size == 0 ? list.contents.length : list.size, 0.65f);
+ this.list = list;
+ for (int i = list.begin ; i < list.begin + list.size ; ++i)
+ {
+ boolean inserted = internalPutIfAbsent(list.contents[i], i);
+ assert inserted;
+ }
+ }
+
+ public ReplicaMap(ReplicaList list, Function<Replica, K> toKey, ObjectIntHashMap<K> map)
+ {
+ this.toKey = toKey;
+ this.list = list;
+ this.map = map;
+ }
+
+ // to be used only by subclasses of AbstractReplicaCollection
+ boolean internalPutIfAbsent(Replica replica, int index)
+ {
+ K key = toKey.apply(replica);
+ int otherIndex = map.put(key, index + 1);
+ if (otherIndex == 0)
+ return true;
+ map.put(key, otherIndex);
+ return false;
+ }
+
+ @Override
+ public boolean containsKey(Object key)
+ {
+ Preconditions.checkNotNull(key);
+ return get((K)key) != null;
+ }
+
+ public Replica get(Object key)
+ {
+ Preconditions.checkNotNull(key);
+ int index = map.get((K)key) - 1;
+ // since this map can be shared between sublists (or snapshots of mutables)
+ // we have to first corroborate that the index we've found is actually within our list's bounds
+ if (index < list.begin || index >= list.begin + list.size)
+ return null;
+ return list.contents[index];
+ }
+
+ @Override
+ public Replica remove(Object key)
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Set<K> keySet()
+ {
+ Set<K> ks = keySet;
+ if (ks == null)
+ keySet = ks = new KeySet();
+ return ks;
+ }
+
+ @Override
+ public Set<Entry<K, Replica>> entrySet()
+ {
+ Set<Entry<K, Replica>> es = entrySet;
+ if (es == null)
+ entrySet = es = new EntrySet();
+ return es;
+ }
+
+ public int size()
+ {
+ return list.size();
+ }
+
+ ReplicaMap<K> forSubList(ReplicaList subList)
+ {
+ assert subList.contents == list.contents;
+ return new ReplicaMap<>(subList, toKey, map);
+ }
+ }
+
+ protected final ReplicaList list;
+ AbstractReplicaCollection(ReplicaList list)
+ {
+ this.list = list;
+ }
+
+ /**
+ * construct a new Builder of our own type, so that we can concatenate
+ * TODO: this isn't terribly pretty, but we need sometimes to select / merge two Endpoints of unknown type;
+ */
+ public abstract Builder<C> newBuilder(int initialCapacity);
+
+ // return a new "sub-collection" with some sub-selection of the contents of this collection
+ abstract C snapshot(ReplicaList newList);
+ // return this object, if it is an immutable snapshot, otherwise returns a copy with these properties
+ public abstract C snapshot();
+
+ /** see {@link ReplicaCollection#subList(int, int)}*/
+ public final C subList(int start, int end)
+ {
+ if (start == 0 && end == size())
+ return snapshot();
+
+ ReplicaList subList;
+ if (start == end) subList = EMPTY_LIST;
+ else subList = list.subList(start, end);
+
+ return snapshot(subList);
+ }
+
+ /** see {@link ReplicaCollection#count(Predicate)}*/
+ public int count(Predicate<Replica> predicate)
+ {
+ int count = 0;
+ for (int i = 0 ; i < list.size() ; ++i)
+ if (predicate.test(list.get(i)))
+ ++count;
+ return count;
+ }
+
+ /** see {@link ReplicaCollection#filter(Predicate)}*/
+ public final C filter(Predicate<Replica> predicate)
+ {
+ return filter(predicate, Integer.MAX_VALUE);
+ }
+
+ /** see {@link ReplicaCollection#filter(Predicate, int)}*/
+ public final C filter(Predicate<Replica> predicate, int limit)
+ {
+ if (isEmpty())
+ return snapshot();
+
+ ReplicaList copy = null;
+ int beginRun = -1, endRun = -1;
+ int i = 0;
+ for (; i < list.size() ; ++i)
+ {
+ Replica replica = list.get(i);
+ if (predicate.test(replica))
+ {
+ if (copy != null)
+ copy.add(replica);
+ else if (beginRun < 0)
+ beginRun = i;
+ else if (endRun > 0)
+ {
+ copy = new ReplicaList(Math.min(limit, (list.size() - i) + (endRun - beginRun)));
+ for (int j = beginRun ; j < endRun ; ++j)
+ copy.add(list.get(j));
+ copy.add(list.get(i));
+ }
+ if (--limit == 0)
+ {
+ ++i;
+ break;
+ }
+ }
+ else if (beginRun >= 0 && endRun < 0)
+ endRun = i;
+ }
+
+ if (beginRun < 0)
+ beginRun = endRun = 0;
+ if (endRun < 0)
+ endRun = i;
+ if (copy == null)
+ return subList(beginRun, endRun);
+ return snapshot(copy);
+ }
+
+ /** see {@link ReplicaCollection#filterLazily(Predicate)}*/
+ public final Iterable<Replica> filterLazily(Predicate<Replica> predicate)
+ {
+ return filterLazily(predicate, Integer.MAX_VALUE);
+ }
+
+ /** see {@link ReplicaCollection#filterLazily(Predicate,int)}*/
+ public final Iterable<Replica> filterLazily(Predicate<Replica> predicate, int limit)
+ {
+ return () -> list.filterIterator(predicate, limit);
+ }
+
+ /** see {@link ReplicaCollection#sorted(Comparator)}*/
+ public final C sorted(Comparator<Replica> comparator)
+ {
+ return snapshot(list.sorted(comparator));
+ }
+
+ public final Replica get(int i)
+ {
+ return list.get(i);
+ }
+
+ public final int size()
+ {
+ return list.size();
+ }
+
+ public final boolean isEmpty()
+ {
+ return list.isEmpty();
+ }
+
+ public final Iterator<Replica> iterator()
+ {
+ return list.iterator();
+ }
+
+ public final Stream<Replica> stream() { return list.stream(); }
+
+ /**
+ * <p>
+ * It's not clear whether {@link AbstractReplicaCollection} should implement the order sensitive {@link Object#equals(Object) equals}
+ * of {@link java.util.List} or the order oblivious {@link Object#equals(Object) equals} of {@link java.util.Set}. We never rely on equality
+ * in the database so rather then leave in a potentially surprising implementation we have it throw {@link UnsupportedOperationException}.
+ * </p>
+ * <p>
+ * Don't implement this and pick one behavior over the other. If you want equality you can static import {@link com.google.common.collect.Iterables#elementsEqual(Iterable, Iterable)}
+ * and use that to get order sensitive equals.
+ * </p>
+ */
+ public final boolean equals(Object o)
+ {
+ throw new UnsupportedOperationException("AbstractReplicaCollection equals unsupported");
+ }
+
+ /**
+ * <p>
+ * It's not clear whether {@link AbstractReplicaCollection} should implement the order sensitive {@link Object#hashCode() hashCode}
+ * of {@link java.util.List} or the order oblivious {@link Object#hashCode() equals} of {@link java.util.Set}. We never rely on hashCode
+ * in the database so rather then leave in a potentially surprising implementation we have it throw {@link UnsupportedOperationException}.
+ * </p>
+ * <p>
+ * Don't implement this and pick one behavior over the other.
+ * </p>
+ */
+ public final int hashCode()
+ {
+ throw new UnsupportedOperationException("AbstractReplicaCollection hashCode unsupported");
+ }
+
+ @Override
+ public final String toString()
+ {
+ return Iterables.toString(list);
+ }
+
+ static <C extends AbstractReplicaCollection<C>> C concat(C replicas, C extraReplicas, Builder.Conflict ignoreConflicts)
+ {
+ if (extraReplicas.isEmpty())
+ return replicas;
+ if (replicas.isEmpty())
+ return extraReplicas;
+ Builder<C> builder = replicas.newBuilder(replicas.size() + extraReplicas.size());
+ builder.addAll(replicas, Builder.Conflict.NONE);
+ builder.addAll(extraReplicas, ignoreConflicts);
+ return builder.build();
+ }
+
+}
diff --git a/src/java/org/apache/cassandra/locator/AbstractReplicationStrategy.java b/src/java/org/apache/cassandra/locator/AbstractReplicationStrategy.java
index cf28bf7..874097d 100644
--- a/src/java/org/apache/cassandra/locator/AbstractReplicationStrategy.java
+++ b/src/java/org/apache/cassandra/locator/AbstractReplicationStrategy.java
@@ -19,15 +19,16 @@
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
-import java.net.InetAddress;
+import java.lang.reflect.Method;
import java.util.*;
import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.HashMultimap;
-import com.google.common.collect.Multimap;
+import com.google.common.base.Preconditions;
+import org.apache.cassandra.locator.ReplicaCollection.Builder.Conflict;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.ConsistencyLevel;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.db.WriteType;
@@ -40,6 +41,7 @@
import org.apache.cassandra.service.DatacenterWriteResponseHandler;
import org.apache.cassandra.service.WriteResponseHandler;
import org.apache.cassandra.utils.FBUtilities;
+
import org.cliffc.high_scale_lib.NonBlockingHashMap;
/**
@@ -72,9 +74,9 @@
// lazy-initialize keyspace itself since we don't create them until after the replication strategies
}
- private final Map<Token, ArrayList<InetAddress>> cachedEndpoints = new NonBlockingHashMap<Token, ArrayList<InetAddress>>();
+ private final Map<Token, EndpointsForRange> cachedReplicas = new NonBlockingHashMap<>();
- public ArrayList<InetAddress> getCachedEndpoints(Token t)
+ public EndpointsForRange getCachedReplicas(Token t)
{
long lastVersion = tokenMetadata.getRingVersion();
@@ -85,13 +87,13 @@
if (lastVersion > lastInvalidatedVersion)
{
logger.trace("clearing cached endpoints");
- cachedEndpoints.clear();
+ cachedReplicas.clear();
lastInvalidatedVersion = lastVersion;
}
}
}
- return cachedEndpoints.get(t);
+ return cachedReplicas.get(t);
}
/**
@@ -101,50 +103,105 @@
* @param searchPosition the position the natural endpoints are requested for
* @return a copy of the natural endpoints for the given token
*/
- public ArrayList<InetAddress> getNaturalEndpoints(RingPosition searchPosition)
+ public EndpointsForToken getNaturalReplicasForToken(RingPosition searchPosition)
+ {
+ return getNaturalReplicas(searchPosition).forToken(searchPosition.getToken());
+ }
+
+ public EndpointsForRange getNaturalReplicas(RingPosition searchPosition)
{
Token searchToken = searchPosition.getToken();
Token keyToken = TokenMetadata.firstToken(tokenMetadata.sortedTokens(), searchToken);
- ArrayList<InetAddress> endpoints = getCachedEndpoints(keyToken);
+ EndpointsForRange endpoints = getCachedReplicas(keyToken);
if (endpoints == null)
{
TokenMetadata tm = tokenMetadata.cachedOnlyTokenMap();
// if our cache got invalidated, it's possible there is a new token to account for too
keyToken = TokenMetadata.firstToken(tm.sortedTokens(), searchToken);
- endpoints = new ArrayList<InetAddress>(calculateNaturalEndpoints(searchToken, tm));
- cachedEndpoints.put(keyToken, endpoints);
+ endpoints = calculateNaturalReplicas(searchToken, tm);
+ cachedReplicas.put(keyToken, endpoints);
}
- return new ArrayList<InetAddress>(endpoints);
+ return endpoints;
+ }
+
+ public Replica getLocalReplicaFor(RingPosition searchPosition)
+ {
+ return getNaturalReplicas(searchPosition)
+ .byEndpoint()
+ .get(FBUtilities.getBroadcastAddressAndPort());
}
/**
- * calculate the natural endpoints for the given token
+ * Calculate the natural endpoints for the given token. Endpoints are returned in the order
+ * they occur in the ring following the searchToken, as defined by the replication strategy.
*
- * @see #getNaturalEndpoints(org.apache.cassandra.dht.RingPosition)
+ * Note that the order of the replicas is _implicitly relied upon_ by the definition of
+ * "primary" range in
+ * {@link org.apache.cassandra.service.StorageService#getPrimaryRangesForEndpoint(String, InetAddressAndPort)}
+ * which is in turn relied on by various components like repair and size estimate calculations.
*
- * @param searchToken the token the natural endpoints are requested for
+ * @see #getNaturalReplicasForToken(org.apache.cassandra.dht.RingPosition)
+ *
+ * @param tokenMetadata the token metadata used to find the searchToken, e.g. contains token to endpoint
+ * mapping information
+ * @param searchToken the token to find the natural endpoints for
* @return a copy of the natural endpoints for the given token
*/
- public abstract List<InetAddress> calculateNaturalEndpoints(Token searchToken, TokenMetadata tokenMetadata);
+ public abstract EndpointsForRange calculateNaturalReplicas(Token searchToken, TokenMetadata tokenMetadata);
- public <T> AbstractWriteResponseHandler<T> getWriteResponseHandler(Collection<InetAddress> naturalEndpoints,
- Collection<InetAddress> pendingEndpoints,
- ConsistencyLevel consistency_level,
+ public <T> AbstractWriteResponseHandler<T> getWriteResponseHandler(ReplicaPlan.ForTokenWrite replicaPlan,
Runnable callback,
WriteType writeType,
long queryStartNanoTime)
{
- if (consistency_level.isDatacenterLocal())
+ return getWriteResponseHandler(replicaPlan, callback, writeType, queryStartNanoTime, DatabaseDescriptor.getIdealConsistencyLevel());
+ }
+
+ public <T> AbstractWriteResponseHandler<T> getWriteResponseHandler(ReplicaPlan.ForTokenWrite replicaPlan,
+ Runnable callback,
+ WriteType writeType,
+ long queryStartNanoTime,
+ ConsistencyLevel idealConsistencyLevel)
+ {
+ AbstractWriteResponseHandler resultResponseHandler;
+ if (replicaPlan.consistencyLevel().isDatacenterLocal())
{
// block for in this context will be localnodes block.
- return new DatacenterWriteResponseHandler<T>(naturalEndpoints, pendingEndpoints, consistency_level, getKeyspace(), callback, writeType, queryStartNanoTime);
+ resultResponseHandler = new DatacenterWriteResponseHandler<T>(replicaPlan, callback, writeType, queryStartNanoTime);
}
- else if (consistency_level == ConsistencyLevel.EACH_QUORUM && (this instanceof NetworkTopologyStrategy))
+ else if (replicaPlan.consistencyLevel() == ConsistencyLevel.EACH_QUORUM && (this instanceof NetworkTopologyStrategy))
{
- return new DatacenterSyncWriteResponseHandler<T>(naturalEndpoints, pendingEndpoints, consistency_level, getKeyspace(), callback, writeType, queryStartNanoTime);
+ resultResponseHandler = new DatacenterSyncWriteResponseHandler<T>(replicaPlan, callback, writeType, queryStartNanoTime);
}
- return new WriteResponseHandler<T>(naturalEndpoints, pendingEndpoints, consistency_level, getKeyspace(), callback, writeType, queryStartNanoTime);
+ else
+ {
+ resultResponseHandler = new WriteResponseHandler<T>(replicaPlan, callback, writeType, queryStartNanoTime);
+ }
+
+ //Check if tracking the ideal consistency level is configured
+ if (idealConsistencyLevel != null)
+ {
+ //If ideal and requested are the same just use this handler to track the ideal consistency level
+ //This is also used so that the ideal consistency level handler when constructed knows it is the ideal
+ //one for tracking purposes
+ if (idealConsistencyLevel == replicaPlan.consistencyLevel())
+ {
+ resultResponseHandler.setIdealCLResponseHandler(resultResponseHandler);
+ }
+ else
+ {
+ //Construct a delegate response handler to use to track the ideal consistency level
+ AbstractWriteResponseHandler idealHandler = getWriteResponseHandler(replicaPlan.withConsistencyLevel(idealConsistencyLevel),
+ callback,
+ writeType,
+ queryStartNanoTime,
+ idealConsistencyLevel);
+ resultResponseHandler.setIdealCLResponseHandler(idealHandler);
+ }
+ }
+
+ return resultResponseHandler;
}
private Keyspace getKeyspace()
@@ -160,7 +217,12 @@
*
* @return the replication factor
*/
- public abstract int getReplicationFactor();
+ public abstract ReplicationFactor getReplicationFactor();
+
+ public boolean hasTransientReplicas()
+ {
+ return getReplicationFactor().hasTransientReplicas();
+ }
/*
* NOTE: this is pretty inefficient. also the inverse (getRangeAddresses) below.
@@ -168,53 +230,81 @@
* (fixing this would probably require merging tokenmetadata into replicationstrategy,
* so we could cache/invalidate cleanly.)
*/
- public Multimap<InetAddress, Range<Token>> getAddressRanges(TokenMetadata metadata)
+ public RangesByEndpoint getAddressReplicas(TokenMetadata metadata)
{
- Multimap<InetAddress, Range<Token>> map = HashMultimap.create();
+ RangesByEndpoint.Builder map = new RangesByEndpoint.Builder();
for (Token token : metadata.sortedTokens())
{
Range<Token> range = metadata.getPrimaryRangeFor(token);
- for (InetAddress ep : calculateNaturalEndpoints(token, metadata))
+ for (Replica replica : calculateNaturalReplicas(token, metadata))
{
- map.put(ep, range);
+ // LocalStrategy always returns (min, min] ranges for it's replicas, so we skip the check here
+ Preconditions.checkState(range.equals(replica.range()) || this instanceof LocalStrategy);
+ map.put(replica.endpoint(), replica);
}
}
- return map;
+ return map.build();
}
- public Multimap<Range<Token>, InetAddress> getRangeAddresses(TokenMetadata metadata)
+ public RangesAtEndpoint getAddressReplicas(TokenMetadata metadata, InetAddressAndPort endpoint)
{
- Multimap<Range<Token>, InetAddress> map = HashMultimap.create();
+ RangesAtEndpoint.Builder builder = RangesAtEndpoint.builder(endpoint);
+ for (Token token : metadata.sortedTokens())
+ {
+ Range<Token> range = metadata.getPrimaryRangeFor(token);
+ Replica replica = calculateNaturalReplicas(token, metadata)
+ .byEndpoint().get(endpoint);
+ if (replica != null)
+ {
+ // LocalStrategy always returns (min, min] ranges for it's replicas, so we skip the check here
+ Preconditions.checkState(range.equals(replica.range()) || this instanceof LocalStrategy);
+ builder.add(replica, Conflict.DUPLICATE);
+ }
+ }
+ return builder.build();
+ }
+
+
+ public EndpointsByRange getRangeAddresses(TokenMetadata metadata)
+ {
+ EndpointsByRange.Builder map = new EndpointsByRange.Builder();
for (Token token : metadata.sortedTokens())
{
Range<Token> range = metadata.getPrimaryRangeFor(token);
- for (InetAddress ep : calculateNaturalEndpoints(token, metadata))
+ for (Replica replica : calculateNaturalReplicas(token, metadata))
{
- map.put(range, ep);
+ // LocalStrategy always returns (min, min] ranges for it's replicas, so we skip the check here
+ Preconditions.checkState(range.equals(replica.range()) || this instanceof LocalStrategy);
+ map.put(range, replica);
}
}
- return map;
+ return map.build();
}
- public Multimap<InetAddress, Range<Token>> getAddressRanges()
+ public RangesByEndpoint getAddressReplicas()
{
- return getAddressRanges(tokenMetadata.cloneOnlyTokenMap());
+ return getAddressReplicas(tokenMetadata.cloneOnlyTokenMap());
}
- public Collection<Range<Token>> getPendingAddressRanges(TokenMetadata metadata, Token pendingToken, InetAddress pendingAddress)
+ public RangesAtEndpoint getAddressReplicas(InetAddressAndPort endpoint)
{
- return getPendingAddressRanges(metadata, Arrays.asList(pendingToken), pendingAddress);
+ return getAddressReplicas(tokenMetadata.cloneOnlyTokenMap(), endpoint);
}
- public Collection<Range<Token>> getPendingAddressRanges(TokenMetadata metadata, Collection<Token> pendingTokens, InetAddress pendingAddress)
+ public RangesAtEndpoint getPendingAddressRanges(TokenMetadata metadata, Token pendingToken, InetAddressAndPort pendingAddress)
+ {
+ return getPendingAddressRanges(metadata, Collections.singleton(pendingToken), pendingAddress);
+ }
+
+ public RangesAtEndpoint getPendingAddressRanges(TokenMetadata metadata, Collection<Token> pendingTokens, InetAddressAndPort pendingAddress)
{
TokenMetadata temp = metadata.cloneOnlyTokenMap();
temp.updateNormalTokens(pendingTokens, pendingAddress);
- return getAddressRanges(temp).get(pendingAddress);
+ return getAddressReplicas(temp, pendingAddress);
}
public abstract void validateOptions() throws ConfigurationException;
@@ -278,6 +368,39 @@
return strategy;
}
+ /**
+ * Before constructing the ARS we first give it a chance to prepare the options map in any way it
+ * would like to. For example datacenter auto-expansion or other templating to make the user interface
+ * more usable. Note that this may mutate the passed strategyOptions Map.
+ *
+ * We do this prior to the construction of the strategyClass itself because at that point the option
+ * map is already immutable and comes from {@link org.apache.cassandra.schema.ReplicationParams}
+ * (and should probably stay that way so we don't start having bugs related to ReplicationParams being mutable).
+ * Instead ARS classes get a static hook here via the prepareOptions(Map, Map) method to mutate the user input
+ * before it becomes an immutable part of the ReplicationParams.
+ *
+ * @param strategyClass The class to call prepareOptions on
+ * @param strategyOptions The proposed strategy options that will be potentially mutated by the prepareOptions
+ * method.
+ * @param previousStrategyOptions In the case of an ALTER statement, the previous strategy options of this class.
+ * This map cannot be mutated.
+ */
+ public static void prepareReplicationStrategyOptions(Class<? extends AbstractReplicationStrategy> strategyClass,
+ Map<String, String> strategyOptions,
+ Map<String, String> previousStrategyOptions)
+ {
+ try
+ {
+ Method method = strategyClass.getDeclaredMethod("prepareOptions", Map.class, Map.class);
+ method.invoke(null, strategyOptions, previousStrategyOptions);
+ }
+ catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException ign)
+ {
+ // If the subclass doesn't specify a prepareOptions method, then that means that it
+ // doesn't want to do anything to the options. So do nothing on reflection related exceptions.
+ }
+ }
+
public static void validateReplicationStrategy(String keyspaceName,
Class<? extends AbstractReplicationStrategy> strategyClass,
TokenMetadata tokenMetadata,
@@ -287,6 +410,10 @@
AbstractReplicationStrategy strategy = createInternal(keyspaceName, strategyClass, tokenMetadata, snitch, strategyOptions);
strategy.validateExpectedOptions();
strategy.validateOptions();
+ if (strategy.hasTransientReplicas() && !DatabaseDescriptor.isTransientReplicationEnabled())
+ {
+ throw new ConfigurationException("Transient replication is disabled. Enable in cassandra.yaml to use.");
+ }
}
public static Class<AbstractReplicationStrategy> getClass(String cls) throws ConfigurationException
@@ -302,25 +429,27 @@
public boolean hasSameSettings(AbstractReplicationStrategy other)
{
- return getClass().equals(other.getClass()) && getReplicationFactor() == other.getReplicationFactor();
+ return getClass().equals(other.getClass()) && getReplicationFactor().equals(other.getReplicationFactor());
}
- protected void validateReplicationFactor(String rf) throws ConfigurationException
+ protected void validateReplicationFactor(String s) throws ConfigurationException
{
try
{
- if (Integer.parseInt(rf) < 0)
+ ReplicationFactor rf = ReplicationFactor.fromString(s);
+ if (rf.hasTransientReplicas())
{
- throw new ConfigurationException("Replication factor must be non-negative; found " + rf);
+ if (DatabaseDescriptor.getNumTokens() > 1)
+ throw new ConfigurationException(String.format("Transient replication is not supported with vnodes yet"));
}
}
- catch (NumberFormatException e2)
+ catch (IllegalArgumentException e)
{
- throw new ConfigurationException("Replication factor must be numeric; found " + rf);
+ throw new ConfigurationException(e.getMessage());
}
}
- private void validateExpectedOptions() throws ConfigurationException
+ protected void validateExpectedOptions() throws ConfigurationException
{
Collection expectedOptions = recognizedOptions();
if (expectedOptions == null)
diff --git a/src/java/org/apache/cassandra/locator/AlibabaCloudSnitch.java b/src/java/org/apache/cassandra/locator/AlibabaCloudSnitch.java
new file mode 100644
index 0000000..729e1b3
--- /dev/null
+++ b/src/java/org/apache/cassandra/locator/AlibabaCloudSnitch.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.locator;
+
+import java.io.DataInputStream;
+import java.io.FilterInputStream;
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.MalformedURLException;
+import java.net.SocketTimeoutException;
+import java.net.URL;
+import java.nio.charset.StandardCharsets;
+import java.util.Map;
+import org.apache.cassandra.db.SystemKeyspace;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.gms.ApplicationState;
+import org.apache.cassandra.gms.EndpointState;
+import org.apache.cassandra.gms.Gossiper;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.utils.FBUtilities;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * A snitch that assumes an ECS region is a DC and an ECS availability_zone
+ * is a rack. This information is available in the config for the node. the
+ * format of the zone-id is like :cn-hangzhou-a where cn means china, hangzhou
+ * means the hangzhou region, a means the az id. We use cn-hangzhou as the dc,
+ * and f as the zone-id.
+ */
+public class AlibabaCloudSnitch extends AbstractNetworkTopologySnitch
+{
+ protected static final Logger logger = LoggerFactory.getLogger(AlibabaCloudSnitch.class);
+ protected static final String ZONE_NAME_QUERY_URL = "http://100.100.100.200/latest/meta-data/zone-id";
+ private static final String DEFAULT_DC = "UNKNOWN-DC";
+ private static final String DEFAULT_RACK = "UNKNOWN-RACK";
+ private Map<InetAddressAndPort, Map<String, String>> savedEndpoints;
+ protected String ecsZone;
+ protected String ecsRegion;
+
+ private static final int HTTP_CONNECT_TIMEOUT = 30000;
+
+
+ public AlibabaCloudSnitch() throws MalformedURLException, IOException
+ {
+ String response = alibabaApiCall(ZONE_NAME_QUERY_URL);
+ String[] splits = response.split("/");
+ String az = splits[splits.length - 1];
+
+ // Split "us-central1-a" or "asia-east1-a" into "us-central1"/"a" and "asia-east1"/"a".
+ splits = az.split("-");
+ ecsZone = splits[splits.length - 1];
+
+ int lastRegionIndex = az.lastIndexOf("-");
+ ecsRegion = az.substring(0, lastRegionIndex);
+
+ String datacenterSuffix = (new SnitchProperties()).get("dc_suffix", "");
+ ecsRegion = ecsRegion.concat(datacenterSuffix);
+ logger.info("AlibabaSnitch using region: {}, zone: {}.", ecsRegion, ecsZone);
+
+ }
+
+ String alibabaApiCall(String url) throws ConfigurationException, IOException, SocketTimeoutException
+ {
+ // Populate the region and zone by introspection, fail if 404 on metadata
+ HttpURLConnection conn = (HttpURLConnection) new URL(url).openConnection();
+ DataInputStream d = null;
+ try
+ {
+ conn.setConnectTimeout(HTTP_CONNECT_TIMEOUT);
+ conn.setRequestMethod("GET");
+
+ int code = conn.getResponseCode();
+ if (code != HttpURLConnection.HTTP_OK)
+ throw new ConfigurationException("AlibabaSnitch was unable to execute the API call. Not an ecs node? and the returun code is " + code);
+
+ // Read the information. I wish I could say (String) conn.getContent() here...
+ int cl = conn.getContentLength();
+ byte[] b = new byte[cl];
+ d = new DataInputStream((FilterInputStream) conn.getContent());
+ d.readFully(b);
+ return new String(b, StandardCharsets.UTF_8);
+ }
+ catch (SocketTimeoutException e)
+ {
+ throw new SocketTimeoutException("Timeout occurred reading a response from the Alibaba ECS metadata");
+ }
+ finally
+ {
+ FileUtils.close(d);
+ conn.disconnect();
+ }
+ }
+
+ @Override
+ public String getRack(InetAddressAndPort endpoint)
+ {
+ if (endpoint.equals(FBUtilities.getBroadcastAddressAndPort()))
+ return ecsZone;
+ EndpointState state = Gossiper.instance.getEndpointStateForEndpoint(endpoint);
+ if (state == null || state.getApplicationState(ApplicationState.RACK) == null)
+ {
+ if (savedEndpoints == null)
+ savedEndpoints = SystemKeyspace.loadDcRackInfo();
+ if (savedEndpoints.containsKey(endpoint))
+ return savedEndpoints.get(endpoint).get("rack");
+ return DEFAULT_RACK;
+ }
+ return state.getApplicationState(ApplicationState.RACK).value;
+
+ }
+
+ @Override
+ public String getDatacenter(InetAddressAndPort endpoint)
+ {
+ if (endpoint.equals(FBUtilities.getBroadcastAddressAndPort()))
+ return ecsRegion;
+ EndpointState state = Gossiper.instance.getEndpointStateForEndpoint(endpoint);
+ if (state == null || state.getApplicationState(ApplicationState.DC) == null)
+ {
+ if (savedEndpoints == null)
+ savedEndpoints = SystemKeyspace.loadDcRackInfo();
+ if (savedEndpoints.containsKey(endpoint))
+ return savedEndpoints.get(endpoint).get("data_center");
+ return DEFAULT_DC;
+ }
+ return state.getApplicationState(ApplicationState.DC).value;
+
+ }
+
+}
diff --git a/src/java/org/apache/cassandra/locator/CloudstackSnitch.java b/src/java/org/apache/cassandra/locator/CloudstackSnitch.java
index ec2e87e..be6d3c4 100644
--- a/src/java/org/apache/cassandra/locator/CloudstackSnitch.java
+++ b/src/java/org/apache/cassandra/locator/CloudstackSnitch.java
@@ -24,7 +24,6 @@
import java.io.IOException;
import java.io.File;
import java.net.HttpURLConnection;
-import java.net.InetAddress;
import java.net.URL;
import java.net.URI;
import java.nio.charset.StandardCharsets;
@@ -56,7 +55,7 @@
protected static final Logger logger = LoggerFactory.getLogger(CloudstackSnitch.class);
protected static final String ZONE_NAME_QUERY_URI = "/latest/meta-data/availability-zone";
- private Map<InetAddress, Map<String, String>> savedEndpoints;
+ private Map<InetAddressAndPort, Map<String, String>> savedEndpoints;
private static final String DEFAULT_DC = "UNKNOWN-DC";
private static final String DEFAULT_RACK = "UNKNOWN-RACK";
@@ -83,9 +82,9 @@
csZoneRack = zone_parts[2];
}
- public String getRack(InetAddress endpoint)
+ public String getRack(InetAddressAndPort endpoint)
{
- if (endpoint.equals(FBUtilities.getBroadcastAddress()))
+ if (endpoint.equals(FBUtilities.getBroadcastAddressAndPort()))
return csZoneRack;
EndpointState state = Gossiper.instance.getEndpointStateForEndpoint(endpoint);
if (state == null || state.getApplicationState(ApplicationState.RACK) == null)
@@ -99,9 +98,9 @@
return state.getApplicationState(ApplicationState.RACK).value;
}
- public String getDatacenter(InetAddress endpoint)
+ public String getDatacenter(InetAddressAndPort endpoint)
{
- if (endpoint.equals(FBUtilities.getBroadcastAddress()))
+ if (endpoint.equals(FBUtilities.getBroadcastAddressAndPort()))
return csZoneDc;
EndpointState state = Gossiper.instance.getEndpointStateForEndpoint(endpoint);
if (state == null || state.getApplicationState(ApplicationState.DC) == null)
diff --git a/src/java/org/apache/cassandra/locator/DynamicEndpointSnitch.java b/src/java/org/apache/cassandra/locator/DynamicEndpointSnitch.java
index 53e4d1d..218bdd6 100644
--- a/src/java/org/apache/cassandra/locator/DynamicEndpointSnitch.java
+++ b/src/java/org/apache/cassandra/locator/DynamicEndpointSnitch.java
@@ -24,6 +24,7 @@
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
import com.codahale.metrics.ExponentiallyDecayingReservoir;
@@ -34,16 +35,16 @@
import org.apache.cassandra.gms.EndpointState;
import org.apache.cassandra.gms.Gossiper;
import org.apache.cassandra.gms.VersionedValue;
+import org.apache.cassandra.net.LatencySubscribers;
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.MBeanWrapper;
-
/**
* A dynamic snitch that sorts endpoints by latency with an adapted phi failure detector
*/
-public class DynamicEndpointSnitch extends AbstractEndpointSnitch implements ILatencySubscriber, DynamicEndpointSnitchMBean
+public class DynamicEndpointSnitch extends AbstractEndpointSnitch implements LatencySubscribers.Subscriber, DynamicEndpointSnitchMBean
{
private static final boolean USE_SEVERITY = !Boolean.getBoolean("cassandra.ignore_dynamic_snitch_severity");
@@ -61,8 +62,8 @@
private String mbeanName;
private boolean registered = false;
- private volatile HashMap<InetAddress, Double> scores = new HashMap<>();
- private final ConcurrentHashMap<InetAddress, ExponentiallyDecayingReservoir> samples = new ConcurrentHashMap<>();
+ private volatile HashMap<InetAddressAndPort, Double> scores = new HashMap<>();
+ private final ConcurrentHashMap<InetAddressAndPort, ExponentiallyDecayingReservoir> samples = new ConcurrentHashMap<>();
public final IEndpointSnitch subsnitch;
@@ -156,67 +157,50 @@
subsnitch.gossiperStarting();
}
- public String getRack(InetAddress endpoint)
+ public String getRack(InetAddressAndPort endpoint)
{
return subsnitch.getRack(endpoint);
}
- public String getDatacenter(InetAddress endpoint)
+ public String getDatacenter(InetAddressAndPort endpoint)
{
return subsnitch.getDatacenter(endpoint);
}
- public List<InetAddress> getSortedListByProximity(final InetAddress address, Collection<InetAddress> addresses)
- {
- List<InetAddress> list = new ArrayList<InetAddress>(addresses);
- sortByProximity(address, list);
- return list;
- }
-
@Override
- public void sortByProximity(final InetAddress address, List<InetAddress> addresses)
+ public <C extends ReplicaCollection<? extends C>> C sortedByProximity(final InetAddressAndPort address, C unsortedAddresses)
{
- assert address.equals(FBUtilities.getBroadcastAddress()); // we only know about ourself
- if (dynamicBadnessThreshold == 0)
- {
- sortByProximityWithScore(address, addresses);
- }
- else
- {
- sortByProximityWithBadness(address, addresses);
- }
+ assert address.equals(FBUtilities.getBroadcastAddressAndPort()); // we only know about ourself
+ return dynamicBadnessThreshold == 0
+ ? sortedByProximityWithScore(address, unsortedAddresses)
+ : sortedByProximityWithBadness(address, unsortedAddresses);
}
- private void sortByProximityWithScore(final InetAddress address, List<InetAddress> addresses)
+ private <C extends ReplicaCollection<? extends C>> C sortedByProximityWithScore(final InetAddressAndPort address, C unsortedAddresses)
{
// Scores can change concurrently from a call to this method. But Collections.sort() expects
// its comparator to be "stable", that is 2 endpoint should compare the same way for the duration
// of the sort() call. As we copy the scores map on write, it is thus enough to alias the current
// version of it during this call.
- final HashMap<InetAddress, Double> scores = this.scores;
- Collections.sort(addresses, new Comparator<InetAddress>()
- {
- public int compare(InetAddress a1, InetAddress a2)
- {
- return compareEndpoints(address, a1, a2, scores);
- }
- });
+ final HashMap<InetAddressAndPort, Double> scores = this.scores;
+ return unsortedAddresses.sorted((r1, r2) -> compareEndpoints(address, r1, r2, scores));
}
- private void sortByProximityWithBadness(final InetAddress address, List<InetAddress> addresses)
+ private <C extends ReplicaCollection<? extends C>> C sortedByProximityWithBadness(final InetAddressAndPort address, C replicas)
{
- if (addresses.size() < 2)
- return;
+ if (replicas.size() < 2)
+ return replicas;
- subsnitch.sortByProximity(address, addresses);
- HashMap<InetAddress, Double> scores = this.scores; // Make sure the score don't change in the middle of the loop below
+ // TODO: avoid copy
+ replicas = subsnitch.sortedByProximity(address, replicas);
+ HashMap<InetAddressAndPort, Double> scores = this.scores; // Make sure the score don't change in the middle of the loop below
// (which wouldn't really matter here but its cleaner that way).
- ArrayList<Double> subsnitchOrderedScores = new ArrayList<>(addresses.size());
- for (InetAddress inet : addresses)
+ ArrayList<Double> subsnitchOrderedScores = new ArrayList<>(replicas.size());
+ for (Replica replica : replicas)
{
- Double score = scores.get(inet);
+ Double score = scores.get(replica.endpoint());
if (score == null)
- continue;
+ score = 0.0;
subsnitchOrderedScores.add(score);
}
@@ -226,22 +210,25 @@
ArrayList<Double> sortedScores = new ArrayList<>(subsnitchOrderedScores);
Collections.sort(sortedScores);
+ // only calculate this once b/c its volatile and shouldn't be modified during the loop either
+ double badnessThreshold = 1.0 + dynamicBadnessThreshold;
Iterator<Double> sortedScoreIterator = sortedScores.iterator();
for (Double subsnitchScore : subsnitchOrderedScores)
{
- if (subsnitchScore > (sortedScoreIterator.next() * (1.0 + dynamicBadnessThreshold)))
+ if (subsnitchScore > (sortedScoreIterator.next() * badnessThreshold))
{
- sortByProximityWithScore(address, addresses);
- return;
+ return sortedByProximityWithScore(address, replicas);
}
}
+
+ return replicas;
}
// Compare endpoints given an immutable snapshot of the scores
- private int compareEndpoints(InetAddress target, InetAddress a1, InetAddress a2, Map<InetAddress, Double> scores)
+ private int compareEndpoints(InetAddressAndPort target, Replica a1, Replica a2, Map<InetAddressAndPort, Double> scores)
{
- Double scored1 = scores.get(a1);
- Double scored2 = scores.get(a2);
+ Double scored1 = scores.get(a1.endpoint());
+ Double scored2 = scores.get(a2.endpoint());
if (scored1 == null)
{
@@ -261,7 +248,7 @@
return 1;
}
- public int compareEndpoints(InetAddress target, InetAddress a1, InetAddress a2)
+ public int compareEndpoints(InetAddressAndPort target, Replica a1, Replica a2)
{
// That function is fundamentally unsafe because the scores can change at any time and so the result of that
// method is not stable for identical arguments. This is why we don't rely on super.sortByProximity() in
@@ -269,7 +256,7 @@
throw new UnsupportedOperationException("You shouldn't wrap the DynamicEndpointSnitch (within itself or otherwise)");
}
- public void receiveTiming(InetAddress host, long latency) // this is cheap
+ public void receiveTiming(InetAddressAndPort host, long latency, TimeUnit unit) // this is cheap
{
ExponentiallyDecayingReservoir sample = samples.get(host);
if (sample == null)
@@ -279,7 +266,7 @@
if (sample == null)
sample = maybeNewSample;
}
- sample.update(latency);
+ sample.update(unit.toMillis(latency));
}
private void updateScores() // this is expensive
@@ -290,30 +277,30 @@
{
if (MessagingService.instance() != null)
{
- MessagingService.instance().register(this);
+ MessagingService.instance().latencySubscribers.subscribe(this);
registered = true;
}
}
double maxLatency = 1;
- Map<InetAddress, Snapshot> snapshots = new HashMap<>(samples.size());
- for (Map.Entry<InetAddress, ExponentiallyDecayingReservoir> entry : samples.entrySet())
+ Map<InetAddressAndPort, Snapshot> snapshots = new HashMap<>(samples.size());
+ for (Map.Entry<InetAddressAndPort, ExponentiallyDecayingReservoir> entry : samples.entrySet())
{
snapshots.put(entry.getKey(), entry.getValue().getSnapshot());
}
// We're going to weight the latency for each host against the worst one we see, to
// arrive at sort of a 'badness percentage' for them. First, find the worst for each:
- HashMap<InetAddress, Double> newScores = new HashMap<>();
- for (Map.Entry<InetAddress, Snapshot> entry : snapshots.entrySet())
+ HashMap<InetAddressAndPort, Double> newScores = new HashMap<>();
+ for (Map.Entry<InetAddressAndPort, Snapshot> entry : snapshots.entrySet())
{
double mean = entry.getValue().getMedian();
if (mean > maxLatency)
maxLatency = mean;
}
// now make another pass to do the weighting based on the maximums we found before
- for (Map.Entry<InetAddress, Snapshot> entry : snapshots.entrySet())
+ for (Map.Entry<InetAddressAndPort, Snapshot> entry : snapshots.entrySet())
{
double score = entry.getValue().getMedian() / maxLatency;
// finally, add the severity without any weighting, since hosts scale this relative to their own load and the size of the task causing the severity.
@@ -333,6 +320,11 @@
public Map<InetAddress, Double> getScores()
{
+ return scores.entrySet().stream().collect(Collectors.toMap(address -> address.getKey().address, Map.Entry::getValue));
+ }
+
+ public Map<InetAddressAndPort, Double> getScoresWithPort()
+ {
return scores;
}
@@ -356,7 +348,7 @@
public List<Double> dumpTimings(String hostname) throws UnknownHostException
{
- InetAddress host = InetAddress.getByName(hostname);
+ InetAddressAndPort host = InetAddressAndPort.getByName(hostname);
ArrayList<Double> timings = new ArrayList<Double>();
ExponentiallyDecayingReservoir sample = samples.get(host);
if (sample != null)
@@ -372,7 +364,7 @@
Gossiper.instance.addLocalApplicationState(ApplicationState.SEVERITY, StorageService.instance.valueFactory.severity(severity));
}
- private double getSeverity(InetAddress endpoint)
+ private double getSeverity(InetAddressAndPort endpoint)
{
EndpointState state = Gossiper.instance.getEndpointStateForEndpoint(endpoint);
if (state == null)
@@ -387,10 +379,10 @@
public double getSeverity()
{
- return getSeverity(FBUtilities.getBroadcastAddress());
+ return getSeverity(FBUtilities.getBroadcastAddressAndPort());
}
- public boolean isWorthMergingForRangeQuery(List<InetAddress> merged, List<InetAddress> l1, List<InetAddress> l2)
+ public boolean isWorthMergingForRangeQuery(ReplicaCollection<?> merged, ReplicaCollection<?> l1, ReplicaCollection<?> l2)
{
if (!subsnitch.isWorthMergingForRangeQuery(merged, l1, l2))
return false;
@@ -410,12 +402,12 @@
}
// Return the max score for the endpoint in the provided list, or -1.0 if no node have a score.
- private double maxScore(List<InetAddress> endpoints)
+ private double maxScore(ReplicaCollection<?> endpoints)
{
double maxScore = -1.0;
- for (InetAddress endpoint : endpoints)
+ for (Replica replica : endpoints)
{
- Double score = scores.get(endpoint);
+ Double score = scores.get(replica.endpoint());
if (score == null)
continue;
@@ -424,4 +416,9 @@
}
return maxScore;
}
+
+ public boolean validate(Set<String> datacenters, Set<String> racks)
+ {
+ return subsnitch.validate(datacenters, racks);
+ }
}
diff --git a/src/java/org/apache/cassandra/locator/DynamicEndpointSnitchMBean.java b/src/java/org/apache/cassandra/locator/DynamicEndpointSnitchMBean.java
index bfafa75..61f0d97 100644
--- a/src/java/org/apache/cassandra/locator/DynamicEndpointSnitchMBean.java
+++ b/src/java/org/apache/cassandra/locator/DynamicEndpointSnitchMBean.java
@@ -24,6 +24,8 @@
public interface DynamicEndpointSnitchMBean
{
+ public Map<InetAddressAndPort, Double> getScoresWithPort();
+ @Deprecated
public Map<InetAddress, Double> getScores();
public int getUpdateInterval();
public int getResetInterval();
diff --git a/src/java/org/apache/cassandra/locator/Ec2MultiRegionSnitch.java b/src/java/org/apache/cassandra/locator/Ec2MultiRegionSnitch.java
index b32ca84..2a6c7e9 100644
--- a/src/java/org/apache/cassandra/locator/Ec2MultiRegionSnitch.java
+++ b/src/java/org/apache/cassandra/locator/Ec2MultiRegionSnitch.java
@@ -19,6 +19,7 @@
import java.io.IOException;
import java.net.InetAddress;
+import java.net.UnknownHostException;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.config.DatabaseDescriptor;
@@ -62,6 +63,16 @@
public void gossiperStarting()
{
super.gossiperStarting();
+ InetAddressAndPort address;
+ try
+ {
+ address = InetAddressAndPort.getByName(localPrivateAddress);
+ }
+ catch (UnknownHostException e)
+ {
+ throw new RuntimeException(e);
+ }
+ Gossiper.instance.addLocalApplicationState(ApplicationState.INTERNAL_ADDRESS_AND_PORT, StorageService.instance.valueFactory.internalAddressAndPort(address));
Gossiper.instance.addLocalApplicationState(ApplicationState.INTERNAL_IP, StorageService.instance.valueFactory.internalIP(localPrivateAddress));
Gossiper.instance.register(new ReconnectableSnitchHelper(this, ec2region, true));
}
diff --git a/src/java/org/apache/cassandra/locator/Ec2Snitch.java b/src/java/org/apache/cassandra/locator/Ec2Snitch.java
index 59eb27b..5e51408 100644
--- a/src/java/org/apache/cassandra/locator/Ec2Snitch.java
+++ b/src/java/org/apache/cassandra/locator/Ec2Snitch.java
@@ -21,11 +21,12 @@
import java.io.FilterInputStream;
import java.io.IOException;
import java.net.HttpURLConnection;
-import java.net.InetAddress;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.util.Map;
+import java.util.Set;
+import com.google.common.annotations.VisibleForTesting;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.db.SystemKeyspace;
@@ -43,30 +44,63 @@
public class Ec2Snitch extends AbstractNetworkTopologySnitch
{
protected static final Logger logger = LoggerFactory.getLogger(Ec2Snitch.class);
- protected static final String ZONE_NAME_QUERY_URL = "http://169.254.169.254/latest/meta-data/placement/availability-zone";
+
+ private static final String SNITCH_PROP_NAMING_SCHEME = "ec2_naming_scheme";
+ static final String EC2_NAMING_LEGACY = "legacy";
+ private static final String EC2_NAMING_STANDARD = "standard";
+
+ private static final String ZONE_NAME_QUERY_URL = "http://169.254.169.254/latest/meta-data/placement/availability-zone";
private static final String DEFAULT_DC = "UNKNOWN-DC";
private static final String DEFAULT_RACK = "UNKNOWN-RACK";
- private Map<InetAddress, Map<String, String>> savedEndpoints;
- protected String ec2zone;
- protected String ec2region;
+
+ final String ec2region;
+ private final String ec2zone;
+ private final boolean usingLegacyNaming;
+
+ private Map<InetAddressAndPort, Map<String, String>> savedEndpoints;
public Ec2Snitch() throws IOException, ConfigurationException
{
+ this(new SnitchProperties());
+ }
+
+ public Ec2Snitch(SnitchProperties props) throws IOException, ConfigurationException
+ {
String az = awsApiCall(ZONE_NAME_QUERY_URL);
- // Split "us-east-1a" or "asia-1a" into "us-east"/"1a" and "asia"/"1a".
- String[] splits = az.split("-");
- ec2zone = splits[splits.length - 1];
- // hack for CASSANDRA-4026
- ec2region = az.substring(0, az.length() - 1);
- if (ec2region.endsWith("1"))
- ec2region = az.substring(0, az.length() - 3);
+ // if using the full naming scheme, region name is created by removing letters from the
+ // end of the availability zone and zone is the full zone name
+ usingLegacyNaming = isUsingLegacyNaming(props);
+ String region;
+ if (usingLegacyNaming)
+ {
+ // Split "us-east-1a" or "asia-1a" into "us-east"/"1a" and "asia"/"1a".
+ String[] splits = az.split("-");
+ ec2zone = splits[splits.length - 1];
- String datacenterSuffix = (new SnitchProperties()).get("dc_suffix", "");
- ec2region = ec2region.concat(datacenterSuffix);
+ // hack for CASSANDRA-4026
+ region = az.substring(0, az.length() - 1);
+ if (region.endsWith("1"))
+ region = az.substring(0, az.length() - 3);
+ }
+ else
+ {
+ // grab the region name, which is embedded in the availability zone name.
+ // thus an AZ of "us-east-1a" yields the region name "us-east-1"
+ region = az.replaceFirst("[a-z]+$","");
+ ec2zone = az;
+ }
+
+ String datacenterSuffix = props.get("dc_suffix", "");
+ ec2region = region.concat(datacenterSuffix);
logger.info("EC2Snitch using region: {}, zone: {}.", ec2region, ec2zone);
}
+ private static boolean isUsingLegacyNaming(SnitchProperties props)
+ {
+ return props.get(SNITCH_PROP_NAMING_SCHEME, EC2_NAMING_STANDARD).equalsIgnoreCase(EC2_NAMING_LEGACY);
+ }
+
String awsApiCall(String url) throws IOException, ConfigurationException
{
// Populate the region and zone by introspection, fail if 404 on metadata
@@ -92,9 +126,9 @@
}
}
- public String getRack(InetAddress endpoint)
+ public String getRack(InetAddressAndPort endpoint)
{
- if (endpoint.equals(FBUtilities.getBroadcastAddress()))
+ if (endpoint.equals(FBUtilities.getBroadcastAddressAndPort()))
return ec2zone;
EndpointState state = Gossiper.instance.getEndpointStateForEndpoint(endpoint);
if (state == null || state.getApplicationState(ApplicationState.RACK) == null)
@@ -108,9 +142,9 @@
return state.getApplicationState(ApplicationState.RACK).value;
}
- public String getDatacenter(InetAddress endpoint)
+ public String getDatacenter(InetAddressAndPort endpoint)
{
- if (endpoint.equals(FBUtilities.getBroadcastAddress()))
+ if (endpoint.equals(FBUtilities.getBroadcastAddressAndPort()))
return ec2region;
EndpointState state = Gossiper.instance.getEndpointStateForEndpoint(endpoint);
if (state == null || state.getApplicationState(ApplicationState.DC) == null)
@@ -123,4 +157,55 @@
}
return state.getApplicationState(ApplicationState.DC).value;
}
+
+ @Override
+ public boolean validate(Set<String> datacenters, Set<String> racks)
+ {
+ return validate(datacenters, racks, usingLegacyNaming);
+ }
+
+ @VisibleForTesting
+ static boolean validate(Set<String> datacenters, Set<String> racks, boolean usingLegacyNaming)
+ {
+ boolean valid = true;
+
+ for (String dc : datacenters)
+ {
+ // predicated on the late-2017 AWS naming 'convention' that all region names end with a digit.
+ // Unfortunately, life isn't that simple. Since we allow custom datacenter suffixes (CASSANDRA-5155),
+ // an operator could conceiveably make the suffix "a", and thus create a region name that looks just like
+ // one of the region's availability zones. (for example, "us-east-1a").
+ // Further, we can't make any assumptions of what that suffix might be by looking at this node's
+ // datacenterSuffix as conceivably their could be many different suffixes in play for a given region.
+ //
+ // It is impossible to distinguish standard and legacy names for datacenters in some cases
+ // as the format didn't change for some regions (us-west-2 for example).
+ // We can still identify as legacy the dc names without a number as a suffix like us-east"
+ boolean dcUsesLegacyFormat = dc.matches("^[a-z]+-[a-z]+$");
+ if (dcUsesLegacyFormat && !usingLegacyNaming)
+ valid = false;
+ }
+
+ for (String rack : racks)
+ {
+ // predicated on late-2017 AWS naming 'convention' that AZs do not have a digit as the first char -
+ // we had that in our legacy AZ (rack) names. Thus we test to see if the rack is in the legacy format.
+ //
+ // NOTE: the allowed custom suffix only applies to datacenter (region) names, not availability zones.
+ boolean rackUsesLegacyFormat = rack.matches("[\\d][a-z]");
+ if (rackUsesLegacyFormat != usingLegacyNaming)
+ valid = false;
+ }
+
+ if (!valid)
+ {
+ logger.error("This ec2-enabled snitch appears to be using the {} naming scheme for regions, " +
+ "but existing nodes in cluster are using the opposite: region(s) = {}, availability zone(s) = {}. " +
+ "Please check the {} property in the {} configuration file for more details.",
+ usingLegacyNaming ? "legacy" : "standard", datacenters, racks,
+ SNITCH_PROP_NAMING_SCHEME, SnitchProperties.RACKDC_PROPERTY_FILENAME);
+ }
+
+ return valid;
+ }
}
diff --git a/src/java/org/apache/cassandra/locator/EndpointSnitchInfo.java b/src/java/org/apache/cassandra/locator/EndpointSnitchInfo.java
index be28b3c..d836cd1 100644
--- a/src/java/org/apache/cassandra/locator/EndpointSnitchInfo.java
+++ b/src/java/org/apache/cassandra/locator/EndpointSnitchInfo.java
@@ -17,12 +17,9 @@
*/
package org.apache.cassandra.locator;
-
-import java.net.InetAddress;
import java.net.UnknownHostException;
import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.MBeanWrapper;
public class EndpointSnitchInfo implements EndpointSnitchInfoMBean
@@ -34,22 +31,22 @@
public String getDatacenter(String host) throws UnknownHostException
{
- return DatabaseDescriptor.getEndpointSnitch().getDatacenter(InetAddress.getByName(host));
+ return DatabaseDescriptor.getEndpointSnitch().getDatacenter(InetAddressAndPort.getByName(host));
}
public String getRack(String host) throws UnknownHostException
{
- return DatabaseDescriptor.getEndpointSnitch().getRack(InetAddress.getByName(host));
+ return DatabaseDescriptor.getEndpointSnitch().getRack(InetAddressAndPort.getByName(host));
}
public String getDatacenter()
{
- return DatabaseDescriptor.getEndpointSnitch().getDatacenter(FBUtilities.getBroadcastAddress());
+ return DatabaseDescriptor.getEndpointSnitch().getLocalDatacenter();
}
public String getRack()
{
- return DatabaseDescriptor.getEndpointSnitch().getRack(FBUtilities.getBroadcastAddress());
+ return DatabaseDescriptor.getEndpointSnitch().getLocalRack();
}
public String getSnitchName()
diff --git a/src/java/org/apache/cassandra/locator/Endpoints.java b/src/java/org/apache/cassandra/locator/Endpoints.java
new file mode 100644
index 0000000..c1a9282
--- /dev/null
+++ b/src/java/org/apache/cassandra/locator/Endpoints.java
@@ -0,0 +1,167 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.locator;
+
+import org.apache.cassandra.locator.ReplicaCollection.Builder.Conflict;
+import org.apache.cassandra.utils.FBUtilities;
+
+import java.util.AbstractList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+
+import com.google.common.collect.Lists;
+
+/**
+ * A collection of Endpoints for a given ring position. This will typically reside in a ReplicaLayout,
+ * representing some subset of the endpoints for the Token or Range
+ * @param <E> The concrete type of Endpoints, that will be returned by the modifying methods
+ */
+public abstract class Endpoints<E extends Endpoints<E>> extends AbstractReplicaCollection<E>
+{
+ static ReplicaMap<InetAddressAndPort> endpointMap(ReplicaList list) { return new ReplicaMap<>(list, Replica::endpoint); }
+ static final ReplicaMap<InetAddressAndPort> EMPTY_MAP = endpointMap(EMPTY_LIST);
+
+ // volatile not needed, as has only final members,
+ // besides (transitively) those that cache objects that themselves have only final members
+ ReplicaMap<InetAddressAndPort> byEndpoint;
+
+ Endpoints(ReplicaList list, ReplicaMap<InetAddressAndPort> byEndpoint)
+ {
+ super(list);
+ this.byEndpoint = byEndpoint;
+ }
+
+ @Override
+ public Set<InetAddressAndPort> endpoints()
+ {
+ return byEndpoint().keySet();
+ }
+
+ public List<InetAddressAndPort> endpointList()
+ {
+ return new AbstractList<InetAddressAndPort>()
+ {
+ public InetAddressAndPort get(int index)
+ {
+ return list.get(index).endpoint();
+ }
+
+ public int size()
+ {
+ return list.size;
+ }
+ };
+ }
+
+ public Map<InetAddressAndPort, Replica> byEndpoint()
+ {
+ ReplicaMap<InetAddressAndPort> map = byEndpoint;
+ if (map == null)
+ byEndpoint = map = endpointMap(list);
+ return map;
+ }
+
+ @Override
+ public boolean contains(Replica replica)
+ {
+ return replica != null
+ && Objects.equals(
+ byEndpoint().get(replica.endpoint()),
+ replica);
+ }
+
+ public E withoutSelf()
+ {
+ InetAddressAndPort self = FBUtilities.getBroadcastAddressAndPort();
+ return filter(r -> !self.equals(r.endpoint()));
+ }
+
+ public Replica selfIfPresent()
+ {
+ InetAddressAndPort self = FBUtilities.getBroadcastAddressAndPort();
+ return byEndpoint().get(self);
+ }
+
+ /**
+ * @return a collection without the provided endpoints, otherwise in the same order as this collection
+ */
+ public E without(Set<InetAddressAndPort> remove)
+ {
+ return filter(r -> !remove.contains(r.endpoint()));
+ }
+
+ /**
+ * @return a collection with only the provided endpoints (ignoring any not present), otherwise in the same order as this collection
+ */
+ public E keep(Set<InetAddressAndPort> keep)
+ {
+ return filter(r -> keep.contains(r.endpoint()));
+ }
+
+ /**
+ * @return a collection containing the Replica from this collection for the provided endpoints, in the order of the provided endpoints
+ */
+ public E select(Iterable<InetAddressAndPort> endpoints, boolean ignoreMissing)
+ {
+ Builder<E> copy = newBuilder(
+ endpoints instanceof Collection<?>
+ ? ((Collection<InetAddressAndPort>) endpoints).size()
+ : size()
+ );
+ Map<InetAddressAndPort, Replica> byEndpoint = byEndpoint();
+ for (InetAddressAndPort endpoint : endpoints)
+ {
+ Replica select = byEndpoint.get(endpoint);
+ if (select == null)
+ {
+ if (!ignoreMissing)
+ throw new IllegalArgumentException(endpoint + " is not present in " + this);
+ continue;
+ }
+ copy.add(select, Builder.Conflict.DUPLICATE);
+ }
+ return copy.build();
+ }
+
+ /**
+ * Care must be taken to ensure no conflicting ranges occur in pending and natural.
+ * Conflicts can occur for two reasons:
+ * 1) due to lack of isolation when reading pending/natural
+ * 2) because a movement that changes the type of replication from transient to full must be handled
+ * differently for reads and writes (with the reader treating it as transient, and writer as full)
+ *
+ * The method {@link ReplicaLayout#haveWriteConflicts} can be used to detect and resolve any issues
+ */
+ public static <E extends Endpoints<E>> E concat(E natural, E pending)
+ {
+ return AbstractReplicaCollection.concat(natural, pending, Conflict.NONE);
+ }
+
+ public static <E extends Endpoints<E>> E append(E replicas, Replica extraReplica)
+ {
+ Builder<E> builder = replicas.newBuilder(replicas.size() + 1);
+ builder.addAll(replicas);
+ builder.add(extraReplica, Conflict.NONE);
+ return builder.build();
+ }
+
+}
diff --git a/src/java/org/apache/cassandra/locator/EndpointsByRange.java b/src/java/org/apache/cassandra/locator/EndpointsByRange.java
new file mode 100644
index 0000000..a1b03b3
--- /dev/null
+++ b/src/java/org/apache/cassandra/locator/EndpointsByRange.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.locator;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Maps;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.locator.ReplicaCollection.Builder.Conflict;
+
+import java.util.Map;
+
+public class EndpointsByRange extends ReplicaMultimap<Range<Token>, EndpointsForRange>
+{
+ public EndpointsByRange(Map<Range<Token>, EndpointsForRange> map)
+ {
+ super(map);
+ }
+
+ public EndpointsForRange get(Range<Token> range)
+ {
+ Preconditions.checkNotNull(range);
+ return map.getOrDefault(range, EndpointsForRange.empty(range));
+ }
+
+ public static class Builder extends ReplicaMultimap.Builder<Range<Token>, EndpointsForRange.Builder>
+ {
+ @Override
+ protected EndpointsForRange.Builder newBuilder(Range<Token> range)
+ {
+ return new EndpointsForRange.Builder(range);
+ }
+
+ // TODO: consider all ignoreDuplicates cases
+ public void putAll(Range<Token> range, EndpointsForRange replicas, Conflict ignoreConflicts)
+ {
+ get(range).addAll(replicas, ignoreConflicts);
+ }
+
+ public EndpointsByRange build()
+ {
+ return new EndpointsByRange(
+ ImmutableMap.copyOf(
+ Maps.transformValues(this.map, EndpointsForRange.Builder::build)));
+ }
+ }
+
+}
diff --git a/src/java/org/apache/cassandra/locator/EndpointsByReplica.java b/src/java/org/apache/cassandra/locator/EndpointsByReplica.java
new file mode 100644
index 0000000..72d8751
--- /dev/null
+++ b/src/java/org/apache/cassandra/locator/EndpointsByReplica.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.locator;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Maps;
+import org.apache.cassandra.locator.ReplicaCollection.Builder.Conflict;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+public class EndpointsByReplica extends ReplicaMultimap<Replica, EndpointsForRange>
+{
+ public EndpointsByReplica(Map<Replica, EndpointsForRange> map)
+ {
+ super(map);
+ }
+
+ public EndpointsForRange get(Replica range)
+ {
+ Preconditions.checkNotNull(range);
+ return map.getOrDefault(range, EndpointsForRange.empty(range.range()));
+ }
+
+ public static class Builder extends ReplicaMultimap.Builder<Replica, EndpointsForRange.Builder>
+ {
+ @Override
+ protected EndpointsForRange.Builder newBuilder(Replica replica)
+ {
+ return new EndpointsForRange.Builder(replica.range());
+ }
+
+ // TODO: consider all ignoreDuplicates cases
+ public void putAll(Replica range, EndpointsForRange replicas, Conflict ignoreConflicts)
+ {
+ map.computeIfAbsent(range, r -> newBuilder(r)).addAll(replicas, ignoreConflicts);
+ }
+
+ public EndpointsByReplica build()
+ {
+ return new EndpointsByReplica(
+ ImmutableMap.copyOf(
+ Maps.transformValues(this.map, EndpointsForRange.Builder::build)));
+ }
+ }
+
+}
diff --git a/src/java/org/apache/cassandra/locator/EndpointsForRange.java b/src/java/org/apache/cassandra/locator/EndpointsForRange.java
new file mode 100644
index 0000000..7039df0
--- /dev/null
+++ b/src/java/org/apache/cassandra/locator/EndpointsForRange.java
@@ -0,0 +1,161 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.locator;
+
+import com.google.common.base.Preconditions;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+
+import java.util.Arrays;
+import java.util.Collection;
+
+import static com.google.common.collect.Iterables.all;
+
+/**
+ * A ReplicaCollection where all Replica are required to cover a range that fully contains the range() defined in the builder().
+ * Endpoints are guaranteed to be unique; on construction, this is enforced unless optionally silenced (in which case
+ * only the first occurrence makes the cut).
+ */
+public class EndpointsForRange extends Endpoints<EndpointsForRange>
+{
+ private final Range<Token> range;
+ private EndpointsForRange(Range<Token> range, ReplicaList list, ReplicaMap<InetAddressAndPort> byEndpoint)
+ {
+ super(list, byEndpoint);
+ this.range = range;
+ assert range != null;
+ }
+
+ public Range<Token> range()
+ {
+ return range;
+ }
+
+ @Override
+ public Builder newBuilder(int initialCapacity)
+ {
+ return new Builder(range, initialCapacity);
+ }
+
+ public EndpointsForToken forToken(Token token)
+ {
+ if (!range.contains(token))
+ throw new IllegalArgumentException(token + " is not contained within " + range);
+ return new EndpointsForToken(token, list, byEndpoint);
+ }
+
+ @Override
+ public EndpointsForRange snapshot()
+ {
+ return this;
+ }
+
+ @Override
+ EndpointsForRange snapshot(ReplicaList newList)
+ {
+ if (newList.isEmpty()) return empty(range);
+ ReplicaMap<InetAddressAndPort> byEndpoint = null;
+ if (this.byEndpoint != null && list.isSubList(newList))
+ byEndpoint = this.byEndpoint.forSubList(newList);
+ return new EndpointsForRange(range, newList, byEndpoint);
+ }
+
+ public static class Builder extends EndpointsForRange implements ReplicaCollection.Builder<EndpointsForRange>
+ {
+ boolean built;
+ public Builder(Range<Token> range) { this(range, 0); }
+ public Builder(Range<Token> range, int capacity) { this(range, new ReplicaList(capacity)); }
+ private Builder(Range<Token> range, ReplicaList list) { super(range, list, endpointMap(list)); }
+
+ public EndpointsForRange.Builder add(Replica replica, Conflict ignoreConflict)
+ {
+ if (built) throw new IllegalStateException();
+ Preconditions.checkNotNull(replica);
+ if (!replica.range().contains(super.range))
+ throw new IllegalArgumentException("Replica " + replica + " does not contain " + super.range);
+
+ if (!super.byEndpoint.internalPutIfAbsent(replica, list.size()))
+ {
+ switch (ignoreConflict)
+ {
+ case DUPLICATE:
+ if (byEndpoint().get(replica.endpoint()).equals(replica))
+ break;
+ case NONE:
+ throw new IllegalArgumentException("Conflicting replica added (expected unique endpoints): "
+ + replica + "; existing: " + byEndpoint().get(replica.endpoint()));
+ case ALL:
+ }
+ return this;
+ }
+
+ list.add(replica);
+ return this;
+ }
+
+ @Override
+ public EndpointsForRange snapshot()
+ {
+ return snapshot(list.subList(0, list.size()));
+ }
+
+ public EndpointsForRange build()
+ {
+ built = true;
+ return new EndpointsForRange(super.range, super.list, super.byEndpoint);
+ }
+ }
+
+ public static Builder builder(Range<Token> range)
+ {
+ return new Builder(range);
+ }
+ public static Builder builder(Range<Token> range, int capacity)
+ {
+ return new Builder(range, capacity);
+ }
+
+ public static EndpointsForRange empty(Range<Token> range)
+ {
+ return new EndpointsForRange(range, EMPTY_LIST, EMPTY_MAP);
+ }
+
+ public static EndpointsForRange of(Replica replica)
+ {
+ // we only use ArrayList or ArrayList.SubList, to ensure callsites are bimorphic
+ ReplicaList one = new ReplicaList(1);
+ one.add(replica);
+ // we can safely use singletonMap, as we only otherwise use LinkedHashMap
+ return new EndpointsForRange(replica.range(), one, endpointMap(one));
+ }
+
+ public static EndpointsForRange of(Replica ... replicas)
+ {
+ return copyOf(Arrays.asList(replicas));
+ }
+
+ public static EndpointsForRange copyOf(Collection<Replica> replicas)
+ {
+ if (replicas.isEmpty())
+ throw new IllegalArgumentException("Collection must be non-empty to copy");
+ Range<Token> range = replicas.iterator().next().range();
+ assert all(replicas, r -> range.equals(r.range()));
+ return builder(range, replicas.size()).addAll(replicas).build();
+ }
+}
diff --git a/src/java/org/apache/cassandra/locator/EndpointsForToken.java b/src/java/org/apache/cassandra/locator/EndpointsForToken.java
new file mode 100644
index 0000000..c709988
--- /dev/null
+++ b/src/java/org/apache/cassandra/locator/EndpointsForToken.java
@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.locator;
+
+import com.google.common.base.Preconditions;
+import org.apache.cassandra.dht.Token;
+
+import java.util.Arrays;
+import java.util.Collection;
+
+/**
+ * A ReplicaCollection where all Replica are required to cover a range that fully contains the token() defined in the builder().
+ * Endpoints are guaranteed to be unique; on construction, this is enforced unless optionally silenced (in which case
+ * only the first occurrence makes the cut).
+ */
+public class EndpointsForToken extends Endpoints<EndpointsForToken>
+{
+ private final Token token;
+
+ EndpointsForToken(Token token, ReplicaList list, ReplicaMap<InetAddressAndPort> byEndpoint)
+ {
+ super(list, byEndpoint);
+ this.token = token;
+ assert token != null;
+ }
+
+ public Token token()
+ {
+ return token;
+ }
+
+ @Override
+ public Builder newBuilder(int initialCapacity)
+ {
+ return new Builder(token, initialCapacity);
+ }
+
+ @Override
+ public EndpointsForToken snapshot()
+ {
+ return this;
+ }
+
+ @Override
+ protected EndpointsForToken snapshot(ReplicaList newList)
+ {
+ if (newList.isEmpty()) return empty(token);
+ ReplicaMap<InetAddressAndPort> byEndpoint = null;
+ if (this.byEndpoint != null && list.isSubList(newList))
+ byEndpoint = this.byEndpoint.forSubList(newList);
+ return new EndpointsForToken(token, newList, byEndpoint);
+ }
+
+ public static class Builder extends EndpointsForToken implements ReplicaCollection.Builder<EndpointsForToken>
+ {
+ boolean built;
+ public Builder(Token token) { this(token, 0); }
+ public Builder(Token token, int capacity) { this(token, new ReplicaList(capacity)); }
+ private Builder(Token token, ReplicaList list) { super(token, list, endpointMap(list)); }
+
+ public EndpointsForToken.Builder add(Replica replica, Conflict ignoreConflict)
+ {
+ if (built) throw new IllegalStateException();
+ Preconditions.checkNotNull(replica);
+ if (!replica.range().contains(super.token))
+ throw new IllegalArgumentException("Replica " + replica + " does not contain " + super.token);
+
+ if (!super.byEndpoint.internalPutIfAbsent(replica, list.size()))
+ {
+ switch (ignoreConflict)
+ {
+ case DUPLICATE:
+ if (byEndpoint().get(replica.endpoint()).equals(replica))
+ break;
+ case NONE:
+ throw new IllegalArgumentException("Conflicting replica added (expected unique endpoints): "
+ + replica + "; existing: " + byEndpoint().get(replica.endpoint()));
+ case ALL:
+ }
+ return this;
+ }
+
+ list.add(replica);
+ return this;
+ }
+
+ @Override
+ public EndpointsForToken snapshot()
+ {
+ return snapshot(list.subList(0, list.size()));
+ }
+
+ public EndpointsForToken build()
+ {
+ built = true;
+ return new EndpointsForToken(super.token, super.list, super.byEndpoint);
+ }
+ }
+
+ public static Builder builder(Token token)
+ {
+ return new Builder(token);
+ }
+ public static Builder builder(Token token, int capacity)
+ {
+ return new Builder(token, capacity);
+ }
+
+ public static EndpointsForToken empty(Token token)
+ {
+ return new EndpointsForToken(token, EMPTY_LIST, EMPTY_MAP);
+ }
+
+ public static EndpointsForToken of(Token token, Replica replica)
+ {
+ // we only use ArrayList or ArrayList.SubList, to ensure callsites are bimorphic
+ ReplicaList one = new ReplicaList(1);
+ one.add(replica);
+ // we can safely use singletonMap, as we only otherwise use LinkedHashMap
+ return new EndpointsForToken(token, one, endpointMap(one));
+ }
+
+ public static EndpointsForToken of(Token token, Replica ... replicas)
+ {
+ return copyOf(token, Arrays.asList(replicas));
+ }
+
+ public static EndpointsForToken copyOf(Token token, Collection<Replica> replicas)
+ {
+ if (replicas.isEmpty()) return empty(token);
+ return builder(token, replicas.size()).addAll(replicas).build();
+ }
+}
diff --git a/src/java/org/apache/cassandra/locator/GoogleCloudSnitch.java b/src/java/org/apache/cassandra/locator/GoogleCloudSnitch.java
index b4d3b19..1e1c500 100644
--- a/src/java/org/apache/cassandra/locator/GoogleCloudSnitch.java
+++ b/src/java/org/apache/cassandra/locator/GoogleCloudSnitch.java
@@ -21,7 +21,6 @@
import java.io.FilterInputStream;
import java.io.IOException;
import java.net.HttpURLConnection;
-import java.net.InetAddress;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.util.Map;
@@ -46,7 +45,7 @@
protected static final String ZONE_NAME_QUERY_URL = "http://metadata.google.internal/computeMetadata/v1/instance/zone";
private static final String DEFAULT_DC = "UNKNOWN-DC";
private static final String DEFAULT_RACK = "UNKNOWN-RACK";
- private Map<InetAddress, Map<String, String>> savedEndpoints;
+ private Map<InetAddressAndPort, Map<String, String>> savedEndpoints;
protected String gceZone;
protected String gceRegion;
@@ -94,9 +93,9 @@
}
}
- public String getRack(InetAddress endpoint)
+ public String getRack(InetAddressAndPort endpoint)
{
- if (endpoint.equals(FBUtilities.getBroadcastAddress()))
+ if (endpoint.equals(FBUtilities.getBroadcastAddressAndPort()))
return gceZone;
EndpointState state = Gossiper.instance.getEndpointStateForEndpoint(endpoint);
if (state == null || state.getApplicationState(ApplicationState.RACK) == null)
@@ -110,9 +109,9 @@
return state.getApplicationState(ApplicationState.RACK).value;
}
- public String getDatacenter(InetAddress endpoint)
+ public String getDatacenter(InetAddressAndPort endpoint)
{
- if (endpoint.equals(FBUtilities.getBroadcastAddress()))
+ if (endpoint.equals(FBUtilities.getBroadcastAddressAndPort()))
return gceRegion;
EndpointState state = Gossiper.instance.getEndpointStateForEndpoint(endpoint);
if (state == null || state.getApplicationState(ApplicationState.DC) == null)
diff --git a/src/java/org/apache/cassandra/locator/GossipingPropertyFileSnitch.java b/src/java/org/apache/cassandra/locator/GossipingPropertyFileSnitch.java
index e2449ae..75b5685 100644
--- a/src/java/org/apache/cassandra/locator/GossipingPropertyFileSnitch.java
+++ b/src/java/org/apache/cassandra/locator/GossipingPropertyFileSnitch.java
@@ -18,7 +18,6 @@
package org.apache.cassandra.locator;
-import java.net.InetAddress;
import java.util.concurrent.atomic.AtomicReference;
import java.util.Map;
@@ -45,7 +44,7 @@
private final boolean preferLocal;
private final AtomicReference<ReconnectableSnitchHelper> snitchHelperReference;
- private Map<InetAddress, Map<String, String>> savedEndpoints;
+ private Map<InetAddressAndPort, Map<String, String>> savedEndpoints;
private static final String DEFAULT_DC = "UNKNOWN_DC";
private static final String DEFAULT_RACK = "UNKNOWN_RACK";
@@ -84,9 +83,9 @@
* @param endpoint the endpoint to process
* @return string of data center
*/
- public String getDatacenter(InetAddress endpoint)
+ public String getDatacenter(InetAddressAndPort endpoint)
{
- if (endpoint.equals(FBUtilities.getBroadcastAddress()))
+ if (endpoint.equals(FBUtilities.getBroadcastAddressAndPort()))
return myDC;
EndpointState epState = Gossiper.instance.getEndpointStateForEndpoint(endpoint);
@@ -112,9 +111,9 @@
* @param endpoint the endpoint to process
* @return string of rack
*/
- public String getRack(InetAddress endpoint)
+ public String getRack(InetAddressAndPort endpoint)
{
- if (endpoint.equals(FBUtilities.getBroadcastAddress()))
+ if (endpoint.equals(FBUtilities.getBroadcastAddressAndPort()))
return myRack;
EndpointState epState = Gossiper.instance.getEndpointStateForEndpoint(endpoint);
@@ -138,8 +137,10 @@
{
super.gossiperStarting();
+ Gossiper.instance.addLocalApplicationState(ApplicationState.INTERNAL_ADDRESS_AND_PORT,
+ StorageService.instance.valueFactory.internalAddressAndPort(FBUtilities.getLocalAddressAndPort()));
Gossiper.instance.addLocalApplicationState(ApplicationState.INTERNAL_IP,
- StorageService.instance.valueFactory.internalIP(FBUtilities.getLocalAddress().getHostAddress()));
+ StorageService.instance.valueFactory.internalIP(FBUtilities.getJustLocalAddress().getHostAddress()));
loadGossiperState();
}
diff --git a/src/java/org/apache/cassandra/locator/IEndpointSnitch.java b/src/java/org/apache/cassandra/locator/IEndpointSnitch.java
index 690b84f..381a642 100644
--- a/src/java/org/apache/cassandra/locator/IEndpointSnitch.java
+++ b/src/java/org/apache/cassandra/locator/IEndpointSnitch.java
@@ -17,42 +17,58 @@
*/
package org.apache.cassandra.locator;
-import java.net.InetAddress;
-import java.util.Collection;
-import java.util.List;
+import java.util.Set;
+
+import org.apache.cassandra.utils.FBUtilities;
/**
- * This interface helps determine location of node in the data center relative to another node.
+ * This interface helps determine location of node in the datacenter relative to another node.
* Give a node A and another node B it can tell if A and B are on the same rack or in the same
- * data center.
+ * datacenter.
*/
public interface IEndpointSnitch
{
/**
- * returns a String repesenting the rack this endpoint belongs to
+ * returns a String representing the rack the given endpoint belongs to
*/
- public String getRack(InetAddress endpoint);
+ public String getRack(InetAddressAndPort endpoint);
/**
- * returns a String representing the datacenter this endpoint belongs to
+ * returns a String representing the rack current endpoint belongs to
*/
- public String getDatacenter(InetAddress endpoint);
+ default public String getLocalRack()
+ {
+ return getRack(FBUtilities.getBroadcastAddressAndPort());
+ }
+
+ /**
+ * returns a String representing the datacenter the given endpoint belongs to
+ */
+ public String getDatacenter(InetAddressAndPort endpoint);
+
+ /**
+ * returns a String representing the datacenter current endpoint belongs to
+ */
+ default public String getLocalDatacenter()
+ {
+ return getDatacenter(FBUtilities.getBroadcastAddressAndPort());
+ }
+
+ default public String getDatacenter(Replica replica)
+ {
+ return getDatacenter(replica.endpoint());
+ }
/**
* returns a new <tt>List</tt> sorted by proximity to the given endpoint
*/
- public List<InetAddress> getSortedListByProximity(InetAddress address, Collection<InetAddress> unsortedAddress);
-
- /**
- * This method will sort the <tt>List</tt> by proximity to the given address.
- */
- public void sortByProximity(InetAddress address, List<InetAddress> addresses);
+ public <C extends ReplicaCollection<? extends C>> C sortedByProximity(final InetAddressAndPort address, C addresses);
/**
* compares two endpoints in relation to the target endpoint, returning as Comparator.compare would
*/
- public int compareEndpoints(InetAddress target, InetAddress a1, InetAddress a2);
+ public int compareEndpoints(InetAddressAndPort target, Replica r1, Replica r2);
/**
* called after Gossiper instance exists immediately before it starts gossiping
@@ -63,5 +79,13 @@
* Returns whether for a range query doing a query against merged is likely
* to be faster than 2 sequential queries, one against l1 followed by one against l2.
*/
- public boolean isWorthMergingForRangeQuery(List<InetAddress> merged, List<InetAddress> l1, List<InetAddress> l2);
+ public boolean isWorthMergingForRangeQuery(ReplicaCollection<?> merged, ReplicaCollection<?> l1, ReplicaCollection<?> l2);
+
+ /**
+ * Determine if the datacenter or rack values in the current node's snitch conflict with those passed in parameters.
+ */
+ default boolean validate(Set<String> datacenters, Set<String> racks)
+ {
+ return true;
+ }
}
diff --git a/src/java/org/apache/cassandra/locator/ILatencySubscriber.java b/src/java/org/apache/cassandra/locator/ILatencySubscriber.java
deleted file mode 100644
index d2ae6db..0000000
--- a/src/java/org/apache/cassandra/locator/ILatencySubscriber.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.locator;
-
-import java.net.InetAddress;
-
-public interface ILatencySubscriber
-{
- public void receiveTiming(InetAddress address, long latency);
-}
diff --git a/src/java/org/apache/cassandra/locator/InOurDcTester.java b/src/java/org/apache/cassandra/locator/InOurDcTester.java
new file mode 100644
index 0000000..514c7ef
--- /dev/null
+++ b/src/java/org/apache/cassandra/locator/InOurDcTester.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.locator;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.utils.FBUtilities;
+import java.util.function.Predicate;
+
+public class InOurDcTester
+{
+ private static ReplicaTester replicas;
+ private static EndpointTester endpoints;
+
+ final String dc;
+ final IEndpointSnitch snitch;
+
+ private InOurDcTester(String dc, IEndpointSnitch snitch)
+ {
+ this.dc = dc;
+ this.snitch = snitch;
+ }
+
+ boolean stale()
+ {
+ return dc != DatabaseDescriptor.getLocalDataCenter()
+ || snitch != DatabaseDescriptor.getEndpointSnitch()
+ // this final clause checks if somehow the snitch/localDc have got out of whack;
+ // presently, this is possible but very unlikely, but this check will also help
+ // resolve races on these global fields as well
+ || !dc.equals(snitch.getLocalDatacenter());
+ }
+
+ private static final class ReplicaTester extends InOurDcTester implements Predicate<Replica>
+ {
+ private ReplicaTester(String dc, IEndpointSnitch snitch)
+ {
+ super(dc, snitch);
+ }
+
+ @Override
+ public boolean test(Replica replica)
+ {
+ return dc.equals(snitch.getDatacenter(replica.endpoint()));
+ }
+ }
+
+ private static final class EndpointTester extends InOurDcTester implements Predicate<InetAddressAndPort>
+ {
+ private EndpointTester(String dc, IEndpointSnitch snitch)
+ {
+ super(dc, snitch);
+ }
+
+ @Override
+ public boolean test(InetAddressAndPort endpoint)
+ {
+ return dc.equals(snitch.getDatacenter(endpoint));
+ }
+ }
+
+ public static Predicate<Replica> replicas()
+ {
+ ReplicaTester cur = replicas;
+ if (cur == null || cur.stale())
+ replicas = cur = new ReplicaTester(DatabaseDescriptor.getLocalDataCenter(), DatabaseDescriptor.getEndpointSnitch());
+ return cur;
+ }
+
+ public static Predicate<InetAddressAndPort> endpoints()
+ {
+ EndpointTester cur = endpoints;
+ if (cur == null || cur.stale())
+ endpoints = cur = new EndpointTester(DatabaseDescriptor.getLocalDataCenter(), DatabaseDescriptor.getEndpointSnitch());
+ return cur;
+ }
+
+}
diff --git a/src/java/org/apache/cassandra/locator/InetAddressAndPort.java b/src/java/org/apache/cassandra/locator/InetAddressAndPort.java
index 6daa2e1..6821f13 100644
--- a/src/java/org/apache/cassandra/locator/InetAddressAndPort.java
+++ b/src/java/org/apache/cassandra/locator/InetAddressAndPort.java
@@ -15,16 +15,24 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.cassandra.locator;
+import java.io.IOException;
import java.io.Serializable;
+import java.net.Inet4Address;
+import java.net.Inet6Address;
import java.net.InetAddress;
import java.net.UnknownHostException;
+import java.nio.ByteBuffer;
import com.google.common.base.Preconditions;
import com.google.common.net.HostAndPort;
+import org.apache.cassandra.io.IVersionedSerializer;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.FastByteOperations;
@@ -41,6 +49,7 @@
* need to sometimes return a port and sometimes not.
*
*/
+@SuppressWarnings("UnstableApiUsage")
public final class InetAddressAndPort implements Comparable<InetAddressAndPort>, Serializable
{
private static final long serialVersionUID = 0;
@@ -65,6 +74,11 @@
this.addressBytes = addressBytes;
}
+ public InetAddressAndPort withPort(int port)
+ {
+ return new InetAddressAndPort(address, addressBytes, port);
+ }
+
private static void validatePortRange(int port)
{
if (port < 0 | port > 65535)
@@ -127,7 +141,7 @@
{
if (withPort)
{
- return HostAndPort.fromParts(address.getHostAddress(), port).toString();
+ return toString(address, port);
}
else
{
@@ -135,6 +149,11 @@
}
}
+ public static String toString(InetAddress address, int port)
+ {
+ return HostAndPort.fromParts(address.getHostAddress(), port).toString();
+ }
+
public static InetAddressAndPort getByName(String name) throws UnknownHostException
{
return getByNameOverrideDefaults(name, null);
@@ -144,8 +163,6 @@
*
* @param name Hostname + optional ports string
* @param port Port to connect on, overridden by values in hostname string, defaults to DatabaseDescriptor default if not specified anywhere.
- * @return
- * @throws UnknownHostException
*/
public static InetAddressAndPort getByNameOverrideDefaults(String name, Integer port) throws UnknownHostException
{
@@ -154,7 +171,7 @@
{
port = hap.getPort();
}
- return getByAddressOverrideDefaults(InetAddress.getByName(hap.getHostText()), port);
+ return getByAddressOverrideDefaults(InetAddress.getByName(hap.getHost()), port);
}
public static InetAddressAndPort getByAddress(byte[] address) throws UnknownHostException
@@ -187,8 +204,128 @@
return new InetAddressAndPort(address, addressBytes, port);
}
+ public static InetAddressAndPort getLoopbackAddress()
+ {
+ return InetAddressAndPort.getByAddress(InetAddress.getLoopbackAddress());
+ }
+
+ public static InetAddressAndPort getLocalHost()
+ {
+ return FBUtilities.getLocalAddressAndPort();
+ }
+
public static void initializeDefaultPort(int port)
{
defaultPort = port;
}
+
+ static int getDefaultPort()
+ {
+ return defaultPort;
+ }
+
+ /*
+ * As of version 4.0 the endpoint description includes a port number as an unsigned short
+ */
+ public static final class Serializer implements IVersionedSerializer<InetAddressAndPort>
+ {
+ public static final int MAXIMUM_SIZE = 19;
+
+ // We put the static instance here, to avoid complexity with dtests.
+ // InetAddressAndPort is one of the only classes we share between instances, which is possible cleanly
+ // because it has no type-dependencies in its public API, however Serializer requires DataOutputPlus, which requires...
+ // and the chain becomes quite unwieldy
+ public static final Serializer inetAddressAndPortSerializer = new Serializer();
+
+ private Serializer() {}
+
+ public void serialize(InetAddressAndPort endpoint, DataOutputPlus out, int version) throws IOException
+ {
+ byte[] buf = endpoint.addressBytes;
+
+ if (version >= MessagingService.VERSION_40)
+ {
+ out.writeByte(buf.length + 2);
+ out.write(buf);
+ out.writeShort(endpoint.port);
+ }
+ else
+ {
+ out.writeByte(buf.length);
+ out.write(buf);
+ }
+ }
+
+ public InetAddressAndPort deserialize(DataInputPlus in, int version) throws IOException
+ {
+ int size = in.readByte() & 0xFF;
+ switch(size)
+ {
+ //The original pre-4.0 serialiation of just an address
+ case 4:
+ case 16:
+ {
+ byte[] bytes = new byte[size];
+ in.readFully(bytes, 0, bytes.length);
+ return getByAddress(bytes);
+ }
+ //Address and one port
+ case 6:
+ case 18:
+ {
+ byte[] bytes = new byte[size - 2];
+ in.readFully(bytes);
+
+ int port = in.readShort() & 0xFFFF;
+ return getByAddressOverrideDefaults(InetAddress.getByAddress(bytes), bytes, port);
+ }
+ default:
+ throw new AssertionError("Unexpected size " + size);
+
+ }
+ }
+
+ /**
+ * Extract {@link InetAddressAndPort} from the provided {@link ByteBuffer} without altering its state.
+ */
+ public InetAddressAndPort extract(ByteBuffer buf, int position) throws IOException
+ {
+ int size = buf.get(position++) & 0xFF;
+ if (size == 4 || size == 16)
+ {
+ byte[] bytes = new byte[size];
+ ByteBufferUtil.copyBytes(buf, position, bytes, 0, size);
+ return getByAddress(bytes);
+ }
+ else if (size == 6 || size == 18)
+ {
+ byte[] bytes = new byte[size - 2];
+ ByteBufferUtil.copyBytes(buf, position, bytes, 0, size - 2);
+ position += (size - 2);
+ int port = buf.getShort(position) & 0xFFFF;
+ return getByAddressOverrideDefaults(InetAddress.getByAddress(bytes), bytes, port);
+ }
+
+ throw new AssertionError("Unexpected pre-4.0 InetAddressAndPort size " + size);
+ }
+
+ public long serializedSize(InetAddressAndPort from, int version)
+ {
+ //4.0 includes a port number
+ if (version >= MessagingService.VERSION_40)
+ {
+ if (from.address instanceof Inet4Address)
+ return 1 + 4 + 2;
+ assert from.address instanceof Inet6Address;
+ return 1 + 16 + 2;
+ }
+ else
+ {
+ if (from.address instanceof Inet4Address)
+ return 1 + 4;
+ assert from.address instanceof Inet6Address;
+ return 1 + 16;
+ }
+ }
+ }
}
diff --git a/src/java/org/apache/cassandra/locator/LocalStrategy.java b/src/java/org/apache/cassandra/locator/LocalStrategy.java
index ae58203..41cc9b0 100644
--- a/src/java/org/apache/cassandra/locator/LocalStrategy.java
+++ b/src/java/org/apache/cassandra/locator/LocalStrategy.java
@@ -17,13 +17,11 @@
*/
package org.apache.cassandra.locator;
-import java.net.InetAddress;
-import java.util.ArrayList;
import java.util.Collections;
import java.util.Collection;
-import java.util.List;
import java.util.Map;
+import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.dht.RingPosition;
import org.apache.cassandra.dht.Token;
@@ -31,32 +29,40 @@
public class LocalStrategy extends AbstractReplicationStrategy
{
+ private static final ReplicationFactor RF = ReplicationFactor.fullOnly(1);
+ private final EndpointsForRange replicas;
+
public LocalStrategy(String keyspaceName, TokenMetadata tokenMetadata, IEndpointSnitch snitch, Map<String, String> configOptions)
{
super(keyspaceName, tokenMetadata, snitch, configOptions);
+ replicas = EndpointsForRange.of(
+ new Replica(FBUtilities.getBroadcastAddressAndPort(),
+ DatabaseDescriptor.getPartitioner().getMinimumToken(),
+ DatabaseDescriptor.getPartitioner().getMinimumToken(),
+ true
+ )
+ );
}
/**
- * We need to override this even if we override calculateNaturalEndpoints,
+ * We need to override this even if we override calculateNaturalReplicas,
* because the default implementation depends on token calculations but
* LocalStrategy may be used before tokens are set up.
*/
@Override
- public ArrayList<InetAddress> getNaturalEndpoints(RingPosition searchPosition)
+ public EndpointsForRange getNaturalReplicas(RingPosition searchPosition)
{
- ArrayList<InetAddress> l = new ArrayList<InetAddress>(1);
- l.add(FBUtilities.getBroadcastAddress());
- return l;
+ return replicas;
}
- public List<InetAddress> calculateNaturalEndpoints(Token token, TokenMetadata metadata)
+ public EndpointsForRange calculateNaturalReplicas(Token token, TokenMetadata metadata)
{
- return Collections.singletonList(FBUtilities.getBroadcastAddress());
+ return replicas;
}
- public int getReplicationFactor()
+ public ReplicationFactor getReplicationFactor()
{
- return 1;
+ return RF;
}
public void validateOptions() throws ConfigurationException
diff --git a/src/java/org/apache/cassandra/locator/NetworkTopologyStrategy.java b/src/java/org/apache/cassandra/locator/NetworkTopologyStrategy.java
index d48dec3..be63ea1 100644
--- a/src/java/org/apache/cassandra/locator/NetworkTopologyStrategy.java
+++ b/src/java/org/apache/cassandra/locator/NetworkTopologyStrategy.java
@@ -17,13 +17,15 @@
*/
package org.apache.cassandra.locator;
-import java.net.InetAddress;
import java.util.*;
import java.util.Map.Entry;
+import org.apache.cassandra.locator.ReplicaCollection.Builder.Conflict;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.cassandra.dht.Datacenters;
+import org.apache.cassandra.dht.Range;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.locator.TokenMetadata.Topology;
@@ -50,28 +52,36 @@
*/
public class NetworkTopologyStrategy extends AbstractReplicationStrategy
{
- private final Map<String, Integer> datacenters;
+ private final Map<String, ReplicationFactor> datacenters;
+ private final ReplicationFactor aggregateRf;
private static final Logger logger = LoggerFactory.getLogger(NetworkTopologyStrategy.class);
+ private static final String REPLICATION_FACTOR = "replication_factor";
public NetworkTopologyStrategy(String keyspaceName, TokenMetadata tokenMetadata, IEndpointSnitch snitch, Map<String, String> configOptions) throws ConfigurationException
{
super(keyspaceName, tokenMetadata, snitch, configOptions);
- Map<String, Integer> newDatacenters = new HashMap<String, Integer>();
+ int replicas = 0;
+ int trans = 0;
+ Map<String, ReplicationFactor> newDatacenters = new HashMap<>();
if (configOptions != null)
{
for (Entry<String, String> entry : configOptions.entrySet())
{
String dc = entry.getKey();
- if (dc.equalsIgnoreCase("replication_factor"))
- throw new ConfigurationException("replication_factor is an option for SimpleStrategy, not NetworkTopologyStrategy");
- Integer replicas = Integer.valueOf(entry.getValue());
- newDatacenters.put(dc, replicas);
+ // prepareOptions should have transformed any "replication_factor" options by now
+ if (dc.equalsIgnoreCase(REPLICATION_FACTOR))
+ throw new ConfigurationException(REPLICATION_FACTOR + " should not appear as an option at construction time for NetworkTopologyStrategy");
+ ReplicationFactor rf = ReplicationFactor.fromString(entry.getValue());
+ replicas += rf.allReplicas;
+ trans += rf.transientReplicas();
+ newDatacenters.put(dc, rf);
}
}
datacenters = Collections.unmodifiableMap(newDatacenters);
- logger.trace("Configured datacenter replicas are {}", FBUtilities.toString(datacenters));
+ aggregateRf = ReplicationFactor.withTransient(replicas, trans);
+ logger.info("Configured datacenter replicas are {}", FBUtilities.toString(datacenters));
}
/**
@@ -80,7 +90,8 @@
private static final class DatacenterEndpoints
{
/** List accepted endpoints get pushed into. */
- Set<InetAddress> endpoints;
+ EndpointsForRange.Builder replicas;
+
/**
* Racks encountered so far. Replicas are put into separate racks while possible.
* For efficiency the set is shared between the instances, using the location pair (dc, rack) to make sure
@@ -91,41 +102,51 @@
/** Number of replicas left to fill from this DC. */
int rfLeft;
int acceptableRackRepeats;
+ int transients;
- DatacenterEndpoints(int rf, int rackCount, int nodeCount, Set<InetAddress> endpoints, Set<Pair<String, String>> racks)
+ DatacenterEndpoints(ReplicationFactor rf, int rackCount, int nodeCount, EndpointsForRange.Builder replicas, Set<Pair<String, String>> racks)
{
- this.endpoints = endpoints;
+ this.replicas = replicas;
this.racks = racks;
// If there aren't enough nodes in this DC to fill the RF, the number of nodes is the effective RF.
- this.rfLeft = Math.min(rf, nodeCount);
+ this.rfLeft = Math.min(rf.allReplicas, nodeCount);
// If there aren't enough racks in this DC to fill the RF, we'll still use at least one node from each rack,
// and the difference is to be filled by the first encountered nodes.
- acceptableRackRepeats = rf - rackCount;
+ acceptableRackRepeats = rf.allReplicas - rackCount;
+
+ // if we have fewer replicas than rf calls for, reduce transients accordingly
+ int reduceTransients = rf.allReplicas - this.rfLeft;
+ transients = Math.max(rf.transientReplicas() - reduceTransients, 0);
+ ReplicationFactor.validate(rfLeft, transients);
}
/**
- * Attempts to add an endpoint to the replicas for this datacenter, adding to the endpoints set if successful.
+ * Attempts to add an endpoint to the replicas for this datacenter, adding to the replicas set if successful.
* Returns true if the endpoint was added, and this datacenter does not require further replicas.
*/
- boolean addEndpointAndCheckIfDone(InetAddress ep, Pair<String,String> location)
+ boolean addEndpointAndCheckIfDone(InetAddressAndPort ep, Pair<String,String> location, Range<Token> replicatedRange)
{
if (done())
return false;
+ if (replicas.endpoints().contains(ep))
+ // Cannot repeat a node.
+ return false;
+
+ Replica replica = new Replica(ep, replicatedRange, rfLeft > transients);
+
if (racks.add(location))
{
// New rack.
--rfLeft;
- boolean added = endpoints.add(ep);
- assert added;
+ replicas.add(replica, Conflict.NONE);
return done();
}
if (acceptableRackRepeats <= 0)
// There must be rfLeft distinct racks left, do not add any more rack repeats.
return false;
- if (!endpoints.add(ep))
- // Cannot repeat a node.
- return false;
+
+ replicas.add(replica, Conflict.NONE);
// Added a node that is from an already met rack to match RF when there aren't enough racks.
--acceptableRackRepeats;
--rfLeft;
@@ -142,48 +163,53 @@
/**
* calculate endpoints in one pass through the tokens by tracking our progress in each DC.
*/
- public List<InetAddress> calculateNaturalEndpoints(Token searchToken, TokenMetadata tokenMetadata)
+ public EndpointsForRange calculateNaturalReplicas(Token searchToken, TokenMetadata tokenMetadata)
{
// we want to preserve insertion order so that the first added endpoint becomes primary
- Set<InetAddress> replicas = new LinkedHashSet<>();
+ ArrayList<Token> sortedTokens = tokenMetadata.sortedTokens();
+ Token replicaEnd = TokenMetadata.firstToken(sortedTokens, searchToken);
+ Token replicaStart = tokenMetadata.getPredecessor(replicaEnd);
+ Range<Token> replicatedRange = new Range<>(replicaStart, replicaEnd);
+
+ EndpointsForRange.Builder builder = new EndpointsForRange.Builder(replicatedRange);
Set<Pair<String, String>> seenRacks = new HashSet<>();
Topology topology = tokenMetadata.getTopology();
// all endpoints in each DC, so we can check when we have exhausted all the members of a DC
- Multimap<String, InetAddress> allEndpoints = topology.getDatacenterEndpoints();
+ Multimap<String, InetAddressAndPort> allEndpoints = topology.getDatacenterEndpoints();
// all racks in a DC so we can check when we have exhausted all racks in a DC
- Map<String, ImmutableMultimap<String, InetAddress>> racks = topology.getDatacenterRacks();
+ Map<String, ImmutableMultimap<String, InetAddressAndPort>> racks = topology.getDatacenterRacks();
assert !allEndpoints.isEmpty() && !racks.isEmpty() : "not aware of any cluster members";
int dcsToFill = 0;
Map<String, DatacenterEndpoints> dcs = new HashMap<>(datacenters.size() * 2);
// Create a DatacenterEndpoints object for each non-empty DC.
- for (Map.Entry<String, Integer> en : datacenters.entrySet())
+ for (Map.Entry<String, ReplicationFactor> en : datacenters.entrySet())
{
String dc = en.getKey();
- int rf = en.getValue();
+ ReplicationFactor rf = en.getValue();
int nodeCount = sizeOrZero(allEndpoints.get(dc));
- if (rf <= 0 || nodeCount <= 0)
+ if (rf.allReplicas <= 0 || nodeCount <= 0)
continue;
- DatacenterEndpoints dcEndpoints = new DatacenterEndpoints(rf, sizeOrZero(racks.get(dc)), nodeCount, replicas, seenRacks);
+ DatacenterEndpoints dcEndpoints = new DatacenterEndpoints(rf, sizeOrZero(racks.get(dc)), nodeCount, builder, seenRacks);
dcs.put(dc, dcEndpoints);
++dcsToFill;
}
- Iterator<Token> tokenIter = TokenMetadata.ringIterator(tokenMetadata.sortedTokens(), searchToken, false);
+ Iterator<Token> tokenIter = TokenMetadata.ringIterator(sortedTokens, searchToken, false);
while (dcsToFill > 0 && tokenIter.hasNext())
{
Token next = tokenIter.next();
- InetAddress ep = tokenMetadata.getEndpoint(next);
+ InetAddressAndPort ep = tokenMetadata.getEndpoint(next);
Pair<String, String> location = topology.getLocation(ep);
DatacenterEndpoints dcEndpoints = dcs.get(location.left);
- if (dcEndpoints != null && dcEndpoints.addEndpointAndCheckIfDone(ep, location))
+ if (dcEndpoints != null && dcEndpoints.addEndpointAndCheckIfDone(ep, location, replicatedRange))
--dcsToFill;
}
- return new ArrayList<>(replicas);
+ return builder.build();
}
private int sizeOrZero(Multimap<?, ?> collection)
@@ -196,18 +222,15 @@
return collection != null ? collection.size() : 0;
}
- public int getReplicationFactor()
+ public ReplicationFactor getReplicationFactor()
{
- int total = 0;
- for (int repFactor : datacenters.values())
- total += repFactor;
- return total;
+ return aggregateRf;
}
- public int getReplicationFactor(String dc)
+ public ReplicationFactor getReplicationFactor(String dc)
{
- Integer replicas = datacenters.get(dc);
- return replicas == null ? 0 : replicas;
+ ReplicationFactor replicas = datacenters.get(dc);
+ return replicas == null ? ReplicationFactor.ZERO : replicas;
}
public Set<String> getDatacenters()
@@ -215,12 +238,66 @@
return datacenters.keySet();
}
+ public Collection<String> recognizedOptions()
+ {
+ // only valid options are valid DC names.
+ return Datacenters.getValidDatacenters();
+ }
+
+ /**
+ * Support datacenter auto-expansion for CASSANDRA-14303. This hook allows us to safely auto-expand
+ * the "replication_factor" options out into the known datacenters. It is called via reflection from
+ * {@link AbstractReplicationStrategy#prepareReplicationStrategyOptions(Class, Map, Map)}.
+ *
+ * @param options The proposed strategy options that will be potentially mutated
+ * @param previousOptions Any previous strategy options in the case of an ALTER statement
+ */
+ protected static void prepareOptions(Map<String, String> options, Map<String, String> previousOptions)
+ {
+ String replication = options.remove(REPLICATION_FACTOR);
+
+ if (replication == null && options.size() == 0)
+ {
+ // Support direct alters from SimpleStrategy to NTS
+ replication = previousOptions.get(REPLICATION_FACTOR);
+ }
+ else if (replication != null)
+ {
+ // When datacenter auto-expansion occurs in e.g. an ALTER statement (meaning that the previousOptions
+ // map is not empty) we choose not to alter existing datacenter replication levels for safety.
+ previousOptions.entrySet().stream()
+ .filter(e -> !e.getKey().equals(REPLICATION_FACTOR)) // SimpleStrategy conversions
+ .forEach(e -> options.putIfAbsent(e.getKey(), e.getValue()));
+ }
+
+ if (replication != null) {
+ ReplicationFactor defaultReplicas = ReplicationFactor.fromString(replication);
+ Datacenters.getValidDatacenters()
+ .forEach(dc -> options.putIfAbsent(dc, defaultReplicas.toParseableString()));
+ }
+
+ options.values().removeAll(Collections.singleton("0"));
+ }
+
+ protected void validateExpectedOptions() throws ConfigurationException
+ {
+ // Do not accept query with no data centers specified.
+ if (this.configOptions.isEmpty())
+ {
+ throw new ConfigurationException("Configuration for at least one datacenter must be present");
+ }
+
+ // Validate the data center names
+ super.validateExpectedOptions();
+ }
+
public void validateOptions() throws ConfigurationException
{
for (Entry<String, String> e : this.configOptions.entrySet())
{
- if (e.getKey().equalsIgnoreCase("replication_factor"))
- throw new ConfigurationException("replication_factor is an option for SimpleStrategy, not NetworkTopologyStrategy");
+ // prepareOptions should have transformed any "replication_factor" by now
+ if (e.getKey().equalsIgnoreCase(REPLICATION_FACTOR))
+ throw new ConfigurationException(REPLICATION_FACTOR + " should not appear as an option to NetworkTopologyStrategy");
validateReplicationFactor(e.getValue());
}
}
diff --git a/src/java/org/apache/cassandra/locator/OldNetworkTopologyStrategy.java b/src/java/org/apache/cassandra/locator/OldNetworkTopologyStrategy.java
deleted file mode 100644
index b9bd767..0000000
--- a/src/java/org/apache/cassandra/locator/OldNetworkTopologyStrategy.java
+++ /dev/null
@@ -1,122 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.locator;
-
-import java.net.InetAddress;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.dht.Token;
-
-/**
- * This Replication Strategy returns the nodes responsible for a given
- * key but respects rack awareness. It places one replica in a
- * different data center from the first (if there is any such data center),
- * the third replica in a different rack in the first datacenter, and
- * any remaining replicas on the first unused nodes on the ring.
- */
-public class OldNetworkTopologyStrategy extends AbstractReplicationStrategy
-{
- public OldNetworkTopologyStrategy(String keyspaceName, TokenMetadata tokenMetadata, IEndpointSnitch snitch, Map<String, String> configOptions)
- {
- super(keyspaceName, tokenMetadata, snitch, configOptions);
- }
-
- public List<InetAddress> calculateNaturalEndpoints(Token token, TokenMetadata metadata)
- {
- int replicas = getReplicationFactor();
- List<InetAddress> endpoints = new ArrayList<InetAddress>(replicas);
- ArrayList<Token> tokens = metadata.sortedTokens();
-
- if (tokens.isEmpty())
- return endpoints;
-
- Iterator<Token> iter = TokenMetadata.ringIterator(tokens, token, false);
- Token primaryToken = iter.next();
- endpoints.add(metadata.getEndpoint(primaryToken));
-
- boolean bDataCenter = false;
- boolean bOtherRack = false;
- while (endpoints.size() < replicas && iter.hasNext())
- {
- // First try to find one in a different data center
- Token t = iter.next();
- if (!snitch.getDatacenter(metadata.getEndpoint(primaryToken)).equals(snitch.getDatacenter(metadata.getEndpoint(t))))
- {
- // If we have already found something in a diff datacenter no need to find another
- if (!bDataCenter)
- {
- endpoints.add(metadata.getEndpoint(t));
- bDataCenter = true;
- }
- continue;
- }
- // Now try to find one on a different rack
- if (!snitch.getRack(metadata.getEndpoint(primaryToken)).equals(snitch.getRack(metadata.getEndpoint(t))) &&
- snitch.getDatacenter(metadata.getEndpoint(primaryToken)).equals(snitch.getDatacenter(metadata.getEndpoint(t))))
- {
- // If we have already found something in a diff rack no need to find another
- if (!bOtherRack)
- {
- endpoints.add(metadata.getEndpoint(t));
- bOtherRack = true;
- }
- }
-
- }
-
- // If we found N number of nodes we are good. This loop wil just exit. Otherwise just
- // loop through the list and add until we have N nodes.
- if (endpoints.size() < replicas)
- {
- iter = TokenMetadata.ringIterator(tokens, token, false);
- while (endpoints.size() < replicas && iter.hasNext())
- {
- Token t = iter.next();
- if (!endpoints.contains(metadata.getEndpoint(t)))
- endpoints.add(metadata.getEndpoint(t));
- }
- }
-
- return endpoints;
- }
-
- public int getReplicationFactor()
- {
- return Integer.parseInt(this.configOptions.get("replication_factor"));
- }
-
- public void validateOptions() throws ConfigurationException
- {
- if (configOptions == null || configOptions.get("replication_factor") == null)
- {
- throw new ConfigurationException("SimpleStrategy requires a replication_factor strategy option.");
- }
- validateReplicationFactor(configOptions.get("replication_factor"));
- }
-
- public Collection<String> recognizedOptions()
- {
- return Collections.<String>singleton("replication_factor");
- }
-}
diff --git a/src/java/org/apache/cassandra/locator/PendingRangeMaps.java b/src/java/org/apache/cassandra/locator/PendingRangeMaps.java
index cfeccc4..f9e3f66 100644
--- a/src/java/org/apache/cassandra/locator/PendingRangeMaps.java
+++ b/src/java/org/apache/cassandra/locator/PendingRangeMaps.java
@@ -23,167 +23,147 @@
import com.google.common.collect.Iterators;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import org.apache.cassandra.locator.ReplicaCollection.Builder.Conflict;
-import java.net.InetAddress;
import java.util.*;
-public class PendingRangeMaps implements Iterable<Map.Entry<Range<Token>, List<InetAddress>>>
+public class PendingRangeMaps implements Iterable<Map.Entry<Range<Token>, EndpointsForRange.Builder>>
{
- private static final Logger logger = LoggerFactory.getLogger(PendingRangeMaps.class);
-
/**
* We have for NavigableMap to be able to search for ranges containing a token efficiently.
*
* First two are for non-wrap-around ranges, and the last two are for wrap-around ranges.
*/
// ascendingMap will sort the ranges by the ascending order of right token
- final NavigableMap<Range<Token>, List<InetAddress>> ascendingMap;
+ private final NavigableMap<Range<Token>, EndpointsForRange.Builder> ascendingMap;
+
/**
* sorting end ascending, if ends are same, sorting begin descending, so that token (end, end) will
* come before (begin, end] with the same end, and (begin, end) will be selected in the tailMap.
*/
- static final Comparator<Range<Token>> ascendingComparator = new Comparator<Range<Token>>()
- {
- @Override
- public int compare(Range<Token> o1, Range<Token> o2)
- {
- int res = o1.right.compareTo(o2.right);
- if (res != 0)
- return res;
+ private static final Comparator<Range<Token>> ascendingComparator = (o1, o2) -> {
+ int res = o1.right.compareTo(o2.right);
+ if (res != 0)
+ return res;
- return o2.left.compareTo(o1.left);
- }
- };
+ return o2.left.compareTo(o1.left);
+ };
// ascendingMap will sort the ranges by the descending order of left token
- final NavigableMap<Range<Token>, List<InetAddress>> descendingMap;
+ private final NavigableMap<Range<Token>, EndpointsForRange.Builder> descendingMap;
+
/**
* sorting begin descending, if begins are same, sorting end descending, so that token (begin, begin) will
* come after (begin, end] with the same begin, and (begin, end) won't be selected in the tailMap.
*/
- static final Comparator<Range<Token>> descendingComparator = new Comparator<Range<Token>>()
- {
- @Override
- public int compare(Range<Token> o1, Range<Token> o2)
- {
- int res = o2.left.compareTo(o1.left);
- if (res != 0)
- return res;
+ private static final Comparator<Range<Token>> descendingComparator = (o1, o2) -> {
+ int res = o2.left.compareTo(o1.left);
+ if (res != 0)
+ return res;
- // if left tokens are same, sort by the descending of the right tokens.
- return o2.right.compareTo(o1.right);
- }
- };
+ // if left tokens are same, sort by the descending of the right tokens.
+ return o2.right.compareTo(o1.right);
+ };
// these two maps are for warp around ranges.
- final NavigableMap<Range<Token>, List<InetAddress>> ascendingMapForWrapAround;
+ private final NavigableMap<Range<Token>, EndpointsForRange.Builder> ascendingMapForWrapAround;
+
/**
* for wrap around range (begin, end], which begin > end.
* Sorting end ascending, if ends are same, sorting begin ascending,
* so that token (end, end) will come before (begin, end] with the same end, and (begin, end] will be selected in
* the tailMap.
*/
- static final Comparator<Range<Token>> ascendingComparatorForWrapAround = new Comparator<Range<Token>>()
- {
- @Override
- public int compare(Range<Token> o1, Range<Token> o2)
- {
- int res = o1.right.compareTo(o2.right);
- if (res != 0)
- return res;
+ private static final Comparator<Range<Token>> ascendingComparatorForWrapAround = (o1, o2) -> {
+ int res = o1.right.compareTo(o2.right);
+ if (res != 0)
+ return res;
- return o1.left.compareTo(o2.left);
- }
+ return o1.left.compareTo(o2.left);
};
- final NavigableMap<Range<Token>, List<InetAddress>> descendingMapForWrapAround;
+ private final NavigableMap<Range<Token>, EndpointsForRange.Builder> descendingMapForWrapAround;
+
/**
* for wrap around ranges, which begin > end.
* Sorting end ascending, so that token (begin, begin) will come after (begin, end] with the same begin,
* and (begin, end) won't be selected in the tailMap.
*/
- static final Comparator<Range<Token>> descendingComparatorForWrapAround = new Comparator<Range<Token>>()
- {
- @Override
- public int compare(Range<Token> o1, Range<Token> o2)
- {
- int res = o2.left.compareTo(o1.left);
- if (res != 0)
- return res;
- return o1.right.compareTo(o2.right);
- }
+ private static final Comparator<Range<Token>> descendingComparatorForWrapAround = (o1, o2) -> {
+ int res = o2.left.compareTo(o1.left);
+ if (res != 0)
+ return res;
+ return o1.right.compareTo(o2.right);
};
public PendingRangeMaps()
{
- this.ascendingMap = new TreeMap<Range<Token>, List<InetAddress>>(ascendingComparator);
- this.descendingMap = new TreeMap<Range<Token>, List<InetAddress>>(descendingComparator);
- this.ascendingMapForWrapAround = new TreeMap<Range<Token>, List<InetAddress>>(ascendingComparatorForWrapAround);
- this.descendingMapForWrapAround = new TreeMap<Range<Token>, List<InetAddress>>(descendingComparatorForWrapAround);
+ this.ascendingMap = new TreeMap<>(ascendingComparator);
+ this.descendingMap = new TreeMap<>(descendingComparator);
+ this.ascendingMapForWrapAround = new TreeMap<>(ascendingComparatorForWrapAround);
+ this.descendingMapForWrapAround = new TreeMap<>(descendingComparatorForWrapAround);
}
static final void addToMap(Range<Token> range,
- InetAddress address,
- NavigableMap<Range<Token>, List<InetAddress>> ascendingMap,
- NavigableMap<Range<Token>, List<InetAddress>> descendingMap)
+ Replica replica,
+ NavigableMap<Range<Token>, EndpointsForRange.Builder> ascendingMap,
+ NavigableMap<Range<Token>, EndpointsForRange.Builder> descendingMap)
{
- List<InetAddress> addresses = ascendingMap.get(range);
- if (addresses == null)
+ EndpointsForRange.Builder replicas = ascendingMap.get(range);
+ if (replicas == null)
{
- addresses = new ArrayList<InetAddress>(1);
- ascendingMap.put(range, addresses);
- descendingMap.put(range, addresses);
+ replicas = new EndpointsForRange.Builder(range,1);
+ ascendingMap.put(range, replicas);
+ descendingMap.put(range, replicas);
}
- addresses.add(address);
+ replicas.add(replica, Conflict.DUPLICATE);
}
- public void addPendingRange(Range<Token> range, InetAddress address)
+ public void addPendingRange(Range<Token> range, Replica replica)
{
if (Range.isWrapAround(range.left, range.right))
{
- addToMap(range, address, ascendingMapForWrapAround, descendingMapForWrapAround);
+ addToMap(range, replica, ascendingMapForWrapAround, descendingMapForWrapAround);
}
else
{
- addToMap(range, address, ascendingMap, descendingMap);
+ addToMap(range, replica, ascendingMap, descendingMap);
}
}
- static final void addIntersections(Set<InetAddress> endpointsToAdd,
- NavigableMap<Range<Token>, List<InetAddress>> smallerMap,
- NavigableMap<Range<Token>, List<InetAddress>> biggerMap)
+ static final void addIntersections(EndpointsForToken.Builder replicasToAdd,
+ NavigableMap<Range<Token>, EndpointsForRange.Builder> smallerMap,
+ NavigableMap<Range<Token>, EndpointsForRange.Builder> biggerMap)
{
// find the intersection of two sets
for (Range<Token> range : smallerMap.keySet())
{
- List<InetAddress> addresses = biggerMap.get(range);
- if (addresses != null)
+ EndpointsForRange.Builder replicas = biggerMap.get(range);
+ if (replicas != null)
{
- endpointsToAdd.addAll(addresses);
+ replicasToAdd.addAll(replicas);
}
}
}
- public Collection<InetAddress> pendingEndpointsFor(Token token)
+ public EndpointsForToken pendingEndpointsFor(Token token)
{
- Set<InetAddress> endpoints = new HashSet<>();
+ EndpointsForToken.Builder replicas = EndpointsForToken.builder(token);
- Range searchRange = new Range(token, token);
+ Range<Token> searchRange = new Range<>(token, token);
// search for non-wrap-around maps
- NavigableMap<Range<Token>, List<InetAddress>> ascendingTailMap = ascendingMap.tailMap(searchRange, true);
- NavigableMap<Range<Token>, List<InetAddress>> descendingTailMap = descendingMap.tailMap(searchRange, false);
+ NavigableMap<Range<Token>, EndpointsForRange.Builder> ascendingTailMap = ascendingMap.tailMap(searchRange, true);
+ NavigableMap<Range<Token>, EndpointsForRange.Builder> descendingTailMap = descendingMap.tailMap(searchRange, false);
// add intersections of two maps
if (ascendingTailMap.size() < descendingTailMap.size())
{
- addIntersections(endpoints, ascendingTailMap, descendingTailMap);
+ addIntersections(replicas, ascendingTailMap, descendingTailMap);
}
else
{
- addIntersections(endpoints, descendingTailMap, ascendingTailMap);
+ addIntersections(replicas, descendingTailMap, ascendingTailMap);
}
// search for wrap-around sets
@@ -191,29 +171,29 @@
descendingTailMap = descendingMapForWrapAround.tailMap(searchRange, false);
// add them since they are all necessary.
- for (Map.Entry<Range<Token>, List<InetAddress>> entry : ascendingTailMap.entrySet())
+ for (Map.Entry<Range<Token>, EndpointsForRange.Builder> entry : ascendingTailMap.entrySet())
{
- endpoints.addAll(entry.getValue());
+ replicas.addAll(entry.getValue());
}
- for (Map.Entry<Range<Token>, List<InetAddress>> entry : descendingTailMap.entrySet())
+ for (Map.Entry<Range<Token>, EndpointsForRange.Builder> entry : descendingTailMap.entrySet())
{
- endpoints.addAll(entry.getValue());
+ replicas.addAll(entry.getValue());
}
- return endpoints;
+ return replicas.build();
}
public String printPendingRanges()
{
StringBuilder sb = new StringBuilder();
- for (Map.Entry<Range<Token>, List<InetAddress>> entry : this)
+ for (Map.Entry<Range<Token>, EndpointsForRange.Builder> entry : this)
{
Range<Token> range = entry.getKey();
- for (InetAddress address : entry.getValue())
+ for (Replica replica : entry.getValue())
{
- sb.append(address).append(':').append(range);
+ sb.append(replica).append(':').append(range);
sb.append(System.getProperty("line.separator"));
}
}
@@ -222,7 +202,7 @@
}
@Override
- public Iterator<Map.Entry<Range<Token>, List<InetAddress>>> iterator()
+ public Iterator<Map.Entry<Range<Token>, EndpointsForRange.Builder>> iterator()
{
return Iterators.concat(ascendingMap.entrySet().iterator(), ascendingMapForWrapAround.entrySet().iterator());
}
diff --git a/src/java/org/apache/cassandra/locator/PropertyFileSnitch.java b/src/java/org/apache/cassandra/locator/PropertyFileSnitch.java
index 8cc6549..3a9b161 100644
--- a/src/java/org/apache/cassandra/locator/PropertyFileSnitch.java
+++ b/src/java/org/apache/cassandra/locator/PropertyFileSnitch.java
@@ -18,7 +18,6 @@
package org.apache.cassandra.locator;
import java.io.InputStream;
-import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.Arrays;
import java.util.HashMap;
@@ -55,7 +54,7 @@
public static final String SNITCH_PROPERTIES_FILENAME = "cassandra-topology.properties";
private static final int DEFAULT_REFRESH_PERIOD_IN_SECONDS = 5;
- private static volatile Map<InetAddress, String[]> endpointMap;
+ private static volatile Map<InetAddressAndPort, String[]> endpointMap;
private static volatile String[] defaultDCRack;
private volatile boolean gossipStarted;
@@ -93,7 +92,7 @@
* @param endpoint endpoint to process
* @return a array of string with the first index being the data center and the second being the rack
*/
- public static String[] getEndpointInfo(InetAddress endpoint)
+ public static String[] getEndpointInfo(InetAddressAndPort endpoint)
{
String[] rawEndpointInfo = getRawEndpointInfo(endpoint);
if (rawEndpointInfo == null)
@@ -101,7 +100,7 @@
return rawEndpointInfo;
}
- private static String[] getRawEndpointInfo(InetAddress endpoint)
+ private static String[] getRawEndpointInfo(InetAddressAndPort endpoint)
{
String[] value = endpointMap.get(endpoint);
if (value == null)
@@ -118,7 +117,7 @@
* @param endpoint the endpoint to process
* @return string of data center
*/
- public String getDatacenter(InetAddress endpoint)
+ public String getDatacenter(InetAddressAndPort endpoint)
{
String[] info = getEndpointInfo(endpoint);
assert info != null : "No location defined for endpoint " + endpoint;
@@ -131,7 +130,7 @@
* @param endpoint the endpoint to process
* @return string of rack
*/
- public String getRack(InetAddress endpoint)
+ public String getRack(InetAddressAndPort endpoint)
{
String[] info = getEndpointInfo(endpoint);
assert info != null : "No location defined for endpoint " + endpoint;
@@ -140,7 +139,7 @@
public void reloadConfiguration(boolean isUpdate) throws ConfigurationException
{
- HashMap<InetAddress, String[]> reloadedMap = new HashMap<>();
+ HashMap<InetAddressAndPort, String[]> reloadedMap = new HashMap<>();
String[] reloadedDefaultDCRack = null;
Properties properties = new Properties();
@@ -168,11 +167,11 @@
}
else
{
- InetAddress host;
+ InetAddressAndPort host;
String hostString = StringUtils.remove(key, '/');
try
{
- host = InetAddress.getByName(hostString);
+ host = InetAddressAndPort.getByName(hostString);
}
catch (UnknownHostException e)
{
@@ -186,15 +185,15 @@
reloadedMap.put(host, token);
}
}
- InetAddress broadcastAddress = FBUtilities.getBroadcastAddress();
+ InetAddressAndPort broadcastAddress = FBUtilities.getBroadcastAddressAndPort();
String[] localInfo = reloadedMap.get(broadcastAddress);
if (reloadedDefaultDCRack == null && localInfo == null)
throw new ConfigurationException(String.format("Snitch definitions at %s do not define a location for " +
"this node's broadcast address %s, nor does it provides a default",
SNITCH_PROPERTIES_FILENAME, broadcastAddress));
- // OutboundTcpConnectionPool.getEndpoint() converts our broadcast address to local,
+ // internode messaging code converts our broadcast address to local,
// make sure we can be found at that as well.
- InetAddress localAddress = FBUtilities.getLocalAddress();
+ InetAddressAndPort localAddress = FBUtilities.getLocalAddressAndPort();
if (!localAddress.equals(broadcastAddress) && !reloadedMap.containsKey(localAddress))
reloadedMap.put(localAddress, localInfo);
@@ -204,7 +203,7 @@
if (logger.isTraceEnabled())
{
StringBuilder sb = new StringBuilder();
- for (Map.Entry<InetAddress, String[]> entry : reloadedMap.entrySet())
+ for (Map.Entry<InetAddressAndPort, String[]> entry : reloadedMap.entrySet())
sb.append(entry.getKey()).append(':').append(Arrays.toString(entry.getValue())).append(", ");
logger.trace("Loaded network topology from property file: {}", StringUtils.removeEnd(sb.toString(), ", "));
}
@@ -231,17 +230,17 @@
* @param reloadedDefaultDCRack - the default dc:rack or null if no default
* @return true if we can continue updating (no live host had dc or rack updated)
*/
- private static boolean livenessCheck(HashMap<InetAddress, String[]> reloadedMap, String[] reloadedDefaultDCRack)
+ private static boolean livenessCheck(HashMap<InetAddressAndPort, String[]> reloadedMap, String[] reloadedDefaultDCRack)
{
// If the default has changed we must check all live hosts but hopefully we will find a live
// host quickly and interrupt the loop. Otherwise we only check the live hosts that were either
// in the old set or in the new set
- Set<InetAddress> hosts = Arrays.equals(defaultDCRack, reloadedDefaultDCRack)
+ Set<InetAddressAndPort> hosts = Arrays.equals(defaultDCRack, reloadedDefaultDCRack)
? Sets.intersection(StorageService.instance.getLiveRingMembers(), // same default
Sets.union(endpointMap.keySet(), reloadedMap.keySet()))
: StorageService.instance.getLiveRingMembers(); // default updated
- for (InetAddress host : hosts)
+ for (InetAddressAndPort host : hosts)
{
String[] origValue = endpointMap.containsKey(host) ? endpointMap.get(host) : defaultDCRack;
String[] updateValue = reloadedMap.containsKey(host) ? reloadedMap.get(host) : reloadedDefaultDCRack;
diff --git a/src/java/org/apache/cassandra/locator/RackInferringSnitch.java b/src/java/org/apache/cassandra/locator/RackInferringSnitch.java
index a6ea1ab..6ae10cc 100644
--- a/src/java/org/apache/cassandra/locator/RackInferringSnitch.java
+++ b/src/java/org/apache/cassandra/locator/RackInferringSnitch.java
@@ -17,21 +17,19 @@
*/
package org.apache.cassandra.locator;
-import java.net.InetAddress;
-
/**
* A simple endpoint snitch implementation that assumes datacenter and rack information is encoded
* in the 2nd and 3rd octets of the ip address, respectively.
*/
public class RackInferringSnitch extends AbstractNetworkTopologySnitch
{
- public String getRack(InetAddress endpoint)
+ public String getRack(InetAddressAndPort endpoint)
{
- return Integer.toString(endpoint.getAddress()[2] & 0xFF, 10);
+ return Integer.toString(endpoint.address.getAddress()[2] & 0xFF, 10);
}
- public String getDatacenter(InetAddress endpoint)
+ public String getDatacenter(InetAddressAndPort endpoint)
{
- return Integer.toString(endpoint.getAddress()[1] & 0xFF, 10);
+ return Integer.toString(endpoint.address.getAddress()[1] & 0xFF, 10);
}
}
diff --git a/src/java/org/apache/cassandra/locator/RangesAtEndpoint.java b/src/java/org/apache/cassandra/locator/RangesAtEndpoint.java
new file mode 100644
index 0000000..33ddffd
--- /dev/null
+++ b/src/java/org/apache/cassandra/locator/RangesAtEndpoint.java
@@ -0,0 +1,317 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.locator;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableSet;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+
+import java.net.UnknownHostException;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.stream.Collector;
+
+import static com.google.common.collect.Iterables.all;
+import static org.apache.cassandra.locator.ReplicaCollection.Builder.Conflict.*;
+
+/**
+ * A ReplicaCollection for Ranges occurring at an endpoint. All Replica will be for the same endpoint,
+ * and must be unique Ranges (though overlapping ranges are presently permitted, these should probably not be permitted to occur)
+ */
+public class RangesAtEndpoint extends AbstractReplicaCollection<RangesAtEndpoint>
+{
+ private static ReplicaMap<Range<Token>> rangeMap(ReplicaList list) { return new ReplicaMap<>(list, Replica::range); }
+ private static final ReplicaMap<Range<Token>> EMPTY_MAP = rangeMap(EMPTY_LIST);
+
+ private final InetAddressAndPort endpoint;
+
+ // volatile not needed, as all of these caching collections have final members,
+ // besides (transitively) those that cache objects that themselves have only final members
+ private ReplicaMap<Range<Token>> byRange;
+ private RangesAtEndpoint onlyFull;
+ private RangesAtEndpoint onlyTransient;
+
+ private RangesAtEndpoint(InetAddressAndPort endpoint, ReplicaList list, ReplicaMap<Range<Token>> byRange)
+ {
+ super(list);
+ this.endpoint = endpoint;
+ this.byRange = byRange;
+ assert endpoint != null;
+ }
+
+ public InetAddressAndPort endpoint()
+ {
+ return endpoint;
+ }
+
+ @Override
+ public Set<InetAddressAndPort> endpoints()
+ {
+ return Collections.unmodifiableSet(list.isEmpty()
+ ? Collections.emptySet()
+ : Collections.singleton(endpoint)
+ );
+ }
+
+ /**
+ * @return a set of all unique Ranges
+ * This method is threadsafe, though it is not synchronised
+ */
+ public Set<Range<Token>> ranges()
+ {
+ return byRange().keySet();
+ }
+
+ /**
+ * @return a map of all Ranges, to their owning Replica instance
+ * This method is threadsafe, though it is not synchronised
+ */
+ public Map<Range<Token>, Replica> byRange()
+ {
+ ReplicaMap<Range<Token>> map = byRange;
+ if (map == null)
+ byRange = map = rangeMap(list);
+ return map;
+ }
+
+ @Override
+ protected RangesAtEndpoint snapshot(ReplicaList newList)
+ {
+ if (newList.isEmpty()) return empty(endpoint);
+ ReplicaMap<Range<Token>> byRange = null;
+ if (this.byRange != null && list.isSubList(newList))
+ byRange = this.byRange.forSubList(newList);
+ return new RangesAtEndpoint(endpoint, newList, byRange);
+ }
+
+ @Override
+ public RangesAtEndpoint snapshot()
+ {
+ return this;
+ }
+
+ @Override
+ public ReplicaCollection.Builder<RangesAtEndpoint> newBuilder(int initialCapacity)
+ {
+ return new Builder(endpoint, initialCapacity);
+ }
+
+ @Override
+ public boolean contains(Replica replica)
+ {
+ return replica != null
+ && Objects.equals(
+ byRange().get(replica.range()),
+ replica);
+ }
+
+ public RangesAtEndpoint onlyFull()
+ {
+ RangesAtEndpoint result = onlyFull;
+ if (onlyFull == null)
+ onlyFull = result = filter(Replica::isFull);
+ return result;
+ }
+
+ public RangesAtEndpoint onlyTransient()
+ {
+ RangesAtEndpoint result = onlyTransient;
+ if (onlyTransient == null)
+ onlyTransient = result = filter(Replica::isTransient);
+ return result;
+ }
+
+ public boolean contains(Range<Token> range, boolean isFull)
+ {
+ Replica replica = byRange().get(range);
+ return replica != null && replica.isFull() == isFull;
+ }
+
+ /**
+ * @return if there are no wrap around ranges contained in this RangesAtEndpoint, return self;
+ * otherwise, return a RangesAtEndpoint covering the same logical portions of the ring, but with those ranges unwrapped
+ */
+ public RangesAtEndpoint unwrap()
+ {
+ int wrapAroundCount = 0;
+ for (Replica replica : this)
+ {
+ if (replica.range().isWrapAround())
+ ++wrapAroundCount;
+ }
+
+ assert wrapAroundCount <= 1;
+ if (wrapAroundCount == 0)
+ return snapshot();
+
+ RangesAtEndpoint.Builder builder = builder(endpoint, size() + wrapAroundCount);
+ for (Replica replica : this)
+ {
+ if (!replica.range().isWrapAround())
+ {
+ builder.add(replica);
+ continue;
+ }
+ for (Range<Token> range : replica.range().unwrap())
+ builder.add(replica.decorateSubrange(range));
+ }
+ return builder.build();
+ }
+
+ public static Collector<Replica, Builder, RangesAtEndpoint> collector(InetAddressAndPort endpoint)
+ {
+ return collector(ImmutableSet.of(), () -> new Builder(endpoint));
+ }
+
+ public static class Builder extends RangesAtEndpoint implements ReplicaCollection.Builder<RangesAtEndpoint>
+ {
+ boolean built;
+ public Builder(InetAddressAndPort endpoint) { this(endpoint, 0); }
+ public Builder(InetAddressAndPort endpoint, int capacity) { this(endpoint, new ReplicaList(capacity)); }
+ private Builder(InetAddressAndPort endpoint, ReplicaList list) { super(endpoint, list, rangeMap(list)); }
+
+ public RangesAtEndpoint.Builder add(Replica replica, Conflict ignoreConflict)
+ {
+ if (built) throw new IllegalStateException();
+ Preconditions.checkNotNull(replica);
+ if (!Objects.equals(super.endpoint, replica.endpoint()))
+ throw new IllegalArgumentException("Replica " + replica + " has incorrect endpoint (expected " + super.endpoint + ")");
+
+ if (!super.byRange.internalPutIfAbsent(replica, list.size()))
+ {
+ switch (ignoreConflict)
+ {
+ case DUPLICATE:
+ if (byRange().get(replica.range()).equals(replica))
+ break;
+ case NONE:
+ throw new IllegalArgumentException("Conflicting replica added (expected unique ranges): "
+ + replica + "; existing: " + byRange().get(replica.range()));
+ case ALL:
+ }
+ return this;
+ }
+
+ list.add(replica);
+ return this;
+ }
+
+ @Override
+ public RangesAtEndpoint snapshot()
+ {
+ return snapshot(list.subList(0, list.size()));
+ }
+
+ public RangesAtEndpoint build()
+ {
+ built = true;
+ return new RangesAtEndpoint(super.endpoint, super.list, super.byRange);
+ }
+ }
+
+ public static Builder builder(InetAddressAndPort endpoint)
+ {
+ return new Builder(endpoint);
+ }
+ public static Builder builder(InetAddressAndPort endpoint, int capacity)
+ {
+ return new Builder(endpoint, capacity);
+ }
+
+ public static RangesAtEndpoint empty(InetAddressAndPort endpoint)
+ {
+ return new RangesAtEndpoint(endpoint, EMPTY_LIST, EMPTY_MAP);
+ }
+
+ public static RangesAtEndpoint of(Replica replica)
+ {
+ ReplicaList one = new ReplicaList(1);
+ one.add(replica);
+ return new RangesAtEndpoint(replica.endpoint(), one, rangeMap(one));
+ }
+
+ public static RangesAtEndpoint of(Replica ... replicas)
+ {
+ return copyOf(Arrays.asList(replicas));
+ }
+
+ public static RangesAtEndpoint copyOf(List<Replica> replicas)
+ {
+ if (replicas.isEmpty())
+ throw new IllegalArgumentException("Must specify a non-empty collection of replicas");
+ return builder(replicas.get(0).endpoint(), replicas.size()).addAll(replicas).build();
+ }
+
+
+ /**
+ * Use of this method to synthesize Replicas is almost always wrong. In repair it turns out the concerns of transient
+ * vs non-transient are handled at a higher level, but eventually repair needs to ask streaming to actually move
+ * the data and at that point it doesn't have a great handle on what the replicas are and it doesn't really matter.
+ *
+ * Streaming expects to be given Replicas with each replica indicating what type of data (transient or not transient)
+ * should be sent.
+ *
+ * So in this one instance we can lie to streaming and pretend all the replicas are full and use a dummy address
+ * and it doesn't matter because streaming doesn't rely on the address for anything other than debugging and full
+ * is a valid value for transientness because streaming is selecting candidate tables from the repair/unrepaired
+ * set already.
+ * @param ranges
+ * @return
+ */
+ @VisibleForTesting
+ public static RangesAtEndpoint toDummyList(Collection<Range<Token>> ranges)
+ {
+ InetAddressAndPort dummy;
+ try
+ {
+ dummy = InetAddressAndPort.getByNameOverrideDefaults("0.0.0.0", 0);
+ }
+ catch (UnknownHostException e)
+ {
+ throw new RuntimeException(e);
+ }
+
+ //For repair we are less concerned with full vs transient since repair is already dealing with those concerns.
+ //Always say full and then if the repair is incremental or not will determine what is streamed.
+ return ranges.stream()
+ .map(range -> new Replica(dummy, range, true))
+ .collect(collector(dummy));
+ }
+
+ public static boolean isDummyList(RangesAtEndpoint ranges)
+ {
+ return all(ranges, range -> range.endpoint().getHostAddress(true).equals("0.0.0.0:0"));
+ }
+
+ /**
+ * @return concatenate two DISJOINT collections together
+ */
+ public static RangesAtEndpoint concat(RangesAtEndpoint replicas, RangesAtEndpoint extraReplicas)
+ {
+ return AbstractReplicaCollection.concat(replicas, extraReplicas, NONE);
+ }
+
+}
diff --git a/src/java/org/apache/cassandra/locator/RangesByEndpoint.java b/src/java/org/apache/cassandra/locator/RangesByEndpoint.java
new file mode 100644
index 0000000..1a71141
--- /dev/null
+++ b/src/java/org/apache/cassandra/locator/RangesByEndpoint.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.locator;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Maps;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+public class RangesByEndpoint extends ReplicaMultimap<InetAddressAndPort, RangesAtEndpoint>
+{
+ public RangesByEndpoint(Map<InetAddressAndPort, RangesAtEndpoint> map)
+ {
+ super(map);
+ }
+
+ public RangesAtEndpoint get(InetAddressAndPort endpoint)
+ {
+ Preconditions.checkNotNull(endpoint);
+ return map.getOrDefault(endpoint, RangesAtEndpoint.empty(endpoint));
+ }
+
+ public static class Builder extends ReplicaMultimap.Builder<InetAddressAndPort, RangesAtEndpoint.Builder>
+ {
+ @Override
+ protected RangesAtEndpoint.Builder newBuilder(InetAddressAndPort endpoint)
+ {
+ return new RangesAtEndpoint.Builder(endpoint);
+ }
+
+ public RangesByEndpoint build()
+ {
+ return new RangesByEndpoint(
+ ImmutableMap.copyOf(
+ Maps.transformValues(this.map, RangesAtEndpoint.Builder::build)));
+ }
+ }
+
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/locator/ReconnectableSnitchHelper.java b/src/java/org/apache/cassandra/locator/ReconnectableSnitchHelper.java
index a6bec0c..dea8c76 100644
--- a/src/java/org/apache/cassandra/locator/ReconnectableSnitchHelper.java
+++ b/src/java/org/apache/cassandra/locator/ReconnectableSnitchHelper.java
@@ -18,14 +18,20 @@
package org.apache.cassandra.locator;
-import java.net.InetAddress;
import java.net.UnknownHostException;
+import com.google.common.annotations.VisibleForTesting;
+
import org.apache.cassandra.gms.*;
+import org.apache.cassandra.net.ConnectionCategory;
import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.net.OutboundConnectionSettings;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static org.apache.cassandra.net.ConnectionType.SMALL_MESSAGES;
+
/**
* Sidekick helper for snitches that want to reconnect from one IP addr for a node to another.
* Typically, this is for situations like EC2 where a node will have a public address and a private address,
@@ -45,11 +51,11 @@
this.preferLocal = preferLocal;
}
- private void reconnect(InetAddress publicAddress, VersionedValue localAddressValue)
+ private void reconnect(InetAddressAndPort publicAddress, VersionedValue localAddressValue)
{
try
{
- reconnect(publicAddress, InetAddress.getByName(localAddressValue.value));
+ reconnect(publicAddress, InetAddressAndPort.getByName(localAddressValue.value), snitch, localDc);
}
catch (UnknownHostException e)
{
@@ -57,50 +63,81 @@
}
}
- private void reconnect(InetAddress publicAddress, InetAddress localAddress)
+ @VisibleForTesting
+ static void reconnect(InetAddressAndPort publicAddress, InetAddressAndPort localAddress, IEndpointSnitch snitch, String localDc)
{
- if (snitch.getDatacenter(publicAddress).equals(localDc)
- && !MessagingService.instance().getConnectionPool(publicAddress).endPoint().equals(localAddress))
+ if (!new OutboundConnectionSettings(publicAddress, localAddress).withDefaults(ConnectionCategory.MESSAGING).authenticate())
{
- MessagingService.instance().getConnectionPool(publicAddress).reset(localAddress);
+ logger.debug("InternodeAuthenticator said don't reconnect to {} on {}", publicAddress, localAddress);
+ return;
+ }
+
+ if (snitch.getDatacenter(publicAddress).equals(localDc))
+ {
+ MessagingService.instance().maybeReconnectWithNewIp(publicAddress, localAddress);
logger.debug("Initiated reconnect to an Internal IP {} for the {}", localAddress, publicAddress);
}
}
- public void beforeChange(InetAddress endpoint, EndpointState currentState, ApplicationState newStateKey, VersionedValue newValue)
+ public void beforeChange(InetAddressAndPort endpoint, EndpointState currentState, ApplicationState newStateKey, VersionedValue newValue)
{
// no-op
}
- public void onJoin(InetAddress endpoint, EndpointState epState)
+ public void onJoin(InetAddressAndPort endpoint, EndpointState epState)
{
- if (preferLocal && !Gossiper.instance.isDeadState(epState) && epState.getApplicationState(ApplicationState.INTERNAL_IP) != null)
- reconnect(endpoint, epState.getApplicationState(ApplicationState.INTERNAL_IP));
+ if (preferLocal && !Gossiper.instance.isDeadState(epState))
+ {
+ VersionedValue address = epState.getApplicationState(ApplicationState.INTERNAL_ADDRESS_AND_PORT);
+ if (address == null)
+ {
+ address = epState.getApplicationState(ApplicationState.INTERNAL_ADDRESS_AND_PORT);
+ }
+ if (address != null)
+ {
+ reconnect(endpoint, address);
+ }
+ }
}
- public void onChange(InetAddress endpoint, ApplicationState state, VersionedValue value)
+ //Skeptical this will always do the right thing all the time port wise. It will converge on the right thing
+ //eventually once INTERNAL_ADDRESS_AND_PORT is populated
+ public void onChange(InetAddressAndPort endpoint, ApplicationState state, VersionedValue value)
{
- if (preferLocal && state == ApplicationState.INTERNAL_IP && !Gossiper.instance.isDeadState(Gossiper.instance.getEndpointStateForEndpoint(endpoint)))
- reconnect(endpoint, value);
+ if (preferLocal && !Gossiper.instance.isDeadState(Gossiper.instance.getEndpointStateForEndpoint(endpoint)))
+ {
+ if (state == ApplicationState.INTERNAL_ADDRESS_AND_PORT)
+ {
+ reconnect(endpoint, value);
+ }
+ else if (state == ApplicationState.INTERNAL_IP &&
+ null == Gossiper.instance.getEndpointStateForEndpoint(endpoint).getApplicationState(ApplicationState.INTERNAL_ADDRESS_AND_PORT))
+ {
+ //Only use INTERNAL_IP if INTERNAL_ADDRESS_AND_PORT is unavailable
+ reconnect(endpoint, value);
+ }
+ }
}
- public void onAlive(InetAddress endpoint, EndpointState state)
+ public void onAlive(InetAddressAndPort endpoint, EndpointState state)
{
- if (preferLocal && state.getApplicationState(ApplicationState.INTERNAL_IP) != null)
- reconnect(endpoint, state.getApplicationState(ApplicationState.INTERNAL_IP));
+ VersionedValue internalIP = state.getApplicationState(ApplicationState.INTERNAL_IP);
+ VersionedValue internalIPAndPorts = state.getApplicationState(ApplicationState.INTERNAL_ADDRESS_AND_PORT);
+ if (preferLocal && internalIP != null)
+ reconnect(endpoint, internalIPAndPorts != null ? internalIPAndPorts : internalIP);
}
- public void onDead(InetAddress endpoint, EndpointState state)
+ public void onDead(InetAddressAndPort endpoint, EndpointState state)
{
// do nothing.
}
- public void onRemove(InetAddress endpoint)
+ public void onRemove(InetAddressAndPort endpoint)
{
// do nothing.
}
- public void onRestart(InetAddress endpoint, EndpointState state)
+ public void onRestart(InetAddressAndPort endpoint, EndpointState state)
{
// do nothing.
}
diff --git a/src/java/org/apache/cassandra/locator/Replica.java b/src/java/org/apache/cassandra/locator/Replica.java
new file mode 100644
index 0000000..4c5f7c6
--- /dev/null
+++ b/src/java/org/apache/cassandra/locator/Replica.java
@@ -0,0 +1,196 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.locator;
+
+import java.util.Objects;
+import java.util.Set;
+
+import com.google.common.base.Preconditions;
+
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.utils.FBUtilities;
+
+/**
+ * A Replica represents an owning node for a copy of a portion of the token ring.
+ *
+ * It consists of:
+ * - the logical token range that is being replicated (i.e. for the first logical replica only, this will be equal
+ * to one of its owned portions of the token ring; all other replicas will have this token range also)
+ * - an endpoint (IP and port)
+ * - whether the range is replicated in full, or transiently (CASSANDRA-14404)
+ *
+ * In general, it is preferred to use a Replica to a Range<Token>, particularly when users of the concept depend on
+ * knowledge of the full/transient status of the copy.
+ *
+ * That means you should avoid unwrapping and rewrapping these things and think hard about subtraction
+ * and such and what the result is WRT to transientness. Definitely avoid creating fake Replicas with misinformation
+ * about endpoints, ranges, or transientness.
+ */
+public final class Replica implements Comparable<Replica>
+{
+ private final Range<Token> range;
+ private final InetAddressAndPort endpoint;
+ private final boolean full;
+
+ public Replica(InetAddressAndPort endpoint, Range<Token> range, boolean full)
+ {
+ Preconditions.checkNotNull(endpoint);
+ Preconditions.checkNotNull(range);
+ this.endpoint = endpoint;
+ this.range = range;
+ this.full = full;
+ }
+
+ public Replica(InetAddressAndPort endpoint, Token start, Token end, boolean full)
+ {
+ this(endpoint, new Range<>(start, end), full);
+ }
+
+ public boolean equals(Object o)
+ {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ Replica replica = (Replica) o;
+ return full == replica.full &&
+ Objects.equals(endpoint, replica.endpoint) &&
+ Objects.equals(range, replica.range);
+ }
+
+ @Override
+ public int compareTo(Replica o)
+ {
+ int c = range.compareTo(o.range);
+ if (c == 0)
+ c = endpoint.compareTo(o.endpoint);
+ if (c == 0)
+ c = Boolean.compare(full, o.full);
+ return c;
+ }
+
+ public int hashCode()
+ {
+ return Objects.hash(endpoint, range, full);
+ }
+
+ @Override
+ public String toString()
+ {
+ return (full ? "Full" : "Transient") + '(' + endpoint() + ',' + range + ')';
+ }
+
+ public final InetAddressAndPort endpoint()
+ {
+ return endpoint;
+ }
+
+ public boolean isSelf()
+ {
+ return endpoint.equals(FBUtilities.getBroadcastAddressAndPort());
+ }
+
+ public Range<Token> range()
+ {
+ return range;
+ }
+
+ public final boolean isFull()
+ {
+ return full;
+ }
+
+ public final boolean isTransient()
+ {
+ return !isFull();
+ }
+
+ /**
+ * This is used exclusively in TokenMetadata to check if a portion of a range is already replicated
+ * by an endpoint so that we only mark as pending the portion that is either not replicated sufficiently (transient
+ * when we need full) or at all.
+ *
+ * If it's not replicated at all it needs to be pending because there is no data.
+ * If it's replicated but only transiently and we need to replicate it fully it must be marked as pending until it
+ * is available fully otherwise a read might treat this replica as full and not read from a full replica that has
+ * the data.
+ */
+ public RangesAtEndpoint subtractSameReplication(RangesAtEndpoint toSubtract)
+ {
+ Set<Range<Token>> subtractedRanges = range().subtractAll(toSubtract.filter(r -> r.isFull() == isFull()).ranges());
+ RangesAtEndpoint.Builder result = RangesAtEndpoint.builder(endpoint, subtractedRanges.size());
+ for (Range<Token> range : subtractedRanges)
+ {
+ result.add(decorateSubrange(range));
+ }
+ return result.build();
+ }
+
+ /**
+ * Don't use this method and ignore transient status unless you are explicitly handling it outside this method.
+ *
+ * This helper method is used by StorageService.calculateStreamAndFetchRanges to perform subtraction.
+ * It ignores transient status because it's already being handled in calculateStreamAndFetchRanges.
+ */
+ public RangesAtEndpoint subtractIgnoreTransientStatus(Range<Token> subtract)
+ {
+ Set<Range<Token>> ranges = this.range.subtract(subtract);
+ RangesAtEndpoint.Builder result = RangesAtEndpoint.builder(endpoint, ranges.size());
+ for (Range<Token> subrange : ranges)
+ result.add(decorateSubrange(subrange));
+ return result.build();
+ }
+
+ public boolean contains(Range<Token> that)
+ {
+ return range().contains(that);
+ }
+
+ public boolean intersectsOnRange(Replica replica)
+ {
+ return range().intersects(replica.range());
+ }
+
+ public Replica decorateSubrange(Range<Token> subrange)
+ {
+ Preconditions.checkArgument(range.contains(subrange));
+ return new Replica(endpoint(), subrange, isFull());
+ }
+
+ public static Replica fullReplica(InetAddressAndPort endpoint, Range<Token> range)
+ {
+ return new Replica(endpoint, range, true);
+ }
+
+ public static Replica fullReplica(InetAddressAndPort endpoint, Token start, Token end)
+ {
+ return fullReplica(endpoint, new Range<>(start, end));
+ }
+
+ public static Replica transientReplica(InetAddressAndPort endpoint, Range<Token> range)
+ {
+ return new Replica(endpoint, range, false);
+ }
+
+ public static Replica transientReplica(InetAddressAndPort endpoint, Token start, Token end)
+ {
+ return transientReplica(endpoint, new Range<>(start, end));
+ }
+
+}
+
diff --git a/src/java/org/apache/cassandra/locator/ReplicaCollection.java b/src/java/org/apache/cassandra/locator/ReplicaCollection.java
new file mode 100644
index 0000000..ec671d5
--- /dev/null
+++ b/src/java/org/apache/cassandra/locator/ReplicaCollection.java
@@ -0,0 +1,170 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.locator;
+
+import java.util.Comparator;
+import java.util.Iterator;
+import java.util.Set;
+import java.util.function.Predicate;
+import java.util.stream.Stream;
+
+/**
+ * A collection like class for Replica objects. Represents both a well defined order on the contained Replica objects,
+ * and efficient methods for accessing the contained Replicas, directly and as a projection onto their endpoints and ranges.
+ */
+public interface ReplicaCollection<C extends ReplicaCollection<C>> extends Iterable<Replica>
+{
+ /**
+ * @return a Set of the endpoints of the contained Replicas.
+ * Iteration order is maintained where there is a 1:1 relationship between endpoint and Replica
+ * Typically this collection offers O(1) access methods, and this is true for all but ReplicaList.
+ */
+ public abstract Set<InetAddressAndPort> endpoints();
+
+ /**
+ * @param i a value in the range [0..size())
+ * @return the i'th Replica, in our iteration order
+ */
+ public abstract Replica get(int i);
+
+ /**
+ * @return the number of Replica contained
+ */
+ public abstract int size();
+
+ /**
+ * @return true iff size() == 0
+ */
+ public abstract boolean isEmpty();
+
+ /**
+ * @return true iff a Replica in this collection is equal to the provided Replica.
+ * Typically this method is expected to take O(1) time, and this is true for all but ReplicaList.
+ */
+ public abstract boolean contains(Replica replica);
+
+ /**
+ * @return the number of replicas that match the predicate
+ */
+ public abstract int count(Predicate<Replica> predicate);
+
+ /**
+ * @return a *eagerly constructed* copy of this collection containing the Replica that match the provided predicate.
+ * An effort will be made to either return ourself, or a subList, where possible.
+ * It is guaranteed that no changes to any upstream Builder will affect the state of the result.
+ */
+ public abstract C filter(Predicate<Replica> predicate);
+
+ /**
+ * @return a *eagerly constructed* copy of this collection containing the Replica that match the provided predicate.
+ * An effort will be made to either return ourself, or a subList, where possible.
+ * It is guaranteed that no changes to any upstream Builder will affect the state of the result.
+ * Only the first maxSize items will be returned.
+ */
+ public abstract C filter(Predicate<Replica> predicate, int maxSize);
+
+ /**
+ * @return a *lazily constructed* Iterable over this collection, containing the Replica that match the provided predicate.
+ */
+ public abstract Iterable<Replica> filterLazily(Predicate<Replica> predicate);
+
+ /**
+ * @return a *lazily constructed* Iterable over this collection, containing the Replica that match the provided predicate.
+ * Only the first maxSize matching items will be returned.
+ */
+ public abstract Iterable<Replica> filterLazily(Predicate<Replica> predicate, int maxSize);
+
+ /**
+ * @return an *eagerly constructed* copy of this collection containing the Replica at positions [start..end);
+ * An effort will be made to either return ourself, or a subList, where possible.
+ * It is guaranteed that no changes to any upstream Builder will affect the state of the result.
+ */
+ public abstract C subList(int start, int end);
+
+ /**
+ * @return an *eagerly constructed* copy of this collection containing the Replica re-ordered according to this comparator
+ * It is guaranteed that no changes to any upstream Builder will affect the state of the result.
+ */
+ public abstract C sorted(Comparator<Replica> comparator);
+
+ public abstract Iterator<Replica> iterator();
+ public abstract Stream<Replica> stream();
+
+ public abstract boolean equals(Object o);
+ public abstract int hashCode();
+ public abstract String toString();
+
+ /**
+ * A mutable (append-only) extension of a ReplicaCollection.
+ * All methods besides add() will return an immutable snapshot of the collection, or the matching items.
+ */
+ public interface Builder<C extends ReplicaCollection<C>> extends ReplicaCollection<C>
+ {
+ /**
+ * @return an Immutable clone that assumes this Builder will never be modified again,
+ * so its contents can be reused.
+ *
+ * This Builder should enforce that it is no longer modified.
+ */
+ public C build();
+
+ /**
+ * @return an Immutable clone that assumes this Builder will be modified again
+ */
+ public C snapshot();
+
+ /**
+ * Passed to add() and addAll() as ignoreConflicts parameter. The meaning of conflict varies by collection type
+ * (for Endpoints, it is a duplicate InetAddressAndPort; for RangesAtEndpoint it is a duplicate Range).
+ */
+ enum Conflict
+ {
+ /** fail on addition of any such conflict */
+ NONE,
+ /** fail on addition of any such conflict where the contents differ (first occurrence and position wins) */
+ DUPLICATE,
+ /** ignore all conflicts (the first occurrence and position wins) */
+ ALL
+ }
+
+ /**
+ * @param replica add this replica to the end of the collection
+ * @param ignoreConflict conflicts to ignore, see {@link Conflict}
+ */
+ Builder<C> add(Replica replica, Conflict ignoreConflict);
+
+ default public Builder<C> add(Replica replica)
+ {
+ return add(replica, Conflict.NONE);
+ }
+
+ default public Builder<C> addAll(Iterable<Replica> replicas, Conflict ignoreConflicts)
+ {
+ for (Replica replica : replicas)
+ add(replica, ignoreConflicts);
+ return this;
+ }
+
+ default public Builder<C> addAll(Iterable<Replica> replicas)
+ {
+ return addAll(replicas, Conflict.NONE);
+ }
+ }
+
+}
diff --git a/src/java/org/apache/cassandra/locator/ReplicaLayout.java b/src/java/org/apache/cassandra/locator/ReplicaLayout.java
new file mode 100644
index 0000000..d44fdd7
--- /dev/null
+++ b/src/java/org/apache/cassandra/locator/ReplicaLayout.java
@@ -0,0 +1,339 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.locator;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.gms.FailureDetector;
+import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.utils.FBUtilities;
+
+import java.util.Set;
+import java.util.function.Predicate;
+
+/**
+ * The relevant replicas for an operation over a given range or token.
+ *
+ * @param <E>
+ */
+public abstract class ReplicaLayout<E extends Endpoints<E>>
+{
+ private final E natural;
+
+ ReplicaLayout(E natural)
+ {
+ this.natural = natural;
+ }
+
+ /**
+ * The 'natural' owners of the ring position(s), as implied by the current ring layout.
+ * This excludes any pending owners, i.e. those that are in the process of taking ownership of a range, but
+ * have not yet finished obtaining their view of the range.
+ */
+ public final E natural()
+ {
+ return natural;
+ }
+
+ /**
+ * All relevant owners of the ring position(s) for this operation, as implied by the current ring layout.
+ * For writes, this will include pending owners, and for reads it will be equivalent to natural()
+ */
+ public E all()
+ {
+ return natural;
+ }
+
+ public String toString()
+ {
+ return "ReplicaLayout [ natural: " + natural + " ]";
+ }
+
+ public static class ForTokenRead extends ReplicaLayout<EndpointsForToken> implements ForToken
+ {
+ public ForTokenRead(EndpointsForToken natural)
+ {
+ super(natural);
+ }
+
+ @Override
+ public Token token()
+ {
+ return natural().token();
+ }
+
+ public ReplicaLayout.ForTokenRead filter(Predicate<Replica> filter)
+ {
+ EndpointsForToken filtered = natural().filter(filter);
+ // AbstractReplicaCollection.filter returns itself if all elements match the filter
+ if (filtered == natural()) return this;
+ return new ReplicaLayout.ForTokenRead(filtered);
+ }
+ }
+
+ public static class ForRangeRead extends ReplicaLayout<EndpointsForRange> implements ForRange
+ {
+ final AbstractBounds<PartitionPosition> range;
+
+ public ForRangeRead(AbstractBounds<PartitionPosition> range, EndpointsForRange natural)
+ {
+ super(natural);
+ this.range = range;
+ }
+
+ @Override
+ public AbstractBounds<PartitionPosition> range()
+ {
+ return range;
+ }
+
+ public ReplicaLayout.ForRangeRead filter(Predicate<Replica> filter)
+ {
+ EndpointsForRange filtered = natural().filter(filter);
+ // AbstractReplicaCollection.filter returns itself if all elements match the filter
+ if (filtered == natural()) return this;
+ return new ReplicaLayout.ForRangeRead(range(), filtered);
+ }
+ }
+
+ public static class ForWrite<E extends Endpoints<E>> extends ReplicaLayout<E>
+ {
+ final E all;
+ final E pending;
+
+ ForWrite(E natural, E pending, E all)
+ {
+ super(natural);
+ assert pending != null && !haveWriteConflicts(natural, pending);
+ if (all == null)
+ all = Endpoints.concat(natural, pending);
+ this.all = all;
+ this.pending = pending;
+ }
+
+ public final E all()
+ {
+ return all;
+ }
+
+ public final E pending()
+ {
+ return pending;
+ }
+
+ public String toString()
+ {
+ return "ReplicaLayout [ natural: " + natural() + ", pending: " + pending + " ]";
+ }
+ }
+
+ public static class ForTokenWrite extends ForWrite<EndpointsForToken> implements ForToken
+ {
+ public ForTokenWrite(EndpointsForToken natural, EndpointsForToken pending)
+ {
+ this(natural, pending, null);
+ }
+ public ForTokenWrite(EndpointsForToken natural, EndpointsForToken pending, EndpointsForToken all)
+ {
+ super(natural, pending, all);
+ }
+
+ @Override
+ public Token token() { return natural().token(); }
+
+ public ReplicaLayout.ForTokenWrite filter(Predicate<Replica> filter)
+ {
+ EndpointsForToken filtered = all().filter(filter);
+ // AbstractReplicaCollection.filter returns itself if all elements match the filter
+ if (filtered == all()) return this;
+ // unique by endpoint, so can for efficiency filter only on endpoint
+ return new ReplicaLayout.ForTokenWrite(
+ natural().keep(filtered.endpoints()),
+ pending().keep(filtered.endpoints()),
+ filtered
+ );
+ }
+ }
+
+ public interface ForRange
+ {
+ public AbstractBounds<PartitionPosition> range();
+ }
+
+ public interface ForToken
+ {
+ public Token token();
+ }
+
+ /**
+ * Gets the 'natural' and 'pending' replicas that own a given token, with no filtering or processing.
+ *
+ * Since a write is intended for all nodes (except, unless necessary, transient replicas), this method's
+ * only responsibility is to fetch the 'natural' and 'pending' replicas, then resolve any conflicts
+ * {@link ReplicaLayout#haveWriteConflicts(Endpoints, Endpoints)}
+ */
+ public static ReplicaLayout.ForTokenWrite forTokenWriteLiveAndDown(Keyspace keyspace, Token token)
+ {
+ // TODO: these should be cached, not the natural replicas
+ // TODO: race condition to fetch these. implications??
+ EndpointsForToken natural = keyspace.getReplicationStrategy().getNaturalReplicasForToken(token);
+ EndpointsForToken pending = StorageService.instance.getTokenMetadata().pendingEndpointsForToken(token, keyspace.getName());
+ return forTokenWrite(natural, pending);
+ }
+
+ public static ReplicaLayout.ForTokenWrite forTokenWrite(EndpointsForToken natural, EndpointsForToken pending)
+ {
+ if (haveWriteConflicts(natural, pending))
+ {
+ natural = resolveWriteConflictsInNatural(natural, pending);
+ pending = resolveWriteConflictsInPending(natural, pending);
+ }
+ return new ReplicaLayout.ForTokenWrite(natural, pending);
+ }
+
+ /**
+ * Detect if we have any endpoint in both pending and full; this can occur either due to races (there is no isolation)
+ * or because an endpoint is transitioning between full and transient replication status.
+ *
+ * We essentially always prefer the full version for writes, because this is stricter.
+ *
+ * For transient->full transitions:
+ *
+ * Since we always write to any pending transient replica, effectively upgrading it to full for the transition duration,
+ * it might at first seem to be OK to continue treating the conflict replica as its 'natural' transient form,
+ * as there is always a quorum of nodes receiving the write. However, ring ownership changes are not atomic or
+ * consistent across the cluster, and it is possible for writers to see different ring states.
+ *
+ * Furthermore, an operator would expect that the full node has received all writes, with no extra need for repair
+ * (as the normal contract dictates) when it completes its transition.
+ *
+ * While we cannot completely eliminate risks due to ring inconsistencies, this approach is the most conservative
+ * available to us today to mitigate, and (we think) the easiest to reason about.
+ *
+ * For full->transient transitions:
+ *
+ * In this case, things are dicier, because in theory we can trigger this change instantly. All we need to do is
+ * drop some data, surely?
+ *
+ * Ring movements can put us in a pickle; any other node could believe us to be full when we have become transient,
+ * and perform a full data request to us that we believe ourselves capable of answering, but that we are not.
+ * If the ring is inconsistent, it's even feasible that a transient request would be made to the node that is losing
+ * its transient status, that also does not know it has yet done so, resulting in all involved nodes being unaware
+ * of the data inconsistency.
+ *
+ * This happens because ring ownership changes are implied by a single node; not all owning nodes get a say in when
+ * the transition takes effect. As such, a node can hold an incorrect belief about its own ownership ranges.
+ *
+ * This race condition is somewhat inherent in present day Cassandra, and there's actually a limit to what we can do about it.
+ * It is a little more dangerous with transient replication, however, because we can completely answer a request without
+ * ever touching a digest, meaning we are less likely to attempt to repair any inconsistency.
+ *
+ * We aren't guaranteed to contact any different nodes for the data requests, of course, though we at least have a chance.
+ *
+ * Note: If we have any pending transient->full movement, we need to move the full replica to our 'natural' bucket
+ * to avoid corrupting our count. This is fine for writes, all we're doing is ensuring we always write to the node,
+ * instead of selectively.
+ *
+ * @param natural
+ * @param pending
+ * @param <E>
+ * @return
+ */
+ static <E extends Endpoints<E>> boolean haveWriteConflicts(E natural, E pending)
+ {
+ Set<InetAddressAndPort> naturalEndpoints = natural.endpoints();
+ for (InetAddressAndPort pendingEndpoint : pending.endpoints())
+ {
+ if (naturalEndpoints.contains(pendingEndpoint))
+ return true;
+ }
+ return false;
+ }
+
+ /**
+ * MUST APPLY FIRST
+ * See {@link ReplicaLayout#haveWriteConflicts}
+ * @return a 'natural' replica collection, that has had its conflicts with pending repaired
+ */
+ @VisibleForTesting
+ static EndpointsForToken resolveWriteConflictsInNatural(EndpointsForToken natural, EndpointsForToken pending)
+ {
+ EndpointsForToken.Builder resolved = natural.newBuilder(natural.size());
+ for (Replica replica : natural)
+ {
+ // always prefer the full natural replica, if there is a conflict
+ if (replica.isTransient())
+ {
+ Replica conflict = pending.byEndpoint().get(replica.endpoint());
+ if (conflict != null)
+ {
+ // it should not be possible to have conflicts of the same replication type for the same range
+ assert conflict.isFull();
+ // If we have any pending transient->full movement, we need to move the full replica to our 'natural' bucket
+ // to avoid corrupting our count
+ resolved.add(conflict);
+ continue;
+ }
+ }
+ resolved.add(replica);
+ }
+ return resolved.build();
+ }
+
+ /**
+ * MUST APPLY SECOND
+ * See {@link ReplicaLayout#haveWriteConflicts}
+ * @return a 'pending' replica collection, that has had its conflicts with natural repaired
+ */
+ @VisibleForTesting
+ static EndpointsForToken resolveWriteConflictsInPending(EndpointsForToken natural, EndpointsForToken pending)
+ {
+ return pending.without(natural.endpoints());
+ }
+
+ /**
+ * @return the read layout for a token - this includes only live natural replicas, i.e. those that are not pending
+ * and not marked down by the failure detector. these are reverse sorted by the badness score of the configured snitch
+ */
+ static ReplicaLayout.ForTokenRead forTokenReadLiveSorted(Keyspace keyspace, Token token)
+ {
+ EndpointsForToken replicas = keyspace.getReplicationStrategy().getNaturalReplicasForToken(token);
+ replicas = DatabaseDescriptor.getEndpointSnitch().sortedByProximity(FBUtilities.getBroadcastAddressAndPort(), replicas);
+ replicas = replicas.filter(FailureDetector.isReplicaAlive);
+ return new ReplicaLayout.ForTokenRead(replicas);
+ }
+
+ /**
+ * TODO: we should really double check that the provided range does not overlap multiple token ring regions
+ * @return the read layout for a range - this includes only live natural replicas, i.e. those that are not pending
+ * and not marked down by the failure detector. these are reverse sorted by the badness score of the configured snitch
+ */
+ static ReplicaLayout.ForRangeRead forRangeReadLiveSorted(Keyspace keyspace, AbstractBounds<PartitionPosition> range)
+ {
+ EndpointsForRange replicas = keyspace.getReplicationStrategy().getNaturalReplicas(range.right);
+ replicas = DatabaseDescriptor.getEndpointSnitch().sortedByProximity(FBUtilities.getBroadcastAddressAndPort(), replicas);
+ replicas = replicas.filter(FailureDetector.isReplicaAlive);
+ return new ReplicaLayout.ForRangeRead(range, replicas);
+ }
+
+}
diff --git a/src/java/org/apache/cassandra/locator/ReplicaMultimap.java b/src/java/org/apache/cassandra/locator/ReplicaMultimap.java
new file mode 100644
index 0000000..5a8551a
--- /dev/null
+++ b/src/java/org/apache/cassandra/locator/ReplicaMultimap.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.locator;
+
+import java.util.AbstractMap;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.stream.Stream;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Iterables;
+
+@VisibleForTesting
+public abstract class ReplicaMultimap<K, C extends ReplicaCollection<?>>
+{
+ final Map<K, C> map;
+ ReplicaMultimap(Map<K, C> map)
+ {
+ this.map = map;
+ }
+
+ public abstract C get(K key);
+
+ public static abstract class Builder
+ <K, B extends ReplicaCollection.Builder<?>>
+
+ {
+ protected abstract B newBuilder(K key);
+
+ final Map<K, B> map;
+ Builder()
+ {
+ this.map = new HashMap<>();
+ }
+
+ public B get(K key)
+ {
+ Preconditions.checkNotNull(key);
+ return map.computeIfAbsent(key, k -> newBuilder(key));
+ }
+
+ public B getIfPresent(K key)
+ {
+ Preconditions.checkNotNull(key);
+ return map.get(key);
+ }
+
+ public void put(K key, Replica replica)
+ {
+ Preconditions.checkNotNull(key);
+ Preconditions.checkNotNull(replica);
+ get(key).add(replica);
+ }
+ }
+
+ public Iterable<Replica> flattenValues()
+ {
+ return Iterables.concat(map.values());
+ }
+
+ public Iterable<Map.Entry<K, Replica>> flattenEntries()
+ {
+ return () -> {
+ Stream<Map.Entry<K, Replica>> s = map.entrySet()
+ .stream()
+ .flatMap(entry -> entry.getValue()
+ .stream()
+ .map(replica -> (Map.Entry<K, Replica>)new AbstractMap.SimpleImmutableEntry<>(entry.getKey(), replica)));
+ return s.iterator();
+ };
+ }
+
+ public boolean isEmpty()
+ {
+ return map.isEmpty();
+ }
+
+ public boolean containsKey(Object key)
+ {
+ return map.containsKey(key);
+ }
+
+ public Set<K> keySet()
+ {
+ return map.keySet();
+ }
+
+ public Set<Map.Entry<K, C>> entrySet()
+ {
+ return map.entrySet();
+ }
+
+ public Map<K, C> asMap()
+ {
+ return map;
+ }
+
+ public boolean equals(Object o)
+ {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ ReplicaMultimap<?, ?> that = (ReplicaMultimap<?, ?>) o;
+ return Objects.equals(map, that.map);
+ }
+
+ public int hashCode()
+ {
+ return map.hashCode();
+ }
+
+ @Override
+ public String toString()
+ {
+ return map.toString();
+ }
+}
diff --git a/src/java/org/apache/cassandra/locator/ReplicaPlan.java b/src/java/org/apache/cassandra/locator/ReplicaPlan.java
new file mode 100644
index 0000000..407db5b
--- /dev/null
+++ b/src/java/org/apache/cassandra/locator/ReplicaPlan.java
@@ -0,0 +1,250 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.locator;
+
+import com.google.common.collect.Iterables;
+import org.apache.cassandra.db.ConsistencyLevel;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.dht.AbstractBounds;
+
+import java.util.function.Predicate;
+
+public abstract class ReplicaPlan<E extends Endpoints<E>>
+{
+ protected final Keyspace keyspace;
+ protected final ConsistencyLevel consistencyLevel;
+
+ // all nodes we will contact via any mechanism, including hints
+ // i.e., for:
+ // - reads, only live natural replicas
+ // ==> live.natural().subList(0, blockFor + initial speculate)
+ // - writes, includes all full, and any pending replicas, (and only any necessary transient ones to make up the difference)
+ // ==> liveAndDown.natural().filter(isFull) ++ liveAndDown.pending() ++ live.natural.filter(isTransient, req)
+ // - paxos, includes all live replicas (natural+pending), for this DC if SERIAL_LOCAL
+ // ==> live.all() (if consistencyLevel.isDCLocal(), then .filter(consistencyLevel.isLocal))
+ private final E contacts;
+
+ ReplicaPlan(Keyspace keyspace, ConsistencyLevel consistencyLevel, E contacts)
+ {
+ assert contacts != null;
+ this.keyspace = keyspace;
+ this.consistencyLevel = consistencyLevel;
+ this.contacts = contacts;
+ }
+
+ public abstract int blockFor();
+
+ public E contacts() { return contacts; }
+
+ // TODO: should this semantically return true if we contain the endpoint, not the exact replica?
+ public boolean contacts(Replica replica) { return contacts.contains(replica); }
+ public Keyspace keyspace() { return keyspace; }
+ public ConsistencyLevel consistencyLevel() { return consistencyLevel; }
+
+ public static abstract class ForRead<E extends Endpoints<E>> extends ReplicaPlan<E>
+ {
+ // all nodes we *could* contacts; typically all natural replicas that are believed to be alive
+ // we will consult this collection to find uncontacted nodes we might contact if we doubt we will meet consistency level
+ private final E candidates;
+
+ ForRead(Keyspace keyspace, ConsistencyLevel consistencyLevel, E candidates, E contact)
+ {
+ super(keyspace, consistencyLevel, contact);
+ this.candidates = candidates;
+ }
+
+ public int blockFor() { return consistencyLevel.blockFor(keyspace); }
+
+ public E candidates() { return candidates; }
+
+ public Replica firstUncontactedCandidate(Predicate<Replica> extraPredicate)
+ {
+ return Iterables.tryFind(candidates(), r -> extraPredicate.test(r) && !contacts(r)).orNull();
+ }
+
+ public Replica lookup(InetAddressAndPort endpoint)
+ {
+ return candidates().byEndpoint().get(endpoint);
+ }
+
+ public String toString()
+ {
+ return "ReplicaPlan.ForRead [ CL: " + consistencyLevel + " keyspace: " + keyspace + " candidates: " + candidates + " contacts: " + contacts() + " ]";
+ }
+ }
+
+ public static class ForTokenRead extends ForRead<EndpointsForToken>
+ {
+ public ForTokenRead(Keyspace keyspace, ConsistencyLevel consistencyLevel, EndpointsForToken candidates, EndpointsForToken contact)
+ {
+ super(keyspace, consistencyLevel, candidates, contact);
+ }
+
+ ForTokenRead withContact(EndpointsForToken newContact)
+ {
+ return new ForTokenRead(keyspace, consistencyLevel, candidates(), newContact);
+ }
+ }
+
+ public static class ForRangeRead extends ForRead<EndpointsForRange>
+ {
+ final AbstractBounds<PartitionPosition> range;
+ final int vnodeCount;
+
+ public ForRangeRead(Keyspace keyspace,
+ ConsistencyLevel consistencyLevel,
+ AbstractBounds<PartitionPosition> range,
+ EndpointsForRange candidates,
+ EndpointsForRange contact,
+ int vnodeCount)
+ {
+ super(keyspace, consistencyLevel, candidates, contact);
+ this.range = range;
+ this.vnodeCount = vnodeCount;
+ }
+
+ public AbstractBounds<PartitionPosition> range() { return range; }
+
+ /**
+ * @return number of vnode ranges covered by the range
+ */
+ public int vnodeCount() { return vnodeCount; }
+
+ ForRangeRead withContact(EndpointsForRange newContact)
+ {
+ return new ForRangeRead(keyspace, consistencyLevel, range, candidates(), newContact, vnodeCount);
+ }
+ }
+
+ public static abstract class ForWrite<E extends Endpoints<E>> extends ReplicaPlan<E>
+ {
+ // TODO: this is only needed because of poor isolation of concerns elsewhere - we can remove it soon, and will do so in a follow-up patch
+ final E pending;
+ final E liveAndDown;
+ final E live;
+
+ ForWrite(Keyspace keyspace, ConsistencyLevel consistencyLevel, E pending, E liveAndDown, E live, E contact)
+ {
+ super(keyspace, consistencyLevel, contact);
+ this.pending = pending;
+ this.liveAndDown = liveAndDown;
+ this.live = live;
+ }
+
+ public int blockFor() { return consistencyLevel.blockForWrite(keyspace, pending()); }
+
+ /** Replicas that a region of the ring is moving to; not yet ready to serve reads, but should receive writes */
+ public E pending() { return pending; }
+ /** Replicas that can participate in the write - this always includes all nodes (pending and natural) in all DCs, except for paxos LOCAL_QUORUM (which is local DC only) */
+ public E liveAndDown() { return liveAndDown; }
+ /** The live replicas present in liveAndDown, usually derived from FailureDetector.isReplicaAlive */
+ public E live() { return live; }
+ /** Calculate which live endpoints we could have contacted, but chose not to */
+ public E liveUncontacted() { return live().filter(r -> !contacts(r)); }
+ /** Test liveness, consistent with the upfront analysis done for this operation (i.e. test membership of live()) */
+ public boolean isAlive(Replica replica) { return live.endpoints().contains(replica.endpoint()); }
+ public Replica lookup(InetAddressAndPort endpoint)
+ {
+ return liveAndDown().byEndpoint().get(endpoint);
+ }
+
+ public String toString()
+ {
+ return "ReplicaPlan.ForWrite [ CL: " + consistencyLevel + " keyspace: " + keyspace + " liveAndDown: " + liveAndDown + " live: " + live + " contacts: " + contacts() + " ]";
+ }
+ }
+
+ public static class ForTokenWrite extends ForWrite<EndpointsForToken>
+ {
+ public ForTokenWrite(Keyspace keyspace, ConsistencyLevel consistencyLevel, EndpointsForToken pending, EndpointsForToken liveAndDown, EndpointsForToken live, EndpointsForToken contact)
+ {
+ super(keyspace, consistencyLevel, pending, liveAndDown, live, contact);
+ }
+
+ private ReplicaPlan.ForTokenWrite copy(ConsistencyLevel newConsistencyLevel, EndpointsForToken newContact)
+ {
+ return new ReplicaPlan.ForTokenWrite(keyspace, newConsistencyLevel, pending(), liveAndDown(), live(), newContact);
+ }
+
+ ForTokenWrite withConsistencyLevel(ConsistencyLevel newConsistencylevel) { return copy(newConsistencylevel, contacts()); }
+ public ForTokenWrite withContact(EndpointsForToken newContact) { return copy(consistencyLevel, newContact); }
+ }
+
+ public static class ForPaxosWrite extends ForWrite<EndpointsForToken>
+ {
+ final int requiredParticipants;
+
+ ForPaxosWrite(Keyspace keyspace, ConsistencyLevel consistencyLevel, EndpointsForToken pending, EndpointsForToken liveAndDown, EndpointsForToken live, EndpointsForToken contact, int requiredParticipants)
+ {
+ super(keyspace, consistencyLevel, pending, liveAndDown, live, contact);
+ this.requiredParticipants = requiredParticipants;
+ }
+
+ public int requiredParticipants() { return requiredParticipants; }
+ }
+
+ /**
+ * Used by AbstractReadExecutor, {Data,Digest}Resolver and ReadRepair to share a ReplicaPlan whose 'contacts' replicas
+ * we progressively modify via various forms of speculation (initial speculation, rr-read and rr-write)
+ *
+ * The internal reference is not volatile, despite being shared between threads. The initial reference provided to
+ * the constructor should be visible by the normal process of sharing data between threads (i.e. executors, etc)
+ * and any updates will either be seen or not seen, perhaps not promptly, but certainly not incompletely.
+ * The contained ReplicaPlan has only final member properties, so it cannot be seen partially initialised.
+ */
+ public interface Shared<E extends Endpoints<E>, P extends ReplicaPlan<E>>
+ {
+ /**
+ * add the provided replica to this shared plan, by updating the internal reference
+ */
+ public void addToContacts(Replica replica);
+ /**
+ * get the shared replica plan, non-volatile (so maybe stale) but no risk of partially initialised
+ */
+ public P get();
+ /**
+ * get the shared replica plan, non-volatile (so maybe stale) but no risk of partially initialised,
+ * but replace its 'contacts' with those provided
+ */
+ public abstract P getWithContacts(E endpoints);
+ }
+
+ public static class SharedForTokenRead implements Shared<EndpointsForToken, ForTokenRead>
+ {
+ private ForTokenRead replicaPlan;
+ SharedForTokenRead(ForTokenRead replicaPlan) { this.replicaPlan = replicaPlan; }
+ public void addToContacts(Replica replica) { replicaPlan = replicaPlan.withContact(Endpoints.append(replicaPlan.contacts(), replica)); }
+ public ForTokenRead get() { return replicaPlan; }
+ public ForTokenRead getWithContacts(EndpointsForToken newContact) { return replicaPlan.withContact(newContact); }
+ }
+
+ public static class SharedForRangeRead implements Shared<EndpointsForRange, ForRangeRead>
+ {
+ private ForRangeRead replicaPlan;
+ SharedForRangeRead(ForRangeRead replicaPlan) { this.replicaPlan = replicaPlan; }
+ public void addToContacts(Replica replica) { replicaPlan = replicaPlan.withContact(Endpoints.append(replicaPlan.contacts(), replica)); }
+ public ForRangeRead get() { return replicaPlan; }
+ public ForRangeRead getWithContacts(EndpointsForRange newContact) { return replicaPlan.withContact(newContact); }
+ }
+
+ public static SharedForTokenRead shared(ForTokenRead replicaPlan) { return new SharedForTokenRead(replicaPlan); }
+ public static SharedForRangeRead shared(ForRangeRead replicaPlan) { return new SharedForRangeRead(replicaPlan); }
+
+}
diff --git a/src/java/org/apache/cassandra/locator/ReplicaPlans.java b/src/java/org/apache/cassandra/locator/ReplicaPlans.java
new file mode 100644
index 0000000..083da7a
--- /dev/null
+++ b/src/java/org/apache/cassandra/locator/ReplicaPlans.java
@@ -0,0 +1,635 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.locator;
+
+import com.carrotsearch.hppc.ObjectIntHashMap;
+import com.carrotsearch.hppc.cursors.ObjectIntCursor;
+import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.ListMultimap;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Multimap;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.ConsistencyLevel;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.exceptions.UnavailableException;
+import org.apache.cassandra.gms.FailureDetector;
+import org.apache.cassandra.schema.SchemaConstants;
+import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.service.reads.AlwaysSpeculativeRetryPolicy;
+import org.apache.cassandra.service.reads.SpeculativeRetryPolicy;
+
+import org.apache.cassandra.utils.FBUtilities;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.function.Consumer;
+import java.util.function.Function;
+import java.util.function.Predicate;
+
+import static com.google.common.collect.Iterables.any;
+import static com.google.common.collect.Iterables.filter;
+import static org.apache.cassandra.db.ConsistencyLevel.EACH_QUORUM;
+import static org.apache.cassandra.db.ConsistencyLevel.eachQuorumForRead;
+import static org.apache.cassandra.db.ConsistencyLevel.eachQuorumForWrite;
+import static org.apache.cassandra.db.ConsistencyLevel.localQuorumFor;
+import static org.apache.cassandra.db.ConsistencyLevel.localQuorumForOurDc;
+import static org.apache.cassandra.locator.Replicas.addToCountPerDc;
+import static org.apache.cassandra.locator.Replicas.countInOurDc;
+import static org.apache.cassandra.locator.Replicas.countPerDc;
+
+public class ReplicaPlans
+{
+ private static final Logger logger = LoggerFactory.getLogger(ReplicaPlans.class);
+
+ public static boolean isSufficientLiveReplicasForRead(Keyspace keyspace, ConsistencyLevel consistencyLevel, Endpoints<?> liveReplicas)
+ {
+ switch (consistencyLevel)
+ {
+ case ANY:
+ // local hint is acceptable, and local node is always live
+ return true;
+ case LOCAL_ONE:
+ return countInOurDc(liveReplicas).hasAtleast(1, 1);
+ case LOCAL_QUORUM:
+ return countInOurDc(liveReplicas).hasAtleast(localQuorumForOurDc(keyspace), 1);
+ case EACH_QUORUM:
+ if (keyspace.getReplicationStrategy() instanceof NetworkTopologyStrategy)
+ {
+ int fullCount = 0;
+ Collection<String> dcs = ((NetworkTopologyStrategy) keyspace.getReplicationStrategy()).getDatacenters();
+ for (ObjectObjectCursor<String, Replicas.ReplicaCount> entry : countPerDc(dcs, liveReplicas))
+ {
+ Replicas.ReplicaCount count = entry.value;
+ if (!count.hasAtleast(localQuorumFor(keyspace, entry.key), 0))
+ return false;
+ fullCount += count.fullReplicas();
+ }
+ return fullCount > 0;
+ }
+ // Fallthough on purpose for SimpleStrategy
+ default:
+ return liveReplicas.size() >= consistencyLevel.blockFor(keyspace)
+ && Replicas.countFull(liveReplicas) > 0;
+ }
+ }
+
+ static void assureSufficientLiveReplicasForRead(Keyspace keyspace, ConsistencyLevel consistencyLevel, Endpoints<?> liveReplicas) throws UnavailableException
+ {
+ assureSufficientLiveReplicas(keyspace, consistencyLevel, liveReplicas, consistencyLevel.blockFor(keyspace), 1);
+ }
+ static void assureSufficientLiveReplicasForWrite(Keyspace keyspace, ConsistencyLevel consistencyLevel, Endpoints<?> allLive, Endpoints<?> pendingWithDown) throws UnavailableException
+ {
+ assureSufficientLiveReplicas(keyspace, consistencyLevel, allLive, consistencyLevel.blockForWrite(keyspace, pendingWithDown), 0);
+ }
+ static void assureSufficientLiveReplicas(Keyspace keyspace, ConsistencyLevel consistencyLevel, Endpoints<?> allLive, int blockFor, int blockForFullReplicas) throws UnavailableException
+ {
+ switch (consistencyLevel)
+ {
+ case ANY:
+ // local hint is acceptable, and local node is always live
+ break;
+ case LOCAL_ONE:
+ {
+ Replicas.ReplicaCount localLive = countInOurDc(allLive);
+ if (!localLive.hasAtleast(blockFor, blockForFullReplicas))
+ throw UnavailableException.create(consistencyLevel, 1, blockForFullReplicas, localLive.allReplicas(), localLive.fullReplicas());
+ break;
+ }
+ case LOCAL_QUORUM:
+ {
+ Replicas.ReplicaCount localLive = countInOurDc(allLive);
+ if (!localLive.hasAtleast(blockFor, blockForFullReplicas))
+ {
+ if (logger.isTraceEnabled())
+ {
+ logger.trace(String.format("Local replicas %s are insufficient to satisfy LOCAL_QUORUM requirement of %d live replicas and %d full replicas in '%s'",
+ allLive.filter(InOurDcTester.replicas()), blockFor, blockForFullReplicas, DatabaseDescriptor.getLocalDataCenter()));
+ }
+ throw UnavailableException.create(consistencyLevel, blockFor, blockForFullReplicas, localLive.allReplicas(), localLive.fullReplicas());
+ }
+ break;
+ }
+ case EACH_QUORUM:
+ if (keyspace.getReplicationStrategy() instanceof NetworkTopologyStrategy)
+ {
+ int total = 0;
+ int totalFull = 0;
+ Collection<String> dcs = ((NetworkTopologyStrategy) keyspace.getReplicationStrategy()).getDatacenters();
+ for (ObjectObjectCursor<String, Replicas.ReplicaCount> entry : countPerDc(dcs, allLive))
+ {
+ int dcBlockFor = localQuorumFor(keyspace, entry.key);
+ Replicas.ReplicaCount dcCount = entry.value;
+ if (!dcCount.hasAtleast(dcBlockFor, 0))
+ throw UnavailableException.create(consistencyLevel, entry.key, dcBlockFor, dcCount.allReplicas(), 0, dcCount.fullReplicas());
+ totalFull += dcCount.fullReplicas();
+ total += dcCount.allReplicas();
+ }
+ if (totalFull < blockForFullReplicas)
+ throw UnavailableException.create(consistencyLevel, blockFor, total, blockForFullReplicas, totalFull);
+ break;
+ }
+ // Fallthough on purpose for SimpleStrategy
+ default:
+ int live = allLive.size();
+ int full = Replicas.countFull(allLive);
+ if (live < blockFor || full < blockForFullReplicas)
+ {
+ if (logger.isTraceEnabled())
+ logger.trace("Live nodes {} do not satisfy ConsistencyLevel ({} required)", Iterables.toString(allLive), blockFor);
+ throw UnavailableException.create(consistencyLevel, blockFor, blockForFullReplicas, live, full);
+ }
+ break;
+ }
+ }
+
+ /**
+ * Construct a ReplicaPlan for writing to exactly one node, with CL.ONE. This node is *assumed* to be alive.
+ */
+ public static ReplicaPlan.ForTokenWrite forSingleReplicaWrite(Keyspace keyspace, Token token, Replica replica)
+ {
+ EndpointsForToken one = EndpointsForToken.of(token, replica);
+ EndpointsForToken empty = EndpointsForToken.empty(token);
+ return new ReplicaPlan.ForTokenWrite(keyspace, ConsistencyLevel.ONE, empty, one, one, one);
+ }
+
+ /**
+ * A forwarding counter write is always sent to a single owning coordinator for the range, by the original coordinator
+ * (if it is not itself an owner)
+ */
+ public static ReplicaPlan.ForTokenWrite forForwardingCounterWrite(Keyspace keyspace, Token token, Replica replica)
+ {
+ return forSingleReplicaWrite(keyspace, token, replica);
+ }
+
+ public static ReplicaPlan.ForTokenWrite forLocalBatchlogWrite()
+ {
+ Token token = DatabaseDescriptor.getPartitioner().getMinimumToken();
+ Keyspace systemKeypsace = Keyspace.open(SchemaConstants.SYSTEM_KEYSPACE_NAME);
+ Replica localSystemReplica = SystemReplicas.getSystemReplica(FBUtilities.getBroadcastAddressAndPort());
+
+ ReplicaLayout.ForTokenWrite liveAndDown = ReplicaLayout.forTokenWrite(
+ EndpointsForToken.of(token, localSystemReplica),
+ EndpointsForToken.empty(token)
+ );
+
+ return forWrite(systemKeypsace, ConsistencyLevel.ONE, liveAndDown, liveAndDown, writeAll);
+ }
+
+ /**
+ * Requires that the provided endpoints are alive. Converts them to their relevant system replicas.
+ * Note that the liveAndDown collection and live are equal to the provided endpoints.
+ *
+ * @param isAny if batch consistency level is ANY, in which case a local node will be picked
+ */
+ public static ReplicaPlan.ForTokenWrite forBatchlogWrite(boolean isAny) throws UnavailableException
+ {
+ // A single case we write not for range or token, but multiple mutations to many tokens
+ Token token = DatabaseDescriptor.getPartitioner().getMinimumToken();
+
+ TokenMetadata.Topology topology = StorageService.instance.getTokenMetadata().cachedOnlyTokenMap().getTopology();
+ IEndpointSnitch snitch = DatabaseDescriptor.getEndpointSnitch();
+ Multimap<String, InetAddressAndPort> localEndpoints = HashMultimap.create(topology.getDatacenterRacks()
+ .get(snitch.getLocalDatacenter()));
+ // Replicas are picked manually:
+ // - replicas should be alive according to the failure detector
+ // - replicas should be in the local datacenter
+ // - choose min(2, number of qualifying candiates above)
+ // - allow the local node to be the only replica only if it's a single-node DC
+ Collection<InetAddressAndPort> chosenEndpoints = filterBatchlogEndpoints(snitch.getLocalRack(), localEndpoints);
+
+ if (chosenEndpoints.isEmpty() && isAny)
+ chosenEndpoints = Collections.singleton(FBUtilities.getBroadcastAddressAndPort());
+
+ ReplicaLayout.ForTokenWrite liveAndDown = ReplicaLayout.forTokenWrite(
+ SystemReplicas.getSystemReplicas(chosenEndpoints).forToken(token),
+ EndpointsForToken.empty(token)
+ );
+
+ // Batchlog is hosted by either one node or two nodes from different racks.
+ ConsistencyLevel consistencyLevel = liveAndDown.all().size() == 1 ? ConsistencyLevel.ONE : ConsistencyLevel.TWO;
+
+ Keyspace systemKeypsace = Keyspace.open(SchemaConstants.SYSTEM_KEYSPACE_NAME);
+
+ // assume that we have already been given live endpoints, and skip applying the failure detector
+ return forWrite(systemKeypsace, consistencyLevel, liveAndDown, liveAndDown, writeAll);
+ }
+
+ private static Collection<InetAddressAndPort> filterBatchlogEndpoints(String localRack,
+ Multimap<String, InetAddressAndPort> endpoints)
+ {
+ return filterBatchlogEndpoints(localRack,
+ endpoints,
+ Collections::shuffle,
+ FailureDetector.isEndpointAlive,
+ ThreadLocalRandom.current()::nextInt);
+ }
+
+ // Collect a list of candidates for batchlog hosting. If possible these will be two nodes from different racks.
+ @VisibleForTesting
+ public static Collection<InetAddressAndPort> filterBatchlogEndpoints(String localRack,
+ Multimap<String, InetAddressAndPort> endpoints,
+ Consumer<List<?>> shuffle,
+ Predicate<InetAddressAndPort> isAlive,
+ Function<Integer, Integer> indexPicker)
+ {
+ // special case for single-node data centers
+ if (endpoints.values().size() == 1)
+ return endpoints.values();
+
+ // strip out dead endpoints and localhost
+ ListMultimap<String, InetAddressAndPort> validated = ArrayListMultimap.create();
+ for (Map.Entry<String, InetAddressAndPort> entry : endpoints.entries())
+ {
+ InetAddressAndPort addr = entry.getValue();
+ if (!addr.equals(FBUtilities.getBroadcastAddressAndPort()) && isAlive.test(addr))
+ validated.put(entry.getKey(), entry.getValue());
+ }
+
+ if (validated.size() <= 2)
+ return validated.values();
+
+ if (validated.size() - validated.get(localRack).size() >= 2)
+ {
+ // we have enough endpoints in other racks
+ validated.removeAll(localRack);
+ }
+
+ if (validated.keySet().size() == 1)
+ {
+ /*
+ * we have only 1 `other` rack to select replicas from (whether it be the local rack or a single non-local rack)
+ * pick two random nodes from there; we are guaranteed to have at least two nodes in the single remaining rack
+ * because of the preceding if block.
+ */
+ List<InetAddressAndPort> otherRack = Lists.newArrayList(validated.values());
+ shuffle.accept(otherRack);
+ return otherRack.subList(0, 2);
+ }
+
+ // randomize which racks we pick from if more than 2 remaining
+ Collection<String> racks;
+ if (validated.keySet().size() == 2)
+ {
+ racks = validated.keySet();
+ }
+ else
+ {
+ racks = Lists.newArrayList(validated.keySet());
+ shuffle.accept((List<?>) racks);
+ }
+
+ // grab a random member of up to two racks
+ List<InetAddressAndPort> result = new ArrayList<>(2);
+ for (String rack : Iterables.limit(racks, 2))
+ {
+ List<InetAddressAndPort> rackMembers = validated.get(rack);
+ result.add(rackMembers.get(indexPicker.apply(rackMembers.size())));
+ }
+
+ return result;
+ }
+
+ public static ReplicaPlan.ForTokenWrite forReadRepair(Token token, ReplicaPlan.ForRead<?> readPlan) throws UnavailableException
+ {
+ return forWrite(readPlan.keyspace, readPlan.consistencyLevel, token, writeReadRepair(readPlan));
+ }
+
+ public static ReplicaPlan.ForTokenWrite forWrite(Keyspace keyspace, ConsistencyLevel consistencyLevel, Token token, Selector selector) throws UnavailableException
+ {
+ return forWrite(keyspace, consistencyLevel, ReplicaLayout.forTokenWriteLiveAndDown(keyspace, token), selector);
+ }
+
+ @VisibleForTesting
+ public static ReplicaPlan.ForTokenWrite forWrite(Keyspace keyspace, ConsistencyLevel consistencyLevel, EndpointsForToken natural, EndpointsForToken pending, Predicate<Replica> isAlive, Selector selector) throws UnavailableException
+ {
+ return forWrite(keyspace, consistencyLevel, ReplicaLayout.forTokenWrite(natural, pending), isAlive, selector);
+ }
+
+ public static ReplicaPlan.ForTokenWrite forWrite(Keyspace keyspace, ConsistencyLevel consistencyLevel, ReplicaLayout.ForTokenWrite liveAndDown, Selector selector) throws UnavailableException
+ {
+ return forWrite(keyspace, consistencyLevel, liveAndDown, FailureDetector.isReplicaAlive, selector);
+ }
+
+ @VisibleForTesting
+ public static ReplicaPlan.ForTokenWrite forWrite(Keyspace keyspace, ConsistencyLevel consistencyLevel, ReplicaLayout.ForTokenWrite liveAndDown, Predicate<Replica> isAlive, Selector selector) throws UnavailableException
+ {
+ ReplicaLayout.ForTokenWrite live = liveAndDown.filter(isAlive);
+ return forWrite(keyspace, consistencyLevel, liveAndDown, live, selector);
+ }
+
+ public static ReplicaPlan.ForTokenWrite forWrite(Keyspace keyspace, ConsistencyLevel consistencyLevel, ReplicaLayout.ForTokenWrite liveAndDown, ReplicaLayout.ForTokenWrite live, Selector selector) throws UnavailableException
+ {
+ EndpointsForToken contacts = selector.select(keyspace, consistencyLevel, liveAndDown, live);
+ assureSufficientLiveReplicasForWrite(keyspace, consistencyLevel, live.all(), liveAndDown.pending());
+ return new ReplicaPlan.ForTokenWrite(keyspace, consistencyLevel, liveAndDown.pending(), liveAndDown.all(), live.all(), contacts);
+ }
+
+ public interface Selector
+ {
+ <E extends Endpoints<E>, L extends ReplicaLayout.ForWrite<E>>
+ E select(Keyspace keyspace, ConsistencyLevel consistencyLevel, L liveAndDown, L live);
+ }
+
+ /**
+ * Select all nodes, transient or otherwise, as targets for the operation.
+ *
+ * This is may no longer be useful once we finish implementing transient replication support, however
+ * it can be of value to stipulate that a location writes to all nodes without regard to transient status.
+ */
+ public static final Selector writeAll = new Selector()
+ {
+ @Override
+ public <E extends Endpoints<E>, L extends ReplicaLayout.ForWrite<E>>
+ E select(Keyspace keyspace, ConsistencyLevel consistencyLevel, L liveAndDown, L live)
+ {
+ return liveAndDown.all();
+ }
+ };
+
+ /**
+ * Select all full nodes, live or down, as write targets. If there are insufficient nodes to complete the write,
+ * but there are live transient nodes, select a sufficient number of these to reach our consistency level.
+ *
+ * Pending nodes are always contacted, whether or not they are full. When a transient replica is undergoing
+ * a pending move to a new node, if we write (transiently) to it, this write would not be replicated to the
+ * pending transient node, and so when completing the move, the write could effectively have not reached the
+ * promised consistency level.
+ */
+ public static final Selector writeNormal = new Selector()
+ {
+ @Override
+ public <E extends Endpoints<E>, L extends ReplicaLayout.ForWrite<E>>
+ E select(Keyspace keyspace, ConsistencyLevel consistencyLevel, L liveAndDown, L live)
+ {
+ if (!any(liveAndDown.all(), Replica::isTransient))
+ return liveAndDown.all();
+
+ ReplicaCollection.Builder<E> contacts = liveAndDown.all().newBuilder(liveAndDown.all().size());
+ contacts.addAll(filter(liveAndDown.natural(), Replica::isFull));
+ contacts.addAll(liveAndDown.pending());
+
+ /**
+ * Per CASSANDRA-14768, we ensure we write to at least a QUORUM of nodes in every DC,
+ * regardless of how many responses we need to wait for and our requested consistencyLevel.
+ * This is to minimally surprise users with transient replication; with normal writes, we
+ * soft-ensure that we reach QUORUM in all DCs we are able to, by writing to every node;
+ * even if we don't wait for ACK, we have in both cases sent sufficient messages.
+ */
+ ObjectIntHashMap<String> requiredPerDc = eachQuorumForWrite(keyspace, liveAndDown.pending());
+ addToCountPerDc(requiredPerDc, live.natural().filter(Replica::isFull), -1);
+ addToCountPerDc(requiredPerDc, live.pending(), -1);
+
+ IEndpointSnitch snitch = DatabaseDescriptor.getEndpointSnitch();
+ for (Replica replica : filter(live.natural(), Replica::isTransient))
+ {
+ String dc = snitch.getDatacenter(replica);
+ if (requiredPerDc.addTo(dc, -1) >= 0)
+ contacts.add(replica);
+ }
+ return contacts.build();
+ }
+ };
+
+ /**
+ * TODO: Transient Replication C-14404/C-14665
+ * TODO: We employ this even when there is no monotonicity to guarantee,
+ * e.g. in case of CL.TWO, CL.ONE with speculation, etc.
+ *
+ * Construct a read-repair write plan to provide monotonicity guarantees on any data we return as part of a read.
+ *
+ * Since this is not a regular write, this is just to guarantee future reads will read this data, we select only
+ * the minimal number of nodes to meet the consistency level, and prefer nodes we contacted on read to minimise
+ * data transfer.
+ */
+ public static Selector writeReadRepair(ReplicaPlan.ForRead<?> readPlan)
+ {
+ return new Selector()
+ {
+ @Override
+ public <E extends Endpoints<E>, L extends ReplicaLayout.ForWrite<E>>
+ E select(Keyspace keyspace, ConsistencyLevel consistencyLevel, L liveAndDown, L live)
+ {
+ assert !any(liveAndDown.all(), Replica::isTransient);
+
+ ReplicaCollection.Builder<E> contacts = live.all().newBuilder(live.all().size());
+ // add all live nodes we might write to that we have already contacted on read
+ contacts.addAll(filter(live.all(), r -> readPlan.contacts().endpoints().contains(r.endpoint())));
+
+ // finally, add sufficient nodes to achieve our consistency level
+ if (consistencyLevel != EACH_QUORUM)
+ {
+ int add = consistencyLevel.blockForWrite(keyspace, liveAndDown.pending()) - contacts.size();
+ if (add > 0)
+ {
+ for (Replica replica : filter(live.all(), r -> !contacts.contains(r)))
+ {
+ contacts.add(replica);
+ if (--add == 0)
+ break;
+ }
+ }
+ }
+ else
+ {
+ ObjectIntHashMap<String> requiredPerDc = eachQuorumForWrite(keyspace, liveAndDown.pending());
+ addToCountPerDc(requiredPerDc, contacts.snapshot(), -1);
+ IEndpointSnitch snitch = DatabaseDescriptor.getEndpointSnitch();
+ for (Replica replica : filter(live.all(), r -> !contacts.contains(r)))
+ {
+ String dc = snitch.getDatacenter(replica);
+ if (requiredPerDc.addTo(dc, -1) >= 0)
+ contacts.add(replica);
+ }
+ }
+ return contacts.build();
+ }
+ };
+ }
+
+ /**
+ * Construct the plan for a paxos round - NOT the write or read consistency level for either the write or comparison,
+ * but for the paxos linearisation agreement.
+ *
+ * This will select all live nodes as the candidates for the operation. Only the required number of participants
+ */
+ public static ReplicaPlan.ForPaxosWrite forPaxos(Keyspace keyspace, DecoratedKey key, ConsistencyLevel consistencyForPaxos) throws UnavailableException
+ {
+ Token tk = key.getToken();
+ ReplicaLayout.ForTokenWrite liveAndDown = ReplicaLayout.forTokenWriteLiveAndDown(keyspace, tk);
+
+ Replicas.temporaryAssertFull(liveAndDown.all()); // TODO CASSANDRA-14547
+
+ if (consistencyForPaxos == ConsistencyLevel.LOCAL_SERIAL)
+ {
+ // TODO: we should cleanup our semantics here, as we're filtering ALL nodes to localDC which is unexpected for ReplicaPlan
+ // Restrict natural and pending to node in the local DC only
+ liveAndDown = liveAndDown.filter(InOurDcTester.replicas());
+ }
+
+ ReplicaLayout.ForTokenWrite live = liveAndDown.filter(FailureDetector.isReplicaAlive);
+
+ // TODO: this should use assureSufficientReplicas
+ int participants = liveAndDown.all().size();
+ int requiredParticipants = participants / 2 + 1; // See CASSANDRA-8346, CASSANDRA-833
+
+ EndpointsForToken contacts = live.all();
+ if (contacts.size() < requiredParticipants)
+ throw UnavailableException.create(consistencyForPaxos, requiredParticipants, contacts.size());
+
+ // We cannot allow CAS operations with 2 or more pending endpoints, see #8346.
+ // Note that we fake an impossible number of required nodes in the unavailable exception
+ // to nail home the point that it's an impossible operation no matter how many nodes are live.
+ if (liveAndDown.pending().size() > 1)
+ throw new UnavailableException(String.format("Cannot perform LWT operation as there is more than one (%d) pending range movement", liveAndDown.all().size()),
+ consistencyForPaxos,
+ participants + 1,
+ contacts.size());
+
+ return new ReplicaPlan.ForPaxosWrite(keyspace, consistencyForPaxos, liveAndDown.pending(), liveAndDown.all(), live.all(), contacts, requiredParticipants);
+ }
+
+
+ private static <E extends Endpoints<E>> E candidatesForRead(ConsistencyLevel consistencyLevel, E liveNaturalReplicas)
+ {
+ return consistencyLevel.isDatacenterLocal()
+ ? liveNaturalReplicas.filter(InOurDcTester.replicas())
+ : liveNaturalReplicas;
+ }
+
+ private static <E extends Endpoints<E>> E contactForEachQuorumRead(Keyspace keyspace, E candidates)
+ {
+ assert keyspace.getReplicationStrategy() instanceof NetworkTopologyStrategy;
+ ObjectIntHashMap<String> perDc = eachQuorumForRead(keyspace);
+
+ final IEndpointSnitch snitch = DatabaseDescriptor.getEndpointSnitch();
+ return candidates.filter(replica -> {
+ String dc = snitch.getDatacenter(replica);
+ return perDc.addTo(dc, -1) >= 0;
+ });
+ }
+
+ private static <E extends Endpoints<E>> E contactForRead(Keyspace keyspace, ConsistencyLevel consistencyLevel, boolean alwaysSpeculate, E candidates)
+ {
+ /*
+ * If we are doing an each quorum query, we have to make sure that the endpoints we select
+ * provide a quorum for each data center. If we are not using a NetworkTopologyStrategy,
+ * we should fall through and grab a quorum in the replication strategy.
+ *
+ * We do not speculate for EACH_QUORUM.
+ *
+ * TODO: this is still very inconistently managed between {LOCAL,EACH}_QUORUM and other consistency levels - should address this in a follow-up
+ */
+ if (consistencyLevel == EACH_QUORUM && keyspace.getReplicationStrategy() instanceof NetworkTopologyStrategy)
+ return contactForEachQuorumRead(keyspace, candidates);
+
+ int count = consistencyLevel.blockFor(keyspace) + (alwaysSpeculate ? 1 : 0);
+ return candidates.subList(0, Math.min(count, candidates.size()));
+ }
+
+
+ /**
+ * Construct a plan for reading from a single node - this permits no speculation or read-repair
+ */
+ public static ReplicaPlan.ForTokenRead forSingleReplicaRead(Keyspace keyspace, Token token, Replica replica)
+ {
+ EndpointsForToken one = EndpointsForToken.of(token, replica);
+ return new ReplicaPlan.ForTokenRead(keyspace, ConsistencyLevel.ONE, one, one);
+ }
+
+ /**
+ * Construct a plan for reading from a single node - this permits no speculation or read-repair
+ */
+ public static ReplicaPlan.ForRangeRead forSingleReplicaRead(Keyspace keyspace, AbstractBounds<PartitionPosition> range, Replica replica, int vnodeCount)
+ {
+ // TODO: this is unsafe, as one.range() may be inconsistent with our supplied range; should refactor Range/AbstractBounds to single class
+ EndpointsForRange one = EndpointsForRange.of(replica);
+ return new ReplicaPlan.ForRangeRead(keyspace, ConsistencyLevel.ONE, range, one, one, vnodeCount);
+ }
+
+ /**
+ * Construct a plan for reading the provided token at the provided consistency level. This translates to a collection of
+ * - candidates who are: alive, replicate the token, and are sorted by their snitch scores
+ * - contacts who are: the first blockFor + (retry == ALWAYS ? 1 : 0) candidates
+ *
+ * The candidate collection can be used for speculation, although at present
+ * it would break EACH_QUORUM to do so without further filtering
+ */
+ public static ReplicaPlan.ForTokenRead forRead(Keyspace keyspace, Token token, ConsistencyLevel consistencyLevel, SpeculativeRetryPolicy retry)
+ {
+ EndpointsForToken candidates = candidatesForRead(consistencyLevel, ReplicaLayout.forTokenReadLiveSorted(keyspace, token).natural());
+ EndpointsForToken contacts = contactForRead(keyspace, consistencyLevel, retry.equals(AlwaysSpeculativeRetryPolicy.INSTANCE), candidates);
+
+ assureSufficientLiveReplicasForRead(keyspace, consistencyLevel, contacts);
+ return new ReplicaPlan.ForTokenRead(keyspace, consistencyLevel, candidates, contacts);
+ }
+
+ /**
+ * Construct a plan for reading the provided range at the provided consistency level. This translates to a collection of
+ * - candidates who are: alive, replicate the range, and are sorted by their snitch scores
+ * - contacts who are: the first blockFor candidates
+ *
+ * There is no speculation for range read queries at present, so we never 'always speculate' here, and a failed response fails the query.
+ */
+ public static ReplicaPlan.ForRangeRead forRangeRead(Keyspace keyspace, ConsistencyLevel consistencyLevel, AbstractBounds<PartitionPosition> range, int vnodeCount)
+ {
+ EndpointsForRange candidates = candidatesForRead(consistencyLevel, ReplicaLayout.forRangeReadLiveSorted(keyspace, range).natural());
+ EndpointsForRange contacts = contactForRead(keyspace, consistencyLevel, false, candidates);
+
+ assureSufficientLiveReplicasForRead(keyspace, consistencyLevel, contacts);
+ return new ReplicaPlan.ForRangeRead(keyspace, consistencyLevel, range, candidates, contacts, vnodeCount);
+ }
+
+ /**
+ * Take two range read plans for adjacent ranges, and check if it is OK (and worthwhile) to combine them into a single plan
+ */
+ public static ReplicaPlan.ForRangeRead maybeMerge(Keyspace keyspace, ConsistencyLevel consistencyLevel, ReplicaPlan.ForRangeRead left, ReplicaPlan.ForRangeRead right)
+ {
+ // TODO: should we be asserting that the ranges are adjacent?
+ AbstractBounds<PartitionPosition> newRange = left.range().withNewRight(right.range().right);
+ EndpointsForRange mergedCandidates = left.candidates().keep(right.candidates().endpoints());
+
+ // Check if there are enough shared endpoints for the merge to be possible.
+ if (!isSufficientLiveReplicasForRead(keyspace, consistencyLevel, mergedCandidates))
+ return null;
+
+ EndpointsForRange contacts = contactForRead(keyspace, consistencyLevel, false, mergedCandidates);
+
+ // Estimate whether merging will be a win or not
+ if (!DatabaseDescriptor.getEndpointSnitch().isWorthMergingForRangeQuery(contacts, left.contacts(), right.contacts()))
+ return null;
+
+ // If we get there, merge this range and the next one
+ return new ReplicaPlan.ForRangeRead(keyspace, consistencyLevel, newRange, mergedCandidates, contacts, left.vnodeCount() + right.vnodeCount());
+ }
+}
diff --git a/src/java/org/apache/cassandra/locator/Replicas.java b/src/java/org/apache/cassandra/locator/Replicas.java
new file mode 100644
index 0000000..1b299cf
--- /dev/null
+++ b/src/java/org/apache/cassandra/locator/Replicas.java
@@ -0,0 +1,162 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.locator;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.function.Predicate;
+
+import com.carrotsearch.hppc.ObjectIntHashMap;
+import com.carrotsearch.hppc.ObjectObjectHashMap;
+import com.google.common.collect.Iterables;
+import org.apache.cassandra.config.DatabaseDescriptor;
+
+import static com.google.common.collect.Iterables.all;
+
+public class Replicas
+{
+
+ public static int countFull(ReplicaCollection<?> replicas)
+ {
+ int count = 0;
+ for (Replica replica : replicas)
+ if (replica.isFull())
+ ++count;
+ return count;
+ }
+
+ public static class ReplicaCount
+ {
+ int fullReplicas;
+ int transientReplicas;
+
+ public int allReplicas()
+ {
+ return fullReplicas + transientReplicas;
+ }
+
+ public int fullReplicas()
+ {
+ return fullReplicas;
+ }
+
+ public int transientReplicas()
+ {
+ return transientReplicas;
+ }
+
+ public void increment(Replica replica)
+ {
+ if (replica.isFull()) ++fullReplicas;
+ else ++transientReplicas;
+ }
+
+ public boolean hasAtleast(int allReplicas, int fullReplicas)
+ {
+ return this.fullReplicas >= fullReplicas
+ && this.allReplicas() >= allReplicas;
+ }
+ }
+
+ public static ReplicaCount countInOurDc(ReplicaCollection<?> replicas)
+ {
+ ReplicaCount count = new ReplicaCount();
+ Predicate<Replica> inOurDc = InOurDcTester.replicas();
+ for (Replica replica : replicas)
+ if (inOurDc.test(replica))
+ count.increment(replica);
+ return count;
+ }
+
+ /**
+ * count the number of full and transient replicas, separately, for each DC
+ */
+ public static ObjectObjectHashMap<String, ReplicaCount> countPerDc(Collection<String> dataCenters, Iterable<Replica> replicas)
+ {
+ ObjectObjectHashMap<String, ReplicaCount> perDc = new ObjectObjectHashMap<>(dataCenters.size());
+ for (String dc: dataCenters)
+ perDc.put(dc, new ReplicaCount());
+
+ IEndpointSnitch snitch = DatabaseDescriptor.getEndpointSnitch();
+ for (Replica replica : replicas)
+ {
+ String dc = snitch.getDatacenter(replica);
+ perDc.get(dc).increment(replica);
+ }
+ return perDc;
+ }
+
+ /**
+ * increment each of the map's DC entries for each matching replica provided
+ */
+ public static void addToCountPerDc(ObjectIntHashMap<String> perDc, Iterable<Replica> replicas, int add)
+ {
+ IEndpointSnitch snitch = DatabaseDescriptor.getEndpointSnitch();
+ for (Replica replica : replicas)
+ {
+ String dc = snitch.getDatacenter(replica);
+ perDc.addTo(dc, add);
+ }
+ }
+
+ /**
+ * A placeholder for areas of the code that cannot yet handle transient replicas, but should do so in future
+ */
+ public static void temporaryAssertFull(Replica replica)
+ {
+ if (!replica.isFull())
+ {
+ throw new UnsupportedOperationException("transient replicas are currently unsupported: " + replica);
+ }
+ }
+
+ /**
+ * A placeholder for areas of the code that cannot yet handle transient replicas, but should do so in future
+ */
+ public static void temporaryAssertFull(Iterable<Replica> replicas)
+ {
+ if (!all(replicas, Replica::isFull))
+ {
+ throw new UnsupportedOperationException("transient replicas are currently unsupported: " + Iterables.toString(replicas));
+ }
+ }
+
+ /**
+ * For areas of the code that should never see a transient replica
+ */
+ public static void assertFull(Iterable<Replica> replicas)
+ {
+ if (!all(replicas, Replica::isFull))
+ {
+ throw new UnsupportedOperationException("transient replicas are currently unsupported: " + Iterables.toString(replicas));
+ }
+ }
+
+ public static List<String> stringify(ReplicaCollection<?> replicas, boolean withPort)
+ {
+ List<String> stringEndpoints = new ArrayList<>(replicas.size());
+ for (Replica replica: replicas)
+ {
+ stringEndpoints.add(replica.endpoint().getHostAddress(withPort));
+ }
+ return stringEndpoints;
+ }
+
+}
diff --git a/src/java/org/apache/cassandra/locator/ReplicationFactor.java b/src/java/org/apache/cassandra/locator/ReplicationFactor.java
new file mode 100644
index 0000000..91ce770
--- /dev/null
+++ b/src/java/org/apache/cassandra/locator/ReplicationFactor.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.locator;
+
+import java.util.List;
+import java.util.Objects;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicates;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.gms.Gossiper;
+import org.apache.cassandra.utils.FBUtilities;
+
+public class ReplicationFactor
+{
+ public static final ReplicationFactor ZERO = new ReplicationFactor(0);
+
+ public final int allReplicas;
+ public final int fullReplicas;
+
+ private ReplicationFactor(int allReplicas, int transientReplicas)
+ {
+ validate(allReplicas, transientReplicas);
+ this.allReplicas = allReplicas;
+ this.fullReplicas = allReplicas - transientReplicas;
+ }
+
+ public int transientReplicas()
+ {
+ return allReplicas - fullReplicas;
+ }
+
+ public boolean hasTransientReplicas()
+ {
+ return allReplicas != fullReplicas;
+ }
+
+ private ReplicationFactor(int allReplicas)
+ {
+ this(allReplicas, 0);
+ }
+
+ static void validate(int totalRF, int transientRF)
+ {
+ Preconditions.checkArgument(transientRF == 0 || DatabaseDescriptor.isTransientReplicationEnabled(),
+ "Transient replication is not enabled on this node");
+ Preconditions.checkArgument(totalRF >= 0,
+ "Replication factor must be non-negative, found %s", totalRF);
+ Preconditions.checkArgument(transientRF == 0 || transientRF < totalRF,
+ "Transient replicas must be zero, or less than total replication factor. For %s/%s", totalRF, transientRF);
+ if (transientRF > 0)
+ {
+ Preconditions.checkArgument(DatabaseDescriptor.getNumTokens() == 1,
+ "Transient nodes are not allowed with multiple tokens");
+ Stream<InetAddressAndPort> endpoints = Stream.concat(Gossiper.instance.getLiveMembers().stream(), Gossiper.instance.getUnreachableMembers().stream());
+ List<InetAddressAndPort> badVersionEndpoints = endpoints.filter(Predicates.not(FBUtilities.getBroadcastAddressAndPort()::equals))
+ .filter(endpoint -> Gossiper.instance.getReleaseVersion(endpoint) != null && Gossiper.instance.getReleaseVersion(endpoint).major < 4)
+ .collect(Collectors.toList());
+ if (!badVersionEndpoints.isEmpty())
+ throw new AssertionError("Transient replication is not supported in mixed version clusters with nodes < 4.0. Bad nodes: " + badVersionEndpoints);
+ }
+ else if (transientRF < 0)
+ {
+ throw new AssertionError(String.format("Amount of transient nodes should be strictly positive, but was: '%d'", transientRF));
+ }
+ }
+
+ public boolean equals(Object o)
+ {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ ReplicationFactor that = (ReplicationFactor) o;
+ return allReplicas == that.allReplicas && fullReplicas == that.fullReplicas;
+ }
+
+ public int hashCode()
+ {
+ return Objects.hash(allReplicas, fullReplicas);
+ }
+
+ public static ReplicationFactor fullOnly(int totalReplicas)
+ {
+ return new ReplicationFactor(totalReplicas);
+ }
+
+ public static ReplicationFactor withTransient(int totalReplicas, int transientReplicas)
+ {
+ return new ReplicationFactor(totalReplicas, transientReplicas);
+ }
+
+ public static ReplicationFactor fromString(String s)
+ {
+ if (s.contains("/"))
+ {
+ String[] parts = s.split("/");
+ Preconditions.checkArgument(parts.length == 2,
+ "Replication factor format is <replicas> or <replicas>/<transient>");
+ return new ReplicationFactor(Integer.valueOf(parts[0]), Integer.valueOf(parts[1]));
+ }
+ else
+ {
+ return new ReplicationFactor(Integer.valueOf(s), 0);
+ }
+ }
+
+ public String toParseableString()
+ {
+ return String.valueOf(allReplicas) + (hasTransientReplicas() ? "/" + transientReplicas() : "");
+ }
+
+ @Override
+ public String toString()
+ {
+ return "rf(" + toParseableString() + ')';
+ }
+}
diff --git a/src/java/org/apache/cassandra/locator/SeedProvider.java b/src/java/org/apache/cassandra/locator/SeedProvider.java
index a013fbb..7efa9e0 100644
--- a/src/java/org/apache/cassandra/locator/SeedProvider.java
+++ b/src/java/org/apache/cassandra/locator/SeedProvider.java
@@ -17,10 +17,9 @@
*/
package org.apache.cassandra.locator;
-import java.net.InetAddress;
import java.util.List;
public interface SeedProvider
{
- List<InetAddress> getSeeds();
+ List<InetAddressAndPort> getSeeds();
}
diff --git a/src/java/org/apache/cassandra/locator/SimpleSeedProvider.java b/src/java/org/apache/cassandra/locator/SimpleSeedProvider.java
index 665261d..fe500b4 100644
--- a/src/java/org/apache/cassandra/locator/SimpleSeedProvider.java
+++ b/src/java/org/apache/cassandra/locator/SimpleSeedProvider.java
@@ -17,7 +17,6 @@
*/
package org.apache.cassandra.locator;
-import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.Collections;
@@ -26,6 +25,7 @@
import org.apache.cassandra.config.Config;
import org.apache.cassandra.config.DatabaseDescriptor;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -35,7 +35,7 @@
public SimpleSeedProvider(Map<String, String> args) {}
- public List<InetAddress> getSeeds()
+ public List<InetAddressAndPort> getSeeds()
{
Config conf;
try
@@ -47,12 +47,14 @@
throw new AssertionError(e);
}
String[] hosts = conf.seed_provider.parameters.get("seeds").split(",", -1);
- List<InetAddress> seeds = new ArrayList<InetAddress>(hosts.length);
+ List<InetAddressAndPort> seeds = new ArrayList<>(hosts.length);
for (String host : hosts)
{
try
{
- seeds.add(InetAddress.getByName(host.trim()));
+ if(!host.trim().isEmpty()) {
+ seeds.add(InetAddressAndPort.getByName(host.trim()));
+ }
}
catch (UnknownHostException ex)
{
diff --git a/src/java/org/apache/cassandra/locator/SimpleSnitch.java b/src/java/org/apache/cassandra/locator/SimpleSnitch.java
index 27648c8..d605b6e 100644
--- a/src/java/org/apache/cassandra/locator/SimpleSnitch.java
+++ b/src/java/org/apache/cassandra/locator/SimpleSnitch.java
@@ -17,9 +17,6 @@
*/
package org.apache.cassandra.locator;
-import java.net.InetAddress;
-import java.util.List;
-
/**
* A simple endpoint snitch implementation that treats Strategy order as proximity,
* allowing non-read-repaired reads to prefer a single endpoint, which improves
@@ -27,23 +24,25 @@
*/
public class SimpleSnitch extends AbstractEndpointSnitch
{
- public String getRack(InetAddress endpoint)
+ public String getRack(InetAddressAndPort endpoint)
{
return "rack1";
}
- public String getDatacenter(InetAddress endpoint)
+ public String getDatacenter(InetAddressAndPort endpoint)
{
return "datacenter1";
}
@Override
- public void sortByProximity(final InetAddress address, List<InetAddress> addresses)
+ public <C extends ReplicaCollection<? extends C>> C sortedByProximity(final InetAddressAndPort address, C unsortedAddress)
{
// Optimization to avoid walking the list
+ return unsortedAddress;
}
- public int compareEndpoints(InetAddress target, InetAddress a1, InetAddress a2)
+ @Override
+ public int compareEndpoints(InetAddressAndPort target, Replica r1, Replica r2)
{
// Making all endpoints equal ensures we won't change the original ordering (since
// Collections.sort is guaranteed to be stable)
diff --git a/src/java/org/apache/cassandra/locator/SimpleStrategy.java b/src/java/org/apache/cassandra/locator/SimpleStrategy.java
index 9a5062b..610ffe1 100644
--- a/src/java/org/apache/cassandra/locator/SimpleStrategy.java
+++ b/src/java/org/apache/cassandra/locator/SimpleStrategy.java
@@ -17,14 +17,15 @@
*/
package org.apache.cassandra.locator;
-import java.net.InetAddress;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Collection;
+import java.util.Comparator;
import java.util.Iterator;
-import java.util.List;
import java.util.Map;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.dht.Range;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.dht.Token;
@@ -37,46 +38,60 @@
*/
public class SimpleStrategy extends AbstractReplicationStrategy
{
+ private static final String REPLICATION_FACTOR = "replication_factor";
+ private final ReplicationFactor rf;
+
public SimpleStrategy(String keyspaceName, TokenMetadata tokenMetadata, IEndpointSnitch snitch, Map<String, String> configOptions)
{
super(keyspaceName, tokenMetadata, snitch, configOptions);
+ validateOptionsInternal(configOptions);
+ this.rf = ReplicationFactor.fromString(this.configOptions.get(REPLICATION_FACTOR));
}
- public List<InetAddress> calculateNaturalEndpoints(Token token, TokenMetadata metadata)
+ public EndpointsForRange calculateNaturalReplicas(Token token, TokenMetadata metadata)
{
- int replicas = getReplicationFactor();
- ArrayList<Token> tokens = metadata.sortedTokens();
- List<InetAddress> endpoints = new ArrayList<InetAddress>(replicas);
+ ArrayList<Token> ring = metadata.sortedTokens();
+ if (ring.isEmpty())
+ return EndpointsForRange.empty(new Range<>(metadata.partitioner.getMinimumToken(), metadata.partitioner.getMinimumToken()));
- if (tokens.isEmpty())
- return endpoints;
+ Token replicaEnd = TokenMetadata.firstToken(ring, token);
+ Token replicaStart = metadata.getPredecessor(replicaEnd);
+ Range<Token> replicaRange = new Range<>(replicaStart, replicaEnd);
+ Iterator<Token> iter = TokenMetadata.ringIterator(ring, token, false);
+
+ EndpointsForRange.Builder replicas = new EndpointsForRange.Builder(replicaRange, rf.allReplicas);
// Add the token at the index by default
- Iterator<Token> iter = TokenMetadata.ringIterator(tokens, token, false);
- while (endpoints.size() < replicas && iter.hasNext())
+ while (replicas.size() < rf.allReplicas && iter.hasNext())
{
- InetAddress ep = metadata.getEndpoint(iter.next());
- if (!endpoints.contains(ep))
- endpoints.add(ep);
+ Token tk = iter.next();
+ InetAddressAndPort ep = metadata.getEndpoint(tk);
+ if (!replicas.endpoints().contains(ep))
+ replicas.add(new Replica(ep, replicaRange, replicas.size() < rf.fullReplicas));
}
- return endpoints;
+
+ return replicas.build();
}
- public int getReplicationFactor()
+ public ReplicationFactor getReplicationFactor()
{
- return Integer.parseInt(this.configOptions.get("replication_factor"));
+ return rf;
+ }
+
+ private final static void validateOptionsInternal(Map<String, String> configOptions) throws ConfigurationException
+ {
+ if (configOptions.get(REPLICATION_FACTOR) == null)
+ throw new ConfigurationException("SimpleStrategy requires a replication_factor strategy option.");
}
public void validateOptions() throws ConfigurationException
{
- String rf = configOptions.get("replication_factor");
- if (rf == null)
- throw new ConfigurationException("SimpleStrategy requires a replication_factor strategy option.");
- validateReplicationFactor(rf);
+ validateOptionsInternal(configOptions);
+ validateReplicationFactor(configOptions.get(REPLICATION_FACTOR));
}
public Collection<String> recognizedOptions()
{
- return Collections.<String>singleton("replication_factor");
+ return Collections.singleton(REPLICATION_FACTOR);
}
}
diff --git a/src/java/org/apache/cassandra/locator/SnitchProperties.java b/src/java/org/apache/cassandra/locator/SnitchProperties.java
index 158feef..afb6804 100644
--- a/src/java/org/apache/cassandra/locator/SnitchProperties.java
+++ b/src/java/org/apache/cassandra/locator/SnitchProperties.java
@@ -30,7 +30,7 @@
private static final Logger logger = LoggerFactory.getLogger(SnitchProperties.class);
public static final String RACKDC_PROPERTY_FILENAME = "cassandra-rackdc.properties";
- private Properties properties;
+ private final Properties properties;
public SnitchProperties()
{
diff --git a/src/java/org/apache/cassandra/locator/SystemReplicas.java b/src/java/org/apache/cassandra/locator/SystemReplicas.java
new file mode 100644
index 0000000..456bae5
--- /dev/null
+++ b/src/java/org/apache/cassandra/locator/SystemReplicas.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.locator;
+
+import java.util.Collection;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+import com.google.common.collect.Collections2;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+
+public class SystemReplicas
+{
+ private static final Map<InetAddressAndPort, Replica> systemReplicas = new ConcurrentHashMap<>();
+ public static final Range<Token> FULL_RANGE = new Range<>(DatabaseDescriptor.getPartitioner().getMinimumToken(),
+ DatabaseDescriptor.getPartitioner().getMinimumToken());
+
+ private static Replica createSystemReplica(InetAddressAndPort endpoint)
+ {
+ return new Replica(endpoint, FULL_RANGE, true);
+ }
+
+ /**
+ * There are a few places where a system function borrows write path functionality, but doesn't otherwise
+ * fit into normal replication strategies (ie: hints and batchlog). So here we provide a replica instance
+ */
+ public static Replica getSystemReplica(InetAddressAndPort endpoint)
+ {
+ return systemReplicas.computeIfAbsent(endpoint, SystemReplicas::createSystemReplica);
+ }
+
+ public static EndpointsForRange getSystemReplicas(Collection<InetAddressAndPort> endpoints)
+ {
+ if (endpoints.isEmpty())
+ return EndpointsForRange.empty(FULL_RANGE);
+
+ return EndpointsForRange.copyOf(Collections2.transform(endpoints, SystemReplicas::getSystemReplica));
+ }
+}
diff --git a/src/java/org/apache/cassandra/locator/TokenMetadata.java b/src/java/org/apache/cassandra/locator/TokenMetadata.java
index 8a1d9d0..c16538b 100644
--- a/src/java/org/apache/cassandra/locator/TokenMetadata.java
+++ b/src/java/org/apache/cassandra/locator/TokenMetadata.java
@@ -17,7 +17,6 @@
*/
package org.apache.cassandra.locator;
-import java.net.InetAddress;
import java.nio.ByteBuffer;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
@@ -28,6 +27,7 @@
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.*;
+import org.apache.cassandra.locator.ReplicaCollection.Builder.Conflict;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -52,12 +52,12 @@
* Each Token is associated with exactly one Address, but each Address may have
* multiple tokens. Hence, the BiMultiValMap collection.
*/
- private final BiMultiValMap<Token, InetAddress> tokenToEndpointMap;
+ private final BiMultiValMap<Token, InetAddressAndPort> tokenToEndpointMap;
/** Maintains endpoint to host ID map of every node in the cluster */
- private final BiMap<InetAddress, UUID> endpointToHostIdMap;
+ private final BiMap<InetAddressAndPort, UUID> endpointToHostIdMap;
- // Prior to CASSANDRA-603, we just had <tt>Map<Range, InetAddress> pendingRanges<tt>,
+ // Prior to CASSANDRA-603, we just had <tt>Map<Range, InetAddressAndPort> pendingRanges<tt>,
// which was added to when a node began bootstrap and removed from when it finished.
//
// This is inadequate when multiple changes are allowed simultaneously. For example,
@@ -70,8 +70,8 @@
//
// So, we made two changes:
//
- // First, we changed pendingRanges to a <tt>Multimap<Range, InetAddress></tt> (now
- // <tt>Map<String, Multimap<Range, InetAddress>></tt>, because replication strategy
+ // First, we changed pendingRanges to a <tt>Multimap<Range, InetAddressAndPort></tt> (now
+ // <tt>Map<String, Multimap<Range, InetAddressAndPort>></tt>, because replication strategy
// and options are per-KeySpace).
//
// Second, we added the bootstrapTokens and leavingEndpoints collections, so we can
@@ -81,17 +81,18 @@
// Finally, note that recording the tokens of joining nodes in bootstrapTokens also
// means we can detect and reject the addition of multiple nodes at the same token
// before one becomes part of the ring.
- private final BiMultiValMap<Token, InetAddress> bootstrapTokens = new BiMultiValMap<>();
+ private final BiMultiValMap<Token, InetAddressAndPort> bootstrapTokens = new BiMultiValMap<>();
- private final BiMap<InetAddress, InetAddress> replacementToOriginal = HashBiMap.create();
+ private final BiMap<InetAddressAndPort, InetAddressAndPort> replacementToOriginal = HashBiMap.create();
// (don't need to record Token here since it's still part of tokenToEndpointMap until it's done leaving)
- private final Set<InetAddress> leavingEndpoints = new HashSet<>();
+ private final Set<InetAddressAndPort> leavingEndpoints = new HashSet<>();
// this is a cache of the calculation from {tokenToEndpointMap, bootstrapTokens, leavingEndpoints}
+ // NOTE: this may contain ranges that conflict with the those implied by sortedTokens when a range is changing its transient status
private final ConcurrentMap<String, PendingRangeMaps> pendingRanges = new ConcurrentHashMap<String, PendingRangeMaps>();
// nodes which are migrating to the new tokens in the ring
- private final Set<Pair<Token, InetAddress>> movingEndpoints = new HashSet<>();
+ private final Set<Pair<Token, InetAddressAndPort>> movingEndpoints = new HashSet<>();
/* Use this lock for manipulating the token map */
private final ReadWriteLock lock = new ReentrantReadWriteLock(true);
@@ -101,26 +102,18 @@
public final IPartitioner partitioner;
- private static final Comparator<InetAddress> inetaddressCmp = new Comparator<InetAddress>()
- {
- public int compare(InetAddress o1, InetAddress o2)
- {
- return ByteBuffer.wrap(o1.getAddress()).compareTo(ByteBuffer.wrap(o2.getAddress()));
- }
- };
-
// signals replication strategies that nodes have joined or left the ring and they need to recompute ownership
private volatile long ringVersion = 0;
public TokenMetadata()
{
- this(SortedBiMultiValMap.<Token, InetAddress>create(null, inetaddressCmp),
- HashBiMap.<InetAddress, UUID>create(),
+ this(SortedBiMultiValMap.<Token, InetAddressAndPort>create(),
+ HashBiMap.create(),
Topology.empty(),
DatabaseDescriptor.getPartitioner());
}
- private TokenMetadata(BiMultiValMap<Token, InetAddress> tokenToEndpointMap, BiMap<InetAddress, UUID> endpointsMap, Topology topology, IPartitioner partitioner)
+ private TokenMetadata(BiMultiValMap<Token, InetAddressAndPort> tokenToEndpointMap, BiMap<InetAddressAndPort, UUID> endpointsMap, Topology topology, IPartitioner partitioner)
{
this.tokenToEndpointMap = tokenToEndpointMap;
this.topology = topology;
@@ -144,7 +137,7 @@
}
/** @return the number of nodes bootstrapping into source's primary range */
- public int pendingRangeChanges(InetAddress source)
+ public int pendingRangeChanges(InetAddressAndPort source)
{
int n = 0;
Collection<Range<Token>> sourceRanges = getPrimaryRangesFor(getTokens(source));
@@ -166,14 +159,14 @@
/**
* Update token map with a single token/endpoint pair in normal state.
*/
- public void updateNormalToken(Token token, InetAddress endpoint)
+ public void updateNormalToken(Token token, InetAddressAndPort endpoint)
{
updateNormalTokens(Collections.singleton(token), endpoint);
}
- public void updateNormalTokens(Collection<Token> tokens, InetAddress endpoint)
+ public void updateNormalTokens(Collection<Token> tokens, InetAddressAndPort endpoint)
{
- Multimap<InetAddress, Token> endpointTokens = HashMultimap.create();
+ Multimap<InetAddressAndPort, Token> endpointTokens = HashMultimap.create();
for (Token token : tokens)
endpointTokens.put(endpoint, token);
updateNormalTokens(endpointTokens);
@@ -185,7 +178,7 @@
* Prefer this whenever there are multiple pairs to update, as each update (whether a single or multiple)
* is expensive (CASSANDRA-3831).
*/
- public void updateNormalTokens(Multimap<InetAddress, Token> endpointTokens)
+ public void updateNormalTokens(Multimap<InetAddressAndPort, Token> endpointTokens)
{
if (endpointTokens.isEmpty())
return;
@@ -195,7 +188,7 @@
{
boolean shouldSortTokens = false;
Topology.Builder topologyBuilder = topology.unbuild();
- for (InetAddress endpoint : endpointTokens.keySet())
+ for (InetAddressAndPort endpoint : endpointTokens.keySet())
{
Collection<Token> tokens = endpointTokens.get(endpoint);
@@ -210,7 +203,7 @@
for (Token token : tokens)
{
- InetAddress prev = tokenToEndpointMap.put(token, endpoint);
+ InetAddressAndPort prev = tokenToEndpointMap.put(token, endpoint);
if (!endpoint.equals(prev))
{
if (prev != null)
@@ -234,7 +227,7 @@
* Store an end-point to host ID mapping. Each ID must be unique, and
* cannot be changed after the fact.
*/
- public void updateHostId(UUID hostId, InetAddress endpoint)
+ public void updateHostId(UUID hostId, InetAddressAndPort endpoint)
{
assert hostId != null;
assert endpoint != null;
@@ -242,7 +235,7 @@
lock.writeLock().lock();
try
{
- InetAddress storedEp = endpointToHostIdMap.inverse().get(hostId);
+ InetAddressAndPort storedEp = endpointToHostIdMap.inverse().get(hostId);
if (storedEp != null)
{
if (!storedEp.equals(endpoint) && (FailureDetector.instance.isAlive(storedEp)))
@@ -257,7 +250,7 @@
UUID storedId = endpointToHostIdMap.get(endpoint);
if ((storedId != null) && (!storedId.equals(hostId)))
logger.warn("Changing {}'s host ID from {} to {}", endpoint, storedId, hostId);
-
+
endpointToHostIdMap.forcePut(endpoint, hostId);
}
finally
@@ -268,7 +261,7 @@
}
/** Return the unique host ID for an end-point. */
- public UUID getHostId(InetAddress endpoint)
+ public UUID getHostId(InetAddressAndPort endpoint)
{
lock.readLock().lock();
try
@@ -282,7 +275,7 @@
}
/** Return the end-point for a unique host ID */
- public InetAddress getEndpointForHostId(UUID hostId)
+ public InetAddressAndPort getEndpointForHostId(UUID hostId)
{
lock.readLock().lock();
try
@@ -296,12 +289,12 @@
}
/** @return a copy of the endpoint-to-id map for read-only operations */
- public Map<InetAddress, UUID> getEndpointToHostIdMapForReading()
+ public Map<InetAddressAndPort, UUID> getEndpointToHostIdMapForReading()
{
lock.readLock().lock();
try
{
- Map<InetAddress, UUID> readMap = new HashMap<>();
+ Map<InetAddressAndPort, UUID> readMap = new HashMap<>();
readMap.putAll(endpointToHostIdMap);
return readMap;
}
@@ -312,17 +305,17 @@
}
@Deprecated
- public void addBootstrapToken(Token token, InetAddress endpoint)
+ public void addBootstrapToken(Token token, InetAddressAndPort endpoint)
{
addBootstrapTokens(Collections.singleton(token), endpoint);
}
- public void addBootstrapTokens(Collection<Token> tokens, InetAddress endpoint)
+ public void addBootstrapTokens(Collection<Token> tokens, InetAddressAndPort endpoint)
{
addBootstrapTokens(tokens, endpoint, null);
}
- private void addBootstrapTokens(Collection<Token> tokens, InetAddress endpoint, InetAddress original)
+ private void addBootstrapTokens(Collection<Token> tokens, InetAddressAndPort endpoint, InetAddressAndPort original)
{
assert tokens != null && !tokens.isEmpty();
assert endpoint != null;
@@ -331,7 +324,7 @@
try
{
- InetAddress oldEndpoint;
+ InetAddressAndPort oldEndpoint;
for (Token token : tokens)
{
@@ -355,7 +348,7 @@
}
}
- public void addReplaceTokens(Collection<Token> replacingTokens, InetAddress newNode, InetAddress oldNode)
+ public void addReplaceTokens(Collection<Token> replacingTokens, InetAddressAndPort newNode, InetAddressAndPort oldNode)
{
assert replacingTokens != null && !replacingTokens.isEmpty();
assert newNode != null && oldNode != null;
@@ -382,7 +375,7 @@
}
}
- public Optional<InetAddress> getReplacementNode(InetAddress endpoint)
+ public Optional<InetAddressAndPort> getReplacementNode(InetAddressAndPort endpoint)
{
lock.readLock().lock();
try
@@ -395,7 +388,7 @@
}
}
- public Optional<InetAddress> getReplacingNode(InetAddress endpoint)
+ public Optional<InetAddressAndPort> getReplacingNode(InetAddressAndPort endpoint)
{
lock.readLock().lock();
try
@@ -424,7 +417,7 @@
}
}
- public void addLeavingEndpoint(InetAddress endpoint)
+ public void addLeavingEndpoint(InetAddressAndPort endpoint)
{
assert endpoint != null;
@@ -444,7 +437,7 @@
* @param token token which is node moving to
* @param endpoint address of the moving node
*/
- public void addMovingEndpoint(Token token, InetAddress endpoint)
+ public void addMovingEndpoint(Token token, InetAddressAndPort endpoint)
{
assert endpoint != null;
@@ -459,7 +452,7 @@
}
}
- public void removeEndpoint(InetAddress endpoint)
+ public void removeEndpoint(InetAddressAndPort endpoint)
{
assert endpoint != null;
@@ -487,7 +480,7 @@
/**
* This is called when the snitch properties for this endpoint are updated, see CASSANDRA-10238.
*/
- public Topology updateTopology(InetAddress endpoint)
+ public Topology updateTopology(InetAddressAndPort endpoint)
{
assert endpoint != null;
@@ -529,14 +522,14 @@
* Remove pair of token/address from moving endpoints
* @param endpoint address of the moving node
*/
- public void removeFromMoving(InetAddress endpoint)
+ public void removeFromMoving(InetAddressAndPort endpoint)
{
assert endpoint != null;
lock.writeLock().lock();
try
{
- for (Pair<Token, InetAddress> pair : movingEndpoints)
+ for (Pair<Token, InetAddressAndPort> pair : movingEndpoints)
{
if (pair.right.equals(endpoint))
{
@@ -553,7 +546,7 @@
}
}
- public Collection<Token> getTokens(InetAddress endpoint)
+ public Collection<Token> getTokens(InetAddressAndPort endpoint)
{
assert endpoint != null;
assert isMember(endpoint); // don't want to return nulls
@@ -570,12 +563,12 @@
}
@Deprecated
- public Token getToken(InetAddress endpoint)
+ public Token getToken(InetAddressAndPort endpoint)
{
return getTokens(endpoint).iterator().next();
}
- public boolean isMember(InetAddress endpoint)
+ public boolean isMember(InetAddressAndPort endpoint)
{
assert endpoint != null;
@@ -590,7 +583,7 @@
}
}
- public boolean isLeaving(InetAddress endpoint)
+ public boolean isLeaving(InetAddressAndPort endpoint)
{
assert endpoint != null;
@@ -605,14 +598,14 @@
}
}
- public boolean isMoving(InetAddress endpoint)
+ public boolean isMoving(InetAddressAndPort endpoint)
{
assert endpoint != null;
lock.readLock().lock();
try
{
- for (Pair<Token, InetAddress> pair : movingEndpoints)
+ for (Pair<Token, InetAddressAndPort> pair : movingEndpoints)
{
if (pair.right.equals(endpoint))
return true;
@@ -637,7 +630,7 @@
lock.readLock().lock();
try
{
- return new TokenMetadata(SortedBiMultiValMap.create(tokenToEndpointMap, null, inetaddressCmp),
+ return new TokenMetadata(SortedBiMultiValMap.create(tokenToEndpointMap),
HashBiMap.create(endpointToHostIdMap),
topology,
partitioner);
@@ -692,9 +685,9 @@
}
}
- private static TokenMetadata removeEndpoints(TokenMetadata allLeftMetadata, Set<InetAddress> leavingEndpoints)
+ private static TokenMetadata removeEndpoints(TokenMetadata allLeftMetadata, Set<InetAddressAndPort> leavingEndpoints)
{
- for (InetAddress endpoint : leavingEndpoints)
+ for (InetAddressAndPort endpoint : leavingEndpoints)
allLeftMetadata.removeEndpoint(endpoint);
return allLeftMetadata;
@@ -713,11 +706,11 @@
{
TokenMetadata metadata = cloneOnlyTokenMap();
- for (InetAddress endpoint : leavingEndpoints)
+ for (InetAddressAndPort endpoint : leavingEndpoints)
metadata.removeEndpoint(endpoint);
- for (Pair<Token, InetAddress> pair : movingEndpoints)
+ for (Pair<Token, InetAddressAndPort> pair : movingEndpoints)
metadata.updateNormalToken(pair.left, pair.right);
return metadata;
@@ -728,7 +721,7 @@
}
}
- public InetAddress getEndpoint(Token token)
+ public InetAddressAndPort getEndpoint(Token token)
{
lock.readLock().lock();
try
@@ -760,24 +753,20 @@
return sortedTokens;
}
- public Multimap<Range<Token>, InetAddress> getPendingRangesMM(String keyspaceName)
+ public EndpointsByRange getPendingRangesMM(String keyspaceName)
{
- Multimap<Range<Token>, InetAddress> map = HashMultimap.create();
+ EndpointsByRange.Builder byRange = new EndpointsByRange.Builder();
PendingRangeMaps pendingRangeMaps = this.pendingRanges.get(keyspaceName);
if (pendingRangeMaps != null)
{
- for (Map.Entry<Range<Token>, List<InetAddress>> entry : pendingRangeMaps)
+ for (Map.Entry<Range<Token>, EndpointsForRange.Builder> entry : pendingRangeMaps)
{
- Range<Token> range = entry.getKey();
- for (InetAddress address : entry.getValue())
- {
- map.put(range, address);
- }
+ byRange.putAll(entry.getKey(), entry.getValue(), Conflict.ALL);
}
}
- return map;
+ return byRange.build();
}
/** a mutable map may be returned but caller should not modify it */
@@ -786,17 +775,18 @@
return this.pendingRanges.get(keyspaceName);
}
- public List<Range<Token>> getPendingRanges(String keyspaceName, InetAddress endpoint)
+ public RangesAtEndpoint getPendingRanges(String keyspaceName, InetAddressAndPort endpoint)
{
- List<Range<Token>> ranges = new ArrayList<>();
- for (Map.Entry<Range<Token>, InetAddress> entry : getPendingRangesMM(keyspaceName).entries())
+ RangesAtEndpoint.Builder builder = RangesAtEndpoint.builder(endpoint);
+ for (Map.Entry<Range<Token>, Replica> entry : getPendingRangesMM(keyspaceName).flattenEntries())
{
- if (entry.getValue().equals(endpoint))
+ Replica replica = entry.getValue();
+ if (replica.endpoint().equals(endpoint))
{
- ranges.add(entry.getKey());
+ builder.add(replica);
}
}
- return ranges;
+ return builder.build();
}
/**
@@ -828,28 +818,32 @@
long startedAt = System.currentTimeMillis();
synchronized (pendingRanges)
{
+ TokenMetadataDiagnostics.pendingRangeCalculationStarted(this, keyspaceName);
+
// create clone of current state
- BiMultiValMap<Token, InetAddress> bootstrapTokensClone;
- Set<InetAddress> leavingEndpointsClone;
- Set<Pair<Token, InetAddress>> movingEndpointsClone;
+ BiMultiValMap<Token, InetAddressAndPort> bootstrapTokensClone;
+ Set<InetAddressAndPort> leavingEndpointsClone;
+ Set<Pair<Token, InetAddressAndPort>> movingEndpointsClone;
TokenMetadata metadata;
lock.readLock().lock();
try
{
+
if (bootstrapTokens.isEmpty() && leavingEndpoints.isEmpty() && movingEndpoints.isEmpty())
{
if (logger.isTraceEnabled())
logger.trace("No bootstrapping, leaving or moving nodes -> empty pending ranges for {}", keyspaceName);
+ if (bootstrapTokens.isEmpty() && leavingEndpoints.isEmpty() && movingEndpoints.isEmpty())
+ {
+ if (logger.isTraceEnabled())
+ logger.trace("No bootstrapping, leaving or moving nodes -> empty pending ranges for {}", keyspaceName);
+ pendingRanges.put(keyspaceName, new PendingRangeMaps());
- pendingRanges.put(keyspaceName, new PendingRangeMaps());
-
- return;
+ return;
+ }
}
- if (logger.isDebugEnabled())
- logger.debug("Starting pending range calculation for {}", keyspaceName);
-
bootstrapTokensClone = new BiMultiValMap<>(this.bootstrapTokens);
leavingEndpointsClone = new HashSet<>(this.leavingEndpoints);
movingEndpointsClone = new HashSet<>(this.movingEndpoints);
@@ -862,6 +856,9 @@
pendingRanges.put(keyspaceName, calculatePendingRanges(strategy, metadata, bootstrapTokensClone,
leavingEndpointsClone, movingEndpointsClone));
+ if (logger.isDebugEnabled())
+ logger.debug("Starting pending range calculation for {}", keyspaceName);
+
long took = System.currentTimeMillis() - startedAt;
if (logger.isDebugEnabled())
@@ -876,50 +873,56 @@
*/
private static PendingRangeMaps calculatePendingRanges(AbstractReplicationStrategy strategy,
TokenMetadata metadata,
- BiMultiValMap<Token, InetAddress> bootstrapTokens,
- Set<InetAddress> leavingEndpoints,
- Set<Pair<Token, InetAddress>> movingEndpoints)
+ BiMultiValMap<Token, InetAddressAndPort> bootstrapTokens,
+ Set<InetAddressAndPort> leavingEndpoints,
+ Set<Pair<Token, InetAddressAndPort>> movingEndpoints)
{
PendingRangeMaps newPendingRanges = new PendingRangeMaps();
- Multimap<InetAddress, Range<Token>> addressRanges = strategy.getAddressRanges(metadata);
+ RangesByEndpoint addressRanges = strategy.getAddressReplicas(metadata);
// Copy of metadata reflecting the situation after all leave operations are finished.
TokenMetadata allLeftMetadata = removeEndpoints(metadata.cloneOnlyTokenMap(), leavingEndpoints);
// get all ranges that will be affected by leaving nodes
- Set<Range<Token>> affectedRanges = new HashSet<Range<Token>>();
- for (InetAddress endpoint : leavingEndpoints)
- affectedRanges.addAll(addressRanges.get(endpoint));
+ Set<Range<Token>> removeAffectedRanges = new HashSet<>();
+ for (InetAddressAndPort endpoint : leavingEndpoints)
+ removeAffectedRanges.addAll(addressRanges.get(endpoint).ranges());
// for each of those ranges, find what new nodes will be responsible for the range when
// all leaving nodes are gone.
- for (Range<Token> range : affectedRanges)
+ for (Range<Token> range : removeAffectedRanges)
{
- Set<InetAddress> currentEndpoints = ImmutableSet.copyOf(strategy.calculateNaturalEndpoints(range.right, metadata));
- Set<InetAddress> newEndpoints = ImmutableSet.copyOf(strategy.calculateNaturalEndpoints(range.right, allLeftMetadata));
- for (InetAddress address : Sets.difference(newEndpoints, currentEndpoints))
+ EndpointsForRange currentReplicas = strategy.calculateNaturalReplicas(range.right, metadata);
+ EndpointsForRange newReplicas = strategy.calculateNaturalReplicas(range.right, allLeftMetadata);
+ for (Replica replica : newReplicas)
{
- newPendingRanges.addPendingRange(range, address);
+ if (currentReplicas.endpoints().contains(replica.endpoint()))
+ continue;
+ newPendingRanges.addPendingRange(range, replica);
}
}
// At this stage newPendingRanges has been updated according to leave operations. We can
// now continue the calculation by checking bootstrapping nodes.
- // For each of the bootstrapping nodes, simply add and remove them one by one to
- // allLeftMetadata and check in between what their ranges would be.
- Multimap<InetAddress, Token> bootstrapAddresses = bootstrapTokens.inverse();
- for (InetAddress endpoint : bootstrapAddresses.keySet())
+ // For each of the bootstrapping nodes, simply add to the allLeftMetadata and check what their
+ // ranges would be. We actually need to clone allLeftMetadata each time as resetting its state
+ // after getting the new pending ranges is not as simple as just removing the bootstrapping
+ // endpoint. If the bootstrapping endpoint constitutes a replacement, removing it after checking
+ // the newly pending ranges means there are now fewer endpoints that there were originally and
+ // causes its next neighbour to take over its primary range which affects the next RF endpoints
+ // in the ring.
+ Multimap<InetAddressAndPort, Token> bootstrapAddresses = bootstrapTokens.inverse();
+ for (InetAddressAndPort endpoint : bootstrapAddresses.keySet())
{
Collection<Token> tokens = bootstrapAddresses.get(endpoint);
-
- allLeftMetadata.updateNormalTokens(tokens, endpoint);
- for (Range<Token> range : strategy.getAddressRanges(allLeftMetadata).get(endpoint))
+ TokenMetadata cloned = allLeftMetadata.cloneOnlyTokenMap();
+ cloned.updateNormalTokens(tokens, endpoint);
+ for (Replica replica : strategy.getAddressReplicas(cloned, endpoint))
{
- newPendingRanges.addPendingRange(range, endpoint);
+ newPendingRanges.addPendingRange(replica.range(), replica);
}
- allLeftMetadata.removeEndpoint(endpoint);
}
// At this stage newPendingRanges has been updated according to leaving and bootstrapping nodes.
@@ -927,41 +930,46 @@
// For each of the moving nodes, we do the same thing we did for bootstrapping:
// simply add and remove them one by one to allLeftMetadata and check in between what their ranges would be.
- for (Pair<Token, InetAddress> moving : movingEndpoints)
+ for (Pair<Token, InetAddressAndPort> moving : movingEndpoints)
{
//Calculate all the ranges which will could be affected. This will include the ranges before and after the move.
- Set<Range<Token>> moveAffectedRanges = new HashSet<>();
- InetAddress endpoint = moving.right; // address of the moving node
+ Set<Replica> moveAffectedReplicas = new HashSet<>();
+ InetAddressAndPort endpoint = moving.right; // address of the moving node
//Add ranges before the move
- for (Range<Token> range : strategy.getAddressRanges(allLeftMetadata).get(endpoint))
+ for (Replica replica : strategy.getAddressReplicas(allLeftMetadata, endpoint))
{
- moveAffectedRanges.add(range);
+ moveAffectedReplicas.add(replica);
}
allLeftMetadata.updateNormalToken(moving.left, endpoint);
//Add ranges after the move
- for (Range<Token> range : strategy.getAddressRanges(allLeftMetadata).get(endpoint))
+ for (Replica replica : strategy.getAddressReplicas(allLeftMetadata, endpoint))
{
- moveAffectedRanges.add(range);
+ moveAffectedReplicas.add(replica);
}
- for(Range<Token> range : moveAffectedRanges)
+ for (Replica replica : moveAffectedReplicas)
{
- Set<InetAddress> currentEndpoints = ImmutableSet.copyOf(strategy.calculateNaturalEndpoints(range.right, metadata));
- Set<InetAddress> newEndpoints = ImmutableSet.copyOf(strategy.calculateNaturalEndpoints(range.right, allLeftMetadata));
- Set<InetAddress> difference = Sets.difference(newEndpoints, currentEndpoints);
- for(final InetAddress address : difference)
+ Set<InetAddressAndPort> currentEndpoints = strategy.calculateNaturalReplicas(replica.range().right, metadata).endpoints();
+ Set<InetAddressAndPort> newEndpoints = strategy.calculateNaturalReplicas(replica.range().right, allLeftMetadata).endpoints();
+ Set<InetAddressAndPort> difference = Sets.difference(newEndpoints, currentEndpoints);
+ for (final InetAddressAndPort address : difference)
{
- Collection<Range<Token>> newRanges = strategy.getAddressRanges(allLeftMetadata).get(address);
- Collection<Range<Token>> oldRanges = strategy.getAddressRanges(metadata).get(address);
- //We want to get rid of any ranges which the node is currently getting.
- newRanges.removeAll(oldRanges);
+ RangesAtEndpoint newReplicas = strategy.getAddressReplicas(allLeftMetadata, address);
+ RangesAtEndpoint oldReplicas = strategy.getAddressReplicas(metadata, address);
- for(Range<Token> newRange : newRanges)
+ // Filter out the things that are already replicated
+ newReplicas = newReplicas.filter(r -> !oldReplicas.contains(r));
+ for (Replica newReplica : newReplicas)
{
- for(Range<Token> pendingRange : newRange.subtractAll(oldRanges))
+ // for correctness on write, we need to treat ranges that are becoming full differently
+ // to those that are presently transient; however reads must continue to use the current view
+ // for ranges that are becoming transient. We could choose to ignore them here, but it's probably
+ // cleaner to ensure this is dealt with at point of use, where we can make a conscious decision
+ // about which to use
+ for (Replica pendingReplica : newReplica.subtractSameReplication(oldReplicas))
{
- newPendingRanges.addPendingRange(pendingRange, address);
+ newPendingRanges.addPendingRange(pendingReplica.range(), pendingReplica);
}
}
}
@@ -1003,7 +1011,7 @@
}
/** @return a copy of the bootstrapping tokens map */
- public BiMultiValMap<Token, InetAddress> getBootstrapTokens()
+ public BiMultiValMap<Token, InetAddressAndPort> getBootstrapTokens()
{
lock.readLock().lock();
try
@@ -1016,7 +1024,7 @@
}
}
- public Set<InetAddress> getAllEndpoints()
+ public Set<InetAddressAndPort> getAllEndpoints()
{
lock.readLock().lock();
try
@@ -1029,8 +1037,21 @@
}
}
+ public int getSizeOfAllEndpoints()
+ {
+ lock.readLock().lock();
+ try
+ {
+ return endpointToHostIdMap.size();
+ }
+ finally
+ {
+ lock.readLock().unlock();
+ }
+ }
+
/** caller should not modify leavingEndpoints */
- public Set<InetAddress> getLeavingEndpoints()
+ public Set<InetAddressAndPort> getLeavingEndpoints()
{
lock.readLock().lock();
try
@@ -1043,11 +1064,24 @@
}
}
+ public int getSizeOfLeavingEndpoints()
+ {
+ lock.readLock().lock();
+ try
+ {
+ return leavingEndpoints.size();
+ }
+ finally
+ {
+ lock.readLock().unlock();
+ }
+ }
+
/**
* Endpoints which are migrating to the new tokens
* @return set of addresses of moving endpoints
*/
- public Set<Pair<Token, InetAddress>> getMovingEndpoints()
+ public Set<Pair<Token, InetAddressAndPort>> getMovingEndpoints()
{
lock.readLock().lock();
try
@@ -1060,6 +1094,19 @@
}
}
+ public int getSizeOfMovingEndpoints()
+ {
+ lock.readLock().lock();
+ try
+ {
+ return movingEndpoints.size();
+ }
+ finally
+ {
+ lock.readLock().unlock();
+ }
+ }
+
public static int firstTokenIndex(final ArrayList<Token> ring, Token start, boolean insertMin)
{
assert ring.size() > 0;
@@ -1148,14 +1195,14 @@
lock.readLock().lock();
try
{
- Multimap<InetAddress, Token> endpointToTokenMap = tokenToEndpointMap.inverse();
- Set<InetAddress> eps = endpointToTokenMap.keySet();
+ Multimap<InetAddressAndPort, Token> endpointToTokenMap = tokenToEndpointMap.inverse();
+ Set<InetAddressAndPort> eps = endpointToTokenMap.keySet();
if (!eps.isEmpty())
{
sb.append("Normal Tokens:");
sb.append(System.getProperty("line.separator"));
- for (InetAddress ep : eps)
+ for (InetAddressAndPort ep : eps)
{
sb.append(ep);
sb.append(':');
@@ -1168,7 +1215,7 @@
{
sb.append("Bootstrapping Tokens:" );
sb.append(System.getProperty("line.separator"));
- for (Map.Entry<Token, InetAddress> entry : bootstrapTokens.entrySet())
+ for (Map.Entry<Token, InetAddressAndPort> entry : bootstrapTokens.entrySet())
{
sb.append(entry.getValue()).append(':').append(entry.getKey());
sb.append(System.getProperty("line.separator"));
@@ -1179,7 +1226,7 @@
{
sb.append("Leaving Endpoints:");
sb.append(System.getProperty("line.separator"));
- for (InetAddress ep : leavingEndpoints)
+ for (InetAddressAndPort ep : leavingEndpoints)
{
sb.append(ep);
sb.append(System.getProperty("line.separator"));
@@ -1213,11 +1260,11 @@
return sb.toString();
}
- public Collection<InetAddress> pendingEndpointsFor(Token token, String keyspaceName)
+ public EndpointsForToken pendingEndpointsForToken(Token token, String keyspaceName)
{
PendingRangeMaps pendingRangeMaps = this.pendingRanges.get(keyspaceName);
if (pendingRangeMaps == null)
- return Collections.emptyList();
+ return EndpointsForToken.empty(token);
return pendingRangeMaps.pendingEndpointsFor(token);
}
@@ -1225,19 +1272,21 @@
/**
* @deprecated retained for benefit of old tests
*/
- public Collection<InetAddress> getWriteEndpoints(Token token, String keyspaceName, Collection<InetAddress> naturalEndpoints)
+ @Deprecated
+ public EndpointsForToken getWriteEndpoints(Token token, String keyspaceName, EndpointsForToken natural)
{
- return ImmutableList.copyOf(Iterables.concat(naturalEndpoints, pendingEndpointsFor(token, keyspaceName)));
+ EndpointsForToken pending = pendingEndpointsForToken(token, keyspaceName);
+ return ReplicaLayout.forTokenWrite(natural, pending).all();
}
/** @return an endpoint to token multimap representation of tokenToEndpointMap (a copy) */
- public Multimap<InetAddress, Token> getEndpointToTokenMapForReading()
+ public Multimap<InetAddressAndPort, Token> getEndpointToTokenMapForReading()
{
lock.readLock().lock();
try
{
- Multimap<InetAddress, Token> cloned = HashMultimap.create();
- for (Map.Entry<Token, InetAddress> entry : tokenToEndpointMap.entrySet())
+ Multimap<InetAddressAndPort, Token> cloned = HashMultimap.create();
+ for (Map.Entry<Token, InetAddressAndPort> entry : tokenToEndpointMap.entrySet())
cloned.put(entry.getValue(), entry.getKey());
return cloned;
}
@@ -1251,12 +1300,12 @@
* @return a (stable copy, won't be modified) Token to Endpoint map for all the normal and bootstrapping nodes
* in the cluster.
*/
- public Map<Token, InetAddress> getNormalAndBootstrappingTokenToEndpointMap()
+ public Map<Token, InetAddressAndPort> getNormalAndBootstrappingTokenToEndpointMap()
{
lock.readLock().lock();
try
{
- Map<Token, InetAddress> map = new HashMap<>(tokenToEndpointMap.size() + bootstrapTokens.size());
+ Map<Token, InetAddressAndPort> map = new HashMap<>(tokenToEndpointMap.size() + bootstrapTokens.size());
map.putAll(tokenToEndpointMap);
map.putAll(bootstrapTokens);
return map;
@@ -1302,18 +1351,18 @@
public static class Topology
{
/** multi-map of DC to endpoints in that DC */
- private final ImmutableMultimap<String, InetAddress> dcEndpoints;
+ private final ImmutableMultimap<String, InetAddressAndPort> dcEndpoints;
/** map of DC to multi-map of rack to endpoints in that rack */
- private final ImmutableMap<String, ImmutableMultimap<String, InetAddress>> dcRacks;
+ private final ImmutableMap<String, ImmutableMultimap<String, InetAddressAndPort>> dcRacks;
/** reverse-lookup map for endpoint to current known dc/rack assignment */
- private final ImmutableMap<InetAddress, Pair<String, String>> currentLocations;
+ private final ImmutableMap<InetAddressAndPort, Pair<String, String>> currentLocations;
private Topology(Builder builder)
{
this.dcEndpoints = ImmutableMultimap.copyOf(builder.dcEndpoints);
- ImmutableMap.Builder<String, ImmutableMultimap<String, InetAddress>> dcRackBuilder = ImmutableMap.builder();
- for (Map.Entry<String, Multimap<String, InetAddress>> entry : builder.dcRacks.entrySet())
+ ImmutableMap.Builder<String, ImmutableMultimap<String, InetAddressAndPort>> dcRackBuilder = ImmutableMap.builder();
+ for (Map.Entry<String, Multimap<String, InetAddressAndPort>> entry : builder.dcRacks.entrySet())
dcRackBuilder.put(entry.getKey(), ImmutableMultimap.copyOf(entry.getValue()));
this.dcRacks = dcRackBuilder.build();
@@ -1323,7 +1372,7 @@
/**
* @return multi-map of DC to endpoints in that DC
*/
- public Multimap<String, InetAddress> getDatacenterEndpoints()
+ public Multimap<String, InetAddressAndPort> getDatacenterEndpoints()
{
return dcEndpoints;
}
@@ -1331,7 +1380,7 @@
/**
* @return map of DC to multi-map of rack to endpoints in that rack
*/
- public ImmutableMap<String, ImmutableMultimap<String, InetAddress>> getDatacenterRacks()
+ public ImmutableMap<String, ImmutableMultimap<String, InetAddressAndPort>> getDatacenterRacks()
{
return dcRacks;
}
@@ -1339,7 +1388,7 @@
/**
* @return The DC and rack of the given endpoint.
*/
- public Pair<String, String> getLocation(InetAddress addr)
+ public Pair<String, String> getLocation(InetAddressAndPort addr)
{
return currentLocations.get(addr);
}
@@ -1362,11 +1411,11 @@
private static class Builder
{
/** multi-map of DC to endpoints in that DC */
- private final Multimap<String, InetAddress> dcEndpoints;
+ private final Multimap<String, InetAddressAndPort> dcEndpoints;
/** map of DC to multi-map of rack to endpoints in that rack */
- private final Map<String, Multimap<String, InetAddress>> dcRacks;
+ private final Map<String, Multimap<String, InetAddressAndPort>> dcRacks;
/** reverse-lookup map for endpoint to current known dc/rack assignment */
- private final Map<InetAddress, Pair<String, String>> currentLocations;
+ private final Map<InetAddressAndPort, Pair<String, String>> currentLocations;
Builder()
{
@@ -1380,7 +1429,7 @@
this.dcEndpoints = HashMultimap.create(from.dcEndpoints);
this.dcRacks = Maps.newHashMapWithExpectedSize(from.dcRacks.size());
- for (Map.Entry<String, ImmutableMultimap<String, InetAddress>> entry : from.dcRacks.entrySet())
+ for (Map.Entry<String, ImmutableMultimap<String, InetAddressAndPort>> entry : from.dcRacks.entrySet())
dcRacks.put(entry.getKey(), HashMultimap.create(entry.getValue()));
this.currentLocations = new HashMap<>(from.currentLocations);
@@ -1389,7 +1438,7 @@
/**
* Stores current DC/rack assignment for ep
*/
- Builder addEndpoint(InetAddress ep)
+ Builder addEndpoint(InetAddressAndPort ep)
{
IEndpointSnitch snitch = DatabaseDescriptor.getEndpointSnitch();
String dc = snitch.getDatacenter(ep);
@@ -1406,12 +1455,12 @@
return this;
}
- private void doAddEndpoint(InetAddress ep, String dc, String rack)
+ private void doAddEndpoint(InetAddressAndPort ep, String dc, String rack)
{
dcEndpoints.put(dc, ep);
if (!dcRacks.containsKey(dc))
- dcRacks.put(dc, HashMultimap.<String, InetAddress>create());
+ dcRacks.put(dc, HashMultimap.<String, InetAddressAndPort>create());
dcRacks.get(dc).put(rack, ep);
currentLocations.put(ep, Pair.create(dc, rack));
@@ -1420,7 +1469,7 @@
/**
* Removes current DC/rack assignment for ep
*/
- Builder removeEndpoint(InetAddress ep)
+ Builder removeEndpoint(InetAddressAndPort ep)
{
if (!currentLocations.containsKey(ep))
return this;
@@ -1429,13 +1478,13 @@
return this;
}
- private void doRemoveEndpoint(InetAddress ep, Pair<String, String> current)
+ private void doRemoveEndpoint(InetAddressAndPort ep, Pair<String, String> current)
{
dcRacks.get(current.left).remove(current.right, ep);
dcEndpoints.remove(current.left, ep);
}
- Builder updateEndpoint(InetAddress ep)
+ Builder updateEndpoint(InetAddressAndPort ep)
{
IEndpointSnitch snitch = DatabaseDescriptor.getEndpointSnitch();
if (snitch == null || !currentLocations.containsKey(ep))
@@ -1451,13 +1500,13 @@
if (snitch == null)
return this;
- for (InetAddress ep : currentLocations.keySet())
+ for (InetAddressAndPort ep : currentLocations.keySet())
updateEndpoint(ep, snitch);
return this;
}
- private void updateEndpoint(InetAddress ep, IEndpointSnitch snitch)
+ private void updateEndpoint(InetAddressAndPort ep, IEndpointSnitch snitch)
{
Pair<String, String> current = currentLocations.get(ep);
String dc = snitch.getDatacenter(ep);
diff --git a/src/java/org/apache/cassandra/locator/TokenMetadataDiagnostics.java b/src/java/org/apache/cassandra/locator/TokenMetadataDiagnostics.java
new file mode 100644
index 0000000..0221f1e
--- /dev/null
+++ b/src/java/org/apache/cassandra/locator/TokenMetadataDiagnostics.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.locator;
+
+import org.apache.cassandra.diag.DiagnosticEventService;
+import org.apache.cassandra.locator.TokenMetadataEvent.TokenMetadataEventType;
+
+/**
+ * Utility methods for events related to {@link TokenMetadata} changes.
+ */
+final class TokenMetadataDiagnostics
+{
+ private static final DiagnosticEventService service = DiagnosticEventService.instance();
+
+ private TokenMetadataDiagnostics()
+ {
+ }
+
+ static void pendingRangeCalculationStarted(TokenMetadata tokenMetadata, String keyspace)
+ {
+ if (isEnabled(TokenMetadataEventType.PENDING_RANGE_CALCULATION_STARTED))
+ service.publish(new TokenMetadataEvent(TokenMetadataEventType.PENDING_RANGE_CALCULATION_STARTED, tokenMetadata, keyspace));
+ }
+
+ private static boolean isEnabled(TokenMetadataEventType type)
+ {
+ return service.isEnabled(TokenMetadataEvent.class, type);
+ }
+
+}
diff --git a/src/java/org/apache/cassandra/locator/TokenMetadataEvent.java b/src/java/org/apache/cassandra/locator/TokenMetadataEvent.java
new file mode 100644
index 0000000..c3ed074
--- /dev/null
+++ b/src/java/org/apache/cassandra/locator/TokenMetadataEvent.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.locator;
+
+import java.io.Serializable;
+import java.util.HashMap;
+
+import org.apache.cassandra.diag.DiagnosticEvent;
+
+/**
+ * Events related to {@link TokenMetadata} changes.
+ */
+public final class TokenMetadataEvent extends DiagnosticEvent
+{
+
+ public enum TokenMetadataEventType
+ {
+ PENDING_RANGE_CALCULATION_STARTED,
+ PENDING_RANGE_CALCULATION_COMPLETED,
+ }
+
+ private final TokenMetadataEventType type;
+ private final TokenMetadata tokenMetadata;
+ private final String keyspace;
+
+ TokenMetadataEvent(TokenMetadataEventType type, TokenMetadata tokenMetadata, String keyspace)
+ {
+ this.type = type;
+ this.tokenMetadata = tokenMetadata;
+ this.keyspace = keyspace;
+ }
+
+ public TokenMetadataEventType getType()
+ {
+ return type;
+ }
+
+ public HashMap<String, Serializable> toMap()
+ {
+ // be extra defensive against nulls and bugs
+ HashMap<String, Serializable> ret = new HashMap<>();
+ ret.put("keyspace", keyspace);
+ ret.put("tokenMetadata", tokenMetadata.toString());
+ return ret;
+ }
+}
diff --git a/src/java/org/apache/cassandra/metrics/AuthMetrics.java b/src/java/org/apache/cassandra/metrics/AuthMetrics.java
deleted file mode 100644
index 57d08ef..0000000
--- a/src/java/org/apache/cassandra/metrics/AuthMetrics.java
+++ /dev/null
@@ -1,40 +0,0 @@
-package org.apache.cassandra.metrics;
-
-import com.codahale.metrics.Meter;
-
-/**
- * Metrics about authentication
- */
-public class AuthMetrics
-{
-
- public static final AuthMetrics instance = new AuthMetrics();
-
- public static void init()
- {
- // no-op, just used to force instance creation
- }
-
- /** Number and rate of successful logins */
- protected final Meter success;
-
- /** Number and rate of login failures */
- protected final Meter failure;
-
- private AuthMetrics()
- {
-
- success = ClientMetrics.instance.registerMeter("AuthSuccess");
- failure = ClientMetrics.instance.registerMeter("AuthFailure");
- }
-
- public void markSuccess()
- {
- success.mark();
- }
-
- public void markFailure()
- {
- failure.mark();
- }
-}
diff --git a/src/java/org/apache/cassandra/metrics/BatchMetrics.java b/src/java/org/apache/cassandra/metrics/BatchMetrics.java
new file mode 100644
index 0000000..9bea162
--- /dev/null
+++ b/src/java/org/apache/cassandra/metrics/BatchMetrics.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.metrics;
+
+import com.codahale.metrics.Histogram;
+
+import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics;
+
+public class BatchMetrics
+{
+ private static final MetricNameFactory factory = new DefaultNameFactory("Batch");
+
+ public final Histogram partitionsPerLoggedBatch;
+ public final Histogram partitionsPerUnloggedBatch;
+ public final Histogram partitionsPerCounterBatch;
+
+ public BatchMetrics()
+ {
+ partitionsPerLoggedBatch = Metrics.histogram(factory.createMetricName("PartitionsPerLoggedBatch"), false);
+ partitionsPerUnloggedBatch = Metrics.histogram(factory.createMetricName("PartitionsPerUnloggedBatch"), false);
+ partitionsPerCounterBatch = Metrics.histogram(factory.createMetricName("PartitionsPerCounterBatch"), false);
+ }
+}
diff --git a/src/java/org/apache/cassandra/metrics/CASClientRequestMetrics.java b/src/java/org/apache/cassandra/metrics/CASClientRequestMetrics.java
index f3f1f64..c6d3921 100644
--- a/src/java/org/apache/cassandra/metrics/CASClientRequestMetrics.java
+++ b/src/java/org/apache/cassandra/metrics/CASClientRequestMetrics.java
@@ -20,31 +20,29 @@
import com.codahale.metrics.Counter;
import com.codahale.metrics.Histogram;
+import com.codahale.metrics.Meter;
import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics;
-
public class CASClientRequestMetrics extends ClientRequestMetrics
{
public final Histogram contention;
- /* Used only for write */
- public final Counter conditionNotMet;
-
public final Counter unfinishedCommit;
+ public final Meter unknownResult;
public CASClientRequestMetrics(String scope)
{
super(scope);
contention = Metrics.histogram(factory.createMetricName("ContentionHistogram"), false);
- conditionNotMet = Metrics.counter(factory.createMetricName("ConditionNotMet"));
- unfinishedCommit = Metrics.counter(factory.createMetricName("UnfinishedCommit"));
+ unfinishedCommit = Metrics.counter(factory.createMetricName("UnfinishedCommit"));
+ unknownResult = Metrics.meter(factory.createMetricName("UnknownResult"));
}
public void release()
{
super.release();
Metrics.remove(factory.createMetricName("ContentionHistogram"));
- Metrics.remove(factory.createMetricName("ConditionNotMet"));
Metrics.remove(factory.createMetricName("UnfinishedCommit"));
+ Metrics.remove(factory.createMetricName("UnknownResult"));
}
}
diff --git a/src/java/org/apache/cassandra/metrics/CASClientWriteRequestMetrics.java b/src/java/org/apache/cassandra/metrics/CASClientWriteRequestMetrics.java
new file mode 100644
index 0000000..5971074
--- /dev/null
+++ b/src/java/org/apache/cassandra/metrics/CASClientWriteRequestMetrics.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.cassandra.metrics;
+
+import com.codahale.metrics.Counter;
+import com.codahale.metrics.Histogram;
+
+import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics;
+
+/**
+ * Metrics for tracking information about CAS write requests.
+ *
+ */
+public class CASClientWriteRequestMetrics extends CASClientRequestMetrics
+{
+ /**
+ * Metric for tracking the mutation sizes in bytes.
+ */
+ public final Histogram mutationSize;
+
+ public final Counter conditionNotMet;
+
+ public CASClientWriteRequestMetrics(String scope)
+ {
+ super(scope);
+ mutationSize = Metrics.histogram(factory.createMetricName("MutationSizeHistogram"), false);
+ conditionNotMet = Metrics.counter(factory.createMetricName("ConditionNotMet"));
+ }
+
+ public void release()
+ {
+ super.release();
+ Metrics.remove(factory.createMetricName("ConditionNotMet"));
+ Metrics.remove(factory.createMetricName("MutationSizeHistogram"));
+ }
+}
diff --git a/src/java/org/apache/cassandra/metrics/CacheMetrics.java b/src/java/org/apache/cassandra/metrics/CacheMetrics.java
index e623dcb..d4a00aa 100644
--- a/src/java/org/apache/cassandra/metrics/CacheMetrics.java
+++ b/src/java/org/apache/cassandra/metrics/CacheMetrics.java
@@ -17,10 +17,10 @@
*/
package org.apache.cassandra.metrics;
-import com.codahale.metrics.Gauge;
-import com.codahale.metrics.Meter;
-import com.codahale.metrics.RatioGauge;
-import org.apache.cassandra.cache.ICache;
+import java.util.function.DoubleSupplier;
+
+import com.codahale.metrics.*;
+import org.apache.cassandra.cache.CacheSize;
import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics;
@@ -31,10 +31,18 @@
{
/** Cache capacity in bytes */
public final Gauge<Long> capacity;
+ /** Total size of cache, in bytes */
+ public final Gauge<Long> size;
+ /** Total number of cache entries */
+ public final Gauge<Integer> entries;
+
/** Total number of cache hits */
public final Meter hits;
+ /** Total number of cache misses */
+ public final Meter misses;
/** Total number of cache requests */
- public final Meter requests;
+ public final Metered requests;
+
/** all time cache hit rate */
public final Gauge<Double> hitRate;
/** 1m hit rate */
@@ -43,10 +51,8 @@
public final Gauge<Double> fiveMinuteHitRate;
/** 15m hit rate */
public final Gauge<Double> fifteenMinuteHitRate;
- /** Total size of cache, in bytes */
- public final Gauge<Long> size;
- /** Total number of cache entries */
- public final Gauge<Integer> entries;
+
+ protected final MetricNameFactory factory;
/**
* Create metrics for given cache.
@@ -54,61 +60,77 @@
* @param type Type of Cache to identify metrics.
* @param cache Cache to measure metrics
*/
- public CacheMetrics(String type, final ICache<?, ?> cache)
+ public CacheMetrics(String type, CacheSize cache)
{
- MetricNameFactory factory = new DefaultNameFactory("Cache", type);
+ factory = new DefaultNameFactory("Cache", type);
- capacity = Metrics.register(factory.createMetricName("Capacity"), new Gauge<Long>()
- {
- public Long getValue()
- {
- return cache.capacity();
- }
- });
+ capacity = Metrics.register(factory.createMetricName("Capacity"), cache::capacity);
+ size = Metrics.register(factory.createMetricName("Size"), cache::weightedSize);
+ entries = Metrics.register(factory.createMetricName("Entries"), cache::size);
+
hits = Metrics.meter(factory.createMetricName("Hits"));
- requests = Metrics.meter(factory.createMetricName("Requests"));
- hitRate = Metrics.register(factory.createMetricName("HitRate"), new RatioGauge()
+ misses = Metrics.meter(factory.createMetricName("Misses"));
+ requests = Metrics.register(factory.createMetricName("Requests"), sumMeters(hits, misses));
+
+ hitRate =
+ Metrics.register(factory.createMetricName("HitRate"),
+ ratioGauge(hits::getCount, requests::getCount));
+ oneMinuteHitRate =
+ Metrics.register(factory.createMetricName("OneMinuteHitRate"),
+ ratioGauge(hits::getOneMinuteRate, requests::getOneMinuteRate));
+ fiveMinuteHitRate =
+ Metrics.register(factory.createMetricName("FiveMinuteHitRate"),
+ ratioGauge(hits::getFiveMinuteRate, requests::getFiveMinuteRate));
+ fifteenMinuteHitRate =
+ Metrics.register(factory.createMetricName("FifteenMinuteHitRate"),
+ ratioGauge(hits::getFifteenMinuteRate, requests::getFifteenMinuteRate));
+ }
+
+ private static Metered sumMeters(Metered first, Metered second)
+ {
+ return new Metered()
+ {
+ @Override
+ public long getCount()
+ {
+ return first.getCount() + second.getCount();
+ }
+
+ @Override
+ public double getMeanRate()
+ {
+ return first.getMeanRate() + second.getMeanRate();
+ }
+
+ @Override
+ public double getOneMinuteRate()
+ {
+ return first.getOneMinuteRate() + second.getOneMinuteRate();
+ }
+
+ @Override
+ public double getFiveMinuteRate()
+ {
+ return first.getFiveMinuteRate() + second.getFiveMinuteRate();
+ }
+
+ @Override
+ public double getFifteenMinuteRate()
+ {
+ return first.getFifteenMinuteRate() + second.getFifteenMinuteRate();
+ }
+ };
+ }
+
+ private static RatioGauge ratioGauge(DoubleSupplier numeratorSupplier, DoubleSupplier denominatorSupplier)
+ {
+ return new RatioGauge()
{
@Override
public Ratio getRatio()
{
- return Ratio.of(hits.getCount(), requests.getCount());
+ return Ratio.of(numeratorSupplier.getAsDouble(), denominatorSupplier.getAsDouble());
}
- });
- oneMinuteHitRate = Metrics.register(factory.createMetricName("OneMinuteHitRate"), new RatioGauge()
- {
- protected Ratio getRatio()
- {
- return Ratio.of(hits.getOneMinuteRate(), requests.getOneMinuteRate());
- }
- });
- fiveMinuteHitRate = Metrics.register(factory.createMetricName("FiveMinuteHitRate"), new RatioGauge()
- {
- protected Ratio getRatio()
- {
- return Ratio.of(hits.getFiveMinuteRate(), requests.getFiveMinuteRate());
- }
- });
- fifteenMinuteHitRate = Metrics.register(factory.createMetricName("FifteenMinuteHitRate"), new RatioGauge()
- {
- protected Ratio getRatio()
- {
- return Ratio.of(hits.getFifteenMinuteRate(), requests.getFifteenMinuteRate());
- }
- });
- size = Metrics.register(factory.createMetricName("Size"), new Gauge<Long>()
- {
- public Long getValue()
- {
- return cache.weightedSize();
- }
- });
- entries = Metrics.register(factory.createMetricName("Entries"), new Gauge<Integer>()
- {
- public Integer getValue()
- {
- return cache.size();
- }
- });
+ };
}
}
diff --git a/src/java/org/apache/cassandra/metrics/CacheMissMetrics.java b/src/java/org/apache/cassandra/metrics/CacheMissMetrics.java
deleted file mode 100644
index 19d61ef..0000000
--- a/src/java/org/apache/cassandra/metrics/CacheMissMetrics.java
+++ /dev/null
@@ -1,114 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.metrics;
-
-import com.codahale.metrics.Gauge;
-import com.codahale.metrics.Meter;
-import com.codahale.metrics.RatioGauge;
-import com.codahale.metrics.Timer;
-import org.apache.cassandra.cache.CacheSize;
-
-import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics;
-
-/**
- * Metrics for {@code ICache}.
- */
-public class CacheMissMetrics
-{
- /** Cache capacity in bytes */
- public final Gauge<Long> capacity;
- /** Total number of cache hits */
- public final Meter misses;
- /** Total number of cache requests */
- public final Meter requests;
- /** Latency of misses */
- public final Timer missLatency;
- /** all time cache hit rate */
- public final Gauge<Double> hitRate;
- /** 1m hit rate */
- public final Gauge<Double> oneMinuteHitRate;
- /** 5m hit rate */
- public final Gauge<Double> fiveMinuteHitRate;
- /** 15m hit rate */
- public final Gauge<Double> fifteenMinuteHitRate;
- /** Total size of cache, in bytes */
- public final Gauge<Long> size;
- /** Total number of cache entries */
- public final Gauge<Integer> entries;
-
- /**
- * Create metrics for given cache.
- *
- * @param type Type of Cache to identify metrics.
- * @param cache Cache to measure metrics
- */
- public CacheMissMetrics(String type, final CacheSize cache)
- {
- MetricNameFactory factory = new DefaultNameFactory("Cache", type);
-
- capacity = Metrics.register(factory.createMetricName("Capacity"), (Gauge<Long>) cache::capacity);
- misses = Metrics.meter(factory.createMetricName("Misses"));
- requests = Metrics.meter(factory.createMetricName("Requests"));
- missLatency = Metrics.timer(factory.createMetricName("MissLatency"));
- hitRate = Metrics.register(factory.createMetricName("HitRate"), new RatioGauge()
- {
- @Override
- public Ratio getRatio()
- {
- long req = requests.getCount();
- long mis = misses.getCount();
- return Ratio.of(req - mis, req);
- }
- });
- oneMinuteHitRate = Metrics.register(factory.createMetricName("OneMinuteHitRate"), new RatioGauge()
- {
- protected Ratio getRatio()
- {
- double req = requests.getOneMinuteRate();
- double mis = misses.getOneMinuteRate();
- return Ratio.of(req - mis, req);
- }
- });
- fiveMinuteHitRate = Metrics.register(factory.createMetricName("FiveMinuteHitRate"), new RatioGauge()
- {
- protected Ratio getRatio()
- {
- double req = requests.getFiveMinuteRate();
- double mis = misses.getFiveMinuteRate();
- return Ratio.of(req - mis, req);
- }
- });
- fifteenMinuteHitRate = Metrics.register(factory.createMetricName("FifteenMinuteHitRate"), new RatioGauge()
- {
- protected Ratio getRatio()
- {
- double req = requests.getFifteenMinuteRate();
- double mis = misses.getFifteenMinuteRate();
- return Ratio.of(req - mis, req);
- }
- });
- size = Metrics.register(factory.createMetricName("Size"), (Gauge<Long>) cache::weightedSize);
- entries = Metrics.register(factory.createMetricName("Entries"), (Gauge<Integer>) cache::size);
- }
-
- public void reset()
- {
- requests.mark(-requests.getCount());
- misses.mark(-misses.getCount());
- }
-}
diff --git a/src/java/org/apache/cassandra/metrics/CassandraMetricsRegistry.java b/src/java/org/apache/cassandra/metrics/CassandraMetricsRegistry.java
index 2181f5c..74c3367 100644
--- a/src/java/org/apache/cassandra/metrics/CassandraMetricsRegistry.java
+++ b/src/java/org/apache/cassandra/metrics/CassandraMetricsRegistry.java
@@ -18,13 +18,19 @@
package org.apache.cassandra.metrics;
import java.lang.reflect.Method;
+import java.util.Collection;
+import java.util.Collections;
import java.util.Locale;
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
-
-import com.codahale.metrics.*;
import javax.management.MalformedObjectNameException;
import javax.management.ObjectName;
+import com.google.common.annotations.VisibleForTesting;
+
+import com.codahale.metrics.*;
import org.apache.cassandra.utils.MBeanWrapper;
/**
@@ -36,6 +42,7 @@
public class CassandraMetricsRegistry extends MetricRegistry
{
public static final CassandraMetricsRegistry Metrics = new CassandraMetricsRegistry();
+ private final Map<String, ThreadPoolMetrics> threadPoolMetrics = new ConcurrentHashMap<>();
private final MBeanWrapper mBeanServer = MBeanWrapper.instance;
@@ -119,6 +126,27 @@
}
}
+ public Collection<ThreadPoolMetrics> allThreadPoolMetrics()
+ {
+ return Collections.unmodifiableCollection(threadPoolMetrics.values());
+ }
+
+ public Optional<ThreadPoolMetrics> getThreadPoolMetrics(String poolName)
+ {
+ return Optional.ofNullable(threadPoolMetrics.get(poolName));
+ }
+
+ ThreadPoolMetrics register(ThreadPoolMetrics metrics)
+ {
+ threadPoolMetrics.put(metrics.poolName, metrics);
+ return metrics;
+ }
+
+ void remove(ThreadPoolMetrics metrics)
+ {
+ threadPoolMetrics.remove(metrics.poolName, metrics);
+ }
+
public <T extends Metric> T register(MetricName name, MetricName aliasName, T metric)
{
T ret = register(name, metric);
@@ -130,11 +158,7 @@
{
boolean removed = remove(name.getMetricName());
- try
- {
- mBeanServer.unregisterMBean(name.getMBeanName());
- } catch (Exception ignore) {}
-
+ mBeanServer.unregisterMBean(name.getMBeanName(), MBeanWrapper.OnException.IGNORE);
return removed;
}
@@ -153,30 +177,20 @@
AbstractBean mbean;
if (metric instanceof Gauge)
- {
mbean = new JmxGauge((Gauge<?>) metric, name);
- } else if (metric instanceof Counter)
- {
+ else if (metric instanceof Counter)
mbean = new JmxCounter((Counter) metric, name);
- } else if (metric instanceof Histogram)
- {
+ else if (metric instanceof Histogram)
mbean = new JmxHistogram((Histogram) metric, name);
- } else if (metric instanceof Meter)
- {
- mbean = new JmxMeter((Meter) metric, name, TimeUnit.SECONDS);
- } else if (metric instanceof Timer)
- {
+ else if (metric instanceof Timer)
mbean = new JmxTimer((Timer) metric, name, TimeUnit.SECONDS, TimeUnit.MICROSECONDS);
- } else
- {
+ else if (metric instanceof Metered)
+ mbean = new JmxMeter((Metered) metric, name, TimeUnit.SECONDS);
+ else
throw new IllegalArgumentException("Unknown metric type: " + metric.getClass());
- }
- try
- {
- mBeanServer.registerMBean(mbean, name);
- }
- catch (Exception ignored) {}
+ if (!mBeanServer.isRegistered(name))
+ mBeanServer.registerMBean(mbean, name, MBeanWrapper.OnException.LOG);
}
private void registerAlias(MetricName existingName, MetricName aliasName)
@@ -189,10 +203,8 @@
private void removeAlias(MetricName name)
{
- try
- {
- MBeanWrapper.instance.unregisterMBean(name.getMBeanName());
- } catch (Exception ignored) {}
+ if (mBeanServer.isRegistered(name.getMBeanName()))
+ MBeanWrapper.instance.unregisterMBean(name.getMBeanName(), MBeanWrapper.OnException.IGNORE);
}
/**
@@ -276,11 +288,14 @@
double get999thPercentile();
long[] values();
+
+ long[] getRecentValues();
}
private static class JmxHistogram extends AbstractBean implements JmxHistogramMBean
{
private final Histogram metric;
+ private long[] last = null;
private JmxHistogram(Histogram metric, ObjectName objectName)
{
@@ -359,6 +374,15 @@
{
return metric.getSnapshot().getValues();
}
+
+ @Override
+ public long[] getRecentValues()
+ {
+ long[] now = metric.getSnapshot().getValues();
+ long[] delta = delta(now, last);
+ last = now;
+ return delta;
+ }
}
public interface JmxCounterMBean extends MetricMBean
@@ -479,6 +503,8 @@
long[] values();
+ long[] getRecentValues();
+
String getDurationUnit();
}
@@ -487,6 +513,7 @@
private final Timer metric;
private final double durationFactor;
private final String durationUnit;
+ private long[] last = null;
private JmxTimer(Timer metric,
ObjectName objectName,
@@ -566,6 +593,15 @@
}
@Override
+ public long[] getRecentValues()
+ {
+ long[] now = metric.getSnapshot().getValues();
+ long[] delta = delta(now, last);
+ last = now;
+ return delta;
+ }
+
+ @Override
public String getDurationUnit()
{
return durationUnit;
@@ -573,6 +609,28 @@
}
/**
+ * Used to determine the changes in a histogram since the last time checked.
+ *
+ * @param now The current histogram
+ * @param last The previous value of the histogram
+ * @return the difference between <i>now</> and <i>last</i>
+ */
+ @VisibleForTesting
+ static long[] delta(long[] now, long[] last)
+ {
+ long[] delta = new long[now.length];
+ if (last == null)
+ {
+ last = new long[now.length];
+ }
+ for(int i = 0; i< now.length; i++)
+ {
+ delta[i] = now[i] - (i < last.length? last[i] : 0);
+ }
+ return delta;
+ }
+
+ /**
* A value class encapsulating a metric's owning class and name.
*/
public static class MetricName implements Comparable<MetricName>
diff --git a/src/java/org/apache/cassandra/metrics/ChunkCacheMetrics.java b/src/java/org/apache/cassandra/metrics/ChunkCacheMetrics.java
new file mode 100644
index 0000000..a3a6928
--- /dev/null
+++ b/src/java/org/apache/cassandra/metrics/ChunkCacheMetrics.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.metrics;
+
+import java.util.concurrent.TimeUnit;
+import javax.annotation.Nonnull;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import com.codahale.metrics.Timer;
+import com.github.benmanes.caffeine.cache.stats.CacheStats;
+import com.github.benmanes.caffeine.cache.stats.StatsCounter;
+import org.apache.cassandra.cache.ChunkCache;
+
+import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics;
+
+/**
+ * Metrics for {@code ICache}.
+ */
+public class ChunkCacheMetrics extends CacheMetrics implements StatsCounter
+{
+ /** Latency of misses */
+ public final Timer missLatency;
+
+ /**
+ * Create metrics for the provided chunk cache.
+ *
+ * @param cache Chunk cache to measure metrics
+ */
+ public ChunkCacheMetrics(ChunkCache cache)
+ {
+ super("ChunkCache", cache);
+ missLatency = Metrics.timer(factory.createMetricName("MissLatency"));
+ }
+
+ @Override
+ public void recordHits(int count)
+ {
+ hits.mark(count);
+ }
+
+ @Override
+ public void recordMisses(int count)
+ {
+ misses.mark(count);
+ }
+
+ @Override
+ public void recordLoadSuccess(long loadTime)
+ {
+ missLatency.update(loadTime, TimeUnit.NANOSECONDS);
+ }
+
+ @Override
+ public void recordLoadFailure(long loadTime)
+ {
+ }
+
+ @Override
+ public void recordEviction()
+ {
+ }
+
+ @Nonnull
+ @Override
+ public CacheStats snapshot()
+ {
+ return new CacheStats(hits.getCount(), misses.getCount(), missLatency.getCount(), 0L, missLatency.getCount(), 0L, 0L);
+ }
+
+ @VisibleForTesting
+ public void reset()
+ {
+ hits.mark(-hits.getCount());
+ misses.mark(-misses.getCount());
+ }
+}
diff --git a/src/java/org/apache/cassandra/metrics/ClientMetrics.java b/src/java/org/apache/cassandra/metrics/ClientMetrics.java
index 67aa05b..7599096 100644
--- a/src/java/org/apache/cassandra/metrics/ClientMetrics.java
+++ b/src/java/org/apache/cassandra/metrics/ClientMetrics.java
@@ -18,27 +18,29 @@
*/
package org.apache.cassandra.metrics;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.concurrent.Callable;
+import java.util.*;
import java.util.concurrent.atomic.AtomicInteger;
import com.codahale.metrics.Gauge;
import com.codahale.metrics.Meter;
+import org.apache.cassandra.transport.ClientStat;
+import org.apache.cassandra.transport.ConnectedClient;
import org.apache.cassandra.transport.Server;
import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics;
-
-public class ClientMetrics
+public final class ClientMetrics
{
- private static final MetricNameFactory factory = new DefaultNameFactory("Client");
public static final ClientMetrics instance = new ClientMetrics();
- private volatile boolean initialized = false;
+ private static final MetricNameFactory factory = new DefaultNameFactory("Client");
+ private volatile boolean initialized = false;
private Collection<Server> servers = Collections.emptyList();
+ private Meter authSuccess;
+ private Meter authFailure;
+
private AtomicInteger pausedConnections;
private Gauge<Integer> pausedConnectionsGauge;
private Meter requestDiscarded;
@@ -47,10 +49,31 @@
{
}
+ public void markAuthSuccess()
+ {
+ authSuccess.mark();
+ }
+
+ public void markAuthFailure()
+ {
+ authFailure.mark();
+ }
+
public void pauseConnection() { pausedConnections.incrementAndGet(); }
public void unpauseConnection() { pausedConnections.decrementAndGet(); }
+
public void markRequestDiscarded() { requestDiscarded.mark(); }
+ public List<ConnectedClient> allConnectedClients()
+ {
+ List<ConnectedClient> clients = new ArrayList<>();
+
+ for (Server server : servers)
+ clients.addAll(server.getConnectedClients());
+
+ return clients;
+ }
+
public synchronized void init(Collection<Server> servers)
{
if (initialized)
@@ -58,7 +81,13 @@
this.servers = servers;
- registerGauge("connectedNativeClients", this::countConnectedClients);
+ registerGauge("connectedNativeClients", this::countConnectedClients);
+ registerGauge("connectedNativeClientsByUser", this::countConnectedClientsByUser);
+ registerGauge("connections", this::connectedClients);
+ registerGauge("clientsByProtocolVersion", this::recentClientStats);
+
+ authSuccess = registerMeter("AuthSuccess");
+ authFailure = registerMeter("AuthFailure");
pausedConnections = new AtomicInteger();
pausedConnectionsGauge = registerGauge("PausedConnections", pausedConnections::get);
@@ -67,35 +96,59 @@
initialized = true;
}
- public void addCounter(String name, final Callable<Integer> provider)
- {
- Metrics.register(factory.createMetricName(name), (Gauge<Integer>) () -> {
- try
- {
- return provider.call();
- } catch (Exception e)
- {
- throw new RuntimeException(e);
- }
- });
- }
-
private int countConnectedClients()
{
int count = 0;
for (Server server : servers)
- count += server.getConnectedClients();
+ count += server.countConnectedClients();
return count;
}
+ private Map<String, Integer> countConnectedClientsByUser()
+ {
+ Map<String, Integer> counts = new HashMap<>();
+
+ for (Server server : servers)
+ {
+ server.countConnectedClientsByUser()
+ .forEach((username, count) -> counts.put(username, counts.getOrDefault(username, 0) + count));
+ }
+
+ return counts;
+ }
+
+ private List<Map<String, String>> connectedClients()
+ {
+ List<Map<String, String>> clients = new ArrayList<>();
+
+ for (Server server : servers)
+ for (ConnectedClient client : server.getConnectedClients())
+ clients.add(client.asMap());
+
+ return clients;
+ }
+
+ private List<Map<String, String>> recentClientStats()
+ {
+ List<Map<String, String>> stats = new ArrayList<>();
+
+ for (Server server : servers)
+ for (ClientStat stat : server.recentClientStats())
+ stats.add(stat.asMap());
+
+ stats.sort(Comparator.comparing(map -> map.get(ClientStat.PROTOCOL_VERSION)));
+
+ return stats;
+ }
+
private <T> Gauge<T> registerGauge(String name, Gauge<T> gauge)
{
return Metrics.register(factory.createMetricName(name), gauge);
}
- public Meter registerMeter(String name)
+ private Meter registerMeter(String name)
{
return Metrics.meter(factory.createMetricName(name));
}
diff --git a/src/java/org/apache/cassandra/metrics/ClientRequestSizeMetrics.java b/src/java/org/apache/cassandra/metrics/ClientRequestSizeMetrics.java
new file mode 100644
index 0000000..41fb162
--- /dev/null
+++ b/src/java/org/apache/cassandra/metrics/ClientRequestSizeMetrics.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.metrics;
+
+
+import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics;
+
+import com.codahale.metrics.Counter;
+import com.codahale.metrics.Histogram;
+
+/**
+ * Metrics to track the size of incoming and outgoing bytes at Cassandra server.
+ */
+public class ClientRequestSizeMetrics
+{
+ private static final String TYPE = "ClientRequestSize";
+ public static final Counter totalBytesRead = Metrics.counter(DefaultNameFactory.createMetricName(TYPE, "IncomingBytes", null));
+ public static final Counter totalBytesWritten = Metrics.counter(DefaultNameFactory.createMetricName(TYPE, "OutgoingBytes", null));
+ public static final Histogram bytesRecievedPerFrame = Metrics.histogram(DefaultNameFactory.createMetricName(TYPE, "BytesRecievedPerFrame", null), true);
+ public static final Histogram bytesTransmittedPerFrame = Metrics.histogram(DefaultNameFactory.createMetricName(TYPE, "BytesTransmittedPerFrame", null), true);
+}
diff --git a/src/java/org/apache/cassandra/metrics/ClientWriteRequestMetrics.java b/src/java/org/apache/cassandra/metrics/ClientWriteRequestMetrics.java
new file mode 100644
index 0000000..50427af
--- /dev/null
+++ b/src/java/org/apache/cassandra/metrics/ClientWriteRequestMetrics.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.cassandra.metrics;
+
+import com.codahale.metrics.Histogram;
+
+import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics;
+
+/**
+ * Metrics for tracking information about write requests.
+ *
+ */
+public class ClientWriteRequestMetrics extends ClientRequestMetrics
+{
+ /**
+ * Metric for tracking the mutation sizes in bytes.
+ */
+ public final Histogram mutationSize;
+
+ public ClientWriteRequestMetrics(String scope)
+ {
+ super(scope);
+ mutationSize = Metrics.histogram(factory.createMetricName("MutationSizeHistogram"), false);
+ }
+
+ public void release()
+ {
+ super.release();
+ Metrics.remove(factory.createMetricName("MutationSizeHistogram"));
+ }
+}
diff --git a/src/java/org/apache/cassandra/metrics/CompactionMetrics.java b/src/java/org/apache/cassandra/metrics/CompactionMetrics.java
index 9aef0f8..46e5940 100644
--- a/src/java/org/apache/cassandra/metrics/CompactionMetrics.java
+++ b/src/java/org/apache/cassandra/metrics/CompactionMetrics.java
@@ -24,25 +24,24 @@
import com.codahale.metrics.Gauge;
import com.codahale.metrics.Meter;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.Schema;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.compaction.ActiveCompactions;
import org.apache.cassandra.db.compaction.CompactionInfo;
import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.TableMetadata;
import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics;
/**
* Metrics for compaction.
*/
-public class CompactionMetrics implements CompactionManager.CompactionExecutorStatsCollector
+public class CompactionMetrics
{
public static final MetricNameFactory factory = new DefaultNameFactory("Compaction");
- // a synchronized identity set of running tasks to their compaction info
- private static final Set<CompactionInfo.Holder> compactions = Collections.synchronizedSet(Collections.newSetFromMap(new IdentityHashMap<CompactionInfo.Holder, Boolean>()));
-
/** Estimated number of compactions remaining to perform */
public final Gauge<Integer> pendingTasks;
/** Estimated number of compactions remaining to perform, group by keyspace and then table name */
@@ -55,6 +54,16 @@
/** Total number of bytes compacted since server [re]start */
public final Counter bytesCompacted;
+
+ /** Total number of compactions that have had sstables drop out of them */
+ public final Counter compactionsReduced;
+
+ /** Total number of sstables that have been dropped out */
+ public final Counter sstablesDropppedFromCompactions;
+
+ /** Total number of compactions which have outright failed due to lack of disk space */
+ public final Counter compactionsAborted;
+
public CompactionMetrics(final ThreadPoolExecutor... collectors)
{
pendingTasks = Metrics.register(factory.createMetricName("PendingTasks"), new Gauge<Integer>()
@@ -69,7 +78,7 @@
n += cfs.getCompactionStrategyManager().getEstimatedRemainingTasks();
}
// add number of currently running compactions
- return n + compactions.size();
+ return n + CompactionManager.instance.active.getCompactions().size();
}
});
@@ -98,27 +107,27 @@
}
// currently running compactions
- for (CompactionInfo.Holder compaction : compactions)
+ for (CompactionInfo.Holder compaction : CompactionManager.instance.active.getCompactions())
{
- CFMetaData metaData = compaction.getCompactionInfo().getCFMetaData();
+ TableMetadata metaData = compaction.getCompactionInfo().getTableMetadata();
if (metaData == null)
{
continue;
}
- if (!resultMap.containsKey(metaData.ksName))
+ if (!resultMap.containsKey(metaData.keyspace))
{
- resultMap.put(metaData.ksName, new HashMap<>());
+ resultMap.put(metaData.keyspace, new HashMap<>());
}
- Map<String, Integer> tableNameToCountMap = resultMap.get(metaData.ksName);
- if (tableNameToCountMap.containsKey(metaData.cfName))
+ Map<String, Integer> tableNameToCountMap = resultMap.get(metaData.keyspace);
+ if (tableNameToCountMap.containsKey(metaData.name))
{
- tableNameToCountMap.put(metaData.cfName,
- tableNameToCountMap.get(metaData.cfName) + 1);
+ tableNameToCountMap.put(metaData.name,
+ tableNameToCountMap.get(metaData.name) + 1);
}
else
{
- tableNameToCountMap.put(metaData.cfName, 1);
+ tableNameToCountMap.put(metaData.name, 1);
}
}
return resultMap;
@@ -137,22 +146,10 @@
});
totalCompactionsCompleted = Metrics.meter(factory.createMetricName("TotalCompactionsCompleted"));
bytesCompacted = Metrics.counter(factory.createMetricName("BytesCompacted"));
- }
- public void beginCompaction(CompactionInfo.Holder ci)
- {
- compactions.add(ci);
- }
-
- public void finishCompaction(CompactionInfo.Holder ci)
- {
- compactions.remove(ci);
- bytesCompacted.inc(ci.getCompactionInfo().getTotal());
- totalCompactionsCompleted.mark();
- }
-
- public static List<CompactionInfo.Holder> getCompactions()
- {
- return new ArrayList<CompactionInfo.Holder>(compactions);
+ // compaction failure metrics
+ compactionsReduced = Metrics.counter(factory.createMetricName("CompactionsReduced"));
+ sstablesDropppedFromCompactions = Metrics.counter(factory.createMetricName("SSTablesDroppedFromCompaction"));
+ compactionsAborted = Metrics.counter(factory.createMetricName("CompactionsAborted"));
}
}
diff --git a/src/java/org/apache/cassandra/metrics/ConnectionMetrics.java b/src/java/org/apache/cassandra/metrics/ConnectionMetrics.java
deleted file mode 100644
index f01c06d..0000000
--- a/src/java/org/apache/cassandra/metrics/ConnectionMetrics.java
+++ /dev/null
@@ -1,157 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.metrics;
-
-import java.net.InetAddress;
-
-import com.codahale.metrics.Gauge;
-import com.codahale.metrics.Meter;
-
-import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics;
-
-
-import org.apache.cassandra.net.OutboundTcpConnectionPool;
-
-/**
- * Metrics for {@link OutboundTcpConnectionPool}.
- */
-public class ConnectionMetrics
-{
- public static final String TYPE_NAME = "Connection";
-
- /** Total number of timeouts happened on this node */
- public static final Meter totalTimeouts = Metrics.meter(DefaultNameFactory.createMetricName(TYPE_NAME, "TotalTimeouts", null));
-
- public final String address;
- /** Pending tasks for large message TCP Connections */
- public final Gauge<Integer> largeMessagePendingTasks;
- /** Completed tasks for large message TCP Connections */
- public final Gauge<Long> largeMessageCompletedTasks;
- /** Dropped tasks for large message TCP Connections */
- public final Gauge<Long> largeMessageDroppedTasks;
- /** Pending tasks for small message TCP Connections */
- public final Gauge<Integer> smallMessagePendingTasks;
- /** Completed tasks for small message TCP Connections */
- public final Gauge<Long> smallMessageCompletedTasks;
- /** Dropped tasks for small message TCP Connections */
- public final Gauge<Long> smallMessageDroppedTasks;
- /** Pending tasks for gossip message TCP Connections */
- public final Gauge<Integer> gossipMessagePendingTasks;
- /** Completed tasks for gossip message TCP Connections */
- public final Gauge<Long> gossipMessageCompletedTasks;
- /** Dropped tasks for gossip message TCP Connections */
- public final Gauge<Long> gossipMessageDroppedTasks;
-
- /** Number of timeouts for specific IP */
- public final Meter timeouts;
-
- private final MetricNameFactory factory;
-
- /**
- * Create metrics for given connection pool.
- *
- * @param ip IP address to use for metrics label
- * @param connectionPool Connection pool
- */
- public ConnectionMetrics(InetAddress ip, final OutboundTcpConnectionPool connectionPool)
- {
- // ipv6 addresses will contain colons, which are invalid in a JMX ObjectName
- address = ip.getHostAddress().replace(':', '.');
-
- factory = new DefaultNameFactory("Connection", address);
-
- largeMessagePendingTasks = Metrics.register(factory.createMetricName("LargeMessagePendingTasks"), new Gauge<Integer>()
- {
- public Integer getValue()
- {
- return connectionPool.largeMessages.getPendingMessages();
- }
- });
- largeMessageCompletedTasks = Metrics.register(factory.createMetricName("LargeMessageCompletedTasks"), new Gauge<Long>()
- {
- public Long getValue()
- {
- return connectionPool.largeMessages.getCompletedMesssages();
- }
- });
- largeMessageDroppedTasks = Metrics.register(factory.createMetricName("LargeMessageDroppedTasks"), new Gauge<Long>()
- {
- public Long getValue()
- {
- return connectionPool.largeMessages.getDroppedMessages();
- }
- });
- smallMessagePendingTasks = Metrics.register(factory.createMetricName("SmallMessagePendingTasks"), new Gauge<Integer>()
- {
- public Integer getValue()
- {
- return connectionPool.smallMessages.getPendingMessages();
- }
- });
- smallMessageCompletedTasks = Metrics.register(factory.createMetricName("SmallMessageCompletedTasks"), new Gauge<Long>()
- {
- public Long getValue()
- {
- return connectionPool.smallMessages.getCompletedMesssages();
- }
- });
- smallMessageDroppedTasks = Metrics.register(factory.createMetricName("SmallMessageDroppedTasks"), new Gauge<Long>()
- {
- public Long getValue()
- {
- return connectionPool.smallMessages.getDroppedMessages();
- }
- });
- gossipMessagePendingTasks = Metrics.register(factory.createMetricName("GossipMessagePendingTasks"), new Gauge<Integer>()
- {
- public Integer getValue()
- {
- return connectionPool.gossipMessages.getPendingMessages();
- }
- });
- gossipMessageCompletedTasks = Metrics.register(factory.createMetricName("GossipMessageCompletedTasks"), new Gauge<Long>()
- {
- public Long getValue()
- {
- return connectionPool.gossipMessages.getCompletedMesssages();
- }
- });
- gossipMessageDroppedTasks = Metrics.register(factory.createMetricName("GossipMessageDroppedTasks"), new Gauge<Long>()
- {
- public Long getValue()
- {
- return connectionPool.gossipMessages.getDroppedMessages();
- }
- });
- timeouts = Metrics.meter(factory.createMetricName("Timeouts"));
- }
-
- public void release()
- {
- Metrics.remove(factory.createMetricName("LargeMessagePendingTasks"));
- Metrics.remove(factory.createMetricName("LargeMessageCompletedTasks"));
- Metrics.remove(factory.createMetricName("LargeMessageDroppedTasks"));
- Metrics.remove(factory.createMetricName("SmallMessagePendingTasks"));
- Metrics.remove(factory.createMetricName("SmallMessageCompletedTasks"));
- Metrics.remove(factory.createMetricName("SmallMessageDroppedTasks"));
- Metrics.remove(factory.createMetricName("GossipMessagePendingTasks"));
- Metrics.remove(factory.createMetricName("GossipMessageCompletedTasks"));
- Metrics.remove(factory.createMetricName("GossipMessageDroppedTasks"));
- Metrics.remove(factory.createMetricName("Timeouts"));
- }
-}
diff --git a/src/java/org/apache/cassandra/metrics/DecayingEstimatedHistogramReservoir.java b/src/java/org/apache/cassandra/metrics/DecayingEstimatedHistogramReservoir.java
index 1bf9dbc..6dd1687 100644
--- a/src/java/org/apache/cassandra/metrics/DecayingEstimatedHistogramReservoir.java
+++ b/src/java/org/apache/cassandra/metrics/DecayingEstimatedHistogramReservoir.java
@@ -21,25 +21,28 @@
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;
-import java.nio.charset.Charset;
-import java.util.Arrays;
+import java.nio.charset.StandardCharsets;
+import java.util.Objects;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLongArray;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.primitives.Ints;
import com.codahale.metrics.Clock;
import com.codahale.metrics.Reservoir;
import com.codahale.metrics.Snapshot;
import org.apache.cassandra.utils.EstimatedHistogram;
+import static java.lang.Math.max;
+import static java.lang.Math.min;
+
/**
* A decaying histogram reservoir where values collected during each minute will be twice as significant as the values
* collected in the previous minute. Measured values are collected in variable sized buckets, using small buckets in the
* lower range and larger buckets in the upper range. Use this histogram when you want to know if the distribution of
* the underlying data stream has changed recently and you want high resolution on values in the lower range.
- *
+ * <p/>
* The histogram use forward decay [1] to make recent values more significant. The forward decay factor will be doubled
* every minute (half-life time set to 60 seconds) [2]. The forward decay landmark is reset every 30 minutes (or at
* first read/update after 30 minutes). During landmark reset, updates and reads in the reservoir will be blocked in a
@@ -47,42 +50,95 @@
* assumption that in an extreme case we would have to collect a metric 1M times for a single bucket each second. By the
* end of the 30:th minute all collected values will roughly add up to 1.000.000 * 60 * pow(2, 30) which can be
* represented with 56 bits giving us some head room in a signed 64 bit long.
- *
- * Internally two reservoirs are maintained, one with decay and one without decay. All public getters in a {@Snapshot}
+ * <p/>
+ * Internally two reservoirs are maintained, one with decay and one without decay. All public getters in a {@link Snapshot}
* will expose the decay functionality with the exception of the {@link Snapshot#getValues()} which will return values
* from the reservoir without decay. This makes it possible for the caller to maintain precise deltas in an interval of
- * its choise.
- *
+ * its choice.
+ * <p/>
* The bucket size starts at 1 and grows by 1.2 each time (rounding and removing duplicates). It goes from 1 to around
* 18T by default (creating 164+1 buckets), which will give a timing resolution from microseconds to roughly 210 days,
* with less precision as the numbers get larger.
- *
+ * <p/>
* The series of values to which the counts in `decayingBuckets` correspond:
* 1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 14, 17, 20, 24, 29, 35, 42, 50, 60, 72 etc.
* Thus, a `decayingBuckets` of [0, 0, 1, 10] would mean we had seen 1 value of 3 and 10 values of 4.
- *
+ * <p/>
* Each bucket represents values from (previous bucket offset, current offset].
- *
- * [1]: http://dimacs.rutgers.edu/~graham/pubs/papers/fwddecay.pdf
- * [2]: https://en.wikipedia.org/wiki/Half-life
- * [3]: https://github.com/dropwizard/metrics/blob/v3.1.2/metrics-core/src/main/java/com/codahale/metrics/ExponentiallyDecayingReservoir.java
+ * <p/>
+ * To reduce contention each logical bucket is striped accross a configurable number of stripes (default: 2). Threads are
+ * assigned to specific stripes. In addition, logical buckets are distributed across the physical storage to reduce conention
+ * when logically adjacent buckets are updated. See CASSANDRA-15213.
+ * <p/>
+ * <ul>
+ * <li>[1]: http://dimacs.rutgers.edu/~graham/pubs/papers/fwddecay.pdf</li>
+ * <li>[2]: https://en.wikipedia.org/wiki/Half-life</li>
+ * <li>[3]: https://github.com/dropwizard/metrics/blob/v3.1.2/metrics-core/src/main/java/com/codahale/metrics/ExponentiallyDecayingReservoir.java</li>
+ * </ul>
*/
public class DecayingEstimatedHistogramReservoir implements Reservoir
{
+
/**
* The default number of decayingBuckets. Use this bucket count to reduce memory allocation for bucket offsets.
*/
public static final int DEFAULT_BUCKET_COUNT = 164;
+ public static final int DEFAULT_STRIPE_COUNT = Integer.parseInt(System.getProperty("cassandra.dehr_stripe_count", "2"));
+ public static final int MAX_BUCKET_COUNT = 237;
public static final boolean DEFAULT_ZERO_CONSIDERATION = false;
+ private static final int[] DISTRIBUTION_PRIMES = new int[] { 17, 19, 23, 29 };
+
// The offsets used with a default sized bucket array without a separate bucket for zero values.
public static final long[] DEFAULT_WITHOUT_ZERO_BUCKET_OFFSETS = EstimatedHistogram.newOffsets(DEFAULT_BUCKET_COUNT, false);
// The offsets used with a default sized bucket array with a separate bucket for zero values.
public static final long[] DEFAULT_WITH_ZERO_BUCKET_OFFSETS = EstimatedHistogram.newOffsets(DEFAULT_BUCKET_COUNT, true);
+ private static final int TABLE_BITS = 4;
+ private static final int TABLE_MASK = -1 >>> (32 - TABLE_BITS);
+ private static final float[] LOG2_TABLE = computeTable(TABLE_BITS);
+ private static final float log2_12_recp = (float) (1d / slowLog2(1.2d));
+
+ private static float[] computeTable(int bits)
+ {
+ float[] table = new float[1 << bits];
+ for (int i = 1 ; i < 1<<bits ; ++i)
+ table[i] = (float) slowLog2(ratio(i, bits));
+ return table;
+ }
+
+ public static float fastLog12(long v)
+ {
+ return fastLog2(v) * log2_12_recp;
+ }
+
+ // returns 0 for all inputs <= 1
+ private static float fastLog2(long v)
+ {
+ v = max(v, 1);
+ int highestBitPosition = 63 - Long.numberOfLeadingZeros(v);
+ v = Long.rotateRight(v, highestBitPosition - TABLE_BITS);
+ int index = (int) (v & TABLE_MASK);
+ float result = LOG2_TABLE[index];
+ result += highestBitPosition;
+ return result;
+ }
+
+ private static double slowLog2(double v)
+ {
+ return Math.log(v) / Math.log(2);
+ }
+
+ private static double ratio(int i, int bits)
+ {
+ return Float.intBitsToFloat((127 << 23) | (i << (23 - bits)));
+ }
+
// Represents the bucket offset as created by {@link EstimatedHistogram#newOffsets()}
+ private final int nStripes;
private final long[] bucketOffsets;
+ private final int distributionPrime;
// decayingBuckets and buckets are one element longer than bucketOffsets -- the last element is values greater than the last offset
private final AtomicLongArray decayingBuckets;
@@ -95,8 +151,6 @@
private final AtomicBoolean rescaling = new AtomicBoolean(false);
private volatile long decayLandmark;
- private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
-
// Wrapper around System.nanoTime() to simplify unit testing.
private final Clock clock;
@@ -106,7 +160,7 @@
*/
public DecayingEstimatedHistogramReservoir()
{
- this(DEFAULT_ZERO_CONSIDERATION, DEFAULT_BUCKET_COUNT, Clock.defaultClock());
+ this(DEFAULT_ZERO_CONSIDERATION, DEFAULT_BUCKET_COUNT, DEFAULT_STRIPE_COUNT, Clock.defaultClock());
}
/**
@@ -117,7 +171,7 @@
*/
public DecayingEstimatedHistogramReservoir(boolean considerZeroes)
{
- this(considerZeroes, DEFAULT_BUCKET_COUNT, Clock.defaultClock());
+ this(considerZeroes, DEFAULT_BUCKET_COUNT, DEFAULT_STRIPE_COUNT, Clock.defaultClock());
}
/**
@@ -127,14 +181,22 @@
* same bucket as 1-value measurements
* @param bucketCount number of buckets used to collect measured values
*/
- public DecayingEstimatedHistogramReservoir(boolean considerZeroes, int bucketCount)
+ public DecayingEstimatedHistogramReservoir(boolean considerZeroes, int bucketCount, int stripes)
{
- this(considerZeroes, bucketCount, Clock.defaultClock());
+ this(considerZeroes, bucketCount, stripes, Clock.defaultClock());
}
@VisibleForTesting
- DecayingEstimatedHistogramReservoir(boolean considerZeroes, int bucketCount, Clock clock)
+ public DecayingEstimatedHistogramReservoir(Clock clock)
{
+ this(DEFAULT_ZERO_CONSIDERATION, DEFAULT_BUCKET_COUNT, DEFAULT_STRIPE_COUNT, clock);
+ }
+
+ @VisibleForTesting
+ DecayingEstimatedHistogramReservoir(boolean considerZeroes, int bucketCount, int stripes, Clock clock)
+ {
+ assert bucketCount <= MAX_BUCKET_COUNT : "bucket count cannot exceed: " + MAX_BUCKET_COUNT;
+
if (bucketCount == DEFAULT_BUCKET_COUNT)
{
if (considerZeroes == true)
@@ -150,10 +212,22 @@
{
bucketOffsets = EstimatedHistogram.newOffsets(bucketCount, considerZeroes);
}
- decayingBuckets = new AtomicLongArray(bucketOffsets.length + 1);
- buckets = new AtomicLongArray(bucketOffsets.length + 1);
+
+ nStripes = stripes;
+ decayingBuckets = new AtomicLongArray((bucketOffsets.length + 1) * nStripes);
+ buckets = new AtomicLongArray((bucketOffsets.length + 1) * nStripes);
this.clock = clock;
decayLandmark = clock.getTime();
+ int distributionPrime = 1;
+ for (int prime : DISTRIBUTION_PRIMES)
+ {
+ if (buckets.length() % prime != 0)
+ {
+ distributionPrime = prime;
+ break;
+ }
+ }
+ this.distributionPrime = distributionPrime;
}
/**
@@ -166,45 +240,70 @@
long now = clock.getTime();
rescaleIfNeeded(now);
- int index = Arrays.binarySearch(bucketOffsets, value);
- if (index < 0)
- {
- // inexact match, take the first bucket higher than n
- index = -index - 1;
- }
- // else exact match; we're good
+ int index = findIndex(bucketOffsets, value);
- lockForRegularUsage();
+ updateBucket(decayingBuckets, index, Math.round(forwardDecayWeight(now)));
+ updateBucket(buckets, index, 1);
+ }
- try
- {
- decayingBuckets.getAndAdd(index, Math.round(forwardDecayWeight(now)));
- }
- finally
- {
- unlockForRegularUsage();
- }
+ public void updateBucket(AtomicLongArray buckets, int index, long value)
+ {
+ int stripe = (int) (Thread.currentThread().getId() & (nStripes - 1));
+ buckets.addAndGet(stripedIndex(index, stripe), value);
+ }
- buckets.getAndIncrement(index);
+ public int stripedIndex(int offsetIndex, int stripe)
+ {
+ return (((offsetIndex * nStripes + stripe) * distributionPrime) % buckets.length());
+ }
+
+ @VisibleForTesting
+ public static int findIndex(long[] bucketOffsets, long value)
+ {
+ // values below zero are nonsense, but we have never failed when presented them
+ value = max(value, 0);
+
+ // The bucket index can be estimated using the equation Math.floor(Math.log(value) / Math.log(1.2))
+
+ // By using an integer domain we effectively squeeze multiple exponents of 1.2 into the same bucket,
+ // so for values > 2, we must "subtract" these exponents from the logarithm to determine which two buckets
+ // to consult (as our approximation otherwise produces a value that is within 1 of the true value)
+ int offset = (value > 2 ? 3 : 1) + (int)bucketOffsets[0];
+
+ // See DecayingEstimatedHistogramResevoirTest#showEstimationWorks and DecayingEstimatedHistogramResevoirTest#testFindIndex()
+ // for a runnable "proof"
+ //
+ // With this assumption, the estimate is calculated and the furthest offset from the estimation is checked
+ // if this bucket does not contain the value then the next one will
+
+ int firstCandidate = max(0, min(bucketOffsets.length - 1, ((int) fastLog12(value)) - offset));
+ return value <= bucketOffsets[firstCandidate] ? firstCandidate : firstCandidate + 1;
}
private double forwardDecayWeight(long now)
{
- return Math.exp(((now - decayLandmark) / 1000L) / MEAN_LIFETIME_IN_S);
+ return Math.exp(((now - decayLandmark) / 1000.0) / MEAN_LIFETIME_IN_S);
}
/**
- * Return the number of buckets where recorded values are stored.
+ * Returns the logical number of buckets where recorded values are stored. The actual number of physical buckets
+ * is size() * stripeCount()
*
* This method does not return the number of recorded values as suggested by the {@link Reservoir} interface.
*
* @return the number of buckets
+ * @see #stripeCount()
*/
public int size()
{
- return decayingBuckets.length();
+ return bucketOffsets.length + 1;
}
+
+ public int stripeCount()
+ {
+ return nStripes;
+ }
/**
* Returns a snapshot of the decaying values in this reservoir.
*
@@ -215,17 +314,7 @@
public Snapshot getSnapshot()
{
rescaleIfNeeded();
-
- lockForRegularUsage();
-
- try
- {
- return new EstimatedHistogramReservoirSnapshot(this);
- }
- finally
- {
- unlockForRegularUsage();
- }
+ return new EstimatedHistogramReservoirSnapshot(this);
}
/**
@@ -234,7 +323,23 @@
@VisibleForTesting
boolean isOverflowed()
{
- return decayingBuckets.get(decayingBuckets.length() - 1) > 0;
+ return bucketValue(bucketOffsets.length, true) > 0;
+ }
+
+ private long bucketValue(int index, boolean withDecay)
+ {
+ long val = 0;
+ AtomicLongArray bs = withDecay ? decayingBuckets : buckets;
+ for (int stripe = 0; stripe < nStripes; stripe++)
+ val += bs.get(stripedIndex(index, stripe));
+
+ return val;
+ }
+
+ @VisibleForTesting
+ long stripedBucketValue(int i, boolean withDecay)
+ {
+ return withDecay ? decayingBuckets.get(i) : buckets.get(i);
}
private void rescaleIfNeeded()
@@ -254,6 +359,7 @@
}
finally
{
+ decayLandmark = now;
rescaling.set(false);
}
}
@@ -262,27 +368,12 @@
private void rescale(long now)
{
- // Check again to make sure that another thread didn't complete rescale already
- if (needRescale(now))
+ // despite striping its safe to rescale each bucket individually
+ final double rescaleFactor = forwardDecayWeight(now);
+ for (int i = 0; i < decayingBuckets.length(); i++)
{
- lockForRescale();
-
- try
- {
- final double rescaleFactor = forwardDecayWeight(now);
- decayLandmark = now;
-
- final int bucketCount = decayingBuckets.length();
- for (int i = 0; i < bucketCount; i++)
- {
- long newValue = Math.round((decayingBuckets.get(i) / rescaleFactor));
- decayingBuckets.set(i, newValue);
- }
- }
- finally
- {
- unlockForRescale();
- }
+ long newValue = Math.round(decayingBuckets.get(i) / rescaleFactor);
+ decayingBuckets.set(i, newValue);
}
}
@@ -294,46 +385,50 @@
@VisibleForTesting
public void clear()
{
- lockForRescale();
-
- try
+ final int bucketCount = decayingBuckets.length();
+ for (int i = 0; i < bucketCount; i++)
{
- final int bucketCount = decayingBuckets.length();
- for (int i = 0; i < bucketCount; i++)
+ decayingBuckets.set(i, 0L);
+ buckets.set(i, 0L);
+ }
+ }
+
+ /**
+ * Replaces current internal values with the given one from a Snapshot. This method is NOT thread safe, values
+ * added at the same time to this reservoir using methods such as update may lose their data
+ */
+ public void rebase(EstimatedHistogramReservoirSnapshot snapshot)
+ {
+ // Check bucket count (a snapshot always has one stripe so the logical bucket count is used
+ if (size() != snapshot.decayingBuckets.length)
+ {
+ throw new IllegalStateException("Unable to merge two DecayingEstimatedHistogramReservoirs with different bucket sizes");
+ }
+
+ // Check bucketOffsets
+ for (int i = 0; i < bucketOffsets.length; i++)
+ {
+ if (bucketOffsets[i] != snapshot.bucketOffsets[i])
{
- decayingBuckets.set(i, 0L);
- buckets.set(i, 0L);
+ throw new IllegalStateException("Merge is only supported with equal bucketOffsets");
}
}
- finally
+
+ this.decayLandmark = snapshot.snapshotLandmark;
+ for (int i = 0; i < size(); i++)
{
- unlockForRescale();
+ // set rebased values in the first stripe and clear out all other data
+ decayingBuckets.set(stripedIndex(i, 0), snapshot.decayingBuckets[i]);
+ buckets.set(stripedIndex(i, 0), snapshot.values[i]);
+ for (int stripe = 1; stripe < nStripes; stripe++)
+ {
+ decayingBuckets.set(stripedIndex(i, stripe), 0);
+ buckets.set(stripedIndex(i, stripe), 0);
+ }
}
+
}
- private void lockForRegularUsage()
- {
- this.lock.readLock().lock();
- }
-
- private void unlockForRegularUsage()
- {
- this.lock.readLock().unlock();
- }
-
- private void lockForRescale()
- {
- this.lock.writeLock().lock();
- }
-
- private void unlockForRescale()
- {
- this.lock.writeLock().unlock();
- }
-
-
- private static final Charset UTF_8 = Charset.forName("UTF-8");
-
/**
* Represents a snapshot of the decaying histogram.
*
@@ -344,19 +439,32 @@
* The decaying buckets will be used for quantile calculations and mean values, but the non decaying buckets will be
* exposed for calls to {@link Snapshot#getValues()}.
*/
- private class EstimatedHistogramReservoirSnapshot extends Snapshot
+ static class EstimatedHistogramReservoirSnapshot extends Snapshot
{
private final long[] decayingBuckets;
+ private final long[] values;
+ private long count;
+ private long snapshotLandmark;
+ private long[] bucketOffsets;
+ private DecayingEstimatedHistogramReservoir reservoir;
public EstimatedHistogramReservoirSnapshot(DecayingEstimatedHistogramReservoir reservoir)
{
- final int length = reservoir.decayingBuckets.length();
- final double rescaleFactor = forwardDecayWeight(clock.getTime());
+ final int length = reservoir.size();
+ final double rescaleFactor = reservoir.forwardDecayWeight(reservoir.clock.getTime());
this.decayingBuckets = new long[length];
+ this.values = new long[length];
+ this.snapshotLandmark = reservoir.decayLandmark;
+ this.bucketOffsets = reservoir.bucketOffsets; // No need to copy, these are immutable
for (int i = 0; i < length; i++)
- this.decayingBuckets[i] = Math.round(reservoir.decayingBuckets.get(i) / rescaleFactor);
+ {
+ this.decayingBuckets[i] = Math.round(reservoir.bucketValue(i, true) / rescaleFactor);
+ this.values[i] = reservoir.bucketValue(i, false);
+ }
+ this.count = count();
+ this.reservoir = reservoir;
}
/**
@@ -399,26 +507,31 @@
*/
public long[] getValues()
{
- final int length = buckets.length();
-
- long[] values = new long[length];
-
- for (int i = 0; i < length; i++)
- values[i] = buckets.get(i);
-
return values;
}
/**
- * Return the number of buckets where recorded values are stored.
- *
- * This method does not return the number of recorded values as suggested by the {@link Snapshot} interface.
- *
- * @return the number of buckets
+ * @see {@link Snapshot#size()}
+ * @return
*/
public int size()
{
- return decayingBuckets.length;
+ return Ints.saturatedCast(count);
+ }
+
+ @VisibleForTesting
+ public long getSnapshotLandmark()
+ {
+ return snapshotLandmark;
+ }
+
+ @VisibleForTesting
+ public Range getBucketingRangeForValue(long value)
+ {
+ int index = findIndex(bucketOffsets, value);
+ long max = bucketOffsets[index];
+ long min = index == 0 ? 0 : 1 + bucketOffsets[index - 1];
+ return new Range(min, max);
}
/**
@@ -437,7 +550,7 @@
/**
* Get the estimated max-value that could have been added to this reservoir.
*
- * As values are collected in variable sized buckets, the actual max value recored in the reservoir may be less
+ * As values are collected in variable sized buckets, the actual max value recorded in the reservoir may be less
* than the value returned.
*
* @return the largest value that could have been added to this reservoir, or Long.MAX_VALUE if the reservoir
@@ -486,7 +599,7 @@
/**
* Get the estimated min-value that could have been added to this reservoir.
*
- * As values are collected in variable sized buckets, the actual min value recored in the reservoir may be
+ * As values are collected in variable sized buckets, the actual min value recorded in the reservoir may be
* higher than the value returned.
*
* @return the smallest value that could have been added to this reservoir
@@ -540,7 +653,7 @@
public void dump(OutputStream output)
{
- try (PrintWriter out = new PrintWriter(new OutputStreamWriter(output, UTF_8)))
+ try (PrintWriter out = new PrintWriter(new OutputStreamWriter(output, StandardCharsets.UTF_8)))
{
int length = decayingBuckets.length;
@@ -550,5 +663,94 @@
}
}
}
+
+ /**
+ * Adds another DecayingEstimatedHistogramReservoir's Snapshot to this one. Both reservoirs must have same bucket definitions. This will rescale both snapshots if needed.
+ *
+ * @param other EstimatedHistogramReservoirSnapshot with identical bucket definition (offsets and length)
+ */
+ public void add(Snapshot other)
+ {
+ if (!(other instanceof EstimatedHistogramReservoirSnapshot))
+ {
+ throw new IllegalStateException("Unable to add other types of Snapshot than another DecayingEstimatedHistogramReservoir");
+ }
+
+ EstimatedHistogramReservoirSnapshot snapshot = (EstimatedHistogramReservoirSnapshot) other;
+
+ if (decayingBuckets.length != snapshot.decayingBuckets.length)
+ {
+ throw new IllegalStateException("Unable to merge two DecayingEstimatedHistogramReservoirs with different bucket sizes");
+ }
+
+ // Check bucketOffsets
+ for (int i = 0; i < bucketOffsets.length; i++)
+ {
+ if (bucketOffsets[i] != snapshot.bucketOffsets[i])
+ {
+ throw new IllegalStateException("Merge is only supported with equal bucketOffsets");
+ }
+ }
+
+ // We need to rescale the reservoirs to the same landmark
+ if (snapshot.snapshotLandmark < snapshotLandmark)
+ {
+ rescaleArray(snapshot.decayingBuckets, (snapshotLandmark - snapshot.snapshotLandmark));
+ }
+ else if (snapshot.snapshotLandmark > snapshotLandmark)
+ {
+ rescaleArray(decayingBuckets, (snapshot.snapshotLandmark - snapshotLandmark));
+ this.snapshotLandmark = snapshot.snapshotLandmark;
+ }
+
+ // Now merge the buckets
+ for (int i = 0; i < snapshot.decayingBuckets.length; i++)
+ {
+ decayingBuckets[i] += snapshot.decayingBuckets[i];
+ values[i] += snapshot.values[i];
+ }
+
+ this.count += snapshot.count;
+ }
+
+ private void rescaleArray(long[] decayingBuckets, long landMarkDifference)
+ {
+ final double rescaleFactor = Math.exp((landMarkDifference / 1000.0) / MEAN_LIFETIME_IN_S);
+ for (int i = 0; i < decayingBuckets.length; i++)
+ {
+ decayingBuckets[i] = Math.round(decayingBuckets[i] / rescaleFactor);
+ }
+ }
+
+ public void rebaseReservoir()
+ {
+ this.reservoir.rebase(this);
+ }
+ }
+
+ static class Range
+ {
+ public final long min;
+ public final long max;
+
+ public Range(long min, long max)
+ {
+ this.min = min;
+ this.max = max;
+ }
+
+ public boolean equals(Object o)
+ {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ Range that = (Range) o;
+ return min == that.min &&
+ max == that.max;
+ }
+
+ public int hashCode()
+ {
+ return Objects.hash(min, max);
+ }
}
}
diff --git a/src/java/org/apache/cassandra/metrics/DroppedMessageMetrics.java b/src/java/org/apache/cassandra/metrics/DroppedMessageMetrics.java
index 794fa9c..8c22778 100644
--- a/src/java/org/apache/cassandra/metrics/DroppedMessageMetrics.java
+++ b/src/java/org/apache/cassandra/metrics/DroppedMessageMetrics.java
@@ -21,6 +21,7 @@
import com.codahale.metrics.Timer;
import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.net.Verb;
import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics;
@@ -38,7 +39,7 @@
/** The cross node dropped latency */
public final Timer crossNodeDroppedLatency;
- public DroppedMessageMetrics(MessagingService.Verb verb)
+ public DroppedMessageMetrics(Verb verb)
{
this(new DefaultNameFactory("DroppedMessage", verb.toString()));
}
diff --git a/src/java/org/apache/cassandra/metrics/FrequencySampler.java b/src/java/org/apache/cassandra/metrics/FrequencySampler.java
new file mode 100644
index 0000000..8a8918b
--- /dev/null
+++ b/src/java/org/apache/cassandra/metrics/FrequencySampler.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.metrics;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.clearspring.analytics.stream.StreamSummary;
+
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+
+/**
+ * Find the most frequent sample. A sample adds to the sum of its key ie
+ * <p>add("x", 10); and add("x", 20); will result in "x" = 30</p> This uses StreamSummary to only store the
+ * approximate cardinality (capacity) of keys. If the number of distinct keys exceed the capacity, the error of the
+ * sample may increase depending on distribution of keys among the total set.
+ *
+ * @param <T>
+ */
+public abstract class FrequencySampler<T> extends Sampler<T>
+{
+ private static final Logger logger = LoggerFactory.getLogger(FrequencySampler.class);
+ private long endTimeNanos = -1;
+
+ private StreamSummary<T> summary;
+
+ /**
+ * Start to record samples
+ *
+ * @param capacity
+ * Number of sample items to keep in memory, the lower this is
+ * the less accurate results are. For best results use value
+ * close to cardinality, but understand the memory trade offs.
+ */
+ public synchronized void beginSampling(int capacity, int durationMillis)
+ {
+ if (endTimeNanos == -1 || clock.now() > endTimeNanos)
+ {
+ summary = new StreamSummary<>(capacity);
+ endTimeNanos = clock.now() + MILLISECONDS.toNanos(durationMillis);
+ }
+ else
+ throw new RuntimeException("Sampling already in progress");
+ }
+
+ /**
+ * Call to stop collecting samples, and gather the results
+ * @param count Number of most frequent items to return
+ */
+ public synchronized List<Sample<T>> finishSampling(int count)
+ {
+ List<Sample<T>> results = Collections.emptyList();
+ if (endTimeNanos != -1)
+ {
+ endTimeNanos = -1;
+ results = summary.topK(count)
+ .stream()
+ .map(c -> new Sample<T>(c.getItem(), c.getCount(), c.getError()))
+ .collect(Collectors.toList());
+ }
+ return results;
+ }
+
+ protected synchronized void insert(final T item, final long value)
+ {
+ // samplerExecutor is single threaded but still need
+ // synchronization against jmx calls to finishSampling
+ if (value > 0 && clock.now() <= endTimeNanos)
+ {
+ try
+ {
+ summary.offer(item, (int) Math.min(value, Integer.MAX_VALUE));
+ } catch (Exception e)
+ {
+ logger.trace("Failure to offer sample", e);
+ }
+ }
+ }
+
+ public boolean isEnabled()
+ {
+ return endTimeNanos != -1 && clock.now() <= endTimeNanos;
+ }
+
+}
+
diff --git a/src/java/org/apache/cassandra/metrics/HintedHandoffMetrics.java b/src/java/org/apache/cassandra/metrics/HintedHandoffMetrics.java
index 51f6569..56888da 100644
--- a/src/java/org/apache/cassandra/metrics/HintedHandoffMetrics.java
+++ b/src/java/org/apache/cassandra/metrics/HintedHandoffMetrics.java
@@ -17,19 +17,20 @@
*/
package org.apache.cassandra.metrics;
-import java.net.InetAddress;
import java.util.Map.Entry;
+import com.google.common.util.concurrent.MoreExecutors;
+
import com.codahale.metrics.Counter;
+import com.github.benmanes.caffeine.cache.Caffeine;
+import com.github.benmanes.caffeine.cache.LoadingCache;
+
import org.apache.cassandra.db.SystemKeyspace;
+import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.utils.UUIDGen;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.CacheLoader;
-import com.google.common.cache.LoadingCache;
-
import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics;
/**
@@ -42,36 +43,28 @@
private static final MetricNameFactory factory = new DefaultNameFactory("HintedHandOffManager");
/** Total number of hints which are not stored, This is not a cache. */
- private final LoadingCache<InetAddress, DifferencingCounter> notStored = CacheBuilder.newBuilder().build(new CacheLoader<InetAddress, DifferencingCounter>()
- {
- public DifferencingCounter load(InetAddress address)
- {
- return new DifferencingCounter(address);
- }
- });
+ private final LoadingCache<InetAddressAndPort, DifferencingCounter> notStored = Caffeine.newBuilder()
+ .executor(MoreExecutors.directExecutor())
+ .build(DifferencingCounter::new);
/** Total number of hints that have been created, This is not a cache. */
- private final LoadingCache<InetAddress, Counter> createdHintCounts = CacheBuilder.newBuilder().build(new CacheLoader<InetAddress, Counter>()
- {
- public Counter load(InetAddress address)
- {
- return Metrics.counter(factory.createMetricName("Hints_created-" + address.getHostAddress().replace(':', '.')));
- }
- });
+ private final LoadingCache<InetAddressAndPort, Counter> createdHintCounts = Caffeine.newBuilder()
+ .executor(MoreExecutors.directExecutor())
+ .build(address -> Metrics.counter(factory.createMetricName("Hints_created-" + address.toString().replace(':', '.'))));
- public void incrCreatedHints(InetAddress address)
+ public void incrCreatedHints(InetAddressAndPort address)
{
- createdHintCounts.getUnchecked(address).inc();
+ createdHintCounts.get(address).inc();
}
- public void incrPastWindow(InetAddress address)
+ public void incrPastWindow(InetAddressAndPort address)
{
- notStored.getUnchecked(address).mark();
+ notStored.get(address).mark();
}
public void log()
{
- for (Entry<InetAddress, DifferencingCounter> entry : notStored.asMap().entrySet())
+ for (Entry<InetAddressAndPort, DifferencingCounter> entry : notStored.asMap().entrySet())
{
long difference = entry.getValue().difference();
if (difference == 0)
@@ -86,9 +79,10 @@
private final Counter meter;
private long reported = 0;
- public DifferencingCounter(InetAddress address)
+ public DifferencingCounter(InetAddressAndPort address)
{
- this.meter = Metrics.counter(factory.createMetricName("Hints_not_stored-" + address.getHostAddress().replace(':', '.')));
+ //This changes the name of the metric, people can update their monitoring when upgrading?
+ this.meter = Metrics.counter(factory.createMetricName("Hints_not_stored-" + address.toString().replace(':', '.')));
}
public long difference()
diff --git a/src/java/org/apache/cassandra/metrics/HintsServiceMetrics.java b/src/java/org/apache/cassandra/metrics/HintsServiceMetrics.java
index ad85281..424f502 100644
--- a/src/java/org/apache/cassandra/metrics/HintsServiceMetrics.java
+++ b/src/java/org/apache/cassandra/metrics/HintsServiceMetrics.java
@@ -17,7 +17,15 @@
*/
package org.apache.cassandra.metrics;
+import com.google.common.util.concurrent.MoreExecutors;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.codahale.metrics.Histogram;
import com.codahale.metrics.Meter;
+import com.github.benmanes.caffeine.cache.Caffeine;
+import com.github.benmanes.caffeine.cache.LoadingCache;
+import org.apache.cassandra.locator.InetAddressAndPort;
import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics;
@@ -26,9 +34,31 @@
*/
public final class HintsServiceMetrics
{
+ private static final Logger logger = LoggerFactory.getLogger(HintsServiceMetrics.class);
+
private static final MetricNameFactory factory = new DefaultNameFactory("HintsService");
public static final Meter hintsSucceeded = Metrics.meter(factory.createMetricName("HintsSucceeded"));
public static final Meter hintsFailed = Metrics.meter(factory.createMetricName("HintsFailed"));
public static final Meter hintsTimedOut = Metrics.meter(factory.createMetricName("HintsTimedOut"));
+
+ /** Histogram of all hint delivery delays */
+ private static final Histogram globalDelayHistogram = Metrics.histogram(factory.createMetricName("Hint_delays"), false);
+
+ /** Histograms per-endpoint of hint delivery delays, This is not a cache. */
+ private static final LoadingCache<InetAddressAndPort, Histogram> delayByEndpoint = Caffeine.newBuilder()
+ .executor(MoreExecutors.directExecutor())
+ .build(address -> Metrics.histogram(factory.createMetricName("Hint_delays-"+address.toString().replace(':', '.')), false));
+
+ public static void updateDelayMetrics(InetAddressAndPort endpoint, long delay)
+ {
+ if (delay <= 0)
+ {
+ logger.warn("Invalid negative latency in hint delivery delay: {}", delay);
+ return;
+ }
+
+ globalDelayHistogram.update(delay);
+ delayByEndpoint.get(endpoint).update(delay);
+ }
}
diff --git a/src/java/org/apache/cassandra/metrics/InternodeInboundMetrics.java b/src/java/org/apache/cassandra/metrics/InternodeInboundMetrics.java
new file mode 100644
index 0000000..cc3c1c0
--- /dev/null
+++ b/src/java/org/apache/cassandra/metrics/InternodeInboundMetrics.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.metrics;
+
+import com.codahale.metrics.Gauge;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.net.InboundMessageHandlers;
+import org.apache.cassandra.metrics.CassandraMetricsRegistry.MetricName;
+
+/**
+ * Metrics for internode connections.
+ */
+public class InternodeInboundMetrics
+{
+ private final MetricName corruptFramesRecovered;
+ private final MetricName corruptFramesUnrecovered;
+ private final MetricName errorBytes;
+ private final MetricName errorCount;
+ private final MetricName expiredBytes;
+ private final MetricName expiredCount;
+ private final MetricName pendingBytes;
+ private final MetricName pendingCount;
+ private final MetricName processedBytes;
+ private final MetricName processedCount;
+ private final MetricName receivedBytes;
+ private final MetricName receivedCount;
+ private final MetricName throttledCount;
+ private final MetricName throttledNanos;
+
+ /**
+ * Create metrics for given inbound message handlers.
+ *
+ * @param peer IP address and port to use for metrics label
+ */
+ public InternodeInboundMetrics(InetAddressAndPort peer, InboundMessageHandlers handlers)
+ {
+ // ipv6 addresses will contain colons, which are invalid in a JMX ObjectName
+ MetricNameFactory factory = new DefaultNameFactory("InboundConnection", peer.toString().replace(':', '_'));
+
+ register(corruptFramesRecovered = factory.createMetricName("CorruptFramesRecovered"), handlers::corruptFramesRecovered);
+ register(corruptFramesUnrecovered = factory.createMetricName("CorruptFramesUnrecovered"), handlers::corruptFramesUnrecovered);
+ register(errorBytes = factory.createMetricName("ErrorBytes"), handlers::errorBytes);
+ register(errorCount = factory.createMetricName("ErrorCount"), handlers::errorCount);
+ register(expiredBytes = factory.createMetricName("ExpiredBytes"), handlers::expiredBytes);
+ register(expiredCount = factory.createMetricName("ExpiredCount"), handlers::expiredCount);
+ register(pendingBytes = factory.createMetricName("ScheduledBytes"), handlers::scheduledBytes);
+ register(pendingCount = factory.createMetricName("ScheduledCount"), handlers::scheduledCount);
+ register(processedBytes = factory.createMetricName("ProcessedBytes"), handlers::processedBytes);
+ register(processedCount = factory.createMetricName("ProcessedCount"), handlers::processedCount);
+ register(receivedBytes = factory.createMetricName("ReceivedBytes"), handlers::receivedBytes);
+ register(receivedCount = factory.createMetricName("ReceivedCount"), handlers::receivedCount);
+ register(throttledCount = factory.createMetricName("ThrottledCount"), handlers::throttledCount);
+ register(throttledNanos = factory.createMetricName("ThrottledNanos"), handlers::throttledNanos);
+ }
+
+ public void release()
+ {
+ remove(corruptFramesRecovered);
+ remove(corruptFramesUnrecovered);
+ remove(errorBytes);
+ remove(errorCount);
+ remove(expiredBytes);
+ remove(expiredCount);
+ remove(pendingBytes);
+ remove(pendingCount);
+ remove(processedBytes);
+ remove(processedCount);
+ remove(receivedBytes);
+ remove(receivedCount);
+ remove(throttledCount);
+ remove(throttledNanos);
+ }
+
+ private static void register(MetricName name, Gauge gauge)
+ {
+ CassandraMetricsRegistry.Metrics.register(name, gauge);
+ }
+
+ private static void remove(MetricName name)
+ {
+ CassandraMetricsRegistry.Metrics.remove(name);
+ }
+}
diff --git a/src/java/org/apache/cassandra/metrics/InternodeOutboundMetrics.java b/src/java/org/apache/cassandra/metrics/InternodeOutboundMetrics.java
new file mode 100644
index 0000000..f04b428
--- /dev/null
+++ b/src/java/org/apache/cassandra/metrics/InternodeOutboundMetrics.java
@@ -0,0 +1,205 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.metrics;
+
+import com.codahale.metrics.Gauge;
+import com.codahale.metrics.Meter;
+import org.apache.cassandra.net.OutboundConnections;
+
+import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics;
+
+import org.apache.cassandra.locator.InetAddressAndPort;
+
+/**
+ * Metrics for internode connections.
+ */
+public class InternodeOutboundMetrics
+{
+ public static final String TYPE_NAME = "Connection";
+
+ /** Total number of callbacks that were not completed successfully for messages that were sent to this node
+ * TODO this was always broken, as it never counted those messages without callbacks? So perhaps we can redefine it. */
+ public static final Meter totalExpiredCallbacks = Metrics.meter(DefaultNameFactory.createMetricName(TYPE_NAME, "TotalTimeouts", null));
+
+ /** Number of timeouts for specific IP */
+ public final Meter expiredCallbacks;
+
+ public final String address;
+ /** Pending tasks for large message TCP Connections */
+ public final Gauge<Integer> largeMessagePendingTasks;
+ /** Pending bytes for large message TCP Connections */
+ public final Gauge<Long> largeMessagePendingBytes;
+ /** Completed tasks for large message TCP Connections */
+ public final Gauge<Long> largeMessageCompletedTasks;
+ /** Completed bytes for large message TCP Connections */
+ public final Gauge<Long> largeMessageCompletedBytes;
+ /** Dropped tasks for large message TCP Connections */
+ public final Gauge<Long> largeMessageDropped;
+ /** Dropped tasks because of timeout for large message TCP Connections */
+ public final Gauge<Long> largeMessageDroppedTasksDueToTimeout;
+ /** Dropped bytes because of timeout for large message TCP Connections */
+ public final Gauge<Long> largeMessageDroppedBytesDueToTimeout;
+ /** Dropped tasks because of overload for large message TCP Connections */
+ public final Gauge<Long> largeMessageDroppedTasksDueToOverload;
+ /** Dropped bytes because of overload for large message TCP Connections */
+ public final Gauge<Long> largeMessageDroppedBytesDueToOverload;
+ /** Dropped tasks because of error for large message TCP Connections */
+ public final Gauge<Long> largeMessageDroppedTasksDueToError;
+ /** Dropped bytes because of error for large message TCP Connections */
+ public final Gauge<Long> largeMessageDroppedBytesDueToError;
+ /** Pending tasks for small message TCP Connections */
+ public final Gauge<Integer> smallMessagePendingTasks;
+ /** Pending bytes for small message TCP Connections */
+ public final Gauge<Long> smallMessagePendingBytes;
+ /** Completed tasks for small message TCP Connections */
+ public final Gauge<Long> smallMessageCompletedTasks;
+ /** Completed bytes for small message TCP Connections */
+ public final Gauge<Long> smallMessageCompletedBytes;
+ /** Dropped tasks for small message TCP Connections */
+ public final Gauge<Long> smallMessageDroppedTasks;
+ /** Dropped tasks because of timeout for small message TCP Connections */
+ public final Gauge<Long> smallMessageDroppedTasksDueToTimeout;
+ /** Dropped bytes because of timeout for small message TCP Connections */
+ public final Gauge<Long> smallMessageDroppedBytesDueToTimeout;
+ /** Dropped tasks because of overload for small message TCP Connections */
+ public final Gauge<Long> smallMessageDroppedTasksDueToOverload;
+ /** Dropped bytes because of overload for small message TCP Connections */
+ public final Gauge<Long> smallMessageDroppedBytesDueToOverload;
+ /** Dropped tasks because of error for small message TCP Connections */
+ public final Gauge<Long> smallMessageDroppedTasksDueToError;
+ /** Dropped bytes because of error for small message TCP Connections */
+ public final Gauge<Long> smallMessageDroppedBytesDueToError;
+ /** Pending tasks for small message TCP Connections */
+ public final Gauge<Integer> urgentMessagePendingTasks;
+ /** Pending bytes for urgent message TCP Connections */
+ public final Gauge<Long> urgentMessagePendingBytes;
+ /** Completed tasks for urgent message TCP Connections */
+ public final Gauge<Long> urgentMessageCompletedTasks;
+ /** Completed bytes for urgent message TCP Connections */
+ public final Gauge<Long> urgentMessageCompletedBytes;
+ /** Dropped tasks for urgent message TCP Connections */
+ public final Gauge<Long> urgentMessageDroppedTasks;
+ /** Dropped tasks because of timeout for urgent message TCP Connections */
+ public final Gauge<Long> urgentMessageDroppedTasksDueToTimeout;
+ /** Dropped bytes because of timeout for urgent message TCP Connections */
+ public final Gauge<Long> urgentMessageDroppedBytesDueToTimeout;
+ /** Dropped tasks because of overload for urgent message TCP Connections */
+ public final Gauge<Long> urgentMessageDroppedTasksDueToOverload;
+ /** Dropped bytes because of overload for urgent message TCP Connections */
+ public final Gauge<Long> urgentMessageDroppedBytesDueToOverload;
+ /** Dropped tasks because of error for urgent message TCP Connections */
+ public final Gauge<Long> urgentMessageDroppedTasksDueToError;
+ /** Dropped bytes because of error for urgent message TCP Connections */
+ public final Gauge<Long> urgentMessageDroppedBytesDueToError;
+
+ private final MetricNameFactory factory;
+
+ /**
+ * Create metrics for given connection pool.
+ *
+ * @param ip IP address to use for metrics label
+ */
+ public InternodeOutboundMetrics(InetAddressAndPort ip, final OutboundConnections messagingPool)
+ {
+ // ipv6 addresses will contain colons, which are invalid in a JMX ObjectName
+ address = ip.toString().replace(':', '_');
+
+ factory = new DefaultNameFactory("Connection", address);
+
+ largeMessagePendingTasks = Metrics.register(factory.createMetricName("LargeMessagePendingTasks"), messagingPool.large::pendingCount);
+ largeMessagePendingBytes = Metrics.register(factory.createMetricName("LargeMessagePendingBytes"), messagingPool.large::pendingBytes);
+ largeMessageCompletedTasks = Metrics.register(factory.createMetricName("LargeMessageCompletedTasks"),messagingPool.large::sentCount);
+ largeMessageCompletedBytes = Metrics.register(factory.createMetricName("LargeMessageCompletedBytes"),messagingPool.large::sentBytes);
+ largeMessageDropped = Metrics.register(factory.createMetricName("LargeMessageDroppedTasks"), messagingPool.large::dropped);
+ largeMessageDroppedTasksDueToOverload = Metrics.register(factory.createMetricName("LargeMessageDroppedTasksDueToOverload"), messagingPool.large::overloadedCount);
+ largeMessageDroppedBytesDueToOverload = Metrics.register(factory.createMetricName("LargeMessageDroppedBytesDueToOverload"), messagingPool.large::overloadedBytes);
+ largeMessageDroppedTasksDueToTimeout = Metrics.register(factory.createMetricName("LargeMessageDroppedTasksDueToTimeout"), messagingPool.large::expiredCount);
+ largeMessageDroppedBytesDueToTimeout = Metrics.register(factory.createMetricName("LargeMessageDroppedBytesDueToTimeout"), messagingPool.large::expiredBytes);
+ largeMessageDroppedTasksDueToError = Metrics.register(factory.createMetricName("LargeMessageDroppedTasksDueToError"), messagingPool.large::errorCount);
+ largeMessageDroppedBytesDueToError = Metrics.register(factory.createMetricName("LargeMessageDroppedBytesDueToError"), messagingPool.large::errorBytes);
+ smallMessagePendingTasks = Metrics.register(factory.createMetricName("SmallMessagePendingTasks"), messagingPool.small::pendingCount);
+ smallMessagePendingBytes = Metrics.register(factory.createMetricName("SmallMessagePendingBytes"), messagingPool.small::pendingBytes);
+ smallMessageCompletedTasks = Metrics.register(factory.createMetricName("SmallMessageCompletedTasks"), messagingPool.small::sentCount);
+ smallMessageCompletedBytes = Metrics.register(factory.createMetricName("SmallMessageCompletedBytes"),messagingPool.small::sentBytes);
+ smallMessageDroppedTasks = Metrics.register(factory.createMetricName("SmallMessageDroppedTasks"), messagingPool.small::dropped);
+ smallMessageDroppedTasksDueToOverload = Metrics.register(factory.createMetricName("SmallMessageDroppedTasksDueToOverload"), messagingPool.small::overloadedCount);
+ smallMessageDroppedBytesDueToOverload = Metrics.register(factory.createMetricName("SmallMessageDroppedBytesDueToOverload"), messagingPool.small::overloadedBytes);
+ smallMessageDroppedTasksDueToTimeout = Metrics.register(factory.createMetricName("SmallMessageDroppedTasksDueToTimeout"), messagingPool.small::expiredCount);
+ smallMessageDroppedBytesDueToTimeout = Metrics.register(factory.createMetricName("SmallMessageDroppedBytesDueToTimeout"), messagingPool.small::expiredBytes);
+ smallMessageDroppedTasksDueToError = Metrics.register(factory.createMetricName("SmallMessageDroppedTasksDueToError"), messagingPool.small::errorCount);
+ smallMessageDroppedBytesDueToError = Metrics.register(factory.createMetricName("SmallMessageDroppedBytesDueToError"), messagingPool.small::errorBytes);
+ urgentMessagePendingTasks = Metrics.register(factory.createMetricName("UrgentMessagePendingTasks"), messagingPool.urgent::pendingCount);
+ urgentMessagePendingBytes = Metrics.register(factory.createMetricName("UrgentMessagePendingBytes"), messagingPool.urgent::pendingBytes);
+ urgentMessageCompletedTasks = Metrics.register(factory.createMetricName("UrgentMessageCompletedTasks"), messagingPool.urgent::sentCount);
+ urgentMessageCompletedBytes = Metrics.register(factory.createMetricName("UrgentMessageCompletedBytes"),messagingPool.urgent::sentBytes);
+ urgentMessageDroppedTasks = Metrics.register(factory.createMetricName("UrgentMessageDroppedTasks"), messagingPool.urgent::dropped);
+ urgentMessageDroppedTasksDueToOverload = Metrics.register(factory.createMetricName("UrgentMessageDroppedTasksDueToOverload"), messagingPool.urgent::overloadedCount);
+ urgentMessageDroppedBytesDueToOverload = Metrics.register(factory.createMetricName("UrgentMessageDroppedBytesDueToOverload"), messagingPool.urgent::overloadedBytes);
+ urgentMessageDroppedTasksDueToTimeout = Metrics.register(factory.createMetricName("UrgentMessageDroppedTasksDueToTimeout"), messagingPool.urgent::expiredCount);
+ urgentMessageDroppedBytesDueToTimeout = Metrics.register(factory.createMetricName("UrgentMessageDroppedBytesDueToTimeout"), messagingPool.urgent::expiredBytes);
+ urgentMessageDroppedTasksDueToError = Metrics.register(factory.createMetricName("UrgentMessageDroppedTasksDueToError"), messagingPool.urgent::errorCount);
+ urgentMessageDroppedBytesDueToError = Metrics.register(factory.createMetricName("UrgentMessageDroppedBytesDueToError"), messagingPool.urgent::errorBytes);
+ expiredCallbacks = Metrics.meter(factory.createMetricName("Timeouts"));
+
+ // deprecated
+ Metrics.register(factory.createMetricName("GossipMessagePendingTasks"), (Gauge<Integer>) messagingPool.urgent::pendingCount);
+ Metrics.register(factory.createMetricName("GossipMessageCompletedTasks"), (Gauge<Long>) messagingPool.urgent::sentCount);
+ Metrics.register(factory.createMetricName("GossipMessageDroppedTasks"), (Gauge<Long>) messagingPool.urgent::dropped);
+ }
+
+ public void release()
+ {
+ Metrics.remove(factory.createMetricName("LargeMessagePendingTasks"));
+ Metrics.remove(factory.createMetricName("LargeMessagePendingBytes"));
+ Metrics.remove(factory.createMetricName("LargeMessageCompletedTasks"));
+ Metrics.remove(factory.createMetricName("LargeMessageCompletedBytes"));
+ Metrics.remove(factory.createMetricName("LargeMessageDroppedTasks"));
+ Metrics.remove(factory.createMetricName("LargeMessageDroppedTasksDueToTimeout"));
+ Metrics.remove(factory.createMetricName("LargeMessageDroppedBytesDueToTimeout"));
+ Metrics.remove(factory.createMetricName("LargeMessageDroppedTasksDueToOverload"));
+ Metrics.remove(factory.createMetricName("LargeMessageDroppedBytesDueToOverload"));
+ Metrics.remove(factory.createMetricName("LargeMessageDroppedTasksDueToError"));
+ Metrics.remove(factory.createMetricName("LargeMessageDroppedBytesDueToError"));
+ Metrics.remove(factory.createMetricName("SmallMessagePendingTasks"));
+ Metrics.remove(factory.createMetricName("SmallMessagePendingBytes"));
+ Metrics.remove(factory.createMetricName("SmallMessageCompletedTasks"));
+ Metrics.remove(factory.createMetricName("SmallMessageCompletedBytes"));
+ Metrics.remove(factory.createMetricName("SmallMessageDroppedTasks"));
+ Metrics.remove(factory.createMetricName("SmallMessageDroppedTasksDueToTimeout"));
+ Metrics.remove(factory.createMetricName("SmallMessageDroppedBytesDueToTimeout"));
+ Metrics.remove(factory.createMetricName("SmallMessageDroppedTasksDueToOverload"));
+ Metrics.remove(factory.createMetricName("SmallMessageDroppedBytesDueToOverload"));
+ Metrics.remove(factory.createMetricName("SmallMessageDroppedTasksDueToError"));
+ Metrics.remove(factory.createMetricName("SmallMessageDroppedBytesDueToError"));
+ Metrics.remove(factory.createMetricName("GossipMessagePendingTasks"));
+ Metrics.remove(factory.createMetricName("GossipMessageCompletedTasks"));
+ Metrics.remove(factory.createMetricName("GossipMessageDroppedTasks"));
+ Metrics.remove(factory.createMetricName("UrgentMessagePendingTasks"));
+ Metrics.remove(factory.createMetricName("UrgentMessagePendingBytes"));
+ Metrics.remove(factory.createMetricName("UrgentMessageCompletedTasks"));
+ Metrics.remove(factory.createMetricName("UrgentMessageCompletedBytes"));
+ Metrics.remove(factory.createMetricName("UrgentMessageDroppedTasks"));
+ Metrics.remove(factory.createMetricName("UrgentMessageDroppedTasksDueToTimeout"));
+ Metrics.remove(factory.createMetricName("UrgentMessageDroppedBytesDueToTimeout"));
+ Metrics.remove(factory.createMetricName("UrgentMessageDroppedTasksDueToOverload"));
+ Metrics.remove(factory.createMetricName("UrgentMessageDroppedBytesDueToOverload"));
+ Metrics.remove(factory.createMetricName("UrgentMessageDroppedTasksDueToError"));
+ Metrics.remove(factory.createMetricName("UrgentMessageDroppedBytesDueToError"));
+ Metrics.remove(factory.createMetricName("Timeouts"));
+ }
+}
diff --git a/src/java/org/apache/cassandra/metrics/KeyspaceMetrics.java b/src/java/org/apache/cassandra/metrics/KeyspaceMetrics.java
index 2e1c384..4af26c0 100644
--- a/src/java/org/apache/cassandra/metrics/KeyspaceMetrics.java
+++ b/src/java/org/apache/cassandra/metrics/KeyspaceMetrics.java
@@ -18,14 +18,18 @@
package org.apache.cassandra.metrics;
import java.util.Set;
+import java.util.function.ToLongFunction;
+import com.codahale.metrics.Counter;
import com.codahale.metrics.Gauge;
import com.codahale.metrics.Histogram;
+import com.codahale.metrics.Meter;
import com.codahale.metrics.Timer;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.metrics.CassandraMetricsRegistry.MetricName;
+import org.apache.cassandra.metrics.TableMetrics.ReleasableMetric;
-import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics;
@@ -91,12 +95,69 @@
public final LatencyMetrics casPropose;
/** CAS Commit metrics */
public final LatencyMetrics casCommit;
+ /** Writes failed ideal consistency **/
+ public final Counter writeFailedIdealCL;
+ /** Ideal CL write latency metrics */
+ public final LatencyMetrics idealCLWriteLatency;
+ /** Speculative retries **/
+ public final Counter speculativeRetries;
+ /** Speculative retry occured but still timed out **/
+ public final Counter speculativeFailedRetries;
+ /** Needed to speculate, but didn't have enough replicas **/
+ public final Counter speculativeInsufficientReplicas;
+ /** Needed to write to a transient replica to satisfy quorum **/
+ public final Counter additionalWrites;
+ /** Number of started repairs as coordinator on this keyspace */
+ public final Counter repairsStarted;
+ /** Number of completed repairs as coordinator on this keyspace */
+ public final Counter repairsCompleted;
+ /** total time spent as a repair coordinator */
+ public final Timer repairTime;
+ /** total time spent preparing for repair */
+ public final Timer repairPrepareTime;
+ /** Time spent anticompacting */
+ public final Timer anticompactionTime;
+ /** total time spent creating merkle trees */
+ public final Timer validationTime;
+ /** total time spent syncing data after repair */
+ public final Timer repairSyncTime;
+ /** histogram over the number of bytes we have validated */
+ public final Histogram bytesValidated;
+ /** histogram over the number of partitions we have validated */
+ public final Histogram partitionsValidated;
+
+ /*
+ * Metrics for inconsistencies detected between repaired data sets across replicas. These
+ * are tracked on the coordinator.
+ */
+
+ /**
+ * Incremented where an inconsistency is detected and there are no pending repair sessions affecting
+ * the data being read, indicating a genuine mismatch between replicas' repaired data sets.
+ */
+ public final Meter confirmedRepairedInconsistencies;
+ /**
+ * Incremented where an inconsistency is detected, but there are pending & uncommitted repair sessions
+ * in play on at least one replica. This may indicate a false positive as the inconsistency could be due to
+ * replicas marking the repair session as committed at slightly different times and so some consider it to
+ * be part of the repaired set whilst others do not.
+ */
+ public final Meter unconfirmedRepairedInconsistencies;
+
+ /**
+ * Tracks the amount overreading of repaired data replicas perform in order to produce digests
+ * at query time. For each query, on a full data read following an initial digest mismatch, the replicas
+ * may read extra repaired data, up to the DataLimit of the command, so that the coordinator can compare
+ * the repaired data on each replica. These are tracked on each replica.
+ */
+ public final Histogram repairedDataTrackingOverreadRows;
+ public final Timer repairedDataTrackingOverreadTime;
public final MetricNameFactory factory;
private Keyspace keyspace;
/** set containing names of all the metrics stored here, for releasing later */
- private Set<String> allMetrics = Sets.newHashSet();
+ private Set<ReleasableMetric> allMetrics = Sets.newHashSet();
/**
* Creates metrics for given {@link ColumnFamilyStore}.
@@ -107,135 +168,73 @@
{
factory = new KeyspaceMetricNameFactory(ks);
keyspace = ks;
- memtableColumnsCount = createKeyspaceGauge("MemtableColumnsCount", new MetricValue()
- {
- public Long getValue(TableMetrics metric)
- {
- return metric.memtableColumnsCount.getValue();
- }
- });
- memtableLiveDataSize = createKeyspaceGauge("MemtableLiveDataSize", new MetricValue()
- {
- public Long getValue(TableMetrics metric)
- {
- return metric.memtableLiveDataSize.getValue();
- }
- });
- memtableOnHeapDataSize = createKeyspaceGauge("MemtableOnHeapDataSize", new MetricValue()
- {
- public Long getValue(TableMetrics metric)
- {
- return metric.memtableOnHeapSize.getValue();
- }
- });
- memtableOffHeapDataSize = createKeyspaceGauge("MemtableOffHeapDataSize", new MetricValue()
- {
- public Long getValue(TableMetrics metric)
- {
- return metric.memtableOffHeapSize.getValue();
- }
- });
- allMemtablesLiveDataSize = createKeyspaceGauge("AllMemtablesLiveDataSize", new MetricValue()
- {
- public Long getValue(TableMetrics metric)
- {
- return metric.allMemtablesLiveDataSize.getValue();
- }
- });
- allMemtablesOnHeapDataSize = createKeyspaceGauge("AllMemtablesOnHeapDataSize", new MetricValue()
- {
- public Long getValue(TableMetrics metric)
- {
- return metric.allMemtablesOnHeapSize.getValue();
- }
- });
- allMemtablesOffHeapDataSize = createKeyspaceGauge("AllMemtablesOffHeapDataSize", new MetricValue()
- {
- public Long getValue(TableMetrics metric)
- {
- return metric.allMemtablesOffHeapSize.getValue();
- }
- });
- memtableSwitchCount = createKeyspaceGauge("MemtableSwitchCount", new MetricValue()
- {
- public Long getValue(TableMetrics metric)
- {
- return metric.memtableSwitchCount.getCount();
- }
- });
- pendingCompactions = createKeyspaceGauge("PendingCompactions", new MetricValue()
- {
- public Long getValue(TableMetrics metric)
- {
- return (long) metric.pendingCompactions.getValue();
- }
- });
- pendingFlushes = createKeyspaceGauge("PendingFlushes", new MetricValue()
- {
- public Long getValue(TableMetrics metric)
- {
- return (long) metric.pendingFlushes.getCount();
- }
- });
- liveDiskSpaceUsed = createKeyspaceGauge("LiveDiskSpaceUsed", new MetricValue()
- {
- public Long getValue(TableMetrics metric)
- {
- return metric.liveDiskSpaceUsed.getCount();
- }
- });
- totalDiskSpaceUsed = createKeyspaceGauge("TotalDiskSpaceUsed", new MetricValue()
- {
- public Long getValue(TableMetrics metric)
- {
- return metric.totalDiskSpaceUsed.getCount();
- }
- });
- bloomFilterDiskSpaceUsed = createKeyspaceGauge("BloomFilterDiskSpaceUsed", new MetricValue()
- {
- public Long getValue(TableMetrics metric)
- {
- return metric.bloomFilterDiskSpaceUsed.getValue();
- }
- });
- bloomFilterOffHeapMemoryUsed = createKeyspaceGauge("BloomFilterOffHeapMemoryUsed", new MetricValue()
- {
- public Long getValue(TableMetrics metric)
- {
- return metric.bloomFilterOffHeapMemoryUsed.getValue();
- }
- });
- indexSummaryOffHeapMemoryUsed = createKeyspaceGauge("IndexSummaryOffHeapMemoryUsed", new MetricValue()
- {
- public Long getValue(TableMetrics metric)
- {
- return metric.indexSummaryOffHeapMemoryUsed.getValue();
- }
- });
- compressionMetadataOffHeapMemoryUsed = createKeyspaceGauge("CompressionMetadataOffHeapMemoryUsed", new MetricValue()
- {
- public Long getValue(TableMetrics metric)
- {
- return metric.compressionMetadataOffHeapMemoryUsed.getValue();
- }
- });
- // latency metrics for TableMetrics to update
- readLatency = new LatencyMetrics(factory, "Read");
- writeLatency = new LatencyMetrics(factory, "Write");
- rangeLatency = new LatencyMetrics(factory, "Range");
- // create histograms for TableMetrics to replicate updates to
- sstablesPerReadHistogram = Metrics.histogram(factory.createMetricName("SSTablesPerReadHistogram"), true);
- tombstoneScannedHistogram = Metrics.histogram(factory.createMetricName("TombstoneScannedHistogram"), false);
- liveScannedHistogram = Metrics.histogram(factory.createMetricName("LiveScannedHistogram"), false);
- colUpdateTimeDeltaHistogram = Metrics.histogram(factory.createMetricName("ColUpdateTimeDeltaHistogram"), false);
- viewLockAcquireTime = Metrics.timer(factory.createMetricName("ViewLockAcquireTime"));
- viewReadTime = Metrics.timer(factory.createMetricName("ViewReadTime"));
- // add manually since histograms do not use createKeyspaceGauge method
- allMetrics.addAll(Lists.newArrayList("SSTablesPerReadHistogram", "TombstoneScannedHistogram", "LiveScannedHistogram"));
+ memtableColumnsCount = createKeyspaceGauge("MemtableColumnsCount",
+ metric -> metric.memtableColumnsCount.getValue());
+ memtableLiveDataSize = createKeyspaceGauge("MemtableLiveDataSize",
+ metric -> metric.memtableLiveDataSize.getValue());
+ memtableOnHeapDataSize = createKeyspaceGauge("MemtableOnHeapDataSize",
+ metric -> metric.memtableOnHeapSize.getValue());
+ memtableOffHeapDataSize = createKeyspaceGauge("MemtableOffHeapDataSize",
+ metric -> metric.memtableOffHeapSize.getValue());
+ allMemtablesLiveDataSize = createKeyspaceGauge("AllMemtablesLiveDataSize",
+ metric -> metric.allMemtablesLiveDataSize.getValue());
+ allMemtablesOnHeapDataSize = createKeyspaceGauge("AllMemtablesOnHeapDataSize",
+ metric -> metric.allMemtablesOnHeapSize.getValue());
+ allMemtablesOffHeapDataSize = createKeyspaceGauge("AllMemtablesOffHeapDataSize",
+ metric -> metric.allMemtablesOffHeapSize.getValue());
+ memtableSwitchCount = createKeyspaceGauge("MemtableSwitchCount",
+ metric -> metric.memtableSwitchCount.getCount());
+ pendingCompactions = createKeyspaceGauge("PendingCompactions", metric -> metric.pendingCompactions.getValue());
+ pendingFlushes = createKeyspaceGauge("PendingFlushes", metric -> metric.pendingFlushes.getCount());
+ liveDiskSpaceUsed = createKeyspaceGauge("LiveDiskSpaceUsed", metric -> metric.liveDiskSpaceUsed.getCount());
+ totalDiskSpaceUsed = createKeyspaceGauge("TotalDiskSpaceUsed", metric -> metric.totalDiskSpaceUsed.getCount());
+ bloomFilterDiskSpaceUsed = createKeyspaceGauge("BloomFilterDiskSpaceUsed",
+ metric -> metric.bloomFilterDiskSpaceUsed.getValue());
+ bloomFilterOffHeapMemoryUsed = createKeyspaceGauge("BloomFilterOffHeapMemoryUsed",
+ metric -> metric.bloomFilterOffHeapMemoryUsed.getValue());
+ indexSummaryOffHeapMemoryUsed = createKeyspaceGauge("IndexSummaryOffHeapMemoryUsed",
+ metric -> metric.indexSummaryOffHeapMemoryUsed.getValue());
+ compressionMetadataOffHeapMemoryUsed = createKeyspaceGauge("CompressionMetadataOffHeapMemoryUsed",
+ metric -> metric.compressionMetadataOffHeapMemoryUsed.getValue());
- casPrepare = new LatencyMetrics(factory, "CasPrepare");
- casPropose = new LatencyMetrics(factory, "CasPropose");
- casCommit = new LatencyMetrics(factory, "CasCommit");
+ // latency metrics for TableMetrics to update
+ readLatency = createLatencyMetrics("Read");
+ writeLatency = createLatencyMetrics("Write");
+ rangeLatency = createLatencyMetrics("Range");
+
+ // create histograms for TableMetrics to replicate updates to
+ sstablesPerReadHistogram = createKeyspaceHistogram("SSTablesPerReadHistogram", true);
+ tombstoneScannedHistogram = createKeyspaceHistogram("TombstoneScannedHistogram", false);
+ liveScannedHistogram = createKeyspaceHistogram("LiveScannedHistogram", false);
+ colUpdateTimeDeltaHistogram = createKeyspaceHistogram("ColUpdateTimeDeltaHistogram", false);
+ viewLockAcquireTime = createKeyspaceTimer("ViewLockAcquireTime");
+ viewReadTime = createKeyspaceTimer("ViewReadTime");
+
+ casPrepare = createLatencyMetrics("CasPrepare");
+ casPropose = createLatencyMetrics("CasPropose");
+ casCommit = createLatencyMetrics("CasCommit");
+ writeFailedIdealCL = createKeyspaceCounter("WriteFailedIdealCL");
+ idealCLWriteLatency = createLatencyMetrics("IdealCLWrite");
+
+ speculativeRetries = createKeyspaceCounter("SpeculativeRetries", metric -> metric.speculativeRetries.getCount());
+ speculativeFailedRetries = createKeyspaceCounter("SpeculativeFailedRetries", metric -> metric.speculativeFailedRetries.getCount());
+ speculativeInsufficientReplicas = createKeyspaceCounter("SpeculativeInsufficientReplicas", metric -> metric.speculativeInsufficientReplicas.getCount());
+ additionalWrites = createKeyspaceCounter("AdditionalWrites", metric -> metric.additionalWrites.getCount());
+ repairsStarted = createKeyspaceCounter("RepairJobsStarted", metric -> metric.repairsStarted.getCount());
+ repairsCompleted = createKeyspaceCounter("RepairJobsCompleted", metric -> metric.repairsCompleted.getCount());
+ repairTime =createKeyspaceTimer("RepairTime");
+ repairPrepareTime = createKeyspaceTimer("RepairPrepareTime");
+ anticompactionTime = createKeyspaceTimer("AntiCompactionTime");
+ validationTime = createKeyspaceTimer("ValidationTime");
+ repairSyncTime = createKeyspaceTimer("RepairSyncTime");
+ partitionsValidated = createKeyspaceHistogram("PartitionsValidated", false);
+ bytesValidated = createKeyspaceHistogram("BytesValidated", false);
+
+ confirmedRepairedInconsistencies = createKeyspaceMeter("RepairedDataInconsistenciesConfirmed");
+ unconfirmedRepairedInconsistencies = createKeyspaceMeter("RepairedDataInconsistenciesUnconfirmed");
+
+ repairedDataTrackingOverreadRows = createKeyspaceHistogram("RepairedOverreadRows", false);
+ repairedDataTrackingOverreadTime = createKeyspaceTimer("RepairedOverreadTime");
}
/**
@@ -243,27 +242,10 @@
*/
public void release()
{
- for(String name : allMetrics)
+ for (ReleasableMetric metric : allMetrics)
{
- Metrics.remove(factory.createMetricName(name));
+ metric.release();
}
- // latency metrics contain multiple metrics internally and need to be released manually
- readLatency.release();
- writeLatency.release();
- rangeLatency.release();
- }
-
- /**
- * Represents a column family metric value.
- */
- private interface MetricValue
- {
- /**
- * get value of a metric
- * @param metric of a column family in this keyspace
- * @return current value of a metric
- */
- public Long getValue(TableMetrics metric);
}
/**
@@ -272,9 +254,9 @@
* @param extractor
* @return Gauge>Long> that computes sum of MetricValue.getValue()
*/
- private Gauge<Long> createKeyspaceGauge(String name, final MetricValue extractor)
+ private Gauge<Long> createKeyspaceGauge(String name, final ToLongFunction<TableMetrics> extractor)
{
- allMetrics.add(name);
+ allMetrics.add(() -> releaseMetric(name));
return Metrics.register(factory.createMetricName(name), new Gauge<Long>()
{
public Long getValue()
@@ -282,13 +264,73 @@
long sum = 0;
for (ColumnFamilyStore cf : keyspace.getColumnFamilyStores())
{
- sum += extractor.getValue(cf.metric);
+ sum += extractor.applyAsLong(cf.metric);
}
return sum;
}
});
}
+ /**
+ * Creates a counter that will sum the current value of a metric for all column families in this keyspace
+ * @param name
+ * @param extractor
+ * @return Counter that computes sum of MetricValue.getValue()
+ */
+ private Counter createKeyspaceCounter(String name, final ToLongFunction<TableMetrics> extractor)
+ {
+ allMetrics.add(() -> releaseMetric(name));
+ return Metrics.register(factory.createMetricName(name), new Counter()
+ {
+ @Override
+ public long getCount()
+ {
+ long sum = 0;
+ for (ColumnFamilyStore cf : keyspace.getColumnFamilyStores())
+ {
+ sum += extractor.applyAsLong(cf.metric);
+ }
+ return sum;
+ }
+ });
+ }
+
+ protected Counter createKeyspaceCounter(String name)
+ {
+ allMetrics.add(() -> releaseMetric(name));
+ return Metrics.counter(factory.createMetricName(name));
+ }
+
+ protected Histogram createKeyspaceHistogram(String name, boolean considerZeroes)
+ {
+ allMetrics.add(() -> releaseMetric(name));
+ return Metrics.histogram(factory.createMetricName(name), considerZeroes);
+ }
+
+ protected Timer createKeyspaceTimer(String name)
+ {
+ allMetrics.add(() -> releaseMetric(name));
+ return Metrics.timer(factory.createMetricName(name));
+ }
+
+ protected Meter createKeyspaceMeter(String name)
+ {
+ allMetrics.add(() -> releaseMetric(name));
+ return Metrics.meter(factory.createMetricName(name));
+ }
+
+ private LatencyMetrics createLatencyMetrics(String name)
+ {
+ LatencyMetrics metric = new LatencyMetrics(factory, name);
+ allMetrics.add(() -> metric.release());
+ return metric;
+ }
+
+ private void releaseMetric(String name)
+ {
+ Metrics.remove(factory.createMetricName(name));
+ }
+
static class KeyspaceMetricNameFactory implements MetricNameFactory
{
private final String keyspaceName;
@@ -298,7 +340,8 @@
this.keyspaceName = ks.getName();
}
- public CassandraMetricsRegistry.MetricName createMetricName(String metricName)
+ @Override
+ public MetricName createMetricName(String metricName)
{
String groupName = TableMetrics.class.getPackage().getName();
@@ -308,7 +351,7 @@
mbeanName.append(",keyspace=").append(keyspaceName);
mbeanName.append(",name=").append(metricName);
- return new CassandraMetricsRegistry.MetricName(groupName, "keyspace", metricName, keyspaceName, mbeanName.toString());
+ return new MetricName(groupName, "keyspace", metricName, keyspaceName, mbeanName.toString());
}
}
}
diff --git a/src/java/org/apache/cassandra/metrics/LatencyMetrics.java b/src/java/org/apache/cassandra/metrics/LatencyMetrics.java
index a1915b1..ab4c9a5 100644
--- a/src/java/org/apache/cassandra/metrics/LatencyMetrics.java
+++ b/src/java/org/apache/cassandra/metrics/LatencyMetrics.java
@@ -17,15 +17,17 @@
*/
package org.apache.cassandra.metrics;
+import java.util.Arrays;
import java.util.List;
import java.util.concurrent.TimeUnit;
-import com.codahale.metrics.Counter;
-import com.codahale.metrics.Timer;
-
-import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
+import com.codahale.metrics.Counter;
+import com.codahale.metrics.Reservoir;
+import com.codahale.metrics.Snapshot;
+import com.codahale.metrics.Timer;
+
import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics;
@@ -35,13 +37,14 @@
public class LatencyMetrics
{
/** Latency */
- public final Timer latency;
+ public final LatencyMetricsTimer latency;
/** Total latency in micro sec */
public final Counter totalLatency;
/** parent metrics to replicate any updates to **/
private List<LatencyMetrics> parents = Lists.newArrayList();
-
+ private List<LatencyMetrics> children = Lists.newArrayList();
+
protected final MetricNameFactory factory;
protected final MetricNameFactory aliasFactory;
protected final String namePrefix;
@@ -86,15 +89,18 @@
this.aliasFactory = aliasFactory;
this.namePrefix = namePrefix;
+ LatencyMetricsTimer timer = new LatencyMetrics.LatencyMetricsTimer(new DecayingEstimatedHistogramReservoir());
+ Counter counter = new LatencyMetricsCounter();
+
if (aliasFactory == null)
{
- latency = Metrics.timer(factory.createMetricName(namePrefix + "Latency"));
- totalLatency = Metrics.counter(factory.createMetricName(namePrefix + "TotalLatency"));
+ latency = Metrics.register(factory.createMetricName(namePrefix + "Latency"), timer);
+ totalLatency = Metrics.register(factory.createMetricName(namePrefix + "TotalLatency"), counter);
}
else
{
- latency = Metrics.timer(factory.createMetricName(namePrefix + "Latency"), aliasFactory.createMetricName(namePrefix + "Latency"));
- totalLatency = Metrics.counter(factory.createMetricName(namePrefix + "TotalLatency"), aliasFactory.createMetricName(namePrefix + "TotalLatency"));
+ latency = Metrics.register(factory.createMetricName(namePrefix + "Latency"), aliasFactory.createMetricName(namePrefix + "Latency"), timer);
+ totalLatency = Metrics.register(factory.createMetricName(namePrefix + "TotalLatency"), aliasFactory.createMetricName(namePrefix + "TotalLatency"), counter);
}
}
@@ -109,7 +115,38 @@
public LatencyMetrics(MetricNameFactory factory, String namePrefix, LatencyMetrics ... parents)
{
this(factory, null, namePrefix);
- this.parents.addAll(ImmutableList.copyOf(parents));
+ this.parents = Arrays.asList(parents);
+ for (LatencyMetrics parent : parents)
+ {
+ parent.addChildren(this);
+ }
+ }
+
+ private void addChildren(LatencyMetrics latencyMetric)
+ {
+ this.children.add(latencyMetric);
+ }
+
+ private synchronized void removeChildren(LatencyMetrics toRelease)
+ {
+ /*
+ Merge details of removed children metrics and add them to our local copy to prevent metrics from going
+ backwards. Synchronized since these methods are not thread safe to prevent multiple simultaneous removals.
+ Will not protect against simultaneous updates, but since these methods are used by linked parent instances only,
+ they should not receive any updates.
+ */
+ this.latency.releasedLatencyCount += toRelease.latency.getCount();
+
+ DecayingEstimatedHistogramReservoir.EstimatedHistogramReservoirSnapshot childSnapshot = (DecayingEstimatedHistogramReservoir.EstimatedHistogramReservoirSnapshot) toRelease.latency.getSnapshot();
+ DecayingEstimatedHistogramReservoir.EstimatedHistogramReservoirSnapshot snapshot = (DecayingEstimatedHistogramReservoir.EstimatedHistogramReservoirSnapshot) this.latency.getSnapshot();
+
+ snapshot.add(childSnapshot);
+ snapshot.rebaseReservoir();
+
+ this.totalLatency.inc(toRelease.totalLatency.getCount());
+
+ // Now we can remove the reference
+ this.children.removeIf(latencyMetrics -> latencyMetrics.equals(toRelease));
}
/** takes nanoseconds **/
@@ -118,14 +155,15 @@
// convert to microseconds. 1 millionth
latency.update(nanos, TimeUnit.NANOSECONDS);
totalLatency.inc(nanos / 1000);
- for(LatencyMetrics parent : parents)
- {
- parent.addNano(nanos);
- }
}
public void release()
{
+ // Notify parent metrics that this metric is being released
+ for (LatencyMetrics parent : this.parents)
+ {
+ parent.removeChildren(this);
+ }
if (aliasFactory == null)
{
Metrics.remove(factory.createMetricName(namePrefix + "Latency"));
@@ -137,4 +175,98 @@
Metrics.remove(factory.createMetricName(namePrefix + "TotalLatency"), aliasFactory.createMetricName(namePrefix + "TotalLatency"));
}
}
+
+ public class LatencyMetricsTimer extends Timer
+ {
+
+ long releasedLatencyCount = 0;
+
+ public LatencyMetricsTimer(Reservoir reservoir)
+ {
+ super(reservoir);
+ }
+
+ @Override
+ public long getCount()
+ {
+ long count = super.getCount() + releasedLatencyCount;
+ for (LatencyMetrics child : children)
+ {
+ count += child.latency.getCount();
+ }
+
+ return count;
+ }
+
+ @Override
+ public double getFifteenMinuteRate()
+ {
+ double rate = super.getFifteenMinuteRate();
+ for (LatencyMetrics child : children)
+ {
+ rate += child.latency.getFifteenMinuteRate();
+ }
+ return rate;
+ }
+
+ @Override
+ public double getFiveMinuteRate()
+ {
+ double rate = super.getFiveMinuteRate();
+ for (LatencyMetrics child : children)
+ {
+ rate += child.latency.getFiveMinuteRate();
+ }
+ return rate;
+ }
+
+ @Override
+ public double getMeanRate()
+ {
+ // Not necessarily 100% accurate, but close enough
+ double rate = super.getMeanRate();
+ for (LatencyMetrics child : children)
+ {
+ rate += child.latency.getMeanRate();
+ }
+ return rate;
+ }
+
+ @Override
+ public double getOneMinuteRate()
+ {
+ double rate = super.getOneMinuteRate();
+ for (LatencyMetrics child : children)
+ {
+ rate += child.latency.getOneMinuteRate();
+ }
+ return rate;
+ }
+
+ @Override
+ public Snapshot getSnapshot()
+ {
+ DecayingEstimatedHistogramReservoir.EstimatedHistogramReservoirSnapshot parent = (DecayingEstimatedHistogramReservoir.EstimatedHistogramReservoirSnapshot) super.getSnapshot();
+ for (LatencyMetrics child : children)
+ {
+ parent.add(child.latency.getSnapshot());
+ }
+
+ return parent;
+ }
+ }
+
+ class LatencyMetricsCounter extends Counter
+ {
+ @Override
+ public long getCount()
+ {
+ long count = super.getCount();
+ for (LatencyMetrics child : children)
+ {
+ count += child.totalLatency.getCount();
+ }
+ return count;
+ }
+ }
}
diff --git a/src/java/org/apache/cassandra/metrics/MaxSampler.java b/src/java/org/apache/cassandra/metrics/MaxSampler.java
new file mode 100644
index 0000000..df24bb9
--- /dev/null
+++ b/src/java/org/apache/cassandra/metrics/MaxSampler.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.metrics;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+
+import com.google.common.collect.MinMaxPriorityQueue;
+
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+
+public abstract class MaxSampler<T> extends Sampler<T>
+{
+ private int capacity;
+ private MinMaxPriorityQueue<Sample<T>> queue;
+ private long endTimeNanos = -1;
+ private final Comparator<Sample<T>> comp = Collections.reverseOrder(Comparator.comparing(p -> p.count));
+
+ public boolean isEnabled()
+ {
+ return endTimeNanos != -1 && clock.now() <= endTimeNanos;
+ }
+
+ public synchronized void beginSampling(int capacity, int durationMillis)
+ {
+ if (endTimeNanos == -1 || clock.now() > endTimeNanos)
+ {
+ endTimeNanos = clock.now() + MILLISECONDS.toNanos(durationMillis);
+ queue = MinMaxPriorityQueue
+ .orderedBy(comp)
+ .maximumSize(Math.max(1, capacity))
+ .create();
+ this.capacity = capacity;
+ }
+ else
+ throw new RuntimeException("Sampling already in progress");
+ }
+
+ public synchronized List<Sample<T>> finishSampling(int count)
+ {
+ List<Sample<T>> result = new ArrayList<>(count);
+ if (endTimeNanos != -1)
+ {
+ endTimeNanos = -1;
+ Sample<T> next;
+ while ((next = queue.poll()) != null && result.size() <= count)
+ result.add(next);
+ }
+ return result;
+ }
+
+ @Override
+ protected synchronized void insert(T item, long value)
+ {
+ if (value > 0 && clock.now() <= endTimeNanos
+ && (queue.isEmpty() || queue.size() < capacity || queue.peekLast().count < value))
+ queue.add(new Sample<T>(item, value, 0));
+ }
+
+}
diff --git a/src/java/org/apache/cassandra/metrics/MessagingMetrics.java b/src/java/org/apache/cassandra/metrics/MessagingMetrics.java
index e126c93..0ea2e10 100644
--- a/src/java/org/apache/cassandra/metrics/MessagingMetrics.java
+++ b/src/java/org/apache/cassandra/metrics/MessagingMetrics.java
@@ -17,43 +17,215 @@
*/
package org.apache.cassandra.metrics;
-import java.net.InetAddress;
+import java.util.EnumMap;
+import java.util.HashMap;
+import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Consumer;
-import org.apache.cassandra.config.DatabaseDescriptor;
+import com.google.common.annotations.VisibleForTesting;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.codahale.metrics.Timer;
+import org.apache.cassandra.concurrent.ScheduledExecutors;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import com.codahale.metrics.Timer;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.net.Message;
+import org.apache.cassandra.net.Verb;
+import org.apache.cassandra.net.InboundMessageHandlers;
+import org.apache.cassandra.net.LatencyConsumer;
+import org.apache.cassandra.utils.StatusLogger;
+
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics;
/**
* Metrics for messages
*/
-public class MessagingMetrics
+public class MessagingMetrics implements InboundMessageHandlers.GlobalMetricCallbacks
{
- private static Logger logger = LoggerFactory.getLogger(MessagingMetrics.class);
private static final MetricNameFactory factory = new DefaultNameFactory("Messaging");
- public final Timer crossNodeLatency;
- public final ConcurrentHashMap<String, Timer> dcLatency;
+ private static final Logger logger = LoggerFactory.getLogger(MessagingMetrics.class);
+ private static final int LOG_DROPPED_INTERVAL_IN_MS = 5000;
+
+ public static class DCLatencyRecorder implements LatencyConsumer
+ {
+ public final Timer dcLatency;
+ public final Timer allLatency;
+
+ DCLatencyRecorder(Timer dcLatency, Timer allLatency)
+ {
+ this.dcLatency = dcLatency;
+ this.allLatency = allLatency;
+ }
+
+ public void accept(long timeTaken, TimeUnit units)
+ {
+ if (timeTaken > 0)
+ {
+ dcLatency.update(timeTaken, units);
+ allLatency.update(timeTaken, units);
+ }
+ }
+ }
+
+ private static final class DroppedForVerb
+ {
+ final DroppedMessageMetrics metrics;
+ final AtomicInteger droppedFromSelf;
+ final AtomicInteger droppedFromPeer;
+
+ DroppedForVerb(Verb verb)
+ {
+ this(new DroppedMessageMetrics(verb));
+ }
+
+ DroppedForVerb(DroppedMessageMetrics metrics)
+ {
+ this.metrics = metrics;
+ this.droppedFromSelf = new AtomicInteger(0);
+ this.droppedFromPeer = new AtomicInteger(0);
+ }
+ }
+
+ private final Timer allLatency;
+ public final ConcurrentHashMap<String, DCLatencyRecorder> dcLatency;
+ public final EnumMap<Verb, Timer> internalLatency;
+
+ // total dropped message counts for server lifetime
+ private final Map<Verb, DroppedForVerb> droppedMessages = new EnumMap<>(Verb.class);
public MessagingMetrics()
{
- crossNodeLatency = Metrics.timer(factory.createMetricName("CrossNodeLatency"));
+ allLatency = Metrics.timer(factory.createMetricName("CrossNodeLatency"));
dcLatency = new ConcurrentHashMap<>();
+ internalLatency = new EnumMap<>(Verb.class);
+ for (Verb verb : Verb.VERBS)
+ internalLatency.put(verb, Metrics.timer(factory.createMetricName(verb + "-WaitLatency")));
+ for (Verb verb : Verb.values())
+ droppedMessages.put(verb, new DroppedForVerb(verb));
}
- public void addTimeTaken(InetAddress from, long timeTaken)
+ public DCLatencyRecorder internodeLatencyRecorder(InetAddressAndPort from)
{
- String dc = DatabaseDescriptor.getEndpointSnitch().getDatacenter(from);
- Timer timer = dcLatency.get(dc);
- if (timer == null)
- {
- timer = dcLatency.computeIfAbsent(dc, k -> Metrics.timer(factory.createMetricName(dc + "-Latency")));
- }
- timer.update(timeTaken, TimeUnit.MILLISECONDS);
- crossNodeLatency.update(timeTaken, TimeUnit.MILLISECONDS);
+ String dcName = DatabaseDescriptor.getEndpointSnitch().getDatacenter(from);
+ DCLatencyRecorder dcUpdater = dcLatency.get(dcName);
+ if (dcUpdater == null)
+ dcUpdater = dcLatency.computeIfAbsent(dcName, k -> new DCLatencyRecorder(Metrics.timer(factory.createMetricName(dcName + "-Latency")), allLatency));
+ return dcUpdater;
}
+
+ public void recordInternalLatency(Verb verb, long timeTaken, TimeUnit units)
+ {
+ if (timeTaken > 0)
+ internalLatency.get(verb).update(timeTaken, units);
+ }
+
+ public void recordSelfDroppedMessage(Verb verb)
+ {
+ recordDroppedMessage(droppedMessages.get(verb), false);
+ }
+
+ public void recordSelfDroppedMessage(Verb verb, long timeElapsed, TimeUnit timeUnit)
+ {
+ recordDroppedMessage(verb, timeElapsed, timeUnit, false);
+ }
+
+ public void recordInternodeDroppedMessage(Verb verb, long timeElapsed, TimeUnit timeUnit)
+ {
+ recordDroppedMessage(verb, timeElapsed, timeUnit, true);
+ }
+
+ public void recordDroppedMessage(Message<?> message, long timeElapsed, TimeUnit timeUnit)
+ {
+ recordDroppedMessage(message.verb(), timeElapsed, timeUnit, message.isCrossNode());
+ }
+
+ public void recordDroppedMessage(Verb verb, long timeElapsed, TimeUnit timeUnit, boolean isCrossNode)
+ {
+ recordDroppedMessage(droppedMessages.get(verb), timeElapsed, timeUnit, isCrossNode);
+ }
+
+ private static void recordDroppedMessage(DroppedForVerb droppedMessages, long timeTaken, TimeUnit units, boolean isCrossNode)
+ {
+ if (isCrossNode)
+ droppedMessages.metrics.crossNodeDroppedLatency.update(timeTaken, units);
+ else
+ droppedMessages.metrics.internalDroppedLatency.update(timeTaken, units);
+ recordDroppedMessage(droppedMessages, isCrossNode);
+ }
+
+ private static void recordDroppedMessage(DroppedForVerb droppedMessages, boolean isCrossNode)
+ {
+ droppedMessages.metrics.dropped.mark();
+ if (isCrossNode)
+ droppedMessages.droppedFromPeer.incrementAndGet();
+ else
+ droppedMessages.droppedFromSelf.incrementAndGet();
+ }
+
+ public void scheduleLogging()
+ {
+ ScheduledExecutors.scheduledTasks.scheduleWithFixedDelay(this::logDroppedMessages,
+ LOG_DROPPED_INTERVAL_IN_MS,
+ LOG_DROPPED_INTERVAL_IN_MS,
+ MILLISECONDS);
+ }
+
+ public Map<String, Integer> getDroppedMessages()
+ {
+ Map<String, Integer> map = new HashMap<>(droppedMessages.size());
+ for (Map.Entry<Verb, DroppedForVerb> entry : droppedMessages.entrySet())
+ map.put(entry.getKey().toString(), (int) entry.getValue().metrics.dropped.getCount());
+ return map;
+ }
+
+ private void logDroppedMessages()
+ {
+ if (resetAndConsumeDroppedErrors(logger::info) > 0)
+ StatusLogger.log();
+ }
+
+ @VisibleForTesting
+ public int resetAndConsumeDroppedErrors(Consumer<String> messageConsumer)
+ {
+ int count = 0;
+ for (Map.Entry<Verb, DroppedForVerb> entry : droppedMessages.entrySet())
+ {
+ Verb verb = entry.getKey();
+ DroppedForVerb droppedForVerb = entry.getValue();
+
+ int droppedInternal = droppedForVerb.droppedFromSelf.getAndSet(0);
+ int droppedCrossNode = droppedForVerb.droppedFromPeer.getAndSet(0);
+ if (droppedInternal > 0 || droppedCrossNode > 0)
+ {
+ messageConsumer.accept(String.format("%s messages were dropped in last %d ms: %d internal and %d cross node."
+ + " Mean internal dropped latency: %d ms and Mean cross-node dropped latency: %d ms",
+ verb,
+ LOG_DROPPED_INTERVAL_IN_MS,
+ droppedInternal,
+ droppedCrossNode,
+ TimeUnit.NANOSECONDS.toMillis((long) droppedForVerb.metrics.internalDroppedLatency.getSnapshot().getMean()),
+ TimeUnit.NANOSECONDS.toMillis((long) droppedForVerb.metrics.crossNodeDroppedLatency.getSnapshot().getMean())));
+ ++count;
+ }
+ }
+ return count;
+ }
+
+ @VisibleForTesting
+ public void resetDroppedMessages(String scope)
+ {
+ for (Verb verb : droppedMessages.keySet())
+ {
+ droppedMessages.put(verb, new DroppedForVerb(new DroppedMessageMetrics(metricName ->
+ new CassandraMetricsRegistry.MetricName("DroppedMessages", metricName, scope)
+ )));
+ }
+ }
+
}
diff --git a/src/java/org/apache/cassandra/metrics/ReadRepairMetrics.java b/src/java/org/apache/cassandra/metrics/ReadRepairMetrics.java
index 9ee1c60..3d00b12 100644
--- a/src/java/org/apache/cassandra/metrics/ReadRepairMetrics.java
+++ b/src/java/org/apache/cassandra/metrics/ReadRepairMetrics.java
@@ -29,6 +29,17 @@
private static final MetricNameFactory factory = new DefaultNameFactory("ReadRepair");
public static final Meter repairedBlocking = Metrics.meter(factory.createMetricName("RepairedBlocking"));
+ public static final Meter repairedAsync = Metrics.meter(factory.createMetricName("RepairedAsync"));
+ public static final Meter reconcileRead = Metrics.meter(factory.createMetricName("ReconcileRead"));
+
+ @Deprecated
public static final Meter repairedBackground = Metrics.meter(factory.createMetricName("RepairedBackground"));
+ @Deprecated
public static final Meter attempted = Metrics.meter(factory.createMetricName("Attempted"));
+
+ // Incremented when additional requests were sent during blocking read repair due to unavailable or slow nodes
+ public static final Meter speculatedRead = Metrics.meter(factory.createMetricName("SpeculatedRead"));
+ public static final Meter speculatedWrite = Metrics.meter(factory.createMetricName("SpeculatedWrite"));
+
+ public static void init() {}
}
diff --git a/src/java/org/apache/cassandra/metrics/RepairMetrics.java b/src/java/org/apache/cassandra/metrics/RepairMetrics.java
new file mode 100644
index 0000000..5b4f67e
--- /dev/null
+++ b/src/java/org/apache/cassandra/metrics/RepairMetrics.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.metrics;
+
+import com.codahale.metrics.Counter;
+
+import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics;
+
+public class RepairMetrics
+{
+ public static final String TYPE_NAME = "Repair";
+ public static final Counter previewFailures = Metrics.counter(DefaultNameFactory.createMetricName(TYPE_NAME, "PreviewFailures", null));
+
+ public static void init()
+ {
+ // noop
+ }
+}
diff --git a/src/java/org/apache/cassandra/metrics/SEPMetrics.java b/src/java/org/apache/cassandra/metrics/SEPMetrics.java
deleted file mode 100644
index 35f02b4..0000000
--- a/src/java/org/apache/cassandra/metrics/SEPMetrics.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.metrics;
-
-import com.codahale.metrics.Counter;
-import com.codahale.metrics.Gauge;
-
-import org.apache.cassandra.concurrent.SEPExecutor;
-
-import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics;
-
-public class SEPMetrics
-{
- /** Number of active tasks. */
- public final Gauge<Integer> activeTasks;
- /** Number of tasks that had blocked before being accepted (or rejected). */
- public final Counter totalBlocked;
- /**
- * Number of tasks currently blocked, waiting to be accepted by
- * the executor (because all threads are busy and the backing queue is full).
- */
- public final Counter currentBlocked;
- /** Number of completed tasks. */
- public final Gauge<Long> completedTasks;
- /** Number of tasks waiting to be executed. */
- public final Gauge<Long> pendingTasks;
- /** Maximum number of threads before it will start queuing tasks */
- public final Gauge<Integer> maxPoolSize;
-
- private MetricNameFactory factory;
-
- /**
- * Create metrics for the given LowSignalExecutor.
- *
- * @param executor Thread pool
- * @param path Type of thread pool
- * @param poolName Name of thread pool to identify metrics
- */
- public SEPMetrics(final SEPExecutor executor, String path, String poolName)
- {
- this.factory = new ThreadPoolMetricNameFactory("ThreadPools", path, poolName);
- activeTasks = Metrics.register(factory.createMetricName("ActiveTasks"), new Gauge<Integer>()
- {
- public Integer getValue()
- {
- return executor.getActiveCount();
- }
- });
- pendingTasks = Metrics.register(factory.createMetricName("PendingTasks"), new Gauge<Long>()
- {
- public Long getValue()
- {
- return executor.getPendingTasks();
- }
- });
- totalBlocked = Metrics.counter(factory.createMetricName("TotalBlockedTasks"));
- currentBlocked = Metrics.counter(factory.createMetricName("CurrentlyBlockedTasks"));
-
- completedTasks = Metrics.register(factory.createMetricName("CompletedTasks"), new Gauge<Long>()
- {
- public Long getValue()
- {
- return executor.getCompletedTasks();
- }
- });
- maxPoolSize = Metrics.register(factory.createMetricName("MaxPoolSize"), new Gauge<Integer>()
- {
- public Integer getValue()
- {
- return executor.maxWorkers;
- }
- });
- }
-
- public void release()
- {
- Metrics.remove(factory.createMetricName("ActiveTasks"));
- Metrics.remove(factory.createMetricName("PendingTasks"));
- Metrics.remove(factory.createMetricName("CompletedTasks"));
- Metrics.remove(factory.createMetricName("TotalBlockedTasks"));
- Metrics.remove(factory.createMetricName("CurrentlyBlockedTasks"));
- Metrics.remove(factory.createMetricName("MaxPoolSize"));
- }
-}
diff --git a/src/java/org/apache/cassandra/metrics/Sampler.java b/src/java/org/apache/cassandra/metrics/Sampler.java
new file mode 100644
index 0000000..cfe3f3b
--- /dev/null
+++ b/src/java/org/apache/cassandra/metrics/Sampler.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.metrics;
+
+import java.io.Serializable;
+import java.util.List;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.cassandra.concurrent.JMXEnabledThreadPoolExecutor;
+import org.apache.cassandra.concurrent.NamedThreadFactory;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.net.Verb;
+import org.apache.cassandra.utils.MonotonicClock;
+
+import com.google.common.annotations.VisibleForTesting;
+
+public abstract class Sampler<T>
+{
+ public enum SamplerType
+ {
+ READS, WRITES, LOCAL_READ_TIME, WRITE_SIZE, CAS_CONTENTIONS
+ }
+
+ @VisibleForTesting
+ MonotonicClock clock = MonotonicClock.approxTime;
+
+ @VisibleForTesting
+ static final ThreadPoolExecutor samplerExecutor = new JMXEnabledThreadPoolExecutor(1, 1,
+ TimeUnit.SECONDS,
+ new ArrayBlockingQueue<Runnable>(1000),
+ new NamedThreadFactory("Sampler"),
+ "internal");
+
+ static
+ {
+ samplerExecutor.setRejectedExecutionHandler((runnable, executor) ->
+ {
+ MessagingService.instance().metrics.recordSelfDroppedMessage(Verb._SAMPLE);
+ });
+ }
+
+ public void addSample(final T item, final int value)
+ {
+ if (isEnabled())
+ samplerExecutor.submit(() -> insert(item, value));
+ }
+
+ protected abstract void insert(T item, long value);
+
+ public abstract boolean isEnabled();
+
+ public abstract void beginSampling(int capacity, int durationMillis);
+
+ public abstract List<Sample<T>> finishSampling(int count);
+
+ public abstract String toString(T value);
+
+ /**
+ * Represents the ranked items collected during a sample period
+ */
+ public static class Sample<S> implements Serializable
+ {
+ public final S value;
+ public final long count;
+ public final long error;
+
+ public Sample(S value, long count, long error)
+ {
+ this.value = value;
+ this.count = count;
+ this.error = error;
+ }
+
+ @Override
+ public String toString()
+ {
+ return "Sample [value=" + value + ", count=" + count + ", error=" + error + "]";
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/metrics/StorageMetrics.java b/src/java/org/apache/cassandra/metrics/StorageMetrics.java
index 12196f7..9399ba6 100644
--- a/src/java/org/apache/cassandra/metrics/StorageMetrics.java
+++ b/src/java/org/apache/cassandra/metrics/StorageMetrics.java
@@ -29,7 +29,8 @@
private static final MetricNameFactory factory = new DefaultNameFactory("Storage");
public static final Counter load = Metrics.counter(factory.createMetricName("Load"));
- public static final Counter exceptions = Metrics.counter(factory.createMetricName("Exceptions"));
+ public static final Counter uncaughtExceptions = Metrics.counter(factory.createMetricName("Exceptions"));
public static final Counter totalHintsInProgress = Metrics.counter(factory.createMetricName("TotalHintsInProgress"));
public static final Counter totalHints = Metrics.counter(factory.createMetricName("TotalHints"));
+ public static final Counter repairExceptions = Metrics.counter(factory.createMetricName("RepairExceptions"));
}
diff --git a/src/java/org/apache/cassandra/metrics/StreamingMetrics.java b/src/java/org/apache/cassandra/metrics/StreamingMetrics.java
index 72e9b23..80a5e13 100644
--- a/src/java/org/apache/cassandra/metrics/StreamingMetrics.java
+++ b/src/java/org/apache/cassandra/metrics/StreamingMetrics.java
@@ -17,11 +17,12 @@
*/
package org.apache.cassandra.metrics;
-import java.net.InetAddress;
import java.util.concurrent.ConcurrentMap;
import com.codahale.metrics.Counter;
+import org.apache.cassandra.locator.InetAddressAndPort;
+
import org.cliffc.high_scale_lib.NonBlockingHashMap;
import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics;
@@ -33,28 +34,44 @@
{
public static final String TYPE_NAME = "Streaming";
- private static final ConcurrentMap<InetAddress, StreamingMetrics> instances = new NonBlockingHashMap<InetAddress, StreamingMetrics>();
+ private static final ConcurrentMap<InetAddressAndPort, StreamingMetrics> instances = new NonBlockingHashMap<>();
public static final Counter activeStreamsOutbound = Metrics.counter(DefaultNameFactory.createMetricName(TYPE_NAME, "ActiveOutboundStreams", null));
public static final Counter totalIncomingBytes = Metrics.counter(DefaultNameFactory.createMetricName(TYPE_NAME, "TotalIncomingBytes", null));
public static final Counter totalOutgoingBytes = Metrics.counter(DefaultNameFactory.createMetricName(TYPE_NAME, "TotalOutgoingBytes", null));
+ public static final Counter totalOutgoingRepairBytes = Metrics.counter(DefaultNameFactory.createMetricName(TYPE_NAME, "TotalOutgoingRepairBytes", null));
+ public static final Counter totalOutgoingRepairSSTables = Metrics.counter(DefaultNameFactory.createMetricName(TYPE_NAME, "TotalOutgoingRepairSSTables", null));
public final Counter incomingBytes;
public final Counter outgoingBytes;
- public static StreamingMetrics get(InetAddress ip)
+ public static StreamingMetrics get(InetAddressAndPort ip)
{
+ /*
+ computeIfAbsent doesn't work for this situation. Since JMX metrics register themselves in their ctor, we need
+ to create the metric exactly once, otherwise we'll get duplicate name exceptions. Although computeIfAbsent is
+ thread safe in the context of the map, it uses compare and swap to add the computed value to the map. This
+ means it eagerly allocates new metric instances, which can cause the jmx name collision we're trying to avoid
+ if multiple calls interleave. So here we use synchronized to ensure we only instantiate metrics exactly once.
+ */
StreamingMetrics metrics = instances.get(ip);
if (metrics == null)
{
- metrics = new StreamingMetrics(ip);
- instances.put(ip, metrics);
+ synchronized (instances)
+ {
+ metrics = instances.get(ip);
+ if (metrics == null)
+ {
+ metrics = new StreamingMetrics(ip);
+ instances.put(ip, metrics);
+ }
+ }
}
return metrics;
}
- public StreamingMetrics(final InetAddress peer)
+ public StreamingMetrics(final InetAddressAndPort peer)
{
- MetricNameFactory factory = new DefaultNameFactory("Streaming", peer.getHostAddress().replace(':', '.'));
+ MetricNameFactory factory = new DefaultNameFactory("Streaming", peer.toString().replace(':', '.'));
incomingBytes = Metrics.counter(factory.createMetricName("IncomingBytes"));
outgoingBytes= Metrics.counter(factory.createMetricName("OutgoingBytes"));
}
diff --git a/src/java/org/apache/cassandra/metrics/TableMetrics.java b/src/java/org/apache/cassandra/metrics/TableMetrics.java
index a02539a..bfb261d 100644
--- a/src/java/org/apache/cassandra/metrics/TableMetrics.java
+++ b/src/java/org/apache/cassandra/metrics/TableMetrics.java
@@ -17,30 +17,45 @@
*/
package org.apache.cassandra.metrics;
+import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics;
+
import java.nio.ByteBuffer;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.EnumMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.TimeUnit;
+import java.util.function.Predicate;
+import com.google.common.collect.Iterables;
import com.google.common.collect.Maps;
-
-import com.codahale.metrics.*;
+import com.google.common.collect.Sets;
import com.codahale.metrics.Timer;
-import org.apache.cassandra.config.Schema;
-import org.apache.cassandra.config.SchemaConstants;
+
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.db.Memtable;
import org.apache.cassandra.db.lifecycle.SSTableSet;
+import org.apache.cassandra.db.lifecycle.View;
import org.apache.cassandra.index.SecondaryIndexManager;
import org.apache.cassandra.io.compress.CompressionMetadata;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
+import org.apache.cassandra.metrics.Sampler.SamplerType;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.SchemaConstants;
import org.apache.cassandra.utils.EstimatedHistogram;
-import org.apache.cassandra.utils.TopKSampler;
+import org.apache.cassandra.utils.Pair;
-import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics;
+import com.codahale.metrics.Counter;
+import com.codahale.metrics.Gauge;
+import com.codahale.metrics.Histogram;
+import com.codahale.metrics.Meter;
+import com.codahale.metrics.Metric;
+import com.codahale.metrics.RatioGauge;
/**
* Metrics for {@link ColumnFamilyStore}.
@@ -92,6 +107,8 @@
public final Gauge<Integer> pendingCompactions;
/** Number of SSTables on disk for this CF */
public final Gauge<Integer> liveSSTableCount;
+ /** Number of SSTables with old version on disk for this CF */
+ public final Gauge<Integer> oldVersionSSTableCount;
/** Disk space used by SSTables belonging to this table */
public final Counter liveDiskSpaceUsed;
/** Total disk space used by SSTables belonging to this table, including obsolete ones waiting to be GC'd */
@@ -138,6 +155,14 @@
public final Counter rowCacheHit;
/** Number of row cache misses */
public final Counter rowCacheMiss;
+ /**
+ * Number of tombstone read failures
+ */
+ public final Counter tombstoneFailures;
+ /**
+ * Number of tombstone read warnings
+ */
+ public final Counter tombstoneWarnings;
/** CAS Prepare metrics */
public final LatencyMetrics casPrepare;
/** CAS Propose metrics */
@@ -146,14 +171,39 @@
public final LatencyMetrics casCommit;
/** percent of the data that is repaired */
public final Gauge<Double> percentRepaired;
+ /** Reports the size of sstables in repaired, unrepaired, and any ongoing repair buckets */
+ public final Gauge<Long> bytesRepaired;
+ public final Gauge<Long> bytesUnrepaired;
+ public final Gauge<Long> bytesPendingRepair;
+ /** Number of started repairs as coordinator on this table */
+ public final Counter repairsStarted;
+ /** Number of completed repairs as coordinator on this table */
+ public final Counter repairsCompleted;
+ /** time spent anticompacting data before participating in a consistent repair */
+ public final TableTimer anticompactionTime;
+ /** time spent creating merkle trees */
+ public final TableTimer validationTime;
+ /** time spent syncing data in a repair */
+ public final TableTimer syncTime;
+ /** approximate number of bytes read while creating merkle trees */
+ public final TableHistogram bytesValidated;
+ /** number of partitions read creating merkle trees */
+ public final TableHistogram partitionsValidated;
+ /** number of bytes read while doing anticompaction */
+ public final Counter bytesAnticompacted;
+ /** number of bytes where the whole sstable was contained in a repairing range so that we only mutated the repair status */
+ public final Counter bytesMutatedAnticompaction;
+ /** ratio of how much we anticompact vs how much we could mutate the repair status*/
+ public final Gauge<Double> mutatedAnticompactionGauge;
public final Timer coordinatorReadLatency;
public final Timer coordinatorScanLatency;
+ public final Timer coordinatorWriteLatency;
/** Time spent waiting for free memtable space, either on- or off-heap */
public final Histogram waitingOnFreeMemtableSpace;
- /** Dropped Mutations Count */
+ @Deprecated
public final Counter droppedMutations;
private final MetricNameFactory factory;
@@ -162,59 +212,134 @@
private static final MetricNameFactory globalAliasFactory = new AllTableMetricNameFactory("ColumnFamily");
public final Counter speculativeRetries;
+ public final Counter speculativeFailedRetries;
+ public final Counter speculativeInsufficientReplicas;
+ public final Gauge<Long> speculativeSampleLatencyNanos;
+
+ public final Counter additionalWrites;
+ public final Gauge<Long> additionalWriteLatencyNanos;
+
+ public final Gauge<Integer> unleveledSSTables;
+
+ /**
+ * Metrics for inconsistencies detected between repaired data sets across replicas. These
+ * are tracked on the coordinator.
+ */
+ // Incremented where an inconsistency is detected and there are no pending repair sessions affecting
+ // the data being read, indicating a genuine mismatch between replicas' repaired data sets.
+ public final TableMeter confirmedRepairedInconsistencies;
+ // Incremented where an inconsistency is detected, but there are pending & uncommitted repair sessions
+ // in play on at least one replica. This may indicate a false positive as the inconsistency could be due to
+ // replicas marking the repair session as committed at slightly different times and so some consider it to
+ // be part of the repaired set whilst others do not.
+ public final TableMeter unconfirmedRepairedInconsistencies;
+
+ // Tracks the amount overreading of repaired data replicas perform in order to produce digests
+ // at query time. For each query, on a full data read following an initial digest mismatch, the replicas
+ // may read extra repaired data, up to the DataLimit of the command, so that the coordinator can compare
+ // the repaired data on each replica. These are tracked on each replica.
+ public final TableHistogram repairedDataTrackingOverreadRows;
+ public final TableTimer repairedDataTrackingOverreadTime;
public final static LatencyMetrics globalReadLatency = new LatencyMetrics(globalFactory, globalAliasFactory, "Read");
public final static LatencyMetrics globalWriteLatency = new LatencyMetrics(globalFactory, globalAliasFactory, "Write");
public final static LatencyMetrics globalRangeLatency = new LatencyMetrics(globalFactory, globalAliasFactory, "Range");
- public final static Gauge<Double> globalPercentRepaired = Metrics.register(globalFactory.createMetricName("PercentRepaired"),
- new Gauge<Double>()
- {
- public Double getValue()
- {
- double repaired = 0;
- double total = 0;
- for (String keyspace : Schema.instance.getNonSystemKeyspaces())
- {
- Keyspace k = Schema.instance.getKeyspaceInstance(keyspace);
- if (SchemaConstants.DISTRIBUTED_KEYSPACE_NAME.equals(k.getName()))
- continue;
- if (k.getReplicationStrategy().getReplicationFactor() < 2)
- continue;
+ /** When sampler activated, will track the most frequently read partitions **/
+ public final Sampler<ByteBuffer> topReadPartitionFrequency;
+ /** When sampler activated, will track the most frequently written to partitions **/
+ public final Sampler<ByteBuffer> topWritePartitionFrequency;
+ /** When sampler activated, will track the largest mutations **/
+ public final Sampler<ByteBuffer> topWritePartitionSize;
+ /** When sampler activated, will track the most frequent partitions with cas contention **/
+ public final Sampler<ByteBuffer> topCasPartitionContention;
+ /** When sampler activated, will track the slowest local reads **/
+ public final Sampler<String> topLocalReadQueryTime;
- for (ColumnFamilyStore cf : k.getColumnFamilyStores())
+ private static Pair<Long, Long> totalNonSystemTablesSize(Predicate<SSTableReader> predicate)
+ {
+ long total = 0;
+ long filtered = 0;
+ for (String keyspace : Schema.instance.getNonSystemKeyspaces())
+ {
+
+ Keyspace k = Schema.instance.getKeyspaceInstance(keyspace);
+ if (SchemaConstants.DISTRIBUTED_KEYSPACE_NAME.equals(k.getName()))
+ continue;
+ if (k.getReplicationStrategy().getReplicationFactor().allReplicas < 2)
+ continue;
+
+ for (ColumnFamilyStore cf : k.getColumnFamilyStores())
+ {
+ if (!SecondaryIndexManager.isIndexColumnFamily(cf.name))
{
- if (!SecondaryIndexManager.isIndexColumnFamily(cf.name))
+ for (SSTableReader sstable : cf.getSSTables(SSTableSet.CANONICAL))
{
- for (SSTableReader sstable : cf.getSSTables(SSTableSet.CANONICAL))
+ if (predicate.test(sstable))
{
- if (sstable.isRepaired())
- {
- repaired += sstable.uncompressedLength();
- }
- total += sstable.uncompressedLength();
+ filtered += sstable.uncompressedLength();
}
+ total += sstable.uncompressedLength();
}
}
}
+ }
+ return Pair.create(filtered, total);
+ }
+
+ public static final Gauge<Double> globalPercentRepaired = Metrics.register(globalFactory.createMetricName("PercentRepaired"),
+ new Gauge<Double>()
+ {
+ public Double getValue()
+ {
+ Pair<Long, Long> result = totalNonSystemTablesSize(SSTableReader::isRepaired);
+ double repaired = result.left;
+ double total = result.right;
return total > 0 ? (repaired / total) * 100 : 100.0;
}
});
+ public static final Gauge<Long> globalBytesRepaired = Metrics.register(globalFactory.createMetricName("BytesRepaired"),
+ new Gauge<Long>()
+ {
+ public Long getValue()
+ {
+ return totalNonSystemTablesSize(SSTableReader::isRepaired).left;
+ }
+ });
+
+ public static final Gauge<Long> globalBytesUnrepaired = Metrics.register(globalFactory.createMetricName("BytesUnrepaired"),
+ new Gauge<Long>()
+ {
+ public Long getValue()
+ {
+ return totalNonSystemTablesSize(s -> !s.isRepaired() && !s.isPendingRepair()).left;
+ }
+ });
+
+ public static final Gauge<Long> globalBytesPendingRepair = Metrics.register(globalFactory.createMetricName("BytesPendingRepair"),
+ new Gauge<Long>()
+ {
+ public Long getValue()
+ {
+ return totalNonSystemTablesSize(SSTableReader::isPendingRepair).left;
+ }
+ });
+
public final Meter readRepairRequests;
public final Meter shortReadProtectionRequests;
public final Meter replicaSideFilteringProtectionRequests;
- public final Map<Sampler, TopKSampler<ByteBuffer>> samplers;
+ public final EnumMap<SamplerType, Sampler<?>> samplers;
/**
* stores metrics that will be rolled into a single global metric
*/
public final static ConcurrentMap<String, Set<Metric>> allTableMetrics = Maps.newConcurrentMap();
/**
- * Stores all metric names created that can be used when unregistering, optionally mapped to an alias name.
+ * Stores all metrics created that can be used when unregistering
*/
- public final static Map<String, String> all = Maps.newHashMap();
+ public final static Set<ReleasableMetric> all = Sets.newHashSet();
private interface GetHistogram
{
@@ -266,11 +391,48 @@
factory = new TableMetricNameFactory(cfs, "Table");
aliasFactory = new TableMetricNameFactory(cfs, "ColumnFamily");
- samplers = Maps.newHashMap();
- for (Sampler sampler : Sampler.values())
+ samplers = new EnumMap<>(SamplerType.class);
+ topReadPartitionFrequency = new FrequencySampler<ByteBuffer>()
{
- samplers.put(sampler, new TopKSampler<>());
- }
+ public String toString(ByteBuffer value)
+ {
+ return cfs.metadata().partitionKeyType.getString(value);
+ }
+ };
+ topWritePartitionFrequency = new FrequencySampler<ByteBuffer>()
+ {
+ public String toString(ByteBuffer value)
+ {
+ return cfs.metadata().partitionKeyType.getString(value);
+ }
+ };
+ topWritePartitionSize = new MaxSampler<ByteBuffer>()
+ {
+ public String toString(ByteBuffer value)
+ {
+ return cfs.metadata().partitionKeyType.getString(value);
+ }
+ };
+ topCasPartitionContention = new FrequencySampler<ByteBuffer>()
+ {
+ public String toString(ByteBuffer value)
+ {
+ return cfs.metadata().partitionKeyType.getString(value);
+ }
+ };
+ topLocalReadQueryTime = new MaxSampler<String>()
+ {
+ public String toString(String value)
+ {
+ return value;
+ }
+ };
+
+ samplers.put(SamplerType.READS, topReadPartitionFrequency);
+ samplers.put(SamplerType.WRITES, topWritePartitionFrequency);
+ samplers.put(SamplerType.WRITE_SIZE, topWritePartitionSize);
+ samplers.put(SamplerType.CAS_CONTENTIONS, topCasPartitionContention);
+ samplers.put(SamplerType.LOCAL_READ_TIME, topLocalReadQueryTime);
memtableColumnsCount = createTableGauge("MemtableColumnsCount", new Gauge<Long>()
{
@@ -331,48 +493,45 @@
}
});
memtableSwitchCount = createTableCounter("MemtableSwitchCount");
- estimatedPartitionSizeHistogram = Metrics.register(factory.createMetricName("EstimatedPartitionSizeHistogram"),
- aliasFactory.createMetricName("EstimatedRowSizeHistogram"),
- new Gauge<long[]>()
- {
- public long[] getValue()
- {
- return combineHistograms(cfs.getSSTables(SSTableSet.CANONICAL), new GetHistogram()
- {
- public EstimatedHistogram getHistogram(SSTableReader reader)
- {
- return reader.getEstimatedPartitionSize();
- }
- });
- }
- });
- estimatedPartitionCount = Metrics.register(factory.createMetricName("EstimatedPartitionCount"),
- aliasFactory.createMetricName("EstimatedRowCount"),
- new Gauge<Long>()
- {
- public Long getValue()
- {
- long memtablePartitions = 0;
- for (Memtable memtable : cfs.getTracker().getView().getAllMemtables())
- memtablePartitions += memtable.partitionCount();
- return SSTableReader.getApproximateKeyCount(cfs.getSSTables(SSTableSet.CANONICAL)) + memtablePartitions;
- }
- });
- estimatedColumnCountHistogram = Metrics.register(factory.createMetricName("EstimatedColumnCountHistogram"),
- aliasFactory.createMetricName("EstimatedColumnCountHistogram"),
- new Gauge<long[]>()
- {
- public long[] getValue()
- {
- return combineHistograms(cfs.getSSTables(SSTableSet.CANONICAL), new GetHistogram()
- {
- public EstimatedHistogram getHistogram(SSTableReader reader)
- {
- return reader.getEstimatedColumnCount();
- }
- });
+ estimatedPartitionSizeHistogram = createTableGauge("EstimatedPartitionSizeHistogram", "EstimatedRowSizeHistogram", new Gauge<long[]>()
+ {
+ public long[] getValue()
+ {
+ return combineHistograms(cfs.getSSTables(SSTableSet.CANONICAL), new GetHistogram()
+ {
+ public EstimatedHistogram getHistogram(SSTableReader reader)
+ {
+ return reader.getEstimatedPartitionSize();
+ }
+ });
}
- });
+ }, null);
+ estimatedPartitionCount = createTableGauge("EstimatedPartitionCount", "EstimatedRowCount", new Gauge<Long>()
+ {
+ public Long getValue()
+ {
+ long memtablePartitions = 0;
+ for (Memtable memtable : cfs.getTracker().getView().getAllMemtables())
+ memtablePartitions += memtable.partitionCount();
+ try(ColumnFamilyStore.RefViewFragment refViewFragment = cfs.selectAndReference(View.selectFunction(SSTableSet.CANONICAL)))
+ {
+ return SSTableReader.getApproximateKeyCount(refViewFragment.sstables) + memtablePartitions;
+ }
+ }
+ }, null);
+ estimatedColumnCountHistogram = createTableGauge("EstimatedColumnCountHistogram", "EstimatedColumnCountHistogram", new Gauge<long[]>()
+ {
+ public long[] getValue()
+ {
+ return combineHistograms(cfs.getSSTables(SSTableSet.CANONICAL), new GetHistogram()
+ {
+ public EstimatedHistogram getHistogram(SSTableReader reader)
+ {
+ return reader.getEstimatedCellPerPartitionCount();
+ }
+ });
+ }
+ }, null);
sstablesPerReadHistogram = createTableHistogram("SSTablesPerReadHistogram", cfs.keyspace.metric.sstablesPerReadHistogram, true);
compressionRatio = createTableGauge("CompressionRatio", new Gauge<Double>()
{
@@ -406,11 +565,52 @@
return total > 0 ? (repaired / total) * 100 : 100.0;
}
});
- readLatency = new LatencyMetrics(factory, "Read", cfs.keyspace.metric.readLatency, globalReadLatency);
- writeLatency = new LatencyMetrics(factory, "Write", cfs.keyspace.metric.writeLatency, globalWriteLatency);
- rangeLatency = new LatencyMetrics(factory, "Range", cfs.keyspace.metric.rangeLatency, globalRangeLatency);
+
+ bytesRepaired = createTableGauge("BytesRepaired", new Gauge<Long>()
+ {
+ public Long getValue()
+ {
+ long size = 0;
+ for (SSTableReader sstable: Iterables.filter(cfs.getSSTables(SSTableSet.CANONICAL), SSTableReader::isRepaired))
+ {
+ size += sstable.uncompressedLength();
+ }
+ return size;
+ }
+ });
+
+ bytesUnrepaired = createTableGauge("BytesUnrepaired", new Gauge<Long>()
+ {
+ public Long getValue()
+ {
+ long size = 0;
+ for (SSTableReader sstable: Iterables.filter(cfs.getSSTables(SSTableSet.CANONICAL), s -> !s.isRepaired() && !s.isPendingRepair()))
+ {
+ size += sstable.uncompressedLength();
+ }
+ return size;
+ }
+ });
+
+ bytesPendingRepair = createTableGauge("BytesPendingRepair", new Gauge<Long>()
+ {
+ public Long getValue()
+ {
+ long size = 0;
+ for (SSTableReader sstable: Iterables.filter(cfs.getSSTables(SSTableSet.CANONICAL), SSTableReader::isPendingRepair))
+ {
+ size += sstable.uncompressedLength();
+ }
+ return size;
+ }
+ });
+
+ readLatency = createLatencyMetrics("Read", cfs.keyspace.metric.readLatency, globalReadLatency);
+ writeLatency = createLatencyMetrics("Write", cfs.keyspace.metric.writeLatency, globalWriteLatency);
+ rangeLatency = createLatencyMetrics("Range", cfs.keyspace.metric.rangeLatency, globalRangeLatency);
pendingFlushes = createTableCounter("PendingFlushes");
bytesFlushed = createTableCounter("BytesFlushed");
+
compactionBytesWritten = createTableCounter("CompactionBytesWritten");
pendingCompactions = createTableGauge("PendingCompactions", new Gauge<Integer>()
{
@@ -426,6 +626,17 @@
return cfs.getTracker().getView().liveSSTables().size();
}
});
+ oldVersionSSTableCount = createTableGauge("OldVersionSSTableCount", new Gauge<Integer>()
+ {
+ public Integer getValue()
+ {
+ int count = 0;
+ for (SSTableReader sstable : cfs.getLiveSSTables())
+ if (!sstable.descriptor.version.isLatestVersion())
+ count++;
+ return count;
+ }
+ });
liveDiskSpaceUsed = createTableCounter("LiveDiskSpaceUsed");
totalDiskSpaceUsed = createTableCounter("TotalDiskSpaceUsed");
minPartitionSize = createTableGauge("MinPartitionSize", "MinRowSize", new Gauge<Long>()
@@ -637,9 +848,14 @@
}
});
speculativeRetries = createTableCounter("SpeculativeRetries");
- keyCacheHitRate = Metrics.register(factory.createMetricName("KeyCacheHitRate"),
- aliasFactory.createMetricName("KeyCacheHitRate"),
- new RatioGauge()
+ speculativeFailedRetries = createTableCounter("SpeculativeFailedRetries");
+ speculativeInsufficientReplicas = createTableCounter("SpeculativeInsufficientReplicas");
+ speculativeSampleLatencyNanos = createTableGauge("SpeculativeSampleLatencyNanos", () -> cfs.sampleReadLatencyNanos);
+
+ additionalWrites = createTableCounter("AdditionalWrites");
+ additionalWriteLatencyNanos = createTableGauge("AdditionalWriteLatencyNanos", () -> cfs.additionalWriteLatencyNanos);
+
+ keyCacheHitRate = createTableGauge("KeyCacheHitRate", "KeyCacheHitRate", new RatioGauge()
{
@Override
public Ratio getRatio()
@@ -662,17 +878,18 @@
requests += sstable.getKeyCacheRequest();
return Math.max(requests, 1); // to avoid NaN.
}
- });
+ }, null);
tombstoneScannedHistogram = createTableHistogram("TombstoneScannedHistogram", cfs.keyspace.metric.tombstoneScannedHistogram, false);
liveScannedHistogram = createTableHistogram("LiveScannedHistogram", cfs.keyspace.metric.liveScannedHistogram, false);
colUpdateTimeDeltaHistogram = createTableHistogram("ColUpdateTimeDeltaHistogram", cfs.keyspace.metric.colUpdateTimeDeltaHistogram, false);
- coordinatorReadLatency = Metrics.timer(factory.createMetricName("CoordinatorReadLatency"));
- coordinatorScanLatency = Metrics.timer(factory.createMetricName("CoordinatorScanLatency"));
- waitingOnFreeMemtableSpace = Metrics.histogram(factory.createMetricName("WaitingOnFreeMemtableSpace"), false);
+ coordinatorReadLatency = createTableTimer("CoordinatorReadLatency");
+ coordinatorScanLatency = createTableTimer("CoordinatorScanLatency");
+ coordinatorWriteLatency = createTableTimer("CoordinatorWriteLatency");
+ waitingOnFreeMemtableSpace = createTableHistogram("WaitingOnFreeMemtableSpace", false);
// We do not want to capture view mutation specific metrics for a view
// They only makes sense to capture on the base table
- if (cfs.metadata.isView())
+ if (cfs.metadata().isView())
{
viewLockAcquireTime = null;
viewReadTime = null;
@@ -693,15 +910,55 @@
rowCacheHitOutOfRange = createTableCounter("RowCacheHitOutOfRange");
rowCacheHit = createTableCounter("RowCacheHit");
rowCacheMiss = createTableCounter("RowCacheMiss");
+
+ tombstoneFailures = createTableCounter("TombstoneFailures");
+ tombstoneWarnings = createTableCounter("TombstoneWarnings");
+
droppedMutations = createTableCounter("DroppedMutations");
- casPrepare = new LatencyMetrics(factory, "CasPrepare", cfs.keyspace.metric.casPrepare);
- casPropose = new LatencyMetrics(factory, "CasPropose", cfs.keyspace.metric.casPropose);
- casCommit = new LatencyMetrics(factory, "CasCommit", cfs.keyspace.metric.casCommit);
+ casPrepare = createLatencyMetrics("CasPrepare", cfs.keyspace.metric.casPrepare);
+ casPropose = createLatencyMetrics("CasPropose", cfs.keyspace.metric.casPropose);
+ casCommit = createLatencyMetrics("CasCommit", cfs.keyspace.metric.casCommit);
+
+ repairsStarted = createTableCounter("RepairJobsStarted");
+ repairsCompleted = createTableCounter("RepairJobsCompleted");
+
+ anticompactionTime = createTableTimer("AnticompactionTime", cfs.keyspace.metric.anticompactionTime);
+ validationTime = createTableTimer("ValidationTime", cfs.keyspace.metric.validationTime);
+ syncTime = createTableTimer("SyncTime", cfs.keyspace.metric.repairSyncTime);
+
+ bytesValidated = createTableHistogram("BytesValidated", cfs.keyspace.metric.bytesValidated, false);
+ partitionsValidated = createTableHistogram("PartitionsValidated", cfs.keyspace.metric.partitionsValidated, false);
+ bytesAnticompacted = createTableCounter("BytesAnticompacted");
+ bytesMutatedAnticompaction = createTableCounter("BytesMutatedAnticompaction");
+ mutatedAnticompactionGauge = createTableGauge("MutatedAnticompactionGauge", () ->
+ {
+ double bytesMutated = bytesMutatedAnticompaction.getCount();
+ double bytesAnticomp = bytesAnticompacted.getCount();
+ if (bytesAnticomp + bytesMutated > 0)
+ return bytesMutated / (bytesAnticomp + bytesMutated);
+ return 0.0;
+ });
readRepairRequests = createTableMeter("ReadRepairRequests");
shortReadProtectionRequests = createTableMeter("ShortReadProtectionRequests");
replicaSideFilteringProtectionRequests = createTableMeter("ReplicaSideFilteringProtectionRequests");
+
+ confirmedRepairedInconsistencies = createTableMeter("RepairedDataInconsistenciesConfirmed", cfs.keyspace.metric.confirmedRepairedInconsistencies);
+ unconfirmedRepairedInconsistencies = createTableMeter("RepairedDataInconsistenciesUnconfirmed", cfs.keyspace.metric.unconfirmedRepairedInconsistencies);
+
+ repairedDataTrackingOverreadRows = createTableHistogram("RepairedDataTrackingOverreadRows", cfs.keyspace.metric.repairedDataTrackingOverreadRows, false);
+ repairedDataTrackingOverreadTime = createTableTimer("RepairedDataTrackingOverreadTime", cfs.keyspace.metric.repairedDataTrackingOverreadTime);
+
+ unleveledSSTables = createTableGauge("UnleveledSSTables", cfs::getUnleveledSSTables, () -> {
+ // global gauge
+ int cnt = 0;
+ for (Metric cfGauge : allTableMetrics.get("UnleveledSSTables"))
+ {
+ cnt += ((Gauge<? extends Number>) cfGauge).getValue().intValue();
+ }
+ return cnt;
+ });
}
public void updateSSTableIterated(int count)
@@ -714,27 +971,10 @@
*/
public void release()
{
- for(Map.Entry<String, String> entry : all.entrySet())
+ for (ReleasableMetric entry : all)
{
- CassandraMetricsRegistry.MetricName name = factory.createMetricName(entry.getKey());
- CassandraMetricsRegistry.MetricName alias = aliasFactory.createMetricName(entry.getValue());
- final Metric metric = Metrics.getMetrics().get(name.getMetricName());
- if (metric != null)
- { // Metric will be null if it's a view metric we are releasing. Views have null for ViewLockAcquireTime and ViewLockReadTime
- allTableMetrics.get(entry.getKey()).remove(metric);
- Metrics.remove(name, alias);
- }
+ entry.release();
}
- readLatency.release();
- writeLatency.release();
- rangeLatency.release();
- Metrics.remove(factory.createMetricName("EstimatedPartitionSizeHistogram"), aliasFactory.createMetricName("EstimatedRowSizeHistogram"));
- Metrics.remove(factory.createMetricName("EstimatedPartitionCount"), aliasFactory.createMetricName("EstimatedRowCount"));
- Metrics.remove(factory.createMetricName("EstimatedColumnCountHistogram"), aliasFactory.createMetricName("EstimatedColumnCountHistogram"));
- Metrics.remove(factory.createMetricName("KeyCacheHitRate"), aliasFactory.createMetricName("KeyCacheHitRate"));
- Metrics.remove(factory.createMetricName("CoordinatorReadLatency"), aliasFactory.createMetricName("CoordinatorReadLatency"));
- Metrics.remove(factory.createMetricName("CoordinatorScanLatency"), aliasFactory.createMetricName("CoordinatorScanLatency"));
- Metrics.remove(factory.createMetricName("WaitingOnFreeMemtableSpace"), aliasFactory.createMetricName("WaitingOnFreeMemtableSpace"));
}
@@ -770,7 +1010,7 @@
protected <G,T> Gauge<T> createTableGauge(String name, String alias, Gauge<T> gauge, Gauge<G> globalGauge)
{
Gauge<T> cfGauge = Metrics.register(factory.createMetricName(name), aliasFactory.createMetricName(alias), gauge);
- if (register(name, alias, cfGauge))
+ if (register(name, alias, cfGauge) && globalGauge != null)
{
Metrics.register(globalFactory.createMetricName(name), globalAliasFactory.createMetricName(alias), globalGauge);
}
@@ -867,6 +1107,18 @@
considerZeroes));
}
+ protected Histogram createTableHistogram(String name, boolean considerZeroes)
+ {
+ return createTableHistogram(name, name, considerZeroes);
+ }
+
+ protected Histogram createTableHistogram(String name, String alias, boolean considerZeroes)
+ {
+ Histogram tableHistogram = Metrics.histogram(factory.createMetricName(name), aliasFactory.createMetricName(alias), considerZeroes);
+ register(name, alias, tableHistogram);
+ return tableHistogram;
+ }
+
protected TableTimer createTableTimer(String name, Timer keyspaceTimer)
{
return createTableTimer(name, name, keyspaceTimer);
@@ -882,6 +1134,40 @@
globalAliasFactory.createMetricName(alias)));
}
+ protected Timer createTableTimer(String name)
+ {
+ return createTableTimer(name, name);
+ }
+
+ protected Timer createTableTimer(String name, String alias)
+ {
+ Timer tableTimer = Metrics.timer(factory.createMetricName(name), aliasFactory.createMetricName(alias));
+ register(name, alias, tableTimer);
+ return tableTimer;
+ }
+
+ protected TableMeter createTableMeter(String name, Meter keyspaceMeter)
+ {
+ return createTableMeter(name, name, keyspaceMeter);
+ }
+
+ protected TableMeter createTableMeter(String name, String alias, Meter keyspaceMeter)
+ {
+ Meter meter = Metrics.meter(factory.createMetricName(name), aliasFactory.createMetricName(alias));
+ register(name, alias, meter);
+ return new TableMeter(meter,
+ keyspaceMeter,
+ Metrics.meter(globalFactory.createMetricName(name),
+ globalAliasFactory.createMetricName(alias)));
+ }
+
+ private LatencyMetrics createLatencyMetrics(String namePrefix, LatencyMetrics ... parents)
+ {
+ LatencyMetrics metric = new LatencyMetrics(factory, namePrefix, parents);
+ all.add(() -> metric.release());
+ return metric;
+ }
+
/**
* Registers a metric to be removed when unloading CF.
* @return true if first time metric with that name has been registered
@@ -890,10 +1176,41 @@
{
boolean ret = allTableMetrics.putIfAbsent(name, ConcurrentHashMap.newKeySet()) == null;
allTableMetrics.get(name).add(metric);
- all.put(name, alias);
+ all.add(() -> releaseMetric(name, alias));
return ret;
}
+ private void releaseMetric(String metricName, String metricAlias)
+ {
+ CassandraMetricsRegistry.MetricName name = factory.createMetricName(metricName);
+ CassandraMetricsRegistry.MetricName alias = aliasFactory.createMetricName(metricAlias);
+ final Metric metric = Metrics.getMetrics().get(name.getMetricName());
+ if (metric != null)
+ { // Metric will be null if we are releasing a view metric. Views have null for ViewLockAcquireTime and ViewLockReadTime
+ allTableMetrics.get(metricName).remove(metric);
+ Metrics.remove(name, alias);
+ }
+ }
+
+ public static class TableMeter
+ {
+ public final Meter[] all;
+ public final Meter table;
+ private TableMeter(Meter table, Meter keyspace, Meter global)
+ {
+ this.table = table;
+ this.all = new Meter[]{table, keyspace, global};
+ }
+
+ public void mark()
+ {
+ for (Meter meter : all)
+ {
+ meter.mark();
+ }
+ }
+ }
+
public static class TableHistogram
{
public final Histogram[] all;
@@ -930,6 +1247,30 @@
timer.update(i, unit);
}
}
+
+ public Context time()
+ {
+ return new Context(all);
+ }
+
+ public static class Context implements AutoCloseable
+ {
+ private final long start;
+ private final Timer [] all;
+
+ private Context(Timer [] all)
+ {
+ this.all = all;
+ start = System.nanoTime();
+ }
+
+ public void close()
+ {
+ long duration = System.nanoTime() - start;
+ for (Timer t : all)
+ t.update(duration, TimeUnit.NANOSECONDS);
+ }
+ }
}
static class TableMetricNameFactory implements MetricNameFactory
@@ -982,8 +1323,9 @@
}
}
- public enum Sampler
+ @FunctionalInterface
+ public interface ReleasableMetric
{
- READS, WRITES
+ void release();
}
}
diff --git a/src/java/org/apache/cassandra/metrics/ThreadPoolMetricNameFactory.java b/src/java/org/apache/cassandra/metrics/ThreadPoolMetricNameFactory.java
deleted file mode 100644
index 7810108..0000000
--- a/src/java/org/apache/cassandra/metrics/ThreadPoolMetricNameFactory.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.metrics;
-
-class ThreadPoolMetricNameFactory implements MetricNameFactory
-{
- private final String type;
- private final String path;
- private final String poolName;
-
- ThreadPoolMetricNameFactory(String type, String path, String poolName)
- {
- this.type = type;
- this.path = path;
- this.poolName = poolName;
- }
-
- public CassandraMetricsRegistry.MetricName createMetricName(String metricName)
- {
- String groupName = ThreadPoolMetrics.class.getPackage().getName();
- StringBuilder mbeanName = new StringBuilder();
- mbeanName.append(groupName).append(":");
- mbeanName.append("type=").append(type);
- mbeanName.append(",path=").append(path);
- mbeanName.append(",scope=").append(poolName);
- mbeanName.append(",name=").append(metricName);
-
- return new CassandraMetricsRegistry.MetricName(groupName, type, metricName, path + "." + poolName, mbeanName.toString());
- }
-}
diff --git a/src/java/org/apache/cassandra/metrics/ThreadPoolMetrics.java b/src/java/org/apache/cassandra/metrics/ThreadPoolMetrics.java
index 268e878..3ba984a 100644
--- a/src/java/org/apache/cassandra/metrics/ThreadPoolMetrics.java
+++ b/src/java/org/apache/cassandra/metrics/ThreadPoolMetrics.java
@@ -17,47 +17,56 @@
*/
package org.apache.cassandra.metrics;
-import java.io.IOException;
-import java.util.Set;
import java.util.concurrent.ThreadPoolExecutor;
import com.codahale.metrics.Counter;
import com.codahale.metrics.Gauge;
-import com.codahale.metrics.JmxReporter;
+import org.apache.cassandra.concurrent.LocalAwareExecutorService;
+import org.apache.cassandra.metrics.CassandraMetricsRegistry.MetricName;
-import javax.management.JMX;
-import javax.management.MBeanServerConnection;
-import javax.management.MalformedObjectNameException;
-import javax.management.ObjectName;
-
-import com.google.common.collect.HashMultimap;
-import com.google.common.collect.Multimap;
+import static java.lang.String.format;
import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics;
-
/**
* Metrics for {@link ThreadPoolExecutor}.
*/
public class ThreadPoolMetrics
{
+ public static final String ACTIVE_TASKS = "ActiveTasks";
+ public static final String PENDING_TASKS = "PendingTasks";
+ public static final String COMPLETED_TASKS = "CompletedTasks";
+ public static final String CURRENTLY_BLOCKED_TASKS = "CurrentlyBlockedTasks";
+ public static final String TOTAL_BLOCKED_TASKS = "TotalBlockedTasks";
+ public static final String MAX_POOL_SIZE = "MaxPoolSize";
+ public static final String MAX_TASKS_QUEUED = "MaxTasksQueued";
+
/** Number of active tasks. */
public final Gauge<Integer> activeTasks;
- /** Number of tasks that had blocked before being accepted (or rejected). */
- public final Counter totalBlocked;
+
+ /** Number of tasks waiting to be executed. */
+ public final Gauge<Integer> pendingTasks;
+
+ /** Number of completed tasks. */
+ public final Gauge<Long> completedTasks;
+
/**
* Number of tasks currently blocked, waiting to be accepted by
* the executor (because all threads are busy and the backing queue is full).
*/
public final Counter currentBlocked;
- /** Number of completed tasks. */
- public final Gauge<Long> completedTasks;
- /** Number of tasks waiting to be executed. */
- public final Gauge<Long> pendingTasks;
+
+ /** Number of tasks that had blocked before being accepted (or rejected). */
+ public final Counter totalBlocked;
+
/** Maximum number of threads before it will start queuing tasks */
public final Gauge<Integer> maxPoolSize;
- private MetricNameFactory factory;
+ /** Maximum number of tasks queued before a task get blocked */
+ public final Gauge<Integer> maxTasksQueued;
+
+ public final String path;
+ public final String poolName;
/**
* Create metrics for given ThreadPoolExecutor.
@@ -66,105 +75,51 @@
* @param path Type of thread pool
* @param poolName Name of thread pool to identify metrics
*/
- public ThreadPoolMetrics(final ThreadPoolExecutor executor, String path, String poolName)
+ public ThreadPoolMetrics(LocalAwareExecutorService executor, String path, String poolName)
{
- this.factory = new ThreadPoolMetricNameFactory("ThreadPools", path, poolName);
+ this.path = path;
+ this.poolName = poolName;
- activeTasks = Metrics.register(factory.createMetricName("ActiveTasks"), new Gauge<Integer>()
- {
- public Integer getValue()
- {
- return executor.getActiveCount();
- }
- });
- totalBlocked = Metrics.counter(factory.createMetricName("TotalBlockedTasks"));
- currentBlocked = Metrics.counter(factory.createMetricName("CurrentlyBlockedTasks"));
- completedTasks = Metrics.register(factory.createMetricName("CompletedTasks"), new Gauge<Long>()
- {
- public Long getValue()
- {
- return executor.getCompletedTaskCount();
- }
- });
- pendingTasks = Metrics.register(factory.createMetricName("PendingTasks"), new Gauge<Long>()
- {
- public Long getValue()
- {
- return executor.getTaskCount() - executor.getCompletedTaskCount();
- }
- });
- maxPoolSize = Metrics.register(factory.createMetricName("MaxPoolSize"), new Gauge<Integer>()
- {
- public Integer getValue()
- {
- return executor.getMaximumPoolSize();
- }
- });
+ totalBlocked = new Counter();
+ currentBlocked = new Counter();
+ activeTasks = executor::getActiveTaskCount;
+ pendingTasks = executor::getPendingTaskCount;
+ completedTasks = executor::getCompletedTaskCount;
+ maxPoolSize = executor::getMaximumPoolSize;
+ maxTasksQueued = executor::getMaxTasksQueued;
+ }
+
+ public ThreadPoolMetrics register()
+ {
+ Metrics.register(makeMetricName(path, poolName, ACTIVE_TASKS), activeTasks);
+ Metrics.register(makeMetricName(path, poolName, PENDING_TASKS), pendingTasks);
+ Metrics.register(makeMetricName(path, poolName, COMPLETED_TASKS), completedTasks);
+ Metrics.register(makeMetricName(path, poolName, CURRENTLY_BLOCKED_TASKS), currentBlocked);
+ Metrics.register(makeMetricName(path, poolName, TOTAL_BLOCKED_TASKS), totalBlocked);
+ Metrics.register(makeMetricName(path, poolName, MAX_POOL_SIZE), maxPoolSize);
+ Metrics.register(makeMetricName(path, poolName, MAX_TASKS_QUEUED), maxTasksQueued);
+ return Metrics.register(this);
}
public void release()
{
- Metrics.remove(factory.createMetricName("ActiveTasks"));
- Metrics.remove(factory.createMetricName("PendingTasks"));
- Metrics.remove(factory.createMetricName("CompletedTasks"));
- Metrics.remove(factory.createMetricName("TotalBlockedTasks"));
- Metrics.remove(factory.createMetricName("CurrentlyBlockedTasks"));
- Metrics.remove(factory.createMetricName("MaxPoolSize"));
+ Metrics.remove(makeMetricName(path, poolName, ACTIVE_TASKS));
+ Metrics.remove(makeMetricName(path, poolName, PENDING_TASKS));
+ Metrics.remove(makeMetricName(path, poolName, COMPLETED_TASKS));
+ Metrics.remove(makeMetricName(path, poolName, CURRENTLY_BLOCKED_TASKS));
+ Metrics.remove(makeMetricName(path, poolName, TOTAL_BLOCKED_TASKS));
+ Metrics.remove(makeMetricName(path, poolName, MAX_POOL_SIZE));
+ Metrics.remove(makeMetricName(path, poolName, MAX_TASKS_QUEUED));
+ Metrics.remove(this);
}
- public static Object getJmxMetric(MBeanServerConnection mbeanServerConn, String jmxPath, String poolName, String metricName)
+ private static MetricName makeMetricName(String path, String poolName, String metricName)
{
- String name = String.format("org.apache.cassandra.metrics:type=ThreadPools,path=%s,scope=%s,name=%s", jmxPath, poolName, metricName);
-
- try
- {
- ObjectName oName = new ObjectName(name);
- if (!mbeanServerConn.isRegistered(oName))
- {
- return "N/A";
- }
-
- switch (metricName)
- {
- case "ActiveTasks":
- case "PendingTasks":
- case "CompletedTasks":
- return JMX.newMBeanProxy(mbeanServerConn, oName, JmxReporter.JmxGaugeMBean.class).getValue();
- case "TotalBlockedTasks":
- case "CurrentlyBlockedTasks":
- return JMX.newMBeanProxy(mbeanServerConn, oName, JmxReporter.JmxCounterMBean.class).getCount();
- default:
- throw new AssertionError("Unknown metric name " + metricName);
- }
- }
- catch (Exception e)
- {
- throw new RuntimeException("Error reading: " + name, e);
- }
+ return new MetricName("org.apache.cassandra.metrics",
+ "ThreadPools",
+ metricName,
+ path + '.' + poolName,
+ format("org.apache.cassandra.metrics:type=ThreadPools,path=%s,scope=%s,name=%s",
+ path, poolName, metricName));
}
-
- public static Multimap<String, String> getJmxThreadPools(MBeanServerConnection mbeanServerConn)
- {
- try
- {
- Multimap<String, String> threadPools = HashMultimap.create();
- Set<ObjectName> threadPoolObjectNames = mbeanServerConn.queryNames(new ObjectName("org.apache.cassandra.metrics:type=ThreadPools,*"),
- null);
- for (ObjectName oName : threadPoolObjectNames)
- {
- threadPools.put(oName.getKeyProperty("path"), oName.getKeyProperty("scope"));
- }
-
- return threadPools;
- }
- catch (MalformedObjectNameException e)
- {
- throw new RuntimeException("Bad query to JMX server: ", e);
- }
- catch (IOException e)
- {
- throw new RuntimeException("Error getting threadpool names from JMX", e);
- }
- }
-
}
diff --git a/src/java/org/apache/cassandra/net/AcceptVersions.java b/src/java/org/apache/cassandra/net/AcceptVersions.java
new file mode 100644
index 0000000..61ae049
--- /dev/null
+++ b/src/java/org/apache/cassandra/net/AcceptVersions.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.net;
+
+/**
+ * Encapsulates minimum and maximum messaging versions that a node accepts.
+ */
+class AcceptVersions
+{
+ final int min, max;
+
+ AcceptVersions(int min, int max)
+ {
+ this.min = min;
+ this.max = max;
+ }
+
+ @Override
+ public boolean equals(Object that)
+ {
+ if (!(that instanceof AcceptVersions))
+ return false;
+
+ return min == ((AcceptVersions) that).min
+ && max == ((AcceptVersions) that).max;
+ }
+}
diff --git a/src/java/org/apache/cassandra/net/AsyncChannelOutputPlus.java b/src/java/org/apache/cassandra/net/AsyncChannelOutputPlus.java
new file mode 100644
index 0000000..163981c
--- /dev/null
+++ b/src/java/org/apache/cassandra/net/AsyncChannelOutputPlus.java
@@ -0,0 +1,268 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.net;
+
+import java.io.IOException;
+import java.nio.channels.WritableByteChannel;
+import java.util.concurrent.locks.LockSupport;
+
+import io.netty.channel.Channel;
+import io.netty.channel.ChannelPromise;
+import org.apache.cassandra.io.util.BufferedDataOutputStreamPlus;
+import org.apache.cassandra.io.util.DataOutputStreamPlus;
+
+import static java.lang.Math.max;
+
+/**
+ * A {@link DataOutputStreamPlus} that writes ASYNCHRONOUSLY to a Netty Channel.
+ *
+ * The close() and flush() methods synchronously wait for pending writes, and will propagate any exceptions
+ * encountered in writing them to the wire.
+ *
+ * The correctness of this class depends on the ChannelPromise we create against a Channel always being completed,
+ * which appears to be a guarantee provided by Netty so long as the event loop is running.
+ *
+ * There are two logical threads accessing the state in this class: the eventLoop of the channel, and the writer
+ * (the writer thread may change, so long as only one utilises the class at any time).
+ * Each thread has exclusive write access to certain state in the class, with the other thread only viewing the state,
+ * simplifying concurrency considerations.
+ */
+public abstract class AsyncChannelOutputPlus extends BufferedDataOutputStreamPlus
+{
+ public static class FlushException extends IOException
+ {
+ public FlushException(String message)
+ {
+ super(message);
+ }
+
+ public FlushException(String message, Throwable cause)
+ {
+ super(message, cause);
+ }
+ }
+
+ final Channel channel;
+
+ /** the number of bytes we have begun flushing; updated only by writer */
+ private volatile long flushing;
+ /** the number of bytes we have finished flushing, successfully or otherwise; updated only by eventLoop */
+ private volatile long flushed;
+ /** the number of bytes we have finished flushing to the network; updated only by eventLoop */
+ private long flushedToNetwork;
+ /** any error that has been thrown during a flush; updated only by eventLoop */
+ private volatile Throwable flushFailed;
+
+ /**
+ * state for pausing until flushing has caught up - store the number of bytes we need to be flushed before
+ * we should be signalled, and store ourselves in {@link #waiting}; once the flushing thread exceeds this many
+ * total bytes flushed, any Thread stored in waiting will be signalled.
+ *
+ * This works exactly like using a WaitQueue, except that we only need to manage a single waiting thread.
+ */
+ private volatile long signalWhenFlushed; // updated only by writer
+ private volatile Thread waiting; // updated only by writer
+
+ public AsyncChannelOutputPlus(Channel channel)
+ {
+ super(null, null);
+ this.channel = channel;
+ }
+
+ /**
+ * Create a ChannelPromise for a flush of the given size.
+ * <p>
+ * This method will not return until the write is permitted by the provided watermarks and in flight bytes,
+ * and on its completion will mark the requested bytes flushed.
+ * <p>
+ * If this method returns normally, the ChannelPromise MUST be writtenAndFlushed, or else completed exceptionally.
+ */
+ protected ChannelPromise beginFlush(int byteCount, int lowWaterMark, int highWaterMark) throws IOException
+ {
+ waitForSpace(byteCount, lowWaterMark, highWaterMark);
+
+ return AsyncChannelPromise.withListener(channel, future -> {
+ if (future.isSuccess() && null == flushFailed)
+ {
+ flushedToNetwork += byteCount;
+ releaseSpace(byteCount);
+ }
+ else if (null == flushFailed)
+ {
+ Throwable cause = future.cause();
+ if (cause == null)
+ {
+ cause = new FlushException("Flush failed for unknown reason");
+ cause.fillInStackTrace();
+ }
+ flushFailed = cause;
+ releaseSpace(flushing - flushed);
+ }
+ else
+ {
+ assert flushing == flushed;
+ }
+ });
+ }
+
+ /**
+ * Imposes our lowWaterMark/highWaterMark constraints, and propagates any exceptions thrown by prior flushes.
+ *
+ * If we currently have lowWaterMark or fewer bytes flushing, we are good to go.
+ * If our new write will not take us over our highWaterMark, we are good to go.
+ * Otherwise we wait until either of these conditions are met.
+ *
+ * This may only be invoked by the writer thread, never by the eventLoop.
+ *
+ * @throws IOException if a prior asynchronous flush failed
+ */
+ private void waitForSpace(int bytesToWrite, int lowWaterMark, int highWaterMark) throws IOException
+ {
+ // decide when we would be willing to carry on writing
+ // we are always writable if we have lowWaterMark or fewer bytes, no matter how many bytes we are flushing
+ // our callers should not be supplying more than (highWaterMark - lowWaterMark) bytes, but we must work correctly if they do
+ int wakeUpWhenFlushing = highWaterMark - bytesToWrite;
+ waitUntilFlushed(max(lowWaterMark, wakeUpWhenFlushing), lowWaterMark);
+ flushing += bytesToWrite;
+ }
+
+ /**
+ * Implementation of waitForSpace, which calculates what flushed points we need to wait for,
+ * parks if necessary and propagates flush failures.
+ *
+ * This may only be invoked by the writer thread, never by the eventLoop.
+ */
+ void waitUntilFlushed(int wakeUpWhenExcessBytesWritten, int signalWhenExcessBytesWritten) throws IOException
+ {
+ // we assume that we are happy to wake up at least as early as we will be signalled; otherwise we will never exit
+ assert signalWhenExcessBytesWritten <= wakeUpWhenExcessBytesWritten;
+ // flushing shouldn't change during this method invocation, so our calculations for signal and flushed are consistent
+ long wakeUpWhenFlushed = flushing - wakeUpWhenExcessBytesWritten;
+ if (flushed < wakeUpWhenFlushed)
+ parkUntilFlushed(wakeUpWhenFlushed, flushing - signalWhenExcessBytesWritten);
+ propagateFailedFlush();
+ }
+
+ /**
+ * Utility method for waitUntilFlushed, which actually parks the current thread until the necessary
+ * number of bytes have been flushed
+ *
+ * This may only be invoked by the writer thread, never by the eventLoop.
+ */
+ protected void parkUntilFlushed(long wakeUpWhenFlushed, long signalWhenFlushed)
+ {
+ assert wakeUpWhenFlushed <= signalWhenFlushed;
+ assert waiting == null;
+ this.waiting = Thread.currentThread();
+ this.signalWhenFlushed = signalWhenFlushed;
+
+ while (flushed < wakeUpWhenFlushed)
+ LockSupport.park();
+ waiting = null;
+ }
+
+ /**
+ * Update our flushed count, and signal any waiters.
+ *
+ * This may only be invoked by the eventLoop, never by the writer thread.
+ */
+ protected void releaseSpace(long bytesFlushed)
+ {
+ long newFlushed = flushed + bytesFlushed;
+ flushed = newFlushed;
+
+ Thread thread = waiting;
+ if (thread != null && signalWhenFlushed <= newFlushed)
+ LockSupport.unpark(thread);
+ }
+
+ private void propagateFailedFlush() throws IOException
+ {
+ Throwable t = flushFailed;
+ if (t != null)
+ {
+ if (SocketFactory.isCausedByConnectionReset(t))
+ throw new FlushException("The channel this output stream was writing to has been closed", t);
+ throw new FlushException("This output stream is in an unsafe state after an asynchronous flush failed", t);
+ }
+ }
+
+ @Override
+ abstract protected void doFlush(int count) throws IOException;
+
+ abstract public long position();
+
+ public long flushed()
+ {
+ // external flushed (that which has had flush() invoked implicitly or otherwise) == internal flushing
+ return flushing;
+ }
+
+ public long flushedToNetwork()
+ {
+ return flushedToNetwork;
+ }
+
+ /**
+ * Perform an asynchronous flush, then waits until all outstanding flushes have completed
+ *
+ * @throws IOException if any flush fails
+ */
+ @Override
+ public void flush() throws IOException
+ {
+ doFlush(0);
+ waitUntilFlushed(0, 0);
+ }
+
+ /**
+ * Flush any remaining writes, and release any buffers.
+ *
+ * The channel is not closed, as it is assumed to be managed externally.
+ *
+ * WARNING: This method requires mutual exclusivity with all other producer methods to run safely.
+ * It should only be invoked by the owning thread, never the eventLoop; the eventLoop should propagate
+ * errors to {@link #flushFailed}, which will propagate them to the producer thread no later than its
+ * final invocation to {@link #close()} or {@link #flush()} (that must not be followed by any further writes).
+ */
+ @Override
+ public void close() throws IOException
+ {
+ try
+ {
+ flush();
+ }
+ finally
+ {
+ discard();
+ }
+ }
+
+ /**
+ * Discard any buffered data, and the buffers that contain it.
+ * May be invoked instead of {@link #close()} if we terminate exceptionally.
+ */
+ public abstract void discard();
+
+ @Override
+ protected WritableByteChannel newDefaultChannel()
+ {
+ throw new UnsupportedOperationException();
+ }
+
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/net/AsyncChannelPromise.java b/src/java/org/apache/cassandra/net/AsyncChannelPromise.java
new file mode 100644
index 0000000..d2c9d0b
--- /dev/null
+++ b/src/java/org/apache/cassandra/net/AsyncChannelPromise.java
@@ -0,0 +1,164 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.net;
+
+import io.netty.channel.Channel;
+import io.netty.channel.ChannelFuture;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.channel.ChannelPromise;
+import io.netty.util.concurrent.Future;
+import io.netty.util.concurrent.GenericFutureListener;
+
+/**
+ * See {@link AsyncPromise} and {@link io.netty.channel.ChannelPromise}
+ *
+ * This class is all boiler plate, just ensuring we return ourselves and invoke the correct Promise method.
+ */
+public class AsyncChannelPromise extends AsyncPromise<Void> implements ChannelPromise
+{
+ private final Channel channel;
+
+ @SuppressWarnings("unused")
+ public AsyncChannelPromise(Channel channel)
+ {
+ super(channel.eventLoop());
+ this.channel = channel;
+ }
+
+ AsyncChannelPromise(Channel channel, GenericFutureListener<? extends Future<? super Void>> listener)
+ {
+ super(channel.eventLoop(), listener);
+ this.channel = channel;
+ }
+
+ public static AsyncChannelPromise withListener(ChannelHandlerContext context, GenericFutureListener<? extends Future<? super Void>> listener)
+ {
+ return withListener(context.channel(), listener);
+ }
+
+ public static AsyncChannelPromise withListener(Channel channel, GenericFutureListener<? extends Future<? super Void>> listener)
+ {
+ return new AsyncChannelPromise(channel, listener);
+ }
+
+ public static ChannelFuture writeAndFlush(ChannelHandlerContext context, Object message, GenericFutureListener<? extends Future<? super Void>> listener)
+ {
+ return context.writeAndFlush(message, withListener(context.channel(), listener));
+ }
+
+ public static ChannelFuture writeAndFlush(Channel channel, Object message, GenericFutureListener<? extends Future<? super Void>> listener)
+ {
+ return channel.writeAndFlush(message, withListener(channel, listener));
+ }
+
+ public static ChannelFuture writeAndFlush(ChannelHandlerContext context, Object message)
+ {
+ return context.writeAndFlush(message, new AsyncChannelPromise(context.channel()));
+ }
+
+ public static ChannelFuture writeAndFlush(Channel channel, Object message)
+ {
+ return channel.writeAndFlush(message, new AsyncChannelPromise(channel));
+ }
+
+ public Channel channel()
+ {
+ return channel;
+ }
+
+ public boolean isVoid()
+ {
+ return false;
+ }
+
+ public ChannelPromise setSuccess()
+ {
+ return setSuccess(null);
+ }
+
+ public ChannelPromise setSuccess(Void v)
+ {
+ super.setSuccess(v);
+ return this;
+ }
+
+ public boolean trySuccess()
+ {
+ return trySuccess(null);
+ }
+
+ public ChannelPromise setFailure(Throwable throwable)
+ {
+ super.setFailure(throwable);
+ return this;
+ }
+
+ public ChannelPromise sync() throws InterruptedException
+ {
+ super.sync();
+ return this;
+ }
+
+ public ChannelPromise syncUninterruptibly()
+ {
+ super.syncUninterruptibly();
+ return this;
+ }
+
+ public ChannelPromise await() throws InterruptedException
+ {
+ super.await();
+ return this;
+ }
+
+ public ChannelPromise awaitUninterruptibly()
+ {
+ super.awaitUninterruptibly();
+ return this;
+ }
+
+ public ChannelPromise addListener(GenericFutureListener<? extends Future<? super Void>> listener)
+ {
+ super.addListener(listener);
+ return this;
+ }
+
+ public ChannelPromise addListeners(GenericFutureListener<? extends Future<? super Void>>... listeners)
+ {
+ super.addListeners(listeners);
+ return this;
+ }
+
+ public ChannelPromise removeListener(GenericFutureListener<? extends Future<? super Void>> listener)
+ {
+ super.removeListener(listener);
+ return this;
+ }
+
+ public ChannelPromise removeListeners(GenericFutureListener<? extends Future<? super Void>>... listeners)
+ {
+ super.removeListeners(listeners);
+ return this;
+ }
+
+ public ChannelPromise unvoid()
+ {
+ return this;
+ }
+}
diff --git a/src/java/org/apache/cassandra/net/AsyncMessageOutputPlus.java b/src/java/org/apache/cassandra/net/AsyncMessageOutputPlus.java
new file mode 100644
index 0000000..8ef0a8f
--- /dev/null
+++ b/src/java/org/apache/cassandra/net/AsyncMessageOutputPlus.java
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.net;
+
+import java.io.IOException;
+import java.nio.channels.ClosedChannelException;
+
+import io.netty.channel.Channel;
+import io.netty.channel.ChannelPromise;
+import io.netty.channel.WriteBufferWaterMark;
+import org.apache.cassandra.io.util.DataOutputStreamPlus;
+
+/**
+ * A {@link DataOutputStreamPlus} that writes ASYNCHRONOUSLY to a Netty Channel.
+ *
+ * Intended as single use, to write one (large) message.
+ *
+ * The close() and flush() methods synchronously wait for pending writes, and will propagate any exceptions
+ * encountered in writing them to the wire.
+ *
+ * The correctness of this class depends on the ChannelPromise we create against a Channel always being completed,
+ * which appears to be a guarantee provided by Netty so long as the event loop is running.
+ */
+public class AsyncMessageOutputPlus extends AsyncChannelOutputPlus
+{
+ /**
+ * the maximum {@link #highWaterMark} and minimum {@link #lowWaterMark} number of bytes we have flushing
+ * during which we should still be writing to the channel.
+ *
+ * i.e., if we are at or below the {@link #lowWaterMark} we should definitely start writing again;
+ * if we are at or above the {@link #highWaterMark} we should definitely stop writing;
+ * if we are inbetween, it is OK to either write or not write
+ *
+ * note that we consider the bytes we are about to write to our high water mark, but not our low.
+ * i.e., we will not begin a write that would take us over our high water mark, unless not doing so would
+ * take us below our low water mark.
+ *
+ * This is somewhat arbitrary accounting, and a meaningless distinction for flushes of a consistent size.
+ */
+ @SuppressWarnings("JavaDoc")
+ private final int highWaterMark;
+ private final int lowWaterMark;
+ private final int bufferSize;
+ private final int messageSize;
+ private boolean closing;
+
+ private final FrameEncoder.PayloadAllocator payloadAllocator;
+ private volatile FrameEncoder.Payload payload;
+
+ AsyncMessageOutputPlus(Channel channel, int bufferSize, int messageSize, FrameEncoder.PayloadAllocator payloadAllocator)
+ {
+ super(channel);
+ WriteBufferWaterMark waterMark = channel.config().getWriteBufferWaterMark();
+ this.lowWaterMark = waterMark.low();
+ this.highWaterMark = waterMark.high();
+ this.messageSize = messageSize;
+ this.bufferSize = Math.min(messageSize, bufferSize);
+ this.payloadAllocator = payloadAllocator;
+ allocateBuffer();
+ }
+
+ private void allocateBuffer()
+ {
+ payload = payloadAllocator.allocate(false, bufferSize);
+ buffer = payload.buffer;
+ }
+
+ @Override
+ protected void doFlush(int count) throws IOException
+ {
+ if (!channel.isOpen())
+ throw new ClosedChannelException();
+
+ // flush the current backing write buffer only if there's any pending data
+ FrameEncoder.Payload flush = payload;
+ int byteCount = flush.length();
+ if (byteCount == 0)
+ return;
+
+ if (byteCount + flushed() > (closing ? messageSize : messageSize - 1))
+ throw new InvalidSerializedSizeException(messageSize, byteCount + flushed());
+
+ flush.finish();
+ ChannelPromise promise = beginFlush(byteCount, lowWaterMark, highWaterMark);
+ channel.writeAndFlush(flush, promise);
+ allocateBuffer();
+ }
+
+ public void close() throws IOException
+ {
+ closing = true;
+ if (flushed() == 0 && payload != null)
+ payload.setSelfContained(true);
+ super.close();
+ }
+
+ public long position()
+ {
+ return flushed() + payload.length();
+ }
+
+ /**
+ * Discard any buffered data, and the buffers that contain it.
+ * May be invoked instead of {@link #close()} if we terminate exceptionally.
+ */
+ public void discard()
+ {
+ if (payload != null)
+ {
+ payload.release();
+ payload = null;
+ buffer = null;
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/net/AsyncOneResponse.java b/src/java/org/apache/cassandra/net/AsyncOneResponse.java
index b7ef227..ba83c84 100644
--- a/src/java/org/apache/cassandra/net/AsyncOneResponse.java
+++ b/src/java/org/apache/cassandra/net/AsyncOneResponse.java
@@ -17,67 +17,31 @@
*/
package org.apache.cassandra.net;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
+import com.google.common.annotations.VisibleForTesting;
+
+import io.netty.util.concurrent.ImmediateEventExecutor;
/**
* A callback specialized for returning a value from a single target; that is, this is for messages
* that we only send to one recipient.
*/
-public class AsyncOneResponse<T> implements IAsyncCallback<T>
+public class AsyncOneResponse<T> extends AsyncPromise<T> implements RequestCallback<T>
{
- private T result;
- private boolean done;
- private final long start = System.nanoTime();
-
- public T get(long timeout, TimeUnit tu) throws TimeoutException
+ public AsyncOneResponse()
{
- timeout = tu.toNanos(timeout);
- boolean interrupted = false;
- try
- {
- synchronized (this)
- {
- while (!done)
- {
- try
- {
- long overallTimeout = timeout - (System.nanoTime() - start);
- if (overallTimeout <= 0)
- {
- throw new TimeoutException("Operation timed out.");
- }
- TimeUnit.NANOSECONDS.timedWait(this, overallTimeout);
- }
- catch (InterruptedException e)
- {
- interrupted = true;
- }
- }
- }
- }
- finally
- {
- if (interrupted)
- {
- Thread.currentThread().interrupt();
- }
- }
- return result;
+ super(ImmediateEventExecutor.INSTANCE);
}
- public synchronized void response(MessageIn<T> response)
+ public void onResponse(Message<T> response)
{
- if (!done)
- {
- result = response.payload;
- done = true;
- this.notifyAll();
- }
+ setSuccess(response.payload);
}
- public boolean isLatencyForSnitch()
+ @VisibleForTesting
+ public static <T> AsyncOneResponse<T> immediate(T value)
{
- return false;
+ AsyncOneResponse<T> response = new AsyncOneResponse<>();
+ response.setSuccess(value);
+ return response;
}
}
diff --git a/src/java/org/apache/cassandra/net/AsyncPromise.java b/src/java/org/apache/cassandra/net/AsyncPromise.java
new file mode 100644
index 0000000..36bc304
--- /dev/null
+++ b/src/java/org/apache/cassandra/net/AsyncPromise.java
@@ -0,0 +1,488 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.net;
+
+import java.util.concurrent.CancellationException;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicReferenceFieldUpdater;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import io.netty.util.concurrent.EventExecutor;
+import io.netty.util.concurrent.Future;
+import io.netty.util.concurrent.GenericFutureListener;
+import io.netty.util.concurrent.Promise;
+import io.netty.util.internal.PlatformDependent;
+import io.netty.util.internal.ThrowableUtil;
+import org.apache.cassandra.utils.concurrent.WaitQueue;
+
+import static java.util.concurrent.atomic.AtomicReferenceFieldUpdater.*;
+
+/**
+ * Netty's DefaultPromise uses a mutex to coordinate notifiers AND waiters between the eventLoop and the other threads.
+ * Since we register cross-thread listeners, this has the potential to block internode messaging for an unknown
+ * number of threads for an unknown period of time, if we are unlucky with the scheduler (which will certainly
+ * happen, just with some unknown but low periodicity)
+ *
+ * At the same time, we manage some other efficiencies:
+ * - We save some space when registering listeners, especially if there is only one listener, as we perform no
+ * extra allocations in this case.
+ * - We permit efficient initial state declaration, avoiding unnecessary CAS or lock acquisitions when mutating
+ * a Promise we are ourselves constructing (and can easily add more; only those we use have been added)
+ *
+ * We can also make some guarantees about our behaviour here, although we primarily mirror Netty.
+ * Specifically, we can guarantee that notifiers are always invoked in the order they are added (which may be true
+ * for netty, but was unclear and is not declared). This is useful for ensuring the correctness of some of our
+ * behaviours in OutboundConnection without having to jump through extra hoops.
+ *
+ * The implementation loosely follows that of Netty's DefaultPromise, with some slight changes; notably that we have
+ * no synchronisation on our listeners, instead using a CoW list that is cleared each time we notify listeners.
+ *
+ * We handle special values slightly differently. We do not use a special value for null, instead using
+ * a special value to indicate the result has not been set yet. This means that once isSuccess() holds,
+ * the result must be a correctly typed object (modulo generics pitfalls).
+ * All special values are also instances of FailureHolder, which simplifies a number of the logical conditions.
+ *
+ * @param <V>
+ */
+public class AsyncPromise<V> implements Promise<V>
+{
+ private static final Logger logger = LoggerFactory.getLogger(AsyncPromise.class);
+
+ private final EventExecutor executor;
+ private volatile Object result;
+ private volatile GenericFutureListener<? extends Future<? super V>> listeners;
+ private volatile WaitQueue waiting;
+ private static final AtomicReferenceFieldUpdater<AsyncPromise, Object> resultUpdater = newUpdater(AsyncPromise.class, Object.class, "result");
+ private static final AtomicReferenceFieldUpdater<AsyncPromise, GenericFutureListener> listenersUpdater = newUpdater(AsyncPromise.class, GenericFutureListener.class, "listeners");
+ private static final AtomicReferenceFieldUpdater<AsyncPromise, WaitQueue> waitingUpdater = newUpdater(AsyncPromise.class, WaitQueue.class, "waiting");
+
+ private static final FailureHolder UNSET = new FailureHolder(null);
+ private static final FailureHolder UNCANCELLABLE = new FailureHolder(null);
+ private static final FailureHolder CANCELLED = new FailureHolder(ThrowableUtil.unknownStackTrace(new CancellationException(), AsyncPromise.class, "cancel(...)"));
+
+ private static final DeferredGenericFutureListener NOTIFYING = future -> {};
+ private static interface DeferredGenericFutureListener<F extends Future<?>> extends GenericFutureListener<F> {}
+
+ private static final class FailureHolder
+ {
+ final Throwable cause;
+ private FailureHolder(Throwable cause)
+ {
+ this.cause = cause;
+ }
+ }
+
+ public AsyncPromise(EventExecutor executor)
+ {
+ this(executor, UNSET);
+ }
+
+ private AsyncPromise(EventExecutor executor, FailureHolder initialState)
+ {
+ this.executor = executor;
+ this.result = initialState;
+ }
+
+ public AsyncPromise(EventExecutor executor, GenericFutureListener<? extends Future<? super V>> listener)
+ {
+ this(executor);
+ this.listeners = listener;
+ }
+
+ AsyncPromise(EventExecutor executor, FailureHolder initialState, GenericFutureListener<? extends Future<? super V>> listener)
+ {
+ this(executor, initialState);
+ this.listeners = listener;
+ }
+
+ public static <V> AsyncPromise<V> uncancellable(EventExecutor executor)
+ {
+ return new AsyncPromise<>(executor, UNCANCELLABLE);
+ }
+
+ public static <V> AsyncPromise<V> uncancellable(EventExecutor executor, GenericFutureListener<? extends Future<? super V>> listener)
+ {
+ return new AsyncPromise<>(executor, UNCANCELLABLE);
+ }
+
+ public Promise<V> setSuccess(V v)
+ {
+ if (!trySuccess(v))
+ throw new IllegalStateException("complete already: " + this);
+ return this;
+ }
+
+ public Promise<V> setFailure(Throwable throwable)
+ {
+ if (!tryFailure(throwable))
+ throw new IllegalStateException("complete already: " + this);
+ return this;
+ }
+
+ public boolean trySuccess(V v)
+ {
+ return trySet(v);
+ }
+
+ public boolean tryFailure(Throwable throwable)
+ {
+ return trySet(new FailureHolder(throwable));
+ }
+
+ public boolean setUncancellable()
+ {
+ if (trySet(UNCANCELLABLE))
+ return true;
+ return result == UNCANCELLABLE;
+ }
+
+ public boolean cancel(boolean b)
+ {
+ return trySet(CANCELLED);
+ }
+
+ /**
+ * Shared implementation of various promise completion methods.
+ * Updates the result if it is possible to do so, returning success/failure.
+ *
+ * If the promise is UNSET the new value will succeed;
+ * if it is UNCANCELLABLE it will succeed only if the new value is not CANCELLED
+ * otherwise it will fail, as isDone() is implied
+ *
+ * If the update succeeds, and the new state implies isDone(), any listeners and waiters will be notified
+ */
+ private boolean trySet(Object v)
+ {
+ while (true)
+ {
+ Object current = result;
+ if (isDone(current) || (current == UNCANCELLABLE && v == CANCELLED))
+ return false;
+ if (resultUpdater.compareAndSet(this, current, v))
+ {
+ if (v != UNCANCELLABLE)
+ {
+ notifyListeners();
+ notifyWaiters();
+ }
+ return true;
+ }
+ }
+ }
+
+ public boolean isSuccess()
+ {
+ return isSuccess(result);
+ }
+
+ private static boolean isSuccess(Object result)
+ {
+ return !(result instanceof FailureHolder);
+ }
+
+ public boolean isCancelled()
+ {
+ return isCancelled(result);
+ }
+
+ private static boolean isCancelled(Object result)
+ {
+ return result == CANCELLED;
+ }
+
+ public boolean isDone()
+ {
+ return isDone(result);
+ }
+
+ private static boolean isDone(Object result)
+ {
+ return result != UNSET && result != UNCANCELLABLE;
+ }
+
+ public boolean isCancellable()
+ {
+ Object result = this.result;
+ return result == UNSET;
+ }
+
+ public Throwable cause()
+ {
+ Object result = this.result;
+ if (result instanceof FailureHolder)
+ return ((FailureHolder) result).cause;
+ return null;
+ }
+
+ /**
+ * if isSuccess(), returns the value, otherwise returns null
+ */
+ @SuppressWarnings("unchecked")
+ public V getNow()
+ {
+ Object result = this.result;
+ if (isSuccess(result))
+ return (V) result;
+ return null;
+ }
+
+ public V get() throws InterruptedException, ExecutionException
+ {
+ await();
+ return getWhenDone();
+ }
+
+ public V get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException
+ {
+ if (!await(timeout, unit))
+ throw new TimeoutException();
+ return getWhenDone();
+ }
+
+ /**
+ * Shared implementation of get() after suitable await(); assumes isDone(), and returns
+ * either the success result or throws the suitable exception under failure
+ */
+ @SuppressWarnings("unchecked")
+ private V getWhenDone() throws ExecutionException
+ {
+ Object result = this.result;
+ if (isSuccess(result))
+ return (V) result;
+ if (result == CANCELLED)
+ throw new CancellationException();
+ throw new ExecutionException(((FailureHolder) result).cause);
+ }
+
+ /**
+ * waits for completion; in case of failure rethrows the original exception without a new wrapping exception
+ * so may cause problems for reporting stack traces
+ */
+ public Promise<V> sync() throws InterruptedException
+ {
+ await();
+ rethrowIfFailed();
+ return this;
+ }
+
+ /**
+ * waits for completion; in case of failure rethrows the original exception without a new wrapping exception
+ * so may cause problems for reporting stack traces
+ */
+ public Promise<V> syncUninterruptibly()
+ {
+ awaitUninterruptibly();
+ rethrowIfFailed();
+ return this;
+ }
+
+ private void rethrowIfFailed()
+ {
+ Throwable cause = this.cause();
+ if (cause != null)
+ {
+ PlatformDependent.throwException(cause);
+ }
+ }
+
+ public Promise<V> addListener(GenericFutureListener<? extends Future<? super V>> listener)
+ {
+ listenersUpdater.accumulateAndGet(this, listener, AsyncPromise::appendListener);
+ if (isDone())
+ notifyListeners();
+ return this;
+ }
+
+ public Promise<V> addListeners(GenericFutureListener<? extends Future<? super V>> ... listeners)
+ {
+ // this could be more efficient if we cared, but we do not
+ return addListener(future -> {
+ for (GenericFutureListener<? extends Future<? super V>> listener : listeners)
+ AsyncPromise.invokeListener((GenericFutureListener<Future<? super V>>)listener, future);
+ });
+ }
+
+ public Promise<V> removeListener(GenericFutureListener<? extends Future<? super V>> listener)
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ public Promise<V> removeListeners(GenericFutureListener<? extends Future<? super V>> ... listeners)
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @SuppressWarnings("unchecked")
+ private void notifyListeners()
+ {
+ if (!executor.inEventLoop())
+ {
+ // submit this method, to guarantee we invoke in the submitted order
+ executor.execute(this::notifyListeners);
+ return;
+ }
+
+ if (listeners == null || listeners instanceof DeferredGenericFutureListener<?>)
+ return; // either no listeners, or we are already notifying listeners, so we'll get to the new one when ready
+
+ // first run our notifiers
+ while (true)
+ {
+ GenericFutureListener listeners = listenersUpdater.getAndSet(this, NOTIFYING);
+ if (listeners != null)
+ invokeListener(listeners, this);
+
+ if (listenersUpdater.compareAndSet(this, NOTIFYING, null))
+ return;
+ }
+ }
+
+ private static <F extends Future<?>> void invokeListener(GenericFutureListener<F> listener, F future)
+ {
+ try
+ {
+ listener.operationComplete(future);
+ }
+ catch (Throwable t)
+ {
+ logger.error("Failed to invoke listener {} to {}", listener, future, t);
+ }
+ }
+
+ private static <F extends Future<?>> GenericFutureListener<F> appendListener(GenericFutureListener<F> prevListener, GenericFutureListener<F> newListener)
+ {
+ GenericFutureListener<F> result = newListener;
+
+ if (prevListener != null && prevListener != NOTIFYING)
+ {
+ result = future -> {
+ invokeListener(prevListener, future);
+ // we will wrap the outer invocation with invokeListener, so no need to do it here too
+ newListener.operationComplete(future);
+ };
+ }
+
+ if (prevListener instanceof DeferredGenericFutureListener<?>)
+ {
+ GenericFutureListener<F> wrap = result;
+ result = (DeferredGenericFutureListener<F>) wrap::operationComplete;
+ }
+
+ return result;
+ }
+
+ public Promise<V> await() throws InterruptedException
+ {
+ await(0L, (signal, nanos) -> { signal.await(); return true; } );
+ return this;
+ }
+
+ public Promise<V> awaitUninterruptibly()
+ {
+ await(0L, (signal, nanos) -> { signal.awaitUninterruptibly(); return true; } );
+ return this;
+ }
+
+ public boolean await(long timeout, TimeUnit unit) throws InterruptedException
+ {
+ return await(unit.toNanos(timeout),
+ (signal, nanos) -> signal.awaitUntil(nanos + System.nanoTime()));
+ }
+
+ public boolean await(long timeoutMillis) throws InterruptedException
+ {
+ return await(timeoutMillis, TimeUnit.MILLISECONDS);
+ }
+
+ public boolean awaitUninterruptibly(long timeout, TimeUnit unit)
+ {
+ return await(unit.toNanos(timeout),
+ (signal, nanos) -> signal.awaitUntilUninterruptibly(nanos + System.nanoTime()));
+ }
+
+ public boolean awaitUninterruptibly(long timeoutMillis)
+ {
+ return awaitUninterruptibly(timeoutMillis, TimeUnit.MILLISECONDS);
+ }
+
+ interface Awaiter<T extends Throwable>
+ {
+ boolean await(WaitQueue.Signal value, long nanos) throws T;
+ }
+
+ /**
+ * A clean way to implement each variant of await using lambdas; we permit a nanos parameter
+ * so that we can implement this without any unnecessary lambda allocations, although not
+ * all implementations need the nanos parameter (i.e. those that wait indefinitely)
+ */
+ private <T extends Throwable> boolean await(long nanos, Awaiter<T> awaiter) throws T
+ {
+ if (isDone())
+ return true;
+
+ WaitQueue.Signal await = registerToWait();
+ if (null != await)
+ return awaiter.await(await, nanos);
+
+ return true;
+ }
+
+ /**
+ * Register a signal that will be notified when the promise is completed;
+ * if the promise becomes completed before this signal is registered, null is returned
+ */
+ private WaitQueue.Signal registerToWait()
+ {
+ WaitQueue waiting = this.waiting;
+ if (waiting == null && !waitingUpdater.compareAndSet(this, null, waiting = new WaitQueue()))
+ waiting = this.waiting;
+ assert waiting != null;
+
+ WaitQueue.Signal signal = waiting.register();
+ if (!isDone())
+ return signal;
+ signal.cancel();
+ return null;
+ }
+
+ private void notifyWaiters()
+ {
+ WaitQueue waiting = this.waiting;
+ if (waiting != null)
+ waiting.signalAll();
+ }
+
+ public String toString()
+ {
+ Object result = this.result;
+ if (isSuccess(result))
+ return "(success: " + result + ')';
+ if (result == UNCANCELLABLE)
+ return "(uncancellable)";
+ if (result == CANCELLED)
+ return "(cancelled)";
+ if (isDone(result))
+ return "(failure: " + ((FailureHolder) result).cause + ')';
+ return "(incomplete)";
+ }
+}
diff --git a/src/java/org/apache/cassandra/net/AsyncStreamingInputPlus.java b/src/java/org/apache/cassandra/net/AsyncStreamingInputPlus.java
new file mode 100644
index 0000000..84fb8ac
--- /dev/null
+++ b/src/java/org/apache/cassandra/net/AsyncStreamingInputPlus.java
@@ -0,0 +1,251 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.net;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.TimeUnit;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.primitives.Ints;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.ByteBufAllocator;
+import io.netty.buffer.Unpooled;
+import io.netty.channel.Channel;
+import org.apache.cassandra.io.util.RebufferingInputStream;
+
+// TODO: rewrite
+public class AsyncStreamingInputPlus extends RebufferingInputStream
+{
+ public static class InputTimeoutException extends IOException
+ {
+ }
+
+ private static final long DEFAULT_REBUFFER_BLOCK_IN_MILLIS = TimeUnit.MINUTES.toMillis(3);
+
+ private final Channel channel;
+
+ /**
+ * The parent, or owning, buffer of the current buffer being read from ({@link super#buffer}).
+ */
+ private ByteBuf currentBuf;
+
+ private final BlockingQueue<ByteBuf> queue;
+
+ private final long rebufferTimeoutNanos;
+
+ private volatile boolean isClosed;
+
+ public AsyncStreamingInputPlus(Channel channel)
+ {
+ this(channel, DEFAULT_REBUFFER_BLOCK_IN_MILLIS, TimeUnit.MILLISECONDS);
+ }
+
+ AsyncStreamingInputPlus(Channel channel, long rebufferTimeout, TimeUnit rebufferTimeoutUnit)
+ {
+ super(Unpooled.EMPTY_BUFFER.nioBuffer());
+ currentBuf = Unpooled.EMPTY_BUFFER;
+
+ queue = new LinkedBlockingQueue<>();
+ rebufferTimeoutNanos = rebufferTimeoutUnit.toNanos(rebufferTimeout);
+
+ this.channel = channel;
+ channel.config().setAutoRead(false);
+ }
+
+ /**
+ * Append a {@link ByteBuf} to the end of the einternal queue.
+ *
+ * Note: it's expected this method is invoked on the netty event loop.
+ */
+ public boolean append(ByteBuf buf) throws IllegalStateException
+ {
+ if (isClosed) return false;
+
+ queue.add(buf);
+
+ /*
+ * it's possible for append() to race with close(), so we need to ensure
+ * that the bytebuf gets released in that scenario
+ */
+ if (isClosed)
+ while ((buf = queue.poll()) != null)
+ buf.release();
+
+ return true;
+ }
+
+ /**
+ * {@inheritDoc}
+ *
+ * Release open buffers and poll the {@link #queue} for more data.
+ * <p>
+ * This is best, and more or less expected, to be invoked on a consuming thread (not the event loop)
+ * becasue if we block on the queue we can't fill it on the event loop (as that's where the buffers are coming from).
+ *
+ * @throws EOFException when no further reading from this instance should occur. Implies this instance is closed.
+ * @throws InputTimeoutException when no new buffers arrive for reading before
+ * the {@link #rebufferTimeoutNanos} elapses while blocking. It's then not safe to reuse this instance again.
+ */
+ @Override
+ protected void reBuffer() throws EOFException, InputTimeoutException
+ {
+ if (queue.isEmpty())
+ channel.read();
+
+ currentBuf.release();
+ currentBuf = null;
+ buffer = null;
+
+ ByteBuf next = null;
+ try
+ {
+ next = queue.poll(rebufferTimeoutNanos, TimeUnit.NANOSECONDS);
+ }
+ catch (InterruptedException ie)
+ {
+ // nop
+ }
+
+ if (null == next)
+ throw new InputTimeoutException();
+
+ if (next == Unpooled.EMPTY_BUFFER) // Unpooled.EMPTY_BUFFER is the indicator that the input is closed
+ throw new EOFException();
+
+ currentBuf = next;
+ buffer = next.nioBuffer();
+ }
+
+ public interface Consumer
+ {
+ int accept(ByteBuffer buffer) throws IOException;
+ }
+
+ /**
+ * Consumes bytes in the stream until the given length
+ */
+ public void consume(Consumer consumer, long length) throws IOException
+ {
+ while (length > 0)
+ {
+ if (!buffer.hasRemaining())
+ reBuffer();
+
+ final int position = buffer.position();
+ final int limit = buffer.limit();
+
+ buffer.limit(position + (int) Math.min(length, limit - position));
+ try
+ {
+ int copied = consumer.accept(buffer);
+ buffer.position(position + copied);
+ length -= copied;
+ }
+ finally
+ {
+ buffer.limit(limit);
+ }
+ }
+ }
+
+ /**
+ * {@inheritDoc}
+ *
+ * As long as this method is invoked on the consuming thread the returned value will be accurate.
+ */
+ @VisibleForTesting
+ public int unsafeAvailable()
+ {
+ long count = buffer != null ? buffer.remaining() : 0;
+ for (ByteBuf buf : queue)
+ count += buf.readableBytes();
+
+ return Ints.checkedCast(count);
+ }
+
+ // TODO:JEB add docs
+ // TL;DR if there's no Bufs open anywhere here, issue a channle read to try and grab data.
+ public void maybeIssueRead()
+ {
+ if (isEmpty())
+ channel.read();
+ }
+
+ public boolean isEmpty()
+ {
+ return queue.isEmpty() && (buffer == null || !buffer.hasRemaining());
+ }
+
+ /**
+ * {@inheritDoc}
+ *
+ * Note: This should invoked on the consuming thread.
+ */
+ @Override
+ public void close()
+ {
+ if (isClosed)
+ return;
+
+ if (currentBuf != null)
+ {
+ currentBuf.release();
+ currentBuf = null;
+ buffer = null;
+ }
+
+ while (true)
+ {
+ try
+ {
+ ByteBuf buf = queue.poll(Long.MAX_VALUE, TimeUnit.NANOSECONDS);
+ if (buf == Unpooled.EMPTY_BUFFER)
+ break;
+ else
+ buf.release();
+ }
+ catch (InterruptedException e)
+ {
+ //
+ }
+ }
+
+ isClosed = true;
+ }
+
+ /**
+ * Mark this stream as closed, but do not release any of the resources.
+ *
+ * Note: this is best to be called from the producer thread.
+ */
+ public void requestClosure()
+ {
+ queue.add(Unpooled.EMPTY_BUFFER);
+ }
+
+ // TODO: let's remove this like we did for AsyncChannelOutputPlus
+ public ByteBufAllocator getAllocator()
+ {
+ return channel.alloc();
+ }
+}
diff --git a/src/java/org/apache/cassandra/net/AsyncStreamingOutputPlus.java b/src/java/org/apache/cassandra/net/AsyncStreamingOutputPlus.java
new file mode 100644
index 0000000..680a9d3
--- /dev/null
+++ b/src/java/org/apache/cassandra/net/AsyncStreamingOutputPlus.java
@@ -0,0 +1,267 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.net;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.ClosedChannelException;
+import java.nio.channels.FileChannel;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import io.netty.channel.Channel;
+import io.netty.channel.ChannelPromise;
+import io.netty.channel.FileRegion;
+import io.netty.channel.WriteBufferWaterMark;
+import io.netty.handler.ssl.SslHandler;
+import org.apache.cassandra.io.compress.BufferType;
+import org.apache.cassandra.io.util.DataOutputStreamPlus;
+import org.apache.cassandra.net.SharedDefaultFileRegion.SharedFileChannel;
+import org.apache.cassandra.streaming.StreamManager.StreamRateLimiter;
+import org.apache.cassandra.utils.memory.BufferPool;
+
+import static java.lang.Math.min;
+
+/**
+ * A {@link DataOutputStreamPlus} that writes ASYNCHRONOUSLY to a Netty Channel.
+ *
+ * The close() and flush() methods synchronously wait for pending writes, and will propagate any exceptions
+ * encountered in writing them to the wire.
+ *
+ * The correctness of this class depends on the ChannelPromise we create against a Channel always being completed,
+ * which appears to be a guarantee provided by Netty so long as the event loop is running.
+ */
+public class AsyncStreamingOutputPlus extends AsyncChannelOutputPlus
+{
+ private static final Logger logger = LoggerFactory.getLogger(AsyncStreamingOutputPlus.class);
+
+ final int defaultLowWaterMark;
+ final int defaultHighWaterMark;
+
+ public AsyncStreamingOutputPlus(Channel channel)
+ {
+ super(channel);
+ WriteBufferWaterMark waterMark = channel.config().getWriteBufferWaterMark();
+ this.defaultLowWaterMark = waterMark.low();
+ this.defaultHighWaterMark = waterMark.high();
+ allocateBuffer();
+ }
+
+ private void allocateBuffer()
+ {
+ // this buffer is only used for small quantities of data
+ buffer = BufferPool.getAtLeast(8 << 10, BufferType.OFF_HEAP);
+ }
+
+ @Override
+ protected void doFlush(int count) throws IOException
+ {
+ if (!channel.isOpen())
+ throw new ClosedChannelException();
+
+ // flush the current backing write buffer only if there's any pending data
+ ByteBuffer flush = buffer;
+ if (flush.position() == 0)
+ return;
+
+ flush.flip();
+ int byteCount = flush.limit();
+ ChannelPromise promise = beginFlush(byteCount, 0, Integer.MAX_VALUE);
+ channel.writeAndFlush(GlobalBufferPoolAllocator.wrap(flush), promise);
+ allocateBuffer();
+ }
+
+ public long position()
+ {
+ return flushed() + buffer.position();
+ }
+
+ public interface BufferSupplier
+ {
+ /**
+ * Request a buffer with at least the given capacity.
+ * This method may only be invoked once, and the lifetime of buffer it returns will be managed
+ * by the AsyncChannelOutputPlus it was created for.
+ */
+ ByteBuffer get(int capacity) throws IOException;
+ }
+
+ public interface Write
+ {
+ /**
+ * Write to a buffer, and flush its contents to the channel.
+ * <p>
+ * The lifetime of the buffer will be managed by the AsyncChannelOutputPlus you issue this Write to.
+ * If the method exits successfully, the contents of the buffer will be written to the channel, otherwise
+ * the buffer will be cleaned and the exception propagated to the caller.
+ */
+ void write(BufferSupplier supplier) throws IOException;
+ }
+
+ /**
+ * Provide a lambda that can request a buffer of suitable size, then fill the buffer and have
+ * that buffer written and flushed to the underlying channel, without having to handle buffer
+ * allocation, lifetime or cleanup, including in case of exceptions.
+ * <p>
+ * Any exception thrown by the Write will be propagated to the caller, after any buffer is cleaned up.
+ */
+ public int writeToChannel(Write write, StreamRateLimiter limiter) throws IOException
+ {
+ doFlush(0);
+ class Holder
+ {
+ ChannelPromise promise;
+ ByteBuffer buffer;
+ }
+ Holder holder = new Holder();
+
+ try
+ {
+ write.write(size -> {
+ if (holder.buffer != null)
+ throw new IllegalStateException("Can only allocate one ByteBuffer");
+ limiter.acquire(size);
+ holder.promise = beginFlush(size, defaultLowWaterMark, defaultHighWaterMark);
+ holder.buffer = BufferPool.get(size, BufferType.OFF_HEAP);
+ return holder.buffer;
+ });
+ }
+ catch (Throwable t)
+ {
+ // we don't currently support cancelling the flush, but at this point we are recoverable if we want
+ if (holder.buffer != null)
+ BufferPool.put(holder.buffer);
+ if (holder.promise != null)
+ holder.promise.tryFailure(t);
+ throw t;
+ }
+
+ ByteBuffer buffer = holder.buffer;
+ BufferPool.putUnusedPortion(buffer);
+
+ int length = buffer.limit();
+ channel.writeAndFlush(GlobalBufferPoolAllocator.wrap(buffer), holder.promise);
+ return length;
+ }
+
+ /**
+ * Writes all data in file channel to stream: <br>
+ * * For zero-copy-streaming, 1MiB at a time, with at most 2MiB in flight at once. <br>
+ * * For streaming with SSL, 64kb at a time, with at most 32+64kb (default low water mark + batch size) in flight. <br>
+ * <p>
+ * This method takes ownership of the provided {@link FileChannel}.
+ * <p>
+ * WARNING: this method blocks only for permission to write to the netty channel; it exits before
+ * the {@link FileRegion}(zero-copy) or {@link ByteBuffer}(ssl) is flushed to the network.
+ */
+ public long writeFileToChannel(FileChannel file, StreamRateLimiter limiter) throws IOException
+ {
+ if (channel.pipeline().get(SslHandler.class) != null)
+ // each batch is loaded into ByteBuffer, 64kb is more BufferPool friendly.
+ return writeFileToChannel(file, limiter, 1 << 16);
+ else
+ // write files in 1MiB chunks, since there may be blocking work performed to fetch it from disk,
+ // the data is never brought in process and is gated by the wire anyway
+ return writeFileToChannelZeroCopy(file, limiter, 1 << 20, 1 << 20, 2 << 20);
+ }
+
+ @VisibleForTesting
+ long writeFileToChannel(FileChannel fc, StreamRateLimiter limiter, int batchSize) throws IOException
+ {
+ final long length = fc.size();
+ long bytesTransferred = 0;
+
+ try
+ {
+ while (bytesTransferred < length)
+ {
+ int toWrite = (int) min(batchSize, length - bytesTransferred);
+ final long position = bytesTransferred;
+
+ writeToChannel(bufferSupplier -> {
+ ByteBuffer outBuffer = bufferSupplier.get(toWrite);
+ long read = fc.read(outBuffer, position);
+ if (read != toWrite)
+ throw new IOException(String.format("could not read required number of bytes from " +
+ "file to be streamed: read %d bytes, wanted %d bytes",
+ read, toWrite));
+ outBuffer.flip();
+ }, limiter);
+
+ if (logger.isTraceEnabled())
+ logger.trace("Writing {} bytes at position {} of {}", toWrite, bytesTransferred, length);
+ bytesTransferred += toWrite;
+ }
+ }
+ finally
+ {
+ // we don't need to wait until byte buffer is flushed by netty
+ fc.close();
+ }
+
+ return bytesTransferred;
+ }
+
+ @VisibleForTesting
+ long writeFileToChannelZeroCopy(FileChannel file, StreamRateLimiter limiter, int batchSize, int lowWaterMark, int highWaterMark) throws IOException
+ {
+ final long length = file.size();
+ long bytesTransferred = 0;
+
+ final SharedFileChannel sharedFile = SharedDefaultFileRegion.share(file);
+ try
+ {
+ while (bytesTransferred < length)
+ {
+ int toWrite = (int) min(batchSize, length - bytesTransferred);
+
+ limiter.acquire(toWrite);
+ ChannelPromise promise = beginFlush(toWrite, lowWaterMark, highWaterMark);
+
+ SharedDefaultFileRegion fileRegion = new SharedDefaultFileRegion(sharedFile, bytesTransferred, toWrite);
+ channel.writeAndFlush(fileRegion, promise);
+
+ if (logger.isTraceEnabled())
+ logger.trace("Writing {} bytes at position {} of {}", toWrite, bytesTransferred, length);
+ bytesTransferred += toWrite;
+ }
+
+ return bytesTransferred;
+ }
+ finally
+ {
+ sharedFile.release();
+ }
+ }
+
+ /**
+ * Discard any buffered data, and the buffers that contain it.
+ * May be invoked instead of {@link #close()} if we terminate exceptionally.
+ */
+ public void discard()
+ {
+ if (buffer != null)
+ {
+ BufferPool.put(buffer);
+ buffer = null;
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/net/BackPressureState.java b/src/java/org/apache/cassandra/net/BackPressureState.java
index 34fd0dd..de19bf3 100644
--- a/src/java/org/apache/cassandra/net/BackPressureState.java
+++ b/src/java/org/apache/cassandra/net/BackPressureState.java
@@ -17,7 +17,7 @@
*/
package org.apache.cassandra.net;
-import java.net.InetAddress;
+import org.apache.cassandra.locator.InetAddressAndPort;
/**
* Interface meant to track the back-pressure state per replica host.
@@ -27,7 +27,7 @@
/**
* Called when a message is sent to a replica.
*/
- void onMessageSent(MessageOut<?> message);
+ void onMessageSent(Message<?> message);
/**
* Called when a response is received from a replica.
@@ -47,5 +47,5 @@
/**
* Returns the host this state refers to.
*/
- InetAddress getHost();
+ InetAddressAndPort getHost();
}
diff --git a/src/java/org/apache/cassandra/net/BackPressureStrategy.java b/src/java/org/apache/cassandra/net/BackPressureStrategy.java
index b61a0a1..6b49495 100644
--- a/src/java/org/apache/cassandra/net/BackPressureStrategy.java
+++ b/src/java/org/apache/cassandra/net/BackPressureStrategy.java
@@ -17,15 +17,17 @@
*/
package org.apache.cassandra.net;
-import java.net.InetAddress;
import java.util.Set;
import java.util.concurrent.TimeUnit;
+import org.apache.cassandra.locator.InetAddressAndPort;
+
/**
* Back-pressure algorithm interface.
- * <br/>
+ * <p>
* For experts usage only. Implementors must provide a constructor accepting a single {@code Map<String, Object>} argument,
* representing any parameters eventually required by the specific implementation.
+ * </p>
*/
public interface BackPressureStrategy<S extends BackPressureState>
{
@@ -38,5 +40,5 @@
/**
* Creates a new {@link BackPressureState} initialized as needed by the specific implementation.
*/
- S newState(InetAddress host);
+ S newState(InetAddressAndPort host);
}
diff --git a/src/java/org/apache/cassandra/net/BufferPoolAllocator.java b/src/java/org/apache/cassandra/net/BufferPoolAllocator.java
new file mode 100644
index 0000000..8782c03
--- /dev/null
+++ b/src/java/org/apache/cassandra/net/BufferPoolAllocator.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.net;
+
+import java.nio.ByteBuffer;
+
+import io.netty.buffer.AbstractByteBufAllocator;
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.Unpooled;
+import io.netty.buffer.UnpooledUnsafeDirectByteBuf;
+import org.apache.cassandra.io.compress.BufferType;
+import org.apache.cassandra.utils.memory.BufferPool;
+
+/**
+ * A trivial wrapper around BufferPool for integrating with Netty, but retaining ownership of pooling behaviour
+ * that is integrated into Cassandra's other pooling.
+ */
+abstract class BufferPoolAllocator extends AbstractByteBufAllocator
+{
+ BufferPoolAllocator()
+ {
+ super(true);
+ }
+
+ @Override
+ public boolean isDirectBufferPooled()
+ {
+ return true;
+ }
+
+ /** shouldn't be invoked */
+ @Override
+ protected ByteBuf newHeapBuffer(int minCapacity, int maxCapacity)
+ {
+ return Unpooled.buffer(minCapacity, maxCapacity);
+ }
+
+ @Override
+ protected ByteBuf newDirectBuffer(int minCapacity, int maxCapacity)
+ {
+ ByteBuf result = new Wrapped(this, getAtLeast(minCapacity));
+ result.clear();
+ return result;
+ }
+
+ ByteBuffer get(int size)
+ {
+ return BufferPool.get(size, BufferType.OFF_HEAP);
+ }
+
+ ByteBuffer getAtLeast(int size)
+ {
+ return BufferPool.getAtLeast(size, BufferType.OFF_HEAP);
+ }
+
+ void put(ByteBuffer buffer)
+ {
+ BufferPool.put(buffer);
+ }
+
+ void putUnusedPortion(ByteBuffer buffer)
+ {
+ BufferPool.putUnusedPortion(buffer);
+ }
+
+ void release()
+ {
+ }
+
+ /**
+ * A simple extension to UnpooledUnsafeDirectByteBuf that returns buffers to BufferPool on deallocate,
+ * and permits extracting the buffer from it to take ownership and use directly.
+ */
+ public static class Wrapped extends UnpooledUnsafeDirectByteBuf
+ {
+ private ByteBuffer wrapped;
+
+ Wrapped(BufferPoolAllocator allocator, ByteBuffer wrap)
+ {
+ super(allocator, wrap, wrap.capacity());
+ wrapped = wrap;
+ }
+
+ @Override
+ public void deallocate()
+ {
+ if (wrapped != null)
+ BufferPool.put(wrapped);
+ }
+
+ public ByteBuffer adopt()
+ {
+ if (refCnt() > 1)
+ throw new IllegalStateException();
+ ByteBuffer adopt = wrapped;
+ adopt.position(readerIndex()).limit(writerIndex());
+ wrapped = null;
+ return adopt;
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/net/CallbackInfo.java b/src/java/org/apache/cassandra/net/CallbackInfo.java
deleted file mode 100644
index ea000ae..0000000
--- a/src/java/org/apache/cassandra/net/CallbackInfo.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.net;
-
-import java.net.InetAddress;
-
-import org.apache.cassandra.io.IVersionedSerializer;
-
-/**
- * Encapsulates the callback information.
- * The ability to set the message is useful in cases for when a hint needs
- * to be written due to a timeout in the response from a replica.
- */
-public class CallbackInfo
-{
- protected final InetAddress target;
- protected final IAsyncCallback callback;
- protected final IVersionedSerializer<?> serializer;
- private final boolean failureCallback;
-
- /**
- * Create CallbackInfo without sent message
- *
- * @param target target to send message
- * @param callback
- * @param serializer serializer to deserialize response message
- * @param failureCallback True when we have a callback to handle failures
- */
- public CallbackInfo(InetAddress target, IAsyncCallback callback, IVersionedSerializer<?> serializer, boolean failureCallback)
- {
- this.target = target;
- this.callback = callback;
- this.serializer = serializer;
- this.failureCallback = failureCallback;
- }
-
- public boolean shouldHint()
- {
- return false;
- }
-
- public boolean isFailureCallback()
- {
- return failureCallback;
- }
-
- public String toString()
- {
- return "CallbackInfo(" +
- "target=" + target +
- ", callback=" + callback +
- ", serializer=" + serializer +
- ", failureCallback=" + failureCallback +
- ')';
- }
-}
diff --git a/src/java/org/apache/cassandra/net/ChunkedInputPlus.java b/src/java/org/apache/cassandra/net/ChunkedInputPlus.java
new file mode 100644
index 0000000..3aad8d9
--- /dev/null
+++ b/src/java/org/apache/cassandra/net/ChunkedInputPlus.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.net;
+
+import java.io.EOFException;
+
+import com.google.common.collect.Iterators;
+import com.google.common.collect.PeekingIterator;
+
+import org.apache.cassandra.io.util.RebufferingInputStream;
+
+/**
+ * A specialised {@link org.apache.cassandra.io.util.DataInputPlus} implementation for deserializing large messages
+ * that are split over multiple {@link FrameDecoder.Frame}s.
+ *
+ * Ensures that every underlying {@link ShareableBytes} frame is released, and promptly so, as frames are consumed.
+ *
+ * {@link #close()} <em>MUST</em> be invoked in the end.
+ */
+class ChunkedInputPlus extends RebufferingInputStream
+{
+ private final PeekingIterator<ShareableBytes> iter;
+
+ private ChunkedInputPlus(PeekingIterator<ShareableBytes> iter)
+ {
+ super(iter.peek().get());
+ this.iter = iter;
+ }
+
+ /**
+ * Creates a {@link ChunkedInputPlus} from the provided {@link ShareableBytes} buffers.
+ *
+ * The provided iterable <em>must</em> contain at least one buffer.
+ */
+ static ChunkedInputPlus of(Iterable<ShareableBytes> buffers)
+ {
+ PeekingIterator<ShareableBytes> iter = Iterators.peekingIterator(buffers.iterator());
+ if (!iter.hasNext())
+ throw new IllegalArgumentException();
+ return new ChunkedInputPlus(iter);
+ }
+
+ @Override
+ protected void reBuffer() throws EOFException
+ {
+ buffer = null;
+ iter.peek().release();
+ iter.next();
+
+ if (!iter.hasNext())
+ throw new EOFException();
+
+ buffer = iter.peek().get();
+ }
+
+ @Override
+ public void close()
+ {
+ buffer = null;
+ iter.forEachRemaining(ShareableBytes::release);
+ }
+
+ /**
+ * Returns the number of unconsumed bytes. Will release any outstanding buffers and consume the underlying iterator.
+ *
+ * Should only be used for sanity checking, once the input is no longer needed, as it will implicitly close the input.
+ */
+ int remainder()
+ {
+ buffer = null;
+
+ int bytes = 0;
+ while (iter.hasNext())
+ {
+ ShareableBytes chunk = iter.peek();
+ bytes += chunk.remaining();
+ chunk.release();
+ iter.next();
+ }
+ return bytes;
+ }
+}
diff --git a/src/java/org/apache/cassandra/net/CompactEndpointSerializationHelper.java b/src/java/org/apache/cassandra/net/CompactEndpointSerializationHelper.java
deleted file mode 100644
index 83bbbf3..0000000
--- a/src/java/org/apache/cassandra/net/CompactEndpointSerializationHelper.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.net;
-
-import java.io.*;
-import java.net.Inet4Address;
-import java.net.Inet6Address;
-import java.net.InetAddress;
-
-public class CompactEndpointSerializationHelper
-{
- public static void serialize(InetAddress endpoint, DataOutput out) throws IOException
- {
- byte[] buf = endpoint.getAddress();
- out.writeByte(buf.length);
- out.write(buf);
- }
-
- public static InetAddress deserialize(DataInput in) throws IOException
- {
- byte[] bytes = new byte[in.readByte()];
- in.readFully(bytes, 0, bytes.length);
- return InetAddress.getByAddress(bytes);
- }
-
- public static int serializedSize(InetAddress from)
- {
- if (from instanceof Inet4Address)
- return 1 + 4;
- assert from instanceof Inet6Address;
- return 1 + 16;
- }
-}
diff --git a/src/java/org/apache/cassandra/net/ConnectionCategory.java b/src/java/org/apache/cassandra/net/ConnectionCategory.java
new file mode 100644
index 0000000..d739e9d
--- /dev/null
+++ b/src/java/org/apache/cassandra/net/ConnectionCategory.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.net;
+
+public enum ConnectionCategory
+{
+ MESSAGING, STREAMING;
+
+ public boolean isStreaming()
+ {
+ return this == STREAMING;
+ }
+
+ public boolean isMessaging()
+ {
+ return this == MESSAGING;
+ }
+}
diff --git a/src/java/org/apache/cassandra/net/ConnectionType.java b/src/java/org/apache/cassandra/net/ConnectionType.java
new file mode 100644
index 0000000..db83d06
--- /dev/null
+++ b/src/java/org/apache/cassandra/net/ConnectionType.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.net;
+
+import java.util.List;
+
+import com.google.common.collect.ImmutableList;
+
+public enum ConnectionType
+{
+ LEGACY_MESSAGES (0), // only used for inbound
+ URGENT_MESSAGES (1),
+ SMALL_MESSAGES (2),
+ LARGE_MESSAGES (3),
+ STREAMING (4);
+
+ public static final List<ConnectionType> MESSAGING_TYPES = ImmutableList.of(URGENT_MESSAGES, SMALL_MESSAGES, LARGE_MESSAGES);
+
+ public final int id;
+
+ ConnectionType(int id)
+ {
+ this.id = id;
+ }
+
+ public int twoBitID()
+ {
+ if (id < 0 || id > 0b11)
+ throw new AssertionError();
+ return id;
+ }
+
+ public boolean isStreaming()
+ {
+ return this == STREAMING;
+ }
+
+ public boolean isMessaging()
+ {
+ return !isStreaming();
+ }
+
+ public ConnectionCategory category()
+ {
+ return this == STREAMING ? ConnectionCategory.STREAMING : ConnectionCategory.MESSAGING;
+ }
+
+ private static final ConnectionType[] values = values();
+
+ public static ConnectionType fromId(int id)
+ {
+ return values[id];
+ }
+}
diff --git a/src/java/org/apache/cassandra/net/Crc.java b/src/java/org/apache/cassandra/net/Crc.java
new file mode 100644
index 0000000..dbd2601
--- /dev/null
+++ b/src/java/org/apache/cassandra/net/Crc.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.net;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.zip.CRC32;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.util.concurrent.FastThreadLocal;
+
+class Crc
+{
+ private static final FastThreadLocal<CRC32> crc32 = new FastThreadLocal<CRC32>()
+ {
+ @Override
+ protected CRC32 initialValue()
+ {
+ return new CRC32();
+ }
+ };
+
+ private static final byte[] initialBytes = new byte[] { (byte) 0xFA, (byte) 0x2D, (byte) 0x55, (byte) 0xCA };
+
+ static final class InvalidCrc extends IOException
+ {
+ InvalidCrc(int read, int computed)
+ {
+ super(String.format("Read %d, Computed %d", read, computed));
+ }
+ }
+
+ static CRC32 crc32()
+ {
+ CRC32 crc = crc32.get();
+ crc.reset();
+ crc.update(initialBytes);
+ return crc;
+ }
+
+ static int computeCrc32(ByteBuf buffer, int startReaderIndex, int endReaderIndex)
+ {
+ CRC32 crc = crc32();
+ crc.update(buffer.internalNioBuffer(startReaderIndex, endReaderIndex - startReaderIndex));
+ return (int) crc.getValue();
+ }
+
+ static int computeCrc32(ByteBuffer buffer, int start, int end)
+ {
+ CRC32 crc = crc32();
+ updateCrc32(crc, buffer, start, end);
+ return (int) crc.getValue();
+ }
+
+ static void updateCrc32(CRC32 crc, ByteBuffer buffer, int start, int end)
+ {
+ int savePosition = buffer.position();
+ int saveLimit = buffer.limit();
+ buffer.limit(end);
+ buffer.position(start);
+ crc.update(buffer);
+ buffer.limit(saveLimit);
+ buffer.position(savePosition);
+ }
+
+ private static final int CRC24_INIT = 0x875060;
+ /**
+ * Polynomial chosen from https://users.ece.cmu.edu/~koopman/crc/index.html, by Philip Koopman
+ *
+ * This webpage claims a copyright to Philip Koopman, which he licenses under the
+ * Creative Commons Attribution 4.0 International License (https://creativecommons.org/licenses/by/4.0)
+ *
+ * It is unclear if this copyright can extend to a 'fact' such as this specific number, particularly
+ * as we do not use Koopman's notation to represent the polynomial, but we anyway attribute his work and
+ * link the terms of his license since they are not incompatible with our usage and we greatly appreciate his work.
+ *
+ * This polynomial provides hamming distance of 8 for messages up to length 105 bits;
+ * we only support 8-64 bits at present, with an expected range of 40-48.
+ */
+ private static final int CRC24_POLY = 0x1974F0B;
+
+ /**
+ * NOTE: the order of bytes must reach the wire in the same order the CRC is computed, with the CRC
+ * immediately following in a trailer. Since we read in least significant byte order, if you
+ * write to a buffer using putInt or putLong, the byte order will be reversed and
+ * you will lose the guarantee of protection from burst corruptions of 24 bits in length.
+ *
+ * Make sure either to write byte-by-byte to the wire, or to use Integer/Long.reverseBytes if you
+ * write to a BIG_ENDIAN buffer.
+ *
+ * See http://users.ece.cmu.edu/~koopman/pubs/ray06_crcalgorithms.pdf
+ *
+ * Complain to the ethernet spec writers, for having inverse bit to byte significance order.
+ *
+ * Note we use the most naive algorithm here. We support at most 8 bytes, and typically supply
+ * 5 or fewer, so any efficiency of a table approach is swallowed by the time to hit L3, even
+ * for a tiny (4bit) table.
+ *
+ * @param bytes an up to 8-byte register containing bytes to compute the CRC over
+ * the bytes AND bits will be read least-significant to most significant.
+ * @param len the number of bytes, greater than 0 and fewer than 9, to be read from bytes
+ * @return the least-significant bit AND byte order crc24 using the CRC24_POLY polynomial
+ */
+ static int crc24(long bytes, int len)
+ {
+ int crc = CRC24_INIT;
+ while (len-- > 0)
+ {
+ crc ^= (bytes & 0xff) << 16;
+ bytes >>= 8;
+
+ for (int i = 0; i < 8; i++)
+ {
+ crc <<= 1;
+ if ((crc & 0x1000000) != 0)
+ crc ^= CRC24_POLY;
+ }
+ }
+ return crc;
+ }
+}
diff --git a/src/java/org/apache/cassandra/net/EndpointMessagingVersions.java b/src/java/org/apache/cassandra/net/EndpointMessagingVersions.java
new file mode 100644
index 0000000..e8cf8f6
--- /dev/null
+++ b/src/java/org/apache/cassandra/net/EndpointMessagingVersions.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.net;
+
+import java.net.UnknownHostException;
+import java.util.concurrent.ConcurrentMap;
+
+import org.cliffc.high_scale_lib.NonBlockingHashMap;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.locator.InetAddressAndPort;
+
+/**
+ * Map of hosts to their known current messaging versions.
+ */
+public class EndpointMessagingVersions
+{
+ private static final Logger logger = LoggerFactory.getLogger(EndpointMessagingVersions.class);
+
+ // protocol versions of the other nodes in the cluster
+ private final ConcurrentMap<InetAddressAndPort, Integer> versions = new NonBlockingHashMap<>();
+
+ /**
+ * @return the last version associated with address, or @param version if this is the first such version
+ */
+ public int set(InetAddressAndPort endpoint, int version)
+ {
+ logger.trace("Setting version {} for {}", version, endpoint);
+
+ Integer v = versions.put(endpoint, version);
+ return v == null ? version : v;
+ }
+
+ public void reset(InetAddressAndPort endpoint)
+ {
+ logger.trace("Resetting version for {}", endpoint);
+ versions.remove(endpoint);
+ }
+
+ /**
+ * Returns the messaging-version as announced by the given node but capped
+ * to the min of the version as announced by the node and {@link MessagingService#current_version}.
+ */
+ public int get(InetAddressAndPort endpoint)
+ {
+ Integer v = versions.get(endpoint);
+ if (v == null)
+ {
+ // we don't know the version. assume current. we'll know soon enough if that was incorrect.
+ logger.trace("Assuming current protocol version for {}", endpoint);
+ return MessagingService.current_version;
+ }
+ else
+ return Math.min(v, MessagingService.current_version);
+ }
+
+ public int get(String endpoint) throws UnknownHostException
+ {
+ return get(InetAddressAndPort.getByName(endpoint));
+ }
+
+ /**
+ * Returns the messaging-version exactly as announced by the given endpoint.
+ */
+ public int getRaw(InetAddressAndPort endpoint)
+ {
+ Integer v = versions.get(endpoint);
+ if (v == null)
+ throw new IllegalStateException("getRawVersion() was called without checking knowsVersion() result first");
+ return v;
+ }
+
+ public boolean knows(InetAddressAndPort endpoint)
+ {
+ return versions.containsKey(endpoint);
+ }
+}
diff --git a/src/java/org/apache/cassandra/net/ForwardingInfo.java b/src/java/org/apache/cassandra/net/ForwardingInfo.java
new file mode 100644
index 0000000..737da48
--- /dev/null
+++ b/src/java/org/apache/cassandra/net/ForwardingInfo.java
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.net;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.function.BiConsumer;
+
+import com.google.common.base.Preconditions;
+import com.google.common.primitives.Ints;
+
+import org.apache.cassandra.db.TypeSizes;
+import org.apache.cassandra.io.IVersionedSerializer;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.locator.InetAddressAndPort;
+
+import static org.apache.cassandra.locator.InetAddressAndPort.Serializer.inetAddressAndPortSerializer;
+import static org.apache.cassandra.net.MessagingService.VERSION_40;
+import static org.apache.cassandra.utils.vint.VIntCoding.computeUnsignedVIntSize;
+
+/**
+ * A container used to store a node -> message_id map for inter-DC write forwarding.
+ * We pick one node in each external DC to forward the message to its local peers.
+ *
+ * TODO: in the next protocol version only serialize peers, message id will become redundant once 3.0 is out of the picture
+ */
+public final class ForwardingInfo implements Serializable
+{
+ final List<InetAddressAndPort> targets;
+ final long[] messageIds;
+
+ public ForwardingInfo(List<InetAddressAndPort> targets, long[] messageIds)
+ {
+ Preconditions.checkArgument(targets.size() == messageIds.length);
+ this.targets = targets;
+ this.messageIds = messageIds;
+ }
+
+ /**
+ * @return {@code true} if all host are to use the same message id, {@code false} otherwise. Starting with 4.0 and
+ * above, we should be reusing the same id, always, but it won't always be true until 3.0/3.11 are phased out.
+ */
+ public boolean useSameMessageID()
+ {
+ if (messageIds.length < 2)
+ return true;
+
+ long id = messageIds[0];
+ for (int i = 1; i < messageIds.length; i++)
+ if (id != messageIds[i])
+ return false;
+
+ return true;
+ }
+
+ /**
+ * Apply the provided consumer to all (host, message_id) pairs.
+ */
+ public void forEach(BiConsumer<Long, InetAddressAndPort> biConsumer)
+ {
+ for (int i = 0; i < messageIds.length; i++)
+ biConsumer.accept(messageIds[i], targets.get(i));
+ }
+
+ static final IVersionedSerializer<ForwardingInfo> serializer = new IVersionedSerializer<ForwardingInfo>()
+ {
+ public void serialize(ForwardingInfo forwardTo, DataOutputPlus out, int version) throws IOException
+ {
+ long[] ids = forwardTo.messageIds;
+ List<InetAddressAndPort> targets = forwardTo.targets;
+
+ int count = ids.length;
+ if (version >= VERSION_40)
+ out.writeUnsignedVInt(count);
+ else
+ out.writeInt(count);
+
+ for (int i = 0; i < count; i++)
+ {
+ inetAddressAndPortSerializer.serialize(targets.get(i), out, version);
+ if (version >= VERSION_40)
+ out.writeUnsignedVInt(ids[i]);
+ else
+ out.writeInt(Ints.checkedCast(ids[i]));
+ }
+ }
+
+ public long serializedSize(ForwardingInfo forwardTo, int version)
+ {
+ long[] ids = forwardTo.messageIds;
+ List<InetAddressAndPort> targets = forwardTo.targets;
+
+ int count = ids.length;
+ long size = version >= VERSION_40 ? computeUnsignedVIntSize(count) : TypeSizes.sizeof(count);
+
+ for (int i = 0; i < count; i++)
+ {
+ size += inetAddressAndPortSerializer.serializedSize(targets.get(i), version);
+ size += version >= VERSION_40 ? computeUnsignedVIntSize(ids[i]) : 4;
+ }
+
+ return size;
+ }
+
+ public ForwardingInfo deserialize(DataInputPlus in, int version) throws IOException
+ {
+ int count = version >= VERSION_40 ? Ints.checkedCast(in.readUnsignedVInt()) : in.readInt();
+
+ long[] ids = new long[count];
+ List<InetAddressAndPort> targets = new ArrayList<>(count);
+
+ for (int i = 0; i < count; i++)
+ {
+ targets.add(inetAddressAndPortSerializer.deserialize(in, version));
+ ids[i] = version >= VERSION_40 ? Ints.checkedCast(in.readUnsignedVInt()) : in.readInt();
+ }
+
+ return new ForwardingInfo(targets, ids);
+ }
+ };
+}
diff --git a/src/java/org/apache/cassandra/net/FrameDecoder.java b/src/java/org/apache/cassandra/net/FrameDecoder.java
new file mode 100644
index 0000000..ed96add
--- /dev/null
+++ b/src/java/org/apache/cassandra/net/FrameDecoder.java
@@ -0,0 +1,400 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.net;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayDeque;
+import java.util.Collection;
+import java.util.Deque;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.channel.ChannelInboundHandlerAdapter;
+import io.netty.channel.ChannelPipeline;
+
+import static org.apache.cassandra.utils.ByteBufferUtil.copyBytes;
+
+/**
+ * A Netty inbound handler that decodes incoming frames and passes them forward to
+ * {@link InboundMessageHandler} for processing.
+ *
+ * Handles work stashing, and together with {@link InboundMessageHandler} - flow control.
+ *
+ * Unlike most Netty inbound handlers, doesn't use the pipeline to talk to its
+ * upstream handler. Instead, a {@link FrameProcessor} must be registered with
+ * the frame decoder, to be invoked on new frames. See {@link #deliver(FrameProcessor)}.
+ *
+ * See {@link #activate(FrameProcessor)}, {@link #reactivate()}, and {@link FrameProcessor}
+ * for flow control implementation.
+ *
+ * Five frame decoders currently exist, one used for each connection depending on flags and messaging version:
+ * 1. {@link FrameDecoderCrc}:
+ no compression; payload is protected by CRC32
+ * 2. {@link FrameDecoderLZ4}:
+ LZ4 compression with custom frame format; payload is protected by CRC32
+ * 3. {@link FrameDecoderUnprotected}:
+ no compression; no integrity protection
+ * 4. {@link FrameDecoderLegacy}:
+ no compression; no integrity protection; turns unframed streams of legacy messages (< 4.0) into frames
+ * 5. {@link FrameDecoderLegacyLZ4}
+ * LZ4 compression using standard LZ4 frame format; groups legacy messages (< 4.0) into frames
+ */
+abstract class FrameDecoder extends ChannelInboundHandlerAdapter
+{
+ private static final FrameProcessor NO_PROCESSOR =
+ frame -> { throw new IllegalStateException("Frame processor invoked on an unregistered FrameDecoder"); };
+
+ private static final FrameProcessor CLOSED_PROCESSOR =
+ frame -> { throw new IllegalStateException("Frame processor invoked on a closed FrameDecoder"); };
+
+ interface FrameProcessor
+ {
+ /**
+ * Frame processor that the frames should be handed off to.
+ *
+ * @return true if more frames can be taken by the processor, false if the decoder should pause until
+ * it's explicitly resumed.
+ */
+ boolean process(Frame frame) throws IOException;
+ }
+
+ abstract static class Frame
+ {
+ final boolean isSelfContained;
+ final int frameSize;
+
+ Frame(boolean isSelfContained, int frameSize)
+ {
+ this.isSelfContained = isSelfContained;
+ this.frameSize = frameSize;
+ }
+
+ abstract void release();
+ abstract boolean isConsumed();
+ }
+
+ /**
+ * The payload bytes of a complete frame, i.e. a frame stripped of its headers and trailers,
+ * with any verification supported by the protocol confirmed.
+ *
+ * If {@code isSelfContained} the payload contains one or more {@link Message}, all of which
+ * may be parsed entirely from the bytes provided. Otherwise, only a part of exactly one
+ * {@link Message} is contained in the payload; it can be relied upon that this partial {@link Message}
+ * will only be delivered in its own unique {@link Frame}.
+ */
+ final static class IntactFrame extends Frame
+ {
+ final ShareableBytes contents;
+
+ IntactFrame(boolean isSelfContained, ShareableBytes contents)
+ {
+ super(isSelfContained, contents.remaining());
+ this.contents = contents;
+ }
+
+ void release()
+ {
+ contents.release();
+ }
+
+ boolean isConsumed()
+ {
+ return !contents.hasRemaining();
+ }
+
+ void consume()
+ {
+ contents.consume();
+ }
+ }
+
+ /**
+ * A corrupted frame was encountered; this represents the knowledge we have about this frame,
+ * and whether or not the stream is recoverable.
+ *
+ * Generally we consider a frame with corrupted header as unrecoverable, and frames with intact header,
+ * but corrupted payload - as recoverable, since we know and can skip payload size.
+ *
+ * {@link InboundMessageHandler} further has its own idea of which frames are and aren't recoverable.
+ * A recoverable {@link CorruptFrame} can be considered unrecoverable by {@link InboundMessageHandler}
+ * if it's the first frame of a large message (isn't self contained).
+ */
+ final static class CorruptFrame extends Frame
+ {
+ final int readCRC, computedCRC;
+
+ CorruptFrame(boolean isSelfContained, int frameSize, int readCRC, int computedCRC)
+ {
+ super(isSelfContained, frameSize);
+ this.readCRC = readCRC;
+ this.computedCRC = computedCRC;
+ }
+
+ static CorruptFrame recoverable(boolean isSelfContained, int frameSize, int readCRC, int computedCRC)
+ {
+ return new CorruptFrame(isSelfContained, frameSize, readCRC, computedCRC);
+ }
+
+ static CorruptFrame unrecoverable(int readCRC, int computedCRC)
+ {
+ return new CorruptFrame(false, Integer.MIN_VALUE, readCRC, computedCRC);
+ }
+
+ boolean isRecoverable()
+ {
+ return frameSize != Integer.MIN_VALUE;
+ }
+
+ void release() { }
+
+ boolean isConsumed()
+ {
+ return true;
+ }
+ }
+
+ protected final BufferPoolAllocator allocator;
+
+ @VisibleForTesting
+ final Deque<Frame> frames = new ArrayDeque<>(4);
+ ByteBuffer stash;
+
+ private boolean isActive;
+ private boolean isClosed;
+ private ChannelHandlerContext ctx;
+ private FrameProcessor processor = NO_PROCESSOR;
+
+ FrameDecoder(BufferPoolAllocator allocator)
+ {
+ this.allocator = allocator;
+ }
+
+ abstract void decode(Collection<Frame> into, ShareableBytes bytes);
+ abstract void addLastTo(ChannelPipeline pipeline);
+
+ /**
+ * For use by InboundMessageHandler (or other upstream handlers) that want to start receiving frames.
+ */
+ void activate(FrameProcessor processor)
+ {
+ if (this.processor != NO_PROCESSOR)
+ throw new IllegalStateException("Attempted to activate an already active FrameDecoder");
+
+ this.processor = processor;
+
+ isActive = true;
+ ctx.read();
+ }
+
+ /**
+ * For use by InboundMessageHandler (or other upstream handlers) that want to resume
+ * receiving frames after previously indicating that processing should be paused.
+ */
+ void reactivate() throws IOException
+ {
+ if (isActive)
+ throw new IllegalStateException("Tried to reactivate an already active FrameDecoder");
+
+ if (deliver(processor))
+ {
+ isActive = true;
+ onExhausted();
+ }
+ }
+
+ /**
+ * For use by InboundMessageHandler (or other upstream handlers) that want to resume
+ * receiving frames after previously indicating that processing should be paused.
+ *
+ * Does not reactivate processing or reading from the wire, but permits processing as many frames (or parts thereof)
+ * that are already waiting as the processor requires.
+ */
+ void processBacklog(FrameProcessor processor) throws IOException
+ {
+ deliver(processor);
+ }
+
+ /**
+ * For use by InboundMessageHandler (or other upstream handlers) that want to permanently
+ * stop receiving frames, e.g. because of an exception caught.
+ */
+ void discard()
+ {
+ isActive = false;
+ processor = CLOSED_PROCESSOR;
+ if (stash != null)
+ {
+ ByteBuffer bytes = stash;
+ stash = null;
+ allocator.put(bytes);
+ }
+ while (!frames.isEmpty())
+ frames.poll().release();
+ }
+
+ /**
+ * Called by Netty pipeline when a new message arrives; we anticipate in normal operation
+ * this will receive messages of type {@link BufferPoolAllocator.Wrapped} or
+ * {@link BufferPoolAllocator.Wrapped}.
+ *
+ * These buffers are unwrapped and passed to {@link #decode(Collection, ShareableBytes)},
+ * which collects decoded frames into {@link #frames}, which we send upstream in {@link #deliver}
+ */
+ @Override
+ public void channelRead(ChannelHandlerContext ctx, Object msg) throws IOException
+ {
+ if (msg instanceof BufferPoolAllocator.Wrapped)
+ {
+ ByteBuffer buf = ((BufferPoolAllocator.Wrapped) msg).adopt();
+ // netty will probably have mis-predicted the space needed
+ allocator.putUnusedPortion(buf);
+ channelRead(ShareableBytes.wrap(buf));
+ }
+ else if (msg instanceof ShareableBytes) // legacy LZ4 decoder
+ {
+ channelRead((ShareableBytes) msg);
+ }
+ else
+ {
+ throw new IllegalArgumentException();
+ }
+ }
+
+ void channelRead(ShareableBytes bytes) throws IOException
+ {
+ decode(frames, bytes);
+
+ if (isActive) isActive = deliver(processor);
+ }
+
+ @Override
+ public void channelReadComplete(ChannelHandlerContext ctx)
+ {
+ if (isActive)
+ onExhausted();
+ }
+
+ /**
+ * Only to be invoked when frames.isEmpty().
+ *
+ * If we have been closed, we will now propagate up the channelInactive notification,
+ * and otherwise we will ask the channel for more data.
+ */
+ private void onExhausted()
+ {
+ if (isClosed)
+ close();
+ else
+ ctx.read();
+ }
+
+ /**
+ * Deliver any waiting frames, including those that were incompletely read last time, to the provided processor
+ * until the processor returns {@code false}, or we finish the backlog.
+ *
+ * Propagate the final return value of the processor.
+ */
+ private boolean deliver(FrameProcessor processor) throws IOException
+ {
+ boolean deliver = true;
+ while (deliver && !frames.isEmpty())
+ {
+ Frame frame = frames.peek();
+ deliver = processor.process(frame);
+
+ assert !deliver || frame.isConsumed();
+ if (deliver || frame.isConsumed())
+ {
+ frames.poll();
+ frame.release();
+ }
+ }
+ return deliver;
+ }
+
+ void stash(ShareableBytes in, int stashLength, int begin, int length)
+ {
+ ByteBuffer out = allocator.getAtLeast(stashLength);
+ copyBytes(in.get(), begin, out, 0, length);
+ out.position(length);
+ stash = out;
+ }
+
+ @Override
+ public void handlerAdded(ChannelHandlerContext ctx)
+ {
+ this.ctx = ctx;
+ ctx.channel().config().setAutoRead(false);
+ }
+
+ @Override
+ public void channelInactive(ChannelHandlerContext ctx)
+ {
+ isClosed = true;
+ if (frames.isEmpty())
+ close();
+ }
+
+ private void close()
+ {
+ discard();
+ ctx.fireChannelInactive();
+ allocator.release();
+ }
+
+ /**
+ * Utility: fill {@code out} from {@code in} up to {@code toOutPosition},
+ * updating the position of both buffers with the result
+ * @return true if there were sufficient bytes to fill to {@code toOutPosition}
+ */
+ static boolean copyToSize(ByteBuffer in, ByteBuffer out, int toOutPosition)
+ {
+ int bytesToSize = toOutPosition - out.position();
+ if (bytesToSize <= 0)
+ return true;
+
+ if (bytesToSize > in.remaining())
+ {
+ out.put(in);
+ return false;
+ }
+
+ copyBytes(in, in.position(), out, out.position(), bytesToSize);
+ in.position(in.position() + bytesToSize);
+ out.position(toOutPosition);
+ return true;
+ }
+
+ /**
+ * @return {@code in} if has sufficient capacity, otherwise
+ * a replacement from {@code BufferPool} that {@code in} is copied into
+ */
+ ByteBuffer ensureCapacity(ByteBuffer in, int capacity)
+ {
+ if (in.capacity() >= capacity)
+ return in;
+
+ ByteBuffer out = allocator.getAtLeast(capacity);
+ in.flip();
+ out.put(in);
+ allocator.put(in);
+ return out;
+ }
+}
diff --git a/src/java/org/apache/cassandra/net/FrameDecoderCrc.java b/src/java/org/apache/cassandra/net/FrameDecoderCrc.java
new file mode 100644
index 0000000..7cd52ac
--- /dev/null
+++ b/src/java/org/apache/cassandra/net/FrameDecoderCrc.java
@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.net;
+
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.util.Collection;
+import java.util.zip.CRC32;
+
+import io.netty.channel.ChannelPipeline;
+
+import static org.apache.cassandra.net.Crc.*;
+import static org.apache.cassandra.net.Crc.updateCrc32;
+
+/**
+ * Framing format that protects integrity of data in movement with CRCs (of both header and payload).
+ *
+ * Every on-wire frame contains:
+ * 1. Payload length (17 bits)
+ * 2. {@code isSelfContained} flag (1 bit)
+ * 3. Header padding (6 bits)
+ * 4. CRC24 of the header (24 bits)
+ * 5. Payload (up to 2 ^ 17 - 1 bits)
+ * 6. Payload CRC32 (32 bits)
+ *
+ * 0 1 2 3
+ * 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ * | Payload Length |C| |
+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ * CRC24 of Header | |
+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +
+ * | |
+ * + +
+ * | Payload |
+ * + +
+ * | |
+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ * | CRC32 of Payload |
+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ */
+final class FrameDecoderCrc extends FrameDecoderWith8bHeader
+{
+ private FrameDecoderCrc(BufferPoolAllocator allocator)
+ {
+ super(allocator);
+ }
+
+ public static FrameDecoderCrc create(BufferPoolAllocator allocator)
+ {
+ return new FrameDecoderCrc(allocator);
+ }
+
+ static final int HEADER_LENGTH = 6;
+ private static final int TRAILER_LENGTH = 4;
+ private static final int HEADER_AND_TRAILER_LENGTH = 10;
+
+ static boolean isSelfContained(long header6b)
+ {
+ return 0 != (header6b & (1L << 17));
+ }
+
+ static int payloadLength(long header6b)
+ {
+ return ((int) header6b) & 0x1FFFF;
+ }
+
+ private static int headerCrc(long header6b)
+ {
+ return ((int) (header6b >>> 24)) & 0xFFFFFF;
+ }
+
+ static long readHeader6b(ByteBuffer frame, int begin)
+ {
+ long header6b;
+ if (frame.limit() - begin >= 8)
+ {
+ header6b = frame.getLong(begin);
+ if (frame.order() == ByteOrder.BIG_ENDIAN)
+ header6b = Long.reverseBytes(header6b);
+ header6b &= 0xffffffffffffL;
+ }
+ else
+ {
+ header6b = 0;
+ for (int i = 0 ; i < HEADER_LENGTH ; ++i)
+ header6b |= (0xffL & frame.get(begin + i)) << (8 * i);
+ }
+ return header6b;
+ }
+
+ static CorruptFrame verifyHeader6b(long header6b)
+ {
+ int computeLengthCrc = crc24(header6b, 3);
+ int readLengthCrc = headerCrc(header6b);
+
+ return readLengthCrc == computeLengthCrc ? null : CorruptFrame.unrecoverable(readLengthCrc, computeLengthCrc);
+ }
+
+ final long readHeader(ByteBuffer frame, int begin)
+ {
+ return readHeader6b(frame, begin);
+ }
+
+ final CorruptFrame verifyHeader(long header6b)
+ {
+ return verifyHeader6b(header6b);
+ }
+
+ final int frameLength(long header6b)
+ {
+ return payloadLength(header6b) + HEADER_AND_TRAILER_LENGTH;
+ }
+
+ final Frame unpackFrame(ShareableBytes bytes, int begin, int end, long header6b)
+ {
+ ByteBuffer in = bytes.get();
+ boolean isSelfContained = isSelfContained(header6b);
+
+ CRC32 crc = crc32();
+ int readFullCrc = in.getInt(end - TRAILER_LENGTH);
+ if (in.order() == ByteOrder.BIG_ENDIAN)
+ readFullCrc = Integer.reverseBytes(readFullCrc);
+
+ updateCrc32(crc, in, begin + HEADER_LENGTH, end - TRAILER_LENGTH);
+ int computeFullCrc = (int) crc.getValue();
+
+ if (readFullCrc != computeFullCrc)
+ return CorruptFrame.recoverable(isSelfContained, (end - begin) - HEADER_AND_TRAILER_LENGTH, readFullCrc, computeFullCrc);
+
+ return new IntactFrame(isSelfContained, bytes.slice(begin + HEADER_LENGTH, end - TRAILER_LENGTH));
+ }
+
+ void decode(Collection<Frame> into, ShareableBytes bytes)
+ {
+ decode(into, bytes, HEADER_LENGTH);
+ }
+
+ void addLastTo(ChannelPipeline pipeline)
+ {
+ pipeline.addLast("frameDecoderCrc", this);
+ }
+}
diff --git a/src/java/org/apache/cassandra/net/FrameDecoderLZ4.java b/src/java/org/apache/cassandra/net/FrameDecoderLZ4.java
new file mode 100644
index 0000000..2b32d18
--- /dev/null
+++ b/src/java/org/apache/cassandra/net/FrameDecoderLZ4.java
@@ -0,0 +1,166 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.net;
+
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.util.Collection;
+import java.util.zip.CRC32;
+
+import io.netty.channel.ChannelPipeline;
+import net.jpountz.lz4.LZ4Factory;
+import net.jpountz.lz4.LZ4SafeDecompressor;
+
+import static org.apache.cassandra.net.Crc.*;
+
+/**
+ * Framing format that compresses payloads with LZ4, and protects integrity of data in movement with CRCs
+ * (of both header and payload).
+ *
+ * Every on-wire frame contains:
+ * 1. Compressed length (17 bits)
+ * 2. Uncompressed length (17 bits)
+ * 3. {@code isSelfContained} flag (1 bit)
+ * 4. Header padding (5 bits)
+ * 5. CRC24 of Header contents (24 bits)
+ * 6. Compressed Payload (up to 2 ^ 17 - 1 bits)
+ * 7. CRC32 of Compressed Payload (32 bits)
+ *
+ * 0 1 2 3
+ * 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ * | Compressed Length | Uncompressed Length
+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ * |C| | CRC24 of Header |
+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ * | |
+ * + +
+ * | Compressed Payload |
+ * + +
+ * | |
+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ * | CRC32 of Compressed Payload |
+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ */
+final class FrameDecoderLZ4 extends FrameDecoderWith8bHeader
+{
+ public static FrameDecoderLZ4 fast(BufferPoolAllocator allocator)
+ {
+ return new FrameDecoderLZ4(allocator, LZ4Factory.fastestInstance().safeDecompressor());
+ }
+
+ private static final int HEADER_LENGTH = 8;
+ private static final int TRAILER_LENGTH = 4;
+ private static final int HEADER_AND_TRAILER_LENGTH = 12;
+
+ private static int compressedLength(long header8b)
+ {
+ return ((int) header8b) & 0x1FFFF;
+ }
+ private static int uncompressedLength(long header8b)
+ {
+ return ((int) (header8b >>> 17)) & 0x1FFFF;
+ }
+ private static boolean isSelfContained(long header8b)
+ {
+ return 0 != (header8b & (1L << 34));
+ }
+ private static int headerCrc(long header8b)
+ {
+ return ((int) (header8b >>> 40)) & 0xFFFFFF;
+ }
+
+ private final LZ4SafeDecompressor decompressor;
+
+ private FrameDecoderLZ4(BufferPoolAllocator allocator, LZ4SafeDecompressor decompressor)
+ {
+ super(allocator);
+ this.decompressor = decompressor;
+ }
+
+ final long readHeader(ByteBuffer frame, int begin)
+ {
+ long header8b = frame.getLong(begin);
+ if (frame.order() == ByteOrder.BIG_ENDIAN)
+ header8b = Long.reverseBytes(header8b);
+ return header8b;
+ }
+
+ final CorruptFrame verifyHeader(long header8b)
+ {
+ int computeLengthCrc = crc24(header8b, 5);
+ int readLengthCrc = headerCrc(header8b);
+
+ return readLengthCrc == computeLengthCrc ? null : CorruptFrame.unrecoverable(readLengthCrc, computeLengthCrc);
+ }
+
+ final int frameLength(long header8b)
+ {
+ return compressedLength(header8b) + HEADER_AND_TRAILER_LENGTH;
+ }
+
+ final Frame unpackFrame(ShareableBytes bytes, int begin, int end, long header8b)
+ {
+ ByteBuffer input = bytes.get();
+
+ boolean isSelfContained = isSelfContained(header8b);
+ int uncompressedLength = uncompressedLength(header8b);
+
+ CRC32 crc = crc32();
+ int readFullCrc = input.getInt(end - TRAILER_LENGTH);
+ if (input.order() == ByteOrder.BIG_ENDIAN)
+ readFullCrc = Integer.reverseBytes(readFullCrc);
+
+ updateCrc32(crc, input, begin + HEADER_LENGTH, end - TRAILER_LENGTH);
+ int computeFullCrc = (int) crc.getValue();
+
+ if (readFullCrc != computeFullCrc)
+ return CorruptFrame.recoverable(isSelfContained, uncompressedLength, readFullCrc, computeFullCrc);
+
+ if (uncompressedLength == 0)
+ {
+ return new IntactFrame(isSelfContained, bytes.slice(begin + HEADER_LENGTH, end - TRAILER_LENGTH));
+ }
+ else
+ {
+ ByteBuffer out = allocator.get(uncompressedLength);
+ try
+ {
+ int sourceLength = end - (begin + HEADER_LENGTH + TRAILER_LENGTH);
+ decompressor.decompress(input, begin + HEADER_LENGTH, sourceLength, out, 0, uncompressedLength);
+ return new IntactFrame(isSelfContained, ShareableBytes.wrap(out));
+ }
+ catch (Throwable t)
+ {
+ allocator.put(out);
+ throw t;
+ }
+ }
+ }
+
+ void decode(Collection<Frame> into, ShareableBytes bytes)
+ {
+ // TODO: confirm in assembly output that we inline the relevant nested method calls
+ decode(into, bytes, HEADER_LENGTH);
+ }
+
+ void addLastTo(ChannelPipeline pipeline)
+ {
+ pipeline.addLast("frameDecoderLZ4", this);
+ }
+}
diff --git a/src/java/org/apache/cassandra/net/FrameDecoderLegacy.java b/src/java/org/apache/cassandra/net/FrameDecoderLegacy.java
new file mode 100644
index 0000000..a3d7bc5
--- /dev/null
+++ b/src/java/org/apache/cassandra/net/FrameDecoderLegacy.java
@@ -0,0 +1,184 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.net;
+
+import java.nio.ByteBuffer;
+import java.util.Collection;
+
+import io.netty.channel.ChannelPipeline;
+
+import static java.lang.Math.max;
+import static org.apache.cassandra.net.OutboundConnections.LARGE_MESSAGE_THRESHOLD;
+
+/**
+ * {@link InboundMessageHandler} operates on frames that adhere to a certain contract
+ * (see {@link FrameDecoder.IntactFrame} and {@link FrameDecoder.CorruptFrame} javadoc).
+ *
+ * Legacy (pre-4.0) messaging protocol does not natively support framing, however. The job
+ * of {@link FrameDecoderLegacy} is turn a raw stream of messages, serialized back to back,
+ * into a sequence of frames that adhere to 4.0+ conventions.
+ */
+class FrameDecoderLegacy extends FrameDecoder
+{
+ private final int messagingVersion;
+
+ private int remainingBytesInLargeMessage = 0;
+
+ FrameDecoderLegacy(BufferPoolAllocator allocator, int messagingVersion)
+ {
+ super(allocator);
+ this.messagingVersion = messagingVersion;
+ }
+
+ final void decode(Collection<Frame> into, ShareableBytes newBytes)
+ {
+ ByteBuffer in = newBytes.get();
+ try
+ {
+ if (stash != null)
+ {
+ int length = Message.serializer.inferMessageSize(stash, 0, stash.position(), messagingVersion);
+ while (length < 0)
+ {
+ if (!in.hasRemaining())
+ return;
+
+ if (stash.position() == stash.capacity())
+ stash = ensureCapacity(stash, stash.capacity() * 2);
+ copyToSize(in, stash, stash.capacity());
+
+ length = Message.serializer.inferMessageSize(stash, 0, stash.position(), messagingVersion);
+ if (length >= 0 && length < stash.position())
+ {
+ int excess = stash.position() - length;
+ in.position(in.position() - excess);
+ stash.position(length);
+ }
+ }
+
+ final boolean isSelfContained;
+ if (length <= LARGE_MESSAGE_THRESHOLD)
+ {
+ isSelfContained = true;
+
+ if (length > stash.capacity())
+ stash = ensureCapacity(stash, length);
+
+ stash.limit(length);
+ allocator.putUnusedPortion(stash); // we may be over capacity from earlier doubling
+ if (!copyToSize(in, stash, length))
+ return;
+ }
+ else
+ {
+ isSelfContained = false;
+ remainingBytesInLargeMessage = length - stash.position();
+
+ stash.limit(stash.position());
+ allocator.putUnusedPortion(stash);
+ }
+
+ stash.flip();
+ assert !isSelfContained || stash.limit() == length;
+ ShareableBytes stashed = ShareableBytes.wrap(stash);
+ into.add(new IntactFrame(isSelfContained, stashed));
+ stash = null;
+ }
+
+ if (remainingBytesInLargeMessage > 0)
+ {
+ if (remainingBytesInLargeMessage >= newBytes.remaining())
+ {
+ remainingBytesInLargeMessage -= newBytes.remaining();
+ into.add(new IntactFrame(false, newBytes.sliceAndConsume(newBytes.remaining())));
+ return;
+ }
+ else
+ {
+ Frame frame = new IntactFrame(false, newBytes.sliceAndConsume(remainingBytesInLargeMessage));
+ remainingBytesInLargeMessage = 0;
+ into.add(frame);
+ }
+ }
+
+ // we loop incrementing our end pointer until we have no more complete messages,
+ // at which point we slice the complete messages, and stash the remainder
+ int begin = in.position();
+ int end = begin;
+ int limit = in.limit();
+
+ if (begin == limit)
+ return;
+
+ while (true)
+ {
+ int length = Message.serializer.inferMessageSize(in, end, limit, messagingVersion);
+
+ if (length >= 0)
+ {
+ if (end + length <= limit)
+ {
+ // we have a complete message, so just bump our end pointer
+ end += length;
+
+ // if we have more bytes, continue to look for another message
+ if (end < limit)
+ continue;
+
+ // otherwise reset length, as we have accounted for it in end
+ length = 0;
+ }
+ }
+
+ // we are done; if we have found any complete messages, slice them all into a single frame
+ if (begin < end)
+ into.add(new IntactFrame(true, newBytes.slice(begin, end)));
+
+ // now consider stashing anything leftover
+ if (length < 0)
+ {
+ stash(newBytes, max(64, limit - end), end, limit - end);
+ }
+ else if (length > LARGE_MESSAGE_THRESHOLD)
+ {
+ remainingBytesInLargeMessage = length - (limit - end);
+ Frame frame = new IntactFrame(false, newBytes.slice(end, limit));
+ into.add(frame);
+ }
+ else if (length > 0)
+ {
+ stash(newBytes, length, end, limit - end);
+ }
+ break;
+ }
+ }
+ catch (Message.InvalidLegacyProtocolMagic e)
+ {
+ into.add(CorruptFrame.unrecoverable(e.read, Message.PROTOCOL_MAGIC));
+ }
+ finally
+ {
+ newBytes.release();
+ }
+ }
+
+ void addLastTo(ChannelPipeline pipeline)
+ {
+ pipeline.addLast("frameDecoderNone", this);
+ }
+}
diff --git a/src/java/org/apache/cassandra/net/FrameDecoderLegacyLZ4.java b/src/java/org/apache/cassandra/net/FrameDecoderLegacyLZ4.java
new file mode 100644
index 0000000..bf6bc17
--- /dev/null
+++ b/src/java/org/apache/cassandra/net/FrameDecoderLegacyLZ4.java
@@ -0,0 +1,378 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.net;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayDeque;
+import java.util.Collection;
+import java.util.Deque;
+
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.channel.ChannelInboundHandlerAdapter;
+import io.netty.channel.ChannelPipeline;
+import io.netty.handler.codec.compression.Lz4FrameDecoder;
+import net.jpountz.lz4.LZ4Factory;
+import net.jpountz.lz4.LZ4SafeDecompressor;
+import net.jpountz.xxhash.XXHash32;
+import net.jpountz.xxhash.XXHashFactory;
+import org.apache.cassandra.utils.memory.BufferPool;
+
+import static java.lang.Integer.reverseBytes;
+import static java.lang.String.format;
+import static org.apache.cassandra.net.LegacyLZ4Constants.*;
+import static org.apache.cassandra.utils.ByteBufferUtil.copyBytes;
+
+/**
+ * A {@link FrameDecoder} consisting of two chained handlers:
+ * 1. A legacy LZ4 block decoder, described below in the description of {@link LZ4Decoder}, followed by
+ * 2. An instance of {@link FrameDecoderLegacy} - transforming the raw messages in the uncompressed stream
+ * into properly formed frames expected by {@link InboundMessageHandler}
+ */
+class FrameDecoderLegacyLZ4 extends FrameDecoderLegacy
+{
+ FrameDecoderLegacyLZ4(BufferPoolAllocator allocator, int messagingVersion)
+ {
+ super(allocator, messagingVersion);
+ }
+
+ @Override
+ void addLastTo(ChannelPipeline pipeline)
+ {
+ pipeline.addLast("legacyLZ4Decoder", new LZ4Decoder(allocator));
+ pipeline.addLast("frameDecoderNone", this);
+ }
+
+ /**
+ * An implementation of LZ4 decoder, used for legacy (3.0, 3.11) connections.
+ *
+ * Netty's provided implementation - {@link Lz4FrameDecoder} couldn't be reused for
+ * two reasons:
+ * 1. It has very poor performance when coupled with xxHash, which we use for legacy connections -
+ * allocating a single-byte array and making a JNI call <em>for every byte of the payload</em>
+ * 2. It was tricky to efficiently integrate with upstream {@link FrameDecoder}, and impossible
+ * to make it play nicely with flow control - Netty's implementation, based on
+ * {@link io.netty.handler.codec.ByteToMessageDecoder}, would potentially keep triggering
+ * reads on its own volition for as long as its last read had no completed frames to supply
+ * - defying our goal to only ever trigger channel reads when explicitly requested
+ *
+ * Since the original LZ4 block format does not contains size of compressed block and size of original data
+ * this encoder uses format like <a href="https://github.com/idelpivnitskiy/lz4-java">LZ4 Java</a> library
+ * written by Adrien Grand and approved by Yann Collet (author of original LZ4 library), as implemented by
+ * Netty's {@link Lz4FrameDecoder}, but adapted for our interaction model.
+ *
+ * 0 1 2 3
+ * 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ * | |
+ * + Magic +
+ * | |
+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ * |T| Compressed Length
+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ * | Uncompressed Length
+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ * | xxHash32 of Uncompressed Payload
+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ * | |
+ * +-+ +
+ * | |
+ * + Payload +
+ * | |
+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ */
+ private static class LZ4Decoder extends ChannelInboundHandlerAdapter
+ {
+ private static final XXHash32 xxhash =
+ XXHashFactory.fastestInstance().hash32();
+
+ private static final LZ4SafeDecompressor decompressor =
+ LZ4Factory.fastestInstance().safeDecompressor();
+
+ private final BufferPoolAllocator allocator;
+
+ LZ4Decoder(BufferPoolAllocator allocator)
+ {
+ this.allocator = allocator;
+ }
+
+ private final Deque<ShareableBytes> frames = new ArrayDeque<>(4);
+
+ // total # of frames decoded between two subsequent invocations of channelReadComplete()
+ private int decodedFrameCount = 0;
+
+ @Override
+ public void channelRead(ChannelHandlerContext ctx, Object msg) throws CorruptLZ4Frame
+ {
+ assert msg instanceof BufferPoolAllocator.Wrapped;
+ ByteBuffer buf = ((BufferPoolAllocator.Wrapped) msg).adopt();
+ // netty will probably have mis-predicted the space needed
+ BufferPool.putUnusedPortion(buf);
+
+ CorruptLZ4Frame error = null;
+ try
+ {
+ decode(frames, ShareableBytes.wrap(buf));
+ }
+ catch (CorruptLZ4Frame e)
+ {
+ error = e;
+ }
+ finally
+ {
+ decodedFrameCount += frames.size();
+ while (!frames.isEmpty())
+ ctx.fireChannelRead(frames.poll());
+ }
+
+ if (null != error)
+ throw error;
+ }
+
+ @Override
+ public void channelReadComplete(ChannelHandlerContext ctx)
+ {
+ /*
+ * If no frames have been decoded from the entire batch of channelRead() calls,
+ * then we must trigger another channel read explicitly, or else risk stalling
+ * forever without bytes to complete the current in-flight frame.
+ */
+ if (null != stash && decodedFrameCount == 0 && !ctx.channel().config().isAutoRead())
+ ctx.read();
+
+ decodedFrameCount = 0;
+ ctx.fireChannelReadComplete();
+ }
+
+ private void decode(Collection<ShareableBytes> into, ShareableBytes newBytes) throws CorruptLZ4Frame
+ {
+ try
+ {
+ doDecode(into, newBytes);
+ }
+ finally
+ {
+ newBytes.release();
+ }
+ }
+
+ private void doDecode(Collection<ShareableBytes> into, ShareableBytes newBytes) throws CorruptLZ4Frame
+ {
+ ByteBuffer in = newBytes.get();
+
+ if (null != stash)
+ {
+ if (!copyToSize(in, stash, HEADER_LENGTH))
+ return;
+
+ header.read(stash, 0);
+ header.validate();
+
+ int frameLength = header.frameLength();
+ stash = ensureCapacity(stash, frameLength);
+
+ if (!copyToSize(in, stash, frameLength))
+ return;
+
+ stash.flip();
+ ShareableBytes stashed = ShareableBytes.wrap(stash);
+ stash = null;
+
+ try
+ {
+ into.add(decompressFrame(stashed, 0, frameLength, header));
+ }
+ finally
+ {
+ stashed.release();
+ }
+ }
+
+ int begin = in.position();
+ int limit = in.limit();
+ while (begin < limit)
+ {
+ int remaining = limit - begin;
+ if (remaining < HEADER_LENGTH)
+ {
+ stash(newBytes, HEADER_LENGTH, begin, remaining);
+ return;
+ }
+
+ header.read(in, begin);
+ header.validate();
+
+ int frameLength = header.frameLength();
+ if (remaining < frameLength)
+ {
+ stash(newBytes, frameLength, begin, remaining);
+ return;
+ }
+
+ into.add(decompressFrame(newBytes, begin, begin + frameLength, header));
+ begin += frameLength;
+ }
+ }
+
+ private ShareableBytes decompressFrame(ShareableBytes bytes, int begin, int end, Header header) throws CorruptLZ4Frame
+ {
+ ByteBuffer buf = bytes.get();
+
+ if (header.uncompressedLength == 0)
+ return bytes.slice(begin + HEADER_LENGTH, end);
+
+ if (!header.isCompressed())
+ {
+ validateChecksum(buf, begin + HEADER_LENGTH, header);
+ return bytes.slice(begin + HEADER_LENGTH, end);
+ }
+
+ ByteBuffer out = allocator.get(header.uncompressedLength);
+ try
+ {
+ int sourceLength = end - (begin + HEADER_LENGTH);
+ decompressor.decompress(buf, begin + HEADER_LENGTH, sourceLength, out, 0, header.uncompressedLength);
+ validateChecksum(out, 0, header);
+ return ShareableBytes.wrap(out);
+ }
+ catch (Throwable t)
+ {
+ BufferPool.put(out);
+ throw t;
+ }
+ }
+
+ private void validateChecksum(ByteBuffer buf, int begin, Header header) throws CorruptLZ4Frame
+ {
+ int checksum = xxhash.hash(buf, begin, header.uncompressedLength, XXHASH_SEED) & XXHASH_MASK;
+ if (checksum != header.checksum)
+ except("Invalid checksum detected: %d (expected: %d)", checksum, header.checksum);
+ }
+
+ @Override
+ public void channelInactive(ChannelHandlerContext ctx)
+ {
+ if (null != stash)
+ {
+ BufferPool.put(stash);
+ stash = null;
+ }
+
+ while (!frames.isEmpty())
+ frames.poll().release();
+
+ ctx.fireChannelInactive();
+ }
+
+ /* reusable container for deserialized header fields */
+ private static final class Header
+ {
+ long magicNumber;
+ byte token;
+ int compressedLength;
+ int uncompressedLength;
+ int checksum;
+
+ int frameLength()
+ {
+ return HEADER_LENGTH + compressedLength;
+ }
+
+ boolean isCompressed()
+ {
+ return (token & 0xF0) == 0x20;
+ }
+
+ int maxUncompressedLength()
+ {
+ return 1 << ((token & 0x0F) + 10);
+ }
+
+ void read(ByteBuffer in, int begin)
+ {
+ magicNumber = in.getLong(begin + MAGIC_NUMBER_OFFSET );
+ token = in.get (begin + TOKEN_OFFSET );
+ compressedLength = reverseBytes(in.getInt (begin + COMPRESSED_LENGTH_OFFSET ));
+ uncompressedLength = reverseBytes(in.getInt (begin + UNCOMPRESSED_LENGTH_OFFSET));
+ checksum = reverseBytes(in.getInt (begin + CHECKSUM_OFFSET ));
+ }
+
+ void validate() throws CorruptLZ4Frame
+ {
+ if (magicNumber != MAGIC_NUMBER)
+ except("Invalid magic number at the beginning of an LZ4 block: %d", magicNumber);
+
+ int blockType = token & 0xF0;
+ if (!(blockType == BLOCK_TYPE_COMPRESSED || blockType == BLOCK_TYPE_NON_COMPRESSED))
+ except("Invalid block type encountered: %d", blockType);
+
+ if (compressedLength < 0 || compressedLength > MAX_BLOCK_LENGTH)
+ except("Invalid compressedLength: %d (expected: 0-%d)", compressedLength, MAX_BLOCK_LENGTH);
+
+ if (uncompressedLength < 0 || uncompressedLength > maxUncompressedLength())
+ except("Invalid uncompressedLength: %d (expected: 0-%d)", uncompressedLength, maxUncompressedLength());
+
+ if ( uncompressedLength == 0 && compressedLength != 0
+ || uncompressedLength != 0 && compressedLength == 0
+ || !isCompressed() && uncompressedLength != compressedLength)
+ {
+ except("Stream corrupted: compressedLength(%d) and decompressedLength(%d) mismatch", compressedLength, uncompressedLength);
+ }
+ }
+ }
+ private final Header header = new Header();
+
+ /**
+ * @return {@code in} if has sufficient capacity, otherwise a replacement from {@code BufferPool} that {@code in} is copied into
+ */
+ private ByteBuffer ensureCapacity(ByteBuffer in, int capacity)
+ {
+ if (in.capacity() >= capacity)
+ return in;
+
+ ByteBuffer out = allocator.getAtLeast(capacity);
+ in.flip();
+ out.put(in);
+ BufferPool.put(in);
+ return out;
+ }
+
+ private ByteBuffer stash;
+
+ private void stash(ShareableBytes in, int stashLength, int begin, int length)
+ {
+ ByteBuffer out = allocator.getAtLeast(stashLength);
+ copyBytes(in.get(), begin, out, 0, length);
+ out.position(length);
+ stash = out;
+ }
+
+ static final class CorruptLZ4Frame extends IOException
+ {
+ CorruptLZ4Frame(String message)
+ {
+ super(message);
+ }
+ }
+
+ private static void except(String format, Object... args) throws CorruptLZ4Frame
+ {
+ throw new CorruptLZ4Frame(format(format, args));
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/net/FrameDecoderUnprotected.java b/src/java/org/apache/cassandra/net/FrameDecoderUnprotected.java
new file mode 100644
index 0000000..44414e3
--- /dev/null
+++ b/src/java/org/apache/cassandra/net/FrameDecoderUnprotected.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.net;
+
+import java.nio.ByteBuffer;
+import java.util.Collection;
+
+import io.netty.channel.ChannelPipeline;
+
+import static org.apache.cassandra.net.FrameDecoderCrc.HEADER_LENGTH;
+import static org.apache.cassandra.net.FrameDecoderCrc.isSelfContained;
+import static org.apache.cassandra.net.FrameDecoderCrc.payloadLength;
+import static org.apache.cassandra.net.FrameDecoderCrc.readHeader6b;
+import static org.apache.cassandra.net.FrameDecoderCrc.verifyHeader6b;
+
+/**
+ * A frame decoder for unprotected frames, i.e. those without any modification or payload protection.
+ * This is non-standard, and useful for systems that have a trusted transport layer that want
+ * to avoid incurring the (very low) cost of computing a CRC. All we do is accumulate the bytes
+ * of the frame, verify the frame header, and pass through the bytes stripped of the header.
+ *
+ * Every on-wire frame contains:
+ * 1. Payload length (17 bits)
+ * 2. {@code isSelfContained} flag (1 bit)
+ * 3. Header padding (6 bits)
+ * 4. CRC24 of the header (24 bits)
+ * 5. Payload (up to 2 ^ 17 - 1 bits)
+ *
+ * 0 1 2 3
+ * 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ * | Payload Length |C| |
+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ * CRC24 of Header | |
+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +
+ * | |
+ * + +
+ * | Payload |
+ * + +
+ * | |
+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ */
+final class FrameDecoderUnprotected extends FrameDecoderWith8bHeader
+{
+ FrameDecoderUnprotected(BufferPoolAllocator allocator)
+ {
+ super(allocator);
+ }
+
+ public static FrameDecoderUnprotected create(BufferPoolAllocator allocator)
+ {
+ return new FrameDecoderUnprotected(allocator);
+ }
+
+ final long readHeader(ByteBuffer frame, int begin)
+ {
+ return readHeader6b(frame, begin);
+ }
+
+ final CorruptFrame verifyHeader(long header6b)
+ {
+ return verifyHeader6b(header6b);
+ }
+
+ final int frameLength(long header6b)
+ {
+ return payloadLength(header6b) + HEADER_LENGTH;
+ }
+
+ final Frame unpackFrame(ShareableBytes bytes, int begin, int end, long header6b)
+ {
+ boolean isSelfContained = isSelfContained(header6b);
+ return new IntactFrame(isSelfContained, bytes.slice(begin + HEADER_LENGTH, end));
+ }
+
+ void decode(Collection<Frame> into, ShareableBytes bytes)
+ {
+ decode(into, bytes, HEADER_LENGTH);
+ }
+
+ void addLastTo(ChannelPipeline pipeline)
+ {
+ pipeline.addLast("frameDecoderUnprotected", this);
+ }
+}
diff --git a/src/java/org/apache/cassandra/net/FrameDecoderWith8bHeader.java b/src/java/org/apache/cassandra/net/FrameDecoderWith8bHeader.java
new file mode 100644
index 0000000..ed87d82
--- /dev/null
+++ b/src/java/org/apache/cassandra/net/FrameDecoderWith8bHeader.java
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.net;
+
+import java.nio.ByteBuffer;
+import java.util.Collection;
+
+import net.nicoulaj.compilecommand.annotations.Inline;
+
+/**
+ * An abstract frame decoder for frames utilising a fixed length header of 8 bytes or smaller.
+ * Implements a generic frame decode method, that is backed by the four abstract methods
+ * (three of which simply decode and verify the header as a long).
+ *
+ * Implementors are expected to declare their implementation methods final, and an outer decode
+ * method implemented to invoke this class' {@link #decode}, so that it may be inlined with the
+ * abstract method implementations then inlined into it.
+ */
+abstract class FrameDecoderWith8bHeader extends FrameDecoder
+{
+ FrameDecoderWith8bHeader(BufferPoolAllocator allocator)
+ {
+ super(allocator);
+ }
+
+ /**
+ * Read a header that is 8 bytes or shorter, without modifying the buffer position.
+ * If your header is longer than this, you will need to implement your own {@link #decode}
+ */
+ abstract long readHeader(ByteBuffer in, int begin);
+ /**
+ * Verify the header, and return an unrecoverable CorruptFrame if it is corrupted
+ * @return null or CorruptFrame.unrecoverable
+ */
+ abstract CorruptFrame verifyHeader(long header);
+
+ /**
+ * Calculate the full frame length from info provided by the header, including the length of the header and any triler
+ */
+ abstract int frameLength(long header);
+
+ /**
+ * Extract a frame known to cover the given range.
+ * If {@code transferOwnership}, the method is responsible for ensuring bytes.release() is invoked at some future point.
+ */
+ abstract Frame unpackFrame(ShareableBytes bytes, int begin, int end, long header);
+
+ /**
+ * Decode a number of frames using the above abstract method implementations.
+ * It is expected for this method to be invoked by the implementing class' {@link #decode(Collection, ShareableBytes)}
+ * so that this implementation will be inlined, and all of the abstract method implementations will also be inlined.
+ */
+ @Inline
+ protected void decode(Collection<Frame> into, ShareableBytes newBytes, int headerLength)
+ {
+ ByteBuffer in = newBytes.get();
+
+ try
+ {
+ if (stash != null)
+ {
+ if (!copyToSize(in, stash, headerLength))
+ return;
+
+ long header = readHeader(stash, 0);
+ CorruptFrame c = verifyHeader(header);
+ if (c != null)
+ {
+ discard();
+ into.add(c);
+ return;
+ }
+
+ int frameLength = frameLength(header);
+ stash = ensureCapacity(stash, frameLength);
+
+ if (!copyToSize(in, stash, frameLength))
+ return;
+
+ stash.flip();
+ ShareableBytes stashed = ShareableBytes.wrap(stash);
+ stash = null;
+
+ try
+ {
+ into.add(unpackFrame(stashed, 0, frameLength, header));
+ }
+ finally
+ {
+ stashed.release();
+ }
+ }
+
+ int begin = in.position();
+ int limit = in.limit();
+ while (begin < limit)
+ {
+ int remaining = limit - begin;
+ if (remaining < headerLength)
+ {
+ stash(newBytes, headerLength, begin, remaining);
+ return;
+ }
+
+ long header = readHeader(in, begin);
+ CorruptFrame c = verifyHeader(header);
+ if (c != null)
+ {
+ into.add(c);
+ return;
+ }
+
+ int frameLength = frameLength(header);
+ if (remaining < frameLength)
+ {
+ stash(newBytes, frameLength, begin, remaining);
+ return;
+ }
+
+ into.add(unpackFrame(newBytes, begin, begin + frameLength, header));
+ begin += frameLength;
+ }
+ }
+ finally
+ {
+ newBytes.release();
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/net/FrameEncoder.java b/src/java/org/apache/cassandra/net/FrameEncoder.java
new file mode 100644
index 0000000..d9df166
--- /dev/null
+++ b/src/java/org/apache/cassandra/net/FrameEncoder.java
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.net;
+
+import java.nio.ByteBuffer;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.channel.ChannelOutboundHandlerAdapter;
+import io.netty.channel.ChannelPromise;
+import org.apache.cassandra.io.compress.BufferType;
+import org.apache.cassandra.utils.memory.BufferPool;
+
+abstract class FrameEncoder extends ChannelOutboundHandlerAdapter
+{
+ /**
+ * An abstraction useful for transparently allocating buffers that can be written to upstream
+ * of the {@code FrameEncoder} without knowledge of the encoder's frame layout, while ensuring
+ * enough space to write the remainder of the frame's contents is reserved.
+ */
+ static class Payload
+ {
+ // isSelfContained is a flag in the Frame API, indicating if the contents consists of only complete messages
+ private boolean isSelfContained;
+ // the buffer to write to
+ final ByteBuffer buffer;
+ // the number of header bytes to reserve
+ final int headerLength;
+ // the number of trailer bytes to reserve
+ final int trailerLength;
+ // an API-misuse detector
+ private boolean isFinished = false;
+
+ Payload(boolean isSelfContained, int payloadCapacity)
+ {
+ this(isSelfContained, payloadCapacity, 0, 0);
+ }
+
+ Payload(boolean isSelfContained, int payloadCapacity, int headerLength, int trailerLength)
+ {
+ this.isSelfContained = isSelfContained;
+ this.headerLength = headerLength;
+ this.trailerLength = trailerLength;
+
+ buffer = BufferPool.getAtLeast(payloadCapacity + headerLength + trailerLength, BufferType.OFF_HEAP);
+ assert buffer.capacity() >= payloadCapacity + headerLength + trailerLength;
+ buffer.position(headerLength);
+ buffer.limit(buffer.capacity() - trailerLength);
+ }
+
+ void setSelfContained(boolean isSelfContained)
+ {
+ this.isSelfContained = isSelfContained;
+ }
+
+ // do not invoke after finish()
+ boolean isEmpty()
+ {
+ assert !isFinished;
+ return buffer.position() == headerLength;
+ }
+
+ // do not invoke after finish()
+ int length()
+ {
+ assert !isFinished;
+ return buffer.position() - headerLength;
+ }
+
+ // do not invoke after finish()
+ int remaining()
+ {
+ assert !isFinished;
+ return buffer.remaining();
+ }
+
+ // do not invoke after finish()
+ void trim(int length)
+ {
+ assert !isFinished;
+ buffer.position(headerLength + length);
+ }
+
+ // may not be written to or queried, after this is invoked; must be passed straight to an encoder (or release called)
+ void finish()
+ {
+ assert !isFinished;
+ isFinished = true;
+ buffer.limit(buffer.position() + trailerLength);
+ buffer.position(0);
+ BufferPool.putUnusedPortion(buffer);
+ }
+
+ void release()
+ {
+ BufferPool.put(buffer);
+ }
+ }
+
+ interface PayloadAllocator
+ {
+ public static final PayloadAllocator simple = Payload::new;
+ Payload allocate(boolean isSelfContained, int capacity);
+ }
+
+ PayloadAllocator allocator()
+ {
+ return PayloadAllocator.simple;
+ }
+
+ /**
+ * Takes ownership of the lifetime of the provided buffer, which can be assumed to be managed by BufferPool
+ */
+ abstract ByteBuf encode(boolean isSelfContained, ByteBuffer buffer);
+
+ public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise promise) throws Exception
+ {
+ if (!(msg instanceof Payload))
+ throw new IllegalStateException("Unexpected type: " + msg);
+
+ Payload payload = (Payload) msg;
+ ByteBuf write = encode(payload.isSelfContained, payload.buffer);
+ ctx.write(write, promise);
+ }
+}
diff --git a/src/java/org/apache/cassandra/net/FrameEncoderCrc.java b/src/java/org/apache/cassandra/net/FrameEncoderCrc.java
new file mode 100644
index 0000000..2d07d6d
--- /dev/null
+++ b/src/java/org/apache/cassandra/net/FrameEncoderCrc.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.net;
+
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.util.zip.CRC32;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.channel.ChannelHandler;
+import org.apache.cassandra.utils.memory.BufferPool;
+
+import static org.apache.cassandra.net.Crc.*;
+
+/**
+ * Please see {@link FrameDecoderCrc} for description of the framing produced by this encoder.
+ */
+@ChannelHandler.Sharable
+class FrameEncoderCrc extends FrameEncoder
+{
+ static final int HEADER_LENGTH = 6;
+ private static final int TRAILER_LENGTH = 4;
+ static final int HEADER_AND_TRAILER_LENGTH = 10;
+
+ static final FrameEncoderCrc instance = new FrameEncoderCrc();
+ static final PayloadAllocator allocator = (isSelfContained, capacity) ->
+ new Payload(isSelfContained, capacity, HEADER_LENGTH, TRAILER_LENGTH);
+
+ PayloadAllocator allocator()
+ {
+ return allocator;
+ }
+
+ static void writeHeader(ByteBuffer frame, boolean isSelfContained, int dataLength)
+ {
+ int header3b = dataLength;
+ if (isSelfContained)
+ header3b |= 1 << 17;
+ int crc = crc24(header3b, 3);
+
+ put3b(frame, 0, header3b);
+ put3b(frame, 3, crc);
+ }
+
+ private static void put3b(ByteBuffer frame, int index, int put3b)
+ {
+ frame.put(index , (byte) put3b );
+ frame.put(index + 1, (byte)(put3b >>> 8) );
+ frame.put(index + 2, (byte)(put3b >>> 16));
+ }
+
+ ByteBuf encode(boolean isSelfContained, ByteBuffer frame)
+ {
+ try
+ {
+ int frameLength = frame.remaining();
+ int dataLength = frameLength - HEADER_AND_TRAILER_LENGTH;
+ if (dataLength >= 1 << 17)
+ throw new IllegalArgumentException("Maximum payload size is 128KiB");
+
+ writeHeader(frame, isSelfContained, dataLength);
+
+ CRC32 crc = crc32();
+ frame.position(HEADER_LENGTH);
+ frame.limit(dataLength + HEADER_LENGTH);
+ crc.update(frame);
+
+ int frameCrc = (int) crc.getValue();
+ if (frame.order() == ByteOrder.BIG_ENDIAN)
+ frameCrc = Integer.reverseBytes(frameCrc);
+
+ frame.limit(frameLength);
+ frame.putInt(frameLength - TRAILER_LENGTH, frameCrc);
+ frame.position(0);
+ return GlobalBufferPoolAllocator.wrap(frame);
+ }
+ catch (Throwable t)
+ {
+ BufferPool.put(frame);
+ throw t;
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/net/FrameEncoderLZ4.java b/src/java/org/apache/cassandra/net/FrameEncoderLZ4.java
new file mode 100644
index 0000000..12351ce
--- /dev/null
+++ b/src/java/org/apache/cassandra/net/FrameEncoderLZ4.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.net;
+
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.util.zip.CRC32;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.channel.ChannelHandler;
+import net.jpountz.lz4.LZ4Compressor;
+import net.jpountz.lz4.LZ4Factory;
+import org.apache.cassandra.io.compress.BufferType;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.memory.BufferPool;
+
+import static org.apache.cassandra.net.Crc.*;
+
+/**
+ * Please see {@link FrameDecoderLZ4} for description of the framing produced by this encoder.
+ */
+@ChannelHandler.Sharable
+class FrameEncoderLZ4 extends FrameEncoder
+{
+ static final FrameEncoderLZ4 fastInstance = new FrameEncoderLZ4(LZ4Factory.fastestInstance().fastCompressor());
+
+ private final LZ4Compressor compressor;
+
+ private FrameEncoderLZ4(LZ4Compressor compressor)
+ {
+ this.compressor = compressor;
+ }
+
+ private static final int HEADER_LENGTH = 8;
+ static final int HEADER_AND_TRAILER_LENGTH = 12;
+
+ private static void writeHeader(ByteBuffer frame, boolean isSelfContained, long compressedLength, long uncompressedLength)
+ {
+ long header5b = compressedLength | (uncompressedLength << 17);
+ if (isSelfContained)
+ header5b |= 1L << 34;
+
+ long crc = crc24(header5b, 5);
+
+ long header8b = header5b | (crc << 40);
+ if (frame.order() == ByteOrder.BIG_ENDIAN)
+ header8b = Long.reverseBytes(header8b);
+
+ frame.putLong(0, header8b);
+ }
+
+ public ByteBuf encode(boolean isSelfContained, ByteBuffer in)
+ {
+ ByteBuffer frame = null;
+ try
+ {
+ int uncompressedLength = in.remaining();
+ if (uncompressedLength >= 1 << 17)
+ throw new IllegalArgumentException("Maximum uncompressed payload size is 128KiB");
+
+ int maxOutputLength = compressor.maxCompressedLength(uncompressedLength);
+ frame = BufferPool.getAtLeast(HEADER_AND_TRAILER_LENGTH + maxOutputLength, BufferType.OFF_HEAP);
+
+ int compressedLength = compressor.compress(in, in.position(), uncompressedLength, frame, HEADER_LENGTH, maxOutputLength);
+
+ if (compressedLength >= uncompressedLength)
+ {
+ ByteBufferUtil.copyBytes(in, in.position(), frame, HEADER_LENGTH, uncompressedLength);
+ compressedLength = uncompressedLength;
+ uncompressedLength = 0;
+ }
+
+ writeHeader(frame, isSelfContained, compressedLength, uncompressedLength);
+
+ CRC32 crc = crc32();
+ frame.position(HEADER_LENGTH);
+ frame.limit(compressedLength + HEADER_LENGTH);
+ crc.update(frame);
+
+ int frameCrc = (int) crc.getValue();
+ if (frame.order() == ByteOrder.BIG_ENDIAN)
+ frameCrc = Integer.reverseBytes(frameCrc);
+ int frameLength = compressedLength + HEADER_AND_TRAILER_LENGTH;
+
+ frame.limit(frameLength);
+ frame.putInt(frameCrc);
+ frame.position(0);
+
+ BufferPool.putUnusedPortion(frame);
+ return GlobalBufferPoolAllocator.wrap(frame);
+ }
+ catch (Throwable t)
+ {
+ if (frame != null)
+ BufferPool.put(frame);
+ throw t;
+ }
+ finally
+ {
+ BufferPool.put(in);
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/net/FrameEncoderLegacy.java b/src/java/org/apache/cassandra/net/FrameEncoderLegacy.java
new file mode 100644
index 0000000..8bfd267
--- /dev/null
+++ b/src/java/org/apache/cassandra/net/FrameEncoderLegacy.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.net;
+
+import java.nio.ByteBuffer;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.channel.ChannelHandler;
+
+/**
+ * A no-op frame encoder: legacy format doesn't support framing. Instead, the byte stream
+ * contains messages, serialized back to back.
+ */
+@ChannelHandler.Sharable
+class FrameEncoderLegacy extends FrameEncoder
+{
+ static final FrameEncoderLegacy instance = new FrameEncoderLegacy();
+
+ ByteBuf encode(boolean isSelfContained, ByteBuffer buffer)
+ {
+ return GlobalBufferPoolAllocator.wrap(buffer);
+ }
+}
diff --git a/src/java/org/apache/cassandra/net/FrameEncoderLegacyLZ4.java b/src/java/org/apache/cassandra/net/FrameEncoderLegacyLZ4.java
new file mode 100644
index 0000000..3b29ecb
--- /dev/null
+++ b/src/java/org/apache/cassandra/net/FrameEncoderLegacyLZ4.java
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.net;
+
+import java.nio.ByteBuffer;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.channel.ChannelHandler;
+import net.jpountz.lz4.LZ4Compressor;
+import net.jpountz.lz4.LZ4Factory;
+import net.jpountz.xxhash.XXHash32;
+import net.jpountz.xxhash.XXHashFactory;
+import org.apache.cassandra.io.compress.BufferType;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.memory.BufferPool;
+
+import static java.lang.Integer.reverseBytes;
+import static java.lang.Math.min;
+import static org.apache.cassandra.net.LegacyLZ4Constants.*;
+
+/**
+ * LZ4 {@link FrameEncoder} implementation for compressed legacy (3.0, 3.11) connections.
+ *
+ * Netty's provided implementation - {@link io.netty.handler.codec.compression.Lz4FrameEncoder} couldn't be reused
+ * for two reasons:
+ * 1. It notifies flushes as successful when they may not be, by flushing an empty buffer ahead
+ * of the compressed buffer
+ * 2. It has very poor performance when coupled with xxHash, which we use for legacy connections -
+ * allocating a single-byte array and making a JNI call <em>for every byte of the payload</em>
+ *
+ * Please see {@link FrameDecoderLegacyLZ4} for the description of the on-wire format of the LZ4 blocks
+ * used by this encoder.
+ */
+@ChannelHandler.Sharable
+class FrameEncoderLegacyLZ4 extends FrameEncoder
+{
+ static final FrameEncoderLegacyLZ4 instance =
+ new FrameEncoderLegacyLZ4(XXHashFactory.fastestInstance().hash32(),
+ LZ4Factory.fastestInstance().fastCompressor());
+
+ private final XXHash32 xxhash;
+ private final LZ4Compressor compressor;
+
+ private FrameEncoderLegacyLZ4(XXHash32 xxhash, LZ4Compressor compressor)
+ {
+ this.xxhash = xxhash;
+ this.compressor = compressor;
+ }
+
+ @Override
+ ByteBuf encode(boolean isSelfContained, ByteBuffer payload)
+ {
+ ByteBuffer frame = null;
+ try
+ {
+ frame = BufferPool.getAtLeast(calculateMaxFrameLength(payload), BufferType.OFF_HEAP);
+
+ int frameOffset = 0;
+ int payloadOffset = 0;
+
+ int payloadLength = payload.remaining();
+ while (payloadOffset < payloadLength)
+ {
+ int blockLength = min(DEFAULT_BLOCK_LENGTH, payloadLength - payloadOffset);
+ frameOffset += compressBlock(frame, frameOffset, payload, payloadOffset, blockLength);
+ payloadOffset += blockLength;
+ }
+
+ frame.limit(frameOffset);
+ BufferPool.putUnusedPortion(frame);
+
+ return GlobalBufferPoolAllocator.wrap(frame);
+ }
+ catch (Throwable t)
+ {
+ if (null != frame)
+ BufferPool.put(frame);
+ throw t;
+ }
+ finally
+ {
+ BufferPool.put(payload);
+ }
+ }
+
+ private int compressBlock(ByteBuffer frame, int frameOffset, ByteBuffer payload, int payloadOffset, int blockLength)
+ {
+ int frameBytesRemaining = frame.limit() - (frameOffset + HEADER_LENGTH);
+ int compressedLength = compressor.compress(payload, payloadOffset, blockLength, frame, frameOffset + HEADER_LENGTH, frameBytesRemaining);
+ if (compressedLength >= blockLength)
+ {
+ ByteBufferUtil.copyBytes(payload, payloadOffset, frame, frameOffset + HEADER_LENGTH, blockLength);
+ compressedLength = blockLength;
+ }
+ int checksum = xxhash.hash(payload, payloadOffset, blockLength, XXHASH_SEED) & XXHASH_MASK;
+ writeHeader(frame, frameOffset, compressedLength, blockLength, checksum);
+ return HEADER_LENGTH + compressedLength;
+ }
+
+ private static final byte TOKEN_NON_COMPRESSED = 0x15;
+ private static final byte TOKEN_COMPRESSED = 0x25;
+
+ private static void writeHeader(ByteBuffer frame, int frameOffset, int compressedLength, int uncompressedLength, int checksum)
+ {
+ byte token = compressedLength == uncompressedLength
+ ? TOKEN_NON_COMPRESSED
+ : TOKEN_COMPRESSED;
+
+ frame.putLong(frameOffset + MAGIC_NUMBER_OFFSET, MAGIC_NUMBER );
+ frame.put (frameOffset + TOKEN_OFFSET, token );
+ frame.putInt (frameOffset + COMPRESSED_LENGTH_OFFSET, reverseBytes(compressedLength) );
+ frame.putInt (frameOffset + UNCOMPRESSED_LENGTH_OFFSET, reverseBytes(uncompressedLength));
+ frame.putInt (frameOffset + CHECKSUM_OFFSET, reverseBytes(checksum) );
+ }
+
+ private int calculateMaxFrameLength(ByteBuffer payload)
+ {
+ int payloadLength = payload.remaining();
+ int blockCount = payloadLength / DEFAULT_BLOCK_LENGTH + (payloadLength % DEFAULT_BLOCK_LENGTH != 0 ? 1 : 0);
+ return compressor.maxCompressedLength(payloadLength) + HEADER_LENGTH * blockCount;
+ }
+}
diff --git a/src/java/org/apache/cassandra/net/FrameEncoderUnprotected.java b/src/java/org/apache/cassandra/net/FrameEncoderUnprotected.java
new file mode 100644
index 0000000..3bca41c
--- /dev/null
+++ b/src/java/org/apache/cassandra/net/FrameEncoderUnprotected.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.net;
+
+import java.nio.ByteBuffer;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.channel.ChannelHandler;
+import org.apache.cassandra.utils.memory.BufferPool;
+
+import static org.apache.cassandra.net.FrameEncoderCrc.HEADER_LENGTH;
+import static org.apache.cassandra.net.FrameEncoderCrc.writeHeader;
+
+/**
+ * A frame encoder that writes frames, just without any modification or payload protection.
+ * This is non-standard, and useful for systems that have a trusted transport layer that want
+ * to avoid incurring the (very low) cost of computing a CRC.
+ *
+ * Please see {@link FrameDecoderUnprotected} for description of the framing produced by this encoder.
+ */
+@ChannelHandler.Sharable
+class FrameEncoderUnprotected extends FrameEncoder
+{
+ static final FrameEncoderUnprotected instance = new FrameEncoderUnprotected();
+ static final PayloadAllocator allocator = (isSelfContained, capacity) ->
+ new Payload(isSelfContained, capacity, HEADER_LENGTH, 0);
+
+ PayloadAllocator allocator()
+ {
+ return allocator;
+ }
+
+ ByteBuf encode(boolean isSelfContained, ByteBuffer frame)
+ {
+ try
+ {
+ int frameLength = frame.remaining();
+ int dataLength = frameLength - HEADER_LENGTH;
+ if (dataLength >= 1 << 17)
+ throw new IllegalArgumentException("Maximum uncompressed payload size is 128KiB");
+
+ writeHeader(frame, isSelfContained, dataLength);
+ return GlobalBufferPoolAllocator.wrap(frame);
+ }
+ catch (Throwable t)
+ {
+ BufferPool.put(frame);
+ throw t;
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/net/FutureCombiner.java b/src/java/org/apache/cassandra/net/FutureCombiner.java
new file mode 100644
index 0000000..dd094bd
--- /dev/null
+++ b/src/java/org/apache/cassandra/net/FutureCombiner.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.net;
+
+import java.util.Collection;
+import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
+import java.util.concurrent.atomic.AtomicReferenceFieldUpdater;
+
+import io.netty.util.concurrent.Future;
+import io.netty.util.concurrent.GenericFutureListener;
+import io.netty.util.concurrent.GlobalEventExecutor;
+import io.netty.util.concurrent.Promise;
+
+/**
+ * Netty's PromiseCombiner is not threadsafe, and we combine futures from multiple event executors.
+ *
+ * This class groups a number of Future into a single logical Future, by registering a listener to each that
+ * decrements a shared counter; if any of them fail, the FutureCombiner is completed with the first cause,
+ * but in all scenario only completes when all underlying future have completed (exceptionally or otherwise)
+ *
+ * This Future is always uncancellable.
+ *
+ * We extend FutureDelegate, and simply provide it an uncancellable Promise that will be completed by the listeners
+ * registered to the input futures.
+ */
+class FutureCombiner extends FutureDelegate<Void>
+{
+ private volatile boolean failed;
+
+ private volatile Throwable firstCause;
+ private static final AtomicReferenceFieldUpdater<FutureCombiner, Throwable> firstCauseUpdater =
+ AtomicReferenceFieldUpdater.newUpdater(FutureCombiner.class, Throwable.class, "firstCause");
+
+ private volatile int waitingOn;
+ private static final AtomicIntegerFieldUpdater<FutureCombiner> waitingOnUpdater =
+ AtomicIntegerFieldUpdater.newUpdater(FutureCombiner.class, "waitingOn");
+
+ FutureCombiner(Collection<? extends Future<?>> combine)
+ {
+ this(AsyncPromise.uncancellable(GlobalEventExecutor.INSTANCE), combine);
+ }
+
+ private FutureCombiner(Promise<Void> combined, Collection<? extends Future<?>> combine)
+ {
+ super(combined);
+
+ if (0 == (waitingOn = combine.size()))
+ combined.trySuccess(null);
+
+ GenericFutureListener<? extends Future<Object>> listener = result ->
+ {
+ if (!result.isSuccess())
+ {
+ firstCauseUpdater.compareAndSet(this, null, result.cause());
+ failed = true;
+ }
+
+ if (0 == waitingOnUpdater.decrementAndGet(this))
+ {
+ if (failed)
+ combined.tryFailure(firstCause);
+ else
+ combined.trySuccess(null);
+ }
+ };
+
+ for (Future<?> future : combine)
+ future.addListener(listener);
+ }
+}
diff --git a/src/java/org/apache/cassandra/net/FutureDelegate.java b/src/java/org/apache/cassandra/net/FutureDelegate.java
new file mode 100644
index 0000000..f04a432
--- /dev/null
+++ b/src/java/org/apache/cassandra/net/FutureDelegate.java
@@ -0,0 +1,145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.net;
+
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import io.netty.util.concurrent.Future;
+import io.netty.util.concurrent.GenericFutureListener;
+
+/**
+ * A delegating future, that we can extend to provide subtly modified behaviour.
+ *
+ * See {@link FutureCombiner} and {@link FutureResult}
+ */
+class FutureDelegate<V> implements Future<V>
+{
+ final Future<V> delegate;
+
+ FutureDelegate(Future<V> delegate)
+ {
+ this.delegate = delegate;
+ }
+
+ public boolean isSuccess()
+ {
+ return delegate.isSuccess();
+ }
+
+ public boolean isCancellable()
+ {
+ return delegate.isCancellable();
+ }
+
+ public Throwable cause()
+ {
+ return delegate.cause();
+ }
+
+ public Future<V> addListener(GenericFutureListener<? extends Future<? super V>> genericFutureListener)
+ {
+ return delegate.addListener(genericFutureListener);
+ }
+
+ public Future<V> addListeners(GenericFutureListener<? extends Future<? super V>>... genericFutureListeners)
+ {
+ return delegate.addListeners(genericFutureListeners);
+ }
+
+ public Future<V> removeListener(GenericFutureListener<? extends Future<? super V>> genericFutureListener)
+ {
+ return delegate.removeListener(genericFutureListener);
+ }
+
+ public Future<V> removeListeners(GenericFutureListener<? extends Future<? super V>>... genericFutureListeners)
+ {
+ return delegate.removeListeners(genericFutureListeners);
+ }
+
+ public Future<V> sync() throws InterruptedException
+ {
+ return delegate.sync();
+ }
+
+ public Future<V> syncUninterruptibly()
+ {
+ return delegate.syncUninterruptibly();
+ }
+
+ public Future<V> await() throws InterruptedException
+ {
+ return delegate.await();
+ }
+
+ public Future<V> awaitUninterruptibly()
+ {
+ return delegate.awaitUninterruptibly();
+ }
+
+ public boolean await(long l, TimeUnit timeUnit) throws InterruptedException
+ {
+ return delegate.await(l, timeUnit);
+ }
+
+ public boolean await(long l) throws InterruptedException
+ {
+ return delegate.await(l);
+ }
+
+ public boolean awaitUninterruptibly(long l, TimeUnit timeUnit)
+ {
+ return delegate.awaitUninterruptibly(l, timeUnit);
+ }
+
+ public boolean awaitUninterruptibly(long l)
+ {
+ return delegate.awaitUninterruptibly(l);
+ }
+
+ public V getNow()
+ {
+ return delegate.getNow();
+ }
+
+ public boolean cancel(boolean b)
+ {
+ return delegate.cancel(b);
+ }
+
+ public boolean isCancelled()
+ {
+ return delegate.isCancelled();
+ }
+
+ public boolean isDone()
+ {
+ return delegate.isDone();
+ }
+
+ public V get() throws InterruptedException, ExecutionException
+ {
+ return delegate.get();
+ }
+
+ public V get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException
+ {
+ return delegate.get(timeout, unit);
+ }
+}
diff --git a/src/java/org/apache/cassandra/net/FutureResult.java b/src/java/org/apache/cassandra/net/FutureResult.java
new file mode 100644
index 0000000..8d43dbe
--- /dev/null
+++ b/src/java/org/apache/cassandra/net/FutureResult.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.net;
+
+import io.netty.util.concurrent.Future;
+
+/**
+ * An abstraction for yielding a result performed by an asynchronous task,
+ * for whom we may wish to offer cancellation,
+ * but no other access to the underlying task
+ */
+class FutureResult<V> extends FutureDelegate<V>
+{
+ private final Future<?> tryCancel;
+
+ /**
+ * @param result the Future that will be completed by {@link #cancel}
+ * @param cancel the Future that is performing the work, and to whom any cancellation attempts will be proxied
+ */
+ FutureResult(Future<V> result, Future<?> cancel)
+ {
+ super(result);
+ this.tryCancel = cancel;
+ }
+
+ @Override
+ public boolean cancel(boolean b)
+ {
+ tryCancel.cancel(true);
+ return delegate.cancel(b);
+ }
+}
diff --git a/src/java/org/apache/cassandra/net/GlobalBufferPoolAllocator.java b/src/java/org/apache/cassandra/net/GlobalBufferPoolAllocator.java
new file mode 100644
index 0000000..66cbc9e
--- /dev/null
+++ b/src/java/org/apache/cassandra/net/GlobalBufferPoolAllocator.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.net;
+
+import java.nio.ByteBuffer;
+
+import io.netty.buffer.ByteBuf;
+import org.apache.cassandra.utils.memory.BufferPool;
+
+/**
+ * Primary {@link ByteBuf} / {@link ByteBuffer} allocator - using the global {@link BufferPool}.
+ */
+class GlobalBufferPoolAllocator extends BufferPoolAllocator
+{
+ static final GlobalBufferPoolAllocator instance = new GlobalBufferPoolAllocator();
+
+ private GlobalBufferPoolAllocator()
+ {
+ super();
+ }
+
+ static ByteBuf wrap(ByteBuffer buffer)
+ {
+ return new Wrapped(instance, buffer);
+ }
+}
diff --git a/src/java/org/apache/cassandra/net/HandshakeProtocol.java b/src/java/org/apache/cassandra/net/HandshakeProtocol.java
new file mode 100644
index 0000000..47d0ec6
--- /dev/null
+++ b/src/java/org/apache/cassandra/net/HandshakeProtocol.java
@@ -0,0 +1,414 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.net;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Objects;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.ByteBufAllocator;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.io.compress.BufferType;
+import org.apache.cassandra.io.util.DataInputBuffer;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputBufferFixed;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.utils.memory.BufferPool;
+
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+import static org.apache.cassandra.locator.InetAddressAndPort.Serializer.inetAddressAndPortSerializer;
+import static org.apache.cassandra.net.MessagingService.VERSION_30;
+import static org.apache.cassandra.net.MessagingService.VERSION_40;
+import static org.apache.cassandra.net.Message.validateLegacyProtocolMagic;
+import static org.apache.cassandra.net.Crc.*;
+import static org.apache.cassandra.net.Crc.computeCrc32;
+import static org.apache.cassandra.net.OutboundConnectionSettings.*;
+
+/**
+ * Messages for the handshake phase of the internode protocol.
+ *
+ * The modern handshake is composed of 2 messages: Initiate and Accept
+ * <p>
+ * The legacy handshake is composed of 3 messages, the first being sent by the initiator of the connection. The other
+ * side then answer with the 2nd message. At that point, if a version mismatch is detected by the connection initiator,
+ * it will simply disconnect and reconnect with a more appropriate version. But if the version is acceptable, the connection
+ * initiator sends the third message of the protocol, after which it considers the connection ready.
+ */
+class HandshakeProtocol
+{
+ static final long TIMEOUT_MILLIS = 3 * DatabaseDescriptor.getRpcTimeout(MILLISECONDS);
+
+ /**
+ * The initial message sent when a node creates a new connection to a remote peer. This message contains:
+ * 1) the {@link Message#PROTOCOL_MAGIC} number (4 bytes).
+ * 2) the connection flags (4 bytes), which encodes:
+ * - the version the initiator thinks should be used for the connection (in practice, either the initiator
+ * version if it's the first time we connect to that remote since startup, or the last version known for that
+ * peer otherwise).
+ * - the "mode" of the connection: whether it is for streaming or for messaging.
+ * - whether compression should be used or not (if it is, compression is enabled _after_ the last message of the
+ * handshake has been sent).
+ * 3) the connection initiator's broadcast address
+ * 4) a CRC protecting the message from corruption
+ * <p>
+ * More precisely, connection flags:
+ * <pre>
+ * {@code
+ * 1 1 1 1 1 1 1 1 1 1 2 2 2 2 2 2 2 2 2 2 3 3
+ * 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ * |C C C M C | REQUEST | MIN | MAX |
+ * |A A M O R | VERSION | SUPPORTED | SUPPORTED |
+ * |T T P D C | (DEPRECATED) | VERSION | VERSION |
+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ * }
+ * </pre>
+ * CAT - QOS category, 2 bits: SMALL, LARGE, URGENT, or LEGACY (unset)
+ * CMP - compression enabled bit
+ * MOD - connection mode; if the bit is on, the connection is for streaming; if the bit is off, it is for inter-node messaging.
+ * CRC - crc enabled bit
+ * VERSION - {@link org.apache.cassandra.net.MessagingService#current_version}
+ */
+ static class Initiate
+ {
+ /** Contains the PROTOCOL_MAGIC (int) and the flags (int). */
+ private static final int MIN_LENGTH = 8;
+ private static final int MAX_LENGTH = 12 + InetAddressAndPort.Serializer.MAXIMUM_SIZE;
+
+ @Deprecated // this is ignored by post40 nodes, i.e. if maxMessagingVersion is set
+ final int requestMessagingVersion;
+ // the messagingVersion bounds the sender will accept to initiate a connection;
+ // if the remote peer supports any, the newest supported version will be selected; otherwise the nearest supported version
+ final AcceptVersions acceptVersions;
+ final ConnectionType type;
+ final Framing framing;
+ final InetAddressAndPort from;
+
+ Initiate(int requestMessagingVersion, AcceptVersions acceptVersions, ConnectionType type, Framing framing, InetAddressAndPort from)
+ {
+ this.requestMessagingVersion = requestMessagingVersion;
+ this.acceptVersions = acceptVersions;
+ this.type = type;
+ this.framing = framing;
+ this.from = from;
+ }
+
+ @VisibleForTesting
+ int encodeFlags()
+ {
+ int flags = 0;
+ if (type.isMessaging())
+ flags |= type.twoBitID();
+ if (type.isStreaming())
+ flags |= 1 << 3;
+
+ // framing id is split over 2nd and 4th bits, for backwards compatibility
+ flags |= ((framing.id & 1) << 2) | ((framing.id & 2) << 3);
+ flags |= (requestMessagingVersion << 8);
+
+ if (requestMessagingVersion < VERSION_40 || acceptVersions.max < VERSION_40)
+ return flags; // for testing, permit serializing as though we are pre40
+
+ flags |= (acceptVersions.min << 16);
+ flags |= (acceptVersions.max << 24);
+ return flags;
+ }
+
+ ByteBuf encode()
+ {
+ ByteBuffer buffer = BufferPool.get(MAX_LENGTH, BufferType.OFF_HEAP);
+ try (DataOutputBufferFixed out = new DataOutputBufferFixed(buffer))
+ {
+ out.writeInt(Message.PROTOCOL_MAGIC);
+ out.writeInt(encodeFlags());
+
+ if (requestMessagingVersion >= VERSION_40 && acceptVersions.max >= VERSION_40)
+ {
+ inetAddressAndPortSerializer.serialize(from, out, requestMessagingVersion);
+ out.writeInt(computeCrc32(buffer, 0, buffer.position()));
+ }
+ buffer.flip();
+ return GlobalBufferPoolAllocator.wrap(buffer);
+ }
+ catch (IOException e)
+ {
+ throw new IllegalStateException(e);
+ }
+ }
+
+ static Initiate maybeDecode(ByteBuf buf) throws IOException
+ {
+ if (buf.readableBytes() < MIN_LENGTH)
+ return null;
+
+ ByteBuffer nio = buf.nioBuffer();
+ int start = nio.position();
+ try (DataInputBuffer in = new DataInputBuffer(nio, false))
+ {
+ validateLegacyProtocolMagic(in.readInt());
+ int flags = in.readInt();
+
+ int requestedMessagingVersion = getBits(flags, 8, 8);
+ int minMessagingVersion = getBits(flags, 16, 8);
+ int maxMessagingVersion = getBits(flags, 24, 8);
+ int framingBits = getBits(flags, 2, 1) | (getBits(flags, 4, 1) << 1);
+ Framing framing = Framing.forId(framingBits);
+
+ boolean isStream = getBits(flags, 3, 1) == 1;
+
+ ConnectionType type = isStream
+ ? ConnectionType.STREAMING
+ : ConnectionType.fromId(getBits(flags, 0, 2));
+
+ InetAddressAndPort from = null;
+
+ if (requestedMessagingVersion >= VERSION_40 && maxMessagingVersion >= MessagingService.VERSION_40)
+ {
+ from = inetAddressAndPortSerializer.deserialize(in, requestedMessagingVersion);
+
+ int computed = computeCrc32(nio, start, nio.position());
+ int read = in.readInt();
+ if (read != computed)
+ throw new InvalidCrc(read, computed);
+ }
+
+ buf.skipBytes(nio.position() - start);
+ return new Initiate(requestedMessagingVersion,
+ minMessagingVersion == 0 && maxMessagingVersion == 0
+ ? null : new AcceptVersions(minMessagingVersion, maxMessagingVersion),
+ type, framing, from);
+
+ }
+ catch (EOFException e)
+ {
+ return null;
+ }
+ }
+
+ @VisibleForTesting
+ @Override
+ public boolean equals(Object other)
+ {
+ if (!(other instanceof Initiate))
+ return false;
+
+ Initiate that = (Initiate)other;
+ return this.type == that.type
+ && this.framing == that.framing
+ && this.requestMessagingVersion == that.requestMessagingVersion
+ && Objects.equals(this.acceptVersions, that.acceptVersions);
+ }
+
+ @Override
+ public String toString()
+ {
+ return String.format("Initiate(request: %d, min: %d, max: %d, type: %s, framing: %b, from: %s)",
+ requestMessagingVersion,
+ acceptVersions == null ? requestMessagingVersion : acceptVersions.min,
+ acceptVersions == null ? requestMessagingVersion : acceptVersions.max,
+ type, framing, from);
+ }
+ }
+
+
+ /**
+ * The second message of the handshake, sent by the node receiving the {@link Initiate} back to the
+ * connection initiator.
+ *
+ * This message contains
+ * 1) the messaging version of the peer sending this message
+ * 2) the negotiated messaging version if one could be accepted by both peers,
+ * or if not the closest version that this peer could support to the ones requested
+ * 3) a CRC protectingn the integrity of the message
+ *
+ * Note that the pre40 equivalent of this message contains ONLY the messaging version of the peer.
+ */
+ static class Accept
+ {
+ /** The messaging version sent by the receiving peer (int). */
+ private static final int MAX_LENGTH = 12;
+
+ final int useMessagingVersion;
+ final int maxMessagingVersion;
+
+ Accept(int useMessagingVersion, int maxMessagingVersion)
+ {
+ this.useMessagingVersion = useMessagingVersion;
+ this.maxMessagingVersion = maxMessagingVersion;
+ }
+
+ ByteBuf encode(ByteBufAllocator allocator)
+ {
+ ByteBuf buffer = allocator.directBuffer(MAX_LENGTH);
+ buffer.clear();
+ buffer.writeInt(maxMessagingVersion);
+ buffer.writeInt(useMessagingVersion);
+ buffer.writeInt(computeCrc32(buffer, 0, 8));
+ return buffer;
+ }
+
+ /**
+ * Respond to pre40 nodes only with our current messagingVersion
+ */
+ static ByteBuf respondPre40(int messagingVersion, ByteBufAllocator allocator)
+ {
+ ByteBuf buffer = allocator.directBuffer(4);
+ buffer.clear();
+ buffer.writeInt(messagingVersion);
+ return buffer;
+ }
+
+ static Accept maybeDecode(ByteBuf in, int handshakeMessagingVersion) throws InvalidCrc
+ {
+ int readerIndex = in.readerIndex();
+ if (in.readableBytes() < 4)
+ return null;
+ int maxMessagingVersion = in.readInt();
+ int useMessagingVersion = 0;
+
+ // if the other node is pre-4.0, it will respond only with its maxMessagingVersion
+ if (maxMessagingVersion < VERSION_40 || handshakeMessagingVersion < VERSION_40)
+ return new Accept(useMessagingVersion, maxMessagingVersion);
+
+ if (in.readableBytes() < 8)
+ {
+ in.readerIndex(readerIndex);
+ return null;
+ }
+ useMessagingVersion = in.readInt();
+
+ // verify crc
+ int computed = computeCrc32(in, readerIndex, readerIndex + 8);
+ int read = in.readInt();
+ if (read != computed)
+ throw new InvalidCrc(read, computed);
+
+ return new Accept(useMessagingVersion, maxMessagingVersion);
+ }
+
+ @VisibleForTesting
+ @Override
+ public boolean equals(Object other)
+ {
+ return other instanceof Accept
+ && this.useMessagingVersion == ((Accept) other).useMessagingVersion
+ && this.maxMessagingVersion == ((Accept) other).maxMessagingVersion;
+ }
+
+ @Override
+ public String toString()
+ {
+ return String.format("Accept(use: %d, max: %d)", useMessagingVersion, maxMessagingVersion);
+ }
+ }
+
+ /**
+ * The third message of the handshake, sent by pre40 nodes on reception of {@link Accept}.
+ * This message contains:
+ * 1) The connection initiator's {@link org.apache.cassandra.net.MessagingService#current_version} (4 bytes).
+ * This indicates the max messaging version supported by this node.
+ * 2) The connection initiator's broadcast address as encoded by {@link InetAddressAndPort.Serializer}.
+ * This can be either 7 bytes for an IPv4 address, or 19 bytes for an IPv6 one, post40.
+ * This can be either 5 bytes for an IPv4 address, or 17 bytes for an IPv6 one, pre40.
+ * <p>
+ * This message concludes the legacy handshake protocol.
+ */
+ static class ConfirmOutboundPre40
+ {
+ private static final int MAX_LENGTH = 4 + InetAddressAndPort.Serializer.MAXIMUM_SIZE;
+
+ final int maxMessagingVersion;
+ final InetAddressAndPort from;
+
+ ConfirmOutboundPre40(int maxMessagingVersion, InetAddressAndPort from)
+ {
+ this.maxMessagingVersion = maxMessagingVersion;
+ this.from = from;
+ }
+
+ ByteBuf encode()
+ {
+ ByteBuffer buffer = BufferPool.get(MAX_LENGTH, BufferType.OFF_HEAP);
+ try (DataOutputBufferFixed out = new DataOutputBufferFixed(buffer))
+ {
+ out.writeInt(maxMessagingVersion);
+ // pre-4.0 nodes should only receive the address, never port, and it's ok to hardcode VERSION_30
+ inetAddressAndPortSerializer.serialize(from, out, VERSION_30);
+ buffer.flip();
+ return GlobalBufferPoolAllocator.wrap(buffer);
+ }
+ catch (IOException e)
+ {
+ throw new IllegalStateException(e);
+ }
+ }
+
+ @SuppressWarnings("resource")
+ static ConfirmOutboundPre40 maybeDecode(ByteBuf in)
+ {
+ ByteBuffer nio = in.nioBuffer();
+ int start = nio.position();
+ DataInputPlus input = new DataInputBuffer(nio, false);
+ try
+ {
+ int version = input.readInt();
+ InetAddressAndPort address = inetAddressAndPortSerializer.deserialize(input, version);
+ in.skipBytes(nio.position() - start);
+ return new ConfirmOutboundPre40(version, address);
+ }
+ catch (EOFException e)
+ {
+ // makes the assumption we didn't have enough bytes to deserialize an IPv6 address,
+ // as we only check the MIN_LENGTH of the buf.
+ return null;
+ }
+ catch (IOException e)
+ {
+ throw new IllegalStateException(e);
+ }
+ }
+
+ @VisibleForTesting
+ @Override
+ public boolean equals(Object other)
+ {
+ if (!(other instanceof ConfirmOutboundPre40))
+ return false;
+
+ ConfirmOutboundPre40 that = (ConfirmOutboundPre40) other;
+ return this.maxMessagingVersion == that.maxMessagingVersion
+ && Objects.equals(this.from, that.from);
+ }
+
+ @Override
+ public String toString()
+ {
+ return String.format("ConfirmOutboundPre40(maxMessagingVersion: %d; address: %s)", maxMessagingVersion, from);
+ }
+ }
+
+ private static int getBits(int packed, int start, int count)
+ {
+ return (packed >>> start) & ~(-1 << count);
+ }
+
+}
diff --git a/src/java/org/apache/cassandra/net/IAsyncCallback.java b/src/java/org/apache/cassandra/net/IAsyncCallback.java
deleted file mode 100644
index 7835079..0000000
--- a/src/java/org/apache/cassandra/net/IAsyncCallback.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.net;
-
-import java.net.InetAddress;
-
-import com.google.common.base.Predicate;
-
-import org.apache.cassandra.gms.FailureDetector;
-
-/**
- * implementors of IAsyncCallback need to make sure that any public methods
- * are threadsafe with respect to response() being called from the message
- * service. In particular, if any shared state is referenced, making
- * response alone synchronized will not suffice.
- */
-public interface IAsyncCallback<T>
-{
- Predicate<InetAddress> isAlive = new Predicate<InetAddress>()
- {
- public boolean apply(InetAddress endpoint)
- {
- return FailureDetector.instance.isAlive(endpoint);
- }
- };
-
- /**
- * @param msg response received.
- */
- void response(MessageIn<T> msg);
-
- /**
- * @return true if this callback is on the read path and its latency should be
- * given as input to the dynamic snitch.
- */
- boolean isLatencyForSnitch();
-
- default boolean supportsBackPressure()
- {
- return false;
- }
-}
diff --git a/src/java/org/apache/cassandra/net/IAsyncCallbackWithFailure.java b/src/java/org/apache/cassandra/net/IAsyncCallbackWithFailure.java
deleted file mode 100644
index 1cd27b6..0000000
--- a/src/java/org/apache/cassandra/net/IAsyncCallbackWithFailure.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.net;
-
-import java.net.InetAddress;
-
-import org.apache.cassandra.exceptions.RequestFailureReason;
-
-public interface IAsyncCallbackWithFailure<T> extends IAsyncCallback<T>
-{
-
- /**
- * Called when there is an exception on the remote node or timeout happens
- */
- void onFailure(InetAddress from, RequestFailureReason failureReason);
-}
diff --git a/src/java/org/apache/cassandra/net/IMessageSink.java b/src/java/org/apache/cassandra/net/IMessageSink.java
deleted file mode 100644
index 5150901..0000000
--- a/src/java/org/apache/cassandra/net/IMessageSink.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.net;
-
-import java.net.InetAddress;
-
-public interface IMessageSink
-{
- /**
- * Allow or drop an outgoing message
- *
- * @return true if the message is allowed, false if it should be dropped
- */
- boolean allowOutgoingMessage(MessageOut message, int id, InetAddress to);
-
- /**
- * Allow or drop an incoming message
- *
- * @return true if the message is allowed, false if it should be dropped
- */
- boolean allowIncomingMessage(MessageIn message, int id);
-}
diff --git a/src/java/org/apache/cassandra/net/IVerbHandler.java b/src/java/org/apache/cassandra/net/IVerbHandler.java
index 0995a68..ac0efe7 100644
--- a/src/java/org/apache/cassandra/net/IVerbHandler.java
+++ b/src/java/org/apache/cassandra/net/IVerbHandler.java
@@ -24,7 +24,6 @@
* The concrete implementation of this interface would provide the functionality
* for a given verb.
*/
-
public interface IVerbHandler<T>
{
/**
@@ -34,7 +33,6 @@
* because the implementation may be synchronized.
*
* @param message - incoming message that needs handling.
- * @param id
*/
- void doVerb(MessageIn<T> message, int id) throws IOException;
+ void doVerb(Message<T> message) throws IOException;
}
diff --git a/src/java/org/apache/cassandra/net/InboundConnectionInitiator.java b/src/java/org/apache/cassandra/net/InboundConnectionInitiator.java
new file mode 100644
index 0000000..4ad3d8c
--- /dev/null
+++ b/src/java/org/apache/cassandra/net/InboundConnectionInitiator.java
@@ -0,0 +1,495 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.net;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.net.SocketAddress;
+import java.util.List;
+import java.util.concurrent.Future;
+import java.util.function.Consumer;
+
+import javax.net.ssl.SSLSession;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import io.netty.bootstrap.ServerBootstrap;
+import io.netty.buffer.ByteBuf;
+import io.netty.channel.Channel;
+import io.netty.channel.ChannelFuture;
+import io.netty.channel.ChannelFutureListener;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.channel.ChannelInitializer;
+import io.netty.channel.ChannelOption;
+import io.netty.channel.ChannelPipeline;
+import io.netty.channel.group.ChannelGroup;
+import io.netty.channel.socket.SocketChannel;
+import io.netty.handler.codec.ByteToMessageDecoder;
+import io.netty.handler.logging.LogLevel;
+import io.netty.handler.logging.LoggingHandler;
+import io.netty.handler.ssl.SslContext;
+import io.netty.handler.ssl.SslHandler;
+import org.apache.cassandra.config.EncryptionOptions;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.net.OutboundConnectionSettings.Framing;
+import org.apache.cassandra.security.SSLFactory;
+import org.apache.cassandra.streaming.async.StreamingInboundHandler;
+import org.apache.cassandra.utils.memory.BufferPool;
+
+import static java.lang.Math.*;
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+import static org.apache.cassandra.net.MessagingService.*;
+import static org.apache.cassandra.net.MessagingService.VERSION_40;
+import static org.apache.cassandra.net.MessagingService.current_version;
+import static org.apache.cassandra.net.MessagingService.minimum_version;
+import static org.apache.cassandra.net.SocketFactory.WIRETRACE;
+import static org.apache.cassandra.net.SocketFactory.encryptionLogStatement;
+import static org.apache.cassandra.net.SocketFactory.newSslHandler;
+
+public class InboundConnectionInitiator
+{
+ private static final Logger logger = LoggerFactory.getLogger(InboundConnectionInitiator.class);
+
+ private static class Initializer extends ChannelInitializer<SocketChannel>
+ {
+ private final InboundConnectionSettings settings;
+ private final ChannelGroup channelGroup;
+ private final Consumer<ChannelPipeline> pipelineInjector;
+
+ Initializer(InboundConnectionSettings settings, ChannelGroup channelGroup,
+ Consumer<ChannelPipeline> pipelineInjector)
+ {
+ this.settings = settings;
+ this.channelGroup = channelGroup;
+ this.pipelineInjector = pipelineInjector;
+ }
+
+ @Override
+ public void initChannel(SocketChannel channel) throws Exception
+ {
+ channelGroup.add(channel);
+
+ channel.config().setOption(ChannelOption.ALLOCATOR, GlobalBufferPoolAllocator.instance);
+ channel.config().setOption(ChannelOption.SO_KEEPALIVE, true);
+ channel.config().setOption(ChannelOption.SO_REUSEADDR, true);
+ channel.config().setOption(ChannelOption.TCP_NODELAY, true); // we only send handshake messages; no point ever delaying
+
+ ChannelPipeline pipeline = channel.pipeline();
+
+ pipelineInjector.accept(pipeline);
+
+ // order of handlers: ssl -> logger -> handshakeHandler
+ // For either unencrypted or transitional modes, allow Ssl optionally.
+ if (settings.encryption.optional)
+ {
+ pipeline.addFirst("ssl", new OptionalSslHandler(settings.encryption));
+ }
+ else
+ {
+ SslContext sslContext = SSLFactory.getOrCreateSslContext(settings.encryption, true, SSLFactory.SocketType.SERVER);
+ InetSocketAddress peer = settings.encryption.require_endpoint_verification ? channel.remoteAddress() : null;
+ SslHandler sslHandler = newSslHandler(channel, sslContext, peer);
+ logger.trace("creating inbound netty SslContext: context={}, engine={}", sslContext.getClass().getName(), sslHandler.engine().getClass().getName());
+ pipeline.addFirst("ssl", sslHandler);
+ }
+
+ if (WIRETRACE)
+ pipeline.addLast("logger", new LoggingHandler(LogLevel.INFO));
+
+ channel.pipeline().addLast("handshake", new Handler(settings));
+
+ }
+ }
+
+ /**
+ * Create a {@link Channel} that listens on the {@code localAddr}. This method will block while trying to bind to the address,
+ * but it does not make a remote call.
+ */
+ private static ChannelFuture bind(Initializer initializer) throws ConfigurationException
+ {
+ logger.info("Listening on {}", initializer.settings);
+
+ ServerBootstrap bootstrap = initializer.settings.socketFactory
+ .newServerBootstrap()
+ .option(ChannelOption.SO_BACKLOG, 1 << 9)
+ .option(ChannelOption.ALLOCATOR, GlobalBufferPoolAllocator.instance)
+ .option(ChannelOption.SO_REUSEADDR, true)
+ .childHandler(initializer);
+
+ int socketReceiveBufferSizeInBytes = initializer.settings.socketReceiveBufferSizeInBytes;
+ if (socketReceiveBufferSizeInBytes > 0)
+ bootstrap.childOption(ChannelOption.SO_RCVBUF, socketReceiveBufferSizeInBytes);
+
+ InetAddressAndPort bind = initializer.settings.bindAddress;
+ ChannelFuture channelFuture = bootstrap.bind(new InetSocketAddress(bind.address, bind.port));
+
+ if (!channelFuture.awaitUninterruptibly().isSuccess())
+ {
+ if (channelFuture.channel().isOpen())
+ channelFuture.channel().close();
+
+ Throwable failedChannelCause = channelFuture.cause();
+
+ String causeString = "";
+ if (failedChannelCause != null && failedChannelCause.getMessage() != null)
+ causeString = failedChannelCause.getMessage();
+
+ if (causeString.contains("in use"))
+ {
+ throw new ConfigurationException(bind + " is in use by another process. Change listen_address:storage_port " +
+ "in cassandra.yaml to values that do not conflict with other services");
+ }
+ // looking at the jdk source, solaris/windows bind failue messages both use the phrase "cannot assign requested address".
+ // windows message uses "Cannot" (with a capital 'C'), and solaris (a/k/a *nux) doe not. hence we search for "annot" <sigh>
+ else if (causeString.contains("annot assign requested address"))
+ {
+ throw new ConfigurationException("Unable to bind to address " + bind
+ + ". Set listen_address in cassandra.yaml to an interface you can bind to, e.g., your private IP address on EC2");
+ }
+ else
+ {
+ throw new ConfigurationException("failed to bind to: " + bind, failedChannelCause);
+ }
+ }
+
+ return channelFuture;
+ }
+
+ public static ChannelFuture bind(InboundConnectionSettings settings, ChannelGroup channelGroup,
+ Consumer<ChannelPipeline> pipelineInjector)
+ {
+ return bind(new Initializer(settings, channelGroup, pipelineInjector));
+ }
+
+ /**
+ * 'Server-side' component that negotiates the internode handshake when establishing a new connection.
+ * This handler will be the first in the netty channel for each incoming connection (secure socket (TLS) notwithstanding),
+ * and once the handshake is successful, it will configure the proper handlers ({@link InboundMessageHandler}
+ * or {@link StreamingInboundHandler}) and remove itself from the working pipeline.
+ */
+ static class Handler extends ByteToMessageDecoder
+ {
+ private final InboundConnectionSettings settings;
+
+ private HandshakeProtocol.Initiate initiate;
+ private HandshakeProtocol.ConfirmOutboundPre40 confirmOutboundPre40;
+
+ /**
+ * A future the essentially places a timeout on how long we'll wait for the peer
+ * to complete the next step of the handshake.
+ */
+ private Future<?> handshakeTimeout;
+
+ Handler(InboundConnectionSettings settings)
+ {
+ this.settings = settings;
+ }
+
+ /**
+ * On registration, immediately schedule a timeout to kill this connection if it does not handshake promptly,
+ * and authenticate the remote address.
+ */
+ public void handlerAdded(ChannelHandlerContext ctx) throws Exception
+ {
+ handshakeTimeout = ctx.executor().schedule(() -> {
+ logger.error("Timeout handshaking with {} (on {})", SocketFactory.addressId(initiate.from, (InetSocketAddress) ctx.channel().remoteAddress()), settings.bindAddress);
+ failHandshake(ctx);
+ }, HandshakeProtocol.TIMEOUT_MILLIS, MILLISECONDS);
+
+ logSsl(ctx);
+ authenticate(ctx.channel().remoteAddress());
+ }
+
+ private void authenticate(SocketAddress socketAddress) throws IOException
+ {
+ if (socketAddress.getClass().getSimpleName().equals("EmbeddedSocketAddress"))
+ return;
+
+ if (!(socketAddress instanceof InetSocketAddress))
+ throw new IOException(String.format("Unexpected SocketAddress type: %s, %s", socketAddress.getClass(), socketAddress));
+
+ InetSocketAddress addr = (InetSocketAddress)socketAddress;
+ if (!settings.authenticate(addr.getAddress(), addr.getPort()))
+ throw new IOException("Authentication failure for inbound connection from peer " + addr);
+ }
+
+ private void logSsl(ChannelHandlerContext ctx)
+ {
+ SslHandler sslHandler = ctx.pipeline().get(SslHandler.class);
+ if (sslHandler != null)
+ {
+ SSLSession session = sslHandler.engine().getSession();
+ logger.info("connection from peer {} to {}, protocol = {}, cipher suite = {}",
+ ctx.channel().remoteAddress(), ctx.channel().localAddress(), session.getProtocol(), session.getCipherSuite());
+ }
+ }
+
+ @Override
+ protected void decode(ChannelHandlerContext ctx, ByteBuf in, List<Object> out) throws Exception
+ {
+ if (initiate == null) initiate(ctx, in);
+ else if (initiate.acceptVersions == null && confirmOutboundPre40 == null) confirmPre40(ctx, in);
+ else throw new IllegalStateException("Should no longer be on pipeline");
+ }
+
+ void initiate(ChannelHandlerContext ctx, ByteBuf in) throws IOException
+ {
+ initiate = HandshakeProtocol.Initiate.maybeDecode(in);
+ if (initiate == null)
+ return;
+
+ logger.trace("Received handshake initiation message from peer {}, message = {}", ctx.channel().remoteAddress(), initiate);
+ if (initiate.acceptVersions != null)
+ {
+ logger.trace("Connection version {} (min {}) from {}", initiate.acceptVersions.max, initiate.acceptVersions.min, initiate.from);
+
+ final AcceptVersions accept;
+
+ if (initiate.type.isStreaming())
+ accept = settings.acceptStreaming;
+ else
+ accept = settings.acceptMessaging;
+
+ int useMessagingVersion = max(accept.min, min(accept.max, initiate.acceptVersions.max));
+ ByteBuf flush = new HandshakeProtocol.Accept(useMessagingVersion, accept.max).encode(ctx.alloc());
+
+ AsyncChannelPromise.writeAndFlush(ctx, flush, (ChannelFutureListener) future -> {
+ if (!future.isSuccess())
+ exceptionCaught(future.channel(), future.cause());
+ });
+
+ if (initiate.acceptVersions.min > accept.max)
+ {
+ logger.info("peer {} only supports messaging versions higher ({}) than this node supports ({})", ctx.channel().remoteAddress(), initiate.acceptVersions.min, current_version);
+ failHandshake(ctx);
+ }
+ else if (initiate.acceptVersions.max < accept.min)
+ {
+ logger.info("peer {} only supports messaging versions lower ({}) than this node supports ({})", ctx.channel().remoteAddress(), initiate.acceptVersions.max, minimum_version);
+ failHandshake(ctx);
+ }
+ else
+ {
+ if (initiate.type.isStreaming())
+ setupStreamingPipeline(initiate.from, ctx);
+ else
+ setupMessagingPipeline(initiate.from, useMessagingVersion, initiate.acceptVersions.max, ctx.pipeline());
+ }
+ }
+ else
+ {
+ int version = initiate.requestMessagingVersion;
+ assert version < VERSION_40 && version >= settings.acceptMessaging.min;
+ logger.trace("Connection version {} from {}", version, ctx.channel().remoteAddress());
+
+ if (initiate.type.isStreaming())
+ {
+ // streaming connections are per-session and have a fixed version. we can't do anything with a wrong-version stream connection, so drop it.
+ if (version != settings.acceptStreaming.max)
+ {
+ logger.warn("Received stream using protocol version {} (my version {}). Terminating connection", version, settings.acceptStreaming.max);
+ failHandshake(ctx);
+ }
+ setupStreamingPipeline(initiate.from, ctx);
+ }
+ else
+ {
+ // if this version is < the MS version the other node is trying
+ // to connect with, the other node will disconnect
+ ByteBuf response = HandshakeProtocol.Accept.respondPre40(settings.acceptMessaging.max, ctx.alloc());
+ AsyncChannelPromise.writeAndFlush(ctx, response,
+ (ChannelFutureListener) future -> {
+ if (!future.isSuccess())
+ exceptionCaught(future.channel(), future.cause());
+ });
+
+ if (version < VERSION_30)
+ throw new IOException(String.format("Unable to read obsolete message version %s from %s; The earliest version supported is 3.0.0", version, ctx.channel().remoteAddress()));
+
+ // we don't setup the messaging pipeline here, as the legacy messaging handshake requires one more message to finish
+ }
+ }
+ }
+
+ /**
+ * Handles the third (and last) message in the internode messaging handshake protocol for pre40 nodes.
+ * Grabs the protocol version and IP addr the peer wants to use.
+ */
+ @VisibleForTesting
+ void confirmPre40(ChannelHandlerContext ctx, ByteBuf in)
+ {
+ confirmOutboundPre40 = HandshakeProtocol.ConfirmOutboundPre40.maybeDecode(in);
+ if (confirmOutboundPre40 == null)
+ return;
+
+ logger.trace("Received third handshake message from peer {}, message = {}", ctx.channel().remoteAddress(), confirmOutboundPre40);
+ setupMessagingPipeline(confirmOutboundPre40.from, initiate.requestMessagingVersion, confirmOutboundPre40.maxMessagingVersion, ctx.pipeline());
+ }
+
+ @Override
+ public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause)
+ {
+ exceptionCaught(ctx.channel(), cause);
+ }
+
+ private void exceptionCaught(Channel channel, Throwable cause)
+ {
+ logger.error("Failed to properly handshake with peer {}. Closing the channel.", channel.remoteAddress(), cause);
+ try
+ {
+ failHandshake(channel);
+ }
+ catch (Throwable t)
+ {
+ logger.error("Unexpected exception in {}.exceptionCaught", this.getClass().getSimpleName(), t);
+ }
+ }
+
+ private void failHandshake(ChannelHandlerContext ctx)
+ {
+ failHandshake(ctx.channel());
+ }
+
+ private void failHandshake(Channel channel)
+ {
+ channel.close();
+ if (handshakeTimeout != null)
+ handshakeTimeout.cancel(true);
+ }
+
+ private void setupStreamingPipeline(InetAddressAndPort from, ChannelHandlerContext ctx)
+ {
+ handshakeTimeout.cancel(true);
+ assert initiate.framing == Framing.UNPROTECTED;
+
+ ChannelPipeline pipeline = ctx.pipeline();
+ Channel channel = ctx.channel();
+
+ if (from == null)
+ {
+ InetSocketAddress address = (InetSocketAddress) channel.remoteAddress();
+ from = InetAddressAndPort.getByAddressOverrideDefaults(address.getAddress(), address.getPort());
+ }
+
+ BufferPool.setRecycleWhenFreeForCurrentThread(false);
+ pipeline.replace(this, "streamInbound", new StreamingInboundHandler(from, current_version, null));
+ }
+
+ @VisibleForTesting
+ void setupMessagingPipeline(InetAddressAndPort from, int useMessagingVersion, int maxMessagingVersion, ChannelPipeline pipeline)
+ {
+ handshakeTimeout.cancel(true);
+ // record the "true" endpoint, i.e. the one the peer is identified with, as opposed to the socket it connected over
+ instance().versions.set(from, maxMessagingVersion);
+
+ BufferPool.setRecycleWhenFreeForCurrentThread(false);
+ BufferPoolAllocator allocator = GlobalBufferPoolAllocator.instance;
+ if (initiate.type == ConnectionType.LARGE_MESSAGES)
+ {
+ // for large messages, swap the global pool allocator for a local one, to optimise utilisation of chunks
+ allocator = new LocalBufferPoolAllocator(pipeline.channel().eventLoop());
+ pipeline.channel().config().setAllocator(allocator);
+ }
+
+ FrameDecoder frameDecoder;
+ switch (initiate.framing)
+ {
+ case LZ4:
+ {
+ if (useMessagingVersion >= VERSION_40)
+ frameDecoder = FrameDecoderLZ4.fast(allocator);
+ else
+ frameDecoder = new FrameDecoderLegacyLZ4(allocator, useMessagingVersion);
+ break;
+ }
+ case CRC:
+ {
+ if (useMessagingVersion >= VERSION_40)
+ {
+ frameDecoder = FrameDecoderCrc.create(allocator);
+ break;
+ }
+ }
+ case UNPROTECTED:
+ {
+ if (useMessagingVersion >= VERSION_40)
+ frameDecoder = new FrameDecoderUnprotected(allocator);
+ else
+ frameDecoder = new FrameDecoderLegacy(allocator, useMessagingVersion);
+ break;
+ }
+ default:
+ throw new AssertionError();
+ }
+
+ frameDecoder.addLastTo(pipeline);
+
+ InboundMessageHandler handler =
+ settings.handlers.apply(from).createHandler(frameDecoder, initiate.type, pipeline.channel(), useMessagingVersion);
+
+ logger.info("{} connection established, version = {}, framing = {}, encryption = {}",
+ handler.id(true),
+ useMessagingVersion,
+ initiate.framing,
+ pipeline.get("ssl") != null ? encryptionLogStatement(settings.encryption) : "disabled");
+
+ pipeline.addLast("deserialize", handler);
+
+ pipeline.remove(this);
+ }
+ }
+
+ private static class OptionalSslHandler extends ByteToMessageDecoder
+ {
+ private final EncryptionOptions.ServerEncryptionOptions encryptionOptions;
+
+ OptionalSslHandler(EncryptionOptions.ServerEncryptionOptions encryptionOptions)
+ {
+ this.encryptionOptions = encryptionOptions;
+ }
+
+ protected void decode(ChannelHandlerContext ctx, ByteBuf in, List<Object> out) throws Exception
+ {
+ if (in.readableBytes() < 5)
+ {
+ // To detect if SSL must be used we need to have at least 5 bytes, so return here and try again
+ // once more bytes a ready.
+ return;
+ }
+
+ if (SslHandler.isEncrypted(in))
+ {
+ // Connection uses SSL/TLS, replace the detection handler with a SslHandler and so use encryption.
+ SslContext sslContext = SSLFactory.getOrCreateSslContext(encryptionOptions, true, SSLFactory.SocketType.SERVER);
+ Channel channel = ctx.channel();
+ InetSocketAddress peer = encryptionOptions.require_endpoint_verification ? (InetSocketAddress) channel.remoteAddress() : null;
+ SslHandler sslHandler = newSslHandler(channel, sslContext, peer);
+ ctx.pipeline().replace(this, "ssl", sslHandler);
+ }
+ else
+ {
+ // Connection use no TLS/SSL encryption, just remove the detection handler and continue without
+ // SslHandler in the pipeline.
+ ctx.pipeline().remove(this);
+ }
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/net/InboundConnectionSettings.java b/src/java/org/apache/cassandra/net/InboundConnectionSettings.java
new file mode 100644
index 0000000..a07395b
--- /dev/null
+++ b/src/java/org/apache/cassandra/net/InboundConnectionSettings.java
@@ -0,0 +1,213 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.net;
+
+import java.net.InetAddress;
+import java.util.function.Function;
+
+import com.google.common.base.Preconditions;
+
+import org.apache.cassandra.auth.IInternodeAuthenticator;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.config.EncryptionOptions.ServerEncryptionOptions;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.utils.FBUtilities;
+
+import static java.lang.String.format;
+import static org.apache.cassandra.net.MessagingService.*;
+
+public class InboundConnectionSettings
+{
+ public final IInternodeAuthenticator authenticator;
+ public final InetAddressAndPort bindAddress;
+ public final ServerEncryptionOptions encryption;
+ public final Integer socketReceiveBufferSizeInBytes;
+ public final Integer applicationReceiveQueueCapacityInBytes;
+ public final AcceptVersions acceptMessaging;
+ public final AcceptVersions acceptStreaming;
+ public final SocketFactory socketFactory;
+ public final Function<InetAddressAndPort, InboundMessageHandlers> handlers;
+
+ private InboundConnectionSettings(IInternodeAuthenticator authenticator,
+ InetAddressAndPort bindAddress,
+ ServerEncryptionOptions encryption,
+ Integer socketReceiveBufferSizeInBytes,
+ Integer applicationReceiveQueueCapacityInBytes,
+ AcceptVersions acceptMessaging,
+ AcceptVersions acceptStreaming,
+ SocketFactory socketFactory,
+ Function<InetAddressAndPort, InboundMessageHandlers> handlers)
+ {
+ this.authenticator = authenticator;
+ this.bindAddress = bindAddress;
+ this.encryption = encryption;
+ this.socketReceiveBufferSizeInBytes = socketReceiveBufferSizeInBytes;
+ this.applicationReceiveQueueCapacityInBytes = applicationReceiveQueueCapacityInBytes;
+ this.acceptMessaging = acceptMessaging;
+ this.acceptStreaming = acceptStreaming;
+ this.socketFactory = socketFactory;
+ this.handlers = handlers;
+ }
+
+ public InboundConnectionSettings()
+ {
+ this(null, null, null, null, null, null, null, null, null);
+ }
+
+ public boolean authenticate(InetAddressAndPort endpoint)
+ {
+ return authenticator.authenticate(endpoint.address, endpoint.port);
+ }
+
+ public boolean authenticate(InetAddress address, int port)
+ {
+ return authenticator.authenticate(address, port);
+ }
+
+ public String toString()
+ {
+ return format("address: (%s), nic: %s, encryption: %s",
+ bindAddress, FBUtilities.getNetworkInterface(bindAddress.address), SocketFactory.encryptionLogStatement(encryption));
+ }
+
+ public InboundConnectionSettings withAuthenticator(IInternodeAuthenticator authenticator)
+ {
+ return new InboundConnectionSettings(authenticator, bindAddress, encryption,
+ socketReceiveBufferSizeInBytes, applicationReceiveQueueCapacityInBytes,
+ acceptMessaging, acceptStreaming, socketFactory, handlers);
+ }
+
+ @SuppressWarnings("unused")
+ public InboundConnectionSettings withBindAddress(InetAddressAndPort bindAddress)
+ {
+ return new InboundConnectionSettings(authenticator, bindAddress, encryption,
+ socketReceiveBufferSizeInBytes, applicationReceiveQueueCapacityInBytes,
+ acceptMessaging, acceptStreaming, socketFactory, handlers);
+ }
+
+ public InboundConnectionSettings withEncryption(ServerEncryptionOptions encryption)
+ {
+ return new InboundConnectionSettings(authenticator, bindAddress, encryption,
+ socketReceiveBufferSizeInBytes, applicationReceiveQueueCapacityInBytes,
+ acceptMessaging, acceptStreaming, socketFactory, handlers);
+ }
+
+ public InboundConnectionSettings withSocketReceiveBufferSizeInBytes(int socketReceiveBufferSizeInBytes)
+ {
+ return new InboundConnectionSettings(authenticator, bindAddress, encryption,
+ socketReceiveBufferSizeInBytes, applicationReceiveQueueCapacityInBytes,
+ acceptMessaging, acceptStreaming, socketFactory, handlers);
+ }
+
+ @SuppressWarnings("unused")
+ public InboundConnectionSettings withApplicationReceiveQueueCapacityInBytes(int applicationReceiveQueueCapacityInBytes)
+ {
+ return new InboundConnectionSettings(authenticator, bindAddress, encryption,
+ socketReceiveBufferSizeInBytes, applicationReceiveQueueCapacityInBytes,
+ acceptMessaging, acceptStreaming, socketFactory, handlers);
+ }
+
+ public InboundConnectionSettings withAcceptMessaging(AcceptVersions acceptMessaging)
+ {
+ return new InboundConnectionSettings(authenticator, bindAddress, encryption,
+ socketReceiveBufferSizeInBytes, applicationReceiveQueueCapacityInBytes,
+ acceptMessaging, acceptStreaming, socketFactory, handlers);
+ }
+
+ public InboundConnectionSettings withAcceptStreaming(AcceptVersions acceptMessaging)
+ {
+ return new InboundConnectionSettings(authenticator, bindAddress, encryption,
+ socketReceiveBufferSizeInBytes, applicationReceiveQueueCapacityInBytes,
+ acceptMessaging, acceptStreaming, socketFactory, handlers);
+ }
+
+ public InboundConnectionSettings withSocketFactory(SocketFactory socketFactory)
+ {
+ return new InboundConnectionSettings(authenticator, bindAddress, encryption,
+ socketReceiveBufferSizeInBytes, applicationReceiveQueueCapacityInBytes,
+ acceptMessaging, acceptStreaming, socketFactory, handlers);
+ }
+
+ public InboundConnectionSettings withHandlers(Function<InetAddressAndPort, InboundMessageHandlers> handlers)
+ {
+ return new InboundConnectionSettings(authenticator, bindAddress, encryption,
+ socketReceiveBufferSizeInBytes, applicationReceiveQueueCapacityInBytes,
+ acceptMessaging, acceptStreaming, socketFactory, handlers);
+ }
+
+ public InboundConnectionSettings withLegacyDefaults()
+ {
+ ServerEncryptionOptions encryption = this.encryption;
+ if (encryption == null)
+ encryption = DatabaseDescriptor.getInternodeMessagingEncyptionOptions();
+ encryption = encryption.withOptional(false);
+
+ return this.withBindAddress(bindAddress.withPort(DatabaseDescriptor.getSSLStoragePort()))
+ .withEncryption(encryption)
+ .withDefaults();
+ }
+
+ // note that connectTo is updated even if specified, in the case of pre40 messaging and using encryption (to update port)
+ public InboundConnectionSettings withDefaults()
+ {
+ // this is for the socket that can be plain, only ssl, or optional plain/ssl
+ if (bindAddress.port != DatabaseDescriptor.getStoragePort() && bindAddress.port != DatabaseDescriptor.getSSLStoragePort())
+ throw new ConfigurationException(format("Local endpoint port %d doesn't match YAML configured port %d or legacy SSL port %d",
+ bindAddress.port, DatabaseDescriptor.getStoragePort(), DatabaseDescriptor.getSSLStoragePort()));
+
+ IInternodeAuthenticator authenticator = this.authenticator;
+ ServerEncryptionOptions encryption = this.encryption;
+ Integer socketReceiveBufferSizeInBytes = this.socketReceiveBufferSizeInBytes;
+ Integer applicationReceiveQueueCapacityInBytes = this.applicationReceiveQueueCapacityInBytes;
+ AcceptVersions acceptMessaging = this.acceptMessaging;
+ AcceptVersions acceptStreaming = this.acceptStreaming;
+ SocketFactory socketFactory = this.socketFactory;
+ Function<InetAddressAndPort, InboundMessageHandlers> handlersFactory = this.handlers;
+
+ if (authenticator == null)
+ authenticator = DatabaseDescriptor.getInternodeAuthenticator();
+
+ if (encryption == null)
+ encryption = DatabaseDescriptor.getInternodeMessagingEncyptionOptions();
+
+ if (socketReceiveBufferSizeInBytes == null)
+ socketReceiveBufferSizeInBytes = DatabaseDescriptor.getInternodeSocketReceiveBufferSizeInBytes();
+
+ if (applicationReceiveQueueCapacityInBytes == null)
+ applicationReceiveQueueCapacityInBytes = DatabaseDescriptor.getInternodeApplicationReceiveQueueCapacityInBytes();
+
+ if (acceptMessaging == null)
+ acceptMessaging = accept_messaging;
+
+ if (acceptStreaming == null)
+ acceptStreaming = accept_streaming;
+
+ if (socketFactory == null)
+ socketFactory = instance().socketFactory;
+
+ if (handlersFactory == null)
+ handlersFactory = instance()::getInbound;
+
+ Preconditions.checkArgument(socketReceiveBufferSizeInBytes == 0 || socketReceiveBufferSizeInBytes >= 1 << 10, "illegal socket send buffer size: " + socketReceiveBufferSizeInBytes);
+ Preconditions.checkArgument(applicationReceiveQueueCapacityInBytes >= 1 << 10, "illegal application receive queue capacity: " + applicationReceiveQueueCapacityInBytes);
+
+ return new InboundConnectionSettings(authenticator, bindAddress, encryption, socketReceiveBufferSizeInBytes, applicationReceiveQueueCapacityInBytes, acceptMessaging, acceptStreaming, socketFactory, handlersFactory);
+ }
+}
diff --git a/src/java/org/apache/cassandra/net/InboundCounters.java b/src/java/org/apache/cassandra/net/InboundCounters.java
new file mode 100644
index 0000000..da035f2
--- /dev/null
+++ b/src/java/org/apache/cassandra/net/InboundCounters.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.net;
+
+import java.util.concurrent.atomic.AtomicLongFieldUpdater;
+
+/**
+ * Aggregates counters for (from, connection type) for the duration of host uptime.
+ *
+ * If contention/false sharing ever become a problem, consider introducing padding.
+ */
+class InboundCounters
+{
+ private volatile long errorCount;
+ private volatile long errorBytes;
+
+ private static final AtomicLongFieldUpdater<InboundCounters> errorCountUpdater =
+ AtomicLongFieldUpdater.newUpdater(InboundCounters.class, "errorCount");
+ private static final AtomicLongFieldUpdater<InboundCounters> errorBytesUpdater =
+ AtomicLongFieldUpdater.newUpdater(InboundCounters.class, "errorBytes");
+
+ void addError(int bytes)
+ {
+ errorCountUpdater.incrementAndGet(this);
+ errorBytesUpdater.addAndGet(this, bytes);
+ }
+
+ long errorCount()
+ {
+ return errorCount;
+ }
+
+ long errorBytes()
+ {
+ return errorBytes;
+ }
+
+ private volatile long expiredCount;
+ private volatile long expiredBytes;
+
+ private static final AtomicLongFieldUpdater<InboundCounters> expiredCountUpdater =
+ AtomicLongFieldUpdater.newUpdater(InboundCounters.class, "expiredCount");
+ private static final AtomicLongFieldUpdater<InboundCounters> expiredBytesUpdater =
+ AtomicLongFieldUpdater.newUpdater(InboundCounters.class, "expiredBytes");
+
+ void addExpired(int bytes)
+ {
+ expiredCountUpdater.incrementAndGet(this);
+ expiredBytesUpdater.addAndGet(this, bytes);
+ }
+
+ long expiredCount()
+ {
+ return expiredCount;
+ }
+
+ long expiredBytes()
+ {
+ return expiredBytes;
+ }
+
+ private volatile long processedCount;
+ private volatile long processedBytes;
+
+ private static final AtomicLongFieldUpdater<InboundCounters> processedCountUpdater =
+ AtomicLongFieldUpdater.newUpdater(InboundCounters.class, "processedCount");
+ private static final AtomicLongFieldUpdater<InboundCounters> processedBytesUpdater =
+ AtomicLongFieldUpdater.newUpdater(InboundCounters.class, "processedBytes");
+
+ void addProcessed(int bytes)
+ {
+ processedCountUpdater.incrementAndGet(this);
+ processedBytesUpdater.addAndGet(this, bytes);
+ }
+
+ long processedCount()
+ {
+ return processedCount;
+ }
+
+ long processedBytes()
+ {
+ return processedBytes;
+ }
+
+ private volatile long scheduledCount;
+ private volatile long scheduledBytes;
+
+ private static final AtomicLongFieldUpdater<InboundCounters> scheduledCountUpdater =
+ AtomicLongFieldUpdater.newUpdater(InboundCounters.class, "scheduledCount");
+ private static final AtomicLongFieldUpdater<InboundCounters> scheduledBytesUpdater =
+ AtomicLongFieldUpdater.newUpdater(InboundCounters.class, "scheduledBytes");
+
+ void addPending(int bytes)
+ {
+ scheduledCountUpdater.incrementAndGet(this);
+ scheduledBytesUpdater.addAndGet(this, bytes);
+ }
+
+ void removePending(int bytes)
+ {
+ scheduledCountUpdater.decrementAndGet(this);
+ scheduledBytesUpdater.addAndGet(this, -bytes);
+ }
+
+ long scheduledCount()
+ {
+ return scheduledCount;
+ }
+
+ long scheduledBytes()
+ {
+ return scheduledBytes;
+ }
+}
diff --git a/src/java/org/apache/cassandra/net/InboundMessageCallbacks.java b/src/java/org/apache/cassandra/net/InboundMessageCallbacks.java
new file mode 100644
index 0000000..ffa4243
--- /dev/null
+++ b/src/java/org/apache/cassandra/net/InboundMessageCallbacks.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.net;
+
+import java.util.concurrent.TimeUnit;
+
+import org.apache.cassandra.concurrent.Stage;
+import org.apache.cassandra.net.Message.Header;
+
+/**
+ * Encapsulates the callbacks that {@link InboundMessageHandler} invokes during the lifecycle of an inbound message
+ * passing through it: from arrival to dispatch to execution.
+ *
+ * The flow will vary slightly between small and large messages. Small messages will be deserialized first and only
+ * then dispatched to one of the {@link Stage} stages for execution, whereas a large message will be dispatched first,
+ * and deserialized in-place on the relevant stage before being immediately processed.
+ *
+ * This difference will only show in case of deserialization failure. For large messages, it's possible for
+ * {@link #onFailedDeserialize(int, Header, Throwable)} to be invoked after {@link #onExecuting(int, Header, long, TimeUnit)},
+ * whereas for small messages it isn't.
+ */
+interface InboundMessageCallbacks
+{
+ /**
+ * Invoked once the header of a message has arrived, small or large.
+ */
+ void onHeaderArrived(int messageSize, Header header, long timeElapsed, TimeUnit unit);
+
+ /**
+ * Invoked once an entire message worth of bytes has arrived, small or large.
+ */
+ void onArrived(int messageSize, Header header, long timeElapsed, TimeUnit unit);
+
+ /**
+ * Invoked if a message arrived too late to be processed, after its expiration. {@code wasCorrupt} might
+ * be set to {@code true} if 1+ corrupt frames were encountered while assembling an expired large message.
+ */
+ void onArrivedExpired(int messageSize, Header header, boolean wasCorrupt, long timeElapsed, TimeUnit unit);
+
+ /**
+ * Invoked if a large message arrived in time, but had one or more of its frames corrupted in flight.
+ */
+ void onArrivedCorrupt(int messageSize, Header header, long timeElapsed, TimeUnit unit);
+
+ /**
+ * Invoked if {@link InboundMessageHandler} was closed before receiving all frames of a large message.
+ * {@code wasCorrupt} will be set to {@code true} if some corrupt frames had been already encountered,
+ * {@code wasExpired} will be set to {@code true} if the message had expired in flight.
+ */
+ void onClosedBeforeArrival(int messageSize, Header header, int bytesReceived, boolean wasCorrupt, boolean wasExpired);
+
+ /**
+ * Invoked if a deserializer threw an exception while attempting to deserialize a message.
+ */
+ void onFailedDeserialize(int messageSize, Header header, Throwable t);
+
+ /**
+ * Invoked just before a message-processing task is scheduled on the appropriate {@link Stage}
+ * for the {@link Verb} of the message.
+ */
+ void onDispatched(int messageSize, Header header);
+
+ /**
+ * Invoked at the very beginning of execution of the message-processing task on the appropriate {@link Stage}.
+ */
+ void onExecuting(int messageSize, Header header, long timeElapsed, TimeUnit unit);
+
+ /**
+ * Invoked upon 'successful' processing of the message. Alternatively, {@link #onExpired(int, Header, long, TimeUnit)}
+ * will be invoked if the message had expired while waiting to be processed in the queue of the {@link Stage}.
+ */
+ void onProcessed(int messageSize, Header header);
+
+ /**
+ * Invoked if the message had expired while waiting to be processed in the queue of the {@link Stage}. Otherwise,
+ * {@link #onProcessed(int, Header)} will be invoked.
+ */
+ void onExpired(int messageSize, Header header, long timeElapsed, TimeUnit unit);
+
+ /**
+ * Invoked at the very end of execution of the message-processing task, no matter the outcome of processing.
+ */
+ void onExecuted(int messageSize, Header header, long timeElapsed, TimeUnit unit);
+}
diff --git a/src/java/org/apache/cassandra/net/InboundMessageHandler.java b/src/java/org/apache/cassandra/net/InboundMessageHandler.java
new file mode 100644
index 0000000..1fc182b
--- /dev/null
+++ b/src/java/org/apache/cassandra/net/InboundMessageHandler.java
@@ -0,0 +1,1193 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.net;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.IdentityHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
+import java.util.concurrent.atomic.AtomicLongFieldUpdater;
+import java.util.function.Consumer;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import io.netty.channel.Channel;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.channel.ChannelInboundHandlerAdapter;
+import io.netty.channel.EventLoop;
+import org.apache.cassandra.concurrent.ExecutorLocals;
+import org.apache.cassandra.concurrent.Stage;
+import org.apache.cassandra.exceptions.IncompatibleSchemaException;
+import org.apache.cassandra.io.util.DataInputBuffer;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.net.Message.Header;
+import org.apache.cassandra.net.FrameDecoder.Frame;
+import org.apache.cassandra.net.FrameDecoder.FrameProcessor;
+import org.apache.cassandra.net.FrameDecoder.IntactFrame;
+import org.apache.cassandra.net.FrameDecoder.CorruptFrame;
+import org.apache.cassandra.net.ResourceLimits.Limit;
+import org.apache.cassandra.tracing.TraceState;
+import org.apache.cassandra.tracing.Tracing;
+import org.apache.cassandra.utils.JVMStabilityInspector;
+import org.apache.cassandra.utils.NoSpamLogger;
+
+import static java.lang.Math.max;
+import static java.lang.Math.min;
+import static java.util.concurrent.TimeUnit.NANOSECONDS;
+import static org.apache.cassandra.net.Crc.*;
+import static org.apache.cassandra.utils.MonotonicClock.approxTime;
+
+/**
+ * Core logic for handling inbound message deserialization and execution (in tandem with {@link FrameDecoder}).
+ *
+ * Handles small and large messages, corruption, flow control, dispatch of message processing onto an appropriate
+ * thread pool.
+ *
+ * # Interaction with {@link FrameDecoder}
+ *
+ * {@link InboundMessageHandler} sits on top of a {@link FrameDecoder} in the Netty pipeline, and is tightly
+ * coupled with it.
+ *
+ * {@link FrameDecoder} decodes inbound frames and relies on a supplied {@link FrameProcessor} to act on them.
+ * {@link InboundMessageHandler} provides two implementations of that interface:
+ * - {@link #process(Frame)} is the default, primary processor, and the primary entry point to this class
+ * - {@link UpToOneMessageFrameProcessor}, supplied to the decoder when the handler is reactivated after being
+ * put in waiting mode due to lack of acquirable reserve memory capacity permits
+ *
+ * Return value of {@link FrameProcessor#process(Frame)} determines whether the decoder should keep processing
+ * frames (if {@code true} is returned) or stop until explicitly reactivated (if {@code false} is). To reactivate
+ * the decoder (once notified of available resource permits), {@link FrameDecoder#reactivate()} is invoked.
+ *
+ * # Frames
+ *
+ * {@link InboundMessageHandler} operates on frames of messages, and there are several kinds of them:
+ * 1. {@link IntactFrame} that are contained. As names suggest, these contain one or multiple fully contained
+ * messages believed to be uncorrupted. Guaranteed to not contain an part of an incomplete message.
+ * See {@link #processFrameOfContainedMessages(ShareableBytes, Limit, Limit)}.
+ * 2. {@link IntactFrame} that are NOT contained. These are uncorrupted parts of a large message split over multiple
+ * parts due to their size. Can represent first or subsequent frame of a large message.
+ * See {@link #processFirstFrameOfLargeMessage(IntactFrame, Limit, Limit)} and
+ * {@link #processSubsequentFrameOfLargeMessage(Frame)}.
+ * 3. {@link CorruptFrame} with corrupt header. These are unrecoverable, and force a connection to be dropped.
+ * 4. {@link CorruptFrame} with a valid header, but corrupt payload. These can be either contained or uncontained.
+ * - contained frames with corrupt payload can be gracefully dropped without dropping the connection
+ * - uncontained frames with corrupt payload can be gracefully dropped unless they represent the first
+ * frame of a new large message, as in that case we don't know how many bytes to skip
+ * See {@link #processCorruptFrame(CorruptFrame)}.
+ *
+ * Fundamental frame invariants:
+ * 1. A contained frame can only have fully-encapsulated messages - 1 to n, that don't cross frame boundaries
+ * 2. An uncontained frame can hold a part of one message only. It can NOT, say, contain end of one large message
+ * and a beginning of another one. All the bytes in an uncontained frame always belong to a single message.
+ *
+ * # Small vs large messages
+ *
+ * A single handler is equipped to process both small and large messages, potentially interleaved, but the logic
+ * differs depending on size. Small messages are deserialized in place, and then handed off to an appropriate
+ * thread pool for processing. Large messages accumulate frames until completion of a message, then hand off
+ * the untouched frames to the correct thread pool for the verb to be deserialized there and immediately processed.
+ *
+ * See {@link LargeMessage} for details of the large-message accumulating state-machine, and {@link ProcessMessage}
+ * and its inheritors for the differences in execution.
+ *
+ * # Flow control (backpressure)
+ *
+ * To prevent nodes from overwhelming and bringing each other to the knees with more inbound messages that
+ * can be processed in a timely manner, {@link InboundMessageHandler} implements a strict flow control policy.
+ *
+ * Before we attempt to process a message fully, we first infer its size from the stream. Then we attempt to
+ * acquire memory permits for a message of that size. If we succeed, then we move on actually process the message.
+ * If we fail, the frame decoder deactivates until sufficient permits are released for the message to be processed
+ * and the handler is activated again. Permits are released back once the message has been fully processed -
+ * after the verb handler has been invoked - on the {@link Stage} for the {@link Verb} of the message.
+ *
+ * Every connection has an exclusive number of permits allocated to it (by default 4MiB). In addition to it,
+ * there is a per-endpoint reserve capacity and a global reserve capacity {@link Limit}, shared between all
+ * connections from the same host and all connections, respectively. So long as long as the handler stays within
+ * its exclusive limit, it doesn't need to tap into reserve capacity.
+ *
+ * If tapping into reserve capacity is necessary, but the handler fails to acquire capacity from either
+ * endpoint of global reserve (and it needs to acquire from both), the handler and its frame decoder become
+ * inactive and register with a {@link WaitQueue} of the appropriate type, depending on which of the reserves
+ * couldn't be tapped into. Once enough messages have finished processing and had their permits released back
+ * to the reserves, {@link WaitQueue} will reactivate the sleeping handlers and they'll resume processing frames.
+ *
+ * The reason we 'split' reserve capacity into two limits - endpoing and global - is to guarantee liveness, and
+ * prevent single endpoint's connections from taking over the whole reserve, starving other connections.
+ *
+ * One permit per byte of serialized message gets acquired. When inflated on-heap, each message will occupy more
+ * than that, necessarily, but despite wide variance, it's a good enough proxy that correlates with on-heap footprint.
+ */
+public class InboundMessageHandler extends ChannelInboundHandlerAdapter implements FrameProcessor
+{
+ private static final Logger logger = LoggerFactory.getLogger(InboundMessageHandler.class);
+ private static final NoSpamLogger noSpamLogger = NoSpamLogger.getLogger(logger, 1L, TimeUnit.SECONDS);
+
+ private static final Message.Serializer serializer = Message.serializer;
+
+ private final FrameDecoder decoder;
+
+ private final ConnectionType type;
+ private final Channel channel;
+ private final InetAddressAndPort self;
+ private final InetAddressAndPort peer;
+ private final int version;
+
+ private final int largeThreshold;
+ private LargeMessage largeMessage;
+
+ private final long queueCapacity;
+ volatile long queueSize = 0L;
+ private static final AtomicLongFieldUpdater<InboundMessageHandler> queueSizeUpdater =
+ AtomicLongFieldUpdater.newUpdater(InboundMessageHandler.class, "queueSize");
+
+ private final Limit endpointReserveCapacity;
+ private final WaitQueue endpointWaitQueue;
+
+ private final Limit globalReserveCapacity;
+ private final WaitQueue globalWaitQueue;
+
+ private final OnHandlerClosed onClosed;
+ private final InboundMessageCallbacks callbacks;
+ private final Consumer<Message<?>> consumer;
+
+ // wait queue handle, non-null if we overrun endpoint or global capacity and request to be resumed once it's released
+ private WaitQueue.Ticket ticket = null;
+
+ long corruptFramesRecovered, corruptFramesUnrecovered;
+ long receivedCount, receivedBytes;
+ long throttledCount, throttledNanos;
+
+ private boolean isClosed;
+
+ InboundMessageHandler(FrameDecoder decoder,
+
+ ConnectionType type,
+ Channel channel,
+ InetAddressAndPort self,
+ InetAddressAndPort peer,
+ int version,
+ int largeThreshold,
+
+ long queueCapacity,
+ Limit endpointReserveCapacity,
+ Limit globalReserveCapacity,
+ WaitQueue endpointWaitQueue,
+ WaitQueue globalWaitQueue,
+
+ OnHandlerClosed onClosed,
+ InboundMessageCallbacks callbacks,
+ Consumer<Message<?>> consumer)
+ {
+ this.decoder = decoder;
+
+ this.type = type;
+ this.channel = channel;
+ this.self = self;
+ this.peer = peer;
+ this.version = version;
+ this.largeThreshold = largeThreshold;
+
+ this.queueCapacity = queueCapacity;
+ this.endpointReserveCapacity = endpointReserveCapacity;
+ this.endpointWaitQueue = endpointWaitQueue;
+ this.globalReserveCapacity = globalReserveCapacity;
+ this.globalWaitQueue = globalWaitQueue;
+
+ this.onClosed = onClosed;
+ this.callbacks = callbacks;
+ this.consumer = consumer;
+ }
+
+ @Override
+ public void channelRead(ChannelHandlerContext ctx, Object msg)
+ {
+ /*
+ * InboundMessageHandler works in tandem with FrameDecoder to implement flow control
+ * and work stashing optimally. We rely on FrameDecoder to invoke the provided
+ * FrameProcessor rather than on the pipeline and invocations of channelRead().
+ * process(Frame) is the primary entry point for this class.
+ */
+ throw new IllegalStateException("InboundMessageHandler doesn't expect channelRead() to be invoked");
+ }
+
+ @Override
+ public void handlerAdded(ChannelHandlerContext ctx)
+ {
+ decoder.activate(this); // the frame decoder starts inactive until explicitly activated by the added inbound message handler
+ }
+
+ @Override
+ public boolean process(Frame frame) throws IOException
+ {
+ if (frame instanceof IntactFrame)
+ return processIntactFrame((IntactFrame) frame, endpointReserveCapacity, globalReserveCapacity);
+
+ processCorruptFrame((CorruptFrame) frame);
+ return true;
+ }
+
+ private boolean processIntactFrame(IntactFrame frame, Limit endpointReserve, Limit globalReserve) throws IOException
+ {
+ if (frame.isSelfContained)
+ return processFrameOfContainedMessages(frame.contents, endpointReserve, globalReserve);
+ else if (null == largeMessage)
+ return processFirstFrameOfLargeMessage(frame, endpointReserve, globalReserve);
+ else
+ return processSubsequentFrameOfLargeMessage(frame);
+ }
+
+ /*
+ * Handle contained messages (not crossing boundaries of the frame) - both small and large, for the inbound
+ * definition of large (breaching the size threshold for what we are willing to process on event-loop vs.
+ * off event-loop).
+ */
+ private boolean processFrameOfContainedMessages(ShareableBytes bytes, Limit endpointReserve, Limit globalReserve) throws IOException
+ {
+ while (bytes.hasRemaining())
+ if (!processOneContainedMessage(bytes, endpointReserve, globalReserve))
+ return false;
+ return true;
+ }
+
+ private boolean processOneContainedMessage(ShareableBytes bytes, Limit endpointReserve, Limit globalReserve) throws IOException
+ {
+ ByteBuffer buf = bytes.get();
+
+ long currentTimeNanos = approxTime.now();
+ Header header = serializer.extractHeader(buf, peer, currentTimeNanos, version);
+ long timeElapsed = currentTimeNanos - header.createdAtNanos;
+ int size = serializer.inferMessageSize(buf, buf.position(), buf.limit(), version);
+
+ if (approxTime.isAfter(currentTimeNanos, header.expiresAtNanos))
+ {
+ callbacks.onHeaderArrived(size, header, timeElapsed, NANOSECONDS);
+ callbacks.onArrivedExpired(size, header, false, timeElapsed, NANOSECONDS);
+ receivedCount++;
+ receivedBytes += size;
+ bytes.skipBytes(size);
+ return true;
+ }
+
+ if (!acquireCapacity(endpointReserve, globalReserve, size, currentTimeNanos, header.expiresAtNanos))
+ return false;
+
+ callbacks.onHeaderArrived(size, header, timeElapsed, NANOSECONDS);
+ callbacks.onArrived(size, header, timeElapsed, NANOSECONDS);
+ receivedCount++;
+ receivedBytes += size;
+
+ if (size <= largeThreshold)
+ processSmallMessage(bytes, size, header);
+ else
+ processLargeMessage(bytes, size, header);
+
+ return true;
+ }
+
+ private void processSmallMessage(ShareableBytes bytes, int size, Header header)
+ {
+ ByteBuffer buf = bytes.get();
+ final int begin = buf.position();
+ final int end = buf.limit();
+ buf.limit(begin + size); // cap to expected message size
+
+ Message<?> message = null;
+ try (DataInputBuffer in = new DataInputBuffer(buf, false))
+ {
+ Message<?> m = serializer.deserialize(in, header, version);
+ if (in.available() > 0) // bytes remaining after deser: deserializer is busted
+ throw new InvalidSerializedSizeException(header.verb, size, size - in.available());
+ message = m;
+ }
+ catch (IncompatibleSchemaException e)
+ {
+ callbacks.onFailedDeserialize(size, header, e);
+ noSpamLogger.info("{} incompatible schema encountered while deserializing a message", id(), e);
+ }
+ catch (Throwable t)
+ {
+ JVMStabilityInspector.inspectThrowable(t, false);
+ callbacks.onFailedDeserialize(size, header, t);
+ logger.error("{} unexpected exception caught while deserializing a message", id(), t);
+ }
+ finally
+ {
+ if (null == message)
+ releaseCapacity(size);
+
+ // no matter what, set position to the beginning of the next message and restore limit, so that
+ // we can always keep on decoding the frame even on failure to deserialize previous message
+ buf.position(begin + size);
+ buf.limit(end);
+ }
+
+ if (null != message)
+ dispatch(new ProcessSmallMessage(message, size));
+ }
+
+ // for various reasons, it's possible for a large message to be contained in a single frame
+ private void processLargeMessage(ShareableBytes bytes, int size, Header header)
+ {
+ new LargeMessage(size, header, bytes.sliceAndConsume(size).share()).schedule();
+ }
+
+ /*
+ * Handling of multi-frame large messages
+ */
+
+ private boolean processFirstFrameOfLargeMessage(IntactFrame frame, Limit endpointReserve, Limit globalReserve) throws IOException
+ {
+ ShareableBytes bytes = frame.contents;
+ ByteBuffer buf = bytes.get();
+
+ long currentTimeNanos = approxTime.now();
+ Header header = serializer.extractHeader(buf, peer, currentTimeNanos, version);
+ int size = serializer.inferMessageSize(buf, buf.position(), buf.limit(), version);
+
+ boolean expired = approxTime.isAfter(currentTimeNanos, header.expiresAtNanos);
+ if (!expired && !acquireCapacity(endpointReserve, globalReserve, size, currentTimeNanos, header.expiresAtNanos))
+ return false;
+
+ callbacks.onHeaderArrived(size, header, currentTimeNanos - header.createdAtNanos, NANOSECONDS);
+ receivedBytes += buf.remaining();
+ largeMessage = new LargeMessage(size, header, expired);
+ largeMessage.supply(frame);
+ return true;
+ }
+
+ private boolean processSubsequentFrameOfLargeMessage(Frame frame)
+ {
+ receivedBytes += frame.frameSize;
+ if (largeMessage.supply(frame))
+ {
+ receivedCount++;
+ largeMessage = null;
+ }
+ return true;
+ }
+
+ /*
+ * We can handle some corrupt frames gracefully without dropping the connection and losing all the
+ * queued up messages, but not others.
+ *
+ * Corrupt frames that *ARE NOT* safe to skip gracefully and require the connection to be dropped:
+ * - any frame with corrupt header (!frame.isRecoverable())
+ * - first corrupt-payload frame of a large message (impossible to infer message size, and without it
+ * impossible to skip the message safely
+ *
+ * Corrupt frames that *ARE* safe to skip gracefully, without reconnecting:
+ * - any self-contained frame with a corrupt payload (but not header): we lose all the messages in the
+ * frame, but that has no effect on subsequent ones
+ * - any non-first payload-corrupt frame of a large message: we know the size of the large message in
+ * flight, so we just skip frames until we've seen all its bytes; we only lose the large message
+ */
+ private void processCorruptFrame(CorruptFrame frame) throws InvalidCrc
+ {
+ if (!frame.isRecoverable())
+ {
+ corruptFramesUnrecovered++;
+ throw new InvalidCrc(frame.readCRC, frame.computedCRC);
+ }
+ else if (frame.isSelfContained)
+ {
+ receivedBytes += frame.frameSize;
+ corruptFramesRecovered++;
+ noSpamLogger.warn("{} invalid, recoverable CRC mismatch detected while reading messages (corrupted self-contained frame)", id());
+ }
+ else if (null == largeMessage) // first frame of a large message
+ {
+ receivedBytes += frame.frameSize;
+ corruptFramesUnrecovered++;
+ noSpamLogger.error("{} invalid, unrecoverable CRC mismatch detected while reading messages (corrupted first frame of a large message)", id());
+ throw new InvalidCrc(frame.readCRC, frame.computedCRC);
+ }
+ else // subsequent frame of a large message
+ {
+ processSubsequentFrameOfLargeMessage(frame);
+ corruptFramesRecovered++;
+ noSpamLogger.warn("{} invalid, recoverable CRC mismatch detected while reading a large message", id());
+ }
+ }
+
+ private void onEndpointReserveCapacityRegained(Limit endpointReserve, long elapsedNanos)
+ {
+ onReserveCapacityRegained(endpointReserve, globalReserveCapacity, elapsedNanos);
+ }
+
+ private void onGlobalReserveCapacityRegained(Limit globalReserve, long elapsedNanos)
+ {
+ onReserveCapacityRegained(endpointReserveCapacity, globalReserve, elapsedNanos);
+ }
+
+ private void onReserveCapacityRegained(Limit endpointReserve, Limit globalReserve, long elapsedNanos)
+ {
+ if (isClosed)
+ return;
+
+ assert channel.eventLoop().inEventLoop();
+
+ ticket = null;
+ throttledNanos += elapsedNanos;
+
+ try
+ {
+ /*
+ * Process up to one message using supplied overriden reserves - one of them pre-allocated,
+ * and guaranteed to be enough for one message - then, if no obstacles enountered, reactivate
+ * the frame decoder using normal reserve capacities.
+ */
+ if (processUpToOneMessage(endpointReserve, globalReserve))
+ decoder.reactivate();
+ }
+ catch (Throwable t)
+ {
+ exceptionCaught(t);
+ }
+ }
+
+ // return true if the handler should be reactivated - if no new hurdles were encountered,
+ // like running out of the other kind of reserve capacity
+ private boolean processUpToOneMessage(Limit endpointReserve, Limit globalReserve) throws IOException
+ {
+ UpToOneMessageFrameProcessor processor = new UpToOneMessageFrameProcessor(endpointReserve, globalReserve);
+ decoder.processBacklog(processor);
+ return processor.isActive;
+ }
+
+ /*
+ * Process at most one message. Won't always be an entire one (if the message in the head of line
+ * is a large one, and there aren't sufficient frames to decode it entirely), but will never be more than one.
+ */
+ private class UpToOneMessageFrameProcessor implements FrameProcessor
+ {
+ private final Limit endpointReserve;
+ private final Limit globalReserve;
+
+ boolean isActive = true;
+ boolean firstFrame = true;
+
+ private UpToOneMessageFrameProcessor(Limit endpointReserve, Limit globalReserve)
+ {
+ this.endpointReserve = endpointReserve;
+ this.globalReserve = globalReserve;
+ }
+
+ @Override
+ public boolean process(Frame frame) throws IOException
+ {
+ if (firstFrame)
+ {
+ if (!(frame instanceof IntactFrame))
+ throw new IllegalStateException("First backlog frame must be intact");
+ firstFrame = false;
+ return processFirstFrame((IntactFrame) frame);
+ }
+
+ return processSubsequentFrame(frame);
+ }
+
+ private boolean processFirstFrame(IntactFrame frame) throws IOException
+ {
+ if (frame.isSelfContained)
+ {
+ isActive = processOneContainedMessage(frame.contents, endpointReserve, globalReserve);
+ return false; // stop after one message
+ }
+ else
+ {
+ isActive = processFirstFrameOfLargeMessage(frame, endpointReserve, globalReserve);
+ return isActive; // continue unless fallen behind coprocessor or ran out of reserve capacity again
+ }
+ }
+
+ private boolean processSubsequentFrame(Frame frame) throws IOException
+ {
+ if (frame instanceof IntactFrame)
+ processSubsequentFrameOfLargeMessage(frame);
+ else
+ processCorruptFrame((CorruptFrame) frame);
+
+ return largeMessage != null; // continue until done with the large message
+ }
+ }
+
+ /**
+ * Try to acquire permits for the inbound message. In case of failure, register with the right wait queue to be
+ * reactivated once permit capacity is regained.
+ */
+ @SuppressWarnings("BooleanMethodIsAlwaysInverted")
+ private boolean acquireCapacity(Limit endpointReserve, Limit globalReserve, int bytes, long currentTimeNanos, long expiresAtNanos)
+ {
+ ResourceLimits.Outcome outcome = acquireCapacity(endpointReserve, globalReserve, bytes);
+
+ if (outcome == ResourceLimits.Outcome.INSUFFICIENT_ENDPOINT)
+ ticket = endpointWaitQueue.register(this, bytes, currentTimeNanos, expiresAtNanos);
+ else if (outcome == ResourceLimits.Outcome.INSUFFICIENT_GLOBAL)
+ ticket = globalWaitQueue.register(this, bytes, currentTimeNanos, expiresAtNanos);
+
+ if (outcome != ResourceLimits.Outcome.SUCCESS)
+ throttledCount++;
+
+ return outcome == ResourceLimits.Outcome.SUCCESS;
+ }
+
+ private ResourceLimits.Outcome acquireCapacity(Limit endpointReserve, Limit globalReserve, int bytes)
+ {
+ long currentQueueSize = queueSize;
+
+ /*
+ * acquireCapacity() is only ever called on the event loop, and as such queueSize is only ever increased
+ * on the event loop. If there is enough capacity, we can safely addAndGet() and immediately return.
+ */
+ if (currentQueueSize + bytes <= queueCapacity)
+ {
+ queueSizeUpdater.addAndGet(this, bytes);
+ return ResourceLimits.Outcome.SUCCESS;
+ }
+
+ // we know we don't have enough local queue capacity for the entire message, so we need to borrow some from reserve capacity
+ long allocatedExcess = min(currentQueueSize + bytes - queueCapacity, bytes);
+
+ if (!globalReserve.tryAllocate(allocatedExcess))
+ return ResourceLimits.Outcome.INSUFFICIENT_GLOBAL;
+
+ if (!endpointReserve.tryAllocate(allocatedExcess))
+ {
+ globalReserve.release(allocatedExcess);
+ globalWaitQueue.signal();
+ return ResourceLimits.Outcome.INSUFFICIENT_ENDPOINT;
+ }
+
+ long newQueueSize = queueSizeUpdater.addAndGet(this, bytes);
+ long actualExcess = max(0, min(newQueueSize - queueCapacity, bytes));
+
+ /*
+ * It's possible that some permits were released at some point after we loaded current queueSize,
+ * and we can satisfy more of the permits using our exclusive per-connection capacity, needing
+ * less than previously estimated from the reserves. If that's the case, release the now unneeded
+ * permit excess back to endpoint/global reserves.
+ */
+ if (actualExcess != allocatedExcess) // actualExcess < allocatedExcess
+ {
+ long excess = allocatedExcess - actualExcess;
+
+ endpointReserve.release(excess);
+ globalReserve.release(excess);
+
+ endpointWaitQueue.signal();
+ globalWaitQueue.signal();
+ }
+
+ return ResourceLimits.Outcome.SUCCESS;
+ }
+
+ private void releaseCapacity(int bytes)
+ {
+ long oldQueueSize = queueSizeUpdater.getAndAdd(this, -bytes);
+ if (oldQueueSize > queueCapacity)
+ {
+ long excess = min(oldQueueSize - queueCapacity, bytes);
+
+ endpointReserveCapacity.release(excess);
+ globalReserveCapacity.release(excess);
+
+ endpointWaitQueue.signal();
+ globalWaitQueue.signal();
+ }
+ }
+
+ /**
+ * Invoked to release capacity for a message that has been fully, successfully processed.
+ *
+ * Normally no different from invoking {@link #releaseCapacity(int)}, but is necessary for the verifier
+ * to be able to delay capacity release for backpressure testing.
+ */
+ @VisibleForTesting
+ protected void releaseProcessedCapacity(int size, Header header)
+ {
+ releaseCapacity(size);
+ }
+
+ @Override
+ public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause)
+ {
+ try
+ {
+ exceptionCaught(cause);
+ }
+ catch (Throwable t)
+ {
+ logger.error("Unexpected exception in {}.exceptionCaught", this.getClass().getSimpleName(), t);
+ }
+ }
+
+ private void exceptionCaught(Throwable cause)
+ {
+ decoder.discard();
+
+ JVMStabilityInspector.inspectThrowable(cause, false);
+
+ if (cause instanceof Message.InvalidLegacyProtocolMagic)
+ logger.error("{} invalid, unrecoverable CRC mismatch detected while reading messages - closing the connection", id());
+ else
+ logger.error("{} unexpected exception caught while processing inbound messages; terminating connection", id(), cause);
+
+ channel.close();
+ }
+
+ @Override
+ public void channelInactive(ChannelHandlerContext ctx)
+ {
+ isClosed = true;
+
+ if (null != largeMessage)
+ largeMessage.abort();
+
+ if (null != ticket)
+ ticket.invalidate();
+
+ onClosed.call(this);
+ }
+
+ private EventLoop eventLoop()
+ {
+ return channel.eventLoop();
+ }
+
+ String id(boolean includeReal)
+ {
+ if (!includeReal)
+ return id();
+
+ return SocketFactory.channelId(peer, (InetSocketAddress) channel.remoteAddress(),
+ self, (InetSocketAddress) channel.localAddress(),
+ type, channel.id().asShortText());
+ }
+
+ String id()
+ {
+ return SocketFactory.channelId(peer, self, type, channel.id().asShortText());
+ }
+
+ /*
+ * A large-message frame-accumulating state machine.
+ *
+ * Collects intact frames until it's has all the bytes necessary to deserialize the large message,
+ * at which point it schedules a task on the appropriate {@link Stage},
+ * a task that deserializes the message and immediately invokes the verb handler.
+ *
+ * Also handles corrupt frames and potential expiry of the large message during accumulation:
+ * if it's taking the frames too long to arrive, there is no point in holding on to the
+ * accumulated frames, or in gathering more - so we release the ones we already have, and
+ * skip any remaining ones, alongside with returning memory permits early.
+ */
+ private class LargeMessage
+ {
+ private final int size;
+ private final Header header;
+
+ private final List<ShareableBytes> buffers = new ArrayList<>();
+ private int received;
+
+ private boolean isExpired;
+ private boolean isCorrupt;
+
+ private LargeMessage(int size, Header header, boolean isExpired)
+ {
+ this.size = size;
+ this.header = header;
+ this.isExpired = isExpired;
+ }
+
+ private LargeMessage(int size, Header header, ShareableBytes bytes)
+ {
+ this(size, header, false);
+ buffers.add(bytes);
+ }
+
+ private void schedule()
+ {
+ dispatch(new ProcessLargeMessage(this));
+ }
+
+ /**
+ * Return true if this was the last frame of the large message.
+ */
+ private boolean supply(Frame frame)
+ {
+ if (frame instanceof IntactFrame)
+ onIntactFrame((IntactFrame) frame);
+ else
+ onCorruptFrame();
+
+ received += frame.frameSize;
+ if (size == received)
+ onComplete();
+ return size == received;
+ }
+
+ private void onIntactFrame(IntactFrame frame)
+ {
+ boolean expires = approxTime.isAfter(header.expiresAtNanos);
+ if (!isExpired && !isCorrupt)
+ {
+ if (!expires)
+ {
+ buffers.add(frame.contents.sliceAndConsume(frame.frameSize).share());
+ return;
+ }
+ releaseBuffersAndCapacity(); // release resources once we transition from normal state to expired
+ }
+ frame.consume();
+ isExpired |= expires;
+ }
+
+ private void onCorruptFrame()
+ {
+ if (!isExpired && !isCorrupt)
+ releaseBuffersAndCapacity(); // release resources once we transition from normal state to corrupt
+ isCorrupt = true;
+ isExpired |= approxTime.isAfter(header.expiresAtNanos);
+ }
+
+ private void onComplete()
+ {
+ long timeElapsed = approxTime.now() - header.createdAtNanos;
+
+ if (!isExpired && !isCorrupt)
+ {
+ callbacks.onArrived(size, header, timeElapsed, NANOSECONDS);
+ schedule();
+ }
+ else if (isExpired)
+ {
+ callbacks.onArrivedExpired(size, header, isCorrupt, timeElapsed, NANOSECONDS);
+ }
+ else
+ {
+ callbacks.onArrivedCorrupt(size, header, timeElapsed, NANOSECONDS);
+ }
+ }
+
+ private void abort()
+ {
+ if (!isExpired && !isCorrupt)
+ releaseBuffersAndCapacity(); // release resources if in normal state when abort() is invoked
+ callbacks.onClosedBeforeArrival(size, header, received, isCorrupt, isExpired);
+ }
+
+ private void releaseBuffers()
+ {
+ buffers.forEach(ShareableBytes::release); buffers.clear();
+ }
+
+ private void releaseBuffersAndCapacity()
+ {
+ releaseBuffers(); releaseCapacity(size);
+ }
+
+ private Message deserialize()
+ {
+ try (ChunkedInputPlus input = ChunkedInputPlus.of(buffers))
+ {
+ Message<?> m = serializer.deserialize(input, header, version);
+ int remainder = input.remainder();
+ if (remainder > 0)
+ throw new InvalidSerializedSizeException(header.verb, size, size - remainder);
+ return m;
+ }
+ catch (IncompatibleSchemaException e)
+ {
+ callbacks.onFailedDeserialize(size, header, e);
+ noSpamLogger.info("{} incompatible schema encountered while deserializing a message", id(), e);
+ }
+ catch (Throwable t)
+ {
+ JVMStabilityInspector.inspectThrowable(t, false);
+ callbacks.onFailedDeserialize(size, header, t);
+ logger.error("{} unexpected exception caught while deserializing a message", id(), t);
+ }
+ finally
+ {
+ buffers.clear(); // closing the input will have ensured that the buffers were released no matter what
+ }
+
+ return null;
+ }
+ }
+
+ /**
+ * Submit a {@link ProcessMessage} task to the appropriate {@link Stage} for the {@link Verb}.
+ */
+ private void dispatch(ProcessMessage task)
+ {
+ Header header = task.header();
+
+ TraceState state = Tracing.instance.initializeFromMessage(header);
+ if (state != null) state.trace("{} message received from {}", header.verb, header.from);
+
+ callbacks.onDispatched(task.size(), header);
+ header.verb.stage.execute(task, ExecutorLocals.create(state));
+ }
+
+ private abstract class ProcessMessage implements Runnable
+ {
+ /**
+ * Actually handle the message. Runs on the appropriate {@link Stage} for the {@link Verb}.
+ *
+ * Small messages will come pre-deserialized. Large messages will be deserialized on the stage,
+ * just in time, and only then processed.
+ */
+ public void run()
+ {
+ Header header = header();
+ long currentTimeNanos = approxTime.now();
+ boolean expired = approxTime.isAfter(currentTimeNanos, header.expiresAtNanos);
+
+ boolean processed = false;
+ try
+ {
+ callbacks.onExecuting(size(), header, currentTimeNanos - header.createdAtNanos, NANOSECONDS);
+
+ if (expired)
+ {
+ callbacks.onExpired(size(), header, currentTimeNanos - header.createdAtNanos, NANOSECONDS);
+ return;
+ }
+
+ Message message = provideMessage();
+ if (null != message)
+ {
+ consumer.accept(message);
+ processed = true;
+ callbacks.onProcessed(size(), header);
+ }
+ }
+ finally
+ {
+ if (processed)
+ releaseProcessedCapacity(size(), header);
+ else
+ releaseCapacity(size());
+
+ releaseResources();
+
+ callbacks.onExecuted(size(), header, approxTime.now() - currentTimeNanos, NANOSECONDS);
+ }
+ }
+
+ abstract int size();
+ abstract Header header();
+ abstract Message provideMessage();
+ void releaseResources() {}
+ }
+
+ private class ProcessSmallMessage extends ProcessMessage
+ {
+ private final int size;
+ private final Message message;
+
+ ProcessSmallMessage(Message message, int size)
+ {
+ this.size = size;
+ this.message = message;
+ }
+
+ int size()
+ {
+ return size;
+ }
+
+ Header header()
+ {
+ return message.header;
+ }
+
+ Message provideMessage()
+ {
+ return message;
+ }
+ }
+
+ private class ProcessLargeMessage extends ProcessMessage
+ {
+ private final LargeMessage message;
+
+ ProcessLargeMessage(LargeMessage message)
+ {
+ this.message = message;
+ }
+
+ int size()
+ {
+ return message.size;
+ }
+
+ Header header()
+ {
+ return message.header;
+ }
+
+ Message provideMessage()
+ {
+ return message.deserialize();
+ }
+
+ @Override
+ void releaseResources()
+ {
+ message.releaseBuffers(); // releases buffers if they haven't been yet (by deserialize() call)
+ }
+ }
+
+ /**
+ * A special-purpose wait queue to park inbound message handlers that failed to allocate
+ * reserve capacity for a message in. Upon such failure a handler registers itself with
+ * a {@link WaitQueue} of the appropriate kind (either ENDPOINT or GLOBAL - if failed
+ * to allocate endpoint or global reserve capacity, respectively), stops processing any
+ * accumulated frames or receiving new ones, and waits - until reactivated.
+ *
+ * Every time permits are returned to an endpoint or global {@link Limit}, the respective
+ * queue gets signalled, and if there are any handlers registered in it, we will attempt
+ * to reactivate as many waiting handlers as current available reserve capacity allows
+ * us to - immediately, on the {@link #signal()}-calling thread. At most one such attempt
+ * will be in progress at any given time.
+ *
+ * Handlers that can be reactivated will be grouped by their {@link EventLoop} and a single
+ * {@link ReactivateHandlers} task will be scheduled per event loop, on the corresponding
+ * event loops.
+ *
+ * When run, the {@link ReactivateHandlers} task will ask each handler in its group to first
+ * process one message - using preallocated reserve capacity - and if no obstacles were met -
+ * reactivate the handlers, this time using their regular reserves.
+ *
+ * See {@link WaitQueue#schedule()}, {@link ReactivateHandlers#run()}, {@link Ticket#reactivateHandler(Limit)}.
+ */
+ public static final class WaitQueue
+ {
+ enum Kind { ENDPOINT, GLOBAL }
+
+ private static final int NOT_RUNNING = 0;
+ @SuppressWarnings("unused")
+ private static final int RUNNING = 1;
+ private static final int RUN_AGAIN = 2;
+
+ private volatile int scheduled;
+ private static final AtomicIntegerFieldUpdater<WaitQueue> scheduledUpdater =
+ AtomicIntegerFieldUpdater.newUpdater(WaitQueue.class, "scheduled");
+
+ private final Kind kind;
+ private final Limit reserveCapacity;
+
+ private final ManyToOneConcurrentLinkedQueue<Ticket> queue = new ManyToOneConcurrentLinkedQueue<>();
+
+ private WaitQueue(Kind kind, Limit reserveCapacity)
+ {
+ this.kind = kind;
+ this.reserveCapacity = reserveCapacity;
+ }
+
+ public static WaitQueue endpoint(Limit endpointReserveCapacity)
+ {
+ return new WaitQueue(Kind.ENDPOINT, endpointReserveCapacity);
+ }
+
+ public static WaitQueue global(Limit globalReserveCapacity)
+ {
+ return new WaitQueue(Kind.GLOBAL, globalReserveCapacity);
+ }
+
+ private Ticket register(InboundMessageHandler handler, int bytesRequested, long registeredAtNanos, long expiresAtNanos)
+ {
+ Ticket ticket = new Ticket(this, handler, bytesRequested, registeredAtNanos, expiresAtNanos);
+ Ticket previous = queue.relaxedPeekLastAndOffer(ticket);
+ if (null == previous || !previous.isWaiting())
+ signal(); // only signal the queue if this handler is first to register
+ return ticket;
+ }
+
+ private void signal()
+ {
+ if (queue.relaxedIsEmpty())
+ return; // we can return early if no handlers have registered with the wait queue
+
+ if (NOT_RUNNING == scheduledUpdater.getAndUpdate(this, i -> min(RUN_AGAIN, i + 1)))
+ {
+ do
+ {
+ schedule();
+ }
+ while (RUN_AGAIN == scheduledUpdater.getAndDecrement(this));
+ }
+ }
+
+ private void schedule()
+ {
+ Map<EventLoop, ReactivateHandlers> tasks = null;
+
+ long currentTimeNanos = approxTime.now();
+
+ Ticket t;
+ while ((t = queue.peek()) != null)
+ {
+ if (!t.call()) // invalidated
+ {
+ queue.remove();
+ continue;
+ }
+
+ boolean isLive = t.isLive(currentTimeNanos);
+ if (isLive && !reserveCapacity.tryAllocate(t.bytesRequested))
+ {
+ if (!t.reset()) // the ticket was invalidated after being called but before now
+ {
+ queue.remove();
+ continue;
+ }
+ break; // TODO: traverse the entire queue to unblock handlers that have expired or invalidated tickets
+ }
+
+ if (null == tasks)
+ tasks = new IdentityHashMap<>();
+
+ queue.remove();
+ tasks.computeIfAbsent(t.handler.eventLoop(), e -> new ReactivateHandlers()).add(t, isLive);
+ }
+
+ if (null != tasks)
+ tasks.forEach(EventLoop::execute);
+ }
+
+ private class ReactivateHandlers implements Runnable
+ {
+ List<Ticket> tickets = new ArrayList<>();
+ long capacity = 0L;
+
+ private void add(Ticket ticket, boolean isLive)
+ {
+ tickets.add(ticket);
+ if (isLive) capacity += ticket.bytesRequested;
+ }
+
+ public void run()
+ {
+ Limit limit = new ResourceLimits.Basic(capacity);
+ try
+ {
+ for (Ticket ticket : tickets)
+ ticket.reactivateHandler(limit);
+ }
+ finally
+ {
+ /*
+ * Free up any unused capacity, if any. Will be non-zero if one or more handlers were closed
+ * when we attempted to run their callback, or used more of their other reserve; or if the first
+ * message in the unprocessed stream has expired in the narrow time window.
+ */
+ long remaining = limit.remaining();
+ if (remaining > 0)
+ {
+ reserveCapacity.release(remaining);
+ signal();
+ }
+ }
+ }
+ }
+
+ private static final class Ticket
+ {
+ private static final int WAITING = 0;
+ private static final int CALLED = 1;
+ private static final int INVALIDATED = 2; // invalidated by a handler that got closed
+
+ private volatile int state;
+ private static final AtomicIntegerFieldUpdater<Ticket> stateUpdater =
+ AtomicIntegerFieldUpdater.newUpdater(Ticket.class, "state");
+
+ private final WaitQueue waitQueue;
+ private final InboundMessageHandler handler;
+ private final int bytesRequested;
+ private final long reigsteredAtNanos;
+ private final long expiresAtNanos;
+
+ private Ticket(WaitQueue waitQueue, InboundMessageHandler handler, int bytesRequested, long registeredAtNanos, long expiresAtNanos)
+ {
+ this.waitQueue = waitQueue;
+ this.handler = handler;
+ this.bytesRequested = bytesRequested;
+ this.reigsteredAtNanos = registeredAtNanos;
+ this.expiresAtNanos = expiresAtNanos;
+ }
+
+ private void reactivateHandler(Limit capacity)
+ {
+ long elapsedNanos = approxTime.now() - reigsteredAtNanos;
+ try
+ {
+ if (waitQueue.kind == Kind.ENDPOINT)
+ handler.onEndpointReserveCapacityRegained(capacity, elapsedNanos);
+ else
+ handler.onGlobalReserveCapacityRegained(capacity, elapsedNanos);
+ }
+ catch (Throwable t)
+ {
+ logger.error("{} exception caught while reactivating a handler", handler.id(), t);
+ }
+ }
+
+ private boolean isWaiting()
+ {
+ return state == WAITING;
+ }
+
+ private boolean isLive(long currentTimeNanos)
+ {
+ return !approxTime.isAfter(currentTimeNanos, expiresAtNanos);
+ }
+
+ private void invalidate()
+ {
+ state = INVALIDATED;
+ waitQueue.signal();
+ }
+
+ private boolean call()
+ {
+ return stateUpdater.compareAndSet(this, WAITING, CALLED);
+ }
+
+ private boolean reset()
+ {
+ return stateUpdater.compareAndSet(this, CALLED, WAITING);
+ }
+ }
+ }
+
+ public interface OnHandlerClosed
+ {
+ void call(InboundMessageHandler handler);
+ }
+}
diff --git a/src/java/org/apache/cassandra/net/InboundMessageHandlers.java b/src/java/org/apache/cassandra/net/InboundMessageHandlers.java
new file mode 100644
index 0000000..4ebd5ad
--- /dev/null
+++ b/src/java/org/apache/cassandra/net/InboundMessageHandlers.java
@@ -0,0 +1,447 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.net;
+
+import java.util.Collection;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLongFieldUpdater;
+import java.util.function.Consumer;
+import java.util.function.ToLongFunction;
+
+import io.netty.channel.Channel;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.metrics.InternodeInboundMetrics;
+import org.apache.cassandra.net.Message.Header;
+import org.apache.cassandra.utils.ApproximateTime;
+
+import static java.util.concurrent.TimeUnit.NANOSECONDS;
+import static org.apache.cassandra.utils.MonotonicClock.approxTime;
+
+/**
+ * An aggregation of {@link InboundMessageHandler}s for all connections from a peer.
+ *
+ * Manages metrics and shared resource limits. Can have multiple connections of a single
+ * type open simultaneousely (legacy in particular).
+ */
+public final class InboundMessageHandlers
+{
+ private final InetAddressAndPort self;
+ private final InetAddressAndPort peer;
+
+ private final int queueCapacity;
+ private final ResourceLimits.Limit endpointReserveCapacity;
+ private final ResourceLimits.Limit globalReserveCapacity;
+
+ private final InboundMessageHandler.WaitQueue endpointWaitQueue;
+ private final InboundMessageHandler.WaitQueue globalWaitQueue;
+
+ private final InboundCounters urgentCounters = new InboundCounters();
+ private final InboundCounters smallCounters = new InboundCounters();
+ private final InboundCounters largeCounters = new InboundCounters();
+ private final InboundCounters legacyCounters = new InboundCounters();
+
+ private final InboundMessageCallbacks urgentCallbacks;
+ private final InboundMessageCallbacks smallCallbacks;
+ private final InboundMessageCallbacks largeCallbacks;
+ private final InboundMessageCallbacks legacyCallbacks;
+
+ private final InternodeInboundMetrics metrics;
+ private final MessageConsumer messageConsumer;
+
+ private final HandlerProvider handlerProvider;
+ private final Collection<InboundMessageHandler> handlers = new CopyOnWriteArrayList<>();
+
+ static class GlobalResourceLimits
+ {
+ final ResourceLimits.Limit reserveCapacity;
+ final InboundMessageHandler.WaitQueue waitQueue;
+
+ GlobalResourceLimits(ResourceLimits.Limit reserveCapacity)
+ {
+ this.reserveCapacity = reserveCapacity;
+ this.waitQueue = InboundMessageHandler.WaitQueue.global(reserveCapacity);
+ }
+ }
+
+ public interface MessageConsumer extends Consumer<Message<?>>
+ {
+ void fail(Message.Header header, Throwable failure);
+ }
+
+ public interface GlobalMetricCallbacks
+ {
+ LatencyConsumer internodeLatencyRecorder(InetAddressAndPort to);
+ void recordInternalLatency(Verb verb, long timeElapsed, TimeUnit timeUnit);
+ void recordInternodeDroppedMessage(Verb verb, long timeElapsed, TimeUnit timeUnit);
+ }
+
+ public InboundMessageHandlers(InetAddressAndPort self,
+ InetAddressAndPort peer,
+ int queueCapacity,
+ long endpointReserveCapacity,
+ GlobalResourceLimits globalResourceLimits,
+ GlobalMetricCallbacks globalMetricCallbacks,
+ MessageConsumer messageConsumer)
+ {
+ this(self, peer, queueCapacity, endpointReserveCapacity, globalResourceLimits, globalMetricCallbacks, messageConsumer, InboundMessageHandler::new);
+ }
+
+ public InboundMessageHandlers(InetAddressAndPort self,
+ InetAddressAndPort peer,
+ int queueCapacity,
+ long endpointReserveCapacity,
+ GlobalResourceLimits globalResourceLimits,
+ GlobalMetricCallbacks globalMetricCallbacks,
+ MessageConsumer messageConsumer,
+ HandlerProvider handlerProvider)
+ {
+ this.self = self;
+ this.peer = peer;
+
+ this.queueCapacity = queueCapacity;
+ this.endpointReserveCapacity = new ResourceLimits.Concurrent(endpointReserveCapacity);
+ this.globalReserveCapacity = globalResourceLimits.reserveCapacity;
+ this.endpointWaitQueue = InboundMessageHandler.WaitQueue.endpoint(this.endpointReserveCapacity);
+ this.globalWaitQueue = globalResourceLimits.waitQueue;
+ this.messageConsumer = messageConsumer;
+
+ this.handlerProvider = handlerProvider;
+
+ urgentCallbacks = makeMessageCallbacks(peer, urgentCounters, globalMetricCallbacks, messageConsumer);
+ smallCallbacks = makeMessageCallbacks(peer, smallCounters, globalMetricCallbacks, messageConsumer);
+ largeCallbacks = makeMessageCallbacks(peer, largeCounters, globalMetricCallbacks, messageConsumer);
+ legacyCallbacks = makeMessageCallbacks(peer, legacyCounters, globalMetricCallbacks, messageConsumer);
+
+ metrics = new InternodeInboundMetrics(peer, this);
+ }
+
+ InboundMessageHandler createHandler(FrameDecoder frameDecoder, ConnectionType type, Channel channel, int version)
+ {
+ InboundMessageHandler handler =
+ handlerProvider.provide(frameDecoder,
+
+ type,
+ channel,
+ self,
+ peer,
+ version,
+ OutboundConnections.LARGE_MESSAGE_THRESHOLD,
+
+ queueCapacity,
+ endpointReserveCapacity,
+ globalReserveCapacity,
+ endpointWaitQueue,
+ globalWaitQueue,
+
+ this::onHandlerClosed,
+ callbacksFor(type),
+ messageConsumer);
+ handlers.add(handler);
+ return handler;
+ }
+
+ void releaseMetrics()
+ {
+ metrics.release();
+ }
+
+ private void onHandlerClosed(InboundMessageHandler handler)
+ {
+ handlers.remove(handler);
+ absorbCounters(handler);
+ }
+
+ /*
+ * Message callbacks
+ */
+
+ private InboundMessageCallbacks callbacksFor(ConnectionType type)
+ {
+ switch (type)
+ {
+ case URGENT_MESSAGES: return urgentCallbacks;
+ case SMALL_MESSAGES: return smallCallbacks;
+ case LARGE_MESSAGES: return largeCallbacks;
+ case LEGACY_MESSAGES: return legacyCallbacks;
+ }
+
+ throw new IllegalArgumentException();
+ }
+
+ private static InboundMessageCallbacks makeMessageCallbacks(InetAddressAndPort peer, InboundCounters counters, GlobalMetricCallbacks globalMetrics, MessageConsumer messageConsumer)
+ {
+ LatencyConsumer internodeLatency = globalMetrics.internodeLatencyRecorder(peer);
+
+ return new InboundMessageCallbacks()
+ {
+ @Override
+ public void onHeaderArrived(int messageSize, Header header, long timeElapsed, TimeUnit unit)
+ {
+ // do not log latency if we are within error bars of zero
+ if (timeElapsed > unit.convert(approxTime.error(), NANOSECONDS))
+ internodeLatency.accept(timeElapsed, unit);
+ }
+
+ @Override
+ public void onArrived(int messageSize, Header header, long timeElapsed, TimeUnit unit)
+ {
+ }
+
+ @Override
+ public void onArrivedExpired(int messageSize, Header header, boolean wasCorrupt, long timeElapsed, TimeUnit unit)
+ {
+ counters.addExpired(messageSize);
+
+ globalMetrics.recordInternodeDroppedMessage(header.verb, timeElapsed, unit);
+ }
+
+ @Override
+ public void onArrivedCorrupt(int messageSize, Header header, long timeElapsed, TimeUnit unit)
+ {
+ counters.addError(messageSize);
+
+ messageConsumer.fail(header, new Crc.InvalidCrc(0, 0)); // could use one of the original exceptions?
+ }
+
+ @Override
+ public void onClosedBeforeArrival(int messageSize, Header header, int bytesReceived, boolean wasCorrupt, boolean wasExpired)
+ {
+ counters.addError(messageSize);
+
+ messageConsumer.fail(header, new InvalidSerializedSizeException(header.verb, messageSize, bytesReceived));
+ }
+
+ @Override
+ public void onExpired(int messageSize, Header header, long timeElapsed, TimeUnit unit)
+ {
+ counters.addExpired(messageSize);
+
+ globalMetrics.recordInternodeDroppedMessage(header.verb, timeElapsed, unit);
+ }
+
+ @Override
+ public void onFailedDeserialize(int messageSize, Header header, Throwable t)
+ {
+ counters.addError(messageSize);
+
+ /*
+ * If an exception is caught during deser, return a failure response immediately
+ * instead of waiting for the callback on the other end to expire.
+ */
+ messageConsumer.fail(header, t);
+ }
+
+ @Override
+ public void onDispatched(int messageSize, Header header)
+ {
+ counters.addPending(messageSize);
+ }
+
+ @Override
+ public void onExecuting(int messageSize, Header header, long timeElapsed, TimeUnit unit)
+ {
+ globalMetrics.recordInternalLatency(header.verb, timeElapsed, unit);
+ }
+
+ @Override
+ public void onExecuted(int messageSize, Header header, long timeElapsed, TimeUnit unit)
+ {
+ counters.removePending(messageSize);
+ }
+
+ @Override
+ public void onProcessed(int messageSize, Header header)
+ {
+ counters.addProcessed(messageSize);
+ }
+ };
+ }
+
+ /*
+ * Aggregated counters
+ */
+
+ InboundCounters countersFor(ConnectionType type)
+ {
+ switch (type)
+ {
+ case URGENT_MESSAGES: return urgentCounters;
+ case SMALL_MESSAGES: return smallCounters;
+ case LARGE_MESSAGES: return largeCounters;
+ case LEGACY_MESSAGES: return legacyCounters;
+ }
+
+ throw new IllegalArgumentException();
+ }
+
+ public long receivedCount()
+ {
+ return sumHandlers(h -> h.receivedCount) + closedReceivedCount;
+ }
+
+ public long receivedBytes()
+ {
+ return sumHandlers(h -> h.receivedBytes) + closedReceivedBytes;
+ }
+
+ public long throttledCount()
+ {
+ return sumHandlers(h -> h.throttledCount) + closedThrottledCount;
+ }
+
+ public long throttledNanos()
+ {
+ return sumHandlers(h -> h.throttledNanos) + closedThrottledNanos;
+ }
+
+ public long usingCapacity()
+ {
+ return sumHandlers(h -> h.queueSize);
+ }
+
+ public long usingEndpointReserveCapacity()
+ {
+ return endpointReserveCapacity.using();
+ }
+
+ public long corruptFramesRecovered()
+ {
+ return sumHandlers(h -> h.corruptFramesRecovered) + closedCorruptFramesRecovered;
+ }
+
+ public long corruptFramesUnrecovered()
+ {
+ return sumHandlers(h -> h.corruptFramesUnrecovered) + closedCorruptFramesUnrecovered;
+ }
+
+ public long errorCount()
+ {
+ return sumCounters(InboundCounters::errorCount);
+ }
+
+ public long errorBytes()
+ {
+ return sumCounters(InboundCounters::errorBytes);
+ }
+
+ public long expiredCount()
+ {
+ return sumCounters(InboundCounters::expiredCount);
+ }
+
+ public long expiredBytes()
+ {
+ return sumCounters(InboundCounters::expiredBytes);
+ }
+
+ public long processedCount()
+ {
+ return sumCounters(InboundCounters::processedCount);
+ }
+
+ public long processedBytes()
+ {
+ return sumCounters(InboundCounters::processedBytes);
+ }
+
+ public long scheduledCount()
+ {
+ return sumCounters(InboundCounters::scheduledCount);
+ }
+
+ public long scheduledBytes()
+ {
+ return sumCounters(InboundCounters::scheduledBytes);
+ }
+
+ /*
+ * 'Archived' counter values, combined for all connections that have been closed.
+ */
+
+ private volatile long closedReceivedCount, closedReceivedBytes;
+
+ private static final AtomicLongFieldUpdater<InboundMessageHandlers> closedReceivedCountUpdater =
+ AtomicLongFieldUpdater.newUpdater(InboundMessageHandlers.class, "closedReceivedCount");
+ private static final AtomicLongFieldUpdater<InboundMessageHandlers> closedReceivedBytesUpdater =
+ AtomicLongFieldUpdater.newUpdater(InboundMessageHandlers.class, "closedReceivedBytes");
+
+ private volatile long closedThrottledCount, closedThrottledNanos;
+
+ private static final AtomicLongFieldUpdater<InboundMessageHandlers> closedThrottledCountUpdater =
+ AtomicLongFieldUpdater.newUpdater(InboundMessageHandlers.class, "closedThrottledCount");
+ private static final AtomicLongFieldUpdater<InboundMessageHandlers> closedThrottledNanosUpdater =
+ AtomicLongFieldUpdater.newUpdater(InboundMessageHandlers.class, "closedThrottledNanos");
+
+ private volatile long closedCorruptFramesRecovered, closedCorruptFramesUnrecovered;
+
+ private static final AtomicLongFieldUpdater<InboundMessageHandlers> closedCorruptFramesRecoveredUpdater =
+ AtomicLongFieldUpdater.newUpdater(InboundMessageHandlers.class, "closedCorruptFramesRecovered");
+ private static final AtomicLongFieldUpdater<InboundMessageHandlers> closedCorruptFramesUnrecoveredUpdater =
+ AtomicLongFieldUpdater.newUpdater(InboundMessageHandlers.class, "closedCorruptFramesUnrecovered");
+
+ private void absorbCounters(InboundMessageHandler handler)
+ {
+ closedReceivedCountUpdater.addAndGet(this, handler.receivedCount);
+ closedReceivedBytesUpdater.addAndGet(this, handler.receivedBytes);
+
+ closedThrottledCountUpdater.addAndGet(this, handler.throttledCount);
+ closedThrottledNanosUpdater.addAndGet(this, handler.throttledNanos);
+
+ closedCorruptFramesRecoveredUpdater.addAndGet(this, handler.corruptFramesRecovered);
+ closedCorruptFramesUnrecoveredUpdater.addAndGet(this, handler.corruptFramesUnrecovered);
+ }
+
+ private long sumHandlers(ToLongFunction<InboundMessageHandler> counter)
+ {
+ long sum = 0L;
+ for (InboundMessageHandler h : handlers)
+ sum += counter.applyAsLong(h);
+ return sum;
+ }
+
+ private long sumCounters(ToLongFunction<InboundCounters> mapping)
+ {
+ return mapping.applyAsLong(urgentCounters)
+ + mapping.applyAsLong(smallCounters)
+ + mapping.applyAsLong(largeCounters)
+ + mapping.applyAsLong(legacyCounters);
+ }
+
+ interface HandlerProvider
+ {
+ InboundMessageHandler provide(FrameDecoder decoder,
+
+ ConnectionType type,
+ Channel channel,
+ InetAddressAndPort self,
+ InetAddressAndPort peer,
+ int version,
+ int largeMessageThreshold,
+
+ int queueCapacity,
+ ResourceLimits.Limit endpointReserveCapacity,
+ ResourceLimits.Limit globalReserveCapacity,
+ InboundMessageHandler.WaitQueue endpointWaitQueue,
+ InboundMessageHandler.WaitQueue globalWaitQueue,
+
+ InboundMessageHandler.OnHandlerClosed onClosed,
+ InboundMessageCallbacks callbacks,
+ Consumer<Message<?>> consumer);
+ }
+}
diff --git a/src/java/org/apache/cassandra/net/InboundSink.java b/src/java/org/apache/cassandra/net/InboundSink.java
new file mode 100644
index 0000000..df63be2
--- /dev/null
+++ b/src/java/org/apache/cassandra/net/InboundSink.java
@@ -0,0 +1,161 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.net;
+
+import java.io.IOException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReferenceFieldUpdater;
+import java.util.function.Predicate;
+
+import org.slf4j.LoggerFactory;
+
+import net.openhft.chronicle.core.util.ThrowingConsumer;
+import org.apache.cassandra.db.filter.TombstoneOverwhelmingException;
+import org.apache.cassandra.exceptions.RequestFailureReason;
+import org.apache.cassandra.index.IndexNotAvailableException;
+import org.apache.cassandra.utils.NoSpamLogger;
+
+/**
+ * A message sink that all inbound messages go through.
+ *
+ * Default sink used by {@link MessagingService} is {@link IVerbHandler#doVerb(Message)}, but it can be overridden
+ * to filter out certain messages, record the fact of attempted delivery, or delay arrival.
+ *
+ * This facility is most useful for test code.
+ *
+ * {@link #accept(Message)} is invoked on a thread belonging to the {@link org.apache.cassandra.concurrent.Stage}
+ * assigned to the {@link Verb} of the message.
+ */
+public class InboundSink implements InboundMessageHandlers.MessageConsumer
+{
+ private static final NoSpamLogger noSpamLogger =
+ NoSpamLogger.getLogger(LoggerFactory.getLogger(InboundSink.class), 1L, TimeUnit.SECONDS);
+
+ private static class Filtered implements ThrowingConsumer<Message<?>, IOException>
+ {
+ final Predicate<Message<?>> condition;
+ final ThrowingConsumer<Message<?>, IOException> next;
+
+ private Filtered(Predicate<Message<?>> condition, ThrowingConsumer<Message<?>, IOException> next)
+ {
+ this.condition = condition;
+ this.next = next;
+ }
+
+ public void accept(Message<?> message) throws IOException
+ {
+ if (condition.test(message))
+ next.accept(message);
+ }
+ }
+
+ @SuppressWarnings("FieldMayBeFinal")
+ private volatile ThrowingConsumer<Message<?>, IOException> sink;
+ private static final AtomicReferenceFieldUpdater<InboundSink, ThrowingConsumer> sinkUpdater
+ = AtomicReferenceFieldUpdater.newUpdater(InboundSink.class, ThrowingConsumer.class, "sink");
+
+ private final MessagingService messaging;
+
+ InboundSink(MessagingService messaging)
+ {
+ this.messaging = messaging;
+ this.sink = message -> message.header.verb.handler().doVerb((Message<Object>) message);
+ }
+
+ public void fail(Message.Header header, Throwable failure)
+ {
+ if (header.callBackOnFailure())
+ {
+ Message response = Message.failureResponse(header.id, header.expiresAtNanos, RequestFailureReason.forException(failure));
+ messaging.send(response, header.from);
+ }
+ }
+
+ public void accept(Message<?> message)
+ {
+ try
+ {
+ sink.accept(message);
+ }
+ catch (Throwable t)
+ {
+ fail(message.header, t);
+
+ if (t instanceof TombstoneOverwhelmingException || t instanceof IndexNotAvailableException)
+ noSpamLogger.error(t.getMessage());
+ else if (t instanceof RuntimeException)
+ throw (RuntimeException) t;
+ else
+ throw new RuntimeException(t);
+ }
+ }
+
+ public void add(Predicate<Message<?>> allow)
+ {
+ sinkUpdater.updateAndGet(this, sink -> new Filtered(allow, sink));
+ }
+
+ public void remove(Predicate<Message<?>> allow)
+ {
+ sinkUpdater.updateAndGet(this, sink -> without(sink, allow));
+ }
+
+ public void clear()
+ {
+ sinkUpdater.updateAndGet(this, InboundSink::clear);
+ }
+
+ @Deprecated // TODO: this is not the correct way to do things
+ public boolean allow(Message<?> message)
+ {
+ return allows(sink, message);
+ }
+
+ private static ThrowingConsumer<Message<?>, IOException> clear(ThrowingConsumer<Message<?>, IOException> sink)
+ {
+ while (sink instanceof Filtered)
+ sink = ((Filtered) sink).next;
+ return sink;
+ }
+
+ private static ThrowingConsumer<Message<?>, IOException> without(ThrowingConsumer<Message<?>, IOException> sink, Predicate<Message<?>> condition)
+ {
+ if (!(sink instanceof Filtered))
+ return sink;
+
+ Filtered filtered = (Filtered) sink;
+ ThrowingConsumer<Message<?>, IOException> next = without(filtered.next, condition);
+ return condition.equals(filtered.condition) ? next
+ : next == filtered.next
+ ? sink
+ : new Filtered(filtered.condition, next);
+ }
+
+ private static boolean allows(ThrowingConsumer<Message<?>, IOException> sink, Message<?> message)
+ {
+ while (sink instanceof Filtered)
+ {
+ Filtered filtered = (Filtered) sink;
+ if (!filtered.condition.test(message))
+ return false;
+ sink = filtered.next;
+ }
+ return true;
+ }
+
+}
diff --git a/src/java/org/apache/cassandra/net/InboundSockets.java b/src/java/org/apache/cassandra/net/InboundSockets.java
new file mode 100644
index 0000000..6fc5f52
--- /dev/null
+++ b/src/java/org/apache/cassandra/net/InboundSockets.java
@@ -0,0 +1,257 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.net;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.function.Consumer;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableList;
+
+import io.netty.channel.Channel;
+import io.netty.channel.ChannelFuture;
+import io.netty.channel.ChannelPipeline;
+import io.netty.channel.group.ChannelGroup;
+import io.netty.channel.group.DefaultChannelGroup;
+import io.netty.util.concurrent.DefaultEventExecutor;
+import io.netty.util.concurrent.Future;
+import io.netty.util.concurrent.GlobalEventExecutor;
+import io.netty.util.concurrent.PromiseNotifier;
+import io.netty.util.concurrent.SucceededFuture;
+import org.apache.cassandra.concurrent.NamedThreadFactory;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.utils.FBUtilities;
+
+class InboundSockets
+{
+ /**
+ * A simple struct to wrap up the components needed for each listening socket.
+ */
+ @VisibleForTesting
+ static class InboundSocket
+ {
+ public final InboundConnectionSettings settings;
+
+ /**
+ * The base {@link Channel} that is doing the socket listen/accept.
+ * Null only until open() is invoked and {@link #binding} has yet to complete.
+ */
+ private volatile Channel listen;
+ /**
+ * Once open() is invoked, this holds the future result of opening the socket,
+ * so that its completion can be waited on. Once complete, it sets itself to null.
+ */
+ private volatile ChannelFuture binding;
+
+ // purely to prevent close racing with open
+ private boolean closedWithoutOpening;
+
+ /**
+ * A group of the open, inbound {@link Channel}s connected to this node. This is mostly interesting so that all of
+ * the inbound connections/channels can be closed when the listening socket itself is being closed.
+ */
+ private final ChannelGroup connections;
+ private final DefaultEventExecutor executor;
+
+ private InboundSocket(InboundConnectionSettings settings)
+ {
+ this.settings = settings;
+ this.executor = new DefaultEventExecutor(new NamedThreadFactory("Listen-" + settings.bindAddress));
+ this.connections = new DefaultChannelGroup(settings.bindAddress.toString(), executor);
+ }
+
+ private Future<Void> open()
+ {
+ return open(pipeline -> {});
+ }
+
+ private Future<Void> open(Consumer<ChannelPipeline> pipelineInjector)
+ {
+ synchronized (this)
+ {
+ if (listen != null)
+ return new SucceededFuture<>(GlobalEventExecutor.INSTANCE, null);
+ if (binding != null)
+ return binding;
+ if (closedWithoutOpening)
+ throw new IllegalStateException();
+ binding = InboundConnectionInitiator.bind(settings, connections, pipelineInjector);
+ }
+
+ return binding.addListener(ignore -> {
+ synchronized (this)
+ {
+ if (binding.isSuccess())
+ listen = binding.channel();
+ binding = null;
+ }
+ });
+ }
+
+ /**
+ * Close this socket and any connections created on it. Once closed, this socket may not be re-opened.
+ *
+ * This may not execute synchronously, so a Future is returned encapsulating its result.
+ * @param shutdownExecutors
+ */
+ private Future<Void> close(Consumer<? super ExecutorService> shutdownExecutors)
+ {
+ AsyncPromise<Void> done = AsyncPromise.uncancellable(GlobalEventExecutor.INSTANCE);
+
+ Runnable close = () -> {
+ List<Future<Void>> closing = new ArrayList<>();
+ if (listen != null)
+ closing.add(listen.close());
+ closing.add(connections.close());
+ new FutureCombiner(closing)
+ .addListener(future -> {
+ executor.shutdownGracefully();
+ shutdownExecutors.accept(executor);
+ })
+ .addListener(new PromiseNotifier<>(done));
+ };
+
+ synchronized (this)
+ {
+ if (listen == null && binding == null)
+ {
+ closedWithoutOpening = true;
+ return new SucceededFuture<>(GlobalEventExecutor.INSTANCE, null);
+ }
+
+ if (listen != null)
+ {
+ close.run();
+ }
+ else
+ {
+ binding.cancel(true);
+ binding.addListener(future -> close.run());
+ }
+
+ return done;
+ }
+ }
+
+ public boolean isOpen()
+ {
+ return listen != null && listen.isOpen();
+ }
+ }
+
+ private final List<InboundSocket> sockets;
+
+ InboundSockets(InboundConnectionSettings template)
+ {
+ this(withDefaultBindAddresses(template));
+ }
+
+ InboundSockets(List<InboundConnectionSettings> templates)
+ {
+ this.sockets = bindings(templates);
+ }
+
+ private static List<InboundConnectionSettings> withDefaultBindAddresses(InboundConnectionSettings template)
+ {
+ ImmutableList.Builder<InboundConnectionSettings> templates = ImmutableList.builder();
+ templates.add(template.withBindAddress(FBUtilities.getLocalAddressAndPort()));
+ if (shouldListenOnBroadcastAddress())
+ templates.add(template.withBindAddress(FBUtilities.getBroadcastAddressAndPort()));
+ return templates.build();
+ }
+
+ private static List<InboundSocket> bindings(List<InboundConnectionSettings> templates)
+ {
+ ImmutableList.Builder<InboundSocket> sockets = ImmutableList.builder();
+ for (InboundConnectionSettings template : templates)
+ addBindings(template, sockets);
+ return sockets.build();
+ }
+
+ private static void addBindings(InboundConnectionSettings template, ImmutableList.Builder<InboundSocket> out)
+ {
+ InboundConnectionSettings settings = template.withDefaults();
+ InboundConnectionSettings legacySettings = template.withLegacyDefaults();
+
+ if (settings.encryption.enable_legacy_ssl_storage_port)
+ {
+ out.add(new InboundSocket(legacySettings));
+
+ /*
+ * If the legacy ssl storage port and storage port match, only bind to the
+ * legacy ssl port. This makes it possible to configure a 4.0 node like a 3.0
+ * node with only the ssl_storage_port if required.
+ */
+ if (settings.bindAddress.equals(legacySettings.bindAddress))
+ return;
+ }
+
+ out.add(new InboundSocket(settings));
+ }
+
+ public Future<Void> open(Consumer<ChannelPipeline> pipelineInjector)
+ {
+ List<Future<Void>> opening = new ArrayList<>();
+ for (InboundSocket socket : sockets)
+ opening.add(socket.open(pipelineInjector));
+
+ return new FutureCombiner(opening);
+ }
+
+ public Future<Void> open()
+ {
+ List<Future<Void>> opening = new ArrayList<>();
+ for (InboundSocket socket : sockets)
+ opening.add(socket.open());
+ return new FutureCombiner(opening);
+ }
+
+ public boolean isListening()
+ {
+ for (InboundSocket socket : sockets)
+ if (socket.isOpen())
+ return true;
+ return false;
+ }
+
+ public Future<Void> close(Consumer<? super ExecutorService> shutdownExecutors)
+ {
+ List<Future<Void>> closing = new ArrayList<>();
+ for (InboundSocket address : sockets)
+ closing.add(address.close(shutdownExecutors));
+ return new FutureCombiner(closing);
+ }
+ public Future<Void> close()
+ {
+ return close(e -> {});
+ }
+
+ private static boolean shouldListenOnBroadcastAddress()
+ {
+ return DatabaseDescriptor.shouldListenOnBroadcastAddress()
+ && !FBUtilities.getLocalAddressAndPort().equals(FBUtilities.getBroadcastAddressAndPort());
+ }
+
+ @VisibleForTesting
+ public List<InboundSocket> sockets()
+ {
+ return sockets;
+ }
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/net/IncomingStreamingConnection.java b/src/java/org/apache/cassandra/net/IncomingStreamingConnection.java
deleted file mode 100644
index b97b836..0000000
--- a/src/java/org/apache/cassandra/net/IncomingStreamingConnection.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.net;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.net.Socket;
-import java.util.Set;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.cassandra.io.util.DataInputPlus;
-import org.apache.cassandra.io.util.DataInputPlus.DataInputStreamPlus;
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.streaming.StreamResultFuture;
-import org.apache.cassandra.streaming.messages.StreamInitMessage;
-import org.apache.cassandra.streaming.messages.StreamMessage;
-
-/**
- * Thread to consume stream init messages.
- */
-public class IncomingStreamingConnection extends Thread implements Closeable
-{
- private static final Logger logger = LoggerFactory.getLogger(IncomingStreamingConnection.class);
-
- private final int version;
- public final Socket socket;
- private final Set<Closeable> group;
-
- public IncomingStreamingConnection(int version, Socket socket, Set<Closeable> group)
- {
- super("STREAM-INIT-" + socket.getRemoteSocketAddress());
- this.version = version;
- this.socket = socket;
- this.group = group;
- }
-
- @Override
- @SuppressWarnings("resource") // Not closing constructed DataInputPlus's as the stream needs to remain open.
- public void run()
- {
- try
- {
- // streaming connections are per-session and have a fixed version.
- // we can't do anything with a wrong-version stream connection, so drop it.
- if (version != StreamMessage.CURRENT_VERSION)
- throw new IOException(String.format("Received stream using protocol version %d (my version %d). Terminating connection", version, StreamMessage.CURRENT_VERSION));
-
- DataInputPlus input = new DataInputStreamPlus(socket.getInputStream());
- StreamInitMessage init = StreamInitMessage.serializer.deserialize(input, version);
-
- //Set SO_TIMEOUT on follower side
- if (!init.isForOutgoing)
- socket.setSoTimeout(DatabaseDescriptor.getStreamingSocketTimeout());
-
- // The initiator makes two connections, one for incoming and one for outgoing.
- // The receiving side distinguish two connections by looking at StreamInitMessage#isForOutgoing.
- // Note: we cannot use the same socket for incoming and outgoing streams because we want to
- // parallelize said streams and the socket is blocking, so we might deadlock.
- StreamResultFuture.initReceivingSide(init.sessionIndex, init.planId, init.description, init.from, this, init.isForOutgoing, version, init.keepSSTableLevel, init.isIncremental);
- }
- catch (Throwable t)
- {
- logger.error("Error while reading from socket from {}.", socket.getRemoteSocketAddress(), t);
- close();
- }
- }
-
- @Override
- public void close()
- {
- try
- {
- if (!socket.isClosed())
- {
- socket.close();
- }
- }
- catch (IOException e)
- {
- logger.debug("Error closing socket", e);
- }
- finally
- {
- group.remove(this);
- }
- }
-}
diff --git a/src/java/org/apache/cassandra/net/IncomingTcpConnection.java b/src/java/org/apache/cassandra/net/IncomingTcpConnection.java
deleted file mode 100644
index c96dc6e..0000000
--- a/src/java/org/apache/cassandra/net/IncomingTcpConnection.java
+++ /dev/null
@@ -1,211 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.net;
-
-import java.io.*;
-import java.net.InetAddress;
-import java.net.Socket;
-import java.net.SocketException;
-import java.nio.channels.Channels;
-import java.nio.channels.ReadableByteChannel;
-import java.util.zip.Checksum;
-import java.util.Set;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import io.netty.util.concurrent.FastThreadLocalThread;
-import net.jpountz.lz4.LZ4BlockInputStream;
-import net.jpountz.lz4.LZ4FastDecompressor;
-import net.jpountz.lz4.LZ4Factory;
-import net.jpountz.xxhash.XXHashFactory;
-
-import org.apache.cassandra.config.Config;
-import org.xerial.snappy.SnappyInputStream;
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.db.UnknownColumnFamilyException;
-import org.apache.cassandra.db.monitoring.ApproximateTime;
-import org.apache.cassandra.io.util.DataInputPlus;
-import org.apache.cassandra.io.util.DataInputPlus.DataInputStreamPlus;
-import org.apache.cassandra.io.util.NIODataInputStream;
-
-public class IncomingTcpConnection extends FastThreadLocalThread implements Closeable
-{
- private static final Logger logger = LoggerFactory.getLogger(IncomingTcpConnection.class);
-
- private static final int BUFFER_SIZE = Integer.getInteger(Config.PROPERTY_PREFIX + ".itc_buffer_size", 1024 * 4);
-
- private final int version;
- private final boolean compressed;
- private final Socket socket;
- private final Set<Closeable> group;
- public InetAddress from;
-
- public IncomingTcpConnection(int version, boolean compressed, Socket socket, Set<Closeable> group)
- {
- super("MessagingService-Incoming-" + socket.getInetAddress());
- this.version = version;
- this.compressed = compressed;
- this.socket = socket;
- this.group = group;
- if (DatabaseDescriptor.getInternodeRecvBufferSize() > 0)
- {
- try
- {
- this.socket.setReceiveBufferSize(DatabaseDescriptor.getInternodeRecvBufferSize());
- }
- catch (SocketException se)
- {
- logger.warn("Failed to set receive buffer size on internode socket.", se);
- }
- }
- }
-
- /**
- * A new connection will either stream or message for its entire lifetime: because streaming
- * bypasses the InputStream implementations to use sendFile, we cannot begin buffering until
- * we've determined the type of the connection.
- */
- @Override
- public void run()
- {
- try
- {
- if (version < MessagingService.VERSION_20)
- throw new UnsupportedOperationException(String.format("Unable to read obsolete message version %s; "
- + "The earliest version supported is 2.0.0",
- version));
-
- receiveMessages();
- }
- catch (EOFException e)
- {
- logger.trace("eof reading from socket; closing", e);
- // connection will be reset so no need to throw an exception.
- }
- catch (UnknownColumnFamilyException e)
- {
- logger.warn("UnknownColumnFamilyException reading from socket; closing", e);
- }
- catch (IOException e)
- {
- logger.trace("IOException reading from socket; closing", e);
- }
- finally
- {
- close();
- }
- }
-
- @Override
- public void close()
- {
- try
- {
- if (logger.isTraceEnabled())
- logger.trace("Closing socket {} - isclosed: {}", socket, socket.isClosed());
- if (!socket.isClosed())
- {
- socket.close();
- }
- }
- catch (IOException e)
- {
- logger.trace("Error closing socket", e);
- }
- finally
- {
- group.remove(this);
- }
- }
-
- @SuppressWarnings("resource") // Not closing constructed DataInputPlus's as the stream needs to remain open.
- private void receiveMessages() throws IOException
- {
- // handshake (true) endpoint versions
- DataOutputStream out = new DataOutputStream(socket.getOutputStream());
- // if this version is < the MS version the other node is trying
- // to connect with, the other node will disconnect
- out.writeInt(MessagingService.current_version);
- out.flush();
-
- // outbound side will reconnect if necessary to upgrade version
- if (version > MessagingService.current_version)
- throw new IOException("Peer-used messaging version " + version + " is larger than max supported " + MessagingService.current_version);
-
- DataInputPlus in = new DataInputStreamPlus(socket.getInputStream());
- int maxVersion = in.readInt();
- from = CompactEndpointSerializationHelper.deserialize(in);
- // record the (true) version of the endpoint
- MessagingService.instance().setVersion(from, maxVersion);
- logger.trace("Set version for {} to {} (will use {})", from, maxVersion, MessagingService.instance().getVersion(from));
-
- if (compressed)
- {
- logger.trace("Upgrading incoming connection to be compressed");
- if (version < MessagingService.VERSION_21)
- {
- in = new DataInputStreamPlus(new SnappyInputStream(socket.getInputStream()));
- }
- else
- {
- LZ4FastDecompressor decompressor = LZ4Factory.fastestInstance().fastDecompressor();
- Checksum checksum = XXHashFactory.fastestInstance().newStreamingHash32(OutboundTcpConnection.LZ4_HASH_SEED).asChecksum();
- in = new DataInputStreamPlus(new LZ4BlockInputStream(socket.getInputStream(),
- decompressor,
- checksum));
- }
- }
- else
- {
- ReadableByteChannel channel = socket.getChannel();
- in = new NIODataInputStream(channel != null ? channel : Channels.newChannel(socket.getInputStream()), BUFFER_SIZE);
- }
-
- while (true)
- {
- MessagingService.validateMagic(in.readInt());
- receiveMessage(in, version);
- }
- }
-
- private InetAddress receiveMessage(DataInputPlus input, int version) throws IOException
- {
- int id;
- if (version < MessagingService.VERSION_20)
- id = Integer.parseInt(input.readUTF());
- else
- id = input.readInt();
- long currentTime = ApproximateTime.currentTimeMillis();
- MessageIn message = MessageIn.read(input, version, id, MessageIn.readConstructionTime(from, input, currentTime));
- if (message == null)
- {
- // callback expired; nothing to do
- return null;
- }
- if (version <= MessagingService.current_version)
- {
- MessagingService.instance().receive(message, id);
- }
- else
- {
- logger.trace("Received connection from newer protocol version {}. Ignoring message", version);
- }
- return message.from;
- }
-}
diff --git a/src/java/org/apache/cassandra/net/InvalidSerializedSizeException.java b/src/java/org/apache/cassandra/net/InvalidSerializedSizeException.java
new file mode 100644
index 0000000..5660fd1
--- /dev/null
+++ b/src/java/org/apache/cassandra/net/InvalidSerializedSizeException.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.net;
+
+import java.io.IOException;
+
+import static java.lang.String.format;
+
+class InvalidSerializedSizeException extends IOException
+{
+ final Verb verb;
+ final long expectedSize;
+ final long actualSizeAtLeast;
+
+ InvalidSerializedSizeException(Verb verb, long expectedSize, long actualSizeAtLeast)
+ {
+ super(format("Invalid serialized size; expected %d, actual size at least %d, for verb %s", expectedSize, actualSizeAtLeast, verb));
+ this.verb = verb;
+ this.expectedSize = expectedSize;
+ this.actualSizeAtLeast = actualSizeAtLeast;
+ }
+
+ InvalidSerializedSizeException(long expectedSize, long actualSizeAtLeast)
+ {
+ super(format("Invalid serialized size; expected %d, actual size at least %d", expectedSize, actualSizeAtLeast));
+ this.verb = null;
+ this.expectedSize = expectedSize;
+ this.actualSizeAtLeast = actualSizeAtLeast;
+ }
+}
diff --git a/src/java/org/apache/cassandra/net/LatencyConsumer.java b/src/java/org/apache/cassandra/net/LatencyConsumer.java
new file mode 100644
index 0000000..3f10d41
--- /dev/null
+++ b/src/java/org/apache/cassandra/net/LatencyConsumer.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.net;
+
+import java.util.concurrent.TimeUnit;
+
+public interface LatencyConsumer
+{
+ void accept(long timeElapsed, TimeUnit unit);
+}
diff --git a/src/java/org/apache/cassandra/net/LatencySubscribers.java b/src/java/org/apache/cassandra/net/LatencySubscribers.java
new file mode 100644
index 0000000..823e6d0
--- /dev/null
+++ b/src/java/org/apache/cassandra/net/LatencySubscribers.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.net;
+
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReferenceFieldUpdater;
+
+import org.apache.cassandra.locator.InetAddressAndPort;
+
+/**
+ * Callback that {@link org.apache.cassandra.locator.DynamicEndpointSnitch} listens to in order
+ * to update host scores.
+ *
+ * FIXME: rename/specialise, since only used by DES?
+ */
+public class LatencySubscribers
+{
+ public interface Subscriber
+ {
+ void receiveTiming(InetAddressAndPort address, long latency, TimeUnit unit);
+ }
+
+ private volatile Subscriber subscribers;
+ private static final AtomicReferenceFieldUpdater<LatencySubscribers, Subscriber> subscribersUpdater
+ = AtomicReferenceFieldUpdater.newUpdater(LatencySubscribers.class, Subscriber.class, "subscribers");
+
+ private static Subscriber merge(Subscriber a, Subscriber b)
+ {
+ if (a == null) return b;
+ if (b == null) return a;
+ return (address, latency, unit) -> {
+ a.receiveTiming(address, latency, unit);
+ b.receiveTiming(address, latency, unit);
+ };
+ }
+
+ public void subscribe(Subscriber subscriber)
+ {
+ subscribersUpdater.accumulateAndGet(this, subscriber, LatencySubscribers::merge);
+ }
+
+ public void add(InetAddressAndPort address, long latency, TimeUnit unit)
+ {
+ Subscriber subscribers = this.subscribers;
+ if (subscribers != null)
+ subscribers.receiveTiming(address, latency, unit);
+ }
+
+ /**
+ * Track latency information for the dynamic snitch
+ *
+ * @param cb the callback associated with this message -- this lets us know if it's a message type we're interested in
+ * @param address the host that replied to the message
+ */
+ public void maybeAdd(RequestCallback cb, InetAddressAndPort address, long latency, TimeUnit unit)
+ {
+ if (cb.trackLatencyForSnitch())
+ add(address, latency, unit);
+ }
+}
diff --git a/src/java/org/apache/cassandra/net/LegacyFlag.java b/src/java/org/apache/cassandra/net/LegacyFlag.java
new file mode 100644
index 0000000..b2781a1
--- /dev/null
+++ b/src/java/org/apache/cassandra/net/LegacyFlag.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.net;
+
+import java.io.IOException;
+
+import com.google.common.base.Preconditions;
+
+import org.apache.cassandra.io.IVersionedSerializer;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+
+/**
+ * Before 4.0 introduced flags field to {@link Message}, we used to encode flags in params field,
+ * using a dummy value (single byte set to 0). From now on, {@link MessageFlag} should be extended
+ * instead.
+ *
+ * Once 3.0/3.11 compatibility is phased out, this class should be removed.
+ */
+@Deprecated
+final class LegacyFlag
+{
+ static final LegacyFlag instance = new LegacyFlag();
+
+ private LegacyFlag()
+ {
+ }
+
+ static IVersionedSerializer<LegacyFlag> serializer = new IVersionedSerializer<LegacyFlag>()
+ {
+ public void serialize(LegacyFlag param, DataOutputPlus out, int version) throws IOException
+ {
+ Preconditions.checkArgument(param == instance);
+ out.write(0);
+ }
+
+ public LegacyFlag deserialize(DataInputPlus in, int version) throws IOException
+ {
+ byte b = in.readByte();
+ assert b == 0;
+ return instance;
+ }
+
+ public long serializedSize(LegacyFlag param, int version)
+ {
+ Preconditions.checkArgument(param == instance);
+ return 1;
+ }
+ };
+}
diff --git a/src/java/org/apache/cassandra/net/LegacyLZ4Constants.java b/src/java/org/apache/cassandra/net/LegacyLZ4Constants.java
new file mode 100644
index 0000000..f4fca44
--- /dev/null
+++ b/src/java/org/apache/cassandra/net/LegacyLZ4Constants.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.net;
+
+abstract class LegacyLZ4Constants
+{
+ static final int XXHASH_SEED = 0x9747B28C;
+
+ static final int HEADER_LENGTH = 8 // magic number
+ + 1 // token
+ + 4 // compressed length
+ + 4 // uncompressed length
+ + 4; // checksum
+
+ static final long MAGIC_NUMBER = (long) 'L' << 56
+ | (long) 'Z' << 48
+ | (long) '4' << 40
+ | (long) 'B' << 32
+ | 'l' << 24
+ | 'o' << 16
+ | 'c' << 8
+ | 'k';
+
+ // offsets of header fields
+ static final int MAGIC_NUMBER_OFFSET = 0;
+ static final int TOKEN_OFFSET = 8;
+ static final int COMPRESSED_LENGTH_OFFSET = 9;
+ static final int UNCOMPRESSED_LENGTH_OFFSET = 13;
+ static final int CHECKSUM_OFFSET = 17;
+
+ static final int DEFAULT_BLOCK_LENGTH = 1 << 15; // 32 KiB
+ static final int MAX_BLOCK_LENGTH = 1 << 25; // 32 MiB
+
+ static final int BLOCK_TYPE_NON_COMPRESSED = 0x10;
+ static final int BLOCK_TYPE_COMPRESSED = 0x20;
+
+ // xxhash to Checksum adapter discards most significant nibble of value ¯\_(ツ)_/¯
+ static final int XXHASH_MASK = 0xFFFFFFF;
+}
diff --git a/src/java/org/apache/cassandra/net/LocalBufferPoolAllocator.java b/src/java/org/apache/cassandra/net/LocalBufferPoolAllocator.java
new file mode 100644
index 0000000..b2d487f
--- /dev/null
+++ b/src/java/org/apache/cassandra/net/LocalBufferPoolAllocator.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.net;
+
+import java.nio.ByteBuffer;
+
+import io.netty.channel.EventLoop;
+import org.apache.cassandra.utils.memory.BufferPool;
+
+/**
+ * Equivalent to {@link GlobalBufferPoolAllocator}, except explicitly using a specified
+ * {@link org.apache.cassandra.utils.memory.BufferPool.LocalPool} to allocate from.
+ *
+ * Exists to facilitate more efficient handling large messages on the inbound path,
+ * used by {@link ConnectionType#LARGE_MESSAGES} connections.
+ */
+class LocalBufferPoolAllocator extends BufferPoolAllocator
+{
+ private final BufferPool.LocalPool pool;
+ private final EventLoop eventLoop;
+
+ LocalBufferPoolAllocator(EventLoop eventLoop)
+ {
+ this.pool = new BufferPool.LocalPool().recycleWhenFree(false);
+ this.eventLoop = eventLoop;
+ }
+
+ @Override
+ ByteBuffer get(int size)
+ {
+ if (!eventLoop.inEventLoop())
+ throw new IllegalStateException("get() called from outside of owning event loop");
+ return pool.get(size);
+ }
+
+ @Override
+ ByteBuffer getAtLeast(int size)
+ {
+ if (!eventLoop.inEventLoop())
+ throw new IllegalStateException("getAtLeast() called from outside of owning event loop");
+ return pool.getAtLeast(size);
+ }
+
+ @Override
+ public void release()
+ {
+ pool.release();
+ }
+}
diff --git a/src/java/org/apache/cassandra/net/ManyToOneConcurrentLinkedQueue.java b/src/java/org/apache/cassandra/net/ManyToOneConcurrentLinkedQueue.java
new file mode 100644
index 0000000..4c73bdc
--- /dev/null
+++ b/src/java/org/apache/cassandra/net/ManyToOneConcurrentLinkedQueue.java
@@ -0,0 +1,350 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.net;
+
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+import java.util.Queue;
+import java.util.concurrent.atomic.AtomicReferenceFieldUpdater;
+import java.util.function.Consumer;
+
+/**
+ * A concurrent many-producers-to-single-consumer linked queue.
+ *
+ * Based roughly on {@link java.util.concurrent.ConcurrentLinkedQueue}, except with simpler/cheaper consumer-side
+ * method implementations ({@link #poll()}, {@link #remove()}, {@link #drain(Consumer)}), and padding added
+ * to prevent false sharing.
+ *
+ * {@link #offer(Object)} provides volatile visibility semantics. {@link #offer(Object)} is lock-free, {@link #poll()}
+ * and all related consumer methods are wait-free.
+ *
+ * In addition to that, provides a {@link #relaxedPeekLastAndOffer(Object)} method that we use to avoid a CAS when
+ * putting message handlers onto the wait queue.
+ */
+class ManyToOneConcurrentLinkedQueue<E> extends ManyToOneConcurrentLinkedQueueHead<E> implements Queue<E>
+{
+ @SuppressWarnings("unused") // pad two cache lines after the head to prevent false sharing
+ protected long p31, p32, p33, p34, p35, p36, p37, p38, p39, p40, p41, p42, p43, p44, p45;
+
+ ManyToOneConcurrentLinkedQueue()
+ {
+ head = tail = new Node<>(null);
+ }
+
+ /**
+ * See {@link #relaxedIsEmpty()}.
+ */
+ @Override
+ public boolean isEmpty()
+ {
+ return relaxedIsEmpty();
+ }
+
+ /**
+ * When invoked by the consumer thread, the answer will always be accurate.
+ * When invoked by a non-consumer thread, it won't always be the case:
+ * - {@code true} result indicates that the queue <em>IS</em> empty, no matter what;
+ * - {@code false} result indicates that the queue <em>MIGHT BE</em> non-empty - the value of {@code head} might
+ * not yet have been made externally visible by the consumer thread.
+ */
+ boolean relaxedIsEmpty()
+ {
+ return null == head.next;
+ }
+
+ @Override
+ public int size()
+ {
+ int size = 0;
+ Node<E> next = head;
+ while (null != (next = next.next))
+ size++;
+ return size;
+ }
+
+ @Override
+ public E peek()
+ {
+ Node<E> next = head.next;
+ if (null == next)
+ return null;
+ return next.item;
+ }
+
+ @Override
+ public E element()
+ {
+ E item = peek();
+ if (null == item)
+ throw new NoSuchElementException("Queue is empty");
+ return item;
+ }
+
+ @Override
+ public E poll()
+ {
+ Node<E> head = this.head;
+ Node<E> next = head.next;
+
+ if (null == next)
+ return null;
+
+ this.lazySetHead(next); // update head reference to next before making previous head node unreachable,
+ head.lazySetNext(head); // to maintain the guarantee of tail being always reachable from head
+
+ E item = next.item;
+ next.item = null;
+ return item;
+ }
+
+ @Override
+ public E remove()
+ {
+ E item = poll();
+ if (null == item)
+ throw new NoSuchElementException("Queue is empty");
+ return item;
+ }
+
+ @Override
+ public boolean remove(Object o)
+ {
+ if (null == o)
+ throw new NullPointerException();
+
+ Node<E> prev = this.head;
+ Node<E> next = prev.next;
+
+ while (null != next)
+ {
+ if (o.equals(next.item))
+ {
+ prev.lazySetNext(next.next); // update prev reference to next before making removed node unreachable,
+ next.lazySetNext(next); // to maintain the guarantee of tail being always reachable from head
+
+ next.item = null;
+ return true;
+ }
+
+ prev = next;
+ next = next.next;
+ }
+
+ return false;
+ }
+
+ /**
+ * Consume the queue in its entirety and feed every item to the provided {@link Consumer}.
+ *
+ * Exists primarily for convenience, and essentially just wraps {@link #poll()} in a loop.
+ * Yields no performance benefit over invoking {@link #poll()} manually - there just isn't
+ * anything to meaningfully amortise on the consumer side of this queue.
+ */
+ void drain(Consumer<E> consumer)
+ {
+ E item;
+ while ((item = poll()) != null)
+ consumer.accept(item);
+ }
+
+ @Override
+ public boolean add(E e)
+ {
+ return offer(e);
+ }
+
+ @Override
+ public boolean offer(E e)
+ {
+ internalOffer(e); return true;
+ }
+
+ /**
+ * Adds the element to the queue and returns the item of the previous tail node.
+ * It's possible for the returned item to already have been consumed.
+ *
+ * @return previously last tail item in the queue, potentially stale
+ */
+ E relaxedPeekLastAndOffer(E e)
+ {
+ return internalOffer(e);
+ }
+
+ /**
+ * internalOffer() is based on {@link java.util.concurrent.ConcurrentLinkedQueue#offer(Object)},
+ * written by Doug Lea and Martin Buchholz with assistance from members of JCP JSR-166 Expert Group
+ * and released to the public domain, as explained at http://creativecommons.org/publicdomain/zero/1.0/
+ */
+ private E internalOffer(E e)
+ {
+ if (null == e)
+ throw new NullPointerException();
+
+ final Node<E> node = new Node<>(e);
+
+ for (Node<E> t = tail, p = t;;)
+ {
+ Node<E> q = p.next;
+ if (q == null)
+ {
+ // p is last node
+ if (p.casNext(null, node))
+ {
+ // successful CAS is the linearization point for e to become an element of this queue and for node to become "live".
+ if (p != t) // hop two nodes at a time
+ casTail(t, node); // failure is ok
+ return p.item;
+ }
+ // lost CAS race to another thread; re-read next
+ }
+ else if (p == q)
+ {
+ /*
+ * We have fallen off list. If tail is unchanged, it will also be off-list, in which case we need to
+ * jump to head, from which all live nodes are always reachable. Else the new tail is a better bet.
+ */
+ p = (t != (t = tail)) ? t : head;
+ }
+ else
+ {
+ // check for tail updates after two hops
+ p = (p != t && t != (t = tail)) ? t : q;
+ }
+ }
+ }
+
+ @Override
+ public boolean contains(Object o)
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Iterator<E> iterator()
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Object[] toArray()
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public <T> T[] toArray(T[] a)
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean containsAll(Collection<?> c)
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean addAll(Collection<? extends E> c)
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean removeAll(Collection<?> c)
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean retainAll(Collection<?> c)
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void clear()
+ {
+ throw new UnsupportedOperationException();
+ }
+}
+
+class ManyToOneConcurrentLinkedQueueHead<E> extends ManyToOneConcurrentLinkedQueuePadding2<E>
+{
+ protected volatile ManyToOneConcurrentLinkedQueue.Node<E> head;
+
+ private static final AtomicReferenceFieldUpdater<ManyToOneConcurrentLinkedQueueHead, Node> headUpdater =
+ AtomicReferenceFieldUpdater.newUpdater(ManyToOneConcurrentLinkedQueueHead.class, Node.class, "head");
+
+ @SuppressWarnings("WeakerAccess")
+ protected void lazySetHead(Node<E> val)
+ {
+ headUpdater.lazySet(this, val);
+ }
+}
+
+class ManyToOneConcurrentLinkedQueuePadding2<E> extends ManyToOneConcurrentLinkedQueueTail<E>
+{
+ @SuppressWarnings("unused") // pad two cache lines between tail and head to prevent false sharing
+ protected long p16, p17, p18, p19, p20, p21, p22, p23, p24, p25, p26, p27, p28, p29, p30;
+}
+
+class ManyToOneConcurrentLinkedQueueTail<E> extends ManyToOneConcurrentLinkedQueuePadding1
+{
+ protected volatile ManyToOneConcurrentLinkedQueue.Node<E> tail;
+
+ private static final AtomicReferenceFieldUpdater<ManyToOneConcurrentLinkedQueueTail, Node> tailUpdater =
+ AtomicReferenceFieldUpdater.newUpdater(ManyToOneConcurrentLinkedQueueTail.class, Node.class, "tail");
+
+ @SuppressWarnings({ "WeakerAccess", "UnusedReturnValue" })
+ protected boolean casTail(Node<E> expect, Node<E> update)
+ {
+ return tailUpdater.compareAndSet(this, expect, update);
+ }
+}
+
+class ManyToOneConcurrentLinkedQueuePadding1
+{
+ @SuppressWarnings("unused") // pad two cache lines before the tail to prevent false sharing
+ protected long p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14, p15;
+
+ static final class Node<E>
+ {
+ E item;
+ volatile Node<E> next;
+
+ private static final AtomicReferenceFieldUpdater<Node, Node> nextUpdater =
+ AtomicReferenceFieldUpdater.newUpdater(Node.class, Node.class, "next");
+
+ Node(E item)
+ {
+ this.item = item;
+ }
+
+ @SuppressWarnings("SameParameterValue")
+ boolean casNext(Node<E> expect, Node<E> update)
+ {
+ return nextUpdater.compareAndSet(this, expect, update);
+ }
+
+ void lazySetNext(Node<E> val)
+ {
+ nextUpdater.lazySet(this, val);
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/net/Message.java b/src/java/org/apache/cassandra/net/Message.java
new file mode 100644
index 0000000..01ba5d4
--- /dev/null
+++ b/src/java/org/apache/cassandra/net/Message.java
@@ -0,0 +1,1355 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.net;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.EnumMap;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import javax.annotation.Nullable;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.primitives.Ints;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.exceptions.RequestFailureReason;
+import org.apache.cassandra.io.IVersionedAsymmetricSerializer;
+import org.apache.cassandra.io.IVersionedSerializer;
+import org.apache.cassandra.io.util.DataInputBuffer;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.tracing.Tracing;
+import org.apache.cassandra.tracing.Tracing.TraceType;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.MonotonicClockTranslation;
+
+import static java.util.concurrent.TimeUnit.MINUTES;
+import static java.util.concurrent.TimeUnit.NANOSECONDS;
+import static org.apache.cassandra.db.TypeSizes.sizeof;
+import static org.apache.cassandra.db.TypeSizes.sizeofUnsignedVInt;
+import static org.apache.cassandra.locator.InetAddressAndPort.Serializer.inetAddressAndPortSerializer;
+import static org.apache.cassandra.net.MessagingService.VERSION_3014;
+import static org.apache.cassandra.net.MessagingService.VERSION_30;
+import static org.apache.cassandra.net.MessagingService.VERSION_40;
+import static org.apache.cassandra.net.MessagingService.instance;
+import static org.apache.cassandra.utils.MonotonicClock.approxTime;
+import static org.apache.cassandra.utils.vint.VIntCoding.computeUnsignedVIntSize;
+import static org.apache.cassandra.utils.vint.VIntCoding.getUnsignedVInt;
+import static org.apache.cassandra.utils.vint.VIntCoding.skipUnsignedVInt;
+
+/**
+ * Immutable main unit of internode communication - what used to be {@code MessageIn} and {@code MessageOut} fused
+ * in one class.
+ *
+ * @param <T> The type of the message payload.
+ */
+public class Message<T>
+{
+ public final Header header;
+ public final T payload;
+
+ private Message(Header header, T payload)
+ {
+ this.header = header;
+ this.payload = payload;
+ }
+
+ /** Sender of the message. */
+ public InetAddressAndPort from()
+ {
+ return header.from;
+ }
+
+ /** Whether the message has crossed the node boundary, that is whether it originated from another node. */
+ public boolean isCrossNode()
+ {
+ return !from().equals(FBUtilities.getBroadcastAddressAndPort());
+ }
+
+ /**
+ * id of the request/message. In 4.0+ can be shared between multiple messages of the same logical request,
+ * whilst in versions above a new id would be allocated for each message sent.
+ */
+ public long id()
+ {
+ return header.id;
+ }
+
+ public Verb verb()
+ {
+ return header.verb;
+ }
+
+ boolean isFailureResponse()
+ {
+ return verb() == Verb.FAILURE_RSP;
+ }
+
+ /**
+ * Creation time of the message. If cross-node timeouts are enabled ({@link DatabaseDescriptor#hasCrossNodeTimeout()},
+ * {@code deserialize()} will use the marshalled value, otherwise will use current time on the deserializing machine.
+ */
+ public long createdAtNanos()
+ {
+ return header.createdAtNanos;
+ }
+
+ public long expiresAtNanos()
+ {
+ return header.expiresAtNanos;
+ }
+
+ /** For how long the message has lived. */
+ public long elapsedSinceCreated(TimeUnit units)
+ {
+ return units.convert(approxTime.now() - createdAtNanos(), NANOSECONDS);
+ }
+
+ public long creationTimeMillis()
+ {
+ return approxTime.translate().toMillisSinceEpoch(createdAtNanos());
+ }
+
+ /** Whether a failure response should be returned upon failure */
+ boolean callBackOnFailure()
+ {
+ return header.callBackOnFailure();
+ }
+
+ /** See CASSANDRA-14145 */
+ public boolean trackRepairedData()
+ {
+ return header.trackRepairedData();
+ }
+
+ /** Used for cross-DC write optimisation - pick one node in the DC and have it relay the write to its local peers */
+ @Nullable
+ public ForwardingInfo forwardTo()
+ {
+ return header.forwardTo();
+ }
+
+ /** The originator of the request - used when forwarding and will differ from {@link #from()} */
+ @Nullable
+ public InetAddressAndPort respondTo()
+ {
+ return header.respondTo();
+ }
+
+ @Nullable
+ public UUID traceSession()
+ {
+ return header.traceSession();
+ }
+
+ @Nullable
+ public TraceType traceType()
+ {
+ return header.traceType();
+ }
+
+ /*
+ * request/response convenience
+ */
+
+ /**
+ * Make a request {@link Message} with supplied verb and payload. Will fill in remaining fields
+ * automatically.
+ *
+ * If you know that you will need to set some params or flags - prefer using variants of {@code out()}
+ * that allow providing them at point of message constructions, rather than allocating new messages
+ * with those added flags and params. See {@code outWithFlag()}, {@code outWithFlags()}, and {@code outWithParam()}
+ * family.
+ */
+ public static <T> Message<T> out(Verb verb, T payload)
+ {
+ assert !verb.isResponse();
+
+ return outWithParam(nextId(), verb, payload, null, null);
+ }
+
+ public static <T> Message<T> out(Verb verb, T payload, long expiresAtNanos)
+ {
+ return outWithParam(nextId(), verb, expiresAtNanos, payload, 0, null, null);
+ }
+
+ public static <T> Message<T> outWithFlag(Verb verb, T payload, MessageFlag flag)
+ {
+ assert !verb.isResponse();
+ return outWithParam(nextId(), verb, 0, payload, flag.addTo(0), null, null);
+ }
+
+ public static <T> Message<T> outWithFlags(Verb verb, T payload, MessageFlag flag1, MessageFlag flag2)
+ {
+ assert !verb.isResponse();
+ return outWithParam(nextId(), verb, 0, payload, flag2.addTo(flag1.addTo(0)), null, null);
+ }
+
+ static <T> Message<T> outWithParam(long id, Verb verb, T payload, ParamType paramType, Object paramValue)
+ {
+ return outWithParam(id, verb, 0, payload, paramType, paramValue);
+ }
+
+ private static <T> Message<T> outWithParam(long id, Verb verb, long expiresAtNanos, T payload, ParamType paramType, Object paramValue)
+ {
+ return outWithParam(id, verb, expiresAtNanos, payload, 0, paramType, paramValue);
+ }
+
+ private static <T> Message<T> outWithParam(long id, Verb verb, long expiresAtNanos, T payload, int flags, ParamType paramType, Object paramValue)
+ {
+ if (payload == null)
+ throw new IllegalArgumentException();
+
+ InetAddressAndPort from = FBUtilities.getBroadcastAddressAndPort();
+ long createdAtNanos = approxTime.now();
+ if (expiresAtNanos == 0)
+ expiresAtNanos = verb.expiresAtNanos(createdAtNanos);
+
+ return new Message<>(new Header(id, verb, from, createdAtNanos, expiresAtNanos, flags, buildParams(paramType, paramValue)), payload);
+ }
+
+ public static <T> Message<T> internalResponse(Verb verb, T payload)
+ {
+ assert verb.isResponse();
+ return outWithParam(0, verb, payload, null, null);
+ }
+
+ /** Builds a response Message with provided payload, and all the right fields inferred from request Message */
+ public <T> Message<T> responseWith(T payload)
+ {
+ return outWithParam(id(), verb().responseVerb, expiresAtNanos(), payload, null, null);
+ }
+
+ /** Builds a response Message with no payload, and all the right fields inferred from request Message */
+ public Message<NoPayload> emptyResponse()
+ {
+ return responseWith(NoPayload.noPayload);
+ }
+
+ /** Builds a failure response Message with an explicit reason, and fields inferred from request Message */
+ public Message<RequestFailureReason> failureResponse(RequestFailureReason reason)
+ {
+ return failureResponse(id(), expiresAtNanos(), reason);
+ }
+
+ static Message<RequestFailureReason> failureResponse(long id, long expiresAtNanos, RequestFailureReason reason)
+ {
+ return outWithParam(id, Verb.FAILURE_RSP, expiresAtNanos, reason, null, null);
+ }
+
+ Message<T> withCallBackOnFailure()
+ {
+ return new Message<>(header.withFlag(MessageFlag.CALL_BACK_ON_FAILURE), payload);
+ }
+
+ public Message<T> withForwardTo(ForwardingInfo peers)
+ {
+ return new Message<>(header.withParam(ParamType.FORWARD_TO, peers), payload);
+ }
+
+ private static final EnumMap<ParamType, Object> NO_PARAMS = new EnumMap<>(ParamType.class);
+
+ private static Map<ParamType, Object> buildParams(ParamType type, Object value)
+ {
+ Map<ParamType, Object> params = NO_PARAMS;
+ if (Tracing.isTracing())
+ params = Tracing.instance.addTraceHeaders(new EnumMap<>(ParamType.class));
+
+ if (type != null)
+ {
+ if (params.isEmpty())
+ params = new EnumMap<>(ParamType.class);
+ params.put(type, value);
+ }
+
+ return params;
+ }
+
+ private static Map<ParamType, Object> addParam(Map<ParamType, Object> params, ParamType type, Object value)
+ {
+ if (type == null)
+ return params;
+
+ params = new EnumMap<>(params);
+ params.put(type, value);
+ return params;
+ }
+
+ /*
+ * id generation
+ */
+
+ private static final long NO_ID = 0L; // this is a valid ID for pre40 nodes
+
+ private static final AtomicInteger nextId = new AtomicInteger(0);
+
+ private static long nextId()
+ {
+ long id;
+ do
+ {
+ id = nextId.incrementAndGet();
+ }
+ while (id == NO_ID);
+
+ return id;
+ }
+
+ /**
+ * WARNING: this is inaccurate for messages from pre40 nodes, which can use 0 as an id (but will do so rarely)
+ */
+ @VisibleForTesting
+ boolean hasId()
+ {
+ return id() != NO_ID;
+ }
+
+ /** we preface every message with this number so the recipient can validate the sender is sane */
+ static final int PROTOCOL_MAGIC = 0xCA552DFA;
+
+ static void validateLegacyProtocolMagic(int magic) throws InvalidLegacyProtocolMagic
+ {
+ if (magic != PROTOCOL_MAGIC)
+ throw new InvalidLegacyProtocolMagic(magic);
+ }
+
+ public static final class InvalidLegacyProtocolMagic extends IOException
+ {
+ public final int read;
+ private InvalidLegacyProtocolMagic(int read)
+ {
+ super(String.format("Read %d, Expected %d", read, PROTOCOL_MAGIC));
+ this.read = read;
+ }
+ }
+
+ public String toString()
+ {
+ return "(from:" + from() + ", type:" + verb().stage + " verb:" + verb() + ')';
+ }
+
+ /**
+ * Split into a separate object to allow partial message deserialization without wasting work and allocation
+ * afterwards, if the entire message is necessary and available.
+ */
+ public static class Header
+ {
+ public final long id;
+ public final Verb verb;
+ public final InetAddressAndPort from;
+ public final long createdAtNanos;
+ public final long expiresAtNanos;
+ private final int flags;
+ private final Map<ParamType, Object> params;
+
+ private Header(long id, Verb verb, InetAddressAndPort from, long createdAtNanos, long expiresAtNanos, int flags, Map<ParamType, Object> params)
+ {
+ this.id = id;
+ this.verb = verb;
+ this.from = from;
+ this.createdAtNanos = createdAtNanos;
+ this.expiresAtNanos = expiresAtNanos;
+ this.flags = flags;
+ this.params = params;
+ }
+
+ Header withFlag(MessageFlag flag)
+ {
+ return new Header(id, verb, from, createdAtNanos, expiresAtNanos, flag.addTo(flags), params);
+ }
+
+ Header withParam(ParamType type, Object value)
+ {
+ return new Header(id, verb, from, createdAtNanos, expiresAtNanos, flags, addParam(params, type, value));
+ }
+
+ boolean callBackOnFailure()
+ {
+ return MessageFlag.CALL_BACK_ON_FAILURE.isIn(flags);
+ }
+
+ boolean trackRepairedData()
+ {
+ return MessageFlag.TRACK_REPAIRED_DATA.isIn(flags);
+ }
+
+ @Nullable
+ ForwardingInfo forwardTo()
+ {
+ return (ForwardingInfo) params.get(ParamType.FORWARD_TO);
+ }
+
+ @Nullable
+ InetAddressAndPort respondTo()
+ {
+ return (InetAddressAndPort) params.get(ParamType.RESPOND_TO);
+ }
+
+ @Nullable
+ public UUID traceSession()
+ {
+ return (UUID) params.get(ParamType.TRACE_SESSION);
+ }
+
+ @Nullable
+ public TraceType traceType()
+ {
+ return (TraceType) params.getOrDefault(ParamType.TRACE_TYPE, TraceType.QUERY);
+ }
+ }
+
+ @SuppressWarnings("WeakerAccess")
+ public static class Builder<T>
+ {
+ private Verb verb;
+ private InetAddressAndPort from;
+ private T payload;
+ private int flags = 0;
+ private final Map<ParamType, Object> params = new EnumMap<>(ParamType.class);
+ private long createdAtNanos;
+ private long expiresAtNanos;
+ private long id;
+
+ private boolean hasId;
+
+ private Builder()
+ {
+ }
+
+ public Builder<T> from(InetAddressAndPort from)
+ {
+ this.from = from;
+ return this;
+ }
+
+ public Builder<T> withPayload(T payload)
+ {
+ this.payload = payload;
+ return this;
+ }
+
+ public Builder<T> withFlag(MessageFlag flag)
+ {
+ flags = flag.addTo(flags);
+ return this;
+ }
+
+ public Builder<T> withFlags(int flags)
+ {
+ this.flags = flags;
+ return this;
+ }
+
+ public Builder<T> withParam(ParamType type, Object value)
+ {
+ params.put(type, value);
+ return this;
+ }
+
+ /**
+ * A shortcut to add tracing params.
+ * Effectively, it is the same as calling {@link #withParam(ParamType, Object)} with tracing params
+ * If there is already tracing params, calling this method overrides any existing ones.
+ */
+ public Builder<T> withTracingParams()
+ {
+ if (Tracing.isTracing())
+ Tracing.instance.addTraceHeaders(params);
+ return this;
+ }
+
+ public Builder<T> withoutParam(ParamType type)
+ {
+ params.remove(type);
+ return this;
+ }
+
+ public Builder<T> withParams(Map<ParamType, Object> params)
+ {
+ this.params.putAll(params);
+ return this;
+ }
+
+ public Builder<T> ofVerb(Verb verb)
+ {
+ this.verb = verb;
+ if (expiresAtNanos == 0 && verb != null && createdAtNanos != 0)
+ expiresAtNanos = verb.expiresAtNanos(createdAtNanos);
+ if (!this.verb.isResponse() && from == null) // default to sending from self if we're a request verb
+ from = FBUtilities.getBroadcastAddressAndPort();
+ return this;
+ }
+
+ public Builder<T> withCreatedAt(long createdAtNanos)
+ {
+ this.createdAtNanos = createdAtNanos;
+ if (expiresAtNanos == 0 && verb != null)
+ expiresAtNanos = verb.expiresAtNanos(createdAtNanos);
+ return this;
+ }
+
+ public Builder<T> withExpiresAt(long expiresAtNanos)
+ {
+ this.expiresAtNanos = expiresAtNanos;
+ return this;
+ }
+
+ public Builder<T> withId(long id)
+ {
+ this.id = id;
+ hasId = true;
+ return this;
+ }
+
+ public Message<T> build()
+ {
+ if (verb == null)
+ throw new IllegalArgumentException();
+ if (from == null)
+ throw new IllegalArgumentException();
+ if (payload == null)
+ throw new IllegalArgumentException();
+
+ return new Message<>(new Header(hasId ? id : nextId(), verb, from, createdAtNanos, expiresAtNanos, flags, params), payload);
+ }
+ }
+
+ public static <T> Builder<T> builder(Message<T> message)
+ {
+ return new Builder<T>().from(message.from())
+ .withId(message.id())
+ .ofVerb(message.verb())
+ .withCreatedAt(message.createdAtNanos())
+ .withExpiresAt(message.expiresAtNanos())
+ .withFlags(message.header.flags)
+ .withParams(message.header.params)
+ .withPayload(message.payload);
+ }
+
+ public static <T> Builder<T> builder(Verb verb, T payload)
+ {
+ return new Builder<T>().ofVerb(verb)
+ .withCreatedAt(approxTime.now())
+ .withPayload(payload);
+ }
+
+ public static final Serializer serializer = new Serializer();
+
+ /**
+ * Each message contains a header with several fixed fields, an optional key-value params section, and then
+ * the message payload itself. Below is a visualization of the layout.
+ *
+ * The params are prefixed by the count of key-value pairs; this value is encoded as unsigned vint.
+ * An individual param has an unsvint id (more specifically, a {@link ParamType}), and a byte array value.
+ * The param value is prefixed with it's length, encoded as an unsigned vint, followed by by the value's bytes.
+ *
+ * Legacy Notes (see {@link Serializer#serialize(Message, DataOutputPlus, int)} for complete details):
+ * - pre 4.0, the IP address was sent along in the header, before the verb. The IP address may be either IPv4 (4 bytes) or IPv6 (16 bytes)
+ * - pre-4.0, the verb was encoded as a 4-byte integer; in 4.0 and up it is an unsigned vint
+ * - pre-4.0, the payloadSize was encoded as a 4-byte integer; in 4.0 and up it is an unsigned vint
+ * - pre-4.0, the count of param key-value pairs was encoded as a 4-byte integer; in 4.0 and up it is an unsigned vint
+ * - pre-4.0, param names were encoded as strings; in 4.0 they are encoded as enum id vints
+ * - pre-4.0, expiry time wasn't encoded at all; in 4.0 it's an unsigned vint
+ * - pre-4.0, message id was an int; in 4.0 and up it's an unsigned vint
+ * - pre-4.0, messages included PROTOCOL MAGIC BYTES; post-4.0, we rely on frame CRCs instead
+ * - pre-4.0, messages would serialize boolean params as dummy ONE_BYTEs; post-4.0 we have a dedicated 'flags' vint
+ *
+ * <pre>
+ * {@code
+ * 1 1 1 1 1 2 2 2 2 2 3
+ * 0 2 4 6 8 0 2 4 6 8 0 2 4 6 8 0
+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ * | Message ID (vint) |
+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ * | Creation timestamp (int) |
+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ * | Expiry (vint) |
+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ * | Verb (vint) |
+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ * | Flags (vint) |
+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ * | Param count (vint) |
+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ * | /
+ * / Params /
+ * / |
+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ * | Payload size (vint) |
+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ * | /
+ * / Payload /
+ * / |
+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ * }
+ * </pre>
+ */
+ public static final class Serializer
+ {
+ private static final int CREATION_TIME_SIZE = 4;
+
+ private Serializer()
+ {
+ }
+
+ public <T> void serialize(Message<T> message, DataOutputPlus out, int version) throws IOException
+ {
+ if (version >= VERSION_40)
+ serializePost40(message, out, version);
+ else
+ serializePre40(message, out, version);
+ }
+
+ public <T> Message<T> deserialize(DataInputPlus in, InetAddressAndPort peer, int version) throws IOException
+ {
+ return version >= VERSION_40 ? deserializePost40(in, peer, version) : deserializePre40(in, version);
+ }
+
+ /**
+ * A partial variant of deserialize, taking in a previously deserialized {@link Header} as an argument.
+ *
+ * Skip deserializing the {@link Header} from the input stream in favour of using the provided header.
+ */
+ public <T> Message<T> deserialize(DataInputPlus in, Header header, int version) throws IOException
+ {
+ return version >= VERSION_40 ? deserializePost40(in, header, version) : deserializePre40(in, header, version);
+ }
+
+ private <T> int serializedSize(Message<T> message, int version)
+ {
+ return version >= VERSION_40 ? serializedSizePost40(message, version) : serializedSizePre40(message, version);
+ }
+
+ /**
+ * Size of the next message in the stream. Returns -1 if there aren't sufficient bytes read yet to determine size.
+ */
+ int inferMessageSize(ByteBuffer buf, int index, int limit, int version) throws InvalidLegacyProtocolMagic
+ {
+ int size = version >= VERSION_40 ? inferMessageSizePost40(buf, index, limit) : inferMessageSizePre40(buf, index, limit);
+ if (size > DatabaseDescriptor.getInternodeMaxMessageSizeInBytes())
+ throw new OversizedMessageException(size);
+ return size;
+ }
+
+ /**
+ * Partially deserialize the message - by only extracting the header and leaving the payload alone.
+ *
+ * To get the rest of the message without repeating the work done here, use {@link #deserialize(DataInputPlus, Header, int)}
+ * method.
+ *
+ * It's assumed that the provided buffer contains all the bytes necessary to deserialize the header fully.
+ */
+ Header extractHeader(ByteBuffer buf, InetAddressAndPort from, long currentTimeNanos, int version) throws IOException
+ {
+ return version >= VERSION_40
+ ? extractHeaderPost40(buf, from, currentTimeNanos, version)
+ : extractHeaderPre40(buf, currentTimeNanos, version);
+ }
+
+ private static long getExpiresAtNanos(long createdAtNanos, long currentTimeNanos, long expirationPeriodNanos)
+ {
+ if (!DatabaseDescriptor.hasCrossNodeTimeout() || createdAtNanos > currentTimeNanos)
+ createdAtNanos = currentTimeNanos;
+ return createdAtNanos + expirationPeriodNanos;
+ }
+
+ /*
+ * 4.0 ser/deser
+ */
+
+ private void serializeHeaderPost40(Header header, DataOutputPlus out, int version) throws IOException
+ {
+ out.writeUnsignedVInt(header.id);
+ // int cast cuts off the high-order half of the timestamp, which we can assume remains
+ // the same between now and when the recipient reconstructs it.
+ out.writeInt((int) approxTime.translate().toMillisSinceEpoch(header.createdAtNanos));
+ out.writeUnsignedVInt(1 + NANOSECONDS.toMillis(header.expiresAtNanos - header.createdAtNanos));
+ out.writeUnsignedVInt(header.verb.id);
+ out.writeUnsignedVInt(header.flags);
+ serializeParams(header.params, out, version);
+ }
+
+ private Header deserializeHeaderPost40(DataInputPlus in, InetAddressAndPort peer, int version) throws IOException
+ {
+ long id = in.readUnsignedVInt();
+ long currentTimeNanos = approxTime.now();
+ MonotonicClockTranslation timeSnapshot = approxTime.translate();
+ long creationTimeNanos = calculateCreationTimeNanos(in.readInt(), timeSnapshot, currentTimeNanos);
+ long expiresAtNanos = getExpiresAtNanos(creationTimeNanos, currentTimeNanos, TimeUnit.MILLISECONDS.toNanos(in.readUnsignedVInt()));
+ Verb verb = Verb.fromId(Ints.checkedCast(in.readUnsignedVInt()));
+ int flags = Ints.checkedCast(in.readUnsignedVInt());
+ Map<ParamType, Object> params = deserializeParams(in, version);
+ return new Header(id, verb, peer, creationTimeNanos, expiresAtNanos, flags, params);
+ }
+
+ private void skipHeaderPost40(DataInputPlus in) throws IOException
+ {
+ skipUnsignedVInt(in); // id
+ in.skipBytesFully(4); // createdAt
+ skipUnsignedVInt(in); // expiresIn
+ skipUnsignedVInt(in); // verb
+ skipUnsignedVInt(in); // flags
+ skipParamsPost40(in); // params
+ }
+
+ private int serializedHeaderSizePost40(Header header, int version)
+ {
+ long size = 0;
+ size += sizeofUnsignedVInt(header.id);
+ size += CREATION_TIME_SIZE;
+ size += sizeofUnsignedVInt(1 + NANOSECONDS.toMillis(header.expiresAtNanos - header.createdAtNanos));
+ size += sizeofUnsignedVInt(header.verb.id);
+ size += sizeofUnsignedVInt(header.flags);
+ size += serializedParamsSize(header.params, version);
+ return Ints.checkedCast(size);
+ }
+
+ private Header extractHeaderPost40(ByteBuffer buf, InetAddressAndPort from, long currentTimeNanos, int version) throws IOException
+ {
+ MonotonicClockTranslation timeSnapshot = approxTime.translate();
+
+ int index = buf.position();
+
+ long id = getUnsignedVInt(buf, index);
+ index += computeUnsignedVIntSize(id);
+
+ int createdAtMillis = buf.getInt(index);
+ index += sizeof(createdAtMillis);
+
+ long expiresInMillis = getUnsignedVInt(buf, index);
+ index += computeUnsignedVIntSize(expiresInMillis);
+
+ Verb verb = Verb.fromId(Ints.checkedCast(getUnsignedVInt(buf, index)));
+ index += computeUnsignedVIntSize(verb.id);
+
+ int flags = Ints.checkedCast(getUnsignedVInt(buf, index));
+ index += computeUnsignedVIntSize(flags);
+
+ Map<ParamType, Object> params = extractParams(buf, index, version);
+
+ long createdAtNanos = calculateCreationTimeNanos(createdAtMillis, timeSnapshot, currentTimeNanos);
+ long expiresAtNanos = getExpiresAtNanos(createdAtNanos, currentTimeNanos, TimeUnit.MILLISECONDS.toNanos(expiresInMillis));
+
+ return new Header(id, verb, from, createdAtNanos, expiresAtNanos, flags, params);
+ }
+
+ private <T> void serializePost40(Message<T> message, DataOutputPlus out, int version) throws IOException
+ {
+ serializeHeaderPost40(message.header, out, version);
+ out.writeUnsignedVInt(message.payloadSize(version));
+ message.verb().serializer().serialize(message.payload, out, version);
+ }
+
+ private <T> Message<T> deserializePost40(DataInputPlus in, InetAddressAndPort peer, int version) throws IOException
+ {
+ Header header = deserializeHeaderPost40(in, peer, version);
+ skipUnsignedVInt(in); // payload size, not needed by payload deserializer
+ T payload = (T) header.verb.serializer().deserialize(in, version);
+ return new Message<>(header, payload);
+ }
+
+ private <T> Message<T> deserializePost40(DataInputPlus in, Header header, int version) throws IOException
+ {
+ skipHeaderPost40(in);
+ skipUnsignedVInt(in); // payload size, not needed by payload deserializer
+ T payload = (T) header.verb.serializer().deserialize(in, version);
+ return new Message<>(header, payload);
+ }
+
+ private <T> int serializedSizePost40(Message<T> message, int version)
+ {
+ long size = 0;
+ size += serializedHeaderSizePost40(message.header, version);
+ int payloadSize = message.payloadSize(version);
+ size += sizeofUnsignedVInt(payloadSize) + payloadSize;
+ return Ints.checkedCast(size);
+ }
+
+ private int inferMessageSizePost40(ByteBuffer buf, int readerIndex, int readerLimit)
+ {
+ int index = readerIndex;
+
+ int idSize = computeUnsignedVIntSize(buf, index, readerLimit);
+ if (idSize < 0)
+ return -1; // not enough bytes to read id
+ index += idSize;
+
+ index += CREATION_TIME_SIZE;
+ if (index > readerLimit)
+ return -1;
+
+ int expirationSize = computeUnsignedVIntSize(buf, index, readerLimit);
+ if (expirationSize < 0)
+ return -1;
+ index += expirationSize;
+
+ int verbIdSize = computeUnsignedVIntSize(buf, index, readerLimit);
+ if (verbIdSize < 0)
+ return -1;
+ index += verbIdSize;
+
+ int flagsSize = computeUnsignedVIntSize(buf, index, readerLimit);
+ if (flagsSize < 0)
+ return -1;
+ index += flagsSize;
+
+ int paramsSize = extractParamsSizePost40(buf, index, readerLimit);
+ if (paramsSize < 0)
+ return -1;
+ index += paramsSize;
+
+ long payloadSize = getUnsignedVInt(buf, index, readerLimit);
+ if (payloadSize < 0)
+ return -1;
+ index += computeUnsignedVIntSize(payloadSize) + payloadSize;
+
+ return index - readerIndex;
+ }
+
+ /*
+ * legacy ser/deser
+ */
+
+ private void serializeHeaderPre40(Header header, DataOutputPlus out, int version) throws IOException
+ {
+ out.writeInt(PROTOCOL_MAGIC);
+ out.writeInt(Ints.checkedCast(header.id));
+ // int cast cuts off the high-order half of the timestamp, which we can assume remains
+ // the same between now and when the recipient reconstructs it.
+ out.writeInt((int) approxTime.translate().toMillisSinceEpoch(header.createdAtNanos));
+ inetAddressAndPortSerializer.serialize(header.from, out, version);
+ out.writeInt(header.verb.toPre40Verb().id);
+ serializeParams(addFlagsToLegacyParams(header.params, header.flags), out, version);
+ }
+
+ private Header deserializeHeaderPre40(DataInputPlus in, int version) throws IOException
+ {
+ validateLegacyProtocolMagic(in.readInt());
+ int id = in.readInt();
+ long currentTimeNanos = approxTime.now();
+ MonotonicClockTranslation timeSnapshot = approxTime.translate();
+ long creationTimeNanos = calculateCreationTimeNanos(in.readInt(), timeSnapshot, currentTimeNanos);
+ InetAddressAndPort from = inetAddressAndPortSerializer.deserialize(in, version);
+ Verb verb = Verb.fromId(in.readInt());
+ Map<ParamType, Object> params = deserializeParams(in, version);
+ int flags = removeFlagsFromLegacyParams(params);
+ return new Header(id, verb, from, creationTimeNanos, verb.expiresAtNanos(creationTimeNanos), flags, params);
+ }
+
+ private static final int PRE_40_MESSAGE_PREFIX_SIZE = 12; // protocol magic + id + createdAt
+
+ private void skipHeaderPre40(DataInputPlus in) throws IOException
+ {
+ in.skipBytesFully(PRE_40_MESSAGE_PREFIX_SIZE); // magic, id, createdAt
+ in.skipBytesFully(in.readByte()); // from
+ in.skipBytesFully(4); // verb
+ skipParamsPre40(in); // params
+ }
+
+ private int serializedHeaderSizePre40(Header header, int version)
+ {
+ long size = 0;
+ size += PRE_40_MESSAGE_PREFIX_SIZE;
+ size += inetAddressAndPortSerializer.serializedSize(header.from, version);
+ size += sizeof(header.verb.id);
+ size += serializedParamsSize(addFlagsToLegacyParams(header.params, header.flags), version);
+ return Ints.checkedCast(size);
+ }
+
+ private Header extractHeaderPre40(ByteBuffer buf, long currentTimeNanos, int version) throws IOException
+ {
+ MonotonicClockTranslation timeSnapshot = approxTime.translate();
+
+ int index = buf.position();
+
+ index += 4; // protocol magic
+
+ long id = buf.getInt(index);
+ index += 4;
+
+ int createdAtMillis = buf.getInt(index);
+ index += 4;
+
+ InetAddressAndPort from = inetAddressAndPortSerializer.extract(buf, index);
+ index += 1 + buf.get(index);
+
+ Verb verb = Verb.fromId(buf.getInt(index));
+ index += 4;
+
+ Map<ParamType, Object> params = extractParams(buf, index, version);
+ int flags = removeFlagsFromLegacyParams(params);
+
+ long createdAtNanos = calculateCreationTimeNanos(createdAtMillis, timeSnapshot, currentTimeNanos);
+ long expiresAtNanos = verb.expiresAtNanos(createdAtNanos);
+
+ return new Header(id, verb, from, createdAtNanos, expiresAtNanos, flags, params);
+ }
+
+ private <T> void serializePre40(Message<T> message, DataOutputPlus out, int version) throws IOException
+ {
+ if (message.isFailureResponse())
+ message = toPre40FailureResponse(message);
+
+ serializeHeaderPre40(message.header, out, version);
+
+ if (message.payload != null && message.payload != NoPayload.noPayload)
+ {
+ int payloadSize = message.payloadSize(version);
+ out.writeInt(payloadSize);
+ message.verb().serializer().serialize(message.payload, out, version);
+ }
+ else
+ {
+ out.writeInt(0);
+ }
+ }
+
+ private <T> Message<T> deserializePre40(DataInputPlus in, int version) throws IOException
+ {
+ Header header = deserializeHeaderPre40(in, version);
+ return deserializePre40(in, header, false, version);
+ }
+
+ private <T> Message<T> deserializePre40(DataInputPlus in, Header header, int version) throws IOException
+ {
+ return deserializePre40(in, header, true, version);
+ }
+
+ private <T> Message<T> deserializePre40(DataInputPlus in, Header header, boolean skipHeader, int version) throws IOException
+ {
+ if (skipHeader)
+ skipHeaderPre40(in);
+
+ IVersionedAsymmetricSerializer<?, T> payloadSerializer = header.verb.serializer();
+ if (null == payloadSerializer)
+ payloadSerializer = instance().callbacks.responseSerializer(header.id, header.from);
+ int payloadSize = in.readInt();
+ T payload = deserializePayloadPre40(in, version, payloadSerializer, payloadSize);
+
+ Message<T> message = new Message<>(header, payload);
+
+ return header.params.containsKey(ParamType.FAILURE_RESPONSE)
+ ? (Message<T>) toPost40FailureResponse(message)
+ : message;
+ }
+
+ private <T> T deserializePayloadPre40(DataInputPlus in, int version, IVersionedAsymmetricSerializer<?, T> serializer, int payloadSize) throws IOException
+ {
+ if (payloadSize == 0 || serializer == null)
+ {
+ // if there's no deserializer for the verb, skip the payload bytes to leave
+ // the stream in a clean state (for the next message)
+ in.skipBytesFully(payloadSize);
+ return null;
+ }
+
+ return serializer.deserialize(in, version);
+ }
+
+ private <T> int serializedSizePre40(Message<T> message, int version)
+ {
+ if (message.isFailureResponse())
+ message = toPre40FailureResponse(message);
+
+ long size = 0;
+ size += serializedHeaderSizePre40(message.header, version);
+ int payloadSize = message.payloadSize(version);
+ size += sizeof(payloadSize);
+ size += payloadSize;
+ return Ints.checkedCast(size);
+ }
+
+ private int inferMessageSizePre40(ByteBuffer buf, int readerIndex, int readerLimit) throws InvalidLegacyProtocolMagic
+ {
+ int index = readerIndex;
+ // protocol magic
+ index += 4;
+ if (index > readerLimit)
+ return -1;
+ validateLegacyProtocolMagic(buf.getInt(index - 4));
+
+ // rest of prefix
+ index += PRE_40_MESSAGE_PREFIX_SIZE - 4;
+ // ip address
+ index += 1;
+ if (index > readerLimit)
+ return -1;
+ index += buf.get(index - 1);
+ // verb
+ index += 4;
+ if (index > readerLimit)
+ return -1;
+
+ int paramsSize = extractParamsSizePre40(buf, index, readerLimit);
+ if (paramsSize < 0)
+ return -1;
+ index += paramsSize;
+
+ // payload
+ index += 4;
+
+ if (index > readerLimit)
+ return -1;
+ index += buf.getInt(index - 4);
+
+ return index - readerIndex;
+ }
+
+ private Message toPre40FailureResponse(Message post40)
+ {
+ Map<ParamType, Object> params = new EnumMap<>(ParamType.class);
+ params.putAll(post40.header.params);
+
+ params.put(ParamType.FAILURE_RESPONSE, LegacyFlag.instance);
+ params.put(ParamType.FAILURE_REASON, post40.payload);
+
+ Header header = new Header(post40.id(), post40.verb().toPre40Verb(), post40.from(), post40.createdAtNanos(), post40.expiresAtNanos(), 0, params);
+ return new Message<>(header, NoPayload.noPayload);
+ }
+
+ private Message<RequestFailureReason> toPost40FailureResponse(Message<?> pre40)
+ {
+ Map<ParamType, Object> params = new EnumMap<>(ParamType.class);
+ params.putAll(pre40.header.params);
+
+ params.remove(ParamType.FAILURE_RESPONSE);
+
+ RequestFailureReason reason = (RequestFailureReason) params.remove(ParamType.FAILURE_REASON);
+ if (null == reason)
+ reason = RequestFailureReason.UNKNOWN;
+
+ Header header = new Header(pre40.id(), Verb.FAILURE_RSP, pre40.from(), pre40.createdAtNanos(), pre40.expiresAtNanos(), pre40.header.flags, params);
+ return new Message<>(header, reason);
+ }
+
+ /*
+ * created at + cross-node
+ */
+
+ private static final long TIMESTAMP_WRAPAROUND_GRACE_PERIOD_START = 0xFFFFFFFFL - MINUTES.toMillis(15L);
+ private static final long TIMESTAMP_WRAPAROUND_GRACE_PERIOD_END = MINUTES.toMillis(15L);
+
+ private static long calculateCreationTimeNanos(int messageTimestampMillis, MonotonicClockTranslation timeSnapshot, long currentTimeNanos)
+ {
+ long currentTimeMillis = timeSnapshot.toMillisSinceEpoch(currentTimeNanos);
+ // Reconstruct the message construction time sent by the remote host (we sent only the lower 4 bytes, assuming the
+ // higher 4 bytes wouldn't change between the sender and receiver)
+ long highBits = currentTimeMillis & 0xFFFFFFFF00000000L;
+
+ long sentLowBits = messageTimestampMillis & 0x00000000FFFFFFFFL;
+ long currentLowBits = currentTimeMillis & 0x00000000FFFFFFFFL;
+
+ // if our sent bits occur within a grace period of a wrap around event,
+ // and our current bits are no more than the same grace period after a wrap around event,
+ // assume a wrap around has occurred, and deduct one highBit
+ if ( sentLowBits > TIMESTAMP_WRAPAROUND_GRACE_PERIOD_START
+ && currentLowBits < TIMESTAMP_WRAPAROUND_GRACE_PERIOD_END)
+ {
+ highBits -= 0x0000000100000000L;
+ }
+
+ long sentTimeMillis = (highBits | sentLowBits);
+ return timeSnapshot.fromMillisSinceEpoch(sentTimeMillis);
+ }
+
+ /*
+ * param ser/deser
+ */
+
+ private Map<ParamType, Object> addFlagsToLegacyParams(Map<ParamType, Object> params, int flags)
+ {
+ if (flags == 0)
+ return params;
+
+ Map<ParamType, Object> extended = new EnumMap<>(ParamType.class);
+ extended.putAll(params);
+
+ if (MessageFlag.CALL_BACK_ON_FAILURE.isIn(flags))
+ extended.put(ParamType.FAILURE_CALLBACK, LegacyFlag.instance);
+
+ if (MessageFlag.TRACK_REPAIRED_DATA.isIn(flags))
+ extended.put(ParamType.TRACK_REPAIRED_DATA, LegacyFlag.instance);
+
+ return extended;
+ }
+
+ private int removeFlagsFromLegacyParams(Map<ParamType, Object> params)
+ {
+ int flags = 0;
+
+ if (null != params.remove(ParamType.FAILURE_CALLBACK))
+ flags = MessageFlag.CALL_BACK_ON_FAILURE.addTo(flags);
+
+ if (null != params.remove(ParamType.TRACK_REPAIRED_DATA))
+ flags = MessageFlag.TRACK_REPAIRED_DATA.addTo(flags);
+
+ return flags;
+ }
+
+ private void serializeParams(Map<ParamType, Object> params, DataOutputPlus out, int version) throws IOException
+ {
+ if (version >= VERSION_40)
+ out.writeUnsignedVInt(params.size());
+ else
+ out.writeInt(params.size());
+
+ for (Map.Entry<ParamType, Object> kv : params.entrySet())
+ {
+ ParamType type = kv.getKey();
+ if (version >= VERSION_40)
+ out.writeUnsignedVInt(type.id);
+ else
+ out.writeUTF(type.legacyAlias);
+
+ IVersionedSerializer serializer = type.serializer;
+ Object value = kv.getValue();
+
+ int length = Ints.checkedCast(serializer.serializedSize(value, version));
+ if (version >= VERSION_40)
+ out.writeUnsignedVInt(length);
+ else
+ out.writeInt(length);
+
+ serializer.serialize(value, out, version);
+ }
+ }
+
+ private Map<ParamType, Object> deserializeParams(DataInputPlus in, int version) throws IOException
+ {
+ int count = version >= VERSION_40 ? Ints.checkedCast(in.readUnsignedVInt()) : in.readInt();
+
+ if (count == 0)
+ return NO_PARAMS;
+
+ Map<ParamType, Object> params = new EnumMap<>(ParamType.class);
+
+ for (int i = 0; i < count; i++)
+ {
+ ParamType type = version >= VERSION_40
+ ? ParamType.lookUpById(Ints.checkedCast(in.readUnsignedVInt()))
+ : ParamType.lookUpByAlias(in.readUTF());
+
+ int length = version >= VERSION_40
+ ? Ints.checkedCast(in.readUnsignedVInt())
+ : in.readInt();
+
+ if (null != type)
+ params.put(type, type.serializer.deserialize(in, version));
+ else
+ in.skipBytesFully(length); // forward compatibiliy with minor version changes
+ }
+
+ return params;
+ }
+
+ /*
+ * Extract post-4.0 params map from a ByteBuffer without modifying it.
+ */
+ private Map<ParamType, Object> extractParams(ByteBuffer buf, int readerIndex, int version) throws IOException
+ {
+ long count = version >= VERSION_40 ? getUnsignedVInt(buf, readerIndex) : buf.getInt(readerIndex);
+
+ if (count == 0)
+ return NO_PARAMS;
+
+ final int position = buf.position();
+ buf.position(readerIndex);
+
+ try (DataInputBuffer in = new DataInputBuffer(buf, false))
+ {
+ return deserializeParams(in, version);
+ }
+ finally
+ {
+ buf.position(position);
+ }
+ }
+
+ private void skipParamsPost40(DataInputPlus in) throws IOException
+ {
+ int count = Ints.checkedCast(in.readUnsignedVInt());
+
+ for (int i = 0; i < count; i++)
+ {
+ skipUnsignedVInt(in);
+ in.skipBytesFully(Ints.checkedCast(in.readUnsignedVInt()));
+ }
+ }
+
+ private void skipParamsPre40(DataInputPlus in) throws IOException
+ {
+ int count = in.readInt();
+
+ for (int i = 0; i < count; i++)
+ {
+ in.skipBytesFully(in.readShort());
+ in.skipBytesFully(in.readInt());
+ }
+ }
+
+ private long serializedParamsSize(Map<ParamType, Object> params, int version)
+ {
+ long size = version >= VERSION_40
+ ? computeUnsignedVIntSize(params.size())
+ : sizeof(params.size());
+
+ for (Map.Entry<ParamType, Object> kv : params.entrySet())
+ {
+ ParamType type = kv.getKey();
+ Object value = kv.getValue();
+
+ long valueLength = type.serializer.serializedSize(value, version);
+
+ if (version >= VERSION_40)
+ size += sizeofUnsignedVInt(type.id) + sizeofUnsignedVInt(valueLength);
+ else
+ size += sizeof(type.legacyAlias) + 4;
+
+ size += valueLength;
+ }
+
+ return size;
+ }
+
+ private int extractParamsSizePost40(ByteBuffer buf, int readerIndex, int readerLimit)
+ {
+ int index = readerIndex;
+
+ long paramsCount = getUnsignedVInt(buf, index, readerLimit);
+ if (paramsCount < 0)
+ return -1;
+ index += computeUnsignedVIntSize(paramsCount);
+
+ for (int i = 0; i < paramsCount; i++)
+ {
+ long type = getUnsignedVInt(buf, index, readerLimit);
+ if (type < 0)
+ return -1;
+ index += computeUnsignedVIntSize(type);
+
+ long length = getUnsignedVInt(buf, index, readerLimit);
+ if (length < 0)
+ return -1;
+ index += computeUnsignedVIntSize(length) + length;
+ }
+
+ return index - readerIndex;
+ }
+
+ private int extractParamsSizePre40(ByteBuffer buf, int readerIndex, int readerLimit)
+ {
+ int index = readerIndex;
+
+ index += 4;
+ if (index > readerLimit)
+ return -1;
+ int paramsCount = buf.getInt(index - 4);
+
+ for (int i = 0; i < paramsCount; i++)
+ {
+ // try to read length and skip to the end of the param name
+ index += 2;
+
+ if (index > readerLimit)
+ return -1;
+ index += buf.getShort(index - 2);
+ // try to read length and skip to the end of the param value
+ index += 4;
+ if (index > readerLimit)
+ return -1;
+ index += buf.getInt(index - 4);
+ }
+
+ return index - readerIndex;
+ }
+
+ private <T> int payloadSize(Message<T> message, int version)
+ {
+ long payloadSize = message.payload != null && message.payload != NoPayload.noPayload
+ ? message.verb().serializer().serializedSize(message.payload, version)
+ : 0;
+ return Ints.checkedCast(payloadSize);
+ }
+ }
+
+ private int serializedSize30;
+ private int serializedSize3014;
+ private int serializedSize40;
+
+ /**
+ * Serialized size of the entire message, for the provided messaging version. Caches the calculated value.
+ */
+ public int serializedSize(int version)
+ {
+ switch (version)
+ {
+ case VERSION_30:
+ if (serializedSize30 == 0)
+ serializedSize30 = serializer.serializedSize(this, VERSION_30);
+ return serializedSize30;
+ case VERSION_3014:
+ if (serializedSize3014 == 0)
+ serializedSize3014 = serializer.serializedSize(this, VERSION_3014);
+ return serializedSize3014;
+ case VERSION_40:
+ if (serializedSize40 == 0)
+ serializedSize40 = serializer.serializedSize(this, VERSION_40);
+ return serializedSize40;
+ default:
+ throw new IllegalStateException();
+ }
+ }
+
+ private int payloadSize30 = -1;
+ private int payloadSize3014 = -1;
+ private int payloadSize40 = -1;
+
+ private int payloadSize(int version)
+ {
+ switch (version)
+ {
+ case VERSION_30:
+ if (payloadSize30 < 0)
+ payloadSize30 = serializer.payloadSize(this, VERSION_30);
+ return payloadSize30;
+ case VERSION_3014:
+ if (payloadSize3014 < 0)
+ payloadSize3014 = serializer.payloadSize(this, VERSION_3014);
+ return payloadSize3014;
+ case VERSION_40:
+ if (payloadSize40 < 0)
+ payloadSize40 = serializer.payloadSize(this, VERSION_40);
+ return payloadSize40;
+ default:
+ throw new IllegalStateException();
+ }
+ }
+
+ static class OversizedMessageException extends RuntimeException
+ {
+ OversizedMessageException(int size)
+ {
+ super("Message of size " + size + " bytes exceeds allowed maximum of " + DatabaseDescriptor.getInternodeMaxMessageSizeInBytes() + " bytes");
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/net/MessageDeliveryTask.java b/src/java/org/apache/cassandra/net/MessageDeliveryTask.java
deleted file mode 100644
index c91e9da..0000000
--- a/src/java/org/apache/cassandra/net/MessageDeliveryTask.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.net;
-
-import java.io.IOException;
-import java.util.EnumSet;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.cassandra.db.filter.TombstoneOverwhelmingException;
-import org.apache.cassandra.exceptions.RequestFailureReason;
-import org.apache.cassandra.gms.Gossiper;
-import org.apache.cassandra.index.IndexNotAvailableException;
-import org.apache.cassandra.io.util.DataOutputBuffer;
-
-public class MessageDeliveryTask implements Runnable
-{
- private static final Logger logger = LoggerFactory.getLogger(MessageDeliveryTask.class);
-
- private final MessageIn message;
- private final int id;
-
- public MessageDeliveryTask(MessageIn message, int id)
- {
- assert message != null;
- this.message = message;
- this.id = id;
- }
-
- public void run()
- {
- MessagingService.Verb verb = message.verb;
- long timeTaken = message.getLifetimeInMS();
- if (MessagingService.DROPPABLE_VERBS.contains(verb)
- && timeTaken > message.getTimeout())
- {
- MessagingService.instance().incrementDroppedMessages(message, timeTaken);
- return;
- }
-
- IVerbHandler verbHandler = MessagingService.instance().getVerbHandler(verb);
- if (verbHandler == null)
- {
- logger.trace("Unknown verb {}", verb);
- return;
- }
-
- try
- {
- verbHandler.doVerb(message, id);
- }
- catch (IOException ioe)
- {
- handleFailure(ioe);
- throw new RuntimeException(ioe);
- }
- catch (TombstoneOverwhelmingException | IndexNotAvailableException e)
- {
- handleFailure(e);
- logger.error(e.getMessage());
- }
- catch (Throwable t)
- {
- handleFailure(t);
- throw t;
- }
-
- if (GOSSIP_VERBS.contains(message.verb))
- Gossiper.instance.setLastProcessedMessageAt(message.constructionTime);
- }
-
- private void handleFailure(Throwable t)
- {
- if (message.doCallbackOnFailure())
- {
- MessageOut response = new MessageOut(MessagingService.Verb.INTERNAL_RESPONSE)
- .withParameter(MessagingService.FAILURE_RESPONSE_PARAM, MessagingService.ONE_BYTE);
-
- if (t instanceof TombstoneOverwhelmingException)
- {
- try (DataOutputBuffer out = new DataOutputBuffer())
- {
- out.writeShort(RequestFailureReason.READ_TOO_MANY_TOMBSTONES.code);
- response = response.withParameter(MessagingService.FAILURE_REASON_PARAM, out.getData());
- }
- catch (IOException ex)
- {
- throw new RuntimeException(ex);
- }
- }
-
- MessagingService.instance().sendReply(response, id, message.from);
- }
- }
-
- private static final EnumSet<MessagingService.Verb> GOSSIP_VERBS = EnumSet.of(MessagingService.Verb.GOSSIP_DIGEST_ACK,
- MessagingService.Verb.GOSSIP_DIGEST_ACK2,
- MessagingService.Verb.GOSSIP_DIGEST_SYN);
-}
diff --git a/src/java/org/apache/cassandra/net/MessageFlag.java b/src/java/org/apache/cassandra/net/MessageFlag.java
new file mode 100644
index 0000000..c74784d
--- /dev/null
+++ b/src/java/org/apache/cassandra/net/MessageFlag.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.net;
+
+import static java.lang.Math.max;
+
+/**
+ * Binary message flags to be passed as {@code flags} field of {@link Message}.
+ */
+public enum MessageFlag
+{
+ /** a failure response should be sent back in case of failure */
+ CALL_BACK_ON_FAILURE (0),
+ /** track repaired data - see CASSANDRA-14145 */
+ TRACK_REPAIRED_DATA (1);
+
+ private final int id;
+
+ MessageFlag(int id)
+ {
+ this.id = id;
+ }
+
+ /**
+ * @return {@code true} if the flag is present in provided flags, {@code false} otherwise
+ */
+ boolean isIn(int flags)
+ {
+ return (flags & (1 << id)) != 0;
+ }
+
+ /**
+ * @return new flags value with this flag added
+ */
+ int addTo(int flags)
+ {
+ return flags | (1 << id);
+ }
+
+ private static final MessageFlag[] idToFlagMap;
+ static
+ {
+ MessageFlag[] flags = values();
+
+ int max = -1;
+ for (MessageFlag flag : flags)
+ max = max(flag.id, max);
+
+ MessageFlag[] idMap = new MessageFlag[max + 1];
+ for (MessageFlag flag : flags)
+ {
+ if (idMap[flag.id] != null)
+ throw new RuntimeException("Two MessageFlag-s that map to the same id: " + flag.id);
+ idMap[flag.id] = flag;
+ }
+ idToFlagMap = idMap;
+ }
+
+ @SuppressWarnings("unused")
+ MessageFlag lookUpById(int id)
+ {
+ if (id < 0)
+ throw new IllegalArgumentException("MessageFlag id must be non-negative (got " + id + ')');
+
+ return id < idToFlagMap.length ? idToFlagMap[id] : null;
+ }
+}
+
diff --git a/src/java/org/apache/cassandra/net/MessageIn.java b/src/java/org/apache/cassandra/net/MessageIn.java
deleted file mode 100644
index d06d515..0000000
--- a/src/java/org/apache/cassandra/net/MessageIn.java
+++ /dev/null
@@ -1,218 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.net;
-
-import java.io.IOException;
-import java.net.InetAddress;
-import java.util.Collections;
-import java.util.Map;
-
-import com.google.common.collect.ImmutableMap;
-
-import org.apache.cassandra.concurrent.Stage;
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.db.monitoring.ApproximateTime;
-import org.apache.cassandra.exceptions.RequestFailureReason;
-import org.apache.cassandra.io.IVersionedSerializer;
-import org.apache.cassandra.io.util.DataInputBuffer;
-import org.apache.cassandra.io.util.DataInputPlus;
-
-public class MessageIn<T>
-{
- public final InetAddress from;
- public final T payload;
- public final Map<String, byte[]> parameters;
- public final MessagingService.Verb verb;
- public final int version;
- public final long constructionTime;
-
- private MessageIn(InetAddress from,
- T payload,
- Map<String, byte[]> parameters,
- MessagingService.Verb verb,
- int version,
- long constructionTime)
- {
- this.from = from;
- this.payload = payload;
- this.parameters = parameters;
- this.verb = verb;
- this.version = version;
- this.constructionTime = constructionTime;
- }
-
- public static <T> MessageIn<T> create(InetAddress from,
- T payload,
- Map<String, byte[]> parameters,
- MessagingService.Verb verb,
- int version,
- long constructionTime)
- {
- return new MessageIn<>(from, payload, parameters, verb, version, constructionTime);
- }
-
- public static <T> MessageIn<T> create(InetAddress from,
- T payload,
- Map<String, byte[]> parameters,
- MessagingService.Verb verb,
- int version)
- {
- return new MessageIn<>(from, payload, parameters, verb, version, ApproximateTime.currentTimeMillis());
- }
-
- public static <T2> MessageIn<T2> read(DataInputPlus in, int version, int id) throws IOException
- {
- return read(in, version, id, ApproximateTime.currentTimeMillis());
- }
-
- public static <T2> MessageIn<T2> read(DataInputPlus in, int version, int id, long constructionTime) throws IOException
- {
- InetAddress from = CompactEndpointSerializationHelper.deserialize(in);
-
- MessagingService.Verb verb = MessagingService.verbValues[in.readInt()];
- int parameterCount = in.readInt();
- Map<String, byte[]> parameters;
- if (parameterCount == 0)
- {
- parameters = Collections.emptyMap();
- }
- else
- {
- ImmutableMap.Builder<String, byte[]> builder = ImmutableMap.builder();
- for (int i = 0; i < parameterCount; i++)
- {
- String key = in.readUTF();
- byte[] value = new byte[in.readInt()];
- in.readFully(value);
- builder.put(key, value);
- }
- parameters = builder.build();
- }
-
- int payloadSize = in.readInt();
- IVersionedSerializer<T2> serializer = (IVersionedSerializer<T2>) MessagingService.instance().verbSerializers.get(verb);
- if (serializer instanceof MessagingService.CallbackDeterminedSerializer)
- {
- CallbackInfo callback = MessagingService.instance().getRegisteredCallback(id);
- if (callback == null)
- {
- // reply for expired callback. we'll have to skip it.
- in.skipBytesFully(payloadSize);
- return null;
- }
- serializer = (IVersionedSerializer<T2>) callback.serializer;
- }
- if (payloadSize == 0 || serializer == null)
- return create(from, null, parameters, verb, version, constructionTime);
-
- T2 payload = serializer.deserialize(in, version);
- return MessageIn.create(from, payload, parameters, verb, version, constructionTime);
- }
-
- public static long readConstructionTime(InetAddress from, DataInputPlus input, long currentTime) throws IOException
- {
- // Reconstruct the message construction time sent by the remote host (we sent only the lower 4 bytes, assuming the
- // higher 4 bytes wouldn't change between the sender and receiver)
- int partial = input.readInt(); // make sure to readInt, even if cross_node_to is not enabled
- long sentConstructionTime = (currentTime & 0xFFFFFFFF00000000L) | (((partial & 0xFFFFFFFFL) << 2) >> 2);
-
- // Because nodes may not have their clock perfectly in sync, it's actually possible the sentConstructionTime is
- // later than the currentTime (the received time). If that's the case, as we definitively know there is a lack
- // of proper synchronziation of the clock, we ignore sentConstructionTime. We also ignore that
- // sentConstructionTime if we're told to.
- long elapsed = currentTime - sentConstructionTime;
- if (elapsed > 0)
- MessagingService.instance().metrics.addTimeTaken(from, elapsed);
-
- boolean useSentTime = DatabaseDescriptor.hasCrossNodeTimeout() && elapsed > 0;
- return useSentTime ? sentConstructionTime : currentTime;
- }
-
- /**
- * Since how long (in milliseconds) the message has lived.
- */
- public long getLifetimeInMS()
- {
- return ApproximateTime.currentTimeMillis() - constructionTime;
- }
-
- /**
- * Whether the message has crossed the node boundary, that is whether it originated from another node.
- *
- */
- public boolean isCrossNode()
- {
- return !from.equals(DatabaseDescriptor.getBroadcastAddress());
- }
-
- public Stage getMessageType()
- {
- return MessagingService.verbStages.get(verb);
- }
-
- public boolean doCallbackOnFailure()
- {
- return parameters.containsKey(MessagingService.FAILURE_CALLBACK_PARAM);
- }
-
- public boolean isFailureResponse()
- {
- return parameters.containsKey(MessagingService.FAILURE_RESPONSE_PARAM);
- }
-
- public boolean containsFailureReason()
- {
- return parameters.containsKey(MessagingService.FAILURE_REASON_PARAM);
- }
-
- public RequestFailureReason getFailureReason()
- {
- if (containsFailureReason())
- {
- try (DataInputBuffer in = new DataInputBuffer(parameters.get(MessagingService.FAILURE_REASON_PARAM)))
- {
- return RequestFailureReason.fromCode(in.readUnsignedShort());
- }
- catch (IOException ex)
- {
- throw new RuntimeException(ex);
- }
- }
- else
- {
- return RequestFailureReason.UNKNOWN;
- }
- }
-
- public long getTimeout()
- {
- return verb.getTimeout();
- }
-
- public long getSlowQueryTimeout()
- {
- return DatabaseDescriptor.getSlowQueryTimeout();
- }
-
- public String toString()
- {
- StringBuilder sbuf = new StringBuilder();
- sbuf.append("FROM:").append(from).append(" TYPE:").append(getMessageType()).append(" VERB:").append(verb);
- return sbuf.toString();
- }
-}
diff --git a/src/java/org/apache/cassandra/net/MessageOut.java b/src/java/org/apache/cassandra/net/MessageOut.java
deleted file mode 100644
index 1d1dd49..0000000
--- a/src/java/org/apache/cassandra/net/MessageOut.java
+++ /dev/null
@@ -1,202 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.net;
-
-import java.io.IOException;
-import java.net.InetAddress;
-import java.util.Collections;
-import java.util.Map;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.ImmutableMap;
-
-import org.apache.cassandra.concurrent.Stage;
-import org.apache.cassandra.db.TypeSizes;
-import org.apache.cassandra.io.IVersionedSerializer;
-import org.apache.cassandra.io.util.DataOutputBuffer;
-import org.apache.cassandra.io.util.DataOutputPlus;
-import org.apache.cassandra.tracing.Tracing;
-import org.apache.cassandra.utils.FBUtilities;
-import static org.apache.cassandra.tracing.Tracing.isTracing;
-
-public class MessageOut<T>
-{
- public final InetAddress from;
- public final MessagingService.Verb verb;
- public final T payload;
- public final IVersionedSerializer<T> serializer;
- public final Map<String, byte[]> parameters;
- private long payloadSize = -1;
- private int payloadSizeVersion = -1;
-
- // we do support messages that just consist of a verb
- public MessageOut(MessagingService.Verb verb)
- {
- this(verb, null, null);
- }
-
- public MessageOut(MessagingService.Verb verb, T payload, IVersionedSerializer<T> serializer)
- {
- this(verb,
- payload,
- serializer,
- isTracing()
- ? Tracing.instance.getTraceHeaders()
- : Collections.<String, byte[]>emptyMap());
- }
-
- private MessageOut(MessagingService.Verb verb, T payload, IVersionedSerializer<T> serializer, Map<String, byte[]> parameters)
- {
- this(FBUtilities.getBroadcastAddress(), verb, payload, serializer, parameters);
- }
-
- @VisibleForTesting
- public MessageOut(InetAddress from, MessagingService.Verb verb, T payload, IVersionedSerializer<T> serializer, Map<String, byte[]> parameters)
- {
- this.from = from;
- this.verb = verb;
- this.payload = payload;
- this.serializer = serializer;
- this.parameters = parameters;
- }
-
- public MessageOut<T> withParameter(String key, byte[] value)
- {
- ImmutableMap.Builder<String, byte[]> builder = ImmutableMap.builder();
- builder.putAll(parameters).put(key, value);
- return new MessageOut<T>(verb, payload, serializer, builder.build());
- }
-
- public Stage getStage()
- {
- return MessagingService.verbStages.get(verb);
- }
-
- public long getTimeout()
- {
- return verb.getTimeout();
- }
-
- public String toString()
- {
- StringBuilder sbuf = new StringBuilder();
- sbuf.append("TYPE:").append(getStage()).append(" VERB:").append(verb);
- return sbuf.toString();
- }
-
- public void serialize(DataOutputPlus out, int version) throws IOException
- {
- serialize(out, from, verb, parameters, payload, serializer, version);
- }
-
- public static <T> void serialize(DataOutputPlus out,
- InetAddress from,
- MessagingService.Verb verb,
- Map<String, byte[]> parameters,
- T payload,
- int version) throws IOException
- {
- IVersionedSerializer<T> serializer = (IVersionedSerializer<T>) MessagingService.instance().verbSerializers.get(verb);
- serialize(out, from, verb, parameters, payload, serializer, version);
- }
-
- public static <T> void serialize(DataOutputPlus out,
- InetAddress from,
- MessagingService.Verb verb,
- Map<String, byte[]> parameters,
- T payload,
- IVersionedSerializer<T> serializer,
- int version) throws IOException
- {
- CompactEndpointSerializationHelper.serialize(from, out);
-
- out.writeInt(MessagingService.Verb.convertForMessagingServiceVersion(verb, version).ordinal());
- out.writeInt(parameters.size());
- for (Map.Entry<String, byte[]> entry : parameters.entrySet())
- {
- out.writeUTF(entry.getKey());
- out.writeInt(entry.getValue().length);
- out.write(entry.getValue());
- }
-
- if (payload != null && serializer != MessagingService.CallbackDeterminedSerializer.instance)
- {
- try (DataOutputBuffer dob = DataOutputBuffer.scratchBuffer.get())
- {
- serializer.serialize(payload, dob, version);
-
- int size = dob.getLength();
- out.writeInt(size);
- out.write(dob.getData(), 0, size);
- }
- }
- else
- {
- out.writeInt(0);
- }
- }
-
- public int serializedSize(int version)
- {
- int size = CompactEndpointSerializationHelper.serializedSize(from);
-
- size += TypeSizes.sizeof(verb.ordinal());
- size += TypeSizes.sizeof(parameters.size());
- for (Map.Entry<String, byte[]> entry : parameters.entrySet())
- {
- size += TypeSizes.sizeof(entry.getKey());
- size += TypeSizes.sizeof(entry.getValue().length);
- size += entry.getValue().length;
- }
-
- long longSize = payloadSize(version);
- assert longSize <= Integer.MAX_VALUE; // larger values are supported in sstables but not messages
- size += TypeSizes.sizeof((int) longSize);
- size += longSize;
- return size;
- }
-
- /**
- * Calculate the size of the payload of this message for the specified protocol version
- * and memoize the result for the specified protocol version. Memoization only covers the protocol
- * version of the first invocation.
- *
- * It is not safe to call payloadSize concurrently from multiple threads unless it has already been invoked
- * once from a single thread and there is a happens before relationship between that invocation and other
- * threads concurrently invoking payloadSize.
- *
- * For instance it would be safe to invokePayload size to make a decision in the thread that created the message
- * and then hand it off to other threads via a thread-safe queue, volatile write, or synchronized/ReentrantLock.
- * @param version Protocol version to use when calculating payload size
- * @return Size of the payload of this message in bytes
- */
- public long payloadSize(int version)
- {
- if (payloadSize == -1)
- {
- payloadSize = payload == null ? 0 : serializer.serializedSize(payload, version);
- payloadSizeVersion = version;
- }
- else if (payloadSizeVersion != version)
- {
- return payload == null ? 0 : serializer.serializedSize(payload, version);
- }
- return payloadSize;
- }
-}
diff --git a/src/java/org/apache/cassandra/net/MessagingService.java b/src/java/org/apache/cassandra/net/MessagingService.java
index dfca087..beec083 100644
--- a/src/java/org/apache/cassandra/net/MessagingService.java
+++ b/src/java/org/apache/cassandra/net/MessagingService.java
@@ -17,464 +17,198 @@
*/
package org.apache.cassandra.net;
-import java.io.*;
-import java.net.*;
-import java.nio.channels.AsynchronousCloseException;
import java.nio.channels.ClosedChannelException;
-import java.nio.channels.ServerSocketChannel;
-import java.util.*;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.CopyOnWriteArraySet;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.stream.Collectors;
-import java.util.stream.StreamSupport;
-
-import javax.net.ssl.SSLHandshakeException;
+import java.util.concurrent.TimeoutException;
import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Function;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
-
-import org.cliffc.high_scale_lib.NonBlockingHashMap;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.cassandra.concurrent.ExecutorLocals;
+import io.netty.util.concurrent.Future;
import org.apache.cassandra.concurrent.ScheduledExecutors;
import org.apache.cassandra.concurrent.Stage;
-import org.apache.cassandra.concurrent.StageManager;
-import org.apache.cassandra.concurrent.LocalAwareExecutorService;
import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.config.EncryptionOptions.ServerEncryptionOptions;
-import org.apache.cassandra.db.*;
-import org.apache.cassandra.batchlog.Batch;
-import org.apache.cassandra.dht.AbstractBounds;
-import org.apache.cassandra.dht.BootStrapper;
-import org.apache.cassandra.dht.IPartitioner;
-import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.db.SystemKeyspace;
import org.apache.cassandra.exceptions.RequestFailureReason;
-import org.apache.cassandra.gms.EchoMessage;
-import org.apache.cassandra.gms.GossipDigestAck;
-import org.apache.cassandra.gms.GossipDigestAck2;
-import org.apache.cassandra.gms.GossipDigestSyn;
-import org.apache.cassandra.hints.HintMessage;
-import org.apache.cassandra.hints.HintResponse;
-import org.apache.cassandra.io.IVersionedSerializer;
-import org.apache.cassandra.io.util.DataInputPlus;
-import org.apache.cassandra.io.util.DataOutputPlus;
-import org.apache.cassandra.io.util.FileUtils;
-import org.apache.cassandra.locator.ILatencySubscriber;
-import org.apache.cassandra.metrics.CassandraMetricsRegistry;
-import org.apache.cassandra.metrics.ConnectionMetrics;
-import org.apache.cassandra.metrics.DroppedMessageMetrics;
-import org.apache.cassandra.metrics.MessagingMetrics;
-import org.apache.cassandra.repair.messages.RepairMessage;
-import org.apache.cassandra.security.SSLFactory;
-import org.apache.cassandra.service.*;
-import org.apache.cassandra.service.paxos.Commit;
-import org.apache.cassandra.service.paxos.PrepareResponse;
-import org.apache.cassandra.tracing.TraceState;
-import org.apache.cassandra.tracing.Tracing;
-import org.apache.cassandra.utils.*;
-import org.apache.cassandra.utils.concurrent.SimpleCondition;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.Replica;
+import org.apache.cassandra.service.AbstractWriteResponseHandler;
+import org.apache.cassandra.utils.ExecutorUtils;
+import org.apache.cassandra.utils.FBUtilities;
-public final class MessagingService implements MessagingServiceMBean
+import static java.util.Collections.synchronizedList;
+import static java.util.concurrent.TimeUnit.MINUTES;
+import static java.util.concurrent.TimeUnit.NANOSECONDS;
+import static org.apache.cassandra.concurrent.Stage.MUTATION;
+import static org.apache.cassandra.utils.Throwables.maybeFail;
+
+/**
+ * MessagingService implements all internode communication - with the exception of SSTable streaming (for now).
+ *
+ * Specifically, it's responsible for dispatch of outbound messages to other nodes and routing of inbound messages
+ * to their appropriate {@link IVerbHandler}.
+ *
+ * <h2>Using MessagingService: sending requests and responses</h2>
+ *
+ * The are two ways to send a {@link Message}, and you should pick one depending on the desired behaviour:
+ * 1. To send a request that expects a response back, use
+ * {@link #sendWithCallback(Message, InetAddressAndPort, RequestCallback)} method. Once a response
+ * message is received, {@link RequestCallback#onResponse(Message)} method will be invoked on the
+ * provided callback - in case of a success response. In case of a failure response (see {@link Verb#FAILURE_RSP}),
+ * or if a response doesn't arrive within verb's configured expiry time,
+ * {@link RequestCallback#onFailure(InetAddressAndPort, RequestFailureReason)} will be invoked instead.
+ * 2. To send a response back, or a message that expects no response, use {@link #send(Message, InetAddressAndPort)}
+ * method.
+ *
+ * See also: {@link Message#out(Verb, Object)}, {@link Message#responseWith(Object)},
+ * and {@link Message#failureResponse(RequestFailureReason)}.
+ *
+ * <h2>Using MessagingService: handling a request</h2>
+ *
+ * As described in the previous section, to handle responses you only need to implement {@link RequestCallback}
+ * interface - so long as your response verb handler is the default {@link ResponseVerbHandler}.
+ *
+ * There are two steps you need to perform to implement request handling:
+ * 1. Create a {@link IVerbHandler} to process incoming requests and responses for the new type (if applicable).
+ * 2. Add a new {@link Verb} to the enum for the new request type, and, if applicable, one for the response message.
+ *
+ * MessagingService will now automatically invoke your handler whenever a {@link Message} with this verb arrives.
+ *
+ * <h1>Architecture of MessagingService</h1>
+ *
+ * <h2>QOS</h2>
+ *
+ * Since our messaging protocol is TCP-based, and also doesn't yet support interleaving messages with each other,
+ * we need a way to prevent head-of-line blocking adversely affecting all messages - in particular, large messages
+ * being in the way of smaller ones. To achive that (somewhat), we maintain three messaging connections to and
+ * from each peer:
+ * - one for large messages - defined as being larger than {@link OutboundConnections#LARGE_MESSAGE_THRESHOLD}
+ * (65KiB by default)
+ * - one for small messages - defined as smaller than that threshold
+ * - and finally, a connection for urgent messages - usually small and/or that are important to arrive
+ * promptly, e.g. gossip-related ones
+ *
+ * <h2>Wire format and framing</h2>
+ *
+ * Small messages are grouped together into frames, and large messages are split over multiple frames.
+ * Framing provides application-level integrity protection to otherwise raw streams of data - we use
+ * CRC24 for frame headers and CRC32 for the entire payload. LZ4 is optionally used for compression.
+ *
+ * You can find the on-wire format description of individual messages in the comments for
+ * {@link Message.Serializer}, alongside with format evolution notes.
+ * For the list and descriptions of available frame decoders see {@link FrameDecoder} comments. You can
+ * find wire format documented in the javadoc of {@link FrameDecoder} implementations:
+ * see {@link FrameDecoderCrc} and {@link FrameDecoderLZ4} in particular.
+ *
+ * <h2>Architecture of outbound messaging</h2>
+ *
+ * {@link OutboundConnection} is the core class implementing outbound connection logic, with
+ * {@link OutboundConnection#enqueue(Message)} being its main entry point. The connections are initiated
+ * by {@link OutboundConnectionInitiator}.
+ *
+ * Netty pipeline for outbound messaging connections generally consists of the following handlers:
+ *
+ * [(optional) SslHandler] <- [FrameEncoder]
+ *
+ * {@link OutboundConnection} handles the entire lifetime of a connection: from the very first handshake
+ * to any necessary reconnects if necessary.
+ *
+ * Message-delivery flow varies depending on the connection type.
+ *
+ * For {@link ConnectionType#SMALL_MESSAGES} and {@link ConnectionType#URGENT_MESSAGES},
+ * {@link Message} serialization and delivery occurs directly on the event loop.
+ * See {@link OutboundConnection.EventLoopDelivery} for details.
+ *
+ * For {@link ConnectionType#LARGE_MESSAGES}, to ensure that servicing large messages doesn't block
+ * timely service of other requests, message serialization is offloaded to a companion thread pool
+ * ({@link SocketFactory#synchronousWorkExecutor}). Most of the work will be performed by
+ * {@link AsyncChannelOutputPlus}. Please see {@link OutboundConnection.LargeMessageDelivery}
+ * for details.
+ *
+ * To prevent fast clients, or slow nodes on the other end of the connection from overwhelming
+ * a host with enqueued, unsent messages on heap, we impose strict limits on how much memory enqueued,
+ * undelivered messages can claim.
+ *
+ * Every individual connection gets an exclusive permit quota to use - 4MiB by default; every endpoint
+ * (group of large, small, and urgent connection) is capped at, by default, at 128MiB of undelivered messages,
+ * and a global limit of 512MiB is imposed on all endpoints combined.
+ *
+ * On an attempt to {@link OutboundConnection#enqueue(Message)}, the connection will attempt to allocate
+ * permits for message-size number of bytes from its exclusive quota; if successful, it will add the
+ * message to the queue; if unsuccessful, it will need to allocate remainder from both endpoint and lobal
+ * reserves, and if it fails to do so, the message will be rejected, and its callbacks, if any,
+ * immediately expired.
+ *
+ * For a more detailed description please see the docs and comments of {@link OutboundConnection}.
+ *
+ * <h2>Architecture of inbound messaging</h2>
+ *
+ * {@link InboundMessageHandler} is the core class implementing inbound connection logic, paired
+ * with {@link FrameDecoder}. Inbound connections are initiated by {@link InboundConnectionInitiator}.
+ * The primary entry points to these classes are {@link FrameDecoder#channelRead(ShareableBytes)}
+ * and {@link InboundMessageHandler#process(FrameDecoder.Frame)}.
+ *
+ * Netty pipeline for inbound messaging connections generally consists of the following handlers:
+ *
+ * [(optional) SslHandler] -> [FrameDecoder] -> [InboundMessageHandler]
+ *
+ * {@link FrameDecoder} is responsible for decoding incoming frames and work stashing; {@link InboundMessageHandler}
+ * then takes decoded frames from the decoder and processes the messages contained in them.
+ *
+ * The flow differs between small and large messages. Small ones are deserialized immediately, and only
+ * then scheduled on the right thread pool for the {@link Verb} for execution. Large messages, OTOH,
+ * aren't deserialized until they are just about to be executed on the appropriate {@link Stage}.
+ *
+ * Similarly to outbound handling, inbound messaging imposes strict memory utilisation limits on individual
+ * endpoints and on global aggregate consumption, and implements simple flow control, to prevent a single
+ * fast endpoint from overwhelming a host.
+ *
+ * Every individual connection gets an exclusive permit quota to use - 4MiB by default; every endpoint
+ * (group of large, small, and urgent connection) is capped at, by default, at 128MiB of unprocessed messages,
+ * and a global limit of 512MiB is imposed on all endpoints combined.
+ *
+ * On arrival of a message header, the handler will attempt to allocate permits for message-size number
+ * of bytes from its exclusive quota; if successful, it will proceed to deserializing and processing the message.
+ * If unsuccessful, the handler will attempt to allocate the remainder from its endpoint and global reserve;
+ * if either allocation is unsuccessful, the handler will cease any further frame processing, and tell
+ * {@link FrameDecoder} to stop reading from the network; subsequently, it will put itself on a special
+ * {@link org.apache.cassandra.net.InboundMessageHandler.WaitQueue}, to be reactivated once more permits
+ * become available.
+ *
+ * For a more detailed description please see the docs and comments of {@link InboundMessageHandler} and
+ * {@link FrameDecoder}.
+ *
+ * <h2>Observability</h2>
+ *
+ * MessagingService exposes diagnostic counters for both outbound and inbound directions - received and sent
+ * bytes and message counts, overload bytes and message count, error bytes and error counts, and many more.
+ *
+ * See {@link org.apache.cassandra.metrics.InternodeInboundMetrics} and
+ * {@link org.apache.cassandra.metrics.InternodeOutboundMetrics} for JMX-exposed counters.
+ *
+ * We also provide {@code system_views.internode_inbound} and {@code system_views.internode_outbound} virtual tables -
+ * implemented in {@link org.apache.cassandra.db.virtual.InternodeInboundTable} and
+ * {@link org.apache.cassandra.db.virtual.InternodeOutboundTable} respectively.
+ */
+public final class MessagingService extends MessagingServiceMBeanImpl
{
- // Required to allow schema migrations while upgrading within the minor 3.0.x/3.x versions to 3.11+.
- // See CASSANDRA-13004 for details.
- public final static boolean FORCE_3_0_PROTOCOL_VERSION = Boolean.getBoolean("cassandra.force_3_0_protocol_version");
-
- public static final String MBEAN_NAME = "org.apache.cassandra.net:type=MessagingService";
+ private static final Logger logger = LoggerFactory.getLogger(MessagingService.class);
// 8 bits version, so don't waste versions
- public static final int VERSION_12 = 6;
- public static final int VERSION_20 = 7;
- public static final int VERSION_21 = 8;
- public static final int VERSION_22 = 9;
public static final int VERSION_30 = 10;
public static final int VERSION_3014 = 11;
- public static final int current_version = FORCE_3_0_PROTOCOL_VERSION ? VERSION_30 : VERSION_3014;
-
- public static final String FAILURE_CALLBACK_PARAM = "CAL_BAC";
- public static final byte[] ONE_BYTE = new byte[1];
- public static final String FAILURE_RESPONSE_PARAM = "FAIL";
- public static final String FAILURE_REASON_PARAM = "FAIL_REASON";
-
- /**
- * we preface every message with this number so the recipient can validate the sender is sane
- */
- public static final int PROTOCOL_MAGIC = 0xCA552DFA;
-
- private boolean allNodesAtLeast22 = true;
- private boolean allNodesAtLeast30 = true;
-
- public final MessagingMetrics metrics = new MessagingMetrics();
-
- /* All verb handler identifiers */
- public enum Verb
- {
- MUTATION
- {
- public long getTimeout()
- {
- return DatabaseDescriptor.getWriteRpcTimeout();
- }
- },
- HINT
- {
- public long getTimeout()
- {
- return DatabaseDescriptor.getWriteRpcTimeout();
- }
- },
- READ_REPAIR
- {
- public long getTimeout()
- {
- return DatabaseDescriptor.getWriteRpcTimeout();
- }
- },
- READ
- {
- public long getTimeout()
- {
- return DatabaseDescriptor.getReadRpcTimeout();
- }
- },
- REQUEST_RESPONSE, // client-initiated reads and writes
- BATCH_STORE
- {
- public long getTimeout()
- {
- return DatabaseDescriptor.getWriteRpcTimeout();
- }
- }, // was @Deprecated STREAM_INITIATE,
- BATCH_REMOVE
- {
- public long getTimeout()
- {
- return DatabaseDescriptor.getWriteRpcTimeout();
- }
- }, // was @Deprecated STREAM_INITIATE_DONE,
- @Deprecated STREAM_REPLY,
- @Deprecated STREAM_REQUEST,
- RANGE_SLICE
- {
- public long getTimeout()
- {
- return DatabaseDescriptor.getRangeRpcTimeout();
- }
- },
- @Deprecated BOOTSTRAP_TOKEN,
- @Deprecated TREE_REQUEST,
- @Deprecated TREE_RESPONSE,
- @Deprecated JOIN,
- GOSSIP_DIGEST_SYN,
- GOSSIP_DIGEST_ACK,
- GOSSIP_DIGEST_ACK2,
- @Deprecated DEFINITIONS_ANNOUNCE,
- DEFINITIONS_UPDATE,
- TRUNCATE
- {
- public long getTimeout()
- {
- return DatabaseDescriptor.getTruncateRpcTimeout();
- }
- },
- SCHEMA_CHECK,
- @Deprecated INDEX_SCAN,
- REPLICATION_FINISHED,
- INTERNAL_RESPONSE, // responses to internal calls
- COUNTER_MUTATION
- {
- public long getTimeout()
- {
- return DatabaseDescriptor.getCounterWriteRpcTimeout();
- }
- },
- @Deprecated STREAMING_REPAIR_REQUEST,
- @Deprecated STREAMING_REPAIR_RESPONSE,
- SNAPSHOT, // Similar to nt snapshot
- MIGRATION_REQUEST,
- GOSSIP_SHUTDOWN,
- _TRACE, // dummy verb so we can use MS.droppedMessagesMap
- ECHO,
- REPAIR_MESSAGE,
- PAXOS_PREPARE
- {
- public long getTimeout()
- {
- return DatabaseDescriptor.getWriteRpcTimeout();
- }
- },
- PAXOS_PROPOSE
- {
- public long getTimeout()
- {
- return DatabaseDescriptor.getWriteRpcTimeout();
- }
- },
- PAXOS_COMMIT
- {
- public long getTimeout()
- {
- return DatabaseDescriptor.getWriteRpcTimeout();
- }
- },
- @Deprecated PAGED_RANGE
- {
- public long getTimeout()
- {
- return DatabaseDescriptor.getRangeRpcTimeout();
- }
- },
- PING,
- // UNUSED verbs were used as padding for backward/forward compatability before 4.0,
- // but it wasn't quite as bullet/future proof as needed. We still need to keep these entries
- // around, at least for a major rev or two (post-4.0). see CASSANDRA-13993 for a discussion.
- // For now, though, the UNUSED are legacy values (placeholders, basically) that should only be used
- // for correctly adding VERBs that need to be emergency additions to 3.0/3.11.
- // We can reclaim them (their id's, to be correct) in future versions, if desired, though.
- UNUSED_2,
- UNUSED_3,
- UNUSED_4,
- UNUSED_5,
- ;
- // remember to add new verbs at the end, since we serialize by ordinal
-
- // This is to support a "late" choice of the verb based on the messaging service version.
- // See CASSANDRA-12249 for more details.
- public static Verb convertForMessagingServiceVersion(Verb verb, int version)
- {
- if (verb == PAGED_RANGE && version >= VERSION_30)
- return RANGE_SLICE;
-
- return verb;
- }
-
- public long getTimeout()
- {
- return DatabaseDescriptor.getRpcTimeout();
- }
- }
-
- public static final Verb[] verbValues = Verb.values();
-
- public static final EnumMap<MessagingService.Verb, Stage> verbStages = new EnumMap<MessagingService.Verb, Stage>(MessagingService.Verb.class)
- {{
- put(Verb.MUTATION, Stage.MUTATION);
- put(Verb.COUNTER_MUTATION, Stage.COUNTER_MUTATION);
- put(Verb.READ_REPAIR, Stage.MUTATION);
- put(Verb.HINT, Stage.MUTATION);
- put(Verb.TRUNCATE, Stage.MUTATION);
- put(Verb.PAXOS_PREPARE, Stage.MUTATION);
- put(Verb.PAXOS_PROPOSE, Stage.MUTATION);
- put(Verb.PAXOS_COMMIT, Stage.MUTATION);
- put(Verb.BATCH_STORE, Stage.MUTATION);
- put(Verb.BATCH_REMOVE, Stage.MUTATION);
-
- put(Verb.READ, Stage.READ);
- put(Verb.RANGE_SLICE, Stage.READ);
- put(Verb.INDEX_SCAN, Stage.READ);
- put(Verb.PAGED_RANGE, Stage.READ);
-
- put(Verb.REQUEST_RESPONSE, Stage.REQUEST_RESPONSE);
- put(Verb.INTERNAL_RESPONSE, Stage.INTERNAL_RESPONSE);
-
- put(Verb.STREAM_REPLY, Stage.MISC); // actually handled by FileStreamTask and streamExecutors
- put(Verb.STREAM_REQUEST, Stage.MISC);
- put(Verb.REPLICATION_FINISHED, Stage.MISC);
- put(Verb.SNAPSHOT, Stage.MISC);
-
- put(Verb.TREE_REQUEST, Stage.ANTI_ENTROPY);
- put(Verb.TREE_RESPONSE, Stage.ANTI_ENTROPY);
- put(Verb.STREAMING_REPAIR_REQUEST, Stage.ANTI_ENTROPY);
- put(Verb.STREAMING_REPAIR_RESPONSE, Stage.ANTI_ENTROPY);
- put(Verb.REPAIR_MESSAGE, Stage.ANTI_ENTROPY);
- put(Verb.GOSSIP_DIGEST_ACK, Stage.GOSSIP);
- put(Verb.GOSSIP_DIGEST_ACK2, Stage.GOSSIP);
- put(Verb.GOSSIP_DIGEST_SYN, Stage.GOSSIP);
- put(Verb.GOSSIP_SHUTDOWN, Stage.GOSSIP);
-
- put(Verb.DEFINITIONS_UPDATE, Stage.MIGRATION);
- put(Verb.SCHEMA_CHECK, Stage.MIGRATION);
- put(Verb.MIGRATION_REQUEST, Stage.MIGRATION);
- put(Verb.INDEX_SCAN, Stage.READ);
- put(Verb.REPLICATION_FINISHED, Stage.MISC);
- put(Verb.SNAPSHOT, Stage.MISC);
- put(Verb.ECHO, Stage.GOSSIP);
-
- put(Verb.UNUSED_2, Stage.INTERNAL_RESPONSE);
- put(Verb.UNUSED_3, Stage.INTERNAL_RESPONSE);
-
- put(Verb.PING, Stage.READ);
- }};
-
- /**
- * Messages we receive in IncomingTcpConnection have a Verb that tells us what kind of message it is.
- * Most of the time, this is enough to determine how to deserialize the message payload.
- * The exception is the REQUEST_RESPONSE verb, which just means "a reply to something you told me to do."
- * Traditionally, this was fine since each VerbHandler knew what type of payload it expected, and
- * handled the deserialization itself. Now that we do that in ITC, to avoid the extra copy to an
- * intermediary byte[] (See CASSANDRA-3716), we need to wire that up to the CallbackInfo object
- * (see below).
- */
- public final EnumMap<Verb, IVersionedSerializer<?>> verbSerializers = new EnumMap<Verb, IVersionedSerializer<?>>(Verb.class)
- {{
- put(Verb.REQUEST_RESPONSE, CallbackDeterminedSerializer.instance);
- put(Verb.INTERNAL_RESPONSE, CallbackDeterminedSerializer.instance);
-
- put(Verb.MUTATION, Mutation.serializer);
- put(Verb.READ_REPAIR, Mutation.serializer);
- put(Verb.READ, ReadCommand.readSerializer);
- put(Verb.RANGE_SLICE, ReadCommand.rangeSliceSerializer);
- put(Verb.PAGED_RANGE, ReadCommand.pagedRangeSerializer);
- put(Verb.BOOTSTRAP_TOKEN, BootStrapper.StringSerializer.instance);
- put(Verb.REPAIR_MESSAGE, RepairMessage.serializer);
- put(Verb.GOSSIP_DIGEST_ACK, GossipDigestAck.serializer);
- put(Verb.GOSSIP_DIGEST_ACK2, GossipDigestAck2.serializer);
- put(Verb.GOSSIP_DIGEST_SYN, GossipDigestSyn.serializer);
- put(Verb.DEFINITIONS_UPDATE, MigrationManager.MigrationsSerializer.instance);
- put(Verb.TRUNCATE, Truncation.serializer);
- put(Verb.REPLICATION_FINISHED, null);
- put(Verb.COUNTER_MUTATION, CounterMutation.serializer);
- put(Verb.SNAPSHOT, SnapshotCommand.serializer);
- put(Verb.ECHO, EchoMessage.serializer);
- put(Verb.PAXOS_PREPARE, Commit.serializer);
- put(Verb.PAXOS_PROPOSE, Commit.serializer);
- put(Verb.PAXOS_COMMIT, Commit.serializer);
- put(Verb.HINT, HintMessage.serializer);
- put(Verb.BATCH_STORE, Batch.serializer);
- put(Verb.BATCH_REMOVE, UUIDSerializer.serializer);
- put(Verb.PING, PingMessage.serializer);
- }};
-
- /**
- * A Map of what kind of serializer to wire up to a REQUEST_RESPONSE callback, based on outbound Verb.
- */
- public static final EnumMap<Verb, IVersionedSerializer<?>> callbackDeserializers = new EnumMap<Verb, IVersionedSerializer<?>>(Verb.class)
- {{
- put(Verb.MUTATION, WriteResponse.serializer);
- put(Verb.HINT, HintResponse.serializer);
- put(Verb.READ_REPAIR, WriteResponse.serializer);
- put(Verb.COUNTER_MUTATION, WriteResponse.serializer);
- put(Verb.RANGE_SLICE, ReadResponse.rangeSliceSerializer);
- put(Verb.PAGED_RANGE, ReadResponse.rangeSliceSerializer);
- put(Verb.READ, ReadResponse.serializer);
- put(Verb.TRUNCATE, TruncateResponse.serializer);
- put(Verb.SNAPSHOT, null);
-
- put(Verb.MIGRATION_REQUEST, MigrationManager.MigrationsSerializer.instance);
- put(Verb.SCHEMA_CHECK, UUIDSerializer.serializer);
- put(Verb.BOOTSTRAP_TOKEN, BootStrapper.StringSerializer.instance);
- put(Verb.REPLICATION_FINISHED, null);
-
- put(Verb.PAXOS_PREPARE, PrepareResponse.serializer);
- put(Verb.PAXOS_PROPOSE, BooleanSerializer.serializer);
-
- put(Verb.BATCH_STORE, WriteResponse.serializer);
- put(Verb.BATCH_REMOVE, WriteResponse.serializer);
- }};
-
- /* This records all the results mapped by message Id */
- private final ExpiringMap<Integer, CallbackInfo> callbacks;
-
- /**
- * a placeholder class that means "deserialize using the callback." We can't implement this without
- * special-case code in InboundTcpConnection because there is no way to pass the message id to IVersionedSerializer.
- */
- static class CallbackDeterminedSerializer implements IVersionedSerializer<Object>
- {
- public static final CallbackDeterminedSerializer instance = new CallbackDeterminedSerializer();
-
- public Object deserialize(DataInputPlus in, int version) throws IOException
- {
- throw new UnsupportedOperationException();
- }
-
- public void serialize(Object o, DataOutputPlus out, int version) throws IOException
- {
- throw new UnsupportedOperationException();
- }
-
- public long serializedSize(Object o, int version)
- {
- throw new UnsupportedOperationException();
- }
- }
-
- /* Lookup table for registering message handlers based on the verb. */
- private final Map<Verb, IVerbHandler> verbHandlers;
-
- private final ConcurrentMap<InetAddress, OutboundTcpConnectionPool> connectionManagers = new NonBlockingHashMap<>();
-
- private static final Logger logger = LoggerFactory.getLogger(MessagingService.class);
- private static final int LOG_DROPPED_INTERVAL_IN_MS = 5000;
-
- private final List<SocketThread> socketThreads = Lists.newArrayList();
- private final SimpleCondition listenGate;
-
- /**
- * Verbs it's okay to drop if the request has been queued longer than the request timeout. These
- * all correspond to client requests or something triggered by them; we don't want to
- * drop internal messages like bootstrap or repair notifications.
- */
- public static final EnumSet<Verb> DROPPABLE_VERBS = EnumSet.of(Verb._TRACE,
- Verb.MUTATION,
- Verb.COUNTER_MUTATION,
- Verb.HINT,
- Verb.READ_REPAIR,
- Verb.READ,
- Verb.RANGE_SLICE,
- Verb.PAGED_RANGE,
- Verb.REQUEST_RESPONSE,
- Verb.BATCH_STORE,
- Verb.BATCH_REMOVE);
-
- private static final class DroppedMessages
- {
- final DroppedMessageMetrics metrics;
- final AtomicInteger droppedInternal;
- final AtomicInteger droppedCrossNode;
-
- DroppedMessages(Verb verb)
- {
- this(new DroppedMessageMetrics(verb));
- }
-
- DroppedMessages(DroppedMessageMetrics metrics)
- {
- this.metrics = metrics;
- this.droppedInternal = new AtomicInteger(0);
- this.droppedCrossNode = new AtomicInteger(0);
- }
- }
-
- @VisibleForTesting
- public void resetDroppedMessagesMap(String scope)
- {
- for (Verb verb : droppedMessagesMap.keySet())
- droppedMessagesMap.put(verb, new DroppedMessages(new DroppedMessageMetrics(metricName -> {
- return new CassandraMetricsRegistry.MetricName("DroppedMessages", metricName, scope);
- })));
- }
-
- // total dropped message counts for server lifetime
- private final Map<Verb, DroppedMessages> droppedMessagesMap = new EnumMap<>(Verb.class);
-
- private final List<ILatencySubscriber> subscribers = new ArrayList<ILatencySubscriber>();
-
- // protocol versions of the other nodes in the cluster
- private final ConcurrentMap<InetAddress, Integer> versions = new NonBlockingHashMap<InetAddress, Integer>();
-
- // message sinks are a testing hook
- private final Set<IMessageSink> messageSinks = new CopyOnWriteArraySet<>();
-
- // back-pressure implementation
- private final BackPressureStrategy backPressure = DatabaseDescriptor.getBackPressureStrategy();
+ public static final int VERSION_40 = 12;
+ public static final int minimum_version = VERSION_30;
+ public static final int current_version = VERSION_40;
+ static AcceptVersions accept_messaging = new AcceptVersions(minimum_version, current_version);
+ static AcceptVersions accept_streaming = new AcceptVersions(current_version, current_version);
private static class MSHandle
{
@@ -486,96 +220,129 @@
return MSHandle.instance;
}
- private static class MSTestHandle
+ public final SocketFactory socketFactory = new SocketFactory();
+ public final LatencySubscribers latencySubscribers = new LatencySubscribers();
+ public final RequestCallbacks callbacks = new RequestCallbacks(this);
+
+ // a public hook for filtering messages intended for delivery to this node
+ public final InboundSink inboundSink = new InboundSink(this);
+
+ // the inbound global reserve limits and associated wait queue
+ private final InboundMessageHandlers.GlobalResourceLimits inboundGlobalReserveLimits = new InboundMessageHandlers.GlobalResourceLimits(
+ new ResourceLimits.Concurrent(DatabaseDescriptor.getInternodeApplicationReceiveQueueReserveGlobalCapacityInBytes()));
+
+ // the socket bindings we accept incoming connections on
+ private final InboundSockets inboundSockets = new InboundSockets(new InboundConnectionSettings()
+ .withHandlers(this::getInbound)
+ .withSocketFactory(socketFactory));
+
+ // a public hook for filtering messages intended for delivery to another node
+ public final OutboundSink outboundSink = new OutboundSink(this::doSend);
+
+ final ResourceLimits.Limit outboundGlobalReserveLimit =
+ new ResourceLimits.Concurrent(DatabaseDescriptor.getInternodeApplicationSendQueueReserveGlobalCapacityInBytes());
+
+ // back-pressure implementation
+ private final BackPressureStrategy backPressure = DatabaseDescriptor.getBackPressureStrategy();
+
+ private volatile boolean isShuttingDown;
+
+ @VisibleForTesting
+ MessagingService(boolean testOnly)
{
- public static final MessagingService instance = new MessagingService(true);
+ super(testOnly);
+ OutboundConnections.scheduleUnusedConnectionMonitoring(this, ScheduledExecutors.scheduledTasks, 1L, TimeUnit.HOURS);
}
- static MessagingService test()
+ /**
+ * Send a non-mutation message to a given endpoint. This method specifies a callback
+ * which is invoked with the actual response.
+ *
+ * @param message message to be sent.
+ * @param to endpoint to which the message needs to be sent
+ * @param cb callback interface which is used to pass the responses or
+ * suggest that a timeout occurred to the invoker of the send().
+ */
+ public void sendWithCallback(Message message, InetAddressAndPort to, RequestCallback cb)
{
- return MSTestHandle.instance;
+ sendWithCallback(message, to, cb, null);
}
- private MessagingService(boolean testOnly)
+ public void sendWithCallback(Message message, InetAddressAndPort to, RequestCallback cb, ConnectionType specifyConnection)
{
- for (Verb verb : DROPPABLE_VERBS)
- droppedMessagesMap.put(verb, new DroppedMessages(verb));
+ callbacks.addWithExpiration(cb, message, to);
+ updateBackPressureOnSend(to, cb, message);
+ if (cb.invokeOnFailure() && !message.callBackOnFailure())
+ message = message.withCallBackOnFailure();
+ send(message, to, specifyConnection);
+ }
- listenGate = new SimpleCondition();
- verbHandlers = new EnumMap<>(Verb.class);
- if (!testOnly)
+ /**
+ * Send a mutation message or a Paxos Commit to a given endpoint. This method specifies a callback
+ * which is invoked with the actual response.
+ * Also holds the message (only mutation messages) to determine if it
+ * needs to trigger a hint (uses StorageProxy for that).
+ *
+ * @param message message to be sent.
+ * @param to endpoint to which the message needs to be sent
+ * @param handler callback interface which is used to pass the responses or
+ * suggest that a timeout occurred to the invoker of the send().
+ */
+ public void sendWriteWithCallback(Message message, Replica to, AbstractWriteResponseHandler<?> handler, boolean allowHints)
+ {
+ assert message.callBackOnFailure();
+ callbacks.addWithExpiration(handler, message, to, handler.consistencyLevel(), allowHints);
+ updateBackPressureOnSend(to.endpoint(), handler, message);
+ send(message, to.endpoint(), null);
+ }
+
+ /**
+ * Send a message to a given endpoint. This method adheres to the fire and forget
+ * style messaging.
+ *
+ * @param message messages to be sent.
+ * @param to endpoint to which the message needs to be sent
+ */
+ public void send(Message message, InetAddressAndPort to)
+ {
+ send(message, to, null);
+ }
+
+ public void send(Message message, InetAddressAndPort to, ConnectionType specifyConnection)
+ {
+ if (logger.isTraceEnabled())
{
- Runnable logDropped = new Runnable()
- {
- public void run()
- {
- logDroppedMessages();
- }
- };
- ScheduledExecutors.scheduledTasks.scheduleWithFixedDelay(logDropped, LOG_DROPPED_INTERVAL_IN_MS, LOG_DROPPED_INTERVAL_IN_MS, TimeUnit.MILLISECONDS);
+ logger.trace("{} sending {} to {}@{}", FBUtilities.getBroadcastAddressAndPort(), message.verb(), message.id(), to);
+
+ if (to.equals(FBUtilities.getBroadcastAddressAndPort()))
+ logger.trace("Message-to-self {} going over MessagingService", message);
}
- Function<Pair<Integer, ExpiringMap.CacheableObject<CallbackInfo>>, ?> timeoutReporter = new Function<Pair<Integer, ExpiringMap.CacheableObject<CallbackInfo>>, Object>()
+ outboundSink.accept(message, to, specifyConnection);
+ }
+
+ private void doSend(Message message, InetAddressAndPort to, ConnectionType specifyConnection)
+ {
+ // expire the callback if the message failed to enqueue (failed to establish a connection or exceeded queue capacity)
+ while (true)
{
- public Object apply(Pair<Integer, ExpiringMap.CacheableObject<CallbackInfo>> pair)
+ OutboundConnections connections = getOutbound(to);
+ try
{
- final CallbackInfo expiredCallbackInfo = pair.right.value;
-
- maybeAddLatency(expiredCallbackInfo.callback, expiredCallbackInfo.target, pair.right.timeout);
-
- ConnectionMetrics.totalTimeouts.mark();
- getConnectionPool(expiredCallbackInfo.target).incrementTimeout();
-
- if (expiredCallbackInfo.callback.supportsBackPressure())
- {
- updateBackPressureOnReceive(expiredCallbackInfo.target, expiredCallbackInfo.callback, true);
- }
-
- if (expiredCallbackInfo.isFailureCallback())
- {
- StageManager.getStage(Stage.INTERNAL_RESPONSE).submit(new Runnable()
- {
- @Override
- public void run()
- {
- ((IAsyncCallbackWithFailure)expiredCallbackInfo.callback).onFailure(expiredCallbackInfo.target, RequestFailureReason.UNKNOWN);
- }
- });
- }
-
- if (expiredCallbackInfo.shouldHint())
- {
- Mutation mutation = ((WriteCallbackInfo) expiredCallbackInfo).mutation();
- return StorageProxy.submitHint(mutation, expiredCallbackInfo.target, null);
- }
-
- return null;
+ connections.enqueue(message, specifyConnection);
+ return;
}
- };
+ catch (ClosedChannelException e)
+ {
+ if (isShuttingDown)
+ return; // just drop the message, and let others clean up
- callbacks = new ExpiringMap<>(DatabaseDescriptor.getMinRpcTimeout(), timeoutReporter);
-
- if (!testOnly)
- {
- MBeanWrapper.instance.registerMBean(this, MBEAN_NAME);
+ // remove the connection and try again
+ channelManagers.remove(to, connections);
+ }
}
}
- public void addMessageSink(IMessageSink sink)
- {
- messageSinks.add(sink);
- }
-
- public void removeMessageSink(IMessageSink sink)
- {
- messageSinks.remove(sink);
- }
-
- public void clearMessageSinks()
- {
- messageSinks.clear();
- }
-
/**
* Updates the back-pressure state on sending to the given host if enabled and the given message callback supports it.
*
@@ -583,12 +350,13 @@
* @param callback The message callback.
* @param message The actual message.
*/
- public void updateBackPressureOnSend(InetAddress host, IAsyncCallback callback, MessageOut<?> message)
+ void updateBackPressureOnSend(InetAddressAndPort host, RequestCallback callback, Message<?> message)
{
if (DatabaseDescriptor.backPressureEnabled() && callback.supportsBackPressure())
{
- BackPressureState backPressureState = getConnectionPool(host).getBackPressureState();
- backPressureState.onMessageSent(message);
+ BackPressureState backPressureState = getBackPressureState(host);
+ if (backPressureState != null)
+ backPressureState.onMessageSent(message);
}
}
@@ -599,11 +367,13 @@
* @param callback The message callback.
* @param timeout True if updated following a timeout, false otherwise.
*/
- public void updateBackPressureOnReceive(InetAddress host, IAsyncCallback callback, boolean timeout)
+ void updateBackPressureOnReceive(InetAddressAndPort host, RequestCallback callback, boolean timeout)
{
if (DatabaseDescriptor.backPressureEnabled() && callback.supportsBackPressure())
{
- BackPressureState backPressureState = getConnectionPool(host).getBackPressureState();
+ BackPressureState backPressureState = getBackPressureState(host);
+ if (backPressureState == null)
+ return;
if (!timeout)
backPressureState.onResponseReceived();
else
@@ -620,357 +390,97 @@
* @param hosts The hosts to apply back-pressure to.
* @param timeoutInNanos The max back-pressure timeout.
*/
- public void applyBackPressure(Iterable<InetAddress> hosts, long timeoutInNanos)
+ public void applyBackPressure(Iterable<InetAddressAndPort> hosts, long timeoutInNanos)
{
if (DatabaseDescriptor.backPressureEnabled())
{
- backPressure.apply(StreamSupport.stream(hosts.spliterator(), false)
- .filter(h -> !h.equals(FBUtilities.getBroadcastAddress()))
- .map(h -> getConnectionPool(h).getBackPressureState())
- .collect(Collectors.toSet()), timeoutInNanos, TimeUnit.NANOSECONDS);
+ Set<BackPressureState> states = new HashSet<>();
+ for (InetAddressAndPort host : hosts)
+ {
+ if (host.equals(FBUtilities.getBroadcastAddressAndPort()))
+ continue;
+ states.add(getOutbound(host).getBackPressureState());
+ }
+ //noinspection unchecked
+ backPressure.apply(states, timeoutInNanos, NANOSECONDS);
}
}
+ BackPressureState getBackPressureState(InetAddressAndPort host)
+ {
+ return getOutbound(host).getBackPressureState();
+ }
+
+ void markExpiredCallback(InetAddressAndPort addr)
+ {
+ OutboundConnections conn = channelManagers.get(addr);
+ if (conn != null)
+ conn.incrementExpiredCallbackCount();
+ }
+
/**
- * Track latency information for the dynamic snitch
+ * Only to be invoked once we believe the endpoint will never be contacted again.
*
- * @param cb the callback associated with this message -- this lets us know if it's a message type we're interested in
- * @param address the host that replied to the message
- * @param latency
+ * We close the connection after a five minute delay, to give asynchronous operations a chance to terminate
*/
- public void maybeAddLatency(IAsyncCallback cb, InetAddress address, long latency)
+ public void closeOutbound(InetAddressAndPort to)
{
- if (cb.isLatencyForSnitch())
- addLatency(address, latency);
- }
-
- public void addLatency(InetAddress address, long latency)
- {
- for (ILatencySubscriber subscriber : subscribers)
- subscriber.receiveTiming(address, latency);
+ OutboundConnections pool = channelManagers.get(to);
+ if (pool != null)
+ pool.scheduleClose(5L, MINUTES, true)
+ .addListener(future -> channelManagers.remove(to, pool));
}
/**
- * called from gossiper when it notices a node is not responding.
+ * Only to be invoked once we believe the connections will never be used again.
*/
- public void convict(InetAddress ep)
+ void closeOutboundNow(OutboundConnections connections)
{
- logger.trace("Resetting pool for {}", ep);
- getConnectionPool(ep).reset();
- }
-
- public void listen()
- {
- callbacks.reset(); // hack to allow tests to stop/restart MS
- listen(FBUtilities.getLocalAddress());
- if (DatabaseDescriptor.shouldListenOnBroadcastAddress()
- && !FBUtilities.getLocalAddress().equals(FBUtilities.getBroadcastAddress()))
- {
- listen(FBUtilities.getBroadcastAddress());
- }
- listenGate.signalAll();
+ connections.close(true).addListener(
+ future -> channelManagers.remove(connections.template().to, connections)
+ );
}
/**
- * Listen on the specified port.
+ * Only to be invoked once we believe the connections will never be used again.
+ */
+ public void removeInbound(InetAddressAndPort from)
+ {
+ InboundMessageHandlers handlers = messageHandlers.remove(from);
+ if (null != handlers)
+ handlers.releaseMetrics();
+ }
+
+ /**
+ * Closes any current open channel/connection to the endpoint, but does not cause any message loss, and we will
+ * try to re-establish connections immediately
+ */
+ public void interruptOutbound(InetAddressAndPort to)
+ {
+ OutboundConnections pool = channelManagers.get(to);
+ if (pool != null)
+ pool.interrupt();
+ }
+
+ /**
+ * Reconnect to the peer using the given {@code addr}. Outstanding messages in each channel will be sent on the
+ * current channel. Typically this function is used for something like EC2 public IP addresses which need to be used
+ * for communication between EC2 regions.
*
- * @param localEp InetAddress whose port to listen on.
+ * @param address IP Address to identify the peer
+ * @param preferredAddress IP Address to use (and prefer) going forward for connecting to the peer
*/
- private void listen(InetAddress localEp) throws ConfigurationException
+ @SuppressWarnings("UnusedReturnValue")
+ public Future<Void> maybeReconnectWithNewIp(InetAddressAndPort address, InetAddressAndPort preferredAddress)
{
- for (ServerSocket ss : getServerSockets(localEp))
- {
- SocketThread th = new SocketThread(ss, "ACCEPT-" + localEp);
- th.start();
- socketThreads.add(th);
- }
- }
+ if (!SystemKeyspace.updatePreferredIP(address, preferredAddress))
+ return null;
- @SuppressWarnings("resource")
- private List<ServerSocket> getServerSockets(InetAddress localEp) throws ConfigurationException
- {
- final List<ServerSocket> ss = new ArrayList<ServerSocket>(2);
- if (DatabaseDescriptor.getServerEncryptionOptions().internode_encryption != ServerEncryptionOptions.InternodeEncryption.none)
- {
- try
- {
- ss.add(SSLFactory.getServerSocket(DatabaseDescriptor.getServerEncryptionOptions(), localEp, DatabaseDescriptor.getSSLStoragePort()));
- }
- catch (IOException e)
- {
- throw new ConfigurationException("Unable to create ssl socket", e);
- }
- // setReuseAddress happens in the factory.
- logger.info("Starting Encrypted Messaging Service on SSL port {}", DatabaseDescriptor.getSSLStoragePort());
- }
+ OutboundConnections messagingPool = channelManagers.get(address);
+ if (messagingPool != null)
+ return messagingPool.reconnectWithNewIp(preferredAddress);
- if (DatabaseDescriptor.getServerEncryptionOptions().internode_encryption != ServerEncryptionOptions.InternodeEncryption.all)
- {
- ServerSocketChannel serverChannel = null;
- try
- {
- serverChannel = ServerSocketChannel.open();
- }
- catch (IOException e)
- {
- throw new RuntimeException(e);
- }
- ServerSocket socket = serverChannel.socket();
- try
- {
- socket.setReuseAddress(true);
- }
- catch (SocketException e)
- {
- FileUtils.closeQuietly(socket);
- throw new ConfigurationException("Insufficient permissions to setReuseAddress", e);
- }
- InetSocketAddress address = new InetSocketAddress(localEp, DatabaseDescriptor.getStoragePort());
- try
- {
- socket.bind(address,500);
- }
- catch (BindException e)
- {
- FileUtils.closeQuietly(socket);
- if (e.getMessage().contains("in use"))
- throw new ConfigurationException(address + " is in use by another process. Change listen_address:storage_port in cassandra.yaml to values that do not conflict with other services");
- else if (e.getMessage().contains("Cannot assign requested address"))
- throw new ConfigurationException("Unable to bind to address " + address
- + ". Set listen_address in cassandra.yaml to an interface you can bind to, e.g., your private IP address on EC2");
- else
- throw new RuntimeException(e);
- }
- catch (IOException e)
- {
- FileUtils.closeQuietly(socket);
- throw new RuntimeException(e);
- }
- String nic = FBUtilities.getNetworkInterface(localEp);
- logger.info("Starting Messaging Service on {}:{}{}", localEp, DatabaseDescriptor.getStoragePort(),
- nic == null? "" : String.format(" (%s)", nic));
- ss.add(socket);
- }
- return ss;
- }
-
- public void waitUntilListening()
- {
- try
- {
- listenGate.await();
- }
- catch (InterruptedException ie)
- {
- logger.trace("await interrupted");
- }
- }
-
- public boolean isListening()
- {
- return listenGate.isSignaled();
- }
-
- public void destroyConnectionPool(InetAddress to)
- {
- OutboundTcpConnectionPool cp = connectionManagers.get(to);
- if (cp == null)
- return;
- cp.close();
- connectionManagers.remove(to);
- }
-
- public OutboundTcpConnectionPool getConnectionPool(InetAddress to)
- {
- OutboundTcpConnectionPool cp = connectionManagers.get(to);
- if (cp == null)
- {
- cp = new OutboundTcpConnectionPool(to, backPressure.newState(to));
- OutboundTcpConnectionPool existingPool = connectionManagers.putIfAbsent(to, cp);
- if (existingPool != null)
- cp = existingPool;
- else
- cp.start();
- }
- cp.waitForStarted();
- return cp;
- }
-
-
- public OutboundTcpConnection getConnection(InetAddress to, MessageOut msg)
- {
- return getConnectionPool(to).getConnection(msg);
- }
-
- /**
- * Register a verb and the corresponding verb handler with the
- * Messaging Service.
- *
- * @param verb
- * @param verbHandler handler for the specified verb
- */
- public void registerVerbHandlers(Verb verb, IVerbHandler verbHandler)
- {
- assert !verbHandlers.containsKey(verb);
- verbHandlers.put(verb, verbHandler);
- }
-
- /**
- * This method returns the verb handler associated with the registered
- * verb. If no handler has been registered then null is returned.
- *
- * @param type for which the verb handler is sought
- * @return a reference to IVerbHandler which is the handler for the specified verb
- */
- public IVerbHandler getVerbHandler(Verb type)
- {
- return verbHandlers.get(type);
- }
-
- public int addCallback(IAsyncCallback cb, MessageOut message, InetAddress to, long timeout, boolean failureCallback)
- {
- assert message.verb != Verb.MUTATION; // mutations need to call the overload with a ConsistencyLevel
- int messageId = nextId();
- CallbackInfo previous = callbacks.put(messageId, new CallbackInfo(to, cb, callbackDeserializers.get(message.verb), failureCallback), timeout);
- assert previous == null : String.format("Callback already exists for id %d! (%s)", messageId, previous);
- return messageId;
- }
-
- public int addCallback(IAsyncCallback cb,
- MessageOut<?> message,
- InetAddress to,
- long timeout,
- ConsistencyLevel consistencyLevel,
- boolean allowHints)
- {
- assert message.verb == Verb.MUTATION
- || message.verb == Verb.COUNTER_MUTATION
- || message.verb == Verb.PAXOS_COMMIT;
- int messageId = nextId();
-
- CallbackInfo previous = callbacks.put(messageId,
- new WriteCallbackInfo(to,
- cb,
- message,
- callbackDeserializers.get(message.verb),
- consistencyLevel,
- allowHints),
- timeout);
- assert previous == null : String.format("Callback already exists for id %d! (%s)", messageId, previous);
- return messageId;
- }
-
- private static final AtomicInteger idGen = new AtomicInteger(0);
-
- private static int nextId()
- {
- return idGen.incrementAndGet();
- }
-
- public int sendRR(MessageOut message, InetAddress to, IAsyncCallback cb)
- {
- return sendRR(message, to, cb, message.getTimeout(), false);
- }
-
- public int sendRRWithFailure(MessageOut message, InetAddress to, IAsyncCallbackWithFailure cb)
- {
- return sendRR(message, to, cb, message.getTimeout(), true);
- }
-
- /**
- * Send a non-mutation message to a given endpoint. This method specifies a callback
- * which is invoked with the actual response.
- *
- * @param message message to be sent.
- * @param to endpoint to which the message needs to be sent
- * @param cb callback interface which is used to pass the responses or
- * suggest that a timeout occurred to the invoker of the send().
- * @param timeout the timeout used for expiration
- * @return an reference to message id used to match with the result
- */
- public int sendRR(MessageOut message, InetAddress to, IAsyncCallback cb, long timeout, boolean failureCallback)
- {
- int id = addCallback(cb, message, to, timeout, failureCallback);
- updateBackPressureOnSend(to, cb, message);
- sendOneWay(failureCallback ? message.withParameter(FAILURE_CALLBACK_PARAM, ONE_BYTE) : message, id, to);
- return id;
- }
-
- /**
- * Send a mutation message or a Paxos Commit to a given endpoint. This method specifies a callback
- * which is invoked with the actual response.
- * Also holds the message (only mutation messages) to determine if it
- * needs to trigger a hint (uses StorageProxy for that).
- *
- * @param message message to be sent.
- * @param to endpoint to which the message needs to be sent
- * @param handler callback interface which is used to pass the responses or
- * suggest that a timeout occurred to the invoker of the send().
- * @return an reference to message id used to match with the result
- */
- public int sendRR(MessageOut<?> message,
- InetAddress to,
- AbstractWriteResponseHandler<?> handler,
- boolean allowHints)
- {
- int id = addCallback(handler, message, to, message.getTimeout(), handler.consistencyLevel, allowHints);
- updateBackPressureOnSend(to, handler, message);
- sendOneWay(message.withParameter(FAILURE_CALLBACK_PARAM, ONE_BYTE), id, to);
- return id;
- }
-
- public void sendOneWay(MessageOut message, InetAddress to)
- {
- sendOneWay(message, nextId(), to);
- }
-
- public void sendReply(MessageOut message, int id, InetAddress to)
- {
- sendOneWay(message, id, to);
- }
-
- /**
- * Send a message to a given endpoint. This method adheres to the fire and forget
- * style messaging.
- *
- * @param message messages to be sent.
- * @param to endpoint to which the message needs to be sent
- */
- public void sendOneWay(MessageOut message, int id, InetAddress to)
- {
- if (logger.isTraceEnabled())
- logger.trace("{} sending {} to {}@{}", FBUtilities.getBroadcastAddress(), message.verb, id, to);
-
- if (to.equals(FBUtilities.getBroadcastAddress()))
- logger.trace("Message-to-self {} going over MessagingService", message);
-
- // message sinks are a testing hook
- for (IMessageSink ms : messageSinks)
- if (!ms.allowOutgoingMessage(message, id, to))
- return;
-
- // get pooled connection (really, connection queue)
- OutboundTcpConnection connection = getConnection(to, message);
-
- // write it
- connection.enqueue(message, id);
- }
-
- public <T> AsyncOneResponse<T> sendRR(MessageOut message, InetAddress to)
- {
- AsyncOneResponse<T> iar = new AsyncOneResponse<T>();
- sendRR(message, to, iar);
- return iar;
- }
-
- public void register(ILatencySubscriber subcriber)
- {
- subscribers.add(subcriber);
- }
-
- public void clearCallbacksUnsafe()
- {
- callbacks.reset();
+ return null;
}
/**
@@ -978,567 +488,105 @@
*/
public void shutdown()
{
- shutdown(true);
+ shutdown(1L, MINUTES, true, true);
}
- public void shutdown(boolean gracefully)
+
+ public void shutdown(long timeout, TimeUnit units, boolean shutdownGracefully, boolean shutdownExecutors)
{
+ isShuttingDown = true;
logger.info("Waiting for messaging service to quiesce");
// We may need to schedule hints on the mutation stage, so it's erroneous to shut down the mutation stage first
- assert !StageManager.getStage(Stage.MUTATION).isShutdown();
+ assert !MUTATION.executor().isShutdown();
- // the important part
- if (!gracefully)
- callbacks.reset();
-
- if (!callbacks.shutdownBlocking())
- logger.warn("Failed to wait for messaging service callbacks shutdown");
-
- // attempt to humor tests that try to stop and restart MS
- try
+ if (shutdownGracefully)
{
- clearMessageSinks();
- for (SocketThread th : socketThreads)
- {
- try
- {
- th.close();
- }
- catch (IOException e)
- {
- // see https://issues.apache.org/jira/browse/CASSANDRA-10545
- handleIOExceptionOnClose(e);
- }
- }
- connectionManagers.values().forEach(OutboundTcpConnectionPool::close);
- }
- catch (IOException e)
- {
- throw new IOError(e);
- }
- }
+ callbacks.shutdownGracefully();
+ List<Future<Void>> closing = new ArrayList<>();
+ for (OutboundConnections pool : channelManagers.values())
+ closing.add(pool.close(true));
- public void receive(MessageIn message, int id)
- {
- TraceState state = Tracing.instance.initializeFromMessage(message);
- if (state != null)
- state.trace("{} message received from {}", message.verb, message.from);
-
- // message sinks are a testing hook
- for (IMessageSink ms : messageSinks)
- if (!ms.allowIncomingMessage(message, id))
- return;
-
- Runnable runnable = new MessageDeliveryTask(message, id);
- LocalAwareExecutorService stage = StageManager.getStage(message.getMessageType());
- assert stage != null : "No stage for message type " + message.verb;
-
- stage.execute(runnable, ExecutorLocals.create(state));
- }
-
- public void setCallbackForTests(int messageId, CallbackInfo callback)
- {
- callbacks.put(messageId, callback);
- }
-
- public CallbackInfo getRegisteredCallback(int messageId)
- {
- return callbacks.get(messageId);
- }
-
- public CallbackInfo removeRegisteredCallback(int messageId)
- {
- return callbacks.remove(messageId);
- }
-
- /**
- * @return System.nanoTime() when callback was created.
- */
- public long getRegisteredCallbackAge(int messageId)
- {
- return callbacks.getAge(messageId);
- }
-
- public static void validateMagic(int magic) throws IOException
- {
- if (magic != PROTOCOL_MAGIC)
- throw new IOException("invalid protocol header");
- }
-
- public static int getBits(int packed, int start, int count)
- {
- return packed >>> (start + 1) - count & ~(-1 << count);
- }
-
- public boolean areAllNodesAtLeast22()
- {
- return allNodesAtLeast22;
- }
-
- public boolean areAllNodesAtLeast30()
- {
- return allNodesAtLeast30;
- }
-
- /**
- * @return the last version associated with address, or @param version if this is the first such version
- */
- public int setVersion(InetAddress endpoint, int version)
- {
- logger.trace("Setting version {} for {}", version, endpoint);
-
- if (version < VERSION_22)
- allNodesAtLeast22 = false;
- if (version < VERSION_30)
- allNodesAtLeast30 = false;
-
- Integer v = versions.put(endpoint, version);
-
- // if the version was increased to 2.2 or later see if the min version across the cluster has changed
- if (v != null && (v < VERSION_30 && version >= VERSION_22))
- refreshAllNodeMinVersions();
-
- return v == null ? version : v;
- }
-
- public void resetVersion(InetAddress endpoint)
- {
- logger.trace("Resetting version for {}", endpoint);
- Integer removed = versions.remove(endpoint);
- if (removed != null && Math.min(removed, current_version) <= VERSION_30)
- refreshAllNodeMinVersions();
- }
-
- private void refreshAllNodeMinVersions()
- {
- boolean anyNodeLowerThan30 = false;
- for (Integer version : versions.values())
- {
- if (version < MessagingService.VERSION_30)
- {
- anyNodeLowerThan30 = true;
- allNodesAtLeast30 = false;
- }
-
- if (version < MessagingService.VERSION_22)
- {
- allNodesAtLeast22 = false;
- return;
- }
- }
- allNodesAtLeast22 = true;
- allNodesAtLeast30 = !anyNodeLowerThan30;
- }
-
- /**
- * Returns the messaging-version as announced by the given node but capped
- * to the min of the version as announced by the node and {@link #current_version}.
- */
- public int getVersion(InetAddress endpoint)
- {
- Integer v = versions.get(endpoint);
- if (v == null)
- {
- // we don't know the version. assume current. we'll know soon enough if that was incorrect.
- logger.trace("Assuming current protocol version for {}", endpoint);
- return MessagingService.current_version;
+ long deadline = System.nanoTime() + units.toNanos(timeout);
+ maybeFail(() -> new FutureCombiner(closing).get(timeout, units),
+ () -> {
+ List<ExecutorService> inboundExecutors = new ArrayList<>();
+ inboundSockets.close(synchronizedList(inboundExecutors)::add).get();
+ ExecutorUtils.awaitTermination(1L, TimeUnit.MINUTES, inboundExecutors);
+ },
+ () -> {
+ if (shutdownExecutors)
+ shutdownExecutors(deadline);
+ },
+ () -> callbacks.awaitTerminationUntil(deadline),
+ inboundSink::clear,
+ outboundSink::clear);
}
else
- return Math.min(v, MessagingService.current_version);
- }
-
- public int getVersion(String endpoint) throws UnknownHostException
- {
- return getVersion(InetAddress.getByName(endpoint));
- }
-
- /**
- * Returns the messaging-version exactly as announced by the given endpoint.
- */
- public int getRawVersion(InetAddress endpoint)
- {
- Integer v = versions.get(endpoint);
- if (v == null)
- throw new IllegalStateException("getRawVersion() was called without checking knowsVersion() result first");
- return v;
- }
-
- public boolean knowsVersion(InetAddress endpoint)
- {
- return versions.containsKey(endpoint);
- }
-
- public void incrementDroppedMutations(Optional<IMutation> mutationOpt, long timeTaken)
- {
- if (mutationOpt.isPresent())
{
- updateDroppedMutationCount(mutationOpt.get());
- }
- incrementDroppedMessages(Verb.MUTATION, timeTaken);
- }
+ callbacks.shutdownNow(false);
+ List<Future<Void>> closing = new ArrayList<>();
+ List<ExecutorService> inboundExecutors = synchronizedList(new ArrayList<ExecutorService>());
+ closing.add(inboundSockets.close(inboundExecutors::add));
+ for (OutboundConnections pool : channelManagers.values())
+ closing.add(pool.close(false));
- public void incrementDroppedMessages(Verb verb)
- {
- incrementDroppedMessages(verb, false);
- }
-
- public void incrementDroppedMessages(Verb verb, long timeTaken)
- {
- incrementDroppedMessages(verb, timeTaken, false);
- }
-
- public void incrementDroppedMessages(MessageIn message, long timeTaken)
- {
- if (message.payload instanceof IMutation)
- {
- updateDroppedMutationCount((IMutation) message.payload);
- }
- incrementDroppedMessages(message.verb, timeTaken, message.isCrossNode());
- }
-
- public void incrementDroppedMessages(Verb verb, long timeTaken, boolean isCrossNode)
- {
- assert DROPPABLE_VERBS.contains(verb) : "Verb " + verb + " should not legally be dropped";
- incrementDroppedMessages(droppedMessagesMap.get(verb), timeTaken, isCrossNode);
- }
-
- public void incrementDroppedMessages(Verb verb, boolean isCrossNode)
- {
- assert DROPPABLE_VERBS.contains(verb) : "Verb " + verb + " should not legally be dropped";
- incrementDroppedMessages(droppedMessagesMap.get(verb), isCrossNode);
- }
-
- private void updateDroppedMutationCount(IMutation mutation)
- {
- assert mutation != null : "Mutation should not be null when updating dropped mutations count";
-
- for (UUID columnFamilyId : mutation.getColumnFamilyIds())
- {
- ColumnFamilyStore cfs = Keyspace.open(mutation.getKeyspaceName()).getColumnFamilyStore(columnFamilyId);
- if (cfs != null)
- {
- cfs.metric.droppedMutations.inc();
- }
+ long deadline = System.nanoTime() + units.toNanos(timeout);
+ maybeFail(() -> new FutureCombiner(closing).get(timeout, units),
+ () -> {
+ if (shutdownExecutors)
+ shutdownExecutors(deadline);
+ },
+ () -> ExecutorUtils.awaitTermination(timeout, units, inboundExecutors),
+ () -> callbacks.awaitTerminationUntil(deadline),
+ inboundSink::clear,
+ outboundSink::clear);
}
}
- private void incrementDroppedMessages(DroppedMessages droppedMessages, long timeTaken, boolean isCrossNode)
+ private void shutdownExecutors(long deadlineNanos) throws TimeoutException, InterruptedException
{
- if (isCrossNode)
- droppedMessages.metrics.crossNodeDroppedLatency.update(timeTaken, TimeUnit.MILLISECONDS);
- else
- droppedMessages.metrics.internalDroppedLatency.update(timeTaken, TimeUnit.MILLISECONDS);
- incrementDroppedMessages(droppedMessages, isCrossNode);
+ socketFactory.shutdownNow();
+ socketFactory.awaitTerminationUntil(deadlineNanos);
}
- private void incrementDroppedMessages(DroppedMessages droppedMessages, boolean isCrossNode)
+ private OutboundConnections getOutbound(InetAddressAndPort to)
{
- droppedMessages.metrics.dropped.mark();
- if (isCrossNode)
- droppedMessages.droppedCrossNode.incrementAndGet();
- else
- droppedMessages.droppedInternal.incrementAndGet();
+ OutboundConnections connections = channelManagers.get(to);
+ if (connections == null)
+ connections = OutboundConnections.tryRegister(channelManagers, to, new OutboundConnectionSettings(to).withDefaults(ConnectionCategory.MESSAGING), backPressure.newState(to));
+ return connections;
}
- private void logDroppedMessages()
+ InboundMessageHandlers getInbound(InetAddressAndPort from)
{
- List<String> logs = getDroppedMessagesLogs();
- for (String log : logs)
- logger.info(log);
+ InboundMessageHandlers handlers = messageHandlers.get(from);
+ if (null != handlers)
+ return handlers;
- if (logs.size() > 0)
- StatusLogger.log();
+ return messageHandlers.computeIfAbsent(from, addr ->
+ new InboundMessageHandlers(FBUtilities.getLocalAddressAndPort(),
+ addr,
+ DatabaseDescriptor.getInternodeApplicationReceiveQueueCapacityInBytes(),
+ DatabaseDescriptor.getInternodeApplicationReceiveQueueReserveEndpointCapacityInBytes(),
+ inboundGlobalReserveLimits, metrics, inboundSink)
+ );
}
@VisibleForTesting
- List<String> getDroppedMessagesLogs()
+ boolean isConnected(InetAddressAndPort address, Message<?> messageOut)
{
- List<String> ret = new ArrayList<>();
- for (Map.Entry<Verb, DroppedMessages> entry : droppedMessagesMap.entrySet())
- {
- Verb verb = entry.getKey();
- DroppedMessages droppedMessages = entry.getValue();
-
- int droppedInternal = droppedMessages.droppedInternal.getAndSet(0);
- int droppedCrossNode = droppedMessages.droppedCrossNode.getAndSet(0);
- if (droppedInternal > 0 || droppedCrossNode > 0)
- {
- ret.add(String.format("%s messages were dropped in last %d ms: %d internal and %d cross node."
- + " Mean internal dropped latency: %d ms and Mean cross-node dropped latency: %d ms",
- verb,
- LOG_DROPPED_INTERVAL_IN_MS,
- droppedInternal,
- droppedCrossNode,
- TimeUnit.NANOSECONDS.toMillis((long)droppedMessages.metrics.internalDroppedLatency.getSnapshot().getMean()),
- TimeUnit.NANOSECONDS.toMillis((long)droppedMessages.metrics.crossNodeDroppedLatency.getSnapshot().getMean())));
- }
- }
- return ret;
+ OutboundConnections pool = channelManagers.get(address);
+ if (pool == null)
+ return false;
+ return pool.connectionFor(messageOut).isConnected();
}
- @VisibleForTesting
- public static class SocketThread extends Thread
+ public void listen()
{
- private final ServerSocket server;
- @VisibleForTesting
- public final Set<Closeable> connections = Sets.newConcurrentHashSet();
-
- SocketThread(ServerSocket server, String name)
- {
- super(name);
- this.server = server;
- }
-
- @SuppressWarnings("resource")
- public void run()
- {
- while (!server.isClosed())
- {
- Socket socket = null;
- try
- {
- socket = server.accept();
- if (!authenticate(socket))
- {
- logger.trace("remote failed to authenticate");
- socket.close();
- continue;
- }
-
- socket.setKeepAlive(true);
- socket.setSoTimeout(2 * OutboundTcpConnection.WAIT_FOR_VERSION_MAX_TIME);
- // determine the connection type to decide whether to buffer
- DataInputStream in = new DataInputStream(socket.getInputStream());
- MessagingService.validateMagic(in.readInt());
- int header = in.readInt();
- boolean isStream = MessagingService.getBits(header, 3, 1) == 1;
- int version = MessagingService.getBits(header, 15, 8);
- logger.trace("Connection version {} from {}", version, socket.getInetAddress());
- socket.setSoTimeout(0);
-
- Thread thread = isStream
- ? new IncomingStreamingConnection(version, socket, connections)
- : new IncomingTcpConnection(version, MessagingService.getBits(header, 2, 1) == 1, socket, connections);
- thread.start();
- connections.add((Closeable) thread);
- }
- catch (AsynchronousCloseException e)
- {
- // this happens when another thread calls close().
- logger.trace("Asynchronous close seen by server thread");
- break;
- }
- catch (ClosedChannelException e)
- {
- logger.trace("MessagingService server thread already closed");
- break;
- }
- catch (SSLHandshakeException e)
- {
- logger.error("SSL handshake error for inbound connection from " + socket, e);
- FileUtils.closeQuietly(socket);
- }
- catch (Throwable t)
- {
- logger.trace("Error reading the socket {}", socket, t);
- FileUtils.closeQuietly(socket);
- }
- }
- logger.info("MessagingService has terminated the accept() thread");
- }
-
- void close() throws IOException
- {
- logger.trace("Closing accept() thread");
-
- try
- {
- server.close();
- }
- catch (IOException e)
- {
- // see https://issues.apache.org/jira/browse/CASSANDRA-8220
- // see https://issues.apache.org/jira/browse/CASSANDRA-12513
- handleIOExceptionOnClose(e);
- }
- for (Closeable connection : connections)
- {
- connection.close();
- }
- }
-
- private boolean authenticate(Socket socket)
- {
- return DatabaseDescriptor.getInternodeAuthenticator().authenticate(socket.getInetAddress(), socket.getPort());
- }
+ inboundSockets.open();
}
- private static void handleIOExceptionOnClose(IOException e) throws IOException
+ public void waitUntilListening() throws InterruptedException
{
- // dirty hack for clean shutdown on OSX w/ Java >= 1.8.0_20
- // see https://bugs.openjdk.java.net/browse/JDK-8050499;
- // also CASSANDRA-12513
- if ("Mac OS X".equals(System.getProperty("os.name")))
- {
- switch (e.getMessage())
- {
- case "Unknown error: 316":
- case "No such file or directory":
- case "Bad file descriptor":
- case "Thread signal failed":
- return;
- }
- }
-
- throw e;
- }
-
- public Map<String, Integer> getLargeMessagePendingTasks()
- {
- Map<String, Integer> pendingTasks = new HashMap<String, Integer>(connectionManagers.size());
- for (Map.Entry<InetAddress, OutboundTcpConnectionPool> entry : connectionManagers.entrySet())
- pendingTasks.put(entry.getKey().getHostAddress(), entry.getValue().largeMessages.getPendingMessages());
- return pendingTasks;
- }
-
- public int getLargeMessagePendingTasks(InetAddress address)
- {
- OutboundTcpConnectionPool connection = connectionManagers.get(address);
- return connection == null ? 0 : connection.largeMessages.getPendingMessages();
- }
-
- public Map<String, Long> getLargeMessageCompletedTasks()
- {
- Map<String, Long> completedTasks = new HashMap<String, Long>(connectionManagers.size());
- for (Map.Entry<InetAddress, OutboundTcpConnectionPool> entry : connectionManagers.entrySet())
- completedTasks.put(entry.getKey().getHostAddress(), entry.getValue().largeMessages.getCompletedMesssages());
- return completedTasks;
- }
-
- public Map<String, Long> getLargeMessageDroppedTasks()
- {
- Map<String, Long> droppedTasks = new HashMap<String, Long>(connectionManagers.size());
- for (Map.Entry<InetAddress, OutboundTcpConnectionPool> entry : connectionManagers.entrySet())
- droppedTasks.put(entry.getKey().getHostAddress(), entry.getValue().largeMessages.getDroppedMessages());
- return droppedTasks;
- }
-
- public Map<String, Integer> getSmallMessagePendingTasks()
- {
- Map<String, Integer> pendingTasks = new HashMap<String, Integer>(connectionManagers.size());
- for (Map.Entry<InetAddress, OutboundTcpConnectionPool> entry : connectionManagers.entrySet())
- pendingTasks.put(entry.getKey().getHostAddress(), entry.getValue().smallMessages.getPendingMessages());
- return pendingTasks;
- }
-
- public Map<String, Long> getSmallMessageCompletedTasks()
- {
- Map<String, Long> completedTasks = new HashMap<String, Long>(connectionManagers.size());
- for (Map.Entry<InetAddress, OutboundTcpConnectionPool> entry : connectionManagers.entrySet())
- completedTasks.put(entry.getKey().getHostAddress(), entry.getValue().smallMessages.getCompletedMesssages());
- return completedTasks;
- }
-
- public Map<String, Long> getSmallMessageDroppedTasks()
- {
- Map<String, Long> droppedTasks = new HashMap<String, Long>(connectionManagers.size());
- for (Map.Entry<InetAddress, OutboundTcpConnectionPool> entry : connectionManagers.entrySet())
- droppedTasks.put(entry.getKey().getHostAddress(), entry.getValue().smallMessages.getDroppedMessages());
- return droppedTasks;
- }
-
- public Map<String, Integer> getGossipMessagePendingTasks()
- {
- Map<String, Integer> pendingTasks = new HashMap<String, Integer>(connectionManagers.size());
- for (Map.Entry<InetAddress, OutboundTcpConnectionPool> entry : connectionManagers.entrySet())
- pendingTasks.put(entry.getKey().getHostAddress(), entry.getValue().gossipMessages.getPendingMessages());
- return pendingTasks;
- }
-
- public Map<String, Long> getGossipMessageCompletedTasks()
- {
- Map<String, Long> completedTasks = new HashMap<String, Long>(connectionManagers.size());
- for (Map.Entry<InetAddress, OutboundTcpConnectionPool> entry : connectionManagers.entrySet())
- completedTasks.put(entry.getKey().getHostAddress(), entry.getValue().gossipMessages.getCompletedMesssages());
- return completedTasks;
- }
-
- public Map<String, Long> getGossipMessageDroppedTasks()
- {
- Map<String, Long> droppedTasks = new HashMap<String, Long>(connectionManagers.size());
- for (Map.Entry<InetAddress, OutboundTcpConnectionPool> entry : connectionManagers.entrySet())
- droppedTasks.put(entry.getKey().getHostAddress(), entry.getValue().gossipMessages.getDroppedMessages());
- return droppedTasks;
- }
-
- public Map<String, Integer> getDroppedMessages()
- {
- Map<String, Integer> map = new HashMap<>(droppedMessagesMap.size());
- for (Map.Entry<Verb, DroppedMessages> entry : droppedMessagesMap.entrySet())
- map.put(entry.getKey().toString(), (int) entry.getValue().metrics.dropped.getCount());
- return map;
- }
-
-
- public long getTotalTimeouts()
- {
- return ConnectionMetrics.totalTimeouts.getCount();
- }
-
- public Map<String, Long> getTimeoutsPerHost()
- {
- Map<String, Long> result = new HashMap<String, Long>(connectionManagers.size());
- for (Map.Entry<InetAddress, OutboundTcpConnectionPool> entry: connectionManagers.entrySet())
- {
- String ip = entry.getKey().getHostAddress();
- long recent = entry.getValue().getTimeouts();
- result.put(ip, recent);
- }
- return result;
- }
-
- public Map<String, Double> getBackPressurePerHost()
- {
- Map<String, Double> map = new HashMap<>(connectionManagers.size());
- for (Map.Entry<InetAddress, OutboundTcpConnectionPool> entry : connectionManagers.entrySet())
- map.put(entry.getKey().getHostAddress(), entry.getValue().getBackPressureState().getBackPressureRateLimit());
-
- return map;
- }
-
- @Override
- public void setBackPressureEnabled(boolean enabled)
- {
- DatabaseDescriptor.setBackPressureEnabled(enabled);
- }
-
- @Override
- public boolean isBackPressureEnabled()
- {
- return DatabaseDescriptor.backPressureEnabled();
- }
-
- public static IPartitioner globalPartitioner()
- {
- return StorageService.instance.getTokenMetadata().partitioner;
- }
-
- public static void validatePartitioner(Collection<? extends AbstractBounds<?>> allBounds)
- {
- for (AbstractBounds<?> bounds : allBounds)
- validatePartitioner(bounds);
- }
-
- public static void validatePartitioner(AbstractBounds<?> bounds)
- {
- if (globalPartitioner() != bounds.left.getPartitioner())
- throw new AssertionError(String.format("Partitioner in bounds serialization. Expected %s, was %s.",
- globalPartitioner().getClass().getName(),
- bounds.left.getPartitioner().getClass().getName()));
- }
-
- @VisibleForTesting
- public List<SocketThread> getSocketThreads()
- {
- return socketThreads;
+ inboundSockets.open().await();
}
}
diff --git a/src/java/org/apache/cassandra/net/MessagingServiceMBean.java b/src/java/org/apache/cassandra/net/MessagingServiceMBean.java
index b2e79e0..732a5ed 100644
--- a/src/java/org/apache/cassandra/net/MessagingServiceMBean.java
+++ b/src/java/org/apache/cassandra/net/MessagingServiceMBean.java
@@ -19,6 +19,7 @@
+import java.io.IOException;
import java.net.UnknownHostException;
import java.util.Map;
@@ -30,47 +31,69 @@
/**
* Pending tasks for large message TCP Connections
*/
+ @Deprecated
public Map<String, Integer> getLargeMessagePendingTasks();
+ public Map<String, Integer> getLargeMessagePendingTasksWithPort();
/**
* Completed tasks for large message) TCP Connections
*/
+ @Deprecated
public Map<String, Long> getLargeMessageCompletedTasks();
+ public Map<String, Long> getLargeMessageCompletedTasksWithPort();
/**
* Dropped tasks for large message TCP Connections
*/
+ @Deprecated
public Map<String, Long> getLargeMessageDroppedTasks();
+ public Map<String, Long> getLargeMessageDroppedTasksWithPort();
+
/**
* Pending tasks for small message TCP Connections
*/
+ @Deprecated
public Map<String, Integer> getSmallMessagePendingTasks();
+ public Map<String, Integer> getSmallMessagePendingTasksWithPort();
+
/**
* Completed tasks for small message TCP Connections
*/
+ @Deprecated
public Map<String, Long> getSmallMessageCompletedTasks();
+ public Map<String, Long> getSmallMessageCompletedTasksWithPort();
+
/**
* Dropped tasks for small message TCP Connections
*/
+ @Deprecated
public Map<String, Long> getSmallMessageDroppedTasks();
+ public Map<String, Long> getSmallMessageDroppedTasksWithPort();
+
/**
* Pending tasks for gossip message TCP Connections
*/
+ @Deprecated
public Map<String, Integer> getGossipMessagePendingTasks();
+ public Map<String, Integer> getGossipMessagePendingTasksWithPort();
/**
* Completed tasks for gossip message TCP Connections
*/
+ @Deprecated
public Map<String, Long> getGossipMessageCompletedTasks();
+ public Map<String, Long> getGossipMessageCompletedTasksWithPort();
/**
* Dropped tasks for gossip message TCP Connections
*/
+ @Deprecated
public Map<String, Long> getGossipMessageDroppedTasks();
+ public Map<String, Long> getGossipMessageDroppedTasksWithPort();
/**
* dropped message counts for server lifetime
@@ -85,12 +108,16 @@
/**
* Number of timeouts per host
*/
+ @Deprecated
public Map<String, Long> getTimeoutsPerHost();
+ public Map<String, Long> getTimeoutsPerHostWithPort();
/**
* Back-pressure rate limiting per host
*/
+ @Deprecated
public Map<String, Double> getBackPressurePerHost();
+ public Map<String, Double> getBackPressurePerHostWithPort();
/**
* Enable/Disable back-pressure
@@ -103,4 +130,6 @@
public boolean isBackPressureEnabled();
public int getVersion(String address) throws UnknownHostException;
+
+ void reloadSslCertificates() throws IOException;
}
diff --git a/src/java/org/apache/cassandra/net/MessagingServiceMBeanImpl.java b/src/java/org/apache/cassandra/net/MessagingServiceMBeanImpl.java
new file mode 100644
index 0000000..b48ae1c
--- /dev/null
+++ b/src/java/org/apache/cassandra/net/MessagingServiceMBeanImpl.java
@@ -0,0 +1,304 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.net;
+
+import java.io.IOException;
+import java.net.UnknownHostException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.config.EncryptionOptions;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.metrics.InternodeOutboundMetrics;
+import org.apache.cassandra.metrics.MessagingMetrics;
+import org.apache.cassandra.security.SSLFactory;
+import org.apache.cassandra.utils.MBeanWrapper;
+
+public class MessagingServiceMBeanImpl implements MessagingServiceMBean
+{
+ public static final String MBEAN_NAME = "org.apache.cassandra.net:type=MessagingService";
+
+ // we use CHM deliberately instead of NBHM, as both are non-blocking for readers (which this map mostly is used for)
+ // and CHM permits prompter GC
+ public final ConcurrentMap<InetAddressAndPort, OutboundConnections> channelManagers = new ConcurrentHashMap<>();
+ public final ConcurrentMap<InetAddressAndPort, InboundMessageHandlers> messageHandlers = new ConcurrentHashMap<>();
+
+ public final EndpointMessagingVersions versions = new EndpointMessagingVersions();
+ public final MessagingMetrics metrics = new MessagingMetrics();
+
+ MessagingServiceMBeanImpl(boolean testOnly)
+ {
+ if (!testOnly)
+ {
+ MBeanWrapper.instance.registerMBean(this, MBEAN_NAME);
+ metrics.scheduleLogging();
+ }
+ }
+
+ @Override
+ public Map<String, Integer> getLargeMessagePendingTasks()
+ {
+ Map<String, Integer> pendingTasks = new HashMap<>(channelManagers.size());
+ for (Map.Entry<InetAddressAndPort, OutboundConnections> entry : channelManagers.entrySet())
+ pendingTasks.put(entry.getKey().toString(false), entry.getValue().large.pendingCount());
+ return pendingTasks;
+ }
+
+ @Override
+ public Map<String, Long> getLargeMessageCompletedTasks()
+ {
+ Map<String, Long> completedTasks = new HashMap<>(channelManagers.size());
+ for (Map.Entry<InetAddressAndPort, OutboundConnections> entry : channelManagers.entrySet())
+ completedTasks.put(entry.getKey().toString(false), entry.getValue().large.sentCount());
+ return completedTasks;
+ }
+
+ @Override
+ public Map<String, Long> getLargeMessageDroppedTasks()
+ {
+ Map<String, Long> droppedTasks = new HashMap<>(channelManagers.size());
+ for (Map.Entry<InetAddressAndPort, OutboundConnections> entry : channelManagers.entrySet())
+ droppedTasks.put(entry.getKey().toString(false), entry.getValue().large.dropped());
+ return droppedTasks;
+ }
+
+ @Override
+ public Map<String, Integer> getSmallMessagePendingTasks()
+ {
+ Map<String, Integer> pendingTasks = new HashMap<>(channelManagers.size());
+ for (Map.Entry<InetAddressAndPort, OutboundConnections> entry : channelManagers.entrySet())
+ pendingTasks.put(entry.getKey().toString(false), entry.getValue().small.pendingCount());
+ return pendingTasks;
+ }
+
+ @Override
+ public Map<String, Long> getSmallMessageCompletedTasks()
+ {
+ Map<String, Long> completedTasks = new HashMap<>(channelManagers.size());
+ for (Map.Entry<InetAddressAndPort, OutboundConnections> entry : channelManagers.entrySet())
+ completedTasks.put(entry.getKey().toString(false), entry.getValue().small.sentCount());
+ return completedTasks;
+ }
+
+ @Override
+ public Map<String, Long> getSmallMessageDroppedTasks()
+ {
+ Map<String, Long> droppedTasks = new HashMap<>(channelManagers.size());
+ for (Map.Entry<InetAddressAndPort, OutboundConnections> entry : channelManagers.entrySet())
+ droppedTasks.put(entry.getKey().toString(false), entry.getValue().small.dropped());
+ return droppedTasks;
+ }
+
+ @Override
+ public Map<String, Integer> getGossipMessagePendingTasks()
+ {
+ Map<String, Integer> pendingTasks = new HashMap<>(channelManagers.size());
+ for (Map.Entry<InetAddressAndPort, OutboundConnections> entry : channelManagers.entrySet())
+ pendingTasks.put(entry.getKey().toString(false), entry.getValue().urgent.pendingCount());
+ return pendingTasks;
+ }
+
+ @Override
+ public Map<String, Long> getGossipMessageCompletedTasks()
+ {
+ Map<String, Long> completedTasks = new HashMap<>(channelManagers.size());
+ for (Map.Entry<InetAddressAndPort, OutboundConnections> entry : channelManagers.entrySet())
+ completedTasks.put(entry.getKey().toString(false), entry.getValue().urgent.sentCount());
+ return completedTasks;
+ }
+
+ @Override
+ public Map<String, Long> getGossipMessageDroppedTasks()
+ {
+ Map<String, Long> droppedTasks = new HashMap<>(channelManagers.size());
+ for (Map.Entry<InetAddressAndPort, OutboundConnections> entry : channelManagers.entrySet())
+ droppedTasks.put(entry.getKey().toString(false), entry.getValue().urgent.dropped());
+ return droppedTasks;
+ }
+
+ @Override
+ public Map<String, Integer> getLargeMessagePendingTasksWithPort()
+ {
+ Map<String, Integer> pendingTasks = new HashMap<>(channelManagers.size());
+ for (Map.Entry<InetAddressAndPort, OutboundConnections> entry : channelManagers.entrySet())
+ pendingTasks.put(entry.getKey().toString(), entry.getValue().large.pendingCount());
+ return pendingTasks;
+ }
+
+ @Override
+ public Map<String, Long> getLargeMessageCompletedTasksWithPort()
+ {
+ Map<String, Long> completedTasks = new HashMap<>(channelManagers.size());
+ for (Map.Entry<InetAddressAndPort, OutboundConnections> entry : channelManagers.entrySet())
+ completedTasks.put(entry.getKey().toString(), entry.getValue().large.sentCount());
+ return completedTasks;
+ }
+
+ @Override
+ public Map<String, Long> getLargeMessageDroppedTasksWithPort()
+ {
+ Map<String, Long> droppedTasks = new HashMap<>(channelManagers.size());
+ for (Map.Entry<InetAddressAndPort, OutboundConnections> entry : channelManagers.entrySet())
+ droppedTasks.put(entry.getKey().toString(), entry.getValue().large.dropped());
+ return droppedTasks;
+ }
+
+ @Override
+ public Map<String, Integer> getSmallMessagePendingTasksWithPort()
+ {
+ Map<String, Integer> pendingTasks = new HashMap<>(channelManagers.size());
+ for (Map.Entry<InetAddressAndPort, OutboundConnections> entry : channelManagers.entrySet())
+ pendingTasks.put(entry.getKey().toString(), entry.getValue().small.pendingCount());
+ return pendingTasks;
+ }
+
+ @Override
+ public Map<String, Long> getSmallMessageCompletedTasksWithPort()
+ {
+ Map<String, Long> completedTasks = new HashMap<>(channelManagers.size());
+ for (Map.Entry<InetAddressAndPort, OutboundConnections> entry : channelManagers.entrySet())
+ completedTasks.put(entry.getKey().toString(), entry.getValue().small.sentCount());
+ return completedTasks;
+ }
+
+ @Override
+ public Map<String, Long> getSmallMessageDroppedTasksWithPort()
+ {
+ Map<String, Long> droppedTasks = new HashMap<>(channelManagers.size());
+ for (Map.Entry<InetAddressAndPort, OutboundConnections> entry : channelManagers.entrySet())
+ droppedTasks.put(entry.getKey().toString(), entry.getValue().small.dropped());
+ return droppedTasks;
+ }
+
+ @Override
+ public Map<String, Integer> getGossipMessagePendingTasksWithPort()
+ {
+ Map<String, Integer> pendingTasks = new HashMap<>(channelManagers.size());
+ for (Map.Entry<InetAddressAndPort, OutboundConnections> entry : channelManagers.entrySet())
+ pendingTasks.put(entry.getKey().toString(), entry.getValue().urgent.pendingCount());
+ return pendingTasks;
+ }
+
+ @Override
+ public Map<String, Long> getGossipMessageCompletedTasksWithPort()
+ {
+ Map<String, Long> completedTasks = new HashMap<>(channelManagers.size());
+ for (Map.Entry<InetAddressAndPort, OutboundConnections> entry : channelManagers.entrySet())
+ completedTasks.put(entry.getKey().toString(), entry.getValue().urgent.sentCount());
+ return completedTasks;
+ }
+
+ @Override
+ public Map<String, Long> getGossipMessageDroppedTasksWithPort()
+ {
+ Map<String, Long> droppedTasks = new HashMap<>(channelManagers.size());
+ for (Map.Entry<InetAddressAndPort, OutboundConnections> entry : channelManagers.entrySet())
+ droppedTasks.put(entry.getKey().toString(), entry.getValue().urgent.dropped());
+ return droppedTasks;
+ }
+
+ @Override
+ public Map<String, Integer> getDroppedMessages()
+ {
+ return metrics.getDroppedMessages();
+ }
+
+ @Override
+ public long getTotalTimeouts()
+ {
+ return InternodeOutboundMetrics.totalExpiredCallbacks.getCount();
+ }
+
+ // these are not messages that time out on sending, but callbacks that timedout without receiving a response
+ @Override
+ public Map<String, Long> getTimeoutsPerHost()
+ {
+ Map<String, Long> result = new HashMap<>(channelManagers.size());
+ for (Map.Entry<InetAddressAndPort, OutboundConnections> entry : channelManagers.entrySet())
+ {
+ String ip = entry.getKey().toString(false);
+ long recent = entry.getValue().expiredCallbacks();
+ result.put(ip, recent);
+ }
+ return result;
+ }
+
+ // these are not messages that time out on sending, but callbacks that timedout without receiving a response
+ @Override
+ public Map<String, Long> getTimeoutsPerHostWithPort()
+ {
+ Map<String, Long> result = new HashMap<>(channelManagers.size());
+ for (Map.Entry<InetAddressAndPort, OutboundConnections> entry : channelManagers.entrySet())
+ {
+ String ip = entry.getKey().toString();
+ long recent = entry.getValue().expiredCallbacks();
+ result.put(ip, recent);
+ }
+ return result;
+ }
+
+ @Override
+ public Map<String, Double> getBackPressurePerHost()
+ {
+ Map<String, Double> map = new HashMap<>(channelManagers.size());
+ for (Map.Entry<InetAddressAndPort, OutboundConnections> entry : channelManagers.entrySet())
+ map.put(entry.getKey().toString(false), entry.getValue().getBackPressureState().getBackPressureRateLimit());
+
+ return map;
+ }
+
+ @Override
+ public Map<String, Double> getBackPressurePerHostWithPort()
+ {
+ Map<String, Double> map = new HashMap<>(channelManagers.size());
+ for (Map.Entry<InetAddressAndPort, OutboundConnections> entry : channelManagers.entrySet())
+ map.put(entry.getKey().toString(false), entry.getValue().getBackPressureState().getBackPressureRateLimit());
+
+ return map;
+ }
+
+ @Override
+ public void setBackPressureEnabled(boolean enabled)
+ {
+ DatabaseDescriptor.setBackPressureEnabled(enabled);
+ }
+
+ @Override
+ public boolean isBackPressureEnabled()
+ {
+ return DatabaseDescriptor.backPressureEnabled();
+ }
+
+ @Override
+ public void reloadSslCertificates() throws IOException
+ {
+ final EncryptionOptions.ServerEncryptionOptions serverOpts = DatabaseDescriptor.getInternodeMessagingEncyptionOptions();
+ final EncryptionOptions clientOpts = DatabaseDescriptor.getNativeProtocolEncryptionOptions();
+ SSLFactory.validateSslCerts(serverOpts, clientOpts);
+ SSLFactory.checkCertFilesForHotReloading(serverOpts, clientOpts);
+ }
+
+ @Override
+ public int getVersion(String address) throws UnknownHostException
+ {
+ return versions.get(address);
+ }
+}
diff --git a/src/java/org/apache/cassandra/net/NoPayload.java b/src/java/org/apache/cassandra/net/NoPayload.java
new file mode 100644
index 0000000..3b2b177
--- /dev/null
+++ b/src/java/org/apache/cassandra/net/NoPayload.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.net;
+
+import org.apache.cassandra.io.IVersionedSerializer;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+
+/**
+ * Empty message payload - primarily used for responses.
+ *
+ * Prefer this singleton to writing one-off specialised classes.
+ */
+public class NoPayload
+{
+ public static final NoPayload noPayload = new NoPayload();
+
+ private NoPayload() {}
+
+ public static final IVersionedSerializer<NoPayload> serializer = new IVersionedSerializer<NoPayload>()
+ {
+ public void serialize(NoPayload noPayload, DataOutputPlus out, int version)
+ {
+ if (noPayload != NoPayload.noPayload)
+ throw new IllegalArgumentException();
+ }
+
+ public NoPayload deserialize(DataInputPlus in, int version)
+ {
+ return noPayload;
+ }
+
+ public long serializedSize(NoPayload noPayload, int version)
+ {
+ return 0;
+ }
+ };
+}
diff --git a/src/java/org/apache/cassandra/net/NoSizeEstimator.java b/src/java/org/apache/cassandra/net/NoSizeEstimator.java
new file mode 100644
index 0000000..848d4f5
--- /dev/null
+++ b/src/java/org/apache/cassandra/net/NoSizeEstimator.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.net;
+
+import io.netty.channel.MessageSizeEstimator;
+
+/**
+ * We want to manage the bytes we have in-flight, so this class asks Netty not to by returning zero for every object.
+ */
+class NoSizeEstimator implements MessageSizeEstimator, MessageSizeEstimator.Handle
+{
+ public static final NoSizeEstimator instance = new NoSizeEstimator();
+ private NoSizeEstimator() {}
+ public Handle newHandle() { return this; }
+ public int size(Object o) { return 0; }
+}
diff --git a/src/java/org/apache/cassandra/net/OutboundConnection.java b/src/java/org/apache/cassandra/net/OutboundConnection.java
new file mode 100644
index 0000000..635f221
--- /dev/null
+++ b/src/java/org/apache/cassandra/net/OutboundConnection.java
@@ -0,0 +1,1753 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.net;
+
+import java.io.IOException;
+import java.net.ConnectException;
+import java.net.InetSocketAddress;
+import java.nio.channels.ClosedChannelException;
+import java.util.Objects;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLongFieldUpdater;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.concurrent.atomic.AtomicReferenceFieldUpdater;
+import java.util.stream.Stream;
+
+import javax.annotation.Nullable;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.util.concurrent.Uninterruptibles;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import io.netty.channel.Channel;
+import io.netty.channel.ChannelFuture;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.channel.ChannelInboundHandlerAdapter;
+import io.netty.channel.EventLoop;
+import io.netty.channel.unix.Errors;
+import io.netty.util.concurrent.Future;
+import io.netty.util.concurrent.Promise;
+import io.netty.util.concurrent.PromiseNotifier;
+import io.netty.util.concurrent.SucceededFuture;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.io.util.DataOutputBufferFixed;
+import org.apache.cassandra.net.OutboundConnectionInitiator.Result.MessagingSuccess;
+import org.apache.cassandra.tracing.Tracing;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.JVMStabilityInspector;
+import org.apache.cassandra.utils.NoSpamLogger;
+
+import static java.lang.Math.max;
+import static java.lang.Math.min;
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+import static org.apache.cassandra.net.MessagingService.current_version;
+import static org.apache.cassandra.net.OutboundConnectionInitiator.*;
+import static org.apache.cassandra.net.OutboundConnections.LARGE_MESSAGE_THRESHOLD;
+import static org.apache.cassandra.net.ResourceLimits.*;
+import static org.apache.cassandra.net.ResourceLimits.Outcome.*;
+import static org.apache.cassandra.net.SocketFactory.*;
+import static org.apache.cassandra.utils.MonotonicClock.approxTime;
+import static org.apache.cassandra.utils.Throwables.isCausedBy;
+
+/**
+ * Represents a connection type to a peer, and handles the state transistions on the connection and the netty {@link Channel}.
+ * The underlying socket is not opened until explicitly requested (by sending a message).
+ *
+ * TODO: complete this description
+ *
+ * Aside from a few administrative methods, the main entry point to sending a message is {@link #enqueue(Message)}.
+ * Any thread may send a message (enqueueing it to {@link #queue}), but only one thread may consume messages from this
+ * queue. There is a single delivery thread - either the event loop, or a companion thread - that has logical ownership
+ * of the queue, but other threads may temporarily take ownership in order to perform book keeping, pruning, etc.,
+ * to ensure system stability.
+ *
+ * {@link Delivery#run()} is the main entry point for consuming messages from the queue, and executes either on the event
+ * loop or on a non-dedicated companion thread. This processing is activated via {@link Delivery#execute()}.
+ *
+ * Almost all internal state maintenance on this class occurs on the eventLoop, a single threaded executor which is
+ * assigned in the constructor. Further details are outlined below in the class. Some behaviours require coordination
+ * between the eventLoop and the companion thread (if any). Some minimal set of behaviours are permitted to occur on
+ * producers to ensure the connection remains healthy and does not overcommit resources.
+ *
+ * All methods are safe to invoke from any thread unless otherwise stated.
+ */
+@SuppressWarnings({ "WeakerAccess", "FieldMayBeFinal", "NonAtomicOperationOnVolatileField", "SameParameterValue" })
+public class OutboundConnection
+{
+ static final Logger logger = LoggerFactory.getLogger(OutboundConnection.class);
+ private static final NoSpamLogger noSpamLogger = NoSpamLogger.getLogger(logger, 30L, TimeUnit.SECONDS);
+
+ private static final AtomicLongFieldUpdater<OutboundConnection> submittedUpdater = AtomicLongFieldUpdater.newUpdater(OutboundConnection.class, "submittedCount");
+ private static final AtomicLongFieldUpdater<OutboundConnection> pendingCountAndBytesUpdater = AtomicLongFieldUpdater.newUpdater(OutboundConnection.class, "pendingCountAndBytes");
+ private static final AtomicLongFieldUpdater<OutboundConnection> overloadedCountUpdater = AtomicLongFieldUpdater.newUpdater(OutboundConnection.class, "overloadedCount");
+ private static final AtomicLongFieldUpdater<OutboundConnection> overloadedBytesUpdater = AtomicLongFieldUpdater.newUpdater(OutboundConnection.class, "overloadedBytes");
+ private static final AtomicReferenceFieldUpdater<OutboundConnection, Future> closingUpdater = AtomicReferenceFieldUpdater.newUpdater(OutboundConnection.class, Future.class, "closing");
+ private static final AtomicReferenceFieldUpdater<OutboundConnection, Future> scheduledCloseUpdater = AtomicReferenceFieldUpdater.newUpdater(OutboundConnection.class, Future.class, "scheduledClose");
+
+ private final EventLoop eventLoop;
+ private final Delivery delivery;
+
+ private final OutboundMessageCallbacks callbacks;
+ private final OutboundDebugCallbacks debug;
+ private final OutboundMessageQueue queue;
+ /** the number of bytes we permit to queue to the network without acquiring any shared resource permits */
+ private final long pendingCapacityInBytes;
+ /** the number of messages and bytes queued for flush to the network,
+ * including those that are being flushed but have not been completed,
+ * packed into a long (top 20 bits for count, bottom 42 for bytes)*/
+ private volatile long pendingCountAndBytes = 0;
+ /** global shared limits that we use only if our local limits are exhausted;
+ * we allocate from here whenever queueSize > queueCapacity */
+ private final EndpointAndGlobal reserveCapacityInBytes;
+
+ private volatile long submittedCount = 0; // updated with cas
+ private volatile long overloadedCount = 0; // updated with cas
+ private volatile long overloadedBytes = 0; // updated with cas
+ private long expiredCount = 0; // updated with queue lock held
+ private long expiredBytes = 0; // updated with queue lock held
+ private long errorCount = 0; // updated only by delivery thread
+ private long errorBytes = 0; // updated by delivery thread only
+ private long sentCount; // updated by delivery thread only
+ private long sentBytes; // updated by delivery thread only
+ private long successfulConnections; // updated by event loop only
+ private long connectionAttempts; // updated by event loop only
+
+ private static final int pendingByteBits = 42;
+ private static boolean isMaxPendingCount(long pendingCountAndBytes)
+ {
+ return (pendingCountAndBytes & (-1L << pendingByteBits)) == (-1L << pendingByteBits);
+ }
+
+ private static int pendingCount(long pendingCountAndBytes)
+ {
+ return (int) (pendingCountAndBytes >>> pendingByteBits);
+ }
+
+ private static long pendingBytes(long pendingCountAndBytes)
+ {
+ return pendingCountAndBytes & (-1L >>> (64 - pendingByteBits));
+ }
+
+ private static long pendingCountAndBytes(long pendingCount, long pendingBytes)
+ {
+ return (pendingCount << pendingByteBits) | pendingBytes;
+ }
+
+ private final ConnectionType type;
+
+ /**
+ * Contains the base settings for this connection, _including_ any defaults filled in.
+ *
+ */
+ private OutboundConnectionSettings template;
+
+ private static class State
+ {
+ static final State CLOSED = new State(Kind.CLOSED);
+
+ enum Kind { ESTABLISHED, CONNECTING, DORMANT, CLOSED }
+
+ final Kind kind;
+
+ State(Kind kind)
+ {
+ this.kind = kind;
+ }
+
+ boolean isEstablished() { return kind == Kind.ESTABLISHED; }
+ boolean isConnecting() { return kind == Kind.CONNECTING; }
+ boolean isDisconnected() { return kind == Kind.CONNECTING || kind == Kind.DORMANT; }
+ boolean isClosed() { return kind == Kind.CLOSED; }
+
+ Established established() { return (Established) this; }
+ Connecting connecting() { return (Connecting) this; }
+ Disconnected disconnected() { return (Disconnected) this; }
+ }
+
+ /**
+ * We have successfully negotiated a channel, and believe it to still be valid.
+ *
+ * Before using this, we should check isConnected() to check the Channel hasn't
+ * become invalid.
+ */
+ private static class Established extends State
+ {
+ final int messagingVersion;
+ final Channel channel;
+ final FrameEncoder.PayloadAllocator payloadAllocator;
+ final OutboundConnectionSettings settings;
+
+ Established(int messagingVersion, Channel channel, FrameEncoder.PayloadAllocator payloadAllocator, OutboundConnectionSettings settings)
+ {
+ super(Kind.ESTABLISHED);
+ this.messagingVersion = messagingVersion;
+ this.channel = channel;
+ this.payloadAllocator = payloadAllocator;
+ this.settings = settings;
+ }
+
+ boolean isConnected() { return channel.isOpen(); }
+ }
+
+ private static class Disconnected extends State
+ {
+ /** Periodic message expiry scheduled while we are disconnected; this will be cancelled and cleared each time we connect */
+ final Future<?> maintenance;
+ Disconnected(Kind kind, Future<?> maintenance)
+ {
+ super(kind);
+ this.maintenance = maintenance;
+ }
+
+ public static Disconnected dormant(Future<?> maintenance)
+ {
+ return new Disconnected(Kind.DORMANT, maintenance);
+ }
+ }
+
+ private static class Connecting extends Disconnected
+ {
+ /**
+ * Currently (or scheduled to) (re)connect; this may be cancelled (if closing) or waited on (for delivery)
+ *
+ * - The work managed by this future is partially performed asynchronously, not necessarily on the eventLoop.
+ * - It is only completed on the eventLoop
+ * - It may not be executing, but might be scheduled to be submitted if {@link #scheduled} is not null
+ */
+ final Future<Result<MessagingSuccess>> attempt;
+
+ /**
+ * If we are retrying to connect with some delay, this represents the scheduled inititation of another attempt
+ */
+ @Nullable
+ final Future<?> scheduled;
+
+ /**
+ * true iff we are retrying to connect after some failure (immediately or following a delay)
+ */
+ final boolean isFailingToConnect;
+
+ Connecting(Disconnected previous, Future<Result<MessagingSuccess>> attempt)
+ {
+ this(previous, attempt, null);
+ }
+
+ Connecting(Disconnected previous, Future<Result<MessagingSuccess>> attempt, Future<?> scheduled)
+ {
+ super(Kind.CONNECTING, previous.maintenance);
+ this.attempt = attempt;
+ this.scheduled = scheduled;
+ this.isFailingToConnect = scheduled != null || (previous.isConnecting() && previous.connecting().isFailingToConnect);
+ }
+
+ /**
+ * Cancel the connection attempt
+ *
+ * No cleanup is needed here, as {@link #attempt} is only completed on the eventLoop,
+ * so we have either already invoked the callbacks and are no longer in {@link #state},
+ * or the {@link OutboundConnectionInitiator} will handle our successful cancellation
+ * when it comes to complete, by closing the channel (if we could not cancel it before then)
+ */
+ void cancel()
+ {
+ if (scheduled != null)
+ scheduled.cancel(true);
+
+ // we guarantee that attempt is only ever completed by the eventLoop
+ boolean cancelled = attempt.cancel(true);
+ assert cancelled;
+ }
+ }
+
+ private volatile State state;
+
+ /** The connection is being permanently closed */
+ private volatile Future<Void> closing;
+ /** The connection is being permanently closed in the near future */
+ private volatile Future<Void> scheduledClose;
+
+ OutboundConnection(ConnectionType type, OutboundConnectionSettings settings, EndpointAndGlobal reserveCapacityInBytes)
+ {
+ this.template = settings.withDefaults(ConnectionCategory.MESSAGING);
+ this.type = type;
+ this.eventLoop = template.socketFactory.defaultGroup().next();
+ this.pendingCapacityInBytes = template.applicationSendQueueCapacityInBytes;
+ this.reserveCapacityInBytes = reserveCapacityInBytes;
+ this.callbacks = template.callbacks;
+ this.debug = template.debug;
+ this.queue = new OutboundMessageQueue(approxTime, this::onExpired);
+ this.delivery = type == ConnectionType.LARGE_MESSAGES
+ ? new LargeMessageDelivery(template.socketFactory.synchronousWorkExecutor)
+ : new EventLoopDelivery();
+ setDisconnected();
+ }
+
+ /**
+ * This is the main entry point for enqueuing a message to be sent to the remote peer.
+ */
+ public void enqueue(Message message) throws ClosedChannelException
+ {
+ if (isClosing())
+ throw new ClosedChannelException();
+
+ final int canonicalSize = canonicalSize(message);
+ if (canonicalSize > DatabaseDescriptor.getInternodeMaxMessageSizeInBytes())
+ throw new Message.OversizedMessageException(canonicalSize);
+
+ submittedUpdater.incrementAndGet(this);
+ switch (acquireCapacity(canonicalSize))
+ {
+ case INSUFFICIENT_ENDPOINT:
+ // if we're overloaded to one endpoint, we may be accumulating expirable messages, so
+ // attempt an expiry to see if this makes room for our newer message.
+ // this is an optimisation only; messages will be expired on ~100ms cycle, and by Delivery when it runs
+ if (queue.maybePruneExpired() && SUCCESS == acquireCapacity(canonicalSize))
+ break;
+ case INSUFFICIENT_GLOBAL:
+ onOverloaded(message);
+ return;
+ }
+
+ queue.add(message);
+ delivery.execute();
+
+ // we might race with the channel closing; if this happens, to ensure this message eventually arrives
+ // we need to remove ourselves from the queue and throw a ClosedChannelException, so that another channel
+ // can be opened in our place to try and send on.
+ if (isClosing() && queue.remove(message))
+ {
+ releaseCapacity(1, canonicalSize);
+ throw new ClosedChannelException();
+ }
+ }
+
+ /**
+ * Try to acquire the necessary resource permits for a number of pending bytes for this connection.
+ *
+ * Since the owner limit is shared amongst multiple connections, our semantics cannot be super trivial.
+ * Were they per-connection, we could simply perform an atomic increment of the queue size, then
+ * allocate any excess we need in the reserve, and on release free everything we see from both.
+ * Since we are coordinating two independent atomic variables we have to track every byte we allocate in reserve
+ * and ensure it is matched by a corresponding released byte. We also need to be sure we do not permit another
+ * releasing thread to release reserve bytes we have not yet - and may never - actually reserve.
+ *
+ * As such, we have to first check if we would need reserve bytes, then allocate them *before* we increment our
+ * queue size. We only increment the queue size if the reserve bytes are definitely not needed, or we could first
+ * obtain them. If in the process of obtaining any reserve bytes the queue size changes, we have some bytes that are
+ * reserved for us, but may be a different number to that we need. So we must continue to track these.
+ *
+ * In the happy path, this is still efficient as we simply CAS
+ */
+ private Outcome acquireCapacity(long bytes)
+ {
+ return acquireCapacity(1, bytes);
+ }
+
+ private Outcome acquireCapacity(long count, long bytes)
+ {
+ long increment = pendingCountAndBytes(count, bytes);
+ long unusedClaimedReserve = 0;
+ Outcome outcome = null;
+ loop: while (true)
+ {
+ long current = pendingCountAndBytes;
+ if (isMaxPendingCount(current))
+ {
+ outcome = INSUFFICIENT_ENDPOINT;
+ break;
+ }
+
+ long next = current + increment;
+ if (pendingBytes(next) <= pendingCapacityInBytes)
+ {
+ if (pendingCountAndBytesUpdater.compareAndSet(this, current, next))
+ {
+ outcome = SUCCESS;
+ break;
+ }
+ continue;
+ }
+
+ State state = this.state;
+ if (state.isConnecting() && state.connecting().isFailingToConnect)
+ {
+ outcome = INSUFFICIENT_ENDPOINT;
+ break;
+ }
+
+ long requiredReserve = min(bytes, pendingBytes(next) - pendingCapacityInBytes);
+ if (unusedClaimedReserve < requiredReserve)
+ {
+ long extraGlobalReserve = requiredReserve - unusedClaimedReserve;
+ switch (outcome = reserveCapacityInBytes.tryAllocate(extraGlobalReserve))
+ {
+ case INSUFFICIENT_ENDPOINT:
+ case INSUFFICIENT_GLOBAL:
+ break loop;
+ case SUCCESS:
+ unusedClaimedReserve += extraGlobalReserve;
+ }
+ }
+
+ if (pendingCountAndBytesUpdater.compareAndSet(this, current, next))
+ {
+ unusedClaimedReserve -= requiredReserve;
+ break;
+ }
+ }
+
+ if (unusedClaimedReserve > 0)
+ reserveCapacityInBytes.release(unusedClaimedReserve);
+
+ return outcome;
+ }
+
+ /**
+ * Mark a number of pending bytes as flushed to the network, releasing their capacity for new outbound messages.
+ */
+ private void releaseCapacity(long count, long bytes)
+ {
+ long decrement = pendingCountAndBytes(count, bytes);
+ long prev = pendingCountAndBytesUpdater.getAndAdd(this, -decrement);
+ if (pendingBytes(prev) > pendingCapacityInBytes)
+ {
+ long excess = min(pendingBytes(prev) - pendingCapacityInBytes, bytes);
+ reserveCapacityInBytes.release(excess);
+ }
+ }
+
+ private void onOverloaded(Message<?> message)
+ {
+ overloadedCountUpdater.incrementAndGet(this);
+ overloadedBytesUpdater.addAndGet(this, canonicalSize(message));
+ noSpamLogger.warn("{} overloaded; dropping {} message (queue: {} local, {} endpoint, {} global)",
+ id(),
+ FBUtilities.prettyPrintMemory(canonicalSize(message)),
+ FBUtilities.prettyPrintMemory(pendingBytes()),
+ FBUtilities.prettyPrintMemory(reserveCapacityInBytes.endpoint.using()),
+ FBUtilities.prettyPrintMemory(reserveCapacityInBytes.global.using()));
+ callbacks.onOverloaded(message, template.to);
+ }
+
+ /**
+ * Take any necessary cleanup action after a message has been selected to be discarded from the queue.
+ *
+ * Only to be invoked while holding OutboundMessageQueue.WithLock
+ */
+ private boolean onExpired(Message<?> message)
+ {
+ releaseCapacity(1, canonicalSize(message));
+ expiredCount += 1;
+ expiredBytes += canonicalSize(message);
+ noSpamLogger.warn("{} dropping message of type {} whose timeout expired before reaching the network", id(), message.verb());
+ callbacks.onExpired(message, template.to);
+ return true;
+ }
+
+ /**
+ * Take any necessary cleanup action after a message has been selected to be discarded from the queue.
+ *
+ * Only to be invoked by the delivery thread
+ */
+ private void onFailedSerialize(Message<?> message, int messagingVersion, int bytesWrittenToNetwork, Throwable t)
+ {
+ JVMStabilityInspector.inspectThrowable(t, false);
+ releaseCapacity(1, canonicalSize(message));
+ errorCount += 1;
+ errorBytes += message.serializedSize(messagingVersion);
+ logger.warn("{} dropping message of type {} due to error", id(), message.verb(), t);
+ callbacks.onFailedSerialize(message, template.to, messagingVersion, bytesWrittenToNetwork, t);
+ }
+
+ /**
+ * Take any necessary cleanup action after a message has been selected to be discarded from the queue on close.
+ * Note that this is only for messages that were queued prior to closing without graceful flush, OR
+ * for those that are unceremoniously dropped when we decide close has been trying to complete for too long.
+ */
+ private void onClosed(Message<?> message)
+ {
+ releaseCapacity(1, canonicalSize(message));
+ callbacks.onDiscardOnClose(message, template.to);
+ }
+
+ /**
+ * Delivery bundles the following:
+ *
+ * - the work that is necessary to actually deliver messages safely, and handle any exceptional states
+ * - the ability to schedule delivery for some time in the future
+ * - the ability to schedule some non-delivery work to happen some time in the future, that is guaranteed
+ * NOT to coincide with delivery for its duration, including any data that is being flushed (e.g. for closing channels)
+ * - this feature is *not* efficient, and should only be used for infrequent operations
+ */
+ private abstract class Delivery extends AtomicInteger implements Runnable
+ {
+ final ExecutorService executor;
+
+ // the AtomicInteger we extend always contains some combination of these bit flags, representing our current run state
+
+ /** Not running, and will not be scheduled again until transitioned to a new state */
+ private static final int STOPPED = 0;
+ /** Currently executing (may only be scheduled to execute, or may be about to terminate);
+ * will stop at end of this run, without rescheduling */
+ private static final int EXECUTING = 1;
+ /** Another execution has been requested; a new execution will begin some time after this state is taken */
+ private static final int EXECUTE_AGAIN = 2;
+ /** We are currently executing and will submit another execution before we terminate */
+ private static final int EXECUTING_AGAIN = EXECUTING | EXECUTE_AGAIN;
+ /** Will begin a new execution some time after this state is taken, but only once some condition is met.
+ * This state will initially be taken in tandem with EXECUTING, but if delivery completes without clearing
+ * the state, the condition will be held on its own until {@link #executeAgain} is invoked */
+ private static final int WAITING_TO_EXECUTE = 4;
+
+ /**
+ * Force all task execution to stop, once any currently in progress work is completed
+ */
+ private volatile boolean terminated;
+
+ /**
+ * Is there asynchronous delivery work in progress.
+ *
+ * This temporarily prevents any {@link #stopAndRun} work from being performed.
+ * Once both inProgress and stopAndRun are set we perform no more delivery work until one is unset,
+ * to ensure we eventually run stopAndRun.
+ *
+ * This should be updated and read only on the Delivery thread.
+ */
+ private boolean inProgress = false;
+
+ /**
+ * Request a task's execution while there is no delivery work in progress.
+ *
+ * This is to permit cleanly tearing down a connection without interrupting any messages that might be in flight.
+ * If stopAndRun is set, we should not enter doRun() until a corresponding setInProgress(false) occurs.
+ */
+ final AtomicReference<Runnable> stopAndRun = new AtomicReference<>();
+
+ Delivery(ExecutorService executor)
+ {
+ this.executor = executor;
+ }
+
+ /**
+ * Ensure that any messages or stopAndRun that were queued prior to this invocation will be seen by at least
+ * one future invocation of the delivery task, unless delivery has already been terminated.
+ */
+ public void execute()
+ {
+ if (get() < EXECUTE_AGAIN && STOPPED == getAndUpdate(i -> i == STOPPED ? EXECUTING: i | EXECUTE_AGAIN))
+ executor.execute(this);
+ }
+
+ private boolean isExecuting(int state)
+ {
+ return 0 != (state & EXECUTING);
+ }
+
+ /**
+ * This method is typically invoked after WAITING_TO_EXECUTE is set.
+ *
+ * However WAITING_TO_EXECUTE does not need to be set; all this method needs to ensure is that
+ * delivery unconditionally performs one new execution promptly.
+ */
+ void executeAgain()
+ {
+ // if we are already executing, set EXECUTING_AGAIN and leave scheduling to the currently running one.
+ // otherwise, set ourselves unconditionally to EXECUTING and schedule ourselves immediately
+ if (!isExecuting(getAndUpdate(i -> !isExecuting(i) ? EXECUTING : EXECUTING_AGAIN)))
+ executor.execute(this);
+ }
+
+ /**
+ * Invoke this when we cannot make further progress now, but we guarantee that we will execute later when we can.
+ * This simply communicates to {@link #run} that we should not schedule ourselves again, just unset the EXECUTING bit.
+ */
+ void promiseToExecuteLater()
+ {
+ set(EXECUTING | WAITING_TO_EXECUTE);
+ }
+
+ /**
+ * Called when exiting {@link #run} to schedule another run if necessary.
+ *
+ * If we are currently executing, we only reschedule if the present state is EXECUTING_AGAIN.
+ * If this is the case, we clear the EXECUTE_AGAIN bit (setting ourselves to EXECUTING), and reschedule.
+ * Otherwise, we clear the EXECUTING bit and terminate, which will set us to either STOPPED or WAITING_TO_EXECUTE
+ * (or possibly WAITING_TO_EXECUTE | EXECUTE_AGAIN, which is logically the same as WAITING_TO_EXECUTE)
+ */
+ private void maybeExecuteAgain()
+ {
+ if (EXECUTING_AGAIN == getAndUpdate(i -> i == EXECUTING_AGAIN ? EXECUTING : (i & ~EXECUTING)))
+ executor.execute(this);
+ }
+
+ /**
+ * No more tasks or delivery will be executed, once any in progress complete.
+ */
+ public void terminate()
+ {
+ terminated = true;
+ }
+
+ /**
+ * Only to be invoked by the Delivery task.
+ *
+ * If true, indicates that we have begun asynchronous delivery work, so that
+ * we cannot safely stopAndRun until it completes.
+ *
+ * Once it completes, we ensure any stopAndRun task has a chance to execute
+ * by ensuring delivery is scheduled.
+ *
+ * If stopAndRun is also set, we should not enter doRun() until a corresponding
+ * setInProgress(false) occurs.
+ */
+ void setInProgress(boolean inProgress)
+ {
+ boolean wasInProgress = this.inProgress;
+ this.inProgress = inProgress;
+ if (!inProgress && wasInProgress)
+ executeAgain();
+ }
+
+ /**
+ * Perform some delivery work.
+ *
+ * Must never be invoked directly, only via {@link #execute()}
+ */
+ public void run()
+ {
+ /* do/while handling setup for {@link #doRun()}, and repeat invocations thereof */
+ while (true)
+ {
+ if (terminated)
+ return;
+
+ if (null != stopAndRun.get())
+ {
+ // if we have an external request to perform, attempt it - if no async delivery is in progress
+
+ if (inProgress)
+ {
+ // if we are in progress, we cannot do anything;
+ // so, exit and rely on setInProgress(false) executing us
+ // (which must happen later, since it must happen on this thread)
+ promiseToExecuteLater();
+ break;
+ }
+
+ stopAndRun.getAndSet(null).run();
+ }
+
+ State state = OutboundConnection.this.state;
+ if (!state.isEstablished() || !state.established().isConnected())
+ {
+ // if we have messages yet to deliver, or a task to run, we need to reconnect and try again
+ // we try to reconnect before running another stopAndRun so that we do not infinite loop in close
+ if (hasPending() || null != stopAndRun.get())
+ {
+ promiseToExecuteLater();
+ requestConnect().addListener(f -> executeAgain());
+ }
+ break;
+ }
+
+ if (!doRun(state.established()))
+ break;
+ }
+
+ maybeExecuteAgain();
+ }
+
+ /**
+ * @return true if we should run again immediately;
+ * always false for eventLoop executor, as want to service other channels
+ */
+ abstract boolean doRun(Established established);
+
+ /**
+ * Schedule a task to run later on the delivery thread while delivery is not in progress,
+ * i.e. there are no bytes in flight to the network buffer.
+ *
+ * Does not guarantee to run promptly if there is no current connection to the remote host.
+ * May wait until a new connection is established, or a connection timeout elapses, before executing.
+ *
+ * Update the shared atomic property containing work we want to interrupt message processing to perform,
+ * the invoke schedule() to be certain it gets run.
+ */
+ void stopAndRun(Runnable run)
+ {
+ stopAndRun.accumulateAndGet(run, OutboundConnection::andThen);
+ execute();
+ }
+
+ /**
+ * Schedule a task to run on the eventLoop, guaranteeing that delivery will not occur while the task is performed.
+ */
+ abstract void stopAndRunOnEventLoop(Runnable run);
+
+ }
+
+ /**
+ * Delivery that runs entirely on the eventLoop
+ *
+ * Since this has single threaded access to most of its environment, it can be simple and efficient, however
+ * it must also have bounded run time, and limit its resource consumption to ensure other channels serviced by the
+ * eventLoop can also make progress.
+ *
+ * This operates on modest buffers, no larger than the {@link OutboundConnections#LARGE_MESSAGE_THRESHOLD} and
+ * filling at most one at a time before writing (potentially asynchronously) to the socket.
+ *
+ * We track the number of bytes we have in flight, ensuring no more than a user-defined maximum at any one time.
+ */
+ class EventLoopDelivery extends Delivery
+ {
+ private int flushingBytes;
+ private boolean isWritable = true;
+
+ EventLoopDelivery()
+ {
+ super(eventLoop);
+ }
+
+ /**
+ * {@link Delivery#doRun}
+ *
+ * Since we are on the eventLoop, in order to ensure other channels are serviced
+ * we never return true to request another run immediately.
+ *
+ * If there is more work to be done, we submit ourselves for execution once the eventLoop has time.
+ */
+ @SuppressWarnings("resource")
+ boolean doRun(Established established)
+ {
+ if (!isWritable)
+ return false;
+
+ // pendingBytes is updated before queue.size() (which triggers notEmpty, and begins delivery),
+ // so it is safe to use it here to exit delivery
+ // this number is inaccurate for old versions, but we don't mind terribly - we'll send at least one message,
+ // and get round to it eventually (though we could add a fudge factor for some room for older versions)
+ int maxSendBytes = (int) min(pendingBytes() - flushingBytes, LARGE_MESSAGE_THRESHOLD);
+ if (maxSendBytes == 0)
+ return false;
+
+ OutboundConnectionSettings settings = established.settings;
+ int messagingVersion = established.messagingVersion;
+
+ FrameEncoder.Payload sending = null;
+ int canonicalSize = 0; // number of bytes we must use for our resource accounting
+ int sendingBytes = 0;
+ int sendingCount = 0;
+ try (OutboundMessageQueue.WithLock withLock = queue.lockOrCallback(approxTime.now(), this::execute))
+ {
+ if (withLock == null)
+ return false; // we failed to acquire the queue lock, so return; we will be scheduled again when the lock is available
+
+ sending = established.payloadAllocator.allocate(true, maxSendBytes);
+ DataOutputBufferFixed out = new DataOutputBufferFixed(sending.buffer);
+
+ Message<?> next;
+ while ( null != (next = withLock.peek()) )
+ {
+ try
+ {
+ int messageSize = next.serializedSize(messagingVersion);
+
+ // actual message size for this version is larger than permitted maximum
+ if (messageSize > DatabaseDescriptor.getInternodeMaxMessageSizeInBytes())
+ throw new Message.OversizedMessageException(messageSize);
+
+ if (messageSize > sending.remaining())
+ {
+ // if we don't have enough room to serialize the next message, we have either
+ // 1) run out of room after writing some messages successfully; this might mean that we are
+ // overflowing our highWaterMark, or that we have just filled our buffer
+ // 2) we have a message that is too large for this connection; this can happen if a message's
+ // size was calculated for the wrong messaging version when enqueued.
+ // In this case we want to write it anyway, so simply allocate a large enough buffer.
+
+ if (sendingBytes > 0)
+ break;
+
+ sending.release();
+ sending = null; // set to null to prevent double-release if we fail to allocate our new buffer
+ sending = established.payloadAllocator.allocate(true, messageSize);
+ //noinspection IOResourceOpenedButNotSafelyClosed
+ out = new DataOutputBufferFixed(sending.buffer);
+ }
+
+ Tracing.instance.traceOutgoingMessage(next, messageSize, settings.connectTo);
+ Message.serializer.serialize(next, out, messagingVersion);
+
+ if (sending.length() != sendingBytes + messageSize)
+ throw new InvalidSerializedSizeException(next.verb(), messageSize, sending.length() - sendingBytes);
+
+ canonicalSize += canonicalSize(next);
+ sendingCount += 1;
+ sendingBytes += messageSize;
+ }
+ catch (Throwable t)
+ {
+ onFailedSerialize(next, messagingVersion, 0, t);
+
+ assert sending != null;
+ // reset the buffer to ignore the message we failed to serialize
+ sending.trim(sendingBytes);
+ }
+ withLock.removeHead(next);
+ }
+ if (0 == sendingBytes)
+ return false;
+
+ sending.finish();
+ debug.onSendSmallFrame(sendingCount, sendingBytes);
+ ChannelFuture flushResult = AsyncChannelPromise.writeAndFlush(established.channel, sending);
+ sending = null;
+
+ if (flushResult.isSuccess())
+ {
+ sentCount += sendingCount;
+ sentBytes += sendingBytes;
+ debug.onSentSmallFrame(sendingCount, sendingBytes);
+ }
+ else
+ {
+ flushingBytes += canonicalSize;
+ setInProgress(true);
+
+ boolean hasOverflowed = flushingBytes >= settings.flushHighWaterMark;
+ if (hasOverflowed)
+ {
+ isWritable = false;
+ promiseToExecuteLater();
+ }
+
+ int releaseBytesFinal = canonicalSize;
+ int sendingBytesFinal = sendingBytes;
+ int sendingCountFinal = sendingCount;
+ flushResult.addListener(future -> {
+
+ releaseCapacity(sendingCountFinal, releaseBytesFinal);
+ flushingBytes -= releaseBytesFinal;
+ if (flushingBytes == 0)
+ setInProgress(false);
+
+ if (!isWritable && flushingBytes <= settings.flushLowWaterMark)
+ {
+ isWritable = true;
+ executeAgain();
+ }
+
+ if (future.isSuccess())
+ {
+ sentCount += sendingCountFinal;
+ sentBytes += sendingBytesFinal;
+ debug.onSentSmallFrame(sendingCountFinal, sendingBytesFinal);
+ }
+ else
+ {
+ errorCount += sendingCountFinal;
+ errorBytes += sendingBytesFinal;
+ invalidateChannel(established, future.cause());
+ debug.onFailedSmallFrame(sendingCountFinal, sendingBytesFinal);
+ }
+ });
+ canonicalSize = 0;
+ }
+ }
+ catch (Throwable t)
+ {
+ errorCount += sendingCount;
+ errorBytes += sendingBytes;
+ invalidateChannel(established, t);
+ }
+ finally
+ {
+ if (canonicalSize > 0)
+ releaseCapacity(sendingCount, canonicalSize);
+
+ if (sending != null)
+ sending.release();
+
+ if (pendingBytes() > flushingBytes && isWritable)
+ execute();
+ }
+
+ return false;
+ }
+
+ void stopAndRunOnEventLoop(Runnable run)
+ {
+ stopAndRun(run);
+ }
+ }
+
+ /**
+ * Delivery that coordinates between the eventLoop and another (non-dedicated) thread
+ *
+ * This is to service messages that are too large to fully serialize on the eventLoop, as they could block
+ * prompt service of other requests. Since our serializers assume blocking IO, the easiest approach is to
+ * ensure a companion thread performs blocking IO that, under the hood, is serviced by async IO on the eventLoop.
+ *
+ * Most of the work here is handed off to {@link AsyncChannelOutputPlus}, with our main job being coordinating
+ * when and what we should run.
+ *
+ * To avoid allocating a huge number of threads across a cluster, we utilise the shared methods of {@link Delivery}
+ * to ensure that only one run() is actually scheduled to run at a time - this permits us to use any {@link ExecutorService}
+ * as a backing, with the number of threads defined only by the maximum concurrency needed to deliver all large messages.
+ * We use a shared caching {@link java.util.concurrent.ThreadPoolExecutor}, and rename the Threads that service
+ * our connection on entry and exit.
+ */
+ class LargeMessageDelivery extends Delivery
+ {
+ static final int DEFAULT_BUFFER_SIZE = 32 * 1024;
+
+ LargeMessageDelivery(ExecutorService executor)
+ {
+ super(executor);
+ }
+
+ /**
+ * A simple wrapper of {@link Delivery#run} to set the current Thread name for the duration of its execution.
+ */
+ public void run()
+ {
+ String threadName, priorThreadName = null;
+ try
+ {
+ priorThreadName = Thread.currentThread().getName();
+ threadName = "Messaging-OUT-" + template.from() + "->" + template.to + '-' + type;
+ Thread.currentThread().setName(threadName);
+
+ super.run();
+ }
+ finally
+ {
+ if (priorThreadName != null)
+ Thread.currentThread().setName(priorThreadName);
+ }
+ }
+
+ @SuppressWarnings({ "resource", "RedundantSuppression" }) // make eclipse warnings go away
+ boolean doRun(Established established)
+ {
+ Message<?> send = queue.tryPoll(approxTime.now(), this::execute);
+ if (send == null)
+ return false;
+
+ AsyncMessageOutputPlus out = null;
+ try
+ {
+ int messageSize = send.serializedSize(established.messagingVersion);
+ out = new AsyncMessageOutputPlus(established.channel, DEFAULT_BUFFER_SIZE, messageSize, established.payloadAllocator);
+ // actual message size for this version is larger than permitted maximum
+ if (messageSize > DatabaseDescriptor.getInternodeMaxMessageSizeInBytes())
+ throw new Message.OversizedMessageException(messageSize);
+
+ Tracing.instance.traceOutgoingMessage(send, messageSize, established.settings.connectTo);
+ Message.serializer.serialize(send, out, established.messagingVersion);
+
+ if (out.position() != messageSize)
+ throw new InvalidSerializedSizeException(send.verb(), messageSize, out.position());
+
+ out.close();
+ sentCount += 1;
+ sentBytes += messageSize;
+ releaseCapacity(1, canonicalSize(send));
+ return hasPending();
+ }
+ catch (Throwable t)
+ {
+ boolean tryAgain = true;
+
+ if (out != null)
+ {
+ out.discard();
+ if (out.flushed() > 0 ||
+ isCausedBy(t, cause -> isConnectionReset(cause)
+ || cause instanceof Errors.NativeIoException
+ || cause instanceof AsyncChannelOutputPlus.FlushException))
+ {
+ // close the channel, and wait for eventLoop to execute
+ disconnectNow(established).awaitUninterruptibly();
+ tryAgain = false;
+ try
+ {
+ // after closing, wait until we are signalled about the in flight writes;
+ // this ensures flushedToNetwork() is correct below
+ out.waitUntilFlushed(0, 0);
+ }
+ catch (Throwable ignore)
+ {
+ // irrelevant
+ }
+ }
+ }
+
+ onFailedSerialize(send, established.messagingVersion, out == null ? 0 : (int) out.flushedToNetwork(), t);
+ return tryAgain;
+ }
+ }
+
+ void stopAndRunOnEventLoop(Runnable run)
+ {
+ stopAndRun(() -> {
+ try
+ {
+ runOnEventLoop(run).await();
+ }
+ catch (InterruptedException e)
+ {
+ throw new RuntimeException(e);
+ }
+ });
+ }
+ }
+
+ /*
+ * Size used for capacity enforcement purposes. Using current messaging version no matter what the peer's version is.
+ */
+ private int canonicalSize(Message<?> message)
+ {
+ return message.serializedSize(current_version);
+ }
+
+ private void invalidateChannel(Established established, Throwable cause)
+ {
+ JVMStabilityInspector.inspectThrowable(cause, false);
+
+ if (state != established)
+ return; // do nothing; channel already invalidated
+
+ if (isCausedByConnectionReset(cause))
+ logger.info("{} channel closed by provider", id(), cause);
+ else
+ logger.error("{} channel in potentially inconsistent state after error; closing", id(), cause);
+
+ disconnectNow(established);
+ }
+
+ /**
+ * Attempt to open a new channel to the remote endpoint.
+ *
+ * Most of the actual work is performed by OutboundConnectionInitiator, this method just manages
+ * our book keeping on either success or failure.
+ *
+ * This method is only to be invoked by the eventLoop, and the inner class' methods should only be evaluated by the eventtLoop
+ */
+ Future<?> initiate()
+ {
+ class Initiate
+ {
+ /**
+ * If we fail to connect, we want to try and connect again before any messages timeout.
+ * However, we update this each time to ensure we do not retry unreasonably often, and settle on a periodicity
+ * that might lead to timeouts in some aggressive systems.
+ */
+ long retryRateMillis = DatabaseDescriptor.getMinRpcTimeout(MILLISECONDS) / 2;
+
+ // our connection settings, possibly updated on retry
+ int messagingVersion = template.endpointToVersion().get(template.to);
+ OutboundConnectionSettings settings;
+
+ /**
+ * If we failed for any reason, try again
+ */
+ void onFailure(Throwable cause)
+ {
+ if (cause instanceof ConnectException)
+ noSpamLogger.info("{} failed to connect", id(), cause);
+ else
+ noSpamLogger.error("{} failed to connect", id(), cause);
+
+ JVMStabilityInspector.inspectThrowable(cause, false);
+
+ if (hasPending())
+ {
+ Promise<Result<MessagingSuccess>> result = new AsyncPromise<>(eventLoop);
+ state = new Connecting(state.disconnected(), result, eventLoop.schedule(() -> attempt(result), max(100, retryRateMillis), MILLISECONDS));
+ retryRateMillis = min(1000, retryRateMillis * 2);
+ }
+ else
+ {
+ // this Initiate will be discarded
+ state = Disconnected.dormant(state.disconnected().maintenance);
+ }
+ }
+
+ void onCompletedHandshake(Result<MessagingSuccess> result)
+ {
+ switch (result.outcome)
+ {
+ case SUCCESS:
+ // it is expected that close, if successful, has already cancelled us; so we do not need to worry about leaking connections
+ assert !state.isClosed();
+
+ MessagingSuccess success = result.success();
+ debug.onConnect(success.messagingVersion, settings);
+ state.disconnected().maintenance.cancel(false);
+
+ FrameEncoder.PayloadAllocator payloadAllocator = success.allocator;
+ Channel channel = success.channel;
+ Established established = new Established(messagingVersion, channel, payloadAllocator, settings);
+ state = established;
+ channel.pipeline().addLast("handleExceptionalStates", new ChannelInboundHandlerAdapter() {
+ @Override
+ public void channelInactive(ChannelHandlerContext ctx)
+ {
+ disconnectNow(established);
+ ctx.fireChannelInactive();
+ }
+
+ @Override
+ public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause)
+ {
+ try
+ {
+ invalidateChannel(established, cause);
+ }
+ catch (Throwable t)
+ {
+ logger.error("Unexpected exception in {}.exceptionCaught", this.getClass().getSimpleName(), t);
+ }
+ }
+ });
+ ++successfulConnections;
+
+ logger.info("{} successfully connected, version = {}, framing = {}, encryption = {}",
+ id(true),
+ success.messagingVersion,
+ settings.framing,
+ encryptionLogStatement(settings.encryption));
+ break;
+
+ case RETRY:
+ if (logger.isTraceEnabled())
+ logger.trace("{} incorrect legacy peer version predicted; reconnecting", id());
+
+ // the messaging version we connected with was incorrect; try again with the one supplied by the remote host
+ messagingVersion = result.retry().withMessagingVersion;
+ settings.endpointToVersion.set(settings.to, messagingVersion);
+
+ initiate();
+ break;
+
+ case INCOMPATIBLE:
+ // we cannot communicate with this peer given its messaging version; mark this as any other failure, and continue trying
+ Throwable t = new IOException(String.format("Incompatible peer: %s, messaging version: %s",
+ settings.to, result.incompatible().maxMessagingVersion));
+ t.fillInStackTrace();
+ onFailure(t);
+ break;
+
+ default:
+ throw new AssertionError();
+ }
+ }
+
+ /**
+ * Initiate all the actions required to establish a working, valid connection. This includes
+ * opening the socket, negotiating the internode messaging handshake, and setting up the working
+ * Netty {@link Channel}. However, this method will not block for all those actions: it will only
+ * kick off the connection attempt, setting the @{link #connecting} future to track its completion.
+ *
+ * Note: this should only be invoked on the event loop.
+ */
+ private void attempt(Promise<Result<MessagingSuccess>> result)
+ {
+ ++connectionAttempts;
+
+ /*
+ * Re-evaluate messagingVersion before re-attempting the connection in case
+ * endpointToVersion were updated. This happens if the outbound connection
+ * is made before the endpointToVersion table is initially constructed or out
+ * of date (e.g. if outbound connections are established for gossip
+ * as a result of an inbound connection) and can result in the wrong outbound
+ * port being selected if configured with enable_legacy_ssl_storage_port=true.
+ */
+ int knownMessagingVersion = messagingVersion();
+ if (knownMessagingVersion != messagingVersion)
+ {
+ logger.trace("Endpoint version changed from {} to {} since connection initialized, updating.",
+ messagingVersion, knownMessagingVersion);
+ messagingVersion = knownMessagingVersion;
+ }
+
+ settings = template;
+ if (messagingVersion > settings.acceptVersions.max)
+ messagingVersion = settings.acceptVersions.max;
+
+ // ensure we connect to the correct SSL port
+ settings = settings.withLegacyPortIfNecessary(messagingVersion);
+
+ initiateMessaging(eventLoop, type, settings, messagingVersion, result)
+ .addListener(future -> {
+ if (future.isCancelled())
+ return;
+ if (future.isSuccess()) //noinspection unchecked
+ onCompletedHandshake((Result<MessagingSuccess>) future.getNow());
+ else
+ onFailure(future.cause());
+ });
+ }
+
+ Future<Result<MessagingSuccess>> initiate()
+ {
+ Promise<Result<MessagingSuccess>> result = new AsyncPromise<>(eventLoop);
+ state = new Connecting(state.disconnected(), result);
+ attempt(result);
+ return result;
+ }
+ }
+
+ return new Initiate().initiate();
+ }
+
+ /**
+ * Returns a future that completes when we are _maybe_ reconnected.
+ *
+ * The connection attempt is guaranteed to have completed (successfully or not) by the time any listeners are invoked,
+ * so if a reconnection attempt is needed, it is already scheduled.
+ */
+ private Future<?> requestConnect()
+ {
+ // we may race with updates to this variable, but this is fine, since we only guarantee that we see a value
+ // that did at some point represent an active connection attempt - if it is stale, it will have been completed
+ // and the caller can retry (or utilise the successfully established connection)
+ {
+ State state = this.state;
+ if (state.isConnecting())
+ return state.connecting().attempt;
+ }
+
+ Promise<Object> promise = AsyncPromise.uncancellable(eventLoop);
+ runOnEventLoop(() -> {
+ if (isClosed()) // never going to connect
+ {
+ promise.tryFailure(new ClosedChannelException());
+ }
+ else if (state.isEstablished() && state.established().isConnected()) // already connected
+ {
+ promise.trySuccess(null);
+ }
+ else
+ {
+ if (state.isEstablished())
+ setDisconnected();
+
+ if (!state.isConnecting())
+ {
+ assert eventLoop.inEventLoop();
+ assert !isConnected();
+ initiate().addListener(new PromiseNotifier<>(promise));
+ }
+ else
+ {
+ state.connecting().attempt.addListener(new PromiseNotifier<>(promise));
+ }
+ }
+ });
+ return promise;
+ }
+
+ /**
+ * Change the IP address on which we connect to the peer. We will attempt to connect to the new address if there
+ * was a previous connection, and new incoming messages as well as existing {@link #queue} messages will be sent there.
+ * Any outstanding messages in the existing channel will still be sent to the previous address (we won't/can't move them from
+ * one channel to another).
+ *
+ * Returns null if the connection is closed.
+ */
+ Future<Void> reconnectWith(OutboundConnectionSettings reconnectWith)
+ {
+ OutboundConnectionSettings newTemplate = reconnectWith.withDefaults(ConnectionCategory.MESSAGING);
+ if (newTemplate.socketFactory != template.socketFactory) throw new IllegalArgumentException();
+ if (newTemplate.callbacks != template.callbacks) throw new IllegalArgumentException();
+ if (!Objects.equals(newTemplate.applicationSendQueueCapacityInBytes, template.applicationSendQueueCapacityInBytes)) throw new IllegalArgumentException();
+ if (!Objects.equals(newTemplate.applicationSendQueueReserveEndpointCapacityInBytes, template.applicationSendQueueReserveEndpointCapacityInBytes)) throw new IllegalArgumentException();
+ if (newTemplate.applicationSendQueueReserveGlobalCapacityInBytes != template.applicationSendQueueReserveGlobalCapacityInBytes) throw new IllegalArgumentException();
+
+ logger.info("{} updating connection settings", id());
+
+ Promise<Void> done = AsyncPromise.uncancellable(eventLoop);
+ delivery.stopAndRunOnEventLoop(() -> {
+ template = newTemplate;
+ // delivery will immediately continue after this, triggering a reconnect if necessary;
+ // this might mean a slight delay for large message delivery, as the connect will be scheduled
+ // asynchronously, so we must wait for a second turn on the eventLoop
+ if (state.isEstablished())
+ {
+ disconnectNow(state.established());
+ }
+ else if (state.isConnecting())
+ {
+ // cancel any in-flight connection attempt and restart with new template
+ state.connecting().cancel();
+ initiate();
+ }
+ done.setSuccess(null);
+ });
+ return done;
+ }
+
+ /**
+ * Close any currently open connection, forcing a reconnect if there are messages outstanding
+ * (or leaving it closed for now otherwise)
+ */
+ public boolean interrupt()
+ {
+ State state = this.state;
+ if (!state.isEstablished())
+ return false;
+
+ disconnectGracefully(state.established());
+ return true;
+ }
+
+ /**
+ * Schedule a safe close of the provided channel, if it has not already been closed.
+ *
+ * This means ensuring that delivery has stopped so that we do not corrupt or interrupt any
+ * in progress transmissions.
+ *
+ * The actual closing of the channel is performed asynchronously, to simplify our internal state management
+ * and promptly get the connection going again; the close is considered to have succeeded as soon as we
+ * have set our internal state.
+ */
+ private void disconnectGracefully(Established closeIfIs)
+ {
+ // delivery will immediately continue after this, triggering a reconnect if necessary;
+ // this might mean a slight delay for large message delivery, as the connect will be scheduled
+ // asynchronously, so we must wait for a second turn on the eventLoop
+ delivery.stopAndRunOnEventLoop(() -> disconnectNow(closeIfIs));
+ }
+
+ /**
+ * The channel is already known to be invalid, so there's no point waiting for a clean break in delivery.
+ *
+ * Delivery will be executed again as soon as we have logically closed the channel; we do not wait
+ * for the channel to actually be closed.
+ *
+ * The Future returned _does_ wait for the channel to be completely closed, so that callers can wait to be sure
+ * all writes have been completed either successfully or not.
+ */
+ private Future<?> disconnectNow(Established closeIfIs)
+ {
+ return runOnEventLoop(() -> {
+ if (state == closeIfIs)
+ {
+ // no need to wait until the channel is closed to set ourselves as disconnected (and potentially open a new channel)
+ setDisconnected();
+ if (hasPending())
+ delivery.execute();
+ closeIfIs.channel.close()
+ .addListener(future -> {
+ if (!future.isSuccess())
+ logger.info("Problem closing channel {}", closeIfIs, future.cause());
+ });
+ }
+ });
+ }
+
+ /**
+ * Schedules regular cleaning of the connection's state while it is disconnected from its remote endpoint.
+ *
+ * To be run only by the eventLoop or in the constructor
+ */
+ private void setDisconnected()
+ {
+ assert state == null || state.isEstablished();
+ state = Disconnected.dormant(eventLoop.scheduleAtFixedRate(queue::maybePruneExpired, 100L, 100L, TimeUnit.MILLISECONDS));
+ }
+
+ /**
+ * Schedule this connection to be permanently closed; only one close may be scheduled,
+ * any future scheduled closes are referred to the original triggering one (which may have a different schedule)
+ */
+ Future<Void> scheduleClose(long time, TimeUnit unit, boolean flushQueue)
+ {
+ Promise<Void> scheduledClose = AsyncPromise.uncancellable(eventLoop);
+ if (!scheduledCloseUpdater.compareAndSet(this, null, scheduledClose))
+ return this.scheduledClose;
+
+ eventLoop.schedule(() -> close(flushQueue).addListener(new PromiseNotifier<>(scheduledClose)), time, unit);
+ return scheduledClose;
+ }
+
+ /**
+ * Permanently close this connection.
+ *
+ * Immediately prevent any new messages from being enqueued - these will throw ClosedChannelException.
+ * The close itself happens asynchronously on the eventLoop, so a Future is returned to help callers
+ * wait for its completion.
+ *
+ * The flushQueue parameter indicates if any outstanding messages should be delivered before closing the connection.
+ *
+ * - If false, any already flushed or in-progress messages are completed, and the remaining messages are cleared
+ * before the connection is promptly torn down.
+ *
+ * - If true, we attempt delivery of all queued messages. If necessary, we will continue to open new connections
+ * to the remote host until they have been delivered. Only if we continue to fail to open a connection for
+ * an extended period of time will we drop any outstanding messages and close the connection.
+ */
+ public Future<Void> close(boolean flushQueue)
+ {
+ // ensure only one close attempt can be in flight
+ Promise<Void> closing = AsyncPromise.uncancellable(eventLoop);
+ if (!closingUpdater.compareAndSet(this, null, closing))
+ return this.closing;
+
+ /*
+ * Now define a cleanup closure, that will be deferred until it is safe to do so.
+ * Once run it:
+ * - immediately _logically_ closes the channel by updating this object's fields, but defers actually closing
+ * - cancels any in-flight connection attempts
+ * - cancels any maintenance work that might be scheduled
+ * - clears any waiting messages on the queue
+ * - terminates the delivery thread
+ * - finally, schedules any open channel's closure, and propagates its completion to the close promise
+ */
+ Runnable eventLoopCleanup = () -> {
+ Runnable onceNotConnecting = () -> {
+ // start by setting ourselves to definitionally closed
+ State state = this.state;
+ this.state = State.CLOSED;
+
+ try
+ {
+ // note that we never clear the queue, to ensure that an enqueue has the opportunity to remove itself
+ // if it raced with close, to potentially requeue the message on a replacement connection
+
+ // we terminate delivery here, to ensure that any listener to {@link connecting} do not schedule more work
+ delivery.terminate();
+
+ // stop periodic cleanup
+ if (state.isDisconnected())
+ {
+ state.disconnected().maintenance.cancel(true);
+ closing.setSuccess(null);
+ }
+ else
+ {
+ assert state.isEstablished();
+ state.established().channel.close()
+ .addListener(new PromiseNotifier<>(closing));
+ }
+ }
+ catch (Throwable t)
+ {
+ // in case of unexpected exception, signal completion and try to close the channel
+ closing.trySuccess(null);
+ try
+ {
+ if (state.isEstablished())
+ state.established().channel.close();
+ }
+ catch (Throwable t2)
+ {
+ t.addSuppressed(t2);
+ logger.error("Failed to close connection cleanly:", t);
+ }
+ throw t;
+ }
+ };
+
+ if (state.isConnecting())
+ {
+ // stop any in-flight connection attempts; these should be running on the eventLoop, so we should
+ // be able to cleanly cancel them, but executing on a listener guarantees correct semantics either way
+ Connecting connecting = state.connecting();
+ connecting.cancel();
+ connecting.attempt.addListener(future -> onceNotConnecting.run());
+ }
+ else
+ {
+ onceNotConnecting.run();
+ }
+ };
+
+ /*
+ * If we want to shutdown gracefully, flushing any outstanding messages, we have to do it very carefully.
+ * Things to note:
+ *
+ * - It is possible flushing messages will require establishing a new connection
+ * (However, if a connection cannot be established, we do not want to keep trying)
+ * - We have to negotiate with a separate thread, so we must be sure it is not in-progress before we stop (like channel close)
+ * - Cleanup must still happen on the eventLoop
+ *
+ * To achieve all of this, we schedule a recurring operation on the delivery thread, executing while delivery
+ * is between messages, that checks if the queue is empty; if it is, it schedules cleanup on the eventLoop.
+ */
+
+ Runnable clearQueue = () ->
+ {
+ CountDownLatch done = new CountDownLatch(1);
+ queue.runEventually(withLock -> {
+ withLock.consume(this::onClosed);
+ done.countDown();
+ });
+ //noinspection UnstableApiUsage
+ Uninterruptibles.awaitUninterruptibly(done);
+ };
+
+ if (flushQueue)
+ {
+ // just keep scheduling on delivery executor a check to see if we're done; there should always be one
+ // delivery attempt between each invocation, unless there is a wider problem with delivery scheduling
+ class FinishDelivery implements Runnable
+ {
+ public void run()
+ {
+ if (!hasPending())
+ delivery.stopAndRunOnEventLoop(eventLoopCleanup);
+ else
+ delivery.stopAndRun(() -> {
+ if (state.isConnecting() && state.connecting().isFailingToConnect)
+ clearQueue.run();
+ run();
+ });
+ }
+ }
+
+ delivery.stopAndRun(new FinishDelivery());
+ }
+ else
+ {
+ delivery.stopAndRunOnEventLoop(() -> {
+ clearQueue.run();
+ eventLoopCleanup.run();
+ });
+ }
+
+ return closing;
+ }
+
+ /**
+ * Run the task immediately if we are the eventLoop, otherwise queue it for execution on the eventLoop.
+ */
+ private Future<?> runOnEventLoop(Runnable runnable)
+ {
+ if (!eventLoop.inEventLoop())
+ return eventLoop.submit(runnable);
+
+ runnable.run();
+ return new SucceededFuture<>(eventLoop, null);
+ }
+
+ public boolean isConnected()
+ {
+ State state = this.state;
+ return state.isEstablished() && state.established().isConnected();
+ }
+
+ boolean isClosing()
+ {
+ return closing != null;
+ }
+
+ boolean isClosed()
+ {
+ return state.isClosed();
+ }
+
+ private String id(boolean includeReal)
+ {
+ State state = this.state;
+ if (!includeReal || !state.isEstablished())
+ return id();
+ Established established = state.established();
+ Channel channel = established.channel;
+ OutboundConnectionSettings settings = established.settings;
+ return SocketFactory.channelId(settings.from, (InetSocketAddress) channel.localAddress(),
+ settings.to, (InetSocketAddress) channel.remoteAddress(),
+ type, channel.id().asShortText());
+ }
+
+ private String id()
+ {
+ State state = this.state;
+ Channel channel = null;
+ OutboundConnectionSettings settings = template;
+ if (state.isEstablished())
+ {
+ channel = state.established().channel;
+ settings = state.established().settings;
+ }
+ String channelId = channel != null ? channel.id().asShortText() : "[no-channel]";
+ return SocketFactory.channelId(settings.from(), settings.to, type, channelId);
+ }
+
+ @Override
+ public String toString()
+ {
+ return id();
+ }
+
+ public boolean hasPending()
+ {
+ return 0 != pendingCountAndBytes;
+ }
+
+ public int pendingCount()
+ {
+ return pendingCount(pendingCountAndBytes);
+ }
+
+ public long pendingBytes()
+ {
+ return pendingBytes(pendingCountAndBytes);
+ }
+
+ public long sentCount()
+ {
+ // not volatile, but shouldn't matter
+ return sentCount;
+ }
+
+ public long sentBytes()
+ {
+ // not volatile, but shouldn't matter
+ return sentBytes;
+ }
+
+ public long submittedCount()
+ {
+ // not volatile, but shouldn't matter
+ return submittedCount;
+ }
+
+ public long dropped()
+ {
+ return overloadedCount + expiredCount;
+ }
+
+ public long overloadedBytes()
+ {
+ return overloadedBytes;
+ }
+
+ public long overloadedCount()
+ {
+ return overloadedCount;
+ }
+
+ public long expiredCount()
+ {
+ return expiredCount;
+ }
+
+ public long expiredBytes()
+ {
+ return expiredBytes;
+ }
+
+ public long errorCount()
+ {
+ return errorCount;
+ }
+
+ public long errorBytes()
+ {
+ return errorBytes;
+ }
+
+ public long successfulConnections()
+ {
+ return successfulConnections;
+ }
+
+ public long connectionAttempts()
+ {
+ return connectionAttempts;
+ }
+
+ private static Runnable andThen(Runnable a, Runnable b)
+ {
+ if (a == null || b == null)
+ return a == null ? b : a;
+ return () -> { a.run(); b.run(); };
+ }
+
+ @VisibleForTesting
+ public ConnectionType type()
+ {
+ return type;
+ }
+
+ @VisibleForTesting
+ OutboundConnectionSettings settings()
+ {
+ State state = this.state;
+ return state.isEstablished() ? state.established().settings : template;
+ }
+
+ @VisibleForTesting
+ int messagingVersion()
+ {
+ State state = this.state;
+ return state.isEstablished() ? state.established().messagingVersion
+ : template.endpointToVersion().get(template.to);
+ }
+
+ @VisibleForTesting
+ void unsafeRunOnDelivery(Runnable run)
+ {
+ delivery.stopAndRun(run);
+ }
+
+ @VisibleForTesting
+ Channel unsafeGetChannel()
+ {
+ State state = this.state;
+ return state.isEstablished() ? state.established().channel : null;
+ }
+
+ @VisibleForTesting
+ boolean unsafeAcquireCapacity(long amount)
+ {
+ return SUCCESS == acquireCapacity(amount);
+ }
+
+ @VisibleForTesting
+ boolean unsafeAcquireCapacity(long count, long amount)
+ {
+ return SUCCESS == acquireCapacity(count, amount);
+ }
+
+ @VisibleForTesting
+ void unsafeReleaseCapacity(long amount)
+ {
+ releaseCapacity(1, amount);
+ }
+
+ @VisibleForTesting
+ void unsafeReleaseCapacity(long count, long amount)
+ {
+ releaseCapacity(count, amount);
+ }
+
+ @VisibleForTesting
+ Limit unsafeGetEndpointReserveLimits()
+ {
+ return reserveCapacityInBytes.endpoint;
+ }
+}
diff --git a/src/java/org/apache/cassandra/net/OutboundConnectionInitiator.java b/src/java/org/apache/cassandra/net/OutboundConnectionInitiator.java
new file mode 100644
index 0000000..5f3eced
--- /dev/null
+++ b/src/java/org/apache/cassandra/net/OutboundConnectionInitiator.java
@@ -0,0 +1,471 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.net;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.nio.channels.ClosedChannelException;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import io.netty.bootstrap.Bootstrap;
+import io.netty.buffer.ByteBuf;
+import io.netty.channel.Channel;
+import io.netty.channel.ChannelHandler;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.channel.ChannelInboundHandler;
+import io.netty.channel.ChannelInitializer;
+import io.netty.channel.ChannelOption;
+import io.netty.channel.ChannelPipeline;
+import io.netty.channel.EventLoop;
+import io.netty.channel.socket.SocketChannel;
+import io.netty.handler.codec.ByteToMessageDecoder;
+
+import io.netty.handler.logging.LogLevel;
+import io.netty.handler.logging.LoggingHandler;
+import io.netty.handler.ssl.SslContext;
+import io.netty.handler.ssl.SslHandler;
+import io.netty.util.concurrent.FailedFuture;
+import io.netty.util.concurrent.Future;
+import io.netty.util.concurrent.Promise;
+import io.netty.util.concurrent.ScheduledFuture;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.net.HandshakeProtocol.Initiate;
+import org.apache.cassandra.net.OutboundConnectionInitiator.Result.MessagingSuccess;
+import org.apache.cassandra.net.OutboundConnectionInitiator.Result.StreamingSuccess;
+import org.apache.cassandra.security.SSLFactory;
+import org.apache.cassandra.utils.JVMStabilityInspector;
+import org.apache.cassandra.utils.memory.BufferPool;
+
+import static java.util.concurrent.TimeUnit.*;
+import static org.apache.cassandra.net.MessagingService.VERSION_40;
+import static org.apache.cassandra.net.HandshakeProtocol.*;
+import static org.apache.cassandra.net.ConnectionType.STREAMING;
+import static org.apache.cassandra.net.OutboundConnectionInitiator.Result.incompatible;
+import static org.apache.cassandra.net.OutboundConnectionInitiator.Result.messagingSuccess;
+import static org.apache.cassandra.net.OutboundConnectionInitiator.Result.retry;
+import static org.apache.cassandra.net.OutboundConnectionInitiator.Result.streamingSuccess;
+import static org.apache.cassandra.net.SocketFactory.*;
+
+/**
+ * A {@link ChannelHandler} to execute the send-side of the internode handshake protocol.
+ * As soon as the handler is added to the channel via {@link ChannelInboundHandler#channelActive(ChannelHandlerContext)}
+ * (which is only invoked if the underlying TCP connection was properly established), the {@link Initiate}
+ * handshake is sent. See {@link HandshakeProtocol} for full details.
+ * <p>
+ * Upon completion of the handshake (on success or fail), the {@link #resultPromise} is completed.
+ * See {@link Result} for details about the different result states.
+ * <p>
+ * This class extends {@link ByteToMessageDecoder}, which is a {@link ChannelInboundHandler}, because this handler
+ * waits for the peer's handshake response (the {@link Accept} of the internode messaging handshake protocol).
+ */
+public class OutboundConnectionInitiator<SuccessType extends OutboundConnectionInitiator.Result.Success>
+{
+ private static final Logger logger = LoggerFactory.getLogger(OutboundConnectionInitiator.class);
+
+ private final ConnectionType type;
+ private final OutboundConnectionSettings settings;
+ private final int requestMessagingVersion; // for pre40 nodes
+ private final Promise<Result<SuccessType>> resultPromise;
+
+ private OutboundConnectionInitiator(ConnectionType type, OutboundConnectionSettings settings,
+ int requestMessagingVersion, Promise<Result<SuccessType>> resultPromise)
+ {
+ this.type = type;
+ this.requestMessagingVersion = requestMessagingVersion;
+ this.settings = settings;
+ this.resultPromise = resultPromise;
+ }
+
+ /**
+ * Initiate a connection with the requested messaging version.
+ * if the other node supports a newer version, or doesn't support this version, we will fail to connect
+ * and try again with the version they reported
+ *
+ * The returned {@code Future} is guaranteed to be completed on the supplied eventLoop.
+ */
+ public static Future<Result<StreamingSuccess>> initiateStreaming(EventLoop eventLoop, OutboundConnectionSettings settings, int requestMessagingVersion)
+ {
+ return new OutboundConnectionInitiator<StreamingSuccess>(STREAMING, settings, requestMessagingVersion, new AsyncPromise<>(eventLoop))
+ .initiate(eventLoop);
+ }
+
+ /**
+ * Initiate a connection with the requested messaging version.
+ * if the other node supports a newer version, or doesn't support this version, we will fail to connect
+ * and try again with the version they reported
+ *
+ * The returned {@code Future} is guaranteed to be completed on the supplied eventLoop.
+ */
+ static Future<Result<MessagingSuccess>> initiateMessaging(EventLoop eventLoop, ConnectionType type, OutboundConnectionSettings settings, int requestMessagingVersion, Promise<Result<MessagingSuccess>> result)
+ {
+ return new OutboundConnectionInitiator<>(type, settings, requestMessagingVersion, result)
+ .initiate(eventLoop);
+ }
+
+ private Future<Result<SuccessType>> initiate(EventLoop eventLoop)
+ {
+ if (logger.isTraceEnabled())
+ logger.trace("creating outbound bootstrap to {}, requestVersion: {}", settings, requestMessagingVersion);
+
+ if (!settings.authenticate())
+ {
+ // interrupt other connections, so they must attempt to re-authenticate
+ MessagingService.instance().interruptOutbound(settings.to);
+ return new FailedFuture<>(eventLoop, new IOException("authentication failed to " + settings.connectToId()));
+ }
+
+ // this is a bit ugly, but is the easiest way to ensure that if we timeout we can propagate a suitable error message
+ // and still guarantee that, if on timing out we raced with success, the successfully created channel is handled
+ AtomicBoolean timedout = new AtomicBoolean();
+ Future<Void> bootstrap = createBootstrap(eventLoop)
+ .connect()
+ .addListener(future -> {
+ eventLoop.execute(() -> {
+ if (!future.isSuccess())
+ {
+ if (future.isCancelled() && !timedout.get())
+ resultPromise.cancel(true);
+ else if (future.isCancelled())
+ resultPromise.tryFailure(new IOException("Timeout handshaking with " + settings.connectToId()));
+ else
+ resultPromise.tryFailure(future.cause());
+ }
+ });
+ });
+
+ ScheduledFuture<?> timeout = eventLoop.schedule(() -> {
+ timedout.set(true);
+ bootstrap.cancel(false);
+ }, TIMEOUT_MILLIS, MILLISECONDS);
+ bootstrap.addListener(future -> timeout.cancel(true));
+
+ // Note that the bootstrap future's listeners may be invoked outside of the eventLoop,
+ // as Epoll failures on connection and disconnect may be run on the GlobalEventExecutor
+ // Since this FutureResult's listeners are all given to our resultPromise, they are guaranteed to be invoked by the eventLoop.
+ return new FutureResult<>(resultPromise, bootstrap);
+ }
+
+ /**
+ * Create the {@link Bootstrap} for connecting to a remote peer. This method does <b>not</b> attempt to connect to the peer,
+ * and thus does not block.
+ */
+ private Bootstrap createBootstrap(EventLoop eventLoop)
+ {
+ Bootstrap bootstrap = settings.socketFactory
+ .newClientBootstrap(eventLoop, settings.tcpUserTimeoutInMS)
+ .option(ChannelOption.ALLOCATOR, GlobalBufferPoolAllocator.instance)
+ .option(ChannelOption.CONNECT_TIMEOUT_MILLIS, settings.tcpConnectTimeoutInMS)
+ .option(ChannelOption.SO_KEEPALIVE, true)
+ .option(ChannelOption.SO_REUSEADDR, true)
+ .option(ChannelOption.TCP_NODELAY, settings.tcpNoDelay)
+ .option(ChannelOption.MESSAGE_SIZE_ESTIMATOR, NoSizeEstimator.instance)
+ .handler(new Initializer());
+
+ if (settings.socketSendBufferSizeInBytes > 0)
+ bootstrap.option(ChannelOption.SO_SNDBUF, settings.socketSendBufferSizeInBytes);
+
+ InetAddressAndPort remoteAddress = settings.connectTo;
+ bootstrap.remoteAddress(new InetSocketAddress(remoteAddress.address, remoteAddress.port));
+ return bootstrap;
+ }
+
+ private class Initializer extends ChannelInitializer<SocketChannel>
+ {
+ public void initChannel(SocketChannel channel) throws Exception
+ {
+ ChannelPipeline pipeline = channel.pipeline();
+
+ // order of handlers: ssl -> logger -> handshakeHandler
+ if (settings.withEncryption())
+ {
+ // check if we should actually encrypt this connection
+ SslContext sslContext = SSLFactory.getOrCreateSslContext(settings.encryption, true, SSLFactory.SocketType.CLIENT);
+ // for some reason channel.remoteAddress() will return null
+ InetAddressAndPort address = settings.to;
+ InetSocketAddress peer = settings.encryption.require_endpoint_verification ? new InetSocketAddress(address.address, address.port) : null;
+ SslHandler sslHandler = newSslHandler(channel, sslContext, peer);
+ logger.trace("creating outbound netty SslContext: context={}, engine={}", sslContext.getClass().getName(), sslHandler.engine().getClass().getName());
+ pipeline.addFirst("ssl", sslHandler);
+ }
+
+ if (WIRETRACE)
+ pipeline.addLast("logger", new LoggingHandler(LogLevel.INFO));
+
+ pipeline.addLast("handshake", new Handler());
+ }
+
+ }
+
+ private class Handler extends ByteToMessageDecoder
+ {
+ /**
+ * {@inheritDoc}
+ *
+ * Invoked when the channel is made active, and sends out the {@link Initiate}.
+ * In the case of streaming, we do not require a full bi-directional handshake; the initial message,
+ * containing the streaming protocol version, is all that is required.
+ */
+ @Override
+ public void channelActive(final ChannelHandlerContext ctx)
+ {
+ Initiate msg = new Initiate(requestMessagingVersion, settings.acceptVersions, type, settings.framing, settings.from);
+ logger.trace("starting handshake with peer {}, msg = {}", settings.connectToId(), msg);
+ AsyncChannelPromise.writeAndFlush(ctx, msg.encode(),
+ future -> { if (!future.isSuccess()) exceptionCaught(ctx, future.cause()); });
+
+ if (type.isStreaming() && requestMessagingVersion < VERSION_40)
+ ctx.pipeline().remove(this);
+
+ ctx.fireChannelActive();
+ }
+
+ @Override
+ public void channelInactive(ChannelHandlerContext ctx) throws Exception
+ {
+ super.channelInactive(ctx);
+ resultPromise.tryFailure(new ClosedChannelException());
+ }
+
+ /**
+ * {@inheritDoc}
+ *
+ * Invoked when we get the response back from the peer, which should contain the second message of the internode messaging handshake.
+ * <p>
+ * If the peer's protocol version does not equal what we were expecting, immediately close the channel (and socket);
+ * do *not* send out the third message of the internode messaging handshake.
+ * We will reconnect on the appropriate protocol version.
+ */
+ @Override
+ protected void decode(ChannelHandlerContext ctx, ByteBuf in, List<Object> out)
+ {
+ try
+ {
+ Accept msg = Accept.maybeDecode(in, requestMessagingVersion);
+ if (msg == null)
+ return;
+
+ int useMessagingVersion = msg.useMessagingVersion;
+ int peerMessagingVersion = msg.maxMessagingVersion;
+ logger.trace("received second handshake message from peer {}, msg = {}", settings.connectTo, msg);
+
+ FrameEncoder frameEncoder = null;
+ Result<SuccessType> result;
+ if (useMessagingVersion > 0)
+ {
+ if (useMessagingVersion < settings.acceptVersions.min || useMessagingVersion > settings.acceptVersions.max)
+ {
+ result = incompatible(useMessagingVersion, peerMessagingVersion);
+ }
+ else
+ {
+ // This is a bit ugly
+ if (type.isMessaging())
+ {
+ switch (settings.framing)
+ {
+ case LZ4:
+ frameEncoder = FrameEncoderLZ4.fastInstance;
+ break;
+ case CRC:
+ frameEncoder = FrameEncoderCrc.instance;
+ break;
+ case UNPROTECTED:
+ frameEncoder = FrameEncoderUnprotected.instance;
+ break;
+ }
+
+ result = (Result<SuccessType>) messagingSuccess(ctx.channel(), useMessagingVersion, frameEncoder.allocator());
+ }
+ else
+ {
+ result = (Result<SuccessType>) streamingSuccess(ctx.channel(), useMessagingVersion);
+ }
+ }
+ }
+ else
+ {
+ assert type.isMessaging();
+
+ // pre40 handshake responses only (can be a post40 node)
+ if (peerMessagingVersion == requestMessagingVersion
+ || peerMessagingVersion > settings.acceptVersions.max) // this clause is for impersonating 3.0 node in testing only
+ {
+ switch (settings.framing)
+ {
+ case CRC:
+ case UNPROTECTED:
+ frameEncoder = FrameEncoderLegacy.instance;
+ break;
+ case LZ4:
+ frameEncoder = FrameEncoderLegacyLZ4.instance;
+ break;
+ }
+
+ result = (Result<SuccessType>) messagingSuccess(ctx.channel(), requestMessagingVersion, frameEncoder.allocator());
+ }
+ else if (peerMessagingVersion < settings.acceptVersions.min)
+ result = incompatible(-1, peerMessagingVersion);
+ else
+ result = retry(peerMessagingVersion);
+
+ if (result.isSuccess())
+ {
+ ConfirmOutboundPre40 message = new ConfirmOutboundPre40(settings.acceptVersions.max, settings.from);
+ AsyncChannelPromise.writeAndFlush(ctx, message.encode());
+ }
+ }
+
+ ChannelPipeline pipeline = ctx.pipeline();
+ if (result.isSuccess())
+ {
+ BufferPool.setRecycleWhenFreeForCurrentThread(false);
+ if (type.isMessaging())
+ {
+ assert frameEncoder != null;
+ pipeline.addLast("frameEncoder", frameEncoder);
+ }
+ pipeline.remove(this);
+ }
+ else
+ {
+ pipeline.close();
+ }
+
+ if (!resultPromise.trySuccess(result) && result.isSuccess())
+ result.success().channel.close();
+ }
+ catch (Throwable t)
+ {
+ exceptionCaught(ctx, t);
+ }
+ }
+
+ @Override
+ public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause)
+ {
+ try
+ {
+ JVMStabilityInspector.inspectThrowable(cause, false);
+ resultPromise.tryFailure(cause);
+ if (isCausedByConnectionReset(cause))
+ logger.info("Failed to connect to peer {}", settings.connectToId(), cause);
+ else
+ logger.error("Failed to handshake with peer {}", settings.connectToId(), cause);
+ ctx.close();
+ }
+ catch (Throwable t)
+ {
+ logger.error("Unexpected exception in {}.exceptionCaught", this.getClass().getSimpleName(), t);
+ }
+ }
+ }
+
+ /**
+ * The result of the handshake. Handshake has 3 possible outcomes:
+ * 1) it can be successful, in which case the channel and version to used is returned in this result.
+ * 2) we may decide to disconnect to reconnect with another protocol version (namely, the version is passed in this result).
+ * 3) we can have a negotiation failure for an unknown reason. (#sadtrombone)
+ */
+ public static class Result<SuccessType extends Result.Success>
+ {
+ /**
+ * Describes the result of receiving the response back from the peer (Message 2 of the handshake)
+ * and implies an action that should be taken.
+ */
+ enum Outcome
+ {
+ SUCCESS, RETRY, INCOMPATIBLE
+ }
+
+ public static class Success<SuccessType extends Success> extends Result<SuccessType>
+ {
+ public final Channel channel;
+ public final int messagingVersion;
+ Success(Channel channel, int messagingVersion)
+ {
+ super(Outcome.SUCCESS);
+ this.channel = channel;
+ this.messagingVersion = messagingVersion;
+ }
+ }
+
+ public static class StreamingSuccess extends Success<StreamingSuccess>
+ {
+ StreamingSuccess(Channel channel, int messagingVersion)
+ {
+ super(channel, messagingVersion);
+ }
+ }
+
+ public static class MessagingSuccess extends Success<MessagingSuccess>
+ {
+ public final FrameEncoder.PayloadAllocator allocator;
+ MessagingSuccess(Channel channel, int messagingVersion, FrameEncoder.PayloadAllocator allocator)
+ {
+ super(channel, messagingVersion);
+ this.allocator = allocator;
+ }
+ }
+
+ static class Retry<SuccessType extends Success> extends Result<SuccessType>
+ {
+ final int withMessagingVersion;
+ Retry(int withMessagingVersion)
+ {
+ super(Outcome.RETRY);
+ this.withMessagingVersion = withMessagingVersion;
+ }
+ }
+
+ static class Incompatible<SuccessType extends Success> extends Result<SuccessType>
+ {
+ final int closestSupportedVersion;
+ final int maxMessagingVersion;
+ Incompatible(int closestSupportedVersion, int maxMessagingVersion)
+ {
+ super(Outcome.INCOMPATIBLE);
+ this.closestSupportedVersion = closestSupportedVersion;
+ this.maxMessagingVersion = maxMessagingVersion;
+ }
+ }
+
+ final Outcome outcome;
+
+ private Result(Outcome outcome)
+ {
+ this.outcome = outcome;
+ }
+
+ boolean isSuccess() { return outcome == Outcome.SUCCESS; }
+ public SuccessType success() { return (SuccessType) this; }
+ static MessagingSuccess messagingSuccess(Channel channel, int messagingVersion, FrameEncoder.PayloadAllocator allocator) { return new MessagingSuccess(channel, messagingVersion, allocator); }
+ static StreamingSuccess streamingSuccess(Channel channel, int messagingVersion) { return new StreamingSuccess(channel, messagingVersion); }
+
+ public Retry retry() { return (Retry) this; }
+ static <SuccessType extends Success> Result<SuccessType> retry(int withMessagingVersion) { return new Retry<>(withMessagingVersion); }
+
+ public Incompatible incompatible() { return (Incompatible) this; }
+ static <SuccessType extends Success> Result<SuccessType> incompatible(int closestSupportedVersion, int maxMessagingVersion) { return new Incompatible(closestSupportedVersion, maxMessagingVersion); }
+ }
+
+}
diff --git a/src/java/org/apache/cassandra/net/OutboundConnectionSettings.java b/src/java/org/apache/cassandra/net/OutboundConnectionSettings.java
new file mode 100644
index 0000000..5f83b6a
--- /dev/null
+++ b/src/java/org/apache/cassandra/net/OutboundConnectionSettings.java
@@ -0,0 +1,524 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.net;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+
+import io.netty.channel.WriteBufferWaterMark;
+import org.apache.cassandra.auth.IInternodeAuthenticator;
+import org.apache.cassandra.config.Config;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.config.EncryptionOptions;
+import org.apache.cassandra.config.EncryptionOptions.ServerEncryptionOptions;
+import org.apache.cassandra.db.SystemKeyspace;
+import org.apache.cassandra.locator.IEndpointSnitch;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.utils.FBUtilities;
+
+import static org.apache.cassandra.config.DatabaseDescriptor.getEndpointSnitch;
+import static org.apache.cassandra.net.MessagingService.VERSION_40;
+import static org.apache.cassandra.net.MessagingService.instance;
+import static org.apache.cassandra.net.SocketFactory.encryptionLogStatement;
+import static org.apache.cassandra.utils.FBUtilities.getBroadcastAddressAndPort;
+
+/**
+ * A collection of settings to be passed around for outbound connections.
+ */
+@SuppressWarnings({ "WeakerAccess", "unused" })
+public class OutboundConnectionSettings
+{
+ private static final String INTRADC_TCP_NODELAY_PROPERTY = Config.PROPERTY_PREFIX + "otc_intradc_tcp_nodelay";
+ /**
+ * Enabled/disable TCP_NODELAY for intradc connections. Defaults to enabled.
+ */
+ private static final boolean INTRADC_TCP_NODELAY = Boolean.parseBoolean(System.getProperty(INTRADC_TCP_NODELAY_PROPERTY, "true"));
+
+ public enum Framing
+ {
+ // for < VERSION_40, implies no framing
+ // for >= VERSION_40, uses simple unprotected frames with header crc but no payload protection
+ UNPROTECTED(0),
+ // for < VERSION_40, uses the jpountz framing format
+ // for >= VERSION_40, uses our framing format with header crc24
+ LZ4(1),
+ // for < VERSION_40, implies UNPROTECTED
+ // for >= VERSION_40, uses simple frames with separate header and payload crc
+ CRC(2);
+
+ public static Framing forId(int id)
+ {
+ switch (id)
+ {
+ case 0: return UNPROTECTED;
+ case 1: return LZ4;
+ case 2: return CRC;
+ }
+ throw new IllegalStateException();
+ }
+
+ final int id;
+ Framing(int id)
+ {
+ this.id = id;
+ }
+ }
+
+ public final IInternodeAuthenticator authenticator;
+ public final InetAddressAndPort to;
+ public final InetAddressAndPort connectTo; // may be represented by a different IP address on this node's local network
+ public final EncryptionOptions encryption;
+ public final Framing framing;
+ public final Integer socketSendBufferSizeInBytes;
+ public final Integer applicationSendQueueCapacityInBytes;
+ public final Integer applicationSendQueueReserveEndpointCapacityInBytes;
+ public final ResourceLimits.Limit applicationSendQueueReserveGlobalCapacityInBytes;
+ public final Boolean tcpNoDelay;
+ public final int flushLowWaterMark, flushHighWaterMark;
+ public final Integer tcpConnectTimeoutInMS;
+ public final Integer tcpUserTimeoutInMS;
+ public final AcceptVersions acceptVersions;
+ public final InetAddressAndPort from;
+ public final SocketFactory socketFactory;
+ public final OutboundMessageCallbacks callbacks;
+ public final OutboundDebugCallbacks debug;
+ public final EndpointMessagingVersions endpointToVersion;
+
+ public OutboundConnectionSettings(InetAddressAndPort to)
+ {
+ this(to, null);
+ }
+
+ public OutboundConnectionSettings(InetAddressAndPort to, InetAddressAndPort preferred)
+ {
+ this(null, to, preferred, null, null, null, null, null, null, null, 1 << 15, 1 << 16, null, null, null, null, null, null, null, null);
+ }
+
+ private OutboundConnectionSettings(IInternodeAuthenticator authenticator,
+ InetAddressAndPort to,
+ InetAddressAndPort connectTo,
+ EncryptionOptions encryption,
+ Framing framing,
+ Integer socketSendBufferSizeInBytes,
+ Integer applicationSendQueueCapacityInBytes,
+ Integer applicationSendQueueReserveEndpointCapacityInBytes,
+ ResourceLimits.Limit applicationSendQueueReserveGlobalCapacityInBytes,
+ Boolean tcpNoDelay,
+ int flushLowWaterMark,
+ int flushHighWaterMark,
+ Integer tcpConnectTimeoutInMS,
+ Integer tcpUserTimeoutInMS,
+ AcceptVersions acceptVersions,
+ InetAddressAndPort from,
+ SocketFactory socketFactory,
+ OutboundMessageCallbacks callbacks,
+ OutboundDebugCallbacks debug,
+ EndpointMessagingVersions endpointToVersion)
+ {
+ Preconditions.checkArgument(socketSendBufferSizeInBytes == null || socketSendBufferSizeInBytes == 0 || socketSendBufferSizeInBytes >= 1 << 10, "illegal socket send buffer size: " + socketSendBufferSizeInBytes);
+ Preconditions.checkArgument(applicationSendQueueCapacityInBytes == null || applicationSendQueueCapacityInBytes >= 1 << 10, "illegal application send queue capacity: " + applicationSendQueueCapacityInBytes);
+ Preconditions.checkArgument(tcpUserTimeoutInMS == null || tcpUserTimeoutInMS >= 0, "tcp user timeout must be non negative: " + tcpUserTimeoutInMS);
+ Preconditions.checkArgument(tcpConnectTimeoutInMS == null || tcpConnectTimeoutInMS > 0, "tcp connect timeout must be positive: " + tcpConnectTimeoutInMS);
+
+ this.authenticator = authenticator;
+ this.to = to;
+ this.connectTo = connectTo;
+ this.encryption = encryption;
+ this.framing = framing;
+ this.socketSendBufferSizeInBytes = socketSendBufferSizeInBytes;
+ this.applicationSendQueueCapacityInBytes = applicationSendQueueCapacityInBytes;
+ this.applicationSendQueueReserveEndpointCapacityInBytes = applicationSendQueueReserveEndpointCapacityInBytes;
+ this.applicationSendQueueReserveGlobalCapacityInBytes = applicationSendQueueReserveGlobalCapacityInBytes;
+ this.tcpNoDelay = tcpNoDelay;
+ this.flushLowWaterMark = flushLowWaterMark;
+ this.flushHighWaterMark = flushHighWaterMark;
+ this.tcpConnectTimeoutInMS = tcpConnectTimeoutInMS;
+ this.tcpUserTimeoutInMS = tcpUserTimeoutInMS;
+ this.acceptVersions = acceptVersions;
+ this.from = from;
+ this.socketFactory = socketFactory;
+ this.callbacks = callbacks;
+ this.debug = debug;
+ this.endpointToVersion = endpointToVersion;
+ }
+
+ public boolean authenticate()
+ {
+ return authenticator.authenticate(to.address, to.port);
+ }
+
+ public boolean withEncryption()
+ {
+ return encryption != null;
+ }
+
+ public String toString()
+ {
+ return String.format("peer: (%s, %s), framing: %s, encryption: %s",
+ to, connectTo, framing, encryptionLogStatement(encryption));
+ }
+
+ public OutboundConnectionSettings withAuthenticator(IInternodeAuthenticator authenticator)
+ {
+ return new OutboundConnectionSettings(authenticator, to, connectTo, encryption, framing,
+ socketSendBufferSizeInBytes, applicationSendQueueCapacityInBytes,
+ applicationSendQueueReserveEndpointCapacityInBytes, applicationSendQueueReserveGlobalCapacityInBytes,
+ tcpNoDelay, flushLowWaterMark, flushHighWaterMark, tcpConnectTimeoutInMS,
+ tcpUserTimeoutInMS, acceptVersions, from, socketFactory, callbacks, debug, endpointToVersion);
+ }
+
+ @SuppressWarnings("unused")
+ public OutboundConnectionSettings toEndpoint(InetAddressAndPort endpoint)
+ {
+ return new OutboundConnectionSettings(authenticator, endpoint, connectTo, encryption, framing,
+ socketSendBufferSizeInBytes, applicationSendQueueCapacityInBytes,
+ applicationSendQueueReserveEndpointCapacityInBytes, applicationSendQueueReserveGlobalCapacityInBytes,
+ tcpNoDelay, flushLowWaterMark, flushHighWaterMark, tcpConnectTimeoutInMS,
+ tcpUserTimeoutInMS, acceptVersions, from, socketFactory, callbacks, debug, endpointToVersion);
+ }
+
+ public OutboundConnectionSettings withConnectTo(InetAddressAndPort connectTo)
+ {
+ return new OutboundConnectionSettings(authenticator, to, connectTo, encryption, framing,
+ socketSendBufferSizeInBytes, applicationSendQueueCapacityInBytes,
+ applicationSendQueueReserveEndpointCapacityInBytes, applicationSendQueueReserveGlobalCapacityInBytes,
+ tcpNoDelay, flushLowWaterMark, flushHighWaterMark, tcpConnectTimeoutInMS,
+ tcpUserTimeoutInMS, acceptVersions, from, socketFactory, callbacks, debug, endpointToVersion);
+ }
+
+ public OutboundConnectionSettings withEncryption(ServerEncryptionOptions encryption)
+ {
+ return new OutboundConnectionSettings(authenticator, to, connectTo, encryption, framing,
+ socketSendBufferSizeInBytes, applicationSendQueueCapacityInBytes,
+ applicationSendQueueReserveEndpointCapacityInBytes, applicationSendQueueReserveGlobalCapacityInBytes,
+ tcpNoDelay, flushLowWaterMark, flushHighWaterMark, tcpConnectTimeoutInMS,
+ tcpUserTimeoutInMS, acceptVersions, from, socketFactory, callbacks, debug, endpointToVersion);
+ }
+
+ @SuppressWarnings("unused")
+ public OutboundConnectionSettings withFraming(Framing framing)
+ {
+ return new OutboundConnectionSettings(authenticator, to, connectTo, encryption, framing, socketSendBufferSizeInBytes, applicationSendQueueCapacityInBytes,
+ applicationSendQueueReserveEndpointCapacityInBytes, applicationSendQueueReserveGlobalCapacityInBytes,
+ tcpNoDelay, flushLowWaterMark, flushHighWaterMark, tcpConnectTimeoutInMS,
+ tcpUserTimeoutInMS, acceptVersions, from, socketFactory, callbacks, debug, endpointToVersion);
+ }
+
+ public OutboundConnectionSettings withSocketSendBufferSizeInBytes(int socketSendBufferSizeInBytes)
+ {
+ return new OutboundConnectionSettings(authenticator, to, connectTo, encryption, framing,
+ socketSendBufferSizeInBytes, applicationSendQueueCapacityInBytes,
+ applicationSendQueueReserveEndpointCapacityInBytes, applicationSendQueueReserveGlobalCapacityInBytes,
+ tcpNoDelay, flushLowWaterMark, flushHighWaterMark, tcpConnectTimeoutInMS,
+ tcpUserTimeoutInMS, acceptVersions, from, socketFactory, callbacks, debug, endpointToVersion);
+ }
+
+ @SuppressWarnings("unused")
+ public OutboundConnectionSettings withApplicationSendQueueCapacityInBytes(int applicationSendQueueCapacityInBytes)
+ {
+ return new OutboundConnectionSettings(authenticator, to, connectTo, encryption, framing,
+ socketSendBufferSizeInBytes, applicationSendQueueCapacityInBytes,
+ applicationSendQueueReserveEndpointCapacityInBytes, applicationSendQueueReserveGlobalCapacityInBytes,
+ tcpNoDelay, flushLowWaterMark, flushHighWaterMark, tcpConnectTimeoutInMS,
+ tcpUserTimeoutInMS, acceptVersions, from, socketFactory, callbacks, debug, endpointToVersion);
+ }
+
+ public OutboundConnectionSettings withApplicationReserveSendQueueCapacityInBytes(Integer applicationReserveSendQueueEndpointCapacityInBytes, ResourceLimits.Limit applicationReserveSendQueueGlobalCapacityInBytes)
+ {
+ return new OutboundConnectionSettings(authenticator, to, connectTo, encryption, framing,
+ socketSendBufferSizeInBytes, applicationSendQueueCapacityInBytes,
+ applicationReserveSendQueueEndpointCapacityInBytes, applicationReserveSendQueueGlobalCapacityInBytes,
+ tcpNoDelay, flushLowWaterMark, flushHighWaterMark, tcpConnectTimeoutInMS,
+ tcpUserTimeoutInMS, acceptVersions, from, socketFactory, callbacks, debug, endpointToVersion);
+ }
+
+ @SuppressWarnings("unused")
+ public OutboundConnectionSettings withTcpNoDelay(boolean tcpNoDelay)
+ {
+ return new OutboundConnectionSettings(authenticator, to, connectTo, encryption, framing,
+ socketSendBufferSizeInBytes, applicationSendQueueCapacityInBytes,
+ applicationSendQueueReserveEndpointCapacityInBytes, applicationSendQueueReserveGlobalCapacityInBytes,
+ tcpNoDelay, flushLowWaterMark, flushHighWaterMark, tcpConnectTimeoutInMS,
+ tcpUserTimeoutInMS, acceptVersions, from, socketFactory, callbacks, debug, endpointToVersion);
+ }
+
+ @SuppressWarnings("unused")
+ public OutboundConnectionSettings withNettyBufferBounds(WriteBufferWaterMark nettyBufferBounds)
+ {
+ return new OutboundConnectionSettings(authenticator, to, connectTo, encryption, framing,
+ socketSendBufferSizeInBytes, applicationSendQueueCapacityInBytes,
+ applicationSendQueueReserveEndpointCapacityInBytes, applicationSendQueueReserveGlobalCapacityInBytes,
+ tcpNoDelay, flushLowWaterMark, flushHighWaterMark, tcpConnectTimeoutInMS,
+ tcpUserTimeoutInMS, acceptVersions, from, socketFactory, callbacks, debug, endpointToVersion);
+ }
+
+ public OutboundConnectionSettings withTcpConnectTimeoutInMS(int tcpConnectTimeoutInMS)
+ {
+ return new OutboundConnectionSettings(authenticator, to, connectTo, encryption, framing,
+ socketSendBufferSizeInBytes, applicationSendQueueCapacityInBytes,
+ applicationSendQueueReserveEndpointCapacityInBytes, applicationSendQueueReserveGlobalCapacityInBytes,
+ tcpNoDelay, flushLowWaterMark, flushHighWaterMark, tcpConnectTimeoutInMS,
+ tcpUserTimeoutInMS, acceptVersions, from, socketFactory, callbacks, debug, endpointToVersion);
+ }
+
+ public OutboundConnectionSettings withTcpUserTimeoutInMS(int tcpUserTimeoutInMS)
+ {
+ return new OutboundConnectionSettings(authenticator, to, connectTo, encryption, framing,
+ socketSendBufferSizeInBytes, applicationSendQueueCapacityInBytes,
+ applicationSendQueueReserveEndpointCapacityInBytes, applicationSendQueueReserveGlobalCapacityInBytes,
+ tcpNoDelay, flushLowWaterMark, flushHighWaterMark, tcpConnectTimeoutInMS,
+ tcpUserTimeoutInMS, acceptVersions, from, socketFactory, callbacks, debug, endpointToVersion);
+ }
+
+ public OutboundConnectionSettings withAcceptVersions(AcceptVersions acceptVersions)
+ {
+ return new OutboundConnectionSettings(authenticator, to, connectTo, encryption, framing,
+ socketSendBufferSizeInBytes, applicationSendQueueCapacityInBytes,
+ applicationSendQueueReserveEndpointCapacityInBytes, applicationSendQueueReserveGlobalCapacityInBytes,
+ tcpNoDelay, flushLowWaterMark, flushHighWaterMark, tcpConnectTimeoutInMS,
+ tcpUserTimeoutInMS, acceptVersions, from, socketFactory, callbacks, debug, endpointToVersion);
+ }
+
+ public OutboundConnectionSettings withFrom(InetAddressAndPort from)
+ {
+ return new OutboundConnectionSettings(authenticator, to, connectTo, encryption, framing,
+ socketSendBufferSizeInBytes, applicationSendQueueCapacityInBytes,
+ applicationSendQueueReserveEndpointCapacityInBytes, applicationSendQueueReserveGlobalCapacityInBytes,
+ tcpNoDelay, flushLowWaterMark, flushHighWaterMark, tcpConnectTimeoutInMS,
+ tcpUserTimeoutInMS, acceptVersions, from, socketFactory, callbacks, debug, endpointToVersion);
+ }
+
+ public OutboundConnectionSettings withSocketFactory(SocketFactory socketFactory)
+ {
+ return new OutboundConnectionSettings(authenticator, to, connectTo, encryption, framing,
+ socketSendBufferSizeInBytes, applicationSendQueueCapacityInBytes,
+ applicationSendQueueReserveEndpointCapacityInBytes, applicationSendQueueReserveGlobalCapacityInBytes,
+ tcpNoDelay, flushLowWaterMark, flushHighWaterMark, tcpConnectTimeoutInMS,
+ tcpUserTimeoutInMS, acceptVersions, from, socketFactory, callbacks, debug, endpointToVersion);
+ }
+
+ public OutboundConnectionSettings withCallbacks(OutboundMessageCallbacks callbacks)
+ {
+ return new OutboundConnectionSettings(authenticator, to, connectTo, encryption, framing,
+ socketSendBufferSizeInBytes, applicationSendQueueCapacityInBytes,
+ applicationSendQueueReserveEndpointCapacityInBytes, applicationSendQueueReserveGlobalCapacityInBytes,
+ tcpNoDelay, flushLowWaterMark, flushHighWaterMark, tcpConnectTimeoutInMS,
+ tcpUserTimeoutInMS, acceptVersions, from, socketFactory, callbacks, debug, endpointToVersion);
+ }
+
+ public OutboundConnectionSettings withDebugCallbacks(OutboundDebugCallbacks debug)
+ {
+ return new OutboundConnectionSettings(authenticator, to, connectTo, encryption, framing,
+ socketSendBufferSizeInBytes, applicationSendQueueCapacityInBytes,
+ applicationSendQueueReserveEndpointCapacityInBytes, applicationSendQueueReserveGlobalCapacityInBytes,
+ tcpNoDelay, flushLowWaterMark, flushHighWaterMark, tcpConnectTimeoutInMS,
+ tcpUserTimeoutInMS, acceptVersions, from, socketFactory, callbacks, debug, endpointToVersion);
+ }
+
+ public OutboundConnectionSettings withDefaultReserveLimits()
+ {
+ Integer applicationReserveSendQueueEndpointCapacityInBytes = this.applicationSendQueueReserveEndpointCapacityInBytes;
+ ResourceLimits.Limit applicationReserveSendQueueGlobalCapacityInBytes = this.applicationSendQueueReserveGlobalCapacityInBytes;
+
+ if (applicationReserveSendQueueEndpointCapacityInBytes == null)
+ applicationReserveSendQueueEndpointCapacityInBytes = DatabaseDescriptor.getInternodeApplicationSendQueueReserveEndpointCapacityInBytes();
+ if (applicationReserveSendQueueGlobalCapacityInBytes == null)
+ applicationReserveSendQueueGlobalCapacityInBytes = MessagingService.instance().outboundGlobalReserveLimit;
+
+ return withApplicationReserveSendQueueCapacityInBytes(applicationReserveSendQueueEndpointCapacityInBytes, applicationReserveSendQueueGlobalCapacityInBytes);
+ }
+
+ public IInternodeAuthenticator authenticator()
+ {
+ return authenticator != null ? authenticator : DatabaseDescriptor.getInternodeAuthenticator();
+ }
+
+ public EndpointMessagingVersions endpointToVersion()
+ {
+ if (endpointToVersion == null)
+ return instance().versions;
+ return endpointToVersion;
+ }
+
+ public InetAddressAndPort from()
+ {
+ return from != null ? from : FBUtilities.getBroadcastAddressAndPort();
+ }
+
+ public OutboundDebugCallbacks debug()
+ {
+ return debug != null ? debug : OutboundDebugCallbacks.NONE;
+ }
+
+ public EncryptionOptions encryption()
+ {
+ return encryption != null ? encryption : defaultEncryptionOptions(to);
+ }
+
+ public SocketFactory socketFactory()
+ {
+ return socketFactory != null ? socketFactory : instance().socketFactory;
+ }
+
+ public OutboundMessageCallbacks callbacks()
+ {
+ return callbacks != null ? callbacks : instance().callbacks;
+ }
+
+ public int socketSendBufferSizeInBytes()
+ {
+ return socketSendBufferSizeInBytes != null ? socketSendBufferSizeInBytes
+ : DatabaseDescriptor.getInternodeSocketSendBufferSizeInBytes();
+ }
+
+ public int applicationSendQueueCapacityInBytes()
+ {
+ return applicationSendQueueCapacityInBytes != null ? applicationSendQueueCapacityInBytes
+ : DatabaseDescriptor.getInternodeApplicationSendQueueCapacityInBytes();
+ }
+
+ public ResourceLimits.Limit applicationSendQueueReserveGlobalCapacityInBytes()
+ {
+ return applicationSendQueueReserveGlobalCapacityInBytes != null ? applicationSendQueueReserveGlobalCapacityInBytes
+ : instance().outboundGlobalReserveLimit;
+ }
+
+ public int applicationSendQueueReserveEndpointCapacityInBytes()
+ {
+ return applicationSendQueueReserveEndpointCapacityInBytes != null ? applicationSendQueueReserveEndpointCapacityInBytes
+ : DatabaseDescriptor.getInternodeApplicationReceiveQueueReserveEndpointCapacityInBytes();
+ }
+
+ public int tcpConnectTimeoutInMS()
+ {
+ return tcpConnectTimeoutInMS != null ? tcpConnectTimeoutInMS
+ : DatabaseDescriptor.getInternodeTcpConnectTimeoutInMS();
+ }
+
+ public int tcpUserTimeoutInMS()
+ {
+ return tcpUserTimeoutInMS != null ? tcpUserTimeoutInMS
+ : DatabaseDescriptor.getInternodeTcpUserTimeoutInMS();
+ }
+
+ public boolean tcpNoDelay()
+ {
+ if (tcpNoDelay != null)
+ return tcpNoDelay;
+
+ if (isInLocalDC(getEndpointSnitch(), getBroadcastAddressAndPort(), to))
+ return INTRADC_TCP_NODELAY;
+
+ return DatabaseDescriptor.getInterDCTcpNoDelay();
+ }
+
+ public AcceptVersions acceptVersions(ConnectionCategory category)
+ {
+ return acceptVersions != null ? acceptVersions
+ : category.isStreaming()
+ ? MessagingService.accept_streaming
+ : MessagingService.accept_messaging;
+ }
+
+ public OutboundConnectionSettings withLegacyPortIfNecessary(int messagingVersion)
+ {
+ return withConnectTo(maybeWithSecurePort(connectTo(), messagingVersion, withEncryption()));
+ }
+
+ public InetAddressAndPort connectTo()
+ {
+ InetAddressAndPort connectTo = this.connectTo;
+ if (connectTo == null)
+ connectTo = SystemKeyspace.getPreferredIP(to);
+ return connectTo;
+ }
+
+ public String connectToId()
+ {
+ return !to.equals(connectTo())
+ ? to.toString()
+ : to.toString() + '(' + connectTo().toString() + ')';
+ }
+
+ public Framing framing(ConnectionCategory category)
+ {
+ if (framing != null)
+ return framing;
+
+ if (category.isStreaming())
+ return Framing.UNPROTECTED;
+
+ return shouldCompressConnection(getEndpointSnitch(), getBroadcastAddressAndPort(), to)
+ ? Framing.LZ4 : Framing.CRC;
+ }
+
+ // note that connectTo is updated even if specified, in the case of pre40 messaging and using encryption (to update port)
+ public OutboundConnectionSettings withDefaults(ConnectionCategory category)
+ {
+ if (to == null)
+ throw new IllegalArgumentException();
+
+ return new OutboundConnectionSettings(authenticator(), to, connectTo(),
+ encryption(), framing(category),
+ socketSendBufferSizeInBytes(), applicationSendQueueCapacityInBytes(),
+ applicationSendQueueReserveEndpointCapacityInBytes(),
+ applicationSendQueueReserveGlobalCapacityInBytes(),
+ tcpNoDelay(), flushLowWaterMark, flushHighWaterMark,
+ tcpConnectTimeoutInMS(), tcpUserTimeoutInMS(), acceptVersions(category),
+ from(), socketFactory(), callbacks(), debug(), endpointToVersion());
+ }
+
+ private static boolean isInLocalDC(IEndpointSnitch snitch, InetAddressAndPort localHost, InetAddressAndPort remoteHost)
+ {
+ String remoteDC = snitch.getDatacenter(remoteHost);
+ String localDC = snitch.getDatacenter(localHost);
+ return remoteDC != null && remoteDC.equals(localDC);
+ }
+
+ @VisibleForTesting
+ static EncryptionOptions defaultEncryptionOptions(InetAddressAndPort endpoint)
+ {
+ ServerEncryptionOptions options = DatabaseDescriptor.getInternodeMessagingEncyptionOptions();
+ return options.shouldEncrypt(endpoint) ? options : null;
+ }
+
+ @VisibleForTesting
+ static boolean shouldCompressConnection(IEndpointSnitch snitch, InetAddressAndPort localHost, InetAddressAndPort remoteHost)
+ {
+ return (DatabaseDescriptor.internodeCompression() == Config.InternodeCompression.all)
+ || ((DatabaseDescriptor.internodeCompression() == Config.InternodeCompression.dc) && !isInLocalDC(snitch, localHost, remoteHost));
+ }
+
+ private static InetAddressAndPort maybeWithSecurePort(InetAddressAndPort address, int messagingVersion, boolean isEncrypted)
+ {
+ if (!isEncrypted || messagingVersion >= VERSION_40)
+ return address;
+
+ // if we don't know the version of the peer, assume it is 4.0 (or higher) as the only time is would be lower
+ // (as in a 3.x version) is during a cluster upgrade (from 3.x to 4.0). In that case the outbound connection will
+ // unfortunately fail - however the peer should connect to this node (at some point), and once we learn it's version, it'll be
+ // in versions map. thus, when we attempt to reconnect to that node, we'll have the version and we can get the correct port.
+ // we will be able to remove this logic at 5.0.
+ // Also as of 4.0 we will propagate the "regular" port (which will support both SSL and non-SSL) via gossip so
+ // for SSL and version 4.0 always connect to the gossiped port because if SSL is enabled it should ALWAYS
+ // listen for SSL on the "regular" port.
+ return address.withPort(DatabaseDescriptor.getSSLStoragePort());
+ }
+
+}
diff --git a/src/java/org/apache/cassandra/net/OutboundConnections.java b/src/java/org/apache/cassandra/net/OutboundConnections.java
new file mode 100644
index 0000000..c900908
--- /dev/null
+++ b/src/java/org/apache/cassandra/net/OutboundConnections.java
@@ -0,0 +1,323 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.net;
+
+import java.nio.channels.ClosedChannelException;
+import java.util.List;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Function;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableList;
+
+import com.carrotsearch.hppc.ObjectObjectHashMap;
+import io.netty.util.concurrent.Future;
+import org.apache.cassandra.config.Config;
+import org.apache.cassandra.gms.Gossiper;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.metrics.InternodeOutboundMetrics;
+import org.apache.cassandra.utils.concurrent.SimpleCondition;
+
+import static org.apache.cassandra.net.MessagingService.current_version;
+import static org.apache.cassandra.net.ConnectionType.URGENT_MESSAGES;
+import static org.apache.cassandra.net.ConnectionType.LARGE_MESSAGES;
+import static org.apache.cassandra.net.ConnectionType.SMALL_MESSAGES;
+
+/**
+ * Groups a set of outbound connections to a given peer, and routes outgoing messages to the appropriate connection
+ * (based upon message's type or size). Contains a {@link OutboundConnection} for each of the
+ * {@link ConnectionType} types.
+ */
+public class OutboundConnections
+{
+ @VisibleForTesting
+ public static final int LARGE_MESSAGE_THRESHOLD = Integer.getInteger(Config.PROPERTY_PREFIX + "otcp_large_message_threshold", 1024 * 64)
+ - Math.max(Math.max(LegacyLZ4Constants.HEADER_LENGTH, FrameEncoderCrc.HEADER_AND_TRAILER_LENGTH), FrameEncoderLZ4.HEADER_AND_TRAILER_LENGTH);
+
+ private final SimpleCondition metricsReady = new SimpleCondition();
+ private volatile InternodeOutboundMetrics metrics;
+ private final BackPressureState backPressureState;
+ private final ResourceLimits.Limit reserveCapacity;
+
+ private OutboundConnectionSettings template;
+ public final OutboundConnection small;
+ public final OutboundConnection large;
+ public final OutboundConnection urgent;
+
+ private OutboundConnections(OutboundConnectionSettings template, BackPressureState backPressureState)
+ {
+ this.backPressureState = backPressureState;
+ this.template = template = template.withDefaultReserveLimits();
+ reserveCapacity = new ResourceLimits.Concurrent(template.applicationSendQueueReserveEndpointCapacityInBytes);
+ ResourceLimits.EndpointAndGlobal reserveCapacityInBytes = new ResourceLimits.EndpointAndGlobal(reserveCapacity, template.applicationSendQueueReserveGlobalCapacityInBytes);
+ this.small = new OutboundConnection(SMALL_MESSAGES, template, reserveCapacityInBytes);
+ this.large = new OutboundConnection(LARGE_MESSAGES, template, reserveCapacityInBytes);
+ this.urgent = new OutboundConnection(URGENT_MESSAGES, template, reserveCapacityInBytes);
+ }
+
+ /**
+ * Select the appropriate connection for the provided message and use it to send the message.
+ */
+ public void enqueue(Message msg, ConnectionType type) throws ClosedChannelException
+ {
+ connectionFor(msg, type).enqueue(msg);
+ }
+
+ static <K> OutboundConnections tryRegister(ConcurrentMap<K, OutboundConnections> in, K key, OutboundConnectionSettings settings, BackPressureState backPressureState)
+ {
+ OutboundConnections connections = in.get(key);
+ if (connections == null)
+ {
+ connections = new OutboundConnections(settings, backPressureState);
+ OutboundConnections existing = in.putIfAbsent(key, connections);
+
+ if (existing == null)
+ {
+ connections.metrics = new InternodeOutboundMetrics(settings.to, connections);
+ connections.metricsReady.signalAll();
+ }
+ else
+ {
+ connections.metricsReady.signalAll();
+ connections.close(false);
+ connections = existing;
+ }
+ }
+ return connections;
+ }
+
+ BackPressureState getBackPressureState()
+ {
+ return backPressureState;
+ }
+
+ /**
+ * Reconnect to the peer using the given {@code addr}. Outstanding messages in each channel will be sent on the
+ * current channel. Typically this function is used for something like EC2 public IP addresses which need to be used
+ * for communication between EC2 regions.
+ *
+ * @param addr IP Address to use (and prefer) going forward for connecting to the peer
+ */
+ synchronized Future<Void> reconnectWithNewIp(InetAddressAndPort addr)
+ {
+ template = template.withConnectTo(addr);
+ return new FutureCombiner(
+ apply(c -> c.reconnectWith(template))
+ );
+ }
+
+ /**
+ * Close the connections permanently
+ *
+ * @param flushQueues {@code true} if existing messages in the queue should be sent before closing.
+ */
+ synchronized Future<Void> scheduleClose(long time, TimeUnit unit, boolean flushQueues)
+ {
+ // immediately release our metrics, so that if we need to re-open immediately we can safely register a new one
+ releaseMetrics();
+ return new FutureCombiner(
+ apply(c -> c.scheduleClose(time, unit, flushQueues))
+ );
+ }
+
+ /**
+ * Close the connections permanently
+ *
+ * @param flushQueues {@code true} if existing messages in the queue should be sent before closing.
+ */
+ synchronized Future<Void> close(boolean flushQueues)
+ {
+ // immediately release our metrics, so that if we need to re-open immediately we can safely register a new one
+ releaseMetrics();
+ return new FutureCombiner(
+ apply(c -> c.close(flushQueues))
+ );
+ }
+
+ private void releaseMetrics()
+ {
+ try
+ {
+ metricsReady.await();
+ }
+ catch (InterruptedException e)
+ {
+ throw new RuntimeException(e);
+ }
+
+ if (metrics != null)
+ metrics.release();
+ }
+
+ /**
+ * Close each netty channel and its socket
+ */
+ void interrupt()
+ {
+ // must return a non-null value for ImmutableList.of()
+ apply(OutboundConnection::interrupt);
+ }
+
+ /**
+ * Apply the given function to each of the connections we are pooling, returning the results as a list
+ */
+ private <V> List<V> apply(Function<OutboundConnection, V> f)
+ {
+ return ImmutableList.of(
+ f.apply(urgent), f.apply(small), f.apply(large)
+ );
+ }
+
+ @VisibleForTesting
+ OutboundConnection connectionFor(Message<?> message)
+ {
+ return connectionFor(message, null);
+ }
+
+ private OutboundConnection connectionFor(Message msg, ConnectionType forceConnection)
+ {
+ return connectionFor(connectionTypeFor(msg, forceConnection));
+ }
+
+ private static ConnectionType connectionTypeFor(Message<?> msg, ConnectionType specifyConnection)
+ {
+ if (specifyConnection != null)
+ return specifyConnection;
+
+ if (msg.verb().priority == Verb.Priority.P0)
+ return URGENT_MESSAGES;
+
+ return msg.serializedSize(current_version) <= LARGE_MESSAGE_THRESHOLD
+ ? SMALL_MESSAGES
+ : LARGE_MESSAGES;
+ }
+
+ @VisibleForTesting
+ final OutboundConnection connectionFor(ConnectionType type)
+ {
+ switch (type)
+ {
+ case SMALL_MESSAGES:
+ return small;
+ case LARGE_MESSAGES:
+ return large;
+ case URGENT_MESSAGES:
+ return urgent;
+ default:
+ throw new IllegalArgumentException("unsupported connection type: " + type);
+ }
+ }
+
+ public long usingReserveBytes()
+ {
+ return reserveCapacity.using();
+ }
+
+ long expiredCallbacks()
+ {
+ return metrics.expiredCallbacks.getCount();
+ }
+
+ void incrementExpiredCallbackCount()
+ {
+ metrics.expiredCallbacks.mark();
+ }
+
+ OutboundConnectionSettings template()
+ {
+ return template;
+ }
+
+ private static class UnusedConnectionMonitor
+ {
+ UnusedConnectionMonitor(MessagingService messagingService)
+ {
+ this.messagingService = messagingService;
+ }
+
+ static class Counts
+ {
+ final long small, large, urgent;
+ Counts(long small, long large, long urgent)
+ {
+ this.small = small;
+ this.large = large;
+ this.urgent = urgent;
+ }
+ }
+
+ final MessagingService messagingService;
+ ObjectObjectHashMap<InetAddressAndPort, Counts> prevEndpointToCounts = new ObjectObjectHashMap<>();
+
+ private void closeUnusedSinceLastRun()
+ {
+ ObjectObjectHashMap<InetAddressAndPort, Counts> curEndpointToCounts = new ObjectObjectHashMap<>();
+ for (OutboundConnections connections : messagingService.channelManagers.values())
+ {
+ Counts cur = new Counts(
+ connections.small.submittedCount(),
+ connections.large.submittedCount(),
+ connections.urgent.submittedCount()
+ );
+ curEndpointToCounts.put(connections.template.to, cur);
+
+ Counts prev = prevEndpointToCounts.get(connections.template.to);
+ if (prev == null)
+ continue;
+
+ if (cur.small != prev.small && cur.large != prev.large && cur.urgent != prev.urgent)
+ continue;
+
+ if (cur.small == prev.small && cur.large == prev.large && cur.urgent == prev.urgent
+ && !Gossiper.instance.isKnownEndpoint(connections.template.to))
+ {
+ // close entirely if no traffic and the endpoint is unknown
+ messagingService.closeOutboundNow(connections);
+ continue;
+ }
+
+ if (cur.small == prev.small)
+ connections.small.interrupt();
+
+ if (cur.large == prev.large)
+ connections.large.interrupt();
+
+ if (cur.urgent == prev.urgent)
+ connections.urgent.interrupt();
+ }
+
+ prevEndpointToCounts = curEndpointToCounts;
+ }
+ }
+
+ static void scheduleUnusedConnectionMonitoring(MessagingService messagingService, ScheduledExecutorService executor, long delay, TimeUnit units)
+ {
+ executor.scheduleWithFixedDelay(new UnusedConnectionMonitor(messagingService)::closeUnusedSinceLastRun, 0L, delay, units);
+ }
+
+ @VisibleForTesting
+ static OutboundConnections unsafeCreate(OutboundConnectionSettings template, BackPressureState backPressureState)
+ {
+ OutboundConnections connections = new OutboundConnections(template, backPressureState);
+ connections.metricsReady.signalAll();
+ return connections;
+ }
+
+}
diff --git a/src/java/org/apache/cassandra/net/OutboundDebugCallbacks.java b/src/java/org/apache/cassandra/net/OutboundDebugCallbacks.java
new file mode 100644
index 0000000..3b83519
--- /dev/null
+++ b/src/java/org/apache/cassandra/net/OutboundDebugCallbacks.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.net;
+
+interface OutboundDebugCallbacks
+{
+ static final OutboundDebugCallbacks NONE = new OutboundDebugCallbacks()
+ {
+ public void onSendSmallFrame(int messageCount, int payloadSizeInBytes) {}
+ public void onSentSmallFrame(int messageCount, int payloadSizeInBytes) {}
+ public void onFailedSmallFrame(int messageCount, int payloadSizeInBytes) {}
+ public void onConnect(int messagingVersion, OutboundConnectionSettings settings) {}
+ };
+
+ /** A complete Frame has been handed to Netty to write to the wire. */
+ void onSendSmallFrame(int messageCount, int payloadSizeInBytes);
+
+ /** A complete Frame has been serialized to the wire */
+ void onSentSmallFrame(int messageCount, int payloadSizeInBytes);
+
+ /** Failed to send an entire frame due to network problems; presumed to be invoked in same order as onSendSmallFrame */
+ void onFailedSmallFrame(int messageCount, int payloadSizeInBytes);
+
+ void onConnect(int messagingVersion, OutboundConnectionSettings settings);
+}
diff --git a/src/java/org/apache/cassandra/net/OutboundMessageCallbacks.java b/src/java/org/apache/cassandra/net/OutboundMessageCallbacks.java
new file mode 100644
index 0000000..abf3f41
--- /dev/null
+++ b/src/java/org/apache/cassandra/net/OutboundMessageCallbacks.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.net;
+
+import org.apache.cassandra.locator.InetAddressAndPort;
+
+interface OutboundMessageCallbacks
+{
+ /** A message was not enqueued to the link because too many messages are already waiting to send */
+ void onOverloaded(Message<?> message, InetAddressAndPort peer);
+
+ /** A message was not serialized to a frame because it had expired */
+ void onExpired(Message<?> message, InetAddressAndPort peer);
+
+ /** A message was not fully or successfully serialized to a frame because an exception was thrown */
+ void onFailedSerialize(Message<?> message, InetAddressAndPort peer, int messagingVersion, int bytesWrittenToNetwork, Throwable failure);
+
+ /** A message was not sent because the connection was forcibly closed */
+ void onDiscardOnClose(Message<?> message, InetAddressAndPort peer);
+}
diff --git a/src/java/org/apache/cassandra/net/OutboundMessageQueue.java b/src/java/org/apache/cassandra/net/OutboundMessageQueue.java
new file mode 100644
index 0000000..3d8bac0
--- /dev/null
+++ b/src/java/org/apache/cassandra/net/OutboundMessageQueue.java
@@ -0,0 +1,528 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.net;
+
+import java.util.Collections;
+import java.util.IdentityHashMap;
+import java.util.Set;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLongFieldUpdater;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.concurrent.atomic.AtomicReferenceFieldUpdater;
+import java.util.function.Consumer;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.util.concurrent.Uninterruptibles;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.utils.MonotonicClock;
+
+import static java.lang.Math.min;
+
+/**
+ * A composite queue holding messages to be delivered by an {@link OutboundConnection}.
+ *
+ * Contains two queues:
+ * 1. An external MPSC {@link ManyToOneConcurrentLinkedQueue} for producers to enqueue messages onto
+ * 2. An internal intermediate {@link PrunableArrayQueue} into which the external queue is
+ * drained with exclusive access and from which actual deliveries happen
+ * The second, intermediate queue exists to enable efficient in-place pruning of expired messages.
+ *
+ * Said pruning will be attempted in several scenarios:
+ * 1. By callers invoking {@link #add(Message)} - if metadata indicates presence of expired messages
+ * in the queue, and if exclusive access can be immediately obtained (non-blockingly)
+ * 2. By {@link OutboundConnection}, periodically, while disconnected
+ * 3. As an optimisation, in an attempt to free up endpoint capacity on {@link OutboundConnection#enqueue(Message)}
+ * if current endpoint reserve was insufficient
+ */
+class OutboundMessageQueue
+{
+ private static final Logger logger = LoggerFactory.getLogger(OutboundMessageQueue.class);
+
+ interface MessageConsumer<Produces extends Throwable>
+ {
+ boolean accept(Message<?> message) throws Produces;
+ }
+
+ private final MonotonicClock clock;
+ private final MessageConsumer<RuntimeException> onExpired;
+
+ private final ManyToOneConcurrentLinkedQueue<Message<?>> externalQueue = new ManyToOneConcurrentLinkedQueue<>();
+ private final PrunableArrayQueue<Message<?>> internalQueue = new PrunableArrayQueue<>(256);
+
+ private volatile long earliestExpiresAt = Long.MAX_VALUE;
+ private volatile long nextExpirationDeadline = Long.MAX_VALUE;
+ private static final AtomicLongFieldUpdater<OutboundMessageQueue> earliestExpiresAtUpdater =
+ AtomicLongFieldUpdater.newUpdater(OutboundMessageQueue.class, "earliestExpiresAt");
+ private static final AtomicLongFieldUpdater<OutboundMessageQueue> nextExpirationDeadlineUpdater =
+ AtomicLongFieldUpdater.newUpdater(OutboundMessageQueue.class, "nextExpirationDeadline");
+
+ OutboundMessageQueue(MonotonicClock clock, MessageConsumer<RuntimeException> onExpired)
+ {
+ this.clock = clock;
+ this.onExpired = onExpired;
+ }
+
+ /**
+ * Add the provided message to the queue. Always succeeds.
+ */
+ void add(Message<?> m)
+ {
+ maybePruneExpired();
+ externalQueue.offer(m);
+ nextExpirationDeadlineUpdater.accumulateAndGet(this,
+ maybeUpdateEarliestExpiresAt(clock.now(), m.expiresAtNanos()),
+ Math::min);
+ }
+
+ /**
+ * Try to obtain the lock; if this fails, a callback will be registered to be invoked when
+ * the lock is relinquished.
+ *
+ * This callback will run WITHOUT ownership of the lock, so must re-obtain the lock.
+ *
+ * @return null if failed to obtain the lock
+ */
+ WithLock lockOrCallback(long nowNanos, Runnable callbackIfDeferred)
+ {
+ if (!lockOrCallback(callbackIfDeferred))
+ return null;
+
+ return new WithLock(nowNanos);
+ }
+
+ /**
+ * Try to obtain the lock. If successful, invoke the provided consumer immediately, otherwise
+ * register it to be invoked when the lock is relinquished.
+ */
+ void runEventually(Consumer<WithLock> runEventually)
+ {
+ try (WithLock withLock = lockOrCallback(clock.now(), () -> runEventually(runEventually)))
+ {
+ if (withLock != null)
+ runEventually.accept(withLock);
+ }
+ }
+
+ /**
+ * If succeeds to obtain the lock, polls the queue, otherwise registers the provided callback
+ * to be invoked when the lock is relinquished.
+ *
+ * May return null when the queue is non-empty - if the lock could not be acquired.
+ */
+ Message<?> tryPoll(long nowNanos, Runnable elseIfDeferred)
+ {
+ try (WithLock withLock = lockOrCallback(nowNanos, elseIfDeferred))
+ {
+ if (withLock == null)
+ return null;
+
+ return withLock.poll();
+ }
+ }
+
+ class WithLock implements AutoCloseable
+ {
+ private final long nowNanos;
+
+ private WithLock(long nowNanos)
+ {
+ this.nowNanos = nowNanos;
+ externalQueue.drain(internalQueue::offer);
+ }
+
+ Message<?> poll()
+ {
+ Message<?> m;
+ while (null != (m = internalQueue.poll()))
+ {
+ if (shouldSend(m, clock, nowNanos))
+ break;
+
+ onExpired.accept(m);
+ }
+
+ return m;
+ }
+
+ void removeHead(Message<?> expectHead)
+ {
+ assert expectHead == internalQueue.peek();
+ internalQueue.poll();
+ }
+
+ Message<?> peek()
+ {
+ Message<?> m;
+ while (null != (m = internalQueue.peek()))
+ {
+ if (shouldSend(m, clock, nowNanos))
+ break;
+
+ internalQueue.poll();
+ onExpired.accept(m);
+ }
+
+ return m;
+ }
+
+ void consume(Consumer<Message<?>> consumer)
+ {
+ Message<?> m;
+ while (null != (m = poll()))
+ consumer.accept(m);
+ }
+
+ @Override
+ public void close()
+ {
+ if (clock.isAfter(nowNanos, nextExpirationDeadline))
+ pruneInternalQueueWithLock(nowNanos);
+
+ unlock();
+ }
+ }
+
+ /**
+ * Call periodically if cannot expect to promptly invoke consume()
+ */
+ boolean maybePruneExpired()
+ {
+ return maybePruneExpired(clock.now());
+ }
+
+ private boolean maybePruneExpired(long nowNanos)
+ {
+ if (clock.isAfter(nowNanos, nextExpirationDeadline))
+ return tryRun(() -> pruneWithLock(nowNanos));
+
+ return false;
+ }
+
+ /**
+ * Update {@code earliestExpiresAt} with the given {@code candidateTime} if less than the current value OR
+ * if the current value is past the current {@code nowNanos} time: this last condition is needed to make sure we keep
+ * tracking the earliest expiry time even while we prune previous values, so that at the end of the pruning task,
+ * we can reconcile between the earliest expiry time recorded at pruning and the one recorded at insert time.
+ */
+ private long maybeUpdateEarliestExpiresAt(long nowNanos, long candidateTime)
+ {
+ return earliestExpiresAtUpdater.accumulateAndGet(this, candidateTime, (oldTime, newTime) -> {
+ if (clock.isAfter(nowNanos, oldTime))
+ return newTime;
+ else
+ return min(oldTime, newTime);
+ });
+ }
+
+ /**
+ * Update {@code nextExpirationDeadline} with the given {@code candidateDeadline} if less than the current
+ * deadline, unless the current deadline is passed in relation to {@code nowNanos}: this is needed
+ * to resolve a race where both {@link #add(org.apache.cassandra.net.Message) } and {@link #pruneInternalQueueWithLock(long) }
+ * try to update the expiration deadline.
+ */
+ private long maybeUpdateNextExpirationDeadline(long nowNanos, long candidateDeadline)
+ {
+ return nextExpirationDeadlineUpdater.accumulateAndGet(this, candidateDeadline, (oldDeadline, newDeadline) -> {
+ if (clock.isAfter(nowNanos, oldDeadline))
+ return newDeadline;
+ else
+ return min(oldDeadline, newDeadline);
+ });
+ }
+
+ /*
+ * Drain external queue into the internal one and prune the latter in-place.
+ */
+ private void pruneWithLock(long nowNanos)
+ {
+ externalQueue.drain(internalQueue::offer);
+ pruneInternalQueueWithLock(nowNanos);
+ }
+
+ /*
+ * Prune the internal queue in-place.
+ */
+ private void pruneInternalQueueWithLock(long nowNanos)
+ {
+ class Pruner implements PrunableArrayQueue.Pruner<Message<?>>
+ {
+ private long earliestExpiresAt = Long.MAX_VALUE;
+
+ public boolean shouldPrune(Message<?> message)
+ {
+ return !shouldSend(message, clock, nowNanos);
+ }
+
+ public void onPruned(Message<?> message)
+ {
+ onExpired.accept(message);
+ }
+
+ public void onKept(Message<?> message)
+ {
+ earliestExpiresAt = min(message.expiresAtNanos(), earliestExpiresAt);
+ }
+ }
+
+ Pruner pruner = new Pruner();
+ internalQueue.prune(pruner);
+
+ maybeUpdateNextExpirationDeadline(nowNanos, maybeUpdateEarliestExpiresAt(nowNanos, pruner.earliestExpiresAt));
+ }
+
+ @VisibleForTesting
+ long nextExpirationIn(long nowNanos, TimeUnit unit)
+ {
+ return unit.convert(nextExpirationDeadline - nowNanos, TimeUnit.NANOSECONDS);
+ }
+
+ private static class Locked implements Runnable
+ {
+ final Runnable run;
+ final Locked next;
+ private Locked(Runnable run, Locked next)
+ {
+ this.run = run;
+ this.next = next;
+ }
+
+ Locked andThen(Runnable next)
+ {
+ return new Locked(next, this);
+ }
+
+ public void run()
+ {
+ Locked cur = this;
+ while (cur != null)
+ {
+ try
+ {
+ cur.run.run();
+ }
+ catch (Throwable t)
+ {
+ logger.error("Unexpected error when executing deferred lock-intending functions", t);
+ }
+ cur = cur.next;
+ }
+ }
+ }
+
+ private static final Locked LOCKED = new Locked(() -> {}, null);
+
+ private volatile Locked locked = null;
+ private static final AtomicReferenceFieldUpdater<OutboundMessageQueue, Locked> lockedUpdater =
+ AtomicReferenceFieldUpdater.newUpdater(OutboundMessageQueue.class, Locked.class, "locked");
+
+ /**
+ * Run runOnceLocked either immediately in the calling thread if we can obtain the lock, or ask the lock's current
+ * owner attempt to run it when the lock is released. This may be passed between a sequence of owners, as the present
+ * owner releases the lock before trying to acquire it again and execute the task.
+ */
+ private void runEventually(Runnable runEventually)
+ {
+ if (!lockOrCallback(() -> runEventually(runEventually)))
+ return;
+
+ try
+ {
+ runEventually.run();
+ }
+ finally
+ {
+ unlock();
+ }
+ }
+
+ /**
+ * If we can immediately obtain the lock, execute runIfLocked and return true;
+ * otherwise do nothing and return false.
+ */
+ private boolean tryRun(Runnable runIfAvailable)
+ {
+ if (!tryLock())
+ return false;
+
+ try
+ {
+ runIfAvailable.run();
+ return true;
+ }
+ finally
+ {
+ unlock();
+ }
+ }
+
+ /**
+ * @return true iff the caller now owns the lock
+ */
+ private boolean tryLock()
+ {
+ return locked == null && lockedUpdater.compareAndSet(this, null, LOCKED);
+ }
+
+ /**
+ * Try to obtain the lock; if this fails, a callback will be registered to be invoked when the lock is relinquished.
+ * This callback will run WITHOUT ownership of the lock, so must re-obtain the lock.
+ *
+ * @return true iff the caller now owns the lock
+ */
+ private boolean lockOrCallback(Runnable callbackWhenAvailable)
+ {
+ if (callbackWhenAvailable == null)
+ return tryLock();
+
+ while (true)
+ {
+ Locked current = locked;
+ if (current == null && lockedUpdater.compareAndSet(this, null, LOCKED))
+ return true;
+ else if (current != null && lockedUpdater.compareAndSet(this, current, current.andThen(callbackWhenAvailable)))
+ return false;
+ }
+ }
+
+ private void unlock()
+ {
+ Locked locked = lockedUpdater.getAndSet(this, null);
+ locked.run();
+ }
+
+
+ /**
+ * While removal happens extremely infrequently, it seems possible for many to still interleave with a connection
+ * being closed, as experimentally we have encountered enough pending removes to overflow the Locked call stack
+ * (prior to making its evaluation iterative).
+ *
+ * While the stack can no longer be exhausted, this suggests a high potential cost for evaluating all removals,
+ * so to ensure system stability we aggregate all pending removes into a single shared object that evaluate
+ * together with only a single lock acquisition.
+ */
+ private volatile RemoveRunner removeRunner = null;
+ private static final AtomicReferenceFieldUpdater<OutboundMessageQueue, RemoveRunner> removeRunnerUpdater =
+ AtomicReferenceFieldUpdater.newUpdater(OutboundMessageQueue.class, RemoveRunner.class, "removeRunner");
+
+ static class Remove
+ {
+ final Message<?> message;
+ final Remove next;
+
+ Remove(Message<?> message, Remove next)
+ {
+ this.message = message;
+ this.next = next;
+ }
+ }
+
+ private class RemoveRunner extends AtomicReference<Remove> implements Runnable
+ {
+ final CountDownLatch done = new CountDownLatch(1);
+ final Set<Message<?>> removed = Collections.newSetFromMap(new IdentityHashMap<>());
+
+ RemoveRunner() { super(new Remove(null, null)); }
+
+ boolean undo(Message<?> message)
+ {
+ return null != updateAndGet(prev -> prev == null ? null : new Remove(message, prev));
+ }
+
+ public void run()
+ {
+ Set<Message<?>> remove = Collections.newSetFromMap(new IdentityHashMap<>());
+ removeRunner = null;
+ Remove undo = getAndSet(null);
+ while (undo.message != null)
+ {
+ remove.add(undo.message);
+ undo = undo.next;
+ }
+
+ class Remover implements PrunableArrayQueue.Pruner<Message<?>>
+ {
+ private long earliestExpiresAt = Long.MAX_VALUE;
+
+ @Override
+ public boolean shouldPrune(Message<?> message)
+ {
+ return remove.contains(message);
+ }
+
+ @Override
+ public void onPruned(Message<?> message)
+ {
+ removed.add(message);
+ }
+
+ @Override
+ public void onKept(Message<?> message)
+ {
+ earliestExpiresAt = min(message.expiresAtNanos(), earliestExpiresAt);
+ }
+ }
+
+ Remover remover = new Remover();
+ externalQueue.drain(internalQueue::offer);
+ internalQueue.prune(remover);
+
+ long nowNanos = clock.now();
+ maybeUpdateNextExpirationDeadline(nowNanos, maybeUpdateEarliestExpiresAt(nowNanos, remover.earliestExpiresAt));
+
+ done.countDown();
+ }
+ }
+
+ /**
+ * Remove the provided Message from the queue, if present.
+ *
+ * WARNING: This is a blocking call.
+ */
+ boolean remove(Message<?> remove)
+ {
+ if (remove == null)
+ throw new NullPointerException();
+
+ RemoveRunner runner;
+ while (true)
+ {
+ runner = removeRunner;
+ if (runner != null && runner.undo(remove))
+ break;
+
+ if (runner == null && removeRunnerUpdater.compareAndSet(this, null, runner = new RemoveRunner()))
+ {
+ runner.undo(remove);
+ runEventually(runner);
+ break;
+ }
+ }
+
+ //noinspection UnstableApiUsage
+ Uninterruptibles.awaitUninterruptibly(runner.done);
+ return runner.removed.contains(remove);
+ }
+
+ private static boolean shouldSend(Message<?> m, MonotonicClock clock, long nowNanos)
+ {
+ return !clock.isAfter(nowNanos, m.expiresAtNanos());
+ }
+}
diff --git a/src/java/org/apache/cassandra/net/OutboundSink.java b/src/java/org/apache/cassandra/net/OutboundSink.java
new file mode 100644
index 0000000..34c72db
--- /dev/null
+++ b/src/java/org/apache/cassandra/net/OutboundSink.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.net;
+
+import java.util.concurrent.atomic.AtomicReferenceFieldUpdater;
+import java.util.function.BiPredicate;
+
+import org.apache.cassandra.locator.InetAddressAndPort;
+
+/**
+ * A message sink that all outbound messages go through.
+ *
+ * Default sink {@link Sink} used by {@link MessagingService} is {@link MessagingService#doSend(Message, InetAddressAndPort, ConnectionType)}, which proceeds to
+ * send messages over the network, but it can be overridden to filter out certain messages, record the fact
+ * of attempted delivery, or delay they delivery.
+ *
+ * This facility is most useful for test code.
+ */
+public class OutboundSink
+{
+ public interface Sink
+ {
+ void accept(Message<?> message, InetAddressAndPort to, ConnectionType connectionType);
+ }
+
+ private static class Filtered implements Sink
+ {
+ final BiPredicate<Message<?>, InetAddressAndPort> condition;
+ final Sink next;
+
+ private Filtered(BiPredicate<Message<?>, InetAddressAndPort> condition, Sink next)
+ {
+ this.condition = condition;
+ this.next = next;
+ }
+
+ public void accept(Message<?> message, InetAddressAndPort to, ConnectionType connectionType)
+ {
+ if (condition.test(message, to))
+ next.accept(message, to, connectionType);
+ }
+ }
+
+ private volatile Sink sink;
+ private static final AtomicReferenceFieldUpdater<OutboundSink, Sink> sinkUpdater
+ = AtomicReferenceFieldUpdater.newUpdater(OutboundSink.class, Sink.class, "sink");
+
+ OutboundSink(Sink sink)
+ {
+ this.sink = sink;
+ }
+
+ public void accept(Message<?> message, InetAddressAndPort to, ConnectionType connectionType)
+ {
+ sink.accept(message, to, connectionType);
+ }
+
+ public void add(BiPredicate<Message<?>, InetAddressAndPort> allow)
+ {
+ sinkUpdater.updateAndGet(this, sink -> new Filtered(allow, sink));
+ }
+
+ public void remove(BiPredicate<Message<?>, InetAddressAndPort> allow)
+ {
+ sinkUpdater.updateAndGet(this, sink -> without(sink, allow));
+ }
+
+ public void clear()
+ {
+ sinkUpdater.updateAndGet(this, OutboundSink::clear);
+ }
+
+ private static Sink clear(Sink sink)
+ {
+ while (sink instanceof OutboundSink.Filtered)
+ sink = ((OutboundSink.Filtered) sink).next;
+ return sink;
+ }
+
+ private static Sink without(Sink sink, BiPredicate<Message<?>, InetAddressAndPort> condition)
+ {
+ if (!(sink instanceof Filtered))
+ return sink;
+
+ Filtered filtered = (Filtered) sink;
+ Sink next = without(filtered.next, condition);
+ return condition.equals(filtered.condition) ? next
+ : next == filtered.next
+ ? sink
+ : new Filtered(filtered.condition, next);
+ }
+
+}
diff --git a/src/java/org/apache/cassandra/net/OutboundTcpConnection.java b/src/java/org/apache/cassandra/net/OutboundTcpConnection.java
deleted file mode 100644
index 4ae62c1..0000000
--- a/src/java/org/apache/cassandra/net/OutboundTcpConnection.java
+++ /dev/null
@@ -1,687 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.net;
-
-import java.io.DataInputStream;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.net.InetAddress;
-import java.net.Socket;
-import java.net.SocketException;
-import java.nio.ByteBuffer;
-import java.nio.channels.Channels;
-import java.nio.channels.WritableByteChannel;
-import java.util.*;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.zip.Checksum;
-
-import javax.net.ssl.SSLHandshakeException;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import io.netty.util.concurrent.FastThreadLocalThread;
-import net.jpountz.lz4.LZ4BlockOutputStream;
-import net.jpountz.lz4.LZ4Compressor;
-import net.jpountz.lz4.LZ4Factory;
-import net.jpountz.xxhash.XXHashFactory;
-
-import org.apache.cassandra.concurrent.NamedThreadFactory;
-import org.apache.cassandra.io.util.DataOutputStreamPlus;
-import org.apache.cassandra.io.util.BufferedDataOutputStreamPlus;
-import org.apache.cassandra.io.util.WrappedDataOutputStreamPlus;
-import org.apache.cassandra.tracing.TraceState;
-import org.apache.cassandra.tracing.Tracing;
-import org.apache.cassandra.utils.CoalescingStrategies;
-import org.apache.cassandra.utils.CoalescingStrategies.Coalescable;
-import org.apache.cassandra.utils.CoalescingStrategies.CoalescingStrategy;
-import org.apache.cassandra.utils.FBUtilities;
-import org.apache.cassandra.utils.JVMStabilityInspector;
-import org.apache.cassandra.utils.NanoTimeToCurrentTimeMillis;
-import org.apache.cassandra.utils.UUIDGen;
-import org.xerial.snappy.SnappyOutputStream;
-import org.apache.cassandra.config.Config;
-import org.apache.cassandra.config.DatabaseDescriptor;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.util.concurrent.Uninterruptibles;
-
-public class OutboundTcpConnection extends FastThreadLocalThread
-{
- private static final Logger logger = LoggerFactory.getLogger(OutboundTcpConnection.class);
-
- private static final String PREFIX = Config.PROPERTY_PREFIX;
-
- /*
- * Enabled/disable TCP_NODELAY for intradc connections. Defaults to enabled.
- */
- private static final String INTRADC_TCP_NODELAY_PROPERTY = PREFIX + "otc_intradc_tcp_nodelay";
- private static final boolean INTRADC_TCP_NODELAY = Boolean.parseBoolean(System.getProperty(INTRADC_TCP_NODELAY_PROPERTY, "true"));
-
- /*
- * Size of buffer in output stream
- */
- private static final String BUFFER_SIZE_PROPERTY = PREFIX + "otc_buffer_size";
- private static final int BUFFER_SIZE = Integer.getInteger(BUFFER_SIZE_PROPERTY, 1024 * 64);
-
- public static final int MAX_COALESCED_MESSAGES = 128;
-
- private static CoalescingStrategy newCoalescingStrategy(String displayName)
- {
- return CoalescingStrategies.newCoalescingStrategy(DatabaseDescriptor.getOtcCoalescingStrategy(),
- DatabaseDescriptor.getOtcCoalescingWindow(),
- logger,
- displayName);
- }
-
- static
- {
- String strategy = DatabaseDescriptor.getOtcCoalescingStrategy();
- switch (strategy)
- {
- case "TIMEHORIZON":
- break;
- case "MOVINGAVERAGE":
- case "FIXED":
- case "DISABLED":
- logger.info("OutboundTcpConnection using coalescing strategy {}", strategy);
- break;
- default:
- //Check that it can be loaded
- newCoalescingStrategy("dummy");
- }
-
- int coalescingWindow = DatabaseDescriptor.getOtcCoalescingWindow();
- if (coalescingWindow != Config.otc_coalescing_window_us_default)
- logger.info("OutboundTcpConnection coalescing window set to {}μs", coalescingWindow);
-
- if (coalescingWindow < 0)
- throw new ExceptionInInitializerError(
- "Value provided for coalescing window must be greater than 0: " + coalescingWindow);
-
- int otc_backlog_expiration_interval_in_ms = DatabaseDescriptor.getOtcBacklogExpirationInterval();
- if (otc_backlog_expiration_interval_in_ms != Config.otc_backlog_expiration_interval_ms_default)
- logger.info("OutboundTcpConnection backlog expiration interval set to to {}ms", otc_backlog_expiration_interval_in_ms);
- }
-
- private static final MessageOut<?> CLOSE_SENTINEL = new MessageOut(MessagingService.Verb.INTERNAL_RESPONSE);
- private volatile boolean isStopped = false;
-
- private static final int OPEN_RETRY_DELAY = 100; // ms between retries
- public static final int WAIT_FOR_VERSION_MAX_TIME = 5000;
- private static final int NO_VERSION = Integer.MIN_VALUE;
-
- static final int LZ4_HASH_SEED = 0x9747b28c;
-
- private final BlockingQueue<QueuedMessage> backlog = new LinkedBlockingQueue<>();
- private static final String BACKLOG_PURGE_SIZE_PROPERTY = PREFIX + "otc_backlog_purge_size";
- @VisibleForTesting
- static final int BACKLOG_PURGE_SIZE = Integer.getInteger(BACKLOG_PURGE_SIZE_PROPERTY, 1024);
- private final AtomicBoolean backlogExpirationActive = new AtomicBoolean(false);
- private volatile long backlogNextExpirationTime;
-
- private final OutboundTcpConnectionPool poolReference;
-
- private final CoalescingStrategy cs;
- private DataOutputStreamPlus out;
- private Socket socket;
- private volatile long completed;
- private final AtomicLong dropped = new AtomicLong();
- private volatile int currentMsgBufferCount = 0;
- private volatile int targetVersion;
-
- public OutboundTcpConnection(OutboundTcpConnectionPool pool, String name)
- {
- super("MessagingService-Outgoing-" + pool.endPoint() + "-" + name);
- this.poolReference = pool;
- cs = newCoalescingStrategy(pool.endPoint().getHostAddress());
-
- // We want to use the most precise version we know because while there is version detection on connect(),
- // the target version might be accessed by the pool (in getConnection()) before we actually connect (as we
- // connect when the first message is submitted). Note however that the only case where we'll connect
- // without knowing the true version of a node is if that node is a seed (otherwise, we can't know a node
- // unless it has been gossiped to us or it has connected to us and in both case this sets the version) and
- // in that case we won't rely on that targetVersion before we're actually connected and so the version
- // detection in connect() will do its job.
- targetVersion = MessagingService.instance().getVersion(pool.endPoint());
- }
-
- private static boolean isLocalDC(InetAddress targetHost)
- {
- String remoteDC = DatabaseDescriptor.getEndpointSnitch().getDatacenter(targetHost);
- String localDC = DatabaseDescriptor.getEndpointSnitch().getDatacenter(FBUtilities.getBroadcastAddress());
- return remoteDC.equals(localDC);
- }
-
- public void enqueue(MessageOut<?> message, int id)
- {
- long nanoTime = System.nanoTime();
- expireMessages(nanoTime);
- try
- {
- backlog.put(new QueuedMessage(message, id, nanoTime));
- }
- catch (InterruptedException e)
- {
- throw new AssertionError(e);
- }
- }
-
- /**
- * This is a helper method for unit testing. Disclaimer: Do not use this method outside unit tests, as
- * this method is iterating the queue which can be an expensive operation (CPU time, queue locking).
- *
- * @return true, if the queue contains at least one expired element
- */
- @VisibleForTesting // (otherwise = VisibleForTesting.NONE)
- boolean backlogContainsExpiredMessages(long nowNanos)
- {
- return backlog.stream().anyMatch(entry -> entry.isTimedOut(nowNanos));
- }
-
- void closeSocket(boolean destroyThread)
- {
- logger.debug("Enqueuing socket close for {}", poolReference.endPoint());
- isStopped = destroyThread; // Exit loop to stop the thread
- backlog.clear();
- // in the "destroyThread = true" case, enqueuing the sentinel is important mostly to unblock the backlog.take()
- // (via the CoalescingStrategy) in case there's a data race between this method enqueuing the sentinel
- // and run() clearing the backlog on connection failure.
- enqueue(CLOSE_SENTINEL, -1);
- }
-
- void softCloseSocket()
- {
- enqueue(CLOSE_SENTINEL, -1);
- }
-
- public int getTargetVersion()
- {
- return targetVersion;
- }
-
- public void run()
- {
- final int drainedMessageSize = MAX_COALESCED_MESSAGES;
- // keeping list (batch) size small for now; that way we don't have an unbounded array (that we never resize)
- final List<QueuedMessage> drainedMessages = new ArrayList<>(drainedMessageSize);
-
- outer:
- while (!isStopped)
- {
- try
- {
- cs.coalesce(backlog, drainedMessages, drainedMessageSize);
- }
- catch (InterruptedException e)
- {
- throw new AssertionError(e);
- }
-
- int count = currentMsgBufferCount = drainedMessages.size();
-
- //The timestamp of the first message has already been provided to the coalescing strategy
- //so skip logging it.
- inner:
- for (QueuedMessage qm : drainedMessages)
- {
- try
- {
- MessageOut<?> m = qm.message;
- if (m == CLOSE_SENTINEL)
- {
- disconnect();
- if (isStopped)
- break outer;
- continue;
- }
-
- if (qm.isTimedOut(System.nanoTime()))
- dropped.incrementAndGet();
- else if (socket != null || connect())
- writeConnected(qm, count == 1 && backlog.isEmpty());
- else
- {
- // Not connected! Clear out the queue, else gossip messages back up. Update dropped
- // statistics accordingly. Hint: The statistics may be slightly too low, if messages
- // are added between the calls of backlog.size() and backlog.clear()
- dropped.addAndGet(backlog.size());
- backlog.clear();
- currentMsgBufferCount = 0;
- break inner;
- }
- }
- catch (Exception e)
- {
- JVMStabilityInspector.inspectThrowable(e);
- // really shouldn't get here, as exception handling in writeConnected() is reasonably robust
- // but we want to catch anything bad we don't drop the messages in the current batch
- logger.error("error processing a message intended for {}", poolReference.endPoint(), e);
- }
- currentMsgBufferCount = --count;
- }
- // Update dropped statistics by the number of unprocessed drainedMessages
- dropped.addAndGet(currentMsgBufferCount);
- drainedMessages.clear();
- }
- }
-
- public int getPendingMessages()
- {
- return backlog.size() + currentMsgBufferCount;
- }
-
- public long getCompletedMesssages()
- {
- return completed;
- }
-
- public long getDroppedMessages()
- {
- return dropped.get();
- }
-
- private boolean shouldCompressConnection()
- {
- // assumes version >= 1.2
- return DatabaseDescriptor.internodeCompression() == Config.InternodeCompression.all
- || (DatabaseDescriptor.internodeCompression() == Config.InternodeCompression.dc && !isLocalDC(poolReference.endPoint()));
- }
-
- private void writeConnected(QueuedMessage qm, boolean flush)
- {
- try
- {
- byte[] sessionBytes = qm.message.parameters.get(Tracing.TRACE_HEADER);
- if (sessionBytes != null)
- {
- UUID sessionId = UUIDGen.getUUID(ByteBuffer.wrap(sessionBytes));
- TraceState state = Tracing.instance.get(sessionId);
- String message = String.format("Sending %s message to %s", qm.message.verb, poolReference.endPoint());
- // session may have already finished; see CASSANDRA-5668
- if (state == null)
- {
- byte[] traceTypeBytes = qm.message.parameters.get(Tracing.TRACE_TYPE);
- Tracing.TraceType traceType = traceTypeBytes == null ? Tracing.TraceType.QUERY : Tracing.TraceType.deserialize(traceTypeBytes[0]);
- Tracing.instance.trace(ByteBuffer.wrap(sessionBytes), message, traceType.getTTL());
- }
- else
- {
- state.trace(message);
- if (qm.message.verb == MessagingService.Verb.REQUEST_RESPONSE)
- Tracing.instance.doneWithNonLocalSession(state);
- }
- }
-
- long timestampMillis = NanoTimeToCurrentTimeMillis.convert(qm.timestampNanos);
- writeInternal(qm.message, qm.id, timestampMillis);
-
- completed++;
- if (flush)
- out.flush();
- }
- catch (Throwable e)
- {
- JVMStabilityInspector.inspectThrowable(e);
- disconnect();
- if (e instanceof IOException || e.getCause() instanceof IOException)
- {
- logger.debug("Error writing to {}", poolReference.endPoint(), e);
-
- // If we haven't retried this message yet, put it back on the queue to retry after re-connecting.
- // See CASSANDRA-5393 and CASSANDRA-12192.
- if (qm.shouldRetry())
- {
- try
- {
- backlog.put(new RetriedQueuedMessage(qm));
- }
- catch (InterruptedException e1)
- {
- throw new AssertionError(e1);
- }
- }
- }
- else
- {
- // Non IO exceptions are likely a programming error so let's not silence them
- logger.error("error writing to {}", poolReference.endPoint(), e);
- }
- }
- }
-
- private void writeInternal(MessageOut<?> message, int id, long timestamp) throws IOException
- {
- out.writeInt(MessagingService.PROTOCOL_MAGIC);
-
- if (targetVersion < MessagingService.VERSION_20)
- out.writeUTF(String.valueOf(id));
- else
- out.writeInt(id);
-
- // int cast cuts off the high-order half of the timestamp, which we can assume remains
- // the same between now and when the recipient reconstructs it.
- out.writeInt((int) timestamp);
- message.serialize(out, targetVersion);
- }
-
- private static void writeHeader(DataOutput out, int version, boolean compressionEnabled) throws IOException
- {
- // 2 bits: unused. used to be "serializer type," which was always Binary
- // 1 bit: compression
- // 1 bit: streaming mode
- // 3 bits: unused
- // 8 bits: version
- // 15 bits: unused
- int header = 0;
- if (compressionEnabled)
- header |= 4;
- header |= (version << 8);
- out.writeInt(header);
- }
-
- private void disconnect()
- {
- if (socket != null)
- {
- try
- {
- socket.close();
- logger.debug("Socket to {} closed", poolReference.endPoint());
- }
- catch (IOException e)
- {
- logger.debug("Exception closing connection to {}", poolReference.endPoint(), e);
- }
- out = null;
- socket = null;
- }
- }
-
- @SuppressWarnings("resource")
- private boolean connect()
- {
- logger.debug("Attempting to connect to {}", poolReference.endPoint());
-
- long start = System.nanoTime();
- long timeout = TimeUnit.MILLISECONDS.toNanos(DatabaseDescriptor.getRpcTimeout());
- while (System.nanoTime() - start < timeout && !isStopped)
- {
- targetVersion = MessagingService.instance().getVersion(poolReference.endPoint());
- try
- {
- socket = poolReference.newSocket();
- socket.setKeepAlive(true);
- if (isLocalDC(poolReference.endPoint()))
- {
- socket.setTcpNoDelay(INTRADC_TCP_NODELAY);
- }
- else
- {
- socket.setTcpNoDelay(DatabaseDescriptor.getInterDCTcpNoDelay());
- }
- if (DatabaseDescriptor.getInternodeSendBufferSize() > 0)
- {
- try
- {
- socket.setSendBufferSize(DatabaseDescriptor.getInternodeSendBufferSize());
- }
- catch (SocketException se)
- {
- logger.warn("Failed to set send buffer size on internode socket.", se);
- }
- }
-
- // SocketChannel may be null when using SSL
- WritableByteChannel ch = socket.getChannel();
- out = new BufferedDataOutputStreamPlus(ch != null ? ch : Channels.newChannel(socket.getOutputStream()), BUFFER_SIZE);
-
- out.writeInt(MessagingService.PROTOCOL_MAGIC);
- writeHeader(out, targetVersion, shouldCompressConnection());
- out.flush();
-
- DataInputStream in = new DataInputStream(socket.getInputStream());
- int maxTargetVersion = handshakeVersion(in);
- if (maxTargetVersion == NO_VERSION)
- {
- // no version is returned, so disconnect an try again: we will either get
- // a different target version (targetVersion < MessagingService.VERSION_12)
- // or if the same version the handshake will finally succeed
- logger.trace("Target max version is {}; no version information yet, will retry", maxTargetVersion);
- disconnect();
- continue;
- }
- else
- {
- MessagingService.instance().setVersion(poolReference.endPoint(), maxTargetVersion);
- }
-
- if (targetVersion > maxTargetVersion)
- {
- logger.trace("Target max version is {}; will reconnect with that version", maxTargetVersion);
- try
- {
- if (DatabaseDescriptor.getSeeds().contains(poolReference.endPoint()))
- logger.warn("Seed gossip version is {}; will not connect with that version", maxTargetVersion);
- }
- catch (Throwable e)
- {
- // If invalid yaml has been added to the config since startup, getSeeds() will throw an AssertionError
- // Additionally, third party seed providers may throw exceptions if network is flakey
- // Regardless of what's thrown, we must catch it, disconnect, and try again
- JVMStabilityInspector.inspectThrowable(e);
- logger.warn("Configuration error prevented outbound connection: {}", e.getLocalizedMessage());
- }
- finally
- {
- disconnect();
- return false;
- }
- }
-
- if (targetVersion < maxTargetVersion && targetVersion < MessagingService.current_version)
- {
- logger.trace("Detected higher max version {} (using {}); will reconnect when queued messages are done",
- maxTargetVersion, targetVersion);
- softCloseSocket();
- }
-
- out.writeInt(MessagingService.current_version);
- CompactEndpointSerializationHelper.serialize(FBUtilities.getBroadcastAddress(), out);
- if (shouldCompressConnection())
- {
- out.flush();
- logger.trace("Upgrading OutputStream to {} to be compressed", poolReference.endPoint());
- if (targetVersion < MessagingService.VERSION_21)
- {
- // Snappy is buffered, so no need for extra buffering output stream
- out = new WrappedDataOutputStreamPlus(new SnappyOutputStream(socket.getOutputStream()));
- }
- else
- {
- // TODO: custom LZ4 OS that supports BB write methods
- LZ4Compressor compressor = LZ4Factory.fastestInstance().fastCompressor();
- Checksum checksum = XXHashFactory.fastestInstance().newStreamingHash32(LZ4_HASH_SEED).asChecksum();
- out = new WrappedDataOutputStreamPlus(new LZ4BlockOutputStream(socket.getOutputStream(),
- 1 << 14, // 16k block size
- compressor,
- checksum,
- true)); // no async flushing
- }
- }
- logger.debug("Done connecting to {}", poolReference.endPoint());
- return true;
- }
- catch (SSLHandshakeException e)
- {
- logger.error("SSL handshake error for outbound connection to " + socket, e);
- disconnect();
- // SSL errors won't be recoverable within timeout period so we'll just abort
- return false;
- }
- catch (IOException e)
- {
- disconnect();
- logger.debug("Unable to connect to {}", poolReference.endPoint(), e);
- Uninterruptibles.sleepUninterruptibly(OPEN_RETRY_DELAY, TimeUnit.MILLISECONDS);
- }
- }
- return false;
- }
-
- private int handshakeVersion(final DataInputStream inputStream)
- {
- final AtomicInteger version = new AtomicInteger(NO_VERSION);
- final CountDownLatch versionLatch = new CountDownLatch(1);
- NamedThreadFactory.createThread(() ->
- {
- try
- {
- logger.info("Handshaking version with {}", poolReference.endPoint());
- version.set(inputStream.readInt());
- }
- catch (IOException ex)
- {
- final String msg = "Cannot handshake version with " + poolReference.endPoint();
- if (logger.isTraceEnabled())
- logger.trace(msg, ex);
- else
- logger.info(msg);
- }
- finally
- {
- //unblock the waiting thread on either success or fail
- versionLatch.countDown();
- }
- }, "HANDSHAKE-" + poolReference.endPoint()).start();
-
- try
- {
- versionLatch.await(WAIT_FOR_VERSION_MAX_TIME, TimeUnit.MILLISECONDS);
- }
- catch (InterruptedException ex)
- {
- throw new AssertionError(ex);
- }
- return version.get();
- }
-
- /**
- * Expire elements from the queue if the queue is pretty full and expiration is not already in progress.
- * This method will only remove droppable expired entries. If no such element exists, nothing is removed from the queue.
- *
- * @param timestampNanos The current time as from System.nanoTime()
- */
- @VisibleForTesting
- void expireMessages(long timestampNanos)
- {
- if (backlog.size() <= BACKLOG_PURGE_SIZE)
- return; // Plenty of space
-
- if (backlogNextExpirationTime - timestampNanos > 0)
- return; // Expiration is not due.
-
- /**
- * Expiration is an expensive process. Iterating the queue locks the queue for both writes and
- * reads during iter.next() and iter.remove(). Thus letting only a single Thread do expiration.
- */
- if (backlogExpirationActive.compareAndSet(false, true))
- {
- try
- {
- Iterator<QueuedMessage> iter = backlog.iterator();
- while (iter.hasNext())
- {
- QueuedMessage qm = iter.next();
- if (!qm.droppable)
- continue;
- if (!qm.isTimedOut(timestampNanos))
- continue;
- iter.remove();
- dropped.incrementAndGet();
- }
-
- if (logger.isTraceEnabled())
- {
- long duration = TimeUnit.NANOSECONDS.toMicros(System.nanoTime() - timestampNanos);
- logger.trace("Expiration of {} took {}μs", getName(), duration);
- }
- }
- finally
- {
- long backlogExpirationIntervalNanos = TimeUnit.MILLISECONDS.toNanos(DatabaseDescriptor.getOtcBacklogExpirationInterval());
- backlogNextExpirationTime = timestampNanos + backlogExpirationIntervalNanos;
- backlogExpirationActive.set(false);
- }
- }
- }
-
- /** messages that have not been retried yet */
- private static class QueuedMessage implements Coalescable
- {
- final MessageOut<?> message;
- final int id;
- final long timestampNanos;
- final boolean droppable;
-
- QueuedMessage(MessageOut<?> message, int id, long timestampNanos)
- {
- this.message = message;
- this.id = id;
- this.timestampNanos = timestampNanos;
- this.droppable = MessagingService.DROPPABLE_VERBS.contains(message.verb);
- }
-
- /** don't drop a non-droppable message just because it's timestamp is expired */
- boolean isTimedOut(long nowNanos)
- {
- long messageTimeoutNanos = TimeUnit.MILLISECONDS.toNanos(message.getTimeout());
- return droppable && nowNanos - timestampNanos > messageTimeoutNanos;
- }
-
- boolean shouldRetry()
- {
- // retry all messages once
- return true;
- }
-
- public long timestampNanos()
- {
- return timestampNanos;
- }
- }
-
- private static class RetriedQueuedMessage extends QueuedMessage
- {
- RetriedQueuedMessage(QueuedMessage msg)
- {
- super(msg.message, msg.id, msg.timestampNanos);
- }
-
- boolean shouldRetry()
- {
- return false;
- }
- }
-}
diff --git a/src/java/org/apache/cassandra/net/OutboundTcpConnectionPool.java b/src/java/org/apache/cassandra/net/OutboundTcpConnectionPool.java
deleted file mode 100644
index 9f9ffee..0000000
--- a/src/java/org/apache/cassandra/net/OutboundTcpConnectionPool.java
+++ /dev/null
@@ -1,224 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.net;
-
-import java.io.IOException;
-import java.net.InetAddress;
-import java.net.InetSocketAddress;
-import java.net.Socket;
-import java.nio.channels.SocketChannel;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.cassandra.concurrent.Stage;
-import org.apache.cassandra.config.Config;
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.db.SystemKeyspace;
-import org.apache.cassandra.locator.IEndpointSnitch;
-import org.apache.cassandra.metrics.ConnectionMetrics;
-import org.apache.cassandra.security.SSLFactory;
-import org.apache.cassandra.utils.FBUtilities;
-
-public class OutboundTcpConnectionPool
-{
- public static final long LARGE_MESSAGE_THRESHOLD =
- Long.getLong(Config.PROPERTY_PREFIX + "otcp_large_message_threshold", 1024 * 64);
-
- // pointer for the real Address.
- private final InetAddress id;
- private final CountDownLatch started;
- public final OutboundTcpConnection smallMessages;
- public final OutboundTcpConnection largeMessages;
- public final OutboundTcpConnection gossipMessages;
-
- // pointer to the reset Address.
- private InetAddress resetEndpoint;
- private ConnectionMetrics metrics;
-
- // back-pressure state linked to this connection:
- private final BackPressureState backPressureState;
-
- OutboundTcpConnectionPool(InetAddress remoteEp, BackPressureState backPressureState)
- {
- id = remoteEp;
- resetEndpoint = SystemKeyspace.getPreferredIP(remoteEp);
- started = new CountDownLatch(1);
-
- smallMessages = new OutboundTcpConnection(this, "Small");
- largeMessages = new OutboundTcpConnection(this, "Large");
- gossipMessages = new OutboundTcpConnection(this, "Gossip");
-
- this.backPressureState = backPressureState;
- }
-
- /**
- * returns the appropriate connection based on message type.
- * returns null if a connection could not be established.
- */
- OutboundTcpConnection getConnection(MessageOut msg)
- {
- if (Stage.GOSSIP == msg.getStage())
- return gossipMessages;
- return msg.payloadSize(smallMessages.getTargetVersion()) > LARGE_MESSAGE_THRESHOLD
- ? largeMessages
- : smallMessages;
- }
-
- public BackPressureState getBackPressureState()
- {
- return backPressureState;
- }
-
- void reset()
- {
- for (OutboundTcpConnection conn : new OutboundTcpConnection[] { smallMessages, largeMessages, gossipMessages })
- conn.closeSocket(false);
- }
-
- public void resetToNewerVersion(int version)
- {
- for (OutboundTcpConnection conn : new OutboundTcpConnection[] { smallMessages, largeMessages, gossipMessages })
- {
- if (version > conn.getTargetVersion())
- conn.softCloseSocket();
- }
- }
-
- /**
- * reconnect to @param remoteEP (after the current message backlog is exhausted).
- * Used by Ec2MultiRegionSnitch to force nodes in the same region to communicate over their private IPs.
- * @param remoteEP
- */
- public void reset(InetAddress remoteEP)
- {
- SystemKeyspace.updatePreferredIP(id, remoteEP);
- resetEndpoint = remoteEP;
- for (OutboundTcpConnection conn : new OutboundTcpConnection[] { smallMessages, largeMessages, gossipMessages })
- conn.softCloseSocket();
-
- // release previous metrics and create new one with reset address
- metrics.release();
- metrics = new ConnectionMetrics(resetEndpoint, this);
- }
-
- public long getTimeouts()
- {
- return metrics.timeouts.getCount();
- }
-
-
- public void incrementTimeout()
- {
- metrics.timeouts.mark();
- }
-
- public Socket newSocket() throws IOException
- {
- return newSocket(endPoint());
- }
-
- @SuppressWarnings("resource") // Closing the socket will close the underlying channel.
- public static Socket newSocket(InetAddress endpoint) throws IOException
- {
- // zero means 'bind on any available port.'
- if (isEncryptedChannel(endpoint))
- {
- return SSLFactory.getSocket(DatabaseDescriptor.getServerEncryptionOptions(), endpoint, DatabaseDescriptor.getSSLStoragePort());
- }
- else
- {
- SocketChannel channel = SocketChannel.open();
- channel.connect(new InetSocketAddress(endpoint, DatabaseDescriptor.getStoragePort()));
- return channel.socket();
- }
- }
-
- public InetAddress endPoint()
- {
- if (id.equals(FBUtilities.getBroadcastAddress()))
- return FBUtilities.getLocalAddress();
- return resetEndpoint;
- }
-
- public static boolean isEncryptedChannel(InetAddress address)
- {
- IEndpointSnitch snitch = DatabaseDescriptor.getEndpointSnitch();
- switch (DatabaseDescriptor.getServerEncryptionOptions().internode_encryption)
- {
- case none:
- return false; // if nothing needs to be encrypted then return immediately.
- case all:
- break;
- case dc:
- if (snitch.getDatacenter(address).equals(snitch.getDatacenter(FBUtilities.getBroadcastAddress())))
- return false;
- break;
- case rack:
- // for rack then check if the DC's are the same.
- if (snitch.getRack(address).equals(snitch.getRack(FBUtilities.getBroadcastAddress()))
- && snitch.getDatacenter(address).equals(snitch.getDatacenter(FBUtilities.getBroadcastAddress())))
- return false;
- break;
- }
- return true;
- }
-
- public void start()
- {
- smallMessages.start();
- largeMessages.start();
- gossipMessages.start();
-
- metrics = new ConnectionMetrics(id, this);
-
- started.countDown();
- }
-
- public void waitForStarted()
- {
- if (started.getCount() == 0)
- return;
-
- boolean error = false;
- try
- {
- if (!started.await(1, TimeUnit.MINUTES))
- error = true;
- }
- catch (InterruptedException e)
- {
- Thread.currentThread().interrupt();
- error = true;
- }
- if (error)
- throw new IllegalStateException(String.format("Connections to %s are not started!", id.getHostAddress()));
- }
-
- public void close()
- {
- // these null guards are simply for tests
- if (largeMessages != null)
- largeMessages.closeSocket(true);
- if (smallMessages != null)
- smallMessages.closeSocket(true);
- if (gossipMessages != null)
- gossipMessages.closeSocket(true);
-
- metrics.release();
- }
-}
diff --git a/src/java/org/apache/cassandra/net/ParamType.java b/src/java/org/apache/cassandra/net/ParamType.java
new file mode 100644
index 0000000..6572348
--- /dev/null
+++ b/src/java/org/apache/cassandra/net/ParamType.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.net;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import javax.annotation.Nullable;
+
+import org.apache.cassandra.exceptions.RequestFailureReason;
+import org.apache.cassandra.io.IVersionedSerializer;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.tracing.Tracing;
+import org.apache.cassandra.utils.UUIDSerializer;
+
+import static java.lang.Math.max;
+import static org.apache.cassandra.locator.InetAddressAndPort.Serializer.inetAddressAndPortSerializer;
+
+/**
+ * Type names and serializers for various parameters that can be put in {@link Message} params map.
+ *
+ * It should be safe to add new params without bumping messaging version - {@link Message} serializer
+ * will skip over any params it doesn't recognise.
+ *
+ * Please don't add boolean params here. Extend and use {@link MessageFlag} instead.
+ */
+public enum ParamType
+{
+ FORWARD_TO (0, "FORWARD_TO", ForwardingInfo.serializer),
+ RESPOND_TO (1, "FORWARD_FROM", inetAddressAndPortSerializer),
+
+ @Deprecated
+ FAILURE_RESPONSE (2, "FAIL", LegacyFlag.serializer),
+ @Deprecated
+ FAILURE_REASON (3, "FAIL_REASON", RequestFailureReason.serializer),
+ @Deprecated
+ FAILURE_CALLBACK (4, "CAL_BAC", LegacyFlag.serializer),
+
+ TRACE_SESSION (5, "TraceSession", UUIDSerializer.serializer),
+ TRACE_TYPE (6, "TraceType", Tracing.traceTypeSerializer),
+
+ @Deprecated
+ TRACK_REPAIRED_DATA (7, "TrackRepaired", LegacyFlag.serializer);
+
+ final int id;
+ @Deprecated final String legacyAlias; // pre-4.0 we used to serialize entire param name string
+ final IVersionedSerializer serializer;
+
+ ParamType(int id, String legacyAlias, IVersionedSerializer serializer)
+ {
+ if (id < 0)
+ throw new IllegalArgumentException("ParamType id must be non-negative");
+
+ this.id = id;
+ this.legacyAlias = legacyAlias;
+ this.serializer = serializer;
+ }
+
+ private static final ParamType[] idToTypeMap;
+ private static final Map<String, ParamType> aliasToTypeMap;
+
+ static
+ {
+ ParamType[] types = values();
+
+ int max = -1;
+ for (ParamType t : types)
+ max = max(t.id, max);
+
+ ParamType[] idMap = new ParamType[max + 1];
+ Map<String, ParamType> aliasMap = new HashMap<>();
+
+ for (ParamType type : types)
+ {
+ if (idMap[type.id] != null)
+ throw new RuntimeException("Two ParamType-s that map to the same id: " + type.id);
+ idMap[type.id] = type;
+
+ if (aliasMap.put(type.legacyAlias, type) != null)
+ throw new RuntimeException("Two ParamType-s that map to the same legacy alias: " + type.legacyAlias);
+ }
+
+ idToTypeMap = idMap;
+ aliasToTypeMap = aliasMap;
+ }
+
+ @Nullable
+ static ParamType lookUpById(int id)
+ {
+ if (id < 0)
+ throw new IllegalArgumentException("ParamType id must be non-negative (got " + id + ')');
+
+ return id < idToTypeMap.length ? idToTypeMap[id] : null;
+ }
+
+ @Nullable
+ static ParamType lookUpByAlias(String alias)
+ {
+ return aliasToTypeMap.get(alias);
+ }
+}
diff --git a/src/java/org/apache/cassandra/net/PingMessage.java b/src/java/org/apache/cassandra/net/PingMessage.java
deleted file mode 100644
index 8eaf23e..0000000
--- a/src/java/org/apache/cassandra/net/PingMessage.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.net;
-
-import java.io.IOException;
-
-import org.apache.cassandra.io.IVersionedSerializer;
-import org.apache.cassandra.io.util.DataInputPlus;
-import org.apache.cassandra.io.util.DataOutputPlus;
-
-/**
- * A backport of the version from 4.0, intentionnaly added as versions 4.0 or greater aren't guaranteed
- * to know the c* versions they communicate with before they connect.
- *
- * It is intentional that no {@link IVerbHandler} is provided as we do not want process the message;
- * the intent is to not break the stream by leaving it in an unclean state, with unconsumed bytes.
- * We do, however, assign a {@link org.apache.cassandra.concurrent.StageManager} stage
- * to maintain proper message flow.
- * See CASSANDRA-13393 for a discussion.
- */
-public class PingMessage
-{
- public static IVersionedSerializer<PingMessage> serializer = new PingMessageSerializer();
-
- public static class PingMessageSerializer implements IVersionedSerializer<PingMessage>
- {
- public void serialize(PingMessage t, DataOutputPlus out, int version)
- {
- throw new UnsupportedOperationException();
- }
-
- public PingMessage deserialize(DataInputPlus in, int version) throws IOException
- {
- // throw away the one byte of the payload
- in.readByte();
- return new PingMessage();
- }
-
- public long serializedSize(PingMessage t, int version)
- {
- return 1;
- }
- }
-}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/net/PingRequest.java b/src/java/org/apache/cassandra/net/PingRequest.java
new file mode 100644
index 0000000..c02bd80
--- /dev/null
+++ b/src/java/org/apache/cassandra/net/PingRequest.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.net;
+
+import java.io.IOException;
+
+import org.apache.cassandra.io.IVersionedSerializer;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+
+import static org.apache.cassandra.net.ConnectionType.URGENT_MESSAGES;
+import static org.apache.cassandra.net.ConnectionType.SMALL_MESSAGES;
+import static org.apache.cassandra.net.ConnectionType.LARGE_MESSAGES;
+
+/**
+ * Indicates to the recipient which {@link ConnectionType} should be used for the response.
+ */
+class PingRequest
+{
+ static final PingRequest forUrgent = new PingRequest(URGENT_MESSAGES);
+ static final PingRequest forSmall = new PingRequest(SMALL_MESSAGES);
+ static final PingRequest forLarge = new PingRequest(LARGE_MESSAGES);
+
+ final ConnectionType connectionType;
+
+ private PingRequest(ConnectionType connectionType)
+ {
+ this.connectionType = connectionType;
+ }
+
+ static IVersionedSerializer<PingRequest> serializer = new IVersionedSerializer<PingRequest>()
+ {
+ public void serialize(PingRequest t, DataOutputPlus out, int version) throws IOException
+ {
+ out.writeByte(t.connectionType.id);
+ }
+
+ public PingRequest deserialize(DataInputPlus in, int version) throws IOException
+ {
+ ConnectionType type = ConnectionType.fromId(in.readByte());
+
+ switch (type)
+ {
+ case URGENT_MESSAGES: return forUrgent;
+ case SMALL_MESSAGES: return forSmall;
+ case LARGE_MESSAGES: return forLarge;
+ }
+
+ throw new IllegalStateException();
+ }
+
+ public long serializedSize(PingRequest t, int version)
+ {
+ return 1;
+ }
+ };
+}
diff --git a/src/java/org/apache/cassandra/net/PingVerbHandler.java b/src/java/org/apache/cassandra/net/PingVerbHandler.java
new file mode 100644
index 0000000..a70cddc
--- /dev/null
+++ b/src/java/org/apache/cassandra/net/PingVerbHandler.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.net;
+
+class PingVerbHandler implements IVerbHandler<PingRequest>
+{
+ static final PingVerbHandler instance = new PingVerbHandler();
+
+ @Override
+ public void doVerb(Message<PingRequest> message)
+ {
+ MessagingService.instance().send(message.emptyResponse(), message.from(), message.payload.connectionType);
+ }
+}
diff --git a/src/java/org/apache/cassandra/net/PrunableArrayQueue.java b/src/java/org/apache/cassandra/net/PrunableArrayQueue.java
new file mode 100644
index 0000000..1fca43c
--- /dev/null
+++ b/src/java/org/apache/cassandra/net/PrunableArrayQueue.java
@@ -0,0 +1,172 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.net;
+
+import java.util.function.Predicate;
+
+/**
+ * A growing array-based queue that allows efficient bulk in-place removal.
+ *
+ * Can think of this queue as if it were an {@link java.util.ArrayDeque} with {@link #prune(Pruner)} method - an efficient
+ * way to prune the queue in-place that is more expressive, and faster than {@link java.util.ArrayDeque#removeIf(Predicate)}.
+ *
+ * The latter has to perform O(n*n) shifts, whereas {@link #prune(Pruner)} only needs O(n) shifts at worst.
+ */
+final class PrunableArrayQueue<E>
+{
+ public interface Pruner<E>
+ {
+ /**
+ * @return whether the element should be pruned
+ * if {@code true}, the element will be removed from the queue, and {@link #onPruned(Object)} will be invoked,
+ * if {@code false}, the element will be kept, and {@link #onKept(Object)} will be invoked.
+ */
+ boolean shouldPrune(E e);
+
+ void onPruned(E e);
+ void onKept(E e);
+ }
+
+ private int capacity;
+ private E[] buffer;
+
+ /*
+ * mask = capacity - 1;
+ * since capacity is a power of 2, value % capacity == value & (capacity - 1) == value & mask
+ */
+ private int mask;
+
+ private int head = 0;
+ private int tail = 0;
+
+ @SuppressWarnings("unchecked")
+ PrunableArrayQueue(int requestedCapacity)
+ {
+ capacity = Math.max(8, findNextPositivePowerOfTwo(requestedCapacity));
+ mask = capacity - 1;
+ buffer = (E[]) new Object[capacity];
+ }
+
+ @SuppressWarnings("UnusedReturnValue")
+ boolean offer(E e)
+ {
+ buffer[tail] = e;
+ if ((tail = (tail + 1) & mask) == head)
+ doubleCapacity();
+ return true;
+ }
+
+ E peek()
+ {
+ return buffer[head];
+ }
+
+ E poll()
+ {
+ E result = buffer[head];
+ if (null == result)
+ return null;
+
+ buffer[head] = null;
+ head = (head + 1) & mask;
+
+ return result;
+ }
+
+ int size()
+ {
+ return (tail - head) & mask;
+ }
+
+ boolean isEmpty()
+ {
+ return head == tail;
+ }
+
+ /**
+ * Prunes the queue using the specified {@link Pruner}
+ *
+ * @return count of removed elements.
+ */
+ int prune(Pruner<E> pruner)
+ {
+ E e;
+ int removed = 0;
+
+ try
+ {
+ int size = size();
+ for (int i = 0; i < size; i++)
+ {
+ /*
+ * We start at the tail and work backwards to minimise the number of copies
+ * as we expect to primarily prune from the front.
+ */
+ int k = (tail - 1 - i) & mask;
+ e = buffer[k];
+
+ if (pruner.shouldPrune(e))
+ {
+ buffer[k] = null;
+ removed++;
+ pruner.onPruned(e);
+ }
+ else
+ {
+ if (removed > 0)
+ {
+ buffer[(k + removed) & mask] = e;
+ buffer[k] = null;
+ }
+ pruner.onKept(e);
+ }
+ }
+ }
+ finally
+ {
+ head = (head + removed) & mask;
+ }
+
+ return removed;
+ }
+
+ @SuppressWarnings("unchecked")
+ private void doubleCapacity()
+ {
+ assert head == tail;
+
+ int newCapacity = capacity << 1;
+ E[] newBuffer = (E[]) new Object[newCapacity];
+
+ int headPortionLen = capacity - head;
+ System.arraycopy(buffer, head, newBuffer, 0, headPortionLen);
+ System.arraycopy(buffer, 0, newBuffer, headPortionLen, tail);
+
+ head = 0;
+ tail = capacity;
+
+ capacity = newCapacity;
+ mask = newCapacity - 1;
+ buffer = newBuffer;
+ }
+
+ private static int findNextPositivePowerOfTwo(int value)
+ {
+ return 1 << (32 - Integer.numberOfLeadingZeros(value - 1));
+ }
+}
diff --git a/src/java/org/apache/cassandra/net/RateBasedBackPressure.java b/src/java/org/apache/cassandra/net/RateBasedBackPressure.java
index 565bf4c..02d8cce 100644
--- a/src/java/org/apache/cassandra/net/RateBasedBackPressure.java
+++ b/src/java/org/apache/cassandra/net/RateBasedBackPressure.java
@@ -17,16 +17,15 @@
*/
package org.apache.cassandra.net;
-import java.net.InetAddress;
import java.util.Map;
import java.util.Set;
-import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
+import com.github.benmanes.caffeine.cache.Cache;
+import com.github.benmanes.caffeine.cache.Caffeine;
import com.google.common.annotations.VisibleForTesting;
-import com.google.common.cache.Cache;
-import com.google.common.cache.CacheBuilder;
import com.google.common.collect.ImmutableMap;
+import com.google.common.util.concurrent.MoreExecutors;
import com.google.common.util.concurrent.RateLimiter;
import org.slf4j.Logger;
@@ -34,11 +33,14 @@
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.config.ParameterizedClass;
+import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.utils.NoSpamLogger;
import org.apache.cassandra.utils.SystemTimeSource;
import org.apache.cassandra.utils.TimeSource;
import org.apache.cassandra.utils.concurrent.IntervalLock;
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+
/**
* Back-pressure algorithm based on rate limiting according to the ratio between incoming and outgoing rates, computed
* over a sliding time window with size equal to write RPC timeout.
@@ -63,7 +65,10 @@
protected final long windowSize;
private final Cache<Set<RateBasedBackPressureState>, IntervalRateLimiter> rateLimiters =
- CacheBuilder.newBuilder().expireAfterAccess(1, TimeUnit.HOURS).build();
+ Caffeine.newBuilder()
+ .expireAfterAccess(1, TimeUnit.HOURS)
+ .executor(MoreExecutors.directExecutor())
+ .build();
enum Flow
{
@@ -81,7 +86,7 @@
public RateBasedBackPressure(Map<String, Object> args)
{
- this(args, new SystemTimeSource(), DatabaseDescriptor.getWriteRpcTimeout());
+ this(args, new SystemTimeSource(), DatabaseDescriptor.getWriteRpcTimeout(MILLISECONDS));
}
@VisibleForTesting
@@ -220,47 +225,40 @@
// Now find the rate limiter corresponding to the replica group represented by these back-pressure states:
if (!states.isEmpty())
{
- try
- {
- // Get the rate limiter:
- IntervalRateLimiter rateLimiter = rateLimiters.get(states, () -> new IntervalRateLimiter(timeSource));
+ // Get the rate limiter:
+ IntervalRateLimiter rateLimiter = rateLimiters.get(states, key -> new IntervalRateLimiter(timeSource));
- // If the back-pressure was updated and we acquire the interval lock for the rate limiter of this group:
- if (isUpdated && rateLimiter.tryIntervalLock(windowSize))
+ // If the back-pressure was updated and we acquire the interval lock for the rate limiter of this group:
+ if (isUpdated && rateLimiter.tryIntervalLock(windowSize))
+ {
+ try
{
- try
- {
- // Update the rate limiter value based on the configured flow:
- if (flow.equals(Flow.FAST))
- rateLimiter.limiter = currentMax;
- else
- rateLimiter.limiter = currentMin;
+ // Update the rate limiter value based on the configured flow:
+ if (flow.equals(Flow.FAST))
+ rateLimiter.limiter = currentMax;
+ else
+ rateLimiter.limiter = currentMin;
- tenSecsNoSpamLogger.info("{} currently applied for remote replicas: {}", rateLimiter.limiter, states);
- }
- finally
- {
- rateLimiter.releaseIntervalLock();
- }
+ tenSecsNoSpamLogger.info("{} currently applied for remote replicas: {}", rateLimiter.limiter, states);
}
- // Assigning a single rate limiter per replica group once per window size allows the back-pressure rate
- // limiting to be stable within the group itself.
+ finally
+ {
+ rateLimiter.releaseIntervalLock();
+ }
+ }
+ // Assigning a single rate limiter per replica group once per window size allows the back-pressure rate
+ // limiting to be stable within the group itself.
- // Finally apply the rate limit with a max pause time equal to the provided timeout minus the
- // response time computed from the incoming rate, to reduce the number of client timeouts by taking into
- // account how long it could take to process responses after back-pressure:
- long responseTimeInNanos = (long) (TimeUnit.NANOSECONDS.convert(1, TimeUnit.SECONDS) / minIncomingRate);
- doRateLimit(rateLimiter.limiter, Math.max(0, TimeUnit.NANOSECONDS.convert(timeout, unit) - responseTimeInNanos));
- }
- catch (ExecutionException ex)
- {
- throw new IllegalStateException(ex);
- }
+ // Finally apply the rate limit with a max pause time equal to the provided timeout minus the
+ // response time computed from the incoming rate, to reduce the number of client timeouts by taking into
+ // account how long it could take to process responses after back-pressure:
+ long responseTimeInNanos = (long) (TimeUnit.NANOSECONDS.convert(1, TimeUnit.SECONDS) / minIncomingRate);
+ doRateLimit(rateLimiter.limiter, Math.max(0, TimeUnit.NANOSECONDS.convert(timeout, unit) - responseTimeInNanos));
}
}
@Override
- public RateBasedBackPressureState newState(InetAddress host)
+ public RateBasedBackPressureState newState(InetAddressAndPort host)
{
return new RateBasedBackPressureState(host, timeSource, windowSize);
}
diff --git a/src/java/org/apache/cassandra/net/RateBasedBackPressureState.java b/src/java/org/apache/cassandra/net/RateBasedBackPressureState.java
index c19f277..a150874 100644
--- a/src/java/org/apache/cassandra/net/RateBasedBackPressureState.java
+++ b/src/java/org/apache/cassandra/net/RateBasedBackPressureState.java
@@ -17,19 +17,18 @@
*/
package org.apache.cassandra.net;
-import java.net.InetAddress;
import java.util.concurrent.TimeUnit;
import com.google.common.util.concurrent.RateLimiter;
+import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.utils.SlidingTimeRate;
import org.apache.cassandra.utils.TimeSource;
import org.apache.cassandra.utils.concurrent.IntervalLock;
/**
* The rate-based back-pressure state, tracked per replica host.
- * <br/><br/>
- *
+ * <p>
* This back-pressure state is made up of the following attributes:
* <ul>
* <li>windowSize: the length of the back-pressure window in milliseconds.</li>
@@ -37,34 +36,32 @@
* <li>outgoingRate: the rate of back-pressure supporting outgoing messages.</li>
* <li>rateLimiter: the rate limiter to eventually apply to outgoing messages.</li>
* </ul>
- * <br/>
* The incomingRate and outgoingRate are updated together when a response is received to guarantee consistency between
* the two.
- * <br/>
+ * <p>
* It also provides methods to exclusively lock/release back-pressure windows at given intervals;
* this allows to apply back-pressure even under concurrent modifications. Please also note a read lock is acquired
* during response processing so that no concurrent rate updates can screw rate computations.
+ * </p>
*/
class RateBasedBackPressureState extends IntervalLock implements BackPressureState
{
- private final InetAddress host;
- private final long windowSize;
+ private final InetAddressAndPort host;
final SlidingTimeRate incomingRate;
final SlidingTimeRate outgoingRate;
final RateLimiter rateLimiter;
- RateBasedBackPressureState(InetAddress host, TimeSource timeSource, long windowSize)
+ RateBasedBackPressureState(InetAddressAndPort host, TimeSource timeSource, long windowSize)
{
super(timeSource);
this.host = host;
- this.windowSize = windowSize;
- this.incomingRate = new SlidingTimeRate(timeSource, this.windowSize, this.windowSize / 10, TimeUnit.MILLISECONDS);
- this.outgoingRate = new SlidingTimeRate(timeSource, this.windowSize, this.windowSize / 10, TimeUnit.MILLISECONDS);
+ this.incomingRate = new SlidingTimeRate(timeSource, windowSize, windowSize / 10, TimeUnit.MILLISECONDS);
+ this.outgoingRate = new SlidingTimeRate(timeSource, windowSize, windowSize / 10, TimeUnit.MILLISECONDS);
this.rateLimiter = RateLimiter.create(Double.POSITIVE_INFINITY);
}
@Override
- public void onMessageSent(MessageOut<?> message) {}
+ public void onMessageSent(Message<?> message) {}
@Override
public void onResponseReceived()
@@ -102,7 +99,7 @@
}
@Override
- public InetAddress getHost()
+ public InetAddressAndPort getHost()
{
return host;
}
diff --git a/src/java/org/apache/cassandra/net/RequestCallback.java b/src/java/org/apache/cassandra/net/RequestCallback.java
new file mode 100644
index 0000000..9ed3a4b
--- /dev/null
+++ b/src/java/org/apache/cassandra/net/RequestCallback.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.net;
+
+import org.apache.cassandra.exceptions.RequestFailureReason;
+import org.apache.cassandra.locator.InetAddressAndPort;
+
+/**
+ * implementors of {@link RequestCallback} need to make sure that any public methods
+ * are threadsafe with respect to {@link #onResponse} being called from the message
+ * service. In particular, if any shared state is referenced, making
+ * response alone synchronized will not suffice.
+ */
+public interface RequestCallback<T>
+{
+ /**
+ * @param msg response received.
+ */
+ void onResponse(Message<T> msg);
+
+ /**
+ * Called when there is an exception on the remote node or timeout happens
+ */
+ default void onFailure(InetAddressAndPort from, RequestFailureReason failureReason)
+ {
+ }
+
+ /**
+ * @return true if the callback should be invoked on failure
+ */
+ default boolean invokeOnFailure()
+ {
+ return false;
+ }
+
+ /**
+ * @return true if this callback is on the read path and its latency should be
+ * given as input to the dynamic snitch.
+ */
+ default boolean trackLatencyForSnitch()
+ {
+ return false;
+ }
+
+ default boolean supportsBackPressure()
+ {
+ return false;
+ }
+}
diff --git a/src/java/org/apache/cassandra/net/RequestCallbacks.java b/src/java/org/apache/cassandra/net/RequestCallbacks.java
new file mode 100644
index 0000000..d5424ed
--- /dev/null
+++ b/src/java/org/apache/cassandra/net/RequestCallbacks.java
@@ -0,0 +1,381 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.net;
+
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeoutException;
+
+import javax.annotation.Nullable;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.concurrent.DebuggableScheduledThreadPoolExecutor;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.ConsistencyLevel;
+import org.apache.cassandra.db.Mutation;
+import org.apache.cassandra.exceptions.RequestFailureReason;
+import org.apache.cassandra.io.IVersionedAsymmetricSerializer;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.Replica;
+import org.apache.cassandra.metrics.InternodeOutboundMetrics;
+import org.apache.cassandra.service.AbstractWriteResponseHandler;
+import org.apache.cassandra.service.StorageProxy;
+import org.apache.cassandra.service.paxos.Commit;
+import org.apache.cassandra.utils.FBUtilities;
+
+import static java.lang.String.format;
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+import static java.util.concurrent.TimeUnit.NANOSECONDS;
+import static org.apache.cassandra.concurrent.Stage.INTERNAL_RESPONSE;
+import static org.apache.cassandra.utils.MonotonicClock.preciseTime;
+
+/**
+ * An expiring map of request callbacks.
+ *
+ * Used to match response (id, peer) pairs to corresponding {@link RequestCallback}s, or, if said responses
+ * don't arrive in a timely manner (within verb's timeout), to expire the callbacks.
+ *
+ * Since we reuse the same request id for multiple messages now, the map is keyed by (id, peer) tuples
+ * rather than just id as it used to before 4.0.
+ */
+public class RequestCallbacks implements OutboundMessageCallbacks
+{
+ private static final Logger logger = LoggerFactory.getLogger(RequestCallbacks.class);
+
+ private final MessagingService messagingService;
+ private final ScheduledExecutorService executor = new DebuggableScheduledThreadPoolExecutor("Callback-Map-Reaper");
+ private final ConcurrentMap<CallbackKey, CallbackInfo> callbacks = new ConcurrentHashMap<>();
+
+ RequestCallbacks(MessagingService messagingService)
+ {
+ this.messagingService = messagingService;
+
+ long expirationInterval = DatabaseDescriptor.getMinRpcTimeout(NANOSECONDS) / 2;
+ executor.scheduleWithFixedDelay(this::expire, expirationInterval, expirationInterval, NANOSECONDS);
+ }
+
+ /**
+ * @return the registered {@link CallbackInfo} for this id and peer, or {@code null} if unset or expired.
+ */
+ @Nullable
+ CallbackInfo get(long id, InetAddressAndPort peer)
+ {
+ return callbacks.get(key(id, peer));
+ }
+
+ /**
+ * Remove and return the {@link CallbackInfo} associated with given id and peer, if known.
+ */
+ @Nullable
+ CallbackInfo remove(long id, InetAddressAndPort peer)
+ {
+ return callbacks.remove(key(id, peer));
+ }
+
+ /**
+ * Register the provided {@link RequestCallback}, inferring expiry and id from the provided {@link Message}.
+ */
+ void addWithExpiration(RequestCallback cb, Message message, InetAddressAndPort to)
+ {
+ // mutations need to call the overload with a ConsistencyLevel
+ assert message.verb() != Verb.MUTATION_REQ && message.verb() != Verb.COUNTER_MUTATION_REQ && message.verb() != Verb.PAXOS_COMMIT_REQ;
+ CallbackInfo previous = callbacks.put(key(message.id(), to), new CallbackInfo(message, to, cb));
+ assert previous == null : format("Callback already exists for id %d/%s! (%s)", message.id(), to, previous);
+ }
+
+ // FIXME: shouldn't need a special overload for writes; hinting should be part of AbstractWriteResponseHandler
+ public void addWithExpiration(AbstractWriteResponseHandler<?> cb,
+ Message<?> message,
+ Replica to,
+ ConsistencyLevel consistencyLevel,
+ boolean allowHints)
+ {
+ assert message.verb() == Verb.MUTATION_REQ || message.verb() == Verb.COUNTER_MUTATION_REQ || message.verb() == Verb.PAXOS_COMMIT_REQ;
+ CallbackInfo previous = callbacks.put(key(message.id(), to.endpoint()), new WriteCallbackInfo(message, to, cb, consistencyLevel, allowHints));
+ assert previous == null : format("Callback already exists for id %d/%s! (%s)", message.id(), to.endpoint(), previous);
+ }
+
+ <T> IVersionedAsymmetricSerializer<?, T> responseSerializer(long id, InetAddressAndPort peer)
+ {
+ CallbackInfo info = get(id, peer);
+ return info == null ? null : info.responseVerb.serializer();
+ }
+
+ @VisibleForTesting
+ public void removeAndRespond(long id, InetAddressAndPort peer, Message message)
+ {
+ CallbackInfo ci = remove(id, peer);
+ if (null != ci) ci.callback.onResponse(message);
+ }
+
+ private void removeAndExpire(long id, InetAddressAndPort peer)
+ {
+ CallbackInfo ci = remove(id, peer);
+ if (null != ci) onExpired(ci);
+ }
+
+ private void expire()
+ {
+ long start = preciseTime.now();
+ int n = 0;
+ for (Map.Entry<CallbackKey, CallbackInfo> entry : callbacks.entrySet())
+ {
+ if (entry.getValue().isReadyToDieAt(start))
+ {
+ if (callbacks.remove(entry.getKey(), entry.getValue()))
+ {
+ n++;
+ onExpired(entry.getValue());
+ }
+ }
+ }
+ logger.trace("Expired {} entries", n);
+ }
+
+ private void forceExpire()
+ {
+ for (Map.Entry<CallbackKey, CallbackInfo> entry : callbacks.entrySet())
+ if (callbacks.remove(entry.getKey(), entry.getValue()))
+ onExpired(entry.getValue());
+ }
+
+ private void onExpired(CallbackInfo info)
+ {
+ messagingService.latencySubscribers.maybeAdd(info.callback, info.peer, info.timeout(), NANOSECONDS);
+
+ InternodeOutboundMetrics.totalExpiredCallbacks.mark();
+ messagingService.markExpiredCallback(info.peer);
+
+ if (info.callback.supportsBackPressure())
+ messagingService.updateBackPressureOnReceive(info.peer, info.callback, true);
+
+ if (info.invokeOnFailure())
+ INTERNAL_RESPONSE.submit(() -> info.callback.onFailure(info.peer, RequestFailureReason.TIMEOUT));
+
+ // FIXME: this has never belonged here, should be part of onFailure() in AbstractWriteResponseHandler
+ if (info.shouldHint())
+ {
+ WriteCallbackInfo writeCallbackInfo = ((WriteCallbackInfo) info);
+ Mutation mutation = writeCallbackInfo.mutation();
+ StorageProxy.submitHint(mutation, writeCallbackInfo.getReplica(), null);
+ }
+ }
+
+ void shutdownNow(boolean expireCallbacks)
+ {
+ executor.shutdownNow();
+ if (expireCallbacks)
+ forceExpire();
+ }
+
+ void shutdownGracefully()
+ {
+ expire();
+ if (!callbacks.isEmpty())
+ executor.schedule(this::shutdownGracefully, 100L, MILLISECONDS);
+ else
+ executor.shutdownNow();
+ }
+
+ void awaitTerminationUntil(long deadlineNanos) throws TimeoutException, InterruptedException
+ {
+ if (!executor.isTerminated())
+ {
+ long wait = deadlineNanos - System.nanoTime();
+ if (wait <= 0 || !executor.awaitTermination(wait, NANOSECONDS))
+ throw new TimeoutException();
+ }
+ }
+
+ @VisibleForTesting
+ public void unsafeClear()
+ {
+ callbacks.clear();
+ }
+
+ private static CallbackKey key(long id, InetAddressAndPort peer)
+ {
+ return new CallbackKey(id, peer);
+ }
+
+ private static class CallbackKey
+ {
+ final long id;
+ final InetAddressAndPort peer;
+
+ CallbackKey(long id, InetAddressAndPort peer)
+ {
+ this.id = id;
+ this.peer = peer;
+ }
+
+ @Override
+ public boolean equals(Object o)
+ {
+ if (!(o instanceof CallbackKey))
+ return false;
+ CallbackKey that = (CallbackKey) o;
+ return this.id == that.id && this.peer.equals(that.peer);
+ }
+
+ @Override
+ public int hashCode()
+ {
+ return Long.hashCode(id) + 31 * peer.hashCode();
+ }
+
+ @Override
+ public String toString()
+ {
+ return "{id:" + id + ", peer:" + peer + '}';
+ }
+ }
+
+ static class CallbackInfo
+ {
+ final long createdAtNanos;
+ final long expiresAtNanos;
+
+ final InetAddressAndPort peer;
+ final RequestCallback callback;
+
+ @Deprecated // for 3.0 compatibility purposes only
+ public final Verb responseVerb;
+
+ private CallbackInfo(Message message, InetAddressAndPort peer, RequestCallback callback)
+ {
+ this.createdAtNanos = message.createdAtNanos();
+ this.expiresAtNanos = message.expiresAtNanos();
+ this.peer = peer;
+ this.callback = callback;
+ this.responseVerb = message.verb().responseVerb;
+ }
+
+ public long timeout()
+ {
+ return expiresAtNanos - createdAtNanos;
+ }
+
+ boolean isReadyToDieAt(long atNano)
+ {
+ return atNano > expiresAtNanos;
+ }
+
+ boolean shouldHint()
+ {
+ return false;
+ }
+
+ boolean invokeOnFailure()
+ {
+ return callback.invokeOnFailure();
+ }
+
+ public String toString()
+ {
+ return "{peer:" + peer + ", callback:" + callback + ", invokeOnFailure:" + invokeOnFailure() + '}';
+ }
+ }
+
+ // FIXME: shouldn't need a specialized container for write callbacks; hinting should be part of
+ // AbstractWriteResponseHandler implementation.
+ static class WriteCallbackInfo extends CallbackInfo
+ {
+ // either a Mutation, or a Paxos Commit (MessageOut)
+ private final Object mutation;
+ private final Replica replica;
+
+ @VisibleForTesting
+ WriteCallbackInfo(Message message, Replica replica, RequestCallback<?> callback, ConsistencyLevel consistencyLevel, boolean allowHints)
+ {
+ super(message, replica.endpoint(), callback);
+ this.mutation = shouldHint(allowHints, message, consistencyLevel) ? message.payload : null;
+ //Local writes shouldn't go through messaging service (https://issues.apache.org/jira/browse/CASSANDRA-10477)
+ //noinspection AssertWithSideEffects
+ assert !peer.equals(FBUtilities.getBroadcastAddressAndPort());
+ this.replica = replica;
+ }
+
+ public boolean shouldHint()
+ {
+ return mutation != null && StorageProxy.shouldHint(replica);
+ }
+
+ public Replica getReplica()
+ {
+ return replica;
+ }
+
+ public Mutation mutation()
+ {
+ return getMutation(mutation);
+ }
+
+ private static Mutation getMutation(Object object)
+ {
+ assert object instanceof Commit || object instanceof Mutation : object;
+ return object instanceof Commit ? ((Commit) object).makeMutation()
+ : (Mutation) object;
+ }
+
+ private static boolean shouldHint(boolean allowHints, Message sentMessage, ConsistencyLevel consistencyLevel)
+ {
+ return allowHints && sentMessage.verb() != Verb.COUNTER_MUTATION_REQ && consistencyLevel != ConsistencyLevel.ANY;
+ }
+ }
+
+ @Override
+ public void onOverloaded(Message<?> message, InetAddressAndPort peer)
+ {
+ removeAndExpire(message, peer);
+ }
+
+ @Override
+ public void onExpired(Message<?> message, InetAddressAndPort peer)
+ {
+ removeAndExpire(message, peer);
+ }
+
+ @Override
+ public void onFailedSerialize(Message<?> message, InetAddressAndPort peer, int messagingVersion, int bytesWrittenToNetwork, Throwable failure)
+ {
+ removeAndExpire(message, peer);
+ }
+
+ @Override
+ public void onDiscardOnClose(Message<?> message, InetAddressAndPort peer)
+ {
+ removeAndExpire(message, peer);
+ }
+
+ private void removeAndExpire(Message message, InetAddressAndPort peer)
+ {
+ removeAndExpire(message.id(), peer);
+
+ /* in case of a write sent to a different DC, also expire all forwarding targets */
+ ForwardingInfo forwardTo = message.forwardTo();
+ if (null != forwardTo)
+ forwardTo.forEach(this::removeAndExpire);
+ }
+}
diff --git a/src/java/org/apache/cassandra/net/ResourceLimits.java b/src/java/org/apache/cassandra/net/ResourceLimits.java
index f8d24d7..4c97c2a 100644
--- a/src/java/org/apache/cassandra/net/ResourceLimits.java
+++ b/src/java/org/apache/cassandra/net/ResourceLimits.java
@@ -36,6 +36,20 @@
long limit();
/**
+ * Sets the total amount of permits represented by this {@link Limit} - the capacity
+ *
+ * If the old limit has been reached and the new limit is large enough to allow for more
+ * permits to be aqcuired, subsequent calls to {@link #allocate(long)} or {@link #tryAllocate(long)}
+ * will succeed.
+ *
+ * If the new limit is lower than the current amount of allocated permits then subsequent calls
+ * to {@link #allocate(long)} or {@link #tryAllocate(long)} will block or fail respectively.
+ *
+ * @return the old limit
+ */
+ long setLimit(long newLimit);
+
+ /**
* @return remaining, unallocated permit amount
*/
long remaining();
@@ -73,7 +87,9 @@
*/
public static class Concurrent implements Limit
{
- private final long limit;
+ private volatile long limit;
+ private static final AtomicLongFieldUpdater<Concurrent> limitUpdater =
+ AtomicLongFieldUpdater.newUpdater(Concurrent.class, "limit");
private volatile long using;
private static final AtomicLongFieldUpdater<Concurrent> usingUpdater =
@@ -89,6 +105,16 @@
return limit;
}
+ public long setLimit(long newLimit)
+ {
+ long oldLimit;
+ do {
+ oldLimit = limit;
+ } while (!limitUpdater.compareAndSet(this, oldLimit, newLimit));
+
+ return oldLimit;
+ }
+
public long remaining()
{
return limit - using;
@@ -139,7 +165,7 @@
*/
static class Basic implements Limit
{
- private final long limit;
+ private long limit;
private long using;
Basic(long limit)
@@ -152,6 +178,14 @@
return limit;
}
+ public long setLimit(long newLimit)
+ {
+ long oldLimit = limit;
+ limit = newLimit;
+
+ return oldLimit;
+ }
+
public long remaining()
{
return limit - using;
diff --git a/src/java/org/apache/cassandra/net/ResponseVerbHandler.java b/src/java/org/apache/cassandra/net/ResponseVerbHandler.java
index fe22e42..e5779ab 100644
--- a/src/java/org/apache/cassandra/net/ResponseVerbHandler.java
+++ b/src/java/org/apache/cassandra/net/ResponseVerbHandler.java
@@ -17,45 +17,50 @@
*/
package org.apache.cassandra.net;
-import java.util.concurrent.TimeUnit;
-
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.cassandra.exceptions.RequestFailureReason;
import org.apache.cassandra.tracing.Tracing;
-public class ResponseVerbHandler implements IVerbHandler
-{
- private static final Logger logger = LoggerFactory.getLogger( ResponseVerbHandler.class );
+import static java.util.concurrent.TimeUnit.NANOSECONDS;
+import static org.apache.cassandra.utils.MonotonicClock.approxTime;
- public void doVerb(MessageIn message, int id)
+class ResponseVerbHandler implements IVerbHandler
+{
+ public static final ResponseVerbHandler instance = new ResponseVerbHandler();
+
+ private static final Logger logger = LoggerFactory.getLogger(ResponseVerbHandler.class);
+
+ @Override
+ public void doVerb(Message message)
{
- long latency = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - MessagingService.instance().getRegisteredCallbackAge(id));
- CallbackInfo callbackInfo = MessagingService.instance().removeRegisteredCallback(id);
+ RequestCallbacks.CallbackInfo callbackInfo = MessagingService.instance().callbacks.remove(message.id(), message.from());
if (callbackInfo == null)
{
String msg = "Callback already removed for {} (from {})";
- logger.trace(msg, id, message.from);
- Tracing.trace(msg, id, message.from);
+ logger.trace(msg, message.id(), message.from());
+ Tracing.trace(msg, message.id(), message.from());
return;
}
- Tracing.trace("Processing response from {}", message.from);
- IAsyncCallback cb = callbackInfo.callback;
+ long latencyNanos = approxTime.now() - callbackInfo.createdAtNanos;
+ Tracing.trace("Processing response from {}", message.from());
+
+ RequestCallback cb = callbackInfo.callback;
if (message.isFailureResponse())
{
- ((IAsyncCallbackWithFailure) cb).onFailure(message.from, message.getFailureReason());
+ cb.onFailure(message.from(), (RequestFailureReason) message.payload);
}
else
{
- //TODO: Should we add latency only in success cases?
- MessagingService.instance().maybeAddLatency(cb, message.from, latency);
- cb.response(message);
+ MessagingService.instance().latencySubscribers.maybeAdd(cb, message.from(), latencyNanos, NANOSECONDS);
+ cb.onResponse(message);
}
if (callbackInfo.callback.supportsBackPressure())
{
- MessagingService.instance().updateBackPressureOnReceive(message.from, cb, false);
+ MessagingService.instance().updateBackPressureOnReceive(message.from(), cb, false);
}
}
}
diff --git a/src/java/org/apache/cassandra/net/ShareableBytes.java b/src/java/org/apache/cassandra/net/ShareableBytes.java
new file mode 100644
index 0000000..e4f2460
--- /dev/null
+++ b/src/java/org/apache/cassandra/net/ShareableBytes.java
@@ -0,0 +1,174 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.net;
+
+import java.nio.ByteBuffer;
+import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
+
+import org.apache.cassandra.utils.memory.BufferPool;
+
+/**
+ * A wrapper for possibly sharing portions of a single, {@link BufferPool} managed, {@link ByteBuffer};
+ * optimised for the case where no sharing is necessary.
+ *
+ * When sharing is necessary, {@link #share()} method must be invoked by the owning thread
+ * before a {@link ShareableBytes} instance can be shared with another thread.
+ */
+class ShareableBytes
+{
+ private final ByteBuffer bytes;
+ private final ShareableBytes owner;
+ private volatile int count;
+
+ private static final int UNSHARED = -1;
+ private static final int RELEASED = 0;
+ private static final AtomicIntegerFieldUpdater<ShareableBytes> countUpdater =
+ AtomicIntegerFieldUpdater.newUpdater(ShareableBytes.class, "count");
+
+ private ShareableBytes(ByteBuffer bytes)
+ {
+ this.count = UNSHARED;
+ this.owner = this;
+ this.bytes = bytes;
+ }
+
+ private ShareableBytes(ShareableBytes owner, ByteBuffer bytes)
+ {
+ this.owner = owner;
+ this.bytes = bytes;
+ }
+
+ ByteBuffer get()
+ {
+ assert owner.count != 0;
+ return bytes;
+ }
+
+ boolean hasRemaining()
+ {
+ return bytes.hasRemaining();
+ }
+
+ int remaining()
+ {
+ return bytes.remaining();
+ }
+
+ void skipBytes(int skipBytes)
+ {
+ bytes.position(bytes.position() + skipBytes);
+ }
+
+ void consume()
+ {
+ bytes.position(bytes.limit());
+ }
+
+ /**
+ * Ensure this ShareableBytes will use atomic operations for updating its count from now on.
+ * The first invocation must occur while the calling thread has exclusive access (though there may be more
+ * than one 'owner', these must all either be owned by the calling thread or otherwise not being used)
+ */
+ ShareableBytes share()
+ {
+ int count = owner.count;
+ if (count < 0)
+ owner.count = -count;
+ return this;
+ }
+
+ private ShareableBytes retain()
+ {
+ owner.doRetain();
+ return this;
+ }
+
+ private void doRetain()
+ {
+ int count = this.count;
+ if (count < 0)
+ {
+ countUpdater.lazySet(this, count - 1);
+ return;
+ }
+
+ while (true)
+ {
+ if (count == RELEASED)
+ throw new IllegalStateException("Attempted to reference an already released SharedByteBuffer");
+
+ if (countUpdater.compareAndSet(this, count, count + 1))
+ return;
+
+ count = this.count;
+ }
+ }
+
+ void release()
+ {
+ owner.doRelease();
+ }
+
+ private void doRelease()
+ {
+ int count = this.count;
+
+ if (count < 0)
+ countUpdater.lazySet(this, count += 1);
+ else if (count > 0)
+ count = countUpdater.decrementAndGet(this);
+ else
+ throw new IllegalStateException("Already released");
+
+ if (count == RELEASED)
+ BufferPool.put(bytes);
+ }
+
+ boolean isReleased()
+ {
+ return owner.count == RELEASED;
+ }
+
+ /**
+ * Create a slice over the next {@code length} bytes, consuming them from our buffer, and incrementing the owner count
+ */
+ ShareableBytes sliceAndConsume(int length)
+ {
+ int begin = bytes.position();
+ int end = begin + length;
+ ShareableBytes result = slice(begin, end);
+ bytes.position(end);
+ return result;
+ }
+
+ /**
+ * Create a new slice, incrementing the number of owners (making it shared if it was previously unshared)
+ */
+ ShareableBytes slice(int begin, int end)
+ {
+ ByteBuffer slice = bytes.duplicate();
+ slice.position(begin).limit(end);
+ return new ShareableBytes(owner.retain(), slice);
+ }
+
+ static ShareableBytes wrap(ByteBuffer buffer)
+ {
+ return new ShareableBytes(buffer);
+ }
+}
+
diff --git a/src/java/org/apache/cassandra/net/SharedDefaultFileRegion.java b/src/java/org/apache/cassandra/net/SharedDefaultFileRegion.java
new file mode 100644
index 0000000..6b47c22
--- /dev/null
+++ b/src/java/org/apache/cassandra/net/SharedDefaultFileRegion.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.net;
+
+import java.nio.channels.FileChannel;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import io.netty.channel.DefaultFileRegion;
+import org.apache.cassandra.utils.concurrent.Ref;
+import org.apache.cassandra.utils.concurrent.RefCounted;
+
+/**
+ * Netty's DefaultFileRegion closes the underlying FileChannel as soon as
+ * the refCnt() for the region drops to zero, this is an implementation of
+ * the DefaultFileRegion that doesn't close the FileChannel.
+ *
+ * See {@link AsyncChannelOutputPlus} for its usage.
+ */
+public class SharedDefaultFileRegion extends DefaultFileRegion
+{
+ public static class SharedFileChannel
+ {
+ // we don't call .ref() on this, because it would generate a lot of PhantomReferences and GC overhead,
+ // but we use it to ensure we can spot memory leaks
+ final Ref<FileChannel> ref;
+ final AtomicInteger refCount = new AtomicInteger(1);
+
+ SharedFileChannel(FileChannel fileChannel)
+ {
+ this.ref = new Ref<>(fileChannel, new RefCounted.Tidy()
+ {
+ public void tidy() throws Exception
+ {
+ // don't mind invoking this on eventLoop, as only used with sendFile which is also blocking
+ // so must use streaming eventLoop
+ fileChannel.close();
+ }
+
+ public String name()
+ {
+ return "SharedFileChannel[" + fileChannel.toString() + ']';
+ }
+ });
+ }
+
+ public void release()
+ {
+ if (0 == refCount.decrementAndGet())
+ ref.release();
+ }
+ }
+
+ private final SharedFileChannel shared;
+ private boolean deallocated = false;
+
+ SharedDefaultFileRegion(SharedFileChannel shared, long position, long count)
+ {
+ super(shared.ref.get(), position, count);
+ this.shared = shared;
+ if (1 >= this.shared.refCount.incrementAndGet())
+ throw new IllegalStateException();
+ }
+
+ @Override
+ protected void deallocate()
+ {
+ if (deallocated)
+ return;
+ deallocated = true;
+ shared.release();
+ }
+
+ public static SharedFileChannel share(FileChannel fileChannel)
+ {
+ return new SharedFileChannel(fileChannel);
+ }
+}
diff --git a/src/java/org/apache/cassandra/net/SocketFactory.java b/src/java/org/apache/cassandra/net/SocketFactory.java
new file mode 100644
index 0000000..da2d461
--- /dev/null
+++ b/src/java/org/apache/cassandra/net/SocketFactory.java
@@ -0,0 +1,300 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.net;
+
+import java.io.IOException;
+import java.net.ConnectException;
+import java.net.InetSocketAddress;
+import java.nio.channels.ClosedChannelException;
+import java.nio.channels.spi.SelectorProvider;
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.TimeoutException;
+import javax.annotation.Nullable;
+import javax.net.ssl.SSLEngine;
+import javax.net.ssl.SSLParameters;
+
+import com.google.common.collect.ImmutableList;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import io.netty.bootstrap.Bootstrap;
+import io.netty.bootstrap.ServerBootstrap;
+import io.netty.channel.Channel;
+import io.netty.channel.ChannelFactory;
+import io.netty.channel.DefaultSelectStrategyFactory;
+import io.netty.channel.EventLoop;
+import io.netty.channel.EventLoopGroup;
+import io.netty.channel.ServerChannel;
+import io.netty.channel.epoll.EpollChannelOption;
+import io.netty.channel.epoll.EpollEventLoopGroup;
+import io.netty.channel.epoll.EpollServerSocketChannel;
+import io.netty.channel.epoll.EpollSocketChannel;
+import io.netty.channel.nio.NioEventLoopGroup;
+import io.netty.channel.socket.nio.NioServerSocketChannel;
+import io.netty.channel.socket.nio.NioSocketChannel;
+import io.netty.channel.unix.Errors;
+import io.netty.handler.ssl.OpenSsl;
+import io.netty.handler.ssl.SslContext;
+import io.netty.handler.ssl.SslHandler;
+import io.netty.util.concurrent.DefaultEventExecutorChooserFactory;
+import io.netty.util.concurrent.DefaultThreadFactory;
+import io.netty.util.concurrent.RejectedExecutionHandlers;
+import io.netty.util.concurrent.ThreadPerTaskExecutor;
+import io.netty.util.internal.logging.InternalLoggerFactory;
+import io.netty.util.internal.logging.Slf4JLoggerFactory;
+import org.apache.cassandra.concurrent.NamedThreadFactory;
+import org.apache.cassandra.config.Config;
+import org.apache.cassandra.config.EncryptionOptions;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.security.SSLFactory;
+import org.apache.cassandra.service.NativeTransportService;
+import org.apache.cassandra.utils.ExecutorUtils;
+import org.apache.cassandra.utils.FBUtilities;
+
+import static io.netty.channel.unix.Errors.ERRNO_ECONNRESET_NEGATIVE;
+import static io.netty.channel.unix.Errors.ERROR_ECONNREFUSED_NEGATIVE;
+import static java.util.concurrent.TimeUnit.SECONDS;
+import static org.apache.cassandra.utils.Throwables.isCausedBy;
+
+/**
+ * A factory for building Netty {@link Channel}s. Channels here are setup with a pipeline to participate
+ * in the internode protocol handshake, either the inbound or outbound side as per the method invoked.
+ */
+public final class SocketFactory
+{
+ private static final Logger logger = LoggerFactory.getLogger(SocketFactory.class);
+
+ private static final int EVENT_THREADS = Integer.getInteger(Config.PROPERTY_PREFIX + "internode-event-threads", FBUtilities.getAvailableProcessors());
+
+ /**
+ * The default task queue used by {@code NioEventLoop} and {@code EpollEventLoop} is {@code MpscUnboundedArrayQueue},
+ * provided by JCTools. While efficient, it has an undesirable quality for a queue backing an event loop: it is
+ * not non-blocking, and can cause the event loop to busy-spin while waiting for a partially completed task
+ * offer, if the producer thread has been suspended mid-offer.
+ *
+ * As it happens, however, we have an MPSC queue implementation that is perfectly fit for this purpose -
+ * {@link ManyToOneConcurrentLinkedQueue}, that is non-blocking, and already used throughout the codebase,
+ * that we can and do use here as well.
+ */
+ enum Provider
+ {
+ NIO
+ {
+ @Override
+ NioEventLoopGroup makeEventLoopGroup(int threadCount, ThreadFactory threadFactory)
+ {
+ return new NioEventLoopGroup(threadCount,
+ new ThreadPerTaskExecutor(threadFactory),
+ DefaultEventExecutorChooserFactory.INSTANCE,
+ SelectorProvider.provider(),
+ DefaultSelectStrategyFactory.INSTANCE,
+ RejectedExecutionHandlers.reject(),
+ capacity -> new ManyToOneConcurrentLinkedQueue<>());
+ }
+
+ @Override
+ ChannelFactory<NioSocketChannel> clientChannelFactory()
+ {
+ return NioSocketChannel::new;
+ }
+
+ @Override
+ ChannelFactory<NioServerSocketChannel> serverChannelFactory()
+ {
+ return NioServerSocketChannel::new;
+ }
+ },
+ EPOLL
+ {
+ @Override
+ EpollEventLoopGroup makeEventLoopGroup(int threadCount, ThreadFactory threadFactory)
+ {
+ return new EpollEventLoopGroup(threadCount,
+ new ThreadPerTaskExecutor(threadFactory),
+ DefaultEventExecutorChooserFactory.INSTANCE,
+ DefaultSelectStrategyFactory.INSTANCE,
+ RejectedExecutionHandlers.reject(),
+ capacity -> new ManyToOneConcurrentLinkedQueue<>());
+ }
+
+ @Override
+ ChannelFactory<EpollSocketChannel> clientChannelFactory()
+ {
+ return EpollSocketChannel::new;
+ }
+
+ @Override
+ ChannelFactory<EpollServerSocketChannel> serverChannelFactory()
+ {
+ return EpollServerSocketChannel::new;
+ }
+ };
+
+ EventLoopGroup makeEventLoopGroup(int threadCount, String threadNamePrefix)
+ {
+ logger.debug("using netty {} event loop for pool prefix {}", name(), threadNamePrefix);
+ return makeEventLoopGroup(threadCount, new DefaultThreadFactory(threadNamePrefix, true));
+ }
+
+ abstract EventLoopGroup makeEventLoopGroup(int threadCount, ThreadFactory threadFactory);
+ abstract ChannelFactory<? extends Channel> clientChannelFactory();
+ abstract ChannelFactory<? extends ServerChannel> serverChannelFactory();
+
+ static Provider optimalProvider()
+ {
+ return NativeTransportService.useEpoll() ? EPOLL : NIO;
+ }
+ }
+
+ /** a useful addition for debugging; simply set to true to get more data in your logs */
+ static final boolean WIRETRACE = false;
+ static
+ {
+ if (WIRETRACE)
+ InternalLoggerFactory.setDefaultFactory(Slf4JLoggerFactory.INSTANCE);
+ }
+
+ private final Provider provider;
+ private final EventLoopGroup acceptGroup;
+ private final EventLoopGroup defaultGroup;
+ // we need a separate EventLoopGroup for outbound streaming because sendFile is blocking
+ private final EventLoopGroup outboundStreamingGroup;
+ final ExecutorService synchronousWorkExecutor = Executors.newCachedThreadPool(new NamedThreadFactory("Messaging-SynchronousWork"));
+
+ SocketFactory()
+ {
+ this(Provider.optimalProvider());
+ }
+
+ SocketFactory(Provider provider)
+ {
+ this.provider = provider;
+ this.acceptGroup = provider.makeEventLoopGroup(1, "Messaging-AcceptLoop");
+ this.defaultGroup = provider.makeEventLoopGroup(EVENT_THREADS, NamedThreadFactory.globalPrefix() + "Messaging-EventLoop");
+ this.outboundStreamingGroup = provider.makeEventLoopGroup(EVENT_THREADS, "Streaming-EventLoop");
+ }
+
+ Bootstrap newClientBootstrap(EventLoop eventLoop, int tcpUserTimeoutInMS)
+ {
+ if (eventLoop == null)
+ throw new IllegalArgumentException("must provide eventLoop");
+
+ Bootstrap bootstrap = new Bootstrap().group(eventLoop).channelFactory(provider.clientChannelFactory());
+
+ if (provider == Provider.EPOLL)
+ bootstrap.option(EpollChannelOption.TCP_USER_TIMEOUT, tcpUserTimeoutInMS);
+
+ return bootstrap;
+ }
+
+ ServerBootstrap newServerBootstrap()
+ {
+ return new ServerBootstrap().group(acceptGroup, defaultGroup).channelFactory(provider.serverChannelFactory());
+ }
+
+ /**
+ * Creates a new {@link SslHandler} from provided SslContext.
+ * @param peer enables endpoint verification for remote address when not null
+ */
+ static SslHandler newSslHandler(Channel channel, SslContext sslContext, @Nullable InetSocketAddress peer)
+ {
+ if (peer == null)
+ return sslContext.newHandler(channel.alloc());
+
+ logger.debug("Creating SSL handler for {}:{}", peer.getHostString(), peer.getPort());
+ SslHandler sslHandler = sslContext.newHandler(channel.alloc(), peer.getHostString(), peer.getPort());
+ SSLEngine engine = sslHandler.engine();
+ SSLParameters sslParameters = engine.getSSLParameters();
+ sslParameters.setEndpointIdentificationAlgorithm("HTTPS");
+ engine.setSSLParameters(sslParameters);
+ return sslHandler;
+ }
+
+ static String encryptionLogStatement(EncryptionOptions options)
+ {
+ if (options == null)
+ return "disabled";
+
+ String encryptionType = SSLFactory.openSslIsAvailable() ? "openssl" : "jdk";
+ return "enabled (" + encryptionType + ')';
+ }
+
+ EventLoopGroup defaultGroup()
+ {
+ return defaultGroup;
+ }
+
+ public EventLoopGroup outboundStreamingGroup()
+ {
+ return outboundStreamingGroup;
+ }
+
+ public void shutdownNow()
+ {
+ acceptGroup.shutdownGracefully(0, 2, SECONDS);
+ defaultGroup.shutdownGracefully(0, 2, SECONDS);
+ outboundStreamingGroup.shutdownGracefully(0, 2, SECONDS);
+ synchronousWorkExecutor.shutdownNow();
+ }
+
+ void awaitTerminationUntil(long deadlineNanos) throws InterruptedException, TimeoutException
+ {
+ List<ExecutorService> groups = ImmutableList.of(acceptGroup, defaultGroup, outboundStreamingGroup, synchronousWorkExecutor);
+ ExecutorUtils.awaitTerminationUntil(deadlineNanos, groups);
+ }
+
+ static boolean isConnectionReset(Throwable t)
+ {
+ if (t instanceof ClosedChannelException)
+ return true;
+ if (t instanceof ConnectException)
+ return true;
+ if (t instanceof Errors.NativeIoException)
+ {
+ int errorCode = ((Errors.NativeIoException) t).expectedErr();
+ return errorCode == ERRNO_ECONNRESET_NEGATIVE || errorCode != ERROR_ECONNREFUSED_NEGATIVE;
+ }
+ return IOException.class == t.getClass() && ("Broken pipe".equals(t.getMessage()) || "Connection reset by peer".equals(t.getMessage()));
+ }
+
+ static boolean isCausedByConnectionReset(Throwable t)
+ {
+ return isCausedBy(t, SocketFactory::isConnectionReset);
+ }
+
+ static String channelId(InetAddressAndPort from, InetSocketAddress realFrom, InetAddressAndPort to, InetSocketAddress realTo, ConnectionType type, String id)
+ {
+ return addressId(from, realFrom) + "->" + addressId(to, realTo) + '-' + type + '-' + id;
+ }
+
+ static String addressId(InetAddressAndPort address, InetSocketAddress realAddress)
+ {
+ String str = address.toString();
+ if (!address.address.equals(realAddress.getAddress()) || address.port != realAddress.getPort())
+ str += '(' + InetAddressAndPort.toString(realAddress.getAddress(), realAddress.getPort()) + ')';
+ return str;
+ }
+
+ static String channelId(InetAddressAndPort from, InetAddressAndPort to, ConnectionType type, String id)
+ {
+ return from + "->" + to + '-' + type + '-' + id;
+ }
+}
diff --git a/src/java/org/apache/cassandra/net/StartupClusterConnectivityChecker.java b/src/java/org/apache/cassandra/net/StartupClusterConnectivityChecker.java
new file mode 100644
index 0000000..b901338
--- /dev/null
+++ b/src/java/org/apache/cassandra/net/StartupClusterConnectivityChecker.java
@@ -0,0 +1,275 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.net;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.SetMultimap;
+import com.google.common.util.concurrent.Uninterruptibles;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.gms.ApplicationState;
+import org.apache.cassandra.gms.EndpointState;
+import org.apache.cassandra.gms.Gossiper;
+import org.apache.cassandra.gms.IEndpointStateChangeSubscriber;
+import org.apache.cassandra.gms.VersionedValue;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.utils.FBUtilities;
+
+import static org.apache.cassandra.net.Verb.PING_REQ;
+import static org.apache.cassandra.net.ConnectionType.LARGE_MESSAGES;
+import static org.apache.cassandra.net.ConnectionType.SMALL_MESSAGES;
+
+public class StartupClusterConnectivityChecker
+{
+ private static final Logger logger = LoggerFactory.getLogger(StartupClusterConnectivityChecker.class);
+
+ private final boolean blockForRemoteDcs;
+ private final long timeoutNanos;
+
+ public static StartupClusterConnectivityChecker create(long timeoutSecs, boolean blockForRemoteDcs)
+ {
+ if (timeoutSecs > 100)
+ logger.warn("setting the block-for-peers timeout (in seconds) to {} might be a bit excessive, but using it nonetheless", timeoutSecs);
+ long timeoutNanos = TimeUnit.SECONDS.toNanos(timeoutSecs);
+
+ return new StartupClusterConnectivityChecker(timeoutNanos, blockForRemoteDcs);
+ }
+
+ @VisibleForTesting
+ StartupClusterConnectivityChecker(long timeoutNanos, boolean blockForRemoteDcs)
+ {
+ this.blockForRemoteDcs = blockForRemoteDcs;
+ this.timeoutNanos = timeoutNanos;
+ }
+
+ /**
+ * @param peers The currently known peers in the cluster; argument is not modified.
+ * @param getDatacenterSource A function for mapping peers to their datacenter.
+ * @return true if the requested percentage of peers are marked ALIVE in gossip and have their connections opened;
+ * else false.
+ */
+ public boolean execute(Set<InetAddressAndPort> peers, Function<InetAddressAndPort, String> getDatacenterSource)
+ {
+ if (peers == null || this.timeoutNanos < 0)
+ return true;
+
+ // make a copy of the set, to avoid mucking with the input (in case it's a sensitive collection)
+ peers = new HashSet<>(peers);
+ InetAddressAndPort localAddress = FBUtilities.getBroadcastAddressAndPort();
+ String localDc = getDatacenterSource.apply(localAddress);
+
+ peers.remove(localAddress);
+ if (peers.isEmpty())
+ return true;
+
+ // make a copy of the datacenter mapping (in case gossip updates happen during this method or some such)
+ Map<InetAddressAndPort, String> peerToDatacenter = new HashMap<>();
+ SetMultimap<String, InetAddressAndPort> datacenterToPeers = HashMultimap.create();
+
+ for (InetAddressAndPort peer : peers)
+ {
+ String datacenter = getDatacenterSource.apply(peer);
+ peerToDatacenter.put(peer, datacenter);
+ datacenterToPeers.put(datacenter, peer);
+ }
+
+ // In the case where we do not want to block startup on remote datacenters (e.g. because clients only use
+ // LOCAL_X consistency levels), we remove all other datacenter hosts from the mapping and we only wait
+ // on the remaining local datacenter.
+ if (!blockForRemoteDcs)
+ {
+ datacenterToPeers.keySet().retainAll(Collections.singleton(localDc));
+ logger.info("Blocking coordination until only a single peer is DOWN in the local datacenter, timeout={}s",
+ TimeUnit.NANOSECONDS.toSeconds(timeoutNanos));
+ }
+ else
+ {
+ logger.info("Blocking coordination until only a single peer is DOWN in each datacenter, timeout={}s",
+ TimeUnit.NANOSECONDS.toSeconds(timeoutNanos));
+ }
+
+ AckMap acks = new AckMap(3);
+ Map<String, CountDownLatch> dcToRemainingPeers = new HashMap<>(datacenterToPeers.size());
+ for (String datacenter: datacenterToPeers.keys())
+ {
+ dcToRemainingPeers.put(datacenter,
+ new CountDownLatch(Math.max(datacenterToPeers.get(datacenter).size() - 1, 0)));
+ }
+
+ long startNanos = System.nanoTime();
+
+ // set up a listener to react to new nodes becoming alive (in gossip), and account for all the nodes that are already alive
+ Set<InetAddressAndPort> alivePeers = Collections.newSetFromMap(new ConcurrentHashMap<>());
+ AliveListener listener = new AliveListener(alivePeers, dcToRemainingPeers, acks, peerToDatacenter::get);
+ Gossiper.instance.register(listener);
+
+ // send out a ping message to open up the non-gossip connections to all peers. Note that this sends the
+ // ping messages to _all_ peers, not just the ones we block for in dcToRemainingPeers.
+ sendPingMessages(peers, dcToRemainingPeers, acks, peerToDatacenter::get);
+
+ for (InetAddressAndPort peer : peers)
+ {
+ if (Gossiper.instance.isAlive(peer) && alivePeers.add(peer) && acks.incrementAndCheck(peer))
+ {
+ String datacenter = peerToDatacenter.get(peer);
+ // We have to check because we might only have the local DC in the map
+ if (dcToRemainingPeers.containsKey(datacenter))
+ dcToRemainingPeers.get(datacenter).countDown();
+ }
+ }
+
+ boolean succeeded = true;
+ for (CountDownLatch countDownLatch : dcToRemainingPeers.values())
+ {
+ long remainingNanos = Math.max(1, timeoutNanos - (System.nanoTime() - startNanos));
+ //noinspection UnstableApiUsage
+ succeeded &= Uninterruptibles.awaitUninterruptibly(countDownLatch, remainingNanos, TimeUnit.NANOSECONDS);
+ }
+
+ Gossiper.instance.unregister(listener);
+
+ Map<String, Long> numDown = dcToRemainingPeers.entrySet().stream()
+ .collect(Collectors.toMap(Map.Entry::getKey,
+ e -> e.getValue().getCount()));
+
+ if (succeeded)
+ {
+ logger.info("Ensured sufficient healthy connections with {} after {} milliseconds",
+ numDown.keySet(), TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos));
+ }
+ else
+ {
+ logger.warn("Timed out after {} milliseconds, was waiting for remaining peers to connect: {}",
+ TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos), numDown);
+ }
+
+ return succeeded;
+ }
+
+ /**
+ * Sends a "connection warmup" message to each peer in the collection, on every {@link ConnectionType}
+ * used for internode messaging (that is not gossip).
+ */
+ private void sendPingMessages(Set<InetAddressAndPort> peers, Map<String, CountDownLatch> dcToRemainingPeers,
+ AckMap acks, Function<InetAddressAndPort, String> getDatacenter)
+ {
+ RequestCallback responseHandler = msg -> {
+ if (acks.incrementAndCheck(msg.from()))
+ {
+ String datacenter = getDatacenter.apply(msg.from());
+ // We have to check because we might only have the local DC in the map
+ if (dcToRemainingPeers.containsKey(datacenter))
+ dcToRemainingPeers.get(datacenter).countDown();
+ }
+ };
+
+ Message<PingRequest> small = Message.out(PING_REQ, PingRequest.forSmall);
+ Message<PingRequest> large = Message.out(PING_REQ, PingRequest.forLarge);
+ for (InetAddressAndPort peer : peers)
+ {
+ MessagingService.instance().sendWithCallback(small, peer, responseHandler, SMALL_MESSAGES);
+ MessagingService.instance().sendWithCallback(large, peer, responseHandler, LARGE_MESSAGES);
+ }
+ }
+
+ /**
+ * A trivial implementation of {@link IEndpointStateChangeSubscriber} that really only cares about
+ * {@link #onAlive(InetAddressAndPort, EndpointState)} invocations.
+ */
+ private static final class AliveListener implements IEndpointStateChangeSubscriber
+ {
+ private final Map<String, CountDownLatch> dcToRemainingPeers;
+ private final Set<InetAddressAndPort> livePeers;
+ private final Function<InetAddressAndPort, String> getDatacenter;
+ private final AckMap acks;
+
+ AliveListener(Set<InetAddressAndPort> livePeers, Map<String, CountDownLatch> dcToRemainingPeers,
+ AckMap acks, Function<InetAddressAndPort, String> getDatacenter)
+ {
+ this.livePeers = livePeers;
+ this.dcToRemainingPeers = dcToRemainingPeers;
+ this.acks = acks;
+ this.getDatacenter = getDatacenter;
+ }
+
+ public void onJoin(InetAddressAndPort endpoint, EndpointState epState)
+ {
+ }
+
+ public void beforeChange(InetAddressAndPort endpoint, EndpointState currentState, ApplicationState newStateKey, VersionedValue newValue)
+ {
+ }
+
+ public void onChange(InetAddressAndPort endpoint, ApplicationState state, VersionedValue value)
+ {
+ }
+
+ public void onAlive(InetAddressAndPort endpoint, EndpointState state)
+ {
+ if (livePeers.add(endpoint) && acks.incrementAndCheck(endpoint))
+ {
+ String datacenter = getDatacenter.apply(endpoint);
+ if (dcToRemainingPeers.containsKey(datacenter))
+ dcToRemainingPeers.get(datacenter).countDown();
+ }
+ }
+
+ public void onDead(InetAddressAndPort endpoint, EndpointState state)
+ {
+ }
+
+ public void onRemove(InetAddressAndPort endpoint)
+ {
+ }
+
+ public void onRestart(InetAddressAndPort endpoint, EndpointState state)
+ {
+ }
+ }
+
+ private static final class AckMap
+ {
+ private final int threshold;
+ private final Map<InetAddressAndPort, AtomicInteger> acks;
+
+ AckMap(int threshold)
+ {
+ this.threshold = threshold;
+ acks = new ConcurrentHashMap<>();
+ }
+
+ boolean incrementAndCheck(InetAddressAndPort address)
+ {
+ return acks.computeIfAbsent(address, addr -> new AtomicInteger(0)).incrementAndGet() == threshold;
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/net/Verb.java b/src/java/org/apache/cassandra/net/Verb.java
new file mode 100644
index 0000000..6ba9ab8
--- /dev/null
+++ b/src/java/org/apache/cassandra/net/Verb.java
@@ -0,0 +1,455 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.net;
+
+import java.lang.reflect.Field;
+import java.lang.reflect.Modifier;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Supplier;
+import java.util.function.ToLongFunction;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableList;
+
+import org.apache.cassandra.batchlog.Batch;
+import org.apache.cassandra.batchlog.BatchRemoveVerbHandler;
+import org.apache.cassandra.batchlog.BatchStoreVerbHandler;
+import org.apache.cassandra.concurrent.Stage;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.CounterMutation;
+import org.apache.cassandra.db.CounterMutationVerbHandler;
+import org.apache.cassandra.db.Mutation;
+import org.apache.cassandra.db.MutationVerbHandler;
+import org.apache.cassandra.db.ReadCommand;
+import org.apache.cassandra.db.ReadCommandVerbHandler;
+import org.apache.cassandra.db.ReadRepairVerbHandler;
+import org.apache.cassandra.db.ReadResponse;
+import org.apache.cassandra.db.SnapshotCommand;
+import org.apache.cassandra.db.TruncateResponse;
+import org.apache.cassandra.db.TruncateVerbHandler;
+import org.apache.cassandra.db.TruncateRequest;
+import org.apache.cassandra.exceptions.RequestFailureReason;
+import org.apache.cassandra.gms.GossipDigestAck;
+import org.apache.cassandra.gms.GossipDigestAck2;
+import org.apache.cassandra.gms.GossipDigestAck2VerbHandler;
+import org.apache.cassandra.gms.GossipDigestAckVerbHandler;
+import org.apache.cassandra.gms.GossipDigestSyn;
+import org.apache.cassandra.gms.GossipDigestSynVerbHandler;
+import org.apache.cassandra.gms.GossipShutdownVerbHandler;
+import org.apache.cassandra.hints.HintMessage;
+import org.apache.cassandra.hints.HintVerbHandler;
+import org.apache.cassandra.io.IVersionedAsymmetricSerializer;
+import org.apache.cassandra.repair.RepairMessageVerbHandler;
+import org.apache.cassandra.repair.messages.AsymmetricSyncRequest;
+import org.apache.cassandra.repair.messages.CleanupMessage;
+import org.apache.cassandra.repair.messages.FailSession;
+import org.apache.cassandra.repair.messages.FinalizeCommit;
+import org.apache.cassandra.repair.messages.FinalizePromise;
+import org.apache.cassandra.repair.messages.FinalizePropose;
+import org.apache.cassandra.repair.messages.PrepareConsistentRequest;
+import org.apache.cassandra.repair.messages.PrepareConsistentResponse;
+import org.apache.cassandra.repair.messages.PrepareMessage;
+import org.apache.cassandra.repair.messages.SnapshotMessage;
+import org.apache.cassandra.repair.messages.StatusRequest;
+import org.apache.cassandra.repair.messages.StatusResponse;
+import org.apache.cassandra.repair.messages.SyncResponse;
+import org.apache.cassandra.repair.messages.SyncRequest;
+import org.apache.cassandra.repair.messages.ValidationResponse;
+import org.apache.cassandra.repair.messages.ValidationRequest;
+import org.apache.cassandra.schema.SchemaPullVerbHandler;
+import org.apache.cassandra.schema.SchemaPushVerbHandler;
+import org.apache.cassandra.schema.SchemaVersionVerbHandler;
+import org.apache.cassandra.utils.BooleanSerializer;
+import org.apache.cassandra.service.EchoVerbHandler;
+import org.apache.cassandra.service.SnapshotVerbHandler;
+import org.apache.cassandra.service.paxos.Commit;
+import org.apache.cassandra.service.paxos.CommitVerbHandler;
+import org.apache.cassandra.service.paxos.PrepareResponse;
+import org.apache.cassandra.service.paxos.PrepareVerbHandler;
+import org.apache.cassandra.service.paxos.ProposeVerbHandler;
+import org.apache.cassandra.streaming.ReplicationDoneVerbHandler;
+import org.apache.cassandra.utils.UUIDSerializer;
+
+import static java.util.concurrent.TimeUnit.NANOSECONDS;
+import static org.apache.cassandra.concurrent.Stage.*;
+import static org.apache.cassandra.concurrent.Stage.INTERNAL_RESPONSE;
+import static org.apache.cassandra.concurrent.Stage.MISC;
+import static org.apache.cassandra.net.VerbTimeouts.*;
+import static org.apache.cassandra.net.Verb.Kind.*;
+import static org.apache.cassandra.net.Verb.Priority.*;
+import static org.apache.cassandra.schema.MigrationManager.MigrationsSerializer;
+
+/**
+ * Note that priorities except P0 are presently unused. P0 corresponds to urgent, i.e. what used to be the "Gossip" connection.
+ */
+public enum Verb
+{
+ MUTATION_RSP (60, P1, writeTimeout, REQUEST_RESPONSE, () -> NoPayload.serializer, () -> ResponseVerbHandler.instance ),
+ MUTATION_REQ (0, P3, writeTimeout, MUTATION, () -> Mutation.serializer, () -> MutationVerbHandler.instance, MUTATION_RSP ),
+ HINT_RSP (61, P1, writeTimeout, REQUEST_RESPONSE, () -> NoPayload.serializer, () -> ResponseVerbHandler.instance ),
+ HINT_REQ (1, P4, writeTimeout, MUTATION, () -> HintMessage.serializer, () -> HintVerbHandler.instance, HINT_RSP ),
+ READ_REPAIR_RSP (62, P1, writeTimeout, REQUEST_RESPONSE, () -> NoPayload.serializer, () -> ResponseVerbHandler.instance ),
+ READ_REPAIR_REQ (2, P1, writeTimeout, MUTATION, () -> Mutation.serializer, () -> ReadRepairVerbHandler.instance, READ_REPAIR_RSP ),
+ BATCH_STORE_RSP (65, P1, writeTimeout, REQUEST_RESPONSE, () -> NoPayload.serializer, () -> ResponseVerbHandler.instance ),
+ BATCH_STORE_REQ (5, P3, writeTimeout, MUTATION, () -> Batch.serializer, () -> BatchStoreVerbHandler.instance, BATCH_STORE_RSP ),
+ BATCH_REMOVE_RSP (66, P1, writeTimeout, REQUEST_RESPONSE, () -> NoPayload.serializer, () -> ResponseVerbHandler.instance ),
+ BATCH_REMOVE_REQ (6, P3, writeTimeout, MUTATION, () -> UUIDSerializer.serializer, () -> BatchRemoveVerbHandler.instance, BATCH_REMOVE_RSP ),
+
+ PAXOS_PREPARE_RSP (93, P2, writeTimeout, REQUEST_RESPONSE, () -> PrepareResponse.serializer, () -> ResponseVerbHandler.instance ),
+ PAXOS_PREPARE_REQ (33, P2, writeTimeout, MUTATION, () -> Commit.serializer, () -> PrepareVerbHandler.instance, PAXOS_PREPARE_RSP ),
+ PAXOS_PROPOSE_RSP (94, P2, writeTimeout, REQUEST_RESPONSE, () -> BooleanSerializer.serializer, () -> ResponseVerbHandler.instance ),
+ PAXOS_PROPOSE_REQ (34, P2, writeTimeout, MUTATION, () -> Commit.serializer, () -> ProposeVerbHandler.instance, PAXOS_PROPOSE_RSP ),
+ PAXOS_COMMIT_RSP (95, P2, writeTimeout, REQUEST_RESPONSE, () -> NoPayload.serializer, () -> ResponseVerbHandler.instance ),
+ PAXOS_COMMIT_REQ (35, P2, writeTimeout, MUTATION, () -> Commit.serializer, () -> CommitVerbHandler.instance, PAXOS_COMMIT_RSP ),
+
+ TRUNCATE_RSP (79, P0, truncateTimeout, REQUEST_RESPONSE, () -> TruncateResponse.serializer, () -> ResponseVerbHandler.instance ),
+ TRUNCATE_REQ (19, P0, truncateTimeout, MUTATION, () -> TruncateRequest.serializer, () -> TruncateVerbHandler.instance, TRUNCATE_RSP ),
+
+ COUNTER_MUTATION_RSP (84, P1, counterTimeout, REQUEST_RESPONSE, () -> NoPayload.serializer, () -> ResponseVerbHandler.instance ),
+ COUNTER_MUTATION_REQ (24, P2, counterTimeout, COUNTER_MUTATION, () -> CounterMutation.serializer, () -> CounterMutationVerbHandler.instance, COUNTER_MUTATION_RSP),
+
+ READ_RSP (63, P2, readTimeout, REQUEST_RESPONSE, () -> ReadResponse.serializer, () -> ResponseVerbHandler.instance ),
+ READ_REQ (3, P3, readTimeout, READ, () -> ReadCommand.serializer, () -> ReadCommandVerbHandler.instance, READ_RSP ),
+ RANGE_RSP (69, P2, rangeTimeout, REQUEST_RESPONSE, () -> ReadResponse.serializer, () -> ResponseVerbHandler.instance ),
+ RANGE_REQ (9, P3, rangeTimeout, READ, () -> ReadCommand.serializer, () -> ReadCommandVerbHandler.instance, RANGE_RSP ),
+
+ GOSSIP_DIGEST_SYN (14, P0, longTimeout, GOSSIP, () -> GossipDigestSyn.serializer, () -> GossipDigestSynVerbHandler.instance ),
+ GOSSIP_DIGEST_ACK (15, P0, longTimeout, GOSSIP, () -> GossipDigestAck.serializer, () -> GossipDigestAckVerbHandler.instance ),
+ GOSSIP_DIGEST_ACK2 (16, P0, longTimeout, GOSSIP, () -> GossipDigestAck2.serializer, () -> GossipDigestAck2VerbHandler.instance ),
+ GOSSIP_SHUTDOWN (29, P0, rpcTimeout, GOSSIP, () -> NoPayload.serializer, () -> GossipShutdownVerbHandler.instance ),
+
+ ECHO_RSP (91, P0, rpcTimeout, GOSSIP, () -> NoPayload.serializer, () -> ResponseVerbHandler.instance ),
+ ECHO_REQ (31, P0, rpcTimeout, GOSSIP, () -> NoPayload.serializer, () -> EchoVerbHandler.instance, ECHO_RSP ),
+ PING_RSP (97, P1, pingTimeout, GOSSIP, () -> NoPayload.serializer, () -> ResponseVerbHandler.instance ),
+ PING_REQ (37, P1, pingTimeout, GOSSIP, () -> PingRequest.serializer, () -> PingVerbHandler.instance, PING_RSP ),
+
+ // P1 because messages can be arbitrarily large or aren't crucial
+ SCHEMA_PUSH_RSP (98, P1, rpcTimeout, MIGRATION, () -> NoPayload.serializer, () -> ResponseVerbHandler.instance ),
+ SCHEMA_PUSH_REQ (18, P1, rpcTimeout, MIGRATION, () -> MigrationsSerializer.instance, () -> SchemaPushVerbHandler.instance, SCHEMA_PUSH_RSP ),
+ SCHEMA_PULL_RSP (88, P1, rpcTimeout, MIGRATION, () -> MigrationsSerializer.instance, () -> ResponseVerbHandler.instance ),
+ SCHEMA_PULL_REQ (28, P1, rpcTimeout, MIGRATION, () -> NoPayload.serializer, () -> SchemaPullVerbHandler.instance, SCHEMA_PULL_RSP ),
+ SCHEMA_VERSION_RSP (80, P1, rpcTimeout, MIGRATION, () -> UUIDSerializer.serializer, () -> ResponseVerbHandler.instance ),
+ SCHEMA_VERSION_REQ (20, P1, rpcTimeout, MIGRATION, () -> NoPayload.serializer, () -> SchemaVersionVerbHandler.instance, SCHEMA_VERSION_RSP ),
+
+ // repair; mostly doesn't use callbacks and sends responses as their own request messages, with matching sessions by uuid; should eventually harmonize and make idiomatic
+ REPAIR_RSP (100, P1, rpcTimeout, REQUEST_RESPONSE, () -> NoPayload.serializer, () -> ResponseVerbHandler.instance ),
+ VALIDATION_RSP (102, P1, rpcTimeout, ANTI_ENTROPY, () -> ValidationResponse.serializer, () -> RepairMessageVerbHandler.instance, REPAIR_RSP ),
+ VALIDATION_REQ (101, P1, rpcTimeout, ANTI_ENTROPY, () -> ValidationRequest.serializer, () -> RepairMessageVerbHandler.instance, REPAIR_RSP ),
+ SYNC_RSP (104, P1, rpcTimeout, ANTI_ENTROPY, () -> SyncResponse.serializer, () -> RepairMessageVerbHandler.instance, REPAIR_RSP ),
+ SYNC_REQ (103, P1, rpcTimeout, ANTI_ENTROPY, () -> SyncRequest.serializer, () -> RepairMessageVerbHandler.instance, REPAIR_RSP ),
+ PREPARE_MSG (105, P1, rpcTimeout, ANTI_ENTROPY, () -> PrepareMessage.serializer, () -> RepairMessageVerbHandler.instance, REPAIR_RSP ),
+ SNAPSHOT_MSG (106, P1, rpcTimeout, ANTI_ENTROPY, () -> SnapshotMessage.serializer, () -> RepairMessageVerbHandler.instance, REPAIR_RSP ),
+ CLEANUP_MSG (107, P1, rpcTimeout, ANTI_ENTROPY, () -> CleanupMessage.serializer, () -> RepairMessageVerbHandler.instance, REPAIR_RSP ),
+ PREPARE_CONSISTENT_RSP (109, P1, rpcTimeout, ANTI_ENTROPY, () -> PrepareConsistentResponse.serializer, () -> RepairMessageVerbHandler.instance, REPAIR_RSP ),
+ PREPARE_CONSISTENT_REQ (108, P1, rpcTimeout, ANTI_ENTROPY, () -> PrepareConsistentRequest.serializer, () -> RepairMessageVerbHandler.instance, REPAIR_RSP ),
+ FINALIZE_PROPOSE_MSG (110, P1, rpcTimeout, ANTI_ENTROPY, () -> FinalizePropose.serializer, () -> RepairMessageVerbHandler.instance, REPAIR_RSP ),
+ FINALIZE_PROMISE_MSG (111, P1, rpcTimeout, ANTI_ENTROPY, () -> FinalizePromise.serializer, () -> RepairMessageVerbHandler.instance, REPAIR_RSP ),
+ FINALIZE_COMMIT_MSG (112, P1, rpcTimeout, ANTI_ENTROPY, () -> FinalizeCommit.serializer, () -> RepairMessageVerbHandler.instance, REPAIR_RSP ),
+ FAILED_SESSION_MSG (113, P1, rpcTimeout, ANTI_ENTROPY, () -> FailSession.serializer, () -> RepairMessageVerbHandler.instance, REPAIR_RSP ),
+ STATUS_RSP (115, P1, rpcTimeout, ANTI_ENTROPY, () -> StatusResponse.serializer, () -> RepairMessageVerbHandler.instance, REPAIR_RSP ),
+ STATUS_REQ (114, P1, rpcTimeout, ANTI_ENTROPY, () -> StatusRequest.serializer, () -> RepairMessageVerbHandler.instance, REPAIR_RSP ),
+ ASYMMETRIC_SYNC_REQ (116, P1, rpcTimeout, ANTI_ENTROPY, () -> AsymmetricSyncRequest.serializer, () -> RepairMessageVerbHandler.instance, REPAIR_RSP ),
+
+ REPLICATION_DONE_RSP (82, P0, rpcTimeout, MISC, () -> NoPayload.serializer, () -> ResponseVerbHandler.instance ),
+ REPLICATION_DONE_REQ (22, P0, rpcTimeout, MISC, () -> NoPayload.serializer, () -> ReplicationDoneVerbHandler.instance, REPLICATION_DONE_RSP),
+ SNAPSHOT_RSP (87, P0, rpcTimeout, MISC, () -> NoPayload.serializer, () -> ResponseVerbHandler.instance ),
+ SNAPSHOT_REQ (27, P0, rpcTimeout, MISC, () -> SnapshotCommand.serializer, () -> SnapshotVerbHandler.instance, SNAPSHOT_RSP ),
+
+ // generic failure response
+ FAILURE_RSP (99, P0, noTimeout, REQUEST_RESPONSE, () -> RequestFailureReason.serializer, () -> ResponseVerbHandler.instance ),
+
+ // dummy verbs
+ _TRACE (30, P1, rpcTimeout, TRACING, () -> NoPayload.serializer, () -> null ),
+ _SAMPLE (42, P1, rpcTimeout, INTERNAL_RESPONSE, () -> NoPayload.serializer, () -> null ),
+ _TEST_1 (10, P0, writeTimeout, IMMEDIATE, () -> NoPayload.serializer, () -> null ),
+ _TEST_2 (11, P1, rpcTimeout, IMMEDIATE, () -> NoPayload.serializer, () -> null ),
+
+ @Deprecated
+ REQUEST_RSP (4, P1, rpcTimeout, REQUEST_RESPONSE, () -> null, () -> ResponseVerbHandler.instance ),
+ @Deprecated
+ INTERNAL_RSP (23, P1, rpcTimeout, INTERNAL_RESPONSE, () -> null, () -> ResponseVerbHandler.instance ),
+
+ // largest used ID: 116
+
+ // CUSTOM VERBS
+ UNUSED_CUSTOM_VERB (CUSTOM,
+ 0, P1, rpcTimeout, INTERNAL_RESPONSE, () -> null, () -> null ),
+
+ ;
+
+ public static final List<Verb> VERBS = ImmutableList.copyOf(Verb.values());
+
+ public enum Priority
+ {
+ P0, // sends on the urgent connection (i.e. for Gossip, Echo)
+ P1, // small or empty responses
+ P2, // larger messages that can be dropped but who have a larger impact on system stability (e.g. READ_REPAIR, READ_RSP)
+ P3,
+ P4
+ }
+
+ public enum Kind
+ {
+ NORMAL,
+ CUSTOM
+ }
+
+ public final int id;
+ public final Priority priority;
+ public final Stage stage;
+ public final Kind kind;
+
+ /**
+ * Messages we receive from peers have a Verb that tells us what kind of message it is.
+ * Most of the time, this is enough to determine how to deserialize the message payload.
+ * The exception is the REQUEST_RSP verb, which just means "a response to something you told me to do."
+ * Traditionally, this was fine since each VerbHandler knew what type of payload it expected, and
+ * handled the deserialization itself. Now that we do that in ITC, to avoid the extra copy to an
+ * intermediary byte[] (See CASSANDRA-3716), we need to wire that up to the CallbackInfo object
+ * (see below).
+ *
+ * NOTE: we use a Supplier to avoid loading the dependent classes until necessary.
+ */
+ private final Supplier<? extends IVersionedAsymmetricSerializer<?, ?>> serializer;
+ private final Supplier<? extends IVerbHandler<?>> handler;
+
+ final Verb responseVerb;
+
+ private final ToLongFunction<TimeUnit> expiration;
+
+
+ /**
+ * Verbs it's okay to drop if the request has been queued longer than the request timeout. These
+ * all correspond to client requests or something triggered by them; we don't want to
+ * drop internal messages like bootstrap or repair notifications.
+ */
+ Verb(int id, Priority priority, ToLongFunction<TimeUnit> expiration, Stage stage, Supplier<? extends IVersionedAsymmetricSerializer<?, ?>> serializer, Supplier<? extends IVerbHandler<?>> handler)
+ {
+ this(id, priority, expiration, stage, serializer, handler, null);
+ }
+
+ Verb(int id, Priority priority, ToLongFunction<TimeUnit> expiration, Stage stage, Supplier<? extends IVersionedAsymmetricSerializer<?, ?>> serializer, Supplier<? extends IVerbHandler<?>> handler, Verb responseVerb)
+ {
+ this(NORMAL, id, priority, expiration, stage, serializer, handler, responseVerb);
+ }
+
+ Verb(Kind kind, int id, Priority priority, ToLongFunction<TimeUnit> expiration, Stage stage, Supplier<? extends IVersionedAsymmetricSerializer<?, ?>> serializer, Supplier<? extends IVerbHandler<?>> handler)
+ {
+ this(kind, id, priority, expiration, stage, serializer, handler, null);
+ }
+
+ Verb(Kind kind, int id, Priority priority, ToLongFunction<TimeUnit> expiration, Stage stage, Supplier<? extends IVersionedAsymmetricSerializer<?, ?>> serializer, Supplier<? extends IVerbHandler<?>> handler, Verb responseVerb)
+ {
+ this.stage = stage;
+ if (id < 0)
+ throw new IllegalArgumentException("Verb id must be non-negative, got " + id + " for verb " + name());
+
+ if (kind == CUSTOM)
+ {
+ if (id > MAX_CUSTOM_VERB_ID)
+ throw new AssertionError("Invalid custom verb id " + id + " - we only allow custom ids between 0 and " + MAX_CUSTOM_VERB_ID);
+ this.id = idForCustomVerb(id);
+ }
+ else
+ {
+ if (id > CUSTOM_VERB_START - MAX_CUSTOM_VERB_ID)
+ throw new AssertionError("Invalid verb id " + id + " - we only allow ids between 0 and " + (CUSTOM_VERB_START - MAX_CUSTOM_VERB_ID));
+ this.id = id;
+ }
+ this.priority = priority;
+ this.serializer = serializer;
+ this.handler = handler;
+ this.responseVerb = responseVerb;
+ this.expiration = expiration;
+ this.kind = kind;
+ }
+
+ public <In, Out> IVersionedAsymmetricSerializer<In, Out> serializer()
+ {
+ return (IVersionedAsymmetricSerializer<In, Out>) serializer.get();
+ }
+
+ public <T> IVerbHandler<T> handler()
+ {
+ return (IVerbHandler<T>) handler.get();
+ }
+
+ public long expiresAtNanos(long nowNanos)
+ {
+ return nowNanos + expiresAfterNanos();
+ }
+
+ public long expiresAfterNanos()
+ {
+ return expiration.applyAsLong(NANOSECONDS);
+ }
+
+ // this is a little hacky, but reduces the number of parameters up top
+ public boolean isResponse()
+ {
+ return handler.get() == ResponseVerbHandler.instance;
+ }
+
+ Verb toPre40Verb()
+ {
+ if (!isResponse())
+ return this;
+ if (priority == P0)
+ return INTERNAL_RSP;
+ return REQUEST_RSP;
+ }
+
+ @VisibleForTesting
+ Supplier<? extends IVerbHandler<?>> unsafeSetHandler(Supplier<? extends IVerbHandler<?>> handler) throws NoSuchFieldException, IllegalAccessException
+ {
+ Supplier<? extends IVerbHandler<?>> original = this.handler;
+ Field field = Verb.class.getDeclaredField("handler");
+ field.setAccessible(true);
+ Field modifiers = Field.class.getDeclaredField("modifiers");
+ modifiers.setAccessible(true);
+ modifiers.setInt(field, field.getModifiers() & ~Modifier.FINAL);
+ field.set(this, handler);
+ return original;
+ }
+
+ @VisibleForTesting
+ Supplier<? extends IVersionedAsymmetricSerializer<?, ?>> unsafeSetSerializer(Supplier<? extends IVersionedAsymmetricSerializer<?, ?>> serializer) throws NoSuchFieldException, IllegalAccessException
+ {
+ Supplier<? extends IVersionedAsymmetricSerializer<?, ?>> original = this.serializer;
+ Field field = Verb.class.getDeclaredField("serializer");
+ field.setAccessible(true);
+ Field modifiers = Field.class.getDeclaredField("modifiers");
+ modifiers.setAccessible(true);
+ modifiers.setInt(field, field.getModifiers() & ~Modifier.FINAL);
+ field.set(this, serializer);
+ return original;
+ }
+
+ @VisibleForTesting
+ ToLongFunction<TimeUnit> unsafeSetExpiration(ToLongFunction<TimeUnit> expiration) throws NoSuchFieldException, IllegalAccessException
+ {
+ ToLongFunction<TimeUnit> original = this.expiration;
+ Field field = Verb.class.getDeclaredField("expiration");
+ field.setAccessible(true);
+ Field modifiers = Field.class.getDeclaredField("modifiers");
+ modifiers.setAccessible(true);
+ modifiers.setInt(field, field.getModifiers() & ~Modifier.FINAL);
+ field.set(this, expiration);
+ return original;
+ }
+
+ // This is the largest number we can store in 2 bytes using VIntCoding (1 bit per byte is used to indicate if there is more data coming).
+ // When generating ids we count *down* from this number
+ private static final int CUSTOM_VERB_START = (1 << (7 * 2)) - 1;
+
+ // Sanity check for the custom verb ids - avoids someone mistakenly adding a custom verb id close to the normal verbs which
+ // could cause a conflict later when new normal verbs are added.
+ private static final int MAX_CUSTOM_VERB_ID = 1000;
+
+ private static final Verb[] idToVerbMap;
+ private static final Verb[] idToCustomVerbMap;
+ private static final int minCustomId;
+
+ static
+ {
+ Verb[] verbs = values();
+ int max = -1;
+ int minCustom = Integer.MAX_VALUE;
+ for (Verb v : verbs)
+ {
+ switch (v.kind)
+ {
+ case NORMAL:
+ max = Math.max(v.id, max);
+ break;
+ case CUSTOM:
+ minCustom = Math.min(v.id, minCustom);
+ break;
+ default:
+ throw new AssertionError("Unsupported Verb Kind: " + v.kind + " for verb " + v);
+ }
+ }
+ minCustomId = minCustom;
+
+ if (minCustom <= max)
+ throw new IllegalStateException("Overlapping verb ids are not allowed");
+
+ Verb[] idMap = new Verb[max + 1];
+ int customCount = minCustom < Integer.MAX_VALUE ? CUSTOM_VERB_START - minCustom : 0;
+ Verb[] customIdMap = new Verb[customCount + 1];
+ for (Verb v : verbs)
+ {
+ switch (v.kind)
+ {
+ case NORMAL:
+ if (idMap[v.id] != null)
+ throw new IllegalArgumentException("cannot have two verbs that map to the same id: " + v + " and " + idMap[v.id]);
+ idMap[v.id] = v;
+ break;
+ case CUSTOM:
+ int relativeId = idForCustomVerb(v.id);
+ if (customIdMap[relativeId] != null)
+ throw new IllegalArgumentException("cannot have two custom verbs that map to the same id: " + v + " and " + customIdMap[relativeId]);
+ customIdMap[relativeId] = v;
+ break;
+ default:
+ throw new AssertionError("Unsupported Verb Kind: " + v.kind + " for verb " + v);
+ }
+ }
+
+ idToVerbMap = idMap;
+ idToCustomVerbMap = customIdMap;
+ }
+
+ public static Verb fromId(int id)
+ {
+ Verb[] verbs = idToVerbMap;
+ if (id >= minCustomId)
+ {
+ id = idForCustomVerb(id);
+ verbs = idToCustomVerbMap;
+ }
+ Verb verb = id >= 0 && id < verbs.length ? verbs[id] : null;
+ if (verb == null)
+ throw new IllegalArgumentException("Unknown verb id " + id);
+ return verb;
+ }
+
+ /**
+ * calculate an id for a custom verb
+ */
+ private static int idForCustomVerb(int id)
+ {
+ return CUSTOM_VERB_START - id;
+ }
+}
+
+@SuppressWarnings("unused")
+class VerbTimeouts
+{
+ static final ToLongFunction<TimeUnit> rpcTimeout = DatabaseDescriptor::getRpcTimeout;
+ static final ToLongFunction<TimeUnit> writeTimeout = DatabaseDescriptor::getWriteRpcTimeout;
+ static final ToLongFunction<TimeUnit> readTimeout = DatabaseDescriptor::getReadRpcTimeout;
+ static final ToLongFunction<TimeUnit> rangeTimeout = DatabaseDescriptor::getRangeRpcTimeout;
+ static final ToLongFunction<TimeUnit> counterTimeout = DatabaseDescriptor::getCounterWriteRpcTimeout;
+ static final ToLongFunction<TimeUnit> truncateTimeout = DatabaseDescriptor::getTruncateRpcTimeout;
+ static final ToLongFunction<TimeUnit> pingTimeout = DatabaseDescriptor::getPingTimeout;
+ static final ToLongFunction<TimeUnit> longTimeout = units -> Math.max(DatabaseDescriptor.getRpcTimeout(units), units.convert(5L, TimeUnit.MINUTES));
+ static final ToLongFunction<TimeUnit> noTimeout = units -> { throw new IllegalStateException(); };
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/net/WriteCallbackInfo.java b/src/java/org/apache/cassandra/net/WriteCallbackInfo.java
deleted file mode 100644
index 9ecc385..0000000
--- a/src/java/org/apache/cassandra/net/WriteCallbackInfo.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.cassandra.net;
-
-import java.net.InetAddress;
-
-import org.apache.cassandra.db.ConsistencyLevel;
-import org.apache.cassandra.db.Mutation;
-import org.apache.cassandra.io.IVersionedSerializer;
-import org.apache.cassandra.service.StorageProxy;
-import org.apache.cassandra.service.paxos.Commit;
-import org.apache.cassandra.utils.FBUtilities;
-
-public class WriteCallbackInfo extends CallbackInfo
-{
- // either a Mutation, or a Paxos Commit (MessageOut)
- private final Object mutation;
-
- public WriteCallbackInfo(InetAddress target,
- IAsyncCallback callback,
- MessageOut message,
- IVersionedSerializer<?> serializer,
- ConsistencyLevel consistencyLevel,
- boolean allowHints)
- {
- super(target, callback, serializer, true);
- assert message != null;
- this.mutation = shouldHint(allowHints, message, consistencyLevel);
- //Local writes shouldn't go through messaging service (https://issues.apache.org/jira/browse/CASSANDRA-10477)
- assert (!target.equals(FBUtilities.getBroadcastAddress()));
- }
-
- public boolean shouldHint()
- {
- return mutation != null && StorageProxy.shouldHint(target);
- }
-
- public Mutation mutation()
- {
- return getMutation(mutation);
- }
-
- private static Mutation getMutation(Object object)
- {
- assert object instanceof Commit || object instanceof Mutation : object;
- return object instanceof Commit ? ((Commit) object).makeMutation()
- : (Mutation) object;
- }
-
- private static Object shouldHint(boolean allowHints, MessageOut sentMessage, ConsistencyLevel consistencyLevel)
- {
- return allowHints
- && sentMessage.verb != MessagingService.Verb.COUNTER_MUTATION
- && consistencyLevel != ConsistencyLevel.ANY
- ? sentMessage.payload : null;
- }
-
-}
diff --git a/src/java/org/apache/cassandra/notifications/SSTableAddedNotification.java b/src/java/org/apache/cassandra/notifications/SSTableAddedNotification.java
index 56d6130..9c95a18 100644
--- a/src/java/org/apache/cassandra/notifications/SSTableAddedNotification.java
+++ b/src/java/org/apache/cassandra/notifications/SSTableAddedNotification.java
@@ -17,13 +17,46 @@
*/
package org.apache.cassandra.notifications;
+import java.util.Optional;
+
+import javax.annotation.Nullable;
+
+import org.apache.cassandra.db.Memtable;
import org.apache.cassandra.io.sstable.format.SSTableReader;
+/**
+ * Notification sent after SSTables are added to their {@link org.apache.cassandra.db.ColumnFamilyStore}.
+ */
public class SSTableAddedNotification implements INotification
{
+ /** The added SSTables */
public final Iterable<SSTableReader> added;
- public SSTableAddedNotification(Iterable<SSTableReader> added)
+
+ /** The memtable from which the tables come when they have been added due to a flush, {@code null} otherwise. */
+ @Nullable
+ private final Memtable memtable;
+
+ /**
+ * Creates a new {@code SSTableAddedNotification} for the specified SSTables and optional memtable.
+ *
+ * @param added the added SSTables
+ * @param memtable the memtable from which the tables come when they have been added due to a memtable flush,
+ * or {@code null} if they don't come from a flush
+ */
+ public SSTableAddedNotification(Iterable<SSTableReader> added, @Nullable Memtable memtable)
{
this.added = added;
+ this.memtable = memtable;
+ }
+
+ /**
+ * Returns the memtable from which the tables come when they have been added due to a memtable flush. If not, an
+ * empty Optional should be returned.
+ *
+ * @return the origin memtable in case of a flush, {@link Optional#empty()} otherwise
+ */
+ public Optional<Memtable> memtable()
+ {
+ return Optional.ofNullable(memtable);
}
}
diff --git a/src/java/org/apache/cassandra/notifications/SSTableMetadataChanged.java b/src/java/org/apache/cassandra/notifications/SSTableMetadataChanged.java
new file mode 100644
index 0000000..83cfe60
--- /dev/null
+++ b/src/java/org/apache/cassandra/notifications/SSTableMetadataChanged.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.notifications;
+
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
+
+public class SSTableMetadataChanged implements INotification
+{
+ public final SSTableReader sstable;
+ public final StatsMetadata oldMetadata;
+
+ public SSTableMetadataChanged(SSTableReader levelChanged, StatsMetadata oldMetadata)
+ {
+ this.sstable = levelChanged;
+ this.oldMetadata = oldMetadata;
+ }
+}
diff --git a/src/java/org/apache/cassandra/repair/AnticompactionTask.java b/src/java/org/apache/cassandra/repair/AnticompactionTask.java
deleted file mode 100644
index 6e6bb65..0000000
--- a/src/java/org/apache/cassandra/repair/AnticompactionTask.java
+++ /dev/null
@@ -1,174 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.repair;
-
-import java.io.IOException;
-import java.net.InetAddress;
-import java.util.Collection;
-import java.util.UUID;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-import com.google.common.util.concurrent.AbstractFuture;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.db.SystemKeyspace;
-import org.apache.cassandra.dht.Range;
-import org.apache.cassandra.dht.Token;
-import org.apache.cassandra.exceptions.RequestFailureReason;
-import org.apache.cassandra.gms.ApplicationState;
-import org.apache.cassandra.gms.EndpointState;
-import org.apache.cassandra.gms.FailureDetector;
-import org.apache.cassandra.gms.IEndpointStateChangeSubscriber;
-import org.apache.cassandra.gms.IFailureDetectionEventListener;
-import org.apache.cassandra.gms.VersionedValue;
-import org.apache.cassandra.net.IAsyncCallbackWithFailure;
-import org.apache.cassandra.net.MessageIn;
-import org.apache.cassandra.net.MessagingService;
-import org.apache.cassandra.repair.messages.AnticompactionRequest;
-import org.apache.cassandra.utils.CassandraVersion;
-
-public class AnticompactionTask extends AbstractFuture<InetAddress> implements Runnable, IEndpointStateChangeSubscriber,
- IFailureDetectionEventListener
-{
- /*
- * Version that anticompaction response is not supported up to.
- * If Cassandra version is more than this, we need to wait for anticompaction response.
- */
- private static final CassandraVersion VERSION_CHECKER = new CassandraVersion("2.1.5");
- private static Logger logger = LoggerFactory.getLogger(AnticompactionTask.class);
-
- private final UUID parentSession;
- private final InetAddress neighbor;
- private final Collection<Range<Token>> successfulRanges;
- private final AtomicBoolean isFinished = new AtomicBoolean(false);
-
- public AnticompactionTask(UUID parentSession, InetAddress neighbor, Collection<Range<Token>> successfulRanges)
- {
- this.parentSession = parentSession;
- this.neighbor = neighbor;
- this.successfulRanges = successfulRanges;
- }
-
- public void run()
- {
- if (FailureDetector.instance.isAlive(neighbor))
- {
- AnticompactionRequest acr = new AnticompactionRequest(parentSession, successfulRanges);
- CassandraVersion peerVersion = SystemKeyspace.getReleaseVersion(neighbor);
- if (peerVersion != null && peerVersion.compareTo(VERSION_CHECKER) > 0)
- {
- MessagingService.instance().sendRR(acr.createMessage(), neighbor, new AnticompactionCallback(this), TimeUnit.DAYS.toMillis(1), true);
- }
- else
- {
- // immediately return after sending request
- MessagingService.instance().sendOneWay(acr.createMessage(), neighbor);
- maybeSetResult(neighbor);
- }
- }
- else
- {
- maybeSetException(new IOException(neighbor + " is down"));
- }
- }
-
- private boolean maybeSetException(Throwable t)
- {
- if (isFinished.compareAndSet(false, true))
- {
- setException(t);
- return true;
- }
- return false;
- }
-
- private boolean maybeSetResult(InetAddress o)
- {
- if (isFinished.compareAndSet(false, true))
- {
- set(o);
- return true;
- }
- return false;
- }
-
- /**
- * Callback for antitcompaction request. Run on INTERNAL_RESPONSE stage.
- */
- public class AnticompactionCallback implements IAsyncCallbackWithFailure
- {
- final AnticompactionTask task;
-
- public AnticompactionCallback(AnticompactionTask task)
- {
- this.task = task;
- }
-
- public void response(MessageIn msg)
- {
- maybeSetResult(msg.from);
- }
-
- public boolean isLatencyForSnitch()
- {
- return false;
- }
-
- public void onFailure(InetAddress from, RequestFailureReason failureReason)
- {
- maybeSetException(new RuntimeException("Anticompaction failed or timed out in " + from));
- }
- }
-
- public void onJoin(InetAddress endpoint, EndpointState epState) {}
- public void beforeChange(InetAddress endpoint, EndpointState currentState, ApplicationState newStateKey, VersionedValue newValue) {}
- public void onChange(InetAddress endpoint, ApplicationState state, VersionedValue value) {}
- public void onAlive(InetAddress endpoint, EndpointState state) {}
- public void onDead(InetAddress endpoint, EndpointState state) {}
-
- public void onRemove(InetAddress endpoint)
- {
- convict(endpoint, Double.MAX_VALUE);
- }
-
- public void onRestart(InetAddress endpoint, EndpointState epState)
- {
- convict(endpoint, Double.MAX_VALUE);
- }
-
- public void convict(InetAddress endpoint, double phi)
- {
- if (!neighbor.equals(endpoint))
- return;
-
- // We want a higher confidence in the failure detection than usual because failing a repair wrongly has a high cost.
- if (phi < 2 * DatabaseDescriptor.getPhiConvictThreshold())
- return;
-
- Exception exception = new IOException(String.format("Endpoint %s died during anti-compaction.", endpoint));
- if (maybeSetException(exception))
- {
- // Though unlikely, it is possible to arrive here multiple time and we want to avoid print an error message twice
- logger.error("[repair #{}] Endpoint {} died during anti-compaction", endpoint, parentSession, exception);
- }
- }
-}
diff --git a/src/java/org/apache/cassandra/repair/AsymmetricRemoteSyncTask.java b/src/java/org/apache/cassandra/repair/AsymmetricRemoteSyncTask.java
new file mode 100644
index 0000000..cf6d84b
--- /dev/null
+++ b/src/java/org/apache/cassandra/repair/AsymmetricRemoteSyncTask.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.repair;
+
+import java.util.List;
+
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.exceptions.RepairException;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.net.Message;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.repair.messages.AsymmetricSyncRequest;
+import org.apache.cassandra.streaming.PreviewKind;
+import org.apache.cassandra.streaming.SessionSummary;
+import org.apache.cassandra.tracing.Tracing;
+import org.apache.cassandra.utils.FBUtilities;
+
+import static org.apache.cassandra.net.Verb.ASYMMETRIC_SYNC_REQ;
+
+/**
+ * AsymmetricRemoteSyncTask sends {@link AsymmetricSyncRequest} to target node to repair(stream)
+ * data with other target replica.
+ *
+ * When AsymmetricRemoteSyncTask receives SyncComplete from the target, task completes.
+ */
+public class AsymmetricRemoteSyncTask extends SyncTask implements CompletableRemoteSyncTask
+{
+ public AsymmetricRemoteSyncTask(RepairJobDesc desc, InetAddressAndPort to, InetAddressAndPort from, List<Range<Token>> differences, PreviewKind previewKind)
+ {
+ super(desc, to, from, differences, previewKind);
+ }
+
+ public void startSync()
+ {
+ InetAddressAndPort local = FBUtilities.getBroadcastAddressAndPort();
+ AsymmetricSyncRequest request = new AsymmetricSyncRequest(desc, local, nodePair.coordinator, nodePair.peer, rangesToSync, previewKind);
+ String message = String.format("Forwarding streaming repair of %d ranges to %s (to be streamed with %s)", request.ranges.size(), request.fetchingNode, request.fetchFrom);
+ Tracing.traceRepair(message);
+ MessagingService.instance().send(Message.out(ASYMMETRIC_SYNC_REQ, request), request.fetchingNode);
+ }
+
+ public void syncComplete(boolean success, List<SessionSummary> summaries)
+ {
+ if (success)
+ {
+ set(stat.withSummaries(summaries));
+ }
+ else
+ {
+ setException(new RepairException(desc, previewKind, String.format("Sync failed between %s and %s", nodePair.coordinator, nodePair.peer)));
+ }
+ }
+
+ @Override
+ public String toString()
+ {
+ return "AsymmetricRemoteSyncTask{" +
+ "rangesToSync=" + rangesToSync +
+ ", nodePair=" + nodePair +
+ '}';
+ }
+}
diff --git a/src/java/org/apache/cassandra/repair/CommonRange.java b/src/java/org/apache/cassandra/repair/CommonRange.java
new file mode 100644
index 0000000..dab77c5
--- /dev/null
+++ b/src/java/org/apache/cassandra/repair/CommonRange.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.repair;
+
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Set;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableSet;
+
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.locator.InetAddressAndPort;
+
+/**
+ * Groups ranges with identical endpoints/transient endpoints
+ */
+public class CommonRange
+{
+ public final ImmutableSet<InetAddressAndPort> endpoints;
+ public final ImmutableSet<InetAddressAndPort> transEndpoints;
+ public final Collection<Range<Token>> ranges;
+
+ public CommonRange(Set<InetAddressAndPort> endpoints, Set<InetAddressAndPort> transEndpoints, Collection<Range<Token>> ranges)
+ {
+ Preconditions.checkArgument(endpoints != null && !endpoints.isEmpty(), "Endpoints can not be empty");
+ Preconditions.checkArgument(transEndpoints != null, "Transient endpoints can not be null");
+ Preconditions.checkArgument(endpoints.containsAll(transEndpoints), "transEndpoints must be a subset of endpoints");
+ Preconditions.checkArgument(ranges != null && !ranges.isEmpty(), "Ranges can not be empty");
+
+ this.endpoints = ImmutableSet.copyOf(endpoints);
+ this.transEndpoints = ImmutableSet.copyOf(transEndpoints);
+ this.ranges = new ArrayList<>(ranges);
+ }
+
+ public boolean matchesEndpoints(Set<InetAddressAndPort> endpoints, Set<InetAddressAndPort> transEndpoints)
+ {
+ // Use strict equality here, as worst thing that can happen is we generate one more stream
+ return this.endpoints.equals(endpoints) && this.transEndpoints.equals(transEndpoints);
+ }
+
+ public boolean equals(Object o)
+ {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ CommonRange that = (CommonRange) o;
+
+ if (!endpoints.equals(that.endpoints)) return false;
+ if (!transEndpoints.equals(that.transEndpoints)) return false;
+ return ranges.equals(that.ranges);
+ }
+
+ public int hashCode()
+ {
+ int result = endpoints.hashCode();
+ result = 31 * result + transEndpoints.hashCode();
+ result = 31 * result + ranges.hashCode();
+ return result;
+ }
+
+ public String toString()
+ {
+ return "CommonRange{" +
+ "endpoints=" + endpoints +
+ ", transEndpoints=" + transEndpoints +
+ ", ranges=" + ranges +
+ '}';
+ }
+}
diff --git a/src/java/org/apache/cassandra/repair/CompletableRemoteSyncTask.java b/src/java/org/apache/cassandra/repair/CompletableRemoteSyncTask.java
new file mode 100644
index 0000000..c4fe6c8
--- /dev/null
+++ b/src/java/org/apache/cassandra/repair/CompletableRemoteSyncTask.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.repair;
+
+import java.util.List;
+
+import org.apache.cassandra.streaming.SessionSummary;
+
+public interface CompletableRemoteSyncTask
+{
+ void syncComplete(boolean success, List<SessionSummary> summaries);
+}
diff --git a/src/java/org/apache/cassandra/repair/KeyspaceRepairManager.java b/src/java/org/apache/cassandra/repair/KeyspaceRepairManager.java
new file mode 100644
index 0000000..0739f10
--- /dev/null
+++ b/src/java/org/apache/cassandra/repair/KeyspaceRepairManager.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.repair;
+
+import java.util.Collection;
+import java.util.UUID;
+import java.util.concurrent.ExecutorService;
+import java.util.function.BooleanSupplier;
+
+import com.google.common.util.concurrent.ListenableFuture;
+
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.locator.RangesAtEndpoint;
+
+/**
+ * Keyspace level hook for repair.
+ */
+public interface KeyspaceRepairManager
+{
+ /**
+ * Isolate the unrepaired ranges of the given tables, and make referenceable by session id. Until each table has
+ * been notified that the repair session has been completed, the data associated with the given session id must
+ * not be combined with repaired or unrepaired data, or data from other repair sessions.
+ */
+ ListenableFuture prepareIncrementalRepair(UUID sessionID,
+ Collection<ColumnFamilyStore> tables,
+ RangesAtEndpoint tokenRanges,
+ ExecutorService executor,
+ BooleanSupplier isCancelled);
+}
diff --git a/src/java/org/apache/cassandra/repair/LocalSyncTask.java b/src/java/org/apache/cassandra/repair/LocalSyncTask.java
index 57a3551..5916401 100644
--- a/src/java/org/apache/cassandra/repair/LocalSyncTask.java
+++ b/src/java/org/apache/cassandra/repair/LocalSyncTask.java
@@ -17,24 +17,30 @@
*/
package org.apache.cassandra.repair;
-import java.net.InetAddress;
+import java.util.Collections;
import java.util.List;
+import java.util.UUID;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.cassandra.db.SystemKeyspace;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
-import org.apache.cassandra.service.ActiveRepairService;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.RangesAtEndpoint;
+import org.apache.cassandra.streaming.PreviewKind;
import org.apache.cassandra.streaming.ProgressInfo;
import org.apache.cassandra.streaming.StreamEvent;
import org.apache.cassandra.streaming.StreamEventHandler;
+import org.apache.cassandra.streaming.StreamOperation;
import org.apache.cassandra.streaming.StreamPlan;
import org.apache.cassandra.streaming.StreamState;
import org.apache.cassandra.tracing.TraceState;
import org.apache.cassandra.tracing.Tracing;
import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.MerkleTrees;
/**
* LocalSyncTask performs streaming between local(coordinator) node and remote replica.
@@ -45,48 +51,72 @@
private static final Logger logger = LoggerFactory.getLogger(LocalSyncTask.class);
- private final long repairedAt;
+ private final UUID pendingRepair;
- private final boolean pullRepair;
+ @VisibleForTesting
+ public final boolean requestRanges;
+ @VisibleForTesting
+ public final boolean transferRanges;
- public LocalSyncTask(RepairJobDesc desc, InetAddress firstEndpoint, InetAddress secondEndpoint, List<Range<Token>> rangesToSync, long repairedAt, boolean pullRepair)
+ public LocalSyncTask(RepairJobDesc desc, InetAddressAndPort local, InetAddressAndPort remote,
+ List<Range<Token>> diff, UUID pendingRepair,
+ boolean requestRanges, boolean transferRanges, PreviewKind previewKind)
{
- super(desc, firstEndpoint, secondEndpoint, rangesToSync);
- this.repairedAt = repairedAt;
- this.pullRepair = pullRepair;
+ super(desc, local, remote, diff, previewKind);
+ Preconditions.checkArgument(requestRanges || transferRanges, "Nothing to do in a sync job");
+ Preconditions.checkArgument(local.equals(FBUtilities.getBroadcastAddressAndPort()));
+
+ this.pendingRepair = pendingRepair;
+ this.requestRanges = requestRanges;
+ this.transferRanges = transferRanges;
+ }
+
+ @VisibleForTesting
+ StreamPlan createStreamPlan()
+ {
+ InetAddressAndPort remote = nodePair.peer;
+
+ StreamPlan plan = new StreamPlan(StreamOperation.REPAIR, 1, false, pendingRepair, previewKind)
+ .listeners(this)
+ .flushBeforeTransfer(pendingRepair == null);
+
+ if (requestRanges)
+ {
+ // see comment on RangesAtEndpoint.toDummyList for why we synthesize replicas here
+ plan.requestRanges(remote, desc.keyspace, RangesAtEndpoint.toDummyList(rangesToSync),
+ RangesAtEndpoint.toDummyList(Collections.emptyList()), desc.columnFamily);
+ }
+
+ if (transferRanges)
+ {
+ // send ranges to the remote node if we are not performing a pull repair
+ // see comment on RangesAtEndpoint.toDummyList for why we synthesize replicas here
+ plan.transferRanges(remote, desc.keyspace, RangesAtEndpoint.toDummyList(rangesToSync), desc.columnFamily);
+ }
+
+ return plan;
}
/**
* Starts sending/receiving our list of differences to/from the remote endpoint: creates a callback
* that will be called out of band once the streams complete.
*/
- protected void startSync(List<Range<Token>> differences)
+ @Override
+ protected void startSync()
{
- InetAddress local = FBUtilities.getBroadcastAddress();
- // We can take anyone of the node as source or destination, however if one is localhost, we put at source to avoid a forwarding
- InetAddress dst = secondEndpoint.equals(local) ? firstEndpoint : secondEndpoint;
- InetAddress preferred = SystemKeyspace.getPreferredIP(dst);
+ InetAddressAndPort remote = nodePair.peer;
- String message = String.format("Performing streaming repair of %d ranges with %s", differences.size(), dst);
- logger.info("[repair #{}] {}", desc.sessionId, message);
- boolean isIncremental = false;
- if (desc.parentSessionId != null)
- {
- ActiveRepairService.ParentRepairSession prs = ActiveRepairService.instance.getParentRepairSession(desc.parentSessionId);
- isIncremental = prs.isIncremental;
- }
+ String message = String.format("Performing streaming repair of %d ranges with %s", rangesToSync.size(), remote);
+ logger.info("{} {}", previewKind.logPrefix(desc.sessionId), message);
Tracing.traceRepair(message);
- StreamPlan plan = new StreamPlan("Repair", repairedAt, 1, false, isIncremental, false).listeners(this)
- .flushBeforeTransfer(true)
- // request ranges from the remote node
- .requestRanges(dst, preferred, desc.keyspace, differences, desc.columnFamily);
- if (!pullRepair)
- {
- // send ranges to the remote node if we are not performing a pull repair
- plan.transferRanges(dst, preferred, desc.keyspace, differences, desc.columnFamily);
- }
- plan.execute();
+ createStreamPlan().execute();
+ }
+
+ @Override
+ public boolean isLocal()
+ {
+ return true;
}
public void handleStreamEvent(StreamEvent event)
@@ -117,14 +147,27 @@
public void onSuccess(StreamState result)
{
- String message = String.format("Sync complete using session %s between %s and %s on %s", desc.sessionId, firstEndpoint, secondEndpoint, desc.columnFamily);
- logger.info("[repair #{}] {}", desc.sessionId, message);
+ String message = String.format("Sync complete using session %s between %s and %s on %s", desc.sessionId, nodePair.coordinator, nodePair.peer, desc.columnFamily);
+ logger.info("{} {}", previewKind.logPrefix(desc.sessionId), message);
Tracing.traceRepair(message);
- set(stat);
+ set(stat.withSummaries(result.createSummaries()));
+ finished();
}
public void onFailure(Throwable t)
{
setException(t);
+ finished();
+ }
+
+ @Override
+ public String toString()
+ {
+ return "LocalSyncTask{" +
+ "requestRanges=" + requestRanges +
+ ", transferRanges=" + transferRanges +
+ ", rangesToSync=" + rangesToSync +
+ ", nodePair=" + nodePair +
+ '}';
}
}
diff --git a/src/java/org/apache/cassandra/repair/NodePair.java b/src/java/org/apache/cassandra/repair/NodePair.java
deleted file mode 100644
index a73c61a..0000000
--- a/src/java/org/apache/cassandra/repair/NodePair.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.repair;
-
-import java.io.IOException;
-import java.net.InetAddress;
-
-import com.google.common.base.Objects;
-
-import org.apache.cassandra.io.IVersionedSerializer;
-import org.apache.cassandra.io.util.DataInputPlus;
-import org.apache.cassandra.io.util.DataOutputPlus;
-import org.apache.cassandra.net.CompactEndpointSerializationHelper;
-
-/**
- * NodePair is used for repair message body to indicate the pair of nodes.
- *
- * @since 2.0
- */
-public class NodePair
-{
- public static IVersionedSerializer<NodePair> serializer = new NodePairSerializer();
-
- public final InetAddress endpoint1;
- public final InetAddress endpoint2;
-
- public NodePair(InetAddress endpoint1, InetAddress endpoint2)
- {
- this.endpoint1 = endpoint1;
- this.endpoint2 = endpoint2;
- }
-
- @Override
- public boolean equals(Object o)
- {
- if (this == o) return true;
- if (o == null || getClass() != o.getClass()) return false;
-
- NodePair nodePair = (NodePair) o;
- return endpoint1.equals(nodePair.endpoint1) && endpoint2.equals(nodePair.endpoint2);
- }
-
- @Override
- public int hashCode()
- {
- return Objects.hashCode(endpoint1, endpoint2);
- }
-
- public static class NodePairSerializer implements IVersionedSerializer<NodePair>
- {
- public void serialize(NodePair nodePair, DataOutputPlus out, int version) throws IOException
- {
- CompactEndpointSerializationHelper.serialize(nodePair.endpoint1, out);
- CompactEndpointSerializationHelper.serialize(nodePair.endpoint2, out);
- }
-
- public NodePair deserialize(DataInputPlus in, int version) throws IOException
- {
- InetAddress ep1 = CompactEndpointSerializationHelper.deserialize(in);
- InetAddress ep2 = CompactEndpointSerializationHelper.deserialize(in);
- return new NodePair(ep1, ep2);
- }
-
- public long serializedSize(NodePair nodePair, int version)
- {
- return 2 * CompactEndpointSerializationHelper.serializedSize(nodePair.endpoint1);
- }
- }
-}
diff --git a/src/java/org/apache/cassandra/repair/RemoteSyncTask.java b/src/java/org/apache/cassandra/repair/RemoteSyncTask.java
deleted file mode 100644
index 5af815a..0000000
--- a/src/java/org/apache/cassandra/repair/RemoteSyncTask.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.repair;
-
-import java.net.InetAddress;
-import java.util.List;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.cassandra.dht.Range;
-import org.apache.cassandra.dht.Token;
-import org.apache.cassandra.exceptions.RepairException;
-import org.apache.cassandra.net.MessagingService;
-import org.apache.cassandra.repair.messages.SyncRequest;
-import org.apache.cassandra.tracing.Tracing;
-import org.apache.cassandra.utils.FBUtilities;
-
-/**
- * RemoteSyncTask sends {@link SyncRequest} to remote(non-coordinator) node
- * to repair(stream) data with other replica.
- *
- * When RemoteSyncTask receives SyncComplete from remote node, task completes.
- */
-public class RemoteSyncTask extends SyncTask
-{
- private static final Logger logger = LoggerFactory.getLogger(RemoteSyncTask.class);
-
- public RemoteSyncTask(RepairJobDesc desc, InetAddress firstEndpoint, InetAddress secondEndpoint, List<Range<Token>> rangesToSync)
- {
- super(desc, firstEndpoint, secondEndpoint, rangesToSync);
- }
-
- protected void startSync(List<Range<Token>> differences)
- {
- InetAddress local = FBUtilities.getBroadcastAddress();
- SyncRequest request = new SyncRequest(desc, local, firstEndpoint, secondEndpoint, differences);
- String message = String.format("Forwarding streaming repair of %d ranges to %s (to be streamed with %s)", request.ranges.size(), request.src, request.dst);
- logger.info("[repair #{}] {}", desc.sessionId, message);
- Tracing.traceRepair(message);
- MessagingService.instance().sendOneWay(request.createMessage(), request.src);
- }
-
- public void syncComplete(boolean success)
- {
- if (success)
- {
- set(stat);
- }
- else
- {
- setException(new RepairException(desc, String.format("Sync failed between %s and %s", firstEndpoint, secondEndpoint)));
- }
- }
-}
diff --git a/src/java/org/apache/cassandra/repair/RepairJob.java b/src/java/org/apache/cassandra/repair/RepairJob.java
index 6f89a86..16eb325 100644
--- a/src/java/org/apache/cassandra/repair/RepairJob.java
+++ b/src/java/org/apache/cassandra/repair/RepairJob.java
@@ -17,16 +17,27 @@
*/
package org.apache.cassandra.repair;
-import java.net.InetAddress;
import java.util.*;
+import java.util.function.Predicate;
+import java.util.function.Function;
+import java.util.stream.Collectors;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
import com.google.common.util.concurrent.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.repair.asymmetric.DifferenceHolder;
+import org.apache.cassandra.repair.asymmetric.HostDifferences;
+import org.apache.cassandra.repair.asymmetric.PreferedNodeFilter;
+import org.apache.cassandra.repair.asymmetric.ReduceHelper;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.streaming.PreviewKind;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.tracing.Tracing;
@@ -39,12 +50,11 @@
*/
public class RepairJob extends AbstractFuture<RepairResult> implements Runnable
{
- private static Logger logger = LoggerFactory.getLogger(RepairJob.class);
+ private static final Logger logger = LoggerFactory.getLogger(RepairJob.class);
private final RepairSession session;
private final RepairJobDesc desc;
private final RepairParallelism parallelismDegree;
- private final long repairedAt;
private final ListeningExecutorService taskExecutor;
/**
@@ -56,12 +66,24 @@
public RepairJob(RepairSession session, String columnFamily)
{
this.session = session;
- this.desc = new RepairJobDesc(session.parentRepairSession, session.getId(), session.keyspace, columnFamily, session.getRanges());
- this.repairedAt = session.repairedAt;
+ this.desc = new RepairJobDesc(session.parentRepairSession, session.getId(), session.keyspace, columnFamily, session.commonRange.ranges);
this.taskExecutor = session.taskExecutor;
this.parallelismDegree = session.parallelismDegree;
}
+ public int getNowInSeconds()
+ {
+ int nowInSeconds = FBUtilities.nowInSeconds();
+ if (session.previewKind == PreviewKind.REPAIRED)
+ {
+ return nowInSeconds + DatabaseDescriptor.getValidationPreviewPurgeHeadStartInSec();
+ }
+ else
+ {
+ return nowInSeconds;
+ }
+ }
+
/**
* Runs repair job.
*
@@ -70,26 +92,39 @@
*/
public void run()
{
- List<InetAddress> allEndpoints = new ArrayList<>(session.endpoints);
- allEndpoints.add(FBUtilities.getBroadcastAddress());
+ Keyspace ks = Keyspace.open(desc.keyspace);
+ ColumnFamilyStore cfs = ks.getColumnFamilyStore(desc.columnFamily);
+ cfs.metric.repairsStarted.inc();
+ List<InetAddressAndPort> allEndpoints = new ArrayList<>(session.commonRange.endpoints);
+ allEndpoints.add(FBUtilities.getBroadcastAddressAndPort());
ListenableFuture<List<TreeResponse>> validations;
// Create a snapshot at all nodes unless we're using pure parallel repairs
if (parallelismDegree != RepairParallelism.PARALLEL)
{
- // Request snapshot to all replica
- List<ListenableFuture<InetAddress>> snapshotTasks = new ArrayList<>(allEndpoints.size());
- for (InetAddress endpoint : allEndpoints)
+ ListenableFuture<List<InetAddressAndPort>> allSnapshotTasks;
+ if (session.isIncremental)
{
- SnapshotTask snapshotTask = new SnapshotTask(desc, endpoint);
- snapshotTasks.add(snapshotTask);
- taskExecutor.execute(snapshotTask);
+ // consistent repair does it's own "snapshotting"
+ allSnapshotTasks = Futures.immediateFuture(allEndpoints);
}
- // When all snapshot complete, send validation requests
- ListenableFuture<List<InetAddress>> allSnapshotTasks = Futures.allAsList(snapshotTasks);
- validations = Futures.transform(allSnapshotTasks, new AsyncFunction<List<InetAddress>, List<TreeResponse>>()
+ else
{
- public ListenableFuture<List<TreeResponse>> apply(List<InetAddress> endpoints)
+ // Request snapshot to all replica
+ List<ListenableFuture<InetAddressAndPort>> snapshotTasks = new ArrayList<>(allEndpoints.size());
+ for (InetAddressAndPort endpoint : allEndpoints)
+ {
+ SnapshotTask snapshotTask = new SnapshotTask(desc, endpoint);
+ snapshotTasks.add(snapshotTask);
+ taskExecutor.execute(snapshotTask);
+ }
+ allSnapshotTasks = Futures.allAsList(snapshotTasks);
+ }
+
+ // When all snapshot complete, send validation requests
+ validations = Futures.transformAsync(allSnapshotTasks, new AsyncFunction<List<InetAddressAndPort>, List<TreeResponse>>()
+ {
+ public ListenableFuture<List<TreeResponse>> apply(List<InetAddressAndPort> endpoints)
{
if (parallelismDegree == RepairParallelism.SEQUENTIAL)
return sendSequentialValidationRequest(endpoints);
@@ -105,21 +140,21 @@
}
// When all validations complete, submit sync tasks
- ListenableFuture<List<SyncStat>> syncResults = Futures.transform(validations, new AsyncFunction<List<TreeResponse>, List<SyncStat>>()
- {
- public ListenableFuture<List<SyncStat>> apply(List<TreeResponse> trees)
- {
- return Futures.allAsList(createSyncTasks(trees, FBUtilities.getLocalAddress()));
- }
- }, taskExecutor);
+ ListenableFuture<List<SyncStat>> syncResults = Futures.transformAsync(validations,
+ session.optimiseStreams && !session.pullRepair ? this::optimisedSyncing : this::standardSyncing,
+ taskExecutor);
// When all sync complete, set the final result
Futures.addCallback(syncResults, new FutureCallback<List<SyncStat>>()
{
public void onSuccess(List<SyncStat> stats)
{
- logger.info("[repair #{}] {} is fully synced", session.getId(), desc.columnFamily);
- SystemDistributedKeyspace.successfulRepairJob(session.getId(), desc.keyspace, desc.columnFamily);
+ if (!session.previewKind.isPreview())
+ {
+ logger.info("{} {}.{} is fully synced", session.previewKind.logPrefix(session.getId()), desc.keyspace, desc.columnFamily);
+ SystemDistributedKeyspace.successfulRepairJob(session.getId(), desc.keyspace, desc.columnFamily);
+ }
+ cfs.metric.repairsCompleted.inc();
set(new RepairResult(desc, stats));
}
@@ -128,19 +163,43 @@
*/
public void onFailure(Throwable t)
{
- logger.warn("[repair #{}] {} sync failed", session.getId(), desc.columnFamily);
- SystemDistributedKeyspace.failedRepairJob(session.getId(), desc.keyspace, desc.columnFamily, t);
+ if (!session.previewKind.isPreview())
+ {
+ logger.warn("{} {}.{} sync failed", session.previewKind.logPrefix(session.getId()), desc.keyspace, desc.columnFamily);
+ SystemDistributedKeyspace.failedRepairJob(session.getId(), desc.keyspace, desc.columnFamily, t);
+ }
+ cfs.metric.repairsCompleted.inc();
setException(t);
}
}, taskExecutor);
-
- // Wait for validation to complete
- Futures.getUnchecked(validations);
}
- @VisibleForTesting
- List<SyncTask> createSyncTasks(List<TreeResponse> trees, InetAddress local)
+ private boolean isTransient(InetAddressAndPort ep)
{
+ return session.commonRange.transEndpoints.contains(ep);
+ }
+
+ private ListenableFuture<List<SyncStat>> standardSyncing(List<TreeResponse> trees)
+ {
+ List<SyncTask> syncTasks = createStandardSyncTasks(desc,
+ trees,
+ FBUtilities.getLocalAddressAndPort(),
+ this::isTransient,
+ session.isIncremental,
+ session.pullRepair,
+ session.previewKind);
+ return executeTasks(syncTasks);
+ }
+
+ static List<SyncTask> createStandardSyncTasks(RepairJobDesc desc,
+ List<TreeResponse> trees,
+ InetAddressAndPort local,
+ Predicate<InetAddressAndPort> isTransient,
+ boolean isIncremental,
+ boolean pullRepair,
+ PreviewKind previewKind)
+ {
+ long startedAt = System.currentTimeMillis();
List<SyncTask> syncTasks = new ArrayList<>();
// We need to difference all trees one against another
for (int i = 0; i < trees.size() - 1; ++i)
@@ -149,46 +208,168 @@
for (int j = i + 1; j < trees.size(); ++j)
{
TreeResponse r2 = trees.get(j);
- SyncTask task;
+
+ // Avoid streming between two tansient replicas
+ if (isTransient.test(r1.endpoint) && isTransient.test(r2.endpoint))
+ continue;
List<Range<Token>> differences = MerkleTrees.difference(r1.trees, r2.trees);
+ // Nothing to do
+ if (differences.isEmpty())
+ continue;
+
+ SyncTask task;
if (r1.endpoint.equals(local) || r2.endpoint.equals(local))
{
- task = new LocalSyncTask(desc, r1.endpoint, r2.endpoint, differences, repairedAt, session.pullRepair);
+ TreeResponse self = r1.endpoint.equals(local) ? r1 : r2;
+ TreeResponse remote = r2.endpoint.equals(local) ? r1 : r2;
+
+ // pull only if local is full
+ boolean requestRanges = !isTransient.test(self.endpoint);
+ // push only if remote is full; additionally check for pull repair
+ boolean transferRanges = !isTransient.test(remote.endpoint) && !pullRepair;
+
+ // Nothing to do
+ if (!requestRanges && !transferRanges)
+ continue;
+
+ task = new LocalSyncTask(desc, self.endpoint, remote.endpoint, differences, isIncremental ? desc.parentSessionId : null,
+ requestRanges, transferRanges, previewKind);
+ }
+ else if (isTransient.test(r1.endpoint) || isTransient.test(r2.endpoint))
+ {
+ // Stream only from transient replica
+ TreeResponse streamFrom = isTransient.test(r1.endpoint) ? r1 : r2;
+ TreeResponse streamTo = isTransient.test(r1.endpoint) ? r2 : r1;
+ task = new AsymmetricRemoteSyncTask(desc, streamTo.endpoint, streamFrom.endpoint, differences, previewKind);
}
else
{
- task = new RemoteSyncTask(desc, r1.endpoint, r2.endpoint, differences);
- // RemoteSyncTask expects SyncComplete message sent back.
- // Register task to RepairSession to receive response.
- session.waitForSync(Pair.create(desc, new NodePair(r1.endpoint, r2.endpoint)), (RemoteSyncTask) task);
+ task = new SymmetricRemoteSyncTask(desc, r1.endpoint, r2.endpoint, differences, previewKind);
}
syncTasks.add(task);
- taskExecutor.submit(task);
+ }
+ trees.get(i).trees.release();
+ }
+ trees.get(trees.size() - 1).trees.release();
+ logger.info("Created {} sync tasks based on {} merkle tree responses for {} (took: {}ms)",
+ syncTasks.size(), trees.size(), desc.parentSessionId, System.currentTimeMillis() - startedAt);
+ return syncTasks;
+ }
+
+ private ListenableFuture<List<SyncStat>> optimisedSyncing(List<TreeResponse> trees)
+ {
+ List<SyncTask> syncTasks = createOptimisedSyncingSyncTasks(desc,
+ trees,
+ FBUtilities.getLocalAddressAndPort(),
+ this::isTransient,
+ this::getDC,
+ session.isIncremental,
+ session.previewKind);
+
+ return executeTasks(syncTasks);
+ }
+
+ @VisibleForTesting
+ ListenableFuture<List<SyncStat>> executeTasks(List<SyncTask> syncTasks)
+ {
+ for (SyncTask task : syncTasks)
+ {
+ if (!task.isLocal())
+ session.trackSyncCompletion(Pair.create(desc, task.nodePair()), (CompletableRemoteSyncTask) task);
+ taskExecutor.submit(task);
+ }
+
+ return Futures.allAsList(syncTasks);
+ }
+
+ static List<SyncTask> createOptimisedSyncingSyncTasks(RepairJobDesc desc,
+ List<TreeResponse> trees,
+ InetAddressAndPort local,
+ Predicate<InetAddressAndPort> isTransient,
+ Function<InetAddressAndPort, String> getDC,
+ boolean isIncremental,
+ PreviewKind previewKind)
+ {
+ long startedAt = System.currentTimeMillis();
+ List<SyncTask> syncTasks = new ArrayList<>();
+ // We need to difference all trees one against another
+ DifferenceHolder diffHolder = new DifferenceHolder(trees);
+
+ logger.debug("diffs = {}", diffHolder);
+ PreferedNodeFilter preferSameDCFilter = (streaming, candidates) ->
+ candidates.stream()
+ .filter(node -> getDC.apply(streaming)
+ .equals(getDC.apply(node)))
+ .collect(Collectors.toSet());
+ ImmutableMap<InetAddressAndPort, HostDifferences> reducedDifferences = ReduceHelper.reduce(diffHolder, preferSameDCFilter);
+
+ for (int i = 0; i < trees.size(); i++)
+ {
+ InetAddressAndPort address = trees.get(i).endpoint;
+
+ // we don't stream to transient replicas
+ if (isTransient.test(address))
+ continue;
+
+ HostDifferences streamsFor = reducedDifferences.get(address);
+ if (streamsFor != null)
+ {
+ Preconditions.checkArgument(streamsFor.get(address).isEmpty(), "We should not fetch ranges from ourselves");
+ for (InetAddressAndPort fetchFrom : streamsFor.hosts())
+ {
+ List<Range<Token>> toFetch = streamsFor.get(fetchFrom);
+ assert !toFetch.isEmpty();
+
+ logger.debug("{} is about to fetch {} from {}", address, toFetch, fetchFrom);
+ SyncTask task;
+ if (address.equals(local))
+ {
+ task = new LocalSyncTask(desc, address, fetchFrom, toFetch, isIncremental ? desc.parentSessionId : null,
+ true, false, previewKind);
+ }
+ else
+ {
+ task = new AsymmetricRemoteSyncTask(desc, address, fetchFrom, toFetch, previewKind);
+ }
+ syncTasks.add(task);
+
+ }
+ }
+ else
+ {
+ logger.debug("Node {} has nothing to stream", address);
}
}
+ logger.info("Created {} optimised sync tasks based on {} merkle tree responses for {} (took: {}ms)",
+ syncTasks.size(), trees.size(), desc.parentSessionId, System.currentTimeMillis() - startedAt);
return syncTasks;
}
+ private String getDC(InetAddressAndPort address)
+ {
+ return DatabaseDescriptor.getEndpointSnitch().getDatacenter(address);
+ }
+
/**
* Creates {@link ValidationTask} and submit them to task executor in parallel.
*
* @param endpoints Endpoint addresses to send validation request
* @return Future that can get all {@link TreeResponse} from replica, if all validation succeed.
*/
- private ListenableFuture<List<TreeResponse>> sendValidationRequest(Collection<InetAddress> endpoints)
+ private ListenableFuture<List<TreeResponse>> sendValidationRequest(Collection<InetAddressAndPort> endpoints)
{
String message = String.format("Requesting merkle trees for %s (to %s)", desc.columnFamily, endpoints);
- logger.info("[repair #{}] {}", desc.sessionId, message);
+ logger.info("{} {}", session.previewKind.logPrefix(desc.sessionId), message);
Tracing.traceRepair(message);
- int gcBefore = Keyspace.open(desc.keyspace).getColumnFamilyStore(desc.columnFamily).gcBefore(FBUtilities.nowInSeconds());
+ int nowInSec = getNowInSeconds();
List<ListenableFuture<TreeResponse>> tasks = new ArrayList<>(endpoints.size());
- for (InetAddress endpoint : endpoints)
+ for (InetAddressAndPort endpoint : endpoints)
{
- ValidationTask task = new ValidationTask(desc, endpoint, gcBefore);
+ ValidationTask task = new ValidationTask(desc, endpoint, nowInSec, session.previewKind);
tasks.add(task);
- session.waitForValidation(Pair.create(desc, endpoint), task);
+ session.trackValidationCompletion(Pair.create(desc, endpoint), task);
taskExecutor.execute(task);
}
return Futures.allAsList(tasks);
@@ -197,38 +378,38 @@
/**
* Creates {@link ValidationTask} and submit them to task executor so that tasks run sequentially.
*/
- private ListenableFuture<List<TreeResponse>> sendSequentialValidationRequest(Collection<InetAddress> endpoints)
+ private ListenableFuture<List<TreeResponse>> sendSequentialValidationRequest(Collection<InetAddressAndPort> endpoints)
{
String message = String.format("Requesting merkle trees for %s (to %s)", desc.columnFamily, endpoints);
- logger.info("[repair #{}] {}", desc.sessionId, message);
+ logger.info("{} {}", session.previewKind.logPrefix(desc.sessionId), message);
Tracing.traceRepair(message);
- int gcBefore = Keyspace.open(desc.keyspace).getColumnFamilyStore(desc.columnFamily).gcBefore(FBUtilities.nowInSeconds());
+ int nowInSec = getNowInSeconds();
List<ListenableFuture<TreeResponse>> tasks = new ArrayList<>(endpoints.size());
- Queue<InetAddress> requests = new LinkedList<>(endpoints);
- InetAddress address = requests.poll();
- ValidationTask firstTask = new ValidationTask(desc, address, gcBefore);
- logger.info("Validating {}", address);
- session.waitForValidation(Pair.create(desc, address), firstTask);
+ Queue<InetAddressAndPort> requests = new LinkedList<>(endpoints);
+ InetAddressAndPort address = requests.poll();
+ ValidationTask firstTask = new ValidationTask(desc, address, nowInSec, session.previewKind);
+ logger.info("{} Validating {}", session.previewKind.logPrefix(desc.sessionId), address);
+ session.trackValidationCompletion(Pair.create(desc, address), firstTask);
tasks.add(firstTask);
ValidationTask currentTask = firstTask;
while (requests.size() > 0)
{
- final InetAddress nextAddress = requests.poll();
- final ValidationTask nextTask = new ValidationTask(desc, nextAddress, gcBefore);
+ final InetAddressAndPort nextAddress = requests.poll();
+ final ValidationTask nextTask = new ValidationTask(desc, nextAddress, nowInSec, session.previewKind);
tasks.add(nextTask);
Futures.addCallback(currentTask, new FutureCallback<TreeResponse>()
{
public void onSuccess(TreeResponse result)
{
- logger.info("Validating {}", nextAddress);
- session.waitForValidation(Pair.create(desc, nextAddress), nextTask);
+ logger.info("{} Validating {}", session.previewKind.logPrefix(desc.sessionId), nextAddress);
+ session.trackValidationCompletion(Pair.create(desc, nextAddress), nextTask);
taskExecutor.execute(nextTask);
}
// failure is handled at root of job chain
public void onFailure(Throwable t) {}
- });
+ }, MoreExecutors.directExecutor());
currentTask = nextTask;
}
// start running tasks
@@ -239,19 +420,19 @@
/**
* Creates {@link ValidationTask} and submit them to task executor so that tasks run sequentially within each dc.
*/
- private ListenableFuture<List<TreeResponse>> sendDCAwareValidationRequest(Collection<InetAddress> endpoints)
+ private ListenableFuture<List<TreeResponse>> sendDCAwareValidationRequest(Collection<InetAddressAndPort> endpoints)
{
String message = String.format("Requesting merkle trees for %s (to %s)", desc.columnFamily, endpoints);
- logger.info("[repair #{}] {}", desc.sessionId, message);
+ logger.info("{} {}", session.previewKind.logPrefix(desc.sessionId), message);
Tracing.traceRepair(message);
- int gcBefore = Keyspace.open(desc.keyspace).getColumnFamilyStore(desc.columnFamily).gcBefore(FBUtilities.nowInSeconds());
+ int nowInSec = getNowInSeconds();
List<ListenableFuture<TreeResponse>> tasks = new ArrayList<>(endpoints.size());
- Map<String, Queue<InetAddress>> requestsByDatacenter = new HashMap<>();
- for (InetAddress endpoint : endpoints)
+ Map<String, Queue<InetAddressAndPort>> requestsByDatacenter = new HashMap<>();
+ for (InetAddressAndPort endpoint : endpoints)
{
String dc = DatabaseDescriptor.getEndpointSnitch().getDatacenter(endpoint);
- Queue<InetAddress> queue = requestsByDatacenter.get(dc);
+ Queue<InetAddressAndPort> queue = requestsByDatacenter.get(dc);
if (queue == null)
{
queue = new LinkedList<>();
@@ -260,32 +441,32 @@
queue.add(endpoint);
}
- for (Map.Entry<String, Queue<InetAddress>> entry : requestsByDatacenter.entrySet())
+ for (Map.Entry<String, Queue<InetAddressAndPort>> entry : requestsByDatacenter.entrySet())
{
- Queue<InetAddress> requests = entry.getValue();
- InetAddress address = requests.poll();
- ValidationTask firstTask = new ValidationTask(desc, address, gcBefore);
- logger.info("Validating {}", address);
- session.waitForValidation(Pair.create(desc, address), firstTask);
+ Queue<InetAddressAndPort> requests = entry.getValue();
+ InetAddressAndPort address = requests.poll();
+ ValidationTask firstTask = new ValidationTask(desc, address, nowInSec, session.previewKind);
+ logger.info("{} Validating {}", session.previewKind.logPrefix(session.getId()), address);
+ session.trackValidationCompletion(Pair.create(desc, address), firstTask);
tasks.add(firstTask);
ValidationTask currentTask = firstTask;
while (requests.size() > 0)
{
- final InetAddress nextAddress = requests.poll();
- final ValidationTask nextTask = new ValidationTask(desc, nextAddress, gcBefore);
+ final InetAddressAndPort nextAddress = requests.poll();
+ final ValidationTask nextTask = new ValidationTask(desc, nextAddress, nowInSec, session.previewKind);
tasks.add(nextTask);
Futures.addCallback(currentTask, new FutureCallback<TreeResponse>()
{
public void onSuccess(TreeResponse result)
{
- logger.info("Validating {}", nextAddress);
- session.waitForValidation(Pair.create(desc, nextAddress), nextTask);
+ logger.info("{} Validating {}", session.previewKind.logPrefix(session.getId()), nextAddress);
+ session.trackValidationCompletion(Pair.create(desc, nextAddress), nextTask);
taskExecutor.execute(nextTask);
}
// failure is handled at root of job chain
public void onFailure(Throwable t) {}
- });
+ }, MoreExecutors.directExecutor());
currentTask = nextTask;
}
// start running tasks
@@ -293,4 +474,4 @@
}
return Futures.allAsList(tasks);
}
-}
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/repair/RepairJobDesc.java b/src/java/org/apache/cassandra/repair/RepairJobDesc.java
index 05adbf9..4aaf655 100644
--- a/src/java/org/apache/cassandra/repair/RepairJobDesc.java
+++ b/src/java/org/apache/cassandra/repair/RepairJobDesc.java
@@ -26,12 +26,14 @@
import org.apache.cassandra.db.TypeSizes;
import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.io.IVersionedSerializer;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.streaming.PreviewKind;
import org.apache.cassandra.utils.UUIDSerializer;
/**
@@ -66,6 +68,11 @@
return "[repair #" + sessionId + " on " + keyspace + "/" + columnFamily + ", " + ranges + "]";
}
+ public String toString(PreviewKind previewKind)
+ {
+ return '[' + previewKind.logPrefix() + " #" + sessionId + " on " + keyspace + "/" + columnFamily + ", " + ranges + "]";
+ }
+
@Override
public boolean equals(Object o)
{
@@ -93,16 +100,14 @@
{
public void serialize(RepairJobDesc desc, DataOutputPlus out, int version) throws IOException
{
- if (version >= MessagingService.VERSION_21)
- {
- out.writeBoolean(desc.parentSessionId != null);
- if (desc.parentSessionId != null)
- UUIDSerializer.serializer.serialize(desc.parentSessionId, out, version);
- }
+ out.writeBoolean(desc.parentSessionId != null);
+ if (desc.parentSessionId != null)
+ UUIDSerializer.serializer.serialize(desc.parentSessionId, out, version);
+
UUIDSerializer.serializer.serialize(desc.sessionId, out, version);
out.writeUTF(desc.keyspace);
out.writeUTF(desc.columnFamily);
- MessagingService.validatePartitioner(desc.ranges);
+ IPartitioner.validate(desc.ranges);
out.writeInt(desc.ranges.size());
for (Range<Token> rt : desc.ranges)
AbstractBounds.tokenSerializer.serialize(rt, out, version);
@@ -111,23 +116,20 @@
public RepairJobDesc deserialize(DataInputPlus in, int version) throws IOException
{
UUID parentSessionId = null;
- if (version >= MessagingService.VERSION_21)
- {
- if (in.readBoolean())
- parentSessionId = UUIDSerializer.serializer.deserialize(in, version);
- }
+ if (in.readBoolean())
+ parentSessionId = UUIDSerializer.serializer.deserialize(in, version);
UUID sessionId = UUIDSerializer.serializer.deserialize(in, version);
String keyspace = in.readUTF();
String columnFamily = in.readUTF();
int nRanges = in.readInt();
- Collection<Range<Token>> ranges = new ArrayList<>();
+ Collection<Range<Token>> ranges = new ArrayList<>(nRanges);
Range<Token> range;
for (int i = 0; i < nRanges; i++)
{
range = (Range<Token>) AbstractBounds.tokenSerializer.deserialize(in,
- MessagingService.globalPartitioner(), version);
+ IPartitioner.global(), version);
ranges.add(range);
}
@@ -136,13 +138,9 @@
public long serializedSize(RepairJobDesc desc, int version)
{
- int size = 0;
- if (version >= MessagingService.VERSION_21)
- {
- size += TypeSizes.sizeof(desc.parentSessionId != null);
- if (desc.parentSessionId != null)
- size += UUIDSerializer.serializer.serializedSize(desc.parentSessionId, version);
- }
+ int size = TypeSizes.sizeof(desc.parentSessionId != null);
+ if (desc.parentSessionId != null)
+ size += UUIDSerializer.serializer.serializedSize(desc.parentSessionId, version);
size += UUIDSerializer.serializer.serializedSize(desc.sessionId, version);
size += TypeSizes.sizeof(desc.keyspace);
size += TypeSizes.sizeof(desc.columnFamily);
diff --git a/src/java/org/apache/cassandra/repair/RepairMessageVerbHandler.java b/src/java/org/apache/cassandra/repair/RepairMessageVerbHandler.java
index 52625bf..47da8bb 100644
--- a/src/java/org/apache/cassandra/repair/RepairMessageVerbHandler.java
+++ b/src/java/org/apache/cassandra/repair/RepairMessageVerbHandler.java
@@ -17,25 +17,22 @@
*/
package org.apache.cassandra.repair;
-import java.net.InetAddress;
import java.util.*;
-import com.google.common.base.Predicate;
-import com.google.common.util.concurrent.ListenableFuture;
-import com.google.common.util.concurrent.MoreExecutors;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.db.ColumnFamilyStore;
-import org.apache.cassandra.db.compaction.CompactionManager;
-import org.apache.cassandra.dht.Bounds;
-import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.exceptions.RequestFailureReason;
import org.apache.cassandra.net.IVerbHandler;
-import org.apache.cassandra.net.MessageIn;
-import org.apache.cassandra.net.MessageOut;
+import org.apache.cassandra.net.Message;
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.repair.messages.*;
+import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.service.ActiveRepairService;
+import org.apache.cassandra.streaming.PreviewKind;
+
+import static org.apache.cassandra.net.Verb.VALIDATION_RSP;
/**
* Handles all repair related message.
@@ -44,122 +41,171 @@
*/
public class RepairMessageVerbHandler implements IVerbHandler<RepairMessage>
{
+ public static RepairMessageVerbHandler instance = new RepairMessageVerbHandler();
+
private static final Logger logger = LoggerFactory.getLogger(RepairMessageVerbHandler.class);
- public void doVerb(final MessageIn<RepairMessage> message, final int id)
+
+ private boolean isIncremental(UUID sessionID)
+ {
+ return ActiveRepairService.instance.consistent.local.isSessionInProgress(sessionID);
+ }
+
+ private PreviewKind previewKind(UUID sessionID)
+ {
+ ActiveRepairService.ParentRepairSession prs = ActiveRepairService.instance.getParentRepairSession(sessionID);
+ return prs != null ? prs.previewKind : PreviewKind.NONE;
+ }
+
+ public void doVerb(final Message<RepairMessage> message)
{
// TODO add cancel/interrupt message
RepairJobDesc desc = message.payload.desc;
try
{
- switch (message.payload.messageType)
+ switch (message.verb())
{
- case PREPARE_MESSAGE:
+ case PREPARE_MSG:
PrepareMessage prepareMessage = (PrepareMessage) message.payload;
logger.debug("Preparing, {}", prepareMessage);
- List<ColumnFamilyStore> columnFamilyStores = new ArrayList<>(prepareMessage.cfIds.size());
- for (UUID cfId : prepareMessage.cfIds)
+ List<ColumnFamilyStore> columnFamilyStores = new ArrayList<>(prepareMessage.tableIds.size());
+ for (TableId tableId : prepareMessage.tableIds)
{
- ColumnFamilyStore columnFamilyStore = ColumnFamilyStore.getIfExists(cfId);
+ ColumnFamilyStore columnFamilyStore = ColumnFamilyStore.getIfExists(tableId);
if (columnFamilyStore == null)
{
logErrorAndSendFailureResponse(String.format("Table with id %s was dropped during prepare phase of repair",
- cfId.toString()), message.from, id);
+ tableId), message);
return;
}
columnFamilyStores.add(columnFamilyStore);
}
ActiveRepairService.instance.registerParentRepairSession(prepareMessage.parentRepairSession,
- message.from,
+ message.from(),
columnFamilyStores,
prepareMessage.ranges,
prepareMessage.isIncremental,
prepareMessage.timestamp,
- prepareMessage.isGlobal);
- MessagingService.instance().sendReply(new MessageOut(MessagingService.Verb.INTERNAL_RESPONSE), id, message.from);
+ prepareMessage.isGlobal,
+ prepareMessage.previewKind);
+ MessagingService.instance().send(message.emptyResponse(), message.from());
break;
- case SNAPSHOT:
+ case SNAPSHOT_MSG:
logger.debug("Snapshotting {}", desc);
final ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(desc.keyspace, desc.columnFamily);
if (cfs == null)
{
- logErrorAndSendFailureResponse(String.format("Table %s.%s was dropped during snapshot phase of repair",
- desc.keyspace, desc.columnFamily), message.from, id);
+ logErrorAndSendFailureResponse(String.format("Table %s.%s was dropped during snapshot phase of repair %s",
+ desc.keyspace, desc.columnFamily, desc.parentSessionId), message);
return;
}
ActiveRepairService.ParentRepairSession prs = ActiveRepairService.instance.getParentRepairSession(desc.parentSessionId);
+ TableRepairManager repairManager = cfs.getRepairManager();
if (prs.isGlobal)
{
- prs.maybeSnapshot(cfs.metadata.cfId, desc.parentSessionId);
+ repairManager.snapshot(desc.parentSessionId.toString(), prs.getRanges(), false);
}
else
{
- cfs.snapshot(desc.sessionId.toString(), new Predicate<SSTableReader>()
- {
- public boolean apply(SSTableReader sstable)
- {
- return sstable != null &&
- !sstable.metadata.isIndex() && // exclude SSTables from 2i
- new Bounds<>(sstable.first.getToken(), sstable.last.getToken()).intersects(desc.ranges);
- }
- }, true, false); //ephemeral snapshot, if repair fails, it will be cleaned next startup
+ repairManager.snapshot(desc.parentSessionId.toString(), desc.ranges, true);
}
- logger.debug("Enqueuing response to snapshot request {} to {}", desc.sessionId, message.from);
- MessagingService.instance().sendReply(new MessageOut(MessagingService.Verb.INTERNAL_RESPONSE), id, message.from);
+ logger.debug("Enqueuing response to snapshot request {} to {}", desc.sessionId, message.from());
+ MessagingService.instance().send(message.emptyResponse(), message.from());
break;
- case VALIDATION_REQUEST:
+ case VALIDATION_REQ:
ValidationRequest validationRequest = (ValidationRequest) message.payload;
logger.debug("Validating {}", validationRequest);
// trigger read-only compaction
ColumnFamilyStore store = ColumnFamilyStore.getIfExists(desc.keyspace, desc.columnFamily);
if (store == null)
{
- logger.error("Table {}.{} was dropped during snapshot phase of repair", desc.keyspace, desc.columnFamily);
- MessagingService.instance().sendOneWay(new ValidationComplete(desc).createMessage(), message.from);
+ logger.error("Table {}.{} was dropped during snapshot phase of repair {}",
+ desc.keyspace, desc.columnFamily, desc.parentSessionId);
+ MessagingService.instance().send(Message.out(VALIDATION_RSP, new ValidationResponse(desc)), message.from());
return;
}
- Validator validator = new Validator(desc, message.from, validationRequest.gcBefore);
- CompactionManager.instance.submitValidation(store, validator);
+ ActiveRepairService.instance.consistent.local.maybeSetRepairing(desc.parentSessionId);
+ Validator validator = new Validator(desc, message.from(), validationRequest.nowInSec,
+ isIncremental(desc.parentSessionId), previewKind(desc.parentSessionId));
+ ValidationManager.instance.submitValidation(store, validator);
break;
- case SYNC_REQUEST:
+ case SYNC_REQ:
// forwarded sync request
SyncRequest request = (SyncRequest) message.payload;
logger.debug("Syncing {}", request);
- long repairedAt = ActiveRepairService.UNREPAIRED_SSTABLE;
- if (desc.parentSessionId != null && ActiveRepairService.instance.getParentRepairSession(desc.parentSessionId) != null)
- repairedAt = ActiveRepairService.instance.getParentRepairSession(desc.parentSessionId).getRepairedAt();
-
- StreamingRepairTask task = new StreamingRepairTask(desc, request, repairedAt);
+ StreamingRepairTask task = new StreamingRepairTask(desc,
+ request.initiator,
+ request.src,
+ request.dst,
+ request.ranges,
+ isIncremental(desc.parentSessionId) ? desc.parentSessionId : null,
+ request.previewKind,
+ false);
task.run();
break;
- case ANTICOMPACTION_REQUEST:
- AnticompactionRequest anticompactionRequest = (AnticompactionRequest) message.payload;
- logger.debug("Got anticompaction request {}", anticompactionRequest);
- ListenableFuture<?> compactionDone = ActiveRepairService.instance.doAntiCompaction(anticompactionRequest.parentRepairSession, anticompactionRequest.successfulRanges);
- compactionDone.addListener(new Runnable()
- {
- @Override
- public void run()
- {
- MessagingService.instance().sendReply(new MessageOut(MessagingService.Verb.INTERNAL_RESPONSE), id, message.from);
- }
- }, MoreExecutors.directExecutor());
+ case ASYMMETRIC_SYNC_REQ:
+ // forwarded sync request
+ AsymmetricSyncRequest asymmetricSyncRequest = (AsymmetricSyncRequest) message.payload;
+ logger.debug("Syncing {}", asymmetricSyncRequest);
+ StreamingRepairTask asymmetricTask = new StreamingRepairTask(desc,
+ asymmetricSyncRequest.initiator,
+ asymmetricSyncRequest.fetchingNode,
+ asymmetricSyncRequest.fetchFrom,
+ asymmetricSyncRequest.ranges,
+ isIncremental(desc.parentSessionId) ? desc.parentSessionId : null,
+ asymmetricSyncRequest.previewKind,
+ true);
+ asymmetricTask.run();
break;
- case CLEANUP:
+ case CLEANUP_MSG:
logger.debug("cleaning up repair");
CleanupMessage cleanup = (CleanupMessage) message.payload;
ActiveRepairService.instance.removeParentRepairSession(cleanup.parentRepairSession);
- MessagingService.instance().sendReply(new MessageOut(MessagingService.Verb.INTERNAL_RESPONSE), id, message.from);
+ MessagingService.instance().send(message.emptyResponse(), message.from());
+ break;
+
+ case PREPARE_CONSISTENT_REQ:
+ ActiveRepairService.instance.consistent.local.handlePrepareMessage(message.from(), (PrepareConsistentRequest) message.payload);
+ break;
+
+ case PREPARE_CONSISTENT_RSP:
+ ActiveRepairService.instance.consistent.coordinated.handlePrepareResponse((PrepareConsistentResponse) message.payload);
+ break;
+
+ case FINALIZE_PROPOSE_MSG:
+ ActiveRepairService.instance.consistent.local.handleFinalizeProposeMessage(message.from(), (FinalizePropose) message.payload);
+ break;
+
+ case FINALIZE_PROMISE_MSG:
+ ActiveRepairService.instance.consistent.coordinated.handleFinalizePromiseMessage((FinalizePromise) message.payload);
+ break;
+
+ case FINALIZE_COMMIT_MSG:
+ ActiveRepairService.instance.consistent.local.handleFinalizeCommitMessage(message.from(), (FinalizeCommit) message.payload);
+ break;
+
+ case FAILED_SESSION_MSG:
+ FailSession failure = (FailSession) message.payload;
+ ActiveRepairService.instance.consistent.coordinated.handleFailSessionMessage(failure);
+ ActiveRepairService.instance.consistent.local.handleFailSessionMessage(message.from(), failure);
+ break;
+
+ case STATUS_REQ:
+ ActiveRepairService.instance.consistent.local.handleStatusRequest(message.from(), (StatusRequest) message.payload);
+ break;
+
+ case STATUS_RSP:
+ ActiveRepairService.instance.consistent.local.handleStatusResponse(message.from(), (StatusResponse) message.payload);
break;
default:
- ActiveRepairService.instance.handleMessage(message.from, message.payload);
+ ActiveRepairService.instance.handleMessage(message);
break;
}
}
@@ -172,11 +218,10 @@
}
}
- private void logErrorAndSendFailureResponse(String errorMessage, InetAddress to, int id)
+ private void logErrorAndSendFailureResponse(String errorMessage, Message<?> respondTo)
{
logger.error(errorMessage);
- MessageOut reply = new MessageOut(MessagingService.Verb.INTERNAL_RESPONSE)
- .withParameter(MessagingService.FAILURE_RESPONSE_PARAM, MessagingService.ONE_BYTE);
- MessagingService.instance().sendReply(reply, id, to);
+ Message reply = respondTo.failureResponse(RequestFailureReason.UNKNOWN);
+ MessagingService.instance().send(reply, respondTo.from());
}
}
diff --git a/src/java/org/apache/cassandra/repair/RepairRunnable.java b/src/java/org/apache/cassandra/repair/RepairRunnable.java
index 35794e2..e5e8e50 100644
--- a/src/java/org/apache/cassandra/repair/RepairRunnable.java
+++ b/src/java/org/apache/cassandra/repair/RepairRunnable.java
@@ -17,25 +17,51 @@
*/
package org.apache.cassandra.repair;
-import java.net.InetAddress;
+import java.io.IOException;
import java.nio.ByteBuffer;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.ExecutorService;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
-import com.google.common.util.concurrent.*;
+import com.google.common.collect.Sets;
+import com.google.common.util.concurrent.AsyncFunction;
+import com.google.common.util.concurrent.FutureCallback;
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.ListeningExecutorService;
+import com.google.common.util.concurrent.MoreExecutors;
import org.apache.commons.lang3.time.DurationFormatUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.cassandra.concurrent.JMXConfigurableThreadPoolExecutor;
+import com.codahale.metrics.Timer;
+import org.apache.cassandra.concurrent.JMXEnabledThreadPoolExecutor;
import org.apache.cassandra.concurrent.NamedThreadFactory;
-import org.apache.cassandra.config.SchemaConstants;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.metrics.RepairMetrics;
+import org.apache.cassandra.db.SnapshotCommand;
+import org.apache.cassandra.gms.FailureDetector;
+import org.apache.cassandra.net.Message;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.net.Verb;
+import org.apache.cassandra.repair.consistent.SyncStatSummary;
+import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.cql3.QueryOptions;
import org.apache.cassandra.cql3.QueryProcessor;
import org.apache.cassandra.cql3.UntypedResultSet;
@@ -44,18 +70,27 @@
import org.apache.cassandra.db.ConsistencyLevel;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.locator.EndpointsForRange;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.Replica;
+import org.apache.cassandra.metrics.StorageMetrics;
+import org.apache.cassandra.repair.consistent.CoordinatorSession;
import org.apache.cassandra.repair.messages.RepairOption;
+import org.apache.cassandra.schema.SchemaConstants;
import org.apache.cassandra.service.ActiveRepairService;
+import org.apache.cassandra.service.ActiveRepairService.ParentRepairStatus;
import org.apache.cassandra.service.ClientState;
import org.apache.cassandra.service.QueryState;
import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.service.reads.repair.RepairedDataVerifier;
+import org.apache.cassandra.streaming.PreviewKind;
import org.apache.cassandra.tracing.TraceKeyspace;
import org.apache.cassandra.tracing.TraceState;
import org.apache.cassandra.tracing.Tracing;
import org.apache.cassandra.transport.messages.ResultMessage;
import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.DiagnosticSnapshotService;
import org.apache.cassandra.utils.FBUtilities;
-import org.apache.cassandra.utils.Pair;
import org.apache.cassandra.utils.UUIDGen;
import org.apache.cassandra.utils.WrappedRunnable;
import org.apache.cassandra.utils.progress.ProgressEvent;
@@ -63,25 +98,38 @@
import org.apache.cassandra.utils.progress.ProgressEventType;
import org.apache.cassandra.utils.progress.ProgressListener;
-public class RepairRunnable extends WrappedRunnable implements ProgressEventNotifier
+public class RepairRunnable implements Runnable, ProgressEventNotifier
{
private static final Logger logger = LoggerFactory.getLogger(RepairRunnable.class);
- private StorageService storageService;
+ private final StorageService storageService;
private final int cmd;
private final RepairOption options;
private final String keyspace;
+ private final String tag;
+ private final AtomicInteger progressCounter = new AtomicInteger();
+ private final int totalProgress;
+
+ private final long creationTimeMillis = System.currentTimeMillis();
+ private final UUID parentSession = UUIDGen.getTimeUUID();
+
private final List<ProgressListener> listeners = new ArrayList<>();
private static final AtomicInteger threadCounter = new AtomicInteger(1);
+ private TraceState traceState;
+
public RepairRunnable(StorageService storageService, int cmd, RepairOption options, String keyspace)
{
this.storageService = storageService;
this.cmd = cmd;
this.options = options;
this.keyspace = keyspace;
+
+ this.tag = "repair:" + cmd;
+ // get valid column families, calculate neighbors, validation, prepare for repair + number of ranges to repair
+ this.totalProgress = 4 + options.getRanges().size();
}
@Override
@@ -96,7 +144,8 @@
listeners.remove(listener);
}
- protected void fireProgressEvent(String tag, ProgressEvent event)
+
+ protected void fireProgressEvent(ProgressEvent event)
{
for (ProgressListener listener : listeners)
{
@@ -104,192 +153,264 @@
}
}
- protected void fireErrorAndComplete(String tag, int progressCount, int totalProgress, String message)
+ public void notification(String msg)
{
- fireProgressEvent(tag, new ProgressEvent(ProgressEventType.ERROR, progressCount, totalProgress, String.format("Repair command #%d failed with error %s", cmd, message)));
- fireProgressEvent(tag, new ProgressEvent(ProgressEventType.COMPLETE, progressCount, totalProgress, String.format("Repair command #%d finished with error", cmd)));
+ logger.info(msg);
+ fireProgressEvent(new ProgressEvent(ProgressEventType.NOTIFICATION, progressCounter.get(), totalProgress, msg));
}
- protected void runMayThrow() throws Exception
+ private void skip(String msg)
{
- final TraceState traceState;
- final UUID parentSession = UUIDGen.getTimeUUID();
- final String tag = "repair:" + cmd;
+ notification("Repair " + parentSession + " skipped: " + msg);
+ success(msg);
+ }
- final AtomicInteger progress = new AtomicInteger();
- final int totalProgress = 4 + options.getRanges().size(); // get valid column families, calculate neighbors, validation, prepare for repair + number of ranges to repair
+ private void success(String msg)
+ {
+ fireProgressEvent(new ProgressEvent(ProgressEventType.SUCCESS, progressCounter.get(), totalProgress, msg));
+ ActiveRepairService.instance.recordRepairStatus(cmd, ActiveRepairService.ParentRepairStatus.COMPLETED,
+ ImmutableList.of(msg));
+ complete(null);
+ }
- String[] columnFamilies = options.getColumnFamilies().toArray(new String[options.getColumnFamilies().size()]);
- Iterable<ColumnFamilyStore> validColumnFamilies;
+ public void notifyError(Throwable error)
+ {
+ // exception should be ignored
+ if (error instanceof SomeRepairFailedException)
+ return;
+ logger.error("Repair {} failed:", parentSession, error);
+
+ StorageMetrics.repairExceptions.inc();
+ String errorMessage = String.format("Repair command #%d failed with error %s", cmd, error.getMessage());
+ fireProgressEvent(new ProgressEvent(ProgressEventType.ERROR, progressCounter.get(), totalProgress, errorMessage));
+
+ // since this can fail, update table only after updating in-memory and notification state
+ maybeStoreParentRepairFailure(error);
+ }
+
+ private void fail(String reason)
+ {
+ if (reason == null)
+ reason = "Some repair failed";
+ String completionMessage = String.format("Repair command #%d finished with error", cmd);
+
+ // Note we rely on the first message being the reason for the failure
+ // when inspecting this state from RepairRunner.queryForCompletedRepair
+ ActiveRepairService.instance.recordRepairStatus(cmd, ParentRepairStatus.FAILED,
+ ImmutableList.of(reason, completionMessage));
+
+ complete(completionMessage);
+ }
+
+ private void complete(String msg)
+ {
+ long durationMillis = System.currentTimeMillis() - creationTimeMillis;
+ if (msg == null)
+ {
+ String duration = DurationFormatUtils.formatDurationWords(durationMillis, true, true);
+ msg = String.format("Repair command #%d finished in %s", cmd, duration);
+ }
+
+ fireProgressEvent(new ProgressEvent(ProgressEventType.COMPLETE, progressCounter.get(), totalProgress, msg));
+ logger.info(options.getPreviewKind().logPrefix(parentSession) + msg);
+
+ ActiveRepairService.instance.removeParentRepairSession(parentSession);
+ TraceState localState = traceState;
+ if (options.isTraced() && localState != null)
+ {
+ for (ProgressListener listener : listeners)
+ localState.removeProgressListener(listener);
+ // Because DebuggableThreadPoolExecutor#afterExecute and this callback
+ // run in a nondeterministic order (within the same thread), the
+ // TraceState may have been nulled out at this point. The TraceState
+ // should be traceState, so just set it without bothering to check if it
+ // actually was nulled out.
+ Tracing.instance.set(localState);
+ Tracing.traceRepair(msg);
+ Tracing.instance.stopSession();
+ }
+
+ Keyspace.open(keyspace).metric.repairTime.update(durationMillis, TimeUnit.MILLISECONDS);
+ }
+
+ public void run()
+ {
try
{
- validColumnFamilies = storageService.getValidColumnFamilies(false, false, keyspace, columnFamilies);
- progress.incrementAndGet();
+ runMayThrow();
}
- catch (IllegalArgumentException e)
+ catch (SkipRepairException e)
{
- logger.error("Repair failed:", e);
- fireErrorAndComplete(tag, progress.get(), totalProgress, e.getMessage());
- return;
+ skip(e.getMessage());
}
+ catch (Exception | Error e)
+ {
+ notifyError(e);
+ fail(e.getMessage());
+ }
+ }
- final long startTime = System.currentTimeMillis();
+ private void runMayThrow() throws Exception
+ {
+ ActiveRepairService.instance.recordRepairStatus(cmd, ParentRepairStatus.IN_PROGRESS, ImmutableList.of());
+
+ List<ColumnFamilyStore> columnFamilies = getColumnFamilies();
+ String[] cfnames = columnFamilies.stream().map(cfs -> cfs.name).toArray(String[]::new);
+
+ this.traceState = maybeCreateTraceState(columnFamilies);
+
+ notifyStarting();
+
+ NeighborsAndRanges neighborsAndRanges = getNeighborsAndRanges();
+
+ maybeStoreParentRepairStart(cfnames);
+
+ prepare(columnFamilies, neighborsAndRanges.allNeighbors, neighborsAndRanges.force);
+
+ repair(cfnames, neighborsAndRanges);
+ }
+
+ private List<ColumnFamilyStore> getColumnFamilies() throws IOException
+ {
+ String[] columnFamilies = options.getColumnFamilies().toArray(new String[options.getColumnFamilies().size()]);
+ Iterable<ColumnFamilyStore> validColumnFamilies = storageService.getValidColumnFamilies(false, false, keyspace, columnFamilies);
+ progressCounter.incrementAndGet();
+
+ if (Iterables.isEmpty(validColumnFamilies))
+ throw new SkipRepairException(String.format("%s Empty keyspace, skipping repair: %s", parentSession, keyspace));
+ return Lists.newArrayList(validColumnFamilies);
+ }
+
+ private TraceState maybeCreateTraceState(Iterable<ColumnFamilyStore> columnFamilyStores)
+ {
+ if (!options.isTraced())
+ return null;
+
+ StringBuilder cfsb = new StringBuilder();
+ for (ColumnFamilyStore cfs : columnFamilyStores)
+ cfsb.append(", ").append(cfs.keyspace.getName()).append(".").append(cfs.name);
+
+ UUID sessionId = Tracing.instance.newSession(Tracing.TraceType.REPAIR);
+ TraceState traceState = Tracing.instance.begin("repair", ImmutableMap.of("keyspace", keyspace, "columnFamilies",
+ cfsb.substring(2)));
+ traceState.enableActivityNotification(tag);
+ for (ProgressListener listener : listeners)
+ traceState.addProgressListener(listener);
+ Thread queryThread = createQueryThread(cmd, sessionId);
+ queryThread.setName("RepairTracePolling");
+ queryThread.start();
+ return traceState;
+ }
+
+ private void notifyStarting()
+ {
String message = String.format("Starting repair command #%d (%s), repairing keyspace %s with %s", cmd, parentSession, keyspace,
options);
logger.info(message);
- if (options.isTraced())
- {
- StringBuilder cfsb = new StringBuilder();
- for (ColumnFamilyStore cfs : validColumnFamilies)
- cfsb.append(", ").append(cfs.keyspace.getName()).append(".").append(cfs.name);
+ Tracing.traceRepair(message);
+ fireProgressEvent(new ProgressEvent(ProgressEventType.START, 0, 100, message));
+ }
- UUID sessionId = Tracing.instance.newSession(Tracing.TraceType.REPAIR);
- traceState = Tracing.instance.begin("repair", ImmutableMap.of("keyspace", keyspace, "columnFamilies",
- cfsb.substring(2)));
- message = message + " tracing with " + sessionId;
- fireProgressEvent(tag, new ProgressEvent(ProgressEventType.START, 0, 100, message));
- Tracing.traceRepair(message);
- traceState.enableActivityNotification(tag);
- for (ProgressListener listener : listeners)
- traceState.addProgressListener(listener);
- Thread queryThread = createQueryThread(cmd, sessionId);
- queryThread.setName("RepairTracePolling");
- queryThread.start();
+ private NeighborsAndRanges getNeighborsAndRanges()
+ {
+ Set<InetAddressAndPort> allNeighbors = new HashSet<>();
+ List<CommonRange> commonRanges = new ArrayList<>();
+
+ //pre-calculate output of getLocalReplicas and pass it to getNeighbors to increase performance and prevent
+ //calculation multiple times
+ Iterable<Range<Token>> keyspaceLocalRanges = storageService.getLocalReplicas(keyspace).ranges();
+
+ for (Range<Token> range : options.getRanges())
+ {
+ EndpointsForRange neighbors = ActiveRepairService.getNeighbors(keyspace, keyspaceLocalRanges, range,
+ options.getDataCenters(),
+ options.getHosts());
+
+ addRangeToNeighbors(commonRanges, range, neighbors);
+ allNeighbors.addAll(neighbors.endpoints());
+ }
+
+ progressCounter.incrementAndGet();
+
+ boolean force = options.isForcedRepair();
+
+ if (force && options.isIncremental())
+ {
+ Set<InetAddressAndPort> actualNeighbors = Sets.newHashSet(Iterables.filter(allNeighbors, FailureDetector.instance::isAlive));
+ force = !allNeighbors.equals(actualNeighbors);
+ allNeighbors = actualNeighbors;
+ }
+ return new NeighborsAndRanges(force, allNeighbors, commonRanges);
+ }
+
+ private void maybeStoreParentRepairStart(String[] cfnames)
+ {
+ if (!options.isPreview())
+ {
+ SystemDistributedKeyspace.startParentRepair(parentSession, keyspace, cfnames, options);
+ }
+ }
+
+ private void maybeStoreParentRepairSuccess(Collection<Range<Token>> successfulRanges)
+ {
+ if (!options.isPreview())
+ {
+ SystemDistributedKeyspace.successfulParentRepair(parentSession, successfulRanges);
+ }
+ }
+
+ private void maybeStoreParentRepairFailure(Throwable error)
+ {
+ if (!options.isPreview())
+ {
+ SystemDistributedKeyspace.failParentRepair(parentSession, error);
+ }
+ }
+
+ private void prepare(List<ColumnFamilyStore> columnFamilies, Set<InetAddressAndPort> allNeighbors, boolean force)
+ {
+ try (Timer.Context ignore = Keyspace.open(keyspace).metric.repairPrepareTime.time())
+ {
+ ActiveRepairService.instance.prepareForRepair(parentSession, FBUtilities.getBroadcastAddressAndPort(), allNeighbors, options, force, columnFamilies);
+ progressCounter.incrementAndGet();
+ }
+ }
+
+ private void repair(String[] cfnames, NeighborsAndRanges neighborsAndRanges)
+ {
+ if (options.isPreview())
+ {
+ previewRepair(parentSession, creationTimeMillis, neighborsAndRanges.commonRanges, cfnames);
+ }
+ else if (options.isIncremental())
+ {
+ incrementalRepair(parentSession, creationTimeMillis, neighborsAndRanges.force, traceState,
+ neighborsAndRanges.allNeighbors, neighborsAndRanges.commonRanges, cfnames);
}
else
{
- fireProgressEvent(tag, new ProgressEvent(ProgressEventType.START, 0, 100, message));
- traceState = null;
+ normalRepair(parentSession, creationTimeMillis, traceState, neighborsAndRanges.commonRanges, cfnames);
}
+ }
- final Set<InetAddress> allNeighbors = new HashSet<>();
- List<Pair<Set<InetAddress>, ? extends Collection<Range<Token>>>> commonRanges = new ArrayList<>();
-
- //pre-calculate output of getLocalRanges and pass it to getNeighbors to increase performance and prevent
- //calculation multiple times
- Collection<Range<Token>> keyspaceLocalRanges = storageService.getLocalRanges(keyspace);
-
- try
- {
- for (Range<Token> range : options.getRanges())
- {
- Set<InetAddress> neighbors = ActiveRepairService.getNeighbors(keyspace, keyspaceLocalRanges, range,
- options.getDataCenters(),
- options.getHosts());
-
- addRangeToNeighbors(commonRanges, range, neighbors);
- allNeighbors.addAll(neighbors);
- }
-
- progress.incrementAndGet();
- }
- catch (IllegalArgumentException e)
- {
- logger.error("Repair failed:", e);
- fireErrorAndComplete(tag, progress.get(), totalProgress, e.getMessage());
- return;
- }
-
- // Validate columnfamilies
- List<ColumnFamilyStore> columnFamilyStores = new ArrayList<>();
- try
- {
- Iterables.addAll(columnFamilyStores, validColumnFamilies);
- progress.incrementAndGet();
- }
- catch (IllegalArgumentException e)
- {
- fireErrorAndComplete(tag, progress.get(), totalProgress, e.getMessage());
- return;
- }
-
- String[] cfnames = new String[columnFamilyStores.size()];
- for (int i = 0; i < columnFamilyStores.size(); i++)
- {
- cfnames[i] = columnFamilyStores.get(i).name;
- }
-
- SystemDistributedKeyspace.startParentRepair(parentSession, keyspace, cfnames, options);
- long repairedAt;
- try
- {
- ActiveRepairService.instance.prepareForRepair(parentSession, FBUtilities.getBroadcastAddress(), allNeighbors, options, columnFamilyStores);
- repairedAt = ActiveRepairService.instance.getParentRepairSession(parentSession).getRepairedAt();
- progress.incrementAndGet();
- }
- catch (Throwable t)
- {
- SystemDistributedKeyspace.failParentRepair(parentSession, t);
- fireErrorAndComplete(tag, progress.get(), totalProgress, t.getMessage());
- return;
- }
+ private void normalRepair(UUID parentSession,
+ long startTime,
+ TraceState traceState,
+ List<CommonRange> commonRanges,
+ String... cfnames)
+ {
// Set up RepairJob executor for this repair command.
- final ListeningExecutorService executor = MoreExecutors.listeningDecorator(new JMXConfigurableThreadPoolExecutor(options.getJobThreads(),
- Integer.MAX_VALUE,
- TimeUnit.SECONDS,
- new LinkedBlockingQueue<Runnable>(),
- new NamedThreadFactory("Repair#" + cmd),
- "internal"));
+ ListeningExecutorService executor = createExecutor();
- List<ListenableFuture<RepairSessionResult>> futures = new ArrayList<>(options.getRanges().size());
- for (Pair<Set<InetAddress>, ? extends Collection<Range<Token>>> p : commonRanges)
- {
- final RepairSession session = ActiveRepairService.instance.submitRepairSession(parentSession,
- p.right,
- keyspace,
- options.getParallelism(),
- p.left,
- repairedAt,
- options.isPullRepair(),
- executor,
- cfnames);
- if (session == null)
- continue;
- // After repair session completes, notify client its result
- Futures.addCallback(session, new FutureCallback<RepairSessionResult>()
- {
- public void onSuccess(RepairSessionResult result)
- {
- /**
- * If the success message below is modified, it must also be updated on
- * {@link org.apache.cassandra.utils.progress.jmx.LegacyJMXProgressSupport}
- * for backward-compatibility support.
- */
- String message = String.format("Repair session %s for range %s finished", session.getId(),
- session.getRanges().toString());
- logger.info(message);
- fireProgressEvent(tag, new ProgressEvent(ProgressEventType.PROGRESS,
- progress.incrementAndGet(),
- totalProgress,
- message));
- }
-
- public void onFailure(Throwable t)
- {
- /**
- * If the failure message below is modified, it must also be updated on
- * {@link org.apache.cassandra.utils.progress.jmx.LegacyJMXProgressSupport}
- * for backward-compatibility support.
- */
- String message = String.format("Repair session %s for range %s failed with error %s",
- session.getId(), session.getRanges().toString(), t.getMessage());
- logger.error(message, t);
- fireProgressEvent(tag, new ProgressEvent(ProgressEventType.PROGRESS,
- progress.incrementAndGet(),
- totalProgress,
- message));
- }
- });
- futures.add(session);
- }
+ // Setting the repairedAt time to UNREPAIRED_SSTABLE causes the repairedAt times to be preserved across streamed sstables
+ final ListenableFuture<List<RepairSessionResult>> allSessions = submitRepairSessions(parentSession, false, executor, commonRanges, cfnames);
// After all repair sessions completes(successful or not),
// run anticompaction if necessary and send finish notice back to client
final Collection<Range<Token>> successfulRanges = new ArrayList<>();
final AtomicBoolean hasFailure = new AtomicBoolean();
- final ListenableFuture<List<RepairSessionResult>> allSessions = Futures.successfulAsList(futures);
- ListenableFuture anticompactionResult = Futures.transform(allSessions, new AsyncFunction<List<RepairSessionResult>, Object>()
+ ListenableFuture repairResult = Futures.transformAsync(allSessions, new AsyncFunction<List<RepairSessionResult>, Object>()
{
@SuppressWarnings("unchecked")
public ListenableFuture apply(List<RepairSessionResult> results)
@@ -297,84 +418,341 @@
// filter out null(=failed) results and get successful ranges
for (RepairSessionResult sessionResult : results)
{
+ logger.debug("Repair result: {}", results);
if (sessionResult != null)
{
- successfulRanges.addAll(sessionResult.ranges);
+ // don't record successful repair if we had to skip ranges
+ if (!sessionResult.skippedReplicas)
+ {
+ successfulRanges.addAll(sessionResult.ranges);
+ }
}
else
{
hasFailure.compareAndSet(false, true);
}
}
- return ActiveRepairService.instance.finishParentSession(parentSession, allNeighbors, successfulRanges);
+ return Futures.immediateFuture(null);
}
- });
- Futures.addCallback(anticompactionResult, new FutureCallback<Object>()
+ }, MoreExecutors.directExecutor());
+ Futures.addCallback(repairResult, new RepairCompleteCallback(parentSession, successfulRanges, startTime, traceState, hasFailure, executor), MoreExecutors.directExecutor());
+ }
+
+ /**
+ * removes dead nodes from common ranges, and exludes ranges left without any participants
+ */
+ @VisibleForTesting
+ static List<CommonRange> filterCommonRanges(List<CommonRange> commonRanges, Set<InetAddressAndPort> liveEndpoints, boolean force)
+ {
+ if (!force)
{
- public void onSuccess(Object result)
+ return commonRanges;
+ }
+ else
+ {
+ List<CommonRange> filtered = new ArrayList<>(commonRanges.size());
+
+ for (CommonRange commonRange : commonRanges)
{
- SystemDistributedKeyspace.successfulParentRepair(parentSession, successfulRanges);
- if (hasFailure.get())
+ Set<InetAddressAndPort> endpoints = ImmutableSet.copyOf(Iterables.filter(commonRange.endpoints, liveEndpoints::contains));
+ Set<InetAddressAndPort> transEndpoints = ImmutableSet.copyOf(Iterables.filter(commonRange.transEndpoints, liveEndpoints::contains));
+ Preconditions.checkState(endpoints.containsAll(transEndpoints), "transEndpoints must be a subset of endpoints");
+
+ // this node is implicitly a participant in this repair, so a single endpoint is ok here
+ if (!endpoints.isEmpty())
{
- fireProgressEvent(tag, new ProgressEvent(ProgressEventType.ERROR, progress.get(), totalProgress,
- "Some repair failed"));
+ filtered.add(new CommonRange(endpoints, transEndpoints, commonRange.ranges));
}
- else
+ }
+ Preconditions.checkState(!filtered.isEmpty(), "Not enough live endpoints for a repair");
+ return filtered;
+ }
+ }
+
+ private void incrementalRepair(UUID parentSession,
+ long startTime,
+ boolean forceRepair,
+ TraceState traceState,
+ Set<InetAddressAndPort> allNeighbors,
+ List<CommonRange> commonRanges,
+ String... cfnames)
+ {
+ // the local node also needs to be included in the set of participants, since coordinator sessions aren't persisted
+ Set<InetAddressAndPort> allParticipants = ImmutableSet.<InetAddressAndPort>builder()
+ .addAll(allNeighbors)
+ .add(FBUtilities.getBroadcastAddressAndPort())
+ .build();
+
+ List<CommonRange> allRanges = filterCommonRanges(commonRanges, allParticipants, forceRepair);
+
+ CoordinatorSession coordinatorSession = ActiveRepairService.instance.consistent.coordinated.registerSession(parentSession, allParticipants, forceRepair);
+ ListeningExecutorService executor = createExecutor();
+ AtomicBoolean hasFailure = new AtomicBoolean(false);
+ ListenableFuture repairResult = coordinatorSession.execute(() -> submitRepairSessions(parentSession, true, executor, allRanges, cfnames),
+ hasFailure);
+ Collection<Range<Token>> ranges = new HashSet<>();
+ for (Collection<Range<Token>> range : Iterables.transform(allRanges, cr -> cr.ranges))
+ {
+ ranges.addAll(range);
+ }
+ Futures.addCallback(repairResult, new RepairCompleteCallback(parentSession, ranges, startTime, traceState, hasFailure, executor), MoreExecutors.directExecutor());
+ }
+
+ private void previewRepair(UUID parentSession,
+ long startTime,
+ List<CommonRange> commonRanges,
+ String... cfnames)
+ {
+
+ logger.debug("Starting preview repair for {}", parentSession);
+ // Set up RepairJob executor for this repair command.
+ ListeningExecutorService executor = createExecutor();
+
+ final ListenableFuture<List<RepairSessionResult>> allSessions = submitRepairSessions(parentSession, false, executor, commonRanges, cfnames);
+
+ Futures.addCallback(allSessions, new FutureCallback<List<RepairSessionResult>>()
+ {
+ public void onSuccess(List<RepairSessionResult> results)
+ {
+ try
{
- fireProgressEvent(tag, new ProgressEvent(ProgressEventType.SUCCESS, progress.get(), totalProgress,
- "Repair completed successfully"));
+ if (results == null || results.stream().anyMatch(s -> s == null))
+ {
+ // something failed
+ fail(null);
+ return;
+ }
+ PreviewKind previewKind = options.getPreviewKind();
+ Preconditions.checkState(previewKind != PreviewKind.NONE, "Preview is NONE");
+ SyncStatSummary summary = new SyncStatSummary(true);
+ summary.consumeSessionResults(results);
+
+ final String message;
+ if (summary.isEmpty())
+ {
+ message = previewKind == PreviewKind.REPAIRED ? "Repaired data is in sync" : "Previewed data was in sync";
+ }
+ else
+ {
+ message = (previewKind == PreviewKind.REPAIRED ? "Repaired data is inconsistent\n" : "Preview complete\n") + summary.toString();
+ RepairMetrics.previewFailures.inc();
+ if (previewKind == PreviewKind.REPAIRED)
+ maybeSnapshotReplicas(parentSession, keyspace, results);
+ }
+ notification(message);
+
+ success("Repair preview completed successfully");
}
- repairComplete();
+ catch (Throwable t)
+ {
+ logger.error("Error completing preview repair", t);
+ onFailure(t);
+ }
+ finally
+ {
+ executor.shutdownNow();
+ }
}
public void onFailure(Throwable t)
{
- fireProgressEvent(tag, new ProgressEvent(ProgressEventType.ERROR, progress.get(), totalProgress, t.getMessage()));
- SystemDistributedKeyspace.failParentRepair(parentSession, t);
- repairComplete();
- }
-
- private void repairComplete()
- {
- String duration = DurationFormatUtils.formatDurationWords(System.currentTimeMillis() - startTime,
- true, true);
- String message = String.format("Repair command #%d finished in %s", cmd, duration);
- fireProgressEvent(tag, new ProgressEvent(ProgressEventType.COMPLETE, progress.get(), totalProgress, message));
- logger.info(message);
- if (options.isTraced() && traceState != null)
- {
- for (ProgressListener listener : listeners)
- traceState.removeProgressListener(listener);
- // Because DebuggableThreadPoolExecutor#afterExecute and this callback
- // run in a nondeterministic order (within the same thread), the
- // TraceState may have been nulled out at this point. The TraceState
- // should be traceState, so just set it without bothering to check if it
- // actually was nulled out.
- Tracing.instance.set(traceState);
- Tracing.traceRepair(message);
- Tracing.instance.stopSession();
- }
+ notifyError(t);
+ fail("Error completing preview repair: " + t.getMessage());
executor.shutdownNow();
}
- });
+ }, MoreExecutors.directExecutor());
}
- private void addRangeToNeighbors(List<Pair<Set<InetAddress>, ? extends Collection<Range<Token>>>> neighborRangeList, Range<Token> range, Set<InetAddress> neighbors)
+ private void maybeSnapshotReplicas(UUID parentSession, String keyspace, List<RepairSessionResult> results)
{
- for (int i = 0; i < neighborRangeList.size(); i++)
- {
- Pair<Set<InetAddress>, ? extends Collection<Range<Token>>> p = neighborRangeList.get(i);
+ if (!DatabaseDescriptor.snapshotOnRepairedDataMismatch())
+ return;
- if (p.left.containsAll(neighbors))
+ try
+ {
+ Set<String> mismatchingTables = new HashSet<>();
+ Set<InetAddressAndPort> nodes = new HashSet<>();
+ for (RepairSessionResult sessionResult : results)
{
- p.right.add(range);
+ for (RepairResult repairResult : emptyIfNull(sessionResult.repairJobResults))
+ {
+ for (SyncStat stat : emptyIfNull(repairResult.stats))
+ {
+ if (stat.numberOfDifferences > 0)
+ mismatchingTables.add(repairResult.desc.columnFamily);
+ // snapshot all replicas, even if they don't have any differences
+ nodes.add(stat.nodes.coordinator);
+ nodes.add(stat.nodes.peer);
+ }
+ }
+ }
+
+ String snapshotName = DiagnosticSnapshotService.getSnapshotName(DiagnosticSnapshotService.REPAIRED_DATA_MISMATCH_SNAPSHOT_PREFIX);
+ for (String table : mismatchingTables)
+ {
+ // we can just check snapshot existence locally since the repair coordinator is always a replica (unlike in the read case)
+ if (!Keyspace.open(keyspace).getColumnFamilyStore(table).snapshotExists(snapshotName))
+ {
+ logger.info("{} Snapshotting {}.{} for preview repair mismatch with tag {} on instances {}",
+ options.getPreviewKind().logPrefix(parentSession),
+ keyspace, table, snapshotName, nodes);
+ DiagnosticSnapshotService.repairedDataMismatch(Keyspace.open(keyspace).getColumnFamilyStore(table).metadata(), nodes);
+ }
+ else
+ {
+ logger.info("{} Not snapshotting {}.{} - snapshot {} exists",
+ options.getPreviewKind().logPrefix(parentSession),
+ keyspace, table, snapshotName);
+ }
+ }
+ }
+ catch (Exception e)
+ {
+ logger.error("{} Failed snapshotting replicas", options.getPreviewKind().logPrefix(parentSession), e);
+ }
+ }
+
+ private static <T> Iterable<T> emptyIfNull(Iterable<T> iter)
+ {
+ if (iter == null)
+ return Collections.emptyList();
+ return iter;
+ }
+
+ private ListenableFuture<List<RepairSessionResult>> submitRepairSessions(UUID parentSession,
+ boolean isIncremental,
+ ListeningExecutorService executor,
+ List<CommonRange> commonRanges,
+ String... cfnames)
+ {
+ List<ListenableFuture<RepairSessionResult>> futures = new ArrayList<>(options.getRanges().size());
+
+ // we do endpoint filtering at the start of an incremental repair,
+ // so repair sessions shouldn't also be checking liveness
+ boolean force = options.isForcedRepair() && !isIncremental;
+ for (CommonRange commonRange : commonRanges)
+ {
+ logger.info("Starting RepairSession for {}", commonRange);
+ RepairSession session = ActiveRepairService.instance.submitRepairSession(parentSession,
+ commonRange,
+ keyspace,
+ options.getParallelism(),
+ isIncremental,
+ options.isPullRepair(),
+ force,
+ options.getPreviewKind(),
+ options.optimiseStreams(),
+ executor,
+ cfnames);
+ if (session == null)
+ continue;
+ Futures.addCallback(session, new RepairSessionCallback(session), MoreExecutors.directExecutor());
+ futures.add(session);
+ }
+ return Futures.successfulAsList(futures);
+ }
+
+ private ListeningExecutorService createExecutor()
+ {
+ return MoreExecutors.listeningDecorator(new JMXEnabledThreadPoolExecutor(options.getJobThreads(),
+ Integer.MAX_VALUE,
+ TimeUnit.SECONDS,
+ new LinkedBlockingQueue<>(),
+ new NamedThreadFactory("Repair#" + cmd),
+ "internal"));
+ }
+
+ private class RepairSessionCallback implements FutureCallback<RepairSessionResult>
+ {
+ private final RepairSession session;
+
+ public RepairSessionCallback(RepairSession session)
+ {
+ this.session = session;
+ }
+
+ public void onSuccess(RepairSessionResult result)
+ {
+ String message = String.format("Repair session %s for range %s finished", session.getId(),
+ session.ranges().toString());
+ logger.info(message);
+ fireProgressEvent(new ProgressEvent(ProgressEventType.PROGRESS,
+ progressCounter.incrementAndGet(),
+ totalProgress,
+ message));
+ }
+
+ public void onFailure(Throwable t)
+ {
+ String message = String.format("Repair session %s for range %s failed with error %s",
+ session.getId(), session.ranges().toString(), t.getMessage());
+ notifyError(new RuntimeException(message, t));
+ }
+ }
+
+ private class RepairCompleteCallback implements FutureCallback<Object>
+ {
+ final UUID parentSession;
+ final Collection<Range<Token>> successfulRanges;
+ final long startTime;
+ final TraceState traceState;
+ final AtomicBoolean hasFailure;
+ final ExecutorService executor;
+
+ public RepairCompleteCallback(UUID parentSession,
+ Collection<Range<Token>> successfulRanges,
+ long startTime,
+ TraceState traceState,
+ AtomicBoolean hasFailure,
+ ExecutorService executor)
+ {
+ this.parentSession = parentSession;
+ this.successfulRanges = successfulRanges;
+ this.startTime = startTime;
+ this.traceState = traceState;
+ this.hasFailure = hasFailure;
+ this.executor = executor;
+ }
+
+ public void onSuccess(Object result)
+ {
+ maybeStoreParentRepairSuccess(successfulRanges);
+ if (hasFailure.get())
+ {
+ fail(null);
+ }
+ else
+ {
+ success("Repair completed successfully");
+ }
+ executor.shutdownNow();
+ }
+
+ public void onFailure(Throwable t)
+ {
+ notifyError(t);
+ fail(t.getMessage());
+ executor.shutdownNow();
+ }
+ }
+
+ private static void addRangeToNeighbors(List<CommonRange> neighborRangeList, Range<Token> range, EndpointsForRange neighbors)
+ {
+ Set<InetAddressAndPort> endpoints = neighbors.endpoints();
+ Set<InetAddressAndPort> transEndpoints = neighbors.filter(Replica::isTransient).endpoints();
+
+ for (CommonRange commonRange : neighborRangeList)
+ {
+ if (commonRange.matchesEndpoints(endpoints, transEndpoints))
+ {
+ commonRange.ranges.add(range);
return;
}
}
List<Range<Token>> ranges = new ArrayList<>();
ranges.add(range);
- neighborRangeList.add(Pair.create(neighbors, ranges));
+ neighborRangeList.add(new CommonRange(endpoints, transEndpoints, ranges));
}
private Thread createQueryThread(final int cmd, final UUID sessionId)
@@ -389,14 +767,14 @@
if (state == null)
throw new Exception("no tracestate");
- String format = "select event_id, source, activity from %s.%s where session_id = ? and event_id > ? and event_id < ?;";
+ String format = "select event_id, source, source_port, activity from %s.%s where session_id = ? and event_id > ? and event_id < ?;";
String query = String.format(format, SchemaConstants.TRACE_KEYSPACE_NAME, TraceKeyspace.EVENTS);
- SelectStatement statement = (SelectStatement) QueryProcessor.parseStatement(query).prepare(ClientState.forInternalCalls()).statement;
+ SelectStatement statement = (SelectStatement) QueryProcessor.parseStatement(query).prepare(ClientState.forInternalCalls());
ByteBuffer sessionIdBytes = ByteBufferUtil.bytes(sessionId);
- InetAddress source = FBUtilities.getBroadcastAddress();
+ InetAddressAndPort source = FBUtilities.getBroadcastAddressAndPort();
- HashSet<UUID>[] seen = new HashSet[] { new HashSet<>(), new HashSet<>() };
+ HashSet<UUID>[] seen = new HashSet[]{ new HashSet<>(), new HashSet<>() };
int si = 0;
UUID uuid;
@@ -430,15 +808,18 @@
for (UntypedResultSet.Row r : result)
{
- if (source.equals(r.getInetAddress("source")))
+ int port = DatabaseDescriptor.getStoragePort();
+ if (r.has("source_port"))
+ port = r.getInt("source_port");
+ InetAddressAndPort eventNode = InetAddressAndPort.getByAddressOverrideDefaults(r.getInetAddress("source"), port);
+ if (source.equals(eventNode))
continue;
if ((uuid = r.getUUID("event_id")).timestamp() > (tcur - 1000) * 10000)
seen[si].add(uuid);
if (seen[si == 0 ? 1 : 0].contains(uuid))
continue;
String message = String.format("%s: %s", r.getInetAddress("source"), r.getString("activity"));
- fireProgressEvent("repair:" + cmd,
- new ProgressEvent(ProgressEventType.NOTIFICATION, 0, 0, message));
+ notification(message);
}
tlast = tcur;
@@ -448,4 +829,26 @@
}
}, "Repair-Runnable-" + threadCounter.incrementAndGet());
}
+
+ private static final class SkipRepairException extends RuntimeException
+ {
+ SkipRepairException(String message)
+ {
+ super(message);
+ }
+ }
+
+ private static final class NeighborsAndRanges
+ {
+ private final boolean force;
+ private final Set<InetAddressAndPort> allNeighbors;
+ private final List<CommonRange> commonRanges;
+
+ private NeighborsAndRanges(boolean force, Set<InetAddressAndPort> allNeighbors, List<CommonRange> commonRanges)
+ {
+ this.force = force;
+ this.allNeighbors = allNeighbors;
+ this.commonRanges = commonRanges;
+ }
+ }
}
diff --git a/src/java/org/apache/cassandra/repair/RepairSession.java b/src/java/org/apache/cassandra/repair/RepairSession.java
index 3d25cbf..2468857 100644
--- a/src/java/org/apache/cassandra/repair/RepairSession.java
+++ b/src/java/org/apache/cassandra/repair/RepairSession.java
@@ -18,7 +18,6 @@
package org.apache.cassandra.repair;
import java.io.IOException;
-import java.net.InetAddress;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
@@ -32,9 +31,18 @@
import org.apache.cassandra.concurrent.DebuggableThreadPoolExecutor;
import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.gms.*;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.repair.consistent.ConsistentSession;
+import org.apache.cassandra.repair.consistent.LocalSession;
+import org.apache.cassandra.repair.consistent.LocalSessions;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.streaming.PreviewKind;
+import org.apache.cassandra.streaming.SessionSummary;
import org.apache.cassandra.tracing.Tracing;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.MerkleTrees;
@@ -53,9 +61,8 @@
* ({@link org.apache.cassandra.repair.ValidationTask}) and waits until all trees are received (in
* validationComplete()).
* </li>
- * <li>Synchronization phase: once all trees are received, the job compares each tree with
- * all the others and creates a {@link SyncTask} for each diverging replica. If there are differences
- * between 2 trees, the concerned SyncTask stream the differences between the 2 endpoints concerned.
+ * <li>Synchronization phase: once all trees are received, the job compares each tree with all the others. If there is
+ * difference between 2 trees, the differences between the 2 endpoints will be streamed with a {@link SyncTask}.
* </li>
* </ol>
* The job is done once all its SyncTasks are done (i.e. have either computed no differences
@@ -73,11 +80,12 @@
* we still first send a message to each node to flush and snapshot data so each merkle tree
* creation is still done on similar data, even if the actual creation is not
* done simulatneously). If not sequential, all merkle tree are requested in parallel.
- * Similarly, if a job is sequential, it will handle one SyncTask at a time, but will handle
+ * Similarly, if a job is sequential, it will handle one SymmetricSyncTask at a time, but will handle
* all of them in parallel otherwise.
*/
public class RepairSession extends AbstractFuture<RepairSessionResult> implements IEndpointStateChangeSubscriber,
- IFailureDetectionEventListener
+ IFailureDetectionEventListener,
+ LocalSessions.Listener
{
private static Logger logger = LoggerFactory.getLogger(RepairSession.class);
@@ -88,44 +96,49 @@
private final String[] cfnames;
public final RepairParallelism parallelismDegree;
public final boolean pullRepair;
+
+ // indicates some replicas were not included in the repair. Only relevant for --force option
+ public final boolean skippedReplicas;
+
/** Range to repair */
- public final Collection<Range<Token>> ranges;
- public final Set<InetAddress> endpoints;
- public final long repairedAt;
+ public final CommonRange commonRange;
+ public final boolean isIncremental;
+ public final PreviewKind previewKind;
private final AtomicBoolean isFailed = new AtomicBoolean(false);
// Each validation task waits response from replica in validating ConcurrentMap (keyed by CF name and endpoint address)
- private final ConcurrentMap<Pair<RepairJobDesc, InetAddress>, ValidationTask> validating = new ConcurrentHashMap<>();
+ private final ConcurrentMap<Pair<RepairJobDesc, InetAddressAndPort>, ValidationTask> validating = new ConcurrentHashMap<>();
// Remote syncing jobs wait response in syncingTasks map
- private final ConcurrentMap<Pair<RepairJobDesc, NodePair>, RemoteSyncTask> syncingTasks = new ConcurrentHashMap<>();
+ private final ConcurrentMap<Pair<RepairJobDesc, SyncNodePair>, CompletableRemoteSyncTask> syncingTasks = new ConcurrentHashMap<>();
// Tasks(snapshot, validate request, differencing, ...) are run on taskExecutor
public final ListeningExecutorService taskExecutor;
+ public final boolean optimiseStreams;
private volatile boolean terminated = false;
/**
* Create new repair session.
- *
* @param parentRepairSession the parent sessions id
* @param id this sessions id
- * @param ranges ranges to repair
+ * @param commonRange ranges to repair
* @param keyspace name of keyspace
* @param parallelismDegree specifies the degree of parallelism when calculating the merkle trees
- * @param endpoints the data centers that should be part of the repair; null for all DCs
- * @param repairedAt when the repair occurred (millis)
* @param pullRepair true if the repair should be one way (from remote host to this host and only applicable between two hosts--see RepairOption)
+ * @param force true if the repair should ignore dead endpoints (instead of failing)
* @param cfnames names of columnfamilies
*/
public RepairSession(UUID parentRepairSession,
UUID id,
- Collection<Range<Token>> ranges,
+ CommonRange commonRange,
String keyspace,
RepairParallelism parallelismDegree,
- Set<InetAddress> endpoints,
- long repairedAt,
+ boolean isIncremental,
boolean pullRepair,
+ boolean force,
+ PreviewKind previewKind,
+ boolean optimiseStreams,
String... cfnames)
{
assert cfnames.length > 0 : "Repairing no column families seems pointless, doesn't it";
@@ -135,14 +148,41 @@
this.parallelismDegree = parallelismDegree;
this.keyspace = keyspace;
this.cfnames = cfnames;
- this.ranges = ranges;
- this.endpoints = endpoints;
- this.repairedAt = repairedAt;
+
+ //If force then filter out dead endpoints
+ boolean forceSkippedReplicas = false;
+ if (force)
+ {
+ logger.debug("force flag set, removing dead endpoints");
+ final Set<InetAddressAndPort> removeCandidates = new HashSet<>();
+ for (final InetAddressAndPort endpoint : commonRange.endpoints)
+ {
+ if (!FailureDetector.instance.isAlive(endpoint))
+ {
+ logger.info("Removing a dead node from Repair due to -force {}", endpoint);
+ removeCandidates.add(endpoint);
+ }
+ }
+ if (!removeCandidates.isEmpty())
+ {
+ // we shouldn't be recording a successful repair if
+ // any replicas are excluded from the repair
+ forceSkippedReplicas = true;
+ Set<InetAddressAndPort> filteredEndpoints = new HashSet<>(commonRange.endpoints);
+ filteredEndpoints.removeAll(removeCandidates);
+ commonRange = new CommonRange(filteredEndpoints, commonRange.transEndpoints, commonRange.ranges);
+ }
+ }
+
+ this.commonRange = commonRange;
+ this.isIncremental = isIncremental;
+ this.previewKind = previewKind;
this.pullRepair = pullRepair;
+ this.skippedReplicas = forceSkippedReplicas;
+ this.optimiseStreams = optimiseStreams;
this.taskExecutor = MoreExecutors.listeningDecorator(createExecutor());
}
- @VisibleForTesting
protected DebuggableThreadPoolExecutor createExecutor()
{
return DebuggableThreadPoolExecutor.createCachedThreadpoolWithMaxSize("RepairJobTask");
@@ -153,17 +193,22 @@
return id;
}
- public Collection<Range<Token>> getRanges()
+ public Collection<Range<Token>> ranges()
{
- return ranges;
+ return commonRange.ranges;
}
- public void waitForValidation(Pair<RepairJobDesc, InetAddress> key, ValidationTask task)
+ public Collection<InetAddressAndPort> endpoints()
+ {
+ return commonRange.endpoints;
+ }
+
+ public void trackValidationCompletion(Pair<RepairJobDesc, InetAddressAndPort> key, ValidationTask task)
{
validating.put(key, task);
}
- public void waitForSync(Pair<RepairJobDesc, NodePair> key, RemoteSyncTask task)
+ public void trackSyncCompletion(Pair<RepairJobDesc, SyncNodePair> key, CompletableRemoteSyncTask task)
{
syncingTasks.put(key, task);
}
@@ -175,7 +220,7 @@
* @param endpoint endpoint that sent merkle tree
* @param trees calculated merkle trees, or null if validation failed
*/
- public void validationComplete(RepairJobDesc desc, InetAddress endpoint, MerkleTrees trees)
+ public void validationComplete(RepairJobDesc desc, InetAddressAndPort endpoint, MerkleTrees trees)
{
ValidationTask task = validating.remove(Pair.create(desc, endpoint));
if (task == null)
@@ -185,33 +230,34 @@
}
String message = String.format("Received merkle tree for %s from %s", desc.columnFamily, endpoint);
- logger.info("[repair #{}] {}", getId(), message);
+ logger.info("{} {}", previewKind.logPrefix(getId()), message);
Tracing.traceRepair(message);
task.treesReceived(trees);
}
/**
- * Notify this session that sync completed/failed with given {@code NodePair}.
+ * Notify this session that sync completed/failed with given {@code SyncNodePair}.
*
* @param desc synced repair job
* @param nodes nodes that completed sync
* @param success true if sync succeeded
*/
- public void syncComplete(RepairJobDesc desc, NodePair nodes, boolean success)
+ public void syncComplete(RepairJobDesc desc, SyncNodePair nodes, boolean success, List<SessionSummary> summaries)
{
- RemoteSyncTask task = syncingTasks.get(Pair.create(desc, nodes));
+ CompletableRemoteSyncTask task = syncingTasks.remove(Pair.create(desc, nodes));
if (task == null)
{
assert terminated;
return;
}
- logger.debug("[repair #{}] Repair completed between {} and {} on {}", getId(), nodes.endpoint1, nodes.endpoint2, desc.columnFamily);
- task.syncComplete(success);
+ if (logger.isDebugEnabled())
+ logger.debug("{} Repair completed between {} and {} on {}", previewKind.logPrefix(getId()), nodes.coordinator, nodes.peer, desc.columnFamily);
+ task.syncComplete(success, summaries);
}
@VisibleForTesting
- Map<Pair<RepairJobDesc, NodePair>, RemoteSyncTask> getSyncingTasks()
+ Map<Pair<RepairJobDesc, SyncNodePair>, CompletableRemoteSyncTask> getSyncingTasks()
{
return Collections.unmodifiableMap(syncingTasks);
}
@@ -219,8 +265,8 @@
private String repairedNodes()
{
StringBuilder sb = new StringBuilder();
- sb.append(FBUtilities.getBroadcastAddress());
- for (InetAddress ep : endpoints)
+ sb.append(FBUtilities.getBroadcastAddressAndPort());
+ for (InetAddressAndPort ep : commonRange.endpoints)
sb.append(", ").append(ep);
return sb.toString();
}
@@ -239,29 +285,39 @@
if (terminated)
return;
- logger.info("[repair #{}] new session: will sync {} on range {} for {}.{}", getId(), repairedNodes(), ranges, keyspace, Arrays.toString(cfnames));
- Tracing.traceRepair("Syncing range {}", ranges);
- SystemDistributedKeyspace.startRepairs(getId(), parentRepairSession, keyspace, cfnames, ranges, endpoints);
-
- if (endpoints.isEmpty())
+ logger.info("{} parentSessionId = {}: new session: will sync {} on range {} for {}.{}",
+ previewKind.logPrefix(getId()), parentRepairSession, repairedNodes(), commonRange, keyspace, Arrays.toString(cfnames));
+ Tracing.traceRepair("Syncing range {}", commonRange);
+ if (!previewKind.isPreview())
{
- logger.info("[repair #{}] {}", getId(), message = String.format("No neighbors to repair with on range %s: session completed", ranges));
+ SystemDistributedKeyspace.startRepairs(getId(), parentRepairSession, keyspace, cfnames, commonRange);
+ }
+
+ if (commonRange.endpoints.isEmpty())
+ {
+ logger.info("{} {}", previewKind.logPrefix(getId()), message = String.format("No neighbors to repair with on range %s: session completed", commonRange));
Tracing.traceRepair(message);
- set(new RepairSessionResult(id, keyspace, ranges, Lists.<RepairResult>newArrayList()));
- SystemDistributedKeyspace.failRepairs(getId(), keyspace, cfnames, new RuntimeException(message));
+ set(new RepairSessionResult(id, keyspace, commonRange.ranges, Lists.<RepairResult>newArrayList(), skippedReplicas));
+ if (!previewKind.isPreview())
+ {
+ SystemDistributedKeyspace.failRepairs(getId(), keyspace, cfnames, new RuntimeException(message));
+ }
return;
}
// Checking all nodes are live
- for (InetAddress endpoint : endpoints)
+ for (InetAddressAndPort endpoint : commonRange.endpoints)
{
- if (!FailureDetector.instance.isAlive(endpoint))
+ if (!FailureDetector.instance.isAlive(endpoint) && !skippedReplicas)
{
message = String.format("Cannot proceed on repair because a neighbor (%s) is dead: session failed", endpoint);
- logger.error("[repair #{}] {}", getId(), message);
+ logger.error("{} {}", previewKind.logPrefix(getId()), message);
Exception e = new IOException(message);
setException(e);
- SystemDistributedKeyspace.failRepairs(getId(), keyspace, cfnames, e);
+ if (!previewKind.isPreview())
+ {
+ SystemDistributedKeyspace.failRepairs(getId(), keyspace, cfnames, e);
+ }
return;
}
}
@@ -281,9 +337,9 @@
public void onSuccess(List<RepairResult> results)
{
// this repair session is completed
- logger.info("[repair #{}] {}", getId(), "Session completed successfully");
- Tracing.traceRepair("Completed sync of range {}", ranges);
- set(new RepairSessionResult(id, keyspace, ranges, results));
+ logger.info("{} {}", previewKind.logPrefix(getId()), "Session completed successfully");
+ Tracing.traceRepair("Completed sync of range {}", commonRange);
+ set(new RepairSessionResult(id, keyspace, commonRange.ranges, results, skippedReplicas));
taskExecutor.shutdown();
// mark this session as terminated
@@ -292,11 +348,11 @@
public void onFailure(Throwable t)
{
- logger.error(String.format("[repair #%s] Session completed with the following error", getId()), t);
+ logger.error("{} Session completed with the following error", previewKind.logPrefix(getId()), t);
Tracing.traceRepair("Session completed with the following error: {}", t);
forceShutdown(t);
}
- });
+ }, MoreExecutors.directExecutor());
}
public void terminate()
@@ -318,25 +374,25 @@
terminate();
}
- public void onJoin(InetAddress endpoint, EndpointState epState) {}
- public void beforeChange(InetAddress endpoint, EndpointState currentState, ApplicationState newStateKey, VersionedValue newValue) {}
- public void onChange(InetAddress endpoint, ApplicationState state, VersionedValue value) {}
- public void onAlive(InetAddress endpoint, EndpointState state) {}
- public void onDead(InetAddress endpoint, EndpointState state) {}
+ public void onJoin(InetAddressAndPort endpoint, EndpointState epState) {}
+ public void beforeChange(InetAddressAndPort endpoint, EndpointState currentState, ApplicationState newStateKey, VersionedValue newValue) {}
+ public void onChange(InetAddressAndPort endpoint, ApplicationState state, VersionedValue value) {}
+ public void onAlive(InetAddressAndPort endpoint, EndpointState state) {}
+ public void onDead(InetAddressAndPort endpoint, EndpointState state) {}
- public void onRemove(InetAddress endpoint)
+ public void onRemove(InetAddressAndPort endpoint)
{
convict(endpoint, Double.MAX_VALUE);
}
- public void onRestart(InetAddress endpoint, EndpointState epState)
+ public void onRestart(InetAddressAndPort endpoint, EndpointState epState)
{
convict(endpoint, Double.MAX_VALUE);
}
- public void convict(InetAddress endpoint, double phi)
+ public void convict(InetAddressAndPort endpoint, double phi)
{
- if (!endpoints.contains(endpoint))
+ if (!commonRange.endpoints.contains(endpoint))
return;
// We want a higher confidence in the failure detection than usual because failing a repair wrongly has a high cost.
@@ -349,8 +405,42 @@
return;
Exception exception = new IOException(String.format("Endpoint %s died", endpoint));
- logger.error(String.format("[repair #%s] session completed with the following error", getId()), exception);
+ logger.error("{} session completed with the following error", previewKind.logPrefix(getId()), exception);
// If a node failed, we stop everything (though there could still be some activity in the background)
forceShutdown(exception);
}
+
+ public void onIRStateChange(LocalSession session)
+ {
+ // we should only be registered as listeners for PreviewKind.REPAIRED, but double check here
+ if (previewKind == PreviewKind.REPAIRED &&
+ session.getState() == ConsistentSession.State.FINALIZED &&
+ includesTables(session.tableIds))
+ {
+ for (Range<Token> range : session.ranges)
+ {
+ if (range.intersects(ranges()))
+ {
+ logger.error("{} An intersecting incremental repair with session id = {} finished, preview repair might not be accurate", previewKind.logPrefix(getId()), session.sessionID);
+ forceShutdown(new Exception("An incremental repair with session id "+session.sessionID+" finished during this preview repair runtime"));
+ return;
+ }
+ }
+ }
+ }
+
+ private boolean includesTables(Set<TableId> tableIds)
+ {
+ Keyspace ks = Keyspace.open(keyspace);
+ if (ks != null)
+ {
+ for (String table : cfnames)
+ {
+ ColumnFamilyStore cfs = ks.getColumnFamilyStore(table);
+ if (tableIds.contains(cfs.metadata.id))
+ return true;
+ }
+ }
+ return false;
+ }
}
diff --git a/src/java/org/apache/cassandra/repair/RepairSessionResult.java b/src/java/org/apache/cassandra/repair/RepairSessionResult.java
index d4fff37..491ab2f 100644
--- a/src/java/org/apache/cassandra/repair/RepairSessionResult.java
+++ b/src/java/org/apache/cassandra/repair/RepairSessionResult.java
@@ -32,12 +32,25 @@
public final String keyspace;
public final Collection<Range<Token>> ranges;
public final Collection<RepairResult> repairJobResults;
+ public final boolean skippedReplicas;
- public RepairSessionResult(UUID sessionId, String keyspace, Collection<Range<Token>> ranges, Collection<RepairResult> repairJobResults)
+ public RepairSessionResult(UUID sessionId, String keyspace, Collection<Range<Token>> ranges, Collection<RepairResult> repairJobResults, boolean skippedReplicas)
{
this.sessionId = sessionId;
this.keyspace = keyspace;
this.ranges = ranges;
this.repairJobResults = repairJobResults;
+ this.skippedReplicas = skippedReplicas;
+ }
+
+ public String toString()
+ {
+ return "RepairSessionResult{" +
+ "sessionId=" + sessionId +
+ ", keyspace='" + keyspace + '\'' +
+ ", ranges=" + ranges +
+ ", repairJobResults=" + repairJobResults +
+ ", skippedReplicas=" + skippedReplicas +
+ '}';
}
}
diff --git a/src/java/org/apache/cassandra/repair/SnapshotTask.java b/src/java/org/apache/cassandra/repair/SnapshotTask.java
index 2b267a7..40e4b3d 100644
--- a/src/java/org/apache/cassandra/repair/SnapshotTask.java
+++ b/src/java/org/apache/cassandra/repair/SnapshotTask.java
@@ -17,27 +17,28 @@
*/
package org.apache.cassandra.repair;
-import java.net.InetAddress;
import java.util.concurrent.RunnableFuture;
-import java.util.concurrent.TimeUnit;
import com.google.common.util.concurrent.AbstractFuture;
import org.apache.cassandra.exceptions.RequestFailureReason;
-import org.apache.cassandra.net.IAsyncCallbackWithFailure;
-import org.apache.cassandra.net.MessageIn;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.net.RequestCallback;
+import org.apache.cassandra.net.Message;
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.repair.messages.SnapshotMessage;
+import static org.apache.cassandra.net.Verb.SNAPSHOT_MSG;
+
/**
* SnapshotTask is a task that sends snapshot request.
*/
-public class SnapshotTask extends AbstractFuture<InetAddress> implements RunnableFuture<InetAddress>
+public class SnapshotTask extends AbstractFuture<InetAddressAndPort> implements RunnableFuture<InetAddressAndPort>
{
private final RepairJobDesc desc;
- private final InetAddress endpoint;
+ private final InetAddressAndPort endpoint;
- public SnapshotTask(RepairJobDesc desc, InetAddress endpoint)
+ SnapshotTask(RepairJobDesc desc, InetAddressAndPort endpoint)
{
this.desc = desc;
this.endpoint = endpoint;
@@ -45,15 +46,15 @@
public void run()
{
- MessagingService.instance().sendRR(new SnapshotMessage(desc).createMessage(),
- endpoint,
- new SnapshotCallback(this), TimeUnit.HOURS.toMillis(1), true);
+ MessagingService.instance().sendWithCallback(Message.out(SNAPSHOT_MSG, new SnapshotMessage(desc)),
+ endpoint,
+ new SnapshotCallback(this));
}
/**
* Callback for snapshot request. Run on INTERNAL_RESPONSE stage.
*/
- static class SnapshotCallback implements IAsyncCallbackWithFailure
+ static class SnapshotCallback implements RequestCallback
{
final SnapshotTask task;
@@ -67,14 +68,20 @@
*
* @param msg response received.
*/
- public void response(MessageIn msg)
+ @Override
+ public void onResponse(Message msg)
{
task.set(task.endpoint);
}
- public boolean isLatencyForSnitch() { return false; }
+ @Override
+ public boolean invokeOnFailure()
+ {
+ return true;
+ }
- public void onFailure(InetAddress from, RequestFailureReason failureReason)
+ @Override
+ public void onFailure(InetAddressAndPort from, RequestFailureReason failureReason)
{
//listener.failedSnapshot();
task.setException(new RuntimeException("Could not create snapshot at " + from));
diff --git a/src/java/org/apache/cassandra/repair/SomeRepairFailedException.java b/src/java/org/apache/cassandra/repair/SomeRepairFailedException.java
new file mode 100644
index 0000000..4b077b8
--- /dev/null
+++ b/src/java/org/apache/cassandra/repair/SomeRepairFailedException.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.repair;
+
+/**
+ * This is a special exception which states "I know something failed but I don't have access to the failure". This
+ * is mostly used to make sure the error notifications are clean and the history table has a meaningful exception.
+ *
+ * The expected behavior is that when this is thrown, this error should be ignored from history table and not used
+ * for notifications
+ */
+public class SomeRepairFailedException extends RuntimeException
+{
+ public static final SomeRepairFailedException INSTANCE = new SomeRepairFailedException();
+}
diff --git a/src/java/org/apache/cassandra/repair/StreamingRepairTask.java b/src/java/org/apache/cassandra/repair/StreamingRepairTask.java
index f5b2b1d..827dce3 100644
--- a/src/java/org/apache/cassandra/repair/StreamingRepairTask.java
+++ b/src/java/org/apache/cassandra/repair/StreamingRepairTask.java
@@ -17,58 +17,78 @@
*/
package org.apache.cassandra.repair;
-import java.net.InetAddress;
+import java.util.UUID;
+import java.util.Collections;
+import java.util.Collection;
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.cassandra.locator.RangesAtEndpoint;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.cassandra.db.SystemKeyspace;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.net.Message;
import org.apache.cassandra.net.MessagingService;
-import org.apache.cassandra.repair.messages.SyncComplete;
-import org.apache.cassandra.repair.messages.SyncRequest;
-import org.apache.cassandra.service.ActiveRepairService;
+import org.apache.cassandra.repair.messages.SyncResponse;
+import org.apache.cassandra.streaming.PreviewKind;
import org.apache.cassandra.streaming.StreamEvent;
import org.apache.cassandra.streaming.StreamEventHandler;
import org.apache.cassandra.streaming.StreamPlan;
import org.apache.cassandra.streaming.StreamState;
+import org.apache.cassandra.streaming.StreamOperation;
+
+import static org.apache.cassandra.net.Verb.SYNC_RSP;
/**
- * StreamingRepairTask performs data streaming between two remote replica which neither is not repair coordinator.
- * Task will send {@link SyncComplete} message back to coordinator upon streaming completion.
+ * StreamingRepairTask performs data streaming between two remote replicas, neither of which is repair coordinator.
+ * Task will send {@link SyncResponse} message back to coordinator upon streaming completion.
*/
public class StreamingRepairTask implements Runnable, StreamEventHandler
{
private static final Logger logger = LoggerFactory.getLogger(StreamingRepairTask.class);
private final RepairJobDesc desc;
- private final SyncRequest request;
- private final long repairedAt;
+ private final boolean asymmetric;
+ private final InetAddressAndPort initiator;
+ private final InetAddressAndPort src;
+ private final InetAddressAndPort dst;
+ private final Collection<Range<Token>> ranges;
+ private final UUID pendingRepair;
+ private final PreviewKind previewKind;
- public StreamingRepairTask(RepairJobDesc desc, SyncRequest request, long repairedAt)
+ public StreamingRepairTask(RepairJobDesc desc, InetAddressAndPort initiator, InetAddressAndPort src, InetAddressAndPort dst, Collection<Range<Token>> ranges, UUID pendingRepair, PreviewKind previewKind, boolean asymmetric)
{
this.desc = desc;
- this.request = request;
- this.repairedAt = repairedAt;
+ this.initiator = initiator;
+ this.src = src;
+ this.dst = dst;
+ this.ranges = ranges;
+ this.asymmetric = asymmetric;
+ this.pendingRepair = pendingRepair;
+ this.previewKind = previewKind;
}
public void run()
{
- InetAddress dest = request.dst;
- InetAddress preferred = SystemKeyspace.getPreferredIP(dest);
- logger.info("[streaming task #{}] Performing streaming repair of {} ranges with {}", desc.sessionId, request.ranges.size(), request.dst);
- boolean isIncremental = false;
- if (desc.parentSessionId != null)
- {
- ActiveRepairService.ParentRepairSession prs = ActiveRepairService.instance.getParentRepairSession(desc.parentSessionId);
- isIncremental = prs.isIncremental;
- }
- new StreamPlan("Repair", repairedAt, 1, false, isIncremental, false).listeners(this)
- .flushBeforeTransfer(true)
- // request ranges from the remote node
- .requestRanges(dest, preferred, desc.keyspace, request.ranges, desc.columnFamily)
- // send ranges to the remote node
- .transferRanges(dest, preferred, desc.keyspace, request.ranges, desc.columnFamily)
- .execute();
+ logger.info("[streaming task #{}] Performing streaming repair of {} ranges with {}", desc.sessionId, ranges.size(), dst);
+ createStreamPlan(dst).execute();
+ }
+
+ @VisibleForTesting
+ StreamPlan createStreamPlan(InetAddressAndPort dest)
+ {
+ StreamPlan sp = new StreamPlan(StreamOperation.REPAIR, 1, false, pendingRepair, previewKind)
+ .listeners(this)
+ .flushBeforeTransfer(pendingRepair == null) // sstables are isolated at the beginning of an incremental repair session, so flushing isn't neccessary
+ // see comment on RangesAtEndpoint.toDummyList for why we synthesize replicas here
+ .requestRanges(dest, desc.keyspace, RangesAtEndpoint.toDummyList(ranges),
+ RangesAtEndpoint.toDummyList(Collections.emptyList()), desc.columnFamily); // request ranges from the remote node
+ if (!asymmetric)
+ // see comment on RangesAtEndpoint.toDummyList for why we synthesize replicas here
+ sp.transferRanges(dest, desc.keyspace, RangesAtEndpoint.toDummyList(ranges), desc.columnFamily); // send ranges to the remote node
+ return sp;
}
public void handleStreamEvent(StreamEvent event)
@@ -78,19 +98,19 @@
}
/**
- * If we succeeded on both stream in and out, reply back to coordinator
+ * If we succeeded on both stream in and out, respond back to coordinator
*/
public void onSuccess(StreamState state)
{
- logger.info("[repair #{}] streaming task succeed, returning response to {}", desc.sessionId, request.initiator);
- MessagingService.instance().sendOneWay(new SyncComplete(desc, request.src, request.dst, true).createMessage(), request.initiator);
+ logger.info("[repair #{}] streaming task succeed, returning response to {}", desc.sessionId, initiator);
+ MessagingService.instance().send(Message.out(SYNC_RSP, new SyncResponse(desc, src, dst, true, state.createSummaries())), initiator);
}
/**
- * If we failed on either stream in or out, reply fail to coordinator
+ * If we failed on either stream in or out, respond fail to coordinator
*/
public void onFailure(Throwable t)
{
- MessagingService.instance().sendOneWay(new SyncComplete(desc, request.src, request.dst, false).createMessage(), request.initiator);
+ MessagingService.instance().send(Message.out(SYNC_RSP, new SyncResponse(desc, src, dst, false, Collections.emptyList())), initiator);
}
}
diff --git a/src/java/org/apache/cassandra/repair/SymmetricRemoteSyncTask.java b/src/java/org/apache/cassandra/repair/SymmetricRemoteSyncTask.java
new file mode 100644
index 0000000..181554a
--- /dev/null
+++ b/src/java/org/apache/cassandra/repair/SymmetricRemoteSyncTask.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.repair;
+
+import java.util.List;
+
+import com.google.common.base.Preconditions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.exceptions.RepairException;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.net.Message;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.repair.messages.RepairMessage;
+import org.apache.cassandra.repair.messages.SyncRequest;
+import org.apache.cassandra.streaming.PreviewKind;
+import org.apache.cassandra.streaming.SessionSummary;
+import org.apache.cassandra.tracing.Tracing;
+import org.apache.cassandra.utils.FBUtilities;
+
+import static org.apache.cassandra.net.Verb.SYNC_REQ;
+
+/**
+ * SymmetricRemoteSyncTask sends {@link SyncRequest} to remote(non-coordinator) node
+ * to repair(stream) data with other replica.
+ *
+ * When SymmetricRemoteSyncTask receives SyncComplete from remote node, task completes.
+ */
+public class SymmetricRemoteSyncTask extends SyncTask implements CompletableRemoteSyncTask
+{
+ private static final Logger logger = LoggerFactory.getLogger(SymmetricRemoteSyncTask.class);
+
+ public SymmetricRemoteSyncTask(RepairJobDesc desc, InetAddressAndPort r1, InetAddressAndPort r2, List<Range<Token>> differences, PreviewKind previewKind)
+ {
+ super(desc, r1, r2, differences, previewKind);
+ }
+
+ void sendRequest(SyncRequest request, InetAddressAndPort to)
+ {
+ MessagingService.instance().send(Message.out(SYNC_REQ, request), to);
+ }
+
+ @Override
+ protected void startSync()
+ {
+ InetAddressAndPort local = FBUtilities.getBroadcastAddressAndPort();
+
+ SyncRequest request = new SyncRequest(desc, local, nodePair.coordinator, nodePair.peer, rangesToSync, previewKind);
+ Preconditions.checkArgument(nodePair.coordinator.equals(request.src));
+ String message = String.format("Forwarding streaming repair of %d ranges to %s (to be streamed with %s)", request.ranges.size(), request.src, request.dst);
+ logger.info("{} {}", previewKind.logPrefix(desc.sessionId), message);
+ Tracing.traceRepair(message);
+ sendRequest(request, request.src);
+ }
+
+ public void syncComplete(boolean success, List<SessionSummary> summaries)
+ {
+ if (success)
+ {
+ set(stat.withSummaries(summaries));
+ }
+ else
+ {
+ setException(new RepairException(desc, previewKind, String.format("Sync failed between %s and %s", nodePair.coordinator, nodePair.peer)));
+ }
+ finished();
+ }
+
+ @Override
+ public String toString()
+ {
+ return "SymmetricRemoteSyncTask{" +
+ "rangesToSync=" + rangesToSync +
+ ", nodePair=" + nodePair +
+ '}';
+ }
+}
diff --git a/src/java/org/apache/cassandra/repair/SyncNodePair.java b/src/java/org/apache/cassandra/repair/SyncNodePair.java
new file mode 100644
index 0000000..e10ad5a
--- /dev/null
+++ b/src/java/org/apache/cassandra/repair/SyncNodePair.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.repair;
+
+import java.io.IOException;
+
+import com.google.common.base.Objects;
+
+import org.apache.cassandra.io.IVersionedSerializer;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.locator.InetAddressAndPort;
+
+import static org.apache.cassandra.locator.InetAddressAndPort.Serializer.inetAddressAndPortSerializer;
+
+/**
+ * SyncNodePair is used for repair message body to indicate the pair of nodes.
+ *
+ * @since 2.0
+ */
+public class SyncNodePair
+{
+ public static IVersionedSerializer<SyncNodePair> serializer = new NodePairSerializer();
+
+ public final InetAddressAndPort coordinator;
+ public final InetAddressAndPort peer;
+
+ public SyncNodePair(InetAddressAndPort coordinator, InetAddressAndPort peer)
+ {
+ this.coordinator = coordinator;
+ this.peer = peer;
+ }
+
+ @Override
+ public boolean equals(Object o)
+ {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ SyncNodePair nodePair = (SyncNodePair) o;
+ return coordinator.equals(nodePair.coordinator) && peer.equals(nodePair.peer);
+ }
+
+ @Override
+ public String toString()
+ {
+ return coordinator.toString() + " - " + peer.toString();
+ }
+
+ @Override
+ public int hashCode()
+ {
+ return Objects.hashCode(coordinator, peer);
+ }
+
+ public static class NodePairSerializer implements IVersionedSerializer<SyncNodePair>
+ {
+ public void serialize(SyncNodePair nodePair, DataOutputPlus out, int version) throws IOException
+ {
+ inetAddressAndPortSerializer.serialize(nodePair.coordinator, out, version);
+ inetAddressAndPortSerializer.serialize(nodePair.peer, out, version);
+ }
+
+ public SyncNodePair deserialize(DataInputPlus in, int version) throws IOException
+ {
+ InetAddressAndPort ep1 = inetAddressAndPortSerializer.deserialize(in, version);
+ InetAddressAndPort ep2 = inetAddressAndPortSerializer.deserialize(in, version);
+ return new SyncNodePair(ep1, ep2);
+ }
+
+ public long serializedSize(SyncNodePair nodePair, int version)
+ {
+ return inetAddressAndPortSerializer.serializedSize(nodePair.coordinator, version)
+ + inetAddressAndPortSerializer.serializedSize(nodePair.peer, version);
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/repair/SyncStat.java b/src/java/org/apache/cassandra/repair/SyncStat.java
index 5721a20..7bb503f 100644
--- a/src/java/org/apache/cassandra/repair/SyncStat.java
+++ b/src/java/org/apache/cassandra/repair/SyncStat.java
@@ -17,17 +17,33 @@
*/
package org.apache.cassandra.repair;
+import java.util.List;
+
+import org.apache.cassandra.streaming.SessionSummary;
+
/**
* Statistics about synchronizing two replica
*/
public class SyncStat
{
- public final NodePair nodes;
- public final long numberOfDifferences;
+ public final SyncNodePair nodes;
+ public final long numberOfDifferences; // TODO: revert to Range<Token>
+ public final List<SessionSummary> summaries;
- public SyncStat(NodePair nodes, long numberOfDifferences)
+ public SyncStat(SyncNodePair nodes, long numberOfDifferences)
+ {
+ this(nodes, numberOfDifferences, null);
+ }
+
+ public SyncStat(SyncNodePair nodes, long numberOfDifferences, List<SessionSummary> summaries)
{
this.nodes = nodes;
this.numberOfDifferences = numberOfDifferences;
+ this.summaries = summaries;
+ }
+
+ public SyncStat withSummaries(List<SessionSummary> summaries)
+ {
+ return new SyncStat(nodes, numberOfDifferences, summaries);
}
}
diff --git a/src/java/org/apache/cassandra/repair/SyncTask.java b/src/java/org/apache/cassandra/repair/SyncTask.java
index c96caf4..d0f1eca 100644
--- a/src/java/org/apache/cassandra/repair/SyncTask.java
+++ b/src/java/org/apache/cassandra/repair/SyncTask.java
@@ -15,70 +15,88 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.cassandra.repair;
-import java.net.InetAddress;
import java.util.List;
+import java.util.concurrent.TimeUnit;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
import com.google.common.util.concurrent.AbstractFuture;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.streaming.PreviewKind;
import org.apache.cassandra.tracing.Tracing;
-/**
- * SyncTask takes the difference of MerkleTrees between two nodes
- * and perform necessary operation to repair replica.
- */
public abstract class SyncTask extends AbstractFuture<SyncStat> implements Runnable
{
private static Logger logger = LoggerFactory.getLogger(SyncTask.class);
protected final RepairJobDesc desc;
- protected final InetAddress firstEndpoint;
- protected final InetAddress secondEndpoint;
+ @VisibleForTesting
+ public final List<Range<Token>> rangesToSync;
+ protected final PreviewKind previewKind;
+ protected final SyncNodePair nodePair;
- private final List<Range<Token>> rangesToSync;
+ protected volatile long startTime = Long.MIN_VALUE;
+ protected final SyncStat stat;
- protected volatile SyncStat stat;
-
- public SyncTask(RepairJobDesc desc, InetAddress firstEndpoint, InetAddress secondEndpoint, List<Range<Token>> rangesToSync)
+ protected SyncTask(RepairJobDesc desc, InetAddressAndPort primaryEndpoint, InetAddressAndPort peer, List<Range<Token>> rangesToSync, PreviewKind previewKind)
{
+ Preconditions.checkArgument(!peer.equals(primaryEndpoint), "Sending and receiving node are the same: %s", peer);
this.desc = desc;
- this.firstEndpoint = firstEndpoint;
- this.secondEndpoint = secondEndpoint;
this.rangesToSync = rangesToSync;
+ this.nodePair = new SyncNodePair(primaryEndpoint, peer);
+ this.previewKind = previewKind;
+ this.stat = new SyncStat(nodePair, rangesToSync.size());
+ }
+
+ protected abstract void startSync();
+
+ public SyncNodePair nodePair()
+ {
+ return nodePair;
}
/**
* Compares trees, and triggers repairs for any ranges that mismatch.
*/
- public void run()
+ public final void run()
{
- stat = new SyncStat(new NodePair(firstEndpoint, secondEndpoint), rangesToSync.size());
+ startTime = System.currentTimeMillis();
+
// choose a repair method based on the significance of the difference
- String format = String.format("[repair #%s] Endpoints %s and %s %%s for %s", desc.sessionId, firstEndpoint, secondEndpoint, desc.columnFamily);
+ String format = String.format("%s Endpoints %s and %s %%s for %s", previewKind.logPrefix(desc.sessionId), nodePair.coordinator, nodePair.peer, desc.columnFamily);
if (rangesToSync.isEmpty())
{
logger.info(String.format(format, "are consistent"));
- Tracing.traceRepair("Endpoint {} is consistent with {} for {}", firstEndpoint, secondEndpoint, desc.columnFamily);
+ Tracing.traceRepair("Endpoint {} is consistent with {} for {}", nodePair.coordinator, nodePair.peer, desc.columnFamily);
set(stat);
return;
}
// non-0 difference: perform streaming repair
logger.info(String.format(format, "have " + rangesToSync.size() + " range(s) out of sync"));
- Tracing.traceRepair("Endpoint {} has {} range(s) out of sync with {} for {}", firstEndpoint, rangesToSync.size(), secondEndpoint, desc.columnFamily);
- startSync(rangesToSync);
+ Tracing.traceRepair("Endpoint {} has {} range(s) out of sync with {} for {}", nodePair.coordinator, rangesToSync.size(), nodePair.peer, desc.columnFamily);
+ startSync();
}
- public SyncStat getCurrentStat()
+ public boolean isLocal()
{
- return stat;
+ return false;
}
- protected abstract void startSync(List<Range<Token>> differences);
+ protected void finished()
+ {
+ if (startTime != Long.MIN_VALUE)
+ Keyspace.open(desc.keyspace).getColumnFamilyStore(desc.columnFamily).metric.syncTime.update(System.currentTimeMillis() - startTime, TimeUnit.MILLISECONDS);
+ }
}
diff --git a/src/java/org/apache/cassandra/repair/SystemDistributedKeyspace.java b/src/java/org/apache/cassandra/repair/SystemDistributedKeyspace.java
index eb2226b..2e3b981 100644
--- a/src/java/org/apache/cassandra/repair/SystemDistributedKeyspace.java
+++ b/src/java/org/apache/cassandra/repair/SystemDistributedKeyspace.java
@@ -19,40 +19,47 @@
import java.io.PrintWriter;
import java.io.StringWriter;
-import java.net.InetAddress;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
-import java.util.concurrent.TimeUnit;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
+import java.util.concurrent.TimeUnit;
import com.google.common.base.Joiner;
import com.google.common.collect.Lists;
+import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Sets;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.config.SchemaConstants;
import org.apache.cassandra.cql3.QueryProcessor;
import org.apache.cassandra.cql3.UntypedResultSet;
+import org.apache.cassandra.cql3.statements.schema.CreateTableStatement;
import org.apache.cassandra.db.ConsistencyLevel;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.gms.Gossiper;
+import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.repair.messages.RepairOption;
+import org.apache.cassandra.schema.CompactionParams;
import org.apache.cassandra.schema.KeyspaceMetadata;
import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.SchemaConstants;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.schema.Tables;
import org.apache.cassandra.utils.FBUtilities;
+import static java.lang.String.format;
+
import static org.apache.cassandra.utils.ByteBufferUtil.bytes;
public final class SystemDistributedKeyspace
@@ -72,8 +79,10 @@
* gen 1: (pre-)add options column to parent_repair_history in 3.0, 3.11
* gen 2: (pre-)add coordinator_port and participants_v2 columns to repair_history in 3.0, 3.11, 4.0
* gen 3: gc_grace_seconds raised from 0 to 10 days in CASSANDRA-12954 in 3.11.0
+ * gen 4: compression chunk length reduced to 16KiB, memtable_flush_period_in_ms now unset on all tables in 4.0
+ * gen 5: add ttl and TWCS to repair_history tables
*/
- public static final long GENERATION = 3;
+ public static final long GENERATION = 5;
public static final String REPAIR_HISTORY = "repair_history";
@@ -81,8 +90,8 @@
public static final String VIEW_BUILD_STATUS = "view_build_status";
- private static final CFMetaData RepairHistory =
- compile(REPAIR_HISTORY,
+ private static final TableMetadata RepairHistory =
+ parse(REPAIR_HISTORY,
"Repair history",
"CREATE TABLE %s ("
+ "keyspace_name text,"
@@ -100,10 +109,14 @@
+ "status text,"
+ "started_at timestamp,"
+ "finished_at timestamp,"
- + "PRIMARY KEY ((keyspace_name, columnfamily_name), id))");
+ + "PRIMARY KEY ((keyspace_name, columnfamily_name), id))")
+ .defaultTimeToLive((int) TimeUnit.DAYS.toSeconds(30))
+ .compaction(CompactionParams.twcs(ImmutableMap.of("compaction_window_unit","DAYS",
+ "compaction_window_size","1")))
+ .build();
- private static final CFMetaData ParentRepairHistory =
- compile(PARENT_REPAIR_HISTORY,
+ private static final TableMetadata ParentRepairHistory =
+ parse(PARENT_REPAIR_HISTORY,
"Repair history",
"CREATE TABLE %s ("
+ "parent_id timeuuid,"
@@ -116,23 +129,27 @@
+ "requested_ranges set<text>,"
+ "successful_ranges set<text>,"
+ "options map<text, text>,"
- + "PRIMARY KEY (parent_id))");
+ + "PRIMARY KEY (parent_id))")
+ .defaultTimeToLive((int) TimeUnit.DAYS.toSeconds(30))
+ .compaction(CompactionParams.twcs(ImmutableMap.of("compaction_window_unit","DAYS",
+ "compaction_window_size","1")))
+ .build();
- private static final CFMetaData ViewBuildStatus =
- compile(VIEW_BUILD_STATUS,
+ private static final TableMetadata ViewBuildStatus =
+ parse(VIEW_BUILD_STATUS,
"Materialized View build status",
"CREATE TABLE %s ("
+ "keyspace_name text,"
+ "view_name text,"
+ "host_id uuid,"
+ "status text,"
- + "PRIMARY KEY ((keyspace_name, view_name), host_id))");
+ + "PRIMARY KEY ((keyspace_name, view_name), host_id))").build();
- private static CFMetaData compile(String name, String description, String schema)
+ private static TableMetadata.Builder parse(String table, String description, String cql)
{
- return CFMetaData.compile(String.format(schema, name), SchemaConstants.DISTRIBUTED_KEYSPACE_NAME)
- .comment(description)
- .gcGraceSeconds((int) TimeUnit.DAYS.toSeconds(10));
+ return CreateTableStatement.parse(format(cql, table), SchemaConstants.DISTRIBUTED_KEYSPACE_NAME)
+ .id(TableId.forSystemTable(SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, table))
+ .comment(description);
}
public static KeyspaceMetadata metadata()
@@ -145,7 +162,7 @@
Collection<Range<Token>> ranges = options.getRanges();
String query = "INSERT INTO %s.%s (parent_id, keyspace_name, columnfamily_names, requested_ranges, started_at, options)"+
" VALUES (%s, '%s', { '%s' }, { '%s' }, toTimestamp(now()), { %s })";
- String fmtQry = String.format(query,
+ String fmtQry = format(query,
SchemaConstants.DISTRIBUTED_KEYSPACE_NAME,
PARENT_REPAIR_HISTORY,
parent_id.toString(),
@@ -168,7 +185,7 @@
if (!first)
map.append(',');
first = false;
- map.append(String.format("'%s': '%s'", entry.getKey(), entry.getValue()));
+ map.append(format("'%s': '%s'", entry.getKey(), entry.getValue()));
}
}
return map.toString();
@@ -181,43 +198,74 @@
StringWriter sw = new StringWriter();
PrintWriter pw = new PrintWriter(sw);
t.printStackTrace(pw);
- String fmtQuery = String.format(query, SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, PARENT_REPAIR_HISTORY, parent_id.toString());
- processSilent(fmtQuery, t.getMessage(), sw.toString());
+ String fmtQuery = format(query, SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, PARENT_REPAIR_HISTORY, parent_id.toString());
+ String message = t.getMessage();
+ processSilent(fmtQuery, message != null ? message : "", sw.toString());
}
public static void successfulParentRepair(UUID parent_id, Collection<Range<Token>> successfulRanges)
{
String query = "UPDATE %s.%s SET finished_at = toTimestamp(now()), successful_ranges = {'%s'} WHERE parent_id=%s";
- String fmtQuery = String.format(query, SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, PARENT_REPAIR_HISTORY, Joiner.on("','").join(successfulRanges), parent_id.toString());
+ String fmtQuery = format(query, SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, PARENT_REPAIR_HISTORY, Joiner.on("','").join(successfulRanges), parent_id.toString());
processSilent(fmtQuery);
}
- public static void startRepairs(UUID id, UUID parent_id, String keyspaceName, String[] cfnames, Collection<Range<Token>> ranges, Iterable<InetAddress> endpoints)
+ public static void startRepairs(UUID id, UUID parent_id, String keyspaceName, String[] cfnames, CommonRange commonRange)
{
- String coordinator = FBUtilities.getBroadcastAddress().getHostAddress();
- Set<String> participants = Sets.newHashSet(coordinator);
+ //Don't record repair history if an upgrade is in progress as version 3 nodes generates errors
+ //due to schema differences
+ boolean includeNewColumns = !Gossiper.instance.haveMajorVersion3Nodes();
- for (InetAddress endpoint : endpoints)
- participants.add(endpoint.getHostAddress());
+ InetAddressAndPort coordinator = FBUtilities.getBroadcastAddressAndPort();
+ Set<String> participants = Sets.newHashSet();
+ Set<String> participants_v2 = Sets.newHashSet();
+
+ for (InetAddressAndPort endpoint : commonRange.endpoints)
+ {
+ participants.add(endpoint.getHostAddress(false));
+ participants_v2.add(endpoint.toString());
+ }
String query =
+ "INSERT INTO %s.%s (keyspace_name, columnfamily_name, id, parent_id, range_begin, range_end, coordinator, coordinator_port, participants, participants_v2, status, started_at) " +
+ "VALUES ( '%s', '%s', %s, %s, '%s', '%s', '%s', %d, { '%s' }, { '%s' }, '%s', toTimestamp(now()))";
+ String queryWithoutNewColumns =
"INSERT INTO %s.%s (keyspace_name, columnfamily_name, id, parent_id, range_begin, range_end, coordinator, participants, status, started_at) " +
- "VALUES ( '%s', '%s', %s, %s, '%s', '%s', '%s', { '%s' }, '%s', toTimestamp(now()))";
+ "VALUES ( '%s', '%s', %s, %s, '%s', '%s', '%s', { '%s' }, '%s', toTimestamp(now()))";
for (String cfname : cfnames)
{
- for (Range<Token> range : ranges)
+ for (Range<Token> range : commonRange.ranges)
{
- String fmtQry = String.format(query, SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, REPAIR_HISTORY,
- keyspaceName,
- cfname,
- id.toString(),
- parent_id.toString(),
- range.left.toString(),
- range.right.toString(),
- coordinator,
- Joiner.on("', '").join(participants),
- RepairState.STARTED.toString());
+ String fmtQry;
+ if (includeNewColumns)
+ {
+ fmtQry = format(query, SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, REPAIR_HISTORY,
+ keyspaceName,
+ cfname,
+ id.toString(),
+ parent_id.toString(),
+ range.left.toString(),
+ range.right.toString(),
+ coordinator.getHostAddress(false),
+ coordinator.port,
+ Joiner.on("', '").join(participants),
+ Joiner.on("', '").join(participants_v2),
+ RepairState.STARTED.toString());
+ }
+ else
+ {
+ fmtQry = format(queryWithoutNewColumns, SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, REPAIR_HISTORY,
+ keyspaceName,
+ cfname,
+ id.toString(),
+ parent_id.toString(),
+ range.left.toString(),
+ range.right.toString(),
+ coordinator.getHostAddress(false),
+ Joiner.on("', '").join(participants),
+ RepairState.STARTED.toString());
+ }
processSilent(fmtQry);
}
}
@@ -232,7 +280,7 @@
public static void successfulRepairJob(UUID id, String keyspaceName, String cfname)
{
String query = "UPDATE %s.%s SET status = '%s', finished_at = toTimestamp(now()) WHERE keyspace_name = '%s' AND columnfamily_name = '%s' AND id = %s";
- String fmtQuery = String.format(query, SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, REPAIR_HISTORY,
+ String fmtQuery = format(query, SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, REPAIR_HISTORY,
RepairState.SUCCESS.toString(),
keyspaceName,
cfname,
@@ -246,7 +294,7 @@
StringWriter sw = new StringWriter();
PrintWriter pw = new PrintWriter(sw);
t.printStackTrace(pw);
- String fmtQry = String.format(query, SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, REPAIR_HISTORY,
+ String fmtQry = format(query, SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, REPAIR_HISTORY,
RepairState.FAILED.toString(),
keyspaceName,
cfname,
@@ -257,7 +305,7 @@
public static void startViewBuild(String keyspace, String view, UUID hostId)
{
String query = "INSERT INTO %s.%s (keyspace_name, view_name, host_id, status) VALUES (?, ?, ?, ?)";
- QueryProcessor.process(String.format(query, SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, VIEW_BUILD_STATUS),
+ QueryProcessor.process(format(query, SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, VIEW_BUILD_STATUS),
ConsistencyLevel.ONE,
Lists.newArrayList(bytes(keyspace),
bytes(view),
@@ -268,7 +316,7 @@
public static void successfulViewBuild(String keyspace, String view, UUID hostId)
{
String query = "UPDATE %s.%s SET status = ? WHERE keyspace_name = ? AND view_name = ? AND host_id = ?";
- QueryProcessor.process(String.format(query, SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, VIEW_BUILD_STATUS),
+ QueryProcessor.process(format(query, SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, VIEW_BUILD_STATUS),
ConsistencyLevel.ONE,
Lists.newArrayList(bytes(BuildStatus.SUCCESS.toString()),
bytes(keyspace),
@@ -282,7 +330,7 @@
UntypedResultSet results;
try
{
- results = QueryProcessor.execute(String.format(query, SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, VIEW_BUILD_STATUS),
+ results = QueryProcessor.execute(format(query, SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, VIEW_BUILD_STATUS),
ConsistencyLevel.ONE,
keyspace,
view);
@@ -304,7 +352,7 @@
public static void setViewRemoved(String keyspaceName, String viewName)
{
String buildReq = "DELETE FROM %s.%s WHERE keyspace_name = ? AND view_name = ?";
- QueryProcessor.executeInternal(String.format(buildReq, SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, VIEW_BUILD_STATUS), keyspaceName, viewName);
+ QueryProcessor.executeInternal(format(buildReq, SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, VIEW_BUILD_STATUS), keyspaceName, viewName);
forceBlockingFlush(VIEW_BUILD_STATUS);
}
@@ -312,7 +360,7 @@
{
try
{
- List<ByteBuffer> valueList = new ArrayList<>();
+ List<ByteBuffer> valueList = new ArrayList<>(values.length);
for (String v : values)
{
valueList.add(bytes(v));
diff --git a/src/java/org/apache/cassandra/repair/TableRepairManager.java b/src/java/org/apache/cassandra/repair/TableRepairManager.java
new file mode 100644
index 0000000..b72af1d
--- /dev/null
+++ b/src/java/org/apache/cassandra/repair/TableRepairManager.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.repair;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.UUID;
+import java.util.concurrent.Callable;
+import java.util.concurrent.Future;
+
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+
+/**
+ * Table level hook for repair
+ */
+public interface TableRepairManager
+{
+ /**
+ * Return a validation iterator for the given parameters. If isIncremental is true, the iterator must only include
+ * data previously isolated for repair with the given parentId. nowInSec should determine whether tombstones shouldn
+ * be purged or not.
+ */
+ ValidationPartitionIterator getValidationIterator(Collection<Range<Token>> ranges, UUID parentId, UUID sessionID, boolean isIncremental, int nowInSec) throws IOException;
+
+ /**
+ * Begin execution of the given validation callable. Which thread pool a validation should run in is an implementation detail.
+ */
+ Future<?> submitValidation(Callable<Object> validation);
+
+ /**
+ * Called when the given incremental session has completed. Because of race and failure conditions, implementors
+ * should not rely only on receiving calls from this method to determine when a session has ended. Implementors
+ * can determine if a session has finished by calling ActiveRepairService.instance.consistent.local.isSessionInProgress.
+ *
+ * Just because a session has completed doesn't mean it's completed succesfully. So implementors need to consult the
+ * repair service at ActiveRepairService.instance.consistent.local.getFinalSessionRepairedAt to get the repairedAt
+ * time. If the repairedAt time is zero, the data for the given session should be demoted back to unrepaired. Otherwise,
+ * it should be promoted to repaired with the given repaired time.
+ */
+ void incrementalSessionCompleted(UUID sessionID);
+
+ /**
+ * For snapshot repairs. A snapshot of the current data for the given ranges should be taken with the given name.
+ * If force is true, a snapshot should be taken even if one already exists with that name.
+ */
+ void snapshot(String name, Collection<Range<Token>> ranges, boolean force);
+}
diff --git a/src/java/org/apache/cassandra/repair/TreeResponse.java b/src/java/org/apache/cassandra/repair/TreeResponse.java
index c898b36..8571caa 100644
--- a/src/java/org/apache/cassandra/repair/TreeResponse.java
+++ b/src/java/org/apache/cassandra/repair/TreeResponse.java
@@ -17,8 +17,7 @@
*/
package org.apache.cassandra.repair;
-import java.net.InetAddress;
-
+import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.utils.MerkleTrees;
/**
@@ -26,10 +25,10 @@
*/
public class TreeResponse
{
- public final InetAddress endpoint;
+ public final InetAddressAndPort endpoint;
public final MerkleTrees trees;
- public TreeResponse(InetAddress endpoint, MerkleTrees trees)
+ public TreeResponse(InetAddressAndPort endpoint, MerkleTrees trees)
{
this.endpoint = endpoint;
this.trees = trees;
diff --git a/src/java/org/apache/cassandra/repair/ValidationManager.java b/src/java/org/apache/cassandra/repair/ValidationManager.java
new file mode 100644
index 0000000..4bbffbf
--- /dev/null
+++ b/src/java/org/apache/cassandra/repair/ValidationManager.java
@@ -0,0 +1,177 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.repair;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Map;
+import java.util.concurrent.Callable;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.metrics.TableMetrics;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.MerkleTree;
+import org.apache.cassandra.utils.MerkleTrees;
+
+public class ValidationManager
+{
+ private static final Logger logger = LoggerFactory.getLogger(ValidationManager.class);
+
+ public static final ValidationManager instance = new ValidationManager();
+
+ private ValidationManager() {}
+
+ private static MerkleTrees createMerkleTrees(ValidationPartitionIterator validationIterator, Collection<Range<Token>> ranges, ColumnFamilyStore cfs)
+ {
+ MerkleTrees tree = new MerkleTrees(cfs.getPartitioner());
+ long allPartitions = validationIterator.estimatedPartitions();
+ Map<Range<Token>, Long> rangePartitionCounts = validationIterator.getRangePartitionCounts();
+
+ // The repair coordinator must hold RF trees in memory at once, so a given validation compaction can only
+ // use 1 / RF of the allowed space.
+ long availableBytes = (DatabaseDescriptor.getRepairSessionSpaceInMegabytes() * 1048576) /
+ cfs.keyspace.getReplicationStrategy().getReplicationFactor().allReplicas;
+
+ for (Range<Token> range : ranges)
+ {
+ long numPartitions = rangePartitionCounts.get(range);
+ double rangeOwningRatio = allPartitions > 0 ? (double)numPartitions / allPartitions : 0;
+ // determine max tree depth proportional to range size to avoid blowing up memory with multiple tress,
+ // capping at a depth that does not exceed our memory budget (CASSANDRA-11390, CASSANDRA-14096)
+ int rangeAvailableBytes = Math.max(1, (int) (rangeOwningRatio * availableBytes));
+ // Try to estimate max tree depth that fits the space budget assuming hashes of 256 bits = 32 bytes
+ // note that estimatedMaxDepthForBytes cannot return a number lower than 1
+ int estimatedMaxDepth = MerkleTree.estimatedMaxDepthForBytes(cfs.getPartitioner(), rangeAvailableBytes, 32);
+ int maxDepth = rangeOwningRatio > 0
+ ? Math.min(estimatedMaxDepth, DatabaseDescriptor.getRepairSessionMaxTreeDepth())
+ : 0;
+ // determine tree depth from number of partitions, capping at max tree depth (CASSANDRA-5263)
+ int depth = numPartitions > 0 ? (int) Math.min(Math.ceil(Math.log(numPartitions) / Math.log(2)), maxDepth) : 0;
+ tree.addMerkleTree((int) Math.pow(2, depth), range);
+ }
+ if (logger.isDebugEnabled())
+ {
+ // MT serialize may take time
+ logger.debug("Created {} merkle trees with merkle trees size {}, {} partitions, {} bytes", tree.ranges().size(), tree.size(), allPartitions, MerkleTrees.serializer.serializedSize(tree, 0));
+ }
+
+ return tree;
+ }
+
+ private static ValidationPartitionIterator getValidationIterator(TableRepairManager repairManager, Validator validator) throws IOException
+ {
+ RepairJobDesc desc = validator.desc;
+ return repairManager.getValidationIterator(desc.ranges, desc.parentSessionId, desc.sessionId, validator.isIncremental, validator.nowInSec);
+ }
+
+ /**
+ * Performs a readonly "compaction" of all sstables in order to validate complete rows,
+ * but without writing the merge result
+ */
+ @SuppressWarnings("resource")
+ private void doValidation(ColumnFamilyStore cfs, Validator validator) throws IOException
+ {
+ // this isn't meant to be race-proof, because it's not -- it won't cause bugs for a CFS to be dropped
+ // mid-validation, or to attempt to validate a droped CFS. this is just a best effort to avoid useless work,
+ // particularly in the scenario where a validation is submitted before the drop, and there are compactions
+ // started prior to the drop keeping some sstables alive. Since validationCompaction can run
+ // concurrently with other compactions, it would otherwise go ahead and scan those again.
+ if (!cfs.isValid())
+ return;
+
+ // Create Merkle trees suitable to hold estimated partitions for the given ranges.
+ // We blindly assume that a partition is evenly distributed on all sstables for now.
+ long start = System.nanoTime();
+ long partitionCount = 0;
+ long estimatedTotalBytes = 0;
+ try (ValidationPartitionIterator vi = getValidationIterator(cfs.getRepairManager(), validator))
+ {
+ MerkleTrees tree = createMerkleTrees(vi, validator.desc.ranges, cfs);
+ try
+ {
+ // validate the CF as we iterate over it
+ validator.prepare(cfs, tree);
+ while (vi.hasNext())
+ {
+ try (UnfilteredRowIterator partition = vi.next())
+ {
+ validator.add(partition);
+ partitionCount++;
+ }
+ }
+ validator.complete();
+ }
+ finally
+ {
+ estimatedTotalBytes = vi.getEstimatedBytes();
+ partitionCount = vi.estimatedPartitions();
+ }
+ }
+ finally
+ {
+ cfs.metric.bytesValidated.update(estimatedTotalBytes);
+ cfs.metric.partitionsValidated.update(partitionCount);
+ }
+ if (logger.isDebugEnabled())
+ {
+ long duration = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start);
+ logger.debug("Validation of {} partitions (~{}) finished in {} msec, for {}",
+ partitionCount,
+ FBUtilities.prettyPrintMemory(estimatedTotalBytes),
+ duration,
+ validator.desc);
+ }
+ }
+
+ /**
+ * Does not mutate data, so is not scheduled.
+ */
+ public Future<?> submitValidation(ColumnFamilyStore cfs, Validator validator)
+ {
+ Callable<Object> validation = new Callable<Object>()
+ {
+ public Object call() throws IOException
+ {
+ try (TableMetrics.TableTimer.Context c = cfs.metric.validationTime.time())
+ {
+ doValidation(cfs, validator);
+ }
+ catch (Throwable e)
+ {
+ // we need to inform the remote end of our failure, otherwise it will hang on repair forever
+ validator.fail();
+ logger.error("Validation failed.", e);
+ throw e;
+ }
+ return this;
+ }
+ };
+
+ return cfs.getRepairManager().submitValidation(validation);
+ }
+}
diff --git a/src/java/org/apache/cassandra/repair/ValidationPartitionIterator.java b/src/java/org/apache/cassandra/repair/ValidationPartitionIterator.java
new file mode 100644
index 0000000..ccfae41
--- /dev/null
+++ b/src/java/org/apache/cassandra/repair/ValidationPartitionIterator.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.repair;
+
+import java.io.IOException;
+import java.util.Map;
+
+import org.apache.cassandra.db.partitions.AbstractUnfilteredPartitionIterator;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+
+public abstract class ValidationPartitionIterator extends AbstractUnfilteredPartitionIterator
+{
+ public abstract long getEstimatedBytes();
+ public abstract long estimatedPartitions();
+ public abstract Map<Range<Token>, Long> getRangePartitionCounts();
+}
diff --git a/src/java/org/apache/cassandra/repair/ValidationTask.java b/src/java/org/apache/cassandra/repair/ValidationTask.java
index 177ad3e..0161acf 100644
--- a/src/java/org/apache/cassandra/repair/ValidationTask.java
+++ b/src/java/org/apache/cassandra/repair/ValidationTask.java
@@ -17,15 +17,18 @@
*/
package org.apache.cassandra.repair;
-import java.net.InetAddress;
-
import com.google.common.util.concurrent.AbstractFuture;
import org.apache.cassandra.exceptions.RepairException;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.net.Message;
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.repair.messages.ValidationRequest;
+import org.apache.cassandra.streaming.PreviewKind;
import org.apache.cassandra.utils.MerkleTrees;
+import static org.apache.cassandra.net.Verb.VALIDATION_REQ;
+
/**
* ValidationTask sends {@link ValidationRequest} to a replica.
* When a replica sends back message, task completes.
@@ -33,14 +36,16 @@
public class ValidationTask extends AbstractFuture<TreeResponse> implements Runnable
{
private final RepairJobDesc desc;
- private final InetAddress endpoint;
- private final int gcBefore;
+ private final InetAddressAndPort endpoint;
+ private final int nowInSec;
+ private final PreviewKind previewKind;
- public ValidationTask(RepairJobDesc desc, InetAddress endpoint, int gcBefore)
+ public ValidationTask(RepairJobDesc desc, InetAddressAndPort endpoint, int nowInSec, PreviewKind previewKind)
{
this.desc = desc;
this.endpoint = endpoint;
- this.gcBefore = gcBefore;
+ this.nowInSec = nowInSec;
+ this.previewKind = previewKind;
}
/**
@@ -48,8 +53,8 @@
*/
public void run()
{
- ValidationRequest request = new ValidationRequest(desc, gcBefore);
- MessagingService.instance().sendOneWay(request.createMessage(), endpoint);
+ ValidationRequest request = new ValidationRequest(desc, nowInSec);
+ MessagingService.instance().send(Message.out(VALIDATION_REQ, request), endpoint);
}
/**
@@ -61,7 +66,7 @@
{
if (trees == null)
{
- setException(new RepairException(desc, "Validation failed in " + endpoint));
+ setException(new RepairException(desc, previewKind, "Validation failed in " + endpoint));
}
else
{
diff --git a/src/java/org/apache/cassandra/repair/Validator.java b/src/java/org/apache/cassandra/repair/Validator.java
index a2a2512..2f71729 100644
--- a/src/java/org/apache/cassandra/repair/Validator.java
+++ b/src/java/org/apache/cassandra/repair/Validator.java
@@ -17,32 +17,44 @@
*/
package org.apache.cassandra.repair;
-import java.net.InetAddress;
-import java.security.MessageDigest;
+import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Random;
import com.google.common.annotations.VisibleForTesting;
-
+import com.google.common.hash.Funnel;
+import com.google.common.hash.HashCode;
+import com.google.common.hash.HashFunction;
+import com.google.common.hash.Hasher;
+import com.google.common.hash.Hashing;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+
import org.apache.cassandra.concurrent.Stage;
-import org.apache.cassandra.concurrent.StageManager;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.Digest;
import org.apache.cassandra.db.rows.UnfilteredRowIterator;
import org.apache.cassandra.db.rows.UnfilteredRowIterators;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.net.Message;
import org.apache.cassandra.net.MessagingService;
-import org.apache.cassandra.repair.messages.ValidationComplete;
+import org.apache.cassandra.repair.messages.ValidationResponse;
+import org.apache.cassandra.streaming.PreviewKind;
+import org.apache.cassandra.service.ActiveRepairService;
import org.apache.cassandra.tracing.Tracing;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.MerkleTree;
import org.apache.cassandra.utils.MerkleTree.RowHash;
import org.apache.cassandra.utils.MerkleTrees;
+import static org.apache.cassandra.net.Verb.VALIDATION_RSP;
+
/**
* Handles the building of a merkle tree for a column family.
*
@@ -56,9 +68,10 @@
private static final Logger logger = LoggerFactory.getLogger(Validator.class);
public final RepairJobDesc desc;
- public final InetAddress initiator;
- public final int gcBefore;
+ public final InetAddressAndPort initiator;
+ public final int nowInSec;
private final boolean evenTreeDistribution;
+ public final boolean isIncremental;
// null when all rows with the min token have been consumed
private long validated;
@@ -70,16 +83,25 @@
// last key seen
private DecoratedKey lastKey;
- public Validator(RepairJobDesc desc, InetAddress initiator, int gcBefore)
+ private final PreviewKind previewKind;
+
+ public Validator(RepairJobDesc desc, InetAddressAndPort initiator, int nowInSec, PreviewKind previewKind)
{
- this(desc, initiator, gcBefore, false);
+ this(desc, initiator, nowInSec, false, false, previewKind);
}
- public Validator(RepairJobDesc desc, InetAddress initiator, int gcBefore, boolean evenTreeDistribution)
+ public Validator(RepairJobDesc desc, InetAddressAndPort initiator, int nowInSec, boolean isIncremental, PreviewKind previewKind)
+ {
+ this(desc, initiator, nowInSec, false, isIncremental, previewKind);
+ }
+
+ public Validator(RepairJobDesc desc, InetAddressAndPort initiator, int nowInSec, boolean evenTreeDistribution, boolean isIncremental, PreviewKind previewKind)
{
this.desc = desc;
this.initiator = initiator;
- this.gcBefore = gcBefore;
+ this.nowInSec = nowInSec;
+ this.isIncremental = isIncremental;
+ this.previewKind = previewKind;
validated = 0;
range = null;
ranges = null;
@@ -128,7 +150,7 @@
}
}
logger.debug("Prepared AEService trees of size {} for {}", trees.size(), desc);
- ranges = tree.invalids();
+ ranges = tree.rangeIterator();
}
/**
@@ -151,7 +173,7 @@
if (!findCorrectRange(lastKey.getToken()))
{
// add the empty hash, and move to the next range
- ranges = trees.invalids();
+ ranges = trees.rangeIterator();
findCorrectRange(lastKey.getToken());
}
@@ -174,54 +196,15 @@
return range.contains(t);
}
- static class CountingDigest extends MessageDigest
- {
- private long count;
- private MessageDigest underlying;
-
- public CountingDigest(MessageDigest underlying)
- {
- super(underlying.getAlgorithm());
- this.underlying = underlying;
- }
-
- @Override
- protected void engineUpdate(byte input)
- {
- underlying.update(input);
- count += 1;
- }
-
- @Override
- protected void engineUpdate(byte[] input, int offset, int len)
- {
- underlying.update(input, offset, len);
- count += len;
- }
-
- @Override
- protected byte[] engineDigest()
- {
- return underlying.digest();
- }
-
- @Override
- protected void engineReset()
- {
- underlying.reset();
- }
-
- }
-
private MerkleTree.RowHash rowHash(UnfilteredRowIterator partition)
{
validated++;
// MerkleTree uses XOR internally, so we want lots of output bits here
- CountingDigest digest = new CountingDigest(FBUtilities.newMessageDigest("SHA-256"));
- UnfilteredRowIterators.digest(null, partition, digest, MessagingService.current_version);
+ Digest digest = Digest.forValidator();
+ UnfilteredRowIterators.digest(partition, digest, MessagingService.current_version);
// only return new hash for merkle tree in case digest was updated - see CASSANDRA-8979
- return digest.count > 0
- ? new MerkleTree.RowHash(partition.partitionKey().getToken(), digest.digest(), digest.count)
+ return digest.inputBytes() > 0
+ ? new MerkleTree.RowHash(partition.partitionKey().getToken(), digest.digest(), digest.inputBytes())
: null;
}
@@ -230,9 +213,7 @@
*/
public void complete()
{
- completeTree();
-
- StageManager.getStage(Stage.ANTI_ENTROPY).execute(this);
+ assert ranges != null : "Validator was not prepared()";
if (logger.isDebugEnabled())
{
@@ -242,20 +223,8 @@
logger.debug("Validated {} partitions for {}. Partition sizes are:", validated, desc.sessionId);
trees.logRowSizePerLeaf(logger);
}
- }
- @VisibleForTesting
- public void completeTree()
- {
- assert ranges != null : "Validator was not prepared()";
-
- ranges = trees.invalids();
-
- while (ranges.hasNext())
- {
- range = ranges.next();
- range.ensureHashInitialised();
- }
+ Stage.ANTI_ENTROPY.execute(this);
}
/**
@@ -266,8 +235,7 @@
public void fail()
{
logger.error("Failed creating a merkle tree for {}, {} (see log for details)", desc, initiator);
- // send fail message only to nodes >= version 2.0
- MessagingService.instance().sendOneWay(new ValidationComplete(desc).createMessage(), initiator);
+ respond(new ValidationResponse(desc));
}
/**
@@ -275,12 +243,51 @@
*/
public void run()
{
- // respond to the request that triggered this validation
- if (!initiator.equals(FBUtilities.getBroadcastAddress()))
+ if (initiatorIsRemote())
{
- logger.info("[repair #{}] Sending completed merkle tree to {} for {}.{}", desc.sessionId, initiator, desc.keyspace, desc.columnFamily);
+ logger.info("{} Sending completed merkle tree to {} for {}.{}", previewKind.logPrefix(desc.sessionId), initiator, desc.keyspace, desc.columnFamily);
Tracing.traceRepair("Sending completed merkle tree to {} for {}.{}", initiator, desc.keyspace, desc.columnFamily);
}
- MessagingService.instance().sendOneWay(new ValidationComplete(desc, trees).createMessage(), initiator);
+ else
+ {
+ logger.info("{} Local completed merkle tree for {} for {}.{}", previewKind.logPrefix(desc.sessionId), initiator, desc.keyspace, desc.columnFamily);
+ Tracing.traceRepair("Local completed merkle tree for {} for {}.{}", initiator, desc.keyspace, desc.columnFamily);
+
+ }
+ respond(new ValidationResponse(desc, trees));
+ }
+
+ private boolean initiatorIsRemote()
+ {
+ return !FBUtilities.getBroadcastAddressAndPort().equals(initiator);
+ }
+
+ private void respond(ValidationResponse response)
+ {
+ if (initiatorIsRemote())
+ {
+ MessagingService.instance().send(Message.out(VALIDATION_RSP, response), initiator);
+ return;
+ }
+
+ /*
+ * For local initiators, DO NOT send the message to self over loopback. This is a wasted ser/de loop
+ * and a ton of garbage. Instead, move the trees off heap and invoke message handler. We could do it
+ * directly, since this method will only be called from {@code Stage.ENTI_ENTROPY}, but we do instead
+ * execute a {@code Runnable} on the stage - in case that assumption ever changes by accident.
+ */
+ Stage.ANTI_ENTROPY.execute(() ->
+ {
+ ValidationResponse movedResponse = response;
+ try
+ {
+ movedResponse = response.tryMoveOffHeap();
+ }
+ catch (IOException e)
+ {
+ logger.error("Failed to move local merkle tree for {} off heap", desc, e);
+ }
+ ActiveRepairService.instance.handleMessage(Message.out(VALIDATION_RSP, movedResponse));
+ });
}
}
diff --git a/src/java/org/apache/cassandra/repair/asymmetric/DifferenceHolder.java b/src/java/org/apache/cassandra/repair/asymmetric/DifferenceHolder.java
new file mode 100644
index 0000000..f85c2eb
--- /dev/null
+++ b/src/java/org/apache/cassandra/repair/asymmetric/DifferenceHolder.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.repair.asymmetric;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableMap;
+
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.repair.TreeResponse;
+import org.apache.cassandra.utils.MerkleTrees;
+
+/**
+ * Just holds all differences between the hosts involved in a repair
+ */
+public class DifferenceHolder
+{
+ private final ImmutableMap<InetAddressAndPort, HostDifferences> differences;
+
+ public DifferenceHolder(List<TreeResponse> trees)
+ {
+ ImmutableMap.Builder<InetAddressAndPort, HostDifferences> diffBuilder = ImmutableMap.builder();
+ for (int i = 0; i < trees.size() - 1; ++i)
+ {
+ TreeResponse r1 = trees.get(i);
+ // create the differences between r1 and all other hosts:
+ HostDifferences hd = new HostDifferences();
+ for (int j = i + 1; j < trees.size(); ++j)
+ {
+ TreeResponse r2 = trees.get(j);
+ hd.add(r2.endpoint, MerkleTrees.difference(r1.trees, r2.trees));
+ }
+ r1.trees.release();
+ // and add them to the diff map
+ diffBuilder.put(r1.endpoint, hd);
+ }
+ trees.get(trees.size() - 1).trees.release();
+ differences = diffBuilder.build();
+ }
+
+ @VisibleForTesting
+ DifferenceHolder(Map<InetAddressAndPort, HostDifferences> differences)
+ {
+ ImmutableMap.Builder<InetAddressAndPort, HostDifferences> diffBuilder = ImmutableMap.builder();
+ diffBuilder.putAll(differences);
+ this.differences = diffBuilder.build();
+ }
+
+ /**
+ * differences only holds one 'side' of the difference - if A and B mismatch, only A will be a key in the map
+ */
+ public Set<InetAddressAndPort> keyHosts()
+ {
+ return differences.keySet();
+ }
+
+ public HostDifferences get(InetAddressAndPort hostWithDifference)
+ {
+ return differences.get(hostWithDifference);
+ }
+
+ public String toString()
+ {
+ return "DifferenceHolder{" +
+ "differences=" + differences +
+ '}';
+ }
+
+ public boolean hasDifferenceBetween(InetAddressAndPort node1, InetAddressAndPort node2, Range<Token> range)
+ {
+ HostDifferences diffsNode1 = differences.get(node1);
+ if (diffsNode1 != null && diffsNode1.hasDifferencesFor(node2, range))
+ return true;
+ HostDifferences diffsNode2 = differences.get(node2);
+ if (diffsNode2 != null && diffsNode2.hasDifferencesFor(node1, range))
+ return true;
+ return false;
+ }
+}
diff --git a/src/java/org/apache/cassandra/repair/asymmetric/HostDifferences.java b/src/java/org/apache/cassandra/repair/asymmetric/HostDifferences.java
new file mode 100644
index 0000000..ab294b9
--- /dev/null
+++ b/src/java/org/apache/cassandra/repair/asymmetric/HostDifferences.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.repair.asymmetric;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.locator.InetAddressAndPort;
+
+/**
+ * Tracks the differences for a single host
+ */
+public class HostDifferences
+{
+ private final Map<InetAddressAndPort, List<Range<Token>>> perHostDifferences = new HashMap<>();
+
+ /**
+ * Adds a set of differences between the node this instance is tracking and endpoint
+ */
+ public void add(InetAddressAndPort endpoint, List<Range<Token>> difference)
+ {
+ perHostDifferences.put(endpoint, difference);
+ }
+
+ public void addSingleRange(InetAddressAndPort remoteNode, Range<Token> rangeToFetch)
+ {
+ perHostDifferences.computeIfAbsent(remoteNode, (x) -> new ArrayList<>()).add(rangeToFetch);
+ }
+
+ /**
+ * Does this instance have differences for range with node2?
+ */
+ public boolean hasDifferencesFor(InetAddressAndPort node2, Range<Token> range)
+ {
+ List<Range<Token>> differences = get(node2);
+ for (Range<Token> diff : differences)
+ {
+ // if the other node has a diff for this range, we know they are not equal.
+ if (range.equals(diff) || range.intersects(diff))
+ return true;
+ }
+ return false;
+ }
+
+ public Set<InetAddressAndPort> hosts()
+ {
+ return perHostDifferences.keySet();
+ }
+
+ public List<Range<Token>> get(InetAddressAndPort differingHost)
+ {
+ return perHostDifferences.getOrDefault(differingHost, Collections.emptyList());
+ }
+
+ public String toString()
+ {
+ return "HostDifferences{" +
+ "perHostDifferences=" + perHostDifferences +
+ '}';
+ }
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/repair/asymmetric/IncomingRepairStreamTracker.java b/src/java/org/apache/cassandra/repair/asymmetric/IncomingRepairStreamTracker.java
new file mode 100644
index 0000000..450336f
--- /dev/null
+++ b/src/java/org/apache/cassandra/repair/asymmetric/IncomingRepairStreamTracker.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.repair.asymmetric;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+
+import com.google.common.collect.ImmutableMap;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.locator.InetAddressAndPort;
+
+/**
+ * Tracks incoming streams for a single host
+ */
+public class IncomingRepairStreamTracker
+{
+ private static final Logger logger = LoggerFactory.getLogger(IncomingRepairStreamTracker.class);
+ private final DifferenceHolder differences;
+ private final Map<Range<Token>, StreamFromOptions> incoming = new HashMap<>();
+
+ public IncomingRepairStreamTracker(DifferenceHolder differences)
+ {
+ this.differences = differences;
+ }
+
+ public String toString()
+ {
+ return "IncomingStreamTracker{" +
+ "incoming=" + incoming +
+ '}';
+ }
+
+ /**
+ * Adds a range to be streamed from streamFromNode
+ *
+ * First the currently tracked ranges are denormalized to make sure that no ranges overlap, then
+ * the streamFromNode is added to the StreamFromOptions for the range
+ *
+ * @param range the range we need to stream from streamFromNode
+ * @param streamFromNode the node we should stream from
+ */
+ public void addIncomingRangeFrom(Range<Token> range, InetAddressAndPort streamFromNode)
+ {
+ logger.trace("adding incoming range {} from {}", range, streamFromNode);
+ Set<Range<Token>> newInput = RangeDenormalizer.denormalize(range, incoming);
+ for (Range<Token> input : newInput)
+ {
+ incoming.computeIfAbsent(input, (newRange) -> new StreamFromOptions(differences, newRange)).add(streamFromNode);
+ }
+ }
+
+ public ImmutableMap<Range<Token>, StreamFromOptions> getIncoming()
+ {
+ return ImmutableMap.copyOf(incoming);
+ }
+}
+
+
+
+
diff --git a/src/java/org/apache/cassandra/repair/asymmetric/PreferedNodeFilter.java b/src/java/org/apache/cassandra/repair/asymmetric/PreferedNodeFilter.java
new file mode 100644
index 0000000..e8ca85d
--- /dev/null
+++ b/src/java/org/apache/cassandra/repair/asymmetric/PreferedNodeFilter.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.repair.asymmetric;
+
+import java.util.Set;
+
+import org.apache.cassandra.locator.InetAddressAndPort;
+
+public interface PreferedNodeFilter
+{
+ public Set<InetAddressAndPort> apply(InetAddressAndPort streamingNode, Set<InetAddressAndPort> toStream);
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/repair/asymmetric/RangeDenormalizer.java b/src/java/org/apache/cassandra/repair/asymmetric/RangeDenormalizer.java
new file mode 100644
index 0000000..f692dd6
--- /dev/null
+++ b/src/java/org/apache/cassandra/repair/asymmetric/RangeDenormalizer.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.repair.asymmetric;
+
+import java.net.InetAddress;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Sets;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+
+public class RangeDenormalizer
+{
+ private static final Logger logger = LoggerFactory.getLogger(RangeDenormalizer.class);
+
+ /**
+ * "Denormalizes" (kind of the opposite of what Range.normalize does) the ranges in the keys of {{incoming}}
+ *
+ * It makes sure that if there is an intersection between {{range}} and some of the ranges in {{incoming.keySet()}}
+ * we know that all intersections are keys in the updated {{incoming}}
+ */
+ public static Set<Range<Token>> denormalize(Range<Token> range, Map<Range<Token>, StreamFromOptions> incoming)
+ {
+ logger.trace("Denormalizing range={} incoming={}", range, incoming);
+ Set<Range<Token>> existingRanges = new HashSet<>(incoming.keySet());
+ Map<Range<Token>, StreamFromOptions> existingOverlappingRanges = new HashMap<>();
+ // remove all overlapping ranges from the incoming map
+ for (Range<Token> existingRange : existingRanges)
+ {
+ if (range.intersects(existingRange))
+ existingOverlappingRanges.put(existingRange, incoming.remove(existingRange));
+ }
+
+ Set<Range<Token>> intersections = intersection(existingRanges, range);
+ Set<Range<Token>> newExisting = Sets.union(subtractFromAllRanges(existingOverlappingRanges.keySet(), range), intersections);
+ Set<Range<Token>> newInput = Sets.union(range.subtractAll(existingOverlappingRanges.keySet()), intersections);
+ assertNonOverLapping(newExisting);
+ assertNonOverLapping(newInput);
+ for (Range<Token> r : newExisting)
+ {
+ for (Map.Entry<Range<Token>, StreamFromOptions> entry : existingOverlappingRanges.entrySet())
+ {
+ if (r.intersects(entry.getKey()))
+ incoming.put(r, entry.getValue().copy(r));
+ }
+ }
+ logger.trace("denormalized {} to {}", range, newInput);
+ logger.trace("denormalized incoming to {}", incoming);
+ assertNonOverLapping(incoming.keySet());
+ return newInput;
+ }
+
+ /**
+ * Subtract the given range from all the input ranges.
+ *
+ * for example:
+ * ranges = [(0, 10], (20, 30]]
+ * and range = (8, 22]
+ *
+ * the result should be [(0, 8], (22, 30]]
+ *
+ */
+ @VisibleForTesting
+ static Set<Range<Token>> subtractFromAllRanges(Collection<Range<Token>> ranges, Range<Token> range)
+ {
+ Set<Range<Token>> result = new HashSet<>();
+ for (Range<Token> r : ranges)
+ result.addAll(r.subtract(range)); // subtract can return two ranges if we remove the middle part
+ return result;
+ }
+
+ /**
+ * Makes sure non of the input ranges are overlapping
+ */
+ private static void assertNonOverLapping(Set<Range<Token>> ranges)
+ {
+ List<Range<Token>> sortedRanges = Range.sort(ranges);
+ Token lastToken = null;
+ for (Range<Token> range : sortedRanges)
+ {
+ if (lastToken != null && lastToken.compareTo(range.left) > 0)
+ {
+ throw new AssertionError("Ranges are overlapping: "+ranges);
+ }
+ lastToken = range.right;
+ }
+ }
+
+ /**
+ * Returns all intersections between the ranges in ranges and the given range
+ */
+ private static Set<Range<Token>> intersection(Collection<Range<Token>> ranges, Range<Token> range)
+ {
+ Set<Range<Token>> result = new HashSet<>();
+ for (Range<Token> r : ranges)
+ result.addAll(range.intersectionWith(r));
+ return result;
+ }
+}
diff --git a/src/java/org/apache/cassandra/repair/asymmetric/ReduceHelper.java b/src/java/org/apache/cassandra/repair/asymmetric/ReduceHelper.java
new file mode 100644
index 0000000..c7d45bf
--- /dev/null
+++ b/src/java/org/apache/cassandra/repair/asymmetric/ReduceHelper.java
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.repair.asymmetric;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableMap;
+
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.locator.InetAddressAndPort;
+
+/**
+ * Basic idea is that we track incoming ranges instead of blindly just exchanging the ranges that mismatch between two nodes
+ *
+ * Say node X has tracked that it will stream range r1 from node Y. Now we see find a diffing range
+ * r1 between node X and Z. When adding r1 from Z as an incoming to X we check if Y and Z are equal on range r (ie, there is
+ * no difference between them). If they are equal X can stream from Y or Z and the end result will be the same.
+ *
+ * The ranges wont match perfectly since we don't iterate over leaves so we always split based on the
+ * smallest range (either the new difference or the existing one)
+ */
+public class ReduceHelper
+{
+ /**
+ * Reduces the differences provided by the merkle trees to a minimum set of differences
+ */
+ public static ImmutableMap<InetAddressAndPort, HostDifferences> reduce(DifferenceHolder differences, PreferedNodeFilter filter)
+ {
+ Map<InetAddressAndPort, IncomingRepairStreamTracker> trackers = createIncomingRepairStreamTrackers(differences);
+ Map<InetAddressAndPort, Integer> outgoingStreamCounts = new HashMap<>();
+ ImmutableMap.Builder<InetAddressAndPort, HostDifferences> mapBuilder = ImmutableMap.builder();
+ for (Map.Entry<InetAddressAndPort, IncomingRepairStreamTracker> trackerEntry : trackers.entrySet())
+ {
+ IncomingRepairStreamTracker tracker = trackerEntry.getValue();
+ HostDifferences rangesToFetch = new HostDifferences();
+ for (Map.Entry<Range<Token>, StreamFromOptions> entry : tracker.getIncoming().entrySet())
+ {
+ Range<Token> rangeToFetch = entry.getKey();
+ for (InetAddressAndPort remoteNode : pickLeastStreaming(trackerEntry.getKey(), entry.getValue(), outgoingStreamCounts, filter))
+ rangesToFetch.addSingleRange(remoteNode, rangeToFetch);
+ }
+ mapBuilder.put(trackerEntry.getKey(), rangesToFetch);
+ }
+
+ return mapBuilder.build();
+ }
+
+ @VisibleForTesting
+ static Map<InetAddressAndPort, IncomingRepairStreamTracker> createIncomingRepairStreamTrackers(DifferenceHolder differences)
+ {
+ Map<InetAddressAndPort, IncomingRepairStreamTracker> trackers = new HashMap<>();
+
+ for (InetAddressAndPort hostWithDifference : differences.keyHosts())
+ {
+ HostDifferences hostDifferences = differences.get(hostWithDifference);
+ for (InetAddressAndPort differingHost : hostDifferences.hosts())
+ {
+ List<Range<Token>> differingRanges = hostDifferences.get(differingHost);
+ // hostWithDifference has mismatching ranges differingRanges with differingHost:
+ for (Range<Token> range : differingRanges)
+ {
+ // a difference means that we need to sync that range between two nodes - add the diffing range to both
+ // hosts:
+ getTracker(differences, trackers, hostWithDifference).addIncomingRangeFrom(range, differingHost);
+ getTracker(differences, trackers, differingHost).addIncomingRangeFrom(range, hostWithDifference);
+ }
+ }
+ }
+ return trackers;
+ }
+
+ private static IncomingRepairStreamTracker getTracker(DifferenceHolder differences,
+ Map<InetAddressAndPort, IncomingRepairStreamTracker> trackers,
+ InetAddressAndPort host)
+ {
+ return trackers.computeIfAbsent(host, (h) -> new IncomingRepairStreamTracker(differences));
+ }
+
+ // greedily pick the nodes doing the least amount of streaming
+ private static Collection<InetAddressAndPort> pickLeastStreaming(InetAddressAndPort streamingNode,
+ StreamFromOptions toStreamFrom,
+ Map<InetAddressAndPort, Integer> outgoingStreamCounts,
+ PreferedNodeFilter filter)
+ {
+ Set<InetAddressAndPort> retSet = new HashSet<>();
+ for (Set<InetAddressAndPort> toStream : toStreamFrom.allStreams())
+ {
+ InetAddressAndPort candidate = null;
+ Set<InetAddressAndPort> prefered = filter.apply(streamingNode, toStream);
+ for (InetAddressAndPort node : prefered)
+ {
+ if (candidate == null || outgoingStreamCounts.getOrDefault(candidate, 0) > outgoingStreamCounts.getOrDefault(node, 0))
+ {
+ candidate = node;
+ }
+ }
+ // ok, found no prefered hosts, try all of them
+ if (candidate == null)
+ {
+ for (InetAddressAndPort node : toStream)
+ {
+ if (candidate == null || outgoingStreamCounts.getOrDefault(candidate, 0) > outgoingStreamCounts.getOrDefault(node, 0))
+ {
+ candidate = node;
+ }
+ }
+ }
+ assert candidate != null;
+ outgoingStreamCounts.put(candidate, outgoingStreamCounts.getOrDefault(candidate, 0) + 1);
+ retSet.add(candidate);
+ }
+ return retSet;
+ }
+}
diff --git a/src/java/org/apache/cassandra/repair/asymmetric/StreamFromOptions.java b/src/java/org/apache/cassandra/repair/asymmetric/StreamFromOptions.java
new file mode 100644
index 0000000..6070983
--- /dev/null
+++ b/src/java/org/apache/cassandra/repair/asymmetric/StreamFromOptions.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.repair.asymmetric;
+
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Sets;
+
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.locator.InetAddressAndPort;
+
+/**
+ * Keeps track of where a node needs to stream a given range from.
+ *
+ * If the remote range is identical on several remote nodes, this class keeps track of them
+ *
+ * These stream from options get 'split' during denormalization - for example if we track range
+ * (100, 200] and we find a new differing range (180, 200] - then the denormalization will create two
+ * new StreamFromOptions (see copy below) with the same streamOptions, one with range (100, 180] and one with (180, 200] - then it
+ * adds the new incoming difference to the StreamFromOptions for the new range (180, 200].
+ */
+public class StreamFromOptions
+{
+ /**
+ * all differences - used to figure out if two nodes are equals on the range
+ */
+ private final DifferenceHolder differences;
+ /**
+ * The range to stream
+ */
+ @VisibleForTesting
+ final Range<Token> range;
+ /**
+ * Contains the hosts to stream from - if two nodes are in the same inner set, they are identical for the range we are handling
+ */
+ private final Set<Set<InetAddressAndPort>> streamOptions = new HashSet<>();
+
+ public StreamFromOptions(DifferenceHolder differences, Range<Token> range)
+ {
+ this(differences, range, Collections.emptySet());
+ }
+
+ private StreamFromOptions(DifferenceHolder differences, Range<Token> range, Set<Set<InetAddressAndPort>> existing)
+ {
+ this.differences = differences;
+ this.range = range;
+ for (Set<InetAddressAndPort> addresses : existing)
+ this.streamOptions.add(Sets.newHashSet(addresses));
+ }
+
+ /**
+ * Add new node to the stream options
+ *
+ * If we have no difference between the new node and a currently tracked on, we know they are matching over the
+ * range we are tracking, then just add it to the set with the identical remote nodes. Otherwise create a new group
+ * of nodes containing this new node.
+ */
+ public void add(InetAddressAndPort streamFromNode)
+ {
+ for (Set<InetAddressAndPort> options : streamOptions)
+ {
+ InetAddressAndPort first = options.iterator().next();
+ if (!differences.hasDifferenceBetween(first, streamFromNode, range))
+ {
+ options.add(streamFromNode);
+ return;
+ }
+ }
+ streamOptions.add(Sets.newHashSet(streamFromNode));
+ }
+
+ public StreamFromOptions copy(Range<Token> withRange)
+ {
+ return new StreamFromOptions(differences, withRange, streamOptions);
+ }
+
+ public Iterable<Set<InetAddressAndPort>> allStreams()
+ {
+ return streamOptions;
+ }
+
+ public String toString()
+ {
+ return "StreamFromOptions{" +
+ ", range=" + range +
+ ", streamOptions=" + streamOptions +
+ '}';
+ }
+}
diff --git a/src/java/org/apache/cassandra/repair/consistent/ConsistentSession.java b/src/java/org/apache/cassandra/repair/consistent/ConsistentSession.java
new file mode 100644
index 0000000..d9ac927
--- /dev/null
+++ b/src/java/org/apache/cassandra/repair/consistent/ConsistentSession.java
@@ -0,0 +1,328 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.repair.consistent;
+
+import java.net.InetAddress;
+import java.util.Collection;
+import java.util.EnumMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.repair.messages.FailSession;
+import org.apache.cassandra.repair.messages.FinalizeCommit;
+import org.apache.cassandra.repair.messages.FinalizePromise;
+import org.apache.cassandra.repair.messages.FinalizePropose;
+import org.apache.cassandra.repair.messages.PrepareConsistentRequest;
+import org.apache.cassandra.repair.messages.PrepareConsistentResponse;
+import org.apache.cassandra.repair.messages.PrepareMessage;
+import org.apache.cassandra.repair.messages.RepairOption;
+import org.apache.cassandra.repair.messages.StatusRequest;
+import org.apache.cassandra.repair.messages.StatusResponse;
+import org.apache.cassandra.repair.messages.ValidationRequest;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.service.ActiveRepairService;
+import org.apache.cassandra.tools.nodetool.RepairAdmin;
+
+/**
+ * Base class for consistent Local and Coordinator sessions
+ *
+ * <p/>
+ * There are 4 stages to a consistent incremental repair.
+ *
+ * <h1>Repair prepare</h1>
+ * First, the normal {@link ActiveRepairService#prepareForRepair(UUID, InetAddressAndPort, Set, RepairOption, boolean, List)} stuff
+ * happens, which sends out {@link PrepareMessage} and creates a {@link ActiveRepairService.ParentRepairSession}
+ * on the coordinator and each of the neighbors.
+ *
+ * <h1>Consistent prepare</h1>
+ * The consistent prepare step promotes the parent repair session to a consistent session, and isolates the sstables
+ * being repaired from other sstables. First, the coordinator sends a {@link PrepareConsistentRequest} message to each repair
+ * participant (including itself). When received, the node creates a {@link LocalSession} instance, sets it's state to
+ * {@code PREPARING}, persists it, and begins a preparing the tables for incremental repair, which segregates the data
+ * being repaired from the rest of the table data. When the preparation completes, the session state is set to
+ * {@code PREPARED}, and a {@link PrepareConsistentResponse} is sent to the coordinator indicating success or failure.
+ * If the pending anti-compaction fails, the local session state is set to {@code FAILED}.
+ * <p/>
+ * (see {@link LocalSessions#handlePrepareMessage(InetAddressAndPort, PrepareConsistentRequest)}
+ * <p/>
+ * Once the coordinator recieves positive {@code PrepareConsistentResponse} messages from all the participants, the
+ * coordinator begins the normal repair process.
+ * <p/>
+ * (see {@link CoordinatorSession#handlePrepareResponse(InetAddressAndPort, boolean)}
+ *
+ * <h1>Repair</h1>
+ * The coordinator runs the normal data repair process against the sstables segregated in the previous step. When a
+ * node recieves a {@link ValidationRequest}, it sets it's local session state to {@code REPAIRING}.
+ * <p/>
+ *
+ * If all of the RepairSessions complete successfully, the coordinator begins the {@code Finalization} process. Otherwise,
+ * it begins the {@code Failure} process.
+ *
+ * <h1>Finalization</h1>
+ * The finalization step finishes the session and promotes the sstables to repaired. The coordinator begins by sending
+ * {@link FinalizePropose} messages to each of the participants. Each participant will set it's state to {@code FINALIZE_PROMISED}
+ * and respond with a {@link FinalizePromise} message. Once the coordinator has received promise messages from all participants,
+ * it will send a {@link FinalizeCommit} message to all of them, ending the coordinator session. When a node receives the
+ * {@code FinalizeCommit} message, it will set it's sessions state to {@code FINALIZED}, completing the {@code LocalSession}.
+ * <p/>
+ *
+ * For the sake of simplicity, finalization does not immediately mark pending repair sstables repaired because of potential
+ * conflicts with in progress compactions. The sstables will be marked repaired as part of the normal compaction process.
+ * <p/>
+ *
+ * On the coordinator side, see {@link CoordinatorSession#finalizePropose()}, {@link CoordinatorSession#handleFinalizePromise(InetAddressAndPort, boolean)},
+ * & {@link CoordinatorSession#finalizeCommit()}
+ * <p/>
+ *
+ * On the local session side, see {@link LocalSessions#handleFinalizeProposeMessage(InetAddressAndPort, FinalizePropose)}
+ * & {@link LocalSessions#handleFinalizeCommitMessage(InetAddressAndPort, FinalizeCommit)}
+ *
+ * <h1>Failure</h1>
+ * If there are any failures or problems during the process above, the session will be failed. When a session is failed,
+ * the coordinator will send {@link FailSession} messages to each of the participants. In some cases (basically those not
+ * including Validation and Sync) errors are reported back to the coordinator by the local session, at which point, it
+ * will send {@code FailSession} messages out.
+ * <p/>
+ * Just as with finalization, sstables aren't immediately moved back to unrepaired, but will be demoted as part of the
+ * normal compaction process.
+ *
+ * <p/>
+ * See {@link LocalSessions#failSession(UUID, boolean)} and {@link CoordinatorSession#fail()}
+ *
+ * <h1>Failure Recovery & Session Cleanup</h1>
+ * There are a few scenarios where sessions can get stuck. If a node fails mid session, or it misses a {@code FailSession}
+ * or {@code FinalizeCommit} message, it will never finish. To address this, there is a cleanup task that runs every
+ * 10 minutes that attempts to complete idle sessions.
+ *
+ * <p/>
+ * If a session is not completed (not {@code FINALIZED} or {@code FAILED}) and there's been no activity on the session for
+ * over an hour, the cleanup task will attempt to finish the session by learning the session state of the other participants.
+ * To do this, it sends a {@link StatusRequest} message to the other session participants. The participants respond with a
+ * {@link StatusResponse} message, notifying the sender of their state. If the sender receives a {@code FAILED} response
+ * from any of the participants, it fails the session locally. If it receives a {@code FINALIZED} response from any of the
+ * participants, it will set it's state to {@code FINALIZED} as well. Since the coordinator won't finalize sessions until
+ * it's received {@code FinalizePromise} messages from <i>all</i> participants, this is safe.
+ *
+ *
+ * <p/>
+ * If a session is not completed, and hasn't had any activity for over a day, the session is auto-failed.
+ *
+ * <p/>
+ * Once a session has been completed for over 2 days, it's deleted.
+ *
+ * <p/>
+ * Operators can also manually fail sessions with {@code nodetool repair_admin --cancel}
+ *
+ * <p/>
+ * See {@link LocalSessions#cleanup()} and {@link RepairAdmin}
+ *
+ */
+public abstract class ConsistentSession
+{
+ /**
+ * The possible states of a {@code ConsistentSession}. The typical progression is {@link State#PREPARING}, {@link State#PREPARED},
+ * {@link State#REPAIRING}, {@link State#FINALIZE_PROMISED}, and {@link State#FINALIZED}. With the exception of {@code FINALIZED},
+ * any state can be transitions to {@link State#FAILED}.
+ */
+ public enum State
+ {
+ PREPARING(0),
+ PREPARED(1),
+ REPAIRING(2),
+ FINALIZE_PROMISED(3),
+ FINALIZED(4),
+ FAILED(5);
+
+ State(int expectedOrdinal)
+ {
+ assert ordinal() == expectedOrdinal;
+ }
+
+ private static final Map<State, Set<State>> transitions = new EnumMap<State, Set<State>>(State.class) {{
+ put(PREPARING, ImmutableSet.of(PREPARED, FAILED));
+ put(PREPARED, ImmutableSet.of(REPAIRING, FAILED));
+ put(REPAIRING, ImmutableSet.of(FINALIZE_PROMISED, FAILED));
+ put(FINALIZE_PROMISED, ImmutableSet.of(FINALIZED, FAILED));
+ put(FINALIZED, ImmutableSet.of());
+ put(FAILED, ImmutableSet.of());
+ }};
+
+ public boolean canTransitionTo(State state)
+ {
+ // redundant transitions are allowed because the failure recovery mechanism can
+ // send redundant status changes out, and they shouldn't throw exceptions
+ return state == this || transitions.get(this).contains(state);
+ }
+
+ public static State valueOf(int ordinal)
+ {
+ return values()[ordinal];
+ }
+ }
+
+ private volatile State state;
+ public final UUID sessionID;
+ public final InetAddressAndPort coordinator;
+ public final ImmutableSet<TableId> tableIds;
+ public final long repairedAt;
+ public final ImmutableSet<Range<Token>> ranges;
+ public final ImmutableSet<InetAddressAndPort> participants;
+
+ ConsistentSession(AbstractBuilder builder)
+ {
+ builder.validate();
+ this.state = builder.state;
+ this.sessionID = builder.sessionID;
+ this.coordinator = builder.coordinator;
+ this.tableIds = ImmutableSet.copyOf(builder.ids);
+ this.repairedAt = builder.repairedAt;
+ this.ranges = ImmutableSet.copyOf(builder.ranges);
+ this.participants = ImmutableSet.copyOf(builder.participants);
+ }
+
+ public State getState()
+ {
+ return state;
+ }
+
+ public void setState(State state)
+ {
+ this.state = state;
+ }
+
+ public boolean equals(Object o)
+ {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ ConsistentSession that = (ConsistentSession) o;
+
+ if (repairedAt != that.repairedAt) return false;
+ if (state != that.state) return false;
+ if (!sessionID.equals(that.sessionID)) return false;
+ if (!coordinator.equals(that.coordinator)) return false;
+ if (!tableIds.equals(that.tableIds)) return false;
+ if (!ranges.equals(that.ranges)) return false;
+ return participants.equals(that.participants);
+ }
+
+ public int hashCode()
+ {
+ int result = state.hashCode();
+ result = 31 * result + sessionID.hashCode();
+ result = 31 * result + coordinator.hashCode();
+ result = 31 * result + tableIds.hashCode();
+ result = 31 * result + (int) (repairedAt ^ (repairedAt >>> 32));
+ result = 31 * result + ranges.hashCode();
+ result = 31 * result + participants.hashCode();
+ return result;
+ }
+
+ public String toString()
+ {
+ return "ConsistentSession{" +
+ "state=" + state +
+ ", sessionID=" + sessionID +
+ ", coordinator=" + coordinator +
+ ", tableIds=" + tableIds +
+ ", repairedAt=" + repairedAt +
+ ", ranges=" + ranges +
+ ", participants=" + participants +
+ '}';
+ }
+
+ abstract static class AbstractBuilder
+ {
+ private State state;
+ private UUID sessionID;
+ private InetAddressAndPort coordinator;
+ private Set<TableId> ids;
+ private long repairedAt;
+ private Collection<Range<Token>> ranges;
+ private Set<InetAddressAndPort> participants;
+
+ void withState(State state)
+ {
+ this.state = state;
+ }
+
+ void withSessionID(UUID sessionID)
+ {
+ this.sessionID = sessionID;
+ }
+
+ void withCoordinator(InetAddressAndPort coordinator)
+ {
+ this.coordinator = coordinator;
+ }
+
+ void withUUIDTableIds(Iterable<UUID> ids)
+ {
+ this.ids = ImmutableSet.copyOf(Iterables.transform(ids, TableId::fromUUID));
+ }
+
+ void withTableIds(Set<TableId> ids)
+ {
+ this.ids = ids;
+ }
+
+ void withRepairedAt(long repairedAt)
+ {
+ this.repairedAt = repairedAt;
+ }
+
+ void withRanges(Collection<Range<Token>> ranges)
+ {
+ this.ranges = ranges;
+ }
+
+ void withParticipants(Set<InetAddressAndPort> peers)
+ {
+ this.participants = peers;
+ }
+
+ void validate()
+ {
+ Preconditions.checkArgument(state != null);
+ Preconditions.checkArgument(sessionID != null);
+ Preconditions.checkArgument(coordinator != null);
+ Preconditions.checkArgument(ids != null);
+ Preconditions.checkArgument(!ids.isEmpty());
+ Preconditions.checkArgument(repairedAt > 0
+ || repairedAt == ActiveRepairService.UNREPAIRED_SSTABLE);
+ Preconditions.checkArgument(ranges != null);
+ Preconditions.checkArgument(!ranges.isEmpty());
+ Preconditions.checkArgument(participants != null);
+ Preconditions.checkArgument(!participants.isEmpty());
+ Preconditions.checkArgument(participants.contains(coordinator));
+ }
+ }
+
+
+}
diff --git a/src/java/org/apache/cassandra/repair/consistent/CoordinatorSession.java b/src/java/org/apache/cassandra/repair/consistent/CoordinatorSession.java
new file mode 100644
index 0000000..9d440c2
--- /dev/null
+++ b/src/java/org/apache/cassandra/repair/consistent/CoordinatorSession.java
@@ -0,0 +1,390 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.repair.consistent;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.Supplier;
+
+import javax.annotation.Nullable;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Iterables;
+import com.google.common.util.concurrent.AsyncFunction;
+import com.google.common.util.concurrent.FutureCallback;
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.MoreExecutors;
+import com.google.common.util.concurrent.SettableFuture;
+import org.apache.commons.lang3.time.DurationFormatUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.net.Message;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.net.Verb;
+import org.apache.cassandra.repair.RepairSessionResult;
+import org.apache.cassandra.repair.SomeRepairFailedException;
+import org.apache.cassandra.repair.messages.FailSession;
+import org.apache.cassandra.repair.messages.FinalizeCommit;
+import org.apache.cassandra.repair.messages.FinalizePropose;
+import org.apache.cassandra.repair.messages.PrepareConsistentRequest;
+import org.apache.cassandra.repair.messages.RepairMessage;
+import org.apache.cassandra.service.ActiveRepairService;
+
+/**
+ * Coordinator side logic and state of a consistent repair session. Like {@link ActiveRepairService.ParentRepairSession},
+ * there is only one {@code CoordinatorSession} per user repair command, regardless of the number of tables and token
+ * ranges involved.
+ */
+public class CoordinatorSession extends ConsistentSession
+{
+ private static final Logger logger = LoggerFactory.getLogger(CoordinatorSession.class);
+
+ private final Map<InetAddressAndPort, State> participantStates = new HashMap<>();
+ private final SettableFuture<Boolean> prepareFuture = SettableFuture.create();
+ private final SettableFuture<Boolean> finalizeProposeFuture = SettableFuture.create();
+
+ private volatile long sessionStart = Long.MIN_VALUE;
+ private volatile long repairStart = Long.MIN_VALUE;
+ private volatile long finalizeStart = Long.MIN_VALUE;
+
+ public CoordinatorSession(Builder builder)
+ {
+ super(builder);
+ for (InetAddressAndPort participant : participants)
+ {
+ participantStates.put(participant, State.PREPARING);
+ }
+ }
+
+ public static class Builder extends AbstractBuilder
+ {
+ public CoordinatorSession build()
+ {
+ validate();
+ return new CoordinatorSession(this);
+ }
+ }
+
+ public static Builder builder()
+ {
+ return new Builder();
+ }
+
+ public void setState(State state)
+ {
+ logger.trace("Setting coordinator state to {} for repair {}", state, sessionID);
+ super.setState(state);
+ }
+
+ @VisibleForTesting
+ synchronized State getParticipantState(InetAddressAndPort participant)
+ {
+ return participantStates.get(participant);
+ }
+
+ public synchronized void setParticipantState(InetAddressAndPort participant, State state)
+ {
+ logger.trace("Setting participant {} to state {} for repair {}", participant, state, sessionID);
+ Preconditions.checkArgument(participantStates.containsKey(participant),
+ "Session %s doesn't include %s",
+ sessionID, participant);
+ Preconditions.checkArgument(participantStates.get(participant).canTransitionTo(state),
+ "Invalid state transition %s -> %s",
+ participantStates.get(participant), state);
+ participantStates.put(participant, state);
+
+ // update coordinator state if all participants are at the value being set
+ if (Iterables.all(participantStates.values(), s -> s == state))
+ {
+ setState(state);
+ }
+ }
+
+ synchronized void setAll(State state)
+ {
+ for (InetAddressAndPort participant : participants)
+ {
+ setParticipantState(participant, state);
+ }
+ }
+
+ synchronized boolean allStates(State state)
+ {
+ return getState() == state && Iterables.all(participantStates.values(), v -> v == state);
+ }
+
+ synchronized boolean hasFailed()
+ {
+ return getState() == State.FAILED || Iterables.any(participantStates.values(), v -> v == State.FAILED);
+ }
+
+ protected void sendMessage(InetAddressAndPort destination, Message<RepairMessage> message)
+ {
+ logger.trace("Sending {} to {}", message.payload, destination);
+ MessagingService.instance().send(message, destination);
+ }
+
+ public ListenableFuture<Boolean> prepare()
+ {
+ Preconditions.checkArgument(allStates(State.PREPARING));
+
+ logger.info("Beginning prepare phase of incremental repair session {}", sessionID);
+ Message<RepairMessage> message =
+ Message.out(Verb.PREPARE_CONSISTENT_REQ, new PrepareConsistentRequest(sessionID, coordinator, participants));
+ for (final InetAddressAndPort participant : participants)
+ {
+ sendMessage(participant, message);
+ }
+ return prepareFuture;
+ }
+
+ public synchronized void handlePrepareResponse(InetAddressAndPort participant, boolean success)
+ {
+ if (!success)
+ {
+ logger.warn("{} failed the prepare phase for incremental repair session {}", participant, sessionID);
+ sendFailureMessageToParticipants();
+ setParticipantState(participant, State.FAILED);
+ }
+ else
+ {
+ logger.trace("Successful prepare response received from {} for repair session {}", participant, sessionID);
+ setParticipantState(participant, State.PREPARED);
+ }
+
+ // don't progress until we've heard from all replicas
+ if(Iterables.any(participantStates.values(), v -> v == State.PREPARING))
+ return;
+
+ if (getState() == State.PREPARED)
+ {
+ logger.info("Incremental repair session {} successfully prepared.", sessionID);
+ prepareFuture.set(true);
+ }
+ else
+ {
+ fail();
+ prepareFuture.set(false);
+ }
+ }
+
+ public synchronized void setRepairing()
+ {
+ setAll(State.REPAIRING);
+ }
+
+ public synchronized ListenableFuture<Boolean> finalizePropose()
+ {
+ Preconditions.checkArgument(allStates(State.REPAIRING));
+ logger.info("Proposing finalization of repair session {}", sessionID);
+ Message<RepairMessage> message = Message.out(Verb.FINALIZE_PROPOSE_MSG, new FinalizePropose(sessionID));
+ for (final InetAddressAndPort participant : participants)
+ {
+ sendMessage(participant, message);
+ }
+ return finalizeProposeFuture;
+ }
+
+ public synchronized void handleFinalizePromise(InetAddressAndPort participant, boolean success)
+ {
+ if (getState() == State.FAILED)
+ {
+ logger.trace("Incremental repair {} has failed, ignoring finalize promise from {}", sessionID, participant);
+ }
+ else if (!success)
+ {
+ logger.warn("Finalization proposal of session {} rejected by {}. Aborting session", sessionID, participant);
+ fail();
+ finalizeProposeFuture.set(false);
+ }
+ else
+ {
+ logger.trace("Successful finalize promise received from {} for repair session {}", participant, sessionID);
+ setParticipantState(participant, State.FINALIZE_PROMISED);
+ if (getState() == State.FINALIZE_PROMISED)
+ {
+ logger.info("Finalization proposal for repair session {} accepted by all participants.", sessionID);
+ finalizeProposeFuture.set(true);
+ }
+ }
+ }
+
+ public synchronized void finalizeCommit()
+ {
+ Preconditions.checkArgument(allStates(State.FINALIZE_PROMISED));
+ logger.info("Committing finalization of repair session {}", sessionID);
+ Message<RepairMessage> message = Message.out(Verb.FINALIZE_COMMIT_MSG, new FinalizeCommit(sessionID));
+ for (final InetAddressAndPort participant : participants)
+ {
+ sendMessage(participant, message);
+ }
+ setAll(State.FINALIZED);
+ logger.info("Incremental repair session {} completed", sessionID);
+ }
+
+ private void sendFailureMessageToParticipants()
+ {
+ Message<RepairMessage> message = Message.out(Verb.FAILED_SESSION_MSG, new FailSession(sessionID));
+ for (final InetAddressAndPort participant : participants)
+ {
+ if (participantStates.get(participant) != State.FAILED)
+ {
+ sendMessage(participant, message);
+ }
+ }
+ }
+
+ public synchronized void fail()
+ {
+ logger.info("Incremental repair session {} failed", sessionID);
+ sendFailureMessageToParticipants();
+ setAll(State.FAILED);
+
+ String exceptionMsg = String.format("Incremental repair session %s has failed", sessionID);
+ finalizeProposeFuture.setException(new RuntimeException(exceptionMsg));
+ prepareFuture.setException(new RuntimeException(exceptionMsg));
+ }
+
+ private static String formatDuration(long then, long now)
+ {
+ if (then == Long.MIN_VALUE || now == Long.MIN_VALUE)
+ {
+ // if neither of the times were initially set, don't return a non-sensical answer
+ return "n/a";
+ }
+ return DurationFormatUtils.formatDurationWords(now - then, true, true);
+ }
+
+ /**
+ * Runs the asynchronous consistent repair session. Actual repair sessions are scheduled via a submitter to make unit testing easier
+ */
+ public ListenableFuture execute(Supplier<ListenableFuture<List<RepairSessionResult>>> sessionSubmitter, AtomicBoolean hasFailure)
+ {
+ logger.info("Beginning coordination of incremental repair session {}", sessionID);
+
+ sessionStart = System.currentTimeMillis();
+ ListenableFuture<Boolean> prepareResult = prepare();
+
+ // run repair sessions normally
+ ListenableFuture<List<RepairSessionResult>> repairSessionResults = Futures.transformAsync(prepareResult, new AsyncFunction<Boolean, List<RepairSessionResult>>()
+ {
+ public ListenableFuture<List<RepairSessionResult>> apply(Boolean success) throws Exception
+ {
+ if (success)
+ {
+ repairStart = System.currentTimeMillis();
+ if (logger.isDebugEnabled())
+ {
+ logger.debug("Incremental repair {} prepare phase completed in {}", sessionID, formatDuration(sessionStart, repairStart));
+ }
+ setRepairing();
+ return sessionSubmitter.get();
+ }
+ else
+ {
+ return Futures.immediateFuture(null);
+ }
+
+ }
+ }, MoreExecutors.directExecutor());
+
+ // mark propose finalization
+ ListenableFuture<Boolean> proposeFuture = Futures.transformAsync(repairSessionResults, new AsyncFunction<List<RepairSessionResult>, Boolean>()
+ {
+ public ListenableFuture<Boolean> apply(List<RepairSessionResult> results) throws Exception
+ {
+ if (results == null || results.isEmpty() || Iterables.any(results, r -> r == null))
+ {
+ finalizeStart = System.currentTimeMillis();
+ if (logger.isDebugEnabled())
+ {
+ logger.debug("Incremental repair {} validation/stream phase completed in {}", sessionID, formatDuration(repairStart, finalizeStart));
+
+ }
+ return Futures.immediateFailedFuture(SomeRepairFailedException.INSTANCE);
+ }
+ else
+ {
+ return finalizePropose();
+ }
+ }
+ }, MoreExecutors.directExecutor());
+
+ // return execution result as set by following callback
+ SettableFuture<Boolean> resultFuture = SettableFuture.create();
+
+ // commit repaired data
+ Futures.addCallback(proposeFuture, new FutureCallback<Boolean>()
+ {
+ public void onSuccess(@Nullable Boolean result)
+ {
+ try
+ {
+ if (result != null && result)
+ {
+ if (logger.isDebugEnabled())
+ {
+ logger.debug("Incremental repair {} finalization phase completed in {}", sessionID, formatDuration(finalizeStart, System.currentTimeMillis()));
+ }
+ finalizeCommit();
+ if (logger.isDebugEnabled())
+ {
+ logger.debug("Incremental repair {} phase completed in {}", sessionID, formatDuration(sessionStart, System.currentTimeMillis()));
+ }
+ }
+ else
+ {
+ hasFailure.set(true);
+ fail();
+ }
+ resultFuture.set(result);
+ }
+ catch (Exception e)
+ {
+ resultFuture.setException(e);
+ }
+ }
+
+ public void onFailure(Throwable t)
+ {
+ try
+ {
+ if (logger.isDebugEnabled())
+ {
+ logger.debug("Incremental repair {} phase failed in {}", sessionID, formatDuration(sessionStart, System.currentTimeMillis()));
+ }
+ hasFailure.set(true);
+ fail();
+ }
+ finally
+ {
+ resultFuture.setException(t);
+ }
+ }
+ }, MoreExecutors.directExecutor());
+
+ return resultFuture;
+ }
+}
diff --git a/src/java/org/apache/cassandra/repair/consistent/CoordinatorSessions.java b/src/java/org/apache/cassandra/repair/consistent/CoordinatorSessions.java
new file mode 100644
index 0000000..b87a2c0
--- /dev/null
+++ b/src/java/org/apache/cassandra/repair/consistent/CoordinatorSessions.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.repair.consistent;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+
+import com.google.common.base.Preconditions;
+
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.repair.messages.FailSession;
+import org.apache.cassandra.repair.messages.FinalizePromise;
+import org.apache.cassandra.repair.messages.PrepareConsistentResponse;
+import org.apache.cassandra.service.ActiveRepairService;
+
+/**
+ * Container for all consistent repair sessions a node is coordinating
+ */
+public class CoordinatorSessions
+{
+ private final Map<UUID, CoordinatorSession> sessions = new HashMap<>();
+
+ protected CoordinatorSession buildSession(CoordinatorSession.Builder builder)
+ {
+ return new CoordinatorSession(builder);
+ }
+
+ public synchronized CoordinatorSession registerSession(UUID sessionId, Set<InetAddressAndPort> participants, boolean isForced)
+ {
+ ActiveRepairService.ParentRepairSession prs = ActiveRepairService.instance.getParentRepairSession(sessionId);
+
+ Preconditions.checkArgument(!sessions.containsKey(sessionId),
+ "A coordinator already exists for session %s", sessionId);
+ Preconditions.checkArgument(!isForced || prs.repairedAt == ActiveRepairService.UNREPAIRED_SSTABLE,
+ "cannot promote data for forced incremental repairs");
+
+ CoordinatorSession.Builder builder = CoordinatorSession.builder();
+ builder.withState(ConsistentSession.State.PREPARING);
+ builder.withSessionID(sessionId);
+ builder.withCoordinator(prs.coordinator);
+
+ builder.withTableIds(prs.getTableIds());
+ builder.withRepairedAt(prs.repairedAt);
+ builder.withRanges(prs.getRanges());
+ builder.withParticipants(participants);
+ CoordinatorSession session = buildSession(builder);
+ sessions.put(session.sessionID, session);
+ return session;
+ }
+
+ public synchronized CoordinatorSession getSession(UUID sessionId)
+ {
+ return sessions.get(sessionId);
+ }
+
+ public void handlePrepareResponse(PrepareConsistentResponse msg)
+ {
+ CoordinatorSession session = getSession(msg.parentSession);
+ if (session != null)
+ {
+ session.handlePrepareResponse(msg.participant, msg.success);
+ }
+ }
+
+ public void handleFinalizePromiseMessage(FinalizePromise msg)
+ {
+ CoordinatorSession session = getSession(msg.sessionID);
+ if (session != null)
+ {
+ session.handleFinalizePromise(msg.participant, msg.promised);
+ }
+ }
+
+ public void handleFailSessionMessage(FailSession msg)
+ {
+ CoordinatorSession session = getSession(msg.sessionID);
+ if (session != null)
+ {
+ session.fail();
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/repair/consistent/LocalSession.java b/src/java/org/apache/cassandra/repair/consistent/LocalSession.java
new file mode 100644
index 0000000..e116a43
--- /dev/null
+++ b/src/java/org/apache/cassandra/repair/consistent/LocalSession.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.repair.consistent;
+
+import com.google.common.base.Preconditions;
+
+import org.apache.cassandra.utils.FBUtilities;
+
+/**
+ * Basically just a record of a local session. All of the local session logic is implemented in {@link LocalSessions}
+ */
+public class LocalSession extends ConsistentSession
+{
+ public final int startedAt;
+ private volatile int lastUpdate;
+
+ public LocalSession(Builder builder)
+ {
+ super(builder);
+ this.startedAt = builder.startedAt;
+ this.lastUpdate = builder.lastUpdate;
+ }
+
+ public boolean isCompleted()
+ {
+ State s = getState();
+ return s == State.FINALIZED || s == State.FAILED;
+ }
+
+ public int getStartedAt()
+ {
+ return startedAt;
+ }
+
+ public int getLastUpdate()
+ {
+ return lastUpdate;
+ }
+
+ public void setLastUpdate()
+ {
+ lastUpdate = FBUtilities.nowInSeconds();
+ }
+
+ public boolean equals(Object o)
+ {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ if (!super.equals(o)) return false;
+
+ LocalSession session = (LocalSession) o;
+
+ if (startedAt != session.startedAt) return false;
+ return lastUpdate == session.lastUpdate;
+ }
+
+ public int hashCode()
+ {
+ int result = super.hashCode();
+ result = 31 * result + startedAt;
+ result = 31 * result + lastUpdate;
+ return result;
+ }
+
+ public String toString()
+ {
+ return "LocalSession{" +
+ "sessionID=" + sessionID +
+ ", state=" + getState() +
+ ", coordinator=" + coordinator +
+ ", tableIds=" + tableIds +
+ ", repairedAt=" + repairedAt +
+ ", ranges=" + ranges +
+ ", participants=" + participants +
+ ", startedAt=" + startedAt +
+ ", lastUpdate=" + lastUpdate +
+ '}';
+ }
+
+ public static class Builder extends AbstractBuilder
+ {
+ private int startedAt;
+ private int lastUpdate;
+
+ public void withStartedAt(int startedAt)
+ {
+ this.startedAt = startedAt;
+ }
+
+ public void withLastUpdate(int lastUpdate)
+ {
+ this.lastUpdate = lastUpdate;
+ }
+
+ void validate()
+ {
+ super.validate();
+ Preconditions.checkArgument(startedAt > 0);
+ Preconditions.checkArgument(lastUpdate > 0);
+ }
+
+ public LocalSession build()
+ {
+ validate();
+ return new LocalSession(this);
+ }
+ }
+
+ public static Builder builder()
+ {
+ return new Builder();
+ }
+}
diff --git a/src/java/org/apache/cassandra/repair/consistent/LocalSessionInfo.java b/src/java/org/apache/cassandra/repair/consistent/LocalSessionInfo.java
new file mode 100644
index 0000000..98b883a
--- /dev/null
+++ b/src/java/org/apache/cassandra/repair/consistent/LocalSessionInfo.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.repair.consistent;
+
+import java.net.InetAddress;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import com.google.common.base.Joiner;
+import com.google.common.collect.Iterables;
+
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.schema.TableMetadata;
+
+/**
+ * helper for JMX management functions
+ */
+public class LocalSessionInfo
+{
+ public static final String SESSION_ID = "SESSION_ID";
+ public static final String STATE = "STATE";
+ public static final String STARTED = "STARTED";
+ public static final String LAST_UPDATE = "LAST_UPDATE";
+ public static final String COORDINATOR = "COORDINATOR";
+ public static final String PARTICIPANTS = "PARTICIPANTS";
+ public static final String PARTICIPANTS_WP = "PARTICIPANTS_WP";
+ public static final String TABLES = "TABLES";
+
+
+ private LocalSessionInfo() {}
+
+ private static String tableString(TableId id)
+ {
+ TableMetadata meta = Schema.instance.getTableMetadata(id);
+ return meta != null ? meta.keyspace + '.' + meta.name : "<null>";
+ }
+
+ static Map<String, String> sessionToMap(LocalSession session)
+ {
+ Map<String, String> m = new HashMap<>();
+ m.put(SESSION_ID, session.sessionID.toString());
+ m.put(STATE, session.getState().toString());
+ m.put(STARTED, Integer.toString(session.getStartedAt()));
+ m.put(LAST_UPDATE, Integer.toString(session.getLastUpdate()));
+ m.put(COORDINATOR, session.coordinator.toString());
+ m.put(PARTICIPANTS, Joiner.on(',').join(Iterables.transform(session.participants.stream().map(peer -> peer.address).collect(Collectors.toList()), InetAddress::getHostAddress)));
+ m.put(PARTICIPANTS_WP, Joiner.on(',').join(Iterables.transform(session.participants, InetAddressAndPort::toString)));
+ m.put(TABLES, Joiner.on(',').join(Iterables.transform(session.tableIds, LocalSessionInfo::tableString)));
+
+ return m;
+ }
+}
diff --git a/src/java/org/apache/cassandra/repair/consistent/LocalSessions.java b/src/java/org/apache/cassandra/repair/consistent/LocalSessions.java
new file mode 100644
index 0000000..a35c50a
--- /dev/null
+++ b/src/java/org/apache/cassandra/repair/consistent/LocalSessions.java
@@ -0,0 +1,890 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.repair.consistent;
+
+import java.io.IOException;
+import java.net.UnknownHostException;
+import java.nio.ByteBuffer;
+import java.time.Instant;
+import java.util.Collection;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.CopyOnWriteArraySet;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.function.BooleanSupplier;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
+import javax.annotation.Nullable;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.common.primitives.Ints;
+import com.google.common.util.concurrent.FutureCallback;
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.MoreExecutors;
+
+import org.apache.cassandra.locator.RangesAtEndpoint;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.locator.Replica;
+import org.apache.cassandra.repair.KeyspaceRepairManager;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.schema.SchemaConstants;
+import org.apache.cassandra.cql3.QueryProcessor;
+import org.apache.cassandra.cql3.UntypedResultSet;
+import org.apache.cassandra.db.SystemKeyspace;
+import org.apache.cassandra.db.marshal.BytesType;
+import org.apache.cassandra.db.marshal.UUIDType;
+import org.apache.cassandra.dht.IPartitioner;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.gms.FailureDetector;
+import org.apache.cassandra.io.util.DataInputBuffer;
+import org.apache.cassandra.io.util.DataOutputBuffer;
+import org.apache.cassandra.net.Message;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.repair.messages.FailSession;
+import org.apache.cassandra.repair.messages.FinalizeCommit;
+import org.apache.cassandra.repair.messages.FinalizePromise;
+import org.apache.cassandra.repair.messages.FinalizePropose;
+import org.apache.cassandra.repair.messages.PrepareConsistentRequest;
+import org.apache.cassandra.repair.messages.PrepareConsistentResponse;
+import org.apache.cassandra.repair.messages.RepairMessage;
+import org.apache.cassandra.repair.messages.StatusRequest;
+import org.apache.cassandra.repair.messages.StatusResponse;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.service.ActiveRepairService;
+import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.utils.FBUtilities;
+
+import static org.apache.cassandra.net.Verb.FAILED_SESSION_MSG;
+import static org.apache.cassandra.net.Verb.FINALIZE_PROMISE_MSG;
+import static org.apache.cassandra.net.Verb.PREPARE_CONSISTENT_RSP;
+import static org.apache.cassandra.net.Verb.STATUS_REQ;
+import static org.apache.cassandra.net.Verb.STATUS_RSP;
+import static org.apache.cassandra.repair.consistent.ConsistentSession.State.*;
+
+/**
+ * Manages all consistent repair sessions a node is participating in.
+ * <p/>
+ * Since sessions need to be loaded, and since we need to handle cases where sessions might not exist, most of the logic
+ * around local sessions is implemented in this class, with the LocalSession class being treated more like a simple struct,
+ * in contrast with {@link CoordinatorSession}
+ */
+public class LocalSessions
+{
+ private static final Logger logger = LoggerFactory.getLogger(LocalSessions.class);
+ private static final Set<Listener> listeners = new CopyOnWriteArraySet<>();
+
+ /**
+ * Amount of time a session can go without any activity before we start checking the status of other
+ * participants to see if we've missed a message
+ */
+ static final int CHECK_STATUS_TIMEOUT = Integer.getInteger("cassandra.repair_status_check_timeout_seconds",
+ Ints.checkedCast(TimeUnit.HOURS.toSeconds(1)));
+
+ /**
+ * Amount of time a session can go without any activity before being automatically set to FAILED
+ */
+ static final int AUTO_FAIL_TIMEOUT = Integer.getInteger("cassandra.repair_fail_timeout_seconds",
+ Ints.checkedCast(TimeUnit.DAYS.toSeconds(1)));
+
+ /**
+ * Amount of time a completed session is kept around after completion before being deleted, this gives
+ * compaction plenty of time to move sstables from successful sessions into the repaired bucket
+ */
+ static final int AUTO_DELETE_TIMEOUT = Integer.getInteger("cassandra.repair_delete_timeout_seconds",
+ Ints.checkedCast(TimeUnit.DAYS.toSeconds(1)));
+ /**
+ * How often LocalSessions.cleanup is run
+ */
+ public static final int CLEANUP_INTERVAL = Integer.getInteger("cassandra.repair_cleanup_interval_seconds",
+ Ints.checkedCast(TimeUnit.MINUTES.toSeconds(10)));
+
+ private static Set<TableId> uuidToTableId(Set<UUID> src)
+ {
+ return ImmutableSet.copyOf(Iterables.transform(src, TableId::fromUUID));
+ }
+
+ private static Set<UUID> tableIdToUuid(Set<TableId> src)
+ {
+ return ImmutableSet.copyOf(Iterables.transform(src, TableId::asUUID));
+ }
+
+ private final String keyspace = SchemaConstants.SYSTEM_KEYSPACE_NAME;
+ private final String table = SystemKeyspace.REPAIRS;
+ private boolean started = false;
+ private volatile ImmutableMap<UUID, LocalSession> sessions = ImmutableMap.of();
+
+ @VisibleForTesting
+ int getNumSessions()
+ {
+ return sessions.size();
+ }
+
+ @VisibleForTesting
+ protected InetAddressAndPort getBroadcastAddressAndPort()
+ {
+ return FBUtilities.getBroadcastAddressAndPort();
+ }
+
+ @VisibleForTesting
+ protected boolean isAlive(InetAddressAndPort address)
+ {
+ return FailureDetector.instance.isAlive(address);
+ }
+
+ @VisibleForTesting
+ protected boolean isNodeInitialized()
+ {
+ return StorageService.instance.isInitialized();
+ }
+
+ public List<Map<String, String>> sessionInfo(boolean all)
+ {
+ Iterable<LocalSession> currentSessions = sessions.values();
+ if (!all)
+ {
+ currentSessions = Iterables.filter(currentSessions, s -> !s.isCompleted());
+ }
+ return Lists.newArrayList(Iterables.transform(currentSessions, LocalSessionInfo::sessionToMap));
+ }
+
+ /**
+ * hook for operators to cancel sessions, cancelling from a non-coordinator is an error, unless
+ * force is set to true. Messages are sent out to other participants, but we don't wait for a response
+ */
+ public void cancelSession(UUID sessionID, boolean force)
+ {
+ logger.info("Cancelling local repair session {}", sessionID);
+ LocalSession session = getSession(sessionID);
+ Preconditions.checkArgument(session != null, "Session {} does not exist", sessionID);
+ Preconditions.checkArgument(force || session.coordinator.equals(getBroadcastAddressAndPort()),
+ "Cancel session %s from it's coordinator (%s) or use --force",
+ sessionID, session.coordinator);
+
+ setStateAndSave(session, FAILED);
+ Message<FailSession> message = Message.out(FAILED_SESSION_MSG, new FailSession(sessionID));
+ for (InetAddressAndPort participant : session.participants)
+ {
+ if (!participant.equals(getBroadcastAddressAndPort()))
+ sendMessage(participant, message);
+ }
+ }
+
+ /**
+ * Loads sessions out of the repairs table and sets state to started
+ */
+ public synchronized void start()
+ {
+ Preconditions.checkArgument(!started, "LocalSessions.start can only be called once");
+ Preconditions.checkArgument(sessions.isEmpty(), "No sessions should be added before start");
+ UntypedResultSet rows = QueryProcessor.executeInternalWithPaging(String.format("SELECT * FROM %s.%s", keyspace, table), 1000);
+ Map<UUID, LocalSession> loadedSessions = new HashMap<>();
+ for (UntypedResultSet.Row row : rows)
+ {
+ try
+ {
+ LocalSession session = load(row);
+ loadedSessions.put(session.sessionID, session);
+ }
+ catch (IllegalArgumentException | NullPointerException e)
+ {
+ logger.warn("Unable to load malformed repair session {}, ignoring", row.has("parent_id") ? row.getUUID("parent_id") : null);
+ }
+ }
+ sessions = ImmutableMap.copyOf(loadedSessions);
+ started = true;
+ }
+
+ public boolean isStarted()
+ {
+ return started;
+ }
+
+ private static boolean shouldCheckStatus(LocalSession session, int now)
+ {
+ return !session.isCompleted() && (now > session.getLastUpdate() + CHECK_STATUS_TIMEOUT);
+ }
+
+ private static boolean shouldFail(LocalSession session, int now)
+ {
+ return !session.isCompleted() && (now > session.getLastUpdate() + AUTO_FAIL_TIMEOUT);
+ }
+
+ private static boolean shouldDelete(LocalSession session, int now)
+ {
+ return session.isCompleted() && (now > session.getLastUpdate() + AUTO_DELETE_TIMEOUT);
+ }
+
+ /**
+ * Auto fails and auto deletes timed out and old sessions
+ * Compaction will clean up the sstables still owned by a deleted session
+ */
+ public void cleanup()
+ {
+ logger.trace("Running LocalSessions.cleanup");
+ if (!isNodeInitialized())
+ {
+ logger.trace("node not initialized, aborting local session cleanup");
+ return;
+ }
+ Set<LocalSession> currentSessions = new HashSet<>(sessions.values());
+ for (LocalSession session : currentSessions)
+ {
+ synchronized (session)
+ {
+ int now = FBUtilities.nowInSeconds();
+ if (shouldFail(session, now))
+ {
+ logger.warn("Auto failing timed out repair session {}", session);
+ failSession(session.sessionID, false);
+ }
+ else if (shouldDelete(session, now))
+ {
+ if (!sessionHasData(session))
+ {
+ logger.info("Auto deleting repair session {}", session);
+ deleteSession(session.sessionID);
+ }
+ else
+ {
+ logger.warn("Skipping delete of LocalSession {} because it still contains sstables", session.sessionID);
+ }
+ }
+ else if (shouldCheckStatus(session, now))
+ {
+ sendStatusRequest(session);
+ }
+ }
+ }
+ }
+
+ private static ByteBuffer serializeRange(Range<Token> range)
+ {
+ int size = (int) Token.serializer.serializedSize(range.left, 0);
+ size += (int) Token.serializer.serializedSize(range.right, 0);
+ try (DataOutputBuffer buffer = new DataOutputBuffer(size))
+ {
+ Token.serializer.serialize(range.left, buffer, 0);
+ Token.serializer.serialize(range.right, buffer, 0);
+ return buffer.buffer();
+ }
+ catch (IOException e)
+ {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private static Set<ByteBuffer> serializeRanges(Set<Range<Token>> ranges)
+ {
+ Set<ByteBuffer> buffers = new HashSet<>(ranges.size());
+ ranges.forEach(r -> buffers.add(serializeRange(r)));
+ return buffers;
+ }
+
+ private static Range<Token> deserializeRange(ByteBuffer bb)
+ {
+ try (DataInputBuffer in = new DataInputBuffer(bb, false))
+ {
+ IPartitioner partitioner = DatabaseDescriptor.getPartitioner();
+ Token left = Token.serializer.deserialize(in, partitioner, 0);
+ Token right = Token.serializer.deserialize(in, partitioner, 0);
+ return new Range<>(left, right);
+ }
+ catch (IOException e)
+ {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private static Set<Range<Token>> deserializeRanges(Set<ByteBuffer> buffers)
+ {
+ Set<Range<Token>> ranges = new HashSet<>(buffers.size());
+ buffers.forEach(bb -> ranges.add(deserializeRange(bb)));
+ return ranges;
+ }
+
+ /**
+ * Save session state to table
+ */
+ @VisibleForTesting
+ void save(LocalSession session)
+ {
+ String query = "INSERT INTO %s.%s " +
+ "(parent_id, " +
+ "started_at, " +
+ "last_update, " +
+ "repaired_at, " +
+ "state, " +
+ "coordinator, " +
+ "coordinator_port, " +
+ "participants, " +
+ "participants_wp," +
+ "ranges, " +
+ "cfids) " +
+ "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)";
+
+ QueryProcessor.executeInternal(String.format(query, keyspace, table),
+ session.sessionID,
+ Date.from(Instant.ofEpochSecond(session.startedAt)),
+ Date.from(Instant.ofEpochSecond(session.getLastUpdate())),
+ Date.from(Instant.ofEpochMilli(session.repairedAt)),
+ session.getState().ordinal(),
+ session.coordinator.address,
+ session.coordinator.port,
+ session.participants.stream().map(participant -> participant.address).collect(Collectors.toSet()),
+ session.participants.stream().map(participant -> participant.toString()).collect(Collectors.toSet()),
+ serializeRanges(session.ranges),
+ tableIdToUuid(session.tableIds));
+ }
+
+ private static int dateToSeconds(Date d)
+ {
+ return Ints.checkedCast(TimeUnit.MILLISECONDS.toSeconds(d.getTime()));
+ }
+
+ private LocalSession load(UntypedResultSet.Row row)
+ {
+ LocalSession.Builder builder = LocalSession.builder();
+ builder.withState(ConsistentSession.State.valueOf(row.getInt("state")));
+ builder.withSessionID(row.getUUID("parent_id"));
+ InetAddressAndPort coordinator = InetAddressAndPort.getByAddressOverrideDefaults(
+ row.getInetAddress("coordinator"),
+ row.getInt("coordinator_port"));
+ builder.withCoordinator(coordinator);
+ builder.withTableIds(uuidToTableId(row.getSet("cfids", UUIDType.instance)));
+ builder.withRepairedAt(row.getTimestamp("repaired_at").getTime());
+ builder.withRanges(deserializeRanges(row.getSet("ranges", BytesType.instance)));
+ //There is no cross version streaming and thus no cross version repair so assume that
+ //any valid repair sessions has the participants_wp column and any that doesn't is malformed
+ Set<String> participants = row.getSet("participants_wp", UTF8Type.instance);
+ builder.withParticipants(participants.stream().map(participant ->
+ {
+ try
+ {
+ return InetAddressAndPort.getByName(participant);
+ }
+ catch (UnknownHostException e)
+ {
+ throw new RuntimeException(e);
+ }
+ }).collect(Collectors.toSet()));
+ builder.withStartedAt(dateToSeconds(row.getTimestamp("started_at")));
+ builder.withLastUpdate(dateToSeconds(row.getTimestamp("last_update")));
+
+ return buildSession(builder);
+ }
+
+ private void deleteRow(UUID sessionID)
+ {
+ String query = "DELETE FROM %s.%s WHERE parent_id=?";
+ QueryProcessor.executeInternal(String.format(query, keyspace, table), sessionID);
+ }
+
+ private void syncTable()
+ {
+ TableId tid = Schema.instance.getTableMetadata(keyspace, table).id;
+ ColumnFamilyStore cfm = Schema.instance.getColumnFamilyStoreInstance(tid);
+ cfm.forceBlockingFlush();
+ }
+
+ /**
+ * Loads a session directly from the table. Should be used for testing only
+ */
+ @VisibleForTesting
+ LocalSession loadUnsafe(UUID sessionId)
+ {
+ String query = "SELECT * FROM %s.%s WHERE parent_id=?";
+ UntypedResultSet result = QueryProcessor.executeInternal(String.format(query, keyspace, table), sessionId);
+ if (result.isEmpty())
+ return null;
+
+ UntypedResultSet.Row row = result.one();
+ return load(row);
+ }
+
+ @VisibleForTesting
+ protected LocalSession buildSession(LocalSession.Builder builder)
+ {
+ return new LocalSession(builder);
+ }
+
+ public LocalSession getSession(UUID sessionID)
+ {
+ return sessions.get(sessionID);
+ }
+
+ @VisibleForTesting
+ synchronized void putSessionUnsafe(LocalSession session)
+ {
+ putSession(session);
+ save(session);
+ }
+
+ private synchronized void putSession(LocalSession session)
+ {
+ Preconditions.checkArgument(!sessions.containsKey(session.sessionID),
+ "LocalSession %s already exists", session.sessionID);
+ Preconditions.checkArgument(started, "sessions cannot be added before LocalSessions is started");
+ sessions = ImmutableMap.<UUID, LocalSession>builder()
+ .putAll(sessions)
+ .put(session.sessionID, session)
+ .build();
+ }
+
+ private synchronized void removeSession(UUID sessionID)
+ {
+ Preconditions.checkArgument(sessionID != null);
+ Map<UUID, LocalSession> temp = new HashMap<>(sessions);
+ temp.remove(sessionID);
+ sessions = ImmutableMap.copyOf(temp);
+ }
+
+ @VisibleForTesting
+ LocalSession createSessionUnsafe(UUID sessionId, ActiveRepairService.ParentRepairSession prs, Set<InetAddressAndPort> peers)
+ {
+ LocalSession.Builder builder = LocalSession.builder();
+ builder.withState(ConsistentSession.State.PREPARING);
+ builder.withSessionID(sessionId);
+ builder.withCoordinator(prs.coordinator);
+
+ builder.withTableIds(prs.getTableIds());
+ builder.withRepairedAt(prs.repairedAt);
+ builder.withRanges(prs.getRanges());
+ builder.withParticipants(peers);
+
+ int now = FBUtilities.nowInSeconds();
+ builder.withStartedAt(now);
+ builder.withLastUpdate(now);
+
+ return buildSession(builder);
+ }
+
+ protected ActiveRepairService.ParentRepairSession getParentRepairSession(UUID sessionID)
+ {
+ return ActiveRepairService.instance.getParentRepairSession(sessionID);
+ }
+
+ protected void sendMessage(InetAddressAndPort destination, Message<? extends RepairMessage> message)
+ {
+ logger.trace("sending {} to {}", message.payload, destination);
+ MessagingService.instance().send(message, destination);
+ }
+
+ private void setStateAndSave(LocalSession session, ConsistentSession.State state)
+ {
+ synchronized (session)
+ {
+ Preconditions.checkArgument(session.getState().canTransitionTo(state),
+ "Invalid state transition %s -> %s",
+ session.getState(), state);
+ logger.trace("Changing LocalSession state from {} -> {} for {}", session.getState(), state, session.sessionID);
+ boolean wasCompleted = session.isCompleted();
+ session.setState(state);
+ session.setLastUpdate();
+ save(session);
+
+ if (session.isCompleted() && !wasCompleted)
+ {
+ sessionCompleted(session);
+ }
+ for (Listener listener : listeners)
+ listener.onIRStateChange(session);
+ }
+ }
+
+ public void failSession(UUID sessionID)
+ {
+ failSession(sessionID, true);
+ }
+
+ public void failSession(UUID sessionID, boolean sendMessage)
+ {
+ LocalSession session = getSession(sessionID);
+ if (session != null)
+ {
+ synchronized (session)
+ {
+ if (session.getState() != FAILED)
+ {
+ logger.info("Failing local repair session {}", sessionID);
+ setStateAndSave(session, FAILED);
+ }
+ }
+ if (sendMessage)
+ {
+ sendMessage(session.coordinator, Message.out(FAILED_SESSION_MSG, new FailSession(sessionID)));
+ }
+ }
+ }
+
+ public synchronized void deleteSession(UUID sessionID)
+ {
+ logger.info("Deleting local repair session {}", sessionID);
+ LocalSession session = getSession(sessionID);
+ Preconditions.checkArgument(session.isCompleted(), "Cannot delete incomplete sessions");
+
+ deleteRow(sessionID);
+ removeSession(sessionID);
+ }
+
+ @VisibleForTesting
+ ListenableFuture prepareSession(KeyspaceRepairManager repairManager,
+ UUID sessionID,
+ Collection<ColumnFamilyStore> tables,
+ RangesAtEndpoint tokenRanges,
+ ExecutorService executor,
+ BooleanSupplier isCancelled)
+ {
+ return repairManager.prepareIncrementalRepair(sessionID, tables, tokenRanges, executor, isCancelled);
+ }
+
+ RangesAtEndpoint filterLocalRanges(String keyspace, Set<Range<Token>> ranges)
+ {
+ RangesAtEndpoint localRanges = StorageService.instance.getLocalReplicas(keyspace);
+ RangesAtEndpoint.Builder builder = RangesAtEndpoint.builder(localRanges.endpoint());
+ for (Range<Token> range : ranges)
+ {
+ for (Replica replica : localRanges)
+ {
+ if (replica.range().equals(range))
+ {
+ builder.add(replica);
+ }
+ else if (replica.contains(range))
+ {
+ builder.add(replica.decorateSubrange(range));
+ }
+ }
+
+ }
+ return builder.build();
+ }
+
+ /**
+ * The PrepareConsistentRequest promotes the parent repair session to a consistent incremental
+ * session, and isolates the data to be repaired from the rest of the table's data
+ *
+ * No response is sent to the repair coordinator until the data preparation / isolation has completed
+ * successfully. If the data preparation fails, a failure message is sent to the coordinator,
+ * cancelling the session.
+ */
+ public void handlePrepareMessage(InetAddressAndPort from, PrepareConsistentRequest request)
+ {
+ logger.trace("received {} from {}", request, from);
+ UUID sessionID = request.parentSession;
+ InetAddressAndPort coordinator = request.coordinator;
+ Set<InetAddressAndPort> peers = request.participants;
+
+ ActiveRepairService.ParentRepairSession parentSession;
+ try
+ {
+ parentSession = getParentRepairSession(sessionID);
+ }
+ catch (Throwable e)
+ {
+ logger.error("Error retrieving ParentRepairSession for session {}, responding with failure", sessionID);
+ sendMessage(coordinator, Message.out(PREPARE_CONSISTENT_RSP, new PrepareConsistentResponse(sessionID, getBroadcastAddressAndPort(), false)));
+ return;
+ }
+
+ LocalSession session = createSessionUnsafe(sessionID, parentSession, peers);
+ putSessionUnsafe(session);
+ logger.info("Beginning local incremental repair session {}", session);
+
+ ExecutorService executor = Executors.newFixedThreadPool(parentSession.getColumnFamilyStores().size());
+
+ KeyspaceRepairManager repairManager = parentSession.getKeyspace().getRepairManager();
+ RangesAtEndpoint tokenRanges = filterLocalRanges(parentSession.getKeyspace().getName(), parentSession.getRanges());
+ ListenableFuture repairPreparation = prepareSession(repairManager, sessionID, parentSession.getColumnFamilyStores(),
+ tokenRanges, executor, () -> session.getState() != PREPARING);
+
+ Futures.addCallback(repairPreparation, new FutureCallback<Object>()
+ {
+ public void onSuccess(@Nullable Object result)
+ {
+ try
+ {
+ logger.info("Prepare phase for incremental repair session {} completed", sessionID);
+ if (session.getState() != FAILED)
+ setStateAndSave(session, PREPARED);
+ else
+ logger.info("Session {} failed before anticompaction completed", sessionID);
+
+ Message<PrepareConsistentResponse> message =
+ Message.out(PREPARE_CONSISTENT_RSP,
+ new PrepareConsistentResponse(sessionID, getBroadcastAddressAndPort(), session.getState() != FAILED));
+ sendMessage(coordinator, message);
+ }
+ finally
+ {
+ executor.shutdown();
+ }
+ }
+
+ public void onFailure(Throwable t)
+ {
+ try
+ {
+ logger.error("Prepare phase for incremental repair session {} failed", sessionID, t);
+ sendMessage(coordinator,
+ Message.out(PREPARE_CONSISTENT_RSP,
+ new PrepareConsistentResponse(sessionID, getBroadcastAddressAndPort(), false)));
+ failSession(sessionID, false);
+ }
+ finally
+ {
+ executor.shutdown();
+ }
+ }
+ }, MoreExecutors.directExecutor());
+ }
+
+ public void maybeSetRepairing(UUID sessionID)
+ {
+ LocalSession session = getSession(sessionID);
+ if (session != null && session.getState() != REPAIRING)
+ {
+ logger.info("Setting local incremental repair session {} to REPAIRING", session);
+ setStateAndSave(session, REPAIRING);
+ }
+ }
+
+ public void handleFinalizeProposeMessage(InetAddressAndPort from, FinalizePropose propose)
+ {
+ logger.trace("received {} from {}", propose, from);
+ UUID sessionID = propose.sessionID;
+ LocalSession session = getSession(sessionID);
+ if (session == null)
+ {
+ logger.info("Received FinalizePropose message for unknown repair session {}, responding with failure", sessionID);
+ sendMessage(from, Message.out(FAILED_SESSION_MSG, new FailSession(sessionID)));
+ return;
+ }
+
+ try
+ {
+ setStateAndSave(session, FINALIZE_PROMISED);
+
+ /*
+ Flushing the repairs table here, *before* responding to the coordinator prevents a scenario where we respond
+ with a promise to the coordinator, but there is a failure before the commit log mutation with the
+ FINALIZE_PROMISED status is synced to disk. This could cause the state for this session to revert to an
+ earlier status on startup, which would prevent the failure recovery mechanism from ever being able to promote
+ this session to FINALIZED, likely creating inconsistencies in the repaired data sets across nodes.
+ */
+ syncTable();
+
+ sendMessage(from, Message.out(FINALIZE_PROMISE_MSG, new FinalizePromise(sessionID, getBroadcastAddressAndPort(), true)));
+ logger.info("Received FinalizePropose message for incremental repair session {}, responded with FinalizePromise", sessionID);
+ }
+ catch (IllegalArgumentException e)
+ {
+ logger.error("Error handling FinalizePropose message for {}", session, e);
+ failSession(sessionID);
+ }
+ }
+
+ @VisibleForTesting
+ protected void sessionCompleted(LocalSession session)
+ {
+ for (TableId tid: session.tableIds)
+ {
+ ColumnFamilyStore cfs = Schema.instance.getColumnFamilyStoreInstance(tid);
+ if (cfs != null)
+ {
+ cfs.getRepairManager().incrementalSessionCompleted(session.sessionID);
+ }
+ }
+ }
+
+ /**
+ * Finalizes the repair session, completing it as successful.
+ *
+ * This only changes the state of the session, it doesn't promote the siloed sstables to repaired. That will happen
+ * as part of the compaction process, and avoids having to worry about in progress compactions interfering with the
+ * promotion.
+ */
+ public void handleFinalizeCommitMessage(InetAddressAndPort from, FinalizeCommit commit)
+ {
+ logger.trace("received {} from {}", commit, from);
+ UUID sessionID = commit.sessionID;
+ LocalSession session = getSession(sessionID);
+ if (session == null)
+ {
+ logger.warn("Ignoring FinalizeCommit message for unknown repair session {}", sessionID);
+ return;
+ }
+
+ setStateAndSave(session, FINALIZED);
+ logger.info("Finalized local repair session {}", sessionID);
+ }
+
+ public void handleFailSessionMessage(InetAddressAndPort from, FailSession msg)
+ {
+ logger.trace("received {} from {}", msg, from);
+ failSession(msg.sessionID, false);
+ }
+
+ public void sendStatusRequest(LocalSession session)
+ {
+ logger.info("Attempting to learn the outcome of unfinished local incremental repair session {}", session.sessionID);
+ Message<StatusRequest> request = Message.out(STATUS_REQ, new StatusRequest(session.sessionID));
+
+ for (InetAddressAndPort participant : session.participants)
+ {
+ if (!getBroadcastAddressAndPort().equals(participant) && isAlive(participant))
+ {
+ sendMessage(participant, request);
+ }
+ }
+ }
+
+ public void handleStatusRequest(InetAddressAndPort from, StatusRequest request)
+ {
+ logger.trace("received {} from {}", request, from);
+ UUID sessionID = request.sessionID;
+ LocalSession session = getSession(sessionID);
+ if (session == null)
+ {
+ logger.warn("Received status request message for unknown session {}", sessionID);
+ sendMessage(from, Message.out(STATUS_RSP, new StatusResponse(sessionID, FAILED)));
+ }
+ else
+ {
+ sendMessage(from, Message.out(STATUS_RSP, new StatusResponse(sessionID, session.getState())));
+ logger.info("Responding to status response message for incremental repair session {} with local state {}", sessionID, session.getState());
+ }
+ }
+
+ public void handleStatusResponse(InetAddressAndPort from, StatusResponse response)
+ {
+ logger.trace("received {} from {}", response, from);
+ UUID sessionID = response.sessionID;
+ LocalSession session = getSession(sessionID);
+ if (session == null)
+ {
+ logger.warn("Received StatusResponse message for unknown repair session {}", sessionID);
+ return;
+ }
+
+ // only change local state if response state is FINALIZED or FAILED, since those are
+ // the only statuses that would indicate we've missed a message completing the session
+ if (response.state == FINALIZED || response.state == FAILED)
+ {
+ setStateAndSave(session, response.state);
+ logger.info("Unfinished local incremental repair session {} set to state {}", sessionID, response.state);
+ }
+ else
+ {
+ logger.info("Received StatusResponse for repair session {} with state {}, which is not actionable. Doing nothing.", sessionID, response.state);
+ }
+ }
+
+ /**
+ * determines if a local session exists, and if it's not finalized or failed
+ */
+ public boolean isSessionInProgress(UUID sessionID)
+ {
+ LocalSession session = getSession(sessionID);
+ return session != null && session.getState() != FINALIZED && session.getState() != FAILED;
+ }
+
+ /**
+ * determines if a local session exists, and if it's in the finalized state
+ */
+ public boolean isSessionFinalized(UUID sessionID)
+ {
+ LocalSession session = getSession(sessionID);
+ return session != null && session.getState() == FINALIZED;
+ }
+
+ /**
+ * determines if a local session exists
+ */
+ public boolean sessionExists(UUID sessionID)
+ {
+ return getSession(sessionID) != null;
+ }
+
+ @VisibleForTesting
+ protected boolean sessionHasData(LocalSession session)
+ {
+ Predicate<TableId> predicate = tid -> {
+ ColumnFamilyStore cfs = Schema.instance.getColumnFamilyStoreInstance(tid);
+ return cfs != null && cfs.getCompactionStrategyManager().hasDataForPendingRepair(session.sessionID);
+
+ };
+ return Iterables.any(session.tableIds, predicate::test);
+ }
+
+ /**
+ * Returns the repairedAt time for a sessions which is unknown, failed, or finalized
+ * calling this for a session which is in progress throws an exception
+ */
+ public long getFinalSessionRepairedAt(UUID sessionID)
+ {
+ LocalSession session = getSession(sessionID);
+ if (session == null || session.getState() == FAILED)
+ {
+ return ActiveRepairService.UNREPAIRED_SSTABLE;
+ }
+ else if (session.getState() == FINALIZED)
+ {
+ return session.repairedAt;
+ }
+ else
+ {
+ throw new IllegalStateException("Cannot get final repaired at value for in progress session: " + session);
+ }
+ }
+
+ public static void registerListener(Listener listener)
+ {
+ listeners.add(listener);
+ }
+
+ public static void unregisterListener(Listener listener)
+ {
+ listeners.remove(listener);
+ }
+
+ public interface Listener
+ {
+ void onIRStateChange(LocalSession session);
+ }
+}
diff --git a/src/java/org/apache/cassandra/repair/consistent/SyncStatSummary.java b/src/java/org/apache/cassandra/repair/consistent/SyncStatSummary.java
new file mode 100644
index 0000000..f8e1bfb
--- /dev/null
+++ b/src/java/org/apache/cassandra/repair/consistent/SyncStatSummary.java
@@ -0,0 +1,245 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.repair.consistent;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+import com.google.common.collect.Lists;
+
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.repair.RepairResult;
+import org.apache.cassandra.repair.RepairSessionResult;
+import org.apache.cassandra.repair.SyncStat;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.streaming.SessionSummary;
+import org.apache.cassandra.streaming.StreamSummary;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.Pair;
+
+import static com.google.common.collect.Iterables.filter;
+
+public class SyncStatSummary
+{
+
+ private static class Session
+ {
+ final InetAddressAndPort src;
+ final InetAddressAndPort dst;
+
+ int files = 0;
+ long bytes = 0;
+ long ranges = 0;
+
+ Session(InetAddressAndPort src, InetAddressAndPort dst)
+ {
+ this.src = src;
+ this.dst = dst;
+ }
+
+ void consumeSummary(StreamSummary summary)
+ {
+ files += summary.files;
+ bytes += summary.totalSize;
+ }
+
+ void consumeSummaries(Collection<StreamSummary> summaries, long numRanges)
+ {
+ summaries.forEach(this::consumeSummary);
+ ranges += numRanges;
+ }
+
+ public String toString()
+ {
+ return String.format("%s -> %s: %s ranges, %s sstables, %s bytes", src, dst, ranges, files, FBUtilities.prettyPrintMemory(bytes));
+ }
+ }
+
+ private static class Table
+ {
+ final String keyspace;
+
+ final String table;
+
+ int files = -1;
+ long bytes = -1;
+ int ranges = -1;
+ boolean totalsCalculated = false;
+
+ final Map<Pair<InetAddressAndPort, InetAddressAndPort>, Session> sessions = new HashMap<>();
+
+ Table(String keyspace, String table)
+ {
+ this.keyspace = keyspace;
+ this.table = table;
+ }
+
+ Session getOrCreate(InetAddressAndPort from, InetAddressAndPort to)
+ {
+ Pair<InetAddressAndPort, InetAddressAndPort> k = Pair.create(from, to);
+ if (!sessions.containsKey(k))
+ {
+ sessions.put(k, new Session(from, to));
+ }
+ return sessions.get(k);
+ }
+
+ void consumeStat(SyncStat stat)
+ {
+ for (SessionSummary summary: stat.summaries)
+ {
+ getOrCreate(summary.coordinator, summary.peer).consumeSummaries(summary.sendingSummaries, stat.numberOfDifferences);
+ getOrCreate(summary.peer, summary.coordinator).consumeSummaries(summary.receivingSummaries, stat.numberOfDifferences);
+ }
+ }
+
+ void consumeStats(List<SyncStat> stats)
+ {
+ filter(stats, s -> s.summaries != null).forEach(this::consumeStat);
+ }
+
+ void calculateTotals()
+ {
+ files = 0;
+ bytes = 0;
+ ranges = 0;
+ for (Session session: sessions.values())
+ {
+ files += session.files;
+ bytes += session.bytes;
+ ranges += session.ranges;
+ }
+ totalsCalculated = true;
+ }
+
+ boolean isCounter()
+ {
+ TableMetadata tmd = Schema.instance.getTableMetadata(keyspace, table);
+ return tmd != null && tmd.isCounter();
+ }
+
+ public String toString()
+ {
+ if (!totalsCalculated)
+ {
+ calculateTotals();
+ }
+ StringBuilder output = new StringBuilder();
+
+ output.append(String.format("%s.%s - %s ranges, %s sstables, %s bytes\n", keyspace, table, ranges, files, FBUtilities.prettyPrintMemory(bytes)));
+ for (Session session: sessions.values())
+ {
+ output.append(" ").append(session.toString()).append('\n');
+ }
+ return output.toString();
+ }
+ }
+
+ private Map<Pair<String, String>, Table> summaries = new HashMap<>();
+ private final boolean isEstimate;
+
+ private int files = -1;
+ private long bytes = -1;
+ private int ranges = -1;
+ private boolean totalsCalculated = false;
+
+ public SyncStatSummary(boolean isEstimate)
+ {
+ this.isEstimate = isEstimate;
+ }
+
+ public void consumeRepairResult(RepairResult result)
+ {
+ Pair<String, String> cf = Pair.create(result.desc.keyspace, result.desc.columnFamily);
+ if (!summaries.containsKey(cf))
+ {
+ summaries.put(cf, new Table(cf.left, cf.right));
+ }
+ summaries.get(cf).consumeStats(result.stats);
+ }
+
+ public void consumeSessionResults(List<RepairSessionResult> results)
+ {
+ if (results != null)
+ {
+ filter(results, Objects::nonNull).forEach(r -> filter(r.repairJobResults, Objects::nonNull).forEach(this::consumeRepairResult));
+ }
+ }
+
+ public boolean isEmpty()
+ {
+ calculateTotals();
+ return files == 0 && bytes == 0 && ranges == 0;
+ }
+
+ private void calculateTotals()
+ {
+ files = 0;
+ bytes = 0;
+ ranges = 0;
+ summaries.values().forEach(Table::calculateTotals);
+ for (Table table: summaries.values())
+ {
+ if (table.isCounter())
+ {
+ continue;
+ }
+ table.calculateTotals();
+ files += table.files;
+ bytes += table.bytes;
+ ranges += table.ranges;
+ }
+ totalsCalculated = true;
+ }
+
+ public String toString()
+ {
+ List<Pair<String, String>> tables = Lists.newArrayList(summaries.keySet());
+ tables.sort((o1, o2) ->
+ {
+ int ks = o1.left.compareTo(o2.left);
+ return ks != 0 ? ks : o1.right.compareTo(o2.right);
+ });
+
+ calculateTotals();
+
+ StringBuilder output = new StringBuilder();
+
+ if (isEstimate)
+ {
+ output.append(String.format("Total estimated streaming: %s ranges, %s sstables, %s bytes\n", ranges, files, FBUtilities.prettyPrintMemory(bytes)));
+ }
+ else
+ {
+ output.append(String.format("Total streaming: %s ranges, %s sstables, %s bytes\n", ranges, files, FBUtilities.prettyPrintMemory(bytes)));
+ }
+
+ for (Pair<String, String> tableName: tables)
+ {
+ Table table = summaries.get(tableName);
+ output.append(table.toString()).append('\n');
+ }
+
+ return output.toString();
+ }
+}
diff --git a/src/java/org/apache/cassandra/repair/messages/AnticompactionRequest.java b/src/java/org/apache/cassandra/repair/messages/AnticompactionRequest.java
deleted file mode 100644
index a29cc87..0000000
--- a/src/java/org/apache/cassandra/repair/messages/AnticompactionRequest.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.repair.messages;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-import java.util.Objects;
-import java.util.UUID;
-
-import org.apache.cassandra.dht.Range;
-import org.apache.cassandra.dht.Token;
-import org.apache.cassandra.io.util.DataInputPlus;
-import org.apache.cassandra.io.util.DataOutputPlus;
-import org.apache.cassandra.net.MessagingService;
-import org.apache.cassandra.utils.UUIDSerializer;
-
-public class AnticompactionRequest extends RepairMessage
-{
- public static MessageSerializer serializer = new AnticompactionRequestSerializer();
- public final UUID parentRepairSession;
- /**
- * Successfully repaired ranges. Does not contain null.
- */
- public final Collection<Range<Token>> successfulRanges;
-
- public AnticompactionRequest(UUID parentRepairSession, Collection<Range<Token>> ranges)
- {
- super(Type.ANTICOMPACTION_REQUEST, null);
- this.parentRepairSession = parentRepairSession;
- this.successfulRanges = ranges;
- }
-
- @Override
- public boolean equals(Object o)
- {
- if (!(o instanceof AnticompactionRequest))
- return false;
- AnticompactionRequest other = (AnticompactionRequest)o;
- return messageType == other.messageType &&
- parentRepairSession.equals(other.parentRepairSession) &&
- successfulRanges.equals(other.successfulRanges);
- }
-
- @Override
- public int hashCode()
- {
- return Objects.hash(messageType, parentRepairSession, successfulRanges);
- }
-
- public static class AnticompactionRequestSerializer implements MessageSerializer<AnticompactionRequest>
- {
- public void serialize(AnticompactionRequest message, DataOutputPlus out, int version) throws IOException
- {
- UUIDSerializer.serializer.serialize(message.parentRepairSession, out, version);
- out.writeInt(message.successfulRanges.size());
- for (Range<Token> r : message.successfulRanges)
- {
- MessagingService.validatePartitioner(r);
- Range.tokenSerializer.serialize(r, out, version);
- }
- }
-
- public AnticompactionRequest deserialize(DataInputPlus in, int version) throws IOException
- {
- UUID parentRepairSession = UUIDSerializer.serializer.deserialize(in, version);
- int rangeCount = in.readInt();
- List<Range<Token>> ranges = new ArrayList<>(rangeCount);
- for (int i = 0; i < rangeCount; i++)
- ranges.add((Range<Token>) Range.tokenSerializer.deserialize(in, MessagingService.globalPartitioner(), version));
- return new AnticompactionRequest(parentRepairSession, ranges);
- }
-
- public long serializedSize(AnticompactionRequest message, int version)
- {
- long size = UUIDSerializer.serializer.serializedSize(message.parentRepairSession, version);
- size += Integer.BYTES; // count of items in successfulRanges
- for (Range<Token> r : message.successfulRanges)
- size += Range.tokenSerializer.serializedSize(r, version);
- return size;
- }
- }
-
- @Override
- public String toString()
- {
- return "AnticompactionRequest{" +
- "parentRepairSession=" + parentRepairSession +
- "} " + super.toString();
- }
-}
diff --git a/src/java/org/apache/cassandra/repair/messages/AsymmetricSyncRequest.java b/src/java/org/apache/cassandra/repair/messages/AsymmetricSyncRequest.java
new file mode 100644
index 0000000..eacc285
--- /dev/null
+++ b/src/java/org/apache/cassandra/repair/messages/AsymmetricSyncRequest.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.repair.messages;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Objects;
+
+import org.apache.cassandra.db.TypeSizes;
+import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.dht.IPartitioner;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.io.IVersionedSerializer;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.repair.RepairJobDesc;
+import org.apache.cassandra.streaming.PreviewKind;
+
+import static org.apache.cassandra.locator.InetAddressAndPort.Serializer.inetAddressAndPortSerializer;
+
+public class AsymmetricSyncRequest extends RepairMessage
+{
+ public final InetAddressAndPort initiator;
+ public final InetAddressAndPort fetchingNode;
+ public final InetAddressAndPort fetchFrom;
+ public final Collection<Range<Token>> ranges;
+ public final PreviewKind previewKind;
+
+ public AsymmetricSyncRequest(RepairJobDesc desc, InetAddressAndPort initiator, InetAddressAndPort fetchingNode, InetAddressAndPort fetchFrom, Collection<Range<Token>> ranges, PreviewKind previewKind)
+ {
+ super(desc);
+ this.initiator = initiator;
+ this.fetchingNode = fetchingNode;
+ this.fetchFrom = fetchFrom;
+ this.ranges = ranges;
+ this.previewKind = previewKind;
+ }
+
+ @Override
+ public boolean equals(Object o)
+ {
+ if (!(o instanceof AsymmetricSyncRequest))
+ return false;
+ AsymmetricSyncRequest req = (AsymmetricSyncRequest)o;
+ return desc.equals(req.desc) &&
+ initiator.equals(req.initiator) &&
+ fetchingNode.equals(req.fetchingNode) &&
+ fetchFrom.equals(req.fetchFrom) &&
+ ranges.equals(req.ranges);
+ }
+
+ @Override
+ public int hashCode()
+ {
+ return Objects.hash(desc, initiator, fetchingNode, fetchFrom, ranges);
+ }
+
+ public static final IVersionedSerializer<AsymmetricSyncRequest> serializer = new IVersionedSerializer<AsymmetricSyncRequest>()
+ {
+ public void serialize(AsymmetricSyncRequest message, DataOutputPlus out, int version) throws IOException
+ {
+ RepairJobDesc.serializer.serialize(message.desc, out, version);
+ inetAddressAndPortSerializer.serialize(message.initiator, out, version);
+ inetAddressAndPortSerializer.serialize(message.fetchingNode, out, version);
+ inetAddressAndPortSerializer.serialize(message.fetchFrom, out, version);
+ out.writeInt(message.ranges.size());
+ for (Range<Token> range : message.ranges)
+ {
+ IPartitioner.validate(range);
+ AbstractBounds.tokenSerializer.serialize(range, out, version);
+ }
+ out.writeInt(message.previewKind.getSerializationVal());
+ }
+
+ public AsymmetricSyncRequest deserialize(DataInputPlus in, int version) throws IOException
+ {
+ RepairJobDesc desc = RepairJobDesc.serializer.deserialize(in, version);
+ InetAddressAndPort owner = inetAddressAndPortSerializer.deserialize(in, version);
+ InetAddressAndPort src = inetAddressAndPortSerializer.deserialize(in, version);
+ InetAddressAndPort dst = inetAddressAndPortSerializer.deserialize(in, version);
+ int rangesCount = in.readInt();
+ List<Range<Token>> ranges = new ArrayList<>(rangesCount);
+ for (int i = 0; i < rangesCount; ++i)
+ ranges.add((Range<Token>) AbstractBounds.tokenSerializer.deserialize(in, IPartitioner.global(), version));
+ PreviewKind previewKind = PreviewKind.deserialize(in.readInt());
+ return new AsymmetricSyncRequest(desc, owner, src, dst, ranges, previewKind);
+ }
+
+ public long serializedSize(AsymmetricSyncRequest message, int version)
+ {
+ long size = RepairJobDesc.serializer.serializedSize(message.desc, version);
+ size += inetAddressAndPortSerializer.serializedSize(message.initiator, version);
+ size += inetAddressAndPortSerializer.serializedSize(message.fetchingNode, version);
+ size += inetAddressAndPortSerializer.serializedSize(message.fetchFrom, version);
+ size += TypeSizes.sizeof(message.ranges.size());
+ for (Range<Token> range : message.ranges)
+ size += AbstractBounds.tokenSerializer.serializedSize(range, version);
+ size += TypeSizes.sizeof(message.previewKind.getSerializationVal());
+ return size;
+ }
+ };
+
+ public String toString()
+ {
+ return "AsymmetricSyncRequest{" +
+ "initiator=" + initiator +
+ ", fetchingNode=" + fetchingNode +
+ ", fetchFrom=" + fetchFrom +
+ ", ranges=" + ranges +
+ ", previewKind=" + previewKind +
+ ", desc="+desc+
+ '}';
+ }
+}
diff --git a/src/java/org/apache/cassandra/repair/messages/CleanupMessage.java b/src/java/org/apache/cassandra/repair/messages/CleanupMessage.java
index 69d147a..5ec7fc6 100644
--- a/src/java/org/apache/cassandra/repair/messages/CleanupMessage.java
+++ b/src/java/org/apache/cassandra/repair/messages/CleanupMessage.java
@@ -21,6 +21,7 @@
import java.util.Objects;
import java.util.UUID;
+import org.apache.cassandra.io.IVersionedSerializer;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.utils.UUIDSerializer;
@@ -32,12 +33,11 @@
*/
public class CleanupMessage extends RepairMessage
{
- public static MessageSerializer serializer = new CleanupMessageSerializer();
public final UUID parentRepairSession;
public CleanupMessage(UUID parentRepairSession)
{
- super(Type.CLEANUP, null);
+ super(null);
this.parentRepairSession = parentRepairSession;
}
@@ -47,17 +47,16 @@
if (!(o instanceof CleanupMessage))
return false;
CleanupMessage other = (CleanupMessage) o;
- return messageType == other.messageType &&
- parentRepairSession.equals(other.parentRepairSession);
+ return parentRepairSession.equals(other.parentRepairSession);
}
@Override
public int hashCode()
{
- return Objects.hash(messageType, parentRepairSession);
+ return Objects.hash(parentRepairSession);
}
- public static class CleanupMessageSerializer implements MessageSerializer<CleanupMessage>
+ public static final IVersionedSerializer<CleanupMessage> serializer = new IVersionedSerializer<CleanupMessage>()
{
public void serialize(CleanupMessage message, DataOutputPlus out, int version) throws IOException
{
@@ -74,5 +73,5 @@
{
return UUIDSerializer.serializer.serializedSize(message.parentRepairSession, version);
}
- }
+ };
}
diff --git a/src/java/org/apache/cassandra/repair/messages/FailSession.java b/src/java/org/apache/cassandra/repair/messages/FailSession.java
new file mode 100644
index 0000000..b8c7ad3
--- /dev/null
+++ b/src/java/org/apache/cassandra/repair/messages/FailSession.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.repair.messages;
+
+import java.io.IOException;
+import java.util.UUID;
+
+import org.apache.cassandra.io.IVersionedSerializer;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.utils.UUIDSerializer;
+
+public class FailSession extends RepairMessage
+{
+ public final UUID sessionID;
+
+ public FailSession(UUID sessionID)
+ {
+ super(null);
+ assert sessionID != null;
+ this.sessionID = sessionID;
+ }
+
+ public boolean equals(Object o)
+ {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ FailSession that = (FailSession) o;
+
+ return sessionID.equals(that.sessionID);
+ }
+
+ public int hashCode()
+ {
+ return sessionID.hashCode();
+ }
+
+ public static final IVersionedSerializer<FailSession> serializer = new IVersionedSerializer<FailSession>()
+ {
+ public void serialize(FailSession msg, DataOutputPlus out, int version) throws IOException
+ {
+ UUIDSerializer.serializer.serialize(msg.sessionID, out, version);
+ }
+
+ public FailSession deserialize(DataInputPlus in, int version) throws IOException
+ {
+ return new FailSession(UUIDSerializer.serializer.deserialize(in, version));
+ }
+
+ public long serializedSize(FailSession msg, int version)
+ {
+ return UUIDSerializer.serializer.serializedSize(msg.sessionID, version);
+ }
+ };
+}
diff --git a/src/java/org/apache/cassandra/repair/messages/FinalizeCommit.java b/src/java/org/apache/cassandra/repair/messages/FinalizeCommit.java
new file mode 100644
index 0000000..bb5cca7
--- /dev/null
+++ b/src/java/org/apache/cassandra/repair/messages/FinalizeCommit.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.repair.messages;
+
+import java.io.IOException;
+import java.util.UUID;
+
+import org.apache.cassandra.io.IVersionedSerializer;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.utils.UUIDSerializer;
+
+public class FinalizeCommit extends RepairMessage
+{
+ public final UUID sessionID;
+
+ public FinalizeCommit(UUID sessionID)
+ {
+ super(null);
+ assert sessionID != null;
+ this.sessionID = sessionID;
+ }
+
+ public boolean equals(Object o)
+ {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ FinalizeCommit that = (FinalizeCommit) o;
+
+ return sessionID.equals(that.sessionID);
+ }
+
+ public int hashCode()
+ {
+ return sessionID.hashCode();
+ }
+
+ public String toString()
+ {
+ return "FinalizeCommit{" +
+ "sessionID=" + sessionID +
+ '}';
+ }
+
+ public static final IVersionedSerializer<FinalizeCommit> serializer = new IVersionedSerializer<FinalizeCommit>()
+ {
+ public void serialize(FinalizeCommit msg, DataOutputPlus out, int version) throws IOException
+ {
+ UUIDSerializer.serializer.serialize(msg.sessionID, out, version);
+ }
+
+ public FinalizeCommit deserialize(DataInputPlus in, int version) throws IOException
+ {
+ return new FinalizeCommit(UUIDSerializer.serializer.deserialize(in, version));
+ }
+
+ public long serializedSize(FinalizeCommit msg, int version)
+ {
+ return UUIDSerializer.serializer.serializedSize(msg.sessionID, version);
+ }
+ };
+}
diff --git a/src/java/org/apache/cassandra/repair/messages/FinalizePromise.java b/src/java/org/apache/cassandra/repair/messages/FinalizePromise.java
new file mode 100644
index 0000000..cfdc07c
--- /dev/null
+++ b/src/java/org/apache/cassandra/repair/messages/FinalizePromise.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.repair.messages;
+
+import java.io.IOException;
+import java.util.UUID;
+
+import org.apache.cassandra.db.TypeSizes;
+import org.apache.cassandra.io.IVersionedSerializer;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.utils.UUIDSerializer;
+
+import static org.apache.cassandra.locator.InetAddressAndPort.Serializer.inetAddressAndPortSerializer;
+
+public class FinalizePromise extends RepairMessage
+{
+ public final UUID sessionID;
+ public final InetAddressAndPort participant;
+ public final boolean promised;
+
+ public FinalizePromise(UUID sessionID, InetAddressAndPort participant, boolean promised)
+ {
+ super(null);
+ assert sessionID != null;
+ assert participant != null;
+ this.sessionID = sessionID;
+ this.participant = participant;
+ this.promised = promised;
+ }
+
+ public boolean equals(Object o)
+ {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ FinalizePromise that = (FinalizePromise) o;
+
+ if (promised != that.promised) return false;
+ if (!sessionID.equals(that.sessionID)) return false;
+ return participant.equals(that.participant);
+ }
+
+ public int hashCode()
+ {
+ int result = sessionID.hashCode();
+ result = 31 * result + participant.hashCode();
+ result = 31 * result + (promised ? 1 : 0);
+ return result;
+ }
+
+ public static final IVersionedSerializer<FinalizePromise> serializer = new IVersionedSerializer<FinalizePromise>()
+ {
+ public void serialize(FinalizePromise msg, DataOutputPlus out, int version) throws IOException
+ {
+ UUIDSerializer.serializer.serialize(msg.sessionID, out, version);
+ inetAddressAndPortSerializer.serialize(msg.participant, out, version);
+ out.writeBoolean(msg.promised);
+ }
+
+ public FinalizePromise deserialize(DataInputPlus in, int version) throws IOException
+ {
+ return new FinalizePromise(UUIDSerializer.serializer.deserialize(in, version),
+ inetAddressAndPortSerializer.deserialize(in, version),
+ in.readBoolean());
+ }
+
+ public long serializedSize(FinalizePromise msg, int version)
+ {
+ long size = UUIDSerializer.serializer.serializedSize(msg.sessionID, version);
+ size += inetAddressAndPortSerializer.serializedSize(msg.participant, version);
+ size += TypeSizes.sizeof(msg.promised);
+ return size;
+ }
+ };
+}
diff --git a/src/java/org/apache/cassandra/repair/messages/FinalizePropose.java b/src/java/org/apache/cassandra/repair/messages/FinalizePropose.java
new file mode 100644
index 0000000..c21dd78
--- /dev/null
+++ b/src/java/org/apache/cassandra/repair/messages/FinalizePropose.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.repair.messages;
+
+import java.io.IOException;
+import java.util.UUID;
+
+import org.apache.cassandra.io.IVersionedSerializer;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.utils.UUIDSerializer;
+
+public class FinalizePropose extends RepairMessage
+{
+ public final UUID sessionID;
+
+ public FinalizePropose(UUID sessionID)
+ {
+ super(null);
+ assert sessionID != null;
+ this.sessionID = sessionID;
+ }
+
+ public boolean equals(Object o)
+ {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ FinalizePropose that = (FinalizePropose) o;
+
+ return sessionID.equals(that.sessionID);
+ }
+
+ public int hashCode()
+ {
+ return sessionID.hashCode();
+ }
+
+ public String toString()
+ {
+ return "FinalizePropose{" +
+ "sessionID=" + sessionID +
+ '}';
+ }
+
+ public static final IVersionedSerializer<FinalizePropose> serializer = new IVersionedSerializer<FinalizePropose>()
+ {
+ public void serialize(FinalizePropose msg, DataOutputPlus out, int version) throws IOException
+ {
+ UUIDSerializer.serializer.serialize(msg.sessionID, out, version);
+ }
+
+ public FinalizePropose deserialize(DataInputPlus in, int version) throws IOException
+ {
+ return new FinalizePropose(UUIDSerializer.serializer.deserialize(in, version));
+ }
+
+ public long serializedSize(FinalizePropose msg, int version)
+ {
+ return UUIDSerializer.serializer.serializedSize(msg.sessionID, version);
+ }
+ };
+}
diff --git a/src/java/org/apache/cassandra/repair/messages/PrepareConsistentRequest.java b/src/java/org/apache/cassandra/repair/messages/PrepareConsistentRequest.java
new file mode 100644
index 0000000..c1be082
--- /dev/null
+++ b/src/java/org/apache/cassandra/repair/messages/PrepareConsistentRequest.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.repair.messages;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.UUID;
+
+import com.google.common.collect.ImmutableSet;
+
+import org.apache.cassandra.db.TypeSizes;
+import org.apache.cassandra.io.IVersionedSerializer;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.utils.UUIDSerializer;
+
+import static org.apache.cassandra.locator.InetAddressAndPort.Serializer.inetAddressAndPortSerializer;
+
+public class PrepareConsistentRequest extends RepairMessage
+{
+ public final UUID parentSession;
+ public final InetAddressAndPort coordinator;
+ public final Set<InetAddressAndPort> participants;
+
+ public PrepareConsistentRequest(UUID parentSession, InetAddressAndPort coordinator, Set<InetAddressAndPort> participants)
+ {
+ super(null);
+ assert parentSession != null;
+ assert coordinator != null;
+ assert participants != null && !participants.isEmpty();
+ this.parentSession = parentSession;
+ this.coordinator = coordinator;
+ this.participants = ImmutableSet.copyOf(participants);
+ }
+
+ public boolean equals(Object o)
+ {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ PrepareConsistentRequest that = (PrepareConsistentRequest) o;
+
+ if (!parentSession.equals(that.parentSession)) return false;
+ if (!coordinator.equals(that.coordinator)) return false;
+ return participants.equals(that.participants);
+ }
+
+ public int hashCode()
+ {
+ int result = parentSession.hashCode();
+ result = 31 * result + coordinator.hashCode();
+ result = 31 * result + participants.hashCode();
+ return result;
+ }
+
+ public String toString()
+ {
+ return "PrepareConsistentRequest{" +
+ "parentSession=" + parentSession +
+ ", coordinator=" + coordinator +
+ ", participants=" + participants +
+ '}';
+ }
+
+ public static final IVersionedSerializer<PrepareConsistentRequest> serializer = new IVersionedSerializer<PrepareConsistentRequest>()
+ {
+ public void serialize(PrepareConsistentRequest request, DataOutputPlus out, int version) throws IOException
+ {
+ UUIDSerializer.serializer.serialize(request.parentSession, out, version);
+ inetAddressAndPortSerializer.serialize(request.coordinator, out, version);
+ out.writeInt(request.participants.size());
+ for (InetAddressAndPort peer : request.participants)
+ {
+ inetAddressAndPortSerializer.serialize(peer, out, version);
+ }
+ }
+
+ public PrepareConsistentRequest deserialize(DataInputPlus in, int version) throws IOException
+ {
+ UUID sessionId = UUIDSerializer.serializer.deserialize(in, version);
+ InetAddressAndPort coordinator = inetAddressAndPortSerializer.deserialize(in, version);
+ int numPeers = in.readInt();
+ Set<InetAddressAndPort> peers = new HashSet<>(numPeers);
+ for (int i = 0; i < numPeers; i++)
+ {
+ InetAddressAndPort peer = inetAddressAndPortSerializer.deserialize(in, version);
+ peers.add(peer);
+ }
+ return new PrepareConsistentRequest(sessionId, coordinator, peers);
+ }
+
+ public long serializedSize(PrepareConsistentRequest request, int version)
+ {
+ long size = UUIDSerializer.serializer.serializedSize(request.parentSession, version);
+ size += inetAddressAndPortSerializer.serializedSize(request.coordinator, version);
+ size += TypeSizes.sizeof(request.participants.size());
+ for (InetAddressAndPort peer : request.participants)
+ {
+ size += inetAddressAndPortSerializer.serializedSize(peer, version);
+ }
+ return size;
+ }
+ };
+}
diff --git a/src/java/org/apache/cassandra/repair/messages/PrepareConsistentResponse.java b/src/java/org/apache/cassandra/repair/messages/PrepareConsistentResponse.java
new file mode 100644
index 0000000..00de77d
--- /dev/null
+++ b/src/java/org/apache/cassandra/repair/messages/PrepareConsistentResponse.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.repair.messages;
+
+import java.io.IOException;
+import java.util.UUID;
+
+import org.apache.cassandra.db.TypeSizes;
+import org.apache.cassandra.io.IVersionedSerializer;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.utils.UUIDSerializer;
+
+import static org.apache.cassandra.locator.InetAddressAndPort.Serializer.inetAddressAndPortSerializer;
+
+public class PrepareConsistentResponse extends RepairMessage
+{
+ public final UUID parentSession;
+ public final InetAddressAndPort participant;
+ public final boolean success;
+
+ public PrepareConsistentResponse(UUID parentSession, InetAddressAndPort participant, boolean success)
+ {
+ super(null);
+ assert parentSession != null;
+ assert participant != null;
+ this.parentSession = parentSession;
+ this.participant = participant;
+ this.success = success;
+ }
+
+ public boolean equals(Object o)
+ {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ PrepareConsistentResponse that = (PrepareConsistentResponse) o;
+
+ if (success != that.success) return false;
+ if (!parentSession.equals(that.parentSession)) return false;
+ return participant.equals(that.participant);
+ }
+
+ public int hashCode()
+ {
+ int result = parentSession.hashCode();
+ result = 31 * result + participant.hashCode();
+ result = 31 * result + (success ? 1 : 0);
+ return result;
+ }
+
+ public static final IVersionedSerializer<PrepareConsistentResponse> serializer = new IVersionedSerializer<PrepareConsistentResponse>()
+ {
+ public void serialize(PrepareConsistentResponse response, DataOutputPlus out, int version) throws IOException
+ {
+ UUIDSerializer.serializer.serialize(response.parentSession, out, version);
+ inetAddressAndPortSerializer.serialize(response.participant, out, version);
+ out.writeBoolean(response.success);
+ }
+
+ public PrepareConsistentResponse deserialize(DataInputPlus in, int version) throws IOException
+ {
+ return new PrepareConsistentResponse(UUIDSerializer.serializer.deserialize(in, version),
+ inetAddressAndPortSerializer.deserialize(in, version),
+ in.readBoolean());
+ }
+
+ public long serializedSize(PrepareConsistentResponse response, int version)
+ {
+ long size = UUIDSerializer.serializer.serializedSize(response.parentSession, version);
+ size += inetAddressAndPortSerializer.serializedSize(response.participant, version);
+ size += TypeSizes.sizeof(response.success);
+ return size;
+ }
+ };
+}
diff --git a/src/java/org/apache/cassandra/repair/messages/PrepareMessage.java b/src/java/org/apache/cassandra/repair/messages/PrepareMessage.java
index b3efeae..9c485bc 100644
--- a/src/java/org/apache/cassandra/repair/messages/PrepareMessage.java
+++ b/src/java/org/apache/cassandra/repair/messages/PrepareMessage.java
@@ -24,35 +24,42 @@
import java.util.Objects;
import java.util.UUID;
+import com.google.common.base.Preconditions;
+
import org.apache.cassandra.db.TypeSizes;
+import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.io.IVersionedSerializer;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.streaming.PreviewKind;
import org.apache.cassandra.utils.UUIDSerializer;
public class PrepareMessage extends RepairMessage
{
- public final static MessageSerializer serializer = new PrepareMessageSerializer();
- public final List<UUID> cfIds;
+ public final List<TableId> tableIds;
public final Collection<Range<Token>> ranges;
public final UUID parentRepairSession;
public final boolean isIncremental;
public final long timestamp;
public final boolean isGlobal;
+ public final PreviewKind previewKind;
- public PrepareMessage(UUID parentRepairSession, List<UUID> cfIds, Collection<Range<Token>> ranges, boolean isIncremental, long timestamp, boolean isGlobal)
+ public PrepareMessage(UUID parentRepairSession, List<TableId> tableIds, Collection<Range<Token>> ranges, boolean isIncremental, long timestamp, boolean isGlobal, PreviewKind previewKind)
{
- super(Type.PREPARE_MESSAGE, null);
+ super(null);
this.parentRepairSession = parentRepairSession;
- this.cfIds = cfIds;
+ this.tableIds = tableIds;
this.ranges = ranges;
this.isIncremental = isIncremental;
this.timestamp = timestamp;
this.isGlobal = isGlobal;
+ this.previewKind = previewKind;
}
@Override
@@ -61,63 +68,74 @@
if (!(o instanceof PrepareMessage))
return false;
PrepareMessage other = (PrepareMessage) o;
- return messageType == other.messageType &&
- parentRepairSession.equals(other.parentRepairSession) &&
+ return parentRepairSession.equals(other.parentRepairSession) &&
isIncremental == other.isIncremental &&
isGlobal == other.isGlobal &&
+ previewKind == other.previewKind &&
timestamp == other.timestamp &&
- cfIds.equals(other.cfIds) &&
+ tableIds.equals(other.tableIds) &&
ranges.equals(other.ranges);
}
@Override
public int hashCode()
{
- return Objects.hash(messageType, parentRepairSession, isGlobal, isIncremental, timestamp, cfIds, ranges);
+ return Objects.hash(parentRepairSession, isGlobal, previewKind, isIncremental, timestamp, tableIds, ranges);
}
- public static class PrepareMessageSerializer implements MessageSerializer<PrepareMessage>
+ private static final String MIXED_MODE_ERROR = "Some nodes involved in repair are on an incompatible major version. " +
+ "Repair is not supported in mixed major version clusters.";
+
+ public static final IVersionedSerializer<PrepareMessage> serializer = new IVersionedSerializer<PrepareMessage>()
{
public void serialize(PrepareMessage message, DataOutputPlus out, int version) throws IOException
{
- out.writeInt(message.cfIds.size());
- for (UUID cfId : message.cfIds)
- UUIDSerializer.serializer.serialize(cfId, out, version);
+ Preconditions.checkArgument(version == MessagingService.current_version, MIXED_MODE_ERROR);
+
+ out.writeInt(message.tableIds.size());
+ for (TableId tableId : message.tableIds)
+ tableId.serialize(out);
UUIDSerializer.serializer.serialize(message.parentRepairSession, out, version);
out.writeInt(message.ranges.size());
for (Range<Token> r : message.ranges)
{
- MessagingService.validatePartitioner(r);
+ IPartitioner.validate(r);
Range.tokenSerializer.serialize(r, out, version);
}
out.writeBoolean(message.isIncremental);
out.writeLong(message.timestamp);
out.writeBoolean(message.isGlobal);
+ out.writeInt(message.previewKind.getSerializationVal());
}
public PrepareMessage deserialize(DataInputPlus in, int version) throws IOException
{
- int cfIdCount = in.readInt();
- List<UUID> cfIds = new ArrayList<>(cfIdCount);
- for (int i = 0; i < cfIdCount; i++)
- cfIds.add(UUIDSerializer.serializer.deserialize(in, version));
+ Preconditions.checkArgument(version == MessagingService.current_version, MIXED_MODE_ERROR);
+
+ int tableIdCount = in.readInt();
+ List<TableId> tableIds = new ArrayList<>(tableIdCount);
+ for (int i = 0; i < tableIdCount; i++)
+ tableIds.add(TableId.deserialize(in));
UUID parentRepairSession = UUIDSerializer.serializer.deserialize(in, version);
int rangeCount = in.readInt();
List<Range<Token>> ranges = new ArrayList<>(rangeCount);
for (int i = 0; i < rangeCount; i++)
- ranges.add((Range<Token>) Range.tokenSerializer.deserialize(in, MessagingService.globalPartitioner(), version));
+ ranges.add((Range<Token>) Range.tokenSerializer.deserialize(in, IPartitioner.global(), version));
boolean isIncremental = in.readBoolean();
long timestamp = in.readLong();
boolean isGlobal = in.readBoolean();
- return new PrepareMessage(parentRepairSession, cfIds, ranges, isIncremental, timestamp, isGlobal);
+ PreviewKind previewKind = PreviewKind.deserialize(in.readInt());
+ return new PrepareMessage(parentRepairSession, tableIds, ranges, isIncremental, timestamp, isGlobal, previewKind);
}
public long serializedSize(PrepareMessage message, int version)
{
+ Preconditions.checkArgument(version == MessagingService.current_version, MIXED_MODE_ERROR);
+
long size;
- size = TypeSizes.sizeof(message.cfIds.size());
- for (UUID cfId : message.cfIds)
- size += UUIDSerializer.serializer.serializedSize(cfId, version);
+ size = TypeSizes.sizeof(message.tableIds.size());
+ for (TableId tableId : message.tableIds)
+ size += tableId.serializedSize();
size += UUIDSerializer.serializer.serializedSize(message.parentRepairSession, version);
size += TypeSizes.sizeof(message.ranges.size());
for (Range<Token> r : message.ranges)
@@ -125,20 +143,21 @@
size += TypeSizes.sizeof(message.isIncremental);
size += TypeSizes.sizeof(message.timestamp);
size += TypeSizes.sizeof(message.isGlobal);
+ size += TypeSizes.sizeof(message.previewKind.getSerializationVal());
return size;
}
- }
+ };
@Override
public String toString()
{
return "PrepareMessage{" +
- "cfIds='" + cfIds + '\'' +
- ", ranges=" + ranges +
- ", parentRepairSession=" + parentRepairSession +
- ", isIncremental="+isIncremental +
- ", timestamp=" + timestamp +
- ", isGlobal=" + isGlobal +
- '}';
+ "tableIds='" + tableIds + '\'' +
+ ", ranges=" + ranges +
+ ", parentRepairSession=" + parentRepairSession +
+ ", isIncremental=" + isIncremental +
+ ", timestamp=" + timestamp +
+ ", isGlobal=" + isGlobal +
+ '}';
}
}
diff --git a/src/java/org/apache/cassandra/repair/messages/RepairMessage.java b/src/java/org/apache/cassandra/repair/messages/RepairMessage.java
index 55fdb66..3137b4e 100644
--- a/src/java/org/apache/cassandra/repair/messages/RepairMessage.java
+++ b/src/java/org/apache/cassandra/repair/messages/RepairMessage.java
@@ -17,13 +17,6 @@
*/
package org.apache.cassandra.repair.messages;
-import java.io.IOException;
-
-import org.apache.cassandra.io.IVersionedSerializer;
-import org.apache.cassandra.io.util.DataInputPlus;
-import org.apache.cassandra.io.util.DataOutputPlus;
-import org.apache.cassandra.net.MessageOut;
-import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.repair.RepairJobDesc;
/**
@@ -33,74 +26,10 @@
*/
public abstract class RepairMessage
{
- public static final IVersionedSerializer<RepairMessage> serializer = new RepairMessageSerializer();
-
- public static interface MessageSerializer<T extends RepairMessage> extends IVersionedSerializer<T> {}
-
- public static enum Type
- {
- VALIDATION_REQUEST(0, ValidationRequest.serializer),
- VALIDATION_COMPLETE(1, ValidationComplete.serializer),
- SYNC_REQUEST(2, SyncRequest.serializer),
- SYNC_COMPLETE(3, SyncComplete.serializer),
- ANTICOMPACTION_REQUEST(4, AnticompactionRequest.serializer),
- PREPARE_MESSAGE(5, PrepareMessage.serializer),
- SNAPSHOT(6, SnapshotMessage.serializer),
- CLEANUP(7, CleanupMessage.serializer);
-
- private final byte type;
- private final MessageSerializer<RepairMessage> serializer;
-
- private Type(int type, MessageSerializer<RepairMessage> serializer)
- {
- this.type = (byte) type;
- this.serializer = serializer;
- }
-
- public static Type fromByte(byte b)
- {
- for (Type t : values())
- {
- if (t.type == b)
- return t;
- }
- throw new IllegalArgumentException("Unknown RepairMessage.Type: " + b);
- }
- }
-
- public final Type messageType;
public final RepairJobDesc desc;
- protected RepairMessage(Type messageType, RepairJobDesc desc)
+ protected RepairMessage(RepairJobDesc desc)
{
- this.messageType = messageType;
this.desc = desc;
}
-
- public MessageOut<RepairMessage> createMessage()
- {
- return new MessageOut<>(MessagingService.Verb.REPAIR_MESSAGE, this, RepairMessage.serializer);
- }
-
- public static class RepairMessageSerializer implements MessageSerializer<RepairMessage>
- {
- public void serialize(RepairMessage message, DataOutputPlus out, int version) throws IOException
- {
- out.write(message.messageType.type);
- message.messageType.serializer.serialize(message, out, version);
- }
-
- public RepairMessage deserialize(DataInputPlus in, int version) throws IOException
- {
- RepairMessage.Type messageType = RepairMessage.Type.fromByte(in.readByte());
- return messageType.serializer.deserialize(in, version);
- }
-
- public long serializedSize(RepairMessage message, int version)
- {
- long size = 1; // for messageType byte
- size += message.messageType.serializer.serializedSize(message, version);
- return size;
- }
- }
}
diff --git a/src/java/org/apache/cassandra/repair/messages/RepairOption.java b/src/java/org/apache/cassandra/repair/messages/RepairOption.java
index 146ab64..adcd776 100644
--- a/src/java/org/apache/cassandra/repair/messages/RepairOption.java
+++ b/src/java/org/apache/cassandra/repair/messages/RepairOption.java
@@ -28,6 +28,7 @@
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.streaming.PreviewKind;
import org.apache.cassandra.repair.RepairParallelism;
import org.apache.cassandra.utils.FBUtilities;
@@ -47,6 +48,9 @@
public static final String TRACE_KEY = "trace";
public static final String SUB_RANGE_REPAIR_KEY = "sub_range_repair";
public static final String PULL_REPAIR_KEY = "pullRepair";
+ public static final String FORCE_REPAIR_KEY = "forceRepair";
+ public static final String PREVIEW = "previewKind";
+ public static final String OPTIMISE_STREAMS_KEY = "optimiseStreams";
// we don't want to push nodes too much for repair
public static final int MAX_JOB_THREADS = 4;
@@ -123,6 +127,17 @@
* This is only allowed if exactly 2 hosts are specified along with a token range that they share.</td>
* <td>false</td>
* </tr>
+ * <tr>
+ * <td>forceRepair</td>
+ * <td>"true" if the repair should continue, even if one of the replicas involved is down.
+ * <td>false</td>
+ * </tr>
+ * <tr>
+ * <td>optimiseStreams</td>
+ * <td>"true" if we should try to optimise the syncing to avoid transfering identical
+ * ranges to the same host multiple times</td>
+ * <td>false</td>
+ * </tr>
* </tbody>
* </table>
*
@@ -136,7 +151,9 @@
RepairParallelism parallelism = RepairParallelism.fromName(options.get(PARALLELISM_KEY));
boolean primaryRange = Boolean.parseBoolean(options.get(PRIMARY_RANGE_KEY));
boolean incremental = Boolean.parseBoolean(options.get(INCREMENTAL_KEY));
+ PreviewKind previewKind = PreviewKind.valueOf(options.getOrDefault(PREVIEW, PreviewKind.NONE.toString()));
boolean trace = Boolean.parseBoolean(options.get(TRACE_KEY));
+ boolean force = Boolean.parseBoolean(options.get(FORCE_REPAIR_KEY));
boolean pullRepair = Boolean.parseBoolean(options.get(PULL_REPAIR_KEY));
int jobThreads = 1;
@@ -153,10 +170,6 @@
Set<Range<Token>> ranges = new HashSet<>();
if (rangesStr != null)
{
- if (incremental)
- logger.warn("Incremental repair can't be requested with subrange repair " +
- "because each subrange repair would generate an anti-compacted table. " +
- "The repair will occur but without anti-compaction.");
StringTokenizer tokenizer = new StringTokenizer(rangesStr, ",");
while (tokenizer.hasMoreTokens())
{
@@ -174,8 +187,9 @@
ranges.add(new Range<>(parsedBeginToken, parsedEndToken));
}
}
+ boolean asymmetricSyncing = Boolean.parseBoolean(options.get(OPTIMISE_STREAMS_KEY));
- RepairOption option = new RepairOption(parallelism, primaryRange, incremental, trace, jobThreads, ranges, !ranges.isEmpty(), pullRepair);
+ RepairOption option = new RepairOption(parallelism, primaryRange, incremental, trace, jobThreads, ranges, !ranges.isEmpty(), pullRepair, force, previewKind, asymmetricSyncing);
// data centers
String dataCentersStr = options.get(DATACENTERS_KEY);
@@ -251,13 +265,16 @@
private final int jobThreads;
private final boolean isSubrangeRepair;
private final boolean pullRepair;
+ private final boolean forceRepair;
+ private final PreviewKind previewKind;
+ private final boolean optimiseStreams;
private final Collection<String> columnFamilies = new HashSet<>();
private final Collection<String> dataCenters = new HashSet<>();
private final Collection<String> hosts = new HashSet<>();
private final Collection<Range<Token>> ranges = new HashSet<>();
- public RepairOption(RepairParallelism parallelism, boolean primaryRange, boolean incremental, boolean trace, int jobThreads, Collection<Range<Token>> ranges, boolean isSubrangeRepair, boolean pullRepair)
+ public RepairOption(RepairParallelism parallelism, boolean primaryRange, boolean incremental, boolean trace, int jobThreads, Collection<Range<Token>> ranges, boolean isSubrangeRepair, boolean pullRepair, boolean forceRepair, PreviewKind previewKind, boolean optimiseStreams)
{
if (FBUtilities.isWindows &&
(DatabaseDescriptor.getDiskAccessMode() != Config.DiskAccessMode.standard || DatabaseDescriptor.getIndexAccessMode() != Config.DiskAccessMode.standard) &&
@@ -276,6 +293,9 @@
this.ranges.addAll(ranges);
this.isSubrangeRepair = isSubrangeRepair;
this.pullRepair = pullRepair;
+ this.forceRepair = forceRepair;
+ this.previewKind = previewKind;
+ this.optimiseStreams = optimiseStreams;
}
public RepairParallelism getParallelism()
@@ -303,6 +323,11 @@
return pullRepair;
}
+ public boolean isForcedRepair()
+ {
+ return forceRepair;
+ }
+
public int getJobThreads()
{
return jobThreads;
@@ -330,7 +355,7 @@
public boolean isGlobal()
{
- return dataCenters.isEmpty() && hosts.isEmpty() && !isSubrangeRepair();
+ return dataCenters.isEmpty() && hosts.isEmpty();
}
public boolean isSubrangeRepair()
@@ -338,24 +363,43 @@
return isSubrangeRepair;
}
- public boolean isInLocalDCOnly() {
+ public PreviewKind getPreviewKind()
+ {
+ return previewKind;
+ }
+
+ public boolean isPreview()
+ {
+ return previewKind.isPreview();
+ }
+
+ public boolean isInLocalDCOnly()
+ {
return dataCenters.size() == 1 && dataCenters.contains(DatabaseDescriptor.getLocalDataCenter());
}
+ public boolean optimiseStreams()
+ {
+ return optimiseStreams;
+ }
+
@Override
public String toString()
{
return "repair options (" +
- "parallelism: " + parallelism +
- ", primary range: " + primaryRange +
- ", incremental: " + incremental +
- ", job threads: " + jobThreads +
- ", ColumnFamilies: " + columnFamilies +
- ", dataCenters: " + dataCenters +
- ", hosts: " + hosts +
- ", # of ranges: " + ranges.size() +
- ", pull repair: " + pullRepair +
- ')';
+ "parallelism: " + parallelism +
+ ", primary range: " + primaryRange +
+ ", incremental: " + incremental +
+ ", job threads: " + jobThreads +
+ ", ColumnFamilies: " + columnFamilies +
+ ", dataCenters: " + dataCenters +
+ ", hosts: " + hosts +
+ ", previewKind: " + previewKind +
+ ", # of ranges: " + ranges.size() +
+ ", pull repair: " + pullRepair +
+ ", force repair: " + forceRepair +
+ ", optimise streams: "+ optimiseStreams +
+ ')';
}
public Map<String, String> asMap()
@@ -372,6 +416,9 @@
options.put(TRACE_KEY, Boolean.toString(trace));
options.put(RANGES_KEY, Joiner.on(",").join(ranges));
options.put(PULL_REPAIR_KEY, Boolean.toString(pullRepair));
+ options.put(FORCE_REPAIR_KEY, Boolean.toString(forceRepair));
+ options.put(PREVIEW, previewKind.toString());
+ options.put(OPTIMISE_STREAMS_KEY, Boolean.toString(optimiseStreams));
return options;
}
}
diff --git a/src/java/org/apache/cassandra/repair/messages/SnapshotMessage.java b/src/java/org/apache/cassandra/repair/messages/SnapshotMessage.java
index d4737d3..c18950a 100644
--- a/src/java/org/apache/cassandra/repair/messages/SnapshotMessage.java
+++ b/src/java/org/apache/cassandra/repair/messages/SnapshotMessage.java
@@ -20,17 +20,16 @@
import java.io.IOException;
import java.util.Objects;
+import org.apache.cassandra.io.IVersionedSerializer;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.repair.RepairJobDesc;
public class SnapshotMessage extends RepairMessage
{
- public final static MessageSerializer serializer = new SnapshotMessageSerializer();
-
public SnapshotMessage(RepairJobDesc desc)
{
- super(Type.SNAPSHOT, desc);
+ super(desc);
}
@Override
@@ -39,16 +38,16 @@
if (!(o instanceof SnapshotMessage))
return false;
SnapshotMessage other = (SnapshotMessage) o;
- return messageType == other.messageType;
+ return desc.equals(other.desc);
}
@Override
public int hashCode()
{
- return Objects.hash(messageType);
+ return Objects.hash(desc);
}
- public static class SnapshotMessageSerializer implements MessageSerializer<SnapshotMessage>
+ public static final IVersionedSerializer<SnapshotMessage> serializer = new IVersionedSerializer<SnapshotMessage>()
{
public void serialize(SnapshotMessage message, DataOutputPlus out, int version) throws IOException
{
@@ -65,5 +64,5 @@
{
return RepairJobDesc.serializer.serializedSize(message.desc, version);
}
- }
+ };
}
diff --git a/src/java/org/apache/cassandra/repair/messages/StatusRequest.java b/src/java/org/apache/cassandra/repair/messages/StatusRequest.java
new file mode 100644
index 0000000..09354e6
--- /dev/null
+++ b/src/java/org/apache/cassandra/repair/messages/StatusRequest.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.repair.messages;
+
+import java.io.IOException;
+import java.util.UUID;
+
+import org.apache.cassandra.io.IVersionedSerializer;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.utils.UUIDSerializer;
+
+public class StatusRequest extends RepairMessage
+{
+ public final UUID sessionID;
+
+ public StatusRequest(UUID sessionID)
+ {
+ super(null);
+ this.sessionID = sessionID;
+ }
+
+ public boolean equals(Object o)
+ {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ StatusRequest request = (StatusRequest) o;
+
+ return sessionID.equals(request.sessionID);
+ }
+
+ public int hashCode()
+ {
+ return sessionID.hashCode();
+ }
+
+ public String toString()
+ {
+ return "StatusRequest{" +
+ "sessionID=" + sessionID +
+ '}';
+ }
+
+ public static final IVersionedSerializer<StatusRequest> serializer = new IVersionedSerializer<StatusRequest>()
+ {
+ public void serialize(StatusRequest msg, DataOutputPlus out, int version) throws IOException
+ {
+ UUIDSerializer.serializer.serialize(msg.sessionID, out, version);
+ }
+
+ public StatusRequest deserialize(DataInputPlus in, int version) throws IOException
+ {
+ return new StatusRequest(UUIDSerializer.serializer.deserialize(in, version));
+ }
+
+ public long serializedSize(StatusRequest msg, int version)
+ {
+ return UUIDSerializer.serializer.serializedSize(msg.sessionID, version);
+ }
+ };
+}
diff --git a/src/java/org/apache/cassandra/repair/messages/StatusResponse.java b/src/java/org/apache/cassandra/repair/messages/StatusResponse.java
new file mode 100644
index 0000000..e62d337
--- /dev/null
+++ b/src/java/org/apache/cassandra/repair/messages/StatusResponse.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.repair.messages;
+
+import java.io.IOException;
+import java.util.UUID;
+
+import org.apache.cassandra.db.TypeSizes;
+import org.apache.cassandra.io.IVersionedSerializer;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.repair.consistent.ConsistentSession;
+import org.apache.cassandra.utils.UUIDSerializer;
+
+public class StatusResponse extends RepairMessage
+{
+ public final UUID sessionID;
+ public final ConsistentSession.State state;
+
+ public StatusResponse(UUID sessionID, ConsistentSession.State state)
+ {
+ super(null);
+ assert sessionID != null;
+ assert state != null;
+ this.sessionID = sessionID;
+ this.state = state;
+ }
+
+ public boolean equals(Object o)
+ {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ StatusResponse that = (StatusResponse) o;
+
+ if (!sessionID.equals(that.sessionID)) return false;
+ return state == that.state;
+ }
+
+ public int hashCode()
+ {
+ int result = sessionID.hashCode();
+ result = 31 * result + state.hashCode();
+ return result;
+ }
+
+ public String toString()
+ {
+ return "StatusResponse{" +
+ "sessionID=" + sessionID +
+ ", state=" + state +
+ '}';
+ }
+
+ public static final IVersionedSerializer<StatusResponse> serializer = new IVersionedSerializer<StatusResponse>()
+ {
+ public void serialize(StatusResponse msg, DataOutputPlus out, int version) throws IOException
+ {
+ UUIDSerializer.serializer.serialize(msg.sessionID, out, version);
+ out.writeInt(msg.state.ordinal());
+ }
+
+ public StatusResponse deserialize(DataInputPlus in, int version) throws IOException
+ {
+ return new StatusResponse(UUIDSerializer.serializer.deserialize(in, version),
+ ConsistentSession.State.valueOf(in.readInt()));
+ }
+
+ public long serializedSize(StatusResponse msg, int version)
+ {
+ return UUIDSerializer.serializer.serializedSize(msg.sessionID, version)
+ + TypeSizes.sizeof(msg.state.ordinal());
+ }
+ };
+}
diff --git a/src/java/org/apache/cassandra/repair/messages/SyncComplete.java b/src/java/org/apache/cassandra/repair/messages/SyncComplete.java
deleted file mode 100644
index 178e710..0000000
--- a/src/java/org/apache/cassandra/repair/messages/SyncComplete.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.repair.messages;
-
-import java.io.IOException;
-import java.net.InetAddress;
-import java.util.Objects;
-
-import org.apache.cassandra.db.TypeSizes;
-import org.apache.cassandra.io.util.DataInputPlus;
-import org.apache.cassandra.io.util.DataOutputPlus;
-import org.apache.cassandra.repair.NodePair;
-import org.apache.cassandra.repair.RepairJobDesc;
-
-/**
- *
- * @since 2.0
- */
-public class SyncComplete extends RepairMessage
-{
- public static final MessageSerializer serializer = new SyncCompleteSerializer();
-
- /** nodes that involved in this sync */
- public final NodePair nodes;
- /** true if sync success, false otherwise */
- public final boolean success;
-
- public SyncComplete(RepairJobDesc desc, NodePair nodes, boolean success)
- {
- super(Type.SYNC_COMPLETE, desc);
- this.nodes = nodes;
- this.success = success;
- }
-
- public SyncComplete(RepairJobDesc desc, InetAddress endpoint1, InetAddress endpoint2, boolean success)
- {
- super(Type.SYNC_COMPLETE, desc);
- this.nodes = new NodePair(endpoint1, endpoint2);
- this.success = success;
- }
-
- @Override
- public boolean equals(Object o)
- {
- if (!(o instanceof SyncComplete))
- return false;
- SyncComplete other = (SyncComplete)o;
- return messageType == other.messageType &&
- desc.equals(other.desc) &&
- success == other.success &&
- nodes.equals(other.nodes);
- }
-
- @Override
- public int hashCode()
- {
- return Objects.hash(messageType, desc, success, nodes);
- }
-
- private static class SyncCompleteSerializer implements MessageSerializer<SyncComplete>
- {
- public void serialize(SyncComplete message, DataOutputPlus out, int version) throws IOException
- {
- RepairJobDesc.serializer.serialize(message.desc, out, version);
- NodePair.serializer.serialize(message.nodes, out, version);
- out.writeBoolean(message.success);
- }
-
- public SyncComplete deserialize(DataInputPlus in, int version) throws IOException
- {
- RepairJobDesc desc = RepairJobDesc.serializer.deserialize(in, version);
- NodePair nodes = NodePair.serializer.deserialize(in, version);
- return new SyncComplete(desc, nodes, in.readBoolean());
- }
-
- public long serializedSize(SyncComplete message, int version)
- {
- long size = RepairJobDesc.serializer.serializedSize(message.desc, version);
- size += NodePair.serializer.serializedSize(message.nodes, version);
- size += TypeSizes.sizeof(message.success);
- return size;
- }
- }
-}
diff --git a/src/java/org/apache/cassandra/repair/messages/SyncRequest.java b/src/java/org/apache/cassandra/repair/messages/SyncRequest.java
index e31cc6c..341455f 100644
--- a/src/java/org/apache/cassandra/repair/messages/SyncRequest.java
+++ b/src/java/org/apache/cassandra/repair/messages/SyncRequest.java
@@ -18,7 +18,6 @@
package org.apache.cassandra.repair.messages;
import java.io.IOException;
-import java.net.InetAddress;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
@@ -26,13 +25,17 @@
import org.apache.cassandra.db.TypeSizes;
import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.io.IVersionedSerializer;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
-import org.apache.cassandra.net.CompactEndpointSerializationHelper;
-import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.repair.RepairJobDesc;
+import org.apache.cassandra.streaming.PreviewKind;
+
+import static org.apache.cassandra.locator.InetAddressAndPort.Serializer.inetAddressAndPortSerializer;
/**
* Body part of SYNC_REQUEST repair message.
@@ -42,20 +45,20 @@
*/
public class SyncRequest extends RepairMessage
{
- public static MessageSerializer serializer = new SyncRequestSerializer();
-
- public final InetAddress initiator;
- public final InetAddress src;
- public final InetAddress dst;
+ public final InetAddressAndPort initiator;
+ public final InetAddressAndPort src;
+ public final InetAddressAndPort dst;
public final Collection<Range<Token>> ranges;
+ public final PreviewKind previewKind;
- public SyncRequest(RepairJobDesc desc, InetAddress initiator, InetAddress src, InetAddress dst, Collection<Range<Token>> ranges)
- {
- super(Type.SYNC_REQUEST, desc);
+ public SyncRequest(RepairJobDesc desc, InetAddressAndPort initiator, InetAddressAndPort src, InetAddressAndPort dst, Collection<Range<Token>> ranges, PreviewKind previewKind)
+ {
+ super(desc);
this.initiator = initiator;
this.src = src;
this.dst = dst;
this.ranges = ranges;
+ this.previewKind = previewKind;
}
@Override
@@ -64,59 +67,62 @@
if (!(o instanceof SyncRequest))
return false;
SyncRequest req = (SyncRequest)o;
- return messageType == req.messageType &&
- desc.equals(req.desc) &&
+ return desc.equals(req.desc) &&
initiator.equals(req.initiator) &&
src.equals(req.src) &&
dst.equals(req.dst) &&
- ranges.equals(req.ranges);
+ ranges.equals(req.ranges) &&
+ previewKind == req.previewKind;
}
@Override
public int hashCode()
{
- return Objects.hash(messageType, desc, initiator, src, dst, ranges);
+ return Objects.hash(desc, initiator, src, dst, ranges, previewKind);
}
- public static class SyncRequestSerializer implements MessageSerializer<SyncRequest>
+ public static final IVersionedSerializer<SyncRequest> serializer = new IVersionedSerializer<SyncRequest>()
{
public void serialize(SyncRequest message, DataOutputPlus out, int version) throws IOException
{
RepairJobDesc.serializer.serialize(message.desc, out, version);
- CompactEndpointSerializationHelper.serialize(message.initiator, out);
- CompactEndpointSerializationHelper.serialize(message.src, out);
- CompactEndpointSerializationHelper.serialize(message.dst, out);
+ inetAddressAndPortSerializer.serialize(message.initiator, out, version);
+ inetAddressAndPortSerializer.serialize(message.src, out, version);
+ inetAddressAndPortSerializer.serialize(message.dst, out, version);
out.writeInt(message.ranges.size());
for (Range<Token> range : message.ranges)
{
- MessagingService.validatePartitioner(range);
+ IPartitioner.validate(range);
AbstractBounds.tokenSerializer.serialize(range, out, version);
}
+ out.writeInt(message.previewKind.getSerializationVal());
}
public SyncRequest deserialize(DataInputPlus in, int version) throws IOException
{
RepairJobDesc desc = RepairJobDesc.serializer.deserialize(in, version);
- InetAddress owner = CompactEndpointSerializationHelper.deserialize(in);
- InetAddress src = CompactEndpointSerializationHelper.deserialize(in);
- InetAddress dst = CompactEndpointSerializationHelper.deserialize(in);
+ InetAddressAndPort owner = inetAddressAndPortSerializer.deserialize(in, version);
+ InetAddressAndPort src = inetAddressAndPortSerializer.deserialize(in, version);
+ InetAddressAndPort dst = inetAddressAndPortSerializer.deserialize(in, version);
int rangesCount = in.readInt();
List<Range<Token>> ranges = new ArrayList<>(rangesCount);
for (int i = 0; i < rangesCount; ++i)
- ranges.add((Range<Token>) AbstractBounds.tokenSerializer.deserialize(in, MessagingService.globalPartitioner(), version));
- return new SyncRequest(desc, owner, src, dst, ranges);
+ ranges.add((Range<Token>) AbstractBounds.tokenSerializer.deserialize(in, IPartitioner.global(), version));
+ PreviewKind previewKind = PreviewKind.deserialize(in.readInt());
+ return new SyncRequest(desc, owner, src, dst, ranges, previewKind);
}
public long serializedSize(SyncRequest message, int version)
{
long size = RepairJobDesc.serializer.serializedSize(message.desc, version);
- size += 3 * CompactEndpointSerializationHelper.serializedSize(message.initiator);
+ size += 3 * inetAddressAndPortSerializer.serializedSize(message.initiator, version);
size += TypeSizes.sizeof(message.ranges.size());
for (Range<Token> range : message.ranges)
size += AbstractBounds.tokenSerializer.serializedSize(range, version);
+ size += TypeSizes.sizeof(message.previewKind.getSerializationVal());
return size;
}
- }
+ };
@Override
public String toString()
@@ -126,6 +132,7 @@
", src=" + src +
", dst=" + dst +
", ranges=" + ranges +
+ ", previewKind=" + previewKind +
"} " + super.toString();
}
}
diff --git a/src/java/org/apache/cassandra/repair/messages/SyncResponse.java b/src/java/org/apache/cassandra/repair/messages/SyncResponse.java
new file mode 100644
index 0000000..e7e7985
--- /dev/null
+++ b/src/java/org/apache/cassandra/repair/messages/SyncResponse.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.repair.messages;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+
+import org.apache.cassandra.db.TypeSizes;
+import org.apache.cassandra.io.IVersionedSerializer;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.repair.SyncNodePair;
+import org.apache.cassandra.repair.RepairJobDesc;
+import org.apache.cassandra.streaming.SessionSummary;
+
+/**
+ *
+ * @since 2.0
+ */
+public class SyncResponse extends RepairMessage
+{
+ /** nodes that involved in this sync */
+ public final SyncNodePair nodes;
+ /** true if sync success, false otherwise */
+ public final boolean success;
+
+ public final List<SessionSummary> summaries;
+
+ public SyncResponse(RepairJobDesc desc, SyncNodePair nodes, boolean success, List<SessionSummary> summaries)
+ {
+ super(desc);
+ this.nodes = nodes;
+ this.success = success;
+ this.summaries = summaries;
+ }
+
+ public SyncResponse(RepairJobDesc desc, InetAddressAndPort endpoint1, InetAddressAndPort endpoint2, boolean success, List<SessionSummary> summaries)
+ {
+ super(desc);
+ this.summaries = summaries;
+ this.nodes = new SyncNodePair(endpoint1, endpoint2);
+ this.success = success;
+ }
+
+ @Override
+ public boolean equals(Object o)
+ {
+ if (!(o instanceof SyncResponse))
+ return false;
+ SyncResponse other = (SyncResponse)o;
+ return desc.equals(other.desc) &&
+ success == other.success &&
+ nodes.equals(other.nodes) &&
+ summaries.equals(other.summaries);
+ }
+
+ @Override
+ public int hashCode()
+ {
+ return Objects.hash(desc, success, nodes, summaries);
+ }
+
+ public static final IVersionedSerializer<SyncResponse> serializer = new IVersionedSerializer<SyncResponse>()
+ {
+ public void serialize(SyncResponse message, DataOutputPlus out, int version) throws IOException
+ {
+ RepairJobDesc.serializer.serialize(message.desc, out, version);
+ SyncNodePair.serializer.serialize(message.nodes, out, version);
+ out.writeBoolean(message.success);
+
+ out.writeInt(message.summaries.size());
+ for (SessionSummary summary: message.summaries)
+ {
+ SessionSummary.serializer.serialize(summary, out, version);
+ }
+ }
+
+ public SyncResponse deserialize(DataInputPlus in, int version) throws IOException
+ {
+ RepairJobDesc desc = RepairJobDesc.serializer.deserialize(in, version);
+ SyncNodePair nodes = SyncNodePair.serializer.deserialize(in, version);
+ boolean success = in.readBoolean();
+
+ int numSummaries = in.readInt();
+ List<SessionSummary> summaries = new ArrayList<>(numSummaries);
+ for (int i=0; i<numSummaries; i++)
+ {
+ summaries.add(SessionSummary.serializer.deserialize(in, version));
+ }
+
+ return new SyncResponse(desc, nodes, success, summaries);
+ }
+
+ public long serializedSize(SyncResponse message, int version)
+ {
+ long size = RepairJobDesc.serializer.serializedSize(message.desc, version);
+ size += SyncNodePair.serializer.serializedSize(message.nodes, version);
+ size += TypeSizes.sizeof(message.success);
+
+ size += TypeSizes.sizeof(message.summaries.size());
+ for (SessionSummary summary: message.summaries)
+ {
+ size += SessionSummary.serializer.serializedSize(summary, version);
+ }
+
+ return size;
+ }
+ };
+}
diff --git a/src/java/org/apache/cassandra/repair/messages/ValidationComplete.java b/src/java/org/apache/cassandra/repair/messages/ValidationComplete.java
deleted file mode 100644
index 704bffb..0000000
--- a/src/java/org/apache/cassandra/repair/messages/ValidationComplete.java
+++ /dev/null
@@ -1,109 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.repair.messages;
-
-import java.io.IOException;
-import java.util.Objects;
-
-import org.apache.cassandra.db.TypeSizes;
-import org.apache.cassandra.io.util.DataInputPlus;
-import org.apache.cassandra.io.util.DataOutputPlus;
-import org.apache.cassandra.repair.RepairJobDesc;
-import org.apache.cassandra.utils.MerkleTrees;
-
-/**
- * ValidationComplete message is sent when validation compaction completed successfully.
- *
- * @since 2.0
- */
-public class ValidationComplete extends RepairMessage
-{
- public static MessageSerializer serializer = new ValidationCompleteSerializer();
-
- /** Merkle hash tree response. Null if validation failed. */
- public final MerkleTrees trees;
-
- public ValidationComplete(RepairJobDesc desc)
- {
- super(Type.VALIDATION_COMPLETE, desc);
- trees = null;
- }
-
- public ValidationComplete(RepairJobDesc desc, MerkleTrees trees)
- {
- super(Type.VALIDATION_COMPLETE, desc);
- assert trees != null;
- this.trees = trees;
- }
-
- public boolean success()
- {
- return trees != null;
- }
-
- @Override
- public boolean equals(Object o)
- {
- if (!(o instanceof ValidationComplete))
- return false;
-
- ValidationComplete other = (ValidationComplete)o;
- return messageType == other.messageType &&
- desc.equals(other.desc);
- }
-
- @Override
- public int hashCode()
- {
- return Objects.hash(messageType, desc);
- }
-
- private static class ValidationCompleteSerializer implements MessageSerializer<ValidationComplete>
- {
- public void serialize(ValidationComplete message, DataOutputPlus out, int version) throws IOException
- {
- RepairJobDesc.serializer.serialize(message.desc, out, version);
- out.writeBoolean(message.success());
- if (message.trees != null)
- MerkleTrees.serializer.serialize(message.trees, out, version);
- }
-
- public ValidationComplete deserialize(DataInputPlus in, int version) throws IOException
- {
- RepairJobDesc desc = RepairJobDesc.serializer.deserialize(in, version);
- boolean success = in.readBoolean();
-
- if (success)
- {
- MerkleTrees trees = MerkleTrees.serializer.deserialize(in, version);
- return new ValidationComplete(desc, trees);
- }
-
- return new ValidationComplete(desc);
- }
-
- public long serializedSize(ValidationComplete message, int version)
- {
- long size = RepairJobDesc.serializer.serializedSize(message.desc, version);
- size += TypeSizes.sizeof(message.success());
- if (message.trees != null)
- size += MerkleTrees.serializer.serializedSize(message.trees, version);
- return size;
- }
- }
-}
diff --git a/src/java/org/apache/cassandra/repair/messages/ValidationRequest.java b/src/java/org/apache/cassandra/repair/messages/ValidationRequest.java
index 0dfab6a..f9a1f4e 100644
--- a/src/java/org/apache/cassandra/repair/messages/ValidationRequest.java
+++ b/src/java/org/apache/cassandra/repair/messages/ValidationRequest.java
@@ -20,6 +20,7 @@
import java.io.IOException;
import org.apache.cassandra.db.TypeSizes;
+import org.apache.cassandra.io.IVersionedSerializer;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.repair.RepairJobDesc;
@@ -31,22 +32,20 @@
*/
public class ValidationRequest extends RepairMessage
{
- public static MessageSerializer serializer = new ValidationRequestSerializer();
+ public final int nowInSec;
- public final int gcBefore;
-
- public ValidationRequest(RepairJobDesc desc, int gcBefore)
+ public ValidationRequest(RepairJobDesc desc, int nowInSec)
{
- super(Type.VALIDATION_REQUEST, desc);
- this.gcBefore = gcBefore;
+ super(desc);
+ this.nowInSec = nowInSec;
}
@Override
public String toString()
{
return "ValidationRequest{" +
- "gcBefore=" + gcBefore +
- "} " + super.toString();
+ "nowInSec=" + nowInSec +
+ "} " + super.toString();
}
@Override
@@ -56,21 +55,21 @@
if (o == null || getClass() != o.getClass()) return false;
ValidationRequest that = (ValidationRequest) o;
- return gcBefore == that.gcBefore;
+ return nowInSec == that.nowInSec;
}
@Override
public int hashCode()
{
- return gcBefore;
+ return nowInSec;
}
- public static class ValidationRequestSerializer implements MessageSerializer<ValidationRequest>
+ public static final IVersionedSerializer<ValidationRequest> serializer = new IVersionedSerializer<ValidationRequest>()
{
public void serialize(ValidationRequest message, DataOutputPlus out, int version) throws IOException
{
RepairJobDesc.serializer.serialize(message.desc, out, version);
- out.writeInt(message.gcBefore);
+ out.writeInt(message.nowInSec);
}
public ValidationRequest deserialize(DataInputPlus dis, int version) throws IOException
@@ -82,8 +81,8 @@
public long serializedSize(ValidationRequest message, int version)
{
long size = RepairJobDesc.serializer.serializedSize(message.desc, version);
- size += TypeSizes.sizeof(message.gcBefore);
+ size += TypeSizes.sizeof(message.nowInSec);
return size;
}
- }
+ };
}
diff --git a/src/java/org/apache/cassandra/repair/messages/ValidationResponse.java b/src/java/org/apache/cassandra/repair/messages/ValidationResponse.java
new file mode 100644
index 0000000..d9f4467
--- /dev/null
+++ b/src/java/org/apache/cassandra/repair/messages/ValidationResponse.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.repair.messages;
+
+import java.io.IOException;
+import java.util.Objects;
+
+import org.apache.cassandra.db.TypeSizes;
+import org.apache.cassandra.io.IVersionedSerializer;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.repair.RepairJobDesc;
+import org.apache.cassandra.utils.MerkleTrees;
+
+/**
+ * ValidationComplete message is sent when validation compaction completed successfully.
+ *
+ * @since 2.0
+ */
+public class ValidationResponse extends RepairMessage
+{
+ /** Merkle hash tree response. Null if validation failed. */
+ public final MerkleTrees trees;
+
+ public ValidationResponse(RepairJobDesc desc)
+ {
+ super(desc);
+ trees = null;
+ }
+
+ public ValidationResponse(RepairJobDesc desc, MerkleTrees trees)
+ {
+ super(desc);
+ assert trees != null;
+ this.trees = trees;
+ }
+
+ public boolean success()
+ {
+ return trees != null;
+ }
+
+ /**
+ * @return a new {@link ValidationResponse} instance with all trees moved off heap, or {@code this}
+ * if it's a failure response.
+ */
+ public ValidationResponse tryMoveOffHeap() throws IOException
+ {
+ return trees == null ? this : new ValidationResponse(desc, trees.tryMoveOffHeap());
+ }
+
+ @Override
+ public boolean equals(Object o)
+ {
+ if (!(o instanceof ValidationResponse))
+ return false;
+
+ ValidationResponse other = (ValidationResponse)o;
+ return desc.equals(other.desc);
+ }
+
+ @Override
+ public int hashCode()
+ {
+ return Objects.hash(desc);
+ }
+
+ public static final IVersionedSerializer<ValidationResponse> serializer = new IVersionedSerializer<ValidationResponse>()
+ {
+ public void serialize(ValidationResponse message, DataOutputPlus out, int version) throws IOException
+ {
+ RepairJobDesc.serializer.serialize(message.desc, out, version);
+ out.writeBoolean(message.success());
+ if (message.trees != null)
+ MerkleTrees.serializer.serialize(message.trees, out, version);
+ }
+
+ public ValidationResponse deserialize(DataInputPlus in, int version) throws IOException
+ {
+ RepairJobDesc desc = RepairJobDesc.serializer.deserialize(in, version);
+ boolean success = in.readBoolean();
+
+ if (success)
+ {
+ MerkleTrees trees = MerkleTrees.serializer.deserialize(in, version);
+ return new ValidationResponse(desc, trees);
+ }
+
+ return new ValidationResponse(desc);
+ }
+
+ public long serializedSize(ValidationResponse message, int version)
+ {
+ long size = RepairJobDesc.serializer.serializedSize(message.desc, version);
+ size += TypeSizes.sizeof(message.success());
+ if (message.trees != null)
+ size += MerkleTrees.serializer.serializedSize(message.trees, version);
+ return size;
+ }
+ };
+}
diff --git a/src/java/org/apache/cassandra/scheduler/IRequestScheduler.java b/src/java/org/apache/cassandra/scheduler/IRequestScheduler.java
deleted file mode 100644
index 798f09e..0000000
--- a/src/java/org/apache/cassandra/scheduler/IRequestScheduler.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.scheduler;
-
-import java.util.concurrent.TimeoutException;
-
-/**
- * Implementors of IRequestScheduler must provide a constructor taking a RequestSchedulerOptions object.
- */
-public interface IRequestScheduler
-{
- /**
- * Queue incoming request threads
- *
- * @param t Thread handing the request
- * @param id Scheduling parameter, an id to distinguish profiles (users/keyspace)
- * @param timeoutMS The max time in milliseconds to spend blocking for a slot
- */
- public void queue(Thread t, String id, long timeoutMS) throws TimeoutException;
-
- /**
- * A convenience method for indicating when a particular request has completed
- * processing, and before a return to the client
- */
- public void release();
-}
diff --git a/src/java/org/apache/cassandra/scheduler/NoScheduler.java b/src/java/org/apache/cassandra/scheduler/NoScheduler.java
deleted file mode 100644
index d3f4ce8..0000000
--- a/src/java/org/apache/cassandra/scheduler/NoScheduler.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.scheduler;
-
-import org.apache.cassandra.config.RequestSchedulerOptions;
-
-
-/**
- * This is basically not having a scheduler, the requests are
- * processed as normally would be handled by the JVM.
- */
-public class NoScheduler implements IRequestScheduler
-{
-
- public NoScheduler(RequestSchedulerOptions options) {}
-
- public NoScheduler() {}
-
- public void queue(Thread t, String id, long timeoutMS) {}
-
- public void release() {}
-}
diff --git a/src/java/org/apache/cassandra/scheduler/RoundRobinScheduler.java b/src/java/org/apache/cassandra/scheduler/RoundRobinScheduler.java
deleted file mode 100644
index fd967f3..0000000
--- a/src/java/org/apache/cassandra/scheduler/RoundRobinScheduler.java
+++ /dev/null
@@ -1,159 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.scheduler;
-
-
-import java.util.Map;
-import java.util.concurrent.Semaphore;
-import java.util.concurrent.TimeoutException;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.cassandra.concurrent.NamedThreadFactory;
-import org.apache.cassandra.config.RequestSchedulerOptions;
-import org.cliffc.high_scale_lib.NonBlockingHashMap;
-
-/**
- * A very basic Round Robin implementation of the RequestScheduler. It handles
- * request groups identified on user/keyspace by placing them in separate
- * queues and servicing a request from each queue in a RoundRobin fashion.
- * It optionally adds weights for each round.
- */
-public class RoundRobinScheduler implements IRequestScheduler
-{
- private static final Logger logger = LoggerFactory.getLogger(RoundRobinScheduler.class);
-
- //Map of queue id to weighted queue
- private final NonBlockingHashMap<String, WeightedQueue> queues;
-
- private final Semaphore taskCount;
-
- // Tracks the count of threads available in all queues
- // Used by the the scheduler thread so we don't need to busy-wait until there is a request to process
- private final Semaphore queueSize = new Semaphore(0, false);
-
- private final int defaultWeight;
- private final Map<String, Integer> weights;
-
- public RoundRobinScheduler(RequestSchedulerOptions options)
- {
- defaultWeight = options.default_weight;
- weights = options.weights;
-
- // the task count is acquired for the first time _after_ releasing a thread, so we pre-decrement
- taskCount = new Semaphore(options.throttle_limit - 1);
-
- queues = new NonBlockingHashMap<String, WeightedQueue>();
- Runnable runnable = () ->
- {
- while (true)
- {
- schedule();
- }
- };
- Thread scheduler = NamedThreadFactory.createThread(runnable, "REQUEST-SCHEDULER");
- scheduler.start();
- logger.info("Started the RoundRobin Request Scheduler");
- }
-
- public void queue(Thread t, String id, long timeoutMS) throws TimeoutException
- {
- WeightedQueue weightedQueue = getWeightedQueue(id);
-
- try
- {
- queueSize.release();
- try
- {
- weightedQueue.put(t, timeoutMS);
- // the scheduler will release us when a slot is available
- }
- catch (TimeoutException | InterruptedException e)
- {
- queueSize.acquireUninterruptibly();
- throw e;
- }
- }
- catch (InterruptedException e)
- {
- throw new RuntimeException("Interrupted while queueing requests", e);
- }
- }
-
- public void release()
- {
- taskCount.release();
- }
-
- private void schedule()
- {
- queueSize.acquireUninterruptibly();
- for (Map.Entry<String,WeightedQueue> request : queues.entrySet())
- {
- WeightedQueue queue = request.getValue();
- //Using the weight, process that many requests at a time (for that scheduler id)
- for (int i=0; i<queue.weight; i++)
- {
- Thread t = queue.poll();
- if (t == null)
- break;
- else
- {
- taskCount.acquireUninterruptibly();
- queueSize.acquireUninterruptibly();
- }
- }
- }
- queueSize.release();
- }
-
- /*
- * Get the Queue for the respective id, if one is not available
- * create a new queue for that corresponding id and return it
- */
- private WeightedQueue getWeightedQueue(String id)
- {
- WeightedQueue weightedQueue = queues.get(id);
- if (weightedQueue != null)
- // queue existed
- return weightedQueue;
-
- WeightedQueue maybenew = new WeightedQueue(id, getWeight(id));
- weightedQueue = queues.putIfAbsent(id, maybenew);
- if (weightedQueue == null)
- {
- return maybenew;
- }
-
- // another thread created the queue
- return weightedQueue;
- }
-
- Semaphore getTaskCount()
- {
- return taskCount;
- }
-
- private int getWeight(String weightingVar)
- {
- return (weights != null && weights.containsKey(weightingVar))
- ? weights.get(weightingVar)
- : defaultWeight;
- }
-}
diff --git a/src/java/org/apache/cassandra/scheduler/WeightedQueue.java b/src/java/org/apache/cassandra/scheduler/WeightedQueue.java
deleted file mode 100644
index 76c7e9d..0000000
--- a/src/java/org/apache/cassandra/scheduler/WeightedQueue.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.scheduler;
-
-import java.util.concurrent.SynchronousQueue;
-import java.util.concurrent.TimeoutException;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.cassandra.metrics.LatencyMetrics;
-
-class WeightedQueue
-{
- private final LatencyMetrics metric;
-
- public final String key;
- public final int weight;
- private final SynchronousQueue<Entry> queue;
- public WeightedQueue(String key, int weight)
- {
- this.key = key;
- this.weight = weight;
- this.queue = new SynchronousQueue<Entry>(true);
- this.metric = new LatencyMetrics("scheduler", "WeightedQueue", key);
- }
-
- public void put(Thread t, long timeoutMS) throws InterruptedException, TimeoutException
- {
- if (!queue.offer(new WeightedQueue.Entry(t), timeoutMS, TimeUnit.MILLISECONDS))
- throw new TimeoutException("Failed to acquire request scheduler slot for '" + key + "'");
- }
-
- public Thread poll()
- {
- Entry e = queue.poll();
- if (e == null)
- return null;
- metric.addNano(System.nanoTime() - e.creationTime);
- return e.thread;
- }
-
- @Override
- public String toString()
- {
- return "RoundRobinScheduler.WeightedQueue(key=" + key + " weight=" + weight + ")";
- }
-
- private final static class Entry
- {
- public final long creationTime = System.nanoTime();
- public final Thread thread;
- public Entry(Thread thread)
- {
- this.thread = thread;
- }
- }
-}
diff --git a/src/java/org/apache/cassandra/schema/ColumnMetadata.java b/src/java/org/apache/cassandra/schema/ColumnMetadata.java
new file mode 100644
index 0000000..ee34be5
--- /dev/null
+++ b/src/java/org/apache/cassandra/schema/ColumnMetadata.java
@@ -0,0 +1,669 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.schema;
+
+import java.nio.ByteBuffer;
+import java.util.*;
+import java.util.function.Predicate;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.MoreObjects;
+import com.google.common.collect.Collections2;
+
+import org.apache.cassandra.cql3.*;
+import org.apache.cassandra.cql3.selection.Selectable;
+import org.apache.cassandra.cql3.selection.Selector;
+import org.apache.cassandra.cql3.selection.SimpleSelector;
+import org.apache.cassandra.db.rows.*;
+import org.apache.cassandra.db.marshal.*;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.serializers.MarshalException;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.github.jamm.Unmetered;
+
+@Unmetered
+public final class ColumnMetadata extends ColumnSpecification implements Selectable, Comparable<ColumnMetadata>
+{
+ public static final Comparator<Object> asymmetricColumnDataComparator =
+ (a, b) -> ((ColumnData) a).column().compareTo((ColumnMetadata) b);
+
+ public static final int NO_POSITION = -1;
+
+ public enum ClusteringOrder
+ {
+ ASC, DESC, NONE
+ }
+
+ /**
+ * The type of CQL3 column this definition represents.
+ * There is 4 main type of CQL3 columns: those parts of the partition key,
+ * those parts of the clustering columns and amongst the others, regular and
+ * static ones.
+ *
+ * IMPORTANT: this enum is serialized as toString() and deserialized by calling
+ * Kind.valueOf(), so do not override toString() or rename existing values.
+ */
+ public enum Kind
+ {
+ // NOTE: if adding a new type, must modify comparisonOrder
+ PARTITION_KEY,
+ CLUSTERING,
+ REGULAR,
+ STATIC;
+
+ public boolean isPrimaryKeyKind()
+ {
+ return this == PARTITION_KEY || this == CLUSTERING;
+ }
+
+ }
+
+ public final Kind kind;
+
+ /*
+ * If the column is a partition key or clustering column, its position relative to
+ * other columns of the same kind. Otherwise, NO_POSITION (-1).
+ *
+ * Note that partition key and clustering columns are numbered separately so
+ * the first clustering column is 0.
+ */
+ private final int position;
+
+ private final Comparator<CellPath> cellPathComparator;
+ private final Comparator<Object> asymmetricCellPathComparator;
+ private final Comparator<? super Cell> cellComparator;
+
+ private int hash;
+
+ /**
+ * These objects are compared frequently, so we encode several of their comparison components
+ * into a single long value so that this can be done efficiently
+ */
+ private final long comparisonOrder;
+
+ private static long comparisonOrder(Kind kind, boolean isComplex, long position, ColumnIdentifier name)
+ {
+ assert position >= 0 && position < 1 << 12;
+ return (((long) kind.ordinal()) << 61)
+ | (isComplex ? 1L << 60 : 0)
+ | (position << 48)
+ | (name.prefixComparison >>> 16);
+ }
+
+ public static ColumnMetadata partitionKeyColumn(TableMetadata table, ByteBuffer name, AbstractType<?> type, int position)
+ {
+ return new ColumnMetadata(table, name, type, position, Kind.PARTITION_KEY);
+ }
+
+ public static ColumnMetadata partitionKeyColumn(String keyspace, String table, String name, AbstractType<?> type, int position)
+ {
+ return new ColumnMetadata(keyspace, table, ColumnIdentifier.getInterned(name, true), type, position, Kind.PARTITION_KEY);
+ }
+
+ public static ColumnMetadata clusteringColumn(TableMetadata table, ByteBuffer name, AbstractType<?> type, int position)
+ {
+ return new ColumnMetadata(table, name, type, position, Kind.CLUSTERING);
+ }
+
+ public static ColumnMetadata clusteringColumn(String keyspace, String table, String name, AbstractType<?> type, int position)
+ {
+ return new ColumnMetadata(keyspace, table, ColumnIdentifier.getInterned(name, true), type, position, Kind.CLUSTERING);
+ }
+
+ public static ColumnMetadata regularColumn(TableMetadata table, ByteBuffer name, AbstractType<?> type)
+ {
+ return new ColumnMetadata(table, name, type, NO_POSITION, Kind.REGULAR);
+ }
+
+ public static ColumnMetadata regularColumn(String keyspace, String table, String name, AbstractType<?> type)
+ {
+ return new ColumnMetadata(keyspace, table, ColumnIdentifier.getInterned(name, true), type, NO_POSITION, Kind.REGULAR);
+ }
+
+ public static ColumnMetadata staticColumn(TableMetadata table, ByteBuffer name, AbstractType<?> type)
+ {
+ return new ColumnMetadata(table, name, type, NO_POSITION, Kind.STATIC);
+ }
+
+ public static ColumnMetadata staticColumn(String keyspace, String table, String name, AbstractType<?> type)
+ {
+ return new ColumnMetadata(keyspace, table, ColumnIdentifier.getInterned(name, true), type, NO_POSITION, Kind.STATIC);
+ }
+
+ public ColumnMetadata(TableMetadata table, ByteBuffer name, AbstractType<?> type, int position, Kind kind)
+ {
+ this(table.keyspace,
+ table.name,
+ ColumnIdentifier.getInterned(name, table.columnDefinitionNameComparator(kind)),
+ type,
+ position,
+ kind);
+ }
+
+ @VisibleForTesting
+ public ColumnMetadata(String ksName,
+ String cfName,
+ ColumnIdentifier name,
+ AbstractType<?> type,
+ int position,
+ Kind kind)
+ {
+ super(ksName, cfName, name, type);
+ assert name != null && type != null && kind != null;
+ assert (position == NO_POSITION) == !kind.isPrimaryKeyKind(); // The position really only make sense for partition and clustering columns (and those must have one),
+ // so make sure we don't sneak it for something else since it'd breaks equals()
+ this.kind = kind;
+ this.position = position;
+ this.cellPathComparator = makeCellPathComparator(kind, type);
+ this.cellComparator = cellPathComparator == null ? ColumnData.comparator : (a, b) -> cellPathComparator.compare(a.path(), b.path());
+ this.asymmetricCellPathComparator = cellPathComparator == null ? null : (a, b) -> cellPathComparator.compare(((Cell)a).path(), (CellPath) b);
+ this.comparisonOrder = comparisonOrder(kind, isComplex(), Math.max(0, position), name);
+ }
+
+ private static Comparator<CellPath> makeCellPathComparator(Kind kind, AbstractType<?> type)
+ {
+ if (kind.isPrimaryKeyKind() || !type.isMultiCell())
+ return null;
+
+ AbstractType<?> nameComparator = type.isCollection()
+ ? ((CollectionType) type).nameComparator()
+ : ((UserType) type).nameComparator();
+
+
+ return (path1, path2) ->
+ {
+ if (path1.size() == 0 || path2.size() == 0)
+ {
+ if (path1 == CellPath.BOTTOM)
+ return path2 == CellPath.BOTTOM ? 0 : -1;
+ if (path1 == CellPath.TOP)
+ return path2 == CellPath.TOP ? 0 : 1;
+ return path2 == CellPath.BOTTOM ? 1 : -1;
+ }
+
+ // This will get more complicated once we have non-frozen UDT and nested collections
+ assert path1.size() == 1 && path2.size() == 1;
+ return nameComparator.compare(path1.get(0), path2.get(0));
+ };
+ }
+
+ public ColumnMetadata copy()
+ {
+ return new ColumnMetadata(ksName, cfName, name, type, position, kind);
+ }
+
+ public ColumnMetadata withNewName(ColumnIdentifier newName)
+ {
+ return new ColumnMetadata(ksName, cfName, newName, type, position, kind);
+ }
+
+ public ColumnMetadata withNewType(AbstractType<?> newType)
+ {
+ return new ColumnMetadata(ksName, cfName, name, newType, position, kind);
+ }
+
+ public boolean isPartitionKey()
+ {
+ return kind == Kind.PARTITION_KEY;
+ }
+
+ public boolean isClusteringColumn()
+ {
+ return kind == Kind.CLUSTERING;
+ }
+
+ public boolean isStatic()
+ {
+ return kind == Kind.STATIC;
+ }
+
+ public boolean isRegular()
+ {
+ return kind == Kind.REGULAR;
+ }
+
+ public ClusteringOrder clusteringOrder()
+ {
+ if (!isClusteringColumn())
+ return ClusteringOrder.NONE;
+
+ return type.isReversed() ? ClusteringOrder.DESC : ClusteringOrder.ASC;
+ }
+
+ public int position()
+ {
+ return position;
+ }
+
+ @Override
+ public boolean equals(Object o)
+ {
+ if (this == o)
+ return true;
+
+ if (!(o instanceof ColumnMetadata))
+ return false;
+
+ ColumnMetadata cd = (ColumnMetadata) o;
+
+ return equalsWithoutType(cd) && type.equals(cd.type);
+ }
+
+ private boolean equalsWithoutType(ColumnMetadata other)
+ {
+ return name.equals(other.name)
+ && kind == other.kind
+ && position == other.position
+ && ksName.equals(other.ksName)
+ && cfName.equals(other.cfName);
+ }
+
+ Optional<Difference> compare(ColumnMetadata other)
+ {
+ if (!equalsWithoutType(other))
+ return Optional.of(Difference.SHALLOW);
+
+ if (type.equals(other.type))
+ return Optional.empty();
+
+ return type.asCQL3Type().toString().equals(other.type.asCQL3Type().toString())
+ ? Optional.of(Difference.DEEP)
+ : Optional.of(Difference.SHALLOW);
+ }
+
+ @Override
+ public int hashCode()
+ {
+ // This achieves the same as Objects.hashcode, but avoids the object array allocation
+ // which features significantly in the allocation profile and caches the result.
+ int result = hash;
+ if (result == 0)
+ {
+ result = 31 + (ksName == null ? 0 : ksName.hashCode());
+ result = 31 * result + (cfName == null ? 0 : cfName.hashCode());
+ result = 31 * result + (name == null ? 0 : name.hashCode());
+ result = 31 * result + (type == null ? 0 : type.hashCode());
+ result = 31 * result + (kind == null ? 0 : kind.hashCode());
+ result = 31 * result + position;
+ hash = result;
+ }
+ return result;
+ }
+
+ @Override
+ public String toString()
+ {
+ return name.toString();
+ }
+
+ public String debugString()
+ {
+ return MoreObjects.toStringHelper(this)
+ .add("name", name)
+ .add("type", type)
+ .add("kind", kind)
+ .add("position", position)
+ .toString();
+ }
+
+ public boolean isPrimaryKeyColumn()
+ {
+ return kind.isPrimaryKeyKind();
+ }
+
+ @Override
+ public boolean selectColumns(Predicate<ColumnMetadata> predicate)
+ {
+ return predicate.test(this);
+ }
+
+ @Override
+ public boolean processesSelection()
+ {
+ return false;
+ }
+
+ /**
+ * Converts the specified column definitions into column identifiers.
+ *
+ * @param definitions the column definitions to convert.
+ * @return the column identifiers corresponding to the specified definitions
+ */
+ public static Collection<ColumnIdentifier> toIdentifiers(Collection<ColumnMetadata> definitions)
+ {
+ return Collections2.transform(definitions, columnDef -> columnDef.name);
+ }
+
+ public int compareTo(ColumnMetadata other)
+ {
+ if (this == other)
+ return 0;
+
+ if (comparisonOrder != other.comparisonOrder)
+ return Long.compare(comparisonOrder, other.comparisonOrder);
+
+ return this.name.compareTo(other.name);
+ }
+
+ public Comparator<CellPath> cellPathComparator()
+ {
+ return cellPathComparator;
+ }
+
+ public Comparator<Object> asymmetricCellPathComparator()
+ {
+ return asymmetricCellPathComparator;
+ }
+
+ public Comparator<? super Cell> cellComparator()
+ {
+ return cellComparator;
+ }
+
+ public boolean isComplex()
+ {
+ return cellPathComparator != null;
+ }
+
+ public boolean isSimple()
+ {
+ return !isComplex();
+ }
+
+ public CellPath.Serializer cellPathSerializer()
+ {
+ // Collections are our only complex so far, so keep it simple
+ return CollectionType.cellPathSerializer;
+ }
+
+ public void validateCell(Cell cell)
+ {
+ if (cell.isTombstone())
+ {
+ if (cell.value().hasRemaining())
+ throw new MarshalException("A tombstone should not have a value");
+ if (cell.path() != null)
+ validateCellPath(cell.path());
+ }
+ else if(type.isUDT())
+ {
+ // To validate a non-frozen UDT field, both the path and the value
+ // are needed, the path being an index into an array of value types.
+ ((UserType)type).validateCell(cell);
+ }
+ else
+ {
+ type.validateCellValue(cell.value());
+ if (cell.path() != null)
+ validateCellPath(cell.path());
+ }
+ }
+
+ private void validateCellPath(CellPath path)
+ {
+ if (!isComplex())
+ throw new MarshalException("Only complex cells should have a cell path");
+
+ assert type.isMultiCell();
+ if (type.isCollection())
+ ((CollectionType)type).nameComparator().validate(path.get(0));
+ else
+ ((UserType)type).nameComparator().validate(path.get(0));
+ }
+
+ public void appendCqlTo(CqlBuilder builder, boolean ignoreStatic)
+ {
+ builder.append(name)
+ .append(' ')
+ .append(type);
+
+ if (isStatic() && !ignoreStatic)
+ builder.append(" static");
+ }
+
+ public static String toCQLString(Iterable<ColumnMetadata> defs)
+ {
+ return toCQLString(defs.iterator());
+ }
+
+ public static String toCQLString(Iterator<ColumnMetadata> defs)
+ {
+ if (!defs.hasNext())
+ return "";
+
+ StringBuilder sb = new StringBuilder();
+ sb.append(defs.next().name);
+ while (defs.hasNext())
+ sb.append(", ").append(defs.next().name);
+ return sb.toString();
+ }
+
+
+ public void appendNameAndOrderTo(CqlBuilder builder)
+ {
+ builder.append(name.toCQLString())
+ .append(' ')
+ .append(clusteringOrder().toString());
+ }
+
+ /**
+ * The type of the cell values for cell belonging to this column.
+ *
+ * This is the same than the column type, except for non-frozen collections where it's the 'valueComparator'
+ * of the collection.
+ *
+ * This method should not be used to get value type of non-frozon UDT.
+ */
+ public AbstractType<?> cellValueType()
+ {
+ assert !(type instanceof UserType && type.isMultiCell());
+ return type instanceof CollectionType && type.isMultiCell()
+ ? ((CollectionType)type).valueComparator()
+ : type;
+ }
+
+ /**
+ * Check if column is counter type.
+ */
+ public boolean isCounterColumn()
+ {
+ if (type instanceof CollectionType) // Possible with, for example, supercolumns
+ return ((CollectionType) type).valueComparator().isCounter();
+ return type.isCounter();
+ }
+
+ public Selector.Factory newSelectorFactory(TableMetadata table, AbstractType<?> expectedType, List<ColumnMetadata> defs, VariableSpecifications boundNames) throws InvalidRequestException
+ {
+ return SimpleSelector.newFactory(this, addAndGetIndex(this, defs));
+ }
+
+ public AbstractType<?> getExactTypeIfKnown(String keyspace)
+ {
+ return type;
+ }
+
+ /**
+ * Because legacy-created tables may have a non-text comparator, we cannot determine the proper 'key' until
+ * we know the comparator. ColumnMetadata.Raw is a placeholder that can be converted to a real ColumnIdentifier
+ * once the comparator is known with prepare(). This should only be used with identifiers that are actual
+ * column names. See CASSANDRA-8178 for more background.
+ */
+ public static abstract class Raw extends Selectable.Raw
+ {
+ /**
+ * Creates a {@code ColumnMetadata.Raw} from an unquoted identifier string.
+ */
+ public static Raw forUnquoted(String text)
+ {
+ return new Literal(text, false);
+ }
+
+ /**
+ * Creates a {@code ColumnMetadata.Raw} from a quoted identifier string.
+ */
+ public static Raw forQuoted(String text)
+ {
+ return new Literal(text, true);
+ }
+
+ /**
+ * Creates a {@code ColumnMetadata.Raw} from a pre-existing {@code ColumnMetadata}
+ * (useful in the rare cases where we already have the column but need
+ * a {@code ColumnMetadata.Raw} for typing purposes).
+ */
+ public static Raw forColumn(ColumnMetadata column)
+ {
+ return new ForColumn(column);
+ }
+
+ /**
+ * Get the identifier corresponding to this raw column, without assuming this is an
+ * existing column (unlike {@link Selectable.Raw#prepare}).
+ */
+ public abstract ColumnIdentifier getIdentifier(TableMetadata table);
+
+ public abstract String rawText();
+
+ @Override
+ public abstract ColumnMetadata prepare(TableMetadata table);
+
+ @Override
+ public final int hashCode()
+ {
+ return toString().hashCode();
+ }
+
+ @Override
+ public final boolean equals(Object o)
+ {
+ if(!(o instanceof Raw))
+ return false;
+
+ Raw that = (Raw)o;
+ return this.toString().equals(that.toString());
+ }
+
+ private static class Literal extends Raw
+ {
+ private final String text;
+
+ public Literal(String rawText, boolean keepCase)
+ {
+ this.text = keepCase ? rawText : rawText.toLowerCase(Locale.US);
+ }
+
+ public ColumnIdentifier getIdentifier(TableMetadata table)
+ {
+ if (!table.isStaticCompactTable())
+ return ColumnIdentifier.getInterned(text, true);
+
+ AbstractType<?> columnNameType = table.staticCompactOrSuperTableColumnNameType();
+ if (columnNameType instanceof UTF8Type)
+ return ColumnIdentifier.getInterned(text, true);
+
+ // We have a legacy-created table with a non-text comparator. Check if we have a matching column, otherwise assume we should use
+ // columnNameType
+ ByteBuffer bufferName = ByteBufferUtil.bytes(text);
+ for (ColumnMetadata def : table.columns())
+ {
+ if (def.name.bytes.equals(bufferName))
+ return def.name;
+ }
+ return ColumnIdentifier.getInterned(columnNameType, columnNameType.fromString(text), text);
+ }
+
+ public ColumnMetadata prepare(TableMetadata table)
+ {
+ if (!table.isStaticCompactTable())
+ return find(table);
+
+ AbstractType<?> columnNameType = table.staticCompactOrSuperTableColumnNameType();
+ if (columnNameType instanceof UTF8Type)
+ return find(table);
+
+ // We have a legacy-created table with a non-text comparator. Check if we have a match column, otherwise assume we should use
+ // columnNameType
+ ByteBuffer bufferName = ByteBufferUtil.bytes(text);
+ for (ColumnMetadata def : table.columns())
+ {
+ if (def.name.bytes.equals(bufferName))
+ return def;
+ }
+ return find(columnNameType.fromString(text), table);
+ }
+
+ private ColumnMetadata find(TableMetadata table)
+ {
+ return find(ByteBufferUtil.bytes(text), table);
+ }
+
+ private ColumnMetadata find(ByteBuffer id, TableMetadata table)
+ {
+ ColumnMetadata def = table.getColumn(id);
+ if (def == null)
+ throw new InvalidRequestException(String.format("Undefined column name %s", toString()));
+ return def;
+ }
+
+ public String rawText()
+ {
+ return text;
+ }
+
+ @Override
+ public String toString()
+ {
+ return ColumnIdentifier.maybeQuote(text);
+ }
+ }
+
+ // Use internally in the rare case where we need a ColumnMetadata.Raw for type-checking but
+ // actually already have the column itself.
+ private static class ForColumn extends Raw
+ {
+ private final ColumnMetadata column;
+
+ private ForColumn(ColumnMetadata column)
+ {
+ this.column = column;
+ }
+
+ public ColumnIdentifier getIdentifier(TableMetadata table)
+ {
+ return column.name;
+ }
+
+ public ColumnMetadata prepare(TableMetadata table)
+ {
+ assert table.getColumn(column.name) != null; // Sanity check that we're not doing something crazy
+ return column;
+ }
+
+ public String rawText()
+ {
+ return column.name.toString();
+ }
+
+ @Override
+ public String toString()
+ {
+ return column.name.toCQLString();
+ }
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/schema/CompactionParams.java b/src/java/org/apache/cassandra/schema/CompactionParams.java
index 73271f1..a9efff3 100644
--- a/src/java/org/apache/cassandra/schema/CompactionParams.java
+++ b/src/java/org/apache/cassandra/schema/CompactionParams.java
@@ -31,6 +31,7 @@
import org.apache.cassandra.db.compaction.AbstractCompactionStrategy;
import org.apache.cassandra.db.compaction.LeveledCompactionStrategy;
import org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy;
+import org.apache.cassandra.db.compaction.TimeWindowCompactionStrategy;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.utils.FBUtilities;
@@ -107,7 +108,7 @@
return new CompactionParams(klass, allOptions, isEnabled, tombstoneOption);
}
- public static CompactionParams scts(Map<String, String> options)
+ public static CompactionParams stcs(Map<String, String> options)
{
return create(SizeTieredCompactionStrategy.class, options);
}
@@ -117,6 +118,11 @@
return create(LeveledCompactionStrategy.class, options);
}
+ public static CompactionParams twcs(Map<String, String> options)
+ {
+ return create(TimeWindowCompactionStrategy.class, options);
+ }
+
public int minCompactionThreshold()
{
String threshold = options.get(Option.MIN_THRESHOLD.toString());
@@ -249,7 +255,7 @@
return create(classFromName(className), options);
}
- private static Class<? extends AbstractCompactionStrategy> classFromName(String name)
+ public static Class<? extends AbstractCompactionStrategy> classFromName(String name)
{
String className = name.contains(".")
? name
diff --git a/src/java/org/apache/cassandra/schema/CompressionParams.java b/src/java/org/apache/cassandra/schema/CompressionParams.java
index f48a688..53760a9 100644
--- a/src/java/org/apache/cassandra/schema/CompressionParams.java
+++ b/src/java/org/apache/cassandra/schema/CompressionParams.java
@@ -20,13 +20,18 @@
import java.io.IOException;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
+import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
+import java.util.concurrent.ThreadLocalRandom;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Objects;
import com.google.common.collect.ImmutableMap;
-import org.apache.commons.lang3.builder.EqualsBuilder;
+
import org.apache.commons.lang3.builder.HashCodeBuilder;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -37,6 +42,7 @@
import org.apache.cassandra.io.compress.*;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.net.MessagingService;
import static java.lang.String.format;
@@ -49,17 +55,30 @@
private static volatile boolean hasLoggedChunkLengthWarning;
private static volatile boolean hasLoggedCrcCheckChanceWarning;
- public static final int DEFAULT_CHUNK_LENGTH = 65536;
+ public static final int DEFAULT_CHUNK_LENGTH = 1024 * 16;
+ public static final double DEFAULT_MIN_COMPRESS_RATIO = 0.0; // Since pre-4.0 versions do not understand the
+ // new compression parameter we can't use a
+ // different default value.
public static final IVersionedSerializer<CompressionParams> serializer = new Serializer();
public static final String CLASS = "class";
public static final String CHUNK_LENGTH_IN_KB = "chunk_length_in_kb";
public static final String ENABLED = "enabled";
+ public static final String MIN_COMPRESS_RATIO = "min_compress_ratio";
public static final CompressionParams DEFAULT = new CompressionParams(LZ4Compressor.create(Collections.<String, String>emptyMap()),
DEFAULT_CHUNK_LENGTH,
+ calcMaxCompressedLength(DEFAULT_CHUNK_LENGTH, DEFAULT_MIN_COMPRESS_RATIO),
+ DEFAULT_MIN_COMPRESS_RATIO,
Collections.emptyMap());
+ public static final CompressionParams NOOP = new CompressionParams(NoopCompressor.create(Collections.emptyMap()),
+ // 4 KiB is often the underlying disk block size
+ 1024 * 4,
+ Integer.MAX_VALUE,
+ DEFAULT_MIN_COMPRESS_RATIO,
+ Collections.emptyMap());
+
private static final String CRC_CHECK_CHANCE_WARNING = "The option crc_check_chance was deprecated as a compression option. " +
"You should specify it as a top-level table option instead";
@@ -68,9 +87,12 @@
@Deprecated public static final String CRC_CHECK_CHANCE = "crc_check_chance";
private final ICompressor sstableCompressor;
- private final Integer chunkLength;
+ private final int chunkLength;
+ private final int maxCompressedLength; // In content we store max length to avoid rounding errors causing compress/decompress mismatch.
+ private final double minCompressRatio; // In configuration we store min ratio, the input parameter.
private final ImmutableMap<String, String> otherOptions; // Unrecognized options, can be used by the compressor
+ // TODO: deprecated, should now be carefully removed. Doesn't affect schema code as it isn't included in equals() and hashCode()
private volatile double crcCheckChance = 1.0;
public static CompressionParams fromMap(Map<String, String> opts)
@@ -94,9 +116,10 @@
sstableCompressionClass = removeSstableCompressionClass(options);
}
- Integer chunkLength = removeChunkLength(options);
+ int chunkLength = removeChunkLength(options);
+ double minCompressRatio = removeMinCompressRatio(options);
- CompressionParams cp = new CompressionParams(sstableCompressionClass, chunkLength, options);
+ CompressionParams cp = new CompressionParams(sstableCompressionClass, options, chunkLength, minCompressRatio);
cp.validate();
return cp;
@@ -109,54 +132,112 @@
public static CompressionParams noCompression()
{
- return new CompressionParams((ICompressor) null, DEFAULT_CHUNK_LENGTH, Collections.emptyMap());
+ return new CompressionParams((ICompressor) null, DEFAULT_CHUNK_LENGTH, Integer.MAX_VALUE, 0.0, Collections.emptyMap());
}
+ // The shorthand methods below are only used for tests. They are a little inconsistent in their choice of
+ // parameters -- this is done on purpose to test out various compression parameter combinations.
+
+ @VisibleForTesting
public static CompressionParams snappy()
{
- return snappy(null);
+ return snappy(DEFAULT_CHUNK_LENGTH);
}
- public static CompressionParams snappy(Integer chunkLength)
+ @VisibleForTesting
+ public static CompressionParams snappy(int chunkLength)
{
- return new CompressionParams(SnappyCompressor.instance, chunkLength, Collections.emptyMap());
+ return snappy(chunkLength, 1.1);
}
+ @VisibleForTesting
+ public static CompressionParams snappy(int chunkLength, double minCompressRatio)
+ {
+ return new CompressionParams(SnappyCompressor.instance, chunkLength, calcMaxCompressedLength(chunkLength, minCompressRatio), minCompressRatio, Collections.emptyMap());
+ }
+
+ @VisibleForTesting
public static CompressionParams deflate()
{
- return deflate(null);
+ return deflate(DEFAULT_CHUNK_LENGTH);
}
- public static CompressionParams deflate(Integer chunkLength)
+ @VisibleForTesting
+ public static CompressionParams deflate(int chunkLength)
{
- return new CompressionParams(DeflateCompressor.instance, chunkLength, Collections.emptyMap());
+ return new CompressionParams(DeflateCompressor.instance, chunkLength, Integer.MAX_VALUE, 0.0, Collections.emptyMap());
}
+ @VisibleForTesting
public static CompressionParams lz4()
{
- return lz4(null);
+ return lz4(DEFAULT_CHUNK_LENGTH);
}
- public static CompressionParams lz4(Integer chunkLength)
+ @VisibleForTesting
+ public static CompressionParams lz4(int chunkLength)
{
- return new CompressionParams(LZ4Compressor.create(Collections.emptyMap()), chunkLength, Collections.emptyMap());
+ return lz4(chunkLength, chunkLength);
}
- public CompressionParams(String sstableCompressorClass, Integer chunkLength, Map<String, String> otherOptions) throws ConfigurationException
+ @VisibleForTesting
+ public static CompressionParams lz4(int chunkLength, int maxCompressedLength)
{
- this(createCompressor(parseCompressorClass(sstableCompressorClass), otherOptions), chunkLength, otherOptions);
+ return new CompressionParams(LZ4Compressor.create(Collections.emptyMap()), chunkLength, maxCompressedLength, calcMinCompressRatio(chunkLength, maxCompressedLength), Collections.emptyMap());
}
- private CompressionParams(ICompressor sstableCompressor, Integer chunkLength, Map<String, String> otherOptions) throws ConfigurationException
+ public static CompressionParams zstd()
+ {
+ return zstd(DEFAULT_CHUNK_LENGTH);
+ }
+
+ public static CompressionParams zstd(Integer chunkLength)
+ {
+ ZstdCompressor compressor = ZstdCompressor.create(Collections.emptyMap());
+ return new CompressionParams(compressor, chunkLength, Integer.MAX_VALUE, DEFAULT_MIN_COMPRESS_RATIO, Collections.emptyMap());
+ }
+
+ @VisibleForTesting
+ public static CompressionParams noop()
+ {
+ NoopCompressor compressor = NoopCompressor.create(Collections.emptyMap());
+ return new CompressionParams(compressor, DEFAULT_CHUNK_LENGTH, Integer.MAX_VALUE, DEFAULT_MIN_COMPRESS_RATIO, Collections.emptyMap());
+ }
+
+ public CompressionParams(String sstableCompressorClass, Map<String, String> otherOptions, int chunkLength, double minCompressRatio) throws ConfigurationException
+ {
+ this(createCompressor(parseCompressorClass(sstableCompressorClass), otherOptions), chunkLength, calcMaxCompressedLength(chunkLength, minCompressRatio), minCompressRatio, otherOptions);
+ }
+
+ static int calcMaxCompressedLength(int chunkLength, double minCompressRatio)
+ {
+ return (int) Math.ceil(Math.min(chunkLength / minCompressRatio, Integer.MAX_VALUE));
+ }
+
+ public CompressionParams(String sstableCompressorClass, int chunkLength, int maxCompressedLength, Map<String, String> otherOptions) throws ConfigurationException
+ {
+ this(createCompressor(parseCompressorClass(sstableCompressorClass), otherOptions), chunkLength, maxCompressedLength, calcMinCompressRatio(chunkLength, maxCompressedLength), otherOptions);
+ }
+
+ static double calcMinCompressRatio(int chunkLength, int maxCompressedLength)
+ {
+ if (maxCompressedLength == Integer.MAX_VALUE)
+ return 0;
+ return chunkLength * 1.0 / maxCompressedLength;
+ }
+
+ private CompressionParams(ICompressor sstableCompressor, int chunkLength, int maxCompressedLength, double minCompressRatio, Map<String, String> otherOptions) throws ConfigurationException
{
this.sstableCompressor = sstableCompressor;
this.chunkLength = chunkLength;
this.otherOptions = ImmutableMap.copyOf(otherOptions);
+ this.minCompressRatio = minCompressRatio;
+ this.maxCompressedLength = maxCompressedLength;
}
public CompressionParams copy()
{
- return new CompressionParams(sstableCompressor, chunkLength, otherOptions);
+ return new CompressionParams(sstableCompressor, chunkLength, maxCompressedLength, minCompressRatio, otherOptions);
}
/**
@@ -184,7 +265,12 @@
public int chunkLength()
{
- return chunkLength == null ? DEFAULT_CHUNK_LENGTH : chunkLength;
+ return chunkLength;
+ }
+
+ public int maxCompressedLength()
+ {
+ return maxCompressedLength;
}
private static Class<?> parseCompressorClass(String className) throws ConfigurationException
@@ -312,7 +398,7 @@
* @param options the options
* @return the chunk length value
*/
- private static Integer removeChunkLength(Map<String, String> options)
+ private static int removeChunkLength(Map<String, String> options)
{
if (options.containsKey(CHUNK_LENGTH_IN_KB))
{
@@ -339,7 +425,23 @@
return parseChunkLength(options.remove(CHUNK_LENGTH_KB));
}
- return null;
+ return DEFAULT_CHUNK_LENGTH;
+ }
+
+ /**
+ * Removes the min compress ratio option from the specified set of option.
+ *
+ * @param options the options
+ * @return the min compress ratio, used to calculate max chunk size to write compressed
+ */
+ private static double removeMinCompressRatio(Map<String, String> options)
+ {
+ String ratio = options.remove(MIN_COMPRESS_RATIO);
+ if (ratio != null)
+ {
+ return Double.parseDouble(ratio);
+ }
+ return DEFAULT_MIN_COMPRESS_RATIO;
}
/**
@@ -420,25 +522,17 @@
public void validate() throws ConfigurationException
{
// if chunk length was not set (chunkLength == null), this is fine, default will be used
- if (chunkLength != null)
- {
- if (chunkLength <= 0)
- throw new ConfigurationException("Invalid negative or null " + CHUNK_LENGTH_IN_KB);
+ if (chunkLength <= 0)
+ throw new ConfigurationException("Invalid negative or null " + CHUNK_LENGTH_IN_KB);
- int c = chunkLength;
- boolean found = false;
- while (c != 0)
- {
- if ((c & 0x01) != 0)
- {
- if (found)
- throw new ConfigurationException(CHUNK_LENGTH_IN_KB + " must be a power of 2");
- else
- found = true;
- }
- c >>= 1;
- }
- }
+ if ((chunkLength & (chunkLength - 1)) != 0)
+ throw new ConfigurationException(CHUNK_LENGTH_IN_KB + " must be a power of 2");
+
+ if (maxCompressedLength < 0)
+ throw new ConfigurationException("Invalid negative " + MIN_COMPRESS_RATIO);
+
+ if (maxCompressedLength > chunkLength && maxCompressedLength < Integer.MAX_VALUE)
+ throw new ConfigurationException(MIN_COMPRESS_RATIO + " can either be 0 or greater than or equal to 1");
}
public Map<String, String> asMap()
@@ -449,6 +543,8 @@
Map<String, String> options = new HashMap<>(otherOptions);
options.put(CLASS, sstableCompressor.getClass().getName());
options.put(CHUNK_LENGTH_IN_KB, chunkLengthInKB());
+ if (minCompressRatio != DEFAULT_MIN_COMPRESS_RATIO)
+ options.put(MIN_COMPRESS_RATIO, String.valueOf(minCompressRatio));
return options;
}
@@ -468,24 +564,28 @@
return crcCheckChance;
}
+ public boolean shouldCheckCrc()
+ {
+ double checkChance = getCrcCheckChance();
+ return checkChance >= 1d ||
+ (checkChance > 0d && checkChance > ThreadLocalRandom.current().nextDouble());
+ }
+
@Override
public boolean equals(Object obj)
{
if (obj == this)
- {
return true;
- }
- else if (obj == null || obj.getClass() != getClass())
- {
+
+ if (!(obj instanceof CompressionParams))
return false;
- }
CompressionParams cp = (CompressionParams) obj;
- return new EqualsBuilder()
- .append(sstableCompressor, cp.sstableCompressor)
- .append(chunkLength(), cp.chunkLength())
- .append(otherOptions, cp.otherOptions)
- .isEquals();
+
+ return Objects.equal(sstableCompressor, cp.sstableCompressor)
+ && chunkLength == cp.chunkLength
+ && otherOptions.equals(cp.otherOptions)
+ && minCompressRatio == cp.minCompressRatio;
}
@Override
@@ -493,8 +593,9 @@
{
return new HashCodeBuilder(29, 1597)
.append(sstableCompressor)
- .append(chunkLength())
+ .append(chunkLength)
.append(otherOptions)
+ .append(minCompressRatio)
.toHashCode();
}
@@ -510,6 +611,11 @@
out.writeUTF(entry.getValue());
}
out.writeInt(parameters.chunkLength());
+ if (version >= MessagingService.VERSION_40)
+ out.writeInt(parameters.maxCompressedLength);
+ else
+ if (parameters.maxCompressedLength != Integer.MAX_VALUE)
+ throw new UnsupportedOperationException("Cannot stream SSTables with uncompressed chunks to pre-4.0 nodes.");
}
public CompressionParams deserialize(DataInputPlus in, int version) throws IOException
@@ -524,10 +630,14 @@
options.put(key, value);
}
int chunkLength = in.readInt();
+ int minCompressRatio = Integer.MAX_VALUE; // Earlier Cassandra cannot use uncompressed chunks.
+ if (version >= MessagingService.VERSION_40)
+ minCompressRatio = in.readInt();
+
CompressionParams parameters;
try
{
- parameters = new CompressionParams(compressorName, chunkLength, options);
+ parameters = new CompressionParams(compressorName, chunkLength, minCompressRatio, options);
}
catch (ConfigurationException e)
{
@@ -546,6 +656,8 @@
size += TypeSizes.sizeof(entry.getValue());
}
size += TypeSizes.sizeof(parameters.chunkLength());
+ if (version >= MessagingService.VERSION_40)
+ size += TypeSizes.sizeof(parameters.maxCompressedLength());
return size;
}
}
diff --git a/src/java/org/apache/cassandra/schema/Diff.java b/src/java/org/apache/cassandra/schema/Diff.java
new file mode 100644
index 0000000..7112c85
--- /dev/null
+++ b/src/java/org/apache/cassandra/schema/Diff.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.schema;
+
+import com.google.common.collect.ImmutableCollection;
+import com.google.common.collect.Iterables;
+
+public class Diff<T extends Iterable, S>
+{
+ public final T created;
+ public final T dropped;
+ public final ImmutableCollection<Altered<S>> altered;
+
+ Diff(T created, T dropped, ImmutableCollection<Altered<S>> altered)
+ {
+ this.created = created;
+ this.dropped = dropped;
+ this.altered = altered;
+ }
+
+ boolean isEmpty()
+ {
+ return Iterables.isEmpty(created) && Iterables.isEmpty(dropped) && Iterables.isEmpty(altered);
+ }
+
+ Iterable<Altered<S>> altered(Difference kind)
+ {
+ return Iterables.filter(altered, a -> a.kind == kind);
+ }
+
+ public static final class Altered<T>
+ {
+ public final T before;
+ public final T after;
+ public final Difference kind;
+
+ Altered(T before, T after, Difference kind)
+ {
+ this.before = before;
+ this.after = after;
+ this.kind = kind;
+ }
+
+ public String toString()
+ {
+ return String.format("%s -> %s (%s)", before, after, kind);
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/schema/Difference.java b/src/java/org/apache/cassandra/schema/Difference.java
new file mode 100644
index 0000000..4f1aea9
--- /dev/null
+++ b/src/java/org/apache/cassandra/schema/Difference.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.schema;
+
+public enum Difference
+{
+ /**
+ * Two schema objects are considered to differ DEEP-ly if one or more of their nested schema objects differ.
+ *
+ * For example, if a table T has a column c of type U, where U is a user defined type, then upon altering U table
+ * T0 (before alter) will differ DEEP-ly from table T1 (after alter).
+ */
+ DEEP,
+
+ /**
+ *
+ * Two schema objects are considered to differ DEEP-ly if their direct structure is altered.
+ *
+ * For example, if a table T is altered to add a new column, a different compaction strategy, or a new description,
+ * then it will differ SHALLOW-ly from the original.
+ */
+ SHALLOW
+}
diff --git a/src/java/org/apache/cassandra/schema/DroppedColumn.java b/src/java/org/apache/cassandra/schema/DroppedColumn.java
new file mode 100644
index 0000000..90dfe65
--- /dev/null
+++ b/src/java/org/apache/cassandra/schema/DroppedColumn.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.schema;
+
+import com.google.common.base.MoreObjects;
+import com.google.common.base.Objects;
+
+public final class DroppedColumn
+{
+ public final ColumnMetadata column;
+ public final long droppedTime; // drop timestamp, in microseconds, yet with millisecond granularity
+
+ public DroppedColumn(ColumnMetadata column, long droppedTime)
+ {
+ this.column = column;
+ this.droppedTime = droppedTime;
+ }
+
+ @Override
+ public boolean equals(Object o)
+ {
+ if (this == o)
+ return true;
+
+ if (!(o instanceof DroppedColumn))
+ return false;
+
+ DroppedColumn dc = (DroppedColumn) o;
+
+ return column.equals(dc.column) && droppedTime == dc.droppedTime;
+ }
+
+ @Override
+ public int hashCode()
+ {
+ return Objects.hashCode(column, droppedTime);
+ }
+
+ @Override
+ public String toString()
+ {
+ return MoreObjects.toStringHelper(this).add("column", column).add("droppedTime", droppedTime).toString();
+ }
+}
diff --git a/src/java/org/apache/cassandra/schema/Functions.java b/src/java/org/apache/cassandra/schema/Functions.java
index a936d81..c5de3b8 100644
--- a/src/java/org/apache/cassandra/schema/Functions.java
+++ b/src/java/org/apache/cassandra/schema/Functions.java
@@ -17,16 +17,21 @@
*/
package org.apache.cassandra.schema;
+import java.nio.ByteBuffer;
import java.util.*;
+import java.util.function.Predicate;
import java.util.stream.Collectors;
import java.util.stream.Stream;
-import com.google.common.collect.ImmutableMultimap;
+import com.google.common.collect.*;
import org.apache.cassandra.cql3.functions.*;
import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.UserType;
-import static com.google.common.collect.Iterables.filter;
+import static java.util.stream.Collectors.toList;
+
+import static com.google.common.collect.Iterables.any;
/**
* An immutable container for a keyspace's UDAs and UDFs (and, in case of {@link org.apache.cassandra.db.SystemKeyspace},
@@ -34,6 +39,21 @@
*/
public final class Functions implements Iterable<Function>
{
+ public enum Filter implements Predicate<Function>
+ {
+ ALL, UDF, UDA;
+
+ public boolean test(Function function)
+ {
+ switch (this)
+ {
+ case UDF: return function instanceof UDFunction;
+ case UDA: return function instanceof UDAggregate;
+ default: return true;
+ }
+ }
+ }
+
private final ImmutableMultimap<FunctionName, Function> functions;
private Functions(Builder builder)
@@ -66,12 +86,17 @@
return functions.values().stream();
}
+ public int size()
+ {
+ return functions.size();
+ }
+
/**
* @return a stream of keyspace's UDFs
*/
public Stream<UDFunction> udfs()
{
- return stream().filter(f -> f instanceof UDFunction).map(f -> (UDFunction) f);
+ return stream().filter(Filter.UDF).map(f -> (UDFunction) f);
}
/**
@@ -79,16 +104,32 @@
*/
public Stream<UDAggregate> udas()
{
- return stream().filter(f -> f instanceof UDAggregate).map(f -> (UDAggregate) f);
+ return stream().filter(Filter.UDA).map(f -> (UDAggregate) f);
+ }
+
+ public Iterable<Function> referencingUserType(ByteBuffer name)
+ {
+ return Iterables.filter(this, f -> f.referencesUserType(name));
+ }
+
+ public Functions withUpdatedUserType(UserType udt)
+ {
+ if (!any(this, f -> f.referencesUserType(udt.name)))
+ return this;
+
+ Collection<UDFunction> udfs = udfs().map(f -> f.withUpdatedUserType(udt)).collect(toList());
+ Collection<UDAggregate> udas = udas().map(f -> f.withUpdatedUserType(udfs, udt)).collect(toList());
+
+ return builder().add(udfs).add(udas).build();
}
/**
- * @return a collection of aggregates that use the provided function as either a state or a final function
+ * @return a stream of aggregates that use the provided function as either a state or a final function
* @param function the referree function
*/
- public Collection<UDAggregate> aggregatesUsingFunction(Function function)
+ public Stream<UDAggregate> aggregatesUsingFunction(Function function)
{
- return udas().filter(uda -> uda.hasReferenceTo(function)).collect(Collectors.toList());
+ return udas().filter(uda -> uda.hasReferenceTo(function));
}
/**
@@ -103,19 +144,59 @@
}
/**
+ * Get all UDFs overloads with the specified name
+ *
+ * @param name fully qualified function name
+ * @return an empty list if the function name is not found; a non-empty collection of {@link UDFunction} otherwise
+ */
+ public Collection<UDFunction> getUdfs(FunctionName name)
+ {
+ return functions.get(name)
+ .stream()
+ .filter(Filter.UDF)
+ .map(f -> (UDFunction) f)
+ .collect(Collectors.toList());
+ }
+
+ /**
+ * Get all UDAs overloads with the specified name
+ *
+ * @param name fully qualified function name
+ * @return an empty list if the function name is not found; a non-empty collection of {@link UDAggregate} otherwise
+ */
+ public Collection<UDAggregate> getUdas(FunctionName name)
+ {
+ return functions.get(name)
+ .stream()
+ .filter(Filter.UDA)
+ .map(f -> (UDAggregate) f)
+ .collect(Collectors.toList());
+ }
+
+ public Optional<Function> find(FunctionName name, List<AbstractType<?>> argTypes)
+ {
+ return find(name, argTypes, Filter.ALL);
+ }
+
+ /**
* Find the function with the specified name
*
* @param name fully qualified function name
* @param argTypes function argument types
* @return an empty {@link Optional} if the function name is not found; a non-empty optional of {@link Function} otherwise
*/
- public Optional<Function> find(FunctionName name, List<AbstractType<?>> argTypes)
+ public Optional<Function> find(FunctionName name, List<AbstractType<?>> argTypes, Filter filter)
{
return get(name).stream()
- .filter(fun -> typesMatch(fun.argTypes(), argTypes))
+ .filter(filter.and(fun -> typesMatch(fun.argTypes(), argTypes)))
.findAny();
}
+ public boolean isEmpty()
+ {
+ return functions.isEmpty();
+ }
+
/*
* We need to compare the CQL3 representation of the type because comparing
* the AbstractType will fail for example if a UDT has been changed.
@@ -129,7 +210,7 @@
* or
* ALTER TYPE foo RENAME ...
*/
- public static boolean typesMatch(AbstractType<?> t1, AbstractType<?> t2)
+ private static boolean typesMatch(AbstractType<?> t1, AbstractType<?> t2)
{
return t1.freeze().asCQL3Type().toString().equals(t2.freeze().asCQL3Type().toString());
}
@@ -159,6 +240,13 @@
return h;
}
+ public Functions filter(Predicate<Function> predicate)
+ {
+ Builder builder = builder();
+ stream().filter(predicate).forEach(builder::add);
+ return builder.build();
+ }
+
/**
* Create a Functions instance with the provided function added
*/
@@ -178,7 +266,19 @@
Function fun =
find(name, argTypes).orElseThrow(() -> new IllegalStateException(String.format("Function %s doesn't exists", name)));
- return builder().add(filter(this, f -> f != fun)).build();
+ return without(fun);
+ }
+
+ public Functions without(Function function)
+ {
+ return builder().add(Iterables.filter(this, f -> f != function)).build();
+ }
+
+ public Functions withAddedOrUpdated(Function function)
+ {
+ return builder().add(Iterables.filter(this, f -> !(f.name().equals(function.name()) && Functions.typesMatch(f.argTypes(), function.argTypes()))))
+ .add(function)
+ .build();
}
@Override
@@ -206,7 +306,7 @@
private Builder()
{
// we need deterministic iteration order; otherwise Functions.equals() breaks down
- functions.orderValuesBy((f1, f2) -> Integer.compare(f1.hashCode(), f2.hashCode()));
+ functions.orderValuesBy(Comparator.comparingInt(Object::hashCode));
}
public Functions build()
@@ -227,10 +327,52 @@
return this;
}
- public Builder add(Iterable<? extends Function> funs)
+ public Builder add(Iterable<? extends Function> funs)
{
funs.forEach(this::add);
return this;
}
}
+
+ @SuppressWarnings("unchecked")
+ static FunctionsDiff<UDFunction> udfsDiff(Functions before, Functions after)
+ {
+ return (FunctionsDiff<UDFunction>) FunctionsDiff.diff(before, after, Filter.UDF);
+ }
+
+ @SuppressWarnings("unchecked")
+ static FunctionsDiff<UDAggregate> udasDiff(Functions before, Functions after)
+ {
+ return (FunctionsDiff<UDAggregate>) FunctionsDiff.diff(before, after, Filter.UDA);
+ }
+
+ public static final class FunctionsDiff<T extends Function> extends Diff<Functions, T>
+ {
+ static final FunctionsDiff NONE = new FunctionsDiff<>(Functions.none(), Functions.none(), ImmutableList.of());
+
+ private FunctionsDiff(Functions created, Functions dropped, ImmutableCollection<Altered<T>> altered)
+ {
+ super(created, dropped, altered);
+ }
+
+ private static FunctionsDiff diff(Functions before, Functions after, Filter filter)
+ {
+ if (before == after)
+ return NONE;
+
+ Functions created = after.filter(filter.and(k -> !before.find(k.name(), k.argTypes(), filter).isPresent()));
+ Functions dropped = before.filter(filter.and(k -> !after.find(k.name(), k.argTypes(), filter).isPresent()));
+
+ ImmutableList.Builder<Altered<Function>> altered = ImmutableList.builder();
+ before.stream().filter(filter).forEach(functionBefore ->
+ {
+ after.find(functionBefore.name(), functionBefore.argTypes(), filter).ifPresent(functionAfter ->
+ {
+ functionBefore.compare(functionAfter).ifPresent(kind -> altered.add(new Altered<>(functionBefore, functionAfter, kind)));
+ });
+ });
+
+ return new FunctionsDiff<>(created, dropped, altered.build());
+ }
+ }
}
diff --git a/src/java/org/apache/cassandra/schema/IndexMetadata.java b/src/java/org/apache/cassandra/schema/IndexMetadata.java
index 04e06ab..81f48ff 100644
--- a/src/java/org/apache/cassandra/schema/IndexMetadata.java
+++ b/src/java/org/apache/cassandra/schema/IndexMetadata.java
@@ -31,10 +31,11 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
-import org.apache.cassandra.cql3.statements.IndexTarget;
+import org.apache.cassandra.cql3.ColumnIdentifier;
+import org.apache.cassandra.cql3.CqlBuilder;
+import org.apache.cassandra.cql3.statements.schema.IndexTarget;
import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.exceptions.UnknownIndexException;
import org.apache.cassandra.index.Index;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
@@ -47,7 +48,7 @@
public final class IndexMetadata
{
private static final Logger logger = LoggerFactory.getLogger(IndexMetadata.class);
-
+
private static final Pattern PATTERN_NON_WORD_CHAR = Pattern.compile("\\W");
private static final Pattern PATTERN_WORD_CHARS = Pattern.compile("\\w+");
@@ -76,56 +77,19 @@
this.kind = kind;
}
- public static IndexMetadata fromLegacyMetadata(CFMetaData cfm,
- ColumnDefinition column,
- String name,
- Kind kind,
- Map<String, String> options)
- {
- Map<String, String> newOptions = new HashMap<>();
- if (options != null)
- newOptions.putAll(options);
-
- IndexTarget target;
- if (newOptions.containsKey(IndexTarget.INDEX_KEYS_OPTION_NAME))
- {
- newOptions.remove(IndexTarget.INDEX_KEYS_OPTION_NAME);
- target = new IndexTarget(column.name, IndexTarget.Type.KEYS);
- }
- else if (newOptions.containsKey(IndexTarget.INDEX_ENTRIES_OPTION_NAME))
- {
- newOptions.remove(IndexTarget.INDEX_KEYS_OPTION_NAME);
- target = new IndexTarget(column.name, IndexTarget.Type.KEYS_AND_VALUES);
- }
- else
- {
- if (column.type.isCollection() && !column.type.isMultiCell())
- {
- target = new IndexTarget(column.name, IndexTarget.Type.FULL);
- }
- else
- {
- target = new IndexTarget(column.name, IndexTarget.Type.VALUES);
- }
- }
- newOptions.put(IndexTarget.TARGET_OPTION_NAME, target.asCqlString(cfm));
- return new IndexMetadata(name, newOptions, kind);
- }
-
public static IndexMetadata fromSchemaMetadata(String name, Kind kind, Map<String, String> options)
{
return new IndexMetadata(name, options, kind);
}
- public static IndexMetadata fromIndexTargets(CFMetaData cfm,
- List<IndexTarget> targets,
+ public static IndexMetadata fromIndexTargets(List<IndexTarget> targets,
String name,
Kind kind,
Map<String, String> options)
{
Map<String, String> newOptions = new HashMap<>(options);
newOptions.put(IndexTarget.TARGET_OPTION_NAME, targets.stream()
- .map(target -> target.asCqlString(cfm))
+ .map(target -> target.asCqlString())
.collect(Collectors.joining(", ")));
return new IndexMetadata(name, newOptions, kind);
}
@@ -135,15 +99,17 @@
return name != null && !name.isEmpty() && PATTERN_WORD_CHARS.matcher(name).matches();
}
- public static String getDefaultIndexName(String cfName, String root)
+ public static String generateDefaultIndexName(String table, ColumnIdentifier column)
{
- if (root == null)
- return PATTERN_NON_WORD_CHAR.matcher(cfName + "_" + "idx").replaceAll("");
- else
- return PATTERN_NON_WORD_CHAR.matcher(cfName + "_" + root + "_idx").replaceAll("");
+ return PATTERN_NON_WORD_CHAR.matcher(table + "_" + column.toString() + "_idx").replaceAll("");
}
- public void validate(CFMetaData cfm)
+ public static String generateDefaultIndexName(String table)
+ {
+ return PATTERN_NON_WORD_CHAR.matcher(table + "_" + "idx").replaceAll("");
+ }
+
+ public void validate(TableMetadata table)
{
if (!isNameValid(name))
throw new ConfigurationException("Illegal index name " + name);
@@ -158,29 +124,25 @@
name, IndexTarget.CUSTOM_INDEX_OPTION_NAME));
String className = options.get(IndexTarget.CUSTOM_INDEX_OPTION_NAME);
Class<Index> indexerClass = FBUtilities.classForName(className, "custom indexer");
- if(!Index.class.isAssignableFrom(indexerClass))
+ if (!Index.class.isAssignableFrom(indexerClass))
throw new ConfigurationException(String.format("Specified Indexer class (%s) does not implement the Indexer interface", className));
- validateCustomIndexOptions(cfm, indexerClass, options);
+ validateCustomIndexOptions(table, indexerClass, options);
}
}
- private void validateCustomIndexOptions(CFMetaData cfm,
- Class<? extends Index> indexerClass,
- Map<String, String> options)
- throws ConfigurationException
+ private void validateCustomIndexOptions(TableMetadata table, Class<? extends Index> indexerClass, Map<String, String> options)
{
try
{
- Map<String, String> filteredOptions =
- Maps.filterKeys(options,key -> !key.equals(IndexTarget.CUSTOM_INDEX_OPTION_NAME));
+ Map<String, String> filteredOptions = Maps.filterKeys(options, key -> !key.equals(IndexTarget.CUSTOM_INDEX_OPTION_NAME));
if (filteredOptions.isEmpty())
return;
- Map<?,?> unknownOptions;
+ Map<?, ?> unknownOptions;
try
{
- unknownOptions = (Map) indexerClass.getMethod("validateOptions", Map.class, CFMetaData.class).invoke(null, filteredOptions, cfm);
+ unknownOptions = (Map) indexerClass.getMethod("validateOptions", Map.class, TableMetadata.class).invoke(null, filteredOptions, table);
}
catch (NoSuchMethodException e)
{
@@ -226,6 +188,7 @@
return kind == Kind.COMPOSITES;
}
+ @Override
public int hashCode()
{
return Objects.hashCode(id, name, kind, options);
@@ -234,9 +197,10 @@
public boolean equalsWithoutName(IndexMetadata other)
{
return Objects.equal(kind, other.kind)
- && Objects.equal(options, other.options);
+ && Objects.equal(options, other.options);
}
+ @Override
public boolean equals(Object obj)
{
if (obj == this)
@@ -245,19 +209,65 @@
if (!(obj instanceof IndexMetadata))
return false;
- IndexMetadata other = (IndexMetadata)obj;
+ IndexMetadata other = (IndexMetadata) obj;
return Objects.equal(id, other.id) && Objects.equal(name, other.name) && equalsWithoutName(other);
}
+ @Override
public String toString()
{
return new ToStringBuilder(this)
- .append("id", id.toString())
- .append("name", name)
- .append("kind", kind)
- .append("options", options)
- .build();
+ .append("id", id.toString())
+ .append("name", name)
+ .append("kind", kind)
+ .append("options", options)
+ .build();
+ }
+
+ public String toCqlString(TableMetadata table)
+ {
+ CqlBuilder builder = new CqlBuilder();
+ appendCqlTo(builder, table);
+ return builder.toString();
+ }
+
+ /**
+ * Appends to the specified builder the CQL used to create this index.
+ *
+ * @param builder the builder to which the CQL myst be appended
+ * @param table the parent table
+ */
+ public void appendCqlTo(CqlBuilder builder, TableMetadata table)
+ {
+ if (isCustom())
+ {
+ Map<String, String> copyOptions = new HashMap<>(options);
+
+ builder.append("CREATE CUSTOM INDEX ")
+ .appendQuotingIfNeeded(name)
+ .append(" ON ")
+ .append(table.toString())
+ .append(" (")
+ .append(copyOptions.remove(IndexTarget.TARGET_OPTION_NAME))
+ .append(") USING ")
+ .appendWithSingleQuotes(copyOptions.remove(IndexTarget.CUSTOM_INDEX_OPTION_NAME));
+
+ if (!copyOptions.isEmpty())
+ builder.append(" WITH OPTIONS = ")
+ .append(options);
+ }
+ else
+ {
+ builder.append("CREATE INDEX ")
+ .appendQuotingIfNeeded(name)
+ .append(" ON ")
+ .append(table.toString())
+ .append(" (")
+ .append(options.get(IndexTarget.TARGET_OPTION_NAME))
+ .append(')');
+ }
+ builder.append(';');
}
public static class Serializer
@@ -267,10 +277,10 @@
UUIDSerializer.serializer.serialize(metadata.id, out, version);
}
- public IndexMetadata deserialize(DataInputPlus in, int version, CFMetaData cfm) throws IOException
+ public IndexMetadata deserialize(DataInputPlus in, int version, TableMetadata table) throws IOException
{
UUID id = UUIDSerializer.serializer.deserialize(in, version);
- return cfm.getIndexes().get(id).orElseThrow(() -> new UnknownIndexException(cfm, id));
+ return table.indexes.get(id).orElseThrow(() -> new UnknownIndexException(table, id));
}
public long serializedSize(IndexMetadata metadata, int version)
diff --git a/src/java/org/apache/cassandra/schema/Indexes.java b/src/java/org/apache/cassandra/schema/Indexes.java
index eb49d39..a83be4b 100644
--- a/src/java/org/apache/cassandra/schema/Indexes.java
+++ b/src/java/org/apache/cassandra/schema/Indexes.java
@@ -15,14 +15,15 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.cassandra.schema;
import java.util.*;
+import java.util.stream.Stream;
import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Sets;
-import org.apache.cassandra.config.Schema;
+import static java.lang.String.format;
import static com.google.common.collect.Iterables.filter;
@@ -35,7 +36,7 @@
* support is added for multiple target columns per-index and for indexes with
* TargetType.ROW
*/
-public class Indexes implements Iterable<IndexMetadata>
+public final class Indexes implements Iterable<IndexMetadata>
{
private final ImmutableMap<String, IndexMetadata> indexesByName;
private final ImmutableMap<UUID, IndexMetadata> indexesById;
@@ -56,11 +57,26 @@
return builder().build();
}
+ public static Indexes of(IndexMetadata... indexes)
+ {
+ return builder().add(indexes).build();
+ }
+
+ public static Indexes of(Iterable<IndexMetadata> indexes)
+ {
+ return builder().add(indexes).build();
+ }
+
public Iterator<IndexMetadata> iterator()
{
return indexesByName.values().iterator();
}
+ public Stream<IndexMetadata> stream()
+ {
+ return indexesById.values().stream();
+ }
+
public int size()
{
return indexesByName.size();
@@ -121,7 +137,7 @@
public Indexes with(IndexMetadata index)
{
if (get(index.name).isPresent())
- throw new IllegalStateException(String.format("Index %s already exists", index.name));
+ throw new IllegalStateException(format("Index %s already exists", index.name));
return builder().add(this).add(index).build();
}
@@ -131,7 +147,7 @@
*/
public Indexes without(String name)
{
- IndexMetadata index = get(name).orElseThrow(() -> new IllegalStateException(String.format("Index %s doesn't exist", name)));
+ IndexMetadata index = get(name).orElseThrow(() -> new IllegalStateException(format("Index %s doesn't exist", name)));
return builder().add(filter(this, v -> v != index)).build();
}
@@ -149,6 +165,11 @@
return this == o || (o instanceof Indexes && indexesByName.equals(((Indexes) o).indexesByName));
}
+ public void validate(TableMetadata table)
+ {
+ indexesByName.values().forEach(i -> i.validate(table));
+ }
+
@Override
public int hashCode()
{
@@ -161,20 +182,6 @@
return indexesByName.values().toString();
}
- public static String getAvailableIndexName(String ksName, String cfName, String indexNameRoot)
- {
-
- KeyspaceMetadata ksm = Schema.instance.getKSMetaData(ksName);
- Set<String> existingNames = ksm == null ? new HashSet<>() : ksm.existingIndexNames(null);
- String baseName = IndexMetadata.getDefaultIndexName(cfName, indexNameRoot);
- String acceptedName = baseName;
- int i = 0;
- while (existingNames.contains(acceptedName))
- acceptedName = baseName + '_' + (++i);
-
- return acceptedName;
- }
-
public static final class Builder
{
final ImmutableMap.Builder<String, IndexMetadata> indexesByName = new ImmutableMap.Builder<>();
@@ -196,6 +203,13 @@
return this;
}
+ public Builder add(IndexMetadata... indexes)
+ {
+ for (IndexMetadata index : indexes)
+ add(index);
+ return this;
+ }
+
public Builder add(Iterable<IndexMetadata> indexes)
{
indexes.forEach(this::add);
diff --git a/src/java/org/apache/cassandra/schema/KeyspaceMetadata.java b/src/java/org/apache/cassandra/schema/KeyspaceMetadata.java
index 4fefd44..23c931f 100644
--- a/src/java/org/apache/cassandra/schema/KeyspaceMetadata.java
+++ b/src/java/org/apache/cassandra/schema/KeyspaceMetadata.java
@@ -27,26 +27,46 @@
import com.google.common.base.Objects;
import com.google.common.collect.Iterables;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.SchemaConstants;
-import org.apache.cassandra.config.ViewDefinition;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.CqlBuilder;
+import org.apache.cassandra.cql3.SchemaElement;
+import org.apache.cassandra.cql3.functions.UDAggregate;
+import org.apache.cassandra.cql3.functions.UDFunction;
+import org.apache.cassandra.db.marshal.UserType;
import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.locator.AbstractReplicationStrategy;
+import org.apache.cassandra.schema.Functions.FunctionsDiff;
+import org.apache.cassandra.schema.Tables.TablesDiff;
+import org.apache.cassandra.schema.Types.TypesDiff;
+import org.apache.cassandra.schema.Views.ViewsDiff;
+import org.apache.cassandra.service.StorageService;
+
+import static java.lang.String.format;
+
+import static com.google.common.collect.Iterables.any;
/**
* An immutable representation of keyspace metadata (name, params, tables, types, and functions).
*/
-public final class KeyspaceMetadata
+public final class KeyspaceMetadata implements SchemaElement
{
+ public enum Kind
+ {
+ REGULAR, VIRTUAL
+ }
+
public final String name;
+ public final Kind kind;
public final KeyspaceParams params;
public final Tables tables;
public final Views views;
public final Types types;
public final Functions functions;
- private KeyspaceMetadata(String name, KeyspaceParams params, Tables tables, Views views, Types types, Functions functions)
+ private KeyspaceMetadata(String name, Kind kind, KeyspaceParams params, Tables tables, Views views, Types types, Functions functions)
{
this.name = name;
+ this.kind = kind;
this.params = params;
this.tables = tables;
this.views = views;
@@ -56,73 +76,118 @@
public static KeyspaceMetadata create(String name, KeyspaceParams params)
{
- return new KeyspaceMetadata(name, params, Tables.none(), Views.none(), Types.none(), Functions.none());
+ return new KeyspaceMetadata(name, Kind.REGULAR, params, Tables.none(), Views.none(), Types.none(), Functions.none());
}
public static KeyspaceMetadata create(String name, KeyspaceParams params, Tables tables)
{
- return new KeyspaceMetadata(name, params, tables, Views.none(), Types.none(), Functions.none());
+ return new KeyspaceMetadata(name, Kind.REGULAR, params, tables, Views.none(), Types.none(), Functions.none());
}
public static KeyspaceMetadata create(String name, KeyspaceParams params, Tables tables, Views views, Types types, Functions functions)
{
- return new KeyspaceMetadata(name, params, tables, views, types, functions);
+ return new KeyspaceMetadata(name, Kind.REGULAR, params, tables, views, types, functions);
+ }
+
+ public static KeyspaceMetadata virtual(String name, Tables tables)
+ {
+ return new KeyspaceMetadata(name, Kind.VIRTUAL, KeyspaceParams.local(), tables, Views.none(), Types.none(), Functions.none());
}
public KeyspaceMetadata withSwapped(KeyspaceParams params)
{
- return new KeyspaceMetadata(name, params, tables, views, types, functions);
+ return new KeyspaceMetadata(name, kind, params, tables, views, types, functions);
}
public KeyspaceMetadata withSwapped(Tables regular)
{
- return new KeyspaceMetadata(name, params, regular, views, types, functions);
+ return new KeyspaceMetadata(name, kind, params, regular, views, types, functions);
}
public KeyspaceMetadata withSwapped(Views views)
{
- return new KeyspaceMetadata(name, params, tables, views, types, functions);
+ return new KeyspaceMetadata(name, kind, params, tables, views, types, functions);
}
public KeyspaceMetadata withSwapped(Types types)
{
- return new KeyspaceMetadata(name, params, tables, views, types, functions);
+ return new KeyspaceMetadata(name, kind, params, tables, views, types, functions);
}
public KeyspaceMetadata withSwapped(Functions functions)
{
- return new KeyspaceMetadata(name, params, tables, views, types, functions);
+ return new KeyspaceMetadata(name, kind, params, tables, views, types, functions);
}
- public Iterable<CFMetaData> tablesAndViews()
+ public boolean isVirtual()
{
- return Iterables.concat(tables, views.metadatas());
+ return kind == Kind.VIRTUAL;
+ }
+
+ /**
+ * Returns a new KeyspaceMetadata with all instances of old UDT replaced with the updated version.
+ * Replaces all instances in tables, views, types, and functions.
+ */
+ public KeyspaceMetadata withUpdatedUserType(UserType udt)
+ {
+ return new KeyspaceMetadata(name,
+ kind,
+ params,
+ tables.withUpdatedUserType(udt),
+ views.withUpdatedUserTypes(udt),
+ types.withUpdatedUserType(udt),
+ functions.withUpdatedUserType(udt));
+ }
+
+ public Iterable<TableMetadata> tablesAndViews()
+ {
+ return Iterables.concat(tables, views.allTableMetadata());
}
@Nullable
- public CFMetaData getTableOrViewNullable(String tableOrViewName)
+ public TableMetadata getTableOrViewNullable(String tableOrViewName)
{
- ViewDefinition view = views.getNullable(tableOrViewName);
+ ViewMetadata view = views.getNullable(tableOrViewName);
return view == null
? tables.getNullable(tableOrViewName)
: view.metadata;
}
- public Set<String> existingIndexNames(String cfToExclude)
+ public boolean hasTable(String tableName)
{
- Set<String> indexNames = new HashSet<>();
- for (CFMetaData table : tables)
- if (cfToExclude == null || !table.cfName.equals(cfToExclude))
- for (IndexMetadata index : table.getIndexes())
- indexNames.add(index.name);
- return indexNames;
+ return tables.get(tableName).isPresent();
}
- public Optional<CFMetaData> findIndexedTable(String indexName)
+ public boolean hasView(String viewName)
{
- for (CFMetaData cfm : tablesAndViews())
- if (cfm.getIndexes().has(indexName))
- return Optional.of(cfm);
+ return views.get(viewName).isPresent();
+ }
+
+ public boolean hasIndex(String indexName)
+ {
+ return any(tables, t -> t.indexes.has(indexName));
+ }
+
+ public String findAvailableIndexName(String baseName)
+ {
+ if (!hasIndex(baseName))
+ return baseName;
+
+ int i = 1;
+ do
+ {
+ String name = baseName + '_' + i++;
+ if (!hasIndex(name))
+ return name;
+ }
+ while (true);
+ }
+
+ public Optional<TableMetadata> findIndexedTable(String indexName)
+ {
+ for (TableMetadata table : tablesAndViews())
+ if (table.indexes.has(indexName))
+ return Optional.of(table);
return Optional.empty();
}
@@ -130,7 +195,7 @@
@Override
public int hashCode()
{
- return Objects.hashCode(name, params, tables, views, functions, types);
+ return Objects.hashCode(name, kind, params, tables, views, functions, types);
}
@Override
@@ -145,6 +210,7 @@
KeyspaceMetadata other = (KeyspaceMetadata) o;
return name.equals(other.name)
+ && kind == other.kind
&& params.equals(other.params)
&& tables.equals(other.tables)
&& views.equals(other.views)
@@ -157,6 +223,7 @@
{
return MoreObjects.toStringHelper(this)
.add("name", name)
+ .add("kind", kind)
.add("params", params)
.add("tables", tables)
.add("views", views)
@@ -165,14 +232,158 @@
.toString();
}
+ @Override
+ public SchemaElementType elementType()
+ {
+ return SchemaElementType.KEYSPACE;
+ }
+
+ @Override
+ public String elementKeyspace()
+ {
+ return name;
+ }
+
+ @Override
+ public String elementName()
+ {
+ return name;
+ }
+
+ @Override
+ public String toCqlString(boolean withInternals)
+ {
+ CqlBuilder builder = new CqlBuilder();
+ if (isVirtual())
+ {
+ builder.append("/*")
+ .newLine()
+ .append("Warning: Keyspace ")
+ .appendQuotingIfNeeded(name)
+ .append(" is a virtual keyspace and cannot be recreated with CQL.")
+ .newLine()
+ .append("Structure, for reference:")
+ .newLine()
+ .append("VIRTUAL KEYSPACE ")
+ .appendQuotingIfNeeded(name)
+ .append(';')
+ .newLine()
+ .append("*/")
+ .toString();
+ }
+ else
+ {
+ builder.append("CREATE KEYSPACE ")
+ .appendQuotingIfNeeded(name)
+ .append(" WITH replication = ");
+
+ params.replication.appendCqlTo(builder);
+
+ builder.append(" AND durable_writes = ")
+ .append(params.durableWrites)
+ .append(';')
+ .toString();
+ }
+ return builder.toString();
+ }
+
public void validate()
{
- if (!CFMetaData.isNameValid(name))
- throw new ConfigurationException(String.format("Keyspace name must not be empty, more than %s characters long, "
- + "or contain non-alphanumeric-underscore characters (got \"%s\")",
- SchemaConstants.NAME_LENGTH,
- name));
+ if (!SchemaConstants.isValidName(name))
+ {
+ throw new ConfigurationException(format("Keyspace name must not be empty, more than %s characters long, "
+ + "or contain non-alphanumeric-underscore characters (got \"%s\")",
+ SchemaConstants.NAME_LENGTH,
+ name));
+ }
+
params.validate(name);
- tablesAndViews().forEach(CFMetaData::validate);
+
+ tablesAndViews().forEach(TableMetadata::validate);
+
+ Set<String> indexNames = new HashSet<>();
+ for (TableMetadata table : tables)
+ {
+ for (IndexMetadata index : table.indexes)
+ {
+ if (indexNames.contains(index.name))
+ throw new ConfigurationException(format("Duplicate index name %s in keyspace %s", index.name, name));
+
+ indexNames.add(index.name);
+ }
+ }
+ }
+
+ public AbstractReplicationStrategy createReplicationStrategy()
+ {
+ return AbstractReplicationStrategy.createReplicationStrategy(name,
+ params.replication.klass,
+ StorageService.instance.getTokenMetadata(),
+ DatabaseDescriptor.getEndpointSnitch(),
+ params.replication.options);
+ }
+
+ static Optional<KeyspaceDiff> diff(KeyspaceMetadata before, KeyspaceMetadata after)
+ {
+ return KeyspaceDiff.diff(before, after);
+ }
+
+ public static final class KeyspaceDiff
+ {
+ public final KeyspaceMetadata before;
+ public final KeyspaceMetadata after;
+
+ public final TablesDiff tables;
+ public final ViewsDiff views;
+ public final TypesDiff types;
+
+ public final FunctionsDiff<UDFunction> udfs;
+ public final FunctionsDiff<UDAggregate> udas;
+
+ private KeyspaceDiff(KeyspaceMetadata before,
+ KeyspaceMetadata after,
+ TablesDiff tables,
+ ViewsDiff views,
+ TypesDiff types,
+ FunctionsDiff<UDFunction> udfs,
+ FunctionsDiff<UDAggregate> udas)
+ {
+ this.before = before;
+ this.after = after;
+ this.tables = tables;
+ this.views = views;
+ this.types = types;
+ this.udfs = udfs;
+ this.udas = udas;
+ }
+
+ private static Optional<KeyspaceDiff> diff(KeyspaceMetadata before, KeyspaceMetadata after)
+ {
+ if (before == after)
+ return Optional.empty();
+
+ if (!before.name.equals(after.name))
+ {
+ String msg = String.format("Attempting to diff two keyspaces with different names ('%s' and '%s')", before.name, after.name);
+ throw new IllegalArgumentException(msg);
+ }
+
+ TablesDiff tables = Tables.diff(before.tables, after.tables);
+ ViewsDiff views = Views.diff(before.views, after.views);
+ TypesDiff types = Types.diff(before.types, after.types);
+
+ @SuppressWarnings("unchecked") FunctionsDiff<UDFunction> udfs = FunctionsDiff.NONE;
+ @SuppressWarnings("unchecked") FunctionsDiff<UDAggregate> udas = FunctionsDiff.NONE;
+ if (before.functions != after.functions)
+ {
+ udfs = Functions.udfsDiff(before.functions, after.functions);
+ udas = Functions.udasDiff(before.functions, after.functions);
+ }
+
+ if (before.params.equals(after.params) && tables.isEmpty() && views.isEmpty() && types.isEmpty() && udfs.isEmpty() && udas.isEmpty())
+ return Optional.empty();
+
+ return Optional.of(new KeyspaceDiff(before, after, tables, views, types, udfs, udas));
+ }
}
}
diff --git a/src/java/org/apache/cassandra/schema/KeyspaceParams.java b/src/java/org/apache/cassandra/schema/KeyspaceParams.java
index 1deaa29..cc46474 100644
--- a/src/java/org/apache/cassandra/schema/KeyspaceParams.java
+++ b/src/java/org/apache/cassandra/schema/KeyspaceParams.java
@@ -31,8 +31,8 @@
public static final boolean DEFAULT_DURABLE_WRITES = true;
/**
- * This determines durable writes for the {@link org.apache.cassandra.config.SchemaConstants#SCHEMA_KEYSPACE_NAME}
- * and {@link org.apache.cassandra.config.SchemaConstants#SYSTEM_KEYSPACE_NAME} keyspaces,
+ * This determines durable writes for the {@link org.apache.cassandra.schema.SchemaConstants#SCHEMA_KEYSPACE_NAME}
+ * and {@link org.apache.cassandra.schema.SchemaConstants#SYSTEM_KEYSPACE_NAME} keyspaces,
* the only reason it is not final is for commitlog unit tests. It should only be changed for testing purposes.
*/
@VisibleForTesting
@@ -74,6 +74,11 @@
return new KeyspaceParams(true, ReplicationParams.simple(replicationFactor));
}
+ public static KeyspaceParams simple(String replicationFactor)
+ {
+ return new KeyspaceParams(true, ReplicationParams.simple(replicationFactor));
+ }
+
public static KeyspaceParams simpleTransient(int replicationFactor)
{
return new KeyspaceParams(false, ReplicationParams.simple(replicationFactor));
diff --git a/src/java/org/apache/cassandra/schema/Keyspaces.java b/src/java/org/apache/cassandra/schema/Keyspaces.java
index 8c0a63e..1938d93 100644
--- a/src/java/org/apache/cassandra/schema/Keyspaces.java
+++ b/src/java/org/apache/cassandra/schema/Keyspaces.java
@@ -18,20 +18,28 @@
package org.apache.cassandra.schema;
import java.util.Iterator;
+import java.util.Optional;
+import java.util.Set;
import java.util.function.Predicate;
import java.util.stream.Stream;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.MapDifference;
-import com.google.common.collect.Maps;
+import javax.annotation.Nullable;
+
+import com.google.common.collect.*;
+
+import org.apache.cassandra.schema.KeyspaceMetadata.KeyspaceDiff;
public final class Keyspaces implements Iterable<KeyspaceMetadata>
{
+ private static final Keyspaces NONE = builder().build();
+
private final ImmutableMap<String, KeyspaceMetadata> keyspaces;
+ private final ImmutableMap<TableId, TableMetadata> tables;
private Keyspaces(Builder builder)
{
keyspaces = builder.keyspaces.build();
+ tables = builder.tables.build();
}
public static Builder builder()
@@ -41,7 +49,7 @@
public static Keyspaces none()
{
- return builder().build();
+ return NONE;
}
public static Keyspaces of(KeyspaceMetadata... keyspaces)
@@ -59,6 +67,44 @@
return keyspaces.values().stream();
}
+ public Set<String> names()
+ {
+ return keyspaces.keySet();
+ }
+
+ /**
+ * Get the keyspace with the specified name
+ *
+ * @param name a non-qualified keyspace name
+ * @return an empty {@link Optional} if the table name is not found; a non-empty optional of {@link KeyspaceMetadata} otherwise
+ */
+ public Optional<KeyspaceMetadata> get(String name)
+ {
+ return Optional.ofNullable(keyspaces.get(name));
+ }
+
+ @Nullable
+ public KeyspaceMetadata getNullable(String name)
+ {
+ return keyspaces.get(name);
+ }
+
+ public boolean containsKeyspace(String name)
+ {
+ return keyspaces.containsKey(name);
+ }
+
+ @Nullable
+ public TableMetadata getTableOrViewNullable(TableId id)
+ {
+ return tables.get(id);
+ }
+
+ public boolean isEmpty()
+ {
+ return keyspaces.isEmpty();
+ }
+
public Keyspaces filter(Predicate<KeyspaceMetadata> predicate)
{
Builder builder = builder();
@@ -66,9 +112,28 @@
return builder.build();
}
- MapDifference<String, KeyspaceMetadata> diff(Keyspaces other)
+ /**
+ * Creates a Keyspaces instance with the keyspace with the provided name removed
+ */
+ public Keyspaces without(String name)
{
- return Maps.difference(keyspaces, other.keyspaces);
+ KeyspaceMetadata keyspace = getNullable(name);
+ if (keyspace == null)
+ throw new IllegalStateException(String.format("Keyspace %s doesn't exists", name));
+
+ return filter(k -> k != keyspace);
+ }
+
+ public Keyspaces withAddedOrUpdated(KeyspaceMetadata keyspace)
+ {
+ return builder().add(Iterables.filter(this, k -> !k.name.equals(keyspace.name)))
+ .add(keyspace)
+ .build();
+ }
+
+ public void validate()
+ {
+ keyspaces.values().forEach(KeyspaceMetadata::validate);
}
@Override
@@ -92,6 +157,7 @@
public static final class Builder
{
private final ImmutableMap.Builder<String, KeyspaceMetadata> keyspaces = new ImmutableMap.Builder<>();
+ private final ImmutableMap.Builder<TableId, TableMetadata> tables = new ImmutableMap.Builder<>();
private Builder()
{
@@ -105,6 +171,10 @@
public Builder add(KeyspaceMetadata keyspace)
{
keyspaces.put(keyspace.name, keyspace);
+
+ keyspace.tables.forEach(t -> tables.put(t.id, t));
+ keyspace.views.forEach(v -> tables.put(v.metadata.id, v.metadata));
+
return this;
}
@@ -121,4 +191,49 @@
return this;
}
}
+
+ static KeyspacesDiff diff(Keyspaces before, Keyspaces after)
+ {
+ return KeyspacesDiff.diff(before, after);
+ }
+
+ public static final class KeyspacesDiff
+ {
+ static final KeyspacesDiff NONE = new KeyspacesDiff(Keyspaces.none(), Keyspaces.none(), ImmutableList.of());
+
+ public final Keyspaces created;
+ public final Keyspaces dropped;
+ public final ImmutableList<KeyspaceDiff> altered;
+
+ private KeyspacesDiff(Keyspaces created, Keyspaces dropped, ImmutableList<KeyspaceDiff> altered)
+ {
+ this.created = created;
+ this.dropped = dropped;
+ this.altered = altered;
+ }
+
+ private static KeyspacesDiff diff(Keyspaces before, Keyspaces after)
+ {
+ if (before == after)
+ return NONE;
+
+ Keyspaces created = after.filter(k -> !before.containsKeyspace(k.name));
+ Keyspaces dropped = before.filter(k -> !after.containsKeyspace(k.name));
+
+ ImmutableList.Builder<KeyspaceDiff> altered = ImmutableList.builder();
+ before.forEach(keyspaceBefore ->
+ {
+ KeyspaceMetadata keyspaceAfter = after.getNullable(keyspaceBefore.name);
+ if (null != keyspaceAfter)
+ KeyspaceMetadata.diff(keyspaceBefore, keyspaceAfter).ifPresent(altered::add);
+ });
+
+ return new KeyspacesDiff(created, dropped, altered.build());
+ }
+
+ public boolean isEmpty()
+ {
+ return created.isEmpty() && dropped.isEmpty() && altered.isEmpty();
+ }
+ }
}
diff --git a/src/java/org/apache/cassandra/schema/LegacySchemaMigrator.java b/src/java/org/apache/cassandra/schema/LegacySchemaMigrator.java
deleted file mode 100644
index c320672..0000000
--- a/src/java/org/apache/cassandra/schema/LegacySchemaMigrator.java
+++ /dev/null
@@ -1,1127 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.schema;
-
-import java.nio.ByteBuffer;
-import java.util.*;
-import java.util.stream.Collectors;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.HashMultimap;
-import com.google.common.collect.ImmutableList;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.cassandra.config.*;
-import org.apache.cassandra.cql3.ColumnIdentifier;
-import org.apache.cassandra.cql3.FieldIdentifier;
-import org.apache.cassandra.cql3.QueryProcessor;
-import org.apache.cassandra.cql3.SuperColumnCompatibility;
-import org.apache.cassandra.cql3.UntypedResultSet;
-import org.apache.cassandra.cql3.functions.FunctionName;
-import org.apache.cassandra.cql3.functions.UDAggregate;
-import org.apache.cassandra.cql3.functions.UDFunction;
-import org.apache.cassandra.db.*;
-import org.apache.cassandra.db.compaction.AbstractCompactionStrategy;
-import org.apache.cassandra.db.marshal.*;
-import org.apache.cassandra.db.rows.RowIterator;
-import org.apache.cassandra.db.rows.UnfilteredRowIterators;
-import org.apache.cassandra.exceptions.InvalidRequestException;
-import org.apache.cassandra.utils.FBUtilities;
-
-import static java.lang.String.format;
-import static org.apache.cassandra.utils.ByteBufferUtil.bytes;
-import static org.apache.cassandra.utils.FBUtilities.fromJsonMap;
-
-/**
- * This majestic class performs migration from legacy (pre-3.0) system.schema_* schema tables to the new and glorious
- * system_schema keyspace.
- *
- * The goal is to not lose any information in the migration - including the timestamps.
- */
-@SuppressWarnings("deprecation")
-public final class LegacySchemaMigrator
-{
- private LegacySchemaMigrator()
- {
- }
-
- private static final Logger logger = LoggerFactory.getLogger(LegacySchemaMigrator.class);
-
- static final List<CFMetaData> LegacySchemaTables =
- ImmutableList.of(SystemKeyspace.LegacyKeyspaces,
- SystemKeyspace.LegacyColumnfamilies,
- SystemKeyspace.LegacyColumns,
- SystemKeyspace.LegacyTriggers,
- SystemKeyspace.LegacyUsertypes,
- SystemKeyspace.LegacyFunctions,
- SystemKeyspace.LegacyAggregates);
-
- public static void migrate()
- {
- // read metadata from the legacy schema tables
- Collection<Keyspace> keyspaces = readSchema();
-
- // if already upgraded, or starting a new 3.0 node, abort early
- if (keyspaces.isEmpty())
- {
- unloadLegacySchemaTables();
- return;
- }
-
- // write metadata to the new schema tables
- logger.info("Moving {} keyspaces from legacy schema tables to the new schema keyspace ({})",
- keyspaces.size(),
- SchemaConstants.SCHEMA_KEYSPACE_NAME);
- keyspaces.forEach(LegacySchemaMigrator::storeKeyspaceInNewSchemaTables);
- keyspaces.forEach(LegacySchemaMigrator::migrateBuiltIndexesForKeyspace);
-
- // flush the new tables before truncating the old ones
- SchemaKeyspace.flush();
-
- // truncate the original tables (will be snapshotted now, and will have been snapshotted by pre-flight checks)
- logger.info("Truncating legacy schema tables");
- truncateLegacySchemaTables();
-
- // remove legacy schema tables from Schema, so that their presence doesn't give the users any wrong ideas
- unloadLegacySchemaTables();
-
- logger.info("Completed migration of legacy schema tables");
- }
-
- private static void migrateBuiltIndexesForKeyspace(Keyspace keyspace)
- {
- keyspace.tables.forEach(LegacySchemaMigrator::migrateBuiltIndexesForTable);
- }
-
- private static void migrateBuiltIndexesForTable(Table table)
- {
- table.metadata.getIndexes().forEach((index) -> migrateIndexBuildStatus(table.metadata.ksName,
- table.metadata.cfName,
- index));
- }
-
- private static void migrateIndexBuildStatus(String keyspace, String table, IndexMetadata index)
- {
- if (SystemKeyspace.isIndexBuilt(keyspace, table + '.' + index.name))
- {
- SystemKeyspace.setIndexBuilt(keyspace, index.name);
- SystemKeyspace.setIndexRemoved(keyspace, table + '.' + index.name);
- }
- }
-
- static void unloadLegacySchemaTables()
- {
- KeyspaceMetadata systemKeyspace = Schema.instance.getKSMetaData(SchemaConstants.SYSTEM_KEYSPACE_NAME);
-
- Tables systemTables = systemKeyspace.tables;
- for (CFMetaData table : LegacySchemaTables)
- systemTables = systemTables.without(table.cfName);
-
- LegacySchemaTables.forEach(Schema.instance::unload);
- LegacySchemaTables.forEach((cfm) -> org.apache.cassandra.db.Keyspace.openAndGetStore(cfm).invalidate());
-
- Schema.instance.setKeyspaceMetadata(systemKeyspace.withSwapped(systemTables));
- }
-
- private static void truncateLegacySchemaTables()
- {
- LegacySchemaTables.forEach(table -> Schema.instance.getColumnFamilyStoreInstance(table.cfId).truncateBlocking());
- }
-
- private static void storeKeyspaceInNewSchemaTables(Keyspace keyspace)
- {
- logger.info("Migrating keyspace {}", keyspace);
-
- Mutation.SimpleBuilder builder = SchemaKeyspace.makeCreateKeyspaceMutation(keyspace.name, keyspace.params, keyspace.timestamp);
- for (Table table : keyspace.tables)
- SchemaKeyspace.addTableToSchemaMutation(table.metadata, true, builder.timestamp(table.timestamp));
-
- for (Type type : keyspace.types)
- SchemaKeyspace.addTypeToSchemaMutation(type.metadata, builder.timestamp(type.timestamp));
-
- for (Function function : keyspace.functions)
- SchemaKeyspace.addFunctionToSchemaMutation(function.metadata, builder.timestamp(function.timestamp));
-
- for (Aggregate aggregate : keyspace.aggregates)
- SchemaKeyspace.addAggregateToSchemaMutation(aggregate.metadata, builder.timestamp(aggregate.timestamp));
-
- builder.build().apply();
- }
-
- /*
- * Read all keyspaces metadata (including nested tables, types, and functions), with their modification timestamps
- */
- private static Collection<Keyspace> readSchema()
- {
- String query = format("SELECT keyspace_name FROM %s.%s", SchemaConstants.SYSTEM_KEYSPACE_NAME, SystemKeyspace.LEGACY_KEYSPACES);
- Collection<String> keyspaceNames = new ArrayList<>();
- query(query).forEach(row -> keyspaceNames.add(row.getString("keyspace_name")));
- keyspaceNames.removeAll(SchemaConstants.LOCAL_SYSTEM_KEYSPACE_NAMES);
-
- Collection<Keyspace> keyspaces = new ArrayList<>();
- keyspaceNames.forEach(name -> keyspaces.add(readKeyspace(name)));
- return keyspaces;
- }
-
- private static Keyspace readKeyspace(String keyspaceName)
- {
- long timestamp = readKeyspaceTimestamp(keyspaceName);
- KeyspaceParams params = readKeyspaceParams(keyspaceName);
-
- Collection<Table> tables = readTables(keyspaceName);
- Collection<Type> types = readTypes(keyspaceName);
- Collection<Function> functions = readFunctions(keyspaceName);
- Functions.Builder functionsBuilder = Functions.builder();
- functions.forEach(udf -> functionsBuilder.add(udf.metadata));
- Collection<Aggregate> aggregates = readAggregates(functionsBuilder.build(), keyspaceName);
-
- return new Keyspace(timestamp, keyspaceName, params, tables, types, functions, aggregates);
- }
-
- /*
- * Reading keyspace params
- */
-
- private static long readKeyspaceTimestamp(String keyspaceName)
- {
- String query = format("SELECT writeTime(durable_writes) AS timestamp FROM %s.%s WHERE keyspace_name = ?",
- SchemaConstants.SYSTEM_KEYSPACE_NAME,
- SystemKeyspace.LEGACY_KEYSPACES);
- return query(query, keyspaceName).one().getLong("timestamp");
- }
-
- private static KeyspaceParams readKeyspaceParams(String keyspaceName)
- {
- String query = format("SELECT * FROM %s.%s WHERE keyspace_name = ?",
- SchemaConstants.SYSTEM_KEYSPACE_NAME,
- SystemKeyspace.LEGACY_KEYSPACES);
- UntypedResultSet.Row row = query(query, keyspaceName).one();
-
- boolean durableWrites = row.getBoolean("durable_writes");
-
- Map<String, String> replication = new HashMap<>();
- replication.putAll(fromJsonMap(row.getString("strategy_options")));
- replication.put(ReplicationParams.CLASS, row.getString("strategy_class"));
-
- return KeyspaceParams.create(durableWrites, replication);
- }
-
- /*
- * Reading tables
- */
-
- private static Collection<Table> readTables(String keyspaceName)
- {
- String query = format("SELECT columnfamily_name FROM %s.%s WHERE keyspace_name = ?",
- SchemaConstants.SYSTEM_KEYSPACE_NAME,
- SystemKeyspace.LEGACY_COLUMNFAMILIES);
- Collection<String> tableNames = new ArrayList<>();
- query(query, keyspaceName).forEach(row -> tableNames.add(row.getString("columnfamily_name")));
-
- Collection<Table> tables = new ArrayList<>();
- tableNames.forEach(name -> tables.add(readTable(keyspaceName, name)));
- return tables;
- }
-
- private static Table readTable(String keyspaceName, String tableName)
- {
- long timestamp = readTableTimestamp(keyspaceName, tableName);
- CFMetaData metadata = readTableMetadata(keyspaceName, tableName);
- return new Table(timestamp, metadata);
- }
-
- private static long readTableTimestamp(String keyspaceName, String tableName)
- {
- String query = format("SELECT writeTime(type) AS timestamp FROM %s.%s WHERE keyspace_name = ? AND columnfamily_name = ?",
- SchemaConstants.SYSTEM_KEYSPACE_NAME,
- SystemKeyspace.LEGACY_COLUMNFAMILIES);
- return query(query, keyspaceName, tableName).one().getLong("timestamp");
- }
-
- private static CFMetaData readTableMetadata(String keyspaceName, String tableName)
- {
- String tableQuery = format("SELECT * FROM %s.%s WHERE keyspace_name = ? AND columnfamily_name = ?",
- SchemaConstants.SYSTEM_KEYSPACE_NAME,
- SystemKeyspace.LEGACY_COLUMNFAMILIES);
- UntypedResultSet.Row tableRow = query(tableQuery, keyspaceName, tableName).one();
-
- String columnsQuery = format("SELECT * FROM %s.%s WHERE keyspace_name = ? AND columnfamily_name = ?",
- SchemaConstants.SYSTEM_KEYSPACE_NAME,
- SystemKeyspace.LEGACY_COLUMNS);
- UntypedResultSet columnRows = query(columnsQuery, keyspaceName, tableName);
-
- String triggersQuery = format("SELECT * FROM %s.%s WHERE keyspace_name = ? AND columnfamily_name = ?",
- SchemaConstants.SYSTEM_KEYSPACE_NAME,
- SystemKeyspace.LEGACY_TRIGGERS);
- UntypedResultSet triggerRows = query(triggersQuery, keyspaceName, tableName);
-
- return decodeTableMetadata(tableName, tableRow, columnRows, triggerRows);
- }
-
- private static CFMetaData decodeTableMetadata(String tableName,
- UntypedResultSet.Row tableRow,
- UntypedResultSet columnRows,
- UntypedResultSet triggerRows)
- {
- String ksName = tableRow.getString("keyspace_name");
- String cfName = tableRow.getString("columnfamily_name");
-
- AbstractType<?> rawComparator = TypeParser.parse(tableRow.getString("comparator"));
- AbstractType<?> subComparator = tableRow.has("subcomparator") ? TypeParser.parse(tableRow.getString("subcomparator")) : null;
-
- boolean isSuper = "super".equals(tableRow.getString("type").toLowerCase(Locale.ENGLISH));
- boolean isCompound = rawComparator instanceof CompositeType || isSuper;
-
- /*
- * Determine whether or not the table is *really* dense
- * We cannot trust is_dense value of true (see CASSANDRA-11502, that fixed the issue for 2.2 only, and not retroactively),
- * but we can trust is_dense value of false.
- */
- Boolean rawIsDense = tableRow.has("is_dense") ? tableRow.getBoolean("is_dense") : null;
- boolean isDense;
- if (rawIsDense != null && !rawIsDense)
- isDense = false;
- else
- isDense = calculateIsDense(rawComparator, columnRows, isSuper);
-
- // now, if switched to sparse, remove redundant compact_value column and the last clustering column,
- // directly copying CASSANDRA-11502 logic. See CASSANDRA-11315.
- Iterable<UntypedResultSet.Row> filteredColumnRows = !isDense && (rawIsDense == null || rawIsDense)
- ? filterOutRedundantRowsForSparse(columnRows, isSuper, isCompound)
- : columnRows;
-
- // We don't really use the default validator but as we have it for backward compatibility, we use it to know if it's a counter table
- AbstractType<?> defaultValidator = TypeParser.parse(tableRow.getString("default_validator"));
- boolean isCounter = defaultValidator instanceof CounterColumnType;
-
- /*
- * With CASSANDRA-5202 we stopped inferring the cf id from the combination of keyspace/table names,
- * and started storing the generated uuids in system.schema_columnfamilies.
- *
- * In 3.0 we SHOULD NOT see tables like that (2.0-created, non-upgraded).
- * But in the off-chance that we do, we generate the deterministic uuid here.
- */
- UUID cfId = tableRow.has("cf_id")
- ? tableRow.getUUID("cf_id")
- : CFMetaData.generateLegacyCfId(ksName, cfName);
-
- boolean isCQLTable = !isSuper && !isDense && isCompound;
- boolean isStaticCompactTable = !isDense && !isCompound;
-
- // Internally, compact tables have a specific layout, see CompactTables. But when upgrading from
- // previous versions, they may not have the expected schema, so detect if we need to upgrade and do
- // it in createColumnsFromColumnRows.
- // We can remove this once we don't support upgrade from versions < 3.0.
- boolean needsUpgrade = !isCQLTable && checkNeedsUpgrade(filteredColumnRows, isSuper, isStaticCompactTable);
-
- List<ColumnDefinition> columnDefs = createColumnsFromColumnRows(filteredColumnRows,
- ksName,
- cfName,
- rawComparator,
- subComparator,
- isSuper,
- isCQLTable,
- isStaticCompactTable,
- needsUpgrade);
-
- if (needsUpgrade)
- {
- addDefinitionForUpgrade(columnDefs,
- ksName,
- cfName,
- isStaticCompactTable,
- isSuper,
- rawComparator,
- subComparator,
- defaultValidator);
- }
-
- CFMetaData cfm = CFMetaData.create(ksName,
- cfName,
- cfId,
- isDense,
- isCompound,
- isSuper,
- isCounter,
- false, // legacy schema did not contain views
- columnDefs,
- DatabaseDescriptor.getPartitioner());
-
- Indexes indexes = createIndexesFromColumnRows(cfm,
- filteredColumnRows,
- ksName,
- cfName,
- rawComparator,
- subComparator,
- isSuper,
- isCQLTable,
- isStaticCompactTable,
- needsUpgrade);
- cfm.indexes(indexes);
-
- if (tableRow.has("dropped_columns"))
- addDroppedColumns(cfm, rawComparator, tableRow.getMap("dropped_columns", UTF8Type.instance, LongType.instance));
-
- return cfm.params(decodeTableParams(tableRow))
- .triggers(createTriggersFromTriggerRows(triggerRows));
- }
-
- /*
- * We call dense a CF for which each component of the comparator is a clustering column, i.e. no
- * component is used to store a regular column names. In other words, non-composite static "thrift"
- * and CQL3 CF are *not* dense.
- * We save whether the table is dense or not during table creation through CQL, but we don't have this
- * information for table just created through thrift, nor for table prior to CASSANDRA-7744, so this
- * method does its best to infer whether the table is dense or not based on other elements.
- */
- private static boolean calculateIsDense(AbstractType<?> comparator, UntypedResultSet columnRows, boolean isSuper)
- {
- /*
- * As said above, this method is only here because we need to deal with thrift upgrades.
- * Once a CF has been "upgraded", i.e. we've rebuilt and save its CQL3 metadata at least once,
- * then we'll have saved the "is_dense" value and will be good to go.
- *
- * But non-upgraded thrift CF (and pre-7744 CF) will have no value for "is_dense", so we need
- * to infer that information without relying on it in that case. And for the most part this is
- * easy, a CF that has at least one REGULAR definition is not dense. But the subtlety is that not
- * having a REGULAR definition may not mean dense because of CQL3 definitions that have only the
- * PRIMARY KEY defined.
- *
- * So we need to recognize those special case CQL3 table with only a primary key. If we have some
- * clustering columns, we're fine as said above. So the only problem is that we cannot decide for
- * sure if a CF without REGULAR columns nor CLUSTERING_COLUMN definition is meant to be dense, or if it
- * has been created in CQL3 by say:
- * CREATE TABLE test (k int PRIMARY KEY)
- * in which case it should not be dense. However, we can limit our margin of error by assuming we are
- * in the latter case only if the comparator is exactly CompositeType(UTF8Type).
- */
- for (UntypedResultSet.Row columnRow : columnRows)
- {
- if ("regular".equals(columnRow.getString("type")))
- return false;
- }
-
- // If we've checked the columns for supercf and found no regulars, it's dense. Relying on the emptiness
- // of the value column is not enough due to index calculation.
- if (isSuper)
- return true;
-
- int maxClusteringIdx = -1;
- for (UntypedResultSet.Row columnRow : columnRows)
- if ("clustering_key".equals(columnRow.getString("type")))
- maxClusteringIdx = Math.max(maxClusteringIdx, columnRow.has("component_index") ? columnRow.getInt("component_index") : 0);
-
- return maxClusteringIdx >= 0
- ? maxClusteringIdx == comparator.componentsCount() - 1
- : !isCQL3OnlyPKComparator(comparator);
- }
-
- private static Iterable<UntypedResultSet.Row> filterOutRedundantRowsForSparse(UntypedResultSet columnRows, boolean isSuper, boolean isCompound)
- {
- Collection<UntypedResultSet.Row> filteredRows = new ArrayList<>();
- for (UntypedResultSet.Row columnRow : columnRows)
- {
- String kind = columnRow.getString("type");
-
- if (!isSuper && "compact_value".equals(kind))
- continue;
-
- if ("clustering_key".equals(kind) && !isSuper && !isCompound)
- continue;
-
- filteredRows.add(columnRow);
- }
-
- return filteredRows;
- }
-
- private static boolean isCQL3OnlyPKComparator(AbstractType<?> comparator)
- {
- if (!(comparator instanceof CompositeType))
- return false;
-
- CompositeType ct = (CompositeType)comparator;
- return ct.types.size() == 1 && ct.types.get(0) instanceof UTF8Type;
- }
-
- private static TableParams decodeTableParams(UntypedResultSet.Row row)
- {
- TableParams.Builder params = TableParams.builder();
-
- params.readRepairChance(row.getDouble("read_repair_chance"))
- .dcLocalReadRepairChance(row.getDouble("local_read_repair_chance"))
- .gcGraceSeconds(row.getInt("gc_grace_seconds"));
-
- if (row.has("comment"))
- params.comment(row.getString("comment"));
-
- if (row.has("memtable_flush_period_in_ms"))
- params.memtableFlushPeriodInMs(row.getInt("memtable_flush_period_in_ms"));
-
- params.caching(cachingFromRow(row.getString("caching")));
-
- if (row.has("default_time_to_live"))
- params.defaultTimeToLive(row.getInt("default_time_to_live"));
-
- if (row.has("speculative_retry"))
- params.speculativeRetry(SpeculativeRetryParam.fromString(row.getString("speculative_retry")));
-
- Map<String, String> compressionParameters = fromJsonMap(row.getString("compression_parameters"));
- String crcCheckChance = compressionParameters.remove("crc_check_chance");
- //crc_check_chance was promoted from a compression property to a top-level property
- if (crcCheckChance != null)
- params.crcCheckChance(Double.parseDouble(crcCheckChance));
-
- params.compression(CompressionParams.fromMap(compressionParameters));
-
- params.compaction(compactionFromRow(row));
-
- if (row.has("min_index_interval"))
- params.minIndexInterval(row.getInt("min_index_interval"));
-
- if (row.has("max_index_interval"))
- params.maxIndexInterval(row.getInt("max_index_interval"));
-
- if (row.has("bloom_filter_fp_chance"))
- params.bloomFilterFpChance(row.getDouble("bloom_filter_fp_chance"));
-
- return params.build();
- }
-
-
- /**
- *
- * 2.1 and newer use JSON'ified map of caching parameters, but older versions had valid Strings
- * NONE, KEYS_ONLY, ROWS_ONLY, and ALL
- *
- * @param caching, the string representing the table's caching options
- * @return CachingParams object corresponding to the input string
- */
- @VisibleForTesting
- public static CachingParams cachingFromRow(String caching)
- {
- switch(caching)
- {
- case "NONE":
- return CachingParams.CACHE_NOTHING;
- case "KEYS_ONLY":
- return CachingParams.CACHE_KEYS;
- case "ROWS_ONLY":
- return new CachingParams(false, Integer.MAX_VALUE);
- case "ALL":
- return CachingParams.CACHE_EVERYTHING;
- default:
- return CachingParams.fromMap(fromJsonMap(caching));
- }
- }
-
- /*
- * The method is needed - to migrate max_compaction_threshold and min_compaction_threshold
- * to the compaction map, where they belong.
- *
- * We must use reflection to validate the options because not every compaction strategy respects and supports
- * the threshold params (LCS doesn't, STCS and DTCS do).
- */
- @SuppressWarnings("unchecked")
- private static CompactionParams compactionFromRow(UntypedResultSet.Row row)
- {
- Class<? extends AbstractCompactionStrategy> klass =
- CFMetaData.createCompactionStrategy(row.getString("compaction_strategy_class"));
- Map<String, String> options = fromJsonMap(row.getString("compaction_strategy_options"));
-
- int minThreshold = row.getInt("min_compaction_threshold");
- int maxThreshold = row.getInt("max_compaction_threshold");
-
- Map<String, String> optionsWithThresholds = new HashMap<>(options);
- optionsWithThresholds.putIfAbsent(CompactionParams.Option.MIN_THRESHOLD.toString(), Integer.toString(minThreshold));
- optionsWithThresholds.putIfAbsent(CompactionParams.Option.MAX_THRESHOLD.toString(), Integer.toString(maxThreshold));
-
- try
- {
- Map<String, String> unrecognizedOptions =
- (Map<String, String>) klass.getMethod("validateOptions", Map.class).invoke(null, optionsWithThresholds);
-
- if (unrecognizedOptions.isEmpty())
- options = optionsWithThresholds;
- }
- catch (Exception e)
- {
- throw new RuntimeException(e);
- }
-
- return CompactionParams.create(klass, options);
- }
-
- // Should only be called on compact tables
- private static boolean checkNeedsUpgrade(Iterable<UntypedResultSet.Row> defs, boolean isSuper, boolean isStaticCompactTable)
- {
- // For SuperColumn tables, re-create a compact value column
- if (isSuper)
- return true;
-
- // For static compact tables, we need to upgrade if the regular definitions haven't been converted to static yet,
- // i.e. if we don't have a static definition yet.
- if (isStaticCompactTable)
- return !hasKind(defs, ColumnDefinition.Kind.STATIC);
-
- // For dense compact tables, we need to upgrade if we don't have a compact value definition
- return !hasRegularColumns(defs);
- }
-
- private static boolean hasRegularColumns(Iterable<UntypedResultSet.Row> columnRows)
- {
- for (UntypedResultSet.Row row : columnRows)
- {
- /*
- * We need to special case and ignore the empty compact column (pre-3.0, COMPACT STORAGE, primary-key only tables),
- * since deserializeKind() will otherwise just return a REGULAR.
- * We want the proper EmptyType regular column to be added by addDefinitionForUpgrade(), so we need
- * checkNeedsUpgrade() to return true in this case.
- * See CASSANDRA-9874.
- */
- if (isEmptyCompactValueColumn(row))
- return false;
-
- if (deserializeKind(row.getString("type")) == ColumnDefinition.Kind.REGULAR)
- return true;
- }
-
- return false;
- }
-
- private static boolean isEmptyCompactValueColumn(UntypedResultSet.Row row)
- {
- return "compact_value".equals(row.getString("type")) && row.getString("column_name").isEmpty();
- }
-
- private static void addDefinitionForUpgrade(List<ColumnDefinition> defs,
- String ksName,
- String cfName,
- boolean isStaticCompactTable,
- boolean isSuper,
- AbstractType<?> rawComparator,
- AbstractType<?> subComparator,
- AbstractType<?> defaultValidator)
- {
- CompactTables.DefaultNames names = CompactTables.defaultNameGenerator(defs);
-
- if (isSuper)
- {
- defs.add(ColumnDefinition.regularDef(ksName, cfName, SuperColumnCompatibility.SUPER_COLUMN_MAP_COLUMN_STR, MapType.getInstance(subComparator, defaultValidator, true)));
- }
- else if (isStaticCompactTable)
- {
- defs.add(ColumnDefinition.clusteringDef(ksName, cfName, names.defaultClusteringName(), rawComparator, 0));
- defs.add(ColumnDefinition.regularDef(ksName, cfName, names.defaultCompactValueName(), defaultValidator));
- }
- else
- {
- // For dense compact tables, we get here if we don't have a compact value column, in which case we should add it.
- // We use EmptyType to recognize that the compact value was not declared by the user (see CreateTableStatement).
- // If user made any writes to this column, compact value column should be initialized as bytes (see CASSANDRA-15778).
- AbstractType<?> compactColumnType = Boolean.getBoolean("cassandra.init_dense_table_compact_value_as_bytes")
- ? BytesType.instance : EmptyType.instance;
- defs.add(ColumnDefinition.regularDef(ksName, cfName, names.defaultCompactValueName(), compactColumnType));
- }
- }
-
- private static boolean hasKind(Iterable<UntypedResultSet.Row> defs, ColumnDefinition.Kind kind)
- {
- for (UntypedResultSet.Row row : defs)
- if (deserializeKind(row.getString("type")) == kind)
- return true;
-
- return false;
- }
-
- /*
- * Prior to 3.0 we used to not store the type of the dropped columns, relying on all collection info being
- * present in the comparator, forever. That allowed us to perform certain validations in AlterTableStatement
- * (namely not allowing to re-add incompatible collection columns, with the same name, but a different type).
- *
- * In 3.0, we no longer preserve the original comparator, and reconstruct it from the columns instead. That means
- * that we should preserve the type of the dropped columns now, and, during migration, fetch the types from
- * the original comparator if necessary.
- */
- private static void addDroppedColumns(CFMetaData cfm, AbstractType<?> comparator, Map<String, Long> droppedTimes)
- {
- AbstractType<?> last = comparator.getComponents().get(comparator.componentsCount() - 1);
- Map<ByteBuffer, CollectionType> collections = last instanceof ColumnToCollectionType
- ? ((ColumnToCollectionType) last).defined
- : Collections.emptyMap();
-
- for (Map.Entry<String, Long> entry : droppedTimes.entrySet())
- {
- String name = entry.getKey();
- ByteBuffer nameBytes = UTF8Type.instance.decompose(name);
- long time = entry.getValue();
-
- AbstractType<?> type = collections.containsKey(nameBytes)
- ? collections.get(nameBytes)
- : BytesType.instance;
-
- cfm.getDroppedColumns().put(nameBytes, new CFMetaData.DroppedColumn(name, null, type, time));
- }
- }
-
- private static List<ColumnDefinition> createColumnsFromColumnRows(Iterable<UntypedResultSet.Row> rows,
- String keyspace,
- String table,
- AbstractType<?> rawComparator,
- AbstractType<?> rawSubComparator,
- boolean isSuper,
- boolean isCQLTable,
- boolean isStaticCompactTable,
- boolean needsUpgrade)
- {
- List<ColumnDefinition> columns = new ArrayList<>();
-
- for (UntypedResultSet.Row row : rows)
- {
- // Skip the empty compact value column. Make addDefinitionForUpgrade() re-add the proper REGULAR one.
- if (isEmptyCompactValueColumn(row))
- continue;
-
- columns.add(createColumnFromColumnRow(row,
- keyspace,
- table,
- rawComparator,
- rawSubComparator,
- isSuper,
- isCQLTable,
- isStaticCompactTable,
- needsUpgrade));
- }
-
- return columns;
- }
-
- private static ColumnDefinition createColumnFromColumnRow(UntypedResultSet.Row row,
- String keyspace,
- String table,
- AbstractType<?> rawComparator,
- AbstractType<?> rawSubComparator,
- boolean isSuper,
- boolean isCQLTable,
- boolean isStaticCompactTable,
- boolean needsUpgrade)
- {
- String rawKind = row.getString("type");
-
- ColumnDefinition.Kind kind = deserializeKind(rawKind);
- if (needsUpgrade && isStaticCompactTable && kind == ColumnDefinition.Kind.REGULAR)
- kind = ColumnDefinition.Kind.STATIC;
-
- int componentIndex = ColumnDefinition.NO_POSITION;
- // Note that the component_index is not useful for non-primary key parts (it never really in fact since there is
- // no particular ordering of non-PK columns, we only used to use it as a simplification but that's not needed
- // anymore)
- if (kind.isPrimaryKeyKind())
- // We use to not have a component index when there was a single partition key, we don't anymore (#10491)
- componentIndex = row.has("component_index") ? row.getInt("component_index") : 0;
-
- // Note: we save the column name as string, but we should not assume that it is an UTF8 name, we
- // we need to use the comparator fromString method
- AbstractType<?> comparator = isCQLTable
- ? UTF8Type.instance
- : CompactTables.columnDefinitionComparator(rawKind, isSuper, rawComparator, rawSubComparator);
- ColumnIdentifier name = ColumnIdentifier.getInterned(comparator.fromString(row.getString("column_name")), comparator);
-
- AbstractType<?> validator = parseType(row.getString("validator"));
-
- // In the 2.x schema we didn't store UDT's with a FrozenType wrapper because they were implicitly frozen. After
- // CASSANDRA-7423 (non-frozen UDTs), this is no longer true, so we need to freeze UDTs and nested freezable
- // types (UDTs and collections) to properly migrate the schema. See CASSANDRA-11609 and CASSANDRA-11613.
- if (validator.isUDT() && validator.isMultiCell())
- validator = validator.freeze();
- else
- validator = validator.freezeNestedMulticellTypes();
-
- return new ColumnDefinition(keyspace, table, name, validator, componentIndex, kind);
- }
-
- private static Indexes createIndexesFromColumnRows(CFMetaData cfm,
- Iterable<UntypedResultSet.Row> rows,
- String keyspace,
- String table,
- AbstractType<?> rawComparator,
- AbstractType<?> rawSubComparator,
- boolean isSuper,
- boolean isCQLTable,
- boolean isStaticCompactTable,
- boolean needsUpgrade)
- {
- Indexes.Builder indexes = Indexes.builder();
-
- for (UntypedResultSet.Row row : rows)
- {
- IndexMetadata.Kind kind = null;
- if (row.has("index_type"))
- kind = IndexMetadata.Kind.valueOf(row.getString("index_type"));
-
- if (kind == null)
- continue;
-
- Map<String, String> indexOptions = null;
- if (row.has("index_options"))
- indexOptions = fromJsonMap(row.getString("index_options"));
-
- if (row.has("index_name"))
- {
- String indexName = row.getString("index_name");
-
- ColumnDefinition column = createColumnFromColumnRow(row,
- keyspace,
- table,
- rawComparator,
- rawSubComparator,
- isSuper,
- isCQLTable,
- isStaticCompactTable,
- needsUpgrade);
-
- indexes.add(IndexMetadata.fromLegacyMetadata(cfm, column, indexName, kind, indexOptions));
- }
- else
- {
- logger.error("Failed to find index name for legacy migration of index on {}.{}", keyspace, table);
- }
- }
-
- return indexes.build();
- }
-
- private static ColumnDefinition.Kind deserializeKind(String kind)
- {
- if ("clustering_key".equalsIgnoreCase(kind))
- return ColumnDefinition.Kind.CLUSTERING;
-
- if ("compact_value".equalsIgnoreCase(kind))
- return ColumnDefinition.Kind.REGULAR;
-
- return Enum.valueOf(ColumnDefinition.Kind.class, kind.toUpperCase());
- }
-
- private static Triggers createTriggersFromTriggerRows(UntypedResultSet rows)
- {
- Triggers.Builder triggers = org.apache.cassandra.schema.Triggers.builder();
- rows.forEach(row -> triggers.add(createTriggerFromTriggerRow(row)));
- return triggers.build();
- }
-
- private static TriggerMetadata createTriggerFromTriggerRow(UntypedResultSet.Row row)
- {
- String name = row.getString("trigger_name");
- String classOption = row.getTextMap("trigger_options").get("class");
- return new TriggerMetadata(name, classOption);
- }
-
- /*
- * Reading user types
- */
-
- private static Collection<Type> readTypes(String keyspaceName)
- {
- String query = format("SELECT type_name FROM %s.%s WHERE keyspace_name = ?",
- SchemaConstants.SYSTEM_KEYSPACE_NAME,
- SystemKeyspace.LEGACY_USERTYPES);
- Collection<String> typeNames = new ArrayList<>();
- query(query, keyspaceName).forEach(row -> typeNames.add(row.getString("type_name")));
-
- Collection<Type> types = new ArrayList<>();
- typeNames.forEach(name -> types.add(readType(keyspaceName, name)));
- return types;
- }
-
- private static Type readType(String keyspaceName, String typeName)
- {
- long timestamp = readTypeTimestamp(keyspaceName, typeName);
- UserType metadata = readTypeMetadata(keyspaceName, typeName);
- return new Type(timestamp, metadata);
- }
-
- /*
- * Unfortunately there is not a single REGULAR column in system.schema_usertypes, so annoyingly we cannot
- * use the writeTime() CQL function, and must resort to a lower level.
- */
- private static long readTypeTimestamp(String keyspaceName, String typeName)
- {
- ColumnFamilyStore store = org.apache.cassandra.db.Keyspace.open(SchemaConstants.SYSTEM_KEYSPACE_NAME)
- .getColumnFamilyStore(SystemKeyspace.LEGACY_USERTYPES);
-
- ClusteringComparator comparator = store.metadata.comparator;
- Slices slices = Slices.with(comparator, Slice.make(comparator, typeName));
- int nowInSec = FBUtilities.nowInSeconds();
- DecoratedKey key = store.metadata.decorateKey(AsciiType.instance.fromString(keyspaceName));
- SinglePartitionReadCommand command = SinglePartitionReadCommand.create(store.metadata, nowInSec, key, slices);
-
- try (ReadExecutionController controller = command.executionController();
- RowIterator partition = UnfilteredRowIterators.filter(command.queryMemtableAndDisk(store, controller), nowInSec))
- {
- return partition.next().primaryKeyLivenessInfo().timestamp();
- }
- }
-
- private static UserType readTypeMetadata(String keyspaceName, String typeName)
- {
- String query = format("SELECT * FROM %s.%s WHERE keyspace_name = ? AND type_name = ?",
- SchemaConstants.SYSTEM_KEYSPACE_NAME,
- SystemKeyspace.LEGACY_USERTYPES);
- UntypedResultSet.Row row = query(query, keyspaceName, typeName).one();
-
- List<FieldIdentifier> names =
- row.getList("field_names", UTF8Type.instance)
- .stream()
- .map(t -> FieldIdentifier.forInternalString(t))
- .collect(Collectors.toList());
-
- List<AbstractType<?>> types =
- row.getList("field_types", UTF8Type.instance)
- .stream()
- .map(LegacySchemaMigrator::parseType)
- .collect(Collectors.toList());
-
- return new UserType(keyspaceName, bytes(typeName), names, types, true);
- }
-
- /*
- * Reading UDFs
- */
-
- private static Collection<Function> readFunctions(String keyspaceName)
- {
- String query = format("SELECT function_name, signature FROM %s.%s WHERE keyspace_name = ?",
- SchemaConstants.SYSTEM_KEYSPACE_NAME,
- SystemKeyspace.LEGACY_FUNCTIONS);
- HashMultimap<String, List<String>> functionSignatures = HashMultimap.create();
- query(query, keyspaceName).forEach(row -> functionSignatures.put(row.getString("function_name"), row.getList("signature", UTF8Type.instance)));
-
- Collection<Function> functions = new ArrayList<>();
- functionSignatures.entries().forEach(pair -> functions.add(readFunction(keyspaceName, pair.getKey(), pair.getValue())));
- return functions;
- }
-
- private static Function readFunction(String keyspaceName, String functionName, List<String> signature)
- {
- long timestamp = readFunctionTimestamp(keyspaceName, functionName, signature);
- UDFunction metadata = readFunctionMetadata(keyspaceName, functionName, signature);
- return new Function(timestamp, metadata);
- }
-
- private static long readFunctionTimestamp(String keyspaceName, String functionName, List<String> signature)
- {
- String query = format("SELECT writeTime(return_type) AS timestamp " +
- "FROM %s.%s " +
- "WHERE keyspace_name = ? AND function_name = ? AND signature = ?",
- SchemaConstants.SYSTEM_KEYSPACE_NAME,
- SystemKeyspace.LEGACY_FUNCTIONS);
- return query(query, keyspaceName, functionName, signature).one().getLong("timestamp");
- }
-
- private static UDFunction readFunctionMetadata(String keyspaceName, String functionName, List<String> signature)
- {
- String query = format("SELECT * FROM %s.%s WHERE keyspace_name = ? AND function_name = ? AND signature = ?",
- SchemaConstants.SYSTEM_KEYSPACE_NAME,
- SystemKeyspace.LEGACY_FUNCTIONS);
- UntypedResultSet.Row row = query(query, keyspaceName, functionName, signature).one();
-
- FunctionName name = new FunctionName(keyspaceName, functionName);
-
- List<ColumnIdentifier> argNames = new ArrayList<>();
- if (row.has("argument_names"))
- for (String arg : row.getList("argument_names", UTF8Type.instance))
- argNames.add(new ColumnIdentifier(arg, true));
-
- List<AbstractType<?>> argTypes = new ArrayList<>();
- if (row.has("argument_types"))
- for (String type : row.getList("argument_types", UTF8Type.instance))
- argTypes.add(parseType(type));
-
- AbstractType<?> returnType = parseType(row.getString("return_type"));
-
- String language = row.getString("language");
- String body = row.getString("body");
- boolean calledOnNullInput = row.getBoolean("called_on_null_input");
-
- try
- {
- return UDFunction.create(name, argNames, argTypes, returnType, calledOnNullInput, language, body);
- }
- catch (InvalidRequestException e)
- {
- return UDFunction.createBrokenFunction(name, argNames, argTypes, returnType, calledOnNullInput, language, body, e);
- }
- }
-
- /*
- * Reading UDAs
- */
-
- private static Collection<Aggregate> readAggregates(Functions functions, String keyspaceName)
- {
- String query = format("SELECT aggregate_name, signature FROM %s.%s WHERE keyspace_name = ?",
- SchemaConstants.SYSTEM_KEYSPACE_NAME,
- SystemKeyspace.LEGACY_AGGREGATES);
- HashMultimap<String, List<String>> aggregateSignatures = HashMultimap.create();
- query(query, keyspaceName).forEach(row -> aggregateSignatures.put(row.getString("aggregate_name"), row.getList("signature", UTF8Type.instance)));
-
- Collection<Aggregate> aggregates = new ArrayList<>();
- aggregateSignatures.entries().forEach(pair -> aggregates.add(readAggregate(functions, keyspaceName, pair.getKey(), pair.getValue())));
- return aggregates;
- }
-
- private static Aggregate readAggregate(Functions functions, String keyspaceName, String aggregateName, List<String> signature)
- {
- long timestamp = readAggregateTimestamp(keyspaceName, aggregateName, signature);
- UDAggregate metadata = readAggregateMetadata(functions, keyspaceName, aggregateName, signature);
- return new Aggregate(timestamp, metadata);
- }
-
- private static long readAggregateTimestamp(String keyspaceName, String aggregateName, List<String> signature)
- {
- String query = format("SELECT writeTime(return_type) AS timestamp " +
- "FROM %s.%s " +
- "WHERE keyspace_name = ? AND aggregate_name = ? AND signature = ?",
- SchemaConstants.SYSTEM_KEYSPACE_NAME,
- SystemKeyspace.LEGACY_AGGREGATES);
- return query(query, keyspaceName, aggregateName, signature).one().getLong("timestamp");
- }
-
- private static UDAggregate readAggregateMetadata(Functions functions, String keyspaceName, String functionName, List<String> signature)
- {
- String query = format("SELECT * FROM %s.%s WHERE keyspace_name = ? AND aggregate_name = ? AND signature = ?",
- SchemaConstants.SYSTEM_KEYSPACE_NAME,
- SystemKeyspace.LEGACY_AGGREGATES);
- UntypedResultSet.Row row = query(query, keyspaceName, functionName, signature).one();
-
- FunctionName name = new FunctionName(keyspaceName, functionName);
-
- List<String> types = row.getList("argument_types", UTF8Type.instance);
-
- List<AbstractType<?>> argTypes = new ArrayList<>();
- if (types != null)
- {
- argTypes = new ArrayList<>(types.size());
- for (String type : types)
- argTypes.add(parseType(type));
- }
-
- AbstractType<?> returnType = parseType(row.getString("return_type"));
-
- FunctionName stateFunc = new FunctionName(keyspaceName, row.getString("state_func"));
- AbstractType<?> stateType = parseType(row.getString("state_type"));
- FunctionName finalFunc = row.has("final_func") ? new FunctionName(keyspaceName, row.getString("final_func")) : null;
- ByteBuffer initcond = row.has("initcond") ? row.getBytes("initcond") : null;
-
- try
- {
- return UDAggregate.create(functions, name, argTypes, returnType, stateFunc, finalFunc, stateType, initcond);
- }
- catch (InvalidRequestException reason)
- {
- return UDAggregate.createBroken(name, argTypes, returnType, initcond, reason);
- }
- }
-
- private static UntypedResultSet query(String query, Object... values)
- {
- return QueryProcessor.executeOnceInternal(query, values);
- }
-
- private static AbstractType<?> parseType(String str)
- {
- return TypeParser.parse(str);
- }
-
- private static final class Keyspace
- {
- final long timestamp;
- final String name;
- final KeyspaceParams params;
- final Collection<Table> tables;
- final Collection<Type> types;
- final Collection<Function> functions;
- final Collection<Aggregate> aggregates;
-
- Keyspace(long timestamp,
- String name,
- KeyspaceParams params,
- Collection<Table> tables,
- Collection<Type> types,
- Collection<Function> functions,
- Collection<Aggregate> aggregates)
- {
- this.timestamp = timestamp;
- this.name = name;
- this.params = params;
- this.tables = tables;
- this.types = types;
- this.functions = functions;
- this.aggregates = aggregates;
- }
- }
-
- private static final class Table
- {
- final long timestamp;
- final CFMetaData metadata;
-
- Table(long timestamp, CFMetaData metadata)
- {
- this.timestamp = timestamp;
- this.metadata = metadata;
- }
- }
-
- private static final class Type
- {
- final long timestamp;
- final UserType metadata;
-
- Type(long timestamp, UserType metadata)
- {
- this.timestamp = timestamp;
- this.metadata = metadata;
- }
- }
-
- private static final class Function
- {
- final long timestamp;
- final UDFunction metadata;
-
- Function(long timestamp, UDFunction metadata)
- {
- this.timestamp = timestamp;
- this.metadata = metadata;
- }
- }
-
- private static final class Aggregate
- {
- final long timestamp;
- final UDAggregate metadata;
-
- Aggregate(long timestamp, UDAggregate metadata)
- {
- this.timestamp = timestamp;
- this.metadata = metadata;
- }
- }
-}
diff --git a/src/java/org/apache/cassandra/schema/MigrationManager.java b/src/java/org/apache/cassandra/schema/MigrationManager.java
new file mode 100644
index 0000000..4f91d94
--- /dev/null
+++ b/src/java/org/apache/cassandra/schema/MigrationManager.java
@@ -0,0 +1,494 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.schema;
+
+import java.io.IOException;
+import java.util.*;
+import java.util.concurrent.*;
+import java.lang.management.ManagementFactory;
+import java.lang.management.RuntimeMXBean;
+
+import com.google.common.util.concurrent.Futures;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.concurrent.ScheduledExecutors;
+import org.apache.cassandra.db.*;
+import org.apache.cassandra.exceptions.AlreadyExistsException;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.gms.*;
+import org.apache.cassandra.io.IVersionedSerializer;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.net.Message;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.schema.Keyspaces.KeyspacesDiff;
+import org.apache.cassandra.utils.FBUtilities;
+
+import static org.apache.cassandra.concurrent.Stage.MIGRATION;
+import static org.apache.cassandra.net.Verb.SCHEMA_PUSH_REQ;
+
+public class MigrationManager
+{
+ private static final Logger logger = LoggerFactory.getLogger(MigrationManager.class);
+
+ public static final MigrationManager instance = new MigrationManager();
+
+ private static final RuntimeMXBean runtimeMXBean = ManagementFactory.getRuntimeMXBean();
+
+ private static final int MIGRATION_DELAY_IN_MS = 60000;
+
+ private static final int MIGRATION_TASK_WAIT_IN_SECONDS = Integer.parseInt(System.getProperty("cassandra.migration_task_wait_in_seconds", "1"));
+
+ private MigrationManager() {}
+
+ public static void scheduleSchemaPull(InetAddressAndPort endpoint, EndpointState state)
+ {
+ UUID schemaVersion = state.getSchemaVersion();
+ if (!endpoint.equals(FBUtilities.getBroadcastAddressAndPort()) && schemaVersion != null)
+ maybeScheduleSchemaPull(schemaVersion, endpoint, state.getApplicationState(ApplicationState.RELEASE_VERSION).value);
+ }
+
+ /**
+ * If versions differ this node sends request with local migration list to the endpoint
+ * and expecting to receive a list of migrations to apply locally.
+ */
+ private static void maybeScheduleSchemaPull(final UUID theirVersion, final InetAddressAndPort endpoint, String releaseVersion)
+ {
+ String ourMajorVersion = FBUtilities.getReleaseVersionMajor();
+ if (!releaseVersion.startsWith(ourMajorVersion))
+ {
+ logger.debug("Not pulling schema because release version in Gossip is not major version {}, it is {}", ourMajorVersion, releaseVersion);
+ return;
+ }
+ if (Schema.instance.getVersion() == null)
+ {
+ logger.debug("Not pulling schema from {}, because local schema version is not known yet",
+ endpoint);
+ SchemaMigrationDiagnostics.unknownLocalSchemaVersion(endpoint, theirVersion);
+ return;
+ }
+ if (Schema.instance.isSameVersion(theirVersion))
+ {
+ logger.debug("Not pulling schema from {}, because schema versions match ({})",
+ endpoint,
+ Schema.schemaVersionToString(theirVersion));
+ SchemaMigrationDiagnostics.versionMatch(endpoint, theirVersion);
+ return;
+ }
+ if (!shouldPullSchemaFrom(endpoint))
+ {
+ logger.debug("Not pulling schema from {}, because versions match ({}/{}), or shouldPullSchemaFrom returned false",
+ endpoint, Schema.instance.getVersion(), theirVersion);
+ SchemaMigrationDiagnostics.skipPull(endpoint, theirVersion);
+ return;
+ }
+
+ if (Schema.instance.isEmpty() || runtimeMXBean.getUptime() < MIGRATION_DELAY_IN_MS)
+ {
+ // If we think we may be bootstrapping or have recently started, submit MigrationTask immediately
+ logger.debug("Immediately submitting migration task for {}, " +
+ "schema versions: local={}, remote={}",
+ endpoint,
+ Schema.schemaVersionToString(Schema.instance.getVersion()),
+ Schema.schemaVersionToString(theirVersion));
+ submitMigrationTask(endpoint);
+ }
+ else
+ {
+ // Include a delay to make sure we have a chance to apply any changes being
+ // pushed out simultaneously. See CASSANDRA-5025
+ Runnable runnable = () ->
+ {
+ // grab the latest version of the schema since it may have changed again since the initial scheduling
+ UUID epSchemaVersion = Gossiper.instance.getSchemaVersion(endpoint);
+ if (epSchemaVersion == null)
+ {
+ logger.debug("epState vanished for {}, not submitting migration task", endpoint);
+ return;
+ }
+ if (Schema.instance.isSameVersion(epSchemaVersion))
+ {
+ logger.debug("Not submitting migration task for {} because our versions match ({})", endpoint, epSchemaVersion);
+ return;
+ }
+ logger.debug("Submitting migration task for {}, schema version mismatch: local={}, remote={}",
+ endpoint,
+ Schema.schemaVersionToString(Schema.instance.getVersion()),
+ Schema.schemaVersionToString(epSchemaVersion));
+ submitMigrationTask(endpoint);
+ };
+ ScheduledExecutors.nonPeriodicTasks.schedule(runnable, MIGRATION_DELAY_IN_MS, TimeUnit.MILLISECONDS);
+ }
+ }
+
+ private static Future<?> submitMigrationTask(InetAddressAndPort endpoint)
+ {
+ /*
+ * Do not de-ref the future because that causes distributed deadlock (CASSANDRA-3832) because we are
+ * running in the gossip stage.
+ */
+ return MIGRATION.submit(new MigrationTask(endpoint));
+ }
+
+ static boolean shouldPullSchemaFrom(InetAddressAndPort endpoint)
+ {
+ /*
+ * Don't request schema from nodes with a differnt or unknonw major version (may have incompatible schema)
+ * Don't request schema from fat clients
+ */
+ return MessagingService.instance().versions.knows(endpoint)
+ && MessagingService.instance().versions.getRaw(endpoint) == MessagingService.current_version
+ && !Gossiper.instance.isGossipOnlyMember(endpoint);
+ }
+
+ private static boolean shouldPushSchemaTo(InetAddressAndPort endpoint)
+ {
+ // only push schema to nodes with known and equal versions
+ return !endpoint.equals(FBUtilities.getBroadcastAddressAndPort())
+ && MessagingService.instance().versions.knows(endpoint)
+ && MessagingService.instance().versions.getRaw(endpoint) == MessagingService.current_version;
+ }
+
+ public static boolean isReadyForBootstrap()
+ {
+ return MigrationTask.getInflightTasks().isEmpty();
+ }
+
+ public static void waitUntilReadyForBootstrap()
+ {
+ CountDownLatch completionLatch;
+ while ((completionLatch = MigrationTask.getInflightTasks().poll()) != null)
+ {
+ try
+ {
+ if (!completionLatch.await(MIGRATION_TASK_WAIT_IN_SECONDS, TimeUnit.SECONDS))
+ logger.error("Migration task failed to complete");
+ }
+ catch (InterruptedException e)
+ {
+ Thread.currentThread().interrupt();
+ logger.error("Migration task was interrupted");
+ }
+ }
+ }
+
+ public static void announceNewKeyspace(KeyspaceMetadata ksm) throws ConfigurationException
+ {
+ announceNewKeyspace(ksm, false);
+ }
+
+ public static void announceNewKeyspace(KeyspaceMetadata ksm, boolean announceLocally) throws ConfigurationException
+ {
+ announceNewKeyspace(ksm, FBUtilities.timestampMicros(), announceLocally);
+ }
+
+ public static void announceNewKeyspace(KeyspaceMetadata ksm, long timestamp, boolean announceLocally) throws ConfigurationException
+ {
+ ksm.validate();
+
+ if (Schema.instance.getKeyspaceMetadata(ksm.name) != null)
+ throw new AlreadyExistsException(ksm.name);
+
+ logger.info("Create new Keyspace: {}", ksm);
+ announce(SchemaKeyspace.makeCreateKeyspaceMutation(ksm, timestamp), announceLocally);
+ }
+
+ public static void announceNewTable(TableMetadata cfm)
+ {
+ announceNewTable(cfm, true, FBUtilities.timestampMicros());
+ }
+
+ /**
+ * Announces the table even if the definition is already know locally.
+ * This should generally be avoided but is used internally when we want to force the most up to date version of
+ * a system table schema (Note that we don't know if the schema we force _is_ the most recent version or not, we
+ * just rely on idempotency to basically ignore that announce if it's not. That's why we can't use announceTableUpdate
+ * it would for instance delete new columns if this is not called with the most up-to-date version)
+ *
+ * Note that this is only safe for system tables where we know the id is fixed and will be the same whatever version
+ * of the definition is used.
+ */
+ public static void forceAnnounceNewTable(TableMetadata cfm)
+ {
+ announceNewTable(cfm, false, 0);
+ }
+
+ private static void announceNewTable(TableMetadata cfm, boolean throwOnDuplicate, long timestamp)
+ {
+ cfm.validate();
+
+ KeyspaceMetadata ksm = Schema.instance.getKeyspaceMetadata(cfm.keyspace);
+ if (ksm == null)
+ throw new ConfigurationException(String.format("Cannot add table '%s' to non existing keyspace '%s'.", cfm.name, cfm.keyspace));
+ // If we have a table or a view which has the same name, we can't add a new one
+ else if (throwOnDuplicate && ksm.getTableOrViewNullable(cfm.name) != null)
+ throw new AlreadyExistsException(cfm.keyspace, cfm.name);
+
+ logger.info("Create new table: {}", cfm);
+ announce(SchemaKeyspace.makeCreateTableMutation(ksm, cfm, timestamp), false);
+ }
+
+ static void announceKeyspaceUpdate(KeyspaceMetadata ksm)
+ {
+ ksm.validate();
+
+ KeyspaceMetadata oldKsm = Schema.instance.getKeyspaceMetadata(ksm.name);
+ if (oldKsm == null)
+ throw new ConfigurationException(String.format("Cannot update non existing keyspace '%s'.", ksm.name));
+
+ logger.info("Update Keyspace '{}' From {} To {}", ksm.name, oldKsm, ksm);
+ announce(SchemaKeyspace.makeCreateKeyspaceMutation(ksm.name, ksm.params, FBUtilities.timestampMicros()), false);
+ }
+
+ public static void announceTableUpdate(TableMetadata tm)
+ {
+ announceTableUpdate(tm, false);
+ }
+
+ public static void announceTableUpdate(TableMetadata updated, boolean announceLocally)
+ {
+ updated.validate();
+
+ TableMetadata current = Schema.instance.getTableMetadata(updated.keyspace, updated.name);
+ if (current == null)
+ throw new ConfigurationException(String.format("Cannot update non existing table '%s' in keyspace '%s'.", updated.name, updated.keyspace));
+ KeyspaceMetadata ksm = Schema.instance.getKeyspaceMetadata(current.keyspace);
+
+ updated.validateCompatibility(current);
+
+ long timestamp = FBUtilities.timestampMicros();
+
+ logger.info("Update table '{}/{}' From {} To {}", current.keyspace, current.name, current, updated);
+ Mutation.SimpleBuilder builder = SchemaKeyspace.makeUpdateTableMutation(ksm, current, updated, timestamp);
+
+ announce(builder, announceLocally);
+ }
+
+ static void announceKeyspaceDrop(String ksName)
+ {
+ KeyspaceMetadata oldKsm = Schema.instance.getKeyspaceMetadata(ksName);
+ if (oldKsm == null)
+ throw new ConfigurationException(String.format("Cannot drop non existing keyspace '%s'.", ksName));
+
+ logger.info("Drop Keyspace '{}'", oldKsm.name);
+ announce(SchemaKeyspace.makeDropKeyspaceMutation(oldKsm, FBUtilities.timestampMicros()), false);
+ }
+
+ public static void announceTableDrop(String ksName, String cfName, boolean announceLocally)
+ {
+ TableMetadata tm = Schema.instance.getTableMetadata(ksName, cfName);
+ if (tm == null)
+ throw new ConfigurationException(String.format("Cannot drop non existing table '%s' in keyspace '%s'.", cfName, ksName));
+ KeyspaceMetadata ksm = Schema.instance.getKeyspaceMetadata(ksName);
+
+ logger.info("Drop table '{}/{}'", tm.keyspace, tm.name);
+ announce(SchemaKeyspace.makeDropTableMutation(ksm, tm, FBUtilities.timestampMicros()), announceLocally);
+ }
+
+ /**
+ * actively announce a new version to active hosts via rpc
+ * @param schema The schema mutation to be applied
+ */
+ private static void announce(Mutation.SimpleBuilder schema, boolean announceLocally)
+ {
+ List<Mutation> mutations = Collections.singletonList(schema.build());
+
+ if (announceLocally)
+ Schema.instance.merge(mutations);
+ else
+ announce(mutations);
+ }
+
+ public static void announce(Mutation change)
+ {
+ announce(Collections.singleton(change));
+ }
+
+ public static void announce(Collection<Mutation> schema)
+ {
+ Future<?> f = MIGRATION.submit(() -> Schema.instance.mergeAndAnnounceVersion(schema));
+
+ Set<InetAddressAndPort> schemaDestinationEndpoints = new HashSet<>();
+ Set<InetAddressAndPort> schemaEndpointsIgnored = new HashSet<>();
+ Message<Collection<Mutation>> message = Message.out(SCHEMA_PUSH_REQ, schema);
+ for (InetAddressAndPort endpoint : Gossiper.instance.getLiveMembers())
+ {
+ if (shouldPushSchemaTo(endpoint))
+ {
+ MessagingService.instance().send(message, endpoint);
+ schemaDestinationEndpoints.add(endpoint);
+ }
+ else
+ {
+ schemaEndpointsIgnored.add(endpoint);
+ }
+ }
+
+ SchemaAnnouncementDiagnostics.schemaMutationsAnnounced(schemaDestinationEndpoints, schemaEndpointsIgnored);
+ FBUtilities.waitOnFuture(f);
+ }
+
+ public static KeyspacesDiff announce(SchemaTransformation transformation, boolean locally)
+ {
+ long now = FBUtilities.timestampMicros();
+
+ Future<Schema.TransformationResult> future =
+ MIGRATION.submit(() -> Schema.instance.transform(transformation, locally, now));
+
+ Schema.TransformationResult result = Futures.getUnchecked(future);
+ if (!result.success)
+ throw result.exception;
+
+ if (locally || result.diff.isEmpty())
+ return result.diff;
+
+ Set<InetAddressAndPort> schemaDestinationEndpoints = new HashSet<>();
+ Set<InetAddressAndPort> schemaEndpointsIgnored = new HashSet<>();
+ Message<Collection<Mutation>> message = Message.out(SCHEMA_PUSH_REQ, result.mutations);
+ for (InetAddressAndPort endpoint : Gossiper.instance.getLiveMembers())
+ {
+ if (shouldPushSchemaTo(endpoint))
+ {
+ MessagingService.instance().send(message, endpoint);
+ schemaDestinationEndpoints.add(endpoint);
+ }
+ else
+ {
+ schemaEndpointsIgnored.add(endpoint);
+ }
+ }
+
+ SchemaAnnouncementDiagnostics.schemaTransformationAnnounced(schemaDestinationEndpoints, schemaEndpointsIgnored,
+ transformation);
+
+ return result.diff;
+ }
+
+ /**
+ * Clear all locally stored schema information and reset schema to initial state.
+ * Called by user (via JMX) who wants to get rid of schema disagreement.
+ */
+ public static void resetLocalSchema()
+ {
+ logger.info("Starting local schema reset...");
+
+ logger.debug("Truncating schema tables...");
+
+ SchemaMigrationDiagnostics.resetLocalSchema();
+
+ SchemaKeyspace.truncate();
+
+ logger.debug("Clearing local schema keyspace definitions...");
+
+ Schema.instance.clear();
+
+ Set<InetAddressAndPort> liveEndpoints = Gossiper.instance.getLiveMembers();
+ liveEndpoints.remove(FBUtilities.getBroadcastAddressAndPort());
+
+ // force migration if there are nodes around
+ for (InetAddressAndPort node : liveEndpoints)
+ {
+ if (shouldPullSchemaFrom(node))
+ {
+ logger.debug("Requesting schema from {}", node);
+ FBUtilities.waitOnFuture(submitMigrationTask(node));
+ break;
+ }
+ }
+
+ logger.info("Local schema reset is complete.");
+ }
+
+ /**
+ * We have a set of non-local, distributed system keyspaces, e.g. system_traces, system_auth, etc.
+ * (see {@link SchemaConstants#REPLICATED_SYSTEM_KEYSPACE_NAMES}), that need to be created on cluster initialisation,
+ * and later evolved on major upgrades (sometimes minor too). This method compares the current known definitions
+ * of the tables (if the keyspace exists) to the expected, most modern ones expected by the running version of C*;
+ * if any changes have been detected, a schema Mutation will be created which, when applied, should make
+ * cluster's view of that keyspace aligned with the expected modern definition.
+ *
+ * @param keyspace the expected modern definition of the keyspace
+ * @param generation timestamp to use for the table changes in the schema mutation
+ *
+ * @return empty Optional if the current definition is up to date, or an Optional with the Mutation that would
+ * bring the schema in line with the expected definition.
+ */
+ public static Optional<Mutation> evolveSystemKeyspace(KeyspaceMetadata keyspace, long generation)
+ {
+ Mutation.SimpleBuilder builder = null;
+
+ KeyspaceMetadata definedKeyspace = Schema.instance.getKeyspaceMetadata(keyspace.name);
+ Tables definedTables = null == definedKeyspace ? Tables.none() : definedKeyspace.tables;
+
+ for (TableMetadata table : keyspace.tables)
+ {
+ if (table.equals(definedTables.getNullable(table.name)))
+ continue;
+
+ if (null == builder)
+ {
+ // for the keyspace definition itself (name, replication, durability) always use generation 0;
+ // this ensures that any changes made to replication by the user will never be overwritten.
+ builder = SchemaKeyspace.makeCreateKeyspaceMutation(keyspace.name, keyspace.params, 0);
+
+ // now set the timestamp to generation, so the tables have the expected timestamp
+ builder.timestamp(generation);
+ }
+
+ // for table definitions always use the provided generation; these tables, unlike their containing
+ // keyspaces, are *NOT* meant to be altered by the user; if their definitions need to change,
+ // the schema must be updated in code, and the appropriate generation must be bumped.
+ SchemaKeyspace.addTableToSchemaMutation(table, true, builder);
+ }
+
+ return builder == null ? Optional.empty() : Optional.of(builder.build());
+ }
+
+ public static class MigrationsSerializer implements IVersionedSerializer<Collection<Mutation>>
+ {
+ public static MigrationsSerializer instance = new MigrationsSerializer();
+
+ public void serialize(Collection<Mutation> schema, DataOutputPlus out, int version) throws IOException
+ {
+ out.writeInt(schema.size());
+ for (Mutation mutation : schema)
+ Mutation.serializer.serialize(mutation, out, version);
+ }
+
+ public Collection<Mutation> deserialize(DataInputPlus in, int version) throws IOException
+ {
+ int count = in.readInt();
+ Collection<Mutation> schema = new ArrayList<>(count);
+
+ for (int i = 0; i < count; i++)
+ schema.add(Mutation.serializer.deserialize(in, version));
+
+ return schema;
+ }
+
+ public long serializedSize(Collection<Mutation> schema, int version)
+ {
+ int size = TypeSizes.sizeof(schema.size());
+ for (Mutation mutation : schema)
+ size += mutation.serializedSize(version);
+ return size;
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/schema/MigrationTask.java b/src/java/org/apache/cassandra/schema/MigrationTask.java
new file mode 100644
index 0000000..3308893
--- /dev/null
+++ b/src/java/org/apache/cassandra/schema/MigrationTask.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.schema;
+
+import java.util.Collection;
+import java.util.EnumSet;
+import java.util.Set;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.CountDownLatch;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.db.Mutation;
+import org.apache.cassandra.db.SystemKeyspace;
+import org.apache.cassandra.db.SystemKeyspace.BootstrapState;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.gms.FailureDetector;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.net.RequestCallback;
+import org.apache.cassandra.net.Message;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.utils.WrappedRunnable;
+
+import static org.apache.cassandra.net.NoPayload.noPayload;
+import static org.apache.cassandra.net.Verb.SCHEMA_PULL_REQ;
+
+final class MigrationTask extends WrappedRunnable
+{
+ private static final Logger logger = LoggerFactory.getLogger(MigrationTask.class);
+
+ private static final ConcurrentLinkedQueue<CountDownLatch> inflightTasks = new ConcurrentLinkedQueue<>();
+
+ private static final Set<BootstrapState> monitoringBootstrapStates = EnumSet.of(BootstrapState.NEEDS_BOOTSTRAP, BootstrapState.IN_PROGRESS);
+
+ private final InetAddressAndPort endpoint;
+
+ MigrationTask(InetAddressAndPort endpoint)
+ {
+ this.endpoint = endpoint;
+ SchemaMigrationDiagnostics.taskCreated(endpoint);
+ }
+
+ static ConcurrentLinkedQueue<CountDownLatch> getInflightTasks()
+ {
+ return inflightTasks;
+ }
+
+ public void runMayThrow() throws Exception
+ {
+ if (!FailureDetector.instance.isAlive(endpoint))
+ {
+ logger.warn("Can't send schema pull request: node {} is down.", endpoint);
+ SchemaMigrationDiagnostics.taskSendAborted(endpoint);
+ return;
+ }
+
+ // There is a chance that quite some time could have passed between now and the MM#maybeScheduleSchemaPull(),
+ // potentially enough for the endpoint node to restart - which is an issue if it does restart upgraded, with
+ // a higher major.
+ if (!MigrationManager.shouldPullSchemaFrom(endpoint))
+ {
+ logger.info("Skipped sending a migration request: node {} has a higher major version now.", endpoint);
+ SchemaMigrationDiagnostics.taskSendAborted(endpoint);
+ return;
+ }
+
+ Message message = Message.out(SCHEMA_PULL_REQ, noPayload);
+
+ final CountDownLatch completionLatch = new CountDownLatch(1);
+
+ RequestCallback<Collection<Mutation>> cb = msg ->
+ {
+ try
+ {
+ Schema.instance.mergeAndAnnounceVersion(msg.payload);
+ }
+ catch (ConfigurationException e)
+ {
+ logger.error("Configuration exception merging remote schema", e);
+ }
+ finally
+ {
+ completionLatch.countDown();
+ }
+ };
+
+ // Only save the latches if we need bootstrap or are bootstrapping
+ if (monitoringBootstrapStates.contains(SystemKeyspace.getBootstrapState()))
+ inflightTasks.offer(completionLatch);
+
+ MessagingService.instance().sendWithCallback(message, endpoint, cb);
+
+ SchemaMigrationDiagnostics.taskRequestSend(endpoint);
+ }
+}
diff --git a/src/java/org/apache/cassandra/schema/ReplicationParams.java b/src/java/org/apache/cassandra/schema/ReplicationParams.java
index 21c029e..048b4ed 100644
--- a/src/java/org/apache/cassandra/schema/ReplicationParams.java
+++ b/src/java/org/apache/cassandra/schema/ReplicationParams.java
@@ -25,6 +25,7 @@
import com.google.common.collect.ImmutableMap;
import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.CqlBuilder;
import org.apache.cassandra.locator.*;
import org.apache.cassandra.service.StorageService;
@@ -51,6 +52,11 @@
return new ReplicationParams(SimpleStrategy.class, ImmutableMap.of("replication_factor", Integer.toString(replicationFactor)));
}
+ static ReplicationParams simple(String replicationFactor)
+ {
+ return new ReplicationParams(SimpleStrategy.class, ImmutableMap.of("replication_factor", replicationFactor));
+ }
+
static ReplicationParams nts(Object... args)
{
assert args.length % 2 == 0;
@@ -58,9 +64,7 @@
Map<String, String> options = new HashMap<>();
for (int i = 0; i < args.length; i += 2)
{
- String dc = (String) args[i];
- Integer rf = (Integer) args[i + 1];
- options.put(dc, rf.toString());
+ options.put((String) args[i], args[i + 1].toString());
}
return new ReplicationParams(NetworkTopologyStrategy.class, options);
@@ -74,11 +78,18 @@
AbstractReplicationStrategy.validateReplicationStrategy(name, klass, tmd, eps, options);
}
- public static ReplicationParams fromMap(Map<String, String> map)
+ public static ReplicationParams fromMap(Map<String, String> map) {
+ return fromMapWithDefaults(map, new HashMap<>());
+ }
+
+ public static ReplicationParams fromMapWithDefaults(Map<String, String> map, Map<String, String> defaults)
{
Map<String, String> options = new HashMap<>(map);
String className = options.remove(CLASS);
+
Class<? extends AbstractReplicationStrategy> klass = AbstractReplicationStrategy.getClass(className);
+ AbstractReplicationStrategy.prepareReplicationStrategyOptions(klass, options, defaults);
+
return new ReplicationParams(klass, options);
}
@@ -118,4 +129,21 @@
helper.add(entry.getKey(), entry.getValue());
return helper.toString();
}
+
+ public void appendCqlTo(CqlBuilder builder)
+ {
+ String classname = "org.apache.cassandra.locator".equals(klass.getPackage().getName()) ? klass.getSimpleName()
+ : klass.getName();
+ builder.append("{'class': ")
+ .appendWithSingleQuotes(classname);
+
+ options.forEach((k, v) -> {
+ builder.append(", ")
+ .appendWithSingleQuotes(k)
+ .append(": ")
+ .appendWithSingleQuotes(v);
+ });
+
+ builder.append('}');
+ }
}
diff --git a/src/java/org/apache/cassandra/schema/Schema.java b/src/java/org/apache/cassandra/schema/Schema.java
new file mode 100644
index 0000000..e2be6ee
--- /dev/null
+++ b/src/java/org/apache/cassandra/schema/Schema.java
@@ -0,0 +1,904 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.schema;
+
+import java.util.*;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.stream.Collectors;
+import javax.annotation.Nullable;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.MapDifference;
+import com.google.common.collect.Sets;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.functions.*;
+import org.apache.cassandra.db.*;
+import org.apache.cassandra.db.commitlog.CommitLog;
+import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.UserType;
+import org.apache.cassandra.db.virtual.VirtualKeyspaceRegistry;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.exceptions.UnknownTableException;
+import org.apache.cassandra.gms.ApplicationState;
+import org.apache.cassandra.gms.Gossiper;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.locator.LocalStrategy;
+import org.apache.cassandra.schema.KeyspaceMetadata.KeyspaceDiff;
+import org.apache.cassandra.schema.Keyspaces.KeyspacesDiff;
+import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.utils.Pair;
+import org.cliffc.high_scale_lib.NonBlockingHashMap;
+
+import static java.lang.String.format;
+
+import static com.google.common.collect.Iterables.size;
+
+public final class Schema
+{
+ public static final Schema instance = new Schema();
+
+ private volatile Keyspaces keyspaces = Keyspaces.none();
+
+ // UUID -> mutable metadata ref map. We have to update these in place every time a table changes.
+ private final Map<TableId, TableMetadataRef> metadataRefs = new NonBlockingHashMap<>();
+
+ // (keyspace name, index name) -> mutable metadata ref map. We have to update these in place every time an index changes.
+ private final Map<Pair<String, String>, TableMetadataRef> indexMetadataRefs = new NonBlockingHashMap<>();
+
+ // Keyspace objects, one per keyspace. Only one instance should ever exist for any given keyspace.
+ private final Map<String, Keyspace> keyspaceInstances = new NonBlockingHashMap<>();
+
+ private volatile UUID version;
+
+ private final List<SchemaChangeListener> changeListeners = new CopyOnWriteArrayList<>();
+
+ /**
+ * Initialize empty schema object and load the hardcoded system tables
+ */
+ private Schema()
+ {
+ if (DatabaseDescriptor.isDaemonInitialized() || DatabaseDescriptor.isToolInitialized())
+ {
+ load(SchemaKeyspace.metadata());
+ load(SystemKeyspace.metadata());
+ }
+ }
+
+ /**
+ * load keyspace (keyspace) definitions, but do not initialize the keyspace instances.
+ * Schema version may be updated as the result.
+ */
+ public void loadFromDisk()
+ {
+ loadFromDisk(true);
+ }
+
+ /**
+ * Load schema definitions from disk.
+ *
+ * @param updateVersion true if schema version needs to be updated
+ */
+ public void loadFromDisk(boolean updateVersion)
+ {
+ SchemaDiagnostics.schemataLoading(this);
+ SchemaKeyspace.fetchNonSystemKeyspaces().forEach(this::load);
+ if (updateVersion)
+ updateVersion();
+ SchemaDiagnostics.schemataLoaded(this);
+ }
+
+ /**
+ * Update (or insert) new keyspace definition
+ *
+ * @param ksm The metadata about keyspace
+ */
+ synchronized public void load(KeyspaceMetadata ksm)
+ {
+ KeyspaceMetadata previous = keyspaces.getNullable(ksm.name);
+
+ if (previous == null)
+ loadNew(ksm);
+ else
+ reload(previous, ksm);
+
+ keyspaces = keyspaces.withAddedOrUpdated(ksm);
+ }
+
+ private void loadNew(KeyspaceMetadata ksm)
+ {
+ ksm.tablesAndViews()
+ .forEach(metadata -> metadataRefs.put(metadata.id, new TableMetadataRef(metadata)));
+
+ ksm.tables
+ .indexTables()
+ .forEach((name, metadata) -> indexMetadataRefs.put(Pair.create(ksm.name, name), new TableMetadataRef(metadata)));
+
+ SchemaDiagnostics.metadataInitialized(this, ksm);
+ }
+
+ private void reload(KeyspaceMetadata previous, KeyspaceMetadata updated)
+ {
+ Keyspace keyspace = getKeyspaceInstance(updated.name);
+ if (null != keyspace)
+ keyspace.setMetadata(updated);
+
+ Tables.TablesDiff tablesDiff = Tables.diff(previous.tables, updated.tables);
+ Views.ViewsDiff viewsDiff = Views.diff(previous.views, updated.views);
+
+ MapDifference<String, TableMetadata> indexesDiff = previous.tables.indexesDiff(updated.tables);
+
+ // clean up after removed entries
+ tablesDiff.dropped.forEach(table -> metadataRefs.remove(table.id));
+ viewsDiff.dropped.forEach(view -> metadataRefs.remove(view.metadata.id));
+
+ indexesDiff.entriesOnlyOnLeft()
+ .values()
+ .forEach(indexTable -> indexMetadataRefs.remove(Pair.create(indexTable.keyspace, indexTable.indexName().get())));
+
+ // load up new entries
+ tablesDiff.created.forEach(table -> metadataRefs.put(table.id, new TableMetadataRef(table)));
+ viewsDiff.created.forEach(view -> metadataRefs.put(view.metadata.id, new TableMetadataRef(view.metadata)));
+
+ indexesDiff.entriesOnlyOnRight()
+ .values()
+ .forEach(indexTable -> indexMetadataRefs.put(Pair.create(indexTable.keyspace, indexTable.indexName().get()), new TableMetadataRef(indexTable)));
+
+ // refresh refs to updated ones
+ tablesDiff.altered.forEach(diff -> metadataRefs.get(diff.after.id).set(diff.after));
+ viewsDiff.altered.forEach(diff -> metadataRefs.get(diff.after.metadata.id).set(diff.after.metadata));
+
+ indexesDiff.entriesDiffering()
+ .values()
+ .stream()
+ .map(MapDifference.ValueDifference::rightValue)
+ .forEach(indexTable -> indexMetadataRefs.get(Pair.create(indexTable.keyspace, indexTable.indexName().get())).set(indexTable));
+
+ SchemaDiagnostics.metadataReloaded(this, previous, updated, tablesDiff, viewsDiff, indexesDiff);
+ }
+
+ public void registerListener(SchemaChangeListener listener)
+ {
+ changeListeners.add(listener);
+ }
+
+ @SuppressWarnings("unused")
+ public void unregisterListener(SchemaChangeListener listener)
+ {
+ changeListeners.remove(listener);
+ }
+
+ /**
+ * Get keyspace instance by name
+ *
+ * @param keyspaceName The name of the keyspace
+ *
+ * @return Keyspace object or null if keyspace was not found
+ */
+ public Keyspace getKeyspaceInstance(String keyspaceName)
+ {
+ return keyspaceInstances.get(keyspaceName);
+ }
+
+ public ColumnFamilyStore getColumnFamilyStoreInstance(TableId id)
+ {
+ TableMetadata metadata = getTableMetadata(id);
+ if (metadata == null)
+ return null;
+
+ Keyspace instance = getKeyspaceInstance(metadata.keyspace);
+ if (instance == null)
+ return null;
+
+ return instance.hasColumnFamilyStore(metadata.id)
+ ? instance.getColumnFamilyStore(metadata.id)
+ : null;
+ }
+
+ /**
+ * Store given Keyspace instance to the schema
+ *
+ * @param keyspace The Keyspace instance to store
+ *
+ * @throws IllegalArgumentException if Keyspace is already stored
+ */
+ public void storeKeyspaceInstance(Keyspace keyspace)
+ {
+ if (keyspaceInstances.containsKey(keyspace.getName()))
+ throw new IllegalArgumentException(String.format("Keyspace %s was already initialized.", keyspace.getName()));
+
+ keyspaceInstances.put(keyspace.getName(), keyspace);
+ }
+
+ /**
+ * Remove keyspace from schema
+ *
+ * @param keyspaceName The name of the keyspace to remove
+ *
+ * @return removed keyspace instance or null if it wasn't found
+ */
+ public Keyspace removeKeyspaceInstance(String keyspaceName)
+ {
+ return keyspaceInstances.remove(keyspaceName);
+ }
+
+ public Keyspaces snapshot()
+ {
+ return keyspaces;
+ }
+
+ /**
+ * Remove keyspace definition from system
+ *
+ * @param ksm The keyspace definition to remove
+ */
+ synchronized void unload(KeyspaceMetadata ksm)
+ {
+ keyspaces = keyspaces.without(ksm.name);
+
+ ksm.tablesAndViews()
+ .forEach(t -> metadataRefs.remove(t.id));
+
+ ksm.tables
+ .indexTables()
+ .keySet()
+ .forEach(name -> indexMetadataRefs.remove(Pair.create(ksm.name, name)));
+
+ SchemaDiagnostics.metadataRemoved(this, ksm);
+ }
+
+ public int getNumberOfTables()
+ {
+ return keyspaces.stream().mapToInt(k -> size(k.tablesAndViews())).sum();
+ }
+
+ public ViewMetadata getView(String keyspaceName, String viewName)
+ {
+ assert keyspaceName != null;
+ KeyspaceMetadata ksm = keyspaces.getNullable(keyspaceName);
+ return (ksm == null) ? null : ksm.views.getNullable(viewName);
+ }
+
+ /**
+ * Get metadata about keyspace by its name
+ *
+ * @param keyspaceName The name of the keyspace
+ *
+ * @return The keyspace metadata or null if it wasn't found
+ */
+ public KeyspaceMetadata getKeyspaceMetadata(String keyspaceName)
+ {
+ assert keyspaceName != null;
+ KeyspaceMetadata keyspace = keyspaces.getNullable(keyspaceName);
+ return null != keyspace ? keyspace : VirtualKeyspaceRegistry.instance.getKeyspaceMetadataNullable(keyspaceName);
+ }
+
+ private Set<String> getNonSystemKeyspacesSet()
+ {
+ return Sets.difference(keyspaces.names(), SchemaConstants.LOCAL_SYSTEM_KEYSPACE_NAMES);
+ }
+
+ /**
+ * @return collection of the non-system keyspaces (note that this count as system only the
+ * non replicated keyspaces, so keyspace like system_traces which are replicated are actually
+ * returned. See getUserKeyspace() below if you don't want those)
+ */
+ public ImmutableList<String> getNonSystemKeyspaces()
+ {
+ return ImmutableList.copyOf(getNonSystemKeyspacesSet());
+ }
+
+ /**
+ * @return a collection of keyspaces that do not use LocalStrategy for replication
+ */
+ public List<String> getNonLocalStrategyKeyspaces()
+ {
+ return keyspaces.stream()
+ .filter(keyspace -> keyspace.params.replication.klass != LocalStrategy.class)
+ .map(keyspace -> keyspace.name)
+ .collect(Collectors.toList());
+ }
+
+ /**
+ * @return collection of the user defined keyspaces
+ */
+ public List<String> getUserKeyspaces()
+ {
+ return ImmutableList.copyOf(Sets.difference(getNonSystemKeyspacesSet(), SchemaConstants.REPLICATED_SYSTEM_KEYSPACE_NAMES));
+ }
+
+ /**
+ * Get metadata about keyspace inner ColumnFamilies
+ *
+ * @param keyspaceName The name of the keyspace
+ *
+ * @return metadata about ColumnFamilies the belong to the given keyspace
+ */
+ public Iterable<TableMetadata> getTablesAndViews(String keyspaceName)
+ {
+ assert keyspaceName != null;
+ KeyspaceMetadata ksm = keyspaces.getNullable(keyspaceName);
+ assert ksm != null;
+ return ksm.tablesAndViews();
+ }
+
+ /**
+ * @return collection of the all keyspace names registered in the system (system and non-system)
+ */
+ public Set<String> getKeyspaces()
+ {
+ return keyspaces.names();
+ }
+
+ /* TableMetadata/Ref query/control methods */
+
+ /**
+ * Given a keyspace name and table/view name, get the table metadata
+ * reference. If the keyspace name or table/view name is not present
+ * this method returns null.
+ *
+ * @return TableMetadataRef object or null if it wasn't found
+ */
+ public TableMetadataRef getTableMetadataRef(String keyspace, String table)
+ {
+ TableMetadata tm = getTableMetadata(keyspace, table);
+ return tm == null
+ ? null
+ : metadataRefs.get(tm.id);
+ }
+
+ public TableMetadataRef getIndexTableMetadataRef(String keyspace, String index)
+ {
+ return indexMetadataRefs.get(Pair.create(keyspace, index));
+ }
+
+ Map<Pair<String, String>, TableMetadataRef> getIndexTableMetadataRefs()
+ {
+ return indexMetadataRefs;
+ }
+
+ /**
+ * Get Table metadata by its identifier
+ *
+ * @param id table or view identifier
+ *
+ * @return metadata about Table or View
+ */
+ public TableMetadataRef getTableMetadataRef(TableId id)
+ {
+ return metadataRefs.get(id);
+ }
+
+ public TableMetadataRef getTableMetadataRef(Descriptor descriptor)
+ {
+ return getTableMetadataRef(descriptor.ksname, descriptor.cfname);
+ }
+
+ Map<TableId, TableMetadataRef> getTableMetadataRefs()
+ {
+ return metadataRefs;
+ }
+
+ /**
+ * Given a keyspace name and table name, get the table
+ * meta data. If the keyspace name or table name is not valid
+ * this function returns null.
+ *
+ * @param keyspace The keyspace name
+ * @param table The table name
+ *
+ * @return TableMetadata object or null if it wasn't found
+ */
+ public TableMetadata getTableMetadata(String keyspace, String table)
+ {
+ assert keyspace != null;
+ assert table != null;
+
+ KeyspaceMetadata ksm = getKeyspaceMetadata(keyspace);
+ return ksm == null
+ ? null
+ : ksm.getTableOrViewNullable(table);
+ }
+
+ @Nullable
+ public TableMetadata getTableMetadata(TableId id)
+ {
+ TableMetadata table = keyspaces.getTableOrViewNullable(id);
+ return null != table ? table : VirtualKeyspaceRegistry.instance.getTableMetadataNullable(id);
+ }
+
+ public TableMetadata validateTable(String keyspaceName, String tableName)
+ {
+ if (tableName.isEmpty())
+ throw new InvalidRequestException("non-empty table is required");
+
+ KeyspaceMetadata keyspace = getKeyspaceMetadata(keyspaceName);
+ if (keyspace == null)
+ throw new KeyspaceNotDefinedException(format("keyspace %s does not exist", keyspaceName));
+
+ TableMetadata metadata = keyspace.getTableOrViewNullable(tableName);
+ if (metadata == null)
+ throw new InvalidRequestException(format("table %s does not exist", tableName));
+
+ return metadata;
+ }
+
+ public TableMetadata getTableMetadata(Descriptor descriptor)
+ {
+ return getTableMetadata(descriptor.ksname, descriptor.cfname);
+ }
+
+ /**
+ * @throws UnknownTableException if the table couldn't be found in the metadata
+ */
+ public TableMetadata getExistingTableMetadata(TableId id) throws UnknownTableException
+ {
+ TableMetadata metadata = getTableMetadata(id);
+ if (metadata != null)
+ return metadata;
+
+ String message =
+ String.format("Couldn't find table with id %s. If a table was just created, this is likely due to the schema"
+ + "not being fully propagated. Please wait for schema agreement on table creation.",
+ id);
+ throw new UnknownTableException(message, id);
+ }
+
+ /* Function helpers */
+
+ /**
+ * Get all function overloads with the specified name
+ *
+ * @param name fully qualified function name
+ * @return an empty list if the keyspace or the function name are not found;
+ * a non-empty collection of {@link Function} otherwise
+ */
+ public Collection<Function> getFunctions(FunctionName name)
+ {
+ if (!name.hasKeyspace())
+ throw new IllegalArgumentException(String.format("Function name must be fully qualified: got %s", name));
+
+ KeyspaceMetadata ksm = getKeyspaceMetadata(name.keyspace);
+ return ksm == null
+ ? Collections.emptyList()
+ : ksm.functions.get(name);
+ }
+
+ /**
+ * Find the function with the specified name
+ *
+ * @param name fully qualified function name
+ * @param argTypes function argument types
+ * @return an empty {@link Optional} if the keyspace or the function name are not found;
+ * a non-empty optional of {@link Function} otherwise
+ */
+ public Optional<Function> findFunction(FunctionName name, List<AbstractType<?>> argTypes)
+ {
+ if (!name.hasKeyspace())
+ throw new IllegalArgumentException(String.format("Function name must be fully quallified: got %s", name));
+
+ KeyspaceMetadata ksm = getKeyspaceMetadata(name.keyspace);
+ return ksm == null
+ ? Optional.empty()
+ : ksm.functions.find(name, argTypes);
+ }
+
+ /* Version control */
+
+ /**
+ * @return current schema version
+ */
+ public UUID getVersion()
+ {
+ return version;
+ }
+
+ /**
+ * Checks whether the given schema version is the same as the current local schema.
+ */
+ public boolean isSameVersion(UUID schemaVersion)
+ {
+ return schemaVersion != null && schemaVersion.equals(version);
+ }
+
+ /**
+ * Checks whether the current schema is empty.
+ */
+ public boolean isEmpty()
+ {
+ return SchemaConstants.emptyVersion.equals(version);
+ }
+
+ /**
+ * Read schema from system keyspace and calculate MD5 digest of every row, resulting digest
+ * will be converted into UUID which would act as content-based version of the schema.
+ */
+ public void updateVersion()
+ {
+ version = SchemaKeyspace.calculateSchemaDigest();
+ SystemKeyspace.updateSchemaVersion(version);
+ SchemaDiagnostics.versionUpdated(this);
+ }
+
+ /*
+ * Like updateVersion, but also announces via gossip
+ */
+ public void updateVersionAndAnnounce()
+ {
+ updateVersion();
+ passiveAnnounceVersion();
+ }
+
+ /**
+ * Announce my version passively over gossip.
+ * Used to notify nodes as they arrive in the cluster.
+ */
+ private void passiveAnnounceVersion()
+ {
+ Gossiper.instance.addLocalApplicationState(ApplicationState.SCHEMA, StorageService.instance.valueFactory.schema(version));
+ SchemaDiagnostics.versionAnnounced(this);
+ }
+
+ /**
+ * Clear all KS/CF metadata and reset version.
+ */
+ public synchronized void clear()
+ {
+ getNonSystemKeyspaces().forEach(k -> unload(getKeyspaceMetadata(k)));
+ updateVersionAndAnnounce();
+ SchemaDiagnostics.schemataCleared(this);
+ }
+
+ /*
+ * Reload schema from local disk. Useful if a user made changes to schema tables by hand, or has suspicion that
+ * in-memory representation got out of sync somehow with what's on disk.
+ */
+ public synchronized void reloadSchemaAndAnnounceVersion()
+ {
+ Keyspaces before = keyspaces.filter(k -> !SchemaConstants.isLocalSystemKeyspace(k.name));
+ Keyspaces after = SchemaKeyspace.fetchNonSystemKeyspaces();
+ merge(Keyspaces.diff(before, after));
+ updateVersionAndAnnounce();
+ }
+
+ /**
+ * Merge remote schema in form of mutations with local and mutate ks/cf metadata objects
+ * (which also involves fs operations on add/drop ks/cf)
+ *
+ * @param mutations the schema changes to apply
+ *
+ * @throws ConfigurationException If one of metadata attributes has invalid value
+ */
+ synchronized void mergeAndAnnounceVersion(Collection<Mutation> mutations)
+ {
+ merge(mutations);
+ updateVersionAndAnnounce();
+ }
+
+ public synchronized TransformationResult transform(SchemaTransformation transformation, boolean locally, long now)
+ {
+ KeyspacesDiff diff;
+ try
+ {
+ Keyspaces before = keyspaces;
+ Keyspaces after = transformation.apply(before);
+ diff = Keyspaces.diff(before, after);
+ }
+ catch (RuntimeException e)
+ {
+ return new TransformationResult(e);
+ }
+
+ if (diff.isEmpty())
+ return new TransformationResult(diff, Collections.emptyList());
+
+ Collection<Mutation> mutations = SchemaKeyspace.convertSchemaDiffToMutations(diff, now);
+ SchemaKeyspace.applyChanges(mutations);
+
+ merge(diff);
+ updateVersion();
+ if (!locally)
+ passiveAnnounceVersion();
+
+ return new TransformationResult(diff, mutations);
+ }
+
+ public static final class TransformationResult
+ {
+ public final boolean success;
+ public final RuntimeException exception;
+ public final KeyspacesDiff diff;
+ public final Collection<Mutation> mutations;
+
+ private TransformationResult(boolean success, RuntimeException exception, KeyspacesDiff diff, Collection<Mutation> mutations)
+ {
+ this.success = success;
+ this.exception = exception;
+ this.diff = diff;
+ this.mutations = mutations;
+ }
+
+ TransformationResult(RuntimeException exception)
+ {
+ this(false, exception, null, null);
+ }
+
+ TransformationResult(KeyspacesDiff diff, Collection<Mutation> mutations)
+ {
+ this(true, null, diff, mutations);
+ }
+ }
+
+ synchronized void merge(Collection<Mutation> mutations)
+ {
+ // only compare the keyspaces affected by this set of schema mutations
+ Set<String> affectedKeyspaces = SchemaKeyspace.affectedKeyspaces(mutations);
+
+ // fetch the current state of schema for the affected keyspaces only
+ Keyspaces before = keyspaces.filter(k -> affectedKeyspaces.contains(k.name));
+
+ // apply the schema mutations
+ SchemaKeyspace.applyChanges(mutations);
+
+ // apply the schema mutations and fetch the new versions of the altered keyspaces
+ Keyspaces after = SchemaKeyspace.fetchKeyspaces(affectedKeyspaces);
+
+ merge(Keyspaces.diff(before, after));
+ }
+
+ private void merge(KeyspacesDiff diff)
+ {
+ diff.dropped.forEach(this::dropKeyspace);
+ diff.created.forEach(this::createKeyspace);
+ diff.altered.forEach(this::alterKeyspace);
+ }
+
+ private void alterKeyspace(KeyspaceDiff delta)
+ {
+ SchemaDiagnostics.keyspaceAltering(this, delta);
+
+ // drop tables and views
+ delta.views.dropped.forEach(this::dropView);
+ delta.tables.dropped.forEach(this::dropTable);
+
+ load(delta.after);
+
+ // add tables and views
+ delta.tables.created.forEach(this::createTable);
+ delta.views.created.forEach(this::createView);
+
+ // update tables and views
+ delta.tables.altered.forEach(diff -> alterTable(diff.after));
+ delta.views.altered.forEach(diff -> alterView(diff.after));
+
+ // deal with all added, and altered views
+ Keyspace.open(delta.after.name).viewManager.reload(true);
+
+ // notify on everything dropped
+ delta.udas.dropped.forEach(uda -> notifyDropAggregate((UDAggregate) uda));
+ delta.udfs.dropped.forEach(udf -> notifyDropFunction((UDFunction) udf));
+ delta.views.dropped.forEach(this::notifyDropView);
+ delta.tables.dropped.forEach(this::notifyDropTable);
+ delta.types.dropped.forEach(this::notifyDropType);
+
+ // notify on everything created
+ delta.types.created.forEach(this::notifyCreateType);
+ delta.tables.created.forEach(this::notifyCreateTable);
+ delta.views.created.forEach(this::notifyCreateView);
+ delta.udfs.created.forEach(udf -> notifyCreateFunction((UDFunction) udf));
+ delta.udas.created.forEach(uda -> notifyCreateAggregate((UDAggregate) uda));
+
+ // notify on everything altered
+ if (!delta.before.params.equals(delta.after.params))
+ notifyAlterKeyspace(delta.before, delta.after);
+ delta.types.altered.forEach(diff -> notifyAlterType(diff.before, diff.after));
+ delta.tables.altered.forEach(diff -> notifyAlterTable(diff.before, diff.after));
+ delta.views.altered.forEach(diff -> notifyAlterView(diff.before, diff.after));
+ delta.udfs.altered.forEach(diff -> notifyAlterFunction(diff.before, diff.after));
+ delta.udas.altered.forEach(diff -> notifyAlterAggregate(diff.before, diff.after));
+ SchemaDiagnostics.keyspaceAltered(this, delta);
+ }
+
+ private void createKeyspace(KeyspaceMetadata keyspace)
+ {
+ SchemaDiagnostics.keyspaceCreating(this, keyspace);
+ load(keyspace);
+ Keyspace.open(keyspace.name);
+
+ notifyCreateKeyspace(keyspace);
+ keyspace.types.forEach(this::notifyCreateType);
+ keyspace.tables.forEach(this::notifyCreateTable);
+ keyspace.views.forEach(this::notifyCreateView);
+ keyspace.functions.udfs().forEach(this::notifyCreateFunction);
+ keyspace.functions.udas().forEach(this::notifyCreateAggregate);
+ SchemaDiagnostics.keyspaceCreated(this, keyspace);
+ }
+
+ private void dropKeyspace(KeyspaceMetadata keyspace)
+ {
+ SchemaDiagnostics.keyspaceDroping(this, keyspace);
+ keyspace.views.forEach(this::dropView);
+ keyspace.tables.forEach(this::dropTable);
+
+ // remove the keyspace from the static instances.
+ Keyspace.clear(keyspace.name);
+ unload(keyspace);
+ Keyspace.writeOrder.awaitNewBarrier();
+
+ keyspace.functions.udas().forEach(this::notifyDropAggregate);
+ keyspace.functions.udfs().forEach(this::notifyDropFunction);
+ keyspace.views.forEach(this::notifyDropView);
+ keyspace.tables.forEach(this::notifyDropTable);
+ keyspace.types.forEach(this::notifyDropType);
+ notifyDropKeyspace(keyspace);
+ SchemaDiagnostics.keyspaceDroped(this, keyspace);
+ }
+
+ private void dropView(ViewMetadata metadata)
+ {
+ Keyspace.open(metadata.keyspace()).viewManager.dropView(metadata.name());
+ dropTable(metadata.metadata);
+ }
+
+ private void dropTable(TableMetadata metadata)
+ {
+ SchemaDiagnostics.tableDropping(this, metadata);
+ ColumnFamilyStore cfs = Keyspace.open(metadata.keyspace).getColumnFamilyStore(metadata.name);
+ assert cfs != null;
+ // make sure all the indexes are dropped, or else.
+ cfs.indexManager.markAllIndexesRemoved();
+ CompactionManager.instance.interruptCompactionFor(Collections.singleton(metadata), (sstable) -> true, true);
+ if (DatabaseDescriptor.isAutoSnapshot())
+ cfs.snapshot(Keyspace.getTimestampedSnapshotNameWithPrefix(cfs.name, ColumnFamilyStore.SNAPSHOT_DROP_PREFIX));
+ CommitLog.instance.forceRecycleAllSegments(Collections.singleton(metadata.id));
+ Keyspace.open(metadata.keyspace).dropCf(metadata.id);
+ SchemaDiagnostics.tableDropped(this, metadata);
+ }
+
+ private void createTable(TableMetadata table)
+ {
+ SchemaDiagnostics.tableCreating(this, table);
+ Keyspace.open(table.keyspace).initCf(metadataRefs.get(table.id), true);
+ SchemaDiagnostics.tableCreated(this, table);
+ }
+
+ private void createView(ViewMetadata view)
+ {
+ Keyspace.open(view.keyspace()).initCf(metadataRefs.get(view.metadata.id), true);
+ }
+
+ private void alterTable(TableMetadata updated)
+ {
+ SchemaDiagnostics.tableAltering(this, updated);
+ Keyspace.open(updated.keyspace).getColumnFamilyStore(updated.name).reload();
+ SchemaDiagnostics.tableAltered(this, updated);
+ }
+
+ private void alterView(ViewMetadata updated)
+ {
+ Keyspace.open(updated.keyspace()).getColumnFamilyStore(updated.name()).reload();
+ }
+
+ private void notifyCreateKeyspace(KeyspaceMetadata ksm)
+ {
+ changeListeners.forEach(l -> l.onCreateKeyspace(ksm.name));
+ }
+
+ private void notifyCreateTable(TableMetadata metadata)
+ {
+ changeListeners.forEach(l -> l.onCreateTable(metadata.keyspace, metadata.name));
+ }
+
+ private void notifyCreateView(ViewMetadata view)
+ {
+ changeListeners.forEach(l -> l.onCreateView(view.keyspace(), view.name()));
+ }
+
+ private void notifyCreateType(UserType ut)
+ {
+ changeListeners.forEach(l -> l.onCreateType(ut.keyspace, ut.getNameAsString()));
+ }
+
+ private void notifyCreateFunction(UDFunction udf)
+ {
+ changeListeners.forEach(l -> l.onCreateFunction(udf.name().keyspace, udf.name().name, udf.argTypes()));
+ }
+
+ private void notifyCreateAggregate(UDAggregate udf)
+ {
+ changeListeners.forEach(l -> l.onCreateAggregate(udf.name().keyspace, udf.name().name, udf.argTypes()));
+ }
+
+ private void notifyAlterKeyspace(KeyspaceMetadata before, KeyspaceMetadata after)
+ {
+ changeListeners.forEach(l -> l.onAlterKeyspace(after.name));
+ }
+
+ private void notifyAlterTable(TableMetadata before, TableMetadata after)
+ {
+ boolean changeAffectedPreparedStatements = before.changeAffectsPreparedStatements(after);
+ changeListeners.forEach(l -> l.onAlterTable(after.keyspace, after.name, changeAffectedPreparedStatements));
+ }
+
+ private void notifyAlterView(ViewMetadata before, ViewMetadata after)
+ {
+ boolean changeAffectedPreparedStatements = before.metadata.changeAffectsPreparedStatements(after.metadata);
+ changeListeners.forEach(l ->l.onAlterView(after.keyspace(), after.name(), changeAffectedPreparedStatements));
+ }
+
+ private void notifyAlterType(UserType before, UserType after)
+ {
+ changeListeners.forEach(l -> l.onAlterType(after.keyspace, after.getNameAsString()));
+ }
+
+ private void notifyAlterFunction(UDFunction before, UDFunction after)
+ {
+ changeListeners.forEach(l -> l.onAlterFunction(after.name().keyspace, after.name().name, after.argTypes()));
+ }
+
+ private void notifyAlterAggregate(UDAggregate before, UDAggregate after)
+ {
+ changeListeners.forEach(l -> l.onAlterAggregate(after.name().keyspace, after.name().name, after.argTypes()));
+ }
+
+ private void notifyDropKeyspace(KeyspaceMetadata ksm)
+ {
+ changeListeners.forEach(l -> l.onDropKeyspace(ksm.name));
+ }
+
+ private void notifyDropTable(TableMetadata metadata)
+ {
+ changeListeners.forEach(l -> l.onDropTable(metadata.keyspace, metadata.name));
+ }
+
+ private void notifyDropView(ViewMetadata view)
+ {
+ changeListeners.forEach(l -> l.onDropView(view.keyspace(), view.name()));
+ }
+
+ private void notifyDropType(UserType ut)
+ {
+ changeListeners.forEach(l -> l.onDropType(ut.keyspace, ut.getNameAsString()));
+ }
+
+ private void notifyDropFunction(UDFunction udf)
+ {
+ changeListeners.forEach(l -> l.onDropFunction(udf.name().keyspace, udf.name().name, udf.argTypes()));
+ }
+
+ private void notifyDropAggregate(UDAggregate udf)
+ {
+ changeListeners.forEach(l -> l.onDropAggregate(udf.name().keyspace, udf.name().name, udf.argTypes()));
+ }
+
+
+ /**
+ * Converts the given schema version to a string. Returns {@code unknown}, if {@code version} is {@code null}
+ * or {@code "(empty)"}, if {@code version} refers to an {@link SchemaConstants#emptyVersion empty) schema.
+ */
+ public static String schemaVersionToString(UUID version)
+ {
+ return version == null
+ ? "unknown"
+ : SchemaConstants.emptyVersion.equals(version)
+ ? "(empty)"
+ : version.toString();
+ }
+}
diff --git a/src/java/org/apache/cassandra/schema/SchemaAnnouncementDiagnostics.java b/src/java/org/apache/cassandra/schema/SchemaAnnouncementDiagnostics.java
new file mode 100644
index 0000000..be60b1b
--- /dev/null
+++ b/src/java/org/apache/cassandra/schema/SchemaAnnouncementDiagnostics.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.schema;
+
+import java.util.Set;
+
+import org.apache.cassandra.diag.DiagnosticEventService;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.schema.SchemaAnnouncementEvent.SchemaAnnouncementEventType;
+
+final class SchemaAnnouncementDiagnostics
+{
+ private static final DiagnosticEventService service = DiagnosticEventService.instance();
+
+ private SchemaAnnouncementDiagnostics()
+ {
+ }
+
+ static void schemaMutationsAnnounced(Set<InetAddressAndPort> schemaDestinationEndpoints, Set<InetAddressAndPort> schemaEndpointsIgnored)
+ {
+ if (isEnabled(SchemaAnnouncementEventType.SCHEMA_MUTATIONS_ANNOUNCED))
+ service.publish(new SchemaAnnouncementEvent(SchemaAnnouncementEventType.SCHEMA_MUTATIONS_ANNOUNCED,
+ schemaDestinationEndpoints, schemaEndpointsIgnored, null, null));
+ }
+
+ public static void schemataMutationsReceived(InetAddressAndPort from)
+ {
+ if (isEnabled(SchemaAnnouncementEventType.SCHEMA_MUTATIONS_RECEIVED))
+ service.publish(new SchemaAnnouncementEvent(SchemaAnnouncementEventType.SCHEMA_MUTATIONS_RECEIVED,
+ null, null, null, from));
+ }
+
+ static void schemaTransformationAnnounced(Set<InetAddressAndPort> schemaDestinationEndpoints, Set<InetAddressAndPort> schemaEndpointsIgnored, SchemaTransformation transformation)
+ {
+ if (isEnabled(SchemaAnnouncementEventType.SCHEMA_TRANSFORMATION_ANNOUNCED))
+ service.publish(new SchemaAnnouncementEvent(SchemaAnnouncementEventType.SCHEMA_TRANSFORMATION_ANNOUNCED,
+ schemaDestinationEndpoints, schemaEndpointsIgnored, transformation, null));
+ }
+
+ private static boolean isEnabled(SchemaAnnouncementEventType type)
+ {
+ return service.isEnabled(SchemaAnnouncementEvent.class, type);
+ }
+}
diff --git a/src/java/org/apache/cassandra/schema/SchemaAnnouncementEvent.java b/src/java/org/apache/cassandra/schema/SchemaAnnouncementEvent.java
new file mode 100644
index 0000000..4e0bd68
--- /dev/null
+++ b/src/java/org/apache/cassandra/schema/SchemaAnnouncementEvent.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.schema;
+
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import javax.annotation.Nullable;
+
+import org.apache.cassandra.audit.AuditLogContext;
+import org.apache.cassandra.cql3.CQLStatement;
+import org.apache.cassandra.diag.DiagnosticEvent;
+import org.apache.cassandra.locator.InetAddressAndPort;
+
+/**
+ * Events emitted by {@link MigrationManager} around propagating schema changes to remote nodes.
+ */
+final class SchemaAnnouncementEvent extends DiagnosticEvent
+{
+ private final SchemaAnnouncementEventType type;
+ @Nullable
+ private final Set<InetAddressAndPort> schemaDestinationEndpoints;
+ @Nullable
+ private final Set<InetAddressAndPort> schemaEndpointsIgnored;
+ @Nullable
+ private final CQLStatement statement;
+ @Nullable
+ private final InetAddressAndPort sender;
+
+ enum SchemaAnnouncementEventType
+ {
+ SCHEMA_MUTATIONS_ANNOUNCED,
+ SCHEMA_TRANSFORMATION_ANNOUNCED,
+ SCHEMA_MUTATIONS_RECEIVED
+ }
+
+ SchemaAnnouncementEvent(SchemaAnnouncementEventType type,
+ @Nullable Set<InetAddressAndPort> schemaDestinationEndpoints,
+ @Nullable Set<InetAddressAndPort> schemaEndpointsIgnored,
+ @Nullable SchemaTransformation transformation,
+ @Nullable InetAddressAndPort sender)
+ {
+ this.type = type;
+ this.schemaDestinationEndpoints = schemaDestinationEndpoints;
+ this.schemaEndpointsIgnored = schemaEndpointsIgnored;
+ if (transformation instanceof CQLStatement) this.statement = (CQLStatement) transformation;
+ else this.statement = null;
+ this.sender = sender;
+ }
+
+ public Enum<?> getType()
+ {
+ return type;
+ }
+
+ public Map<String, Serializable> toMap()
+ {
+ HashMap<String, Serializable> ret = new HashMap<>();
+ if (schemaDestinationEndpoints != null)
+ {
+ Set<String> eps = schemaDestinationEndpoints.stream().map(InetAddressAndPort::toString).collect(Collectors.toSet());
+ ret.put("endpointDestinations", new HashSet<>(eps));
+ }
+ if (schemaEndpointsIgnored != null)
+ {
+ Set<String> eps = schemaEndpointsIgnored.stream().map(InetAddressAndPort::toString).collect(Collectors.toSet());
+ ret.put("endpointIgnored", new HashSet<>(eps));
+ }
+ if (statement != null)
+ {
+ AuditLogContext logContext = statement.getAuditLogContext();
+ if (logContext != null)
+ {
+ HashMap<String, String> log = new HashMap<>();
+ if (logContext.auditLogEntryType != null) log.put("type", logContext.auditLogEntryType.name());
+ if (logContext.keyspace != null) log.put("keyspace", logContext.keyspace);
+ if (logContext.scope != null) log.put("table", logContext.scope);
+ ret.put("statement", log);
+ }
+ }
+ if (sender != null) ret.put("sender", sender.toString());
+ return ret;
+ }
+}
diff --git a/src/java/org/apache/cassandra/schema/SchemaChangeListener.java b/src/java/org/apache/cassandra/schema/SchemaChangeListener.java
new file mode 100644
index 0000000..4390309
--- /dev/null
+++ b/src/java/org/apache/cassandra/schema/SchemaChangeListener.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.schema;
+
+import java.util.List;
+
+import org.apache.cassandra.db.marshal.AbstractType;
+
+public abstract class SchemaChangeListener
+{
+ public void onCreateKeyspace(String keyspace)
+ {
+ }
+
+ public void onCreateTable(String keyspace, String table)
+ {
+ }
+
+ public void onCreateView(String keyspace, String view)
+ {
+ onCreateTable(keyspace, view);
+ }
+
+ public void onCreateType(String keyspace, String type)
+ {
+ }
+
+ public void onCreateFunction(String keyspace, String function, List<AbstractType<?>> argumentTypes)
+ {
+ }
+
+ public void onCreateAggregate(String keyspace, String aggregate, List<AbstractType<?>> argumentTypes)
+ {
+ }
+
+ public void onAlterKeyspace(String keyspace)
+ {
+ }
+
+ // the boolean flag indicates whether the change that triggered this event may have a substantive
+ // impact on statements using the column family.
+ public void onAlterTable(String keyspace, String table, boolean affectsStatements)
+ {
+ }
+
+ public void onAlterView(String keyspace, String view, boolean affectsStataments)
+ {
+ onAlterTable(keyspace, view, affectsStataments);
+ }
+
+ public void onAlterType(String keyspace, String type)
+ {
+ }
+
+ public void onAlterFunction(String keyspace, String function, List<AbstractType<?>> argumentTypes)
+ {
+ }
+
+ public void onAlterAggregate(String keyspace, String aggregate, List<AbstractType<?>> argumentTypes)
+ {
+ }
+
+ public void onDropKeyspace(String keyspace)
+ {
+ }
+
+ public void onDropTable(String keyspace, String table)
+ {
+ }
+
+ public void onDropView(String keyspace, String view)
+ {
+ onDropTable(keyspace, view);
+ }
+
+ public void onDropType(String keyspace, String type)
+ {
+ }
+
+ public void onDropFunction(String keyspace, String function, List<AbstractType<?>> argumentTypes)
+ {
+ }
+
+ public void onDropAggregate(String keyspace, String aggregate, List<AbstractType<?>> argumentTypes)
+ {
+ }
+}
diff --git a/src/java/org/apache/cassandra/schema/SchemaConstants.java b/src/java/org/apache/cassandra/schema/SchemaConstants.java
new file mode 100644
index 0000000..7b6b7de
--- /dev/null
+++ b/src/java/org/apache/cassandra/schema/SchemaConstants.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.schema;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Set;
+import java.util.UUID;
+import java.util.regex.Pattern;
+
+import com.google.common.collect.ImmutableSet;
+
+import org.apache.cassandra.db.Digest;
+
+public final class SchemaConstants
+{
+ public static final Pattern PATTERN_WORD_CHARS = Pattern.compile("\\w+");
+
+ public static final String SYSTEM_KEYSPACE_NAME = "system";
+ public static final String SCHEMA_KEYSPACE_NAME = "system_schema";
+
+ public static final String TRACE_KEYSPACE_NAME = "system_traces";
+ public static final String AUTH_KEYSPACE_NAME = "system_auth";
+ public static final String DISTRIBUTED_KEYSPACE_NAME = "system_distributed";
+
+ public static final String VIRTUAL_SCHEMA = "system_virtual_schema";
+
+ public static final String VIRTUAL_VIEWS = "system_views";
+
+ /* system keyspace names (the ones with LocalStrategy replication strategy) */
+ public static final Set<String> LOCAL_SYSTEM_KEYSPACE_NAMES =
+ ImmutableSet.of(SYSTEM_KEYSPACE_NAME, SCHEMA_KEYSPACE_NAME);
+
+ /* replicate system keyspace names (the ones with a "true" replication strategy) */
+ public static final Set<String> REPLICATED_SYSTEM_KEYSPACE_NAMES =
+ ImmutableSet.of(TRACE_KEYSPACE_NAME, AUTH_KEYSPACE_NAME, DISTRIBUTED_KEYSPACE_NAME);
+ /**
+ * longest permissible KS or CF name. Our main concern is that filename not be more than 255 characters;
+ * the filename will contain both the KS and CF names. Since non-schema-name components only take up
+ * ~64 characters, we could allow longer names than this, but on Windows, the entire path should be not greater than
+ * 255 characters, so a lower limit here helps avoid problems. See CASSANDRA-4110.
+ */
+ public static final int NAME_LENGTH = 48;
+
+ // 59adb24e-f3cd-3e02-97f0-5b395827453f
+ public static final UUID emptyVersion;
+
+ public static final List<String> LEGACY_AUTH_TABLES = Arrays.asList("credentials", "users", "permissions");
+
+ public static boolean isValidName(String name)
+ {
+ return name != null && !name.isEmpty() && name.length() <= NAME_LENGTH && PATTERN_WORD_CHARS.matcher(name).matches();
+ }
+
+ static
+ {
+ emptyVersion = UUID.nameUUIDFromBytes(Digest.forSchema().digest());
+ }
+
+ /**
+ * @return whether or not the keyspace is a really system one (w/ LocalStrategy, unmodifiable, hardcoded)
+ */
+ public static boolean isLocalSystemKeyspace(String keyspaceName)
+ {
+ return LOCAL_SYSTEM_KEYSPACE_NAMES.contains(keyspaceName.toLowerCase());
+ }
+
+ /**
+ * @return whether or not the keyspace is a replicated system ks (system_auth, system_traces, system_distributed)
+ */
+ public static boolean isReplicatedSystemKeyspace(String keyspaceName)
+ {
+ return REPLICATED_SYSTEM_KEYSPACE_NAMES.contains(keyspaceName.toLowerCase());
+ }
+
+ /**
+ * Checks if the keyspace is a virtual system keyspace.
+ * @return {@code true} if the keyspace is a virtual system keyspace, {@code false} otherwise.
+ */
+ public static boolean isVirtualSystemKeyspace(String keyspaceName)
+ {
+ return VIRTUAL_SCHEMA.equals(keyspaceName.toLowerCase()) || VIRTUAL_VIEWS.equals(keyspaceName.toLowerCase());
+ }
+
+ /**
+ * Checks if the keyspace is a system keyspace (local replicated or virtual).
+ * @return {@code true} if the keyspace is a system keyspace, {@code false} otherwise.
+ */
+ public static boolean isSystemKeyspace(String keyspaceName)
+ {
+ return isLocalSystemKeyspace(keyspaceName)
+ || isReplicatedSystemKeyspace(keyspaceName)
+ || isVirtualSystemKeyspace(keyspaceName);
+ }
+}
diff --git a/src/java/org/apache/cassandra/schema/SchemaDiagnostics.java b/src/java/org/apache/cassandra/schema/SchemaDiagnostics.java
new file mode 100644
index 0000000..12b8409
--- /dev/null
+++ b/src/java/org/apache/cassandra/schema/SchemaDiagnostics.java
@@ -0,0 +1,178 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.schema;
+
+import com.google.common.collect.MapDifference;
+
+import org.apache.cassandra.diag.DiagnosticEventService;
+import org.apache.cassandra.schema.SchemaEvent.SchemaEventType;
+
+final class SchemaDiagnostics
+{
+ private static final DiagnosticEventService service = DiagnosticEventService.instance();
+
+ private SchemaDiagnostics()
+ {
+ }
+
+ static void metadataInitialized(Schema schema, KeyspaceMetadata ksmUpdate)
+ {
+ if (isEnabled(SchemaEventType.KS_METADATA_LOADED))
+ service.publish(new SchemaEvent(SchemaEventType.KS_METADATA_LOADED, schema, ksmUpdate, null, null, null, null, null, null));
+ }
+
+ static void metadataReloaded(Schema schema, KeyspaceMetadata previous, KeyspaceMetadata ksmUpdate, Tables.TablesDiff tablesDiff, Views.ViewsDiff viewsDiff, MapDifference<String,TableMetadata> indexesDiff)
+ {
+ if (isEnabled(SchemaEventType.KS_METADATA_RELOADED))
+ service.publish(new SchemaEvent(SchemaEventType.KS_METADATA_RELOADED, schema, ksmUpdate, previous,
+ null, null, tablesDiff, viewsDiff, indexesDiff));
+ }
+
+ static void metadataRemoved(Schema schema, KeyspaceMetadata ksmUpdate)
+ {
+ if (isEnabled(SchemaEventType.KS_METADATA_REMOVED))
+ service.publish(new SchemaEvent(SchemaEventType.KS_METADATA_REMOVED, schema, ksmUpdate,
+ null, null, null, null, null, null));
+ }
+
+ static void versionUpdated(Schema schema)
+ {
+ if (isEnabled(SchemaEventType.VERSION_UPDATED))
+ service.publish(new SchemaEvent(SchemaEventType.VERSION_UPDATED, schema,
+ null, null, null, null, null, null, null));
+ }
+
+ static void keyspaceCreating(Schema schema, KeyspaceMetadata keyspace)
+ {
+ if (isEnabled(SchemaEventType.KS_CREATING))
+ service.publish(new SchemaEvent(SchemaEventType.KS_CREATING, schema, keyspace,
+ null, null, null, null, null, null));
+ }
+
+ static void keyspaceCreated(Schema schema, KeyspaceMetadata keyspace)
+ {
+ if (isEnabled(SchemaEventType.KS_CREATED))
+ service.publish(new SchemaEvent(SchemaEventType.KS_CREATED, schema, keyspace,
+ null, null, null, null, null, null));
+ }
+
+ static void keyspaceAltering(Schema schema, KeyspaceMetadata.KeyspaceDiff delta)
+ {
+ if (isEnabled(SchemaEventType.KS_ALTERING))
+ service.publish(new SchemaEvent(SchemaEventType.KS_ALTERING, schema, delta.after,
+ delta.before, delta, null, null, null, null));
+ }
+
+ static void keyspaceAltered(Schema schema, KeyspaceMetadata.KeyspaceDiff delta)
+ {
+ if (isEnabled(SchemaEventType.KS_ALTERED))
+ service.publish(new SchemaEvent(SchemaEventType.KS_ALTERED, schema, delta.after,
+ delta.before, delta, null, null, null, null));
+ }
+
+ static void keyspaceDroping(Schema schema, KeyspaceMetadata keyspace)
+ {
+ if (isEnabled(SchemaEventType.KS_DROPPING))
+ service.publish(new SchemaEvent(SchemaEventType.KS_DROPPING, schema, keyspace,
+ null, null, null, null, null, null));
+ }
+
+ static void keyspaceDroped(Schema schema, KeyspaceMetadata keyspace)
+ {
+ if (isEnabled(SchemaEventType.KS_DROPPED))
+ service.publish(new SchemaEvent(SchemaEventType.KS_DROPPED, schema, keyspace,
+ null, null, null, null, null, null));
+ }
+
+ static void schemataLoading(Schema schema)
+ {
+ if (isEnabled(SchemaEventType.SCHEMATA_LOADING))
+ service.publish(new SchemaEvent(SchemaEventType.SCHEMATA_LOADING, schema, null,
+ null, null, null, null, null, null));
+ }
+
+ static void schemataLoaded(Schema schema)
+ {
+ if (isEnabled(SchemaEventType.SCHEMATA_LOADED))
+ service.publish(new SchemaEvent(SchemaEventType.SCHEMATA_LOADED, schema, null,
+ null, null, null, null, null, null));
+ }
+
+ static void versionAnnounced(Schema schema)
+ {
+ if (isEnabled(SchemaEventType.VERSION_ANOUNCED))
+ service.publish(new SchemaEvent(SchemaEventType.VERSION_ANOUNCED, schema, null,
+ null, null, null, null, null, null));
+ }
+
+ static void schemataCleared(Schema schema)
+ {
+ if (isEnabled(SchemaEventType.SCHEMATA_CLEARED))
+ service.publish(new SchemaEvent(SchemaEventType.SCHEMATA_CLEARED, schema, null,
+ null, null, null, null, null, null));
+ }
+
+ static void tableCreating(Schema schema, TableMetadata table)
+ {
+ if (isEnabled(SchemaEventType.TABLE_CREATING))
+ service.publish(new SchemaEvent(SchemaEventType.TABLE_CREATING, schema, null,
+ null, null, table, null, null, null));
+ }
+
+ static void tableCreated(Schema schema, TableMetadata table)
+ {
+ if (isEnabled(SchemaEventType.TABLE_CREATED))
+ service.publish(new SchemaEvent(SchemaEventType.TABLE_CREATED, schema, null,
+ null, null, table, null, null, null));
+ }
+
+ static void tableAltering(Schema schema, TableMetadata table)
+ {
+ if (isEnabled(SchemaEventType.TABLE_ALTERING))
+ service.publish(new SchemaEvent(SchemaEventType.TABLE_ALTERING, schema, null,
+ null, null, table, null, null, null));
+ }
+
+ static void tableAltered(Schema schema, TableMetadata table)
+ {
+ if (isEnabled(SchemaEventType.TABLE_ALTERED))
+ service.publish(new SchemaEvent(SchemaEventType.TABLE_ALTERED, schema, null,
+ null, null, table, null, null, null));
+ }
+
+ static void tableDropping(Schema schema, TableMetadata table)
+ {
+ if (isEnabled(SchemaEventType.TABLE_DROPPING))
+ service.publish(new SchemaEvent(SchemaEventType.TABLE_DROPPING, schema, null,
+ null, null, table, null, null, null));
+ }
+
+ static void tableDropped(Schema schema, TableMetadata table)
+ {
+ if (isEnabled(SchemaEventType.TABLE_DROPPED))
+ service.publish(new SchemaEvent(SchemaEventType.TABLE_DROPPED, schema, null,
+ null, null, table, null, null, null));
+ }
+
+ private static boolean isEnabled(SchemaEventType type)
+ {
+ return service.isEnabled(SchemaEvent.class, type);
+ }
+
+}
diff --git a/src/java/org/apache/cassandra/schema/SchemaEvent.java b/src/java/org/apache/cassandra/schema/SchemaEvent.java
new file mode 100644
index 0000000..00c8136
--- /dev/null
+++ b/src/java/org/apache/cassandra/schema/SchemaEvent.java
@@ -0,0 +1,319 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.schema;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.stream.Collectors;
+import javax.annotation.Nullable;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Lists;
+import com.google.common.collect.MapDifference;
+
+import org.apache.cassandra.diag.DiagnosticEvent;
+import org.apache.cassandra.utils.Pair;
+
+public final class SchemaEvent extends DiagnosticEvent
+{
+ private final SchemaEventType type;
+
+ private final HashSet<String> keyspaces;
+ private final HashMap<String, String> indexTables;
+ private final HashMap<String, String> tables;
+ private final ArrayList<String> nonSystemKeyspaces;
+ private final ArrayList<String> userKeyspaces;
+ private final int numberOfTables;
+ private final UUID version;
+
+ @Nullable
+ private final KeyspaceMetadata ksUpdate;
+ @Nullable
+ private final KeyspaceMetadata previous;
+ @Nullable
+ private final KeyspaceMetadata.KeyspaceDiff ksDiff;
+ @Nullable
+ private final TableMetadata tableUpdate;
+ @Nullable
+ private final Tables.TablesDiff tablesDiff;
+ @Nullable
+ private final Views.ViewsDiff viewsDiff;
+ @Nullable
+ private final MapDifference<String,TableMetadata> indexesDiff;
+
+ public enum SchemaEventType
+ {
+ KS_METADATA_LOADED,
+ KS_METADATA_RELOADED,
+ KS_METADATA_REMOVED,
+ VERSION_UPDATED,
+ VERSION_ANOUNCED,
+ KS_CREATING,
+ KS_CREATED,
+ KS_ALTERING,
+ KS_ALTERED,
+ KS_DROPPING,
+ KS_DROPPED,
+ TABLE_CREATING,
+ TABLE_CREATED,
+ TABLE_ALTERING,
+ TABLE_ALTERED,
+ TABLE_DROPPING,
+ TABLE_DROPPED,
+ SCHEMATA_LOADING,
+ SCHEMATA_LOADED,
+ SCHEMATA_CLEARED
+ }
+
+ SchemaEvent(SchemaEventType type, Schema schema, @Nullable KeyspaceMetadata ksUpdate,
+ @Nullable KeyspaceMetadata previous, @Nullable KeyspaceMetadata.KeyspaceDiff ksDiff,
+ @Nullable TableMetadata tableUpdate, @Nullable Tables.TablesDiff tablesDiff,
+ @Nullable Views.ViewsDiff viewsDiff, @Nullable MapDifference<String,TableMetadata> indexesDiff)
+ {
+ this.type = type;
+ this.ksUpdate = ksUpdate;
+ this.previous = previous;
+ this.ksDiff = ksDiff;
+ this.tableUpdate = tableUpdate;
+ this.tablesDiff = tablesDiff;
+ this.viewsDiff = viewsDiff;
+ this.indexesDiff = indexesDiff;
+
+ this.keyspaces = new HashSet<>(schema.getKeyspaces());
+ this.nonSystemKeyspaces = new ArrayList<>(schema.getNonSystemKeyspaces());
+ this.userKeyspaces = new ArrayList<>(schema.getUserKeyspaces());
+ this.numberOfTables = schema.getNumberOfTables();
+ this.version = schema.getVersion();
+
+ Map<Pair<String, String>, TableMetadataRef> indexTableMetadataRefs = schema.getIndexTableMetadataRefs();
+ Map<String, String> indexTables = indexTableMetadataRefs.entrySet().stream()
+ .collect(Collectors.toMap(e -> e.getKey().left + ',' +
+ e.getKey().right,
+ e -> e.getValue().id.toHexString() + ',' +
+ e.getValue().keyspace + ',' +
+ e.getValue().name));
+ this.indexTables = new HashMap<>(indexTables);
+ Map<TableId, TableMetadataRef> tableMetadataRefs = schema.getTableMetadataRefs();
+ Map<String, String> tables = tableMetadataRefs.entrySet().stream()
+ .collect(Collectors.toMap(e -> e.getKey().toHexString(),
+ e -> e.getValue().id.toHexString() + ',' +
+ e.getValue().keyspace + ',' +
+ e.getValue().name));
+ this.tables = new HashMap<>(tables);
+ }
+
+ public SchemaEventType getType()
+ {
+ return type;
+ }
+
+ public Map<String, Serializable> toMap()
+ {
+ HashMap<String, Serializable> ret = new HashMap<>();
+ ret.put("keyspaces", this.keyspaces);
+ ret.put("nonSystemKeyspaces", this.nonSystemKeyspaces);
+ ret.put("userKeyspaces", this.userKeyspaces);
+ ret.put("numberOfTables", this.numberOfTables);
+ ret.put("version", this.version);
+ ret.put("tables", this.tables);
+ ret.put("indexTables", this.indexTables);
+ if (ksUpdate != null) ret.put("ksMetadataUpdate", repr(ksUpdate));
+ if (previous != null) ret.put("ksMetadataPrevious", repr(previous));
+ if (ksDiff != null)
+ {
+ HashMap<String, Serializable> ks = new HashMap<>();
+ ks.put("before", repr(ksDiff.before));
+ ks.put("after", repr(ksDiff.after));
+ ks.put("tables", repr(ksDiff.tables));
+ ks.put("views", repr(ksDiff.views));
+ ks.put("types", repr(ksDiff.types));
+ ks.put("udas", repr(ksDiff.udas));
+ ks.put("udfs", repr(ksDiff.udfs));
+ ret.put("ksDiff", ks);
+ }
+ if (tableUpdate != null) ret.put("tableMetadataUpdate", repr(tableUpdate));
+ if (tablesDiff != null) ret.put("tablesDiff", repr(tablesDiff));
+ if (viewsDiff != null) ret.put("viewsDiff", repr(viewsDiff));
+ if (indexesDiff != null) ret.put("indexesDiff", Lists.newArrayList(indexesDiff.entriesDiffering().keySet()));
+ return ret;
+ }
+
+ private HashMap<String, Serializable> repr(Diff<?, ?> diff)
+ {
+ HashMap<String, Serializable> ret = new HashMap<>();
+ if (diff.created != null) ret.put("created", diff.created.toString());
+ if (diff.dropped != null) ret.put("dropped", diff.dropped.toString());
+ if (diff.altered != null)
+ ret.put("created", Lists.newArrayList(diff.altered.stream().map(Diff.Altered::toString).iterator()));
+ return ret;
+ }
+
+ private HashMap<String, Serializable> repr(KeyspaceMetadata ksm)
+ {
+ HashMap<String, Serializable> ret = new HashMap<>();
+ ret.put("name", ksm.name);
+ if (ksm.kind != null) ret.put("kind", ksm.kind.name());
+ if (ksm.params != null) ret.put("params", ksm.params.toString());
+ if (ksm.tables != null) ret.put("tables", ksm.tables.toString());
+ if (ksm.views != null) ret.put("views", ksm.views.toString());
+ if (ksm.functions != null) ret.put("functions", ksm.functions.toString());
+ if (ksm.types != null) ret.put("types", ksm.types.toString());
+ return ret;
+ }
+
+ private HashMap<String, Serializable> repr(TableMetadata table)
+ {
+ HashMap<String, Serializable> ret = new HashMap<>();
+ ret.put("id", table.id.toHexString());
+ ret.put("name", table.name);
+ ret.put("keyspace", table.keyspace);
+ ret.put("partitioner", table.partitioner.toString());
+ ret.put("kind", table.kind.name());
+ ret.put("flags", Lists.newArrayList(table.flags.stream().map(Enum::name).iterator()));
+ ret.put("params", repr(table.params));
+ ret.put("indexes", Lists.newArrayList(table.indexes.stream().map(this::repr).iterator()));
+ ret.put("triggers", Lists.newArrayList(repr(table.triggers)));
+ ret.put("columns", Lists.newArrayList(table.columns.values().stream().map(this::repr).iterator()));
+ ret.put("droppedColumns", Lists.newArrayList(table.droppedColumns.values().stream().map(this::repr).iterator()));
+ ret.put("isCompactTable", table.isCompactTable());
+ ret.put("isCompound", table.isCompound());
+ ret.put("isCounter", table.isCounter());
+ ret.put("isCQLTable", table.isCQLTable());
+ ret.put("isDense", table.isDense());
+ ret.put("isIndex", table.isIndex());
+ ret.put("isStaticCompactTable", table.isStaticCompactTable());
+ ret.put("isSuper", table.isSuper());
+ ret.put("isView", table.isView());
+ ret.put("isVirtual", table.isVirtual());
+ return ret;
+ }
+
+ private HashMap<String, Serializable> repr(TableParams params)
+ {
+ HashMap<String, Serializable> ret = new HashMap<>();
+ if (params == null) return ret;
+ ret.put("minIndexInterval", params.minIndexInterval);
+ ret.put("maxIndexInterval", params.maxIndexInterval);
+ ret.put("defaultTimeToLive", params.defaultTimeToLive);
+ ret.put("gcGraceSeconds", params.gcGraceSeconds);
+ ret.put("bloomFilterFpChance", params.bloomFilterFpChance);
+ ret.put("cdc", params.cdc);
+ ret.put("crcCheckChance", params.crcCheckChance);
+ ret.put("memtableFlushPeriodInMs", params.memtableFlushPeriodInMs);
+ ret.put("comment", params.comment);
+ ret.put("caching", repr(params.caching));
+ ret.put("compaction", repr(params.compaction));
+ ret.put("compression", repr(params.compression));
+ if (params.speculativeRetry != null) ret.put("speculativeRetry", params.speculativeRetry.kind().name());
+ return ret;
+ }
+
+ private HashMap<String, Serializable> repr(CachingParams caching)
+ {
+ HashMap<String, Serializable> ret = new HashMap<>();
+ if (caching == null) return ret;
+ ret.putAll(caching.asMap());
+ return ret;
+ }
+
+ private HashMap<String, Serializable> repr(CompactionParams comp)
+ {
+ HashMap<String, Serializable> ret = new HashMap<>();
+ if (comp == null) return ret;
+ ret.putAll(comp.asMap());
+ return ret;
+ }
+
+ private HashMap<String, Serializable> repr(CompressionParams compr)
+ {
+ HashMap<String, Serializable> ret = new HashMap<>();
+ if (compr == null) return ret;
+ ret.putAll(compr.asMap());
+ return ret;
+ }
+
+ private HashMap<String, Serializable> repr(IndexMetadata index)
+ {
+ HashMap<String, Serializable> ret = new HashMap<>();
+ if (index == null) return ret;
+ ret.put("name", index.name);
+ ret.put("kind", index.kind.name());
+ ret.put("id", index.id);
+ ret.put("options", new HashMap<>(index.options));
+ ret.put("isCustom", index.isCustom());
+ ret.put("isKeys", index.isKeys());
+ ret.put("isComposites", index.isComposites());
+ return ret;
+ }
+
+ private List<Map<String, Serializable>> repr(Triggers triggers)
+ {
+ List<Map<String, Serializable>> ret = new ArrayList<>();
+ if (triggers == null) return ret;
+ Iterator<TriggerMetadata> iter = triggers.iterator();
+ while (iter.hasNext()) ret.add(repr(iter.next()));
+ return ret;
+ }
+
+ private HashMap<String, Serializable> repr(TriggerMetadata trigger)
+ {
+ HashMap<String, Serializable> ret = new HashMap<>();
+ if (trigger == null) return ret;
+ ret.put("name", trigger.name);
+ ret.put("classOption", trigger.classOption);
+ return ret;
+ }
+
+ private HashMap<String, Serializable> repr(ColumnMetadata col)
+ {
+ HashMap<String, Serializable> ret = new HashMap<>();
+ if (col == null) return ret;
+ ret.put("name", col.name.toString());
+ ret.put("kind", col.kind.name());
+ ret.put("type", col.type.toString());
+ ret.put("ksName", col.ksName);
+ ret.put("cfName", col.cfName);
+ ret.put("position", col.position());
+ ret.put("clusteringOrder", col.clusteringOrder().name());
+ ret.put("isComplex", col.isComplex());
+ ret.put("isStatic", col.isStatic());
+ ret.put("isPrimaryKeyColumn", col.isPrimaryKeyColumn());
+ ret.put("isSimple", col.isSimple());
+ ret.put("isPartitionKey", col.isPartitionKey());
+ ret.put("isClusteringColumn", col.isClusteringColumn());
+ ret.put("isCounterColumn", col.isCounterColumn());
+ ret.put("isRegular", col.isRegular());
+ return ret;
+ }
+
+ private HashMap<String, Serializable> repr(DroppedColumn column)
+ {
+ HashMap<String, Serializable> ret = new HashMap<>();
+ if (column == null) return ret;
+ ret.put("droppedTime", column.droppedTime);
+ ret.put("column", repr(column.column));
+ return ret;
+ }
+}
diff --git a/src/java/org/apache/cassandra/schema/SchemaKeyspace.java b/src/java/org/apache/cassandra/schema/SchemaKeyspace.java
index 8b7ac84..a10156c 100644
--- a/src/java/org/apache/cassandra/schema/SchemaKeyspace.java
+++ b/src/java/org/apache/cassandra/schema/SchemaKeyspace.java
@@ -19,8 +19,6 @@
import java.nio.ByteBuffer;
import java.nio.charset.CharacterCodingException;
-import java.security.MessageDigest;
-import java.security.NoSuchAlgorithmException;
import java.util.*;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
@@ -31,31 +29,32 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.antlr.runtime.RecognitionException;
import org.apache.cassandra.config.*;
-import org.apache.cassandra.config.CFMetaData.DroppedColumn;
-import org.apache.cassandra.config.ColumnDefinition.ClusteringOrder;
import org.apache.cassandra.cql3.*;
import org.apache.cassandra.cql3.functions.*;
-import org.apache.cassandra.cql3.statements.SelectStatement;
+import org.apache.cassandra.cql3.statements.schema.CreateTableStatement;
import org.apache.cassandra.db.*;
+import org.apache.cassandra.db.filter.ColumnFilter;
import org.apache.cassandra.db.marshal.*;
import org.apache.cassandra.db.partitions.*;
import org.apache.cassandra.db.rows.*;
-import org.apache.cassandra.db.filter.ColumnFilter;
-import org.apache.cassandra.db.view.View;
-import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.service.reads.SpeculativeRetryPolicy;
+import org.apache.cassandra.schema.ColumnMetadata.ClusteringOrder;
+import org.apache.cassandra.schema.Keyspaces.KeyspacesDiff;
+import org.apache.cassandra.service.reads.repair.ReadRepairStrategy;
import org.apache.cassandra.transport.ProtocolVersion;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FBUtilities;
-import org.apache.cassandra.utils.Pair;
import static java.lang.String.format;
import static java.util.stream.Collectors.toList;
+import static java.util.stream.Collectors.toSet;
+
import static org.apache.cassandra.cql3.QueryProcessor.executeInternal;
import static org.apache.cassandra.cql3.QueryProcessor.executeOnceInternal;
-import static org.apache.cassandra.schema.CQLTypeParser.parse;
/**
* system_schema.* tables and methods for manipulating them.
@@ -104,171 +103,170 @@
*/
private static final Set<String> TABLES_WITH_CDC_ADDED = ImmutableSet.of(TABLES, VIEWS);
+ private static final TableMetadata Keyspaces =
+ parse(KEYSPACES,
+ "keyspace definitions",
+ "CREATE TABLE %s ("
+ + "keyspace_name text,"
+ + "durable_writes boolean,"
+ + "replication frozen<map<text, text>>,"
+ + "PRIMARY KEY ((keyspace_name)))");
- /**
- * Until we upgrade the messaging service version, that is version 4.0, we must preserve the old order (before CASSANDRA-12213)
- * for digest calculations, otherwise the nodes will never agree on the schema during a rolling upgrade, see CASSANDRA-13559.
- */
- public static final ImmutableList<String> ALL_FOR_DIGEST =
- ImmutableList.of(KEYSPACES, TABLES, COLUMNS, TRIGGERS, VIEWS, TYPES, FUNCTIONS, AGGREGATES, INDEXES);
+ private static final TableMetadata Tables =
+ parse(TABLES,
+ "table definitions",
+ "CREATE TABLE %s ("
+ + "keyspace_name text,"
+ + "table_name text,"
+ + "bloom_filter_fp_chance double,"
+ + "caching frozen<map<text, text>>,"
+ + "comment text,"
+ + "compaction frozen<map<text, text>>,"
+ + "compression frozen<map<text, text>>,"
+ + "crc_check_chance double,"
+ + "dclocal_read_repair_chance double," // no longer used, left for drivers' sake
+ + "default_time_to_live int,"
+ + "extensions frozen<map<text, blob>>,"
+ + "flags frozen<set<text>>," // SUPER, COUNTER, DENSE, COMPOUND
+ + "gc_grace_seconds int,"
+ + "id uuid,"
+ + "max_index_interval int,"
+ + "memtable_flush_period_in_ms int,"
+ + "min_index_interval int,"
+ + "read_repair_chance double," // no longer used, left for drivers' sake
+ + "speculative_retry text,"
+ + "additional_write_policy text,"
+ + "cdc boolean,"
+ + "read_repair text,"
+ + "PRIMARY KEY ((keyspace_name), table_name))");
- private static final CFMetaData Keyspaces =
- compile(KEYSPACES,
- "keyspace definitions",
- "CREATE TABLE %s ("
- + "keyspace_name text,"
- + "durable_writes boolean,"
- + "replication frozen<map<text, text>>,"
- + "PRIMARY KEY ((keyspace_name)))");
+ private static final TableMetadata Columns =
+ parse(COLUMNS,
+ "column definitions",
+ "CREATE TABLE %s ("
+ + "keyspace_name text,"
+ + "table_name text,"
+ + "column_name text,"
+ + "clustering_order text,"
+ + "column_name_bytes blob,"
+ + "kind text,"
+ + "position int,"
+ + "type text,"
+ + "PRIMARY KEY ((keyspace_name), table_name, column_name))");
- private static final CFMetaData Tables =
- compile(TABLES,
- "table definitions",
- "CREATE TABLE %s ("
- + "keyspace_name text,"
- + "table_name text,"
- + "bloom_filter_fp_chance double,"
- + "caching frozen<map<text, text>>,"
- + "comment text,"
- + "compaction frozen<map<text, text>>,"
- + "compression frozen<map<text, text>>,"
- + "crc_check_chance double,"
- + "dclocal_read_repair_chance double,"
- + "default_time_to_live int,"
- + "extensions frozen<map<text, blob>>,"
- + "flags frozen<set<text>>," // SUPER, COUNTER, DENSE, COMPOUND
- + "gc_grace_seconds int,"
- + "id uuid,"
- + "max_index_interval int,"
- + "memtable_flush_period_in_ms int,"
- + "min_index_interval int,"
- + "read_repair_chance double,"
- + "speculative_retry text,"
- + "cdc boolean,"
- + "PRIMARY KEY ((keyspace_name), table_name))");
+ private static final TableMetadata DroppedColumns =
+ parse(DROPPED_COLUMNS,
+ "dropped column registry",
+ "CREATE TABLE %s ("
+ + "keyspace_name text,"
+ + "table_name text,"
+ + "column_name text,"
+ + "dropped_time timestamp,"
+ + "kind text,"
+ + "type text,"
+ + "PRIMARY KEY ((keyspace_name), table_name, column_name))");
- private static final CFMetaData Columns =
- compile(COLUMNS,
- "column definitions",
- "CREATE TABLE %s ("
- + "keyspace_name text,"
- + "table_name text,"
- + "column_name text,"
- + "clustering_order text,"
- + "column_name_bytes blob,"
- + "kind text,"
- + "position int,"
- + "type text,"
- + "PRIMARY KEY ((keyspace_name), table_name, column_name))");
+ private static final TableMetadata Triggers =
+ parse(TRIGGERS,
+ "trigger definitions",
+ "CREATE TABLE %s ("
+ + "keyspace_name text,"
+ + "table_name text,"
+ + "trigger_name text,"
+ + "options frozen<map<text, text>>,"
+ + "PRIMARY KEY ((keyspace_name), table_name, trigger_name))");
- private static final CFMetaData DroppedColumns =
- compile(DROPPED_COLUMNS,
- "dropped column registry",
- "CREATE TABLE %s ("
- + "keyspace_name text,"
- + "table_name text,"
- + "column_name text,"
- + "dropped_time timestamp,"
- + "kind text,"
- + "type text,"
- + "PRIMARY KEY ((keyspace_name), table_name, column_name))");
+ private static final TableMetadata Views =
+ parse(VIEWS,
+ "view definitions",
+ "CREATE TABLE %s ("
+ + "keyspace_name text,"
+ + "view_name text,"
+ + "base_table_id uuid,"
+ + "base_table_name text,"
+ + "where_clause text,"
+ + "bloom_filter_fp_chance double,"
+ + "caching frozen<map<text, text>>,"
+ + "comment text,"
+ + "compaction frozen<map<text, text>>,"
+ + "compression frozen<map<text, text>>,"
+ + "crc_check_chance double,"
+ + "dclocal_read_repair_chance double," // no longer used, left for drivers' sake
+ + "default_time_to_live int,"
+ + "extensions frozen<map<text, blob>>,"
+ + "gc_grace_seconds int,"
+ + "id uuid,"
+ + "include_all_columns boolean,"
+ + "max_index_interval int,"
+ + "memtable_flush_period_in_ms int,"
+ + "min_index_interval int,"
+ + "read_repair_chance double," // no longer used, left for drivers' sake
+ + "speculative_retry text,"
+ + "additional_write_policy text,"
+ + "cdc boolean,"
+ + "read_repair text,"
+ + "PRIMARY KEY ((keyspace_name), view_name))");
- private static final CFMetaData Triggers =
- compile(TRIGGERS,
- "trigger definitions",
- "CREATE TABLE %s ("
- + "keyspace_name text,"
- + "table_name text,"
- + "trigger_name text,"
- + "options frozen<map<text, text>>,"
- + "PRIMARY KEY ((keyspace_name), table_name, trigger_name))");
+ private static final TableMetadata Indexes =
+ parse(INDEXES,
+ "secondary index definitions",
+ "CREATE TABLE %s ("
+ + "keyspace_name text,"
+ + "table_name text,"
+ + "index_name text,"
+ + "kind text,"
+ + "options frozen<map<text, text>>,"
+ + "PRIMARY KEY ((keyspace_name), table_name, index_name))");
- private static final CFMetaData Views =
- compile(VIEWS,
- "view definitions",
- "CREATE TABLE %s ("
- + "keyspace_name text,"
- + "view_name text,"
- + "base_table_id uuid,"
- + "base_table_name text,"
- + "where_clause text,"
- + "bloom_filter_fp_chance double,"
- + "caching frozen<map<text, text>>,"
- + "comment text,"
- + "compaction frozen<map<text, text>>,"
- + "compression frozen<map<text, text>>,"
- + "crc_check_chance double,"
- + "dclocal_read_repair_chance double,"
- + "default_time_to_live int,"
- + "extensions frozen<map<text, blob>>,"
- + "gc_grace_seconds int,"
- + "id uuid,"
- + "include_all_columns boolean,"
- + "max_index_interval int,"
- + "memtable_flush_period_in_ms int,"
- + "min_index_interval int,"
- + "read_repair_chance double,"
- + "speculative_retry text,"
- + "cdc boolean,"
- + "PRIMARY KEY ((keyspace_name), view_name))");
+ private static final TableMetadata Types =
+ parse(TYPES,
+ "user defined type definitions",
+ "CREATE TABLE %s ("
+ + "keyspace_name text,"
+ + "type_name text,"
+ + "field_names frozen<list<text>>,"
+ + "field_types frozen<list<text>>,"
+ + "PRIMARY KEY ((keyspace_name), type_name))");
- private static final CFMetaData Indexes =
- compile(INDEXES,
- "secondary index definitions",
- "CREATE TABLE %s ("
- + "keyspace_name text,"
- + "table_name text,"
- + "index_name text,"
- + "kind text,"
- + "options frozen<map<text, text>>,"
- + "PRIMARY KEY ((keyspace_name), table_name, index_name))");
+ private static final TableMetadata Functions =
+ parse(FUNCTIONS,
+ "user defined function definitions",
+ "CREATE TABLE %s ("
+ + "keyspace_name text,"
+ + "function_name text,"
+ + "argument_types frozen<list<text>>,"
+ + "argument_names frozen<list<text>>,"
+ + "body text,"
+ + "language text,"
+ + "return_type text,"
+ + "called_on_null_input boolean,"
+ + "PRIMARY KEY ((keyspace_name), function_name, argument_types))");
- private static final CFMetaData Types =
- compile(TYPES,
- "user defined type definitions",
- "CREATE TABLE %s ("
- + "keyspace_name text,"
- + "type_name text,"
- + "field_names frozen<list<text>>,"
- + "field_types frozen<list<text>>,"
- + "PRIMARY KEY ((keyspace_name), type_name))");
+ private static final TableMetadata Aggregates =
+ parse(AGGREGATES,
+ "user defined aggregate definitions",
+ "CREATE TABLE %s ("
+ + "keyspace_name text,"
+ + "aggregate_name text,"
+ + "argument_types frozen<list<text>>,"
+ + "final_func text,"
+ + "initcond text,"
+ + "return_type text,"
+ + "state_func text,"
+ + "state_type text,"
+ + "PRIMARY KEY ((keyspace_name), aggregate_name, argument_types))");
- private static final CFMetaData Functions =
- compile(FUNCTIONS,
- "user defined function definitions",
- "CREATE TABLE %s ("
- + "keyspace_name text,"
- + "function_name text,"
- + "argument_types frozen<list<text>>,"
- + "argument_names frozen<list<text>>,"
- + "body text,"
- + "language text,"
- + "return_type text,"
- + "called_on_null_input boolean,"
- + "PRIMARY KEY ((keyspace_name), function_name, argument_types))");
-
- private static final CFMetaData Aggregates =
- compile(AGGREGATES,
- "user defined aggregate definitions",
- "CREATE TABLE %s ("
- + "keyspace_name text,"
- + "aggregate_name text,"
- + "argument_types frozen<list<text>>,"
- + "final_func text,"
- + "initcond text,"
- + "return_type text,"
- + "state_func text,"
- + "state_type text,"
- + "PRIMARY KEY ((keyspace_name), aggregate_name, argument_types))");
-
- public static final List<CFMetaData> ALL_TABLE_METADATA =
+ private static final List<TableMetadata> ALL_TABLE_METADATA =
ImmutableList.of(Keyspaces, Tables, Columns, Triggers, DroppedColumns, Views, Types, Functions, Aggregates, Indexes);
- private static CFMetaData compile(String name, String description, String schema)
+ private static TableMetadata parse(String name, String description, String cql)
{
- return CFMetaData.compile(String.format(schema, name), SchemaConstants.SCHEMA_KEYSPACE_NAME)
- .comment(description)
- .gcGraceSeconds((int) TimeUnit.DAYS.toSeconds(7));
+ return CreateTableStatement.parse(format(cql, name), SchemaConstants.SCHEMA_KEYSPACE_NAME)
+ .id(TableId.forSystemTable(SchemaConstants.SCHEMA_KEYSPACE_NAME, name))
+ .gcGraceSeconds((int) TimeUnit.DAYS.toSeconds(7))
+ .memtableFlushPeriod((int) TimeUnit.HOURS.toMillis(1))
+ .comment(description)
+ .build();
}
public static KeyspaceMetadata metadata()
@@ -276,13 +274,51 @@
return KeyspaceMetadata.create(SchemaConstants.SCHEMA_KEYSPACE_NAME, KeyspaceParams.local(), org.apache.cassandra.schema.Tables.of(ALL_TABLE_METADATA));
}
+ static Collection<Mutation> convertSchemaDiffToMutations(KeyspacesDiff diff, long timestamp)
+ {
+ Map<String, Mutation> mutations = new HashMap<>();
+
+ diff.created.forEach(k -> mutations.put(k.name, makeCreateKeyspaceMutation(k, timestamp).build()));
+ diff.dropped.forEach(k -> mutations.put(k.name, makeDropKeyspaceMutation(k, timestamp).build()));
+ diff.altered.forEach(kd ->
+ {
+ KeyspaceMetadata ks = kd.after;
+
+ Mutation.SimpleBuilder builder = makeCreateKeyspaceMutation(ks.name, ks.params, timestamp);
+
+ kd.types.created.forEach(t -> addTypeToSchemaMutation(t, builder));
+ kd.types.dropped.forEach(t -> addDropTypeToSchemaMutation(t, builder));
+ kd.types.altered(Difference.SHALLOW).forEach(td -> addTypeToSchemaMutation(td.after, builder));
+
+ kd.tables.created.forEach(t -> addTableToSchemaMutation(t, true, builder));
+ kd.tables.dropped.forEach(t -> addDropTableToSchemaMutation(t, builder));
+ kd.tables.altered(Difference.SHALLOW).forEach(td -> addAlterTableToSchemaMutation(td.before, td.after, builder));
+
+ kd.views.created.forEach(v -> addViewToSchemaMutation(v, true, builder));
+ kd.views.dropped.forEach(v -> addDropViewToSchemaMutation(v, builder));
+ kd.views.altered(Difference.SHALLOW).forEach(vd -> addAlterViewToSchemaMutation(vd.before, vd.after, builder));
+
+ kd.udfs.created.forEach(f -> addFunctionToSchemaMutation((UDFunction) f, builder));
+ kd.udfs.dropped.forEach(f -> addDropFunctionToSchemaMutation((UDFunction) f, builder));
+ kd.udfs.altered(Difference.SHALLOW).forEach(fd -> addFunctionToSchemaMutation(fd.after, builder));
+
+ kd.udas.created.forEach(a -> addAggregateToSchemaMutation((UDAggregate) a, builder));
+ kd.udas.dropped.forEach(a -> addDropAggregateToSchemaMutation((UDAggregate) a, builder));
+ kd.udas.altered(Difference.SHALLOW).forEach(ad -> addAggregateToSchemaMutation(ad.after, builder));
+
+ mutations.put(ks.name, builder.build());
+ });
+
+ return mutations.values();
+ }
+
/**
* Add entries to system_schema.* for the hardcoded system keyspaces
*/
public static void saveSystemKeyspacesSchema()
{
- KeyspaceMetadata system = Schema.instance.getKSMetaData(SchemaConstants.SYSTEM_KEYSPACE_NAME);
- KeyspaceMetadata schema = Schema.instance.getKSMetaData(SchemaConstants.SCHEMA_KEYSPACE_NAME);
+ KeyspaceMetadata system = Schema.instance.getKeyspaceMetadata(SchemaConstants.SYSTEM_KEYSPACE_NAME);
+ KeyspaceMetadata schema = Schema.instance.getKeyspaceMetadata(SchemaConstants.SCHEMA_KEYSPACE_NAME);
long timestamp = FBUtilities.timestampMicros();
@@ -304,7 +340,7 @@
ALL.reverse().forEach(table -> getSchemaCFS(table).truncateBlocking());
}
- static void flush()
+ private static void flush()
{
if (!DatabaseDescriptor.isUnsafeSystem())
ALL.forEach(table -> FBUtilities.waitOnFuture(getSchemaCFS(table).forceFlush()));
@@ -313,33 +349,11 @@
/**
* Read schema from system keyspace and calculate MD5 digest of every row, resulting digest
* will be converted into UUID which would act as content-based version of the schema.
- *
- * This implementation is special cased for 3.11 as it returns the schema digests for 3.11
- * <em>and</em> 3.0 - i.e. with and without the beloved {@code cdc} column.
*/
- public static Pair<UUID, UUID> calculateSchemaDigest()
+ static UUID calculateSchemaDigest()
{
- Set<ByteBuffer> cdc = Collections.singleton(ByteBufferUtil.bytes("cdc"));
-
- return calculateSchemaDigest(cdc);
- }
-
- @VisibleForTesting
- static Pair<UUID, UUID> calculateSchemaDigest(Set<ByteBuffer> columnsToExclude)
- {
- MessageDigest digest;
- MessageDigest digest30;
- try
- {
- digest = MessageDigest.getInstance("MD5");
- digest30 = MessageDigest.getInstance("MD5");
- }
- catch (NoSuchAlgorithmException e)
- {
- throw new RuntimeException(e);
- }
-
- for (String table : ALL_FOR_DIGEST)
+ Digest digest = Digest.forSchema();
+ for (String table : ALL)
{
ReadCommand cmd = getReadCommandForTableSchema(table);
try (ReadExecutionController executionController = cmd.executionController();
@@ -350,15 +364,12 @@
try (RowIterator partition = schema.next())
{
if (!isSystemKeyspaceSchemaPartition(partition.partitionKey()))
- {
- RowIterators.digest(partition, digest, digest30, columnsToExclude);
- }
+ RowIterators.digest(partition, digest);
}
}
}
}
-
- return Pair.create(UUID.nameUUIDFromBytes(digest.digest()), UUID.nameUUIDFromBytes(digest30.digest()));
+ return UUID.nameUUIDFromBytes(digest.digest());
}
/**
@@ -377,20 +388,20 @@
private static ReadCommand getReadCommandForTableSchema(String schemaTableName)
{
ColumnFamilyStore cfs = getSchemaCFS(schemaTableName);
- return PartitionRangeReadCommand.allDataRead(cfs.metadata, FBUtilities.nowInSeconds());
+ return PartitionRangeReadCommand.allDataRead(cfs.metadata(), FBUtilities.nowInSeconds());
}
- public static Collection<Mutation> convertSchemaToMutations()
+ static Collection<Mutation> convertSchemaToMutations()
{
- Map<DecoratedKey, Mutation> mutationMap = new HashMap<>();
+ Map<DecoratedKey, Mutation.PartitionUpdateCollector> mutationMap = new HashMap<>();
for (String table : ALL)
convertSchemaToMutations(mutationMap, table);
- return mutationMap.values();
+ return mutationMap.values().stream().map(Mutation.PartitionUpdateCollector::build).collect(Collectors.toList());
}
- private static void convertSchemaToMutations(Map<DecoratedKey, Mutation> mutationMap, String schemaTableName)
+ private static void convertSchemaToMutations(Map<DecoratedKey, Mutation.PartitionUpdateCollector> mutationMap, String schemaTableName)
{
ReadCommand cmd = getReadCommandForTableSchema(schemaTableName);
try (ReadExecutionController executionController = cmd.executionController();
@@ -404,14 +415,8 @@
continue;
DecoratedKey key = partition.partitionKey();
- Mutation mutation = mutationMap.get(key);
- if (mutation == null)
- {
- mutation = new Mutation(SchemaConstants.SCHEMA_KEYSPACE_NAME, key);
- mutationMap.put(key, mutation);
- }
-
- mutation.add(makeUpdateForSchema(partition, cmd.columnFilter()));
+ Mutation.PartitionUpdateCollector puCollector = mutationMap.computeIfAbsent(key, k -> new Mutation.PartitionUpdateCollector(SchemaConstants.SCHEMA_KEYSPACE_NAME, key));
+ puCollector.add(makeUpdateForSchema(partition, cmd.columnFilter()));
}
}
}
@@ -427,13 +432,13 @@
// This method is used during schema migration tasks, and if cdc is disabled, we want to force excluding the
// 'cdc' column from the TABLES/VIEWS schema table because it is problematic if received by older nodes (see #12236
// and #12697). Otherwise though, we just simply "buffer" the content of the partition into a PartitionUpdate.
- if (DatabaseDescriptor.isCDCEnabled() || !TABLES_WITH_CDC_ADDED.contains(partition.metadata().cfName))
+ if (DatabaseDescriptor.isCDCEnabled() || !TABLES_WITH_CDC_ADDED.contains(partition.metadata().name))
return PartitionUpdate.fromIterator(partition, filter);
// We want to skip the 'cdc' column. A simple solution for that is based on the fact that
// 'PartitionUpdate.fromIterator()' will ignore any columns that are marked as 'fetched' but not 'queried'.
- ColumnFilter.Builder builder = ColumnFilter.allColumnsBuilder(partition.metadata());
- for (ColumnDefinition column : filter.fetchedColumns())
+ ColumnFilter.Builder builder = ColumnFilter.allRegularColumnsBuilder(partition.metadata());
+ for (ColumnMetadata column : filter.fetchedColumns())
{
if (!column.name.toString().equals("cdc"))
builder.add(column);
@@ -451,14 +456,15 @@
* Schema entities to mutations
*/
- private static DecoratedKey decorate(CFMetaData metadata, Object value)
+ @SuppressWarnings("unchecked")
+ private static DecoratedKey decorate(TableMetadata metadata, Object value)
{
- return metadata.decorateKey(((AbstractType)metadata.getKeyValidator()).decompose(value));
+ return metadata.partitioner.decorateKey(((AbstractType) metadata.partitionKeyType).decompose(value));
}
- public static Mutation.SimpleBuilder makeCreateKeyspaceMutation(String name, KeyspaceParams params, long timestamp)
+ static Mutation.SimpleBuilder makeCreateKeyspaceMutation(String name, KeyspaceParams params, long timestamp)
{
- Mutation.SimpleBuilder builder = Mutation.simpleBuilder(Keyspaces.ksName, decorate(Keyspaces, name))
+ Mutation.SimpleBuilder builder = Mutation.simpleBuilder(Keyspaces.keyspace, decorate(Keyspaces, name))
.timestamp(timestamp);
builder.update(Keyspaces)
@@ -469,7 +475,7 @@
return builder;
}
- public static Mutation.SimpleBuilder makeCreateKeyspaceMutation(KeyspaceMetadata keyspace, long timestamp)
+ static Mutation.SimpleBuilder makeCreateKeyspaceMutation(KeyspaceMetadata keyspace, long timestamp)
{
Mutation.SimpleBuilder builder = makeCreateKeyspaceMutation(keyspace.name, keyspace.params, timestamp);
@@ -482,26 +488,18 @@
return builder;
}
- public static Mutation.SimpleBuilder makeDropKeyspaceMutation(KeyspaceMetadata keyspace, long timestamp)
+ static Mutation.SimpleBuilder makeDropKeyspaceMutation(KeyspaceMetadata keyspace, long timestamp)
{
Mutation.SimpleBuilder builder = Mutation.simpleBuilder(SchemaConstants.SCHEMA_KEYSPACE_NAME, decorate(Keyspaces, keyspace.name))
.timestamp(timestamp);
- for (CFMetaData schemaTable : ALL_TABLE_METADATA)
+ for (TableMetadata schemaTable : ALL_TABLE_METADATA)
builder.update(schemaTable).delete();
return builder;
}
- public static Mutation.SimpleBuilder makeCreateTypeMutation(KeyspaceMetadata keyspace, UserType type, long timestamp)
- {
- // Include the serialized keyspace in case the target node missed a CREATE KEYSPACE migration (see CASSANDRA-5631).
- Mutation.SimpleBuilder builder = makeCreateKeyspaceMutation(keyspace.name, keyspace.params, timestamp);
- addTypeToSchemaMutation(type, builder);
- return builder;
- }
-
- static void addTypeToSchemaMutation(UserType type, Mutation.SimpleBuilder mutation)
+ private static void addTypeToSchemaMutation(UserType type, Mutation.SimpleBuilder mutation)
{
mutation.update(Types)
.row(type.getNameAsString())
@@ -509,15 +507,12 @@
.add("field_types", type.fieldTypes().stream().map(AbstractType::asCQL3Type).map(CQL3Type::toString).collect(toList()));
}
- public static Mutation.SimpleBuilder dropTypeFromSchemaMutation(KeyspaceMetadata keyspace, UserType type, long timestamp)
+ private static void addDropTypeToSchemaMutation(UserType type, Mutation.SimpleBuilder builder)
{
- // Include the serialized keyspace in case the target node missed a CREATE KEYSPACE migration (see CASSANDRA-5631).
- Mutation.SimpleBuilder builder = makeCreateKeyspaceMutation(keyspace.name, keyspace.params, timestamp);
builder.update(Types).row(type.name).delete();
- return builder;
}
- public static Mutation.SimpleBuilder makeCreateTableMutation(KeyspaceMetadata keyspace, CFMetaData table, long timestamp)
+ static Mutation.SimpleBuilder makeCreateTableMutation(KeyspaceMetadata keyspace, TableMetadata table, long timestamp)
{
// Include the serialized keyspace in case the target node missed a CREATE KEYSPACE migration (see CASSANDRA-5631).
Mutation.SimpleBuilder builder = makeCreateKeyspaceMutation(keyspace.name, keyspace.params, timestamp);
@@ -525,27 +520,27 @@
return builder;
}
- public static void addTableToSchemaMutation(CFMetaData table, boolean withColumnsAndTriggers, Mutation.SimpleBuilder builder)
+ static void addTableToSchemaMutation(TableMetadata table, boolean withColumnsAndTriggers, Mutation.SimpleBuilder builder)
{
Row.SimpleBuilder rowBuilder = builder.update(Tables)
- .row(table.cfName)
- .add("id", table.cfId)
- .add("flags", CFMetaData.flagsToStrings(table.flags()));
+ .row(table.name)
+ .add("id", table.id.asUUID())
+ .add("flags", TableMetadata.Flag.toStringSet(table.flags));
addTableParamsToRowBuilder(table.params, rowBuilder);
if (withColumnsAndTriggers)
{
- for (ColumnDefinition column : table.allColumns())
+ for (ColumnMetadata column : table.columns())
addColumnToSchemaMutation(table, column, builder);
- for (CFMetaData.DroppedColumn column : table.getDroppedColumns().values())
+ for (DroppedColumn column : table.droppedColumns.values())
addDroppedColumnToSchemaMutation(table, column, builder);
- for (TriggerMetadata trigger : table.getTriggers())
+ for (TriggerMetadata trigger : table.triggers)
addTriggerToSchemaMutation(table, trigger, builder);
- for (IndexMetadata index : table.getIndexes())
+ for (IndexMetadata index : table.indexes)
addIndexToSchemaMutation(table, index, builder);
}
}
@@ -554,18 +549,20 @@
{
builder.add("bloom_filter_fp_chance", params.bloomFilterFpChance)
.add("comment", params.comment)
- .add("dclocal_read_repair_chance", params.dcLocalReadRepairChance)
+ .add("dclocal_read_repair_chance", 0.0) // no longer used, left for drivers' sake
.add("default_time_to_live", params.defaultTimeToLive)
.add("gc_grace_seconds", params.gcGraceSeconds)
.add("max_index_interval", params.maxIndexInterval)
.add("memtable_flush_period_in_ms", params.memtableFlushPeriodInMs)
.add("min_index_interval", params.minIndexInterval)
- .add("read_repair_chance", params.readRepairChance)
+ .add("read_repair_chance", 0.0) // no longer used, left for drivers' sake
.add("speculative_retry", params.speculativeRetry.toString())
+ .add("additional_write_policy", params.additionalWritePolicy.toString())
.add("crc_check_chance", params.crcCheckChance)
.add("caching", params.caching.asMap())
.add("compaction", params.compaction.asMap())
.add("compression", params.compression.asMap())
+ .add("read_repair", params.readRepair.toString())
.add("extensions", params.extensions);
// Only add CDC-enabled flag to schema if it's enabled on the node. This is to work around RTE's post-8099 if a 3.8+
@@ -574,43 +571,37 @@
builder.add("cdc", params.cdc);
}
- public static Mutation.SimpleBuilder makeUpdateTableMutation(KeyspaceMetadata keyspace,
- CFMetaData oldTable,
- CFMetaData newTable,
- long timestamp)
+ private static void addAlterTableToSchemaMutation(TableMetadata oldTable, TableMetadata newTable, Mutation.SimpleBuilder builder)
{
- Mutation.SimpleBuilder builder = makeCreateKeyspaceMutation(keyspace.name, keyspace.params, timestamp);
-
addTableToSchemaMutation(newTable, false, builder);
- MapDifference<ByteBuffer, ColumnDefinition> columnDiff = Maps.difference(oldTable.getColumnMetadata(),
- newTable.getColumnMetadata());
+ MapDifference<ByteBuffer, ColumnMetadata> columnDiff = Maps.difference(oldTable.columns, newTable.columns);
// columns that are no longer needed
- for (ColumnDefinition column : columnDiff.entriesOnlyOnLeft().values())
+ for (ColumnMetadata column : columnDiff.entriesOnlyOnLeft().values())
dropColumnFromSchemaMutation(oldTable, column, builder);
// newly added columns
- for (ColumnDefinition column : columnDiff.entriesOnlyOnRight().values())
+ for (ColumnMetadata column : columnDiff.entriesOnlyOnRight().values())
addColumnToSchemaMutation(newTable, column, builder);
// old columns with updated attributes
for (ByteBuffer name : columnDiff.entriesDiffering().keySet())
- addColumnToSchemaMutation(newTable, newTable.getColumnDefinition(name), builder);
+ addColumnToSchemaMutation(newTable, newTable.getColumn(name), builder);
// dropped columns
- MapDifference<ByteBuffer, CFMetaData.DroppedColumn> droppedColumnDiff =
- Maps.difference(oldTable.getDroppedColumns(), newTable.getDroppedColumns());
+ MapDifference<ByteBuffer, DroppedColumn> droppedColumnDiff =
+ Maps.difference(oldTable.droppedColumns, newTable.droppedColumns);
// newly dropped columns
- for (CFMetaData.DroppedColumn column : droppedColumnDiff.entriesOnlyOnRight().values())
+ for (DroppedColumn column : droppedColumnDiff.entriesOnlyOnRight().values())
addDroppedColumnToSchemaMutation(newTable, column, builder);
// columns added then dropped again
for (ByteBuffer name : droppedColumnDiff.entriesDiffering().keySet())
- addDroppedColumnToSchemaMutation(newTable, newTable.getDroppedColumns().get(name), builder);
+ addDroppedColumnToSchemaMutation(newTable, newTable.droppedColumns.get(name), builder);
- MapDifference<String, TriggerMetadata> triggerDiff = triggersDiff(oldTable.getTriggers(), newTable.getTriggers());
+ MapDifference<String, TriggerMetadata> triggerDiff = triggersDiff(oldTable.triggers, newTable.triggers);
// dropped triggers
for (TriggerMetadata trigger : triggerDiff.entriesOnlyOnLeft().values())
@@ -620,8 +611,7 @@
for (TriggerMetadata trigger : triggerDiff.entriesOnlyOnRight().values())
addTriggerToSchemaMutation(newTable, trigger, builder);
- MapDifference<String, IndexMetadata> indexesDiff = indexesDiff(oldTable.getIndexes(),
- newTable.getIndexes());
+ MapDifference<String, IndexMetadata> indexesDiff = indexesDiff(oldTable.indexes, newTable.indexes);
// dropped indexes
for (IndexMetadata index : indexesDiff.entriesOnlyOnLeft().values())
@@ -634,7 +624,15 @@
// updated indexes need to be updated
for (MapDifference.ValueDifference<IndexMetadata> diff : indexesDiff.entriesDiffering().values())
addUpdatedIndexToSchemaMutation(newTable, diff.rightValue(), builder);
+ }
+ static Mutation.SimpleBuilder makeUpdateTableMutation(KeyspaceMetadata keyspace,
+ TableMetadata oldTable,
+ TableMetadata newTable,
+ long timestamp)
+ {
+ Mutation.SimpleBuilder builder = makeCreateKeyspaceMutation(keyspace.name, keyspace.params, timestamp);
+ addAlterTableToSchemaMutation(oldTable, newTable, builder);
return builder;
}
@@ -660,36 +658,39 @@
return Maps.difference(beforeMap, afterMap);
}
- public static Mutation.SimpleBuilder makeDropTableMutation(KeyspaceMetadata keyspace, CFMetaData table, long timestamp)
+ static Mutation.SimpleBuilder makeDropTableMutation(KeyspaceMetadata keyspace, TableMetadata table, long timestamp)
{
// Include the serialized keyspace in case the target node missed a CREATE KEYSPACE migration (see CASSANDRA-5631).
Mutation.SimpleBuilder builder = makeCreateKeyspaceMutation(keyspace.name, keyspace.params, timestamp);
-
- builder.update(Tables).row(table.cfName).delete();
-
- for (ColumnDefinition column : table.allColumns())
- dropColumnFromSchemaMutation(table, column, builder);
-
- for (CFMetaData.DroppedColumn column : table.getDroppedColumns().values())
- dropDroppedColumnFromSchemaMutation(table, column, timestamp, builder);
-
- for (TriggerMetadata trigger : table.getTriggers())
- dropTriggerFromSchemaMutation(table, trigger, builder);
-
- for (IndexMetadata index : table.getIndexes())
- dropIndexFromSchemaMutation(table, index, builder);
-
+ addDropTableToSchemaMutation(table, builder);
return builder;
}
- private static void addColumnToSchemaMutation(CFMetaData table, ColumnDefinition column, Mutation.SimpleBuilder builder)
+ private static void addDropTableToSchemaMutation(TableMetadata table, Mutation.SimpleBuilder builder)
+ {
+ builder.update(Tables).row(table.name).delete();
+
+ for (ColumnMetadata column : table.columns())
+ dropColumnFromSchemaMutation(table, column, builder);
+
+ for (TriggerMetadata trigger : table.triggers)
+ dropTriggerFromSchemaMutation(table, trigger, builder);
+
+ for (DroppedColumn column : table.droppedColumns.values())
+ dropDroppedColumnFromSchemaMutation(table, column, builder);
+
+ for (IndexMetadata index : table.indexes)
+ dropIndexFromSchemaMutation(table, index, builder);
+ }
+
+ private static void addColumnToSchemaMutation(TableMetadata table, ColumnMetadata column, Mutation.SimpleBuilder builder)
{
AbstractType<?> type = column.type;
if (type instanceof ReversedType)
type = ((ReversedType) type).baseType;
builder.update(Columns)
- .row(table.cfName, column.name.toString())
+ .row(table.name, column.name.toString())
.add("column_name_bytes", column.name.bytes)
.add("kind", column.kind.toString().toLowerCase())
.add("position", column.position())
@@ -697,158 +698,113 @@
.add("type", type.asCQL3Type().toString());
}
- private static void dropColumnFromSchemaMutation(CFMetaData table, ColumnDefinition column, Mutation.SimpleBuilder builder)
+ private static void dropColumnFromSchemaMutation(TableMetadata table, ColumnMetadata column, Mutation.SimpleBuilder builder)
{
// Note: we do want to use name.toString(), not name.bytes directly for backward compatibility (For CQL3, this won't make a difference).
- builder.update(Columns).row(table.cfName, column.name.toString()).delete();
+ builder.update(Columns).row(table.name, column.name.toString()).delete();
}
- private static void addDroppedColumnToSchemaMutation(CFMetaData table, CFMetaData.DroppedColumn column, Mutation.SimpleBuilder builder)
+ private static void addDroppedColumnToSchemaMutation(TableMetadata table, DroppedColumn column, Mutation.SimpleBuilder builder)
{
builder.update(DroppedColumns)
- .row(table.cfName, column.name)
+ .row(table.name, column.column.name.toString())
.add("dropped_time", new Date(TimeUnit.MICROSECONDS.toMillis(column.droppedTime)))
- .add("kind", null != column.kind ? column.kind.toString().toLowerCase() : null)
- .add("type", expandUserTypes(column.type).asCQL3Type().toString());
+ .add("type", column.column.type.asCQL3Type().toString())
+ .add("kind", column.column.kind.toString().toLowerCase());
}
- private static void dropDroppedColumnFromSchemaMutation(CFMetaData table, DroppedColumn column, long timestamp, Mutation.SimpleBuilder builder)
+ private static void dropDroppedColumnFromSchemaMutation(TableMetadata table, DroppedColumn column, Mutation.SimpleBuilder builder)
{
- builder.update(DroppedColumns).row(table.cfName, column.name).delete();
+ builder.update(DroppedColumns).row(table.name, column.column.name.toString()).delete();
}
- private static void addTriggerToSchemaMutation(CFMetaData table, TriggerMetadata trigger, Mutation.SimpleBuilder builder)
+ private static void addTriggerToSchemaMutation(TableMetadata table, TriggerMetadata trigger, Mutation.SimpleBuilder builder)
{
builder.update(Triggers)
- .row(table.cfName, trigger.name)
+ .row(table.name, trigger.name)
.add("options", Collections.singletonMap("class", trigger.classOption));
}
- private static void dropTriggerFromSchemaMutation(CFMetaData table, TriggerMetadata trigger, Mutation.SimpleBuilder builder)
+ private static void dropTriggerFromSchemaMutation(TableMetadata table, TriggerMetadata trigger, Mutation.SimpleBuilder builder)
{
- builder.update(Triggers).row(table.cfName, trigger.name).delete();
+ builder.update(Triggers).row(table.name, trigger.name).delete();
}
- public static Mutation.SimpleBuilder makeCreateViewMutation(KeyspaceMetadata keyspace, ViewDefinition view, long timestamp)
+ private static void addViewToSchemaMutation(ViewMetadata view, boolean includeColumns, Mutation.SimpleBuilder builder)
{
- // Include the serialized keyspace in case the target node missed a CREATE KEYSPACE migration (see CASSANDRA-5631).
- Mutation.SimpleBuilder builder = makeCreateKeyspaceMutation(keyspace.name, keyspace.params, timestamp);
- addViewToSchemaMutation(view, true, builder);
- return builder;
- }
-
- private static void addViewToSchemaMutation(ViewDefinition view, boolean includeColumns, Mutation.SimpleBuilder builder)
- {
- CFMetaData table = view.metadata;
+ TableMetadata table = view.metadata;
Row.SimpleBuilder rowBuilder = builder.update(Views)
- .row(view.viewName)
+ .row(view.name())
.add("include_all_columns", view.includeAllColumns)
- .add("base_table_id", view.baseTableId)
- .add("base_table_name", view.baseTableMetadata().cfName)
- .add("where_clause", view.whereClause)
- .add("id", table.cfId);
+ .add("base_table_id", view.baseTableId.asUUID())
+ .add("base_table_name", view.baseTableName)
+ .add("where_clause", view.whereClause.toString())
+ .add("id", table.id.asUUID());
addTableParamsToRowBuilder(table.params, rowBuilder);
if (includeColumns)
{
- for (ColumnDefinition column : table.allColumns())
+ for (ColumnMetadata column : table.columns())
addColumnToSchemaMutation(table, column, builder);
- for (CFMetaData.DroppedColumn column : table.getDroppedColumns().values())
+ for (DroppedColumn column : table.droppedColumns.values())
addDroppedColumnToSchemaMutation(table, column, builder);
}
}
- public static Mutation.SimpleBuilder makeDropViewMutation(KeyspaceMetadata keyspace, ViewDefinition view, long timestamp)
+ private static void addDropViewToSchemaMutation(ViewMetadata view, Mutation.SimpleBuilder builder)
{
- // Include the serialized keyspace in case the target node missed a CREATE KEYSPACE migration (see CASSANDRA-5631).
- Mutation.SimpleBuilder builder = makeCreateKeyspaceMutation(keyspace.name, keyspace.params, timestamp);
+ builder.update(Views).row(view.name()).delete();
- builder.update(Views).row(view.viewName).delete();
-
- CFMetaData table = view.metadata;
- for (ColumnDefinition column : table.allColumns())
+ TableMetadata table = view.metadata;
+ for (ColumnMetadata column : table.columns())
dropColumnFromSchemaMutation(table, column, builder);
-
- for (IndexMetadata index : table.getIndexes())
- dropIndexFromSchemaMutation(table, index, builder);
-
- return builder;
}
- public static Mutation.SimpleBuilder makeUpdateViewMutation(Mutation.SimpleBuilder builder,
- ViewDefinition oldView,
- ViewDefinition newView)
+ private static void addAlterViewToSchemaMutation(ViewMetadata before, ViewMetadata after, Mutation.SimpleBuilder builder)
{
- addViewToSchemaMutation(newView, false, builder);
+ addViewToSchemaMutation(after, false, builder);
- MapDifference<ByteBuffer, ColumnDefinition> columnDiff = Maps.difference(oldView.metadata.getColumnMetadata(),
- newView.metadata.getColumnMetadata());
+ MapDifference<ByteBuffer, ColumnMetadata> columnDiff = Maps.difference(before.metadata.columns, after.metadata.columns);
// columns that are no longer needed
- for (ColumnDefinition column : columnDiff.entriesOnlyOnLeft().values())
- dropColumnFromSchemaMutation(oldView.metadata, column, builder);
+ for (ColumnMetadata column : columnDiff.entriesOnlyOnLeft().values())
+ dropColumnFromSchemaMutation(before.metadata, column, builder);
// newly added columns
- for (ColumnDefinition column : columnDiff.entriesOnlyOnRight().values())
- addColumnToSchemaMutation(newView.metadata, column, builder);
+ for (ColumnMetadata column : columnDiff.entriesOnlyOnRight().values())
+ addColumnToSchemaMutation(after.metadata, column, builder);
// old columns with updated attributes
for (ByteBuffer name : columnDiff.entriesDiffering().keySet())
- addColumnToSchemaMutation(newView.metadata, newView.metadata.getColumnDefinition(name), builder);
-
- // dropped columns
- MapDifference<ByteBuffer, CFMetaData.DroppedColumn> droppedColumnDiff =
- Maps.difference(oldView.metadata.getDroppedColumns(), oldView.metadata.getDroppedColumns());
-
- // newly dropped columns
- for (CFMetaData.DroppedColumn column : droppedColumnDiff.entriesOnlyOnRight().values())
- addDroppedColumnToSchemaMutation(oldView.metadata, column, builder);
-
- // columns added then dropped again
- for (ByteBuffer name : droppedColumnDiff.entriesDiffering().keySet())
- addDroppedColumnToSchemaMutation(newView.metadata, newView.metadata.getDroppedColumns().get(name), builder);
-
- return builder;
+ addColumnToSchemaMutation(after.metadata, after.metadata.getColumn(name), builder);
}
- private static void addIndexToSchemaMutation(CFMetaData table,
- IndexMetadata index,
- Mutation.SimpleBuilder builder)
+ private static void addIndexToSchemaMutation(TableMetadata table, IndexMetadata index, Mutation.SimpleBuilder builder)
{
builder.update(Indexes)
- .row(table.cfName, index.name)
+ .row(table.name, index.name)
.add("kind", index.kind.toString())
.add("options", index.options);
}
- private static void dropIndexFromSchemaMutation(CFMetaData table,
- IndexMetadata index,
- Mutation.SimpleBuilder builder)
+ private static void dropIndexFromSchemaMutation(TableMetadata table, IndexMetadata index, Mutation.SimpleBuilder builder)
{
- builder.update(Indexes).row(table.cfName, index.name).delete();
+ builder.update(Indexes).row(table.name, index.name).delete();
}
- private static void addUpdatedIndexToSchemaMutation(CFMetaData table,
+ private static void addUpdatedIndexToSchemaMutation(TableMetadata table,
IndexMetadata index,
Mutation.SimpleBuilder builder)
{
addIndexToSchemaMutation(table, index, builder);
}
- public static Mutation.SimpleBuilder makeCreateFunctionMutation(KeyspaceMetadata keyspace, UDFunction function, long timestamp)
- {
- // Include the serialized keyspace in case the target node missed a CREATE KEYSPACE migration (see CASSANDRA-5631).
- Mutation.SimpleBuilder builder = makeCreateKeyspaceMutation(keyspace.name, keyspace.params, timestamp);
- addFunctionToSchemaMutation(function, builder);
- return builder;
- }
-
- static void addFunctionToSchemaMutation(UDFunction function, Mutation.SimpleBuilder builder)
+ private static void addFunctionToSchemaMutation(UDFunction function, Mutation.SimpleBuilder builder)
{
builder.update(Functions)
- .row(function.name().name, functionArgumentsList(function))
+ .row(function.name().name, function.argumentsList())
.add("body", function.body())
.add("language", function.language())
.add("return_type", function.returnType().asCQL3Type().toString())
@@ -868,35 +824,15 @@
}
}
- private static List<String> functionArgumentsList(AbstractFunction fun)
+ private static void addDropFunctionToSchemaMutation(UDFunction function, Mutation.SimpleBuilder builder)
{
- return fun.argTypes()
- .stream()
- .map(AbstractType::asCQL3Type)
- .map(CQL3Type::toString)
- .collect(toList());
+ builder.update(Functions).row(function.name().name, function.argumentsList()).delete();
}
- public static Mutation.SimpleBuilder makeDropFunctionMutation(KeyspaceMetadata keyspace, UDFunction function, long timestamp)
- {
- // Include the serialized keyspace in case the target node missed a CREATE KEYSPACE migration (see CASSANDRA-5631).
- Mutation.SimpleBuilder builder = makeCreateKeyspaceMutation(keyspace.name, keyspace.params, timestamp);
- builder.update(Functions).row(function.name().name, functionArgumentsList(function)).delete();
- return builder;
- }
-
- public static Mutation.SimpleBuilder makeCreateAggregateMutation(KeyspaceMetadata keyspace, UDAggregate aggregate, long timestamp)
- {
- // Include the serialized keyspace in case the target node missed a CREATE KEYSPACE migration (see CASSANDRA-5631).
- Mutation.SimpleBuilder builder = makeCreateKeyspaceMutation(keyspace.name, keyspace.params, timestamp);
- addAggregateToSchemaMutation(aggregate, builder);
- return builder;
- }
-
- static void addAggregateToSchemaMutation(UDAggregate aggregate, Mutation.SimpleBuilder builder)
+ private static void addAggregateToSchemaMutation(UDAggregate aggregate, Mutation.SimpleBuilder builder)
{
builder.update(Aggregates)
- .row(aggregate.name().name, functionArgumentsList(aggregate))
+ .row(aggregate.name().name, aggregate.argumentsList())
.add("return_type", aggregate.returnType().asCQL3Type().toString())
.add("state_func", aggregate.stateFunction().name().name)
.add("state_type", aggregate.stateType().asCQL3Type().toString())
@@ -907,19 +843,16 @@
: null);
}
- public static Mutation.SimpleBuilder makeDropAggregateMutation(KeyspaceMetadata keyspace, UDAggregate aggregate, long timestamp)
+ private static void addDropAggregateToSchemaMutation(UDAggregate aggregate, Mutation.SimpleBuilder builder)
{
- // Include the serialized keyspace in case the target node missed a CREATE KEYSPACE migration (see CASSANDRA-5631).
- Mutation.SimpleBuilder builder = makeCreateKeyspaceMutation(keyspace.name, keyspace.params, timestamp);
- builder.update(Aggregates).row(aggregate.name().name, functionArgumentsList(aggregate)).delete();
- return builder;
+ builder.update(Aggregates).row(aggregate.name().name, aggregate.argumentsList()).delete();
}
/*
* Fetching schema
*/
- public static Keyspaces fetchNonSystemKeyspaces()
+ static Keyspaces fetchNonSystemKeyspaces()
{
return fetchKeyspacesWithout(SchemaConstants.LOCAL_SYSTEM_KEYSPACE_NAMES);
}
@@ -938,20 +871,6 @@
return keyspaces.build();
}
- private static Keyspaces fetchKeyspacesOnly(Set<String> includedKeyspaceNames)
- {
- /*
- * We know the keyspace names we are going to query, but we still want to run the SELECT IN
- * query, to filter out the keyspaces that had been dropped by the applied mutation set.
- */
- String query = format("SELECT keyspace_name FROM %s.%s WHERE keyspace_name IN ?", SchemaConstants.SCHEMA_KEYSPACE_NAME, KEYSPACES);
-
- Keyspaces.Builder keyspaces = org.apache.cassandra.schema.Keyspaces.builder();
- for (UntypedResultSet.Row row : query(query, new ArrayList<>(includedKeyspaceNames)))
- keyspaces.add(fetchKeyspace(row.getString("keyspace_name")));
- return keyspaces.build();
- }
-
private static KeyspaceMetadata fetchKeyspace(String keyspaceName)
{
KeyspaceParams params = fetchKeyspaceParams(keyspaceName);
@@ -1026,7 +945,7 @@
return tables.build();
}
- private static CFMetaData fetchTable(String keyspaceName, String tableName, Types types)
+ private static TableMetadata fetchTable(String keyspaceName, String tableName, Types types)
{
String query = String.format("SELECT * FROM %s.%s WHERE keyspace_name = ? AND table_name = ?", SchemaConstants.SCHEMA_KEYSPACE_NAME, TABLES);
UntypedResultSet rows = query(query, keyspaceName, tableName);
@@ -1034,43 +953,24 @@
throw new RuntimeException(String.format("%s:%s not found in the schema definitions keyspace.", keyspaceName, tableName));
UntypedResultSet.Row row = rows.one();
- UUID id = row.getUUID("id");
+ Set<TableMetadata.Flag> flags = TableMetadata.Flag.fromStringSet(row.getFrozenSet("flags", UTF8Type.instance));
- Set<CFMetaData.Flag> flags = CFMetaData.flagsFromStrings(row.getFrozenSet("flags", UTF8Type.instance));
-
- boolean isSuper = flags.contains(CFMetaData.Flag.SUPER);
- boolean isCounter = flags.contains(CFMetaData.Flag.COUNTER);
- boolean isDense = flags.contains(CFMetaData.Flag.DENSE);
- boolean isCompound = flags.contains(CFMetaData.Flag.COMPOUND);
-
- List<ColumnDefinition> columns = fetchColumns(keyspaceName, tableName, types);
- if (!columns.stream().anyMatch(ColumnDefinition::isPartitionKey))
+ if (!TableMetadata.Flag.isCQLCompatible(flags))
{
- String msg = String.format("Table %s.%s did not have any partition key columns in the schema tables", keyspaceName, tableName);
- throw new AssertionError(msg);
+ throw new IllegalArgumentException(TableMetadata.COMPACT_STORAGE_HALT_MESSAGE);
}
- Map<ByteBuffer, CFMetaData.DroppedColumn> droppedColumns = fetchDroppedColumns(keyspaceName, tableName);
- Indexes indexes = fetchIndexes(keyspaceName, tableName);
- Triggers triggers = fetchTriggers(keyspaceName, tableName);
-
- return CFMetaData.create(keyspaceName,
- tableName,
- id,
- isDense,
- isCompound,
- isSuper,
- isCounter,
- false,
- columns,
- DatabaseDescriptor.getPartitioner())
- .params(createTableParamsFromRow(row))
- .droppedColumns(droppedColumns)
- .indexes(indexes)
- .triggers(triggers);
+ return TableMetadata.builder(keyspaceName, tableName, TableId.fromUUID(row.getUUID("id")))
+ .flags(flags)
+ .params(createTableParamsFromRow(row))
+ .addColumns(fetchColumns(keyspaceName, tableName, types))
+ .droppedColumns(fetchDroppedColumns(keyspaceName, tableName))
+ .indexes(fetchIndexes(keyspaceName, tableName))
+ .triggers(fetchTriggers(keyspaceName, tableName))
+ .build();
}
- public static TableParams createTableParamsFromRow(UntypedResultSet.Row row)
+ static TableParams createTableParamsFromRow(UntypedResultSet.Row row)
{
return TableParams.builder()
.bloomFilterFpChance(row.getDouble("bloom_filter_fp_chance"))
@@ -1078,83 +978,89 @@
.comment(row.getString("comment"))
.compaction(CompactionParams.fromMap(row.getFrozenTextMap("compaction")))
.compression(CompressionParams.fromMap(row.getFrozenTextMap("compression")))
- .dcLocalReadRepairChance(row.getDouble("dclocal_read_repair_chance"))
.defaultTimeToLive(row.getInt("default_time_to_live"))
.extensions(row.getFrozenMap("extensions", UTF8Type.instance, BytesType.instance))
.gcGraceSeconds(row.getInt("gc_grace_seconds"))
.maxIndexInterval(row.getInt("max_index_interval"))
.memtableFlushPeriodInMs(row.getInt("memtable_flush_period_in_ms"))
.minIndexInterval(row.getInt("min_index_interval"))
- .readRepairChance(row.getDouble("read_repair_chance"))
.crcCheckChance(row.getDouble("crc_check_chance"))
- .speculativeRetry(SpeculativeRetryParam.fromString(row.getString("speculative_retry")))
- .cdc(row.has("cdc") ? row.getBoolean("cdc") : false)
+ .speculativeRetry(SpeculativeRetryPolicy.fromString(row.getString("speculative_retry")))
+ .additionalWritePolicy(row.has("additional_write_policy") ?
+ SpeculativeRetryPolicy.fromString(row.getString("additional_write_policy")) :
+ SpeculativeRetryPolicy.fromString("99PERCENTILE"))
+ .cdc(row.has("cdc") && row.getBoolean("cdc"))
+ .readRepair(getReadRepairStrategy(row))
.build();
}
- private static List<ColumnDefinition> fetchColumns(String keyspace, String table, Types types)
+ private static List<ColumnMetadata> fetchColumns(String keyspace, String table, Types types)
{
String query = format("SELECT * FROM %s.%s WHERE keyspace_name = ? AND table_name = ?", SchemaConstants.SCHEMA_KEYSPACE_NAME, COLUMNS);
UntypedResultSet columnRows = query(query, keyspace, table);
if (columnRows.isEmpty())
- throw new MissingColumns("Columns not found in schema table for " + keyspace + "." + table);
+ throw new MissingColumns("Columns not found in schema table for " + keyspace + '.' + table);
- List<ColumnDefinition> columns = new ArrayList<>();
+ List<ColumnMetadata> columns = new ArrayList<>();
columnRows.forEach(row -> columns.add(createColumnFromRow(row, types)));
- if (columns.stream().noneMatch(ColumnDefinition::isPartitionKey))
+ if (columns.stream().noneMatch(ColumnMetadata::isPartitionKey))
throw new MissingColumns("No partition key columns found in schema table for " + keyspace + "." + table);
return columns;
}
- public static ColumnDefinition createColumnFromRow(UntypedResultSet.Row row, Types types)
+ static ColumnMetadata createColumnFromRow(UntypedResultSet.Row row, Types types)
{
String keyspace = row.getString("keyspace_name");
String table = row.getString("table_name");
- ColumnDefinition.Kind kind = ColumnDefinition.Kind.valueOf(row.getString("kind").toUpperCase());
+ ColumnMetadata.Kind kind = ColumnMetadata.Kind.valueOf(row.getString("kind").toUpperCase());
int position = row.getInt("position");
ClusteringOrder order = ClusteringOrder.valueOf(row.getString("clustering_order").toUpperCase());
- AbstractType<?> type = parse(keyspace, row.getString("type"), types);
+ AbstractType<?> type = CQLTypeParser.parse(keyspace, row.getString("type"), types);
if (order == ClusteringOrder.DESC)
type = ReversedType.getInstance(type);
ColumnIdentifier name = new ColumnIdentifier(row.getBytes("column_name_bytes"), row.getString("column_name"));
- return new ColumnDefinition(keyspace, table, name, type, position, kind);
+ return new ColumnMetadata(keyspace, table, name, type, position, kind);
}
- private static Map<ByteBuffer, CFMetaData.DroppedColumn> fetchDroppedColumns(String keyspace, String table)
+ private static Map<ByteBuffer, DroppedColumn> fetchDroppedColumns(String keyspace, String table)
{
String query = format("SELECT * FROM %s.%s WHERE keyspace_name = ? AND table_name = ?", SchemaConstants.SCHEMA_KEYSPACE_NAME, DROPPED_COLUMNS);
- Map<ByteBuffer, CFMetaData.DroppedColumn> columns = new HashMap<>();
+ Map<ByteBuffer, DroppedColumn> columns = new HashMap<>();
for (UntypedResultSet.Row row : query(query, keyspace, table))
{
- CFMetaData.DroppedColumn column = createDroppedColumnFromRow(row);
- columns.put(UTF8Type.instance.decompose(column.name), column);
+ DroppedColumn column = createDroppedColumnFromRow(row);
+ columns.put(column.column.name.bytes, column);
}
return columns;
}
- private static CFMetaData.DroppedColumn createDroppedColumnFromRow(UntypedResultSet.Row row)
+ private static DroppedColumn createDroppedColumnFromRow(UntypedResultSet.Row row)
{
String keyspace = row.getString("keyspace_name");
+ String table = row.getString("table_name");
String name = row.getString("column_name");
-
- ColumnDefinition.Kind kind =
- row.has("kind") ? ColumnDefinition.Kind.valueOf(row.getString("kind").toUpperCase())
- : null;
/*
* we never store actual UDT names in dropped column types (so that we can safely drop types if nothing refers to
* them anymore), so before storing dropped columns in schema we expand UDTs to tuples. See expandUserTypes method.
* Because of that, we can safely pass Types.none() to parse()
*/
- AbstractType<?> type = parse(keyspace, row.getString("type"), org.apache.cassandra.schema.Types.none());
+ AbstractType<?> type = CQLTypeParser.parse(keyspace, row.getString("type"), org.apache.cassandra.schema.Types.none());
+ ColumnMetadata.Kind kind = row.has("kind")
+ ? ColumnMetadata.Kind.valueOf(row.getString("kind").toUpperCase())
+ : ColumnMetadata.Kind.REGULAR;
+ assert kind == ColumnMetadata.Kind.REGULAR || kind == ColumnMetadata.Kind.STATIC
+ : "Unexpected dropped column kind: " + kind.toString();
+
+ ColumnMetadata column = new ColumnMetadata(keyspace, table, ColumnIdentifier.getInterned(name, true), type, ColumnMetadata.NO_POSITION, kind);
long droppedTime = TimeUnit.MILLISECONDS.toMicros(row.getLong("dropped_time"));
- return new CFMetaData.DroppedColumn(name, kind, type, droppedTime);
+ return new DroppedColumn(column, droppedTime);
}
private static Indexes fetchIndexes(String keyspace, String table)
@@ -1194,11 +1100,11 @@
Views.Builder views = org.apache.cassandra.schema.Views.builder();
for (UntypedResultSet.Row row : query(query, keyspaceName))
- views.add(fetchView(keyspaceName, row.getString("view_name"), types));
+ views.put(fetchView(keyspaceName, row.getString("view_name"), types));
return views.build();
}
- private static ViewDefinition fetchView(String keyspaceName, String viewName, Types types)
+ private static ViewMetadata fetchView(String keyspaceName, String viewName, Types types)
{
String query = String.format("SELECT * FROM %s.%s WHERE keyspace_name = ? AND view_name = ?", SchemaConstants.SCHEMA_KEYSPACE_NAME, VIEWS);
UntypedResultSet rows = query(query, keyspaceName, viewName);
@@ -1206,54 +1112,51 @@
throw new RuntimeException(String.format("%s:%s not found in the schema definitions keyspace.", keyspaceName, viewName));
UntypedResultSet.Row row = rows.one();
- UUID id = row.getUUID("id");
- UUID baseTableId = row.getUUID("base_table_id");
+ TableId baseTableId = TableId.fromUUID(row.getUUID("base_table_id"));
String baseTableName = row.getString("base_table_name");
boolean includeAll = row.getBoolean("include_all_columns");
- String whereClause = row.getString("where_clause");
+ String whereClauseString = row.getString("where_clause");
- List<ColumnDefinition> columns = fetchColumns(keyspaceName, viewName, types);
+ List<ColumnMetadata> columns = fetchColumns(keyspaceName, viewName, types);
- Map<ByteBuffer, CFMetaData.DroppedColumn> droppedColumns = fetchDroppedColumns(keyspaceName, viewName);
+ TableMetadata metadata =
+ TableMetadata.builder(keyspaceName, viewName, TableId.fromUUID(row.getUUID("id")))
+ .kind(TableMetadata.Kind.VIEW)
+ .addColumns(columns)
+ .droppedColumns(fetchDroppedColumns(keyspaceName, viewName))
+ .params(createTableParamsFromRow(row))
+ .build();
- CFMetaData cfm = CFMetaData.create(keyspaceName,
- viewName,
- id,
- false,
- true,
- false,
- false,
- true,
- columns,
- DatabaseDescriptor.getPartitioner())
- .params(createTableParamsFromRow(row))
- .droppedColumns(droppedColumns);
+ WhereClause whereClause;
- String rawSelect = View.buildSelectStatement(baseTableName, columns, whereClause);
- SelectStatement.RawStatement rawStatement = (SelectStatement.RawStatement) QueryProcessor.parseStatement(rawSelect);
+ try
+ {
+ whereClause = WhereClause.parse(whereClauseString);
+ }
+ catch (RecognitionException e)
+ {
+ throw new RuntimeException(format("Unexpected error while parsing materialized view's where clause for '%s' (got %s)", viewName, whereClauseString));
+ }
- return new ViewDefinition(keyspaceName, viewName, baseTableId, baseTableName, includeAll, rawStatement, whereClause, cfm);
+ return new ViewMetadata(baseTableId, baseTableName, includeAll, whereClause, metadata);
}
private static Functions fetchFunctions(String keyspaceName, Types types)
{
- Functions udfs = fetchUDFs(keyspaceName, types);
- Functions udas = fetchUDAs(keyspaceName, udfs, types);
+ Collection<UDFunction> udfs = fetchUDFs(keyspaceName, types);
+ Collection<UDAggregate> udas = fetchUDAs(keyspaceName, udfs, types);
- return org.apache.cassandra.schema.Functions.builder()
- .add(udfs)
- .add(udas)
- .build();
+ return org.apache.cassandra.schema.Functions.builder().add(udfs).add(udas).build();
}
- private static Functions fetchUDFs(String keyspaceName, Types types)
+ private static Collection<UDFunction> fetchUDFs(String keyspaceName, Types types)
{
String query = format("SELECT * FROM %s.%s WHERE keyspace_name = ?", SchemaConstants.SCHEMA_KEYSPACE_NAME, FUNCTIONS);
- Functions.Builder functions = org.apache.cassandra.schema.Functions.builder();
+ Collection<UDFunction> functions = new ArrayList<>();
for (UntypedResultSet.Row row : query(query, keyspaceName))
functions.add(createUDFFromRow(row, types));
- return functions.build();
+ return functions;
}
private static UDFunction createUDFFromRow(UntypedResultSet.Row row, Types types)
@@ -1268,14 +1171,18 @@
List<AbstractType<?>> argTypes = new ArrayList<>();
for (String type : row.getFrozenList("argument_types", UTF8Type.instance))
- argTypes.add(parse(ksName, type, types));
+ argTypes.add(CQLTypeParser.parse(ksName, type, types));
- AbstractType<?> returnType = parse(ksName, row.getString("return_type"), types);
+ AbstractType<?> returnType = CQLTypeParser.parse(ksName, row.getString("return_type"), types);
String language = row.getString("language");
String body = row.getString("body");
boolean calledOnNullInput = row.getBoolean("called_on_null_input");
+ /*
+ * TODO: find a way to get rid of Schema.instance dependency; evaluate if the opimisation below makes a difference
+ * in the first place. Remove if it isn't.
+ */
org.apache.cassandra.cql3.functions.Function existing = Schema.instance.findFunction(name, argTypes).orElse(null);
if (existing instanceof UDFunction)
{
@@ -1284,7 +1191,8 @@
// statement, since CreateFunctionStatement needs to execute UDFunction.create but schema migration
// also needs that (since it needs to handle its own change).
UDFunction udf = (UDFunction) existing;
- if (udf.argNames().equals(argNames) && // arg types checked in Functions.find call
+ if (udf.argNames().equals(argNames) &&
+ udf.argTypes().equals(argTypes) &&
udf.returnType().equals(returnType) &&
!udf.isAggregate() &&
udf.language().equals(language) &&
@@ -1307,17 +1215,16 @@
}
}
- private static Functions fetchUDAs(String keyspaceName, Functions udfs, Types types)
+ private static Collection<UDAggregate> fetchUDAs(String keyspaceName, Collection<UDFunction> udfs, Types types)
{
String query = format("SELECT * FROM %s.%s WHERE keyspace_name = ?", SchemaConstants.SCHEMA_KEYSPACE_NAME, AGGREGATES);
- Functions.Builder aggregates = org.apache.cassandra.schema.Functions.builder();
- for (UntypedResultSet.Row row : query(query, keyspaceName))
- aggregates.add(createUDAFromRow(row, udfs, types));
- return aggregates.build();
+ Collection<UDAggregate> aggregates = new ArrayList<>();
+ query(query, keyspaceName).forEach(row -> aggregates.add(createUDAFromRow(row, udfs, types)));
+ return aggregates;
}
- private static UDAggregate createUDAFromRow(UntypedResultSet.Row row, Functions functions, Types types)
+ private static UDAggregate createUDAFromRow(UntypedResultSet.Row row, Collection<UDFunction> functions, Types types)
{
String ksName = row.getString("keyspace_name");
String functionName = row.getString("aggregate_name");
@@ -1326,24 +1233,18 @@
List<AbstractType<?>> argTypes =
row.getFrozenList("argument_types", UTF8Type.instance)
.stream()
- .map(t -> parse(ksName, t, types))
+ .map(t -> CQLTypeParser.parse(ksName, t, types))
.collect(toList());
- AbstractType<?> returnType = parse(ksName, row.getString("return_type"), types);
+ AbstractType<?> returnType = CQLTypeParser.parse(ksName, row.getString("return_type"), types);
FunctionName stateFunc = new FunctionName(ksName, (row.getString("state_func")));
+
FunctionName finalFunc = row.has("final_func") ? new FunctionName(ksName, row.getString("final_func")) : null;
- AbstractType<?> stateType = row.has("state_type") ? parse(ksName, row.getString("state_type"), types) : null;
+ AbstractType<?> stateType = row.has("state_type") ? CQLTypeParser.parse(ksName, row.getString("state_type"), types) : null;
ByteBuffer initcond = row.has("initcond") ? Terms.asBytes(ksName, row.getString("initcond"), stateType) : null;
- try
- {
- return UDAggregate.create(functions, name, argTypes, returnType, stateFunc, finalFunc, stateType, initcond);
- }
- catch (InvalidRequestException reason)
- {
- return UDAggregate.createBroken(name, argTypes, returnType, initcond, reason);
- }
+ return UDAggregate.create(functions, name, argTypes, returnType, stateFunc, finalFunc, stateType, initcond);
}
private static UntypedResultSet query(String query, Object... variables)
@@ -1355,180 +1256,36 @@
* Merging schema
*/
- /*
- * Reload schema from local disk. Useful if a user made changes to schema tables by hand, or has suspicion that
- * in-memory representation got out of sync somehow with what's on disk.
- */
- public static synchronized void reloadSchemaAndAnnounceVersion()
- {
- Keyspaces before = Schema.instance.getReplicatedKeyspaces();
- Keyspaces after = fetchNonSystemKeyspaces();
- mergeSchema(before, after);
- Schema.instance.updateVersionAndAnnounce();
- }
-
/**
- * Merge remote schema in form of mutations with local and mutate ks/cf metadata objects
- * (which also involves fs operations on add/drop ks/cf)
- *
- * @param mutations the schema changes to apply
- *
- * @throws ConfigurationException If one of metadata attributes has invalid value
+ * Computes the set of names of keyspaces affected by the provided schema mutations.
*/
- public static synchronized void mergeSchemaAndAnnounceVersion(Collection<Mutation> mutations) throws ConfigurationException
- {
- mergeSchema(mutations);
- Schema.instance.updateVersionAndAnnounce();
- }
-
- public static synchronized void mergeSchema(Collection<Mutation> mutations)
+ static Set<String> affectedKeyspaces(Collection<Mutation> mutations)
{
// only compare the keyspaces affected by this set of schema mutations
- Set<String> affectedKeyspaces =
- mutations.stream()
- .map(m -> UTF8Type.instance.compose(m.key().getKey()))
- .collect(Collectors.toSet());
+ return mutations.stream()
+ .map(m -> UTF8Type.instance.compose(m.key().getKey()))
+ .collect(toSet());
+ }
- // fetch the current state of schema for the affected keyspaces only
- Keyspaces before = Schema.instance.getKeyspaces(affectedKeyspaces);
-
- // apply the schema mutations and flush
+ static void applyChanges(Collection<Mutation> mutations)
+ {
mutations.forEach(Mutation::apply);
- if (FLUSH_SCHEMA_TABLES)
- flush();
-
- // fetch the new state of schema from schema tables (not applied to Schema.instance yet)
- Keyspaces after = fetchKeyspacesOnly(affectedKeyspaces);
-
- mergeSchema(before, after);
+ if (SchemaKeyspace.FLUSH_SCHEMA_TABLES)
+ SchemaKeyspace.flush();
}
- private static synchronized void mergeSchema(Keyspaces before, Keyspaces after)
+ static Keyspaces fetchKeyspaces(Set<String> toFetch)
{
- MapDifference<String, KeyspaceMetadata> keyspacesDiff = before.diff(after);
+ /*
+ * We know the keyspace names we are going to query, but we still want to run the SELECT IN
+ * query, to filter out the keyspaces that had been dropped by the applied mutation set.
+ */
+ String query = format("SELECT keyspace_name FROM %s.%s WHERE keyspace_name IN ?", SchemaConstants.SCHEMA_KEYSPACE_NAME, KEYSPACES);
- // dropped keyspaces
- for (KeyspaceMetadata keyspace : keyspacesDiff.entriesOnlyOnLeft().values())
- {
- keyspace.functions.udas().forEach(Schema.instance::dropAggregate);
- keyspace.functions.udfs().forEach(Schema.instance::dropFunction);
- keyspace.views.forEach(v -> Schema.instance.dropView(v.ksName, v.viewName));
- keyspace.tables.forEach(t -> Schema.instance.dropTable(t.ksName, t.cfName));
- keyspace.types.forEach(Schema.instance::dropType);
- Schema.instance.dropKeyspace(keyspace.name);
- }
-
- // new keyspaces
- for (KeyspaceMetadata keyspace : keyspacesDiff.entriesOnlyOnRight().values())
- {
- Schema.instance.addKeyspace(KeyspaceMetadata.create(keyspace.name, keyspace.params));
- keyspace.types.forEach(Schema.instance::addType);
- keyspace.tables.forEach(Schema.instance::addTable);
- keyspace.views.forEach(Schema.instance::addView);
- keyspace.functions.udfs().forEach(Schema.instance::addFunction);
- keyspace.functions.udas().forEach(Schema.instance::addAggregate);
- }
-
- // updated keyspaces
- for (Map.Entry<String, MapDifference.ValueDifference<KeyspaceMetadata>> diff : keyspacesDiff.entriesDiffering().entrySet())
- updateKeyspace(diff.getKey(), diff.getValue().leftValue(), diff.getValue().rightValue());
- }
-
- private static void updateKeyspace(String keyspaceName, KeyspaceMetadata keyspaceBefore, KeyspaceMetadata keyspaceAfter)
- {
- // calculate the deltas
- MapDifference<String, CFMetaData> tablesDiff = keyspaceBefore.tables.diff(keyspaceAfter.tables);
- MapDifference<String, ViewDefinition> viewsDiff = keyspaceBefore.views.diff(keyspaceAfter.views);
- MapDifference<ByteBuffer, UserType> typesDiff = keyspaceBefore.types.diff(keyspaceAfter.types);
-
- Map<Pair<FunctionName, List<String>>, UDFunction> udfsBefore = new HashMap<>();
- keyspaceBefore.functions.udfs().forEach(f -> udfsBefore.put(Pair.create(f.name(), functionArgumentsList(f)), f));
- Map<Pair<FunctionName, List<String>>, UDFunction> udfsAfter = new HashMap<>();
- keyspaceAfter.functions.udfs().forEach(f -> udfsAfter.put(Pair.create(f.name(), functionArgumentsList(f)), f));
- MapDifference<Pair<FunctionName, List<String>>, UDFunction> udfsDiff = Maps.difference(udfsBefore, udfsAfter);
-
- Map<Pair<FunctionName, List<String>>, UDAggregate> udasBefore = new HashMap<>();
- keyspaceBefore.functions.udas().forEach(f -> udasBefore.put(Pair.create(f.name(), functionArgumentsList(f)), f));
- Map<Pair<FunctionName, List<String>>, UDAggregate> udasAfter = new HashMap<>();
- keyspaceAfter.functions.udas().forEach(f -> udasAfter.put(Pair.create(f.name(), functionArgumentsList(f)), f));
- MapDifference<Pair<FunctionName, List<String>>, UDAggregate> udasDiff = Maps.difference(udasBefore, udasAfter);
-
- // update keyspace params, if changed
- if (!keyspaceBefore.params.equals(keyspaceAfter.params))
- Schema.instance.updateKeyspace(keyspaceName, keyspaceAfter.params);
-
- // drop everything removed
- udasDiff.entriesOnlyOnLeft().values().forEach(Schema.instance::dropAggregate);
- udfsDiff.entriesOnlyOnLeft().values().forEach(Schema.instance::dropFunction);
- viewsDiff.entriesOnlyOnLeft().values().forEach(v -> Schema.instance.dropView(v.ksName, v.viewName));
- tablesDiff.entriesOnlyOnLeft().values().forEach(t -> Schema.instance.dropTable(t.ksName, t.cfName));
- typesDiff.entriesOnlyOnLeft().values().forEach(Schema.instance::dropType);
-
- // add everything created
- typesDiff.entriesOnlyOnRight().values().forEach(Schema.instance::addType);
- tablesDiff.entriesOnlyOnRight().values().forEach(Schema.instance::addTable);
- viewsDiff.entriesOnlyOnRight().values().forEach(Schema.instance::addView);
- udfsDiff.entriesOnlyOnRight().values().forEach(Schema.instance::addFunction);
- udasDiff.entriesOnlyOnRight().values().forEach(Schema.instance::addAggregate);
-
- // update everything altered
- for (MapDifference.ValueDifference<UserType> diff : typesDiff.entriesDiffering().values())
- Schema.instance.updateType(diff.rightValue());
- for (MapDifference.ValueDifference<CFMetaData> diff : tablesDiff.entriesDiffering().values())
- Schema.instance.updateTable(diff.rightValue());
- for (MapDifference.ValueDifference<ViewDefinition> diff : viewsDiff.entriesDiffering().values())
- Schema.instance.updateView(diff.rightValue());
- for (MapDifference.ValueDifference<UDFunction> diff : udfsDiff.entriesDiffering().values())
- Schema.instance.updateFunction(diff.rightValue());
- for (MapDifference.ValueDifference<UDAggregate> diff : udasDiff.entriesDiffering().values())
- Schema.instance.updateAggregate(diff.rightValue());
- }
-
- /*
- * Type parsing and transformation
- */
-
- /*
- * Recursively replaces any instances of UserType with an equivalent TupleType.
- * We do it for dropped_columns, to allow safely dropping unused user types without retaining any references
- * in dropped_columns.
- */
- @VisibleForTesting
- public static AbstractType<?> expandUserTypes(AbstractType<?> original)
- {
- if (original instanceof UserType)
- return new TupleType(expandUserTypes(((UserType) original).fieldTypes()));
-
- if (original instanceof TupleType)
- return new TupleType(expandUserTypes(((TupleType) original).allTypes()));
-
- if (original instanceof ListType<?>)
- return ListType.getInstance(expandUserTypes(((ListType<?>) original).getElementsType()), original.isMultiCell());
-
- if (original instanceof MapType<?,?>)
- {
- MapType<?, ?> mt = (MapType<?, ?>) original;
- return MapType.getInstance(expandUserTypes(mt.getKeysType()), expandUserTypes(mt.getValuesType()), mt.isMultiCell());
- }
-
- if (original instanceof SetType<?>)
- return SetType.getInstance(expandUserTypes(((SetType<?>) original).getElementsType()), original.isMultiCell());
-
- // this is very unlikely to ever happen, but it's better to be safe than sorry
- if (original instanceof ReversedType<?>)
- return ReversedType.getInstance(expandUserTypes(((ReversedType) original).baseType));
-
- if (original instanceof CompositeType)
- return CompositeType.getInstance(expandUserTypes(original.getComponents()));
-
- return original;
- }
-
- private static List<AbstractType<?>> expandUserTypes(List<AbstractType<?>> types)
- {
- return types.stream()
- .map(SchemaKeyspace::expandUserTypes)
- .collect(toList());
+ Keyspaces.Builder keyspaces = org.apache.cassandra.schema.Keyspaces.builder();
+ for (UntypedResultSet.Row row : query(query, new ArrayList<>(toFetch)))
+ keyspaces.add(fetchKeyspace(row.getString("keyspace_name")));
+ return keyspaces.build();
}
@VisibleForTesting
@@ -1539,4 +1296,11 @@
super(message);
}
}
+
+ private static ReadRepairStrategy getReadRepairStrategy(UntypedResultSet.Row row)
+ {
+ return row.has("read_repair")
+ ? ReadRepairStrategy.fromString(row.getString("read_repair"))
+ : ReadRepairStrategy.BLOCKING;
+ }
}
diff --git a/src/java/org/apache/cassandra/schema/SchemaMigrationDiagnostics.java b/src/java/org/apache/cassandra/schema/SchemaMigrationDiagnostics.java
new file mode 100644
index 0000000..62f1768
--- /dev/null
+++ b/src/java/org/apache/cassandra/schema/SchemaMigrationDiagnostics.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.schema;
+
+import java.util.UUID;
+
+import org.apache.cassandra.diag.DiagnosticEventService;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.schema.SchemaMigrationEvent.MigrationManagerEventType;
+
+final class SchemaMigrationDiagnostics
+{
+ private static final DiagnosticEventService service = DiagnosticEventService.instance();
+
+ private SchemaMigrationDiagnostics()
+ {
+ }
+
+ static void unknownLocalSchemaVersion(InetAddressAndPort endpoint, UUID theirVersion)
+ {
+ if (isEnabled(MigrationManagerEventType.UNKNOWN_LOCAL_SCHEMA_VERSION))
+ service.publish(new SchemaMigrationEvent(MigrationManagerEventType.UNKNOWN_LOCAL_SCHEMA_VERSION, endpoint,
+ theirVersion));
+ }
+
+ static void versionMatch(InetAddressAndPort endpoint, UUID theirVersion)
+ {
+ if (isEnabled(MigrationManagerEventType.VERSION_MATCH))
+ service.publish(new SchemaMigrationEvent(MigrationManagerEventType.VERSION_MATCH, endpoint, theirVersion));
+ }
+
+ static void skipPull(InetAddressAndPort endpoint, UUID theirVersion)
+ {
+ if (isEnabled(MigrationManagerEventType.SKIP_PULL))
+ service.publish(new SchemaMigrationEvent(MigrationManagerEventType.SKIP_PULL, endpoint, theirVersion));
+ }
+
+ static void resetLocalSchema()
+ {
+ if (isEnabled(MigrationManagerEventType.RESET_LOCAL_SCHEMA))
+ service.publish(new SchemaMigrationEvent(MigrationManagerEventType.RESET_LOCAL_SCHEMA, null, null));
+ }
+
+ static void taskCreated(InetAddressAndPort endpoint)
+ {
+ if (isEnabled(MigrationManagerEventType.TASK_CREATED))
+ service.publish(new SchemaMigrationEvent(MigrationManagerEventType.TASK_CREATED, endpoint, null));
+ }
+
+ static void taskSendAborted(InetAddressAndPort endpoint)
+ {
+ if (isEnabled(MigrationManagerEventType.TASK_SEND_ABORTED))
+ service.publish(new SchemaMigrationEvent(MigrationManagerEventType.TASK_SEND_ABORTED, endpoint, null));
+ }
+
+ static void taskRequestSend(InetAddressAndPort endpoint)
+ {
+ if (isEnabled(MigrationManagerEventType.TASK_REQUEST_SEND))
+ service.publish(new SchemaMigrationEvent(MigrationManagerEventType.TASK_REQUEST_SEND,
+ endpoint, null));
+ }
+
+ private static boolean isEnabled(MigrationManagerEventType type)
+ {
+ return service.isEnabled(SchemaMigrationEvent.class, type);
+ }
+}
diff --git a/src/java/org/apache/cassandra/schema/SchemaMigrationEvent.java b/src/java/org/apache/cassandra/schema/SchemaMigrationEvent.java
new file mode 100644
index 0000000..45844b3
--- /dev/null
+++ b/src/java/org/apache/cassandra/schema/SchemaMigrationEvent.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.schema;
+
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Queue;
+import java.util.UUID;
+import java.util.concurrent.CountDownLatch;
+
+import javax.annotation.Nullable;
+
+import org.apache.cassandra.db.SystemKeyspace;
+import org.apache.cassandra.diag.DiagnosticEvent;
+import org.apache.cassandra.gms.FailureDetector;
+import org.apache.cassandra.gms.Gossiper;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.net.MessagingService;
+
+/**
+ * Internal events emitted by {@link MigrationManager}.
+ */
+final class SchemaMigrationEvent extends DiagnosticEvent
+{
+ private final MigrationManagerEventType type;
+ @Nullable
+ private final InetAddressAndPort endpoint;
+ @Nullable
+ private final UUID endpointSchemaVersion;
+ private final UUID localSchemaVersion;
+ private final Integer localMessagingVersion;
+ private final SystemKeyspace.BootstrapState bootstrapState;
+ @Nullable
+ private Integer inflightTaskCount;
+ @Nullable
+ private Integer endpointMessagingVersion;
+ @Nullable
+ private Boolean endpointGossipOnlyMember;
+ @Nullable
+ private Boolean isAlive;
+
+ enum MigrationManagerEventType
+ {
+ UNKNOWN_LOCAL_SCHEMA_VERSION,
+ VERSION_MATCH,
+ SKIP_PULL,
+ RESET_LOCAL_SCHEMA,
+ TASK_CREATED,
+ TASK_SEND_ABORTED,
+ TASK_REQUEST_SEND
+ }
+
+ SchemaMigrationEvent(MigrationManagerEventType type,
+ @Nullable InetAddressAndPort endpoint, @Nullable UUID endpointSchemaVersion)
+ {
+ this.type = type;
+ this.endpoint = endpoint;
+ this.endpointSchemaVersion = endpointSchemaVersion;
+
+ localSchemaVersion = Schema.instance.getVersion();
+ localMessagingVersion = MessagingService.current_version;
+
+ Queue<CountDownLatch> inflightTasks = MigrationTask.getInflightTasks();
+ if (inflightTasks != null)
+ inflightTaskCount = inflightTasks.size();
+
+ this.bootstrapState = SystemKeyspace.getBootstrapState();
+
+ if (endpoint == null) return;
+
+ if (MessagingService.instance().versions.knows(endpoint))
+ endpointMessagingVersion = MessagingService.instance().versions.getRaw(endpoint);
+
+ endpointGossipOnlyMember = Gossiper.instance.isGossipOnlyMember(endpoint);
+ this.isAlive = FailureDetector.instance.isAlive(endpoint);
+ }
+
+ public Enum<?> getType()
+ {
+ return type;
+ }
+
+ public Map<String, Serializable> toMap()
+ {
+ HashMap<String, Serializable> ret = new HashMap<>();
+ if (endpoint != null) ret.put("endpoint", endpoint.getHostAddress(true));
+ ret.put("endpointSchemaVersion", Schema.schemaVersionToString(endpointSchemaVersion));
+ ret.put("localSchemaVersion", Schema.schemaVersionToString(localSchemaVersion));
+ if (endpointMessagingVersion != null) ret.put("endpointMessagingVersion", endpointMessagingVersion);
+ if (localMessagingVersion != null) ret.put("localMessagingVersion", localMessagingVersion);
+ if (endpointGossipOnlyMember != null) ret.put("endpointGossipOnlyMember", endpointGossipOnlyMember);
+ if (isAlive != null) ret.put("endpointIsAlive", isAlive);
+ if (bootstrapState != null) ret.put("bootstrapState", bootstrapState.name());
+ if (inflightTaskCount != null) ret.put("inflightTaskCount", inflightTaskCount);
+ return ret;
+ }
+}
diff --git a/src/java/org/apache/cassandra/schema/SchemaPullVerbHandler.java b/src/java/org/apache/cassandra/schema/SchemaPullVerbHandler.java
new file mode 100644
index 0000000..ed30792
--- /dev/null
+++ b/src/java/org/apache/cassandra/schema/SchemaPullVerbHandler.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.schema;
+
+import java.util.Collection;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.db.Mutation;
+import org.apache.cassandra.net.IVerbHandler;
+import org.apache.cassandra.net.Message;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.net.NoPayload;
+
+/**
+ * Sends it's current schema state in form of mutations in response to the remote node's request.
+ * Such a request is made when one of the nodes, by means of Gossip, detects schema disagreement in the ring.
+ */
+public final class SchemaPullVerbHandler implements IVerbHandler<NoPayload>
+{
+ public static final SchemaPullVerbHandler instance = new SchemaPullVerbHandler();
+
+ private static final Logger logger = LoggerFactory.getLogger(SchemaPullVerbHandler.class);
+
+ public void doVerb(Message<NoPayload> message)
+ {
+ logger.trace("Received schema pull request from {}", message.from());
+ Message<Collection<Mutation>> response = message.responseWith(SchemaKeyspace.convertSchemaToMutations());
+ MessagingService.instance().send(response, message.from());
+ }
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/schema/SchemaPushVerbHandler.java b/src/java/org/apache/cassandra/schema/SchemaPushVerbHandler.java
new file mode 100644
index 0000000..f2f0faf
--- /dev/null
+++ b/src/java/org/apache/cassandra/schema/SchemaPushVerbHandler.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.schema;
+
+import java.util.Collection;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.concurrent.Stage;
+import org.apache.cassandra.db.Mutation;
+import org.apache.cassandra.net.IVerbHandler;
+import org.apache.cassandra.net.Message;
+
+/**
+ * Called when node receives updated schema state from the schema migration coordinator node.
+ * Such happens when user makes local schema migration on one of the nodes in the ring
+ * (which is going to act as coordinator) and that node sends (pushes) it's updated schema state
+ * (in form of mutations) to all the alive nodes in the cluster.
+ */
+public final class SchemaPushVerbHandler implements IVerbHandler<Collection<Mutation>>
+{
+ public static final SchemaPushVerbHandler instance = new SchemaPushVerbHandler();
+
+ private static final Logger logger = LoggerFactory.getLogger(SchemaPushVerbHandler.class);
+
+ public void doVerb(final Message<Collection<Mutation>> message)
+ {
+ logger.trace("Received schema push request from {}", message.from());
+
+ SchemaAnnouncementDiagnostics.schemataMutationsReceived(message.from());
+ Stage.MIGRATION.submit(() -> Schema.instance.mergeAndAnnounceVersion(message.payload));
+ }
+}
diff --git a/src/java/org/apache/cassandra/schema/SchemaTransformation.java b/src/java/org/apache/cassandra/schema/SchemaTransformation.java
new file mode 100644
index 0000000..c19ac7c
--- /dev/null
+++ b/src/java/org/apache/cassandra/schema/SchemaTransformation.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.schema;
+
+public interface SchemaTransformation
+{
+ /**
+ * Apply a statement transformation to a schema snapshot.
+ *
+ * Implementing methods should be side-effect free.
+ *
+ * @param schema Keyspaces to base the transformation on
+ * @return Keyspaces transformed by the statement
+ */
+ Keyspaces apply(Keyspaces schema);
+}
diff --git a/src/java/org/apache/cassandra/schema/SchemaVersionVerbHandler.java b/src/java/org/apache/cassandra/schema/SchemaVersionVerbHandler.java
new file mode 100644
index 0000000..80090de
--- /dev/null
+++ b/src/java/org/apache/cassandra/schema/SchemaVersionVerbHandler.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.schema;
+
+import java.util.UUID;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.net.IVerbHandler;
+import org.apache.cassandra.net.Message;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.net.NoPayload;
+
+public final class SchemaVersionVerbHandler implements IVerbHandler<NoPayload>
+{
+ public static final SchemaVersionVerbHandler instance = new SchemaVersionVerbHandler();
+
+ private final Logger logger = LoggerFactory.getLogger(SchemaVersionVerbHandler.class);
+
+ public void doVerb(Message<NoPayload> message)
+ {
+ logger.trace("Received schema version request from {}", message.from());
+ Message<UUID> response = message.responseWith(Schema.instance.getVersion());
+ MessagingService.instance().send(response, message.from());
+ }
+}
diff --git a/src/java/org/apache/cassandra/schema/SpeculativeRetryParam.java b/src/java/org/apache/cassandra/schema/SpeculativeRetryParam.java
deleted file mode 100644
index 43447f0..0000000
--- a/src/java/org/apache/cassandra/schema/SpeculativeRetryParam.java
+++ /dev/null
@@ -1,161 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.schema;
-
-import java.text.DecimalFormat;
-import java.util.concurrent.TimeUnit;
-import java.util.Locale;
-
-import com.google.common.base.Objects;
-
-import org.apache.cassandra.exceptions.ConfigurationException;
-
-import static java.lang.String.format;
-
-public final class SpeculativeRetryParam
-{
- public enum Kind
- {
- NONE, CUSTOM, PERCENTILE, ALWAYS
- }
-
- public static final SpeculativeRetryParam NONE = none();
- public static final SpeculativeRetryParam ALWAYS = always();
- public static final SpeculativeRetryParam DEFAULT = percentile(99);
-
- private final Kind kind;
- private final double value;
-
- // pre-processed (divided by 100 for PERCENTILE), multiplied by 1M for CUSTOM (to nanos)
- private final double threshold;
-
- private SpeculativeRetryParam(Kind kind, double value)
- {
- this.kind = kind;
- this.value = value;
-
- if (kind == Kind.PERCENTILE)
- threshold = value / 100;
- else if (kind == Kind.CUSTOM)
- threshold = TimeUnit.MILLISECONDS.toNanos((long) value);
- else
- threshold = value;
- }
-
- public Kind kind()
- {
- return kind;
- }
-
- public double threshold()
- {
- return threshold;
- }
-
- public static SpeculativeRetryParam none()
- {
- return new SpeculativeRetryParam(Kind.NONE, 0);
- }
-
- public static SpeculativeRetryParam always()
- {
- return new SpeculativeRetryParam(Kind.ALWAYS, 0);
- }
-
- public static SpeculativeRetryParam custom(double value)
- {
- return new SpeculativeRetryParam(Kind.CUSTOM, value);
- }
-
- public static SpeculativeRetryParam percentile(double value)
- {
- return new SpeculativeRetryParam(Kind.PERCENTILE, value);
- }
-
- public static SpeculativeRetryParam fromString(String value)
- {
- if (value.toLowerCase(Locale.ENGLISH).endsWith("ms"))
- {
- try
- {
- return custom(Double.parseDouble(value.substring(0, value.length() - "ms".length())));
- }
- catch (IllegalArgumentException e)
- {
- throw new ConfigurationException(format("Invalid value %s for option '%s'", value, TableParams.Option.SPECULATIVE_RETRY));
- }
- }
-
- if (value.toUpperCase(Locale.ENGLISH).endsWith(Kind.PERCENTILE.toString()))
- {
- double threshold;
- try
- {
- threshold = Double.parseDouble(value.substring(0, value.length() - Kind.PERCENTILE.toString().length()));
- }
- catch (IllegalArgumentException e)
- {
- throw new ConfigurationException(format("Invalid value %s for option '%s'", value, TableParams.Option.SPECULATIVE_RETRY));
- }
-
- if (threshold >= 0.0 && threshold <= 100.0)
- return percentile(threshold);
-
- throw new ConfigurationException(format("Invalid value %s for PERCENTILE option '%s': must be between 0.0 and 100.0",
- value,
- TableParams.Option.SPECULATIVE_RETRY));
- }
-
- if (value.equals(Kind.NONE.toString()))
- return NONE;
-
- if (value.equals(Kind.ALWAYS.toString()))
- return ALWAYS;
-
- throw new ConfigurationException(format("Invalid value %s for option '%s'", value, TableParams.Option.SPECULATIVE_RETRY));
- }
-
- @Override
- public boolean equals(Object o)
- {
- if (!(o instanceof SpeculativeRetryParam))
- return false;
- SpeculativeRetryParam srp = (SpeculativeRetryParam) o;
- return kind == srp.kind && threshold == srp.threshold;
- }
-
- @Override
- public int hashCode()
- {
- return Objects.hashCode(kind, threshold);
- }
-
- @Override
- public String toString()
- {
- switch (kind)
- {
- case CUSTOM:
- return format("%sms", value);
- case PERCENTILE:
- return format("%sPERCENTILE", new DecimalFormat("#.#####").format(value));
- default: // NONE and ALWAYS
- return kind.toString();
- }
- }
-}
diff --git a/src/java/org/apache/cassandra/schema/TableId.java b/src/java/org/apache/cassandra/schema/TableId.java
new file mode 100644
index 0000000..695147f
--- /dev/null
+++ b/src/java/org/apache/cassandra/schema/TableId.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.schema;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.UUID;
+
+import org.apache.commons.lang3.ArrayUtils;
+
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.UUIDGen;
+
+/**
+ * The unique identifier of a table.
+ * <p>
+ * This is essentially a UUID, but we wrap it as it's used quite a bit in the code and having a nicely named class make
+ * the code more readable.
+ */
+public class TableId
+{
+ private final UUID id;
+
+ private TableId(UUID id)
+ {
+ this.id = id;
+ }
+
+ public static TableId fromUUID(UUID id)
+ {
+ return new TableId(id);
+ }
+
+ public static TableId generate()
+ {
+ return new TableId(UUIDGen.getTimeUUID());
+ }
+
+ public static TableId fromString(String idString)
+ {
+ return new TableId(UUID.fromString(idString));
+ }
+
+ /**
+ * Creates the UUID of a system table.
+ *
+ * This is deterministically based on the table name as system tables are hardcoded and initialized independently
+ * on each node (they don't go through a CREATE), but we still want them to have the same ID everywhere.
+ *
+ * We shouldn't use this for any other table.
+ */
+ public static TableId forSystemTable(String keyspace, String table)
+ {
+ assert SchemaConstants.isLocalSystemKeyspace(keyspace) || SchemaConstants.isReplicatedSystemKeyspace(keyspace);
+ return new TableId(UUID.nameUUIDFromBytes(ArrayUtils.addAll(keyspace.getBytes(), table.getBytes())));
+ }
+
+ public String toHexString()
+ {
+ return ByteBufferUtil.bytesToHex(ByteBufferUtil.bytes(id));
+ }
+
+ public UUID asUUID()
+ {
+ return id;
+ }
+
+ @Override
+ public final int hashCode()
+ {
+ return id.hashCode();
+ }
+
+ @Override
+ public final boolean equals(Object o)
+ {
+ return this == o || (o instanceof TableId && this.id.equals(((TableId) o).id));
+ }
+
+ @Override
+ public String toString()
+ {
+ return id.toString();
+ }
+
+ public void serialize(DataOutput out) throws IOException
+ {
+ out.writeLong(id.getMostSignificantBits());
+ out.writeLong(id.getLeastSignificantBits());
+ }
+
+ public int serializedSize()
+ {
+ return 16;
+ }
+
+ public static TableId deserialize(DataInput in) throws IOException
+ {
+ return new TableId(new UUID(in.readLong(), in.readLong()));
+ }
+}
diff --git a/src/java/org/apache/cassandra/schema/TableMetadata.java b/src/java/org/apache/cassandra/schema/TableMetadata.java
new file mode 100644
index 0000000..bebb65e
--- /dev/null
+++ b/src/java/org/apache/cassandra/schema/TableMetadata.java
@@ -0,0 +1,1389 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.schema;
+
+import java.nio.ByteBuffer;
+import java.util.*;
+import java.util.Map.Entry;
+
+import javax.annotation.Nullable;
+
+import com.google.common.base.MoreObjects;
+import com.google.common.collect.*;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.auth.DataResource;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.ColumnIdentifier;
+import org.apache.cassandra.cql3.CqlBuilder;
+import org.apache.cassandra.cql3.SchemaElement;
+import org.apache.cassandra.db.*;
+import org.apache.cassandra.db.marshal.*;
+import org.apache.cassandra.dht.IPartitioner;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.service.reads.SpeculativeRetryPolicy;
+import org.apache.cassandra.utils.AbstractIterator;
+import org.github.jamm.Unmetered;
+
+import static com.google.common.collect.Iterables.any;
+import static com.google.common.collect.Iterables.transform;
+import static java.lang.String.format;
+import static java.util.stream.Collectors.toList;
+import static java.util.stream.Collectors.toSet;
+import static org.apache.cassandra.schema.IndexMetadata.isNameValid;
+
+@Unmetered
+public final class TableMetadata implements SchemaElement
+{
+ private static final Logger logger = LoggerFactory.getLogger(TableMetadata.class);
+ private static final ImmutableSet<Flag> DEFAULT_CQL_FLAGS = ImmutableSet.of(Flag.COMPOUND);
+ private static final ImmutableSet<Flag> DEPRECATED_CS_FLAGS = ImmutableSet.of(Flag.DENSE, Flag.SUPER);
+
+ public static final String COMPACT_STORAGE_HALT_MESSAGE =
+ "Compact Tables are not allowed in Cassandra starting with 4.0 version. " +
+ "Use `ALTER ... DROP COMPACT STORAGE` command supplied in 3.x/3.11 Cassandra " +
+ "in order to migrate off Compact Storage.";
+
+ private static final String COMPACT_STORAGE_DEPRECATION_MESSAGE =
+ "Incorrect set of flags is was detected in table {}.{}: '{}'. \n" +
+ "Starting with version 4.0, '{}' flags are deprecated and every table has to have COMPOUND flag. \n" +
+ "Forcing the following set of flags: '{}'";
+
+ public enum Flag
+ {
+ SUPER, COUNTER, DENSE, COMPOUND;
+
+ public static boolean isCQLCompatible(Set<Flag> flags)
+ {
+ return !flags.contains(Flag.DENSE) && !flags.contains(Flag.SUPER) && flags.contains(Flag.COMPOUND);
+ }
+
+ public static Set<Flag> fromStringSet(Set<String> strings)
+ {
+ return strings.stream().map(String::toUpperCase).map(Flag::valueOf).collect(toSet());
+ }
+
+ public static Set<String> toStringSet(Set<Flag> flags)
+ {
+ return flags.stream().map(Flag::toString).map(String::toLowerCase).collect(toSet());
+ }
+ }
+
+ public enum Kind
+ {
+ REGULAR, INDEX, VIEW, VIRTUAL
+ }
+
+ public final String keyspace;
+ public final String name;
+ public final TableId id;
+
+ public final IPartitioner partitioner;
+ public final Kind kind;
+ public final TableParams params;
+ public final ImmutableSet<Flag> flags;
+
+ @Nullable
+ private final String indexName; // derived from table name
+
+ /*
+ * All CQL3 columns definition are stored in the columns map.
+ * On top of that, we keep separated collection of each kind of definition, to
+ * 1) allow easy access to each kind and
+ * 2) for the partition key and clustering key ones, those list are ordered by the "component index" of the elements.
+ */
+ public final ImmutableMap<ByteBuffer, DroppedColumn> droppedColumns;
+ final ImmutableMap<ByteBuffer, ColumnMetadata> columns;
+
+ private final ImmutableList<ColumnMetadata> partitionKeyColumns;
+ private final ImmutableList<ColumnMetadata> clusteringColumns;
+ private final RegularAndStaticColumns regularAndStaticColumns;
+
+ public final Indexes indexes;
+ public final Triggers triggers;
+
+ // derived automatically from flags and columns
+ public final AbstractType<?> partitionKeyType;
+ public final ClusteringComparator comparator;
+
+ /*
+ * For dense tables, this alias the single non-PK column the table contains (since it can only have one). We keep
+ * that as convenience to access that column more easily (but we could replace calls by regularAndStaticColumns().iterator().next()
+ * for those tables in practice).
+ */
+ public final ColumnMetadata compactValueColumn;
+
+ // performance hacks; TODO see if all are really necessary
+ public final DataResource resource;
+
+ private TableMetadata(Builder builder)
+ {
+ if (!Flag.isCQLCompatible(builder.flags))
+ {
+ flags = ImmutableSet.copyOf(Sets.union(Sets.difference(builder.flags, DEPRECATED_CS_FLAGS), DEFAULT_CQL_FLAGS));
+ logger.warn(COMPACT_STORAGE_DEPRECATION_MESSAGE, builder.keyspace, builder.name, builder.flags, DEPRECATED_CS_FLAGS, flags);
+ }
+ else
+ {
+ flags = Sets.immutableEnumSet(builder.flags);
+ }
+ keyspace = builder.keyspace;
+ name = builder.name;
+ id = builder.id;
+
+ partitioner = builder.partitioner;
+ kind = builder.kind;
+ params = builder.params.build();
+
+ indexName = kind == Kind.INDEX ? name.substring(name.indexOf('.') + 1) : null;
+
+ droppedColumns = ImmutableMap.copyOf(builder.droppedColumns);
+ Collections.sort(builder.partitionKeyColumns);
+ partitionKeyColumns = ImmutableList.copyOf(builder.partitionKeyColumns);
+ Collections.sort(builder.clusteringColumns);
+ clusteringColumns = ImmutableList.copyOf(builder.clusteringColumns);
+ regularAndStaticColumns = RegularAndStaticColumns.builder().addAll(builder.regularAndStaticColumns).build();
+ columns = ImmutableMap.copyOf(builder.columns);
+
+ indexes = builder.indexes;
+ triggers = builder.triggers;
+
+ partitionKeyType = partitionKeyColumns.size() == 1
+ ? partitionKeyColumns.get(0).type
+ : CompositeType.getInstance(transform(partitionKeyColumns, t -> t.type));
+
+ comparator = new ClusteringComparator(transform(clusteringColumns, c -> c.type));
+
+ compactValueColumn = isCompactTable()
+ ? CompactTables.getCompactValueColumn(regularAndStaticColumns, isSuper())
+ : null;
+
+ resource = DataResource.table(keyspace, name);
+ }
+
+ public static Builder builder(String keyspace, String table)
+ {
+ return new Builder(keyspace, table);
+ }
+
+ public static Builder builder(String keyspace, String table, TableId id)
+ {
+ return new Builder(keyspace, table, id);
+ }
+
+ public Builder unbuild()
+ {
+ return builder(keyspace, name, id)
+ .partitioner(partitioner)
+ .kind(kind)
+ .params(params)
+ .flags(flags)
+ .addColumns(columns())
+ .droppedColumns(droppedColumns)
+ .indexes(indexes)
+ .triggers(triggers);
+ }
+
+ public boolean isIndex()
+ {
+ return kind == Kind.INDEX;
+ }
+
+ public TableMetadata withSwapped(TableParams params)
+ {
+ return unbuild().params(params).build();
+ }
+
+ public TableMetadata withSwapped(Triggers triggers)
+ {
+ return unbuild().triggers(triggers).build();
+ }
+
+ public TableMetadata withSwapped(Indexes indexes)
+ {
+ return unbuild().indexes(indexes).build();
+ }
+
+ public boolean isView()
+ {
+ return kind == Kind.VIEW;
+ }
+
+ public boolean isVirtual()
+ {
+ return kind == Kind.VIRTUAL;
+ }
+
+ public Optional<String> indexName()
+ {
+ return Optional.ofNullable(indexName);
+ }
+
+ /*
+ * We call dense a CF for which each component of the comparator is a clustering column, i.e. no
+ * component is used to store a regular column names. In other words, non-composite static "thrift"
+ * and CQL3 CF are *not* dense.
+ */
+ public boolean isDense()
+ {
+ return flags.contains(Flag.DENSE);
+ }
+
+ public boolean isCompound()
+ {
+ return flags.contains(Flag.COMPOUND);
+ }
+
+ public boolean isSuper()
+ {
+ return flags.contains(Flag.SUPER);
+ }
+
+ public boolean isCounter()
+ {
+ return flags.contains(Flag.COUNTER);
+ }
+
+ public boolean isCQLTable()
+ {
+ return !isSuper() && !isDense() && isCompound();
+ }
+
+ public boolean isCompactTable()
+ {
+ return !isCQLTable();
+ }
+
+ public boolean isStaticCompactTable()
+ {
+ return !isSuper() && !isDense() && !isCompound();
+ }
+
+ public ImmutableCollection<ColumnMetadata> columns()
+ {
+ return columns.values();
+ }
+
+ public Iterable<ColumnMetadata> primaryKeyColumns()
+ {
+ return Iterables.concat(partitionKeyColumns, clusteringColumns);
+ }
+
+ public ImmutableList<ColumnMetadata> partitionKeyColumns()
+ {
+ return partitionKeyColumns;
+ }
+
+ public ImmutableList<ColumnMetadata> clusteringColumns()
+ {
+ return clusteringColumns;
+ }
+
+ public ImmutableList<ColumnMetadata> createStatementClusteringColumns()
+ {
+ return isStaticCompactTable() ? ImmutableList.of() : clusteringColumns;
+ }
+
+ public RegularAndStaticColumns regularAndStaticColumns()
+ {
+ return regularAndStaticColumns;
+ }
+
+ public Columns regularColumns()
+ {
+ return regularAndStaticColumns.regulars;
+ }
+
+ public Columns staticColumns()
+ {
+ return regularAndStaticColumns.statics;
+ }
+
+ /*
+ * An iterator over all column definitions but that respect the order of a SELECT *.
+ * This also "hide" the clustering/regular columns for a non-CQL3 non-dense table for backward compatibility
+ * sake.
+ */
+ public Iterator<ColumnMetadata> allColumnsInSelectOrder()
+ {
+ boolean isStaticCompactTable = isStaticCompactTable();
+ boolean noNonPkColumns = isCompactTable() && CompactTables.hasEmptyCompactValue(this);
+
+ Iterator<ColumnMetadata> partitionKeyIter = partitionKeyColumns.iterator();
+ Iterator<ColumnMetadata> clusteringIter =
+ isStaticCompactTable ? Collections.emptyIterator() : clusteringColumns.iterator();
+ Iterator<ColumnMetadata> otherColumns =
+ noNonPkColumns
+ ? Collections.emptyIterator()
+ : (isStaticCompactTable ? staticColumns().selectOrderIterator()
+ : regularAndStaticColumns.selectOrderIterator());
+
+ return columnsIterator(partitionKeyIter, clusteringIter, otherColumns);
+ }
+
+ /**
+ * Returns an iterator over all column definitions that respect the order of the CREATE statement.
+ */
+ public Iterator<ColumnMetadata> allColumnsInCreateOrder()
+ {
+ boolean isStaticCompactTable = isStaticCompactTable();
+ boolean noNonPkColumns = isCompactTable() && CompactTables.hasEmptyCompactValue(this);
+
+ Iterator<ColumnMetadata> partitionKeyIter = partitionKeyColumns.iterator();
+ Iterator<ColumnMetadata> clusteringIter = createStatementClusteringColumns().iterator();
+ Iterator<ColumnMetadata> otherColumns =
+ noNonPkColumns
+ ? Collections.emptyIterator()
+ : (isStaticCompactTable ? staticColumns().iterator()
+ : regularAndStaticColumns.iterator());
+
+ return columnsIterator(partitionKeyIter, clusteringIter, otherColumns);
+ }
+
+ private static Iterator<ColumnMetadata> columnsIterator(Iterator<ColumnMetadata> partitionKeys,
+ Iterator<ColumnMetadata> clusteringColumns,
+ Iterator<ColumnMetadata> otherColumns)
+ {
+ return new AbstractIterator<ColumnMetadata>()
+ {
+ protected ColumnMetadata computeNext()
+ {
+ if (partitionKeys.hasNext())
+ return partitionKeys.next();
+
+ if (clusteringColumns.hasNext())
+ return clusteringColumns.next();
+
+ return otherColumns.hasNext() ? otherColumns.next() : endOfData();
+ }
+ };
+ }
+
+ /**
+ * Returns the ColumnMetadata for {@code name}.
+ */
+ public ColumnMetadata getColumn(ColumnIdentifier name)
+ {
+ return columns.get(name.bytes);
+ }
+
+ /*
+ * In general it is preferable to work with ColumnIdentifier to make it
+ * clear that we are talking about a CQL column, not a cell name, but there
+ * is a few cases where all we have is a ByteBuffer (when dealing with IndexExpression
+ * for instance) so...
+ */
+ public ColumnMetadata getColumn(ByteBuffer name)
+ {
+ return columns.get(name);
+ }
+
+ public ColumnMetadata getDroppedColumn(ByteBuffer name)
+ {
+ DroppedColumn dropped = droppedColumns.get(name);
+ return dropped == null ? null : dropped.column;
+ }
+
+ /**
+ * Returns a "fake" ColumnMetadata corresponding to the dropped column {@code name}
+ * of {@code null} if there is no such dropped column.
+ *
+ * @param name - the column name
+ * @param isStatic - whether the column was a static column, if known
+ */
+ public ColumnMetadata getDroppedColumn(ByteBuffer name, boolean isStatic)
+ {
+ DroppedColumn dropped = droppedColumns.get(name);
+ if (dropped == null)
+ return null;
+
+ if (isStatic && !dropped.column.isStatic())
+ return ColumnMetadata.staticColumn(this, name, dropped.column.type);
+
+ return dropped.column;
+ }
+
+ public boolean hasStaticColumns()
+ {
+ return !staticColumns().isEmpty();
+ }
+
+ public void validate()
+ {
+ if (!isNameValid(keyspace))
+ except("Keyspace name must not be empty, more than %s characters long, or contain non-alphanumeric-underscore characters (got \"%s\")", SchemaConstants.NAME_LENGTH, keyspace);
+
+ if (!isNameValid(name))
+ except("Table name must not be empty, more than %s characters long, or contain non-alphanumeric-underscore characters (got \"%s\")", SchemaConstants.NAME_LENGTH, name);
+
+ params.validate();
+
+ if (partitionKeyColumns.stream().anyMatch(c -> c.type.isCounter()))
+ except("PRIMARY KEY columns cannot contain counters");
+
+ // Mixing counter with non counter columns is not supported (#2614)
+ if (isCounter())
+ {
+ for (ColumnMetadata column : regularAndStaticColumns)
+ if (!(column.type.isCounter()) && !CompactTables.isSuperColumnMapColumn(column))
+ except("Cannot have a non counter column (\"%s\") in a counter table", column.name);
+ }
+ else
+ {
+ for (ColumnMetadata column : regularAndStaticColumns)
+ if (column.type.isCounter())
+ except("Cannot have a counter column (\"%s\") in a non counter column table", column.name);
+ }
+
+ // All tables should have a partition key
+ if (partitionKeyColumns.isEmpty())
+ except("Missing partition keys for table %s", toString());
+
+ // A compact table should always have a clustering
+ if (isCompactTable() && clusteringColumns.isEmpty())
+ except("For table %s, isDense=%b, isCompound=%b, clustering=%s", toString(), isDense(), isCompound(), clusteringColumns);
+
+ if (!indexes.isEmpty() && isSuper())
+ except("Secondary indexes are not supported on super column families");
+
+ indexes.validate(this);
+ }
+
+ void validateCompatibility(TableMetadata previous)
+ {
+ if (isIndex())
+ return;
+
+ if (!previous.keyspace.equals(keyspace))
+ except("Keyspace mismatch (found %s; expected %s)", keyspace, previous.keyspace);
+
+ if (!previous.name.equals(name))
+ except("Table mismatch (found %s; expected %s)", name, previous.name);
+
+ if (!previous.id.equals(id))
+ except("Table ID mismatch (found %s; expected %s)", id, previous.id);
+
+ if (!previous.flags.equals(flags))
+ except("Table type mismatch (found %s; expected %s)", flags, previous.flags);
+
+ if (previous.partitionKeyColumns.size() != partitionKeyColumns.size())
+ {
+ except("Partition keys of different length (found %s; expected %s)",
+ partitionKeyColumns.size(),
+ previous.partitionKeyColumns.size());
+ }
+
+ for (int i = 0; i < partitionKeyColumns.size(); i++)
+ {
+ if (!partitionKeyColumns.get(i).type.isCompatibleWith(previous.partitionKeyColumns.get(i).type))
+ {
+ except("Partition key column mismatch (found %s; expected %s)",
+ partitionKeyColumns.get(i).type,
+ previous.partitionKeyColumns.get(i).type);
+ }
+ }
+
+ if (previous.clusteringColumns.size() != clusteringColumns.size())
+ {
+ except("Clustering columns of different length (found %s; expected %s)",
+ clusteringColumns.size(),
+ previous.clusteringColumns.size());
+ }
+
+ for (int i = 0; i < clusteringColumns.size(); i++)
+ {
+ if (!clusteringColumns.get(i).type.isCompatibleWith(previous.clusteringColumns.get(i).type))
+ {
+ except("Clustering column mismatch (found %s; expected %s)",
+ clusteringColumns.get(i).type,
+ previous.clusteringColumns.get(i).type);
+ }
+ }
+
+ for (ColumnMetadata previousColumn : previous.regularAndStaticColumns)
+ {
+ ColumnMetadata column = getColumn(previousColumn.name);
+ if (column != null && !column.type.isCompatibleWith(previousColumn.type))
+ except("Column mismatch (found %s; expected %s)", column, previousColumn);
+ }
+ }
+
+ public ClusteringComparator partitionKeyAsClusteringComparator()
+ {
+ return new ClusteringComparator(partitionKeyColumns.stream().map(c -> c.type).collect(toList()));
+ }
+
+ /**
+ * The type to use to compare column names in "static compact"
+ * tables or superColum ones.
+ * <p>
+ * This exists because for historical reasons, "static compact" tables as
+ * well as super column ones can have non-UTF8 column names.
+ * <p>
+ * This method should only be called for superColumn tables and "static
+ * compact" ones. For any other table, all column names are UTF8.
+ */
+ AbstractType<?> staticCompactOrSuperTableColumnNameType()
+ {
+ if (isSuper())
+ {
+ assert compactValueColumn != null && compactValueColumn.type instanceof MapType;
+ return ((MapType) compactValueColumn.type).nameComparator();
+ }
+
+ assert isStaticCompactTable();
+ return clusteringColumns.get(0).type;
+ }
+
+ public AbstractType<?> columnDefinitionNameComparator(ColumnMetadata.Kind kind)
+ {
+ return (isSuper() && kind == ColumnMetadata.Kind.REGULAR) || (isStaticCompactTable() && kind == ColumnMetadata.Kind.STATIC)
+ ? staticCompactOrSuperTableColumnNameType()
+ : UTF8Type.instance;
+ }
+
+ /**
+ * Generate a table name for an index corresponding to the given column.
+ * This is NOT the same as the index's name! This is only used in sstable filenames and is not exposed to users.
+ *
+ * @param info A definition of the column with index
+ *
+ * @return name of the index table
+ */
+ public String indexTableName(IndexMetadata info)
+ {
+ // TODO simplify this when info.index_name is guaranteed to be set
+ return name + Directories.SECONDARY_INDEX_NAME_SEPARATOR + info.name;
+ }
+
+ /**
+ * @return true if the change as made impacts queries/updates on the table,
+ * e.g. any columns or indexes were added, removed, or altered; otherwise, false is returned.
+ * Used to determine whether prepared statements against this table need to be re-prepared.
+ */
+ boolean changeAffectsPreparedStatements(TableMetadata updated)
+ {
+ return !partitionKeyColumns.equals(updated.partitionKeyColumns)
+ || !clusteringColumns.equals(updated.clusteringColumns)
+ || !regularAndStaticColumns.equals(updated.regularAndStaticColumns)
+ || !indexes.equals(updated.indexes)
+ || params.defaultTimeToLive != updated.params.defaultTimeToLive
+ || params.gcGraceSeconds != updated.params.gcGraceSeconds;
+ }
+
+ /**
+ * There is a couple of places in the code where we need a TableMetadata object and don't have one readily available
+ * and know that only the keyspace and name matter. This creates such "fake" metadata. Use only if you know what
+ * you're doing.
+ */
+ public static TableMetadata minimal(String keyspace, String name)
+ {
+ return TableMetadata.builder(keyspace, name)
+ .addPartitionKeyColumn("key", BytesType.instance)
+ .build();
+ }
+
+ public TableMetadata updateIndexTableMetadata(TableParams baseTableParams)
+ {
+ TableParams.Builder builder = baseTableParams.unbuild().gcGraceSeconds(0);
+
+ // Depends on parent's cache setting, turn on its index table's cache.
+ // Row caching is never enabled; see CASSANDRA-5732
+ builder.caching(baseTableParams.caching.cacheKeys() ? CachingParams.CACHE_KEYS : CachingParams.CACHE_NOTHING);
+
+ return unbuild().params(builder.build()).build();
+ }
+
+ boolean referencesUserType(ByteBuffer name)
+ {
+ return any(columns(), c -> c.type.referencesUserType(name));
+ }
+
+ public TableMetadata withUpdatedUserType(UserType udt)
+ {
+ if (!referencesUserType(udt.name))
+ return this;
+
+ Builder builder = unbuild();
+ columns().forEach(c -> builder.alterColumnType(c.name, c.type.withUpdatedUserType(udt)));
+
+ return builder.build();
+ }
+
+ private void except(String format, Object... args)
+ {
+ throw new ConfigurationException(keyspace + "." + name + ": " + format(format, args));
+ }
+
+ @Override
+ public boolean equals(Object o)
+ {
+ if (this == o)
+ return true;
+
+ if (!(o instanceof TableMetadata))
+ return false;
+
+ TableMetadata tm = (TableMetadata) o;
+
+ return equalsWithoutColumns(tm) && columns.equals(tm.columns);
+ }
+
+ private boolean equalsWithoutColumns(TableMetadata tm)
+ {
+ return keyspace.equals(tm.keyspace)
+ && name.equals(tm.name)
+ && id.equals(tm.id)
+ && partitioner.equals(tm.partitioner)
+ && kind == tm.kind
+ && params.equals(tm.params)
+ && flags.equals(tm.flags)
+ && droppedColumns.equals(tm.droppedColumns)
+ && indexes.equals(tm.indexes)
+ && triggers.equals(tm.triggers);
+ }
+
+ Optional<Difference> compare(TableMetadata other)
+ {
+ return equalsWithoutColumns(other)
+ ? compareColumns(other.columns)
+ : Optional.of(Difference.SHALLOW);
+ }
+
+ private Optional<Difference> compareColumns(Map<ByteBuffer, ColumnMetadata> other)
+ {
+ if (!columns.keySet().equals(other.keySet()))
+ return Optional.of(Difference.SHALLOW);
+
+ boolean differsDeeply = false;
+
+ for (Map.Entry<ByteBuffer, ColumnMetadata> entry : columns.entrySet())
+ {
+ ColumnMetadata thisColumn = entry.getValue();
+ ColumnMetadata thatColumn = other.get(entry.getKey());
+
+ Optional<Difference> difference = thisColumn.compare(thatColumn);
+ if (difference.isPresent())
+ {
+ switch (difference.get())
+ {
+ case SHALLOW:
+ return difference;
+ case DEEP:
+ differsDeeply = true;
+ }
+ }
+ }
+
+ return differsDeeply ? Optional.of(Difference.DEEP) : Optional.empty();
+ }
+
+ @Override
+ public int hashCode()
+ {
+ return Objects.hash(keyspace, name, id, partitioner, kind, params, flags, columns, droppedColumns, indexes, triggers);
+ }
+
+ @Override
+ public String toString()
+ {
+ return String.format("%s.%s", ColumnIdentifier.maybeQuote(keyspace), ColumnIdentifier.maybeQuote(name));
+ }
+
+ public String toDebugString()
+ {
+ return MoreObjects.toStringHelper(this)
+ .add("keyspace", keyspace)
+ .add("table", name)
+ .add("id", id)
+ .add("partitioner", partitioner)
+ .add("kind", kind)
+ .add("params", params)
+ .add("flags", flags)
+ .add("columns", columns())
+ .add("droppedColumns", droppedColumns.values())
+ .add("indexes", indexes)
+ .add("triggers", triggers)
+ .toString();
+ }
+
+ public static final class Builder
+ {
+ final String keyspace;
+ final String name;
+
+ private TableId id;
+
+ private IPartitioner partitioner;
+ private Kind kind = Kind.REGULAR;
+ private TableParams.Builder params = TableParams.builder();
+
+ // Setting compound as default as "normal" CQL tables are compound and that's what we want by default
+ private Set<Flag> flags = EnumSet.of(Flag.COMPOUND);
+ private Triggers triggers = Triggers.none();
+ private Indexes indexes = Indexes.none();
+
+ private final Map<ByteBuffer, DroppedColumn> droppedColumns = new HashMap<>();
+ private final Map<ByteBuffer, ColumnMetadata> columns = new HashMap<>();
+ private final List<ColumnMetadata> partitionKeyColumns = new ArrayList<>();
+ private final List<ColumnMetadata> clusteringColumns = new ArrayList<>();
+ private final List<ColumnMetadata> regularAndStaticColumns = new ArrayList<>();
+
+ private Builder(String keyspace, String name, TableId id)
+ {
+ this.keyspace = keyspace;
+ this.name = name;
+ this.id = id;
+ }
+
+ private Builder(String keyspace, String name)
+ {
+ this.keyspace = keyspace;
+ this.name = name;
+ }
+
+ public TableMetadata build()
+ {
+ if (partitioner == null)
+ partitioner = DatabaseDescriptor.getPartitioner();
+
+ if (id == null)
+ id = TableId.generate();
+
+ return new TableMetadata(this);
+ }
+
+ public Builder id(TableId val)
+ {
+ id = val;
+ return this;
+ }
+
+ public Builder partitioner(IPartitioner val)
+ {
+ partitioner = val;
+ return this;
+ }
+
+ public Builder kind(Kind val)
+ {
+ kind = val;
+ return this;
+ }
+
+ public Builder params(TableParams val)
+ {
+ params = val.unbuild();
+ return this;
+ }
+
+ public Builder bloomFilterFpChance(double val)
+ {
+ params.bloomFilterFpChance(val);
+ return this;
+ }
+
+ public Builder caching(CachingParams val)
+ {
+ params.caching(val);
+ return this;
+ }
+
+ public Builder comment(String val)
+ {
+ params.comment(val);
+ return this;
+ }
+
+ public Builder compaction(CompactionParams val)
+ {
+ params.compaction(val);
+ return this;
+ }
+
+ public Builder compression(CompressionParams val)
+ {
+ params.compression(val);
+ return this;
+ }
+
+ public Builder defaultTimeToLive(int val)
+ {
+ params.defaultTimeToLive(val);
+ return this;
+ }
+
+ public Builder gcGraceSeconds(int val)
+ {
+ params.gcGraceSeconds(val);
+ return this;
+ }
+
+ public Builder maxIndexInterval(int val)
+ {
+ params.maxIndexInterval(val);
+ return this;
+ }
+
+ public Builder memtableFlushPeriod(int val)
+ {
+ params.memtableFlushPeriodInMs(val);
+ return this;
+ }
+
+ public Builder minIndexInterval(int val)
+ {
+ params.minIndexInterval(val);
+ return this;
+ }
+
+ public Builder crcCheckChance(double val)
+ {
+ params.crcCheckChance(val);
+ return this;
+ }
+
+ public Builder speculativeRetry(SpeculativeRetryPolicy val)
+ {
+ params.speculativeRetry(val);
+ return this;
+ }
+
+ public Builder additionalWritePolicy(SpeculativeRetryPolicy val)
+ {
+ params.additionalWritePolicy(val);
+ return this;
+ }
+
+ public Builder extensions(Map<String, ByteBuffer> val)
+ {
+ params.extensions(val);
+ return this;
+ }
+
+ public Builder flags(Set<Flag> val)
+ {
+ flags = val;
+ return this;
+ }
+
+ public Builder isSuper(boolean val)
+ {
+ return flag(Flag.SUPER, val);
+ }
+
+ public Builder isCounter(boolean val)
+ {
+ return flag(Flag.COUNTER, val);
+ }
+
+ public Builder isDense(boolean val)
+ {
+ return flag(Flag.DENSE, val);
+ }
+
+ public Builder isCompound(boolean val)
+ {
+ return flag(Flag.COMPOUND, val);
+ }
+
+ private Builder flag(Flag flag, boolean set)
+ {
+ if (set) flags.add(flag); else flags.remove(flag);
+ return this;
+ }
+
+ public Builder triggers(Triggers val)
+ {
+ triggers = val;
+ return this;
+ }
+
+ public Builder indexes(Indexes val)
+ {
+ indexes = val;
+ return this;
+ }
+
+ public Builder addPartitionKeyColumn(String name, AbstractType type)
+ {
+ return addPartitionKeyColumn(ColumnIdentifier.getInterned(name, false), type);
+ }
+
+ public Builder addPartitionKeyColumn(ColumnIdentifier name, AbstractType type)
+ {
+ return addColumn(new ColumnMetadata(keyspace, this.name, name, type, partitionKeyColumns.size(), ColumnMetadata.Kind.PARTITION_KEY));
+ }
+
+ public Builder addClusteringColumn(String name, AbstractType type)
+ {
+ return addClusteringColumn(ColumnIdentifier.getInterned(name, false), type);
+ }
+
+ public Builder addClusteringColumn(ColumnIdentifier name, AbstractType type)
+ {
+ return addColumn(new ColumnMetadata(keyspace, this.name, name, type, clusteringColumns.size(), ColumnMetadata.Kind.CLUSTERING));
+ }
+
+ public Builder addRegularColumn(String name, AbstractType type)
+ {
+ return addRegularColumn(ColumnIdentifier.getInterned(name, false), type);
+ }
+
+ public Builder addRegularColumn(ColumnIdentifier name, AbstractType type)
+ {
+ return addColumn(new ColumnMetadata(keyspace, this.name, name, type, ColumnMetadata.NO_POSITION, ColumnMetadata.Kind.REGULAR));
+ }
+
+ public Builder addStaticColumn(String name, AbstractType type)
+ {
+ return addStaticColumn(ColumnIdentifier.getInterned(name, false), type);
+ }
+
+ public Builder addStaticColumn(ColumnIdentifier name, AbstractType type)
+ {
+ return addColumn(new ColumnMetadata(keyspace, this.name, name, type, ColumnMetadata.NO_POSITION, ColumnMetadata.Kind.STATIC));
+ }
+
+ public Builder addColumn(ColumnMetadata column)
+ {
+ if (columns.containsKey(column.name.bytes))
+ throw new IllegalArgumentException();
+
+ switch (column.kind)
+ {
+ case PARTITION_KEY:
+ partitionKeyColumns.add(column);
+ Collections.sort(partitionKeyColumns);
+ break;
+ case CLUSTERING:
+ column.type.checkComparable();
+ clusteringColumns.add(column);
+ Collections.sort(clusteringColumns);
+ break;
+ default:
+ regularAndStaticColumns.add(column);
+ }
+
+ columns.put(column.name.bytes, column);
+
+ return this;
+ }
+
+ Builder addColumns(Iterable<ColumnMetadata> columns)
+ {
+ columns.forEach(this::addColumn);
+ return this;
+ }
+
+ public Builder droppedColumns(Map<ByteBuffer, DroppedColumn> droppedColumns)
+ {
+ this.droppedColumns.clear();
+ this.droppedColumns.putAll(droppedColumns);
+ return this;
+ }
+
+ /**
+ * Records a deprecated column for a system table.
+ */
+ public Builder recordDeprecatedSystemColumn(String name, AbstractType<?> type)
+ {
+ // As we play fast and loose with the removal timestamp, make sure this is misued for a non system table.
+ assert SchemaConstants.isLocalSystemKeyspace(keyspace);
+ recordColumnDrop(ColumnMetadata.regularColumn(keyspace, this.name, name, type), Long.MAX_VALUE);
+ return this;
+ }
+
+ public Builder recordColumnDrop(ColumnMetadata column, long timeMicros)
+ {
+ droppedColumns.put(column.name.bytes, new DroppedColumn(column.withNewType(column.type.expandUserTypes()), timeMicros));
+ return this;
+ }
+
+ public Iterable<ColumnMetadata> columns()
+ {
+ return columns.values();
+ }
+
+ public Set<String> columnNames()
+ {
+ return columns.values().stream().map(c -> c.name.toString()).collect(toSet());
+ }
+
+ public ColumnMetadata getColumn(ColumnIdentifier identifier)
+ {
+ return columns.get(identifier.bytes);
+ }
+
+ public ColumnMetadata getColumn(ByteBuffer name)
+ {
+ return columns.get(name);
+ }
+
+ public boolean hasRegularColumns()
+ {
+ return regularAndStaticColumns.stream().anyMatch(ColumnMetadata::isRegular);
+ }
+
+ /*
+ * The following methods all assume a Builder with valid set of partition key, clustering, regular and static columns.
+ */
+
+ public Builder removeRegularOrStaticColumn(ColumnIdentifier identifier)
+ {
+ ColumnMetadata column = columns.get(identifier.bytes);
+ if (column == null || column.isPrimaryKeyColumn())
+ throw new IllegalArgumentException();
+
+ columns.remove(identifier.bytes);
+ regularAndStaticColumns.remove(column);
+
+ return this;
+ }
+
+ public Builder renamePrimaryKeyColumn(ColumnIdentifier from, ColumnIdentifier to)
+ {
+ if (columns.containsKey(to.bytes))
+ throw new IllegalArgumentException();
+
+ ColumnMetadata column = columns.get(from.bytes);
+ if (column == null || !column.isPrimaryKeyColumn())
+ throw new IllegalArgumentException();
+
+ ColumnMetadata newColumn = column.withNewName(to);
+ if (column.isPartitionKey())
+ partitionKeyColumns.set(column.position(), newColumn);
+ else
+ clusteringColumns.set(column.position(), newColumn);
+
+ columns.remove(from.bytes);
+ columns.put(to.bytes, newColumn);
+
+ return this;
+ }
+
+ Builder alterColumnType(ColumnIdentifier name, AbstractType<?> type)
+ {
+ ColumnMetadata column = columns.get(name.bytes);
+ if (column == null)
+ throw new IllegalArgumentException();
+
+ ColumnMetadata newColumn = column.withNewType(type);
+
+ switch (column.kind)
+ {
+ case PARTITION_KEY:
+ partitionKeyColumns.set(column.position(), newColumn);
+ break;
+ case CLUSTERING:
+ clusteringColumns.set(column.position(), newColumn);
+ break;
+ case REGULAR:
+ case STATIC:
+ regularAndStaticColumns.remove(column);
+ regularAndStaticColumns.add(newColumn);
+ break;
+ }
+
+ columns.put(column.name.bytes, newColumn);
+
+ return this;
+ }
+ }
+
+ /**
+ * A table with strict liveness filters/ignores rows without PK liveness info,
+ * effectively tying the row liveness to its primary key liveness.
+ *
+ * Currently this is only used by views with normal base column as PK column
+ * so updates to other columns do not make the row live when the base column
+ * is not live. See CASSANDRA-11500.
+ *
+ * TODO: does not belong here, should be gone
+ */
+ public boolean enforceStrictLiveness()
+ {
+ return isView() && Keyspace.open(keyspace).viewManager.getByName(name).enforceStrictLiveness();
+ }
+
+ /**
+ * Returns the names of all the user types referenced by this table.
+ *
+ * @return the names of all the user types referenced by this table.
+ */
+ public Set<ByteBuffer> getReferencedUserTypes()
+ {
+ Set<ByteBuffer> types = new LinkedHashSet<>();
+ columns().forEach(c -> addUserTypes(c.type, types));
+ return types;
+ }
+
+ /**
+ * Find all user types used by the specified type and add them to the set.
+ *
+ * @param type the type to check for user types.
+ * @param types the set of UDT names to which to add new user types found in {@code type}. Note that the
+ * insertion ordering is important and ensures that if a user type A uses another user type B, then B will appear
+ * before A in iteration order.
+ */
+ private static void addUserTypes(AbstractType<?> type, Set<ByteBuffer> types)
+ {
+ // Reach into subtypes first, so that if the type is a UDT, it's dependencies are recreated first.
+ type.subTypes().forEach(t -> addUserTypes(t, types));
+
+ if (type.isUDT())
+ types.add(((UserType)type).name);
+ }
+
+ @Override
+ public SchemaElementType elementType()
+ {
+ return SchemaElementType.TABLE;
+ }
+
+ @Override
+ public String elementKeyspace()
+ {
+ return keyspace;
+ }
+
+ @Override
+ public String elementName()
+ {
+ return name;
+ }
+
+ @Override
+ public String toCqlString(boolean withInternals)
+ {
+ CqlBuilder builder = new CqlBuilder(2048);
+ appendCqlTo(builder, withInternals, withInternals, false);
+ return builder.toString();
+ }
+
+ public String toCqlString(boolean includeDroppedColumns,
+ boolean internals,
+ boolean ifNotExists)
+ {
+ CqlBuilder builder = new CqlBuilder(2048);
+ appendCqlTo(builder, includeDroppedColumns, internals, ifNotExists);
+ return builder.toString();
+ }
+
+ public void appendCqlTo(CqlBuilder builder,
+ boolean includeDroppedColumns,
+ boolean internals,
+ boolean ifNotExists)
+ {
+ assert !isView();
+
+ String createKeyword = "CREATE";
+ if (!isCQLTable())
+ {
+ builder.append("/*")
+ .newLine()
+ .append("Warning: Table ")
+ .append(toString())
+ .append(" omitted because it has constructs not compatible with CQL (was created via legacy API).")
+ .newLine()
+ .append("Approximate structure, for reference:")
+ .newLine()
+ .append("(this should not be used to reproduce this schema)")
+ .newLine()
+ .newLine();
+ }
+ else if (isVirtual())
+ {
+ builder.append(String.format("/*\n" +
+ "Warning: Table %s is a virtual table and cannot be recreated with CQL.\n" +
+ "Structure, for reference:\n",
+ toString()));
+ createKeyword = "VIRTUAL";
+ }
+
+ builder.append(createKeyword)
+ .append(" TABLE ");
+
+ if (ifNotExists)
+ builder.append("IF NOT EXISTS ");
+
+ builder.append(toString())
+ .append(" (")
+ .newLine()
+ .increaseIndent();
+
+ boolean hasSingleColumnPrimaryKey = partitionKeyColumns.size() == 1 && clusteringColumns.isEmpty();
+
+ appendColumnDefinitions(builder, includeDroppedColumns, hasSingleColumnPrimaryKey);
+
+ if (!hasSingleColumnPrimaryKey)
+ appendPrimaryKey(builder);
+
+ builder.decreaseIndent()
+ .append(')');
+
+ appendTableOptions(builder, internals);
+
+ builder.decreaseIndent();
+
+ if (!isCQLTable() || isVirtual())
+ {
+ builder.newLine()
+ .append("*/");
+ }
+
+ if (includeDroppedColumns)
+ appendDropColumns(builder);
+ }
+
+ private void appendColumnDefinitions(CqlBuilder builder,
+ boolean includeDroppedColumns,
+ boolean hasSingleColumnPrimaryKey)
+ {
+ Iterator<ColumnMetadata> iter = allColumnsInCreateOrder();
+ while (iter.hasNext())
+ {
+ ColumnMetadata column = iter.next();
+
+ // If the column has been re-added after a drop, we don't include it right away. Instead, we'll add the
+ // dropped one first below, then we'll issue the DROP and then the actual ADD for this column, thus
+ // simulating the proper sequence of events.
+ if (includeDroppedColumns && droppedColumns.containsKey(column.name.bytes))
+ continue;
+
+ column.appendCqlTo(builder, isStaticCompactTable());
+
+ if (hasSingleColumnPrimaryKey && column.isPartitionKey())
+ builder.append(" PRIMARY KEY");
+
+ if (!hasSingleColumnPrimaryKey || (includeDroppedColumns && !droppedColumns.isEmpty()) || iter.hasNext())
+ builder.append(',');
+
+ builder.newLine();
+ }
+
+ if (includeDroppedColumns)
+ {
+ Iterator<DroppedColumn> iterDropped = droppedColumns.values().iterator();
+ while (iterDropped.hasNext())
+ {
+ DroppedColumn dropped = iterDropped.next();
+ dropped.column.appendCqlTo(builder, isStaticCompactTable());
+
+ if (!hasSingleColumnPrimaryKey || iter.hasNext())
+ builder.append(',');
+
+ builder.newLine();
+ }
+ }
+ }
+
+ void appendPrimaryKey(CqlBuilder builder)
+ {
+ List<ColumnMetadata> partitionKeyColumns = partitionKeyColumns();
+ List<ColumnMetadata> clusteringColumns = createStatementClusteringColumns();
+
+ builder.append("PRIMARY KEY (");
+ if (partitionKeyColumns.size() > 1)
+ {
+ builder.append('(')
+ .appendWithSeparators(partitionKeyColumns, (b, c) -> b.append(c.name), ", ")
+ .append(')');
+ }
+ else
+ {
+ builder.append(partitionKeyColumns.get(0).name);
+ }
+
+ if (!clusteringColumns.isEmpty())
+ builder.append(", ")
+ .appendWithSeparators(clusteringColumns, (b, c) -> b.append(c.name), ", ");
+
+ builder.append(')')
+ .newLine();
+ }
+
+ void appendTableOptions(CqlBuilder builder, boolean internals)
+ {
+ builder.append(" WITH ")
+ .increaseIndent();
+
+ if (internals)
+ builder.append("ID = ")
+ .append(id.toString())
+ .newLine()
+ .append("AND ");
+
+ if (isCompactTable())
+ builder.append("COMPACT STORAGE")
+ .newLine()
+ .append("AND ");
+
+ List<ColumnMetadata> clusteringColumns = createStatementClusteringColumns();
+ if (!clusteringColumns.isEmpty())
+ {
+ builder.append("CLUSTERING ORDER BY (")
+ .appendWithSeparators(clusteringColumns, (b, c) -> c.appendNameAndOrderTo(b), ", ")
+ .append(')')
+ .newLine()
+ .append("AND ");
+ }
+
+ if (isVirtual())
+ {
+ builder.append("comment = ").appendWithSingleQuotes(params.comment);
+ }
+ else
+ {
+ params.appendCqlTo(builder);
+ }
+ builder.append(";");
+ }
+
+ private void appendDropColumns(CqlBuilder builder)
+ {
+ for (Entry<ByteBuffer, DroppedColumn> entry : droppedColumns.entrySet())
+ {
+ DroppedColumn dropped = entry.getValue();
+
+ builder.newLine()
+ .append("ALTER TABLE ")
+ .append(toString())
+ .append(" DROP ")
+ .append(dropped.column.name)
+ .append(" USING TIMESTAMP ")
+ .append(dropped.droppedTime)
+ .append(';');
+
+ ColumnMetadata column = getColumn(entry.getKey());
+ if (column != null)
+ {
+ builder.newLine()
+ .append("ALTER TABLE ")
+ .append(toString())
+ .append(" ADD ");
+
+ column.appendCqlTo(builder, false);
+
+ builder.append(';');
+ }
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/schema/TableMetadataRef.java b/src/java/org/apache/cassandra/schema/TableMetadataRef.java
new file mode 100644
index 0000000..3c45594
--- /dev/null
+++ b/src/java/org/apache/cassandra/schema/TableMetadataRef.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.schema;
+
+import org.github.jamm.Unmetered;
+
+/**
+ * Encapsulates a volatile reference to an immutable {@link TableMetadata} instance.
+ *
+ * Used in classes that need up-to-date metadata to avoid the cost of looking up {@link Schema} hashmaps.
+ */
+@Unmetered
+public final class TableMetadataRef
+{
+ public final TableId id;
+ public final String keyspace;
+ public final String name;
+
+ private volatile TableMetadata metadata;
+
+ TableMetadataRef(TableMetadata metadata)
+ {
+ this.metadata = metadata;
+
+ id = metadata.id;
+ keyspace = metadata.keyspace;
+ name = metadata.name;
+ }
+
+ /**
+ * Create a new ref to the passed {@link TableMetadata} for use by offline tools only.
+ *
+ * @param metadata {@link TableMetadata} to reference
+ * @return a new TableMetadataRef instance linking to the passed {@link TableMetadata}
+ */
+ public static TableMetadataRef forOfflineTools(TableMetadata metadata)
+ {
+ return new TableMetadataRef(metadata);
+ }
+
+ public TableMetadata get()
+ {
+ return metadata;
+ }
+
+ /**
+ * Update the reference with the most current version of {@link TableMetadata}
+ * <p>
+ * Must only be used by methods in {@link Schema}, *DO NOT* make public
+ * even for testing purposes, it isn't safe.
+ */
+ void set(TableMetadata metadata)
+ {
+ metadata.validateCompatibility(get());
+ this.metadata = metadata;
+ }
+
+ @Override
+ public String toString()
+ {
+ return get().toString();
+ }
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/schema/TableParams.java b/src/java/org/apache/cassandra/schema/TableParams.java
index 3750aa1..c99a2f3 100644
--- a/src/java/org/apache/cassandra/schema/TableParams.java
+++ b/src/java/org/apache/cassandra/schema/TableParams.java
@@ -19,16 +19,23 @@
import java.nio.ByteBuffer;
import java.util.Map;
+import java.util.Map.Entry;
import com.google.common.base.MoreObjects;
import com.google.common.base.Objects;
import com.google.common.collect.ImmutableMap;
import org.apache.cassandra.cql3.Attributes;
+import org.apache.cassandra.cql3.CqlBuilder;
import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.service.reads.PercentileSpeculativeRetryPolicy;
+import org.apache.cassandra.service.reads.SpeculativeRetryPolicy;
+import org.apache.cassandra.service.reads.repair.ReadRepairStrategy;
import org.apache.cassandra.utils.BloomCalculations;
+import org.apache.cassandra.utils.ByteBufferUtil;
import static java.lang.String.format;
+import static java.util.stream.Collectors.toMap;
public final class TableParams
{
@@ -41,17 +48,17 @@
COMMENT,
COMPACTION,
COMPRESSION,
- DCLOCAL_READ_REPAIR_CHANCE,
DEFAULT_TIME_TO_LIVE,
EXTENSIONS,
GC_GRACE_SECONDS,
MAX_INDEX_INTERVAL,
MEMTABLE_FLUSH_PERIOD_IN_MS,
MIN_INDEX_INTERVAL,
- READ_REPAIR_CHANCE,
SPECULATIVE_RETRY,
+ ADDITIONAL_WRITE_POLICY,
CRC_CHECK_CHANCE,
- CDC;
+ CDC,
+ READ_REPAIR;
@Override
public String toString()
@@ -60,19 +67,7 @@
}
}
- public static final String DEFAULT_COMMENT = "";
- public static final double DEFAULT_READ_REPAIR_CHANCE = 0.0;
- public static final double DEFAULT_DCLOCAL_READ_REPAIR_CHANCE = 0.1;
- public static final int DEFAULT_GC_GRACE_SECONDS = 864000; // 10 days
- public static final int DEFAULT_DEFAULT_TIME_TO_LIVE = 0;
- public static final int DEFAULT_MEMTABLE_FLUSH_PERIOD_IN_MS = 0;
- public static final int DEFAULT_MIN_INDEX_INTERVAL = 128;
- public static final int DEFAULT_MAX_INDEX_INTERVAL = 2048;
- public static final double DEFAULT_CRC_CHECK_CHANCE = 1.0;
-
public final String comment;
- public final double readRepairChance;
- public final double dcLocalReadRepairChance;
public final double bloomFilterFpChance;
public final double crcCheckChance;
public final int gcGraceSeconds;
@@ -80,18 +75,18 @@
public final int memtableFlushPeriodInMs;
public final int minIndexInterval;
public final int maxIndexInterval;
- public final SpeculativeRetryParam speculativeRetry;
+ public final SpeculativeRetryPolicy speculativeRetry;
+ public final SpeculativeRetryPolicy additionalWritePolicy;
public final CachingParams caching;
public final CompactionParams compaction;
public final CompressionParams compression;
public final ImmutableMap<String, ByteBuffer> extensions;
public final boolean cdc;
+ public final ReadRepairStrategy readRepair;
private TableParams(Builder builder)
{
comment = builder.comment;
- readRepairChance = builder.readRepairChance;
- dcLocalReadRepairChance = builder.dcLocalReadRepairChance;
bloomFilterFpChance = builder.bloomFilterFpChance == null
? builder.compaction.defaultBloomFilterFbChance()
: builder.bloomFilterFpChance;
@@ -102,11 +97,13 @@
minIndexInterval = builder.minIndexInterval;
maxIndexInterval = builder.maxIndexInterval;
speculativeRetry = builder.speculativeRetry;
+ additionalWritePolicy = builder.additionalWritePolicy;
caching = builder.caching;
compaction = builder.compaction;
compression = builder.compression;
extensions = builder.extensions;
cdc = builder.cdc;
+ readRepair = builder.readRepair;
}
public static Builder builder()
@@ -121,17 +118,22 @@
.comment(params.comment)
.compaction(params.compaction)
.compression(params.compression)
- .dcLocalReadRepairChance(params.dcLocalReadRepairChance)
.crcCheckChance(params.crcCheckChance)
.defaultTimeToLive(params.defaultTimeToLive)
.gcGraceSeconds(params.gcGraceSeconds)
.maxIndexInterval(params.maxIndexInterval)
.memtableFlushPeriodInMs(params.memtableFlushPeriodInMs)
.minIndexInterval(params.minIndexInterval)
- .readRepairChance(params.readRepairChance)
.speculativeRetry(params.speculativeRetry)
+ .additionalWritePolicy(params.additionalWritePolicy)
.extensions(params.extensions)
- .cdc(params.cdc);
+ .cdc(params.cdc)
+ .readRepair(params.readRepair);
+ }
+
+ public Builder unbuild()
+ {
+ return builder(this);
}
public void validate()
@@ -148,20 +150,6 @@
bloomFilterFpChance);
}
- if (dcLocalReadRepairChance < 0 || dcLocalReadRepairChance > 1.0)
- {
- fail("%s must be larger than or equal to 0 and smaller than or equal to 1.0 (got %s)",
- Option.DCLOCAL_READ_REPAIR_CHANCE,
- dcLocalReadRepairChance);
- }
-
- if (readRepairChance < 0 || readRepairChance > 1.0)
- {
- fail("%s must be larger than or equal to 0 and smaller than or equal to 1.0 (got %s)",
- Option.READ_REPAIR_CHANCE,
- readRepairChance);
- }
-
if (crcCheckChance < 0 || crcCheckChance > 1.0)
{
fail("%s must be larger than or equal to 0 and smaller than or equal to 1.0 (got %s)",
@@ -211,8 +199,6 @@
TableParams p = (TableParams) o;
return comment.equals(p.comment)
- && readRepairChance == p.readRepairChance
- && dcLocalReadRepairChance == p.dcLocalReadRepairChance
&& bloomFilterFpChance == p.bloomFilterFpChance
&& crcCheckChance == p.crcCheckChance
&& gcGraceSeconds == p.gcGraceSeconds
@@ -225,15 +211,14 @@
&& compaction.equals(p.compaction)
&& compression.equals(p.compression)
&& extensions.equals(p.extensions)
- && cdc == p.cdc;
+ && cdc == p.cdc
+ && readRepair == p.readRepair;
}
@Override
public int hashCode()
{
return Objects.hashCode(comment,
- readRepairChance,
- dcLocalReadRepairChance,
bloomFilterFpChance,
crcCheckChance,
gcGraceSeconds,
@@ -246,7 +231,8 @@
compaction,
compression,
extensions,
- cdc);
+ cdc,
+ readRepair);
}
@Override
@@ -254,8 +240,6 @@
{
return MoreObjects.toStringHelper(this)
.add(Option.COMMENT.toString(), comment)
- .add(Option.READ_REPAIR_CHANCE.toString(), readRepairChance)
- .add(Option.DCLOCAL_READ_REPAIR_CHANCE.toString(), dcLocalReadRepairChance)
.add(Option.BLOOM_FILTER_FP_CHANCE.toString(), bloomFilterFpChance)
.add(Option.CRC_CHECK_CHANCE.toString(), crcCheckChance)
.add(Option.GC_GRACE_SECONDS.toString(), gcGraceSeconds)
@@ -269,27 +253,69 @@
.add(Option.COMPRESSION.toString(), compression)
.add(Option.EXTENSIONS.toString(), extensions)
.add(Option.CDC.toString(), cdc)
+ .add(Option.READ_REPAIR.toString(), readRepair)
.toString();
}
+ public void appendCqlTo(CqlBuilder builder)
+ {
+ // option names should be in alphabetical order
+ builder.append("additional_write_policy = ").appendWithSingleQuotes(additionalWritePolicy.toString())
+ .newLine()
+ .append("AND bloom_filter_fp_chance = ").append(bloomFilterFpChance)
+ .newLine()
+ .append("AND caching = ").append(caching.asMap())
+ .newLine()
+ .append("AND cdc = ").append(cdc)
+ .newLine()
+ .append("AND comment = ").appendWithSingleQuotes(comment)
+ .newLine()
+ .append("AND compaction = ").append(compaction.asMap())
+ .newLine()
+ .append("AND compression = ").append(compression.asMap())
+ .newLine()
+ .append("AND crc_check_chance = ").append(crcCheckChance)
+ .newLine()
+ .append("AND default_time_to_live = ").append(defaultTimeToLive)
+ .newLine()
+ .append("AND extensions = ").append(extensions.entrySet()
+ .stream()
+ .collect(toMap(Entry::getKey,
+ e -> "0x" + ByteBufferUtil.bytesToHex(e.getValue()))),
+ false)
+ .newLine()
+ .append("AND gc_grace_seconds = ").append(gcGraceSeconds)
+ .newLine()
+ .append("AND max_index_interval = ").append(maxIndexInterval)
+ .newLine()
+ .append("AND memtable_flush_period_in_ms = ").append(memtableFlushPeriodInMs)
+ .newLine()
+ .append("AND min_index_interval = ").append(minIndexInterval)
+ .newLine()
+ .append("AND read_repair = ").appendWithSingleQuotes(readRepair.toString())
+ .newLine()
+ .append("AND speculative_retry = ").appendWithSingleQuotes(speculativeRetry.toString());
+
+ }
+
public static final class Builder
{
- private String comment = DEFAULT_COMMENT;
- private double readRepairChance = DEFAULT_READ_REPAIR_CHANCE;
- private double dcLocalReadRepairChance = DEFAULT_DCLOCAL_READ_REPAIR_CHANCE;
+ private String comment = "";
private Double bloomFilterFpChance;
- public Double crcCheckChance = DEFAULT_CRC_CHECK_CHANCE;
- private int gcGraceSeconds = DEFAULT_GC_GRACE_SECONDS;
- private int defaultTimeToLive = DEFAULT_DEFAULT_TIME_TO_LIVE;
- private int memtableFlushPeriodInMs = DEFAULT_MEMTABLE_FLUSH_PERIOD_IN_MS;
- private int minIndexInterval = DEFAULT_MIN_INDEX_INTERVAL;
- private int maxIndexInterval = DEFAULT_MAX_INDEX_INTERVAL;
- private SpeculativeRetryParam speculativeRetry = SpeculativeRetryParam.DEFAULT;
+ private double crcCheckChance = 1.0;
+ private int gcGraceSeconds = 864000; // 10 days
+ private int defaultTimeToLive = 0;
+ private int memtableFlushPeriodInMs = 0;
+ private int minIndexInterval = 128;
+ private int maxIndexInterval = 2048;
+ private SpeculativeRetryPolicy speculativeRetry = PercentileSpeculativeRetryPolicy.NINETY_NINE_P;
+ private SpeculativeRetryPolicy additionalWritePolicy = PercentileSpeculativeRetryPolicy.NINETY_NINE_P;
private CachingParams caching = CachingParams.DEFAULT;
private CompactionParams compaction = CompactionParams.DEFAULT;
private CompressionParams compression = CompressionParams.DEFAULT;
private ImmutableMap<String, ByteBuffer> extensions = ImmutableMap.of();
private boolean cdc;
+ private ReadRepairStrategy readRepair = ReadRepairStrategy.BLOCKING;
public Builder()
{
@@ -306,18 +332,6 @@
return this;
}
- public Builder readRepairChance(double val)
- {
- readRepairChance = val;
- return this;
- }
-
- public Builder dcLocalReadRepairChance(double val)
- {
- dcLocalReadRepairChance = val;
- return this;
- }
-
public Builder bloomFilterFpChance(double val)
{
bloomFilterFpChance = val;
@@ -360,12 +374,18 @@
return this;
}
- public Builder speculativeRetry(SpeculativeRetryParam val)
+ public Builder speculativeRetry(SpeculativeRetryPolicy val)
{
speculativeRetry = val;
return this;
}
+ public Builder additionalWritePolicy(SpeculativeRetryPolicy val)
+ {
+ additionalWritePolicy = val;
+ return this;
+ }
+
public Builder caching(CachingParams val)
{
caching = val;
@@ -390,6 +410,12 @@
return this;
}
+ public Builder readRepair(ReadRepairStrategy val)
+ {
+ readRepair = val;
+ return this;
+ }
+
public Builder extensions(Map<String, ByteBuffer> val)
{
extensions = ImmutableMap.copyOf(val);
diff --git a/src/java/org/apache/cassandra/schema/Tables.java b/src/java/org/apache/cassandra/schema/Tables.java
index 4f728d4..33ed1b8 100644
--- a/src/java/org/apache/cassandra/schema/Tables.java
+++ b/src/java/org/apache/cassandra/schema/Tables.java
@@ -17,29 +17,41 @@
*/
package org.apache.cassandra.schema;
+import java.nio.ByteBuffer;
+import java.util.HashMap;
import java.util.Iterator;
+import java.util.Map;
import java.util.Optional;
+import java.util.function.Predicate;
+import java.util.stream.Stream;
+import java.util.stream.StreamSupport;
import javax.annotation.Nullable;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.MapDifference;
-import com.google.common.collect.Maps;
+import com.google.common.collect.*;
-import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.db.marshal.UserType;
+import org.apache.cassandra.index.internal.CassandraIndex;
-import static com.google.common.collect.Iterables.filter;
+import static com.google.common.collect.Iterables.any;
+import static com.google.common.collect.Iterables.transform;
/**
* An immutable container for a keyspace's Tables.
*/
-public final class Tables implements Iterable<CFMetaData>
+public final class Tables implements Iterable<TableMetadata>
{
- private final ImmutableMap<String, CFMetaData> tables;
+ private static final Tables NONE = builder().build();
+
+ private final ImmutableMap<String, TableMetadata> tables;
+ private final ImmutableMap<TableId, TableMetadata> tablesById;
+ private final ImmutableMap<String, TableMetadata> indexTables;
private Tables(Builder builder)
{
tables = builder.tables.build();
+ tablesById = builder.tablesById.build();
+ indexTables = builder.indexTables.build();
}
public static Builder builder()
@@ -49,24 +61,39 @@
public static Tables none()
{
- return builder().build();
+ return NONE;
}
- public static Tables of(CFMetaData... tables)
+ public static Tables of(TableMetadata... tables)
{
return builder().add(tables).build();
}
- public static Tables of(Iterable<CFMetaData> tables)
+ public static Tables of(Iterable<TableMetadata> tables)
{
return builder().add(tables).build();
}
- public Iterator<CFMetaData> iterator()
+ public Iterator<TableMetadata> iterator()
{
return tables.values().iterator();
}
+ public Stream<TableMetadata> stream()
+ {
+ return StreamSupport.stream(spliterator(), false);
+ }
+
+ public Iterable<TableMetadata> referencingUserType(ByteBuffer name)
+ {
+ return Iterables.filter(tables.values(), t -> t.referencesUserType(name));
+ }
+
+ ImmutableMap<String, TableMetadata> indexTables()
+ {
+ return indexTables;
+ }
+
public int size()
{
return tables.size();
@@ -76,9 +103,9 @@
* Get the table with the specified name
*
* @param name a non-qualified table name
- * @return an empty {@link Optional} if the table name is not found; a non-empty optional of {@link CFMetaData} otherwise
+ * @return an empty {@link Optional} if the table name is not found; a non-empty optional of {@link TableMetadataRef} otherwise
*/
- public Optional<CFMetaData> get(String name)
+ public Optional<TableMetadata> get(String name)
{
return Optional.ofNullable(tables.get(name));
}
@@ -87,39 +114,80 @@
* Get the table with the specified name
*
* @param name a non-qualified table name
- * @return null if the table name is not found; the found {@link CFMetaData} otherwise
+ * @return null if the table name is not found; the found {@link TableMetadataRef} otherwise
*/
@Nullable
- public CFMetaData getNullable(String name)
+ public TableMetadata getNullable(String name)
{
return tables.get(name);
}
+ @Nullable
+ TableMetadata getNullable(TableId id)
+ {
+ return tablesById.get(id);
+ }
+
+ boolean containsTable(TableId id)
+ {
+ return tablesById.containsKey(id);
+ }
+
+ public Tables filter(Predicate<TableMetadata> predicate)
+ {
+ Builder builder = builder();
+ tables.values().stream().filter(predicate).forEach(builder::add);
+ return builder.build();
+ }
+
/**
* Create a Tables instance with the provided table added
*/
- public Tables with(CFMetaData table)
+ public Tables with(TableMetadata table)
{
- if (get(table.cfName).isPresent())
- throw new IllegalStateException(String.format("Table %s already exists", table.cfName));
+ if (get(table.name).isPresent())
+ throw new IllegalStateException(String.format("Table %s already exists", table.name));
return builder().add(this).add(table).build();
}
+ public Tables withSwapped(TableMetadata table)
+ {
+ return without(table.name).with(table);
+ }
+
/**
* Creates a Tables instance with the table with the provided name removed
*/
public Tables without(String name)
{
- CFMetaData table =
+ TableMetadata table =
get(name).orElseThrow(() -> new IllegalStateException(String.format("Table %s doesn't exists", name)));
- return builder().add(filter(this, t -> t != table)).build();
+ return without(table);
}
- MapDifference<String, CFMetaData> diff(Tables other)
+ public Tables without(TableMetadata table)
{
- return Maps.difference(tables, other.tables);
+ return filter(t -> t != table);
+ }
+
+ public Tables withUpdatedUserType(UserType udt)
+ {
+ return any(this, t -> t.referencesUserType(udt.name))
+ ? builder().add(transform(this, t -> t.withUpdatedUserType(udt))).build()
+ : this;
+ }
+
+ MapDifference<String, TableMetadata> indexesDiff(Tables other)
+ {
+ Map<String, TableMetadata> thisIndexTables = new HashMap<>();
+ this.indexTables.values().forEach(t -> thisIndexTables.put(t.indexName().get(), t));
+
+ Map<String, TableMetadata> otherIndexTables = new HashMap<>();
+ other.indexTables.values().forEach(t -> otherIndexTables.put(t.indexName().get(), t));
+
+ return Maps.difference(thisIndexTables, otherIndexTables);
}
@Override
@@ -142,7 +210,9 @@
public static final class Builder
{
- final ImmutableMap.Builder<String, CFMetaData> tables = new ImmutableMap.Builder<>();
+ final ImmutableMap.Builder<String, TableMetadata> tables = new ImmutableMap.Builder<>();
+ final ImmutableMap.Builder<TableId, TableMetadata> tablesById = new ImmutableMap.Builder<>();
+ final ImmutableMap.Builder<String, TableMetadata> indexTables = new ImmutableMap.Builder<>();
private Builder()
{
@@ -153,23 +223,66 @@
return new Tables(this);
}
- public Builder add(CFMetaData table)
+ public Builder add(TableMetadata table)
{
- tables.put(table.cfName, table);
+ tables.put(table.name, table);
+
+ tablesById.put(table.id, table);
+
+ table.indexes
+ .stream()
+ .filter(i -> !i.isCustom())
+ .map(i -> CassandraIndex.indexCfsMetadata(table, i))
+ .forEach(i -> indexTables.put(i.indexName().get(), i));
+
return this;
}
- public Builder add(CFMetaData... tables)
+ public Builder add(TableMetadata... tables)
{
- for (CFMetaData table : tables)
+ for (TableMetadata table : tables)
add(table);
return this;
}
- public Builder add(Iterable<CFMetaData> tables)
+ public Builder add(Iterable<TableMetadata> tables)
{
tables.forEach(this::add);
return this;
}
}
+
+ static TablesDiff diff(Tables before, Tables after)
+ {
+ return TablesDiff.diff(before, after);
+ }
+
+ public static final class TablesDiff extends Diff<Tables, TableMetadata>
+ {
+ private final static TablesDiff NONE = new TablesDiff(Tables.none(), Tables.none(), ImmutableList.of());
+
+ private TablesDiff(Tables created, Tables dropped, ImmutableCollection<Altered<TableMetadata>> altered)
+ {
+ super(created, dropped, altered);
+ }
+
+ private static TablesDiff diff(Tables before, Tables after)
+ {
+ if (before == after)
+ return NONE;
+
+ Tables created = after.filter(t -> !before.containsTable(t.id));
+ Tables dropped = before.filter(t -> !after.containsTable(t.id));
+
+ ImmutableList.Builder<Altered<TableMetadata>> altered = ImmutableList.builder();
+ before.forEach(tableBefore ->
+ {
+ TableMetadata tableAfter = after.getNullable(tableBefore.id);
+ if (null != tableAfter)
+ tableBefore.compare(tableAfter).ifPresent(kind -> altered.add(new Altered<>(tableBefore, tableAfter, kind)));
+ });
+
+ return new TablesDiff(created, dropped, altered.build());
+ }
+ }
}
diff --git a/src/java/org/apache/cassandra/schema/Triggers.java b/src/java/org/apache/cassandra/schema/Triggers.java
index bb39f1f..5e10722 100644
--- a/src/java/org/apache/cassandra/schema/Triggers.java
+++ b/src/java/org/apache/cassandra/schema/Triggers.java
@@ -43,6 +43,16 @@
return builder().build();
}
+ public static Triggers of(TriggerMetadata... triggers)
+ {
+ return builder().add(triggers).build();
+ }
+
+ public static Triggers of(Iterable<TriggerMetadata> triggers)
+ {
+ return builder().add(triggers).build();
+ }
+
public Iterator<TriggerMetadata> iterator()
{
return triggers.values().iterator();
@@ -128,6 +138,13 @@
return this;
}
+ public Builder add(TriggerMetadata... triggers)
+ {
+ for (TriggerMetadata trigger : triggers)
+ add(trigger);
+ return this;
+ }
+
public Builder add(Iterable<TriggerMetadata> triggers)
{
triggers.forEach(this::add);
diff --git a/src/java/org/apache/cassandra/schema/Types.java b/src/java/org/apache/cassandra/schema/Types.java
index c2d8aac..76694cc 100644
--- a/src/java/org/apache/cassandra/schema/Types.java
+++ b/src/java/org/apache/cassandra/schema/Types.java
@@ -19,6 +19,9 @@
import java.nio.ByteBuffer;
import java.util.*;
+import java.util.function.Predicate;
+import java.util.stream.Stream;
+import java.util.stream.StreamSupport;
import javax.annotation.Nullable;
@@ -31,8 +34,11 @@
import org.apache.cassandra.exceptions.ConfigurationException;
import static java.lang.String.format;
-import static com.google.common.collect.Iterables.filter;
import static java.util.stream.Collectors.toList;
+
+import static com.google.common.collect.Iterables.any;
+import static com.google.common.collect.Iterables.transform;
+
import static org.apache.cassandra.utils.ByteBufferUtil.bytes;
/**
@@ -82,6 +88,27 @@
return types.values().iterator();
}
+ public Stream<UserType> stream()
+ {
+ return StreamSupport.stream(spliterator(), false);
+ }
+
+ /**
+ * Returns a stream of user types sorted by dependencies
+ * @return a stream of user types sorted by dependencies
+ */
+ public Stream<UserType> sortedStream()
+ {
+ Set<ByteBuffer> sorted = new LinkedHashSet<>();
+ types.values().forEach(t -> addUserTypes(t, sorted));
+ return sorted.stream().map(n -> types.get(n));
+ }
+
+ public Iterable<UserType> referencingUserType(ByteBuffer name)
+ {
+ return Iterables.filter(types.values(), t -> t.referencesUserType(name) && !t.name.equals(name));
+ }
+
/**
* Get the type with the specified name
*
@@ -105,6 +132,18 @@
return types.get(name);
}
+ boolean containsType(ByteBuffer name)
+ {
+ return types.containsKey(name);
+ }
+
+ Types filter(Predicate<UserType> predicate)
+ {
+ Builder builder = builder();
+ types.values().stream().filter(predicate).forEach(builder::add);
+ return builder.build();
+ }
+
/**
* Create a Types instance with the provided type added
*/
@@ -124,12 +163,19 @@
UserType type =
get(name).orElseThrow(() -> new IllegalStateException(format("Type %s doesn't exists", name)));
- return builder().add(filter(this, t -> t != type)).build();
+ return without(type);
}
- MapDifference<ByteBuffer, UserType> diff(Types other)
+ public Types without(UserType type)
{
- return Maps.difference(types, other.types);
+ return filter(t -> t != type);
+ }
+
+ public Types withUpdatedUserType(UserType udt)
+ {
+ return any(this, t -> t.referencesUserType(udt.name))
+ ? builder().add(transform(this, t -> t.withUpdatedUserType(udt))).build()
+ : this;
}
@Override
@@ -155,7 +201,7 @@
if (!thisNext.getKey().equals(otherNext.getKey()))
return false;
- if (!thisNext.getValue().equals(otherNext.getValue(), true)) // ignore freezing
+ if (!thisNext.getValue().equals(otherNext.getValue()))
return false;
}
return true;
@@ -173,6 +219,36 @@
return types.values().toString();
}
+ /**
+ * Sorts the types by dependencies.
+ *
+ * @param types the types to sort
+ * @return the types sorted by dependencies and names
+ */
+ private static Set<ByteBuffer> sortByDependencies(Collection<UserType> types)
+ {
+ Set<ByteBuffer> sorted = new LinkedHashSet<>();
+ types.stream().forEach(t -> addUserTypes(t, sorted));
+ return sorted;
+ }
+
+ /**
+ * Find all user types used by the specified type and add them to the set.
+ *
+ * @param type the type to check for user types.
+ * @param types the set of UDT names to which to add new user types found in {@code type}. Note that the
+ * insertion ordering is important and ensures that if a user type A uses another user type B, then B will appear
+ * before A in iteration order.
+ */
+ private static void addUserTypes(AbstractType<?> type, Set<ByteBuffer> types)
+ {
+ // Reach into subtypes first, so that if the type is a UDT, it's dependencies are recreated first.
+ type.subTypes().forEach(t -> addUserTypes(t, types));
+
+ if (type.isUDT())
+ types.add(((UserType) type).name);
+ }
+
public static final class Builder
{
final ImmutableSortedMap.Builder<ByteBuffer, UserType> types = ImmutableSortedMap.naturalOrder();
@@ -231,7 +307,7 @@
/*
* build a DAG of UDT dependencies
*/
- Map<RawUDT, Integer> vertices = new HashMap<>(); // map values are numbers of referenced types
+ Map<RawUDT, Integer> vertices = Maps.newHashMapWithExpectedSize(definitions.size()); // map values are numbers of referenced types
for (RawUDT udt : definitions)
vertices.put(udt, 0);
@@ -305,7 +381,7 @@
{
List<FieldIdentifier> preparedFieldNames =
fieldNames.stream()
- .map(t -> FieldIdentifier.forInternalString(t))
+ .map(FieldIdentifier::forInternalString)
.collect(toList());
List<AbstractType<?>> preparedFieldTypes =
@@ -329,4 +405,38 @@
}
}
}
+
+ static TypesDiff diff(Types before, Types after)
+ {
+ return TypesDiff.diff(before, after);
+ }
+
+ static final class TypesDiff extends Diff<Types, UserType>
+ {
+ private static final TypesDiff NONE = new TypesDiff(Types.none(), Types.none(), ImmutableList.of());
+
+ private TypesDiff(Types created, Types dropped, ImmutableCollection<Altered<UserType>> altered)
+ {
+ super(created, dropped, altered);
+ }
+
+ private static TypesDiff diff(Types before, Types after)
+ {
+ if (before == after)
+ return NONE;
+
+ Types created = after.filter(t -> !before.containsType(t.name));
+ Types dropped = before.filter(t -> !after.containsType(t.name));
+
+ ImmutableList.Builder<Altered<UserType>> altered = ImmutableList.builder();
+ before.forEach(typeBefore ->
+ {
+ UserType typeAfter = after.getNullable(typeBefore.name);
+ if (null != typeAfter)
+ typeBefore.compare(typeAfter).ifPresent(kind -> altered.add(new Altered<>(typeBefore, typeAfter, kind)));
+ });
+
+ return new TypesDiff(created, dropped, altered.build());
+ }
+ }
}
diff --git a/src/java/org/apache/cassandra/schema/UnknownIndexException.java b/src/java/org/apache/cassandra/schema/UnknownIndexException.java
deleted file mode 100644
index 5daf631..0000000
--- a/src/java/org/apache/cassandra/schema/UnknownIndexException.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.schema;
-
-import java.io.IOException;
-import java.util.UUID;
-
-import org.apache.cassandra.config.CFMetaData;
-
-/**
- * Exception thrown when we read an index id from a serialized ReadCommand and no corresponding IndexMetadata
- * can be found in the CFMetaData#indexes collection. Note that this is an internal exception and is not meant
- * to be user facing, the node reading the ReadCommand should proceed as if no index id were present.
- */
-public class UnknownIndexException extends IOException
-{
- public final UUID indexId;
- public UnknownIndexException(CFMetaData metadata, UUID id)
- {
- super(String.format("Unknown index %s for table %s.%s", id.toString(), metadata.ksName, metadata.cfName));
- indexId = id;
- }
-}
diff --git a/src/java/org/apache/cassandra/schema/ViewMetadata.java b/src/java/org/apache/cassandra/schema/ViewMetadata.java
new file mode 100644
index 0000000..3fbf728
--- /dev/null
+++ b/src/java/org/apache/cassandra/schema/ViewMetadata.java
@@ -0,0 +1,239 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.schema;
+
+import java.nio.ByteBuffer;
+import java.util.Optional;
+
+import org.apache.commons.lang3.builder.HashCodeBuilder;
+import org.apache.commons.lang3.builder.ToStringBuilder;
+
+import org.apache.cassandra.cql3.*;
+import org.apache.cassandra.db.marshal.UserType;
+
+public final class ViewMetadata implements SchemaElement
+{
+ public final TableId baseTableId;
+ public final String baseTableName;
+
+ public final boolean includeAllColumns;
+ public final TableMetadata metadata;
+
+ public final WhereClause whereClause;
+
+ /**
+ * @param baseTableId Internal ID of the table which this view is based off of
+ * @param includeAllColumns Whether to include all columns or not
+ */
+ public ViewMetadata(TableId baseTableId,
+ String baseTableName,
+ boolean includeAllColumns,
+ WhereClause whereClause,
+ TableMetadata metadata)
+ {
+ this.baseTableId = baseTableId;
+ this.baseTableName = baseTableName;
+ this.includeAllColumns = includeAllColumns;
+ this.whereClause = whereClause;
+ this.metadata = metadata;
+ }
+
+ public String keyspace()
+ {
+ return metadata.keyspace;
+ }
+
+ public String name()
+ {
+ return metadata.name;
+ }
+
+ /**
+ * @return true if the view specified by this definition will include the column, false otherwise
+ */
+ public boolean includes(ColumnIdentifier column)
+ {
+ return metadata.getColumn(column) != null;
+ }
+
+ public ViewMetadata copy(TableMetadata newMetadata)
+ {
+ return new ViewMetadata(baseTableId, baseTableName, includeAllColumns, whereClause, newMetadata);
+ }
+
+ public TableMetadata baseTableMetadata()
+ {
+ return Schema.instance.getTableMetadata(baseTableId);
+ }
+
+ @Override
+ public boolean equals(Object o)
+ {
+ if (this == o)
+ return true;
+
+ if (!(o instanceof ViewMetadata))
+ return false;
+
+ ViewMetadata other = (ViewMetadata) o;
+ return baseTableId.equals(other.baseTableId)
+ && includeAllColumns == other.includeAllColumns
+ && whereClause.equals(other.whereClause)
+ && metadata.equals(other.metadata);
+ }
+
+ Optional<Difference> compare(ViewMetadata other)
+ {
+ if (!baseTableId.equals(other.baseTableId) || includeAllColumns != other.includeAllColumns || !whereClause.equals(other.whereClause))
+ return Optional.of(Difference.SHALLOW);
+
+ return metadata.compare(other.metadata);
+ }
+
+ @Override
+ public int hashCode()
+ {
+ return new HashCodeBuilder(29, 1597)
+ .append(baseTableId)
+ .append(includeAllColumns)
+ .append(whereClause)
+ .append(metadata)
+ .toHashCode();
+ }
+
+ @Override
+ public String toString()
+ {
+ return new ToStringBuilder(this)
+ .append("baseTableId", baseTableId)
+ .append("baseTableName", baseTableName)
+ .append("includeAllColumns", includeAllColumns)
+ .append("whereClause", whereClause)
+ .append("metadata", metadata)
+ .toString();
+ }
+
+ public boolean referencesUserType(ByteBuffer name)
+ {
+ return metadata.referencesUserType(name);
+ }
+
+ public ViewMetadata withUpdatedUserType(UserType udt)
+ {
+ return referencesUserType(udt.name)
+ ? copy(metadata.withUpdatedUserType(udt))
+ : this;
+ }
+
+ public ViewMetadata withRenamedPrimaryKeyColumn(ColumnIdentifier from, ColumnIdentifier to)
+ {
+ // convert whereClause to Relations, rename ids in Relations, then convert back to whereClause
+ ColumnMetadata.Raw rawFrom = ColumnMetadata.Raw.forQuoted(from.toString());
+ ColumnMetadata.Raw rawTo = ColumnMetadata.Raw.forQuoted(to.toString());
+
+ return new ViewMetadata(baseTableId,
+ baseTableName,
+ includeAllColumns,
+ whereClause.renameIdentifier(rawFrom, rawTo),
+ metadata.unbuild().renamePrimaryKeyColumn(from, to).build());
+ }
+
+ public ViewMetadata withAddedRegularColumn(ColumnMetadata column)
+ {
+ return new ViewMetadata(baseTableId,
+ baseTableName,
+ includeAllColumns,
+ whereClause,
+ metadata.unbuild().addColumn(column).build());
+ }
+
+ public void appendCqlTo(CqlBuilder builder,
+ boolean internals,
+ boolean ifNotExists)
+ {
+ builder.append("CREATE MATERIALIZED VIEW ");
+
+ if (ifNotExists)
+ builder.append("IF NOT EXISTS ");
+
+ builder.append(metadata.toString())
+ .append(" AS")
+ .newLine()
+ .increaseIndent()
+ .append("SELECT ");
+
+ if (includeAllColumns)
+ {
+ builder.append('*');
+ }
+ else
+ {
+ builder.appendWithSeparators(metadata.allColumnsInSelectOrder(), (b, c) -> b.append(c.name), ", ");
+ }
+
+ builder.newLine()
+ .append("FROM ")
+ .appendQuotingIfNeeded(metadata.keyspace)
+ .append('.')
+ .appendQuotingIfNeeded(baseTableName)
+ .newLine()
+ .append("WHERE ")
+ .append(whereClause.toString())
+ .newLine();
+
+ metadata.appendPrimaryKey(builder);
+
+ builder.decreaseIndent();
+
+ metadata.appendTableOptions(builder, internals);
+ }
+
+ @Override
+ public SchemaElementType elementType()
+ {
+ return SchemaElementType.MATERIALIZED_VIEW;
+ }
+
+ @Override
+ public String elementKeyspace()
+ {
+ return keyspace();
+ }
+
+ @Override
+ public String elementName()
+ {
+ return name();
+ }
+
+ @Override
+ public String toCqlString(boolean withInternals)
+ {
+ CqlBuilder builder = new CqlBuilder(2048);
+ appendCqlTo(builder, withInternals, false);
+ return builder.toString();
+ }
+
+ public String toCqlString(boolean internals,
+ boolean ifNotExists)
+ {
+ CqlBuilder builder = new CqlBuilder(2048);
+ appendCqlTo(builder, internals, ifNotExists);
+ return builder.toString();
+ }
+}
diff --git a/src/java/org/apache/cassandra/schema/Views.java b/src/java/org/apache/cassandra/schema/Views.java
index b8fdd4b..f926c07 100644
--- a/src/java/org/apache/cassandra/schema/Views.java
+++ b/src/java/org/apache/cassandra/schema/Views.java
@@ -15,32 +15,34 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.cassandra.schema;
-
+import java.util.HashMap;
import java.util.Iterator;
+import java.util.Map;
import java.util.Optional;
+import java.util.function.Predicate;
+import java.util.stream.Stream;
+import java.util.stream.StreamSupport;
import javax.annotation.Nullable;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.MapDifference;
-import com.google.common.collect.Maps;
+import com.google.common.collect.*;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ViewDefinition;
+import org.apache.cassandra.db.marshal.UserType;
-import static com.google.common.collect.Iterables.filter;
+import static com.google.common.collect.Iterables.any;
+import static com.google.common.collect.Iterables.transform;
-public final class Views implements Iterable<ViewDefinition>
+public final class Views implements Iterable<ViewMetadata>
{
- private final ImmutableMap<String, ViewDefinition> views;
+ private static final Views NONE = builder().build();
+
+ private final ImmutableMap<String, ViewMetadata> views;
private Views(Builder builder)
{
- views = builder.views.build();
+ views = ImmutableMap.copyOf(builder.views);
}
public static Builder builder()
@@ -48,17 +50,22 @@
return new Builder();
}
- public static Views none()
+ public Builder unbuild()
{
- return builder().build();
+ return builder().put(this);
}
- public Iterator<ViewDefinition> iterator()
+ public static Views none()
+ {
+ return NONE;
+ }
+
+ public Iterator<ViewMetadata> iterator()
{
return views.values().iterator();
}
- public Iterable<CFMetaData> metadatas()
+ Iterable<TableMetadata> allTableMetadata()
{
return Iterables.transform(views.values(), view -> view.metadata);
}
@@ -73,13 +80,28 @@
return views.isEmpty();
}
+ public Iterable<ViewMetadata> forTable(TableId tableId)
+ {
+ return Iterables.filter(this, v -> v.baseTableId.equals(tableId));
+ }
+
+ public Stream<ViewMetadata> stream()
+ {
+ return StreamSupport.stream(spliterator(), false);
+ }
+
+ public Stream<ViewMetadata> stream(TableId tableId)
+ {
+ return stream().filter(v -> v.baseTableId.equals(tableId));
+ }
+
/**
* Get the materialized view with the specified name
*
* @param name a non-qualified materialized view name
- * @return an empty {@link Optional} if the materialized view name is not found; a non-empty optional of {@link ViewDefinition} otherwise
+ * @return an empty {@link Optional} if the materialized view name is not found; a non-empty optional of {@link ViewMetadata} otherwise
*/
- public Optional<ViewDefinition> get(String name)
+ public Optional<ViewMetadata> get(String name)
{
return Optional.ofNullable(views.get(name));
}
@@ -88,23 +110,40 @@
* Get the view with the specified name
*
* @param name a non-qualified view name
- * @return null if the view name is not found; the found {@link ViewDefinition} otherwise
+ * @return null if the view name is not found; the found {@link ViewMetadata} otherwise
*/
@Nullable
- public ViewDefinition getNullable(String name)
+ public ViewMetadata getNullable(String name)
{
return views.get(name);
}
+ boolean containsView(String name)
+ {
+ return views.containsKey(name);
+ }
+
+ Views filter(Predicate<ViewMetadata> predicate)
+ {
+ Builder builder = builder();
+ views.values().stream().filter(predicate).forEach(builder::put);
+ return builder.build();
+ }
+
/**
* Create a MaterializedViews instance with the provided materialized view added
*/
- public Views with(ViewDefinition view)
+ public Views with(ViewMetadata view)
{
- if (get(view.viewName).isPresent())
- throw new IllegalStateException(String.format("Materialized View %s already exists", view.viewName));
+ if (get(view.name()).isPresent())
+ throw new IllegalStateException(String.format("Materialized View %s already exists", view.name()));
- return builder().add(this).add(view).build();
+ return builder().put(this).put(view).build();
+ }
+
+ public Views withSwapped(ViewMetadata view)
+ {
+ return without(view.name()).with(view);
}
/**
@@ -112,23 +151,17 @@
*/
public Views without(String name)
{
- ViewDefinition materializedView =
+ ViewMetadata materializedView =
get(name).orElseThrow(() -> new IllegalStateException(String.format("Materialized View %s doesn't exists", name)));
- return builder().add(filter(this, v -> v != materializedView)).build();
+ return filter(v -> v != materializedView);
}
- /**
- * Creates a MaterializedViews instance which contains an updated materialized view
- */
- public Views replace(ViewDefinition view, CFMetaData cfm)
+ Views withUpdatedUserTypes(UserType udt)
{
- return without(view.viewName).with(view);
- }
-
- MapDifference<String, ViewDefinition> diff(Views other)
- {
- return Maps.difference(views, other.views);
+ return any(this, v -> v.referencesUserType(udt.name))
+ ? builder().put(transform(this, v -> v.withUpdatedUserType(udt))).build()
+ : this;
}
@Override
@@ -151,7 +184,7 @@
public static final class Builder
{
- final ImmutableMap.Builder<String, ViewDefinition> views = new ImmutableMap.Builder<>();
+ final Map<String, ViewMetadata> views = new HashMap<>();
private Builder()
{
@@ -162,17 +195,61 @@
return new Views(this);
}
-
- public Builder add(ViewDefinition view)
+ public ViewMetadata get(String name)
{
- views.put(view.viewName, view);
+ return views.get(name);
+ }
+
+ public Builder put(ViewMetadata view)
+ {
+ views.put(view.name(), view);
return this;
}
- public Builder add(Iterable<ViewDefinition> views)
+ public Builder remove(String name)
{
- views.forEach(this::add);
+ views.remove(name);
return this;
}
+
+ public Builder put(Iterable<ViewMetadata> views)
+ {
+ views.forEach(this::put);
+ return this;
+ }
+ }
+
+ static ViewsDiff diff(Views before, Views after)
+ {
+ return ViewsDiff.diff(before, after);
+ }
+
+ static final class ViewsDiff extends Diff<Views, ViewMetadata>
+ {
+ private static final ViewsDiff NONE = new ViewsDiff(Views.none(), Views.none(), ImmutableList.of());
+
+ private ViewsDiff(Views created, Views dropped, ImmutableCollection<Altered<ViewMetadata>> altered)
+ {
+ super(created, dropped, altered);
+ }
+
+ private static ViewsDiff diff(Views before, Views after)
+ {
+ if (before == after)
+ return NONE;
+
+ Views created = after.filter(v -> !before.containsView(v.name()));
+ Views dropped = before.filter(v -> !after.containsView(v.name()));
+
+ ImmutableList.Builder<Altered<ViewMetadata>> altered = ImmutableList.builder();
+ before.forEach(viewBefore ->
+ {
+ ViewMetadata viewAfter = after.getNullable(viewBefore.name());
+ if (null != viewAfter)
+ viewBefore.compare(viewAfter).ifPresent(kind -> altered.add(new Altered<>(viewBefore, viewAfter, kind)));
+ });
+
+ return new ViewsDiff(created, dropped, altered.build());
+ }
}
}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/security/CipherFactory.java b/src/java/org/apache/cassandra/security/CipherFactory.java
index 3f5c5f3..3c13629 100644
--- a/src/java/org/apache/cassandra/security/CipherFactory.java
+++ b/src/java/org/apache/cassandra/security/CipherFactory.java
@@ -25,17 +25,17 @@
import java.security.NoSuchAlgorithmException;
import java.security.SecureRandom;
import java.util.Arrays;
-import java.util.concurrent.ExecutionException;
+import java.util.concurrent.CompletionException;
import javax.crypto.Cipher;
import javax.crypto.NoSuchPaddingException;
import javax.crypto.spec.IvParameterSpec;
import com.google.common.annotations.VisibleForTesting;
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.CacheLoader;
-import com.google.common.cache.LoadingCache;
-import com.google.common.cache.RemovalListener;
-import com.google.common.cache.RemovalNotification;
+import com.google.common.util.concurrent.MoreExecutors;
+
+import com.github.benmanes.caffeine.cache.Caffeine;
+import com.github.benmanes.caffeine.cache.LoadingCache;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -79,26 +79,19 @@
throw new RuntimeException("couldn't load cipher factory", e);
}
- cache = CacheBuilder.newBuilder() // by default cache is unbounded
+ cache = Caffeine.newBuilder() // by default cache is unbounded
.maximumSize(64) // a value large enough that we should never even get close (so nothing gets evicted)
- .concurrencyLevel(Runtime.getRuntime().availableProcessors())
- .removalListener(new RemovalListener<String, Key>()
+ .executor(MoreExecutors.directExecutor())
+ .removalListener((key, value, cause) ->
{
- public void onRemoval(RemovalNotification<String, Key> notice)
- {
- // maybe reload the key? (to avoid the reload being on the user's dime)
- logger.info("key {} removed from cipher key cache", notice.getKey());
- }
+ // maybe reload the key? (to avoid the reload being on the user's dime)
+ logger.info("key {} removed from cipher key cache", key);
})
- .build(new CacheLoader<String, Key>()
- {
- @Override
- public Key load(String alias) throws Exception
- {
- logger.info("loading secret key for alias {}", alias);
- return keyProvider.getSecretKey(alias);
- }
- });
+ .build(alias ->
+ {
+ logger.info("loading secret key for alias {}", alias);
+ return keyProvider.getSecretKey(alias);
+ });
}
public Cipher getEncryptor(String transformation, String keyAlias) throws IOException
@@ -148,7 +141,7 @@
{
return cache.get(keyAlias);
}
- catch (ExecutionException e)
+ catch (CompletionException e)
{
if (e.getCause() instanceof IOException)
throw (IOException)e.getCause();
diff --git a/src/java/org/apache/cassandra/security/JKSKeyProvider.java b/src/java/org/apache/cassandra/security/JKSKeyProvider.java
index db7a2b9..cea7b23 100644
--- a/src/java/org/apache/cassandra/security/JKSKeyProvider.java
+++ b/src/java/org/apache/cassandra/security/JKSKeyProvider.java
@@ -17,7 +17,9 @@
*/
package org.apache.cassandra.security;
-import java.io.FileInputStream;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.io.InputStream;
import java.io.IOException;
import java.security.Key;
import java.security.KeyStore;
@@ -46,7 +48,7 @@
{
this.options = options;
logger.info("initializing keystore from file {}", options.get(PROP_KEYSTORE));
- try (FileInputStream inputStream = new FileInputStream(options.get(PROP_KEYSTORE)))
+ try (InputStream inputStream = Files.newInputStream(Paths.get(options.get(PROP_KEYSTORE))))
{
store = KeyStore.getInstance(options.get(PROP_KEYSTORE_TYPE));
store.load(inputStream, options.get(PROP_KEYSTORE_PW).toCharArray());
diff --git a/src/java/org/apache/cassandra/security/SSLFactory.java b/src/java/org/apache/cassandra/security/SSLFactory.java
index 7216e2c..e51ce46 100644
--- a/src/java/org/apache/cassandra/security/SSLFactory.java
+++ b/src/java/org/apache/cassandra/security/SSLFactory.java
@@ -18,163 +18,196 @@
package org.apache.cassandra.security;
-import java.io.FileInputStream;
+import java.io.File;
import java.io.IOException;
-import java.net.InetAddress;
-import java.net.InetSocketAddress;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Paths;
import java.security.KeyStore;
import java.security.cert.X509Certificate;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.Enumeration;
import java.util.List;
-
+import java.util.Objects;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.TimeUnit;
import javax.net.ssl.KeyManagerFactory;
import javax.net.ssl.SSLContext;
-import javax.net.ssl.SSLParameters;
-import javax.net.ssl.SSLServerSocket;
import javax.net.ssl.SSLSocket;
import javax.net.ssl.TrustManager;
import javax.net.ssl.TrustManagerFactory;
-import org.apache.cassandra.config.EncryptionOptions;
-import org.apache.cassandra.io.util.FileUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Predicates;
+import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
import com.google.common.collect.Sets;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import io.netty.handler.ssl.ClientAuth;
+import io.netty.handler.ssl.OpenSsl;
+import io.netty.handler.ssl.SslContext;
+import io.netty.handler.ssl.SslContextBuilder;
+import io.netty.handler.ssl.SslProvider;
+import io.netty.handler.ssl.SupportedCipherSuiteFilter;
+import io.netty.util.ReferenceCountUtil;
+import org.apache.cassandra.concurrent.ScheduledExecutors;
+import org.apache.cassandra.config.Config;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.config.EncryptionOptions;
/**
- * A Factory for providing and setting up Client and Server SSL wrapped
- * Socket and ServerSocket
+ * A Factory for providing and setting up client {@link SSLSocket}s. Also provides
+ * methods for creating both JSSE {@link SSLContext} instances as well as netty {@link SslContext} instances.
+ * <p>
+ * Netty {@link SslContext} instances are expensive to create (as well as to destroy) and consume a lof of resources
+ * (especially direct memory), but instances can be reused across connections (assuming the SSL params are the same).
+ * Hence we cache created instances in {@link #cachedSslContexts}.
*/
public final class SSLFactory
{
private static final Logger logger = LoggerFactory.getLogger(SSLFactory.class);
- private static boolean checkedExpiry = false;
- public static SSLServerSocket getServerSocket(EncryptionOptions options, InetAddress address, int port) throws IOException
+ /**
+ * Indicates if the process holds the inbound/listening end of the socket ({@link SocketType#SERVER})), or the
+ * outbound side ({@link SocketType#CLIENT}).
+ */
+ public enum SocketType
{
- SSLContext ctx = createSSLContext(options, true);
- SSLServerSocket serverSocket = (SSLServerSocket)ctx.getServerSocketFactory().createServerSocket();
- try
+ SERVER, CLIENT
+ }
+
+ @VisibleForTesting
+ static volatile boolean checkedExpiry = false;
+
+ // Isolate calls to OpenSsl.isAvailable to allow in-jvm dtests to disable tcnative openssl
+ // support. It creates a circular reference that prevents the instance class loader from being
+ // garbage collected.
+ static private final boolean openSslIsAvailable;
+ static
+ {
+ if (Boolean.getBoolean(Config.PROPERTY_PREFIX + "disable_tcactive_openssl"))
{
- serverSocket.setReuseAddress(true);
- prepareSocket(serverSocket, options);
- serverSocket.bind(new InetSocketAddress(address, port), 500);
- return serverSocket;
+ openSslIsAvailable = false;
}
- catch (IllegalArgumentException | SecurityException | IOException e)
+ else
{
- serverSocket.close();
- throw e;
+ openSslIsAvailable = OpenSsl.isAvailable();
+ }
+ }
+ public static boolean openSslIsAvailable()
+ {
+ return openSslIsAvailable;
+ }
+
+ /**
+ * Cached references of SSL Contexts
+ */
+ private static final ConcurrentHashMap<CacheKey, SslContext> cachedSslContexts = new ConcurrentHashMap<>();
+
+ /**
+ * List of files that trigger hot reloading of SSL certificates
+ */
+ private static volatile List<HotReloadableFile> hotReloadableFiles = ImmutableList.of();
+
+ /**
+ * Default initial delay for hot reloading
+ */
+ public static final int DEFAULT_HOT_RELOAD_INITIAL_DELAY_SEC = 600;
+
+ /**
+ * Default periodic check delay for hot reloading
+ */
+ public static final int DEFAULT_HOT_RELOAD_PERIOD_SEC = 600;
+
+ /**
+ * State variable to maintain initialization invariant
+ */
+ private static boolean isHotReloadingInitialized = false;
+
+ /**
+ * Helper class for hot reloading SSL Contexts
+ */
+ private static class HotReloadableFile
+ {
+ private final File file;
+ private volatile long lastModTime;
+
+ HotReloadableFile(String path)
+ {
+ file = new File(path);
+ lastModTime = file.lastModified();
+ }
+
+ boolean shouldReload()
+ {
+ long curModTime = file.lastModified();
+ boolean result = curModTime != lastModTime;
+ lastModTime = curModTime;
+ return result;
+ }
+
+ @Override
+ public String toString()
+ {
+ return "HotReloadableFile{" +
+ "file=" + file +
+ ", lastModTime=" + lastModTime +
+ '}';
}
}
- /** Create a socket and connect */
- public static SSLSocket getSocket(EncryptionOptions options, InetAddress address, int port, InetAddress localAddress, int localPort) throws IOException
- {
- SSLContext ctx = createSSLContext(options, true);
- SSLSocket socket = (SSLSocket) ctx.getSocketFactory().createSocket(address, port, localAddress, localPort);
- try
- {
- prepareSocket(socket, options);
- return socket;
- }
- catch (IllegalArgumentException e)
- {
- socket.close();
- throw e;
- }
- }
-
- /** Create a socket and connect, using any local address */
- public static SSLSocket getSocket(EncryptionOptions options, InetAddress address, int port) throws IOException
- {
- SSLContext ctx = createSSLContext(options, true);
- SSLSocket socket = (SSLSocket) ctx.getSocketFactory().createSocket(address, port);
- try
- {
- prepareSocket(socket, options);
- return socket;
- }
- catch (IllegalArgumentException e)
- {
- socket.close();
- throw e;
- }
- }
-
- /** Just create a socket */
- public static SSLSocket getSocket(EncryptionOptions options) throws IOException
- {
- SSLContext ctx = createSSLContext(options, true);
- SSLSocket socket = (SSLSocket) ctx.getSocketFactory().createSocket();
- try
- {
- prepareSocket(socket, options);
- return socket;
- }
- catch (IllegalArgumentException e)
- {
- socket.close();
- throw e;
- }
- }
-
- /** Sets relevant socket options specified in encryption settings */
- private static void prepareSocket(SSLServerSocket serverSocket, EncryptionOptions options)
- {
- String[] suites = filterCipherSuites(serverSocket.getSupportedCipherSuites(), options.cipher_suites);
- if(options.require_endpoint_verification)
- {
- SSLParameters sslParameters = serverSocket.getSSLParameters();
- sslParameters.setEndpointIdentificationAlgorithm("HTTPS");
- serverSocket.setSSLParameters(sslParameters);
- }
- serverSocket.setEnabledCipherSuites(suites);
- serverSocket.setNeedClientAuth(options.require_client_auth);
- }
-
- /** Sets relevant socket options specified in encryption settings */
- private static void prepareSocket(SSLSocket socket, EncryptionOptions options)
- {
- String[] suites = filterCipherSuites(socket.getSupportedCipherSuites(), options.cipher_suites);
- if(options.require_endpoint_verification)
- {
- SSLParameters sslParameters = socket.getSSLParameters();
- sslParameters.setEndpointIdentificationAlgorithm("HTTPS");
- socket.setSSLParameters(sslParameters);
- }
- socket.setEnabledCipherSuites(suites);
- }
-
+ /**
+ * Create a JSSE {@link SSLContext}.
+ */
@SuppressWarnings("resource")
public static SSLContext createSSLContext(EncryptionOptions options, boolean buildTruststore) throws IOException
{
- FileInputStream tsf = null;
- FileInputStream ksf = null;
- SSLContext ctx;
+ TrustManager[] trustManagers = null;
+ if (buildTruststore)
+ trustManagers = buildTrustManagerFactory(options).getTrustManagers();
+
+ KeyManagerFactory kmf = buildKeyManagerFactory(options);
+
try
{
- ctx = SSLContext.getInstance(options.protocol);
- TrustManager[] trustManagers = null;
+ SSLContext ctx = SSLContext.getInstance(options.protocol);
+ ctx.init(kmf.getKeyManagers(), trustManagers, null);
+ return ctx;
+ }
+ catch (Exception e)
+ {
+ throw new IOException("Error creating/initializing the SSL Context", e);
+ }
+ }
- if(buildTruststore)
- {
- tsf = new FileInputStream(options.truststore);
- TrustManagerFactory tmf = TrustManagerFactory.getInstance(options.algorithm);
- KeyStore ts = KeyStore.getInstance(options.store_type);
- ts.load(tsf, options.truststore_password.toCharArray());
- tmf.init(ts);
- trustManagers = tmf.getTrustManagers();
- }
+ static TrustManagerFactory buildTrustManagerFactory(EncryptionOptions options) throws IOException
+ {
+ try (InputStream tsf = Files.newInputStream(Paths.get(options.truststore)))
+ {
+ TrustManagerFactory tmf = TrustManagerFactory.getInstance(
+ options.algorithm == null ? TrustManagerFactory.getDefaultAlgorithm() : options.algorithm);
+ KeyStore ts = KeyStore.getInstance(options.store_type);
+ ts.load(tsf, options.truststore_password.toCharArray());
+ tmf.init(ts);
+ return tmf;
+ }
+ catch (Exception e)
+ {
+ throw new IOException("failed to build trust manager store for secure connections", e);
+ }
+ }
- ksf = new FileInputStream(options.keystore);
- KeyManagerFactory kmf = KeyManagerFactory.getInstance(options.algorithm);
+ static KeyManagerFactory buildKeyManagerFactory(EncryptionOptions options) throws IOException
+ {
+ try (InputStream ksf = Files.newInputStream(Paths.get(options.keystore)))
+ {
+ KeyManagerFactory kmf = KeyManagerFactory.getInstance(
+ options.algorithm == null ? KeyManagerFactory.getDefaultAlgorithm() : options.algorithm);
KeyStore ks = KeyStore.getInstance(options.store_type);
ks.load(ksf, options.keystore_password.toCharArray());
if (!checkedExpiry)
@@ -192,20 +225,12 @@
checkedExpiry = true;
}
kmf.init(ks, options.keystore_password.toCharArray());
-
- ctx.init(kmf.getKeyManagers(), trustManagers, null);
-
+ return kmf;
}
catch (Exception e)
{
- throw new IOException("Error creating the initializing the SSL Context", e);
+ throw new IOException("failed to build trust manager store for secure connections", e);
}
- finally
- {
- FileUtils.closeQuietly(tsf);
- FileUtils.closeQuietly(ksf);
- }
- return ctx;
}
public static String[] filterCipherSuites(String[] supported, String[] desired)
@@ -222,4 +247,221 @@
}
return ret;
}
+
+ /**
+ * get a netty {@link SslContext} instance
+ */
+ public static SslContext getOrCreateSslContext(EncryptionOptions options, boolean buildTruststore,
+ SocketType socketType) throws IOException
+ {
+ return getOrCreateSslContext(options, buildTruststore, socketType, openSslIsAvailable());
+ }
+
+ /**
+ * Get a netty {@link SslContext} instance.
+ */
+ @VisibleForTesting
+ static SslContext getOrCreateSslContext(EncryptionOptions options,
+ boolean buildTruststore,
+ SocketType socketType,
+ boolean useOpenSsl) throws IOException
+ {
+ CacheKey key = new CacheKey(options, socketType, useOpenSsl);
+ SslContext sslContext;
+
+ sslContext = cachedSslContexts.get(key);
+ if (sslContext != null)
+ return sslContext;
+
+ sslContext = createNettySslContext(options, buildTruststore, socketType, useOpenSsl);
+
+ SslContext previous = cachedSslContexts.putIfAbsent(key, sslContext);
+ if (previous == null)
+ return sslContext;
+
+ ReferenceCountUtil.release(sslContext);
+ return previous;
+ }
+
+ /**
+ * Create a Netty {@link SslContext}
+ */
+ static SslContext createNettySslContext(EncryptionOptions options, boolean buildTruststore,
+ SocketType socketType, boolean useOpenSsl) throws IOException
+ {
+ /*
+ There is a case where the netty/openssl combo might not support using KeyManagerFactory. specifically,
+ I've seen this with the netty-tcnative dynamic openssl implementation. using the netty-tcnative static-boringssl
+ works fine with KeyManagerFactory. If we want to support all of the netty-tcnative options, we would need
+ to fall back to passing in a file reference for both a x509 and PKCS#8 private key file in PEM format (see
+ {@link SslContextBuilder#forServer(File, File, String)}). However, we are not supporting that now to keep
+ the config/yaml API simple.
+ */
+ KeyManagerFactory kmf = buildKeyManagerFactory(options);
+ SslContextBuilder builder;
+ if (socketType == SocketType.SERVER)
+ {
+ builder = SslContextBuilder.forServer(kmf);
+ builder.clientAuth(options.require_client_auth ? ClientAuth.REQUIRE : ClientAuth.NONE);
+ }
+ else
+ {
+ builder = SslContextBuilder.forClient().keyManager(kmf);
+ }
+
+ builder.sslProvider(useOpenSsl ? SslProvider.OPENSSL : SslProvider.JDK);
+
+ // only set the cipher suites if the opertor has explicity configured values for it; else, use the default
+ // for each ssl implemention (jdk or openssl)
+ if (options.cipher_suites != null && !options.cipher_suites.isEmpty())
+ builder.ciphers(options.cipher_suites, SupportedCipherSuiteFilter.INSTANCE);
+
+ if (buildTruststore)
+ builder.trustManager(buildTrustManagerFactory(options));
+
+ return builder.build();
+ }
+
+ /**
+ * Performs a lightweight check whether the certificate files have been refreshed.
+ *
+ * @throws IllegalStateException if {@link #initHotReloading(EncryptionOptions.ServerEncryptionOptions, EncryptionOptions, boolean)}
+ * is not called first
+ */
+ public static void checkCertFilesForHotReloading(EncryptionOptions.ServerEncryptionOptions serverOpts,
+ EncryptionOptions clientOpts)
+ {
+ if (!isHotReloadingInitialized)
+ throw new IllegalStateException("Hot reloading functionality has not been initialized.");
+
+ logger.debug("Checking whether certificates have been updated {}", hotReloadableFiles);
+
+ if (hotReloadableFiles.stream().anyMatch(HotReloadableFile::shouldReload))
+ {
+ logger.info("SSL certificates have been updated. Reseting the ssl contexts for new connections.");
+ try
+ {
+ validateSslCerts(serverOpts, clientOpts);
+ cachedSslContexts.clear();
+ }
+ catch(Exception e)
+ {
+ logger.error("Failed to hot reload the SSL Certificates! Please check the certificate files.", e);
+ }
+ }
+ }
+
+ /**
+ * Determines whether to hot reload certificates and schedules a periodic task for it.
+ *
+ * @param serverOpts Server encryption options (Internode)
+ * @param clientOpts Client encryption options (Native Protocol)
+ */
+ public static synchronized void initHotReloading(EncryptionOptions.ServerEncryptionOptions serverOpts,
+ EncryptionOptions clientOpts,
+ boolean force) throws IOException
+ {
+ if (isHotReloadingInitialized && !force)
+ return;
+
+ logger.debug("Initializing hot reloading SSLContext");
+
+ validateSslCerts(serverOpts, clientOpts);
+
+ List<HotReloadableFile> fileList = new ArrayList<>();
+
+ if (serverOpts != null && serverOpts.isEnabled())
+ {
+ fileList.add(new HotReloadableFile(serverOpts.keystore));
+ fileList.add(new HotReloadableFile(serverOpts.truststore));
+ }
+
+ if (clientOpts != null && clientOpts.isEnabled())
+ {
+ fileList.add(new HotReloadableFile(clientOpts.keystore));
+ fileList.add(new HotReloadableFile(clientOpts.truststore));
+ }
+
+ hotReloadableFiles = ImmutableList.copyOf(fileList);
+
+ if (!isHotReloadingInitialized)
+ {
+ ScheduledExecutors.scheduledTasks
+ .scheduleWithFixedDelay(() -> checkCertFilesForHotReloading(
+ DatabaseDescriptor.getInternodeMessagingEncyptionOptions(),
+ DatabaseDescriptor.getNativeProtocolEncryptionOptions()),
+ DEFAULT_HOT_RELOAD_INITIAL_DELAY_SEC,
+ DEFAULT_HOT_RELOAD_PERIOD_SEC, TimeUnit.SECONDS);
+ }
+
+ isHotReloadingInitialized = true;
+ }
+
+
+ /**
+ * Sanity checks all certificates to ensure we can actually load them
+ */
+ public static void validateSslCerts(EncryptionOptions.ServerEncryptionOptions serverOpts, EncryptionOptions clientOpts) throws IOException
+ {
+ try
+ {
+ // Ensure we're able to create both server & client SslContexts
+ if (serverOpts != null && serverOpts.isEnabled())
+ {
+ createNettySslContext(serverOpts, true, SocketType.SERVER, openSslIsAvailable());
+ createNettySslContext(serverOpts, true, SocketType.CLIENT, openSslIsAvailable());
+ }
+ }
+ catch (Exception e)
+ {
+ throw new IOException("Failed to create SSL context using server_encryption_options!", e);
+ }
+
+ try
+ {
+ // Ensure we're able to create both server & client SslContexts
+ if (clientOpts != null && clientOpts.isEnabled())
+ {
+ createNettySslContext(clientOpts, clientOpts.require_client_auth, SocketType.SERVER, openSslIsAvailable());
+ createNettySslContext(clientOpts, clientOpts.require_client_auth, SocketType.CLIENT, openSslIsAvailable());
+ }
+ }
+ catch (Exception e)
+ {
+ throw new IOException("Failed to create SSL context using client_encryption_options!", e);
+ }
+ }
+
+ static class CacheKey
+ {
+ private final EncryptionOptions encryptionOptions;
+ private final SocketType socketType;
+ private final boolean useOpenSSL;
+
+ public CacheKey(EncryptionOptions encryptionOptions, SocketType socketType, boolean useOpenSSL)
+ {
+ this.encryptionOptions = encryptionOptions;
+ this.socketType = socketType;
+ this.useOpenSSL = useOpenSSL;
+ }
+
+ public boolean equals(Object o)
+ {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ CacheKey cacheKey = (CacheKey) o;
+ return (socketType == cacheKey.socketType &&
+ useOpenSSL == cacheKey.useOpenSSL &&
+ Objects.equals(encryptionOptions, cacheKey.encryptionOptions));
+ }
+
+ public int hashCode()
+ {
+ int result = 0;
+ result += 31 * socketType.hashCode();
+ result += 31 * encryptionOptions.hashCode();
+ result += 31 * Boolean.hashCode(useOpenSSL);
+ return result;
+ }
+ }
}
diff --git a/src/java/org/apache/cassandra/security/ThreadAwareSecurityManager.java b/src/java/org/apache/cassandra/security/ThreadAwareSecurityManager.java
index c9402f1..86c8b5b 100644
--- a/src/java/org/apache/cassandra/security/ThreadAwareSecurityManager.java
+++ b/src/java/org/apache/cassandra/security/ThreadAwareSecurityManager.java
@@ -18,6 +18,7 @@
package org.apache.cassandra.security;
+import java.lang.reflect.ReflectPermission;
import java.security.AccessControlException;
import java.security.AllPermission;
import java.security.CodeSource;
@@ -66,6 +67,12 @@
private static final RuntimePermission MODIFY_THREAD_PERMISSION = new RuntimePermission("modifyThread");
private static final RuntimePermission MODIFY_THREADGROUP_PERMISSION = new RuntimePermission("modifyThreadGroup");
+ // Nashorn / Java 11
+ private static final RuntimePermission NASHORN_GLOBAL_PERMISSION = new RuntimePermission("nashorn.createGlobal");
+ private static final ReflectPermission SUPPRESS_ACCESS_CHECKS_PERMISSION = new ReflectPermission("suppressAccessChecks");
+ private static final RuntimePermission DYNALINK_LOOKUP_PERMISSION = new RuntimePermission("dynalink.getLookup");
+ private static final RuntimePermission GET_CLASSLOADER_PERMISSION = new RuntimePermission("getClassLoader");
+
private static volatile boolean installed;
public static void install()
@@ -103,7 +110,11 @@
switch (codesource.getLocation().getProtocol())
{
- case "file":
+ case "jar": // One-JAR or Uno-Jar source
+ if (!codesource.getLocation().getPath().startsWith("file:")) {
+ return perms;
+ } // else fall through and add AllPermission()
+ case "file": // Standard file system source
// All JARs and class files reside on the file system - we can safely
// assume that these classes are "good".
perms.add(new AllPermission());
@@ -126,7 +137,9 @@
switch (codesource.getLocation().getProtocol())
{
- case "file":
+ case "jar": // One-JAR or Uno-Jar source
+ return codesource.getLocation().getPath().startsWith("file:");
+ case "file": // Standard file system source
// All JARs and class files reside on the file system - we can safely
// assume that these classes are "good".
return true;
@@ -189,6 +202,16 @@
if (CHECK_MEMBER_ACCESS_PERMISSION.equals(perm))
return;
+ // Nashorn / Java 11
+ if (NASHORN_GLOBAL_PERMISSION.equals(perm))
+ return;
+ if (SUPPRESS_ACCESS_CHECKS_PERMISSION.equals(perm))
+ return;
+ if (DYNALINK_LOOKUP_PERMISSION.equals(perm))
+ return;
+ if (GET_CLASSLOADER_PERMISSION.equals(perm))
+ return;
+
super.checkPermission(perm);
}
@@ -208,7 +231,5 @@
RuntimePermission perm = new RuntimePermission("accessClassInPackage." + pkg);
throw new AccessControlException("access denied: " + perm, perm);
}
-
- super.checkPackageAccess(pkg);
}
}
diff --git a/src/java/org/apache/cassandra/serializers/ByteSerializer.java b/src/java/org/apache/cassandra/serializers/ByteSerializer.java
index 9d34fbc..8428e62 100644
--- a/src/java/org/apache/cassandra/serializers/ByteSerializer.java
+++ b/src/java/org/apache/cassandra/serializers/ByteSerializer.java
@@ -28,12 +28,12 @@
public Byte deserialize(ByteBuffer bytes)
{
- return bytes == null || bytes.remaining() == 0 ? null : bytes.get(bytes.position());
+ return bytes == null || bytes.remaining() == 0 ? null : ByteBufferUtil.toByte(bytes);
}
public ByteBuffer serialize(Byte value)
{
- return value == null ? ByteBufferUtil.EMPTY_BYTE_BUFFER : ByteBuffer.allocate(1).put(0, value);
+ return value == null ? ByteBufferUtil.EMPTY_BYTE_BUFFER : ByteBufferUtil.bytes(value);
}
public void validate(ByteBuffer bytes) throws MarshalException
diff --git a/src/java/org/apache/cassandra/serializers/CollectionSerializer.java b/src/java/org/apache/cassandra/serializers/CollectionSerializer.java
index 95a0388..3efdef9 100644
--- a/src/java/org/apache/cassandra/serializers/CollectionSerializer.java
+++ b/src/java/org/apache/cassandra/serializers/CollectionSerializer.java
@@ -23,6 +23,7 @@
import java.util.List;
import org.apache.cassandra.transport.ProtocolVersion;
+import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.utils.ByteBufferUtil;
public abstract class CollectionSerializer<T> implements TypeSerializer<T>
@@ -71,7 +72,7 @@
protected static void writeCollectionSize(ByteBuffer output, int elements, ProtocolVersion version)
{
- output.putInt(elements);
+ output.putInt(elements);
}
public static int readCollectionSize(ByteBuffer input, ProtocolVersion version)
@@ -105,8 +106,67 @@
return ByteBufferUtil.readBytes(input, size);
}
+ protected static void skipValue(ByteBuffer input, ProtocolVersion version)
+ {
+ int size = input.getInt();
+ input.position(input.position() + size);
+ }
+
public static int sizeOfValue(ByteBuffer value, ProtocolVersion version)
{
return value == null ? 4 : 4 + value.remaining();
}
+
+ /**
+ * Extract an element from a serialized collection.
+ * <p>
+ * Note that this is only supported to sets and maps. For sets, this mostly ends up being
+ * a check for the presence of the provide key: it will return the key if it's present and
+ * {@code null} otherwise.
+ *
+ * @param collection the serialized collection. This cannot be {@code null}.
+ * @param key the key to extract (This cannot be {@code null} nor {@code ByteBufferUtil.UNSET_BYTE_BUFFER}).
+ * @param comparator the type to use to compare the {@code key} value to those
+ * in the collection.
+ * @return the value associated with {@code key} if one exists, {@code null} otherwise
+ */
+ public abstract ByteBuffer getSerializedValue(ByteBuffer collection, ByteBuffer key, AbstractType<?> comparator);
+
+ /**
+ * Returns the slice of a collection directly from its serialized value.
+ * <p>If the slice contains no elements an empty collection will be returned for frozen collections, and a
+ * {@code null} one for non-frozen collections.</p>
+ *
+ * @param collection the serialized collection. This cannot be {@code null}.
+ * @param from the left bound of the slice to extract. This cannot be {@code null} but if this is
+ * {@code ByteBufferUtil.UNSET_BYTE_BUFFER}, then the returned slice starts at the beginning
+ * of {@code collection}.
+ * @param comparator the type to use to compare the {@code from} and {@code to} values to those
+ * in the collection.
+ * @param frozen {@code true} if the collection is a frozen one, {@code false} otherwise
+ * @return a serialized collection corresponding to slice {@code [from, to]} of {@code collection}.
+ */
+ public abstract ByteBuffer getSliceFromSerialized(ByteBuffer collection,
+ ByteBuffer from,
+ ByteBuffer to,
+ AbstractType<?> comparator,
+ boolean frozen);
+
+ /**
+ * Creates a new serialized map composed from the data from {@code input} between {@code startPos}
+ * (inclusive) and {@code endPos} (exclusive), assuming that data holds {@code count} elements.
+ */
+ protected ByteBuffer copyAsNewCollection(ByteBuffer input, int count, int startPos, int endPos, ProtocolVersion version)
+ {
+ int sizeLen = sizeOfCollectionSize(count, version);
+ if (count == 0)
+ return ByteBuffer.allocate(sizeLen);
+
+ int bodyLen = endPos - startPos;
+ ByteBuffer output = ByteBuffer.allocate(sizeLen + bodyLen);
+ writeCollectionSize(output, count, version);
+ output.position(0);
+ ByteBufferUtil.copyBytes(input, startPos, output, sizeLen, bodyLen);
+ return output;
+ }
}
diff --git a/src/java/org/apache/cassandra/serializers/EmptySerializer.java b/src/java/org/apache/cassandra/serializers/EmptySerializer.java
index 352ef2c..733e179 100644
--- a/src/java/org/apache/cassandra/serializers/EmptySerializer.java
+++ b/src/java/org/apache/cassandra/serializers/EmptySerializer.java
@@ -40,11 +40,7 @@
public void validate(ByteBuffer bytes) throws MarshalException
{
if (bytes.remaining() > 0)
- {
- throw new MarshalException("EmptyType only accept empty values. " +
- "A non-empty value can be a result of a Thrift write into CQL-created dense table. " +
- "See CASSANDRA-15778 for details.");
- }
+ throw new MarshalException("EmptyType only accept empty values");
}
public String toString(Void value)
diff --git a/src/java/org/apache/cassandra/serializers/ListSerializer.java b/src/java/org/apache/cassandra/serializers/ListSerializer.java
index 44c33a6..dd0bc9e 100644
--- a/src/java/org/apache/cassandra/serializers/ListSerializer.java
+++ b/src/java/org/apache/cassandra/serializers/ListSerializer.java
@@ -18,27 +18,27 @@
package org.apache.cassandra.serializers;
-import org.apache.cassandra.transport.ProtocolVersion;
-
import java.nio.BufferUnderflowException;
import java.nio.ByteBuffer;
import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.transport.ProtocolVersion;
public class ListSerializer<T> extends CollectionSerializer<List<T>>
{
// interning instances
- private static final Map<TypeSerializer<?>, ListSerializer> instances = new HashMap<TypeSerializer<?>, ListSerializer>();
+ private static final ConcurrentMap<TypeSerializer<?>, ListSerializer> instances = new ConcurrentHashMap<TypeSerializer<?>, ListSerializer>();
public final TypeSerializer<T> elements;
- public static synchronized <T> ListSerializer<T> getInstance(TypeSerializer<T> elements)
+ public static <T> ListSerializer<T> getInstance(TypeSerializer<T> elements)
{
ListSerializer<T> t = instances.get(elements);
if (t == null)
- {
- t = new ListSerializer<T>(elements);
- instances.put(elements, t);
- }
+ t = instances.computeIfAbsent(elements, k -> new ListSerializer<>(k) );
return t;
}
@@ -168,4 +168,22 @@
{
return (Class) List.class;
}
+
+ @Override
+ public ByteBuffer getSerializedValue(ByteBuffer collection, ByteBuffer key, AbstractType<?> comparator)
+ {
+ // We don't allow selecting an element of a list so we don't need this.
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public ByteBuffer getSliceFromSerialized(ByteBuffer collection,
+ ByteBuffer from,
+ ByteBuffer to,
+ AbstractType<?> comparator,
+ boolean frozen)
+ {
+ // We don't allow slicing of list so we don't need this.
+ throw new UnsupportedOperationException();
+ }
}
diff --git a/src/java/org/apache/cassandra/serializers/MapSerializer.java b/src/java/org/apache/cassandra/serializers/MapSerializer.java
index 232a1b0..726d8a7 100644
--- a/src/java/org/apache/cassandra/serializers/MapSerializer.java
+++ b/src/java/org/apache/cassandra/serializers/MapSerializer.java
@@ -21,29 +21,29 @@
import java.nio.BufferUnderflowException;
import java.nio.ByteBuffer;
import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.transport.ProtocolVersion;
+import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.Pair;
public class MapSerializer<K, V> extends CollectionSerializer<Map<K, V>>
{
// interning instances
- private static final Map<Pair<TypeSerializer<?>, TypeSerializer<?>>, MapSerializer> instances = new HashMap<Pair<TypeSerializer<?>, TypeSerializer<?>>, MapSerializer>();
+ private static final ConcurrentMap<Pair<TypeSerializer<?>, TypeSerializer<?>>, MapSerializer> instances = new ConcurrentHashMap<Pair<TypeSerializer<?>, TypeSerializer<?>>, MapSerializer>();
public final TypeSerializer<K> keys;
public final TypeSerializer<V> values;
private final Comparator<Pair<ByteBuffer, ByteBuffer>> comparator;
- public static synchronized <K, V> MapSerializer<K, V> getInstance(TypeSerializer<K> keys, TypeSerializer<V> values, Comparator<ByteBuffer> comparator)
+ public static <K, V> MapSerializer<K, V> getInstance(TypeSerializer<K> keys, TypeSerializer<V> values, Comparator<ByteBuffer> comparator)
{
Pair<TypeSerializer<?>, TypeSerializer<?>> p = Pair.<TypeSerializer<?>, TypeSerializer<?>>create(keys, values);
MapSerializer<K, V> t = instances.get(p);
if (t == null)
- {
- t = new MapSerializer<K, V>(keys, values, comparator);
- instances.put(p, t);
- }
+ t = instances.computeIfAbsent(p, k -> new MapSerializer<>(k.left, k.right, comparator) );
return t;
}
@@ -133,29 +133,24 @@
}
}
- /**
- * Given a serialized map, gets the value associated with a given key.
- * @param serializedMap a serialized map
- * @param serializedKey a serialized key
- * @param keyType the key type for the map
- * @return the value associated with the key if one exists, null otherwise
- */
- public ByteBuffer getSerializedValue(ByteBuffer serializedMap, ByteBuffer serializedKey, AbstractType keyType)
+ @Override
+ public ByteBuffer getSerializedValue(ByteBuffer collection, ByteBuffer key, AbstractType<?> comparator)
{
try
{
- ByteBuffer input = serializedMap.duplicate();
+ ByteBuffer input = collection.duplicate();
int n = readCollectionSize(input, ProtocolVersion.V3);
for (int i = 0; i < n; i++)
{
ByteBuffer kbb = readValue(input, ProtocolVersion.V3);
- ByteBuffer vbb = readValue(input, ProtocolVersion.V3);
- int comparison = keyType.compare(kbb, serializedKey);
+ int comparison = comparator.compareForCQL(kbb, key);
if (comparison == 0)
- return vbb;
+ return readValue(input, ProtocolVersion.V3);
else if (comparison > 0)
// since the map is in sorted order, we know we've gone too far and the element doesn't exist
return null;
+ else // comparison < 0
+ skipValue(input, ProtocolVersion.V3);
}
return null;
}
@@ -165,6 +160,76 @@
}
}
+ @Override
+ public ByteBuffer getSliceFromSerialized(ByteBuffer collection,
+ ByteBuffer from,
+ ByteBuffer to,
+ AbstractType<?> comparator,
+ boolean frozen)
+ {
+ if (from == ByteBufferUtil.UNSET_BYTE_BUFFER && to == ByteBufferUtil.UNSET_BYTE_BUFFER)
+ return collection;
+
+ try
+ {
+ ByteBuffer input = collection.duplicate();
+ int n = readCollectionSize(input, ProtocolVersion.V3);
+ int startPos = input.position();
+ int count = 0;
+ boolean inSlice = from == ByteBufferUtil.UNSET_BYTE_BUFFER;
+
+ for (int i = 0; i < n; i++)
+ {
+ int pos = input.position();
+ ByteBuffer kbb = readValue(input, ProtocolVersion.V3); // key
+
+ // If we haven't passed the start already, check if we have now
+ if (!inSlice)
+ {
+ int comparison = comparator.compareForCQL(from, kbb);
+ if (comparison <= 0)
+ {
+ // We're now within the slice
+ inSlice = true;
+ startPos = pos;
+ }
+ else
+ {
+ // We're before the slice so we know we don't care about this element
+ skipValue(input, ProtocolVersion.V3); // value
+ continue;
+ }
+ }
+
+ // Now check if we're done
+ int comparison = to == ByteBufferUtil.UNSET_BYTE_BUFFER ? -1 : comparator.compareForCQL(kbb, to);
+ if (comparison > 0)
+ {
+ // We're done and shouldn't include the key we just read
+ input.position(pos);
+ break;
+ }
+
+ // Otherwise, we'll include that element
+ skipValue(input, ProtocolVersion.V3); // value
+ ++count;
+
+ // But if we know if was the last of the slice, we break early
+ if (comparison == 0)
+ break;
+ }
+
+ if (count == 0 && !frozen)
+ return null;
+
+ return copyAsNewCollection(collection, count, startPos, input.position(), ProtocolVersion.V3);
+ }
+ catch (BufferUnderflowException e)
+ {
+ throw new MarshalException("Not enough bytes to read a map");
+ }
+ }
+
public String toString(Map<K, V> value)
{
StringBuilder sb = new StringBuilder();
diff --git a/src/java/org/apache/cassandra/serializers/SetSerializer.java b/src/java/org/apache/cassandra/serializers/SetSerializer.java
index 52286b0..8a3ac15 100644
--- a/src/java/org/apache/cassandra/serializers/SetSerializer.java
+++ b/src/java/org/apache/cassandra/serializers/SetSerializer.java
@@ -21,25 +21,27 @@
import java.nio.BufferUnderflowException;
import java.nio.ByteBuffer;
import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
import org.apache.cassandra.transport.ProtocolVersion;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.utils.ByteBufferUtil;
+
public class SetSerializer<T> extends CollectionSerializer<Set<T>>
{
// interning instances
- private static final Map<TypeSerializer<?>, SetSerializer> instances = new HashMap<TypeSerializer<?>, SetSerializer>();
+ private static final ConcurrentMap<TypeSerializer<?>, SetSerializer> instances = new ConcurrentHashMap<TypeSerializer<?>, SetSerializer>();
public final TypeSerializer<T> elements;
private final Comparator<ByteBuffer> comparator;
- public static synchronized <T> SetSerializer<T> getInstance(TypeSerializer<T> elements, Comparator<ByteBuffer> elementComparator)
+ public static <T> SetSerializer<T> getInstance(TypeSerializer<T> elements, Comparator<ByteBuffer> elementComparator)
{
SetSerializer<T> t = instances.get(elements);
if (t == null)
- {
- t = new SetSerializer<T>(elements, elementComparator);
- instances.put(elements, t);
- }
+ t = instances.computeIfAbsent(elements, k -> new SetSerializer<>(k, elementComparator) );
return t;
}
@@ -141,4 +143,98 @@
{
return (Class) Set.class;
}
+
+ @Override
+ public ByteBuffer getSerializedValue(ByteBuffer collection, ByteBuffer key, AbstractType<?> comparator)
+ {
+ try
+ {
+ ByteBuffer input = collection.duplicate();
+ int n = readCollectionSize(input, ProtocolVersion.V3);
+ for (int i = 0; i < n; i++)
+ {
+ ByteBuffer value = readValue(input, ProtocolVersion.V3);
+ int comparison = comparator.compareForCQL(value, key);
+ if (comparison == 0)
+ return value;
+ else if (comparison > 0)
+ // since the set is in sorted order, we know we've gone too far and the element doesn't exist
+ return null;
+ // else, we're before the element so continue
+ }
+ return null;
+ }
+ catch (BufferUnderflowException e)
+ {
+ throw new MarshalException("Not enough bytes to read a set");
+ }
+ }
+
+ @Override
+ public ByteBuffer getSliceFromSerialized(ByteBuffer collection,
+ ByteBuffer from,
+ ByteBuffer to,
+ AbstractType<?> comparator,
+ boolean frozen)
+ {
+ if (from == ByteBufferUtil.UNSET_BYTE_BUFFER && to == ByteBufferUtil.UNSET_BYTE_BUFFER)
+ return collection;
+
+ try
+ {
+ ByteBuffer input = collection.duplicate();
+ int n = readCollectionSize(input, ProtocolVersion.V3);
+ int startPos = input.position();
+ int count = 0;
+ boolean inSlice = from == ByteBufferUtil.UNSET_BYTE_BUFFER;
+
+ for (int i = 0; i < n; i++)
+ {
+ int pos = input.position();
+ ByteBuffer value = readValue(input, ProtocolVersion.V3);
+
+ // If we haven't passed the start already, check if we have now
+ if (!inSlice)
+ {
+ int comparison = comparator.compareForCQL(from, value);
+ if (comparison <= 0)
+ {
+ // We're now within the slice
+ inSlice = true;
+ startPos = pos;
+ }
+ else
+ {
+ // We're before the slice so we know we don't care about this value
+ continue;
+ }
+ }
+
+ // Now check if we're done
+ int comparison = to == ByteBufferUtil.UNSET_BYTE_BUFFER ? -1 : comparator.compareForCQL(value, to);
+ if (comparison > 0)
+ {
+ // We're done and shouldn't include the value we just read
+ input.position(pos);
+ break;
+ }
+
+ // Otherwise, we'll include that value
+ ++count;
+
+ // But if we know if was the last of the slice, we break early
+ if (comparison == 0)
+ break;
+ }
+
+ if (count == 0 && !frozen)
+ return null;
+
+ return copyAsNewCollection(collection, count, startPos, input.position(), ProtocolVersion.V3);
+ }
+ catch (BufferUnderflowException e)
+ {
+ throw new MarshalException("Not enough bytes to read a set");
+ }
+ }
}
diff --git a/src/java/org/apache/cassandra/serializers/SimpleDateSerializer.java b/src/java/org/apache/cassandra/serializers/SimpleDateSerializer.java
index 075094c..fbb5087 100644
--- a/src/java/org/apache/cassandra/serializers/SimpleDateSerializer.java
+++ b/src/java/org/apache/cassandra/serializers/SimpleDateSerializer.java
@@ -18,22 +18,26 @@
package org.apache.cassandra.serializers;
import java.nio.ByteBuffer;
+import java.time.Duration;
+import java.time.Instant;
+import java.time.LocalDate;
+import java.time.ZonedDateTime;
+import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeParseException;
import java.util.concurrent.TimeUnit;
import java.util.regex.Pattern;
-import org.joda.time.DateTime;
-import org.joda.time.DateTimeZone;
-import org.joda.time.LocalDate;
-import org.joda.time.format.DateTimeFormat;
-import org.joda.time.format.DateTimeFormatter;
-
import org.apache.cassandra.utils.ByteBufferUtil;
+import static java.time.ZoneOffset.UTC;
+import static java.time.format.ResolverStyle.STRICT;
+
// For byte-order comparability, we shift by Integer.MIN_VALUE and treat the data as an unsigned integer ranging from
// min date to max date w/epoch sitting in the center @ 2^31
public class SimpleDateSerializer implements TypeSerializer<Integer>
{
- private static final DateTimeFormatter formatter = DateTimeFormat.forPattern("yyyy-MM-dd").withZone(DateTimeZone.UTC);
+ private static final DateTimeFormatter formatter =
+ DateTimeFormatter.ISO_LOCAL_DATE.withZone(UTC).withResolverStyle(STRICT);
private static final long minSupportedDateMillis = TimeUnit.DAYS.toMillis(Integer.MIN_VALUE);
private static final long maxSupportedDateMillis = TimeUnit.DAYS.toMillis(Integer.MAX_VALUE);
private static final long maxSupportedDays = (long)Math.pow(2,32) - 1;
@@ -57,53 +61,57 @@
// Raw day value in unsigned int form, epoch @ 2^31
if (rawPattern.matcher(source).matches())
{
- try
- {
- long result = Long.parseLong(source);
-
- if (result < 0 || result > maxSupportedDays)
- throw new NumberFormatException("Input out of bounds: " + source);
-
- // Shift > epoch days into negative portion of Integer result for byte order comparability
- if (result >= Integer.MAX_VALUE)
- result -= byteOrderShift;
-
- return (int) result;
- }
- catch (NumberFormatException e)
- {
- throw new MarshalException(String.format("Unable to make unsigned int (for date) from: '%s'", source), e);
- }
+ return parseRaw(source);
}
// Attempt to parse as date string
try
{
- DateTime parsed = formatter.parseDateTime(source);
- long millis = parsed.getMillis();
+ LocalDate parsed = formatter.parse(source, LocalDate::from);
+ long millis = parsed.atStartOfDay(UTC).toInstant().toEpochMilli();
if (millis < minSupportedDateMillis)
- throw new MarshalException(String.format("Input date %s is less than min supported date %s", source, new LocalDate(minSupportedDateMillis).toString()));
+ throw new MarshalException(String.format("Input date %s is less than min supported date %s", source,
+ ZonedDateTime.ofInstant(Instant.ofEpochMilli(minSupportedDateMillis), UTC).toString()));
if (millis > maxSupportedDateMillis)
- throw new MarshalException(String.format("Input date %s is greater than max supported date %s", source, new LocalDate(maxSupportedDateMillis).toString()));
+ throw new MarshalException(String.format("Input date %s is greater than max supported date %s", source,
+ ZonedDateTime.ofInstant(Instant.ofEpochMilli(maxSupportedDateMillis), UTC).toString()));
return timeInMillisToDay(millis);
}
- catch (IllegalArgumentException e1)
+ catch (DateTimeParseException| ArithmeticException e1)
{
throw new MarshalException(String.format("Unable to coerce '%s' to a formatted date (long)", source), e1);
}
}
+ private static int parseRaw(String source) {
+ try
+ {
+ long result = Long.parseLong(source);
+
+ if (result < 0 || result > maxSupportedDays)
+ throw new NumberFormatException("Input out of bounds: " + source);
+
+ // Shift > epoch days into negative portion of Integer result for byte order comparability
+ if (result >= Integer.MAX_VALUE)
+ result -= byteOrderShift;
+
+ return (int) result;
+ }
+ catch (NumberFormatException | DateTimeParseException e)
+ {
+ throw new MarshalException(String.format("Unable to make unsigned int (for date) from: '%s'", source), e);
+ }
+ }
+
public static int timeInMillisToDay(long millis)
{
- Integer result = (int) TimeUnit.MILLISECONDS.toDays(millis);
- result -= Integer.MIN_VALUE;
- return result;
+ return (int) (Duration.ofMillis(millis).toDays() - Integer.MIN_VALUE);
}
public static long dayToTimeInMillis(int days)
{
- return TimeUnit.DAYS.toMillis(days - Integer.MIN_VALUE);
+ return Duration.ofDays(days + Integer.MIN_VALUE).toMillis();
}
public void validate(ByteBuffer bytes) throws MarshalException
@@ -117,7 +125,7 @@
if (value == null)
return "";
- return formatter.print(new LocalDate(dayToTimeInMillis(value), DateTimeZone.UTC));
+ return Instant.ofEpochMilli(dayToTimeInMillis(value)).atZone(UTC).format(formatter);
}
public Class<Integer> getType()
diff --git a/src/java/org/apache/cassandra/service/AbstractReadExecutor.java b/src/java/org/apache/cassandra/service/AbstractReadExecutor.java
deleted file mode 100644
index 572ae6e..0000000
--- a/src/java/org/apache/cassandra/service/AbstractReadExecutor.java
+++ /dev/null
@@ -1,344 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.service;
-
-import java.net.InetAddress;
-import java.util.Collection;
-import java.util.List;
-import java.util.concurrent.TimeUnit;
-
-import com.google.common.collect.Iterables;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.cassandra.concurrent.Stage;
-import org.apache.cassandra.concurrent.StageManager;
-import org.apache.cassandra.config.ReadRepairDecision;
-import org.apache.cassandra.db.ColumnFamilyStore;
-import org.apache.cassandra.db.ConsistencyLevel;
-import org.apache.cassandra.db.ReadCommand;
-import org.apache.cassandra.db.SinglePartitionReadCommand;
-import org.apache.cassandra.db.Keyspace;
-import org.apache.cassandra.db.partitions.PartitionIterator;
-import org.apache.cassandra.exceptions.ReadFailureException;
-import org.apache.cassandra.exceptions.ReadTimeoutException;
-import org.apache.cassandra.exceptions.UnavailableException;
-import org.apache.cassandra.metrics.ReadRepairMetrics;
-import org.apache.cassandra.net.MessageOut;
-import org.apache.cassandra.net.MessagingService;
-import org.apache.cassandra.schema.SpeculativeRetryParam;
-import org.apache.cassandra.service.StorageProxy.LocalReadRunnable;
-import org.apache.cassandra.tracing.TraceState;
-import org.apache.cassandra.tracing.Tracing;
-
-/**
- * Sends a read request to the replicas needed to satisfy a given ConsistencyLevel.
- *
- * Optionally, may perform additional requests to provide redundancy against replica failure:
- * AlwaysSpeculatingReadExecutor will always send a request to one extra replica, while
- * SpeculatingReadExecutor will wait until it looks like the original request is in danger
- * of timing out before performing extra reads.
- */
-public abstract class AbstractReadExecutor
-{
- private static final Logger logger = LoggerFactory.getLogger(AbstractReadExecutor.class);
-
- protected final ReadCommand command;
- protected final List<InetAddress> targetReplicas;
- protected final ReadCallback handler;
- protected final TraceState traceState;
-
- AbstractReadExecutor(Keyspace keyspace, ReadCommand command, ConsistencyLevel consistencyLevel, List<InetAddress> targetReplicas, long queryStartNanoTime)
- {
- this.command = command;
- this.targetReplicas = targetReplicas;
- this.handler = new ReadCallback(new DigestResolver(keyspace, command, consistencyLevel, targetReplicas.size()), consistencyLevel, command, targetReplicas, queryStartNanoTime);
- this.traceState = Tracing.instance.get();
-
- // Set the digest version (if we request some digests). This is the smallest version amongst all our target replicas since new nodes
- // knows how to produce older digest but the reverse is not true.
- // TODO: we need this when talking with pre-3.0 nodes. So if we preserve the digest format moving forward, we can get rid of this once
- // we stop being compatible with pre-3.0 nodes.
- int digestVersion = MessagingService.current_version;
- for (InetAddress replica : targetReplicas)
- digestVersion = Math.min(digestVersion, MessagingService.instance().getVersion(replica));
- command.setDigestVersion(digestVersion);
- }
-
- protected void makeDataRequests(Iterable<InetAddress> endpoints)
- {
- makeRequests(command, endpoints);
-
- }
-
- protected void makeDigestRequests(Iterable<InetAddress> endpoints)
- {
- makeRequests(command.copyAsDigestQuery(), endpoints);
- }
-
- private void makeRequests(ReadCommand readCommand, Iterable<InetAddress> endpoints)
- {
- boolean hasLocalEndpoint = false;
-
- for (InetAddress endpoint : endpoints)
- {
- if (StorageProxy.canDoLocalRequest(endpoint))
- {
- hasLocalEndpoint = true;
- continue;
- }
-
- if (traceState != null)
- traceState.trace("reading {} from {}", readCommand.isDigestQuery() ? "digest" : "data", endpoint);
- logger.trace("reading {} from {}", readCommand.isDigestQuery() ? "digest" : "data", endpoint);
- MessageOut<ReadCommand> message = readCommand.createMessage(MessagingService.instance().getVersion(endpoint));
- MessagingService.instance().sendRRWithFailure(message, endpoint, handler);
- }
-
- // We delay the local (potentially blocking) read till the end to avoid stalling remote requests.
- if (hasLocalEndpoint)
- {
- logger.trace("reading {} locally", readCommand.isDigestQuery() ? "digest" : "data");
- StageManager.getStage(Stage.READ).maybeExecuteImmediately(new LocalReadRunnable(command, handler));
- }
- }
-
- /**
- * Perform additional requests if it looks like the original will time out. May block while it waits
- * to see if the original requests are answered first.
- */
- public abstract void maybeTryAdditionalReplicas();
-
- /**
- * Get the replicas involved in the [finished] request.
- *
- * @return target replicas + the extra replica, *IF* we speculated.
- */
- public abstract Collection<InetAddress> getContactedReplicas();
-
- /**
- * send the initial set of requests
- */
- public abstract void executeAsync();
-
- /**
- * wait for an answer. Blocks until success or timeout, so it is caller's
- * responsibility to call maybeTryAdditionalReplicas first.
- */
- public PartitionIterator get() throws ReadFailureException, ReadTimeoutException, DigestMismatchException
- {
- return handler.get();
- }
-
- /**
- * @return an executor appropriate for the configured speculative read policy
- */
- public static AbstractReadExecutor getReadExecutor(SinglePartitionReadCommand command, ConsistencyLevel consistencyLevel, long queryStartNanoTime) throws UnavailableException
- {
- Keyspace keyspace = Keyspace.open(command.metadata().ksName);
- List<InetAddress> allReplicas = StorageProxy.getLiveSortedEndpoints(keyspace, command.partitionKey());
- // 11980: Excluding EACH_QUORUM reads from potential RR, so that we do not miscount DC responses
- ReadRepairDecision repairDecision = consistencyLevel == ConsistencyLevel.EACH_QUORUM
- ? ReadRepairDecision.NONE
- : command.metadata().newReadRepairDecision();
- List<InetAddress> targetReplicas = consistencyLevel.filterForQuery(keyspace, allReplicas, repairDecision);
-
- // Throw UAE early if we don't have enough replicas.
- consistencyLevel.assureSufficientLiveNodes(keyspace, targetReplicas);
-
- if (repairDecision != ReadRepairDecision.NONE)
- {
- Tracing.trace("Read-repair {}", repairDecision);
- ReadRepairMetrics.attempted.mark();
- }
-
- ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(command.metadata().cfId);
- SpeculativeRetryParam retry = cfs.metadata.params.speculativeRetry;
-
- // Speculative retry is disabled *OR* there are simply no extra replicas to speculate.
- // 11980: Disable speculative retry if using EACH_QUORUM in order to prevent miscounting DC responses
- if (retry.equals(SpeculativeRetryParam.NONE)
- || consistencyLevel == ConsistencyLevel.EACH_QUORUM
- || consistencyLevel.blockFor(keyspace) == allReplicas.size())
- return new NeverSpeculatingReadExecutor(keyspace, command, consistencyLevel, targetReplicas, queryStartNanoTime);
-
- if (targetReplicas.size() == allReplicas.size())
- {
- // CL.ALL, RRD.GLOBAL or RRD.DC_LOCAL and a single-DC.
- // We are going to contact every node anyway, so ask for 2 full data requests instead of 1, for redundancy
- // (same amount of requests in total, but we turn 1 digest request into a full blown data request).
- return new AlwaysSpeculatingReadExecutor(keyspace, cfs, command, consistencyLevel, targetReplicas, queryStartNanoTime);
- }
-
- // RRD.NONE or RRD.DC_LOCAL w/ multiple DCs.
- InetAddress extraReplica = allReplicas.get(targetReplicas.size());
- // With repair decision DC_LOCAL all replicas/target replicas may be in different order, so
- // we might have to find a replacement that's not already in targetReplicas.
- if (repairDecision == ReadRepairDecision.DC_LOCAL && targetReplicas.contains(extraReplica))
- {
- for (InetAddress address : allReplicas)
- {
- if (!targetReplicas.contains(address))
- {
- extraReplica = address;
- break;
- }
- }
- }
- targetReplicas.add(extraReplica);
-
- if (retry.equals(SpeculativeRetryParam.ALWAYS))
- return new AlwaysSpeculatingReadExecutor(keyspace, cfs, command, consistencyLevel, targetReplicas, queryStartNanoTime);
- else // PERCENTILE or CUSTOM.
- return new SpeculatingReadExecutor(keyspace, cfs, command, consistencyLevel, targetReplicas, queryStartNanoTime);
- }
-
- public static class NeverSpeculatingReadExecutor extends AbstractReadExecutor
- {
- public NeverSpeculatingReadExecutor(Keyspace keyspace, ReadCommand command, ConsistencyLevel consistencyLevel, List<InetAddress> targetReplicas, long queryStartNanoTime)
- {
- super(keyspace, command, consistencyLevel, targetReplicas, queryStartNanoTime);
- }
-
- public void executeAsync()
- {
- makeDataRequests(targetReplicas.subList(0, 1));
- if (targetReplicas.size() > 1)
- makeDigestRequests(targetReplicas.subList(1, targetReplicas.size()));
- }
-
- public void maybeTryAdditionalReplicas()
- {
- // no-op
- }
-
- public Collection<InetAddress> getContactedReplicas()
- {
- return targetReplicas;
- }
- }
-
- private static class SpeculatingReadExecutor extends AbstractReadExecutor
- {
- private final ColumnFamilyStore cfs;
- private volatile boolean speculated = false;
-
- public SpeculatingReadExecutor(Keyspace keyspace,
- ColumnFamilyStore cfs,
- ReadCommand command,
- ConsistencyLevel consistencyLevel,
- List<InetAddress> targetReplicas,
- long queryStartNanoTime)
- {
- super(keyspace, command, consistencyLevel, targetReplicas, queryStartNanoTime);
- this.cfs = cfs;
- }
-
- public void executeAsync()
- {
- // if CL + RR result in covering all replicas, getReadExecutor forces AlwaysSpeculating. So we know
- // that the last replica in our list is "extra."
- List<InetAddress> initialReplicas = targetReplicas.subList(0, targetReplicas.size() - 1);
-
- if (handler.blockfor < initialReplicas.size())
- {
- // We're hitting additional targets for read repair. Since our "extra" replica is the least-
- // preferred by the snitch, we do an extra data read to start with against a replica more
- // likely to reply; better to let RR fail than the entire query.
- makeDataRequests(initialReplicas.subList(0, 2));
- if (initialReplicas.size() > 2)
- makeDigestRequests(initialReplicas.subList(2, initialReplicas.size()));
- }
- else
- {
- // not doing read repair; all replies are important, so it doesn't matter which nodes we
- // perform data reads against vs digest.
- makeDataRequests(initialReplicas.subList(0, 1));
- if (initialReplicas.size() > 1)
- makeDigestRequests(initialReplicas.subList(1, initialReplicas.size()));
- }
- }
-
- public void maybeTryAdditionalReplicas()
- {
- // no latency information, or we're overloaded
- if (cfs.sampleLatencyNanos > TimeUnit.MILLISECONDS.toNanos(command.getTimeout()))
- return;
-
- if (!handler.await(cfs.sampleLatencyNanos, TimeUnit.NANOSECONDS))
- {
- // Could be waiting on the data, or on enough digests.
- ReadCommand retryCommand = command;
- if (handler.resolver.isDataPresent())
- retryCommand = command.copyAsDigestQuery();
-
- InetAddress extraReplica = Iterables.getLast(targetReplicas);
- if (traceState != null)
- traceState.trace("speculating read retry on {}", extraReplica);
- logger.trace("speculating read retry on {}", extraReplica);
- int version = MessagingService.instance().getVersion(extraReplica);
- MessagingService.instance().sendRRWithFailure(retryCommand.createMessage(version), extraReplica, handler);
- speculated = true;
-
- cfs.metric.speculativeRetries.inc();
- }
- }
-
- public Collection<InetAddress> getContactedReplicas()
- {
- return speculated
- ? targetReplicas
- : targetReplicas.subList(0, targetReplicas.size() - 1);
- }
- }
-
- private static class AlwaysSpeculatingReadExecutor extends AbstractReadExecutor
- {
- private final ColumnFamilyStore cfs;
-
- public AlwaysSpeculatingReadExecutor(Keyspace keyspace,
- ColumnFamilyStore cfs,
- ReadCommand command,
- ConsistencyLevel consistencyLevel,
- List<InetAddress> targetReplicas,
- long queryStartNanoTime)
- {
- super(keyspace, command, consistencyLevel, targetReplicas, queryStartNanoTime);
- this.cfs = cfs;
- }
-
- public void maybeTryAdditionalReplicas()
- {
- // no-op
- }
-
- public Collection<InetAddress> getContactedReplicas()
- {
- return targetReplicas;
- }
-
- @Override
- public void executeAsync()
- {
- makeDataRequests(targetReplicas.subList(0, targetReplicas.size() > 1 ? 2 : 1));
- if (targetReplicas.size() > 2)
- makeDigestRequests(targetReplicas.subList(2, targetReplicas.size()));
- cfs.metric.speculativeRetries.inc();
- }
- }
-}
diff --git a/src/java/org/apache/cassandra/service/AbstractWriteResponseHandler.java b/src/java/org/apache/cassandra/service/AbstractWriteResponseHandler.java
index 8c30b89..b1eb5b3 100644
--- a/src/java/org/apache/cassandra/service/AbstractWriteResponseHandler.java
+++ b/src/java/org/apache/cassandra/service/AbstractWriteResponseHandler.java
@@ -17,61 +17,77 @@
*/
package org.apache.cassandra.service;
-import java.net.InetAddress;
-import java.util.Collection;
+import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
+import java.util.stream.Collectors;
-import com.google.common.collect.Iterables;
+import org.apache.cassandra.db.ConsistencyLevel;
+import org.apache.cassandra.locator.EndpointsForToken;
+import org.apache.cassandra.locator.ReplicaPlan;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.db.ConsistencyLevel;
-import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.IMutation;
import org.apache.cassandra.db.WriteType;
-import org.apache.cassandra.exceptions.*;
-import org.apache.cassandra.net.IAsyncCallbackWithFailure;
-import org.apache.cassandra.net.MessageIn;
+import org.apache.cassandra.exceptions.RequestFailureReason;
+import org.apache.cassandra.exceptions.WriteFailureException;
+import org.apache.cassandra.exceptions.WriteTimeoutException;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.net.RequestCallback;
+import org.apache.cassandra.net.Message;
+import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.utils.concurrent.SimpleCondition;
-public abstract class AbstractWriteResponseHandler<T> implements IAsyncCallbackWithFailure<T>
-{
- protected static final Logger logger = LoggerFactory.getLogger( AbstractWriteResponseHandler.class );
+import static java.util.concurrent.TimeUnit.NANOSECONDS;
+
+public abstract class AbstractWriteResponseHandler<T> implements RequestCallback<T>
+{
+ protected static final Logger logger = LoggerFactory.getLogger(AbstractWriteResponseHandler.class);
+
+ //Count down until all responses and expirations have occured before deciding whether the ideal CL was reached.
+ private AtomicInteger responsesAndExpirations;
private final SimpleCondition condition = new SimpleCondition();
- protected final Keyspace keyspace;
- protected final Collection<InetAddress> naturalEndpoints;
- public final ConsistencyLevel consistencyLevel;
+ protected final ReplicaPlan.ForTokenWrite replicaPlan;
+
protected final Runnable callback;
- protected final Collection<InetAddress> pendingEndpoints;
protected final WriteType writeType;
private static final AtomicIntegerFieldUpdater<AbstractWriteResponseHandler> failuresUpdater
- = AtomicIntegerFieldUpdater.newUpdater(AbstractWriteResponseHandler.class, "failures");
+ = AtomicIntegerFieldUpdater.newUpdater(AbstractWriteResponseHandler.class, "failures");
private volatile int failures = 0;
- private final Map<InetAddress, RequestFailureReason> failureReasonByEndpoint;
+ private final Map<InetAddressAndPort, RequestFailureReason> failureReasonByEndpoint;
private final long queryStartNanoTime;
private volatile boolean supportsBackPressure = true;
/**
- * @param callback A callback to be called when the write is successful.
+ * Delegate to another WriteResponseHandler or possibly this one to track if the ideal consistency level was reached.
+ * Will be set to null if ideal CL was not configured
+ * Will be set to an AWRH delegate if ideal CL was configured
+ * Will be same as "this" if this AWRH is the ideal consistency level
+ */
+ private AbstractWriteResponseHandler idealCLDelegate;
+
+ /**
+ * We don't want to increment the writeFailedIdealCL if we didn't achieve the original requested CL
+ */
+ private boolean requestedCLAchieved = false;
+
+ /**
+ * @param callback A callback to be called when the write is successful.
* @param queryStartNanoTime
*/
- protected AbstractWriteResponseHandler(Keyspace keyspace,
- Collection<InetAddress> naturalEndpoints,
- Collection<InetAddress> pendingEndpoints,
- ConsistencyLevel consistencyLevel,
+ protected AbstractWriteResponseHandler(ReplicaPlan.ForTokenWrite replicaPlan,
Runnable callback,
WriteType writeType,
long queryStartNanoTime)
{
- this.keyspace = keyspace;
- this.pendingEndpoints = pendingEndpoints;
- this.consistencyLevel = consistencyLevel;
- this.naturalEndpoints = naturalEndpoints;
+ this.replicaPlan = replicaPlan;
this.callback = callback;
this.writeType = writeType;
this.failureReasonByEndpoint = new ConcurrentHashMap<>();
@@ -80,12 +96,12 @@
public void get() throws WriteTimeoutException, WriteFailureException
{
- long timeout = currentTimeout();
+ long timeoutNanos = currentTimeoutNanos();
boolean success;
try
{
- success = condition.await(timeout, TimeUnit.NANOSECONDS);
+ success = condition.await(timeoutNanos, NANOSECONDS);
}
catch (InterruptedException ex)
{
@@ -94,52 +110,117 @@
if (!success)
{
- int blockedFor = totalBlockFor();
+ int blockedFor = blockFor();
int acks = ackCount();
// It's pretty unlikely, but we can race between exiting await above and here, so
// that we could now have enough acks. In that case, we "lie" on the acks count to
// avoid sending confusing info to the user (see CASSANDRA-6491).
if (acks >= blockedFor)
acks = blockedFor - 1;
- throw new WriteTimeoutException(writeType, consistencyLevel, acks, blockedFor);
+ throw new WriteTimeoutException(writeType, replicaPlan.consistencyLevel(), acks, blockedFor);
}
- if (totalBlockFor() + failures > totalEndpoints())
+ if (blockFor() + failures > candidateReplicaCount())
{
- throw new WriteFailureException(consistencyLevel, ackCount(), totalBlockFor(), writeType, failureReasonByEndpoint);
+ throw new WriteFailureException(replicaPlan.consistencyLevel(), ackCount(), blockFor(), writeType, failureReasonByEndpoint);
}
}
- public final long currentTimeout()
+ public final long currentTimeoutNanos()
{
long requestTimeout = writeType == WriteType.COUNTER
- ? DatabaseDescriptor.getCounterWriteRpcTimeout()
- : DatabaseDescriptor.getWriteRpcTimeout();
- return TimeUnit.MILLISECONDS.toNanos(requestTimeout) - (System.nanoTime() - queryStartNanoTime);
+ ? DatabaseDescriptor.getCounterWriteRpcTimeout(NANOSECONDS)
+ : DatabaseDescriptor.getWriteRpcTimeout(NANOSECONDS);
+ return requestTimeout - (System.nanoTime() - queryStartNanoTime);
}
/**
- * @return the minimum number of endpoints that must reply.
+ * Set a delegate ideal CL write response handler. Note that this could be the same as this
+ * if the ideal CL and requested CL are the same.
*/
- protected int totalBlockFor()
+ public void setIdealCLResponseHandler(AbstractWriteResponseHandler handler)
+ {
+ this.idealCLDelegate = handler;
+ idealCLDelegate.responsesAndExpirations = new AtomicInteger(replicaPlan.contacts().size());
+ }
+
+ /**
+ * This logs the response but doesn't do any further processing related to this write response handler
+ * on whether the CL was achieved. Only call this after the subclass has completed all it's processing
+ * since the subclass instance may be queried to find out if the CL was achieved.
+ */
+ protected final void logResponseToIdealCLDelegate(Message<T> m)
+ {
+ //Tracking ideal CL was not configured
+ if (idealCLDelegate == null)
+ {
+ return;
+ }
+
+ if (idealCLDelegate == this)
+ {
+ //Processing of the message was already done since this is the handler for the
+ //ideal consistency level. Just decrement the counter.
+ decrementResponseOrExpired();
+ }
+ else
+ {
+ //Let the delegate do full processing, this will loop back into the branch above
+ //with idealCLDelegate == this, because the ideal write handler idealCLDelegate will always
+ //be set to this in the delegate.
+ idealCLDelegate.onResponse(m);
+ }
+ }
+
+ public final void expired()
+ {
+ //Tracking ideal CL was not configured
+ if (idealCLDelegate == null)
+ {
+ return;
+ }
+
+ //The requested CL matched ideal CL so reuse this object
+ if (idealCLDelegate == this)
+ {
+ decrementResponseOrExpired();
+ }
+ else
+ {
+ //Have the delegate track the expired response
+ idealCLDelegate.decrementResponseOrExpired();
+ }
+ }
+
+ /**
+ * @return the minimum number of endpoints that must respond.
+ */
+ protected int blockFor()
{
// During bootstrap, we have to include the pending endpoints or we may fail the consistency level
// guarantees (see #833)
- return consistencyLevel.blockFor(keyspace) + pendingEndpoints.size();
+ return replicaPlan.blockFor();
}
/**
- * @return the total number of endpoints the request has been sent to.
+ * TODO: this method is brittle for its purpose of deciding when we should fail a query;
+ * this needs to be CL aware, and of which nodes are live/down
+ * @return the total number of endpoints the request can been sent to.
*/
- protected int totalEndpoints()
+ protected int candidateReplicaCount()
{
- return naturalEndpoints.size() + pendingEndpoints.size();
+ return replicaPlan.liveAndDown().size();
+ }
+
+ public ConsistencyLevel consistencyLevel()
+ {
+ return replicaPlan.consistencyLevel();
}
/**
- * @return true if the message counts towards the totalBlockFor() threshold
+ * @return true if the message counts towards the blockFor() threshold
*/
- protected boolean waitingFor(InetAddress from)
+ protected boolean waitingFor(InetAddressAndPort from)
{
return true;
}
@@ -149,37 +230,47 @@
*/
protected abstract int ackCount();
- /** null message means "response from local write" */
- public abstract void response(MessageIn<T> msg);
-
- public void assureSufficientLiveNodes() throws UnavailableException
- {
- consistencyLevel.assureSufficientLiveNodes(keyspace, Iterables.filter(Iterables.concat(naturalEndpoints, pendingEndpoints), isAlive));
- }
+ /**
+ * null message means "response from local write"
+ */
+ public abstract void onResponse(Message<T> msg);
protected void signal()
{
+ //The ideal CL should only count as a strike if the requested CL was achieved.
+ //If the requested CL is not achieved it's fine for the ideal CL to also not be achieved.
+ if (idealCLDelegate != null)
+ {
+ idealCLDelegate.requestedCLAchieved = true;
+ }
+
condition.signalAll();
if (callback != null)
callback.run();
}
@Override
- public void onFailure(InetAddress from, RequestFailureReason failureReason)
+ public void onFailure(InetAddressAndPort from, RequestFailureReason failureReason)
{
logger.trace("Got failure from {}", from);
int n = waitingFor(from)
- ? failuresUpdater.incrementAndGet(this)
- : failures;
+ ? failuresUpdater.incrementAndGet(this)
+ : failures;
failureReasonByEndpoint.put(from, failureReason);
- if (totalBlockFor() + n > totalEndpoints())
+ if (blockFor() + n > candidateReplicaCount())
signal();
}
@Override
+ public boolean invokeOnFailure()
+ {
+ return true;
+ }
+
+ @Override
public boolean supportsBackPressure()
{
return supportsBackPressure;
@@ -189,4 +280,65 @@
{
this.supportsBackPressure = supportsBackPressure;
}
+
+ /**
+ * Decrement the counter for all responses/expirations and if the counter
+ * hits 0 check to see if the ideal consistency level (this write response handler)
+ * was reached using the signal.
+ */
+ private final void decrementResponseOrExpired()
+ {
+ int decrementedValue = responsesAndExpirations.decrementAndGet();
+ if (decrementedValue == 0)
+ {
+ // The condition being signaled is a valid proxy for the CL being achieved
+ // Only mark it as failed if the requested CL was achieved.
+ if (!condition.isSignaled() && requestedCLAchieved)
+ {
+ replicaPlan.keyspace().metric.writeFailedIdealCL.inc();
+ }
+ else
+ {
+ replicaPlan.keyspace().metric.idealCLWriteLatency.addNano(System.nanoTime() - queryStartNanoTime);
+ }
+ }
+ }
+
+ /**
+ * Cheap Quorum backup. If we failed to reach quorum with our initial (full) nodes, reach out to other nodes.
+ */
+ public void maybeTryAdditionalReplicas(IMutation mutation, StorageProxy.WritePerformer writePerformer, String localDC)
+ {
+ EndpointsForToken uncontacted = replicaPlan.liveUncontacted();
+ if (uncontacted.isEmpty())
+ return;
+
+ long timeout = Long.MAX_VALUE;
+ List<ColumnFamilyStore> cfs = mutation.getTableIds().stream()
+ .map(Schema.instance::getColumnFamilyStoreInstance)
+ .collect(Collectors.toList());
+ for (ColumnFamilyStore cf : cfs)
+ timeout = Math.min(timeout, cf.additionalWriteLatencyNanos);
+
+ // no latency information, or we're overloaded
+ if (timeout > mutation.getTimeout(NANOSECONDS))
+ return;
+
+ try
+ {
+ if (!condition.await(timeout, NANOSECONDS))
+ {
+ for (ColumnFamilyStore cf : cfs)
+ cf.metric.additionalWrites.inc();
+
+ writePerformer.apply(mutation, replicaPlan.withContact(uncontacted),
+ (AbstractWriteResponseHandler<IMutation>) this,
+ localDC);
+ }
+ }
+ catch (InterruptedException ex)
+ {
+ throw new AssertionError(ex);
+ }
+ }
}
diff --git a/src/java/org/apache/cassandra/service/ActiveRepairService.java b/src/java/org/apache/cassandra/service/ActiveRepairService.java
index 626aa91..d386d37 100644
--- a/src/java/org/apache/cassandra/service/ActiveRepairService.java
+++ b/src/java/org/apache/cassandra/service/ActiveRepairService.java
@@ -18,30 +18,34 @@
package org.apache.cassandra.service;
import java.io.IOException;
-import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.*;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicBoolean;
-import com.google.common.base.Predicate;
-import com.google.common.collect.ImmutableMap;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
import com.google.common.collect.Multimap;
-import com.google.common.collect.Sets;
import com.google.common.util.concurrent.AbstractFuture;
-import com.google.common.util.concurrent.Futures;
-import com.google.common.util.concurrent.ListenableFuture;
+
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.common.util.concurrent.MoreExecutors;
+import org.apache.cassandra.locator.EndpointsByRange;
+import org.apache.cassandra.locator.EndpointsForRange;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.cassandra.concurrent.JMXEnabledThreadPoolExecutor;
+import org.apache.cassandra.concurrent.NamedThreadFactory;
+import org.apache.cassandra.concurrent.ScheduledExecutors;
+import org.apache.cassandra.config.Config;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.ColumnFamilyStore;
-import org.apache.cassandra.db.compaction.CompactionManager;
-import org.apache.cassandra.db.lifecycle.SSTableSet;
-import org.apache.cassandra.db.lifecycle.View;
-import org.apache.cassandra.dht.Bounds;
+import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.exceptions.RequestFailureReason;
@@ -53,23 +57,29 @@
import org.apache.cassandra.gms.IEndpointStateChangeSubscriber;
import org.apache.cassandra.gms.IFailureDetectionEventListener;
import org.apache.cassandra.gms.VersionedValue;
-import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.locator.TokenMetadata;
-import org.apache.cassandra.net.IAsyncCallbackWithFailure;
-import org.apache.cassandra.net.MessageIn;
-import org.apache.cassandra.net.MessageOut;
+import org.apache.cassandra.metrics.RepairMetrics;
+import org.apache.cassandra.net.RequestCallback;
+import org.apache.cassandra.net.Message;
import org.apache.cassandra.net.MessagingService;
-import org.apache.cassandra.repair.AnticompactionTask;
+import org.apache.cassandra.repair.CommonRange;
+import org.apache.cassandra.streaming.PreviewKind;
import org.apache.cassandra.repair.RepairJobDesc;
import org.apache.cassandra.repair.RepairParallelism;
import org.apache.cassandra.repair.RepairSession;
+import org.apache.cassandra.repair.consistent.CoordinatorSessions;
+import org.apache.cassandra.repair.consistent.LocalSessions;
import org.apache.cassandra.repair.messages.*;
-import org.apache.cassandra.utils.CassandraVersion;
-import org.apache.cassandra.utils.Clock;
+import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.MBeanWrapper;
+import org.apache.cassandra.utils.Pair;
import org.apache.cassandra.utils.UUIDGen;
-import org.apache.cassandra.utils.concurrent.Ref;
-import org.apache.cassandra.utils.concurrent.Refs;
+
+import static com.google.common.collect.Iterables.concat;
+import static com.google.common.collect.Iterables.transform;
+import static org.apache.cassandra.net.Verb.PREPARE_MSG;
/**
* ActiveRepairService is the starting point for manual "active" repairs.
@@ -85,27 +95,30 @@
* The creation of a repair session is done through the submitRepairSession that
* returns a future on the completion of that session.
*/
-public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFailureDetectionEventListener
+public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFailureDetectionEventListener, ActiveRepairServiceMBean
{
- /**
- * @deprecated this statuses are from the previous JMX notification service,
- * which will be deprecated on 4.0. For statuses of the new notification
- * service, see {@link org.apache.cassandra.streaming.StreamEvent.ProgressEvent}
- */
- @Deprecated
- public static enum Status
- {
- STARTED, SESSION_SUCCESS, SESSION_FAILED, FINISHED
- }
- private boolean registeredForEndpointChanges = false;
- public static CassandraVersion SUPPORTS_GLOBAL_PREPARE_FLAG_VERSION = new CassandraVersion("2.2.1");
+ public enum ParentRepairStatus
+ {
+ IN_PROGRESS, COMPLETED, FAILED
+ }
+
+ public static class ConsistentSessions
+ {
+ public final LocalSessions local = new LocalSessions();
+ public final CoordinatorSessions coordinated = new CoordinatorSessions();
+ }
+
+ public final ConsistentSessions consistent = new ConsistentSessions();
+
+ private boolean registeredForEndpointChanges = false;
private static final Logger logger = LoggerFactory.getLogger(ActiveRepairService.class);
// singleton enforcement
public static final ActiveRepairService instance = new ActiveRepairService(FailureDetector.instance, Gossiper.instance);
public static final long UNREPAIRED_SSTABLE = 0;
+ public static final UUID NO_PENDING_REPAIR = null;
/**
* A map of active coordinator session.
@@ -114,13 +127,109 @@
private final ConcurrentMap<UUID, ParentRepairSession> parentRepairSessions = new ConcurrentHashMap<>();
+ static
+ {
+ RepairMetrics.init();
+ }
+
+ public static class RepairCommandExecutorHandle
+ {
+ private static final ThreadPoolExecutor repairCommandExecutor =
+ initializeExecutor(DatabaseDescriptor.getRepairCommandPoolSize(),
+ DatabaseDescriptor.getRepairCommandPoolFullStrategy());
+ }
+
+ @VisibleForTesting
+ static ThreadPoolExecutor initializeExecutor(int maxPoolSize, Config.RepairCommandPoolFullStrategy strategy)
+ {
+ int corePoolSize = 1;
+ BlockingQueue<Runnable> queue;
+ if (strategy == Config.RepairCommandPoolFullStrategy.reject)
+ {
+ // new threads will be created on demand up to max pool
+ // size so we can leave corePoolSize at 1 to start with
+ queue = new SynchronousQueue<>();
+ }
+ else
+ {
+ // new threads are only created if > corePoolSize threads are running
+ // and the queue is full, so set corePoolSize to the desired max as the
+ // queue will _never_ be full. Idle core threads will eventually time
+ // out and may be re-created if/when subsequent tasks are submitted.
+ corePoolSize = maxPoolSize;
+ queue = new LinkedBlockingQueue<>();
+ }
+
+ ThreadPoolExecutor executor = new JMXEnabledThreadPoolExecutor(corePoolSize,
+ maxPoolSize,
+ 1,
+ TimeUnit.HOURS,
+ queue,
+ new NamedThreadFactory("Repair-Task"),
+ "internal",
+ new ThreadPoolExecutor.AbortPolicy());
+ // allow idle core threads to be terminated
+ executor.allowCoreThreadTimeOut(true);
+ return executor;
+ }
+
+ public static ThreadPoolExecutor repairCommandExecutor()
+ {
+ return RepairCommandExecutorHandle.repairCommandExecutor;
+ }
+
private final IFailureDetector failureDetector;
private final Gossiper gossiper;
+ private final Cache<Integer, Pair<ParentRepairStatus, List<String>>> repairStatusByCmd;
public ActiveRepairService(IFailureDetector failureDetector, Gossiper gossiper)
{
this.failureDetector = failureDetector;
this.gossiper = gossiper;
+ this.repairStatusByCmd = CacheBuilder.newBuilder()
+ .expireAfterWrite(
+ Long.getLong("cassandra.parent_repair_status_expiry_seconds",
+ TimeUnit.SECONDS.convert(1, TimeUnit.DAYS)), TimeUnit.SECONDS)
+ // using weight wouldn't work so well, since it doesn't reflect mutation of cached data
+ // see https://github.com/google/guava/wiki/CachesExplained
+ // We assume each entry is unlikely to be much more than 100 bytes, so bounding the size should be sufficient.
+ .maximumSize(Long.getLong("cassandra.parent_repair_status_cache_size", 100_000))
+ .build();
+
+ MBeanWrapper.instance.registerMBean(this, MBEAN_NAME);
+ }
+
+ public void start()
+ {
+ consistent.local.start();
+ ScheduledExecutors.optionalTasks.scheduleAtFixedRate(consistent.local::cleanup, 0,
+ LocalSessions.CLEANUP_INTERVAL,
+ TimeUnit.SECONDS);
+ }
+
+ @Override
+ public List<Map<String, String>> getSessions(boolean all)
+ {
+ return consistent.local.sessionInfo(all);
+ }
+
+ @Override
+ public void failSession(String session, boolean force)
+ {
+ UUID sessionID = UUID.fromString(session);
+ consistent.local.cancelSession(sessionID, force);
+ }
+
+ @Override
+ public void setRepairSessionSpaceInMegabytes(int sizeInMegabytes)
+ {
+ DatabaseDescriptor.setRepairSessionSpaceInMegabytes(sizeInMegabytes);
+ }
+
+ @Override
+ public int getRepairSessionSpaceInMegabytes()
+ {
+ return DatabaseDescriptor.getRepairSessionSpaceInMegabytes();
}
/**
@@ -129,27 +238,34 @@
* @return Future for asynchronous call or null if there is no need to repair
*/
public RepairSession submitRepairSession(UUID parentRepairSession,
- Collection<Range<Token>> range,
+ CommonRange range,
String keyspace,
RepairParallelism parallelismDegree,
- Set<InetAddress> endpoints,
- long repairedAt,
+ boolean isIncremental,
boolean pullRepair,
+ boolean force,
+ PreviewKind previewKind,
+ boolean optimiseStreams,
ListeningExecutorService executor,
String... cfnames)
{
- if (endpoints.isEmpty())
+ if (range.endpoints.isEmpty())
return null;
if (cfnames.length == 0)
return null;
- final RepairSession session = new RepairSession(parentRepairSession, UUIDGen.getTimeUUID(), range, keyspace, parallelismDegree, endpoints, repairedAt, pullRepair, cfnames);
+ final RepairSession session = new RepairSession(parentRepairSession, UUIDGen.getTimeUUID(), range, keyspace,
+ parallelismDegree, isIncremental, pullRepair, force,
+ previewKind, optimiseStreams, cfnames);
sessions.put(session.getId(), session);
// register listeners
registerOnFdAndGossip(session);
+ if (session.previewKind == PreviewKind.REPAIRED)
+ LocalSessions.registerListener(session);
+
// remove session at completion
session.addListener(new Runnable()
{
@@ -159,12 +275,23 @@
public void run()
{
sessions.remove(session.getId());
+ LocalSessions.unregisterListener(session);
}
}, MoreExecutors.directExecutor());
session.start(executor);
return session;
}
+ public boolean getUseOffheapMerkleTrees()
+ {
+ return DatabaseDescriptor.useOffheapMerkleTrees();
+ }
+
+ public void setUseOffheapMerkleTrees(boolean value)
+ {
+ DatabaseDescriptor.useOffheapMerkleTrees(value);
+ }
+
private <T extends AbstractFuture &
IEndpointStateChangeSubscriber &
IFailureDetectionEventListener> void registerOnFdAndGossip(final T task)
@@ -183,7 +310,7 @@
failureDetector.unregisterFailureDetectionEventListener(task);
gossiper.unregister(task);
}
- }, MoreExecutors.sameThreadExecutor());
+ }, MoreExecutors.directExecutor());
}
public synchronized void terminateSessions()
@@ -196,6 +323,17 @@
parentRepairSessions.clear();
}
+ public void recordRepairStatus(int cmd, ParentRepairStatus parentRepairStatus, List<String> messages)
+ {
+ repairStatusByCmd.put(cmd, Pair.create(parentRepairStatus, messages));
+ }
+
+
+ Pair<ParentRepairStatus, List<String>> getRepairStatus(Integer cmd)
+ {
+ return repairStatusByCmd.getIfPresent(cmd);
+ }
+
/**
* Return all of the neighbors with whom we share the provided range.
*
@@ -206,12 +344,12 @@
*
* @return neighbors with whom we share the provided range
*/
- public static Set<InetAddress> getNeighbors(String keyspaceName, Collection<Range<Token>> keyspaceLocalRanges,
- Range<Token> toRepair, Collection<String> dataCenters,
- Collection<String> hosts)
+ public static EndpointsForRange getNeighbors(String keyspaceName, Iterable<Range<Token>> keyspaceLocalRanges,
+ Range<Token> toRepair, Collection<String> dataCenters,
+ Collection<String> hosts)
{
StorageService ss = StorageService.instance;
- Map<Range<Token>, List<InetAddress>> replicaSets = ss.getRangeToAddressMap(keyspaceName);
+ EndpointsByRange replicaSets = ss.getRangeToAddressMap(keyspaceName);
Range<Token> rangeSuperSet = null;
for (Range<Token> range : keyspaceLocalRanges)
{
@@ -229,33 +367,26 @@
}
}
if (rangeSuperSet == null || !replicaSets.containsKey(rangeSuperSet))
- return Collections.emptySet();
+ return EndpointsForRange.empty(toRepair);
- Set<InetAddress> neighbors = new HashSet<>(replicaSets.get(rangeSuperSet));
- neighbors.remove(FBUtilities.getBroadcastAddress());
+ EndpointsForRange neighbors = replicaSets.get(rangeSuperSet).withoutSelf();
if (dataCenters != null && !dataCenters.isEmpty())
{
TokenMetadata.Topology topology = ss.getTokenMetadata().cloneOnlyTokenMap().getTopology();
- Set<InetAddress> dcEndpoints = Sets.newHashSet();
- Multimap<String,InetAddress> dcEndpointsMap = topology.getDatacenterEndpoints();
- for (String dc : dataCenters)
- {
- Collection<InetAddress> c = dcEndpointsMap.get(dc);
- if (c != null)
- dcEndpoints.addAll(c);
- }
- return Sets.intersection(neighbors, dcEndpoints);
+ Multimap<String, InetAddressAndPort> dcEndpointsMap = topology.getDatacenterEndpoints();
+ Iterable<InetAddressAndPort> dcEndpoints = concat(transform(dataCenters, dcEndpointsMap::get));
+ return neighbors.select(dcEndpoints, true);
}
else if (hosts != null && !hosts.isEmpty())
{
- Set<InetAddress> specifiedHost = new HashSet<>();
+ Set<InetAddressAndPort> specifiedHost = new HashSet<>();
for (final String host : hosts)
{
try
{
- final InetAddress endpoint = InetAddress.getByName(host.trim());
- if (endpoint.equals(FBUtilities.getBroadcastAddress()) || neighbors.contains(endpoint))
+ final InetAddressAndPort endpoint = InetAddressAndPort.getByName(host.trim());
+ if (endpoint.equals(FBUtilities.getBroadcastAddressAndPort()) || neighbors.endpoints().contains(endpoint))
specifiedHost.add(endpoint);
}
catch (UnknownHostException e)
@@ -264,7 +395,7 @@
}
}
- if (!specifiedHost.contains(FBUtilities.getBroadcastAddress()))
+ if (!specifiedHost.contains(FBUtilities.getBroadcastAddressAndPort()))
throw new IllegalArgumentException("The current host must be part of the repair");
if (specifiedHost.size() <= 1)
@@ -275,66 +406,95 @@
throw new IllegalArgumentException(String.format(msg, hosts, toRepair, neighbors));
}
- specifiedHost.remove(FBUtilities.getBroadcastAddress());
- return specifiedHost;
-
+ specifiedHost.remove(FBUtilities.getBroadcastAddressAndPort());
+ return neighbors.keep(specifiedHost);
}
return neighbors;
}
- public UUID prepareForRepair(UUID parentRepairSession, InetAddress coordinator, Set<InetAddress> endpoints, RepairOption options, List<ColumnFamilyStore> columnFamilyStores)
+ /**
+ * we only want to set repairedAt for incremental repairs including all replicas for a token range. For non-global
+ * incremental repairs, forced incremental repairs, and full repairs, the UNREPAIRED_SSTABLE value will prevent
+ * sstables from being promoted to repaired or preserve the repairedAt/pendingRepair values, respectively.
+ */
+ static long getRepairedAt(RepairOption options, boolean force)
{
- long timestamp = Clock.instance.currentTimeMillis();
- registerParentRepairSession(parentRepairSession, coordinator, columnFamilyStores, options.getRanges(), options.isIncremental(), timestamp, options.isGlobal());
+ // we only want to set repairedAt for incremental repairs including all replicas for a token range. For non-global incremental repairs, full repairs, the UNREPAIRED_SSTABLE value will prevent
+ // sstables from being promoted to repaired or preserve the repairedAt/pendingRepair values, respectively. For forced repairs, repairedAt time is only set to UNREPAIRED_SSTABLE if we actually
+ // end up skipping replicas
+ if (options.isIncremental() && options.isGlobal() && ! force)
+ {
+ return System.currentTimeMillis();
+ }
+ else
+ {
+ return ActiveRepairService.UNREPAIRED_SSTABLE;
+ }
+ }
+
+ public UUID prepareForRepair(UUID parentRepairSession, InetAddressAndPort coordinator, Set<InetAddressAndPort> endpoints, RepairOption options, boolean isForcedRepair, List<ColumnFamilyStore> columnFamilyStores)
+ {
+ long repairedAt = getRepairedAt(options, isForcedRepair);
+ registerParentRepairSession(parentRepairSession, coordinator, columnFamilyStores, options.getRanges(), options.isIncremental(), repairedAt, options.isGlobal(), options.getPreviewKind());
final CountDownLatch prepareLatch = new CountDownLatch(endpoints.size());
final AtomicBoolean status = new AtomicBoolean(true);
final Set<String> failedNodes = Collections.synchronizedSet(new HashSet<String>());
- IAsyncCallbackWithFailure callback = new IAsyncCallbackWithFailure()
+ RequestCallback callback = new RequestCallback()
{
- public void response(MessageIn msg)
+ @Override
+ public void onResponse(Message msg)
{
prepareLatch.countDown();
}
- public boolean isLatencyForSnitch()
- {
- return false;
- }
-
- public void onFailure(InetAddress from, RequestFailureReason failureReason)
+ @Override
+ public void onFailure(InetAddressAndPort from, RequestFailureReason failureReason)
{
status.set(false);
- failedNodes.add(from.getHostAddress());
+ failedNodes.add(from.toString());
prepareLatch.countDown();
}
+
+ @Override
+ public boolean invokeOnFailure()
+ {
+ return true;
+ }
};
- List<UUID> cfIds = new ArrayList<>(columnFamilyStores.size());
+ List<TableId> tableIds = new ArrayList<>(columnFamilyStores.size());
for (ColumnFamilyStore cfs : columnFamilyStores)
- cfIds.add(cfs.metadata.cfId);
+ tableIds.add(cfs.metadata.id);
- for (InetAddress neighbour : endpoints)
+ for (InetAddressAndPort neighbour : endpoints)
{
if (FailureDetector.instance.isAlive(neighbour))
{
- PrepareMessage message = new PrepareMessage(parentRepairSession, cfIds, options.getRanges(), options.isIncremental(), timestamp, options.isGlobal());
- MessageOut<RepairMessage> msg = message.createMessage();
- MessagingService.instance().sendRR(msg, neighbour, callback, TimeUnit.HOURS.toMillis(1), true);
+ PrepareMessage message = new PrepareMessage(parentRepairSession, tableIds, options.getRanges(), options.isIncremental(), repairedAt, options.isGlobal(), options.getPreviewKind());
+ Message<RepairMessage> msg = Message.out(PREPARE_MSG, message);
+ MessagingService.instance().sendWithCallback(msg, neighbour, callback);
}
else
{
- // bailout early to avoid potentially waiting for a long time.
- failRepair(parentRepairSession, "Endpoint not alive: " + neighbour);
+ // we pre-filter the endpoints we want to repair for forced incremental repairs. So if any of the
+ // remaining ones go down, we still want to fail so we don't create repair sessions that can't complete
+ if (isForcedRepair && !options.isIncremental())
+ {
+ prepareLatch.countDown();
+ }
+ else
+ {
+ // bailout early to avoid potentially waiting for a long time.
+ failRepair(parentRepairSession, "Endpoint not alive: " + neighbour);
+ }
+
}
}
-
try
{
- // Failed repair is expensive so we wait for longer time.
- if (!prepareLatch.await(1, TimeUnit.HOURS)) {
+ if (!prepareLatch.await(DatabaseDescriptor.getRpcTimeout(TimeUnit.MILLISECONDS), TimeUnit.MILLISECONDS))
failRepair(parentRepairSession, "Did not get replies from all endpoints.");
- }
}
catch (InterruptedException e)
{
@@ -354,8 +514,9 @@
throw new RuntimeException(errorMsg);
}
- public synchronized void registerParentRepairSession(UUID parentRepairSession, InetAddress coordinator, List<ColumnFamilyStore> columnFamilyStores, Collection<Range<Token>> ranges, boolean isIncremental, long timestamp, boolean isGlobal)
+ public synchronized void registerParentRepairSession(UUID parentRepairSession, InetAddressAndPort coordinator, List<ColumnFamilyStore> columnFamilyStores, Collection<Range<Token>> ranges, boolean isIncremental, long repairedAt, boolean isGlobal, PreviewKind previewKind)
{
+ assert isIncremental || repairedAt == ActiveRepairService.UNREPAIRED_SSTABLE;
if (!registeredForEndpointChanges)
{
Gossiper.instance.register(this);
@@ -365,44 +526,10 @@
if (!parentRepairSessions.containsKey(parentRepairSession))
{
- parentRepairSessions.put(parentRepairSession, new ParentRepairSession(coordinator, columnFamilyStores, ranges, isIncremental, timestamp, isGlobal));
+ parentRepairSessions.put(parentRepairSession, new ParentRepairSession(coordinator, columnFamilyStores, ranges, isIncremental, repairedAt, isGlobal, previewKind));
}
}
- public Set<SSTableReader> currentlyRepairing(UUID cfId, UUID parentRepairSession)
- {
- Set<SSTableReader> repairing = new HashSet<>();
- for (Map.Entry<UUID, ParentRepairSession> entry : parentRepairSessions.entrySet())
- {
- Collection<SSTableReader> sstables = entry.getValue().getActiveSSTables(cfId);
- if (sstables != null && !entry.getKey().equals(parentRepairSession))
- repairing.addAll(sstables);
- }
- return repairing;
- }
-
- /**
- * Run final process of repair.
- * This removes all resources held by parent repair session, after performing anti compaction if necessary.
- *
- * @param parentSession Parent session ID
- * @param neighbors Repair participants (not including self)
- * @param successfulRanges Ranges that repaired successfully
- */
- public synchronized ListenableFuture finishParentSession(UUID parentSession, Set<InetAddress> neighbors, Collection<Range<Token>> successfulRanges)
- {
- List<ListenableFuture<?>> tasks = new ArrayList<>(neighbors.size() + 1);
- for (InetAddress neighbor : neighbors)
- {
- AnticompactionTask task = new AnticompactionTask(parentSession, neighbor, successfulRanges);
- registerOnFdAndGossip(task);
- tasks.add(task);
- task.run(); // 'run' is just sending message
- }
- tasks.add(doAntiCompaction(parentSession, successfulRanges));
- return Futures.successfulAsList(tasks);
- }
-
public ParentRepairSession getParentRepairSession(UUID parentSessionId)
{
ParentRepairSession session = parentRepairSessions.get(parentSessionId);
@@ -425,77 +552,33 @@
public synchronized ParentRepairSession removeParentRepairSession(UUID parentSessionId)
{
String snapshotName = parentSessionId.toString();
- for (ColumnFamilyStore cfs : getParentRepairSession(parentSessionId).columnFamilyStores.values())
+ ParentRepairSession session = parentRepairSessions.remove(parentSessionId);
+ if (session == null)
+ return null;
+ for (ColumnFamilyStore cfs : session.columnFamilyStores.values())
{
if (cfs.snapshotExists(snapshotName))
cfs.clearSnapshot(snapshotName);
}
- return parentRepairSessions.remove(parentSessionId);
+ return session;
}
- /**
- * Submit anti-compaction jobs to CompactionManager.
- * When all jobs are done, parent repair session is removed whether those are suceeded or not.
- *
- * @param parentRepairSession parent repair session ID
- * @return Future result of all anti-compaction jobs.
- */
- @SuppressWarnings("resource")
- public ListenableFuture<List<Object>> doAntiCompaction(final UUID parentRepairSession, Collection<Range<Token>> successfulRanges)
+ public void handleMessage(Message<? extends RepairMessage> message)
{
- assert parentRepairSession != null;
- ParentRepairSession prs = getParentRepairSession(parentRepairSession);
- //A repair will be marked as not global if it is a subrange repair to avoid many small anti-compactions
- //in addition to other scenarios such as repairs not involving all DCs or hosts
- if (!prs.isGlobal)
- {
- logger.info("[repair #{}] Not a global repair, will not do anticompaction", parentRepairSession);
- removeParentRepairSession(parentRepairSession);
- return Futures.immediateFuture(Collections.emptyList());
- }
- assert prs.ranges.containsAll(successfulRanges) : "Trying to perform anticompaction on unknown ranges";
-
- List<ListenableFuture<?>> futures = new ArrayList<>();
- // if we don't have successful repair ranges, then just skip anticompaction
- if (!successfulRanges.isEmpty())
- {
- for (Map.Entry<UUID, ColumnFamilyStore> columnFamilyStoreEntry : prs.columnFamilyStores.entrySet())
- {
- Refs<SSTableReader> sstables = prs.getActiveRepairedSSTableRefsForAntiCompaction(columnFamilyStoreEntry.getKey(), parentRepairSession);
- ColumnFamilyStore cfs = columnFamilyStoreEntry.getValue();
- futures.add(CompactionManager.instance.submitAntiCompaction(cfs, successfulRanges, sstables, prs.repairedAt, parentRepairSession));
- }
- }
-
- ListenableFuture<List<Object>> allAntiCompactionResults = Futures.successfulAsList(futures);
- allAntiCompactionResults.addListener(new Runnable()
- {
- @Override
- public void run()
- {
- removeParentRepairSession(parentRepairSession);
- }
- }, MoreExecutors.directExecutor());
-
- return allAntiCompactionResults;
- }
-
- public void handleMessage(InetAddress endpoint, RepairMessage message)
- {
- RepairJobDesc desc = message.desc;
+ RepairJobDesc desc = message.payload.desc;
RepairSession session = sessions.get(desc.sessionId);
if (session == null)
return;
- switch (message.messageType)
+ switch (message.verb())
{
- case VALIDATION_COMPLETE:
- ValidationComplete validation = (ValidationComplete) message;
- session.validationComplete(desc, endpoint, validation.trees);
+ case VALIDATION_RSP:
+ ValidationResponse validation = (ValidationResponse) message.payload;
+ session.validationComplete(desc, message.from(), validation.trees);
break;
- case SYNC_COMPLETE:
+ case SYNC_RSP:
// one of replica is synced.
- SyncComplete sync = (SyncComplete) message;
- session.syncComplete(desc, sync.nodes, sync.success);
+ SyncResponse sync = (SyncResponse) message.payload;
+ session.syncComplete(desc, sync.nodes, sync.success, sync.summaries);
break;
default:
break;
@@ -506,217 +589,61 @@
* We keep a ParentRepairSession around for the duration of the entire repair, for example, on a 256 token vnode rf=3 cluster
* we would have 768 RepairSession but only one ParentRepairSession. We use the PRS to avoid anticompacting the sstables
* 768 times, instead we take all repaired ranges at the end of the repair and anticompact once.
- *
- * We do an optimistic marking of sstables - when we start an incremental repair we mark all unrepaired sstables as
- * repairing (@see markSSTablesRepairing), then while the repair is ongoing compactions might remove those sstables,
- * and when it is time for anticompaction we will only anticompact the sstables that are still on disk.
- *
- * Note that validation and streaming do not care about which sstables we have marked as repairing - they operate on
- * all unrepaired sstables (if it is incremental), otherwise we would not get a correct repair.
*/
public static class ParentRepairSession
{
- private final Map<UUID, ColumnFamilyStore> columnFamilyStores = new HashMap<>();
+ private final Keyspace keyspace;
+ private final Map<TableId, ColumnFamilyStore> columnFamilyStores = new HashMap<>();
private final Collection<Range<Token>> ranges;
- public final Map<UUID, Set<String>> sstableMap = new HashMap<>();
public final boolean isIncremental;
public final boolean isGlobal;
public final long repairedAt;
- public final InetAddress coordinator;
- /**
- * Indicates whether we have marked sstables as repairing. Can only be done once per table per ParentRepairSession
- */
- private final Set<UUID> marked = new HashSet<>();
+ public final InetAddressAndPort coordinator;
+ public final PreviewKind previewKind;
- public ParentRepairSession(InetAddress coordinator, List<ColumnFamilyStore> columnFamilyStores, Collection<Range<Token>> ranges, boolean isIncremental, long repairedAt, boolean isGlobal)
+ public ParentRepairSession(InetAddressAndPort coordinator, List<ColumnFamilyStore> columnFamilyStores, Collection<Range<Token>> ranges, boolean isIncremental, long repairedAt, boolean isGlobal, PreviewKind previewKind)
{
this.coordinator = coordinator;
+ Set<Keyspace> keyspaces = new HashSet<>();
for (ColumnFamilyStore cfs : columnFamilyStores)
{
- this.columnFamilyStores.put(cfs.metadata.cfId, cfs);
- sstableMap.put(cfs.metadata.cfId, new HashSet<String>());
+ keyspaces.add(cfs.keyspace);
+ this.columnFamilyStores.put(cfs.metadata.id, cfs);
}
+
+ Preconditions.checkArgument(keyspaces.size() == 1, "repair sessions cannot operate on multiple keyspaces");
+ this.keyspace = Iterables.getOnlyElement(keyspaces);
+
this.ranges = ranges;
this.repairedAt = repairedAt;
this.isIncremental = isIncremental;
this.isGlobal = isGlobal;
+ this.previewKind = previewKind;
}
- /**
- * Mark sstables repairing - either all sstables or only the unrepaired ones depending on
- *
- * whether this is an incremental or full repair
- *
- * @param cfId the column family
- * @param parentSessionId the parent repair session id, used to make sure we don't start multiple repairs over the same sstables
- */
- public synchronized void markSSTablesRepairing(UUID cfId, UUID parentSessionId)
+ public boolean isPreview()
{
- if (!marked.contains(cfId))
- {
- List<SSTableReader> sstables = columnFamilyStores.get(cfId).select(View.select(SSTableSet.CANONICAL, (s) -> !isIncremental || !s.isRepaired())).sstables;
- Set<SSTableReader> currentlyRepairing = ActiveRepairService.instance.currentlyRepairing(cfId, parentSessionId);
- if (!Sets.intersection(currentlyRepairing, Sets.newHashSet(sstables)).isEmpty())
- {
- logger.error("Cannot start multiple repair sessions over the same sstables");
- throw new RuntimeException("Cannot start multiple repair sessions over the same sstables");
- }
- addSSTables(cfId, sstables);
- marked.add(cfId);
- }
+ return previewKind != PreviewKind.NONE;
}
- /**
- * Get the still active sstables we should run anticompaction on
- *
- * note that validation and streaming do not call this method - they have to work on the actual active sstables on the node, we only call this
- * to know which sstables are still there that were there when we started the repair
- *
- * @param cfId
- * @param parentSessionId for checking if there exists a snapshot for this repair
- * @return
- */
- @SuppressWarnings("resource")
- public synchronized Refs<SSTableReader> getActiveRepairedSSTableRefsForAntiCompaction(UUID cfId, UUID parentSessionId)
+ public Collection<ColumnFamilyStore> getColumnFamilyStores()
{
- assert marked.contains(cfId);
- if (!columnFamilyStores.containsKey(cfId))
- throw new RuntimeException("Not possible to get sstables for anticompaction for " + cfId);
- boolean isSnapshotRepair = columnFamilyStores.get(cfId).snapshotExists(parentSessionId.toString());
- ImmutableMap.Builder<SSTableReader, Ref<SSTableReader>> references = ImmutableMap.builder();
- Iterable<SSTableReader> sstables = isSnapshotRepair ? getSSTablesForSnapshotRepair(cfId, parentSessionId) : getActiveSSTables(cfId);
- // we check this above - if columnFamilyStores contains the cfId sstables will not be null
- assert sstables != null;
- for (SSTableReader sstable : sstables)
- {
- Ref<SSTableReader> ref = sstable.tryRef();
- if (ref == null)
- sstableMap.get(cfId).remove(sstable.getFilename());
- else
- references.put(sstable, ref);
- }
- return new Refs<>(references.build());
+ return ImmutableSet.<ColumnFamilyStore>builder().addAll(columnFamilyStores.values()).build();
}
- /**
- * If we are running a snapshot repair we need to find the 'real' sstables when we start anticompaction
- *
- * We use the generation of the sstables as identifiers instead of the file name to avoid having to parse out the
- * actual filename.
- *
- * @param cfId
- * @param parentSessionId
- * @return
- */
- private Set<SSTableReader> getSSTablesForSnapshotRepair(UUID cfId, UUID parentSessionId)
+ public Keyspace getKeyspace()
{
- Set<SSTableReader> activeSSTables = new HashSet<>();
- ColumnFamilyStore cfs = columnFamilyStores.get(cfId);
- if (cfs == null)
- return null;
-
- Set<Integer> snapshotGenerations = new HashSet<>();
- try (Refs<SSTableReader> snapshottedSSTables = cfs.getSnapshotSSTableReader(parentSessionId.toString()))
- {
- for (SSTableReader sstable : snapshottedSSTables)
- {
- snapshotGenerations.add(sstable.descriptor.generation);
- }
- }
- catch (IOException e)
- {
- throw new RuntimeException(e);
- }
- for (SSTableReader sstable : cfs.getSSTables(SSTableSet.CANONICAL))
- if (snapshotGenerations.contains(sstable.descriptor.generation))
- activeSSTables.add(sstable);
- return activeSSTables;
+ return keyspace;
}
- public synchronized void maybeSnapshot(UUID cfId, UUID parentSessionId)
+ public Set<TableId> getTableIds()
{
- String snapshotName = parentSessionId.toString();
- if (!columnFamilyStores.get(cfId).snapshotExists(snapshotName))
- {
- Set<SSTableReader> snapshottedSSTables = columnFamilyStores.get(cfId).snapshot(snapshotName, new Predicate<SSTableReader>()
- {
- public boolean apply(SSTableReader sstable)
- {
- return sstable != null &&
- (!isIncremental || !sstable.isRepaired()) &&
- !(sstable.metadata.isIndex()) && // exclude SSTables from 2i
- new Bounds<>(sstable.first.getToken(), sstable.last.getToken()).intersects(ranges);
- }
- }, true, false);
-
- if (isAlreadyRepairing(cfId, parentSessionId, snapshottedSSTables))
- {
- columnFamilyStores.get(cfId).clearSnapshot(parentSessionId.toString());
- logger.error("Cannot start multiple repair sessions over the same sstables");
- throw new RuntimeException("Cannot start multiple repair sessions over the same sstables");
- }
- addSSTables(cfId, snapshottedSSTables);
- marked.add(cfId);
- }
+ return ImmutableSet.copyOf(transform(getColumnFamilyStores(), cfs -> cfs.metadata.id));
}
-
- /**
- * Compares other repairing sstables *generation* to the ones we just snapshotted
- *
- * we compare generations since the sstables have different paths due to snapshot names
- *
- * @param cfId id of the column family store
- * @param parentSessionId parent repair session
- * @param sstables the newly snapshotted sstables
- * @return
- */
- private boolean isAlreadyRepairing(UUID cfId, UUID parentSessionId, Collection<SSTableReader> sstables)
+ public Set<Range<Token>> getRanges()
{
- Set<SSTableReader> currentlyRepairing = ActiveRepairService.instance.currentlyRepairing(cfId, parentSessionId);
- Set<Integer> currentlyRepairingGenerations = new HashSet<>();
- Set<Integer> newRepairingGenerations = new HashSet<>();
- for (SSTableReader sstable : currentlyRepairing)
- currentlyRepairingGenerations.add(sstable.descriptor.generation);
- for (SSTableReader sstable : sstables)
- newRepairingGenerations.add(sstable.descriptor.generation);
-
- return !Sets.intersection(currentlyRepairingGenerations, newRepairingGenerations).isEmpty();
- }
-
- private Set<SSTableReader> getActiveSSTables(UUID cfId)
- {
- if (!columnFamilyStores.containsKey(cfId))
- return null;
-
- Set<String> repairedSSTables = sstableMap.get(cfId);
- Set<SSTableReader> activeSSTables = new HashSet<>();
- Set<String> activeSSTableNames = new HashSet<>();
- ColumnFamilyStore cfs = columnFamilyStores.get(cfId);
- for (SSTableReader sstable : cfs.getSSTables(SSTableSet.CANONICAL))
- {
- if (repairedSSTables.contains(sstable.getFilename()))
- {
- activeSSTables.add(sstable);
- activeSSTableNames.add(sstable.getFilename());
- }
- }
- sstableMap.put(cfId, activeSSTableNames);
- return activeSSTables;
- }
-
- private void addSSTables(UUID cfId, Collection<SSTableReader> sstables)
- {
- for (SSTableReader sstable : sstables)
- sstableMap.get(cfId).add(sstable.getFilename());
- }
-
-
- public long getRepairedAt()
- {
- if (isGlobal)
- return repairedAt;
- return ActiveRepairService.UNREPAIRED_SSTABLE;
+ return ImmutableSet.copyOf(ranges);
}
@Override
@@ -725,7 +652,6 @@
return "ParentRepairSession{" +
"columnFamilyStores=" + columnFamilyStores +
", ranges=" + ranges +
- ", sstableMap=" + sstableMap +
", repairedAt=" + repairedAt +
'}';
}
@@ -735,18 +661,18 @@
If the coordinator node dies we should remove the parent repair session from the other nodes.
This uses the same notifications as we get in RepairSession
*/
- public void onJoin(InetAddress endpoint, EndpointState epState) {}
- public void beforeChange(InetAddress endpoint, EndpointState currentState, ApplicationState newStateKey, VersionedValue newValue) {}
- public void onChange(InetAddress endpoint, ApplicationState state, VersionedValue value) {}
- public void onAlive(InetAddress endpoint, EndpointState state) {}
- public void onDead(InetAddress endpoint, EndpointState state) {}
+ public void onJoin(InetAddressAndPort endpoint, EndpointState epState) {}
+ public void beforeChange(InetAddressAndPort endpoint, EndpointState currentState, ApplicationState newStateKey, VersionedValue newValue) {}
+ public void onChange(InetAddressAndPort endpoint, ApplicationState state, VersionedValue value) {}
+ public void onAlive(InetAddressAndPort endpoint, EndpointState state) {}
+ public void onDead(InetAddressAndPort endpoint, EndpointState state) {}
- public void onRemove(InetAddress endpoint)
+ public void onRemove(InetAddressAndPort endpoint)
{
convict(endpoint, Double.MAX_VALUE);
}
- public void onRestart(InetAddress endpoint, EndpointState state)
+ public void onRestart(InetAddressAndPort endpoint, EndpointState state)
{
convict(endpoint, Double.MAX_VALUE);
}
@@ -760,7 +686,7 @@
* @param ep endpoint to be convicted
* @param phi the value of phi with with ep was convicted
*/
- public void convict(InetAddress ep, double phi)
+ public void convict(InetAddressAndPort ep, double phi)
{
// We want a higher confidence in the failure detection than usual because failing a repair wrongly has a high cost.
if (phi < 2 * DatabaseDescriptor.getPhiConvictThreshold() || parentRepairSessions.isEmpty())
diff --git a/src/java/org/apache/cassandra/service/ActiveRepairServiceMBean.java b/src/java/org/apache/cassandra/service/ActiveRepairServiceMBean.java
new file mode 100644
index 0000000..d967280
--- /dev/null
+++ b/src/java/org/apache/cassandra/service/ActiveRepairServiceMBean.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service;
+
+import java.util.List;
+import java.util.Map;
+
+public interface ActiveRepairServiceMBean
+{
+ public static final String MBEAN_NAME = "org.apache.cassandra.db:type=RepairService";
+
+ public List<Map<String, String>> getSessions(boolean all);
+ public void failSession(String session, boolean force);
+
+ public void setRepairSessionSpaceInMegabytes(int sizeInMegabytes);
+ public int getRepairSessionSpaceInMegabytes();
+
+ public boolean getUseOffheapMerkleTrees();
+ public void setUseOffheapMerkleTrees(boolean value);
+}
diff --git a/src/java/org/apache/cassandra/service/AsyncRepairCallback.java b/src/java/org/apache/cassandra/service/AsyncRepairCallback.java
deleted file mode 100644
index d613f3d..0000000
--- a/src/java/org/apache/cassandra/service/AsyncRepairCallback.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.service;
-
-import java.util.concurrent.atomic.AtomicInteger;
-
-import org.apache.cassandra.concurrent.Stage;
-import org.apache.cassandra.concurrent.StageManager;
-import org.apache.cassandra.db.ReadResponse;
-import org.apache.cassandra.net.IAsyncCallback;
-import org.apache.cassandra.net.MessageIn;
-import org.apache.cassandra.utils.WrappedRunnable;
-
-public class AsyncRepairCallback implements IAsyncCallback<ReadResponse>
-{
- private final DataResolver repairResolver;
- private final int blockfor;
- protected final AtomicInteger received = new AtomicInteger(0);
-
- public AsyncRepairCallback(DataResolver repairResolver, int blockfor)
- {
- this.repairResolver = repairResolver;
- this.blockfor = blockfor;
- }
-
- public void response(MessageIn<ReadResponse> message)
- {
- repairResolver.preprocess(message);
- if (received.incrementAndGet() == blockfor)
- {
- StageManager.getStage(Stage.READ_REPAIR).execute(new WrappedRunnable()
- {
- protected void runMayThrow()
- {
- repairResolver.compareResponses();
- }
- });
- }
- }
-
- public boolean isLatencyForSnitch()
- {
- return true;
- }
-}
diff --git a/src/java/org/apache/cassandra/service/BatchlogResponseHandler.java b/src/java/org/apache/cassandra/service/BatchlogResponseHandler.java
index 420b715..b28f468 100644
--- a/src/java/org/apache/cassandra/service/BatchlogResponseHandler.java
+++ b/src/java/org/apache/cassandra/service/BatchlogResponseHandler.java
@@ -18,14 +18,13 @@
package org.apache.cassandra.service;
-import java.net.InetAddress;
import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
import org.apache.cassandra.exceptions.RequestFailureReason;
import org.apache.cassandra.exceptions.WriteFailureException;
import org.apache.cassandra.exceptions.WriteTimeoutException;
-import org.apache.cassandra.net.MessageIn;
-import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.net.Message;
public class BatchlogResponseHandler<T> extends AbstractWriteResponseHandler<T>
{
@@ -37,7 +36,7 @@
public BatchlogResponseHandler(AbstractWriteResponseHandler<T> wrapped, int requiredBeforeFinish, BatchlogCleanup cleanup, long queryStartNanoTime)
{
- super(wrapped.keyspace, wrapped.naturalEndpoints, wrapped.pendingEndpoints, wrapped.consistencyLevel, wrapped.callback, wrapped.writeType, queryStartNanoTime);
+ super(wrapped.replicaPlan, wrapped.callback, wrapped.writeType, queryStartNanoTime);
this.wrapped = wrapped;
this.requiredBeforeFinish = requiredBeforeFinish;
this.cleanup = cleanup;
@@ -48,26 +47,21 @@
return wrapped.ackCount();
}
- public void response(MessageIn<T> msg)
+ public void onResponse(Message<T> msg)
{
- wrapped.response(msg);
+ wrapped.onResponse(msg);
if (requiredBeforeFinishUpdater.decrementAndGet(this) == 0)
cleanup.ackMutation();
}
- public boolean isLatencyForSnitch()
- {
- return wrapped.isLatencyForSnitch();
- }
-
- public void onFailure(InetAddress from, RequestFailureReason failureReason)
+ public void onFailure(InetAddressAndPort from, RequestFailureReason failureReason)
{
wrapped.onFailure(from, failureReason);
}
- public void assureSufficientLiveNodes()
+ public boolean invokeOnFailure()
{
- wrapped.assureSufficientLiveNodes();
+ return wrapped.invokeOnFailure();
}
public void get() throws WriteTimeoutException, WriteFailureException
@@ -75,17 +69,17 @@
wrapped.get();
}
- protected int totalBlockFor()
+ protected int blockFor()
{
- return wrapped.totalBlockFor();
+ return wrapped.blockFor();
}
- protected int totalEndpoints()
+ protected int candidateReplicaCount()
{
- return wrapped.totalEndpoints();
+ return wrapped.candidateReplicaCount();
}
- protected boolean waitingFor(InetAddress from)
+ protected boolean waitingFor(InetAddressAndPort from)
{
return wrapped.waitingFor(from);
}
diff --git a/src/java/org/apache/cassandra/service/CASRequest.java b/src/java/org/apache/cassandra/service/CASRequest.java
index 1db100d..88fb9bd 100644
--- a/src/java/org/apache/cassandra/service/CASRequest.java
+++ b/src/java/org/apache/cassandra/service/CASRequest.java
@@ -17,7 +17,7 @@
*/
package org.apache.cassandra.service;
-import org.apache.cassandra.db.SinglePartitionReadCommand;
+import org.apache.cassandra.db.SinglePartitionReadQuery;
import org.apache.cassandra.db.partitions.FilteredPartition;
import org.apache.cassandra.db.partitions.PartitionUpdate;
import org.apache.cassandra.exceptions.InvalidRequestException;
@@ -30,7 +30,7 @@
/**
* The command to use to fetch the value to compare for the CAS.
*/
- public SinglePartitionReadCommand readCommand(int nowInSec);
+ public SinglePartitionReadQuery readCommand(int nowInSec);
/**
* Returns whether the provided CF, that represents the values fetched using the
diff --git a/src/java/org/apache/cassandra/service/CacheService.java b/src/java/org/apache/cassandra/service/CacheService.java
index 62d820f..4a2e3d5 100644
--- a/src/java/org/apache/cassandra/service/CacheService.java
+++ b/src/java/org/apache/cassandra/service/CacheService.java
@@ -28,25 +28,24 @@
import com.google.common.util.concurrent.Futures;
-import org.apache.cassandra.db.lifecycle.SSTableSet;
-import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+
import org.apache.cassandra.cache.*;
import org.apache.cassandra.cache.AutoSavingCache.CacheSerializer;
import org.apache.cassandra.concurrent.Stage;
-import org.apache.cassandra.concurrent.StageManager;
-import org.apache.cassandra.config.ColumnDefinition;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.*;
-import org.apache.cassandra.db.rows.*;
+import org.apache.cassandra.db.context.CounterContext;
import org.apache.cassandra.db.filter.*;
+import org.apache.cassandra.db.lifecycle.SSTableSet;
import org.apache.cassandra.db.partitions.CachedBTreePartition;
import org.apache.cassandra.db.partitions.CachedPartition;
-import org.apache.cassandra.db.context.CounterContext;
+import org.apache.cassandra.db.rows.*;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
-import org.apache.cassandra.io.sstable.format.big.BigFormat;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.MBeanWrapper;
@@ -104,7 +103,7 @@
// as values are constant size we can use singleton weigher
// where 48 = 40 bytes (average size of the key) + 8 bytes (size of value)
ICache<KeyCacheKey, RowIndexEntry> kc;
- kc = ConcurrentLinkedHashCache.create(keyCacheInMemoryCapacity);
+ kc = CaffeineCache.create(keyCacheInMemoryCapacity);
AutoSavingCache<KeyCacheKey, RowIndexEntry> keyCache = new AutoSavingCache<>(kc, CacheType.KEY_CACHE, new KeyCacheSerializer());
int keyCacheKeysToSave = DatabaseDescriptor.getKeyCacheKeysToSave();
@@ -153,7 +152,7 @@
long capacity = DatabaseDescriptor.getCounterCacheSizeInMB() * 1024 * 1024;
AutoSavingCache<CounterCacheKey, ClockAndCount> cache =
- new AutoSavingCache<>(ConcurrentLinkedHashCache.<CounterCacheKey, ClockAndCount>create(capacity),
+ new AutoSavingCache<>(CaffeineCache.create(capacity),
CacheType.COUNTER_CACHE,
new CounterCacheSerializer());
@@ -255,13 +254,13 @@
keyCache.clear();
}
- public void invalidateKeyCacheForCf(Pair<String, String> ksAndCFName)
+ public void invalidateKeyCacheForCf(TableMetadata tableMetadata)
{
Iterator<KeyCacheKey> keyCacheIterator = keyCache.keyIterator();
while (keyCacheIterator.hasNext())
{
KeyCacheKey key = keyCacheIterator.next();
- if (key.ksAndCFName.equals(ksAndCFName))
+ if (key.sameTable(tableMetadata))
keyCacheIterator.remove();
}
}
@@ -271,24 +270,24 @@
rowCache.clear();
}
- public void invalidateRowCacheForCf(Pair<String, String> ksAndCFName)
+ public void invalidateRowCacheForCf(TableMetadata tableMetadata)
{
Iterator<RowCacheKey> rowCacheIterator = rowCache.keyIterator();
while (rowCacheIterator.hasNext())
{
- RowCacheKey rowCacheKey = rowCacheIterator.next();
- if (rowCacheKey.ksAndCFName.equals(ksAndCFName))
+ RowCacheKey key = rowCacheIterator.next();
+ if (key.sameTable(tableMetadata))
rowCacheIterator.remove();
}
}
- public void invalidateCounterCacheForCf(Pair<String, String> ksAndCFName)
+ public void invalidateCounterCacheForCf(TableMetadata tableMetadata)
{
Iterator<CounterCacheKey> counterCacheIterator = counterCache.keyIterator();
while (counterCacheIterator.hasNext())
{
- CounterCacheKey counterCacheKey = counterCacheIterator.next();
- if (counterCacheKey.ksAndCFName.equals(ksAndCFName))
+ CounterCacheKey key = counterCacheIterator.next();
+ if (key.sameTable(tableMetadata))
counterCacheIterator.remove();
}
}
@@ -343,60 +342,31 @@
{
public void serialize(CounterCacheKey key, DataOutputPlus out, ColumnFamilyStore cfs) throws IOException
{
- assert(cfs.metadata.isCounter());
- out.write(cfs.metadata.ksAndCFBytes);
- ByteBufferUtil.writeWithLength(key.partitionKey, out);
- ByteBufferUtil.writeWithLength(key.cellName, out);
+ assert(cfs.metadata().isCounter());
+ TableMetadata tableMetadata = cfs.metadata();
+ tableMetadata.id.serialize(out);
+ out.writeUTF(tableMetadata.indexName().orElse(""));
+ key.write(out);
}
public Future<Pair<CounterCacheKey, ClockAndCount>> deserialize(DataInputPlus in, final ColumnFamilyStore cfs) throws IOException
{
//Keyspace and CF name are deserialized by AutoSaving cache and used to fetch the CFS provided as a
//parameter so they aren't deserialized here, even though they are serialized by this serializer
- final ByteBuffer partitionKey = ByteBufferUtil.readWithLength(in);
- final ByteBuffer cellName = ByteBufferUtil.readWithLength(in);
- if (cfs == null || !cfs.metadata.isCounter() || !cfs.isCounterCacheEnabled())
+ if (cfs == null)
return null;
- assert(cfs.metadata.isCounter());
- return StageManager.getStage(Stage.READ).submit(new Callable<Pair<CounterCacheKey, ClockAndCount>>()
+ final CounterCacheKey cacheKey = CounterCacheKey.read(cfs.metadata(), in);
+ if (!cfs.metadata().isCounter() || !cfs.isCounterCacheEnabled())
+ return null;
+
+ return Stage.READ.submit(new Callable<Pair<CounterCacheKey, ClockAndCount>>()
{
public Pair<CounterCacheKey, ClockAndCount> call() throws Exception
{
- DecoratedKey key = cfs.decorateKey(partitionKey);
- LegacyLayout.LegacyCellName name = LegacyLayout.decodeCellName(cfs.metadata, cellName);
- ColumnDefinition column = name.column;
- CellPath path = name.collectionElement == null ? null : CellPath.create(name.collectionElement);
-
- int nowInSec = FBUtilities.nowInSeconds();
- ColumnFilter.Builder builder = ColumnFilter.selectionBuilder();
- if (path == null)
- builder.add(column);
- else
- builder.select(column, path);
-
- ClusteringIndexFilter filter = new ClusteringIndexNamesFilter(FBUtilities.<Clustering>singleton(name.clustering, cfs.metadata.comparator), false);
- SinglePartitionReadCommand cmd = SinglePartitionReadCommand.create(cfs.metadata, nowInSec, key, builder.build(), filter);
- try (ReadExecutionController controller = cmd.executionController();
- RowIterator iter = UnfilteredRowIterators.filter(cmd.queryMemtableAndDisk(cfs, controller), nowInSec))
- {
- Cell cell;
- if (column.isStatic())
- {
- cell = iter.staticRow().getCell(column);
- }
- else
- {
- if (!iter.hasNext())
- return null;
- cell = iter.next().getCell(column);
- }
-
- if (cell == null)
- return null;
-
- ClockAndCount clockAndCount = CounterContext.instance().getLocalClockAndCount(cell.value());
- return Pair.create(CounterCacheKey.create(cfs.metadata.ksAndCFName, partitionKey, name.clustering, column, path), clockAndCount);
- }
+ ByteBuffer value = cacheKey.readCounterValue(cfs);
+ return value == null
+ ? null
+ : Pair.create(cacheKey, CounterContext.instance().getLocalClockAndCount(value));
}
});
}
@@ -407,7 +377,9 @@
public void serialize(RowCacheKey key, DataOutputPlus out, ColumnFamilyStore cfs) throws IOException
{
assert(!cfs.isIndex());//Shouldn't have row cache entries for indexes
- out.write(cfs.metadata.ksAndCFBytes);
+ TableMetadata tableMetadata = cfs.metadata();
+ tableMetadata.id.serialize(out);
+ out.writeUTF(tableMetadata.indexName().orElse(""));
ByteBufferUtil.writeWithLength(key.key, out);
}
@@ -418,20 +390,20 @@
final ByteBuffer buffer = ByteBufferUtil.readWithLength(in);
if (cfs == null || !cfs.isRowCacheEnabled())
return null;
- final int rowsToCache = cfs.metadata.params.caching.rowsPerPartitionToCache();
+ final int rowsToCache = cfs.metadata().params.caching.rowsPerPartitionToCache();
assert(!cfs.isIndex());//Shouldn't have row cache entries for indexes
- return StageManager.getStage(Stage.READ).submit(new Callable<Pair<RowCacheKey, IRowCacheEntry>>()
+ return Stage.READ.submit(new Callable<Pair<RowCacheKey, IRowCacheEntry>>()
{
public Pair<RowCacheKey, IRowCacheEntry> call() throws Exception
{
DecoratedKey key = cfs.decorateKey(buffer);
int nowInSec = FBUtilities.nowInSeconds();
- SinglePartitionReadCommand cmd = SinglePartitionReadCommand.fullPartitionRead(cfs.metadata, nowInSec, key);
+ SinglePartitionReadCommand cmd = SinglePartitionReadCommand.fullPartitionRead(cfs.metadata(), nowInSec, key);
try (ReadExecutionController controller = cmd.executionController(); UnfilteredRowIterator iter = cmd.queryMemtableAndDisk(cfs, controller))
{
CachedPartition toCache = CachedBTreePartition.create(DataLimits.cqlLimits(rowsToCache).filter(iter, nowInSec, true), nowInSec);
- return Pair.create(new RowCacheKey(cfs.metadata.ksAndCFName, key), (IRowCacheEntry)toCache);
+ return Pair.create(new RowCacheKey(cfs.metadata(), key), toCache);
}
}
});
@@ -442,21 +414,19 @@
{
public void serialize(KeyCacheKey key, DataOutputPlus out, ColumnFamilyStore cfs) throws IOException
{
- //Don't serialize old format entries since we didn't bother to implement serialization of both for simplicity
- //https://issues.apache.org/jira/browse/CASSANDRA-10778
- if (!key.desc.version.storeRows()) return;
-
RowIndexEntry entry = CacheService.instance.keyCache.getInternal(key);
if (entry == null)
return;
- out.write(cfs.metadata.ksAndCFBytes);
+ TableMetadata tableMetadata = cfs.metadata();
+ tableMetadata.id.serialize(out);
+ out.writeUTF(tableMetadata.indexName().orElse(""));
ByteBufferUtil.writeWithLength(key.key, out);
out.writeInt(key.desc.generation);
out.writeBoolean(true);
- SerializationHeader header = new SerializationHeader(false, cfs.metadata, cfs.metadata.partitionColumns(), EncodingStats.NO_STATS);
- key.desc.getFormat().getIndexSerializer(cfs.metadata, key.desc.version, header).serializeForCache(entry, out);
+ SerializationHeader header = new SerializationHeader(false, cfs.metadata(), cfs.metadata().regularAndStaticColumns(), EncodingStats.NO_STATS);
+ key.desc.getFormat().getIndexSerializer(cfs.metadata(), key.desc.version, header).serializeForCache(entry, out);
}
public Future<Pair<KeyCacheKey, RowIndexEntry>> deserialize(DataInputPlus input, ColumnFamilyStore cfs) throws IOException
@@ -479,14 +449,14 @@
// wrong is during upgrade, in which case we fail at deserialization. This is not a huge deal however since 1) this is unlikely enough that
// this won't affect many users (if any) and only once, 2) this doesn't prevent the node from starting and 3) CASSANDRA-10219 shows that this
// part of the code has been broken for a while without anyone noticing (it is, btw, still broken until CASSANDRA-10219 is fixed).
- RowIndexEntry.Serializer.skipForCache(input, BigFormat.instance.getLatestVersion());
+ RowIndexEntry.Serializer.skipForCache(input);
return null;
}
- RowIndexEntry.IndexSerializer<?> indexSerializer = reader.descriptor.getFormat().getIndexSerializer(reader.metadata,
+ RowIndexEntry.IndexSerializer<?> indexSerializer = reader.descriptor.getFormat().getIndexSerializer(reader.metadata(),
reader.descriptor.version,
reader.header);
RowIndexEntry<?> entry = indexSerializer.deserializeForCache(input);
- return Futures.immediateFuture(Pair.create(new KeyCacheKey(cfs.metadata.ksAndCFName, reader.descriptor, key), entry));
+ return Futures.immediateFuture(Pair.create(new KeyCacheKey(cfs.metadata(), reader.descriptor, key), entry));
}
private SSTableReader findDesc(int generation, Iterable<SSTableReader> collection)
diff --git a/src/java/org/apache/cassandra/service/CassandraDaemon.java b/src/java/org/apache/cassandra/service/CassandraDaemon.java
index 1f93262..10d9cea 100644
--- a/src/java/org/apache/cassandra/service/CassandraDaemon.java
+++ b/src/java/org/apache/cassandra/service/CassandraDaemon.java
@@ -26,7 +26,6 @@
import java.net.UnknownHostException;
import java.util.List;
import java.util.concurrent.TimeUnit;
-import javax.management.MBeanServer;
import javax.management.ObjectName;
import javax.management.StandardMBean;
import javax.management.remote.JMXConnectorServer;
@@ -45,19 +44,23 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.cassandra.batchlog.LegacyBatchlogMigrator;
+import org.apache.cassandra.audit.AuditLogManager;
import org.apache.cassandra.concurrent.ScheduledExecutors;
-import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.db.virtual.SystemViewsKeyspace;
+import org.apache.cassandra.db.virtual.VirtualKeyspaceRegistry;
+import org.apache.cassandra.db.virtual.VirtualSchemaKeyspace;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.net.StartupClusterConnectivityChecker;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.config.Schema;
-import org.apache.cassandra.config.SchemaConstants;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.SchemaConstants;
import org.apache.cassandra.cql3.QueryProcessor;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.commitlog.CommitLog;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.exceptions.StartupException;
import org.apache.cassandra.gms.Gossiper;
-import org.apache.cassandra.hints.LegacyHintsMigrator;
import org.apache.cassandra.io.FSError;
import org.apache.cassandra.io.sstable.CorruptSSTableException;
import org.apache.cassandra.io.sstable.SSTableHeaderFix;
@@ -65,12 +68,12 @@
import org.apache.cassandra.metrics.CassandraMetricsRegistry;
import org.apache.cassandra.metrics.DefaultNameFactory;
import org.apache.cassandra.metrics.StorageMetrics;
-import org.apache.cassandra.schema.LegacySchemaMigrator;
-import org.apache.cassandra.thrift.ThriftServer;
import org.apache.cassandra.tracing.Tracing;
import org.apache.cassandra.utils.*;
import org.apache.cassandra.security.ThreadAwareSecurityManager;
+import static java.util.concurrent.TimeUnit.NANOSECONDS;
+
/**
* The <code>CassandraDaemon</code> is an abstraction for a Cassandra daemon
* service, which defines not only a way to activate and deactivate it, but also
@@ -159,7 +162,6 @@
static final CassandraDaemon instance = new CassandraDaemon();
- public Server thriftServer;
private NativeTransportService nativeTransportService;
private JMXConnectorServer jmxServer;
@@ -202,6 +204,8 @@
NativeLibrary.tryMlockall();
+ CommitLog.instance.start();
+
try
{
startupChecks.verify();
@@ -213,10 +217,7 @@
try
{
- if (SystemKeyspace.snapshotOnVersionChange())
- {
- SystemKeyspace.migrateDataDirs();
- }
+ SystemKeyspace.snapshotOnVersionChange();
}
catch (IOException e)
{
@@ -227,46 +228,25 @@
// This should be the first write to SystemKeyspace (CASSANDRA-11742)
SystemKeyspace.persistLocalMetadata();
- Thread.setDefaultUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler()
- {
- public void uncaughtException(Thread t, Throwable e)
- {
- StorageMetrics.exceptions.inc();
- logger.error("Exception in thread " + t, e);
- Tracing.trace("Exception in thread {}", t, e);
- for (Throwable e2 = e; e2 != null; e2 = e2.getCause())
- {
- JVMStabilityInspector.inspectThrowable(e2);
+ Thread.setDefaultUncaughtExceptionHandler(CassandraDaemon::uncaughtException);
- if (e2 instanceof FSError)
- {
- if (e2 != e) // make sure FSError gets logged exactly once.
- logger.error("Exception in thread " + t, e2);
- FileUtils.handleFSError((FSError) e2);
- }
-
- if (e2 instanceof CorruptSSTableException)
- {
- if (e2 != e)
- logger.error("Exception in thread " + t, e2);
- FileUtils.handleCorruptSSTable((CorruptSSTableException) e2);
- }
- }
- }
- });
-
- /*
- * Migrate pre-3.0 keyspaces, tables, types, functions, and aggregates, to their new 3.0 storage.
- * We don't (and can't) wait for commit log replay here, but we don't need to - all schema changes force
- * explicit memtable flushes.
- */
- LegacySchemaMigrator.migrate();
+ SystemKeyspaceMigrator40.migrate();
// Populate token metadata before flushing, for token-aware sstable partitioning (#6696)
StorageService.instance.populateTokenMetadata();
- // load schema from disk
- Schema.instance.loadFromDisk();
+ try
+ {
+ // load schema from disk
+ Schema.instance.loadFromDisk();
+ }
+ catch (Exception e)
+ {
+ logger.error("Error while loading schema: ", e);
+ throw e;
+ }
+
+ setupVirtualKeyspaces();
SSTableHeaderFix.fixNonFrozenUDTIfUpgradeFrom30();
@@ -277,7 +257,7 @@
if (keyspaceName.equals(SchemaConstants.SYSTEM_KEYSPACE_NAME))
continue;
- for (CFMetaData cfm : Schema.instance.getTablesAndViews(keyspaceName))
+ for (TableMetadata cfm : Schema.instance.getTablesAndViews(keyspaceName))
{
try
{
@@ -307,6 +287,7 @@
}
}
+
try
{
loadRowAndKeyCacheAsync().get();
@@ -340,12 +321,6 @@
// Re-populate token metadata after commit log recover (new peers might be loaded onto system keyspace #10293)
StorageService.instance.populateTokenMetadata();
- // migrate any legacy (pre-3.0) hints from system.hints table into the new store
- new LegacyHintsMigrator(DatabaseDescriptor.getHintsDirectory(), DatabaseDescriptor.getMaxHintsFileSize()).migrate();
-
- // migrate any legacy (pre-3.0) batch entries from system.batchlog to system.batches (new table format)
- LegacyBatchlogMigrator.migrate();
-
SystemKeyspace.finishStartup();
// Clean up system.size_estimates entries left lying around from missed keyspace drops (CASSANDRA-14905)
@@ -357,6 +332,8 @@
if (sizeRecorderInterval > 0)
ScheduledExecutors.optionalTasks.scheduleWithFixedDelay(SizeEstimatesRecorder.instance, 30, sizeRecorderInterval, TimeUnit.SECONDS);
+ ActiveRepairService.instance.start();
+
// Prepared statements
QueryProcessor.preloadPreparedStatement();
@@ -414,7 +391,7 @@
ScheduledExecutors.optionalTasks.schedule(viewRebuild, StorageService.RING_DELAY, TimeUnit.MILLISECONDS);
- if (!FBUtilities.getBroadcastAddress().equals(InetAddress.getLoopbackAddress()))
+ if (!FBUtilities.getBroadcastAddressAndPort().equals(InetAddressAndPort.getLoopbackAddress()))
Gossiper.waitToSettle();
// re-enable auto-compaction after gossip is settled, so correct disk boundaries are used
@@ -427,26 +404,44 @@
store.reload(); //reload CFs in case there was a change of disk boundaries
if (store.getCompactionStrategyManager().shouldBeEnabled())
{
- store.enableAutoCompaction();
+ if (DatabaseDescriptor.getAutocompactionOnStartupEnabled())
+ {
+ store.enableAutoCompaction();
+ }
+ else
+ {
+ logger.info("Not enabling compaction for {}.{}; autocompaction_on_startup_enabled is set to false", store.keyspace.getName(), store.name);
+ }
}
}
}
}
+ AuditLogManager.instance.initialize();
+
// schedule periodic background compaction task submission. this is simply a backstop against compactions stalling
// due to scheduling errors or race conditions
ScheduledExecutors.optionalTasks.scheduleWithFixedDelay(ColumnFamilyStore.getBackgroundCompactionTaskSubmitter(), 5, 1, TimeUnit.MINUTES);
- // Thrift
- InetAddress rpcAddr = DatabaseDescriptor.getRpcAddress();
- int rpcPort = DatabaseDescriptor.getRpcPort();
- int listenBacklog = DatabaseDescriptor.getRpcListenBacklog();
- thriftServer = new ThriftServer(rpcAddr, rpcPort, listenBacklog);
+ // schedule periodic recomputation of speculative retry thresholds
+ ScheduledExecutors.optionalTasks.scheduleWithFixedDelay(
+ () -> Keyspace.all().forEach(k -> k.getColumnFamilyStores().forEach(ColumnFamilyStore::updateSpeculationThreshold)),
+ DatabaseDescriptor.getReadRpcTimeout(NANOSECONDS),
+ DatabaseDescriptor.getReadRpcTimeout(NANOSECONDS),
+ NANOSECONDS
+ );
+
initializeNativeTransport();
completeSetup();
}
+ public void setupVirtualKeyspaces()
+ {
+ VirtualKeyspaceRegistry.instance.register(VirtualSchemaKeyspace.instance);
+ VirtualKeyspaceRegistry.instance.register(SystemViewsKeyspace.instance);
+ }
+
public void initializeNativeTransport()
{
// Native transport
@@ -454,6 +449,32 @@
nativeTransportService = new NativeTransportService();
}
+ @VisibleForTesting
+ public static void uncaughtException(Thread t, Throwable e)
+ {
+ StorageMetrics.uncaughtExceptions.inc();
+ logger.error("Exception in thread " + t, e);
+ Tracing.trace("Exception in thread {}", t, e);
+ for (Throwable e2 = e; e2 != null; e2 = e2.getCause())
+ {
+ JVMStabilityInspector.inspectThrowable(e2);
+
+ if (e2 instanceof FSError)
+ {
+ if (e2 != e) // make sure FSError gets logged exactly once.
+ logger.error("Exception in thread " + t, e2);
+ FileUtils.handleFSError((FSError) e2);
+ }
+
+ if (e2 instanceof CorruptSSTableException)
+ {
+ if (e2 != e)
+ logger.error("Exception in thread " + t, e2);
+ FileUtils.handleCorruptSSTable((CorruptSSTableException) e2);
+ }
+ }
+ }
+
/*
* Asynchronously load the row and key cache in one off threads and return a compound future of the result.
* Error handling is pushed into the cache load since cache loads are allowed to fail and are handled by logging.
@@ -487,7 +508,7 @@
{
try
{
- logger.info("Hostname: {}", InetAddress.getLocalHost().getHostName());
+ logger.info("Hostname: {}", InetAddress.getLocalHost().getHostName() + ":" + DatabaseDescriptor.getStoragePort() + ":" + DatabaseDescriptor.getSSLStoragePort());
}
catch (UnknownHostException e1)
{
@@ -530,15 +551,32 @@
*/
public void start()
{
- try
+ StartupClusterConnectivityChecker connectivityChecker = StartupClusterConnectivityChecker.create(DatabaseDescriptor.getBlockForPeersTimeoutInSeconds(),
+ DatabaseDescriptor.getBlockForPeersInRemoteDatacenters());
+ connectivityChecker.execute(Gossiper.instance.getEndpoints(), DatabaseDescriptor.getEndpointSnitch()::getDatacenter);
+
+ // We only start transports if bootstrap has completed and we're not in survey mode,
+ // OR if we are in survey mode and streaming has completed but we're not using auth
+ // OR if we have not joined the ring yet.
+ if (StorageService.instance.hasJoined())
{
- validateTransportsCanStart();
- }
- catch (IllegalStateException isx)
- {
- // If there are any errors, we just log and return in this case
- logger.info(isx.getMessage());
- return;
+ if (StorageService.instance.isSurveyMode())
+ {
+ if (StorageService.instance.isBootstrapMode() || DatabaseDescriptor.getAuthenticator().requireAuthentication())
+ {
+ logger.info("Not starting client transports in write_survey mode as it's bootstrapping or " +
+ "auth is enabled");
+ return;
+ }
+ }
+ else
+ {
+ if (!SystemKeyspace.bootstrapComplete())
+ {
+ logger.info("Not starting client transports as bootstrap has not completed");
+ return;
+ }
+ }
}
String nativeFlag = System.getProperty("cassandra.start_native_transport");
@@ -549,12 +587,6 @@
}
else
logger.info("Not starting native transport as requested. Use JMX (StorageService->startNativeTransport()) or nodetool (enablebinary) to start it");
-
- String rpcFlag = System.getProperty("cassandra.start_rpc");
- if ((rpcFlag != null && Boolean.parseBoolean(rpcFlag)) || (rpcFlag == null && DatabaseDescriptor.startRpc()))
- thriftServer.start();
- else
- logger.info("Not starting RPC server as requested. Use JMX (StorageService->startRPCServer()) or nodetool (enablethrift) to start it");
}
/**
@@ -567,8 +599,6 @@
// On linux, this doesn't entirely shut down Cassandra, just the RPC server.
// jsvc takes care of taking the rest down
logger.info("Cassandra shutting down...");
- if (thriftServer != null)
- thriftServer.stop();
if (nativeTransportService != null)
nativeTransportService.destroy();
StorageService.instance.setRpcReady(false);
@@ -648,7 +678,7 @@
catch (Throwable e)
{
boolean logStackTrace =
- e instanceof ConfigurationException ? ((ConfigurationException)e).logStackTrace : true;
+ e instanceof ConfigurationException ? ((ConfigurationException)e).logStackTrace : true;
System.out.println("Exception (" + e.getClass().getName() + ") encountered during startup: " + e.getMessage());
@@ -723,16 +753,6 @@
return nativeTransportService != null ? nativeTransportService.isRunning() : false;
}
- public int getMaxNativeProtocolVersion()
- {
- return nativeTransportService.getMaxProtocolVersion();
- }
-
- public void refreshMaxNativeProtocolVersion()
- {
- if (nativeTransportService != null)
- nativeTransportService.refreshMaxNegotiableProtocolVersion();
- }
/**
* A convenience method to stop and destroy the daemon in one shot.
@@ -758,6 +778,11 @@
instance.activate();
}
+ public void clearConnectionHistory()
+ {
+ nativeTransportService.clearConnectionHistory();
+ }
+
private void exitOrFail(int code, String message)
{
exitOrFail(code, message, null);
@@ -810,5 +835,7 @@
* Returns whether the server is currently running.
*/
public boolean isRunning();
+
+ public void clearConnectionHistory();
}
}
diff --git a/src/java/org/apache/cassandra/service/ClientState.java b/src/java/org/apache/cassandra/service/ClientState.java
index 15262d7..496cabd 100644
--- a/src/java/org/apache/cassandra/service/ClientState.java
+++ b/src/java/org/apache/cassandra/service/ClientState.java
@@ -17,10 +17,12 @@
*/
package org.apache.cassandra.service;
+import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.SocketAddress;
import java.util.Arrays;
import java.util.HashSet;
+import java.util.Optional;
import java.util.Set;
import java.util.concurrent.atomic.AtomicLong;
@@ -28,23 +30,25 @@
import org.slf4j.LoggerFactory;
import org.apache.cassandra.auth.*;
-import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.db.virtual.VirtualSchemaKeyspace;
+import org.apache.cassandra.exceptions.RequestExecutionException;
+import org.apache.cassandra.exceptions.RequestValidationException;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.schema.TableMetadataRef;
import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.config.Schema;
-import org.apache.cassandra.config.SchemaConstants;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.SchemaConstants;
import org.apache.cassandra.cql3.QueryHandler;
import org.apache.cassandra.cql3.QueryProcessor;
import org.apache.cassandra.cql3.functions.Function;
import org.apache.cassandra.db.SystemKeyspace;
+import org.apache.cassandra.dht.Datacenters;
import org.apache.cassandra.exceptions.AuthenticationException;
import org.apache.cassandra.exceptions.InvalidRequestException;
-import org.apache.cassandra.exceptions.RequestExecutionException;
import org.apache.cassandra.exceptions.UnauthorizedException;
import org.apache.cassandra.schema.SchemaKeyspace;
-import org.apache.cassandra.thrift.ThriftValidation;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.JVMStabilityInspector;
-import org.apache.cassandra.utils.CassandraVersion;
/**
* State related to a client connection.
@@ -52,20 +56,23 @@
public class ClientState
{
private static final Logger logger = LoggerFactory.getLogger(ClientState.class);
- public static final CassandraVersion DEFAULT_CQL_VERSION = org.apache.cassandra.cql3.QueryProcessor.CQL_VERSION;
private static final Set<IResource> READABLE_SYSTEM_RESOURCES = new HashSet<>();
private static final Set<IResource> PROTECTED_AUTH_RESOURCES = new HashSet<>();
- private static final Set<IResource> DROPPABLE_SYSTEM_AUTH_TABLES = new HashSet<>();
+
static
{
// We want these system cfs to be always readable to authenticated users since many tools rely on them
// (nodetool, cqlsh, bulkloader, etc.)
- for (String cf : Arrays.asList(SystemKeyspace.LOCAL, SystemKeyspace.PEERS))
+ for (String cf : Arrays.asList(SystemKeyspace.LOCAL, SystemKeyspace.LEGACY_PEERS, SystemKeyspace.PEERS_V2))
READABLE_SYSTEM_RESOURCES.add(DataResource.table(SchemaConstants.SYSTEM_KEYSPACE_NAME, cf));
+ // make all schema tables readable by default (required by the drivers)
SchemaKeyspace.ALL.forEach(table -> READABLE_SYSTEM_RESOURCES.add(DataResource.table(SchemaConstants.SCHEMA_KEYSPACE_NAME, table)));
+ // make all virtual schema tables readable by default as well
+ VirtualSchemaKeyspace.instance.tables().forEach(t -> READABLE_SYSTEM_RESOURCES.add(t.metadata().resource));
+
// neither clients nor tools need authentication/authorization
if (DatabaseDescriptor.isDaemonInitialized())
{
@@ -73,22 +80,12 @@
PROTECTED_AUTH_RESOURCES.addAll(DatabaseDescriptor.getAuthorizer().protectedResources());
PROTECTED_AUTH_RESOURCES.addAll(DatabaseDescriptor.getRoleManager().protectedResources());
}
-
- DROPPABLE_SYSTEM_AUTH_TABLES.add(DataResource.table(SchemaConstants.AUTH_KEYSPACE_NAME, PasswordAuthenticator.LEGACY_CREDENTIALS_TABLE));
- DROPPABLE_SYSTEM_AUTH_TABLES.add(DataResource.table(SchemaConstants.AUTH_KEYSPACE_NAME, CassandraRoleManager.LEGACY_USERS_TABLE));
- DROPPABLE_SYSTEM_AUTH_TABLES.add(DataResource.table(SchemaConstants.AUTH_KEYSPACE_NAME, CassandraAuthorizer.USER_PERMISSIONS));
}
// Current user for the session
private volatile AuthenticatedUser user;
private volatile String keyspace;
- /**
- * Force Compact Tables to be represented as CQL ones for the current client session (simulates
- * ALTER .. DROP COMPACT STORAGE but only for this session)
- */
- private volatile boolean noCompactMode;
-
private static final QueryHandler cqlQueryHandler;
static
{
@@ -117,6 +114,10 @@
// The remote address of the client - null for internal clients.
private final InetSocketAddress remoteAddress;
+ // Driver String for the client
+ private volatile String driverName;
+ private volatile String driverVersion;
+
// The biggest timestamp that was returned by getTimestamp/assigned to a query. This is global to ensure that the
// timestamp assigned are strictly monotonic on a node, which is likely what user expect intuitively (more likely,
// most new user will intuitively expect timestamp to be strictly monotonic cluster-wise, but while that last part
@@ -140,6 +141,16 @@
this.user = AuthenticatedUser.ANONYMOUS_USER;
}
+ protected ClientState(ClientState source)
+ {
+ this.isInternal = source.isInternal;
+ this.remoteAddress = source.remoteAddress;
+ this.user = source.user;
+ this.keyspace = source.keyspace;
+ this.driverName = source.driverName;
+ this.driverVersion = source.driverVersion;
+ }
+
/**
* @return a ClientState object for internal C* calls (not limited by any kind of auth).
*/
@@ -148,8 +159,15 @@
return new ClientState();
}
+ public static ClientState forInternalCalls(String keyspace)
+ {
+ ClientState state = new ClientState();
+ state.setKeyspace(keyspace);
+ return state;
+ }
+
/**
- * @return a ClientState object for external clients (thrift/native protocol users).
+ * @return a ClientState object for external clients (native protocol users).
*/
public static ClientState forExternalCalls(SocketAddress remoteAddress)
{
@@ -157,10 +175,26 @@
}
/**
+ * Clone this ClientState object, but use the provided keyspace instead of the
+ * keyspace in this ClientState object.
+ *
+ * @return a new ClientState object if the keyspace argument is non-null. Otherwise do not clone
+ * and return this ClientState object.
+ */
+ public ClientState cloneWithKeyspaceIfSet(String keyspace)
+ {
+ if (keyspace == null)
+ return this;
+ ClientState clientState = new ClientState(this);
+ clientState.setKeyspace(keyspace);
+ return clientState;
+ }
+
+ /**
* This clock guarantees that updates for the same ClientState will be ordered
* in the sequence seen, even if multiple updates happen in the same millisecond.
*/
- public long getTimestamp()
+ public static long getTimestamp()
{
while (true)
{
@@ -230,6 +264,26 @@
}
}
+ public Optional<String> getDriverName()
+ {
+ return Optional.ofNullable(driverName);
+ }
+
+ public Optional<String> getDriverVersion()
+ {
+ return Optional.ofNullable(driverVersion);
+ }
+
+ public void setDriverName(String driverName)
+ {
+ this.driverName = driverName;
+ }
+
+ public void setDriverVersion(String driverVersion)
+ {
+ this.driverVersion = driverVersion;
+ }
+
public static QueryHandler getCQLQueryHandler()
{
return cqlQueryHandler;
@@ -240,6 +294,11 @@
return remoteAddress;
}
+ InetAddress getClientAddress()
+ {
+ return isInternal ? null : remoteAddress.getAddress();
+ }
+
public String getRawKeyspace()
{
return keyspace;
@@ -252,33 +311,20 @@
return keyspace;
}
- public void setKeyspace(String ks) throws InvalidRequestException
+ public void setKeyspace(String ks)
{
// Skip keyspace validation for non-authenticated users. Apparently, some client libraries
// call set_keyspace() before calling login(), and we have to handle that.
- if (user != null && Schema.instance.getKSMetaData(ks) == null)
+ if (user != null && Schema.instance.getKeyspaceMetadata(ks) == null)
throw new InvalidRequestException("Keyspace '" + ks + "' does not exist");
keyspace = ks;
}
- public void setNoCompactMode()
- {
- this.noCompactMode = true;
- }
-
- public boolean isNoCompactMode()
- {
- return noCompactMode;
- }
-
/**
* Attempts to login the given user.
*/
- public void login(AuthenticatedUser user) throws AuthenticationException
+ public void login(AuthenticatedUser user)
{
- // Login privilege is not inherited via granted roles, so just
- // verify that the role with the credentials that were actually
- // supplied has it
if (user.isAnonymous() || canLogin(user))
this.user = user;
else
@@ -289,40 +335,43 @@
{
try
{
- return DatabaseDescriptor.getRoleManager().canLogin(user.getPrimaryRole());
- } catch (RequestExecutionException e) {
+ return user.canLogin();
+ }
+ catch (RequestExecutionException | RequestValidationException e)
+ {
throw new AuthenticationException("Unable to perform authentication: " + e.getMessage(), e);
}
}
- public void hasAllKeyspacesAccess(Permission perm) throws UnauthorizedException
+ public void ensureAllKeyspacesPermission(Permission perm)
{
if (isInternal)
return;
validateLogin();
- ensureHasPermission(perm, DataResource.root());
+ ensurePermission(perm, DataResource.root());
}
- public void hasKeyspaceAccess(String keyspace, Permission perm) throws UnauthorizedException, InvalidRequestException
+ public void ensureKeyspacePermission(String keyspace, Permission perm)
{
- hasAccess(keyspace, perm, DataResource.keyspace(keyspace));
+ ensurePermission(keyspace, perm, DataResource.keyspace(keyspace));
}
- public void hasColumnFamilyAccess(String keyspace, String columnFamily, Permission perm)
- throws UnauthorizedException, InvalidRequestException
+ public void ensureTablePermission(String keyspace, String table, Permission perm)
{
- ThriftValidation.validateColumnFamily(keyspace, columnFamily);
- hasAccess(keyspace, perm, DataResource.table(keyspace, columnFamily));
+ ensurePermission(keyspace, perm, DataResource.table(keyspace, table));
}
- public void hasColumnFamilyAccess(CFMetaData cfm, Permission perm)
- throws UnauthorizedException, InvalidRequestException
+ public void ensureTablePermission(TableMetadataRef tableRef, Permission perm)
{
- hasAccess(cfm.ksName, perm, cfm.resource);
+ ensureTablePermission(tableRef.get(), perm);
}
- private void hasAccess(String keyspace, Permission perm, DataResource resource)
- throws UnauthorizedException, InvalidRequestException
+ public void ensureTablePermission(TableMetadata table, Permission perm)
+ {
+ ensurePermission(table.keyspace, perm, table.resource);
+ }
+
+ private void ensurePermission(String keyspace, Permission perm, DataResource resource)
{
validateKeyspace(keyspace);
@@ -339,11 +388,10 @@
if (PROTECTED_AUTH_RESOURCES.contains(resource))
if ((perm == Permission.CREATE) || (perm == Permission.ALTER) || (perm == Permission.DROP))
throw new UnauthorizedException(String.format("%s schema is protected", resource));
-
- ensureHasPermission(perm, resource);
+ ensurePermission(perm, resource);
}
- public void ensureHasPermission(Permission perm, IResource resource) throws UnauthorizedException
+ public void ensurePermission(Permission perm, IResource resource)
{
if (!DatabaseDescriptor.getAuthorizer().requireAuthorization())
return;
@@ -353,12 +401,12 @@
if (((FunctionResource)resource).getKeyspace().equals(SchemaConstants.SYSTEM_KEYSPACE_NAME))
return;
- checkPermissionOnResourceChain(perm, resource);
+ ensurePermissionOnResourceChain(perm, resource);
}
- // Convenience method called from checkAccess method of CQLStatement
+ // Convenience method called from authorize method of CQLStatement
// Also avoids needlessly creating lots of FunctionResource objects
- public void ensureHasPermission(Permission permission, Function function)
+ public void ensurePermission(Permission permission, Function function)
{
// Save creating a FunctionResource is we don't need to
if (!DatabaseDescriptor.getAuthorizer().requireAuthorization())
@@ -368,12 +416,12 @@
if (function.isNative())
return;
- checkPermissionOnResourceChain(permission, FunctionResource.function(function.name().keyspace,
- function.name().name,
- function.argTypes()));
+ ensurePermissionOnResourceChain(permission, FunctionResource.function(function.name().keyspace,
+ function.name().name,
+ function.argTypes()));
}
- private void checkPermissionOnResourceChain(Permission perm, IResource resource)
+ private void ensurePermissionOnResourceChain(Permission perm, IResource resource)
{
for (IResource r : Resources.chain(resource))
if (authorize(r).contains(perm))
@@ -385,7 +433,7 @@
resource));
}
- private void preventSystemKSSchemaModification(String keyspace, DataResource resource, Permission perm) throws UnauthorizedException
+ private void preventSystemKSSchemaModification(String keyspace, DataResource resource, Permission perm)
{
// we only care about DDL statements
if (perm != Permission.ALTER && perm != Permission.DROP && perm != Permission.CREATE)
@@ -401,35 +449,37 @@
if (perm == Permission.ALTER && resource.isKeyspaceLevel())
return;
- // allow users with sufficient privileges to drop legacy tables in replicated system keyspaces
- if (perm == Permission.DROP && DROPPABLE_SYSTEM_AUTH_TABLES.contains(resource))
- return;
-
// prevent all other modifications of replicated system keyspaces
throw new UnauthorizedException(String.format("Cannot %s %s", perm, resource));
}
}
- public void validateLogin() throws UnauthorizedException
+ public void validateLogin()
{
if (user == null)
+ {
throw new UnauthorizedException("You have not logged in");
+ }
+ else if (!user.hasLocalAccess())
+ {
+ throw new UnauthorizedException(String.format("You do not have access to this datacenter (%s)", Datacenters.thisDatacenter()));
+ }
}
- public void ensureNotAnonymous() throws UnauthorizedException
+ public void ensureNotAnonymous()
{
validateLogin();
if (user.isAnonymous())
throw new UnauthorizedException("You have to be logged in and not anonymous to perform this request");
}
- public void ensureIsSuper(String message) throws UnauthorizedException
+ public void ensureIsSuperuser(String message)
{
if (DatabaseDescriptor.getAuthenticator().requireAuthentication() && (user == null || !user.isSuper()))
throw new UnauthorizedException(message);
}
- private static void validateKeyspace(String keyspace) throws InvalidRequestException
+ private static void validateKeyspace(String keyspace)
{
if (keyspace == null)
throw new InvalidRequestException("You have not set a keyspace for this session");
@@ -440,11 +490,6 @@
return user;
}
- public static CassandraVersion[] getCQLSupportedVersion()
- {
- return new CassandraVersion[]{ QueryProcessor.CQL_VERSION };
- }
-
private Set<Permission> authorize(IResource resource)
{
return user.getPermissions(resource);
diff --git a/src/java/org/apache/cassandra/service/DataResolver.java b/src/java/org/apache/cassandra/service/DataResolver.java
deleted file mode 100644
index 3a2d54d..0000000
--- a/src/java/org/apache/cassandra/service/DataResolver.java
+++ /dev/null
@@ -1,949 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.service;
-
-import java.net.InetAddress;
-import java.util.*;
-import java.util.concurrent.TimeoutException;
-import java.util.function.UnaryOperator;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Joiner;
-import com.google.common.collect.Iterables;
-
-import org.apache.cassandra.concurrent.Stage;
-import org.apache.cassandra.concurrent.StageManager;
-import org.apache.cassandra.config.*;
-import org.apache.cassandra.cql3.statements.IndexTarget;
-import org.apache.cassandra.db.*;
-import org.apache.cassandra.db.filter.*;
-import org.apache.cassandra.db.filter.DataLimits.Counter;
-import org.apache.cassandra.db.partitions.*;
-import org.apache.cassandra.db.rows.*;
-import org.apache.cassandra.db.transform.*;
-import org.apache.cassandra.dht.AbstractBounds;
-import org.apache.cassandra.dht.ExcludingBounds;
-import org.apache.cassandra.dht.Range;
-import org.apache.cassandra.exceptions.ReadTimeoutException;
-import org.apache.cassandra.index.sasi.SASIIndex;
-import org.apache.cassandra.net.*;
-import org.apache.cassandra.schema.IndexMetadata;
-import org.apache.cassandra.tracing.Tracing;
-import org.apache.cassandra.utils.FBUtilities;
-
-public class DataResolver extends ResponseResolver
-{
- private static final boolean DROP_OVERSIZED_READ_REPAIR_MUTATIONS =
- Boolean.getBoolean("cassandra.drop_oversized_readrepair_mutations");
-
- @VisibleForTesting
- final List<AsyncOneResponse> repairResults = Collections.synchronizedList(new ArrayList<>());
- private final long queryStartNanoTime;
- private final boolean enforceStrictLiveness;
-
- DataResolver(Keyspace keyspace, ReadCommand command, ConsistencyLevel consistency, int maxResponseCount, long queryStartNanoTime)
- {
- super(keyspace, command, consistency, maxResponseCount);
- this.queryStartNanoTime = queryStartNanoTime;
- this.enforceStrictLiveness = command.metadata().enforceStrictLiveness();
- }
-
- public PartitionIterator getData()
- {
- ReadResponse response = responses.iterator().next().payload;
- return UnfilteredPartitionIterators.filter(response.makeIterator(command), command.nowInSec());
- }
-
- public boolean isDataPresent()
- {
- return !responses.isEmpty();
- }
-
- public void compareResponses()
- {
- // We need to fully consume the results to trigger read repairs if appropriate
- try (PartitionIterator iterator = resolve())
- {
- PartitionIterators.consume(iterator);
- }
- }
-
- public PartitionIterator resolve()
- {
- if (!needsReplicaFilteringProtection())
- {
- ResolveContext context = new ResolveContext(responses.size());
- return resolveWithReadRepair(context,
- i -> shortReadProtectedResponse(i, context),
- UnaryOperator.identity());
- }
-
- return resolveWithReplicaFilteringProtection();
- }
-
- private boolean needsReplicaFilteringProtection()
- {
- if (command.rowFilter().isEmpty())
- return false;
-
- IndexMetadata indexDef = command.indexMetadata();
- if (indexDef != null && indexDef.isCustom())
- {
- String className = indexDef.options.get(IndexTarget.CUSTOM_INDEX_OPTION_NAME);
- return !SASIIndex.class.getName().equals(className);
- }
-
- return true;
- }
-
- private class ResolveContext
- {
- private final InetAddress[] sources;
- private final DataLimits.Counter mergedResultCounter;
-
- private ResolveContext(int count)
- {
- assert count <= responses.size();
- this.sources = new InetAddress[count];
- for (int i = 0; i < count; i++)
- sources[i] = responses.get(i).from;
- this.mergedResultCounter = command.limits().newCounter(command.nowInSec(),
- true,
- command.selectsFullPartition(),
- enforceStrictLiveness);
- }
-
- private boolean needShortReadProtection()
- {
- // If we have only one result, there is no read repair to do and we can't get short reads
- // Also, so-called "short reads" stems from nodes returning only a subset of the results they have for a
- // partition due to the limit, but that subset not being enough post-reconciliation. So if we don't have limit,
- // don't bother protecting against short reads.
- return sources.length > 1 && !command.limits().isUnlimited();
- }
- }
-
- @FunctionalInterface
- private interface ResponseProvider
- {
- UnfilteredPartitionIterator getResponse(int i);
- }
-
- private UnfilteredPartitionIterator shortReadProtectedResponse(int i, ResolveContext context)
- {
- UnfilteredPartitionIterator originalResponse = responses.get(i).payload.makeIterator(command);
-
- return context.needShortReadProtection()
- ? extendWithShortReadProtection(originalResponse, context.sources[i], context.mergedResultCounter)
- : originalResponse;
- }
-
- private PartitionIterator resolveWithReadRepair(ResolveContext context,
- ResponseProvider responseProvider,
- UnaryOperator<PartitionIterator> preCountFilter)
- {
- return resolveInternal(context, new RepairMergeListener(context.sources), responseProvider, preCountFilter);
- }
-
- private PartitionIterator resolveWithReplicaFilteringProtection()
- {
- // Protecting against inconsistent replica filtering (some replica returning a row that is outdated but that
- // wouldn't be removed by normal reconciliation because up-to-date replica have filtered the up-to-date version
- // of that row) works in 3 steps:
- // 1) we read the full response just to collect rows that may be outdated (the ones we got from some
- // replica but didn't got any response for other; it could be those other replica have filtered a more
- // up-to-date result). In doing so, we do not count any of such "potentially outdated" row towards the
- // query limit. This simulate the worst case scenario where all those "potentially outdated" rows are
- // indeed outdated, and thus make sure we are guaranteed to read enough results (thanks to short read
- // protection).
- // 2) we query all the replica/rows we need to rule out whether those "potentially outdated" rows are outdated
- // or not.
- // 3) we re-read cached copies of each replica response using the "normal" read path merge with read-repair,
- // but where for each replica we use their original response _plus_ the additional rows queried in the
- // previous step (and apply the command#rowFilter() on the full result). Since the first phase has
- // pessimistically collected enough results for the case where all potentially outdated results are indeed
- // outdated, we shouldn't need further short-read protection requests during this phase.
-
- // We could get more responses while this method runs, which is ok (we're happy to ignore any response not here
- // at the beginning of this method), so grab the response count once and use that through the method.
- int count = responses.size();
- // We need separate contexts, as each context has his own counter
- ResolveContext firstPhaseContext = new ResolveContext(count);
- ResolveContext secondPhaseContext = new ResolveContext(count);
- ReplicaFilteringProtection rfp = new ReplicaFilteringProtection(keyspace, command, consistency, queryStartNanoTime, firstPhaseContext.sources);
- PartitionIterator firstPhasePartitions = resolveInternal(firstPhaseContext,
- rfp.mergeController(),
- i -> shortReadProtectedResponse(i, firstPhaseContext),
- UnaryOperator.identity());
-
- // Consume the first phase partitions to populate the replica filtering protection with both those materialized
- // partitions and the primary keys to be fetched.
- PartitionIterators.consume(firstPhasePartitions);
- firstPhasePartitions.close();
-
- // After reading the entire query results the protection helper should have cached all the partitions so we can
- // clear the responses accumulator for the sake of memory usage, given that the second phase might take long if
- // it needs to query replicas.
- responses.clearUnsafe();
-
- return resolveWithReadRepair(secondPhaseContext,
- rfp::queryProtectedPartitions,
- results -> command.rowFilter().filter(results, command.metadata(), command.nowInSec()));
- }
-
- private PartitionIterator resolveInternal(ResolveContext context,
- UnfilteredPartitionIterators.MergeListener mergeListener,
- ResponseProvider responseProvider,
- UnaryOperator<PartitionIterator> preCountFilter)
- {
- int count = context.sources.length;
- List<UnfilteredPartitionIterator> results = new ArrayList<>(count);
- for (int i = 0; i < count; i++)
- results.add(responseProvider.getResponse(i));
-
- /*
- * Even though every response, individually, will honor the limit, it is possible that we will, after the merge,
- * have more rows than the client requested. To make sure that we still conform to the original limit,
- * we apply a top-level post-reconciliation counter to the merged partition iterator.
- *
- * Short read protection logic (ShortReadRowsProtection.moreContents()) relies on this counter to be applied
- * to the current partition to work. For this reason we have to apply the counter transformation before
- * empty partition discard logic kicks in - for it will eagerly consume the iterator.
- *
- * That's why the order here is: 1) merge; 2) filter rows; 3) count; 4) discard empty partitions
- *
- * See CASSANDRA-13747 for more details.
- */
-
- UnfilteredPartitionIterator merged = UnfilteredPartitionIterators.merge(results, command.nowInSec(), mergeListener);
- FilteredPartitions filtered =
- FilteredPartitions.filter(merged, new Filter(command.nowInSec(), command.metadata().enforceStrictLiveness()));
- PartitionIterator counted = Transformation.apply(preCountFilter.apply(filtered), context.mergedResultCounter);
-
- return command.isForThrift()
- ? counted
- : Transformation.apply(counted, new EmptyPartitionsDiscarder());
- }
-
- private class RepairMergeListener implements UnfilteredPartitionIterators.MergeListener
- {
- private final InetAddress[] sources;
-
- private RepairMergeListener(InetAddress[] sources)
- {
- this.sources = sources;
- }
-
- public UnfilteredRowIterators.MergeListener getRowMergeListener(DecoratedKey partitionKey, List<UnfilteredRowIterator> versions)
- {
- return new MergeListener(partitionKey, columns(versions), isReversed(versions));
- }
-
- private PartitionColumns columns(List<UnfilteredRowIterator> versions)
- {
- Columns statics = Columns.NONE;
- Columns regulars = Columns.NONE;
- for (UnfilteredRowIterator iter : versions)
- {
- if (iter == null)
- continue;
-
- PartitionColumns cols = iter.columns();
- statics = statics.mergeTo(cols.statics);
- regulars = regulars.mergeTo(cols.regulars);
- }
- return new PartitionColumns(statics, regulars);
- }
-
- private boolean isReversed(List<UnfilteredRowIterator> versions)
- {
- for (UnfilteredRowIterator iter : versions)
- {
- if (iter == null)
- continue;
-
- // Everything will be in the same order
- return iter.isReverseOrder();
- }
-
- assert false : "Expected at least one iterator";
- return false;
- }
-
- public void close()
- {
- try
- {
- FBUtilities.waitOnFutures(repairResults, DatabaseDescriptor.getWriteRpcTimeout());
- }
- catch (TimeoutException ex)
- {
- // We got all responses, but timed out while repairing
- int blockFor = consistency.blockFor(keyspace);
- if (Tracing.isTracing())
- Tracing.trace("Timed out while read-repairing after receiving all {} data and digest responses", blockFor);
- else
- logger.debug("Timeout while read-repairing after receiving all {} data and digest responses", blockFor);
-
- throw new ReadTimeoutException(consistency, blockFor-1, blockFor, true);
- }
- }
-
- private class MergeListener implements UnfilteredRowIterators.MergeListener
- {
- private final DecoratedKey partitionKey;
- private final PartitionColumns columns;
- private final boolean isReversed;
- private final PartitionUpdate[] repairs = new PartitionUpdate[sources.length];
-
- private final Row.Builder[] currentRows = new Row.Builder[sources.length];
- private final RowDiffListener diffListener;
-
- // The partition level deletion for the merge row.
- private DeletionTime partitionLevelDeletion;
- // When merged has a currently open marker, its time. null otherwise.
- private DeletionTime mergedDeletionTime;
- // For each source, the time of the current deletion as known by the source.
- private final DeletionTime[] sourceDeletionTime = new DeletionTime[sources.length];
- // For each source, record if there is an open range to send as repair, and from where.
- private final ClusteringBound[] markerToRepair = new ClusteringBound[sources.length];
-
- private MergeListener(DecoratedKey partitionKey, PartitionColumns columns, boolean isReversed)
- {
- this.partitionKey = partitionKey;
- this.columns = columns;
- this.isReversed = isReversed;
-
- this.diffListener = new RowDiffListener()
- {
- public void onPrimaryKeyLivenessInfo(int i, Clustering clustering, LivenessInfo merged, LivenessInfo original)
- {
- if (merged != null && !merged.equals(original))
- currentRow(i, clustering).addPrimaryKeyLivenessInfo(merged);
- }
-
- public void onDeletion(int i, Clustering clustering, Row.Deletion merged, Row.Deletion original)
- {
- if (merged != null && !merged.equals(original))
- currentRow(i, clustering).addRowDeletion(merged);
- }
-
- public void onComplexDeletion(int i, Clustering clustering, ColumnDefinition column, DeletionTime merged, DeletionTime original)
- {
- if (merged != null && !merged.equals(original))
- currentRow(i, clustering).addComplexDeletion(column, merged);
- }
-
- public void onCell(int i, Clustering clustering, Cell merged, Cell original)
- {
- if (merged != null && !merged.equals(original) && isQueried(merged))
- currentRow(i, clustering).addCell(merged);
- }
-
- private boolean isQueried(Cell cell)
- {
- // When we read, we may have some cell that have been fetched but are not selected by the user. Those cells may
- // have empty values as optimization (see CASSANDRA-10655) and hence they should not be included in the read-repair.
- // This is fine since those columns are not actually requested by the user and are only present for the sake of CQL
- // semantic (making sure we can always distinguish between a row that doesn't exist from one that do exist but has
- /// no value for the column requested by the user) and so it won't be unexpected by the user that those columns are
- // not repaired.
- ColumnDefinition column = cell.column();
- ColumnFilter filter = command.columnFilter();
- return column.isComplex() ? filter.fetchedCellIsQueried(column, cell.path()) : filter.fetchedColumnIsQueried(column);
- }
- };
- }
-
- private PartitionUpdate update(int i)
- {
- if (repairs[i] == null)
- repairs[i] = new PartitionUpdate(command.metadata(), partitionKey, columns, 1);
- return repairs[i];
- }
-
- /**
- * The partition level deletion with with which source {@code i} is currently repaired, or
- * {@code DeletionTime.LIVE} if the source is not repaired on the partition level deletion (meaning it was
- * up to date on it). The output* of this method is only valid after the call to
- * {@link #onMergedPartitionLevelDeletion}.
- */
- private DeletionTime partitionLevelRepairDeletion(int i)
- {
- return repairs[i] == null ? DeletionTime.LIVE : repairs[i].partitionLevelDeletion();
- }
-
- private Row.Builder currentRow(int i, Clustering clustering)
- {
- if (currentRows[i] == null)
- {
- currentRows[i] = BTreeRow.sortedBuilder();
- currentRows[i].newRow(clustering);
- }
- return currentRows[i];
- }
-
- public void onMergedPartitionLevelDeletion(DeletionTime mergedDeletion, DeletionTime[] versions)
- {
- this.partitionLevelDeletion = mergedDeletion;
- for (int i = 0; i < versions.length; i++)
- {
- if (mergedDeletion.supersedes(versions[i]))
- update(i).addPartitionDeletion(mergedDeletion);
- }
- }
-
- public Row onMergedRows(Row merged, Row[] versions)
- {
- // If a row was shadowed post merged, it must be by a partition level or range tombstone, and we handle
- // those case directly in their respective methods (in other words, it would be inefficient to send a row
- // deletion as repair when we know we've already send a partition level or range tombstone that covers it).
- if (merged.isEmpty())
- return merged;
-
- Rows.diff(diffListener, merged, versions);
- for (int i = 0; i < currentRows.length; i++)
- {
- if (currentRows[i] != null)
- update(i).add(currentRows[i].build());
- }
- Arrays.fill(currentRows, null);
-
- return merged;
- }
-
- private DeletionTime currentDeletion()
- {
- return mergedDeletionTime == null ? partitionLevelDeletion : mergedDeletionTime;
- }
-
- public void onMergedRangeTombstoneMarkers(RangeTombstoneMarker merged, RangeTombstoneMarker[] versions)
- {
- try
- {
- // The code for merging range tombstones is a tad complex and we had the assertions there triggered
- // unexpectedly in a few occasions (CASSANDRA-13237, CASSANDRA-13719). It's hard to get insights
- // when that happen without more context that what the assertion errors give us however, hence the
- // catch here that basically gather as much as context as reasonable.
- internalOnMergedRangeTombstoneMarkers(merged, versions);
- }
- catch (AssertionError e)
- {
- // The following can be pretty verbose, but it's really only triggered if a bug happen, so we'd
- // rather get more info to debug than not.
- CFMetaData table = command.metadata();
- String details = String.format("Error merging RTs on %s.%s: command=%s, reversed=%b, merged=%s, versions=%s, sources={%s}, responses:%n %s",
- table.ksName, table.cfName,
- command.toCQLString(),
- isReversed,
- merged == null ? "null" : merged.toString(table),
- '[' + Joiner.on(", ").join(Iterables.transform(Arrays.asList(versions), rt -> rt == null ? "null" : rt.toString(table))) + ']',
- Arrays.toString(sources),
- makeResponsesDebugString());
- throw new AssertionError(details, e);
- }
- }
-
- private String makeResponsesDebugString()
- {
- return Joiner.on(",\n")
- .join(Iterables.transform(getMessages(), m -> m.from + " => " + m.payload.toDebugString(command, partitionKey)));
- }
-
- private void internalOnMergedRangeTombstoneMarkers(RangeTombstoneMarker merged, RangeTombstoneMarker[] versions)
- {
- // The current deletion as of dealing with this marker.
- DeletionTime currentDeletion = currentDeletion();
-
- for (int i = 0; i < versions.length; i++)
- {
- RangeTombstoneMarker marker = versions[i];
-
- // Update what the source now thinks is the current deletion
- if (marker != null)
- sourceDeletionTime[i] = marker.isOpen(isReversed) ? marker.openDeletionTime(isReversed) : null;
-
- // If merged == null, some of the source is opening or closing a marker
- if (merged == null)
- {
- // but if it's not this source, move to the next one
- if (marker == null)
- continue;
-
- // We have a close and/or open marker for a source, with nothing corresponding in merged.
- // Because merged is a superset, this imply that we have a current deletion (being it due to an
- // early opening in merged or a partition level deletion) and that this deletion will still be
- // active after that point. Further whatever deletion was open or is open by this marker on the
- // source, that deletion cannot supersedes the current one.
- //
- // But while the marker deletion (before and/or after this point) cannot supersede the current
- // deletion, we want to know if it's equal to it (both before and after), because in that case
- // the source is up to date and we don't want to include repair.
- //
- // So in practice we have 2 possible case:
- // 1) the source was up-to-date on deletion up to that point: then it won't be from that point
- // on unless it's a boundary and the new opened deletion time is also equal to the current
- // deletion (note that this implies the boundary has the same closing and opening deletion
- // time, which should generally not happen, but can due to legacy reading code not avoiding
- // this for a while, see CASSANDRA-13237).
- // 2) the source wasn't up-to-date on deletion up to that point and it may now be (if it isn't
- // we just have nothing to do for that marker).
- assert !currentDeletion.isLive() : currentDeletion.toString();
-
- // Is the source up to date on deletion? It's up to date if it doesn't have an open RT repair
- // nor an "active" partition level deletion (where "active" means that it's greater or equal
- // to the current deletion: if the source has a repaired partition deletion lower than the
- // current deletion, this means the current deletion is due to a previously open range tombstone,
- // and if the source isn't currently repaired for that RT, then it means it's up to date on it).
- DeletionTime partitionRepairDeletion = partitionLevelRepairDeletion(i);
- if (markerToRepair[i] == null && currentDeletion.supersedes(partitionRepairDeletion))
- {
- /*
- * Since there is an ongoing merged deletion, the only way we don't have an open repair for
- * this source is that it had a range open with the same deletion as current marker,
- * and the marker is closing it.
- */
- assert marker.isClose(isReversed) && currentDeletion.equals(marker.closeDeletionTime(isReversed))
- : String.format("currentDeletion=%s, marker=%s", currentDeletion, marker.toString(command.metadata()));
-
- // and so unless it's a boundary whose opening deletion time is still equal to the current
- // deletion (see comment above for why this can actually happen), we have to repair the source
- // from that point on.
- if (!(marker.isOpen(isReversed) && currentDeletion.equals(marker.openDeletionTime(isReversed))))
- markerToRepair[i] = marker.closeBound(isReversed).invert();
- }
- // In case 2) above, we only have something to do if the source is up-to-date after that point
- // (which, since the source isn't up-to-date before that point, means we're opening a new deletion
- // that is equal to the current one).
- else if (marker.isOpen(isReversed) && currentDeletion.equals(marker.openDeletionTime(isReversed)))
- {
- closeOpenMarker(i, marker.openBound(isReversed).invert());
- }
- }
- else
- {
- // We have a change of current deletion in merged (potentially to/from no deletion at all).
-
- if (merged.isClose(isReversed))
- {
- // We're closing the merged range. If we're recorded that this should be repaird for the
- // source, close and add said range to the repair to send.
- if (markerToRepair[i] != null)
- closeOpenMarker(i, merged.closeBound(isReversed));
-
- }
-
- if (merged.isOpen(isReversed))
- {
- // If we're opening a new merged range (or just switching deletion), then unless the source
- // is up to date on that deletion (note that we've updated what the source deleteion is
- // above), we'll have to sent the range to the source.
- DeletionTime newDeletion = merged.openDeletionTime(isReversed);
- DeletionTime sourceDeletion = sourceDeletionTime[i];
- if (!newDeletion.equals(sourceDeletion))
- markerToRepair[i] = merged.openBound(isReversed);
- }
- }
- }
-
- if (merged != null)
- mergedDeletionTime = merged.isOpen(isReversed) ? merged.openDeletionTime(isReversed) : null;
- }
-
- private void closeOpenMarker(int i, ClusteringBound close)
- {
- ClusteringBound open = markerToRepair[i];
- update(i).add(new RangeTombstone(Slice.make(isReversed ? close : open, isReversed ? open : close), currentDeletion()));
- markerToRepair[i] = null;
- }
-
- public void close()
- {
- for (int i = 0; i < repairs.length; i++)
- if (null != repairs[i])
- sendRepairMutation(repairs[i], sources[i]);
- }
-
- private void sendRepairMutation(PartitionUpdate partition, InetAddress destination)
- {
- Mutation mutation = new Mutation(partition);
- int messagingVersion = MessagingService.instance().getVersion(destination);
-
- int mutationSize = (int) Mutation.serializer.serializedSize(mutation, messagingVersion);
- int maxMutationSize = DatabaseDescriptor.getMaxMutationSize();
-
- if (mutationSize <= maxMutationSize)
- {
- Tracing.trace("Sending read-repair-mutation to {}", destination);
- // use a separate verb here to avoid writing hints on timeouts
- MessageOut<Mutation> message = mutation.createMessage(MessagingService.Verb.READ_REPAIR);
- repairResults.add(MessagingService.instance().sendRR(message, destination));
- ColumnFamilyStore.metricsFor(command.metadata().cfId).readRepairRequests.mark();
- }
- else if (DROP_OVERSIZED_READ_REPAIR_MUTATIONS)
- {
- logger.debug("Encountered an oversized ({}/{}) read repair mutation for table {}.{}, key {}, node {}",
- mutationSize,
- maxMutationSize,
- command.metadata().ksName,
- command.metadata().cfName,
- command.metadata().getKeyValidator().getString(partitionKey.getKey()),
- destination);
- }
- else
- {
- logger.warn("Encountered an oversized ({}/{}) read repair mutation for table {}.{}, key {}, node {}",
- mutationSize,
- maxMutationSize,
- command.metadata().ksName,
- command.metadata().cfName,
- command.metadata().getKeyValidator().getString(partitionKey.getKey()),
- destination);
-
- int blockFor = consistency.blockFor(keyspace);
- Tracing.trace("Timed out while read-repairing after receiving all {} data and digest responses", blockFor);
- throw new ReadTimeoutException(consistency, blockFor - 1, blockFor, true);
- }
- }
- }
- }
-
- private UnfilteredPartitionIterator extendWithShortReadProtection(UnfilteredPartitionIterator partitions,
- InetAddress source,
- DataLimits.Counter mergedResultCounter)
- {
- DataLimits.Counter singleResultCounter =
- command.limits().newCounter(command.nowInSec(), false, command.selectsFullPartition(), enforceStrictLiveness).onlyCount();
-
- ShortReadPartitionsProtection protection =
- new ShortReadPartitionsProtection(source, singleResultCounter, mergedResultCounter, queryStartNanoTime);
-
- /*
- * The order of extention and transformations is important here. Extending with more partitions has to happen
- * first due to the way BaseIterator.hasMoreContents() works: only transformations applied after extension will
- * be called on the first partition of the extended iterator.
- *
- * Additionally, we want singleResultCounter to be applied after SRPP, so that its applyToPartition() method will
- * be called last, after the extension done by SRRP.applyToPartition() call. That way we preserve the same order
- * when it comes to calling SRRP.moreContents() and applyToRow() callbacks.
- *
- * See ShortReadPartitionsProtection.applyToPartition() for more details.
- */
-
- // extend with moreContents() only if it's a range read command with no partition key specified
- if (!command.isLimitedToOnePartition())
- partitions = MorePartitions.extend(partitions, protection); // register SRPP.moreContents()
-
- partitions = Transformation.apply(partitions, protection); // register SRPP.applyToPartition()
- partitions = Transformation.apply(partitions, singleResultCounter); // register the per-source counter
-
- return partitions;
- }
-
- /*
- * We have a potential short read if the result from a given node contains the requested number of rows
- * (i.e. it has stopped returning results due to the limit), but some of them haven't
- * made it into the final post-reconciliation result due to other nodes' row, range, and/or partition tombstones.
- *
- * If that is the case, then that node may have more rows that we should fetch, as otherwise we could
- * ultimately return fewer rows than required. Also, those additional rows may contain tombstones which
- * which we also need to fetch as they may shadow rows or partitions from other replicas' results, which we would
- * otherwise return incorrectly.
- */
- private class ShortReadPartitionsProtection extends Transformation<UnfilteredRowIterator> implements MorePartitions<UnfilteredPartitionIterator>
- {
- private final InetAddress source;
-
- private final DataLimits.Counter singleResultCounter; // unmerged per-source counter
- private final DataLimits.Counter mergedResultCounter; // merged end-result counter
-
- private DecoratedKey lastPartitionKey; // key of the last observed partition
-
- private boolean partitionsFetched; // whether we've seen any new partitions since iteration start or last moreContents() call
-
- private final long queryStartNanoTime;
-
- private ShortReadPartitionsProtection(InetAddress source,
- DataLimits.Counter singleResultCounter,
- DataLimits.Counter mergedResultCounter,
- long queryStartNanoTime)
- {
- this.source = source;
- this.singleResultCounter = singleResultCounter;
- this.mergedResultCounter = mergedResultCounter;
- this.queryStartNanoTime = queryStartNanoTime;
- }
-
- @Override
- public UnfilteredRowIterator applyToPartition(UnfilteredRowIterator partition)
- {
- partitionsFetched = true;
-
- lastPartitionKey = partition.partitionKey();
-
- /*
- * Extend for moreContents() then apply protection to track lastClustering by applyToRow().
- *
- * If we don't apply the transformation *after* extending the partition with MoreRows,
- * applyToRow() method of protection will not be called on the first row of the new extension iterator.
- */
- ShortReadRowsProtection protection = new ShortReadRowsProtection(partition.metadata(), partition.partitionKey());
- return Transformation.apply(MoreRows.extend(partition, protection), protection);
- }
-
- /*
- * We only get here once all the rows and partitions in this iterator have been iterated over, and so
- * if the node had returned the requested number of rows but we still get here, then some results were
- * skipped during reconciliation.
- */
- public UnfilteredPartitionIterator moreContents()
- {
- // never try to request additional partitions from replicas if our reconciled partitions are already filled to the limit
- assert !mergedResultCounter.isDone();
-
- // we do not apply short read protection when we have no limits at all
- assert !command.limits().isUnlimited();
-
- /*
- * If this is a single partition read command or an (indexed) partition range read command with
- * a partition key specified, then we can't and shouldn't try fetch more partitions.
- */
- assert !command.isLimitedToOnePartition();
-
- /*
- * If the returned result doesn't have enough rows/partitions to satisfy even the original limit, don't ask for more.
- *
- * Can only take the short cut if there is no per partition limit set. Otherwise it's possible to hit false
- * positives due to some rows being uncounted for in certain scenarios (see CASSANDRA-13911).
- */
- if (!singleResultCounter.isDone() && command.limits().perPartitionCount() == DataLimits.NO_LIMIT)
- return null;
-
- /*
- * Either we had an empty iterator as the initial response, or our moreContents() call got us an empty iterator.
- * There is no point to ask the replica for more rows - it has no more in the requested range.
- */
- if (!partitionsFetched)
- return null;
- partitionsFetched = false;
-
- /*
- * We are going to fetch one partition at a time for thrift and potentially more for CQL.
- * The row limit will either be set to the per partition limit - if the command has no total row limit set, or
- * the total # of rows remaining - if it has some. If we don't grab enough rows in some of the partitions,
- * then future ShortReadRowsProtection.moreContents() calls will fetch the missing ones.
- */
- int toQuery = command.limits().count() != DataLimits.NO_LIMIT
- ? command.limits().count() - counted(mergedResultCounter)
- : command.limits().perPartitionCount();
-
- ColumnFamilyStore.metricsFor(command.metadata().cfId).shortReadProtectionRequests.mark();
- Tracing.trace("Requesting {} extra rows from {} for short read protection", toQuery, source);
-
- PartitionRangeReadCommand cmd = makeFetchAdditionalPartitionReadCommand(toQuery);
- return executeReadCommand(cmd);
- }
-
- // Counts the number of rows for regular queries and the number of groups for GROUP BY queries
- private int counted(Counter counter)
- {
- return command.limits().isGroupByLimit()
- ? counter.rowCounted()
- : counter.counted();
- }
-
- private PartitionRangeReadCommand makeFetchAdditionalPartitionReadCommand(int toQuery)
- {
- PartitionRangeReadCommand cmd = (PartitionRangeReadCommand) command;
-
- DataLimits newLimits = cmd.limits().forShortReadRetry(toQuery);
-
- AbstractBounds<PartitionPosition> bounds = cmd.dataRange().keyRange();
- AbstractBounds<PartitionPosition> newBounds = bounds.inclusiveRight()
- ? new Range<>(lastPartitionKey, bounds.right)
- : new ExcludingBounds<>(lastPartitionKey, bounds.right);
- DataRange newDataRange = cmd.dataRange().forSubRange(newBounds);
-
- return cmd.withUpdatedLimitsAndDataRange(newLimits, newDataRange);
- }
-
- private class ShortReadRowsProtection extends Transformation implements MoreRows<UnfilteredRowIterator>
- {
- private final CFMetaData metadata;
- private final DecoratedKey partitionKey;
-
- private Clustering lastClustering; // clustering of the last observed row
-
- private int lastCounted = 0; // last seen recorded # before attempting to fetch more rows
- private int lastFetched = 0; // # rows returned by last attempt to get more (or by the original read command)
- private int lastQueried = 0; // # extra rows requested from the replica last time
-
- private ShortReadRowsProtection(CFMetaData metadata, DecoratedKey partitionKey)
- {
- this.metadata = metadata;
- this.partitionKey = partitionKey;
- }
-
- @Override
- public Row applyToRow(Row row)
- {
- lastClustering = row.clustering();
- return row;
- }
-
- /*
- * We only get here once all the rows in this iterator have been iterated over, and so if the node
- * had returned the requested number of rows but we still get here, then some results were skipped
- * during reconciliation.
- */
- public UnfilteredRowIterator moreContents()
- {
- // never try to request additional rows from replicas if our reconciled partition is already filled to the limit
- assert !mergedResultCounter.isDoneForPartition();
-
- // we do not apply short read protection when we have no limits at all
- assert !command.limits().isUnlimited();
-
- /*
- * If the returned partition doesn't have enough rows to satisfy even the original limit, don't ask for more.
- *
- * Can only take the short cut if there is no per partition limit set. Otherwise it's possible to hit false
- * positives due to some rows being uncounted for in certain scenarios (see CASSANDRA-13911).
- */
- if (!singleResultCounter.isDoneForPartition() && command.limits().perPartitionCount() == DataLimits.NO_LIMIT)
- return null;
-
- /*
- * If the replica has no live rows in the partition, don't try to fetch more.
- *
- * Note that the previous branch [if (!singleResultCounter.isDoneForPartition()) return null] doesn't
- * always cover this scenario:
- * isDoneForPartition() is defined as [isDone() || rowInCurrentPartition >= perPartitionLimit],
- * and will return true if isDone() returns true, even if there are 0 rows counted in the current partition.
- *
- * This can happen with a range read if after 1+ rounds of short read protection requests we managed to fetch
- * enough extra rows for other partitions to satisfy the singleResultCounter's total row limit, but only
- * have tombstones in the current partition.
- *
- * One other way we can hit this condition is when the partition only has a live static row and no regular
- * rows. In that scenario the counter will remain at 0 until the partition is closed - which happens after
- * the moreContents() call.
- */
- if (countedInCurrentPartition(singleResultCounter) == 0)
- return null;
-
- /*
- * This is a table with no clustering columns, and has at most one row per partition - with EMPTY clustering.
- * We already have the row, so there is no point in asking for more from the partition.
- */
- if (Clustering.EMPTY == lastClustering)
- return null;
-
- lastFetched = countedInCurrentPartition(singleResultCounter) - lastCounted;
- lastCounted = countedInCurrentPartition(singleResultCounter);
-
- // getting back fewer rows than we asked for means the partition on the replica has been fully consumed
- if (lastQueried > 0 && lastFetched < lastQueried)
- return null;
-
- /*
- * At this point we know that:
- * 1. the replica returned [repeatedly?] as many rows as we asked for and potentially has more
- * rows in the partition
- * 2. at least one of those returned rows was shadowed by a tombstone returned from another
- * replica
- * 3. we haven't satisfied the client's limits yet, and should attempt to query for more rows to
- * avoid a short read
- *
- * In the ideal scenario, we would get exactly min(a, b) or fewer rows from the next request, where a and b
- * are defined as follows:
- * [a] limits.count() - mergedResultCounter.counted()
- * [b] limits.perPartitionCount() - mergedResultCounter.countedInCurrentPartition()
- *
- * It would be naive to query for exactly that many rows, as it's possible and not unlikely
- * that some of the returned rows would also be shadowed by tombstones from other hosts.
- *
- * Note: we don't know, nor do we care, how many rows from the replica made it into the reconciled result;
- * we can only tell how many in total we queried for, and that [0, mrc.countedInCurrentPartition()) made it.
- *
- * In general, our goal should be to minimise the number of extra requests - *not* to minimise the number
- * of rows fetched: there is a high transactional cost for every individual request, but a relatively low
- * marginal cost for each extra row requested.
- *
- * As such it's better to overfetch than to underfetch extra rows from a host; but at the same
- * time we want to respect paging limits and not blow up spectacularly.
- *
- * Note: it's ok to retrieve more rows that necessary since singleResultCounter is not stopping and only
- * counts.
- *
- * With that in mind, we'll just request the minimum of (count(), perPartitionCount()) limits.
- *
- * See CASSANDRA-13794 for more details.
- */
- lastQueried = Math.min(command.limits().count(), command.limits().perPartitionCount());
-
- ColumnFamilyStore.metricsFor(metadata.cfId).shortReadProtectionRequests.mark();
- Tracing.trace("Requesting {} extra rows from {} for short read protection", lastQueried, source);
-
- SinglePartitionReadCommand cmd = makeFetchAdditionalRowsReadCommand(lastQueried);
- return UnfilteredPartitionIterators.getOnlyElement(executeReadCommand(cmd), cmd);
- }
-
- // Counts the number of rows for regular queries and the number of groups for GROUP BY queries
- private int countedInCurrentPartition(Counter counter)
- {
- return command.limits().isGroupByLimit()
- ? counter.rowCountedInCurrentPartition()
- : counter.countedInCurrentPartition();
- }
-
- private SinglePartitionReadCommand makeFetchAdditionalRowsReadCommand(int toQuery)
- {
- ClusteringIndexFilter filter = command.clusteringIndexFilter(partitionKey);
- if (null != lastClustering)
- filter = filter.forPaging(metadata.comparator, lastClustering, false);
-
- return SinglePartitionReadCommand.create(command.isForThrift(),
- command.metadata(),
- command.nowInSec(),
- command.columnFilter(),
- command.rowFilter(),
- command.limits().forShortReadRetry(toQuery),
- partitionKey,
- filter,
- command.indexMetadata());
- }
- }
-
- private UnfilteredPartitionIterator executeReadCommand(ReadCommand cmd)
- {
- DataResolver resolver = new DataResolver(keyspace, cmd, ConsistencyLevel.ONE, 1, queryStartNanoTime);
- ReadCallback handler = new ReadCallback(resolver, ConsistencyLevel.ONE, cmd, Collections.singletonList(source), queryStartNanoTime);
-
- if (StorageProxy.canDoLocalRequest(source))
- StageManager.getStage(Stage.READ).maybeExecuteImmediately(new StorageProxy.LocalReadRunnable(cmd, handler));
- else
- MessagingService.instance().sendRRWithFailure(cmd.createMessage(MessagingService.current_version), source, handler);
-
- // We don't call handler.get() because we want to preserve tombstones since we're still in the middle of merging node results.
- handler.awaitResults();
- assert resolver.responses.size() == 1;
- return resolver.responses.get(0).payload.makeIterator(command);
- }
- }
-}
diff --git a/src/java/org/apache/cassandra/service/DatacenterSyncWriteResponseHandler.java b/src/java/org/apache/cassandra/service/DatacenterSyncWriteResponseHandler.java
index 9584611..389dcd5 100644
--- a/src/java/org/apache/cassandra/service/DatacenterSyncWriteResponseHandler.java
+++ b/src/java/org/apache/cassandra/service/DatacenterSyncWriteResponseHandler.java
@@ -17,17 +17,16 @@
*/
package org.apache.cassandra.service;
-import java.net.InetAddress;
-import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.locator.IEndpointSnitch;
import org.apache.cassandra.locator.NetworkTopologyStrategy;
-import org.apache.cassandra.net.MessageIn;
+import org.apache.cassandra.locator.Replica;
+import org.apache.cassandra.locator.ReplicaPlan;
+import org.apache.cassandra.net.Message;
import org.apache.cassandra.db.ConsistencyLevel;
import org.apache.cassandra.db.WriteType;
@@ -41,60 +40,66 @@
private final Map<String, AtomicInteger> responses = new HashMap<String, AtomicInteger>();
private final AtomicInteger acks = new AtomicInteger(0);
- public DatacenterSyncWriteResponseHandler(Collection<InetAddress> naturalEndpoints,
- Collection<InetAddress> pendingEndpoints,
- ConsistencyLevel consistencyLevel,
- Keyspace keyspace,
+ public DatacenterSyncWriteResponseHandler(ReplicaPlan.ForTokenWrite replicaPlan,
Runnable callback,
WriteType writeType,
long queryStartNanoTime)
{
// Response is been managed by the map so make it 1 for the superclass.
- super(keyspace, naturalEndpoints, pendingEndpoints, consistencyLevel, callback, writeType, queryStartNanoTime);
- assert consistencyLevel == ConsistencyLevel.EACH_QUORUM;
+ super(replicaPlan, callback, writeType, queryStartNanoTime);
+ assert replicaPlan.consistencyLevel() == ConsistencyLevel.EACH_QUORUM;
- NetworkTopologyStrategy strategy = (NetworkTopologyStrategy) keyspace.getReplicationStrategy();
-
- for (String dc : strategy.getDatacenters())
+ if (replicaPlan.keyspace().getReplicationStrategy() instanceof NetworkTopologyStrategy)
{
- int rf = strategy.getReplicationFactor(dc);
- responses.put(dc, new AtomicInteger((rf / 2) + 1));
+ NetworkTopologyStrategy strategy = (NetworkTopologyStrategy) replicaPlan.keyspace().getReplicationStrategy();
+ for (String dc : strategy.getDatacenters())
+ {
+ int rf = strategy.getReplicationFactor(dc).allReplicas;
+ responses.put(dc, new AtomicInteger((rf / 2) + 1));
+ }
+ }
+ else
+ {
+ responses.put(DatabaseDescriptor.getLocalDataCenter(), new AtomicInteger(ConsistencyLevel.quorumFor(replicaPlan.keyspace())));
}
// During bootstrap, we have to include the pending endpoints or we may fail the consistency level
// guarantees (see #833)
- for (InetAddress pending : pendingEndpoints)
+ for (Replica pending : replicaPlan.pending())
{
responses.get(snitch.getDatacenter(pending)).incrementAndGet();
}
}
- public void response(MessageIn<T> message)
+ public void onResponse(Message<T> message)
{
- String dataCenter = message == null
- ? DatabaseDescriptor.getLocalDataCenter()
- : snitch.getDatacenter(message.from);
-
- responses.get(dataCenter).getAndDecrement();
- acks.incrementAndGet();
-
- for (AtomicInteger i : responses.values())
+ try
{
- if (i.get() > 0)
- return;
- }
+ String dataCenter = message == null
+ ? DatabaseDescriptor.getLocalDataCenter()
+ : snitch.getDatacenter(message.from());
- // all the quorum conditions are met
- signal();
+ responses.get(dataCenter).getAndDecrement();
+ acks.incrementAndGet();
+
+ for (AtomicInteger i : responses.values())
+ {
+ if (i.get() > 0)
+ return;
+ }
+
+ // all the quorum conditions are met
+ signal();
+ }
+ finally
+ {
+ //Must be last after all subclass processing
+ logResponseToIdealCLDelegate(message);
+ }
}
protected int ackCount()
{
return acks.get();
}
-
- public boolean isLatencyForSnitch()
- {
- return false;
- }
}
diff --git a/src/java/org/apache/cassandra/service/DatacenterWriteResponseHandler.java b/src/java/org/apache/cassandra/service/DatacenterWriteResponseHandler.java
index 2309e87..a9583a3 100644
--- a/src/java/org/apache/cassandra/service/DatacenterWriteResponseHandler.java
+++ b/src/java/org/apache/cassandra/service/DatacenterWriteResponseHandler.java
@@ -17,49 +17,48 @@
*/
package org.apache.cassandra.service;
-import java.net.InetAddress;
-import java.util.Collection;
-
-import org.apache.cassandra.db.Keyspace;
-import org.apache.cassandra.net.MessageIn;
-import org.apache.cassandra.db.ConsistencyLevel;
import org.apache.cassandra.db.WriteType;
+import org.apache.cassandra.locator.InOurDcTester;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.ReplicaPlan;
+import org.apache.cassandra.net.Message;
+
+import java.util.function.Predicate;
/**
* This class blocks for a quorum of responses _in the local datacenter only_ (CL.LOCAL_QUORUM).
*/
public class DatacenterWriteResponseHandler<T> extends WriteResponseHandler<T>
{
- public DatacenterWriteResponseHandler(Collection<InetAddress> naturalEndpoints,
- Collection<InetAddress> pendingEndpoints,
- ConsistencyLevel consistencyLevel,
- Keyspace keyspace,
+ private final Predicate<InetAddressAndPort> waitingFor = InOurDcTester.endpoints();
+
+ public DatacenterWriteResponseHandler(ReplicaPlan.ForTokenWrite replicaPlan,
Runnable callback,
WriteType writeType,
long queryStartNanoTime)
{
- super(naturalEndpoints, pendingEndpoints, consistencyLevel, keyspace, callback, writeType, queryStartNanoTime);
- assert consistencyLevel.isDatacenterLocal();
+ super(replicaPlan, callback, writeType, queryStartNanoTime);
+ assert replicaPlan.consistencyLevel().isDatacenterLocal();
}
@Override
- public void response(MessageIn<T> message)
+ public void onResponse(Message<T> message)
{
- if (message == null || waitingFor(message.from))
- super.response(message);
+ if (message == null || waitingFor(message.from()))
+ {
+ super.onResponse(message);
+ }
+ else
+ {
+ //WriteResponseHandler.response will call logResonseToIdealCLDelegate so only do it if not calling WriteResponseHandler.response.
+ //Must be last after all subclass processing
+ logResponseToIdealCLDelegate(message);
+ }
}
@Override
- protected int totalBlockFor()
+ protected boolean waitingFor(InetAddressAndPort from)
{
- // during bootstrap, include pending endpoints (only local here) in the count
- // or we may fail the consistency level guarantees (see #833, #8058)
- return consistencyLevel.blockFor(keyspace) + consistencyLevel.countLocalEndpoints(pendingEndpoints);
- }
-
- @Override
- protected boolean waitingFor(InetAddress from)
- {
- return consistencyLevel.isLocal(from);
+ return waitingFor.test(from);
}
}
diff --git a/src/java/org/apache/cassandra/service/DigestMismatchException.java b/src/java/org/apache/cassandra/service/DigestMismatchException.java
deleted file mode 100644
index 18d5939..0000000
--- a/src/java/org/apache/cassandra/service/DigestMismatchException.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.service;
-
-import java.nio.ByteBuffer;
-
-import org.apache.cassandra.db.DecoratedKey;
-import org.apache.cassandra.utils.ByteBufferUtil;
-
-@SuppressWarnings("serial")
-public class DigestMismatchException extends Exception
-{
- public DigestMismatchException(DecoratedKey key, ByteBuffer digest1, ByteBuffer digest2)
- {
- super(String.format("Mismatch for key %s (%s vs %s)",
- key.toString(),
- ByteBufferUtil.bytesToHex(digest1),
- ByteBufferUtil.bytesToHex(digest2)));
- }
-}
diff --git a/src/java/org/apache/cassandra/service/DigestResolver.java b/src/java/org/apache/cassandra/service/DigestResolver.java
deleted file mode 100644
index 6a528e9..0000000
--- a/src/java/org/apache/cassandra/service/DigestResolver.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.service;
-
-import java.nio.ByteBuffer;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.cassandra.db.*;
-import org.apache.cassandra.db.partitions.PartitionIterator;
-import org.apache.cassandra.db.partitions.UnfilteredPartitionIterators;
-import org.apache.cassandra.net.MessageIn;
-
-public class DigestResolver extends ResponseResolver
-{
- private volatile ReadResponse dataResponse;
-
- public DigestResolver(Keyspace keyspace, ReadCommand command, ConsistencyLevel consistency, int maxResponseCount)
- {
- super(keyspace, command, consistency, maxResponseCount);
- }
-
- @Override
- public void preprocess(MessageIn<ReadResponse> message)
- {
- super.preprocess(message);
- if (dataResponse == null && !message.payload.isDigestResponse())
- dataResponse = message.payload;
- }
-
- /**
- * Special case of resolve() so that CL.ONE reads never throw DigestMismatchException in the foreground
- */
- public PartitionIterator getData()
- {
- assert isDataPresent();
- return UnfilteredPartitionIterators.filter(dataResponse.makeIterator(command), command.nowInSec());
- }
-
- /*
- * This method handles two different scenarios:
- *
- * a) we're handling the initial read of data from the closest replica + digests
- * from the rest. In this case we check the digests against each other,
- * throw an exception if there is a mismatch, otherwise return the data row.
- *
- * b) we're checking additional digests that arrived after the minimum to handle
- * the requested ConsistencyLevel, i.e. asynchronous read repair check
- */
- public PartitionIterator resolve() throws DigestMismatchException
- {
- if (responses.size() == 1)
- return getData();
-
- if (logger.isTraceEnabled())
- logger.trace("resolving {} responses", responses.size());
-
- compareResponses();
-
- return UnfilteredPartitionIterators.filter(dataResponse.makeIterator(command), command.nowInSec());
- }
-
- public void compareResponses() throws DigestMismatchException
- {
- long start = System.nanoTime();
-
- // validate digests against each other; throw immediately on mismatch.
- ByteBuffer digest = null;
- for (MessageIn<ReadResponse> message : responses)
- {
- ReadResponse response = message.payload;
-
- ByteBuffer newDigest = response.digest(command);
- if (digest == null)
- digest = newDigest;
- else if (!digest.equals(newDigest))
- // rely on the fact that only single partition queries use digests
- throw new DigestMismatchException(((SinglePartitionReadCommand)command).partitionKey(), digest, newDigest);
- }
-
- if (logger.isTraceEnabled())
- logger.trace("resolve: {} ms.", TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start));
- }
-
- public boolean isDataPresent()
- {
- return dataResponse != null;
- }
-}
diff --git a/src/java/org/apache/cassandra/service/EchoVerbHandler.java b/src/java/org/apache/cassandra/service/EchoVerbHandler.java
index d0c435e..76f23d4 100644
--- a/src/java/org/apache/cassandra/service/EchoVerbHandler.java
+++ b/src/java/org/apache/cassandra/service/EchoVerbHandler.java
@@ -19,24 +19,23 @@
* under the License.
*
*/
-
-
-import org.apache.cassandra.gms.EchoMessage;
import org.apache.cassandra.net.IVerbHandler;
-import org.apache.cassandra.net.MessageIn;
-import org.apache.cassandra.net.MessageOut;
+import org.apache.cassandra.net.Message;
import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.net.NoPayload;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-public class EchoVerbHandler implements IVerbHandler<EchoMessage>
+public class EchoVerbHandler implements IVerbHandler<NoPayload>
{
+ public static final EchoVerbHandler instance = new EchoVerbHandler();
+
private static final Logger logger = LoggerFactory.getLogger(EchoVerbHandler.class);
- public void doVerb(MessageIn<EchoMessage> message, int id)
+ public void doVerb(Message<NoPayload> message)
{
- MessageOut<EchoMessage> echoMessage = new MessageOut<EchoMessage>(MessagingService.Verb.REQUEST_RESPONSE, EchoMessage.instance, EchoMessage.serializer);
- logger.trace("Sending a EchoMessage reply {}", message.from);
- MessagingService.instance().sendReply(echoMessage, id, message.from);
+ logger.trace("Sending ECHO_RSP to {}", message.from());
+ MessagingService.instance().send(message.emptyResponse(), message.from());
}
}
diff --git a/src/java/org/apache/cassandra/service/EmbeddedCassandraService.java b/src/java/org/apache/cassandra/service/EmbeddedCassandraService.java
index 2515259..9172eab 100644
--- a/src/java/org/apache/cassandra/service/EmbeddedCassandraService.java
+++ b/src/java/org/apache/cassandra/service/EmbeddedCassandraService.java
@@ -20,8 +20,7 @@
import java.io.IOException;
/**
- * An embedded, in-memory cassandra storage service that listens
- * on the thrift interface as configured in cassandra.yaml
+ * An embedded, in-memory cassandra storage service.
* This kind of service is useful when running unit tests of
* services using cassandra for example.
*
@@ -51,4 +50,9 @@
cassandraDaemon.init(null);
cassandraDaemon.start();
}
+
+ public void stop()
+ {
+ cassandraDaemon.stop();
+ }
}
diff --git a/src/java/org/apache/cassandra/service/GCInspector.java b/src/java/org/apache/cassandra/service/GCInspector.java
index 787d79a..02fd720 100644
--- a/src/java/org/apache/cassandra/service/GCInspector.java
+++ b/src/java/org/apache/cassandra/service/GCInspector.java
@@ -17,6 +17,7 @@
*/
package org.apache.cassandra.service;
+import java.io.IOException;
import java.lang.management.GarbageCollectorMXBean;
import java.lang.management.ManagementFactory;
import java.lang.management.MemoryUsage;
@@ -27,7 +28,11 @@
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
+import javax.management.InstanceAlreadyExistsException;
+import javax.management.MBeanRegistrationException;
import javax.management.MBeanServer;
+import javax.management.MalformedObjectNameException;
+import javax.management.NotCompliantMBeanException;
import javax.management.Notification;
import javax.management.NotificationListener;
import javax.management.ObjectName;
@@ -49,9 +54,8 @@
{
public static final String MBEAN_NAME = "org.apache.cassandra.service:type=GCInspector";
private static final Logger logger = LoggerFactory.getLogger(GCInspector.class);
- final static long MIN_LOG_DURATION = DatabaseDescriptor.getGCLogThreshold();
- final static long GC_WARN_THRESHOLD_IN_MS = DatabaseDescriptor.getGCWarnThreshold();
- final static long STAT_THRESHOLD = GC_WARN_THRESHOLD_IN_MS != 0 ? GC_WARN_THRESHOLD_IN_MS : MIN_LOG_DURATION;
+ private volatile long gcLogThreshholdInMs = DatabaseDescriptor.getGCLogThreshold();
+ private volatile long gcWarnThreasholdInMs = DatabaseDescriptor.getGCWarnThreshold();
/*
* The field from java.nio.Bits that tracks the total number of allocated
@@ -59,13 +63,23 @@
*/
final static Field BITS_TOTAL_CAPACITY;
+
static
{
Field temp = null;
try
{
Class<?> bitsClass = Class.forName("java.nio.Bits");
- Field f = bitsClass.getDeclaredField("totalCapacity");
+ Field f;
+ try
+ {
+ f = bitsClass.getDeclaredField("totalCapacity");
+ }
+ catch (NoSuchFieldException ex)
+ {
+ // in Java11 it changed name to "TOTAL_CAPACITY"
+ f = bitsClass.getDeclaredField("TOTAL_CAPACITY");
+ }
f.setAccessible(true);
temp = f;
}
@@ -147,10 +161,11 @@
GarbageCollectorMXBean gc = ManagementFactory.newPlatformMXBeanProxy(mbs, name.getCanonicalName(), GarbageCollectorMXBean.class);
gcStates.put(gc.getName(), new GCState(gc, assumeGCIsPartiallyConcurrent(gc), assumeGCIsOldGen(gc)));
}
-
- MBeanWrapper.instance.registerMBean(this, new ObjectName(MBEAN_NAME));
+ ObjectName me = new ObjectName(MBEAN_NAME);
+ if (!mbs.isRegistered(me))
+ MBeanWrapper.instance.registerMBean(this, new ObjectName(MBEAN_NAME));
}
- catch (Exception e)
+ catch (MalformedObjectNameException | IOException e)
{
throw new RuntimeException(e);
}
@@ -277,16 +292,15 @@
if (state.compareAndSet(prev, new State(duration, bytes, prev)))
break;
}
-
- String st = sb.toString();
- if (GC_WARN_THRESHOLD_IN_MS != 0 && duration > GC_WARN_THRESHOLD_IN_MS)
- logger.warn(st);
- else if (duration > MIN_LOG_DURATION)
- logger.info(st);
+
+ if (gcWarnThreasholdInMs != 0 && duration > gcWarnThreasholdInMs)
+ logger.warn(sb.toString());
+ else if (duration > gcLogThreshholdInMs)
+ logger.info(sb.toString());
else if (logger.isTraceEnabled())
- logger.trace(st);
+ logger.trace(sb.toString());
- if (duration > STAT_THRESHOLD)
+ if (duration > this.getStatusThresholdInMs())
StatusLogger.log();
// if we just finished an old gen collection and we're still using a lot of memory, try to reduce the pressure
@@ -329,4 +343,40 @@
return -1;
}
}
+
+ public void setGcWarnThresholdInMs(long threshold)
+ {
+ if (threshold < 0)
+ throw new IllegalArgumentException("Threshold must be greater than or equal to 0");
+ if (threshold != 0 && threshold <= gcLogThreshholdInMs)
+ throw new IllegalArgumentException("Threshold must be greater than gcLogTreasholdInMs which is currently "
+ + gcLogThreshholdInMs);
+ gcWarnThreasholdInMs = threshold;
+ }
+
+ public long getGcWarnThresholdInMs()
+ {
+ return gcWarnThreasholdInMs;
+ }
+
+ public void setGcLogThresholdInMs(long threshold)
+ {
+ if (threshold <= 0)
+ throw new IllegalArgumentException("Threashold must be greater than 0");
+ if (gcWarnThreasholdInMs != 0 && threshold > gcWarnThreasholdInMs)
+ throw new IllegalArgumentException("Threashold must be less than gcWarnTreasholdInMs which is currently "
+ + gcWarnThreasholdInMs);
+ gcLogThreshholdInMs = threshold;
+ }
+
+ public long getGcLogThresholdInMs()
+ {
+ return gcLogThreshholdInMs;
+ }
+
+ public long getStatusThresholdInMs()
+ {
+ return gcWarnThreasholdInMs != 0 ? gcWarnThreasholdInMs : gcLogThreshholdInMs;
+ }
+
}
diff --git a/src/java/org/apache/cassandra/service/GCInspectorMXBean.java b/src/java/org/apache/cassandra/service/GCInspectorMXBean.java
index c26a67c..08795ed 100644
--- a/src/java/org/apache/cassandra/service/GCInspectorMXBean.java
+++ b/src/java/org/apache/cassandra/service/GCInspectorMXBean.java
@@ -20,6 +20,13 @@
public interface GCInspectorMXBean
{
- // returns { interval (ms), max(gc real time (ms)), sum(gc real time (ms)), sum((gc real time (ms))^2), sum(gc bytes), count(gc) }
- public double[] getAndResetStats();
+ /* @returns { interval (ms), max(gc real time (ms)), sum(gc real time (ms)), sum((gc real time (ms))^2), sum(gc bytes), count(gc) }
+ *
+ */
+ double[] getAndResetStats();
+ void setGcWarnThresholdInMs(long threshold);
+ long getGcWarnThresholdInMs();
+ void setGcLogThresholdInMs(long threshold);
+ long getGcLogThresholdInMs();
+ long getStatusThresholdInMs();
}
diff --git a/src/java/org/apache/cassandra/service/IEndpointLifecycleSubscriber.java b/src/java/org/apache/cassandra/service/IEndpointLifecycleSubscriber.java
index 24cb3d7..bc49d3b 100644
--- a/src/java/org/apache/cassandra/service/IEndpointLifecycleSubscriber.java
+++ b/src/java/org/apache/cassandra/service/IEndpointLifecycleSubscriber.java
@@ -17,7 +17,7 @@
*/
package org.apache.cassandra.service;
-import java.net.InetAddress;
+import org.apache.cassandra.locator.InetAddressAndPort;
/**
* Interface on which interested parties can be notified of high level endpoint
@@ -35,33 +35,33 @@
*
* @param endpoint the newly added endpoint.
*/
- public void onJoinCluster(InetAddress endpoint);
+ public void onJoinCluster(InetAddressAndPort endpoint);
/**
* Called when a new node leave the cluster (decommission or removeToken).
*
* @param endpoint the endpoint that is leaving.
*/
- public void onLeaveCluster(InetAddress endpoint);
+ public void onLeaveCluster(InetAddressAndPort endpoint);
/**
* Called when a node is marked UP.
*
* @param endpoint the endpoint marked UP.
*/
- public void onUp(InetAddress endpoint);
+ public void onUp(InetAddressAndPort endpoint);
/**
* Called when a node is marked DOWN.
*
* @param endpoint the endpoint marked DOWN.
*/
- public void onDown(InetAddress endpoint);
+ public void onDown(InetAddressAndPort endpoint);
/**
* Called when a node has moved (to a new token).
*
* @param endpoint the endpoint that has moved.
*/
- public void onMove(InetAddress endpoint);
+ public void onMove(InetAddressAndPort endpoint);
}
diff --git a/src/java/org/apache/cassandra/service/LoadBroadcaster.java b/src/java/org/apache/cassandra/service/LoadBroadcaster.java
index 945dd2f..35c0b62 100644
--- a/src/java/org/apache/cassandra/service/LoadBroadcaster.java
+++ b/src/java/org/apache/cassandra/service/LoadBroadcaster.java
@@ -17,12 +17,12 @@
*/
package org.apache.cassandra.service;
-import java.net.InetAddress;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.TimeUnit;
+import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.metrics.StorageMetrics;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -38,21 +38,21 @@
private static final Logger logger = LoggerFactory.getLogger(LoadBroadcaster.class);
- private ConcurrentMap<InetAddress, Double> loadInfo = new ConcurrentHashMap<InetAddress, java.lang.Double>();
+ private ConcurrentMap<InetAddressAndPort, Double> loadInfo = new ConcurrentHashMap<>();
private LoadBroadcaster()
{
Gossiper.instance.register(this);
}
- public void onChange(InetAddress endpoint, ApplicationState state, VersionedValue value)
+ public void onChange(InetAddressAndPort endpoint, ApplicationState state, VersionedValue value)
{
if (state != ApplicationState.LOAD)
return;
loadInfo.put(endpoint, Double.valueOf(value.value));
}
- public void onJoin(InetAddress endpoint, EndpointState epState)
+ public void onJoin(InetAddressAndPort endpoint, EndpointState epState)
{
VersionedValue localValue = epState.getApplicationState(ApplicationState.LOAD);
if (localValue != null)
@@ -61,20 +61,20 @@
}
}
- public void beforeChange(InetAddress endpoint, EndpointState currentState, ApplicationState newStateKey, VersionedValue newValue) {}
+ public void beforeChange(InetAddressAndPort endpoint, EndpointState currentState, ApplicationState newStateKey, VersionedValue newValue) {}
- public void onAlive(InetAddress endpoint, EndpointState state) {}
+ public void onAlive(InetAddressAndPort endpoint, EndpointState state) {}
- public void onDead(InetAddress endpoint, EndpointState state) {}
+ public void onDead(InetAddressAndPort endpoint, EndpointState state) {}
- public void onRestart(InetAddress endpoint, EndpointState state) {}
+ public void onRestart(InetAddressAndPort endpoint, EndpointState state) {}
- public void onRemove(InetAddress endpoint)
+ public void onRemove(InetAddressAndPort endpoint)
{
loadInfo.remove(endpoint);
}
- public Map<InetAddress, Double> getLoadInfo()
+ public Map<InetAddressAndPort, Double> getLoadInfo()
{
return Collections.unmodifiableMap(loadInfo);
}
diff --git a/src/java/org/apache/cassandra/service/MigrationListener.java b/src/java/org/apache/cassandra/service/MigrationListener.java
deleted file mode 100644
index 19d2592..0000000
--- a/src/java/org/apache/cassandra/service/MigrationListener.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.service;
-
-import java.util.List;
-
-import org.apache.cassandra.db.marshal.AbstractType;
-
-public abstract class MigrationListener
-{
- public void onCreateKeyspace(String ksName)
- {
- }
-
- public void onCreateColumnFamily(String ksName, String cfName)
- {
- }
-
- public void onCreateView(String ksName, String viewName)
- {
- onCreateColumnFamily(ksName, viewName);
- }
-
- public void onCreateUserType(String ksName, String typeName)
- {
- }
-
- public void onCreateFunction(String ksName, String functionName, List<AbstractType<?>> argTypes)
- {
- }
-
- public void onCreateAggregate(String ksName, String aggregateName, List<AbstractType<?>> argTypes)
- {
- }
-
- public void onUpdateKeyspace(String ksName)
- {
- }
-
- // the boolean flag indicates whether the change that triggered this event may have a substantive
- // impact on statements using the column family.
- public void onUpdateColumnFamily(String ksName, String cfName, boolean affectsStatements)
- {
- }
-
- public void onUpdateView(String ksName, String viewName, boolean columnsDidChange)
- {
- onUpdateColumnFamily(ksName, viewName, columnsDidChange);
- }
-
- public void onUpdateUserType(String ksName, String typeName)
- {
- }
-
- public void onUpdateFunction(String ksName, String functionName, List<AbstractType<?>> argTypes)
- {
- }
-
- public void onUpdateAggregate(String ksName, String aggregateName, List<AbstractType<?>> argTypes)
- {
- }
-
- public void onDropKeyspace(String ksName)
- {
- }
-
- public void onDropColumnFamily(String ksName, String cfName)
- {
- }
-
- public void onDropView(String ksName, String viewName)
- {
- onDropColumnFamily(ksName, viewName);
- }
-
- public void onDropUserType(String ksName, String typeName)
- {
- }
-
- public void onDropFunction(String ksName, String functionName, List<AbstractType<?>> argTypes)
- {
- }
-
- public void onDropAggregate(String ksName, String aggregateName, List<AbstractType<?>> argTypes)
- {
- }
-}
diff --git a/src/java/org/apache/cassandra/service/MigrationManager.java b/src/java/org/apache/cassandra/service/MigrationManager.java
deleted file mode 100644
index 019bc67..0000000
--- a/src/java/org/apache/cassandra/service/MigrationManager.java
+++ /dev/null
@@ -1,729 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.service;
-
-import java.io.IOException;
-import java.net.InetAddress;
-import java.util.*;
-import java.util.concurrent.*;
-import java.lang.management.ManagementFactory;
-import java.lang.management.RuntimeMXBean;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.cassandra.concurrent.ScheduledExecutors;
-import org.apache.cassandra.concurrent.Stage;
-import org.apache.cassandra.concurrent.StageManager;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.Schema;
-import org.apache.cassandra.config.ViewDefinition;
-import org.apache.cassandra.cql3.functions.UDAggregate;
-import org.apache.cassandra.cql3.functions.UDFunction;
-import org.apache.cassandra.db.*;
-import org.apache.cassandra.db.marshal.UserType;
-import org.apache.cassandra.exceptions.AlreadyExistsException;
-import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.gms.*;
-import org.apache.cassandra.io.IVersionedSerializer;
-import org.apache.cassandra.io.util.DataInputPlus;
-import org.apache.cassandra.io.util.DataOutputPlus;
-import org.apache.cassandra.net.MessageOut;
-import org.apache.cassandra.net.MessagingService;
-import org.apache.cassandra.schema.KeyspaceMetadata;
-import org.apache.cassandra.schema.SchemaKeyspace;
-import org.apache.cassandra.schema.Tables;
-import org.apache.cassandra.utils.FBUtilities;
-import org.apache.cassandra.utils.WrappedRunnable;
-
-public class MigrationManager
-{
- private static final Logger logger = LoggerFactory.getLogger(MigrationManager.class);
-
- public static final MigrationManager instance = new MigrationManager();
-
- private static final RuntimeMXBean runtimeMXBean = ManagementFactory.getRuntimeMXBean();
-
- public static final int MIGRATION_DELAY_IN_MS = 60000;
-
- private static final int MIGRATION_TASK_WAIT_IN_SECONDS = Integer.parseInt(System.getProperty("cassandra.migration_task_wait_in_seconds", "1"));
-
- private final List<MigrationListener> listeners = new CopyOnWriteArrayList<>();
-
- private MigrationManager() {}
-
- public void register(MigrationListener listener)
- {
- listeners.add(listener);
- }
-
- public void unregister(MigrationListener listener)
- {
- listeners.remove(listener);
- }
-
- public static void scheduleSchemaPull(InetAddress endpoint, EndpointState state)
- {
- UUID schemaVersion = state.getSchemaVersion();
- if (!endpoint.equals(FBUtilities.getBroadcastAddress()) && schemaVersion != null)
- maybeScheduleSchemaPull(schemaVersion, endpoint, state.getApplicationState(ApplicationState.RELEASE_VERSION).value);
- }
-
- /**
- * If versions differ this node sends request with local migration list to the endpoint
- * and expecting to receive a list of migrations to apply locally.
- */
- private static void maybeScheduleSchemaPull(final UUID theirVersion, final InetAddress endpoint, String releaseVersion)
- {
- String ourMajorVersion = FBUtilities.getReleaseVersionMajor();
- if (!releaseVersion.startsWith(ourMajorVersion))
- {
- logger.debug("Not pulling schema because release version in Gossip is not major version {}, it is {}", ourMajorVersion, releaseVersion);
- return;
- }
- if (Schema.instance.getVersion() == null)
- {
- logger.debug("Not pulling schema from {}, because local schama version is not known yet",
- endpoint);
- return;
- }
- if (Schema.instance.isSameVersion(theirVersion))
- {
- logger.debug("Not pulling schema from {}, because schema versions match: " +
- "local/real={}, local/compatible={}, remote={}",
- endpoint,
- Schema.schemaVersionToString(Schema.instance.getRealVersion()),
- Schema.schemaVersionToString(Schema.instance.getAltVersion()),
- Schema.schemaVersionToString(theirVersion));
- return;
- }
- if (!shouldPullSchemaFrom(endpoint))
- {
- logger.debug("Not pulling schema because versions match or shouldPullSchemaFrom returned false");
- return;
- }
-
- if (Schema.instance.isEmpty() || runtimeMXBean.getUptime() < MIGRATION_DELAY_IN_MS)
- {
- // If we think we may be bootstrapping or have recently started, submit MigrationTask immediately
- logger.debug("Immediately submitting migration task for {}, " +
- "schema versions: local/real={}, local/compatible={}, remote={}",
- endpoint,
- Schema.schemaVersionToString(Schema.instance.getRealVersion()),
- Schema.schemaVersionToString(Schema.instance.getAltVersion()),
- Schema.schemaVersionToString(theirVersion));
- submitMigrationTask(endpoint);
- }
- else
- {
- // Include a delay to make sure we have a chance to apply any changes being
- // pushed out simultaneously. See CASSANDRA-5025
- Runnable runnable = () ->
- {
- // grab the latest version of the schema since it may have changed again since the initial scheduling
- UUID epSchemaVersion = Gossiper.instance.getSchemaVersion(endpoint);
- if (epSchemaVersion == null)
- {
- logger.debug("epState vanished for {}, not submitting migration task", endpoint);
- return;
- }
- if (Schema.instance.isSameVersion(epSchemaVersion))
- {
- logger.debug("Not submitting migration task for {} because our versions match ({})", endpoint, epSchemaVersion);
- return;
- }
- logger.debug("submitting migration task for {}, schema version mismatch: local/real={}, local/compatible={}, remote={}",
- endpoint,
- Schema.schemaVersionToString(Schema.instance.getRealVersion()),
- Schema.schemaVersionToString(Schema.instance.getAltVersion()),
- Schema.schemaVersionToString(epSchemaVersion));
- submitMigrationTask(endpoint);
- };
- ScheduledExecutors.nonPeriodicTasks.schedule(runnable, MIGRATION_DELAY_IN_MS, TimeUnit.MILLISECONDS);
- }
- }
-
- private static Future<?> submitMigrationTask(InetAddress endpoint)
- {
- /*
- * Do not de-ref the future because that causes distributed deadlock (CASSANDRA-3832) because we are
- * running in the gossip stage.
- */
- return StageManager.getStage(Stage.MIGRATION).submit(new MigrationTask(endpoint));
- }
-
- public static boolean shouldPullSchemaFrom(InetAddress endpoint)
- {
- /*
- * Don't request schema from nodes with a differnt or unknonw major version (may have incompatible schema)
- * Don't request schema from fat clients
- */
- return MessagingService.instance().knowsVersion(endpoint)
- && is30Compatible(MessagingService.instance().getRawVersion(endpoint))
- && !Gossiper.instance.isGossipOnlyMember(endpoint);
- }
-
- // Since 3.0.14 protocol contains only a CASSANDRA-13004 bugfix, it is safe to accept schema changes
- // from both 3.0 and 3.0.14.
- private static boolean is30Compatible(int version)
- {
- return version == MessagingService.current_version || version == MessagingService.VERSION_3014;
- }
-
- public static boolean isReadyForBootstrap()
- {
- return MigrationTask.getInflightTasks().isEmpty();
- }
-
- public static void waitUntilReadyForBootstrap()
- {
- CountDownLatch completionLatch;
- while ((completionLatch = MigrationTask.getInflightTasks().poll()) != null)
- {
- try
- {
- if (!completionLatch.await(MIGRATION_TASK_WAIT_IN_SECONDS, TimeUnit.SECONDS))
- logger.error("Migration task failed to complete");
- }
- catch (InterruptedException e)
- {
- Thread.currentThread().interrupt();
- logger.error("Migration task was interrupted");
- }
- }
- }
-
- public void notifyCreateKeyspace(KeyspaceMetadata ksm)
- {
- for (MigrationListener listener : listeners)
- listener.onCreateKeyspace(ksm.name);
- }
-
- public void notifyCreateColumnFamily(CFMetaData cfm)
- {
- for (MigrationListener listener : listeners)
- listener.onCreateColumnFamily(cfm.ksName, cfm.cfName);
- }
-
- public void notifyCreateView(ViewDefinition view)
- {
- for (MigrationListener listener : listeners)
- listener.onCreateView(view.ksName, view.viewName);
- }
-
- public void notifyCreateUserType(UserType ut)
- {
- for (MigrationListener listener : listeners)
- listener.onCreateUserType(ut.keyspace, ut.getNameAsString());
- }
-
- public void notifyCreateFunction(UDFunction udf)
- {
- for (MigrationListener listener : listeners)
- listener.onCreateFunction(udf.name().keyspace, udf.name().name, udf.argTypes());
- }
-
- public void notifyCreateAggregate(UDAggregate udf)
- {
- for (MigrationListener listener : listeners)
- listener.onCreateAggregate(udf.name().keyspace, udf.name().name, udf.argTypes());
- }
-
- public void notifyUpdateKeyspace(KeyspaceMetadata ksm)
- {
- for (MigrationListener listener : listeners)
- listener.onUpdateKeyspace(ksm.name);
- }
-
- public void notifyUpdateColumnFamily(CFMetaData cfm, boolean columnsDidChange)
- {
- for (MigrationListener listener : listeners)
- listener.onUpdateColumnFamily(cfm.ksName, cfm.cfName, columnsDidChange);
- }
-
- public void notifyUpdateView(ViewDefinition view, boolean columnsDidChange)
- {
- for (MigrationListener listener : listeners)
- listener.onUpdateView(view.ksName, view.viewName, columnsDidChange);
- }
-
- public void notifyUpdateUserType(UserType ut)
- {
- for (MigrationListener listener : listeners)
- listener.onUpdateUserType(ut.keyspace, ut.getNameAsString());
-
- // FIXME: remove when we get rid of AbstractType in metadata. Doesn't really belong anywhere.
- Schema.instance.getKSMetaData(ut.keyspace).functions.udfs().forEach(f -> f.userTypeUpdated(ut.keyspace, ut.getNameAsString()));
- }
-
- public void notifyUpdateFunction(UDFunction udf)
- {
- for (MigrationListener listener : listeners)
- listener.onUpdateFunction(udf.name().keyspace, udf.name().name, udf.argTypes());
- }
-
- public void notifyUpdateAggregate(UDAggregate udf)
- {
- for (MigrationListener listener : listeners)
- listener.onUpdateAggregate(udf.name().keyspace, udf.name().name, udf.argTypes());
- }
-
- public void notifyDropKeyspace(KeyspaceMetadata ksm)
- {
- for (MigrationListener listener : listeners)
- listener.onDropKeyspace(ksm.name);
- }
-
- public void notifyDropColumnFamily(CFMetaData cfm)
- {
- for (MigrationListener listener : listeners)
- listener.onDropColumnFamily(cfm.ksName, cfm.cfName);
- }
-
- public void notifyDropView(ViewDefinition view)
- {
- for (MigrationListener listener : listeners)
- listener.onDropView(view.ksName, view.viewName);
- }
-
- public void notifyDropUserType(UserType ut)
- {
- for (MigrationListener listener : listeners)
- listener.onDropUserType(ut.keyspace, ut.getNameAsString());
- }
-
- public void notifyDropFunction(UDFunction udf)
- {
- for (MigrationListener listener : listeners)
- listener.onDropFunction(udf.name().keyspace, udf.name().name, udf.argTypes());
- }
-
- public void notifyDropAggregate(UDAggregate udf)
- {
- for (MigrationListener listener : listeners)
- listener.onDropAggregate(udf.name().keyspace, udf.name().name, udf.argTypes());
- }
-
- public static void announceNewKeyspace(KeyspaceMetadata ksm) throws ConfigurationException
- {
- announceNewKeyspace(ksm, false);
- }
-
- public static void announceNewKeyspace(KeyspaceMetadata ksm, boolean announceLocally) throws ConfigurationException
- {
- announceNewKeyspace(ksm, FBUtilities.timestampMicros(), announceLocally);
- }
-
- public static void announceNewKeyspace(KeyspaceMetadata ksm, long timestamp, boolean announceLocally) throws ConfigurationException
- {
- ksm.validate();
-
- if (Schema.instance.getKSMetaData(ksm.name) != null)
- throw new AlreadyExistsException(ksm.name);
-
- logger.info("Create new Keyspace: {}", ksm);
- announce(SchemaKeyspace.makeCreateKeyspaceMutation(ksm, timestamp), announceLocally);
- }
-
- public static void announceNewColumnFamily(CFMetaData cfm) throws ConfigurationException
- {
- announceNewColumnFamily(cfm, false);
- }
-
- public static void announceNewColumnFamily(CFMetaData cfm, boolean announceLocally) throws ConfigurationException
- {
- announceNewColumnFamily(cfm, announceLocally, true);
- }
-
- private static void announceNewColumnFamily(CFMetaData cfm, boolean announceLocally, boolean throwOnDuplicate) throws ConfigurationException
- {
- announceNewColumnFamily(cfm, announceLocally, throwOnDuplicate, FBUtilities.timestampMicros());
- }
-
- private static void announceNewColumnFamily(CFMetaData cfm, boolean announceLocally, boolean throwOnDuplicate, long timestamp) throws ConfigurationException
- {
- cfm.validate();
-
- KeyspaceMetadata ksm = Schema.instance.getKSMetaData(cfm.ksName);
- if (ksm == null)
- throw new ConfigurationException(String.format("Cannot add table '%s' to non existing keyspace '%s'.", cfm.cfName, cfm.ksName));
- // If we have a table or a view which has the same name, we can't add a new one
- else if (throwOnDuplicate && ksm.getTableOrViewNullable(cfm.cfName) != null)
- throw new AlreadyExistsException(cfm.ksName, cfm.cfName);
-
- logger.info("Create new table: {}", cfm);
- announce(SchemaKeyspace.makeCreateTableMutation(ksm, cfm, timestamp), announceLocally);
- }
-
- public static void announceNewView(ViewDefinition view, boolean announceLocally) throws ConfigurationException
- {
- view.metadata.validate();
-
- KeyspaceMetadata ksm = Schema.instance.getKSMetaData(view.ksName);
- if (ksm == null)
- throw new ConfigurationException(String.format("Cannot add table '%s' to non existing keyspace '%s'.", view.viewName, view.ksName));
- else if (ksm.getTableOrViewNullable(view.viewName) != null)
- throw new AlreadyExistsException(view.ksName, view.viewName);
-
- logger.info("Create new view: {}", view);
- announce(SchemaKeyspace.makeCreateViewMutation(ksm, view, FBUtilities.timestampMicros()), announceLocally);
- }
-
- public static void announceNewType(UserType newType, boolean announceLocally)
- {
- KeyspaceMetadata ksm = Schema.instance.getKSMetaData(newType.keyspace);
- announce(SchemaKeyspace.makeCreateTypeMutation(ksm, newType, FBUtilities.timestampMicros()), announceLocally);
- }
-
- public static void announceNewFunction(UDFunction udf, boolean announceLocally)
- {
- logger.info("Create scalar function '{}'", udf.name());
- KeyspaceMetadata ksm = Schema.instance.getKSMetaData(udf.name().keyspace);
- announce(SchemaKeyspace.makeCreateFunctionMutation(ksm, udf, FBUtilities.timestampMicros()), announceLocally);
- }
-
- public static void announceNewAggregate(UDAggregate udf, boolean announceLocally)
- {
- logger.info("Create aggregate function '{}'", udf.name());
- KeyspaceMetadata ksm = Schema.instance.getKSMetaData(udf.name().keyspace);
- announce(SchemaKeyspace.makeCreateAggregateMutation(ksm, udf, FBUtilities.timestampMicros()), announceLocally);
- }
-
- public static void announceKeyspaceUpdate(KeyspaceMetadata ksm) throws ConfigurationException
- {
- announceKeyspaceUpdate(ksm, false);
- }
-
- public static void announceKeyspaceUpdate(KeyspaceMetadata ksm, boolean announceLocally) throws ConfigurationException
- {
- ksm.validate();
-
- KeyspaceMetadata oldKsm = Schema.instance.getKSMetaData(ksm.name);
- if (oldKsm == null)
- throw new ConfigurationException(String.format("Cannot update non existing keyspace '%s'.", ksm.name));
-
- logger.info("Update Keyspace '{}' From {} To {}", ksm.name, oldKsm, ksm);
- announce(SchemaKeyspace.makeCreateKeyspaceMutation(ksm.name, ksm.params, FBUtilities.timestampMicros()), announceLocally);
- }
-
- public static void announceColumnFamilyUpdate(CFMetaData cfm) throws ConfigurationException
- {
- announceColumnFamilyUpdate(cfm, false);
- }
-
- public static void announceColumnFamilyUpdate(CFMetaData cfm, boolean announceLocally) throws ConfigurationException
- {
- announceColumnFamilyUpdate(cfm, null, announceLocally);
- }
-
- public static void announceColumnFamilyUpdate(CFMetaData cfm, Collection<ViewDefinition> views, boolean announceLocally) throws ConfigurationException
- {
- cfm.validate();
-
- CFMetaData oldCfm = Schema.instance.getCFMetaData(cfm.ksName, cfm.cfName);
- if (oldCfm == null)
- throw new ConfigurationException(String.format("Cannot update non existing table '%s' in keyspace '%s'.", cfm.cfName, cfm.ksName));
- KeyspaceMetadata ksm = Schema.instance.getKSMetaData(cfm.ksName);
-
- oldCfm.validateCompatibility(cfm);
-
- long timestamp = FBUtilities.timestampMicros();
-
- logger.info("Update table '{}/{}' From {} To {}", cfm.ksName, cfm.cfName, oldCfm, cfm);
- Mutation.SimpleBuilder builder = SchemaKeyspace.makeUpdateTableMutation(ksm, oldCfm, cfm, timestamp);
-
- if (views != null)
- views.forEach(view -> addViewUpdateToMutationBuilder(view, builder));
-
- announce(builder, announceLocally);
- }
-
- public static void announceViewUpdate(ViewDefinition view, boolean announceLocally) throws ConfigurationException
- {
- KeyspaceMetadata ksm = Schema.instance.getKSMetaData(view.ksName);
- long timestamp = FBUtilities.timestampMicros();
- Mutation.SimpleBuilder builder = SchemaKeyspace.makeCreateKeyspaceMutation(ksm.name, ksm.params, timestamp);
- addViewUpdateToMutationBuilder(view, builder);
- announce(builder, announceLocally);
- }
-
- private static void addViewUpdateToMutationBuilder(ViewDefinition view, Mutation.SimpleBuilder builder)
- {
- view.metadata.validate();
-
- ViewDefinition oldView = Schema.instance.getView(view.ksName, view.viewName);
- if (oldView == null)
- throw new ConfigurationException(String.format("Cannot update non existing materialized view '%s' in keyspace '%s'.", view.viewName, view.ksName));
-
- oldView.metadata.validateCompatibility(view.metadata);
-
- logger.info("Update view '{}/{}' From {} To {}", view.ksName, view.viewName, oldView, view);
- SchemaKeyspace.makeUpdateViewMutation(builder, oldView, view);
- }
-
- public static void announceTypeUpdate(UserType updatedType, boolean announceLocally)
- {
- logger.info("Update type '{}.{}' to {}", updatedType.keyspace, updatedType.getNameAsString(), updatedType);
- announceNewType(updatedType, announceLocally);
- }
-
- public static void announceKeyspaceDrop(String ksName) throws ConfigurationException
- {
- announceKeyspaceDrop(ksName, false);
- }
-
- public static void announceKeyspaceDrop(String ksName, boolean announceLocally) throws ConfigurationException
- {
- KeyspaceMetadata oldKsm = Schema.instance.getKSMetaData(ksName);
- if (oldKsm == null)
- throw new ConfigurationException(String.format("Cannot drop non existing keyspace '%s'.", ksName));
-
- logger.info("Drop Keyspace '{}'", oldKsm.name);
- announce(SchemaKeyspace.makeDropKeyspaceMutation(oldKsm, FBUtilities.timestampMicros()), announceLocally);
- }
-
- public static void announceColumnFamilyDrop(String ksName, String cfName) throws ConfigurationException
- {
- announceColumnFamilyDrop(ksName, cfName, false);
- }
-
- public static void announceColumnFamilyDrop(String ksName, String cfName, boolean announceLocally) throws ConfigurationException
- {
- CFMetaData oldCfm = Schema.instance.getCFMetaData(ksName, cfName);
- if (oldCfm == null)
- throw new ConfigurationException(String.format("Cannot drop non existing table '%s' in keyspace '%s'.", cfName, ksName));
- KeyspaceMetadata ksm = Schema.instance.getKSMetaData(ksName);
-
- logger.info("Drop table '{}/{}'", oldCfm.ksName, oldCfm.cfName);
- announce(SchemaKeyspace.makeDropTableMutation(ksm, oldCfm, FBUtilities.timestampMicros()), announceLocally);
- }
-
- public static void announceViewDrop(String ksName, String viewName, boolean announceLocally) throws ConfigurationException
- {
- ViewDefinition view = Schema.instance.getView(ksName, viewName);
- if (view == null)
- throw new ConfigurationException(String.format("Cannot drop non existing materialized view '%s' in keyspace '%s'.", viewName, ksName));
- KeyspaceMetadata ksm = Schema.instance.getKSMetaData(ksName);
-
- logger.info("Drop table '{}/{}'", view.ksName, view.viewName);
- announce(SchemaKeyspace.makeDropViewMutation(ksm, view, FBUtilities.timestampMicros()), announceLocally);
- }
-
- public static void announceTypeDrop(UserType droppedType)
- {
- announceTypeDrop(droppedType, false);
- }
-
- public static void announceTypeDrop(UserType droppedType, boolean announceLocally)
- {
- KeyspaceMetadata ksm = Schema.instance.getKSMetaData(droppedType.keyspace);
- announce(SchemaKeyspace.dropTypeFromSchemaMutation(ksm, droppedType, FBUtilities.timestampMicros()), announceLocally);
- }
-
- public static void announceFunctionDrop(UDFunction udf, boolean announceLocally)
- {
- logger.info("Drop scalar function overload '{}' args '{}'", udf.name(), udf.argTypes());
- KeyspaceMetadata ksm = Schema.instance.getKSMetaData(udf.name().keyspace);
- announce(SchemaKeyspace.makeDropFunctionMutation(ksm, udf, FBUtilities.timestampMicros()), announceLocally);
- }
-
- public static void announceAggregateDrop(UDAggregate udf, boolean announceLocally)
- {
- logger.info("Drop aggregate function overload '{}' args '{}'", udf.name(), udf.argTypes());
- KeyspaceMetadata ksm = Schema.instance.getKSMetaData(udf.name().keyspace);
- announce(SchemaKeyspace.makeDropAggregateMutation(ksm, udf, FBUtilities.timestampMicros()), announceLocally);
- }
-
- static void announceGlobally(Mutation change)
- {
- announceGlobally(Collections.singletonList(change));
- }
-
- static void announceGlobally(Collection<Mutation> change)
- {
- FBUtilities.waitOnFuture(announce(change));
- }
-
- /**
- * actively announce a new version to active hosts via rpc
- * @param schema The schema mutation to be applied
- */
- private static void announce(Mutation.SimpleBuilder schema, boolean announceLocally)
- {
- List<Mutation> mutations = Collections.singletonList(schema.build());
-
- if (announceLocally)
- SchemaKeyspace.mergeSchema(mutations);
- else
- FBUtilities.waitOnFuture(announce(mutations));
- }
-
- private static void pushSchemaMutation(InetAddress endpoint, Collection<Mutation> schema)
- {
- MessageOut<Collection<Mutation>> msg = new MessageOut<>(MessagingService.Verb.DEFINITIONS_UPDATE,
- schema,
- MigrationsSerializer.instance);
- MessagingService.instance().sendOneWay(msg, endpoint);
- }
-
- // Returns a future on the local application of the schema
- private static Future<?> announce(final Collection<Mutation> schema)
- {
- Future<?> f = StageManager.getStage(Stage.MIGRATION).submit(new WrappedRunnable()
- {
- protected void runMayThrow() throws ConfigurationException
- {
- SchemaKeyspace.mergeSchemaAndAnnounceVersion(schema);
- }
- });
-
- for (InetAddress endpoint : Gossiper.instance.getLiveMembers())
- {
- // only push schema to nodes with known and equal versions
- if (!endpoint.equals(FBUtilities.getBroadcastAddress()) &&
- MessagingService.instance().knowsVersion(endpoint) &&
- is30Compatible(MessagingService.instance().getRawVersion(endpoint)))
- pushSchemaMutation(endpoint, schema);
- }
-
- return f;
- }
-
- /**
- * Announce my version passively over gossip.
- * Used to notify nodes as they arrive in the cluster.
- *
- * @param version The schema version to announce
- * @param compatible flag whether {@code version} is a 3.0 compatible version
- */
- public static void passiveAnnounce(UUID version, boolean compatible)
- {
- Gossiper.instance.addLocalApplicationState(ApplicationState.SCHEMA, StorageService.instance.valueFactory.schema(version));
- logger.debug("Gossiping my {} schema version {}",
- compatible ? "3.0 compatible" : "3.11",
- Schema.schemaVersionToString(version));
- }
-
- /**
- * Clear all locally stored schema information and reset schema to initial state.
- * Called by user (via JMX) who wants to get rid of schema disagreement.
- */
- public static void resetLocalSchema()
- {
- logger.info("Starting local schema reset...");
-
- logger.debug("Truncating schema tables...");
-
- SchemaKeyspace.truncate();
-
- logger.debug("Clearing local schema keyspace definitions...");
-
- Schema.instance.clear();
-
- Set<InetAddress> liveEndpoints = Gossiper.instance.getLiveMembers();
- liveEndpoints.remove(FBUtilities.getBroadcastAddress());
-
- // force migration if there are nodes around
- for (InetAddress node : liveEndpoints)
- {
- if (shouldPullSchemaFrom(node))
- {
- logger.debug("Requesting schema from {}", node);
- FBUtilities.waitOnFuture(submitMigrationTask(node));
- break;
- }
- }
-
- logger.info("Local schema reset is complete.");
- }
-
- /**
- * We have a set of non-local, distributed system keyspaces, e.g. system_traces, system_auth, etc.
- * (see {@link Schema#REPLICATED_SYSTEM_KEYSPACE_NAMES}), that need to be created on cluster initialisation,
- * and later evolved on major upgrades (sometimes minor too). This method compares the current known definitions
- * of the tables (if the keyspace exists) to the expected, most modern ones expected by the running version of C*;
- * if any changes have been detected, a schema Mutation will be created which, when applied, should make
- * cluster's view of that keyspace aligned with the expected modern definition.
- *
- * @param keyspace the expected modern definition of the keyspace
- * @param generation timestamp to use for the table changes in the schema mutation
- *
- * @return empty Optional if the current definition is up to date, or an Optional with the Mutation that would
- * bring the schema in line with the expected definition.
- */
- static Optional<Mutation> evolveSystemKeyspace(KeyspaceMetadata keyspace, long generation)
- {
- Mutation.SimpleBuilder builder = null;
-
- KeyspaceMetadata definedKeyspace = Schema.instance.getKSMetaData(keyspace.name);
- Tables definedTables = null == definedKeyspace ? Tables.none() : definedKeyspace.tables;
-
- for (CFMetaData table : keyspace.tables)
- {
- if (table.equals(definedTables.getNullable(table.cfName)))
- continue;
-
- if (null == builder)
- {
- // for the keyspace definition itself (name, replication, durability) always use generation 0;
- // this ensures that any changes made to replication by the user will never be overwritten.
- builder = SchemaKeyspace.makeCreateKeyspaceMutation(keyspace.name, keyspace.params, 0);
-
- // now set the timestamp to generation, so the tables have the expected timestamp
- builder.timestamp(generation);
- }
-
- // for table definitions always use the provided generation; these tables, unlike their containing
- // keyspaces, are *NOT* meant to be altered by the user; if their definitions need to change,
- // the schema must be updated in code, and the appropriate generation must be bumped.
- SchemaKeyspace.addTableToSchemaMutation(table, true, builder);
- }
-
- return builder == null ? Optional.empty() : Optional.of(builder.build());
- }
-
- public static class MigrationsSerializer implements IVersionedSerializer<Collection<Mutation>>
- {
- public static MigrationsSerializer instance = new MigrationsSerializer();
-
- public void serialize(Collection<Mutation> schema, DataOutputPlus out, int version) throws IOException
- {
- out.writeInt(schema.size());
- for (Mutation mutation : schema)
- Mutation.serializer.serialize(mutation, out, version);
- }
-
- public Collection<Mutation> deserialize(DataInputPlus in, int version) throws IOException
- {
- int count = in.readInt();
- Collection<Mutation> schema = new ArrayList<>(count);
-
- for (int i = 0; i < count; i++)
- schema.add(Mutation.serializer.deserialize(in, version));
-
- return schema;
- }
-
- public long serializedSize(Collection<Mutation> schema, int version)
- {
- int size = TypeSizes.sizeof(schema.size());
- for (Mutation mutation : schema)
- size += Mutation.serializer.serializedSize(mutation, version);
- return size;
- }
- }
-}
diff --git a/src/java/org/apache/cassandra/service/MigrationTask.java b/src/java/org/apache/cassandra/service/MigrationTask.java
deleted file mode 100644
index db65c20..0000000
--- a/src/java/org/apache/cassandra/service/MigrationTask.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.service;
-
-import java.net.InetAddress;
-import java.util.Collection;
-import java.util.EnumSet;
-import java.util.Set;
-import java.util.concurrent.ConcurrentLinkedQueue;
-import java.util.concurrent.CountDownLatch;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.cassandra.db.Mutation;
-import org.apache.cassandra.db.SystemKeyspace;
-import org.apache.cassandra.db.SystemKeyspace.BootstrapState;
-import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.gms.FailureDetector;
-import org.apache.cassandra.net.IAsyncCallback;
-import org.apache.cassandra.net.MessageIn;
-import org.apache.cassandra.net.MessageOut;
-import org.apache.cassandra.net.MessagingService;
-import org.apache.cassandra.schema.SchemaKeyspace;
-import org.apache.cassandra.utils.WrappedRunnable;
-
-
-class MigrationTask extends WrappedRunnable
-{
- private static final Logger logger = LoggerFactory.getLogger(MigrationTask.class);
-
- private static final ConcurrentLinkedQueue<CountDownLatch> inflightTasks = new ConcurrentLinkedQueue<>();
-
- private static final Set<BootstrapState> monitoringBootstrapStates = EnumSet.of(BootstrapState.NEEDS_BOOTSTRAP, BootstrapState.IN_PROGRESS);
-
- private final InetAddress endpoint;
-
- MigrationTask(InetAddress endpoint)
- {
- this.endpoint = endpoint;
- }
-
- public static ConcurrentLinkedQueue<CountDownLatch> getInflightTasks()
- {
- return inflightTasks;
- }
-
- public void runMayThrow() throws Exception
- {
- if (!FailureDetector.instance.isAlive(endpoint))
- {
- logger.warn("Can't send schema pull request: node {} is down.", endpoint);
- return;
- }
-
- // There is a chance that quite some time could have passed between now and the MM#maybeScheduleSchemaPull(),
- // potentially enough for the endpoint node to restart - which is an issue if it does restart upgraded, with
- // a higher major.
- if (!MigrationManager.shouldPullSchemaFrom(endpoint))
- {
- logger.info("Skipped sending a migration request: node {} has a higher major version now.", endpoint);
- return;
- }
-
- MessageOut message = new MessageOut<>(MessagingService.Verb.MIGRATION_REQUEST, null, MigrationManager.MigrationsSerializer.instance);
-
- final CountDownLatch completionLatch = new CountDownLatch(1);
-
- IAsyncCallback<Collection<Mutation>> cb = new IAsyncCallback<Collection<Mutation>>()
- {
- @Override
- public void response(MessageIn<Collection<Mutation>> message)
- {
- try
- {
- SchemaKeyspace.mergeSchemaAndAnnounceVersion(message.payload);
- }
- catch (ConfigurationException e)
- {
- logger.error("Configuration exception merging remote schema", e);
- }
- finally
- {
- completionLatch.countDown();
- }
- }
-
- public boolean isLatencyForSnitch()
- {
- return false;
- }
- };
-
- // Only save the latches if we need bootstrap or are bootstrapping
- if (monitoringBootstrapStates.contains(SystemKeyspace.getBootstrapState()))
- inflightTasks.offer(completionLatch);
-
- MessagingService.instance().sendRR(message, endpoint, cb);
- }
-}
diff --git a/src/java/org/apache/cassandra/service/NativeTransportService.java b/src/java/org/apache/cassandra/service/NativeTransportService.java
index c58bb5e..6cdce3f 100644
--- a/src/java/org/apache/cassandra/service/NativeTransportService.java
+++ b/src/java/org/apache/cassandra/service/NativeTransportService.java
@@ -24,6 +24,7 @@
import java.util.concurrent.TimeUnit;
import com.google.common.annotations.VisibleForTesting;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -31,13 +32,11 @@
import io.netty.channel.epoll.Epoll;
import io.netty.channel.epoll.EpollEventLoopGroup;
import io.netty.channel.nio.NioEventLoopGroup;
-import io.netty.util.concurrent.EventExecutor;
import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.metrics.AuthMetrics;
import org.apache.cassandra.metrics.ClientMetrics;
-import org.apache.cassandra.transport.ConfiguredLimit;
import org.apache.cassandra.transport.Message;
import org.apache.cassandra.transport.Server;
+import org.apache.cassandra.utils.NativeLibrary;
/**
* Handles native transport server lifecycle and associated resources. Lazily initialized.
@@ -51,7 +50,6 @@
private boolean initialized = false;
private EventLoopGroup workerGroup;
- private ConfiguredLimit protocolVersionLimit;
/**
* Creates netty thread pools and event loops.
@@ -73,18 +71,15 @@
logger.info("Netty using Java NIO event loop");
}
- protocolVersionLimit = ConfiguredLimit.newLimit();
-
int nativePort = DatabaseDescriptor.getNativeTransportPort();
int nativePortSSL = DatabaseDescriptor.getNativeTransportPortSSL();
InetAddress nativeAddr = DatabaseDescriptor.getRpcAddress();
org.apache.cassandra.transport.Server.Builder builder = new org.apache.cassandra.transport.Server.Builder()
.withEventLoopGroup(workerGroup)
- .withProtocolVersionLimit(protocolVersionLimit)
.withHost(nativeAddr);
- if (!DatabaseDescriptor.getClientEncryptionOptions().enabled)
+ if (!DatabaseDescriptor.getNativeProtocolEncryptionOptions().isEnabled())
{
servers = Collections.singleton(builder.withSSL(false).withPort(nativePort).build());
}
@@ -107,11 +102,8 @@
}
}
- // register metrics
ClientMetrics.instance.init(servers);
- AuthMetrics.init();
-
initialized = true;
}
@@ -146,26 +138,16 @@
Message.Dispatcher.shutdown();
}
- public int getMaxProtocolVersion()
- {
- return protocolVersionLimit.getMaxVersion().asInt();
- }
-
- public void refreshMaxNegotiableProtocolVersion()
- {
- // lowering the max negotiable protocol version is only safe if we haven't already
- // allowed clients to connect with a higher version. This still allows the max
- // version to be raised, as that is safe.
- if (initialized)
- protocolVersionLimit.updateMaxSupportedVersion();
- }
-
/**
- * @return intend to use epoll bassed event looping
+ * @return intend to use epoll based event looping
*/
public static boolean useEpoll()
{
final boolean enableEpoll = Boolean.parseBoolean(System.getProperty("cassandra.native.epoll.enabled", "true"));
+
+ if (enableEpoll && !Epoll.isAvailable() && NativeLibrary.osType == NativeLibrary.OSType.LINUX)
+ logger.warn("epoll not available", Epoll.unavailabilityCause());
+
return enableEpoll && Epoll.isAvailable();
}
@@ -190,4 +172,10 @@
{
return servers;
}
+
+ public void clearConnectionHistory()
+ {
+ for (Server server : servers)
+ server.clearConnectionHistory();
+ }
}
diff --git a/src/java/org/apache/cassandra/service/PendingRangeCalculatorService.java b/src/java/org/apache/cassandra/service/PendingRangeCalculatorService.java
index 1334611..1c6b183 100644
--- a/src/java/org/apache/cassandra/service/PendingRangeCalculatorService.java
+++ b/src/java/org/apache/cassandra/service/PendingRangeCalculatorService.java
@@ -20,7 +20,7 @@
import org.apache.cassandra.concurrent.JMXEnabledThreadPoolExecutor;
import org.apache.cassandra.concurrent.NamedThreadFactory;
-import org.apache.cassandra.config.Schema;
+import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.locator.AbstractReplicationStrategy;
import org.apache.cassandra.utils.ExecutorUtils;
@@ -28,16 +28,12 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.Arrays;
import java.util.List;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicInteger;
import com.google.common.annotations.VisibleForTesting;
-import static org.apache.cassandra.utils.ExecutorUtils.awaitTermination;
-import static org.apache.cassandra.utils.ExecutorUtils.shutdownNow;
-
public class PendingRangeCalculatorService
{
public static final PendingRangeCalculatorService instance = new PendingRangeCalculatorService();
@@ -53,28 +49,35 @@
public PendingRangeCalculatorService()
{
- executor.setRejectedExecutionHandler(new RejectedExecutionHandler()
- {
- public void rejectedExecution(Runnable r, ThreadPoolExecutor e)
+ executor.setRejectedExecutionHandler((r, e) ->
{
+ PendingRangeCalculatorServiceDiagnostics.taskRejected(instance, updateJobs);
PendingRangeCalculatorService.instance.finishUpdate();
}
- }
);
}
private static class PendingRangeTask implements Runnable
{
+ private final AtomicInteger updateJobs;
+
+ PendingRangeTask(AtomicInteger updateJobs)
+ {
+ this.updateJobs = updateJobs;
+ }
+
public void run()
{
try
{
+ PendingRangeCalculatorServiceDiagnostics.taskStarted(instance, updateJobs);
long start = System.currentTimeMillis();
List<String> keyspaces = Schema.instance.getNonLocalStrategyKeyspaces();
for (String keyspaceName : keyspaces)
calculatePendingRanges(Keyspace.open(keyspaceName).getReplicationStrategy(), keyspaceName);
if (logger.isTraceEnabled())
logger.trace("Finished PendingRangeTask for {} keyspaces in {}ms", keyspaces.size(), System.currentTimeMillis() - start);
+ PendingRangeCalculatorServiceDiagnostics.taskFinished(instance, updateJobs);
}
finally
{
@@ -85,13 +88,15 @@
private void finishUpdate()
{
- updateJobs.decrementAndGet();
+ int jobs = updateJobs.decrementAndGet();
+ PendingRangeCalculatorServiceDiagnostics.taskCountChanged(instance, jobs);
}
public void update()
{
- updateJobs.incrementAndGet();
- executor.submit(new PendingRangeTask());
+ int jobs = updateJobs.incrementAndGet();
+ PendingRangeCalculatorServiceDiagnostics.taskCountChanged(instance, jobs);
+ executor.execute(new PendingRangeTask(updateJobs));
}
public void blockUntilFinished()
@@ -118,7 +123,7 @@
}
@VisibleForTesting
- public void shutdownExecutor(long timeout, TimeUnit unit) throws InterruptedException, TimeoutException
+ public void shutdownAndWait(long timeout, TimeUnit unit) throws InterruptedException, TimeoutException
{
ExecutorUtils.shutdownNowAndWait(timeout, unit, executor);
}
diff --git a/src/java/org/apache/cassandra/service/PendingRangeCalculatorServiceDiagnostics.java b/src/java/org/apache/cassandra/service/PendingRangeCalculatorServiceDiagnostics.java
new file mode 100644
index 0000000..ec09e3f
--- /dev/null
+++ b/src/java/org/apache/cassandra/service/PendingRangeCalculatorServiceDiagnostics.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service;
+
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.cassandra.diag.DiagnosticEventService;
+import org.apache.cassandra.service.PendingRangeCalculatorServiceEvent.PendingRangeCalculatorServiceEventType;
+
+/**
+ * Utility methods for diagnostic events related to {@link PendingRangeCalculatorService}.
+ */
+final class PendingRangeCalculatorServiceDiagnostics
+{
+ private static final DiagnosticEventService service = DiagnosticEventService.instance();
+
+ private PendingRangeCalculatorServiceDiagnostics()
+ {
+ }
+
+ static void taskStarted(PendingRangeCalculatorService calculatorService, AtomicInteger taskCount)
+ {
+ if (isEnabled(PendingRangeCalculatorServiceEventType.TASK_STARTED))
+ service.publish(new PendingRangeCalculatorServiceEvent(PendingRangeCalculatorServiceEventType.TASK_STARTED,
+ calculatorService,
+ taskCount.get()));
+ }
+
+ static void taskFinished(PendingRangeCalculatorService calculatorService, AtomicInteger taskCount)
+ {
+ if (isEnabled(PendingRangeCalculatorServiceEventType.TASK_FINISHED_SUCCESSFULLY))
+ service.publish(new PendingRangeCalculatorServiceEvent(PendingRangeCalculatorServiceEventType.TASK_FINISHED_SUCCESSFULLY,
+ calculatorService,
+ taskCount.get()));
+ }
+
+ static void taskRejected(PendingRangeCalculatorService calculatorService, AtomicInteger taskCount)
+ {
+ if (isEnabled(PendingRangeCalculatorServiceEventType.TASK_EXECUTION_REJECTED))
+ service.publish(new PendingRangeCalculatorServiceEvent(PendingRangeCalculatorServiceEventType.TASK_EXECUTION_REJECTED,
+ calculatorService,
+ taskCount.get()));
+ }
+
+ static void taskCountChanged(PendingRangeCalculatorService calculatorService, int taskCount)
+ {
+ if (isEnabled(PendingRangeCalculatorServiceEventType.TASK_COUNT_CHANGED))
+ service.publish(new PendingRangeCalculatorServiceEvent(PendingRangeCalculatorServiceEventType.TASK_COUNT_CHANGED,
+ calculatorService,
+ taskCount));
+ }
+
+ private static boolean isEnabled(PendingRangeCalculatorServiceEventType type)
+ {
+ return service.isEnabled(PendingRangeCalculatorServiceEvent.class, type);
+ }
+}
diff --git a/src/java/org/apache/cassandra/service/PendingRangeCalculatorServiceEvent.java b/src/java/org/apache/cassandra/service/PendingRangeCalculatorServiceEvent.java
new file mode 100644
index 0000000..3024149
--- /dev/null
+++ b/src/java/org/apache/cassandra/service/PendingRangeCalculatorServiceEvent.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service;
+
+import java.io.Serializable;
+import java.util.HashMap;
+
+import org.apache.cassandra.diag.DiagnosticEvent;
+
+/**
+ * Events related to {@link PendingRangeCalculatorService}.
+ */
+final class PendingRangeCalculatorServiceEvent extends DiagnosticEvent
+{
+ private final PendingRangeCalculatorServiceEventType type;
+ private final PendingRangeCalculatorService source;
+ private final int taskCount;
+
+ public enum PendingRangeCalculatorServiceEventType
+ {
+ TASK_STARTED,
+ TASK_FINISHED_SUCCESSFULLY,
+ TASK_EXECUTION_REJECTED,
+ TASK_COUNT_CHANGED
+ }
+
+ PendingRangeCalculatorServiceEvent(PendingRangeCalculatorServiceEventType type,
+ PendingRangeCalculatorService service,
+ int taskCount)
+ {
+ this.type = type;
+ this.source = service;
+ this.taskCount = taskCount;
+ }
+
+ public int getTaskCount()
+ {
+ return taskCount;
+ }
+
+ public PendingRangeCalculatorServiceEventType getType()
+ {
+ return type;
+ }
+
+ public HashMap<String, Serializable> toMap()
+ {
+ // be extra defensive against nulls and bugs
+ HashMap<String, Serializable> ret = new HashMap<>();
+ ret.put("taskCount", taskCount);
+ return ret;
+ }
+}
diff --git a/src/java/org/apache/cassandra/service/QueryState.java b/src/java/org/apache/cassandra/service/QueryState.java
index f0ae3b2..adb13b5 100644
--- a/src/java/org/apache/cassandra/service/QueryState.java
+++ b/src/java/org/apache/cassandra/service/QueryState.java
@@ -18,26 +18,35 @@
package org.apache.cassandra.service;
import java.net.InetAddress;
-import java.nio.ByteBuffer;
-import java.util.Map;
-import java.util.UUID;
-import java.util.concurrent.ThreadLocalRandom;
-import org.apache.cassandra.tracing.Tracing;
+import org.apache.cassandra.transport.ClientStat;
+import org.apache.cassandra.utils.FBUtilities;
/**
- * Represents the state related to a given query.
+ * Primarily used as a recorder for server-generated timestamps (timestamp, in microseconds, and nowInSeconds - in, well, seconds).
+ *
+ * The goal is to be able to use a single consistent server-generated value for both timestamps across the whole request,
+ * and later be able to inspect QueryState for the generated values - for logging or other purposes.
*/
public class QueryState
{
private final ClientState clientState;
- private volatile UUID preparedTracingSession;
+
+ private long timestamp = Long.MIN_VALUE;
+ private int nowInSeconds = Integer.MIN_VALUE;
public QueryState(ClientState clientState)
{
this.clientState = clientState;
}
+ public QueryState(ClientState clientState, long timestamp, int nowInSeconds)
+ {
+ this(clientState);
+ this.timestamp = timestamp;
+ this.nowInSeconds = nowInSeconds;
+ }
+
/**
* @return a QueryState object for internal C* calls (not limited by any kind of auth).
*/
@@ -46,54 +55,64 @@
return new QueryState(ClientState.forInternalCalls());
}
+ /**
+ * Generate, cache, and record a timestamp value on the server-side.
+ *
+ * Used in reads for all live and expiring cells, and all kinds of deletion infos.
+ *
+ * Shouldn't be used directly. {@link org.apache.cassandra.cql3.QueryOptions#getTimestamp(QueryState)} should be used
+ * by all consumers.
+ *
+ * @return server-generated, recorded timestamp in seconds
+ */
+ public long getTimestamp()
+ {
+ if (timestamp == Long.MIN_VALUE)
+ timestamp = ClientState.getTimestamp();
+ return timestamp;
+ }
+
+ /**
+ * Generate, cache, and record a nowInSeconds value on the server-side.
+ *
+ * In writes is used for calculating localDeletionTime for tombstones and expiring cells and other deletion infos.
+ * In reads used to determine liveness of expiring cells and rows.
+ *
+ * Shouldn't be used directly. {@link org.apache.cassandra.cql3.QueryOptions#getNowInSeconds(QueryState)} should be used
+ * by all consumers.
+ *
+ * @return server-generated, recorded timestamp in seconds
+ */
+ public int getNowInSeconds()
+ {
+ if (nowInSeconds == Integer.MIN_VALUE)
+ nowInSeconds = FBUtilities.nowInSeconds();
+ return nowInSeconds;
+ }
+
+ /**
+ * @return server-generated timestamp value, if one had been requested, or Long.MIN_VALUE otherwise
+ */
+ public long generatedTimestamp()
+ {
+ return timestamp;
+ }
+
+ /**
+ * @return server-generated nowInSeconds value, if one had been requested, or Integer.MIN_VALUE otherwise
+ */
+ public int generatedNowInSeconds()
+ {
+ return nowInSeconds;
+ }
+
public ClientState getClientState()
{
return clientState;
}
- /**
- * This clock guarantees that updates for the same QueryState will be ordered
- * in the sequence seen, even if multiple updates happen in the same millisecond.
- */
- public long getTimestamp()
- {
- return clientState.getTimestamp();
- }
-
- public boolean traceNextQuery()
- {
- if (preparedTracingSession != null)
- {
- return true;
- }
-
- double traceProbability = StorageService.instance.getTraceProbability();
- return traceProbability != 0 && ThreadLocalRandom.current().nextDouble() < traceProbability;
- }
-
- public void prepareTracingSession(UUID sessionId)
- {
- this.preparedTracingSession = sessionId;
- }
-
- public void createTracingSession(Map<String,ByteBuffer> customPayload)
- {
- UUID session = this.preparedTracingSession;
- if (session == null)
- {
- Tracing.instance.newSession(customPayload);
- }
- else
- {
- Tracing.instance.newSession(session, customPayload);
- this.preparedTracingSession = null;
- }
- }
-
public InetAddress getClientAddress()
{
- return clientState.isInternal
- ? null
- : clientState.getRemoteAddress().getAddress();
+ return clientState.getClientAddress();
}
}
diff --git a/src/java/org/apache/cassandra/service/RangeRelocator.java b/src/java/org/apache/cassandra/service/RangeRelocator.java
new file mode 100644
index 0000000..b63c105
--- /dev/null
+++ b/src/java/org/apache/cassandra/service/RangeRelocator.java
@@ -0,0 +1,323 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.Future;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Multimap;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.RangeStreamer;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.gms.FailureDetector;
+import org.apache.cassandra.locator.AbstractReplicationStrategy;
+import org.apache.cassandra.locator.EndpointsByReplica;
+import org.apache.cassandra.locator.EndpointsForRange;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.RangesAtEndpoint;
+import org.apache.cassandra.locator.RangesByEndpoint;
+import org.apache.cassandra.locator.Replica;
+import org.apache.cassandra.locator.TokenMetadata;
+import org.apache.cassandra.streaming.StreamOperation;
+import org.apache.cassandra.streaming.StreamPlan;
+import org.apache.cassandra.streaming.StreamState;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.Pair;
+
+@VisibleForTesting
+public class RangeRelocator
+{
+ private static final Logger logger = LoggerFactory.getLogger(StorageService.class);
+
+ private final StreamPlan streamPlan = new StreamPlan(StreamOperation.RELOCATION);
+ private final InetAddressAndPort localAddress = FBUtilities.getBroadcastAddressAndPort();
+ private final TokenMetadata tokenMetaCloneAllSettled;
+ // clone to avoid concurrent modification in calculateNaturalReplicas
+ private final TokenMetadata tokenMetaClone;
+ private final Collection<Token> tokens;
+ private final List<String> keyspaceNames;
+
+
+ RangeRelocator(Collection<Token> tokens, List<String> keyspaceNames, TokenMetadata tmd)
+ {
+ this.tokens = tokens;
+ this.keyspaceNames = keyspaceNames;
+ this.tokenMetaCloneAllSettled = tmd.cloneAfterAllSettled();
+ // clone to avoid concurrent modification in calculateNaturalReplicas
+ this.tokenMetaClone = tmd.cloneOnlyTokenMap();
+ }
+
+ @VisibleForTesting
+ public RangeRelocator()
+ {
+ this.tokens = null;
+ this.keyspaceNames = null;
+ this.tokenMetaCloneAllSettled = null;
+ this.tokenMetaClone = null;
+ }
+
+ /**
+ * Wrapper that supplies accessors to the real implementations of the various dependencies for this method
+ */
+ private static Multimap<InetAddressAndPort, RangeStreamer.FetchReplica> calculateRangesToFetchWithPreferredEndpoints(RangesAtEndpoint fetchRanges,
+ AbstractReplicationStrategy strategy,
+ String keyspace,
+ TokenMetadata tmdBefore,
+ TokenMetadata tmdAfter)
+ {
+ EndpointsByReplica preferredEndpoints =
+ RangeStreamer.calculateRangesToFetchWithPreferredEndpoints(DatabaseDescriptor.getEndpointSnitch()::sortedByProximity,
+ strategy,
+ fetchRanges,
+ StorageService.useStrictConsistency,
+ tmdBefore,
+ tmdAfter,
+ keyspace,
+ Arrays.asList(new RangeStreamer.FailureDetectorSourceFilter(FailureDetector.instance),
+ new RangeStreamer.ExcludeLocalNodeFilter()));
+ return RangeStreamer.convertPreferredEndpointsToWorkMap(preferredEndpoints);
+ }
+
+ /**
+ * calculating endpoints to stream current ranges to if needed
+ * in some situations node will handle current ranges as part of the new ranges
+ **/
+ public static RangesByEndpoint calculateRangesToStreamWithEndpoints(RangesAtEndpoint streamRanges,
+ AbstractReplicationStrategy strat,
+ TokenMetadata tmdBefore,
+ TokenMetadata tmdAfter)
+ {
+ RangesByEndpoint.Builder endpointRanges = new RangesByEndpoint.Builder();
+ for (Replica toStream : streamRanges)
+ {
+ //If the range we are sending is full only send it to the new full replica
+ //There will also be a new transient replica we need to send the data to, but not
+ //the repaired data
+ EndpointsForRange oldEndpoints = strat.calculateNaturalReplicas(toStream.range().right, tmdBefore);
+ EndpointsForRange newEndpoints = strat.calculateNaturalReplicas(toStream.range().right, tmdAfter);
+ logger.debug("Need to stream {}, current endpoints {}, new endpoints {}", toStream, oldEndpoints, newEndpoints);
+
+ for (Replica newEndpoint : newEndpoints)
+ {
+ Replica oldEndpoint = oldEndpoints.byEndpoint().get(newEndpoint.endpoint());
+
+ // Nothing to do
+ if (newEndpoint.equals(oldEndpoint))
+ continue;
+
+ // Completely new range for this endpoint
+ if (oldEndpoint == null)
+ {
+ if (toStream.isTransient() && newEndpoint.isFull())
+ throw new AssertionError(String.format("Need to stream %s, but only have %s which is transient and not full", newEndpoint, toStream));
+
+ for (Range<Token> intersection : newEndpoint.range().intersectionWith(toStream.range()))
+ {
+ endpointRanges.put(newEndpoint.endpoint(), newEndpoint.decorateSubrange(intersection));
+ }
+ }
+ else
+ {
+ Set<Range<Token>> subsToStream = Collections.singleton(toStream.range());
+
+ //First subtract what we already have
+ if (oldEndpoint.isFull() == newEndpoint.isFull() || oldEndpoint.isFull())
+ subsToStream = toStream.range().subtract(oldEndpoint.range());
+
+ //Now we only stream what is still replicated
+ subsToStream.stream()
+ .flatMap(range -> range.intersectionWith(newEndpoint.range()).stream())
+ .forEach(tokenRange -> endpointRanges.put(newEndpoint.endpoint(), newEndpoint.decorateSubrange(tokenRange)));
+ }
+ }
+ }
+ return endpointRanges.build();
+ }
+
+ public void calculateToFromStreams()
+ {
+ logger.debug("Current tmd: {}, Updated tmd: {}", tokenMetaClone, tokenMetaCloneAllSettled);
+
+ for (String keyspace : keyspaceNames)
+ {
+ // replication strategy of the current keyspace
+ AbstractReplicationStrategy strategy = Keyspace.open(keyspace).getReplicationStrategy();
+
+ logger.info("Calculating ranges to stream and request for keyspace {}", keyspace);
+ //From what I have seen we only ever call this with a single token from StorageService.move(Token)
+ for (Token newToken : tokens)
+ {
+ Collection<Token> currentTokens = tokenMetaClone.getTokens(localAddress);
+ if (currentTokens.size() > 1 || currentTokens.isEmpty())
+ {
+ throw new AssertionError("Unexpected current tokens: " + currentTokens);
+ }
+
+ // calculated parts of the ranges to request/stream from/to nodes in the ring
+ Pair<RangesAtEndpoint, RangesAtEndpoint> streamAndFetchOwnRanges;
+
+ //In the single node token move there is nothing to do and Range subtraction is broken
+ //so it's easier to just identify this case up front.
+ if (tokenMetaClone.getTopology().getDatacenterEndpoints().get(DatabaseDescriptor.getEndpointSnitch().getLocalDatacenter()).size() > 1)
+ {
+ // getting collection of the currently used ranges by this keyspace
+ RangesAtEndpoint currentReplicas = strategy.getAddressReplicas(localAddress);
+
+ // collection of ranges which this node will serve after move to the new token
+ RangesAtEndpoint updatedReplicas = strategy.getPendingAddressRanges(tokenMetaClone, newToken, localAddress);
+
+ streamAndFetchOwnRanges = calculateStreamAndFetchRanges(currentReplicas, updatedReplicas);
+ }
+ else
+ {
+ streamAndFetchOwnRanges = Pair.create(RangesAtEndpoint.empty(localAddress), RangesAtEndpoint.empty(localAddress));
+ }
+
+ RangesByEndpoint rangesToStream = calculateRangesToStreamWithEndpoints(streamAndFetchOwnRanges.left, strategy, tokenMetaClone, tokenMetaCloneAllSettled);
+ logger.info("Endpoint ranges to stream to " + rangesToStream);
+
+ // stream ranges
+ for (InetAddressAndPort address : rangesToStream.keySet())
+ {
+ logger.debug("Will stream range {} of keyspace {} to endpoint {}", rangesToStream.get(address), keyspace, address);
+ RangesAtEndpoint ranges = rangesToStream.get(address);
+ streamPlan.transferRanges(address, keyspace, ranges);
+ }
+
+ Multimap<InetAddressAndPort, RangeStreamer.FetchReplica> rangesToFetch = calculateRangesToFetchWithPreferredEndpoints(streamAndFetchOwnRanges.right, strategy, keyspace, tokenMetaClone, tokenMetaCloneAllSettled);
+
+ // stream requests
+ rangesToFetch.asMap().forEach((address, sourceAndOurReplicas) -> {
+ RangesAtEndpoint full = sourceAndOurReplicas.stream()
+ .filter(pair -> pair.remote.isFull())
+ .map(pair -> pair.local)
+ .collect(RangesAtEndpoint.collector(localAddress));
+ RangesAtEndpoint trans = sourceAndOurReplicas.stream()
+ .filter(pair -> pair.remote.isTransient())
+ .map(pair -> pair.local)
+ .collect(RangesAtEndpoint.collector(localAddress));
+ logger.debug("Will request range {} of keyspace {} from endpoint {}", rangesToFetch.get(address), keyspace, address);
+ streamPlan.requestRanges(address, keyspace, full, trans);
+ });
+
+ logger.debug("Keyspace {}: work map {}.", keyspace, rangesToFetch);
+ }
+ }
+ }
+
+ /**
+ * Calculate pair of ranges to stream/fetch for given two range collections
+ * (current ranges for keyspace and ranges after move to new token)
+ *
+ * With transient replication the added wrinkle is that if a range transitions from full to transient then
+ * we need to stream the range despite the fact that we are retaining it as transient. Some replica
+ * somewhere needs to transition from transient to full and we will be the source.
+ *
+ * If the range is transient and is transitioning to full then always fetch even if the range was already transient
+ * since a transiently replicated obviously needs to fetch data to become full.
+ *
+ * This why there is a continue after checking for instersection because intersection is not sufficient reason
+ * to do the subtraction since we might need to stream/fetch data anyways.
+ *
+ * @param currentRanges collection of the ranges by current token
+ * @param updatedRanges collection of the ranges after token is changed
+ * @return pair of ranges to stream/fetch for given current and updated range collections
+ */
+ public static Pair<RangesAtEndpoint, RangesAtEndpoint> calculateStreamAndFetchRanges(RangesAtEndpoint currentRanges, RangesAtEndpoint updatedRanges)
+ {
+ RangesAtEndpoint.Builder toStream = RangesAtEndpoint.builder(currentRanges.endpoint());
+ RangesAtEndpoint.Builder toFetch = RangesAtEndpoint.builder(currentRanges.endpoint());
+ logger.debug("Calculating toStream");
+ computeRanges(currentRanges, updatedRanges, toStream);
+
+ logger.debug("Calculating toFetch");
+ computeRanges(updatedRanges, currentRanges, toFetch);
+
+ logger.debug("To stream {}", toStream);
+ logger.debug("To fetch {}", toFetch);
+ return Pair.create(toStream.build(), toFetch.build());
+ }
+
+ private static void computeRanges(RangesAtEndpoint srcRanges, RangesAtEndpoint dstRanges, RangesAtEndpoint.Builder ranges)
+ {
+ for (Replica src : srcRanges)
+ {
+ boolean intersect = false;
+ RangesAtEndpoint remainder = null;
+ for (Replica dst : dstRanges)
+ {
+ logger.debug("Comparing {} and {}", src, dst);
+ // Stream the full range if there's no intersection
+ if (!src.intersectsOnRange(dst))
+ continue;
+
+ // If we're transitioning from full to transient
+ if (src.isFull() && dst.isTransient())
+ continue;
+
+ if (remainder == null)
+ {
+ remainder = src.subtractIgnoreTransientStatus(dst.range());
+ }
+ else
+ {
+ // Re-subtract ranges to avoid overstreaming in cases when the single range is split or merged
+ RangesAtEndpoint.Builder newRemainder = new RangesAtEndpoint.Builder(remainder.endpoint());
+ for (Replica replica : remainder)
+ newRemainder.addAll(replica.subtractIgnoreTransientStatus(dst.range()));
+ remainder = newRemainder.build();
+ }
+ intersect = true;
+ }
+
+ if (!intersect)
+ {
+ assert remainder == null;
+ logger.debug(" Doesn't intersect adding {}", src);
+ ranges.add(src); // should stream whole old range
+ }
+ else
+ {
+ ranges.addAll(remainder);
+ logger.debug(" Intersects adding {}", remainder);
+ }
+ }
+ }
+
+ public Future<StreamState> stream()
+ {
+ return streamPlan.execute();
+ }
+
+ public boolean streamsNeeded()
+ {
+ return !streamPlan.isEmpty();
+ }
+}
diff --git a/src/java/org/apache/cassandra/service/ReadCallback.java b/src/java/org/apache/cassandra/service/ReadCallback.java
deleted file mode 100644
index b312852..0000000
--- a/src/java/org/apache/cassandra/service/ReadCallback.java
+++ /dev/null
@@ -1,272 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.service;
-
-import java.net.InetAddress;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
-
-import org.apache.commons.lang3.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.cassandra.concurrent.Stage;
-import org.apache.cassandra.concurrent.StageManager;
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.db.*;
-import org.apache.cassandra.db.partitions.PartitionIterator;
-import org.apache.cassandra.exceptions.RequestFailureReason;
-import org.apache.cassandra.db.transform.DuplicateRowChecker;
-import org.apache.cassandra.exceptions.ReadFailureException;
-import org.apache.cassandra.exceptions.ReadTimeoutException;
-import org.apache.cassandra.exceptions.UnavailableException;
-import org.apache.cassandra.metrics.ReadRepairMetrics;
-import org.apache.cassandra.net.IAsyncCallbackWithFailure;
-import org.apache.cassandra.net.MessageIn;
-import org.apache.cassandra.net.MessageOut;
-import org.apache.cassandra.net.MessagingService;
-import org.apache.cassandra.db.ConsistencyLevel;
-import org.apache.cassandra.tracing.TraceState;
-import org.apache.cassandra.tracing.Tracing;
-import org.apache.cassandra.utils.FBUtilities;
-import org.apache.cassandra.utils.concurrent.SimpleCondition;
-
-public class ReadCallback implements IAsyncCallbackWithFailure<ReadResponse>
-{
- protected static final Logger logger = LoggerFactory.getLogger( ReadCallback.class );
-
- public final ResponseResolver resolver;
- private final SimpleCondition condition = new SimpleCondition();
- private final long queryStartNanoTime;
- final int blockfor;
- final List<InetAddress> endpoints;
- private final ReadCommand command;
- private final ConsistencyLevel consistencyLevel;
- private static final AtomicIntegerFieldUpdater<ReadCallback> recievedUpdater
- = AtomicIntegerFieldUpdater.newUpdater(ReadCallback.class, "received");
- private volatile int received = 0;
- private static final AtomicIntegerFieldUpdater<ReadCallback> failuresUpdater
- = AtomicIntegerFieldUpdater.newUpdater(ReadCallback.class, "failures");
- private volatile int failures = 0;
- private final Map<InetAddress, RequestFailureReason> failureReasonByEndpoint;
-
- private final Keyspace keyspace; // TODO push this into ConsistencyLevel?
-
- /**
- * Constructor when response count has to be calculated and blocked for.
- */
- public ReadCallback(ResponseResolver resolver, ConsistencyLevel consistencyLevel, ReadCommand command, List<InetAddress> filteredEndpoints, long queryStartNanoTime)
- {
- this(resolver,
- consistencyLevel,
- consistencyLevel.blockFor(Keyspace.open(command.metadata().ksName)),
- command,
- Keyspace.open(command.metadata().ksName),
- filteredEndpoints,
- queryStartNanoTime);
- }
-
- public ReadCallback(ResponseResolver resolver, ConsistencyLevel consistencyLevel, int blockfor, ReadCommand command, Keyspace keyspace, List<InetAddress> endpoints, long queryStartNanoTime)
- {
- this.command = command;
- this.keyspace = keyspace;
- this.blockfor = blockfor;
- this.consistencyLevel = consistencyLevel;
- this.resolver = resolver;
- this.queryStartNanoTime = queryStartNanoTime;
- this.endpoints = endpoints;
- this.failureReasonByEndpoint = new ConcurrentHashMap<>();
- // we don't support read repair (or rapid read protection) for range scans yet (CASSANDRA-6897)
- assert !(command instanceof PartitionRangeReadCommand) || blockfor >= endpoints.size();
-
- if (logger.isTraceEnabled())
- logger.trace("Blockfor is {}; setting up requests to {}", blockfor, StringUtils.join(this.endpoints, ","));
- }
-
- public boolean await(long timePastStart, TimeUnit unit)
- {
- long time = unit.toNanos(timePastStart) - (System.nanoTime() - queryStartNanoTime);
- try
- {
- return condition.await(time, TimeUnit.NANOSECONDS);
- }
- catch (InterruptedException ex)
- {
- throw new AssertionError(ex);
- }
- }
-
- public void awaitResults() throws ReadFailureException, ReadTimeoutException
- {
- boolean signaled = await(command.getTimeout(), TimeUnit.MILLISECONDS);
- boolean failed = blockfor + failures > endpoints.size();
- if (signaled && !failed)
- return;
-
- if (Tracing.isTracing())
- {
- String gotData = received > 0 ? (resolver.isDataPresent() ? " (including data)" : " (only digests)") : "";
- Tracing.trace("{}; received {} of {} responses{}", new Object[]{ (failed ? "Failed" : "Timed out"), received, blockfor, gotData });
- }
- else if (logger.isDebugEnabled())
- {
- String gotData = received > 0 ? (resolver.isDataPresent() ? " (including data)" : " (only digests)") : "";
- logger.debug("{}; received {} of {} responses{}", new Object[]{ (failed ? "Failed" : "Timed out"), received, blockfor, gotData });
- }
-
- // Same as for writes, see AbstractWriteResponseHandler
- throw failed
- ? new ReadFailureException(consistencyLevel, received, blockfor, resolver.isDataPresent(), failureReasonByEndpoint)
- : new ReadTimeoutException(consistencyLevel, received, blockfor, resolver.isDataPresent());
- }
-
-
- public PartitionIterator get() throws ReadFailureException, ReadTimeoutException, DigestMismatchException
- {
- awaitResults();
-
- PartitionIterator result = blockfor == 1 ? resolver.getData() : resolver.resolve();
- if (logger.isTraceEnabled())
- logger.trace("Read: {} ms.", TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - queryStartNanoTime));
- return DuplicateRowChecker.duringRead(result, endpoints);
- }
-
- public int blockFor()
- {
- return blockfor;
- }
-
- public void response(MessageIn<ReadResponse> message)
- {
- resolver.preprocess(message);
- int n = waitingFor(message.from)
- ? recievedUpdater.incrementAndGet(this)
- : received;
- if (n >= blockfor && resolver.isDataPresent())
- {
- condition.signalAll();
- // kick off a background digest comparison if this is a result that (may have) arrived after
- // the original resolve that get() kicks off as soon as the condition is signaled
- if (blockfor < endpoints.size() && n == endpoints.size())
- {
- TraceState traceState = Tracing.instance.get();
- if (traceState != null)
- traceState.trace("Initiating read-repair");
- StageManager.getStage(Stage.READ_REPAIR).execute(new AsyncRepairRunner(traceState, queryStartNanoTime));
- }
- }
- }
-
- /**
- * @return true if the message counts towards the blockfor threshold
- */
- private boolean waitingFor(InetAddress from)
- {
- return consistencyLevel.isDatacenterLocal()
- ? DatabaseDescriptor.getLocalDataCenter().equals(DatabaseDescriptor.getEndpointSnitch().getDatacenter(from))
- : true;
- }
-
- /**
- * @return the current number of received responses
- */
- public int getReceivedCount()
- {
- return received;
- }
-
- public void response(ReadResponse result)
- {
- MessageIn<ReadResponse> message = MessageIn.create(FBUtilities.getBroadcastAddress(),
- result,
- Collections.<String, byte[]>emptyMap(),
- MessagingService.Verb.INTERNAL_RESPONSE,
- MessagingService.current_version);
- response(message);
- }
-
- public void assureSufficientLiveNodes() throws UnavailableException
- {
- consistencyLevel.assureSufficientLiveNodes(keyspace, endpoints);
- }
-
- public boolean isLatencyForSnitch()
- {
- return true;
- }
-
- private class AsyncRepairRunner implements Runnable
- {
- private final TraceState traceState;
- private final long queryStartNanoTime;
-
- public AsyncRepairRunner(TraceState traceState, long queryStartNanoTime)
- {
- this.traceState = traceState;
- this.queryStartNanoTime = queryStartNanoTime;
- }
-
- public void run()
- {
- // If the resolver is a DigestResolver, we need to do a full data read if there is a mismatch.
- // Otherwise, resolve will send the repairs directly if needs be (and in that case we should never
- // get a digest mismatch).
- try
- {
- resolver.compareResponses();
- }
- catch (DigestMismatchException e)
- {
- assert resolver instanceof DigestResolver;
-
- if (traceState != null)
- traceState.trace("Digest mismatch: {}", e.toString());
- if (logger.isDebugEnabled())
- logger.debug("Digest mismatch:", e);
-
- ReadRepairMetrics.repairedBackground.mark();
-
- final DataResolver repairResolver = new DataResolver(keyspace, command, consistencyLevel, endpoints.size(), queryStartNanoTime);
- AsyncRepairCallback repairHandler = new AsyncRepairCallback(repairResolver, endpoints.size());
-
- for (InetAddress endpoint : endpoints)
- {
- MessageOut<ReadCommand> message = command.createMessage(MessagingService.instance().getVersion(endpoint));
- MessagingService.instance().sendRR(message, endpoint, repairHandler);
- }
- }
- }
- }
-
- @Override
- public void onFailure(InetAddress from, RequestFailureReason failureReason)
- {
- int n = waitingFor(from)
- ? failuresUpdater.incrementAndGet(this)
- : failures;
-
- failureReasonByEndpoint.put(from, failureReason);
-
- if (blockfor + n > endpoints.size())
- condition.signalAll();
- }
-}
diff --git a/src/java/org/apache/cassandra/service/ReplicaFilteringProtection.java b/src/java/org/apache/cassandra/service/ReplicaFilteringProtection.java
deleted file mode 100644
index 428c603..0000000
--- a/src/java/org/apache/cassandra/service/ReplicaFilteringProtection.java
+++ /dev/null
@@ -1,468 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.service;
-
-import java.net.InetAddress;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.List;
-import java.util.NavigableSet;
-import java.util.SortedMap;
-import java.util.TreeMap;
-import java.util.stream.Collectors;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.cassandra.concurrent.Stage;
-import org.apache.cassandra.concurrent.StageManager;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.db.Clustering;
-import org.apache.cassandra.db.ColumnFamilyStore;
-import org.apache.cassandra.db.Columns;
-import org.apache.cassandra.db.ConsistencyLevel;
-import org.apache.cassandra.db.DecoratedKey;
-import org.apache.cassandra.db.DeletionTime;
-import org.apache.cassandra.db.Keyspace;
-import org.apache.cassandra.db.PartitionColumns;
-import org.apache.cassandra.db.ReadCommand;
-import org.apache.cassandra.db.SinglePartitionReadCommand;
-import org.apache.cassandra.db.filter.ClusteringIndexFilter;
-import org.apache.cassandra.db.filter.ClusteringIndexNamesFilter;
-import org.apache.cassandra.db.filter.DataLimits;
-import org.apache.cassandra.db.filter.RowFilter;
-import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
-import org.apache.cassandra.db.partitions.UnfilteredPartitionIterators;
-import org.apache.cassandra.db.rows.EncodingStats;
-import org.apache.cassandra.db.rows.RangeTombstoneMarker;
-import org.apache.cassandra.db.rows.Row;
-import org.apache.cassandra.db.rows.Rows;
-import org.apache.cassandra.db.rows.Unfiltered;
-import org.apache.cassandra.db.rows.UnfilteredRowIterator;
-import org.apache.cassandra.db.rows.UnfilteredRowIterators;
-import org.apache.cassandra.exceptions.ReadTimeoutException;
-import org.apache.cassandra.exceptions.UnavailableException;
-import org.apache.cassandra.metrics.TableMetrics;
-import org.apache.cassandra.net.MessagingService;
-import org.apache.cassandra.tracing.Tracing;
-import org.apache.cassandra.utils.btree.BTreeSet;
-
-/**
- * Helper in charge of collecting additional queries to be done on the coordinator to protect against invalid results
- * being included due to replica-side filtering (secondary indexes or {@code ALLOW * FILTERING}).
- * <p>
- * When using replica-side filtering with CL>ONE, a replica can send a stale result satisfying the filter, while updated
- * replicas won't send a corresponding tombstone to discard that result during reconciliation. This helper identifies
- * the rows in a replica response that don't have a corresponding row in other replica responses, and requests them by
- * primary key to the "silent" replicas in a second fetch round.
- * <p>
- * See CASSANDRA-8272 and CASSANDRA-8273 for further details.
- */
-class ReplicaFilteringProtection
-{
- private static final Logger logger = LoggerFactory.getLogger(ReplicaFilteringProtection.class);
-
- private final Keyspace keyspace;
- private final ReadCommand command;
- private final ConsistencyLevel consistency;
- private final long queryStartNanoTime;
- private final InetAddress[] sources;
- private final TableMetrics tableMetrics;
-
- /**
- * Per-source primary keys of the rows that might be outdated so they need to be fetched.
- * For outdated static rows we use an empty builder to signal it has to be queried.
- */
- private final List<SortedMap<DecoratedKey, BTreeSet.Builder<Clustering>>> rowsToFetch;
-
- /**
- * Per-source list of all the partitions seen by the merge listener, to be merged with the extra fetched rows.
- */
- private final List<List<PartitionBuilder>> originalPartitions;
-
- ReplicaFilteringProtection(Keyspace keyspace,
- ReadCommand command,
- ConsistencyLevel consistency,
- long queryStartNanoTime,
- InetAddress[] sources)
- {
- this.keyspace = keyspace;
- this.command = command;
- this.consistency = consistency;
- this.queryStartNanoTime = queryStartNanoTime;
- this.sources = sources;
- this.rowsToFetch = new ArrayList<>(sources.length);
- this.originalPartitions = new ArrayList<>(sources.length);
-
- for (InetAddress ignored : sources)
- {
- rowsToFetch.add(new TreeMap<>());
- originalPartitions.add(new ArrayList<>());
- }
-
- tableMetrics = ColumnFamilyStore.metricsFor(command.metadata().cfId);
- }
-
- private BTreeSet.Builder<Clustering> getOrCreateToFetch(int source, DecoratedKey partitionKey)
- {
- return rowsToFetch.get(source).computeIfAbsent(partitionKey, k -> BTreeSet.builder(command.metadata().comparator));
- }
-
- /**
- * Returns the protected results for the specified replica. These are generated fetching the extra rows and merging
- * them with the cached original filtered results for that replica.
- *
- * @param source the source
- * @return the protected results for the specified replica
- */
- UnfilteredPartitionIterator queryProtectedPartitions(int source)
- {
- UnfilteredPartitionIterator original = makeIterator(originalPartitions.get(source));
- SortedMap<DecoratedKey, BTreeSet.Builder<Clustering>> toFetch = rowsToFetch.get(source);
-
- if (toFetch.isEmpty())
- return original;
-
- // TODO: this would be more efficient if we had multi-key queries internally
- List<UnfilteredPartitionIterator> fetched = toFetch.keySet()
- .stream()
- .map(k -> querySourceOnKey(source, k))
- .collect(Collectors.toList());
-
- return UnfilteredPartitionIterators.merge(Arrays.asList(original, UnfilteredPartitionIterators.concat(fetched)),
- command.nowInSec(), null);
- }
-
- private UnfilteredPartitionIterator querySourceOnKey(int i, DecoratedKey key)
- {
- BTreeSet.Builder<Clustering> builder = rowsToFetch.get(i).get(key);
- assert builder != null; // We're calling this on the result of rowsToFetch.get(i).keySet()
-
- InetAddress source = sources[i];
- NavigableSet<Clustering> clusterings = builder.build();
- tableMetrics.replicaSideFilteringProtectionRequests.mark();
- if (logger.isTraceEnabled())
- logger.trace("Requesting rows {} in partition {} from {} for replica-side filtering protection",
- clusterings, key, source);
- Tracing.trace("Requesting {} rows in partition {} from {} for replica-side filtering protection",
- clusterings.size(), key, source);
-
- // build the read command taking into account that we could be requesting only in the static row
- DataLimits limits = clusterings.isEmpty() ? DataLimits.cqlLimits(1) : DataLimits.NONE;
- ClusteringIndexFilter filter = new ClusteringIndexNamesFilter(clusterings, command.isReversed());
- SinglePartitionReadCommand cmd = SinglePartitionReadCommand.create(command.metadata(),
- command.nowInSec(),
- command.columnFilter(),
- RowFilter.NONE,
- limits,
- key,
- filter);
- try
- {
- return executeReadCommand(cmd, source);
- }
- catch (ReadTimeoutException e)
- {
- int blockFor = consistency.blockFor(keyspace);
- throw new ReadTimeoutException(consistency, blockFor - 1, blockFor, true);
- }
- catch (UnavailableException e)
- {
- int blockFor = consistency.blockFor(keyspace);
- throw new UnavailableException(consistency, blockFor, blockFor - 1);
- }
- }
-
- private UnfilteredPartitionIterator executeReadCommand(ReadCommand cmd, InetAddress source)
- {
- DataResolver resolver = new DataResolver(keyspace, cmd, ConsistencyLevel.ONE, 1, queryStartNanoTime);
- ReadCallback handler = new ReadCallback(resolver, ConsistencyLevel.ONE, cmd, Collections.singletonList(source), queryStartNanoTime);
-
- if (StorageProxy.canDoLocalRequest(source))
- StageManager.getStage(Stage.READ).maybeExecuteImmediately(new StorageProxy.LocalReadRunnable(cmd, handler));
- else
- MessagingService.instance().sendRRWithFailure(cmd.createMessage(MessagingService.current_version), source, handler);
-
- // We don't call handler.get() because we want to preserve tombstones
- handler.awaitResults();
- assert resolver.responses.size() == 1;
- return resolver.responses.get(0).payload.makeIterator(command);
- }
-
- /**
- * Returns a merge listener that skips the merged rows for which any of the replicas doesn't have a version,
- * pessimistically assuming that they are outdated. It is intended to be used during a first merge of per-replica
- * query results to ensure we fetch enough results from the replicas to ensure we don't miss any potentially
- * outdated result.
- * <p>
- * The listener will track both the accepted data and the primary keys of the rows that are considered as outdated.
- * That way, once the query results would have been merged using this listener, further calls to
- * {@link #queryProtectedPartitions(int)} will use the collected data to return a copy of the
- * data originally collected from the specified replica, completed with the potentially outdated rows.
- */
- UnfilteredPartitionIterators.MergeListener mergeController()
- {
- return (partitionKey, versions) -> {
-
- PartitionBuilder[] builders = new PartitionBuilder[sources.length];
-
- for (int i = 0; i < sources.length; i++)
- builders[i] = new PartitionBuilder(partitionKey, columns(versions), stats(versions));
-
- return new UnfilteredRowIterators.MergeListener()
- {
- @Override
- public void onMergedPartitionLevelDeletion(DeletionTime mergedDeletion, DeletionTime[] versions)
- {
- // cache the deletion time versions to be able to regenerate the original row iterator
- for (int i = 0; i < versions.length; i++)
- builders[i].setDeletionTime(versions[i]);
- }
-
- @Override
- public Row onMergedRows(Row merged, Row[] versions)
- {
- // cache the row versions to be able to regenerate the original row iterator
- for (int i = 0; i < versions.length; i++)
- builders[i].addRow(versions[i]);
-
- if (merged.isEmpty())
- return merged;
-
- boolean isPotentiallyOutdated = false;
- boolean isStatic = merged.isStatic();
- for (int i = 0; i < versions.length; i++)
- {
- Row version = versions[i];
- if (version == null || (isStatic && version.isEmpty()))
- {
- isPotentiallyOutdated = true;
- BTreeSet.Builder<Clustering> toFetch = getOrCreateToFetch(i, partitionKey);
- // Note that for static, we shouldn't add the clustering to the clustering set (the
- // ClusteringIndexNamesFilter we'll build from this later does not expect it), but the fact
- // we created a builder in the first place will act as a marker that the static row must be
- // fetched, even if no other rows are added for this partition.
- if (!isStatic)
- toFetch.add(merged.clustering());
- }
- }
-
- // If the row is potentially outdated (because some replica didn't send anything and so it _may_ be
- // an outdated result that is only present because other replica have filtered the up-to-date result
- // out), then we skip the row. In other words, the results of the initial merging of results by this
- // protection assume the worst case scenario where every row that might be outdated actually is.
- // This ensures that during this first phase (collecting additional row to fetch) we are guaranteed
- // to look at enough data to ultimately fulfill the query limit.
- return isPotentiallyOutdated ? null : merged;
- }
-
- @Override
- public void onMergedRangeTombstoneMarkers(RangeTombstoneMarker merged, RangeTombstoneMarker[] versions)
- {
- // cache the marker versions to be able to regenerate the original row iterator
- for (int i = 0; i < versions.length; i++)
- builders[i].addRangeTombstoneMarker(versions[i]);
- }
-
- @Override
- public void close()
- {
- for (int i = 0; i < sources.length; i++)
- originalPartitions.get(i).add(builders[i]);
- }
- };
- };
- }
-
- private static PartitionColumns columns(List<UnfilteredRowIterator> versions)
- {
- Columns statics = Columns.NONE;
- Columns regulars = Columns.NONE;
- for (UnfilteredRowIterator iter : versions)
- {
- if (iter == null)
- continue;
-
- PartitionColumns cols = iter.columns();
- statics = statics.mergeTo(cols.statics);
- regulars = regulars.mergeTo(cols.regulars);
- }
- return new PartitionColumns(statics, regulars);
- }
-
- private static EncodingStats stats(List<UnfilteredRowIterator> iterators)
- {
- EncodingStats stats = EncodingStats.NO_STATS;
- for (UnfilteredRowIterator iter : iterators)
- {
- if (iter == null)
- continue;
-
- stats = stats.mergeWith(iter.stats());
- }
- return stats;
- }
-
- private UnfilteredPartitionIterator makeIterator(List<PartitionBuilder> builders)
- {
- return new UnfilteredPartitionIterator()
- {
- final Iterator<PartitionBuilder> iterator = builders.iterator();
-
- @Override
- public boolean isForThrift()
- {
- return command.isForThrift();
- }
-
- @Override
- public CFMetaData metadata()
- {
- return command.metadata();
- }
-
- @Override
- public void close()
- {
- // nothing to do here
- }
-
- @Override
- public boolean hasNext()
- {
- return iterator.hasNext();
- }
-
- @Override
- public UnfilteredRowIterator next()
- {
- return iterator.next().build();
- }
- };
- }
-
- private class PartitionBuilder
- {
- private final DecoratedKey partitionKey;
- private final PartitionColumns columns;
- private final EncodingStats stats;
- private DeletionTime deletionTime;
- private Row staticRow = Rows.EMPTY_STATIC_ROW;
- private final List<Unfiltered> contents = new ArrayList<>();
-
- private PartitionBuilder(DecoratedKey partitionKey, PartitionColumns columns, EncodingStats stats)
- {
- this.partitionKey = partitionKey;
- this.columns = columns;
- this.stats = stats;
- }
-
- private void setDeletionTime(DeletionTime deletionTime)
- {
- this.deletionTime = deletionTime;
- }
-
- private void addRow(Row row)
- {
- if (row == null)
- return;
-
- if (row.isStatic())
- staticRow = row;
- else
- contents.add(row);
- }
-
- private void addRangeTombstoneMarker(RangeTombstoneMarker marker)
- {
- if (marker != null)
- contents.add(marker);
- }
-
- private UnfilteredRowIterator build()
- {
- return new UnfilteredRowIterator()
- {
- final Iterator<Unfiltered> iterator = contents.iterator();
-
- @Override
- public DeletionTime partitionLevelDeletion()
- {
- return deletionTime;
- }
-
- @Override
- public EncodingStats stats()
- {
- return stats;
- }
-
- @Override
- public CFMetaData metadata()
- {
- return command.metadata();
- }
-
- @Override
- public boolean isReverseOrder()
- {
- return command.isReversed();
- }
-
- @Override
- public PartitionColumns columns()
- {
- return columns;
- }
-
- @Override
- public DecoratedKey partitionKey()
- {
- return partitionKey;
- }
-
- @Override
- public Row staticRow()
- {
- return staticRow;
- }
-
- @Override
- public void close()
- {
- // nothing to do here
- }
-
- @Override
- public boolean hasNext()
- {
- return iterator.hasNext();
- }
-
- @Override
- public Unfiltered next()
- {
- return iterator.next();
- }
- };
- }
- }
-}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/service/ResponseResolver.java b/src/java/org/apache/cassandra/service/ResponseResolver.java
deleted file mode 100644
index 81b18b6..0000000
--- a/src/java/org/apache/cassandra/service/ResponseResolver.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.service;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.cassandra.db.*;
-import org.apache.cassandra.db.partitions.*;
-import org.apache.cassandra.net.MessageIn;
-import org.apache.cassandra.utils.concurrent.Accumulator;
-
-public abstract class ResponseResolver
-{
- protected static final Logger logger = LoggerFactory.getLogger(ResponseResolver.class);
-
- protected final Keyspace keyspace;
- protected final ReadCommand command;
- protected final ConsistencyLevel consistency;
-
- // Accumulator gives us non-blocking thread-safety with optimal algorithmic constraints
- protected final Accumulator<MessageIn<ReadResponse>> responses;
-
- public ResponseResolver(Keyspace keyspace, ReadCommand command, ConsistencyLevel consistency, int maxResponseCount)
- {
- this.keyspace = keyspace;
- this.command = command;
- this.consistency = consistency;
- this.responses = new Accumulator<>(maxResponseCount);
- }
-
- public abstract PartitionIterator getData();
- public abstract PartitionIterator resolve() throws DigestMismatchException;
-
- /**
- * Compares received responses, potentially triggering a digest mismatch (for a digest resolver) and read-repairs
- * (for a data resolver).
- * <p>
- * This is functionally equivalent to calling {@link #resolve()} and consuming the result, but can be slightly more
- * efficient in some case due to the fact that we don't care about the result itself. This is used when doing
- * asynchronous read-repairs.
- *
- * @throws DigestMismatchException if it's a digest resolver and the responses don't match.
- */
- public abstract void compareResponses() throws DigestMismatchException;
-
- public abstract boolean isDataPresent();
-
- public void preprocess(MessageIn<ReadResponse> message)
- {
- responses.add(message);
- }
-
- public Iterable<MessageIn<ReadResponse>> getMessages()
- {
- return responses;
- }
-}
diff --git a/src/java/org/apache/cassandra/service/SnapshotVerbHandler.java b/src/java/org/apache/cassandra/service/SnapshotVerbHandler.java
index 179abeb..1309d6e 100644
--- a/src/java/org/apache/cassandra/service/SnapshotVerbHandler.java
+++ b/src/java/org/apache/cassandra/service/SnapshotVerbHandler.java
@@ -17,22 +17,27 @@
*/
package org.apache.cassandra.service;
+import java.util.concurrent.Executor;
+import java.util.concurrent.Executors;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.SnapshotCommand;
import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.net.IVerbHandler;
-import org.apache.cassandra.net.MessageIn;
-import org.apache.cassandra.net.MessageOut;
+import org.apache.cassandra.net.Message;
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.utils.DiagnosticSnapshotService;
public class SnapshotVerbHandler implements IVerbHandler<SnapshotCommand>
{
+ public static final SnapshotVerbHandler instance = new SnapshotVerbHandler();
private static final Logger logger = LoggerFactory.getLogger(SnapshotVerbHandler.class);
- public void doVerb(MessageIn<SnapshotCommand> message, int id)
+ public void doVerb(Message<SnapshotCommand> message)
{
SnapshotCommand command = message.payload;
if (command.clear_snapshot)
@@ -41,11 +46,14 @@
}
else if (DiagnosticSnapshotService.isDiagnosticSnapshotRequest(command))
{
- DiagnosticSnapshotService.snapshot(command, message.from);
+ DiagnosticSnapshotService.snapshot(command, message.from());
}
else
+ {
Keyspace.open(command.keyspace).getColumnFamilyStore(command.column_family).snapshot(command.snapshot_name);
- logger.debug("Enqueuing response to snapshot request {} to {}", command.snapshot_name, message.from);
- MessagingService.instance().sendReply(new MessageOut(MessagingService.Verb.INTERNAL_RESPONSE), id, message.from);
+ }
+
+ logger.debug("Enqueuing response to snapshot request {} to {}", command.snapshot_name, message.from());
+ MessagingService.instance().send(message.emptyResponse(), message.from());
}
}
diff --git a/src/java/org/apache/cassandra/service/StartupChecks.java b/src/java/org/apache/cassandra/service/StartupChecks.java
index cb10ab4..7754c6f 100644
--- a/src/java/org/apache/cassandra/service/StartupChecks.java
+++ b/src/java/org/apache/cassandra/service/StartupChecks.java
@@ -29,22 +29,23 @@
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Joiner;
+import com.google.common.base.Throwables;
import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.cassandra.config.CFMetaData;
+import net.jpountz.lz4.LZ4Factory;
+import org.apache.cassandra.cql3.QueryProcessor;
+import org.apache.cassandra.cql3.UntypedResultSet;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.config.Config;
import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.config.Schema;
-import org.apache.cassandra.config.SchemaConstants;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.SchemaConstants;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Directories;
import org.apache.cassandra.db.SystemKeyspace;
-import org.apache.cassandra.cql3.QueryProcessor;
-import org.apache.cassandra.cql3.UntypedResultSet;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.exceptions.StartupException;
import org.apache.cassandra.io.sstable.Descriptor;
@@ -84,6 +85,7 @@
// always want the system keyspace check run last, as this actually loads the schema for that
// keyspace. All other checks should not require any schema initialization.
private final List<StartupCheck> DEFAULT_TESTS = ImmutableList.of(checkJemalloc,
+ checkLz4Native,
checkValidLaunchDate,
checkJMXPorts,
checkJMXProperties,
@@ -141,6 +143,17 @@
}
};
+ public static final StartupCheck checkLz4Native = () -> {
+ try
+ {
+ LZ4Factory.nativeInstance(); // make sure native loads
+ }
+ catch (AssertionError | LinkageError e)
+ {
+ logger.warn("lz4-java was unable to load native libraries; this will lower the performance of lz4 (network/sstables/etc.): {}", Throwables.getRootCause(e).getMessage());
+ }
+ };
+
public static final StartupCheck checkValidLaunchDate = new StartupCheck()
{
/**
@@ -355,14 +368,15 @@
FileVisitor<Path> sstableVisitor = new SimpleFileVisitor<Path>()
{
- public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
+ public FileVisitResult visitFile(Path path, BasicFileAttributes attrs)
{
- if (!Descriptor.isValidFile(file.getFileName().toString()))
+ File file = path.toFile();
+ if (!Descriptor.isValidFile(file))
return FileVisitResult.CONTINUE;
try
{
- if (!Descriptor.fromFilename(file.toString()).isCompatible())
+ if (!Descriptor.fromFilename(file).isCompatible())
invalid.add(file.toString());
}
catch (Exception e)
@@ -414,7 +428,7 @@
// we do a one-off scrub of the system keyspace first; we can't load the list of the rest of the keyspaces,
// until system keyspace is opened.
- for (CFMetaData cfm : Schema.instance.getTablesAndViews(SchemaConstants.SYSTEM_KEYSPACE_NAME))
+ for (TableMetadata cfm : Schema.instance.getTablesAndViews(SchemaConstants.SYSTEM_KEYSPACE_NAME))
ColumnFamilyStore.scrubDataDirectories(cfm);
try
@@ -423,7 +437,7 @@
}
catch (ConfigurationException e)
{
- throw new StartupException(100, "Fatal exception during initialization", e);
+ throw new StartupException(StartupException.ERR_WRONG_CONFIG, "Fatal exception during initialization", e);
}
}
};
@@ -437,7 +451,7 @@
String storedDc = SystemKeyspace.getDatacenter();
if (storedDc != null)
{
- String currentDc = DatabaseDescriptor.getEndpointSnitch().getDatacenter(FBUtilities.getBroadcastAddress());
+ String currentDc = DatabaseDescriptor.getEndpointSnitch().getLocalDatacenter();
if (!storedDc.equals(currentDc))
{
String formatMessage = "Cannot start node if snitch's data center (%s) differs from previous data center (%s). " +
@@ -459,7 +473,7 @@
String storedRack = SystemKeyspace.getRack();
if (storedRack != null)
{
- String currentRack = DatabaseDescriptor.getEndpointSnitch().getRack(FBUtilities.getBroadcastAddress());
+ String currentRack = DatabaseDescriptor.getEndpointSnitch().getLocalRack();
if (!storedRack.equals(currentRack))
{
String formatMessage = "Cannot start node if snitch's rack (%s) differs from previous rack (%s). " +
@@ -472,14 +486,17 @@
}
};
- public static final StartupCheck checkLegacyAuthTables = () -> checkLegacyAuthTablesMessage().ifPresent(logger::warn);
-
- static final Set<String> LEGACY_AUTH_TABLES = ImmutableSet.of("credentials", "users", "permissions");
+ public static final StartupCheck checkLegacyAuthTables = () ->
+ {
+ Optional<String> errMsg = checkLegacyAuthTablesMessage();
+ if (errMsg.isPresent())
+ throw new StartupException(StartupException.ERR_WRONG_CONFIG, errMsg.get());
+ };
@VisibleForTesting
static Optional<String> checkLegacyAuthTablesMessage()
{
- List<String> existing = new ArrayList<>(LEGACY_AUTH_TABLES).stream().filter((legacyAuthTable) ->
+ List<String> existing = new ArrayList<>(SchemaConstants.LEGACY_AUTH_TABLES).stream().filter((legacyAuthTable) ->
{
UntypedResultSet result = QueryProcessor.executeOnceInternal(String.format("SELECT table_name FROM %s.%s WHERE keyspace_name='%s' AND table_name='%s'",
SchemaConstants.SCHEMA_KEYSPACE_NAME,
diff --git a/src/java/org/apache/cassandra/service/StorageProxy.java b/src/java/org/apache/cassandra/service/StorageProxy.java
index b1e0696..0aedd3d 100644
--- a/src/java/org/apache/cassandra/service/StorageProxy.java
+++ b/src/java/org/apache/cassandra/service/StorageProxy.java
@@ -17,61 +17,150 @@
*/
package org.apache.cassandra.service;
-import java.io.IOException;
-import java.net.InetAddress;
import java.nio.ByteBuffer;
-import java.util.*;
-import java.util.concurrent.*;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.EnumMap;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Future;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
-import com.google.common.base.Predicate;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
import com.google.common.cache.CacheLoader;
-import com.google.common.collect.*;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Iterators;
+import com.google.common.collect.PeekingIterator;
import com.google.common.primitives.Ints;
import com.google.common.util.concurrent.Uninterruptibles;
-
import org.apache.commons.lang3.StringUtils;
-
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.batchlog.Batch;
import org.apache.cassandra.batchlog.BatchlogManager;
-import org.apache.cassandra.batchlog.LegacyBatchlogMigrator;
import org.apache.cassandra.concurrent.Stage;
-import org.apache.cassandra.concurrent.StageManager;
-import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.config.Schema;
-import org.apache.cassandra.config.SchemaConstants;
-import org.apache.cassandra.db.*;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.ConsistencyLevel;
+import org.apache.cassandra.db.CounterMutation;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.HintedHandOffManager;
+import org.apache.cassandra.db.IMutation;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.Mutation;
+import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.db.PartitionRangeReadCommand;
+import org.apache.cassandra.db.ReadCommand;
+import org.apache.cassandra.db.ReadExecutionController;
+import org.apache.cassandra.db.ReadResponse;
+import org.apache.cassandra.db.SinglePartitionReadCommand;
+import org.apache.cassandra.db.TruncateRequest;
+import org.apache.cassandra.db.WriteType;
import org.apache.cassandra.db.filter.DataLimits;
import org.apache.cassandra.db.filter.TombstoneOverwhelmingException;
-import org.apache.cassandra.db.partitions.*;
+import org.apache.cassandra.db.partitions.FilteredPartition;
+import org.apache.cassandra.db.partitions.PartitionIterator;
+import org.apache.cassandra.db.partitions.PartitionIterators;
+import org.apache.cassandra.db.partitions.PartitionUpdate;
+import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
import org.apache.cassandra.db.rows.RowIterator;
import org.apache.cassandra.db.view.ViewUtils;
-import org.apache.cassandra.dht.*;
-import org.apache.cassandra.exceptions.*;
-import org.apache.cassandra.gms.FailureDetector;
+import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.dht.Bounds;
+import org.apache.cassandra.dht.RingPosition;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.exceptions.CasWriteTimeoutException;
+import org.apache.cassandra.exceptions.CasWriteUnknownResultException;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.exceptions.IsBootstrappingException;
+import org.apache.cassandra.exceptions.OverloadedException;
+import org.apache.cassandra.exceptions.ReadFailureException;
+import org.apache.cassandra.exceptions.ReadTimeoutException;
+import org.apache.cassandra.exceptions.RequestFailureException;
+import org.apache.cassandra.exceptions.RequestFailureReason;
+import org.apache.cassandra.exceptions.RequestTimeoutException;
+import org.apache.cassandra.exceptions.UnavailableException;
+import org.apache.cassandra.exceptions.WriteFailureException;
+import org.apache.cassandra.exceptions.WriteTimeoutException;
import org.apache.cassandra.gms.Gossiper;
import org.apache.cassandra.hints.Hint;
import org.apache.cassandra.hints.HintsService;
import org.apache.cassandra.index.Index;
-import org.apache.cassandra.io.util.DataOutputBuffer;
-import org.apache.cassandra.locator.*;
-import org.apache.cassandra.metrics.*;
-import org.apache.cassandra.net.*;
+import org.apache.cassandra.locator.AbstractReplicationStrategy;
+import org.apache.cassandra.locator.EndpointsForRange;
+import org.apache.cassandra.locator.EndpointsForToken;
+import org.apache.cassandra.locator.IEndpointSnitch;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.LocalStrategy;
+import org.apache.cassandra.locator.Replica;
+import org.apache.cassandra.locator.ReplicaLayout;
+import org.apache.cassandra.locator.ReplicaPlan;
+import org.apache.cassandra.locator.ReplicaPlans;
+import org.apache.cassandra.locator.Replicas;
+import org.apache.cassandra.locator.TokenMetadata;
+import org.apache.cassandra.metrics.CASClientRequestMetrics;
+import org.apache.cassandra.metrics.CASClientWriteRequestMetrics;
+import org.apache.cassandra.metrics.ClientRequestMetrics;
+import org.apache.cassandra.metrics.ClientWriteRequestMetrics;
+import org.apache.cassandra.metrics.ReadRepairMetrics;
+import org.apache.cassandra.metrics.StorageMetrics;
+import org.apache.cassandra.metrics.ViewWriteMetrics;
+import org.apache.cassandra.net.ForwardingInfo;
+import org.apache.cassandra.net.Message;
+import org.apache.cassandra.net.MessageFlag;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.net.RequestCallback;
+import org.apache.cassandra.net.Verb;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.SchemaConstants;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.service.paxos.Commit;
import org.apache.cassandra.service.paxos.PaxosState;
import org.apache.cassandra.service.paxos.PrepareCallback;
import org.apache.cassandra.service.paxos.ProposeCallback;
-import org.apache.cassandra.net.MessagingService.Verb;
+import org.apache.cassandra.service.reads.AbstractReadExecutor;
+import org.apache.cassandra.service.reads.DataResolver;
+import org.apache.cassandra.service.reads.ReadCallback;
+import org.apache.cassandra.service.reads.repair.ReadRepair;
import org.apache.cassandra.tracing.Tracing;
import org.apache.cassandra.triggers.TriggerExecutor;
-import org.apache.cassandra.utils.*;
import org.apache.cassandra.utils.AbstractIterator;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.MBeanWrapper;
+import org.apache.cassandra.utils.MonotonicClock;
+import org.apache.cassandra.utils.Pair;
+import org.apache.cassandra.utils.UUIDGen;
+
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+import static java.util.concurrent.TimeUnit.NANOSECONDS;
+import static org.apache.cassandra.net.NoPayload.noPayload;
+import static org.apache.cassandra.net.Verb.BATCH_STORE_REQ;
+import static org.apache.cassandra.net.Verb.MUTATION_REQ;
+import static org.apache.cassandra.net.Verb.PAXOS_COMMIT_REQ;
+import static org.apache.cassandra.net.Verb.PAXOS_PREPARE_REQ;
+import static org.apache.cassandra.net.Verb.PAXOS_PROPOSE_REQ;
+import static org.apache.cassandra.net.Verb.TRUNCATE_REQ;
+import static org.apache.cassandra.service.BatchlogResponseHandler.BatchlogCleanup;
+import static org.apache.cassandra.service.paxos.PrepareVerbHandler.doPrepare;
+import static org.apache.cassandra.service.paxos.ProposeVerbHandler.doPropose;
public class StorageProxy implements StorageProxyMBean
{
@@ -87,21 +176,21 @@
public static final StorageProxy instance = new StorageProxy();
private static volatile int maxHintsInProgress = 128 * FBUtilities.getAvailableProcessors();
- private static final CacheLoader<InetAddress, AtomicInteger> hintsInProgress = new CacheLoader<InetAddress, AtomicInteger>()
+ private static final CacheLoader<InetAddressAndPort, AtomicInteger> hintsInProgress = new CacheLoader<InetAddressAndPort, AtomicInteger>()
{
- public AtomicInteger load(InetAddress inetAddress)
+ public AtomicInteger load(InetAddressAndPort inetAddress)
{
return new AtomicInteger(0);
}
};
private static final ClientRequestMetrics readMetrics = new ClientRequestMetrics("Read");
private static final ClientRequestMetrics rangeMetrics = new ClientRequestMetrics("RangeSlice");
- private static final ClientRequestMetrics writeMetrics = new ClientRequestMetrics("Write");
- private static final CASClientRequestMetrics casWriteMetrics = new CASClientRequestMetrics("CASWrite");
+ private static final ClientWriteRequestMetrics writeMetrics = new ClientWriteRequestMetrics("Write");
+ private static final CASClientWriteRequestMetrics casWriteMetrics = new CASClientWriteRequestMetrics("CASWrite");
private static final CASClientRequestMetrics casReadMetrics = new CASClientRequestMetrics("CASRead");
private static final ViewWriteMetrics viewWriteMetrics = new ViewWriteMetrics("ViewWrite");
private static final Map<ConsistencyLevel, ClientRequestMetrics> readMetricsMap = new EnumMap<>(ConsistencyLevel.class);
- private static final Map<ConsistencyLevel, ClientRequestMetrics> writeMetricsMap = new EnumMap<>(ConsistencyLevel.class);
+ private static final Map<ConsistencyLevel, ClientWriteRequestMetrics> writeMetricsMap = new EnumMap<>(ConsistencyLevel.class);
private static final double CONCURRENT_SUBREQUESTS_MARGIN = 0.10;
@@ -124,18 +213,10 @@
HintsService.instance.registerMBean();
HintedHandOffManager.instance.registerMBean();
- standardWritePerformer = new WritePerformer()
+ standardWritePerformer = (mutation, targets, responseHandler, localDataCenter) ->
{
- public void apply(IMutation mutation,
- Iterable<InetAddress> targets,
- AbstractWriteResponseHandler<IMutation> responseHandler,
- String localDataCenter,
- ConsistencyLevel consistency_level)
- throws OverloadedException
- {
- assert mutation instanceof Mutation;
- sendToHintedEndpoints((Mutation) mutation, targets, responseHandler, localDataCenter, Stage.MUTATION);
- }
+ assert mutation instanceof Mutation;
+ sendToHintedReplicas((Mutation) mutation, targets, responseHandler, localDataCenter, Stage.MUTATION);
};
/*
@@ -144,36 +225,28 @@
* but on the latter case, the verb handler already run on the COUNTER_MUTATION stage, so we must not execute the
* underlying on the stage otherwise we risk a deadlock. Hence two different performer.
*/
- counterWritePerformer = new WritePerformer()
+ counterWritePerformer = (mutation, targets, responseHandler, localDataCenter) ->
{
- public void apply(IMutation mutation,
- Iterable<InetAddress> targets,
- AbstractWriteResponseHandler<IMutation> responseHandler,
- String localDataCenter,
- ConsistencyLevel consistencyLevel)
- {
- counterWriteTask(mutation, targets, responseHandler, localDataCenter).run();
- }
+ EndpointsForToken selected = targets.contacts().withoutSelf();
+ Replicas.temporaryAssertFull(selected); // TODO CASSANDRA-14548
+ counterWriteTask(mutation, targets.withContact(selected), responseHandler, localDataCenter).run();
};
- counterWriteOnCoordinatorPerformer = new WritePerformer()
+ counterWriteOnCoordinatorPerformer = (mutation, targets, responseHandler, localDataCenter) ->
{
- public void apply(IMutation mutation,
- Iterable<InetAddress> targets,
- AbstractWriteResponseHandler<IMutation> responseHandler,
- String localDataCenter,
- ConsistencyLevel consistencyLevel)
- {
- StageManager.getStage(Stage.COUNTER_MUTATION)
- .execute(counterWriteTask(mutation, targets, responseHandler, localDataCenter));
- }
+ EndpointsForToken selected = targets.contacts().withoutSelf();
+ Replicas.temporaryAssertFull(selected); // TODO CASSANDRA-14548
+ Stage.COUNTER_MUTATION.executor()
+ .execute(counterWriteTask(mutation, targets.withContact(selected), responseHandler, localDataCenter));
};
for(ConsistencyLevel level : ConsistencyLevel.values())
{
readMetricsMap.put(level, new ClientRequestMetrics("Read-" + level.name()));
- writeMetricsMap.put(level, new ClientRequestMetrics("Write-" + level.name()));
+ writeMetricsMap.put(level, new ClientWriteRequestMetrics("Write-" + level.name()));
}
+
+ ReadRepairMetrics.init();
}
/**
@@ -190,7 +263,7 @@
* 1. Prepare: the coordinator generates a ballot (timeUUID in our case) and asks replicas to (a) promise
* not to accept updates from older ballots and (b) tell us about the most recent update it has already
* accepted.
- * 2. Accept: if a majority of replicas reply, the coordinator asks replicas to accept the value of the
+ * 2. Accept: if a majority of replicas respond, the coordinator asks replicas to accept the value of the
* highest proposal ballot it heard about, or a new value if no in-progress proposals were reported.
* 3. Commit (Learn): if a majority of replicas acknowledge the accept request, we can commit the new
* value.
@@ -224,33 +297,31 @@
ConsistencyLevel consistencyForPaxos,
ConsistencyLevel consistencyForCommit,
ClientState state,
+ int nowInSeconds,
long queryStartNanoTime)
- throws UnavailableException, IsBootstrappingException, RequestFailureException, RequestTimeoutException, InvalidRequestException
+ throws UnavailableException, IsBootstrappingException, RequestFailureException, RequestTimeoutException, InvalidRequestException, CasWriteUnknownResultException
{
final long startTimeForMetrics = System.nanoTime();
+ TableMetadata metadata = Schema.instance.getTableMetadata(keyspaceName, cfName);
int contentions = 0;
try
{
consistencyForPaxos.validateForCas();
consistencyForCommit.validateForCasCommit(keyspaceName);
- CFMetaData metadata = Schema.instance.getCFMetaData(keyspaceName, cfName);
-
- long timeout = TimeUnit.MILLISECONDS.toNanos(DatabaseDescriptor.getCasContentionTimeout());
- while (System.nanoTime() - queryStartNanoTime < timeout)
+ long timeoutNanos = DatabaseDescriptor.getCasContentionTimeout(NANOSECONDS);
+ while (System.nanoTime() - queryStartNanoTime < timeoutNanos)
{
// for simplicity, we'll do a single liveness check at the start of each attempt
- Pair<List<InetAddress>, Integer> p = getPaxosParticipants(metadata, key, consistencyForPaxos);
- List<InetAddress> liveEndpoints = p.left;
- int requiredParticipants = p.right;
+ ReplicaPlan.ForPaxosWrite replicaPlan = ReplicaPlans.forPaxos(Keyspace.open(keyspaceName), key, consistencyForPaxos);
- final Pair<UUID, Integer> pair = beginAndRepairPaxos(queryStartNanoTime, key, metadata, liveEndpoints, requiredParticipants, consistencyForPaxos, consistencyForCommit, true, state);
- final UUID ballot = pair.left;
- contentions += pair.right;
+ final PaxosBallotAndContention pair = beginAndRepairPaxos(queryStartNanoTime, key, metadata, replicaPlan, consistencyForPaxos, consistencyForCommit, true, state);
+ final UUID ballot = pair.ballot;
+ contentions += pair.contentions;
// read the current values and check they validate the conditions
Tracing.trace("Reading existing values for CAS precondition");
- SinglePartitionReadCommand readCommand = request.readCommand(FBUtilities.nowInSeconds());
+ SinglePartitionReadCommand readCommand = (SinglePartitionReadCommand) request.readCommand(nowInSeconds);
ConsistencyLevel readConsistency = consistencyForPaxos == ConsistencyLevel.LOCAL_SERIAL ? ConsistencyLevel.LOCAL_QUORUM : ConsistencyLevel.QUORUM;
FilteredPartition current;
@@ -270,6 +341,10 @@
// TODO turn null updates into delete?
PartitionUpdate updates = request.makeUpdates(current);
+ long size = updates.dataSize();
+ casWriteMetrics.mutationSize.update(size);
+ writeMetricsMap.get(consistencyForPaxos).mutationSize.update(size);
+
// Apply triggers to cas updates. A consideration here is that
// triggers emit Mutations, and so a given trigger implementation
// may generate mutations for partitions other than the one this
@@ -282,7 +357,7 @@
Commit proposal = Commit.newProposal(ballot, updates);
Tracing.trace("CAS precondition is met; proposing client-requested updates for {}", ballot);
- if (proposePaxos(proposal, liveEndpoints, requiredParticipants, true, consistencyForPaxos, queryStartNanoTime))
+ if (proposePaxos(proposal, replicaPlan, true, queryStartNanoTime))
{
commitPaxos(proposal, consistencyForCommit, true, queryStartNanoTime);
Tracing.trace("CAS successful");
@@ -291,13 +366,24 @@
Tracing.trace("Paxos proposal not accepted (pre-empted by a higher ballot)");
contentions++;
- Uninterruptibles.sleepUninterruptibly(ThreadLocalRandom.current().nextInt(100), TimeUnit.MILLISECONDS);
+ Uninterruptibles.sleepUninterruptibly(ThreadLocalRandom.current().nextInt(100), MILLISECONDS);
// continue to retry
}
throw new WriteTimeoutException(WriteType.CAS, consistencyForPaxos, 0, consistencyForPaxos.blockFor(Keyspace.open(keyspaceName)));
}
- catch (WriteTimeoutException|ReadTimeoutException e)
+ catch (CasWriteUnknownResultException e)
+ {
+ casWriteMetrics.unknownResult.mark();
+ throw e;
+ }
+ catch (WriteTimeoutException wte)
+ {
+ casWriteMetrics.timeouts.mark();
+ writeMetricsMap.get(consistencyForPaxos).timeouts.mark();
+ throw new CasWriteTimeoutException(wte.writeType, wte.consistency, wte.received, wte.blockFor, contentions);
+ }
+ catch (ReadTimeoutException e)
{
casWriteMetrics.timeouts.mark();
writeMetricsMap.get(consistencyForPaxos).timeouts.mark();
@@ -309,7 +395,7 @@
writeMetricsMap.get(consistencyForPaxos).failures.mark();
throw e;
}
- catch(UnavailableException e)
+ catch (UnavailableException e)
{
casWriteMetrics.unavailables.mark();
writeMetricsMap.get(consistencyForPaxos).unavailables.mark();
@@ -318,6 +404,7 @@
finally
{
recordCasContention(contentions);
+ Keyspace.open(keyspaceName).getColumnFamilyStore(cfName).metric.topCasPartitionContention.addSample(key.getKey(), contentions);
final long latency = System.nanoTime() - startTimeForMetrics;
casWriteMetrics.addNano(latency);
writeMetricsMap.get(consistencyForPaxos).addNano(latency);
@@ -330,71 +417,27 @@
casWriteMetrics.contention.update(contentions);
}
- private static Predicate<InetAddress> sameDCPredicateFor(final String dc)
- {
- final IEndpointSnitch snitch = DatabaseDescriptor.getEndpointSnitch();
- return new Predicate<InetAddress>()
- {
- public boolean apply(InetAddress host)
- {
- return dc.equals(snitch.getDatacenter(host));
- }
- };
- }
-
- private static Pair<List<InetAddress>, Integer> getPaxosParticipants(CFMetaData cfm, DecoratedKey key, ConsistencyLevel consistencyForPaxos) throws UnavailableException
- {
- Token tk = key.getToken();
- List<InetAddress> naturalEndpoints = StorageService.instance.getNaturalEndpoints(cfm.ksName, tk);
- Collection<InetAddress> pendingEndpoints = StorageService.instance.getTokenMetadata().pendingEndpointsFor(tk, cfm.ksName);
- if (consistencyForPaxos == ConsistencyLevel.LOCAL_SERIAL)
- {
- // Restrict naturalEndpoints and pendingEndpoints to node in the local DC only
- String localDc = DatabaseDescriptor.getEndpointSnitch().getDatacenter(FBUtilities.getBroadcastAddress());
- Predicate<InetAddress> isLocalDc = sameDCPredicateFor(localDc);
- naturalEndpoints = ImmutableList.copyOf(Iterables.filter(naturalEndpoints, isLocalDc));
- pendingEndpoints = ImmutableList.copyOf(Iterables.filter(pendingEndpoints, isLocalDc));
- }
- int participants = pendingEndpoints.size() + naturalEndpoints.size();
- int requiredParticipants = participants / 2 + 1; // See CASSANDRA-8346, CASSANDRA-833
- List<InetAddress> liveEndpoints = ImmutableList.copyOf(Iterables.filter(Iterables.concat(naturalEndpoints, pendingEndpoints), IAsyncCallback.isAlive));
- if (liveEndpoints.size() < requiredParticipants)
- throw new UnavailableException(consistencyForPaxos, requiredParticipants, liveEndpoints.size());
-
- // We cannot allow CAS operations with 2 or more pending endpoints, see #8346.
- // Note that we fake an impossible number of required nodes in the unavailable exception
- // to nail home the point that it's an impossible operation no matter how many nodes are live.
- if (pendingEndpoints.size() > 1)
- throw new UnavailableException(String.format("Cannot perform LWT operation as there is more than one (%d) pending range movement", pendingEndpoints.size()),
- consistencyForPaxos,
- participants + 1,
- liveEndpoints.size());
-
- return Pair.create(liveEndpoints, requiredParticipants);
- }
-
/**
* begin a Paxos session by sending a prepare request and completing any in-progress requests seen in the replies
*
* @return the Paxos ballot promised by the replicas if no in-progress requests were seen and a quorum of
* nodes have seen the mostRecentCommit. Otherwise, return null.
*/
- private static Pair<UUID, Integer> beginAndRepairPaxos(long queryStartNanoTime,
- DecoratedKey key,
- CFMetaData metadata,
- List<InetAddress> liveEndpoints,
- int requiredParticipants,
- ConsistencyLevel consistencyForPaxos,
- ConsistencyLevel consistencyForCommit,
- final boolean isWrite,
- ClientState state)
+ private static PaxosBallotAndContention beginAndRepairPaxos(long queryStartNanoTime,
+ DecoratedKey key,
+ TableMetadata metadata,
+ ReplicaPlan.ForPaxosWrite paxosPlan,
+ ConsistencyLevel consistencyForPaxos,
+ ConsistencyLevel consistencyForCommit,
+ final boolean isWrite,
+ ClientState state)
throws WriteTimeoutException, WriteFailureException
{
- long timeout = TimeUnit.MILLISECONDS.toNanos(DatabaseDescriptor.getCasContentionTimeout());
+ long timeoutNanos = DatabaseDescriptor.getCasContentionTimeout(NANOSECONDS);
PrepareCallback summary = null;
int contentions = 0;
- while (System.nanoTime() - queryStartNanoTime < timeout)
+ while (System.nanoTime() - queryStartNanoTime < timeoutNanos)
{
// We want a timestamp that is guaranteed to be unique for that node (so that the ballot is globally unique), but if we've got a prepare rejected
// already we also want to make sure we pick a timestamp that has a chance to be promised, i.e. one that is greater that the most recently known
@@ -409,13 +452,13 @@
// prepare
Tracing.trace("Preparing {}", ballot);
Commit toPrepare = Commit.newPrepare(key, metadata, ballot);
- summary = preparePaxos(toPrepare, liveEndpoints, requiredParticipants, consistencyForPaxos, queryStartNanoTime);
+ summary = preparePaxos(toPrepare, paxosPlan, queryStartNanoTime);
if (!summary.promised)
{
Tracing.trace("Some replicas have already promised a higher ballot than ours; aborting");
contentions++;
// sleep a random amount to give the other proposer a chance to finish
- Uninterruptibles.sleepUninterruptibly(ThreadLocalRandom.current().nextInt(100), TimeUnit.MILLISECONDS);
+ Uninterruptibles.sleepUninterruptibly(ThreadLocalRandom.current().nextInt(100), MILLISECONDS);
continue;
}
@@ -432,7 +475,7 @@
else
casReadMetrics.unfinishedCommit.inc();
Commit refreshedInProgress = Commit.newProposal(ballot, inProgress.update);
- if (proposePaxos(refreshedInProgress, liveEndpoints, requiredParticipants, false, consistencyForPaxos, queryStartNanoTime))
+ if (proposePaxos(refreshedInProgress, paxosPlan, false, queryStartNanoTime))
{
try
{
@@ -450,7 +493,7 @@
Tracing.trace("Some replicas have already promised a higher ballot than ours; aborting");
// sleep a random amount to give the other proposer a chance to finish
contentions++;
- Uninterruptibles.sleepUninterruptibly(ThreadLocalRandom.current().nextInt(100), TimeUnit.MILLISECONDS);
+ Uninterruptibles.sleepUninterruptibly(ThreadLocalRandom.current().nextInt(100), MILLISECONDS);
}
continue;
}
@@ -460,7 +503,7 @@
// Since we waited for quorum nodes, if some of them haven't seen the last commit (which may just be a timing issue, but may also
// mean we lost messages), we pro-actively "repair" those nodes, and retry.
int nowInSec = Ints.checkedCast(TimeUnit.MICROSECONDS.toSeconds(ballotMicros));
- Iterable<InetAddress> missingMRC = summary.replicasMissingMostRecentCommit(metadata, nowInSec);
+ Iterable<InetAddressAndPort> missingMRC = summary.replicasMissingMostRecentCommit(metadata, nowInSec);
if (Iterables.size(missingMRC) > 0)
{
Tracing.trace("Repairing replicas that missed the most recent commit");
@@ -472,49 +515,90 @@
continue;
}
- return Pair.create(ballot, contentions);
+ return new PaxosBallotAndContention(ballot, contentions);
}
recordCasContention(contentions);
- throw new WriteTimeoutException(WriteType.CAS, consistencyForPaxos, 0, consistencyForPaxos.blockFor(Keyspace.open(metadata.ksName)));
+ throw new WriteTimeoutException(WriteType.CAS, consistencyForPaxos, 0, consistencyForPaxos.blockFor(Keyspace.open(metadata.keyspace)));
}
/**
* Unlike commitPaxos, this does not wait for replies
*/
- private static void sendCommit(Commit commit, Iterable<InetAddress> replicas)
+ private static void sendCommit(Commit commit, Iterable<InetAddressAndPort> replicas)
{
- MessageOut<Commit> message = new MessageOut<Commit>(MessagingService.Verb.PAXOS_COMMIT, commit, Commit.serializer);
- for (InetAddress target : replicas)
- MessagingService.instance().sendOneWay(message, target);
+ Message<Commit> message = Message.out(PAXOS_COMMIT_REQ, commit);
+ for (InetAddressAndPort target : replicas)
+ MessagingService.instance().send(message, target);
}
- private static PrepareCallback preparePaxos(Commit toPrepare, List<InetAddress> endpoints, int requiredParticipants, ConsistencyLevel consistencyForPaxos, long queryStartNanoTime)
+ private static PrepareCallback preparePaxos(Commit toPrepare, ReplicaPlan.ForPaxosWrite replicaPlan, long queryStartNanoTime)
throws WriteTimeoutException
{
- PrepareCallback callback = new PrepareCallback(toPrepare.update.partitionKey(), toPrepare.update.metadata(), requiredParticipants, consistencyForPaxos, queryStartNanoTime);
- MessageOut<Commit> message = new MessageOut<Commit>(MessagingService.Verb.PAXOS_PREPARE, toPrepare, Commit.serializer);
- for (InetAddress target : endpoints)
- MessagingService.instance().sendRR(message, target, callback);
+ PrepareCallback callback = new PrepareCallback(toPrepare.update.partitionKey(), toPrepare.update.metadata(), replicaPlan.requiredParticipants(), replicaPlan.consistencyLevel(), queryStartNanoTime);
+ Message<Commit> message = Message.out(PAXOS_PREPARE_REQ, toPrepare);
+ for (Replica replica: replicaPlan.contacts())
+ {
+ if (replica.isSelf())
+ {
+ PAXOS_PREPARE_REQ.stage.execute(() -> {
+ try
+ {
+ callback.onResponse(message.responseWith(doPrepare(toPrepare)));
+ }
+ catch (Exception ex)
+ {
+ logger.error("Failed paxos prepare locally", ex);
+ }
+ });
+ }
+ else
+ {
+ MessagingService.instance().sendWithCallback(message, replica.endpoint(), callback);
+ }
+ }
callback.await();
return callback;
}
- private static boolean proposePaxos(Commit proposal, List<InetAddress> endpoints, int requiredParticipants, boolean timeoutIfPartial, ConsistencyLevel consistencyLevel, long queryStartNanoTime)
- throws WriteTimeoutException
+ /**
+ * Propose the {@param proposal} accoding to the {@param replicaPlan}.
+ * When {@param backoffIfPartial} is true, the proposer backs off when seeing the proposal being accepted by some but not a quorum.
+ * The result of the cooresponding CAS in uncertain as the accepted proposal may or may not be spread to other nodes in later rounds.
+ */
+ private static boolean proposePaxos(Commit proposal, ReplicaPlan.ForPaxosWrite replicaPlan, boolean backoffIfPartial, long queryStartNanoTime)
+ throws WriteTimeoutException, CasWriteUnknownResultException
{
- ProposeCallback callback = new ProposeCallback(endpoints.size(), requiredParticipants, !timeoutIfPartial, consistencyLevel, queryStartNanoTime);
- MessageOut<Commit> message = new MessageOut<Commit>(MessagingService.Verb.PAXOS_PROPOSE, proposal, Commit.serializer);
- for (InetAddress target : endpoints)
- MessagingService.instance().sendRR(message, target, callback);
-
+ ProposeCallback callback = new ProposeCallback(replicaPlan.contacts().size(), replicaPlan.requiredParticipants(), !backoffIfPartial, replicaPlan.consistencyLevel(), queryStartNanoTime);
+ Message<Commit> message = Message.out(PAXOS_PROPOSE_REQ, proposal);
+ for (Replica replica : replicaPlan.contacts())
+ {
+ if (replica.isSelf())
+ {
+ PAXOS_PROPOSE_REQ.stage.execute(() -> {
+ try
+ {
+ Message<Boolean> response = message.responseWith(doPropose(proposal));
+ callback.onResponse(response);
+ }
+ catch (Exception ex)
+ {
+ logger.error("Failed paxos propose locally", ex);
+ }
+ });
+ }
+ else
+ {
+ MessagingService.instance().sendWithCallback(message, replica.endpoint(), callback);
+ }
+ }
callback.await();
if (callback.isSuccessful())
return true;
- if (timeoutIfPartial && !callback.isFullyRefused())
- throw new WriteTimeoutException(WriteType.CAS, consistencyLevel, callback.getAcceptCount(), requiredParticipants);
+ if (backoffIfPartial && !callback.isFullyRefused())
+ throw new CasWriteUnknownResultException(replicaPlan.consistencyLevel(), callback.getAcceptCount(), replicaPlan.requiredParticipants());
return false;
}
@@ -522,42 +606,50 @@
private static void commitPaxos(Commit proposal, ConsistencyLevel consistencyLevel, boolean allowHints, long queryStartNanoTime) throws WriteTimeoutException
{
boolean shouldBlock = consistencyLevel != ConsistencyLevel.ANY;
- Keyspace keyspace = Keyspace.open(proposal.update.metadata().ksName);
+ Keyspace keyspace = Keyspace.open(proposal.update.metadata().keyspace);
Token tk = proposal.update.partitionKey().getToken();
- List<InetAddress> naturalEndpoints = StorageService.instance.getNaturalEndpoints(keyspace.getName(), tk);
- Collection<InetAddress> pendingEndpoints = StorageService.instance.getTokenMetadata().pendingEndpointsFor(tk, keyspace.getName());
AbstractWriteResponseHandler<Commit> responseHandler = null;
+ // NOTE: this ReplicaPlan is a lie, this usage of ReplicaPlan could do with being clarified - the selected() collection is essentially (I think) never used
+ ReplicaPlan.ForTokenWrite replicaPlan = ReplicaPlans.forWrite(keyspace, consistencyLevel, tk, ReplicaPlans.writeAll);
if (shouldBlock)
{
AbstractReplicationStrategy rs = keyspace.getReplicationStrategy();
- responseHandler = rs.getWriteResponseHandler(naturalEndpoints, pendingEndpoints, consistencyLevel, null, WriteType.SIMPLE, queryStartNanoTime);
+ responseHandler = rs.getWriteResponseHandler(replicaPlan, null, WriteType.SIMPLE, queryStartNanoTime);
responseHandler.setSupportsBackPressure(false);
}
- MessageOut<Commit> message = new MessageOut<Commit>(MessagingService.Verb.PAXOS_COMMIT, proposal, Commit.serializer);
- for (InetAddress destination : Iterables.concat(naturalEndpoints, pendingEndpoints))
+ Message<Commit> message = Message.outWithFlag(PAXOS_COMMIT_REQ, proposal, MessageFlag.CALL_BACK_ON_FAILURE);
+ for (Replica replica : replicaPlan.liveAndDown())
{
- checkHintOverload(destination);
+ InetAddressAndPort destination = replica.endpoint();
+ checkHintOverload(replica);
- if (FailureDetector.instance.isAlive(destination))
+ if (replicaPlan.isAlive(replica))
{
if (shouldBlock)
{
- if (canDoLocalRequest(destination))
- commitPaxosLocal(message, responseHandler);
+ if (replica.isSelf())
+ commitPaxosLocal(replica, message, responseHandler);
else
- MessagingService.instance().sendRR(message, destination, responseHandler, allowHints && shouldHint(destination));
+ MessagingService.instance().sendWriteWithCallback(message, replica, responseHandler, allowHints && shouldHint(replica));
}
else
{
- MessagingService.instance().sendOneWay(message, destination);
+ MessagingService.instance().send(message, destination);
}
}
- else if (allowHints && shouldHint(destination))
+ else
{
- submitHint(proposal.makeMutation(), destination, null);
+ if (responseHandler != null)
+ {
+ responseHandler.expired();
+ }
+ if (allowHints && shouldHint(replica))
+ {
+ submitHint(proposal.makeMutation(), replica, null);
+ }
}
}
@@ -570,9 +662,9 @@
* submit a fake one that executes immediately on the mutation stage, but generates the necessary backpressure
* signal for hints
*/
- private static void commitPaxosLocal(final MessageOut<Commit> message, final AbstractWriteResponseHandler<?> responseHandler)
+ private static void commitPaxosLocal(Replica localReplica, final Message<Commit> message, final AbstractWriteResponseHandler<?> responseHandler)
{
- StageManager.getStage(MessagingService.verbStages.get(MessagingService.Verb.PAXOS_COMMIT)).maybeExecuteImmediately(new LocalMutationRunnable()
+ PAXOS_COMMIT_REQ.stage.maybeExecuteImmediately(new LocalMutationRunnable(localReplica)
{
public void runMayThrow()
{
@@ -580,20 +672,20 @@
{
PaxosState.commit(message.payload);
if (responseHandler != null)
- responseHandler.response(null);
+ responseHandler.onResponse(null);
}
catch (Exception ex)
{
if (!(ex instanceof WriteTimeoutException))
- logger.error("Failed to apply paxos commit locally : {}", ex);
- responseHandler.onFailure(FBUtilities.getBroadcastAddress(), RequestFailureReason.UNKNOWN);
+ logger.error("Failed to apply paxos commit locally : ", ex);
+ responseHandler.onFailure(FBUtilities.getBroadcastAddressAndPort(), RequestFailureReason.forException(ex));
}
}
@Override
protected Verb verb()
{
- return MessagingService.Verb.PAXOS_COMMIT;
+ return PAXOS_COMMIT_REQ;
}
});
}
@@ -605,42 +697,44 @@
* the data across to some other replica.
*
* @param mutations the mutations to be applied across the replicas
- * @param consistency_level the consistency level for the operation
+ * @param consistencyLevel the consistency level for the operation
* @param queryStartNanoTime the value of System.nanoTime() when the query started to be processed
*/
- public static void mutate(Collection<? extends IMutation> mutations, ConsistencyLevel consistency_level, long queryStartNanoTime)
+ public static void mutate(List<? extends IMutation> mutations, ConsistencyLevel consistencyLevel, long queryStartNanoTime)
throws UnavailableException, OverloadedException, WriteTimeoutException, WriteFailureException
{
Tracing.trace("Determining replicas for mutation");
- final String localDataCenter = DatabaseDescriptor.getEndpointSnitch().getDatacenter(FBUtilities.getBroadcastAddress());
+ final String localDataCenter = DatabaseDescriptor.getEndpointSnitch().getLocalDatacenter();
long startTime = System.nanoTime();
+
List<AbstractWriteResponseHandler<IMutation>> responseHandlers = new ArrayList<>(mutations.size());
+ WriteType plainWriteType = mutations.size() <= 1 ? WriteType.SIMPLE : WriteType.UNLOGGED_BATCH;
try
{
for (IMutation mutation : mutations)
{
if (mutation instanceof CounterMutation)
- {
responseHandlers.add(mutateCounter((CounterMutation)mutation, localDataCenter, queryStartNanoTime));
- }
else
- {
- WriteType wt = mutations.size() <= 1 ? WriteType.SIMPLE : WriteType.UNLOGGED_BATCH;
- responseHandlers.add(performWrite(mutation, consistency_level, localDataCenter, standardWritePerformer, null, wt, queryStartNanoTime));
- }
+ responseHandlers.add(performWrite(mutation, consistencyLevel, localDataCenter, standardWritePerformer, null, plainWriteType, queryStartNanoTime));
+ }
+
+ // upgrade to full quorum any failed cheap quorums
+ for (int i = 0 ; i < mutations.size() ; ++i)
+ {
+ if (!(mutations.get(i) instanceof CounterMutation)) // at the moment, only non-counter writes support cheap quorums
+ responseHandlers.get(i).maybeTryAdditionalReplicas(mutations.get(i), standardWritePerformer, localDataCenter);
}
// wait for writes. throws TimeoutException if necessary
for (AbstractWriteResponseHandler<IMutation> responseHandler : responseHandlers)
- {
responseHandler.get();
- }
}
catch (WriteTimeoutException|WriteFailureException ex)
{
- if (consistency_level == ConsistencyLevel.ANY)
+ if (consistencyLevel == ConsistencyLevel.ANY)
{
hintMutations(mutations);
}
@@ -649,7 +743,7 @@
if (ex instanceof WriteFailureException)
{
writeMetrics.failures.mark();
- writeMetricsMap.get(consistency_level).failures.mark();
+ writeMetricsMap.get(consistencyLevel).failures.mark();
WriteFailureException fe = (WriteFailureException)ex;
Tracing.trace("Write failure; received {} of {} required replies, failed {} requests",
fe.received, fe.blockFor, fe.failureReasonByEndpoint.size());
@@ -657,7 +751,7 @@
else
{
writeMetrics.timeouts.mark();
- writeMetricsMap.get(consistency_level).timeouts.mark();
+ writeMetricsMap.get(consistencyLevel).timeouts.mark();
WriteTimeoutException te = (WriteTimeoutException)ex;
Tracing.trace("Write timeout; received {} of {} required replies", te.received, te.blockFor);
}
@@ -667,14 +761,14 @@
catch (UnavailableException e)
{
writeMetrics.unavailables.mark();
- writeMetricsMap.get(consistency_level).unavailables.mark();
+ writeMetricsMap.get(consistencyLevel).unavailables.mark();
Tracing.trace("Unavailable");
throw e;
}
catch (OverloadedException e)
{
writeMetrics.unavailables.mark();
- writeMetricsMap.get(consistency_level).unavailables.mark();
+ writeMetricsMap.get(consistencyLevel).unavailables.mark();
Tracing.trace("Overloaded");
throw e;
}
@@ -682,7 +776,8 @@
{
long latency = System.nanoTime() - startTime;
writeMetrics.addNano(latency);
- writeMetricsMap.get(consistency_level).addNano(latency);
+ writeMetricsMap.get(consistencyLevel).addNano(latency);
+ updateCoordinatorWriteLatencyTableMetric(mutations, latency);
}
}
@@ -709,26 +804,23 @@
String keyspaceName = mutation.getKeyspaceName();
Token token = mutation.key().getToken();
- Iterable<InetAddress> endpoints = StorageService.instance.getNaturalAndPendingEndpoints(keyspaceName, token);
- ArrayList<InetAddress> endpointsToHint = new ArrayList<>(Iterables.size(endpoints));
-
// local writes can timeout, but cannot be dropped (see LocalMutationRunnable and CASSANDRA-6510),
// so there is no need to hint or retry.
- for (InetAddress target : endpoints)
- if (!target.equals(FBUtilities.getBroadcastAddress()) && shouldHint(target))
- endpointsToHint.add(target);
+ EndpointsForToken replicasToHint = ReplicaLayout.forTokenWriteLiveAndDown(Keyspace.open(keyspaceName), token)
+ .all()
+ .filter(StorageProxy::shouldHint);
- submitHint(mutation, endpointsToHint, null);
+ submitHint(mutation, replicasToHint, null);
}
public boolean appliesLocally(Mutation mutation)
{
String keyspaceName = mutation.getKeyspaceName();
Token token = mutation.key().getToken();
- InetAddress local = FBUtilities.getBroadcastAddress();
+ InetAddressAndPort local = FBUtilities.getBroadcastAddressAndPort();
- return StorageService.instance.getNaturalEndpoints(keyspaceName, token).contains(local)
- || StorageService.instance.getTokenMetadata().pendingEndpointsFor(token, keyspaceName).contains(local);
+ return ReplicaLayout.forTokenWriteLiveAndDown(Keyspace.open(keyspaceName), token)
+ .all().endpoints().contains(local);
}
/**
@@ -744,7 +836,7 @@
throws UnavailableException, OverloadedException, WriteTimeoutException
{
Tracing.trace("Determining replicas for mutation");
- final String localDataCenter = DatabaseDescriptor.getEndpointSnitch().getDatacenter(FBUtilities.getBroadcastAddress());
+ final String localDataCenter = DatabaseDescriptor.getEndpointSnitch().getLocalDatacenter();
long startTime = System.nanoTime();
@@ -769,21 +861,22 @@
ConsistencyLevel consistencyLevel = ConsistencyLevel.ONE;
//Since the base -> view replication is 1:1 we only need to store the BL locally
- final Collection<InetAddress> batchlogEndpoints = Collections.singleton(FBUtilities.getBroadcastAddress());
- BatchlogResponseHandler.BatchlogCleanup cleanup = new BatchlogResponseHandler.BatchlogCleanup(mutations.size(), () -> asyncRemoveFromBatchlog(batchlogEndpoints, batchUUID));
+ ReplicaPlan.ForTokenWrite replicaPlan = ReplicaPlans.forLocalBatchlogWrite();
+ BatchlogCleanup cleanup = new BatchlogCleanup(mutations.size(),
+ () -> asyncRemoveFromBatchlog(replicaPlan, batchUUID));
// add a handler for each mutation - includes checking availability, but doesn't initiate any writes, yet
for (Mutation mutation : mutations)
{
String keyspaceName = mutation.getKeyspaceName();
Token tk = mutation.key().getToken();
- Optional<InetAddress> pairedEndpoint = ViewUtils.getViewNaturalEndpoint(keyspaceName, baseToken, tk);
- Collection<InetAddress> pendingEndpoints = StorageService.instance.getTokenMetadata().pendingEndpointsFor(tk, keyspaceName);
+ Optional<Replica> pairedEndpoint = ViewUtils.getViewNaturalEndpoint(keyspaceName, baseToken, tk);
+ EndpointsForToken pendingReplicas = StorageService.instance.getTokenMetadata().pendingEndpointsForToken(tk, keyspaceName);
// if there are no paired endpoints there are probably range movements going on, so we write to the local batchlog to replay later
if (!pairedEndpoint.isPresent())
{
- if (pendingEndpoints.isEmpty())
+ if (pendingReplicas.isEmpty())
logger.warn("Received base materialized view mutation for key {} that does not belong " +
"to this node. There is probably a range movement happening (move or decommission)," +
"but this node hasn't updated its ring metadata yet. Adding mutation to " +
@@ -795,8 +888,8 @@
// When local node is the endpoint we can just apply the mutation locally,
// unless there are pending endpoints, in which case we want to do an ordinary
// write so the view mutation is sent to the pending endpoint
- if (pairedEndpoint.get().equals(FBUtilities.getBroadcastAddress()) && StorageService.instance.isJoined()
- && pendingEndpoints.isEmpty())
+ if (pairedEndpoint.get().isSelf() && StorageService.instance.isJoined()
+ && pendingReplicas.isEmpty())
{
try
{
@@ -815,7 +908,8 @@
wrappers.add(wrapViewBatchResponseHandler(mutation,
consistencyLevel,
consistencyLevel,
- Collections.singletonList(pairedEndpoint.get()),
+ EndpointsForToken.of(tk, pairedEndpoint.get()),
+ pendingReplicas,
baseComplete,
WriteType.BATCH,
cleanup,
@@ -839,7 +933,7 @@
}
@SuppressWarnings("unchecked")
- public static void mutateWithTriggers(Collection<? extends IMutation> mutations,
+ public static void mutateWithTriggers(List<? extends IMutation> mutations,
ConsistencyLevel consistencyLevel,
boolean mutateAtomically,
long queryStartNanoTime)
@@ -851,6 +945,10 @@
.viewManager
.updatesAffectView(mutations, true);
+ long size = IMutation.dataSize(mutations);
+ writeMetrics.mutationSize.update(size);
+ writeMetricsMap.get(consistencyLevel).mutationSize.update(size);
+
if (augmented != null)
mutateAtomically(augmented, consistencyLevel, updatesView, queryStartNanoTime);
else
@@ -883,7 +981,9 @@
long startTime = System.nanoTime();
List<WriteResponseHandlerWrapper> wrappers = new ArrayList<WriteResponseHandlerWrapper>(mutations.size());
- String localDataCenter = DatabaseDescriptor.getEndpointSnitch().getDatacenter(FBUtilities.getBroadcastAddress());
+
+ if (mutations.stream().anyMatch(mutation -> Keyspace.open(mutation.getKeyspaceName()).getReplicationStrategy().hasTransientReplicas()))
+ throw new AssertionError("Logged batches are unsupported with transient replication");
try
{
@@ -901,10 +1001,11 @@
batchConsistencyLevel = consistency_level;
}
- final BatchlogEndpoints batchlogEndpoints = getBatchlogEndpoints(localDataCenter, batchConsistencyLevel);
+ ReplicaPlan.ForTokenWrite replicaPlan = ReplicaPlans.forBatchlogWrite(batchConsistencyLevel == ConsistencyLevel.ANY);
+
final UUID batchUUID = UUIDGen.getTimeUUID();
- BatchlogResponseHandler.BatchlogCleanup cleanup = new BatchlogResponseHandler.BatchlogCleanup(mutations.size(),
- () -> asyncRemoveFromBatchlog(batchlogEndpoints, batchUUID, queryStartNanoTime));
+ BatchlogCleanup cleanup = new BatchlogCleanup(mutations.size(),
+ () -> asyncRemoveFromBatchlog(replicaPlan, batchUUID));
// add a handler for each mutation - includes checking availability, but doesn't initiate any writes, yet
for (Mutation mutation : mutations)
@@ -916,15 +1017,14 @@
cleanup,
queryStartNanoTime);
// exit early if we can't fulfill the CL at this time.
- wrapper.handler.assureSufficientLiveNodes();
wrappers.add(wrapper);
}
// write to the batchlog
- syncWriteToBatchlog(mutations, batchlogEndpoints, batchUUID, queryStartNanoTime);
+ syncWriteToBatchlog(mutations, replicaPlan, batchUUID, queryStartNanoTime);
// now actually perform the writes and wait for them to complete
- syncWriteBatchedMutations(wrappers, localDataCenter, Stage.MUTATION);
+ syncWriteBatchedMutations(wrappers, Stage.MUTATION);
}
catch (UnavailableException e)
{
@@ -952,74 +1052,65 @@
long latency = System.nanoTime() - startTime;
writeMetrics.addNano(latency);
writeMetricsMap.get(consistency_level).addNano(latency);
-
+ updateCoordinatorWriteLatencyTableMetric(mutations, latency);
}
}
- public static boolean canDoLocalRequest(InetAddress replica)
+ private static void updateCoordinatorWriteLatencyTableMetric(Collection<? extends IMutation> mutations, long latency)
{
- return replica.equals(FBUtilities.getBroadcastAddress());
+ if (null == mutations)
+ {
+ return;
+ }
+
+ try
+ {
+ //We could potentially pass a callback into performWrite. And add callback provision for mutateCounter or mutateAtomically (sendToHintedEndPoints)
+ //However, Trade off between write metric per CF accuracy vs performance hit due to callbacks. Similar issue exists with CoordinatorReadLatency metric.
+ mutations.stream()
+ .flatMap(m -> m.getTableIds().stream().map(tableId -> Keyspace.open(m.getKeyspaceName()).getColumnFamilyStore(tableId)))
+ .distinct()
+ .forEach(store -> store.metric.coordinatorWriteLatency.update(latency, TimeUnit.NANOSECONDS));
+ }
+ catch (Exception ex)
+ {
+ logger.warn("Exception occurred updating coordinatorWriteLatency metric", ex);
+ }
}
- private static void syncWriteToBatchlog(Collection<Mutation> mutations, BatchlogEndpoints endpoints, UUID uuid, long queryStartNanoTime)
+ private static void syncWriteToBatchlog(Collection<Mutation> mutations, ReplicaPlan.ForTokenWrite replicaPlan, UUID uuid, long queryStartNanoTime)
throws WriteTimeoutException, WriteFailureException
{
- WriteResponseHandler<?> handler = new WriteResponseHandler<>(endpoints.all,
- Collections.<InetAddress>emptyList(),
- endpoints.all.size() == 1 ? ConsistencyLevel.ONE : ConsistencyLevel.TWO,
- Keyspace.open(SchemaConstants.SYSTEM_KEYSPACE_NAME),
- null,
- WriteType.BATCH_LOG,
- queryStartNanoTime);
+ WriteResponseHandler<?> handler = new WriteResponseHandler(replicaPlan,
+ WriteType.BATCH_LOG,
+ queryStartNanoTime);
Batch batch = Batch.createLocal(uuid, FBUtilities.timestampMicros(), mutations);
+ Message<Batch> message = Message.out(BATCH_STORE_REQ, batch);
+ for (Replica replica : replicaPlan.liveAndDown())
+ {
+ logger.trace("Sending batchlog store request {} to {} for {} mutations", batch.id, replica, batch.size());
- if (!endpoints.current.isEmpty())
- syncWriteToBatchlog(handler, batch, endpoints.current);
-
- if (!endpoints.legacy.isEmpty())
- LegacyBatchlogMigrator.syncWriteToBatchlog(handler, batch, endpoints.legacy);
-
+ if (replica.isSelf())
+ performLocally(Stage.MUTATION, replica, () -> BatchlogManager.store(batch), handler);
+ else
+ MessagingService.instance().sendWithCallback(message, replica.endpoint(), handler);
+ }
handler.get();
}
- private static void syncWriteToBatchlog(WriteResponseHandler<?> handler, Batch batch, Collection<InetAddress> endpoints)
- throws WriteTimeoutException, WriteFailureException
+ private static void asyncRemoveFromBatchlog(ReplicaPlan.ForTokenWrite replicaPlan, UUID uuid)
{
- MessageOut<Batch> message = new MessageOut<>(MessagingService.Verb.BATCH_STORE, batch, Batch.serializer);
-
- for (InetAddress target : endpoints)
- {
- logger.trace("Sending batchlog store request {} to {} for {} mutations", batch.id, target, batch.size());
-
- if (canDoLocalRequest(target))
- performLocally(Stage.MUTATION, Optional.empty(), () -> BatchlogManager.store(batch), handler);
- else
- MessagingService.instance().sendRR(message, target, handler);
- }
- }
-
- private static void asyncRemoveFromBatchlog(BatchlogEndpoints endpoints, UUID uuid, long queryStartNanoTime)
- {
- if (!endpoints.current.isEmpty())
- asyncRemoveFromBatchlog(endpoints.current, uuid);
-
- if (!endpoints.legacy.isEmpty())
- LegacyBatchlogMigrator.asyncRemoveFromBatchlog(endpoints.legacy, uuid, queryStartNanoTime);
- }
-
- private static void asyncRemoveFromBatchlog(Collection<InetAddress> endpoints, UUID uuid)
- {
- MessageOut<UUID> message = new MessageOut<>(MessagingService.Verb.BATCH_REMOVE, uuid, UUIDSerializer.serializer);
- for (InetAddress target : endpoints)
+ Message<UUID> message = Message.out(Verb.BATCH_REMOVE_REQ, uuid);
+ for (Replica target : replicaPlan.contacts())
{
if (logger.isTraceEnabled())
logger.trace("Sending batchlog remove request {} to {}", uuid, target);
- if (canDoLocalRequest(target))
- performLocally(Stage.MUTATION, () -> BatchlogManager.remove(uuid));
+ if (target.isSelf())
+ performLocally(Stage.MUTATION, target, () -> BatchlogManager.remove(uuid));
else
- MessagingService.instance().sendOneWay(message, target);
+ MessagingService.instance().send(message, target.endpoint());
}
}
@@ -1027,29 +1118,32 @@
{
for (WriteResponseHandlerWrapper wrapper : wrappers)
{
- Iterable<InetAddress> endpoints = Iterables.concat(wrapper.handler.naturalEndpoints, wrapper.handler.pendingEndpoints);
+ Replicas.temporaryAssertFull(wrapper.handler.replicaPlan.liveAndDown()); // TODO: CASSANDRA-14549
+ ReplicaPlan.ForTokenWrite replicas = wrapper.handler.replicaPlan.withContact(wrapper.handler.replicaPlan.liveAndDown());
try
{
- sendToHintedEndpoints(wrapper.mutation, endpoints, wrapper.handler, localDataCenter, stage);
+ sendToHintedReplicas(wrapper.mutation, replicas, wrapper.handler, localDataCenter, stage);
}
catch (OverloadedException | WriteTimeoutException e)
{
- wrapper.handler.onFailure(FBUtilities.getBroadcastAddress(), RequestFailureReason.UNKNOWN);
+ wrapper.handler.onFailure(FBUtilities.getBroadcastAddressAndPort(), RequestFailureReason.forException(e));
}
}
}
- private static void syncWriteBatchedMutations(List<WriteResponseHandlerWrapper> wrappers, String localDataCenter, Stage stage)
+ private static void syncWriteBatchedMutations(List<WriteResponseHandlerWrapper> wrappers, Stage stage)
throws WriteTimeoutException, OverloadedException
{
+ String localDataCenter = DatabaseDescriptor.getEndpointSnitch().getLocalDatacenter();
+
for (WriteResponseHandlerWrapper wrapper : wrappers)
{
- Iterable<InetAddress> endpoints = Iterables.concat(wrapper.handler.naturalEndpoints, wrapper.handler.pendingEndpoints);
- sendToHintedEndpoints(wrapper.mutation, endpoints, wrapper.handler, localDataCenter, stage);
+ EndpointsForToken sendTo = wrapper.handler.replicaPlan.liveAndDown();
+ Replicas.temporaryAssertFull(sendTo); // TODO: CASSANDRA-14549
+ sendToHintedReplicas(wrapper.mutation, wrapper.handler.replicaPlan.withContact(sendTo), wrapper.handler, localDataCenter, stage);
}
-
for (WriteResponseHandlerWrapper wrapper : wrappers)
wrapper.handler.get();
}
@@ -1061,7 +1155,7 @@
* responses based on consistency level.
*
* @param mutation the mutation to be applied
- * @param consistency_level the consistency level for the write operation
+ * @param consistencyLevel the consistency level for the write operation
* @param performer the WritePerformer in charge of appliying the mutation
* given the list of write endpoints (either standardWritePerformer for
* standard writes or counterWritePerformer for counter writes).
@@ -1069,33 +1163,29 @@
* @param queryStartNanoTime the value of System.nanoTime() when the query started to be processed
*/
public static AbstractWriteResponseHandler<IMutation> performWrite(IMutation mutation,
- ConsistencyLevel consistency_level,
+ ConsistencyLevel consistencyLevel,
String localDataCenter,
WritePerformer performer,
Runnable callback,
WriteType writeType,
long queryStartNanoTime)
- throws UnavailableException, OverloadedException
{
String keyspaceName = mutation.getKeyspaceName();
- AbstractReplicationStrategy rs = Keyspace.open(keyspaceName).getReplicationStrategy();
+ Keyspace keyspace = Keyspace.open(keyspaceName);
+ AbstractReplicationStrategy rs = keyspace.getReplicationStrategy();
Token tk = mutation.key().getToken();
- List<InetAddress> naturalEndpoints = StorageService.instance.getNaturalEndpoints(keyspaceName, tk);
- Collection<InetAddress> pendingEndpoints = StorageService.instance.getTokenMetadata().pendingEndpointsFor(tk, keyspaceName);
- AbstractWriteResponseHandler<IMutation> responseHandler = rs.getWriteResponseHandler(naturalEndpoints, pendingEndpoints, consistency_level, callback, writeType, queryStartNanoTime);
+ ReplicaPlan.ForTokenWrite replicaPlan = ReplicaPlans.forWrite(keyspace, consistencyLevel, tk, ReplicaPlans.writeNormal);
+ AbstractWriteResponseHandler<IMutation> responseHandler = rs.getWriteResponseHandler(replicaPlan, callback, writeType, queryStartNanoTime);
- // exit early if we can't fulfill the CL at this time
- responseHandler.assureSufficientLiveNodes();
-
- performer.apply(mutation, Iterables.concat(naturalEndpoints, pendingEndpoints), responseHandler, localDataCenter, consistency_level);
+ performer.apply(mutation, replicaPlan, responseHandler, localDataCenter);
return responseHandler;
}
// same as performWrites except does not initiate writes (but does perform availability checks).
private static WriteResponseHandlerWrapper wrapBatchResponseHandler(Mutation mutation,
- ConsistencyLevel consistency_level,
+ ConsistencyLevel consistencyLevel,
ConsistencyLevel batchConsistencyLevel,
WriteType writeType,
BatchlogResponseHandler.BatchlogCleanup cleanup,
@@ -1103,11 +1193,10 @@
{
Keyspace keyspace = Keyspace.open(mutation.getKeyspaceName());
AbstractReplicationStrategy rs = keyspace.getReplicationStrategy();
- String keyspaceName = mutation.getKeyspaceName();
Token tk = mutation.key().getToken();
- List<InetAddress> naturalEndpoints = StorageService.instance.getNaturalEndpoints(keyspaceName, tk);
- Collection<InetAddress> pendingEndpoints = StorageService.instance.getTokenMetadata().pendingEndpointsFor(tk, keyspaceName);
- AbstractWriteResponseHandler<IMutation> writeHandler = rs.getWriteResponseHandler(naturalEndpoints, pendingEndpoints, consistency_level, null, writeType, queryStartNanoTime);
+
+ ReplicaPlan.ForTokenWrite replicaPlan = ReplicaPlans.forWrite(keyspace, consistencyLevel, tk, ReplicaPlans.writeNormal);
+ AbstractWriteResponseHandler<IMutation> writeHandler = rs.getWriteResponseHandler(replicaPlan,null, writeType, queryStartNanoTime);
BatchlogResponseHandler<IMutation> batchHandler = new BatchlogResponseHandler<>(writeHandler, batchConsistencyLevel.blockFor(keyspace), cleanup, queryStartNanoTime);
return new WriteResponseHandlerWrapper(batchHandler, mutation);
}
@@ -1117,9 +1206,10 @@
* Keeps track of ViewWriteMetrics
*/
private static WriteResponseHandlerWrapper wrapViewBatchResponseHandler(Mutation mutation,
- ConsistencyLevel consistency_level,
+ ConsistencyLevel consistencyLevel,
ConsistencyLevel batchConsistencyLevel,
- List<InetAddress> naturalEndpoints,
+ EndpointsForToken naturalEndpoints,
+ EndpointsForToken pendingEndpoints,
AtomicLong baseComplete,
WriteType writeType,
BatchlogResponseHandler.BatchlogCleanup cleanup,
@@ -1127,12 +1217,13 @@
{
Keyspace keyspace = Keyspace.open(mutation.getKeyspaceName());
AbstractReplicationStrategy rs = keyspace.getReplicationStrategy();
- String keyspaceName = mutation.getKeyspaceName();
- Token tk = mutation.key().getToken();
- Collection<InetAddress> pendingEndpoints = StorageService.instance.getTokenMetadata().pendingEndpointsFor(tk, keyspaceName);
- AbstractWriteResponseHandler<IMutation> writeHandler = rs.getWriteResponseHandler(naturalEndpoints, pendingEndpoints, consistency_level, () -> {
+
+ ReplicaLayout.ForTokenWrite liveAndDown = ReplicaLayout.forTokenWrite(naturalEndpoints, pendingEndpoints);
+ ReplicaPlan.ForTokenWrite replicaPlan = ReplicaPlans.forWrite(keyspace, consistencyLevel, liveAndDown, ReplicaPlans.writeAll);
+
+ AbstractWriteResponseHandler<IMutation> writeHandler = rs.getWriteResponseHandler(replicaPlan, () -> {
long delay = Math.max(0, System.currentTimeMillis() - baseComplete.get());
- viewWriteMetrics.viewWriteLatency.update(delay, TimeUnit.MILLISECONDS);
+ viewWriteMetrics.viewWriteLatency.update(delay, MILLISECONDS);
}, writeType, queryStartNanoTime);
BatchlogResponseHandler<IMutation> batchHandler = new ViewWriteMetricsWrapped(writeHandler, batchConsistencyLevel.blockFor(keyspace), cleanup, queryStartNanoTime);
return new WriteResponseHandlerWrapper(batchHandler, mutation);
@@ -1151,57 +1242,6 @@
}
}
- /*
- * A class to filter batchlog endpoints into legacy endpoints (version < 3.0) or not.
- */
- private static final class BatchlogEndpoints
- {
- public final Collection<InetAddress> all;
- public final Collection<InetAddress> current;
- public final Collection<InetAddress> legacy;
-
- BatchlogEndpoints(Collection<InetAddress> endpoints)
- {
- all = endpoints;
- current = new ArrayList<>(2);
- legacy = new ArrayList<>(2);
-
- for (InetAddress ep : endpoints)
- {
- if (MessagingService.instance().getVersion(ep) >= MessagingService.VERSION_30)
- current.add(ep);
- else
- legacy.add(ep);
- }
- }
- }
-
- /*
- * Replicas are picked manually:
- * - replicas should be alive according to the failure detector
- * - replicas should be in the local datacenter
- * - choose min(2, number of qualifying candiates above)
- * - allow the local node to be the only replica only if it's a single-node DC
- */
- private static BatchlogEndpoints getBatchlogEndpoints(String localDataCenter, ConsistencyLevel consistencyLevel)
- throws UnavailableException
- {
- TokenMetadata.Topology topology = StorageService.instance.getTokenMetadata().cachedOnlyTokenMap().getTopology();
- Multimap<String, InetAddress> localEndpoints = HashMultimap.create(topology.getDatacenterRacks().get(localDataCenter));
- String localRack = DatabaseDescriptor.getEndpointSnitch().getRack(FBUtilities.getBroadcastAddress());
-
- Collection<InetAddress> chosenEndpoints = new BatchlogManager.EndpointFilter(localRack, localEndpoints).filter();
- if (chosenEndpoints.isEmpty())
- {
- if (consistencyLevel == ConsistencyLevel.ANY)
- return new BatchlogEndpoints(Collections.singleton(FBUtilities.getBroadcastAddress()));
-
- throw new UnavailableException(ConsistencyLevel.ONE, 1, 0);
- }
-
- return new BatchlogEndpoints(chosenEndpoints);
- }
-
/**
* Send the mutations to the right targets, write it locally if it corresponds or writes a hint when the node
* is not available.
@@ -1219,42 +1259,42 @@
*
* @throws OverloadedException if the hints cannot be written/enqueued
*/
- public static void sendToHintedEndpoints(final Mutation mutation,
- Iterable<InetAddress> targets,
- AbstractWriteResponseHandler<IMutation> responseHandler,
- String localDataCenter,
- Stage stage)
+ public static void sendToHintedReplicas(final Mutation mutation,
+ ReplicaPlan.ForTokenWrite plan,
+ AbstractWriteResponseHandler<IMutation> responseHandler,
+ String localDataCenter,
+ Stage stage)
throws OverloadedException
{
- int targetsSize = Iterables.size(targets);
-
// this dc replicas:
- Collection<InetAddress> localDc = null;
+ Collection<Replica> localDc = null;
// extra-datacenter replicas, grouped by dc
- Map<String, Collection<InetAddress>> dcGroups = null;
+ Map<String, Collection<Replica>> dcGroups = null;
// only need to create a Message for non-local writes
- MessageOut<Mutation> message = null;
+ Message<Mutation> message = null;
boolean insertLocal = false;
- ArrayList<InetAddress> endpointsToHint = null;
+ Replica localReplica = null;
+ Collection<Replica> endpointsToHint = null;
- List<InetAddress> backPressureHosts = null;
+ List<InetAddressAndPort> backPressureHosts = null;
- for (InetAddress destination : targets)
+ for (Replica destination : plan.contacts())
{
checkHintOverload(destination);
- if (FailureDetector.instance.isAlive(destination))
+ if (plan.isAlive(destination))
{
- if (canDoLocalRequest(destination))
+ if (destination.isSelf())
{
insertLocal = true;
+ localReplica = destination;
}
else
{
// belongs on a different server
if (message == null)
- message = mutation.createMessage();
+ message = Message.outWithFlag(MUTATION_REQ, mutation, MessageFlag.CALL_BACK_ON_FAILURE);
String dc = DatabaseDescriptor.getEndpointSnitch().getDatacenter(destination);
@@ -1263,36 +1303,36 @@
if (localDataCenter.equals(dc))
{
if (localDc == null)
- localDc = new ArrayList<>(targetsSize);
+ localDc = new ArrayList<>(plan.contacts().size());
localDc.add(destination);
}
else
{
- Collection<InetAddress> messages = (dcGroups != null) ? dcGroups.get(dc) : null;
+ if (dcGroups == null)
+ dcGroups = new HashMap<>();
+
+ Collection<Replica> messages = dcGroups.get(dc);
if (messages == null)
- {
- messages = new ArrayList<>(3); // most DCs will have <= 3 replicas
- if (dcGroups == null)
- dcGroups = new HashMap<>();
- dcGroups.put(dc, messages);
- }
+ messages = dcGroups.computeIfAbsent(dc, (v) -> new ArrayList<>(3)); // most DCs will have <= 3 replicas
messages.add(destination);
}
if (backPressureHosts == null)
- backPressureHosts = new ArrayList<>(targetsSize);
+ backPressureHosts = new ArrayList<>(plan.contacts().size());
- backPressureHosts.add(destination);
+ backPressureHosts.add(destination.endpoint());
}
}
else
{
+ //Immediately mark the response as expired since the request will not be sent
+ responseHandler.expired();
if (shouldHint(destination))
{
if (endpointsToHint == null)
- endpointsToHint = new ArrayList<>(targetsSize);
+ endpointsToHint = new ArrayList<>();
endpointsToHint.add(destination);
}
@@ -1300,28 +1340,31 @@
}
if (backPressureHosts != null)
- MessagingService.instance().applyBackPressure(backPressureHosts, responseHandler.currentTimeout());
+ MessagingService.instance().applyBackPressure(backPressureHosts, responseHandler.currentTimeoutNanos());
if (endpointsToHint != null)
- submitHint(mutation, endpointsToHint, responseHandler);
+ submitHint(mutation, EndpointsForToken.copyOf(mutation.key().getToken(), endpointsToHint), responseHandler);
if (insertLocal)
- performLocally(stage, Optional.of(mutation), mutation::apply, responseHandler);
+ {
+ Preconditions.checkNotNull(localReplica);
+ performLocally(stage, localReplica, mutation::apply, responseHandler);
+ }
if (localDc != null)
{
- for (InetAddress destination : localDc)
- MessagingService.instance().sendRR(message, destination, responseHandler, true);
+ for (Replica destination : localDc)
+ MessagingService.instance().sendWriteWithCallback(message, destination, responseHandler, true);
}
if (dcGroups != null)
{
// for each datacenter, send the message to one node to relay the write to other replicas
- for (Collection<InetAddress> dcTargets : dcGroups.values())
- sendMessagesToNonlocalDC(message, dcTargets, responseHandler);
+ for (Collection<Replica> dcTargets : dcGroups.values())
+ sendMessagesToNonlocalDC(message, EndpointsForToken.copyOf(mutation.key().getToken(), dcTargets), responseHandler);
}
}
- private static void checkHintOverload(InetAddress destination)
+ private static void checkHintOverload(Replica destination)
{
// avoid OOMing due to excess hints. we need to do this check even for "live" nodes, since we can
// still generate hints for those if it's overloaded or simply dead but not yet known-to-be-dead.
@@ -1329,53 +1372,52 @@
// a small number of nodes causing problems, so we should avoid shutting down writes completely to
// healthy nodes. Any node with no hintsInProgress is considered healthy.
if (StorageMetrics.totalHintsInProgress.getCount() > maxHintsInProgress
- && (getHintsInProgressFor(destination).get() > 0 && shouldHint(destination)))
+ && (getHintsInProgressFor(destination.endpoint()).get() > 0 && shouldHint(destination)))
{
throw new OverloadedException("Too many in flight hints: " + StorageMetrics.totalHintsInProgress.getCount() +
" destination: " + destination +
- " destination hints: " + getHintsInProgressFor(destination).get());
+ " destination hints: " + getHintsInProgressFor(destination.endpoint()).get());
}
}
- private static void sendMessagesToNonlocalDC(MessageOut<? extends IMutation> message,
- Collection<InetAddress> targets,
+ /*
+ * Send the message to the first replica of targets, and have it forward the message to others in its DC
+ */
+ private static void sendMessagesToNonlocalDC(Message<? extends IMutation> message,
+ EndpointsForToken targets,
AbstractWriteResponseHandler<IMutation> handler)
{
- Iterator<InetAddress> iter = targets.iterator();
- InetAddress target = iter.next();
+ final Replica target;
- // Add the other destinations of the same message as a FORWARD_HEADER entry
- try(DataOutputBuffer out = new DataOutputBuffer())
+ if (targets.size() > 1)
{
- out.writeInt(targets.size() - 1);
- while (iter.hasNext())
+ target = targets.get(ThreadLocalRandom.current().nextInt(0, targets.size()));
+ EndpointsForToken forwardToReplicas = targets.filter(r -> r != target, targets.size());
+
+ for (Replica replica : forwardToReplicas)
{
- InetAddress destination = iter.next();
- CompactEndpointSerializationHelper.serialize(destination, out);
- int id = MessagingService.instance().addCallback(handler,
- message,
- destination,
- message.getTimeout(),
- handler.consistencyLevel,
- true);
- out.writeInt(id);
- logger.trace("Adding FWD message to {}@{}", id, destination);
+ MessagingService.instance().callbacks.addWithExpiration(handler, message, replica, handler.replicaPlan.consistencyLevel(), true);
+ logger.trace("Adding FWD message to {}@{}", message.id(), replica);
}
- message = message.withParameter(Mutation.FORWARD_TO, out.getData());
- // send the combined message + forward headers
- int id = MessagingService.instance().sendRR(message, target, handler, true);
- logger.trace("Sending message to {}@{}", id, target);
+
+ // starting with 4.0, use the same message id for all replicas
+ long[] messageIds = new long[forwardToReplicas.size()];
+ Arrays.fill(messageIds, message.id());
+
+ message = message.withForwardTo(new ForwardingInfo(forwardToReplicas.endpointList(), messageIds));
}
- catch (IOException e)
+ else
{
- // DataOutputBuffer is in-memory, doesn't throw IOException
- throw new AssertionError(e);
+ target = targets.get(0);
}
+
+ MessagingService.instance().sendWriteWithCallback(message, target, handler, true);
+ logger.trace("Sending message to {}@{}", message.id(), target);
}
- private static void performLocally(Stage stage, final Runnable runnable)
+ private static void performLocally(Stage stage, Replica localReplica, final Runnable runnable)
{
- StageManager.getStage(stage).maybeExecuteImmediately(new LocalMutationRunnable()
+ stage.maybeExecuteImmediately(new LocalMutationRunnable(localReplica)
{
public void runMayThrow()
{
@@ -1385,41 +1427,41 @@
}
catch (Exception ex)
{
- logger.error("Failed to apply mutation locally : {}", ex);
+ logger.error("Failed to apply mutation locally : ", ex);
}
}
@Override
protected Verb verb()
{
- return MessagingService.Verb.MUTATION;
+ return Verb.MUTATION_REQ;
}
});
}
- private static void performLocally(Stage stage, Optional<IMutation> mutation, final Runnable runnable, final IAsyncCallbackWithFailure<?> handler)
+ private static void performLocally(Stage stage, Replica localReplica, final Runnable runnable, final RequestCallback<?> handler)
{
- StageManager.getStage(stage).maybeExecuteImmediately(new LocalMutationRunnable(mutation)
+ stage.maybeExecuteImmediately(new LocalMutationRunnable(localReplica)
{
public void runMayThrow()
{
try
{
runnable.run();
- handler.response(null);
+ handler.onResponse(null);
}
catch (Exception ex)
{
if (!(ex instanceof WriteTimeoutException))
- logger.error("Failed to apply mutation locally : {}", ex);
- handler.onFailure(FBUtilities.getBroadcastAddress(), RequestFailureReason.UNKNOWN);
+ logger.error("Failed to apply mutation locally : ", ex);
+ handler.onFailure(FBUtilities.getBroadcastAddressAndPort(), RequestFailureReason.forException(ex));
}
}
@Override
protected Verb verb()
{
- return MessagingService.Verb.MUTATION;
+ return Verb.MUTATION_REQ;
}
});
}
@@ -1440,9 +1482,9 @@
*/
public static AbstractWriteResponseHandler<IMutation> mutateCounter(CounterMutation cm, String localDataCenter, long queryStartNanoTime) throws UnavailableException, OverloadedException
{
- InetAddress endpoint = findSuitableEndpoint(cm.getKeyspaceName(), cm.key(), localDataCenter, cm.consistency());
+ Replica replica = findSuitableReplica(cm.getKeyspaceName(), cm.key(), localDataCenter, cm.consistency());
- if (endpoint.equals(FBUtilities.getBroadcastAddress()))
+ if (replica.isSelf())
{
return applyCounterMutationOnCoordinator(cm, localDataCenter, queryStartNanoTime);
}
@@ -1450,18 +1492,19 @@
{
// Exit now if we can't fulfill the CL here instead of forwarding to the leader replica
String keyspaceName = cm.getKeyspaceName();
- AbstractReplicationStrategy rs = Keyspace.open(keyspaceName).getReplicationStrategy();
+ Keyspace keyspace = Keyspace.open(keyspaceName);
Token tk = cm.key().getToken();
- List<InetAddress> naturalEndpoints = StorageService.instance.getNaturalEndpoints(keyspaceName, tk);
- Collection<InetAddress> pendingEndpoints = StorageService.instance.getTokenMetadata().pendingEndpointsFor(tk, keyspaceName);
- rs.getWriteResponseHandler(naturalEndpoints, pendingEndpoints, cm.consistency(), null, WriteType.COUNTER, queryStartNanoTime).assureSufficientLiveNodes();
+ // we build this ONLY to perform the sufficiency check that happens on construction
+ ReplicaPlans.forWrite(keyspace, cm.consistency(), tk, ReplicaPlans.writeAll);
// Forward the actual update to the chosen leader replica
- AbstractWriteResponseHandler<IMutation> responseHandler = new WriteResponseHandler<>(endpoint, WriteType.COUNTER, queryStartNanoTime);
+ AbstractWriteResponseHandler<IMutation> responseHandler = new WriteResponseHandler<>(ReplicaPlans.forForwardingCounterWrite(keyspace, tk, replica),
+ WriteType.COUNTER, queryStartNanoTime);
- Tracing.trace("Enqueuing counter update to {}", endpoint);
- MessagingService.instance().sendRR(cm.makeMutationMessage(), endpoint, responseHandler, false);
+ Tracing.trace("Enqueuing counter update to {}", replica);
+ Message message = Message.outWithFlag(Verb.COUNTER_MUTATION_REQ, cm, MessageFlag.CALL_BACK_ON_FAILURE);
+ MessagingService.instance().sendWriteWithCallback(message, replica, responseHandler, false);
return responseHandler;
}
}
@@ -1476,38 +1519,37 @@
* is unclear we want to mix those latencies with read latencies, so this
* may be a bit involved.
*/
- private static InetAddress findSuitableEndpoint(String keyspaceName, DecoratedKey key, String localDataCenter, ConsistencyLevel cl) throws UnavailableException
+ private static Replica findSuitableReplica(String keyspaceName, DecoratedKey key, String localDataCenter, ConsistencyLevel cl) throws UnavailableException
{
Keyspace keyspace = Keyspace.open(keyspaceName);
IEndpointSnitch snitch = DatabaseDescriptor.getEndpointSnitch();
- List<InetAddress> endpoints = new ArrayList<>();
- StorageService.instance.getLiveNaturalEndpoints(keyspace, key, endpoints);
+ EndpointsForToken replicas = keyspace.getReplicationStrategy().getNaturalReplicasForToken(key);
// CASSANDRA-13043: filter out those endpoints not accepting clients yet, maybe because still bootstrapping
- endpoints.removeIf(endpoint -> !StorageService.instance.isRpcReady(endpoint));
+ replicas = replicas.filter(replica -> StorageService.instance.isRpcReady(replica.endpoint()));
// TODO have a way to compute the consistency level
- if (endpoints.isEmpty())
- throw new UnavailableException(cl, cl.blockFor(keyspace), 0);
+ if (replicas.isEmpty())
+ throw UnavailableException.create(cl, cl.blockFor(keyspace), 0);
- List<InetAddress> localEndpoints = new ArrayList<>(endpoints.size());
+ List<Replica> localReplicas = new ArrayList<>(replicas.size());
- for (InetAddress endpoint : endpoints)
- if (snitch.getDatacenter(endpoint).equals(localDataCenter))
- localEndpoints.add(endpoint);
+ for (Replica replica : replicas)
+ if (snitch.getDatacenter(replica).equals(localDataCenter))
+ localReplicas.add(replica);
- if (localEndpoints.isEmpty())
+ if (localReplicas.isEmpty())
{
// If the consistency required is local then we should not involve other DCs
if (cl.isDatacenterLocal())
- throw new UnavailableException(cl, cl.blockFor(keyspace), 0);
+ throw UnavailableException.create(cl, cl.blockFor(keyspace), 0);
// No endpoint in local DC, pick the closest endpoint according to the snitch
- snitch.sortByProximity(FBUtilities.getBroadcastAddress(), endpoints);
- return endpoints.get(0);
+ replicas = snitch.sortedByProximity(FBUtilities.getBroadcastAddressAndPort(), replicas);
+ return replicas.get(0);
}
- return localEndpoints.get(ThreadLocalRandom.current().nextInt(localEndpoints.size()));
+ return localReplicas.get(ThreadLocalRandom.current().nextInt(localReplicas.size()));
}
// Must be called on a replica of the mutation. This replica becomes the
@@ -1527,11 +1569,11 @@
}
private static Runnable counterWriteTask(final IMutation mutation,
- final Iterable<InetAddress> targets,
+ final ReplicaPlan.ForTokenWrite replicaPlan,
final AbstractWriteResponseHandler<IMutation> responseHandler,
final String localDataCenter)
{
- return new DroppableRunnable(MessagingService.Verb.COUNTER_MUTATION)
+ return new DroppableRunnable(Verb.COUNTER_MUTATION_REQ)
{
@Override
public void runMayThrow() throws OverloadedException, WriteTimeoutException
@@ -1539,12 +1581,8 @@
assert mutation instanceof CounterMutation;
Mutation result = ((CounterMutation) mutation).applyCounterMutation();
- responseHandler.response(null);
-
- Set<InetAddress> remotes = Sets.difference(ImmutableSet.copyOf(targets),
- ImmutableSet.of(FBUtilities.getBroadcastAddress()));
- if (!remotes.isEmpty())
- sendToHintedEndpoints(result, remotes, responseHandler, localDataCenter, Stage.COUNTER_MUTATION);
+ responseHandler.onResponse(null);
+ sendToHintedReplicas(result, replicaPlan, responseHandler, localDataCenter, Stage.COUNTER_MUTATION);
}
};
}
@@ -1552,7 +1590,7 @@
private static boolean systemKeyspaceQuery(List<? extends ReadCommand> cmds)
{
for (ReadCommand cmd : cmds)
- if (!SchemaConstants.isLocalSystemKeyspace(cmd.metadata().ksName))
+ if (!SchemaConstants.isLocalSystemKeyspace(cmd.metadata().keyspace))
return false;
return true;
}
@@ -1584,7 +1622,7 @@
public static PartitionIterator read(SinglePartitionReadCommand.Group group, ConsistencyLevel consistencyLevel, ClientState state, long queryStartNanoTime)
throws UnavailableException, IsBootstrappingException, ReadFailureException, ReadTimeoutException, InvalidRequestException
{
- if (StorageService.instance.isBootstrapMode() && !systemKeyspaceQuery(group.commands))
+ if (StorageService.instance.isBootstrapMode() && !systemKeyspaceQuery(group.queries))
{
readMetrics.unavailables.mark();
readMetricsMap.get(consistencyLevel).unavailables.mark();
@@ -1600,21 +1638,19 @@
throws InvalidRequestException, UnavailableException, ReadFailureException, ReadTimeoutException
{
assert state != null;
- if (group.commands.size() > 1)
+ if (group.queries.size() > 1)
throw new InvalidRequestException("SERIAL/LOCAL_SERIAL consistency may only be requested for one partition at a time");
long start = System.nanoTime();
- SinglePartitionReadCommand command = group.commands.get(0);
- CFMetaData metadata = command.metadata();
+ SinglePartitionReadCommand command = group.queries.get(0);
+ TableMetadata metadata = command.metadata();
DecoratedKey key = command.partitionKey();
PartitionIterator result = null;
try
{
// make sure any in-progress paxos writes are done (i.e., committed to a majority of replicas), before performing a quorum read
- Pair<List<InetAddress>, Integer> p = getPaxosParticipants(metadata, key, consistencyLevel);
- List<InetAddress> liveEndpoints = p.left;
- int requiredParticipants = p.right;
+ ReplicaPlan.ForPaxosWrite replicaPlan = ReplicaPlans.forPaxos(Keyspace.open(metadata.keyspace), key, consistencyLevel);
// does the work of applying in-progress writes; throws UAE or timeout if it can't
final ConsistencyLevel consistencyForCommitOrFetch = consistencyLevel == ConsistencyLevel.LOCAL_SERIAL
@@ -1623,20 +1659,20 @@
try
{
- final Pair<UUID, Integer> pair = beginAndRepairPaxos(start, key, metadata, liveEndpoints, requiredParticipants, consistencyLevel, consistencyForCommitOrFetch, false, state);
- if (pair.right > 0)
- casReadMetrics.contention.update(pair.right);
+ final PaxosBallotAndContention pair = beginAndRepairPaxos(start, key, metadata, replicaPlan, consistencyLevel, consistencyForCommitOrFetch, false, state);
+ if (pair.contentions > 0)
+ casReadMetrics.contention.update(pair.contentions);
}
catch (WriteTimeoutException e)
{
- throw new ReadTimeoutException(consistencyLevel, 0, consistencyLevel.blockFor(Keyspace.open(metadata.ksName)), false);
+ throw new ReadTimeoutException(consistencyLevel, 0, consistencyLevel.blockFor(Keyspace.open(metadata.keyspace)), false);
}
catch (WriteFailureException e)
{
throw new ReadFailureException(consistencyLevel, e.received, e.blockFor, false, e.failureReasonByEndpoint);
}
- result = fetchRows(group.commands, consistencyForCommitOrFetch, queryStartNanoTime);
+ result = fetchRows(group.queries, consistencyForCommitOrFetch, queryStartNanoTime);
}
catch (UnavailableException e)
{
@@ -1665,7 +1701,7 @@
readMetrics.addNano(latency);
casReadMetrics.addNano(latency);
readMetricsMap.get(consistencyLevel).addNano(latency);
- Keyspace.open(metadata.ksName).getColumnFamilyStore(metadata.cfName).metric.coordinatorReadLatency.update(latency, TimeUnit.NANOSECONDS);
+ Keyspace.open(metadata.keyspace).getColumnFamilyStore(metadata.name).metric.coordinatorReadLatency.update(latency, TimeUnit.NANOSECONDS);
}
return result;
@@ -1678,13 +1714,13 @@
long start = System.nanoTime();
try
{
- PartitionIterator result = fetchRows(group.commands, consistencyLevel, queryStartNanoTime);
+ PartitionIterator result = fetchRows(group.queries, consistencyLevel, queryStartNanoTime);
// Note that the only difference between the command in a group must be the partition key on which
// they applied.
- boolean enforceStrictLiveness = group.commands.get(0).metadata().enforceStrictLiveness();
+ boolean enforceStrictLiveness = group.queries.get(0).metadata().enforceStrictLiveness();
// If we have more than one command, then despite each read command honoring the limit, the total result
// might not honor it and so we should enforce it
- if (group.commands.size() > 1)
+ if (group.queries.size() > 1)
result = group.limits().filter(result, group.nowInSec(), group.selectsFullPartition(), enforceStrictLiveness);
return result;
}
@@ -1712,11 +1748,39 @@
readMetrics.addNano(latency);
readMetricsMap.get(consistencyLevel).addNano(latency);
// TODO avoid giving every command the same latency number. Can fix this in CASSADRA-5329
- for (ReadCommand command : group.commands)
+ for (ReadCommand command : group.queries)
Keyspace.openAndGetStore(command.metadata()).metric.coordinatorReadLatency.update(latency, TimeUnit.NANOSECONDS);
}
}
+ private static PartitionIterator concatAndBlockOnRepair(List<PartitionIterator> iterators, List<ReadRepair> repairs)
+ {
+ PartitionIterator concatenated = PartitionIterators.concat(iterators);
+
+ if (repairs.isEmpty())
+ return concatenated;
+
+ return new PartitionIterator()
+ {
+ public void close()
+ {
+ concatenated.close();
+ repairs.forEach(ReadRepair::maybeSendAdditionalWrites);
+ repairs.forEach(ReadRepair::awaitWrites);
+ }
+
+ public boolean hasNext()
+ {
+ return concatenated.hasNext();
+ }
+
+ public RowIterator next()
+ {
+ return concatenated.next();
+ }
+ };
+ }
+
/**
* This function executes local and remote reads, and blocks for the results:
*
@@ -1733,142 +1797,71 @@
{
int cmdCount = commands.size();
- SinglePartitionReadLifecycle[] reads = new SinglePartitionReadLifecycle[cmdCount];
- for (int i = 0; i < cmdCount; i++)
- reads[i] = new SinglePartitionReadLifecycle(commands.get(i), consistencyLevel, queryStartNanoTime);
+ AbstractReadExecutor[] reads = new AbstractReadExecutor[cmdCount];
- for (int i = 0; i < cmdCount; i++)
- reads[i].doInitialQueries();
+ // Get the replica locations, sorted by response time according to the snitch, and create a read executor
+ // for type of speculation we'll use in this read
+ for (int i=0; i<cmdCount; i++)
+ {
+ reads[i] = AbstractReadExecutor.getReadExecutor(commands.get(i), consistencyLevel, queryStartNanoTime);
+ }
- for (int i = 0; i < cmdCount; i++)
+ // sends a data request to the closest replica, and a digest request to the others. If we have a speculating
+ // read executoe, we'll only send read requests to enough replicas to satisfy the consistency level
+ for (int i=0; i<cmdCount; i++)
+ {
+ reads[i].executeAsync();
+ }
+
+ // if we have a speculating read executor and it looks like we may not receive a response from the initial
+ // set of replicas we sent messages to, speculatively send an additional messages to an un-contacted replica
+ for (int i=0; i<cmdCount; i++)
+ {
reads[i].maybeTryAdditionalReplicas();
+ }
- for (int i = 0; i < cmdCount; i++)
- reads[i].awaitResultsAndRetryOnDigestMismatch();
+ // wait for enough responses to meet the consistency level. If there's a digest mismatch, begin the read
+ // repair process by sending full data reads to all replicas we received responses from.
+ for (int i=0; i<cmdCount; i++)
+ {
+ reads[i].awaitResponses();
+ }
- for (int i = 0; i < cmdCount; i++)
- if (!reads[i].isDone())
- reads[i].maybeAwaitFullDataRead();
+ // read repair - if it looks like we may not receive enough full data responses to meet CL, send
+ // an additional request to any remaining replicas we haven't contacted (if there are any)
+ for (int i=0; i<cmdCount; i++)
+ {
+ reads[i].maybeSendAdditionalDataRequests();
+ }
+ // read repair - block on full data responses
+ for (int i=0; i<cmdCount; i++)
+ {
+ reads[i].awaitReadRepair();
+ }
+
+ // if we didn't do a read repair, return the contents of the data response, if we did do a read
+ // repair, merge the full data reads
List<PartitionIterator> results = new ArrayList<>(cmdCount);
- for (int i = 0; i < cmdCount; i++)
+ List<ReadRepair> repairs = new ArrayList<>(cmdCount);
+ for (int i=0; i<cmdCount; i++)
{
- assert reads[i].isDone();
results.add(reads[i].getResult());
+ repairs.add(reads[i].getReadRepair());
}
- return PartitionIterators.concat(results);
+ // if we did a read repair, assemble repair mutation and block on them
+ return concatAndBlockOnRepair(results, repairs);
}
- private static class SinglePartitionReadLifecycle
- {
- private final SinglePartitionReadCommand command;
- private final AbstractReadExecutor executor;
- private final ConsistencyLevel consistency;
- private final long queryStartNanoTime;
-
- private PartitionIterator result;
- private ReadCallback repairHandler;
-
- SinglePartitionReadLifecycle(SinglePartitionReadCommand command, ConsistencyLevel consistency, long queryStartNanoTime)
- {
- this.command = command;
- this.executor = AbstractReadExecutor.getReadExecutor(command, consistency, queryStartNanoTime);
- this.consistency = consistency;
- this.queryStartNanoTime = queryStartNanoTime;
- }
-
- boolean isDone()
- {
- return result != null;
- }
-
- void doInitialQueries()
- {
- executor.executeAsync();
- }
-
- void maybeTryAdditionalReplicas()
- {
- executor.maybeTryAdditionalReplicas();
- }
-
- void awaitResultsAndRetryOnDigestMismatch() throws ReadFailureException, ReadTimeoutException
- {
- try
- {
- result = executor.get();
- }
- catch (DigestMismatchException ex)
- {
- Tracing.trace("Digest mismatch: {}", ex);
-
- ReadRepairMetrics.repairedBlocking.mark();
-
- // Do a full data read to resolve the correct response (and repair node that need be)
- Keyspace keyspace = Keyspace.open(command.metadata().ksName);
- DataResolver resolver = new DataResolver(keyspace, command, ConsistencyLevel.ALL, executor.handler.endpoints.size(), queryStartNanoTime);
- repairHandler = new ReadCallback(resolver,
- ConsistencyLevel.ALL,
- executor.getContactedReplicas().size(),
- command,
- keyspace,
- executor.handler.endpoints,
- queryStartNanoTime);
-
- for (InetAddress endpoint : executor.getContactedReplicas())
- {
- MessageOut<ReadCommand> message = command.createMessage(MessagingService.instance().getVersion(endpoint));
- Tracing.trace("Enqueuing full data read to {}", endpoint);
- MessagingService.instance().sendRRWithFailure(message, endpoint, repairHandler);
- }
- }
- }
-
- void maybeAwaitFullDataRead() throws ReadTimeoutException
- {
- // There wasn't a digest mismatch, we're good
- if (repairHandler == null)
- return;
-
- // Otherwise, get the result from the full-data read and check that it's not a short read
- try
- {
- result = repairHandler.get();
- }
- catch (DigestMismatchException e)
- {
- throw new AssertionError(e); // full data requested from each node here, no digests should be sent
- }
- catch (ReadTimeoutException e)
- {
- if (Tracing.isTracing())
- Tracing.trace("Timed out waiting on digest mismatch repair requests");
- else
- logger.trace("Timed out waiting on digest mismatch repair requests");
- // the caught exception here will have CL.ALL from the repair command,
- // not whatever CL the initial command was at (CASSANDRA-7947)
- int blockFor = consistency.blockFor(Keyspace.open(command.metadata().ksName));
- throw new ReadTimeoutException(consistency, blockFor-1, blockFor, true);
- }
- }
-
- PartitionIterator getResult()
- {
- assert result != null;
- return result;
- }
- }
-
- static class LocalReadRunnable extends DroppableRunnable
+ public static class LocalReadRunnable extends DroppableRunnable
{
private final ReadCommand command;
private final ReadCallback handler;
- private final long start = System.nanoTime();
- LocalReadRunnable(ReadCommand command, ReadCallback handler)
+ public LocalReadRunnable(ReadCommand command, ReadCallback handler)
{
- super(MessagingService.Verb.READ);
+ super(Verb.READ_REQ);
this.command = command;
this.handler = handler;
}
@@ -1877,7 +1870,7 @@
{
try
{
- command.setMonitoringTime(constructionTime, false, verb.getTimeout(), DatabaseDescriptor.getSlowQueryTimeout());
+ command.setMonitoringTime(approxCreationTimeNanos, false, verb.expiresAfterNanos(), DatabaseDescriptor.getSlowQueryTimeout(NANOSECONDS));
ReadResponse response;
try (ReadExecutionController executionController = command.executionController();
@@ -1892,96 +1885,48 @@
}
else
{
- MessagingService.instance().incrementDroppedMessages(verb, System.currentTimeMillis() - constructionTime);
- handler.onFailure(FBUtilities.getBroadcastAddress(), RequestFailureReason.UNKNOWN);
+ MessagingService.instance().metrics.recordSelfDroppedMessage(verb, MonotonicClock.approxTime.now() - approxCreationTimeNanos, NANOSECONDS);
+ handler.onFailure(FBUtilities.getBroadcastAddressAndPort(), RequestFailureReason.UNKNOWN);
}
- MessagingService.instance().addLatency(FBUtilities.getBroadcastAddress(), TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start));
+ MessagingService.instance().latencySubscribers.add(FBUtilities.getBroadcastAddressAndPort(), MonotonicClock.approxTime.now() - approxCreationTimeNanos, NANOSECONDS);
}
catch (Throwable t)
{
if (t instanceof TombstoneOverwhelmingException)
{
- handler.onFailure(FBUtilities.getBroadcastAddress(), RequestFailureReason.READ_TOO_MANY_TOMBSTONES);
+ handler.onFailure(FBUtilities.getBroadcastAddressAndPort(), RequestFailureReason.READ_TOO_MANY_TOMBSTONES);
logger.error(t.getMessage());
}
else
{
- handler.onFailure(FBUtilities.getBroadcastAddress(), RequestFailureReason.UNKNOWN);
+ handler.onFailure(FBUtilities.getBroadcastAddressAndPort(), RequestFailureReason.UNKNOWN);
throw t;
}
}
}
}
- public static List<InetAddress> getLiveSortedEndpoints(Keyspace keyspace, ByteBuffer key)
- {
- return getLiveSortedEndpoints(keyspace, StorageService.instance.getTokenMetadata().decorateKey(key));
- }
-
- public static List<InetAddress> getLiveSortedEndpoints(Keyspace keyspace, RingPosition pos)
- {
- List<InetAddress> liveEndpoints = StorageService.instance.getLiveNaturalEndpoints(keyspace, pos);
- DatabaseDescriptor.getEndpointSnitch().sortByProximity(FBUtilities.getBroadcastAddress(), liveEndpoints);
- return liveEndpoints;
- }
-
- private static List<InetAddress> intersection(List<InetAddress> l1, List<InetAddress> l2)
- {
- // Note: we don't use Guava Sets.intersection() for 3 reasons:
- // 1) retainAll would be inefficient if l1 and l2 are large but in practice both are the replicas for a range and
- // so will be very small (< RF). In that case, retainAll is in fact more efficient.
- // 2) we do ultimately need a list so converting everything to sets don't make sense
- // 3) l1 and l2 are sorted by proximity. The use of retainAll maintain that sorting in the result, while using sets wouldn't.
- List<InetAddress> inter = new ArrayList<InetAddress>(l1);
- inter.retainAll(l2);
- return inter;
- }
-
/**
- * Estimate the number of result rows (either cql3 rows or "thrift" rows, as called for by the command) per
- * range in the ring based on our local data. This assumes that ranges are uniformly distributed across the cluster
- * and that the queried data is also uniformly distributed.
+ * Estimate the number of result rows per range in the ring based on our local data.
+ * <p>
+ * This assumes that ranges are uniformly distributed across the cluster and
+ * that the queried data is also uniformly distributed.
*/
private static float estimateResultsPerRange(PartitionRangeReadCommand command, Keyspace keyspace)
{
- ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(command.metadata().cfId);
+ ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(command.metadata().id);
Index index = command.getIndex(cfs);
float maxExpectedResults = index == null
? command.limits().estimateTotalResults(cfs)
: index.getEstimatedResultRows();
// adjust maxExpectedResults by the number of tokens this node has and the replication factor for this ks
- return (maxExpectedResults / DatabaseDescriptor.getNumTokens()) / keyspace.getReplicationStrategy().getReplicationFactor();
+ return (maxExpectedResults / DatabaseDescriptor.getNumTokens()) / keyspace.getReplicationStrategy().getReplicationFactor().allReplicas;
}
@VisibleForTesting
- public static class RangeForQuery
- {
- public final AbstractBounds<PartitionPosition> range;
- public final List<InetAddress> liveEndpoints;
- public final List<InetAddress> filteredEndpoints;
- public final int vnodeCount;
-
- public RangeForQuery(AbstractBounds<PartitionPosition> range,
- List<InetAddress> liveEndpoints,
- List<InetAddress> filteredEndpoints,
- int vnodeCount)
- {
- this.range = range;
- this.liveEndpoints = liveEndpoints;
- this.filteredEndpoints = filteredEndpoints;
- this.vnodeCount = vnodeCount;
- }
-
- public int vnodeCount()
- {
- return vnodeCount;
- }
- }
-
- @VisibleForTesting
- public static class RangeIterator extends AbstractIterator<RangeForQuery>
+ public static class RangeIterator extends AbstractIterator<ReplicaPlan.ForRangeRead>
{
private final Keyspace keyspace;
private final ConsistencyLevel consistency;
@@ -2005,40 +1950,34 @@
return rangeCount;
}
- protected RangeForQuery computeNext()
+ protected ReplicaPlan.ForRangeRead computeNext()
{
if (!ranges.hasNext())
return endOfData();
- AbstractBounds<PartitionPosition> range = ranges.next();
- List<InetAddress> liveEndpoints = getLiveSortedEndpoints(keyspace, range.right);
- return new RangeForQuery(range,
- liveEndpoints,
- consistency.filterForQuery(keyspace, liveEndpoints),
- 1);
+ return ReplicaPlans.forRangeRead(keyspace, consistency, ranges.next(), 1);
}
}
- @VisibleForTesting
- public static class RangeMerger extends AbstractIterator<RangeForQuery>
+ public static class RangeMerger extends AbstractIterator<ReplicaPlan.ForRangeRead>
{
private final Keyspace keyspace;
private final ConsistencyLevel consistency;
- private final PeekingIterator<RangeForQuery> ranges;
+ private final PeekingIterator<ReplicaPlan.ForRangeRead> ranges;
- public RangeMerger(Iterator<RangeForQuery> iterator, Keyspace keyspace, ConsistencyLevel consistency)
+ public RangeMerger(Iterator<ReplicaPlan.ForRangeRead> iterator, Keyspace keyspace, ConsistencyLevel consistency)
{
this.keyspace = keyspace;
this.consistency = consistency;
this.ranges = Iterators.peekingIterator(iterator);
}
- protected RangeForQuery computeNext()
+ protected ReplicaPlan.ForRangeRead computeNext()
{
if (!ranges.hasNext())
return endOfData();
- RangeForQuery current = ranges.next();
+ ReplicaPlan.ForRangeRead current = ranges.next();
// getRestrictedRange has broken the queried range into per-[vnode] token ranges, but this doesn't take
// the replication factor into account. If the intersection of live endpoints for 2 consecutive ranges
@@ -2050,26 +1989,15 @@
// Note: it would be slightly more efficient to have CFS.getRangeSlice on the destination nodes unwraps
// the range if necessary and deal with it. However, we can't start sending wrapped range without breaking
// wire compatibility, so It's likely easier not to bother;
- if (current.range.right.isMinimum())
+ if (current.range().right.isMinimum())
break;
- RangeForQuery next = ranges.peek();
-
- List<InetAddress> merged = intersection(current.liveEndpoints, next.liveEndpoints);
-
- // Check if there is enough endpoint for the merge to be possible.
- if (!consistency.isSufficientLiveNodes(keyspace, merged))
+ ReplicaPlan.ForRangeRead next = ranges.peek();
+ ReplicaPlan.ForRangeRead merged = ReplicaPlans.maybeMerge(keyspace, consistency, current, next);
+ if (merged == null)
break;
- List<InetAddress> filteredMerged = consistency.filterForQuery(keyspace, merged);
-
- // Estimate whether merging will be a win or not
- if (!DatabaseDescriptor.getEndpointSnitch().isWorthMergingForRangeQuery(filteredMerged, current.filteredEndpoints, next.filteredEndpoints))
- break;
-
- // If we get there, merge this range and the next one
- int vnodeCount = current.vnodeCount + next.vnodeCount;
- current = new RangeForQuery(current.range.withNewRight(next.range.right), merged, filteredMerged, vnodeCount);
+ current = merged;
ranges.next(); // consume the range we just merged since we've only peeked so far
}
return current;
@@ -2078,12 +2006,16 @@
private static class SingleRangeResponse extends AbstractIterator<RowIterator> implements PartitionIterator
{
+ private final DataResolver resolver;
private final ReadCallback handler;
+ private final ReadRepair readRepair;
private PartitionIterator result;
- private SingleRangeResponse(ReadCallback handler)
+ private SingleRangeResponse(DataResolver resolver, ReadCallback handler, ReadRepair readRepair)
{
+ this.resolver = resolver;
this.handler = handler;
+ this.readRepair = readRepair;
}
private void waitForResponse() throws ReadTimeoutException
@@ -2091,14 +2023,8 @@
if (result != null)
return;
- try
- {
- result = handler.get();
- }
- catch (DigestMismatchException e)
- {
- throw new AssertionError(e); // no digests in range slices yet
- }
+ handler.awaitResults();
+ result = resolver.resolve();
}
protected RowIterator computeNext()
@@ -2116,11 +2042,9 @@
public static class RangeCommandIterator extends AbstractIterator<RowIterator> implements PartitionIterator
{
- private final Iterator<RangeForQuery> ranges;
+ private final Iterator<ReplicaPlan.ForRangeRead> ranges;
private final int totalRangeCount;
private final PartitionRangeReadCommand command;
- private final Keyspace keyspace;
- private final ConsistencyLevel consistency;
private final boolean enforceStrictLiveness;
private final long startTime;
@@ -2136,13 +2060,11 @@
private int rangesQueried;
private int batchesRequested = 0;
- public RangeCommandIterator(Iterator<RangeForQuery> ranges,
+ public RangeCommandIterator(Iterator<ReplicaPlan.ForRangeRead> ranges,
PartitionRangeReadCommand command,
int concurrencyFactor,
int maxConcurrencyFactor,
int totalRangeCount,
- Keyspace keyspace,
- ConsistencyLevel consistency,
long queryStartNanoTime)
{
this.command = command;
@@ -2151,8 +2073,6 @@
this.startTime = System.nanoTime();
this.ranges = ranges;
this.totalRangeCount = totalRangeCount;
- this.consistency = consistency;
- this.keyspace = keyspace;
this.queryStartNanoTime = queryStartNanoTime;
this.enforceStrictLiveness = command.metadata().enforceStrictLiveness();
}
@@ -2170,6 +2090,7 @@
// else, sends the next batch of concurrent queries (after having close the previous iterator)
if (sentQueryIterator != null)
{
+ liveReturned += counter.counted();
sentQueryIterator.close();
// It's not the first batch of queries and we're not done, so we we can use what has been
@@ -2229,59 +2150,84 @@
/**
* Queries the provided sub-range.
*
- * @param toQuery the subRange to query.
+ * @param replicaPlan the subRange to query.
* @param isFirst in the case where multiple queries are sent in parallel, whether that's the first query on
* that batch or not. The reason it matters is that whe paging queries, the command (more specifically the
* {@code DataLimits}) may have "state" information and that state may only be valid for the first query (in
* that it's the query that "continues" whatever we're previously queried).
*/
- private SingleRangeResponse query(RangeForQuery toQuery, boolean isFirst)
+ private SingleRangeResponse query(ReplicaPlan.ForRangeRead replicaPlan, boolean isFirst)
{
- PartitionRangeReadCommand rangeCommand = command.forSubRange(toQuery.range, isFirst);
-
- DataResolver resolver = new DataResolver(keyspace, rangeCommand, consistency, toQuery.filteredEndpoints.size(), queryStartNanoTime);
-
- int blockFor = consistency.blockFor(keyspace);
- int minResponses = Math.min(toQuery.filteredEndpoints.size(), blockFor);
- List<InetAddress> minimalEndpoints = toQuery.filteredEndpoints.subList(0, minResponses);
- ReadCallback handler = new ReadCallback(resolver, consistency, rangeCommand, minimalEndpoints, queryStartNanoTime);
-
- handler.assureSufficientLiveNodes();
-
- if (toQuery.filteredEndpoints.size() == 1 && canDoLocalRequest(toQuery.filteredEndpoints.get(0)))
+ PartitionRangeReadCommand rangeCommand = command.forSubRange(replicaPlan.range(), isFirst);
+ // If enabled, request repaired data tracking info from full replicas but
+ // only if there are multiple full replicas to compare results from
+ if (DatabaseDescriptor.getRepairedDataTrackingForRangeReadsEnabled()
+ && replicaPlan.contacts().filter(Replica::isFull).size() > 1)
{
- StageManager.getStage(Stage.READ).execute(new LocalReadRunnable(rangeCommand, handler));
+ command.trackRepairedStatus();
+ rangeCommand.trackRepairedStatus();
+ }
+
+ ReplicaPlan.SharedForRangeRead sharedReplicaPlan = ReplicaPlan.shared(replicaPlan);
+ ReadRepair<EndpointsForRange, ReplicaPlan.ForRangeRead> readRepair
+ = ReadRepair.create(command, sharedReplicaPlan, queryStartNanoTime);
+ DataResolver<EndpointsForRange, ReplicaPlan.ForRangeRead> resolver
+ = new DataResolver<>(rangeCommand, sharedReplicaPlan, readRepair, queryStartNanoTime);
+ ReadCallback<EndpointsForRange, ReplicaPlan.ForRangeRead> handler
+ = new ReadCallback<>(resolver, rangeCommand, sharedReplicaPlan, queryStartNanoTime);
+
+
+ if (replicaPlan.contacts().size() == 1 && replicaPlan.contacts().get(0).isSelf())
+ {
+ Stage.READ.execute(new LocalReadRunnable(rangeCommand, handler));
}
else
{
- for (InetAddress endpoint : toQuery.filteredEndpoints)
+ for (Replica replica : replicaPlan.contacts())
{
- MessageOut<ReadCommand> message = rangeCommand.createMessage(MessagingService.instance().getVersion(endpoint));
- Tracing.trace("Enqueuing request to {}", endpoint);
- MessagingService.instance().sendRRWithFailure(message, endpoint, handler);
+ Tracing.trace("Enqueuing request to {}", replica);
+ ReadCommand command = replica.isFull() ? rangeCommand : rangeCommand.copyAsTransientQuery(replica);
+ Message<ReadCommand> message = command.createMessage(command.isTrackingRepairedStatus() && replica.isFull());
+ MessagingService.instance().sendWithCallback(message, replica.endpoint(), handler);
}
}
- return new SingleRangeResponse(handler);
+ return new SingleRangeResponse(resolver, handler, readRepair);
}
private PartitionIterator sendNextRequests()
{
List<PartitionIterator> concurrentQueries = new ArrayList<>(concurrencyFactor);
- for (int i = 0; i < concurrencyFactor && ranges.hasNext();)
+ List<ReadRepair> readRepairs = new ArrayList<>(concurrencyFactor);
+
+ try
{
- RangeForQuery range = ranges.next();
- concurrentQueries.add(query(range, i == 0));
- rangesQueried += range.vnodeCount();
- i += range.vnodeCount();
+ for (int i = 0; i < concurrencyFactor && ranges.hasNext();)
+ {
+ ReplicaPlan.ForRangeRead range = ranges.next();
+
+ @SuppressWarnings("resource") // response will be closed by concatAndBlockOnRepair, or in the catch block below
+ SingleRangeResponse response = query(range, i == 0);
+ concurrentQueries.add(response);
+ readRepairs.add(response.readRepair);
+ // due to RangeMerger, coordinator may fetch more ranges than required by concurrency factor.
+ rangesQueried += range.vnodeCount();
+ i += range.vnodeCount();
+ }
+ batchesRequested++;
}
- batchesRequested++;
+ catch (Throwable t)
+ {
+ for (PartitionIterator response: concurrentQueries)
+ response.close();
+ throw t;
+ }
Tracing.trace("Submitted {} concurrent range requests", concurrentQueries.size());
// We want to count the results for the sake of updating the concurrency factor (see updateConcurrencyFactor) but we don't want to
// enforce any particular limit at this point (this could break code than rely on postReconciliationProcessing), hence the DataLimits.NONE.
counter = DataLimits.NONE.newCounter(command.nowInSec(), true, command.selectsFullPartition(), enforceStrictLiveness);
- return counter.applyTo(PartitionIterators.concat(concurrentQueries));
+ return counter.applyTo(concatAndBlockOnRepair(concurrentQueries, readRepairs));
}
public void close()
@@ -2317,7 +2263,7 @@
{
Tracing.trace("Computing ranges to query");
- Keyspace keyspace = Keyspace.open(command.metadata().ksName);
+ Keyspace keyspace = Keyspace.open(command.metadata().keyspace);
RangeIterator ranges = new RangeIterator(command, keyspace, consistencyLevel);
// our estimate of how many result rows there will be per-range
@@ -2327,8 +2273,8 @@
resultsPerRange -= resultsPerRange * CONCURRENT_SUBREQUESTS_MARGIN;
int maxConcurrencyFactor = Math.min(ranges.rangeCount(), MAX_CONCURRENT_RANGE_REQUESTS);
int concurrencyFactor = resultsPerRange == 0.0
- ? 1
- : Math.max(1, Math.min(maxConcurrencyFactor, (int) Math.ceil(command.limits().count() / resultsPerRange)));
+ ? 1
+ : Math.max(1, Math.min(maxConcurrencyFactor, (int) Math.ceil(command.limits().count() / resultsPerRange)));
logger.trace("Estimated result rows per range: {}; requested rows: {}, ranges.size(): {}; concurrent range requests: {}",
resultsPerRange, command.limits().count(), ranges.rangeCount(), concurrencyFactor);
Tracing.trace("Submitting range requests on {} ranges with a concurrency of {} ({} rows per range expected)", ranges.rangeCount(), concurrencyFactor, resultsPerRange);
@@ -2340,8 +2286,6 @@
concurrencyFactor,
maxConcurrencyFactor,
ranges.rangeCount(),
- keyspace,
- consistencyLevel,
queryStartNanoTime);
return command.limits().filter(command.postReconciliationProcessing(rangeCommandIterator),
command.nowInSec(),
@@ -2351,7 +2295,12 @@
public Map<String, List<String>> getSchemaVersions()
{
- return describeSchemaVersions();
+ return describeSchemaVersions(false);
+ }
+
+ public Map<String, List<String>> getSchemaVersionsWithPort()
+ {
+ return describeSchemaVersions(true);
}
/**
@@ -2359,36 +2308,28 @@
* migration id. This is useful for determining if a schema change has propagated through the cluster. Disagreement
* is assumed if any node fails to respond.
*/
- public static Map<String, List<String>> describeSchemaVersions()
+ public static Map<String, List<String>> describeSchemaVersions(boolean withPort)
{
final String myVersion = Schema.instance.getVersion().toString();
- final Map<InetAddress, UUID> versions = new ConcurrentHashMap<InetAddress, UUID>();
- final Set<InetAddress> liveHosts = Gossiper.instance.getLiveMembers();
+ final Map<InetAddressAndPort, UUID> versions = new ConcurrentHashMap<>();
+ final Set<InetAddressAndPort> liveHosts = Gossiper.instance.getLiveMembers();
final CountDownLatch latch = new CountDownLatch(liveHosts.size());
- IAsyncCallback<UUID> cb = new IAsyncCallback<UUID>()
+ RequestCallback<UUID> cb = message ->
{
- public void response(MessageIn<UUID> message)
- {
- // record the response from the remote node.
- versions.put(message.from, message.payload);
- latch.countDown();
- }
-
- public boolean isLatencyForSnitch()
- {
- return false;
- }
+ // record the response from the remote node.
+ versions.put(message.from(), message.payload);
+ latch.countDown();
};
- // an empty message acts as a request to the SchemaCheckVerbHandler.
- MessageOut message = new MessageOut(MessagingService.Verb.SCHEMA_CHECK);
- for (InetAddress endpoint : liveHosts)
- MessagingService.instance().sendRR(message, endpoint, cb);
+ // an empty message acts as a request to the SchemaVersionVerbHandler.
+ Message message = Message.out(Verb.SCHEMA_VERSION_REQ, noPayload);
+ for (InetAddressAndPort endpoint : liveHosts)
+ MessagingService.instance().sendWithCallback(message, endpoint, cb);
try
{
// wait for as long as possible. timeout-1s if possible.
- latch.await(DatabaseDescriptor.getRpcTimeout(), TimeUnit.MILLISECONDS);
+ latch.await(DatabaseDescriptor.getRpcTimeout(NANOSECONDS), NANOSECONDS);
}
catch (InterruptedException ex)
{
@@ -2397,8 +2338,8 @@
// maps versions to hosts that are on that version.
Map<String, List<String>> results = new HashMap<String, List<String>>();
- Iterable<InetAddress> allHosts = Iterables.concat(Gossiper.instance.getLiveMembers(), Gossiper.instance.getUnreachableMembers());
- for (InetAddress host : allHosts)
+ Iterable<InetAddressAndPort> allHosts = Iterables.concat(Gossiper.instance.getLiveMembers(), Gossiper.instance.getUnreachableMembers());
+ for (InetAddressAndPort host : allHosts)
{
UUID version = versions.get(host);
String stringVersion = version == null ? UNREACHABLE : version.toString();
@@ -2408,7 +2349,7 @@
hosts = new ArrayList<String>();
results.put(stringVersion, hosts);
}
- hosts.add(host.getHostAddress());
+ hosts.add(host.getHostAddress(withPort));
}
// we're done: the results map is ready to return to the client. the rest is just debug logging:
@@ -2516,32 +2457,30 @@
DatabaseDescriptor.setMaxHintWindow(ms);
}
- public static boolean shouldHint(InetAddress ep)
+ public static boolean shouldHint(Replica replica)
{
- if (DatabaseDescriptor.hintedHandoffEnabled())
- {
- Set<String> disabledDCs = DatabaseDescriptor.hintedHandoffDisabledDCs();
- if (!disabledDCs.isEmpty())
- {
- final String dc = DatabaseDescriptor.getEndpointSnitch().getDatacenter(ep);
- if (disabledDCs.contains(dc))
- {
- Tracing.trace("Not hinting {} since its data center {} has been disabled {}", ep, dc, disabledDCs);
- return false;
- }
- }
- boolean hintWindowExpired = Gossiper.instance.getEndpointDowntime(ep) > DatabaseDescriptor.getMaxHintWindow();
- if (hintWindowExpired)
- {
- HintsService.instance.metrics.incrPastWindow(ep);
- Tracing.trace("Not hinting {} which has been down {} ms", ep, Gossiper.instance.getEndpointDowntime(ep));
- }
- return !hintWindowExpired;
- }
- else
- {
+ if (!DatabaseDescriptor.hintedHandoffEnabled())
return false;
+ if (replica.isTransient() || replica.isSelf())
+ return false;
+
+ Set<String> disabledDCs = DatabaseDescriptor.hintedHandoffDisabledDCs();
+ if (!disabledDCs.isEmpty())
+ {
+ final String dc = DatabaseDescriptor.getEndpointSnitch().getDatacenter(replica);
+ if (disabledDCs.contains(dc))
+ {
+ Tracing.trace("Not hinting {} since its data center {} has been disabled {}", replica, dc, disabledDCs);
+ return false;
+ }
}
+ boolean hintWindowExpired = Gossiper.instance.getEndpointDowntime(replica.endpoint()) > DatabaseDescriptor.getMaxHintWindow();
+ if (hintWindowExpired)
+ {
+ HintsService.instance.metrics.incrPastWindow(replica.endpoint());
+ Tracing.trace("Not hinting {} which has been down {} ms", replica, Gossiper.instance.getEndpointDowntime(replica.endpoint()));
+ }
+ return !hintWindowExpired;
}
/**
@@ -2562,20 +2501,19 @@
// invoked by an admin, for simplicity we require that all nodes are up
// to perform the operation.
int liveMembers = Gossiper.instance.getLiveMembers().size();
- throw new UnavailableException(ConsistencyLevel.ALL, liveMembers + Gossiper.instance.getUnreachableMembers().size(), liveMembers);
+ throw UnavailableException.create(ConsistencyLevel.ALL, liveMembers + Gossiper.instance.getUnreachableMembers().size(), liveMembers);
}
- Set<InetAddress> allEndpoints = StorageService.instance.getLiveRingMembers(true);
+ Set<InetAddressAndPort> allEndpoints = StorageService.instance.getLiveRingMembers(true);
int blockFor = allEndpoints.size();
final TruncateResponseHandler responseHandler = new TruncateResponseHandler(blockFor);
// Send out the truncate calls and track the responses with the callbacks.
Tracing.trace("Enqueuing truncate messages to hosts {}", allEndpoints);
- final Truncation truncation = new Truncation(keyspace, cfname);
- MessageOut<Truncation> message = truncation.createMessage();
- for (InetAddress endpoint : allEndpoints)
- MessagingService.instance().sendRR(message, endpoint, responseHandler);
+ Message<TruncateRequest> message = Message.out(TRUNCATE_REQ, new TruncateRequest(keyspace, cfname));
+ for (InetAddressAndPort endpoint : allEndpoints)
+ MessagingService.instance().sendWithCallback(message, endpoint, responseHandler);
// Wait for all
try
@@ -2601,10 +2539,9 @@
public interface WritePerformer
{
public void apply(IMutation mutation,
- Iterable<InetAddress> targets,
+ ReplicaPlan.ForTokenWrite targets,
AbstractWriteResponseHandler<IMutation> responseHandler,
- String localDataCenter,
- ConsistencyLevel consistencyLevel) throws OverloadedException;
+ String localDataCenter) throws OverloadedException;
}
/**
@@ -2615,12 +2552,12 @@
public ViewWriteMetricsWrapped(AbstractWriteResponseHandler<IMutation> writeHandler, int i, BatchlogCleanup cleanup, long queryStartNanoTime)
{
super(writeHandler, i, cleanup, queryStartNanoTime);
- viewWriteMetrics.viewReplicasAttempted.inc(totalEndpoints());
+ viewWriteMetrics.viewReplicasAttempted.inc(candidateReplicaCount());
}
- public void response(MessageIn<IMutation> msg)
+ public void onResponse(Message<IMutation> msg)
{
- super.response(msg);
+ super.onResponse(msg);
viewWriteMetrics.viewReplicasSuccess.inc();
}
}
@@ -2630,21 +2567,23 @@
*/
private static abstract class DroppableRunnable implements Runnable
{
- final long constructionTime;
- final MessagingService.Verb verb;
+ final long approxCreationTimeNanos;
+ final Verb verb;
- public DroppableRunnable(MessagingService.Verb verb)
+ public DroppableRunnable(Verb verb)
{
- this.constructionTime = System.currentTimeMillis();
+ this.approxCreationTimeNanos = MonotonicClock.approxTime.now();
this.verb = verb;
}
public final void run()
{
- long timeTaken = System.currentTimeMillis() - constructionTime;
- if (timeTaken > verb.getTimeout())
+ long approxCurrentTimeNanos = MonotonicClock.approxTime.now();
+ long expirationTimeNanos = verb.expiresAtNanos(approxCreationTimeNanos);
+ if (approxCurrentTimeNanos > expirationTimeNanos)
{
- MessagingService.instance().incrementDroppedMessages(verb, timeTaken);
+ long timeTakenNanos = approxCurrentTimeNanos - approxCreationTimeNanos;
+ MessagingService.instance().metrics.recordSelfDroppedMessage(verb, timeTakenNanos, NANOSECONDS);
return;
}
try
@@ -2666,30 +2605,26 @@
*/
private static abstract class LocalMutationRunnable implements Runnable
{
- private final long constructionTime = System.currentTimeMillis();
+ private final long approxCreationTimeNanos = MonotonicClock.approxTime.now();
- private final Optional<IMutation> mutationOpt;
+ private final Replica localReplica;
- public LocalMutationRunnable(Optional<IMutation> mutationOpt)
+ LocalMutationRunnable(Replica localReplica)
{
- this.mutationOpt = mutationOpt;
- }
-
- public LocalMutationRunnable()
- {
- this.mutationOpt = Optional.empty();
+ this.localReplica = localReplica;
}
public final void run()
{
- final MessagingService.Verb verb = verb();
- long mutationTimeout = verb.getTimeout();
- long timeTaken = System.currentTimeMillis() - constructionTime;
- if (timeTaken > mutationTimeout)
+ final Verb verb = verb();
+ long nowNanos = MonotonicClock.approxTime.now();
+ long expirationTimeNanos = verb.expiresAtNanos(approxCreationTimeNanos);
+ if (nowNanos > expirationTimeNanos)
{
- if (MessagingService.DROPPABLE_VERBS.contains(verb))
- MessagingService.instance().incrementDroppedMutations(mutationOpt, timeTaken);
- HintRunnable runnable = new HintRunnable(Collections.singleton(FBUtilities.getBroadcastAddress()))
+ long timeTakenNanos = nowNanos - approxCreationTimeNanos;
+ MessagingService.instance().metrics.recordSelfDroppedMessage(Verb.MUTATION_REQ, timeTakenNanos, NANOSECONDS);
+
+ HintRunnable runnable = new HintRunnable(EndpointsForToken.of(localReplica.range().right, localReplica))
{
protected void runMayThrow() throws Exception
{
@@ -2710,7 +2645,7 @@
}
}
- abstract protected MessagingService.Verb verb();
+ abstract protected Verb verb();
abstract protected void runMayThrow() throws Exception;
}
@@ -2720,9 +2655,9 @@
*/
private abstract static class HintRunnable implements Runnable
{
- public final Collection<InetAddress> targets;
+ public final EndpointsForToken targets;
- protected HintRunnable(Collection<InetAddress> targets)
+ protected HintRunnable(EndpointsForToken targets)
{
this.targets = targets;
}
@@ -2740,7 +2675,7 @@
finally
{
StorageMetrics.totalHintsInProgress.dec(targets.size());
- for (InetAddress target : targets)
+ for (InetAddressAndPort target : targets.endpoints())
getHintsInProgressFor(target).decrementAndGet();
}
}
@@ -2774,7 +2709,7 @@
logger.warn("Some hints were not written before shutdown. This is not supposed to happen. You should (a) run repair, and (b) file a bug report");
}
- private static AtomicInteger getHintsInProgressFor(InetAddress destination)
+ private static AtomicInteger getHintsInProgressFor(InetAddressAndPort destination)
{
try
{
@@ -2786,22 +2721,23 @@
}
}
- public static Future<Void> submitHint(Mutation mutation, InetAddress target, AbstractWriteResponseHandler<IMutation> responseHandler)
+ public static Future<Void> submitHint(Mutation mutation, Replica target, AbstractWriteResponseHandler<IMutation> responseHandler)
{
- return submitHint(mutation, Collections.singleton(target), responseHandler);
+ return submitHint(mutation, EndpointsForToken.of(target.range().right, target), responseHandler);
}
public static Future<Void> submitHint(Mutation mutation,
- Collection<InetAddress> targets,
+ EndpointsForToken targets,
AbstractWriteResponseHandler<IMutation> responseHandler)
{
+ Replicas.assertFull(targets); // hints should not be written for transient replicas
HintRunnable runnable = new HintRunnable(targets)
{
public void runMayThrow()
{
- Set<InetAddress> validTargets = new HashSet<>(targets.size());
+ Set<InetAddressAndPort> validTargets = new HashSet<>(targets.size());
Set<UUID> hostIds = new HashSet<>(targets.size());
- for (InetAddress target : targets)
+ for (InetAddressAndPort target : targets.endpoints())
{
UUID hostId = StorageService.instance.getHostIdForEndpoint(target);
if (hostId != null)
@@ -2816,8 +2752,8 @@
HintsService.instance.write(hostIds, Hint.create(mutation, System.currentTimeMillis()));
validTargets.forEach(HintsService.instance.metrics::incrCreatedHints);
// Notify the handler only for CL == ANY
- if (responseHandler != null && responseHandler.consistencyLevel == ConsistencyLevel.ANY)
- responseHandler.response(null);
+ if (responseHandler != null && responseHandler.replicaPlan.consistencyLevel() == ConsistencyLevel.ANY)
+ responseHandler.onResponse(null);
}
};
@@ -2827,30 +2763,30 @@
private static Future<Void> submitHint(HintRunnable runnable)
{
StorageMetrics.totalHintsInProgress.inc(runnable.targets.size());
- for (InetAddress target : runnable.targets)
- getHintsInProgressFor(target).incrementAndGet();
- return (Future<Void>) StageManager.getStage(Stage.MUTATION).submit(runnable);
+ for (Replica target : runnable.targets)
+ getHintsInProgressFor(target.endpoint()).incrementAndGet();
+ return (Future<Void>) Stage.MUTATION.submit(runnable);
}
- public Long getRpcTimeout() { return DatabaseDescriptor.getRpcTimeout(); }
+ public Long getRpcTimeout() { return DatabaseDescriptor.getRpcTimeout(MILLISECONDS); }
public void setRpcTimeout(Long timeoutInMillis) { DatabaseDescriptor.setRpcTimeout(timeoutInMillis); }
- public Long getReadRpcTimeout() { return DatabaseDescriptor.getReadRpcTimeout(); }
+ public Long getReadRpcTimeout() { return DatabaseDescriptor.getReadRpcTimeout(MILLISECONDS); }
public void setReadRpcTimeout(Long timeoutInMillis) { DatabaseDescriptor.setReadRpcTimeout(timeoutInMillis); }
- public Long getWriteRpcTimeout() { return DatabaseDescriptor.getWriteRpcTimeout(); }
+ public Long getWriteRpcTimeout() { return DatabaseDescriptor.getWriteRpcTimeout(MILLISECONDS); }
public void setWriteRpcTimeout(Long timeoutInMillis) { DatabaseDescriptor.setWriteRpcTimeout(timeoutInMillis); }
- public Long getCounterWriteRpcTimeout() { return DatabaseDescriptor.getCounterWriteRpcTimeout(); }
+ public Long getCounterWriteRpcTimeout() { return DatabaseDescriptor.getCounterWriteRpcTimeout(MILLISECONDS); }
public void setCounterWriteRpcTimeout(Long timeoutInMillis) { DatabaseDescriptor.setCounterWriteRpcTimeout(timeoutInMillis); }
- public Long getCasContentionTimeout() { return DatabaseDescriptor.getCasContentionTimeout(); }
+ public Long getCasContentionTimeout() { return DatabaseDescriptor.getCasContentionTimeout(MILLISECONDS); }
public void setCasContentionTimeout(Long timeoutInMillis) { DatabaseDescriptor.setCasContentionTimeout(timeoutInMillis); }
- public Long getRangeRpcTimeout() { return DatabaseDescriptor.getRangeRpcTimeout(); }
+ public Long getRangeRpcTimeout() { return DatabaseDescriptor.getRangeRpcTimeout(MILLISECONDS); }
public void setRangeRpcTimeout(Long timeoutInMillis) { DatabaseDescriptor.setRangeRpcTimeout(timeoutInMillis); }
- public Long getTruncateRpcTimeout() { return DatabaseDescriptor.getTruncateRpcTimeout(); }
+ public Long getTruncateRpcTimeout() { return DatabaseDescriptor.getTruncateRpcTimeout(MILLISECONDS); }
public void setTruncateRpcTimeout(Long timeoutInMillis) { DatabaseDescriptor.setTruncateRpcTimeout(timeoutInMillis); }
public Long getNativeTransportMaxConcurrentConnections() { return DatabaseDescriptor.getNativeTransportMaxConcurrentConnections(); }
@@ -2881,12 +2817,126 @@
return Schema.instance.getNumberOfTables();
}
- public int getOtcBacklogExpirationInterval() {
- return DatabaseDescriptor.getOtcBacklogExpirationInterval();
+ public String getIdealConsistencyLevel()
+ {
+ return DatabaseDescriptor.getIdealConsistencyLevel().toString();
}
- public void setOtcBacklogExpirationInterval(int intervalInMillis) {
- DatabaseDescriptor.setOtcBacklogExpirationInterval(intervalInMillis);
+ public String setIdealConsistencyLevel(String cl)
+ {
+ ConsistencyLevel original = DatabaseDescriptor.getIdealConsistencyLevel();
+ ConsistencyLevel newCL = ConsistencyLevel.valueOf(cl.trim().toUpperCase());
+ DatabaseDescriptor.setIdealConsistencyLevel(newCL);
+ return String.format("Updating ideal consistency level new value: %s old value %s", newCL, original.toString());
+ }
+
+ @Deprecated
+ public int getOtcBacklogExpirationInterval() {
+ return 0;
+ }
+
+ @Deprecated
+ public void setOtcBacklogExpirationInterval(int intervalInMillis) { }
+
+ @Override
+ public void enableRepairedDataTrackingForRangeReads()
+ {
+ DatabaseDescriptor.setRepairedDataTrackingForRangeReadsEnabled(true);
+ }
+
+ @Override
+ public void disableRepairedDataTrackingForRangeReads()
+ {
+ DatabaseDescriptor.setRepairedDataTrackingForRangeReadsEnabled(false);
+ }
+
+ @Override
+ public boolean getRepairedDataTrackingEnabledForRangeReads()
+ {
+ return DatabaseDescriptor.getRepairedDataTrackingForRangeReadsEnabled();
+ }
+
+ @Override
+ public void enableRepairedDataTrackingForPartitionReads()
+ {
+ DatabaseDescriptor.setRepairedDataTrackingForPartitionReadsEnabled(true);
+ }
+
+ @Override
+ public void disableRepairedDataTrackingForPartitionReads()
+ {
+ DatabaseDescriptor.setRepairedDataTrackingForPartitionReadsEnabled(false);
+ }
+
+ @Override
+ public boolean getRepairedDataTrackingEnabledForPartitionReads()
+ {
+ return DatabaseDescriptor.getRepairedDataTrackingForPartitionReadsEnabled();
+ }
+
+ @Override
+ public void enableReportingUnconfirmedRepairedDataMismatches()
+ {
+ DatabaseDescriptor.reportUnconfirmedRepairedDataMismatches(true);
+ }
+
+ @Override
+ public void disableReportingUnconfirmedRepairedDataMismatches()
+ {
+ DatabaseDescriptor.reportUnconfirmedRepairedDataMismatches(false);
+ }
+
+ @Override
+ public boolean getReportingUnconfirmedRepairedDataMismatchesEnabled()
+ {
+ return DatabaseDescriptor.reportUnconfirmedRepairedDataMismatches();
+ }
+
+ @Override
+ public boolean getSnapshotOnRepairedDataMismatchEnabled()
+ {
+ return DatabaseDescriptor.snapshotOnRepairedDataMismatch();
+ }
+
+ @Override
+ public void enableSnapshotOnRepairedDataMismatch()
+ {
+ DatabaseDescriptor.setSnapshotOnRepairedDataMismatch(true);
+ }
+
+ @Override
+ public void disableSnapshotOnRepairedDataMismatch()
+ {
+ DatabaseDescriptor.setSnapshotOnRepairedDataMismatch(false);
+ }
+
+ static class PaxosBallotAndContention
+ {
+ final UUID ballot;
+ final int contentions;
+
+ PaxosBallotAndContention(UUID ballot, int contentions)
+ {
+ this.ballot = ballot;
+ this.contentions = contentions;
+ }
+
+ @Override
+ public final int hashCode()
+ {
+ int hashCode = 31 + (ballot == null ? 0 : ballot.hashCode());
+ return 31 * hashCode * this.contentions;
+ }
+
+ @Override
+ public final boolean equals(Object o)
+ {
+ if(!(o instanceof PaxosBallotAndContention))
+ return false;
+ PaxosBallotAndContention that = (PaxosBallotAndContention)o;
+ // handles nulls properly
+ return Objects.equals(ballot, that.ballot) && contentions == that.contentions;
+ }
}
@Override
diff --git a/src/java/org/apache/cassandra/service/StorageProxyMBean.java b/src/java/org/apache/cassandra/service/StorageProxyMBean.java
index cdf07f4..e3cde4b 100644
--- a/src/java/org/apache/cassandra/service/StorageProxyMBean.java
+++ b/src/java/org/apache/cassandra/service/StorageProxyMBean.java
@@ -21,7 +21,6 @@
import java.util.Map;
import java.util.Set;
-import org.apache.cassandra.config.DatabaseDescriptor;
public interface StorageProxyMBean
{
@@ -61,14 +60,39 @@
public long getReadRepairRepairedBlocking();
public long getReadRepairRepairedBackground();
+ @Deprecated
public int getOtcBacklogExpirationInterval();
+ @Deprecated
public void setOtcBacklogExpirationInterval(int intervalInMillis);
/** Returns each live node's schema version */
- public Map<String, List<String>> getSchemaVersions();
+ @Deprecated public Map<String, List<String>> getSchemaVersions();
+ public Map<String, List<String>> getSchemaVersionsWithPort();
public int getNumberOfTables();
+ public String getIdealConsistencyLevel();
+ public String setIdealConsistencyLevel(String cl);
+
+ /**
+ * Tracking and reporting of variances in the repaired data set across replicas at read time
+ */
+ void enableRepairedDataTrackingForRangeReads();
+ void disableRepairedDataTrackingForRangeReads();
+ boolean getRepairedDataTrackingEnabledForRangeReads();
+
+ void enableRepairedDataTrackingForPartitionReads();
+ void disableRepairedDataTrackingForPartitionReads();
+ boolean getRepairedDataTrackingEnabledForPartitionReads();
+
+ void enableReportingUnconfirmedRepairedDataMismatches();
+ void disableReportingUnconfirmedRepairedDataMismatches();
+ boolean getReportingUnconfirmedRepairedDataMismatchesEnabled();
+
+ void enableSnapshotOnRepairedDataMismatch();
+ void disableSnapshotOnRepairedDataMismatch();
+ boolean getSnapshotOnRepairedDataMismatchEnabled();
+
void enableSnapshotOnDuplicateRowDetection();
void disableSnapshotOnDuplicateRowDetection();
boolean getSnapshotOnDuplicateRowDetectionEnabled();
diff --git a/src/java/org/apache/cassandra/service/StorageService.java b/src/java/org/apache/cassandra/service/StorageService.java
index d3c30a0..a1b3b82 100644
--- a/src/java/org/apache/cassandra/service/StorageService.java
+++ b/src/java/org/apache/cassandra/service/StorageService.java
@@ -21,6 +21,7 @@
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.nio.ByteBuffer;
+import java.nio.file.Paths;
import java.util.*;
import java.util.Map.Entry;
import java.util.concurrent.*;
@@ -28,49 +29,57 @@
import java.util.concurrent.atomic.AtomicInteger;
import java.util.regex.MatchResult;
import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
import java.util.stream.StreamSupport;
import javax.annotation.Nullable;
import javax.management.*;
+import javax.management.openmbean.CompositeData;
+import javax.management.openmbean.OpenDataException;
import javax.management.openmbean.TabularData;
import javax.management.openmbean.TabularDataSupport;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.base.Predicate;
+import com.google.common.base.Predicates;
import com.google.common.collect.*;
import com.google.common.util.concurrent.*;
+import org.apache.cassandra.config.ParameterizedClass;
+import org.apache.cassandra.dht.RangeStreamer.FetchReplica;
+import org.apache.cassandra.fql.FullQueryLogger;
+import org.apache.cassandra.fql.FullQueryLoggerOptions;
+import org.apache.cassandra.locator.ReplicaCollection.Builder.Conflict;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.cassandra.audit.AuditLogManager;
+import org.apache.cassandra.audit.AuditLogOptions;
import org.apache.cassandra.auth.AuthKeyspace;
-import org.apache.cassandra.auth.AuthMigrationListener;
-import org.apache.cassandra.batchlog.BatchRemoveVerbHandler;
-import org.apache.cassandra.batchlog.BatchStoreVerbHandler;
+import org.apache.cassandra.auth.AuthSchemaChangeListener;
import org.apache.cassandra.batchlog.BatchlogManager;
+import org.apache.cassandra.concurrent.ExecutorLocals;
import org.apache.cassandra.concurrent.NamedThreadFactory;
import org.apache.cassandra.concurrent.ScheduledExecutors;
import org.apache.cassandra.concurrent.Stage;
-import org.apache.cassandra.concurrent.StageManager;
-import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.config.Config;
import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.config.Schema;
-import org.apache.cassandra.config.SchemaConstants;
-import org.apache.cassandra.config.ViewDefinition;
+import org.apache.cassandra.cql3.QueryProcessor;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.commitlog.CommitLog;
import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.db.compaction.Verifier;
import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.db.virtual.VirtualKeyspaceRegistry;
import org.apache.cassandra.dht.*;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token.TokenFactory;
import org.apache.cassandra.exceptions.*;
import org.apache.cassandra.gms.*;
-import org.apache.cassandra.hints.HintVerbHandler;
import org.apache.cassandra.hints.HintsService;
import org.apache.cassandra.io.sstable.SSTableLoader;
import org.apache.cassandra.io.util.FileUtils;
@@ -81,29 +90,39 @@
import org.apache.cassandra.repair.messages.RepairOption;
import org.apache.cassandra.schema.CompactionParams.TombstoneOption;
import org.apache.cassandra.schema.KeyspaceMetadata;
-import org.apache.cassandra.schema.SchemaKeyspace;
-import org.apache.cassandra.service.paxos.CommitVerbHandler;
-import org.apache.cassandra.service.paxos.PrepareVerbHandler;
-import org.apache.cassandra.service.paxos.ProposeVerbHandler;
+import org.apache.cassandra.schema.MigrationManager;
+import org.apache.cassandra.schema.ReplicationParams;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.SchemaConstants;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.schema.TableMetadataRef;
+import org.apache.cassandra.schema.ViewMetadata;
import org.apache.cassandra.streaming.*;
-import org.apache.cassandra.thrift.EndpointDetails;
-import org.apache.cassandra.thrift.TokenRange;
-import org.apache.cassandra.thrift.cassandraConstants;
import org.apache.cassandra.tracing.TraceKeyspace;
import org.apache.cassandra.transport.ProtocolVersion;
+import org.apache.cassandra.transport.Server;
import org.apache.cassandra.utils.*;
import org.apache.cassandra.utils.logging.LoggingSupportFactory;
import org.apache.cassandra.utils.progress.ProgressEvent;
import org.apache.cassandra.utils.progress.ProgressEventType;
+import org.apache.cassandra.utils.progress.ProgressListener;
+import org.apache.cassandra.utils.progress.jmx.JMXBroadcastExecutor;
import org.apache.cassandra.utils.progress.jmx.JMXProgressSupport;
-import org.apache.cassandra.utils.progress.jmx.LegacyJMXProgressSupport;
+import static com.google.common.collect.Iterables.transform;
+import static com.google.common.collect.Iterables.tryFind;
import static java.util.Arrays.asList;
+import static java.util.Arrays.stream;
import static java.util.concurrent.TimeUnit.MINUTES;
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+import static java.util.concurrent.TimeUnit.NANOSECONDS;
import static java.util.stream.Collectors.toList;
+import static java.util.stream.Collectors.toMap;
import static org.apache.cassandra.index.SecondaryIndexManager.getIndexName;
import static org.apache.cassandra.index.SecondaryIndexManager.isIndexColumnFamily;
-import static org.apache.cassandra.service.MigrationManager.evolveSystemKeyspace;
+import static org.apache.cassandra.net.NoPayload.noPayload;
+import static org.apache.cassandra.net.Verb.REPLICATION_DONE_REQ;
+import static org.apache.cassandra.schema.MigrationManager.evolveSystemKeyspace;
/**
* This abstraction contains the token/identifier of this node
@@ -119,15 +138,6 @@
private final JMXProgressSupport progressSupport = new JMXProgressSupport(this);
- /**
- * @deprecated backward support to previous notification interface
- * Will be removed on 4.0
- */
- @Deprecated
- private final LegacyJMXProgressSupport legacyProgressSupport;
-
- private static final AtomicInteger threadCounter = new AtomicInteger(1);
-
private static int getRingDelay()
{
String newdelay = System.getProperty("cassandra.ring_delay_ms");
@@ -163,31 +173,45 @@
return isShutdown;
}
- public Collection<Range<Token>> getLocalRanges(String keyspaceName)
+ public RangesAtEndpoint getLocalReplicas(String keyspaceName)
{
- return getRangesForEndpoint(keyspaceName, FBUtilities.getBroadcastAddress());
+ return Keyspace.open(keyspaceName).getReplicationStrategy()
+ .getAddressReplicas(FBUtilities.getBroadcastAddressAndPort());
+ }
+
+ public List<Range<Token>> getLocalAndPendingRanges(String ks)
+ {
+ InetAddressAndPort broadcastAddress = FBUtilities.getBroadcastAddressAndPort();
+ Keyspace keyspace = Keyspace.open(ks);
+ List<Range<Token>> ranges = new ArrayList<>();
+ for (Replica r : keyspace.getReplicationStrategy().getAddressReplicas(broadcastAddress))
+ ranges.add(r.range());
+ for (Replica r : getTokenMetadata().getPendingRanges(ks, broadcastAddress))
+ ranges.add(r.range());
+ return ranges;
}
public Collection<Range<Token>> getPrimaryRanges(String keyspace)
{
- return getPrimaryRangesForEndpoint(keyspace, FBUtilities.getBroadcastAddress());
+ return getPrimaryRangesForEndpoint(keyspace, FBUtilities.getBroadcastAddressAndPort());
}
public Collection<Range<Token>> getPrimaryRangesWithinDC(String keyspace)
{
- return getPrimaryRangeForEndpointWithinDC(keyspace, FBUtilities.getBroadcastAddress());
+ return getPrimaryRangeForEndpointWithinDC(keyspace, FBUtilities.getBroadcastAddressAndPort());
}
- private final Set<InetAddress> replicatingNodes = Collections.synchronizedSet(new HashSet<InetAddress>());
+ private final Set<InetAddressAndPort> replicatingNodes = Sets.newConcurrentHashSet();
private CassandraDaemon daemon;
- private InetAddress removingNode;
+ private InetAddressAndPort removingNode;
/* Are we starting this node in bootstrap mode? */
private volatile boolean isBootstrapMode;
/* we bootstrap but do NOT join the ring unless told to do so */
- private boolean isSurveyMode = Boolean.parseBoolean(System.getProperty("cassandra.write_survey", "false"));
+ private boolean isSurveyMode = Boolean.parseBoolean(System.getProperty
+ ("cassandra.write_survey", "false"));
/* true if node is rebuilding and receiving data */
private final AtomicBoolean isRebuilding = new AtomicBoolean();
private final AtomicBoolean isDecommissioning = new AtomicBoolean();
@@ -216,7 +240,7 @@
private Collection<Token> bootstrapTokens = null;
// true when keeping strict consistency while bootstrapping
- private static final boolean useStrictConsistency = Boolean.parseBoolean(System.getProperty("cassandra.consistent.rangemovement", "true"));
+ public static final boolean useStrictConsistency = Boolean.parseBoolean(System.getProperty("cassandra.consistent.rangemovement", "true"));
private static final boolean allowSimultaneousMoves = Boolean.parseBoolean(System.getProperty("cassandra.consistent.simultaneousmoves.allow","false"));
private static final boolean joinRing = Boolean.parseBoolean(System.getProperty("cassandra.join_ring", "true"));
private boolean replacing;
@@ -242,7 +266,7 @@
SystemKeyspace.updateTokens(tokens);
Collection<Token> localTokens = getLocalTokens();
setGossipTokens(localTokens);
- tokenMetadata.updateNormalTokens(tokens, FBUtilities.getBroadcastAddress());
+ tokenMetadata.updateNormalTokens(tokens, FBUtilities.getBroadcastAddressAndPort());
setMode(Mode.NORMAL, false);
}
@@ -250,54 +274,19 @@
{
List<Pair<ApplicationState, VersionedValue>> states = new ArrayList<Pair<ApplicationState, VersionedValue>>();
states.add(Pair.create(ApplicationState.TOKENS, valueFactory.tokens(tokens)));
+ states.add(Pair.create(ApplicationState.STATUS_WITH_PORT, valueFactory.normal(tokens)));
states.add(Pair.create(ApplicationState.STATUS, valueFactory.normal(tokens)));
Gossiper.instance.addLocalApplicationStates(states);
}
public StorageService()
{
- // use dedicated executor for sending JMX notifications
- super(Executors.newSingleThreadExecutor());
+ // use dedicated executor for handling JMX notifications
+ super(JMXBroadcastExecutor.executor);
jmxObjectName = "org.apache.cassandra.db:type=StorageService";
MBeanWrapper.instance.registerMBean(this, jmxObjectName);
MBeanWrapper.instance.registerMBean(StreamManager.instance, StreamManager.OBJECT_NAME);
-
- legacyProgressSupport = new LegacyJMXProgressSupport(this, jmxObjectName);
-
- /* register the verb handlers */
- MessagingService.instance().registerVerbHandlers(MessagingService.Verb.MUTATION, new MutationVerbHandler());
- MessagingService.instance().registerVerbHandlers(MessagingService.Verb.READ_REPAIR, new ReadRepairVerbHandler());
- MessagingService.instance().registerVerbHandlers(MessagingService.Verb.READ, new ReadCommandVerbHandler());
- MessagingService.instance().registerVerbHandlers(MessagingService.Verb.RANGE_SLICE, new RangeSliceVerbHandler());
- MessagingService.instance().registerVerbHandlers(MessagingService.Verb.PAGED_RANGE, new RangeSliceVerbHandler());
- MessagingService.instance().registerVerbHandlers(MessagingService.Verb.COUNTER_MUTATION, new CounterMutationVerbHandler());
- MessagingService.instance().registerVerbHandlers(MessagingService.Verb.TRUNCATE, new TruncateVerbHandler());
- MessagingService.instance().registerVerbHandlers(MessagingService.Verb.PAXOS_PREPARE, new PrepareVerbHandler());
- MessagingService.instance().registerVerbHandlers(MessagingService.Verb.PAXOS_PROPOSE, new ProposeVerbHandler());
- MessagingService.instance().registerVerbHandlers(MessagingService.Verb.PAXOS_COMMIT, new CommitVerbHandler());
- MessagingService.instance().registerVerbHandlers(MessagingService.Verb.HINT, new HintVerbHandler());
-
- // see BootStrapper for a summary of how the bootstrap verbs interact
- MessagingService.instance().registerVerbHandlers(MessagingService.Verb.REPLICATION_FINISHED, new ReplicationFinishedVerbHandler());
- MessagingService.instance().registerVerbHandlers(MessagingService.Verb.REQUEST_RESPONSE, new ResponseVerbHandler());
- MessagingService.instance().registerVerbHandlers(MessagingService.Verb.INTERNAL_RESPONSE, new ResponseVerbHandler());
- MessagingService.instance().registerVerbHandlers(MessagingService.Verb.REPAIR_MESSAGE, new RepairMessageVerbHandler());
- MessagingService.instance().registerVerbHandlers(MessagingService.Verb.GOSSIP_SHUTDOWN, new GossipShutdownVerbHandler());
-
- MessagingService.instance().registerVerbHandlers(MessagingService.Verb.GOSSIP_DIGEST_SYN, new GossipDigestSynVerbHandler());
- MessagingService.instance().registerVerbHandlers(MessagingService.Verb.GOSSIP_DIGEST_ACK, new GossipDigestAckVerbHandler());
- MessagingService.instance().registerVerbHandlers(MessagingService.Verb.GOSSIP_DIGEST_ACK2, new GossipDigestAck2VerbHandler());
-
- MessagingService.instance().registerVerbHandlers(MessagingService.Verb.DEFINITIONS_UPDATE, new DefinitionsUpdateVerbHandler());
- MessagingService.instance().registerVerbHandlers(MessagingService.Verb.SCHEMA_CHECK, new SchemaCheckVerbHandler());
- MessagingService.instance().registerVerbHandlers(MessagingService.Verb.MIGRATION_REQUEST, new MigrationRequestVerbHandler());
-
- MessagingService.instance().registerVerbHandlers(MessagingService.Verb.SNAPSHOT, new SnapshotVerbHandler());
- MessagingService.instance().registerVerbHandlers(MessagingService.Verb.ECHO, new EchoVerbHandler());
-
- MessagingService.instance().registerVerbHandlers(MessagingService.Verb.BATCH_STORE, new BatchStoreVerbHandler());
- MessagingService.instance().registerVerbHandlers(MessagingService.Verb.BATCH_REMOVE, new BatchRemoveVerbHandler());
}
public void registerDaemon(CassandraDaemon daemon)
@@ -357,54 +346,6 @@
return Gossiper.instance.isEnabled();
}
- // should only be called via JMX
- public synchronized void startRPCServer()
- {
- checkServiceAllowedToStart("thrift");
-
- if (daemon == null)
- {
- throw new IllegalStateException("No configured daemon");
- }
-
- // We only start transports if bootstrap has completed and we're not in survey mode, OR if we are in
- // survey mode and streaming has completed but we're not using auth.
- // OR if we have not joined the ring yet.
- if (StorageService.instance.hasJoined() &&
- ((!StorageService.instance.isSurveyMode() && !SystemKeyspace.bootstrapComplete()) ||
- (StorageService.instance.isSurveyMode() && StorageService.instance.isBootstrapMode())))
- {
- throw new IllegalStateException("Node is not yet bootstrapped completely. Use nodetool to check bootstrap state and resume. For more, see `nodetool help bootstrap`");
- }
- else if (StorageService.instance.hasJoined() && StorageService.instance.isSurveyMode() &&
- DatabaseDescriptor.getAuthenticator().requireAuthentication())
- {
- // Auth isn't initialised until we join the ring, so if we're in survey mode auth will always fail.
- throw new IllegalStateException("Not starting RPC server as write_survey mode and authentication is enabled");
- }
-
- daemon.thriftServer.start();
- }
-
- public void stopRPCServer()
- {
- if (daemon == null)
- {
- throw new IllegalStateException("No configured daemon");
- }
- if (daemon.thriftServer != null)
- daemon.thriftServer.stop();
- }
-
- public boolean isRPCServerRunning()
- {
- if ((daemon == null) || (daemon.thriftServer == null))
- {
- return false;
- }
- return daemon.thriftServer.isRunning();
- }
-
public synchronized void startNativeTransport()
{
checkServiceAllowedToStart("native transport");
@@ -442,21 +383,16 @@
return daemon.isNativeTransportRunning();
}
- public int getMaxNativeProtocolVersion()
+ @Override
+ public void enableNativeTransportOldProtocolVersions()
{
- if (daemon == null)
- {
- throw new IllegalStateException("No configured daemon");
- }
- return daemon.getMaxNativeProtocolVersion();
+ DatabaseDescriptor.setNativeTransportAllowOlderProtocols(true);
}
- private void refreshMaxNativeProtocolVersion()
+ @Override
+ public void disableNativeTransportOldProtocolVersions()
{
- if (daemon != null)
- {
- daemon.refreshMaxNativeProtocolVersion();
- }
+ DatabaseDescriptor.setNativeTransportAllowOlderProtocols(false);
}
public void stopTransports()
@@ -466,11 +402,6 @@
logger.error("Stopping gossiper");
stopGossiping();
}
- if (isRPCServerRunning())
- {
- logger.error("Stopping RPC server");
- stopRPCServer();
- }
if (isNativeTransportRunning())
{
logger.error("Stopping native transport");
@@ -486,12 +417,11 @@
* they get the Gossip shutdown message, so even if
* we don't get time to broadcast this, it is not a problem.
*
- * See {@link Gossiper#markAsShutdown(InetAddress)}
+ * See {@link Gossiper#markAsShutdown(InetAddressAndPort)}
*/
private void shutdownClientServers()
{
setRpcReady(false);
- stopRPCServer();
stopNativeTransport();
}
@@ -502,7 +432,7 @@
MessagingService.instance().shutdown();
// give it a second so that task accepted before the MessagingService shutdown gets submitted to the stage (to avoid RejectedExecutionException)
Uninterruptibles.sleepUninterruptibly(1, TimeUnit.SECONDS);
- StageManager.shutdownNow();
+ Stage.shutdownNow();
}
public boolean isInitialized()
@@ -543,13 +473,15 @@
"To perform this operation, please restart with " +
"-Dcassandra.allow_unsafe_replace=true");
- InetAddress replaceAddress = DatabaseDescriptor.getReplaceAddress();
+ InetAddressAndPort replaceAddress = DatabaseDescriptor.getReplaceAddress();
logger.info("Gathering node replacement information for {}", replaceAddress);
- Map<InetAddress, EndpointState> epStates = Gossiper.instance.doShadowRound();
+ Map<InetAddressAndPort, EndpointState> epStates = Gossiper.instance.doShadowRound();
// as we've completed the shadow round of gossip, we should be able to find the node we're replacing
if (epStates.get(replaceAddress) == null)
throw new RuntimeException(String.format("Cannot replace_address %s because it doesn't exist in gossip", replaceAddress));
+ validateEndpointSnitch(epStates.values().iterator());
+
try
{
VersionedValue tokensVersionedValue = epStates.get(replaceAddress).getApplicationState(ApplicationState.TOKENS);
@@ -574,7 +506,7 @@
return localHostId;
}
- private synchronized void checkForEndpointCollision(UUID localHostId, Set<InetAddress> peers) throws ConfigurationException
+ private synchronized void checkForEndpointCollision(UUID localHostId, Set<InetAddressAndPort> peers) throws ConfigurationException
{
if (Boolean.getBoolean("cassandra.allow_unsafe_join"))
{
@@ -583,30 +515,39 @@
}
logger.debug("Starting shadow gossip round to check for endpoint collision");
- Map<InetAddress, EndpointState> epStates = Gossiper.instance.doShadowRound(peers);
+ Map<InetAddressAndPort, EndpointState> epStates = Gossiper.instance.doShadowRound(peers);
- if (epStates.isEmpty() && DatabaseDescriptor.getSeeds().contains(FBUtilities.getBroadcastAddress()))
+ if (epStates.isEmpty() && DatabaseDescriptor.getSeeds().contains(FBUtilities.getBroadcastAddressAndPort()))
logger.info("Unable to gossip with any peers but continuing anyway since node is in its own seed list");
// If bootstrapping, check whether any previously known status for the endpoint makes it unsafe to do so.
// If not bootstrapping, compare the host id for this endpoint learned from gossip (if any) with the local
// one, which was either read from system.local or generated at startup. If a learned id is present &
// doesn't match the local, then the node needs replacing
- if (!Gossiper.instance.isSafeForStartup(FBUtilities.getBroadcastAddress(), localHostId, shouldBootstrap(), epStates))
+ if (!Gossiper.instance.isSafeForStartup(FBUtilities.getBroadcastAddressAndPort(), localHostId, shouldBootstrap(), epStates))
{
throw new RuntimeException(String.format("A node with address %s already exists, cancelling join. " +
"Use cassandra.replace_address if you want to replace this node.",
- FBUtilities.getBroadcastAddress()));
+ FBUtilities.getBroadcastAddressAndPort()));
}
+ validateEndpointSnitch(epStates.values().iterator());
+
if (shouldBootstrap() && useStrictConsistency && !allowSimultaneousMoves())
{
- for (Map.Entry<InetAddress, EndpointState> entry : epStates.entrySet())
+ for (Map.Entry<InetAddressAndPort, EndpointState> entry : epStates.entrySet())
{
// ignore local node or empty status
- if (entry.getKey().equals(FBUtilities.getBroadcastAddress()) || entry.getValue().getApplicationState(ApplicationState.STATUS) == null)
+ if (entry.getKey().equals(FBUtilities.getBroadcastAddressAndPort()) || (entry.getValue().getApplicationState(ApplicationState.STATUS_WITH_PORT) == null & entry.getValue().getApplicationState(ApplicationState.STATUS) == null))
continue;
- String[] pieces = splitValue(entry.getValue().getApplicationState(ApplicationState.STATUS));
+
+ VersionedValue value = entry.getValue().getApplicationState(ApplicationState.STATUS_WITH_PORT);
+ if (value == null)
+ {
+ value = entry.getValue().getApplicationState(ApplicationState.STATUS);
+ }
+
+ String[] pieces = splitValue(value);
assert (pieces.length > 0);
String state = pieces[0];
if (state.equals(VersionedValue.STATUS_BOOTSTRAPPING) || state.equals(VersionedValue.STATUS_LEAVING) || state.equals(VersionedValue.STATUS_MOVING))
@@ -615,6 +556,28 @@
}
}
+ private static void validateEndpointSnitch(Iterator<EndpointState> endpointStates)
+ {
+ Set<String> datacenters = new HashSet<>();
+ Set<String> racks = new HashSet<>();
+ while (endpointStates.hasNext())
+ {
+ EndpointState state = endpointStates.next();
+ VersionedValue val = state.getApplicationState(ApplicationState.DC);
+ if (val != null)
+ datacenters.add(val.value);
+ val = state.getApplicationState(ApplicationState.RACK);
+ if (val != null)
+ racks.add(val.value);
+ }
+
+ IEndpointSnitch snitch = DatabaseDescriptor.getEndpointSnitch();
+ if (!snitch.validate(datacenters, racks))
+ {
+ throw new IllegalStateException();
+ }
+ }
+
private boolean allowSimultaneousMoves()
{
return allowSimultaneousMoves && DatabaseDescriptor.getNumTokens() == 1;
@@ -628,8 +591,7 @@
Gossiper.instance.register(this);
Gossiper.instance.start((int) (System.currentTimeMillis() / 1000)); // needed for node-ring gathering.
Gossiper.instance.addLocalApplicationState(ApplicationState.NET_VERSION, valueFactory.networkVersion());
- if (!MessagingService.instance().isListening())
- MessagingService.instance().listen();
+ MessagingService.instance().listen();
}
public void populateTokenMetadata()
@@ -637,10 +599,10 @@
if (Boolean.parseBoolean(System.getProperty("cassandra.load_ring_state", "true")))
{
logger.info("Populating token metadata from system tables");
- Multimap<InetAddress, Token> loadedTokens = SystemKeyspace.loadTokens();
+ Multimap<InetAddressAndPort, Token> loadedTokens = SystemKeyspace.loadTokens();
if (!shouldBootstrap()) // if we have not completed bootstrapping, we should not add ourselves as a normal token
- loadedTokens.putAll(FBUtilities.getBroadcastAddress(), SystemKeyspace.getSavedTokens());
- for (InetAddress ep : loadedTokens.keySet())
+ loadedTokens.putAll(FBUtilities.getBroadcastAddressAndPort(), SystemKeyspace.getSavedTokens());
+ for (InetAddressAndPort ep : loadedTokens.keySet())
tokenMetadata.updateNormalTokens(loadedTokens.get(ep), ep);
logger.info("Token metadata: {}", tokenMetadata);
@@ -655,9 +617,7 @@
public synchronized void initServer(int delay) throws ConfigurationException
{
logger.info("Cassandra version: {}", FBUtilities.getReleaseVersionString());
- logger.info("Thrift API version: {}", cassandraConstants.VERSION);
- logger.info("CQL supported versions: {} (default: {})",
- StringUtils.join(ClientState.getCQLSupportedVersion(), ", "), ClientState.DEFAULT_CQL_VERSION);
+ logger.info("CQL version: {}", QueryProcessor.CQL_VERSION);
logger.info("Native protocol supported versions: {} (default: {})",
StringUtils.join(ProtocolVersion.supportedVersions(), ", "), ProtocolVersion.CURRENT);
@@ -722,10 +682,11 @@
Collection<Token> tokens = SystemKeyspace.getSavedTokens();
if (!tokens.isEmpty())
{
- tokenMetadata.updateNormalTokens(tokens, FBUtilities.getBroadcastAddress());
+ tokenMetadata.updateNormalTokens(tokens, FBUtilities.getBroadcastAddressAndPort());
// order is important here, the gossiper can fire in between adding these two states. It's ok to send TOKENS without STATUS, but *not* vice versa.
List<Pair<ApplicationState, VersionedValue>> states = new ArrayList<Pair<ApplicationState, VersionedValue>>();
states.add(Pair.create(ApplicationState.TOKENS, valueFactory.tokens(tokens)));
+ states.add(Pair.create(ApplicationState.STATUS_WITH_PORT, valueFactory.hibernate(true)));
states.add(Pair.create(ApplicationState.STATUS, valueFactory.hibernate(true)));
Gossiper.instance.addLocalApplicationStates(states);
}
@@ -741,11 +702,11 @@
if (Boolean.parseBoolean(System.getProperty("cassandra.load_ring_state", "true")))
{
logger.info("Loading persisted ring state");
- Multimap<InetAddress, Token> loadedTokens = SystemKeyspace.loadTokens();
- Map<InetAddress, UUID> loadedHostIds = SystemKeyspace.loadHostIds();
- for (InetAddress ep : loadedTokens.keySet())
+ Multimap<InetAddressAndPort, Token> loadedTokens = SystemKeyspace.loadTokens();
+ Map<InetAddressAndPort, UUID> loadedHostIds = SystemKeyspace.loadHostIds();
+ for (InetAddressAndPort ep : loadedTokens.keySet())
{
- if (ep.equals(FBUtilities.getBroadcastAddress()))
+ if (ep.equals(FBUtilities.getBroadcastAddressAndPort()))
{
// entry has been mistakenly added, delete it
SystemKeyspace.removeEndpoint(ep);
@@ -789,11 +750,10 @@
public static boolean isSeed()
{
- return DatabaseDescriptor.getSeeds().contains(FBUtilities.getBroadcastAddress());
+ return DatabaseDescriptor.getSeeds().contains(FBUtilities.getBroadcastAddressAndPort());
}
- @VisibleForTesting
- public void prepareToJoin() throws ConfigurationException
+ private void prepareToJoin() throws ConfigurationException
{
if (!joined)
{
@@ -813,8 +773,7 @@
if (DatabaseDescriptor.getReplaceTokens().size() > 0 || DatabaseDescriptor.getReplaceNode() != null)
throw new RuntimeException("Replace method removed; use cassandra.replace_address instead");
- if (!MessagingService.instance().isListening())
- MessagingService.instance().listen();
+ MessagingService.instance().listen();
UUID localHostId = SystemKeyspace.getLocalHostId();
@@ -836,6 +795,7 @@
"the node to be replaced ({}). If the previous node has been down for longer than max_hint_window_in_ms, " +
"repair must be run after the replacement process in order to make this node consistent.",
DatabaseDescriptor.getReplaceAddress());
+ appStates.put(ApplicationState.STATUS_WITH_PORT, valueFactory.hibernate(true));
appStates.put(ApplicationState.STATUS, valueFactory.hibernate(true));
}
}
@@ -857,10 +817,11 @@
// for bootstrap to get the load info it needs.
// (we won't be part of the storage ring though until we add a counterId to our state, below.)
// Seed the host ID-to-endpoint map with our own ID.
- getTokenMetadata().updateHostId(localHostId, FBUtilities.getBroadcastAddress());
+ getTokenMetadata().updateHostId(localHostId, FBUtilities.getBroadcastAddressAndPort());
appStates.put(ApplicationState.NET_VERSION, valueFactory.networkVersion());
appStates.put(ApplicationState.HOST_ID, valueFactory.hostId(localHostId));
- appStates.put(ApplicationState.RPC_ADDRESS, valueFactory.rpcaddress(FBUtilities.getBroadcastRpcAddress()));
+ appStates.put(ApplicationState.NATIVE_ADDRESS_AND_PORT, valueFactory.nativeaddressAndPort(FBUtilities.getBroadcastNativeAddressAndPort()));
+ appStates.put(ApplicationState.RPC_ADDRESS, valueFactory.rpcaddress(FBUtilities.getJustBroadcastNativeAddress()));
appStates.put(ApplicationState.RELEASE_VERSION, valueFactory.releaseVersion());
// load the persisted ring state. This used to be done earlier in the init process,
@@ -888,9 +849,9 @@
for (int i = 0; i < delay; i += 1000)
{
// if we see schema, we can proceed to the next check directly
- if (!Schema.instance.getVersion().equals(SchemaConstants.emptyVersion))
+ if (!Schema.instance.isEmpty())
{
- logger.debug("got schema: {}", Schema.instance.getVersion());
+ logger.debug("current schema version: {}", Schema.instance.getVersion());
break;
}
Uninterruptibles.sleepUninterruptibly(1, TimeUnit.SECONDS);
@@ -905,9 +866,8 @@
}
}
- @VisibleForTesting
- public void joinTokenRing(int delay) throws ConfigurationException
-{
+ private void joinTokenRing(int delay) throws ConfigurationException
+ {
joined = true;
// We bootstrap if we haven't successfully bootstrapped before, as long as we are not a seed.
@@ -919,16 +879,16 @@
//
// We attempted to replace this with a schema-presence check, but you need a meaningful sleep
// to get schema info from gossip which defeats the purpose. See CASSANDRA-4427 for the gory details.
- Set<InetAddress> current = new HashSet<>();
+ Set<InetAddressAndPort> current = new HashSet<>();
if (logger.isDebugEnabled())
{
logger.debug("Bootstrap variables: {} {} {} {}",
DatabaseDescriptor.isAutoBootstrap(),
SystemKeyspace.bootstrapInProgress(),
SystemKeyspace.bootstrapComplete(),
- DatabaseDescriptor.getSeeds().contains(FBUtilities.getBroadcastAddress()));
+ DatabaseDescriptor.getSeeds().contains(FBUtilities.getBroadcastAddressAndPort()));
}
- if (DatabaseDescriptor.isAutoBootstrap() && !SystemKeyspace.bootstrapComplete() && DatabaseDescriptor.getSeeds().contains(FBUtilities.getBroadcastAddress()))
+ if (DatabaseDescriptor.isAutoBootstrap() && !SystemKeyspace.bootstrapComplete() && DatabaseDescriptor.getSeeds().contains(FBUtilities.getBroadcastAddressAndPort()))
{
logger.info("This node will not auto bootstrap because it is configured to be a seed node.");
}
@@ -953,8 +913,8 @@
if (useStrictConsistency && !allowSimultaneousMoves() &&
(
tokenMetadata.getBootstrapTokens().valueSet().size() > 0 ||
- tokenMetadata.getLeavingEndpoints().size() > 0 ||
- tokenMetadata.getMovingEndpoints().size() > 0
+ tokenMetadata.getSizeOfLeavingEndpoints() > 0 ||
+ tokenMetadata.getSizeOfMovingEndpoints() > 0
))
{
String bootstrapTokens = StringUtils.join(tokenMetadata.getBootstrapTokens().valueSet(), ',');
@@ -966,13 +926,13 @@
// get bootstrap tokens
if (!replacing)
{
- if (tokenMetadata.isMember(FBUtilities.getBroadcastAddress()))
+ if (tokenMetadata.isMember(FBUtilities.getBroadcastAddressAndPort()))
{
String s = "This node is already a member of the token ring; bootstrap aborted. (If replacing a dead node, remove the old one from the ring first.)";
throw new UnsupportedOperationException(s);
}
setMode(Mode.JOINING, "getting bootstrap token", true);
- bootstrapTokens = BootStrapper.getBootstrapTokens(tokenMetadata, FBUtilities.getBroadcastAddress(), delay);
+ bootstrapTokens = BootStrapper.getBootstrapTokens(tokenMetadata, FBUtilities.getBroadcastAddressAndPort(), delay);
}
else
{
@@ -992,7 +952,7 @@
// check for operator errors...
for (Token token : bootstrapTokens)
{
- InetAddress existing = tokenMetadata.getEndpoint(token);
+ InetAddressAndPort existing = tokenMetadata.getEndpoint(token);
if (existing != null)
{
long nanoDelay = delay * 1000000L;
@@ -1028,7 +988,7 @@
bootstrapTokens = SystemKeyspace.getSavedTokens();
if (bootstrapTokens.isEmpty())
{
- bootstrapTokens = BootStrapper.getBootstrapTokens(tokenMetadata, FBUtilities.getBroadcastAddress(), delay);
+ bootstrapTokens = BootStrapper.getBootstrapTokens(tokenMetadata, FBUtilities.getBroadcastAddressAndPort(), delay);
}
else
{
@@ -1050,7 +1010,7 @@
if (!current.isEmpty())
{
Gossiper.runInGossipStageBlocking(() -> {
- for (InetAddress existing : current)
+ for (InetAddressAndPort existing : current)
Gossiper.instance.replacedEndpoint(existing);
});
}
@@ -1072,20 +1032,20 @@
@VisibleForTesting
public void ensureTraceKeyspace()
{
- evolveSystemKeyspace(TraceKeyspace.metadata(), TraceKeyspace.GENERATION).ifPresent(MigrationManager::announceGlobally);
+ evolveSystemKeyspace(TraceKeyspace.metadata(), TraceKeyspace.GENERATION).ifPresent(MigrationManager::announce);
}
public static boolean isReplacingSameAddress()
{
- InetAddress replaceAddress = DatabaseDescriptor.getReplaceAddress();
- return replaceAddress != null && replaceAddress.equals(FBUtilities.getBroadcastAddress());
+ InetAddressAndPort replaceAddress = DatabaseDescriptor.getReplaceAddress();
+ return replaceAddress != null && replaceAddress.equals(FBUtilities.getBroadcastAddressAndPort());
}
public void gossipSnitchInfo()
{
IEndpointSnitch snitch = DatabaseDescriptor.getEndpointSnitch();
- String dc = snitch.getDatacenter(FBUtilities.getBroadcastAddress());
- String rack = snitch.getRack(FBUtilities.getBroadcastAddress());
+ String dc = snitch.getLocalDatacenter();
+ String rack = snitch.getLocalRack();
Gossiper.instance.addLocalApplicationState(ApplicationState.DC, StorageService.instance.valueFactory.datacenter(dc));
Gossiper.instance.addLocalApplicationState(ApplicationState.RACK, StorageService.instance.valueFactory.rack(rack));
}
@@ -1116,9 +1076,9 @@
// node can join the ring even if isBootstrapMode is true which should not happen
if (!isBootstrapMode())
{
- isSurveyMode = false;
logger.info("Leaving write survey mode and joining ring at operator request");
finishJoiningRing(resumedBootstrap, SystemKeyspace.getSavedTokens());
+ isSurveyMode = false;
daemon.start();
}
else
@@ -1157,12 +1117,13 @@
if (!authSetupCalled.getAndSet(true))
{
if (setUpSchema)
- evolveSystemKeyspace(AuthKeyspace.metadata(), AuthKeyspace.GENERATION).ifPresent(MigrationManager::announceGlobally);
+ evolveSystemKeyspace(AuthKeyspace.metadata(), AuthKeyspace.GENERATION).ifPresent(MigrationManager::announce);
DatabaseDescriptor.getRoleManager().setup();
DatabaseDescriptor.getAuthenticator().setup();
DatabaseDescriptor.getAuthorizer().setup();
- MigrationManager.instance.register(new AuthMigrationListener());
+ DatabaseDescriptor.getNetworkAuthorizer().setup();
+ Schema.instance.registerListener(new AuthSchemaChangeListener());
authSetupComplete = true;
}
}
@@ -1181,12 +1142,12 @@
evolveSystemKeyspace( AuthKeyspace.metadata(), AuthKeyspace.GENERATION).ifPresent(changes::add);
if (!changes.isEmpty())
- MigrationManager.announceGlobally(changes);
+ MigrationManager.announce(changes);
}
public boolean isJoined()
{
- return tokenMetadata.isMember(FBUtilities.getBroadcastAddress()) && !isSurveyMode;
+ return tokenMetadata.isMember(FBUtilities.getBroadcastAddressAndPort()) && !isSurveyMode;
}
public void rebuild(String sourceDc)
@@ -1216,24 +1177,24 @@
{
RangeStreamer streamer = new RangeStreamer(tokenMetadata,
null,
- FBUtilities.getBroadcastAddress(),
- "Rebuild",
+ FBUtilities.getBroadcastAddressAndPort(),
+ StreamOperation.REBUILD,
useStrictConsistency && !replacing,
DatabaseDescriptor.getEndpointSnitch(),
streamStateStore,
- false);
- streamer.addSourceFilter(new RangeStreamer.FailureDetectorSourceFilter(FailureDetector.instance));
+ false,
+ DatabaseDescriptor.getStreamingConnectionsPerHost());
if (sourceDc != null)
streamer.addSourceFilter(new RangeStreamer.SingleDatacenterFilter(DatabaseDescriptor.getEndpointSnitch(), sourceDc));
if (keyspace == null)
{
for (String keyspaceName : Schema.instance.getNonLocalStrategyKeyspaces())
- streamer.addRanges(keyspaceName, getLocalRanges(keyspaceName));
+ streamer.addRanges(keyspaceName, getLocalReplicas(keyspaceName));
}
else if (tokens == null)
{
- streamer.addRanges(keyspace, getLocalRanges(keyspace));
+ streamer.addRanges(keyspace, getLocalReplicas(keyspace));
}
else
{
@@ -1255,14 +1216,16 @@
}
// Ensure all specified ranges are actually ranges owned by this host
- Collection<Range<Token>> localRanges = getLocalRanges(keyspace);
+ RangesAtEndpoint localReplicas = getLocalReplicas(keyspace);
+ RangesAtEndpoint.Builder streamRanges = new RangesAtEndpoint.Builder(FBUtilities.getBroadcastAddressAndPort(), ranges.size());
for (Range<Token> specifiedRange : ranges)
{
boolean foundParentRange = false;
- for (Range<Token> localRange : localRanges)
+ for (Replica localReplica : localReplicas)
{
- if (localRange.contains(specifiedRange))
+ if (localReplica.contains(specifiedRange))
{
+ streamRanges.add(localReplica.decorateSubrange(specifiedRange));
foundParentRange = true;
break;
}
@@ -1276,13 +1239,13 @@
if (specificSources != null)
{
String[] stringHosts = specificSources.split(",");
- Set<InetAddress> sources = new HashSet<>(stringHosts.length);
+ Set<InetAddressAndPort> sources = new HashSet<>(stringHosts.length);
for (String stringHost : stringHosts)
{
try
{
- InetAddress endpoint = InetAddress.getByName(stringHost);
- if (FBUtilities.getBroadcastAddress().equals(endpoint))
+ InetAddressAndPort endpoint = InetAddressAndPort.getByName(stringHost);
+ if (FBUtilities.getBroadcastAddressAndPort().equals(endpoint))
{
throw new IllegalArgumentException("This host was specified as a source for rebuilding. Sources for a rebuild can only be other nodes in the cluster.");
}
@@ -1296,7 +1259,7 @@
streamer.addSourceFilter(new RangeStreamer.AllowedSourcesFilter(sources));
}
- streamer.addRanges(keyspace, ranges);
+ streamer.addRanges(keyspace, streamRanges.build());
}
StreamResultFuture resultFuture = streamer.fetchAsync();
@@ -1328,7 +1291,7 @@
public long getRpcTimeout()
{
- return DatabaseDescriptor.getRpcTimeout();
+ return DatabaseDescriptor.getRpcTimeout(MILLISECONDS);
}
public void setReadRpcTimeout(long value)
@@ -1339,7 +1302,7 @@
public long getReadRpcTimeout()
{
- return DatabaseDescriptor.getReadRpcTimeout();
+ return DatabaseDescriptor.getReadRpcTimeout(MILLISECONDS);
}
public void setRangeRpcTimeout(long value)
@@ -1350,7 +1313,7 @@
public long getRangeRpcTimeout()
{
- return DatabaseDescriptor.getRangeRpcTimeout();
+ return DatabaseDescriptor.getRangeRpcTimeout(MILLISECONDS);
}
public void setWriteRpcTimeout(long value)
@@ -1361,7 +1324,29 @@
public long getWriteRpcTimeout()
{
- return DatabaseDescriptor.getWriteRpcTimeout();
+ return DatabaseDescriptor.getWriteRpcTimeout(MILLISECONDS);
+ }
+
+ public void setInternodeTcpConnectTimeoutInMS(int value)
+ {
+ DatabaseDescriptor.setInternodeTcpConnectTimeoutInMS(value);
+ logger.info("set internode tcp connect timeout to {} ms", value);
+ }
+
+ public int getInternodeTcpConnectTimeoutInMS()
+ {
+ return DatabaseDescriptor.getInternodeTcpConnectTimeoutInMS();
+ }
+
+ public void setInternodeTcpUserTimeoutInMS(int value)
+ {
+ DatabaseDescriptor.setInternodeTcpUserTimeoutInMS(value);
+ logger.info("set internode tcp user timeout to {} ms", value);
+ }
+
+ public int getInternodeTcpUserTimeoutInMS()
+ {
+ return DatabaseDescriptor.getInternodeTcpUserTimeoutInMS();
}
public void setCounterWriteRpcTimeout(long value)
@@ -1372,7 +1357,7 @@
public long getCounterWriteRpcTimeout()
{
- return DatabaseDescriptor.getCounterWriteRpcTimeout();
+ return DatabaseDescriptor.getCounterWriteRpcTimeout(MILLISECONDS);
}
public void setCasContentionTimeout(long value)
@@ -1383,7 +1368,7 @@
public long getCasContentionTimeout()
{
- return DatabaseDescriptor.getCasContentionTimeout();
+ return DatabaseDescriptor.getCasContentionTimeout(MILLISECONDS);
}
public void setTruncateRpcTimeout(long value)
@@ -1394,18 +1379,7 @@
public long getTruncateRpcTimeout()
{
- return DatabaseDescriptor.getTruncateRpcTimeout();
- }
-
- public void setStreamingSocketTimeout(int value)
- {
- DatabaseDescriptor.setStreamingSocketTimeout(value);
- logger.info("set streaming socket timeout to {} ms", value);
- }
-
- public int getStreamingSocketTimeout()
- {
- return DatabaseDescriptor.getStreamingSocketTimeout();
+ return DatabaseDescriptor.getTruncateRpcTimeout(MILLISECONDS);
}
public void setStreamThroughputMbPerSec(int value)
@@ -1442,6 +1416,17 @@
CompactionManager.instance.setRate(value);
}
+ public int getBatchlogReplayThrottleInKB()
+ {
+ return DatabaseDescriptor.getBatchlogReplayThrottleInKB();
+ }
+
+ public void setBatchlogReplayThrottleInKB(int throttleInKB)
+ {
+ DatabaseDescriptor.setBatchlogReplayThrottleInKB(throttleInKB);
+ BatchlogManager.instance.setRate(throttleInKB);
+ }
+
public int getConcurrentCompactors()
{
return DatabaseDescriptor.getConcurrentCompactors();
@@ -1455,6 +1440,63 @@
CompactionManager.instance.setConcurrentCompactors(value);
}
+ public void bypassConcurrentValidatorsLimit()
+ {
+ logger.info("Enabling the ability to set concurrent validations to an unlimited value");
+ DatabaseDescriptor.allowUnlimitedConcurrentValidations = true ;
+ }
+
+ public void enforceConcurrentValidatorsLimit()
+ {
+ logger.info("Disabling the ability to set concurrent validations to an unlimited value");
+ DatabaseDescriptor.allowUnlimitedConcurrentValidations = false ;
+ }
+
+ public boolean isConcurrentValidatorsLimitEnforced()
+ {
+ return DatabaseDescriptor.allowUnlimitedConcurrentValidations;
+ }
+
+ public int getConcurrentValidators()
+ {
+ return DatabaseDescriptor.getConcurrentValidations();
+ }
+
+ public void setConcurrentValidators(int value)
+ {
+ int concurrentCompactors = DatabaseDescriptor.getConcurrentCompactors();
+ if (value > concurrentCompactors && !DatabaseDescriptor.allowUnlimitedConcurrentValidations)
+ throw new IllegalArgumentException(
+ String.format("Cannot set concurrent_validations greater than concurrent_compactors (%d)",
+ concurrentCompactors));
+
+ if (value <= 0)
+ {
+ logger.info("Using default value of concurrent_compactors ({}) for concurrent_validations", concurrentCompactors);
+ value = concurrentCompactors;
+ }
+ else
+ {
+ logger.info("Setting concurrent_validations to {}", value);
+ }
+
+ DatabaseDescriptor.setConcurrentValidations(value);
+ CompactionManager.instance.setConcurrentValidations();
+ }
+
+ public int getConcurrentViewBuilders()
+ {
+ return DatabaseDescriptor.getConcurrentViewBuilders();
+ }
+
+ public void setConcurrentViewBuilders(int value)
+ {
+ if (value <= 0)
+ throw new IllegalArgumentException("Number of concurrent view builders should be greater than 0.");
+ DatabaseDescriptor.setConcurrentViewBuilders(value);
+ CompactionManager.instance.setConcurrentViewBuilders(DatabaseDescriptor.getConcurrentViewBuilders());
+ }
+
public boolean isIncrementalBackupsEnabled()
{
return DatabaseDescriptor.isIncrementalBackupsEnabled();
@@ -1499,17 +1541,20 @@
// if not an existing token then bootstrap
List<Pair<ApplicationState, VersionedValue>> states = new ArrayList<>();
states.add(Pair.create(ApplicationState.TOKENS, valueFactory.tokens(tokens)));
+ states.add(Pair.create(ApplicationState.STATUS_WITH_PORT, replacing?
+ valueFactory.bootReplacingWithPort(DatabaseDescriptor.getReplaceAddress()) :
+ valueFactory.bootstrapping(tokens)));
states.add(Pair.create(ApplicationState.STATUS, replacing?
- valueFactory.bootReplacing(DatabaseDescriptor.getReplaceAddress()) :
+ valueFactory.bootReplacing(DatabaseDescriptor.getReplaceAddress().address) :
valueFactory.bootstrapping(tokens)));
Gossiper.instance.addLocalApplicationStates(states);
setMode(Mode.JOINING, "sleeping " + RING_DELAY + " ms for pending range setup", true);
- Uninterruptibles.sleepUninterruptibly(RING_DELAY, TimeUnit.MILLISECONDS);
+ Uninterruptibles.sleepUninterruptibly(RING_DELAY, MILLISECONDS);
}
else
{
// Dont set any state for the node which is bootstrapping the existing token...
- tokenMetadata.updateNormalTokens(tokens, FBUtilities.getBroadcastAddress());
+ tokenMetadata.updateNormalTokens(tokens, FBUtilities.getBroadcastAddressAndPort());
SystemKeyspace.removeEndpoint(DatabaseDescriptor.getReplaceAddress());
}
if (!Gossiper.instance.seenAnySeed())
@@ -1525,7 +1570,7 @@
invalidateDiskBoundaries();
setMode(Mode.JOINING, "Starting to bootstrap...", true);
- BootStrapper bootstrapper = new BootStrapper(FBUtilities.getBroadcastAddress(), tokens, tokenMetadata);
+ BootStrapper bootstrapper = new BootStrapper(FBUtilities.getBroadcastAddressAndPort(), tokens, tokenMetadata);
bootstrapper.addProgressListener(progressSupport);
ListenableFuture<StreamState> bootstrapStream = bootstrapper.bootstrap(streamStateStore, useStrictConsistency && !replacing); // handles token update
try
@@ -1562,8 +1607,8 @@
private void markViewsAsBuilt() {
for (String keyspace : Schema.instance.getUserKeyspaces())
{
- for (ViewDefinition view: Schema.instance.getKSMetaData(keyspace).views)
- SystemKeyspace.finishViewBuildStatus(view.ksName, view.viewName);
+ for (ViewMetadata view: Schema.instance.getKeyspaceMetadata(keyspace).views)
+ SystemKeyspace.finishViewBuildStatus(view.keyspace(), view.name());
}
}
@@ -1584,7 +1629,7 @@
// get bootstrap tokens saved in system keyspace
final Collection<Token> tokens = SystemKeyspace.getSavedTokens();
// already bootstrapped ranges are filtered during bootstrap
- BootStrapper bootstrapper = new BootStrapper(FBUtilities.getBroadcastAddress(), tokens, tokenMetadata);
+ BootStrapper bootstrapper = new BootStrapper(FBUtilities.getBroadcastAddressAndPort(), tokens, tokenMetadata);
bootstrapper.addProgressListener(progressSupport);
ListenableFuture<StreamState> bootstrapStream = bootstrapper.bootstrap(streamStateStore, useStrictConsistency && !replacing); // handles token update
Futures.addCallback(bootstrapStream, new FutureCallback<StreamState>()
@@ -1634,7 +1679,7 @@
progressSupport.progress("bootstrap", new ProgressEvent(ProgressEventType.ERROR, 1, 1, message));
progressSupport.progress("bootstrap", new ProgressEvent(ProgressEventType.COMPLETE, 1, 1, "Resume bootstrap complete"));
}
- });
+ }, MoreExecutors.directExecutor());
return true;
}
else
@@ -1644,6 +1689,21 @@
}
}
+ public Map<String,List<Integer>> getConcurrency(List<String> stageNames)
+ {
+ Stream<Stage> stageStream = stageNames.isEmpty() ? stream(Stage.values()) : stageNames.stream().map(Stage::fromPoolName);
+ return stageStream.collect(toMap(s -> s.jmxName,
+ s -> Arrays.asList(s.getCorePoolSize(), s.getMaximumPoolSize())));
+ }
+
+ public void setConcurrency(String threadPoolName, int newCorePoolSize, int newMaximumPoolSize)
+ {
+ Stage stage = Stage.fromPoolName(threadPoolName);
+ if (newCorePoolSize >= 0)
+ stage.setCorePoolSize(newCorePoolSize);
+ stage.setMaximumPoolSize(newMaximumPoolSize);
+ }
+
public boolean isBootstrapMode()
{
return isBootstrapMode;
@@ -1654,35 +1714,67 @@
return tokenMetadata;
}
+ public Map<List<String>, List<String>> getRangeToEndpointMap(String keyspace)
+ {
+ return getRangeToEndpointMap(keyspace, false);
+ }
+
+ public Map<List<String>, List<String>> getRangeToEndpointWithPortMap(String keyspace)
+ {
+ return getRangeToEndpointMap(keyspace, true);
+ }
+
/**
* for a keyspace, return the ranges and corresponding listen addresses.
* @param keyspace
* @return the endpoint map
*/
- public Map<List<String>, List<String>> getRangeToEndpointMap(String keyspace)
+ public Map<List<String>, List<String>> getRangeToEndpointMap(String keyspace, boolean withPort)
{
/* All the ranges for the tokens */
Map<List<String>, List<String>> map = new HashMap<>();
- for (Map.Entry<Range<Token>,List<InetAddress>> entry : getRangeToAddressMap(keyspace).entrySet())
+ for (Map.Entry<Range<Token>, EndpointsForRange> entry : getRangeToAddressMap(keyspace).entrySet())
{
- map.put(entry.getKey().asList(), stringify(entry.getValue()));
+ map.put(entry.getKey().asList(), Replicas.stringify(entry.getValue(), withPort));
}
return map;
}
/**
- * Return the rpc address associated with an endpoint as a string.
+ * Return the native address associated with an endpoint as a string.
* @param endpoint The endpoint to get rpc address for
- * @return the rpc address
+ * @return the native address
*/
- public String getRpcaddress(InetAddress endpoint)
+ public String getNativeaddress(InetAddressAndPort endpoint, boolean withPort)
{
- if (endpoint.equals(FBUtilities.getBroadcastAddress()))
- return FBUtilities.getBroadcastRpcAddress().getHostAddress();
+ if (endpoint.equals(FBUtilities.getBroadcastAddressAndPort()))
+ return FBUtilities.getBroadcastNativeAddressAndPort().toString(withPort);
+ else if (Gossiper.instance.getEndpointStateForEndpoint(endpoint).getApplicationState(ApplicationState.NATIVE_ADDRESS_AND_PORT) != null)
+ {
+ try
+ {
+ InetAddressAndPort address = InetAddressAndPort.getByName(Gossiper.instance.getEndpointStateForEndpoint(endpoint).getApplicationState(ApplicationState.NATIVE_ADDRESS_AND_PORT).value);
+ return address.getHostAddress(withPort);
+ }
+ catch (UnknownHostException e)
+ {
+ throw new RuntimeException(e);
+ }
+ }
else if (Gossiper.instance.getEndpointStateForEndpoint(endpoint).getApplicationState(ApplicationState.RPC_ADDRESS) == null)
- return endpoint.getHostAddress();
+ return endpoint.address.getHostAddress() + ":" + DatabaseDescriptor.getNativeTransportPort();
else
- return Gossiper.instance.getEndpointStateForEndpoint(endpoint).getApplicationState(ApplicationState.RPC_ADDRESS).value;
+ return Gossiper.instance.getEndpointStateForEndpoint(endpoint).getApplicationState(ApplicationState.RPC_ADDRESS).value + ":" + DatabaseDescriptor.getNativeTransportPort();
+ }
+
+ public Map<List<String>, List<String>> getRangeToRpcaddressMap(String keyspace)
+ {
+ return getRangeToNativeaddressMap(keyspace, false);
+ }
+
+ public Map<List<String>, List<String>> getRangeToNativeaddressWithPortMap(String keyspace)
+ {
+ return getRangeToNativeaddressMap(keyspace, true);
}
/**
@@ -1690,16 +1782,16 @@
* @param keyspace
* @return the endpoint map
*/
- public Map<List<String>, List<String>> getRangeToRpcaddressMap(String keyspace)
+ private Map<List<String>, List<String>> getRangeToNativeaddressMap(String keyspace, boolean withPort)
{
/* All the ranges for the tokens */
Map<List<String>, List<String>> map = new HashMap<>();
- for (Map.Entry<Range<Token>, List<InetAddress>> entry : getRangeToAddressMap(keyspace).entrySet())
+ for (Map.Entry<Range<Token>, EndpointsForRange> entry : getRangeToAddressMap(keyspace).entrySet())
{
List<String> rpcaddrs = new ArrayList<>(entry.getValue().size());
- for (InetAddress endpoint: entry.getValue())
+ for (Replica replicas: entry.getValue())
{
- rpcaddrs.add(getRpcaddress(endpoint));
+ rpcaddrs.add(getNativeaddress(replicas.endpoint(), withPort));
}
map.put(entry.getKey().asList(), rpcaddrs);
}
@@ -1708,44 +1800,47 @@
public Map<List<String>, List<String>> getPendingRangeToEndpointMap(String keyspace)
{
+ return getPendingRangeToEndpointMap(keyspace, false);
+ }
+
+ public Map<List<String>, List<String>> getPendingRangeToEndpointWithPortMap(String keyspace)
+ {
+ return getPendingRangeToEndpointMap(keyspace, true);
+ }
+
+ private Map<List<String>, List<String>> getPendingRangeToEndpointMap(String keyspace, boolean withPort)
+ {
// some people just want to get a visual representation of things. Allow null and set it to the first
// non-system keyspace.
if (keyspace == null)
keyspace = Schema.instance.getNonLocalStrategyKeyspaces().get(0);
Map<List<String>, List<String>> map = new HashMap<>();
- for (Map.Entry<Range<Token>, Collection<InetAddress>> entry : tokenMetadata.getPendingRangesMM(keyspace).asMap().entrySet())
+ for (Map.Entry<Range<Token>, EndpointsForRange> entry : tokenMetadata.getPendingRangesMM(keyspace).asMap().entrySet())
{
- List<InetAddress> l = new ArrayList<>(entry.getValue());
- map.put(entry.getKey().asList(), stringify(l));
+ map.put(entry.getKey().asList(), Replicas.stringify(entry.getValue(), withPort));
}
return map;
}
- public Map<Range<Token>, List<InetAddress>> getRangeToAddressMap(String keyspace)
+ public EndpointsByRange getRangeToAddressMap(String keyspace)
{
return getRangeToAddressMap(keyspace, tokenMetadata.sortedTokens());
}
- public Map<Range<Token>, List<InetAddress>> getRangeToAddressMapInLocalDC(String keyspace)
+ public EndpointsByRange getRangeToAddressMapInLocalDC(String keyspace)
{
- Predicate<InetAddress> isLocalDC = new Predicate<InetAddress>()
- {
- public boolean apply(InetAddress address)
- {
- return isLocalDC(address);
- }
- };
+ Predicate<Replica> isLocalDC = replica -> isLocalDC(replica.endpoint());
- Map<Range<Token>, List<InetAddress>> origMap = getRangeToAddressMap(keyspace, getTokensInLocalDC());
- Map<Range<Token>, List<InetAddress>> filteredMap = Maps.newHashMap();
- for (Map.Entry<Range<Token>, List<InetAddress>> entry : origMap.entrySet())
+ EndpointsByRange origMap = getRangeToAddressMap(keyspace, getTokensInLocalDC());
+ Map<Range<Token>, EndpointsForRange> filteredMap = Maps.newHashMap();
+ for (Map.Entry<Range<Token>, EndpointsForRange> entry : origMap.entrySet())
{
- List<InetAddress> endpointsInLocalDC = Lists.newArrayList(Collections2.filter(entry.getValue(), isLocalDC));
+ EndpointsForRange endpointsInLocalDC = entry.getValue().filter(isLocalDC);
filteredMap.put(entry.getKey(), endpointsInLocalDC);
}
- return filteredMap;
+ return new EndpointsByRange(filteredMap);
}
private List<Token> getTokensInLocalDC()
@@ -1753,21 +1848,21 @@
List<Token> filteredTokens = Lists.newArrayList();
for (Token token : tokenMetadata.sortedTokens())
{
- InetAddress endpoint = tokenMetadata.getEndpoint(token);
+ InetAddressAndPort endpoint = tokenMetadata.getEndpoint(token);
if (isLocalDC(endpoint))
filteredTokens.add(token);
}
return filteredTokens;
}
- private boolean isLocalDC(InetAddress targetHost)
+ private boolean isLocalDC(InetAddressAndPort targetHost)
{
String remoteDC = DatabaseDescriptor.getEndpointSnitch().getDatacenter(targetHost);
- String localDC = DatabaseDescriptor.getEndpointSnitch().getDatacenter(FBUtilities.getBroadcastAddress());
+ String localDC = DatabaseDescriptor.getEndpointSnitch().getLocalDatacenter();
return remoteDC.equals(localDC);
}
- private Map<Range<Token>, List<InetAddress>> getRangeToAddressMap(String keyspace, List<Token> sortedTokens)
+ private EndpointsByRange getRangeToAddressMap(String keyspace, List<Token> sortedTokens)
{
// some people just want to get a visual representation of things. Allow null and set it to the first
// non-system keyspace.
@@ -1779,6 +1874,16 @@
}
+ public List<String> describeRingJMX(String keyspace) throws IOException
+ {
+ return describeRingJMX(keyspace, false);
+ }
+
+ public List<String> describeRingWithPortJMX(String keyspace) throws IOException
+ {
+ return describeRingJMX(keyspace,true);
+ }
+
/**
* The same as {@code describeRing(String)} but converts TokenRange to the String for JMX compatibility
*
@@ -1786,12 +1891,12 @@
*
* @return a List of TokenRange(s) converted to String for the given keyspace
*/
- public List<String> describeRingJMX(String keyspace) throws IOException
+ private List<String> describeRingJMX(String keyspace, boolean withPort) throws IOException
{
List<TokenRange> tokenRanges;
try
{
- tokenRanges = describeRing(keyspace);
+ tokenRanges = describeRing(keyspace, false, withPort);
}
catch (InvalidRequestException e)
{
@@ -1800,7 +1905,7 @@
List<String> result = new ArrayList<>(tokenRanges.size());
for (TokenRange tokenRange : tokenRanges)
- result.add(tokenRange.toString());
+ result.add(tokenRange.toString(withPort));
return result;
}
@@ -1816,7 +1921,7 @@
*/
public List<TokenRange> describeRing(String keyspace) throws InvalidRequestException
{
- return describeRing(keyspace, false);
+ return describeRing(keyspace, false, false);
}
/**
@@ -1824,10 +1929,10 @@
*/
public List<TokenRange> describeLocalRing(String keyspace) throws InvalidRequestException
{
- return describeRing(keyspace, true);
+ return describeRing(keyspace, true, false);
}
- private List<TokenRange> describeRing(String keyspace, boolean includeOnlyLocalDC) throws InvalidRequestException
+ private List<TokenRange> describeRing(String keyspace, boolean includeOnlyLocalDC, boolean withPort) throws InvalidRequestException
{
if (!Schema.instance.getKeyspaces().contains(keyspace))
throw new InvalidRequestException("No such keyspace: " + keyspace);
@@ -1838,64 +1943,49 @@
List<TokenRange> ranges = new ArrayList<>();
Token.TokenFactory tf = getTokenFactory();
- Map<Range<Token>, List<InetAddress>> rangeToAddressMap =
+ EndpointsByRange rangeToAddressMap =
includeOnlyLocalDC
? getRangeToAddressMapInLocalDC(keyspace)
: getRangeToAddressMap(keyspace);
- for (Map.Entry<Range<Token>, List<InetAddress>> entry : rangeToAddressMap.entrySet())
- {
- Range<Token> range = entry.getKey();
- List<InetAddress> addresses = entry.getValue();
- List<String> endpoints = new ArrayList<>(addresses.size());
- List<String> rpc_endpoints = new ArrayList<>(addresses.size());
- List<EndpointDetails> epDetails = new ArrayList<>(addresses.size());
-
- for (InetAddress endpoint : addresses)
- {
- EndpointDetails details = new EndpointDetails();
- details.host = endpoint.getHostAddress();
- details.datacenter = DatabaseDescriptor.getEndpointSnitch().getDatacenter(endpoint);
- details.rack = DatabaseDescriptor.getEndpointSnitch().getRack(endpoint);
-
- endpoints.add(details.host);
- rpc_endpoints.add(getRpcaddress(endpoint));
-
- epDetails.add(details);
- }
-
- TokenRange tr = new TokenRange(tf.toString(range.left.getToken()), tf.toString(range.right.getToken()), endpoints)
- .setEndpoint_details(epDetails)
- .setRpc_endpoints(rpc_endpoints);
-
- ranges.add(tr);
- }
+ for (Map.Entry<Range<Token>, EndpointsForRange> entry : rangeToAddressMap.entrySet())
+ ranges.add(TokenRange.create(tf, entry.getKey(), ImmutableList.copyOf(entry.getValue().endpoints()), withPort));
return ranges;
}
public Map<String, String> getTokenToEndpointMap()
{
- Map<Token, InetAddress> mapInetAddress = tokenMetadata.getNormalAndBootstrappingTokenToEndpointMap();
+ return getTokenToEndpointMap(false);
+ }
+
+ public Map<String, String> getTokenToEndpointWithPortMap()
+ {
+ return getTokenToEndpointMap(true);
+ }
+
+ private Map<String, String> getTokenToEndpointMap(boolean withPort)
+ {
+ Map<Token, InetAddressAndPort> mapInetAddress = tokenMetadata.getNormalAndBootstrappingTokenToEndpointMap();
// in order to preserve tokens in ascending order, we use LinkedHashMap here
Map<String, String> mapString = new LinkedHashMap<>(mapInetAddress.size());
List<Token> tokens = new ArrayList<>(mapInetAddress.keySet());
Collections.sort(tokens);
for (Token token : tokens)
{
- mapString.put(token.toString(), mapInetAddress.get(token).getHostAddress());
+ mapString.put(token.toString(), mapInetAddress.get(token).getHostAddress(withPort));
}
return mapString;
}
public String getLocalHostId()
{
- return getTokenMetadata().getHostId(FBUtilities.getBroadcastAddress()).toString();
+ return getTokenMetadata().getHostId(FBUtilities.getBroadcastAddressAndPort()).toString();
}
public UUID getLocalHostUUID()
{
- return getTokenMetadata().getHostId(FBUtilities.getBroadcastAddress());
+ return getTokenMetadata().getHostId(FBUtilities.getBroadcastAddressAndPort());
}
public Map<String, String> getHostIdMap()
@@ -1903,19 +1993,40 @@
return getEndpointToHostId();
}
+
public Map<String, String> getEndpointToHostId()
{
+ return getEndpointToHostId(false);
+ }
+
+ public Map<String, String> getEndpointWithPortToHostId()
+ {
+ return getEndpointToHostId(true);
+ }
+
+ private Map<String, String> getEndpointToHostId(boolean withPort)
+ {
Map<String, String> mapOut = new HashMap<>();
- for (Map.Entry<InetAddress, UUID> entry : getTokenMetadata().getEndpointToHostIdMapForReading().entrySet())
- mapOut.put(entry.getKey().getHostAddress(), entry.getValue().toString());
+ for (Map.Entry<InetAddressAndPort, UUID> entry : getTokenMetadata().getEndpointToHostIdMapForReading().entrySet())
+ mapOut.put(entry.getKey().getHostAddress(withPort), entry.getValue().toString());
return mapOut;
}
public Map<String, String> getHostIdToEndpoint()
{
+ return getHostIdToEndpoint(false);
+ }
+
+ public Map<String, String> getHostIdToEndpointWithPort()
+ {
+ return getHostIdToEndpoint(true);
+ }
+
+ private Map<String, String> getHostIdToEndpoint(boolean withPort)
+ {
Map<String, String> mapOut = new HashMap<>();
- for (Map.Entry<InetAddress, UUID> entry : getTokenMetadata().getEndpointToHostIdMapForReading().entrySet())
- mapOut.put(entry.getValue().toString(), entry.getKey().getHostAddress());
+ for (Map.Entry<InetAddressAndPort, UUID> entry : getTokenMetadata().getEndpointToHostIdMapForReading().entrySet())
+ mapOut.put(entry.getValue().toString(), entry.getKey().getHostAddress(withPort));
return mapOut;
}
@@ -1925,17 +2036,16 @@
* @param ranges
* @return mapping of ranges to the replicas responsible for them.
*/
- private Map<Range<Token>, List<InetAddress>> constructRangeToEndpointMap(String keyspace, List<Range<Token>> ranges)
+ private EndpointsByRange constructRangeToEndpointMap(String keyspace, List<Range<Token>> ranges)
{
- Map<Range<Token>, List<InetAddress>> rangeToEndpointMap = new HashMap<>(ranges.size());
+ AbstractReplicationStrategy strategy = Keyspace.open(keyspace).getReplicationStrategy();
+ Map<Range<Token>, EndpointsForRange> rangeToEndpointMap = new HashMap<>(ranges.size());
for (Range<Token> range : ranges)
- {
- rangeToEndpointMap.put(range, Keyspace.open(keyspace).getReplicationStrategy().getNaturalEndpoints(range.right));
- }
- return rangeToEndpointMap;
+ rangeToEndpointMap.put(range, strategy.getNaturalReplicas(range.right));
+ return new EndpointsByRange(rangeToEndpointMap);
}
- public void beforeChange(InetAddress endpoint, EndpointState currentState, ApplicationState newStateKey, VersionedValue newValue)
+ public void beforeChange(InetAddressAndPort endpoint, EndpointState currentState, ApplicationState newStateKey, VersionedValue newValue)
{
// no-op
}
@@ -1972,9 +2082,9 @@
* Note: Any time a node state changes from STATUS_NORMAL, it will not be visible to new nodes. So it follows that
* you should never bootstrap a new node during a removenode, decommission or move.
*/
- public void onChange(InetAddress endpoint, ApplicationState state, VersionedValue value)
+ public void onChange(InetAddressAndPort endpoint, ApplicationState state, VersionedValue value)
{
- if (state == ApplicationState.STATUS)
+ if (state == ApplicationState.STATUS || state == ApplicationState.STATUS_WITH_PORT)
{
String[] pieces = splitValue(value);
assert (pieces.length > 0);
@@ -2021,24 +2131,34 @@
if (getTokenMetadata().isMember(endpoint))
{
- final ExecutorService executor = StageManager.getStage(Stage.MUTATION);
switch (state)
{
case RELEASE_VERSION:
- SystemKeyspace.updatePeerReleaseVersion(endpoint, value.value, this::refreshMaxNativeProtocolVersion, executor);
+ SystemKeyspace.updatePeerInfo(endpoint, "release_version", value.value);
break;
case DC:
updateTopology(endpoint);
- SystemKeyspace.updatePeerInfo(endpoint, "data_center", value.value, executor);
+ SystemKeyspace.updatePeerInfo(endpoint, "data_center", value.value);
break;
case RACK:
updateTopology(endpoint);
- SystemKeyspace.updatePeerInfo(endpoint, "rack", value.value, executor);
+ SystemKeyspace.updatePeerInfo(endpoint, "rack", value.value);
break;
case RPC_ADDRESS:
try
{
- SystemKeyspace.updatePeerInfo(endpoint, "rpc_address", InetAddress.getByName(value.value), executor);
+ SystemKeyspace.updatePeerInfo(endpoint, "rpc_address", InetAddress.getByName(value.value));
+ }
+ catch (UnknownHostException e)
+ {
+ throw new RuntimeException(e);
+ }
+ break;
+ case NATIVE_ADDRESS_AND_PORT:
+ try
+ {
+ InetAddressAndPort address = InetAddressAndPort.getByName(value.value);
+ SystemKeyspace.updatePeerNativeAddress(endpoint, address);
}
catch (UnknownHostException e)
{
@@ -2046,11 +2166,11 @@
}
break;
case SCHEMA:
- SystemKeyspace.updatePeerInfo(endpoint, "schema_version", UUID.fromString(value.value), executor);
+ SystemKeyspace.updatePeerInfo(endpoint, "schema_version", UUID.fromString(value.value));
MigrationManager.instance.scheduleSchemaPull(endpoint, epState);
break;
case HOST_ID:
- SystemKeyspace.updatePeerInfo(endpoint, "host_id", UUID.fromString(value.value), executor);
+ SystemKeyspace.updatePeerInfo(endpoint, "host_id", UUID.fromString(value.value));
break;
case RPC_READY:
notifyRpcChange(endpoint, epState.isRpcReady());
@@ -2068,11 +2188,11 @@
return value.value.split(VersionedValue.DELIMITER_STR, -1);
}
- private void updateNetVersion(InetAddress endpoint, VersionedValue value)
+ private void updateNetVersion(InetAddressAndPort endpoint, VersionedValue value)
{
try
{
- MessagingService.instance().setVersion(endpoint, Integer.parseInt(value.value));
+ MessagingService.instance().versions.set(endpoint, Integer.parseInt(value.value));
}
catch (NumberFormatException e)
{
@@ -2080,7 +2200,7 @@
}
}
- public void updateTopology(InetAddress endpoint)
+ public void updateTopology(InetAddressAndPort endpoint)
{
if (getTokenMetadata().isMember(endpoint))
{
@@ -2093,27 +2213,41 @@
getTokenMetadata().updateTopology();
}
- private void updatePeerInfo(InetAddress endpoint)
+ private void updatePeerInfo(InetAddressAndPort endpoint)
{
EndpointState epState = Gossiper.instance.getEndpointStateForEndpoint(endpoint);
- final ExecutorService executor = StageManager.getStage(Stage.MUTATION);
+ InetAddress native_address = null;
+ int native_port = DatabaseDescriptor.getNativeTransportPort();
+
for (Map.Entry<ApplicationState, VersionedValue> entry : epState.states())
{
switch (entry.getKey())
{
case RELEASE_VERSION:
- SystemKeyspace.updatePeerReleaseVersion(endpoint, entry.getValue().value, this::refreshMaxNativeProtocolVersion, executor);
+ SystemKeyspace.updatePeerInfo(endpoint, "release_version", entry.getValue().value);
break;
case DC:
- SystemKeyspace.updatePeerInfo(endpoint, "data_center", entry.getValue().value, executor);
+ SystemKeyspace.updatePeerInfo(endpoint, "data_center", entry.getValue().value);
break;
case RACK:
- SystemKeyspace.updatePeerInfo(endpoint, "rack", entry.getValue().value, executor);
+ SystemKeyspace.updatePeerInfo(endpoint, "rack", entry.getValue().value);
break;
case RPC_ADDRESS:
try
{
- SystemKeyspace.updatePeerInfo(endpoint, "rpc_address", InetAddress.getByName(entry.getValue().value), executor);
+ native_address = InetAddress.getByName(entry.getValue().value);
+ }
+ catch (UnknownHostException e)
+ {
+ throw new RuntimeException(e);
+ }
+ break;
+ case NATIVE_ADDRESS_AND_PORT:
+ try
+ {
+ InetAddressAndPort address = InetAddressAndPort.getByName(entry.getValue().value);
+ native_address = address.address;
+ native_port = address.port;
}
catch (UnknownHostException e)
{
@@ -2121,16 +2255,24 @@
}
break;
case SCHEMA:
- SystemKeyspace.updatePeerInfo(endpoint, "schema_version", UUID.fromString(entry.getValue().value), executor);
+ SystemKeyspace.updatePeerInfo(endpoint, "schema_version", UUID.fromString(entry.getValue().value));
break;
case HOST_ID:
- SystemKeyspace.updatePeerInfo(endpoint, "host_id", UUID.fromString(entry.getValue().value), executor);
+ SystemKeyspace.updatePeerInfo(endpoint, "host_id", UUID.fromString(entry.getValue().value));
break;
}
}
+
+ //Some tests won't set all the states
+ if (native_address != null)
+ {
+ SystemKeyspace.updatePeerNativeAddress(endpoint,
+ InetAddressAndPort.getByAddressOverrideDefaults(native_address,
+ native_port));
+ }
}
- private void notifyRpcChange(InetAddress endpoint, boolean ready)
+ private void notifyRpcChange(InetAddressAndPort endpoint, boolean ready)
{
if (ready)
notifyUp(endpoint);
@@ -2138,7 +2280,7 @@
notifyDown(endpoint);
}
- private void notifyUp(InetAddress endpoint)
+ private void notifyUp(InetAddressAndPort endpoint)
{
if (!isRpcReady(endpoint) || !Gossiper.instance.isAlive(endpoint))
return;
@@ -2147,13 +2289,13 @@
subscriber.onUp(endpoint);
}
- private void notifyDown(InetAddress endpoint)
+ private void notifyDown(InetAddressAndPort endpoint)
{
for (IEndpointLifecycleSubscriber subscriber : lifecycleSubscribers)
subscriber.onDown(endpoint);
}
- private void notifyJoined(InetAddress endpoint)
+ private void notifyJoined(InetAddressAndPort endpoint)
{
if (!isStatus(endpoint, VersionedValue.STATUS_NORMAL))
return;
@@ -2162,28 +2304,26 @@
subscriber.onJoinCluster(endpoint);
}
- private void notifyMoved(InetAddress endpoint)
+ private void notifyMoved(InetAddressAndPort endpoint)
{
for (IEndpointLifecycleSubscriber subscriber : lifecycleSubscribers)
subscriber.onMove(endpoint);
}
- private void notifyLeft(InetAddress endpoint)
+ private void notifyLeft(InetAddressAndPort endpoint)
{
for (IEndpointLifecycleSubscriber subscriber : lifecycleSubscribers)
subscriber.onLeaveCluster(endpoint);
}
- private boolean isStatus(InetAddress endpoint, String status)
+ private boolean isStatus(InetAddressAndPort endpoint, String status)
{
EndpointState state = Gossiper.instance.getEndpointStateForEndpoint(endpoint);
return state != null && state.getStatus().equals(status);
}
- public boolean isRpcReady(InetAddress endpoint)
+ public boolean isRpcReady(InetAddressAndPort endpoint)
{
- if (MessagingService.instance().getVersion(endpoint) < MessagingService.VERSION_22)
- return true;
EndpointState state = Gossiper.instance.getEndpointStateForEndpoint(endpoint);
return state != null && state.isRpcReady();
}
@@ -2198,7 +2338,7 @@
*/
public void setRpcReady(boolean value)
{
- EndpointState state = Gossiper.instance.getEndpointStateForEndpoint(FBUtilities.getBroadcastAddress());
+ EndpointState state = Gossiper.instance.getEndpointStateForEndpoint(FBUtilities.getBroadcastAddressAndPort());
// if value is false we're OK with a null state, if it is true we are not.
assert !value || state != null;
@@ -2206,7 +2346,7 @@
Gossiper.instance.addLocalApplicationState(ApplicationState.RPC_READY, valueFactory.rpcReady(value));
}
- private Collection<Token> getTokensFor(InetAddress endpoint)
+ private Collection<Token> getTokensFor(InetAddressAndPort endpoint)
{
try
{
@@ -2231,7 +2371,7 @@
*
* @param endpoint bootstrapping node
*/
- private void handleStateBootstrap(InetAddress endpoint)
+ private void handleStateBootstrap(InetAddressAndPort endpoint)
{
Collection<Token> tokens;
// explicitly check for TOKENS, because a bootstrapping node might be bootstrapping in legacy mode; that is, not using vnodes and no token specified
@@ -2261,12 +2401,12 @@
tokenMetadata.updateHostId(Gossiper.instance.getHostId(endpoint), endpoint);
}
- private void handleStateBootreplacing(InetAddress newNode, String[] pieces)
+ private void handleStateBootreplacing(InetAddressAndPort newNode, String[] pieces)
{
- InetAddress oldNode;
+ InetAddressAndPort oldNode;
try
{
- oldNode = InetAddress.getByName(pieces[1]);
+ oldNode = InetAddressAndPort.getByName(pieces[1]);
}
catch (Exception e)
{
@@ -2279,7 +2419,7 @@
throw new RuntimeException(String.format("Node %s is trying to replace alive node %s.", newNode, oldNode));
}
- Optional<InetAddress> replacingNode = tokenMetadata.getReplacingNode(newNode);
+ Optional<InetAddressAndPort> replacingNode = tokenMetadata.getReplacingNode(newNode);
if (replacingNode.isPresent() && !replacingNode.get().equals(oldNode))
{
throw new RuntimeException(String.format("Node %s is already replacing %s but is trying to replace %s.",
@@ -2297,7 +2437,7 @@
tokenMetadata.updateHostId(Gossiper.instance.getHostId(newNode), newNode);
}
- private void ensureUpToDateTokenMetadata(String status, InetAddress endpoint)
+ private void ensureUpToDateTokenMetadata(String status, InetAddressAndPort endpoint)
{
Set<Token> tokens = new TreeSet<>(getTokensFor(endpoint));
@@ -2319,12 +2459,12 @@
}
}
- private void updateTokenMetadata(InetAddress endpoint, Iterable<Token> tokens)
+ private void updateTokenMetadata(InetAddressAndPort endpoint, Iterable<Token> tokens)
{
updateTokenMetadata(endpoint, tokens, new HashSet<>());
}
- private void updateTokenMetadata(InetAddress endpoint, Iterable<Token> tokens, Set<InetAddress> endpointsToRemove)
+ private void updateTokenMetadata(InetAddressAndPort endpoint, Iterable<Token> tokens, Set<InetAddressAndPort> endpointsToRemove)
{
Set<Token> tokensToUpdateInMetadata = new HashSet<>();
Set<Token> tokensToUpdateInSystemKeyspace = new HashSet<>();
@@ -2332,7 +2472,7 @@
for (final Token token : tokens)
{
// we don't want to update if this node is responsible for the token and it has a later startup time than endpoint.
- InetAddress currentOwner = tokenMetadata.getEndpoint(token);
+ InetAddressAndPort currentOwner = tokenMetadata.getEndpoint(token);
if (currentOwner == null)
{
logger.debug("New node {} at token {}", endpoint, token);
@@ -2352,7 +2492,7 @@
// currentOwner is no longer current, endpoint is. Keep track of these moves, because when
// a host no longer has any tokens, we'll want to remove it.
- Multimap<InetAddress, Token> epToTokenCopy = getTokenMetadata().getEndpointToTokenMapForReading();
+ Multimap<InetAddressAndPort, Token> epToTokenCopy = getTokenMetadata().getEndpointToTokenMapForReading();
epToTokenCopy.get(currentOwner).remove(token);
if (epToTokenCopy.get(currentOwner).isEmpty())
endpointsToRemove.add(currentOwner);
@@ -2366,26 +2506,25 @@
}
tokenMetadata.updateNormalTokens(tokensToUpdateInMetadata, endpoint);
- for (InetAddress ep : endpointsToRemove)
+ for (InetAddressAndPort ep : endpointsToRemove)
{
removeEndpoint(ep);
if (replacing && ep.equals(DatabaseDescriptor.getReplaceAddress()))
Gossiper.instance.replacementQuarantine(ep); // quarantine locally longer than normally; see CASSANDRA-8260
}
if (!tokensToUpdateInSystemKeyspace.isEmpty())
- SystemKeyspace.updateTokens(endpoint, tokensToUpdateInSystemKeyspace, StageManager.getStage(Stage.MUTATION));
+ SystemKeyspace.updateTokens(endpoint, tokensToUpdateInSystemKeyspace);
}
-
/**
* Handle node move to normal state. That is, node is entering token ring and participating
* in reads.
*
* @param endpoint node
*/
- private void handleStateNormal(final InetAddress endpoint, final String status)
+ private void handleStateNormal(final InetAddressAndPort endpoint, final String status)
{
Collection<Token> tokens = getTokensFor(endpoint);
- Set<InetAddress> endpointsToRemove = new HashSet<>();
+ Set<InetAddressAndPort> endpointsToRemove = new HashSet<>();
if (logger.isDebugEnabled())
logger.debug("Node {} state {}, token {}", endpoint, status, tokens);
@@ -2398,7 +2537,7 @@
endpoint,
Gossiper.instance.getEndpointStateForEndpoint(endpoint));
- Optional<InetAddress> replacingNode = tokenMetadata.getReplacingNode(endpoint);
+ Optional<InetAddressAndPort> replacingNode = tokenMetadata.getReplacingNode(endpoint);
if (replacingNode.isPresent())
{
assert !endpoint.equals(replacingNode.get()) : "Pending replacement endpoint with same address is not supported";
@@ -2411,7 +2550,7 @@
endpointsToRemove.add(replacingNode.get());
}
- Optional<InetAddress> replacementNode = tokenMetadata.getReplacementNode(endpoint);
+ Optional<InetAddressAndPort> replacementNode = tokenMetadata.getReplacementNode(endpoint);
if (replacementNode.isPresent())
{
logger.warn("Node {} is currently being replaced by node {}.", endpoint, replacementNode.get());
@@ -2420,7 +2559,7 @@
updatePeerInfo(endpoint);
// Order Matters, TM.updateHostID() should be called before TM.updateNormalToken(), (see CASSANDRA-4300).
UUID hostId = Gossiper.instance.getHostId(endpoint);
- InetAddress existing = tokenMetadata.getEndpointForHostId(hostId);
+ InetAddressAndPort existing = tokenMetadata.getEndpointForHostId(hostId);
if (replacing && isReplacingSameAddress() && Gossiper.instance.getEndpointStateForEndpoint(DatabaseDescriptor.getReplaceAddress()) != null
&& (hostId.equals(Gossiper.instance.getHostId(DatabaseDescriptor.getReplaceAddress()))))
logger.warn("Not updating token metadata for {} because I am replacing it", endpoint);
@@ -2428,7 +2567,7 @@
{
if (existing != null && !existing.equals(endpoint))
{
- if (existing.equals(FBUtilities.getBroadcastAddress()))
+ if (existing.equals(FBUtilities.getBroadcastAddressAndPort()))
{
logger.warn("Not updating host ID {} for {} because it's mine", hostId, endpoint);
tokenMetadata.removeEndpoint(endpoint);
@@ -2476,7 +2615,7 @@
*
* @param endpoint node
*/
- private void handleStateLeaving(InetAddress endpoint)
+ private void handleStateLeaving(InetAddressAndPort endpoint)
{
// If the node is previously unknown or tokens do not match, update tokenmetadata to
// have this node as 'normal' (it must have been using this token before the
@@ -2496,7 +2635,7 @@
* @param endpoint If reason for leaving is decommission, endpoint is the leaving node.
* @param pieces STATE_LEFT,token
*/
- private void handleStateLeft(InetAddress endpoint, String[] pieces)
+ private void handleStateLeft(InetAddressAndPort endpoint, String[] pieces)
{
assert pieces.length >= 2;
Collection<Token> tokens = getTokensFor(endpoint);
@@ -2513,7 +2652,7 @@
* @param endpoint moving endpoint address
* @param pieces STATE_MOVING, token
*/
- private void handleStateMoving(InetAddress endpoint, String[] pieces)
+ private void handleStateMoving(InetAddressAndPort endpoint, String[] pieces)
{
ensureUpToDateTokenMetadata(VersionedValue.STATUS_MOVING, endpoint);
@@ -2534,11 +2673,11 @@
* @param endpoint node
* @param pieces either REMOVED_TOKEN (node is gone) or REMOVING_TOKEN (replicas need to be restored)
*/
- private void handleStateRemoving(InetAddress endpoint, String[] pieces)
+ private void handleStateRemoving(InetAddressAndPort endpoint, String[] pieces)
{
assert (pieces.length > 0);
- if (endpoint.equals(FBUtilities.getBroadcastAddress()))
+ if (endpoint.equals(FBUtilities.getBroadcastAddressAndPort()))
{
logger.info("Received removenode gossip about myself. Is this node rejoining after an explicit removenode?");
try
@@ -2586,7 +2725,7 @@
}
}
- private void excise(Collection<Token> tokens, InetAddress endpoint)
+ private void excise(Collection<Token> tokens, InetAddressAndPort endpoint)
{
logger.info("Removing tokens {} for {}", tokens, endpoint);
@@ -2595,8 +2734,8 @@
{
// enough time for writes to expire and MessagingService timeout reporter callback to fire, which is where
// hints are mostly written from - using getMinRpcTimeout() / 2 for the interval.
- long delay = DatabaseDescriptor.getMinRpcTimeout() + DatabaseDescriptor.getWriteRpcTimeout();
- ScheduledExecutors.optionalTasks.schedule(() -> HintsService.instance.excise(hostId), delay, TimeUnit.MILLISECONDS);
+ long delay = DatabaseDescriptor.getMinRpcTimeout(MILLISECONDS) + DatabaseDescriptor.getWriteRpcTimeout(MILLISECONDS);
+ ScheduledExecutors.optionalTasks.schedule(() -> HintsService.instance.excise(hostId), delay, MILLISECONDS);
}
removeEndpoint(endpoint);
@@ -2607,20 +2746,20 @@
PendingRangeCalculatorService.instance.update();
}
- private void excise(Collection<Token> tokens, InetAddress endpoint, long expireTime)
+ private void excise(Collection<Token> tokens, InetAddressAndPort endpoint, long expireTime)
{
addExpireTimeIfFound(endpoint, expireTime);
excise(tokens, endpoint);
}
/** unlike excise we just need this endpoint gone without going through any notifications **/
- private void removeEndpoint(InetAddress endpoint)
+ private void removeEndpoint(InetAddressAndPort endpoint)
{
Gossiper.runInGossipStageBlocking(() -> Gossiper.instance.removeEndpoint(endpoint));
SystemKeyspace.removeEndpoint(endpoint);
}
- protected void addExpireTimeIfFound(InetAddress endpoint, long expireTime)
+ protected void addExpireTimeIfFound(InetAddressAndPort endpoint, long expireTime)
{
if (expireTime != 0L)
{
@@ -2637,32 +2776,56 @@
* Finds living endpoints responsible for the given ranges
*
* @param keyspaceName the keyspace ranges belong to
- * @param ranges the ranges to find sources for
+ * @param leavingReplicas the ranges to find sources for
* @return multimap of addresses to ranges the address is responsible for
*/
- private Multimap<InetAddress, Range<Token>> getNewSourceRanges(String keyspaceName, Set<Range<Token>> ranges)
+ private Multimap<InetAddressAndPort, FetchReplica> getNewSourceReplicas(String keyspaceName, Set<LeavingReplica> leavingReplicas)
{
- InetAddress myAddress = FBUtilities.getBroadcastAddress();
- Multimap<Range<Token>, InetAddress> rangeAddresses = Keyspace.open(keyspaceName).getReplicationStrategy().getRangeAddresses(tokenMetadata.cloneOnlyTokenMap());
- Multimap<InetAddress, Range<Token>> sourceRanges = HashMultimap.create();
+ InetAddressAndPort myAddress = FBUtilities.getBroadcastAddressAndPort();
+ EndpointsByRange rangeReplicas = Keyspace.open(keyspaceName).getReplicationStrategy().getRangeAddresses(tokenMetadata.cloneOnlyTokenMap());
+ Multimap<InetAddressAndPort, FetchReplica> sourceRanges = HashMultimap.create();
IFailureDetector failureDetector = FailureDetector.instance;
+ logger.debug("Getting new source replicas for {}", leavingReplicas);
+
// find alive sources for our new ranges
- for (Range<Token> range : ranges)
+ for (LeavingReplica leaver : leavingReplicas)
{
- Collection<InetAddress> possibleRanges = rangeAddresses.get(range);
+ //We need this to find the replicas from before leaving to supply the data
+ Replica leavingReplica = leaver.leavingReplica;
+ //We need this to know what to fetch and what the transient status is
+ Replica ourReplica = leaver.ourReplica;
+ //If we are going to be a full replica only consider full replicas
+ Predicate<Replica> replicaFilter = ourReplica.isFull() ? Replica::isFull : Predicates.alwaysTrue();
+ Predicate<Replica> notSelf = replica -> !replica.endpoint().equals(myAddress);
+ EndpointsForRange possibleReplicas = rangeReplicas.get(leavingReplica.range());
+ logger.info("Possible replicas for newReplica {} are {}", ourReplica, possibleReplicas);
IEndpointSnitch snitch = DatabaseDescriptor.getEndpointSnitch();
- List<InetAddress> sources = snitch.getSortedListByProximity(myAddress, possibleRanges);
+ EndpointsForRange sortedPossibleReplicas = snitch.sortedByProximity(myAddress, possibleReplicas);
+ logger.info("Sorted possible replicas starts as {}", sortedPossibleReplicas);
+ Optional<Replica> myCurrentReplica = tryFind(possibleReplicas, replica -> replica.endpoint().equals(myAddress)).toJavaUtil();
- assert (!sources.contains(myAddress));
+ boolean transientToFull = myCurrentReplica.isPresent() && myCurrentReplica.get().isTransient() && ourReplica.isFull();
+ assert !sortedPossibleReplicas.endpoints().contains(myAddress) || transientToFull : String.format("My address %s, sortedPossibleReplicas %s, myCurrentReplica %s, myNewReplica %s", myAddress, sortedPossibleReplicas, myCurrentReplica, ourReplica);
- for (InetAddress source : sources)
+ //Originally this didn't log if it couldn't restore replication and that seems wrong
+ boolean foundLiveReplica = false;
+ for (Replica possibleReplica : sortedPossibleReplicas.filter(Predicates.and(replicaFilter, notSelf)))
{
- if (failureDetector.isAlive(source))
+ if (failureDetector.isAlive(possibleReplica.endpoint()))
{
- sourceRanges.put(source, range);
+ foundLiveReplica = true;
+ sourceRanges.put(possibleReplica.endpoint(), new FetchReplica(ourReplica, possibleReplica));
break;
}
+ else
+ {
+ logger.debug("Skipping down replica {}", possibleReplica);
+ }
+ }
+ if (!foundLiveReplica)
+ {
+ logger.warn("Didn't find live replica to restore replication for " + ourReplica);
}
}
return sourceRanges;
@@ -2673,25 +2836,68 @@
*
* @param remote node to send notification to
*/
- private void sendReplicationNotification(InetAddress remote)
+ private void sendReplicationNotification(InetAddressAndPort remote)
{
// notify the remote token
- MessageOut msg = new MessageOut(MessagingService.Verb.REPLICATION_FINISHED);
+ Message msg = Message.out(REPLICATION_DONE_REQ, noPayload);
IFailureDetector failureDetector = FailureDetector.instance;
if (logger.isDebugEnabled())
logger.debug("Notifying {} of replication completion\n", remote);
while (failureDetector.isAlive(remote))
{
- AsyncOneResponse iar = MessagingService.instance().sendRR(msg, remote);
- try
- {
- iar.get(DatabaseDescriptor.getRpcTimeout(), TimeUnit.MILLISECONDS);
- return; // done
- }
- catch(TimeoutException e)
- {
- // try again
- }
+ AsyncOneResponse ior = new AsyncOneResponse();
+ MessagingService.instance().sendWithCallback(msg, remote, ior);
+
+ if (!ior.awaitUninterruptibly(DatabaseDescriptor.getRpcTimeout(NANOSECONDS), NANOSECONDS))
+ continue; // try again if we timeout
+
+ if (!ior.isSuccess())
+ throw new AssertionError(ior.cause());
+
+ return;
+ }
+ }
+
+ private static class LeavingReplica
+ {
+ //The node that is leaving
+ private final Replica leavingReplica;
+
+ //Our range and transient status
+ private final Replica ourReplica;
+
+ public LeavingReplica(Replica leavingReplica, Replica ourReplica)
+ {
+ Preconditions.checkNotNull(leavingReplica);
+ Preconditions.checkNotNull(ourReplica);
+ this.leavingReplica = leavingReplica;
+ this.ourReplica = ourReplica;
+ }
+
+ public boolean equals(Object o)
+ {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ LeavingReplica that = (LeavingReplica) o;
+
+ if (!leavingReplica.equals(that.leavingReplica)) return false;
+ return ourReplica.equals(that.ourReplica);
+ }
+
+ public int hashCode()
+ {
+ int result = leavingReplica.hashCode();
+ result = 31 * result + ourReplica.hashCode();
+ return result;
+ }
+
+ public String toString()
+ {
+ return "LeavingReplica{" +
+ "leavingReplica=" + leavingReplica +
+ ", ourReplica=" + ourReplica +
+ '}';
}
}
@@ -2705,41 +2911,54 @@
*
* @param endpoint the node that left
*/
- private void restoreReplicaCount(InetAddress endpoint, final InetAddress notifyEndpoint)
+ private void restoreReplicaCount(InetAddressAndPort endpoint, final InetAddressAndPort notifyEndpoint)
{
- Multimap<String, Map.Entry<InetAddress, Collection<Range<Token>>>> rangesToFetch = HashMultimap.create();
+ Map<String, Multimap<InetAddressAndPort, FetchReplica>> replicasToFetch = new HashMap<>();
- InetAddress myAddress = FBUtilities.getBroadcastAddress();
+ InetAddressAndPort myAddress = FBUtilities.getBroadcastAddressAndPort();
for (String keyspaceName : Schema.instance.getNonLocalStrategyKeyspaces())
{
- Multimap<Range<Token>, InetAddress> changedRanges = getChangedRangesForLeaving(keyspaceName, endpoint);
- Set<Range<Token>> myNewRanges = new HashSet<>();
- for (Map.Entry<Range<Token>, InetAddress> entry : changedRanges.entries())
+ logger.debug("Restoring replica count for keyspace {}", keyspaceName);
+ EndpointsByReplica changedReplicas = getChangedReplicasForLeaving(keyspaceName, endpoint, tokenMetadata, Keyspace.open(keyspaceName).getReplicationStrategy());
+ Set<LeavingReplica> myNewReplicas = new HashSet<>();
+ for (Map.Entry<Replica, Replica> entry : changedReplicas.flattenEntries())
{
- if (entry.getValue().equals(myAddress))
- myNewRanges.add(entry.getKey());
+ Replica replica = entry.getValue();
+ if (replica.endpoint().equals(myAddress))
+ {
+ //Maybe we don't technically need to fetch transient data from somewhere
+ //but it's probably not a lot and it probably makes things a hair more resilient to people
+ //not running repair when they should.
+ myNewReplicas.add(new LeavingReplica(entry.getKey(), entry.getValue()));
+ }
}
- Multimap<InetAddress, Range<Token>> sourceRanges = getNewSourceRanges(keyspaceName, myNewRanges);
- for (Map.Entry<InetAddress, Collection<Range<Token>>> entry : sourceRanges.asMap().entrySet())
- {
- rangesToFetch.put(keyspaceName, entry);
- }
+ logger.debug("Changed replicas for leaving {}, myNewReplicas {}", changedReplicas, myNewReplicas);
+ replicasToFetch.put(keyspaceName, getNewSourceReplicas(keyspaceName, myNewReplicas));
}
- StreamPlan stream = new StreamPlan("Restore replica count");
- for (String keyspaceName : rangesToFetch.keySet())
- {
- for (Map.Entry<InetAddress, Collection<Range<Token>>> entry : rangesToFetch.get(keyspaceName))
- {
- InetAddress source = entry.getKey();
- InetAddress preferred = SystemKeyspace.getPreferredIP(source);
- Collection<Range<Token>> ranges = entry.getValue();
+ StreamPlan stream = new StreamPlan(StreamOperation.RESTORE_REPLICA_COUNT);
+ replicasToFetch.forEach((keyspaceName, sources) -> {
+ logger.debug("Requesting keyspace {} sources", keyspaceName);
+ sources.asMap().forEach((sourceAddress, fetchReplicas) -> {
+ logger.debug("Source and our replicas are {}", fetchReplicas);
+ //Remember whether this node is providing the full or transient replicas for this range. We are going
+ //to pass streaming the local instance of Replica for the range which doesn't tell us anything about the source
+ //By encoding it as two separate sets we retain this information about the source.
+ RangesAtEndpoint full = fetchReplicas.stream()
+ .filter(f -> f.remote.isFull())
+ .map(f -> f.local)
+ .collect(RangesAtEndpoint.collector(myAddress));
+ RangesAtEndpoint transientReplicas = fetchReplicas.stream()
+ .filter(f -> f.remote.isTransient())
+ .map(f -> f.local)
+ .collect(RangesAtEndpoint.collector(myAddress));
if (logger.isDebugEnabled())
- logger.debug("Requesting from {} ranges {}", source, StringUtils.join(ranges, ", "));
- stream.requestRanges(source, preferred, keyspaceName, ranges);
- }
- }
+ logger.debug("Requesting from {} full replicas {} transient replicas {}", sourceAddress, StringUtils.join(full, ", "), StringUtils.join(transientReplicas, ", "));
+
+ stream.requestRanges(sourceAddress, keyspaceName, full, transientReplicas);
+ });
+ });
StreamResultFuture future = stream.execute();
Futures.addCallback(future, new FutureCallback<StreamState>()
{
@@ -2754,24 +2973,39 @@
// We still want to send the notification
sendReplicationNotification(notifyEndpoint);
}
- });
+ }, MoreExecutors.directExecutor());
}
+ /**
+ * This is used in three contexts, graceful decomission, and restoreReplicaCount/removeNode.
+ * Graceful decomission should never lose data and it's going to be important that transient data
+ * is streamed to at least one other node from this one for each range.
+ *
+ * For ranges this node replicates its removal should cause a new replica to be selected either as transient or full
+ * for every range. So I believe the current code doesn't have to do anything special because it will engage in streaming
+ * for every range it replicates to at least one other node and that should propagate the transient data that was here.
+ * When I graphed this out on paper the result of removal looked correct and there are no issues such as
+ * this node needing to create a full replica for a range it transiently replicates because what is created is just another
+ * transient replica to replace this node.
+ * @param keyspaceName
+ * @param endpoint
+ * @return
+ */
// needs to be modified to accept either a keyspace or ARS.
- private Multimap<Range<Token>, InetAddress> getChangedRangesForLeaving(String keyspaceName, InetAddress endpoint)
+ static EndpointsByReplica getChangedReplicasForLeaving(String keyspaceName, InetAddressAndPort endpoint, TokenMetadata tokenMetadata, AbstractReplicationStrategy strat)
{
// First get all ranges the leaving endpoint is responsible for
- Collection<Range<Token>> ranges = getRangesForEndpoint(keyspaceName, endpoint);
+ RangesAtEndpoint replicas = strat.getAddressReplicas(endpoint);
if (logger.isDebugEnabled())
- logger.debug("Node {} ranges [{}]", endpoint, StringUtils.join(ranges, ", "));
+ logger.debug("Node {} replicas [{}]", endpoint, StringUtils.join(replicas, ", "));
- Map<Range<Token>, List<InetAddress>> currentReplicaEndpoints = new HashMap<>(ranges.size());
+ Map<Replica, EndpointsForRange> currentReplicaEndpoints = Maps.newHashMapWithExpectedSize(replicas.size());
// Find (for each range) all nodes that store replicas for these ranges as well
TokenMetadata metadata = tokenMetadata.cloneOnlyTokenMap(); // don't do this in the loop! #7758
- for (Range<Token> range : ranges)
- currentReplicaEndpoints.put(range, Keyspace.open(keyspaceName).getReplicationStrategy().calculateNaturalEndpoints(range.right, metadata));
+ for (Replica replica : replicas)
+ currentReplicaEndpoints.put(replica, strat.calculateNaturalReplicas(replica.range().right, metadata));
TokenMetadata temp = tokenMetadata.cloneAfterAllLeft();
@@ -2780,29 +3014,46 @@
if (temp.isMember(endpoint))
temp.removeEndpoint(endpoint);
- Multimap<Range<Token>, InetAddress> changedRanges = HashMultimap.create();
+ EndpointsByReplica.Builder changedRanges = new EndpointsByReplica.Builder();
// Go through the ranges and for each range check who will be
// storing replicas for these ranges when the leaving endpoint
// is gone. Whoever is present in newReplicaEndpoints list, but
// not in the currentReplicaEndpoints list, will be needing the
// range.
- for (Range<Token> range : ranges)
+ for (Replica replica : replicas)
{
- Collection<InetAddress> newReplicaEndpoints = Keyspace.open(keyspaceName).getReplicationStrategy().calculateNaturalEndpoints(range.right, temp);
- newReplicaEndpoints.removeAll(currentReplicaEndpoints.get(range));
+ EndpointsForRange newReplicaEndpoints = strat.calculateNaturalReplicas(replica.range().right, temp);
+ newReplicaEndpoints = newReplicaEndpoints.filter(newReplica -> {
+ Optional<Replica> currentReplicaOptional =
+ tryFind(currentReplicaEndpoints.get(replica),
+ currentReplica -> newReplica.endpoint().equals(currentReplica.endpoint())
+ ).toJavaUtil();
+ //If it is newly replicating then yes we must do something to get the data there
+ if (!currentReplicaOptional.isPresent())
+ return true;
+
+ Replica currentReplica = currentReplicaOptional.get();
+ //This transition requires streaming to occur
+ //Full -> transient is handled by nodetool cleanup
+ //transient -> transient and full -> full don't require any action
+ if (currentReplica.isTransient() && newReplica.isFull())
+ return true;
+ return false;
+ });
+
if (logger.isDebugEnabled())
if (newReplicaEndpoints.isEmpty())
- logger.debug("Range {} already in all replicas", range);
+ logger.debug("Replica {} already in all replicas", replica);
else
- logger.debug("Range {} will be responsibility of {}", range, StringUtils.join(newReplicaEndpoints, ", "));
- changedRanges.putAll(range, newReplicaEndpoints);
+ logger.debug("Replica {} will be responsibility of {}", replica, StringUtils.join(newReplicaEndpoints, ", "));
+ changedRanges.putAll(replica, newReplicaEndpoints, Conflict.NONE);
}
- return changedRanges;
+ return changedRanges.build();
}
- public void onJoin(InetAddress endpoint, EndpointState epState)
+ public void onJoin(InetAddressAndPort endpoint, EndpointState epState)
{
for (Map.Entry<ApplicationState, VersionedValue> entry : epState.states())
{
@@ -2811,7 +3062,7 @@
MigrationManager.instance.scheduleSchemaPull(endpoint, epState);
}
- public void onAlive(InetAddress endpoint, EndpointState state)
+ public void onAlive(InetAddressAndPort endpoint, EndpointState state)
{
MigrationManager.instance.scheduleSchemaPull(endpoint, state);
@@ -2819,19 +3070,21 @@
notifyUp(endpoint);
}
- public void onRemove(InetAddress endpoint)
+ public void onRemove(InetAddressAndPort endpoint)
{
tokenMetadata.removeEndpoint(endpoint);
PendingRangeCalculatorService.instance.update();
}
- public void onDead(InetAddress endpoint, EndpointState state)
+ public void onDead(InetAddressAndPort endpoint, EndpointState state)
{
- MessagingService.instance().convict(endpoint);
+ // interrupt any outbound connection; if the node is failing and we cannot reconnect,
+ // this will rapidly lower the number of bytes we are willing to queue to the node
+ MessagingService.instance().interruptOutbound(endpoint);
notifyDown(endpoint);
}
- public void onRestart(InetAddress endpoint, EndpointState state)
+ public void onRestart(InetAddressAndPort endpoint, EndpointState state)
{
// If we have restarted before the node was even marked down, we need to reset the connection pool
if (state.isAlive())
@@ -2850,15 +3103,25 @@
return FileUtils.stringifyFileSize(StorageMetrics.load.getCount());
}
+ public Map<String, String> getLoadMapWithPort()
+ {
+ return getLoadMap(true);
+ }
+
public Map<String, String> getLoadMap()
{
+ return getLoadMap(false);
+ }
+
+ private Map<String, String> getLoadMap(boolean withPort)
+ {
Map<String, String> map = new HashMap<>();
- for (Map.Entry<InetAddress,Double> entry : LoadBroadcaster.instance.getLoadInfo().entrySet())
+ for (Map.Entry<InetAddressAndPort,Double> entry : LoadBroadcaster.instance.getLoadInfo().entrySet())
{
- map.put(entry.getKey().getHostAddress(), FileUtils.stringifyFileSize(entry.getValue()));
+ map.put(entry.getKey().getHostAddress(withPort), FileUtils.stringifyFileSize(entry.getValue()));
}
// gossiper doesn't see its own updates, so we need to special-case the local node
- map.put(FBUtilities.getBroadcastAddress().getHostAddress(), getLoadString());
+ map.put(withPort ? FBUtilities.getJustBroadcastAddress().getHostAddress() : FBUtilities.getBroadcastAddressAndPort().toString(), getLoadString());
return map;
}
@@ -2876,13 +3139,13 @@
}
@Nullable
- public InetAddress getEndpointForHostId(UUID hostId)
+ public InetAddressAndPort getEndpointForHostId(UUID hostId)
{
return tokenMetadata.getEndpointForHostId(hostId);
}
@Nullable
- public UUID getHostIdForEndpoint(InetAddress address)
+ public UUID getHostIdForEndpoint(InetAddressAndPort address)
{
return tokenMetadata.getHostId(address);
}
@@ -2891,15 +3154,15 @@
public List<String> getTokens()
{
- return getTokens(FBUtilities.getBroadcastAddress());
+ return getTokens(FBUtilities.getBroadcastAddressAndPort());
}
public List<String> getTokens(String endpoint) throws UnknownHostException
{
- return getTokens(InetAddress.getByName(endpoint));
+ return getTokens(InetAddressAndPort.getByName(endpoint));
}
- private List<String> getTokens(InetAddress endpoint)
+ private List<String> getTokens(InetAddressAndPort endpoint)
{
List<String> strTokens = new ArrayList<>();
for (Token tok : getTokenMetadata().getTokens(endpoint))
@@ -2917,42 +3180,84 @@
return Schema.instance.getVersion().toString();
}
- public List<String> getLeavingNodes()
+ public String getKeyspaceReplicationInfo(String keyspaceName)
{
- return stringify(tokenMetadata.getLeavingEndpoints());
+ Keyspace keyspaceInstance = Schema.instance.getKeyspaceInstance(keyspaceName);
+ if (keyspaceInstance == null)
+ throw new IllegalArgumentException(); // ideally should never happen
+ ReplicationParams replicationParams = keyspaceInstance.getMetadata().params.replication;
+ String replicationInfo = replicationParams.klass.getSimpleName() + " " + replicationParams.options.toString();
+ return replicationInfo;
}
+ @Deprecated
+ public List<String> getLeavingNodes()
+ {
+ return stringify(tokenMetadata.getLeavingEndpoints(), false);
+ }
+
+ public List<String> getLeavingNodesWithPort()
+ {
+ return stringify(tokenMetadata.getLeavingEndpoints(), true);
+ }
+
+ @Deprecated
public List<String> getMovingNodes()
{
List<String> endpoints = new ArrayList<>();
- for (Pair<Token, InetAddress> node : tokenMetadata.getMovingEndpoints())
+ for (Pair<Token, InetAddressAndPort> node : tokenMetadata.getMovingEndpoints())
{
- endpoints.add(node.right.getHostAddress());
+ endpoints.add(node.right.address.getHostAddress());
}
return endpoints;
}
+ public List<String> getMovingNodesWithPort()
+ {
+ List<String> endpoints = new ArrayList<>();
+
+ for (Pair<Token, InetAddressAndPort> node : tokenMetadata.getMovingEndpoints())
+ {
+ endpoints.add(node.right.toString());
+ }
+
+ return endpoints;
+ }
+
+
public List<String> getJoiningNodes()
{
- return stringify(tokenMetadata.getBootstrapTokens().valueSet());
+ return stringify(tokenMetadata.getBootstrapTokens().valueSet(), false);
+ }
+
+ @Deprecated
+ public List<String> getJoiningNodesWithPort()
+ {
+ return stringify(tokenMetadata.getBootstrapTokens().valueSet(), true);
}
public List<String> getLiveNodes()
{
- return stringify(Gossiper.instance.getLiveMembers());
+ return stringify(Gossiper.instance.getLiveMembers(), false);
}
- public Set<InetAddress> getLiveRingMembers()
+ @Deprecated
+ public List<String> getLiveNodesWithPort()
+ {
+ return stringify(Gossiper.instance.getLiveMembers(), true);
+ }
+
+ public Set<InetAddressAndPort> getLiveRingMembers()
{
return getLiveRingMembers(false);
}
- public Set<InetAddress> getLiveRingMembers(boolean excludeDeadStates)
+ public Set<InetAddressAndPort> getLiveRingMembers(boolean excludeDeadStates)
{
- Set<InetAddress> ret = new HashSet<>();
- for (InetAddress ep : Gossiper.instance.getLiveMembers())
+ Set<InetAddressAndPort> ret = new HashSet<>();
+ for (InetAddressAndPort ep : Gossiper.instance.getLiveMembers())
{
if (excludeDeadStates)
{
@@ -2968,9 +3273,15 @@
}
+ @Deprecated
public List<String> getUnreachableNodes()
{
- return stringify(Gossiper.instance.getUnreachableMembers());
+ return stringify(Gossiper.instance.getUnreachableMembers(), false);
+ }
+
+ public List<String> getUnreachableNodesWithPort()
+ {
+ return stringify(Gossiper.instance.getUnreachableMembers(), true);
}
public String[] getAllDataFileLocations()
@@ -2991,19 +3302,19 @@
return FileUtils.getCanonicalPath(DatabaseDescriptor.getSavedCachesLocation());
}
- private List<String> stringify(Iterable<InetAddress> endpoints)
+ private List<String> stringify(Iterable<InetAddressAndPort> endpoints, boolean withPort)
{
List<String> stringEndpoints = new ArrayList<>();
- for (InetAddress ep : endpoints)
+ for (InetAddressAndPort ep : endpoints)
{
- stringEndpoints.add(ep.getHostAddress());
+ stringEndpoints.add(ep.getHostAddress(withPort));
}
return stringEndpoints;
}
public int getCurrentGenerationNumber()
{
- return Gossiper.instance.getCurrentGenerationNumber(FBUtilities.getBroadcastAddress());
+ return Gossiper.instance.getCurrentGenerationNumber(FBUtilities.getBroadcastAddressAndPort());
}
public int forceKeyspaceCleanup(String keyspaceName, String... tables) throws IOException, ExecutionException, InterruptedException
@@ -3053,12 +3364,25 @@
return status.statusCode;
}
+ @Deprecated
public int verify(boolean extendedVerify, String keyspaceName, String... tableNames) throws IOException, ExecutionException, InterruptedException
{
+ return verify(extendedVerify, false, false, false, false, false, keyspaceName, tableNames);
+ }
+
+ public int verify(boolean extendedVerify, boolean checkVersion, boolean diskFailurePolicy, boolean mutateRepairStatus, boolean checkOwnsTokens, boolean quick, String keyspaceName, String... tableNames) throws IOException, ExecutionException, InterruptedException
+ {
CompactionManager.AllSSTableOpStatus status = CompactionManager.AllSSTableOpStatus.SUCCESSFUL;
+ Verifier.Options options = Verifier.options().invokeDiskFailurePolicy(diskFailurePolicy)
+ .extendedVerification(extendedVerify)
+ .checkVersion(checkVersion)
+ .mutateRepairStatus(mutateRepairStatus)
+ .checkOwnsTokens(checkOwnsTokens)
+ .quick(quick).build();
+ logger.info("Verifying {}.{} with options = {}", keyspaceName, Arrays.toString(tableNames), options);
for (ColumnFamilyStore cfStore : getValidColumnFamilies(false, false, keyspaceName, tableNames))
{
- CompactionManager.AllSSTableOpStatus oneStatus = cfStore.verify(extendedVerify);
+ CompactionManager.AllSSTableOpStatus oneStatus = cfStore.verify(options);
if (oneStatus != CompactionManager.AllSSTableOpStatus.SUCCESSFUL)
status = oneStatus;
}
@@ -3299,12 +3623,18 @@
}
- private Keyspace getValidKeyspace(String keyspaceName) throws IOException
+ private void verifyKeyspaceIsValid(String keyspaceName)
{
+ if (null != VirtualKeyspaceRegistry.instance.getKeyspaceNullable(keyspaceName))
+ throw new IllegalArgumentException("Cannot perform any operations against virtual keyspace " + keyspaceName);
+
if (!Schema.instance.getKeyspaces().contains(keyspaceName))
- {
- throw new IOException("Keyspace " + keyspaceName + " does not exist");
- }
+ throw new IllegalArgumentException("Keyspace " + keyspaceName + " does not exist");
+ }
+
+ private Keyspace getValidKeyspace(String keyspaceName)
+ {
+ verifyKeyspaceIsValid(keyspaceName);
return Keyspace.open(keyspaceName);
}
@@ -3341,12 +3671,9 @@
Map<String, TabularData> snapshotMap = new HashMap<>();
for (Keyspace keyspace : Keyspace.all())
{
- if (SchemaConstants.isLocalSystemKeyspace(keyspace.getName()))
- continue;
-
for (ColumnFamilyStore cfStore : keyspace.getColumnFamilyStores())
{
- for (Map.Entry<String, Pair<Long,Long>> snapshotDetail : cfStore.getSnapshotDetails().entrySet())
+ for (Map.Entry<String, Directories.SnapshotSizeDetails> snapshotDetail : cfStore.getSnapshotDetails().entrySet())
{
TabularDataSupport data = (TabularDataSupport)snapshotMap.get(snapshotDetail.getKey());
if (data == null)
@@ -3355,7 +3682,7 @@
snapshotMap.put(snapshotDetail.getKey(), data);
}
- SnapshotDetailsTabularData.from(snapshotDetail.getKey(), keyspace.getName(), cfStore.getColumnFamilyName(), snapshotDetail, data);
+ SnapshotDetailsTabularData.from(snapshotDetail.getKey(), keyspace.getName(), cfStore.getTableName(), snapshotDetail, data);
}
}
}
@@ -3387,24 +3714,7 @@
public void cleanupSizeEstimates()
{
- SetMultimap<String, String> sizeEstimates = SystemKeyspace.getTablesWithSizeEstimates();
-
- for (Entry<String, Collection<String>> tablesByKeyspace : sizeEstimates.asMap().entrySet())
- {
- String keyspace = tablesByKeyspace.getKey();
- if (!Schema.instance.getKeyspaces().contains(keyspace))
- {
- SystemKeyspace.clearSizeEstimates(keyspace);
- }
- else
- {
- for (String table : tablesByKeyspace.getValue())
- {
- if (!Schema.instance.hasCF(Pair.create(keyspace, table)))
- SystemKeyspace.clearSizeEstimates(keyspace, table);
- }
- }
- }
+ SystemKeyspace.clearAllEstimates();
}
/**
@@ -3437,6 +3747,11 @@
public int repairAsync(String keyspace, Map<String, String> repairSpec)
{
+ return repair(keyspace, repairSpec, Collections.emptyList()).left;
+ }
+
+ public Pair<Integer, Future<?>> repair(String keyspace, Map<String, String> repairSpec, List<ProgressListener> listeners)
+ {
RepairOption option = RepairOption.parse(repairSpec, tokenMetadata.partitioner);
// if ranges are not specified
if (option.getRanges().isEmpty())
@@ -3454,173 +3769,14 @@
}
else
{
- option.getRanges().addAll(getLocalRanges(keyspace));
+ Iterables.addAll(option.getRanges(), getLocalReplicas(keyspace).onlyFull().ranges());
}
}
- return forceRepairAsync(keyspace, option, false);
- }
+ if (option.getRanges().isEmpty() || Keyspace.open(keyspace).getReplicationStrategy().getReplicationFactor().allReplicas < 2)
+ return Pair.create(0, Futures.immediateFuture(null));
- @Deprecated
- public int forceRepairAsync(String keyspace,
- boolean isSequential,
- Collection<String> dataCenters,
- Collection<String> hosts,
- boolean primaryRange,
- boolean fullRepair,
- String... tableNames)
- {
- return forceRepairAsync(keyspace, isSequential ? RepairParallelism.SEQUENTIAL.ordinal() : RepairParallelism.PARALLEL.ordinal(), dataCenters, hosts, primaryRange, fullRepair, tableNames);
- }
-
- @Deprecated
- public int forceRepairAsync(String keyspace,
- int parallelismDegree,
- Collection<String> dataCenters,
- Collection<String> hosts,
- boolean primaryRange,
- boolean fullRepair,
- String... tableNames)
- {
- if (parallelismDegree < 0 || parallelismDegree > RepairParallelism.values().length - 1)
- {
- throw new IllegalArgumentException("Invalid parallelism degree specified: " + parallelismDegree);
- }
- RepairParallelism parallelism = RepairParallelism.values()[parallelismDegree];
- if (FBUtilities.isWindows && parallelism != RepairParallelism.PARALLEL)
- {
- logger.warn("Snapshot-based repair is not yet supported on Windows. Reverting to parallel repair.");
- parallelism = RepairParallelism.PARALLEL;
- }
-
- RepairOption options = new RepairOption(parallelism, primaryRange, !fullRepair, false, 1, Collections.<Range<Token>>emptyList(), false, false);
- if (dataCenters != null)
- {
- options.getDataCenters().addAll(dataCenters);
- }
- if (hosts != null)
- {
- options.getHosts().addAll(hosts);
- }
- if (primaryRange)
- {
- // when repairing only primary range, neither dataCenters nor hosts can be set
- if (options.getDataCenters().isEmpty() && options.getHosts().isEmpty())
- options.getRanges().addAll(getPrimaryRanges(keyspace));
- // except dataCenters only contain local DC (i.e. -local)
- else if (options.getDataCenters().size() == 1 && options.getDataCenters().contains(DatabaseDescriptor.getLocalDataCenter()))
- options.getRanges().addAll(getPrimaryRangesWithinDC(keyspace));
- else
- throw new IllegalArgumentException("You need to run primary range repair on all nodes in the cluster.");
- }
- else
- {
- options.getRanges().addAll(getLocalRanges(keyspace));
- }
- if (tableNames != null)
- {
- for (String table : tableNames)
- {
- options.getColumnFamilies().add(table);
- }
- }
- return forceRepairAsync(keyspace, options, true);
- }
-
- @Deprecated
- public int forceRepairAsync(String keyspace,
- boolean isSequential,
- boolean isLocal,
- boolean primaryRange,
- boolean fullRepair,
- String... tableNames)
- {
- Set<String> dataCenters = null;
- if (isLocal)
- {
- dataCenters = Sets.newHashSet(DatabaseDescriptor.getLocalDataCenter());
- }
- return forceRepairAsync(keyspace, isSequential, dataCenters, null, primaryRange, fullRepair, tableNames);
- }
-
- @Deprecated
- public int forceRepairRangeAsync(String beginToken,
- String endToken,
- String keyspaceName,
- boolean isSequential,
- Collection<String> dataCenters,
- Collection<String> hosts,
- boolean fullRepair,
- String... tableNames)
- {
- return forceRepairRangeAsync(beginToken, endToken, keyspaceName,
- isSequential ? RepairParallelism.SEQUENTIAL.ordinal() : RepairParallelism.PARALLEL.ordinal(),
- dataCenters, hosts, fullRepair, tableNames);
- }
-
- @Deprecated
- public int forceRepairRangeAsync(String beginToken,
- String endToken,
- String keyspaceName,
- int parallelismDegree,
- Collection<String> dataCenters,
- Collection<String> hosts,
- boolean fullRepair,
- String... tableNames)
- {
- if (parallelismDegree < 0 || parallelismDegree > RepairParallelism.values().length - 1)
- {
- throw new IllegalArgumentException("Invalid parallelism degree specified: " + parallelismDegree);
- }
- RepairParallelism parallelism = RepairParallelism.values()[parallelismDegree];
- if (FBUtilities.isWindows && parallelism != RepairParallelism.PARALLEL)
- {
- logger.warn("Snapshot-based repair is not yet supported on Windows. Reverting to parallel repair.");
- parallelism = RepairParallelism.PARALLEL;
- }
-
- if (!fullRepair)
- logger.warn("Incremental repair can't be requested with subrange repair " +
- "because each subrange repair would generate an anti-compacted table. " +
- "The repair will occur but without anti-compaction.");
- Collection<Range<Token>> repairingRange = createRepairRangeFrom(beginToken, endToken);
-
- RepairOption options = new RepairOption(parallelism, false, !fullRepair, false, 1, repairingRange, true, false);
- if (dataCenters != null)
- {
- options.getDataCenters().addAll(dataCenters);
- }
- if (hosts != null)
- {
- options.getHosts().addAll(hosts);
- }
- if (tableNames != null)
- {
- for (String table : tableNames)
- {
- options.getColumnFamilies().add(table);
- }
- }
-
- logger.info("starting user-requested repair of range {} for keyspace {} and column families {}",
- repairingRange, keyspaceName, tableNames);
- return forceRepairAsync(keyspaceName, options, true);
- }
-
- @Deprecated
- public int forceRepairRangeAsync(String beginToken,
- String endToken,
- String keyspaceName,
- boolean isSequential,
- boolean isLocal,
- boolean fullRepair,
- String... tableNames)
- {
- Set<String> dataCenters = null;
- if (isLocal)
- {
- dataCenters = Sets.newHashSet(DatabaseDescriptor.getLocalDataCenter());
- }
- return forceRepairRangeAsync(beginToken, endToken, keyspaceName, isSequential, dataCenters, null, fullRepair, tableNames);
+ int cmd = nextRepairCommand.incrementAndGet();
+ return Pair.create(cmd, ActiveRepairService.repairCommandExecutor().submit(createRepairTask(cmd, keyspace, option, listeners)));
}
/**
@@ -3666,17 +3822,7 @@
return tokenMetadata.partitioner.getTokenFactory();
}
- public int forceRepairAsync(String keyspace, RepairOption options, boolean legacy)
- {
- if (options.getRanges().isEmpty() || Keyspace.open(keyspace).getReplicationStrategy().getReplicationFactor() < 2)
- return 0;
-
- int cmd = nextRepairCommand.incrementAndGet();
- NamedThreadFactory.createThread(createRepairTask(cmd, keyspace, options, legacy), "Repair-Task-" + threadCounter.incrementAndGet()).start();
- return cmd;
- }
-
- private FutureTask<Object> createRepairTask(final int cmd, final String keyspace, final RepairOption options, boolean legacy)
+ private FutureTask<Object> createRepairTask(final int cmd, final String keyspace, final RepairOption options, List<ProgressListener> listeners)
{
if (!options.getDataCenters().isEmpty() && !options.getDataCenters().contains(DatabaseDescriptor.getLocalDataCenter()))
{
@@ -3685,8 +3831,24 @@
RepairRunnable task = new RepairRunnable(this, cmd, options, keyspace);
task.addProgressListener(progressSupport);
- if (legacy)
- task.addProgressListener(legacyProgressSupport);
+ for (ProgressListener listener : listeners)
+ task.addProgressListener(listener);
+
+ if (options.isTraced())
+ {
+ Runnable r = () ->
+ {
+ try
+ {
+ task.run();
+ }
+ finally
+ {
+ ExecutorLocals.set(null);
+ }
+ };
+ return new FutureTask<>(r, null);
+ }
return new FutureTask<>(task, null);
}
@@ -3695,6 +3857,14 @@
ActiveRepairService.instance.terminateSessions();
}
+ @Nullable
+ public List<String> getParentRepairStatus(int cmd)
+ {
+ Pair<ActiveRepairService.ParentRepairStatus, List<String>> pair = ActiveRepairService.instance.getRepairStatus(cmd);
+ return pair == null ? null :
+ ImmutableList.<String>builder().add(pair.left.name()).addAll(pair.right).build();
+ }
+
public void setRepairSessionMaxTreeDepth(int depth)
{
DatabaseDescriptor.setRepairSessionMaxTreeDepth(depth);
@@ -3711,22 +3881,25 @@
* Get the "primary ranges" for the specified keyspace and endpoint.
* "Primary ranges" are the ranges that the node is responsible for storing replica primarily.
* The node that stores replica primarily is defined as the first node returned
- * by {@link AbstractReplicationStrategy#calculateNaturalEndpoints}.
+ * by {@link AbstractReplicationStrategy#calculateNaturalReplicas}.
*
* @param keyspace Keyspace name to check primary ranges
* @param ep endpoint we are interested in.
* @return primary ranges for the specified endpoint.
*/
- public Collection<Range<Token>> getPrimaryRangesForEndpoint(String keyspace, InetAddress ep)
+ public Collection<Range<Token>> getPrimaryRangesForEndpoint(String keyspace, InetAddressAndPort ep)
{
AbstractReplicationStrategy strategy = Keyspace.open(keyspace).getReplicationStrategy();
Collection<Range<Token>> primaryRanges = new HashSet<>();
TokenMetadata metadata = tokenMetadata.cloneOnlyTokenMap();
for (Token token : metadata.sortedTokens())
{
- List<InetAddress> endpoints = strategy.calculateNaturalEndpoints(token, metadata);
- if (endpoints.size() > 0 && endpoints.get(0).equals(ep))
+ EndpointsForRange replicas = strategy.calculateNaturalReplicas(token, metadata);
+ if (replicas.size() > 0 && replicas.get(0).endpoint().equals(ep))
+ {
+ Preconditions.checkState(replicas.get(0).isFull());
primaryRanges.add(new Range<>(metadata.getPredecessor(token), token));
+ }
}
return primaryRanges;
}
@@ -3734,27 +3907,27 @@
/**
* Get the "primary ranges" within local DC for the specified keyspace and endpoint.
*
- * @see #getPrimaryRangesForEndpoint(String, java.net.InetAddress)
+ * @see #getPrimaryRangesForEndpoint(String, InetAddressAndPort)
* @param keyspace Keyspace name to check primary ranges
* @param referenceEndpoint endpoint we are interested in.
* @return primary ranges within local DC for the specified endpoint.
*/
- public Collection<Range<Token>> getPrimaryRangeForEndpointWithinDC(String keyspace, InetAddress referenceEndpoint)
+ public Collection<Range<Token>> getPrimaryRangeForEndpointWithinDC(String keyspace, InetAddressAndPort referenceEndpoint)
{
TokenMetadata metadata = tokenMetadata.cloneOnlyTokenMap();
String localDC = DatabaseDescriptor.getEndpointSnitch().getDatacenter(referenceEndpoint);
- Collection<InetAddress> localDcNodes = metadata.getTopology().getDatacenterEndpoints().get(localDC);
+ Collection<InetAddressAndPort> localDcNodes = metadata.getTopology().getDatacenterEndpoints().get(localDC);
AbstractReplicationStrategy strategy = Keyspace.open(keyspace).getReplicationStrategy();
Collection<Range<Token>> localDCPrimaryRanges = new HashSet<>();
for (Token token : metadata.sortedTokens())
{
- List<InetAddress> endpoints = strategy.calculateNaturalEndpoints(token, metadata);
- for (InetAddress endpoint : endpoints)
+ EndpointsForRange replicas = strategy.calculateNaturalReplicas(token, metadata);
+ for (Replica replica : replicas)
{
- if (localDcNodes.contains(endpoint))
+ if (localDcNodes.contains(replica.endpoint()))
{
- if (endpoint.equals(referenceEndpoint))
+ if (replica.endpoint().equals(referenceEndpoint))
{
localDCPrimaryRanges.add(new Range<>(metadata.getPredecessor(token), token));
}
@@ -3766,14 +3939,30 @@
return localDCPrimaryRanges;
}
- /**
- * Get all ranges an endpoint is responsible for (by keyspace)
- * @param ep endpoint we are interested in.
- * @return ranges for the specified endpoint.
- */
- Collection<Range<Token>> getRangesForEndpoint(String keyspaceName, InetAddress ep)
+ public Collection<Range<Token>> getLocalPrimaryRange()
{
- return Keyspace.open(keyspaceName).getReplicationStrategy().getAddressRanges().get(ep);
+ return getLocalPrimaryRangeForEndpoint(FBUtilities.getBroadcastAddressAndPort());
+ }
+
+ public Collection<Range<Token>> getLocalPrimaryRangeForEndpoint(InetAddressAndPort referenceEndpoint)
+ {
+ IEndpointSnitch snitch = DatabaseDescriptor.getEndpointSnitch();
+ TokenMetadata tokenMetadata = this.tokenMetadata.cloneOnlyTokenMap();
+ String dc = snitch.getDatacenter(referenceEndpoint);
+ Set<Token> tokens = new HashSet<>(tokenMetadata.getTokens(referenceEndpoint));
+
+ // filter tokens to the single DC
+ List<Token> filteredTokens = Lists.newArrayList();
+ for (Token token : tokenMetadata.sortedTokens())
+ {
+ InetAddressAndPort endpoint = tokenMetadata.getEndpoint(token);
+ if (dc.equals(snitch.getDatacenter(endpoint)))
+ filteredTokens.add(token);
+ }
+
+ return getAllRanges(filteredTokens).stream()
+ .filter(t -> tokens.contains(t.right))
+ .collect(Collectors.toList());
}
/**
@@ -3811,82 +4000,52 @@
* @param key key for which we need to find the endpoint
* @return the endpoint responsible for this key
*/
+ @Deprecated
public List<InetAddress> getNaturalEndpoints(String keyspaceName, String cf, String key)
{
- KeyspaceMetadata ksMetaData = Schema.instance.getKSMetaData(keyspaceName);
+ EndpointsForToken replicas = getNaturalReplicasForToken(keyspaceName, cf, key);
+ List<InetAddress> inetList = new ArrayList<>(replicas.size());
+ replicas.forEach(r -> inetList.add(r.endpoint().address));
+ return inetList;
+ }
+
+ public List<String> getNaturalEndpointsWithPort(String keyspaceName, String cf, String key)
+ {
+ return Replicas.stringify(getNaturalReplicasForToken(keyspaceName, cf, key), true);
+ }
+
+ @Deprecated
+ public List<InetAddress> getNaturalEndpoints(String keyspaceName, ByteBuffer key)
+ {
+ EndpointsForToken replicas = getNaturalReplicasForToken(keyspaceName, key);
+ List<InetAddress> inetList = new ArrayList<>(replicas.size());
+ replicas.forEach(r -> inetList.add(r.endpoint().address));
+ return inetList;
+ }
+
+ public List<String> getNaturalEndpointsWithPort(String keyspaceName, ByteBuffer key)
+ {
+ EndpointsForToken replicas = getNaturalReplicasForToken(keyspaceName, key);
+ return Replicas.stringify(replicas, true);
+ }
+
+ public EndpointsForToken getNaturalReplicasForToken(String keyspaceName, String cf, String key)
+ {
+ KeyspaceMetadata ksMetaData = Schema.instance.getKeyspaceMetadata(keyspaceName);
if (ksMetaData == null)
throw new IllegalArgumentException("Unknown keyspace '" + keyspaceName + "'");
- CFMetaData cfMetaData = ksMetaData.getTableOrViewNullable(cf);
- if (cfMetaData == null)
+ TableMetadata metadata = ksMetaData.getTableOrViewNullable(cf);
+ if (metadata == null)
throw new IllegalArgumentException("Unknown table '" + cf + "' in keyspace '" + keyspaceName + "'");
- return getNaturalEndpoints(keyspaceName, tokenMetadata.partitioner.getToken(cfMetaData.getKeyValidator().fromString(key)));
+ return getNaturalReplicasForToken(keyspaceName, metadata.partitionKeyType.fromString(key));
}
- public List<InetAddress> getNaturalEndpoints(String keyspaceName, ByteBuffer key)
+ public EndpointsForToken getNaturalReplicasForToken(String keyspaceName, ByteBuffer key)
{
- return getNaturalEndpoints(keyspaceName, tokenMetadata.partitioner.getToken(key));
- }
-
- /**
- * This method returns the N endpoints that are responsible for storing the
- * specified key i.e for replication.
- *
- * @param keyspaceName keyspace name also known as keyspace
- * @param pos position for which we need to find the endpoint
- * @return the endpoint responsible for this token
- */
- public List<InetAddress> getNaturalEndpoints(String keyspaceName, RingPosition pos)
- {
- return Keyspace.open(keyspaceName).getReplicationStrategy().getNaturalEndpoints(pos);
- }
-
- /**
- * Returns the endpoints currently responsible for storing the token plus pending ones
- */
- public Iterable<InetAddress> getNaturalAndPendingEndpoints(String keyspaceName, Token token)
- {
- return Iterables.concat(getNaturalEndpoints(keyspaceName, token), tokenMetadata.pendingEndpointsFor(token, keyspaceName));
- }
-
- /**
- * This method attempts to return N endpoints that are responsible for storing the
- * specified key i.e for replication.
- *
- * @param keyspace keyspace name also known as keyspace
- * @param key key for which we need to find the endpoint
- * @return the endpoint responsible for this key
- */
- public List<InetAddress> getLiveNaturalEndpoints(Keyspace keyspace, ByteBuffer key)
- {
- return getLiveNaturalEndpoints(keyspace, tokenMetadata.decorateKey(key));
- }
-
- public List<InetAddress> getLiveNaturalEndpoints(Keyspace keyspace, RingPosition pos)
- {
- List<InetAddress> liveEps = new ArrayList<>();
- getLiveNaturalEndpoints(keyspace, pos, liveEps);
- return liveEps;
- }
-
- /**
- * This method attempts to return N endpoints that are responsible for storing the
- * specified key i.e for replication.
- *
- * @param keyspace keyspace name also known as keyspace
- * @param pos position for which we need to find the endpoint
- * @param liveEps the list of endpoints to mutate
- */
- public void getLiveNaturalEndpoints(Keyspace keyspace, RingPosition pos, List<InetAddress> liveEps)
- {
- List<InetAddress> endpoints = keyspace.getReplicationStrategy().getNaturalEndpoints(pos);
-
- for (InetAddress endpoint : endpoints)
- {
- if (FailureDetector.instance.isAlive(endpoint))
- liveEps.add(endpoint);
- }
+ Token token = tokenMetadata.partitioner.getToken(key);
+ return Keyspace.open(keyspaceName).getReplicationStrategy().getNaturalReplicasForToken(token);
}
public void setLoggingLevel(String classQualifier, String rawLevel) throws Exception
@@ -3935,7 +4094,7 @@
Token token = tokens.get(index);
Range<Token> range = new Range<>(prevToken, token);
// always return an estimate > 0 (see CASSANDRA-7322)
- splits.add(Pair.create(range, Math.max(cfs.metadata.params.minIndexInterval, cfs.estimatedKeysForRange(range))));
+ splits.add(Pair.create(range, Math.max(cfs.metadata().params.minIndexInterval, cfs.estimatedKeysForRange(range))));
prevToken = token;
}
return splits;
@@ -3965,20 +4124,25 @@
*/
private void startLeaving()
{
+ Gossiper.instance.addLocalApplicationState(ApplicationState.STATUS_WITH_PORT, valueFactory.leaving(getLocalTokens()));
Gossiper.instance.addLocalApplicationState(ApplicationState.STATUS, valueFactory.leaving(getLocalTokens()));
- tokenMetadata.addLeavingEndpoint(FBUtilities.getBroadcastAddress());
+ tokenMetadata.addLeavingEndpoint(FBUtilities.getBroadcastAddressAndPort());
PendingRangeCalculatorService.instance.update();
}
- public void decommission() throws InterruptedException
+ public void decommission(boolean force) throws InterruptedException
{
- if (!tokenMetadata.isMember(FBUtilities.getBroadcastAddress()))
- throw new UnsupportedOperationException("local node is not a member of the token ring yet");
- if (tokenMetadata.cloneAfterAllLeft().sortedTokens().size() < 2)
- throw new UnsupportedOperationException("no other normal nodes in the ring; decommission would be pointless");
- if (operationMode != Mode.LEAVING && operationMode != Mode.NORMAL)
- throw new UnsupportedOperationException("Node in " + operationMode + " state; wait for status to become normal or restart");
- if (isDecommissioning.compareAndSet(true, true))
+ TokenMetadata metadata = tokenMetadata.cloneAfterAllLeft();
+ if (operationMode != Mode.LEAVING)
+ {
+ if (!tokenMetadata.isMember(FBUtilities.getBroadcastAddressAndPort()))
+ throw new UnsupportedOperationException("local node is not a member of the token ring yet");
+ if (metadata.getAllEndpoints().size() < 2)
+ throw new UnsupportedOperationException("no other normal nodes in the ring; decommission would be pointless");
+ if (operationMode != Mode.NORMAL)
+ throw new UnsupportedOperationException("Node in " + operationMode + " state; wait for status to become normal or restart");
+ }
+ if (!isDecommissioning.compareAndSet(false, true))
throw new IllegalStateException("Node is still decommissioning. Check nodetool netstats.");
if (logger.isDebugEnabled())
@@ -3987,10 +4151,38 @@
try
{
PendingRangeCalculatorService.instance.blockUntilFinished();
- for (String keyspaceName : Schema.instance.getNonLocalStrategyKeyspaces())
+
+ String dc = DatabaseDescriptor.getEndpointSnitch().getLocalDatacenter();
+
+ if (operationMode != Mode.LEAVING) // If we're already decommissioning there is no point checking RF/pending ranges
{
- if (tokenMetadata.getPendingRanges(keyspaceName, FBUtilities.getBroadcastAddress()).size() > 0)
- throw new UnsupportedOperationException("data is currently moving to this node; unable to leave the ring");
+ int rf, numNodes;
+ for (String keyspaceName : Schema.instance.getNonLocalStrategyKeyspaces())
+ {
+ if (!force)
+ {
+ Keyspace keyspace = Keyspace.open(keyspaceName);
+ if (keyspace.getReplicationStrategy() instanceof NetworkTopologyStrategy)
+ {
+ NetworkTopologyStrategy strategy = (NetworkTopologyStrategy) keyspace.getReplicationStrategy();
+ rf = strategy.getReplicationFactor(dc).allReplicas;
+ numNodes = metadata.getTopology().getDatacenterEndpoints().get(dc).size();
+ }
+ else
+ {
+ numNodes = metadata.getAllEndpoints().size();
+ rf = keyspace.getReplicationStrategy().getReplicationFactor().allReplicas;
+ }
+
+ if (numNodes <= rf)
+ throw new UnsupportedOperationException("Not enough live nodes to maintain replication factor in keyspace "
+ + keyspaceName + " (RF = " + rf + ", N = " + numNodes + ")."
+ + " Perform a forceful decommission to ignore.");
+ }
+ // TODO: do we care about fixing transient/full self-movements here? probably
+ if (tokenMetadata.getPendingRanges(keyspaceName, FBUtilities.getBroadcastAddressAndPort()).size() > 0)
+ throw new UnsupportedOperationException("data is currently moving to this node; unable to leave the ring");
+ }
}
startLeaving();
@@ -4012,7 +4204,7 @@
{
logger.info("failed to shutdown message service: {}", ioe);
}
- StageManager.shutdownNow();
+ Stage.shutdownNow();
SystemKeyspace.setBootstrapState(SystemKeyspace.BootstrapState.DECOMMISSIONED);
setMode(Mode.DECOMMISSIONED, true);
// let op be responsible for killing the process
@@ -4038,22 +4230,23 @@
private void leaveRing()
{
SystemKeyspace.setBootstrapState(SystemKeyspace.BootstrapState.NEEDS_BOOTSTRAP);
- tokenMetadata.removeEndpoint(FBUtilities.getBroadcastAddress());
+ tokenMetadata.removeEndpoint(FBUtilities.getBroadcastAddressAndPort());
PendingRangeCalculatorService.instance.update();
+ Gossiper.instance.addLocalApplicationState(ApplicationState.STATUS_WITH_PORT, valueFactory.left(getLocalTokens(),Gossiper.computeExpireTime()));
Gossiper.instance.addLocalApplicationState(ApplicationState.STATUS, valueFactory.left(getLocalTokens(),Gossiper.computeExpireTime()));
int delay = Math.max(RING_DELAY, Gossiper.intervalInMillis * 2);
logger.info("Announcing that I have left the ring for {}ms", delay);
- Uninterruptibles.sleepUninterruptibly(delay, TimeUnit.MILLISECONDS);
+ Uninterruptibles.sleepUninterruptibly(delay, MILLISECONDS);
}
private void unbootstrap(Runnable onFinish) throws ExecutionException, InterruptedException
{
- Map<String, Multimap<Range<Token>, InetAddress>> rangesToStream = new HashMap<>();
+ Map<String, EndpointsByReplica> rangesToStream = new HashMap<>();
for (String keyspaceName : Schema.instance.getNonLocalStrategyKeyspaces())
{
- Multimap<Range<Token>, InetAddress> rangesMM = getChangedRangesForLeaving(keyspaceName, FBUtilities.getBroadcastAddress());
+ EndpointsByReplica rangesMM = getChangedReplicasForLeaving(keyspaceName, FBUtilities.getBroadcastAddressAndPort(), tokenMetadata, Keyspace.open(keyspaceName).getReplicationStrategy());
if (logger.isDebugEnabled())
logger.debug("Ranges needing transfer are [{}]", StringUtils.join(rangesMM.keySet(), ","));
@@ -4089,20 +4282,22 @@
return HintsService.instance.transferHints(this::getPreferredHintsStreamTarget);
}
+ private static EndpointsForRange getStreamCandidates(Collection<InetAddressAndPort> endpoints)
+ {
+ endpoints = endpoints.stream()
+ .filter(endpoint -> FailureDetector.instance.isAlive(endpoint) && !FBUtilities.getBroadcastAddressAndPort().equals(endpoint))
+ .collect(Collectors.toList());
+
+ return SystemReplicas.getSystemReplicas(endpoints);
+ }
/**
* Find the best target to stream hints to. Currently the closest peer according to the snitch
*/
private UUID getPreferredHintsStreamTarget()
{
- List<InetAddress> candidates = new ArrayList<>(StorageService.instance.getTokenMetadata().cloneAfterAllLeft().getAllEndpoints());
- candidates.remove(FBUtilities.getBroadcastAddress());
- for (Iterator<InetAddress> iter = candidates.iterator(); iter.hasNext(); )
- {
- InetAddress address = iter.next();
- if (!FailureDetector.instance.isAlive(address))
- iter.remove();
- }
+ Set<InetAddressAndPort> endpoints = StorageService.instance.getTokenMetadata().cloneAfterAllLeft().getAllEndpoints();
+ EndpointsForRange candidates = getStreamCandidates(endpoints);
if (candidates.isEmpty())
{
logger.warn("Unable to stream hints since no live endpoints seen");
@@ -4111,8 +4306,8 @@
else
{
// stream to the closest peer as chosen by the snitch
- DatabaseDescriptor.getEndpointSnitch().sortByProximity(FBUtilities.getBroadcastAddress(), candidates);
- InetAddress hintsDestinationHost = candidates.get(0);
+ candidates = DatabaseDescriptor.getEndpointSnitch().sortedByProximity(FBUtilities.getBroadcastAddressAndPort(), candidates);
+ InetAddressAndPort hintsDestinationHost = candidates.get(0).endpoint();
return tokenMetadata.getHostId(hintsDestinationHost);
}
}
@@ -4146,7 +4341,7 @@
throw new IOException("target token " + newToken + " is already owned by another node.");
// address of the current node
- InetAddress localAddress = FBUtilities.getBroadcastAddress();
+ InetAddressAndPort localAddress = FBUtilities.getBroadcastAddressAndPort();
// This doesn't make any sense in a vnodes environment.
if (getTokenMetadata().getTokens(localAddress).size() > 1)
@@ -4161,17 +4356,20 @@
// checking if data is moving to this node
for (String keyspaceName : keyspacesToProcess)
{
+ // TODO: do we care about fixing transient/full self-movements here?
if (tokenMetadata.getPendingRanges(keyspaceName, localAddress).size() > 0)
throw new UnsupportedOperationException("data is currently moving to this node; unable to leave the ring");
}
+ Gossiper.instance.addLocalApplicationState(ApplicationState.STATUS_WITH_PORT, valueFactory.moving(newToken));
Gossiper.instance.addLocalApplicationState(ApplicationState.STATUS, valueFactory.moving(newToken));
setMode(Mode.MOVING, String.format("Moving %s from %s to %s.", localAddress, getLocalTokens().iterator().next(), newToken), true);
setMode(Mode.MOVING, String.format("Sleeping %s ms before start streaming/fetching ranges", RING_DELAY), true);
- Uninterruptibles.sleepUninterruptibly(RING_DELAY, TimeUnit.MILLISECONDS);
+ Uninterruptibles.sleepUninterruptibly(RING_DELAY, MILLISECONDS);
- RangeRelocator relocator = new RangeRelocator(Collections.singleton(newToken), keyspacesToProcess);
+ RangeRelocator relocator = new RangeRelocator(Collections.singleton(newToken), keyspacesToProcess, tokenMetadata);
+ relocator.calculateToFromStreams();
if (relocator.streamsNeeded())
{
@@ -4196,162 +4394,39 @@
logger.debug("Successfully moved to new token {}", getLocalTokens().iterator().next());
}
- private class RangeRelocator
+ public String getRemovalStatus()
{
- private final StreamPlan streamPlan = new StreamPlan("Relocation");
+ return getRemovalStatus(false);
+ }
- private RangeRelocator(Collection<Token> tokens, List<String> keyspaceNames)
- {
- calculateToFromStreams(tokens, keyspaceNames);
- }
-
- private void calculateToFromStreams(Collection<Token> newTokens, List<String> keyspaceNames)
- {
- InetAddress localAddress = FBUtilities.getBroadcastAddress();
- IEndpointSnitch snitch = DatabaseDescriptor.getEndpointSnitch();
- TokenMetadata tokenMetaCloneAllSettled = tokenMetadata.cloneAfterAllSettled();
- // clone to avoid concurrent modification in calculateNaturalEndpoints
- TokenMetadata tokenMetaClone = tokenMetadata.cloneOnlyTokenMap();
-
- for (String keyspace : keyspaceNames)
- {
- // replication strategy of the current keyspace
- AbstractReplicationStrategy strategy = Keyspace.open(keyspace).getReplicationStrategy();
- Multimap<InetAddress, Range<Token>> endpointToRanges = strategy.getAddressRanges();
-
- logger.debug("Calculating ranges to stream and request for keyspace {}", keyspace);
- for (Token newToken : newTokens)
- {
- // getting collection of the currently used ranges by this keyspace
- Collection<Range<Token>> currentRanges = endpointToRanges.get(localAddress);
- // collection of ranges which this node will serve after move to the new token
- Collection<Range<Token>> updatedRanges = strategy.getPendingAddressRanges(tokenMetaClone, newToken, localAddress);
-
- // ring ranges and endpoints associated with them
- // this used to determine what nodes should we ping about range data
- Multimap<Range<Token>, InetAddress> rangeAddresses = strategy.getRangeAddresses(tokenMetaClone);
-
- // calculated parts of the ranges to request/stream from/to nodes in the ring
- Pair<Set<Range<Token>>, Set<Range<Token>>> rangesPerKeyspace = calculateStreamAndFetchRanges(currentRanges, updatedRanges);
-
- /**
- * In this loop we are going through all ranges "to fetch" and determining
- * nodes in the ring responsible for data we are interested in
- */
- Multimap<Range<Token>, InetAddress> rangesToFetchWithPreferredEndpoints = ArrayListMultimap.create();
- for (Range<Token> toFetch : rangesPerKeyspace.right)
- {
- for (Range<Token> range : rangeAddresses.keySet())
- {
- if (range.contains(toFetch))
- {
- List<InetAddress> endpoints = null;
-
- if (useStrictConsistency)
- {
- Set<InetAddress> oldEndpoints = Sets.newHashSet(rangeAddresses.get(range));
- Set<InetAddress> newEndpoints = Sets.newHashSet(strategy.calculateNaturalEndpoints(toFetch.right, tokenMetaCloneAllSettled));
-
- //Due to CASSANDRA-5953 we can have a higher RF then we have endpoints.
- //So we need to be careful to only be strict when endpoints == RF
- if (oldEndpoints.size() == strategy.getReplicationFactor())
- {
- oldEndpoints.removeAll(newEndpoints);
-
- //No relocation required
- if (oldEndpoints.isEmpty())
- continue;
-
- assert oldEndpoints.size() == 1 : "Expected 1 endpoint but found " + oldEndpoints.size();
- }
-
- endpoints = Lists.newArrayList(oldEndpoints.iterator().next());
- }
- else
- {
- endpoints = snitch.getSortedListByProximity(localAddress, rangeAddresses.get(range));
- }
-
- // storing range and preferred endpoint set
- rangesToFetchWithPreferredEndpoints.putAll(toFetch, endpoints);
- }
- }
-
- Collection<InetAddress> addressList = rangesToFetchWithPreferredEndpoints.get(toFetch);
- if (addressList == null || addressList.isEmpty())
- continue;
-
- if (useStrictConsistency)
- {
- if (addressList.size() > 1)
- throw new IllegalStateException("Multiple strict sources found for " + toFetch);
-
- InetAddress sourceIp = addressList.iterator().next();
- if (Gossiper.instance.isEnabled() && !Gossiper.instance.getEndpointStateForEndpoint(sourceIp).isAlive())
- throw new RuntimeException("A node required to move the data consistently is down ("+sourceIp+"). If you wish to move the data from a potentially inconsistent replica, restart the node with -Dcassandra.consistent.rangemovement=false");
- }
- }
-
- // calculating endpoints to stream current ranges to if needed
- // in some situations node will handle current ranges as part of the new ranges
- Multimap<InetAddress, Range<Token>> endpointRanges = HashMultimap.create();
- for (Range<Token> toStream : rangesPerKeyspace.left)
- {
- Set<InetAddress> currentEndpoints = ImmutableSet.copyOf(strategy.calculateNaturalEndpoints(toStream.right, tokenMetaClone));
- Set<InetAddress> newEndpoints = ImmutableSet.copyOf(strategy.calculateNaturalEndpoints(toStream.right, tokenMetaCloneAllSettled));
- logger.debug("Range: {} Current endpoints: {} New endpoints: {}", toStream, currentEndpoints, newEndpoints);
- for (InetAddress address : Sets.difference(newEndpoints, currentEndpoints))
- {
- logger.debug("Range {} has new owner {}", toStream, address);
- endpointRanges.put(address, toStream);
- }
- }
-
- // stream ranges
- for (InetAddress address : endpointRanges.keySet())
- {
- logger.debug("Will stream range {} of keyspace {} to endpoint {}", endpointRanges.get(address), keyspace, address);
- InetAddress preferred = SystemKeyspace.getPreferredIP(address);
- streamPlan.transferRanges(address, preferred, keyspace, endpointRanges.get(address));
- }
-
- // stream requests
- Multimap<InetAddress, Range<Token>> workMap = RangeStreamer.getWorkMap(rangesToFetchWithPreferredEndpoints, keyspace, FailureDetector.instance, useStrictConsistency);
- for (InetAddress address : workMap.keySet())
- {
- logger.debug("Will request range {} of keyspace {} from endpoint {}", workMap.get(address), keyspace, address);
- InetAddress preferred = SystemKeyspace.getPreferredIP(address);
- streamPlan.requestRanges(address, preferred, keyspace, workMap.get(address));
- }
-
- logger.debug("Keyspace {}: work map {}.", keyspace, workMap);
- }
- }
- }
-
- public Future<StreamState> stream()
- {
- return streamPlan.execute();
- }
-
- public boolean streamsNeeded()
- {
- return !streamPlan.isEmpty();
- }
+ public String getRemovalStatusWithPort()
+ {
+ return getRemovalStatus(true);
}
/**
* Get the status of a token removal.
*/
- public String getRemovalStatus()
+ private String getRemovalStatus(boolean withPort)
{
if (removingNode == null)
{
return "No token removals in process.";
}
+
+ Collection toFormat = replicatingNodes;
+ if (!withPort)
+ {
+ toFormat = new ArrayList(replicatingNodes.size());
+ for (InetAddressAndPort node : replicatingNodes)
+ {
+ toFormat.add(node.toString(false));
+ }
+ }
+
return String.format("Removing token (%s). Waiting for replication confirmation from [%s].",
tokenMetadata.getToken(removingNode),
- StringUtils.join(replicatingNodes, ","));
+ StringUtils.join(toFormat, ","));
}
/**
@@ -4361,10 +4436,10 @@
*/
public void forceRemoveCompletion()
{
- if (!replicatingNodes.isEmpty() || !tokenMetadata.getLeavingEndpoints().isEmpty())
+ if (!replicatingNodes.isEmpty() || tokenMetadata.getSizeOfLeavingEndpoints() > 0)
{
logger.warn("Removal not confirmed for for {}", StringUtils.join(this.replicatingNodes, ","));
- for (InetAddress endpoint : tokenMetadata.getLeavingEndpoints())
+ for (InetAddressAndPort endpoint : tokenMetadata.getLeavingEndpoints())
{
UUID hostId = tokenMetadata.getHostId(endpoint);
Gossiper.instance.advertiseTokenRemoved(endpoint, hostId);
@@ -4390,10 +4465,10 @@
*/
public void removeNode(String hostIdString)
{
- InetAddress myAddress = FBUtilities.getBroadcastAddress();
+ InetAddressAndPort myAddress = FBUtilities.getBroadcastAddressAndPort();
UUID localHostId = tokenMetadata.getHostId(myAddress);
UUID hostId = UUID.fromString(hostIdString);
- InetAddress endpoint = tokenMetadata.getEndpointForHostId(hostId);
+ InetAddressAndPort endpoint = tokenMetadata.getEndpointForHostId(hostId);
if (endpoint == null)
throw new UnsupportedOperationException("Host ID not found.");
@@ -4420,14 +4495,14 @@
for (String keyspaceName : Schema.instance.getNonLocalStrategyKeyspaces())
{
// if the replication factor is 1 the data is lost so we shouldn't wait for confirmation
- if (Keyspace.open(keyspaceName).getReplicationStrategy().getReplicationFactor() == 1)
+ if (Keyspace.open(keyspaceName).getReplicationStrategy().getReplicationFactor().allReplicas == 1)
continue;
// get all ranges that change ownership (that is, a node needs
// to take responsibility for new range)
- Multimap<Range<Token>, InetAddress> changedRanges = getChangedRangesForLeaving(keyspaceName, endpoint);
+ EndpointsByReplica changedRanges = getChangedReplicasForLeaving(keyspaceName, endpoint, tokenMetadata, Keyspace.open(keyspaceName).getReplicationStrategy());
IFailureDetector failureDetector = FailureDetector.instance;
- for (InetAddress ep : changedRanges.values())
+ for (InetAddressAndPort ep : transform(changedRanges.flattenValues(), Replica::endpoint))
{
if (failureDetector.isAlive(ep))
replicatingNodes.add(ep);
@@ -4447,10 +4522,10 @@
// kick off streaming commands
restoreReplicaCount(endpoint, myAddress);
- // wait for ReplicationFinishedVerbHandler to signal we're done
+ // wait for ReplicationDoneVerbHandler to signal we're done
while (!replicatingNodes.isEmpty())
{
- Uninterruptibles.sleepUninterruptibly(100, TimeUnit.MILLISECONDS);
+ Uninterruptibles.sleepUninterruptibly(100, MILLISECONDS);
}
excise(tokens, endpoint);
@@ -4462,7 +4537,7 @@
removingNode = null;
}
- public void confirmReplication(InetAddress node)
+ public void confirmReplication(InetAddressAndPort node)
{
// replicatingNodes can be empty in the case where this node used to be a removal coordinator,
// but restarted before all 'replication finished' messages arrived. In that case, we'll
@@ -4522,9 +4597,9 @@
protected synchronized void drain(boolean isFinalShutdown) throws IOException, InterruptedException, ExecutionException
{
- ExecutorService counterMutationStage = StageManager.getStage(Stage.COUNTER_MUTATION);
- ExecutorService viewMutationStage = StageManager.getStage(Stage.VIEW_MUTATION);
- ExecutorService mutationStage = StageManager.getStage(Stage.MUTATION);
+ ExecutorService counterMutationStage = Stage.COUNTER_MUTATION.executor();
+ ExecutorService viewMutationStage = Stage.VIEW_MUTATION.executor();
+ ExecutorService mutationStage = Stage.MUTATION.executor();
if (mutationStage.isTerminated()
&& counterMutationStage.isTerminated()
@@ -4747,6 +4822,8 @@
public void truncate(String keyspace, String table) throws TimeoutException, IOException
{
+ verifyKeyspaceIsValid(keyspace);
+
try
{
StorageProxy.truncateBlocking(keyspace, table);
@@ -4765,12 +4842,30 @@
Map<InetAddress, Float> nodeMap = new LinkedHashMap<>();
for (Map.Entry<Token, Float> entry : tokenMap.entrySet())
{
- InetAddress endpoint = tokenMetadata.getEndpoint(entry.getKey());
+ InetAddressAndPort endpoint = tokenMetadata.getEndpoint(entry.getKey());
Float tokenOwnership = entry.getValue();
- if (nodeMap.containsKey(endpoint))
- nodeMap.put(endpoint, nodeMap.get(endpoint) + tokenOwnership);
+ if (nodeMap.containsKey(endpoint.address))
+ nodeMap.put(endpoint.address, nodeMap.get(endpoint.address) + tokenOwnership);
else
- nodeMap.put(endpoint, tokenOwnership);
+ nodeMap.put(endpoint.address, tokenOwnership);
+ }
+ return nodeMap;
+ }
+
+ public Map<String, Float> getOwnershipWithPort()
+ {
+ List<Token> sortedTokens = tokenMetadata.sortedTokens();
+ // describeOwnership returns tokens in an unspecified order, let's re-order them
+ Map<Token, Float> tokenMap = new TreeMap<Token, Float>(tokenMetadata.partitioner.describeOwnership(sortedTokens));
+ Map<String, Float> nodeMap = new LinkedHashMap<>();
+ for (Map.Entry<Token, Float> entry : tokenMap.entrySet())
+ {
+ InetAddressAndPort endpoint = tokenMetadata.getEndpoint(entry.getKey());
+ Float tokenOwnership = entry.getValue();
+ if (nodeMap.containsKey(endpoint.toString()))
+ nodeMap.put(endpoint.toString(), nodeMap.get(endpoint.toString()) + tokenOwnership);
+ else
+ nodeMap.put(endpoint.toString(), tokenOwnership);
}
return nodeMap;
}
@@ -4784,7 +4879,7 @@
*
* @throws IllegalStateException when node is not configured properly.
*/
- public LinkedHashMap<InetAddress, Float> effectiveOwnership(String keyspace) throws IllegalStateException
+ private LinkedHashMap<InetAddressAndPort, Float> getEffectiveOwnership(String keyspace)
{
AbstractReplicationStrategy strategy;
if (keyspace != null)
@@ -4824,28 +4919,27 @@
TokenMetadata metadata = tokenMetadata.cloneOnlyTokenMap();
- Collection<Collection<InetAddress>> endpointsGroupedByDc = new ArrayList<>();
+ Collection<Collection<InetAddressAndPort>> endpointsGroupedByDc = new ArrayList<>();
// mapping of dc's to nodes, use sorted map so that we get dcs sorted
- SortedMap<String, Collection<InetAddress>> sortedDcsToEndpoints = new TreeMap<>();
- sortedDcsToEndpoints.putAll(metadata.getTopology().getDatacenterEndpoints().asMap());
- for (Collection<InetAddress> endpoints : sortedDcsToEndpoints.values())
+ SortedMap<String, Collection<InetAddressAndPort>> sortedDcsToEndpoints = new TreeMap<>(metadata.getTopology().getDatacenterEndpoints().asMap());
+ for (Collection<InetAddressAndPort> endpoints : sortedDcsToEndpoints.values())
endpointsGroupedByDc.add(endpoints);
Map<Token, Float> tokenOwnership = tokenMetadata.partitioner.describeOwnership(tokenMetadata.sortedTokens());
- LinkedHashMap<InetAddress, Float> finalOwnership = Maps.newLinkedHashMap();
+ LinkedHashMap<InetAddressAndPort, Float> finalOwnership = Maps.newLinkedHashMap();
- Multimap<InetAddress, Range<Token>> endpointToRanges = strategy.getAddressRanges();
+ RangesByEndpoint endpointToRanges = strategy.getAddressReplicas();
// calculate ownership per dc
- for (Collection<InetAddress> endpoints : endpointsGroupedByDc)
+ for (Collection<InetAddressAndPort> endpoints : endpointsGroupedByDc)
{
// calculate the ownership with replication and add the endpoint to the final ownership map
- for (InetAddress endpoint : endpoints)
+ for (InetAddressAndPort endpoint : endpoints)
{
float ownership = 0.0f;
- for (Range<Token> range : endpointToRanges.get(endpoint))
+ for (Replica replica : endpointToRanges.get(endpoint))
{
- if (tokenOwnership.containsKey(range.right))
- ownership += tokenOwnership.get(range.right);
+ if (tokenOwnership.containsKey(replica.range().right))
+ ownership += tokenOwnership.get(replica.range().right);
}
finalOwnership.put(endpoint, ownership);
}
@@ -4853,6 +4947,22 @@
return finalOwnership;
}
+ public LinkedHashMap<InetAddress, Float> effectiveOwnership(String keyspace) throws IllegalStateException
+ {
+ LinkedHashMap<InetAddressAndPort, Float> result = getEffectiveOwnership(keyspace);
+ LinkedHashMap<InetAddress, Float> asInets = new LinkedHashMap<>();
+ result.entrySet().stream().forEachOrdered(entry -> asInets.put(entry.getKey().address, entry.getValue()));
+ return asInets;
+ }
+
+ public LinkedHashMap<String, Float> effectiveOwnershipWithPort(String keyspace) throws IllegalStateException
+ {
+ LinkedHashMap<InetAddressAndPort, Float> result = getEffectiveOwnership(keyspace);
+ LinkedHashMap<String, Float> asStrings = new LinkedHashMap<>();
+ result.entrySet().stream().forEachOrdered(entry -> asStrings.put(entry.getKey().toString(), entry.getValue()));
+ return asStrings;
+ }
+
public List<String> getKeyspaces()
{
List<String> keyspaceNamesList = new ArrayList<>(Schema.instance.getKeyspaces());
@@ -4870,25 +4980,33 @@
return Collections.unmodifiableList(Schema.instance.getNonLocalStrategyKeyspaces());
}
- public Map<String, String> getViewBuildStatuses(String keyspace, String view)
+ public Map<String, String> getViewBuildStatuses(String keyspace, String view, boolean withPort)
{
Map<UUID, String> coreViewStatus = SystemDistributedKeyspace.viewStatus(keyspace, view);
- Map<InetAddress, UUID> hostIdToEndpoint = tokenMetadata.getEndpointToHostIdMapForReading();
+ Map<InetAddressAndPort, UUID> hostIdToEndpoint = tokenMetadata.getEndpointToHostIdMapForReading();
Map<String, String> result = new HashMap<>();
- for (Map.Entry<InetAddress, UUID> entry : hostIdToEndpoint.entrySet())
+ for (Map.Entry<InetAddressAndPort, UUID> entry : hostIdToEndpoint.entrySet())
{
UUID hostId = entry.getValue();
- InetAddress endpoint = entry.getKey();
- result.put(endpoint.toString(),
- coreViewStatus.containsKey(hostId)
- ? coreViewStatus.get(hostId)
- : "UNKNOWN");
+ InetAddressAndPort endpoint = entry.getKey();
+ result.put(endpoint.toString(withPort),
+ coreViewStatus.getOrDefault(hostId, "UNKNOWN"));
}
return Collections.unmodifiableMap(result);
}
+ public Map<String, String> getViewBuildStatuses(String keyspace, String view)
+ {
+ return getViewBuildStatuses(keyspace, view, false);
+ }
+
+ public Map<String, String> getViewBuildStatusesWithPort(String keyspace, String view)
+ {
+ return getViewBuildStatuses(keyspace, view, true);
+ }
+
public void setDynamicUpdateInterval(int dynamicUpdateInterval)
{
if (DatabaseDescriptor.getEndpointSnitch() instanceof DynamicEndpointSnitch)
@@ -4976,128 +5094,68 @@
}
/**
- * Seed data to the endpoints that will be responsible for it at the future
+ * Send data to the endpoints that will be responsible for it in the future
*
* @param rangesToStreamByKeyspace keyspaces and data ranges with endpoints included for each
* @return async Future for whether stream was success
*/
- private Future<StreamState> streamRanges(Map<String, Multimap<Range<Token>, InetAddress>> rangesToStreamByKeyspace)
+ private Future<StreamState> streamRanges(Map<String, EndpointsByReplica> rangesToStreamByKeyspace)
{
// First, we build a list of ranges to stream to each host, per table
- Map<String, Map<InetAddress, List<Range<Token>>>> sessionsToStreamByKeyspace = new HashMap<>();
+ Map<String, RangesByEndpoint> sessionsToStreamByKeyspace = new HashMap<>();
- for (Map.Entry<String, Multimap<Range<Token>, InetAddress>> entry : rangesToStreamByKeyspace.entrySet())
+ for (Map.Entry<String, EndpointsByReplica> entry : rangesToStreamByKeyspace.entrySet())
{
String keyspace = entry.getKey();
- Multimap<Range<Token>, InetAddress> rangesWithEndpoints = entry.getValue();
+ EndpointsByReplica rangesWithEndpoints = entry.getValue();
if (rangesWithEndpoints.isEmpty())
continue;
- Map<InetAddress, Set<Range<Token>>> transferredRangePerKeyspace = SystemKeyspace.getTransferredRanges("Unbootstrap",
- keyspace,
- StorageService.instance.getTokenMetadata().partitioner);
- Map<InetAddress, List<Range<Token>>> rangesPerEndpoint = new HashMap<>();
- for (Map.Entry<Range<Token>, InetAddress> endPointEntry : rangesWithEndpoints.entries())
+ //Description is always Unbootstrap? Is that right?
+ Map<InetAddressAndPort, Set<Range<Token>>> transferredRangePerKeyspace = SystemKeyspace.getTransferredRanges("Unbootstrap",
+ keyspace,
+ StorageService.instance.getTokenMetadata().partitioner);
+ RangesByEndpoint.Builder replicasPerEndpoint = new RangesByEndpoint.Builder();
+ for (Map.Entry<Replica, Replica> endPointEntry : rangesWithEndpoints.flattenEntries())
{
- Range<Token> range = endPointEntry.getKey();
- InetAddress endpoint = endPointEntry.getValue();
-
- Set<Range<Token>> transferredRanges = transferredRangePerKeyspace.get(endpoint);
- if (transferredRanges != null && transferredRanges.contains(range))
+ Replica local = endPointEntry.getKey();
+ Replica remote = endPointEntry.getValue();
+ Set<Range<Token>> transferredRanges = transferredRangePerKeyspace.get(remote.endpoint());
+ if (transferredRanges != null && transferredRanges.contains(local.range()))
{
- logger.debug("Skipping transferred range {} of keyspace {}, endpoint {}", range, keyspace, endpoint);
+ logger.debug("Skipping transferred range {} of keyspace {}, endpoint {}", local, keyspace, remote);
continue;
}
- List<Range<Token>> curRanges = rangesPerEndpoint.get(endpoint);
- if (curRanges == null)
- {
- curRanges = new LinkedList<>();
- rangesPerEndpoint.put(endpoint, curRanges);
- }
- curRanges.add(range);
+ replicasPerEndpoint.put(remote.endpoint(), remote.decorateSubrange(local.range()));
}
- sessionsToStreamByKeyspace.put(keyspace, rangesPerEndpoint);
+ sessionsToStreamByKeyspace.put(keyspace, replicasPerEndpoint.build());
}
- StreamPlan streamPlan = new StreamPlan("Unbootstrap");
+ StreamPlan streamPlan = new StreamPlan(StreamOperation.DECOMMISSION);
- // Vinculate StreamStateStore to current StreamPlan to update transferred ranges per StreamSession
+ // Vinculate StreamStateStore to current StreamPlan to update transferred rangeas per StreamSession
streamPlan.listeners(streamStateStore);
- for (Map.Entry<String, Map<InetAddress, List<Range<Token>>>> entry : sessionsToStreamByKeyspace.entrySet())
+ for (Map.Entry<String, RangesByEndpoint> entry : sessionsToStreamByKeyspace.entrySet())
{
String keyspaceName = entry.getKey();
- Map<InetAddress, List<Range<Token>>> rangesPerEndpoint = entry.getValue();
+ RangesByEndpoint replicasPerEndpoint = entry.getValue();
- for (Map.Entry<InetAddress, List<Range<Token>>> rangesEntry : rangesPerEndpoint.entrySet())
+ for (Map.Entry<InetAddressAndPort, RangesAtEndpoint> rangesEntry : replicasPerEndpoint.asMap().entrySet())
{
- List<Range<Token>> ranges = rangesEntry.getValue();
- InetAddress newEndpoint = rangesEntry.getKey();
- InetAddress preferred = SystemKeyspace.getPreferredIP(newEndpoint);
+ RangesAtEndpoint replicas = rangesEntry.getValue();
+ InetAddressAndPort newEndpoint = rangesEntry.getKey();
// TODO each call to transferRanges re-flushes, this is potentially a lot of waste
- streamPlan.transferRanges(newEndpoint, preferred, keyspaceName, ranges);
+ streamPlan.transferRanges(newEndpoint, keyspaceName, replicas);
}
}
return streamPlan.execute();
}
- /**
- * Calculate pair of ranges to stream/fetch for given two range collections
- * (current ranges for keyspace and ranges after move to new token)
- *
- * @param current collection of the ranges by current token
- * @param updated collection of the ranges after token is changed
- * @return pair of ranges to stream/fetch for given current and updated range collections
- */
- public Pair<Set<Range<Token>>, Set<Range<Token>>> calculateStreamAndFetchRanges(Collection<Range<Token>> current, Collection<Range<Token>> updated)
- {
- Set<Range<Token>> toStream = new HashSet<>();
- Set<Range<Token>> toFetch = new HashSet<>();
-
-
- for (Range<Token> r1 : current)
- {
- boolean intersect = false;
- for (Range<Token> r2 : updated)
- {
- if (r1.intersects(r2))
- {
- // adding difference ranges to fetch from a ring
- toStream.addAll(r1.subtract(r2));
- intersect = true;
- }
- }
- if (!intersect)
- {
- toStream.add(r1); // should seed whole old range
- }
- }
-
- for (Range<Token> r2 : updated)
- {
- boolean intersect = false;
- for (Range<Token> r1 : current)
- {
- if (r2.intersects(r1))
- {
- // adding difference ranges to fetch from a ring
- toFetch.addAll(r2.subtract(r1));
- intersect = true;
- }
- }
- if (!intersect)
- {
- toFetch.add(r2); // should fetch whole old range
- }
- }
-
- return Pair.create(toStream, toFetch);
- }
-
public void bulkLoad(String directory)
{
try
@@ -5131,10 +5189,12 @@
this.keyspace = keyspace;
try
{
- for (Map.Entry<Range<Token>, List<InetAddress>> entry : StorageService.instance.getRangeToAddressMap(keyspace).entrySet())
+ for (Map.Entry<Range<Token>, EndpointsForRange> entry : StorageService.instance.getRangeToAddressMap(keyspace).entrySet())
{
Range<Token> range = entry.getKey();
- for (InetAddress endpoint : entry.getValue())
+ EndpointsForRange replicas = entry.getValue();
+ Replicas.temporaryAssertFull(replicas);
+ for (InetAddressAndPort endpoint : replicas.endpoints())
addRangeForEndpoint(range, endpoint);
}
}
@@ -5144,9 +5204,9 @@
}
}
- public CFMetaData getTableMetadata(String tableName)
+ public TableMetadataRef getTableMetadata(String tableName)
{
- return Schema.instance.getCFMetaData(keyspace, tableName);
+ return Schema.instance.getTableMetadataRef(keyspace, tableName);
}
};
@@ -5161,10 +5221,12 @@
/**
* #{@inheritDoc}
*/
+ @Deprecated
public void loadNewSSTables(String ksName, String cfName)
{
if (!isInitialized())
throw new RuntimeException("Not yet initialized, can't load new sstables");
+ verifyKeyspaceIsValid(ksName);
ColumnFamilyStore.loadNewSSTables(ksName, cfName);
}
@@ -5176,7 +5238,7 @@
List<DecoratedKey> keys = new ArrayList<>();
for (Keyspace keyspace : Keyspace.nonLocalStrategy())
{
- for (Range<Token> range : getPrimaryRangesForEndpoint(keyspace.getName(), FBUtilities.getBroadcastAddress()))
+ for (Range<Token> range : getPrimaryRangesForEndpoint(keyspace.getName(), FBUtilities.getBroadcastAddressAndPort()))
keys.addAll(keySamples(keyspace.getColumnFamilyStores(), range));
}
@@ -5186,6 +5248,44 @@
return sampledKeys;
}
+ /*
+ * { "sampler_name": [ {table: "", count: i, error: i, value: ""}, ... ] }
+ */
+ @Override
+ public Map<String, List<CompositeData>> samplePartitions(int durationMillis, int capacity, int count,
+ List<String> samplers) throws OpenDataException
+ {
+ ConcurrentHashMap<String, List<CompositeData>> result = new ConcurrentHashMap<>();
+ for (String sampler : samplers)
+ {
+ for (ColumnFamilyStore table : ColumnFamilyStore.all())
+ {
+ table.beginLocalSampling(sampler, capacity, durationMillis);
+ }
+ }
+ Uninterruptibles.sleepUninterruptibly(durationMillis, MILLISECONDS);
+
+ for (String sampler : samplers)
+ {
+ List<CompositeData> topk = new ArrayList<>();
+ for (ColumnFamilyStore table : ColumnFamilyStore.all())
+ {
+ topk.addAll(table.finishLocalSampling(sampler, count));
+ }
+ Collections.sort(topk, new Ordering<CompositeData>()
+ {
+ public int compare(CompositeData left, CompositeData right)
+ {
+ return Long.compare((long) right.get("count"), (long) left.get("count"));
+ }
+ });
+ // sublist is not serializable for jmx
+ topk = new ArrayList<>(topk.subList(0, Math.min(topk.size(), count)));
+ result.put(sampler, topk);
+ }
+ return result;
+ }
+
public void rebuildSecondaryIndex(String ksName, String cfName, String... idxNames)
{
String[] indices = asList(idxNames).stream()
@@ -5203,7 +5303,7 @@
public void reloadLocalSchema()
{
- SchemaKeyspace.reloadSchemaAndAnnounceVersion();
+ Schema.instance.reloadSchemaAndAnnounceVersion();
}
public void setTraceProbability(double probability)
@@ -5216,6 +5316,11 @@
return traceProbability;
}
+ public boolean shouldTraceProbablistically()
+ {
+ return traceProbability != 0 && ThreadLocalRandom.current().nextDouble() < traceProbability;
+ }
+
public void disableAutoCompaction(String ks, String... tables) throws IOException
{
for (ColumnFamilyStore cfs : getValidColumnFamilies(true, true, ks, tables))
@@ -5234,6 +5339,14 @@
}
}
+ public Map<String, Boolean> getAutoCompactionStatus(String ks, String... tables) throws IOException
+ {
+ Map<String, Boolean> status = new HashMap<String, Boolean>();
+ for (ColumnFamilyStore cfs : getValidColumnFamilies(true, true, ks, tables))
+ status.put(cfs.getTableName(), cfs.isAutoCompactionDisabled());
+ return status;
+ }
+
/** Returns the name of the cluster */
public String getClusterName()
{
@@ -5246,6 +5359,26 @@
return DatabaseDescriptor.getPartitionerName();
}
+ public void setSSTablePreemptiveOpenIntervalInMB(int intervalInMB)
+ {
+ DatabaseDescriptor.setSSTablePreemptiveOpenIntervalInMB(intervalInMB);
+ }
+
+ public int getSSTablePreemptiveOpenIntervalInMB()
+ {
+ return DatabaseDescriptor.getSSTablePreemptiveOpenIntervalInMB();
+ }
+
+ public boolean getMigrateKeycacheOnCompaction()
+ {
+ return DatabaseDescriptor.shouldMigrateKeycacheOnCompaction();
+ }
+
+ public void setMigrateKeycacheOnCompaction(boolean invalidateKeyCacheOnCompaction)
+ {
+ DatabaseDescriptor.setMigrateKeycacheOnCompaction(invalidateKeyCacheOnCompaction);
+ }
+
public int getTombstoneWarnThreshold()
{
return DatabaseDescriptor.getTombstoneWarnThreshold();
@@ -5266,6 +5399,17 @@
DatabaseDescriptor.setTombstoneFailureThreshold(threshold);
}
+ public int getColumnIndexCacheSize()
+ {
+ return DatabaseDescriptor.getColumnIndexCacheSizeInKB();
+ }
+
+ public void setColumnIndexCacheSize(int cacheSizeInKB)
+ {
+ DatabaseDescriptor.setColumnIndexCacheSize(cacheSizeInKB);
+ logger.info("Updated column_index_cache_size_in_kb to {}", cacheSizeInKB);
+ }
+
public int getBatchSizeFailureThreshold()
{
return DatabaseDescriptor.getBatchSizeFailThresholdInKB();
@@ -5274,6 +5418,53 @@
public void setBatchSizeFailureThreshold(int threshold)
{
DatabaseDescriptor.setBatchSizeFailThresholdInKB(threshold);
+ logger.info("Updated batch_size_fail_threshold_in_kb to {}", threshold);
+ }
+
+ public int getBatchSizeWarnThreshold()
+ {
+ return DatabaseDescriptor.getBatchSizeWarnThresholdInKB();
+ }
+
+ public void setBatchSizeWarnThreshold(int threshold)
+ {
+ DatabaseDescriptor.setBatchSizeWarnThresholdInKB(threshold);
+ logger.info("Updated batch_size_warn_threshold_in_kb to {}", threshold);
+ }
+
+ public int getInitialRangeTombstoneListAllocationSize()
+ {
+ return DatabaseDescriptor.getInitialRangeTombstoneListAllocationSize();
+ }
+
+ public void setInitialRangeTombstoneListAllocationSize(int size)
+ {
+ if (size < 0 || size > 1024)
+ {
+ throw new IllegalStateException("Not updating initial_range_tombstone_allocation_size as it must be in the range [0, 1024] inclusive");
+ }
+ int originalSize = DatabaseDescriptor.getInitialRangeTombstoneListAllocationSize();
+ DatabaseDescriptor.setInitialRangeTombstoneListAllocationSize(size);
+ logger.info("Updated initial_range_tombstone_allocation_size from {} to {}", originalSize, size);
+ }
+
+ public double getRangeTombstoneResizeListGrowthFactor()
+ {
+ return DatabaseDescriptor.getRangeTombstoneListGrowthFactor();
+ }
+
+ public void setRangeTombstoneListResizeGrowthFactor(double growthFactor) throws IllegalStateException
+ {
+ if (growthFactor < 1.2 || growthFactor > 5)
+ {
+ throw new IllegalStateException("Not updating range_tombstone_resize_factor as growth factor must be in the range [1.2, 5.0] inclusive");
+ }
+ else
+ {
+ double originalGrowthFactor = DatabaseDescriptor.getRangeTombstoneListGrowthFactor();
+ DatabaseDescriptor.setRangeTombstoneListGrowthFactor(growthFactor);
+ logger.info("Updated range_tombstone_resize_factor from {} to {}", originalGrowthFactor, growthFactor);
+ }
}
public void setHintedHandoffThrottleInKB(int throttleInKB)
@@ -5282,6 +5473,92 @@
logger.info("Updated hinted_handoff_throttle_in_kb to {}", throttleInKB);
}
+ @Override
+ public void clearConnectionHistory()
+ {
+ daemon.clearConnectionHistory();
+ logger.info("Cleared connection history");
+ }
+ public void disableAuditLog()
+ {
+ AuditLogManager.instance.disableAuditLog();
+ logger.info("Auditlog is disabled");
+ }
+
+ public void enableAuditLog(String loggerName, String includedKeyspaces, String excludedKeyspaces, String includedCategories, String excludedCategories,
+ String includedUsers, String excludedUsers) throws ConfigurationException, IllegalStateException
+ {
+ enableAuditLog(loggerName, Collections.emptyMap(), includedKeyspaces, excludedKeyspaces, includedCategories, excludedCategories, includedUsers, excludedUsers);
+ }
+
+ public void enableAuditLog(String loggerName, Map<String, String> parameters, String includedKeyspaces, String excludedKeyspaces, String includedCategories, String excludedCategories,
+ String includedUsers, String excludedUsers) throws ConfigurationException, IllegalStateException
+ {
+ loggerName = loggerName != null ? loggerName : DatabaseDescriptor.getAuditLoggingOptions().logger.class_name;
+
+ Preconditions.checkNotNull(loggerName, "cassandra.yaml did not have logger in audit_logging_option and not set as parameter");
+ Preconditions.checkState(FBUtilities.isAuditLoggerClassExists(loggerName), "Unable to find AuditLogger class: "+loggerName);
+
+ AuditLogOptions auditLogOptions = new AuditLogOptions();
+ auditLogOptions.enabled = true;
+ auditLogOptions.logger = new ParameterizedClass(loggerName, parameters);
+ auditLogOptions.included_keyspaces = includedKeyspaces != null ? includedKeyspaces : DatabaseDescriptor.getAuditLoggingOptions().included_keyspaces;
+ auditLogOptions.excluded_keyspaces = excludedKeyspaces != null ? excludedKeyspaces : DatabaseDescriptor.getAuditLoggingOptions().excluded_keyspaces;
+ auditLogOptions.included_categories = includedCategories != null ? includedCategories : DatabaseDescriptor.getAuditLoggingOptions().included_categories;
+ auditLogOptions.excluded_categories = excludedCategories != null ? excludedCategories : DatabaseDescriptor.getAuditLoggingOptions().excluded_categories;
+ auditLogOptions.included_users = includedUsers != null ? includedUsers : DatabaseDescriptor.getAuditLoggingOptions().included_users;
+ auditLogOptions.excluded_users = excludedUsers != null ? excludedUsers : DatabaseDescriptor.getAuditLoggingOptions().excluded_users;
+
+ AuditLogManager.instance.enable(auditLogOptions);
+
+ logger.info("AuditLog is enabled with logger: [{}], included_keyspaces: [{}], excluded_keyspaces: [{}], " +
+ "included_categories: [{}], excluded_categories: [{}], included_users: [{}], "
+ + "excluded_users: [{}], archive_command: [{}]", auditLogOptions.logger, auditLogOptions.included_keyspaces, auditLogOptions.excluded_keyspaces,
+ auditLogOptions.included_categories, auditLogOptions.excluded_categories, auditLogOptions.included_users, auditLogOptions.excluded_users,
+ auditLogOptions.archive_command);
+
+ }
+
+ public boolean isAuditLogEnabled()
+ {
+ return AuditLogManager.instance.isEnabled();
+ }
+
+ public String getCorruptedTombstoneStrategy()
+ {
+ return DatabaseDescriptor.getCorruptedTombstoneStrategy().toString();
+ }
+
+ public void setCorruptedTombstoneStrategy(String strategy)
+ {
+ DatabaseDescriptor.setCorruptedTombstoneStrategy(Config.CorruptedTombstoneStrategy.valueOf(strategy));
+ logger.info("Setting corrupted tombstone strategy to {}", strategy);
+ }
+
+ @Override
+ public long getNativeTransportMaxConcurrentRequestsInBytes()
+ {
+ return Server.EndpointPayloadTracker.getGlobalLimit();
+ }
+
+ @Override
+ public void setNativeTransportMaxConcurrentRequestsInBytes(long newLimit)
+ {
+ Server.EndpointPayloadTracker.setGlobalLimit(newLimit);
+ }
+
+ @Override
+ public long getNativeTransportMaxConcurrentRequestsInBytesPerIp()
+ {
+ return Server.EndpointPayloadTracker.getEndpointLimit();
+ }
+
+ @Override
+ public void setNativeTransportMaxConcurrentRequestsInBytesPerIp(long newLimit)
+ {
+ Server.EndpointPayloadTracker.setEndpointLimit(newLimit);
+ }
+
@VisibleForTesting
public void shutdownServer()
{
@@ -5290,4 +5567,32 @@
Runtime.getRuntime().removeShutdownHook(drainOnShutdown);
}
}
+
+ @Override
+ public void enableFullQueryLogger(String path, String rollCycle, Boolean blocking, int maxQueueWeight, long maxLogSize, String archiveCommand, int maxArchiveRetries)
+ {
+ FullQueryLoggerOptions fqlOptions = DatabaseDescriptor.getFullQueryLogOptions();
+ path = path != null ? path : fqlOptions.log_dir;
+ rollCycle = rollCycle != null ? rollCycle : fqlOptions.roll_cycle;
+ blocking = blocking != null ? blocking : fqlOptions.block;
+ maxQueueWeight = maxQueueWeight != Integer.MIN_VALUE ? maxQueueWeight : fqlOptions.max_queue_weight;
+ maxLogSize = maxLogSize != Long.MIN_VALUE ? maxLogSize : fqlOptions.max_log_size;
+ archiveCommand = archiveCommand != null ? archiveCommand : fqlOptions.archive_command;
+ maxArchiveRetries = maxArchiveRetries != Integer.MIN_VALUE ? maxArchiveRetries : fqlOptions.max_archive_retries;
+
+ Preconditions.checkNotNull(path, "cassandra.yaml did not set log_dir and not set as parameter");
+ FullQueryLogger.instance.enable(Paths.get(path), rollCycle, blocking, maxQueueWeight, maxLogSize, archiveCommand, maxArchiveRetries);
+ }
+
+ @Override
+ public void resetFullQueryLogger()
+ {
+ FullQueryLogger.instance.reset(DatabaseDescriptor.getFullQueryLogOptions().log_dir);
+ }
+
+ @Override
+ public void stopFullQueryLogger()
+ {
+ FullQueryLogger.instance.stop();
+ }
}
diff --git a/src/java/org/apache/cassandra/service/StorageServiceMBean.java b/src/java/org/apache/cassandra/service/StorageServiceMBean.java
index 4548fa8..9664769 100644
--- a/src/java/org/apache/cassandra/service/StorageServiceMBean.java
+++ b/src/java/org/apache/cassandra/service/StorageServiceMBean.java
@@ -18,18 +18,24 @@
package org.apache.cassandra.service;
import java.io.IOException;
+import java.io.Serializable;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.nio.ByteBuffer;
-import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeoutException;
-
+import javax.annotation.Nullable;
import javax.management.NotificationEmitter;
+import javax.management.openmbean.CompositeData;
+import javax.management.openmbean.OpenDataException;
import javax.management.openmbean.TabularData;
+import org.apache.cassandra.db.ColumnFamilyStoreMBean;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.utils.Pair;
+
public interface StorageServiceMBean extends NotificationEmitter
{
/**
@@ -38,7 +44,8 @@
*
* @return set of IP addresses, as Strings
*/
- public List<String> getLiveNodes();
+ @Deprecated public List<String> getLiveNodes();
+ public List<String> getLiveNodesWithPort();
/**
* Retrieve the list of unreachable nodes in the cluster, as determined
@@ -46,28 +53,32 @@
*
* @return set of IP addresses, as Strings
*/
- public List<String> getUnreachableNodes();
+ @Deprecated public List<String> getUnreachableNodes();
+ public List<String> getUnreachableNodesWithPort();
/**
* Retrieve the list of nodes currently bootstrapping into the ring.
*
* @return set of IP addresses, as Strings
*/
- public List<String> getJoiningNodes();
+ @Deprecated public List<String> getJoiningNodes();
+ public List<String> getJoiningNodesWithPort();
/**
* Retrieve the list of nodes currently leaving the ring.
*
* @return set of IP addresses, as Strings
*/
- public List<String> getLeavingNodes();
+ @Deprecated public List<String> getLeavingNodes();
+ public List<String> getLeavingNodesWithPort();
/**
* Retrieve the list of nodes currently moving in the ring.
*
* @return set of IP addresses, as Strings
*/
- public List<String> getMovingNodes();
+ @Deprecated public List<String> getMovingNodes();
+ public List<String> getMovingNodesWithPort();
/**
* Fetch string representations of the tokens for this node.
@@ -96,6 +107,11 @@
*/
public String getSchemaVersion();
+ /**
+ * Fetch the replication factor for a given keyspace.
+ * @return An integer that represents replication factor for the given keyspace.
+ */
+ public String getKeyspaceReplicationInfo(String keyspaceName);
/**
* Get the list of all data file locations from conf
@@ -121,7 +137,8 @@
*
* @return mapping of ranges to end points
*/
- public Map<List<String>, List<String>> getRangeToEndpointMap(String keyspace);
+ @Deprecated public Map<List<String>, List<String>> getRangeToEndpointMap(String keyspace);
+ public Map<List<String>, List<String>> getRangeToEndpointWithPortMap(String keyspace);
/**
* Retrieve a map of range to rpc addresses that describe the ring topology
@@ -129,7 +146,8 @@
*
* @return mapping of ranges to rpc addresses
*/
- public Map<List<String>, List<String>> getRangeToRpcaddressMap(String keyspace);
+ @Deprecated public Map<List<String>, List<String>> getRangeToRpcaddressMap(String keyspace);
+ public Map<List<String>, List<String>> getRangeToNativeaddressWithPortMap(String keyspace);
/**
* The same as {@code describeRing(String)} but converts TokenRange to the String for JMX compatibility
@@ -138,14 +156,16 @@
*
* @return a List of TokenRange(s) converted to String for the given keyspace
*/
- public List <String> describeRingJMX(String keyspace) throws IOException;
+ @Deprecated public List <String> describeRingJMX(String keyspace) throws IOException;
+ public List<String> describeRingWithPortJMX(String keyspace) throws IOException;
/**
* Retrieve a map of pending ranges to endpoints that describe the ring topology
* @param keyspace the keyspace to get the pending range map for.
* @return a map of pending ranges to endpoints
*/
- public Map<List<String>, List<String>> getPendingRangeToEndpointMap(String keyspace);
+ @Deprecated public Map<List<String>, List<String>> getPendingRangeToEndpointMap(String keyspace);
+ public Map<List<String>, List<String>> getPendingRangeToEndpointWithPortMap(String keyspace);
/**
* Retrieve a map of tokens to endpoints, including the bootstrapping
@@ -153,7 +173,8 @@
*
* @return a map of tokens to endpoints in ascending order
*/
- public Map<String, String> getTokenToEndpointMap();
+ @Deprecated public Map<String, String> getTokenToEndpointMap();
+ public Map<String, String> getTokenToEndpointWithPortMap();
/** Retrieve this hosts unique ID */
public String getLocalHostId();
@@ -163,16 +184,19 @@
public Map<String, String> getHostIdMap();
/** Retrieve the mapping of endpoint to host ID */
- public Map<String, String> getEndpointToHostId();
+ @Deprecated public Map<String, String> getEndpointToHostId();
+ public Map<String, String> getEndpointWithPortToHostId();
/** Retrieve the mapping of host ID to endpoint */
- public Map<String, String> getHostIdToEndpoint();
+ @Deprecated public Map<String, String> getHostIdToEndpoint();
+ public Map<String, String> getHostIdToEndpointWithPort();
/** Human-readable load value */
public String getLoadString();
/** Human-readable load value. Keys are IP addresses. */
- public Map<String, String> getLoadMap();
+ @Deprecated public Map<String, String> getLoadMap();
+ public Map<String, String> getLoadMapWithPort();
/**
* Return the generation value for this node.
@@ -190,8 +214,10 @@
* @param key - key for which we need to find the endpoint return value -
* the endpoint responsible for this key
*/
- public List<InetAddress> getNaturalEndpoints(String keyspaceName, String cf, String key);
- public List<InetAddress> getNaturalEndpoints(String keyspaceName, ByteBuffer key);
+ @Deprecated public List<InetAddress> getNaturalEndpoints(String keyspaceName, String cf, String key);
+ public List<String> getNaturalEndpointsWithPort(String keyspaceName, String cf, String key);
+ @Deprecated public List<InetAddress> getNaturalEndpoints(String keyspaceName, ByteBuffer key);
+ public List<String> getNaturalEndpointsWithPort(String keysapceName, ByteBuffer key);
/**
* @deprecated use {@link #takeSnapshot(String tag, Map options, String... entities)} instead.
@@ -293,6 +319,7 @@
* The entire sstable will be read to ensure each cell validates if extendedVerify is true
*/
public int verify(boolean extendedVerify, String keyspaceName, String... tableNames) throws IOException, ExecutionException, InterruptedException;
+ public int verify(boolean extendedVerify, boolean checkVersion, boolean diskFailurePolicy, boolean mutateRepairStatus, boolean checkOwnsTokens, boolean quick, String keyspaceName, String... tableNames) throws IOException, ExecutionException, InterruptedException;
/**
* Rewrite all sstables to the latest version.
@@ -330,55 +357,6 @@
*/
public int repairAsync(String keyspace, Map<String, String> options);
- /**
- * @deprecated use {@link #repairAsync(String keyspace, Map options)} instead.
- */
- @Deprecated
- public int forceRepairAsync(String keyspace, boolean isSequential, Collection<String> dataCenters, Collection<String> hosts, boolean primaryRange, boolean fullRepair, String... tableNames) throws IOException;
-
- /**
- * Invoke repair asynchronously.
- * You can track repair progress by subscribing JMX notification sent from this StorageServiceMBean.
- * Notification format is:
- * type: "repair"
- * userObject: int array of length 2, [0]=command number, [1]=ordinal of ActiveRepairService.Status
- *
- * @deprecated use {@link #repairAsync(String keyspace, Map options)} instead.
- *
- * @param parallelismDegree 0: sequential, 1: parallel, 2: DC parallel
- * @return Repair command number, or 0 if nothing to repair
- */
- @Deprecated
- public int forceRepairAsync(String keyspace, int parallelismDegree, Collection<String> dataCenters, Collection<String> hosts, boolean primaryRange, boolean fullRepair, String... tableNames);
-
- /**
- * @deprecated use {@link #repairAsync(String keyspace, Map options)} instead.
- */
- @Deprecated
- public int forceRepairRangeAsync(String beginToken, String endToken, String keyspaceName, boolean isSequential, Collection<String> dataCenters, Collection<String> hosts, boolean fullRepair, String... tableNames) throws IOException;
-
- /**
- * Same as forceRepairAsync, but handles a specified range
- *
- * @deprecated use {@link #repairAsync(String keyspace, Map options)} instead.
- *
- * @param parallelismDegree 0: sequential, 1: parallel, 2: DC parallel
- */
- @Deprecated
- public int forceRepairRangeAsync(String beginToken, String endToken, String keyspaceName, int parallelismDegree, Collection<String> dataCenters, Collection<String> hosts, boolean fullRepair, String... tableNames);
-
- /**
- * @deprecated use {@link #repairAsync(String keyspace, Map options)} instead.
- */
- @Deprecated
- public int forceRepairAsync(String keyspace, boolean isSequential, boolean isLocal, boolean primaryRange, boolean fullRepair, String... tableNames);
-
- /**
- * @deprecated use {@link #repairAsync(String keyspace, Map options)} instead.
- */
- @Deprecated
- public int forceRepairRangeAsync(String beginToken, String endToken, String keyspaceName, boolean isSequential, boolean isLocal, boolean fullRepair, String... tableNames);
-
public void forceTerminateAllRepairSessions();
public void setRepairSessionMaxTreeDepth(int depth);
@@ -386,9 +364,19 @@
public int getRepairSessionMaxTreeDepth();
/**
- * transfer this node's data to other machines and remove it from service.
+ * Get the status of a given parent repair session.
+ * @param cmd the int reference returned when issuing the repair
+ * @return status of parent repair from enum {@link org.apache.cassandra.repair.RepairRunnable.Status}
+ * followed by final message or messages of the session
*/
- public void decommission() throws InterruptedException;
+ @Nullable
+ public List<String> getParentRepairStatus(int cmd);
+
+ /**
+ * transfer this node's data to other machines and remove it from service.
+ * @param force Decommission even if this will reduce N to be less than RF.
+ */
+ public void decommission(boolean force) throws InterruptedException;
/**
* @param newToken token to move this node to.
@@ -405,7 +393,8 @@
/**
* Get the status of a token removal.
*/
- public String getRemovalStatus();
+ @Deprecated public String getRemovalStatus();
+ public String getRemovalStatusWithPort();
/**
* Force a remove operation to finish.
@@ -460,7 +449,8 @@
* given a list of tokens (representing the nodes in the cluster), returns
* a mapping from {@code "token -> %age of cluster owned by that token"}
*/
- public Map<InetAddress, Float> getOwnership();
+ @Deprecated public Map<InetAddress, Float> getOwnership();
+ public Map<String, Float> getOwnershipWithPort();
/**
* Effective ownership is % of the data each node owns given the keyspace
@@ -469,7 +459,8 @@
* in the cluster have the same replication strategies and if yes then we will
* use the first else a empty Map is returned.
*/
- public Map<InetAddress, Float> effectiveOwnership(String keyspace) throws IllegalStateException;
+ @Deprecated public Map<InetAddress, Float> effectiveOwnership(String keyspace) throws IllegalStateException;
+ public Map<String, Float> effectiveOwnershipWithPort(String keyspace) throws IllegalStateException;
public List<String> getKeyspaces();
@@ -477,7 +468,8 @@
public List<String> getNonLocalStrategyKeyspaces();
- public Map<String, String> getViewBuildStatuses(String keyspace, String view);
+ @Deprecated public Map<String, String> getViewBuildStatuses(String keyspace, String view);
+ public Map<String, String> getViewBuildStatusesWithPort(String keyspace, String view);
/**
* Change endpointsnitch class and dynamic-ness (and dynamic attributes) at runtime.
@@ -522,25 +514,24 @@
// to determine if initialization has completed
public boolean isInitialized();
- // allows a user to disable thrift
- public void stopRPCServer();
-
- // allows a user to reenable thrift
- public void startRPCServer();
-
- // to determine if thrift is running
- public boolean isRPCServerRunning();
-
public void stopNativeTransport();
public void startNativeTransport();
public boolean isNativeTransportRunning();
+ public void enableNativeTransportOldProtocolVersions();
+ public void disableNativeTransportOldProtocolVersions();
+
+ // sets limits on number of concurrent requests in flights in number of bytes
+ public long getNativeTransportMaxConcurrentRequestsInBytes();
+ public void setNativeTransportMaxConcurrentRequestsInBytes(long newLimit);
+ public long getNativeTransportMaxConcurrentRequestsInBytesPerIp();
+ public void setNativeTransportMaxConcurrentRequestsInBytesPerIp(long newLimit);
+
// allows a node that have been started without joining the ring to join it
public void joinRing() throws IOException;
public boolean isJoined();
public boolean isDrained();
public boolean isDraining();
-
/** Check if currently bootstrapping.
* Note this becomes false before {@link org.apache.cassandra.db.SystemKeyspace#bootstrapComplete()} is called,
* as setting bootstrap to complete is called only when the node joins the ring.
@@ -560,6 +551,12 @@
public void setWriteRpcTimeout(long value);
public long getWriteRpcTimeout();
+ public void setInternodeTcpConnectTimeoutInMS(int value);
+ public int getInternodeTcpConnectTimeoutInMS();
+
+ public void setInternodeTcpUserTimeoutInMS(int value);
+ public int getInternodeTcpUserTimeoutInMS();
+
public void setCounterWriteRpcTimeout(long value);
public long getCounterWriteRpcTimeout();
@@ -569,9 +566,6 @@
public void setTruncateRpcTimeout(long value);
public long getTruncateRpcTimeout();
- public void setStreamingSocketTimeout(int value);
- public int getStreamingSocketTimeout();
-
public void setStreamThroughputMbPerSec(int value);
public int getStreamThroughputMbPerSec();
@@ -581,9 +575,28 @@
public int getCompactionThroughputMbPerSec();
public void setCompactionThroughputMbPerSec(int value);
+ public int getBatchlogReplayThrottleInKB();
+ public void setBatchlogReplayThrottleInKB(int value);
+
public int getConcurrentCompactors();
public void setConcurrentCompactors(int value);
+ public void bypassConcurrentValidatorsLimit();
+ public void enforceConcurrentValidatorsLimit();
+ public boolean isConcurrentValidatorsLimitEnforced();
+
+ public int getConcurrentValidators();
+ public void setConcurrentValidators(int value);
+
+ public int getSSTablePreemptiveOpenIntervalInMB();
+ public void setSSTablePreemptiveOpenIntervalInMB(int intervalInMB);
+
+ public boolean getMigrateKeycacheOnCompaction();
+ public void setMigrateKeycacheOnCompaction(boolean invalidateKeyCacheOnCompaction);
+
+ public int getConcurrentViewBuilders();
+ public void setConcurrentViewBuilders(int value);
+
public boolean isIncrementalBackupsEnabled();
public void setIncrementalBackupsEnabled(boolean value);
@@ -622,7 +635,10 @@
*
* @param ksName The parent keyspace name
* @param tableName The ColumnFamily name where SSTables belong
+ *
+ * @see ColumnFamilyStoreMBean#loadNewSSTables()
*/
+ @Deprecated
public void loadNewSSTables(String ksName, String tableName);
/**
@@ -645,7 +661,7 @@
public void reloadLocalSchema();
/**
- * Enables/Disables tracing for the whole system. Only thrift requests can start tracing currently.
+ * Enables/Disables tracing for the whole system.
*
* @param probability
* ]0,1[ will enable tracing on a partial number of requests with the provided probability. 0 will
@@ -653,6 +669,8 @@
*/
public void setTraceProbability(double probability);
+ public Map<String, List<CompositeData>> samplePartitions(int duration, int capacity, int count, List<String> samplers) throws OpenDataException;
+
/**
* Returns the configured tracing probability.
*/
@@ -660,6 +678,7 @@
void disableAutoCompaction(String ks, String ... tables) throws IOException;
void enableAutoCompaction(String ks, String ... tables) throws IOException;
+ Map<String, Boolean> getAutoCompactionStatus(String ks, String... tables) throws IOException;
public void deliverHints(String host) throws UnknownHostException;
@@ -678,11 +697,21 @@
/** Sets the threshold for abandoning queries with many tombstones */
public void setTombstoneFailureThreshold(int tombstoneDebugThreshold);
+ /** Returns the threshold for skipping the column index when caching partition info **/
+ public int getColumnIndexCacheSize();
+ /** Sets the threshold for skipping the column index when caching partition info **/
+ public void setColumnIndexCacheSize(int cacheSizeInKB);
+
/** Returns the threshold for rejecting queries due to a large batch size */
public int getBatchSizeFailureThreshold();
/** Sets the threshold for rejecting queries due to a large batch size */
public void setBatchSizeFailureThreshold(int batchSizeDebugThreshold);
+ /** Returns the threshold for warning queries due to a large batch size */
+ public int getBatchSizeWarnThreshold();
+ /** Sets the threshold for warning queries due to a large batch size */
+ public void setBatchSizeWarnThreshold(int batchSizeDebugThreshold);
+
/** Sets the hinted handoff throttle in kb per second, per delivery thread. */
public void setHintedHandoffThrottleInKB(int throttleInKB);
@@ -694,6 +723,66 @@
*/
public boolean resumeBootstrap();
- /** Returns the max version that this node will negotiate for native protocol connections */
- public int getMaxNativeProtocolVersion();
+ /** Gets the concurrency settings for processing stages*/
+ static class StageConcurrency implements Serializable
+ {
+ public final int corePoolSize;
+ public final int maximumPoolSize;
+
+ public StageConcurrency(int corePoolSize, int maximumPoolSize)
+ {
+ this.corePoolSize = corePoolSize;
+ this.maximumPoolSize = maximumPoolSize;
+ }
+
+ }
+ public Map<String, List<Integer>> getConcurrency(List<String> stageNames);
+
+ /** Sets the concurrency setting for processing stages */
+ public void setConcurrency(String threadPoolName, int newCorePoolSize, int newMaximumPoolSize);
+
+ /** Clears the history of clients that have connected in the past **/
+ void clearConnectionHistory();
+ public void disableAuditLog();
+ public void enableAuditLog(String loggerName, Map<String, String> parameters, String includedKeyspaces, String excludedKeyspaces, String includedCategories, String excludedCategories, String includedUsers, String excludedUsers) throws ConfigurationException;
+ public void enableAuditLog(String loggerName, String includedKeyspaces, String excludedKeyspaces, String includedCategories, String excludedCategories, String includedUsers, String excludedUsers) throws ConfigurationException;
+ public boolean isAuditLogEnabled();
+ public String getCorruptedTombstoneStrategy();
+ public void setCorruptedTombstoneStrategy(String strategy);
+
+ /**
+ * Start the fully query logger.
+ * @param path Path where the full query log will be stored. If null cassandra.yaml value is used.
+ * @param rollCycle How often to create a new file for query data (MINUTELY, DAILY, HOURLY)
+ * @param blocking Whether threads submitting queries to the query log should block if they can't be drained to the filesystem or alternatively drops samples and log
+ * @param maxQueueWeight How many bytes of query data to queue before blocking or dropping samples
+ * @param maxLogSize How many bytes of log data to store before dropping segments. Might not be respected if a log file hasn't rolled so it can be deleted.
+ * @param archiveCommand executable archiving the rolled log files,
+ * @param maxArchiveRetries max number of times to retry a failing archive command
+ *
+ */
+ public void enableFullQueryLogger(String path, String rollCycle, Boolean blocking, int maxQueueWeight, long maxLogSize, @Nullable String archiveCommand, int maxArchiveRetries);
+
+ /**
+ * Disable the full query logger if it is enabled.
+ * Also delete any generated files in the last used full query log path as well as the one configure in cassandra.yaml
+ */
+ public void resetFullQueryLogger();
+
+ /**
+ * Stop logging queries but leave any generated files on disk.
+ */
+ public void stopFullQueryLogger();
+
+ /** Sets the initial allocation size of backing arrays for new RangeTombstoneList objects */
+ public void setInitialRangeTombstoneListAllocationSize(int size);
+
+ /** Returns the initial allocation size of backing arrays for new RangeTombstoneList objects */
+ public int getInitialRangeTombstoneListAllocationSize();
+
+ /** Sets the resize factor to use when growing/resizing a RangeTombstoneList */
+ public void setRangeTombstoneListResizeGrowthFactor(double growthFactor);
+
+ /** Returns the resize factor to use when growing/resizing a RangeTombstoneList */
+ public double getRangeTombstoneResizeListGrowthFactor();
}
diff --git a/src/java/org/apache/cassandra/service/TokenRange.java b/src/java/org/apache/cassandra/service/TokenRange.java
new file mode 100644
index 0000000..37971f5
--- /dev/null
+++ b/src/java/org/apache/cassandra/service/TokenRange.java
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.service;
+
+import java.util.*;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.locator.IEndpointSnitch;
+import org.apache.cassandra.locator.InetAddressAndPort;
+
+/**
+ * Holds token range informations for the sake of {@link StorageService#describeRing}.
+ *
+ * This class mostly exists for the sake of {@link StorageService#describeRing},
+ * which used to rely on a thrift class which this is the equivalent of. This is
+ * the reason this class behave how it does and the reason for the format
+ * of {@code toString()} in particular (used by
+ * {@link StorageService#describeRingJMX}). This class probably have no other
+ * good uses than providing backward compatibility.
+ */
+public class TokenRange
+{
+ private final Token.TokenFactory tokenFactory;
+
+ public final Range<Token> range;
+ public final List<EndpointDetails> endpoints;
+
+ private TokenRange(Token.TokenFactory tokenFactory, Range<Token> range, List<EndpointDetails> endpoints)
+ {
+ this.tokenFactory = tokenFactory;
+ this.range = range;
+ this.endpoints = endpoints;
+ }
+
+ private String toStr(Token tk)
+ {
+ return tokenFactory.toString(tk);
+ }
+
+ public static TokenRange create(Token.TokenFactory tokenFactory, Range<Token> range, List<InetAddressAndPort> endpoints, boolean withPorts)
+ {
+ List<EndpointDetails> details = new ArrayList<>(endpoints.size());
+ IEndpointSnitch snitch = DatabaseDescriptor.getEndpointSnitch();
+ for (InetAddressAndPort ep : endpoints)
+ details.add(new EndpointDetails(ep,
+ StorageService.instance.getNativeaddress(ep, withPorts),
+ snitch.getDatacenter(ep),
+ snitch.getRack(ep)));
+ return new TokenRange(tokenFactory, range, details);
+ }
+
+ @Override
+ public String toString()
+ {
+ return toString(false);
+ }
+
+ public String toString(boolean withPorts)
+ {
+ StringBuilder sb = new StringBuilder("TokenRange(");
+
+ sb.append("start_token:").append(toStr(range.left));
+ sb.append(", end_token:").append(toStr(range.right));
+
+ List<String> hosts = new ArrayList<>(endpoints.size());
+ List<String> rpcs = new ArrayList<>(endpoints.size());
+ List<String> endpointDetails = new ArrayList<>(endpoints.size());
+ for (EndpointDetails ep : endpoints)
+ {
+ hosts.add(ep.host.getHostAddress(withPorts));
+ rpcs.add(ep.nativeAddress);
+ endpointDetails.add(ep.toString(withPorts));
+ }
+
+ sb.append(", endpoints:").append(hosts);
+ sb.append(", rpc_endpoints:").append(rpcs);
+ sb.append(", endpoint_details:").append(endpointDetails);
+
+ sb.append(")");
+ return sb.toString();
+ }
+
+ public static class EndpointDetails
+ {
+ public final InetAddressAndPort host;
+ public final String nativeAddress;
+ public final String datacenter;
+ public final String rack;
+
+ private EndpointDetails(InetAddressAndPort host, String nativeAddress, String datacenter, String rack)
+ {
+ // dc and rack can be null, but host shouldn't
+ assert host != null;
+ this.host = host;
+ this.nativeAddress = nativeAddress;
+ this.datacenter = datacenter;
+ this.rack = rack;
+ }
+
+ @Override
+ public String toString()
+ {
+ return toString(false);
+ }
+
+ public String toString(boolean withPorts)
+ {
+ // Format matters for backward compatibility with describeRing()
+ String dcStr = datacenter == null ? "" : String.format(", datacenter:%s", datacenter);
+ String rackStr = rack == null ? "" : String.format(", rack:%s", rack);
+ return String.format("EndpointDetails(host:%s%s%s)", host.getHostAddress(withPorts), dcStr, rackStr);
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/service/TruncateResponseHandler.java b/src/java/org/apache/cassandra/service/TruncateResponseHandler.java
index cce8ecc..bcd7426 100644
--- a/src/java/org/apache/cassandra/service/TruncateResponseHandler.java
+++ b/src/java/org/apache/cassandra/service/TruncateResponseHandler.java
@@ -17,7 +17,6 @@
*/
package org.apache.cassandra.service;
-import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicInteger;
@@ -25,11 +24,13 @@
import org.slf4j.LoggerFactory;
import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.net.IAsyncCallback;
-import org.apache.cassandra.net.MessageIn;
+import org.apache.cassandra.net.RequestCallback;
+import org.apache.cassandra.net.Message;
import org.apache.cassandra.utils.concurrent.SimpleCondition;
-public class TruncateResponseHandler implements IAsyncCallback
+import static java.util.concurrent.TimeUnit.NANOSECONDS;
+
+public class TruncateResponseHandler implements RequestCallback
{
protected static final Logger logger = LoggerFactory.getLogger(TruncateResponseHandler.class);
protected final SimpleCondition condition = new SimpleCondition();
@@ -49,11 +50,11 @@
public void get() throws TimeoutException
{
- long timeout = TimeUnit.MILLISECONDS.toNanos(DatabaseDescriptor.getTruncateRpcTimeout()) - (System.nanoTime() - start);
+ long timeoutNanos = DatabaseDescriptor.getTruncateRpcTimeout(NANOSECONDS) - (System.nanoTime() - start);
boolean success;
try
{
- success = condition.await(timeout, TimeUnit.NANOSECONDS); // TODO truncate needs a much longer timeout
+ success = condition.await(timeoutNanos, NANOSECONDS); // TODO truncate needs a much longer timeout
}
catch (InterruptedException ex)
{
@@ -66,15 +67,10 @@
}
}
- public void response(MessageIn message)
+ public void onResponse(Message message)
{
responses.incrementAndGet();
if (responses.get() >= responseCount)
condition.signalAll();
}
-
- public boolean isLatencyForSnitch()
- {
- return false;
- }
}
diff --git a/src/java/org/apache/cassandra/service/WriteResponseHandler.java b/src/java/org/apache/cassandra/service/WriteResponseHandler.java
index 46e4e93..94f5a80 100644
--- a/src/java/org/apache/cassandra/service/WriteResponseHandler.java
+++ b/src/java/org/apache/cassandra/service/WriteResponseHandler.java
@@ -17,18 +17,13 @@
*/
package org.apache.cassandra.service;
-import java.net.InetAddress;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
+import org.apache.cassandra.locator.ReplicaPlan;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.cassandra.db.Keyspace;
-import org.apache.cassandra.net.MessageIn;
-import org.apache.cassandra.db.ConsistencyLevel;
+import org.apache.cassandra.net.Message;
import org.apache.cassandra.db.WriteType;
/**
@@ -42,41 +37,32 @@
private static final AtomicIntegerFieldUpdater<WriteResponseHandler> responsesUpdater
= AtomicIntegerFieldUpdater.newUpdater(WriteResponseHandler.class, "responses");
- public WriteResponseHandler(Collection<InetAddress> writeEndpoints,
- Collection<InetAddress> pendingEndpoints,
- ConsistencyLevel consistencyLevel,
- Keyspace keyspace,
+ public WriteResponseHandler(ReplicaPlan.ForTokenWrite replicaPlan,
Runnable callback,
WriteType writeType,
long queryStartNanoTime)
{
- super(keyspace, writeEndpoints, pendingEndpoints, consistencyLevel, callback, writeType, queryStartNanoTime);
- responses = totalBlockFor();
+ super(replicaPlan, callback, writeType, queryStartNanoTime);
+ responses = blockFor();
}
- public WriteResponseHandler(InetAddress endpoint, WriteType writeType, Runnable callback, long queryStartNanoTime)
+ public WriteResponseHandler(ReplicaPlan.ForTokenWrite replicaPlan, WriteType writeType, long queryStartNanoTime)
{
- this(Arrays.asList(endpoint), Collections.<InetAddress>emptyList(), ConsistencyLevel.ONE, null, callback, writeType, queryStartNanoTime);
+ this(replicaPlan, null, writeType, queryStartNanoTime);
}
- public WriteResponseHandler(InetAddress endpoint, WriteType writeType, long queryStartNanoTime)
- {
- this(endpoint, writeType, null, queryStartNanoTime);
- }
-
- public void response(MessageIn<T> m)
+ public void onResponse(Message<T> m)
{
if (responsesUpdater.decrementAndGet(this) == 0)
signal();
+ //Must be last after all subclass processing
+ //The two current subclasses both assume logResponseToIdealCLDelegate is called
+ //here.
+ logResponseToIdealCLDelegate(m);
}
protected int ackCount()
{
- return totalBlockFor() - responses;
- }
-
- public boolean isLatencyForSnitch()
- {
- return false;
+ return blockFor() - responses;
}
}
diff --git a/src/java/org/apache/cassandra/service/pager/AbstractQueryPager.java b/src/java/org/apache/cassandra/service/pager/AbstractQueryPager.java
index fa3f262..3faa253 100644
--- a/src/java/org/apache/cassandra/service/pager/AbstractQueryPager.java
+++ b/src/java/org/apache/cassandra/service/pager/AbstractQueryPager.java
@@ -17,18 +17,18 @@
*/
package org.apache.cassandra.service.pager;
-import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.rows.*;
import org.apache.cassandra.db.partitions.*;
import org.apache.cassandra.db.filter.DataLimits;
import org.apache.cassandra.db.transform.Transformation;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.service.ClientState;
import org.apache.cassandra.transport.ProtocolVersion;
-abstract class AbstractQueryPager implements QueryPager
+abstract class AbstractQueryPager<T extends ReadQuery> implements QueryPager
{
- protected final ReadCommand command;
+ protected final T query;
protected final DataLimits limits;
protected final ProtocolVersion protocolVersion;
private final boolean enforceStrictLiveness;
@@ -43,12 +43,12 @@
private boolean exhausted;
- protected AbstractQueryPager(ReadCommand command, ProtocolVersion protocolVersion)
+ protected AbstractQueryPager(T query, ProtocolVersion protocolVersion)
{
- this.command = command;
+ this.query = query;
this.protocolVersion = protocolVersion;
- this.limits = command.limits();
- this.enforceStrictLiveness = command.metadata().enforceStrictLiveness();
+ this.limits = query.limits();
+ this.enforceStrictLiveness = query.metadata().enforceStrictLiveness();
this.remaining = limits.count();
this.remainingInPartition = limits.perPartitionCount();
@@ -56,7 +56,7 @@
public ReadExecutionController executionController()
{
- return command.executionController();
+ return query.executionController();
}
public PartitionIterator fetchPage(int pageSize, ConsistencyLevel consistency, ClientState clientState, long queryStartNanoTime)
@@ -65,14 +65,14 @@
return EmptyIterators.partition();
pageSize = Math.min(pageSize, remaining);
- Pager pager = new RowPager(limits.forPaging(pageSize), command.nowInSec());
- ReadCommand readCommand = nextPageReadCommand(pageSize);
- if (readCommand == null)
+ Pager pager = new RowPager(limits.forPaging(pageSize), query.nowInSec());
+ ReadQuery readQuery = nextPageReadQuery(pageSize);
+ if (readQuery == null)
{
exhausted = true;
return EmptyIterators.partition();
}
- return Transformation.apply(readCommand.execute(consistency, clientState, queryStartNanoTime), pager);
+ return Transformation.apply(readQuery.execute(consistency, clientState, queryStartNanoTime), pager);
}
public PartitionIterator fetchPageInternal(int pageSize, ReadExecutionController executionController)
@@ -81,30 +81,30 @@
return EmptyIterators.partition();
pageSize = Math.min(pageSize, remaining);
- RowPager pager = new RowPager(limits.forPaging(pageSize), command.nowInSec());
- ReadCommand readCommand = nextPageReadCommand(pageSize);
- if (readCommand == null)
+ RowPager pager = new RowPager(limits.forPaging(pageSize), query.nowInSec());
+ ReadQuery readQuery = nextPageReadQuery(pageSize);
+ if (readQuery == null)
{
exhausted = true;
return EmptyIterators.partition();
}
- return Transformation.apply(readCommand.executeInternal(executionController), pager);
+ return Transformation.apply(readQuery.executeInternal(executionController), pager);
}
- public UnfilteredPartitionIterator fetchPageUnfiltered(CFMetaData cfm, int pageSize, ReadExecutionController executionController)
+ public UnfilteredPartitionIterator fetchPageUnfiltered(TableMetadata metadata, int pageSize, ReadExecutionController executionController)
{
if (isExhausted())
- return EmptyIterators.unfilteredPartition(cfm, false);
+ return EmptyIterators.unfilteredPartition(metadata);
pageSize = Math.min(pageSize, remaining);
- UnfilteredPager pager = new UnfilteredPager(limits.forPaging(pageSize), command.nowInSec());
- ReadCommand readCommand = nextPageReadCommand(pageSize);
- if (readCommand == null)
+ UnfilteredPager pager = new UnfilteredPager(limits.forPaging(pageSize), query.nowInSec());
+ ReadQuery readQuery = nextPageReadQuery(pageSize);
+ if (readQuery == null)
{
exhausted = true;
- return EmptyIterators.unfilteredPartition(cfm, false);
+ return EmptyIterators.unfilteredPartition(metadata);
}
- return Transformation.apply(readCommand.executeLocally(executionController), pager);
+ return Transformation.apply(readQuery.executeLocally(executionController), pager);
}
private class UnfilteredPager extends Pager<Unfiltered>
@@ -145,7 +145,7 @@
private Pager(DataLimits pageLimits, int nowInSec)
{
- this.counter = pageLimits.newCounter(nowInSec, true, command.selectsFullPartition(), enforceStrictLiveness);
+ this.counter = pageLimits.newCounter(nowInSec, true, query.selectsFullPartition(), enforceStrictLiveness);
this.pageLimits = pageLimits;
}
@@ -246,7 +246,7 @@
return remainingInPartition;
}
- protected abstract ReadCommand nextPageReadCommand(int pageSize);
+ protected abstract T nextPageReadQuery(int pageSize);
protected abstract void recordLast(DecoratedKey key, Row row);
protected abstract boolean isPreviouslyReturnedPartition(DecoratedKey key);
}
diff --git a/src/java/org/apache/cassandra/service/pager/AggregationQueryPager.java b/src/java/org/apache/cassandra/service/pager/AggregationQueryPager.java
index f9a8cda..5ac01b2 100644
--- a/src/java/org/apache/cassandra/service/pager/AggregationQueryPager.java
+++ b/src/java/org/apache/cassandra/service/pager/AggregationQueryPager.java
@@ -20,7 +20,7 @@
import java.nio.ByteBuffer;
import java.util.NoSuchElementException;
-import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.aggregation.GroupingState;
import org.apache.cassandra.db.filter.DataLimits;
@@ -319,7 +319,7 @@
this.rowIterator = delegate;
}
- public CFMetaData metadata()
+ public TableMetadata metadata()
{
return rowIterator.metadata();
}
@@ -329,7 +329,7 @@
return rowIterator.isReverseOrder();
}
- public PartitionColumns columns()
+ public RegularAndStaticColumns columns()
{
return rowIterator.columns();
}
diff --git a/src/java/org/apache/cassandra/service/pager/MultiPartitionPager.java b/src/java/org/apache/cassandra/service/pager/MultiPartitionPager.java
index 9dae11c..ca16967 100644
--- a/src/java/org/apache/cassandra/service/pager/MultiPartitionPager.java
+++ b/src/java/org/apache/cassandra/service/pager/MultiPartitionPager.java
@@ -31,20 +31,20 @@
import org.apache.cassandra.service.ClientState;
/**
- * Pager over a list of ReadCommand.
+ * Pager over a list of SinglePartitionReadQuery.
*
- * Note that this is not easy to make efficient. Indeed, we need to page the first command fully before
- * returning results from the next one, but if the result returned by each command is small (compared to pageSize),
- * paging the commands one at a time under-performs compared to parallelizing. On the other, if we parallelize
- * and each command raised pageSize results, we'll end up with commands.size() * pageSize results in memory, which
+ * Note that this is not easy to make efficient. Indeed, we need to page the first query fully before
+ * returning results from the next one, but if the result returned by each query is small (compared to pageSize),
+ * paging the queries one at a time under-performs compared to parallelizing. On the other, if we parallelize
+ * and each query raised pageSize results, we'll end up with queries.size() * pageSize results in memory, which
* defeats the purpose of paging.
*
- * For now, we keep it simple (somewhat) and just do one command at a time. Provided that we make sure to not
+ * For now, we keep it simple (somewhat) and just do one query at a time. Provided that we make sure to not
* create a pager unless we need to, this is probably fine. Though if we later want to get fancy, we could use the
- * cfs meanPartitionSize to decide if parallelizing some of the command might be worth it while being confident we don't
+ * cfs meanPartitionSize to decide if parallelizing some of the query might be worth it while being confident we don't
* blow out memory.
*/
-public class MultiPartitionPager implements QueryPager
+public class MultiPartitionPager<T extends SinglePartitionReadQuery> implements QueryPager
{
private final SinglePartitionPager[] pagers;
private final DataLimits limit;
@@ -54,33 +54,33 @@
private int remaining;
private int current;
- public MultiPartitionPager(SinglePartitionReadCommand.Group group, PagingState state, ProtocolVersion protocolVersion)
+ public MultiPartitionPager(SinglePartitionReadQuery.Group<T> group, PagingState state, ProtocolVersion protocolVersion)
{
this.limit = group.limits();
this.nowInSec = group.nowInSec();
int i = 0;
- // If it's not the beginning (state != null), we need to find where we were and skip previous commands
+ // If it's not the beginning (state != null), we need to find where we were and skip previous queries
// since they are done.
if (state != null)
- for (; i < group.commands.size(); i++)
- if (group.commands.get(i).partitionKey().getKey().equals(state.partitionKey))
+ for (; i < group.queries.size(); i++)
+ if (group.queries.get(i).partitionKey().getKey().equals(state.partitionKey))
break;
- if (i >= group.commands.size())
+ if (i >= group.queries.size())
{
pagers = null;
return;
}
- pagers = new SinglePartitionPager[group.commands.size() - i];
+ pagers = new SinglePartitionPager[group.queries.size() - i];
// 'i' is on the first non exhausted pager for the previous page (or the first one)
- SinglePartitionReadCommand command = group.commands.get(i);
- pagers[0] = command.getPager(state, protocolVersion);
+ T query = group.queries.get(i);
+ pagers[0] = query.getPager(state, protocolVersion);
// Following ones haven't been started yet
- for (int j = i + 1; j < group.commands.size(); j++)
- pagers[j - i] = group.commands.get(j).getPager(null, protocolVersion);
+ for (int j = i + 1; j < group.queries.size(); j++)
+ pagers[j - i] = group.queries.get(j).getPager(null, protocolVersion);
remaining = state == null ? limit.count() : state.remaining;
}
@@ -103,11 +103,11 @@
SinglePartitionPager[] newPagers = Arrays.copyOf(pagers, pagers.length);
newPagers[current] = newPagers[current].withUpdatedLimit(newLimits);
- return new MultiPartitionPager(newPagers,
- newLimits,
- nowInSec,
- remaining,
- current);
+ return new MultiPartitionPager<T>(newPagers,
+ newLimits,
+ nowInSec,
+ remaining,
+ current);
}
public PagingState state()
diff --git a/src/java/org/apache/cassandra/service/pager/PagingState.java b/src/java/org/apache/cassandra/service/pager/PagingState.java
index 9b7eccf..8df2366 100644
--- a/src/java/org/apache/cassandra/service/pager/PagingState.java
+++ b/src/java/org/apache/cassandra/service/pager/PagingState.java
@@ -24,17 +24,18 @@
import com.google.common.annotations.VisibleForTesting;
import com.google.common.primitives.Ints;
-import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.db.Clustering;
-import org.apache.cassandra.db.LegacyLayout;
+import org.apache.cassandra.db.CompactTables;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.db.marshal.BytesType;
+import org.apache.cassandra.db.marshal.CompositeType;
import org.apache.cassandra.db.rows.Cell;
import org.apache.cassandra.db.rows.Row;
import org.apache.cassandra.io.util.DataInputBuffer;
import org.apache.cassandra.io.util.DataOutputBuffer;
import org.apache.cassandra.io.util.DataOutputBufferFixed;
import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.transport.ProtocolException;
import org.apache.cassandra.transport.ProtocolVersion;
@@ -332,7 +333,7 @@
this.protocolVersion = protocolVersion;
}
- private static List<AbstractType<?>> makeClusteringTypes(CFMetaData metadata)
+ private static List<AbstractType<?>> makeClusteringTypes(TableMetadata metadata)
{
// This is the types that will be used when serializing the clustering in the paging state. We can't really use the actual clustering
// types however because we can't guarantee that there won't be a schema change between when we send the paging state and get it back,
@@ -346,7 +347,7 @@
return l;
}
- public static RowMark create(CFMetaData metadata, Row row, ProtocolVersion protocolVersion)
+ public static RowMark create(TableMetadata metadata, Row row, ProtocolVersion protocolVersion)
{
ByteBuffer mark;
if (protocolVersion.isSmallerOrEqualTo(ProtocolVersion.V3))
@@ -360,12 +361,12 @@
// If the last returned row has no cell, this means in 2.1/2.2 terms that we stopped on the row
// marker. Note that this shouldn't happen if the table is COMPACT.
assert !metadata.isCompactTable();
- mark = LegacyLayout.encodeCellName(metadata, row.clustering(), EMPTY_BYTE_BUFFER, null);
+ mark = encodeCellName(metadata, row.clustering(), EMPTY_BYTE_BUFFER, null);
}
else
{
Cell cell = cells.next();
- mark = LegacyLayout.encodeCellName(metadata, row.clustering(), cell.column().name.bytes, cell.column().isComplex() ? cell.path().get(0) : null);
+ mark = encodeCellName(metadata, row.clustering(), cell.column().name.bytes, cell.column().isComplex() ? cell.path().get(0) : null);
}
}
else
@@ -377,16 +378,90 @@
return new RowMark(mark, protocolVersion);
}
- public Clustering clustering(CFMetaData metadata)
+ public Clustering clustering(TableMetadata metadata)
{
if (mark == null)
return null;
return protocolVersion.isSmallerOrEqualTo(ProtocolVersion.V3)
- ? LegacyLayout.decodeClustering(metadata, mark)
+ ? decodeClustering(metadata, mark)
: Clustering.serializer.deserialize(mark, MessagingService.VERSION_30, makeClusteringTypes(metadata));
}
+ // Old (pre-3.0) encoding of cells. We need that for the protocol v3 as that is how things where encoded
+ private static ByteBuffer encodeCellName(TableMetadata metadata, Clustering clustering, ByteBuffer columnName, ByteBuffer collectionElement)
+ {
+ boolean isStatic = clustering == Clustering.STATIC_CLUSTERING;
+
+ if (!metadata.isCompound())
+ {
+ if (isStatic)
+ return columnName;
+
+ assert clustering.size() == 1 : "Expected clustering size to be 1, but was " + clustering.size();
+ return clustering.get(0);
+ }
+
+ // We use comparator.size() rather than clustering.size() because of static clusterings
+ int clusteringSize = metadata.comparator.size();
+ int size = clusteringSize + (metadata.isDense() ? 0 : 1) + (collectionElement == null ? 0 : 1);
+ if (metadata.isSuper())
+ size = clusteringSize + 1;
+ ByteBuffer[] values = new ByteBuffer[size];
+ for (int i = 0; i < clusteringSize; i++)
+ {
+ if (isStatic)
+ {
+ values[i] = EMPTY_BYTE_BUFFER;
+ continue;
+ }
+
+ ByteBuffer v = clustering.get(i);
+ // we can have null (only for dense compound tables for backward compatibility reasons) but that
+ // means we're done and should stop there as far as building the composite is concerned.
+ if (v == null)
+ return CompositeType.build(Arrays.copyOfRange(values, 0, i));
+
+ values[i] = v;
+ }
+
+ if (metadata.isSuper())
+ {
+ // We need to set the "column" (in thrift terms) name, i.e. the value corresponding to the subcomparator.
+ // What it is depends if this a cell for a declared "static" column or a "dynamic" column part of the
+ // super-column internal map.
+ assert columnName != null; // This should never be null for supercolumns, see decodeForSuperColumn() above
+ values[clusteringSize] = columnName.equals(CompactTables.SUPER_COLUMN_MAP_COLUMN)
+ ? collectionElement
+ : columnName;
+ }
+ else
+ {
+ if (!metadata.isDense())
+ values[clusteringSize] = columnName;
+ if (collectionElement != null)
+ values[clusteringSize + 1] = collectionElement;
+ }
+
+ return CompositeType.build(isStatic, values);
+ }
+
+ private static Clustering decodeClustering(TableMetadata metadata, ByteBuffer value)
+ {
+ int csize = metadata.comparator.size();
+ if (csize == 0)
+ return Clustering.EMPTY;
+
+ if (metadata.isCompound() && CompositeType.isStaticName(value))
+ return Clustering.STATIC_CLUSTERING;
+
+ List<ByteBuffer> components = metadata.isCompound()
+ ? CompositeType.splitName(value)
+ : Collections.singletonList(value);
+
+ return Clustering.make(components.subList(0, Math.min(csize, components.size())).toArray(new ByteBuffer[csize]));
+ }
+
@Override
public final int hashCode()
{
diff --git a/src/java/org/apache/cassandra/service/pager/PartitionRangeQueryPager.java b/src/java/org/apache/cassandra/service/pager/PartitionRangeQueryPager.java
index 75f76cb..4f1e0e7 100644
--- a/src/java/org/apache/cassandra/service/pager/PartitionRangeQueryPager.java
+++ b/src/java/org/apache/cassandra/service/pager/PartitionRangeQueryPager.java
@@ -21,40 +21,36 @@
import org.apache.cassandra.db.filter.DataLimits;
import org.apache.cassandra.db.rows.Row;
import org.apache.cassandra.dht.*;
-import org.apache.cassandra.exceptions.RequestExecutionException;
import org.apache.cassandra.transport.ProtocolVersion;
/**
- * Pages a PartitionRangeReadCommand.
- *
- * Note: this only work for CQL3 queries for now (because thrift queries expect
- * a different limit on the rows than on the columns, which complicates it).
+ * Pages a PartitionRangeReadQuery.
*/
-public class PartitionRangeQueryPager extends AbstractQueryPager
+public class PartitionRangeQueryPager extends AbstractQueryPager<PartitionRangeReadQuery>
{
private volatile DecoratedKey lastReturnedKey;
private volatile PagingState.RowMark lastReturnedRow;
- public PartitionRangeQueryPager(PartitionRangeReadCommand command, PagingState state, ProtocolVersion protocolVersion)
+ public PartitionRangeQueryPager(PartitionRangeReadQuery query, PagingState state, ProtocolVersion protocolVersion)
{
- super(command, protocolVersion);
+ super(query, protocolVersion);
if (state != null)
{
- lastReturnedKey = command.metadata().decorateKey(state.partitionKey);
+ lastReturnedKey = query.metadata().partitioner.decorateKey(state.partitionKey);
lastReturnedRow = state.rowMark;
restoreState(lastReturnedKey, state.remaining, state.remainingInPartition);
}
}
- public PartitionRangeQueryPager(ReadCommand command,
+ public PartitionRangeQueryPager(PartitionRangeReadQuery query,
ProtocolVersion protocolVersion,
DecoratedKey lastReturnedKey,
PagingState.RowMark lastReturnedRow,
int remaining,
int remainingInPartition)
{
- super(command, protocolVersion);
+ super(query, protocolVersion);
this.lastReturnedKey = lastReturnedKey;
this.lastReturnedRow = lastReturnedRow;
restoreState(lastReturnedKey, remaining, remainingInPartition);
@@ -62,7 +58,7 @@
public PartitionRangeQueryPager withUpdatedLimit(DataLimits newLimits)
{
- return new PartitionRangeQueryPager(command.withUpdatedLimit(newLimits),
+ return new PartitionRangeQueryPager(query.withUpdatedLimit(newLimits),
protocolVersion,
lastReturnedKey,
lastReturnedRow,
@@ -77,16 +73,16 @@
: new PagingState(lastReturnedKey.getKey(), lastReturnedRow, maxRemaining(), remainingInPartition());
}
- protected ReadCommand nextPageReadCommand(int pageSize)
- throws RequestExecutionException
+ @Override
+ protected PartitionRangeReadQuery nextPageReadQuery(int pageSize)
{
DataLimits limits;
- DataRange fullRange = ((PartitionRangeReadCommand)command).dataRange();
+ DataRange fullRange = query.dataRange();
DataRange pageRange;
if (lastReturnedKey == null)
{
pageRange = fullRange;
- limits = command.limits().forPaging(pageSize);
+ limits = query.limits().forPaging(pageSize);
}
// if the last key was the one of the end of the range we know that we are done
else if (lastReturnedKey.equals(fullRange.keyRange().right) && remainingInPartition() == 0 && lastReturnedRow == null)
@@ -96,24 +92,21 @@
else
{
// We want to include the last returned key only if we haven't achieved our per-partition limit, otherwise, don't bother.
- // note that the distinct check should only be hit when getting queries in a mixed mode cluster where a 2.1/2.2-serialized
- // PagingState is sent to a 3.0 node - in that case we get remainingInPartition = Integer.MAX_VALUE and we include
- // duplicate keys. For standard non-mixed operation remainingInPartition will always be 0 for DISTINCT queries.
- boolean includeLastKey = remainingInPartition() > 0 && lastReturnedRow != null && !command.limits().isDistinct();
+ boolean includeLastKey = remainingInPartition() > 0 && lastReturnedRow != null;
AbstractBounds<PartitionPosition> bounds = makeKeyBounds(lastReturnedKey, includeLastKey);
if (includeLastKey)
{
- pageRange = fullRange.forPaging(bounds, command.metadata().comparator, lastReturnedRow.clustering(command.metadata()), false);
- limits = command.limits().forPaging(pageSize, lastReturnedKey.getKey(), remainingInPartition());
+ pageRange = fullRange.forPaging(bounds, query.metadata().comparator, lastReturnedRow.clustering(query.metadata()), false);
+ limits = query.limits().forPaging(pageSize, lastReturnedKey.getKey(), remainingInPartition());
}
else
{
pageRange = fullRange.forSubRange(bounds);
- limits = command.limits().forPaging(pageSize);
+ limits = query.limits().forPaging(pageSize);
}
}
- return ((PartitionRangeReadCommand) command).withUpdatedLimitsAndDataRange(limits, pageRange);
+ return query.withUpdatedLimitsAndDataRange(limits, pageRange);
}
protected void recordLast(DecoratedKey key, Row last)
@@ -122,7 +115,7 @@
{
lastReturnedKey = key;
if (last.clustering() != Clustering.STATIC_CLUSTERING)
- lastReturnedRow = PagingState.RowMark.create(command.metadata(), last, protocolVersion);
+ lastReturnedRow = PagingState.RowMark.create(query.metadata(), last, protocolVersion);
}
}
@@ -134,18 +127,16 @@
private AbstractBounds<PartitionPosition> makeKeyBounds(PartitionPosition lastReturnedKey, boolean includeLastKey)
{
- AbstractBounds<PartitionPosition> bounds = ((PartitionRangeReadCommand)command).dataRange().keyRange();
+ AbstractBounds<PartitionPosition> bounds = query.dataRange().keyRange();
if (bounds instanceof Range || bounds instanceof Bounds)
{
return includeLastKey
- ? new Bounds<PartitionPosition>(lastReturnedKey, bounds.right)
- : new Range<PartitionPosition>(lastReturnedKey, bounds.right);
+ ? new Bounds<>(lastReturnedKey, bounds.right)
+ : new Range<>(lastReturnedKey, bounds.right);
}
- else
- {
- return includeLastKey
- ? new IncludingExcludingBounds<PartitionPosition>(lastReturnedKey, bounds.right)
- : new ExcludingBounds<PartitionPosition>(lastReturnedKey, bounds.right);
- }
+
+ return includeLastKey
+ ? new IncludingExcludingBounds<>(lastReturnedKey, bounds.right)
+ : new ExcludingBounds<>(lastReturnedKey, bounds.right);
}
}
diff --git a/src/java/org/apache/cassandra/service/pager/QueryPagers.java b/src/java/org/apache/cassandra/service/pager/QueryPagers.java
deleted file mode 100644
index 1a70864..0000000
--- a/src/java/org/apache/cassandra/service/pager/QueryPagers.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.service.pager;
-
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.db.*;
-import org.apache.cassandra.db.filter.*;
-import org.apache.cassandra.db.partitions.*;
-import org.apache.cassandra.exceptions.RequestExecutionException;
-import org.apache.cassandra.exceptions.RequestValidationException;
-import org.apache.cassandra.service.ClientState;
-import org.apache.cassandra.transport.ProtocolVersion;
-
-/**
- * Static utility methods for paging.
- */
-public class QueryPagers
-{
- private QueryPagers() {};
-
- /**
- * Convenience method that count (live) cells/rows for a given slice of a row, but page underneath.
- */
- public static int countPaged(CFMetaData metadata,
- DecoratedKey key,
- ColumnFilter columnFilter,
- ClusteringIndexFilter filter,
- DataLimits limits,
- ConsistencyLevel consistencyLevel,
- ClientState state,
- final int pageSize,
- int nowInSec,
- boolean isForThrift,
- long queryStartNanoTime) throws RequestValidationException, RequestExecutionException
- {
- SinglePartitionReadCommand command = SinglePartitionReadCommand.create(isForThrift, metadata, nowInSec, columnFilter, RowFilter.NONE, limits, key, filter);
- final SinglePartitionPager pager = new SinglePartitionPager(command, null, ProtocolVersion.CURRENT);
-
- int count = 0;
- while (!pager.isExhausted())
- {
- try (PartitionIterator iter = pager.fetchPage(pageSize, consistencyLevel, state, queryStartNanoTime))
- {
- DataLimits.Counter counter = limits.newCounter(nowInSec, true, command.selectsFullPartition(), metadata.enforceStrictLiveness());
- PartitionIterators.consume(counter.applyTo(iter));
- count += counter.counted();
- }
- }
- return count;
- }
-}
diff --git a/src/java/org/apache/cassandra/service/pager/SinglePartitionPager.java b/src/java/org/apache/cassandra/service/pager/SinglePartitionPager.java
index e400fb6..93a0265 100644
--- a/src/java/org/apache/cassandra/service/pager/SinglePartitionPager.java
+++ b/src/java/org/apache/cassandra/service/pager/SinglePartitionPager.java
@@ -29,40 +29,36 @@
*
* For use by MultiPartitionPager.
*/
-public class SinglePartitionPager extends AbstractQueryPager
+public class SinglePartitionPager extends AbstractQueryPager<SinglePartitionReadQuery>
{
- private final SinglePartitionReadCommand command;
-
private volatile PagingState.RowMark lastReturned;
- public SinglePartitionPager(SinglePartitionReadCommand command, PagingState state, ProtocolVersion protocolVersion)
+ public SinglePartitionPager(SinglePartitionReadQuery query, PagingState state, ProtocolVersion protocolVersion)
{
- super(command, protocolVersion);
- this.command = command;
+ super(query, protocolVersion);
if (state != null)
{
lastReturned = state.rowMark;
- restoreState(command.partitionKey(), state.remaining, state.remainingInPartition);
+ restoreState(query.partitionKey(), state.remaining, state.remainingInPartition);
}
}
- private SinglePartitionPager(SinglePartitionReadCommand command,
+ private SinglePartitionPager(SinglePartitionReadQuery query,
ProtocolVersion protocolVersion,
PagingState.RowMark rowMark,
int remaining,
int remainingInPartition)
{
- super(command, protocolVersion);
- this.command = command;
+ super(query, protocolVersion);
this.lastReturned = rowMark;
- restoreState(command.partitionKey(), remaining, remainingInPartition);
+ restoreState(query.partitionKey(), remaining, remainingInPartition);
}
@Override
public SinglePartitionPager withUpdatedLimit(DataLimits newLimits)
{
- return new SinglePartitionPager(command.withUpdatedLimit(newLimits),
+ return new SinglePartitionPager(query.withUpdatedLimit(newLimits),
protocolVersion,
lastReturned,
maxRemaining(),
@@ -71,12 +67,12 @@
public ByteBuffer key()
{
- return command.partitionKey().getKey();
+ return query.partitionKey().getKey();
}
public DataLimits limits()
{
- return command.limits();
+ return query.limits();
}
public PagingState state()
@@ -86,19 +82,21 @@
: new PagingState(null, lastReturned, maxRemaining(), remainingInPartition());
}
- protected ReadCommand nextPageReadCommand(int pageSize)
+ @Override
+ protected SinglePartitionReadQuery nextPageReadQuery(int pageSize)
{
- Clustering clustering = lastReturned == null ? null : lastReturned.clustering(command.metadata());
- DataLimits limits = (lastReturned == null || command.isForThrift()) ? limits().forPaging(pageSize)
- : limits().forPaging(pageSize, key(), remainingInPartition());
+ Clustering clustering = lastReturned == null ? null : lastReturned.clustering(query.metadata());
+ DataLimits limits = lastReturned == null
+ ? limits().forPaging(pageSize)
+ : limits().forPaging(pageSize, key(), remainingInPartition());
- return command.forPaging(clustering, limits);
+ return query.forPaging(clustering, limits);
}
protected void recordLast(DecoratedKey key, Row last)
{
if (last != null && last.clustering() != Clustering.STATIC_CLUSTERING)
- lastReturned = PagingState.RowMark.create(command.metadata(), last, protocolVersion);
+ lastReturned = PagingState.RowMark.create(query.metadata(), last, protocolVersion);
}
protected boolean isPreviouslyReturnedPartition(DecoratedKey key)
diff --git a/src/java/org/apache/cassandra/service/paxos/AbstractPaxosCallback.java b/src/java/org/apache/cassandra/service/paxos/AbstractPaxosCallback.java
index 90bfc5d..ab24f50 100644
--- a/src/java/org/apache/cassandra/service/paxos/AbstractPaxosCallback.java
+++ b/src/java/org/apache/cassandra/service/paxos/AbstractPaxosCallback.java
@@ -1,4 +1,3 @@
-package org.apache.cassandra.service.paxos;
/*
*
* Licensed to the Apache Software Foundation (ASF) under one
@@ -19,18 +18,19 @@
* under the License.
*
*/
-
+package org.apache.cassandra.service.paxos;
import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.TimeUnit;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.ConsistencyLevel;
import org.apache.cassandra.db.WriteType;
import org.apache.cassandra.exceptions.WriteTimeoutException;
-import org.apache.cassandra.net.IAsyncCallback;
+import org.apache.cassandra.net.RequestCallback;
-public abstract class AbstractPaxosCallback<T> implements IAsyncCallback<T>
+import static java.util.concurrent.TimeUnit.NANOSECONDS;
+
+public abstract class AbstractPaxosCallback<T> implements RequestCallback<T>
{
protected final CountDownLatch latch;
protected final int targets;
@@ -45,11 +45,6 @@
this.queryStartNanoTime = queryStartNanoTime;
}
- public boolean isLatencyForSnitch()
- {
- return false;
- }
-
public int getResponseCount()
{
return (int) (targets - latch.getCount());
@@ -59,8 +54,8 @@
{
try
{
- long timeout = TimeUnit.MILLISECONDS.toNanos(DatabaseDescriptor.getWriteRpcTimeout()) - (System.nanoTime() - queryStartNanoTime);
- if (!latch.await(timeout, TimeUnit.NANOSECONDS))
+ long timeout = DatabaseDescriptor.getWriteRpcTimeout(NANOSECONDS) - (System.nanoTime() - queryStartNanoTime);
+ if (!latch.await(timeout, NANOSECONDS))
throw new WriteTimeoutException(WriteType.CAS, consistency, getResponseCount(), targets);
}
catch (InterruptedException ex)
diff --git a/src/java/org/apache/cassandra/service/paxos/Commit.java b/src/java/org/apache/cassandra/service/paxos/Commit.java
index af94869..ed0eb9b 100644
--- a/src/java/org/apache/cassandra/service/paxos/Commit.java
+++ b/src/java/org/apache/cassandra/service/paxos/Commit.java
@@ -22,20 +22,17 @@
import java.io.IOException;
-import java.nio.ByteBuffer;
import java.util.UUID;
import com.google.common.base.Objects;
-import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.rows.*;
import org.apache.cassandra.db.partitions.PartitionUpdate;
import org.apache.cassandra.io.IVersionedSerializer;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
-import org.apache.cassandra.net.MessagingService;
-import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.UUIDGen;
import org.apache.cassandra.utils.UUIDSerializer;
@@ -55,18 +52,18 @@
this.update = update;
}
- public static Commit newPrepare(DecoratedKey key, CFMetaData metadata, UUID ballot)
+ public static Commit newPrepare(DecoratedKey key, TableMetadata metadata, UUID ballot)
{
return new Commit(ballot, PartitionUpdate.emptyUpdate(metadata, key));
}
public static Commit newProposal(UUID ballot, PartitionUpdate update)
{
- update.updateAllTimestamp(UUIDGen.microsTimestamp(ballot));
- return new Commit(ballot, update);
+ PartitionUpdate withNewTimestamp = new PartitionUpdate.Builder(update, 0).updateAllTimestamp(UUIDGen.microsTimestamp(ballot)).build();
+ return new Commit(ballot, withNewTimestamp);
}
- public static Commit emptyCommit(DecoratedKey key, CFMetaData metadata)
+ public static Commit emptyCommit(DecoratedKey key, TableMetadata metadata)
{
return new Commit(UUIDGen.minTimeUUID(0), PartitionUpdate.emptyUpdate(metadata, key));
}
@@ -113,32 +110,20 @@
{
public void serialize(Commit commit, DataOutputPlus out, int version) throws IOException
{
- if (version < MessagingService.VERSION_30)
- ByteBufferUtil.writeWithShortLength(commit.update.partitionKey().getKey(), out);
-
UUIDSerializer.serializer.serialize(commit.ballot, out, version);
PartitionUpdate.serializer.serialize(commit.update, out, version);
}
public Commit deserialize(DataInputPlus in, int version) throws IOException
{
- ByteBuffer key = null;
- if (version < MessagingService.VERSION_30)
- key = ByteBufferUtil.readWithShortLength(in);
-
UUID ballot = UUIDSerializer.serializer.deserialize(in, version);
- PartitionUpdate update = PartitionUpdate.serializer.deserialize(in, version, SerializationHelper.Flag.LOCAL, key);
+ PartitionUpdate update = PartitionUpdate.serializer.deserialize(in, version, DeserializationHelper.Flag.LOCAL);
return new Commit(ballot, update);
}
public long serializedSize(Commit commit, int version)
{
- int size = 0;
- if (version < MessagingService.VERSION_30)
- size += ByteBufferUtil.serializedSizeWithShortLength(commit.update.partitionKey().getKey());
-
- return size
- + UUIDSerializer.serializer.serializedSize(commit.ballot, version)
+ return UUIDSerializer.serializer.serializedSize(commit.ballot, version)
+ PartitionUpdate.serializer.serializedSize(commit.update, version);
}
}
diff --git a/src/java/org/apache/cassandra/service/paxos/CommitVerbHandler.java b/src/java/org/apache/cassandra/service/paxos/CommitVerbHandler.java
index a702a4d..12466dd 100644
--- a/src/java/org/apache/cassandra/service/paxos/CommitVerbHandler.java
+++ b/src/java/org/apache/cassandra/service/paxos/CommitVerbHandler.java
@@ -20,19 +20,20 @@
*/
package org.apache.cassandra.service.paxos;
-import org.apache.cassandra.db.WriteResponse;
import org.apache.cassandra.net.IVerbHandler;
-import org.apache.cassandra.net.MessageIn;
+import org.apache.cassandra.net.Message;
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.tracing.Tracing;
public class CommitVerbHandler implements IVerbHandler<Commit>
{
- public void doVerb(MessageIn<Commit> message, int id)
+ public static final CommitVerbHandler instance = new CommitVerbHandler();
+
+ public void doVerb(Message<Commit> message)
{
PaxosState.commit(message.payload);
- Tracing.trace("Enqueuing acknowledge to {}", message.from);
- MessagingService.instance().sendReply(WriteResponse.createMessage(), id, message.from);
+ Tracing.trace("Enqueuing acknowledge to {}", message.from());
+ MessagingService.instance().send(message.emptyResponse(), message.from());
}
}
diff --git a/src/java/org/apache/cassandra/service/paxos/PaxosState.java b/src/java/org/apache/cassandra/service/paxos/PaxosState.java
index ee1ba6a..6e02435 100644
--- a/src/java/org/apache/cassandra/service/paxos/PaxosState.java
+++ b/src/java/org/apache/cassandra/service/paxos/PaxosState.java
@@ -1,5 +1,5 @@
/*
- *
+ *
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -7,27 +7,24 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
- *
+ *
*/
package org.apache.cassandra.service.paxos;
-import java.util.concurrent.ExecutionException;
import java.util.concurrent.locks.Lock;
-import com.google.common.base.Throwables;
import com.google.common.util.concurrent.Striped;
-import com.google.common.util.concurrent.Uninterruptibles;
-import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.*;
import org.apache.cassandra.tracing.Tracing;
@@ -41,7 +38,7 @@
private final Commit accepted;
private final Commit mostRecentCommit;
- public PaxosState(DecoratedKey key, CFMetaData metadata)
+ public PaxosState(DecoratedKey key, TableMetadata metadata)
{
this(Commit.emptyCommit(key, metadata), Commit.emptyCommit(key, metadata), Commit.emptyCommit(key, metadata));
}
@@ -49,7 +46,7 @@
public PaxosState(Commit promised, Commit accepted, Commit mostRecentCommit)
{
assert promised.update.partitionKey().equals(accepted.update.partitionKey()) && accepted.update.partitionKey().equals(mostRecentCommit.update.partitionKey());
- assert promised.update.metadata() == accepted.update.metadata() && accepted.update.metadata() == mostRecentCommit.update.metadata();
+ assert promised.update.metadata().id.equals(accepted.update.metadata().id) && accepted.update.metadata().id.equals(mostRecentCommit.update.metadata().id);
this.promised = promised;
this.accepted = accepted;
@@ -92,7 +89,7 @@
}
finally
{
- Keyspace.open(toPrepare.update.metadata().ksName).getColumnFamilyStore(toPrepare.update.metadata().cfId).metric.casPrepare.addNano(System.nanoTime() - start);
+ Keyspace.open(toPrepare.update.metadata().keyspace).getColumnFamilyStore(toPrepare.update.metadata().id).metric.casPrepare.addNano(System.nanoTime() - start);
}
}
@@ -127,7 +124,7 @@
}
finally
{
- Keyspace.open(proposal.update.metadata().ksName).getColumnFamilyStore(proposal.update.metadata().cfId).metric.casPropose.addNano(System.nanoTime() - start);
+ Keyspace.open(proposal.update.metadata().keyspace).getColumnFamilyStore(proposal.update.metadata().id).metric.casPropose.addNano(System.nanoTime() - start);
}
}
@@ -143,7 +140,7 @@
// erase the in-progress update.
// The table may have been truncated since the proposal was initiated. In that case, we
// don't want to perform the mutation and potentially resurrect truncated data
- if (UUIDGen.unixTimestamp(proposal.ballot) >= SystemKeyspace.getTruncatedAt(proposal.update.metadata().cfId))
+ if (UUIDGen.unixTimestamp(proposal.ballot) >= SystemKeyspace.getTruncatedAt(proposal.update.metadata().id))
{
Tracing.trace("Committing proposal {}", proposal);
Mutation mutation = proposal.makeMutation();
@@ -158,7 +155,7 @@
}
finally
{
- Keyspace.open(proposal.update.metadata().ksName).getColumnFamilyStore(proposal.update.metadata().cfId).metric.casCommit.addNano(System.nanoTime() - start);
+ Keyspace.open(proposal.update.metadata().keyspace).getColumnFamilyStore(proposal.update.metadata().id).metric.casCommit.addNano(System.nanoTime() - start);
}
}
}
diff --git a/src/java/org/apache/cassandra/service/paxos/PrepareCallback.java b/src/java/org/apache/cassandra/service/paxos/PrepareCallback.java
index 5915eab..26890a9 100644
--- a/src/java/org/apache/cassandra/service/paxos/PrepareCallback.java
+++ b/src/java/org/apache/cassandra/service/paxos/PrepareCallback.java
@@ -21,21 +21,22 @@
*/
-import java.net.InetAddress;
import java.util.Collections;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import com.google.common.base.Predicate;
import com.google.common.collect.Iterables;
+
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.db.ConsistencyLevel;
import org.apache.cassandra.db.DecoratedKey;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.db.SystemKeyspace;
-import org.apache.cassandra.net.MessageIn;
+import org.apache.cassandra.net.Message;
import org.apache.cassandra.utils.UUIDGen;
public class PrepareCallback extends AbstractPaxosCallback<PrepareResponse>
@@ -47,21 +48,21 @@
public Commit mostRecentInProgressCommit;
public Commit mostRecentInProgressCommitWithUpdate;
- private final Map<InetAddress, Commit> commitsByReplica = new ConcurrentHashMap<InetAddress, Commit>();
+ private final Map<InetAddressAndPort, Commit> commitsByReplica = new ConcurrentHashMap<>();
- public PrepareCallback(DecoratedKey key, CFMetaData metadata, int targets, ConsistencyLevel consistency, long queryStartNanoTime)
+ public PrepareCallback(DecoratedKey key, TableMetadata metadata, int targets, ConsistencyLevel consistency, long queryStartNanoTime)
{
super(targets, consistency, queryStartNanoTime);
- // need to inject the right key in the empty commit so comparing with empty commits in the reply works as expected
+ // need to inject the right key in the empty commit so comparing with empty commits in the response works as expected
mostRecentCommit = Commit.emptyCommit(key, metadata);
mostRecentInProgressCommit = Commit.emptyCommit(key, metadata);
mostRecentInProgressCommitWithUpdate = Commit.emptyCommit(key, metadata);
}
- public synchronized void response(MessageIn<PrepareResponse> message)
+ public synchronized void onResponse(Message<PrepareResponse> message)
{
PrepareResponse response = message.payload;
- logger.trace("Prepare response {} from {}", response, message.from);
+ logger.trace("Prepare response {} from {}", response, message.from());
// In case of clock skew, another node could be proposing with ballot that are quite a bit
// older than our own. In that case, we record the more recent commit we've received to make
@@ -77,7 +78,7 @@
return;
}
- commitsByReplica.put(message.from, response.mostRecentCommit);
+ commitsByReplica.put(message.from(), response.mostRecentCommit);
if (response.mostRecentCommit.isAfter(mostRecentCommit))
mostRecentCommit = response.mostRecentCommit;
@@ -89,7 +90,7 @@
latch.countDown();
}
- public Iterable<InetAddress> replicasMissingMostRecentCommit(CFMetaData metadata, int nowInSec)
+ public Iterable<InetAddressAndPort> replicasMissingMostRecentCommit(TableMetadata metadata, int nowInSec)
{
// In general, we need every replicas that have answered to the prepare (a quorum) to agree on the MRC (see
// coment in StorageProxy.beginAndRepairPaxos(), but basically we need to make sure at least a quorum of nodes
@@ -104,9 +105,9 @@
if (UUIDGen.unixTimestampInSec(mostRecentCommit.ballot) + paxosTtlSec < nowInSec)
return Collections.emptySet();
- return Iterables.filter(commitsByReplica.keySet(), new Predicate<InetAddress>()
+ return Iterables.filter(commitsByReplica.keySet(), new Predicate<InetAddressAndPort>()
{
- public boolean apply(InetAddress inetAddress)
+ public boolean apply(InetAddressAndPort inetAddress)
{
return (!commitsByReplica.get(inetAddress).ballot.equals(mostRecentCommit.ballot));
}
diff --git a/src/java/org/apache/cassandra/service/paxos/PrepareResponse.java b/src/java/org/apache/cassandra/service/paxos/PrepareResponse.java
index f843b8d..4c7becc 100644
--- a/src/java/org/apache/cassandra/service/paxos/PrepareResponse.java
+++ b/src/java/org/apache/cassandra/service/paxos/PrepareResponse.java
@@ -22,16 +22,11 @@
import java.io.IOException;
-import java.util.UUID;
import org.apache.cassandra.db.TypeSizes;
-import org.apache.cassandra.db.rows.SerializationHelper;
-import org.apache.cassandra.db.partitions.PartitionUpdate;
import org.apache.cassandra.io.IVersionedSerializer;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
-import org.apache.cassandra.net.MessagingService;
-import org.apache.cassandra.utils.UUIDSerializer;
public class PrepareResponse
{
@@ -50,7 +45,7 @@
public PrepareResponse(boolean promised, Commit inProgressCommit, Commit mostRecentCommit)
{
assert inProgressCommit.update.partitionKey().equals(mostRecentCommit.update.partitionKey());
- assert inProgressCommit.update.metadata() == mostRecentCommit.update.metadata();
+ assert inProgressCommit.update.metadata().id.equals(mostRecentCommit.update.metadata().id);
this.promised = promised;
this.mostRecentCommit = mostRecentCommit;
@@ -69,51 +64,22 @@
{
out.writeBoolean(response.promised);
Commit.serializer.serialize(response.inProgressCommit, out, version);
-
- if (version < MessagingService.VERSION_30)
- {
- UUIDSerializer.serializer.serialize(response.mostRecentCommit.ballot, out, version);
- PartitionUpdate.serializer.serialize(response.mostRecentCommit.update, out, version);
- }
- else
- {
- Commit.serializer.serialize(response.mostRecentCommit, out, version);
- }
+ Commit.serializer.serialize(response.mostRecentCommit, out, version);
}
public PrepareResponse deserialize(DataInputPlus in, int version) throws IOException
{
boolean success = in.readBoolean();
Commit inProgress = Commit.serializer.deserialize(in, version);
- Commit mostRecent;
- if (version < MessagingService.VERSION_30)
- {
- UUID ballot = UUIDSerializer.serializer.deserialize(in, version);
- PartitionUpdate update = PartitionUpdate.serializer.deserialize(in, version, SerializationHelper.Flag.LOCAL, inProgress.update.partitionKey());
- mostRecent = new Commit(ballot, update);
- }
- else
- {
- mostRecent = Commit.serializer.deserialize(in, version);
- }
+ Commit mostRecent = Commit.serializer.deserialize(in, version);
return new PrepareResponse(success, inProgress, mostRecent);
}
public long serializedSize(PrepareResponse response, int version)
{
- long size = TypeSizes.sizeof(response.promised)
- + Commit.serializer.serializedSize(response.inProgressCommit, version);
-
- if (version < MessagingService.VERSION_30)
- {
- size += UUIDSerializer.serializer.serializedSize(response.mostRecentCommit.ballot, version);
- size += PartitionUpdate.serializer.serializedSize(response.mostRecentCommit.update, version);
- }
- else
- {
- size += Commit.serializer.serializedSize(response.mostRecentCommit, version);
- }
- return size;
+ return TypeSizes.sizeof(response.promised)
+ + Commit.serializer.serializedSize(response.inProgressCommit, version)
+ + Commit.serializer.serializedSize(response.mostRecentCommit, version);
}
}
}
diff --git a/src/java/org/apache/cassandra/service/paxos/PrepareVerbHandler.java b/src/java/org/apache/cassandra/service/paxos/PrepareVerbHandler.java
index 50a537f..157630f 100644
--- a/src/java/org/apache/cassandra/service/paxos/PrepareVerbHandler.java
+++ b/src/java/org/apache/cassandra/service/paxos/PrepareVerbHandler.java
@@ -19,19 +19,22 @@
* under the License.
*
*/
-
-
import org.apache.cassandra.net.IVerbHandler;
-import org.apache.cassandra.net.MessageIn;
-import org.apache.cassandra.net.MessageOut;
+import org.apache.cassandra.net.Message;
import org.apache.cassandra.net.MessagingService;
public class PrepareVerbHandler implements IVerbHandler<Commit>
{
- public void doVerb(MessageIn<Commit> message, int id)
+ public static PrepareVerbHandler instance = new PrepareVerbHandler();
+
+ public static PrepareResponse doPrepare(Commit toPrepare)
{
- PrepareResponse response = PaxosState.prepare(message.payload);
- MessageOut<PrepareResponse> reply = new MessageOut<PrepareResponse>(MessagingService.Verb.REQUEST_RESPONSE, response, PrepareResponse.serializer);
- MessagingService.instance().sendReply(reply, id, message.from);
+ return PaxosState.prepare(toPrepare);
+ }
+
+ public void doVerb(Message<Commit> message)
+ {
+ Message<PrepareResponse> reply = message.responseWith(doPrepare(message.payload));
+ MessagingService.instance().send(reply, message.from());
}
}
diff --git a/src/java/org/apache/cassandra/service/paxos/ProposeCallback.java b/src/java/org/apache/cassandra/service/paxos/ProposeCallback.java
index c9cb1f0..7e755a0 100644
--- a/src/java/org/apache/cassandra/service/paxos/ProposeCallback.java
+++ b/src/java/org/apache/cassandra/service/paxos/ProposeCallback.java
@@ -27,7 +27,7 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.cassandra.net.MessageIn;
+import org.apache.cassandra.net.Message;
/**
* ProposeCallback has two modes of operation, controlled by the failFast parameter.
@@ -35,7 +35,7 @@
* In failFast mode, we will return a failure as soon as a majority of nodes reject
* the proposal. This is used when replaying a proposal from an earlier leader.
*
- * Otherwise, we wait for either all replicas to reply or until we achieve
+ * Otherwise, we wait for either all replicas to respond or until we achieve
* the desired quorum. We continue to wait for all replicas even after we know we cannot succeed
* because we need to know if no node at all have accepted or if at least one has.
* In the former case, a proposer is guaranteed no-one will
@@ -57,9 +57,9 @@
this.failFast = failFast;
}
- public void response(MessageIn<Boolean> msg)
+ public void onResponse(Message<Boolean> msg)
{
- logger.trace("Propose response {} from {}", msg.payload, msg.from);
+ logger.trace("Propose response {} from {}", msg.payload, msg.from());
if (msg.payload)
accepts.incrementAndGet();
diff --git a/src/java/org/apache/cassandra/service/paxos/ProposeVerbHandler.java b/src/java/org/apache/cassandra/service/paxos/ProposeVerbHandler.java
index 536ff5a..5a20b67 100644
--- a/src/java/org/apache/cassandra/service/paxos/ProposeVerbHandler.java
+++ b/src/java/org/apache/cassandra/service/paxos/ProposeVerbHandler.java
@@ -19,20 +19,22 @@
* under the License.
*
*/
-
-
import org.apache.cassandra.net.IVerbHandler;
-import org.apache.cassandra.net.MessageIn;
-import org.apache.cassandra.net.MessageOut;
+import org.apache.cassandra.net.Message;
import org.apache.cassandra.net.MessagingService;
-import org.apache.cassandra.utils.BooleanSerializer;
public class ProposeVerbHandler implements IVerbHandler<Commit>
{
- public void doVerb(MessageIn<Commit> message, int id)
+ public static final ProposeVerbHandler instance = new ProposeVerbHandler();
+
+ public static Boolean doPropose(Commit proposal)
{
- Boolean response = PaxosState.propose(message.payload);
- MessageOut<Boolean> reply = new MessageOut<Boolean>(MessagingService.Verb.REQUEST_RESPONSE, response, BooleanSerializer.serializer);
- MessagingService.instance().sendReply(reply, id, message.from);
+ return PaxosState.propose(proposal);
+ }
+
+ public void doVerb(Message<Commit> message)
+ {
+ Message<Boolean> reply = message.responseWith(doPropose(message.payload));
+ MessagingService.instance().send(reply, message.from());
}
}
diff --git a/src/java/org/apache/cassandra/service/reads/AbstractReadExecutor.java b/src/java/org/apache/cassandra/service/reads/AbstractReadExecutor.java
new file mode 100644
index 0000000..8907e74
--- /dev/null
+++ b/src/java/org/apache/cassandra/service/reads/AbstractReadExecutor.java
@@ -0,0 +1,435 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.service.reads;
+
+import com.google.common.base.Preconditions;
+
+import com.google.common.base.Predicates;
+
+import org.apache.cassandra.db.transform.DuplicateRowChecker;
+import org.apache.cassandra.locator.ReplicaPlan;
+import org.apache.cassandra.locator.ReplicaPlans;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.concurrent.Stage;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.ConsistencyLevel;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.ReadCommand;
+import org.apache.cassandra.db.SinglePartitionReadCommand;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.partitions.PartitionIterator;
+import org.apache.cassandra.exceptions.ReadFailureException;
+import org.apache.cassandra.exceptions.ReadTimeoutException;
+import org.apache.cassandra.exceptions.UnavailableException;
+import org.apache.cassandra.locator.EndpointsForToken;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.Replica;
+import org.apache.cassandra.locator.ReplicaCollection;
+import org.apache.cassandra.net.Message;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.service.reads.repair.ReadRepair;
+import org.apache.cassandra.service.StorageProxy.LocalReadRunnable;
+import org.apache.cassandra.tracing.TraceState;
+import org.apache.cassandra.tracing.Tracing;
+
+import static com.google.common.collect.Iterables.all;
+import static java.util.concurrent.TimeUnit.NANOSECONDS;
+
+/**
+ * Sends a read request to the replicas needed to satisfy a given ConsistencyLevel.
+ *
+ * Optionally, may perform additional requests to provide redundancy against replica failure:
+ * AlwaysSpeculatingReadExecutor will always send a request to one extra replica, while
+ * SpeculatingReadExecutor will wait until it looks like the original request is in danger
+ * of timing out before performing extra reads.
+ */
+public abstract class AbstractReadExecutor
+{
+ private static final Logger logger = LoggerFactory.getLogger(AbstractReadExecutor.class);
+
+ protected final ReadCommand command;
+ private final ReplicaPlan.SharedForTokenRead replicaPlan;
+ protected final ReadRepair<EndpointsForToken, ReplicaPlan.ForTokenRead> readRepair;
+ protected final DigestResolver<EndpointsForToken, ReplicaPlan.ForTokenRead> digestResolver;
+ protected final ReadCallback<EndpointsForToken, ReplicaPlan.ForTokenRead> handler;
+ protected final TraceState traceState;
+ protected final ColumnFamilyStore cfs;
+ protected final long queryStartNanoTime;
+ private final int initialDataRequestCount;
+ protected volatile PartitionIterator result = null;
+
+ AbstractReadExecutor(ColumnFamilyStore cfs, ReadCommand command, ReplicaPlan.ForTokenRead replicaPlan, int initialDataRequestCount, long queryStartNanoTime)
+ {
+ this.command = command;
+ this.replicaPlan = ReplicaPlan.shared(replicaPlan);
+ this.initialDataRequestCount = initialDataRequestCount;
+ // the ReadRepair and DigestResolver both need to see our updated
+ this.readRepair = ReadRepair.create(command, this.replicaPlan, queryStartNanoTime);
+ this.digestResolver = new DigestResolver<>(command, this.replicaPlan, queryStartNanoTime);
+ this.handler = new ReadCallback<>(digestResolver, command, this.replicaPlan, queryStartNanoTime);
+ this.cfs = cfs;
+ this.traceState = Tracing.instance.get();
+ this.queryStartNanoTime = queryStartNanoTime;
+
+
+ // Set the digest version (if we request some digests). This is the smallest version amongst all our target replicas since new nodes
+ // knows how to produce older digest but the reverse is not true.
+ // TODO: we need this when talking with pre-3.0 nodes. So if we preserve the digest format moving forward, we can get rid of this once
+ // we stop being compatible with pre-3.0 nodes.
+ int digestVersion = MessagingService.current_version;
+ for (Replica replica : replicaPlan.contacts())
+ digestVersion = Math.min(digestVersion, MessagingService.instance().versions.get(replica.endpoint()));
+ command.setDigestVersion(digestVersion);
+ }
+
+ public DecoratedKey getKey()
+ {
+ Preconditions.checkState(command instanceof SinglePartitionReadCommand,
+ "Can only get keys for SinglePartitionReadCommand");
+ return ((SinglePartitionReadCommand) command).partitionKey();
+ }
+
+ public ReadRepair getReadRepair()
+ {
+ return readRepair;
+ }
+
+ protected void makeFullDataRequests(ReplicaCollection<?> replicas)
+ {
+ assert all(replicas, Replica::isFull);
+ makeRequests(command, replicas);
+ }
+
+ protected void makeTransientDataRequests(Iterable<Replica> replicas)
+ {
+ makeRequests(command.copyAsTransientQuery(replicas), replicas);
+ }
+
+ protected void makeDigestRequests(Iterable<Replica> replicas)
+ {
+ assert all(replicas, Replica::isFull);
+ // only send digest requests to full replicas, send data requests instead to the transient replicas
+ makeRequests(command.copyAsDigestQuery(replicas), replicas);
+ }
+
+ private void makeRequests(ReadCommand readCommand, Iterable<Replica> replicas)
+ {
+ boolean hasLocalEndpoint = false;
+ Message<ReadCommand> message = null;
+
+ for (Replica replica: replicas)
+ {
+ assert replica.isFull() || readCommand.acceptsTransient();
+
+ InetAddressAndPort endpoint = replica.endpoint();
+ if (replica.isSelf())
+ {
+ hasLocalEndpoint = true;
+ continue;
+ }
+
+ if (traceState != null)
+ traceState.trace("reading {} from {}", readCommand.isDigestQuery() ? "digest" : "data", endpoint);
+
+ if (null == message)
+ message = readCommand.createMessage(false);
+
+ MessagingService.instance().sendWithCallback(message, endpoint, handler);
+ }
+
+ // We delay the local (potentially blocking) read till the end to avoid stalling remote requests.
+ if (hasLocalEndpoint)
+ {
+ logger.trace("reading {} locally", readCommand.isDigestQuery() ? "digest" : "data");
+ Stage.READ.maybeExecuteImmediately(new LocalReadRunnable(command, handler));
+ }
+ }
+
+ /**
+ * Perform additional requests if it looks like the original will time out. May block while it waits
+ * to see if the original requests are answered first.
+ */
+ public abstract void maybeTryAdditionalReplicas();
+
+ /**
+ * send the initial set of requests
+ */
+ public void executeAsync()
+ {
+ EndpointsForToken selected = replicaPlan().contacts();
+ EndpointsForToken fullDataRequests = selected.filter(Replica::isFull, initialDataRequestCount);
+ makeFullDataRequests(fullDataRequests);
+ makeTransientDataRequests(selected.filterLazily(Replica::isTransient));
+ makeDigestRequests(selected.filterLazily(r -> r.isFull() && !fullDataRequests.contains(r)));
+ }
+
+ /**
+ * @return an executor appropriate for the configured speculative read policy
+ */
+ public static AbstractReadExecutor getReadExecutor(SinglePartitionReadCommand command, ConsistencyLevel consistencyLevel, long queryStartNanoTime) throws UnavailableException
+ {
+ Keyspace keyspace = Keyspace.open(command.metadata().keyspace);
+ ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(command.metadata().id);
+ SpeculativeRetryPolicy retry = cfs.metadata().params.speculativeRetry;
+
+ ReplicaPlan.ForTokenRead replicaPlan = ReplicaPlans.forRead(keyspace, command.partitionKey().getToken(), consistencyLevel, retry);
+
+ // Speculative retry is disabled *OR*
+ // 11980: Disable speculative retry if using EACH_QUORUM in order to prevent miscounting DC responses
+ if (retry.equals(NeverSpeculativeRetryPolicy.INSTANCE) || consistencyLevel == ConsistencyLevel.EACH_QUORUM)
+ return new NeverSpeculatingReadExecutor(cfs, command, replicaPlan, queryStartNanoTime, false);
+
+ // There are simply no extra replicas to speculate.
+ // Handle this separately so it can record failed attempts to speculate due to lack of replicas
+ if (replicaPlan.contacts().size() == replicaPlan.candidates().size())
+ {
+ boolean recordFailedSpeculation = consistencyLevel != ConsistencyLevel.ALL;
+ return new NeverSpeculatingReadExecutor(cfs, command, replicaPlan, queryStartNanoTime, recordFailedSpeculation);
+ }
+
+ if (retry.equals(AlwaysSpeculativeRetryPolicy.INSTANCE))
+ return new AlwaysSpeculatingReadExecutor(cfs, command, replicaPlan, queryStartNanoTime);
+ else // PERCENTILE or CUSTOM.
+ return new SpeculatingReadExecutor(cfs, command, replicaPlan, queryStartNanoTime);
+ }
+
+ /**
+ * Returns true if speculation should occur and if it should then block until it is time to
+ * send the speculative reads
+ */
+ boolean shouldSpeculateAndMaybeWait()
+ {
+ // no latency information, or we're overloaded
+ if (cfs.sampleReadLatencyNanos > command.getTimeout(NANOSECONDS))
+ return false;
+
+ return !handler.await(cfs.sampleReadLatencyNanos, NANOSECONDS);
+ }
+
+ ReplicaPlan.ForTokenRead replicaPlan()
+ {
+ return replicaPlan.get();
+ }
+
+ void onReadTimeout() {}
+
+ public static class NeverSpeculatingReadExecutor extends AbstractReadExecutor
+ {
+ /**
+ * If never speculating due to lack of replicas
+ * log it is as a failure if it should have happened
+ * but couldn't due to lack of replicas
+ */
+ private final boolean logFailedSpeculation;
+
+ public NeverSpeculatingReadExecutor(ColumnFamilyStore cfs, ReadCommand command, ReplicaPlan.ForTokenRead replicaPlan, long queryStartNanoTime, boolean logFailedSpeculation)
+ {
+ super(cfs, command, replicaPlan, 1, queryStartNanoTime);
+ this.logFailedSpeculation = logFailedSpeculation;
+ }
+
+ public void maybeTryAdditionalReplicas()
+ {
+ if (shouldSpeculateAndMaybeWait() && logFailedSpeculation)
+ {
+ cfs.metric.speculativeInsufficientReplicas.inc();
+ }
+ }
+ }
+
+ static class SpeculatingReadExecutor extends AbstractReadExecutor
+ {
+ private volatile boolean speculated = false;
+
+ public SpeculatingReadExecutor(ColumnFamilyStore cfs,
+ ReadCommand command,
+ ReplicaPlan.ForTokenRead replicaPlan,
+ long queryStartNanoTime)
+ {
+ // We're hitting additional targets for read repair (??). Since our "extra" replica is the least-
+ // preferred by the snitch, we do an extra data read to start with against a replica more
+ // likely to respond; better to let RR fail than the entire query.
+ super(cfs, command, replicaPlan, replicaPlan.blockFor() < replicaPlan.contacts().size() ? 2 : 1, queryStartNanoTime);
+ }
+
+ public void maybeTryAdditionalReplicas()
+ {
+ if (shouldSpeculateAndMaybeWait())
+ {
+ //Handle speculation stats first in case the callback fires immediately
+ cfs.metric.speculativeRetries.inc();
+ speculated = true;
+
+ ReplicaPlan.ForTokenRead replicaPlan = replicaPlan();
+ ReadCommand retryCommand;
+ Replica extraReplica;
+ if (handler.resolver.isDataPresent())
+ {
+ extraReplica = replicaPlan.firstUncontactedCandidate(Predicates.alwaysTrue());
+
+ // we should only use a SpeculatingReadExecutor if we have an extra replica to speculate against
+ assert extraReplica != null;
+
+ retryCommand = extraReplica.isTransient()
+ ? command.copyAsTransientQuery(extraReplica)
+ : command.copyAsDigestQuery(extraReplica);
+ }
+ else
+ {
+ extraReplica = replicaPlan.firstUncontactedCandidate(Replica::isFull);
+ retryCommand = command;
+ if (extraReplica == null)
+ {
+ cfs.metric.speculativeInsufficientReplicas.inc();
+ // cannot safely speculate a new data request, without more work - requests assumed to be
+ // unique per endpoint, and we have no full nodes left to speculate against
+ return;
+ }
+ }
+
+ // we must update the plan to include this new node, else when we come to read-repair, we may not include this
+ // speculated response in the data requests we make again, and we will not be able to 'speculate' an extra repair read,
+ // nor would we be able to speculate a new 'write' if the repair writes are insufficient
+ super.replicaPlan.addToContacts(extraReplica);
+
+ if (traceState != null)
+ traceState.trace("speculating read retry on {}", extraReplica);
+ logger.trace("speculating read retry on {}", extraReplica);
+ MessagingService.instance().sendWithCallback(retryCommand.createMessage(false), extraReplica.endpoint(), handler);
+ }
+ }
+
+ @Override
+ void onReadTimeout()
+ {
+ //Shouldn't be possible to get here without first attempting to speculate even if the
+ //timing is bad
+ assert speculated;
+ cfs.metric.speculativeFailedRetries.inc();
+ }
+ }
+
+ private static class AlwaysSpeculatingReadExecutor extends AbstractReadExecutor
+ {
+ public AlwaysSpeculatingReadExecutor(ColumnFamilyStore cfs,
+ ReadCommand command,
+ ReplicaPlan.ForTokenRead replicaPlan,
+ long queryStartNanoTime)
+ {
+ // presumably, we speculate an extra data request here in case it is our data request that fails to respond,
+ // and there are no more nodes to consult
+ super(cfs, command, replicaPlan, replicaPlan.contacts().size() > 1 ? 2 : 1, queryStartNanoTime);
+ }
+
+ public void maybeTryAdditionalReplicas()
+ {
+ // no-op
+ }
+
+ @Override
+ public void executeAsync()
+ {
+ super.executeAsync();
+ cfs.metric.speculativeRetries.inc();
+ }
+
+ @Override
+ void onReadTimeout()
+ {
+ cfs.metric.speculativeFailedRetries.inc();
+ }
+ }
+
+ public void setResult(PartitionIterator result)
+ {
+ Preconditions.checkState(this.result == null, "Result can only be set once");
+ this.result = DuplicateRowChecker.duringRead(result, this.replicaPlan.get().candidates().endpointList());
+ }
+
+ /**
+ * Wait for the CL to be satisfied by responses
+ */
+ public void awaitResponses() throws ReadTimeoutException
+ {
+ try
+ {
+ handler.awaitResults();
+ }
+ catch (ReadTimeoutException e)
+ {
+ try
+ {
+ onReadTimeout();
+ }
+ finally
+ {
+ throw e;
+ }
+ }
+
+ // return immediately, or begin a read repair
+ if (digestResolver.responsesMatch())
+ {
+ setResult(digestResolver.getData());
+ }
+ else
+ {
+ Tracing.trace("Digest mismatch: Mismatch for key {}", getKey());
+ readRepair.startRepair(digestResolver, this::setResult);
+ }
+ }
+
+ public void awaitReadRepair() throws ReadTimeoutException
+ {
+ try
+ {
+ readRepair.awaitReads();
+ }
+ catch (ReadTimeoutException e)
+ {
+ if (Tracing.isTracing())
+ Tracing.trace("Timed out waiting on digest mismatch repair requests");
+ else
+ logger.trace("Timed out waiting on digest mismatch repair requests");
+ // the caught exception here will have CL.ALL from the repair command,
+ // not whatever CL the initial command was at (CASSANDRA-7947)
+ throw new ReadTimeoutException(replicaPlan().consistencyLevel(), handler.blockFor - 1, handler.blockFor, true);
+ }
+ }
+
+ boolean isDone()
+ {
+ return result != null;
+ }
+
+ public void maybeSendAdditionalDataRequests()
+ {
+ if (isDone())
+ return;
+
+ readRepair.maybeSendAdditionalReads();
+ }
+
+ public PartitionIterator getResult() throws ReadFailureException, ReadTimeoutException
+ {
+ Preconditions.checkState(result != null, "Result must be set first");
+ return result;
+ }
+}
diff --git a/src/java/org/apache/cassandra/service/reads/AlwaysSpeculativeRetryPolicy.java b/src/java/org/apache/cassandra/service/reads/AlwaysSpeculativeRetryPolicy.java
new file mode 100644
index 0000000..a6092fb
--- /dev/null
+++ b/src/java/org/apache/cassandra/service/reads/AlwaysSpeculativeRetryPolicy.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.service.reads;
+
+import com.google.common.base.Objects;
+
+import com.codahale.metrics.Snapshot;
+
+public class AlwaysSpeculativeRetryPolicy implements SpeculativeRetryPolicy
+{
+ public static final AlwaysSpeculativeRetryPolicy INSTANCE = new AlwaysSpeculativeRetryPolicy();
+
+ private AlwaysSpeculativeRetryPolicy()
+ {
+ }
+
+ @Override
+ public long calculateThreshold(Snapshot latency, long existingValue)
+ {
+ return 0;
+ }
+
+ @Override
+ public Kind kind()
+ {
+ return Kind.ALWAYS;
+ }
+
+ @Override
+ public boolean equals(Object obj)
+ {
+ return obj instanceof AlwaysSpeculativeRetryPolicy;
+ }
+
+ @Override
+ public int hashCode()
+ {
+ return Objects.hashCode(kind());
+ }
+
+ @Override
+ public String toString()
+ {
+ return Kind.ALWAYS.toString();
+ }
+
+ static boolean stringMatches(String str)
+ {
+ return str.equalsIgnoreCase("ALWAYS");
+ }
+}
diff --git a/src/java/org/apache/cassandra/service/reads/DataResolver.java b/src/java/org/apache/cassandra/service/reads/DataResolver.java
new file mode 100644
index 0000000..30427d0
--- /dev/null
+++ b/src/java/org/apache/cassandra/service/reads/DataResolver.java
@@ -0,0 +1,405 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.service.reads;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+import java.util.function.UnaryOperator;
+
+import com.google.common.base.Joiner;
+
+import org.apache.cassandra.cql3.statements.schema.IndexTarget;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.DeletionTime;
+import org.apache.cassandra.db.ReadCommand;
+import org.apache.cassandra.db.ReadResponse;
+import org.apache.cassandra.db.filter.DataLimits;
+import org.apache.cassandra.db.partitions.PartitionIterator;
+import org.apache.cassandra.db.partitions.PartitionIterators;
+import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
+import org.apache.cassandra.db.partitions.UnfilteredPartitionIterators;
+import org.apache.cassandra.db.rows.RangeTombstoneMarker;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.db.rows.UnfilteredRowIterators;
+import org.apache.cassandra.db.transform.EmptyPartitionsDiscarder;
+import org.apache.cassandra.db.transform.Filter;
+import org.apache.cassandra.db.transform.FilteredPartitions;
+import org.apache.cassandra.db.transform.Transformation;
+import org.apache.cassandra.index.sasi.SASIIndex;
+import org.apache.cassandra.locator.Endpoints;
+import org.apache.cassandra.locator.ReplicaPlan;
+import org.apache.cassandra.net.Message;
+import org.apache.cassandra.schema.IndexMetadata;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.service.reads.repair.ReadRepair;
+import org.apache.cassandra.service.reads.repair.RepairedDataTracker;
+import org.apache.cassandra.service.reads.repair.RepairedDataVerifier;
+
+import static com.google.common.collect.Iterables.*;
+
+public class DataResolver<E extends Endpoints<E>, P extends ReplicaPlan.ForRead<E>> extends ResponseResolver<E, P>
+{
+ private final boolean enforceStrictLiveness;
+ private final ReadRepair<E, P> readRepair;
+
+ public DataResolver(ReadCommand command, ReplicaPlan.Shared<E, P> replicaPlan, ReadRepair<E, P> readRepair, long queryStartNanoTime)
+ {
+ super(command, replicaPlan, queryStartNanoTime);
+ this.enforceStrictLiveness = command.metadata().enforceStrictLiveness();
+ this.readRepair = readRepair;
+ }
+
+ public PartitionIterator getData()
+ {
+ ReadResponse response = responses.get(0).payload;
+ return UnfilteredPartitionIterators.filter(response.makeIterator(command), command.nowInSec());
+ }
+
+ public boolean isDataPresent()
+ {
+ return !responses.isEmpty();
+ }
+
+ public PartitionIterator resolve()
+ {
+ // We could get more responses while this method runs, which is ok (we're happy to ignore any response not here
+ // at the beginning of this method), so grab the response count once and use that through the method.
+ Collection<Message<ReadResponse>> messages = responses.snapshot();
+ assert !any(messages, msg -> msg.payload.isDigestResponse());
+
+ E replicas = replicaPlan().candidates().select(transform(messages, Message::from), false);
+
+ // If requested, inspect each response for a digest of the replica's repaired data set
+ RepairedDataTracker repairedDataTracker = command.isTrackingRepairedStatus()
+ ? new RepairedDataTracker(getRepairedDataVerifier(command))
+ : null;
+ if (repairedDataTracker != null)
+ {
+ messages.forEach(msg -> {
+ if (msg.payload.mayIncludeRepairedDigest() && replicas.byEndpoint().get(msg.from()).isFull())
+ {
+ repairedDataTracker.recordDigest(msg.from(),
+ msg.payload.repairedDataDigest(),
+ msg.payload.isRepairedDigestConclusive());
+ }
+ });
+ }
+
+ if (!needsReplicaFilteringProtection())
+ {
+ ResolveContext context = new ResolveContext(replicas);
+ return resolveWithReadRepair(context,
+ i -> shortReadProtectedResponse(i, context),
+ UnaryOperator.identity(),
+ repairedDataTracker);
+ }
+
+ return resolveWithReplicaFilteringProtection(replicas, repairedDataTracker);
+ }
+
+ private boolean needsReplicaFilteringProtection()
+ {
+ if (command.rowFilter().isEmpty())
+ return false;
+
+ IndexMetadata indexDef = command.indexMetadata();
+ if (indexDef != null && indexDef.isCustom())
+ {
+ String className = indexDef.options.get(IndexTarget.CUSTOM_INDEX_OPTION_NAME);
+ return !SASIIndex.class.getName().equals(className);
+ }
+
+ return true;
+ }
+
+ private class ResolveContext
+ {
+ private final E replicas;
+ private final DataLimits.Counter mergedResultCounter;
+
+ private ResolveContext(E replicas)
+ {
+ this.replicas = replicas;
+ this.mergedResultCounter = command.limits().newCounter(command.nowInSec(),
+ true,
+ command.selectsFullPartition(),
+ enforceStrictLiveness);
+ }
+
+ private boolean needsReadRepair()
+ {
+ return replicas.size() > 1;
+ }
+
+ private boolean needShortReadProtection()
+ {
+ // If we have only one result, there is no read repair to do and we can't get short reads
+ // Also, so-called "short reads" stems from nodes returning only a subset of the results they have for a
+ // partition due to the limit, but that subset not being enough post-reconciliation. So if we don't have limit,
+ // don't bother protecting against short reads.
+ return replicas.size() > 1 && !command.limits().isUnlimited();
+ }
+ }
+
+ @FunctionalInterface
+ private interface ResponseProvider
+ {
+ UnfilteredPartitionIterator getResponse(int i);
+ }
+
+ private UnfilteredPartitionIterator shortReadProtectedResponse(int i, ResolveContext context)
+ {
+ UnfilteredPartitionIterator originalResponse = responses.get(i).payload.makeIterator(command);
+
+ return context.needShortReadProtection()
+ ? ShortReadProtection.extend(context.replicas.get(i),
+ originalResponse,
+ command,
+ context.mergedResultCounter,
+ queryStartNanoTime,
+ enforceStrictLiveness)
+ : originalResponse;
+ }
+
+ private PartitionIterator resolveWithReadRepair(ResolveContext context,
+ ResponseProvider responseProvider,
+ UnaryOperator<PartitionIterator> preCountFilter,
+ RepairedDataTracker repairedDataTracker)
+ {
+ UnfilteredPartitionIterators.MergeListener listener = null;
+ if (context.needsReadRepair())
+ {
+ P sources = replicaPlan.getWithContacts(context.replicas);
+ listener = wrapMergeListener(readRepair.getMergeListener(sources), sources, repairedDataTracker);
+ }
+
+ return resolveInternal(context, listener, responseProvider, preCountFilter);
+ }
+
+ @SuppressWarnings("resource")
+ private PartitionIterator resolveWithReplicaFilteringProtection(E replicas, RepairedDataTracker repairedDataTracker)
+ {
+ // Protecting against inconsistent replica filtering (some replica returning a row that is outdated but that
+ // wouldn't be removed by normal reconciliation because up-to-date replica have filtered the up-to-date version
+ // of that row) works in 3 steps:
+ // 1) we read the full response just to collect rows that may be outdated (the ones we got from some
+ // replica but didn't got any response for other; it could be those other replica have filtered a more
+ // up-to-date result). In doing so, we do not count any of such "potentially outdated" row towards the
+ // query limit. This simulate the worst case scenario where all those "potentially outdated" rows are
+ // indeed outdated, and thus make sure we are guaranteed to read enough results (thanks to short read
+ // protection).
+ // 2) we query all the replica/rows we need to rule out whether those "potentially outdated" rows are outdated
+ // or not.
+ // 3) we re-read cached copies of each replica response using the "normal" read path merge with read-repair,
+ // but where for each replica we use their original response _plus_ the additional rows queried in the
+ // previous step (and apply the command#rowFilter() on the full result). Since the first phase has
+ // pessimistically collected enough results for the case where all potentially outdated results are indeed
+ // outdated, we shouldn't need further short-read protection requests during this phase.
+
+ // We need separate contexts, as each context has his own counter
+ ResolveContext firstPhaseContext = new ResolveContext(replicas);
+ ResolveContext secondPhaseContext = new ResolveContext(replicas);
+ ReplicaFilteringProtection<E> rfp = new ReplicaFilteringProtection<>(replicaPlan().keyspace(),
+ command,
+ replicaPlan().consistencyLevel(),
+ queryStartNanoTime,
+ firstPhaseContext.replicas);
+ PartitionIterator firstPhasePartitions = resolveInternal(firstPhaseContext,
+ rfp.mergeController(),
+ i -> shortReadProtectedResponse(i, firstPhaseContext),
+ UnaryOperator.identity());
+
+ // Consume the first phase partitions to populate the replica filtering protection with both those materialized
+ // partitions and the primary keys to be fetched.
+ PartitionIterators.consume(firstPhasePartitions);
+ firstPhasePartitions.close();
+
+ // After reading the entire query results the protection helper should have cached all the partitions so we can
+ // clear the responses accumulator for the sake of memory usage, given that the second phase might take long if
+ // it needs to query replicas.
+ responses.clearUnsafe();
+
+ return resolveWithReadRepair(secondPhaseContext,
+ rfp::queryProtectedPartitions,
+ results -> command.rowFilter().filter(results, command.metadata(), command.nowInSec()),
+ repairedDataTracker);
+ }
+
+ @SuppressWarnings("resource")
+ private PartitionIterator resolveInternal(ResolveContext context,
+ UnfilteredPartitionIterators.MergeListener mergeListener,
+ ResponseProvider responseProvider,
+ UnaryOperator<PartitionIterator> preCountFilter)
+ {
+ int count = context.replicas.size();
+ List<UnfilteredPartitionIterator> results = new ArrayList<>(count);
+ for (int i = 0; i < count; i++)
+ results.add(responseProvider.getResponse(i));
+
+ /*
+ * Even though every response, individually, will honor the limit, it is possible that we will, after the merge,
+ * have more rows than the client requested. To make sure that we still conform to the original limit,
+ * we apply a top-level post-reconciliation counter to the merged partition iterator.
+ *
+ * Short read protection logic (ShortReadRowsProtection.moreContents()) relies on this counter to be applied
+ * to the current partition to work. For this reason we have to apply the counter transformation before
+ * empty partition discard logic kicks in - for it will eagerly consume the iterator.
+ *
+ * That's why the order here is: 1) merge; 2) filter rows; 3) count; 4) discard empty partitions
+ *
+ * See CASSANDRA-13747 for more details.
+ */
+
+ UnfilteredPartitionIterator merged = UnfilteredPartitionIterators.merge(results, mergeListener);
+ FilteredPartitions filtered = FilteredPartitions.filter(merged, new Filter(command.nowInSec(), command.metadata().enforceStrictLiveness()));
+ PartitionIterator counted = Transformation.apply(preCountFilter.apply(filtered), context.mergedResultCounter);
+ return Transformation.apply(counted, new EmptyPartitionsDiscarder());
+ }
+
+ protected RepairedDataVerifier getRepairedDataVerifier(ReadCommand command)
+ {
+ return RepairedDataVerifier.verifier(command);
+ }
+
+ private String makeResponsesDebugString(DecoratedKey partitionKey)
+ {
+ return Joiner.on(",\n").join(transform(getMessages().snapshot(), m -> m.from() + " => " + m.payload.toDebugString(command, partitionKey)));
+ }
+
+ private UnfilteredPartitionIterators.MergeListener wrapMergeListener(UnfilteredPartitionIterators.MergeListener partitionListener,
+ P sources,
+ RepairedDataTracker repairedDataTracker)
+ {
+ // Avoid wrapping no-op listener as it doesn't throw, unless we're tracking repaired status
+ // in which case we need to inject the tracker & verify on close
+ if (partitionListener == UnfilteredPartitionIterators.MergeListener.NOOP)
+ {
+ if (repairedDataTracker == null)
+ return partitionListener;
+
+ return new UnfilteredPartitionIterators.MergeListener()
+ {
+
+ public UnfilteredRowIterators.MergeListener getRowMergeListener(DecoratedKey partitionKey, List<UnfilteredRowIterator> versions)
+ {
+ return UnfilteredRowIterators.MergeListener.NOOP;
+ }
+
+ public void close()
+ {
+ repairedDataTracker.verify();
+ }
+ };
+ }
+
+ return new UnfilteredPartitionIterators.MergeListener()
+ {
+ public UnfilteredRowIterators.MergeListener getRowMergeListener(DecoratedKey partitionKey, List<UnfilteredRowIterator> versions)
+ {
+ UnfilteredRowIterators.MergeListener rowListener = partitionListener.getRowMergeListener(partitionKey, versions);
+
+ return new UnfilteredRowIterators.MergeListener()
+ {
+ public void onMergedPartitionLevelDeletion(DeletionTime mergedDeletion, DeletionTime[] versions)
+ {
+ try
+ {
+ rowListener.onMergedPartitionLevelDeletion(mergedDeletion, versions);
+ }
+ catch (AssertionError e)
+ {
+ // The following can be pretty verbose, but it's really only triggered if a bug happen, so we'd
+ // rather get more info to debug than not.
+ TableMetadata table = command.metadata();
+ String details = String.format("Error merging partition level deletion on %s: merged=%s, versions=%s, sources={%s}, debug info:%n %s",
+ table,
+ mergedDeletion == null ? "null" : mergedDeletion.toString(),
+ '[' + Joiner.on(", ").join(transform(Arrays.asList(versions), rt -> rt == null ? "null" : rt.toString())) + ']',
+ sources.contacts(),
+ makeResponsesDebugString(partitionKey));
+ throw new AssertionError(details, e);
+ }
+ }
+
+ public Row onMergedRows(Row merged, Row[] versions)
+ {
+ try
+ {
+ return rowListener.onMergedRows(merged, versions);
+ }
+ catch (AssertionError e)
+ {
+ // The following can be pretty verbose, but it's really only triggered if a bug happen, so we'd
+ // rather get more info to debug than not.
+ TableMetadata table = command.metadata();
+ String details = String.format("Error merging rows on %s: merged=%s, versions=%s, sources={%s}, debug info:%n %s",
+ table,
+ merged == null ? "null" : merged.toString(table),
+ '[' + Joiner.on(", ").join(transform(Arrays.asList(versions), rt -> rt == null ? "null" : rt.toString(table))) + ']',
+ sources.contacts(),
+ makeResponsesDebugString(partitionKey));
+ throw new AssertionError(details, e);
+ }
+ }
+
+ public void onMergedRangeTombstoneMarkers(RangeTombstoneMarker merged, RangeTombstoneMarker[] versions)
+ {
+ try
+ {
+ // The code for merging range tombstones is a tad complex and we had the assertions there triggered
+ // unexpectedly in a few occasions (CASSANDRA-13237, CASSANDRA-13719). It's hard to get insights
+ // when that happen without more context that what the assertion errors give us however, hence the
+ // catch here that basically gather as much as context as reasonable.
+ rowListener.onMergedRangeTombstoneMarkers(merged, versions);
+ }
+ catch (AssertionError e)
+ {
+
+ // The following can be pretty verbose, but it's really only triggered if a bug happen, so we'd
+ // rather get more info to debug than not.
+ TableMetadata table = command.metadata();
+ String details = String.format("Error merging RTs on %s: merged=%s, versions=%s, sources={%s}, debug info:%n %s",
+ table,
+ merged == null ? "null" : merged.toString(table),
+ '[' + Joiner.on(", ").join(transform(Arrays.asList(versions), rt -> rt == null ? "null" : rt.toString(table))) + ']',
+ sources.contacts(),
+ makeResponsesDebugString(partitionKey));
+ throw new AssertionError(details, e);
+ }
+
+ }
+
+ public void close()
+ {
+ rowListener.close();
+ }
+ };
+ }
+
+ public void close()
+ {
+ partitionListener.close();
+ if (repairedDataTracker != null)
+ repairedDataTracker.verify();
+ }
+ };
+ }
+}
diff --git a/src/java/org/apache/cassandra/service/reads/DigestResolver.java b/src/java/org/apache/cassandra/service/reads/DigestResolver.java
new file mode 100644
index 0000000..dbb761b
--- /dev/null
+++ b/src/java/org/apache/cassandra/service/reads/DigestResolver.java
@@ -0,0 +1,166 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.service.reads;
+
+import java.nio.ByteBuffer;
+import java.util.Collection;
+import java.util.concurrent.TimeUnit;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+
+import org.apache.cassandra.db.ReadCommand;
+import org.apache.cassandra.db.ReadResponse;
+import org.apache.cassandra.db.SinglePartitionReadCommand;
+import org.apache.cassandra.db.partitions.PartitionIterator;
+import org.apache.cassandra.db.partitions.UnfilteredPartitionIterators;
+import org.apache.cassandra.locator.Endpoints;
+import org.apache.cassandra.locator.Replica;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.ReplicaPlan;
+import org.apache.cassandra.net.Message;
+import org.apache.cassandra.service.reads.repair.NoopReadRepair;
+import org.apache.cassandra.utils.ByteBufferUtil;
+
+import static com.google.common.collect.Iterables.any;
+
+public class DigestResolver<E extends Endpoints<E>, P extends ReplicaPlan.ForRead<E>> extends ResponseResolver<E, P>
+{
+ private volatile Message<ReadResponse> dataResponse;
+
+ public DigestResolver(ReadCommand command, ReplicaPlan.Shared<E, P> replicaPlan, long queryStartNanoTime)
+ {
+ super(command, replicaPlan, queryStartNanoTime);
+ Preconditions.checkArgument(command instanceof SinglePartitionReadCommand,
+ "DigestResolver can only be used with SinglePartitionReadCommand commands");
+ }
+
+ @Override
+ public void preprocess(Message<ReadResponse> message)
+ {
+ super.preprocess(message);
+ Replica replica = replicaPlan().lookup(message.from());
+ if (dataResponse == null && !message.payload.isDigestResponse() && replica.isFull())
+ dataResponse = message;
+ }
+
+ @VisibleForTesting
+ public boolean hasTransientResponse()
+ {
+ return hasTransientResponse(responses.snapshot());
+ }
+
+ private boolean hasTransientResponse(Collection<Message<ReadResponse>> responses)
+ {
+ return any(responses,
+ msg -> !msg.payload.isDigestResponse()
+ && replicaPlan().lookup(msg.from()).isTransient());
+ }
+
+ public PartitionIterator getData()
+ {
+ assert isDataPresent();
+
+ Collection<Message<ReadResponse>> responses = this.responses.snapshot();
+
+ if (!hasTransientResponse(responses))
+ {
+ return UnfilteredPartitionIterators.filter(dataResponse.payload.makeIterator(command), command.nowInSec());
+ }
+ else
+ {
+ // This path can be triggered only if we've got responses from full replicas and they match, but
+ // transient replica response still contains data, which needs to be reconciled.
+ DataResolver<E, P> dataResolver
+ = new DataResolver<>(command, replicaPlan, NoopReadRepair.instance, queryStartNanoTime);
+
+ dataResolver.preprocess(dataResponse);
+ // Reconcile with transient replicas
+ for (Message<ReadResponse> response : responses)
+ {
+ Replica replica = replicaPlan().lookup(response.from());
+ if (replica.isTransient())
+ dataResolver.preprocess(response);
+ }
+
+ return dataResolver.resolve();
+ }
+ }
+
+ public boolean responsesMatch()
+ {
+ long start = System.nanoTime();
+
+ // validate digests against each other; return false immediately on mismatch.
+ ByteBuffer digest = null;
+ Collection<Message<ReadResponse>> snapshot = responses.snapshot();
+ if (snapshot.size() <= 1)
+ return true;
+
+ // TODO: should also not calculate if only one full node
+ for (Message<ReadResponse> message : snapshot)
+ {
+ if (replicaPlan().lookup(message.from()).isTransient())
+ continue;
+
+ ByteBuffer newDigest = message.payload.digest(command);
+ if (digest == null)
+ digest = newDigest;
+ else if (!digest.equals(newDigest))
+ // rely on the fact that only single partition queries use digests
+ return false;
+ }
+
+ if (logger.isTraceEnabled())
+ logger.trace("responsesMatch: {} ms.", TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start));
+
+ return true;
+ }
+
+ public boolean isDataPresent()
+ {
+ return dataResponse != null;
+ }
+
+ public DigestResolverDebugResult[] getDigestsByEndpoint()
+ {
+ DigestResolverDebugResult[] ret = new DigestResolverDebugResult[responses.size()];
+ for (int i = 0; i < responses.size(); i++)
+ {
+ Message<ReadResponse> message = responses.get(i);
+ ReadResponse response = message.payload;
+ String digestHex = ByteBufferUtil.bytesToHex(response.digest(command));
+ ret[i] = new DigestResolverDebugResult(message.from(), digestHex, message.payload.isDigestResponse());
+ }
+ return ret;
+ }
+
+ public static class DigestResolverDebugResult
+ {
+ public InetAddressAndPort from;
+ public String digestHex;
+ public boolean isDigestResponse;
+
+ private DigestResolverDebugResult(InetAddressAndPort from, String digestHex, boolean isDigestResponse)
+ {
+ this.from = from;
+ this.digestHex = digestHex;
+ this.isDigestResponse = isDigestResponse;
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/service/reads/FixedSpeculativeRetryPolicy.java b/src/java/org/apache/cassandra/service/reads/FixedSpeculativeRetryPolicy.java
new file mode 100644
index 0000000..9bbeb12
--- /dev/null
+++ b/src/java/org/apache/cassandra/service/reads/FixedSpeculativeRetryPolicy.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.service.reads;
+
+import java.util.concurrent.TimeUnit;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import com.google.common.base.Objects;
+
+import com.codahale.metrics.Snapshot;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.schema.TableParams;
+
+public class FixedSpeculativeRetryPolicy implements SpeculativeRetryPolicy
+{
+ private static final Pattern PATTERN = Pattern.compile("^(?<val>[0-9.]+)ms$", Pattern.CASE_INSENSITIVE);
+
+ private final int speculateAtMilliseconds;
+
+ FixedSpeculativeRetryPolicy(int speculateAtMilliseconds)
+ {
+ this.speculateAtMilliseconds = speculateAtMilliseconds;
+ }
+
+ @Override
+ public long calculateThreshold(Snapshot latency, long existingValue)
+ {
+ return TimeUnit.MILLISECONDS.toNanos(speculateAtMilliseconds);
+ }
+
+ @Override
+ public Kind kind()
+ {
+ return Kind.FIXED;
+ }
+
+ @Override
+ public boolean equals(Object obj)
+ {
+ if (!(obj instanceof FixedSpeculativeRetryPolicy))
+ return false;
+ FixedSpeculativeRetryPolicy rhs = (FixedSpeculativeRetryPolicy) obj;
+ return speculateAtMilliseconds == rhs.speculateAtMilliseconds;
+ }
+
+ @Override
+ public int hashCode()
+ {
+ return Objects.hashCode(kind(), speculateAtMilliseconds);
+ }
+
+ @Override
+ public String toString()
+ {
+ return String.format("%dms", speculateAtMilliseconds);
+ }
+
+ static FixedSpeculativeRetryPolicy fromString(String str)
+ {
+ Matcher matcher = PATTERN.matcher(str);
+
+ if (!matcher.matches())
+ throw new IllegalArgumentException();
+
+ String val = matcher.group("val");
+ try
+ {
+ // historically we've always parsed this as double, but treated as int; so we keep doing it for compatibility
+ return new FixedSpeculativeRetryPolicy((int) Double.parseDouble(val));
+ }
+ catch (IllegalArgumentException e)
+ {
+ throw new ConfigurationException(String.format("Invalid value %s for option '%s'", str, TableParams.Option.SPECULATIVE_RETRY));
+ }
+ }
+
+ static boolean stringMatches(String str)
+ {
+ return PATTERN.matcher(str).matches();
+ }
+}
diff --git a/src/java/org/apache/cassandra/service/reads/HybridSpeculativeRetryPolicy.java b/src/java/org/apache/cassandra/service/reads/HybridSpeculativeRetryPolicy.java
new file mode 100644
index 0000000..8228c45
--- /dev/null
+++ b/src/java/org/apache/cassandra/service/reads/HybridSpeculativeRetryPolicy.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.service.reads;
+
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import com.google.common.base.Objects;
+
+import com.codahale.metrics.Snapshot;
+import com.codahale.metrics.Timer;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.schema.TableParams;
+
+public class HybridSpeculativeRetryPolicy implements SpeculativeRetryPolicy
+{
+ private static final Pattern PATTERN =
+ Pattern.compile("^(?<fun>MIN|MAX)\\((?<val1>[0-9.]+[a-z]+)\\s*,\\s*(?<val2>[0-9.]+[a-z]+)\\)$",
+ Pattern.CASE_INSENSITIVE);
+
+ public enum Function
+ {
+ MIN, MAX;
+
+ long call(long val1, long val2)
+ {
+ return this == MIN ? Math.min(val1, val2) : Math.max(val1, val2);
+ }
+ }
+
+ private final PercentileSpeculativeRetryPolicy percentilePolicy;
+ private final FixedSpeculativeRetryPolicy fixedPolicy;
+ private final Function function;
+
+ HybridSpeculativeRetryPolicy(PercentileSpeculativeRetryPolicy percentilePolicy,
+ FixedSpeculativeRetryPolicy fixedPolicy,
+ Function function)
+ {
+ this.percentilePolicy = percentilePolicy;
+ this.fixedPolicy = fixedPolicy;
+ this.function = function;
+ }
+
+ @Override
+ public long calculateThreshold(Snapshot latency, long existingValue)
+ {
+ if (latency.size() <= 0)
+ return existingValue;
+ return function.call(percentilePolicy.calculateThreshold(latency, existingValue), fixedPolicy.calculateThreshold(latency, existingValue));
+ }
+
+ @Override
+ public Kind kind()
+ {
+ return Kind.HYBRID;
+ }
+
+ @Override
+ public boolean equals(Object obj)
+ {
+ if (!(obj instanceof HybridSpeculativeRetryPolicy))
+ return false;
+ HybridSpeculativeRetryPolicy rhs = (HybridSpeculativeRetryPolicy) obj;
+ return function == rhs.function
+ && Objects.equal(percentilePolicy, rhs.percentilePolicy)
+ && Objects.equal(fixedPolicy, rhs.fixedPolicy);
+ }
+
+ @Override
+ public int hashCode()
+ {
+ return Objects.hashCode(function, percentilePolicy, fixedPolicy);
+ }
+
+ @Override
+ public String toString()
+ {
+ return String.format("%s(%s,%s)", function, percentilePolicy, fixedPolicy);
+ }
+
+ static HybridSpeculativeRetryPolicy fromString(String str)
+ {
+ Matcher matcher = PATTERN.matcher(str);
+
+ if (!matcher.matches())
+ throw new IllegalArgumentException();
+
+ String val1 = matcher.group("val1");
+ String val2 = matcher.group("val2");
+
+ SpeculativeRetryPolicy value1, value2;
+ try
+ {
+ value1 = SpeculativeRetryPolicy.fromString(val1);
+ value2 = SpeculativeRetryPolicy.fromString(val2);
+ }
+ catch (ConfigurationException e)
+ {
+ throw new ConfigurationException(String.format("Invalid value %s for option '%s'", str, TableParams.Option.SPECULATIVE_RETRY));
+ }
+
+ if (value1.kind() == value2.kind())
+ {
+ throw new ConfigurationException(String.format("Invalid value %s for option '%s': MIN()/MAX() arguments " +
+ "should be of different types, but both are of type %s",
+ str, TableParams.Option.SPECULATIVE_RETRY, value1.kind()));
+ }
+
+ SpeculativeRetryPolicy policy1 = value1 instanceof PercentileSpeculativeRetryPolicy ? value1 : value2;
+ SpeculativeRetryPolicy policy2 = value1 instanceof FixedSpeculativeRetryPolicy ? value1 : value2;
+
+ Function function = Function.valueOf(matcher.group("fun").toUpperCase());
+ return new HybridSpeculativeRetryPolicy((PercentileSpeculativeRetryPolicy) policy1, (FixedSpeculativeRetryPolicy) policy2, function);
+ }
+
+ static boolean stringMatches(String str)
+ {
+ return PATTERN.matcher(str).matches();
+ }
+}
diff --git a/src/java/org/apache/cassandra/service/reads/NeverSpeculativeRetryPolicy.java b/src/java/org/apache/cassandra/service/reads/NeverSpeculativeRetryPolicy.java
new file mode 100644
index 0000000..1211142
--- /dev/null
+++ b/src/java/org/apache/cassandra/service/reads/NeverSpeculativeRetryPolicy.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.service.reads;
+
+import com.google.common.base.Objects;
+
+import com.codahale.metrics.Snapshot;
+
+public class NeverSpeculativeRetryPolicy implements SpeculativeRetryPolicy
+{
+ public static final NeverSpeculativeRetryPolicy INSTANCE = new NeverSpeculativeRetryPolicy();
+
+ private NeverSpeculativeRetryPolicy()
+ {
+ }
+
+ @Override
+ public long calculateThreshold(Snapshot latency, long existingValue)
+ {
+ return Long.MAX_VALUE;
+ }
+
+ @Override
+ public Kind kind()
+ {
+ return Kind.NEVER;
+ }
+
+ @Override
+ public boolean equals(Object obj)
+ {
+ return obj instanceof NeverSpeculativeRetryPolicy;
+ }
+
+ @Override
+ public int hashCode()
+ {
+ return Objects.hashCode(kind());
+ }
+
+ @Override
+ public String toString()
+ {
+ return Kind.NEVER.toString();
+ }
+
+ static boolean stringMatches(String str)
+ {
+ return str.equalsIgnoreCase("NEVER") || str.equalsIgnoreCase("NONE");
+ }
+}
diff --git a/src/java/org/apache/cassandra/service/reads/PercentileSpeculativeRetryPolicy.java b/src/java/org/apache/cassandra/service/reads/PercentileSpeculativeRetryPolicy.java
new file mode 100644
index 0000000..ffd473e
--- /dev/null
+++ b/src/java/org/apache/cassandra/service/reads/PercentileSpeculativeRetryPolicy.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.service.reads;
+
+import java.text.DecimalFormat;
+import java.text.DecimalFormatSymbols;
+import java.util.Locale;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import com.google.common.base.Objects;
+
+import com.codahale.metrics.Snapshot;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.schema.TableParams;
+
+public class PercentileSpeculativeRetryPolicy implements SpeculativeRetryPolicy
+{
+ public static final PercentileSpeculativeRetryPolicy NINETY_NINE_P = new PercentileSpeculativeRetryPolicy(99.0);
+
+ private static final Pattern PATTERN = Pattern.compile("^(?<val>[0-9.]+)p(ercentile)?$", Pattern.CASE_INSENSITIVE);
+ /**
+ * The pattern above uses dot as decimal separator, so we use {@link Locale#ENGLISH} to enforce that. (CASSANDRA-14374)
+ */
+ private static final DecimalFormat FORMATTER = new DecimalFormat("#.####", new DecimalFormatSymbols(Locale.ENGLISH));
+
+ private final double percentile;
+
+ public PercentileSpeculativeRetryPolicy(double percentile)
+ {
+ this.percentile = percentile;
+ }
+
+ @Override
+ public long calculateThreshold(Snapshot latency, long existingValue)
+ {
+ if (latency.size() <= 0)
+ return existingValue;
+ return (long) latency.getValue(percentile / 100);
+ }
+
+ @Override
+ public Kind kind()
+ {
+ return Kind.PERCENTILE;
+ }
+
+ @Override
+ public boolean equals(Object obj)
+ {
+ if (!(obj instanceof PercentileSpeculativeRetryPolicy))
+ return false;
+ PercentileSpeculativeRetryPolicy rhs = (PercentileSpeculativeRetryPolicy) obj;
+ return percentile == rhs.percentile;
+ }
+
+ @Override
+ public int hashCode()
+ {
+ return Objects.hashCode(kind(), percentile);
+ }
+
+ @Override
+ public String toString()
+ {
+ return String.format("%sp", FORMATTER.format(percentile));
+ }
+
+ static PercentileSpeculativeRetryPolicy fromString(String str)
+ {
+ Matcher matcher = PATTERN.matcher(str);
+
+ if (!matcher.matches())
+ throw new IllegalArgumentException();
+
+ String val = matcher.group("val");
+
+ double percentile;
+ try
+ {
+ percentile = Double.parseDouble(val);
+ }
+ catch (IllegalArgumentException e)
+ {
+ throw new ConfigurationException(String.format("Invalid value %s for option '%s'", str, TableParams.Option.SPECULATIVE_RETRY));
+ }
+
+ if (percentile <= 0.0 || percentile >= 100.0)
+ {
+ throw new ConfigurationException(String.format("Invalid value %s for PERCENTILE option '%s': must be between (0.0 and 100.0)",
+ str, TableParams.Option.SPECULATIVE_RETRY));
+ }
+
+ return new PercentileSpeculativeRetryPolicy(percentile);
+ }
+
+ static boolean stringMatches(String str)
+ {
+ return PATTERN.matcher(str).matches();
+ }
+}
diff --git a/src/java/org/apache/cassandra/service/reads/ReadCallback.java b/src/java/org/apache/cassandra/service/reads/ReadCallback.java
new file mode 100644
index 0000000..2968dbc
--- /dev/null
+++ b/src/java/org/apache/cassandra/service/reads/ReadCallback.java
@@ -0,0 +1,179 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.service.reads;
+
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.locator.ReplicaPlan;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.db.PartitionRangeReadCommand;
+import org.apache.cassandra.db.ReadCommand;
+import org.apache.cassandra.db.ReadResponse;
+import org.apache.cassandra.exceptions.ReadFailureException;
+import org.apache.cassandra.exceptions.ReadTimeoutException;
+import org.apache.cassandra.exceptions.RequestFailureReason;
+import org.apache.cassandra.locator.Endpoints;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.net.RequestCallback;
+import org.apache.cassandra.net.Message;
+import org.apache.cassandra.net.Verb;
+import org.apache.cassandra.tracing.Tracing;
+import org.apache.cassandra.utils.concurrent.SimpleCondition;
+
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+
+public class ReadCallback<E extends Endpoints<E>, P extends ReplicaPlan.ForRead<E>> implements RequestCallback<ReadResponse>
+{
+ protected static final Logger logger = LoggerFactory.getLogger( ReadCallback.class );
+
+ public final ResponseResolver resolver;
+ final SimpleCondition condition = new SimpleCondition();
+ private final long queryStartNanoTime;
+ final int blockFor; // TODO: move to replica plan as well?
+ // this uses a plain reference, but is initialised before handoff to any other threads; the later updates
+ // may not be visible to the threads immediately, but ReplicaPlan only contains final fields, so they will never see an uninitialised object
+ final ReplicaPlan.Shared<E, P> replicaPlan;
+ private final ReadCommand command;
+ private static final AtomicIntegerFieldUpdater<ReadCallback> recievedUpdater
+ = AtomicIntegerFieldUpdater.newUpdater(ReadCallback.class, "received");
+ private volatile int received = 0;
+ private static final AtomicIntegerFieldUpdater<ReadCallback> failuresUpdater
+ = AtomicIntegerFieldUpdater.newUpdater(ReadCallback.class, "failures");
+ private volatile int failures = 0;
+ private final Map<InetAddressAndPort, RequestFailureReason> failureReasonByEndpoint;
+
+ public ReadCallback(ResponseResolver resolver, ReadCommand command, ReplicaPlan.Shared<E, P> replicaPlan, long queryStartNanoTime)
+ {
+ this.command = command;
+ this.resolver = resolver;
+ this.queryStartNanoTime = queryStartNanoTime;
+ this.replicaPlan = replicaPlan;
+ this.blockFor = replicaPlan.get().blockFor();
+ this.failureReasonByEndpoint = new ConcurrentHashMap<>();
+ // we don't support read repair (or rapid read protection) for range scans yet (CASSANDRA-6897)
+ assert !(command instanceof PartitionRangeReadCommand) || blockFor >= replicaPlan().contacts().size();
+
+ if (logger.isTraceEnabled())
+ logger.trace("Blockfor is {}; setting up requests to {}", blockFor, this.replicaPlan);
+ }
+
+ protected P replicaPlan()
+ {
+ return replicaPlan.get();
+ }
+
+ public boolean await(long timePastStart, TimeUnit unit)
+ {
+ long time = unit.toNanos(timePastStart) - (System.nanoTime() - queryStartNanoTime);
+ try
+ {
+ return condition.await(time, TimeUnit.NANOSECONDS);
+ }
+ catch (InterruptedException ex)
+ {
+ throw new AssertionError(ex);
+ }
+ }
+
+ public void awaitResults() throws ReadFailureException, ReadTimeoutException
+ {
+ boolean signaled = await(command.getTimeout(MILLISECONDS), TimeUnit.MILLISECONDS);
+ boolean failed = failures > 0 && blockFor + failures > replicaPlan().contacts().size();
+ if (signaled && !failed)
+ return;
+
+ if (Tracing.isTracing())
+ {
+ String gotData = received > 0 ? (resolver.isDataPresent() ? " (including data)" : " (only digests)") : "";
+ Tracing.trace("{}; received {} of {} responses{}", new Object[]{ (failed ? "Failed" : "Timed out"), received, blockFor, gotData });
+ }
+ else if (logger.isDebugEnabled())
+ {
+ String gotData = received > 0 ? (resolver.isDataPresent() ? " (including data)" : " (only digests)") : "";
+ logger.debug("{}; received {} of {} responses{}", new Object[]{ (failed ? "Failed" : "Timed out"), received, blockFor, gotData });
+ }
+
+ // Same as for writes, see AbstractWriteResponseHandler
+ throw failed
+ ? new ReadFailureException(replicaPlan().consistencyLevel(), received, blockFor, resolver.isDataPresent(), failureReasonByEndpoint)
+ : new ReadTimeoutException(replicaPlan().consistencyLevel(), received, blockFor, resolver.isDataPresent());
+ }
+
+ public int blockFor()
+ {
+ return blockFor;
+ }
+
+ public void onResponse(Message<ReadResponse> message)
+ {
+ resolver.preprocess(message);
+ int n = waitingFor(message.from())
+ ? recievedUpdater.incrementAndGet(this)
+ : received;
+
+ if (n >= blockFor && resolver.isDataPresent())
+ condition.signalAll();
+ }
+
+ /**
+ * @return true if the message counts towards the blockFor threshold
+ */
+ private boolean waitingFor(InetAddressAndPort from)
+ {
+ return !replicaPlan().consistencyLevel().isDatacenterLocal() || DatabaseDescriptor.getLocalDataCenter().equals(DatabaseDescriptor.getEndpointSnitch().getDatacenter(from));
+ }
+
+ public void response(ReadResponse result)
+ {
+ Verb kind = command.isRangeRequest() ? Verb.RANGE_RSP : Verb.READ_RSP;
+ Message<ReadResponse> message = Message.internalResponse(kind, result);
+ onResponse(message);
+ }
+
+
+ @Override
+ public boolean trackLatencyForSnitch()
+ {
+ return true;
+ }
+
+ @Override
+ public void onFailure(InetAddressAndPort from, RequestFailureReason failureReason)
+ {
+ int n = waitingFor(from)
+ ? failuresUpdater.incrementAndGet(this)
+ : failures;
+
+ failureReasonByEndpoint.put(from, failureReason);
+
+ if (blockFor + n > replicaPlan().contacts().size())
+ condition.signalAll();
+ }
+
+ @Override
+ public boolean invokeOnFailure()
+ {
+ return true;
+ }
+}
diff --git a/src/java/org/apache/cassandra/service/reads/ReplicaFilteringProtection.java b/src/java/org/apache/cassandra/service/reads/ReplicaFilteringProtection.java
new file mode 100644
index 0000000..b4a3cb5
--- /dev/null
+++ b/src/java/org/apache/cassandra/service/reads/ReplicaFilteringProtection.java
@@ -0,0 +1,480 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.reads;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.List;
+import java.util.NavigableSet;
+import java.util.SortedMap;
+import java.util.TreeMap;
+import java.util.stream.Collectors;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.concurrent.Stage;
+import org.apache.cassandra.db.Clustering;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Columns;
+import org.apache.cassandra.db.ConsistencyLevel;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.DeletionTime;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.ReadCommand;
+import org.apache.cassandra.db.RegularAndStaticColumns;
+import org.apache.cassandra.db.SinglePartitionReadCommand;
+import org.apache.cassandra.db.filter.ClusteringIndexFilter;
+import org.apache.cassandra.db.filter.ClusteringIndexNamesFilter;
+import org.apache.cassandra.db.filter.DataLimits;
+import org.apache.cassandra.db.filter.RowFilter;
+import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
+import org.apache.cassandra.db.partitions.UnfilteredPartitionIterators;
+import org.apache.cassandra.db.rows.EncodingStats;
+import org.apache.cassandra.db.rows.RangeTombstoneMarker;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.db.rows.Rows;
+import org.apache.cassandra.db.rows.Unfiltered;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.db.rows.UnfilteredRowIterators;
+import org.apache.cassandra.exceptions.ReadTimeoutException;
+import org.apache.cassandra.exceptions.UnavailableException;
+import org.apache.cassandra.locator.Endpoints;
+import org.apache.cassandra.locator.Replica;
+import org.apache.cassandra.locator.ReplicaPlan;
+import org.apache.cassandra.locator.ReplicaPlans;
+import org.apache.cassandra.metrics.TableMetrics;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.service.StorageProxy;
+import org.apache.cassandra.service.reads.repair.NoopReadRepair;
+import org.apache.cassandra.tracing.Tracing;
+import org.apache.cassandra.utils.btree.BTreeSet;
+
+/**
+ * Helper in charge of collecting additional queries to be done on the coordinator to protect against invalid results
+ * being included due to replica-side filtering (secondary indexes or {@code ALLOW * FILTERING}).
+ * <p>
+ * When using replica-side filtering with CL>ONE, a replica can send a stale result satisfying the filter, while updated
+ * replicas won't send a corresponding tombstone to discard that result during reconciliation. This helper identifies
+ * the rows in a replica response that don't have a corresponding row in other replica responses, and requests them by
+ * primary key to the "silent" replicas in a second fetch round.
+ * <p>
+ * See CASSANDRA-8272 and CASSANDRA-8273 for further details.
+ */
+class ReplicaFilteringProtection<E extends Endpoints<E>>
+{
+ private static final Logger logger = LoggerFactory.getLogger(ReplicaFilteringProtection.class);
+
+ private final Keyspace keyspace;
+ private final ReadCommand command;
+ private final ConsistencyLevel consistency;
+ private final long queryStartNanoTime;
+ private final E sources;
+ private final TableMetrics tableMetrics;
+
+ /**
+ * Per-source primary keys of the rows that might be outdated so they need to be fetched.
+ * For outdated static rows we use an empty builder to signal it has to be queried.
+ */
+ private final List<SortedMap<DecoratedKey, BTreeSet.Builder<Clustering>>> rowsToFetch;
+
+ /**
+ * Per-source list of all the partitions seen by the merge listener, to be merged with the extra fetched rows.
+ */
+ private final List<List<PartitionBuilder>> originalPartitions;
+
+ ReplicaFilteringProtection(Keyspace keyspace,
+ ReadCommand command,
+ ConsistencyLevel consistency,
+ long queryStartNanoTime,
+ E sources)
+ {
+ this.keyspace = keyspace;
+ this.command = command;
+ this.consistency = consistency;
+ this.queryStartNanoTime = queryStartNanoTime;
+ this.sources = sources;
+ this.rowsToFetch = new ArrayList<>(sources.size());
+ this.originalPartitions = new ArrayList<>(sources.size());
+
+ for (Replica ignored : sources)
+ {
+ rowsToFetch.add(new TreeMap<>());
+ originalPartitions.add(new ArrayList<>());
+ }
+
+ tableMetrics = ColumnFamilyStore.metricsFor(command.metadata().id);
+ }
+
+ private BTreeSet.Builder<Clustering> getOrCreateToFetch(int source, DecoratedKey partitionKey)
+ {
+ return rowsToFetch.get(source).computeIfAbsent(partitionKey, k -> BTreeSet.builder(command.metadata().comparator));
+ }
+
+ /**
+ * Returns the protected results for the specified replica. These are generated fetching the extra rows and merging
+ * them with the cached original filtered results for that replica.
+ *
+ * @param source the source
+ * @return the protected results for the specified replica
+ */
+ UnfilteredPartitionIterator queryProtectedPartitions(int source)
+ {
+ UnfilteredPartitionIterator original = makeIterator(originalPartitions.get(source));
+ SortedMap<DecoratedKey, BTreeSet.Builder<Clustering>> toFetch = rowsToFetch.get(source);
+
+ if (toFetch.isEmpty())
+ return original;
+
+ // TODO: this would be more efficient if we had multi-key queries internally
+ List<UnfilteredPartitionIterator> fetched = toFetch.keySet()
+ .stream()
+ .map(k -> querySourceOnKey(source, k))
+ .collect(Collectors.toList());
+
+ return UnfilteredPartitionIterators.merge(Arrays.asList(original, UnfilteredPartitionIterators.concat(fetched)), null);
+ }
+
+ private UnfilteredPartitionIterator querySourceOnKey(int i, DecoratedKey key)
+ {
+ BTreeSet.Builder<Clustering> builder = rowsToFetch.get(i).get(key);
+ assert builder != null; // We're calling this on the result of rowsToFetch.get(i).keySet()
+
+ Replica source = sources.get(i);
+ NavigableSet<Clustering> clusterings = builder.build();
+ tableMetrics.replicaSideFilteringProtectionRequests.mark();
+ if (logger.isTraceEnabled())
+ logger.trace("Requesting rows {} in partition {} from {} for replica-side filtering protection",
+ clusterings, key, source);
+ Tracing.trace("Requesting {} rows in partition {} from {} for replica-side filtering protection",
+ clusterings.size(), key, source);
+
+ // build the read command taking into account that we could be requesting only in the static row
+ DataLimits limits = clusterings.isEmpty() ? DataLimits.cqlLimits(1) : DataLimits.NONE;
+ ClusteringIndexFilter filter = new ClusteringIndexNamesFilter(clusterings, command.isReversed());
+ SinglePartitionReadCommand cmd = SinglePartitionReadCommand.create(command.metadata(),
+ command.nowInSec(),
+ command.columnFilter(),
+ RowFilter.NONE,
+ limits,
+ key,
+ filter);
+
+ ReplicaPlan.ForTokenRead replicaPlan = ReplicaPlans.forSingleReplicaRead(keyspace, key.getToken(), source);
+ ReplicaPlan.SharedForTokenRead sharedReplicaPlan = ReplicaPlan.shared(replicaPlan);
+ try
+ {
+ return executeReadCommand(cmd, source, sharedReplicaPlan);
+ }
+ catch (ReadTimeoutException e)
+ {
+ int blockFor = consistency.blockFor(keyspace);
+ throw new ReadTimeoutException(consistency, blockFor - 1, blockFor, true);
+ }
+ catch (UnavailableException e)
+ {
+ int blockFor = consistency.blockFor(keyspace);
+ throw UnavailableException.create(consistency, blockFor, blockFor - 1);
+ }
+ }
+
+ private <E extends Endpoints<E>, P extends ReplicaPlan.ForRead<E>>
+ UnfilteredPartitionIterator executeReadCommand(ReadCommand cmd, Replica source, ReplicaPlan.Shared<E, P> replicaPlan)
+ {
+ DataResolver<E, P> resolver = new DataResolver<>(cmd, replicaPlan, (NoopReadRepair<E, P>)NoopReadRepair.instance, queryStartNanoTime);
+ ReadCallback<E, P> handler = new ReadCallback<>(resolver, cmd, replicaPlan, queryStartNanoTime);
+
+ if (source.isSelf())
+ {
+ Stage.READ.maybeExecuteImmediately(new StorageProxy.LocalReadRunnable(cmd, handler));
+ }
+ else
+ {
+ if (source.isTransient())
+ cmd = cmd.copyAsTransientQuery(source);
+ MessagingService.instance().sendWithCallback(cmd.createMessage(false), source.endpoint(), handler);
+ }
+
+ // We don't call handler.get() because we want to preserve tombstones since we're still in the middle of merging node results.
+ handler.awaitResults();
+ assert resolver.getMessages().size() == 1;
+ return resolver.getMessages().get(0).payload.makeIterator(command);
+ }
+
+ /**
+ * Returns a merge listener that skips the merged rows for which any of the replicas doesn't have a version,
+ * pessimistically assuming that they are outdated. It is intended to be used during a first merge of per-replica
+ * query results to ensure we fetch enough results from the replicas to ensure we don't miss any potentially
+ * outdated result.
+ * <p>
+ * The listener will track both the accepted data and the primary keys of the rows that are considered as outdated.
+ * That way, once the query results would have been merged using this listener, further calls to
+ * {@link #queryProtectedPartitions(int)} will use the collected data to return a copy of the
+ * data originally collected from the specified replica, completed with the potentially outdated rows.
+ */
+ UnfilteredPartitionIterators.MergeListener mergeController()
+ {
+ return (partitionKey, versions) -> {
+
+ PartitionBuilder[] builders = new PartitionBuilder[sources.size()];
+
+ for (int i = 0; i < sources.size(); i++)
+ builders[i] = new PartitionBuilder(command, partitionKey, columns(versions), stats(versions));
+
+ return new UnfilteredRowIterators.MergeListener()
+ {
+ @Override
+ public void onMergedPartitionLevelDeletion(DeletionTime mergedDeletion, DeletionTime[] versions)
+ {
+ // cache the deletion time versions to be able to regenerate the original row iterator
+ for (int i = 0; i < versions.length; i++)
+ builders[i].setDeletionTime(versions[i]);
+ }
+
+ @Override
+ public Row onMergedRows(Row merged, Row[] versions)
+ {
+ // cache the row versions to be able to regenerate the original row iterator
+ for (int i = 0; i < versions.length; i++)
+ builders[i].addRow(versions[i]);
+
+ if (merged.isEmpty())
+ return merged;
+
+ boolean isPotentiallyOutdated = false;
+ boolean isStatic = merged.isStatic();
+ for (int i = 0; i < versions.length; i++)
+ {
+ Row version = versions[i];
+ if (version == null || (isStatic && version.isEmpty()))
+ {
+ isPotentiallyOutdated = true;
+ BTreeSet.Builder<Clustering> toFetch = getOrCreateToFetch(i, partitionKey);
+ // Note that for static, we shouldn't add the clustering to the clustering set (the
+ // ClusteringIndexNamesFilter we'll build from this later does not expect it), but the fact
+ // we created a builder in the first place will act as a marker that the static row must be
+ // fetched, even if no other rows are added for this partition.
+ if (!isStatic)
+ toFetch.add(merged.clustering());
+ }
+ }
+
+ // If the row is potentially outdated (because some replica didn't send anything and so it _may_ be
+ // an outdated result that is only present because other replica have filtered the up-to-date result
+ // out), then we skip the row. In other words, the results of the initial merging of results by this
+ // protection assume the worst case scenario where every row that might be outdated actually is.
+ // This ensures that during this first phase (collecting additional row to fetch) we are guaranteed
+ // to look at enough data to ultimately fulfill the query limit.
+ return isPotentiallyOutdated ? null : merged;
+ }
+
+ @Override
+ public void onMergedRangeTombstoneMarkers(RangeTombstoneMarker merged, RangeTombstoneMarker[] versions)
+ {
+ // cache the marker versions to be able to regenerate the original row iterator
+ for (int i = 0; i < versions.length; i++)
+ builders[i].addRangeTombstoneMarker(versions[i]);
+ }
+
+ @Override
+ public void close()
+ {
+ for (int i = 0; i < sources.size(); i++)
+ originalPartitions.get(i).add(builders[i]);
+ }
+ };
+ };
+ }
+
+ private static RegularAndStaticColumns columns(List<UnfilteredRowIterator> versions)
+ {
+ Columns statics = Columns.NONE;
+ Columns regulars = Columns.NONE;
+ for (UnfilteredRowIterator iter : versions)
+ {
+ if (iter == null)
+ continue;
+
+ RegularAndStaticColumns cols = iter.columns();
+ statics = statics.mergeTo(cols.statics);
+ regulars = regulars.mergeTo(cols.regulars);
+ }
+ return new RegularAndStaticColumns(statics, regulars);
+ }
+
+ private static EncodingStats stats(List<UnfilteredRowIterator> iterators)
+ {
+ EncodingStats stats = EncodingStats.NO_STATS;
+ for (UnfilteredRowIterator iter : iterators)
+ {
+ if (iter == null)
+ continue;
+
+ stats = stats.mergeWith(iter.stats());
+ }
+ return stats;
+ }
+
+ private UnfilteredPartitionIterator makeIterator(List<PartitionBuilder> builders)
+ {
+ return new UnfilteredPartitionIterator()
+ {
+ final Iterator<PartitionBuilder> iterator = builders.iterator();
+
+ @Override
+ public TableMetadata metadata()
+ {
+ return command.metadata();
+ }
+
+ @Override
+ public void close()
+ {
+ // nothing to do here
+ }
+
+ @Override
+ public boolean hasNext()
+ {
+ return iterator.hasNext();
+ }
+
+ @Override
+ public UnfilteredRowIterator next()
+ {
+ return iterator.next().build();
+ }
+ };
+ }
+
+ private static class PartitionBuilder
+ {
+ private final ReadCommand command;
+ private final DecoratedKey partitionKey;
+ private final RegularAndStaticColumns columns;
+ private final EncodingStats stats;
+
+ private DeletionTime deletionTime;
+ private Row staticRow = Rows.EMPTY_STATIC_ROW;
+ private final List<Unfiltered> contents = new ArrayList<>();
+
+ private PartitionBuilder(ReadCommand command,
+ DecoratedKey partitionKey,
+ RegularAndStaticColumns columns,
+ EncodingStats stats)
+ {
+ this.command = command;
+ this.partitionKey = partitionKey;
+ this.columns = columns;
+ this.stats = stats;
+ }
+
+ private void setDeletionTime(DeletionTime deletionTime)
+ {
+ this.deletionTime = deletionTime;
+ }
+
+ private void addRow(Row row)
+ {
+ if (row == null)
+ return;
+
+ if (row.isStatic())
+ staticRow = row;
+ else
+ contents.add(row);
+ }
+
+ private void addRangeTombstoneMarker(RangeTombstoneMarker marker)
+ {
+ if (marker != null)
+ contents.add(marker);
+ }
+
+ private UnfilteredRowIterator build()
+ {
+ return new UnfilteredRowIterator()
+ {
+ final Iterator<Unfiltered> iterator = contents.iterator();
+
+ @Override
+ public DeletionTime partitionLevelDeletion()
+ {
+ return deletionTime;
+ }
+
+ @Override
+ public EncodingStats stats()
+ {
+ return stats;
+ }
+
+ @Override
+ public TableMetadata metadata()
+ {
+ return command.metadata();
+ }
+
+ @Override
+ public boolean isReverseOrder()
+ {
+ return command.isReversed();
+ }
+
+ @Override
+ public RegularAndStaticColumns columns()
+ {
+ return columns;
+ }
+
+ @Override
+ public DecoratedKey partitionKey()
+ {
+ return partitionKey;
+ }
+
+ @Override
+ public Row staticRow()
+ {
+ return staticRow;
+ }
+
+ @Override
+ public void close()
+ {
+ // nothing to do here
+ }
+
+ @Override
+ public boolean hasNext()
+ {
+ return iterator.hasNext();
+ }
+
+ @Override
+ public Unfiltered next()
+ {
+ return iterator.next();
+ }
+ };
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/service/reads/ResponseResolver.java b/src/java/org/apache/cassandra/service/reads/ResponseResolver.java
new file mode 100644
index 0000000..6ae19ac
--- /dev/null
+++ b/src/java/org/apache/cassandra/service/reads/ResponseResolver.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.service.reads;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.db.ReadCommand;
+import org.apache.cassandra.db.ReadResponse;
+import org.apache.cassandra.locator.Endpoints;
+import org.apache.cassandra.locator.ReplicaPlan;
+import org.apache.cassandra.net.Message;
+import org.apache.cassandra.utils.concurrent.Accumulator;
+
+public abstract class ResponseResolver<E extends Endpoints<E>, P extends ReplicaPlan.ForRead<E>>
+{
+ protected static final Logger logger = LoggerFactory.getLogger(ResponseResolver.class);
+
+ protected final ReadCommand command;
+ protected final ReplicaPlan.Shared<E, P> replicaPlan;
+
+ // Accumulator gives us non-blocking thread-safety with optimal algorithmic constraints
+ protected final Accumulator<Message<ReadResponse>> responses;
+ protected final long queryStartNanoTime;
+
+ public ResponseResolver(ReadCommand command, ReplicaPlan.Shared<E, P> replicaPlan, long queryStartNanoTime)
+ {
+ this.command = command;
+ this.replicaPlan = replicaPlan;
+ this.responses = new Accumulator<>(replicaPlan.get().candidates().size());
+ this.queryStartNanoTime = queryStartNanoTime;
+ }
+
+ protected P replicaPlan()
+ {
+ return replicaPlan.get();
+ }
+
+ public abstract boolean isDataPresent();
+
+ public void preprocess(Message<ReadResponse> message)
+ {
+ if (replicaPlan().lookup(message.from()).isTransient() &&
+ message.payload.isDigestResponse())
+ throw new IllegalArgumentException("Digest response received from transient replica");
+
+ try
+ {
+ responses.add(message);
+ }
+ catch (IllegalStateException e)
+ {
+ logger.error("Encountered error while trying to preprocess the message {}, in command {}, replica plan: {}",
+ message, command, replicaPlan);
+ throw e;
+ }
+ }
+
+ public Accumulator<Message<ReadResponse>> getMessages()
+ {
+ return responses;
+ }
+}
diff --git a/src/java/org/apache/cassandra/service/reads/ShortReadPartitionsProtection.java b/src/java/org/apache/cassandra/service/reads/ShortReadPartitionsProtection.java
new file mode 100644
index 0000000..59edc5a
--- /dev/null
+++ b/src/java/org/apache/cassandra/service/reads/ShortReadPartitionsProtection.java
@@ -0,0 +1,199 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.reads;
+
+import org.apache.cassandra.locator.Endpoints;
+import org.apache.cassandra.locator.ReplicaPlan;
+import org.apache.cassandra.locator.ReplicaPlans;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.concurrent.Stage;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DataRange;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.db.PartitionRangeReadCommand;
+import org.apache.cassandra.db.ReadCommand;
+import org.apache.cassandra.db.filter.DataLimits;
+import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.db.transform.MorePartitions;
+import org.apache.cassandra.db.transform.MoreRows;
+import org.apache.cassandra.db.transform.Transformation;
+import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.dht.ExcludingBounds;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.locator.Replica;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.service.reads.repair.NoopReadRepair;
+import org.apache.cassandra.service.StorageProxy;
+import org.apache.cassandra.tracing.Tracing;
+
+public class ShortReadPartitionsProtection extends Transformation<UnfilteredRowIterator> implements MorePartitions<UnfilteredPartitionIterator>
+{
+ private static final Logger logger = LoggerFactory.getLogger(ShortReadPartitionsProtection.class);
+ private final ReadCommand command;
+ private final Replica source;
+
+ private final DataLimits.Counter singleResultCounter; // unmerged per-source counter
+ private final DataLimits.Counter mergedResultCounter; // merged end-result counter
+
+ private DecoratedKey lastPartitionKey; // key of the last observed partition
+
+ private boolean partitionsFetched; // whether we've seen any new partitions since iteration start or last moreContents() call
+
+ private final long queryStartNanoTime;
+
+ public ShortReadPartitionsProtection(ReadCommand command, Replica source,
+ DataLimits.Counter singleResultCounter,
+ DataLimits.Counter mergedResultCounter,
+ long queryStartNanoTime)
+ {
+ this.command = command;
+ this.source = source;
+ this.singleResultCounter = singleResultCounter;
+ this.mergedResultCounter = mergedResultCounter;
+ this.queryStartNanoTime = queryStartNanoTime;
+ }
+
+ @Override
+ public UnfilteredRowIterator applyToPartition(UnfilteredRowIterator partition)
+ {
+ partitionsFetched = true;
+
+ lastPartitionKey = partition.partitionKey();
+
+ /*
+ * Extend for moreContents() then apply protection to track lastClustering by applyToRow().
+ *
+ * If we don't apply the transformation *after* extending the partition with MoreRows,
+ * applyToRow() method of protection will not be called on the first row of the new extension iterator.
+ */
+ ReplicaPlan.ForTokenRead replicaPlan = ReplicaPlans.forSingleReplicaRead(Keyspace.open(command.metadata().keyspace), partition.partitionKey().getToken(), source);
+ ReplicaPlan.SharedForTokenRead sharedReplicaPlan = ReplicaPlan.shared(replicaPlan);
+ ShortReadRowsProtection protection = new ShortReadRowsProtection(partition.partitionKey(),
+ command, source,
+ (cmd) -> executeReadCommand(cmd, sharedReplicaPlan),
+ singleResultCounter,
+ mergedResultCounter);
+ return Transformation.apply(MoreRows.extend(partition, protection), protection);
+ }
+
+ /*
+ * We only get here once all the rows and partitions in this iterator have been iterated over, and so
+ * if the node had returned the requested number of rows but we still get here, then some results were
+ * skipped during reconciliation.
+ */
+ public UnfilteredPartitionIterator moreContents()
+ {
+ // never try to request additional partitions from replicas if our reconciled partitions are already filled to the limit
+ assert !mergedResultCounter.isDone();
+
+ // we do not apply short read protection when we have no limits at all
+ assert !command.limits().isUnlimited();
+
+ /*
+ * If this is a single partition read command or an (indexed) partition range read command with
+ * a partition key specified, then we can't and shouldn't try fetch more partitions.
+ */
+ assert !command.isLimitedToOnePartition();
+
+ /*
+ * If the returned result doesn't have enough rows/partitions to satisfy even the original limit, don't ask for more.
+ *
+ * Can only take the short cut if there is no per partition limit set. Otherwise it's possible to hit false
+ * positives due to some rows being uncounted for in certain scenarios (see CASSANDRA-13911).
+ */
+ if (!singleResultCounter.isDone() && command.limits().perPartitionCount() == DataLimits.NO_LIMIT)
+ return null;
+
+ /*
+ * Either we had an empty iterator as the initial response, or our moreContents() call got us an empty iterator.
+ * There is no point to ask the replica for more rows - it has no more in the requested range.
+ */
+ if (!partitionsFetched)
+ return null;
+ partitionsFetched = false;
+
+ /*
+ * We are going to fetch one partition at a time for thrift and potentially more for CQL.
+ * The row limit will either be set to the per partition limit - if the command has no total row limit set, or
+ * the total # of rows remaining - if it has some. If we don't grab enough rows in some of the partitions,
+ * then future ShortReadRowsProtection.moreContents() calls will fetch the missing ones.
+ */
+ int toQuery = command.limits().count() != DataLimits.NO_LIMIT
+ ? command.limits().count() - counted(mergedResultCounter)
+ : command.limits().perPartitionCount();
+
+ ColumnFamilyStore.metricsFor(command.metadata().id).shortReadProtectionRequests.mark();
+ Tracing.trace("Requesting {} extra rows from {} for short read protection", toQuery, source);
+ logger.info("Requesting {} extra rows from {} for short read protection", toQuery, source);
+
+ return makeAndExecuteFetchAdditionalPartitionReadCommand(toQuery);
+ }
+
+ // Counts the number of rows for regular queries and the number of groups for GROUP BY queries
+ private int counted(DataLimits.Counter counter)
+ {
+ return command.limits().isGroupByLimit()
+ ? counter.rowCounted()
+ : counter.counted();
+ }
+
+ private UnfilteredPartitionIterator makeAndExecuteFetchAdditionalPartitionReadCommand(int toQuery)
+ {
+ PartitionRangeReadCommand cmd = (PartitionRangeReadCommand) command;
+
+ DataLimits newLimits = cmd.limits().forShortReadRetry(toQuery);
+
+ AbstractBounds<PartitionPosition> bounds = cmd.dataRange().keyRange();
+ AbstractBounds<PartitionPosition> newBounds = bounds.inclusiveRight()
+ ? new Range<>(lastPartitionKey, bounds.right)
+ : new ExcludingBounds<>(lastPartitionKey, bounds.right);
+ DataRange newDataRange = cmd.dataRange().forSubRange(newBounds);
+
+ ReplicaPlan.ForRangeRead replicaPlan = ReplicaPlans.forSingleReplicaRead(Keyspace.open(command.metadata().keyspace), cmd.dataRange().keyRange(), source, 1);
+ return executeReadCommand(cmd.withUpdatedLimitsAndDataRange(newLimits, newDataRange), ReplicaPlan.shared(replicaPlan));
+ }
+
+ private <E extends Endpoints<E>, P extends ReplicaPlan.ForRead<E>>
+ UnfilteredPartitionIterator executeReadCommand(ReadCommand cmd, ReplicaPlan.Shared<E, P> replicaPlan)
+ {
+ DataResolver<E, P> resolver = new DataResolver<>(cmd, replicaPlan, (NoopReadRepair<E, P>)NoopReadRepair.instance, queryStartNanoTime);
+ ReadCallback<E, P> handler = new ReadCallback<>(resolver, cmd, replicaPlan, queryStartNanoTime);
+
+ if (source.isSelf())
+ {
+ Stage.READ.maybeExecuteImmediately(new StorageProxy.LocalReadRunnable(cmd, handler));
+ }
+ else
+ {
+ if (source.isTransient())
+ cmd = cmd.copyAsTransientQuery(source);
+ MessagingService.instance().sendWithCallback(cmd.createMessage(false), source.endpoint(), handler);
+ }
+
+ // We don't call handler.get() because we want to preserve tombstones since we're still in the middle of merging node results.
+ handler.awaitResults();
+ assert resolver.getMessages().size() == 1;
+ return resolver.getMessages().get(0).payload.makeIterator(command);
+ }
+}
diff --git a/src/java/org/apache/cassandra/service/reads/ShortReadProtection.java b/src/java/org/apache/cassandra/service/reads/ShortReadProtection.java
new file mode 100644
index 0000000..1a454f9
--- /dev/null
+++ b/src/java/org/apache/cassandra/service/reads/ShortReadProtection.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.reads;
+
+import java.net.InetAddress;
+
+
+import org.apache.cassandra.db.ReadCommand;
+import org.apache.cassandra.db.filter.DataLimits;
+import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
+import org.apache.cassandra.db.transform.MorePartitions;
+import org.apache.cassandra.db.transform.Transformation;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.Replica;
+
+/**
+ * We have a potential short read if the result from a given node contains the requested number of rows
+ * (i.e. it has stopped returning results due to the limit), but some of them haven't
+ * made it into the final post-reconciliation result due to other nodes' row, range, and/or partition tombstones.
+ *
+ * If that is the case, then that node may have more rows that we should fetch, as otherwise we could
+ * ultimately return fewer rows than required. Also, those additional rows may contain tombstones which
+ * which we also need to fetch as they may shadow rows or partitions from other replicas' results, which we would
+ * otherwise return incorrectly.
+ */
+public class ShortReadProtection
+{
+ @SuppressWarnings("resource")
+ public static UnfilteredPartitionIterator extend(Replica source, UnfilteredPartitionIterator partitions,
+ ReadCommand command, DataLimits.Counter mergedResultCounter,
+ long queryStartNanoTime, boolean enforceStrictLiveness)
+ {
+ DataLimits.Counter singleResultCounter = command.limits().newCounter(command.nowInSec(),
+ false,
+ command.selectsFullPartition(),
+ enforceStrictLiveness).onlyCount();
+
+ ShortReadPartitionsProtection protection = new ShortReadPartitionsProtection(command, source, singleResultCounter, mergedResultCounter, queryStartNanoTime);
+
+ /*
+ * The order of extention and transformations is important here. Extending with more partitions has to happen
+ * first due to the way BaseIterator.hasMoreContents() works: only transformations applied after extension will
+ * be called on the first partition of the extended iterator.
+ *
+ * Additionally, we want singleResultCounter to be applied after SRPP, so that its applyToPartition() method will
+ * be called last, after the extension done by SRRP.applyToPartition() call. That way we preserve the same order
+ * when it comes to calling SRRP.moreContents() and applyToRow() callbacks.
+ *
+ * See ShortReadPartitionsProtection.applyToPartition() for more details.
+ */
+
+ // extend with moreContents() only if it's a range read command with no partition key specified
+ if (!command.isLimitedToOnePartition())
+ partitions = MorePartitions.extend(partitions, protection); // register SRPP.moreContents()
+
+ partitions = Transformation.apply(partitions, protection); // register SRPP.applyToPartition()
+ partitions = Transformation.apply(partitions, singleResultCounter); // register the per-source counter
+
+ return partitions;
+ }
+}
diff --git a/src/java/org/apache/cassandra/service/reads/ShortReadRowsProtection.java b/src/java/org/apache/cassandra/service/reads/ShortReadRowsProtection.java
new file mode 100644
index 0000000..8dc7fc7
--- /dev/null
+++ b/src/java/org/apache/cassandra/service/reads/ShortReadRowsProtection.java
@@ -0,0 +1,197 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.reads;
+
+import java.util.function.Function;
+
+import org.apache.cassandra.db.Clustering;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.ReadCommand;
+import org.apache.cassandra.db.SinglePartitionReadCommand;
+import org.apache.cassandra.db.filter.ClusteringIndexFilter;
+import org.apache.cassandra.db.filter.DataLimits;
+import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
+import org.apache.cassandra.db.partitions.UnfilteredPartitionIterators;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.db.transform.MoreRows;
+import org.apache.cassandra.db.transform.Transformation;
+import org.apache.cassandra.locator.Replica;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.tracing.Tracing;
+
+class ShortReadRowsProtection extends Transformation implements MoreRows<UnfilteredRowIterator>
+{
+ private final ReadCommand command;
+ private final Replica source;
+ private final DataLimits.Counter singleResultCounter; // unmerged per-source counter
+ private final DataLimits.Counter mergedResultCounter; // merged end-result counter
+ private final Function<ReadCommand, UnfilteredPartitionIterator> commandExecutor;
+ private final TableMetadata metadata;
+ private final DecoratedKey partitionKey;
+
+ private Clustering lastClustering; // clustering of the last observed row
+
+ private int lastCounted = 0; // last seen recorded # before attempting to fetch more rows
+ private int lastFetched = 0; // # rows returned by last attempt to get more (or by the original read command)
+ private int lastQueried = 0; // # extra rows requested from the replica last time
+
+ ShortReadRowsProtection(DecoratedKey partitionKey, ReadCommand command, Replica source,
+ Function<ReadCommand, UnfilteredPartitionIterator> commandExecutor,
+ DataLimits.Counter singleResultCounter, DataLimits.Counter mergedResultCounter)
+ {
+ this.command = command;
+ this.source = source;
+ this.commandExecutor = commandExecutor;
+ this.singleResultCounter = singleResultCounter;
+ this.mergedResultCounter = mergedResultCounter;
+ this.metadata = command.metadata();
+ this.partitionKey = partitionKey;
+ }
+
+ @Override
+ public Row applyToRow(Row row)
+ {
+ lastClustering = row.clustering();
+ return row;
+ }
+
+ /*
+ * We only get here once all the rows in this iterator have been iterated over, and so if the node
+ * had returned the requested number of rows but we still get here, then some results were skipped
+ * during reconciliation.
+ */
+ public UnfilteredRowIterator moreContents()
+ {
+ // never try to request additional rows from replicas if our reconciled partition is already filled to the limit
+ assert !mergedResultCounter.isDoneForPartition();
+
+ // we do not apply short read protection when we have no limits at all
+ assert !command.limits().isUnlimited();
+
+ /*
+ * If the returned partition doesn't have enough rows to satisfy even the original limit, don't ask for more.
+ *
+ * Can only take the short cut if there is no per partition limit set. Otherwise it's possible to hit false
+ * positives due to some rows being uncounted for in certain scenarios (see CASSANDRA-13911).
+ */
+ if (!singleResultCounter.isDoneForPartition() && command.limits().perPartitionCount() == DataLimits.NO_LIMIT)
+ return null;
+
+ /*
+ * If the replica has no live rows in the partition, don't try to fetch more.
+ *
+ * Note that the previous branch [if (!singleResultCounter.isDoneForPartition()) return null] doesn't
+ * always cover this scenario:
+ * isDoneForPartition() is defined as [isDone() || rowInCurrentPartition >= perPartitionLimit],
+ * and will return true if isDone() returns true, even if there are 0 rows counted in the current partition.
+ *
+ * This can happen with a range read if after 1+ rounds of short read protection requests we managed to fetch
+ * enough extra rows for other partitions to satisfy the singleResultCounter's total row limit, but only
+ * have tombstones in the current partition.
+ *
+ * One other way we can hit this condition is when the partition only has a live static row and no regular
+ * rows. In that scenario the counter will remain at 0 until the partition is closed - which happens after
+ * the moreContents() call.
+ */
+ if (countedInCurrentPartition(singleResultCounter) == 0)
+ return null;
+
+ /*
+ * This is a table with no clustering columns, and has at most one row per partition - with EMPTY clustering.
+ * We already have the row, so there is no point in asking for more from the partition.
+ */
+ if (Clustering.EMPTY == lastClustering)
+ return null;
+
+ lastFetched = countedInCurrentPartition(singleResultCounter) - lastCounted;
+ lastCounted = countedInCurrentPartition(singleResultCounter);
+
+ // getting back fewer rows than we asked for means the partition on the replica has been fully consumed
+ if (lastQueried > 0 && lastFetched < lastQueried)
+ return null;
+
+ /*
+ * At this point we know that:
+ * 1. the replica returned [repeatedly?] as many rows as we asked for and potentially has more
+ * rows in the partition
+ * 2. at least one of those returned rows was shadowed by a tombstone returned from another
+ * replica
+ * 3. we haven't satisfied the client's limits yet, and should attempt to query for more rows to
+ * avoid a short read
+ *
+ * In the ideal scenario, we would get exactly min(a, b) or fewer rows from the next request, where a and b
+ * are defined as follows:
+ * [a] limits.count() - mergedResultCounter.counted()
+ * [b] limits.perPartitionCount() - mergedResultCounter.countedInCurrentPartition()
+ *
+ * It would be naive to query for exactly that many rows, as it's possible and not unlikely
+ * that some of the returned rows would also be shadowed by tombstones from other hosts.
+ *
+ * Note: we don't know, nor do we care, how many rows from the replica made it into the reconciled result;
+ * we can only tell how many in total we queried for, and that [0, mrc.countedInCurrentPartition()) made it.
+ *
+ * In general, our goal should be to minimise the number of extra requests - *not* to minimise the number
+ * of rows fetched: there is a high transactional cost for every individual request, but a relatively low
+ * marginal cost for each extra row requested.
+ *
+ * As such it's better to overfetch than to underfetch extra rows from a host; but at the same
+ * time we want to respect paging limits and not blow up spectacularly.
+ *
+ * Note: it's ok to retrieve more rows that necessary since singleResultCounter is not stopping and only
+ * counts.
+ *
+ * With that in mind, we'll just request the minimum of (count(), perPartitionCount()) limits.
+ *
+ * See CASSANDRA-13794 for more details.
+ */
+ lastQueried = Math.min(command.limits().count(), command.limits().perPartitionCount());
+
+ ColumnFamilyStore.metricsFor(metadata.id).shortReadProtectionRequests.mark();
+ Tracing.trace("Requesting {} extra rows from {} for short read protection", lastQueried, source);
+
+ SinglePartitionReadCommand cmd = makeFetchAdditionalRowsReadCommand(lastQueried);
+ return UnfilteredPartitionIterators.getOnlyElement(commandExecutor.apply(cmd), cmd);
+ }
+
+ // Counts the number of rows for regular queries and the number of groups for GROUP BY queries
+ private int countedInCurrentPartition(DataLimits.Counter counter)
+ {
+ return command.limits().isGroupByLimit()
+ ? counter.rowCountedInCurrentPartition()
+ : counter.countedInCurrentPartition();
+ }
+
+ private SinglePartitionReadCommand makeFetchAdditionalRowsReadCommand(int toQuery)
+ {
+ ClusteringIndexFilter filter = command.clusteringIndexFilter(partitionKey);
+ if (null != lastClustering)
+ filter = filter.forPaging(metadata.comparator, lastClustering, false);
+
+ return SinglePartitionReadCommand.create(command.metadata(),
+ command.nowInSec(),
+ command.columnFilter(),
+ command.rowFilter(),
+ command.limits().forShortReadRetry(toQuery),
+ partitionKey,
+ filter,
+ command.indexMetadata());
+ }
+}
diff --git a/src/java/org/apache/cassandra/service/reads/SpeculativeRetryPolicy.java b/src/java/org/apache/cassandra/service/reads/SpeculativeRetryPolicy.java
new file mode 100644
index 0000000..e09ff51
--- /dev/null
+++ b/src/java/org/apache/cassandra/service/reads/SpeculativeRetryPolicy.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.service.reads;
+
+import com.codahale.metrics.Snapshot;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.schema.TableParams;
+
+public interface SpeculativeRetryPolicy
+{
+ public enum Kind
+ {
+ NEVER, FIXED, PERCENTILE, HYBRID, ALWAYS
+ }
+
+ long calculateThreshold(Snapshot latency, long existingValue);
+
+ Kind kind();
+
+ public static SpeculativeRetryPolicy fromString(String str)
+ {
+ if (AlwaysSpeculativeRetryPolicy.stringMatches(str))
+ return AlwaysSpeculativeRetryPolicy.INSTANCE;
+
+ if (NeverSpeculativeRetryPolicy.stringMatches(str))
+ return NeverSpeculativeRetryPolicy.INSTANCE;
+
+ if (PercentileSpeculativeRetryPolicy.stringMatches(str))
+ return PercentileSpeculativeRetryPolicy.fromString(str);
+
+ if (FixedSpeculativeRetryPolicy.stringMatches(str))
+ return FixedSpeculativeRetryPolicy.fromString(str);
+
+ if (HybridSpeculativeRetryPolicy.stringMatches(str))
+ return HybridSpeculativeRetryPolicy.fromString(str);
+
+ throw new ConfigurationException(String.format("Invalid value %s for option '%s'", str, TableParams.Option.SPECULATIVE_RETRY));
+ }
+}
diff --git a/src/java/org/apache/cassandra/service/reads/repair/AbstractReadRepair.java b/src/java/org/apache/cassandra/service/reads/repair/AbstractReadRepair.java
new file mode 100644
index 0000000..1b08877
--- /dev/null
+++ b/src/java/org/apache/cassandra/service/reads/repair/AbstractReadRepair.java
@@ -0,0 +1,182 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.reads.repair;
+
+import java.util.function.Consumer;
+
+import com.google.common.base.Preconditions;
+
+import com.codahale.metrics.Meter;
+import com.google.common.base.Predicates;
+
+import org.apache.cassandra.concurrent.Stage;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.ConsistencyLevel;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.ReadCommand;
+import org.apache.cassandra.db.SinglePartitionReadCommand;
+import org.apache.cassandra.db.partitions.PartitionIterator;
+import org.apache.cassandra.exceptions.ReadTimeoutException;
+import org.apache.cassandra.locator.Endpoints;
+import org.apache.cassandra.locator.Replica;
+import org.apache.cassandra.locator.ReplicaPlan;
+import org.apache.cassandra.metrics.ReadRepairMetrics;
+import org.apache.cassandra.net.Message;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.service.StorageProxy;
+import org.apache.cassandra.service.reads.DataResolver;
+import org.apache.cassandra.service.reads.DigestResolver;
+import org.apache.cassandra.service.reads.ReadCallback;
+import org.apache.cassandra.tracing.Tracing;
+
+import static java.util.concurrent.TimeUnit.NANOSECONDS;
+
+public abstract class AbstractReadRepair<E extends Endpoints<E>, P extends ReplicaPlan.ForRead<E>>
+ implements ReadRepair<E, P>
+{
+ protected final ReadCommand command;
+ protected final long queryStartNanoTime;
+ protected final ReplicaPlan.Shared<E, P> replicaPlan;
+ protected final ColumnFamilyStore cfs;
+
+ private volatile DigestRepair digestRepair = null;
+
+ private static class DigestRepair
+ {
+ private final DataResolver dataResolver;
+ private final ReadCallback readCallback;
+ private final Consumer<PartitionIterator> resultConsumer;
+
+ public DigestRepair(DataResolver dataResolver, ReadCallback readCallback, Consumer<PartitionIterator> resultConsumer)
+ {
+ this.dataResolver = dataResolver;
+ this.readCallback = readCallback;
+ this.resultConsumer = resultConsumer;
+ }
+ }
+
+ public AbstractReadRepair(ReadCommand command,
+ ReplicaPlan.Shared<E, P> replicaPlan,
+ long queryStartNanoTime)
+ {
+ this.command = command;
+ this.queryStartNanoTime = queryStartNanoTime;
+ this.replicaPlan = replicaPlan;
+ this.cfs = Keyspace.openAndGetStore(command.metadata());
+ }
+
+ protected P replicaPlan()
+ {
+ return replicaPlan.get();
+ }
+
+ void sendReadCommand(Replica to, ReadCallback readCallback, boolean speculative)
+ {
+ ReadCommand command = this.command;
+
+ if (to.isSelf())
+ {
+ Stage.READ.maybeExecuteImmediately(new StorageProxy.LocalReadRunnable(command, readCallback));
+ return;
+ }
+
+ if (to.isTransient())
+ {
+ // It's OK to send queries to transient nodes during RR, as we may have contacted them for their data request initially
+ // So long as we don't use these to generate repair mutations, we're fine, and this is enforced by requiring
+ // ReadOnlyReadRepair for transient keyspaces.
+ command = command.copyAsTransientQuery(to);
+ }
+
+ if (Tracing.isTracing())
+ {
+ String type;
+ if (speculative) type = to.isFull() ? "speculative full" : "speculative transient";
+ else type = to.isFull() ? "full" : "transient";
+ Tracing.trace("Enqueuing {} data read to {}", type, to);
+ }
+ // if enabled, request additional info about repaired data from any full replicas
+ Message<ReadCommand> message = command.createMessage(command.isTrackingRepairedStatus() && to.isFull());
+ MessagingService.instance().sendWithCallback(message, to.endpoint(), readCallback);
+ }
+
+ abstract Meter getRepairMeter();
+
+ // digestResolver isn't used here because we resend read requests to all participants
+ public void startRepair(DigestResolver<E, P> digestResolver, Consumer<PartitionIterator> resultConsumer)
+ {
+ getRepairMeter().mark();
+
+ // Do a full data read to resolve the correct response (and repair node that need be)
+ DataResolver<E, P> resolver = new DataResolver<>(command, replicaPlan, this, queryStartNanoTime);
+ ReadCallback<E, P> readCallback = new ReadCallback<>(resolver, command, replicaPlan, queryStartNanoTime);
+
+ digestRepair = new DigestRepair(resolver, readCallback, resultConsumer);
+
+ // if enabled, request additional info about repaired data from any full replicas
+ if (DatabaseDescriptor.getRepairedDataTrackingForPartitionReadsEnabled())
+ command.trackRepairedStatus();
+
+ for (Replica replica : replicaPlan().contacts())
+ sendReadCommand(replica, readCallback, false);
+
+ ReadRepairDiagnostics.startRepair(this, replicaPlan(), digestResolver);
+ }
+
+ public void awaitReads() throws ReadTimeoutException
+ {
+ DigestRepair repair = digestRepair;
+ if (repair == null)
+ return;
+
+ repair.readCallback.awaitResults();
+ repair.resultConsumer.accept(digestRepair.dataResolver.resolve());
+ }
+
+ private boolean shouldSpeculate()
+ {
+ ConsistencyLevel consistency = replicaPlan().consistencyLevel();
+ ConsistencyLevel speculativeCL = consistency.isDatacenterLocal() ? ConsistencyLevel.LOCAL_QUORUM : ConsistencyLevel.QUORUM;
+ return consistency != ConsistencyLevel.EACH_QUORUM
+ && consistency.satisfies(speculativeCL, cfs.keyspace)
+ && cfs.sampleReadLatencyNanos <= command.getTimeout(NANOSECONDS);
+ }
+
+ public void maybeSendAdditionalReads()
+ {
+ Preconditions.checkState(command instanceof SinglePartitionReadCommand,
+ "maybeSendAdditionalReads can only be called for SinglePartitionReadCommand");
+ DigestRepair repair = digestRepair;
+ if (repair == null)
+ return;
+
+ if (shouldSpeculate() && !repair.readCallback.await(cfs.sampleReadLatencyNanos, NANOSECONDS))
+ {
+ Replica uncontacted = replicaPlan().firstUncontactedCandidate(Predicates.alwaysTrue());
+ if (uncontacted == null)
+ return;
+
+ replicaPlan.addToContacts(uncontacted);
+ sendReadCommand(uncontacted, repair.readCallback, true);
+ ReadRepairMetrics.speculatedRead.mark();
+ ReadRepairDiagnostics.speculatedRead(this, uncontacted.endpoint(), replicaPlan());
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/service/reads/repair/BlockingPartitionRepair.java b/src/java/org/apache/cassandra/service/reads/repair/BlockingPartitionRepair.java
new file mode 100644
index 0000000..edcf14d
--- /dev/null
+++ b/src/java/org/apache/cassandra/service/reads/repair/BlockingPartitionRepair.java
@@ -0,0 +1,261 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.reads.repair;
+
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Predicate;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicates;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.common.util.concurrent.AbstractFuture;
+
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.ConsistencyLevel;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.Mutation;
+import org.apache.cassandra.db.partitions.PartitionUpdate;
+import org.apache.cassandra.locator.Endpoints;
+import org.apache.cassandra.locator.EndpointsForToken;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.Replica;
+import org.apache.cassandra.locator.ReplicaPlan;
+import org.apache.cassandra.locator.Replicas;
+import org.apache.cassandra.locator.InOurDcTester;
+import org.apache.cassandra.metrics.ReadRepairMetrics;
+import org.apache.cassandra.net.RequestCallback;
+import org.apache.cassandra.net.Message;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.tracing.Tracing;
+
+import static org.apache.cassandra.net.Verb.*;
+
+public class BlockingPartitionRepair
+ extends AbstractFuture<Object> implements RequestCallback<Object>
+{
+ private final DecoratedKey key;
+ private final ReplicaPlan.ForTokenWrite writePlan;
+ private final Map<Replica, Mutation> pendingRepairs;
+ private final CountDownLatch latch;
+ private final Predicate<InetAddressAndPort> shouldBlockOn;
+
+ private volatile long mutationsSentTime;
+
+ public BlockingPartitionRepair(DecoratedKey key, Map<Replica, Mutation> repairs, ReplicaPlan.ForTokenWrite writePlan)
+ {
+ this(key, repairs, writePlan,
+ writePlan.consistencyLevel().isDatacenterLocal() ? InOurDcTester.endpoints() : Predicates.alwaysTrue());
+ }
+ public BlockingPartitionRepair(DecoratedKey key, Map<Replica, Mutation> repairs, ReplicaPlan.ForTokenWrite writePlan, Predicate<InetAddressAndPort> shouldBlockOn)
+ {
+ this.key = key;
+ this.pendingRepairs = new ConcurrentHashMap<>(repairs);
+ this.writePlan = writePlan;
+ this.shouldBlockOn = shouldBlockOn;
+
+ int blockFor = writePlan.blockFor();
+ // here we remove empty repair mutations from the block for total, since
+ // we're not sending them mutations
+ for (Replica participant : writePlan.contacts())
+ {
+ // remote dcs can sometimes get involved in dc-local reads. We want to repair
+ // them if they do, but they shouldn't interfere with blocking the client read.
+ if (!repairs.containsKey(participant) && shouldBlockOn.test(participant.endpoint()))
+ blockFor--;
+ }
+
+ // there are some cases where logically identical data can return different digests
+ // For read repair, this would result in ReadRepairHandler being called with a map of
+ // empty mutations. If we'd also speculated on either of the read stages, the number
+ // of empty mutations would be greater than blockFor, causing the latch ctor to throw
+ // an illegal argument exception due to a negative start value. So here we clamp it 0
+ latch = new CountDownLatch(Math.max(blockFor, 0));
+ }
+
+ int blockFor()
+ {
+ return writePlan.blockFor();
+ }
+
+ @VisibleForTesting
+ int waitingOn()
+ {
+ return (int) latch.getCount();
+ }
+
+ @VisibleForTesting
+ void ack(InetAddressAndPort from)
+ {
+ if (shouldBlockOn.test(from))
+ {
+ pendingRepairs.remove(writePlan.lookup(from));
+ latch.countDown();
+ }
+ }
+
+ @Override
+ public void onResponse(Message<Object> msg)
+ {
+ ack(msg.from());
+ }
+
+ private static PartitionUpdate extractUpdate(Mutation mutation)
+ {
+ return Iterables.getOnlyElement(mutation.getPartitionUpdates());
+ }
+
+ /**
+ * Combine the contents of any unacked repair into a single update
+ */
+ private PartitionUpdate mergeUnackedUpdates()
+ {
+ // recombinate the updates
+ List<PartitionUpdate> updates = Lists.newArrayList(Iterables.transform(pendingRepairs.values(), BlockingPartitionRepair::extractUpdate));
+ return updates.isEmpty() ? null : PartitionUpdate.merge(updates);
+ }
+
+ @VisibleForTesting
+ protected void sendRR(Message<Mutation> message, InetAddressAndPort endpoint)
+ {
+ MessagingService.instance().sendWithCallback(message, endpoint, this);
+ }
+
+ public void sendInitialRepairs()
+ {
+ mutationsSentTime = System.nanoTime();
+ Replicas.assertFull(pendingRepairs.keySet());
+
+ for (Map.Entry<Replica, Mutation> entry: pendingRepairs.entrySet())
+ {
+ Replica destination = entry.getKey();
+ Preconditions.checkArgument(destination.isFull(), "Can't send repairs to transient replicas: %s", destination);
+ Mutation mutation = entry.getValue();
+ TableId tableId = extractUpdate(mutation).metadata().id;
+
+ Tracing.trace("Sending read-repair-mutation to {}", destination);
+ // use a separate verb here to avoid writing hints on timeouts
+ sendRR(Message.out(READ_REPAIR_REQ, mutation), destination.endpoint());
+ ColumnFamilyStore.metricsFor(tableId).readRepairRequests.mark();
+
+ if (!shouldBlockOn.test(destination.endpoint()))
+ pendingRepairs.remove(destination);
+ ReadRepairDiagnostics.sendInitialRepair(this, destination.endpoint(), mutation);
+ }
+ }
+
+ /**
+ * Wait for the repair to complete util a future time
+ * If the {@param timeoutAt} is a past time, the method returns immediately with the repair result.
+ * @param timeoutAt, future time
+ * @param timeUnit, the time unit of the future time
+ * @return true if repair is done; otherwise, false.
+ */
+ public boolean awaitRepairsUntil(long timeoutAt, TimeUnit timeUnit)
+ {
+ long timeoutAtNanos = timeUnit.toNanos(timeoutAt);
+ long remaining = timeoutAtNanos - System.nanoTime();
+ try
+ {
+ return latch.await(remaining, TimeUnit.NANOSECONDS);
+ }
+ catch (InterruptedException e)
+ {
+ throw new AssertionError(e);
+ }
+ }
+
+ private static int msgVersionIdx(int version)
+ {
+ return version - MessagingService.minimum_version;
+ }
+
+ /**
+ * If it looks like we might not receive acks for all the repair mutations we sent out, combine all
+ * the unacked mutations and send them to the minority of nodes not involved in the read repair data
+ * read / write cycle. We will accept acks from them in lieu of acks from the initial mutations sent
+ * out, so long as we receive the same number of acks as repair mutations transmitted. This prevents
+ * misbehaving nodes from killing a quorum read, while continuing to guarantee monotonic quorum reads
+ */
+ public void maybeSendAdditionalWrites(long timeout, TimeUnit timeoutUnit)
+ {
+ if (awaitRepairsUntil(timeout + timeoutUnit.convert(mutationsSentTime, TimeUnit.NANOSECONDS), timeoutUnit))
+ return;
+
+ EndpointsForToken newCandidates = writePlan.liveUncontacted();
+ if (newCandidates.isEmpty())
+ return;
+
+ PartitionUpdate update = mergeUnackedUpdates();
+ if (update == null)
+ // final response was received between speculate
+ // timeout and call to get unacked mutation.
+ return;
+
+ ReadRepairMetrics.speculatedWrite.mark();
+
+ Mutation[] versionedMutations = new Mutation[msgVersionIdx(MessagingService.current_version) + 1];
+
+ for (Replica replica : newCandidates)
+ {
+ int versionIdx = msgVersionIdx(MessagingService.instance().versions.get(replica.endpoint()));
+
+ Mutation mutation = versionedMutations[versionIdx];
+
+ if (mutation == null)
+ {
+ mutation = BlockingReadRepairs.createRepairMutation(update, writePlan.consistencyLevel(), replica.endpoint(), true);
+ versionedMutations[versionIdx] = mutation;
+ }
+
+ if (mutation == null)
+ {
+ // the mutation is too large to send.
+ ReadRepairDiagnostics.speculatedWriteOversized(this, replica.endpoint());
+ continue;
+ }
+
+ Tracing.trace("Sending speculative read-repair-mutation to {}", replica);
+ sendRR(Message.out(READ_REPAIR_REQ, mutation), replica.endpoint());
+ ReadRepairDiagnostics.speculatedWrite(this, replica.endpoint(), mutation);
+ }
+ }
+
+ Keyspace getKeyspace()
+ {
+ return writePlan.keyspace();
+ }
+
+ DecoratedKey getKey()
+ {
+ return key;
+ }
+
+ ConsistencyLevel getConsistency()
+ {
+ return writePlan.consistencyLevel();
+ }
+}
diff --git a/src/java/org/apache/cassandra/service/reads/repair/BlockingReadRepair.java b/src/java/org/apache/cassandra/service/reads/repair/BlockingReadRepair.java
new file mode 100644
index 0000000..fdc8b50
--- /dev/null
+++ b/src/java/org/apache/cassandra/service/reads/repair/BlockingReadRepair.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.reads.repair;
+
+import java.util.Map;
+import java.util.Queue;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.cassandra.db.DecoratedKey;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.codahale.metrics.Meter;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.Mutation;
+import org.apache.cassandra.db.ReadCommand;
+import org.apache.cassandra.db.partitions.UnfilteredPartitionIterators;
+import org.apache.cassandra.exceptions.ReadTimeoutException;
+import org.apache.cassandra.locator.Endpoints;
+import org.apache.cassandra.locator.Replica;
+import org.apache.cassandra.locator.ReplicaPlan;
+import org.apache.cassandra.metrics.ReadRepairMetrics;
+import org.apache.cassandra.tracing.Tracing;
+
+import static java.util.concurrent.TimeUnit.NANOSECONDS;
+
+/**
+ * 'Classic' read repair. Doesn't allow the client read to return until
+ * updates have been written to nodes needing correction. Breaks write
+ * atomicity in some situations
+ */
+public class BlockingReadRepair<E extends Endpoints<E>, P extends ReplicaPlan.ForRead<E>>
+ extends AbstractReadRepair<E, P>
+{
+ private static final Logger logger = LoggerFactory.getLogger(BlockingReadRepair.class);
+
+ protected final Queue<BlockingPartitionRepair> repairs = new ConcurrentLinkedQueue<>();
+
+ BlockingReadRepair(ReadCommand command, ReplicaPlan.Shared<E, P> replicaPlan, long queryStartNanoTime)
+ {
+ super(command, replicaPlan, queryStartNanoTime);
+ }
+
+ public UnfilteredPartitionIterators.MergeListener getMergeListener(P replicaPlan)
+ {
+ return new PartitionIteratorMergeListener<>(replicaPlan, command, this);
+ }
+
+ @Override
+ Meter getRepairMeter()
+ {
+ return ReadRepairMetrics.repairedBlocking;
+ }
+
+ @Override
+ public void maybeSendAdditionalWrites()
+ {
+ for (BlockingPartitionRepair repair: repairs)
+ {
+ repair.maybeSendAdditionalWrites(cfs.additionalWriteLatencyNanos, TimeUnit.NANOSECONDS);
+ }
+ }
+
+ @Override
+ public void awaitWrites()
+ {
+ BlockingPartitionRepair timedOut = null;
+ for (BlockingPartitionRepair repair : repairs)
+ {
+ if (!repair.awaitRepairsUntil(DatabaseDescriptor.getReadRpcTimeout(NANOSECONDS) + queryStartNanoTime, NANOSECONDS))
+ {
+ timedOut = repair;
+ break;
+ }
+ }
+ if (timedOut != null)
+ {
+ // We got all responses, but timed out while repairing;
+ // pick one of the repairs to throw, as this is better than completely manufacturing the error message
+ int blockFor = timedOut.blockFor();
+ int received = Math.min(blockFor - timedOut.waitingOn(), blockFor - 1);
+ if (Tracing.isTracing())
+ Tracing.trace("Timed out while read-repairing after receiving all {} data and digest responses", blockFor);
+ else
+ logger.debug("Timeout while read-repairing after receiving all {} data and digest responses", blockFor);
+
+ throw new ReadTimeoutException(replicaPlan().consistencyLevel(), received, blockFor, true);
+ }
+ }
+
+ @Override
+ public void repairPartition(DecoratedKey partitionKey, Map<Replica, Mutation> mutations, ReplicaPlan.ForTokenWrite writePlan)
+ {
+ BlockingPartitionRepair blockingRepair = new BlockingPartitionRepair(partitionKey, mutations, writePlan);
+ blockingRepair.sendInitialRepairs();
+ repairs.add(blockingRepair);
+ }
+}
diff --git a/src/java/org/apache/cassandra/service/reads/repair/BlockingReadRepairs.java b/src/java/org/apache/cassandra/service/reads/repair/BlockingReadRepairs.java
new file mode 100644
index 0000000..68d1b4c
--- /dev/null
+++ b/src/java/org/apache/cassandra/service/reads/repair/BlockingReadRepairs.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.reads.repair;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.db.ConsistencyLevel;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.Mutation;
+import org.apache.cassandra.db.MutationExceededMaxSizeException;
+import org.apache.cassandra.db.partitions.PartitionUpdate;
+import org.apache.cassandra.exceptions.ReadTimeoutException;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.tracing.Tracing;
+
+import static org.apache.cassandra.db.IMutation.MAX_MUTATION_SIZE;
+
+public class BlockingReadRepairs
+{
+ private static final Logger logger = LoggerFactory.getLogger(BlockingReadRepairs.class);
+
+ private static final boolean DROP_OVERSIZED_READ_REPAIR_MUTATIONS =
+ Boolean.getBoolean("cassandra.drop_oversized_readrepair_mutations");
+
+ /**
+ * Create a read repair mutation from the given update, if the mutation is not larger than the maximum
+ * mutation size, otherwise return null. Or, if we're configured to be strict, throw an exception.
+ */
+ public static Mutation createRepairMutation(PartitionUpdate update, ConsistencyLevel consistency, InetAddressAndPort destination, boolean suppressException)
+ {
+ if (update == null)
+ return null;
+
+ DecoratedKey key = update.partitionKey();
+ Mutation mutation = new Mutation(update);
+ int messagingVersion = MessagingService.instance().versions.get(destination);
+
+ try
+ {
+ mutation.validateSize(messagingVersion, 0);
+ return mutation;
+ }
+ catch (MutationExceededMaxSizeException e)
+ {
+ Keyspace keyspace = Keyspace.open(mutation.getKeyspaceName());
+ TableMetadata metadata = update.metadata();
+
+ if (DROP_OVERSIZED_READ_REPAIR_MUTATIONS)
+ {
+ logger.debug("Encountered an oversized ({}/{}) read repair mutation for table {}, key {}, node {}",
+ e.mutationSize,
+ MAX_MUTATION_SIZE,
+ metadata,
+ metadata.partitionKeyType.getString(key.getKey()),
+ destination);
+ }
+ else
+ {
+ logger.warn("Encountered an oversized ({}/{}) read repair mutation for table {}, key {}, node {}",
+ e.mutationSize,
+ MAX_MUTATION_SIZE,
+ metadata,
+ metadata.partitionKeyType.getString(key.getKey()),
+ destination);
+
+ if (!suppressException)
+ {
+ int blockFor = consistency.blockFor(keyspace);
+ Tracing.trace("Timed out while read-repairing after receiving all {} data and digest responses", blockFor);
+ throw new ReadTimeoutException(consistency, blockFor - 1, blockFor, true);
+ }
+ }
+ return null;
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/service/reads/repair/NoopReadRepair.java b/src/java/org/apache/cassandra/service/reads/repair/NoopReadRepair.java
new file mode 100644
index 0000000..2f82c22
--- /dev/null
+++ b/src/java/org/apache/cassandra/service/reads/repair/NoopReadRepair.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.reads.repair;
+
+import java.util.Map;
+import java.util.function.Consumer;
+
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.Mutation;
+import org.apache.cassandra.db.partitions.PartitionIterator;
+import org.apache.cassandra.db.partitions.UnfilteredPartitionIterators;
+import org.apache.cassandra.exceptions.ReadTimeoutException;
+import org.apache.cassandra.locator.Endpoints;
+import org.apache.cassandra.locator.Replica;
+import org.apache.cassandra.locator.ReplicaPlan;
+import org.apache.cassandra.service.reads.DigestResolver;
+
+/**
+ * Bypasses the read repair path for short read protection and testing
+ */
+public class NoopReadRepair<E extends Endpoints<E>, P extends ReplicaPlan.ForRead<E>>
+ implements ReadRepair<E, P>
+{
+ public static final NoopReadRepair instance = new NoopReadRepair();
+
+ private NoopReadRepair() {}
+
+ @Override
+ public UnfilteredPartitionIterators.MergeListener getMergeListener(P replicas)
+ {
+ return UnfilteredPartitionIterators.MergeListener.NOOP;
+ }
+
+ @Override
+ public void startRepair(DigestResolver<E, P> digestResolver, Consumer<PartitionIterator> resultConsumer)
+ {
+ resultConsumer.accept(digestResolver.getData());
+ }
+
+ public void awaitReads() throws ReadTimeoutException
+ {
+ }
+
+ @Override
+ public void maybeSendAdditionalReads()
+ {
+
+ }
+
+ @Override
+ public void maybeSendAdditionalWrites()
+ {
+
+ }
+
+ @Override
+ public void awaitWrites()
+ {
+
+ }
+
+ @Override
+ public void repairPartition(DecoratedKey partitionKey, Map<Replica, Mutation> mutations, ReplicaPlan.ForTokenWrite writePlan)
+ {
+
+ }
+}
diff --git a/src/java/org/apache/cassandra/service/reads/repair/PartitionIteratorMergeListener.java b/src/java/org/apache/cassandra/service/reads/repair/PartitionIteratorMergeListener.java
new file mode 100644
index 0000000..7247704
--- /dev/null
+++ b/src/java/org/apache/cassandra/service/reads/repair/PartitionIteratorMergeListener.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.reads.repair;
+
+import java.util.List;
+
+import org.apache.cassandra.db.Columns;
+import org.apache.cassandra.db.ConsistencyLevel;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.ReadCommand;
+import org.apache.cassandra.db.RegularAndStaticColumns;
+import org.apache.cassandra.db.partitions.UnfilteredPartitionIterators;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.db.rows.UnfilteredRowIterators;
+import org.apache.cassandra.locator.Endpoints;
+import org.apache.cassandra.locator.ReplicaPlan;
+
+public class PartitionIteratorMergeListener<E extends Endpoints<E>>
+ implements UnfilteredPartitionIterators.MergeListener
+{
+ private final ReplicaPlan.ForRead<E> replicaPlan;
+ private final ReadCommand command;
+ private final ReadRepair readRepair;
+
+ public PartitionIteratorMergeListener(ReplicaPlan.ForRead<E> replicaPlan, ReadCommand command, ReadRepair readRepair)
+ {
+ this.replicaPlan = replicaPlan;
+ this.command = command;
+ this.readRepair = readRepair;
+ }
+
+ public UnfilteredRowIterators.MergeListener getRowMergeListener(DecoratedKey partitionKey, List<UnfilteredRowIterator> versions)
+ {
+ return new RowIteratorMergeListener<>(partitionKey, columns(versions), isReversed(versions), replicaPlan, command, readRepair);
+ }
+
+ protected RegularAndStaticColumns columns(List<UnfilteredRowIterator> versions)
+ {
+ Columns statics = Columns.NONE;
+ Columns regulars = Columns.NONE;
+ for (UnfilteredRowIterator iter : versions)
+ {
+ if (iter == null)
+ continue;
+
+ RegularAndStaticColumns cols = iter.columns();
+ statics = statics.mergeTo(cols.statics);
+ regulars = regulars.mergeTo(cols.regulars);
+ }
+ return new RegularAndStaticColumns(statics, regulars);
+ }
+
+ protected boolean isReversed(List<UnfilteredRowIterator> versions)
+ {
+ for (UnfilteredRowIterator iter : versions)
+ {
+ if (iter == null)
+ continue;
+
+ // Everything will be in the same order
+ return iter.isReverseOrder();
+ }
+
+ assert false : "Expected at least one iterator";
+ return false;
+ }
+
+ public void close()
+ {
+ }
+}
+
diff --git a/src/java/org/apache/cassandra/service/reads/repair/PartitionRepairEvent.java b/src/java/org/apache/cassandra/service/reads/repair/PartitionRepairEvent.java
new file mode 100644
index 0000000..04abbcf
--- /dev/null
+++ b/src/java/org/apache/cassandra/service/reads/repair/PartitionRepairEvent.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.reads.repair;
+
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.Map;
+
+import javax.annotation.Nullable;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import org.apache.cassandra.db.ConsistencyLevel;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.Mutation;
+import org.apache.cassandra.diag.DiagnosticEvent;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.utils.ByteBufferUtil;
+
+final class PartitionRepairEvent extends DiagnosticEvent
+{
+ private final PartitionRepairEventType type;
+ @VisibleForTesting
+ final InetAddressAndPort destination;
+ @Nullable
+ private final Keyspace keyspace;
+ @Nullable
+ private final DecoratedKey key;
+ @Nullable
+ private final ConsistencyLevel consistency;
+ @Nullable
+ @VisibleForTesting
+ String mutationSummary;
+
+ enum PartitionRepairEventType
+ {
+ SEND_INITIAL_REPAIRS,
+ SPECULATED_WRITE,
+ UPDATE_OVERSIZED
+ }
+
+ PartitionRepairEvent(PartitionRepairEventType type, BlockingPartitionRepair partitionRepair,
+ InetAddressAndPort destination, Mutation mutation)
+ {
+ this.type = type;
+ this.destination = destination;
+ this.keyspace = partitionRepair.getKeyspace();
+ this.consistency = partitionRepair.getConsistency();
+ this.key = partitionRepair.getKey();
+ if (mutation != null)
+ {
+ try
+ {
+ this.mutationSummary = mutation.toString();
+ }
+ catch (Exception e)
+ {
+ this.mutationSummary = String.format("<Mutation.toString(): %s>", e.getMessage());
+ }
+ }
+ }
+
+ public PartitionRepairEventType getType()
+ {
+ return type;
+ }
+
+ public Map<String, Serializable> toMap()
+ {
+ HashMap<String, Serializable> ret = new HashMap<>();
+ if (keyspace != null) ret.put("keyspace", keyspace.getName());
+ if (key != null)
+ {
+ ret.put("key", key.getKey() == null ? "null" : ByteBufferUtil.bytesToHex(key.getKey()));
+ ret.put("token", key.getToken().toString());
+ }
+ if (consistency != null) ret.put("consistency", consistency.name());
+
+ ret.put("destination", destination.toString());
+
+ if (mutationSummary != null) ret.put("mutation", mutationSummary);
+
+ return ret;
+ }
+}
diff --git a/src/java/org/apache/cassandra/service/reads/repair/ReadOnlyReadRepair.java b/src/java/org/apache/cassandra/service/reads/repair/ReadOnlyReadRepair.java
new file mode 100644
index 0000000..d9293fb
--- /dev/null
+++ b/src/java/org/apache/cassandra/service/reads/repair/ReadOnlyReadRepair.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.reads.repair;
+
+import java.util.Map;
+
+import com.codahale.metrics.Meter;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.Mutation;
+import org.apache.cassandra.db.ReadCommand;
+import org.apache.cassandra.db.partitions.UnfilteredPartitionIterators;
+import org.apache.cassandra.locator.Endpoints;
+import org.apache.cassandra.locator.Replica;
+import org.apache.cassandra.locator.ReplicaPlan;
+import org.apache.cassandra.metrics.ReadRepairMetrics;
+
+/**
+ * Only performs the collection of data responses and reconciliation of them, doesn't send repair mutations
+ * to replicas. This preserves write atomicity, but doesn't provide monotonic quorum reads
+ */
+public class ReadOnlyReadRepair<E extends Endpoints<E>, P extends ReplicaPlan.ForRead<E>>
+ extends AbstractReadRepair<E, P>
+{
+ ReadOnlyReadRepair(ReadCommand command, ReplicaPlan.Shared<E, P> replicaPlan, long queryStartNanoTime)
+ {
+ super(command, replicaPlan, queryStartNanoTime);
+ }
+
+ @Override
+ public UnfilteredPartitionIterators.MergeListener getMergeListener(P replicaPlan)
+ {
+ return UnfilteredPartitionIterators.MergeListener.NOOP;
+ }
+
+ @Override
+ Meter getRepairMeter()
+ {
+ return ReadRepairMetrics.reconcileRead;
+ }
+
+ @Override
+ public void maybeSendAdditionalWrites()
+ {
+
+ }
+
+ @Override
+ public void repairPartition(DecoratedKey partitionKey, Map<Replica, Mutation> mutations, ReplicaPlan.ForTokenWrite writePlan)
+ {
+ throw new UnsupportedOperationException("ReadOnlyReadRepair shouldn't be trying to repair partitions");
+ }
+
+ @Override
+ public void awaitWrites()
+ {
+
+ }
+}
diff --git a/src/java/org/apache/cassandra/service/reads/repair/ReadRepair.java b/src/java/org/apache/cassandra/service/reads/repair/ReadRepair.java
new file mode 100644
index 0000000..4747651
--- /dev/null
+++ b/src/java/org/apache/cassandra/service/reads/repair/ReadRepair.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.service.reads.repair;
+
+import java.util.Map;
+import java.util.function.Consumer;
+import java.util.function.Supplier;
+
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.locator.Endpoints;
+
+import org.apache.cassandra.db.Mutation;
+import org.apache.cassandra.db.ReadCommand;
+import org.apache.cassandra.db.partitions.PartitionIterator;
+import org.apache.cassandra.db.partitions.UnfilteredPartitionIterators;
+import org.apache.cassandra.exceptions.ReadTimeoutException;
+import org.apache.cassandra.locator.Replica;
+import org.apache.cassandra.locator.ReplicaPlan;
+import org.apache.cassandra.service.reads.DigestResolver;
+
+public interface ReadRepair<E extends Endpoints<E>, P extends ReplicaPlan.ForRead<E>>
+{
+ public interface Factory
+ {
+ <E extends Endpoints<E>, P extends ReplicaPlan.ForRead<E>>
+ ReadRepair<E, P> create(ReadCommand command, ReplicaPlan.Shared<E, P> replicaPlan, long queryStartNanoTime);
+ }
+
+ static <E extends Endpoints<E>, P extends ReplicaPlan.ForRead<E>>
+ ReadRepair<E, P> create(ReadCommand command, ReplicaPlan.Shared<E, P> replicaPlan, long queryStartNanoTime)
+ {
+ return command.metadata().params.readRepair.create(command, replicaPlan, queryStartNanoTime);
+ }
+
+ /**
+ * Used by DataResolver to generate corrections as the partition iterator is consumed
+ */
+ UnfilteredPartitionIterators.MergeListener getMergeListener(P replicaPlan);
+
+ /**
+ * Called when the digests from the initial read don't match. Reads may block on the
+ * repair started by this method.
+ * @param digestResolver supplied so we can get the original data response
+ * @param resultConsumer hook for the repair to set it's result on completion
+ */
+ public void startRepair(DigestResolver<E, P> digestResolver, Consumer<PartitionIterator> resultConsumer);
+
+ /**
+ * Block on the reads (or timeout) sent out in {@link ReadRepair#startRepair}
+ */
+ public void awaitReads() throws ReadTimeoutException;
+
+ /**
+ * if it looks like we might not receive data requests from everyone in time, send additional requests
+ * to additional replicas not contacted in the initial full data read. If the collection of nodes that
+ * end up responding in time end up agreeing on the data, and we don't consider the response from the
+ * disagreeing replica that triggered the read repair, that's ok, since the disagreeing data would not
+ * have been successfully written and won't be included in the response the the client, preserving the
+ * expectation of monotonic quorum reads
+ */
+ public void maybeSendAdditionalReads();
+
+ /**
+ * If it looks like we might not receive acks for all the repair mutations we sent out, combine all
+ * the unacked mutations and send them to the minority of nodes not involved in the read repair data
+ * read / write cycle. We will accept acks from them in lieu of acks from the initial mutations sent
+ * out, so long as we receive the same number of acks as repair mutations transmitted. This prevents
+ * misbehaving nodes from killing a quorum read, while continuing to guarantee monotonic quorum reads
+ */
+ public void maybeSendAdditionalWrites();
+
+ /**
+ * Block on any mutations (or timeout) we sent out to repair replicas in {@link ReadRepair#repairPartition}
+ */
+ public void awaitWrites();
+
+ /**
+ * Repairs a partition _after_ receiving data responses. This method receives replica list, since
+ * we will block repair only on the replicas that have responded.
+ */
+ void repairPartition(DecoratedKey partitionKey, Map<Replica, Mutation> mutations, ReplicaPlan.ForTokenWrite writePlan);
+}
diff --git a/src/java/org/apache/cassandra/service/reads/repair/ReadRepairDiagnostics.java b/src/java/org/apache/cassandra/service/reads/repair/ReadRepairDiagnostics.java
new file mode 100644
index 0000000..b9167bd
--- /dev/null
+++ b/src/java/org/apache/cassandra/service/reads/repair/ReadRepairDiagnostics.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.reads.repair;
+
+import java.util.Collections;
+
+import com.google.common.collect.Lists;
+
+import org.apache.cassandra.db.Mutation;
+import org.apache.cassandra.diag.DiagnosticEventService;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.ReplicaLayout;
+import org.apache.cassandra.locator.ReplicaPlan;
+import org.apache.cassandra.service.reads.DigestResolver;
+import org.apache.cassandra.service.reads.repair.PartitionRepairEvent.PartitionRepairEventType;
+import org.apache.cassandra.service.reads.repair.ReadRepairEvent.ReadRepairEventType;
+
+final class ReadRepairDiagnostics
+{
+ private static final DiagnosticEventService service = DiagnosticEventService.instance();
+
+ private ReadRepairDiagnostics()
+ {
+ }
+
+ static void startRepair(AbstractReadRepair readRepair, ReplicaPlan.ForRead<?> fullPlan, DigestResolver digestResolver)
+ {
+ if (service.isEnabled(ReadRepairEvent.class, ReadRepairEventType.START_REPAIR))
+ service.publish(new ReadRepairEvent(ReadRepairEventType.START_REPAIR,
+ readRepair,
+ fullPlan.contacts().endpoints(),
+ fullPlan.candidates().endpoints(), digestResolver));
+ }
+
+ static void speculatedRead(AbstractReadRepair readRepair, InetAddressAndPort endpoint,
+ ReplicaPlan.ForRead<?> fullPlan)
+ {
+ if (service.isEnabled(ReadRepairEvent.class, ReadRepairEventType.SPECULATED_READ))
+ service.publish(new ReadRepairEvent(ReadRepairEventType.SPECULATED_READ,
+ readRepair, Collections.singletonList(endpoint),
+ Lists.newArrayList(fullPlan.candidates().endpoints()), null));
+ }
+
+ static void sendInitialRepair(BlockingPartitionRepair partitionRepair, InetAddressAndPort destination, Mutation mutation)
+ {
+ if (service.isEnabled(PartitionRepairEvent.class, PartitionRepairEventType.SEND_INITIAL_REPAIRS))
+ service.publish(new PartitionRepairEvent(PartitionRepairEventType.SEND_INITIAL_REPAIRS, partitionRepair,
+ destination, mutation));
+ }
+
+ static void speculatedWrite(BlockingPartitionRepair partitionRepair, InetAddressAndPort destination, Mutation mutation)
+ {
+ if (service.isEnabled(PartitionRepairEvent.class, PartitionRepairEventType.SPECULATED_WRITE))
+ service.publish(new PartitionRepairEvent(PartitionRepairEventType.SPECULATED_WRITE, partitionRepair,
+ destination, mutation));
+ }
+
+ static void speculatedWriteOversized(BlockingPartitionRepair partitionRepair, InetAddressAndPort destination)
+ {
+ if (service.isEnabled(PartitionRepairEvent.class, PartitionRepairEventType.UPDATE_OVERSIZED))
+ service.publish(new PartitionRepairEvent(PartitionRepairEventType.UPDATE_OVERSIZED, partitionRepair,
+ destination, null));
+ }
+}
diff --git a/src/java/org/apache/cassandra/service/reads/repair/ReadRepairEvent.java b/src/java/org/apache/cassandra/service/reads/repair/ReadRepairEvent.java
new file mode 100644
index 0000000..5cec802
--- /dev/null
+++ b/src/java/org/apache/cassandra/service/reads/repair/ReadRepairEvent.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.reads.repair;
+
+import java.io.Serializable;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import javax.annotation.Nullable;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import org.apache.cassandra.db.ConsistencyLevel;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.diag.DiagnosticEvent;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.service.reads.DigestResolver;
+import org.apache.cassandra.service.reads.DigestResolver.DigestResolverDebugResult;
+import org.apache.cassandra.service.reads.SpeculativeRetryPolicy;
+
+final class ReadRepairEvent extends DiagnosticEvent
+{
+
+ private final ReadRepairEventType type;
+ private final Keyspace keyspace;
+ private final String tableName;
+ private final String cqlCommand;
+ private final ConsistencyLevel consistency;
+ private final SpeculativeRetryPolicy.Kind speculativeRetry;
+ @VisibleForTesting
+ final Collection<InetAddressAndPort> destinations;
+ @VisibleForTesting
+ final Collection<InetAddressAndPort> allEndpoints;
+ @Nullable
+ private final DigestResolverDebugResult[] digestsByEndpoint;
+
+ enum ReadRepairEventType
+ {
+ START_REPAIR,
+ SPECULATED_READ
+ }
+
+ ReadRepairEvent(ReadRepairEventType type, AbstractReadRepair readRepair, Collection<InetAddressAndPort> destinations,
+ Collection<InetAddressAndPort> allEndpoints, DigestResolver digestResolver)
+ {
+ this.keyspace = readRepair.cfs.keyspace;
+ this.tableName = readRepair.cfs.getTableName();
+ this.cqlCommand = readRepair.command.toCQLString();
+ this.consistency = readRepair.replicaPlan().consistencyLevel();
+ this.speculativeRetry = readRepair.cfs.metadata().params.speculativeRetry.kind();
+ this.destinations = destinations;
+ this.allEndpoints = allEndpoints;
+ this.digestsByEndpoint = digestResolver != null ? digestResolver.getDigestsByEndpoint() : null;
+ this.type = type;
+ }
+
+ public ReadRepairEventType getType()
+ {
+ return type;
+ }
+
+ public Map<String, Serializable> toMap()
+ {
+ HashMap<String, Serializable> ret = new HashMap<>();
+
+ ret.put("keyspace", keyspace.getName());
+ ret.put("table", tableName);
+ ret.put("command", cqlCommand);
+ ret.put("consistency", consistency.name());
+ ret.put("speculativeRetry", speculativeRetry.name());
+
+ Set<String> eps = destinations.stream().map(InetAddressAndPort::toString).collect(Collectors.toSet());
+ ret.put("endpointDestinations", new HashSet<>(eps));
+
+ if (digestsByEndpoint != null)
+ {
+ HashMap<String, Serializable> digestsMap = new HashMap<>();
+ for (DigestResolverDebugResult digestsByEndpoint : digestsByEndpoint)
+ {
+ HashMap<String, Serializable> digests = new HashMap<>();
+ digests.put("digestHex", digestsByEndpoint.digestHex);
+ digests.put("isDigestResponse", digestsByEndpoint.isDigestResponse);
+ digestsMap.put(digestsByEndpoint.from.toString(), digests);
+ }
+ ret.put("digestsByEndpoint", digestsMap);
+ }
+ if (allEndpoints != null)
+ {
+ eps = allEndpoints.stream().map(InetAddressAndPort::toString).collect(Collectors.toSet());
+ ret.put("allEndpoints", new HashSet<>(eps));
+ }
+ return ret;
+ }
+}
diff --git a/src/java/org/apache/cassandra/service/reads/repair/ReadRepairStrategy.java b/src/java/org/apache/cassandra/service/reads/repair/ReadRepairStrategy.java
new file mode 100644
index 0000000..7a4b795
--- /dev/null
+++ b/src/java/org/apache/cassandra/service/reads/repair/ReadRepairStrategy.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.reads.repair;
+
+import org.apache.cassandra.db.ReadCommand;
+import org.apache.cassandra.locator.Endpoints;
+import org.apache.cassandra.locator.ReplicaLayout;
+import org.apache.cassandra.locator.ReplicaPlan;
+
+public enum ReadRepairStrategy implements ReadRepair.Factory
+{
+ NONE
+ {
+ public <E extends Endpoints<E>, P extends ReplicaPlan.ForRead<E>>
+ ReadRepair<E, P> create(ReadCommand command, ReplicaPlan.Shared<E, P> replicaPlan, long queryStartNanoTime)
+ {
+ return new ReadOnlyReadRepair<>(command, replicaPlan, queryStartNanoTime);
+ }
+ },
+
+ BLOCKING
+ {
+ public <E extends Endpoints<E>, P extends ReplicaPlan.ForRead<E>>
+ ReadRepair<E, P> create(ReadCommand command, ReplicaPlan.Shared<E, P> replicaPlan, long queryStartNanoTime)
+ {
+ return new BlockingReadRepair<>(command, replicaPlan, queryStartNanoTime);
+ }
+ };
+
+ public static ReadRepairStrategy fromString(String s)
+ {
+ return valueOf(s.toUpperCase());
+ }
+}
diff --git a/src/java/org/apache/cassandra/service/reads/repair/RepairedDataTracker.java b/src/java/org/apache/cassandra/service/reads/repair/RepairedDataTracker.java
new file mode 100644
index 0000000..5024e86
--- /dev/null
+++ b/src/java/org/apache/cassandra/service/reads/repair/RepairedDataTracker.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.reads.repair;
+
+import java.nio.ByteBuffer;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+
+import com.google.common.base.MoreObjects;
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.Multimap;
+
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.utils.ByteBufferUtil;
+
+public class RepairedDataTracker
+{
+ private final RepairedDataVerifier verifier;
+
+ public final Multimap<ByteBuffer, InetAddressAndPort> digests = HashMultimap.create();
+ public final Set<InetAddressAndPort> inconclusiveDigests = new HashSet<>();
+
+ public RepairedDataTracker(RepairedDataVerifier verifier)
+ {
+ this.verifier = verifier;
+ }
+
+ public void recordDigest(InetAddressAndPort source, ByteBuffer digest, boolean isConclusive)
+ {
+ digests.put(digest, source);
+ if (!isConclusive)
+ inconclusiveDigests.add(source);
+ }
+
+ public void verify()
+ {
+ verifier.verify(this);
+ }
+
+ public String toString()
+ {
+ return MoreObjects.toStringHelper(this)
+ .add("digests", hexDigests())
+ .add("inconclusive", inconclusiveDigests).toString();
+ }
+
+ private Map<String, Collection<InetAddressAndPort>> hexDigests()
+ {
+ Map<String, Collection<InetAddressAndPort>> hexDigests = new HashMap<>();
+ digests.asMap().forEach((k, v) -> hexDigests.put(ByteBufferUtil.bytesToHex(k), v));
+ return hexDigests;
+ }
+
+ public boolean equals(Object o)
+ {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ RepairedDataTracker that = (RepairedDataTracker) o;
+ return Objects.equals(digests, that.digests) &&
+ Objects.equals(inconclusiveDigests, that.inconclusiveDigests);
+ }
+
+ public int hashCode()
+ {
+ return Objects.hash(digests, inconclusiveDigests);
+ }
+}
diff --git a/src/java/org/apache/cassandra/service/reads/repair/RepairedDataVerifier.java b/src/java/org/apache/cassandra/service/reads/repair/RepairedDataVerifier.java
new file mode 100644
index 0000000..d1cff11
--- /dev/null
+++ b/src/java/org/apache/cassandra/service/reads/repair/RepairedDataVerifier.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.reads.repair;
+
+import java.time.LocalDate;
+import java.time.format.DateTimeFormatter;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.ReadCommand;
+import org.apache.cassandra.db.SnapshotCommand;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.metrics.TableMetrics;
+import org.apache.cassandra.net.Message;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.net.Verb;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.service.SnapshotVerbHandler;
+import org.apache.cassandra.tracing.Tracing;
+import org.apache.cassandra.utils.DiagnosticSnapshotService;
+import org.apache.cassandra.utils.NoSpamLogger;
+
+public interface RepairedDataVerifier
+{
+ public void verify(RepairedDataTracker tracker);
+
+ static RepairedDataVerifier verifier(ReadCommand command)
+ {
+ return DatabaseDescriptor.snapshotOnRepairedDataMismatch() ? snapshotting(command) : simple(command);
+ }
+
+ static RepairedDataVerifier simple(ReadCommand command)
+ {
+ return new SimpleVerifier(command);
+ }
+
+ static RepairedDataVerifier snapshotting(ReadCommand command)
+ {
+ return new SnapshottingVerifier(command);
+ }
+
+ static class SimpleVerifier implements RepairedDataVerifier
+ {
+ private static final Logger logger = LoggerFactory.getLogger(SimpleVerifier.class);
+ protected final ReadCommand command;
+
+ private static final String INCONSISTENCY_WARNING = "Detected mismatch between repaired datasets for table {}.{} during read of {}. {}";
+
+ SimpleVerifier(ReadCommand command)
+ {
+ this.command = command;
+ }
+
+ @Override
+ public void verify(RepairedDataTracker tracker)
+ {
+ Tracing.trace("Verifying repaired data tracker {}", tracker);
+
+ // some mismatch occurred between the repaired datasets on the replicas
+ if (tracker.digests.keySet().size() > 1)
+ {
+ // if any of the digests should be considered inconclusive, because there were
+ // pending repair sessions which had not yet been committed or unrepaired partition
+ // deletes which meant some sstables were skipped during reads, mark the inconsistency
+ // as confirmed
+ if (tracker.inconclusiveDigests.isEmpty())
+ {
+ TableMetrics metrics = ColumnFamilyStore.metricsFor(command.metadata().id);
+ metrics.confirmedRepairedInconsistencies.mark();
+ NoSpamLogger.log(logger, NoSpamLogger.Level.WARN, 1, TimeUnit.MINUTES,
+ INCONSISTENCY_WARNING, command.metadata().keyspace,
+ command.metadata().name, command.toString(), tracker);
+ }
+ else if (DatabaseDescriptor.reportUnconfirmedRepairedDataMismatches())
+ {
+ TableMetrics metrics = ColumnFamilyStore.metricsFor(command.metadata().id);
+ metrics.unconfirmedRepairedInconsistencies.mark();
+ NoSpamLogger.log(logger, NoSpamLogger.Level.WARN, 1, TimeUnit.MINUTES,
+ INCONSISTENCY_WARNING, command.metadata().keyspace,
+ command.metadata().name, command.toString(), tracker);
+ }
+ }
+ }
+ }
+
+ static class SnapshottingVerifier extends SimpleVerifier
+ {
+ private static final Logger logger = LoggerFactory.getLogger(SnapshottingVerifier.class);
+ private static final String SNAPSHOTTING_WARNING = "Issuing snapshot command for mismatch between repaired datasets for table {}.{} during read of {}. {}";
+
+ SnapshottingVerifier(ReadCommand command)
+ {
+ super(command);
+ }
+
+ public void verify(RepairedDataTracker tracker)
+ {
+ super.verify(tracker);
+ if (tracker.digests.keySet().size() > 1)
+ {
+ if (tracker.inconclusiveDigests.isEmpty() || DatabaseDescriptor.reportUnconfirmedRepairedDataMismatches())
+ {
+ logger.warn(SNAPSHOTTING_WARNING, command.metadata().keyspace, command.metadata().name, command.toString(), tracker);
+ DiagnosticSnapshotService.repairedDataMismatch(command.metadata(), tracker.digests.values());
+ }
+ }
+ }
+ }
+}
+
diff --git a/src/java/org/apache/cassandra/service/reads/repair/RowIteratorMergeListener.java b/src/java/org/apache/cassandra/service/reads/repair/RowIteratorMergeListener.java
new file mode 100644
index 0000000..12d27d2
--- /dev/null
+++ b/src/java/org/apache/cassandra/service/reads/repair/RowIteratorMergeListener.java
@@ -0,0 +1,388 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.reads.repair;
+
+import java.util.Arrays;
+import java.util.BitSet;
+import java.util.Map;
+import java.util.function.Consumer;
+
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Maps;
+
+import com.carrotsearch.hppc.ObjectIntHashMap;
+import net.nicoulaj.compilecommand.annotations.Inline;
+import org.apache.cassandra.db.Clustering;
+import org.apache.cassandra.db.ClusteringBound;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.DeletionTime;
+import org.apache.cassandra.db.LivenessInfo;
+import org.apache.cassandra.db.Mutation;
+import org.apache.cassandra.db.RangeTombstone;
+import org.apache.cassandra.db.ReadCommand;
+import org.apache.cassandra.db.RegularAndStaticColumns;
+import org.apache.cassandra.db.Slice;
+import org.apache.cassandra.db.filter.ColumnFilter;
+import org.apache.cassandra.db.partitions.PartitionUpdate;
+import org.apache.cassandra.db.rows.BTreeRow;
+import org.apache.cassandra.db.rows.Cell;
+import org.apache.cassandra.db.rows.RangeTombstoneMarker;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.db.rows.RowDiffListener;
+import org.apache.cassandra.db.rows.Rows;
+import org.apache.cassandra.db.rows.UnfilteredRowIterators;
+import org.apache.cassandra.locator.Endpoints;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.Replica;
+import org.apache.cassandra.locator.ReplicaPlan;
+import org.apache.cassandra.locator.ReplicaPlans;
+import org.apache.cassandra.schema.ColumnMetadata;
+
+public class RowIteratorMergeListener<E extends Endpoints<E>>
+ implements UnfilteredRowIterators.MergeListener
+{
+ private final DecoratedKey partitionKey;
+ private final RegularAndStaticColumns columns;
+ private final boolean isReversed;
+ private final ReadCommand command;
+
+ private final BitSet writeBackTo;
+ private final boolean buildFullDiff;
+ /** the repairs we will send to each source, suffixed by a complete repair of all differences, if {@link #buildFullDiff} */
+ private final PartitionUpdate.Builder[] repairs;
+ private final Row.Builder[] currentRows;
+ private final RowDiffListener diffListener;
+ private final ReplicaPlan.ForRead<E> readPlan;
+ private final ReplicaPlan.ForTokenWrite writePlan;
+
+ // The partition level deletion for the merge row.
+ private DeletionTime partitionLevelDeletion;
+ // When merged has a currently open marker, its time. null otherwise.
+ private DeletionTime mergedDeletionTime;
+ // For each source, the time of the current deletion as known by the source.
+ private final DeletionTime[] sourceDeletionTime;
+ // For each source, record if there is an open range to send as repair, and from where.
+ private final ClusteringBound[] markerToRepair;
+
+ private final ReadRepair readRepair;
+
+ public RowIteratorMergeListener(DecoratedKey partitionKey, RegularAndStaticColumns columns, boolean isReversed, ReplicaPlan.ForRead<E> readPlan, ReadCommand command, ReadRepair readRepair)
+ {
+ this.partitionKey = partitionKey;
+ this.columns = columns;
+ this.isReversed = isReversed;
+ this.readPlan = readPlan;
+ this.writePlan = ReplicaPlans.forReadRepair(partitionKey.getToken(), readPlan);
+
+ int size = readPlan.contacts().size();
+ this.writeBackTo = new BitSet(size);
+ {
+ int i = 0;
+ for (Replica replica : readPlan.contacts())
+ {
+ if (writePlan.contacts().endpoints().contains(replica.endpoint()))
+ writeBackTo.set(i);
+ ++i;
+ }
+ }
+ // If we are contacting any nodes we didn't read from, we are likely handling a range movement.
+ // In this case we need to send all differences to these nodes, as we do not (with present design) know which
+ // node they bootstrapped from, and so which data we need to duplicate.
+ // In reality, there will be situations where we are simply sending the same number of writes to different nodes
+ // and in this case we could probably avoid building a full difference, and only ensure each write makes it to
+ // some other node, but it is probably not worth special casing this scenario.
+ this.buildFullDiff = Iterables.any(writePlan.contacts().endpoints(), e -> !readPlan.contacts().endpoints().contains(e));
+ this.repairs = new PartitionUpdate.Builder[size + (buildFullDiff ? 1 : 0)];
+ this.currentRows = new Row.Builder[size];
+ this.sourceDeletionTime = new DeletionTime[size];
+ this.markerToRepair = new ClusteringBound[size];
+ this.command = command;
+ this.readRepair = readRepair;
+
+ this.diffListener = new RowDiffListener()
+ {
+ public void onPrimaryKeyLivenessInfo(int i, Clustering clustering, LivenessInfo merged, LivenessInfo original)
+ {
+ if (merged != null && !merged.equals(original))
+ currentRow(i, clustering).addPrimaryKeyLivenessInfo(merged);
+ }
+
+ public void onDeletion(int i, Clustering clustering, Row.Deletion merged, Row.Deletion original)
+ {
+ if (merged != null && !merged.equals(original))
+ currentRow(i, clustering).addRowDeletion(merged);
+ }
+
+ public void onComplexDeletion(int i, Clustering clustering, ColumnMetadata column, DeletionTime merged, DeletionTime original)
+ {
+ if (merged != null && !merged.equals(original))
+ currentRow(i, clustering).addComplexDeletion(column, merged);
+ }
+
+ public void onCell(int i, Clustering clustering, Cell merged, Cell original)
+ {
+ if (merged != null && !merged.equals(original) && isQueried(merged))
+ currentRow(i, clustering).addCell(merged);
+ }
+
+ private boolean isQueried(Cell cell)
+ {
+ // When we read, we may have some cell that have been fetched but are not selected by the user. Those cells may
+ // have empty values as optimization (see CASSANDRA-10655) and hence they should not be included in the read-repair.
+ // This is fine since those columns are not actually requested by the user and are only present for the sake of CQL
+ // semantic (making sure we can always distinguish between a row that doesn't exist from one that do exist but has
+ /// no value for the column requested by the user) and so it won't be unexpected by the user that those columns are
+ // not repaired.
+ ColumnMetadata column = cell.column();
+ ColumnFilter filter = RowIteratorMergeListener.this.command.columnFilter();
+ return column.isComplex() ? filter.fetchedCellIsQueried(column, cell.path()) : filter.fetchedColumnIsQueried(column);
+ }
+ };
+ }
+
+ /**
+ * The partition level deletion with with which source {@code i} is currently repaired, or
+ * {@code DeletionTime.LIVE} if the source is not repaired on the partition level deletion (meaning it was
+ * up to date on it). The output* of this method is only valid after the call to
+ * {@link #onMergedPartitionLevelDeletion}.
+ */
+ private DeletionTime partitionLevelRepairDeletion(int i)
+ {
+ return repairs[i] == null ? DeletionTime.LIVE : repairs[i].partitionLevelDeletion();
+ }
+
+ private Row.Builder currentRow(int i, Clustering clustering)
+ {
+ if (currentRows[i] == null)
+ {
+ currentRows[i] = BTreeRow.sortedBuilder();
+ currentRows[i].newRow(clustering);
+ }
+ return currentRows[i];
+ }
+
+ @Inline
+ private void applyToPartition(int i, Consumer<PartitionUpdate.Builder> f)
+ {
+ if (writeBackTo.get(i))
+ {
+ if (repairs[i] == null)
+ repairs[i] = new PartitionUpdate.Builder(command.metadata(), partitionKey, columns, 1);
+ f.accept(repairs[i]);
+ }
+ if (buildFullDiff)
+ {
+ if (repairs[repairs.length - 1] == null)
+ repairs[repairs.length - 1] = new PartitionUpdate.Builder(command.metadata(), partitionKey, columns, 1);
+ f.accept(repairs[repairs.length - 1]);
+ }
+ }
+
+ public void onMergedPartitionLevelDeletion(DeletionTime mergedDeletion, DeletionTime[] versions)
+ {
+ this.partitionLevelDeletion = mergedDeletion;
+ for (int i = 0; i < versions.length; i++)
+ {
+ if (mergedDeletion.supersedes(versions[i]))
+ applyToPartition(i, p -> p.addPartitionDeletion(mergedDeletion));
+ }
+ }
+
+ public Row onMergedRows(Row merged, Row[] versions)
+ {
+ // If a row was shadowed post merged, it must be by a partition level or range tombstone, and we handle
+ // those case directly in their respective methods (in other words, it would be inefficient to send a row
+ // deletion as repair when we know we've already send a partition level or range tombstone that covers it).
+ if (merged.isEmpty())
+ return merged;
+
+ Rows.diff(diffListener, merged, versions);
+ for (int i = 0; i < currentRows.length; i++)
+ {
+ if (currentRows[i] != null)
+ {
+ Row row = currentRows[i].build();
+ applyToPartition(i, p -> p.add(row));
+ }
+ }
+ Arrays.fill(currentRows, null);
+
+ return merged;
+ }
+
+ private DeletionTime currentDeletion()
+ {
+ return mergedDeletionTime == null ? partitionLevelDeletion : mergedDeletionTime;
+ }
+
+ public void onMergedRangeTombstoneMarkers(RangeTombstoneMarker merged, RangeTombstoneMarker[] versions)
+ {
+ // The current deletion as of dealing with this marker.
+ DeletionTime currentDeletion = currentDeletion();
+
+ for (int i = 0; i < versions.length; i++)
+ {
+ RangeTombstoneMarker marker = versions[i];
+
+ // Update what the source now thinks is the current deletion
+ if (marker != null)
+ sourceDeletionTime[i] = marker.isOpen(isReversed) ? marker.openDeletionTime(isReversed) : null;
+
+ // If merged == null, some of the source is opening or closing a marker
+ if (merged == null)
+ {
+ // but if it's not this source, move to the next one
+ if (marker == null)
+ continue;
+
+ // We have a close and/or open marker for a source, with nothing corresponding in merged.
+ // Because merged is a superset, this imply that we have a current deletion (being it due to an
+ // early opening in merged or a partition level deletion) and that this deletion will still be
+ // active after that point. Further whatever deletion was open or is open by this marker on the
+ // source, that deletion cannot supersedes the current one.
+ //
+ // But while the marker deletion (before and/or after this point) cannot supersede the current
+ // deletion, we want to know if it's equal to it (both before and after), because in that case
+ // the source is up to date and we don't want to include repair.
+ //
+ // So in practice we have 2 possible case:
+ // 1) the source was up-to-date on deletion up to that point: then it won't be from that point
+ // on unless it's a boundary and the new opened deletion time is also equal to the current
+ // deletion (note that this implies the boundary has the same closing and opening deletion
+ // time, which should generally not happen, but can due to legacy reading code not avoiding
+ // this for a while, see CASSANDRA-13237).
+ // 2) the source wasn't up-to-date on deletion up to that point and it may now be (if it isn't
+ // we just have nothing to do for that marker).
+ assert !currentDeletion.isLive() : currentDeletion.toString();
+
+ // Is the source up to date on deletion? It's up to date if it doesn't have an open RT repair
+ // nor an "active" partition level deletion (where "active" means that it's greater or equal
+ // to the current deletion: if the source has a repaired partition deletion lower than the
+ // current deletion, this means the current deletion is due to a previously open range tombstone,
+ // and if the source isn't currently repaired for that RT, then it means it's up to date on it).
+ DeletionTime partitionRepairDeletion = partitionLevelRepairDeletion(i);
+ if (markerToRepair[i] == null && currentDeletion.supersedes(partitionRepairDeletion))
+ {
+ /*
+ * Since there is an ongoing merged deletion, the only two ways we don't have an open repair for
+ * this source are that:
+ *
+ * 1) it had a range open with the same deletion as current marker, and the marker is coming from
+ * a short read protection response - repeating the open RT bound, or
+ * 2) it had a range open with the same deletion as current marker, and the marker is closing it.
+ */
+ if (!marker.isBoundary() && marker.isOpen(isReversed)) // (1)
+ {
+ assert currentDeletion.equals(marker.openDeletionTime(isReversed))
+ : String.format("currentDeletion=%s, marker=%s", currentDeletion, marker.toString(command.metadata()));
+ }
+ else // (2)
+ {
+ assert marker.isClose(isReversed) && currentDeletion.equals(marker.closeDeletionTime(isReversed))
+ : String.format("currentDeletion=%s, marker=%s", currentDeletion, marker.toString(command.metadata()));
+ }
+
+ // and so unless it's a boundary whose opening deletion time is still equal to the current
+ // deletion (see comment above for why this can actually happen), we have to repair the source
+ // from that point on.
+ if (!(marker.isOpen(isReversed) && currentDeletion.equals(marker.openDeletionTime(isReversed))))
+ markerToRepair[i] = marker.closeBound(isReversed).invert();
+ }
+ // In case 2) above, we only have something to do if the source is up-to-date after that point
+ // (which, since the source isn't up-to-date before that point, means we're opening a new deletion
+ // that is equal to the current one).
+ else if (marker.isOpen(isReversed) && currentDeletion.equals(marker.openDeletionTime(isReversed)))
+ {
+ closeOpenMarker(i, marker.openBound(isReversed).invert());
+ }
+ }
+ else
+ {
+ // We have a change of current deletion in merged (potentially to/from no deletion at all).
+
+ if (merged.isClose(isReversed))
+ {
+ // We're closing the merged range. If we're recorded that this should be repaird for the
+ // source, close and add said range to the repair to send.
+ if (markerToRepair[i] != null)
+ closeOpenMarker(i, merged.closeBound(isReversed));
+
+ }
+
+ if (merged.isOpen(isReversed))
+ {
+ // If we're opening a new merged range (or just switching deletion), then unless the source
+ // is up to date on that deletion (note that we've updated what the source deleteion is
+ // above), we'll have to sent the range to the source.
+ DeletionTime newDeletion = merged.openDeletionTime(isReversed);
+ DeletionTime sourceDeletion = sourceDeletionTime[i];
+ if (!newDeletion.equals(sourceDeletion))
+ markerToRepair[i] = merged.openBound(isReversed);
+ }
+ }
+ }
+
+ if (merged != null)
+ mergedDeletionTime = merged.isOpen(isReversed) ? merged.openDeletionTime(isReversed) : null;
+ }
+
+ private void closeOpenMarker(int i, ClusteringBound close)
+ {
+ ClusteringBound open = markerToRepair[i];
+ RangeTombstone rt = new RangeTombstone(Slice.make(isReversed ? close : open, isReversed ? open : close), currentDeletion());
+ applyToPartition(i, p -> p.add(rt));
+ markerToRepair[i] = null;
+ }
+
+ public void close()
+ {
+ boolean hasRepairs = false;
+ for (int i = 0 ; !hasRepairs && i < repairs.length ; ++i)
+ hasRepairs = repairs[i] != null;
+ if (!hasRepairs)
+ return;
+
+ PartitionUpdate fullDiffRepair = null;
+ if (buildFullDiff && repairs[repairs.length - 1] != null)
+ fullDiffRepair = repairs[repairs.length - 1].build();
+
+ Map<Replica, Mutation> mutations = Maps.newHashMapWithExpectedSize(writePlan.contacts().size());
+ ObjectIntHashMap<InetAddressAndPort> sourceIds = new ObjectIntHashMap<>(((repairs.length + 1) * 4) / 3);
+ for (int i = 0 ; i < readPlan.contacts().size() ; ++i)
+ sourceIds.put(readPlan.contacts().get(i).endpoint(), 1 + i);
+
+ for (Replica replica : writePlan.contacts())
+ {
+ PartitionUpdate update = null;
+ int i = -1 + sourceIds.get(replica.endpoint());
+ if (i < 0)
+ update = fullDiffRepair;
+ else if (repairs[i] != null)
+ update = repairs[i].build();
+
+ Mutation mutation = BlockingReadRepairs.createRepairMutation(update, readPlan.consistencyLevel(), replica.endpoint(), false);
+ if (mutation == null)
+ continue;
+
+ mutations.put(replica, mutation);
+ }
+
+ readRepair.repairPartition(partitionKey, mutations, writePlan);
+ }
+}
diff --git a/src/java/org/apache/cassandra/streaming/ConnectionHandler.java b/src/java/org/apache/cassandra/streaming/ConnectionHandler.java
deleted file mode 100644
index 556748d..0000000
--- a/src/java/org/apache/cassandra/streaming/ConnectionHandler.java
+++ /dev/null
@@ -1,423 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.streaming;
-
-import java.io.BufferedOutputStream;
-import java.io.IOException;
-import java.net.Socket;
-import java.net.SocketException;
-import java.nio.ByteBuffer;
-import java.nio.channels.Channels;
-import java.nio.channels.ReadableByteChannel;
-import java.nio.channels.WritableByteChannel;
-import java.util.Collection;
-import java.util.Comparator;
-import java.util.concurrent.PriorityBlockingQueue;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicReference;
-
-import com.google.common.util.concurrent.Futures;
-import com.google.common.util.concurrent.ListenableFuture;
-import com.google.common.util.concurrent.SettableFuture;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import io.netty.util.concurrent.FastThreadLocalThread;
-import org.apache.cassandra.io.util.DataOutputStreamPlus;
-import org.apache.cassandra.io.util.BufferedDataOutputStreamPlus;
-import org.apache.cassandra.io.util.WrappedDataOutputStreamPlus;
-import org.apache.cassandra.net.IncomingStreamingConnection;
-import org.apache.cassandra.streaming.messages.StreamInitMessage;
-import org.apache.cassandra.streaming.messages.StreamMessage;
-import org.apache.cassandra.utils.FBUtilities;
-import org.apache.cassandra.utils.JVMStabilityInspector;
-
-/**
- * ConnectionHandler manages incoming/outgoing message exchange for the {@link StreamSession}.
- *
- * <p>
- * Internally, ConnectionHandler manages thread to receive incoming {@link StreamMessage} and thread to
- * send outgoing message. Messages are encoded/decoded on those thread and handed to
- * {@link StreamSession#messageReceived(org.apache.cassandra.streaming.messages.StreamMessage)}.
- */
-public class ConnectionHandler
-{
- private static final Logger logger = LoggerFactory.getLogger(ConnectionHandler.class);
-
- private final StreamSession session;
-
- private IncomingMessageHandler incoming;
- private OutgoingMessageHandler outgoing;
-
- ConnectionHandler(StreamSession session, int incomingSocketTimeout)
- {
- this.session = session;
- this.incoming = new IncomingMessageHandler(session, incomingSocketTimeout);
- this.outgoing = new OutgoingMessageHandler(session);
- }
-
- /**
- * Set up incoming message handler and initiate streaming.
- *
- * This method is called once on initiator.
- *
- * @throws IOException
- */
- @SuppressWarnings("resource")
- public void initiate() throws IOException
- {
- logger.debug("[Stream #{}] Sending stream init for incoming stream", session.planId());
- Socket incomingSocket = session.createConnection();
- incoming.start(incomingSocket, StreamMessage.CURRENT_VERSION, true);
-
- logger.debug("[Stream #{}] Sending stream init for outgoing stream", session.planId());
- Socket outgoingSocket = session.createConnection();
- outgoing.start(outgoingSocket, StreamMessage.CURRENT_VERSION, true);
- }
-
- /**
- * Set up outgoing message handler on receiving side.
- *
- * @param connection Incoming connection to use for {@link OutgoingMessageHandler}.
- * @param version Streaming message version
- * @throws IOException
- */
- public void initiateOnReceivingSide(IncomingStreamingConnection connection, boolean isForOutgoing, int version) throws IOException
- {
- if (isForOutgoing)
- outgoing.start(connection, version);
- else
- incoming.start(connection, version);
- }
-
- public ListenableFuture<?> close()
- {
- logger.debug("[Stream #{}] Closing stream connection handler on {}", session.planId(), session.peer);
-
- ListenableFuture<?> inClosed = closeIncoming();
- ListenableFuture<?> outClosed = closeOutgoing();
-
- return Futures.allAsList(inClosed, outClosed);
- }
-
- public ListenableFuture<?> closeOutgoing()
- {
- return outgoing == null ? Futures.immediateFuture(null) : outgoing.close();
- }
-
- public ListenableFuture<?> closeIncoming()
- {
- return incoming == null ? Futures.immediateFuture(null) : incoming.close();
- }
-
- /**
- * Enqueue messages to be sent.
- *
- * @param messages messages to send
- */
- public void sendMessages(Collection<? extends StreamMessage> messages)
- {
- for (StreamMessage message : messages)
- sendMessage(message);
- }
-
- public void sendMessage(StreamMessage message)
- {
- if (outgoing.isClosed())
- throw new RuntimeException("Outgoing stream handler has been closed");
-
- outgoing.enqueue(message);
- }
-
- /**
- * @return true if outgoing connection is opened and ready to send messages
- */
- public boolean isOutgoingConnected()
- {
- return outgoing != null && !outgoing.isClosed();
- }
-
- abstract static class MessageHandler implements Runnable
- {
- protected final StreamSession session;
-
- protected int protocolVersion;
- private final boolean isOutgoingHandler;
- protected Socket socket;
-
- private final AtomicReference<SettableFuture<?>> closeFuture = new AtomicReference<>();
- private IncomingStreamingConnection incomingConnection;
-
- protected MessageHandler(StreamSession session, boolean isOutgoingHandler)
- {
- this.session = session;
- this.isOutgoingHandler = isOutgoingHandler;
- }
-
- protected abstract String name();
-
- @SuppressWarnings("resource")
- protected static DataOutputStreamPlus getWriteChannel(Socket socket) throws IOException
- {
- WritableByteChannel out = socket.getChannel();
- // socket channel is null when encrypted(SSL)
- if (out == null)
- return new WrappedDataOutputStreamPlus(new BufferedOutputStream(socket.getOutputStream()));
- return new BufferedDataOutputStreamPlus(out);
- }
-
- protected static ReadableByteChannel getReadChannel(Socket socket) throws IOException
- {
- //we do this instead of socket.getChannel() so socketSoTimeout is respected
- return Channels.newChannel(socket.getInputStream());
- }
-
- @SuppressWarnings("resource")
- private void sendInitMessage() throws IOException
- {
- StreamInitMessage message = new StreamInitMessage(
- FBUtilities.getBroadcastAddress(),
- session.sessionIndex(),
- session.planId(),
- session.description(),
- !isOutgoingHandler,
- session.keepSSTableLevel(),
- session.isIncremental());
- ByteBuffer messageBuf = message.createMessage(false, protocolVersion);
- DataOutputStreamPlus out = getWriteChannel(socket);
- out.write(messageBuf);
- out.flush();
- }
-
- public void start(IncomingStreamingConnection connection, int protocolVersion) throws IOException
- {
- this.incomingConnection = connection;
- start(connection.socket, protocolVersion, false);
- }
-
- public void start(Socket socket, int protocolVersion, boolean initiator) throws IOException
- {
- this.socket = socket;
- this.protocolVersion = protocolVersion;
- if (initiator)
- sendInitMessage();
-
- new FastThreadLocalThread(this, name() + "-" + socket.getRemoteSocketAddress()).start();
- }
-
- public ListenableFuture<?> close()
- {
- // Assume it wasn't closed. Not a huge deal if we create a future on a race
- SettableFuture<?> future = SettableFuture.create();
- return closeFuture.compareAndSet(null, future)
- ? future
- : closeFuture.get();
- }
-
- public boolean isClosed()
- {
- return closeFuture.get() != null;
- }
-
- protected void signalCloseDone()
- {
- if (!isClosed())
- close();
-
- closeFuture.get().set(null);
-
- // We can now close the socket
- if (incomingConnection != null)
- {
- //this will close the underlying socket and remove it
- //from active MessagingService connections (CASSANDRA-11854)
- incomingConnection.close();
- }
- else
- {
- //this is an outgoing connection not registered in the MessagingService
- //so we can close the socket directly
- try
- {
- socket.close();
- }
- catch (IOException e)
- {
- // Erroring out while closing shouldn't happen but is not really a big deal, so just log
- // it at DEBUG and ignore otherwise.
- logger.debug("Unexpected error while closing streaming connection", e);
- }
- }
- }
- }
-
- /**
- * Incoming streaming message handler
- */
- static class IncomingMessageHandler extends MessageHandler
- {
- private final int socketTimeout;
-
- IncomingMessageHandler(StreamSession session, int socketTimeout)
- {
- super(session, false);
- this.socketTimeout = socketTimeout;
- }
-
- @Override
- public void start(Socket socket, int version, boolean initiator) throws IOException
- {
- try
- {
- socket.setSoTimeout(socketTimeout);
- }
- catch (SocketException e)
- {
- logger.warn("Could not set incoming socket timeout to {}", socketTimeout, e);
- }
- super.start(socket, version, initiator);
- }
-
- protected String name()
- {
- return "STREAM-IN";
- }
-
- @SuppressWarnings("resource")
- public void run()
- {
- try
- {
- ReadableByteChannel in = getReadChannel(socket);
- while (!isClosed())
- {
- // receive message
- StreamMessage message = StreamMessage.deserialize(in, protocolVersion, session);
- logger.debug("[Stream #{}] Received {}", session.planId(), message);
- // Might be null if there is an error during streaming (see FileMessage.deserialize). It's ok
- // to ignore here since we'll have asked for a retry.
- if (message != null)
- {
- session.messageReceived(message);
- }
- }
- }
- catch (Throwable t)
- {
- JVMStabilityInspector.inspectThrowable(t);
- session.onError(t);
- }
- finally
- {
- signalCloseDone();
- }
- }
- }
-
- /**
- * Outgoing file transfer thread
- */
- static class OutgoingMessageHandler extends MessageHandler
- {
- /*
- * All out going messages are queued up into messageQueue.
- * The size will grow when received streaming request.
- *
- * Queue is also PriorityQueue so that prior messages can go out fast.
- */
- private final PriorityBlockingQueue<StreamMessage> messageQueue = new PriorityBlockingQueue<>(64, new Comparator<StreamMessage>()
- {
- public int compare(StreamMessage o1, StreamMessage o2)
- {
- return o2.getPriority() - o1.getPriority();
- }
- });
-
- OutgoingMessageHandler(StreamSession session)
- {
- super(session, true);
- }
-
- protected String name()
- {
- return "STREAM-OUT";
- }
-
- public void enqueue(StreamMessage message)
- {
- messageQueue.put(message);
- }
-
- @SuppressWarnings("resource")
- public void run()
- {
- try
- {
- DataOutputStreamPlus out = getWriteChannel(socket);
-
- StreamMessage next;
- while (!isClosed())
- {
- if ((next = messageQueue.poll(1, TimeUnit.SECONDS)) != null)
- {
- logger.debug("[Stream #{}] Sending {}", session.planId(), next);
- sendMessage(out, next);
- if (next.type == StreamMessage.Type.SESSION_FAILED)
- close();
- }
- }
-
- // Sends the last messages on the queue
- while ((next = messageQueue.poll()) != null)
- sendMessage(out, next);
- }
- catch (InterruptedException e)
- {
- throw new AssertionError(e);
- }
- catch (Throwable e)
- {
- session.onError(e);
- }
- finally
- {
- signalCloseDone();
- }
- }
-
- private void sendMessage(DataOutputStreamPlus out, StreamMessage message)
- {
- try
- {
- StreamMessage.serialize(message, out, protocolVersion, session);
- out.flush();
- message.sent();
- }
- catch (SocketException e)
- {
- session.onError(e);
- close();
- }
- catch (IOException e)
- {
- session.onError(e);
- }
- }
- }
-}
diff --git a/src/java/org/apache/cassandra/streaming/DefaultConnectionFactory.java b/src/java/org/apache/cassandra/streaming/DefaultConnectionFactory.java
index 7e9dfd3..5f2163f 100644
--- a/src/java/org/apache/cassandra/streaming/DefaultConnectionFactory.java
+++ b/src/java/org/apache/cassandra/streaming/DefaultConnectionFactory.java
@@ -15,66 +15,44 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.cassandra.streaming;
import java.io.IOException;
-import java.net.InetAddress;
-import java.net.Socket;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import com.google.common.annotations.VisibleForTesting;
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.net.OutboundTcpConnectionPool;
+import io.netty.channel.Channel;
+import io.netty.channel.EventLoop;
+import io.netty.util.concurrent.Future;
+import org.apache.cassandra.net.ConnectionCategory;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.net.OutboundConnectionInitiator.Result;
+import org.apache.cassandra.net.OutboundConnectionInitiator.Result.StreamingSuccess;
+import org.apache.cassandra.net.OutboundConnectionSettings;
+
+import static org.apache.cassandra.net.OutboundConnectionInitiator.initiateStreaming;
public class DefaultConnectionFactory implements StreamConnectionFactory
{
- private static final Logger logger = LoggerFactory.getLogger(DefaultConnectionFactory.class);
+ @VisibleForTesting
+ public static int MAX_CONNECT_ATTEMPTS = 3;
- private static final int MAX_CONNECT_ATTEMPTS = 3;
-
- /**
- * Connect to peer and start exchanging message.
- * When connect attempt fails, this retries for maximum of MAX_CONNECT_ATTEMPTS times.
- *
- * @param peer the peer to connect to.
- * @return the created socket.
- *
- * @throws IOException when connection failed.
- */
- public Socket createConnection(InetAddress peer) throws IOException
+ @Override
+ public Channel createConnection(OutboundConnectionSettings template, int messagingVersion) throws IOException
{
+ EventLoop eventLoop = MessagingService.instance().socketFactory.outboundStreamingGroup().next();
+
int attempts = 0;
while (true)
{
- Socket socket = null;
- try
- {
- socket = OutboundTcpConnectionPool.newSocket(peer);
- socket.setSoTimeout(DatabaseDescriptor.getStreamingSocketTimeout());
- socket.setKeepAlive(true);
- return socket;
- }
- catch (IOException e)
- {
- if (socket != null)
- {
- socket.close();
- }
- if (++attempts >= MAX_CONNECT_ATTEMPTS)
- throw e;
+ Future<Result<StreamingSuccess>> result = initiateStreaming(eventLoop, template.withDefaults(ConnectionCategory.STREAMING), messagingVersion);
+ result.awaitUninterruptibly(); // initiate has its own timeout, so this is "guaranteed" to return relatively promptly
+ if (result.isSuccess())
+ return result.getNow().success().channel;
- long waitms = DatabaseDescriptor.getRpcTimeout() * (long)Math.pow(2, attempts);
- logger.warn("Failed attempt {} to connect to {}. Retrying in {} ms. ({})", attempts, peer, waitms, e);
- try
- {
- Thread.sleep(waitms);
- }
- catch (InterruptedException wtf)
- {
- throw new IOException("interrupted", wtf);
- }
- }
+ if (++attempts == MAX_CONNECT_ATTEMPTS)
+ throw new IOException("failed to connect to " + template.to + " for streaming data", result.cause());
}
}
}
diff --git a/src/java/org/apache/cassandra/streaming/IncomingStream.java b/src/java/org/apache/cassandra/streaming/IncomingStream.java
new file mode 100644
index 0000000..0733249
--- /dev/null
+++ b/src/java/org/apache/cassandra/streaming/IncomingStream.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.streaming;
+
+import java.io.IOException;
+
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.schema.TableId;
+
+/**
+ * The counterpart of {@link OutgoingStream} on the receiving side.
+ *
+ * Data streamed in can (and should) be persisted, but must not be included in the table's
+ * live data set until added by {@link StreamReceiver}. If the stream fails, the stream receiver will
+ * delete the streamed data, but implementations still need to handle the case where it's process dies
+ * during streaming, and it has data left around on startup, in which case it should be deleted.
+ */
+public interface IncomingStream
+{
+
+ /**
+ * Read in the stream data.
+ */
+ void read(DataInputPlus inputPlus, int version) throws IOException;
+
+ String getName();
+ long getSize();
+ int getNumFiles();
+ TableId getTableId();
+
+ /**
+ * @return stream session used to receive given file
+ */
+ StreamSession session();
+}
diff --git a/src/java/org/apache/cassandra/streaming/OutgoingStream.java b/src/java/org/apache/cassandra/streaming/OutgoingStream.java
new file mode 100644
index 0000000..4a58cae
--- /dev/null
+++ b/src/java/org/apache/cassandra/streaming/OutgoingStream.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.streaming;
+
+import java.io.IOException;
+import java.util.UUID;
+
+import org.apache.cassandra.io.util.DataOutputStreamPlus;
+import org.apache.cassandra.schema.TableId;
+
+/**
+ * Some subset of data to be streamed. Implementations handle writing out their data via the write method.
+ * On the receiving end, {@link IncomingStream} streams the data in.
+ *
+ * All the data contained in a given stream needs to have the same repairedAt timestamp (or 0) and pendingRepair
+ * id (or null).
+ */
+public interface OutgoingStream
+{
+ /**
+ * Write the streams data into the socket
+ */
+ void write(StreamSession session, DataOutputStreamPlus output, int version) throws IOException;
+
+ /**
+ * Release any resources held by the stream
+ */
+ void finish();
+
+ long getRepairedAt();
+ UUID getPendingRepair();
+
+ String getName();
+ long getSize();
+ TableId getTableId();
+ int getNumFiles();
+}
diff --git a/src/java/org/apache/cassandra/streaming/PreviewKind.java b/src/java/org/apache/cassandra/streaming/PreviewKind.java
new file mode 100644
index 0000000..b5467de
--- /dev/null
+++ b/src/java/org/apache/cassandra/streaming/PreviewKind.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.streaming;
+
+import java.util.UUID;
+
+import com.google.common.base.Predicate;
+import com.google.common.base.Predicates;
+
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
+import org.apache.cassandra.repair.consistent.ConsistentSession;
+import org.apache.cassandra.repair.consistent.LocalSession;
+import org.apache.cassandra.service.ActiveRepairService;
+
+public enum PreviewKind
+{
+ NONE(0, (sstable) -> {
+ throw new RuntimeException("Can't get preview predicate for preview kind NONE");
+ }),
+ ALL(1, Predicates.alwaysTrue()),
+ UNREPAIRED(2, sstable -> !sstable.isRepaired()),
+ REPAIRED(3, new PreviewRepairedSSTablePredicate());
+
+ private final int serializationVal;
+ private final Predicate<SSTableReader> predicate;
+
+ PreviewKind(int serializationVal, Predicate<SSTableReader> predicate)
+ {
+ assert ordinal() == serializationVal;
+ this.serializationVal = serializationVal;
+ this.predicate = predicate;
+ }
+
+ public int getSerializationVal()
+ {
+ return serializationVal;
+ }
+
+ public static PreviewKind deserialize(int serializationVal)
+ {
+ return values()[serializationVal];
+ }
+
+ public boolean isPreview()
+ {
+ return this != NONE;
+ }
+
+ public String logPrefix()
+ {
+ return isPreview() ? "preview repair" : "repair";
+ }
+
+ public String logPrefix(UUID sessionId)
+ {
+ return '[' + logPrefix() + " #" + sessionId.toString() + ']';
+ }
+
+ public Predicate<SSTableReader> predicate()
+ {
+ return predicate;
+ }
+
+ private static class PreviewRepairedSSTablePredicate implements Predicate<SSTableReader>
+ {
+ public boolean apply(SSTableReader sstable)
+ {
+ // grab the metadata before checking pendingRepair since this can be nulled out at any time
+ StatsMetadata sstableMetadata = sstable.getSSTableMetadata();
+ if (sstableMetadata.pendingRepair != null)
+ {
+ LocalSession session = ActiveRepairService.instance.consistent.local.getSession(sstableMetadata.pendingRepair);
+ if (session == null)
+ return false;
+ else if (session.getState() == ConsistentSession.State.FINALIZED)
+ return true;
+ else if (session.getState() != ConsistentSession.State.FAILED)
+ throw new IllegalStateException(String.format("SSTable %s is marked pending for non-finalized incremental repair session %s, failing preview repair", sstable, sstableMetadata.pendingRepair));
+ }
+ return sstableMetadata.repairedAt != ActiveRepairService.UNREPAIRED_SSTABLE;
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/streaming/ProgressInfo.java b/src/java/org/apache/cassandra/streaming/ProgressInfo.java
index fdd3e97..ac91855 100644
--- a/src/java/org/apache/cassandra/streaming/ProgressInfo.java
+++ b/src/java/org/apache/cassandra/streaming/ProgressInfo.java
@@ -18,12 +18,13 @@
package org.apache.cassandra.streaming;
import java.io.Serializable;
-import java.net.InetAddress;
import com.google.common.base.Objects;
+import org.apache.cassandra.locator.InetAddressAndPort;
+
/**
- * ProgressInfo contains file transfer progress.
+ * ProgressInfo contains stream transfer progress.
*/
public class ProgressInfo implements Serializable
{
@@ -48,14 +49,14 @@
}
}
- public final InetAddress peer;
+ public final InetAddressAndPort peer;
public final int sessionIndex;
public final String fileName;
public final Direction direction;
public final long currentBytes;
public final long totalBytes;
- public ProgressInfo(InetAddress peer, int sessionIndex, String fileName, Direction direction, long currentBytes, long totalBytes)
+ public ProgressInfo(InetAddressAndPort peer, int sessionIndex, String fileName, Direction direction, long currentBytes, long totalBytes)
{
assert totalBytes > 0;
@@ -68,7 +69,7 @@
}
/**
- * @return true if file transfer is completed
+ * @return true if transfer is completed
*/
public boolean isCompleted()
{
@@ -102,13 +103,18 @@
@Override
public String toString()
{
+ return toString(false);
+ }
+
+ public String toString(boolean withPorts)
+ {
StringBuilder sb = new StringBuilder(fileName);
sb.append(" ").append(currentBytes);
sb.append("/").append(totalBytes).append(" bytes");
sb.append("(").append(currentBytes*100/totalBytes).append("%) ");
sb.append(direction == Direction.OUT ? "sent to " : "received from ");
sb.append("idx:").append(sessionIndex);
- sb.append(peer);
+ sb.append(peer.toString(withPorts));
return sb.toString();
}
}
diff --git a/src/java/org/apache/cassandra/streaming/ReplicationDoneVerbHandler.java b/src/java/org/apache/cassandra/streaming/ReplicationDoneVerbHandler.java
new file mode 100644
index 0000000..7d73b11
--- /dev/null
+++ b/src/java/org/apache/cassandra/streaming/ReplicationDoneVerbHandler.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.streaming;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.net.IVerbHandler;
+import org.apache.cassandra.net.Message;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.service.StorageService;
+
+public class ReplicationDoneVerbHandler implements IVerbHandler
+{
+ public static ReplicationDoneVerbHandler instance = new ReplicationDoneVerbHandler();
+
+ private static final Logger logger = LoggerFactory.getLogger(ReplicationDoneVerbHandler.class);
+
+ public void doVerb(Message msg)
+ {
+ StorageService.instance.confirmReplication(msg.from());
+ logger.debug("Replying to {}@{}", msg.id(), msg.from());
+ MessagingService.instance().send(msg.emptyResponse(), msg.from());
+ }
+}
diff --git a/src/java/org/apache/cassandra/streaming/ReplicationFinishedVerbHandler.java b/src/java/org/apache/cassandra/streaming/ReplicationFinishedVerbHandler.java
deleted file mode 100644
index ce8a921..0000000
--- a/src/java/org/apache/cassandra/streaming/ReplicationFinishedVerbHandler.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.streaming;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.cassandra.net.IVerbHandler;
-import org.apache.cassandra.net.MessageIn;
-import org.apache.cassandra.net.MessageOut;
-import org.apache.cassandra.net.MessagingService;
-import org.apache.cassandra.service.StorageService;
-
-public class ReplicationFinishedVerbHandler implements IVerbHandler
-{
- private static final Logger logger = LoggerFactory.getLogger(ReplicationFinishedVerbHandler.class);
-
- public void doVerb(MessageIn msg, int id)
- {
- StorageService.instance.confirmReplication(msg.from);
- MessageOut response = new MessageOut(MessagingService.Verb.INTERNAL_RESPONSE);
- if (logger.isDebugEnabled())
- logger.debug("Replying to {}@{}", id, msg.from);
- MessagingService.instance().sendReply(response, id, msg.from);
- }
-}
diff --git a/src/java/org/apache/cassandra/streaming/SessionInfo.java b/src/java/org/apache/cassandra/streaming/SessionInfo.java
index 3bcb20c..4b4bbed 100644
--- a/src/java/org/apache/cassandra/streaming/SessionInfo.java
+++ b/src/java/org/apache/cassandra/streaming/SessionInfo.java
@@ -18,7 +18,6 @@
package org.apache.cassandra.streaming;
import java.io.Serializable;
-import java.net.InetAddress;
import java.util.Collection;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
@@ -27,14 +26,17 @@
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.utils.FBUtilities;
+
/**
* Stream session info.
*/
public final class SessionInfo implements Serializable
{
- public final InetAddress peer;
+ public final InetAddressAndPort peer;
public final int sessionIndex;
- public final InetAddress connecting;
+ public final InetAddressAndPort connecting;
/** Immutable collection of receiving summaries */
public final Collection<StreamSummary> receivingSummaries;
/** Immutable collection of sending summaries*/
@@ -45,9 +47,9 @@
private final Map<String, ProgressInfo> receivingFiles;
private final Map<String, ProgressInfo> sendingFiles;
- public SessionInfo(InetAddress peer,
+ public SessionInfo(InetAddressAndPort peer,
int sessionIndex,
- InetAddress connecting,
+ InetAddressAndPort connecting,
Collection<StreamSummary> receivingSummaries,
Collection<StreamSummary> sendingSummaries,
StreamSession.State state)
@@ -68,7 +70,7 @@
}
/**
- * Update progress of receiving/sending file.
+ * Update progress of receiving/sending stream.
*
* @param newProgress new progress info
*/
@@ -155,11 +157,11 @@
return getTotalSizes(sendingSummaries);
}
- private long getTotalSizeInProgress(Collection<ProgressInfo> files)
+ private long getTotalSizeInProgress(Collection<ProgressInfo> streams)
{
long total = 0;
- for (ProgressInfo file : files)
- total += file.currentBytes;
+ for (ProgressInfo stream : streams)
+ total += stream.currentBytes;
return total;
}
@@ -190,4 +192,9 @@
});
return Iterables.size(completed);
}
+
+ public SessionSummary createSummary()
+ {
+ return new SessionSummary(FBUtilities.getBroadcastAddressAndPort(), peer, receivingSummaries, sendingSummaries);
+ }
}
diff --git a/src/java/org/apache/cassandra/streaming/SessionSummary.java b/src/java/org/apache/cassandra/streaming/SessionSummary.java
new file mode 100644
index 0000000..5b168a0
--- /dev/null
+++ b/src/java/org/apache/cassandra/streaming/SessionSummary.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.streaming;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+import org.apache.cassandra.db.TypeSizes;
+import org.apache.cassandra.io.IVersionedSerializer;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.InetAddressAndPort.Serializer;
+
+import static org.apache.cassandra.locator.InetAddressAndPort.Serializer.inetAddressAndPortSerializer;
+
+public class SessionSummary
+{
+ public final InetAddressAndPort coordinator;
+ public final InetAddressAndPort peer;
+ /** Immutable collection of receiving summaries */
+ public final Collection<StreamSummary> receivingSummaries;
+ /** Immutable collection of sending summaries*/
+ public final Collection<StreamSummary> sendingSummaries;
+
+ public SessionSummary(InetAddressAndPort coordinator, InetAddressAndPort peer,
+ Collection<StreamSummary> receivingSummaries,
+ Collection<StreamSummary> sendingSummaries)
+ {
+ assert coordinator != null;
+ assert peer != null;
+ assert receivingSummaries != null;
+ assert sendingSummaries != null;
+
+ this.coordinator = coordinator;
+ this.peer = peer;
+ this.receivingSummaries = receivingSummaries;
+ this.sendingSummaries = sendingSummaries;
+ }
+
+ public boolean equals(Object o)
+ {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ SessionSummary summary = (SessionSummary) o;
+
+ if (!coordinator.equals(summary.coordinator)) return false;
+ if (!peer.equals(summary.peer)) return false;
+ if (!receivingSummaries.equals(summary.receivingSummaries)) return false;
+ return sendingSummaries.equals(summary.sendingSummaries);
+ }
+
+ public int hashCode()
+ {
+ int result = coordinator.hashCode();
+ result = 31 * result + peer.hashCode();
+ result = 31 * result + receivingSummaries.hashCode();
+ result = 31 * result + sendingSummaries.hashCode();
+ return result;
+ }
+
+ public static IVersionedSerializer<SessionSummary> serializer = new IVersionedSerializer<SessionSummary>()
+ {
+ public void serialize(SessionSummary summary, DataOutputPlus out, int version) throws IOException
+ {
+ inetAddressAndPortSerializer.serialize(summary.coordinator, out, version);
+ inetAddressAndPortSerializer.serialize(summary.peer, out, version);
+
+ out.writeInt(summary.receivingSummaries.size());
+ for (StreamSummary streamSummary: summary.receivingSummaries)
+ {
+ StreamSummary.serializer.serialize(streamSummary, out, version);
+ }
+
+ out.writeInt(summary.sendingSummaries.size());
+ for (StreamSummary streamSummary: summary.sendingSummaries)
+ {
+ StreamSummary.serializer.serialize(streamSummary, out, version);
+ }
+ }
+
+ public SessionSummary deserialize(DataInputPlus in, int version) throws IOException
+ {
+ InetAddressAndPort coordinator = inetAddressAndPortSerializer.deserialize(in, version);
+ InetAddressAndPort peer = inetAddressAndPortSerializer.deserialize(in, version);
+
+ int numRcvd = in.readInt();
+ List<StreamSummary> receivingSummaries = new ArrayList<>(numRcvd);
+ for (int i=0; i<numRcvd; i++)
+ {
+ receivingSummaries.add(StreamSummary.serializer.deserialize(in, version));
+ }
+
+ int numSent = in.readInt();
+ List<StreamSummary> sendingSummaries = new ArrayList<>(numRcvd);
+ for (int i=0; i<numSent; i++)
+ {
+ sendingSummaries.add(StreamSummary.serializer.deserialize(in, version));
+ }
+
+ return new SessionSummary(coordinator, peer, receivingSummaries, sendingSummaries);
+ }
+
+ public long serializedSize(SessionSummary summary, int version)
+ {
+ long size = 0;
+ size += inetAddressAndPortSerializer.serializedSize(summary.coordinator, version);
+ size += inetAddressAndPortSerializer.serializedSize(summary.peer, version);
+
+ size += TypeSizes.sizeof(summary.receivingSummaries.size());
+ for (StreamSummary streamSummary: summary.receivingSummaries)
+ {
+ size += StreamSummary.serializer.serializedSize(streamSummary, version);
+ }
+ size += TypeSizes.sizeof(summary.sendingSummaries.size());
+ for (StreamSummary streamSummary: summary.sendingSummaries)
+ {
+ size += StreamSummary.serializer.serializedSize(streamSummary, version);
+ }
+ return size;
+ }
+ };
+}
diff --git a/src/java/org/apache/cassandra/streaming/StreamConnectionFactory.java b/src/java/org/apache/cassandra/streaming/StreamConnectionFactory.java
index dd99611..95208e4 100644
--- a/src/java/org/apache/cassandra/streaming/StreamConnectionFactory.java
+++ b/src/java/org/apache/cassandra/streaming/StreamConnectionFactory.java
@@ -15,16 +15,15 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.cassandra.streaming;
import java.io.IOException;
-import java.net.InetAddress;
-import java.net.Socket;
-/**
- * Interface that creates connection used by streaming.
- */
+import io.netty.channel.Channel;
+import org.apache.cassandra.net.OutboundConnectionSettings;
+
public interface StreamConnectionFactory
{
- Socket createConnection(InetAddress peer) throws IOException;
+ Channel createConnection(OutboundConnectionSettings template, int messagingVersion) throws IOException;
}
diff --git a/src/java/org/apache/cassandra/streaming/StreamCoordinator.java b/src/java/org/apache/cassandra/streaming/StreamCoordinator.java
index b801ecc..e590e96 100644
--- a/src/java/org/apache/cassandra/streaming/StreamCoordinator.java
+++ b/src/java/org/apache/cassandra/streaming/StreamCoordinator.java
@@ -17,20 +17,14 @@
*/
package org.apache.cassandra.streaming;
-import java.net.InetAddress;
import java.util.*;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.ImmutableList;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.cassandra.concurrent.DebuggableThreadPoolExecutor;
-import org.apache.cassandra.utils.ExecutorUtils;
-import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.locator.InetAddressAndPort;
+
/**
* {@link StreamCoordinator} is a helper class that abstracts away maintaining multiple
@@ -43,27 +37,27 @@
{
private static final Logger logger = LoggerFactory.getLogger(StreamCoordinator.class);
- // Executor strictly for establishing the initial connections. Once we're connected to the other end the rest of the
- // streaming is handled directly by the ConnectionHandler's incoming and outgoing threads.
- private static final DebuggableThreadPoolExecutor streamExecutor = DebuggableThreadPoolExecutor.createWithFixedPoolSize("StreamConnectionEstablisher",
- FBUtilities.getAvailableProcessors());
private final boolean connectSequentially;
- private Map<InetAddress, HostStreamingData> peerSessions = new HashMap<>();
+ private final Map<InetAddressAndPort, HostStreamingData> peerSessions = new HashMap<>();
+ private final StreamOperation streamOperation;
private final int connectionsPerHost;
+ private final boolean follower;
private StreamConnectionFactory factory;
- private final boolean keepSSTableLevel;
- private final boolean isIncremental;
private Iterator<StreamSession> sessionsToConnect = null;
+ private final UUID pendingRepair;
+ private final PreviewKind previewKind;
- public StreamCoordinator(int connectionsPerHost, boolean keepSSTableLevel, boolean isIncremental,
- StreamConnectionFactory factory, boolean connectSequentially)
+ public StreamCoordinator(StreamOperation streamOperation, int connectionsPerHost, StreamConnectionFactory factory,
+ boolean follower, boolean connectSequentially, UUID pendingRepair, PreviewKind previewKind)
{
+ this.streamOperation = streamOperation;
this.connectionsPerHost = connectionsPerHost;
this.factory = factory;
- this.keepSSTableLevel = keepSSTableLevel;
- this.isIncremental = isIncremental;
+ this.follower = follower;
this.connectSequentially = connectSequentially;
+ this.pendingRepair = pendingRepair;
+ this.previewKind = previewKind;
}
public void setConnectionFactory(StreamConnectionFactory factory)
@@ -94,9 +88,9 @@
return results;
}
- public boolean isReceiving()
+ public boolean isFollower()
{
- return connectionsPerHost == 0;
+ return follower;
}
public void connect(StreamResultFuture future)
@@ -145,26 +139,32 @@
if (sessionsToConnect.hasNext())
{
StreamSession next = sessionsToConnect.next();
- logger.debug("Connecting next session {} with {}.", next.planId(), next.peer.getHostAddress());
- streamExecutor.execute(new StreamSessionConnector(next));
+ if (logger.isDebugEnabled())
+ logger.debug("Connecting next session {} with {}.", next.planId(), next.peer.toString());
+ startSession(next);
}
else
logger.debug("Finished connecting all sessions");
}
- public synchronized Set<InetAddress> getPeers()
+ public synchronized Set<InetAddressAndPort> getPeers()
{
return new HashSet<>(peerSessions.keySet());
}
- public synchronized StreamSession getOrCreateNextSession(InetAddress peer, InetAddress connecting)
+ public synchronized StreamSession getOrCreateNextSession(InetAddressAndPort peer)
{
- return getOrCreateHostData(peer).getOrCreateNextSession(peer, connecting);
+ return getOrCreateHostData(peer).getOrCreateNextSession(peer);
}
- public synchronized StreamSession getOrCreateSessionById(InetAddress peer, int id, InetAddress connecting)
+ public synchronized StreamSession getOrCreateSessionById(InetAddressAndPort peer, int id)
{
- return getOrCreateHostData(peer).getOrCreateSessionById(peer, id, connecting);
+ return getOrCreateHostData(peer).getOrCreateSessionById(peer, id);
+ }
+
+ public StreamSession getSessionById(InetAddressAndPort peer, int id)
+ {
+ return getHostData(peer).getSessionById(id);
}
public synchronized void updateProgress(ProgressInfo info)
@@ -188,63 +188,60 @@
return result;
}
- public synchronized void transferFiles(InetAddress to, Collection<StreamSession.SSTableStreamingSections> sstableDetails)
+ public synchronized void transferStreams(InetAddressAndPort to, Collection<OutgoingStream> streams)
{
HostStreamingData sessionList = getOrCreateHostData(to);
if (connectionsPerHost > 1)
{
- List<List<StreamSession.SSTableStreamingSections>> buckets = sliceSSTableDetails(sstableDetails);
+ List<Collection<OutgoingStream>> buckets = bucketStreams(streams);
- for (List<StreamSession.SSTableStreamingSections> subList : buckets)
+ for (Collection<OutgoingStream> bucket : buckets)
{
- StreamSession session = sessionList.getOrCreateNextSession(to, to);
- session.addTransferFiles(subList);
+ StreamSession session = sessionList.getOrCreateNextSession(to);
+ session.addTransferStreams(bucket);
}
}
else
{
- StreamSession session = sessionList.getOrCreateNextSession(to, to);
- session.addTransferFiles(sstableDetails);
+ StreamSession session = sessionList.getOrCreateNextSession(to);
+ session.addTransferStreams(streams);
}
}
- private List<List<StreamSession.SSTableStreamingSections>> sliceSSTableDetails(Collection<StreamSession.SSTableStreamingSections> sstableDetails)
+ private List<Collection<OutgoingStream>> bucketStreams(Collection<OutgoingStream> streams)
{
// There's no point in divvying things up into more buckets than we have sstableDetails
- int targetSlices = Math.min(sstableDetails.size(), connectionsPerHost);
- int step = Math.round((float) sstableDetails.size() / (float) targetSlices);
+ int targetSlices = Math.min(streams.size(), connectionsPerHost);
+ int step = Math.round((float) streams.size() / (float) targetSlices);
int index = 0;
- List<List<StreamSession.SSTableStreamingSections>> result = new ArrayList<>();
- List<StreamSession.SSTableStreamingSections> slice = null;
- Iterator<StreamSession.SSTableStreamingSections> iter = sstableDetails.iterator();
- while (iter.hasNext())
- {
- StreamSession.SSTableStreamingSections streamSession = iter.next();
+ List<Collection<OutgoingStream>> result = new ArrayList<>();
+ List<OutgoingStream> slice = null;
+ for (OutgoingStream stream: streams)
+ {
if (index % step == 0)
{
slice = new ArrayList<>();
result.add(slice);
}
- slice.add(streamSession);
+ slice.add(stream);
++index;
- iter.remove();
}
-
return result;
}
- private HostStreamingData getHostData(InetAddress peer)
+ private HostStreamingData getHostData(InetAddressAndPort peer)
{
HostStreamingData data = peerSessions.get(peer);
+
if (data == null)
throw new IllegalArgumentException("Unknown peer requested: " + peer);
return data;
}
- private HostStreamingData getOrCreateHostData(InetAddress peer)
+ private HostStreamingData getOrCreateHostData(InetAddressAndPort peer)
{
HostStreamingData data = peerSessions.get(peer);
if (data == null)
@@ -255,26 +252,21 @@
return data;
}
- private static class StreamSessionConnector implements Runnable
+ public UUID getPendingRepair()
{
- private final StreamSession session;
- public StreamSessionConnector(StreamSession session)
- {
- this.session = session;
- }
+ return pendingRepair;
+ }
- @Override
- public void run()
- {
- session.start();
- logger.info("[Stream #{}, ID#{}] Beginning stream session with {}", session.planId(), session.sessionIndex(), session.peer);
- }
+ private void startSession(StreamSession session)
+ {
+ session.start();
+ logger.info("[Stream #{}, ID#{}] Beginning stream session with {}", session.planId(), session.sessionIndex(), session.peer);
}
private class HostStreamingData
{
- private Map<Integer, StreamSession> streamSessions = new HashMap<>();
- private Map<Integer, SessionInfo> sessionInfos = new HashMap<>();
+ private final Map<Integer, StreamSession> streamSessions = new HashMap<>();
+ private final Map<Integer, SessionInfo> sessionInfos = new HashMap<>();
private int lastReturned = -1;
@@ -282,19 +274,19 @@
{
for (StreamSession session : streamSessions.values())
{
- StreamSession.State state = session.state();
- if (!state.isFinalState())
+ if (!session.state().isFinalState())
return true;
}
return false;
}
- public StreamSession getOrCreateNextSession(InetAddress peer, InetAddress connecting)
+ public StreamSession getOrCreateNextSession(InetAddressAndPort peer)
{
// create
if (streamSessions.size() < connectionsPerHost)
{
- StreamSession session = new StreamSession(peer, connecting, factory, streamSessions.size(), keepSSTableLevel, isIncremental);
+ StreamSession session = new StreamSession(streamOperation, peer, factory, isFollower(), streamSessions.size(),
+ pendingRepair, previewKind);
streamSessions.put(++lastReturned, session);
sessionInfos.put(lastReturned, session.getSessionInfo());
return session;
@@ -313,7 +305,7 @@
{
for (StreamSession session : streamSessions.values())
{
- streamExecutor.execute(new StreamSessionConnector(session));
+ startSession(session);
}
}
@@ -322,18 +314,23 @@
return Collections.unmodifiableCollection(streamSessions.values());
}
- public StreamSession getOrCreateSessionById(InetAddress peer, int id, InetAddress connecting)
+ public StreamSession getOrCreateSessionById(InetAddressAndPort peer, int id)
{
StreamSession session = streamSessions.get(id);
if (session == null)
{
- session = new StreamSession(peer, connecting, factory, id, keepSSTableLevel, isIncremental);
+ session = new StreamSession(streamOperation, peer, factory, isFollower(), id, pendingRepair, previewKind);
streamSessions.put(id, session);
sessionInfos.put(id, session.getSessionInfo());
}
return session;
}
+ public StreamSession getSessionById(int id)
+ {
+ return streamSessions.get(id);
+ }
+
public void updateProgress(ProgressInfo info)
{
sessionInfos.get(info.sessionIndex).updateProgress(info);
@@ -348,12 +345,11 @@
{
return sessionInfos.values();
}
- }
- @VisibleForTesting
- public static void shutdownAndWait(long timeout, TimeUnit unit) throws InterruptedException, TimeoutException
- {
- ExecutorUtils.shutdownAndWait(timeout, unit, streamExecutor);
+ @VisibleForTesting
+ public void shutdown()
+ {
+ streamSessions.values().forEach(ss -> ss.sessionFailed());
+ }
}
-
}
diff --git a/src/java/org/apache/cassandra/streaming/StreamEvent.java b/src/java/org/apache/cassandra/streaming/StreamEvent.java
index 49172fb..7ecd081 100644
--- a/src/java/org/apache/cassandra/streaming/StreamEvent.java
+++ b/src/java/org/apache/cassandra/streaming/StreamEvent.java
@@ -17,7 +17,6 @@
*/
package org.apache.cassandra.streaming;
-import java.net.InetAddress;
import java.util.Collections;
import java.util.Map;
import java.util.Set;
@@ -27,6 +26,7 @@
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.locator.InetAddressAndPort;
public abstract class StreamEvent
{
@@ -48,11 +48,11 @@
public static class SessionCompleteEvent extends StreamEvent
{
- public final InetAddress peer;
+ public final InetAddressAndPort peer;
public final boolean success;
public final int sessionIndex;
public final Set<StreamRequest> requests;
- public final String description;
+ public final StreamOperation streamOperation;
public final Map<String, Set<Range<Token>>> transferredRangesPerKeyspace;
public SessionCompleteEvent(StreamSession session)
@@ -62,7 +62,7 @@
this.success = session.isSuccess();
this.sessionIndex = session.sessionIndex();
this.requests = ImmutableSet.copyOf(session.requests);
- this.description = session.description();
+ this.streamOperation = session.streamOperation();
this.transferredRangesPerKeyspace = Collections.unmodifiableMap(session.transferredRangesPerKeyspace);
}
}
diff --git a/src/java/org/apache/cassandra/streaming/StreamHook.java b/src/java/org/apache/cassandra/streaming/StreamHook.java
index d610297..86b5182 100644
--- a/src/java/org/apache/cassandra/streaming/StreamHook.java
+++ b/src/java/org/apache/cassandra/streaming/StreamHook.java
@@ -18,19 +18,17 @@
package org.apache.cassandra.streaming;
-import org.apache.cassandra.db.ColumnFamilyStore;
-import org.apache.cassandra.io.sstable.SSTableMultiWriter;
-import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.streaming.messages.OutgoingFileMessage;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.streaming.messages.OutgoingStreamMessage;
import org.apache.cassandra.utils.FBUtilities;
public interface StreamHook
{
public static final StreamHook instance = createHook();
- public OutgoingFileMessage reportOutgoingFile(StreamSession session, SSTableReader sstable, OutgoingFileMessage message);
+ public OutgoingStreamMessage reportOutgoingStream(StreamSession session, OutgoingStream stream, OutgoingStreamMessage message);
public void reportStreamFuture(StreamSession session, StreamResultFuture future);
- public void reportIncomingFile(ColumnFamilyStore cfs, SSTableMultiWriter writer, StreamSession session, int sequenceNumber);
+ public void reportIncomingStream(TableId tableId, IncomingStream stream, StreamSession session, int sequenceNumber);
static StreamHook createHook()
{
@@ -43,14 +41,14 @@
{
return new StreamHook()
{
- public OutgoingFileMessage reportOutgoingFile(StreamSession session, SSTableReader sstable, OutgoingFileMessage message)
+ public OutgoingStreamMessage reportOutgoingStream(StreamSession session, OutgoingStream stream, OutgoingStreamMessage message)
{
return message;
}
public void reportStreamFuture(StreamSession session, StreamResultFuture future) {}
- public void reportIncomingFile(ColumnFamilyStore cfs, SSTableMultiWriter writer, StreamSession session, int sequenceNumber) {}
+ public void reportIncomingStream(TableId tableId, IncomingStream stream, StreamSession session, int sequenceNumber) {}
};
}
}
diff --git a/src/java/org/apache/cassandra/streaming/StreamManager.java b/src/java/org/apache/cassandra/streaming/StreamManager.java
index 52652c0..fa157a8 100644
--- a/src/java/org/apache/cassandra/streaming/StreamManager.java
+++ b/src/java/org/apache/cassandra/streaming/StreamManager.java
@@ -17,11 +17,9 @@
*/
package org.apache.cassandra.streaming;
-import java.net.InetAddress;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
-
import javax.management.ListenerNotFoundException;
import javax.management.MBeanNotificationInfo;
import javax.management.NotificationFilter;
@@ -36,6 +34,7 @@
import org.cliffc.high_scale_lib.NonBlockingHashMap;
import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.streaming.management.StreamEventJMXNotifier;
import org.apache.cassandra.streaming.management.StreamStateCompositeData;
@@ -56,7 +55,7 @@
*
* @return StreamRateLimiter with rate limit set based on peer location.
*/
- public static StreamRateLimiter getRateLimiter(InetAddress peer)
+ public static StreamRateLimiter getRateLimiter(InetAddressAndPort peer)
{
return new StreamRateLimiter(peer);
}
@@ -68,7 +67,7 @@
private static final RateLimiter interDCLimiter = RateLimiter.create(Double.MAX_VALUE);
private final boolean isLocalDC;
- public StreamRateLimiter(InetAddress peer)
+ public StreamRateLimiter(InetAddressAndPort peer)
{
double throughput = DatabaseDescriptor.getStreamThroughputOutboundMegabitsPerSec() * BYTES_PER_MEGABIT;
mayUpdateThroughput(throughput, limiter);
@@ -107,12 +106,12 @@
* We manage them in two different maps to distinguish plan from initiated ones to
* receiving ones withing the same JVM.
*/
- private final Map<UUID, StreamResultFuture> initiatedStreams = new NonBlockingHashMap<>();
- private final Map<UUID, StreamResultFuture> receivingStreams = new NonBlockingHashMap<>();
+ private final Map<UUID, StreamResultFuture> initiatorStreams = new NonBlockingHashMap<>();
+ private final Map<UUID, StreamResultFuture> followerStreams = new NonBlockingHashMap<>();
public Set<CompositeData> getCurrentStreams()
{
- return Sets.newHashSet(Iterables.transform(Iterables.concat(initiatedStreams.values(), receivingStreams.values()), new Function<StreamResultFuture, CompositeData>()
+ return Sets.newHashSet(Iterables.transform(Iterables.concat(initiatorStreams.values(), followerStreams.values()), new Function<StreamResultFuture, CompositeData>()
{
public CompositeData apply(StreamResultFuture input)
{
@@ -121,7 +120,7 @@
}));
}
- public void register(final StreamResultFuture result)
+ public void registerInitiator(final StreamResultFuture result)
{
result.addEventListener(notifier);
// Make sure we remove the stream on completion (whether successful or not)
@@ -129,14 +128,14 @@
{
public void run()
{
- initiatedStreams.remove(result.planId);
+ initiatorStreams.remove(result.planId);
}
}, MoreExecutors.directExecutor());
- initiatedStreams.put(result.planId, result);
+ initiatorStreams.put(result.planId, result);
}
- public void registerReceiving(final StreamResultFuture result)
+ public StreamResultFuture registerFollower(final StreamResultFuture result)
{
result.addEventListener(notifier);
// Make sure we remove the stream on completion (whether successful or not)
@@ -144,16 +143,17 @@
{
public void run()
{
- receivingStreams.remove(result.planId);
+ followerStreams.remove(result.planId);
}
}, MoreExecutors.directExecutor());
- receivingStreams.put(result.planId, result);
+ StreamResultFuture previous = followerStreams.putIfAbsent(result.planId, result);
+ return previous == null ? result : previous;
}
public StreamResultFuture getReceivingStream(UUID planId)
{
- return receivingStreams.get(planId);
+ return followerStreams.get(planId);
}
public void addNotificationListener(NotificationListener listener, NotificationFilter filter, Object handback)
@@ -175,4 +175,19 @@
{
return notifier.getNotificationInfo();
}
+
+ public StreamSession findSession(InetAddressAndPort peer, UUID planId, int sessionIndex, boolean searchInitiatorSessions)
+ {
+ Map<UUID, StreamResultFuture> streams = searchInitiatorSessions ? initiatorStreams : followerStreams;
+ return findSession(streams, peer, planId, sessionIndex);
+ }
+
+ private StreamSession findSession(Map<UUID, StreamResultFuture> streams, InetAddressAndPort peer, UUID planId, int sessionIndex)
+ {
+ StreamResultFuture streamResultFuture = streams.get(planId);
+ if (streamResultFuture == null)
+ return null;
+
+ return streamResultFuture.getSession(peer, sessionIndex);
+ }
}
diff --git a/src/java/org/apache/cassandra/streaming/StreamOperation.java b/src/java/org/apache/cassandra/streaming/StreamOperation.java
new file mode 100644
index 0000000..8151b47
--- /dev/null
+++ b/src/java/org/apache/cassandra/streaming/StreamOperation.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.streaming;
+
+public enum StreamOperation
+{
+ OTHER("Other"), // Fallback to avoid null types when deserializing from string
+ RESTORE_REPLICA_COUNT("Restore replica count", false), // Handles removeNode
+ DECOMMISSION("Unbootstrap", false),
+ RELOCATION("Relocation", false),
+ BOOTSTRAP("Bootstrap", false),
+ REBUILD("Rebuild", false),
+ BULK_LOAD("Bulk Load"),
+ REPAIR("Repair");
+
+ private final String description;
+ private final boolean requiresViewBuild;
+
+
+ StreamOperation(String description) {
+ this(description, true);
+ }
+
+ /**
+ * @param description The operation description
+ * @param requiresViewBuild Whether this operation requires views to be updated if it involves a base table
+ */
+ StreamOperation(String description, boolean requiresViewBuild) {
+ this.description = description;
+ this.requiresViewBuild = requiresViewBuild;
+ }
+
+ public static StreamOperation fromString(String text) {
+ for (StreamOperation b : StreamOperation.values()) {
+ if (b.description.equalsIgnoreCase(text)) {
+ return b;
+ }
+ }
+
+ return OTHER;
+ }
+
+ public String getDescription() {
+ return description;
+ }
+
+ /**
+ * Wether this operation requires views to be updated
+ */
+ public boolean requiresViewBuild()
+ {
+ return this.requiresViewBuild;
+ }
+}
diff --git a/src/java/org/apache/cassandra/streaming/StreamPlan.java b/src/java/org/apache/cassandra/streaming/StreamPlan.java
index e9d43cb..60845fa 100644
--- a/src/java/org/apache/cassandra/streaming/StreamPlan.java
+++ b/src/java/org/apache/cassandra/streaming/StreamPlan.java
@@ -17,14 +17,18 @@
*/
package org.apache.cassandra.streaming;
-import java.net.InetAddress;
import java.util.*;
-import org.apache.cassandra.dht.Range;
-import org.apache.cassandra.dht.Token;
-import org.apache.cassandra.service.ActiveRepairService;
+import com.google.common.annotations.VisibleForTesting;
+
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.RangesAtEndpoint;
+import org.apache.cassandra.locator.Replica;
import org.apache.cassandra.utils.UUIDGen;
+import static com.google.common.collect.Iterables.all;
+import static org.apache.cassandra.service.ActiveRepairService.NO_PENDING_REPAIR;
+
/**
* {@link StreamPlan} is a helper class that builds StreamOperation of given configuration.
*
@@ -32,11 +36,10 @@
*/
public class StreamPlan
{
- public static final String[] EMPTY_COLUMN_FAMILIES = new String[0];
+ private static final String[] EMPTY_COLUMN_FAMILIES = new String[0];
private final UUID planId = UUIDGen.getTimeUUID();
- private final String description;
+ private final StreamOperation streamOperation;
private final List<StreamEventHandler> handlers = new ArrayList<>();
- private final long repairedAt;
private final StreamCoordinator coordinator;
private boolean flushBeforeTransfer = true;
@@ -44,112 +47,98 @@
/**
* Start building stream plan.
*
- * @param description Stream type that describes this StreamPlan
+ * @param streamOperation Stream streamOperation that describes this StreamPlan
*/
- public StreamPlan(String description)
+ public StreamPlan(StreamOperation streamOperation)
{
- this(description, ActiveRepairService.UNREPAIRED_SSTABLE, 1, false, false, false);
+ this(streamOperation, 1, false, NO_PENDING_REPAIR, PreviewKind.NONE);
}
- public StreamPlan(String description, boolean keepSSTableLevels, boolean connectSequentially)
+ public StreamPlan(StreamOperation streamOperation, boolean connectSequentially)
{
- this(description, ActiveRepairService.UNREPAIRED_SSTABLE, 1, keepSSTableLevels, false, connectSequentially);
+ this(streamOperation, 1, connectSequentially, NO_PENDING_REPAIR, PreviewKind.NONE);
}
- public StreamPlan(String description, long repairedAt, int connectionsPerHost, boolean keepSSTableLevels,
- boolean isIncremental, boolean connectSequentially)
+ public StreamPlan(StreamOperation streamOperation, int connectionsPerHost,
+ boolean connectSequentially, UUID pendingRepair, PreviewKind previewKind)
{
- this.description = description;
- this.repairedAt = repairedAt;
- this.coordinator = new StreamCoordinator(connectionsPerHost, keepSSTableLevels, isIncremental, new DefaultConnectionFactory(),
- connectSequentially);
+ this.streamOperation = streamOperation;
+ this.coordinator = new StreamCoordinator(streamOperation, connectionsPerHost, new DefaultConnectionFactory(),
+ false, connectSequentially, pendingRepair, previewKind);
}
/**
* Request data in {@code keyspace} and {@code ranges} from specific node.
*
+ * Here, we have to encode both _local_ range transientness (encoded in Replica itself, in RangesAtEndpoint)
+ * and _remote_ (source) range transientmess, which is encoded by splitting ranges into full and transient.
+ *
+ * At the other end the distinction between full and transient is ignored it just used the transient status
+ * of the Replica objects we send to determine what to send. The real reason we have this split down to
+ * StreamRequest is that on completion StreamRequest is used to write to the system table tracking
+ * what has already been streamed. At that point since we only have the local Replica instances so we don't
+ * know what we got from the remote. We preserve that here by splitting based on the remotes transient
+ * status.
+ *
* @param from endpoint address to fetch data from.
- * @param connecting Actual connecting address for the endpoint
* @param keyspace name of keyspace
- * @param ranges ranges to fetch
+ * @param fullRanges ranges to fetch that from provides the full version of
+ * @param transientRanges ranges to fetch that from provides only transient data of
* @return this object for chaining
*/
- public StreamPlan requestRanges(InetAddress from, InetAddress connecting, String keyspace, Collection<Range<Token>> ranges)
+ public StreamPlan requestRanges(InetAddressAndPort from, String keyspace, RangesAtEndpoint fullRanges, RangesAtEndpoint transientRanges)
{
- return requestRanges(from, connecting, keyspace, ranges, EMPTY_COLUMN_FAMILIES);
+ return requestRanges(from, keyspace, fullRanges, transientRanges, EMPTY_COLUMN_FAMILIES);
}
/**
* Request data in {@code columnFamilies} under {@code keyspace} and {@code ranges} from specific node.
*
* @param from endpoint address to fetch data from.
- * @param connecting Actual connecting address for the endpoint
* @param keyspace name of keyspace
- * @param ranges ranges to fetch
+ * @param fullRanges ranges to fetch that from provides the full data for
+ * @param transientRanges ranges to fetch that from provides only transient data for
* @param columnFamilies specific column families
* @return this object for chaining
*/
- public StreamPlan requestRanges(InetAddress from, InetAddress connecting, String keyspace, Collection<Range<Token>> ranges, String... columnFamilies)
+ public StreamPlan requestRanges(InetAddressAndPort from, String keyspace, RangesAtEndpoint fullRanges, RangesAtEndpoint transientRanges, String... columnFamilies)
{
- StreamSession session = coordinator.getOrCreateNextSession(from, connecting);
- session.addStreamRequest(keyspace, ranges, Arrays.asList(columnFamilies), repairedAt);
+ //It should either be a dummy address for repair or if it's a bootstrap/move/rebuild it should be this node
+ assert all(fullRanges, Replica::isSelf) || RangesAtEndpoint.isDummyList(fullRanges) : fullRanges.toString();
+ assert all(transientRanges, Replica::isSelf) || RangesAtEndpoint.isDummyList(transientRanges) : transientRanges.toString();
+
+ StreamSession session = coordinator.getOrCreateNextSession(from);
+ session.addStreamRequest(keyspace, fullRanges, transientRanges, Arrays.asList(columnFamilies));
return this;
}
/**
* Add transfer task to send data of specific {@code columnFamilies} under {@code keyspace} and {@code ranges}.
*
- * @see #transferRanges(java.net.InetAddress, java.net.InetAddress, String, java.util.Collection, String...)
- */
- public StreamPlan transferRanges(InetAddress to, String keyspace, Collection<Range<Token>> ranges, String... columnFamilies)
- {
- return transferRanges(to, to, keyspace, ranges, columnFamilies);
- }
-
- /**
- * Add transfer task to send data of specific keyspace and ranges.
- *
* @param to endpoint address of receiver
- * @param connecting Actual connecting address of the endpoint
* @param keyspace name of keyspace
- * @param ranges ranges to send
- * @return this object for chaining
- */
- public StreamPlan transferRanges(InetAddress to, InetAddress connecting, String keyspace, Collection<Range<Token>> ranges)
- {
- return transferRanges(to, connecting, keyspace, ranges, EMPTY_COLUMN_FAMILIES);
- }
-
- /**
- * Add transfer task to send data of specific {@code columnFamilies} under {@code keyspace} and {@code ranges}.
- *
- * @param to endpoint address of receiver
- * @param connecting Actual connecting address of the endpoint
- * @param keyspace name of keyspace
- * @param ranges ranges to send
+ * @param replicas ranges to send
* @param columnFamilies specific column families
* @return this object for chaining
*/
- public StreamPlan transferRanges(InetAddress to, InetAddress connecting, String keyspace, Collection<Range<Token>> ranges, String... columnFamilies)
+ public StreamPlan transferRanges(InetAddressAndPort to, String keyspace, RangesAtEndpoint replicas, String... columnFamilies)
{
- StreamSession session = coordinator.getOrCreateNextSession(to, connecting);
- session.addTransferRanges(keyspace, ranges, Arrays.asList(columnFamilies), flushBeforeTransfer, repairedAt);
+ StreamSession session = coordinator.getOrCreateNextSession(to);
+ session.addTransferRanges(keyspace, replicas, Arrays.asList(columnFamilies), flushBeforeTransfer);
return this;
}
/**
- * Add transfer task to send given SSTable files.
+ * Add transfer task to send given streams
*
* @param to endpoint address of receiver
- * @param sstableDetails sstables with file positions and estimated key count.
- * this collection will be modified to remove those files that are successfully handed off
+ * @param streams streams to send
* @return this object for chaining
*/
- public StreamPlan transferFiles(InetAddress to, Collection<StreamSession.SSTableStreamingSections> sstableDetails)
+ public StreamPlan transferStreams(InetAddressAndPort to, Collection<OutgoingStream> streams)
{
- coordinator.transferFiles(to, sstableDetails);
+ coordinator.transferStreams(to, streams);
return this;
-
}
public StreamPlan listeners(StreamEventHandler handler, StreamEventHandler... handlers)
@@ -187,7 +176,7 @@
*/
public StreamResultFuture execute()
{
- return StreamResultFuture.init(planId, description, handlers, coordinator);
+ return StreamResultFuture.createInitiator(planId, streamOperation, handlers, coordinator);
}
/**
@@ -202,4 +191,20 @@
this.flushBeforeTransfer = flushBeforeTransfer;
return this;
}
+
+ public UUID getPendingRepair()
+ {
+ return coordinator.getPendingRepair();
+ }
+
+ public boolean getFlushBeforeTransfer()
+ {
+ return flushBeforeTransfer;
+ }
+
+ @VisibleForTesting
+ public StreamCoordinator getCoordinator()
+ {
+ return coordinator;
+ }
}
diff --git a/src/java/org/apache/cassandra/streaming/StreamReader.java b/src/java/org/apache/cassandra/streaming/StreamReader.java
deleted file mode 100644
index dbd5a4a..0000000
--- a/src/java/org/apache/cassandra/streaming/StreamReader.java
+++ /dev/null
@@ -1,337 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.streaming;
-
-import java.io.*;
-import java.nio.channels.Channels;
-import java.nio.channels.ReadableByteChannel;
-import java.util.Collection;
-import java.util.UUID;
-
-import com.google.common.base.Throwables;
-import com.google.common.collect.UnmodifiableIterator;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.ning.compress.lzf.LZFInputStream;
-
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.Schema;
-import org.apache.cassandra.db.*;
-import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
-import org.apache.cassandra.db.rows.*;
-import org.apache.cassandra.io.sstable.SSTableMultiWriter;
-import org.apache.cassandra.io.sstable.SSTableSimpleIterator;
-import org.apache.cassandra.io.sstable.format.RangeAwareSSTableWriter;
-import org.apache.cassandra.io.sstable.format.SSTableFormat;
-import org.apache.cassandra.io.sstable.format.Version;
-import org.apache.cassandra.io.util.RewindableDataInputStreamPlus;
-import org.apache.cassandra.io.util.DataInputPlus;
-import org.apache.cassandra.net.MessagingService;
-import org.apache.cassandra.streaming.messages.FileMessageHeader;
-import org.apache.cassandra.utils.ByteBufferUtil;
-import org.apache.cassandra.io.util.TrackedInputStream;
-import org.apache.cassandra.utils.FBUtilities;
-import org.apache.cassandra.utils.Pair;
-
-/**
- * StreamReader reads from stream and writes to SSTable.
- */
-public class StreamReader
-{
- private static final Logger logger = LoggerFactory.getLogger(StreamReader.class);
- protected final UUID cfId;
- protected final long estimatedKeys;
- protected final Collection<Pair<Long, Long>> sections;
- protected final StreamSession session;
- protected final Version inputVersion;
- protected final long repairedAt;
- protected final SSTableFormat.Type format;
- protected final int sstableLevel;
- protected final SerializationHeader.Component header;
- protected final int fileSeqNum;
-
- public StreamReader(FileMessageHeader header, StreamSession session)
- {
- this.session = session;
- this.cfId = header.cfId;
- this.estimatedKeys = header.estimatedKeys;
- this.sections = header.sections;
- this.inputVersion = header.version;
- this.repairedAt = header.repairedAt;
- this.format = header.format;
- this.sstableLevel = header.sstableLevel;
- this.header = header.header;
- this.fileSeqNum = header.sequenceNumber;
- }
-
- /**
- * @param channel where this reads data from
- * @return SSTable transferred
- * @throws IOException if reading the remote sstable fails. Will throw an RTE if local write fails.
- */
- @SuppressWarnings("resource") // channel needs to remain open, streams on top of it can't be closed
- public SSTableMultiWriter read(ReadableByteChannel channel) throws IOException
- {
- long totalSize = totalSize();
-
- Pair<String, String> kscf = Schema.instance.getCF(cfId);
- ColumnFamilyStore cfs = null;
- if (kscf != null)
- cfs = Keyspace.open(kscf.left).getColumnFamilyStore(kscf.right);
-
- if (kscf == null || cfs == null)
- {
- // schema was dropped during streaming
- throw new IOException("CF " + cfId + " was dropped during streaming");
- }
-
- logger.debug("[Stream #{}] Start receiving file #{} from {}, repairedAt = {}, size = {}, ks = '{}', table = '{}'.",
- session.planId(), fileSeqNum, session.peer, repairedAt, totalSize, cfs.keyspace.getName(),
- cfs.getColumnFamilyName());
-
- TrackedInputStream in = new TrackedInputStream(new LZFInputStream(Channels.newInputStream(channel)));
- StreamDeserializer deserializer = new StreamDeserializer(cfs.metadata, in, inputVersion, getHeader(cfs.metadata),
- totalSize, session.planId());
- SSTableMultiWriter writer = null;
- try
- {
- writer = createWriter(cfs, totalSize, repairedAt, format);
- while (in.getBytesRead() < totalSize)
- {
- writePartition(deserializer, writer);
- // TODO move this to BytesReadTracker
- session.progress(writer.getFilename(), ProgressInfo.Direction.IN, in.getBytesRead(), totalSize);
- }
- logger.debug("[Stream #{}] Finished receiving file #{} from {} readBytes = {}, totalSize = {}",
- session.planId(), fileSeqNum, session.peer, FBUtilities.prettyPrintMemory(in.getBytesRead()), FBUtilities.prettyPrintMemory(totalSize));
- return writer;
- }
- catch (Throwable e)
- {
- if (deserializer != null)
- logger.warn("[Stream {}] Error while reading partition {} from stream on ks='{}' and table='{}'.",
- session.planId(), deserializer.partitionKey(), cfs.keyspace.getName(), cfs.getTableName(), e);
- if (writer != null)
- {
- writer.abort(e);
- }
- throw Throwables.propagate(e);
- }
- finally
- {
- if (deserializer != null)
- deserializer.cleanup();
- }
- }
-
- protected SerializationHeader getHeader(CFMetaData metadata)
- {
- return header != null? header.toHeader(metadata) : null; //pre-3.0 sstable have no SerializationHeader
- }
-
- protected SSTableMultiWriter createWriter(ColumnFamilyStore cfs, long totalSize, long repairedAt, SSTableFormat.Type format) throws IOException
- {
- Directories.DataDirectory localDir = cfs.getDirectories().getWriteableLocation(totalSize);
- if (localDir == null)
- throw new IOException(String.format("Insufficient disk space to store %s", FBUtilities.prettyPrintMemory(totalSize)));
-
- LifecycleNewTracker lifecycleNewTracker = session.getReceivingTask(cfId).createLifecycleNewTracker();
- RangeAwareSSTableWriter writer = new RangeAwareSSTableWriter(cfs, estimatedKeys, repairedAt, format, sstableLevel, totalSize, lifecycleNewTracker, getHeader(cfs.metadata));
- StreamHook.instance.reportIncomingFile(cfs, writer, session, fileSeqNum);
- return writer;
- }
-
- protected long totalSize()
- {
- long size = 0;
- for (Pair<Long, Long> section : sections)
- size += section.right - section.left;
- return size;
- }
-
- protected void writePartition(StreamDeserializer deserializer, SSTableMultiWriter writer) throws IOException
- {
- writer.append(deserializer.newPartition());
- deserializer.checkForExceptions();
- }
-
- public static class StreamDeserializer extends UnmodifiableIterator<Unfiltered> implements UnfilteredRowIterator
- {
- public static final int INITIAL_MEM_BUFFER_SIZE = Integer.getInteger("cassandra.streamdes.initial_mem_buffer_size", 32768);
- public static final int MAX_MEM_BUFFER_SIZE = Integer.getInteger("cassandra.streamdes.max_mem_buffer_size", 1048576);
- public static final int MAX_SPILL_FILE_SIZE = Integer.getInteger("cassandra.streamdes.max_spill_file_size", Integer.MAX_VALUE);
-
- public static final String BUFFER_FILE_PREFIX = "buf";
- public static final String BUFFER_FILE_SUFFIX = "dat";
-
- private final CFMetaData metadata;
- private final DataInputPlus in;
- private final SerializationHeader header;
- private final SerializationHelper helper;
-
- private DecoratedKey key;
- private DeletionTime partitionLevelDeletion;
- private SSTableSimpleIterator iterator;
- private Row staticRow;
- private IOException exception;
-
- public StreamDeserializer(CFMetaData metadata, InputStream in, Version version, SerializationHeader header,
- long totalSize, UUID sessionId) throws IOException
- {
- this.metadata = metadata;
- // streaming pre-3.0 sstables require mark/reset support from source stream
- if (version.correspondingMessagingVersion() < MessagingService.VERSION_30)
- {
- logger.trace("Initializing rewindable input stream for reading legacy sstable with {} bytes with following " +
- "parameters: initial_mem_buffer_size={}, max_mem_buffer_size={}, max_spill_file_size={}.",
- totalSize, INITIAL_MEM_BUFFER_SIZE, MAX_MEM_BUFFER_SIZE, MAX_SPILL_FILE_SIZE);
- File bufferFile = getTempBufferFile(metadata, totalSize, sessionId);
- this.in = new RewindableDataInputStreamPlus(in, INITIAL_MEM_BUFFER_SIZE, MAX_MEM_BUFFER_SIZE, bufferFile, MAX_SPILL_FILE_SIZE);
- } else
- this.in = new DataInputPlus.DataInputStreamPlus(in);
- this.helper = new SerializationHelper(metadata, version.correspondingMessagingVersion(), SerializationHelper.Flag.PRESERVE_SIZE);
- this.header = header;
- }
-
- public StreamDeserializer newPartition() throws IOException
- {
- key = metadata.decorateKey(ByteBufferUtil.readWithShortLength(in));
- partitionLevelDeletion = DeletionTime.serializer.deserialize(in);
- iterator = SSTableSimpleIterator.create(metadata, in, header, helper, partitionLevelDeletion);
- staticRow = iterator.readStaticRow();
- return this;
- }
-
- public CFMetaData metadata()
- {
- return metadata;
- }
-
- public PartitionColumns columns()
- {
- // We don't know which columns we'll get so assume it can be all of them
- return metadata.partitionColumns();
- }
-
- public boolean isReverseOrder()
- {
- return false;
- }
-
- public DecoratedKey partitionKey()
- {
- return key;
- }
-
- public DeletionTime partitionLevelDeletion()
- {
- return partitionLevelDeletion;
- }
-
- public Row staticRow()
- {
- return staticRow;
- }
-
- public EncodingStats stats()
- {
- return header.stats();
- }
-
- public boolean hasNext()
- {
- try
- {
- return iterator.hasNext();
- }
- catch (IOError e)
- {
- if (e.getCause() != null && e.getCause() instanceof IOException)
- {
- exception = (IOException)e.getCause();
- return false;
- }
- throw e;
- }
- }
-
- public Unfiltered next()
- {
- // Note that in practice we know that IOException will be thrown by hasNext(), because that's
- // where the actual reading happens, so we don't bother catching RuntimeException here (contrarily
- // to what we do in hasNext)
- Unfiltered unfiltered = iterator.next();
- return metadata.isCounter() && unfiltered.kind() == Unfiltered.Kind.ROW
- ? maybeMarkLocalToBeCleared((Row) unfiltered)
- : unfiltered;
- }
-
- private Row maybeMarkLocalToBeCleared(Row row)
- {
- return metadata.isCounter() ? row.markCounterLocalToBeCleared() : row;
- }
-
- public void checkForExceptions() throws IOException
- {
- if (exception != null)
- throw exception;
- }
-
- public void close()
- {
- }
-
- /* We have a separate cleanup method because sometimes close is called before exhausting the
- StreamDeserializer (for instance, when enclosed in an try-with-resources wrapper, such as in
- BigTableWriter.append()).
- */
- public void cleanup()
- {
- if (in instanceof RewindableDataInputStreamPlus)
- {
- try
- {
- ((RewindableDataInputStreamPlus) in).close(false);
- }
- catch (IOException e)
- {
- logger.warn("Error while closing RewindableDataInputStreamPlus.", e);
- }
- }
- }
-
- private static File getTempBufferFile(CFMetaData metadata, long totalSize, UUID sessionId) throws IOException
- {
- ColumnFamilyStore cfs = Keyspace.open(metadata.ksName).getColumnFamilyStore(metadata.cfName);
- if (cfs == null)
- {
- // schema was dropped during streaming
- throw new RuntimeException(String.format("CF %s.%s was dropped during streaming", metadata.ksName, metadata.cfName));
- }
-
- long maxSize = Math.min(MAX_SPILL_FILE_SIZE, totalSize);
- File tmpDir = cfs.getDirectories().getTemporaryWriteableDirectoryAsFile(maxSize);
- if (tmpDir == null)
- throw new IOException(String.format("No sufficient disk space to stream legacy sstable from {}.{}. " +
- "Required disk space: %s.", FBUtilities.prettyPrintMemory(maxSize)));
- return new File(tmpDir, String.format("%s-%s.%s", BUFFER_FILE_PREFIX, sessionId, BUFFER_FILE_SUFFIX));
- }
- }
-}
diff --git a/src/java/org/apache/cassandra/streaming/StreamReceiveException.java b/src/java/org/apache/cassandra/streaming/StreamReceiveException.java
new file mode 100644
index 0000000..54b365a
--- /dev/null
+++ b/src/java/org/apache/cassandra/streaming/StreamReceiveException.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.streaming;
+
+public class StreamReceiveException extends RuntimeException
+{
+ public final StreamSession session;
+
+ public StreamReceiveException(StreamSession session, String msg)
+ {
+ super(msg);
+ this.session = session;
+ }
+
+ public StreamReceiveException(StreamSession session, Throwable t)
+ {
+ super(t);
+ this.session = session;
+ }
+}
diff --git a/src/java/org/apache/cassandra/streaming/StreamReceiveTask.java b/src/java/org/apache/cassandra/streaming/StreamReceiveTask.java
index 5388dd6..25977a5 100644
--- a/src/java/org/apache/cassandra/streaming/StreamReceiveTask.java
+++ b/src/java/org/apache/cassandra/streaming/StreamReceiveTask.java
@@ -17,41 +17,22 @@
*/
package org.apache.cassandra.streaming;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-import java.util.Set;
-import java.util.UUID;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
-import com.google.common.collect.Iterables;
-
-import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-
import org.apache.cassandra.concurrent.NamedThreadFactory;
-import org.apache.cassandra.config.Schema;
import org.apache.cassandra.db.ColumnFamilyStore;
-import org.apache.cassandra.db.Keyspace;
-import org.apache.cassandra.db.Mutation;
-import org.apache.cassandra.db.compaction.OperationType;
-import org.apache.cassandra.db.filter.ColumnFilter;
-import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
-import org.apache.cassandra.db.partitions.PartitionUpdate;
-import org.apache.cassandra.db.rows.UnfilteredRowIterator;
-import org.apache.cassandra.db.view.View;
-import org.apache.cassandra.dht.Bounds;
-import org.apache.cassandra.dht.Token;
-import org.apache.cassandra.io.sstable.ISSTableScanner;
-import org.apache.cassandra.io.sstable.SSTable;
-import org.apache.cassandra.io.sstable.SSTableMultiWriter;
-import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.utils.JVMStabilityInspector;
-import org.apache.cassandra.utils.Pair;
-import org.apache.cassandra.utils.Throwables;
-import org.apache.cassandra.utils.concurrent.Refs;
+
+import static org.apache.cassandra.utils.ExecutorUtils.awaitTermination;
+import static org.apache.cassandra.utils.ExecutorUtils.shutdown;
/**
* Task that manages receiving files for the session for certain ColumnFamily.
@@ -62,64 +43,54 @@
private static final ExecutorService executor = Executors.newCachedThreadPool(new NamedThreadFactory("StreamReceiveTask"));
- // number of files to receive
- private final int totalFiles;
- // total size of files to receive
- private final long totalSize;
+ private final StreamReceiver receiver;
- // Transaction tracking new files received
- private final LifecycleTransaction txn;
+ // number of streams to receive
+ private final int totalStreams;
+
+ // total size of streams to receive
+ private final long totalSize;
// true if task is done (either completed or aborted)
private volatile boolean done = false;
- // holds references to SSTables received
- protected Collection<SSTableReader> sstables;
+ private int remoteStreamsReceived = 0;
+ private long bytesReceived = 0;
- private int remoteSSTablesReceived = 0;
-
- public StreamReceiveTask(StreamSession session, UUID cfId, int totalFiles, long totalSize)
+ public StreamReceiveTask(StreamSession session, TableId tableId, int totalStreams, long totalSize)
{
- super(session, cfId);
- this.totalFiles = totalFiles;
+ super(session, tableId);
+ this.receiver = ColumnFamilyStore.getIfExists(tableId).getStreamManager().createStreamReceiver(session, totalStreams);
+ this.totalStreams = totalStreams;
this.totalSize = totalSize;
- // this is an "offline" transaction, as we currently manually expose the sstables once done;
- // this should be revisited at a later date, so that LifecycleTransaction manages all sstable state changes
- this.txn = LifecycleTransaction.offline(OperationType.STREAM);
- this.sstables = new ArrayList<>(totalFiles);
}
/**
- * Process received file.
+ * Process received stream.
*
- * @param sstable SSTable file received.
+ * @param stream Stream received.
*/
- public synchronized void received(SSTableMultiWriter sstable)
+ public synchronized void received(IncomingStream stream)
{
+ Preconditions.checkState(!session.isPreview(), "we should never receive sstables when previewing");
+
if (done)
{
- logger.warn("[{}] Received sstable {} on already finished stream received task. Aborting sstable.", session.planId(),
- sstable.getFilename());
- Throwables.maybeFail(sstable.abort(null));
+ logger.warn("[{}] Received stream {} on already finished stream received task. Aborting stream.", session.planId(),
+ stream.getName());
+ receiver.discardStream(stream);
return;
}
- remoteSSTablesReceived++;
- assert cfId.equals(sstable.getCfId());
+ remoteStreamsReceived += stream.getNumFiles();
+ bytesReceived += stream.getSize();
+ Preconditions.checkArgument(tableId.equals(stream.getTableId()));
+ logger.debug("received {} of {} total files {} of total bytes {}", remoteStreamsReceived, totalStreams,
+ bytesReceived, totalSize);
- Collection<SSTableReader> finished = null;
- try
- {
- finished = sstable.finish(true);
- }
- catch (Throwable t)
- {
- Throwables.maybeFail(sstable.abort(t));
- }
- txn.update(finished, false);
- sstables.addAll(finished);
+ receiver.received(stream);
- if (remoteSSTablesReceived == totalFiles)
+ if (remoteStreamsReceived == totalStreams)
{
done = true;
executor.submit(new OnCompletionRunnable(this));
@@ -128,7 +99,7 @@
public int getTotalNumberOfFiles()
{
- return totalFiles;
+ return totalStreams;
}
public long getTotalSize()
@@ -136,39 +107,11 @@
return totalSize;
}
- /**
- * @return a LifecycleNewTracker whose operations are synchronised on this StreamReceiveTask.
- */
- public synchronized LifecycleNewTracker createLifecycleNewTracker()
+ public synchronized StreamReceiver getReceiver()
{
if (done)
- throw new RuntimeException(String.format("Stream receive task %s of cf %s already finished.", session.planId(), cfId));
-
- return new LifecycleNewTracker()
- {
- @Override
- public void trackNew(SSTable table)
- {
- synchronized (StreamReceiveTask.this)
- {
- txn.trackNew(table);
- }
- }
-
- @Override
- public void untrackNew(SSTable table)
- {
- synchronized (StreamReceiveTask.this)
- {
- txn.untrackNew(table);
- }
- }
-
- public OperationType opType()
- {
- return txn.opType();
- }
- };
+ throw new RuntimeException(String.format("Stream receive task %s of cf %s already finished.", session.planId(), tableId));
+ return receiver;
}
private static class OnCompletionRunnable implements Runnable
@@ -182,97 +125,17 @@
public void run()
{
- boolean hasViews = false;
- boolean hasCDC = false;
- ColumnFamilyStore cfs = null;
try
{
- Pair<String, String> kscf = Schema.instance.getCF(task.cfId);
- if (kscf == null)
+ if (ColumnFamilyStore.getIfExists(task.tableId) == null)
{
// schema was dropped during streaming
- task.sstables.clear();
- task.abortTransaction();
+ task.receiver.abort();
task.session.taskCompleted(task);
return;
}
- cfs = Keyspace.open(kscf.left).getColumnFamilyStore(kscf.right);
- hasViews = !Iterables.isEmpty(View.findAll(kscf.left, kscf.right));
- hasCDC = cfs.metadata.params.cdc;
- Collection<SSTableReader> readers = task.sstables;
-
- try (Refs<SSTableReader> refs = Refs.ref(readers))
- {
- /*
- * We have a special path for views and for CDC.
- *
- * For views, since the view requires cleaning up any pre-existing state, we must put all partitions
- * through the same write path as normal mutations. This also ensures any 2is are also updated.
- *
- * For CDC-enabled tables, we want to ensure that the mutations are run through the CommitLog so they
- * can be archived by the CDC process on discard.
- */
- if (hasViews || hasCDC)
- {
- for (SSTableReader reader : readers)
- {
- Keyspace ks = Keyspace.open(reader.getKeyspaceName());
- try (ISSTableScanner scanner = reader.getScanner())
- {
- while (scanner.hasNext())
- {
- try (UnfilteredRowIterator rowIterator = scanner.next())
- {
- Mutation m = new Mutation(PartitionUpdate.fromIterator(rowIterator, ColumnFilter.all(cfs.metadata)));
-
- // MV *can* be applied unsafe if there's no CDC on the CFS as we flush below
- // before transaction is done.
- //
- // If the CFS has CDC, however, these updates need to be written to the CommitLog
- // so they get archived into the cdc_raw folder
- ks.apply(m, hasCDC, true, false);
- }
- }
- }
- }
- }
- else
- {
- task.finishTransaction();
-
- logger.debug("[Stream #{}] Received {} sstables from {} ({})", task.session.planId(), readers.size(), task.session.peer, readers);
- // add sstables and build secondary indexes
- cfs.addSSTables(readers);
- cfs.indexManager.buildAllIndexesBlocking(readers);
-
- //invalidate row and counter cache
- if (cfs.isRowCacheEnabled() || cfs.metadata.isCounter())
- {
- List<Bounds<Token>> boundsToInvalidate = new ArrayList<>(readers.size());
- readers.forEach(sstable -> boundsToInvalidate.add(new Bounds<Token>(sstable.first.getToken(), sstable.last.getToken())));
- Set<Bounds<Token>> nonOverlappingBounds = Bounds.getNonOverlappingBounds(boundsToInvalidate);
-
- if (cfs.isRowCacheEnabled())
- {
- int invalidatedKeys = cfs.invalidateRowCache(nonOverlappingBounds);
- if (invalidatedKeys > 0)
- logger.debug("[Stream #{}] Invalidated {} row cache entries on table {}.{} after stream " +
- "receive task completed.", task.session.planId(), invalidatedKeys,
- cfs.keyspace.getName(), cfs.getTableName());
- }
-
- if (cfs.metadata.isCounter())
- {
- int invalidatedKeys = cfs.invalidateCounterCache(nonOverlappingBounds);
- if (invalidatedKeys > 0)
- logger.debug("[Stream #{}] Invalidated {} counter cache entries on table {}.{} after stream " +
- "receive task completed.", task.session.planId(), invalidatedKeys,
- cfs.keyspace.getName(), cfs.getTableName());
- }
- }
- }
- }
+ task.receiver.finished();
task.session.taskCompleted(task);
}
catch (Throwable t)
@@ -282,14 +145,7 @@
}
finally
{
- // We don't keep the streamed sstables since we've applied them manually so we abort the txn and delete
- // the streamed sstables.
- if (hasViews || hasCDC)
- {
- if (cfs != null)
- cfs.forceBlockingFlush();
- task.abortTransaction();
- }
+ task.receiver.cleanup();
}
}
}
@@ -306,17 +162,13 @@
return;
done = true;
- abortTransaction();
- sstables.clear();
+ receiver.abort();
}
- private synchronized void abortTransaction()
+ @VisibleForTesting
+ public static void shutdownAndWait(long timeout, TimeUnit unit) throws InterruptedException, TimeoutException
{
- txn.abort();
- }
-
- private synchronized void finishTransaction()
- {
- txn.finish();
+ shutdown(executor);
+ awaitTermination(timeout, unit, executor);
}
}
diff --git a/src/java/org/apache/cassandra/streaming/StreamReceiver.java b/src/java/org/apache/cassandra/streaming/StreamReceiver.java
new file mode 100644
index 0000000..bc357ef
--- /dev/null
+++ b/src/java/org/apache/cassandra/streaming/StreamReceiver.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.streaming;
+
+/**
+ * StreamReceiver acts as a staging area for incoming data. Received data
+ * ends up here, and is kept separate from the live data until all streams
+ * for a session have been received successfully
+ */
+public interface StreamReceiver
+{
+ /**
+ * Called after we've finished receiving stream data. The data covered by the given stream should
+ * be kept isolated from the live dataset for it's table.
+ */
+ void received(IncomingStream stream);
+
+ /**
+ * This is called when we've received stream data we can't add to the received set for some reason,
+ * usually when we've received data for a session which has been closed. The data backing this stream
+ * should be deleted, and any resources associated with the given stream should be released.
+ */
+ void discardStream(IncomingStream stream);
+
+ /**
+ * Called when something went wrong with a stream session. All data associated with this receiver
+ * should be deleted, and any associated resources should be cleaned up
+ */
+ void abort();
+
+ /**
+ * Called when a stream session has succesfully completed. All stream data being held by this receiver
+ * should be added to the live data sets for their respective tables before this method returns.
+ */
+ void finished();
+
+ /**
+ * Called after finished has returned and we've sent any messages to other nodes. Mainly for
+ * signaling that mvs and cdc should cleanup.
+ */
+ void cleanup();
+}
diff --git a/src/java/org/apache/cassandra/streaming/StreamRequest.java b/src/java/org/apache/cassandra/streaming/StreamRequest.java
index 93726e7..0c8542f 100644
--- a/src/java/org/apache/cassandra/streaming/StreamRequest.java
+++ b/src/java/org/apache/cassandra/streaming/StreamRequest.java
@@ -24,27 +24,41 @@
import java.util.List;
import org.apache.cassandra.db.TypeSizes;
+import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.io.IVersionedSerializer;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
-import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.RangesAtEndpoint;
+import org.apache.cassandra.locator.Replica;
+
+import static org.apache.cassandra.locator.InetAddressAndPort.Serializer.inetAddressAndPortSerializer;
public class StreamRequest
{
public static final IVersionedSerializer<StreamRequest> serializer = new StreamRequestSerializer();
public final String keyspace;
- public final Collection<Range<Token>> ranges;
+ //Full replicas and transient replicas are split based on the transient status of the remote we are fetching
+ //from. We preserve this distinction so on completion we can log to a system table whether we got the data transiently
+ //or fully from some remote. This is an important distinction for resumable bootstrap. The Replicas in these collections
+ //are local replicas (or dummy if this is triggered by repair) and don't encode the necessary information about
+ //what the remote provided.
+ public final RangesAtEndpoint full;
+ public final RangesAtEndpoint transientReplicas;
public final Collection<String> columnFamilies = new HashSet<>();
- public final long repairedAt;
- public StreamRequest(String keyspace, Collection<Range<Token>> ranges, Collection<String> columnFamilies, long repairedAt)
+
+ public StreamRequest(String keyspace, RangesAtEndpoint full, RangesAtEndpoint transientReplicas, Collection<String> columnFamilies)
{
this.keyspace = keyspace;
- this.ranges = ranges;
+ if (!full.endpoint().equals(transientReplicas.endpoint()))
+ throw new IllegalStateException("Mismatching endpoints: " + full + ", " + transientReplicas);
+
+ this.full = full;
+ this.transientReplicas = transientReplicas;
this.columnFamilies.addAll(columnFamilies);
- this.repairedAt = repairedAt;
}
public static class StreamRequestSerializer implements IVersionedSerializer<StreamRequest>
@@ -52,52 +66,82 @@
public void serialize(StreamRequest request, DataOutputPlus out, int version) throws IOException
{
out.writeUTF(request.keyspace);
- out.writeLong(request.repairedAt);
- out.writeInt(request.ranges.size());
- for (Range<Token> range : request.ranges)
- {
- MessagingService.validatePartitioner(range);
- Token.serializer.serialize(range.left, out, version);
- Token.serializer.serialize(range.right, out, version);
- }
out.writeInt(request.columnFamilies.size());
+
+ inetAddressAndPortSerializer.serialize(request.full.endpoint(), out, version);
+ serializeReplicas(request.full, out, version);
+ serializeReplicas(request.transientReplicas, out, version);
for (String cf : request.columnFamilies)
out.writeUTF(cf);
}
+ private void serializeReplicas(RangesAtEndpoint replicas, DataOutputPlus out, int version) throws IOException
+ {
+ out.writeInt(replicas.size());
+
+ for (Replica replica : replicas)
+ {
+ IPartitioner.validate(replica.range());
+ Token.serializer.serialize(replica.range().left, out, version);
+ Token.serializer.serialize(replica.range().right, out, version);
+ }
+ }
+
public StreamRequest deserialize(DataInputPlus in, int version) throws IOException
{
String keyspace = in.readUTF();
- long repairedAt = in.readLong();
- int rangeCount = in.readInt();
- List<Range<Token>> ranges = new ArrayList<>(rangeCount);
- for (int i = 0; i < rangeCount; i++)
- {
- Token left = Token.serializer.deserialize(in, MessagingService.globalPartitioner(), version);
- Token right = Token.serializer.deserialize(in, MessagingService.globalPartitioner(), version);
- ranges.add(new Range<>(left, right));
- }
int cfCount = in.readInt();
+ InetAddressAndPort endpoint = inetAddressAndPortSerializer.deserialize(in, version);
+
+ RangesAtEndpoint full = deserializeReplicas(in, version, endpoint, true);
+ RangesAtEndpoint transientReplicas = deserializeReplicas(in, version, endpoint, false);
List<String> columnFamilies = new ArrayList<>(cfCount);
for (int i = 0; i < cfCount; i++)
columnFamilies.add(in.readUTF());
- return new StreamRequest(keyspace, ranges, columnFamilies, repairedAt);
+ return new StreamRequest(keyspace, full, transientReplicas, columnFamilies);
+ }
+
+ RangesAtEndpoint deserializeReplicas(DataInputPlus in, int version, InetAddressAndPort endpoint, boolean isFull) throws IOException
+ {
+ int replicaCount = in.readInt();
+
+ RangesAtEndpoint.Builder replicas = RangesAtEndpoint.builder(endpoint, replicaCount);
+ for (int i = 0; i < replicaCount; i++)
+ {
+ //TODO, super need to review the usage of streaming vs not streaming endpoint serialization helper
+ //to make sure I'm not using the wrong one some of the time, like do repair messages use the
+ //streaming version?
+ Token left = Token.serializer.deserialize(in, IPartitioner.global(), version);
+ Token right = Token.serializer.deserialize(in, IPartitioner.global(), version);
+ replicas.add(new Replica(endpoint, new Range<>(left, right), isFull));
+ }
+ return replicas.build();
}
public long serializedSize(StreamRequest request, int version)
{
int size = TypeSizes.sizeof(request.keyspace);
- size += TypeSizes.sizeof(request.repairedAt);
- size += TypeSizes.sizeof(request.ranges.size());
- for (Range<Token> range : request.ranges)
- {
- size += Token.serializer.serializedSize(range.left, version);
- size += Token.serializer.serializedSize(range.right, version);
- }
size += TypeSizes.sizeof(request.columnFamilies.size());
+ size += inetAddressAndPortSerializer.serializedSize(request.full.endpoint(), version);
+ size += replicasSerializedSize(request.transientReplicas, version);
+ size += replicasSerializedSize(request.full, version);
for (String cf : request.columnFamilies)
size += TypeSizes.sizeof(cf);
return size;
}
+
+ private long replicasSerializedSize(RangesAtEndpoint replicas, int version)
+ {
+ long size = 0;
+ size += TypeSizes.sizeof(replicas.size());
+
+ for (Replica replica : replicas)
+ {
+ size += Token.serializer.serializedSize(replica.range().left, version);
+ size += Token.serializer.serializedSize(replica.range().right, version);
+ }
+ return size;
+ }
+
}
}
diff --git a/src/java/org/apache/cassandra/streaming/StreamResultFuture.java b/src/java/org/apache/cassandra/streaming/StreamResultFuture.java
index 481e93d..89a6cf1 100644
--- a/src/java/org/apache/cassandra/streaming/StreamResultFuture.java
+++ b/src/java/org/apache/cassandra/streaming/StreamResultFuture.java
@@ -17,17 +17,18 @@
*/
package org.apache.cassandra.streaming;
-import java.io.IOException;
-import java.net.InetAddress;
-import java.util.*;
+import java.util.Collection;
+import java.util.UUID;
import java.util.concurrent.ConcurrentLinkedQueue;
import com.google.common.util.concurrent.AbstractFuture;
import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.MoreExecutors;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.cassandra.net.IncomingStreamingConnection;
+import io.netty.channel.Channel;
+import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.utils.FBUtilities;
/**
@@ -48,46 +49,45 @@
private static final Logger logger = LoggerFactory.getLogger(StreamResultFuture.class);
public final UUID planId;
- public final String description;
+ public final StreamOperation streamOperation;
private final StreamCoordinator coordinator;
private final Collection<StreamEventHandler> eventListeners = new ConcurrentLinkedQueue<>();
/**
- * Create new StreamResult of given {@code planId} and type.
+ * Create new StreamResult of given {@code planId} and streamOperation.
*
* Constructor is package private. You need to use {@link StreamPlan#execute()} to get the instance.
*
* @param planId Stream plan ID
- * @param description Stream description
+ * @param streamOperation Stream streamOperation
*/
- private StreamResultFuture(UUID planId, String description, StreamCoordinator coordinator)
+ private StreamResultFuture(UUID planId, StreamOperation streamOperation, StreamCoordinator coordinator)
{
this.planId = planId;
- this.description = description;
+ this.streamOperation = streamOperation;
this.coordinator = coordinator;
// if there is no session to listen to, we immediately set result for returning
- if (!coordinator.isReceiving() && !coordinator.hasActiveSessions())
+ if (!coordinator.isFollower() && !coordinator.hasActiveSessions())
set(getCurrentState());
}
- private StreamResultFuture(UUID planId, String description, boolean keepSSTableLevels, boolean isIncremental)
+ private StreamResultFuture(UUID planId, StreamOperation streamOperation, UUID pendingRepair, PreviewKind previewKind)
{
- this(planId, description, new StreamCoordinator(0, keepSSTableLevels, isIncremental,
- new DefaultConnectionFactory(), false));
+ this(planId, streamOperation, new StreamCoordinator(streamOperation, 0, new DefaultConnectionFactory(), true, false, pendingRepair, previewKind));
}
- static StreamResultFuture init(UUID planId, String description, Collection<StreamEventHandler> listeners,
- StreamCoordinator coordinator)
+ public static StreamResultFuture createInitiator(UUID planId, StreamOperation streamOperation, Collection<StreamEventHandler> listeners,
+ StreamCoordinator coordinator)
{
- StreamResultFuture future = createAndRegister(planId, description, coordinator);
+ StreamResultFuture future = createAndRegisterInitiator(planId, streamOperation, coordinator);
if (listeners != null)
{
for (StreamEventHandler listener : listeners)
future.addEventListener(listener);
}
- logger.info("[Stream #{}] Executing streaming plan for {}", planId, description);
+ logger.info("[Stream #{}] Executing streaming plan for {}", planId, streamOperation.getDescription());
// Initialize and start all sessions
for (final StreamSession session : coordinator.getAllStreamSessions())
@@ -100,47 +100,52 @@
return future;
}
- public static synchronized StreamResultFuture initReceivingSide(int sessionIndex,
- UUID planId,
- String description,
- InetAddress from,
- IncomingStreamingConnection connection,
- boolean isForOutgoing,
- int version,
- boolean keepSSTableLevel,
- boolean isIncremental) throws IOException
+ public static synchronized StreamResultFuture createFollower(int sessionIndex,
+ UUID planId,
+ StreamOperation streamOperation,
+ InetAddressAndPort from,
+ Channel channel,
+ UUID pendingRepair,
+ PreviewKind previewKind)
{
StreamResultFuture future = StreamManager.instance.getReceivingStream(planId);
if (future == null)
{
- logger.info("[Stream #{} ID#{}] Creating new streaming plan for {}", planId, sessionIndex, description);
+ logger.info("[Stream #{} ID#{}] Creating new streaming plan for {} from {} channel.remote {} channel.local {}" +
+ " channel.id {}", planId, sessionIndex, streamOperation.getDescription(),
+ from, channel.remoteAddress(), channel.localAddress(), channel.id());
// The main reason we create a StreamResultFuture on the receiving side is for JMX exposure.
- future = new StreamResultFuture(planId, description, keepSSTableLevel, isIncremental);
- StreamManager.instance.registerReceiving(future);
+ future = new StreamResultFuture(planId, streamOperation, pendingRepair, previewKind);
+ StreamManager.instance.registerFollower(future);
}
- future.attachConnection(from, sessionIndex, connection, isForOutgoing, version);
- logger.info("[Stream #{}, ID#{}] Received streaming plan for {}", planId, sessionIndex, description);
+ future.attachConnection(from, sessionIndex, channel);
+ logger.info("[Stream #{}, ID#{}] Received streaming plan for {} from {} channel.remote {} channel.local {} channel.id {}",
+ planId, sessionIndex, streamOperation.getDescription(), from, channel.remoteAddress(), channel.localAddress(), channel.id());
return future;
}
- private static StreamResultFuture createAndRegister(UUID planId, String description, StreamCoordinator coordinator)
+ private static StreamResultFuture createAndRegisterInitiator(UUID planId, StreamOperation streamOperation, StreamCoordinator coordinator)
{
- StreamResultFuture future = new StreamResultFuture(planId, description, coordinator);
- StreamManager.instance.register(future);
+ StreamResultFuture future = new StreamResultFuture(planId, streamOperation, coordinator);
+ StreamManager.instance.registerInitiator(future);
return future;
}
- private void attachConnection(InetAddress from, int sessionIndex, IncomingStreamingConnection connection, boolean isForOutgoing, int version) throws IOException
+ public StreamCoordinator getCoordinator()
{
- StreamSession session = coordinator.getOrCreateSessionById(from, sessionIndex, connection.socket.getInetAddress());
+ return coordinator;
+ }
+
+ private void attachConnection(InetAddressAndPort from, int sessionIndex, Channel channel)
+ {
+ StreamSession session = coordinator.getOrCreateSessionById(from, sessionIndex);
session.init(this);
- session.handler.initiateOnReceivingSide(connection, isForOutgoing, version);
}
public void addEventListener(StreamEventHandler listener)
{
- Futures.addCallback(this, listener);
+ Futures.addCallback(this, listener, MoreExecutors.directExecutor());
eventListeners.add(listener);
}
@@ -149,7 +154,7 @@
*/
public StreamState getCurrentState()
{
- return new StreamState(planId, description, coordinator.getAllSessionInfo());
+ return new StreamState(planId, streamOperation, coordinator.getAllSessionInfo());
}
@Override
@@ -222,6 +227,11 @@
}
}
+ public StreamSession getSession(InetAddressAndPort peer, int sessionIndex)
+ {
+ return coordinator.getSessionById(peer, sessionIndex);
+ }
+
/**
* We can't use {@link StreamCoordinator#hasActiveSessions()} directly because {@link this#maybeComplete()}
* relies on the snapshotted state from {@link StreamCoordinator} and not the {@link StreamSession} state
diff --git a/src/java/org/apache/cassandra/streaming/StreamSession.java b/src/java/org/apache/cassandra/streaming/StreamSession.java
index 08cd22d..f59eaa5 100644
--- a/src/java/org/apache/cassandra/streaming/StreamSession.java
+++ b/src/java/org/apache/cassandra/streaming/StreamSession.java
@@ -17,129 +17,134 @@
*/
package org.apache.cassandra.streaming;
-import java.io.IOException;
-import java.net.InetAddress;
-import java.net.Socket;
+import java.io.EOFException;
import java.net.SocketTimeoutException;
import java.util.*;
import java.util.concurrent.*;
-import java.util.concurrent.atomic.AtomicBoolean;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.*;
-import org.apache.cassandra.concurrent.DebuggableScheduledThreadPoolExecutor;
-import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
-import org.apache.cassandra.db.lifecycle.SSTableIntervalTree;
-import org.apache.cassandra.db.lifecycle.SSTableSet;
-import org.apache.cassandra.db.lifecycle.View;
-import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.concurrent.ScheduledExecutors;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.locator.RangesAtEndpoint;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.cassandra.config.DatabaseDescriptor;
+import io.netty.channel.Channel;
+import io.netty.channel.ChannelId;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Keyspace;
-import org.apache.cassandra.db.PartitionPosition;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.gms.*;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.Replica;
import org.apache.cassandra.metrics.StreamingMetrics;
-import org.apache.cassandra.service.ActiveRepairService;
+import org.apache.cassandra.net.OutboundConnectionSettings;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.streaming.async.NettyStreamingMessageSender;
import org.apache.cassandra.streaming.messages.*;
-import org.apache.cassandra.utils.CassandraVersion;
-import org.apache.cassandra.utils.ExecutorUtils;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.JVMStabilityInspector;
-import org.apache.cassandra.utils.Pair;
-import org.apache.cassandra.utils.concurrent.Ref;
-import org.apache.cassandra.utils.concurrent.Refs;
+
+import static com.google.common.collect.Iterables.all;
+import static org.apache.cassandra.net.MessagingService.current_version;
/**
- * Handles the streaming a one or more section of one of more sstables to and from a specific
- * remote node.
- *
- * Both this node and the remote one will create a similar symmetrical StreamSession. A streaming
+ * Handles the streaming a one or more streams to and from a specific remote node.
+ *<p/>
+ * Both this node and the remote one will create a similar symmetrical {@link StreamSession}. A streaming
* session has the following life-cycle:
+ *<pre>
+ * 1. Session Initialization
*
- * 1. Connections Initialization
- *
- * (a) A node (the initiator in the following) create a new StreamSession, initialize it (init())
- * and then start it (start()). Start will create a {@link ConnectionHandler} that will create
- * two connections to the remote node (the follower in the following) with whom to stream and send
- * a StreamInit message. The first connection will be the incoming connection for the
- * initiator, and the second connection will be the outgoing.
- * (b) Upon reception of that StreamInit message, the follower creates its own StreamSession,
- * initialize it if it still does not exist, and attach connecting socket to its ConnectionHandler
- * according to StreamInit message's isForOutgoing flag.
- * (d) When the both incoming and outgoing connections are established, StreamSession calls
- * StreamSession#onInitializationComplete method to start the streaming prepare phase
- * (StreamResultFuture.startStreaming()).
+ * (a) A node (the initiator in the following) create a new {@link StreamSession},
+ * initialize it {@link #init(StreamResultFuture)}, and then start it ({@link #start()}).
+ * Starting a session causes a {@link StreamInitMessage} to be sent.
+ * (b) Upon reception of that {@link StreamInitMessage}, the follower creates its own {@link StreamSession},
+ * and initializes it if it still does not exist.
+ * (c) After the initiator sends the {@link StreamInitMessage}, it invokes
+ * {@link StreamSession#onInitializationComplete()} to start the streaming prepare phase.
*
* 2. Streaming preparation phase
*
- * (a) This phase is started when the initiator onInitializationComplete() method is called. This method sends a
- * PrepareMessage that includes what files/sections this node will stream to the follower
- * (stored in a StreamTransferTask, each column family has it's own transfer task) and what
- * the follower needs to stream back (StreamReceiveTask, same as above). If the initiator has
- * nothing to receive from the follower, it goes directly to its Streaming phase. Otherwise,
- * it waits for the follower PrepareMessage.
- * (b) Upon reception of the PrepareMessage, the follower records which files/sections it will receive
- * and send back its own PrepareMessage with a summary of the files/sections that will be sent to
- * the initiator (prepare()). After having sent that message, the follower goes to its Streamning
- * phase.
- * (c) When the initiator receives the follower PrepareMessage, it records which files/sections it will
- * receive and then goes to his own Streaming phase.
+ * (a) A {@link PrepareSynMessage} is sent that includes a) what files/sections this node will stream to the follower
+ * (stored locally in a {@link StreamTransferTask}, one for each table) and b) what the follower needs to
+ * stream back (stored locally in a {@link StreamReceiveTask}, one for each table).
+ * (b) Upon reception of the {@link PrepareSynMessage}, the follower records which files/sections it will receive
+ * and send back a {@link PrepareSynAckMessage}, which contains a summary of the files/sections that will be sent to
+ * the initiator.
+ * (c) When the initiator receives the {@link PrepareSynAckMessage}, it records which files/sections it will
+ * receive, and then goes to it's Streaming phase (see next section). If the intiator is to receive files,
+ * it sends a {@link PrepareAckMessage} to the follower to indicate that it can start streaming to the initiator.
+ * (d) (Optional) If the follower receives a {@link PrepareAckMessage}, it enters it's Streaming phase.
*
* 3. Streaming phase
*
- * (a) The streaming phase is started by each node (the sender in the follower, but note that each side
- * of the StreamSession may be sender for some of the files) involved by calling startStreamingFiles().
- * This will sequentially send a FileMessage for each file of each SteamTransferTask. Each FileMessage
- * consists of a FileMessageHeader that indicates which file is coming and then start streaming the
- * content for that file (StreamWriter in FileMessage.serialize()). When a file is fully sent, the
- * fileSent() method is called for that file. If all the files for a StreamTransferTask are sent
- * (StreamTransferTask.complete()), the task is marked complete (taskCompleted()).
- * (b) On the receiving side, a SSTable will be written for the incoming file (StreamReader in
- * FileMessage.deserialize()) and once the FileMessage is fully received, the file will be marked as
- * complete (received()). When all files for the StreamReceiveTask have been received, the sstables
- * are added to the CFS (and 2ndary index are built, StreamReceiveTask.complete()) and the task
- * is marked complete (taskCompleted())
- * (b) If during the streaming of a particular file an error occurs on the receiving end of a stream
- * (FileMessage.deserialize), the node will send a SessionFailedMessage to the sender and close the stream session.
- * (c) When all transfer and receive tasks for a session are complete, the move to the Completion phase
- * (maybeCompleted()).
+ * (a) The streaming phase is started at each node by calling {@link StreamSession#startStreamingFiles(boolean)}.
+ * This will send, sequentially on each outbound streaming connection (see {@link NettyStreamingMessageSender}),
+ * an {@link OutgoingStreamMessage} for each stream in each of the {@link StreamTransferTask}.
+ * Each {@link OutgoingStreamMessage} consists of a {@link StreamMessageHeader} that contains metadata about
+ * the stream, followed by the stream content itself. Once all the files for a {@link StreamTransferTask} are sent,
+ * the task is marked complete {@link StreamTransferTask#complete(int)}.
+ * (b) On the receiving side, the incoming data is written to disk, and once the stream is fully received,
+ * it will be marked as complete ({@link StreamReceiveTask#received(IncomingStream)}). When all streams
+ * for the {@link StreamReceiveTask} have been received, the data is added to the CFS (and 2ndary indexes/MV are built),
+ * and the task is marked complete ({@link #taskCompleted(StreamReceiveTask)}).
+ * (b) If during the streaming of a particular stream an error occurs on the receiving end of a stream
+ * (it may be either the initiator or the follower), the node will send a {@link SessionFailedMessage}
+ * to the sender and close the stream session.
+ * (c) When all transfer and receive tasks for a session are complete, the session moves to the Completion phase
+ * ({@link #maybeCompleted()}).
*
* 4. Completion phase
*
- * (a) When a node has finished all transfer and receive task, it enter the completion phase (maybeCompleted()).
- * If it had already received a CompleteMessage from the other side (it is in the WAIT_COMPLETE state), that
- * session is done is is closed (closeSession()). Otherwise, the node switch to the WAIT_COMPLETE state and
- * send a CompleteMessage to the other side.
+ * (a) When the initiator finishes streaming, it enters the {@link StreamSession.State#WAIT_COMPLETE} state, and waits
+ * for the follower to send a {@link CompleteMessage} once it finishes streaming too. Once the {@link CompleteMessage}
+ * is received, initiator sets its own state to {@link StreamSession.State#COMPLETE} and closes all channels attached
+ * to this session.
+ *
+ * </pre>
+ *
+ * In brief, the message passing looks like this (I for initiator, F for follwer):
+ * <pre>
+ * (session init)
+ * I: StreamInitMessage
+ * (session prepare)
+ * I: PrepareSynMessage
+ * F: PrepareSynAckMessage
+ * I: PrepareAckMessage
+ * (stream - this can happen in both directions)
+ * I: OutgoingStreamMessage
+ * F: ReceivedMessage
+ * (completion)
+ * F: CompleteMessage
+ *</pre>
+ *
+ * All messages which derive from {@link StreamMessage} are sent by the standard internode messaging
+ * (via {@link org.apache.cassandra.net.MessagingService}, while the actual files themselves are sent by a special
+ * "streaming" connection type. See {@link NettyStreamingMessageSender} for details. Because of the asynchronous
*/
public class StreamSession implements IEndpointStateChangeSubscriber
{
-
- /**
- * Version where keep-alive support was added
- */
- private static final CassandraVersion STREAM_KEEP_ALIVE = new CassandraVersion("3.10");
private static final Logger logger = LoggerFactory.getLogger(StreamSession.class);
- private static final DebuggableScheduledThreadPoolExecutor keepAliveExecutor = new DebuggableScheduledThreadPoolExecutor("StreamKeepAliveExecutor");
- static {
- // Immediately remove keep-alive task when cancelled.
- keepAliveExecutor.setRemoveOnCancelPolicy(true);
- }
+
+ // for test purpose to record received message and state transition
+ public volatile static MessageStateSink sink = MessageStateSink.NONE;
+
+ private final StreamOperation streamOperation;
/**
* Streaming endpoint.
*
- * Each {@code StreamSession} is identified by this InetAddress which is broadcast address of the node streaming.
+ * Each {@code StreamSession} is identified by this InetAddressAndPort which is broadcast address of the node streaming.
*/
- public final InetAddress peer;
+ public final InetAddressAndPort peer;
+ private final OutboundConnectionSettings template;
+
private final int index;
- /** Actual connecting address. Can be the same as {@linkplain #peer}. */
- public final InetAddress connecting;
// should not be null when session is started
private StreamResultFuture streamResult;
@@ -148,23 +153,48 @@
protected final Set<StreamRequest> requests = Sets.newConcurrentHashSet();
// streaming tasks are created and managed per ColumnFamily ID
@VisibleForTesting
- protected final ConcurrentHashMap<UUID, StreamTransferTask> transfers = new ConcurrentHashMap<>();
+ protected final ConcurrentHashMap<TableId, StreamTransferTask> transfers = new ConcurrentHashMap<>();
// data receivers, filled after receiving prepare message
- private final Map<UUID, StreamReceiveTask> receivers = new ConcurrentHashMap<>();
+ private final Map<TableId, StreamReceiveTask> receivers = new ConcurrentHashMap<>();
private final StreamingMetrics metrics;
- /* can be null when session is created in remote */
- private final StreamConnectionFactory factory;
- public final Map<String, Set<Range<Token>>> transferredRangesPerKeyspace = new HashMap<>();
+ final Map<String, Set<Range<Token>>> transferredRangesPerKeyspace = new HashMap<>();
- public final ConnectionHandler handler;
+ private final boolean isFollower;
+ private final NettyStreamingMessageSender messageSender;
+ // contains both inbound and outbound channels
+ private final ConcurrentMap<ChannelId, Channel> channels = new ConcurrentHashMap<>();
- private AtomicBoolean isAborted = new AtomicBoolean(false);
- private final boolean keepSSTableLevel;
- private final boolean isIncremental;
- private ScheduledFuture<?> keepAliveFuture = null;
+ // "maybeCompleted()" should be executed at most once. Because it can be executed asynchronously by IO
+ // threads(serialization/deserialization) and stream messaging processing thread, causing connection closed before
+ // receiving peer's CompleteMessage.
+ private boolean maybeCompleted = false;
+ private Future closeFuture;
- public static enum State
+ private final UUID pendingRepair;
+ private final PreviewKind previewKind;
+
+ /**
+ * State Transition:
+ *
+ * <pre>
+ * +------------------+----------> FAILED <--------------------+
+ * | | ^ |
+ * | | | initiator |
+ * INITIALIZED --> PREPARING --> STREAMING ------------> WAIT_COMPLETE ----> COMPLETED
+ * | | | ^ ^
+ * | | | follower | |
+ * | | +-------------------------)-----------------+
+ * | | | |
+ * | | if preview | |
+ * | +----------------------------------------+ |
+ * | nothing to request or to transfer |
+ * +-----------------------------------------------------------------------------+
+ * nothing to request or to transfer
+ *
+ * </pre>
+ */
+ public enum State
{
INITIALIZED(false),
PREPARING(false),
@@ -185,32 +215,35 @@
*/
public boolean isFinalState()
{
- return finalState;
+ return finalState;
}
}
private volatile State state = State.INITIALIZED;
- private volatile boolean completeSent = false;
/**
* Create new streaming session with the peer.
- *
- * @param peer Address of streaming peer
- * @param connecting Actual connecting address
- * @param factory is used for establishing connection
*/
- public StreamSession(InetAddress peer, InetAddress connecting, StreamConnectionFactory factory, int index, boolean keepSSTableLevel, boolean isIncremental)
+ public StreamSession(StreamOperation streamOperation, InetAddressAndPort peer, StreamConnectionFactory factory,
+ boolean isFollower, int index, UUID pendingRepair, PreviewKind previewKind)
{
+ this.streamOperation = streamOperation;
this.peer = peer;
- this.connecting = connecting;
+ this.template = new OutboundConnectionSettings(peer);
+ this.isFollower = isFollower;
this.index = index;
- this.factory = factory;
- this.handler = new ConnectionHandler(this, isKeepAliveSupported()?
- (int)TimeUnit.SECONDS.toMillis(2 * DatabaseDescriptor.getStreamingKeepAlivePeriod()) :
- DatabaseDescriptor.getStreamingSocketTimeout());
- this.metrics = StreamingMetrics.get(connecting);
- this.keepSSTableLevel = keepSSTableLevel;
- this.isIncremental = isIncremental;
+
+ this.messageSender = new NettyStreamingMessageSender(this, template, factory, current_version, previewKind.isPreview());
+ this.metrics = StreamingMetrics.get(peer);
+ this.pendingRepair = pendingRepair;
+ this.previewKind = previewKind;
+
+ logger.debug("Creating stream session to {} as {}", template, isFollower ? "follower" : "initiator");
+ }
+
+ public boolean isFollower()
+ {
+ return isFollower;
}
public UUID planId()
@@ -223,32 +256,35 @@
return index;
}
- public String description()
+ public StreamOperation streamOperation()
{
- return streamResult == null ? null : streamResult.description;
+ return streamResult == null ? null : streamResult.streamOperation;
}
- public boolean keepSSTableLevel()
+ public StreamOperation getStreamOperation()
{
- return keepSSTableLevel;
+ return streamOperation;
}
- public boolean isIncremental()
+ public UUID getPendingRepair()
{
- return isIncremental;
+ return pendingRepair;
}
-
- StreamReceiveTask getReceivingTask(UUID cfId)
+ public boolean isPreview()
{
- assert receivers.containsKey(cfId);
- return receivers.get(cfId);
+ return previewKind.isPreview();
}
- private boolean isKeepAliveSupported()
+ public PreviewKind getPreviewKind()
{
- CassandraVersion peerVersion = Gossiper.instance.getReleaseVersion(peer);
- return STREAM_KEEP_ALIVE.isSupportedBy(peerVersion);
+ return previewKind;
+ }
+
+ public StreamReceiver getAggregator(TableId tableId)
+ {
+ assert receivers.containsKey(tableId) : "Missing tableId " + tableId;
+ return receivers.get(tableId).getReceiver();
}
/**
@@ -261,13 +297,53 @@
{
this.streamResult = streamResult;
StreamHook.instance.reportStreamFuture(this, streamResult);
-
- if (isKeepAliveSupported())
- scheduleKeepAliveTask();
- else
- logger.debug("Peer {} does not support keep-alive.", peer);
}
+ /**
+ * Attach a channel to this session upon receiving the first inbound message.
+ *
+ * @param channel The channel to attach.
+ * @param isControlChannel If the channel is the one to send control messages to.
+ * @return False if the channel was already attached, true otherwise.
+ */
+ public synchronized boolean attachInbound(Channel channel, boolean isControlChannel)
+ {
+ failIfFinished();
+
+ if (!messageSender.hasControlChannel() && isControlChannel)
+ messageSender.injectControlMessageChannel(channel);
+
+ channel.closeFuture().addListener(ignored -> onChannelClose(channel));
+ return channels.putIfAbsent(channel.id(), channel) == null;
+ }
+
+ /**
+ * Attach a channel to this session upon sending the first outbound message.
+ *
+ * @param channel The channel to attach.
+ * @return False if the channel was already attached, true otherwise.
+ */
+ public synchronized boolean attachOutbound(Channel channel)
+ {
+ failIfFinished();
+
+ channel.closeFuture().addListener(ignored -> onChannelClose(channel));
+ return channels.putIfAbsent(channel.id(), channel) == null;
+ }
+
+ /**
+ * On channel closing, if no channels are left just close the message sender; this must be closed last to ensure
+ * keep alive messages are sent until the very end of the streaming session.
+ */
+ private synchronized void onChannelClose(Channel channel)
+ {
+ if (channels.remove(channel.id()) != null && channels.isEmpty())
+ messageSender.close();
+ }
+
+ /**
+ * invoked by the node that begins the stream session (it may be sending files, receiving files, or both)
+ */
public void start()
{
if (requests.isEmpty() && transfers.isEmpty())
@@ -281,8 +357,8 @@
{
logger.info("[Stream #{}] Starting streaming to {}{}", planId(),
peer,
- peer.equals(connecting) ? "" : " through " + connecting);
- handler.initiate();
+ template.connectTo == null ? "" : " through " + template.connectTo);
+ messageSender.initialize();
onInitializationComplete();
}
catch (Exception e)
@@ -292,60 +368,54 @@
}
}
- public Socket createConnection() throws IOException
- {
- assert factory != null;
- return factory.createConnection(connecting);
- }
-
/**
* Request data fetch task to this session.
*
+ * Here, we have to encode both _local_ range transientness (encoded in Replica itself, in RangesAtEndpoint)
+ * and _remote_ (source) range transientmess, which is encoded by splitting ranges into full and transient.
+ *
* @param keyspace Requesting keyspace
- * @param ranges Ranges to retrieve data
+ * @param fullRanges Ranges to retrieve data that will return full data from the source
+ * @param transientRanges Ranges to retrieve data that will return transient data from the source
* @param columnFamilies ColumnFamily names. Can be empty if requesting all CF under the keyspace.
*/
- public void addStreamRequest(String keyspace, Collection<Range<Token>> ranges, Collection<String> columnFamilies, long repairedAt)
+ public void addStreamRequest(String keyspace, RangesAtEndpoint fullRanges, RangesAtEndpoint transientRanges, Collection<String> columnFamilies)
{
- requests.add(new StreamRequest(keyspace, ranges, columnFamilies, repairedAt));
+ //It should either be a dummy address for repair or if it's a bootstrap/move/rebuild it should be this node
+ assert all(fullRanges, Replica::isSelf) || RangesAtEndpoint.isDummyList(fullRanges) : fullRanges.toString();
+ assert all(transientRanges, Replica::isSelf) || RangesAtEndpoint.isDummyList(transientRanges) : transientRanges.toString();
+
+ requests.add(new StreamRequest(keyspace, fullRanges, transientRanges, columnFamilies));
}
/**
* Set up transfer for specific keyspace/ranges/CFs
*
- * Used in repair - a streamed sstable in repair will be marked with the given repairedAt time
- *
* @param keyspace Transfer keyspace
- * @param ranges Transfer ranges
+ * @param replicas Transfer ranges
* @param columnFamilies Transfer ColumnFamilies
* @param flushTables flush tables?
- * @param repairedAt the time the repair started.
*/
- public synchronized void addTransferRanges(String keyspace, Collection<Range<Token>> ranges, Collection<String> columnFamilies, boolean flushTables, long repairedAt)
+ synchronized void addTransferRanges(String keyspace, RangesAtEndpoint replicas, Collection<String> columnFamilies, boolean flushTables)
{
failIfFinished();
Collection<ColumnFamilyStore> stores = getColumnFamilyStores(keyspace, columnFamilies);
if (flushTables)
flushSSTables(stores);
- List<Range<Token>> normalizedRanges = Range.normalize(ranges);
- List<SSTableStreamingSections> sections = getSSTableSectionsForRanges(normalizedRanges, stores, repairedAt, isIncremental);
- try
+ //Was it safe to remove this normalize, sorting seems not to matter, merging? Maybe we should have?
+ //Do we need to unwrap here also or is that just making it worse?
+ //Range and if it's transient
+ RangesAtEndpoint unwrappedRanges = replicas.unwrap();
+ List<OutgoingStream> streams = getOutgoingStreamsForRanges(unwrappedRanges, stores, pendingRepair, previewKind);
+ addTransferStreams(streams);
+ Set<Range<Token>> toBeUpdated = transferredRangesPerKeyspace.get(keyspace);
+ if (toBeUpdated == null)
{
- addTransferFiles(sections);
- Set<Range<Token>> toBeUpdated = transferredRangesPerKeyspace.get(keyspace);
- if (toBeUpdated == null)
- {
- toBeUpdated = new HashSet<>();
- }
- toBeUpdated.addAll(ranges);
- transferredRangesPerKeyspace.put(keyspace, toBeUpdated);
+ toBeUpdated = new HashSet<>();
}
- finally
- {
- for (SSTableStreamingSections release : sections)
- release.ref.release();
- }
+ toBeUpdated.addAll(replicas.ranges());
+ transferredRangesPerKeyspace.put(keyspace, toBeUpdated);
}
private void failIfFinished()
@@ -371,130 +441,83 @@
}
@VisibleForTesting
- public static List<SSTableStreamingSections> getSSTableSectionsForRanges(Collection<Range<Token>> ranges, Collection<ColumnFamilyStore> stores, long overriddenRepairedAt, final boolean isIncremental)
+ public List<OutgoingStream> getOutgoingStreamsForRanges(RangesAtEndpoint replicas, Collection<ColumnFamilyStore> stores, UUID pendingRepair, PreviewKind previewKind)
{
- Refs<SSTableReader> refs = new Refs<>();
+ List<OutgoingStream> streams = new ArrayList<>();
try
{
- for (ColumnFamilyStore cfStore : stores)
+ for (ColumnFamilyStore cfs: stores)
{
- final List<Range<PartitionPosition>> keyRanges = new ArrayList<>(ranges.size());
- for (Range<Token> range : ranges)
- keyRanges.add(Range.makeRowRange(range));
- refs.addAll(cfStore.selectAndReference(view -> {
- Set<SSTableReader> sstables = Sets.newHashSet();
- SSTableIntervalTree intervalTree = SSTableIntervalTree.build(view.select(SSTableSet.CANONICAL));
- for (Range<PartitionPosition> keyRange : keyRanges)
- {
- // keyRange excludes its start, while sstableInBounds is inclusive (of both start and end).
- // This is fine however, because keyRange has been created from a token range through Range.makeRowRange (see above).
- // And that later method uses the Token.maxKeyBound() method to creates the range, which return a "fake" key that
- // sort after all keys having the token. That "fake" key cannot however be equal to any real key, so that even
- // including keyRange.left will still exclude any key having the token of the original token range, and so we're
- // still actually selecting what we wanted.
- for (SSTableReader sstable : View.sstablesInBounds(keyRange.left, keyRange.right, intervalTree))
- {
- if (!isIncremental || !sstable.isRepaired())
- sstables.add(sstable);
- }
- }
-
- if (logger.isDebugEnabled())
- logger.debug("ViewFilter for {}/{} sstables", sstables.size(), Iterables.size(view.select(SSTableSet.CANONICAL)));
- return sstables;
- }).refs);
+ streams.addAll(cfs.getStreamManager().createOutgoingStreams(this, replicas, pendingRepair, previewKind));
}
-
- List<SSTableStreamingSections> sections = new ArrayList<>(refs.size());
- for (SSTableReader sstable : refs)
- {
- long repairedAt = overriddenRepairedAt;
- if (overriddenRepairedAt == ActiveRepairService.UNREPAIRED_SSTABLE)
- repairedAt = sstable.getSSTableMetadata().repairedAt;
- sections.add(new SSTableStreamingSections(refs.get(sstable),
- sstable.getPositionsForRanges(ranges),
- sstable.estimatedKeysForRanges(ranges),
- repairedAt));
- }
- return sections;
}
catch (Throwable t)
{
- refs.release();
+ streams.forEach(OutgoingStream::finish);
throw t;
}
+ return streams;
}
- public synchronized void addTransferFiles(Collection<SSTableStreamingSections> sstableDetails)
+ synchronized void addTransferStreams(Collection<OutgoingStream> streams)
{
failIfFinished();
- Iterator<SSTableStreamingSections> iter = sstableDetails.iterator();
- while (iter.hasNext())
+ for (OutgoingStream stream: streams)
{
- SSTableStreamingSections details = iter.next();
- if (details.sections.isEmpty())
- {
- // A reference was acquired on the sstable and we won't stream it
- details.ref.release();
- iter.remove();
- continue;
- }
-
- UUID cfId = details.ref.get().metadata.cfId;
- StreamTransferTask task = transfers.get(cfId);
+ TableId tableId = stream.getTableId();
+ StreamTransferTask task = transfers.get(tableId);
if (task == null)
{
//guarantee atomicity
- StreamTransferTask newTask = new StreamTransferTask(this, cfId);
- task = transfers.putIfAbsent(cfId, newTask);
+ StreamTransferTask newTask = new StreamTransferTask(this, tableId);
+ task = transfers.putIfAbsent(tableId, newTask);
if (task == null)
task = newTask;
}
- task.addTransferFile(details.ref, details.estimatedKeys, details.sections, details.repairedAt);
- iter.remove();
+ task.addTransferStream(stream);
}
}
- public static class SSTableStreamingSections
+ private synchronized Future closeSession(State finalState)
{
- public final Ref<SSTableReader> ref;
- public final List<Pair<Long, Long>> sections;
- public final long estimatedKeys;
- public final long repairedAt;
+ // it's session is already closed
+ if (closeFuture != null)
+ return closeFuture;
- public SSTableStreamingSections(Ref<SSTableReader> ref, List<Pair<Long, Long>> sections, long estimatedKeys, long repairedAt)
+ state(finalState);
+
+ List<Future> futures = new ArrayList<>();
+
+ // ensure aborting the tasks do not happen on the network IO thread (read: netty event loop)
+ // as we don't want any blocking disk IO to stop the network thread
+ if (finalState == State.FAILED)
+ futures.add(ScheduledExecutors.nonPeriodicTasks.submit(this::abortTasks));
+
+ // Channels should only be closed by the initiator; but, if this session closed
+ // due to failure, channels should be always closed regardless, even if this is not the initator.
+ if (!isFollower || state != State.COMPLETE)
{
- this.ref = ref;
- this.sections = sections;
- this.estimatedKeys = estimatedKeys;
- this.repairedAt = repairedAt;
+ logger.debug("[Stream #{}] Will close attached channels {}", planId(), channels);
+ channels.values().forEach(channel -> futures.add(channel.close()));
}
+
+ sink.onClose(peer);
+ streamResult.handleSessionComplete(this);
+ closeFuture = FBUtilities.allOf(futures);
+
+ return closeFuture;
}
- private synchronized void closeSession(State finalState)
+ private void abortTasks()
{
- if (isAborted.compareAndSet(false, true))
+ try
{
- state(finalState);
-
- if (finalState == State.FAILED)
- {
- for (StreamTask task : Iterables.concat(receivers.values(), transfers.values()))
- task.abort();
- }
-
- if (keepAliveFuture != null)
- {
- logger.debug("[Stream #{}] Finishing keep-alive task.", planId());
- keepAliveFuture.cancel(false);
- keepAliveFuture = null;
- }
-
- // Note that we shouldn't block on this close because this method is called on the handler
- // incoming thread (so we would deadlock).
- handler.close();
-
- streamResult.handleSessionComplete(this);
+ receivers.values().forEach(StreamReceiveTask::abort);
+ transfers.values().forEach(StreamTransferTask::abort);
+ }
+ catch (Exception e)
+ {
+ logger.warn("[Stream #{}] failed to abort some streaming tasks", planId(), e);
}
}
@@ -505,6 +528,10 @@
*/
public void state(State newState)
{
+ if (logger.isTraceEnabled())
+ logger.trace("[Stream #{}] Changing session state from {} to {}", planId(), state, newState);
+
+ sink.recordState(peer, newState);
state = newState;
}
@@ -516,6 +543,11 @@
return state;
}
+ public NettyStreamingMessageSender getMessageSender()
+ {
+ return messageSender;
+ }
+
/**
* Return if this session completed successfully.
*
@@ -526,31 +558,50 @@
return state == State.COMPLETE;
}
- public void messageReceived(StreamMessage message)
+ public synchronized void messageReceived(StreamMessage message)
{
+ if (message.type != StreamMessage.Type.KEEP_ALIVE)
+ failIfFinished();
+
+ sink.recordMessage(peer, message.type);
+
switch (message.type)
{
- case PREPARE:
- PrepareMessage msg = (PrepareMessage) message;
+ case STREAM_INIT:
+ // at follower, nop
+ break;
+ case PREPARE_SYN:
+ // at follower
+ PrepareSynMessage msg = (PrepareSynMessage) message;
prepare(msg.requests, msg.summaries);
break;
-
- case FILE:
- receive((IncomingFileMessage) message);
+ case PREPARE_SYNACK:
+ // at initiator
+ prepareSynAck((PrepareSynAckMessage) message);
break;
-
+ case PREPARE_ACK:
+ // at follower
+ prepareAck((PrepareAckMessage) message);
+ break;
+ case STREAM:
+ receive((IncomingStreamMessage) message);
+ break;
case RECEIVED:
ReceivedMessage received = (ReceivedMessage) message;
- received(received.cfId, received.sequenceNumber);
+ received(received.tableId, received.sequenceNumber);
break;
-
case COMPLETE:
+ // at initiator
complete();
break;
-
+ case KEEP_ALIVE:
+ // NOP - we only send/receive the KEEP_ALIVE to force the TCP connection to remain open
+ break;
case SESSION_FAILED:
sessionFailed();
break;
+ default:
+ throw new AssertionError("unhandled StreamMessage type: " + message.getClass().getName());
}
}
@@ -561,55 +612,78 @@
{
// send prepare message
state(State.PREPARING);
- PrepareMessage prepare = new PrepareMessage();
+ PrepareSynMessage prepare = new PrepareSynMessage();
prepare.requests.addAll(requests);
+ long totalBytesToStream = 0;
+ long totalSSTablesStreamed = 0;
for (StreamTransferTask task : transfers.values())
+ {
+ totalBytesToStream += task.getTotalSize();
+ totalSSTablesStreamed += task.getTotalNumberOfFiles();
prepare.summaries.add(task.getSummary());
- handler.sendMessage(prepare);
+ }
- // if we don't need to prepare for receiving stream, start sending files immediately
- if (requests.isEmpty())
- startStreamingFiles();
+ if(StreamOperation.REPAIR == getStreamOperation())
+ {
+ StreamingMetrics.totalOutgoingRepairBytes.inc(totalBytesToStream);
+ StreamingMetrics.totalOutgoingRepairSSTables.inc(totalSSTablesStreamed);
+ }
+
+ messageSender.sendMessage(prepare);
}
- /**l
- * Call back for handling exception during streaming.
- *
- * @param e thrown exception
+ /**
+ * Signal an error to this stream session: if it's an EOF exception, it tries to understand if the socket was closed
+ * after completion or because the peer was down, otherwise sends a {@link SessionFailedMessage} and closes
+ * the session as {@link State#FAILED}.
*/
- public void onError(Throwable e)
+ public synchronized Future onError(Throwable e)
{
+ boolean isEofException = e instanceof EOFException;
+ if (isEofException)
+ {
+ if (state.finalState)
+ {
+ logger.debug("[Stream #{}] Socket closed after session completed with state {}", planId(), state);
+
+ return null;
+ }
+ else
+ {
+ logger.error("[Stream #{}] Socket closed before session completion, peer {} is probably down.",
+ planId(),
+ peer.address.getHostAddress(),
+ e);
+
+ return closeSession(State.FAILED);
+ }
+ }
+
logError(e);
// send session failure message
- if (handler.isOutgoingConnected())
- handler.sendMessage(new SessionFailedMessage());
+ if (messageSender.connected())
+ messageSender.sendMessage(new SessionFailedMessage());
// fail session
- closeSession(State.FAILED);
+ return closeSession(State.FAILED);
}
private void logError(Throwable e)
{
if (e instanceof SocketTimeoutException)
{
- if (isKeepAliveSupported())
- logger.error("[Stream #{}] Did not receive response from peer {}{} for {} secs. Is peer down? " +
- "If not, maybe try increasing streaming_keep_alive_period_in_secs.", planId(),
- peer.getHostAddress(),
- peer.equals(connecting) ? "" : " through " + connecting.getHostAddress(),
- 2 * DatabaseDescriptor.getStreamingKeepAlivePeriod(),
- e);
- else
- logger.error("[Stream #{}] Streaming socket timed out. This means the session peer stopped responding or " +
- "is still processing received data. If there is no sign of failure in the other end or a very " +
- "dense table is being transferred you may want to increase streaming_socket_timeout_in_ms " +
- "property. Current value is {}ms.", planId(), DatabaseDescriptor.getStreamingSocketTimeout(), e);
+ logger.error("[Stream #{}] Did not receive response from peer {}{} for {} secs. Is peer down? " +
+ "If not, maybe try increasing streaming_keep_alive_period_in_secs.", planId(),
+ peer.getHostAddress(true),
+ template.connectTo == null ? "" : " through " + template.connectTo.getHostAddress(true),
+ 2 * DatabaseDescriptor.getStreamingKeepAlivePeriod(),
+ e);
}
else
{
logger.error("[Stream #{}] Streaming error occurred on session with peer {}{}", planId(),
- peer.getHostAddress(),
- peer.equals(connecting) ? "" : " through " + connecting.getHostAddress(),
- e);
+ peer.getHostAddress(true),
+ template.connectTo == null ? "" : " through " + template.connectTo.getHostAddress(true),
+ e);
}
}
@@ -620,56 +694,96 @@
{
// prepare tasks
state(State.PREPARING);
+ ScheduledExecutors.nonPeriodicTasks.execute(() -> prepareAsync(requests, summaries));
+ }
+
+ /**
+ * Finish preparing the session. This method is blocking (memtables are flushed in {@link #addTransferRanges}),
+ * so the logic should not execute on the main IO thread (read: netty event loop).
+ */
+ private void prepareAsync(Collection<StreamRequest> requests, Collection<StreamSummary> summaries)
+ {
for (StreamRequest request : requests)
- addTransferRanges(request.keyspace, request.ranges, request.columnFamilies, true, request.repairedAt); // always flush on stream request
+ addTransferRanges(request.keyspace, RangesAtEndpoint.concat(request.full, request.transientReplicas), request.columnFamilies, true); // always flush on stream request
for (StreamSummary summary : summaries)
prepareReceiving(summary);
- // send back prepare message if prepare message contains stream request
- if (!requests.isEmpty())
- {
- PrepareMessage prepare = new PrepareMessage();
+ PrepareSynAckMessage prepareSynAck = new PrepareSynAckMessage();
+ if (!peer.equals(FBUtilities.getBroadcastAddressAndPort()))
for (StreamTransferTask task : transfers.values())
- prepare.summaries.add(task.getSummary());
- handler.sendMessage(prepare);
+ prepareSynAck.summaries.add(task.getSummary());
+ messageSender.sendMessage(prepareSynAck);
+
+ streamResult.handleSessionPrepared(this);
+
+ if (isPreview())
+ completePreview();
+ else
+ maybeCompleted();
+ }
+
+ private void prepareSynAck(PrepareSynAckMessage msg)
+ {
+ if (!msg.summaries.isEmpty())
+ {
+ for (StreamSummary summary : msg.summaries)
+ prepareReceiving(summary);
+
+ // only send the (final) ACK if we are expecting the peer to send this node (the initiator) some files
+ if (!isPreview())
+ messageSender.sendMessage(new PrepareAckMessage());
}
- // if there are files to stream
- if (!maybeCompleted())
- startStreamingFiles();
+ if (isPreview())
+ completePreview();
+ else
+ startStreamingFiles(true);
+ }
+
+ private void prepareAck(PrepareAckMessage msg)
+ {
+ if (isPreview())
+ throw new RuntimeException(String.format("[Stream #%s] Cannot receive PrepareAckMessage for preview session", planId()));
+ startStreamingFiles(true);
}
/**
- * Call back after sending FileMessageHeader.
+ * Call back after sending StreamMessageHeader.
*
- * @param header sent header
+ * @param message sent stream message
*/
- public void fileSent(FileMessageHeader header)
+ public void streamSent(OutgoingStreamMessage message)
{
- long headerSize = header.size();
+ long headerSize = message.stream.getSize();
StreamingMetrics.totalOutgoingBytes.inc(headerSize);
metrics.outgoingBytes.inc(headerSize);
// schedule timeout for receiving ACK
- StreamTransferTask task = transfers.get(header.cfId);
+ StreamTransferTask task = transfers.get(message.header.tableId);
if (task != null)
{
- task.scheduleTimeout(header.sequenceNumber, 12, TimeUnit.HOURS);
+ task.scheduleTimeout(message.header.sequenceNumber, 12, TimeUnit.HOURS);
}
}
/**
- * Call back after receiving FileMessageHeader.
+ * Call back after receiving a stream.
*
- * @param message received file
+ * @param message received stream
*/
- public void receive(IncomingFileMessage message)
+ public void receive(IncomingStreamMessage message)
{
- long headerSize = message.header.size();
+ if (isPreview())
+ {
+ throw new RuntimeException(String.format("[Stream #%s] Cannot receive files for preview session", planId()));
+ }
+
+ long headerSize = message.stream.getSize();
StreamingMetrics.totalIncomingBytes.inc(headerSize);
metrics.incomingBytes.inc(headerSize);
// send back file received message
- handler.sendMessage(new ReceivedMessage(message.header.cfId, message.header.sequenceNumber));
- receivers.get(message.header.cfId).received(message.sstable);
+ messageSender.sendMessage(new ReceivedMessage(message.header.tableId, message.header.sequenceNumber));
+ StreamHook.instance.reportIncomingStream(message.header.tableId, message.stream, this, message.header.sequenceNumber);
+ receivers.get(message.header.tableId).received(message.stream);
}
public void progress(String filename, ProgressInfo.Direction direction, long bytes, long total)
@@ -678,9 +792,9 @@
streamResult.handleProgress(progress);
}
- public void received(UUID cfId, int sequenceNumber)
+ public void received(TableId tableId, int sequenceNumber)
{
- transfers.get(cfId).complete(sequenceNumber);
+ transfers.get(tableId).complete(sequenceNumber);
}
/**
@@ -688,30 +802,49 @@
*/
public synchronized void complete()
{
- if (state == State.WAIT_COMPLETE)
+ logger.debug("[Stream #{}] handling Complete message, state = {}", planId(), state);
+
+ if (!isFollower)
{
- if (!completeSent)
- {
- handler.sendMessage(new CompleteMessage());
- completeSent = true;
- }
- closeSession(State.COMPLETE);
+ if (state == State.WAIT_COMPLETE)
+ closeSession(State.COMPLETE);
+ else
+ state(State.WAIT_COMPLETE);
}
else
{
- state(State.WAIT_COMPLETE);
- handler.closeIncoming();
+ // pre-4.0 nodes should not be connected via streaming, see {@link MessagingService#accept_streaming}
+ throw new IllegalStateException(String.format("[Stream #%s] Complete message can be only received by the initiator!", planId()));
}
}
- private synchronized void scheduleKeepAliveTask()
+ /**
+ * Synchronize both {@link #complete()} and {@link #maybeCompleted()} to avoid racing
+ */
+ private synchronized boolean maybeCompleted()
{
- if (keepAliveFuture == null)
+ if (!(receivers.isEmpty() && transfers.isEmpty()))
+ return false;
+
+ // if already executed once, skip it
+ if (maybeCompleted)
+ return true;
+
+ maybeCompleted = true;
+ if (!isFollower)
{
- int keepAlivePeriod = DatabaseDescriptor.getStreamingKeepAlivePeriod();
- logger.debug("[Stream #{}] Scheduling keep-alive task with {}s period.", planId(), keepAlivePeriod);
- keepAliveFuture = keepAliveExecutor.scheduleAtFixedRate(new KeepAliveTask(), 0, keepAlivePeriod, TimeUnit.SECONDS);
+ if (state == State.WAIT_COMPLETE)
+ closeSession(State.COMPLETE);
+ else
+ state(State.WAIT_COMPLETE);
}
+ else
+ {
+ messageSender.sendMessage(new CompleteMessage());
+ closeSession(State.COMPLETE);
+ }
+
+ return true;
}
/**
@@ -719,7 +852,7 @@
*/
public synchronized void sessionFailed()
{
- logger.error("[Stream #{}] Remote peer {} failed stream session.", planId(), peer.getHostAddress());
+ logger.error("[Stream #{}] Remote peer {} failed stream session.", planId(), peer.toString());
closeSession(State.FAILED);
}
@@ -734,63 +867,54 @@
List<StreamSummary> transferSummaries = Lists.newArrayList();
for (StreamTask transfer : transfers.values())
transferSummaries.add(transfer.getSummary());
- return new SessionInfo(peer, index, connecting, receivingSummaries, transferSummaries, state);
+ // TODO: the connectTo treatment here is peculiar, and needs thinking about - since the connection factory can change it
+ return new SessionInfo(peer, index, template.connectTo == null ? peer : template.connectTo, receivingSummaries, transferSummaries, state);
}
public synchronized void taskCompleted(StreamReceiveTask completedTask)
{
- receivers.remove(completedTask.cfId);
+ receivers.remove(completedTask.tableId);
maybeCompleted();
}
public synchronized void taskCompleted(StreamTransferTask completedTask)
{
- transfers.remove(completedTask.cfId);
+ transfers.remove(completedTask.tableId);
maybeCompleted();
}
- public void onJoin(InetAddress endpoint, EndpointState epState) {}
- public void beforeChange(InetAddress endpoint, EndpointState currentState, ApplicationState newStateKey, VersionedValue newValue) {}
- public void onChange(InetAddress endpoint, ApplicationState state, VersionedValue value) {}
- public void onAlive(InetAddress endpoint, EndpointState state) {}
- public void onDead(InetAddress endpoint, EndpointState state) {}
+ public void onJoin(InetAddressAndPort endpoint, EndpointState epState) {}
+ public void beforeChange(InetAddressAndPort endpoint, EndpointState currentState, ApplicationState newStateKey, VersionedValue newValue) {}
+ public void onChange(InetAddressAndPort endpoint, ApplicationState state, VersionedValue value) {}
+ public void onAlive(InetAddressAndPort endpoint, EndpointState state) {}
+ public void onDead(InetAddressAndPort endpoint, EndpointState state) {}
- public void onRemove(InetAddress endpoint)
+ public void onRemove(InetAddressAndPort endpoint)
{
- logger.error("[Stream #{}] Session failed because remote peer {} has left.", planId(), peer.getHostAddress());
+ logger.error("[Stream #{}] Session failed because remote peer {} has left.", planId(), peer.toString());
closeSession(State.FAILED);
}
- public void onRestart(InetAddress endpoint, EndpointState epState)
+ public void onRestart(InetAddressAndPort endpoint, EndpointState epState)
{
- logger.error("[Stream #{}] Session failed because remote peer {} was restarted.", planId(), peer.getHostAddress());
+ logger.error("[Stream #{}] Session failed because remote peer {} was restarted.", planId(), peer.toString());
closeSession(State.FAILED);
}
- private boolean maybeCompleted()
+ private void completePreview()
{
- boolean completed = receivers.isEmpty() && transfers.isEmpty();
- if (completed)
+ try
{
- if (state == State.WAIT_COMPLETE)
- {
- if (!completeSent)
- {
- handler.sendMessage(new CompleteMessage());
- completeSent = true;
- }
- closeSession(State.COMPLETE);
- }
- else
- {
- // notify peer that this session is completed
- handler.sendMessage(new CompleteMessage());
- completeSent = true;
- state(State.WAIT_COMPLETE);
- handler.closeOutgoing();
- }
+ state(State.WAIT_COMPLETE);
+ closeSession(State.COMPLETE);
}
- return completed;
+ finally
+ {
+ // aborting the tasks here needs to be the last thing we do so that we accurately report
+ // expected streaming, but don't leak any resources held by the task
+ for (StreamTask task : Iterables.concat(receivers.values(), transfers.values()))
+ task.abort();
+ }
}
/**
@@ -805,60 +929,89 @@
FBUtilities.waitOnFutures(flushes);
}
- private synchronized void prepareReceiving(StreamSummary summary)
+ @VisibleForTesting
+ public synchronized void prepareReceiving(StreamSummary summary)
{
failIfFinished();
if (summary.files > 0)
- receivers.put(summary.cfId, new StreamReceiveTask(this, summary.cfId, summary.files, summary.totalSize));
+ receivers.put(summary.tableId, new StreamReceiveTask(this, summary.tableId, summary.files, summary.totalSize));
}
- private void startStreamingFiles()
+ private void startStreamingFiles(boolean notifyPrepared)
{
- streamResult.handleSessionPrepared(this);
+ if (notifyPrepared)
+ streamResult.handleSessionPrepared(this);
state(State.STREAMING);
+
for (StreamTransferTask task : transfers.values())
{
- Collection<OutgoingFileMessage> messages = task.getFileMessages();
- if (messages.size() > 0)
- handler.sendMessages(messages);
- else
- taskCompleted(task); // there is no file to send
- }
- }
-
- class KeepAliveTask implements Runnable
- {
- private KeepAliveMessage last = null;
-
- public void run()
- {
- //to avoid jamming the message queue, we only send if the last one was sent
- if (last == null || last.wasSent())
+ Collection<OutgoingStreamMessage> messages = task.getFileMessages();
+ if (!messages.isEmpty())
{
- logger.trace("[Stream #{}] Sending keep-alive to {}.", planId(), peer);
- last = new KeepAliveMessage();
- try
+ for (OutgoingStreamMessage ofm : messages)
{
- handler.sendMessage(last);
- }
- catch (RuntimeException e) //connection handler is closed
- {
- logger.debug("[Stream #{}] Could not send keep-alive message (perhaps stream session is finished?).", planId(), e);
+ // pass the session planId/index to the OFM (which is only set at init(), after the transfers have already been created)
+ ofm.header.addSessionInfo(this);
+ messageSender.sendMessage(ofm);
}
}
else
{
- logger.trace("[Stream #{}] Skip sending keep-alive to {} (previous was not yet sent).", planId(), peer);
+ taskCompleted(task); // there are no files to send
}
}
+ maybeCompleted();
}
@VisibleForTesting
- public static void shutdownAndWait(long timeout, TimeUnit unit) throws InterruptedException, TimeoutException
+ public int getNumRequests()
{
- List<ExecutorService> executors = ImmutableList.of(keepAliveExecutor);
- ExecutorUtils.shutdownNow(executors);
- ExecutorUtils.awaitTermination(timeout, unit, executors);
+ return requests.size();
+ }
+
+ @VisibleForTesting
+ public int getNumTransfers()
+ {
+ return transferredRangesPerKeyspace.size();
+ }
+
+ @VisibleForTesting
+ public static interface MessageStateSink
+ {
+ static final MessageStateSink NONE = new MessageStateSink() {
+ @Override
+ public void recordState(InetAddressAndPort from, State state)
+ {
+ }
+
+ @Override
+ public void recordMessage(InetAddressAndPort from, StreamMessage.Type message)
+ {
+ }
+
+ @Override
+ public void onClose(InetAddressAndPort from)
+ {
+ }
+ };
+
+ /**
+ * @param from peer that is connected in the stream session
+ * @param state new state to change to
+ */
+ public void recordState(InetAddressAndPort from, StreamSession.State state);
+
+ /**
+ * @param from peer that sends the given message
+ * @param message stream message sent by peer
+ */
+ public void recordMessage(InetAddressAndPort from, StreamMessage.Type message);
+
+ /**
+ *
+ * @param from peer that is being disconnected
+ */
+ public void onClose(InetAddressAndPort from);
}
}
diff --git a/src/java/org/apache/cassandra/streaming/StreamState.java b/src/java/org/apache/cassandra/streaming/StreamState.java
index db50c2a..be37677 100644
--- a/src/java/org/apache/cassandra/streaming/StreamState.java
+++ b/src/java/org/apache/cassandra/streaming/StreamState.java
@@ -18,11 +18,13 @@
package org.apache.cassandra.streaming;
import java.io.Serializable;
+import java.util.List;
import java.util.Set;
import java.util.UUID;
import com.google.common.base.Predicate;
import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
/**
* Current snapshot of streaming progress.
@@ -30,14 +32,14 @@
public class StreamState implements Serializable
{
public final UUID planId;
- public final String description;
+ public final StreamOperation streamOperation;
public final Set<SessionInfo> sessions;
- public StreamState(UUID planId, String description, Set<SessionInfo> sessions)
+ public StreamState(UUID planId, StreamOperation streamOperation, Set<SessionInfo> sessions)
{
this.planId = planId;
- this.description = description;
this.sessions = sessions;
+ this.streamOperation = streamOperation;
}
public boolean hasFailedSession()
@@ -50,4 +52,9 @@
}
});
}
+
+ public List<SessionSummary> createSummaries()
+ {
+ return Lists.newArrayList(Iterables.transform(sessions, SessionInfo::createSummary));
+ }
}
diff --git a/src/java/org/apache/cassandra/streaming/StreamSummary.java b/src/java/org/apache/cassandra/streaming/StreamSummary.java
index c427283..3f957c6 100644
--- a/src/java/org/apache/cassandra/streaming/StreamSummary.java
+++ b/src/java/org/apache/cassandra/streaming/StreamSummary.java
@@ -19,7 +19,6 @@
import java.io.IOException;
import java.io.Serializable;
-import java.util.UUID;
import com.google.common.base.Objects;
@@ -27,8 +26,7 @@
import org.apache.cassandra.io.IVersionedSerializer;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
-import org.apache.cassandra.net.MessagingService;
-import org.apache.cassandra.utils.UUIDSerializer;
+import org.apache.cassandra.schema.TableId;
/**
* Summary of streaming.
@@ -37,7 +35,7 @@
{
public static final IVersionedSerializer<StreamSummary> serializer = new StreamSummarySerializer();
- public final UUID cfId;
+ public final TableId tableId;
/**
* Number of files to transfer. Can be 0 if nothing to transfer for some streaming request.
@@ -45,9 +43,9 @@
public final int files;
public final long totalSize;
- public StreamSummary(UUID cfId, int files, long totalSize)
+ public StreamSummary(TableId tableId, int files, long totalSize)
{
- this.cfId = cfId;
+ this.tableId = tableId;
this.files = files;
this.totalSize = totalSize;
}
@@ -58,20 +56,20 @@
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
StreamSummary summary = (StreamSummary) o;
- return files == summary.files && totalSize == summary.totalSize && cfId.equals(summary.cfId);
+ return files == summary.files && totalSize == summary.totalSize && tableId.equals(summary.tableId);
}
@Override
public int hashCode()
{
- return Objects.hashCode(cfId, files, totalSize);
+ return Objects.hashCode(tableId, files, totalSize);
}
@Override
public String toString()
{
final StringBuilder sb = new StringBuilder("StreamSummary{");
- sb.append("path=").append(cfId);
+ sb.append("path=").append(tableId);
sb.append(", files=").append(files);
sb.append(", totalSize=").append(totalSize);
sb.append('}');
@@ -80,25 +78,24 @@
public static class StreamSummarySerializer implements IVersionedSerializer<StreamSummary>
{
- // arbitrary version is fine for UUIDSerializer for now...
public void serialize(StreamSummary summary, DataOutputPlus out, int version) throws IOException
{
- UUIDSerializer.serializer.serialize(summary.cfId, out, MessagingService.current_version);
+ summary.tableId.serialize(out);
out.writeInt(summary.files);
out.writeLong(summary.totalSize);
}
public StreamSummary deserialize(DataInputPlus in, int version) throws IOException
{
- UUID cfId = UUIDSerializer.serializer.deserialize(in, MessagingService.current_version);
+ TableId tableId = TableId.deserialize(in);
int files = in.readInt();
long totalSize = in.readLong();
- return new StreamSummary(cfId, files, totalSize);
+ return new StreamSummary(tableId, files, totalSize);
}
public long serializedSize(StreamSummary summary, int version)
{
- long size = UUIDSerializer.serializer.serializedSize(summary.cfId, MessagingService.current_version);
+ long size = summary.tableId.serializedSize();
size += TypeSizes.sizeof(summary.files);
size += TypeSizes.sizeof(summary.totalSize);
return size;
diff --git a/src/java/org/apache/cassandra/streaming/StreamTask.java b/src/java/org/apache/cassandra/streaming/StreamTask.java
index ac72cff..1e22c34 100644
--- a/src/java/org/apache/cassandra/streaming/StreamTask.java
+++ b/src/java/org/apache/cassandra/streaming/StreamTask.java
@@ -17,7 +17,7 @@
*/
package org.apache.cassandra.streaming;
-import java.util.UUID;
+import org.apache.cassandra.schema.TableId;
/**
* StreamTask is an abstraction of the streaming task performed over specific ColumnFamily.
@@ -27,12 +27,12 @@
/** StreamSession that this task belongs */
protected final StreamSession session;
- protected final UUID cfId;
+ protected final TableId tableId;
- protected StreamTask(StreamSession session, UUID cfId)
+ protected StreamTask(StreamSession session, TableId tableId)
{
this.session = session;
- this.cfId = cfId;
+ this.tableId = tableId;
}
/**
@@ -56,6 +56,6 @@
*/
public StreamSummary getSummary()
{
- return new StreamSummary(cfId, getTotalNumberOfFiles(), getTotalSize());
+ return new StreamSummary(tableId, getTotalNumberOfFiles(), getTotalSize());
}
}
diff --git a/src/java/org/apache/cassandra/streaming/StreamTransferTask.java b/src/java/org/apache/cassandra/streaming/StreamTransferTask.java
index 4f313c3..0f7a834 100644
--- a/src/java/org/apache/cassandra/streaming/StreamTransferTask.java
+++ b/src/java/org/apache/cassandra/streaming/StreamTransferTask.java
@@ -17,54 +17,67 @@
*/
package org.apache.cassandra.streaming;
-import java.util.*;
-import java.util.concurrent.*;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicInteger;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
import com.google.common.base.Throwables;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.apache.cassandra.concurrent.NamedThreadFactory;
-import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.streaming.messages.OutgoingFileMessage;
-import org.apache.cassandra.utils.Pair;
-import org.apache.cassandra.utils.concurrent.Ref;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.streaming.messages.OutgoingStreamMessage;
+
+import static org.apache.cassandra.utils.ExecutorUtils.awaitTermination;
+import static org.apache.cassandra.utils.ExecutorUtils.shutdown;
/**
- * StreamTransferTask sends sections of SSTable files in certain ColumnFamily.
+ * StreamTransferTask sends streams for a given table
*/
public class StreamTransferTask extends StreamTask
{
+ private static final Logger logger = LoggerFactory.getLogger(StreamTransferTask.class);
private static final ScheduledExecutorService timeoutExecutor = Executors.newSingleThreadScheduledExecutor(new NamedThreadFactory("StreamingTransferTaskTimeouts"));
private final AtomicInteger sequenceNumber = new AtomicInteger(0);
private boolean aborted = false;
@VisibleForTesting
- protected final Map<Integer, OutgoingFileMessage> files = new HashMap<>();
+ protected final Map<Integer, OutgoingStreamMessage> streams = new HashMap<>();
private final Map<Integer, ScheduledFuture> timeoutTasks = new HashMap<>();
- private long totalSize;
+ private long totalSize = 0;
+ private int totalFiles = 0;
- public StreamTransferTask(StreamSession session, UUID cfId)
+ public StreamTransferTask(StreamSession session, TableId tableId)
{
- super(session, cfId);
+ super(session, tableId);
}
- public synchronized void addTransferFile(Ref<SSTableReader> ref, long estimatedKeys, List<Pair<Long, Long>> sections, long repairedAt)
+ public synchronized void addTransferStream(OutgoingStream stream)
{
- assert ref.get() != null && cfId.equals(ref.get().metadata.cfId);
- OutgoingFileMessage message = new OutgoingFileMessage(ref, sequenceNumber.getAndIncrement(), estimatedKeys, sections, repairedAt, session.keepSSTableLevel());
- message = StreamHook.instance.reportOutgoingFile(session, ref.get(), message);
- files.put(message.header.sequenceNumber, message);
- totalSize += message.header.size();
+ Preconditions.checkArgument(tableId.equals(stream.getTableId()));
+ OutgoingStreamMessage message = new OutgoingStreamMessage(tableId, session, stream, sequenceNumber.getAndIncrement());
+ message = StreamHook.instance.reportOutgoingStream(session, stream, message);
+ streams.put(message.header.sequenceNumber, message);
+ totalSize += message.stream.getSize();
+ totalFiles += message.stream.getNumFiles();
}
/**
- * Received ACK for file at {@code sequenceNumber}.
+ * Received ACK for stream at {@code sequenceNumber}.
*
- * @param sequenceNumber sequence number of file
+ * @param sequenceNumber sequence number of stream
*/
public void complete(int sequenceNumber)
{
@@ -75,11 +88,12 @@
if (timeout != null)
timeout.cancel(false);
- OutgoingFileMessage file = files.remove(sequenceNumber);
- if (file != null)
- file.complete();
+ OutgoingStreamMessage stream = streams.remove(sequenceNumber);
+ if (stream != null)
+ stream.complete();
- signalComplete = files.isEmpty();
+ logger.debug("recevied sequenceNumber {}, remaining files {}", sequenceNumber, streams.keySet());
+ signalComplete = streams.isEmpty();
}
// all file sent, notify session this task is complete.
@@ -98,11 +112,11 @@
timeoutTasks.clear();
Throwable fail = null;
- for (OutgoingFileMessage file : files.values())
+ for (OutgoingStreamMessage stream : streams.values())
{
try
{
- file.complete();
+ stream.complete();
}
catch (Throwable t)
{
@@ -110,14 +124,14 @@
else fail.addSuppressed(t);
}
}
- files.clear();
+ streams.clear();
if (fail != null)
Throwables.propagate(fail);
}
public synchronized int getTotalNumberOfFiles()
{
- return files.size();
+ return totalFiles;
}
public long getTotalSize()
@@ -125,35 +139,35 @@
return totalSize;
}
- public synchronized Collection<OutgoingFileMessage> getFileMessages()
+ public synchronized Collection<OutgoingStreamMessage> getFileMessages()
{
// We may race between queuing all those messages and the completion of the completion of
// the first ones. So copy the values to avoid a ConcurrentModificationException
- return new ArrayList<>(files.values());
+ return new ArrayList<>(streams.values());
}
- public synchronized OutgoingFileMessage createMessageForRetry(int sequenceNumber)
+ public synchronized OutgoingStreamMessage createMessageForRetry(int sequenceNumber)
{
// remove previous time out task to be rescheduled later
ScheduledFuture future = timeoutTasks.remove(sequenceNumber);
if (future != null)
future.cancel(false);
- return files.get(sequenceNumber);
+ return streams.get(sequenceNumber);
}
/**
- * Schedule timeout task to release reference for file sent.
+ * Schedule timeout task to release reference for stream sent.
* When not receiving ACK after sending to receiver in given time,
* the task will release reference.
*
- * @param sequenceNumber sequence number of file sent.
+ * @param sequenceNumber sequence number of stream sent.
* @param time time to timeout
* @param unit unit of given time
* @return scheduled future for timeout task
*/
public synchronized ScheduledFuture scheduleTimeout(final int sequenceNumber, long time, TimeUnit unit)
{
- if (!files.containsKey(sequenceNumber))
+ if (!streams.containsKey(sequenceNumber))
return null;
ScheduledFuture future = timeoutExecutor.schedule(new Runnable()
@@ -173,4 +187,11 @@
assert prev == null;
return future;
}
+
+ @VisibleForTesting
+ public static void shutdownAndWait(long timeout, TimeUnit units) throws InterruptedException, TimeoutException
+ {
+ shutdown(timeoutExecutor);
+ awaitTermination(timeout, units, timeoutExecutor);
+ }
}
diff --git a/src/java/org/apache/cassandra/streaming/StreamWriter.java b/src/java/org/apache/cassandra/streaming/StreamWriter.java
deleted file mode 100644
index 6c86c8b..0000000
--- a/src/java/org/apache/cassandra/streaming/StreamWriter.java
+++ /dev/null
@@ -1,158 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.streaming;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.util.Collection;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.ning.compress.lzf.LZFOutputStream;
-
-import org.apache.cassandra.io.sstable.Component;
-import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.io.util.DataIntegrityMetadata;
-import org.apache.cassandra.io.util.DataIntegrityMetadata.ChecksumValidator;
-import org.apache.cassandra.io.util.DataOutputStreamPlus;
-import org.apache.cassandra.io.util.RandomAccessReader;
-import org.apache.cassandra.streaming.StreamManager.StreamRateLimiter;
-import org.apache.cassandra.utils.FBUtilities;
-import org.apache.cassandra.utils.Pair;
-
-/**
- * StreamWriter writes given section of the SSTable to given channel.
- */
-public class StreamWriter
-{
- private static final int DEFAULT_CHUNK_SIZE = 64 * 1024;
-
- private static final Logger logger = LoggerFactory.getLogger(StreamWriter.class);
-
- protected final SSTableReader sstable;
- protected final Collection<Pair<Long, Long>> sections;
- protected final StreamRateLimiter limiter;
- protected final StreamSession session;
-
- private OutputStream compressedOutput;
-
- // allocate buffer to use for transfers only once
- private byte[] transferBuffer;
-
- public StreamWriter(SSTableReader sstable, Collection<Pair<Long, Long>> sections, StreamSession session)
- {
- this.session = session;
- this.sstable = sstable;
- this.sections = sections;
- this.limiter = StreamManager.getRateLimiter(session.peer);
- }
-
- /**
- * Stream file of specified sections to given channel.
- *
- * StreamWriter uses LZF compression on wire to decrease size to transfer.
- *
- * @param output where this writes data to
- * @throws IOException on any I/O error
- */
- public void write(DataOutputStreamPlus output) throws IOException
- {
- long totalSize = totalSize();
- logger.debug("[Stream #{}] Start streaming file {} to {}, repairedAt = {}, totalSize = {}", session.planId(),
- sstable.getFilename(), session.peer, sstable.getSSTableMetadata().repairedAt, totalSize);
-
- try(RandomAccessReader file = sstable.openDataReader();
- ChecksumValidator validator = new File(sstable.descriptor.filenameFor(Component.CRC)).exists()
- ? DataIntegrityMetadata.checksumValidator(sstable.descriptor)
- : null;)
- {
- transferBuffer = validator == null ? new byte[DEFAULT_CHUNK_SIZE] : new byte[validator.chunkSize];
-
- // setting up data compression stream
- compressedOutput = new LZFOutputStream(output);
- long progress = 0L;
-
- // stream each of the required sections of the file
- for (Pair<Long, Long> section : sections)
- {
- long start = validator == null ? section.left : validator.chunkStart(section.left);
- int readOffset = (int) (section.left - start);
- // seek to the beginning of the section
- file.seek(start);
- if (validator != null)
- validator.seek(start);
-
- // length of the section to read
- long length = section.right - start;
- // tracks write progress
- long bytesRead = 0;
- while (bytesRead < length)
- {
- long lastBytesRead = write(file, validator, readOffset, length, bytesRead);
- bytesRead += lastBytesRead;
- progress += (lastBytesRead - readOffset);
- session.progress(sstable.descriptor.filenameFor(Component.DATA), ProgressInfo.Direction.OUT, progress, totalSize);
- readOffset = 0;
- }
-
- // make sure that current section is sent
- compressedOutput.flush();
- }
- logger.debug("[Stream #{}] Finished streaming file {} to {}, bytesTransferred = {}, totalSize = {}",
- session.planId(), sstable.getFilename(), session.peer, FBUtilities.prettyPrintMemory(progress), FBUtilities.prettyPrintMemory(totalSize));
- }
- }
-
- protected long totalSize()
- {
- long size = 0;
- for (Pair<Long, Long> section : sections)
- size += section.right - section.left;
- return size;
- }
-
- /**
- * Sequentially read bytes from the file and write them to the output stream
- *
- * @param reader The file reader to read from
- * @param validator validator to verify data integrity
- * @param start number of bytes to skip transfer, but include for validation.
- * @param length The full length that should be read from {@code reader}
- * @param bytesTransferred Number of bytes already read out of {@code length}
- *
- * @return Number of bytes read
- *
- * @throws java.io.IOException on any I/O error
- */
- protected long write(RandomAccessReader reader, ChecksumValidator validator, int start, long length, long bytesTransferred) throws IOException
- {
- int toTransfer = (int) Math.min(transferBuffer.length, length - bytesTransferred);
- int minReadable = (int) Math.min(transferBuffer.length, reader.length() - reader.getFilePointer());
-
- reader.readFully(transferBuffer, 0, minReadable);
- if (validator != null)
- validator.validate(transferBuffer, 0, minReadable);
-
- limiter.acquire(toTransfer - start);
- compressedOutput.write(transferBuffer, start, (toTransfer - start));
-
- return toTransfer;
- }
-}
diff --git a/src/java/org/apache/cassandra/streaming/StreamingMessageSender.java b/src/java/org/apache/cassandra/streaming/StreamingMessageSender.java
new file mode 100644
index 0000000..96e7626
--- /dev/null
+++ b/src/java/org/apache/cassandra/streaming/StreamingMessageSender.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.streaming;
+
+import java.io.IOException;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.streaming.messages.StreamMessage;
+
+public interface StreamingMessageSender
+{
+ void initialize() throws IOException;
+
+ void sendMessage(StreamMessage message) throws IOException;
+
+ boolean connected();
+
+ void close();
+}
diff --git a/src/java/org/apache/cassandra/streaming/TableStreamManager.java b/src/java/org/apache/cassandra/streaming/TableStreamManager.java
new file mode 100644
index 0000000..d97fabc
--- /dev/null
+++ b/src/java/org/apache/cassandra/streaming/TableStreamManager.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.streaming;
+
+import java.util.Collection;
+import java.util.UUID;
+
+import org.apache.cassandra.locator.RangesAtEndpoint;
+import org.apache.cassandra.streaming.messages.StreamMessageHeader;
+
+/**
+ * The main streaming hook for a storage implementation.
+ *
+ * From here, the streaming system can get instances of {@link StreamReceiver}, {@link IncomingStream},
+ * and {@link OutgoingStream}, which expose the interfaces into the the underlying storage implementation
+ * needed to make streaming work.
+ */
+public interface TableStreamManager
+{
+ /**
+ * Creates a {@link StreamReceiver} for the given session, expecting the given number of streams
+ */
+ StreamReceiver createStreamReceiver(StreamSession session, int totalStreams);
+
+ /**
+ * Creates an {@link IncomingStream} for the given header
+ */
+ IncomingStream prepareIncomingStream(StreamSession session, StreamMessageHeader header);
+
+ /**
+ * Returns a collection of {@link OutgoingStream}s that contains the data selected by the
+ * given replicas, pendingRepair, and preview.
+ *
+ * There aren't any requirements on how data is divided between the outgoing streams
+ */
+ Collection<OutgoingStream> createOutgoingStreams(StreamSession session,
+ RangesAtEndpoint replicas,
+ UUID pendingRepair,
+ PreviewKind previewKind);
+}
diff --git a/src/java/org/apache/cassandra/streaming/async/NettyStreamingMessageSender.java b/src/java/org/apache/cassandra/streaming/async/NettyStreamingMessageSender.java
new file mode 100644
index 0000000..fba56f5
--- /dev/null
+++ b/src/java/org/apache/cassandra/streaming/async/NettyStreamingMessageSender.java
@@ -0,0 +1,561 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.streaming.async;
+
+import java.io.IOError;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.ClosedByInterruptException;
+import java.util.Collection;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Throwables;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.channel.Channel;
+import io.netty.channel.ChannelFuture;
+import io.netty.channel.ChannelPipeline;
+import io.netty.util.AttributeKey;
+import io.netty.util.concurrent.Future;
+import io.netty.util.concurrent.GenericFutureListener;
+import org.apache.cassandra.concurrent.DebuggableThreadPoolExecutor;
+import org.apache.cassandra.concurrent.NamedThreadFactory;
+import org.apache.cassandra.config.Config;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.io.util.DataOutputBufferFixed;
+import org.apache.cassandra.io.util.DataOutputStreamPlus;
+import org.apache.cassandra.net.AsyncChannelPromise;
+import org.apache.cassandra.net.OutboundConnectionSettings;
+import org.apache.cassandra.net.AsyncStreamingOutputPlus;
+import org.apache.cassandra.streaming.StreamConnectionFactory;
+import org.apache.cassandra.streaming.StreamSession;
+import org.apache.cassandra.streaming.StreamingMessageSender;
+import org.apache.cassandra.streaming.messages.IncomingStreamMessage;
+import org.apache.cassandra.streaming.messages.KeepAliveMessage;
+import org.apache.cassandra.streaming.messages.OutgoingStreamMessage;
+import org.apache.cassandra.streaming.messages.StreamInitMessage;
+import org.apache.cassandra.streaming.messages.StreamMessage;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.JVMStabilityInspector;
+
+/**
+ * Responsible for sending {@link StreamMessage}s to a given peer. We manage an array of netty {@link Channel}s
+ * for sending {@link OutgoingStreamMessage} instances; all other {@link StreamMessage} types are sent via
+ * a special control channel. The reason for this is to treat those messages carefully and not let them get stuck
+ * behind a stream transfer.
+ *
+ * One of the challenges when sending streams is we might need to delay shipping the stream if:
+ *
+ * - we've exceeded our network I/O use due to rate limiting (at the cassandra level)
+ * - the receiver isn't keeping up, which causes the local TCP socket buffer to not empty, which causes epoll writes to not
+ * move any bytes to the socket, which causes buffers to stick around in user-land (a/k/a cassandra) memory.
+ *
+ * When those conditions occur, it's easy enough to reschedule processing the stream once the resources pick up
+ * (we acquire the permits from the rate limiter, or the socket drains). However, we need to ensure that
+ * no other messages are submitted to the same channel while the current stream is still being processed.
+ */
+public class NettyStreamingMessageSender implements StreamingMessageSender
+{
+ private static final Logger logger = LoggerFactory.getLogger(NettyStreamingMessageSender.class);
+
+ private static final int DEFAULT_MAX_PARALLEL_TRANSFERS = FBUtilities.getAvailableProcessors();
+ private static final int MAX_PARALLEL_TRANSFERS = Integer.parseInt(System.getProperty(Config.PROPERTY_PREFIX + "streaming.session.parallelTransfers", Integer.toString(DEFAULT_MAX_PARALLEL_TRANSFERS)));
+
+ private static final long DEFAULT_CLOSE_WAIT_IN_MILLIS = TimeUnit.MINUTES.toMillis(5);
+
+ // a simple mechansim for allowing a degree of fairnes across multiple sessions
+ private static final Semaphore fileTransferSemaphore = new Semaphore(DEFAULT_MAX_PARALLEL_TRANSFERS, true);
+
+ private final StreamSession session;
+ private final boolean isPreview;
+ private final int streamingVersion;
+ private final OutboundConnectionSettings template;
+ private final StreamConnectionFactory factory;
+
+ private volatile boolean closed;
+
+ /**
+ * A special {@link Channel} for sending non-stream streaming messages, basically anything that isn't an
+ * {@link OutgoingStreamMessage} (or an {@link IncomingStreamMessage}, but a node doesn't send that, it's only received).
+ */
+ private volatile Channel controlMessageChannel;
+
+ // note: this really doesn't need to be a LBQ, just something that's thread safe
+ private final Collection<ScheduledFuture<?>> channelKeepAlives = new LinkedBlockingQueue<>();
+
+ private final ThreadPoolExecutor fileTransferExecutor;
+
+ /**
+ * A mapping of each {@link #fileTransferExecutor} thread to a channel that can be written to (on that thread).
+ */
+ private final ConcurrentMap<Thread, Channel> threadToChannelMap = new ConcurrentHashMap<>();
+
+ /**
+ * A netty channel attribute used to indicate if a channel is currently transferring a stream. This is primarily used
+ * to indicate to the {@link KeepAliveTask} if it is safe to send a {@link KeepAliveMessage}, as sending the
+ * (application level) keep-alive in the middle of a stream would be bad news.
+ */
+ @VisibleForTesting
+ static final AttributeKey<Boolean> TRANSFERRING_FILE_ATTR = AttributeKey.valueOf("transferringFile");
+
+ public NettyStreamingMessageSender(StreamSession session, OutboundConnectionSettings template, StreamConnectionFactory factory, int streamingVersion, boolean isPreview)
+ {
+ this.session = session;
+ this.streamingVersion = streamingVersion;
+ this.template = template;
+ this.factory = factory;
+ this.isPreview = isPreview;
+
+ String name = session.peer.toString().replace(':', '.');
+ fileTransferExecutor = new DebuggableThreadPoolExecutor(1, MAX_PARALLEL_TRANSFERS, 1L, TimeUnit.SECONDS, new LinkedBlockingQueue<>(),
+ new NamedThreadFactory("NettyStreaming-Outbound-" + name));
+ fileTransferExecutor.allowCoreThreadTimeOut(true);
+ }
+
+ @Override
+ public void initialize()
+ {
+ StreamInitMessage message = new StreamInitMessage(FBUtilities.getBroadcastAddressAndPort(),
+ session.sessionIndex(),
+ session.planId(),
+ session.streamOperation(),
+ session.getPendingRepair(),
+ session.getPreviewKind());
+ sendMessage(message);
+ }
+
+ public boolean hasControlChannel()
+ {
+ return controlMessageChannel != null;
+ }
+
+ /**
+ * Used by follower to setup control message channel created by initiator
+ */
+ public void injectControlMessageChannel(Channel channel)
+ {
+ this.controlMessageChannel = channel;
+ channel.attr(TRANSFERRING_FILE_ATTR).set(Boolean.FALSE);
+ scheduleKeepAliveTask(channel);
+ }
+
+ /**
+ * Used by initiator to setup control message channel connecting to follower
+ */
+ private void setupControlMessageChannel() throws IOException
+ {
+ if (controlMessageChannel == null)
+ {
+ /*
+ * Inbound handlers are needed:
+ * a) for initiator's control channel(the first outbound channel) to receive follower's message.
+ * b) for streaming receiver (note: both initiator and follower can receive streaming files) to reveive files,
+ * in {@link Handler#setupStreamingPipeline}
+ */
+ controlMessageChannel = createChannel(true);
+ scheduleKeepAliveTask(controlMessageChannel);
+ }
+ }
+
+ private void scheduleKeepAliveTask(Channel channel)
+ {
+ int keepAlivePeriod = DatabaseDescriptor.getStreamingKeepAlivePeriod();
+ if (logger.isDebugEnabled())
+ logger.debug("{} Scheduling keep-alive task with {}s period.", createLogTag(session, channel), keepAlivePeriod);
+
+ KeepAliveTask task = new KeepAliveTask(channel, session);
+ ScheduledFuture<?> scheduledFuture = channel.eventLoop().scheduleAtFixedRate(task, 0, keepAlivePeriod, TimeUnit.SECONDS);
+ channelKeepAlives.add(scheduledFuture);
+ task.future = scheduledFuture;
+ }
+
+ private Channel createChannel(boolean isInboundHandlerNeeded) throws IOException
+ {
+ Channel channel = factory.createConnection(template, streamingVersion);
+ session.attachOutbound(channel);
+
+ if (isInboundHandlerNeeded)
+ {
+ ChannelPipeline pipeline = channel.pipeline();
+ pipeline.addLast("stream", new StreamingInboundHandler(template.to, streamingVersion, session));
+ }
+ channel.attr(TRANSFERRING_FILE_ATTR).set(Boolean.FALSE);
+ logger.debug("Creating channel id {} local {} remote {}", channel.id(), channel.localAddress(), channel.remoteAddress());
+ return channel;
+ }
+
+ static String createLogTag(StreamSession session, Channel channel)
+ {
+ StringBuilder sb = new StringBuilder(64);
+ sb.append("[Stream");
+
+ if (session != null)
+ sb.append(" #").append(session.planId());
+
+ if (channel != null)
+ sb.append(" channel: ").append(channel.id());
+
+ sb.append(']');
+ return sb.toString();
+ }
+
+ @Override
+ public void sendMessage(StreamMessage message)
+ {
+ if (closed)
+ throw new RuntimeException("stream has been closed, cannot send " + message);
+
+ if (message instanceof OutgoingStreamMessage)
+ {
+ if (isPreview)
+ throw new RuntimeException("Cannot send stream data messages for preview streaming sessions");
+ if (logger.isDebugEnabled())
+ logger.debug("{} Sending {}", createLogTag(session, null), message);
+ fileTransferExecutor.submit(new FileStreamTask((OutgoingStreamMessage)message));
+ return;
+ }
+
+ try
+ {
+ setupControlMessageChannel();
+ sendControlMessage(controlMessageChannel, message, future -> onControlMessageComplete(future, message));
+ }
+ catch (Exception e)
+ {
+ close();
+ session.onError(e);
+ }
+ }
+
+ private void sendControlMessage(Channel channel, StreamMessage message, GenericFutureListener listener) throws IOException
+ {
+ if (logger.isDebugEnabled())
+ logger.debug("{} Sending {}", createLogTag(session, channel), message);
+
+ // we anticipate that the control messages are rather small, so allocating a ByteBuf shouldn't blow out of memory.
+ long messageSize = StreamMessage.serializedSize(message, streamingVersion);
+ if (messageSize > 1 << 30)
+ {
+ throw new IllegalStateException(String.format("%s something is seriously wrong with the calculated stream control message's size: %d bytes, type is %s",
+ createLogTag(session, channel), messageSize, message.type));
+ }
+
+ // as control messages are (expected to be) small, we can simply allocate a ByteBuf here, wrap it, and send via the channel
+ ByteBuf buf = channel.alloc().directBuffer((int) messageSize, (int) messageSize);
+ ByteBuffer nioBuf = buf.nioBuffer(0, (int) messageSize);
+ @SuppressWarnings("resource")
+ DataOutputBufferFixed out = new DataOutputBufferFixed(nioBuf);
+ StreamMessage.serialize(message, out, streamingVersion, session);
+ assert nioBuf.position() == nioBuf.limit();
+ buf.writerIndex(nioBuf.position());
+
+ AsyncChannelPromise.writeAndFlush(channel, buf, listener);
+ }
+
+ /**
+ * Decides what to do after a {@link StreamMessage} is processed.
+ *
+ * Note: this is called from the netty event loop.
+ *
+ * @return null if the message was processed sucessfully; else, a {@link java.util.concurrent.Future} to indicate
+ * the status of aborting any remaining tasks in the session.
+ */
+ java.util.concurrent.Future onControlMessageComplete(Future<?> future, StreamMessage msg)
+ {
+ ChannelFuture channelFuture = (ChannelFuture)future;
+ Throwable cause = future.cause();
+ if (cause == null)
+ return null;
+
+ Channel channel = channelFuture.channel();
+ logger.error("{} failed to send a stream message/data to peer {}: msg = {}",
+ createLogTag(session, channel), template.to, msg, future.cause());
+
+ // StreamSession will invoke close(), but we have to mark this sender as closed so the session doesn't try
+ // to send any failure messages
+ return session.onError(cause);
+ }
+
+ class FileStreamTask implements Runnable
+ {
+ /**
+ * Time interval, in minutes, to wait between logging a message indicating that we're waiting on a semaphore
+ * permit to become available.
+ */
+ private static final int SEMAPHORE_UNAVAILABLE_LOG_INTERVAL = 3;
+
+ /**
+ * Even though we expect only an {@link OutgoingStreamMessage} at runtime, the type here is {@link StreamMessage}
+ * to facilitate simpler testing.
+ */
+ private final StreamMessage msg;
+
+ FileStreamTask(OutgoingStreamMessage ofm)
+ {
+ this.msg = ofm;
+ }
+
+ /**
+ * For testing purposes
+ */
+ FileStreamTask(StreamMessage msg)
+ {
+ this.msg = msg;
+ }
+
+ @Override
+ public void run()
+ {
+ if (!acquirePermit(SEMAPHORE_UNAVAILABLE_LOG_INTERVAL))
+ return;
+
+ Channel channel = null;
+ try
+ {
+ channel = getOrCreateChannel();
+ if (!channel.attr(TRANSFERRING_FILE_ATTR).compareAndSet(false, true))
+ throw new IllegalStateException("channel's transferring state is currently set to true. refusing to start new stream");
+
+ // close the DataOutputStreamPlus as we're done with it - but don't close the channel
+ try (DataOutputStreamPlus outPlus = new AsyncStreamingOutputPlus(channel))
+ {
+ StreamMessage.serialize(msg, outPlus, streamingVersion, session);
+ }
+ finally
+ {
+ channel.attr(TRANSFERRING_FILE_ATTR).set(Boolean.FALSE);
+ }
+ }
+ catch (Exception e)
+ {
+ session.onError(e);
+ }
+ catch (Throwable t)
+ {
+ if (closed && Throwables.getRootCause(t) instanceof ClosedByInterruptException && fileTransferExecutor.isShutdown())
+ {
+ logger.debug("{} Streaming channel was closed due to the executor pool being shutdown", createLogTag(session, channel));
+ }
+ else
+ {
+ JVMStabilityInspector.inspectThrowable(t);
+ if (!session.state().isFinalState())
+ session.onError(t);
+ }
+ }
+ finally
+ {
+ fileTransferSemaphore.release();
+ }
+ }
+
+ boolean acquirePermit(int logInterval)
+ {
+ long logIntervalNanos = TimeUnit.MINUTES.toNanos(logInterval);
+ long timeOfLastLogging = System.nanoTime();
+ while (true)
+ {
+ if (closed)
+ return false;
+ try
+ {
+ if (fileTransferSemaphore.tryAcquire(1, TimeUnit.SECONDS))
+ return true;
+
+ // log a helpful message to operators in case they are wondering why a given session might not be making progress.
+ long now = System.nanoTime();
+ if (now - timeOfLastLogging > logIntervalNanos)
+ {
+ timeOfLastLogging = now;
+ OutgoingStreamMessage ofm = (OutgoingStreamMessage)msg;
+
+ if (logger.isInfoEnabled())
+ logger.info("{} waiting to acquire a permit to begin streaming {}. This message logs every {} minutes",
+ createLogTag(session, null), ofm.getName(), logInterval);
+ }
+ }
+ catch (InterruptedException ie)
+ {
+ //ignore
+ }
+ }
+ }
+
+ private Channel getOrCreateChannel()
+ {
+ Thread currentThread = Thread.currentThread();
+ try
+ {
+ Channel channel = threadToChannelMap.get(currentThread);
+ if (channel != null)
+ return channel;
+
+ channel = createChannel(false);
+ threadToChannelMap.put(currentThread, channel);
+ return channel;
+ }
+ catch (Exception e)
+ {
+ throw new IOError(e);
+ }
+ }
+
+ private void onError(Throwable t)
+ {
+ try
+ {
+ session.onError(t).get(DEFAULT_CLOSE_WAIT_IN_MILLIS, TimeUnit.MILLISECONDS);
+ }
+ catch (Exception e)
+ {
+ // nop - let the Throwable param be the main failure point here, and let session handle it
+ }
+ }
+
+ /**
+ * For testing purposes
+ */
+ void injectChannel(Channel channel)
+ {
+ Thread currentThread = Thread.currentThread();
+ if (threadToChannelMap.get(currentThread) != null)
+ throw new IllegalStateException("previous channel already set");
+
+ threadToChannelMap.put(currentThread, channel);
+ }
+
+ /**
+ * For testing purposes
+ */
+ void unsetChannel()
+ {
+ threadToChannelMap.remove(Thread.currentThread());
+ }
+ }
+
+ /**
+ * Periodically sends the {@link KeepAliveMessage}.
+ *
+ * NOTE: this task, and the callback function {@link #keepAliveListener(Future)} is executed in the netty event loop.
+ */
+ class KeepAliveTask implements Runnable
+ {
+ private final Channel channel;
+ private final StreamSession session;
+
+ /**
+ * A reference to the scheduled task for this instance so that it may be cancelled.
+ */
+ ScheduledFuture<?> future;
+
+ KeepAliveTask(Channel channel, StreamSession session)
+ {
+ this.channel = channel;
+ this.session = session;
+ }
+
+ public void run()
+ {
+ // if the channel has been closed, cancel the scheduled task and return
+ if (!channel.isOpen() || closed)
+ {
+ future.cancel(false);
+ return;
+ }
+
+ // if the channel is currently processing streaming, skip this execution. As this task executes
+ // on the event loop, even if there is a race with a FileStreamTask which changes the channel attribute
+ // after we check it, the FileStreamTask cannot send out any bytes as this KeepAliveTask is executing
+ // on the event loop (and FileStreamTask publishes it's buffer to the channel, consumed after we're done here).
+ if (channel.attr(TRANSFERRING_FILE_ATTR).get())
+ return;
+
+ try
+ {
+ if (logger.isTraceEnabled())
+ logger.trace("{} Sending keep-alive to {}.", createLogTag(session, channel), session.peer);
+ sendControlMessage(channel, new KeepAliveMessage(), this::keepAliveListener);
+ }
+ catch (IOException ioe)
+ {
+ future.cancel(false);
+ }
+ }
+
+ private void keepAliveListener(Future<? super Void> future)
+ {
+ if (future.isSuccess() || future.isCancelled())
+ return;
+
+ if (logger.isDebugEnabled())
+ logger.debug("{} Could not send keep-alive message (perhaps stream session is finished?).",
+ createLogTag(session, channel), future.cause());
+ }
+ }
+
+ /**
+ * For testing purposes only.
+ */
+ public void setClosed()
+ {
+ closed = true;
+ }
+
+ void setControlMessageChannel(Channel channel)
+ {
+ controlMessageChannel = channel;
+ }
+
+ int semaphoreAvailablePermits()
+ {
+ return fileTransferSemaphore.availablePermits();
+ }
+
+ @Override
+ public boolean connected()
+ {
+ return !closed && (controlMessageChannel == null || controlMessageChannel.isOpen());
+ }
+
+ @Override
+ public void close()
+ {
+ if (closed)
+ return;
+
+ closed = true;
+ if (logger.isDebugEnabled())
+ logger.debug("{} Closing stream connection channels on {}", createLogTag(session, null), template.to);
+ for (ScheduledFuture<?> future : channelKeepAlives)
+ future.cancel(false);
+ channelKeepAlives.clear();
+
+ threadToChannelMap.clear();
+ fileTransferExecutor.shutdownNow();
+ }
+}
diff --git a/src/java/org/apache/cassandra/streaming/async/StreamCompressionSerializer.java b/src/java/org/apache/cassandra/streaming/async/StreamCompressionSerializer.java
new file mode 100644
index 0000000..fc1bde2
--- /dev/null
+++ b/src/java/org/apache/cassandra/streaming/async/StreamCompressionSerializer.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.streaming.async;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.ReadableByteChannel;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.ByteBufAllocator;
+import net.jpountz.lz4.LZ4Compressor;
+import net.jpountz.lz4.LZ4SafeDecompressor;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.net.AsyncStreamingOutputPlus;
+
+import static org.apache.cassandra.net.MessagingService.current_version;
+
+/**
+ * A serialiazer for stream compressed files (see package-level documentation). Much like a typical compressed
+ * output stream, this class operates on buffers or chunks of the data at a a time. The format for each compressed
+ * chunk is as follows:
+ *
+ * - int - compressed payload length
+ * - int - uncompressed payload length
+ * - bytes - compressed payload
+ */
+public class StreamCompressionSerializer
+{
+ private final ByteBufAllocator allocator;
+
+ public StreamCompressionSerializer(ByteBufAllocator allocator)
+ {
+ this.allocator = allocator;
+ }
+
+ /**
+ * Length of heaer data, which includes compressed length, uncompressed length.
+ */
+ private static final int HEADER_LENGTH = 8;
+
+ public static AsyncStreamingOutputPlus.Write serialize(LZ4Compressor compressor, ByteBuffer in, int version)
+ {
+ assert version == current_version;
+ return bufferSupplier -> {
+ int uncompressedLength = in.remaining();
+ int maxLength = compressor.maxCompressedLength(uncompressedLength);
+ ByteBuffer out = bufferSupplier.get(maxLength);
+ out.position(HEADER_LENGTH);
+ compressor.compress(in, out);
+ int compressedLength = out.position() - HEADER_LENGTH;
+ out.putInt(0, compressedLength);
+ out.putInt(4, uncompressedLength);
+ out.flip();
+ };
+ }
+
+ /**
+ * @return A buffer with decompressed data.
+ */
+ public ByteBuf deserialize(LZ4SafeDecompressor decompressor, DataInputPlus in, int version) throws IOException
+ {
+ final int compressedLength = in.readInt();
+ final int uncompressedLength = in.readInt();
+
+ // there's no guarantee the next compressed block is contained within one buffer in the input,
+ // so hence we need a 'staging' buffer to get all the bytes into one contiguous buffer for the decompressor
+ ByteBuf compressed = null;
+ ByteBuf uncompressed = null;
+ try
+ {
+ final ByteBuffer compressedNioBuffer;
+
+ // ReadableByteChannel allows us to keep the bytes off-heap because we pass a ByteBuffer to RBC.read(BB),
+ // DataInputPlus.read() takes a byte array (thus, an on-heap array).
+ if (in instanceof ReadableByteChannel)
+ {
+ compressed = allocator.directBuffer(compressedLength);
+ compressedNioBuffer = compressed.nioBuffer(0, compressedLength);
+ int readLength = ((ReadableByteChannel) in).read(compressedNioBuffer);
+ assert readLength == compressedNioBuffer.position();
+ compressedNioBuffer.flip();
+ }
+ else
+ {
+ byte[] compressedBytes = new byte[compressedLength];
+ in.readFully(compressedBytes);
+ compressedNioBuffer = ByteBuffer.wrap(compressedBytes);
+ }
+
+ uncompressed = allocator.directBuffer(uncompressedLength);
+ ByteBuffer uncompressedNioBuffer = uncompressed.nioBuffer(0, uncompressedLength);
+ decompressor.decompress(compressedNioBuffer, uncompressedNioBuffer);
+ uncompressed.writerIndex(uncompressedLength);
+ return uncompressed;
+ }
+ catch (Exception e)
+ {
+ if (uncompressed != null)
+ uncompressed.release();
+
+ if (e instanceof IOException)
+ throw e;
+ throw new IOException(e);
+ }
+ finally
+ {
+ if (compressed != null)
+ compressed.release();
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/streaming/async/StreamingInboundHandler.java b/src/java/org/apache/cassandra/streaming/async/StreamingInboundHandler.java
new file mode 100644
index 0000000..3b9c172
--- /dev/null
+++ b/src/java/org/apache/cassandra/streaming/async/StreamingInboundHandler.java
@@ -0,0 +1,256 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.streaming.async;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.TimeUnit;
+import javax.annotation.Nullable;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.util.concurrent.Uninterruptibles;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.channel.Channel;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.channel.ChannelInboundHandlerAdapter;
+import io.netty.util.ReferenceCountUtil;
+import io.netty.util.concurrent.FastThreadLocalThread;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.net.AsyncStreamingInputPlus;
+import org.apache.cassandra.streaming.StreamReceiveException;
+import org.apache.cassandra.streaming.StreamSession;
+import org.apache.cassandra.streaming.messages.KeepAliveMessage;
+import org.apache.cassandra.streaming.messages.StreamInitMessage;
+import org.apache.cassandra.streaming.messages.StreamMessage;
+import org.apache.cassandra.utils.JVMStabilityInspector;
+
+import static org.apache.cassandra.streaming.async.NettyStreamingMessageSender.createLogTag;
+
+/**
+ * Handles the inbound side of streaming messages and stream data. From the incoming data, we derserialize the message
+ * including the actual stream data itself. Because the reading and deserialization of streams is a blocking affair,
+ * we can't block the netty event loop. Thus we have a background thread perform all the blocking deserialization.
+ */
+public class StreamingInboundHandler extends ChannelInboundHandlerAdapter
+{
+ private static final Logger logger = LoggerFactory.getLogger(StreamingInboundHandler.class);
+ private static volatile boolean trackInboundHandlers = false;
+ private static Collection<StreamingInboundHandler> inboundHandlers;
+ private final InetAddressAndPort remoteAddress;
+ private final int protocolVersion;
+
+ private final StreamSession session;
+
+ /**
+ * A collection of {@link ByteBuf}s that are yet to be processed. Incoming buffers are first dropped into this
+ * structure, and then consumed.
+ * <p>
+ * For thread safety, this structure's resources are released on the consuming thread
+ * (via {@link AsyncStreamingInputPlus#close()},
+ * but the producing side calls {@link AsyncStreamingInputPlus#requestClosure()} to notify the input that is should close.
+ */
+ private AsyncStreamingInputPlus buffers;
+
+ private volatile boolean closed;
+
+ public StreamingInboundHandler(InetAddressAndPort remoteAddress, int protocolVersion, @Nullable StreamSession session)
+ {
+ this.remoteAddress = remoteAddress;
+ this.protocolVersion = protocolVersion;
+ this.session = session;
+ if (trackInboundHandlers)
+ inboundHandlers.add(this);
+ }
+
+ @Override
+ @SuppressWarnings("resource")
+ public void handlerAdded(ChannelHandlerContext ctx)
+ {
+ buffers = new AsyncStreamingInputPlus(ctx.channel());
+ Thread blockingIOThread = new FastThreadLocalThread(new StreamDeserializingTask(session, ctx.channel()),
+ String.format("Stream-Deserializer-%s-%s", remoteAddress.toString(), ctx.channel().id()));
+ blockingIOThread.setDaemon(true);
+ blockingIOThread.start();
+ }
+
+ @Override
+ public void channelRead(ChannelHandlerContext ctx, Object message)
+ {
+ if (closed || !(message instanceof ByteBuf) || !buffers.append((ByteBuf) message))
+ ReferenceCountUtil.release(message);
+ }
+
+ @Override
+ public void channelInactive(ChannelHandlerContext ctx)
+ {
+ close();
+ ctx.fireChannelInactive();
+ }
+
+ void close()
+ {
+ closed = true;
+ buffers.requestClosure();
+ if (trackInboundHandlers)
+ inboundHandlers.remove(this);
+ }
+
+ @Override
+ public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause)
+ {
+ if (cause instanceof IOException)
+ logger.trace("connection problem while streaming", cause);
+ else
+ logger.warn("exception occurred while in processing streaming data", cause);
+ close();
+ }
+
+ /**
+ * For testing only!!
+ */
+ void setPendingBuffers(AsyncStreamingInputPlus bufChannel)
+ {
+ this.buffers = bufChannel;
+ }
+
+ /**
+ * The task that performs the actual deserialization.
+ */
+ class StreamDeserializingTask implements Runnable
+ {
+ private final Channel channel;
+
+ @VisibleForTesting
+ StreamSession session;
+
+ StreamDeserializingTask(StreamSession session, Channel channel)
+ {
+ this.session = session;
+ this.channel = channel;
+ }
+
+ @Override
+ public void run()
+ {
+ try
+ {
+ while (true)
+ {
+ buffers.maybeIssueRead();
+
+ // do a check of available bytes and possibly sleep some amount of time (then continue).
+ // this way we can break out of run() sanely or we end up blocking indefintely in StreamMessage.deserialize()
+ while (buffers.isEmpty())
+ {
+ if (closed)
+ return;
+
+ Uninterruptibles.sleepUninterruptibly(400, TimeUnit.MILLISECONDS);
+ }
+
+ StreamMessage message = StreamMessage.deserialize(buffers, protocolVersion);
+
+ // keep-alives don't necessarily need to be tied to a session (they could be arrive before or after
+ // wrt session lifecycle, due to races), just log that we received the message and carry on
+ if (message instanceof KeepAliveMessage)
+ {
+ if (logger.isDebugEnabled())
+ logger.debug("{} Received {}", createLogTag(session, channel), message);
+ continue;
+ }
+
+ if (session == null)
+ session = deriveSession(message);
+
+ if (logger.isDebugEnabled())
+ logger.debug("{} Received {}", createLogTag(session, channel), message);
+
+ session.messageReceived(message);
+ }
+ }
+ catch (Throwable t)
+ {
+ JVMStabilityInspector.inspectThrowable(t);
+ if (session != null)
+ {
+ session.onError(t);
+ }
+ else if (t instanceof StreamReceiveException)
+ {
+ ((StreamReceiveException)t).session.onError(t);
+ }
+ else
+ {
+ logger.error("{} stream operation from {} failed", createLogTag(session, channel), remoteAddress, t);
+ }
+ }
+ finally
+ {
+ channel.close();
+ closed = true;
+
+ if (buffers != null)
+ {
+ // request closure again as the original request could have raced with receiving a
+ // message and been consumed in the message receive loop above. Otherweise
+ // buffers could hang indefinitely on the queue.poll.
+ buffers.requestClosure();
+ buffers.close();
+ }
+ }
+ }
+
+ StreamSession deriveSession(StreamMessage message)
+ {
+ // StreamInitMessage starts a new channel here, but IncomingStreamMessage needs a session
+ // to be established a priori
+ StreamSession streamSession = message.getOrCreateSession(channel);
+
+ // Attach this channel to the session: this only happens upon receiving the first init message as a follower;
+ // in all other cases, no new control channel will be added, as the proper control channel will be already attached.
+ streamSession.attachInbound(channel, message instanceof StreamInitMessage);
+ return streamSession;
+ }
+ }
+
+ /** Shutdown for in-JVM tests. For any other usage, tracking of active inbound streaming handlers
+ * should be revisted first and in-JVM shutdown refactored with it.
+ * This does not prevent new inbound handlers being added after shutdown, nor is not thread-safe
+ * around new inbound handlers being opened during shutdown.
+ */
+ @VisibleForTesting
+ public static void shutdown()
+ {
+ assert trackInboundHandlers : "in-JVM tests required tracking of inbound streaming handlers";
+
+ inboundHandlers.forEach(StreamingInboundHandler::close);
+ inboundHandlers.clear();
+ }
+
+ public static void trackInboundHandlers()
+ {
+ inboundHandlers = Collections.newSetFromMap(new ConcurrentHashMap<>());
+ trackInboundHandlers = true;
+ }
+}
diff --git a/src/java/org/apache/cassandra/streaming/async/package-info.java b/src/java/org/apache/cassandra/streaming/async/package-info.java
new file mode 100644
index 0000000..9455c7c
--- /dev/null
+++ b/src/java/org/apache/cassandra/streaming/async/package-info.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * <h1>Non-blocking streaming with netty</h1>
+ * This document describes the implementation details of the streaming protocol. A listener for a streaming
+ * session listens on the same socket as internode messaging, and participates in the same handshake protocol
+ * That protocol is described in the package-level documentation for {@link org.apache.cassandra.net.async}, and
+ * thus not here.
+ *
+ * Streaming 2.0 was implemented as CASSANDRA-5286. Streaming 2.0 used (the equivalent of) a single thread and
+ * a single socket to transfer sstables sequentially to a peer (either as part of a repair, bootstrap, and so on).
+ * Part of the motivation for switching to netty and a non-blocking model as to enable stream transfers to occur
+ * in parallel for a given session.
+ *
+ * Thus, a more detailed approach is required for stream session management.
+ *
+ * <h2>Session setup and management</h2>
+ *
+ * The full details of the session lifecycle are documented in {@link org.apache.cassandra.streaming.StreamSession}.
+ *
+ */
+package org.apache.cassandra.streaming.async;
+
diff --git a/src/java/org/apache/cassandra/streaming/compress/CompressedInputStream.java b/src/java/org/apache/cassandra/streaming/compress/CompressedInputStream.java
deleted file mode 100644
index 8a32d7a..0000000
--- a/src/java/org/apache/cassandra/streaming/compress/CompressedInputStream.java
+++ /dev/null
@@ -1,235 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.streaming.compress;
-
-import java.io.EOFException;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.Iterator;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.ThreadLocalRandom;
-import java.util.function.Supplier;
-
-import com.google.common.collect.Iterators;
-import com.google.common.primitives.Ints;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import io.netty.util.concurrent.FastThreadLocalThread;
-import org.apache.cassandra.io.compress.CompressionMetadata;
-import org.apache.cassandra.utils.ChecksumType;
-import org.apache.cassandra.utils.WrappedRunnable;
-
-/**
- * InputStream which reads data from underlining source with given {@link CompressionInfo}.
- */
-public class CompressedInputStream extends InputStream
-{
-
- private static final Logger logger = LoggerFactory.getLogger(CompressedInputStream.class);
-
- private final CompressionInfo info;
- // chunk buffer
- private final BlockingQueue<byte[]> dataBuffer;
- private final Supplier<Double> crcCheckChanceSupplier;
-
- // uncompressed bytes
- private final byte[] buffer;
-
- // offset from the beginning of the buffer
- protected long bufferOffset = 0;
- // current position in stream
- private long current = 0;
- // number of bytes in the buffer that are actually valid
- protected int validBufferBytes = -1;
-
- private final ChecksumType checksumType;
-
- // raw checksum bytes
- private final byte[] checksumBytes = new byte[4];
-
- /**
- * Indicates there was a problem when reading from source stream.
- * When this is added to the <code>dataBuffer</code> by the stream Reader,
- * it is expected that the <code>readException</code> variable is populated
- * with the cause of the error when reading from source stream, so it is
- * thrown to the consumer on subsequent read operation.
- */
- private static final byte[] POISON_PILL = new byte[0];
-
- protected volatile IOException readException = null;
-
- private long totalCompressedBytesRead;
-
- /**
- * @param source Input source to read compressed data from
- * @param info Compression info
- */
- public CompressedInputStream(InputStream source, CompressionInfo info, ChecksumType checksumType, Supplier<Double> crcCheckChanceSupplier)
- {
- this.info = info;
- this.buffer = new byte[info.parameters.chunkLength()];
- // buffer is limited to store up to 1024 chunks
- this.dataBuffer = new ArrayBlockingQueue<>(Math.min(info.chunks.length, 1024));
- this.crcCheckChanceSupplier = crcCheckChanceSupplier;
- this.checksumType = checksumType;
-
- new FastThreadLocalThread(new Reader(source, info, dataBuffer)).start();
- }
-
- private void decompressNextChunk() throws IOException
- {
- if (readException != null)
- throw readException;
-
- try
- {
- byte[] compressedWithCRC = dataBuffer.take();
- if (compressedWithCRC == POISON_PILL)
- {
- assert readException != null;
- throw readException;
- }
- decompress(compressedWithCRC);
- }
- catch (InterruptedException e)
- {
- throw new EOFException("No chunk available");
- }
- }
-
- @Override
- public int read() throws IOException
- {
- if (current >= bufferOffset + buffer.length || validBufferBytes == -1)
- decompressNextChunk();
-
- assert current >= bufferOffset && current < bufferOffset + validBufferBytes;
-
- return ((int) buffer[(int) (current++ - bufferOffset)]) & 0xff;
- }
-
- @Override
- public int read(byte[] b, int off, int len) throws IOException
- {
- long nextCurrent = current + len;
-
- if (current >= bufferOffset + buffer.length || validBufferBytes == -1)
- decompressNextChunk();
-
- assert nextCurrent >= bufferOffset;
-
- int read = 0;
- while (read < len)
- {
- int nextLen = Math.min((len - read), (int)((bufferOffset + validBufferBytes) - current));
-
- System.arraycopy(buffer, (int)(current - bufferOffset), b, off + read, nextLen);
- read += nextLen;
-
- current += nextLen;
- if (read != len)
- decompressNextChunk();
- }
-
- return len;
- }
-
- public void position(long position)
- {
- assert position >= current : "stream can only read forward.";
- current = position;
- }
-
- private void decompress(byte[] compressed) throws IOException
- {
- // uncompress
- validBufferBytes = info.parameters.getSstableCompressor().uncompress(compressed, 0, compressed.length - checksumBytes.length, buffer, 0);
- totalCompressedBytesRead += compressed.length;
-
- // validate crc randomly
- if (this.crcCheckChanceSupplier.get() >= 1d ||
- this.crcCheckChanceSupplier.get() > ThreadLocalRandom.current().nextDouble())
- {
- int checksum = (int) checksumType.of(compressed, 0, compressed.length - checksumBytes.length);
-
- System.arraycopy(compressed, compressed.length - checksumBytes.length, checksumBytes, 0, checksumBytes.length);
- if (Ints.fromByteArray(checksumBytes) != checksum)
- throw new IOException("CRC unmatched");
- }
-
- // buffer offset is always aligned
- bufferOffset = current & ~(buffer.length - 1);
- }
-
- public long getTotalCompressedBytesRead()
- {
- return totalCompressedBytesRead;
- }
-
- class Reader extends WrappedRunnable
- {
- private final InputStream source;
- private final Iterator<CompressionMetadata.Chunk> chunks;
- private final BlockingQueue<byte[]> dataBuffer;
-
- Reader(InputStream source, CompressionInfo info, BlockingQueue<byte[]> dataBuffer)
- {
- this.source = source;
- this.chunks = Iterators.forArray(info.chunks);
- this.dataBuffer = dataBuffer;
- }
-
- protected void runMayThrow() throws Exception
- {
- byte[] compressedWithCRC;
- while (chunks.hasNext())
- {
- CompressionMetadata.Chunk chunk = chunks.next();
-
- int readLength = chunk.length + 4; // read with CRC
- compressedWithCRC = new byte[readLength];
-
- int bufferRead = 0;
- while (bufferRead < readLength)
- {
- try
- {
- int r = source.read(compressedWithCRC, bufferRead, readLength - bufferRead);
- if (r < 0)
- {
- readException = new EOFException("No chunk available");
- dataBuffer.put(POISON_PILL);
- return; // throw exception where we consume dataBuffer
- }
- bufferRead += r;
- }
- catch (IOException e)
- {
- logger.warn("Error while reading compressed input stream.", e);
- readException = e;
- dataBuffer.put(POISON_PILL);
- return; // throw exception where we consume dataBuffer
- }
- }
- dataBuffer.put(compressedWithCRC);
- }
- }
- }
-}
diff --git a/src/java/org/apache/cassandra/streaming/compress/CompressedStreamReader.java b/src/java/org/apache/cassandra/streaming/compress/CompressedStreamReader.java
deleted file mode 100644
index 70b5765..0000000
--- a/src/java/org/apache/cassandra/streaming/compress/CompressedStreamReader.java
+++ /dev/null
@@ -1,145 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.streaming.compress;
-
-import java.io.IOException;
-import java.nio.channels.Channels;
-import java.nio.channels.ReadableByteChannel;
-
-import com.google.common.base.Throwables;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.cassandra.config.Schema;
-import org.apache.cassandra.db.ColumnFamilyStore;
-import org.apache.cassandra.db.Keyspace;
-import org.apache.cassandra.io.compress.CompressionMetadata;
-import org.apache.cassandra.io.sstable.SSTableMultiWriter;
-import org.apache.cassandra.io.util.TrackedInputStream;
-import org.apache.cassandra.streaming.ProgressInfo;
-import org.apache.cassandra.streaming.StreamReader;
-import org.apache.cassandra.streaming.StreamSession;
-import org.apache.cassandra.streaming.messages.FileMessageHeader;
-import org.apache.cassandra.utils.FBUtilities;
-import org.apache.cassandra.utils.Pair;
-
-import static org.apache.cassandra.utils.Throwables.extractIOExceptionCause;
-
-/**
- * StreamReader that reads from streamed compressed SSTable
- */
-public class CompressedStreamReader extends StreamReader
-{
- private static final Logger logger = LoggerFactory.getLogger(CompressedStreamReader.class);
-
- protected final CompressionInfo compressionInfo;
-
- public CompressedStreamReader(FileMessageHeader header, StreamSession session)
- {
- super(header, session);
- this.compressionInfo = header.compressionInfo;
- }
-
- /**
- * @return SSTable transferred
- * @throws java.io.IOException if reading the remote sstable fails. Will throw an RTE if local write fails.
- */
- @Override
- @SuppressWarnings("resource") // channel needs to remain open, streams on top of it can't be closed
- public SSTableMultiWriter read(ReadableByteChannel channel) throws IOException
- {
- long totalSize = totalSize();
-
- Pair<String, String> kscf = Schema.instance.getCF(cfId);
- ColumnFamilyStore cfs = null;
- if (kscf != null)
- cfs = Keyspace.open(kscf.left).getColumnFamilyStore(kscf.right);
-
- if (kscf == null || cfs == null)
- {
- // schema was dropped during streaming
- throw new IOException("CF " + cfId + " was dropped during streaming");
- }
-
- logger.debug("[Stream #{}] Start receiving file #{} from {}, repairedAt = {}, size = {}, ks = '{}', table = '{}'.",
- session.planId(), fileSeqNum, session.peer, repairedAt, totalSize, cfs.keyspace.getName(),
- cfs.getColumnFamilyName());
-
- CompressedInputStream cis = new CompressedInputStream(Channels.newInputStream(channel), compressionInfo,
- inputVersion.compressedChecksumType(), cfs::getCrcCheckChance);
- TrackedInputStream in = new TrackedInputStream(cis);
-
- StreamDeserializer deserializer = new StreamDeserializer(cfs.metadata, in, inputVersion, getHeader(cfs.metadata),
- totalSize, session.planId());
- SSTableMultiWriter writer = null;
- try
- {
- writer = createWriter(cfs, totalSize, repairedAt, format);
- String filename = writer.getFilename();
- int sectionIdx = 0;
- for (Pair<Long, Long> section : sections)
- {
- assert cis.getTotalCompressedBytesRead() <= totalSize;
- long sectionLength = section.right - section.left;
-
- logger.trace("[Stream #{}] Reading section {} with length {} from stream.", session.planId(), sectionIdx++, sectionLength);
- // skip to beginning of section inside chunk
- cis.position(section.left);
- in.reset(0);
-
- while (in.getBytesRead() < sectionLength)
- {
- writePartition(deserializer, writer);
- // when compressed, report total bytes of compressed chunks read since remoteFile.size is the sum of chunks transferred
- session.progress(filename, ProgressInfo.Direction.IN, cis.getTotalCompressedBytesRead(), totalSize);
- }
- }
- logger.debug("[Stream #{}] Finished receiving file #{} from {} readBytes = {}, totalSize = {}", session.planId(), fileSeqNum,
- session.peer, FBUtilities.prettyPrintMemory(cis.getTotalCompressedBytesRead()), FBUtilities.prettyPrintMemory(totalSize));
- return writer;
- }
- catch (Throwable e)
- {
- if (deserializer != null)
- logger.warn("[Stream {}] Error while reading partition {} from stream on ks='{}' and table='{}'.",
- session.planId(), deserializer.partitionKey(), cfs.keyspace.getName(), cfs.getTableName());
- if (writer != null)
- {
- writer.abort(e);
- }
- if (extractIOExceptionCause(e).isPresent())
- throw e;
- throw Throwables.propagate(e);
- }
- finally
- {
- if (deserializer != null)
- deserializer.cleanup();
- }
- }
-
- @Override
- protected long totalSize()
- {
- long size = 0;
- // calculate total length of transferring chunks
- for (CompressionMetadata.Chunk chunk : compressionInfo.chunks)
- size += chunk.length + 4; // 4 bytes for CRC
- return size;
- }
-}
diff --git a/src/java/org/apache/cassandra/streaming/compress/CompressedStreamWriter.java b/src/java/org/apache/cassandra/streaming/compress/CompressedStreamWriter.java
deleted file mode 100644
index 185ab22..0000000
--- a/src/java/org/apache/cassandra/streaming/compress/CompressedStreamWriter.java
+++ /dev/null
@@ -1,135 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.streaming.compress;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.cassandra.io.compress.CompressionMetadata;
-import org.apache.cassandra.io.sstable.Component;
-import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.io.util.ChannelProxy;
-import org.apache.cassandra.io.util.DataOutputStreamPlus;
-import org.apache.cassandra.streaming.ProgressInfo;
-import org.apache.cassandra.streaming.StreamSession;
-import org.apache.cassandra.streaming.StreamWriter;
-import org.apache.cassandra.utils.FBUtilities;
-import org.apache.cassandra.utils.Pair;
-
-/**
- * StreamWriter for compressed SSTable.
- */
-public class CompressedStreamWriter extends StreamWriter
-{
- public static final int CHUNK_SIZE = 10 * 1024 * 1024;
-
- private static final Logger logger = LoggerFactory.getLogger(CompressedStreamWriter.class);
-
- private final CompressionInfo compressionInfo;
-
- public CompressedStreamWriter(SSTableReader sstable, Collection<Pair<Long, Long>> sections, CompressionInfo compressionInfo, StreamSession session)
- {
- super(sstable, sections, session);
- this.compressionInfo = compressionInfo;
- }
-
- @Override
- public void write(DataOutputStreamPlus out) throws IOException
- {
- long totalSize = totalSize();
- logger.debug("[Stream #{}] Start streaming file {} to {}, repairedAt = {}, totalSize = {}", session.planId(),
- sstable.getFilename(), session.peer, sstable.getSSTableMetadata().repairedAt, totalSize);
- try (ChannelProxy fc = sstable.getDataChannel().sharedCopy())
- {
- long progress = 0L;
- // calculate chunks to transfer. we want to send continuous chunks altogether.
- List<Pair<Long, Long>> sections = getTransferSections(compressionInfo.chunks);
-
- int sectionIdx = 0;
-
- // stream each of the required sections of the file
- for (final Pair<Long, Long> section : sections)
- {
- // length of the section to stream
- long length = section.right - section.left;
-
- logger.trace("[Stream #{}] Writing section {} with length {} to stream.", session.planId(), sectionIdx++, length);
-
- // tracks write progress
- long bytesTransferred = 0;
- while (bytesTransferred < length)
- {
- final long bytesTransferredFinal = bytesTransferred;
- final int toTransfer = (int) Math.min(CHUNK_SIZE, length - bytesTransferred);
- limiter.acquire(toTransfer);
- long lastWrite = out.applyToChannel((wbc) -> fc.transferTo(section.left + bytesTransferredFinal, toTransfer, wbc));
- bytesTransferred += lastWrite;
- progress += lastWrite;
- session.progress(sstable.descriptor.filenameFor(Component.DATA), ProgressInfo.Direction.OUT, progress, totalSize);
- }
- }
- logger.debug("[Stream #{}] Finished streaming file {} to {}, bytesTransferred = {}, totalSize = {}",
- session.planId(), sstable.getFilename(), session.peer, FBUtilities.prettyPrintMemory(progress), FBUtilities.prettyPrintMemory(totalSize));
- }
- }
-
- @Override
- protected long totalSize()
- {
- long size = 0;
- // calculate total length of transferring chunks
- for (CompressionMetadata.Chunk chunk : compressionInfo.chunks)
- size += chunk.length + 4; // 4 bytes for CRC
- return size;
- }
-
- // chunks are assumed to be sorted by offset
- private List<Pair<Long, Long>> getTransferSections(CompressionMetadata.Chunk[] chunks)
- {
- List<Pair<Long, Long>> transferSections = new ArrayList<>();
- Pair<Long, Long> lastSection = null;
- for (CompressionMetadata.Chunk chunk : chunks)
- {
- if (lastSection != null)
- {
- if (chunk.offset == lastSection.right)
- {
- // extend previous section to end of this chunk
- lastSection = Pair.create(lastSection.left, chunk.offset + chunk.length + 4); // 4 bytes for CRC
- }
- else
- {
- transferSections.add(lastSection);
- lastSection = Pair.create(chunk.offset, chunk.offset + chunk.length + 4);
- }
- }
- else
- {
- lastSection = Pair.create(chunk.offset, chunk.offset + chunk.length + 4);
- }
- }
- if (lastSection != null)
- transferSections.add(lastSection);
- return transferSections;
- }
-}
diff --git a/src/java/org/apache/cassandra/streaming/compress/CompressionInfo.java b/src/java/org/apache/cassandra/streaming/compress/CompressionInfo.java
deleted file mode 100644
index bd0c2d5..0000000
--- a/src/java/org/apache/cassandra/streaming/compress/CompressionInfo.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.streaming.compress;
-
-import java.io.IOException;
-
-import org.apache.cassandra.db.TypeSizes;
-import org.apache.cassandra.io.IVersionedSerializer;
-import org.apache.cassandra.io.compress.CompressionMetadata;
-import org.apache.cassandra.schema.CompressionParams;
-import org.apache.cassandra.io.util.DataInputPlus;
-import org.apache.cassandra.io.util.DataOutputPlus;
-
-/**
- * Container that carries compression parameters and chunks to decompress data from stream.
- */
-public class CompressionInfo
-{
- public static final IVersionedSerializer<CompressionInfo> serializer = new CompressionInfoSerializer();
-
- public final CompressionMetadata.Chunk[] chunks;
- public final CompressionParams parameters;
-
- public CompressionInfo(CompressionMetadata.Chunk[] chunks, CompressionParams parameters)
- {
- assert chunks != null && parameters != null;
- this.chunks = chunks;
- this.parameters = parameters;
- }
-
- static class CompressionInfoSerializer implements IVersionedSerializer<CompressionInfo>
- {
- public void serialize(CompressionInfo info, DataOutputPlus out, int version) throws IOException
- {
- if (info == null)
- {
- out.writeInt(-1);
- return;
- }
-
- int chunkCount = info.chunks.length;
- out.writeInt(chunkCount);
- for (int i = 0; i < chunkCount; i++)
- CompressionMetadata.Chunk.serializer.serialize(info.chunks[i], out, version);
- // compression params
- CompressionParams.serializer.serialize(info.parameters, out, version);
- }
-
- public CompressionInfo deserialize(DataInputPlus in, int version) throws IOException
- {
- // chunks
- int chunkCount = in.readInt();
- if (chunkCount < 0)
- return null;
-
- CompressionMetadata.Chunk[] chunks = new CompressionMetadata.Chunk[chunkCount];
- for (int i = 0; i < chunkCount; i++)
- chunks[i] = CompressionMetadata.Chunk.serializer.deserialize(in, version);
-
- // compression params
- CompressionParams parameters = CompressionParams.serializer.deserialize(in, version);
- return new CompressionInfo(chunks, parameters);
- }
-
- public long serializedSize(CompressionInfo info, int version)
- {
- if (info == null)
- return TypeSizes.sizeof(-1);
-
- // chunks
- int chunkCount = info.chunks.length;
- long size = TypeSizes.sizeof(chunkCount);
- for (int i = 0; i < chunkCount; i++)
- size += CompressionMetadata.Chunk.serializer.serializedSize(info.chunks[i], version);
- // compression params
- size += CompressionParams.serializer.serializedSize(info.parameters, version);
- return size;
- }
- }
-}
diff --git a/src/java/org/apache/cassandra/streaming/compress/StreamCompressionInputStream.java b/src/java/org/apache/cassandra/streaming/compress/StreamCompressionInputStream.java
new file mode 100644
index 0000000..ceed532
--- /dev/null
+++ b/src/java/org/apache/cassandra/streaming/compress/StreamCompressionInputStream.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.streaming.compress;
+
+import java.io.IOException;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.ByteBufAllocator;
+import io.netty.buffer.PooledByteBufAllocator;
+import io.netty.buffer.Unpooled;
+import net.jpountz.lz4.LZ4Factory;
+import net.jpountz.lz4.LZ4SafeDecompressor;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.RebufferingInputStream;
+import org.apache.cassandra.net.AsyncStreamingInputPlus;
+import org.apache.cassandra.streaming.async.StreamCompressionSerializer;
+
+public class StreamCompressionInputStream extends RebufferingInputStream implements AutoCloseable
+{
+ /**
+ * The stream which contains buffers of compressed data that came from the peer.
+ */
+ private final DataInputPlus dataInputPlus;
+
+ private final LZ4SafeDecompressor decompressor;
+ private final int protocolVersion;
+ private final StreamCompressionSerializer deserializer;
+
+ /**
+ * The parent, or owning, buffer of the current buffer being read from ({@link super#buffer}).
+ */
+ private ByteBuf currentBuf;
+
+ public StreamCompressionInputStream(DataInputPlus dataInputPlus, int protocolVersion)
+ {
+ super(Unpooled.EMPTY_BUFFER.nioBuffer());
+ currentBuf = Unpooled.EMPTY_BUFFER;
+
+ this.dataInputPlus = dataInputPlus;
+ this.protocolVersion = protocolVersion;
+ this.decompressor = LZ4Factory.fastestInstance().safeDecompressor();
+
+ ByteBufAllocator allocator = dataInputPlus instanceof AsyncStreamingInputPlus
+ ? ((AsyncStreamingInputPlus)dataInputPlus).getAllocator()
+ : PooledByteBufAllocator.DEFAULT;
+ deserializer = new StreamCompressionSerializer(allocator);
+ }
+
+ @Override
+ public void reBuffer() throws IOException
+ {
+ currentBuf.release();
+ currentBuf = deserializer.deserialize(decompressor, dataInputPlus, protocolVersion);
+ buffer = currentBuf.nioBuffer(0, currentBuf.readableBytes());
+ }
+
+ /**
+ * {@inheritDoc}
+ *
+ * Close resources except {@link #dataInputPlus} as that needs to remain open for other streaming activity.
+ */
+ @Override
+ public void close()
+ {
+ currentBuf.release();
+ }
+}
diff --git a/src/java/org/apache/cassandra/streaming/management/ProgressInfoCompositeData.java b/src/java/org/apache/cassandra/streaming/management/ProgressInfoCompositeData.java
index b9e6951..a1fa19f 100644
--- a/src/java/org/apache/cassandra/streaming/management/ProgressInfoCompositeData.java
+++ b/src/java/org/apache/cassandra/streaming/management/ProgressInfoCompositeData.java
@@ -17,7 +17,6 @@
*/
package org.apache.cassandra.streaming.management;
-import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.HashMap;
import java.util.Map;
@@ -26,12 +25,14 @@
import com.google.common.base.Throwables;
+import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.streaming.ProgressInfo;
public class ProgressInfoCompositeData
{
private static final String[] ITEM_NAMES = new String[]{"planId",
"peer",
+ "peer storage port",
"sessionIndex",
"fileName",
"direction",
@@ -39,14 +40,16 @@
"totalBytes"};
private static final String[] ITEM_DESCS = new String[]{"String representation of Plan ID",
"Session peer",
+ "Session peer storage port",
"Index of session",
- "Name of the file",
+ "Name of the stream",
"Direction('IN' or 'OUT')",
"Current bytes transferred",
"Total bytes to transfer"};
private static final OpenType<?>[] ITEM_TYPES = new OpenType[]{SimpleType.STRING,
SimpleType.STRING,
SimpleType.INTEGER,
+ SimpleType.INTEGER,
SimpleType.STRING,
SimpleType.STRING,
SimpleType.LONG,
@@ -73,12 +76,13 @@
{
Map<String, Object> valueMap = new HashMap<>();
valueMap.put(ITEM_NAMES[0], planId.toString());
- valueMap.put(ITEM_NAMES[1], progressInfo.peer.getHostAddress());
- valueMap.put(ITEM_NAMES[2], progressInfo.sessionIndex);
- valueMap.put(ITEM_NAMES[3], progressInfo.fileName);
- valueMap.put(ITEM_NAMES[4], progressInfo.direction.name());
- valueMap.put(ITEM_NAMES[5], progressInfo.currentBytes);
- valueMap.put(ITEM_NAMES[6], progressInfo.totalBytes);
+ valueMap.put(ITEM_NAMES[1], progressInfo.peer.address.getHostAddress());
+ valueMap.put(ITEM_NAMES[2], progressInfo.peer.port);
+ valueMap.put(ITEM_NAMES[3], progressInfo.sessionIndex);
+ valueMap.put(ITEM_NAMES[4], progressInfo.fileName);
+ valueMap.put(ITEM_NAMES[5], progressInfo.direction.name());
+ valueMap.put(ITEM_NAMES[6], progressInfo.currentBytes);
+ valueMap.put(ITEM_NAMES[7], progressInfo.totalBytes);
try
{
return new CompositeDataSupport(COMPOSITE_TYPE, valueMap);
@@ -94,12 +98,12 @@
Object[] values = cd.getAll(ITEM_NAMES);
try
{
- return new ProgressInfo(InetAddress.getByName((String) values[1]),
- (int) values[2],
- (String) values[3],
- ProgressInfo.Direction.valueOf((String)values[4]),
- (long) values[5],
- (long) values[6]);
+ return new ProgressInfo(InetAddressAndPort.getByNameOverrideDefaults((String) values[1], (Integer)values[2]),
+ (int) values[3],
+ (String) values[4],
+ ProgressInfo.Direction.valueOf((String)values[5]),
+ (long) values[6],
+ (long) values[7]);
}
catch (UnknownHostException e)
{
diff --git a/src/java/org/apache/cassandra/streaming/management/SessionCompleteEventCompositeData.java b/src/java/org/apache/cassandra/streaming/management/SessionCompleteEventCompositeData.java
index 516582a..1c0d8c5 100644
--- a/src/java/org/apache/cassandra/streaming/management/SessionCompleteEventCompositeData.java
+++ b/src/java/org/apache/cassandra/streaming/management/SessionCompleteEventCompositeData.java
@@ -29,12 +29,15 @@
{
private static final String[] ITEM_NAMES = new String[]{"planId",
"peer",
+ "peer storage port",
"success"};
private static final String[] ITEM_DESCS = new String[]{"Plan ID",
"Session peer",
+ "Session peer storage port",
"Indicates whether session was successful"};
private static final OpenType<?>[] ITEM_TYPES = new OpenType[]{SimpleType.STRING,
SimpleType.STRING,
+ SimpleType.INTEGER,
SimpleType.BOOLEAN};
public static final CompositeType COMPOSITE_TYPE;
@@ -58,8 +61,9 @@
{
Map<String, Object> valueMap = new HashMap<>();
valueMap.put(ITEM_NAMES[0], event.planId.toString());
- valueMap.put(ITEM_NAMES[1], event.peer.getHostAddress());
- valueMap.put(ITEM_NAMES[2], event.success);
+ valueMap.put(ITEM_NAMES[1], event.peer.address.getHostAddress());
+ valueMap.put(ITEM_NAMES[2], event.peer.port);
+ valueMap.put(ITEM_NAMES[3], event.success);
try
{
return new CompositeDataSupport(COMPOSITE_TYPE, valueMap);
diff --git a/src/java/org/apache/cassandra/streaming/management/SessionInfoCompositeData.java b/src/java/org/apache/cassandra/streaming/management/SessionInfoCompositeData.java
index a6762a8..d20eaf5 100644
--- a/src/java/org/apache/cassandra/streaming/management/SessionInfoCompositeData.java
+++ b/src/java/org/apache/cassandra/streaming/management/SessionInfoCompositeData.java
@@ -17,7 +17,6 @@
*/
package org.apache.cassandra.streaming.management;
-import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.*;
import javax.management.openmbean.*;
@@ -27,6 +26,7 @@
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
+import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.streaming.ProgressInfo;
import org.apache.cassandra.streaming.SessionInfo;
import org.apache.cassandra.streaming.StreamSession;
@@ -36,7 +36,9 @@
{
private static final String[] ITEM_NAMES = new String[]{"planId",
"peer",
+ "peer_port",
"connecting",
+ "connecting_port",
"receivingSummaries",
"sendingSummaries",
"state",
@@ -45,7 +47,9 @@
"sessionIndex"};
private static final String[] ITEM_DESCS = new String[]{"Plan ID",
"Session peer",
+ "Session peer storage port",
"Connecting address",
+ "Connecting storage port",
"Summaries of receiving data",
"Summaries of sending data",
"Current session state",
@@ -61,7 +65,9 @@
{
ITEM_TYPES = new OpenType[]{SimpleType.STRING,
SimpleType.STRING,
+ SimpleType.INTEGER,
SimpleType.STRING,
+ SimpleType.INTEGER,
ArrayType.getArrayType(StreamSummaryCompositeData.COMPOSITE_TYPE),
ArrayType.getArrayType(StreamSummaryCompositeData.COMPOSITE_TYPE),
SimpleType.STRING,
@@ -84,8 +90,10 @@
{
Map<String, Object> valueMap = new HashMap<>();
valueMap.put(ITEM_NAMES[0], planId.toString());
- valueMap.put(ITEM_NAMES[1], sessionInfo.peer.getHostAddress());
- valueMap.put(ITEM_NAMES[2], sessionInfo.connecting.getHostAddress());
+ valueMap.put(ITEM_NAMES[1], sessionInfo.peer.address.getHostAddress());
+ valueMap.put(ITEM_NAMES[2], sessionInfo.peer.port);
+ valueMap.put(ITEM_NAMES[3], sessionInfo.connecting.address.getHostAddress());
+ valueMap.put(ITEM_NAMES[4], sessionInfo.connecting.port);
Function<StreamSummary, CompositeData> fromStreamSummary = new Function<StreamSummary, CompositeData>()
{
public CompositeData apply(StreamSummary input)
@@ -93,9 +101,9 @@
return StreamSummaryCompositeData.toCompositeData(input);
}
};
- valueMap.put(ITEM_NAMES[3], toArrayOfCompositeData(sessionInfo.receivingSummaries, fromStreamSummary));
- valueMap.put(ITEM_NAMES[4], toArrayOfCompositeData(sessionInfo.sendingSummaries, fromStreamSummary));
- valueMap.put(ITEM_NAMES[5], sessionInfo.state.name());
+ valueMap.put(ITEM_NAMES[5], toArrayOfCompositeData(sessionInfo.receivingSummaries, fromStreamSummary));
+ valueMap.put(ITEM_NAMES[6], toArrayOfCompositeData(sessionInfo.sendingSummaries, fromStreamSummary));
+ valueMap.put(ITEM_NAMES[7], sessionInfo.state.name());
Function<ProgressInfo, CompositeData> fromProgressInfo = new Function<ProgressInfo, CompositeData>()
{
public CompositeData apply(ProgressInfo input)
@@ -103,9 +111,9 @@
return ProgressInfoCompositeData.toCompositeData(planId, input);
}
};
- valueMap.put(ITEM_NAMES[6], toArrayOfCompositeData(sessionInfo.getReceivingFiles(), fromProgressInfo));
- valueMap.put(ITEM_NAMES[7], toArrayOfCompositeData(sessionInfo.getSendingFiles(), fromProgressInfo));
- valueMap.put(ITEM_NAMES[8], sessionInfo.sessionIndex);
+ valueMap.put(ITEM_NAMES[8], toArrayOfCompositeData(sessionInfo.getReceivingFiles(), fromProgressInfo));
+ valueMap.put(ITEM_NAMES[9], toArrayOfCompositeData(sessionInfo.getSendingFiles(), fromProgressInfo));
+ valueMap.put(ITEM_NAMES[10], sessionInfo.sessionIndex);
try
{
return new CompositeDataSupport(COMPOSITE_TYPE, valueMap);
@@ -121,11 +129,11 @@
assert cd.getCompositeType().equals(COMPOSITE_TYPE);
Object[] values = cd.getAll(ITEM_NAMES);
- InetAddress peer, connecting;
+ InetAddressAndPort peer, connecting;
try
{
- peer = InetAddress.getByName((String) values[1]);
- connecting = InetAddress.getByName((String) values[2]);
+ peer = InetAddressAndPort.getByNameOverrideDefaults((String) values[1], (Integer)values[2]);
+ connecting = InetAddressAndPort.getByNameOverrideDefaults((String) values[3], (Integer)values[4]);
}
catch (UnknownHostException e)
{
@@ -139,11 +147,11 @@
}
};
SessionInfo info = new SessionInfo(peer,
- (int)values[8],
+ (int)values[10],
connecting,
- fromArrayOfCompositeData((CompositeData[]) values[3], toStreamSummary),
- fromArrayOfCompositeData((CompositeData[]) values[4], toStreamSummary),
- StreamSession.State.valueOf((String) values[5]));
+ fromArrayOfCompositeData((CompositeData[]) values[5], toStreamSummary),
+ fromArrayOfCompositeData((CompositeData[]) values[6], toStreamSummary),
+ StreamSession.State.valueOf((String) values[7]));
Function<CompositeData, ProgressInfo> toProgressInfo = new Function<CompositeData, ProgressInfo>()
{
public ProgressInfo apply(CompositeData input)
@@ -151,11 +159,11 @@
return ProgressInfoCompositeData.fromCompositeData(input);
}
};
- for (ProgressInfo progress : fromArrayOfCompositeData((CompositeData[]) values[6], toProgressInfo))
+ for (ProgressInfo progress : fromArrayOfCompositeData((CompositeData[]) values[8], toProgressInfo))
{
info.updateProgress(progress);
}
- for (ProgressInfo progress : fromArrayOfCompositeData((CompositeData[]) values[7], toProgressInfo))
+ for (ProgressInfo progress : fromArrayOfCompositeData((CompositeData[]) values[9], toProgressInfo))
{
info.updateProgress(progress);
}
diff --git a/src/java/org/apache/cassandra/streaming/management/StreamStateCompositeData.java b/src/java/org/apache/cassandra/streaming/management/StreamStateCompositeData.java
index e25ab1a..de88762 100644
--- a/src/java/org/apache/cassandra/streaming/management/StreamStateCompositeData.java
+++ b/src/java/org/apache/cassandra/streaming/management/StreamStateCompositeData.java
@@ -28,6 +28,7 @@
import org.apache.cassandra.streaming.SessionInfo;
import org.apache.cassandra.streaming.StreamState;
+import org.apache.cassandra.streaming.StreamOperation;
/**
*/
@@ -73,7 +74,7 @@
{
Map<String, Object> valueMap = new HashMap<>();
valueMap.put(ITEM_NAMES[0], streamState.planId.toString());
- valueMap.put(ITEM_NAMES[1], streamState.description);
+ valueMap.put(ITEM_NAMES[1], streamState.streamOperation.getDescription());
CompositeData[] sessions = new CompositeData[streamState.sessions.size()];
Lists.newArrayList(Iterables.transform(streamState.sessions, new Function<SessionInfo, CompositeData>()
@@ -121,7 +122,7 @@
assert cd.getCompositeType().equals(COMPOSITE_TYPE);
Object[] values = cd.getAll(ITEM_NAMES);
UUID planId = UUID.fromString((String) values[0]);
- String description = (String) values[1];
+ String typeString = (String) values[1];
Set<SessionInfo> sessions = Sets.newHashSet(Iterables.transform(Arrays.asList((CompositeData[]) values[2]),
new Function<CompositeData, SessionInfo>()
{
@@ -130,6 +131,6 @@
return SessionInfoCompositeData.fromCompositeData(input);
}
}));
- return new StreamState(planId, description, sessions);
+ return new StreamState(planId, StreamOperation.fromString(typeString), sessions);
}
}
diff --git a/src/java/org/apache/cassandra/streaming/management/StreamSummaryCompositeData.java b/src/java/org/apache/cassandra/streaming/management/StreamSummaryCompositeData.java
index 9ef23ab..a1f2496 100644
--- a/src/java/org/apache/cassandra/streaming/management/StreamSummaryCompositeData.java
+++ b/src/java/org/apache/cassandra/streaming/management/StreamSummaryCompositeData.java
@@ -19,18 +19,18 @@
import java.util.HashMap;
import java.util.Map;
-import java.util.UUID;
import javax.management.openmbean.*;
import com.google.common.base.Throwables;
+import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.streaming.StreamSummary;
/**
*/
public class StreamSummaryCompositeData
{
- private static final String[] ITEM_NAMES = new String[]{"cfId",
+ private static final String[] ITEM_NAMES = new String[]{"tableId",
"files",
"totalSize"};
private static final String[] ITEM_DESCS = new String[]{"ColumnFamilu ID",
@@ -60,7 +60,7 @@
public static CompositeData toCompositeData(StreamSummary streamSummary)
{
Map<String, Object> valueMap = new HashMap<>();
- valueMap.put(ITEM_NAMES[0], streamSummary.cfId.toString());
+ valueMap.put(ITEM_NAMES[0], streamSummary.tableId.toString());
valueMap.put(ITEM_NAMES[1], streamSummary.files);
valueMap.put(ITEM_NAMES[2], streamSummary.totalSize);
try
@@ -76,7 +76,7 @@
public static StreamSummary fromCompositeData(CompositeData cd)
{
Object[] values = cd.getAll(ITEM_NAMES);
- return new StreamSummary(UUID.fromString((String) values[0]),
+ return new StreamSummary(TableId.fromString((String) values[0]),
(int) values[1],
(long) values[2]);
}
diff --git a/src/java/org/apache/cassandra/streaming/messages/CompleteMessage.java b/src/java/org/apache/cassandra/streaming/messages/CompleteMessage.java
index 44ff553..83d95e0 100644
--- a/src/java/org/apache/cassandra/streaming/messages/CompleteMessage.java
+++ b/src/java/org/apache/cassandra/streaming/messages/CompleteMessage.java
@@ -17,8 +17,7 @@
*/
package org.apache.cassandra.streaming.messages;
-import java.nio.channels.ReadableByteChannel;
-
+import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputStreamPlus;
import org.apache.cassandra.streaming.StreamSession;
@@ -26,12 +25,17 @@
{
public static Serializer<CompleteMessage> serializer = new Serializer<CompleteMessage>()
{
- public CompleteMessage deserialize(ReadableByteChannel in, int version, StreamSession session)
+ public CompleteMessage deserialize(DataInputPlus in, int version)
{
return new CompleteMessage();
}
public void serialize(CompleteMessage message, DataOutputStreamPlus out, int version, StreamSession session) {}
+
+ public long serializedSize(CompleteMessage message, int version)
+ {
+ return 0;
+ }
};
public CompleteMessage()
diff --git a/src/java/org/apache/cassandra/streaming/messages/FileMessageHeader.java b/src/java/org/apache/cassandra/streaming/messages/FileMessageHeader.java
deleted file mode 100644
index 232727d..0000000
--- a/src/java/org/apache/cassandra/streaming/messages/FileMessageHeader.java
+++ /dev/null
@@ -1,271 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.streaming.messages;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.UUID;
-
-import org.apache.cassandra.db.SerializationHeader;
-import org.apache.cassandra.db.TypeSizes;
-import org.apache.cassandra.io.compress.CompressionMetadata;
-import org.apache.cassandra.io.sstable.format.SSTableFormat;
-import org.apache.cassandra.io.util.DataInputPlus;
-import org.apache.cassandra.io.util.DataOutputPlus;
-import org.apache.cassandra.io.sstable.format.Version;
-import org.apache.cassandra.net.MessagingService;
-import org.apache.cassandra.streaming.compress.CompressionInfo;
-import org.apache.cassandra.utils.Pair;
-import org.apache.cassandra.utils.UUIDSerializer;
-
-/**
- * StreamingFileHeader is appended before sending actual data to describe what it's sending.
- */
-public class FileMessageHeader
-{
- public static FileMessageHeaderSerializer serializer = new FileMessageHeaderSerializer();
-
- public final UUID cfId;
- public final int sequenceNumber;
- /** SSTable version */
- public final Version version;
-
- /** SSTable format **/
- public final SSTableFormat.Type format;
- public final long estimatedKeys;
- public final List<Pair<Long, Long>> sections;
- /**
- * Compression info for SSTable to send. Can be null if SSTable is not compressed.
- * On sender, this field is always null to avoid holding large number of Chunks.
- * Use compressionMetadata instead.
- */
- public final CompressionInfo compressionInfo;
- private final CompressionMetadata compressionMetadata;
- public final long repairedAt;
- public final int sstableLevel;
- public final SerializationHeader.Component header;
-
- /* cached size value */
- private transient final long size;
-
- public FileMessageHeader(UUID cfId,
- int sequenceNumber,
- Version version,
- SSTableFormat.Type format,
- long estimatedKeys,
- List<Pair<Long, Long>> sections,
- CompressionInfo compressionInfo,
- long repairedAt,
- int sstableLevel,
- SerializationHeader.Component header)
- {
- this.cfId = cfId;
- this.sequenceNumber = sequenceNumber;
- this.version = version;
- this.format = format;
- this.estimatedKeys = estimatedKeys;
- this.sections = sections;
- this.compressionInfo = compressionInfo;
- this.compressionMetadata = null;
- this.repairedAt = repairedAt;
- this.sstableLevel = sstableLevel;
- this.header = header;
- this.size = calculateSize();
- }
-
- public FileMessageHeader(UUID cfId,
- int sequenceNumber,
- Version version,
- SSTableFormat.Type format,
- long estimatedKeys,
- List<Pair<Long, Long>> sections,
- CompressionMetadata compressionMetadata,
- long repairedAt,
- int sstableLevel,
- SerializationHeader.Component header)
- {
- this.cfId = cfId;
- this.sequenceNumber = sequenceNumber;
- this.version = version;
- this.format = format;
- this.estimatedKeys = estimatedKeys;
- this.sections = sections;
- this.compressionInfo = null;
- this.compressionMetadata = compressionMetadata;
- this.repairedAt = repairedAt;
- this.sstableLevel = sstableLevel;
- this.header = header;
- this.size = calculateSize();
- }
-
- public boolean isCompressed()
- {
- return compressionInfo != null || compressionMetadata != null;
- }
-
- /**
- * @return total file size to transfer in bytes
- */
- public long size()
- {
- return size;
- }
-
- private long calculateSize()
- {
- long transferSize = 0;
- if (compressionInfo != null)
- {
- // calculate total length of transferring chunks
- for (CompressionMetadata.Chunk chunk : compressionInfo.chunks)
- transferSize += chunk.length + 4; // 4 bytes for CRC
- }
- else if (compressionMetadata != null)
- {
- transferSize = compressionMetadata.getTotalSizeForSections(sections);
- }
- else
- {
- for (Pair<Long, Long> section : sections)
- transferSize += section.right - section.left;
- }
- return transferSize;
- }
-
- @Override
- public String toString()
- {
- final StringBuilder sb = new StringBuilder("Header (");
- sb.append("cfId: ").append(cfId);
- sb.append(", #").append(sequenceNumber);
- sb.append(", version: ").append(version);
- sb.append(", format: ").append(format);
- sb.append(", estimated keys: ").append(estimatedKeys);
- sb.append(", transfer size: ").append(size());
- sb.append(", compressed?: ").append(isCompressed());
- sb.append(", repairedAt: ").append(repairedAt);
- sb.append(", level: ").append(sstableLevel);
- sb.append(')');
- return sb.toString();
- }
-
- @Override
- public boolean equals(Object o)
- {
- if (this == o) return true;
- if (o == null || getClass() != o.getClass()) return false;
- FileMessageHeader that = (FileMessageHeader) o;
- return sequenceNumber == that.sequenceNumber && cfId.equals(that.cfId);
- }
-
- @Override
- public int hashCode()
- {
- int result = cfId.hashCode();
- result = 31 * result + sequenceNumber;
- return result;
- }
-
- static class FileMessageHeaderSerializer
- {
- public CompressionInfo serialize(FileMessageHeader header, DataOutputPlus out, int version) throws IOException
- {
- UUIDSerializer.serializer.serialize(header.cfId, out, version);
- out.writeInt(header.sequenceNumber);
- out.writeUTF(header.version.toString());
-
- //We can't stream to a node that doesn't understand a new sstable format
- if (version < StreamMessage.VERSION_22 && header.format != SSTableFormat.Type.LEGACY && header.format != SSTableFormat.Type.BIG)
- throw new UnsupportedOperationException("Can't stream non-legacy sstables to nodes < 2.2");
-
- if (version >= StreamMessage.VERSION_22)
- out.writeUTF(header.format.name);
-
- out.writeLong(header.estimatedKeys);
- out.writeInt(header.sections.size());
- for (Pair<Long, Long> section : header.sections)
- {
- out.writeLong(section.left);
- out.writeLong(section.right);
- }
- // construct CompressionInfo here to avoid holding large number of Chunks on heap.
- CompressionInfo compressionInfo = null;
- if (header.compressionMetadata != null)
- compressionInfo = new CompressionInfo(header.compressionMetadata.getChunksForSections(header.sections), header.compressionMetadata.parameters);
- CompressionInfo.serializer.serialize(compressionInfo, out, version);
- out.writeLong(header.repairedAt);
- out.writeInt(header.sstableLevel);
-
- if (version >= StreamMessage.VERSION_30 && header.version.storeRows())
- SerializationHeader.serializer.serialize(header.version, header.header, out);
- return compressionInfo;
- }
-
- public FileMessageHeader deserialize(DataInputPlus in, int version) throws IOException
- {
- UUID cfId = UUIDSerializer.serializer.deserialize(in, MessagingService.current_version);
- int sequenceNumber = in.readInt();
- Version sstableVersion = SSTableFormat.Type.current().info.getVersion(in.readUTF());
-
- SSTableFormat.Type format = SSTableFormat.Type.LEGACY;
- if (version >= StreamMessage.VERSION_22)
- format = SSTableFormat.Type.validate(in.readUTF());
-
- long estimatedKeys = in.readLong();
- int count = in.readInt();
- List<Pair<Long, Long>> sections = new ArrayList<>(count);
- for (int k = 0; k < count; k++)
- sections.add(Pair.create(in.readLong(), in.readLong()));
- CompressionInfo compressionInfo = CompressionInfo.serializer.deserialize(in, MessagingService.current_version);
- long repairedAt = in.readLong();
- int sstableLevel = in.readInt();
- SerializationHeader.Component header = version >= StreamMessage.VERSION_30 && sstableVersion.storeRows()
- ? SerializationHeader.serializer.deserialize(sstableVersion, in)
- : null;
-
- return new FileMessageHeader(cfId, sequenceNumber, sstableVersion, format, estimatedKeys, sections, compressionInfo, repairedAt, sstableLevel, header);
- }
-
- public long serializedSize(FileMessageHeader header, int version)
- {
- long size = UUIDSerializer.serializer.serializedSize(header.cfId, version);
- size += TypeSizes.sizeof(header.sequenceNumber);
- size += TypeSizes.sizeof(header.version.toString());
-
- if (version >= StreamMessage.VERSION_22)
- size += TypeSizes.sizeof(header.format.name);
-
- size += TypeSizes.sizeof(header.estimatedKeys);
-
- size += TypeSizes.sizeof(header.sections.size());
- for (Pair<Long, Long> section : header.sections)
- {
- size += TypeSizes.sizeof(section.left);
- size += TypeSizes.sizeof(section.right);
- }
- size += CompressionInfo.serializer.serializedSize(header.compressionInfo, version);
- size += TypeSizes.sizeof(header.sstableLevel);
-
- if (version >= StreamMessage.VERSION_30)
- size += SerializationHeader.serializer.serializedSize(header.version, header.header);
-
- return size;
- }
- }
-}
diff --git a/src/java/org/apache/cassandra/streaming/messages/IncomingFileMessage.java b/src/java/org/apache/cassandra/streaming/messages/IncomingFileMessage.java
deleted file mode 100644
index 66480d4..0000000
--- a/src/java/org/apache/cassandra/streaming/messages/IncomingFileMessage.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.streaming.messages;
-
-import java.io.IOException;
-import java.nio.channels.Channels;
-import java.nio.channels.ReadableByteChannel;
-import java.util.Optional;
-
-import org.apache.cassandra.io.sstable.SSTableMultiWriter;
-import org.apache.cassandra.io.util.DataInputPlus;
-import org.apache.cassandra.io.util.DataInputPlus.DataInputStreamPlus;
-
-import org.apache.cassandra.io.util.DataOutputStreamPlus;
-import org.apache.cassandra.streaming.StreamReader;
-import org.apache.cassandra.streaming.StreamSession;
-import org.apache.cassandra.streaming.compress.CompressedStreamReader;
-import org.apache.cassandra.utils.JVMStabilityInspector;
-
-import static org.apache.cassandra.utils.Throwables.extractIOExceptionCause;
-
-/**
- * IncomingFileMessage is used to receive the part(or whole) of a SSTable data file.
- */
-public class IncomingFileMessage extends StreamMessage
-{
- public static Serializer<IncomingFileMessage> serializer = new Serializer<IncomingFileMessage>()
- {
- @SuppressWarnings("resource")
- public IncomingFileMessage deserialize(ReadableByteChannel in, int version, StreamSession session) throws IOException
- {
- DataInputPlus input = new DataInputStreamPlus(Channels.newInputStream(in));
- FileMessageHeader header = FileMessageHeader.serializer.deserialize(input, version);
- StreamReader reader = !header.isCompressed() ? new StreamReader(header, session)
- : new CompressedStreamReader(header, session);
-
- try
- {
- return new IncomingFileMessage(reader.read(in), header);
- }
- catch (Throwable t)
- {
- JVMStabilityInspector.inspectThrowable(t);
- throw t;
- }
- }
-
- public void serialize(IncomingFileMessage message, DataOutputStreamPlus out, int version, StreamSession session)
- {
- throw new UnsupportedOperationException("Not allowed to call serialize on an incoming file");
- }
- };
-
- public FileMessageHeader header;
- public SSTableMultiWriter sstable;
-
- public IncomingFileMessage(SSTableMultiWriter sstable, FileMessageHeader header)
- {
- super(Type.FILE);
- this.header = header;
- this.sstable = sstable;
- }
-
- @Override
- public String toString()
- {
- return "File (" + header + ", file: " + sstable.getFilename() + ")";
- }
-}
-
diff --git a/src/java/org/apache/cassandra/streaming/messages/IncomingStreamMessage.java b/src/java/org/apache/cassandra/streaming/messages/IncomingStreamMessage.java
new file mode 100644
index 0000000..e268747
--- /dev/null
+++ b/src/java/org/apache/cassandra/streaming/messages/IncomingStreamMessage.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.streaming.messages;
+
+import java.io.IOException;
+import java.util.Objects;
+
+import io.netty.channel.Channel;
+
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.io.util.DataInputPlus;
+
+import org.apache.cassandra.io.util.DataOutputStreamPlus;
+import org.apache.cassandra.streaming.IncomingStream;
+import org.apache.cassandra.streaming.StreamManager;
+import org.apache.cassandra.streaming.StreamReceiveException;
+import org.apache.cassandra.streaming.StreamSession;
+import org.apache.cassandra.utils.JVMStabilityInspector;
+
+public class IncomingStreamMessage extends StreamMessage
+{
+ public static Serializer<IncomingStreamMessage> serializer = new Serializer<IncomingStreamMessage>()
+ {
+ @SuppressWarnings("resource")
+ public IncomingStreamMessage deserialize(DataInputPlus input, int version) throws IOException
+ {
+ StreamMessageHeader header = StreamMessageHeader.serializer.deserialize(input, version);
+ StreamSession session = StreamManager.instance.findSession(header.sender, header.planId, header.sessionIndex, header.sendByFollower);
+ if (session == null)
+ throw new IllegalStateException(String.format("unknown stream session: %s - %d", header.planId, header.sessionIndex));
+ ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(header.tableId);
+ if (cfs == null)
+ throw new StreamReceiveException(session, "CF " + header.tableId + " was dropped during streaming");
+
+ try
+ {
+ IncomingStream incomingData = cfs.getStreamManager().prepareIncomingStream(session, header);
+ incomingData.read(input, version);
+
+ return new IncomingStreamMessage(incomingData, header);
+ }
+ catch (Throwable t)
+ {
+ JVMStabilityInspector.inspectThrowable(t);
+ throw new StreamReceiveException(session, t);
+ }
+ }
+
+ public void serialize(IncomingStreamMessage message, DataOutputStreamPlus out, int version, StreamSession session)
+ {
+ throw new UnsupportedOperationException("Not allowed to call serialize on an incoming stream");
+ }
+
+ public long serializedSize(IncomingStreamMessage message, int version)
+ {
+ throw new UnsupportedOperationException("Not allowed to call serializedSize on an incoming stream");
+ }
+ };
+
+ public final StreamMessageHeader header;
+ public final IncomingStream stream;
+
+ public IncomingStreamMessage(IncomingStream stream, StreamMessageHeader header)
+ {
+ super(Type.STREAM);
+ this.stream = stream;
+ this.header = header;
+ }
+
+ @Override
+ public StreamSession getOrCreateSession(Channel channel)
+ {
+ return stream.session();
+ }
+
+ @Override
+ public String toString()
+ {
+ return "IncomingStreamMessage{" +
+ "header=" + header +
+ ", stream=" + stream +
+ '}';
+ }
+
+ public boolean equals(Object o)
+ {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ IncomingStreamMessage that = (IncomingStreamMessage) o;
+ return Objects.equals(header, that.header) &&
+ Objects.equals(stream, that.stream);
+ }
+
+ public int hashCode()
+ {
+
+ return Objects.hash(header, stream);
+ }
+}
+
diff --git a/src/java/org/apache/cassandra/streaming/messages/KeepAliveMessage.java b/src/java/org/apache/cassandra/streaming/messages/KeepAliveMessage.java
index bfdc72e..5352b3b 100644
--- a/src/java/org/apache/cassandra/streaming/messages/KeepAliveMessage.java
+++ b/src/java/org/apache/cassandra/streaming/messages/KeepAliveMessage.java
@@ -19,8 +19,8 @@
package org.apache.cassandra.streaming.messages;
import java.io.IOException;
-import java.nio.channels.ReadableByteChannel;
+import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputStreamPlus;
import org.apache.cassandra.streaming.StreamSession;
@@ -28,13 +28,18 @@
{
public static Serializer<KeepAliveMessage> serializer = new Serializer<KeepAliveMessage>()
{
- public KeepAliveMessage deserialize(ReadableByteChannel in, int version, StreamSession session) throws IOException
+ public KeepAliveMessage deserialize(DataInputPlus in, int version) throws IOException
{
return new KeepAliveMessage();
}
public void serialize(KeepAliveMessage message, DataOutputStreamPlus out, int version, StreamSession session)
{}
+
+ public long serializedSize(KeepAliveMessage message, int version)
+ {
+ return 0;
+ }
};
public KeepAliveMessage()
diff --git a/src/java/org/apache/cassandra/streaming/messages/OutgoingFileMessage.java b/src/java/org/apache/cassandra/streaming/messages/OutgoingFileMessage.java
deleted file mode 100644
index 6723d17..0000000
--- a/src/java/org/apache/cassandra/streaming/messages/OutgoingFileMessage.java
+++ /dev/null
@@ -1,143 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.streaming.messages;
-
-import java.io.IOException;
-import java.nio.channels.ReadableByteChannel;
-import java.util.List;
-
-import com.google.common.annotations.VisibleForTesting;
-
-import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.io.util.DataOutputStreamPlus;
-import org.apache.cassandra.streaming.StreamSession;
-import org.apache.cassandra.streaming.StreamWriter;
-import org.apache.cassandra.streaming.compress.CompressedStreamWriter;
-import org.apache.cassandra.streaming.compress.CompressionInfo;
-import org.apache.cassandra.utils.Pair;
-import org.apache.cassandra.utils.concurrent.Ref;
-
-/**
- * OutgoingFileMessage is used to transfer the part(or whole) of a SSTable data file.
- */
-public class OutgoingFileMessage extends StreamMessage
-{
- public static Serializer<OutgoingFileMessage> serializer = new Serializer<OutgoingFileMessage>()
- {
- public OutgoingFileMessage deserialize(ReadableByteChannel in, int version, StreamSession session)
- {
- throw new UnsupportedOperationException("Not allowed to call deserialize on an outgoing file");
- }
-
- public void serialize(OutgoingFileMessage message, DataOutputStreamPlus out, int version, StreamSession session) throws IOException
- {
- message.startTransfer();
- try
- {
- message.serialize(out, version, session);
- session.fileSent(message.header);
- }
- finally
- {
- message.finishTransfer();
- }
- }
- };
-
- public final FileMessageHeader header;
- private final Ref<SSTableReader> ref;
- private final String filename;
- private boolean completed = false;
- private boolean transferring = false;
-
- public OutgoingFileMessage(Ref<SSTableReader> ref, int sequenceNumber, long estimatedKeys, List<Pair<Long, Long>> sections, long repairedAt, boolean keepSSTableLevel)
- {
- super(Type.FILE);
- this.ref = ref;
-
- SSTableReader sstable = ref.get();
- filename = sstable.getFilename();
- this.header = new FileMessageHeader(sstable.metadata.cfId,
- sequenceNumber,
- sstable.descriptor.version,
- sstable.descriptor.formatType,
- estimatedKeys,
- sections,
- sstable.compression ? sstable.getCompressionMetadata() : null,
- repairedAt,
- keepSSTableLevel ? sstable.getSSTableLevel() : 0,
- sstable.header == null ? null : sstable.header.toComponent());
- }
-
- public synchronized void serialize(DataOutputStreamPlus out, int version, StreamSession session) throws IOException
- {
- if (completed)
- {
- return;
- }
-
- CompressionInfo compressionInfo = FileMessageHeader.serializer.serialize(header, out, version);
-
- final SSTableReader reader = ref.get();
- StreamWriter writer = compressionInfo == null ?
- new StreamWriter(reader, header.sections, session) :
- new CompressedStreamWriter(reader, header.sections,
- compressionInfo, session);
- writer.write(out);
- }
-
- @VisibleForTesting
- public synchronized void finishTransfer()
- {
- transferring = false;
- //session was aborted mid-transfer, now it's safe to release
- if (completed)
- {
- ref.release();
- }
- }
-
- @VisibleForTesting
- public synchronized void startTransfer()
- {
- if (completed)
- throw new RuntimeException(String.format("Transfer of file %s already completed or aborted (perhaps session failed?).",
- filename));
- transferring = true;
- }
-
- public synchronized void complete()
- {
- if (!completed)
- {
- completed = true;
- //release only if not transferring
- if (!transferring)
- {
- ref.release();
- }
- }
- }
-
- @Override
- public String toString()
- {
- return "File (" + header + ", file: " + filename + ")";
- }
-}
-
diff --git a/src/java/org/apache/cassandra/streaming/messages/OutgoingStreamMessage.java b/src/java/org/apache/cassandra/streaming/messages/OutgoingStreamMessage.java
new file mode 100644
index 0000000..702e806
--- /dev/null
+++ b/src/java/org/apache/cassandra/streaming/messages/OutgoingStreamMessage.java
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.streaming.messages;
+
+import java.io.IOException;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputStreamPlus;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.streaming.OutgoingStream;
+import org.apache.cassandra.streaming.StreamSession;
+import org.apache.cassandra.utils.FBUtilities;
+
+public class OutgoingStreamMessage extends StreamMessage
+{
+ public static Serializer<OutgoingStreamMessage> serializer = new Serializer<OutgoingStreamMessage>()
+ {
+ public OutgoingStreamMessage deserialize(DataInputPlus in, int version)
+ {
+ throw new UnsupportedOperationException("Not allowed to call deserialize on an outgoing stream");
+ }
+
+ public void serialize(OutgoingStreamMessage message, DataOutputStreamPlus out, int version, StreamSession session) throws IOException
+ {
+ message.startTransfer();
+ try
+ {
+ message.serialize(out, version, session);
+ session.streamSent(message);
+ }
+ finally
+ {
+ message.finishTransfer();
+ }
+ }
+
+ public long serializedSize(OutgoingStreamMessage message, int version)
+ {
+ return 0;
+ }
+ };
+
+ public final StreamMessageHeader header;
+ public final OutgoingStream stream;
+ private boolean completed = false;
+ private boolean transferring = false;
+
+ public OutgoingStreamMessage(TableId tableId, StreamSession session, OutgoingStream stream, int sequenceNumber)
+ {
+ super(Type.STREAM);
+
+ this.stream = stream;
+ this.header = new StreamMessageHeader(tableId,
+ FBUtilities.getBroadcastAddressAndPort(),
+ session.planId(),
+ session.isFollower(),
+ session.sessionIndex(),
+ sequenceNumber,
+ stream.getRepairedAt(),
+ stream.getPendingRepair());
+ }
+
+ public synchronized void serialize(DataOutputStreamPlus out, int version, StreamSession session) throws IOException
+ {
+ if (completed)
+ {
+ return;
+ }
+ StreamMessageHeader.serializer.serialize(header, out, version);
+ stream.write(session, out, version);
+ }
+
+ @VisibleForTesting
+ public synchronized void finishTransfer()
+ {
+ transferring = false;
+ //session was aborted mid-transfer, now it's safe to release
+ if (completed)
+ {
+ stream.finish();
+ }
+ }
+
+ @VisibleForTesting
+ public synchronized void startTransfer()
+ {
+ if (completed)
+ throw new RuntimeException(String.format("Transfer of stream %s already completed or aborted (perhaps session failed?).",
+ stream));
+ transferring = true;
+ }
+
+ public synchronized void complete()
+ {
+ if (!completed)
+ {
+ completed = true;
+ //release only if not transferring
+ if (!transferring)
+ {
+ stream.finish();
+ }
+ }
+ }
+
+ @Override
+ public String toString()
+ {
+ return "OutgoingStreamMessage{" +
+ "header=" + header +
+ ", stream=" + stream +
+ '}';
+ }
+
+ public String getName()
+ {
+ return stream.getName();
+ }
+}
+
diff --git a/src/java/org/apache/cassandra/streaming/messages/PrepareAckMessage.java b/src/java/org/apache/cassandra/streaming/messages/PrepareAckMessage.java
new file mode 100644
index 0000000..97fdff7
--- /dev/null
+++ b/src/java/org/apache/cassandra/streaming/messages/PrepareAckMessage.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.streaming.messages;
+
+import java.io.IOException;
+
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputStreamPlus;
+import org.apache.cassandra.streaming.StreamSession;
+
+public class PrepareAckMessage extends StreamMessage
+{
+ public static Serializer<PrepareAckMessage> serializer = new Serializer<PrepareAckMessage>()
+ {
+ public void serialize(PrepareAckMessage message, DataOutputStreamPlus out, int version, StreamSession session) throws IOException
+ {
+ //nop
+ }
+
+ public PrepareAckMessage deserialize(DataInputPlus in, int version) throws IOException
+ {
+ return new PrepareAckMessage();
+ }
+
+ public long serializedSize(PrepareAckMessage message, int version)
+ {
+ return 0;
+ }
+ };
+
+ public PrepareAckMessage()
+ {
+ super(Type.PREPARE_ACK);
+ }
+
+ @Override
+ public String toString()
+ {
+ return "Prepare ACK";
+ }
+}
diff --git a/src/java/org/apache/cassandra/streaming/messages/PrepareMessage.java b/src/java/org/apache/cassandra/streaming/messages/PrepareMessage.java
deleted file mode 100644
index 1f53be7..0000000
--- a/src/java/org/apache/cassandra/streaming/messages/PrepareMessage.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.streaming.messages;
-
-import java.io.*;
-import java.nio.channels.Channels;
-import java.nio.channels.ReadableByteChannel;
-import java.util.ArrayList;
-import java.util.Collection;
-
-import org.apache.cassandra.io.util.DataInputPlus;
-import org.apache.cassandra.io.util.DataInputPlus.DataInputStreamPlus;
-import org.apache.cassandra.io.util.DataOutputStreamPlus;
-import org.apache.cassandra.streaming.StreamRequest;
-import org.apache.cassandra.streaming.StreamSession;
-import org.apache.cassandra.streaming.StreamSummary;
-
-public class PrepareMessage extends StreamMessage
-{
- public static Serializer<PrepareMessage> serializer = new Serializer<PrepareMessage>()
- {
- @SuppressWarnings("resource") // Not closing constructed DataInputPlus's as the channel needs to remain open.
- public PrepareMessage deserialize(ReadableByteChannel in, int version, StreamSession session) throws IOException
- {
- DataInputPlus input = new DataInputStreamPlus(Channels.newInputStream(in));
- PrepareMessage message = new PrepareMessage();
- // requests
- int numRequests = input.readInt();
- for (int i = 0; i < numRequests; i++)
- message.requests.add(StreamRequest.serializer.deserialize(input, version));
- // summaries
- int numSummaries = input.readInt();
- for (int i = 0; i < numSummaries; i++)
- message.summaries.add(StreamSummary.serializer.deserialize(input, version));
- return message;
- }
-
- public void serialize(PrepareMessage message, DataOutputStreamPlus out, int version, StreamSession session) throws IOException
- {
- // requests
- out.writeInt(message.requests.size());
- for (StreamRequest request : message.requests)
- StreamRequest.serializer.serialize(request, out, version);
- // summaries
- out.writeInt(message.summaries.size());
- for (StreamSummary summary : message.summaries)
- StreamSummary.serializer.serialize(summary, out, version);
- }
- };
-
- /**
- * Streaming requests
- */
- public final Collection<StreamRequest> requests = new ArrayList<>();
-
- /**
- * Summaries of streaming out
- */
- public final Collection<StreamSummary> summaries = new ArrayList<>();
-
- public PrepareMessage()
- {
- super(Type.PREPARE);
- }
-
- @Override
- public String toString()
- {
- final StringBuilder sb = new StringBuilder("Prepare (");
- sb.append(requests.size()).append(" requests, ");
- int totalFile = 0;
- for (StreamSummary summary : summaries)
- totalFile += summary.files;
- sb.append(" ").append(totalFile).append(" files");
- sb.append('}');
- return sb.toString();
- }
-}
diff --git a/src/java/org/apache/cassandra/streaming/messages/PrepareSynAckMessage.java b/src/java/org/apache/cassandra/streaming/messages/PrepareSynAckMessage.java
new file mode 100644
index 0000000..4e5e8fb
--- /dev/null
+++ b/src/java/org/apache/cassandra/streaming/messages/PrepareSynAckMessage.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.streaming.messages;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputStreamPlus;
+import org.apache.cassandra.streaming.StreamSession;
+import org.apache.cassandra.streaming.StreamSummary;
+
+public class PrepareSynAckMessage extends StreamMessage
+{
+ public static Serializer<PrepareSynAckMessage> serializer = new Serializer<PrepareSynAckMessage>()
+ {
+ public void serialize(PrepareSynAckMessage message, DataOutputStreamPlus out, int version, StreamSession session) throws IOException
+ {
+ out.writeInt(message.summaries.size());
+ for (StreamSummary summary : message.summaries)
+ StreamSummary.serializer.serialize(summary, out, version);
+ }
+
+ public PrepareSynAckMessage deserialize(DataInputPlus input, int version) throws IOException
+ {
+ PrepareSynAckMessage message = new PrepareSynAckMessage();
+ int numSummaries = input.readInt();
+ for (int i = 0; i < numSummaries; i++)
+ message.summaries.add(StreamSummary.serializer.deserialize(input, version));
+ return message;
+ }
+
+ public long serializedSize(PrepareSynAckMessage message, int version)
+ {
+ long size = 4; // count of requests and count of summaries
+ for (StreamSummary summary : message.summaries)
+ size += StreamSummary.serializer.serializedSize(summary, version);
+ return size;
+ }
+ };
+
+ /**
+ * Summaries of streaming out
+ */
+ public final Collection<StreamSummary> summaries = new ArrayList<>();
+
+ public PrepareSynAckMessage()
+ {
+ super(Type.PREPARE_SYNACK);
+ }
+
+ @Override
+ public String toString()
+ {
+ final StringBuilder sb = new StringBuilder("Prepare SYNACK (");
+ int totalFile = 0;
+ for (StreamSummary summary : summaries)
+ totalFile += summary.files;
+ sb.append(" ").append(totalFile).append(" files");
+ sb.append('}');
+ return sb.toString();
+ }
+}
diff --git a/src/java/org/apache/cassandra/streaming/messages/PrepareSynMessage.java b/src/java/org/apache/cassandra/streaming/messages/PrepareSynMessage.java
new file mode 100644
index 0000000..e378af7
--- /dev/null
+++ b/src/java/org/apache/cassandra/streaming/messages/PrepareSynMessage.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.streaming.messages;
+
+import java.io.*;
+import java.util.ArrayList;
+import java.util.Collection;
+
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputStreamPlus;
+import org.apache.cassandra.streaming.StreamRequest;
+import org.apache.cassandra.streaming.StreamSession;
+import org.apache.cassandra.streaming.StreamSummary;
+
+public class PrepareSynMessage extends StreamMessage
+{
+ public static Serializer<PrepareSynMessage> serializer = new Serializer<PrepareSynMessage>()
+ {
+ public PrepareSynMessage deserialize(DataInputPlus input, int version) throws IOException
+ {
+ PrepareSynMessage message = new PrepareSynMessage();
+ // requests
+ int numRequests = input.readInt();
+ for (int i = 0; i < numRequests; i++)
+ message.requests.add(StreamRequest.serializer.deserialize(input, version));
+ // summaries
+ int numSummaries = input.readInt();
+ for (int i = 0; i < numSummaries; i++)
+ message.summaries.add(StreamSummary.serializer.deserialize(input, version));
+ return message;
+ }
+
+ public long serializedSize(PrepareSynMessage message, int version)
+ {
+ long size = 4 + 4; // count of requests and count of summaries
+ for (StreamRequest request : message.requests)
+ size += StreamRequest.serializer.serializedSize(request, version);
+ for (StreamSummary summary : message.summaries)
+ size += StreamSummary.serializer.serializedSize(summary, version);
+ return size;
+ }
+
+ public void serialize(PrepareSynMessage message, DataOutputStreamPlus out, int version, StreamSession session) throws IOException
+ {
+ // requests
+ out.writeInt(message.requests.size());
+ for (StreamRequest request : message.requests)
+ StreamRequest.serializer.serialize(request, out, version);
+ // summaries
+ out.writeInt(message.summaries.size());
+ for (StreamSummary summary : message.summaries)
+ StreamSummary.serializer.serialize(summary, out, version);
+ }
+ };
+
+ /**
+ * Streaming requests
+ */
+ public final Collection<StreamRequest> requests = new ArrayList<>();
+
+ /**
+ * Summaries of streaming out
+ */
+ public final Collection<StreamSummary> summaries = new ArrayList<>();
+
+ public PrepareSynMessage()
+ {
+ super(Type.PREPARE_SYN);
+ }
+
+ @Override
+ public String toString()
+ {
+ final StringBuilder sb = new StringBuilder("Prepare SYN (");
+ sb.append(requests.size()).append(" requests, ");
+ int totalFile = 0;
+ for (StreamSummary summary : summaries)
+ totalFile += summary.files;
+ sb.append(" ").append(totalFile).append(" files");
+ sb.append('}');
+ return sb.toString();
+ }
+}
diff --git a/src/java/org/apache/cassandra/streaming/messages/ReceivedMessage.java b/src/java/org/apache/cassandra/streaming/messages/ReceivedMessage.java
index 251b9c8..ff2cdec 100644
--- a/src/java/org/apache/cassandra/streaming/messages/ReceivedMessage.java
+++ b/src/java/org/apache/cassandra/streaming/messages/ReceivedMessage.java
@@ -18,42 +18,41 @@
package org.apache.cassandra.streaming.messages;
import java.io.*;
-import java.nio.channels.Channels;
-import java.nio.channels.ReadableByteChannel;
-import java.util.UUID;
import org.apache.cassandra.io.util.DataInputPlus;
-import org.apache.cassandra.io.util.DataInputPlus.DataInputStreamPlus;
import org.apache.cassandra.io.util.DataOutputStreamPlus;
-import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.streaming.StreamSession;
-import org.apache.cassandra.utils.UUIDSerializer;
public class ReceivedMessage extends StreamMessage
{
public static Serializer<ReceivedMessage> serializer = new Serializer<ReceivedMessage>()
{
@SuppressWarnings("resource") // Not closing constructed DataInputPlus's as the channel needs to remain open.
- public ReceivedMessage deserialize(ReadableByteChannel in, int version, StreamSession session) throws IOException
+ public ReceivedMessage deserialize(DataInputPlus input, int version) throws IOException
{
- DataInputPlus input = new DataInputStreamPlus(Channels.newInputStream(in));
- return new ReceivedMessage(UUIDSerializer.serializer.deserialize(input, MessagingService.current_version), input.readInt());
+ return new ReceivedMessage(TableId.deserialize(input), input.readInt());
}
public void serialize(ReceivedMessage message, DataOutputStreamPlus out, int version, StreamSession session) throws IOException
{
- UUIDSerializer.serializer.serialize(message.cfId, out, MessagingService.current_version);
+ message.tableId.serialize(out);
out.writeInt(message.sequenceNumber);
}
+
+ public long serializedSize(ReceivedMessage message, int version)
+ {
+ return message.tableId.serializedSize() + 4;
+ }
};
- public final UUID cfId;
+ public final TableId tableId;
public final int sequenceNumber;
- public ReceivedMessage(UUID cfId, int sequenceNumber)
+ public ReceivedMessage(TableId tableId, int sequenceNumber)
{
super(Type.RECEIVED);
- this.cfId = cfId;
+ this.tableId = tableId;
this.sequenceNumber = sequenceNumber;
}
@@ -61,7 +60,7 @@
public String toString()
{
final StringBuilder sb = new StringBuilder("Received (");
- sb.append(cfId).append(", #").append(sequenceNumber).append(')');
+ sb.append(tableId).append(", #").append(sequenceNumber).append(')');
return sb.toString();
}
}
diff --git a/src/java/org/apache/cassandra/streaming/messages/RetryMessage.java b/src/java/org/apache/cassandra/streaming/messages/RetryMessage.java
deleted file mode 100644
index 047fb06..0000000
--- a/src/java/org/apache/cassandra/streaming/messages/RetryMessage.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.streaming.messages;
-
-import java.io.*;
-import java.nio.channels.Channels;
-import java.nio.channels.ReadableByteChannel;
-import java.util.UUID;
-
-import org.apache.cassandra.io.util.DataInputPlus;
-import org.apache.cassandra.io.util.DataInputPlus.DataInputStreamPlus;
-import org.apache.cassandra.io.util.DataOutputStreamPlus;
-import org.apache.cassandra.net.MessagingService;
-import org.apache.cassandra.streaming.StreamSession;
-import org.apache.cassandra.utils.UUIDSerializer;
-
-/**
- * @deprecated retry support removed on CASSANDRA-10992
- */
-@Deprecated
-public class RetryMessage extends StreamMessage
-{
- public static Serializer<RetryMessage> serializer = new Serializer<RetryMessage>()
- {
- @SuppressWarnings("resource") // Not closing constructed DataInputPlus's as the channel needs to remain open.
- public RetryMessage deserialize(ReadableByteChannel in, int version, StreamSession session) throws IOException
- {
- DataInputPlus input = new DataInputStreamPlus(Channels.newInputStream(in));
- return new RetryMessage(UUIDSerializer.serializer.deserialize(input, MessagingService.current_version), input.readInt());
- }
-
- public void serialize(RetryMessage message, DataOutputStreamPlus out, int version, StreamSession session) throws IOException
- {
- UUIDSerializer.serializer.serialize(message.cfId, out, MessagingService.current_version);
- out.writeInt(message.sequenceNumber);
- }
- };
-
- public final UUID cfId;
- public final int sequenceNumber;
-
- public RetryMessage(UUID cfId, int sequenceNumber)
- {
- super(Type.RETRY);
- this.cfId = cfId;
- this.sequenceNumber = sequenceNumber;
- }
-
- @Override
- public String toString()
- {
- final StringBuilder sb = new StringBuilder("Retry (");
- sb.append(cfId).append(", #").append(sequenceNumber).append(')');
- return sb.toString();
- }
-}
diff --git a/src/java/org/apache/cassandra/streaming/messages/SessionFailedMessage.java b/src/java/org/apache/cassandra/streaming/messages/SessionFailedMessage.java
index 4a5b6df..ca10bcc 100644
--- a/src/java/org/apache/cassandra/streaming/messages/SessionFailedMessage.java
+++ b/src/java/org/apache/cassandra/streaming/messages/SessionFailedMessage.java
@@ -17,8 +17,7 @@
*/
package org.apache.cassandra.streaming.messages;
-import java.nio.channels.ReadableByteChannel;
-
+import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputStreamPlus;
import org.apache.cassandra.streaming.StreamSession;
@@ -26,12 +25,17 @@
{
public static Serializer<SessionFailedMessage> serializer = new Serializer<SessionFailedMessage>()
{
- public SessionFailedMessage deserialize(ReadableByteChannel in, int version, StreamSession session)
+ public SessionFailedMessage deserialize(DataInputPlus in, int version)
{
return new SessionFailedMessage();
}
public void serialize(SessionFailedMessage message, DataOutputStreamPlus out, int version, StreamSession session) {}
+
+ public long serializedSize(SessionFailedMessage message, int version)
+ {
+ return 0;
+ }
};
public SessionFailedMessage()
diff --git a/src/java/org/apache/cassandra/streaming/messages/StreamInitMessage.java b/src/java/org/apache/cassandra/streaming/messages/StreamInitMessage.java
index 6d807e9..0d6ef47 100644
--- a/src/java/org/apache/cassandra/streaming/messages/StreamInitMessage.java
+++ b/src/java/org/apache/cassandra/streaming/messages/StreamInitMessage.java
@@ -18,125 +18,110 @@
package org.apache.cassandra.streaming.messages;
import java.io.IOException;
-import java.net.InetAddress;
-import java.nio.ByteBuffer;
import java.util.UUID;
+import io.netty.channel.Channel;
+
import org.apache.cassandra.db.TypeSizes;
-import org.apache.cassandra.io.IVersionedSerializer;
import org.apache.cassandra.io.util.DataInputPlus;
-import org.apache.cassandra.io.util.DataOutputBuffer;
-import org.apache.cassandra.io.util.DataOutputBufferFixed;
-import org.apache.cassandra.io.util.DataOutputPlus;
-import org.apache.cassandra.net.CompactEndpointSerializationHelper;
+import org.apache.cassandra.io.util.DataOutputStreamPlus;
+import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.streaming.StreamOperation;
+import org.apache.cassandra.streaming.PreviewKind;
+import org.apache.cassandra.streaming.StreamResultFuture;
+import org.apache.cassandra.streaming.StreamSession;
import org.apache.cassandra.utils.UUIDSerializer;
+import static org.apache.cassandra.locator.InetAddressAndPort.Serializer.inetAddressAndPortSerializer;
+
/**
* StreamInitMessage is first sent from the node where {@link org.apache.cassandra.streaming.StreamSession} is started,
* to initiate corresponding {@link org.apache.cassandra.streaming.StreamSession} on the other side.
*/
-public class StreamInitMessage
+public class StreamInitMessage extends StreamMessage
{
- public static IVersionedSerializer<StreamInitMessage> serializer = new StreamInitMessageSerializer();
+ public static Serializer<StreamInitMessage> serializer = new StreamInitMessageSerializer();
- public final InetAddress from;
+ public final InetAddressAndPort from;
public final int sessionIndex;
public final UUID planId;
- public final String description;
+ public final StreamOperation streamOperation;
- // true if this init message is to connect for outgoing message on receiving side
- public final boolean isForOutgoing;
- public final boolean keepSSTableLevel;
- public final boolean isIncremental;
+ public final UUID pendingRepair;
+ public final PreviewKind previewKind;
- public StreamInitMessage(InetAddress from, int sessionIndex, UUID planId, String description, boolean isForOutgoing, boolean keepSSTableLevel, boolean isIncremental)
+ public StreamInitMessage(InetAddressAndPort from, int sessionIndex, UUID planId, StreamOperation streamOperation,
+ UUID pendingRepair, PreviewKind previewKind)
{
+ super(Type.STREAM_INIT);
this.from = from;
this.sessionIndex = sessionIndex;
this.planId = planId;
- this.description = description;
- this.isForOutgoing = isForOutgoing;
- this.keepSSTableLevel = keepSSTableLevel;
- this.isIncremental = isIncremental;
+ this.streamOperation = streamOperation;
+ this.pendingRepair = pendingRepair;
+ this.previewKind = previewKind;
}
- /**
- * Create serialized message.
- *
- * @param compress true if message is compressed
- * @param version Streaming protocol version
- * @return serialized message in ByteBuffer format
- */
- public ByteBuffer createMessage(boolean compress, int version)
+ @Override
+ public StreamSession getOrCreateSession(Channel channel)
{
- int header = 0;
- // set compression bit.
- if (compress)
- header |= 4;
- // set streaming bit
- header |= 8;
- // Setting up the version bit
- header |= (version << 8);
-
- byte[] bytes;
- try
- {
- int size = (int)StreamInitMessage.serializer.serializedSize(this, version);
- try (DataOutputBuffer buffer = new DataOutputBufferFixed(size))
- {
- StreamInitMessage.serializer.serialize(this, buffer, version);
- bytes = buffer.getData();
- }
- }
- catch (IOException e)
- {
- throw new RuntimeException(e);
- }
- assert bytes.length > 0;
-
- ByteBuffer buffer = ByteBuffer.allocate(4 + 4 + bytes.length);
- buffer.putInt(MessagingService.PROTOCOL_MAGIC);
- buffer.putInt(header);
- buffer.put(bytes);
- buffer.flip();
- return buffer;
+ return StreamResultFuture.createFollower(sessionIndex, planId, streamOperation, from, channel, pendingRepair, previewKind)
+ .getSession(from, sessionIndex);
}
- private static class StreamInitMessageSerializer implements IVersionedSerializer<StreamInitMessage>
+ @Override
+ public String toString()
{
- public void serialize(StreamInitMessage message, DataOutputPlus out, int version) throws IOException
+ StringBuilder sb = new StringBuilder(128);
+ sb.append("StreamInitMessage: from = ").append(from);
+ sb.append(", planId = ").append(planId).append(", session index = ").append(sessionIndex);
+ return sb.toString();
+ }
+
+ private static class StreamInitMessageSerializer implements Serializer<StreamInitMessage>
+ {
+ public void serialize(StreamInitMessage message, DataOutputStreamPlus out, int version, StreamSession session) throws IOException
{
- CompactEndpointSerializationHelper.serialize(message.from, out);
+ inetAddressAndPortSerializer.serialize(message.from, out, version);
out.writeInt(message.sessionIndex);
UUIDSerializer.serializer.serialize(message.planId, out, MessagingService.current_version);
- out.writeUTF(message.description);
- out.writeBoolean(message.isForOutgoing);
- out.writeBoolean(message.keepSSTableLevel);
- out.writeBoolean(message.isIncremental);
+ out.writeUTF(message.streamOperation.getDescription());
+
+ out.writeBoolean(message.pendingRepair != null);
+ if (message.pendingRepair != null)
+ {
+ UUIDSerializer.serializer.serialize(message.pendingRepair, out, MessagingService.current_version);
+ }
+ out.writeInt(message.previewKind.getSerializationVal());
}
public StreamInitMessage deserialize(DataInputPlus in, int version) throws IOException
{
- InetAddress from = CompactEndpointSerializationHelper.deserialize(in);
+ InetAddressAndPort from = inetAddressAndPortSerializer.deserialize(in, version);
int sessionIndex = in.readInt();
UUID planId = UUIDSerializer.serializer.deserialize(in, MessagingService.current_version);
String description = in.readUTF();
- boolean sentByInitiator = in.readBoolean();
- boolean keepSSTableLevel = in.readBoolean();
- boolean isIncremental = in.readBoolean();
- return new StreamInitMessage(from, sessionIndex, planId, description, sentByInitiator, keepSSTableLevel, isIncremental);
+
+ UUID pendingRepair = in.readBoolean() ? UUIDSerializer.serializer.deserialize(in, version) : null;
+ PreviewKind previewKind = PreviewKind.deserialize(in.readInt());
+ return new StreamInitMessage(from, sessionIndex, planId, StreamOperation.fromString(description),
+ pendingRepair, previewKind);
}
public long serializedSize(StreamInitMessage message, int version)
{
- long size = CompactEndpointSerializationHelper.serializedSize(message.from);
+ long size = inetAddressAndPortSerializer.serializedSize(message.from, version);
size += TypeSizes.sizeof(message.sessionIndex);
size += UUIDSerializer.serializer.serializedSize(message.planId, MessagingService.current_version);
- size += TypeSizes.sizeof(message.description);
- size += TypeSizes.sizeof(message.isForOutgoing);
- size += TypeSizes.sizeof(message.keepSSTableLevel);
- size += TypeSizes.sizeof(message.isIncremental);
+ size += TypeSizes.sizeof(message.streamOperation.getDescription());
+ size += TypeSizes.sizeof(message.pendingRepair != null);
+ if (message.pendingRepair != null)
+ {
+ size += UUIDSerializer.serializer.serializedSize(message.pendingRepair, MessagingService.current_version);
+ }
+ size += TypeSizes.sizeof(message.previewKind.getSerializationVal());
+
return size;
}
}
diff --git a/src/java/org/apache/cassandra/streaming/messages/StreamMessage.java b/src/java/org/apache/cassandra/streaming/messages/StreamMessage.java
index 7487aaf..e2f08fd 100644
--- a/src/java/org/apache/cassandra/streaming/messages/StreamMessage.java
+++ b/src/java/org/apache/cassandra/streaming/messages/StreamMessage.java
@@ -18,13 +18,15 @@
package org.apache.cassandra.streaming.messages;
import java.io.IOException;
-import java.net.SocketException;
-import java.nio.ByteBuffer;
-import java.nio.channels.ReadableByteChannel;
+import io.netty.channel.Channel;
+
+import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputStreamPlus;
import org.apache.cassandra.streaming.StreamSession;
+import static java.lang.Math.max;
+
/**
* StreamMessage is an abstract base class that every messages in streaming protocol inherit.
*
@@ -32,99 +34,91 @@
*/
public abstract class StreamMessage
{
- /** Streaming protocol version */
- public static final int VERSION_20 = 2;
- public static final int VERSION_22 = 3;
- public static final int VERSION_30 = 4;
- public static final int CURRENT_VERSION = VERSION_30;
-
- private transient volatile boolean sent = false;
-
public static void serialize(StreamMessage message, DataOutputStreamPlus out, int version, StreamSession session) throws IOException
{
- ByteBuffer buff = ByteBuffer.allocate(1);
- // message type
- buff.put(message.type.type);
- buff.flip();
- out.write(buff);
+ out.writeByte(message.type.id);
message.type.outSerializer.serialize(message, out, version, session);
}
- public static StreamMessage deserialize(ReadableByteChannel in, int version, StreamSession session) throws IOException
+ public static long serializedSize(StreamMessage message, int version) throws IOException
{
- ByteBuffer buff = ByteBuffer.allocate(1);
- int readBytes = in.read(buff);
- if (readBytes > 0)
- {
- buff.flip();
- Type type = Type.get(buff.get());
- return type.inSerializer.deserialize(in, version, session);
- }
- else if (readBytes == 0)
- {
- // input socket buffer was not filled yet
- return null;
- }
- else
- {
- // possibly socket gets closed
- throw new SocketException("End-of-stream reached");
- }
+ return 1 + message.type.outSerializer.serializedSize(message, version);
}
- public void sent()
+ public static StreamMessage deserialize(DataInputPlus in, int version) throws IOException
{
- sent = true;
- }
-
- public boolean wasSent()
- {
- return sent;
+ Type type = Type.lookupById(in.readByte());
+ return type.inSerializer.deserialize(in, version);
}
/** StreamMessage serializer */
public static interface Serializer<V extends StreamMessage>
{
- V deserialize(ReadableByteChannel in, int version, StreamSession session) throws IOException;
+ V deserialize(DataInputPlus in, int version) throws IOException;
void serialize(V message, DataOutputStreamPlus out, int version, StreamSession session) throws IOException;
+ long serializedSize(V message, int version) throws IOException;
}
/** StreamMessage types */
- public static enum Type
+ public enum Type
{
- PREPARE(1, 5, PrepareMessage.serializer),
- FILE(2, 0, IncomingFileMessage.serializer, OutgoingFileMessage.serializer),
- RECEIVED(3, 4, ReceivedMessage.serializer),
- RETRY(4, 4, RetryMessage.serializer),
- COMPLETE(5, 1, CompleteMessage.serializer),
- SESSION_FAILED(6, 5, SessionFailedMessage.serializer),
- KEEP_ALIVE(7, 5, KeepAliveMessage.serializer);
+ PREPARE_SYN (1, 5, PrepareSynMessage.serializer ),
+ STREAM (2, 0, IncomingStreamMessage.serializer, OutgoingStreamMessage.serializer),
+ RECEIVED (3, 4, ReceivedMessage.serializer ),
+ COMPLETE (5, 1, CompleteMessage.serializer ),
+ SESSION_FAILED (6, 5, SessionFailedMessage.serializer),
+ KEEP_ALIVE (7, 5, KeepAliveMessage.serializer ),
+ PREPARE_SYNACK (8, 5, PrepareSynAckMessage.serializer),
+ PREPARE_ACK (9, 5, PrepareAckMessage.serializer ),
+ STREAM_INIT (10, 5, StreamInitMessage.serializer );
- public static Type get(byte type)
+ private static final Type[] idToTypeMap;
+
+ static
{
- for (Type t : Type.values())
+ Type[] values = values();
+
+ int max = Integer.MIN_VALUE;
+ for (Type t : values)
+ max = max(t.id, max);
+
+ Type[] idMap = new Type[max + 1];
+ for (Type t : values)
{
- if (t.type == type)
- return t;
+ if (idMap[t.id] != null)
+ throw new RuntimeException("Two StreamMessage Types map to the same id: " + t.id);
+ idMap[t.id] = t;
}
- throw new IllegalArgumentException("Unknown type " + type);
+
+ idToTypeMap = idMap;
}
- private final byte type;
+ public static Type lookupById(int id)
+ {
+ if (id < 0 || id >= idToTypeMap.length)
+ throw new IllegalArgumentException("Invalid type id: " + id);
+
+ return idToTypeMap[id];
+ }
+
+ public final int id;
public final int priority;
+
public final Serializer<StreamMessage> inSerializer;
public final Serializer<StreamMessage> outSerializer;
- @SuppressWarnings("unchecked")
- private Type(int type, int priority, Serializer serializer)
+ Type(int id, int priority, Serializer serializer)
{
- this(type, priority, serializer, serializer);
+ this(id, priority, serializer, serializer);
}
@SuppressWarnings("unchecked")
- private Type(int type, int priority, Serializer inSerializer, Serializer outSerializer)
+ Type(int id, int priority, Serializer inSerializer, Serializer outSerializer)
{
- this.type = (byte) type;
+ if (id < 0 || id > Byte.MAX_VALUE)
+ throw new IllegalArgumentException("StreamMessage Type id must be non-negative and less than " + Byte.MAX_VALUE);
+
+ this.id = id;
this.priority = priority;
this.inSerializer = inSerializer;
this.outSerializer = outSerializer;
@@ -145,4 +139,13 @@
{
return type.priority;
}
+
+ /**
+ * Get or create a {@link StreamSession} based on this stream message data: not all stream messages support this,
+ * so the default implementation just throws an exception.
+ */
+ public StreamSession getOrCreateSession(Channel channel)
+ {
+ throw new UnsupportedOperationException("Not supported by streaming messages of type: " + this.getClass());
+ }
}
diff --git a/src/java/org/apache/cassandra/streaming/messages/StreamMessageHeader.java b/src/java/org/apache/cassandra/streaming/messages/StreamMessageHeader.java
new file mode 100644
index 0000000..30afbb8
--- /dev/null
+++ b/src/java/org/apache/cassandra/streaming/messages/StreamMessageHeader.java
@@ -0,0 +1,155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.streaming.messages;
+
+import java.io.IOException;
+import java.util.UUID;
+
+import com.google.common.base.Objects;
+
+import org.apache.cassandra.db.TypeSizes;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.streaming.StreamSession;
+import org.apache.cassandra.utils.UUIDSerializer;
+
+import static org.apache.cassandra.locator.InetAddressAndPort.Serializer.inetAddressAndPortSerializer;
+
+/**
+ * StreamMessageHeader is appended before sending actual data to describe what it's sending.
+ */
+public class StreamMessageHeader
+{
+ public static FileMessageHeaderSerializer serializer = new FileMessageHeaderSerializer();
+
+ public final TableId tableId;
+ public UUID planId;
+ // it tells us if the file was sent by a follower stream session
+ public final boolean sendByFollower;
+ public int sessionIndex;
+ public final int sequenceNumber;
+ public final long repairedAt;
+ public final UUID pendingRepair;
+ public final InetAddressAndPort sender;
+
+ public StreamMessageHeader(TableId tableId,
+ InetAddressAndPort sender,
+ UUID planId,
+ boolean sendByFollower,
+ int sessionIndex,
+ int sequenceNumber,
+ long repairedAt,
+ UUID pendingRepair)
+ {
+ this.tableId = tableId;
+ this.sender = sender;
+ this.planId = planId;
+ this.sendByFollower = sendByFollower;
+ this.sessionIndex = sessionIndex;
+ this.sequenceNumber = sequenceNumber;
+ this.repairedAt = repairedAt;
+ this.pendingRepair = pendingRepair;
+ }
+
+ @Override
+ public String toString()
+ {
+ final StringBuilder sb = new StringBuilder("Header (");
+ sb.append("tableId: ").append(tableId);
+ sb.append(", #").append(sequenceNumber);
+ sb.append(", repairedAt: ").append(repairedAt);
+ sb.append(", pendingRepair: ").append(pendingRepair);
+ sb.append(", sendByFollower: ").append(sendByFollower);
+ sb.append(')');
+ return sb.toString();
+ }
+
+ @Override
+ public boolean equals(Object o)
+ {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ StreamMessageHeader that = (StreamMessageHeader) o;
+ return sendByFollower == that.sendByFollower &&
+ sequenceNumber == that.sequenceNumber &&
+ Objects.equal(tableId, that.tableId);
+ }
+
+ @Override
+ public int hashCode()
+ {
+ return Objects.hashCode(tableId, sendByFollower, sequenceNumber);
+ }
+
+ public void addSessionInfo(StreamSession session)
+ {
+ planId = session.planId();
+ sessionIndex = session.sessionIndex();
+ }
+
+ public static class FileMessageHeaderSerializer
+ {
+ public void serialize(StreamMessageHeader header, DataOutputPlus out, int version) throws IOException
+ {
+ header.tableId.serialize(out);
+ inetAddressAndPortSerializer.serialize(header.sender, out, version);
+ UUIDSerializer.serializer.serialize(header.planId, out, version);
+ out.writeBoolean(header.sendByFollower);
+ out.writeInt(header.sessionIndex);
+ out.writeInt(header.sequenceNumber);
+ out.writeLong(header.repairedAt);
+ out.writeBoolean(header.pendingRepair != null);
+ if (header.pendingRepair != null)
+ {
+ UUIDSerializer.serializer.serialize(header.pendingRepair, out, version);
+ }
+ }
+
+ public StreamMessageHeader deserialize(DataInputPlus in, int version) throws IOException
+ {
+ TableId tableId = TableId.deserialize(in);
+ InetAddressAndPort sender = inetAddressAndPortSerializer.deserialize(in, version);
+ UUID planId = UUIDSerializer.serializer.deserialize(in, MessagingService.current_version);
+ boolean sendByFollower = in.readBoolean();
+ int sessionIndex = in.readInt();
+ int sequenceNumber = in.readInt();
+ long repairedAt = in.readLong();
+ UUID pendingRepair = in.readBoolean() ? UUIDSerializer.serializer.deserialize(in, version) : null;
+
+ return new StreamMessageHeader(tableId, sender, planId, sendByFollower, sessionIndex, sequenceNumber, repairedAt, pendingRepair);
+ }
+
+ public long serializedSize(StreamMessageHeader header, int version)
+ {
+ long size = header.tableId.serializedSize();
+ size += inetAddressAndPortSerializer.serializedSize(header.sender, version);
+ size += UUIDSerializer.serializer.serializedSize(header.planId, version);
+ size += TypeSizes.sizeof(header.sendByFollower);
+ size += TypeSizes.sizeof(header.sessionIndex);
+ size += TypeSizes.sizeof(header.sequenceNumber);
+ size += TypeSizes.sizeof(header.repairedAt);
+ size += TypeSizes.sizeof(header.pendingRepair != null);
+ size += header.pendingRepair != null ? UUIDSerializer.serializer.serializedSize(header.pendingRepair, version) : 0;
+
+ return size;
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/thrift/CassandraServer.java b/src/java/org/apache/cassandra/thrift/CassandraServer.java
deleted file mode 100644
index 444a938..0000000
--- a/src/java/org/apache/cassandra/thrift/CassandraServer.java
+++ /dev/null
@@ -1,2649 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.thrift;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.nio.charset.CharacterCodingException;
-import java.nio.charset.StandardCharsets;
-import java.util.*;
-import java.util.concurrent.Callable;
-import java.util.concurrent.TimeoutException;
-import java.util.zip.DataFormatException;
-import java.util.zip.Inflater;
-
-import com.google.common.base.Joiner;
-import com.google.common.collect.*;
-import com.google.common.primitives.Longs;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.cassandra.auth.Permission;
-import org.apache.cassandra.config.*;
-import org.apache.cassandra.cql3.QueryOptions;
-import org.apache.cassandra.cql3.SuperColumnCompatibility;
-import org.apache.cassandra.cql3.statements.ParsedStatement;
-import org.apache.cassandra.db.*;
-import org.apache.cassandra.db.context.CounterContext;
-import org.apache.cassandra.db.filter.*;
-import org.apache.cassandra.db.marshal.AbstractType;
-import org.apache.cassandra.db.marshal.TimeUUIDType;
-import org.apache.cassandra.db.partitions.*;
-import org.apache.cassandra.db.rows.*;
-import org.apache.cassandra.db.view.View;
-import org.apache.cassandra.dht.*;
-import org.apache.cassandra.dht.Range;
-import org.apache.cassandra.exceptions.*;
-import org.apache.cassandra.io.util.DataOutputBuffer;
-import org.apache.cassandra.locator.DynamicEndpointSnitch;
-import org.apache.cassandra.metrics.ClientMetrics;
-import org.apache.cassandra.scheduler.IRequestScheduler;
-import org.apache.cassandra.schema.KeyspaceMetadata;
-import org.apache.cassandra.serializers.MarshalException;
-import org.apache.cassandra.service.*;
-import org.apache.cassandra.service.pager.QueryPagers;
-import org.apache.cassandra.tracing.Tracing;
-import org.apache.cassandra.utils.*;
-import org.apache.cassandra.utils.btree.BTreeSet;
-import org.apache.thrift.TException;
-
-public class CassandraServer implements Cassandra.Iface
-{
- private static final Logger logger = LoggerFactory.getLogger(CassandraServer.class);
-
- private final static int COUNT_PAGE_SIZE = 1024;
-
- private final static List<ColumnOrSuperColumn> EMPTY_COLUMNS = Collections.emptyList();
-
- /*
- * RequestScheduler to perform the scheduling of incoming requests
- */
- private final IRequestScheduler requestScheduler;
-
- public CassandraServer()
- {
- requestScheduler = DatabaseDescriptor.getRequestScheduler();
- registerMetrics();
- }
-
- public ThriftClientState state()
- {
- return ThriftSessionManager.instance.currentSession();
- }
-
- protected PartitionIterator read(List<SinglePartitionReadCommand> commands, org.apache.cassandra.db.ConsistencyLevel consistency_level, ClientState cState, long queryStartNanoTime)
- throws org.apache.cassandra.exceptions.InvalidRequestException, UnavailableException, TimedOutException
- {
- try
- {
- schedule(DatabaseDescriptor.getReadRpcTimeout());
- try
- {
- return StorageProxy.read(new SinglePartitionReadCommand.Group(commands, DataLimits.NONE), consistency_level, cState, queryStartNanoTime);
- }
- finally
- {
- release();
- }
- }
- catch (RequestExecutionException e)
- {
- throw ThriftConversion.rethrow(e);
- }
- }
-
- public List<ColumnOrSuperColumn> thriftifyColumns(CFMetaData metadata, Iterator<LegacyLayout.LegacyCell> cells)
- {
- ArrayList<ColumnOrSuperColumn> thriftColumns = new ArrayList<>();
- while (cells.hasNext())
- {
- LegacyLayout.LegacyCell cell = cells.next();
- thriftColumns.add(thriftifyColumnWithName(metadata, cell, cell.name.encode(metadata)));
- }
- return thriftColumns;
- }
-
- private ColumnOrSuperColumn thriftifyColumnWithName(CFMetaData metadata, LegacyLayout.LegacyCell cell, ByteBuffer newName)
- {
- if (cell.isCounter())
- return new ColumnOrSuperColumn().setCounter_column(thriftifySubCounter(metadata, cell).setName(newName));
- else
- return new ColumnOrSuperColumn().setColumn(thriftifySubColumn(cell, newName));
- }
-
- private Column thriftifySubColumn(CFMetaData metadata, LegacyLayout.LegacyCell cell)
- {
- return thriftifySubColumn(cell, cell.name.encode(metadata));
- }
-
- private Column thriftifySubColumn(LegacyLayout.LegacyCell cell, ByteBuffer name)
- {
- assert !cell.isCounter();
-
- Column thrift_column = new Column(name).setValue(cell.value).setTimestamp(cell.timestamp);
- if (cell.isExpiring())
- thrift_column.setTtl(cell.ttl);
- return thrift_column;
- }
-
- private List<Column> thriftifyColumnsAsColumns(CFMetaData metadata, Iterator<LegacyLayout.LegacyCell> cells)
- {
- List<Column> thriftColumns = new ArrayList<>();
- while (cells.hasNext())
- thriftColumns.add(thriftifySubColumn(metadata, cells.next()));
- return thriftColumns;
- }
-
- private CounterColumn thriftifySubCounter(CFMetaData metadata, LegacyLayout.LegacyCell cell)
- {
- assert cell.isCounter();
- return new CounterColumn(cell.name.encode(metadata), CounterContext.instance().total(cell.value));
- }
-
- private List<ColumnOrSuperColumn> thriftifySuperColumns(CFMetaData metadata,
- Iterator<LegacyLayout.LegacyCell> cells,
- boolean subcolumnsOnly,
- boolean isCounterCF,
- boolean reversed)
- {
- if (subcolumnsOnly)
- {
- ArrayList<ColumnOrSuperColumn> thriftSuperColumns = new ArrayList<>();
- while (cells.hasNext())
- {
- LegacyLayout.LegacyCell cell = cells.next();
- thriftSuperColumns.add(thriftifyColumnWithName(metadata, cell, cell.name.superColumnSubName()));
- }
- // Generally, cells come reversed if the query is reverse. However, this is not the case within a super column because
- // internally a super column is a map within a row and those are never returned reversed.
- if (reversed)
- Collections.reverse(thriftSuperColumns);
- return thriftSuperColumns;
- }
- else
- {
- if (isCounterCF)
- return thriftifyCounterSuperColumns(metadata, cells, reversed);
- else
- return thriftifySuperColumns(cells, reversed);
- }
- }
-
- private List<ColumnOrSuperColumn> thriftifySuperColumns(Iterator<LegacyLayout.LegacyCell> cells, boolean reversed)
- {
- ArrayList<ColumnOrSuperColumn> thriftSuperColumns = new ArrayList<>();
- SuperColumn current = null;
- while (cells.hasNext())
- {
- LegacyLayout.LegacyCell cell = cells.next();
- ByteBuffer scName = cell.name.superColumnName();
- if (current == null || !scName.equals(current.bufferForName()))
- {
- // Generally, cells come reversed if the query is reverse. However, this is not the case within a super column because
- // internally a super column is a map within a row and those are never returned reversed.
- if (current != null && reversed)
- Collections.reverse(current.columns);
-
- current = new SuperColumn(scName, new ArrayList<>());
- thriftSuperColumns.add(new ColumnOrSuperColumn().setSuper_column(current));
- }
- current.getColumns().add(thriftifySubColumn(cell, cell.name.superColumnSubName()));
- }
-
- if (current != null && reversed)
- Collections.reverse(current.columns);
-
- return thriftSuperColumns;
- }
-
- private List<ColumnOrSuperColumn> thriftifyCounterSuperColumns(CFMetaData metadata, Iterator<LegacyLayout.LegacyCell> cells, boolean reversed)
- {
- ArrayList<ColumnOrSuperColumn> thriftSuperColumns = new ArrayList<>();
- CounterSuperColumn current = null;
- while (cells.hasNext())
- {
- LegacyLayout.LegacyCell cell = cells.next();
- ByteBuffer scName = cell.name.superColumnName();
- if (current == null || !scName.equals(current.bufferForName()))
- {
- // Generally, cells come reversed if the query is reverse. However, this is not the case within a super column because
- // internally a super column is a map within a row and those are never returned reversed.
- if (current != null && reversed)
- Collections.reverse(current.columns);
-
- current = new CounterSuperColumn(scName, new ArrayList<>());
- thriftSuperColumns.add(new ColumnOrSuperColumn().setCounter_super_column(current));
- }
- current.getColumns().add(thriftifySubCounter(metadata, cell).setName(cell.name.superColumnSubName()));
- }
- return thriftSuperColumns;
- }
-
- private List<ColumnOrSuperColumn> thriftifyPartition(RowIterator partition, boolean subcolumnsOnly, boolean reversed, int cellLimit)
- {
- if (partition.isEmpty())
- return EMPTY_COLUMNS;
-
- Iterator<LegacyLayout.LegacyCell> cells = LegacyLayout.fromRowIterator(partition).right;
- List<ColumnOrSuperColumn> result;
- if (partition.metadata().isSuper())
- {
- boolean isCounterCF = partition.metadata().isCounter();
- result = thriftifySuperColumns(partition.metadata(), cells, subcolumnsOnly, isCounterCF, reversed);
- }
- else
- {
- result = thriftifyColumns(partition.metadata(), cells);
- }
-
- // Thrift count cells, but internally we only count them at "row" boundaries, which means that if the limit stops in the middle
- // of an internal row we'll include a few additional cells. So trim it here.
- return result.size() > cellLimit
- ? result.subList(0, cellLimit)
- : result;
- }
-
- private Map<ByteBuffer, List<ColumnOrSuperColumn>> getSlice(List<SinglePartitionReadCommand> commands, boolean subColumnsOnly, int cellLimit, org.apache.cassandra.db.ConsistencyLevel consistency_level, ClientState cState, long queryStartNanoTime)
- throws org.apache.cassandra.exceptions.InvalidRequestException, UnavailableException, TimedOutException
- {
- try (PartitionIterator results = read(commands, consistency_level, cState, queryStartNanoTime))
- {
- Map<ByteBuffer, List<ColumnOrSuperColumn>> columnFamiliesMap = new HashMap<>();
- while (results.hasNext())
- {
- try (RowIterator iter = results.next())
- {
- List<ColumnOrSuperColumn> thriftifiedColumns = thriftifyPartition(iter, subColumnsOnly, iter.isReverseOrder(), cellLimit);
- columnFamiliesMap.put(iter.partitionKey().getKey(), thriftifiedColumns);
- }
- }
- return columnFamiliesMap;
- }
- }
-
- public List<ColumnOrSuperColumn> get_slice(ByteBuffer key, ColumnParent column_parent, SlicePredicate predicate, ConsistencyLevel consistency_level)
- throws InvalidRequestException, UnavailableException, TimedOutException
- {
- long queryStartNanoTime = System.nanoTime();
- if (startSessionIfRequested())
- {
- Map<String, String> traceParameters = ImmutableMap.of("key", ByteBufferUtil.bytesToHex(key),
- "column_parent", column_parent.toString(),
- "predicate", predicate.toString(),
- "consistency_level", consistency_level.name());
- Tracing.instance.begin("get_slice", traceParameters);
- }
- else
- {
- logger.trace("get_slice");
- }
-
- try
- {
- ClientState cState = state();
- String keyspace = cState.getKeyspace();
- state().hasColumnFamilyAccess(keyspace, column_parent.column_family, Permission.SELECT);
- List<ColumnOrSuperColumn> result = getSliceInternal(keyspace, key, column_parent, FBUtilities.nowInSeconds(), predicate, consistency_level, cState, queryStartNanoTime);
- return result == null ? Collections.<ColumnOrSuperColumn>emptyList() : result;
- }
- catch (RequestValidationException e)
- {
- throw ThriftConversion.toThrift(e);
- }
- finally
- {
- Tracing.instance.stopSession();
- }
- }
-
- private List<ColumnOrSuperColumn> getSliceInternal(String keyspace,
- ByteBuffer key,
- ColumnParent column_parent,
- int nowInSec,
- SlicePredicate predicate,
- ConsistencyLevel consistency_level,
- ClientState cState,
- long queryStartNanoTime)
- throws org.apache.cassandra.exceptions.InvalidRequestException, UnavailableException, TimedOutException
- {
- return multigetSliceInternal(keyspace, Collections.singletonList(key), column_parent, nowInSec, predicate, consistency_level, cState, queryStartNanoTime).get(key);
- }
-
- public Map<ByteBuffer, List<ColumnOrSuperColumn>> multiget_slice(List<ByteBuffer> keys, ColumnParent column_parent, SlicePredicate predicate, ConsistencyLevel consistency_level)
- throws InvalidRequestException, UnavailableException, TimedOutException
- {
- long queryStartNanoTime = System.nanoTime();
- if (startSessionIfRequested())
- {
- List<String> keysList = Lists.newArrayList();
- for (ByteBuffer key : keys)
- keysList.add(ByteBufferUtil.bytesToHex(key));
- Map<String, String> traceParameters = ImmutableMap.of("keys", keysList.toString(),
- "column_parent", column_parent.toString(),
- "predicate", predicate.toString(),
- "consistency_level", consistency_level.name());
- Tracing.instance.begin("multiget_slice", traceParameters);
- }
- else
- {
- logger.trace("multiget_slice");
- }
-
- try
- {
- ClientState cState = state();
- String keyspace = cState.getKeyspace();
- cState.hasColumnFamilyAccess(keyspace, column_parent.column_family, Permission.SELECT);
- return multigetSliceInternal(keyspace, keys, column_parent, FBUtilities.nowInSeconds(), predicate, consistency_level, cState, queryStartNanoTime);
- }
- catch (RequestValidationException e)
- {
- throw ThriftConversion.toThrift(e);
- }
- finally
- {
- Tracing.instance.stopSession();
- }
- }
-
- private ClusteringIndexFilter toInternalFilter(CFMetaData metadata, ColumnParent parent, SliceRange range)
- {
- if (metadata.isSuper() && parent.isSetSuper_column())
- return new ClusteringIndexNamesFilter(FBUtilities.singleton(Clustering.make(parent.bufferForSuper_column()), metadata.comparator), range.reversed);
- else
- return new ClusteringIndexSliceFilter(makeSlices(metadata, range), range.reversed);
- }
-
- private Slices makeSlices(CFMetaData metadata, SliceRange range)
- {
- // Note that in thrift, the bounds are reversed if the query is reversed, but not internally.
- ByteBuffer start = range.reversed ? range.finish : range.start;
- ByteBuffer finish = range.reversed ? range.start : range.finish;
- return Slices.with(metadata.comparator, Slice.make(LegacyLayout.decodeSliceBound(metadata, start, true).bound, LegacyLayout.decodeSliceBound(metadata, finish, false).bound));
- }
-
- private ClusteringIndexFilter toInternalFilter(CFMetaData metadata, ColumnParent parent, SlicePredicate predicate)
- throws org.apache.cassandra.exceptions.InvalidRequestException
- {
- try
- {
- if (predicate.column_names != null)
- {
- if (metadata.isSuper())
- {
- if (parent.isSetSuper_column())
- {
- return new ClusteringIndexNamesFilter(FBUtilities.singleton(Clustering.make(parent.bufferForSuper_column()), metadata.comparator), false);
- }
- else
- {
- NavigableSet<Clustering> clusterings = new TreeSet<>(metadata.comparator);
- for (ByteBuffer bb : predicate.column_names)
- clusterings.add(Clustering.make(bb));
- return new ClusteringIndexNamesFilter(clusterings, false);
- }
- }
- else
- {
- NavigableSet<Clustering> clusterings = new TreeSet<>(metadata.comparator);
- for (ByteBuffer bb : predicate.column_names)
- {
- LegacyLayout.LegacyCellName name = LegacyLayout.decodeCellName(metadata, parent.bufferForSuper_column(), bb);
-
- if (!name.clustering.equals(Clustering.STATIC_CLUSTERING))
- clusterings.add(name.clustering);
- }
-
- // clusterings cannot include STATIC_CLUSTERING, so if the names filter is for static columns, clusterings
- // will be empty. However, by requesting the static columns in our ColumnFilter, this will still work.
- return new ClusteringIndexNamesFilter(clusterings, false);
- }
- }
- else
- {
- return toInternalFilter(metadata, parent, predicate.slice_range);
- }
- }
- catch (UnknownColumnException e)
- {
- throw new org.apache.cassandra.exceptions.InvalidRequestException(e.getMessage());
- }
- }
-
- private ColumnFilter makeColumnFilter(CFMetaData metadata, ColumnParent parent, SliceRange range)
- {
- if (metadata.isSuper() && parent.isSetSuper_column())
- {
- // We want a slice of the dynamic columns
- ColumnFilter.Builder builder = ColumnFilter.selectionBuilder();
- ColumnDefinition def = metadata.compactValueColumn();
- ByteBuffer start = range.reversed ? range.finish : range.start;
- ByteBuffer finish = range.reversed ? range.start : range.finish;
- builder.slice(def, start.hasRemaining() ? CellPath.create(start) : CellPath.BOTTOM, finish.hasRemaining() ? CellPath.create(finish) : CellPath.TOP);
-
- if (metadata.isDense())
- return builder.build();
-
- // We also want to add any staticly defined column if it's within the range
- AbstractType<?> cmp = metadata.thriftColumnNameType();
-
- for (ColumnDefinition column : metadata.partitionColumns())
- {
- if (SuperColumnCompatibility.isSuperColumnMapColumn(column))
- continue;
-
- ByteBuffer name = column.name.bytes;
- if (cmp.compare(name, start) < 0 || cmp.compare(finish, name) > 0)
- continue;
-
- builder.add(column);
- }
- return builder.build();
- }
- return makeColumnFilter(metadata, makeSlices(metadata, range));
- }
-
- private ColumnFilter makeColumnFilter(CFMetaData metadata, Slices slices)
- {
- PartitionColumns columns = metadata.partitionColumns();
- if (metadata.isStaticCompactTable() && !columns.statics.isEmpty())
- {
- PartitionColumns.Builder builder = PartitionColumns.builder();
- builder.addAll(columns.regulars);
- // We only want to include the static columns that are selected by the slices
- for (ColumnDefinition def : columns.statics)
- {
- if (slices.selects(Clustering.make(def.name.bytes)))
- builder.add(def);
- }
- columns = builder.build();
- }
- return ColumnFilter.selection(columns);
- }
-
- private ColumnFilter makeColumnFilter(CFMetaData metadata, ColumnParent parent, SlicePredicate predicate)
- throws org.apache.cassandra.exceptions.InvalidRequestException
- {
- try
- {
- if (predicate.column_names != null)
- {
- if (metadata.isSuper())
- {
- if (parent.isSetSuper_column())
- {
- ColumnFilter.Builder builder = ColumnFilter.selectionBuilder();
- ColumnDefinition dynamicDef = metadata.compactValueColumn();
- for (ByteBuffer bb : predicate.column_names)
- {
- ColumnDefinition staticDef = metadata.getColumnDefinition(bb);
- if (staticDef == null)
- builder.select(dynamicDef, CellPath.create(bb));
- else
- builder.add(staticDef);
- }
- return builder.build();
- }
- else
- {
- return ColumnFilter.all(metadata);
- }
- }
- else
- {
- PartitionColumns.Builder builder = PartitionColumns.builder();
- for (ByteBuffer bb : predicate.column_names)
- {
- LegacyLayout.LegacyCellName name = LegacyLayout.decodeCellName(metadata, parent.bufferForSuper_column(), bb);
- builder.add(name.column);
- }
-
- if (metadata.isStaticCompactTable())
- builder.add(metadata.compactValueColumn());
-
- return ColumnFilter.selection(builder.build());
- }
- }
- else
- {
- return makeColumnFilter(metadata, parent, predicate.slice_range);
- }
- }
- catch (UnknownColumnException e)
- {
- throw new org.apache.cassandra.exceptions.InvalidRequestException(e.getMessage());
- }
- }
-
- private DataLimits getLimits(int partitionLimit, boolean countSuperColumns, SlicePredicate predicate)
- {
- int cellsPerPartition = predicate.slice_range == null ? Integer.MAX_VALUE : predicate.slice_range.count;
- return getLimits(partitionLimit, countSuperColumns, cellsPerPartition);
- }
-
- private DataLimits getLimits(int partitionLimit, boolean countSuperColumns, int perPartitionCount)
- {
- return countSuperColumns
- ? DataLimits.superColumnCountingLimits(partitionLimit, perPartitionCount)
- : DataLimits.thriftLimits(partitionLimit, perPartitionCount);
- }
-
- private Map<ByteBuffer, List<ColumnOrSuperColumn>> multigetSliceInternal(String keyspace,
- List<ByteBuffer> keys,
- ColumnParent column_parent,
- int nowInSec,
- SlicePredicate predicate,
- ConsistencyLevel consistency_level,
- ClientState cState,
- long queryStartNanoTime)
- throws org.apache.cassandra.exceptions.InvalidRequestException, UnavailableException, TimedOutException
- {
- CFMetaData metadata = ThriftValidation.validateColumnFamily(keyspace, column_parent.column_family);
- ThriftValidation.validateColumnParent(metadata, column_parent);
- ThriftValidation.validatePredicate(metadata, column_parent, predicate);
-
- org.apache.cassandra.db.ConsistencyLevel consistencyLevel = ThriftConversion.fromThrift(consistency_level);
- consistencyLevel.validateForRead(keyspace);
-
- List<SinglePartitionReadCommand> commands = new ArrayList<>(keys.size());
- ColumnFilter columnFilter = makeColumnFilter(metadata, column_parent, predicate);
- ClusteringIndexFilter filter = toInternalFilter(metadata, column_parent, predicate);
- DataLimits limits = getLimits(1, metadata.isSuper() && !column_parent.isSetSuper_column(), predicate);
-
- for (ByteBuffer key: keys)
- {
- ThriftValidation.validateKey(metadata, key);
- DecoratedKey dk = metadata.decorateKey(key);
- commands.add(SinglePartitionReadCommand.create(true, metadata, nowInSec, columnFilter, RowFilter.NONE, limits, dk, filter));
- }
-
- return getSlice(commands, column_parent.isSetSuper_column(), limits.perPartitionCount(), consistencyLevel, cState, queryStartNanoTime);
- }
-
- public ColumnOrSuperColumn get(ByteBuffer key, ColumnPath column_path, ConsistencyLevel consistency_level)
- throws InvalidRequestException, NotFoundException, UnavailableException, TimedOutException
- {
- long queryStartNanoTime = System.nanoTime();
- if (startSessionIfRequested())
- {
- Map<String, String> traceParameters = ImmutableMap.of("key", ByteBufferUtil.bytesToHex(key),
- "column_path", column_path.toString(),
- "consistency_level", consistency_level.name());
- Tracing.instance.begin("get", traceParameters);
- }
- else
- {
- logger.trace("get");
- }
-
- try
- {
- ThriftClientState cState = state();
- String keyspace = cState.getKeyspace();
- cState.hasColumnFamilyAccess(keyspace, column_path.column_family, Permission.SELECT);
-
- CFMetaData metadata = ThriftValidation.validateColumnFamily(keyspace, column_path.column_family);
- ThriftValidation.validateColumnPath(metadata, column_path);
- org.apache.cassandra.db.ConsistencyLevel consistencyLevel = ThriftConversion.fromThrift(consistency_level);
- consistencyLevel.validateForRead(keyspace);
-
- ThriftValidation.validateKey(metadata, key);
-
- ColumnFilter columns;
- ClusteringIndexFilter filter;
- if (metadata.isSuper())
- {
- if (column_path.column == null)
- {
- // Selects a full super column
- columns = ColumnFilter.all(metadata);
- }
- else
- {
- // Selects a single column within a super column
- ColumnFilter.Builder builder = ColumnFilter.selectionBuilder();
- ColumnDefinition staticDef = metadata.getColumnDefinition(column_path.column);
- ColumnDefinition dynamicDef = metadata.compactValueColumn();
-
- if (staticDef != null)
- builder.add(staticDef);
- // Note that even if there is a staticDef, we still query the dynamicDef since we can't guarantee the static one hasn't
- // been created after data has been inserted for that definition
- builder.select(dynamicDef, CellPath.create(column_path.column));
- columns = builder.build();
- }
- filter = new ClusteringIndexNamesFilter(FBUtilities.singleton(Clustering.make(column_path.super_column), metadata.comparator),
- false);
- }
- else
- {
- LegacyLayout.LegacyCellName cellname = LegacyLayout.decodeCellName(metadata, column_path.super_column, column_path.column);
- if (cellname.clustering == Clustering.STATIC_CLUSTERING)
- {
- // Same as above: even if we're querying a static column, we still query the equivalent dynamic column and value as some
- // values might have been created post creation of the column (ThriftResultMerger then ensures we get only one result).
- ColumnFilter.Builder builder = ColumnFilter.selectionBuilder();
- builder.add(cellname.column);
- builder.add(metadata.compactValueColumn());
- columns = builder.build();
- filter = new ClusteringIndexNamesFilter(FBUtilities.singleton(Clustering.make(column_path.column), metadata.comparator), false);
- }
- else
- {
- columns = ColumnFilter.selection(PartitionColumns.of(cellname.column));
- filter = new ClusteringIndexNamesFilter(FBUtilities.singleton(cellname.clustering, metadata.comparator), false);
- }
- }
-
- DecoratedKey dk = metadata.decorateKey(key);
- SinglePartitionReadCommand command = SinglePartitionReadCommand.create(true, metadata, FBUtilities.nowInSeconds(), columns, RowFilter.NONE, DataLimits.NONE, dk, filter);
-
- try (RowIterator result = PartitionIterators.getOnlyElement(read(Arrays.asList(command), consistencyLevel, cState, queryStartNanoTime), command))
- {
- if (!result.hasNext())
- throw new NotFoundException();
-
- List<ColumnOrSuperColumn> tcolumns = thriftifyPartition(result, metadata.isSuper() && column_path.column != null, result.isReverseOrder(), 1);
- if (tcolumns.isEmpty())
- throw new NotFoundException();
- assert tcolumns.size() == 1;
- return tcolumns.get(0);
- }
- }
- catch (UnknownColumnException e)
- {
- throw new InvalidRequestException(e.getMessage());
- }
- catch (RequestValidationException e)
- {
- throw ThriftConversion.toThrift(e);
- }
- finally
- {
- Tracing.instance.stopSession();
- }
- }
-
- public int get_count(ByteBuffer key, ColumnParent column_parent, SlicePredicate predicate, ConsistencyLevel consistency_level)
- throws InvalidRequestException, UnavailableException, TimedOutException
- {
- long queryStartNanoTime = System.nanoTime();
- if (startSessionIfRequested())
- {
- Map<String, String> traceParameters = ImmutableMap.of("key", ByteBufferUtil.bytesToHex(key),
- "column_parent", column_parent.toString(),
- "predicate", predicate.toString(),
- "consistency_level", consistency_level.name());
- Tracing.instance.begin("get_count", traceParameters);
- }
- else
- {
- logger.trace("get_count");
- }
-
- try
- {
- ThriftClientState cState = state();
- String keyspace = cState.getKeyspace();
- cState.hasColumnFamilyAccess(keyspace, column_parent.column_family, Permission.SELECT);
- Keyspace keyspaceName = Keyspace.open(keyspace);
- ColumnFamilyStore cfs = keyspaceName.getColumnFamilyStore(column_parent.column_family);
- int nowInSec = FBUtilities.nowInSeconds();
-
- if (predicate.column_names != null)
- return getSliceInternal(keyspace, key, column_parent, nowInSec, predicate, consistency_level, cState, queryStartNanoTime).size();
-
- int pageSize;
- // request by page if this is a large row
- if (cfs.getMeanColumns() > 0)
- {
- int averageColumnSize = (int) (cfs.metric.meanPartitionSize.getValue() / cfs.getMeanColumns());
- pageSize = Math.min(COUNT_PAGE_SIZE, 4 * 1024 * 1024 / averageColumnSize);
- pageSize = Math.max(2, pageSize);
- logger.trace("average row column size is {}; using pageSize of {}", averageColumnSize, pageSize);
- }
- else
- {
- pageSize = COUNT_PAGE_SIZE;
- }
-
- SliceRange sliceRange = predicate.slice_range == null
- ? new SliceRange(ByteBufferUtil.EMPTY_BYTE_BUFFER, ByteBufferUtil.EMPTY_BYTE_BUFFER, false, Integer.MAX_VALUE)
- : predicate.slice_range;
-
- ColumnFilter columnFilter;
- ClusteringIndexFilter filter;
- CFMetaData metadata = cfs.metadata;
- if (metadata.isSuper() && !column_parent.isSetSuper_column())
- {
- // If we count on a super column table without having set the super column name, we're in fact interested by the count of super columns
- columnFilter = ColumnFilter.all(metadata);
- filter = new ClusteringIndexSliceFilter(makeSlices(metadata, sliceRange), sliceRange.reversed);
- }
- else
- {
- columnFilter = makeColumnFilter(metadata, column_parent, sliceRange);
- filter = toInternalFilter(metadata, column_parent, sliceRange);
- }
-
- DataLimits limits = getLimits(1, metadata.isSuper() && !column_parent.isSetSuper_column(), predicate);
- DecoratedKey dk = metadata.decorateKey(key);
-
- return QueryPagers.countPaged(metadata,
- dk,
- columnFilter,
- filter,
- limits,
- ThriftConversion.fromThrift(consistency_level),
- cState,
- pageSize,
- nowInSec,
- true,
- queryStartNanoTime);
- }
- catch (IllegalArgumentException e)
- {
- // CASSANDRA-5701
- throw new InvalidRequestException(e.getMessage());
- }
- catch (RequestExecutionException e)
- {
- throw ThriftConversion.rethrow(e);
- }
- catch (RequestValidationException e)
- {
- throw ThriftConversion.toThrift(e);
- }
- finally
- {
- Tracing.instance.stopSession();
- }
- }
-
- public Map<ByteBuffer, Integer> multiget_count(List<ByteBuffer> keys, ColumnParent column_parent, SlicePredicate predicate, ConsistencyLevel consistency_level)
- throws InvalidRequestException, UnavailableException, TimedOutException
- {
- long queryStartNanoTime = System.nanoTime();
- if (startSessionIfRequested())
- {
- List<String> keysList = Lists.newArrayList();
- for (ByteBuffer key : keys)
- {
- keysList.add(ByteBufferUtil.bytesToHex(key));
- }
- Map<String, String> traceParameters = ImmutableMap.of("keys", keysList.toString(),
- "column_parent", column_parent.toString(),
- "predicate", predicate.toString(),
- "consistency_level", consistency_level.name());
- Tracing.instance.begin("multiget_count", traceParameters);
- }
- else
- {
- logger.trace("multiget_count");
- }
-
- try
- {
- ThriftClientState cState = state();
- String keyspace = cState.getKeyspace();
- cState.hasColumnFamilyAccess(keyspace, column_parent.column_family, Permission.SELECT);
-
- Map<ByteBuffer, Integer> counts = new HashMap<>();
- Map<ByteBuffer, List<ColumnOrSuperColumn>> columnFamiliesMap = multigetSliceInternal(keyspace,
- keys,
- column_parent,
- FBUtilities.nowInSeconds(),
- predicate,
- consistency_level,
- cState,
- queryStartNanoTime);
-
- for (Map.Entry<ByteBuffer, List<ColumnOrSuperColumn>> cf : columnFamiliesMap.entrySet())
- counts.put(cf.getKey(), cf.getValue().size());
- return counts;
- }
- catch (RequestValidationException e)
- {
- throw ThriftConversion.toThrift(e);
- }
- finally
- {
- Tracing.instance.stopSession();
- }
- }
-
- private Cell cellFromColumn(CFMetaData metadata, LegacyLayout.LegacyCellName name, Column column)
- {
- CellPath path = name.collectionElement == null ? null : CellPath.create(name.collectionElement);
- int ttl = getTtl(metadata, column);
- return ttl == LivenessInfo.NO_TTL
- ? BufferCell.live(name.column, column.timestamp, column.value, path)
- : BufferCell.expiring(name.column, column.timestamp, ttl, FBUtilities.nowInSeconds(), column.value, path);
- }
-
- private int getTtl(CFMetaData metadata,Column column)
- {
- if (!column.isSetTtl())
- return metadata.params.defaultTimeToLive;
-
- if (column.ttl == LivenessInfo.NO_TTL && metadata.params.defaultTimeToLive != LivenessInfo.NO_TTL)
- return LivenessInfo.NO_TTL;
-
- return column.ttl;
- }
-
- private void internal_insert(ByteBuffer key, ColumnParent column_parent, Column column, ConsistencyLevel consistency_level, long queryStartNanoTime)
- throws RequestValidationException, UnavailableException, TimedOutException
- {
- ThriftClientState cState = state();
- String keyspace = cState.getKeyspace();
- cState.hasColumnFamilyAccess(keyspace, column_parent.column_family, Permission.MODIFY);
-
- CFMetaData metadata = ThriftValidation.validateColumnFamily(keyspace, column_parent.column_family, false);
- if (metadata.isView())
- throw new org.apache.cassandra.exceptions.InvalidRequestException("Cannot modify Materialized Views directly");
-
- ThriftValidation.validateKey(metadata, key);
- ThriftValidation.validateColumnParent(metadata, column_parent);
- // SuperColumn field is usually optional, but not when we're inserting
- if (metadata.isSuper() && column_parent.super_column == null)
- {
- throw new org.apache.cassandra.exceptions.InvalidRequestException("missing mandatory super column name for super CF " + column_parent.column_family);
- }
- ThriftValidation.validateColumnNames(metadata, column_parent, Collections.singletonList(column.name));
- ThriftValidation.validateColumnData(metadata, column_parent.super_column, column);
-
- org.apache.cassandra.db.Mutation mutation;
- try
- {
- LegacyLayout.LegacyCellName name = LegacyLayout.decodeCellName(metadata, column_parent.super_column, column.name);
- Cell cell = cellFromColumn(metadata, name, column);
- PartitionUpdate update = PartitionUpdate.singleRowUpdate(metadata, key, BTreeRow.singleCellRow(name.clustering, cell));
-
- // Indexed column values cannot be larger than 64K. See CASSANDRA-3057/4240 for more details
- Keyspace.open(metadata.ksName).getColumnFamilyStore(metadata.cfName).indexManager.validate(update);
-
- mutation = new org.apache.cassandra.db.Mutation(update);
- }
- catch (MarshalException|UnknownColumnException e)
- {
- throw new org.apache.cassandra.exceptions.InvalidRequestException(e.getMessage());
- }
- doInsert(consistency_level, Collections.singletonList(mutation), queryStartNanoTime);
- }
-
- public void insert(ByteBuffer key, ColumnParent column_parent, Column column, ConsistencyLevel consistency_level)
- throws InvalidRequestException, UnavailableException, TimedOutException
- {
- long queryStartNanoTime = System.nanoTime();
- if (startSessionIfRequested())
- {
- Map<String, String> traceParameters = ImmutableMap.of("key", ByteBufferUtil.bytesToHex(key),
- "column_parent", column_parent.toString(),
- "column", column.toString(),
- "consistency_level", consistency_level.name());
- Tracing.instance.begin("insert", traceParameters);
- }
- else
- {
- logger.trace("insert");
- }
-
- try
- {
- internal_insert(key, column_parent, column, consistency_level, queryStartNanoTime);
- }
- catch (RequestValidationException e)
- {
- throw ThriftConversion.toThrift(e);
- }
- finally
- {
- Tracing.instance.stopSession();
- }
- }
-
- public CASResult cas(ByteBuffer key,
- String column_family,
- List<Column> expected,
- List<Column> updates,
- ConsistencyLevel serial_consistency_level,
- ConsistencyLevel commit_consistency_level)
- throws InvalidRequestException, UnavailableException, TimedOutException
- {
- long queryStartNanoTime = System.nanoTime();
- if (startSessionIfRequested())
- {
- ImmutableMap.Builder<String,String> builder = ImmutableMap.builder();
- builder.put("key", ByteBufferUtil.bytesToHex(key));
- builder.put("column_family", column_family);
- builder.put("old", expected.toString());
- builder.put("updates", updates.toString());
- builder.put("consistency_level", commit_consistency_level.name());
- builder.put("serial_consistency_level", serial_consistency_level.name());
- Map<String,String> traceParameters = builder.build();
-
- Tracing.instance.begin("cas", traceParameters);
- }
- else
- {
- logger.trace("cas");
- }
-
- try
- {
- ThriftClientState cState = state();
- String keyspace = cState.getKeyspace();
- cState.hasColumnFamilyAccess(keyspace, column_family, Permission.MODIFY);
- // CAS updates can be used to simulate a get request, so should require Permission.SELECT.
- cState.hasColumnFamilyAccess(keyspace, column_family, Permission.SELECT);
-
- CFMetaData metadata = ThriftValidation.validateColumnFamily(keyspace, column_family, false);
- if (metadata.isView())
- throw new org.apache.cassandra.exceptions.InvalidRequestException("Cannot modify Materialized Views directly");
-
- ThriftValidation.validateKey(metadata, key);
- if (metadata.isSuper())
- throw new org.apache.cassandra.exceptions.InvalidRequestException("CAS does not support supercolumns");
-
- Iterable<ByteBuffer> names = Iterables.transform(updates, column -> column.name);
- ThriftValidation.validateColumnNames(metadata, new ColumnParent(column_family), names);
- for (Column column : updates)
- ThriftValidation.validateColumnData(metadata, null, column);
-
- DecoratedKey dk = metadata.decorateKey(key);
- int nowInSec = FBUtilities.nowInSeconds();
-
- PartitionUpdate partitionUpdates = PartitionUpdate.fromIterator(LegacyLayout.toRowIterator(metadata, dk, toLegacyCells(metadata, updates, nowInSec).iterator(), nowInSec), ColumnFilter.all(metadata));
- // Indexed column values cannot be larger than 64K. See CASSANDRA-3057/4240 for more details
- Keyspace.open(metadata.ksName).getColumnFamilyStore(metadata.cfName).indexManager.validate(partitionUpdates);
-
- schedule(DatabaseDescriptor.getWriteRpcTimeout());
- try (RowIterator result = StorageProxy.cas(cState.getKeyspace(),
- column_family,
- dk,
- new ThriftCASRequest(toLegacyCells(metadata, expected, nowInSec), partitionUpdates, nowInSec),
- ThriftConversion.fromThrift(serial_consistency_level),
- ThriftConversion.fromThrift(commit_consistency_level),
- cState,
- queryStartNanoTime))
- {
- return result == null
- ? new CASResult(true)
- : new CASResult(false).setCurrent_values(thriftifyColumnsAsColumns(metadata, LegacyLayout.fromRowIterator(result).right));
- }
- }
- catch (UnknownColumnException e)
- {
- throw new InvalidRequestException(e.getMessage());
- }
- catch (RequestTimeoutException e)
- {
- throw ThriftConversion.toThrift(e);
- }
- catch (RequestValidationException e)
- {
- throw ThriftConversion.toThrift(e);
- }
- catch (RequestExecutionException e)
- {
- throw ThriftConversion.rethrow(e);
- }
- finally
- {
- Tracing.instance.stopSession();
- }
- }
-
- private LegacyLayout.LegacyCell toLegacyCell(CFMetaData metadata, Column column, int nowInSec) throws UnknownColumnException
- {
- return toLegacyCell(metadata, null, column, nowInSec);
- }
-
- private LegacyLayout.LegacyCell toLegacyCell(CFMetaData metadata, ByteBuffer superColumnName, Column column, int nowInSec)
- throws UnknownColumnException
- {
- return column.ttl > 0
- ? LegacyLayout.LegacyCell.expiring(metadata, superColumnName, column.name, column.value, column.timestamp, column.ttl, nowInSec)
- : LegacyLayout.LegacyCell.regular(metadata, superColumnName, column.name, column.value, column.timestamp);
- }
-
- private LegacyLayout.LegacyCell toLegacyDeletion(CFMetaData metadata, ByteBuffer name, long timestamp, int nowInSec)
- throws UnknownColumnException
- {
- return toLegacyDeletion(metadata, null, name, timestamp, nowInSec);
- }
-
- private LegacyLayout.LegacyCell toLegacyDeletion(CFMetaData metadata, ByteBuffer superColumnName, ByteBuffer name, long timestamp, int nowInSec)
- throws UnknownColumnException
- {
- return LegacyLayout.LegacyCell.tombstone(metadata, superColumnName, name, timestamp, nowInSec);
- }
-
- private LegacyLayout.LegacyCell toCounterLegacyCell(CFMetaData metadata, CounterColumn column)
- throws UnknownColumnException
- {
- return toCounterLegacyCell(metadata, null, column);
- }
-
- private LegacyLayout.LegacyCell toCounterLegacyCell(CFMetaData metadata, ByteBuffer superColumnName, CounterColumn column)
- throws UnknownColumnException
- {
- return LegacyLayout.LegacyCell.counterUpdate(metadata, superColumnName, column.name, column.value);
- }
-
- private void sortAndMerge(CFMetaData metadata, List<LegacyLayout.LegacyCell> cells, int nowInSec)
- {
- Collections.sort(cells, LegacyLayout.legacyCellComparator(metadata));
-
- // After sorting, if we have multiple cells for the same "cellname", we want to merge those together.
- Comparator<LegacyLayout.LegacyCellName> comparator = LegacyLayout.legacyCellNameComparator(metadata, false);
-
- int previous = 0; // The last element that was set
- for (int current = 1; current < cells.size(); current++)
- {
- LegacyLayout.LegacyCell pc = cells.get(previous);
- LegacyLayout.LegacyCell cc = cells.get(current);
-
- // There is really only 2 possible comparison: < 0 or == 0 since we've sorted already
- int cmp = comparator.compare(pc.name, cc.name);
- if (cmp == 0)
- {
- // current and previous are the same cell. Merge current into previous
- // (and so previous + 1 will be "free").
- Conflicts.Resolution res;
- if (metadata.isCounter())
- {
- res = Conflicts.resolveCounter(pc.timestamp, pc.isLive(nowInSec), pc.value,
- cc.timestamp, cc.isLive(nowInSec), cc.value);
-
- }
- else
- {
- res = Conflicts.resolveRegular(pc.timestamp, pc.isLive(nowInSec), pc.localDeletionTime, pc.value,
- cc.timestamp, cc.isLive(nowInSec), cc.localDeletionTime, cc.value);
- }
-
- switch (res)
- {
- case LEFT_WINS:
- // The previous cell wins, we'll just ignore current
- break;
- case RIGHT_WINS:
- cells.set(previous, cc);
- break;
- case MERGE:
- assert metadata.isCounter();
- ByteBuffer merged = Conflicts.mergeCounterValues(pc.value, cc.value);
- cells.set(previous, LegacyLayout.LegacyCell.counter(pc.name, merged));
- break;
- }
- }
- else
- {
- // cell.get(previous) < cells.get(current), so move current just after previous if needs be
- ++previous;
- if (previous != current)
- cells.set(previous, cc);
- }
- }
-
- // The last element we want is previous, so trim anything after that
- for (int i = cells.size() - 1; i > previous; i--)
- cells.remove(i);
- }
-
- private List<LegacyLayout.LegacyCell> toLegacyCells(CFMetaData metadata, List<Column> columns, int nowInSec)
- throws UnknownColumnException
- {
- List<LegacyLayout.LegacyCell> cells = new ArrayList<>(columns.size());
- for (Column column : columns)
- cells.add(toLegacyCell(metadata, column, nowInSec));
-
- sortAndMerge(metadata, cells, nowInSec);
- return cells;
- }
-
- private List<IMutation> createMutationList(ConsistencyLevel consistency_level,
- Map<ByteBuffer,Map<String,List<Mutation>>> mutation_map,
- boolean allowCounterMutations)
- throws RequestValidationException, InvalidRequestException
- {
- List<IMutation> mutations = new ArrayList<>();
- ThriftClientState cState = state();
- String keyspace = cState.getKeyspace();
- int nowInSec = FBUtilities.nowInSeconds();
-
- for (Map.Entry<ByteBuffer, Map<String, List<Mutation>>> mutationEntry: mutation_map.entrySet())
- {
- ByteBuffer key = mutationEntry.getKey();
-
- // We need to separate mutation for standard cf and counter cf (that will be encapsulated in a
- // CounterMutation) because it doesn't follow the same code path
- org.apache.cassandra.db.Mutation standardMutation = null;
- org.apache.cassandra.db.Mutation counterMutation = null;
-
- Map<String, List<Mutation>> columnFamilyToMutations = mutationEntry.getValue();
- for (Map.Entry<String, List<Mutation>> columnFamilyMutations : columnFamilyToMutations.entrySet())
- {
- String cfName = columnFamilyMutations.getKey();
- List<Mutation> muts = columnFamilyMutations.getValue();
-
- cState.hasColumnFamilyAccess(keyspace, cfName, Permission.MODIFY);
-
- CFMetaData metadata = ThriftValidation.validateColumnFamily(keyspace, cfName);
- if (metadata.isView())
- throw new org.apache.cassandra.exceptions.InvalidRequestException("Cannot modify Materialized Views directly");
-
- ThriftValidation.validateKey(metadata, key);
- if (metadata.isCounter())
- ThriftConversion.fromThrift(consistency_level).validateCounterForWrite(metadata);
-
- LegacyLayout.LegacyDeletionInfo delInfo = LegacyLayout.LegacyDeletionInfo.live();
- List<LegacyLayout.LegacyCell> cells = new ArrayList<>();
- for (Mutation m : muts)
- {
- ThriftValidation.validateMutation(metadata, m);
-
- if (m.deletion != null)
- {
- deleteColumnOrSuperColumn(delInfo, cells, metadata, m.deletion, nowInSec);
- }
- if (m.column_or_supercolumn != null)
- {
- addColumnOrSuperColumn(cells, metadata, m.column_or_supercolumn, nowInSec);
- }
- }
-
- sortAndMerge(metadata, cells, nowInSec);
- DecoratedKey dk = metadata.decorateKey(key);
- PartitionUpdate update = PartitionUpdate.fromIterator(LegacyLayout.toUnfilteredRowIterator(metadata, dk, delInfo, cells.iterator()), ColumnFilter.all(metadata));
-
- // Indexed column values cannot be larger than 64K. See CASSANDRA-3057/4240 for more details
- Keyspace.open(metadata.ksName).getColumnFamilyStore(metadata.cfName).indexManager.validate(update);
-
- org.apache.cassandra.db.Mutation mutation;
- if (metadata.isCounter())
- {
- counterMutation = counterMutation == null ? new org.apache.cassandra.db.Mutation(keyspace, dk) : counterMutation;
- mutation = counterMutation;
- }
- else
- {
- standardMutation = standardMutation == null ? new org.apache.cassandra.db.Mutation(keyspace, dk) : standardMutation;
- mutation = standardMutation;
- }
- mutation.add(update);
- }
- if (standardMutation != null && !standardMutation.isEmpty())
- mutations.add(standardMutation);
-
- if (counterMutation != null && !counterMutation.isEmpty())
- {
- if (allowCounterMutations)
- mutations.add(new CounterMutation(counterMutation, ThriftConversion.fromThrift(consistency_level)));
- else
- throw new org.apache.cassandra.exceptions.InvalidRequestException("Counter mutations are not allowed in atomic batches");
- }
- }
-
- return mutations;
- }
-
- private void addColumnOrSuperColumn(List<LegacyLayout.LegacyCell> cells, CFMetaData cfm, ColumnOrSuperColumn cosc, int nowInSec)
- throws InvalidRequestException
- {
- try
- {
- if (cosc.super_column != null)
- {
- for (Column column : cosc.super_column.columns)
- cells.add(toLegacyCell(cfm, cosc.super_column.name, column, nowInSec));
- }
- else if (cosc.column != null)
- {
- cells.add(toLegacyCell(cfm, cosc.column, nowInSec));
- }
- else if (cosc.counter_super_column != null)
- {
- for (CounterColumn column : cosc.counter_super_column.columns)
- cells.add(toCounterLegacyCell(cfm, cosc.counter_super_column.name, column));
- }
- else // cosc.counter_column != null
- {
- cells.add(toCounterLegacyCell(cfm, cosc.counter_column));
- }
- }
- catch (UnknownColumnException e)
- {
- throw new InvalidRequestException(e.getMessage());
- }
- }
-
- private void addRange(CFMetaData cfm, LegacyLayout.LegacyDeletionInfo delInfo, ClusteringBound start, ClusteringBound end, long timestamp, int nowInSec)
- {
- delInfo.add(cfm, new RangeTombstone(Slice.make(start, end), new DeletionTime(timestamp, nowInSec)));
- }
-
- private void deleteColumnOrSuperColumn(LegacyLayout.LegacyDeletionInfo delInfo, List<LegacyLayout.LegacyCell> cells, CFMetaData cfm, Deletion del, int nowInSec)
- throws InvalidRequestException
- {
- if (del.predicate != null && del.predicate.column_names != null)
- {
- for (ByteBuffer c : del.predicate.column_names)
- {
- try
- {
- if (del.super_column == null && cfm.isSuper())
- addRange(cfm, delInfo, ClusteringBound.inclusiveStartOf(c), ClusteringBound.inclusiveEndOf(c), del.timestamp, nowInSec);
- else if (del.super_column != null)
- cells.add(toLegacyDeletion(cfm, del.super_column, c, del.timestamp, nowInSec));
- else
- cells.add(toLegacyDeletion(cfm, c, del.timestamp, nowInSec));
- }
- catch (UnknownColumnException e)
- {
- throw new InvalidRequestException(e.getMessage());
- }
- }
- }
- else if (del.predicate != null && del.predicate.slice_range != null)
- {
- if (del.super_column == null)
- {
- LegacyLayout.LegacyBound start = LegacyLayout.decodeTombstoneBound(cfm, del.predicate.getSlice_range().start, true);
- LegacyLayout.LegacyBound end = LegacyLayout.decodeTombstoneBound(cfm, del.predicate.getSlice_range().finish, false);
- delInfo.add(cfm, new LegacyLayout.LegacyRangeTombstone(start, end, new DeletionTime(del.timestamp, nowInSec)));
- }
- else
- {
- // Since we use a map for subcolumns, we would need range tombstone for collections to support this.
- // And while we may want those some day, this require a bit of additional work. And since super columns
- // are basically deprecated since a long time, and range tombstone on them has been only very recently
- // added so that no thrift driver actually supports it to the best of my knowledge, it's likely ok to
- // discontinue support for this. If it turns out that this is blocking the update of someone, we can
- // decide then if we want to tackle the addition of range tombstone for collections then.
- throw new InvalidRequestException("Cannot delete a range of subcolumns in a super column");
- }
- }
- else
- {
- if (del.super_column != null)
- addRange(cfm, delInfo, ClusteringBound.inclusiveStartOf(del.super_column), ClusteringBound.inclusiveEndOf(del.super_column), del.timestamp, nowInSec);
- else
- delInfo.add(new DeletionTime(del.timestamp, nowInSec));
- }
- }
-
- public void batch_mutate(Map<ByteBuffer,Map<String,List<Mutation>>> mutation_map, ConsistencyLevel consistency_level)
- throws InvalidRequestException, UnavailableException, TimedOutException
- {
- long queryStartNanoTime = System.nanoTime();
- if (startSessionIfRequested())
- {
- Map<String, String> traceParameters = Maps.newLinkedHashMap();
- for (Map.Entry<ByteBuffer, Map<String, List<Mutation>>> mutationEntry : mutation_map.entrySet())
- {
- traceParameters.put(ByteBufferUtil.bytesToHex(mutationEntry.getKey()),
- Joiner.on(";").withKeyValueSeparator(":").join(mutationEntry.getValue()));
- }
- traceParameters.put("consistency_level", consistency_level.name());
- Tracing.instance.begin("batch_mutate", traceParameters);
- }
- else
- {
- logger.trace("batch_mutate");
- }
-
- try
- {
- doInsert(consistency_level, createMutationList(consistency_level, mutation_map, true), queryStartNanoTime);
- }
- catch (RequestValidationException e)
- {
- throw ThriftConversion.toThrift(e);
- }
- finally
- {
- Tracing.instance.stopSession();
- }
- }
-
- public void atomic_batch_mutate(Map<ByteBuffer,Map<String,List<Mutation>>> mutation_map, ConsistencyLevel consistency_level)
- throws InvalidRequestException, UnavailableException, TimedOutException
- {
- long queryStartNanoTime = System.nanoTime();
- if (startSessionIfRequested())
- {
- Map<String, String> traceParameters = Maps.newLinkedHashMap();
- for (Map.Entry<ByteBuffer, Map<String, List<Mutation>>> mutationEntry : mutation_map.entrySet())
- {
- traceParameters.put(ByteBufferUtil.bytesToHex(mutationEntry.getKey()),
- Joiner.on(";").withKeyValueSeparator(":").join(mutationEntry.getValue()));
- }
- traceParameters.put("consistency_level", consistency_level.name());
- Tracing.instance.begin("atomic_batch_mutate", traceParameters);
- }
- else
- {
- logger.trace("atomic_batch_mutate");
- }
-
- try
- {
- doInsert(consistency_level, createMutationList(consistency_level, mutation_map, false), true, queryStartNanoTime);
- }
- catch (RequestValidationException e)
- {
- throw ThriftConversion.toThrift(e);
- }
- finally
- {
- Tracing.instance.stopSession();
- }
- }
-
- private void internal_remove(ByteBuffer key, ColumnPath column_path, long timestamp, ConsistencyLevel consistency_level, boolean isCommutativeOp, long queryStartNanoTime)
- throws RequestValidationException, UnavailableException, TimedOutException
- {
- ThriftClientState cState = state();
- String keyspace = cState.getKeyspace();
- cState.hasColumnFamilyAccess(keyspace, column_path.column_family, Permission.MODIFY);
-
- CFMetaData metadata = ThriftValidation.validateColumnFamily(keyspace, column_path.column_family, isCommutativeOp);
- if (metadata.isView())
- throw new org.apache.cassandra.exceptions.InvalidRequestException("Cannot modify Materialized Views directly");
-
- ThriftValidation.validateKey(metadata, key);
- ThriftValidation.validateColumnPathOrParent(metadata, column_path);
- if (isCommutativeOp)
- ThriftConversion.fromThrift(consistency_level).validateCounterForWrite(metadata);
-
- DecoratedKey dk = metadata.decorateKey(key);
-
- int nowInSec = FBUtilities.nowInSeconds();
- PartitionUpdate update;
- if (column_path.super_column == null && column_path.column == null)
- {
- update = PartitionUpdate.fullPartitionDelete(metadata, dk, timestamp, nowInSec);
- }
- else if (column_path.super_column != null && column_path.column == null)
- {
- Row row = BTreeRow.emptyDeletedRow(Clustering.make(column_path.super_column), Row.Deletion.regular(new DeletionTime(timestamp, nowInSec)));
- update = PartitionUpdate.singleRowUpdate(metadata, dk, row);
- }
- else
- {
- try
- {
- LegacyLayout.LegacyCellName name = LegacyLayout.decodeCellName(metadata, column_path.super_column, column_path.column);
- CellPath path = name.collectionElement == null ? null : CellPath.create(name.collectionElement);
- Cell cell = BufferCell.tombstone(name.column, timestamp, nowInSec, path);
- update = PartitionUpdate.singleRowUpdate(metadata, dk, BTreeRow.singleCellRow(name.clustering, cell));
- }
- catch (UnknownColumnException e)
- {
- throw new org.apache.cassandra.exceptions.InvalidRequestException(e.getMessage());
- }
- }
-
- org.apache.cassandra.db.Mutation mutation = new org.apache.cassandra.db.Mutation(update);
-
- if (isCommutativeOp)
- doInsert(consistency_level, Collections.singletonList(new CounterMutation(mutation, ThriftConversion.fromThrift(consistency_level))), queryStartNanoTime);
- else
- doInsert(consistency_level, Collections.singletonList(mutation), queryStartNanoTime);
- }
-
- public void remove(ByteBuffer key, ColumnPath column_path, long timestamp, ConsistencyLevel consistency_level)
- throws InvalidRequestException, UnavailableException, TimedOutException
- {
- long queryStartNanoTime = System.nanoTime();
- if (startSessionIfRequested())
- {
- Map<String, String> traceParameters = ImmutableMap.of("key", ByteBufferUtil.bytesToHex(key),
- "column_path", column_path.toString(),
- "timestamp", timestamp + "",
- "consistency_level", consistency_level.name());
- Tracing.instance.begin("remove", traceParameters);
- }
- else
- {
- logger.trace("remove");
- }
-
- try
- {
- internal_remove(key, column_path, timestamp, consistency_level, false, queryStartNanoTime);
- }
- catch (RequestValidationException e)
- {
- throw ThriftConversion.toThrift(e);
- }
- finally
- {
- Tracing.instance.stopSession();
- }
- }
-
- private void doInsert(ConsistencyLevel consistency_level, List<? extends IMutation> mutations, long queryStartNanoTime)
- throws UnavailableException, TimedOutException, org.apache.cassandra.exceptions.InvalidRequestException
- {
- doInsert(consistency_level, mutations, false, queryStartNanoTime);
- }
-
- private void doInsert(ConsistencyLevel consistency_level, List<? extends IMutation> mutations, boolean mutateAtomically, long queryStartNanoTime)
- throws UnavailableException, TimedOutException, org.apache.cassandra.exceptions.InvalidRequestException
- {
- org.apache.cassandra.db.ConsistencyLevel consistencyLevel = ThriftConversion.fromThrift(consistency_level);
- consistencyLevel.validateForWrite(state().getKeyspace());
- if (mutations.isEmpty())
- return;
-
- long timeout = Long.MAX_VALUE;
- for (IMutation m : mutations)
- timeout = Longs.min(timeout, m.getTimeout());
-
- schedule(timeout);
- try
- {
- StorageProxy.mutateWithTriggers(mutations, consistencyLevel, mutateAtomically, queryStartNanoTime);
- }
- catch (RequestExecutionException e)
- {
- ThriftConversion.rethrow(e);
- }
- finally
- {
- release();
- }
- }
-
- private void validateLogin() throws InvalidRequestException
- {
- try
- {
- state().validateLogin();
- }
- catch (UnauthorizedException e)
- {
- throw new InvalidRequestException(e.getMessage());
- }
- }
-
- public KsDef describe_keyspace(String keyspaceName) throws NotFoundException, InvalidRequestException
- {
- validateLogin();
-
- KeyspaceMetadata ksm = Schema.instance.getKSMetaData(keyspaceName);
- if (ksm == null)
- throw new NotFoundException();
-
- return ThriftConversion.toThrift(ksm);
- }
-
- public List<KeySlice> get_range_slices(ColumnParent column_parent, SlicePredicate predicate, KeyRange range, ConsistencyLevel consistency_level)
- throws InvalidRequestException, UnavailableException, TException, TimedOutException
- {
- long queryStartNanoTime = System.nanoTime();
- if (startSessionIfRequested())
- {
- Map<String, String> traceParameters = ImmutableMap.of(
- "column_parent", column_parent.toString(),
- "predicate", predicate.toString(),
- "range", range.toString(),
- "consistency_level", consistency_level.name());
- Tracing.instance.begin("get_range_slices", traceParameters);
- }
- else
- {
- logger.trace("range_slice");
- }
-
- try
- {
- ThriftClientState cState = state();
- String keyspace = cState.getKeyspace();
- cState.hasColumnFamilyAccess(keyspace, column_parent.column_family, Permission.SELECT);
-
- CFMetaData metadata = ThriftValidation.validateColumnFamily(keyspace, column_parent.column_family);
- ThriftValidation.validateColumnParent(metadata, column_parent);
- ThriftValidation.validatePredicate(metadata, column_parent, predicate);
- ThriftValidation.validateKeyRange(metadata, column_parent.super_column, range);
-
- org.apache.cassandra.db.ConsistencyLevel consistencyLevel = ThriftConversion.fromThrift(consistency_level);
- consistencyLevel.validateForRead(keyspace);
-
- IPartitioner p = metadata.partitioner;
- AbstractBounds<PartitionPosition> bounds;
- if (range.start_key == null)
- {
- Token.TokenFactory tokenFactory = p.getTokenFactory();
- Token left = tokenFactory.fromString(range.start_token);
- Token right = tokenFactory.fromString(range.end_token);
- bounds = Range.makeRowRange(left, right);
- }
- else
- {
- PartitionPosition end = range.end_key == null
- ? p.getTokenFactory().fromString(range.end_token).maxKeyBound()
- : PartitionPosition.ForKey.get(range.end_key, p);
- bounds = new Bounds<>(PartitionPosition.ForKey.get(range.start_key, p), end);
- }
- int nowInSec = FBUtilities.nowInSeconds();
- schedule(DatabaseDescriptor.getRangeRpcTimeout());
- try
- {
- ColumnFilter columns = makeColumnFilter(metadata, column_parent, predicate);
- ClusteringIndexFilter filter = toInternalFilter(metadata, column_parent, predicate);
- DataLimits limits = getLimits(range.count, metadata.isSuper() && !column_parent.isSetSuper_column(), predicate);
-
- PartitionRangeReadCommand cmd =
- PartitionRangeReadCommand.create(true,
- metadata,
- nowInSec,
- columns,
- ThriftConversion.rowFilterFromThrift(metadata, range.row_filter),
- limits,
- new DataRange(bounds, filter));
-
- try (PartitionIterator results = StorageProxy.getRangeSlice(cmd, consistencyLevel, queryStartNanoTime))
- {
- assert results != null;
- return thriftifyKeySlices(results, column_parent, limits.perPartitionCount());
- }
- }
- finally
- {
- release();
- }
- }
- catch (RequestValidationException e)
- {
- throw ThriftConversion.toThrift(e);
- }
- catch (RequestExecutionException e)
- {
- throw ThriftConversion.rethrow(e);
- }
- finally
- {
- Tracing.instance.stopSession();
- }
- }
-
- public List<KeySlice> get_paged_slice(String column_family, KeyRange range, ByteBuffer start_column, ConsistencyLevel consistency_level)
- throws InvalidRequestException, UnavailableException, TimedOutException, TException
- {
- long queryStartNanoTime = System.nanoTime();
- if (startSessionIfRequested())
- {
- Map<String, String> traceParameters = ImmutableMap.of("column_family", column_family,
- "range", range.toString(),
- "start_column", ByteBufferUtil.bytesToHex(start_column),
- "consistency_level", consistency_level.name());
- Tracing.instance.begin("get_paged_slice", traceParameters);
- }
- else
- {
- logger.trace("get_paged_slice");
- }
-
- try
- {
-
- ThriftClientState cState = state();
- String keyspace = cState.getKeyspace();
- cState.hasColumnFamilyAccess(keyspace, column_family, Permission.SELECT);
-
- CFMetaData metadata = ThriftValidation.validateColumnFamily(keyspace, column_family);
- ThriftValidation.validateKeyRange(metadata, null, range);
-
- org.apache.cassandra.db.ConsistencyLevel consistencyLevel = ThriftConversion.fromThrift(consistency_level);
- consistencyLevel.validateForRead(keyspace);
-
- IPartitioner p = metadata.partitioner;
- AbstractBounds<PartitionPosition> bounds;
- if (range.start_key == null)
- {
- // (token, key) is unsupported, assume (token, token)
- Token.TokenFactory tokenFactory = p.getTokenFactory();
- Token left = tokenFactory.fromString(range.start_token);
- Token right = tokenFactory.fromString(range.end_token);
- bounds = Range.makeRowRange(left, right);
- }
- else
- {
- PartitionPosition end = range.end_key == null
- ? p.getTokenFactory().fromString(range.end_token).maxKeyBound()
- : PartitionPosition.ForKey.get(range.end_key, p);
- bounds = new Bounds<>(PartitionPosition.ForKey.get(range.start_key, p), end);
- }
-
- if (range.row_filter != null && !range.row_filter.isEmpty())
- throw new InvalidRequestException("Cross-row paging is not supported along with index clauses");
-
- int nowInSec = FBUtilities.nowInSeconds();
- schedule(DatabaseDescriptor.getRangeRpcTimeout());
- try
- {
- ClusteringIndexFilter filter = new ClusteringIndexSliceFilter(Slices.ALL, false);
- DataLimits limits = getLimits(range.count, true, Integer.MAX_VALUE);
- Clustering pageFrom = metadata.isSuper()
- ? Clustering.make(start_column)
- : LegacyLayout.decodeCellName(metadata, start_column).clustering;
-
- PartitionRangeReadCommand cmd =
- PartitionRangeReadCommand.create(true,
- metadata,
- nowInSec,
- ColumnFilter.all(metadata),
- RowFilter.NONE,
- limits,
- new DataRange(bounds, filter).forPaging(bounds, metadata.comparator, pageFrom, true));
-
- try (PartitionIterator results = StorageProxy.getRangeSlice(cmd, consistencyLevel, queryStartNanoTime))
- {
- return thriftifyKeySlices(results, new ColumnParent(column_family), limits.perPartitionCount());
- }
- }
- catch (UnknownColumnException e)
- {
- throw new InvalidRequestException(e.getMessage());
- }
- finally
- {
- release();
- }
- }
- catch (RequestValidationException e)
- {
- throw ThriftConversion.toThrift(e);
- }
- catch (RequestExecutionException e)
- {
- throw ThriftConversion.rethrow(e);
- }
- finally
- {
- Tracing.instance.stopSession();
- }
- }
-
- private List<KeySlice> thriftifyKeySlices(PartitionIterator results, ColumnParent column_parent, int cellLimit)
- {
- try (PartitionIterator iter = results)
- {
- List<KeySlice> keySlices = new ArrayList<>();
- while (iter.hasNext())
- {
- try (RowIterator partition = iter.next())
- {
- List<ColumnOrSuperColumn> thriftifiedColumns = thriftifyPartition(partition, column_parent.super_column != null, partition.isReverseOrder(), cellLimit);
- keySlices.add(new KeySlice(partition.partitionKey().getKey(), thriftifiedColumns));
- }
- }
-
- return keySlices;
- }
- }
-
- public List<KeySlice> get_indexed_slices(ColumnParent column_parent, IndexClause index_clause, SlicePredicate column_predicate, ConsistencyLevel consistency_level)
- throws InvalidRequestException, UnavailableException, TimedOutException, TException
- {
- long queryStartNanoTime = System.nanoTime();
- if (startSessionIfRequested())
- {
- Map<String, String> traceParameters = ImmutableMap.of("column_parent", column_parent.toString(),
- "index_clause", index_clause.toString(),
- "slice_predicate", column_predicate.toString(),
- "consistency_level", consistency_level.name());
- Tracing.instance.begin("get_indexed_slices", traceParameters);
- }
- else
- {
- logger.trace("scan");
- }
-
- try
- {
- ThriftClientState cState = state();
- String keyspace = cState.getKeyspace();
- cState.hasColumnFamilyAccess(keyspace, column_parent.column_family, Permission.SELECT);
- CFMetaData metadata = ThriftValidation.validateColumnFamily(keyspace, column_parent.column_family, false);
- ThriftValidation.validateColumnParent(metadata, column_parent);
- ThriftValidation.validatePredicate(metadata, column_parent, column_predicate);
- ThriftValidation.validateIndexClauses(metadata, index_clause);
- org.apache.cassandra.db.ConsistencyLevel consistencyLevel = ThriftConversion.fromThrift(consistency_level);
- consistencyLevel.validateForRead(keyspace);
-
- IPartitioner p = metadata.partitioner;
- AbstractBounds<PartitionPosition> bounds = new Bounds<>(PartitionPosition.ForKey.get(index_clause.start_key, p),
- p.getMinimumToken().minKeyBound());
-
- int nowInSec = FBUtilities.nowInSeconds();
- ColumnFilter columns = makeColumnFilter(metadata, column_parent, column_predicate);
- ClusteringIndexFilter filter = toInternalFilter(metadata, column_parent, column_predicate);
- DataLimits limits = getLimits(index_clause.count, metadata.isSuper() && !column_parent.isSetSuper_column(), column_predicate);
-
- PartitionRangeReadCommand cmd =
- PartitionRangeReadCommand.create(true,
- metadata,
- nowInSec,
- columns,
- ThriftConversion.rowFilterFromThrift(metadata, index_clause.expressions),
- limits,
- new DataRange(bounds, filter));
-
- // If there's a secondary index that the command can use, have it validate the request parameters.
- cmd.maybeValidateIndex();
-
- try (PartitionIterator results = StorageProxy.getRangeSlice(cmd, consistencyLevel, queryStartNanoTime))
- {
- return thriftifyKeySlices(results, column_parent, limits.perPartitionCount());
- }
- }
- catch (RequestValidationException e)
- {
- throw ThriftConversion.toThrift(e);
- }
- catch (RequestExecutionException e)
- {
- throw ThriftConversion.rethrow(e);
- }
- finally
- {
- Tracing.instance.stopSession();
- }
- }
-
- public List<KsDef> describe_keyspaces() throws TException, InvalidRequestException
- {
- validateLogin();
-
- Set<String> keyspaces = Schema.instance.getKeyspaces();
- List<KsDef> ksset = new ArrayList<>(keyspaces.size());
- for (String ks : keyspaces)
- {
- try
- {
- ksset.add(describe_keyspace(ks));
- }
- catch (NotFoundException nfe)
- {
- logger.info("Failed to find metadata for keyspace '{}'. Continuing... ", ks);
- }
- }
- return ksset;
- }
-
- public String describe_cluster_name() throws TException
- {
- return DatabaseDescriptor.getClusterName();
- }
-
- public String describe_version() throws TException
- {
- return cassandraConstants.VERSION;
- }
-
- public List<TokenRange> describe_ring(String keyspace) throws InvalidRequestException
- {
- try
- {
- return StorageService.instance.describeRing(keyspace);
- }
- catch (RequestValidationException e)
- {
- throw ThriftConversion.toThrift(e);
- }
- }
-
- @Override
- public List<TokenRange> describe_local_ring(String keyspace) throws InvalidRequestException, TException
- {
- try
- {
- return StorageService.instance.describeLocalRing(keyspace);
- }
- catch (RequestValidationException e)
- {
- throw ThriftConversion.toThrift(e);
- }
- }
-
- public Map<String, String> describe_token_map() throws InvalidRequestException
- {
- return StorageService.instance.getTokenToEndpointMap();
- }
-
- public String describe_partitioner() throws TException
- {
- return StorageService.instance.getPartitionerName();
- }
-
- public String describe_snitch() throws TException
- {
- if (DatabaseDescriptor.getEndpointSnitch() instanceof DynamicEndpointSnitch)
- return ((DynamicEndpointSnitch)DatabaseDescriptor.getEndpointSnitch()).subsnitch.getClass().getName();
- return DatabaseDescriptor.getEndpointSnitch().getClass().getName();
- }
-
- @Deprecated
- public List<String> describe_splits(String cfName, String start_token, String end_token, int keys_per_split)
- throws TException, InvalidRequestException
- {
- List<CfSplit> splits = describe_splits_ex(cfName, start_token, end_token, keys_per_split);
- List<String> result = new ArrayList<>(splits.size() + 1);
-
- result.add(splits.get(0).getStart_token());
- for (CfSplit cfSplit : splits)
- result.add(cfSplit.getEnd_token());
-
- return result;
- }
-
- public List<CfSplit> describe_splits_ex(String cfName, String start_token, String end_token, int keys_per_split)
- throws InvalidRequestException, TException
- {
- try
- {
- Token.TokenFactory tf = StorageService.instance.getTokenFactory();
- Range<Token> tr = new Range<Token>(tf.fromString(start_token), tf.fromString(end_token));
- List<Pair<Range<Token>, Long>> splits =
- StorageService.instance.getSplits(state().getKeyspace(), cfName, tr, keys_per_split);
- List<CfSplit> result = new ArrayList<>(splits.size());
- for (Pair<Range<Token>, Long> split : splits)
- result.add(new CfSplit(split.left.left.toString(), split.left.right.toString(), split.right));
- return result;
- }
- catch (RequestValidationException e)
- {
- throw ThriftConversion.toThrift(e);
- }
- }
-
- public void login(AuthenticationRequest auth_request) throws TException
- {
- try
- {
- state().login(DatabaseDescriptor.getAuthenticator().legacyAuthenticate(auth_request.getCredentials()));
- }
- catch (org.apache.cassandra.exceptions.AuthenticationException e)
- {
- throw ThriftConversion.toThrift(e);
- }
- }
-
- /**
- * Schedule the current thread for access to the required services
- */
- private void schedule(long timeoutMS) throws UnavailableException
- {
- try
- {
- requestScheduler.queue(Thread.currentThread(), state().getSchedulingValue(), timeoutMS);
- }
- catch (TimeoutException e)
- {
- throw new UnavailableException();
- }
- }
-
- /**
- * Release count for the used up resources
- */
- private void release()
- {
- requestScheduler.release();
- }
-
- public String system_add_column_family(CfDef cf_def) throws TException
- {
- logger.trace("add_column_family");
-
- try
- {
- ClientState cState = state();
- String keyspace = cState.getKeyspace();
- cState.hasKeyspaceAccess(keyspace, Permission.CREATE);
- cf_def.unsetId(); // explicitly ignore any id set by client (Hector likes to set zero)
- CFMetaData cfm = ThriftConversion.fromThrift(cf_def);
- cfm.params.compaction.validate();
-
- if (!cfm.getTriggers().isEmpty())
- state().ensureIsSuper("Only superusers are allowed to add triggers.");
-
- MigrationManager.announceNewColumnFamily(cfm);
- return Schema.instance.getVersion().toString();
- }
- catch (RequestValidationException e)
- {
- throw ThriftConversion.toThrift(e);
- }
- }
-
- public String system_drop_column_family(String column_family)
- throws InvalidRequestException, SchemaDisagreementException, TException
- {
- logger.trace("drop_column_family");
-
- ThriftClientState cState = state();
-
- try
- {
- String keyspace = cState.getKeyspace();
- cState.hasColumnFamilyAccess(keyspace, column_family, Permission.DROP);
-
- CFMetaData metadata = ThriftValidation.validateColumnFamily(keyspace, column_family);
- if (metadata.isView())
- throw new org.apache.cassandra.exceptions.InvalidRequestException("Cannot drop Materialized Views from Thrift");
-
- MigrationManager.announceColumnFamilyDrop(keyspace, column_family);
- return Schema.instance.getVersion().toString();
- }
- catch (RequestValidationException e)
- {
- throw ThriftConversion.toThrift(e);
- }
- }
-
- public String system_add_keyspace(KsDef ks_def)
- throws InvalidRequestException, SchemaDisagreementException, TException
- {
- logger.trace("add_keyspace");
-
- try
- {
- ThriftValidation.validateKeyspaceNotSystem(ks_def.name);
- state().hasAllKeyspacesAccess(Permission.CREATE);
- ThriftValidation.validateKeyspaceNotYetExisting(ks_def.name);
-
- // generate a meaningful error if the user setup keyspace and/or column definition incorrectly
- for (CfDef cf : ks_def.cf_defs)
- {
- if (!cf.getKeyspace().equals(ks_def.getName()))
- {
- throw new InvalidRequestException("CfDef (" + cf.getName() +") had a keyspace definition that did not match KsDef");
- }
- }
-
- Collection<CFMetaData> cfDefs = new ArrayList<CFMetaData>(ks_def.cf_defs.size());
- for (CfDef cf_def : ks_def.cf_defs)
- {
- cf_def.unsetId(); // explicitly ignore any id set by client (same as system_add_column_family)
- CFMetaData cfm = ThriftConversion.fromThrift(cf_def);
-
- if (!cfm.getTriggers().isEmpty())
- state().ensureIsSuper("Only superusers are allowed to add triggers.");
-
- cfDefs.add(cfm);
- }
- MigrationManager.announceNewKeyspace(ThriftConversion.fromThrift(ks_def, cfDefs.toArray(new CFMetaData[cfDefs.size()])));
- return Schema.instance.getVersion().toString();
- }
- catch (RequestValidationException e)
- {
- throw ThriftConversion.toThrift(e);
- }
- }
-
- public String system_drop_keyspace(String keyspace)
- throws InvalidRequestException, SchemaDisagreementException, TException
- {
- logger.trace("drop_keyspace");
-
- try
- {
- ThriftValidation.validateKeyspaceNotSystem(keyspace);
- state().hasKeyspaceAccess(keyspace, Permission.DROP);
-
- MigrationManager.announceKeyspaceDrop(keyspace);
- return Schema.instance.getVersion().toString();
- }
- catch (RequestValidationException e)
- {
- throw ThriftConversion.toThrift(e);
- }
- }
-
- /** update an existing keyspace, but do not allow column family modifications.
- * @throws SchemaDisagreementException
- */
- public String system_update_keyspace(KsDef ks_def)
- throws InvalidRequestException, SchemaDisagreementException, TException
- {
- logger.trace("update_keyspace");
-
- try
- {
- ThriftValidation.validateKeyspaceNotSystem(ks_def.name);
- state().hasKeyspaceAccess(ks_def.name, Permission.ALTER);
- ThriftValidation.validateKeyspace(ks_def.name);
- if (ks_def.getCf_defs() != null && ks_def.getCf_defs().size() > 0)
- throw new InvalidRequestException("Keyspace update must not contain any table definitions.");
-
- MigrationManager.announceKeyspaceUpdate(ThriftConversion.fromThrift(ks_def));
- return Schema.instance.getVersion().toString();
- }
- catch (RequestValidationException e)
- {
- throw ThriftConversion.toThrift(e);
- }
- }
-
- public String system_update_column_family(CfDef cf_def)
- throws InvalidRequestException, SchemaDisagreementException, TException
- {
- logger.trace("update_column_family");
-
- try
- {
- if (cf_def.keyspace == null || cf_def.name == null)
- throw new InvalidRequestException("Keyspace and CF name must be set.");
-
- state().hasColumnFamilyAccess(cf_def.keyspace, cf_def.name, Permission.ALTER);
- CFMetaData oldCfm = Schema.instance.getCFMetaData(cf_def.keyspace, cf_def.name);
-
- if (oldCfm == null)
- throw new InvalidRequestException("Could not find table definition to modify.");
-
- if (oldCfm.isView())
- throw new InvalidRequestException("Cannot modify Materialized View table " + oldCfm.cfName + " as it may break the schema. You should use cqlsh to modify Materialized View tables instead.");
- if (!Iterables.isEmpty(View.findAll(cf_def.keyspace, cf_def.name)))
- throw new InvalidRequestException("Cannot modify table with Materialized View " + oldCfm.cfName + " as it may break the schema. You should use cqlsh to modify tables with Materialized Views instead.");
-
- if (!oldCfm.isThriftCompatible())
- throw new InvalidRequestException("Cannot modify CQL3 table " + oldCfm.cfName + " as it may break the schema. You should use cqlsh to modify CQL3 tables instead.");
-
- CFMetaData cfm = ThriftConversion.fromThriftForUpdate(cf_def, oldCfm);
- cfm.params.compaction.validate();
-
- if (!oldCfm.getTriggers().equals(cfm.getTriggers()))
- state().ensureIsSuper("Only superusers are allowed to add or remove triggers.");
-
- MigrationManager.announceColumnFamilyUpdate(cfm);
- return Schema.instance.getVersion().toString();
- }
- catch (RequestValidationException e)
- {
- throw ThriftConversion.toThrift(e);
- }
- }
-
- public void truncate(String cfname) throws InvalidRequestException, UnavailableException, TimedOutException, TException
- {
- ClientState cState = state();
-
- try
- {
- String keyspace = cState.getKeyspace();
- cState.hasColumnFamilyAccess(keyspace, cfname, Permission.MODIFY);
- CFMetaData metadata = ThriftValidation.validateColumnFamily(keyspace, cfname, false);
- if (metadata.isView())
- throw new org.apache.cassandra.exceptions.InvalidRequestException("Cannot truncate Materialized Views");
-
- if (startSessionIfRequested())
- {
- Tracing.instance.begin("truncate", ImmutableMap.of("cf", cfname, "ks", keyspace));
- }
- else
- {
- logger.trace("truncating {}.{}", cState.getKeyspace(), cfname);
- }
-
- schedule(DatabaseDescriptor.getTruncateRpcTimeout());
- try
- {
- StorageProxy.truncateBlocking(cState.getKeyspace(), cfname);
- }
- finally
- {
- release();
- }
- }
- catch (RequestValidationException e)
- {
- throw ThriftConversion.toThrift(e);
- }
- catch (org.apache.cassandra.exceptions.UnavailableException e)
- {
- throw ThriftConversion.toThrift(e);
- }
- catch (TimeoutException e)
- {
- throw new TimedOutException();
- }
- finally
- {
- Tracing.instance.stopSession();
- }
- }
-
- public void set_keyspace(String keyspace) throws InvalidRequestException, TException
- {
- try
- {
- state().setKeyspace(keyspace);
- }
- catch (RequestValidationException e)
- {
- throw ThriftConversion.toThrift(e);
- }
- }
-
- public Map<String, List<String>> describe_schema_versions() throws TException, InvalidRequestException
- {
- logger.trace("checking schema agreement");
- return StorageProxy.describeSchemaVersions();
- }
-
- // counter methods
-
- public void add(ByteBuffer key, ColumnParent column_parent, CounterColumn column, ConsistencyLevel consistency_level)
- throws InvalidRequestException, UnavailableException, TimedOutException, TException
- {
- long queryStartNanoTime = System.nanoTime();
- if (startSessionIfRequested())
- {
- Map<String, String> traceParameters = ImmutableMap.of("column_parent", column_parent.toString(),
- "column", column.toString(),
- "consistency_level", consistency_level.name());
- Tracing.instance.begin("add", traceParameters);
- }
- else
- {
- logger.trace("add");
- }
-
- try
- {
- ClientState cState = state();
- String keyspace = cState.getKeyspace();
-
- cState.hasColumnFamilyAccess(keyspace, column_parent.column_family, Permission.MODIFY);
-
- CFMetaData metadata = ThriftValidation.validateColumnFamily(keyspace, column_parent.column_family, true);
- if (metadata.isView())
- throw new org.apache.cassandra.exceptions.InvalidRequestException("Cannot modify Materialized Views directly");
-
- ThriftValidation.validateKey(metadata, key);
- ThriftConversion.fromThrift(consistency_level).validateCounterForWrite(metadata);
- ThriftValidation.validateColumnParent(metadata, column_parent);
- // SuperColumn field is usually optional, but not when we're adding
- if (metadata.isSuper() && column_parent.super_column == null)
- throw new InvalidRequestException("missing mandatory super column name for super CF " + column_parent.column_family);
-
- ThriftValidation.validateColumnNames(metadata, column_parent, Arrays.asList(column.name));
-
- try
- {
- LegacyLayout.LegacyCellName name = LegacyLayout.decodeCellName(metadata, column_parent.super_column, column.name);
-
- // See UpdateParameters.addCounter() for more details on this
- ByteBuffer value = CounterContext.instance().createUpdate(column.value);
- CellPath path = name.collectionElement == null ? null : CellPath.create(name.collectionElement);
- Cell cell = BufferCell.live(name.column, FBUtilities.timestampMicros(), value, path);
-
- PartitionUpdate update = PartitionUpdate.singleRowUpdate(metadata, key, BTreeRow.singleCellRow(name.clustering, cell));
-
- org.apache.cassandra.db.Mutation mutation = new org.apache.cassandra.db.Mutation(update);
- doInsert(consistency_level, Arrays.asList(new CounterMutation(mutation, ThriftConversion.fromThrift(consistency_level))), queryStartNanoTime);
- }
- catch (MarshalException|UnknownColumnException e)
- {
- throw new InvalidRequestException(e.getMessage());
- }
- }
- catch (RequestValidationException e)
- {
- throw ThriftConversion.toThrift(e);
- }
- finally
- {
- Tracing.instance.stopSession();
- }
- }
-
- public void remove_counter(ByteBuffer key, ColumnPath path, ConsistencyLevel consistency_level)
- throws InvalidRequestException, UnavailableException, TimedOutException, TException
- {
- long queryStartNanoTime = System.nanoTime();
- if (startSessionIfRequested())
- {
- Map<String, String> traceParameters = ImmutableMap.of("key", ByteBufferUtil.bytesToHex(key),
- "column_path", path.toString(),
- "consistency_level", consistency_level.name());
- Tracing.instance.begin("remove_counter", traceParameters);
- }
- else
- {
- logger.trace("remove_counter");
- }
-
- try
- {
- internal_remove(key, path, FBUtilities.timestampMicros(), consistency_level, true, queryStartNanoTime);
- }
- catch (RequestValidationException e)
- {
- throw ThriftConversion.toThrift(e);
- }
- finally
- {
- Tracing.instance.stopSession();
- }
- }
-
- private static String uncompress(ByteBuffer query, Compression compression) throws InvalidRequestException
- {
- String queryString = null;
-
- // Decompress the query string.
- try
- {
- switch (compression)
- {
- case GZIP:
- DataOutputBuffer decompressed = new DataOutputBuffer();
- byte[] outBuffer = new byte[1024], inBuffer = new byte[1024];
-
- Inflater decompressor = new Inflater();
-
- int lenRead = 0;
- while (true)
- {
- if (decompressor.needsInput())
- lenRead = query.remaining() < 1024 ? query.remaining() : 1024;
- query.get(inBuffer, 0, lenRead);
- decompressor.setInput(inBuffer, 0, lenRead);
-
- int lenWrite = 0;
- while ((lenWrite = decompressor.inflate(outBuffer)) != 0)
- decompressed.write(outBuffer, 0, lenWrite);
-
- if (decompressor.finished())
- break;
- }
-
- decompressor.end();
-
- queryString = new String(decompressed.getData(), 0, decompressed.getLength(), StandardCharsets.UTF_8);
- break;
- case NONE:
- try
- {
- queryString = ByteBufferUtil.string(query);
- }
- catch (CharacterCodingException ex)
- {
- throw new InvalidRequestException(ex.getMessage());
- }
- break;
- }
- }
- catch (DataFormatException e)
- {
- throw new InvalidRequestException("Error deflating query string.");
- }
- catch (IOException e)
- {
- throw new AssertionError(e);
- }
- return queryString;
- }
-
- public CqlResult execute_cql_query(ByteBuffer query, Compression compression) throws TException
- {
- throw new InvalidRequestException("CQL2 has been removed in Cassandra 3.0. Please use CQL3 instead");
- }
-
- public CqlResult execute_cql3_query(ByteBuffer query, Compression compression, ConsistencyLevel cLevel) throws TException
- {
- try
- {
- long queryStartNanoTime = System.nanoTime();
- String queryString = uncompress(query, compression);
- if (startSessionIfRequested())
- {
- Tracing.instance.begin("execute_cql3_query",
- ImmutableMap.of("query", queryString,
- "consistency_level", cLevel.name()));
- }
- else
- {
- logger.trace("execute_cql3_query");
- }
-
- ThriftClientState cState = state();
- return ClientState.getCQLQueryHandler().process(queryString,
- cState.getQueryState(),
- QueryOptions.fromThrift(ThriftConversion.fromThrift(cLevel),
- Collections.<ByteBuffer>emptyList()),
- null,
- queryStartNanoTime).toThriftResult();
- }
- catch (RequestExecutionException e)
- {
- throw ThriftConversion.rethrow(e);
- }
- catch (RequestValidationException e)
- {
- throw ThriftConversion.toThrift(e);
- }
- finally
- {
- Tracing.instance.stopSession();
- }
- }
-
- public CqlPreparedResult prepare_cql_query(ByteBuffer query, Compression compression) throws TException
- {
- throw new InvalidRequestException("CQL2 has been removed in Cassandra 3.0. Please use CQL3 instead");
- }
-
- public CqlPreparedResult prepare_cql3_query(ByteBuffer query, Compression compression) throws TException
- {
- logger.trace("prepare_cql3_query");
-
- String queryString = uncompress(query, compression);
- ThriftClientState cState = state();
-
- try
- {
- cState.validateLogin();
- return ClientState.getCQLQueryHandler().prepare(queryString, cState.getQueryState(), null).toThriftPreparedResult();
- }
- catch (RequestValidationException e)
- {
- throw ThriftConversion.toThrift(e);
- }
- }
-
- public CqlResult execute_prepared_cql_query(int itemId, List<ByteBuffer> bindVariables) throws TException
- {
- throw new InvalidRequestException("CQL2 has been removed in Cassandra 3.0. Please use CQL3 instead");
- }
-
- public CqlResult execute_prepared_cql3_query(int itemId, List<ByteBuffer> bindVariables, ConsistencyLevel cLevel) throws TException
- {
- long queryStartNanoTime = System.nanoTime();
- if (startSessionIfRequested())
- {
- // TODO we don't have [typed] access to CQL bind variables here. CASSANDRA-4560 is open to add support.
- Tracing.instance.begin("execute_prepared_cql3_query", ImmutableMap.of("consistency_level", cLevel.name()));
- }
- else
- {
- logger.trace("execute_prepared_cql3_query");
- }
-
- try
- {
- ThriftClientState cState = state();
- ParsedStatement.Prepared prepared = ClientState.getCQLQueryHandler().getPreparedForThrift(itemId);
-
- if (prepared == null)
- throw new InvalidRequestException(String.format("Prepared query with ID %d not found" +
- " (either the query was not prepared on this host (maybe the host has been restarted?)" +
- " or you have prepared too many queries and it has been evicted from the internal cache)",
- itemId));
- logger.trace("Retrieved prepared statement #{} with {} bind markers", itemId, prepared.statement.getBoundTerms());
-
- return ClientState.getCQLQueryHandler().processPrepared(prepared.statement,
- cState.getQueryState(),
- QueryOptions.fromThrift(ThriftConversion.fromThrift(cLevel), bindVariables),
- null,
- queryStartNanoTime).toThriftResult();
- }
- catch (RequestExecutionException e)
- {
- throw ThriftConversion.rethrow(e);
- }
- catch (RequestValidationException e)
- {
- throw ThriftConversion.toThrift(e);
- }
- finally
- {
- Tracing.instance.stopSession();
- }
- }
-
- @Override
- public List<ColumnOrSuperColumn> get_multi_slice(MultiSliceRequest request)
- throws InvalidRequestException, UnavailableException, TimedOutException
- {
- long queryStartNanoTime = System.nanoTime();
- if (startSessionIfRequested())
- {
- Map<String, String> traceParameters = ImmutableMap.of("key", ByteBufferUtil.bytesToHex(request.key),
- "column_parent", request.column_parent.toString(),
- "consistency_level", request.consistency_level.name(),
- "count", String.valueOf(request.count),
- "column_slices", request.column_slices.toString());
- Tracing.instance.begin("get_multi_slice", traceParameters);
- }
- else
- {
- logger.trace("get_multi_slice");
- }
- try
- {
- ClientState cState = state();
- String keyspace = cState.getKeyspace();
- state().hasColumnFamilyAccess(keyspace, request.getColumn_parent().column_family, Permission.SELECT);
- CFMetaData metadata = ThriftValidation.validateColumnFamily(keyspace, request.getColumn_parent().column_family);
- if (metadata.isSuper())
- throw new org.apache.cassandra.exceptions.InvalidRequestException("get_multi_slice does not support super columns");
- ThriftValidation.validateColumnParent(metadata, request.getColumn_parent());
- org.apache.cassandra.db.ConsistencyLevel consistencyLevel = ThriftConversion.fromThrift(request.getConsistency_level());
- consistencyLevel.validateForRead(keyspace);
-
- Slices.Builder builder = new Slices.Builder(metadata.comparator, request.getColumn_slices().size());
- for (int i = 0 ; i < request.getColumn_slices().size() ; i++)
- {
- fixOptionalSliceParameters(request.getColumn_slices().get(i));
- ClusteringBound start = LegacyLayout.decodeSliceBound(metadata, request.getColumn_slices().get(i).start, true).bound;
- ClusteringBound finish = LegacyLayout.decodeSliceBound(metadata, request.getColumn_slices().get(i).finish, false).bound;
-
- int compare = metadata.comparator.compare(start, finish);
- if (!request.reversed && compare > 0)
- throw new InvalidRequestException(String.format("Column slice at index %d had start greater than finish", i));
- else if (request.reversed && compare < 0)
- throw new InvalidRequestException(String.format("Reversed column slice at index %d had start less than finish", i));
-
- builder.add(request.reversed ? Slice.make(finish, start) : Slice.make(start, finish));
- }
-
- Slices slices = builder.build();
- ColumnFilter columns = makeColumnFilter(metadata, slices);
- ClusteringIndexSliceFilter filter = new ClusteringIndexSliceFilter(slices, request.reversed);
- DataLimits limits = getLimits(1, false, request.count);
-
- ThriftValidation.validateKey(metadata, request.key);
- DecoratedKey dk = metadata.decorateKey(request.key);
- SinglePartitionReadCommand cmd = SinglePartitionReadCommand.create(true, metadata, FBUtilities.nowInSeconds(), columns, RowFilter.NONE, limits, dk, filter);
- return getSlice(Collections.<SinglePartitionReadCommand>singletonList(cmd),
- false,
- limits.perPartitionCount(),
- consistencyLevel,
- cState,
- queryStartNanoTime).entrySet().iterator().next().getValue();
- }
- catch (RequestValidationException e)
- {
- throw ThriftConversion.toThrift(e);
- }
- finally
- {
- Tracing.instance.stopSession();
- }
- }
-
- /**
- * Set the to start-of end-of value of "" for start and finish.
- * @param columnSlice
- */
- private static void fixOptionalSliceParameters(org.apache.cassandra.thrift.ColumnSlice columnSlice)
- {
- if (!columnSlice.isSetStart())
- columnSlice.setStart(new byte[0]);
- if (!columnSlice.isSetFinish())
- columnSlice.setFinish(new byte[0]);
- }
-
- /*
- * No-op since 3.0.
- */
- public void set_cql_version(String version)
- {
- }
-
- public ByteBuffer trace_next_query() throws TException
- {
- UUID sessionId = UUIDGen.getTimeUUID();
- state().getQueryState().prepareTracingSession(sessionId);
- return TimeUUIDType.instance.decompose(sessionId);
- }
-
- private boolean startSessionIfRequested()
- {
- if (state().getQueryState().traceNextQuery())
- {
- state().getQueryState().createTracingSession(Collections.EMPTY_MAP);
- return true;
- }
- return false;
- }
-
- private void registerMetrics()
- {
- ClientMetrics.instance.addCounter("connectedThriftClients", new Callable<Integer>()
- {
- @Override
- public Integer call() throws Exception
- {
- return ThriftSessionManager.instance.getConnectedClients();
- }
- });
- }
-
- private static class ThriftCASRequest implements CASRequest
- {
- private final CFMetaData metadata;
- private final DecoratedKey key;
- private final List<LegacyLayout.LegacyCell> expected;
- private final PartitionUpdate updates;
- private final int nowInSec;
-
- private ThriftCASRequest(List<LegacyLayout.LegacyCell> expected, PartitionUpdate updates, int nowInSec)
- {
- this.metadata = updates.metadata();
- this.key = updates.partitionKey();
- this.expected = expected;
- this.updates = updates;
- this.nowInSec = nowInSec;
- }
-
- public SinglePartitionReadCommand readCommand(int nowInSec)
- {
- if (expected.isEmpty())
- {
- // We want to know if the partition exists, so just fetch a single cell.
- ClusteringIndexSliceFilter filter = new ClusteringIndexSliceFilter(Slices.ALL, false);
- DataLimits limits = DataLimits.thriftLimits(1, 1);
- return SinglePartitionReadCommand.create(true, metadata, nowInSec, ColumnFilter.all(metadata), RowFilter.NONE, limits, key, filter);
- }
-
- // Gather the clustering for the expected values and query those.
- BTreeSet.Builder<Clustering> clusterings = BTreeSet.builder(metadata.comparator);
- FilteredPartition expectedPartition =
- FilteredPartition.create(LegacyLayout.toRowIterator(metadata, key, expected.iterator(), nowInSec));
-
- for (Row row : expectedPartition)
- clusterings.add(row.clustering());
-
- PartitionColumns columns = expectedPartition.staticRow().isEmpty()
- ? metadata.partitionColumns().withoutStatics()
- : metadata.partitionColumns();
- ClusteringIndexNamesFilter filter = new ClusteringIndexNamesFilter(clusterings.build(), false);
- return SinglePartitionReadCommand.create(true, metadata, nowInSec, ColumnFilter.selection(columns), RowFilter.NONE, DataLimits.NONE, key, filter);
- }
-
- public boolean appliesTo(FilteredPartition current)
- {
- if (expected.isEmpty())
- return current.isEmpty();
- else if (current.isEmpty())
- return false;
-
- // Push the expected results through ThriftResultsMerger to translate any static
- // columns into clusterings. The current partition is retrieved in the same so
- // unless they're both handled the same, they won't match.
- FilteredPartition expectedPartition =
- FilteredPartition.create(
- UnfilteredRowIterators.filter(
- ThriftResultsMerger.maybeWrap(expectedToUnfilteredRowIterator(), nowInSec), nowInSec));
-
- // Check that for everything we expected, the fetched values exists and correspond.
- for (Row e : expectedPartition)
- {
- Row c = current.getRow(e.clustering());
- if (c == null)
- return false;
-
- SearchIterator<ColumnDefinition, ColumnData> searchIter = c.searchIterator();
- for (ColumnData expectedData : e)
- {
- ColumnDefinition column = expectedData.column();
- ColumnData currentData = searchIter.next(column);
- if (currentData == null)
- return false;
-
- if (column.isSimple())
- {
- if (!((Cell)currentData).value().equals(((Cell)expectedData).value()))
- return false;
- }
- else
- {
- ComplexColumnData currentComplexData = (ComplexColumnData)currentData;
- for (Cell expectedCell : (ComplexColumnData)expectedData)
- {
- Cell currentCell = currentComplexData.getCell(expectedCell.path());
- if (currentCell == null || !currentCell.value().equals(expectedCell.value()))
- return false;
- }
- }
- }
- }
- return true;
- }
-
- public PartitionUpdate makeUpdates(FilteredPartition current)
- {
- return updates;
- }
-
- private UnfilteredRowIterator expectedToUnfilteredRowIterator()
- {
- return LegacyLayout.toUnfilteredRowIterator(metadata, key, LegacyLayout.LegacyDeletionInfo.live(), expected.iterator());
- }
- }
-}
diff --git a/src/java/org/apache/cassandra/thrift/CustomTNonBlockingServer.java b/src/java/org/apache/cassandra/thrift/CustomTNonBlockingServer.java
deleted file mode 100644
index 8221a83..0000000
--- a/src/java/org/apache/cassandra/thrift/CustomTNonBlockingServer.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.thrift;
-
-import java.net.InetSocketAddress;
-
-import java.nio.channels.SelectionKey;
-
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.thrift.server.TNonblockingServer;
-import org.apache.thrift.server.TServer;
-import org.apache.thrift.transport.TNonblockingServerTransport;
-import org.apache.thrift.transport.TNonblockingSocket;
-import org.apache.thrift.transport.TNonblockingTransport;
-import org.apache.thrift.transport.TTransportException;
-
-public class CustomTNonBlockingServer extends TNonblockingServer
-{
- public CustomTNonBlockingServer(Args args)
- {
- super(args);
- }
-
- @Override
- @SuppressWarnings("resource")
- protected boolean requestInvoke(FrameBuffer frameBuffer)
- {
- TNonblockingSocket socket = (TNonblockingSocket)((CustomFrameBuffer)frameBuffer).getTransport();
- ThriftSessionManager.instance.setCurrentSocket(socket.getSocketChannel().socket().getRemoteSocketAddress());
- frameBuffer.invoke();
- return true;
- }
-
- public static class Factory implements TServerFactory
- {
- @SuppressWarnings("resource")
- public TServer buildTServer(Args args)
- {
- if (DatabaseDescriptor.getClientEncryptionOptions().enabled)
- throw new RuntimeException("Client SSL is not supported for non-blocking sockets. Please remove client ssl from the configuration.");
-
- final InetSocketAddress addr = args.addr;
- TNonblockingServerTransport serverTransport;
- try
- {
- serverTransport = new TCustomNonblockingServerSocket(addr, args.keepAlive, args.sendBufferSize, args.recvBufferSize);
- }
- catch (TTransportException e)
- {
- throw new RuntimeException(String.format("Unable to create thrift socket to %s:%s", addr.getAddress(), addr.getPort()), e);
- }
-
- // This is single threaded hence the invocation will be all
- // in one thread.
- TNonblockingServer.Args serverArgs = new TNonblockingServer.Args(serverTransport).inputTransportFactory(args.inTransportFactory)
- .outputTransportFactory(args.outTransportFactory)
- .inputProtocolFactory(args.tProtocolFactory)
- .outputProtocolFactory(args.tProtocolFactory)
- .processor(args.processor);
- return new CustomTNonBlockingServer(serverArgs);
- }
- }
-
- public class CustomFrameBuffer extends FrameBuffer
- {
- public CustomFrameBuffer(final TNonblockingTransport trans,
- final SelectionKey selectionKey,
- final AbstractSelectThread selectThread)
- {
- super(trans, selectionKey, selectThread);
- }
-
- public TNonblockingTransport getTransport()
- {
- return this.trans_;
- }
- }
-}
diff --git a/src/java/org/apache/cassandra/thrift/CustomTThreadPoolServer.java b/src/java/org/apache/cassandra/thrift/CustomTThreadPoolServer.java
deleted file mode 100644
index 31aa689..0000000
--- a/src/java/org/apache/cassandra/thrift/CustomTThreadPoolServer.java
+++ /dev/null
@@ -1,290 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.thrift;
-
-import java.net.InetSocketAddress;
-import java.net.SocketAddress;
-import java.net.SocketTimeoutException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.RejectedExecutionException;
-import java.util.concurrent.SynchronousQueue;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import javax.net.ssl.SSLServerSocket;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.cassandra.concurrent.NamedThreadFactory;
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.config.EncryptionOptions.ClientEncryptionOptions;
-import org.apache.cassandra.utils.JVMStabilityInspector;
-import org.apache.cassandra.security.SSLFactory;
-import org.apache.thrift.TException;
-import org.apache.thrift.TProcessor;
-import org.apache.thrift.protocol.TProtocol;
-import org.apache.thrift.server.TServer;
-import org.apache.thrift.server.TThreadPoolServer;
-import org.apache.thrift.transport.TSSLTransportFactory;
-import org.apache.thrift.transport.TServerSocket;
-import org.apache.thrift.transport.TServerTransport;
-import org.apache.thrift.transport.TTransport;
-import org.apache.thrift.transport.TTransportException;
-import org.apache.thrift.transport.TSSLTransportFactory.TSSLTransportParameters;
-
-import com.google.common.util.concurrent.Uninterruptibles;
-
-
-/**
- * Slightly modified version of the Apache Thrift TThreadPoolServer.
- * <p>
- * This allows passing an executor so you have more control over the actual
- * behavior of the tasks being run.
- * </p>
- * Newer version of Thrift should make this obsolete.
- */
-public class CustomTThreadPoolServer extends TServer
-{
-
- private static final Logger logger = LoggerFactory.getLogger(CustomTThreadPoolServer.class.getName());
-
- // Executor service for handling client connections
- private final ExecutorService executorService;
-
- // Flag for stopping the server
- private volatile boolean stopped;
-
- // Server options
- private final TThreadPoolServer.Args args;
-
- //Track and Limit the number of connected clients
- private final AtomicInteger activeClients;
-
-
- public CustomTThreadPoolServer(TThreadPoolServer.Args args, ExecutorService executorService)
- {
- super(args);
- this.executorService = executorService;
- this.stopped = false;
- this.args = args;
- this.activeClients = new AtomicInteger(0);
- }
-
- @SuppressWarnings("resource")
- public void serve()
- {
- try
- {
- serverTransport_.listen();
- }
- catch (TTransportException ttx)
- {
- logger.error("Error occurred during listening.", ttx);
- return;
- }
-
- while (!stopped)
- {
- // block until we are under max clients
- while (activeClients.get() >= args.maxWorkerThreads)
- {
- Uninterruptibles.sleepUninterruptibly(10, TimeUnit.MILLISECONDS);
- }
-
- try
- {
- TTransport client = serverTransport_.accept();
- activeClients.incrementAndGet();
- WorkerProcess wp = new WorkerProcess(client);
- executorService.execute(wp);
- }
- catch (TTransportException ttx)
- {
- if (ttx.getCause() instanceof SocketTimeoutException) // thrift sucks
- continue;
-
- if (!stopped)
- {
- logger.warn("Transport error occurred during acceptance of message.", ttx);
- }
- }
- catch (RejectedExecutionException e)
- {
- // worker thread decremented activeClients but hadn't finished exiting
- logger.trace("Dropping client connection because our limit of {} has been reached", args.maxWorkerThreads);
- continue;
- }
-
- if (activeClients.get() >= args.maxWorkerThreads)
- logger.warn("Maximum number of clients {} reached", args.maxWorkerThreads);
- }
-
- executorService.shutdown();
- // Thrift's default shutdown waits for the WorkerProcess threads to complete. We do not,
- // because doing that allows a client to hold our shutdown "hostage" by simply not sending
- // another message after stop is called (since process will block indefinitely trying to read
- // the next meessage header).
- //
- // The "right" fix would be to update thrift to set a socket timeout on client connections
- // (and tolerate unintentional timeouts until stopped is set). But this requires deep
- // changes to the code generator, so simply setting these threads to daemon (in our custom
- // CleaningThreadPool) and ignoring them after shutdown is good enough.
- //
- // Remember, our goal on shutdown is not necessarily that each client request we receive
- // gets answered first [to do that, you should redirect clients to a different coordinator
- // first], but rather (1) to make sure that for each update we ack as successful, we generate
- // hints for any non-responsive replicas, and (2) to make sure that we quickly stop
- // accepting client connections so shutdown can continue. Not waiting for the WorkerProcess
- // threads here accomplishes (2); MessagingService's shutdown method takes care of (1).
- //
- // See CASSANDRA-3335 and CASSANDRA-3727.
- }
-
- public void stop()
- {
- stopped = true;
- serverTransport_.interrupt();
- }
-
- private class WorkerProcess implements Runnable
- {
-
- /**
- * Client that this services.
- */
- private TTransport client_;
-
- /**
- * Default constructor.
- *
- * @param client Transport to process
- */
- private WorkerProcess(TTransport client)
- {
- client_ = client;
- }
-
- /**
- * Loops on processing a client forever
- */
- public void run()
- {
- TProcessor processor = null;
- TProtocol inputProtocol = null;
- TProtocol outputProtocol = null;
- SocketAddress socket = null;
- try (TTransport inputTransport = inputTransportFactory_.getTransport(client_);
- TTransport outputTransport = outputTransportFactory_.getTransport(client_))
- {
- socket = ((TCustomSocket) client_).getSocket().getRemoteSocketAddress();
- ThriftSessionManager.instance.setCurrentSocket(socket);
- processor = processorFactory_.getProcessor(client_);
-
- inputProtocol = inputProtocolFactory_.getProtocol(inputTransport);
- outputProtocol = outputProtocolFactory_.getProtocol(outputTransport);
- // we check stopped first to make sure we're not supposed to be shutting
- // down. this is necessary for graceful shutdown. (but not sufficient,
- // since process() can take arbitrarily long waiting for client input.
- // See comments at the end of serve().)
- while (!stopped && processor.process(inputProtocol, outputProtocol))
- {
- inputProtocol = inputProtocolFactory_.getProtocol(inputTransport);
- outputProtocol = outputProtocolFactory_.getProtocol(outputTransport);
- }
- }
- catch (TTransportException ttx)
- {
- // Assume the client died and continue silently
- // Log at debug to allow debugging of "frame too large" errors (see CASSANDRA-3142).
- logger.trace("Thrift transport error occurred during processing of message.", ttx);
- }
- catch (TException tx)
- {
- logger.error("Thrift error occurred during processing of message.", tx);
- }
- catch (Exception e)
- {
- JVMStabilityInspector.inspectThrowable(e);
- logger.error("Error occurred during processing of message.", e);
- }
- finally
- {
- if (socket != null)
- ThriftSessionManager.instance.connectionComplete(socket);
-
- activeClients.decrementAndGet();
- }
- }
- }
-
- public static class Factory implements TServerFactory
- {
- @SuppressWarnings("resource")
- public TServer buildTServer(Args args)
- {
- final InetSocketAddress addr = args.addr;
- TServerTransport serverTransport;
- try
- {
- final ClientEncryptionOptions clientEnc = DatabaseDescriptor.getClientEncryptionOptions();
- if (clientEnc.enabled)
- {
- logger.info("enabling encrypted thrift connections between client and server");
- TSSLTransportParameters params = new TSSLTransportParameters(clientEnc.protocol, new String[0]);
- params.setKeyStore(clientEnc.keystore, clientEnc.keystore_password);
- if (clientEnc.require_client_auth)
- {
- params.setTrustStore(clientEnc.truststore, clientEnc.truststore_password);
- params.requireClientAuth(true);
- }
- TServerSocket sslServer = TSSLTransportFactory.getServerSocket(addr.getPort(), 0, addr.getAddress(), params);
- SSLServerSocket sslServerSocket = (SSLServerSocket) sslServer.getServerSocket();
- String[] suites = SSLFactory.filterCipherSuites(sslServerSocket.getSupportedCipherSuites(), clientEnc.cipher_suites);
- sslServerSocket.setEnabledCipherSuites(suites);
- serverTransport = new TCustomServerSocket(sslServerSocket, args.keepAlive, args.sendBufferSize, args.recvBufferSize);
- }
- else
- {
- serverTransport = new TCustomServerSocket(addr, args.keepAlive, args.sendBufferSize, args.recvBufferSize, args.listenBacklog);
- }
- }
- catch (TTransportException e)
- {
- throw new RuntimeException(String.format("Unable to create thrift socket to %s:%s", addr.getAddress(), addr.getPort()), e);
- }
- // ThreadPool Server and will be invocation per connection basis...
- TThreadPoolServer.Args serverArgs = new TThreadPoolServer.Args(serverTransport)
- .minWorkerThreads(DatabaseDescriptor.getRpcMinThreads())
- .maxWorkerThreads(DatabaseDescriptor.getRpcMaxThreads())
- .inputTransportFactory(args.inTransportFactory)
- .outputTransportFactory(args.outTransportFactory)
- .inputProtocolFactory(args.tProtocolFactory)
- .outputProtocolFactory(args.tProtocolFactory)
- .processor(args.processor);
- ExecutorService executorService = new ThreadPoolExecutor(serverArgs.minWorkerThreads,
- serverArgs.maxWorkerThreads,
- 60,
- TimeUnit.SECONDS,
- new SynchronousQueue<Runnable>(),
- new NamedThreadFactory("Thrift"));
- return new CustomTThreadPoolServer(serverArgs, executorService);
- }
- }
-}
diff --git a/src/java/org/apache/cassandra/thrift/ITransportFactory.java b/src/java/org/apache/cassandra/thrift/ITransportFactory.java
deleted file mode 100644
index 7a65728..0000000
--- a/src/java/org/apache/cassandra/thrift/ITransportFactory.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-package org.apache.cassandra.thrift;
-
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.thrift.transport.TTransport;
-
-/**
- * Transport factory for establishing thrift connections from clients to a remote server.
- */
-public interface ITransportFactory
-{
- static final String PROPERTY_KEY = "cassandra.client.transport.factory";
-
- /**
- * Opens a client transport to a thrift server.
- * Example:
- *
- * <pre>
- * TTransport transport = clientTransportFactory.openTransport(address, port);
- * Cassandra.Iface client = new Cassandra.Client(new BinaryProtocol(transport));
- * </pre>
- *
- * @param host fully qualified hostname of the server
- * @param port RPC port of the server
- * @return open and ready to use transport
- * @throws Exception implementation defined; usually throws TTransportException or IOException
- * if the connection cannot be established
- */
- TTransport openTransport(String host, int port) throws Exception;
-
- /**
- * Sets an implementation defined set of options.
- * Keys in this map must conform to the set set returned by ITransportFactory#supportedOptions.
- * @param options option map
- */
- void setOptions(Map<String, String> options);
-
- /**
- * @return set of options supported by this transport factory implementation
- */
- Set<String> supportedOptions();
-}
-
diff --git a/src/java/org/apache/cassandra/thrift/SSLTransportFactory.java b/src/java/org/apache/cassandra/thrift/SSLTransportFactory.java
deleted file mode 100644
index ea74b94..0000000
--- a/src/java/org/apache/cassandra/thrift/SSLTransportFactory.java
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.thrift;
-
-import com.google.common.collect.Sets;
-import org.apache.thrift.transport.TFramedTransport;
-import org.apache.thrift.transport.TSSLTransportFactory;
-import org.apache.thrift.transport.TTransport;
-
-import java.util.Map;
-import java.util.Set;
-
-public class SSLTransportFactory implements ITransportFactory
-{
- public static final int DEFAULT_MAX_FRAME_SIZE = 15 * 1024 * 1024; // 15 MiB
-
- public static final String TRUSTSTORE = "enc.truststore";
- public static final String TRUSTSTORE_PASSWORD = "enc.truststore.password";
- public static final String KEYSTORE = "enc.keystore";
- public static final String KEYSTORE_PASSWORD = "enc.keystore.password";
- public static final String PROTOCOL = "enc.protocol";
- public static final String CIPHER_SUITES = "enc.cipher.suites";
- public static final int SOCKET_TIMEOUT = 0;
-
- private static final Set<String> SUPPORTED_OPTIONS = Sets.newHashSet(TRUSTSTORE,
- TRUSTSTORE_PASSWORD,
- KEYSTORE,
- KEYSTORE_PASSWORD,
- PROTOCOL,
- CIPHER_SUITES);
-
- private String truststore;
- private String truststorePassword;
- private String keystore;
- private String keystorePassword;
- private String protocol;
- private String[] cipherSuites;
-
- @Override
- @SuppressWarnings("resource")
- public TTransport openTransport(String host, int port) throws Exception
- {
- TSSLTransportFactory.TSSLTransportParameters params = new TSSLTransportFactory.TSSLTransportParameters(protocol, cipherSuites);
- params.setTrustStore(truststore, truststorePassword);
- if (null != keystore)
- params.setKeyStore(keystore, keystorePassword);
- TTransport trans = TSSLTransportFactory.getClientSocket(host, port, SOCKET_TIMEOUT, params);
- return new TFramedTransport(trans, DEFAULT_MAX_FRAME_SIZE);
- }
-
- @Override
- public void setOptions(Map<String, String> options)
- {
- if (options.containsKey(TRUSTSTORE))
- truststore = options.get(TRUSTSTORE);
- if (options.containsKey(TRUSTSTORE_PASSWORD))
- truststorePassword = options.get(TRUSTSTORE_PASSWORD);
- if (options.containsKey(KEYSTORE))
- keystore = options.get(KEYSTORE);
- if (options.containsKey(KEYSTORE_PASSWORD))
- keystorePassword = options.get(KEYSTORE_PASSWORD);
- if (options.containsKey(PROTOCOL))
- protocol = options.get(PROTOCOL);
- if (options.containsKey(CIPHER_SUITES))
- cipherSuites = options.get(CIPHER_SUITES).split(",");
- }
-
- @Override
- public Set<String> supportedOptions()
- {
- return SUPPORTED_OPTIONS;
- }
-}
diff --git a/src/java/org/apache/cassandra/thrift/TCustomNonblockingServerSocket.java b/src/java/org/apache/cassandra/thrift/TCustomNonblockingServerSocket.java
deleted file mode 100644
index a430721..0000000
--- a/src/java/org/apache/cassandra/thrift/TCustomNonblockingServerSocket.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.thrift;
-
-import java.net.InetSocketAddress;
-import java.net.Socket;
-import java.net.SocketException;
-
-import org.apache.thrift.transport.TNonblockingServerSocket;
-import org.apache.thrift.transport.TNonblockingSocket;
-import org.apache.thrift.transport.TTransportException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TCustomNonblockingServerSocket extends TNonblockingServerSocket
-{
- private static final Logger logger = LoggerFactory.getLogger(TCustomNonblockingServerSocket.class);
- private final boolean keepAlive;
- private final Integer sendBufferSize;
- private final Integer recvBufferSize;
-
- public TCustomNonblockingServerSocket(InetSocketAddress bindAddr, boolean keepAlive, Integer sendBufferSize, Integer recvBufferSize) throws TTransportException
- {
- super(bindAddr);
- this.keepAlive = keepAlive;
- this.sendBufferSize = sendBufferSize;
- this.recvBufferSize = recvBufferSize;
- }
-
- @Override
- @SuppressWarnings("resource")
- protected TNonblockingSocket acceptImpl() throws TTransportException
- {
- TNonblockingSocket tsocket = super.acceptImpl();
- if (tsocket == null || tsocket.getSocketChannel() == null)
- return tsocket;
- Socket socket = tsocket.getSocketChannel().socket();
- try
- {
- socket.setKeepAlive(this.keepAlive);
- }
- catch (SocketException se)
- {
- logger.warn("Failed to set keep-alive on Thrift socket.", se);
- }
-
- if (this.sendBufferSize != null)
- {
- try
- {
- socket.setSendBufferSize(this.sendBufferSize.intValue());
- }
- catch (SocketException se)
- {
- logger.warn("Failed to set send buffer size on Thrift socket.", se);
- }
- }
-
- if (this.recvBufferSize != null)
- {
- try
- {
- socket.setReceiveBufferSize(this.recvBufferSize.intValue());
- }
- catch (SocketException se)
- {
- logger.warn("Failed to set receive buffer size on Thrift socket.", se);
- }
- }
- return tsocket;
- }
-}
diff --git a/src/java/org/apache/cassandra/thrift/TCustomServerSocket.java b/src/java/org/apache/cassandra/thrift/TCustomServerSocket.java
deleted file mode 100644
index 8e27481..0000000
--- a/src/java/org/apache/cassandra/thrift/TCustomServerSocket.java
+++ /dev/null
@@ -1,189 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.thrift;
-
-
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.net.ServerSocket;
-import java.net.Socket;
-import java.net.SocketException;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.thrift.transport.TServerTransport;
-import org.apache.thrift.transport.TTransportException;
-
-/**
- * Extends Thrift's TServerSocket to allow customization of various desirable TCP properties.
- */
-public class TCustomServerSocket extends TServerTransport
-{
-
- private static final Logger logger = LoggerFactory.getLogger(TCustomServerSocket.class);
-
- /**
- * Underlying serversocket object
- */
- private ServerSocket serverSocket = null;
-
- private final boolean keepAlive;
- private final Integer sendBufferSize;
- private final Integer recvBufferSize;
-
- /**
- * Allows fine-tuning of the server socket including keep-alive, reuse of addresses, send and receive buffer sizes.
- *
- * @param bindAddr
- * @param keepAlive
- * @param sendBufferSize
- * @param recvBufferSize
- * @throws TTransportException
- */
- public TCustomServerSocket(InetSocketAddress bindAddr, boolean keepAlive, Integer sendBufferSize,
- Integer recvBufferSize, Integer listenBacklog)
- throws TTransportException
- {
- try
- {
- // Make server socket
- serverSocket = new ServerSocket();
- // Prevent 2MSL delay problem on server restarts
- serverSocket.setReuseAddress(true);
- // Bind to listening port
- serverSocket.bind(bindAddr, listenBacklog);
- }
- catch (IOException ioe)
- {
- serverSocket = null;
- throw new TTransportException("Could not create ServerSocket on address " + bindAddr + ".");
- }
-
- this.keepAlive = keepAlive;
- this.sendBufferSize = sendBufferSize;
- this.recvBufferSize = recvBufferSize;
- }
-
- public TCustomServerSocket(ServerSocket socket, boolean keepAlive, Integer sendBufferSize, Integer recvBufferSize)
- {
- this.serverSocket = socket;
- this.keepAlive = keepAlive;
- this.sendBufferSize = sendBufferSize;
- this.recvBufferSize = recvBufferSize;
- }
-
- @Override
- @SuppressWarnings("resource")
- protected TCustomSocket acceptImpl() throws TTransportException
- {
-
- if (serverSocket == null)
- throw new TTransportException(TTransportException.NOT_OPEN, "No underlying server socket.");
-
- TCustomSocket tsocket = null;
- Socket socket = null;
- try
- {
- socket = serverSocket.accept();
- tsocket = new TCustomSocket(socket);
- tsocket.setTimeout(0);
- }
- catch (IOException iox)
- {
- throw new TTransportException(iox);
- }
-
- try
- {
- socket.setKeepAlive(this.keepAlive);
- }
- catch (SocketException se)
- {
- logger.warn("Failed to set keep-alive on Thrift socket.", se);
- }
-
- if (this.sendBufferSize != null)
- {
- try
- {
- socket.setSendBufferSize(this.sendBufferSize.intValue());
- }
- catch (SocketException se)
- {
- logger.warn("Failed to set send buffer size on Thrift socket.", se);
- }
- }
-
- if (this.recvBufferSize != null)
- {
- try
- {
- socket.setReceiveBufferSize(this.recvBufferSize.intValue());
- }
- catch (SocketException se)
- {
- logger.warn("Failed to set receive buffer size on Thrift socket.", se);
- }
- }
-
- return tsocket;
- }
-
- @Override
- public void listen() throws TTransportException
- {
- // Make sure not to block on accept
- if (serverSocket != null)
- {
- try
- {
- serverSocket.setSoTimeout(100);
- }
- catch (SocketException sx)
- {
- logger.error("Could not set socket timeout.", sx);
- }
- }
- }
-
- @Override
- public void close()
- {
- if (serverSocket != null)
- {
- try
- {
- serverSocket.close();
- }
- catch (IOException iox)
- {
- logger.warn("Could not close server socket.", iox);
- }
- serverSocket = null;
- }
- }
-
- @Override
- public void interrupt()
- {
- // The thread-safeness of this is dubious, but Java documentation suggests
- // that it is safe to do this from a different thread context
- close();
- }
-}
diff --git a/src/java/org/apache/cassandra/thrift/TCustomSocket.java b/src/java/org/apache/cassandra/thrift/TCustomSocket.java
deleted file mode 100644
index 08a9770..0000000
--- a/src/java/org/apache/cassandra/thrift/TCustomSocket.java
+++ /dev/null
@@ -1,246 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.thrift;
-
-
-import java.io.BufferedInputStream;
-import java.io.BufferedOutputStream;
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.net.Socket;
-import java.net.SocketException;
-
-import org.apache.thrift.transport.TIOStreamTransport;
-import org.apache.thrift.transport.TTransportException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Socket implementation of the TTransport interface.
- *
- * Adds socket buffering
- *
- */
-public class TCustomSocket extends TIOStreamTransport
-{
-
- private static final Logger LOGGER = LoggerFactory.getLogger(TCustomSocket.class.getName());
-
- /**
- * Wrapped Socket object
- */
- private Socket socket = null;
-
- /**
- * Remote host
- */
- private String host = null;
-
- /**
- * Remote port
- */
- private int port = 0;
-
- /**
- * Socket timeout
- */
- private int timeout = 0;
-
- /**
- * Constructor that takes an already created socket.
- *
- * @param socket Already created socket object
- * @throws TTransportException if there is an error setting up the streams
- */
- public TCustomSocket(Socket socket) throws TTransportException
- {
- this.socket = socket;
- try
- {
- socket.setSoLinger(false, 0);
- socket.setTcpNoDelay(true);
- }
- catch (SocketException sx)
- {
- LOGGER.warn("Could not configure socket.", sx);
- }
-
- if (isOpen())
- {
- try
- {
- inputStream_ = new BufferedInputStream(socket.getInputStream(), 1024);
- outputStream_ = new BufferedOutputStream(socket.getOutputStream(), 1024);
- }
- catch (IOException iox)
- {
- close();
- throw new TTransportException(TTransportException.NOT_OPEN, iox);
- }
- }
- }
-
- /**
- * Creates a new unconnected socket that will connect to the given host
- * on the given port.
- *
- * @param host Remote host
- * @param port Remote port
- */
- public TCustomSocket(String host, int port)
- {
- this(host, port, 0);
- }
-
- /**
- * Creates a new unconnected socket that will connect to the given host
- * on the given port.
- *
- * @param host Remote host
- * @param port Remote port
- * @param timeout Socket timeout
- */
- public TCustomSocket(String host, int port, int timeout)
- {
- this.host = host;
- this.port = port;
- this.timeout = timeout;
- initSocket();
- }
-
- /**
- * Initializes the socket object
- */
- private void initSocket()
- {
- socket = new Socket();
- try
- {
- socket.setSoLinger(false, 0);
- socket.setTcpNoDelay(true);
- socket.setSoTimeout(timeout);
- }
- catch (SocketException sx)
- {
- LOGGER.error("Could not configure socket.", sx);
- }
- }
-
- /**
- * Sets the socket timeout
- *
- * @param timeout Milliseconds timeout
- */
- public void setTimeout(int timeout)
- {
- this.timeout = timeout;
- try
- {
- socket.setSoTimeout(timeout);
- }
- catch (SocketException sx)
- {
- LOGGER.warn("Could not set socket timeout.", sx);
- }
- }
-
- /**
- * Returns a reference to the underlying socket.
- */
- public Socket getSocket()
- {
- if (socket == null)
- {
- initSocket();
- }
- return socket;
- }
-
- /**
- * Checks whether the socket is connected.
- */
- public boolean isOpen()
- {
- if (socket == null)
- {
- return false;
- }
- return socket.isConnected();
- }
-
- /**
- * Connects the socket, creating a new socket object if necessary.
- */
- public void open() throws TTransportException
- {
- if (isOpen())
- {
- throw new TTransportException(TTransportException.ALREADY_OPEN, "Socket already connected.");
- }
-
- if (host.length() == 0)
- {
- throw new TTransportException(TTransportException.NOT_OPEN, "Cannot open null host.");
- }
- if (port <= 0)
- {
- throw new TTransportException(TTransportException.NOT_OPEN, "Cannot open without port.");
- }
-
- if (socket == null)
- {
- initSocket();
- }
-
- try
- {
- socket.connect(new InetSocketAddress(host, port), timeout);
- inputStream_ = new BufferedInputStream(socket.getInputStream(), 1024);
- outputStream_ = new BufferedOutputStream(socket.getOutputStream(), 1024);
- }
- catch (IOException iox)
- {
- close();
- throw new TTransportException(TTransportException.NOT_OPEN, iox);
- }
- }
-
- /**
- * Closes the socket.
- */
- public void close()
- {
- // Close the underlying streams
- super.close();
-
- // Close the socket
- if (socket != null)
- {
- try
- {
- socket.close();
- }
- catch (IOException iox)
- {
- LOGGER.warn("Could not close socket.", iox);
- }
- socket = null;
- }
- }
-
-}
diff --git a/src/java/org/apache/cassandra/thrift/TFramedTransportFactory.java b/src/java/org/apache/cassandra/thrift/TFramedTransportFactory.java
deleted file mode 100644
index 7bf0b96..0000000
--- a/src/java/org/apache/cassandra/thrift/TFramedTransportFactory.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-package org.apache.cassandra.thrift;
-
-import java.util.Collections;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.thrift.transport.TFramedTransport;
-import org.apache.thrift.transport.TSocket;
-import org.apache.thrift.transport.TTransport;
-import org.apache.thrift.transport.TTransportException;
-
-public class TFramedTransportFactory implements ITransportFactory
-{
- private static final String THRIFT_FRAMED_TRANSPORT_SIZE_IN_MB = "cassandra.thrift.framed.size_mb";
- private int thriftFramedTransportSizeMb = 15; // 15Mb is the default for C* & Hadoop ConfigHelper
-
- @SuppressWarnings("resource")
- public TTransport openTransport(String host, int port) throws TTransportException
- {
- TSocket socket = new TSocket(host, port);
- TTransport transport = new TFramedTransport(socket, thriftFramedTransportSizeMb * 1024 * 1024);
- transport.open();
- return transport;
- }
-
- public void setOptions(Map<String, String> options)
- {
- if (options.containsKey(THRIFT_FRAMED_TRANSPORT_SIZE_IN_MB))
- thriftFramedTransportSizeMb = Integer.parseInt(options.get(THRIFT_FRAMED_TRANSPORT_SIZE_IN_MB));
- }
-
- public Set<String> supportedOptions()
- {
- return Collections.singleton(THRIFT_FRAMED_TRANSPORT_SIZE_IN_MB);
- }
-}
diff --git a/src/java/org/apache/cassandra/thrift/THsHaDisruptorServer.java b/src/java/org/apache/cassandra/thrift/THsHaDisruptorServer.java
deleted file mode 100644
index 37bc440..0000000
--- a/src/java/org/apache/cassandra/thrift/THsHaDisruptorServer.java
+++ /dev/null
@@ -1,109 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.cassandra.thrift;
-
-import java.net.InetSocketAddress;
-import java.util.concurrent.SynchronousQueue;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-
-import com.thinkaurelius.thrift.Message;
-import com.thinkaurelius.thrift.TDisruptorServer;
-import org.apache.cassandra.concurrent.JMXEnabledThreadPoolExecutor;
-import org.apache.cassandra.concurrent.NamedThreadFactory;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.thrift.server.TServer;
-import org.apache.thrift.transport.TNonblockingServerTransport;
-import org.apache.thrift.transport.TNonblockingSocket;
-import org.apache.thrift.transport.TTransportException;
-
-public class THsHaDisruptorServer extends TDisruptorServer
-{
- private static final Logger logger = LoggerFactory.getLogger(THsHaDisruptorServer.class.getName());
-
- /**
- * All the arguments to Non Blocking Server will apply here. In addition,
- * executor pool will be responsible for creating the internal threads which
- * will process the data. threads for selection usually are equal to the
- * number of cpu's
- */
- public THsHaDisruptorServer(Args args)
- {
- super(args);
- logger.info("Starting up {}", this);
- }
-
- @Override
- protected void beforeInvoke(Message buffer)
- {
- TNonblockingSocket socket = (TNonblockingSocket) buffer.transport;
- ThriftSessionManager.instance.setCurrentSocket(socket.getSocketChannel().socket().getRemoteSocketAddress());
- }
-
- public void beforeClose(Message buffer)
- {
- TNonblockingSocket socket = (TNonblockingSocket) buffer.transport;
- ThriftSessionManager.instance.connectionComplete(socket.getSocketChannel().socket().getRemoteSocketAddress());
- }
-
- public static class Factory implements TServerFactory
- {
- @SuppressWarnings("resource")
- public TServer buildTServer(Args args)
- {
- if (DatabaseDescriptor.getClientEncryptionOptions().enabled)
- throw new RuntimeException("Client SSL is not supported for non-blocking sockets (hsha). Please remove client ssl from the configuration.");
-
- final InetSocketAddress addr = args.addr;
- TNonblockingServerTransport serverTransport;
- try
- {
- serverTransport = new TCustomNonblockingServerSocket(addr, args.keepAlive, args.sendBufferSize, args.recvBufferSize);
- }
- catch (TTransportException e)
- {
- throw new RuntimeException(String.format("Unable to create thrift socket to %s:%s", addr.getAddress(), addr.getPort()), e);
- }
-
- ThreadPoolExecutor invoker = new JMXEnabledThreadPoolExecutor(DatabaseDescriptor.getRpcMinThreads(),
- DatabaseDescriptor.getRpcMaxThreads(),
- 60L,
- TimeUnit.SECONDS,
- new SynchronousQueue<Runnable>(),
- new NamedThreadFactory("RPC-Thread"), "RPC-THREAD-POOL");
-
- com.thinkaurelius.thrift.util.TBinaryProtocol.Factory protocolFactory = new com.thinkaurelius.thrift.util.TBinaryProtocol.Factory(true, true);
-
- TDisruptorServer.Args serverArgs = new TDisruptorServer.Args(serverTransport).useHeapBasedAllocation(true)
- .inputTransportFactory(args.inTransportFactory)
- .outputTransportFactory(args.outTransportFactory)
- .inputProtocolFactory(protocolFactory)
- .outputProtocolFactory(protocolFactory)
- .processor(args.processor)
- .maxFrameSizeInBytes(DatabaseDescriptor.getThriftFramedTransportSize())
- .invocationExecutor(invoker)
- .alwaysReallocateBuffers(true);
-
- return new THsHaDisruptorServer(serverArgs);
- }
- }
-}
diff --git a/src/java/org/apache/cassandra/thrift/TServerCustomFactory.java b/src/java/org/apache/cassandra/thrift/TServerCustomFactory.java
deleted file mode 100644
index 5a272dd..0000000
--- a/src/java/org/apache/cassandra/thrift/TServerCustomFactory.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.cassandra.thrift;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.thrift.server.TServer;
-
-/**
- * Helper implementation to create a thrift TServer based on one of the common types we support (sync, hsha),
- * or a custom type by setting the fully qualified java class name in the rpc_server_type setting.
- */
-public class TServerCustomFactory implements TServerFactory
-{
- private static Logger logger = LoggerFactory.getLogger(TServerCustomFactory.class);
- private final String serverType;
-
- public TServerCustomFactory(String serverType)
- {
- assert serverType != null;
- this.serverType = serverType;
- }
-
- public TServer buildTServer(TServerFactory.Args args)
- {
- TServer server;
- if (ThriftServer.ThriftServerType.SYNC.equalsIgnoreCase(serverType))
- {
- server = new CustomTThreadPoolServer.Factory().buildTServer(args);
- }
- else if(ThriftServer.ThriftServerType.ASYNC.equalsIgnoreCase(serverType))
- {
- server = new CustomTNonBlockingServer.Factory().buildTServer(args);
- logger.info("Using non-blocking/asynchronous thrift server on {} : {}", args.addr.getHostName(), args.addr.getPort());
- }
- else if(ThriftServer.ThriftServerType.HSHA.equalsIgnoreCase(serverType))
- {
- server = new THsHaDisruptorServer.Factory().buildTServer(args);
- logger.info("Using custom half-sync/half-async thrift server on {} : {}", args.addr.getHostName(), args.addr.getPort());
- }
- else
- {
- TServerFactory serverFactory;
- try
- {
- serverFactory = (TServerFactory) Class.forName(serverType).newInstance();
- }
- catch (Exception e)
- {
- throw new RuntimeException("Failed to instantiate server factory:" + serverType, e);
- }
- server = serverFactory.buildTServer(args);
- logger.info("Using custom thrift server {} on {} : {}", server.getClass().getName(), args.addr.getHostName(), args.addr.getPort());
- }
- return server;
- }
-}
diff --git a/src/java/org/apache/cassandra/thrift/TServerFactory.java b/src/java/org/apache/cassandra/thrift/TServerFactory.java
deleted file mode 100644
index 09014ce..0000000
--- a/src/java/org/apache/cassandra/thrift/TServerFactory.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.cassandra.thrift;
-
-import java.net.InetSocketAddress;
-
-import org.apache.thrift.TProcessor;
-import org.apache.thrift.protocol.TProtocolFactory;
-import org.apache.thrift.server.TServer;
-import org.apache.thrift.transport.TTransportFactory;
-
-public interface TServerFactory
-{
- TServer buildTServer(Args args);
-
- public static class Args
- {
- public InetSocketAddress addr;
- public Integer listenBacklog;
- public TProcessor processor;
- public TProtocolFactory tProtocolFactory;
- public TTransportFactory inTransportFactory;
- public TTransportFactory outTransportFactory;
- public Integer sendBufferSize;
- public Integer recvBufferSize;
- public boolean keepAlive;
- }
-}
diff --git a/src/java/org/apache/cassandra/thrift/ThriftClientState.java b/src/java/org/apache/cassandra/thrift/ThriftClientState.java
deleted file mode 100644
index 6a3c50f..0000000
--- a/src/java/org/apache/cassandra/thrift/ThriftClientState.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.thrift;
-
-import java.net.InetSocketAddress;
-
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.service.ClientState;
-import org.apache.cassandra.service.QueryState;
-
-/**
- * ClientState used by thrift that also provide a QueryState.
- *
- * Thrift is intrinsically synchronous so there could be only one query per
- * client at a given time. So ClientState and QueryState can be merge into the
- * same object.
- */
-public class ThriftClientState extends ClientState
-{
- private final QueryState queryState;
-
- public ThriftClientState(InetSocketAddress remoteAddress)
- {
- super(remoteAddress);
- this.queryState = new QueryState(this);
- }
-
- public QueryState getQueryState()
- {
- return queryState;
- }
-
- public String getSchedulingValue()
- {
- switch(DatabaseDescriptor.getRequestSchedulerId())
- {
- case keyspace: return getRawKeyspace();
- }
- return "default";
- }
-}
diff --git a/src/java/org/apache/cassandra/thrift/ThriftConversion.java b/src/java/org/apache/cassandra/thrift/ThriftConversion.java
deleted file mode 100644
index 0f40d22..0000000
--- a/src/java/org/apache/cassandra/thrift/ThriftConversion.java
+++ /dev/null
@@ -1,734 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.thrift;
-
-import java.util.*;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Strings;
-import com.google.common.collect.Maps;
-
-import org.apache.cassandra.config.*;
-import org.apache.cassandra.cql3.ColumnIdentifier;
-import org.apache.cassandra.cql3.Operator;
-import org.apache.cassandra.cql3.SuperColumnCompatibility;
-import org.apache.cassandra.cql3.statements.IndexTarget;
-import org.apache.cassandra.db.CompactTables;
-import org.apache.cassandra.db.LegacyLayout;
-import org.apache.cassandra.db.WriteType;
-import org.apache.cassandra.db.compaction.AbstractCompactionStrategy;
-import org.apache.cassandra.db.filter.RowFilter;
-import org.apache.cassandra.db.marshal.*;
-import org.apache.cassandra.exceptions.*;
-import org.apache.cassandra.index.TargetParser;
-import org.apache.cassandra.io.compress.ICompressor;
-import org.apache.cassandra.locator.AbstractReplicationStrategy;
-import org.apache.cassandra.locator.LocalStrategy;
-import org.apache.cassandra.schema.*;
-import org.apache.cassandra.serializers.MarshalException;
-import org.apache.cassandra.utils.ByteBufferUtil;
-import org.apache.cassandra.utils.Pair;
-import org.apache.cassandra.utils.UUIDGen;
-
-/**
- * Static utility methods to convert internal structure to and from thrift ones.
- */
-public class ThriftConversion
-{
- public static org.apache.cassandra.db.ConsistencyLevel fromThrift(ConsistencyLevel cl)
- {
- switch (cl)
- {
- case ANY: return org.apache.cassandra.db.ConsistencyLevel.ANY;
- case ONE: return org.apache.cassandra.db.ConsistencyLevel.ONE;
- case TWO: return org.apache.cassandra.db.ConsistencyLevel.TWO;
- case THREE: return org.apache.cassandra.db.ConsistencyLevel.THREE;
- case QUORUM: return org.apache.cassandra.db.ConsistencyLevel.QUORUM;
- case ALL: return org.apache.cassandra.db.ConsistencyLevel.ALL;
- case LOCAL_QUORUM: return org.apache.cassandra.db.ConsistencyLevel.LOCAL_QUORUM;
- case EACH_QUORUM: return org.apache.cassandra.db.ConsistencyLevel.EACH_QUORUM;
- case SERIAL: return org.apache.cassandra.db.ConsistencyLevel.SERIAL;
- case LOCAL_SERIAL: return org.apache.cassandra.db.ConsistencyLevel.LOCAL_SERIAL;
- case LOCAL_ONE: return org.apache.cassandra.db.ConsistencyLevel.LOCAL_ONE;
- }
- throw new AssertionError();
- }
-
- public static ConsistencyLevel toThrift(org.apache.cassandra.db.ConsistencyLevel cl)
- {
- switch (cl)
- {
- case ANY: return ConsistencyLevel.ANY;
- case ONE: return ConsistencyLevel.ONE;
- case TWO: return ConsistencyLevel.TWO;
- case THREE: return ConsistencyLevel.THREE;
- case QUORUM: return ConsistencyLevel.QUORUM;
- case ALL: return ConsistencyLevel.ALL;
- case LOCAL_QUORUM: return ConsistencyLevel.LOCAL_QUORUM;
- case EACH_QUORUM: return ConsistencyLevel.EACH_QUORUM;
- case SERIAL: return ConsistencyLevel.SERIAL;
- case LOCAL_SERIAL: return ConsistencyLevel.LOCAL_SERIAL;
- case LOCAL_ONE: return ConsistencyLevel.LOCAL_ONE;
- }
- throw new AssertionError();
- }
-
- // We never return, but returning a RuntimeException allows to write "throw rethrow(e)" without java complaining
- // for methods that have a return value.
- public static RuntimeException rethrow(RequestExecutionException e) throws UnavailableException, TimedOutException
- {
- if (e instanceof RequestFailureException)
- throw toThrift((RequestFailureException)e);
- else if (e instanceof RequestTimeoutException)
- throw toThrift((RequestTimeoutException)e);
- else
- throw new UnavailableException();
- }
-
- public static InvalidRequestException toThrift(RequestValidationException e)
- {
- return new InvalidRequestException(e.getMessage());
- }
-
- public static UnavailableException toThrift(org.apache.cassandra.exceptions.UnavailableException e)
- {
- return new UnavailableException();
- }
-
- public static AuthenticationException toThrift(org.apache.cassandra.exceptions.AuthenticationException e)
- {
- return new AuthenticationException(e.getMessage());
- }
-
- public static TimedOutException toThrift(RequestTimeoutException e)
- {
- TimedOutException toe = new TimedOutException();
- if (e instanceof WriteTimeoutException)
- {
- WriteTimeoutException wte = (WriteTimeoutException)e;
- toe.setAcknowledged_by(wte.received);
- if (wte.writeType == WriteType.BATCH_LOG)
- toe.setAcknowledged_by_batchlog(false);
- else if (wte.writeType == WriteType.BATCH)
- toe.setAcknowledged_by_batchlog(true);
- else if (wte.writeType == WriteType.CAS)
- toe.setPaxos_in_progress(true);
- }
- return toe;
- }
-
- // Thrift does not support RequestFailureExceptions, so we translate them into timeouts
- public static TimedOutException toThrift(RequestFailureException e)
- {
- return new TimedOutException();
- }
-
- public static RowFilter rowFilterFromThrift(CFMetaData metadata, List<IndexExpression> exprs)
- {
- if (exprs == null || exprs.isEmpty())
- return RowFilter.NONE;
-
- RowFilter converted = RowFilter.forThrift(exprs.size());
- for (IndexExpression expr : exprs)
- converted.addThriftExpression(metadata, expr.column_name, Operator.valueOf(expr.op.name()), expr.value);
- return converted;
- }
-
- public static KeyspaceMetadata fromThrift(KsDef ksd, CFMetaData... cfDefs) throws ConfigurationException
- {
- Class<? extends AbstractReplicationStrategy> cls = AbstractReplicationStrategy.getClass(ksd.strategy_class);
- if (cls.equals(LocalStrategy.class))
- throw new ConfigurationException("Unable to use given strategy class: LocalStrategy is reserved for internal use.");
-
- Map<String, String> replicationMap = new HashMap<>();
- if (ksd.strategy_options != null)
- replicationMap.putAll(ksd.strategy_options);
- replicationMap.put(ReplicationParams.CLASS, cls.getName());
-
- return KeyspaceMetadata.create(ksd.name, KeyspaceParams.create(ksd.durable_writes, replicationMap), Tables.of(cfDefs));
- }
-
- public static KsDef toThrift(KeyspaceMetadata ksm)
- {
- List<CfDef> cfDefs = new ArrayList<>();
- for (CFMetaData cfm : ksm.tables) // do not include views
- if (cfm.isThriftCompatible()) // Don't expose CF that cannot be correctly handle by thrift; see CASSANDRA-4377 for further details
- cfDefs.add(toThrift(cfm));
-
- KsDef ksdef = new KsDef(ksm.name, ksm.params.replication.klass.getName(), cfDefs);
- ksdef.setStrategy_options(ksm.params.replication.options);
- ksdef.setDurable_writes(ksm.params.durableWrites);
-
- return ksdef;
- }
-
- public static CFMetaData fromThrift(CfDef cf_def)
- throws org.apache.cassandra.exceptions.InvalidRequestException, ConfigurationException
- {
- // This is a creation: the table is dense if it doesn't define any column_metadata
- boolean isDense = cf_def.column_metadata == null || cf_def.column_metadata.isEmpty();
- return internalFromThrift(cf_def, true, Collections.<ColumnDefinition>emptyList(), isDense);
- }
-
- public static CFMetaData fromThriftForUpdate(CfDef cf_def, CFMetaData toUpdate)
- throws org.apache.cassandra.exceptions.InvalidRequestException, ConfigurationException
- {
- return internalFromThrift(cf_def, false, toUpdate.allColumns(), toUpdate.isDense());
- }
-
- private static boolean isSuper(String thriftColumnType)
- throws org.apache.cassandra.exceptions.InvalidRequestException
- {
- switch (thriftColumnType.toLowerCase(Locale.ENGLISH))
- {
- case "standard": return false;
- case "super": return true;
- default: throw new org.apache.cassandra.exceptions.InvalidRequestException("Invalid column type " + thriftColumnType);
- }
- }
-
- /**
- * Convert a thrift CfDef.
- * <p>,
- * This is used both for creation and update of CF.
- *
- * @param cf_def the thrift CfDef to convert.
- * @param isCreation whether that is a new table creation or not.
- * @param previousCQLMetadata if it is not a table creation, the previous
- * definitions of the tables (which we use to preserve the CQL metadata).
- * If it is a table creation, this will be empty.
- * @param isDense whether the table is dense or not.
- *
- * @return the converted table definition.
- */
- private static CFMetaData internalFromThrift(CfDef cf_def,
- boolean isCreation,
- Collection<ColumnDefinition> previousCQLMetadata,
- boolean isDense)
- throws org.apache.cassandra.exceptions.InvalidRequestException, ConfigurationException
- {
- applyImplicitDefaults(cf_def);
-
- try
- {
- boolean isSuper = isSuper(cf_def.column_type);
- AbstractType<?> rawComparator = TypeParser.parse(cf_def.comparator_type);
- AbstractType<?> subComparator = isSuper
- ? cf_def.subcomparator_type == null ? BytesType.instance : TypeParser.parse(cf_def.subcomparator_type)
- : null;
-
- AbstractType<?> keyValidator = cf_def.isSetKey_validation_class() ? TypeParser.parse(cf_def.key_validation_class) : BytesType.instance;
- AbstractType<?> defaultValidator = TypeParser.parse(cf_def.default_validation_class);
-
- // Convert the definitions from the input CfDef
- List<ColumnDefinition> defs = fromThrift(cf_def.keyspace, cf_def.name, rawComparator, subComparator, cf_def.column_metadata);
-
- // Add the keyAlias if there is one, since that's a CQL metadata that thrift can actually change (for
- // historical reasons)
- boolean hasKeyAlias = cf_def.isSetKey_alias() && keyValidator != null && !(keyValidator instanceof CompositeType);
- if (hasKeyAlias)
- defs.add(ColumnDefinition.partitionKeyDef(cf_def.keyspace, cf_def.name, UTF8Type.instance.getString(cf_def.key_alias), keyValidator, 0));
-
- // Now add any CQL metadata that we want to copy, skipping the keyAlias if there was one
- for (ColumnDefinition def : previousCQLMetadata)
- {
- // isPartOfCellName basically means 'is not just a CQL metadata'
- if (def.isPartOfCellName(false, isSuper))
- continue;
-
- if (def.kind == ColumnDefinition.Kind.PARTITION_KEY && hasKeyAlias)
- continue;
-
- defs.add(def);
- }
-
- UUID cfId = Schema.instance.getId(cf_def.keyspace, cf_def.name);
- if (cfId == null)
- cfId = UUIDGen.getTimeUUID();
-
- boolean isCompound = !isSuper && (rawComparator instanceof CompositeType);
- boolean isCounter = defaultValidator instanceof CounterColumnType;
-
- // If it's a thrift table creation, adds the default CQL metadata for the new table
- if (isCreation)
- {
- addDefaultCQLMetadata(defs,
- cf_def.keyspace,
- cf_def.name,
- hasKeyAlias ? null : keyValidator,
- rawComparator,
- subComparator,
- defaultValidator,
- isDense);
- }
-
- // We do not allow Thrift views, so we always set it to false
- boolean isView = false;
-
- CFMetaData newCFMD = CFMetaData.create(cf_def.keyspace,
- cf_def.name,
- cfId,
- isDense,
- isCompound,
- isSuper,
- isCounter,
- isView,
- defs,
- DatabaseDescriptor.getPartitioner());
-
- // Convert any secondary indexes defined in the thrift column_metadata
- newCFMD.indexes(indexDefsFromThrift(newCFMD,
- cf_def.keyspace,
- cf_def.name,
- rawComparator,
- subComparator,
- cf_def.column_metadata));
-
- if (cf_def.isSetGc_grace_seconds())
- newCFMD.gcGraceSeconds(cf_def.gc_grace_seconds);
-
- newCFMD.compaction(compactionParamsFromThrift(cf_def));
-
- if (cf_def.isSetBloom_filter_fp_chance())
- newCFMD.bloomFilterFpChance(cf_def.bloom_filter_fp_chance);
- if (cf_def.isSetMemtable_flush_period_in_ms())
- newCFMD.memtableFlushPeriod(cf_def.memtable_flush_period_in_ms);
- if (cf_def.isSetCaching() || cf_def.isSetCells_per_row_to_cache())
- newCFMD.caching(cachingFromThrift(cf_def.caching, cf_def.cells_per_row_to_cache));
- if (cf_def.isSetRead_repair_chance())
- newCFMD.readRepairChance(cf_def.read_repair_chance);
- if (cf_def.isSetDefault_time_to_live())
- newCFMD.defaultTimeToLive(cf_def.default_time_to_live);
- if (cf_def.isSetDclocal_read_repair_chance())
- newCFMD.dcLocalReadRepairChance(cf_def.dclocal_read_repair_chance);
- if (cf_def.isSetMin_index_interval())
- newCFMD.minIndexInterval(cf_def.min_index_interval);
- if (cf_def.isSetMax_index_interval())
- newCFMD.maxIndexInterval(cf_def.max_index_interval);
- if (cf_def.isSetSpeculative_retry())
- newCFMD.speculativeRetry(SpeculativeRetryParam.fromString(cf_def.speculative_retry));
- if (cf_def.isSetTriggers())
- newCFMD.triggers(triggerDefinitionsFromThrift(cf_def.triggers));
- if (cf_def.isSetComment())
- newCFMD.comment(cf_def.comment);
- if (cf_def.isSetCompression_options())
- newCFMD.compression(compressionParametersFromThrift(cf_def.compression_options));
-
- return newCFMD;
- }
- catch (SyntaxException | MarshalException e)
- {
- throw new ConfigurationException(e.getMessage());
- }
- }
-
- @SuppressWarnings("unchecked")
- private static CompactionParams compactionParamsFromThrift(CfDef cf_def)
- {
- Class<? extends AbstractCompactionStrategy> klass =
- CFMetaData.createCompactionStrategy(cf_def.compaction_strategy);
- Map<String, String> options = new HashMap<>(cf_def.compaction_strategy_options);
-
- int minThreshold = cf_def.min_compaction_threshold;
- int maxThreshold = cf_def.max_compaction_threshold;
-
- if (CompactionParams.supportsThresholdParams(klass))
- {
- options.putIfAbsent(CompactionParams.Option.MIN_THRESHOLD.toString(), Integer.toString(minThreshold));
- options.putIfAbsent(CompactionParams.Option.MAX_THRESHOLD.toString(), Integer.toString(maxThreshold));
- }
-
- return CompactionParams.create(klass, options);
- }
-
- private static CompressionParams compressionParametersFromThrift(Map<String, String> compression_options)
- {
- CompressionParams compressionParameter = CompressionParams.fromMap(compression_options);
- compressionParameter.validate();
- return compressionParameter;
- }
-
- private static void addDefaultCQLMetadata(Collection<ColumnDefinition> defs,
- String ks,
- String cf,
- AbstractType<?> keyValidator,
- AbstractType<?> comparator,
- AbstractType<?> subComparator,
- AbstractType<?> defaultValidator,
- boolean isDense)
- {
- CompactTables.DefaultNames names = CompactTables.defaultNameGenerator(defs);
- if (keyValidator != null)
- {
- if (keyValidator instanceof CompositeType)
- {
- List<AbstractType<?>> subTypes = ((CompositeType)keyValidator).types;
- for (int i = 0; i < subTypes.size(); i++)
- defs.add(ColumnDefinition.partitionKeyDef(ks, cf, names.defaultPartitionKeyName(), subTypes.get(i), i));
- }
- else
- {
- defs.add(ColumnDefinition.partitionKeyDef(ks, cf, names.defaultPartitionKeyName(), keyValidator, 0));
- }
- }
-
- if (subComparator != null)
- {
- // SuperColumn tables: we use a special map to hold dynamic values within a given super column
- defs.add(ColumnDefinition.clusteringDef(ks, cf, names.defaultClusteringName(), comparator, 0));
- defs.add(ColumnDefinition.regularDef(ks, cf, SuperColumnCompatibility.SUPER_COLUMN_MAP_COLUMN_STR, MapType.getInstance(subComparator, defaultValidator, true)));
- if (isDense)
- {
- defs.add(ColumnDefinition.clusteringDef(ks, cf, names.defaultClusteringName(), subComparator, 1));
- defs.add(ColumnDefinition.regularDef(ks, cf, names.defaultCompactValueName(), defaultValidator));
- }
- }
- else
- {
- List<AbstractType<?>> subTypes = comparator instanceof CompositeType
- ? ((CompositeType)comparator).types
- : Collections.<AbstractType<?>>singletonList(comparator);
-
- for (int i = 0; i < subTypes.size(); i++)
- defs.add(ColumnDefinition.clusteringDef(ks, cf, names.defaultClusteringName(), subTypes.get(i), i));
-
- defs.add(ColumnDefinition.regularDef(ks, cf, names.defaultCompactValueName(), defaultValidator));
- }
- }
-
- /* applies implicit defaults to cf definition. useful in updates */
- @SuppressWarnings("deprecation")
- private static void applyImplicitDefaults(org.apache.cassandra.thrift.CfDef cf_def)
- {
- if (!cf_def.isSetComment())
- cf_def.setComment("");
- if (!cf_def.isSetMin_compaction_threshold())
- cf_def.setMin_compaction_threshold(CompactionParams.DEFAULT_MIN_THRESHOLD);
- if (!cf_def.isSetMax_compaction_threshold())
- cf_def.setMax_compaction_threshold(CompactionParams.DEFAULT_MAX_THRESHOLD);
- if (!cf_def.isSetCompaction_strategy())
- cf_def.setCompaction_strategy(CompactionParams.DEFAULT.klass().getSimpleName());
- if (!cf_def.isSetCompaction_strategy_options())
- cf_def.setCompaction_strategy_options(Collections.emptyMap());
- if (!cf_def.isSetCompression_options())
- cf_def.setCompression_options(Collections.singletonMap(CompressionParams.SSTABLE_COMPRESSION, CompressionParams.DEFAULT.klass().getCanonicalName()));
- if (!cf_def.isSetDefault_time_to_live())
- cf_def.setDefault_time_to_live(TableParams.DEFAULT_DEFAULT_TIME_TO_LIVE);
- if (!cf_def.isSetDclocal_read_repair_chance())
- cf_def.setDclocal_read_repair_chance(TableParams.DEFAULT_DCLOCAL_READ_REPAIR_CHANCE);
-
- // if index_interval was set, use that for the min_index_interval default
- if (!cf_def.isSetMin_index_interval())
- {
- if (cf_def.isSetIndex_interval())
- cf_def.setMin_index_interval(cf_def.getIndex_interval());
- else
- cf_def.setMin_index_interval(TableParams.DEFAULT_MIN_INDEX_INTERVAL);
- }
-
- if (!cf_def.isSetMax_index_interval())
- {
- // ensure the max is at least as large as the min
- cf_def.setMax_index_interval(Math.max(cf_def.min_index_interval, TableParams.DEFAULT_MAX_INDEX_INTERVAL));
- }
- }
-
- public static CfDef toThrift(CFMetaData cfm)
- {
- CfDef def = new CfDef(cfm.ksName, cfm.cfName);
- def.setColumn_type(cfm.isSuper() ? "Super" : "Standard");
-
- if (cfm.isSuper())
- {
- def.setComparator_type(cfm.comparator.subtype(0).toString());
- def.setSubcomparator_type(cfm.thriftColumnNameType().toString());
- }
- else
- {
- def.setComparator_type(LegacyLayout.makeLegacyComparator(cfm).toString());
- }
-
- def.setComment(cfm.params.comment);
- def.setRead_repair_chance(cfm.params.readRepairChance);
- def.setDclocal_read_repair_chance(cfm.params.dcLocalReadRepairChance);
- def.setGc_grace_seconds(cfm.params.gcGraceSeconds);
- def.setDefault_validation_class(cfm.makeLegacyDefaultValidator().toString());
- def.setKey_validation_class(cfm.getKeyValidator().toString());
- def.setMin_compaction_threshold(cfm.params.compaction.minCompactionThreshold());
- def.setMax_compaction_threshold(cfm.params.compaction.maxCompactionThreshold());
- // We only return the alias if only one is set since thrift don't know about multiple key aliases
- if (cfm.partitionKeyColumns().size() == 1)
- def.setKey_alias(cfm.partitionKeyColumns().get(0).name.bytes);
- def.setColumn_metadata(columnDefinitionsToThrift(cfm, cfm.allColumns()));
- def.setCompaction_strategy(cfm.params.compaction.klass().getName());
- def.setCompaction_strategy_options(cfm.params.compaction.options());
- def.setCompression_options(compressionParametersToThrift(cfm.params.compression));
- def.setBloom_filter_fp_chance(cfm.params.bloomFilterFpChance);
- def.setMin_index_interval(cfm.params.minIndexInterval);
- def.setMax_index_interval(cfm.params.maxIndexInterval);
- def.setMemtable_flush_period_in_ms(cfm.params.memtableFlushPeriodInMs);
- def.setCaching(toThrift(cfm.params.caching));
- def.setCells_per_row_to_cache(toThriftCellsPerRow(cfm.params.caching));
- def.setDefault_time_to_live(cfm.params.defaultTimeToLive);
- def.setSpeculative_retry(cfm.params.speculativeRetry.toString());
- def.setTriggers(triggerDefinitionsToThrift(cfm.getTriggers()));
-
- return def;
- }
-
- public static ColumnDefinition fromThrift(String ksName,
- String cfName,
- AbstractType<?> thriftComparator,
- AbstractType<?> thriftSubcomparator,
- ColumnDef thriftColumnDef)
- throws SyntaxException, ConfigurationException
- {
- boolean isSuper = thriftSubcomparator != null;
- // For super columns, the componentIndex is 1 because the ColumnDefinition applies to the column component.
- AbstractType<?> comparator = thriftSubcomparator == null ? thriftComparator : thriftSubcomparator;
- try
- {
- comparator.validate(thriftColumnDef.name);
- }
- catch (MarshalException e)
- {
- throw new ConfigurationException(String.format("Column name %s is not valid for comparator %s", ByteBufferUtil.bytesToHex(thriftColumnDef.name), comparator));
- }
-
- // In our generic layout, we store thrift defined columns as static, but this doesn't work for super columns so we
- // use a regular definition (and "dynamic" columns are handled in a map).
- ColumnDefinition.Kind kind = isSuper ? ColumnDefinition.Kind.REGULAR : ColumnDefinition.Kind.STATIC;
- return new ColumnDefinition(ksName,
- cfName,
- ColumnIdentifier.getInterned(ByteBufferUtil.clone(thriftColumnDef.name), comparator),
- TypeParser.parse(thriftColumnDef.validation_class),
- ColumnDefinition.NO_POSITION,
- kind);
- }
-
- private static List<ColumnDefinition> fromThrift(String ksName,
- String cfName,
- AbstractType<?> thriftComparator,
- AbstractType<?> thriftSubcomparator,
- List<ColumnDef> thriftDefs)
- throws SyntaxException, ConfigurationException
- {
- if (thriftDefs == null)
- return new ArrayList<>();
-
- List<ColumnDefinition> defs = new ArrayList<>(thriftDefs.size());
- for (ColumnDef thriftColumnDef : thriftDefs)
- defs.add(fromThrift(ksName, cfName, thriftComparator, thriftSubcomparator, thriftColumnDef));
-
- return defs;
- }
-
- private static Indexes indexDefsFromThrift(CFMetaData cfm,
- String ksName,
- String cfName,
- AbstractType<?> thriftComparator,
- AbstractType<?> thriftSubComparator,
- List<ColumnDef> thriftDefs)
- {
- if (thriftDefs == null)
- return Indexes.none();
-
- Set<String> indexNames = new HashSet<>();
- Indexes.Builder indexes = Indexes.builder();
- for (ColumnDef def : thriftDefs)
- {
- if (def.isSetIndex_type())
- {
- ColumnDefinition column = fromThrift(ksName, cfName, thriftComparator, thriftSubComparator, def);
-
- String indexName = def.getIndex_name();
- // add a generated index name if none was supplied
- if (Strings.isNullOrEmpty(indexName))
- indexName = Indexes.getAvailableIndexName(ksName, cfName, column.name.toString());
-
- if (indexNames.contains(indexName))
- throw new ConfigurationException("Duplicate index name " + indexName);
-
- indexNames.add(indexName);
-
- Map<String, String> indexOptions = def.getIndex_options();
- if (indexOptions != null && indexOptions.containsKey(IndexTarget.TARGET_OPTION_NAME))
- throw new ConfigurationException("Reserved index option 'target' cannot be used");
-
- IndexMetadata.Kind kind = IndexMetadata.Kind.valueOf(def.index_type.name());
-
- indexes.add(IndexMetadata.fromLegacyMetadata(cfm, column, indexName, kind, indexOptions));
- }
- }
- return indexes.build();
- }
-
- @VisibleForTesting
- public static ColumnDef toThrift(CFMetaData cfMetaData, ColumnDefinition column)
- {
- ColumnDef cd = new ColumnDef();
-
- cd.setName(ByteBufferUtil.clone(column.name.bytes));
- cd.setValidation_class(column.type.toString());
-
- // we include the index in the ColumnDef iff its targets are compatible with
- // pre-3.0 indexes AND it is the only index defined on the given column, that is:
- // * it is the only index on the column (i.e. with this column as its target)
- // * it has only a single target, which matches the pattern for pre-3.0 indexes
- // i.e. keys/values/entries/full, with exactly 1 argument that matches the
- // column name OR a simple column name (for indexes on non-collection columns)
- // n.b. it's a guess that using a pre-compiled regex and checking the group is
- // cheaper than compiling a new regex for each column, but as this isn't on
- // any hot path this hasn't been verified yet.
- IndexMetadata matchedIndex = null;
- for (IndexMetadata index : cfMetaData.getIndexes())
- {
- Pair<ColumnDefinition, IndexTarget.Type> target = TargetParser.parse(cfMetaData, index);
- if (target.left.equals(column))
- {
- // we already found an index for this column, we've no option but to
- // ignore both of them (and any others we've yet to find)
- if (matchedIndex != null)
- return cd;
-
- matchedIndex = index;
- }
- }
-
- if (matchedIndex != null)
- {
- cd.setIndex_type(org.apache.cassandra.thrift.IndexType.valueOf(matchedIndex.kind.name()));
- cd.setIndex_name(matchedIndex.name);
- Map<String, String> filteredOptions = Maps.filterKeys(matchedIndex.options,
- s -> !IndexTarget.TARGET_OPTION_NAME.equals(s));
- cd.setIndex_options(filteredOptions.isEmpty()
- ? null
- : Maps.newHashMap(filteredOptions));
- }
-
- return cd;
- }
-
- private static List<ColumnDef> columnDefinitionsToThrift(CFMetaData metadata, Collection<ColumnDefinition> columns)
- {
- List<ColumnDef> thriftDefs = new ArrayList<>(columns.size());
- for (ColumnDefinition def : columns)
- if (def.isPartOfCellName(metadata.isCQLTable(), metadata.isSuper()))
- thriftDefs.add(ThriftConversion.toThrift(metadata, def));
- return thriftDefs;
- }
-
- private static Triggers triggerDefinitionsFromThrift(List<TriggerDef> thriftDefs)
- {
- Triggers.Builder triggers = Triggers.builder();
- for (TriggerDef thriftDef : thriftDefs)
- triggers.add(new TriggerMetadata(thriftDef.getName(), thriftDef.getOptions().get(TriggerMetadata.CLASS)));
- return triggers.build();
- }
-
- private static List<TriggerDef> triggerDefinitionsToThrift(Triggers triggers)
- {
- List<TriggerDef> thriftDefs = new ArrayList<>();
- for (TriggerMetadata def : triggers)
- {
- TriggerDef td = new TriggerDef();
- td.setName(def.name);
- td.setOptions(Collections.singletonMap(TriggerMetadata.CLASS, def.classOption));
- thriftDefs.add(td);
- }
- return thriftDefs;
- }
-
- @SuppressWarnings("deprecation")
- public static Map<String, String> compressionParametersToThrift(CompressionParams parameters)
- {
- if (!parameters.isEnabled())
- return Collections.emptyMap();
-
- Map<String, String> options = new HashMap<>(parameters.getOtherOptions());
- Class<? extends ICompressor> klass = parameters.getSstableCompressor().getClass();
- options.put(CompressionParams.SSTABLE_COMPRESSION, klass.getName());
- options.put(CompressionParams.CHUNK_LENGTH_KB, parameters.chunkLengthInKB());
- return options;
- }
-
- private static String toThrift(CachingParams caching)
- {
- if (caching.cacheRows() && caching.cacheKeys())
- return "ALL";
-
- if (caching.cacheRows())
- return "ROWS_ONLY";
-
- if (caching.cacheKeys())
- return "KEYS_ONLY";
-
- return "NONE";
- }
-
- private static CachingParams cachingFromTrhfit(String caching)
- {
- switch (caching.toUpperCase(Locale.ENGLISH))
- {
- case "ALL":
- return CachingParams.CACHE_EVERYTHING;
- case "ROWS_ONLY":
- return new CachingParams(false, Integer.MAX_VALUE);
- case "KEYS_ONLY":
- return CachingParams.CACHE_KEYS;
- case "NONE":
- return CachingParams.CACHE_NOTHING;
- default:
- throw new ConfigurationException(String.format("Invalid value %s for caching parameter", caching));
- }
- }
-
- private static String toThriftCellsPerRow(CachingParams caching)
- {
- return caching.cacheAllRows()
- ? "ALL"
- : String.valueOf(caching.rowsPerPartitionToCache());
- }
-
- private static int fromThriftCellsPerRow(String value)
- {
- return "ALL".equals(value)
- ? Integer.MAX_VALUE
- : Integer.parseInt(value);
- }
-
- public static CachingParams cachingFromThrift(String caching, String cellsPerRow)
- {
- boolean cacheKeys = true;
- int rowsPerPartitionToCache = 0;
-
- // if we get a caching string from thrift it is legacy, "ALL", "KEYS_ONLY" etc
- if (caching != null)
- {
- CachingParams parsed = cachingFromTrhfit(caching);
- cacheKeys = parsed.cacheKeys();
- rowsPerPartitionToCache = parsed.rowsPerPartitionToCache();
- }
-
- // if we get cells_per_row from thrift, it is either "ALL" or "<number of cells to cache>".
- if (cellsPerRow != null && rowsPerPartitionToCache > 0)
- rowsPerPartitionToCache = fromThriftCellsPerRow(cellsPerRow);
-
- return new CachingParams(cacheKeys, rowsPerPartitionToCache);
- }
-}
diff --git a/src/java/org/apache/cassandra/thrift/ThriftResultsMerger.java b/src/java/org/apache/cassandra/thrift/ThriftResultsMerger.java
deleted file mode 100644
index a14409e..0000000
--- a/src/java/org/apache/cassandra/thrift/ThriftResultsMerger.java
+++ /dev/null
@@ -1,294 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.thrift;
-
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.NoSuchElementException;
-
-import org.apache.cassandra.db.transform.Transformation;
-import org.apache.cassandra.utils.AbstractIterator;
-import com.google.common.collect.Iterators;
-import com.google.common.collect.PeekingIterator;
-
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
-import org.apache.cassandra.db.*;
-import org.apache.cassandra.db.rows.*;
-import org.apache.cassandra.db.marshal.AbstractType;
-import org.apache.cassandra.db.marshal.MapType;
-import org.apache.cassandra.db.partitions.*;
-
-/**
- * Given an iterator on a partition of a compact table, this return an iterator that merges the
- * static row columns with the other results.
- *
- * Compact tables stores thrift column_metadata as static columns (see CompactTables for
- * details). When reading for thrift however, we want to merge those static values with other
- * results because:
- * 1) on thrift, all "columns" are sorted together, whether or not they are declared
- * column_metadata.
- * 2) it's possible that a table add a value for a "dynamic" column, and later that column
- * is statically defined. Merging "static" and "dynamic" columns make sure we don't miss
- * a value prior to the column declaration.
- *
- * For example, if a thrift table declare 2 columns "c1" and "c5" and the results from a query
- * is:
- * Partition: static: { c1: 3, c5: 4 }
- * "a" : { value : 2 }
- * "c3": { value : 8 }
- * "c7": { value : 1 }
- * then this class transform it into:
- * Partition: "a" : { value : 2 }
- * "c1": { value : 3 }
- * "c3": { value : 8 }
- * "c5": { value : 4 }
- * "c7": { value : 1 }
- */
-public class ThriftResultsMerger extends Transformation<UnfilteredRowIterator>
-{
- private final int nowInSec;
-
- private ThriftResultsMerger(int nowInSec)
- {
- this.nowInSec = nowInSec;
- }
-
- public static UnfilteredPartitionIterator maybeWrap(UnfilteredPartitionIterator iterator, CFMetaData metadata, int nowInSec)
- {
- if (!metadata.isStaticCompactTable() && !metadata.isSuper())
- return iterator;
-
- return Transformation.apply(iterator, new ThriftResultsMerger(nowInSec));
- }
-
- public static UnfilteredRowIterator maybeWrap(UnfilteredRowIterator iterator, int nowInSec)
- {
- if (!iterator.metadata().isStaticCompactTable() && !iterator.metadata().isSuper())
- return iterator;
-
- return iterator.metadata().isSuper()
- ? Transformation.apply(iterator, new SuperColumnsPartitionMerger(iterator, nowInSec))
- : new PartitionMerger(iterator, nowInSec);
- }
-
- @Override
- public UnfilteredRowIterator applyToPartition(UnfilteredRowIterator iter)
- {
- return iter.metadata().isSuper()
- ? Transformation.apply(iter, new SuperColumnsPartitionMerger(iter, nowInSec))
- : new PartitionMerger(iter, nowInSec);
- }
-
- private static class PartitionMerger extends WrappingUnfilteredRowIterator
- {
- private final int nowInSec;
-
- // We initialize lazily to avoid having this iterator fetch the wrapped iterator before it's actually asked for it.
- private boolean isInit;
-
- private Iterator<Cell> staticCells;
-
- private final Row.Builder builder;
- private Row nextToMerge;
- private Unfiltered nextFromWrapped;
-
- private PartitionMerger(UnfilteredRowIterator results, int nowInSec)
- {
- super(results);
- assert results.metadata().isStaticCompactTable();
- this.nowInSec = nowInSec;
- this.builder = BTreeRow.sortedBuilder();
- }
-
- private void init()
- {
- assert !isInit;
- Row staticRow = super.staticRow();
- assert !staticRow.hasComplex();
-
- staticCells = staticRow.cells().iterator();
- updateNextToMerge();
- isInit = true;
- }
-
- @Override
- public Row staticRow()
- {
- return Rows.EMPTY_STATIC_ROW;
- }
-
- @Override
- public boolean hasNext()
- {
- if (!isInit)
- init();
-
- return nextFromWrapped != null || nextToMerge != null || super.hasNext();
- }
-
- @Override
- public Unfiltered next()
- {
- if (!isInit)
- init();
-
- if (nextFromWrapped == null && super.hasNext())
- nextFromWrapped = super.next();
-
- if (nextFromWrapped == null)
- {
- if (nextToMerge == null)
- throw new NoSuchElementException();
-
- return consumeNextToMerge();
- }
-
- if (nextToMerge == null)
- return consumeNextWrapped();
-
- int cmp = metadata().comparator.compare(nextToMerge, nextFromWrapped);
- if (cmp < 0)
- return consumeNextToMerge();
- if (cmp > 0)
- return consumeNextWrapped();
-
- // Same row, so merge them
- assert nextFromWrapped instanceof Row;
- return Rows.merge((Row)consumeNextWrapped(), consumeNextToMerge(), nowInSec);
- }
-
- private Unfiltered consumeNextWrapped()
- {
- Unfiltered toReturn = nextFromWrapped;
- nextFromWrapped = null;
- return toReturn;
- }
-
- private Row consumeNextToMerge()
- {
- Row toReturn = nextToMerge;
- updateNextToMerge();
- return toReturn;
- }
-
- private void updateNextToMerge()
- {
- if (!staticCells.hasNext())
- {
- // Nothing more to merge.
- nextToMerge = null;
- return;
- }
-
- Cell cell = staticCells.next();
-
- // Given a static cell, the equivalent row uses the column name as clustering and the value as unique cell value.
- builder.newRow(Clustering.make(cell.column().name.bytes));
- builder.addCell(new BufferCell(metadata().compactValueColumn(), cell.timestamp(), cell.ttl(), cell.localDeletionTime(), cell.value(), cell.path()));
- nextToMerge = builder.build();
- }
- }
-
- private static class SuperColumnsPartitionMerger extends Transformation
- {
- private final int nowInSec;
- private final Row.Builder builder;
- private final ColumnDefinition superColumnMapColumn;
- private final AbstractType<?> columnComparator;
-
- private SuperColumnsPartitionMerger(UnfilteredRowIterator applyTo, int nowInSec)
- {
- assert applyTo.metadata().isSuper();
- this.nowInSec = nowInSec;
-
- this.superColumnMapColumn = applyTo.metadata().compactValueColumn();
- assert superColumnMapColumn != null && superColumnMapColumn.type instanceof MapType;
-
- this.builder = BTreeRow.sortedBuilder();
- this.columnComparator = ((MapType)superColumnMapColumn.type).nameComparator();
- }
-
- @Override
- public Row applyToRow(Row row)
- {
- PeekingIterator<Cell> staticCells = Iterators.peekingIterator(simpleCellsIterator(row));
- if (!staticCells.hasNext())
- return row;
-
- builder.newRow(row.clustering());
-
- ComplexColumnData complexData = row.getComplexColumnData(superColumnMapColumn);
-
- PeekingIterator<Cell> dynamicCells;
- if (complexData == null)
- {
- dynamicCells = Iterators.peekingIterator(Collections.<Cell>emptyIterator());
- }
- else
- {
- dynamicCells = Iterators.peekingIterator(complexData.iterator());
- builder.addComplexDeletion(superColumnMapColumn, complexData.complexDeletion());
- }
-
- while (staticCells.hasNext() && dynamicCells.hasNext())
- {
- Cell staticCell = staticCells.peek();
- Cell dynamicCell = dynamicCells.peek();
- int cmp = columnComparator.compare(staticCell.column().name.bytes, dynamicCell.path().get(0));
- if (cmp < 0)
- builder.addCell(makeDynamicCell(staticCells.next()));
- else if (cmp > 0)
- builder.addCell(dynamicCells.next());
- else
- builder.addCell(Cells.reconcile(makeDynamicCell(staticCells.next()), dynamicCells.next(), nowInSec));
- }
-
- while (staticCells.hasNext())
- builder.addCell(makeDynamicCell(staticCells.next()));
- while (dynamicCells.hasNext())
- builder.addCell(dynamicCells.next());
-
- return builder.build();
- }
-
- private Cell makeDynamicCell(Cell staticCell)
- {
- return new BufferCell(superColumnMapColumn, staticCell.timestamp(), staticCell.ttl(), staticCell.localDeletionTime(), staticCell.value(), CellPath.create(staticCell.column().name.bytes));
- }
-
- private Iterator<Cell> simpleCellsIterator(Row row)
- {
- final Iterator<Cell> cells = row.cells().iterator();
- return new AbstractIterator<Cell>()
- {
- protected Cell computeNext()
- {
- if (cells.hasNext())
- {
- Cell cell = cells.next();
- if (cell.column().isSimple())
- return cell;
- }
- return endOfData();
- }
- };
- }
- }
-}
-
diff --git a/src/java/org/apache/cassandra/thrift/ThriftServer.java b/src/java/org/apache/cassandra/thrift/ThriftServer.java
deleted file mode 100644
index 4aa5736..0000000
--- a/src/java/org/apache/cassandra/thrift/ThriftServer.java
+++ /dev/null
@@ -1,150 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.thrift;
-
-import java.net.InetAddress;
-import java.net.InetSocketAddress;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.cassandra.service.CassandraDaemon;
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.thrift.TProcessor;
-import org.apache.thrift.protocol.TBinaryProtocol;
-import org.apache.thrift.server.TServer;
-import org.apache.thrift.transport.TFramedTransport;
-import org.apache.thrift.transport.TTransportFactory;
-
-public class ThriftServer implements CassandraDaemon.Server
-{
- private static final Logger logger = LoggerFactory.getLogger(ThriftServer.class);
-
- protected final InetAddress address;
- protected final int port;
- protected final int backlog;
- private volatile ThriftServerThread server;
-
- public ThriftServer(InetAddress address, int port, int backlog)
- {
- this.address = address;
- this.port = port;
- this.backlog = backlog;
- }
-
- public void start()
- {
- if (server == null)
- {
- CassandraServer iface = getCassandraServer();
- server = new ThriftServerThread(address, port, backlog, getProcessor(iface), getTransportFactory());
- server.start();
- }
- }
-
- public synchronized void stop()
- {
- if (server != null)
- {
- server.stopServer();
- try
- {
- server.join();
- }
- catch (InterruptedException e)
- {
- logger.error("Interrupted while waiting thrift server to stop", e);
- }
- server = null;
- }
- }
-
- public boolean isRunning()
- {
- return server != null;
- }
-
- /*
- * These methods are intended to be overridden to provide custom implementations.
- */
- protected CassandraServer getCassandraServer()
- {
- return new CassandraServer();
- }
-
- protected TProcessor getProcessor(CassandraServer server)
- {
- return new Cassandra.Processor<Cassandra.Iface>(server);
- }
-
- protected TTransportFactory getTransportFactory()
- {
- int tFramedTransportSize = DatabaseDescriptor.getThriftFramedTransportSize();
- return new TFramedTransport.Factory(tFramedTransportSize);
- }
-
- /**
- * Simple class to run the thrift connection accepting code in separate
- * thread of control.
- */
- private static class ThriftServerThread extends Thread
- {
- private final TServer serverEngine;
-
- public ThriftServerThread(InetAddress listenAddr,
- int listenPort,
- int listenBacklog,
- TProcessor processor,
- TTransportFactory transportFactory)
- {
- // now we start listening for clients
- logger.info("Binding thrift service to {}:{}", listenAddr, listenPort);
-
- TServerFactory.Args args = new TServerFactory.Args();
- args.tProtocolFactory = new TBinaryProtocol.Factory(true, true);
- args.addr = new InetSocketAddress(listenAddr, listenPort);
- args.listenBacklog = listenBacklog;
- args.processor = processor;
- args.keepAlive = DatabaseDescriptor.getRpcKeepAlive();
- args.sendBufferSize = DatabaseDescriptor.getRpcSendBufferSize();
- args.recvBufferSize = DatabaseDescriptor.getRpcRecvBufferSize();
- args.inTransportFactory = transportFactory;
- args.outTransportFactory = transportFactory;
- serverEngine = new TServerCustomFactory(DatabaseDescriptor.getRpcServerType()).buildTServer(args);
- }
-
- public void run()
- {
- logger.info("Listening for thrift clients...");
- serverEngine.serve();
- }
-
- public void stopServer()
- {
- logger.info("Stop listening to thrift clients");
- serverEngine.stop();
- }
- }
-
- public static final class ThriftServerType
- {
- public final static String SYNC = "sync";
- public final static String ASYNC = "async";
- public final static String HSHA = "hsha";
- }
-}
diff --git a/src/java/org/apache/cassandra/thrift/ThriftSessionManager.java b/src/java/org/apache/cassandra/thrift/ThriftSessionManager.java
deleted file mode 100644
index 60da3b4..0000000
--- a/src/java/org/apache/cassandra/thrift/ThriftSessionManager.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.thrift;
-
-import java.net.InetSocketAddress;
-import java.net.SocketAddress;
-import java.util.concurrent.ConcurrentHashMap;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import io.netty.util.concurrent.FastThreadLocal;
-
-/**
- * Encapsulates the current client state (session).
- *
- * We rely on the Thrift server to tell us what socket it is
- * executing a request for via setCurrentSocket, after which currentSession can do its job anywhere.
- */
-public class ThriftSessionManager
-{
- private static final Logger logger = LoggerFactory.getLogger(ThriftSessionManager.class);
- public final static ThriftSessionManager instance = new ThriftSessionManager();
-
- private final FastThreadLocal<SocketAddress> remoteSocket = new FastThreadLocal<>();
- private final ConcurrentHashMap<SocketAddress, ThriftClientState> activeSocketSessions = new ConcurrentHashMap<>();
-
- /**
- * @param socket the address on which the current thread will work on requests for until further notice
- */
- public void setCurrentSocket(SocketAddress socket)
- {
- remoteSocket.set(socket);
- }
-
- /**
- * @return the current session for the most recently given socket on this thread
- */
- public ThriftClientState currentSession()
- {
- SocketAddress socket = remoteSocket.get();
- assert socket != null;
-
- ThriftClientState cState = activeSocketSessions.get(socket);
- if (cState == null)
- {
- //guarantee atomicity
- ThriftClientState newState = new ThriftClientState((InetSocketAddress)socket);
- cState = activeSocketSessions.putIfAbsent(socket, newState);
- if (cState == null)
- cState = newState;
- }
- return cState;
- }
-
- /**
- * The connection associated with @param socket is permanently finished.
- */
- public void connectionComplete(SocketAddress socket)
- {
- assert socket != null;
- activeSocketSessions.remove(socket);
- if (logger.isTraceEnabled())
- logger.trace("ClientState removed for socket addr {}", socket);
- }
-
- public int getConnectedClients()
- {
- return activeSocketSessions.size();
- }
-}
diff --git a/src/java/org/apache/cassandra/thrift/ThriftValidation.java b/src/java/org/apache/cassandra/thrift/ThriftValidation.java
deleted file mode 100644
index 1b61fd5..0000000
--- a/src/java/org/apache/cassandra/thrift/ThriftValidation.java
+++ /dev/null
@@ -1,686 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.thrift;
-
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import java.util.Comparator;
-import java.util.List;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
-import org.apache.cassandra.config.Schema;
-import org.apache.cassandra.config.SchemaConstants;
-import org.apache.cassandra.cql3.Attributes;
-import org.apache.cassandra.cql3.Operator;
-import org.apache.cassandra.db.*;
-import org.apache.cassandra.db.marshal.AbstractType;
-import org.apache.cassandra.dht.IPartitioner;
-import org.apache.cassandra.dht.Token;
-import org.apache.cassandra.index.Index;
-import org.apache.cassandra.index.SecondaryIndexManager;
-import org.apache.cassandra.serializers.MarshalException;
-import org.apache.cassandra.utils.ByteBufferUtil;
-import org.apache.cassandra.utils.FBUtilities;
-
-/**
- * This has a lot of building blocks for CassandraServer to call to make sure it has valid input
- * -- ensuring column names conform to the declared comparator, for instance.
- *
- * The methods here mostly try to do just one part of the validation so they can be combined
- * for different needs -- supercolumns vs regular, range slices vs named, batch vs single-column.
- * (ValidateColumnPath is the main exception in that it includes keyspace and CF validation.)
- */
-public class ThriftValidation
-{
- private static final Logger logger = LoggerFactory.getLogger(ThriftValidation.class);
-
- public static void validateKey(CFMetaData metadata, ByteBuffer key) throws org.apache.cassandra.exceptions.InvalidRequestException
- {
- if (key == null || key.remaining() == 0)
- {
- throw new org.apache.cassandra.exceptions.InvalidRequestException("Key may not be empty");
- }
-
- // check that key can be handled by FBUtilities.writeShortByteArray
- if (key.remaining() > FBUtilities.MAX_UNSIGNED_SHORT)
- {
- throw new org.apache.cassandra.exceptions.InvalidRequestException("Key length of " + key.remaining() +
- " is longer than maximum of " +
- FBUtilities.MAX_UNSIGNED_SHORT);
- }
-
- try
- {
- metadata.getKeyValidator().validate(key);
- }
- catch (MarshalException e)
- {
- throw new org.apache.cassandra.exceptions.InvalidRequestException(e.getMessage());
- }
- }
-
- public static void validateKeyspace(String keyspaceName) throws KeyspaceNotDefinedException
- {
- if (!Schema.instance.getKeyspaces().contains(keyspaceName))
- {
- throw new KeyspaceNotDefinedException("Keyspace " + keyspaceName + " does not exist");
- }
- }
-
- public static CFMetaData validateColumnFamily(String keyspaceName, String cfName, boolean isCommutativeOp) throws org.apache.cassandra.exceptions.InvalidRequestException
- {
- CFMetaData metadata = validateColumnFamily(keyspaceName, cfName);
-
- if (isCommutativeOp)
- {
- if (!metadata.isCounter())
- throw new org.apache.cassandra.exceptions.InvalidRequestException("invalid operation for non commutative table " + cfName);
- }
- else
- {
- if (metadata.isCounter())
- throw new org.apache.cassandra.exceptions.InvalidRequestException("invalid operation for commutative table " + cfName);
- }
- return metadata;
- }
-
- // To be used when the operation should be authorized whether this is a counter CF or not
- public static CFMetaData validateColumnFamily(String keyspaceName, String cfName) throws org.apache.cassandra.exceptions.InvalidRequestException
- {
- return validateColumnFamilyWithCompactMode(keyspaceName, cfName, false);
- }
-
- public static CFMetaData validateColumnFamilyWithCompactMode(String keyspaceName, String cfName, boolean noCompactMode) throws org.apache.cassandra.exceptions.InvalidRequestException
- {
- validateKeyspace(keyspaceName);
- if (cfName.isEmpty())
- throw new org.apache.cassandra.exceptions.InvalidRequestException("non-empty table is required");
-
- CFMetaData metadata = Schema.instance.getCFMetaData(keyspaceName, cfName);
- if (metadata == null)
- throw new org.apache.cassandra.exceptions.InvalidRequestException("unconfigured table " + cfName);
-
- if (metadata.isCompactTable() && noCompactMode)
- return metadata.asNonCompact();
- else
- return metadata;
- }
-
- /**
- * validates all parts of the path to the column, including the column name
- */
- public static void validateColumnPath(CFMetaData metadata, ColumnPath column_path) throws org.apache.cassandra.exceptions.InvalidRequestException
- {
- if (!metadata.isSuper())
- {
- if (column_path.super_column != null)
- {
- throw new org.apache.cassandra.exceptions.InvalidRequestException("supercolumn parameter is invalid for standard CF " + metadata.cfName);
- }
- if (column_path.column == null)
- {
- throw new org.apache.cassandra.exceptions.InvalidRequestException("column parameter is not optional for standard CF " + metadata.cfName);
- }
- }
- else
- {
- if (column_path.super_column == null)
- throw new org.apache.cassandra.exceptions.InvalidRequestException("supercolumn parameter is not optional for super CF " + metadata.cfName);
- }
- if (column_path.column != null)
- {
- validateColumnNames(metadata, column_path.super_column, Arrays.asList(column_path.column));
- }
- if (column_path.super_column != null)
- {
- validateColumnNames(metadata, (ByteBuffer)null, Arrays.asList(column_path.super_column));
- }
- }
-
- public static void validateColumnParent(CFMetaData metadata, ColumnParent column_parent) throws org.apache.cassandra.exceptions.InvalidRequestException
- {
- if (!metadata.isSuper())
- {
- if (column_parent.super_column != null)
- {
- throw new org.apache.cassandra.exceptions.InvalidRequestException("table alone is required for standard CF " + metadata.cfName);
- }
- }
-
- if (column_parent.super_column != null)
- {
- validateColumnNames(metadata, (ByteBuffer)null, Arrays.asList(column_parent.super_column));
- }
- }
-
- // column_path_or_parent is a ColumnPath for remove, where the "column" is optional even for a standard CF
- static void validateColumnPathOrParent(CFMetaData metadata, ColumnPath column_path_or_parent) throws org.apache.cassandra.exceptions.InvalidRequestException
- {
- if (metadata.isSuper())
- {
- if (column_path_or_parent.super_column == null && column_path_or_parent.column != null)
- {
- throw new org.apache.cassandra.exceptions.InvalidRequestException("A column cannot be specified without specifying a super column for removal on super CF "
- + metadata.cfName);
- }
- }
- else
- {
- if (column_path_or_parent.super_column != null)
- {
- throw new org.apache.cassandra.exceptions.InvalidRequestException("supercolumn may not be specified for standard CF " + metadata.cfName);
- }
- }
- if (column_path_or_parent.column != null)
- {
- validateColumnNames(metadata, column_path_or_parent.super_column, Arrays.asList(column_path_or_parent.column));
- }
- if (column_path_or_parent.super_column != null)
- {
- validateColumnNames(metadata, (ByteBuffer)null, Arrays.asList(column_path_or_parent.super_column));
- }
- }
-
- private static AbstractType<?> getThriftColumnNameComparator(CFMetaData metadata, ByteBuffer superColumnName)
- {
- if (!metadata.isSuper())
- return LegacyLayout.makeLegacyComparator(metadata);
-
- if (superColumnName == null)
- {
- // comparator for super column name
- return metadata.comparator.subtype(0);
- }
- else
- {
- // comparator for sub columns
- return metadata.thriftColumnNameType();
- }
- }
-
- /**
- * Validates the column names but not the parent path or data
- */
- private static void validateColumnNames(CFMetaData metadata, ByteBuffer superColumnName, Iterable<ByteBuffer> column_names)
- throws org.apache.cassandra.exceptions.InvalidRequestException
- {
- int maxNameLength = LegacyLayout.MAX_CELL_NAME_LENGTH;
-
- if (superColumnName != null)
- {
- if (superColumnName.remaining() > maxNameLength)
- throw new org.apache.cassandra.exceptions.InvalidRequestException("supercolumn name length must not be greater than " + maxNameLength);
- if (superColumnName.remaining() == 0)
- throw new org.apache.cassandra.exceptions.InvalidRequestException("supercolumn name must not be empty");
- if (!metadata.isSuper())
- throw new org.apache.cassandra.exceptions.InvalidRequestException("supercolumn specified to table " + metadata.cfName + " containing normal columns");
- }
- AbstractType<?> comparator = getThriftColumnNameComparator(metadata, superColumnName);
- boolean isCQL3Table = !metadata.isThriftCompatible();
- for (ByteBuffer name : column_names)
- {
- if (name.remaining() > maxNameLength)
- throw new org.apache.cassandra.exceptions.InvalidRequestException("column name length must not be greater than " + maxNameLength);
- if (name.remaining() == 0)
- throw new org.apache.cassandra.exceptions.InvalidRequestException("column name must not be empty");
- try
- {
- comparator.validate(name);
- }
- catch (MarshalException e)
- {
- throw new org.apache.cassandra.exceptions.InvalidRequestException(e.getMessage());
- }
-
- if (isCQL3Table)
- {
- try
- {
- LegacyLayout.LegacyCellName cname = LegacyLayout.decodeCellName(metadata, name);
- assert cname.clustering.size() == metadata.comparator.size();
-
- // CQL3 table don't support having only part of their composite column names set
- for (int i = 0; i < cname.clustering.size(); i++)
- {
- if (cname.clustering.get(i) == null)
- throw new org.apache.cassandra.exceptions.InvalidRequestException(String.format("Not enough components (found %d but %d expected) for column name since %s is a CQL3 table",
- i, metadata.comparator.size() + 1, metadata.cfName));
- }
-
-
-
- // On top of that, if we have a collection component, the (CQL3) column must be a collection
- if (cname.column != null && cname.collectionElement != null && !cname.column.type.isCollection())
- throw new org.apache.cassandra.exceptions.InvalidRequestException(String.format("Invalid collection component, %s is not a collection", cname.column.name));
- }
- catch (IllegalArgumentException | UnknownColumnException e)
- {
- throw new org.apache.cassandra.exceptions.InvalidRequestException(String.format("Error validating cell name for CQL3 table %s: %s", metadata.cfName, e.getMessage()));
- }
- }
- }
- }
-
- public static void validateColumnNames(CFMetaData metadata, ColumnParent column_parent, Iterable<ByteBuffer> column_names) throws org.apache.cassandra.exceptions.InvalidRequestException
- {
- validateColumnNames(metadata, column_parent.super_column, column_names);
- }
-
- public static void validateRange(CFMetaData metadata, ColumnParent column_parent, SliceRange range) throws org.apache.cassandra.exceptions.InvalidRequestException
- {
- if (range.count < 0)
- throw new org.apache.cassandra.exceptions.InvalidRequestException("get_slice requires non-negative count");
-
- int maxNameLength = LegacyLayout.MAX_CELL_NAME_LENGTH;
- if (range.start.remaining() > maxNameLength)
- throw new org.apache.cassandra.exceptions.InvalidRequestException("range start length cannot be larger than " + maxNameLength);
- if (range.finish.remaining() > maxNameLength)
- throw new org.apache.cassandra.exceptions.InvalidRequestException("range finish length cannot be larger than " + maxNameLength);
-
- AbstractType<?> comparator = getThriftColumnNameComparator(metadata, column_parent.super_column);
- try
- {
- comparator.validate(range.start);
- comparator.validate(range.finish);
- }
- catch (MarshalException e)
- {
- throw new org.apache.cassandra.exceptions.InvalidRequestException(e.getMessage());
- }
-
- Comparator<ByteBuffer> orderedComparator = range.isReversed() ? comparator.reverseComparator : comparator;
- if (range.start.remaining() > 0
- && range.finish.remaining() > 0
- && orderedComparator.compare(range.start, range.finish) > 0)
- {
- throw new org.apache.cassandra.exceptions.InvalidRequestException("range finish must come after start in the order of traversal");
- }
- }
-
- public static void validateColumnOrSuperColumn(CFMetaData metadata, ColumnOrSuperColumn cosc)
- throws org.apache.cassandra.exceptions.InvalidRequestException
- {
- boolean isCommutative = metadata.isCounter();
-
- int nulls = 0;
- if (cosc.column == null) nulls++;
- if (cosc.super_column == null) nulls++;
- if (cosc.counter_column == null) nulls++;
- if (cosc.counter_super_column == null) nulls++;
-
- if (nulls != 3)
- throw new org.apache.cassandra.exceptions.InvalidRequestException("ColumnOrSuperColumn must have one (and only one) of column, super_column, counter and counter_super_column");
-
- if (cosc.column != null)
- {
- if (isCommutative)
- throw new org.apache.cassandra.exceptions.InvalidRequestException("invalid operation for commutative table " + metadata.cfName);
-
- validateTtl(metadata, cosc.column);
- validateColumnPath(metadata, new ColumnPath(metadata.cfName).setSuper_column((ByteBuffer)null).setColumn(cosc.column.name));
- validateColumnData(metadata, null, cosc.column);
- }
-
- if (cosc.super_column != null)
- {
- if (isCommutative)
- throw new org.apache.cassandra.exceptions.InvalidRequestException("invalid operation for commutative table " + metadata.cfName);
-
- for (Column c : cosc.super_column.columns)
- {
- validateColumnPath(metadata, new ColumnPath(metadata.cfName).setSuper_column(cosc.super_column.name).setColumn(c.name));
- validateColumnData(metadata, cosc.super_column.name, c);
- }
- }
-
- if (cosc.counter_column != null)
- {
- if (!isCommutative)
- throw new org.apache.cassandra.exceptions.InvalidRequestException("invalid operation for non commutative table " + metadata.cfName);
-
- validateColumnPath(metadata, new ColumnPath(metadata.cfName).setSuper_column((ByteBuffer)null).setColumn(cosc.counter_column.name));
- }
-
- if (cosc.counter_super_column != null)
- {
- if (!isCommutative)
- throw new org.apache.cassandra.exceptions.InvalidRequestException("invalid operation for non commutative table " + metadata.cfName);
-
- for (CounterColumn c : cosc.counter_super_column.columns)
- validateColumnPath(metadata, new ColumnPath(metadata.cfName).setSuper_column(cosc.counter_super_column.name).setColumn(c.name));
- }
- }
-
- private static void validateTtl(CFMetaData metadata, Column column) throws org.apache.cassandra.exceptions.InvalidRequestException
- {
- if (column.isSetTtl())
- {
- if (column.ttl < 0)
- throw new org.apache.cassandra.exceptions.InvalidRequestException("ttl must be greater or equal to 0");
-
- if (column.ttl > Attributes.MAX_TTL)
- throw new org.apache.cassandra.exceptions.InvalidRequestException(String.format("ttl is too large. requested (%d) maximum (%d)", column.ttl, Attributes.MAX_TTL));
- ExpirationDateOverflowHandling.maybeApplyExpirationDateOverflowPolicy(metadata, column.ttl, false);
- }
- else
- {
- ExpirationDateOverflowHandling.maybeApplyExpirationDateOverflowPolicy(metadata, metadata.params.defaultTimeToLive, true);
- // if it's not set, then it should be zero -- here we are just checking to make sure Thrift doesn't change that contract with us.
- assert column.ttl == 0;
- }
- }
-
- public static void validateMutation(CFMetaData metadata, Mutation mut)
- throws org.apache.cassandra.exceptions.InvalidRequestException
- {
- ColumnOrSuperColumn cosc = mut.column_or_supercolumn;
- Deletion del = mut.deletion;
-
- int nulls = 0;
- if (cosc == null) nulls++;
- if (del == null) nulls++;
-
- if (nulls != 1)
- {
- throw new org.apache.cassandra.exceptions.InvalidRequestException("mutation must have one and only one of column_or_supercolumn or deletion");
- }
-
- if (cosc != null)
- {
- validateColumnOrSuperColumn(metadata, cosc);
- }
- else
- {
- validateDeletion(metadata, del);
- }
- }
-
- public static void validateDeletion(CFMetaData metadata, Deletion del) throws org.apache.cassandra.exceptions.InvalidRequestException
- {
-
- if (del.super_column != null)
- validateColumnNames(metadata, (ByteBuffer)null, Arrays.asList(del.super_column));
-
- if (del.predicate != null)
- validateSlicePredicate(metadata, del.super_column, del.predicate);
-
- if (!metadata.isSuper() && del.super_column != null)
- {
- String msg = String.format("Deletion of super columns is not possible on a standard table (KeySpace=%s Table=%s Deletion=%s)", metadata.ksName, metadata.cfName, del);
- throw new org.apache.cassandra.exceptions.InvalidRequestException(msg);
- }
-
- if (metadata.isCounter())
- {
- // forcing server timestamp even if a timestamp was set for coherence with other counter operation
- del.timestamp = FBUtilities.timestampMicros();
- }
- else if (!del.isSetTimestamp())
- {
- throw new org.apache.cassandra.exceptions.InvalidRequestException("Deletion timestamp is not optional for non commutative table " + metadata.cfName);
- }
- }
-
- public static void validateSlicePredicate(CFMetaData metadata, ByteBuffer scName, SlicePredicate predicate) throws org.apache.cassandra.exceptions.InvalidRequestException
- {
- if (predicate.column_names == null && predicate.slice_range == null)
- throw new org.apache.cassandra.exceptions.InvalidRequestException("A SlicePredicate must be given a list of Columns, a SliceRange, or both");
-
- if (predicate.slice_range != null)
- validateRange(metadata, new ColumnParent(metadata.cfName).setSuper_column(scName), predicate.slice_range);
-
- if (predicate.column_names != null)
- validateColumnNames(metadata, scName, predicate.column_names);
- }
-
- /**
- * Validates the data part of the column (everything in the column object but the name, which is assumed to be valid)
- */
- public static void validateColumnData(CFMetaData metadata, ByteBuffer scName, Column column) throws org.apache.cassandra.exceptions.InvalidRequestException
- {
- validateTtl(metadata, column);
- if (!column.isSetValue())
- throw new org.apache.cassandra.exceptions.InvalidRequestException("Column value is required");
- if (!column.isSetTimestamp())
- throw new org.apache.cassandra.exceptions.InvalidRequestException("Column timestamp is required");
-
- try
- {
- LegacyLayout.LegacyCellName cn = LegacyLayout.decodeCellName(metadata, scName, column.name);
- if (cn.column.isPrimaryKeyColumn())
- throw new org.apache.cassandra.exceptions.InvalidRequestException(String.format("Cannot add primary key column %s to partition update", cn.column.name));
-
- cn.column.type.validateCellValue(column.value);
- }
- catch (UnknownColumnException e)
- {
- throw new org.apache.cassandra.exceptions.InvalidRequestException(e.getMessage());
- }
- catch (MarshalException me)
- {
- if (logger.isTraceEnabled())
- logger.trace("rejecting invalid value {}", ByteBufferUtil.bytesToHex(summarize(column.value)));
-
- throw new org.apache.cassandra.exceptions.InvalidRequestException(String.format("(%s) [%s][%s][%s] failed validation",
- me.getMessage(),
- metadata.ksName,
- metadata.cfName,
- (getThriftColumnNameComparator(metadata, scName)).getString(column.name)));
- }
-
- }
-
- /**
- * Return, at most, the first 64K of the buffer. This avoids very large column values being
- * logged in their entirety.
- */
- private static ByteBuffer summarize(ByteBuffer buffer)
- {
- int MAX = Short.MAX_VALUE;
- if (buffer.remaining() <= MAX)
- return buffer;
- return (ByteBuffer) buffer.slice().limit(buffer.position() + MAX);
- }
-
-
- public static void validatePredicate(CFMetaData metadata, ColumnParent column_parent, SlicePredicate predicate)
- throws org.apache.cassandra.exceptions.InvalidRequestException
- {
- if (predicate.column_names == null && predicate.slice_range == null)
- throw new org.apache.cassandra.exceptions.InvalidRequestException("predicate column_names and slice_range may not both be null");
- if (predicate.column_names != null && predicate.slice_range != null)
- throw new org.apache.cassandra.exceptions.InvalidRequestException("predicate column_names and slice_range may not both be present");
-
- if (predicate.getSlice_range() != null)
- validateRange(metadata, column_parent, predicate.slice_range);
- else
- validateColumnNames(metadata, column_parent, predicate.column_names);
- }
-
- public static void validateKeyRange(CFMetaData metadata, ByteBuffer superColumn, KeyRange range) throws org.apache.cassandra.exceptions.InvalidRequestException
- {
- if ((range.start_key == null) == (range.start_token == null)
- || (range.end_key == null) == (range.end_token == null))
- {
- throw new org.apache.cassandra.exceptions.InvalidRequestException("exactly one each of {start key, start token} and {end key, end token} must be specified");
- }
-
- // (key, token) is supported (for wide-row CFRR) but not (token, key)
- if (range.start_token != null && range.end_key != null)
- throw new org.apache.cassandra.exceptions.InvalidRequestException("start token + end key is not a supported key range");
-
- IPartitioner p = metadata.partitioner;
-
- if (range.start_key != null && range.end_key != null)
- {
- Token startToken = p.getToken(range.start_key);
- Token endToken = p.getToken(range.end_key);
- if (startToken.compareTo(endToken) > 0 && !endToken.isMinimum())
- {
- if (p.preservesOrder())
- throw new org.apache.cassandra.exceptions.InvalidRequestException("start key must sort before (or equal to) finish key in your partitioner!");
- else
- throw new org.apache.cassandra.exceptions.InvalidRequestException("start key's token sorts after end key's token. this is not allowed; you probably should not specify end key at all except with an ordered partitioner");
- }
- }
- else if (range.start_key != null && range.end_token != null)
- {
- // start_token/end_token can wrap, but key/token should not
- PartitionPosition stop = p.getTokenFactory().fromString(range.end_token).maxKeyBound();
- if (PartitionPosition.ForKey.get(range.start_key, p).compareTo(stop) > 0 && !stop.isMinimum())
- throw new org.apache.cassandra.exceptions.InvalidRequestException("Start key's token sorts after end token");
- }
-
- validateFilterClauses(metadata, range.row_filter);
-
- if (!isEmpty(range.row_filter) && superColumn != null)
- {
- throw new org.apache.cassandra.exceptions.InvalidRequestException("super columns are not supported for indexing");
- }
-
- if (range.count <= 0)
- {
- throw new org.apache.cassandra.exceptions.InvalidRequestException("maxRows must be positive");
- }
- }
-
- private static boolean isEmpty(List<IndexExpression> clause)
- {
- return clause == null || clause.isEmpty();
- }
-
- public static void validateIndexClauses(CFMetaData metadata, IndexClause index_clause)
- throws org.apache.cassandra.exceptions.InvalidRequestException
- {
- if (index_clause.expressions.isEmpty())
- throw new org.apache.cassandra.exceptions.InvalidRequestException("index clause list may not be empty");
-
- if (!validateFilterClauses(metadata, index_clause.expressions))
- throw new org.apache.cassandra.exceptions.InvalidRequestException("No indexed columns present in index clause with operator EQ");
- }
-
- // return true if index_clause contains an indexed columns with operator EQ
- public static boolean validateFilterClauses(CFMetaData metadata, List<IndexExpression> index_clause)
- throws org.apache.cassandra.exceptions.InvalidRequestException
- {
- if (isEmpty(index_clause))
- // no filter to apply
- return false;
-
- SecondaryIndexManager idxManager = Keyspace.open(metadata.ksName).getColumnFamilyStore(metadata.cfName).indexManager;
- AbstractType<?> nameValidator = getThriftColumnNameComparator(metadata, null);
-
- boolean isIndexed = false;
- for (IndexExpression expression : index_clause)
- {
- try
- {
- nameValidator.validate(expression.column_name);
- }
- catch (MarshalException me)
- {
- throw new org.apache.cassandra.exceptions.InvalidRequestException(String.format("[%s]=[%s] failed name validation (%s)",
- ByteBufferUtil.bytesToHex(expression.column_name),
- ByteBufferUtil.bytesToHex(expression.value),
- me.getMessage()));
- }
-
- if (expression.value.remaining() > 0xFFFF)
- throw new org.apache.cassandra.exceptions.InvalidRequestException("Index expression values may not be larger than 64K");
-
- ColumnDefinition def = metadata.getColumnDefinition(expression.column_name);
- if (def == null)
- {
- if (!metadata.isCompactTable())
- throw new org.apache.cassandra.exceptions.InvalidRequestException(String.format("Unknown column %s", nameValidator.getString(expression.column_name)));
-
- def = metadata.compactValueColumn();
- }
-
- try
- {
- def.type.validate(expression.value);
- }
- catch (MarshalException me)
- {
- throw new org.apache.cassandra.exceptions.InvalidRequestException(String.format("[%s]=[%s] failed value validation (%s)",
- ByteBufferUtil.bytesToHex(expression.column_name),
- ByteBufferUtil.bytesToHex(expression.value),
- me.getMessage()));
- }
-
- for(Index index : idxManager.listIndexes())
- isIndexed |= index.supportsExpression(def, Operator.valueOf(expression.op.name()));
- }
-
- return isIndexed;
- }
-
- public static void validateKeyspaceNotYetExisting(String newKsName) throws org.apache.cassandra.exceptions.InvalidRequestException
- {
- // keyspace names must be unique case-insensitively because the keyspace name becomes the directory
- // where we store CF sstables. Names that differ only in case would thus cause problems on
- // case-insensitive filesystems (NTFS, most installations of HFS+).
- for (String ksName : Schema.instance.getKeyspaces())
- {
- if (ksName.equalsIgnoreCase(newKsName))
- throw new org.apache.cassandra.exceptions.InvalidRequestException(String.format("Keyspace names must be case-insensitively unique (\"%s\" conflicts with \"%s\")",
- newKsName,
- ksName));
- }
- }
-
- public static void validateKeyspaceNotSystem(String modifiedKeyspace) throws org.apache.cassandra.exceptions.InvalidRequestException
- {
- if (SchemaConstants.isLocalSystemKeyspace(modifiedKeyspace))
- throw new org.apache.cassandra.exceptions.InvalidRequestException(String.format("%s keyspace is not user-modifiable", modifiedKeyspace));
- }
-
- //public static IDiskAtomFilter asIFilter(SlicePredicate sp, CFMetaData metadata, ByteBuffer superColumn)
- //{
- // SliceRange sr = sp.slice_range;
- // IDiskAtomFilter filter;
-
- // CellNameType comparator = metadata.isSuper()
- // ? new SimpleDenseCellNameType(metadata.comparator.subtype(superColumn == null ? 0 : 1))
- // : metadata.comparator;
- // if (sr == null)
- // {
-
- // SortedSet<CellName> ss = new TreeSet<CellName>(comparator);
- // for (ByteBuffer bb : sp.column_names)
- // ss.add(comparator.cellFromByteBuffer(bb));
- // filter = new NamesQueryFilter(ss);
- // }
- // else
- // {
- // filter = new SliceQueryFilter(comparator.fromByteBuffer(sr.start),
- // comparator.fromByteBuffer(sr.finish),
- // sr.reversed,
- // sr.count);
- // }
-
- // if (metadata.isSuper())
- // filter = SuperColumns.fromSCFilter(metadata.comparator, superColumn, filter);
- // return filter;
- //}
-}
diff --git a/src/java/org/apache/cassandra/tools/AuditLogViewer.java b/src/java/org/apache/cassandra/tools/AuditLogViewer.java
new file mode 100644
index 0000000..f1a6e37
--- /dev/null
+++ b/src/java/org/apache/cassandra/tools/AuditLogViewer.java
@@ -0,0 +1,259 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.tools;
+
+import java.io.File;
+import java.util.Arrays;
+import java.util.List;
+import java.util.function.Consumer;
+import java.util.stream.Collectors;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.GnuParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+
+import net.openhft.chronicle.core.io.IORuntimeException;
+import net.openhft.chronicle.queue.ChronicleQueueBuilder;
+import net.openhft.chronicle.queue.ExcerptTailer;
+import net.openhft.chronicle.queue.RollCycles;
+import net.openhft.chronicle.queue.impl.single.SingleChronicleQueue;
+import net.openhft.chronicle.threads.Pauser;
+import net.openhft.chronicle.wire.ReadMarshallable;
+import net.openhft.chronicle.wire.WireIn;
+import org.apache.cassandra.audit.BinAuditLogger;
+import org.apache.cassandra.utils.binlog.BinLog;
+
+/**
+ * Tool to view the contenst of AuditLog files in human readable format. Default implementation for AuditLog files
+ * logs audit messages in {@link org.apache.cassandra.utils.binlog.BinLog} format, this tool prints the contens of
+ * binary audit log files in text format.
+ */
+public class AuditLogViewer
+{
+ private static final String TOOL_NAME = "auditlogviewer";
+ private static final String ROLL_CYCLE = "roll_cycle";
+ private static final String FOLLOW = "follow";
+ private static final String IGNORE = "ignore";
+ private static final String HELP_OPTION = "help";
+
+ public static void main(String[] args)
+ {
+ AuditLogViewerOptions options = AuditLogViewerOptions.parseArgs(args);
+
+ try
+ {
+ dump(options.pathList, options.rollCycle, options.follow, options.ignoreUnsupported, System.out::print);
+ }
+ catch (Exception e)
+ {
+ System.err.println(e.getMessage());
+ System.exit(1);
+ }
+ }
+
+ static void dump(List<String> pathList, String rollCycle, boolean follow, boolean ignoreUnsupported, Consumer<String> displayFun)
+ {
+ //Backoff strategy for spinning on the queue, not aggressive at all as this doesn't need to be low latency
+ Pauser pauser = Pauser.millis(100);
+ List<ExcerptTailer> tailers = pathList.stream()
+ .distinct()
+ .map(path -> ChronicleQueueBuilder.single(new File(path)).readOnly(true).rollCycle(RollCycles.valueOf(rollCycle)).build())
+ .map(SingleChronicleQueue::createTailer)
+ .collect(Collectors.toList());
+ boolean hadWork = true;
+ while (hadWork)
+ {
+ hadWork = false;
+ for (ExcerptTailer tailer : tailers)
+ {
+ while (tailer.readDocument(new DisplayRecord(ignoreUnsupported, displayFun)))
+ {
+ hadWork = true;
+ }
+ }
+
+ if (follow)
+ {
+ if (!hadWork)
+ {
+ //Chronicle queue doesn't support blocking so use this backoff strategy
+ pauser.pause();
+ }
+ //Don't terminate the loop even if there wasn't work
+ hadWork = true;
+ }
+ }
+ }
+
+ private static class DisplayRecord implements ReadMarshallable
+ {
+ private final boolean ignoreUnsupported;
+ private final Consumer<String> displayFun;
+
+ DisplayRecord(boolean ignoreUnsupported, Consumer<String> displayFun)
+ {
+ this.ignoreUnsupported = ignoreUnsupported;
+ this.displayFun = displayFun;
+ }
+
+ public void readMarshallable(WireIn wireIn) throws IORuntimeException
+ {
+ int version = wireIn.read(BinLog.VERSION).int16();
+ if (!isSupportedVersion(version))
+ {
+ return;
+ }
+ String type = wireIn.read(BinLog.TYPE).text();
+ if (!isSupportedType(type))
+ {
+ return;
+ }
+
+ StringBuilder sb = new StringBuilder();
+ sb.append("Type: ")
+ .append(type)
+ .append(System.lineSeparator())
+ .append("LogMessage: ")
+ .append(wireIn.read(BinAuditLogger.AUDITLOG_MESSAGE).text())
+ .append(System.lineSeparator());
+
+ displayFun.accept(sb.toString());
+ }
+
+ private boolean isSupportedVersion(int version)
+ {
+ if (version <= BinAuditLogger.CURRENT_VERSION)
+ {
+ return true;
+ }
+
+ if (ignoreUnsupported)
+ {
+ return false;
+ }
+
+ throw new IORuntimeException("Unsupported record version [" + version
+ + "] - highest supported version is [" + BinAuditLogger.CURRENT_VERSION + ']');
+ }
+
+ private boolean isSupportedType(String type)
+ {
+ if (BinAuditLogger.AUDITLOG_TYPE.equals(type))
+ {
+ return true;
+ }
+
+ if (ignoreUnsupported)
+ {
+ return false;
+ }
+
+ throw new IORuntimeException("Unsupported record type field [" + type
+ + "] - supported type is [" + BinAuditLogger.AUDITLOG_TYPE + ']');
+ }
+ }
+
+ private static class AuditLogViewerOptions
+ {
+ private final List<String> pathList;
+ private String rollCycle = "HOURLY";
+ private boolean follow;
+ private boolean ignoreUnsupported;
+
+ private AuditLogViewerOptions(String[] pathList)
+ {
+ this.pathList = Arrays.asList(pathList);
+ }
+
+ static AuditLogViewerOptions parseArgs(String cmdArgs[])
+ {
+ CommandLineParser parser = new GnuParser();
+ Options options = getCmdLineOptions();
+ try
+ {
+ CommandLine cmd = parser.parse(options, cmdArgs, false);
+
+ if (cmd.hasOption(HELP_OPTION))
+ {
+ printUsage(options);
+ System.exit(0);
+ }
+
+ String[] args = cmd.getArgs();
+ if (args.length <= 0)
+ {
+ System.err.println("Audit log files directory path is a required argument.");
+ printUsage(options);
+ System.exit(1);
+ }
+
+ AuditLogViewerOptions opts = new AuditLogViewerOptions(args);
+
+ opts.follow = cmd.hasOption(FOLLOW);
+
+ opts.ignoreUnsupported = cmd.hasOption(IGNORE);
+
+ if (cmd.hasOption(ROLL_CYCLE))
+ {
+ opts.rollCycle = cmd.getOptionValue(ROLL_CYCLE);
+ }
+
+ return opts;
+ }
+ catch (ParseException e)
+ {
+ errorMsg(e.getMessage(), options);
+ return null;
+ }
+ }
+
+ static void errorMsg(String msg, Options options)
+ {
+ System.err.println(msg);
+ printUsage(options);
+ System.exit(1);
+ }
+
+ static Options getCmdLineOptions()
+ {
+ Options options = new Options();
+
+ options.addOption(new Option("r", ROLL_CYCLE, true, "How often to roll the log file was rolled. May be necessary for Chronicle to correctly parse file names. (MINUTELY, HOURLY, DAILY). Default HOURLY."));
+ options.addOption(new Option("f", FOLLOW, false, "Upon reacahing the end of the log continue indefinitely waiting for more records"));
+ options.addOption(new Option("i", IGNORE, false, "Silently ignore unsupported records"));
+ options.addOption(new Option("h", HELP_OPTION, false, "display this help message"));
+
+ return options;
+ }
+
+ static void printUsage(Options options)
+ {
+ String usage = String.format("%s <path1> [<path2>...<pathN>] [options]", TOOL_NAME);
+ StringBuilder header = new StringBuilder();
+ header.append("--\n");
+ header.append("View the audit log contents in human readable format");
+ header.append("\n--\n");
+ header.append("Options are:");
+ new HelpFormatter().printHelp(usage, header.toString(), options, "");
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/tools/BulkLoadConnectionFactory.java b/src/java/org/apache/cassandra/tools/BulkLoadConnectionFactory.java
index aa23f45..177f811 100644
--- a/src/java/org/apache/cassandra/tools/BulkLoadConnectionFactory.java
+++ b/src/java/org/apache/cassandra/tools/BulkLoadConnectionFactory.java
@@ -15,52 +15,40 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.cassandra.tools;
import java.io.IOException;
-import java.net.InetAddress;
-import java.net.InetSocketAddress;
-import java.net.Socket;
-import java.nio.channels.SocketChannel;
+import io.netty.channel.Channel;
import org.apache.cassandra.config.EncryptionOptions;
-import org.apache.cassandra.security.SSLFactory;
+import org.apache.cassandra.net.OutboundConnectionSettings;
+import org.apache.cassandra.streaming.DefaultConnectionFactory;
import org.apache.cassandra.streaming.StreamConnectionFactory;
-import org.apache.cassandra.utils.FBUtilities;
-public class BulkLoadConnectionFactory implements StreamConnectionFactory
+public class BulkLoadConnectionFactory extends DefaultConnectionFactory implements StreamConnectionFactory
{
+ // TODO: what is this unused variable for?
private final boolean outboundBindAny;
- private final int storagePort;
private final int secureStoragePort;
private final EncryptionOptions.ServerEncryptionOptions encryptionOptions;
- public BulkLoadConnectionFactory(int storagePort, int secureStoragePort, EncryptionOptions.ServerEncryptionOptions encryptionOptions, boolean outboundBindAny)
+ public BulkLoadConnectionFactory(int secureStoragePort, EncryptionOptions.ServerEncryptionOptions encryptionOptions, boolean outboundBindAny)
{
- this.storagePort = storagePort;
this.secureStoragePort = secureStoragePort;
this.encryptionOptions = encryptionOptions;
this.outboundBindAny = outboundBindAny;
}
- public Socket createConnection(InetAddress peer) throws IOException
+ public Channel createConnection(OutboundConnectionSettings template, int messagingVersion) throws IOException
{
// Connect to secure port for all peers if ServerEncryptionOptions is configured other than 'none'
// When 'all', 'dc' and 'rack', server nodes always have SSL port open, and since thin client like sstableloader
// does not know which node is in which dc/rack, connecting to SSL port is always the option.
+
if (encryptionOptions != null && encryptionOptions.internode_encryption != EncryptionOptions.ServerEncryptionOptions.InternodeEncryption.none)
- {
- if (outboundBindAny)
- return SSLFactory.getSocket(encryptionOptions, peer, secureStoragePort);
- else
- return SSLFactory.getSocket(encryptionOptions, peer, secureStoragePort, FBUtilities.getLocalAddress(), 0);
- }
- else
- {
- Socket socket = SocketChannel.open(new InetSocketAddress(peer, storagePort)).socket();
- if (outboundBindAny && !socket.isBound())
- socket.bind(new InetSocketAddress(FBUtilities.getLocalAddress(), 0));
- return socket;
- }
+ template = template.withConnectTo(template.to.withPort(secureStoragePort));
+
+ return super.createConnection(template, messagingVersion);
}
}
diff --git a/src/java/org/apache/cassandra/tools/BulkLoader.java b/src/java/org/apache/cassandra/tools/BulkLoader.java
index 0f1c555..c08881b 100644
--- a/src/java/org/apache/cassandra/tools/BulkLoader.java
+++ b/src/java/org/apache/cassandra/tools/BulkLoader.java
@@ -18,7 +18,7 @@
package org.apache.cassandra.tools;
import java.io.IOException;
-import java.net.InetAddress;
+import java.net.InetSocketAddress;
import java.util.Set;
import javax.net.ssl.SSLContext;
@@ -33,6 +33,7 @@
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.config.EncryptionOptions;
import org.apache.cassandra.io.sstable.SSTableLoader;
+import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.security.SSLFactory;
import org.apache.cassandra.streaming.*;
import org.apache.cassandra.utils.FBUtilities;
@@ -56,14 +57,14 @@
options.directory.getAbsoluteFile(),
new ExternalClient(
options.hosts,
- options.nativePort,
- options.authProvider,
options.storagePort,
+ options.authProvider,
options.sslStoragePort,
options.serverEncOptions,
buildSSLOptions(options.clientEncOptions)),
handler,
- options.connectionsPerHost);
+ options.connectionsPerHost,
+ options.targetKeyspace);
DatabaseDescriptor.setStreamThroughputOutboundMegabitsPerSec(options.throttle);
DatabaseDescriptor.setInterDCStreamThroughputOutboundMegabitsPerSec(options.interDcThrottle);
StreamResultFuture future = null;
@@ -104,7 +105,6 @@
// Give sockets time to gracefully close
Thread.sleep(1000);
- // System.exit(0); // We need that to stop non daemonized threads
}
catch (Exception e)
{
@@ -125,7 +125,7 @@
private long peak = 0;
private int totalFiles = 0;
- private final Multimap<InetAddress, SessionInfo> sessionsByHost = HashMultimap.create();
+ private final Multimap<InetAddressAndPort, SessionInfo> sessionsByHost = HashMultimap.create();
public ProgressIndicator()
{
@@ -166,7 +166,7 @@
boolean updateTotalFiles = totalFiles == 0;
// recalculate progress across all sessions in all hosts and display
- for (InetAddress peer : sessionsByHost.keySet())
+ for (InetAddressAndPort peer : sessionsByHost.keySet())
{
sb.append("[").append(peer).append("]");
@@ -243,10 +243,10 @@
}
}
- private static SSLOptions buildSSLOptions(EncryptionOptions.ClientEncryptionOptions clientEncryptionOptions)
+ private static SSLOptions buildSSLOptions(EncryptionOptions clientEncryptionOptions)
{
- if (!clientEncryptionOptions.enabled)
+ if (!clientEncryptionOptions.isEnabled())
{
return null;
}
@@ -263,26 +263,23 @@
return JdkSSLOptions.builder()
.withSSLContext(sslContext)
- .withCipherSuites(clientEncryptionOptions.cipher_suites)
+ .withCipherSuites(clientEncryptionOptions.cipher_suites.toArray(new String[0]))
.build();
}
static class ExternalClient extends NativeSSTableLoaderClient
{
- private final int storagePort;
private final int sslStoragePort;
private final EncryptionOptions.ServerEncryptionOptions serverEncOptions;
- public ExternalClient(Set<InetAddress> hosts,
- int port,
- AuthProvider authProvider,
+ public ExternalClient(Set<InetSocketAddress> hosts,
int storagePort,
+ AuthProvider authProvider,
int sslStoragePort,
EncryptionOptions.ServerEncryptionOptions serverEncryptionOptions,
SSLOptions sslOptions)
{
- super(hosts, port, authProvider, sslOptions);
- this.storagePort = storagePort;
+ super(hosts, storagePort, authProvider, sslOptions);
this.sslStoragePort = sslStoragePort;
serverEncOptions = serverEncryptionOptions;
}
@@ -290,7 +287,7 @@
@Override
public StreamConnectionFactory getConnectionFactory()
{
- return new BulkLoadConnectionFactory(storagePort, sslStoragePort, serverEncOptions, false);
+ return new BulkLoadConnectionFactory(sslStoragePort, serverEncOptions, false);
}
}
@@ -313,6 +310,23 @@
}
/**
+ * Add option with argument and argument name that accepts being defined multiple times as a list
+ * @param opt shortcut for option name
+ * @param longOpt complete option name
+ * @param argName argument name
+ * @param description description of the option
+ * @return updated Options object
+ */
+ public Options addOptionList(String opt, String longOpt, String argName, String description)
+ {
+ Option option = new Option(opt, longOpt, true, description);
+ option.setArgName(argName);
+ option.setArgs(Option.UNLIMITED_VALUES);
+
+ return addOption(option);
+ }
+
+ /**
* Add option without argument
* @param opt shortcut for option name
* @param longOpt complete option name
diff --git a/src/java/org/apache/cassandra/tools/INodeProbeFactory.java b/src/java/org/apache/cassandra/tools/INodeProbeFactory.java
index abc1651..fec4a2b 100644
--- a/src/java/org/apache/cassandra/tools/INodeProbeFactory.java
+++ b/src/java/org/apache/cassandra/tools/INodeProbeFactory.java
@@ -25,4 +25,18 @@
NodeProbe create(String host, int port) throws IOException;
NodeProbe create(String host, int port, String username, String password) throws IOException;
-}
\ No newline at end of file
+}
+
+class NodeProbeFactory implements INodeProbeFactory
+{
+
+ public NodeProbe create(String host, int port) throws IOException
+ {
+ return new NodeProbe(host, port);
+ }
+
+ public NodeProbe create(String host, int port, String username, String password) throws IOException
+ {
+ return new NodeProbe(host, port, username, password);
+ }
+}
diff --git a/src/java/org/apache/cassandra/tools/JsonTransformer.java b/src/java/org/apache/cassandra/tools/JsonTransformer.java
index 315bb7f..c538722 100644
--- a/src/java/org/apache/cassandra/tools/JsonTransformer.java
+++ b/src/java/org/apache/cassandra/tools/JsonTransformer.java
@@ -30,11 +30,14 @@
import java.util.concurrent.TimeUnit;
import java.util.stream.Stream;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.core.util.DefaultPrettyPrinter;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
+import com.fasterxml.jackson.core.util.DefaultPrettyPrinter.Indenter;
+import com.fasterxml.jackson.core.util.MinimalPrettyPrinter;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.db.marshal.CollectionType;
@@ -50,10 +53,10 @@
import org.apache.cassandra.db.rows.Unfiltered;
import org.apache.cassandra.db.rows.UnfilteredRowIterator;
import org.apache.cassandra.io.sstable.ISSTableScanner;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.transport.ProtocolVersion;
import org.apache.cassandra.utils.ByteBufferUtil;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
public final class JsonTransformer
{
@@ -68,7 +71,7 @@
private final CompactIndenter arrayIndenter = new CompactIndenter();
- private final CFMetaData metadata;
+ private final TableMetadata metadata;
private final ISSTableScanner currentScanner;
@@ -76,36 +79,55 @@
private long currentPosition = 0;
- private JsonTransformer(JsonGenerator json, ISSTableScanner currentScanner, boolean rawTime, CFMetaData metadata)
+ private JsonTransformer(JsonGenerator json, ISSTableScanner currentScanner, boolean rawTime, TableMetadata metadata, boolean isJsonLines)
{
this.json = json;
this.metadata = metadata;
this.currentScanner = currentScanner;
this.rawTime = rawTime;
- DefaultPrettyPrinter prettyPrinter = new DefaultPrettyPrinter();
- prettyPrinter.indentObjectsWith(objectIndenter);
- prettyPrinter.indentArraysWith(arrayIndenter);
- json.setPrettyPrinter(prettyPrinter);
+ if (isJsonLines)
+ {
+ MinimalPrettyPrinter minimalPrettyPrinter = new MinimalPrettyPrinter();
+ minimalPrettyPrinter.setRootValueSeparator("\n");
+ json.setPrettyPrinter(minimalPrettyPrinter);
+ }
+ else
+ {
+ DefaultPrettyPrinter prettyPrinter = new DefaultPrettyPrinter();
+ prettyPrinter.indentObjectsWith(objectIndenter);
+ prettyPrinter.indentArraysWith(arrayIndenter);
+ json.setPrettyPrinter(prettyPrinter);
+ }
}
- public static void toJson(ISSTableScanner currentScanner, Stream<UnfilteredRowIterator> partitions, boolean rawTime, CFMetaData metadata, OutputStream out)
+ public static void toJson(ISSTableScanner currentScanner, Stream<UnfilteredRowIterator> partitions, boolean rawTime, TableMetadata metadata, OutputStream out)
throws IOException
{
- try (JsonGenerator json = jsonFactory.createJsonGenerator(new OutputStreamWriter(out, StandardCharsets.UTF_8)))
+ try (JsonGenerator json = jsonFactory.createGenerator(new OutputStreamWriter(out, StandardCharsets.UTF_8)))
{
- JsonTransformer transformer = new JsonTransformer(json, currentScanner, rawTime, metadata);
+ JsonTransformer transformer = new JsonTransformer(json, currentScanner, rawTime, metadata, false);
json.writeStartArray();
partitions.forEach(transformer::serializePartition);
json.writeEndArray();
}
}
- public static void keysToJson(ISSTableScanner currentScanner, Stream<DecoratedKey> keys, boolean rawTime, CFMetaData metadata, OutputStream out) throws IOException
+ public static void toJsonLines(ISSTableScanner currentScanner, Stream<UnfilteredRowIterator> partitions, boolean rawTime, TableMetadata metadata, OutputStream out)
+ throws IOException
{
- try (JsonGenerator json = jsonFactory.createJsonGenerator(new OutputStreamWriter(out, StandardCharsets.UTF_8)))
+ try (JsonGenerator json = jsonFactory.createGenerator(new OutputStreamWriter(out, StandardCharsets.UTF_8)))
{
- JsonTransformer transformer = new JsonTransformer(json, currentScanner, rawTime, metadata);
+ JsonTransformer transformer = new JsonTransformer(json, currentScanner, rawTime, metadata, true);
+ partitions.forEach(transformer::serializePartition);
+ }
+ }
+
+ public static void keysToJson(ISSTableScanner currentScanner, Stream<DecoratedKey> keys, boolean rawTime, TableMetadata metadata, OutputStream out) throws IOException
+ {
+ try (JsonGenerator json = jsonFactory.createGenerator(new OutputStreamWriter(out, StandardCharsets.UTF_8)))
+ {
+ JsonTransformer transformer = new JsonTransformer(json, currentScanner, rawTime, metadata, false);
json.writeStartArray();
keys.forEach(transformer::serializePartitionKey);
json.writeEndArray();
@@ -119,7 +141,7 @@
private void serializePartitionKey(DecoratedKey key)
{
- AbstractType<?> keyValidator = metadata.getKeyValidator();
+ AbstractType<?> keyValidator = metadata.partitionKeyType;
objectIndenter.setCompact(true);
try
{
@@ -221,9 +243,10 @@
json.writeEndObject();
}
+
catch (IOException e)
{
- String key = metadata.getKeyValidator().getString(partition.partitionKey().getKey());
+ String key = metadata.partitionKeyType.getString(partition.partitionKey().getKey());
logger.error("Fatal error parsing partition: {}", key, e);
}
}
@@ -334,10 +357,10 @@
objectIndenter.setCompact(true);
json.writeStartArray();
arrayIndenter.setCompact(true);
- List<ColumnDefinition> clusteringColumns = metadata.clusteringColumns();
+ List<ColumnMetadata> clusteringColumns = metadata.clusteringColumns();
for (int i = 0; i < clusteringColumns.size(); i++)
{
- ColumnDefinition column = clusteringColumns.get(i);
+ ColumnMetadata column = clusteringColumns.get(i);
if (i >= clustering.size())
{
json.writeString("*");
@@ -498,7 +521,7 @@
}
/**
- * A specialized {@link com.fasterxml.jackson.core.util.DefaultPrettyPrinter.Indenter} that enables a 'compact' mode which puts all subsequent json values on the same
+ * A specialized {@link Indenter} that enables a 'compact' mode which puts all subsequent json values on the same
* line. This is manipulated via {@link CompactIndenter#setCompact(boolean)}
*/
private static final class CompactIndenter extends DefaultPrettyPrinter.NopIndenter
diff --git a/src/java/org/apache/cassandra/tools/LoaderOptions.java b/src/java/org/apache/cassandra/tools/LoaderOptions.java
index b3110f2..686c834 100644
--- a/src/java/org/apache/cassandra/tools/LoaderOptions.java
+++ b/src/java/org/apache/cassandra/tools/LoaderOptions.java
@@ -27,13 +27,18 @@
import java.util.HashSet;
import java.util.Set;
+import com.google.common.base.Throwables;
+import com.google.common.net.HostAndPort;
+
import org.apache.cassandra.config.*;
import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.tools.BulkLoader.CmdLineOptions;
import com.datastax.driver.core.AuthProvider;
import com.datastax.driver.core.PlainTextAuthProvider;
import org.apache.commons.cli.*;
+import org.apache.commons.lang3.StringUtils;
public class LoaderOptions
{
@@ -54,6 +59,7 @@
public static final String THROTTLE_MBITS = "throttle";
public static final String INTER_DC_THROTTLE_MBITS = "inter-dc-throttle";
public static final String TOOL_NAME = "sstableloader";
+ public static final String TARGET_KEYSPACE = "target-keyspace";
/* client encryption options */
public static final String SSL_TRUSTSTORE = "truststore";
@@ -77,11 +83,12 @@
public final int interDcThrottle;
public final int storagePort;
public final int sslStoragePort;
- public final EncryptionOptions.ClientEncryptionOptions clientEncOptions;
+ public final EncryptionOptions clientEncOptions;
public final int connectionsPerHost;
public final EncryptionOptions.ServerEncryptionOptions serverEncOptions;
- public final Set<InetAddress> hosts;
- public final Set<InetAddress> ignores;
+ public final Set<InetSocketAddress> hosts;
+ public final Set<InetAddressAndPort> ignores;
+ public final String targetKeyspace;
LoaderOptions(Builder builder)
{
@@ -102,6 +109,7 @@
serverEncOptions = builder.serverEncOptions;
hosts = builder.hosts;
ignores = builder.ignores;
+ targetKeyspace = builder.targetKeyspace;
}
static class Builder
@@ -119,11 +127,14 @@
int interDcThrottle = 0;
int storagePort;
int sslStoragePort;
- EncryptionOptions.ClientEncryptionOptions clientEncOptions = new EncryptionOptions.ClientEncryptionOptions();
+ EncryptionOptions clientEncOptions = new EncryptionOptions();
int connectionsPerHost = 1;
EncryptionOptions.ServerEncryptionOptions serverEncOptions = new EncryptionOptions.ServerEncryptionOptions();
- Set<InetAddress> hosts = new HashSet<>();
- Set<InetAddress> ignores = new HashSet<>();
+ Set<InetAddress> hostsArg = new HashSet<>();
+ Set<InetAddress> ignoresArg = new HashSet<>();
+ Set<InetSocketAddress> hosts = new HashSet<>();
+ Set<InetAddressAndPort> ignores = new HashSet<>();
+ String targetKeyspace;
Builder()
{
@@ -133,6 +144,23 @@
public LoaderOptions build()
{
constructAuthProvider();
+
+ try
+ {
+ for (InetAddress host : hostsArg)
+ {
+ hosts.add(new InetSocketAddress(host, nativePort));
+ }
+ for (InetAddress host : ignoresArg)
+ {
+ ignores.add(InetAddressAndPort.getByNameOverrideDefaults(host.getHostAddress(), storagePort));
+ }
+ }
+ catch (UnknownHostException e)
+ {
+ Throwables.propagate(e);
+ }
+
return new LoaderOptions(this);
}
@@ -208,7 +236,7 @@
return this;
}
- public Builder encOptions(EncryptionOptions.ClientEncryptionOptions encOptions)
+ public Builder encOptions(EncryptionOptions encOptions)
{
this.clientEncOptions = encOptions;
return this;
@@ -226,26 +254,51 @@
return this;
}
+ @Deprecated
public Builder hosts(Set<InetAddress> hosts)
{
- this.hosts = hosts;
+ this.hostsArg.addAll(hosts);
+ return this;
+ }
+
+ public Builder hostsAndNativePort(Set<InetSocketAddress> hosts)
+ {
+ this.hosts.addAll(hosts);
return this;
}
public Builder host(InetAddress host)
{
+ hostsArg.add(host);
+ return this;
+ }
+
+ public Builder hostAndNativePort(InetSocketAddress host)
+ {
hosts.add(host);
return this;
}
public Builder ignore(Set<InetAddress> ignores)
{
- this.ignores = ignores;
+ this.ignoresArg.addAll(ignores);
+ return this;
+ }
+
+ public Builder ignoresAndInternalPorts(Set<InetAddressAndPort> ignores)
+ {
+ this.ignores.addAll(ignores);
return this;
}
public Builder ignore(InetAddress ignore)
{
+ ignoresArg.add(ignore);
+ return this;
+ }
+
+ public Builder ignoreAndInternalPorts(InetAddressAndPort ignore)
+ {
ignores.add(ignore);
return this;
}
@@ -312,47 +365,6 @@
authProviderName = cmd.getOptionValue(AUTH_PROVIDER_OPTION);
}
- if (cmd.hasOption(INITIAL_HOST_ADDRESS_OPTION))
- {
- String[] nodes = cmd.getOptionValue(INITIAL_HOST_ADDRESS_OPTION).split(",");
- try
- {
- for (String node : nodes)
- {
- hosts.add(InetAddress.getByName(node.trim()));
- }
- } catch (UnknownHostException e)
- {
- errorMsg("Unknown host: " + e.getMessage(), options);
- }
-
- } else
- {
- System.err.println("Initial hosts must be specified (-d)");
- printUsage(options);
- System.exit(1);
- }
-
- if (cmd.hasOption(IGNORE_NODES_OPTION))
- {
- String[] nodes = cmd.getOptionValue(IGNORE_NODES_OPTION).split(",");
- try
- {
- for (String node : nodes)
- {
- ignores.add(InetAddress.getByName(node.trim()));
- }
- } catch (UnknownHostException e)
- {
- errorMsg("Unknown host: " + e.getMessage(), options);
- }
- }
-
- if (cmd.hasOption(CONNECTIONS_PER_HOST))
- {
- connectionsPerHost = Integer.parseInt(cmd.getOptionValue(CONNECTIONS_PER_HOST));
- }
-
// try to load config file first, so that values can be
// rewritten with other option values.
// otherwise use default config.
@@ -374,10 +386,54 @@
config.inter_dc_stream_throughput_outbound_megabits_per_sec = 0;
}
+
+ if (cmd.hasOption(INITIAL_HOST_ADDRESS_OPTION))
+ {
+ String[] nodes = cmd.getOptionValue(INITIAL_HOST_ADDRESS_OPTION).split(",");
+ try
+ {
+ for (String node : nodes)
+ {
+ HostAndPort hap = HostAndPort.fromString(node);
+ hosts.add(new InetSocketAddress(InetAddress.getByName(hap.getHost()), hap.getPortOrDefault(nativePort)));
+ }
+ } catch (UnknownHostException e)
+ {
+ errorMsg("Unknown host: " + e.getMessage(), options);
+ }
+
+ } else
+ {
+ System.err.println("Initial hosts must be specified (-d)");
+ printUsage(options);
+ System.exit(1);
+ }
+
if (cmd.hasOption(STORAGE_PORT_OPTION))
storagePort = Integer.parseInt(cmd.getOptionValue(STORAGE_PORT_OPTION));
else
storagePort = config.storage_port;
+
+ if (cmd.hasOption(IGNORE_NODES_OPTION))
+ {
+ String[] nodes = cmd.getOptionValue(IGNORE_NODES_OPTION).split(",");
+ try
+ {
+ for (String node : nodes)
+ {
+ ignores.add(InetAddressAndPort.getByNameOverrideDefaults(node.trim(), storagePort));
+ }
+ } catch (UnknownHostException e)
+ {
+ errorMsg("Unknown host: " + e.getMessage(), options);
+ }
+ }
+
+ if (cmd.hasOption(CONNECTIONS_PER_HOST))
+ {
+ connectionsPerHost = Integer.parseInt(cmd.getOptionValue(CONNECTIONS_PER_HOST));
+ }
+
if (cmd.hasOption(SSL_STORAGE_PORT_OPTION))
sslStoragePort = Integer.parseInt(cmd.getOptionValue(SSL_STORAGE_PORT_OPTION));
else
@@ -399,7 +455,7 @@
if (cmd.hasOption(SSL_TRUSTSTORE) || cmd.hasOption(SSL_TRUSTSTORE_PW) ||
cmd.hasOption(SSL_KEYSTORE) || cmd.hasOption(SSL_KEYSTORE_PW))
{
- clientEncOptions.enabled = true;
+ clientEncOptions = clientEncOptions.withEnabled(true);
}
if (cmd.hasOption(NATIVE_PORT_OPTION))
@@ -408,7 +464,7 @@
}
else
{
- if (config.native_transport_port_ssl != null && (config.client_encryption_options.enabled || clientEncOptions.enabled))
+ if (config.native_transport_port_ssl != null && (config.client_encryption_options.isEnabled() || clientEncOptions.isEnabled()))
nativePort = config.native_transport_port_ssl;
else
nativePort = config.native_transport_port;
@@ -416,47 +472,54 @@
if (cmd.hasOption(SSL_TRUSTSTORE))
{
- clientEncOptions.truststore = cmd.getOptionValue(SSL_TRUSTSTORE);
+ clientEncOptions = clientEncOptions.withTrustStore(cmd.getOptionValue(SSL_TRUSTSTORE));
}
if (cmd.hasOption(SSL_TRUSTSTORE_PW))
{
- clientEncOptions.truststore_password = cmd.getOptionValue(SSL_TRUSTSTORE_PW);
+ clientEncOptions = clientEncOptions.withTrustStorePassword(cmd.getOptionValue(SSL_TRUSTSTORE_PW));
}
if (cmd.hasOption(SSL_KEYSTORE))
{
- clientEncOptions.keystore = cmd.getOptionValue(SSL_KEYSTORE);
// if a keystore was provided, lets assume we'll need to use
- // it
- clientEncOptions.require_client_auth = true;
+ clientEncOptions = clientEncOptions.withKeyStore(cmd.getOptionValue(SSL_KEYSTORE))
+ .withRequireClientAuth(true);
}
if (cmd.hasOption(SSL_KEYSTORE_PW))
{
- clientEncOptions.keystore_password = cmd.getOptionValue(SSL_KEYSTORE_PW);
+ clientEncOptions = clientEncOptions.withKeyStorePassword(cmd.getOptionValue(SSL_KEYSTORE_PW));
}
if (cmd.hasOption(SSL_PROTOCOL))
{
- clientEncOptions.protocol = cmd.getOptionValue(SSL_PROTOCOL);
+ clientEncOptions = clientEncOptions.withProtocol(cmd.getOptionValue(SSL_PROTOCOL));
}
if (cmd.hasOption(SSL_ALGORITHM))
{
- clientEncOptions.algorithm = cmd.getOptionValue(SSL_ALGORITHM);
+ clientEncOptions = clientEncOptions.withAlgorithm(cmd.getOptionValue(SSL_ALGORITHM));
}
if (cmd.hasOption(SSL_STORE_TYPE))
{
- clientEncOptions.store_type = cmd.getOptionValue(SSL_STORE_TYPE);
+ clientEncOptions = clientEncOptions.withStoreType(cmd.getOptionValue(SSL_STORE_TYPE));
}
if (cmd.hasOption(SSL_CIPHER_SUITES))
{
- clientEncOptions.cipher_suites = cmd.getOptionValue(SSL_CIPHER_SUITES).split(",");
+ clientEncOptions = clientEncOptions.withCipherSuites(cmd.getOptionValue(SSL_CIPHER_SUITES).split(","));
}
+ if (cmd.hasOption(TARGET_KEYSPACE))
+ {
+ targetKeyspace = cmd.getOptionValue(TARGET_KEYSPACE);
+ if (StringUtils.isBlank(targetKeyspace))
+ {
+ errorMsg("Empty keyspace is not supported.", options);
+ }
+ }
return this;
}
catch (ParseException | ConfigurationException | MalformedURLException e)
@@ -558,10 +621,11 @@
options.addOption("ks", SSL_KEYSTORE, "KEYSTORE", "Client SSL: full path to keystore");
options.addOption("kspw", SSL_KEYSTORE_PW, "KEYSTORE-PASSWORD", "Client SSL: password of the keystore");
options.addOption("prtcl", SSL_PROTOCOL, "PROTOCOL", "Client SSL: connections protocol to use (default: TLS)");
- options.addOption("alg", SSL_ALGORITHM, "ALGORITHM", "Client SSL: algorithm (default: SunX509)");
+ options.addOption("alg", SSL_ALGORITHM, "ALGORITHM", "Client SSL: algorithm");
options.addOption("st", SSL_STORE_TYPE, "STORE-TYPE", "Client SSL: type of store");
options.addOption("ciphers", SSL_CIPHER_SUITES, "CIPHER-SUITES", "Client SSL: comma-separated list of encryption suites to use");
options.addOption("f", CONFIG_PATH, "path to config file", "cassandra.yaml file path for streaming throughput and client/server SSL.");
+ options.addOption("k", TARGET_KEYSPACE, "target keyspace name", "target keyspace name");
return options;
}
diff --git a/src/java/org/apache/cassandra/tools/NodeProbe.java b/src/java/org/apache/cassandra/tools/NodeProbe.java
index 2c4e409..4ea301b 100644
--- a/src/java/org/apache/cassandra/tools/NodeProbe.java
+++ b/src/java/org/apache/cassandra/tools/NodeProbe.java
@@ -42,6 +42,7 @@
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
+import javax.annotation.Nullable;
import javax.management.JMX;
import javax.management.MBeanServerConnection;
import javax.management.MalformedObjectNameException;
@@ -59,6 +60,7 @@
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.ColumnFamilyStoreMBean;
import org.apache.cassandra.db.HintedHandOffManagerMBean;
+import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.db.compaction.CompactionManager;
import org.apache.cassandra.db.compaction.CompactionManagerMBean;
import org.apache.cassandra.gms.FailureDetector;
@@ -69,13 +71,12 @@
import org.apache.cassandra.locator.DynamicEndpointSnitchMBean;
import org.apache.cassandra.locator.EndpointSnitchInfoMBean;
import org.apache.cassandra.metrics.CassandraMetricsRegistry;
-import org.apache.cassandra.metrics.TableMetrics.Sampler;
import org.apache.cassandra.metrics.StorageMetrics;
import org.apache.cassandra.metrics.TableMetrics;
import org.apache.cassandra.metrics.ThreadPoolMetrics;
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.net.MessagingServiceMBean;
-import org.apache.cassandra.schema.CompactionParams.TombstoneOption;
+import org.apache.cassandra.service.ActiveRepairServiceMBean;
import org.apache.cassandra.service.CacheService;
import org.apache.cassandra.service.CacheServiceMBean;
import org.apache.cassandra.service.GCInspector;
@@ -87,14 +88,17 @@
import org.apache.cassandra.streaming.StreamState;
import org.apache.cassandra.streaming.management.StreamStateCompositeData;
+import com.codahale.metrics.JmxReporter;
import com.google.common.base.Function;
import com.google.common.base.Strings;
+import com.google.common.collect.HashMultimap;
import com.google.common.collect.Iterables;
import com.google.common.collect.Maps;
import com.google.common.collect.Multimap;
import com.google.common.collect.Sets;
import com.google.common.util.concurrent.Uninterruptibles;
import org.apache.cassandra.tools.nodetool.GetTimeout;
+import org.apache.cassandra.utils.NativeLibrary;
/**
* JMX client operations for Cassandra.
@@ -104,11 +108,15 @@
private static final String fmtUrl = "service:jmx:rmi:///jndi/rmi://[%s]:%d/jmxrmi";
private static final String ssObjName = "org.apache.cassandra.db:type=StorageService";
private static final int defaultPort = 7199;
+
+ static long JMX_NOTIFICATION_POLL_INTERVAL_SECONDS = Long.getLong("cassandra.nodetool.jmx_notification_poll_interval_seconds", TimeUnit.SECONDS.convert(5, TimeUnit.MINUTES));
+
final String host;
final int port;
private String username;
private String password;
+
protected JMXConnector jmxc;
protected MBeanServerConnection mbeanServerConn;
protected CompactionManagerMBean compactionProxy;
@@ -124,6 +132,7 @@
protected StorageProxyMBean spProxy;
protected HintedHandOffManagerMBean hhProxy;
protected BatchlogManagerMBean bmProxy;
+ protected ActiveRepairServiceMBean arsProxy;
private boolean failed;
/**
@@ -187,7 +196,7 @@
protected void connect() throws IOException
{
JMXServiceURL jmxUrl = new JMXServiceURL(String.format(fmtUrl, host, port));
- Map<String,Object> env = new HashMap<String,Object>();
+ Map<String, Object> env = new HashMap<String, Object>();
if (username != null)
{
String[] creds = { username, password };
@@ -223,6 +232,8 @@
gossProxy = JMX.newMBeanProxy(mbeanServerConn, name, GossiperMBean.class);
name = new ObjectName(BatchlogManager.MBEAN_NAME);
bmProxy = JMX.newMBeanProxy(mbeanServerConn, name, BatchlogManagerMBean.class);
+ name = new ObjectName(ActiveRepairServiceMBean.MBEAN_NAME);
+ arsProxy = JMX.newMBeanProxy(mbeanServerConn, name, ActiveRepairServiceMBean.class);
}
catch (MalformedObjectNameException e)
{
@@ -267,9 +278,9 @@
return ssProxy.scrub(disableSnapshot, skipCorrupted, checkData, reinsertOverflowedTTL, jobs, keyspaceName, tables);
}
- public int verify(boolean extendedVerify, String keyspaceName, String... tableNames) throws IOException, ExecutionException, InterruptedException
+ public int verify(boolean extendedVerify, boolean checkVersion, boolean diskFailurePolicy, boolean mutateRepairStatus, boolean checkOwnsTokens, boolean quick, String keyspaceName, String... tableNames) throws IOException, ExecutionException, InterruptedException
{
- return ssProxy.verify(extendedVerify, keyspaceName, tableNames);
+ return ssProxy.verify(extendedVerify, checkVersion, diskFailurePolicy, mutateRepairStatus, checkOwnsTokens, quick, keyspaceName, tableNames);
}
public int upgradeSSTables(String keyspaceName, boolean excludeCurrentVersion, int jobs, String... tableNames) throws IOException, ExecutionException, InterruptedException
@@ -322,9 +333,9 @@
}
}
- public void verify(PrintStream out, boolean extendedVerify, String keyspaceName, String... tableNames) throws IOException, ExecutionException, InterruptedException
+ public void verify(PrintStream out, boolean extendedVerify, boolean checkVersion, boolean diskFailurePolicy, boolean mutateRepairStatus, boolean checkOwnsTokens, boolean quick, String keyspaceName, String... tableNames) throws IOException, ExecutionException, InterruptedException
{
- switch (verify(extendedVerify, keyspaceName, tableNames))
+ switch (verify(extendedVerify, checkVersion, diskFailurePolicy, mutateRepairStatus, checkOwnsTokens, quick, keyspaceName, tableNames))
{
case 1:
failed = true;
@@ -388,6 +399,11 @@
ssProxy.forceKeyspaceFlush(keyspaceName, tableNames);
}
+ public String getKeyspaceReplicationInfo(String keyspaceName)
+ {
+ return ssProxy.getKeyspaceReplicationInfo(keyspaceName);
+ }
+
public void repairAsync(final PrintStream out, final String keyspace, Map<String, String> options) throws IOException
{
RepairRunner runner = new RepairRunner(out, ssProxy, keyspace, options);
@@ -400,7 +416,7 @@
}
catch (Exception e)
{
- throw new IOException(e) ;
+ throw new IOException(e);
}
finally
{
@@ -416,19 +432,23 @@
}
}
}
+ public Map<String, List<CompositeData>> getPartitionSample(int capacity, int durationMillis, int count, List<String> samplers) throws OpenDataException
+ {
+ return ssProxy.samplePartitions(durationMillis, capacity, count, samplers);
+ }
- public Map<Sampler, CompositeData> getPartitionSample(String ks, String cf, int capacity, int duration, int count, List<Sampler> samplers) throws OpenDataException
+ public Map<String, List<CompositeData>> getPartitionSample(String ks, String cf, int capacity, int durationMillis, int count, List<String> samplers) throws OpenDataException
{
ColumnFamilyStoreMBean cfsProxy = getCfsProxy(ks, cf);
- for(Sampler sampler : samplers)
+ for(String sampler : samplers)
{
- cfsProxy.beginLocalSampling(sampler.name(), capacity);
+ cfsProxy.beginLocalSampling(sampler, capacity, durationMillis);
}
- Uninterruptibles.sleepUninterruptibly(duration, TimeUnit.MILLISECONDS);
- Map<Sampler, CompositeData> result = Maps.newHashMap();
- for(Sampler sampler : samplers)
+ Uninterruptibles.sleepUninterruptibly(durationMillis, TimeUnit.MILLISECONDS);
+ Map<String, List<CompositeData>> result = Maps.newHashMap();
+ for(String sampler : samplers)
{
- result.put(sampler, cfsProxy.finishLocalSampling(sampler.name(), count));
+ result.put(sampler, cfsProxy.finishLocalSampling(sampler, count));
}
return result;
}
@@ -453,39 +473,39 @@
ssProxy.drain();
}
- public Map<String, String> getTokenToEndpointMap()
+ public Map<String, String> getTokenToEndpointMap(boolean withPort)
{
- return ssProxy.getTokenToEndpointMap();
+ return withPort ? ssProxy.getTokenToEndpointWithPortMap() : ssProxy.getTokenToEndpointMap();
}
- public List<String> getLiveNodes()
+ public List<String> getLiveNodes(boolean withPort)
{
- return ssProxy.getLiveNodes();
+ return withPort ? ssProxy.getLiveNodesWithPort() : ssProxy.getLiveNodes();
}
- public List<String> getJoiningNodes()
+ public List<String> getJoiningNodes(boolean withPort)
{
- return ssProxy.getJoiningNodes();
+ return withPort ? ssProxy.getJoiningNodesWithPort() : ssProxy.getJoiningNodes();
}
- public List<String> getLeavingNodes()
+ public List<String> getLeavingNodes(boolean withPort)
{
- return ssProxy.getLeavingNodes();
+ return withPort ? ssProxy.getLeavingNodesWithPort() : ssProxy.getLeavingNodes();
}
- public List<String> getMovingNodes()
+ public List<String> getMovingNodes(boolean withPort)
{
- return ssProxy.getMovingNodes();
+ return withPort ? ssProxy.getMovingNodesWithPort() : ssProxy.getMovingNodes();
}
- public List<String> getUnreachableNodes()
+ public List<String> getUnreachableNodes(boolean withPort)
{
- return ssProxy.getUnreachableNodes();
+ return withPort ? ssProxy.getUnreachableNodesWithPort() : ssProxy.getUnreachableNodes();
}
- public Map<String, String> getLoadMap()
+ public Map<String, String> getLoadMap(boolean withPort)
{
- return ssProxy.getLoadMap();
+ return withPort ? ssProxy.getLoadMapWithPort() : ssProxy.getLoadMap();
}
public Map<InetAddress, Float> getOwnership()
@@ -493,11 +513,26 @@
return ssProxy.getOwnership();
}
+ public Map<String, Float> getOwnershipWithPort()
+ {
+ return ssProxy.getOwnershipWithPort();
+ }
+
public Map<InetAddress, Float> effectiveOwnership(String keyspace) throws IllegalStateException
{
return ssProxy.effectiveOwnership(keyspace);
}
+ public Map<String, Float> effectiveOwnershipWithPort(String keyspace) throws IllegalStateException
+ {
+ return ssProxy.effectiveOwnershipWithPort(keyspace);
+ }
+
+ public MBeanServerConnection getMbeanServerConn()
+ {
+ return mbeanServerConn;
+ }
+
public CacheServiceMBean getCacheServiceMBean()
{
String cachePath = "org.apache.cassandra.db:type=Caches";
@@ -560,9 +595,9 @@
return ssProxy.getLocalHostId();
}
- public Map<String, String> getHostIdMap()
+ public Map<String, String> getHostIdMap(boolean withPort)
{
- return ssProxy.getEndpointToHostId();
+ return withPort ? ssProxy.getEndpointWithPortToHostId() : ssProxy.getEndpointToHostId();
}
public String getLoadString()
@@ -679,9 +714,9 @@
ssProxy.joinRing();
}
- public void decommission() throws InterruptedException
+ public void decommission(boolean force) throws InterruptedException
{
- ssProxy.decommission();
+ ssProxy.decommission(force);
}
public void move(String newToken) throws IOException
@@ -694,9 +729,9 @@
ssProxy.removeNode(token);
}
- public String getRemovalStatus()
+ public String getRemovalStatus(boolean withPort)
{
- return ssProxy.getRemovalStatus();
+ return withPort ? ssProxy.getRemovalStatusWithPort() : ssProxy.getRemovalStatus();
}
public void forceRemoveCompletion()
@@ -709,6 +744,16 @@
gossProxy.assassinateEndpoint(address);
}
+ public List<String> reloadSeeds()
+ {
+ return gossProxy.reloadSeeds();
+ }
+
+ public List<String> getSeeds()
+ {
+ return gossProxy.getSeeds();
+ }
+
/**
* Set the compaction threshold
*
@@ -731,6 +776,11 @@
ssProxy.enableAutoCompaction(ks, tableNames);
}
+ public Map<String, Boolean> getAutoCompactionDisabled(String ks, String ... tableNames) throws IOException
+ {
+ return ssProxy.getAutoCompactionStatus(ks, tableNames);
+ }
+
public void setIncrementalBackupsEnabled(boolean enabled)
{
ssProxy.setIncrementalBackupsEnabled(enabled);
@@ -762,6 +812,11 @@
ssProxy.setHintedHandoffThrottleInKB(throttleInKB);
}
+ public List<String> getEndpointsWithPort(String keyspace, String cf, String key)
+ {
+ return ssProxy.getNaturalEndpointsWithPort(keyspace, cf, key);
+ }
+
public List<InetAddress> getEndpoints(String keyspace, String cf, String key)
{
return ssProxy.getNaturalEndpoints(keyspace, cf, key);
@@ -868,11 +923,6 @@
return spProxy;
}
- public MessagingServiceMBean getMessagingServiceProxy()
- {
- return msProxy;
- }
-
public StorageServiceMBean getStorageService() {
return ssProxy;
}
@@ -1027,21 +1077,6 @@
return ssProxy.isGossipRunning();
}
- public void stopThriftServer()
- {
- ssProxy.stopRPCServer();
- }
-
- public void startThriftServer()
- {
- ssProxy.startRPCServer();
- }
-
- public boolean isThriftServerRunning()
- {
- return ssProxy.isRPCServerRunning();
- }
-
public void stopCassandraDaemon()
{
ssProxy.stopDaemon();
@@ -1062,6 +1097,16 @@
return ssProxy.getCompactionThroughputMbPerSec();
}
+ public void setBatchlogReplayThrottle(int value)
+ {
+ ssProxy.setBatchlogReplayThrottleInKB(value);
+ }
+
+ public int getBatchlogReplayThrottle()
+ {
+ return ssProxy.getBatchlogReplayThrottleInKB();
+ }
+
public void setConcurrentCompactors(int value)
{
ssProxy.setConcurrentCompactors(value);
@@ -1072,6 +1117,26 @@
return ssProxy.getConcurrentCompactors();
}
+ public void setConcurrentViewBuilders(int value)
+ {
+ ssProxy.setConcurrentViewBuilders(value);
+ }
+
+ public int getConcurrentViewBuilders()
+ {
+ return ssProxy.getConcurrentViewBuilders();
+ }
+
+ public void setMaxHintWindow(int value)
+ {
+ spProxy.setMaxHintWindow(value);
+ }
+
+ public int getMaxHintWindow()
+ {
+ return spProxy.getMaxHintWindow();
+ }
+
public long getTimeout(String type)
{
switch (type)
@@ -1090,8 +1155,10 @@
return ssProxy.getCasContentionTimeout();
case "truncate":
return ssProxy.getTruncateRpcTimeout();
- case "streamingsocket":
- return (long) ssProxy.getStreamingSocketTimeout();
+ case "internodeconnect":
+ return ssProxy.getInternodeTcpConnectTimeoutInMS();
+ case "internodeuser":
+ return ssProxy.getInternodeTcpUserTimeoutInMS();
default:
throw new RuntimeException("Timeout type requires one of (" + GetTimeout.TIMEOUT_TYPES + ")");
}
@@ -1114,7 +1181,7 @@
public int getExceptionCount()
{
- return (int)StorageMetrics.exceptions.getCount();
+ return (int)StorageMetrics.uncaughtExceptions.getCount();
}
public Map<String, Integer> getDroppedMessages()
@@ -1122,19 +1189,25 @@
return msProxy.getDroppedMessages();
}
+ @Deprecated
public void loadNewSSTables(String ksName, String cfName)
{
ssProxy.loadNewSSTables(ksName, cfName);
}
+ public List<String> importNewSSTables(String ksName, String cfName, Set<String> srcPaths, boolean resetLevel, boolean clearRepaired, boolean verifySSTables, boolean verifyTokens, boolean invalidateCaches, boolean extendedVerify)
+ {
+ return getCfsProxy(ksName, cfName).importNewSSTables(srcPaths, resetLevel, clearRepaired, verifySSTables, verifyTokens, invalidateCaches, extendedVerify);
+ }
+
public void rebuildIndex(String ksName, String cfName, String... idxNames)
{
ssProxy.rebuildSecondaryIndex(ksName, cfName, idxNames);
}
- public String getGossipInfo()
+ public String getGossipInfo(boolean withPort)
{
- return fdProxy.getAllEndpointStates();
+ return withPort ? fdProxy.getAllEndpointStatesWithPort() : fdProxy.getAllEndpointStates();
}
public void stop(String string)
@@ -1170,10 +1243,11 @@
case "truncate":
ssProxy.setTruncateRpcTimeout(value);
break;
- case "streamingsocket":
- if (value > Integer.MAX_VALUE)
- throw new RuntimeException("streamingsocket timeout must be less than " + Integer.MAX_VALUE);
- ssProxy.setStreamingSocketTimeout((int) value);
+ case "internodeconnect":
+ ssProxy.setInternodeTcpConnectTimeoutInMS((int) value);
+ break;
+ case "internodeuser":
+ ssProxy.setInternodeTcpUserTimeoutInMS((int) value);
break;
default:
throw new RuntimeException("Timeout type requires one of (" + GetTimeout.TIMEOUT_TYPES + ")");
@@ -1205,9 +1279,9 @@
return ssProxy.getSchemaVersion();
}
- public List<String> describeRing(String keyspaceName) throws IOException
+ public List<String> describeRing(String keyspaceName, boolean withPort) throws IOException
{
- return ssProxy.describeRingJMX(keyspaceName);
+ return withPort ? ssProxy.describeRingWithPortJMX(keyspaceName) : ssProxy.describeRingJMX(keyspaceName);
}
public void rebuild(String sourceDc, String keyspace, String tokens, String specificSources)
@@ -1235,6 +1309,11 @@
return failed;
}
+ public void failed()
+ {
+ this.failed = true;
+ }
+
public long getReadRepairAttempted()
{
return spProxy.getReadRepairAttempted();
@@ -1294,9 +1373,64 @@
}
}
+ private static Multimap<String, String> getJmxThreadPools(MBeanServerConnection mbeanServerConn)
+ {
+ try
+ {
+ Multimap<String, String> threadPools = HashMultimap.create();
+
+ Set<ObjectName> threadPoolObjectNames = mbeanServerConn.queryNames(
+ new ObjectName("org.apache.cassandra.metrics:type=ThreadPools,*"),
+ null);
+
+ for (ObjectName oName : threadPoolObjectNames)
+ {
+ threadPools.put(oName.getKeyProperty("path"), oName.getKeyProperty("scope"));
+ }
+
+ return threadPools;
+ }
+ catch (MalformedObjectNameException e)
+ {
+ throw new RuntimeException("Bad query to JMX server: ", e);
+ }
+ catch (IOException e)
+ {
+ throw new RuntimeException("Error getting threadpool names from JMX", e);
+ }
+ }
+
public Object getThreadPoolMetric(String pathName, String poolName, String metricName)
{
- return ThreadPoolMetrics.getJmxMetric(mbeanServerConn, pathName, poolName, metricName);
+ String name = String.format("org.apache.cassandra.metrics:type=ThreadPools,path=%s,scope=%s,name=%s",
+ pathName, poolName, metricName);
+
+ try
+ {
+ ObjectName oName = new ObjectName(name);
+ if (!mbeanServerConn.isRegistered(oName))
+ {
+ return "N/A";
+ }
+
+ switch (metricName)
+ {
+ case ThreadPoolMetrics.ACTIVE_TASKS:
+ case ThreadPoolMetrics.PENDING_TASKS:
+ case ThreadPoolMetrics.COMPLETED_TASKS:
+ case ThreadPoolMetrics.MAX_POOL_SIZE:
+ return JMX.newMBeanProxy(mbeanServerConn, oName, JmxReporter.JmxGaugeMBean.class).getValue();
+ case ThreadPoolMetrics.TOTAL_BLOCKED_TASKS:
+ case ThreadPoolMetrics.CURRENTLY_BLOCKED_TASKS:
+ return JMX.newMBeanProxy(mbeanServerConn, oName, JmxReporter.JmxCounterMBean.class).getCount();
+ default:
+ throw new AssertionError("Unknown metric name " + metricName);
+ }
+ }
+ catch (Exception e)
+ {
+ throw new RuntimeException("Error reading: " + name, e);
+ }
}
/**
@@ -1305,7 +1439,7 @@
*/
public Multimap<String, String> getThreadPools()
{
- return ThreadPoolMetrics.getJmxThreadPools(mbeanServerConn);
+ return getJmxThreadPools(mbeanServerConn);
}
public int getNumberOfTables()
@@ -1351,6 +1485,7 @@
case "EstimatedPartitionCount":
case "KeyCacheHitRate":
case "LiveSSTableCount":
+ case "OldVersionSSTableCount":
case "MaxPartitionSize":
case "MeanPartitionSize":
case "MemtableColumnsCount":
@@ -1358,6 +1493,9 @@
case "MemtableOffHeapSize":
case "MinPartitionSize":
case "PercentRepaired":
+ case "BytesRepaired":
+ case "BytesUnrepaired":
+ case "BytesPendingRepair":
case "RecentBloomFilterFalsePositives":
case "RecentBloomFilterFalseRatio":
case "SnapshotsSize":
@@ -1408,6 +1546,20 @@
}
}
+ public CassandraMetricsRegistry.JmxTimerMBean getMessagingQueueWaitMetrics(String verb)
+ {
+ try
+ {
+ return JMX.newMBeanProxy(mbeanServerConn,
+ new ObjectName("org.apache.cassandra.metrics:name=" + verb + "-WaitLatency,type=Messaging"),
+ CassandraMetricsRegistry.JmxTimerMBean.class);
+ }
+ catch (MalformedObjectNameException e)
+ {
+ throw new RuntimeException(e);
+ }
+ }
+
/**
* Retrieve Proxy metrics
* @param metricName CompletedTasks, PendingTasks, BytesCompacted or TotalCompactionsCompleted.
@@ -1444,6 +1596,33 @@
/**
* Retrieve Proxy metrics
+ * @param metricName
+ */
+ public Object getClientMetric(String metricName)
+ {
+ try
+ {
+ switch(metricName)
+ {
+ case "connections": // List<Map<String,String>> - list of all native connections and their properties
+ case "connectedNativeClients": // number of connected native clients
+ case "connectedNativeClientsByUser": // number of native clients by username
+ case "clientsByProtocolVersion": // number of native clients by username
+ return JMX.newMBeanProxy(mbeanServerConn,
+ new ObjectName("org.apache.cassandra.metrics:type=Client,name=" + metricName),
+ CassandraMetricsRegistry.JmxGaugeMBean.class).getValue();
+ default:
+ throw new RuntimeException("Unknown client metric " + metricName);
+ }
+ }
+ catch (MalformedObjectNameException e)
+ {
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * Retrieve Proxy metrics
* @param metricName Exceptions, Load, TotalHints or TotalHintsInProgress.
*/
public long getStorageMetric(String metricName)
@@ -1500,7 +1679,7 @@
}
catch (Exception e)
{
- throw new RuntimeException("Error setting log for " + classQualifier +" on level " + level +". Please check logback configuration and ensure to have <jmxConfigurator /> set", e);
+ throw new RuntimeException("Error setting log for " + classQualifier + " on level " + level + ". Please check logback configuration and ensure to have <jmxConfigurator /> set", e);
}
}
@@ -1509,6 +1688,11 @@
return ssProxy.getLoggingLevels();
}
+ public long getPid()
+ {
+ return NativeLibrary.getProcessID();
+ }
+
public void resumeBootstrap(PrintStream out) throws IOException
{
BootstrapMonitor monitor = new BootstrapMonitor(out);
@@ -1546,6 +1730,16 @@
}
}
+ public Map<String, List<Integer>> getMaximumPoolSizes(List<String> stageNames)
+ {
+ return ssProxy.getConcurrency(stageNames);
+ }
+
+ public void setConcurrency(String stageName, int coreThreads, int maxConcurrency)
+ {
+ ssProxy.setConcurrency(stageName, coreThreads, maxConcurrency);
+ }
+
public void replayBatchlog() throws IOException
{
try
@@ -1558,17 +1752,77 @@
}
}
- public TabularData getFailureDetectorPhilValues()
+ public TabularData getFailureDetectorPhilValues(boolean withPort)
{
try
{
- return fdProxy.getPhiValues();
+ return withPort ? fdProxy.getPhiValuesWithPort() : fdProxy.getPhiValues();
}
catch (OpenDataException e)
{
throw new RuntimeException(e);
}
}
+
+ public ActiveRepairServiceMBean getRepairServiceProxy()
+ {
+ return arsProxy;
+ }
+
+ public void reloadSslCerts() throws IOException
+ {
+ msProxy.reloadSslCertificates();
+ }
+
+ public void clearConnectionHistory()
+ {
+ ssProxy.clearConnectionHistory();
+ }
+
+ public void disableAuditLog()
+ {
+ ssProxy.disableAuditLog();
+ }
+
+ public void enableAuditLog(String loggerName, Map<String, String> parameters, String includedKeyspaces ,String excludedKeyspaces ,String includedCategories ,String excludedCategories ,String includedUsers ,String excludedUsers)
+ {
+ ssProxy.enableAuditLog(loggerName, parameters, includedKeyspaces, excludedKeyspaces, includedCategories, excludedCategories, includedUsers, excludedUsers);
+ }
+
+ public void enableAuditLog(String loggerName, String includedKeyspaces ,String excludedKeyspaces ,String includedCategories ,String excludedCategories ,String includedUsers ,String excludedUsers)
+ {
+ this.enableAuditLog(loggerName, Collections.emptyMap(), includedKeyspaces, excludedKeyspaces, includedCategories, excludedCategories, includedUsers, excludedUsers);
+ }
+
+ public void enableOldProtocolVersions()
+ {
+ ssProxy.enableNativeTransportOldProtocolVersions();
+ }
+
+ public void disableOldProtocolVersions()
+ {
+ ssProxy.disableNativeTransportOldProtocolVersions();
+ }
+
+ public MessagingServiceMBean getMessagingServiceProxy()
+ {
+ return msProxy;
+ }
+
+ public void enableFullQueryLogger(String path, String rollCycle, Boolean blocking, int maxQueueWeight, long maxLogSize, @Nullable String archiveCommand, int maxArchiveRetries)
+ {
+ ssProxy.enableFullQueryLogger(path, rollCycle, blocking, maxQueueWeight, maxLogSize, archiveCommand, maxArchiveRetries);
+ }
+
+ public void stopFullQueryLogger()
+ {
+ ssProxy.stopFullQueryLogger();
+ }
+
+ public void resetFullQueryLogger()
+ {
+ ssProxy.resetFullQueryLogger();
+ }
}
class ColumnFamilyStoreMBeanIterator implements Iterator<Map.Entry<String, ColumnFamilyStoreMBean>>
@@ -1592,8 +1846,8 @@
return keyspaceNameCmp;
// get CF name and split it for index name
- String e1CF[] = e1.getValue().getColumnFamilyName().split("\\.");
- String e2CF[] = e2.getValue().getColumnFamilyName().split("\\.");
+ String e1CF[] = e1.getValue().getTableName().split("\\.");
+ String e2CF[] = e2.getValue().getTableName().split("\\.");
assert e1CF.length <= 2 && e2CF.length <= 2 : "unexpected split count for table name";
//if neither are indexes, just compare CF names
diff --git a/src/java/org/apache/cassandra/tools/NodeProbeFactory.java b/src/java/org/apache/cassandra/tools/NodeProbeFactory.java
deleted file mode 100644
index 577bcfa..0000000
--- a/src/java/org/apache/cassandra/tools/NodeProbeFactory.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.tools;
-
-import java.io.IOException;
-
-public class NodeProbeFactory implements INodeProbeFactory
-{
- public NodeProbe create(String host, int port) throws IOException
- {
- return new NodeProbe(host, port);
- }
-
- public NodeProbe create(String host, int port, String username, String password) throws IOException
- {
- return new NodeProbe(host, port, username, password);
- }
-}
diff --git a/src/java/org/apache/cassandra/tools/NodeTool.java b/src/java/org/apache/cassandra/tools/NodeTool.java
index f716573..bf5e5cc 100644
--- a/src/java/org/apache/cassandra/tools/NodeTool.java
+++ b/src/java/org/apache/cassandra/tools/NodeTool.java
@@ -17,34 +17,64 @@
*/
package org.apache.cassandra.tools;
-import java.io.*;
-import java.net.InetAddress;
-import java.net.UnknownHostException;
-import java.text.SimpleDateFormat;
-import java.util.*;
-import java.util.Map.Entry;
-import java.util.function.Consumer;
-
-import com.google.common.base.Joiner;
-import com.google.common.base.Throwables;
-import com.google.common.collect.*;
-
-import io.airlift.command.*;
-
-import org.apache.cassandra.locator.EndpointSnitchInfoMBean;
-import org.apache.cassandra.tools.nodetool.*;
-import org.apache.cassandra.utils.FBUtilities;
-
import static com.google.common.base.Throwables.getStackTraceAsString;
import static com.google.common.collect.Iterables.toArray;
import static com.google.common.collect.Lists.newArrayList;
import static java.lang.Integer.parseInt;
import static java.lang.String.format;
import static org.apache.commons.lang3.ArrayUtils.EMPTY_STRING_ARRAY;
-import static org.apache.commons.lang3.StringUtils.*;
+import static org.apache.commons.lang3.StringUtils.EMPTY;
+import static org.apache.commons.lang3.StringUtils.isEmpty;
+import static org.apache.commons.lang3.StringUtils.isNotEmpty;
+
+import java.io.Console;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileWriter;
+import java.io.IOError;
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Scanner;
+import java.util.SortedMap;
+import java.util.function.Consumer;
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Throwables;
+
+import org.apache.cassandra.locator.EndpointSnitchInfoMBean;
+import org.apache.cassandra.tools.nodetool.*;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.tools.nodetool.Sjk;
+
+import com.google.common.collect.Maps;
+
+import io.airlift.airline.Cli;
+import io.airlift.airline.Help;
+import io.airlift.airline.Option;
+import io.airlift.airline.OptionType;
+import io.airlift.airline.ParseArgumentsMissingException;
+import io.airlift.airline.ParseArgumentsUnexpectedException;
+import io.airlift.airline.ParseCommandMissingException;
+import io.airlift.airline.ParseCommandUnrecognizedException;
+import io.airlift.airline.ParseOptionConversionException;
+import io.airlift.airline.ParseOptionMissingException;
+import io.airlift.airline.ParseOptionMissingValueException;
public class NodeTool
{
+ static
+ {
+ FBUtilities.preventIllegalAccessWarnings();
+ }
+
private static final String HISTORYFILE = "nodetool.history";
private final INodeProbeFactory nodeProbeFactory;
@@ -72,6 +102,7 @@
TableHistograms.class,
Cleanup.class,
ClearSnapshot.class,
+ ClientStats.class,
Compact.class,
Scrub.class,
Verify.class,
@@ -89,17 +120,23 @@
EnableGossip.class,
DisableGossip.class,
EnableHandoff.class,
- EnableThrift.class,
+ EnableFullQueryLog.class,
+ DisableFullQueryLog.class,
GcStats.class,
+ GetBatchlogReplayTrottle.class,
GetCompactionThreshold.class,
GetCompactionThroughput.class,
+ GetConcurrency.class,
GetTimeout.class,
GetStreamThroughput.class,
GetTraceProbability.class,
GetInterDCStreamThroughput.class,
GetEndpoints.class,
+ GetSeeds.class,
GetSSTables.class,
+ GetMaxHintWindow.class,
GossipInfo.class,
+ Import.class,
InvalidateKeyCache.class,
InvalidateRowCache.class,
InvalidateCounterCache.class,
@@ -107,31 +144,41 @@
Move.class,
PauseHandoff.class,
ResumeHandoff.class,
+ ProfileLoad.class,
ProxyHistograms.class,
Rebuild.class,
Refresh.class,
RemoveNode.class,
Assassinate.class,
+ ReloadSeeds.class,
+ ResetFullQueryLog.class,
Repair.class,
+ RepairAdmin.class,
ReplayBatchlog.class,
SetCacheCapacity.class,
+ SetConcurrency.class,
SetHintedHandoffThrottleInKB.class,
+ SetBatchlogReplayThrottle.class,
SetCompactionThreshold.class,
SetCompactionThroughput.class,
GetConcurrentCompactors.class,
SetConcurrentCompactors.class,
+ GetConcurrentViewBuilders.class,
+ SetConcurrentViewBuilders.class,
+ SetConcurrency.class,
SetTimeout.class,
SetStreamThroughput.class,
SetInterDCStreamThroughput.class,
SetTraceProbability.class,
+ SetMaxHintWindow.class,
Snapshot.class,
ListSnapshots.class,
Status.class,
StatusBinary.class,
StatusGossip.class,
- StatusThrift.class,
StatusBackup.class,
StatusHandoff.class,
+ StatusAutoCompaction.class,
Stop.class,
StopDaemon.class,
Version.class,
@@ -144,7 +191,6 @@
ReloadLocalSchema.class,
ReloadTriggers.class,
SetCacheKeysToSave.class,
- DisableThrift.class,
DisableHandoff.class,
Drain.class,
TruncateHints.class,
@@ -152,12 +198,18 @@
TopPartitions.class,
SetLoggingLevel.class,
GetLoggingLevels.class,
+ Sjk.class,
DisableHintsForDC.class,
EnableHintsForDC.class,
FailureDetectorInfo.class,
RefreshSizeEstimates.class,
RelocateSSTables.class,
- ViewBuildStatus.class
+ ViewBuildStatus.class,
+ ReloadSslCertificates.class,
+ EnableAuditLog.class,
+ DisableAuditLog.class,
+ EnableOldProtocolVersions.class,
+ DisableOldProtocolVersions.class
);
Cli.CliBuilder<Consumer<INodeProbeFactory>> builder = Cli.builder("nodetool");
@@ -260,6 +312,9 @@
@Option(type = OptionType.GLOBAL, name = {"-pwf", "--password-file"}, description = "Path to the JMX password file")
private String passwordFilePath = EMPTY;
+ @Option(type = OptionType.GLOBAL, name = { "-pp", "--print-port"}, description = "Operate in 4.0 mode with hosts disambiguated by port number", arity = 0)
+ protected boolean printPort = false;
+
private INodeProbeFactory nodeProbeFactory;
public void accept(INodeProbeFactory nodeProbeFactory)
@@ -417,4 +472,27 @@
}
return ownershipByDc;
}
+
+ public static SortedMap<String, SetHostStatWithPort> getOwnershipByDcWithPort(NodeProbe probe, boolean resolveIp,
+ Map<String, String> tokenToEndpoint,
+ Map<String, Float> ownerships)
+ {
+ SortedMap<String, SetHostStatWithPort> ownershipByDc = Maps.newTreeMap();
+ EndpointSnitchInfoMBean epSnitchInfo = probe.getEndpointSnitchInfoProxy();
+ try
+ {
+ for (Entry<String, String> tokenAndEndPoint : tokenToEndpoint.entrySet())
+ {
+ String dc = epSnitchInfo.getDatacenter(tokenAndEndPoint.getValue());
+ if (!ownershipByDc.containsKey(dc))
+ ownershipByDc.put(dc, new SetHostStatWithPort(resolveIp));
+ ownershipByDc.get(dc).add(tokenAndEndPoint.getKey(), tokenAndEndPoint.getValue(), ownerships);
+ }
+ }
+ catch (UnknownHostException e)
+ {
+ throw new RuntimeException(e);
+ }
+ return ownershipByDc;
+ }
}
diff --git a/src/java/org/apache/cassandra/tools/ReloadSslCertificates.java b/src/java/org/apache/cassandra/tools/ReloadSslCertificates.java
new file mode 100644
index 0000000..a572648
--- /dev/null
+++ b/src/java/org/apache/cassandra/tools/ReloadSslCertificates.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.tools;
+
+import java.io.IOException;
+
+import io.airlift.airline.Command;
+
+@Command(name = "reloadssl", description = "Signals Cassandra to reload SSL certificates")
+public class ReloadSslCertificates extends NodeTool.NodeToolCmd
+{
+ @Override
+ public void execute(NodeProbe probe)
+ {
+ try
+ {
+ probe.reloadSslCerts();
+ }
+ catch (IOException e)
+ {
+ throw new RuntimeException("Failed to reload SSL certificates. Please check the SSL certificates", e);
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/tools/RepairRunner.java b/src/java/org/apache/cassandra/tools/RepairRunner.java
index 961916a..593bc26 100644
--- a/src/java/org/apache/cassandra/tools/RepairRunner.java
+++ b/src/java/org/apache/cassandra/tools/RepairRunner.java
@@ -20,9 +20,15 @@
import java.io.IOException;
import java.io.PrintStream;
import java.text.SimpleDateFormat;
+import java.util.List;
import java.util.Map;
+import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Condition;
+import com.google.common.base.Throwables;
+
+import org.apache.cassandra.repair.messages.RepairOption;
+import org.apache.cassandra.service.ActiveRepairService;
import org.apache.cassandra.service.StorageServiceMBean;
import org.apache.cassandra.utils.concurrent.SimpleCondition;
import org.apache.cassandra.utils.progress.ProgressEvent;
@@ -37,10 +43,9 @@
private final StorageServiceMBean ssProxy;
private final String keyspace;
private final Map<String, String> options;
- private final Condition condition = new SimpleCondition();
+ private final SimpleCondition condition = new SimpleCondition();
private int cmd;
- private volatile boolean hasNotificationLost;
private volatile Exception error;
public RepairRunner(PrintStream out, StorageServiceMBean ssProxy, String keyspace, Map<String, String> options)
@@ -57,20 +62,28 @@
if (cmd <= 0)
{
// repairAsync can only return 0 for replication factor 1.
- String message = String.format("[%s] Replication factor is 1. No repair is needed for keyspace '%s'", format.format(System.currentTimeMillis()), keyspace);
- out.println(message);
+ String message = String.format("Replication factor is 1. No repair is needed for keyspace '%s'", keyspace);
+ printMessage(message);
}
else
{
- condition.await();
+ while (!condition.await(NodeProbe.JMX_NOTIFICATION_POLL_INTERVAL_SECONDS, TimeUnit.SECONDS))
+ {
+ queryForCompletedRepair(String.format("After waiting for poll interval of %s seconds",
+ NodeProbe.JMX_NOTIFICATION_POLL_INTERVAL_SECONDS));
+ }
+ Exception error = this.error;
+ if (error == null)
+ {
+ // notifications are lossy so its possible to see complete and not error; request latest state
+ // from the server
+ queryForCompletedRepair("condition satisfied");
+ error = this.error;
+ }
if (error != null)
{
throw error;
}
- if (hasNotificationLost)
- {
- out.println(String.format("There were some lost notification(s). You should check server log for repair status of keyspace %s", keyspace));
- }
}
}
@@ -83,7 +96,11 @@
@Override
public void handleNotificationLost(long timestamp, String message)
{
- hasNotificationLost = true;
+ if (cmd > 0)
+ {
+ // Check to see if the lost notification was a completion message
+ queryForCompletedRepair("After receiving lost notification");
+ }
}
@Override
@@ -96,8 +113,8 @@
public void handleConnectionFailed(long timestamp, String message)
{
error = new IOException(String.format("[%s] JMX connection closed. You should check server log for repair status of keyspace %s"
- + "(Subsequent keyspaces are not going to be repaired).",
- format.format(timestamp), keyspace));
+ + "(Subsequent keyspaces are not going to be repaired).",
+ format.format(timestamp), keyspace));
condition.signalAll();
}
@@ -105,19 +122,73 @@
public void progress(String tag, ProgressEvent event)
{
ProgressEventType type = event.getType();
- String message = String.format("[%s] %s", format.format(System.currentTimeMillis()), event.getMessage());
+ String message = event.getMessage();
if (type == ProgressEventType.PROGRESS)
{
- message = message + " (progress: " + (int)event.getProgressPercentage() + "%)";
+ message = message + " (progress: " + (int) event.getProgressPercentage() + "%)";
}
- out.println(message);
+ printMessage(message);
if (type == ProgressEventType.ERROR)
{
- error = new RuntimeException("Repair job has failed with the error message: " + message);
+ error = new RuntimeException(String.format("Repair job has failed with the error message: %s. " +
+ "Check the logs on the repair participants for further details",
+ message));
}
if (type == ProgressEventType.COMPLETE)
{
condition.signalAll();
}
}
+
+
+ private void queryForCompletedRepair(String triggeringCondition)
+ {
+ List<String> status = ssProxy.getParentRepairStatus(cmd);
+ String queriedString = "queried for parent session status and";
+ if (status == null)
+ {
+ String message = String.format("%s %s couldn't find repair status for cmd: %s", triggeringCondition,
+ queriedString, cmd);
+ printMessage(message);
+ }
+ else
+ {
+ ActiveRepairService.ParentRepairStatus parentRepairStatus = ActiveRepairService.ParentRepairStatus.valueOf(status.get(0));
+ List<String> messages = status.subList(1, status.size());
+ switch (parentRepairStatus)
+ {
+ case COMPLETED:
+ case FAILED:
+ printMessage(String.format("%s %s discovered repair %s.",
+ triggeringCondition,
+ queriedString, parentRepairStatus.name().toLowerCase()));
+ if (parentRepairStatus == ActiveRepairService.ParentRepairStatus.FAILED)
+ {
+ error = new IOException(messages.get(0));
+ }
+ printMessages(messages);
+ condition.signalAll();
+ break;
+ case IN_PROGRESS:
+ break;
+ default:
+ printMessage(String.format("WARNING Encountered unexpected RepairRunnable.ParentRepairStatus: %s", parentRepairStatus));
+ printMessages(messages);
+ break;
+ }
+ }
+ }
+
+ private void printMessages(List<String> messages)
+ {
+ for (String message : messages)
+ {
+ printMessage(message);
+ }
+ }
+
+ private void printMessage(String message)
+ {
+ out.println(String.format("[%s] %s", this.format.format(System.currentTimeMillis()), message));
+ }
}
diff --git a/src/java/org/apache/cassandra/tools/SSTableExpiredBlockers.java b/src/java/org/apache/cassandra/tools/SSTableExpiredBlockers.java
index 1f407cb..56c57d9 100644
--- a/src/java/org/apache/cassandra/tools/SSTableExpiredBlockers.java
+++ b/src/java/org/apache/cassandra/tools/SSTableExpiredBlockers.java
@@ -17,7 +17,6 @@
*/
package org.apache.cassandra.tools;
-import java.io.IOException;
import java.io.PrintStream;
import java.util.Collections;
import java.util.HashSet;
@@ -27,8 +26,8 @@
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.Multimap;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.Schema;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Directories;
import org.apache.cassandra.db.Keyspace;
@@ -46,7 +45,7 @@
*/
public class SSTableExpiredBlockers
{
- public static void main(String[] args) throws IOException
+ public static void main(String[] args)
{
PrintStream out = System.out;
if (args.length < 2)
@@ -61,11 +60,7 @@
String columnfamily = args[args.length - 1];
Schema.instance.loadFromDisk(false);
- CFMetaData metadata = Schema.instance.getCFMetaData(keyspace, columnfamily);
- if (metadata == null)
- throw new IllegalArgumentException(String.format("Unknown keyspace/table %s.%s",
- keyspace,
- columnfamily));
+ TableMetadata metadata = Schema.instance.validateTable(keyspace, columnfamily);
Keyspace ks = Keyspace.openWithoutSSTables(keyspace);
ColumnFamilyStore cfs = ks.getColumnFamilyStore(columnfamily);
diff --git a/src/java/org/apache/cassandra/tools/SSTableExport.java b/src/java/org/apache/cassandra/tools/SSTableExport.java
index 40ddbe8..394f4b6 100644
--- a/src/java/org/apache/cassandra/tools/SSTableExport.java
+++ b/src/java/org/apache/cassandra/tools/SSTableExport.java
@@ -19,30 +19,36 @@
import java.io.File;
import java.io.IOException;
-import java.util.*;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
import java.util.concurrent.atomic.AtomicLong;
import java.util.stream.Collectors;
import java.util.stream.Stream;
-import java.util.stream.StreamSupport;
-import org.apache.commons.cli.*;
-
-import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.cql3.ColumnIdentifier;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.PartitionPosition;
-import org.apache.cassandra.db.SerializationHeader;
-import org.apache.cassandra.db.marshal.UTF8Type;
import org.apache.cassandra.db.rows.UnfilteredRowIterator;
-import org.apache.cassandra.dht.*;
+import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.dht.Bounds;
+import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.io.sstable.ISSTableScanner;
import org.apache.cassandra.io.sstable.KeyIterator;
import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.apache.commons.cli.PosixParser;
import org.apache.cassandra.io.sstable.metadata.MetadataComponent;
import org.apache.cassandra.io.sstable.metadata.MetadataType;
+import org.apache.cassandra.schema.TableMetadataRef;
import org.apache.cassandra.utils.FBUtilities;
/**
@@ -50,19 +56,24 @@
*/
public class SSTableExport
{
+ static
+ {
+ FBUtilities.preventIllegalAccessWarnings();
+ }
private static final String KEY_OPTION = "k";
private static final String DEBUG_OUTPUT_OPTION = "d";
private static final String EXCLUDE_KEY_OPTION = "x";
private static final String ENUMERATE_KEYS_OPTION = "e";
private static final String RAW_TIMESTAMPS = "t";
+ private static final String PARTITION_JSON_LINES = "l";
private static final Options options = new Options();
private static CommandLine cmd;
static
{
- DatabaseDescriptor.toolInitialization();
+ DatabaseDescriptor.clientInitialization();
Option optKey = new Option(KEY_OPTION, true, "Partition key");
// Number of times -k <key> can be passed on the command line.
@@ -82,48 +93,9 @@
Option rawTimestamps = new Option(RAW_TIMESTAMPS, false, "Print raw timestamps instead of iso8601 date strings");
options.addOption(rawTimestamps);
- }
- /**
- * Construct table schema from info stored in SSTable's Stats.db
- *
- * @param desc SSTable's descriptor
- * @return Restored CFMetaData
- * @throws IOException when Stats.db cannot be read
- */
- public static CFMetaData metadataFromSSTable(Descriptor desc) throws IOException
- {
- if (!desc.version.storeRows())
- throw new IOException("pre-3.0 SSTable is not supported.");
-
- EnumSet<MetadataType> types = EnumSet.of(MetadataType.STATS, MetadataType.HEADER);
- Map<MetadataType, MetadataComponent> sstableMetadata = desc.getMetadataSerializer().deserialize(desc, types);
- SerializationHeader.Component header = (SerializationHeader.Component) sstableMetadata.get(MetadataType.HEADER);
- IPartitioner partitioner = FBUtilities.newPartitioner(desc);
-
- CFMetaData.Builder builder = CFMetaData.Builder.create("keyspace", "table").withPartitioner(partitioner);
- header.getStaticColumns().entrySet().stream()
- .forEach(entry -> {
- ColumnIdentifier ident = ColumnIdentifier.getInterned(UTF8Type.instance.getString(entry.getKey()), true);
- builder.addStaticColumn(ident, entry.getValue());
- });
- header.getRegularColumns().entrySet().stream()
- .forEach(entry -> {
- ColumnIdentifier ident = ColumnIdentifier.getInterned(UTF8Type.instance.getString(entry.getKey()), true);
- builder.addRegularColumn(ident, entry.getValue());
- });
- builder.addPartitionKey("PartitionKey", header.getKeyType());
- for (int i = 0; i < header.getClusteringTypes().size(); i++)
- {
- builder.addClusteringColumn("clustering" + (i > 0 ? i : ""), header.getClusteringTypes().get(i));
- }
- return builder.build();
- }
-
- private static <T> Stream<T> iterToStream(Iterator<T> iter)
- {
- Spliterator<T> splititer = Spliterators.spliteratorUnknownSize(iter, Spliterator.IMMUTABLE);
- return StreamSupport.stream(splititer, false);
+ Option partitionJsonLines= new Option(PARTITION_JSON_LINES, false, "Output json lines, by partition");
+ options.addOption(partitionJsonLines);
}
/**
@@ -134,6 +106,7 @@
* @throws ConfigurationException
* on configuration failure (wrong params given)
*/
+ @SuppressWarnings("resource")
public static void main(String[] args) throws ConfigurationException
{
CommandLineParser parser = new PosixParser();
@@ -162,11 +135,6 @@
: cmd.getOptionValues(EXCLUDE_KEY_OPTION)));
String ssTableFileName = new File(cmd.getArgs()[0]).getAbsolutePath();
- if (Descriptor.isLegacyFile(new File(ssTableFileName)))
- {
- System.err.println("Unsupported legacy sstable");
- System.exit(1);
- }
if (!new File(ssTableFileName).exists())
{
System.err.println("Cannot find file " + ssTableFileName);
@@ -175,12 +143,12 @@
Descriptor desc = Descriptor.fromFilename(ssTableFileName);
try
{
- CFMetaData metadata = metadataFromSSTable(desc);
+ TableMetadata metadata = Util.metadataFromSSTable(desc);
if (cmd.hasOption(ENUMERATE_KEYS_OPTION))
{
try (KeyIterator iter = new KeyIterator(desc, metadata))
{
- JsonTransformer.keysToJson(null, iterToStream(iter),
+ JsonTransformer.keysToJson(null, Util.iterToStream(iter),
cmd.hasOption(RAW_TIMESTAMPS),
metadata,
System.out);
@@ -188,14 +156,14 @@
}
else
{
- SSTableReader sstable = SSTableReader.openNoValidation(desc, metadata);
+ SSTableReader sstable = SSTableReader.openNoValidation(desc, TableMetadataRef.forOfflineTools(metadata));
IPartitioner partitioner = sstable.getPartitioner();
final ISSTableScanner currentScanner;
if ((keys != null) && (keys.length > 0))
{
List<AbstractBounds<PartitionPosition>> bounds = Arrays.stream(keys)
.filter(key -> !excludes.contains(key))
- .map(metadata.getKeyValidator()::fromString)
+ .map(metadata.partitionKeyType::fromString)
.map(partitioner::decorateKey)
.sorted()
.map(DecoratedKey::getToken)
@@ -206,8 +174,8 @@
{
currentScanner = sstable.getScanner();
}
- Stream<UnfilteredRowIterator> partitions = iterToStream(currentScanner).filter(i ->
- excludes.isEmpty() || !excludes.contains(metadata.getKeyValidator().getString(i.partitionKey().getKey()))
+ Stream<UnfilteredRowIterator> partitions = Util.iterToStream(currentScanner).filter(i ->
+ excludes.isEmpty() || !excludes.contains(metadata.partitionKeyType.getString(i.partitionKey().getKey()))
);
if (cmd.hasOption(DEBUG_OUTPUT_OPTION))
{
@@ -218,23 +186,27 @@
if (!partition.partitionLevelDeletion().isLive())
{
- System.out.println("[" + metadata.getKeyValidator().getString(partition.partitionKey().getKey()) + "]@" +
+ System.out.println("[" + metadata.partitionKeyType.getString(partition.partitionKey().getKey()) + "]@" +
position.get() + " " + partition.partitionLevelDeletion());
}
if (!partition.staticRow().isEmpty())
{
- System.out.println("[" + metadata.getKeyValidator().getString(partition.partitionKey().getKey()) + "]@" +
+ System.out.println("[" + metadata.partitionKeyType.getString(partition.partitionKey().getKey()) + "]@" +
position.get() + " " + partition.staticRow().toString(metadata, true));
}
partition.forEachRemaining(row ->
{
System.out.println(
- "[" + metadata.getKeyValidator().getString(partition.partitionKey().getKey()) + "]@"
- + position.get() + " " + row.toString(metadata, false, true));
+ "[" + metadata.partitionKeyType.getString(partition.partitionKey().getKey()) + "]@"
+ + position.get() + " " + row.toString(metadata, false, true));
position.set(currentScanner.getCurrentPosition());
});
});
}
+ else if (cmd.hasOption(PARTITION_JSON_LINES))
+ {
+ JsonTransformer.toJsonLines(currentScanner, partitions, cmd.hasOption(RAW_TIMESTAMPS), metadata, System.out);
+ }
else
{
JsonTransformer.toJson(currentScanner, partitions, cmd.hasOption(RAW_TIMESTAMPS), metadata, System.out);
diff --git a/src/java/org/apache/cassandra/tools/SSTableLevelResetter.java b/src/java/org/apache/cassandra/tools/SSTableLevelResetter.java
index 915edf1..3a66ef9 100644
--- a/src/java/org/apache/cassandra/tools/SSTableLevelResetter.java
+++ b/src/java/org/apache/cassandra/tools/SSTableLevelResetter.java
@@ -21,7 +21,7 @@
import java.util.Map;
import java.util.Set;
-import org.apache.cassandra.config.Schema;
+import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Directories;
import org.apache.cassandra.db.Keyspace;
@@ -69,7 +69,7 @@
String keyspaceName = args[1];
String columnfamily = args[2];
// validate columnfamily
- if (Schema.instance.getCFMetaData(keyspaceName, columnfamily) == null)
+ if (Schema.instance.getTableMetadataRef(keyspaceName, columnfamily) == null)
{
System.err.println("ColumnFamily not found: " + keyspaceName + "/" + columnfamily);
System.exit(1);
diff --git a/src/java/org/apache/cassandra/tools/SSTableMetadataViewer.java b/src/java/org/apache/cassandra/tools/SSTableMetadataViewer.java
old mode 100644
new mode 100755
index b405fad..6366fd5
--- a/src/java/org/apache/cassandra/tools/SSTableMetadataViewer.java
+++ b/src/java/org/apache/cassandra/tools/SSTableMetadataViewer.java
@@ -17,25 +17,51 @@
*/
package org.apache.cassandra.tools;
-import java.io.*;
+import static org.apache.cassandra.tools.Util.BLUE;
+import static org.apache.cassandra.tools.Util.CYAN;
+import static org.apache.cassandra.tools.Util.RESET;
+import static org.apache.cassandra.tools.Util.WHITE;
+import static org.apache.commons.lang3.time.DurationFormatUtils.formatDurationWords;
+
+import java.io.DataInputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.io.PrintWriter;
import java.nio.ByteBuffer;
+import java.nio.file.Files;
import java.util.Arrays;
+import java.util.Comparator;
import java.util.EnumSet;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
+import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.SerializationHeader;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.db.marshal.UTF8Type;
import org.apache.cassandra.db.rows.EncodingStats;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.db.rows.Unfiltered;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.io.compress.CompressionMetadata;
import org.apache.cassandra.io.sstable.Component;
import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.ISSTableScanner;
import org.apache.cassandra.io.sstable.IndexSummary;
-import org.apache.cassandra.io.sstable.metadata.*;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.metadata.CompactionMetadata;
+import org.apache.cassandra.io.sstable.metadata.MetadataComponent;
+import org.apache.cassandra.io.sstable.metadata.MetadataType;
+import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
+import org.apache.cassandra.io.sstable.metadata.ValidationMetadata;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.schema.TableMetadataRef;
+import org.apache.cassandra.tools.Util.TermHistogram;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.Pair;
import org.apache.commons.cli.CommandLine;
@@ -46,179 +72,501 @@
import org.apache.commons.cli.ParseException;
import org.apache.commons.cli.PosixParser;
+import com.google.common.collect.MinMaxPriorityQueue;
+import org.apache.commons.lang3.time.DurationFormatUtils;
+
/**
* Shows the contents of sstable metadata
*/
public class SSTableMetadataViewer
{
- private static final String GCGS_KEY = "gc_grace_seconds";
+ private static final Options options = new Options();
+ private static CommandLine cmd;
+ private static final String COLORS = "c";
+ private static final String UNICODE = "u";
+ private static final String GCGS_KEY = "g";
+ private static final String TIMESTAMP_UNIT = "t";
+ private static final String SCAN = "s";
+ private static Comparator<ValuedByteBuffer> VCOMP = Comparator.comparingLong(ValuedByteBuffer::getValue).reversed();
- /**
- * @param args a list of sstables whose metadata we're interested in
- */
- public static void main(String[] args) throws IOException
+ static
{
- PrintStream out = System.out;
- Option optGcgs = new Option(null, GCGS_KEY, true, "The "+GCGS_KEY+" to use when calculating droppable tombstones");
+ DatabaseDescriptor.clientInitialization();
+ }
- Options options = new Options();
- options.addOption(optGcgs);
- CommandLine cmd = null;
- CommandLineParser parser = new PosixParser();
- try
- {
- cmd = parser.parse(options, args);
- }
- catch (ParseException e)
- {
- printHelp(options, out);
- }
+ boolean color;
+ boolean unicode;
+ int gc;
+ PrintStream out;
+ String[] files;
+ TimeUnit tsUnit;
- if (cmd.getArgs().length == 0)
- {
- printHelp(options, out);
- }
- int gcgs = Integer.parseInt(cmd.getOptionValue(GCGS_KEY, "0"));
- Util.initDatabaseDescriptor();
+ public SSTableMetadataViewer()
+ {
+ this(true, true, 0, TimeUnit.MICROSECONDS, System.out);
+ }
- for (String fname : cmd.getArgs())
+ public SSTableMetadataViewer(boolean color, boolean unicode, int gc, TimeUnit tsUnit, PrintStream out)
+ {
+ this.color = color;
+ this.tsUnit = tsUnit;
+ this.unicode = unicode;
+ this.out = out;
+ this.gc = gc;
+ }
+
+ public static String deletion(long time)
+ {
+ if (time == 0 || time == Integer.MAX_VALUE)
{
- if (new File(fname).exists())
+ return "no tombstones";
+ }
+ return toDateString(time, TimeUnit.SECONDS);
+ }
+
+ public static String toDateString(long time, TimeUnit unit)
+ {
+ if (time == 0)
+ {
+ return null;
+ }
+ return new java.text.SimpleDateFormat("MM/dd/yyyy HH:mm:ss").format(new java.util.Date(unit.toMillis(time)));
+ }
+
+ public static String toDurationString(long duration, TimeUnit unit)
+ {
+ if (duration == 0)
+ {
+ return null;
+ }
+ else if (duration == Integer.MAX_VALUE)
+ {
+ return "never";
+ }
+ return formatDurationWords(unit.toMillis(duration), true, true);
+ }
+
+ public static String toByteString(long bytes)
+ {
+ if (bytes == 0)
+ return null;
+ else if (bytes < 1024)
+ return bytes + " B";
+
+ int exp = (int) (Math.log(bytes) / Math.log(1024));
+ char pre = "kMGTP".charAt(exp - 1);
+ return String.format("%.1f %sB", bytes / Math.pow(1024, exp), pre);
+ }
+
+ public String scannedOverviewOutput(String key, long value)
+ {
+ StringBuilder sb = new StringBuilder();
+ if (color) sb.append(CYAN);
+ sb.append('[');
+ if (color) sb.append(RESET);
+ sb.append(key);
+ if (color) sb.append(CYAN);
+ sb.append("] ");
+ if (color) sb.append(RESET);
+ sb.append(value);
+ return sb.toString();
+ }
+
+ private void printScannedOverview(Descriptor descriptor, StatsMetadata stats) throws IOException
+ {
+ TableMetadata cfm = Util.metadataFromSSTable(descriptor);
+ SSTableReader reader = SSTableReader.openNoValidation(descriptor, TableMetadataRef.forOfflineTools(cfm));
+ try (ISSTableScanner scanner = reader.getScanner())
+ {
+ long bytes = scanner.getLengthInBytes();
+ MinMaxPriorityQueue<ValuedByteBuffer> widestPartitions = MinMaxPriorityQueue
+ .orderedBy(VCOMP)
+ .maximumSize(5)
+ .create();
+ MinMaxPriorityQueue<ValuedByteBuffer> largestPartitions = MinMaxPriorityQueue
+ .orderedBy(VCOMP)
+ .maximumSize(5)
+ .create();
+ MinMaxPriorityQueue<ValuedByteBuffer> mostTombstones = MinMaxPriorityQueue
+ .orderedBy(VCOMP)
+ .maximumSize(5)
+ .create();
+ long partitionCount = 0;
+ long rowCount = 0;
+ long tombstoneCount = 0;
+ long cellCount = 0;
+ double totalCells = stats.totalColumnsSet;
+ int lastPercent = 0;
+ long lastPercentTime = 0;
+ while (scanner.hasNext())
{
- Descriptor descriptor = Descriptor.fromFilename(fname);
- Map<MetadataType, MetadataComponent> metadata = descriptor.getMetadataSerializer().deserialize(descriptor, EnumSet.allOf(MetadataType.class));
- ValidationMetadata validation = (ValidationMetadata) metadata.get(MetadataType.VALIDATION);
- StatsMetadata stats = (StatsMetadata) metadata.get(MetadataType.STATS);
- CompactionMetadata compaction = (CompactionMetadata) metadata.get(MetadataType.COMPACTION);
- CompressionMetadata compression = null;
- File compressionFile = new File(descriptor.filenameFor(Component.COMPRESSION_INFO));
- if (compressionFile.exists())
- compression = CompressionMetadata.create(fname);
- SerializationHeader.Component header = (SerializationHeader.Component) metadata.get(MetadataType.HEADER);
-
- out.printf("SSTable: %s%n", descriptor);
- if (validation != null)
+ try (UnfilteredRowIterator partition = scanner.next())
{
- out.printf("Partitioner: %s%n", validation.partitioner);
- out.printf("Bloom Filter FP chance: %f%n", validation.bloomFilterFPChance);
- }
- if (stats != null)
- {
- out.printf("Minimum timestamp: %s%n", stats.minTimestamp);
- out.printf("Maximum timestamp: %s%n", stats.maxTimestamp);
- out.printf("SSTable min local deletion time: %s%n", stats.minLocalDeletionTime);
- out.printf("SSTable max local deletion time: %s%n", stats.maxLocalDeletionTime);
- out.printf("Compressor: %s%n", compression != null ? compression.compressor().getClass().getName() : "-");
- if (compression != null)
- out.printf("Compression ratio: %s%n", stats.compressionRatio);
- out.printf("TTL min: %s%n", stats.minTTL);
- out.printf("TTL max: %s%n", stats.maxTTL);
- if (validation != null && header != null)
- printMinMaxToken(descriptor, FBUtilities.newPartitioner(descriptor), header.getKeyType(), out);
-
- if (header != null && header.getClusteringTypes().size() == stats.minClusteringValues.size())
+ long psize = 0;
+ long pcount = 0;
+ int ptombcount = 0;
+ partitionCount++;
+ if (!partition.staticRow().isEmpty())
{
- List<AbstractType<?>> clusteringTypes = header.getClusteringTypes();
- List<ByteBuffer> minClusteringValues = stats.minClusteringValues;
- List<ByteBuffer> maxClusteringValues = stats.maxClusteringValues;
- String[] minValues = new String[clusteringTypes.size()];
- String[] maxValues = new String[clusteringTypes.size()];
- for (int i = 0; i < clusteringTypes.size(); i++)
+ rowCount++;
+ pcount++;
+ psize += partition.staticRow().dataSize();
+ }
+ if (!partition.partitionLevelDeletion().isLive())
+ {
+ tombstoneCount++;
+ ptombcount++;
+ }
+ while (partition.hasNext())
+ {
+ Unfiltered unfiltered = partition.next();
+ switch (unfiltered.kind())
{
- minValues[i] = clusteringTypes.get(i).getString(minClusteringValues.get(i));
- maxValues[i] = clusteringTypes.get(i).getString(maxClusteringValues.get(i));
+ case ROW:
+ rowCount++;
+ Row row = (Row) unfiltered;
+ psize += row.dataSize();
+ pcount++;
+ for (org.apache.cassandra.db.rows.Cell cell : row.cells())
+ {
+ cellCount++;
+ double percentComplete = Math.min(1.0, cellCount / totalCells);
+ if (lastPercent != (int) (percentComplete * 100) &&
+ (System.currentTimeMillis() - lastPercentTime) > 1000)
+ {
+ lastPercentTime = System.currentTimeMillis();
+ lastPercent = (int) (percentComplete * 100);
+ if (color)
+ out.printf("\r%sAnalyzing SSTable... %s%s %s(%%%s)", BLUE, CYAN,
+ Util.progress(percentComplete, 30, unicode),
+ RESET,
+ (int) (percentComplete * 100));
+ else
+ out.printf("\rAnalyzing SSTable... %s (%%%s)",
+ Util.progress(percentComplete, 30, unicode),
+ (int) (percentComplete * 100));
+ out.flush();
+ }
+ if (cell.isTombstone())
+ {
+ tombstoneCount++;
+ ptombcount++;
+ }
+ }
+ break;
+ case RANGE_TOMBSTONE_MARKER:
+ tombstoneCount++;
+ ptombcount++;
+ break;
}
- out.printf("minClustringValues: %s%n", Arrays.toString(minValues));
- out.printf("maxClustringValues: %s%n", Arrays.toString(maxValues));
}
- out.printf("Estimated droppable tombstones: %s%n", stats.getEstimatedDroppableTombstoneRatio((int) (System.currentTimeMillis() / 1000) - gcgs));
- out.printf("SSTable Level: %d%n", stats.sstableLevel);
- out.printf("Repaired at: %d%n", stats.repairedAt);
- out.printf("Replay positions covered: %s%n", stats.commitLogIntervals);
- out.printf("totalColumnsSet: %s%n", stats.totalColumnsSet);
- out.printf("totalRows: %s%n", stats.totalRows);
- out.println("Estimated tombstone drop times:");
- for (Map.Entry<Number, long[]> entry : stats.estimatedTombstoneDropTime.getAsMap().entrySet())
- {
- out.printf("%-10s:%10s%n",entry.getKey().intValue(), entry.getValue()[0]);
- }
- printHistograms(stats, out);
- }
- if (compaction != null)
- {
- out.printf("Estimated cardinality: %s%n", compaction.cardinalityEstimator.cardinality());
- }
- if (header != null)
- {
- EncodingStats encodingStats = header.getEncodingStats();
- AbstractType<?> keyType = header.getKeyType();
- List<AbstractType<?>> clusteringTypes = header.getClusteringTypes();
- Map<ByteBuffer, AbstractType<?>> staticColumns = header.getStaticColumns();
- Map<String, String> statics = staticColumns.entrySet().stream()
- .collect(Collectors.toMap(
- e -> UTF8Type.instance.getString(e.getKey()),
- e -> e.getValue().toString()));
- Map<ByteBuffer, AbstractType<?>> regularColumns = header.getRegularColumns();
- Map<String, String> regulars = regularColumns.entrySet().stream()
- .collect(Collectors.toMap(
- e -> UTF8Type.instance.getString(e.getKey()),
- e -> e.getValue().toString()));
-
- out.printf("EncodingStats minTTL: %s%n", encodingStats.minTTL);
- out.printf("EncodingStats minLocalDeletionTime: %s%n", encodingStats.minLocalDeletionTime);
- out.printf("EncodingStats minTimestamp: %s%n", encodingStats.minTimestamp);
- out.printf("KeyType: %s%n", keyType.toString());
- out.printf("ClusteringTypes: %s%n", clusteringTypes.toString());
- out.printf("StaticColumns: {%s}%n", FBUtilities.toString(statics));
- out.printf("RegularColumns: {%s}%n", FBUtilities.toString(regulars));
+ widestPartitions.add(new ValuedByteBuffer(partition.partitionKey().getKey(), pcount));
+ largestPartitions.add(new ValuedByteBuffer(partition.partitionKey().getKey(), psize));
+ mostTombstones.add(new ValuedByteBuffer(partition.partitionKey().getKey(), ptombcount));
}
}
- else
+
+ out.printf("\r%80s\r", " ");
+ field("Size", bytes);
+ field("Partitions", partitionCount);
+ field("Rows", rowCount);
+ field("Tombstones", tombstoneCount);
+ field("Cells", cellCount);
+ field("Widest Partitions", "");
+ Util.iterToStream(widestPartitions.iterator()).sorted(VCOMP).forEach(p ->
+ {
+ out.println(" " + scannedOverviewOutput(cfm.partitionKeyType.getString(p.buffer), p.value));
+ });
+ field("Largest Partitions", "");
+ Util.iterToStream(largestPartitions.iterator()).sorted(VCOMP).forEach(p ->
+ {
+ out.print(" ");
+ out.print(scannedOverviewOutput(cfm.partitionKeyType.getString(p.buffer), p.value));
+ if (color)
+ out.print(WHITE);
+ out.print(" (");
+ out.print(toByteString(p.value));
+ out.print(")");
+ if (color)
+ out.print(RESET);
+ out.println();
+ });
+ StringBuilder tleaders = new StringBuilder();
+ Util.iterToStream(mostTombstones.iterator()).sorted(VCOMP).forEach(p ->
+ {
+ if (p.value > 0)
+ {
+ tleaders.append(" ");
+ tleaders.append(scannedOverviewOutput(cfm.partitionKeyType.getString(p.buffer), p.value));
+ tleaders.append(System.lineSeparator());
+ }
+ });
+ String tombstoneLeaders = tleaders.toString();
+ if (tombstoneLeaders.length() > 10)
{
- out.println("No such file: " + fname);
+ field("Tombstone Leaders", "");
+ out.print(tombstoneLeaders);
}
}
- }
-
- private static void printHelp(Options options, PrintStream out)
- {
- out.println();
- new HelpFormatter().printHelp("Usage: sstablemetadata [--"+GCGS_KEY+" n] <sstable filenames>", "Dump contents of given SSTable to standard output in JSON format.", options, "");
- System.exit(1);
- }
-
- private static void printHistograms(StatsMetadata metadata, PrintStream out)
- {
- long[] offsets = metadata.estimatedPartitionSize.getBucketOffsets();
- long[] ersh = metadata.estimatedPartitionSize.getBuckets(false);
- long[] ecch = metadata.estimatedColumnCount.getBuckets(false);
-
- out.println(String.format("%-10s%18s%18s",
- "Count", "Row Size", "Cell Count"));
-
- for (int i = 0; i < offsets.length; i++)
+ finally
{
- out.println(String.format("%-10d%18s%18s",
- offsets[i],
- (i < ersh.length ? ersh[i] : ""),
- (i < ecch.length ? ecch[i] : "")));
+ reader.selfRef().ensureReleased();
}
}
- private static void printMinMaxToken(Descriptor descriptor, IPartitioner partitioner, AbstractType<?> keyType, PrintStream out) throws IOException
+ private void printSStableMetadata(String fname, boolean scan) throws IOException
+ {
+ Descriptor descriptor = Descriptor.fromFilename(fname);
+ Map<MetadataType, MetadataComponent> metadata = descriptor.getMetadataSerializer()
+ .deserialize(descriptor, EnumSet.allOf(MetadataType.class));
+ ValidationMetadata validation = (ValidationMetadata) metadata.get(MetadataType.VALIDATION);
+ StatsMetadata stats = (StatsMetadata) metadata.get(MetadataType.STATS);
+ CompactionMetadata compaction = (CompactionMetadata) metadata.get(MetadataType.COMPACTION);
+ CompressionMetadata compression = null;
+ File compressionFile = new File(descriptor.filenameFor(Component.COMPRESSION_INFO));
+ if (compressionFile.exists())
+ compression = CompressionMetadata.create(fname);
+ SerializationHeader.Component header = (SerializationHeader.Component) metadata
+ .get(MetadataType.HEADER);
+
+ field("SSTable", descriptor);
+ if (scan && descriptor.version.getVersion().compareTo("ma") >= 0)
+ {
+ printScannedOverview(descriptor, stats);
+ }
+ if (validation != null)
+ {
+ field("Partitioner", validation.partitioner);
+ field("Bloom Filter FP chance", validation.bloomFilterFPChance);
+ }
+ if (stats != null)
+ {
+ field("Minimum timestamp", stats.minTimestamp, toDateString(stats.minTimestamp, tsUnit));
+ field("Maximum timestamp", stats.maxTimestamp, toDateString(stats.maxTimestamp, tsUnit));
+ field("SSTable min local deletion time", stats.minLocalDeletionTime, deletion(stats.minLocalDeletionTime));
+ field("SSTable max local deletion time", stats.maxLocalDeletionTime, deletion(stats.maxLocalDeletionTime));
+ field("Compressor", compression != null ? compression.compressor().getClass().getName() : "-");
+ if (compression != null)
+ field("Compression ratio", stats.compressionRatio);
+ field("TTL min", stats.minTTL, toDurationString(stats.minTTL, TimeUnit.SECONDS));
+ field("TTL max", stats.maxTTL, toDurationString(stats.maxTTL, TimeUnit.SECONDS));
+
+ if (validation != null && header != null)
+ printMinMaxToken(descriptor, FBUtilities.newPartitioner(descriptor), header.getKeyType());
+
+ if (header != null && header.getClusteringTypes().size() == stats.minClusteringValues.size())
+ {
+ List<AbstractType<?>> clusteringTypes = header.getClusteringTypes();
+ List<ByteBuffer> minClusteringValues = stats.minClusteringValues;
+ List<ByteBuffer> maxClusteringValues = stats.maxClusteringValues;
+ String[] minValues = new String[clusteringTypes.size()];
+ String[] maxValues = new String[clusteringTypes.size()];
+ for (int i = 0; i < clusteringTypes.size(); i++)
+ {
+ minValues[i] = clusteringTypes.get(i).getString(minClusteringValues.get(i));
+ maxValues[i] = clusteringTypes.get(i).getString(maxClusteringValues.get(i));
+ }
+ field("minClusteringValues", Arrays.toString(minValues));
+ field("maxClusteringValues", Arrays.toString(maxValues));
+ }
+ field("Estimated droppable tombstones",
+ stats.getEstimatedDroppableTombstoneRatio((int) (System.currentTimeMillis() / 1000) - this.gc));
+ field("SSTable Level", stats.sstableLevel);
+ field("Repaired at", stats.repairedAt, toDateString(stats.repairedAt, TimeUnit.MILLISECONDS));
+ field("Pending repair", stats.pendingRepair);
+ field("Replay positions covered", stats.commitLogIntervals);
+ field("totalColumnsSet", stats.totalColumnsSet);
+ field("totalRows", stats.totalRows);
+ field("Estimated tombstone drop times", "");
+
+ TermHistogram estDropped = new TermHistogram(stats.estimatedTombstoneDropTime,
+ "Drop Time",
+ offset -> String.format("%d %s",
+ offset,
+ Util.wrapQuiet(toDateString(offset, TimeUnit.SECONDS),
+ color)),
+ String::valueOf);
+ estDropped.printHistogram(out, color, unicode);
+ field("Partition Size", "");
+ TermHistogram rowSize = new TermHistogram(stats.estimatedPartitionSize,
+ "Size (bytes)",
+ offset -> String.format("%d %s",
+ offset,
+ Util.wrapQuiet(toByteString(offset), color)),
+ String::valueOf);
+ rowSize.printHistogram(out, color, unicode);
+ field("Column Count", "");
+ TermHistogram cellCount = new TermHistogram(stats.estimatedCellPerPartitionCount,
+ "Columns",
+ String::valueOf,
+ String::valueOf);
+ cellCount.printHistogram(out, color, unicode);
+ }
+ if (compaction != null)
+ {
+ field("Estimated cardinality", compaction.cardinalityEstimator.cardinality());
+ }
+ if (header != null)
+ {
+ EncodingStats encodingStats = header.getEncodingStats();
+ AbstractType<?> keyType = header.getKeyType();
+ List<AbstractType<?>> clusteringTypes = header.getClusteringTypes();
+ Map<ByteBuffer, AbstractType<?>> staticColumns = header.getStaticColumns();
+ Map<String, String> statics = staticColumns.entrySet().stream()
+ .collect(Collectors.toMap(e -> UTF8Type.instance.getString(e.getKey()),
+ e -> e.getValue().toString()));
+ Map<ByteBuffer, AbstractType<?>> regularColumns = header.getRegularColumns();
+ Map<String, String> regulars = regularColumns.entrySet().stream()
+ .collect(Collectors.toMap(e -> UTF8Type.instance.getString(e.getKey()),
+ e -> e.getValue().toString()));
+
+ field("EncodingStats minTTL", encodingStats.minTTL,
+ toDurationString(encodingStats.minTTL, TimeUnit.SECONDS));
+ field("EncodingStats minLocalDeletionTime", encodingStats.minLocalDeletionTime,
+ toDateString(encodingStats.minLocalDeletionTime, TimeUnit.SECONDS));
+ field("EncodingStats minTimestamp", encodingStats.minTimestamp,
+ toDateString(encodingStats.minTimestamp, tsUnit));
+ field("KeyType", keyType.toString());
+ field("ClusteringTypes", clusteringTypes.toString());
+ field("StaticColumns", FBUtilities.toString(statics));
+ field("RegularColumns", FBUtilities.toString(regulars));
+ field("IsTransient", stats.isTransient);
+ }
+ }
+
+ private void field(String field, Object value)
+ {
+ field(field, value, null);
+ }
+
+ private void field(String field, Object value, String comment)
+ {
+ StringBuilder sb = new StringBuilder();
+ if (color) sb.append(BLUE);
+ sb.append(field);
+ if (color) sb.append(CYAN);
+ sb.append(": ");
+ if (color) sb.append(RESET);
+ sb.append(value == null? "--" : value.toString());
+
+ if (comment != null)
+ {
+ if (color) sb.append(WHITE);
+ sb.append(" (");
+ sb.append(comment);
+ sb.append(")");
+ if (color) sb.append(RESET);
+ }
+ this.out.println(sb.toString());
+ }
+
+ private static void printUsage()
+ {
+ try (PrintWriter errWriter = new PrintWriter(System.err, true))
+ {
+ HelpFormatter formatter = new HelpFormatter();
+ formatter.printHelp(errWriter, 120, "sstablemetadata <options> <sstable...>",
+ String.format("%nDump information about SSTable[s] for Apache Cassandra 3.x%nOptions:"),
+ options, 2, 1, "", true);
+ errWriter.println();
+ }
+ }
+
+ private void printMinMaxToken(Descriptor descriptor, IPartitioner partitioner, AbstractType<?> keyType)
+ throws IOException
{
File summariesFile = new File(descriptor.filenameFor(Component.SUMMARY));
if (!summariesFile.exists())
return;
- try (DataInputStream iStream = new DataInputStream(new FileInputStream(summariesFile)))
+ try (DataInputStream iStream = new DataInputStream(Files.newInputStream(summariesFile.toPath())))
{
- Pair<DecoratedKey, DecoratedKey> firstLast = new IndexSummary.IndexSummarySerializer().deserializeFirstLastKey(iStream, partitioner, descriptor.version.hasSamplingLevel());
- out.printf("First token: %s (key=%s)%n", firstLast.left.getToken(), keyType.getString(firstLast.left.getKey()));
- out.printf("Last token: %s (key=%s)%n", firstLast.right.getToken(), keyType.getString(firstLast.right.getKey()));
+ Pair<DecoratedKey, DecoratedKey> firstLast = new IndexSummary.IndexSummarySerializer()
+ .deserializeFirstLastKey(iStream, partitioner);
+ field("First token", firstLast.left.getToken(), keyType.getString(firstLast.left.getKey()));
+ field("Last token", firstLast.right.getToken(), keyType.getString(firstLast.right.getKey()));
}
}
+ /**
+ * @param args
+ * a list of sstables whose metadata we're interested in
+ */
+ public static void main(String[] args) throws IOException
+ {
+ CommandLineParser parser = new PosixParser();
+
+ Option disableColors = new Option(COLORS, "colors", false, "Use ANSI color sequences");
+ disableColors.setOptionalArg(true);
+ options.addOption(disableColors);
+ Option unicode = new Option(UNICODE, "unicode", false, "Use unicode to draw histograms and progress bars");
+ unicode.setOptionalArg(true);
+
+ options.addOption(unicode);
+ Option gcgs = new Option(GCGS_KEY, "gc_grace_seconds", true, "Time to use when calculating droppable tombstones");
+ gcgs.setOptionalArg(true);
+ options.addOption(gcgs);
+ Option tsUnit = new Option(TIMESTAMP_UNIT, "timestamp_unit", true, "Time unit that cell timestamps are written with");
+ tsUnit.setOptionalArg(true);
+ options.addOption(tsUnit);
+
+ Option scanEnabled = new Option(SCAN, "scan", false,
+ "Full sstable scan for additional details. Only available in 3.0+ sstables. Defaults: false");
+ scanEnabled.setOptionalArg(true);
+ options.addOption(scanEnabled);
+ try
+ {
+ cmd = parser.parse(options, args);
+ }
+ catch (ParseException e1)
+ {
+ System.err.println(e1.getMessage());
+ printUsage();
+ System.exit(1);
+ }
+
+ if (cmd.getArgs().length < 1)
+ {
+ System.err.println("You must supply at least one sstable");
+ printUsage();
+ System.exit(1);
+ }
+ boolean enabledColors = cmd.hasOption(COLORS);
+ boolean enabledUnicode = cmd.hasOption(UNICODE);
+ boolean fullScan = cmd.hasOption(SCAN);
+ int gc = Integer.parseInt(cmd.getOptionValue(GCGS_KEY, "0"));
+ TimeUnit ts = TimeUnit.valueOf(cmd.getOptionValue(TIMESTAMP_UNIT, "MICROSECONDS"));
+ SSTableMetadataViewer metawriter = new SSTableMetadataViewer(enabledColors, enabledUnicode, gc, ts, System.out);
+ for (String fname : cmd.getArgs())
+ {
+ File sstable = new File(fname);
+ if (sstable.exists())
+ {
+ metawriter.printSStableMetadata(sstable.getAbsolutePath(), fullScan);
+ }
+ else
+ {
+ System.out.println("No such file: " + fname);
+ }
+ }
+ }
+
+ private static class ValuedByteBuffer
+ {
+ public long value;
+ public ByteBuffer buffer;
+
+ public ValuedByteBuffer(ByteBuffer buffer, long value)
+ {
+ this.value = value;
+ this.buffer = buffer;
+ }
+
+ public long getValue()
+ {
+ return value;
+ }
+ }
}
diff --git a/src/java/org/apache/cassandra/tools/SSTableOfflineRelevel.java b/src/java/org/apache/cassandra/tools/SSTableOfflineRelevel.java
index 9f0395b..b88bf0a 100644
--- a/src/java/org/apache/cassandra/tools/SSTableOfflineRelevel.java
+++ b/src/java/org/apache/cassandra/tools/SSTableOfflineRelevel.java
@@ -34,7 +34,7 @@
import com.google.common.collect.Multimap;
import com.google.common.collect.SetMultimap;
-import org.apache.cassandra.config.Schema;
+import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.Directories;
@@ -92,8 +92,8 @@
String columnfamily = args[args.length - 1];
Schema.instance.loadFromDisk(false);
- if (Schema.instance.getCFMetaData(keyspace, columnfamily) == null)
- throw new IllegalArgumentException(String.format("Unknown keyspace/columnFamily %s.%s",
+ if (Schema.instance.getTableMetadataRef(keyspace, columnfamily) == null)
+ throw new IllegalArgumentException(String.format("Unknown keyspace/table %s.%s",
keyspace,
columnfamily));
diff --git a/src/java/org/apache/cassandra/tools/SSTableRepairedAtSetter.java b/src/java/org/apache/cassandra/tools/SSTableRepairedAtSetter.java
index 413ec4d..31d80fa 100644
--- a/src/java/org/apache/cassandra/tools/SSTableRepairedAtSetter.java
+++ b/src/java/org/apache/cassandra/tools/SSTableRepairedAtSetter.java
@@ -25,11 +25,8 @@
import java.util.Arrays;
import java.util.List;
-import org.apache.cassandra.config.Config;
-import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.io.sstable.Component;
import org.apache.cassandra.io.sstable.Descriptor;
-import org.apache.cassandra.service.ActiveRepairService;
/**
* Set repairedAt status on a given set of sstables.
@@ -82,21 +79,20 @@
for (String fname: fileNames)
{
Descriptor descriptor = Descriptor.fromFilename(fname);
- if (descriptor.version.hasRepairedAt())
+ if (!descriptor.version.isCompatible())
{
- if (setIsRepaired)
- {
- FileTime f = Files.getLastModifiedTime(new File(descriptor.filenameFor(Component.DATA)).toPath());
- descriptor.getMetadataSerializer().mutateRepairedAt(descriptor, f.toMillis());
- }
- else
- {
- descriptor.getMetadataSerializer().mutateRepairedAt(descriptor, ActiveRepairService.UNREPAIRED_SSTABLE);
- }
+ System.err.println("SSTable " + fname + " is in a old and unsupported format");
+ continue;
+ }
+
+ if (setIsRepaired)
+ {
+ FileTime f = Files.getLastModifiedTime(new File(descriptor.filenameFor(Component.DATA)).toPath());
+ descriptor.getMetadataSerializer().mutateRepairMetadata(descriptor, f.toMillis(), null, false);
}
else
{
- System.err.println("SSTable " + fname + " does not have repaired property, run upgradesstables");
+ descriptor.getMetadataSerializer().mutateRepairMetadata(descriptor, 0, null, false);
}
}
}
diff --git a/src/java/org/apache/cassandra/tools/StandaloneSSTableUtil.java b/src/java/org/apache/cassandra/tools/StandaloneSSTableUtil.java
index 2e8ee0b..9a7847a 100644
--- a/src/java/org/apache/cassandra/tools/StandaloneSSTableUtil.java
+++ b/src/java/org/apache/cassandra/tools/StandaloneSSTableUtil.java
@@ -18,9 +18,8 @@
*/
package org.apache.cassandra.tools;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.config.Schema;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Directories;
import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
@@ -30,6 +29,7 @@
import java.io.File;
import java.io.IOException;
import java.util.function.BiFunction;
+import java.util.function.BiPredicate;
import static org.apache.cassandra.tools.BulkLoader.CmdLineOptions;
@@ -52,7 +52,7 @@
Util.initDatabaseDescriptor();
Schema.instance.loadFromDisk(false);
- CFMetaData metadata = Schema.instance.getCFMetaData(options.keyspaceName, options.cfName);
+ TableMetadata metadata = Schema.instance.getTableMetadata(options.keyspaceName, options.cfName);
if (metadata == null)
throw new IllegalArgumentException(String.format("Unknown keyspace/table %s.%s",
options.keyspaceName,
@@ -82,9 +82,9 @@
}
}
- private static void listFiles(Options options, CFMetaData metadata, OutputHandler handler) throws IOException
+ private static void listFiles(Options options, TableMetadata metadata, OutputHandler handler) throws IOException
{
- Directories directories = new Directories(metadata, ColumnFamilyStore.getInitialDirectories());
+ Directories directories = new Directories(metadata);
for (File dir : directories.getCFDirectories())
{
@@ -93,7 +93,7 @@
}
}
- private static BiFunction<File, Directories.FileType, Boolean> getFilter(Options options)
+ private static BiPredicate<File, Directories.FileType> getFilter(Options options)
{
return (file, type) ->
{
diff --git a/src/java/org/apache/cassandra/tools/StandaloneScrubber.java b/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
index 2643438..d9d8db1 100644
--- a/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
+++ b/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
@@ -29,7 +29,7 @@
import com.google.common.collect.Lists;
import org.apache.commons.cli.*;
-import org.apache.cassandra.config.Schema;
+import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Directories;
import org.apache.cassandra.db.Keyspace;
@@ -71,7 +71,7 @@
// load keyspace descriptions.
Schema.instance.loadFromDisk(false);
- if (Schema.instance.getKSMetaData(options.keyspaceName) == null)
+ if (Schema.instance.getKeyspaceMetadata(options.keyspaceName) == null)
throw new IllegalArgumentException(String.format("Unknown keyspace %s", options.keyspaceName));
// Do not load sstables since they might be broken
@@ -121,7 +121,7 @@
SSTableHeaderFix.Builder headerFixBuilder = SSTableHeaderFix.builder()
.logToList(logOutput)
- .schemaCallback(() -> Schema.instance::getCFMetaData);
+ .schemaCallback(() -> Schema.instance::getTableMetadata);
if (options.headerFixMode == Options.HeaderFixMode.VALIDATE)
headerFixBuilder = headerFixBuilder.dryRun();
diff --git a/src/java/org/apache/cassandra/tools/StandaloneSplitter.java b/src/java/org/apache/cassandra/tools/StandaloneSplitter.java
index 1e57ff4..7a60b6f 100644
--- a/src/java/org/apache/cassandra/tools/StandaloneSplitter.java
+++ b/src/java/org/apache/cassandra/tools/StandaloneSplitter.java
@@ -22,7 +22,7 @@
import java.util.*;
import java.util.concurrent.TimeUnit;
-import org.apache.cassandra.config.Schema;
+import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.commons.cli.*;
@@ -35,7 +35,6 @@
import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
import org.apache.cassandra.io.sstable.*;
import org.apache.cassandra.utils.JVMStabilityInspector;
-import org.apache.cassandra.utils.Pair;
import static org.apache.cassandra.tools.BulkLoader.CmdLineOptions;
@@ -70,12 +69,11 @@
continue;
}
- Pair<Descriptor, Component> pair = SSTable.tryComponentFromFilename(file.getParentFile(), file.getName());
- if (pair == null) {
+ Descriptor desc = SSTable.tryDescriptorFromFilename(file);
+ if (desc == null) {
System.out.println("Skipping non sstable file " + file);
continue;
}
- Descriptor desc = pair.left;
if (ksName == null)
ksName = desc.ksname;
diff --git a/src/java/org/apache/cassandra/tools/StandaloneUpgrader.java b/src/java/org/apache/cassandra/tools/StandaloneUpgrader.java
index e55b3a8..ed25e42 100644
--- a/src/java/org/apache/cassandra/tools/StandaloneUpgrader.java
+++ b/src/java/org/apache/cassandra/tools/StandaloneUpgrader.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file * distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
@@ -22,7 +22,6 @@
import org.apache.commons.cli.*;
-import org.apache.cassandra.config.Schema;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Directories;
import org.apache.cassandra.db.Keyspace;
@@ -33,6 +32,7 @@
import org.apache.cassandra.io.sstable.*;
import org.apache.cassandra.io.sstable.format.SSTableFormat;
import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.utils.JVMStabilityInspector;
import org.apache.cassandra.utils.OutputHandler;
@@ -55,7 +55,7 @@
// load keyspace descriptions.
Schema.instance.loadFromDisk(false);
- if (Schema.instance.getCFMetaData(options.keyspace, options.cf) == null)
+ if (Schema.instance.getTableMetadataRef(options.keyspace, options.cf) == null)
throw new IllegalArgumentException(String.format("Unknown keyspace/table %s.%s",
options.keyspace,
options.cf));
diff --git a/src/java/org/apache/cassandra/tools/StandaloneVerifier.java b/src/java/org/apache/cassandra/tools/StandaloneVerifier.java
index ee55dd5..9074418 100644
--- a/src/java/org/apache/cassandra/tools/StandaloneVerifier.java
+++ b/src/java/org/apache/cassandra/tools/StandaloneVerifier.java
@@ -18,11 +18,14 @@
*/
package org.apache.cassandra.tools;
-import org.apache.cassandra.config.Schema;
+import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Directories;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.db.compaction.*;
+import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
import org.apache.cassandra.io.sstable.Component;
import org.apache.cassandra.io.sstable.CorruptSSTableException;
import org.apache.cassandra.io.sstable.Descriptor;
@@ -33,6 +36,8 @@
import java.util.*;
import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
import static org.apache.cassandra.tools.BulkLoader.CmdLineOptions;
@@ -43,11 +48,16 @@
private static final String EXTENDED_OPTION = "extended";
private static final String DEBUG_OPTION = "debug";
private static final String HELP_OPTION = "help";
+ private static final String CHECK_VERSION = "check_version";
+ private static final String MUTATE_REPAIR_STATUS = "mutate_repair_status";
+ private static final String QUICK = "quick";
+ private static final String TOKEN_RANGE = "token_range";
public static void main(String args[])
{
Options options = Options.parseArgs(args);
Util.initDatabaseDescriptor();
+ System.out.println("sstableverify using the following options: " + options);
try
{
@@ -56,7 +66,7 @@
boolean hasFailed = false;
- if (Schema.instance.getCFMetaData(options.keyspaceName, options.cfName) == null)
+ if (Schema.instance.getTableMetadataRef(options.keyspaceName, options.cfName) == null)
throw new IllegalArgumentException(String.format("Unknown keyspace/table %s.%s",
options.keyspaceName,
options.cfName));
@@ -68,8 +78,6 @@
OutputHandler handler = new OutputHandler.SystemOutput(options.verbose, options.debug);
Directories.SSTableLister lister = cfs.getDirectories().sstableLister(Directories.OnTxnErr.THROW).skipTemporary(true);
- boolean extended = options.extended;
-
List<SSTableReader> sstables = new ArrayList<>();
// Verify sstables
@@ -92,15 +100,22 @@
e.printStackTrace(System.err);
}
}
-
+ Verifier.Options verifyOptions = Verifier.options().invokeDiskFailurePolicy(false)
+ .extendedVerification(options.extended)
+ .checkVersion(options.checkVersion)
+ .mutateRepairStatus(options.mutateRepairStatus)
+ .checkOwnsTokens(!options.tokens.isEmpty())
+ .tokenLookup(ignore -> options.tokens)
+ .build();
+ handler.output("Running verifier with the following options: " + verifyOptions);
for (SSTableReader sstable : sstables)
{
try
{
- try (Verifier verifier = new Verifier(cfs, sstable, handler, true))
+ try (Verifier verifier = new Verifier(cfs, sstable, handler, true, verifyOptions))
{
- verifier.verify(extended);
+ verifier.verify();
}
catch (CorruptSSTableException cs)
{
@@ -136,6 +151,10 @@
public boolean debug;
public boolean verbose;
public boolean extended;
+ public boolean checkVersion;
+ public boolean mutateRepairStatus;
+ public boolean quick;
+ public Collection<Range<Token>> tokens;
private Options(String keyspaceName, String cfName)
{
@@ -174,6 +193,20 @@
opts.debug = cmd.hasOption(DEBUG_OPTION);
opts.verbose = cmd.hasOption(VERBOSE_OPTION);
opts.extended = cmd.hasOption(EXTENDED_OPTION);
+ opts.checkVersion = cmd.hasOption(CHECK_VERSION);
+ opts.mutateRepairStatus = cmd.hasOption(MUTATE_REPAIR_STATUS);
+ opts.quick = cmd.hasOption(QUICK);
+
+ if (cmd.hasOption(TOKEN_RANGE))
+ {
+ opts.tokens = Stream.of(cmd.getOptionValues(TOKEN_RANGE))
+ .map(StandaloneVerifier::parseTokenRange)
+ .collect(Collectors.toSet());
+ }
+ else
+ {
+ opts.tokens = Collections.emptyList();
+ }
return opts;
}
@@ -184,6 +217,21 @@
}
}
+ public String toString()
+ {
+ return "Options{" +
+ "keyspaceName='" + keyspaceName + '\'' +
+ ", cfName='" + cfName + '\'' +
+ ", debug=" + debug +
+ ", verbose=" + verbose +
+ ", extended=" + extended +
+ ", checkVersion=" + checkVersion +
+ ", mutateRepairStatus=" + mutateRepairStatus +
+ ", quick=" + quick +
+ ", tokens=" + tokens +
+ '}';
+ }
+
private static void errorMsg(String msg, CmdLineOptions options)
{
System.err.println(msg);
@@ -198,6 +246,10 @@
options.addOption("e", EXTENDED_OPTION, "extended verification");
options.addOption("v", VERBOSE_OPTION, "verbose output");
options.addOption("h", HELP_OPTION, "display this help message");
+ options.addOption("c", CHECK_VERSION, "make sure sstables are the latest version");
+ options.addOption("r", MUTATE_REPAIR_STATUS, "don't mutate repair status");
+ options.addOption("q", QUICK, "do a quick check, don't read all data");
+ options.addOptionList("t", TOKEN_RANGE, "range", "long token range of the format left,right. This may be provided multiple times to define multiple different ranges");
return options;
}
@@ -212,4 +264,14 @@
new HelpFormatter().printHelp(usage, header.toString(), options, "");
}
}
+
+ private static Range<Token> parseTokenRange(String line)
+ {
+ String[] split = line.split(",");
+ if (split.length != 2)
+ throw new IllegalArgumentException("Unable to parse token range from " + line + "; format is left,right but saw " + split.length + " parts");
+ long left = Long.parseLong(split[0]);
+ long right = Long.parseLong(split[1]);
+ return new Range<>(new Murmur3Partitioner.LongToken(left), new Murmur3Partitioner.LongToken(right));
+ }
}
diff --git a/src/java/org/apache/cassandra/tools/Util.java b/src/java/org/apache/cassandra/tools/Util.java
index 76011a9..db664aa 100644
--- a/src/java/org/apache/cassandra/tools/Util.java
+++ b/src/java/org/apache/cassandra/tools/Util.java
@@ -18,11 +18,250 @@
package org.apache.cassandra.tools;
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.exceptions.ConfigurationException;
+import static java.lang.String.format;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.util.EnumSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Spliterator;
+import java.util.Spliterators;
+import java.util.TreeMap;
+import java.util.Map.Entry;
+import java.util.function.LongFunction;
+import java.util.stream.Stream;
+import java.util.stream.StreamSupport;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.ColumnIdentifier;
+import org.apache.cassandra.db.SerializationHeader;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.dht.IPartitioner;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.metadata.MetadataComponent;
+import org.apache.cassandra.io.sstable.metadata.MetadataType;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.utils.EstimatedHistogram;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.streamhist.TombstoneHistogram;
+
+import com.google.common.base.Strings;
+import com.google.common.collect.Lists;
+
+@SuppressWarnings("serial")
public final class Util
{
+ static final String RESET = "\u001B[0m";
+ static final String BLUE = "\u001B[34m";
+ static final String CYAN = "\u001B[36m";
+ static final String WHITE = "\u001B[37m";
+ private static final List<String> ANSI_COLORS = Lists.newArrayList(RESET, BLUE, CYAN, WHITE);
+
+ private static final String FULL_BAR_UNICODE = Strings.repeat("\u2593", 30);
+ private static final String EMPTY_BAR_UNICODE = Strings.repeat("\u2591", 30);
+ private static final String FULL_BAR_ASCII = Strings.repeat("#", 30);
+ private static final String EMPTY_BAR_ASCII = Strings.repeat("-", 30);
+
+ private static final TreeMap<Double, String> BARS_UNICODE = new TreeMap<Double, String>()
+ {{
+ this.put(1.0, "\u2589"); // full, actually using 7/8th for bad font impls of fullblock
+ this.put(7.0 / 8.0, "\u2589"); // 7/8ths left block
+ this.put(3.0 / 4.0, "\u258A"); // 3/4th block
+ this.put(5.0 / 8.0, "\u258B"); // 5/8th
+ this.put(3.0 / 8.0, "\u258D"); // three eighths, skips 1/2 due to font inconsistencies
+ this.put(1.0 / 4.0, "\u258E"); // 1/4th
+ this.put(1.0 / 8.0, "\u258F"); // 1/8th
+ }};
+
+ private static final TreeMap<Double, String> BARS_ASCII = new TreeMap<Double, String>()
+ {{
+ this.put(1.00, "O");
+ this.put(0.75, "o");
+ this.put(0.30, ".");
+ }};
+
+ private static TreeMap<Double, String> barmap(boolean unicode)
+ {
+ return unicode ? BARS_UNICODE : BARS_ASCII;
+ }
+
+ public static String progress(double percentComplete, int width, boolean unicode)
+ {
+ assert percentComplete >= 0 && percentComplete <= 1;
+ int cols = (int) (percentComplete * width);
+ return (unicode ? FULL_BAR_UNICODE : FULL_BAR_ASCII).substring(width - cols) +
+ (unicode ? EMPTY_BAR_UNICODE : EMPTY_BAR_ASCII ).substring(cols);
+ }
+
+ public static String stripANSI(String string)
+ {
+ return ANSI_COLORS.stream().reduce(string, (a, b) -> a.replace(b, ""));
+ }
+
+ public static int countANSI(String string)
+ {
+ return string.length() - stripANSI(string).length();
+ }
+
+ public static String wrapQuiet(String toWrap, boolean color)
+ {
+ if (Strings.isNullOrEmpty(toWrap))
+ {
+ return "";
+ }
+ StringBuilder sb = new StringBuilder();
+ if (color) sb.append(WHITE);
+ sb.append("(");
+ sb.append(toWrap);
+ sb.append(")");
+ if (color) sb.append(RESET);
+ return sb.toString();
+ }
+
+ public static class TermHistogram
+ {
+ public long max;
+ public long min;
+ public double sum;
+ int maxCountLength = 5;
+ int maxOffsetLength = 5;
+ Map<? extends Number, Long> histogram;
+ LongFunction<String> offsetName;
+ LongFunction<String> countName;
+ String title;
+
+ public TermHistogram(Map<? extends Number, Long> histogram,
+ String title,
+ LongFunction<String> offsetName,
+ LongFunction<String> countName)
+ {
+ this.offsetName = offsetName;
+ this.countName = countName;
+ this.histogram = histogram;
+ this.title = title;
+ maxOffsetLength = title.length();
+ histogram.entrySet().stream().forEach(e ->
+ {
+ max = Math.max(max, e.getValue());
+ min = Math.min(min, e.getValue());
+ sum += e.getValue();
+ // find max width, but remove ansi sequences first
+ maxCountLength = Math.max(maxCountLength, stripANSI(countName.apply(e.getValue())).length());
+ maxOffsetLength = Math.max(maxOffsetLength, stripANSI(offsetName.apply(e.getKey().longValue())).length());
+ });
+ }
+
+ public TermHistogram(TombstoneHistogram histogram,
+ String title,
+ LongFunction<String> offsetName,
+ LongFunction<String> countName)
+ {
+ this(new TreeMap<Number, Long>()
+ {
+ {
+ histogram.forEach((point, value) -> {
+ this.put(point, (long) value);
+ });
+ }
+ }, title, offsetName, countName);
+ }
+
+ public TermHistogram(EstimatedHistogram histogram,
+ String title,
+ LongFunction<String> offsetName,
+ LongFunction<String> countName)
+ {
+ this(new TreeMap<Number, Long>()
+ {
+ {
+ long[] counts = histogram.getBuckets(false);
+ long[] offsets = histogram.getBucketOffsets();
+ for (int i = 0; i < counts.length; i++)
+ {
+ long e = counts[i];
+ if (e > 0)
+ {
+ put(offsets[i], e);
+ }
+ }
+ }
+ }, title, offsetName, countName);
+ }
+
+ public String bar(long count, int length, String color, boolean unicode)
+ {
+ if (color == null) color = "";
+ StringBuilder sb = new StringBuilder(color);
+ long barVal = count;
+ int intWidth = (int) (barVal * 1.0 / max * length);
+ double remainderWidth = (barVal * 1.0 / max * length) - intWidth;
+ sb.append(Strings.repeat(barmap(unicode).get(1.0), intWidth));
+
+ if (barmap(unicode).floorKey(remainderWidth) != null)
+ sb.append(barmap(unicode).get(barmap(unicode).floorKey(remainderWidth)));
+
+ if(!Strings.isNullOrEmpty(color))
+ sb.append(RESET);
+
+ return sb.toString();
+ }
+
+ public void printHistogram(PrintStream out, boolean color, boolean unicode)
+ {
+ // String.format includes ansi sequences in the count, so need to modify the lengths
+ int offsetTitleLength = color ? maxOffsetLength + BLUE.length() : maxOffsetLength;
+ out.printf(" %-" + offsetTitleLength + "s %s %-" + maxCountLength + "s %s %sHistogram%s %n",
+ color ? BLUE + title : title,
+ color ? CYAN + "|" + BLUE : "|",
+ "Count",
+ wrapQuiet("%", color),
+ color ? BLUE : "",
+ color ? RESET : "");
+ histogram.entrySet().stream().forEach(e ->
+ {
+ String offset = offsetName.apply(e.getKey().longValue());
+ long count = e.getValue();
+ String histo = bar(count, 30, color? WHITE : null, unicode);
+ int mol = color ? maxOffsetLength + countANSI(offset) : maxOffsetLength;
+ int mcl = color ? maxCountLength + countANSI(countName.apply(count)) : maxCountLength;
+ out.printf(" %-" + mol + "s %s %" + mcl + "s %s %s%n",
+ offset,
+ color ? CYAN + "|" + RESET : "|",
+ countName.apply(count),
+ wrapQuiet(String.format("%3s", (int) (100 * ((double) count / sum))), color),
+ histo);
+ });
+ EstimatedHistogram eh = new EstimatedHistogram(165);
+ for (Entry<? extends Number, Long> e : histogram.entrySet())
+ {
+ eh.add(e.getKey().longValue(), e.getValue());
+ }
+ String[] percentiles = new String[]{"50th", "75th", "95th", "98th", "99th", "Min", "Max"};
+ long[] data = new long[]
+ {
+ eh.percentile(.5),
+ eh.percentile(.75),
+ eh.percentile(.95),
+ eh.percentile(.98),
+ eh.percentile(.99),
+ eh.min(),
+ eh.max(),
+ };
+ out.println((color ? BLUE : "") + " Percentiles" + (color ? RESET : ""));
+
+ for (int i = 0; i < percentiles.length; i++)
+ {
+ out.println(format(" %s%-10s%s%s",
+ (color ? BLUE : ""),
+ percentiles[i],
+ (color ? RESET : ""),
+ offsetName.apply(data[i])));
+ }
+ }
+ }
private Util()
{
}
@@ -54,4 +293,47 @@
}
}
}
+
+ public static <T> Stream<T> iterToStream(Iterator<T> iter)
+ {
+ Spliterator<T> splititer = Spliterators.spliteratorUnknownSize(iter, Spliterator.IMMUTABLE);
+ return StreamSupport.stream(splititer, false);
+ }
+
+ /**
+ * Construct table schema from info stored in SSTable's Stats.db
+ *
+ * @param desc SSTable's descriptor
+ * @return Restored CFMetaData
+ * @throws IOException when Stats.db cannot be read
+ */
+ public static TableMetadata metadataFromSSTable(Descriptor desc) throws IOException
+ {
+ if (desc.version.getVersion().compareTo("ma") < 0)
+ throw new IOException("pre-3.0 SSTable is not supported.");
+
+ EnumSet<MetadataType> types = EnumSet.of(MetadataType.STATS, MetadataType.HEADER);
+ Map<MetadataType, MetadataComponent> sstableMetadata = desc.getMetadataSerializer().deserialize(desc, types);
+ SerializationHeader.Component header = (SerializationHeader.Component) sstableMetadata.get(MetadataType.HEADER);
+
+ IPartitioner partitioner = FBUtilities.newPartitioner(desc);
+
+ TableMetadata.Builder builder = TableMetadata.builder("keyspace", "table").partitioner(partitioner);
+ header.getStaticColumns().entrySet().stream()
+ .forEach(entry -> {
+ ColumnIdentifier ident = ColumnIdentifier.getInterned(UTF8Type.instance.getString(entry.getKey()), true);
+ builder.addStaticColumn(ident, entry.getValue());
+ });
+ header.getRegularColumns().entrySet().stream()
+ .forEach(entry -> {
+ ColumnIdentifier ident = ColumnIdentifier.getInterned(UTF8Type.instance.getString(entry.getKey()), true);
+ builder.addRegularColumn(ident, entry.getValue());
+ });
+ builder.addPartitionKeyColumn("PartitionKey", header.getKeyType());
+ for (int i = 0; i < header.getClusteringTypes().size(); i++)
+ {
+ builder.addClusteringColumn("clustering" + (i > 0 ? i : ""), header.getClusteringTypes().get(i));
+ }
+ return builder.build();
+ }
}
diff --git a/src/java/org/apache/cassandra/tools/nodetool/Assassinate.java b/src/java/org/apache/cassandra/tools/nodetool/Assassinate.java
index 56fec44..a075ded 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/Assassinate.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/Assassinate.java
@@ -18,8 +18,8 @@
package org.apache.cassandra.tools.nodetool;
import static org.apache.commons.lang3.StringUtils.EMPTY;
-import io.airlift.command.Arguments;
-import io.airlift.command.Command;
+import io.airlift.airline.Arguments;
+import io.airlift.airline.Command;
import java.net.UnknownHostException;
diff --git a/src/java/org/apache/cassandra/tools/nodetool/BootstrapResume.java b/src/java/org/apache/cassandra/tools/nodetool/BootstrapResume.java
index bb47e10..7be9173 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/BootstrapResume.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/BootstrapResume.java
@@ -17,7 +17,7 @@
*/
package org.apache.cassandra.tools.nodetool;
-import io.airlift.command.Command;
+import io.airlift.airline.Command;
import java.io.IOError;
import java.io.IOException;
diff --git a/src/java/org/apache/cassandra/tools/nodetool/CfHistograms.java b/src/java/org/apache/cassandra/tools/nodetool/CfHistograms.java
index 69d3b4a..8fdf803 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/CfHistograms.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/CfHistograms.java
@@ -17,7 +17,7 @@
*/
package org.apache.cassandra.tools.nodetool;
-import io.airlift.command.Command;
+import io.airlift.airline.Command;
/**
* @deprecated use TableHistograms
diff --git a/src/java/org/apache/cassandra/tools/nodetool/CfStats.java b/src/java/org/apache/cassandra/tools/nodetool/CfStats.java
index 15c72ba..2d27ea0 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/CfStats.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/CfStats.java
@@ -17,7 +17,7 @@
*/
package org.apache.cassandra.tools.nodetool;
-import io.airlift.command.Command;
+import io.airlift.airline.Command;
/**
* @deprecated use TableStats
diff --git a/src/java/org/apache/cassandra/tools/nodetool/Cleanup.java b/src/java/org/apache/cassandra/tools/nodetool/Cleanup.java
index 3e6fa23..200d255 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/Cleanup.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/Cleanup.java
@@ -17,14 +17,14 @@
*/
package org.apache.cassandra.tools.nodetool;
-import io.airlift.command.Arguments;
-import io.airlift.command.Command;
+import io.airlift.airline.Arguments;
+import io.airlift.airline.Command;
import java.util.ArrayList;
import java.util.List;
-import io.airlift.command.Option;
-import org.apache.cassandra.config.SchemaConstants;
+import io.airlift.airline.Option;
+import org.apache.cassandra.schema.SchemaConstants;
import org.apache.cassandra.tools.NodeProbe;
import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
diff --git a/src/java/org/apache/cassandra/tools/nodetool/ClearSnapshot.java b/src/java/org/apache/cassandra/tools/nodetool/ClearSnapshot.java
index 7167bd9..12322d0 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/ClearSnapshot.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/ClearSnapshot.java
@@ -20,9 +20,9 @@
import static com.google.common.collect.Iterables.toArray;
import static org.apache.commons.lang3.StringUtils.EMPTY;
import static org.apache.commons.lang3.StringUtils.join;
-import io.airlift.command.Arguments;
-import io.airlift.command.Command;
-import io.airlift.command.Option;
+import io.airlift.airline.Arguments;
+import io.airlift.airline.Command;
+import io.airlift.airline.Option;
import java.io.IOException;
import java.util.ArrayList;
@@ -40,9 +40,18 @@
@Option(title = "snapshot_name", name = "-t", description = "Remove the snapshot with a given name")
private String snapshotName = EMPTY;
+ @Option(title = "clear_all_snapshots", name = "--all", description = "Removes all snapshots")
+ private boolean clearAllSnapshots = false;
+
@Override
public void execute(NodeProbe probe)
{
+ if(snapshotName.isEmpty() && !clearAllSnapshots)
+ throw new RuntimeException("Specify snapshot name or --all");
+
+ if(!snapshotName.isEmpty() && clearAllSnapshots)
+ throw new RuntimeException("Specify only one of snapshot name or --all");
+
StringBuilder sb = new StringBuilder();
sb.append("Requested clearing snapshot(s) for ");
@@ -52,7 +61,9 @@
else
sb.append("[").append(join(keyspaces, ", ")).append("]");
- if (!snapshotName.isEmpty())
+ if (snapshotName.isEmpty())
+ sb.append(" with [all snapshots]");
+ else
sb.append(" with snapshot name [").append(snapshotName).append("]");
System.out.println(sb.toString());
diff --git a/src/java/org/apache/cassandra/tools/nodetool/ClientStats.java b/src/java/org/apache/cassandra/tools/nodetool/ClientStats.java
new file mode 100644
index 0000000..3bf46b4
--- /dev/null
+++ b/src/java/org/apache/cassandra/tools/nodetool/ClientStats.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.tools.nodetool;
+
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import io.airlift.airline.Command;
+import io.airlift.airline.Option;
+import org.apache.cassandra.tools.NodeProbe;
+import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
+import org.apache.cassandra.tools.nodetool.formatter.TableBuilder;
+import org.apache.cassandra.transport.ClientStat;
+import org.apache.cassandra.transport.ConnectedClient;
+
+@Command(name = "clientstats", description = "Print information about connected clients")
+public class ClientStats extends NodeToolCmd
+{
+ @Option(title = "list_connections", name = "--all", description = "Lists all connections")
+ private boolean listConnections = false;
+
+ @Option(title = "by_protocol", name = "--by-protocol", description = "Lists most recent client connections by protocol version")
+ private boolean connectionsByProtocolVersion = false;
+
+ @Option(title = "clear_history", name = "--clear-history", description = "Clear the history of connected clients")
+ private boolean clearConnectionHistory = false;
+
+ @Override
+ public void execute(NodeProbe probe)
+ {
+ if (clearConnectionHistory)
+ {
+ System.out.println("Clearing connection history");
+ probe.clearConnectionHistory();
+ return;
+ }
+
+ if (connectionsByProtocolVersion)
+ {
+ SimpleDateFormat sdf = new SimpleDateFormat("MMM dd, yyyy HH:mm:ss");
+
+ System.out.println("Clients by protocol version");
+ System.out.println();
+
+ List<Map<String, String>> clients = (List<Map<String, String>>) probe.getClientMetric("clientsByProtocolVersion");
+
+ if (!clients.isEmpty())
+ {
+ TableBuilder table = new TableBuilder();
+ table.add("Protocol-Version", "IP-Address", "Last-Seen");
+
+ for (Map<String, String> client : clients)
+ {
+ table.add(client.get(ClientStat.PROTOCOL_VERSION),
+ client.get(ClientStat.INET_ADDRESS),
+ sdf.format(new Date(Long.valueOf(client.get(ClientStat.LAST_SEEN_TIME)))));
+ }
+
+ table.printTo(System.out);
+ System.out.println();
+ }
+
+ return;
+ }
+
+ if (listConnections)
+ {
+ List<Map<String, String>> clients = (List<Map<String, String>>) probe.getClientMetric("connections");
+ if (!clients.isEmpty())
+ {
+ TableBuilder table = new TableBuilder();
+ table.add("Address", "SSL", "Cipher", "Protocol", "Version", "User", "Keyspace", "Requests", "Driver-Name", "Driver-Version");
+ for (Map<String, String> conn : clients)
+ {
+ table.add(conn.get(ConnectedClient.ADDRESS),
+ conn.get(ConnectedClient.SSL),
+ conn.get(ConnectedClient.CIPHER),
+ conn.get(ConnectedClient.PROTOCOL),
+ conn.get(ConnectedClient.VERSION),
+ conn.get(ConnectedClient.USER),
+ conn.get(ConnectedClient.KEYSPACE),
+ conn.get(ConnectedClient.REQUESTS),
+ conn.get(ConnectedClient.DRIVER_NAME),
+ conn.get(ConnectedClient.DRIVER_VERSION));
+ }
+ table.printTo(System.out);
+ System.out.println();
+ }
+ }
+
+ Map<String, Integer> connectionsByUser = (Map<String, Integer>) probe.getClientMetric("connectedNativeClientsByUser");
+ int total = connectionsByUser.values().stream().reduce(0, Integer::sum);
+ System.out.println("Total connected clients: " + total);
+ System.out.println();
+ TableBuilder table = new TableBuilder();
+ table.add("User", "Connections");
+ for (Entry<String, Integer> entry : connectionsByUser.entrySet())
+ {
+ table.add(entry.getKey(), entry.getValue().toString());
+ }
+ table.printTo(System.out);
+ }
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/tools/nodetool/Compact.java b/src/java/org/apache/cassandra/tools/nodetool/Compact.java
index ef10a83..7278ead 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/Compact.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/Compact.java
@@ -19,9 +19,9 @@
import static org.apache.commons.lang3.StringUtils.EMPTY;
-import io.airlift.command.Arguments;
-import io.airlift.command.Command;
-import io.airlift.command.Option;
+import io.airlift.airline.Arguments;
+import io.airlift.airline.Command;
+import io.airlift.airline.Option;
import java.util.ArrayList;
import java.util.List;
diff --git a/src/java/org/apache/cassandra/tools/nodetool/CompactionHistory.java b/src/java/org/apache/cassandra/tools/nodetool/CompactionHistory.java
index 8d24845..8896db0 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/CompactionHistory.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/CompactionHistory.java
@@ -17,8 +17,8 @@
*/
package org.apache.cassandra.tools.nodetool;
-import io.airlift.command.Command;
-import io.airlift.command.Option;
+import io.airlift.airline.Command;
+import io.airlift.airline.Option;
import org.apache.cassandra.tools.NodeProbe;
import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
import org.apache.cassandra.tools.nodetool.stats.CompactionHistoryHolder;
diff --git a/src/java/org/apache/cassandra/tools/nodetool/CompactionStats.java b/src/java/org/apache/cassandra/tools/nodetool/CompactionStats.java
index d615516..497fe24 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/CompactionStats.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/CompactionStats.java
@@ -22,8 +22,8 @@
import java.util.Map;
import java.util.Map.Entry;
-import io.airlift.command.Command;
-import io.airlift.command.Option;
+import io.airlift.airline.Command;
+import io.airlift.airline.Option;
import org.apache.cassandra.db.compaction.CompactionInfo;
import org.apache.cassandra.db.compaction.CompactionManagerMBean;
@@ -82,17 +82,17 @@
table.add("id", "compaction type", "keyspace", "table", "completed", "total", "unit", "progress");
for (Map<String, String> c : compactions)
{
- long total = Long.parseLong(c.get("total"));
- long completed = Long.parseLong(c.get("completed"));
- String taskType = c.get("taskType");
- String keyspace = c.get("keyspace");
- String columnFamily = c.get("columnfamily");
- String unit = c.get("unit");
+ long total = Long.parseLong(c.get(CompactionInfo.TOTAL));
+ long completed = Long.parseLong(c.get(CompactionInfo.COMPLETED));
+ String taskType = c.get(CompactionInfo.TASK_TYPE);
+ String keyspace = c.get(CompactionInfo.KEYSPACE);
+ String columnFamily = c.get(CompactionInfo.COLUMNFAMILY);
+ String unit = c.get(CompactionInfo.UNIT);
boolean toFileSize = humanReadable && Unit.isFileSize(unit);
String completedStr = toFileSize ? FileUtils.stringifyFileSize(completed) : Long.toString(completed);
String totalStr = toFileSize ? FileUtils.stringifyFileSize(total) : Long.toString(total);
String percentComplete = total == 0 ? "n/a" : new DecimalFormat("0.00").format((double) completed / total * 100) + "%";
- String id = c.get("compactionId");
+ String id = c.get(CompactionInfo.COMPACTION_ID);
table.add(id, taskType, keyspace, columnFamily, completedStr, totalStr, unit, percentComplete);
if (taskType.equals(OperationType.COMPACTION.toString()))
remainingBytes += total - completed;
diff --git a/src/java/org/apache/cassandra/tools/nodetool/Decommission.java b/src/java/org/apache/cassandra/tools/nodetool/Decommission.java
index 34890e0..0e58687 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/Decommission.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/Decommission.java
@@ -17,7 +17,8 @@
*/
package org.apache.cassandra.tools.nodetool;
-import io.airlift.command.Command;
+import io.airlift.airline.Command;
+import io.airlift.airline.Option;
import org.apache.cassandra.tools.NodeProbe;
import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
@@ -25,12 +26,18 @@
@Command(name = "decommission", description = "Decommission the *node I am connecting to*")
public class Decommission extends NodeToolCmd
{
+
+ @Option(title = "force",
+ name = {"-f", "--force"},
+ description = "Force decommission of this node even when it reduces the number of replicas to below configured RF")
+ private boolean force = false;
+
@Override
public void execute(NodeProbe probe)
{
try
{
- probe.decommission();
+ probe.decommission(force);
} catch (InterruptedException e)
{
throw new RuntimeException("Error decommissioning node", e);
diff --git a/src/java/org/apache/cassandra/tools/nodetool/DescribeCluster.java b/src/java/org/apache/cassandra/tools/nodetool/DescribeCluster.java
index 4178998..6a9f023 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/DescribeCluster.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/DescribeCluster.java
@@ -17,19 +17,29 @@
*/
package org.apache.cassandra.tools.nodetool;
-import static java.lang.String.format;
-import io.airlift.command.Command;
-
+import java.util.Collection;
import java.util.List;
import java.util.Map;
+import java.util.SortedMap;
+import com.google.common.collect.ArrayListMultimap;
+
+import io.airlift.airline.Command;
import org.apache.cassandra.locator.DynamicEndpointSnitch;
+import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.tools.NodeProbe;
+import org.apache.cassandra.tools.NodeTool;
import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
+import static java.lang.String.format;
+
@Command(name = "describecluster", description = "Print the name, snitch, partitioner and schema version of a cluster")
public class DescribeCluster extends NodeToolCmd
{
+ private boolean resolveIp = false;
+ private String keyspace = null;
+ private Collection<String> joiningNodes, leavingNodes, movingNodes, liveNodes, unreachableNodes;
+
@Override
public void execute(NodeProbe probe)
{
@@ -49,10 +59,88 @@
// display schema version for each node
System.out.println("\tSchema versions:");
- Map<String, List<String>> schemaVersions = probe.getSpProxy().getSchemaVersions();
+ Map<String, List<String>> schemaVersions = printPort ? probe.getSpProxy().getSchemaVersionsWithPort() : probe.getSpProxy().getSchemaVersions();
for (String version : schemaVersions.keySet())
{
System.out.println(format("\t\t%s: %s%n", version, schemaVersions.get(version)));
}
+
+ // Collect status information of all nodes
+ boolean withPort = true;
+ joiningNodes = probe.getJoiningNodes(withPort);
+ leavingNodes = probe.getLeavingNodes(withPort);
+ movingNodes = probe.getMovingNodes(withPort);
+ liveNodes = probe.getLiveNodes(withPort);
+ unreachableNodes = probe.getUnreachableNodes(withPort);
+
+ // Get the list of all keyspaces
+ List<String> keyspaces = probe.getKeyspaces();
+
+ System.out.println("Stats for all nodes:");
+ System.out.println("\tLive: " + liveNodes.size());
+ System.out.println("\tJoining: " + joiningNodes.size());
+ System.out.println("\tMoving: " + movingNodes.size());
+ System.out.println("\tLeaving: " + leavingNodes.size());
+ System.out.println("\tUnreachable: " + unreachableNodes.size());
+
+ Map<String, String> tokensToEndpoints = probe.getTokenToEndpointMap(withPort);
+ Map<String, Float> ownerships = null;
+ try
+ {
+ ownerships = probe.effectiveOwnershipWithPort(keyspace);
+ }
+ catch (IllegalStateException ex)
+ {
+ ownerships = probe.getOwnershipWithPort();
+ System.out.println("Error: " + ex.getMessage());
+ }
+ catch (IllegalArgumentException ex)
+ {
+ System.out.println("%nError: " + ex.getMessage());
+ System.exit(1);
+ }
+
+ SortedMap<String, SetHostStatWithPort> dcs = NodeTool.getOwnershipByDcWithPort(probe, resolveIp, tokensToEndpoints, ownerships);
+
+ System.out.println("\nData Centers: ");
+ for (Map.Entry<String, SetHostStatWithPort> dc : dcs.entrySet())
+ {
+ System.out.print("\t" + dc.getKey());
+
+ ArrayListMultimap<InetAddressAndPort, HostStatWithPort> hostToTokens = ArrayListMultimap.create();
+ for (HostStatWithPort stat : dc.getValue())
+ hostToTokens.put(stat.endpointWithPort, stat);
+
+ int totalNodes = 0; // total number of nodes in a datacenter
+ int downNodes = 0; // number of down nodes in a datacenter
+
+ for (InetAddressAndPort endpoint : hostToTokens.keySet())
+ {
+ totalNodes++;
+ if (unreachableNodes.contains(endpoint.toString()))
+ downNodes++;
+ }
+ System.out.print(" #Nodes: " + totalNodes);
+ System.out.println(" #Down: " + downNodes);
+ }
+
+ // display database version for each node
+ System.out.println("\nDatabase versions:");
+ Map<String, List<String>> databaseVersions = probe.getGossProxy().getReleaseVersionsWithPort();
+ for (String version : databaseVersions.keySet())
+ {
+ System.out.println(format("\t%s: %s%n", version, databaseVersions.get(version)));
+ }
+
+ System.out.println("Keyspaces:");
+ for (String keyspaceName : keyspaces)
+ {
+ String replicationInfo = probe.getKeyspaceReplicationInfo(keyspaceName);
+ if (replicationInfo == null)
+ {
+ System.out.println("something went wrong for keyspace: " + keyspaceName);
+ }
+ System.out.println("\t" + keyspaceName + " -> Replication class: " + replicationInfo);
+ }
}
-}
\ No newline at end of file
+}
diff --git a/src/java/org/apache/cassandra/tools/nodetool/DescribeRing.java b/src/java/org/apache/cassandra/tools/nodetool/DescribeRing.java
index a120ffe..ef8c97e 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/DescribeRing.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/DescribeRing.java
@@ -18,8 +18,8 @@
package org.apache.cassandra.tools.nodetool;
import static org.apache.commons.lang3.StringUtils.EMPTY;
-import io.airlift.command.Arguments;
-import io.airlift.command.Command;
+import io.airlift.airline.Arguments;
+import io.airlift.airline.Command;
import java.io.IOException;
@@ -39,7 +39,7 @@
System.out.println("TokenRange: ");
try
{
- for (String tokenRangeString : probe.describeRing(keyspace))
+ for (String tokenRangeString : probe.describeRing(keyspace, printPort))
{
System.out.println("\t" + tokenRangeString);
}
@@ -48,4 +48,4 @@
throw new RuntimeException(e);
}
}
-}
\ No newline at end of file
+}
diff --git a/src/java/org/apache/cassandra/tools/nodetool/DisableAuditLog.java b/src/java/org/apache/cassandra/tools/nodetool/DisableAuditLog.java
new file mode 100644
index 0000000..35653ae
--- /dev/null
+++ b/src/java/org/apache/cassandra/tools/nodetool/DisableAuditLog.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.tools.nodetool;
+
+import io.airlift.airline.Command;
+import org.apache.cassandra.tools.NodeProbe;
+import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
+
+@Command(name = "disableauditlog", description = "Disable the audit log")
+public class DisableAuditLog extends NodeToolCmd
+{
+ @Override
+ public void execute(NodeProbe probe)
+ {
+ probe.disableAuditLog();
+ }
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/tools/nodetool/DisableAutoCompaction.java b/src/java/org/apache/cassandra/tools/nodetool/DisableAutoCompaction.java
index 4d35ded..b9fc7d6 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/DisableAutoCompaction.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/DisableAutoCompaction.java
@@ -17,8 +17,8 @@
*/
package org.apache.cassandra.tools.nodetool;
-import io.airlift.command.Arguments;
-import io.airlift.command.Command;
+import io.airlift.airline.Arguments;
+import io.airlift.airline.Command;
import java.io.IOException;
import java.util.ArrayList;
diff --git a/src/java/org/apache/cassandra/tools/nodetool/DisableBackup.java b/src/java/org/apache/cassandra/tools/nodetool/DisableBackup.java
index 74e7f50..4b0bfbe 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/DisableBackup.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/DisableBackup.java
@@ -17,7 +17,7 @@
*/
package org.apache.cassandra.tools.nodetool;
-import io.airlift.command.Command;
+import io.airlift.airline.Command;
import org.apache.cassandra.tools.NodeProbe;
import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
diff --git a/src/java/org/apache/cassandra/tools/nodetool/DisableBinary.java b/src/java/org/apache/cassandra/tools/nodetool/DisableBinary.java
index dee319b..463f2b0 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/DisableBinary.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/DisableBinary.java
@@ -17,7 +17,7 @@
*/
package org.apache.cassandra.tools.nodetool;
-import io.airlift.command.Command;
+import io.airlift.airline.Command;
import org.apache.cassandra.tools.NodeProbe;
import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
diff --git a/src/java/org/apache/cassandra/tools/nodetool/DisableFullQueryLog.java b/src/java/org/apache/cassandra/tools/nodetool/DisableFullQueryLog.java
new file mode 100644
index 0000000..8820e5f
--- /dev/null
+++ b/src/java/org/apache/cassandra/tools/nodetool/DisableFullQueryLog.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.tools.nodetool;
+
+import io.airlift.airline.Command;
+import org.apache.cassandra.tools.NodeProbe;
+import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
+
+@Command(name = "disablefullquerylog", description = "Disable the full query log")
+public class DisableFullQueryLog extends NodeToolCmd
+{
+ @Override
+ public void execute(NodeProbe probe)
+ {
+ probe.stopFullQueryLogger();
+ }
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/tools/nodetool/DisableGossip.java b/src/java/org/apache/cassandra/tools/nodetool/DisableGossip.java
index 32448c9..6f950bb 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/DisableGossip.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/DisableGossip.java
@@ -17,7 +17,7 @@
*/
package org.apache.cassandra.tools.nodetool;
-import io.airlift.command.Command;
+import io.airlift.airline.Command;
import org.apache.cassandra.tools.NodeProbe;
import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
diff --git a/src/java/org/apache/cassandra/tools/nodetool/DisableHandoff.java b/src/java/org/apache/cassandra/tools/nodetool/DisableHandoff.java
index 11cd754..d7ec35f 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/DisableHandoff.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/DisableHandoff.java
@@ -17,7 +17,7 @@
*/
package org.apache.cassandra.tools.nodetool;
-import io.airlift.command.Command;
+import io.airlift.airline.Command;
import org.apache.cassandra.tools.NodeProbe;
import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
diff --git a/src/java/org/apache/cassandra/tools/nodetool/DisableHintsForDC.java b/src/java/org/apache/cassandra/tools/nodetool/DisableHintsForDC.java
index 7072318..d65c70b 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/DisableHintsForDC.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/DisableHintsForDC.java
@@ -20,8 +20,8 @@
import java.util.ArrayList;
import java.util.List;
-import io.airlift.command.Arguments;
-import io.airlift.command.Command;
+import io.airlift.airline.Arguments;
+import io.airlift.airline.Command;
import org.apache.cassandra.tools.NodeProbe;
import org.apache.cassandra.tools.NodeTool;
diff --git a/src/java/org/apache/cassandra/tools/nodetool/DisableOldProtocolVersions.java b/src/java/org/apache/cassandra/tools/nodetool/DisableOldProtocolVersions.java
new file mode 100644
index 0000000..2083062
--- /dev/null
+++ b/src/java/org/apache/cassandra/tools/nodetool/DisableOldProtocolVersions.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.tools.nodetool;
+
+import io.airlift.airline.Command;
+import org.apache.cassandra.tools.NodeProbe;
+import org.apache.cassandra.tools.NodeTool;
+
+@Command(name = "disableoldprotocolversions", description = "Disable old protocol versions")
+public class DisableOldProtocolVersions extends NodeTool.NodeToolCmd
+{
+ @Override
+ public void execute(NodeProbe probe)
+ {
+ probe.disableOldProtocolVersions();
+ }
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/tools/nodetool/DisableThrift.java b/src/java/org/apache/cassandra/tools/nodetool/DisableThrift.java
deleted file mode 100644
index 148b195..0000000
--- a/src/java/org/apache/cassandra/tools/nodetool/DisableThrift.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.tools.nodetool;
-
-import io.airlift.command.Command;
-
-import org.apache.cassandra.tools.NodeProbe;
-import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
-
-@Command(name = "disablethrift", description = "Disable thrift server")
-public class DisableThrift extends NodeToolCmd
-{
- @Override
- public void execute(NodeProbe probe)
- {
- probe.stopThriftServer();
- }
-}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/tools/nodetool/Drain.java b/src/java/org/apache/cassandra/tools/nodetool/Drain.java
index 5562e6d..eaa537a 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/Drain.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/Drain.java
@@ -17,7 +17,7 @@
*/
package org.apache.cassandra.tools.nodetool;
-import io.airlift.command.Command;
+import io.airlift.airline.Command;
import java.io.IOException;
import java.util.concurrent.ExecutionException;
diff --git a/src/java/org/apache/cassandra/tools/nodetool/EnableAuditLog.java b/src/java/org/apache/cassandra/tools/nodetool/EnableAuditLog.java
new file mode 100644
index 0000000..c71d210
--- /dev/null
+++ b/src/java/org/apache/cassandra/tools/nodetool/EnableAuditLog.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.tools.nodetool;
+
+import io.airlift.airline.Command;
+import io.airlift.airline.Option;
+import org.apache.cassandra.tools.NodeProbe;
+import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
+
+@Command(name = "enableauditlog", description = "Enable the audit log")
+public class EnableAuditLog extends NodeToolCmd
+{
+ @Option(title = "logger", name = { "--logger" }, description = "Logger name to be used for AuditLogging. Default BinAuditLogger. If not set the value from cassandra.yaml will be used")
+ private String logger = null;
+
+ @Option(title = "included_keyspaces", name = { "--included-keyspaces" }, description = "Comma separated list of keyspaces to be included for audit log. If not set the value from cassandra.yaml will be used")
+ private String included_keyspaces = null;
+
+ @Option(title = "excluded_keyspaces", name = { "--excluded-keyspaces" }, description = "Comma separated list of keyspaces to be excluded for audit log. If not set the value from cassandra.yaml will be used")
+ private String excluded_keyspaces = null;
+
+ @Option(title = "included_categories", name = { "--included-categories" }, description = "Comma separated list of Audit Log Categories to be included for audit log. If not set the value from cassandra.yaml will be used")
+ private String included_categories = null;
+
+ @Option(title = "excluded_categories", name = { "--excluded-categories" }, description = "Comma separated list of Audit Log Categories to be excluded for audit log. If not set the value from cassandra.yaml will be used")
+ private String excluded_categories = null;
+
+ @Option(title = "included_users", name = { "--included-users" }, description = "Comma separated list of users to be included for audit log. If not set the value from cassandra.yaml will be used")
+ private String included_users = null;
+
+ @Option(title = "excluded_users", name = { "--excluded-users" }, description = "Comma separated list of users to be excluded for audit log. If not set the value from cassandra.yaml will be used")
+ private String excluded_users = null;
+
+ @Override
+ public void execute(NodeProbe probe)
+ {
+ probe.enableAuditLog(logger, included_keyspaces, excluded_keyspaces, included_categories, excluded_categories, included_users, excluded_users);
+ }
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/tools/nodetool/EnableAutoCompaction.java b/src/java/org/apache/cassandra/tools/nodetool/EnableAutoCompaction.java
index c758df8..795ab13 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/EnableAutoCompaction.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/EnableAutoCompaction.java
@@ -17,8 +17,8 @@
*/
package org.apache.cassandra.tools.nodetool;
-import io.airlift.command.Arguments;
-import io.airlift.command.Command;
+import io.airlift.airline.Arguments;
+import io.airlift.airline.Command;
import java.io.IOException;
import java.util.ArrayList;
diff --git a/src/java/org/apache/cassandra/tools/nodetool/EnableBackup.java b/src/java/org/apache/cassandra/tools/nodetool/EnableBackup.java
index 4847fa5..d1773d9 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/EnableBackup.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/EnableBackup.java
@@ -17,7 +17,7 @@
*/
package org.apache.cassandra.tools.nodetool;
-import io.airlift.command.Command;
+import io.airlift.airline.Command;
import org.apache.cassandra.tools.NodeProbe;
import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
diff --git a/src/java/org/apache/cassandra/tools/nodetool/EnableBinary.java b/src/java/org/apache/cassandra/tools/nodetool/EnableBinary.java
index f1d5d9c..506945f 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/EnableBinary.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/EnableBinary.java
@@ -17,7 +17,7 @@
*/
package org.apache.cassandra.tools.nodetool;
-import io.airlift.command.Command;
+import io.airlift.airline.Command;
import org.apache.cassandra.tools.NodeProbe;
import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
diff --git a/src/java/org/apache/cassandra/tools/nodetool/EnableFullQueryLog.java b/src/java/org/apache/cassandra/tools/nodetool/EnableFullQueryLog.java
new file mode 100644
index 0000000..9873e5a
--- /dev/null
+++ b/src/java/org/apache/cassandra/tools/nodetool/EnableFullQueryLog.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.tools.nodetool;
+
+import io.airlift.airline.Command;
+import io.airlift.airline.Option;
+import org.apache.cassandra.tools.NodeProbe;
+import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
+
+@Command(name = "enablefullquerylog", description = "Enable full query logging, defaults for the options are configured in cassandra.yaml")
+public class EnableFullQueryLog extends NodeToolCmd
+{
+ @Option(title = "roll_cycle", name = {"--roll-cycle"}, description = "How often to roll the log file (MINUTELY, HOURLY, DAILY).")
+ private String rollCycle = null;
+
+ @Option(title = "blocking", name = {"--blocking"}, description = "If the queue is full whether to block producers or drop samples [true|false].")
+ private String blocking = null;
+
+ @Option(title = "max_queue_weight", name = {"--max-queue-weight"}, description = "Maximum number of bytes of query data to queue to disk before blocking or dropping samples.")
+ private int maxQueueWeight = Integer.MIN_VALUE;
+
+ @Option(title = "max_log_size", name = {"--max-log-size"}, description = "How many bytes of log data to store before dropping segments. Might not be respected if a log file hasn't rolled so it can be deleted.")
+ private long maxLogSize = Long.MIN_VALUE;
+
+ @Option(title = "path", name = {"--path"}, description = "Path to store the full query log at. Will have it's contents recursively deleted.")
+ private String path = null;
+
+ @Option(title = "archive_command", name = {"--archive-command"}, description = "Command that will handle archiving rolled full query log files." +
+ " Format is \"/path/to/script.sh %path\" where %path will be replaced with the file to archive")
+ private String archiveCommand = null;
+
+ @Option(title = "archive_retries", name = {"--max-archive-retries"}, description = "Max number of archive retries.")
+ private int archiveRetries = Integer.MIN_VALUE;
+
+ @Override
+ public void execute(NodeProbe probe)
+ {
+ Boolean bblocking = null;
+ if (blocking != null)
+ {
+ if (!blocking.equalsIgnoreCase("TRUE") && !blocking.equalsIgnoreCase("FALSE"))
+ throw new IllegalArgumentException("Invalid [" + blocking + "]. Blocking only accepts 'true' or 'false'.");
+ else
+ bblocking = Boolean.parseBoolean(blocking);
+ }
+ probe.enableFullQueryLogger(path, rollCycle, bblocking, maxQueueWeight, maxLogSize, archiveCommand, archiveRetries);
+ }
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/tools/nodetool/EnableGossip.java b/src/java/org/apache/cassandra/tools/nodetool/EnableGossip.java
index 16a9f4b..900c427 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/EnableGossip.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/EnableGossip.java
@@ -17,7 +17,7 @@
*/
package org.apache.cassandra.tools.nodetool;
-import io.airlift.command.Command;
+import io.airlift.airline.Command;
import org.apache.cassandra.tools.NodeProbe;
import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
diff --git a/src/java/org/apache/cassandra/tools/nodetool/EnableHandoff.java b/src/java/org/apache/cassandra/tools/nodetool/EnableHandoff.java
index 149c0fc..bccf7e7 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/EnableHandoff.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/EnableHandoff.java
@@ -17,7 +17,7 @@
*/
package org.apache.cassandra.tools.nodetool;
-import io.airlift.command.Command;
+import io.airlift.airline.Command;
import org.apache.cassandra.tools.NodeProbe;
import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
diff --git a/src/java/org/apache/cassandra/tools/nodetool/EnableHintsForDC.java b/src/java/org/apache/cassandra/tools/nodetool/EnableHintsForDC.java
index 1979ebd..97e40e0 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/EnableHintsForDC.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/EnableHintsForDC.java
@@ -20,8 +20,8 @@
import java.util.ArrayList;
import java.util.List;
-import io.airlift.command.Arguments;
-import io.airlift.command.Command;
+import io.airlift.airline.Arguments;
+import io.airlift.airline.Command;
import org.apache.cassandra.tools.NodeProbe;
import org.apache.cassandra.tools.NodeTool;
diff --git a/src/java/org/apache/cassandra/tools/nodetool/EnableOldProtocolVersions.java b/src/java/org/apache/cassandra/tools/nodetool/EnableOldProtocolVersions.java
new file mode 100644
index 0000000..f6d5be5
--- /dev/null
+++ b/src/java/org/apache/cassandra/tools/nodetool/EnableOldProtocolVersions.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.tools.nodetool;
+
+import io.airlift.airline.Command;
+import org.apache.cassandra.tools.NodeProbe;
+import org.apache.cassandra.tools.NodeTool;
+
+
+@Command(name = "enableoldprotocolversions", description = "Enable old protocol versions")
+public class EnableOldProtocolVersions extends NodeTool.NodeToolCmd
+{
+ @Override
+ public void execute(NodeProbe probe)
+ {
+ probe.enableOldProtocolVersions();
+ }
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/tools/nodetool/EnableThrift.java b/src/java/org/apache/cassandra/tools/nodetool/EnableThrift.java
deleted file mode 100644
index 780b36d..0000000
--- a/src/java/org/apache/cassandra/tools/nodetool/EnableThrift.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.tools.nodetool;
-
-import io.airlift.command.Command;
-
-import org.apache.cassandra.tools.NodeProbe;
-import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
-
-@Command(name = "enablethrift", description = "Reenable thrift server")
-public class EnableThrift extends NodeToolCmd
-{
- @Override
- public void execute(NodeProbe probe)
- {
- probe.startThriftServer();
- }
-}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/tools/nodetool/FailureDetectorInfo.java b/src/java/org/apache/cassandra/tools/nodetool/FailureDetectorInfo.java
index 3c0303d..c1b2192 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/FailureDetectorInfo.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/FailureDetectorInfo.java
@@ -17,7 +17,7 @@
*/
package org.apache.cassandra.tools.nodetool;
-import io.airlift.command.Command;
+import io.airlift.airline.Command;
import java.util.List;
@@ -33,7 +33,7 @@
@Override
public void execute(NodeProbe probe)
{
- TabularData data = probe.getFailureDetectorPhilValues();
+ TabularData data = probe.getFailureDetectorPhilValues(printPort);
System.out.printf("%10s,%16s%n", "Endpoint", "Phi");
for (Object o : data.keySet())
{
diff --git a/src/java/org/apache/cassandra/tools/nodetool/Flush.java b/src/java/org/apache/cassandra/tools/nodetool/Flush.java
index f768615..c83e420 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/Flush.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/Flush.java
@@ -17,8 +17,8 @@
*/
package org.apache.cassandra.tools.nodetool;
-import io.airlift.command.Arguments;
-import io.airlift.command.Command;
+import io.airlift.airline.Arguments;
+import io.airlift.airline.Command;
import java.util.ArrayList;
import java.util.List;
diff --git a/src/java/org/apache/cassandra/tools/nodetool/GarbageCollect.java b/src/java/org/apache/cassandra/tools/nodetool/GarbageCollect.java
index baa245f..5b8cd6e 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/GarbageCollect.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/GarbageCollect.java
@@ -17,9 +17,9 @@
*/
package org.apache.cassandra.tools.nodetool;
-import io.airlift.command.Arguments;
-import io.airlift.command.Command;
-import io.airlift.command.Option;
+import io.airlift.airline.Arguments;
+import io.airlift.airline.Command;
+import io.airlift.airline.Option;
import java.util.ArrayList;
import java.util.List;
diff --git a/src/java/org/apache/cassandra/tools/nodetool/GcStats.java b/src/java/org/apache/cassandra/tools/nodetool/GcStats.java
index dd38fe7..07ae6d9 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/GcStats.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/GcStats.java
@@ -20,7 +20,7 @@
import org.apache.cassandra.tools.NodeProbe;
import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
-import io.airlift.command.Command;
+import io.airlift.airline.Command;
@Command(name = "gcstats", description = "Print GC Statistics")
public class GcStats extends NodeToolCmd
diff --git a/src/java/org/apache/cassandra/tools/nodetool/GetBatchlogReplayTrottle.java b/src/java/org/apache/cassandra/tools/nodetool/GetBatchlogReplayTrottle.java
new file mode 100644
index 0000000..661c495
--- /dev/null
+++ b/src/java/org/apache/cassandra/tools/nodetool/GetBatchlogReplayTrottle.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.tools.nodetool;
+
+import io.airlift.airline.Command;
+import org.apache.cassandra.tools.NodeProbe;
+import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
+
+@Command(name = "getbatchlogreplaythrottle", description = "Print batchlog replay throttle in KB/s. " +
+ "This is reduced proportionally to the number of nodes in the cluster.")
+public class GetBatchlogReplayTrottle extends NodeToolCmd
+{
+ @Override
+ public void execute(NodeProbe probe)
+ {
+ System.out.println("Batchlog replay throttle: " + probe.getBatchlogReplayThrottle() + " KB/s");
+ }
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/tools/nodetool/GetCompactionThreshold.java b/src/java/org/apache/cassandra/tools/nodetool/GetCompactionThreshold.java
index 6c629de..589b1b3 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/GetCompactionThreshold.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/GetCompactionThreshold.java
@@ -18,8 +18,8 @@
package org.apache.cassandra.tools.nodetool;
import static com.google.common.base.Preconditions.checkArgument;
-import io.airlift.command.Arguments;
-import io.airlift.command.Command;
+import io.airlift.airline.Arguments;
+import io.airlift.airline.Command;
import java.util.ArrayList;
import java.util.List;
diff --git a/src/java/org/apache/cassandra/tools/nodetool/GetCompactionThroughput.java b/src/java/org/apache/cassandra/tools/nodetool/GetCompactionThroughput.java
index c3af184..a7df4d1 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/GetCompactionThroughput.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/GetCompactionThroughput.java
@@ -17,7 +17,7 @@
*/
package org.apache.cassandra.tools.nodetool;
-import io.airlift.command.Command;
+import io.airlift.airline.Command;
import org.apache.cassandra.tools.NodeProbe;
import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
diff --git a/src/java/org/apache/cassandra/tools/nodetool/GetConcurrency.java b/src/java/org/apache/cassandra/tools/nodetool/GetConcurrency.java
new file mode 100644
index 0000000..ede2908
--- /dev/null
+++ b/src/java/org/apache/cassandra/tools/nodetool/GetConcurrency.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.tools.nodetool;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import io.airlift.airline.Arguments;
+import io.airlift.airline.Command;
+import org.apache.cassandra.tools.NodeProbe;
+import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
+
+@Command(name = "getconcurrency", description = "Get maximum concurrency for processing stages")
+public class GetConcurrency extends NodeToolCmd
+{
+ @Arguments(title = "[stage-names]",
+ usage = "[stage-names]",
+ description = "optional list of stage names, otherwise display all stages")
+ private List<String> args = new ArrayList<>();
+
+ @Override
+ public void execute(NodeProbe probe)
+ {
+ System.out.printf("%-25s%16s%16s%n", "Stage", "CorePoolSize", "MaximumPoolSize");
+ probe.getMaximumPoolSizes(args).entrySet().stream()
+ .sorted(Map.Entry.comparingByKey())
+ .forEach(entry ->
+ System.out.printf("%-25s%16d%16d%n",
+ entry.getKey(),
+ entry.getValue().get(0),
+ entry.getValue().get(1)));
+
+ }
+}
diff --git a/src/java/org/apache/cassandra/tools/nodetool/GetConcurrentCompactors.java b/src/java/org/apache/cassandra/tools/nodetool/GetConcurrentCompactors.java
index 8f4d5e4..6aa4d8b 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/GetConcurrentCompactors.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/GetConcurrentCompactors.java
@@ -18,7 +18,7 @@
package org.apache.cassandra.tools.nodetool;
-import io.airlift.command.Command;
+import io.airlift.airline.Command;
import org.apache.cassandra.tools.NodeProbe;
import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
diff --git a/src/java/org/apache/cassandra/tools/nodetool/GetConcurrentViewBuilders.java b/src/java/org/apache/cassandra/tools/nodetool/GetConcurrentViewBuilders.java
new file mode 100644
index 0000000..c189fb0
--- /dev/null
+++ b/src/java/org/apache/cassandra/tools/nodetool/GetConcurrentViewBuilders.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.tools.nodetool;
+
+import io.airlift.airline.Command;
+import org.apache.cassandra.tools.NodeProbe;
+import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
+
+@Command(name = "getconcurrentviewbuilders", description = "Get the number of concurrent view builders in the system")
+public class GetConcurrentViewBuilders extends NodeToolCmd
+{
+ protected void execute(NodeProbe probe)
+ {
+ System.out.println("Current number of concurrent view builders in the system is: \n" +
+ probe.getConcurrentViewBuilders());
+ }
+}
diff --git a/src/java/org/apache/cassandra/tools/nodetool/GetEndpoints.java b/src/java/org/apache/cassandra/tools/nodetool/GetEndpoints.java
index 49d2148..78065c4 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/GetEndpoints.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/GetEndpoints.java
@@ -18,8 +18,8 @@
package org.apache.cassandra.tools.nodetool;
import static com.google.common.base.Preconditions.checkArgument;
-import io.airlift.command.Arguments;
-import io.airlift.command.Command;
+import io.airlift.airline.Arguments;
+import io.airlift.airline.Command;
import java.net.InetAddress;
import java.util.ArrayList;
@@ -42,10 +42,20 @@
String table = args.get(1);
String key = args.get(2);
- List<InetAddress> endpoints = probe.getEndpoints(ks, table, key);
- for (InetAddress endpoint : endpoints)
+ if (printPort)
{
- System.out.println(endpoint.getHostAddress());
+ for (String endpoint : probe.getEndpointsWithPort(ks, table, key))
+ {
+ System.out.println(endpoint);
+ }
+ }
+ else
+ {
+ List<InetAddress> endpoints = probe.getEndpoints(ks, table, key);
+ for (InetAddress endpoint : endpoints)
+ {
+ System.out.println(endpoint.getHostAddress());
+ }
}
}
-}
\ No newline at end of file
+}
diff --git a/src/java/org/apache/cassandra/tools/nodetool/GetInterDCStreamThroughput.java b/src/java/org/apache/cassandra/tools/nodetool/GetInterDCStreamThroughput.java
index 4c354c0..039814e 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/GetInterDCStreamThroughput.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/GetInterDCStreamThroughput.java
@@ -17,7 +17,7 @@
*/
package org.apache.cassandra.tools.nodetool;
-import io.airlift.command.Command;
+import io.airlift.airline.Command;
import org.apache.cassandra.tools.NodeProbe;
import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
diff --git a/src/java/org/apache/cassandra/tools/nodetool/GetLoggingLevels.java b/src/java/org/apache/cassandra/tools/nodetool/GetLoggingLevels.java
index 7ce0017..90d6817 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/GetLoggingLevels.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/GetLoggingLevels.java
@@ -17,7 +17,7 @@
*/
package org.apache.cassandra.tools.nodetool;
-import io.airlift.command.Command;
+import io.airlift.airline.Command;
import java.util.Map;
diff --git a/src/java/org/apache/cassandra/tools/nodetool/GetMaxHintWindow.java b/src/java/org/apache/cassandra/tools/nodetool/GetMaxHintWindow.java
new file mode 100644
index 0000000..7bc1a30
--- /dev/null
+++ b/src/java/org/apache/cassandra/tools/nodetool/GetMaxHintWindow.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.tools.nodetool;
+
+import io.airlift.airline.Command;
+import org.apache.cassandra.tools.NodeProbe;
+import org.apache.cassandra.tools.NodeTool;
+
+@Command(name = "getmaxhintwindow", description = "Print the max hint window in ms")
+public class GetMaxHintWindow extends NodeTool.NodeToolCmd
+{
+ @Override
+ public void execute(NodeProbe probe)
+ {
+ System.out.println("Current max hint window: " + probe.getMaxHintWindow() + " ms");
+ }
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/tools/nodetool/GetSSTables.java b/src/java/org/apache/cassandra/tools/nodetool/GetSSTables.java
index 849ad94..e43c2bf 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/GetSSTables.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/GetSSTables.java
@@ -18,13 +18,13 @@
package org.apache.cassandra.tools.nodetool;
import static com.google.common.base.Preconditions.checkArgument;
-import io.airlift.command.Arguments;
-import io.airlift.command.Command;
+import io.airlift.airline.Arguments;
+import io.airlift.airline.Command;
import java.util.ArrayList;
import java.util.List;
-import io.airlift.command.Option;
+import io.airlift.airline.Option;
import org.apache.cassandra.tools.NodeProbe;
import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
diff --git a/src/java/org/apache/cassandra/tools/nodetool/GetSeeds.java b/src/java/org/apache/cassandra/tools/nodetool/GetSeeds.java
new file mode 100644
index 0000000..207363c
--- /dev/null
+++ b/src/java/org/apache/cassandra/tools/nodetool/GetSeeds.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.tools.nodetool;
+
+import java.util.List;
+
+import io.airlift.airline.Command;
+
+import org.apache.cassandra.tools.NodeProbe;
+import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
+
+@Command(name = "getseeds", description = "Get the currently in use seed node IP list excluding the node IP")
+public class GetSeeds extends NodeToolCmd
+{
+ @Override
+ public void execute(NodeProbe probe)
+ {
+ List<String> seedList = probe.getSeeds();
+ if (seedList.isEmpty())
+ {
+ System.out.println("Seed node list does not contain any remote node IPs");
+ }
+ else
+ {
+ System.out.println("Current list of seed node IPs, excluding the current node's IP: " + String.join(" ", seedList));
+ }
+
+ }
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/tools/nodetool/GetStreamThroughput.java b/src/java/org/apache/cassandra/tools/nodetool/GetStreamThroughput.java
index 437eb54..b76d14b 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/GetStreamThroughput.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/GetStreamThroughput.java
@@ -17,7 +17,7 @@
*/
package org.apache.cassandra.tools.nodetool;
-import io.airlift.command.Command;
+import io.airlift.airline.Command;
import org.apache.cassandra.tools.NodeProbe;
import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
diff --git a/src/java/org/apache/cassandra/tools/nodetool/GetTimeout.java b/src/java/org/apache/cassandra/tools/nodetool/GetTimeout.java
index b12c9a7..9f99ac6 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/GetTimeout.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/GetTimeout.java
@@ -17,8 +17,8 @@
*/
package org.apache.cassandra.tools.nodetool;
-import io.airlift.command.Arguments;
-import io.airlift.command.Command;
+import io.airlift.airline.Arguments;
+import io.airlift.airline.Command;
import java.util.ArrayList;
import java.util.List;
@@ -31,7 +31,7 @@
@Command(name = "gettimeout", description = "Print the timeout of the given type in ms")
public class GetTimeout extends NodeToolCmd
{
- public static final String TIMEOUT_TYPES = "read, range, write, counterwrite, cascontention, truncate, streamingsocket, misc (general rpc_timeout_in_ms)";
+ public static final String TIMEOUT_TYPES = "read, range, write, counterwrite, cascontention, truncate, internodeconnect, internodeuser, misc (general rpc_timeout_in_ms)";
@Arguments(usage = "<timeout_type>", description = "The timeout type, one of (" + TIMEOUT_TYPES + ")")
private List<String> args = new ArrayList<>();
diff --git a/src/java/org/apache/cassandra/tools/nodetool/GetTraceProbability.java b/src/java/org/apache/cassandra/tools/nodetool/GetTraceProbability.java
index 3940790..374ab2c 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/GetTraceProbability.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/GetTraceProbability.java
@@ -17,7 +17,7 @@
*/
package org.apache.cassandra.tools.nodetool;
-import io.airlift.command.Command;
+import io.airlift.airline.Command;
import org.apache.cassandra.tools.NodeProbe;
import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
diff --git a/src/java/org/apache/cassandra/tools/nodetool/GossipInfo.java b/src/java/org/apache/cassandra/tools/nodetool/GossipInfo.java
index 2acfcf1..24c0634 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/GossipInfo.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/GossipInfo.java
@@ -17,7 +17,7 @@
*/
package org.apache.cassandra.tools.nodetool;
-import io.airlift.command.Command;
+import io.airlift.airline.Command;
import org.apache.cassandra.tools.NodeProbe;
import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
@@ -28,6 +28,6 @@
@Override
public void execute(NodeProbe probe)
{
- System.out.println(probe.getGossipInfo());
+ System.out.println(probe.getGossipInfo(printPort));
}
-}
\ No newline at end of file
+}
diff --git a/src/java/org/apache/cassandra/tools/nodetool/HostStatWithPort.java b/src/java/org/apache/cassandra/tools/nodetool/HostStatWithPort.java
new file mode 100644
index 0000000..4849fd1
--- /dev/null
+++ b/src/java/org/apache/cassandra/tools/nodetool/HostStatWithPort.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.tools.nodetool;
+
+import org.apache.cassandra.locator.InetAddressAndPort;
+
+public class HostStatWithPort extends HostStat
+{
+ public final InetAddressAndPort endpointWithPort;
+
+ public HostStatWithPort(String token, InetAddressAndPort endpoint, boolean resolveIp, Float owns)
+ {
+ super(token, endpoint.address, resolveIp, owns);
+ this.endpointWithPort = endpoint;
+ }
+
+ public String ipOrDns()
+ {
+ return ipOrDns(true);
+ }
+
+ public String ipOrDns(boolean withPort)
+ {
+ if (!withPort)
+ return super.ipOrDns();
+
+ return resolveIp ?
+ endpointWithPort.address.getHostName() + ':' + endpointWithPort.port :
+ endpointWithPort.toString();
+ }
+}
diff --git a/src/java/org/apache/cassandra/tools/nodetool/Import.java b/src/java/org/apache/cassandra/tools/nodetool/Import.java
new file mode 100644
index 0000000..7315c3e
--- /dev/null
+++ b/src/java/org/apache/cassandra/tools/nodetool/Import.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.tools.nodetool;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import io.airlift.airline.Arguments;
+import io.airlift.airline.Command;
+
+import io.airlift.airline.Option;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+
+import com.google.common.collect.Lists;
+
+import org.apache.cassandra.tools.NodeProbe;
+import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
+
+@Command(name = "import", description = "Import new SSTables to the system")
+public class Import extends NodeToolCmd
+{
+ @Arguments(usage = "<keyspace> <table> <directory> ...", description = "The keyspace, table name and directories to import sstables from")
+ private List<String> args = new ArrayList<>();
+
+ @Option(title = "keep_level",
+ name = {"-l", "--keep-level"},
+ description = "Keep the level on the new sstables")
+ private boolean keepLevel = false;
+
+ @Option(title = "keep_repaired",
+ name = {"-r", "--keep-repaired"},
+ description = "Keep any repaired information from the sstables")
+ private boolean keepRepaired = false;
+
+ @Option(title = "no_verify_sstables",
+ name = {"-v", "--no-verify"},
+ description = "Don't verify new sstables")
+ private boolean noVerify = false;
+
+ @Option(title = "no_verify_tokens",
+ name = {"-t", "--no-tokens"},
+ description = "Don't verify that all tokens in the new sstable are owned by the current node")
+ private boolean noVerifyTokens = false;
+
+ @Option(title = "no_invalidate_caches",
+ name = {"-c", "--no-invalidate-caches"},
+ description = "Don't invalidate the row cache when importing")
+ private boolean noInvalidateCaches = false;
+
+ @Option(title = "quick",
+ name = {"-q", "--quick"},
+ description = "Do a quick import without verifying sstables, clearing row cache or checking in which data directory to put the file")
+ private boolean quick = false;
+
+ @Option(title = "extended_verify",
+ name = {"-e", "--extended-verify"},
+ description = "Run an extended verify, verifying all values in the new sstables")
+ private boolean extendedVerify = false;
+
+ @Override
+ public void execute(NodeProbe probe)
+ {
+ checkArgument(args.size() >= 3, "import requires keyspace, table name and directories");
+
+ if (quick)
+ {
+ System.out.println("Doing a quick import - skipping sstable verification and row cache invalidation");
+ noVerifyTokens = true;
+ noInvalidateCaches = true;
+ noVerify = true;
+ extendedVerify = false;
+ }
+ List<String> srcPaths = Lists.newArrayList(args.subList(2, args.size()));
+ List<String> failedDirs = probe.importNewSSTables(args.get(0), args.get(1), new HashSet<>(srcPaths), !keepLevel, !keepRepaired, !noVerify, !noVerifyTokens, !noInvalidateCaches, extendedVerify);
+ if (!failedDirs.isEmpty())
+ {
+ System.err.println("Some directories failed to import, check server logs for details:");
+ for (String directory : failedDirs)
+ System.err.println(directory);
+ System.exit(1);
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/tools/nodetool/Info.java b/src/java/org/apache/cassandra/tools/nodetool/Info.java
index 032e47f..6ff5037 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/Info.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/Info.java
@@ -17,8 +17,8 @@
*/
package org.apache.cassandra.tools.nodetool;
-import io.airlift.command.Command;
-import io.airlift.command.Option;
+import io.airlift.airline.Command;
+import io.airlift.airline.Option;
import java.lang.management.MemoryUsage;
import java.util.Iterator;
@@ -47,7 +47,6 @@
System.out.printf("%-23s: %s%n", "ID", probe.getLocalHostId());
System.out.printf("%-23s: %s%n", "Gossip active", gossipInitialized);
- System.out.printf("%-23s: %s%n", "Thrift active", probe.isThriftServerRunning());
System.out.printf("%-23s: %s%n", "Native Transport active", probe.isNativeTransportRunning());
System.out.printf("%-23s: %s%n", "Load", probe.getLoadString());
if (gossipInitialized)
@@ -174,7 +173,7 @@
{
Entry<String, ColumnFamilyStoreMBean> entry = cfamilies.next();
String keyspaceName = entry.getKey();
- String cfName = entry.getValue().getColumnFamilyName();
+ String cfName = entry.getValue().getTableName();
offHeapMemUsedInBytes += (Long) probe.getColumnFamilyMetric(keyspaceName, cfName, "MemtableOffHeapSize");
offHeapMemUsedInBytes += (Long) probe.getColumnFamilyMetric(keyspaceName, cfName, "BloomFilterOffHeapMemoryUsed");
diff --git a/src/java/org/apache/cassandra/tools/nodetool/InvalidateCounterCache.java b/src/java/org/apache/cassandra/tools/nodetool/InvalidateCounterCache.java
index a5f0ebc..aef77bd 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/InvalidateCounterCache.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/InvalidateCounterCache.java
@@ -17,7 +17,7 @@
*/
package org.apache.cassandra.tools.nodetool;
-import io.airlift.command.Command;
+import io.airlift.airline.Command;
import org.apache.cassandra.tools.NodeProbe;
import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
diff --git a/src/java/org/apache/cassandra/tools/nodetool/InvalidateKeyCache.java b/src/java/org/apache/cassandra/tools/nodetool/InvalidateKeyCache.java
index 70abd53..cfe7d2f 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/InvalidateKeyCache.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/InvalidateKeyCache.java
@@ -17,7 +17,7 @@
*/
package org.apache.cassandra.tools.nodetool;
-import io.airlift.command.Command;
+import io.airlift.airline.Command;
import org.apache.cassandra.tools.NodeProbe;
import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
diff --git a/src/java/org/apache/cassandra/tools/nodetool/InvalidateRowCache.java b/src/java/org/apache/cassandra/tools/nodetool/InvalidateRowCache.java
index 149f80b..7357e27 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/InvalidateRowCache.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/InvalidateRowCache.java
@@ -17,7 +17,7 @@
*/
package org.apache.cassandra.tools.nodetool;
-import io.airlift.command.Command;
+import io.airlift.airline.Command;
import org.apache.cassandra.tools.NodeProbe;
import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
diff --git a/src/java/org/apache/cassandra/tools/nodetool/Join.java b/src/java/org/apache/cassandra/tools/nodetool/Join.java
index a4a7cad..c77559c 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/Join.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/Join.java
@@ -18,7 +18,7 @@
package org.apache.cassandra.tools.nodetool;
import static com.google.common.base.Preconditions.checkState;
-import io.airlift.command.Command;
+import io.airlift.airline.Command;
import java.io.IOException;
@@ -42,4 +42,4 @@
throw new RuntimeException("Error during joining the ring", e);
}
}
-}
\ No newline at end of file
+}
diff --git a/src/java/org/apache/cassandra/tools/nodetool/ListSnapshots.java b/src/java/org/apache/cassandra/tools/nodetool/ListSnapshots.java
index 84ce84b..9bc62d1 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/ListSnapshots.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/ListSnapshots.java
@@ -22,14 +22,14 @@
import java.util.Set;
import javax.management.openmbean.TabularData;
-import io.airlift.command.Command;
+import io.airlift.airline.Command;
import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.tools.NodeProbe;
import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
import org.apache.cassandra.tools.nodetool.formatter.TableBuilder;
-@Command(name = "listsnapshots", description = "Lists all the snapshots along with the size on disk and true size.")
+@Command(name = "listsnapshots", description = "Lists all the snapshots along with the size on disk and true size. True size is the total size of all SSTables which are not backed up to disk. Size on disk is total size of the snapshot on disk. Total TrueDiskSpaceUsed does not make any SSTable deduplication.")
public class ListSnapshots extends NodeToolCmd
{
@Override
@@ -70,4 +70,4 @@
throw new RuntimeException("Error during list snapshot", e);
}
}
-}
\ No newline at end of file
+}
diff --git a/src/java/org/apache/cassandra/tools/nodetool/Move.java b/src/java/org/apache/cassandra/tools/nodetool/Move.java
index fc6b1bf..8654d25 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/Move.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/Move.java
@@ -18,8 +18,8 @@
package org.apache.cassandra.tools.nodetool;
import static org.apache.commons.lang3.StringUtils.EMPTY;
-import io.airlift.command.Arguments;
-import io.airlift.command.Command;
+import io.airlift.airline.Arguments;
+import io.airlift.airline.Command;
import java.io.IOException;
diff --git a/src/java/org/apache/cassandra/tools/nodetool/NetStats.java b/src/java/org/apache/cassandra/tools/nodetool/NetStats.java
index 0c250aa..c0500ca 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/NetStats.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/NetStats.java
@@ -17,8 +17,8 @@
*/
package org.apache.cassandra.tools.nodetool;
-import io.airlift.command.Command;
-import io.airlift.command.Option;
+import io.airlift.airline.Command;
+import io.airlift.airline.Option;
import java.util.Set;
@@ -47,14 +47,14 @@
System.out.println("Not sending any streams.");
for (StreamState status : statuses)
{
- System.out.printf("%s %s%n", status.description, status.planId.toString());
+ System.out.printf("%s %s%n", status.streamOperation.getDescription(), status.planId.toString());
for (SessionInfo info : status.sessions)
{
- System.out.printf(" %s", info.peer.toString());
+ System.out.printf(" %s", info.peer.toString(printPort));
// print private IP when it is used
if (!info.peer.equals(info.connecting))
{
- System.out.printf(" (using %s)", info.connecting.toString());
+ System.out.printf(" (using %s)", info.connecting.toString(printPort));
}
System.out.printf("%n");
if (!info.receivingSummaries.isEmpty())
@@ -65,7 +65,7 @@
System.out.printf(" Receiving %d files, %d bytes total. Already received %d files, %d bytes total%n", info.getTotalFilesToReceive(), info.getTotalSizeToReceive(), info.getTotalFilesReceived(), info.getTotalSizeReceived());
for (ProgressInfo progress : info.getReceivingFiles())
{
- System.out.printf(" %s%n", progress.toString());
+ System.out.printf(" %s%n", progress.toString(printPort));
}
}
if (!info.sendingSummaries.isEmpty())
@@ -76,7 +76,7 @@
System.out.printf(" Sending %d files, %d bytes total. Already sent %d files, %d bytes total%n", info.getTotalFilesToSend(), info.getTotalSizeToSend(), info.getTotalFilesSent(), info.getTotalSizeSent());
for (ProgressInfo progress : info.getSendingFiles())
{
- System.out.printf(" %s%n", progress.toString());
+ System.out.printf(" %s%n", progress.toString(printPort));
}
}
}
diff --git a/src/java/org/apache/cassandra/tools/nodetool/PauseHandoff.java b/src/java/org/apache/cassandra/tools/nodetool/PauseHandoff.java
index ed1f655..4ec70d8 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/PauseHandoff.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/PauseHandoff.java
@@ -17,7 +17,7 @@
*/
package org.apache.cassandra.tools.nodetool;
-import io.airlift.command.Command;
+import io.airlift.airline.Command;
import org.apache.cassandra.tools.NodeProbe;
import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
diff --git a/src/java/org/apache/cassandra/tools/nodetool/ProfileLoad.java b/src/java/org/apache/cassandra/tools/nodetool/ProfileLoad.java
new file mode 100644
index 0000000..7037969
--- /dev/null
+++ b/src/java/org/apache/cassandra/tools/nodetool/ProfileLoad.java
@@ -0,0 +1,192 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.tools.nodetool;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static org.apache.commons.lang3.StringUtils.join;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.stream.Collectors;
+
+import javax.management.openmbean.CompositeData;
+import javax.management.openmbean.OpenDataException;
+
+import org.apache.cassandra.metrics.Sampler.SamplerType;
+import org.apache.cassandra.tools.NodeProbe;
+import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
+import org.apache.cassandra.tools.nodetool.formatter.TableBuilder;
+import org.apache.cassandra.utils.Pair;
+
+import com.google.common.collect.Lists;
+
+import io.airlift.airline.Arguments;
+import io.airlift.airline.Command;
+import io.airlift.airline.Option;
+
+@Command(name = "profileload", description = "Low footprint profiling of activity for a period of time")
+public class ProfileLoad extends NodeToolCmd
+{
+ @Arguments(usage = "<keyspace> <cfname> <duration>", description = "The keyspace, column family name, and duration in milliseconds")
+ private List<String> args = new ArrayList<>();
+
+ @Option(name = "-s", description = "Capacity of the sampler, higher for more accuracy (Default: 256)")
+ private int capacity = 256;
+
+ @Option(name = "-k", description = "Number of the top samples to list (Default: 10)")
+ private int topCount = 10;
+
+ @Option(name = "-a", description = "Comma separated list of samplers to use (Default: all)")
+ private String samplers = join(SamplerType.values(), ',');
+
+ @Override
+ public void execute(NodeProbe probe)
+ {
+ checkArgument(args.size() == 3 || args.size() == 1 || args.size() == 0, "Invalid arguments, either [keyspace table duration] or [duration] or no args");
+ checkArgument(topCount < capacity, "TopK count (-k) option must be smaller then the summary capacity (-s)");
+ String keyspace = null;
+ String table = null;
+ Integer durationMillis = 10000;
+ if(args.size() == 3)
+ {
+ keyspace = args.get(0);
+ table = args.get(1);
+ durationMillis = Integer.valueOf(args.get(2));
+ }
+ else if (args.size() == 1)
+ {
+ durationMillis = Integer.valueOf(args.get(0));
+ }
+ // generate the list of samplers
+ List<String> targets = Lists.newArrayList();
+ List<String> available = Arrays.stream(SamplerType.values()).map(Enum::toString).collect(Collectors.toList());
+ for (String s : samplers.split(","))
+ {
+ String sampler = s.trim().toUpperCase();
+ checkArgument(available.contains(sampler), String.format("'%s' sampler is not available from: %s", s, Arrays.toString(SamplerType.values())));
+ targets.add(sampler);
+ }
+
+ Map<String, List<CompositeData>> results;
+ try
+ {
+ if (keyspace == null)
+ results = probe.getPartitionSample(capacity, durationMillis, topCount, targets);
+ else
+ results = probe.getPartitionSample(keyspace, table, capacity, durationMillis, topCount, targets);
+
+ } catch (OpenDataException e)
+ {
+ throw new RuntimeException(e);
+ }
+
+ AtomicBoolean first = new AtomicBoolean(true);
+ ResultBuilder rb = new ResultBuilder(first, results, targets);
+
+ for(String sampler : Lists.newArrayList("READS", "WRITES", "CAS_CONTENTIONS"))
+ {
+ rb.forType(SamplerType.valueOf(sampler), "Frequency of " + sampler.toLowerCase().replaceAll("_", " ") + " by partition")
+ .addColumn("Table", "table")
+ .addColumn("Partition", "value")
+ .addColumn("Count", "count")
+ .addColumn("+/-", "error")
+ .print();
+ }
+
+ rb.forType(SamplerType.WRITE_SIZE, "Max mutation size by partition")
+ .addColumn("Table", "table")
+ .addColumn("Partition", "value")
+ .addColumn("Bytes", "count")
+ .print();
+
+ rb.forType(SamplerType.LOCAL_READ_TIME, "Longest read query times")
+ .addColumn("Query", "value")
+ .addColumn("Microseconds", "count")
+ .print();
+ }
+
+ private class ResultBuilder
+ {
+ private SamplerType type;
+ private String description;
+ private AtomicBoolean first;
+ private Map<String, List<CompositeData>> results;
+ private List<String> targets;
+ private List<Pair<String, String>> dataKeys;
+
+ public ResultBuilder(AtomicBoolean first, Map<String, List<CompositeData>> results, List<String> targets)
+ {
+ super();
+ this.first = first;
+ this.results = results;
+ this.targets = targets;
+ this.dataKeys = new ArrayList<>();
+ this.dataKeys.add(Pair.create(" ", " "));
+ }
+
+ public ResultBuilder forType(SamplerType type, String description)
+ {
+ ResultBuilder rb = new ResultBuilder(first, results, targets);
+ rb.type = type;
+ rb.description = description;
+ return rb;
+ }
+
+ public ResultBuilder addColumn(String title, String key)
+ {
+ this.dataKeys.add(Pair.create(title, key));
+ return this;
+ }
+
+ private String get(CompositeData cd, String key)
+ {
+ if (cd.containsKey(key))
+ return cd.get(key).toString();
+ return key;
+ }
+
+ public void print()
+ {
+ if (targets.contains(type.toString()))
+ {
+ if (!first.get())
+ System.out.println();
+ first.set(false);
+ System.out.println(description + ':');
+ TableBuilder out = new TableBuilder();
+ out.add(dataKeys.stream().map(p -> p.left).collect(Collectors.toList()).toArray(new String[] {}));
+ List<CompositeData> topk = results.get(type.toString());
+ for (CompositeData cd : topk)
+ {
+ out.add(dataKeys.stream().map(p -> get(cd, p.right)).collect(Collectors.toList()).toArray(new String[] {}));
+ }
+ if (topk.size() == 0)
+ {
+ System.out.println(" Nothing recorded during sampling period...");
+ }
+ else
+ {
+ out.printTo(System.out);
+ }
+ }
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/tools/nodetool/ProxyHistograms.java b/src/java/org/apache/cassandra/tools/nodetool/ProxyHistograms.java
index 656e7ed..620d75a 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/ProxyHistograms.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/ProxyHistograms.java
@@ -18,7 +18,7 @@
package org.apache.cassandra.tools.nodetool;
import static java.lang.String.format;
-import io.airlift.command.Command;
+import io.airlift.airline.Command;
import org.apache.cassandra.tools.NodeProbe;
import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
diff --git a/src/java/org/apache/cassandra/tools/nodetool/RangeKeySample.java b/src/java/org/apache/cassandra/tools/nodetool/RangeKeySample.java
index e079a4b..1ca2aa9 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/RangeKeySample.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/RangeKeySample.java
@@ -17,7 +17,7 @@
*/
package org.apache.cassandra.tools.nodetool;
-import io.airlift.command.Command;
+import io.airlift.airline.Command;
import java.util.List;
diff --git a/src/java/org/apache/cassandra/tools/nodetool/Rebuild.java b/src/java/org/apache/cassandra/tools/nodetool/Rebuild.java
index b27e674..a083cde 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/Rebuild.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/Rebuild.java
@@ -17,9 +17,9 @@
*/
package org.apache.cassandra.tools.nodetool;
-import io.airlift.command.Arguments;
-import io.airlift.command.Command;
-import io.airlift.command.Option;
+import io.airlift.airline.Arguments;
+import io.airlift.airline.Command;
+import io.airlift.airline.Option;
import org.apache.cassandra.tools.NodeProbe;
import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
diff --git a/src/java/org/apache/cassandra/tools/nodetool/RebuildIndex.java b/src/java/org/apache/cassandra/tools/nodetool/RebuildIndex.java
index 5fd7327..4a6b071 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/RebuildIndex.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/RebuildIndex.java
@@ -19,8 +19,8 @@
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.collect.Iterables.toArray;
-import io.airlift.command.Arguments;
-import io.airlift.command.Command;
+import io.airlift.airline.Arguments;
+import io.airlift.airline.Command;
import java.util.ArrayList;
import java.util.List;
diff --git a/src/java/org/apache/cassandra/tools/nodetool/Refresh.java b/src/java/org/apache/cassandra/tools/nodetool/Refresh.java
index 153255c..3c8d4c9 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/Refresh.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/Refresh.java
@@ -18,8 +18,8 @@
package org.apache.cassandra.tools.nodetool;
import static com.google.common.base.Preconditions.checkArgument;
-import io.airlift.command.Arguments;
-import io.airlift.command.Command;
+import io.airlift.airline.Arguments;
+import io.airlift.airline.Command;
import java.util.ArrayList;
import java.util.List;
@@ -36,6 +36,7 @@
@Override
public void execute(NodeProbe probe)
{
+ System.out.println("nodetool refresh is deprecated, use nodetool import instead");
checkArgument(args.size() == 2, "refresh requires ks and cf args");
probe.loadNewSSTables(args.get(0), args.get(1));
}
diff --git a/src/java/org/apache/cassandra/tools/nodetool/RefreshSizeEstimates.java b/src/java/org/apache/cassandra/tools/nodetool/RefreshSizeEstimates.java
index 870c7b4..586b451 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/RefreshSizeEstimates.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/RefreshSizeEstimates.java
@@ -18,7 +18,7 @@
package org.apache.cassandra.tools.nodetool;
-import io.airlift.command.Command;
+import io.airlift.airline.Command;
import org.apache.cassandra.tools.NodeProbe;
import org.apache.cassandra.tools.NodeTool;
diff --git a/src/java/org/apache/cassandra/tools/nodetool/ReloadLocalSchema.java b/src/java/org/apache/cassandra/tools/nodetool/ReloadLocalSchema.java
index 78fbf2d..d4b1c69 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/ReloadLocalSchema.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/ReloadLocalSchema.java
@@ -17,7 +17,7 @@
*/
package org.apache.cassandra.tools.nodetool;
-import io.airlift.command.Command;
+import io.airlift.airline.Command;
import org.apache.cassandra.tools.NodeProbe;
import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
diff --git a/src/java/org/apache/cassandra/tools/nodetool/ReloadSeeds.java b/src/java/org/apache/cassandra/tools/nodetool/ReloadSeeds.java
new file mode 100644
index 0000000..b9682cf
--- /dev/null
+++ b/src/java/org/apache/cassandra/tools/nodetool/ReloadSeeds.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.tools.nodetool;
+
+import java.util.List;
+
+import io.airlift.airline.Command;
+
+import org.apache.cassandra.tools.NodeProbe;
+import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
+
+@Command(name = "reloadseeds", description = "Reload the seed node list from the seed node provider")
+public class ReloadSeeds extends NodeToolCmd
+{
+ @Override
+ public void execute(NodeProbe probe)
+ {
+ List<String> seedList = probe.reloadSeeds();
+ if (seedList == null)
+ {
+ System.out.println("Failed to reload the seed node list.");
+ }
+ else if (seedList.isEmpty())
+ {
+ System.out.println("Seed node list does not contain any remote node IPs");
+ }
+ else
+ {
+ System.out.println("Updated seed node IP list, excluding the current node's IP: " + String.join(" ", seedList));
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/tools/nodetool/ReloadTriggers.java b/src/java/org/apache/cassandra/tools/nodetool/ReloadTriggers.java
index 416aff0..6ca90fb 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/ReloadTriggers.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/ReloadTriggers.java
@@ -17,7 +17,7 @@
*/
package org.apache.cassandra.tools.nodetool;
-import io.airlift.command.Command;
+import io.airlift.airline.Command;
import org.apache.cassandra.tools.NodeProbe;
import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
diff --git a/src/java/org/apache/cassandra/tools/nodetool/RelocateSSTables.java b/src/java/org/apache/cassandra/tools/nodetool/RelocateSSTables.java
index 7c3066c..853b1d3 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/RelocateSSTables.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/RelocateSSTables.java
@@ -20,9 +20,9 @@
import java.util.ArrayList;
import java.util.List;
-import io.airlift.command.Arguments;
-import io.airlift.command.Command;
-import io.airlift.command.Option;
+import io.airlift.airline.Arguments;
+import io.airlift.airline.Command;
+import io.airlift.airline.Option;
import org.apache.cassandra.tools.NodeProbe;
import org.apache.cassandra.tools.NodeTool;
diff --git a/src/java/org/apache/cassandra/tools/nodetool/RemoveNode.java b/src/java/org/apache/cassandra/tools/nodetool/RemoveNode.java
index 848049e..0acb313 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/RemoveNode.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/RemoveNode.java
@@ -18,8 +18,8 @@
package org.apache.cassandra.tools.nodetool;
import static org.apache.commons.lang3.StringUtils.EMPTY;
-import io.airlift.command.Arguments;
-import io.airlift.command.Command;
+import io.airlift.airline.Arguments;
+import io.airlift.airline.Command;
import org.apache.cassandra.tools.NodeProbe;
import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
@@ -36,10 +36,10 @@
switch (removeOperation)
{
case "status":
- System.out.println("RemovalStatus: " + probe.getRemovalStatus());
+ System.out.println("RemovalStatus: " + probe.getRemovalStatus(printPort));
break;
case "force":
- System.out.println("RemovalStatus: " + probe.getRemovalStatus());
+ System.out.println("RemovalStatus: " + probe.getRemovalStatus(printPort));
probe.forceRemoveCompletion();
break;
default:
@@ -47,4 +47,4 @@
break;
}
}
-}
\ No newline at end of file
+}
diff --git a/src/java/org/apache/cassandra/tools/nodetool/Repair.java b/src/java/org/apache/cassandra/tools/nodetool/Repair.java
index 350601a..990d241 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/Repair.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/Repair.java
@@ -19,9 +19,9 @@
import static com.google.common.collect.Lists.newArrayList;
import static org.apache.commons.lang3.StringUtils.EMPTY;
-import io.airlift.command.Arguments;
-import io.airlift.command.Command;
-import io.airlift.command.Option;
+import io.airlift.airline.Arguments;
+import io.airlift.airline.Command;
+import io.airlift.airline.Option;
import java.util.ArrayList;
import java.util.HashMap;
@@ -31,7 +31,8 @@
import com.google.common.collect.Sets;
-import org.apache.cassandra.config.SchemaConstants;
+import org.apache.cassandra.schema.SchemaConstants;
+import org.apache.cassandra.streaming.PreviewKind;
import org.apache.cassandra.repair.RepairParallelism;
import org.apache.cassandra.repair.messages.RepairOption;
import org.apache.cassandra.tools.NodeProbe;
@@ -61,10 +62,10 @@
@Option(title = "specific_host", name = {"-hosts", "--in-hosts"}, description = "Use -hosts to repair specific hosts")
private List<String> specificHosts = new ArrayList<>();
- @Option(title = "start_token", name = {"-st", "--start-token"}, description = "Use -st to specify a token at which the repair range starts")
+ @Option(title = "start_token", name = {"-st", "--start-token"}, description = "Use -st to specify a token at which the repair range starts (exclusive)")
private String startToken = EMPTY;
- @Option(title = "end_token", name = {"-et", "--end-token"}, description = "Use -et to specify a token at which repair range ends")
+ @Option(title = "end_token", name = {"-et", "--end-token"}, description = "Use -et to specify a token at which repair range ends (inclusive)")
private String endToken = EMPTY;
@Option(title = "primary_range", name = {"-pr", "--partitioner-range"}, description = "Use -pr to repair only the first range returned by the partitioner")
@@ -73,6 +74,15 @@
@Option(title = "full", name = {"-full", "--full"}, description = "Use -full to issue a full repair.")
private boolean fullRepair = false;
+ @Option(title = "force", name = {"-force", "--force"}, description = "Use -force to filter out down endpoints")
+ private boolean force = false;
+
+ @Option(title = "preview", name = {"-prv", "--preview"}, description = "Determine ranges and amount of data to be streamed, but don't actually perform repair")
+ private boolean preview = false;
+
+ @Option(title = "validate", name = {"-vd", "--validate"}, description = "Checks that repaired data is in sync between nodes. Out of sync repaired data indicates a full repair should be run.")
+ private boolean validate = false;
+
@Option(title = "job_threads", name = {"-j", "--job-threads"}, description = "Number of threads to run repair jobs. " +
"Usually this means number of CFs to repair concurrently. " +
"WARNING: increasing this puts more load on repairing nodes, so be careful. (default: 1, max: 4)")
@@ -84,6 +94,30 @@
@Option(title = "pull_repair", name = {"-pl", "--pull"}, description = "Use --pull to perform a one way repair where data is only streamed from a remote node to this node.")
private boolean pullRepair = false;
+ @Option(title = "optimise_streams", name = {"-os", "--optimise-streams"}, description = "Use --optimise-streams to try to reduce the number of streams we do (EXPERIMENTAL, see CASSANDRA-3200).")
+ private boolean optimiseStreams = false;
+
+
+ private PreviewKind getPreviewKind()
+ {
+ if (validate)
+ {
+ return PreviewKind.REPAIRED;
+ }
+ else if (preview && fullRepair)
+ {
+ return PreviewKind.ALL;
+ }
+ else if (preview)
+ {
+ return PreviewKind.UNREPAIRED;
+ }
+ else
+ {
+ return PreviewKind.NONE;
+ }
+ }
+
@Override
public void execute(NodeProbe probe)
{
@@ -112,6 +146,9 @@
options.put(RepairOption.TRACE_KEY, Boolean.toString(trace));
options.put(RepairOption.COLUMNFAMILIES_KEY, StringUtils.join(cfnames, ","));
options.put(RepairOption.PULL_REPAIR_KEY, Boolean.toString(pullRepair));
+ options.put(RepairOption.FORCE_REPAIR_KEY, Boolean.toString(force));
+ options.put(RepairOption.PREVIEW, getPreviewKind().toString());
+ options.put(RepairOption.OPTIMISE_STREAMS_KEY, Boolean.toString(optimiseStreams));
if (!startToken.isEmpty() || !endToken.isEmpty())
{
options.put(RepairOption.RANGES_KEY, startToken + ":" + endToken);
diff --git a/src/java/org/apache/cassandra/tools/nodetool/RepairAdmin.java b/src/java/org/apache/cassandra/tools/nodetool/RepairAdmin.java
new file mode 100644
index 0000000..ba3cf62
--- /dev/null
+++ b/src/java/org/apache/cassandra/tools/nodetool/RepairAdmin.java
@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.tools.nodetool;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+
+import io.airlift.airline.Command;
+import io.airlift.airline.Option;
+import org.apache.cassandra.repair.consistent.LocalSessionInfo;
+import org.apache.cassandra.service.ActiveRepairServiceMBean;
+import org.apache.cassandra.tools.NodeProbe;
+import org.apache.cassandra.tools.NodeTool;
+import org.apache.cassandra.utils.FBUtilities;
+
+/**
+ * Supports listing and failing incremental repair sessions
+ */
+@Command(name = "repair_admin", description = "list and fail incremental repair sessions")
+public class RepairAdmin extends NodeTool.NodeToolCmd
+{
+ @Option(title = "list", name = {"-l", "--list"}, description = "list repair sessions (default behavior)")
+ private boolean list = false;
+
+ @Option(title = "all", name = {"-a", "--all"}, description = "include completed and failed sessions")
+ private boolean all = false;
+
+ @Option(title = "cancel", name = {"-x", "--cancel"}, description = "cancel an incremental repair session")
+ private String cancel = null;
+
+ @Option(title = "force", name = {"-f", "--force"}, description = "cancel repair session from a node other than the repair coordinator." +
+ " Attempting to cancel FINALIZED or FAILED sessions is an error.")
+ private boolean force = false;
+
+ private static final List<String> header = Lists.newArrayList("id",
+ "state",
+ "last activity",
+ "coordinator",
+ "participants",
+ "participants_wp");
+
+
+ private List<String> sessionValues(Map<String, String> session, int now)
+ {
+ int updated = Integer.parseInt(session.get(LocalSessionInfo.LAST_UPDATE));
+ return Lists.newArrayList(session.get(LocalSessionInfo.SESSION_ID),
+ session.get(LocalSessionInfo.STATE),
+ Integer.toString(now - updated) + " (s)",
+ session.get(LocalSessionInfo.COORDINATOR),
+ session.get(LocalSessionInfo.PARTICIPANTS),
+ session.get(LocalSessionInfo.PARTICIPANTS_WP));
+ }
+
+ private void listSessions(ActiveRepairServiceMBean repairServiceProxy)
+ {
+ Preconditions.checkArgument(cancel == null);
+ Preconditions.checkArgument(!force, "-f/--force only valid for session cancel");
+ List<Map<String, String>> sessions = repairServiceProxy.getSessions(all);
+ if (sessions.isEmpty())
+ {
+ System.out.println("no sessions");
+
+ }
+ else
+ {
+ List<List<String>> rows = new ArrayList<>();
+ rows.add(header);
+ int now = FBUtilities.nowInSeconds();
+ for (Map<String, String> session : sessions)
+ {
+ rows.add(sessionValues(session, now));
+ }
+
+ // get max col widths
+ int[] widths = new int[header.size()];
+ for (List<String> row : rows)
+ {
+ assert row.size() == widths.length;
+ for (int i = 0; i < widths.length; i++)
+ {
+ widths[i] = Math.max(widths[i], row.get(i).length());
+ }
+ }
+
+ List<String> fmts = new ArrayList<>(widths.length);
+ for (int i = 0; i < widths.length; i++)
+ {
+ fmts.add("%-" + Integer.toString(widths[i]) + "s");
+ }
+
+
+ // print
+ for (List<String> row : rows)
+ {
+ List<String> formatted = new ArrayList<>(row.size());
+ for (int i = 0; i < widths.length; i++)
+ {
+ formatted.add(String.format(fmts.get(i), row.get(i)));
+ }
+ System.out.println(Joiner.on(" | ").join(formatted));
+ }
+ }
+ }
+
+ private void cancelSession(ActiveRepairServiceMBean repairServiceProxy)
+ {
+ Preconditions.checkArgument(!list);
+ Preconditions.checkArgument(!all, "-a/--all only valid for session list");
+ repairServiceProxy.failSession(cancel, force);
+ }
+
+ protected void execute(NodeProbe probe)
+ {
+ if (list && cancel != null)
+ {
+ throw new RuntimeException("Can either list, or cancel sessions, not both");
+ }
+ else if (cancel != null)
+ {
+ cancelSession(probe.getRepairServiceProxy());
+ }
+ else
+ {
+ // default
+ listSessions(probe.getRepairServiceProxy());
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/tools/nodetool/ReplayBatchlog.java b/src/java/org/apache/cassandra/tools/nodetool/ReplayBatchlog.java
index e3dcbd4..23c31f7 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/ReplayBatchlog.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/ReplayBatchlog.java
@@ -21,7 +21,7 @@
import java.io.IOError;
import java.io.IOException;
-import io.airlift.command.Command;
+import io.airlift.airline.Command;
import org.apache.cassandra.tools.NodeProbe;
import org.apache.cassandra.tools.NodeTool;
diff --git a/src/java/org/apache/cassandra/tools/nodetool/ResetFullQueryLog.java b/src/java/org/apache/cassandra/tools/nodetool/ResetFullQueryLog.java
new file mode 100644
index 0000000..786852d
--- /dev/null
+++ b/src/java/org/apache/cassandra/tools/nodetool/ResetFullQueryLog.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.tools.nodetool;
+
+import io.airlift.airline.Command;
+import org.apache.cassandra.tools.NodeProbe;
+import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
+
+@Command(name = "resetfullquerylog", description = "Stop the full query log and clean files in the configured full query log directory from cassandra.yaml as well as JMX")
+public class ResetFullQueryLog extends NodeToolCmd
+{
+ @Override
+ public void execute(NodeProbe probe)
+ {
+ probe.resetFullQueryLogger();
+ }
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/tools/nodetool/ResetLocalSchema.java b/src/java/org/apache/cassandra/tools/nodetool/ResetLocalSchema.java
index 43280ab..708636f 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/ResetLocalSchema.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/ResetLocalSchema.java
@@ -17,7 +17,7 @@
*/
package org.apache.cassandra.tools.nodetool;
-import io.airlift.command.Command;
+import io.airlift.airline.Command;
import java.io.IOException;
diff --git a/src/java/org/apache/cassandra/tools/nodetool/ResumeHandoff.java b/src/java/org/apache/cassandra/tools/nodetool/ResumeHandoff.java
index 584ab64..a3984f8 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/ResumeHandoff.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/ResumeHandoff.java
@@ -17,7 +17,7 @@
*/
package org.apache.cassandra.tools.nodetool;
-import io.airlift.command.Command;
+import io.airlift.airline.Command;
import org.apache.cassandra.tools.NodeProbe;
import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
diff --git a/src/java/org/apache/cassandra/tools/nodetool/Ring.java b/src/java/org/apache/cassandra/tools/nodetool/Ring.java
index c6b9af1..20cb890 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/Ring.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/Ring.java
@@ -18,9 +18,9 @@
package org.apache.cassandra.tools.nodetool;
import static java.lang.String.format;
-import io.airlift.command.Arguments;
-import io.airlift.command.Command;
-import io.airlift.command.Option;
+import io.airlift.airline.Arguments;
+import io.airlift.airline.Command;
+import io.airlift.airline.Option;
import java.net.InetAddress;
import java.net.UnknownHostException;
@@ -51,59 +51,103 @@
@Override
public void execute(NodeProbe probe)
{
- Map<String, String> tokensToEndpoints = probe.getTokenToEndpointMap();
- LinkedHashMultimap<String, String> endpointsToTokens = LinkedHashMultimap.create();
- boolean haveVnodes = false;
- for (Map.Entry<String, String> entry : tokensToEndpoints.entrySet())
- {
- haveVnodes |= endpointsToTokens.containsKey(entry.getValue());
- endpointsToTokens.put(entry.getValue(), entry.getKey());
- }
-
- int maxAddressLength = Collections.max(endpointsToTokens.keys(), new Comparator<String>()
- {
- @Override
- public int compare(String first, String second)
- {
- return Integer.compare(first.length(), second.length());
- }
- }).length();
-
- String formatPlaceholder = "%%-%ds %%-12s%%-7s%%-8s%%-16s%%-20s%%-44s%%n";
- String format = format(formatPlaceholder, maxAddressLength);
-
- StringBuilder errors = new StringBuilder();
- boolean showEffectiveOwnership = true;
- // Calculate per-token ownership of the ring
- Map<InetAddress, Float> ownerships;
try
{
- ownerships = probe.effectiveOwnership(keyspace);
- }
- catch (IllegalStateException ex)
+ Map<String, String> tokensToEndpoints = probe.getTokenToEndpointMap(printPort);
+ LinkedHashMultimap<String, String> endpointsToTokens = LinkedHashMultimap.create();
+ boolean haveVnodes = false;
+ for (Map.Entry<String, String> entry : tokensToEndpoints.entrySet())
+ {
+ haveVnodes |= endpointsToTokens.containsKey(entry.getValue());
+ endpointsToTokens.put(entry.getValue(), entry.getKey());
+ }
+
+ int maxAddressLength = Collections.max(endpointsToTokens.keys(), new Comparator<String>()
+ {
+ @Override
+ public int compare(String first, String second)
+ {
+ return Integer.compare(first.length(), second.length());
+ }
+ }).length();
+
+ String formatPlaceholder = "%%-%ds %%-12s%%-7s%%-8s%%-16s%%-20s%%-44s%%n";
+ String format = format(formatPlaceholder, maxAddressLength);
+
+ StringBuilder errors = new StringBuilder();
+ boolean showEffectiveOwnership = true;
+
+ if (printPort)
+ {
+ // Calculate per-token ownership of the ring
+ Map<String, Float> ownerships;
+ try
+ {
+ ownerships = probe.effectiveOwnershipWithPort(keyspace);
+ }
+ catch (IllegalStateException ex)
+ {
+ ownerships = probe.getOwnershipWithPort();
+ errors.append("Note: ").append(ex.getMessage()).append("%n");
+ showEffectiveOwnership = false;
+ }
+ catch (IllegalArgumentException ex)
+ {
+ System.out.printf("%nError: %s%n", ex.getMessage());
+ return;
+ }
+
+
+ System.out.println();
+ for (Entry<String, SetHostStatWithPort> entry : NodeTool.getOwnershipByDcWithPort(probe, resolveIp, tokensToEndpoints, ownerships).entrySet())
+ printDc(probe, format, entry.getKey(), endpointsToTokens, entry.getValue(), showEffectiveOwnership);
+
+ if (haveVnodes)
+ {
+ System.out.println(" Warning: \"nodetool ring\" is used to output all the tokens of a node.");
+ System.out.println(" To view status related info of a node use \"nodetool status\" instead.\n");
+ }
+
+ System.out.printf("%n " + errors.toString());
+ }
+ else
+ {
+ // Calculate per-token ownership of the ring
+ Map<InetAddress, Float> ownerships;
+ try
+ {
+ ownerships = probe.effectiveOwnership(keyspace);
+ }
+ catch (IllegalStateException ex)
+ {
+ ownerships = probe.getOwnership();
+ errors.append("Note: ").append(ex.getMessage()).append("%n");
+ showEffectiveOwnership = false;
+ }
+ catch (IllegalArgumentException ex)
+ {
+ System.out.printf("%nError: %s%n", ex.getMessage());
+ return;
+ }
+
+
+ System.out.println();
+ for (Entry<String, SetHostStat> entry : NodeTool.getOwnershipByDc(probe, resolveIp, tokensToEndpoints, ownerships).entrySet())
+ printDc(probe, format, entry.getKey(), endpointsToTokens, entry.getValue(), showEffectiveOwnership);
+
+ if (haveVnodes)
+ {
+ System.out.println(" Warning: \"nodetool ring\" is used to output all the tokens of a node.");
+ System.out.println(" To view status related info of a node use \"nodetool status\" instead.\n");
+ }
+
+ System.out.printf("%n " + errors.toString());
+ }
+ } catch (Exception e)
{
- ownerships = probe.getOwnership();
- errors.append("Note: ").append(ex.getMessage()).append("%n");
- showEffectiveOwnership = false;
+ e.printStackTrace();
+ throw e;
}
- catch (IllegalArgumentException ex)
- {
- System.out.printf("%nError: %s%n", ex.getMessage());
- return;
- }
-
-
- System.out.println();
- for (Entry<String, SetHostStat> entry : NodeTool.getOwnershipByDc(probe, resolveIp, tokensToEndpoints, ownerships).entrySet())
- printDc(probe, format, entry.getKey(), endpointsToTokens, entry.getValue(),showEffectiveOwnership);
-
- if (haveVnodes)
- {
- System.out.println(" Warning: \"nodetool ring\" is used to output all the tokens of a node.");
- System.out.println(" To view status related info of a node use \"nodetool status\" instead.\n");
- }
-
- System.out.printf("%n " + errors.toString());
}
private void printDc(NodeProbe probe, String format,
@@ -111,12 +155,12 @@
LinkedHashMultimap<String, String> endpointsToTokens,
SetHostStat hoststats,boolean showEffectiveOwnership)
{
- Collection<String> liveNodes = probe.getLiveNodes();
- Collection<String> deadNodes = probe.getUnreachableNodes();
- Collection<String> joiningNodes = probe.getJoiningNodes();
- Collection<String> leavingNodes = probe.getLeavingNodes();
- Collection<String> movingNodes = probe.getMovingNodes();
- Map<String, String> loadMap = probe.getLoadMap();
+ Collection<String> liveNodes = probe.getLiveNodes(false);
+ Collection<String> deadNodes = probe.getUnreachableNodes(false);
+ Collection<String> joiningNodes = probe.getJoiningNodes(false);
+ Collection<String> leavingNodes = probe.getLeavingNodes(false);
+ Collection<String> movingNodes = probe.getMovingNodes(false);
+ Map<String, String> loadMap = probe.getLoadMap(false);
System.out.println("Datacenter: " + dc);
System.out.println("==========");
@@ -174,4 +218,73 @@
}
System.out.println();
}
+
+ private void printDc(NodeProbe probe, String format,
+ String dc,
+ LinkedHashMultimap<String, String> endpointsToTokens,
+ SetHostStatWithPort hoststats,boolean showEffectiveOwnership)
+ {
+ Collection<String> liveNodes = probe.getLiveNodes(true);
+ Collection<String> deadNodes = probe.getUnreachableNodes(true);
+ Collection<String> joiningNodes = probe.getJoiningNodes(true);
+ Collection<String> leavingNodes = probe.getLeavingNodes(true);
+ Collection<String> movingNodes = probe.getMovingNodes(true);
+ Map<String, String> loadMap = probe.getLoadMap(true);
+
+ System.out.println("Datacenter: " + dc);
+ System.out.println("==========");
+
+ // get the total amount of replicas for this dc and the last token in this dc's ring
+ List<String> tokens = new ArrayList<>();
+ String lastToken = "";
+
+ for (HostStatWithPort stat : hoststats)
+ {
+ tokens.addAll(endpointsToTokens.get(stat.endpoint.toString()));
+ lastToken = tokens.get(tokens.size() - 1);
+ }
+
+ System.out.printf(format, "Address", "Rack", "Status", "State", "Load", "Owns", "Token");
+
+ if (hoststats.size() > 1)
+ System.out.printf(format, "", "", "", "", "", "", lastToken);
+ else
+ System.out.println();
+
+ for (HostStatWithPort stat : hoststats)
+ {
+ String endpoint = stat.endpoint.toString();
+ String rack;
+ try
+ {
+ rack = probe.getEndpointSnitchInfoProxy().getRack(endpoint);
+ }
+ catch (UnknownHostException e)
+ {
+ rack = "Unknown";
+ }
+
+ String status = liveNodes.contains(endpoint)
+ ? "Up"
+ : deadNodes.contains(endpoint)
+ ? "Down"
+ : "?";
+
+ String state = "Normal";
+
+ if (joiningNodes.contains(endpoint))
+ state = "Joining";
+ else if (leavingNodes.contains(endpoint))
+ state = "Leaving";
+ else if (movingNodes.contains(endpoint))
+ state = "Moving";
+
+ String load = loadMap.containsKey(endpoint)
+ ? loadMap.get(endpoint)
+ : "?";
+ String owns = stat.owns != null && showEffectiveOwnership? new DecimalFormat("##0.00%").format(stat.owns) : "?";
+ System.out.printf(format, stat.ipOrDns(), rack, status, state, load, owns, stat.token);
+ }
+ System.out.println();
+ }
}
diff --git a/src/java/org/apache/cassandra/tools/nodetool/Scrub.java b/src/java/org/apache/cassandra/tools/nodetool/Scrub.java
index 263291d..7c5ff00 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/Scrub.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/Scrub.java
@@ -17,9 +17,9 @@
*/
package org.apache.cassandra.tools.nodetool;
-import io.airlift.command.Arguments;
-import io.airlift.command.Command;
-import io.airlift.command.Option;
+import io.airlift.airline.Arguments;
+import io.airlift.airline.Command;
+import io.airlift.airline.Option;
import java.util.ArrayList;
import java.util.List;
diff --git a/src/java/org/apache/cassandra/tools/nodetool/SetBatchlogReplayThrottle.java b/src/java/org/apache/cassandra/tools/nodetool/SetBatchlogReplayThrottle.java
new file mode 100644
index 0000000..65bb8f5
--- /dev/null
+++ b/src/java/org/apache/cassandra/tools/nodetool/SetBatchlogReplayThrottle.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.tools.nodetool;
+
+import io.airlift.airline.Arguments;
+import io.airlift.airline.Command;
+import org.apache.cassandra.tools.NodeProbe;
+import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
+
+@Command(name = "setbatchlogreplaythrottle", description = "Set batchlog replay throttle in KB per second, or 0 to disable throttling. " +
+ "This will be reduced proportionally to the number of nodes in the cluster.")
+public class SetBatchlogReplayThrottle extends NodeToolCmd
+{
+ @Arguments(title = "batchlog_replay_throttle", usage = "<value_in_kb_per_sec>", description = "Value in KB per second, 0 to disable throttling", required = true)
+ private Integer batchlogReplayThrottle = null;
+
+ @Override
+ public void execute(NodeProbe probe)
+ {
+ probe.setBatchlogReplayThrottle(batchlogReplayThrottle);
+ }
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/tools/nodetool/SetCacheCapacity.java b/src/java/org/apache/cassandra/tools/nodetool/SetCacheCapacity.java
index 6c280d8..461f6ae 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/SetCacheCapacity.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/SetCacheCapacity.java
@@ -18,8 +18,8 @@
package org.apache.cassandra.tools.nodetool;
import static com.google.common.base.Preconditions.checkArgument;
-import io.airlift.command.Arguments;
-import io.airlift.command.Command;
+import io.airlift.airline.Arguments;
+import io.airlift.airline.Command;
import java.util.ArrayList;
import java.util.List;
diff --git a/src/java/org/apache/cassandra/tools/nodetool/SetCacheKeysToSave.java b/src/java/org/apache/cassandra/tools/nodetool/SetCacheKeysToSave.java
index 12a4570..18197e6 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/SetCacheKeysToSave.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/SetCacheKeysToSave.java
@@ -18,8 +18,8 @@
package org.apache.cassandra.tools.nodetool;
import static com.google.common.base.Preconditions.checkArgument;
-import io.airlift.command.Arguments;
-import io.airlift.command.Command;
+import io.airlift.airline.Arguments;
+import io.airlift.airline.Command;
import java.util.ArrayList;
import java.util.List;
diff --git a/src/java/org/apache/cassandra/tools/nodetool/SetCompactionThreshold.java b/src/java/org/apache/cassandra/tools/nodetool/SetCompactionThreshold.java
index 304f2b7..56e558f 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/SetCompactionThreshold.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/SetCompactionThreshold.java
@@ -19,8 +19,8 @@
import static com.google.common.base.Preconditions.checkArgument;
import static java.lang.Integer.parseInt;
-import io.airlift.command.Arguments;
-import io.airlift.command.Command;
+import io.airlift.airline.Arguments;
+import io.airlift.airline.Command;
import java.util.ArrayList;
import java.util.List;
diff --git a/src/java/org/apache/cassandra/tools/nodetool/SetCompactionThroughput.java b/src/java/org/apache/cassandra/tools/nodetool/SetCompactionThroughput.java
index 0111a20..80e7222 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/SetCompactionThroughput.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/SetCompactionThroughput.java
@@ -17,8 +17,8 @@
*/
package org.apache.cassandra.tools.nodetool;
-import io.airlift.command.Arguments;
-import io.airlift.command.Command;
+import io.airlift.airline.Arguments;
+import io.airlift.airline.Command;
import org.apache.cassandra.tools.NodeProbe;
import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
diff --git a/src/java/org/apache/cassandra/tools/nodetool/SetConcurrency.java b/src/java/org/apache/cassandra/tools/nodetool/SetConcurrency.java
new file mode 100644
index 0000000..3ce94e1
--- /dev/null
+++ b/src/java/org/apache/cassandra/tools/nodetool/SetConcurrency.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.tools.nodetool;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import io.airlift.airline.Arguments;
+import io.airlift.airline.Command;
+import org.apache.cassandra.tools.NodeProbe;
+import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+@Command(name = "setconcurrency", description = "Set maximum concurrency for processing stage")
+public class SetConcurrency extends NodeToolCmd
+{
+ @Arguments(title = "<pool-name> <maximum-concurrency> | <stage-name> <core-pool> <maximum-concurrency>",
+ usage = "<stage-name> <maximum-concurrency> | <stage-name> <core-pool> <maximum-concurrency>",
+ description = "Set concurrency for processing stage",
+ required = true)
+ private List<String> args = new ArrayList<>();
+
+ @Override
+ public void execute(NodeProbe probe)
+ {
+ checkArgument(args.size() >= 2 && args.size() <= 3, "setconcurrency requires stage name, optional core pool size and maximum concurrency");
+
+ int corePoolSize = args.size() == 2 ? -1 : Integer.valueOf(args.get(1));
+ int maximumPoolSize = args.size() == 2 ? Integer.valueOf(args.get(1)) : Integer.valueOf(args.get(2));
+
+ checkArgument(args.size() == 2 || corePoolSize >= 0, "Core pool size must be non-negative");
+ checkArgument(maximumPoolSize >= 0, "Maximum pool size must be non-negative");
+
+ try
+ {
+ probe.setConcurrency(args.get(0), corePoolSize, maximumPoolSize);
+ }
+ catch (IllegalArgumentException e)
+ {
+ String message = e.getMessage() != null ? e.getMessage() : "invalid pool size";
+ System.out.println("Unable to set concurrency: " + message);
+ System.exit(1);
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/tools/nodetool/SetConcurrentCompactors.java b/src/java/org/apache/cassandra/tools/nodetool/SetConcurrentCompactors.java
index 56fafe1..5980e99 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/SetConcurrentCompactors.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/SetConcurrentCompactors.java
@@ -19,8 +19,8 @@
package org.apache.cassandra.tools.nodetool;
import static com.google.common.base.Preconditions.checkArgument;
-import io.airlift.command.Arguments;
-import io.airlift.command.Command;
+import io.airlift.airline.Arguments;
+import io.airlift.airline.Command;
import org.apache.cassandra.tools.NodeProbe;
import org.apache.cassandra.tools.NodeTool;
diff --git a/src/java/org/apache/cassandra/tools/nodetool/SetConcurrentViewBuilders.java b/src/java/org/apache/cassandra/tools/nodetool/SetConcurrentViewBuilders.java
new file mode 100644
index 0000000..96adf2c
--- /dev/null
+++ b/src/java/org/apache/cassandra/tools/nodetool/SetConcurrentViewBuilders.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.tools.nodetool;
+
+import io.airlift.airline.Arguments;
+import io.airlift.airline.Command;
+import org.apache.cassandra.tools.NodeProbe;
+import org.apache.cassandra.tools.NodeTool;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+@Command(name = "setconcurrentviewbuilders", description = "Set the number of concurrent view builders in the system")
+public class SetConcurrentViewBuilders extends NodeTool.NodeToolCmd
+{
+ @Arguments(title = "concurrent_view_builders", usage = "<value>", description = "Number of concurrent view builders, greater than 0.", required = true)
+ private Integer concurrentViewBuilders = null;
+
+ protected void execute(NodeProbe probe)
+ {
+ checkArgument(concurrentViewBuilders > 0, "concurrent_view_builders should be great than 0.");
+ probe.setConcurrentViewBuilders(concurrentViewBuilders);
+ }
+}
diff --git a/src/java/org/apache/cassandra/tools/nodetool/SetHintedHandoffThrottleInKB.java b/src/java/org/apache/cassandra/tools/nodetool/SetHintedHandoffThrottleInKB.java
index d20ff3f..feb945b 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/SetHintedHandoffThrottleInKB.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/SetHintedHandoffThrottleInKB.java
@@ -17,8 +17,8 @@
*/
package org.apache.cassandra.tools.nodetool;
-import io.airlift.command.Arguments;
-import io.airlift.command.Command;
+import io.airlift.airline.Arguments;
+import io.airlift.airline.Command;
import org.apache.cassandra.tools.NodeProbe;
import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
diff --git a/src/java/org/apache/cassandra/tools/nodetool/SetHostStatWithPort.java b/src/java/org/apache/cassandra/tools/nodetool/SetHostStatWithPort.java
new file mode 100644
index 0000000..6ac0258
--- /dev/null
+++ b/src/java/org/apache/cassandra/tools/nodetool/SetHostStatWithPort.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.tools.nodetool;
+
+import java.net.UnknownHostException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.cassandra.locator.InetAddressAndPort;
+
+public class SetHostStatWithPort implements Iterable<HostStatWithPort>
+{
+ final List<HostStatWithPort> hostStats = new ArrayList<>();
+ final boolean resolveIp;
+
+ public SetHostStatWithPort(boolean resolveIp)
+ {
+ this.resolveIp = resolveIp;
+ }
+
+ public int size()
+ {
+ return hostStats.size();
+ }
+
+ @Override
+ public Iterator<HostStatWithPort> iterator()
+ {
+ return hostStats.iterator();
+ }
+
+ public void add(String token, String host, Map<String, Float> ownerships) throws UnknownHostException
+ {
+ InetAddressAndPort endpoint = InetAddressAndPort.getByName(host);
+ Float owns = ownerships.get(endpoint.toString());
+ hostStats.add(new HostStatWithPort(token, endpoint, resolveIp, owns));
+ }
+}
diff --git a/src/java/org/apache/cassandra/tools/nodetool/SetInterDCStreamThroughput.java b/src/java/org/apache/cassandra/tools/nodetool/SetInterDCStreamThroughput.java
index 41ce43a..e2e606c 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/SetInterDCStreamThroughput.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/SetInterDCStreamThroughput.java
@@ -17,8 +17,8 @@
*/
package org.apache.cassandra.tools.nodetool;
-import io.airlift.command.Arguments;
-import io.airlift.command.Command;
+import io.airlift.airline.Arguments;
+import io.airlift.airline.Command;
import org.apache.cassandra.tools.NodeProbe;
import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
diff --git a/src/java/org/apache/cassandra/tools/nodetool/SetLoggingLevel.java b/src/java/org/apache/cassandra/tools/nodetool/SetLoggingLevel.java
index d11d48f..8d9ad90 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/SetLoggingLevel.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/SetLoggingLevel.java
@@ -18,26 +18,86 @@
package org.apache.cassandra.tools.nodetool;
import static org.apache.commons.lang3.StringUtils.EMPTY;
-import io.airlift.command.Arguments;
-import io.airlift.command.Command;
+import io.airlift.airline.Arguments;
+import io.airlift.airline.Command;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
+import com.google.common.collect.Lists;
+
import org.apache.cassandra.tools.NodeProbe;
import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
-@Command(name = "setlogginglevel", description = "Set the log level threshold for a given class. If both class and level are empty/null, it will reset to the initial configuration")
+@Command(name = "setlogginglevel", description = "Set the log level threshold for a given component or class. Will reset to the initial configuration if called with no parameters.")
public class SetLoggingLevel extends NodeToolCmd
{
- @Arguments(usage = "<class> <level>", description = "The class to change the level for and the log level threshold to set (can be empty)")
+ @Arguments(usage = "<component|class> <level>", description = "The component or class to change the level for and the log level threshold to set. Will reset to initial level if omitted. "
+ + "Available components: bootstrap, compaction, repair, streaming, cql, ring")
private List<String> args = new ArrayList<>();
@Override
public void execute(NodeProbe probe)
{
- String classQualifier = args.size() >= 1 ? args.get(0) : EMPTY;
+ String target = args.size() >= 1 ? args.get(0) : EMPTY;
String level = args.size() == 2 ? args.get(1) : EMPTY;
- probe.setLoggingLevel(classQualifier, level);
+
+ List<String> classQualifiers = Collections.singletonList(target);
+ if (target.equals("bootstrap"))
+ {
+ classQualifiers = Lists.newArrayList(
+ "org.apache.cassandra.gms",
+ "org.apache.cassandra.hints",
+ "org.apache.cassandra.schema",
+ "org.apache.cassandra.service.StorageService",
+ "org.apache.cassandra.db.SystemKeyspace",
+ "org.apache.cassandra.batchlog.BatchlogManager",
+ "org.apache.cassandra.net.MessagingService");
+ }
+ else if (target.equals("repair"))
+ {
+ classQualifiers = Lists.newArrayList(
+ "org.apache.cassandra.repair",
+ "org.apache.cassandra.db.compaction.CompactionManager",
+ "org.apache.cassandra.service.SnapshotVerbHandler");
+ }
+ else if (target.equals("streaming"))
+ {
+ classQualifiers = Lists.newArrayList(
+ "org.apache.cassandra.streaming",
+ "org.apache.cassandra.dht.RangeStreamer");
+ }
+ else if (target.equals("compaction"))
+ {
+ classQualifiers = Lists.newArrayList(
+ "org.apache.cassandra.db.compaction",
+ "org.apache.cassandra.db.ColumnFamilyStore",
+ "org.apache.cassandra.io.sstable.IndexSummaryRedistribution");
+ }
+ else if (target.equals("cql"))
+ {
+ classQualifiers = Lists.newArrayList(
+ "org.apache.cassandra.cql3",
+ "org.apache.cassandra.auth",
+ "org.apache.cassandra.batchlog",
+ "org.apache.cassandra.net.ResponseVerbHandler",
+ "org.apache.cassandra.service.AbstractReadExecutor",
+ "org.apache.cassandra.service.AbstractWriteResponseHandler",
+ "org.apache.cassandra.service.paxos",
+ "org.apache.cassandra.service.ReadCallback",
+ "org.apache.cassandra.service.ResponseResolver");
+ }
+ else if (target.equals("ring"))
+ {
+ classQualifiers = Lists.newArrayList(
+ "org.apache.cassandra.gms",
+ "org.apache.cassandra.service.PendingRangeCalculatorService",
+ "org.apache.cassandra.service.LoadBroadcaster",
+ "org.apache.cassandra.transport.Server");
+ }
+
+ for (String classQualifier : classQualifiers)
+ probe.setLoggingLevel(classQualifier, level);
}
}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/tools/nodetool/SetMaxHintWindow.java b/src/java/org/apache/cassandra/tools/nodetool/SetMaxHintWindow.java
new file mode 100644
index 0000000..ace1a49
--- /dev/null
+++ b/src/java/org/apache/cassandra/tools/nodetool/SetMaxHintWindow.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.tools.nodetool;
+
+import io.airlift.airline.Arguments;
+import io.airlift.airline.Command;
+import org.apache.cassandra.tools.NodeProbe;
+import org.apache.cassandra.tools.NodeTool;
+
+@Command(name = "setmaxhintwindow", description = "Set the specified max hint window in ms")
+public class SetMaxHintWindow extends NodeTool.NodeToolCmd
+{
+ @Arguments(title = "max_hint_window", usage = "<value_in_ms>", description = "Value of maxhintwindow in ms", required = true)
+ private Integer maxHintWindow = null;
+
+ @Override
+ public void execute(NodeProbe probe)
+ {
+ probe.setMaxHintWindow(maxHintWindow);
+ }
+}
diff --git a/src/java/org/apache/cassandra/tools/nodetool/SetStreamThroughput.java b/src/java/org/apache/cassandra/tools/nodetool/SetStreamThroughput.java
index 8055872..069a6e9 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/SetStreamThroughput.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/SetStreamThroughput.java
@@ -17,8 +17,8 @@
*/
package org.apache.cassandra.tools.nodetool;
-import io.airlift.command.Arguments;
-import io.airlift.command.Command;
+import io.airlift.airline.Arguments;
+import io.airlift.airline.Command;
import org.apache.cassandra.tools.NodeProbe;
import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
diff --git a/src/java/org/apache/cassandra/tools/nodetool/SetTimeout.java b/src/java/org/apache/cassandra/tools/nodetool/SetTimeout.java
index 0b99efd..06b859b 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/SetTimeout.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/SetTimeout.java
@@ -17,8 +17,8 @@
*/
package org.apache.cassandra.tools.nodetool;
-import io.airlift.command.Arguments;
-import io.airlift.command.Command;
+import io.airlift.airline.Arguments;
+import io.airlift.airline.Command;
import java.util.ArrayList;
import java.util.List;
diff --git a/src/java/org/apache/cassandra/tools/nodetool/SetTraceProbability.java b/src/java/org/apache/cassandra/tools/nodetool/SetTraceProbability.java
index ebef1a4..e081980 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/SetTraceProbability.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/SetTraceProbability.java
@@ -18,8 +18,8 @@
package org.apache.cassandra.tools.nodetool;
import static com.google.common.base.Preconditions.checkArgument;
-import io.airlift.command.Arguments;
-import io.airlift.command.Command;
+import io.airlift.airline.Arguments;
+import io.airlift.airline.Command;
import org.apache.cassandra.tools.NodeProbe;
import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
diff --git a/src/java/org/apache/cassandra/tools/nodetool/Sjk.java b/src/java/org/apache/cassandra/tools/nodetool/Sjk.java
new file mode 100644
index 0000000..f394d38
--- /dev/null
+++ b/src/java/org/apache/cassandra/tools/nodetool/Sjk.java
@@ -0,0 +1,485 @@
+
+package org.apache.cassandra.tools.nodetool;
+
+import java.io.File;
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.net.URL;
+import java.net.URLDecoder;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Enumeration;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.jar.JarEntry;
+import java.util.jar.JarFile;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import javax.management.MBeanServerConnection;
+
+import com.beust.jcommander.JCommander;
+import com.beust.jcommander.ParameterDescription;
+import com.beust.jcommander.Parameterized;
+import io.airlift.airline.Arguments;
+import io.airlift.airline.Command;
+import org.gridkit.jvmtool.JmxConnectionInfo;
+import org.gridkit.jvmtool.cli.CommandLauncher;
+
+import org.apache.cassandra.tools.NodeProbe;
+import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
+
+@Command(name = "sjk", description = "Run commands of 'Swiss Java Knife'. Run 'nodetool sjk --help' for more information.")
+public class Sjk extends NodeToolCmd
+{
+ @Arguments(description = "Arguments passed as is to 'Swiss Java Knife'.")
+ private List<String> args;
+
+ private final Wrapper wrapper = new Wrapper();
+
+ public void run()
+ {
+ wrapper.prepare(args != null ? args.toArray(new String[0]) : new String[]{"help"});
+
+ if (!wrapper.requiresMbeanServerConn())
+ {
+ // SJK command does not require an MBeanServerConnection, so just invoke it
+ wrapper.run(null);
+ }
+ else
+ {
+ // invoke common nodetool handling to establish MBeanServerConnection
+ super.run();
+ }
+ }
+
+ public void sequenceRun(NodeProbe probe)
+ {
+ wrapper.prepare(args != null ? args.toArray(new String[0]) : new String[]{"help"});
+ if (!wrapper.run(probe))
+ probe.failed();
+ }
+
+ protected void execute(NodeProbe probe)
+ {
+ if (!wrapper.run(probe))
+ probe.failed();
+ }
+
+ /**
+ * Adopted copy of {@link org.gridkit.jvmtool.SJK} from <a href="https://github.com/aragozin/jvm-tools">https://github.com/aragozin/jvm-tools</a>.
+ */
+ public static class Wrapper extends CommandLauncher
+ {
+ boolean suppressSystemExit;
+
+ private final Map<String, Runnable> commands = new HashMap<>();
+
+ private JCommander parser;
+
+ private Runnable cmd;
+
+ public void suppressSystemExit()
+ {
+ suppressSystemExit = true;
+ super.suppressSystemExit();
+ }
+
+ public boolean start(String[] args)
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ public void prepare(String[] args)
+ {
+ try
+ {
+
+ parser = new JCommander(this);
+
+ addCommands();
+
+ fixCommands();
+
+ try
+ {
+ parser.parse(args);
+ }
+ catch (Exception e)
+ {
+ failAndPrintUsage(e.toString());
+ }
+
+ if (isHelp())
+ {
+ String cmd = parser.getParsedCommand();
+ if (cmd == null)
+ {
+ parser.usage();
+ }
+ else
+ {
+ parser.usage(cmd);
+ }
+ }
+ else if (isListCommands())
+ {
+ for (String cmd : commands.keySet())
+ {
+ System.out.println(String.format("%8s - %s", cmd, parser.getCommandDescription(cmd)));
+ }
+ }
+ else
+ {
+
+ cmd = commands.get(parser.getParsedCommand());
+
+ if (cmd == null)
+ {
+ failAndPrintUsage();
+ }
+ }
+ }
+ catch (CommandAbortedError error)
+ {
+ for (String m : error.messages)
+ {
+ logError(m);
+ }
+ if (isVerbose() && error.getCause() != null)
+ {
+ logTrace(error.getCause());
+ }
+ if (error.printUsage && parser != null)
+ {
+ if (parser.getParsedCommand() != null)
+ {
+ parser.usage(parser.getParsedCommand());
+ }
+ else
+ {
+ parser.usage();
+ }
+ }
+ }
+ catch (Throwable e)
+ {
+ e.printStackTrace();
+ }
+ }
+
+ public boolean run(final NodeProbe probe)
+ {
+ try
+ {
+ setJmxConnInfo(probe);
+
+ if (cmd != null)
+ cmd.run();
+
+ return true;
+ }
+ catch (CommandAbortedError error)
+ {
+ for (String m : error.messages)
+ {
+ logError(m);
+ }
+ if (isVerbose() && error.getCause() != null)
+ {
+ logTrace(error.getCause());
+ }
+ if (error.printUsage && parser != null)
+ {
+ if (parser.getParsedCommand() != null)
+ {
+ parser.usage(parser.getParsedCommand());
+ }
+ else
+ {
+ parser.usage();
+ }
+ }
+ return true;
+ }
+ catch (Throwable e)
+ {
+ e.printStackTrace();
+ }
+
+ // abnormal termination
+ return false;
+ }
+
+ private void setJmxConnInfo(final NodeProbe probe) throws IllegalAccessException
+ {
+ Field f = jmxConnectionInfoField(cmd);
+ if (f != null)
+ {
+ f.setAccessible(true);
+ f.set(cmd, new JmxConnectionInfo(this)
+ {
+ public MBeanServerConnection getMServer()
+ {
+ return probe.getMbeanServerConn();
+ }
+ });
+ }
+ f = pidField(cmd);
+ if (f != null)
+ {
+ long pid = probe.getPid();
+
+ f.setAccessible(true);
+ if (f.getType() == int.class)
+ f.setInt(cmd, (int) pid);
+ if (f.getType() == long.class)
+ f.setLong(cmd, pid);
+ if (f.getType() == String.class)
+ f.set(cmd, Long.toString(pid));
+ }
+ }
+
+ private boolean isHelp()
+ {
+ try
+ {
+ Field f = CommandLauncher.class.getDeclaredField("help");
+ f.setAccessible(true);
+ return f.getBoolean(this);
+ }
+ catch (Exception e)
+ {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private boolean isListCommands()
+ {
+ try
+ {
+ Field f = CommandLauncher.class.getDeclaredField("listCommands");
+ f.setAccessible(true);
+ return f.getBoolean(this);
+ }
+ catch (Exception e)
+ {
+ throw new RuntimeException(e);
+ }
+ }
+
+ protected List<String> getCommandPackages()
+ {
+ return Collections.singletonList("org.gridkit.jvmtool.cmd");
+ }
+
+ private void addCommands() throws InstantiationException, IllegalAccessException
+ {
+ for (String pack : getCommandPackages())
+ {
+ for (Class<?> c : findClasses(pack))
+ {
+ if (CommandLauncher.CmdRef.class.isAssignableFrom(c))
+ {
+ CommandLauncher.CmdRef cmd = (CommandLauncher.CmdRef) c.newInstance();
+ String cmdName = cmd.getCommandName();
+ Runnable cmdTask = cmd.newCommand(this);
+ if (commands.containsKey(cmdName))
+ {
+ fail("Ambiguous implementation for '" + cmdName + '\'');
+ }
+ commands.put(cmdName, cmdTask);
+ parser.addCommand(cmdName, cmdTask);
+ }
+ }
+ }
+ }
+
+ private void fixCommands() throws Exception
+ {
+ List<Field> fields = Arrays.asList(JCommander.class.getDeclaredField("m_fields"),
+ JCommander.class.getDeclaredField("m_requiredFields"));
+ for (Field f : fields)
+ f.setAccessible(true);
+
+ for (JCommander cmdr : parser.getCommands().values())
+ {
+ for (Field field : fields) {
+ Map<Parameterized, ParameterDescription> fieldsMap = (Map<Parameterized, ParameterDescription>) field.get(cmdr);
+ for (Iterator<Map.Entry<Parameterized, ParameterDescription>> iPar = fieldsMap.entrySet().iterator(); iPar.hasNext(); )
+ {
+ Map.Entry<Parameterized, ParameterDescription> par = iPar.next();
+ switch (par.getKey().getName())
+ {
+ // JmxConnectionInfo fields
+ case "pid":
+ case "sockAddr":
+ case "user":
+ case "password":
+ //
+ case "verbose":
+ case "help":
+ case "listCommands":
+ iPar.remove();
+ break;
+ }
+ }
+ }
+ }
+ }
+
+ boolean requiresMbeanServerConn()
+ {
+ return jmxConnectionInfoField(cmd) != null || pidField(cmd) != null;
+ }
+
+ private static Field jmxConnectionInfoField(Runnable cmd)
+ {
+ if (cmd == null)
+ return null;
+
+ for (Field f : cmd.getClass().getDeclaredFields())
+ {
+ if (f.getType() == JmxConnectionInfo.class)
+ {
+ return f;
+ }
+ }
+ return null;
+ }
+
+ private static Field pidField(Runnable cmd)
+ {
+ if (cmd == null)
+ return null;
+
+ for (Field f : cmd.getClass().getDeclaredFields())
+ {
+ if ("pid".equals(f.getName()) &&
+ (f.getType() == int.class || f.getType() == long.class || f.getType() == String.class))
+ {
+ return f;
+ }
+ }
+ return null;
+ }
+
+ private static List<Class<?>> findClasses(String packageName)
+ {
+ // TODO this will probably fail with JPMS/Jigsaw
+
+ List<Class<?>> result = new ArrayList<>();
+ try
+ {
+ String path = packageName.replace('.', '/');
+ for (String f : findFiles(path))
+ {
+ if (f.endsWith(".class") && f.indexOf('$') < 0)
+ {
+ f = f.substring(0, f.length() - ".class".length());
+ f = f.replace('/', '.');
+ result.add(Class.forName(f));
+ }
+ }
+ return result;
+ }
+ catch (Exception e)
+ {
+ throw new RuntimeException(e);
+ }
+ }
+
+ static List<String> findFiles(String path) throws IOException
+ {
+ List<String> result = new ArrayList<>();
+ ClassLoader cl = Thread.currentThread().getContextClassLoader();
+ Enumeration<URL> en = cl.getResources(path);
+ while (en.hasMoreElements())
+ {
+ URL u = en.nextElement();
+ listFiles(result, u, path);
+ }
+ return result;
+ }
+
+ static void listFiles(List<String> results, URL packageURL, String path) throws IOException
+ {
+ if (packageURL.getProtocol().equals("jar"))
+ {
+ String jarFileName;
+ Enumeration<JarEntry> jarEntries;
+ String entryName;
+
+ // build jar file name, then loop through zipped entries
+ jarFileName = URLDecoder.decode(packageURL.getFile(), "UTF-8");
+ jarFileName = jarFileName.substring(5, jarFileName.indexOf('!'));
+ try (JarFile jf = new JarFile(jarFileName))
+ {
+ jarEntries = jf.entries();
+ while (jarEntries.hasMoreElements())
+ {
+ entryName = jarEntries.nextElement().getName();
+ if (entryName.startsWith(path))
+ {
+ results.add(entryName);
+ }
+ }
+ }
+ }
+ else
+ {
+ // loop through files in classpath
+ File dir = new File(packageURL.getFile());
+ String cp = dir.getCanonicalPath();
+ File root = dir;
+ while (true)
+ {
+ if (cp.equals(new File(root, path).getCanonicalPath()))
+ {
+ break;
+ }
+ root = root.getParentFile();
+ }
+ listFiles(results, root, dir);
+ }
+ }
+
+ static void listFiles(List<String> names, File root, File dir)
+ {
+ String rootPath = root.getAbsolutePath();
+ if (dir.exists() && dir.isDirectory())
+ {
+ for (File file : dir.listFiles())
+ {
+ if (file.isDirectory())
+ {
+ listFiles(names, root, file);
+ }
+ else
+ {
+ String name = file.getAbsolutePath().substring(rootPath.length() + 1);
+ name = name.replace('\\', '/');
+ names.add(name);
+ }
+ }
+ }
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/tools/nodetool/Snapshot.java b/src/java/org/apache/cassandra/tools/nodetool/Snapshot.java
index 8941ec1..495ee9d 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/Snapshot.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/Snapshot.java
@@ -19,9 +19,9 @@
import static com.google.common.collect.Iterables.toArray;
import static org.apache.commons.lang3.StringUtils.join;
-import io.airlift.command.Arguments;
-import io.airlift.command.Command;
-import io.airlift.command.Option;
+import io.airlift.airline.Arguments;
+import io.airlift.airline.Command;
+import io.airlift.airline.Option;
import java.io.IOException;
import java.util.ArrayList;
@@ -71,7 +71,7 @@
else
{
throw new IOException(
- "When specifying the Keyspace columfamily list for a snapshot, you should not specify columnfamily");
+ "When specifying the Keyspace table list (using -kt,--kt-list,-kc,--kc.list), you must not also specify keyspaces to snapshot");
}
if (!snapshotName.isEmpty())
sb.append(" with snapshot name [").append(snapshotName).append("]");
diff --git a/src/java/org/apache/cassandra/tools/nodetool/Status.java b/src/java/org/apache/cassandra/tools/nodetool/Status.java
index 24c5c74..fe11335 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/Status.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/Status.java
@@ -17,27 +17,29 @@
*/
package org.apache.cassandra.tools.nodetool;
-import io.airlift.command.Arguments;
-import io.airlift.command.Command;
-import io.airlift.command.Option;
+import io.airlift.airline.Arguments;
+import io.airlift.airline.Command;
+import io.airlift.airline.Option;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.text.DecimalFormat;
import java.util.Collection;
-import java.util.HashSet;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
-import java.util.Set;
import java.util.SortedMap;
import org.apache.cassandra.locator.EndpointSnitchInfoMBean;
+import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.tools.NodeProbe;
import org.apache.cassandra.tools.NodeTool;
import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
+import org.apache.cassandra.tools.nodetool.formatter.TableBuilder;
import com.google.common.collect.ArrayListMultimap;
+@SuppressWarnings("UseOfSystemOutOrSystemErr")
@Command(name = "status", description = "Print cluster information (state, load, IDs, ...)")
public class Status extends NodeToolCmd
{
@@ -48,7 +50,6 @@
private boolean resolveIp = false;
private boolean isTokenPerNode = true;
- private String format = null;
private Collection<String> joiningNodes, leavingNodes, movingNodes, liveNodes, unreachableNodes;
private Map<String, String> loadMap, hostIDMap;
private EndpointSnitchInfoMBean epSnitchInfo;
@@ -56,28 +57,29 @@
@Override
public void execute(NodeProbe probe)
{
- joiningNodes = probe.getJoiningNodes();
- leavingNodes = probe.getLeavingNodes();
- movingNodes = probe.getMovingNodes();
- loadMap = probe.getLoadMap();
- Map<String, String> tokensToEndpoints = probe.getTokenToEndpointMap();
- liveNodes = probe.getLiveNodes();
- unreachableNodes = probe.getUnreachableNodes();
- hostIDMap = probe.getHostIdMap();
+ joiningNodes = probe.getJoiningNodes(printPort);
+ leavingNodes = probe.getLeavingNodes(printPort);
+ movingNodes = probe.getMovingNodes(printPort);
+ loadMap = probe.getLoadMap(printPort);
+ Map<String, String> tokensToEndpoints = probe.getTokenToEndpointMap(printPort);
+ liveNodes = probe.getLiveNodes(printPort);
+ unreachableNodes = probe.getUnreachableNodes(printPort);
+ hostIDMap = probe.getHostIdMap(printPort);
epSnitchInfo = probe.getEndpointSnitchInfoProxy();
StringBuilder errors = new StringBuilder();
+ TableBuilder.SharedTable sharedTable = new TableBuilder.SharedTable(" ");
- Map<InetAddress, Float> ownerships = null;
+ Map<String, Float> ownerships = null;
boolean hasEffectiveOwns = false;
try
{
- ownerships = probe.effectiveOwnership(keyspace);
+ ownerships = probe.effectiveOwnershipWithPort(keyspace);
hasEffectiveOwns = true;
}
catch (IllegalStateException e)
{
- ownerships = probe.getOwnership();
+ ownerships = probe.getOwnershipWithPort();
errors.append("Note: ").append(e.getMessage()).append("%n");
}
catch (IllegalArgumentException ex)
@@ -86,17 +88,39 @@
System.exit(1);
}
- SortedMap<String, SetHostStat> dcs = NodeTool.getOwnershipByDc(probe, resolveIp, tokensToEndpoints, ownerships);
+ SortedMap<String, SetHostStatWithPort> dcs = NodeTool.getOwnershipByDcWithPort(probe, resolveIp, tokensToEndpoints, ownerships);
// More tokens than nodes (aka vnodes)?
- if (dcs.values().size() < tokensToEndpoints.keySet().size())
+ if (dcs.size() < tokensToEndpoints.size())
isTokenPerNode = false;
- int maxAddressLength = computeMaxAddressLength(dcs);
-
// Datacenters
- for (Map.Entry<String, SetHostStat> dc : dcs.entrySet())
+ for (Map.Entry<String, SetHostStatWithPort> dc : dcs.entrySet())
{
+ TableBuilder tableBuilder = sharedTable.next();
+ addNodesHeader(hasEffectiveOwns, tableBuilder);
+
+ ArrayListMultimap<String, HostStatWithPort> hostToTokens = ArrayListMultimap.create();
+ for (HostStatWithPort stat : dc.getValue())
+ hostToTokens.put(stat.ipOrDns(printPort), stat);
+
+ for (String endpoint : hostToTokens.keySet())
+ {
+ Float owns = ownerships.get(endpoint);
+ List<HostStatWithPort> tokens = hostToTokens.get(endpoint);
+ addNode(endpoint, owns, tokens.get(0).ipOrDns(printPort), tokens.get(0).token, tokens.size(),
+ hasEffectiveOwns, tableBuilder);
+ }
+ }
+
+ Iterator<TableBuilder> results = sharedTable.complete().iterator();
+ boolean first = true;
+ for (Map.Entry<String, SetHostStatWithPort> dc : dcs.entrySet())
+ {
+ if (!first) {
+ System.out.println();
+ }
+ first = false;
String dcHeader = String.format("Datacenter: %s%n", dc.getKey());
System.out.print(dcHeader);
for (int i = 0; i < (dcHeader.length() - 1); i++) System.out.print('=');
@@ -105,54 +129,27 @@
// Legend
System.out.println("Status=Up/Down");
System.out.println("|/ State=Normal/Leaving/Joining/Moving");
-
- printNodesHeader(hasEffectiveOwns, isTokenPerNode, maxAddressLength);
-
- ArrayListMultimap<InetAddress, HostStat> hostToTokens = ArrayListMultimap.create();
- for (HostStat stat : dc.getValue())
- hostToTokens.put(stat.endpoint, stat);
-
- for (InetAddress endpoint : hostToTokens.keySet())
- {
- Float owns = ownerships.get(endpoint);
- List<HostStat> tokens = hostToTokens.get(endpoint);
- printNode(endpoint.getHostAddress(), owns, tokens, hasEffectiveOwns, isTokenPerNode, maxAddressLength);
- }
+ TableBuilder dcTable = results.next();
+ dcTable.printTo(System.out);
}
System.out.printf("%n" + errors);
-
}
- private int computeMaxAddressLength(Map<String, SetHostStat> dcs)
+ private void addNodesHeader(boolean hasEffectiveOwns, TableBuilder tableBuilder)
{
- int maxAddressLength = 0;
-
- Set<InetAddress> seenHosts = new HashSet<>();
- for (SetHostStat stats : dcs.values())
- for (HostStat stat : stats)
- if (seenHosts.add(stat.endpoint))
- maxAddressLength = Math.max(maxAddressLength, stat.ipOrDns().length());
-
- return maxAddressLength;
- }
-
- private void printNodesHeader(boolean hasEffectiveOwns, boolean isTokenPerNode, int maxAddressLength)
- {
- String fmt = getFormat(hasEffectiveOwns, isTokenPerNode, maxAddressLength);
String owns = hasEffectiveOwns ? "Owns (effective)" : "Owns";
if (isTokenPerNode)
- System.out.printf(fmt, "-", "-", "Address", "Load", owns, "Host ID", "Token", "Rack");
+ tableBuilder.add("--", "Address", "Load", owns, "Host ID", "Token", "Rack");
else
- System.out.printf(fmt, "-", "-", "Address", "Load", "Tokens", owns, "Host ID", "Rack");
+ tableBuilder.add("--", "Address", "Load", "Tokens", owns, "Host ID", "Rack");
}
- private void printNode(String endpoint, Float owns, List<HostStat> tokens, boolean hasEffectiveOwns,
- boolean isTokenPerNode, int maxAddressLength)
+ private void addNode(String endpoint, Float owns, String epDns, String token, int size, boolean hasEffectiveOwns,
+ TableBuilder tableBuilder)
{
- String status, state, load, strOwns, hostID, rack, fmt;
- fmt = getFormat(hasEffectiveOwns, isTokenPerNode, maxAddressLength);
+ String status, state, load, strOwns, hostID, rack;
if (liveNodes.contains(endpoint)) status = "U";
else if (unreachableNodes.contains(endpoint)) status = "D";
else status = "?";
@@ -161,7 +158,8 @@
else if (movingNodes.contains(endpoint)) state = "M";
else state = "N";
- load = loadMap.containsKey(endpoint) ? loadMap.get(endpoint) : "?";
+ String statusAndState = status.concat(state);
+ load = loadMap.getOrDefault(endpoint, "?");
strOwns = owns != null && hasEffectiveOwns ? new DecimalFormat("##0.0%").format(owns) : "?";
hostID = hostIDMap.get(endpoint);
@@ -173,36 +171,14 @@
throw new RuntimeException(e);
}
- String endpointDns = tokens.get(0).ipOrDns();
if (isTokenPerNode)
- System.out.printf(fmt, status, state, endpointDns, load, strOwns, hostID, tokens.get(0).token, rack);
- else
- System.out.printf(fmt, status, state, endpointDns, load, tokens.size(), strOwns, hostID, rack);
- }
-
- private String getFormat(boolean hasEffectiveOwns, boolean isTokenPerNode, int maxAddressLength)
- {
- if (format == null)
{
- StringBuilder buf = new StringBuilder();
- String addressPlaceholder = String.format("%%-%ds ", maxAddressLength);
- buf.append("%s%s "); // status
- buf.append(addressPlaceholder); // address
- buf.append("%-9s "); // load
- if (!isTokenPerNode)
- buf.append("%-11s "); // "Tokens"
- if (hasEffectiveOwns)
- buf.append("%-16s "); // "Owns (effective)"
- else
- buf.append("%-6s "); // "Owns
- buf.append("%-36s "); // Host ID
- if (isTokenPerNode)
- buf.append("%-39s "); // token
- buf.append("%s%n"); // "Rack"
-
- format = buf.toString();
+ tableBuilder.add(statusAndState, epDns, load, strOwns, hostID, token, rack);
}
-
- return format;
+ else
+ {
+ tableBuilder.add(statusAndState, epDns, load, String.valueOf(size), strOwns, hostID, rack);
+ }
}
+
}
diff --git a/src/java/org/apache/cassandra/tools/nodetool/StatusAutoCompaction.java b/src/java/org/apache/cassandra/tools/nodetool/StatusAutoCompaction.java
new file mode 100644
index 0000000..4322bb8
--- /dev/null
+++ b/src/java/org/apache/cassandra/tools/nodetool/StatusAutoCompaction.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.tools.nodetool;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import io.airlift.airline.Arguments;
+import io.airlift.airline.Command;
+import io.airlift.airline.Option;
+import org.apache.cassandra.tools.NodeProbe;
+import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
+import org.apache.cassandra.tools.nodetool.formatter.TableBuilder;
+
+@Command(name = "statusautocompaction", description = "status of autocompaction of the given keyspace and table")
+public class StatusAutoCompaction extends NodeToolCmd
+{
+ @Arguments(usage = "[<keyspace> <tables>...]", description = "The keyspace followed by one or many tables")
+ private List<String> args = new ArrayList<>();
+
+ @Option(title = "show_all", name = { "-a", "--all" }, description = "Show auto compaction status for each keyspace/table")
+ private boolean showAll = false;
+
+ @Override
+ public void execute(NodeProbe probe)
+ {
+ List<String> keyspaces = parseOptionalKeyspace(args, probe);
+ String[] tableNames = parseOptionalTables(args);
+
+ boolean allDisabled = true;
+ boolean allEnabled = true;
+ TableBuilder table = new TableBuilder();
+ table.add("Keyspace", "Table", "Status");
+ try
+ {
+ for (String keyspace : keyspaces)
+ {
+ Map<String, Boolean> statuses = probe.getAutoCompactionDisabled(keyspace, tableNames);
+ for (Map.Entry<String, Boolean> status : statuses.entrySet())
+ {
+ String tableName = status.getKey();
+ boolean disabled = status.getValue();
+ allDisabled &= disabled;
+ allEnabled &= !disabled;
+ table.add(keyspace, tableName, !disabled ? "running" : "not running");
+ }
+ }
+ if (showAll)
+ table.printTo(System.out);
+ else
+ System.out.println(allEnabled ? "running" :
+ allDisabled ? "not running" : "partially running");
+ }
+ catch (IOException e)
+ {
+ throw new RuntimeException("Error occurred during status-auto-compaction", e);
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/tools/nodetool/StatusBackup.java b/src/java/org/apache/cassandra/tools/nodetool/StatusBackup.java
index 49a6750..84fa8a5 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/StatusBackup.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/StatusBackup.java
@@ -17,7 +17,7 @@
*/
package org.apache.cassandra.tools.nodetool;
-import io.airlift.command.Command;
+import io.airlift.airline.Command;
import org.apache.cassandra.tools.NodeProbe;
import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
diff --git a/src/java/org/apache/cassandra/tools/nodetool/StatusBinary.java b/src/java/org/apache/cassandra/tools/nodetool/StatusBinary.java
index d4fae14..45fe6c3 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/StatusBinary.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/StatusBinary.java
@@ -17,7 +17,7 @@
*/
package org.apache.cassandra.tools.nodetool;
-import io.airlift.command.Command;
+import io.airlift.airline.Command;
import org.apache.cassandra.tools.NodeProbe;
import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
diff --git a/src/java/org/apache/cassandra/tools/nodetool/StatusGossip.java b/src/java/org/apache/cassandra/tools/nodetool/StatusGossip.java
index e40df8d..b6a1164 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/StatusGossip.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/StatusGossip.java
@@ -17,7 +17,7 @@
*/
package org.apache.cassandra.tools.nodetool;
-import io.airlift.command.Command;
+import io.airlift.airline.Command;
import org.apache.cassandra.tools.NodeProbe;
import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
diff --git a/src/java/org/apache/cassandra/tools/nodetool/StatusHandoff.java b/src/java/org/apache/cassandra/tools/nodetool/StatusHandoff.java
index 65f6729..bee161d 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/StatusHandoff.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/StatusHandoff.java
@@ -17,7 +17,7 @@
*/
package org.apache.cassandra.tools.nodetool;
-import io.airlift.command.Command;
+import io.airlift.airline.Command;
import org.apache.cassandra.tools.NodeProbe;
import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
diff --git a/src/java/org/apache/cassandra/tools/nodetool/StatusThrift.java b/src/java/org/apache/cassandra/tools/nodetool/StatusThrift.java
deleted file mode 100644
index 0cb17d2..0000000
--- a/src/java/org/apache/cassandra/tools/nodetool/StatusThrift.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.tools.nodetool;
-
-import io.airlift.command.Command;
-
-import org.apache.cassandra.tools.NodeProbe;
-import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
-
-@Command(name = "statusthrift", description = "Status of thrift server")
-public class StatusThrift extends NodeToolCmd
-{
- @Override
- public void execute(NodeProbe probe)
- {
- System.out.println(
- probe.isThriftServerRunning()
- ? "running"
- : "not running");
- }
-}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/tools/nodetool/Stop.java b/src/java/org/apache/cassandra/tools/nodetool/Stop.java
index 7d785ab..aa05d75 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/Stop.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/Stop.java
@@ -17,9 +17,9 @@
*/
package org.apache.cassandra.tools.nodetool;
-import io.airlift.command.Arguments;
-import io.airlift.command.Command;
-import io.airlift.command.Option;
+import io.airlift.airline.Arguments;
+import io.airlift.airline.Command;
+import io.airlift.airline.Option;
import org.apache.cassandra.db.compaction.OperationType;
import org.apache.cassandra.tools.NodeProbe;
diff --git a/src/java/org/apache/cassandra/tools/nodetool/StopDaemon.java b/src/java/org/apache/cassandra/tools/nodetool/StopDaemon.java
index 24c8920..6e453da 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/StopDaemon.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/StopDaemon.java
@@ -17,7 +17,7 @@
*/
package org.apache.cassandra.tools.nodetool;
-import io.airlift.command.Command;
+import io.airlift.airline.Command;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.tools.NodeProbe;
diff --git a/src/java/org/apache/cassandra/tools/nodetool/TableHistograms.java b/src/java/org/apache/cassandra/tools/nodetool/TableHistograms.java
index 8f4ffa6..cb3b946 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/TableHistograms.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/TableHistograms.java
@@ -19,122 +19,154 @@
import static com.google.common.base.Preconditions.checkArgument;
import static java.lang.String.format;
-import io.airlift.command.Arguments;
-import io.airlift.command.Command;
+import io.airlift.airline.Arguments;
+import io.airlift.airline.Command;
import java.util.ArrayList;
+import java.util.Iterator;
import java.util.List;
+import java.util.Map;
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.Multimap;
+
+import org.apache.cassandra.db.ColumnFamilyStoreMBean;
import org.apache.cassandra.metrics.CassandraMetricsRegistry;
import org.apache.cassandra.tools.NodeProbe;
import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
import org.apache.cassandra.utils.EstimatedHistogram;
+
import org.apache.commons.lang3.ArrayUtils;
@Command(name = "tablehistograms", description = "Print statistic histograms for a given table")
public class TableHistograms extends NodeToolCmd
{
- @Arguments(usage = "<keyspace> <table> | <keyspace.table>", description = "The keyspace and table name")
+ @Arguments(usage = "[<keyspace> <table> | <keyspace.table>]", description = "The keyspace and table name")
private List<String> args = new ArrayList<>();
@Override
public void execute(NodeProbe probe)
{
- String keyspace = null, table = null;
+ Multimap<String, String> tablesList = HashMultimap.create();
+
+ // a <keyspace, set<table>> mapping for verification or as reference if none provided
+ Multimap<String, String> allTables = HashMultimap.create();
+ Iterator<Map.Entry<String, ColumnFamilyStoreMBean>> tableMBeans = probe.getColumnFamilyStoreMBeanProxies();
+ while (tableMBeans.hasNext())
+ {
+ Map.Entry<String, ColumnFamilyStoreMBean> entry = tableMBeans.next();
+ allTables.put(entry.getKey(), entry.getValue().getTableName());
+ }
+
if (args.size() == 2)
{
- keyspace = args.get(0);
- table = args.get(1);
+ tablesList.put(args.get(0), args.get(1));
}
else if (args.size() == 1)
{
String[] input = args.get(0).split("\\.");
checkArgument(input.length == 2, "tablehistograms requires keyspace and table name arguments");
- keyspace = input[0];
- table = input[1];
+ tablesList.put(input[0], input[1]);
}
else
{
- checkArgument(false, "tablehistograms requires keyspace and table name arguments");
+ // use all tables
+ tablesList = allTables;
}
- // calculate percentile of row size and column count
- long[] estimatedPartitionSize = (long[]) probe.getColumnFamilyMetric(keyspace, table, "EstimatedPartitionSizeHistogram");
- long[] estimatedColumnCount = (long[]) probe.getColumnFamilyMetric(keyspace, table, "EstimatedColumnCountHistogram");
-
- // build arrays to store percentile values
- double[] estimatedRowSizePercentiles = new double[7];
- double[] estimatedColumnCountPercentiles = new double[7];
- double[] offsetPercentiles = new double[]{0.5, 0.75, 0.95, 0.98, 0.99};
-
- if (ArrayUtils.isEmpty(estimatedPartitionSize) || ArrayUtils.isEmpty(estimatedColumnCount))
+ // verify that all tables to list exist
+ for (String keyspace : tablesList.keys())
{
- System.err.println("No SSTables exists, unable to calculate 'Partition Size' and 'Cell Count' percentiles");
-
- for (int i = 0; i < 7; i++)
+ for (String table : tablesList.get(keyspace))
{
- estimatedRowSizePercentiles[i] = Double.NaN;
- estimatedColumnCountPercentiles[i] = Double.NaN;
+ if (!allTables.containsEntry(keyspace, table))
+ throw new IllegalArgumentException("Unknown table " + keyspace + '.' + table);
}
}
- else
- {
- EstimatedHistogram partitionSizeHist = new EstimatedHistogram(estimatedPartitionSize);
- EstimatedHistogram columnCountHist = new EstimatedHistogram(estimatedColumnCount);
- if (partitionSizeHist.isOverflowed())
+ for (String keyspace : tablesList.keys())
+ {
+ for (String table : tablesList.get(keyspace))
{
- System.err.println(String.format("Row sizes are larger than %s, unable to calculate percentiles", partitionSizeHist.getLargestBucketOffset()));
- for (int i = 0; i < offsetPercentiles.length; i++)
+ // calculate percentile of row size and column count
+ long[] estimatedPartitionSize = (long[]) probe.getColumnFamilyMetric(keyspace, table, "EstimatedPartitionSizeHistogram");
+ long[] estimatedColumnCount = (long[]) probe.getColumnFamilyMetric(keyspace, table, "EstimatedColumnCountHistogram");
+
+ // build arrays to store percentile values
+ double[] estimatedRowSizePercentiles = new double[7];
+ double[] estimatedColumnCountPercentiles = new double[7];
+ double[] offsetPercentiles = new double[]{0.5, 0.75, 0.95, 0.98, 0.99};
+
+ if (ArrayUtils.isEmpty(estimatedPartitionSize) || ArrayUtils.isEmpty(estimatedColumnCount))
+ {
+ System.out.println("No SSTables exists, unable to calculate 'Partition Size' and 'Cell Count' percentiles");
+
+ for (int i = 0; i < 7; i++)
+ {
estimatedRowSizePercentiles[i] = Double.NaN;
- }
- else
- {
- for (int i = 0; i < offsetPercentiles.length; i++)
- estimatedRowSizePercentiles[i] = partitionSizeHist.percentile(offsetPercentiles[i]);
- }
+ estimatedColumnCountPercentiles[i] = Double.NaN;
+ }
+ }
+ else
+ {
+ EstimatedHistogram partitionSizeHist = new EstimatedHistogram(estimatedPartitionSize);
+ EstimatedHistogram columnCountHist = new EstimatedHistogram(estimatedColumnCount);
- if (columnCountHist.isOverflowed())
- {
- System.err.println(String.format("Column counts are larger than %s, unable to calculate percentiles", columnCountHist.getLargestBucketOffset()));
- for (int i = 0; i < estimatedColumnCountPercentiles.length; i++)
- estimatedColumnCountPercentiles[i] = Double.NaN;
- }
- else
- {
- for (int i = 0; i < offsetPercentiles.length; i++)
- estimatedColumnCountPercentiles[i] = columnCountHist.percentile(offsetPercentiles[i]);
- }
+ if (partitionSizeHist.isOverflowed())
+ {
+ System.out.println(String.format("Row sizes are larger than %s, unable to calculate percentiles", partitionSizeHist.getLargestBucketOffset()));
+ for (int i = 0; i < offsetPercentiles.length; i++)
+ estimatedRowSizePercentiles[i] = Double.NaN;
+ }
+ else
+ {
+ for (int i = 0; i < offsetPercentiles.length; i++)
+ estimatedRowSizePercentiles[i] = partitionSizeHist.percentile(offsetPercentiles[i]);
+ }
- // min value
- estimatedRowSizePercentiles[5] = partitionSizeHist.min();
- estimatedColumnCountPercentiles[5] = columnCountHist.min();
- // max value
- estimatedRowSizePercentiles[6] = partitionSizeHist.max();
- estimatedColumnCountPercentiles[6] = columnCountHist.max();
+ if (columnCountHist.isOverflowed())
+ {
+ System.out.println(String.format("Column counts are larger than %s, unable to calculate percentiles", columnCountHist.getLargestBucketOffset()));
+ for (int i = 0; i < estimatedColumnCountPercentiles.length; i++)
+ estimatedColumnCountPercentiles[i] = Double.NaN;
+ }
+ else
+ {
+ for (int i = 0; i < offsetPercentiles.length; i++)
+ estimatedColumnCountPercentiles[i] = columnCountHist.percentile(offsetPercentiles[i]);
+ }
+
+ // min value
+ estimatedRowSizePercentiles[5] = partitionSizeHist.min();
+ estimatedColumnCountPercentiles[5] = columnCountHist.min();
+ // max value
+ estimatedRowSizePercentiles[6] = partitionSizeHist.max();
+ estimatedColumnCountPercentiles[6] = columnCountHist.max();
+ }
+
+ String[] percentiles = new String[]{"50%", "75%", "95%", "98%", "99%", "Min", "Max"};
+ double[] readLatency = probe.metricPercentilesAsArray((CassandraMetricsRegistry.JmxTimerMBean) probe.getColumnFamilyMetric(keyspace, table, "ReadLatency"));
+ double[] writeLatency = probe.metricPercentilesAsArray((CassandraMetricsRegistry.JmxTimerMBean) probe.getColumnFamilyMetric(keyspace, table, "WriteLatency"));
+ double[] sstablesPerRead = probe.metricPercentilesAsArray((CassandraMetricsRegistry.JmxHistogramMBean) probe.getColumnFamilyMetric(keyspace, table, "SSTablesPerReadHistogram"));
+
+ System.out.println(format("%s/%s histograms", keyspace, table));
+ System.out.println(format("%-10s%18s%18s%18s%18s%18s",
+ "Percentile", "Read Latency", "Write Latency", "SSTables", "Partition Size", "Cell Count"));
+ System.out.println(format("%-10s%18s%18s%18s%18s%18s",
+ "", "(micros)", "(micros)", "", "(bytes)", ""));
+
+ for (int i = 0; i < percentiles.length; i++)
+ {
+ System.out.println(format("%-10s%18.2f%18.2f%18.2f%18.0f%18.0f",
+ percentiles[i],
+ readLatency[i],
+ writeLatency[i],
+ sstablesPerRead[i],
+ estimatedRowSizePercentiles[i],
+ estimatedColumnCountPercentiles[i]));
+ }
+ System.out.println();
+ }
}
-
- String[] percentiles = new String[]{"50%", "75%", "95%", "98%", "99%", "Min", "Max"};
- double[] readLatency = probe.metricPercentilesAsArray((CassandraMetricsRegistry.JmxTimerMBean) probe.getColumnFamilyMetric(keyspace, table, "ReadLatency"));
- double[] writeLatency = probe.metricPercentilesAsArray((CassandraMetricsRegistry.JmxTimerMBean) probe.getColumnFamilyMetric(keyspace, table, "WriteLatency"));
- double[] sstablesPerRead = probe.metricPercentilesAsArray((CassandraMetricsRegistry.JmxHistogramMBean) probe.getColumnFamilyMetric(keyspace, table, "SSTablesPerReadHistogram"));
-
- System.out.println(format("%s/%s histograms", keyspace, table));
- System.out.println(format("%-10s%10s%18s%18s%18s%18s",
- "Percentile", "SSTables", "Write Latency", "Read Latency", "Partition Size", "Cell Count"));
- System.out.println(format("%-10s%10s%18s%18s%18s%18s",
- "", "", "(micros)", "(micros)", "(bytes)", ""));
-
- for (int i = 0; i < percentiles.length; i++)
- {
- System.out.println(format("%-10s%10.2f%18.2f%18.2f%18.0f%18.0f",
- percentiles[i],
- sstablesPerRead[i],
- writeLatency[i],
- readLatency[i],
- estimatedRowSizePercentiles[i],
- estimatedColumnCountPercentiles[i]));
- }
- System.out.println();
}
}
diff --git a/src/java/org/apache/cassandra/tools/nodetool/TableStats.java b/src/java/org/apache/cassandra/tools/nodetool/TableStats.java
index b67f7c4..5f5651b 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/TableStats.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/TableStats.java
@@ -18,9 +18,10 @@
package org.apache.cassandra.tools.nodetool;
import java.util.*;
-import io.airlift.command.Arguments;
-import io.airlift.command.Command;
-import io.airlift.command.Option;
+
+import io.airlift.airline.Arguments;
+import io.airlift.airline.Command;
+import io.airlift.airline.Option;
import org.apache.cassandra.tools.NodeProbe;
import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
@@ -45,6 +46,28 @@
description = "Output format (json, yaml)")
private String outputFormat = "";
+ @Option(title = "sort_key",
+ name = {"-s", "--sort"},
+ description = "Sort tables by specified sort key (average_live_cells_per_slice_last_five_minutes, "
+ + "average_tombstones_per_slice_last_five_minutes, bloom_filter_false_positives, "
+ + "bloom_filter_false_ratio, bloom_filter_off_heap_memory_used, bloom_filter_space_used, "
+ + "compacted_partition_maximum_bytes, compacted_partition_mean_bytes, "
+ + "compacted_partition_minimum_bytes, compression_metadata_off_heap_memory_used, "
+ + "dropped_mutations, full_name, index_summary_off_heap_memory_used, local_read_count, "
+ + "local_read_latency_ms, local_write_latency_ms, "
+ + "maximum_live_cells_per_slice_last_five_minutes, "
+ + "maximum_tombstones_per_slice_last_five_minutes, memtable_cell_count, memtable_data_size, "
+ + "memtable_off_heap_memory_used, memtable_switch_count, number_of_partitions_estimate, "
+ + "off_heap_memory_used_total, pending_flushes, percent_repaired, read_latency, reads, "
+ + "space_used_by_snapshots_total, space_used_live, space_used_total, "
+ + "sstable_compression_ratio, sstable_count, table_name, write_latency, writes)")
+ private String sortKey = "";
+
+ @Option(title = "top",
+ name = {"-t", "--top"},
+ description = "Show only the top K tables for the sort key (specify the number K of tables to be shown")
+ private int top = 0;
+
@Override
public void execute(NodeProbe probe)
{
@@ -53,9 +76,25 @@
throw new IllegalArgumentException("arguments for -F are json,yaml only.");
}
- StatsHolder holder = new TableStatsHolder(probe, humanReadable, ignore, tableNames);
+ if (!sortKey.isEmpty() && !Arrays.asList(StatsTableComparator.supportedSortKeys).contains(sortKey))
+ {
+ throw new IllegalArgumentException(String.format("argument for sort must be one of: %s",
+ String.join(", ", StatsTableComparator.supportedSortKeys)));
+ }
+
+ if (top > 0 && sortKey.isEmpty())
+ {
+ throw new IllegalArgumentException("cannot filter top K tables without specifying a sort key.");
+ }
+
+ if (top < 0)
+ {
+ throw new IllegalArgumentException("argument for top must be a positive integer.");
+ }
+
+ StatsHolder holder = new TableStatsHolder(probe, humanReadable, ignore, tableNames, sortKey, top);
// print out the keyspace and table statistics
- StatsPrinter printer = TableStatsPrinter.from(outputFormat);
+ StatsPrinter printer = TableStatsPrinter.from(outputFormat, !sortKey.isEmpty());
printer.print(holder, System.out);
}
diff --git a/src/java/org/apache/cassandra/tools/nodetool/TopPartitions.java b/src/java/org/apache/cassandra/tools/nodetool/TopPartitions.java
index 931ac62..66ad574 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/TopPartitions.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/TopPartitions.java
@@ -17,101 +17,10 @@
*/
package org.apache.cassandra.tools.nodetool;
-import static com.google.common.base.Preconditions.checkArgument;
-import static org.apache.commons.lang3.StringUtils.join;
-import io.airlift.command.Arguments;
-import io.airlift.command.Command;
-import io.airlift.command.Option;
+import io.airlift.airline.Command;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-
-import javax.management.openmbean.CompositeData;
-import javax.management.openmbean.OpenDataException;
-import javax.management.openmbean.TabularDataSupport;
-
-import org.apache.cassandra.metrics.TableMetrics;
-import org.apache.cassandra.metrics.TableMetrics.Sampler;
-import org.apache.cassandra.tools.NodeProbe;
-import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
-
-import com.google.common.collect.Lists;
-import com.google.common.collect.Ordering;
-
-@Command(name = "toppartitions", description = "Sample and print the most active partitions for a given column family")
-public class TopPartitions extends NodeToolCmd
+@Command(name = "toppartitions", description = "Sample and print the most active partitions")
+@Deprecated
+public class TopPartitions extends ProfileLoad
{
- @Arguments(usage = "<keyspace> <cfname> <duration>", description = "The keyspace, column family name, and duration in milliseconds")
- private List<String> args = new ArrayList<>();
- @Option(name = "-s", description = "Capacity of stream summary, closer to the actual cardinality of partitions will yield more accurate results (Default: 256)")
- private int size = 256;
- @Option(name = "-k", description = "Number of the top partitions to list (Default: 10)")
- private int topCount = 10;
- @Option(name = "-a", description = "Comma separated list of samplers to use (Default: all)")
- private String samplers = join(TableMetrics.Sampler.values(), ',');
- @Override
- public void execute(NodeProbe probe)
- {
- checkArgument(args.size() == 3, "toppartitions requires keyspace, column family name, and duration");
- checkArgument(topCount < size, "TopK count (-k) option must be smaller then the summary capacity (-s)");
- String keyspace = args.get(0);
- String cfname = args.get(1);
- Integer duration = Integer.valueOf(args.get(2));
- // generate the list of samplers
- List<Sampler> targets = Lists.newArrayList();
- for (String s : samplers.split(","))
- {
- try
- {
- targets.add(Sampler.valueOf(s.toUpperCase()));
- } catch (Exception e)
- {
- throw new IllegalArgumentException(s + " is not a valid sampler, choose one of: " + join(Sampler.values(), ", "));
- }
- }
-
- Map<Sampler, CompositeData> results;
- try
- {
- results = probe.getPartitionSample(keyspace, cfname, size, duration, topCount, targets);
- } catch (OpenDataException e)
- {
- throw new RuntimeException(e);
- }
- boolean first = true;
- for(Entry<Sampler, CompositeData> result : results.entrySet())
- {
- CompositeData sampling = result.getValue();
- // weird casting for http://bugs.sun.com/view_bug.do?bug_id=6548436
- List<CompositeData> topk = (List<CompositeData>) (Object) Lists.newArrayList(((TabularDataSupport) sampling.get("partitions")).values());
- Collections.sort(topk, new Ordering<CompositeData>()
- {
- public int compare(CompositeData left, CompositeData right)
- {
- return Long.compare((long) right.get("count"), (long) left.get("count"));
- }
- });
- if(!first)
- System.out.println();
- System.out.println(result.getKey().toString()+ " Sampler:");
- System.out.printf(" Cardinality: ~%d (%d capacity)%n", sampling.get("cardinality"), size);
- System.out.printf(" Top %d partitions:%n", topCount);
- if (topk.size() == 0)
- {
- System.out.println("\tNothing recorded during sampling period...");
- } else
- {
- int offset = 0;
- for (CompositeData entry : topk)
- offset = Math.max(offset, entry.get("string").toString().length());
- System.out.printf("\t%-" + offset + "s%10s%10s%n", "Partition", "Count", "+/-");
- for (CompositeData entry : topk)
- System.out.printf("\t%-" + offset + "s%10d%10d%n", entry.get("string").toString(), entry.get("count"), entry.get("error"));
- }
- first = false;
- }
- }
-}
\ No newline at end of file
+}
diff --git a/src/java/org/apache/cassandra/tools/nodetool/TpStats.java b/src/java/org/apache/cassandra/tools/nodetool/TpStats.java
index 0cf8e50..307c78d 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/TpStats.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/TpStats.java
@@ -17,9 +17,11 @@
*/
package org.apache.cassandra.tools.nodetool;
-import io.airlift.command.Command;
+import javax.management.MBeanServerConnection;
-import io.airlift.command.Option;
+import io.airlift.airline.Command;
+
+import io.airlift.airline.Option;
import org.apache.cassandra.tools.NodeProbe;
import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
import org.apache.cassandra.tools.nodetool.stats.TpStatsHolder;
diff --git a/src/java/org/apache/cassandra/tools/nodetool/TruncateHints.java b/src/java/org/apache/cassandra/tools/nodetool/TruncateHints.java
index bcd554f..a3a0049 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/TruncateHints.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/TruncateHints.java
@@ -18,8 +18,8 @@
package org.apache.cassandra.tools.nodetool;
import static org.apache.commons.lang3.StringUtils.EMPTY;
-import io.airlift.command.Arguments;
-import io.airlift.command.Command;
+import io.airlift.airline.Arguments;
+import io.airlift.airline.Command;
import org.apache.cassandra.tools.NodeProbe;
import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
diff --git a/src/java/org/apache/cassandra/tools/nodetool/UpgradeSSTable.java b/src/java/org/apache/cassandra/tools/nodetool/UpgradeSSTable.java
index 82866e0..c957c8c 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/UpgradeSSTable.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/UpgradeSSTable.java
@@ -17,9 +17,9 @@
*/
package org.apache.cassandra.tools.nodetool;
-import io.airlift.command.Arguments;
-import io.airlift.command.Command;
-import io.airlift.command.Option;
+import io.airlift.airline.Arguments;
+import io.airlift.airline.Command;
+import io.airlift.airline.Option;
import java.util.ArrayList;
import java.util.List;
diff --git a/src/java/org/apache/cassandra/tools/nodetool/Verify.java b/src/java/org/apache/cassandra/tools/nodetool/Verify.java
index c449366..28b91fd 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/Verify.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/Verify.java
@@ -17,9 +17,9 @@
*/
package org.apache.cassandra.tools.nodetool;
-import io.airlift.command.Arguments;
-import io.airlift.command.Command;
-import io.airlift.command.Option;
+import io.airlift.airline.Arguments;
+import io.airlift.airline.Command;
+import io.airlift.airline.Option;
import java.util.ArrayList;
import java.util.List;
@@ -34,21 +34,52 @@
private List<String> args = new ArrayList<>();
@Option(title = "extended_verify",
- name = {"-e", "--extended-verify"},
- description = "Verify each cell data, beyond simply checking sstable checksums")
+ name = {"-e", "--extended-verify"},
+ description = "Verify each cell data, beyond simply checking sstable checksums")
private boolean extendedVerify = false;
+ @Option(title = "check_version",
+ name = {"-c", "--check-version"},
+ description = "Also check that all sstables are the latest version")
+ private boolean checkVersion = false;
+
+ @Option(title = "dfp",
+ name = {"-d", "--dfp"},
+ description = "Invoke the disk failure policy if a corrupt sstable is found")
+ private boolean diskFailurePolicy = false;
+
+ @Option(title = "repair_status_change",
+ name = {"-r", "--rsc"},
+ description = "Mutate the repair status on corrupt sstables")
+ private boolean mutateRepairStatus = false;
+
+ @Option(title = "check_owns_tokens",
+ name = {"-t", "--check-tokens"},
+ description = "Verify that all tokens in sstables are owned by this node")
+ private boolean checkOwnsTokens = false;
+
+ @Option(title = "quick",
+ name = {"-q", "--quick"},
+ description = "Do a quick check - avoid reading all data to verify checksums")
+ private boolean quick = false;
+
@Override
public void execute(NodeProbe probe)
{
List<String> keyspaces = parseOptionalKeyspace(args, probe);
String[] tableNames = parseOptionalTables(args);
+ if (checkOwnsTokens && !extendedVerify)
+ {
+ System.out.println("Token verification requires --extended-verify");
+ System.exit(1);
+ }
+
for (String keyspace : keyspaces)
{
try
{
- probe.verify(System.out, extendedVerify, keyspace, tableNames);
+ probe.verify(System.out, extendedVerify, checkVersion, diskFailurePolicy, mutateRepairStatus, checkOwnsTokens, quick, keyspace, tableNames);
} catch (Exception e)
{
throw new RuntimeException("Error occurred during verifying", e);
diff --git a/src/java/org/apache/cassandra/tools/nodetool/Version.java b/src/java/org/apache/cassandra/tools/nodetool/Version.java
index 2495508..395a247 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/Version.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/Version.java
@@ -17,7 +17,7 @@
*/
package org.apache.cassandra.tools.nodetool;
-import io.airlift.command.Command;
+import io.airlift.airline.Command;
import org.apache.cassandra.tools.NodeProbe;
import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
diff --git a/src/java/org/apache/cassandra/tools/nodetool/ViewBuildStatus.java b/src/java/org/apache/cassandra/tools/nodetool/ViewBuildStatus.java
index 0696396..b432b68 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/ViewBuildStatus.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/ViewBuildStatus.java
@@ -22,8 +22,8 @@
import java.util.List;
import java.util.Map;
-import io.airlift.command.Arguments;
-import io.airlift.command.Command;
+import io.airlift.airline.Arguments;
+import io.airlift.airline.Command;
import org.apache.cassandra.tools.NodeProbe;
import org.apache.cassandra.tools.NodeTool;
import org.apache.cassandra.tools.nodetool.formatter.TableBuilder;
diff --git a/src/java/org/apache/cassandra/tools/nodetool/formatter/TableBuilder.java b/src/java/org/apache/cassandra/tools/nodetool/formatter/TableBuilder.java
index a56e52e..2557a7d 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/formatter/TableBuilder.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/formatter/TableBuilder.java
@@ -20,8 +20,11 @@
import java.io.PrintStream;
import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
import java.util.List;
import java.util.Objects;
+import java.util.stream.Collectors;
import javax.annotation.Nonnull;
/**
@@ -41,8 +44,8 @@
*/
public class TableBuilder
{
- // column delimiter char
- private final char columnDelimiter;
+ // column delimiter
+ private final String columnDelimiter;
private int[] maximumColumnWidth;
private final List<String[]> rows = new ArrayList<>();
@@ -54,9 +57,21 @@
public TableBuilder(char columnDelimiter)
{
+ this(String.valueOf(columnDelimiter));
+ }
+
+ public TableBuilder(String columnDelimiter)
+ {
this.columnDelimiter = columnDelimiter;
}
+ private TableBuilder(TableBuilder base, int[] maximumColumnWidth)
+ {
+ this(base.columnDelimiter);
+ this.maximumColumnWidth = maximumColumnWidth;
+ this.rows.addAll(base.rows);
+ }
+
public void add(@Nonnull String... row)
{
Objects.requireNonNull(row);
@@ -100,4 +115,56 @@
out.println();
}
}
+
+ /**
+ * Share max offsets across multiple TableBuilders
+ */
+ public static class SharedTable {
+ private List<TableBuilder> tables = new ArrayList<>();
+ private final String columnDelimiter;
+
+ public SharedTable()
+ {
+ this(' ');
+ }
+
+ public SharedTable(char columnDelimiter)
+ {
+ this(String.valueOf(columnDelimiter));
+ }
+
+ public SharedTable(String columnDelimiter)
+ {
+ this.columnDelimiter = columnDelimiter;
+ }
+
+ public TableBuilder next()
+ {
+ TableBuilder next = new TableBuilder(columnDelimiter);
+ tables.add(next);
+ return next;
+ }
+
+ public List<TableBuilder> complete()
+ {
+ if (tables.size() == 0)
+ return Collections.emptyList();
+
+ final int columns = tables.stream()
+ .max(Comparator.comparing(tb -> tb.maximumColumnWidth.length))
+ .get().maximumColumnWidth.length;
+
+ final int[] maximumColumnWidth = new int[columns];
+ for (TableBuilder tb : tables)
+ {
+ for (int i = 0; i < tb.maximumColumnWidth.length; i++)
+ {
+ maximumColumnWidth[i] = Math.max(tb.maximumColumnWidth[i], maximumColumnWidth[i]);
+ }
+ }
+ return tables.stream()
+ .map(tb -> new TableBuilder(tb, maximumColumnWidth))
+ .collect(Collectors.toList());
+ }
+ }
}
diff --git a/src/java/org/apache/cassandra/tools/nodetool/stats/StatsTable.java b/src/java/org/apache/cassandra/tools/nodetool/stats/StatsTable.java
index 87bc527..01d2164 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/stats/StatsTable.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/stats/StatsTable.java
@@ -23,10 +23,13 @@
public class StatsTable
{
- public String name;
+ public String fullName;
+ public String keyspaceName;
+ public String tableName;
public boolean isIndex;
public boolean isLeveledSstable = false;
public Object sstableCount;
+ public Object oldSSTableCount;
public String spaceUsedLive;
public String spaceUsedTotal;
public String spaceUsedBySnapshotsTotal;
@@ -57,6 +60,9 @@
public long compactedPartitionMaximumBytes;
public long compactedPartitionMeanBytes;
public double percentRepaired;
+ public long bytesRepaired;
+ public long bytesUnrepaired;
+ public long bytesPendingRepair;
public double averageLiveCellsPerSliceLastFiveMinutes;
public long maximumLiveCellsPerSliceLastFiveMinutes;
public double averageTombstonesPerSliceLastFiveMinutes;
diff --git a/src/java/org/apache/cassandra/tools/nodetool/stats/StatsTableComparator.java b/src/java/org/apache/cassandra/tools/nodetool/stats/StatsTableComparator.java
new file mode 100644
index 0000000..065d89e
--- /dev/null
+++ b/src/java/org/apache/cassandra/tools/nodetool/stats/StatsTableComparator.java
@@ -0,0 +1,336 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.tools.nodetool.stats;
+
+import java.util.Comparator;
+
+import org.apache.cassandra.io.util.FileUtils;
+
+/**
+ * Comparator to sort StatsTables by a named statistic.
+ */
+public class StatsTableComparator implements Comparator
+{
+
+ /**
+ * Name of the stat that will be used as the sort key.
+ */
+ private final String sortKey;
+
+ /**
+ * Whether data size stats are printed human readable.
+ */
+ private final boolean humanReadable;
+
+ /**
+ * Whether sorting should be done ascending.
+ */
+ private final boolean ascending;
+
+ /**
+ * Names of supported sort keys as they should be specified on the command line.
+ */
+ public static final String[] supportedSortKeys = { "average_live_cells_per_slice_last_five_minutes",
+ "average_tombstones_per_slice_last_five_minutes",
+ "bloom_filter_false_positives", "bloom_filter_false_ratio",
+ "bloom_filter_off_heap_memory_used", "bloom_filter_space_used",
+ "compacted_partition_maximum_bytes",
+ "compacted_partition_mean_bytes",
+ "compacted_partition_minimum_bytes",
+ "compression_metadata_off_heap_memory_used", "dropped_mutations",
+ "full_name", "index_summary_off_heap_memory_used",
+ "local_read_count", "local_read_latency_ms",
+ "local_write_latency_ms",
+ "maximum_live_cells_per_slice_last_five_minutes",
+ "maximum_tombstones_per_slice_last_five_minutes",
+ "memtable_cell_count", "memtable_data_size",
+ "memtable_off_heap_memory_used", "memtable_switch_count",
+ "number_of_partitions_estimate", "off_heap_memory_used_total",
+ "pending_flushes", "percent_repaired", "read_latency", "reads",
+ "space_used_by_snapshots_total", "space_used_live",
+ "space_used_total", "sstable_compression_ratio", "sstable_count",
+ "table_name", "write_latency", "writes" };
+
+ public StatsTableComparator(String sortKey, boolean humanReadable)
+ {
+ this(sortKey, humanReadable, false);
+ }
+
+ public StatsTableComparator(String sortKey, boolean humanReadable, boolean ascending)
+ {
+ this.sortKey = sortKey;
+ this.humanReadable = humanReadable;
+ this.ascending = ascending;
+ }
+
+ /**
+ * Compare stats represented as doubles
+ */
+ private int compareDoubles(double x, double y)
+ {
+ int sign = ascending ? 1 : -1;
+ if (Double.isNaN(x) && !Double.isNaN(y))
+ return sign * Double.valueOf(0D).compareTo(Double.valueOf(y));
+ else if (!Double.isNaN(x) && Double.isNaN(y))
+ return sign * Double.valueOf(x).compareTo(Double.valueOf(0D));
+ else if (Double.isNaN(x) && Double.isNaN(y))
+ return 0;
+ else
+ return sign * Double.valueOf(x).compareTo(Double.valueOf(y));
+ }
+
+ /**
+ * Compare file size stats represented as strings
+ */
+ private int compareFileSizes(String x, String y)
+ {
+ int sign = ascending ? 1 : -1;
+ if (null == x && null != y)
+ return sign * -1;
+ else if (null != x && null == y)
+ return sign;
+ else if (null == x && null == y)
+ return 0;
+ long sizeX = humanReadable ? FileUtils.parseFileSize(x) : Long.valueOf(x);
+ long sizeY = humanReadable ? FileUtils.parseFileSize(y) : Long.valueOf(y);
+ return sign * Long.compare(sizeX, sizeY);
+ }
+
+ /**
+ * Compare StatsTable instances based on this instance's sortKey.
+ */
+ public int compare(Object x, Object y)
+ {
+ if (!(x instanceof StatsTable && y instanceof StatsTable))
+ throw new ClassCastException(String.format("StatsTableComparator cannot compare %s and %s",
+ x.getClass().toString(), y.getClass().toString()));
+ if (x == null || y == null)
+ throw new NullPointerException("StatsTableComparator cannot compare null objects");
+ StatsTable stx = (StatsTable) x;
+ StatsTable sty = (StatsTable) y;
+ int sign = ascending ? 1 : -1;
+ int result = 0;
+ if (sortKey.equals("average_live_cells_per_slice_last_five_minutes"))
+ {
+ result = compareDoubles(stx.averageLiveCellsPerSliceLastFiveMinutes,
+ sty.averageLiveCellsPerSliceLastFiveMinutes);
+ }
+ else if (sortKey.equals("average_tombstones_per_slice_last_five_minutes"))
+ {
+ result = compareDoubles(stx.averageTombstonesPerSliceLastFiveMinutes,
+ sty.averageTombstonesPerSliceLastFiveMinutes);
+ }
+ else if (sortKey.equals("bloom_filter_false_positives"))
+ {
+ result = sign * ((Long) stx.bloomFilterFalsePositives)
+ .compareTo((Long) sty.bloomFilterFalsePositives);
+ }
+ else if (sortKey.equals("bloom_filter_false_ratio"))
+ {
+ result = compareDoubles((Double) stx.bloomFilterFalseRatio,
+ (Double) sty.bloomFilterFalseRatio);
+ }
+ else if (sortKey.equals("bloom_filter_off_heap_memory_used"))
+ {
+ if (stx.bloomFilterOffHeapUsed && !sty.bloomFilterOffHeapUsed)
+ return sign;
+ else if (!stx.bloomFilterOffHeapUsed && sty.bloomFilterOffHeapUsed)
+ return sign * -1;
+ else if (!stx.bloomFilterOffHeapUsed && !sty.bloomFilterOffHeapUsed)
+ result = 0;
+ else
+ {
+ result = compareFileSizes(stx.bloomFilterOffHeapMemoryUsed,
+ sty.bloomFilterOffHeapMemoryUsed);
+ }
+ }
+ else if (sortKey.equals("bloom_filter_space_used"))
+ {
+ result = compareFileSizes(stx.bloomFilterSpaceUsed,
+ sty.bloomFilterSpaceUsed);
+ }
+ else if (sortKey.equals("compacted_partition_maximum_bytes"))
+ {
+ result = sign * Long.valueOf(stx.compactedPartitionMaximumBytes)
+ .compareTo(Long.valueOf(sty.compactedPartitionMaximumBytes));
+ }
+ else if (sortKey.equals("compacted_partition_mean_bytes"))
+ {
+ result = sign * Long.valueOf(stx.compactedPartitionMeanBytes)
+ .compareTo(Long.valueOf(sty.compactedPartitionMeanBytes));
+ }
+ else if (sortKey.equals("compacted_partition_minimum_bytes"))
+ {
+ result = sign * Long.valueOf(stx.compactedPartitionMinimumBytes)
+ .compareTo(Long.valueOf(sty.compactedPartitionMinimumBytes));
+ }
+ else if (sortKey.equals("compression_metadata_off_heap_memory_used"))
+ {
+ if (stx.compressionMetadataOffHeapUsed && !sty.compressionMetadataOffHeapUsed)
+ return sign;
+ else if (!stx.compressionMetadataOffHeapUsed && sty.compressionMetadataOffHeapUsed)
+ return sign * -1;
+ else if (!stx.compressionMetadataOffHeapUsed && !sty.compressionMetadataOffHeapUsed)
+ result = 0;
+ else
+ {
+ result = compareFileSizes(stx.compressionMetadataOffHeapMemoryUsed,
+ sty.compressionMetadataOffHeapMemoryUsed);
+ }
+ }
+ else if (sortKey.equals("dropped_mutations"))
+ {
+ result = compareFileSizes(stx.droppedMutations, sty.droppedMutations);
+ }
+ else if (sortKey.equals("full_name"))
+ {
+ return sign * stx.fullName.compareTo(sty.fullName);
+ }
+ else if (sortKey.equals("index_summary_off_heap_memory_used"))
+ {
+ if (stx.indexSummaryOffHeapUsed && !sty.indexSummaryOffHeapUsed)
+ return sign;
+ else if (!stx.indexSummaryOffHeapUsed && sty.indexSummaryOffHeapUsed)
+ return sign * -1;
+ else if (!stx.indexSummaryOffHeapUsed && !sty.indexSummaryOffHeapUsed)
+ result = 0;
+ else
+ {
+ result = compareFileSizes(stx.indexSummaryOffHeapMemoryUsed,
+ sty.indexSummaryOffHeapMemoryUsed);
+ }
+ }
+ else if (sortKey.equals("local_read_count") || sortKey.equals("reads"))
+ {
+ result = sign * Long.valueOf(stx.localReadCount)
+ .compareTo(Long.valueOf(sty.localReadCount));
+ }
+ else if (sortKey.equals("local_read_latency_ms") || sortKey.equals("read_latency"))
+ {
+ result = compareDoubles(stx.localReadLatencyMs, sty.localReadLatencyMs);
+ }
+ else if (sortKey.equals("local_write_count") || sortKey.equals("writes"))
+ {
+ result = sign * Long.valueOf(stx.localWriteCount)
+ .compareTo(Long.valueOf(sty.localWriteCount));
+ }
+ else if (sortKey.equals("local_write_latency_ms") || sortKey.equals("write_latency"))
+ {
+ result = compareDoubles(stx.localWriteLatencyMs, sty.localWriteLatencyMs);
+ }
+ else if (sortKey.equals("maximum_live_cells_per_slice_last_five_minutes"))
+ {
+ result = sign * Long.valueOf(stx.maximumLiveCellsPerSliceLastFiveMinutes)
+ .compareTo(Long.valueOf(sty.maximumLiveCellsPerSliceLastFiveMinutes));
+ }
+ else if (sortKey.equals("maximum_tombstones_per_slice_last_five_minutes"))
+ {
+ result = sign * Long.valueOf(stx.maximumTombstonesPerSliceLastFiveMinutes)
+ .compareTo(Long.valueOf(sty.maximumTombstonesPerSliceLastFiveMinutes));
+ }
+ else if (sortKey.equals("memtable_cell_count"))
+ {
+ result = sign * ((Long) stx.memtableCellCount)
+ .compareTo((Long) sty.memtableCellCount);
+ }
+ else if (sortKey.equals("memtable_data_size"))
+ {
+ result = compareFileSizes(stx.memtableDataSize, sty.memtableDataSize);
+ }
+ else if (sortKey.equals("memtable_off_heap_memory_used"))
+ {
+ if (stx.memtableOffHeapUsed && !sty.memtableOffHeapUsed)
+ return sign;
+ else if (!stx.memtableOffHeapUsed && sty.memtableOffHeapUsed)
+ return sign * -1;
+ else if (!stx.memtableOffHeapUsed && !sty.memtableOffHeapUsed)
+ result = 0;
+ else
+ {
+ result = compareFileSizes(stx.memtableOffHeapMemoryUsed,
+ sty.memtableOffHeapMemoryUsed);
+ }
+ }
+ else if (sortKey.equals("memtable_switch_count"))
+ {
+ result = sign * ((Long) stx.memtableSwitchCount)
+ .compareTo((Long) sty.memtableSwitchCount);
+ }
+ else if (sortKey.equals("number_of_partitions_estimate"))
+ {
+ result = sign * ((Long) stx.numberOfPartitionsEstimate)
+ .compareTo((Long) sty.numberOfPartitionsEstimate);
+ }
+ else if (sortKey.equals("off_heap_memory_used_total"))
+ {
+ if (stx.offHeapUsed && !sty.offHeapUsed)
+ return sign;
+ else if (!stx.offHeapUsed && sty.offHeapUsed)
+ return sign * -1;
+ else if (!stx.offHeapUsed && !sty.offHeapUsed)
+ result = 0;
+ else
+ {
+ result = compareFileSizes(stx.offHeapMemoryUsedTotal,
+ sty.offHeapMemoryUsedTotal);
+ }
+ }
+ else if (sortKey.equals("pending_flushes"))
+ {
+ result = sign * ((Long) stx.pendingFlushes)
+ .compareTo((Long) sty.pendingFlushes);
+ }
+ else if (sortKey.equals("percent_repaired"))
+ {
+ result = compareDoubles(stx.percentRepaired, sty.percentRepaired);
+ }
+ else if (sortKey.equals("space_used_by_snapshots_total"))
+ {
+ result = compareFileSizes(stx.spaceUsedBySnapshotsTotal,
+ sty.spaceUsedBySnapshotsTotal);
+ }
+ else if (sortKey.equals("space_used_live"))
+ {
+ result = compareFileSizes(stx.spaceUsedLive, sty.spaceUsedLive);
+ }
+ else if (sortKey.equals("space_used_total"))
+ {
+ result = compareFileSizes(stx.spaceUsedTotal, sty.spaceUsedTotal);
+ }
+ else if (sortKey.equals("sstable_compression_ratio"))
+ {
+ result = compareDoubles((Double) stx.sstableCompressionRatio,
+ (Double) sty.sstableCompressionRatio);
+ }
+ else if (sortKey.equals("sstable_count"))
+ {
+ result = sign * ((Integer) stx.sstableCount)
+ .compareTo((Integer) sty.sstableCount);
+ }
+ else if (sortKey.equals("table_name"))
+ {
+ return sign * stx.tableName.compareTo(sty.tableName);
+ }
+ else
+ {
+ throw new IllegalStateException(String.format("Unsupported sort key: %s", sortKey));
+ }
+ return (result == 0) ? stx.fullName.compareTo(sty.fullName) : result;
+ }
+}
diff --git a/src/java/org/apache/cassandra/tools/nodetool/stats/TableStatsHolder.java b/src/java/org/apache/cassandra/tools/nodetool/stats/TableStatsHolder.java
index 19ab53c..624484f 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/stats/TableStatsHolder.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/stats/TableStatsHolder.java
@@ -20,10 +20,9 @@
import java.util.*;
-import javax.management.InstanceNotFoundException;
-
import com.google.common.collect.ArrayListMultimap;
+import javax.management.InstanceNotFoundException;
import org.apache.cassandra.db.*;
import org.apache.cassandra.io.util.*;
import org.apache.cassandra.metrics.*;
@@ -33,17 +32,41 @@
{
public final List<StatsKeyspace> keyspaces;
public final int numberOfTables;
+ public final boolean humanReadable;
+ public final String sortKey;
+ public final int top;
- public TableStatsHolder(NodeProbe probe, boolean humanReadable, boolean ignore, List<String> tableNames)
+ public TableStatsHolder(NodeProbe probe, boolean humanReadable, boolean ignore, List<String> tableNames, String sortKey, int top)
{
this.keyspaces = new ArrayList<>();
- this.numberOfTables = probe.getNumberOfTables();
- this.initializeKeyspaces(probe, humanReadable, ignore, tableNames);
+ this.humanReadable = humanReadable;
+ this.sortKey = sortKey;
+ this.top = top;
+ if (!this.isTestTableStatsHolder())
+ {
+ this.numberOfTables = probe.getNumberOfTables();
+ this.initializeKeyspaces(probe, ignore, tableNames);
+ }
+ else
+ {
+ this.numberOfTables = 0;
+ }
}
@Override
public Map<String, Object> convert2Map()
{
+ if (sortKey.isEmpty())
+ return convertAllToMap();
+ else
+ return convertSortedFilteredSubsetToMap();
+ }
+
+ /**
+ * @returns Map<String, Object> a nested HashMap of keyspaces, their tables, and the tables' statistics.
+ */
+ private Map<String, Object> convertAllToMap()
+ {
HashMap<String, Object> mpRet = new HashMap<>();
mpRet.put("total_number_of_tables", numberOfTables);
for (StatsKeyspace keyspace : keyspaces)
@@ -62,51 +85,8 @@
Map<String, Map<String, Object>> mpTables = new HashMap<>();
for (StatsTable table : tables)
{
- Map<String, Object> mpTable = new HashMap<>();
-
- mpTable.put("sstables_in_each_level", table.sstablesInEachLevel);
- mpTable.put("space_used_live", table.spaceUsedLive);
- mpTable.put("space_used_total", table.spaceUsedTotal);
- mpTable.put("space_used_by_snapshots_total", table.spaceUsedBySnapshotsTotal);
- if (table.offHeapUsed)
- mpTable.put("off_heap_memory_used_total", table.offHeapMemoryUsedTotal);
- mpTable.put("sstable_compression_ratio", table.sstableCompressionRatio);
- mpTable.put("number_of_partitions_estimate", table.numberOfPartitionsEstimate);
- mpTable.put("memtable_cell_count", table.memtableCellCount);
- mpTable.put("memtable_data_size", table.memtableDataSize);
- if (table.memtableOffHeapUsed)
- mpTable.put("memtable_off_heap_memory_used", table.memtableOffHeapMemoryUsed);
- mpTable.put("memtable_switch_count", table.memtableSwitchCount);
- mpTable.put("local_read_count", table.localReadCount);
- mpTable.put("local_read_latency_ms", String.format("%01.3f", table.localReadLatencyMs));
- mpTable.put("local_write_count", table.localWriteCount);
- mpTable.put("local_write_latency_ms", String.format("%01.3f", table.localWriteLatencyMs));
- mpTable.put("pending_flushes", table.pendingFlushes);
- mpTable.put("percent_repaired", table.percentRepaired);
- mpTable.put("bloom_filter_false_positives", table.bloomFilterFalsePositives);
- mpTable.put("bloom_filter_false_ratio", String.format("%01.5f", table.bloomFilterFalseRatio));
- mpTable.put("bloom_filter_space_used", table.bloomFilterSpaceUsed);
- if (table.bloomFilterOffHeapUsed)
- mpTable.put("bloom_filter_off_heap_memory_used", table.bloomFilterOffHeapMemoryUsed);
- if (table.indexSummaryOffHeapUsed)
- mpTable.put("index_summary_off_heap_memory_used", table.indexSummaryOffHeapMemoryUsed);
- if (table.compressionMetadataOffHeapUsed)
- mpTable.put("compression_metadata_off_heap_memory_used",
- table.compressionMetadataOffHeapMemoryUsed);
- mpTable.put("compacted_partition_minimum_bytes", table.compactedPartitionMinimumBytes);
- mpTable.put("compacted_partition_maximum_bytes", table.compactedPartitionMaximumBytes);
- mpTable.put("compacted_partition_mean_bytes", table.compactedPartitionMeanBytes);
- mpTable.put("average_live_cells_per_slice_last_five_minutes",
- table.averageLiveCellsPerSliceLastFiveMinutes);
- mpTable.put("maximum_live_cells_per_slice_last_five_minutes",
- table.maximumLiveCellsPerSliceLastFiveMinutes);
- mpTable.put("average_tombstones_per_slice_last_five_minutes",
- table.averageTombstonesPerSliceLastFiveMinutes);
- mpTable.put("maximum_tombstones_per_slice_last_five_minutes",
- table.maximumTombstonesPerSliceLastFiveMinutes);
- mpTable.put("dropped_mutations", table.droppedMutations);
-
- mpTables.put(table.name, mpTable);
+ Map<String, Object> mpTable = convertStatsTableToMap(table);
+ mpTables.put(table.tableName, mpTable);
}
mpKeyspace.put("tables", mpTables);
mpRet.put(keyspace.name, mpKeyspace);
@@ -114,7 +94,71 @@
return mpRet;
}
- private void initializeKeyspaces(NodeProbe probe, boolean humanReadable, boolean ignore, List<String> tableNames)
+ /**
+ * @returns Map<String, Object> a nested HashMap of the sorted and filtered table names and the HashMaps of their statistics.
+ */
+ private Map<String, Object> convertSortedFilteredSubsetToMap()
+ {
+ HashMap<String, Object> mpRet = new HashMap<>();
+ mpRet.put("total_number_of_tables", numberOfTables);
+ List<StatsTable> sortedFilteredTables = getSortedFilteredTables();
+ for (StatsTable table : sortedFilteredTables)
+ {
+ String tableDisplayName = table.keyspaceName + "." + table.tableName;
+ Map<String, Object> mpTable = convertStatsTableToMap(table);
+ mpRet.put(tableDisplayName, mpTable);
+ }
+ return mpRet;
+ }
+
+ private Map<String, Object> convertStatsTableToMap(StatsTable table)
+ {
+ Map<String, Object> mpTable = new HashMap<>();
+ mpTable.put("sstables_in_each_level", table.sstablesInEachLevel);
+ mpTable.put("space_used_live", table.spaceUsedLive);
+ mpTable.put("space_used_total", table.spaceUsedTotal);
+ mpTable.put("space_used_by_snapshots_total", table.spaceUsedBySnapshotsTotal);
+ if (table.offHeapUsed)
+ mpTable.put("off_heap_memory_used_total", table.offHeapMemoryUsedTotal);
+ mpTable.put("sstable_compression_ratio", table.sstableCompressionRatio);
+ mpTable.put("number_of_partitions_estimate", table.numberOfPartitionsEstimate);
+ mpTable.put("memtable_cell_count", table.memtableCellCount);
+ mpTable.put("memtable_data_size", table.memtableDataSize);
+ if (table.memtableOffHeapUsed)
+ mpTable.put("memtable_off_heap_memory_used", table.memtableOffHeapMemoryUsed);
+ mpTable.put("memtable_switch_count", table.memtableSwitchCount);
+ mpTable.put("local_read_count", table.localReadCount);
+ mpTable.put("local_read_latency_ms", String.format("%01.3f", table.localReadLatencyMs));
+ mpTable.put("local_write_count", table.localWriteCount);
+ mpTable.put("local_write_latency_ms", String.format("%01.3f", table.localWriteLatencyMs));
+ mpTable.put("pending_flushes", table.pendingFlushes);
+ mpTable.put("percent_repaired", table.percentRepaired);
+ mpTable.put("bloom_filter_false_positives", table.bloomFilterFalsePositives);
+ mpTable.put("bloom_filter_false_ratio", String.format("%01.5f", table.bloomFilterFalseRatio));
+ mpTable.put("bloom_filter_space_used", table.bloomFilterSpaceUsed);
+ if (table.bloomFilterOffHeapUsed)
+ mpTable.put("bloom_filter_off_heap_memory_used", table.bloomFilterOffHeapMemoryUsed);
+ if (table.indexSummaryOffHeapUsed)
+ mpTable.put("index_summary_off_heap_memory_used", table.indexSummaryOffHeapMemoryUsed);
+ if (table.compressionMetadataOffHeapUsed)
+ mpTable.put("compression_metadata_off_heap_memory_used",
+ table.compressionMetadataOffHeapMemoryUsed);
+ mpTable.put("compacted_partition_minimum_bytes", table.compactedPartitionMinimumBytes);
+ mpTable.put("compacted_partition_maximum_bytes", table.compactedPartitionMaximumBytes);
+ mpTable.put("compacted_partition_mean_bytes", table.compactedPartitionMeanBytes);
+ mpTable.put("average_live_cells_per_slice_last_five_minutes",
+ table.averageLiveCellsPerSliceLastFiveMinutes);
+ mpTable.put("maximum_live_cells_per_slice_last_five_minutes",
+ table.maximumLiveCellsPerSliceLastFiveMinutes);
+ mpTable.put("average_tombstones_per_slice_last_five_minutes",
+ table.averageTombstonesPerSliceLastFiveMinutes);
+ mpTable.put("maximum_tombstones_per_slice_last_five_minutes",
+ table.maximumTombstonesPerSliceLastFiveMinutes);
+ mpTable.put("dropped_mutations", table.droppedMutations);
+ return mpTable;
+ }
+
+ private void initializeKeyspaces(NodeProbe probe, boolean ignore, List<String> tableNames)
{
OptionFilter filter = new OptionFilter(ignore, tableNames);
ArrayListMultimap<String, ColumnFamilyStoreMBean> selectedTableMbeans = ArrayListMultimap.create();
@@ -160,9 +204,12 @@
{
String tableName = table.getTableName();
StatsTable statsTable = new StatsTable();
- statsTable.name = tableName;
+ statsTable.fullName = keyspaceName + "." + tableName;
+ statsTable.keyspaceName = keyspaceName;
+ statsTable.tableName = tableName;
statsTable.isIndex = tableName.contains(".");
statsTable.sstableCount = probe.getColumnFamilyMetric(keyspaceName, tableName, "LiveSSTableCount");
+ statsTable.oldSSTableCount = probe.getColumnFamilyMetric(keyspaceName, tableName, "OldVersionSSTableCount");
int[] leveledSStables = table.getSSTableCountPerLevel();
if (leveledSStables != null)
{
@@ -287,6 +334,24 @@
}
/**
+ * Sort and filter this TableStatHolder's tables as specified by its sortKey and top attributes.
+ */
+ public List<StatsTable> getSortedFilteredTables() {
+ List<StatsTable> tables = new ArrayList<>();
+ for (StatsKeyspace keyspace : keyspaces)
+ tables.addAll(keyspace.tables);
+ Collections.sort(tables, new StatsTableComparator(sortKey, humanReadable));
+ int k = (tables.size() >= top) ? top : tables.size();
+ if (k > 0)
+ tables = tables.subList(0, k);
+ return tables;
+ }
+
+ protected boolean isTestTableStatsHolder() {
+ return false;
+ }
+
+ /**
* Used for filtering keyspaces and tables to be displayed using the tablestats command.
*/
private static class OptionFilter
diff --git a/src/java/org/apache/cassandra/tools/nodetool/stats/TableStatsPrinter.java b/src/java/org/apache/cassandra/tools/nodetool/stats/TableStatsPrinter.java
index e1e7b42..0998c2a 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/stats/TableStatsPrinter.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/stats/TableStatsPrinter.java
@@ -23,7 +23,7 @@
public class TableStatsPrinter
{
- public static StatsPrinter from(String format)
+ public static StatsPrinter from(String format, boolean sorted)
{
switch (format)
{
@@ -32,10 +32,16 @@
case "yaml":
return new StatsPrinter.YamlPrinter();
default:
- return new DefaultPrinter();
+ if (sorted)
+ return new SortedDefaultPrinter();
+ else
+ return new DefaultPrinter();
}
}
+ /**
+ * A StatsPrinter to print stats in a keyspace-centric way, nesting stats for each table under their parent keyspaces.
+ */
private static class DefaultPrinter implements StatsPrinter<TableStatsHolder>
{
@Override
@@ -59,56 +65,87 @@
List<StatsTable> tables = keyspace.tables;
for (StatsTable table : tables)
{
- out.println("\t\tTable" + (table.isIndex ? " (index): " : ": ") + table.name);
- out.println("\t\tSSTable count: " + table.sstableCount);
- if (table.isLeveledSstable)
- out.println("\t\tSSTables in each level: [" + String.join(", ",
- table.sstablesInEachLevel) + "]");
-
- out.println("\t\tSpace used (live): " + table.spaceUsedLive);
- out.println("\t\tSpace used (total): " + table.spaceUsedTotal);
- out.println("\t\tSpace used by snapshots (total): " + table.spaceUsedBySnapshotsTotal);
-
- if (table.offHeapUsed)
- out.println("\t\tOff heap memory used (total): " + table.offHeapMemoryUsedTotal);
- out.println("\t\tSSTable Compression Ratio: " + table.sstableCompressionRatio);
- out.println("\t\tNumber of partitions (estimate): " + table.numberOfPartitionsEstimate);
- out.println("\t\tMemtable cell count: " + table.memtableCellCount);
- out.println("\t\tMemtable data size: " + table.memtableDataSize);
-
- if (table.memtableOffHeapUsed)
- out.println("\t\tMemtable off heap memory used: " + table.memtableOffHeapMemoryUsed);
- out.println("\t\tMemtable switch count: " + table.memtableSwitchCount);
- out.println("\t\tLocal read count: " + table.localReadCount);
- out.printf("\t\tLocal read latency: %01.3f ms%n", table.localReadLatencyMs);
- out.println("\t\tLocal write count: " + table.localWriteCount);
- out.printf("\t\tLocal write latency: %01.3f ms%n", table.localWriteLatencyMs);
- out.println("\t\tPending flushes: " + table.pendingFlushes);
- out.println("\t\tPercent repaired: " + table.percentRepaired);
-
- out.println("\t\tBloom filter false positives: " + table.bloomFilterFalsePositives);
- out.printf("\t\tBloom filter false ratio: %01.5f%n", table.bloomFilterFalseRatio);
- out.println("\t\tBloom filter space used: " + table.bloomFilterSpaceUsed);
-
- if (table.bloomFilterOffHeapUsed)
- out.println("\t\tBloom filter off heap memory used: " + table.bloomFilterOffHeapMemoryUsed);
- if (table.indexSummaryOffHeapUsed)
- out.println("\t\tIndex summary off heap memory used: " + table.indexSummaryOffHeapMemoryUsed);
- if (table.compressionMetadataOffHeapUsed)
- out.println("\t\tCompression metadata off heap memory used: " + table.compressionMetadataOffHeapMemoryUsed);
-
- out.println("\t\tCompacted partition minimum bytes: " + table.compactedPartitionMinimumBytes);
- out.println("\t\tCompacted partition maximum bytes: " + table.compactedPartitionMaximumBytes);
- out.println("\t\tCompacted partition mean bytes: " + table.compactedPartitionMeanBytes);
- out.println("\t\tAverage live cells per slice (last five minutes): " + table.averageLiveCellsPerSliceLastFiveMinutes);
- out.println("\t\tMaximum live cells per slice (last five minutes): " + table.maximumLiveCellsPerSliceLastFiveMinutes);
- out.println("\t\tAverage tombstones per slice (last five minutes): " + table.averageTombstonesPerSliceLastFiveMinutes);
- out.println("\t\tMaximum tombstones per slice (last five minutes): " + table.maximumTombstonesPerSliceLastFiveMinutes);
- out.println("\t\tDropped Mutations: " + table.droppedMutations);
- out.println("");
+ printStatsTable(table, table.tableName, "\t\t", out);
}
out.println("----------------");
}
}
+
+ protected void printStatsTable(StatsTable table, String tableDisplayName, String indent, PrintStream out)
+ {
+ out.println(indent + "Table" + (table.isIndex ? " (index): " : ": ") + tableDisplayName);
+ out.println(indent + "SSTable count: " + table.sstableCount);
+ out.println(indent + "Old SSTable count: " + table.oldSSTableCount);
+ if (table.isLeveledSstable)
+ out.println(indent + "SSTables in each level: [" + String.join(", ",
+ table.sstablesInEachLevel) + "]");
+
+ out.println(indent + "Space used (live): " + table.spaceUsedLive);
+ out.println(indent + "Space used (total): " + table.spaceUsedTotal);
+ out.println(indent + "Space used by snapshots (total): " + table.spaceUsedBySnapshotsTotal);
+
+ if (table.offHeapUsed)
+ out.println(indent + "Off heap memory used (total): " + table.offHeapMemoryUsedTotal);
+ out.println(indent + "SSTable Compression Ratio: " + table.sstableCompressionRatio);
+ out.println(indent + "Number of partitions (estimate): " + table.numberOfPartitionsEstimate);
+ out.println(indent + "Memtable cell count: " + table.memtableCellCount);
+ out.println(indent + "Memtable data size: " + table.memtableDataSize);
+
+ if (table.memtableOffHeapUsed)
+ out.println(indent + "Memtable off heap memory used: " + table.memtableOffHeapMemoryUsed);
+ out.println(indent + "Memtable switch count: " + table.memtableSwitchCount);
+ out.println(indent + "Local read count: " + table.localReadCount);
+ out.printf(indent + "Local read latency: %01.3f ms%n", table.localReadLatencyMs);
+ out.println(indent + "Local write count: " + table.localWriteCount);
+ out.printf(indent + "Local write latency: %01.3f ms%n", table.localWriteLatencyMs);
+ out.println(indent + "Pending flushes: " + table.pendingFlushes);
+ out.println(indent + "Percent repaired: " + table.percentRepaired);
+
+ out.println(indent + "Bloom filter false positives: " + table.bloomFilterFalsePositives);
+ out.printf(indent + "Bloom filter false ratio: %01.5f%n", table.bloomFilterFalseRatio);
+ out.println(indent + "Bloom filter space used: " + table.bloomFilterSpaceUsed);
+
+ if (table.bloomFilterOffHeapUsed)
+ out.println(indent + "Bloom filter off heap memory used: " + table.bloomFilterOffHeapMemoryUsed);
+ if (table.indexSummaryOffHeapUsed)
+ out.println(indent + "Index summary off heap memory used: " + table.indexSummaryOffHeapMemoryUsed);
+ if (table.compressionMetadataOffHeapUsed)
+ out.println(indent + "Compression metadata off heap memory used: " + table.compressionMetadataOffHeapMemoryUsed);
+
+ out.println(indent + "Compacted partition minimum bytes: " + table.compactedPartitionMinimumBytes);
+ out.println(indent + "Compacted partition maximum bytes: " + table.compactedPartitionMaximumBytes);
+ out.println(indent + "Compacted partition mean bytes: " + table.compactedPartitionMeanBytes);
+ out.println(indent + "Average live cells per slice (last five minutes): " + table.averageLiveCellsPerSliceLastFiveMinutes);
+ out.println(indent + "Maximum live cells per slice (last five minutes): " + table.maximumLiveCellsPerSliceLastFiveMinutes);
+ out.println(indent + "Average tombstones per slice (last five minutes): " + table.averageTombstonesPerSliceLastFiveMinutes);
+ out.println(indent + "Maximum tombstones per slice (last five minutes): " + table.maximumTombstonesPerSliceLastFiveMinutes);
+ out.println(indent + "Dropped Mutations: " + table.droppedMutations);
+ out.println("");
+ }
+ }
+
+ /**
+ * A StatsPrinter to print stats in a sorted, table-centric way.
+ */
+ private static class SortedDefaultPrinter extends DefaultPrinter
+ {
+ @Override
+ public void print(TableStatsHolder data, PrintStream out)
+ {
+ List<StatsTable> tables = data.getSortedFilteredTables();
+ String totalTablesSummary = String.format("Total number of tables: %d", data.numberOfTables);
+ if (data.top > 0)
+ {
+ int k = (data.top <= data.numberOfTables) ? data.top : data.numberOfTables;
+ totalTablesSummary += String.format(" (showing top %d by %s)", k, data.sortKey);
+ }
+ out.println(totalTablesSummary);
+ out.println("----------------");
+ for (StatsTable table : tables)
+ {
+ printStatsTable(table, table.keyspaceName + "." + table.tableName, "\t", out);
+ }
+ out.println("----------------");
+ }
}
}
diff --git a/src/java/org/apache/cassandra/tools/nodetool/stats/TpStatsHolder.java b/src/java/org/apache/cassandra/tools/nodetool/stats/TpStatsHolder.java
index d70b4dd..f3e91dc 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/stats/TpStatsHolder.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/stats/TpStatsHolder.java
@@ -39,6 +39,7 @@
HashMap<String, Object> result = new HashMap<>();
HashMap<String, Map<String, Object>> threadPools = new HashMap<>();
HashMap<String, Object> droppedMessage = new HashMap<>();
+ HashMap<String, double[]> waitLatencies = new HashMap<>();
for (Map.Entry<String, String> tp : probe.getThreadPools().entries())
{
@@ -53,8 +54,20 @@
result.put("ThreadPools", threadPools);
for (Map.Entry<String, Integer> entry : probe.getDroppedMessages().entrySet())
+ {
droppedMessage.put(entry.getKey(), entry.getValue());
+ try
+ {
+ waitLatencies.put(entry.getKey(), probe.metricPercentilesAsArray(probe.getMessagingQueueWaitMetrics(entry.getKey())));
+ }
+ catch (RuntimeException e)
+ {
+ // ignore the exceptions when fetching metrics
+ }
+ }
+
result.put("DroppedMessage", droppedMessage);
+ result.put("WaitLatencies", waitLatencies);
return result;
}
diff --git a/src/java/org/apache/cassandra/tools/nodetool/stats/TpStatsPrinter.java b/src/java/org/apache/cassandra/tools/nodetool/stats/TpStatsPrinter.java
index b874746..86bdf28 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/stats/TpStatsPrinter.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/stats/TpStatsPrinter.java
@@ -20,7 +20,6 @@
import java.io.PrintStream;
import java.util.Collections;
-import java.util.HashMap;
import java.util.Map;
public class TpStatsPrinter
@@ -61,12 +60,25 @@
values.get("TotalBlockedTasks"));
}
- out.printf("%n%-20s%10s%n", "Message type", "Dropped");
+ out.printf("%n%-20s%10s%18s%18s%18s%18s%n", "Message type", "Dropped", "", "Latency waiting in queue (micros)", "", "");
+ out.printf("%-20s%10s%18s%18s%18s%18s%n", "", "", "50%", "95%", "99%", "Max");
Map<Object, Object> droppedMessages = convertData.get("DroppedMessage") instanceof Map<?, ?> ? (Map)convertData.get("DroppedMessage") : Collections.emptyMap();
+ Map<Object, double[]> waitLatencies = convertData.get("WaitLatencies") instanceof Map<?, ?> ? (Map)convertData.get("WaitLatencies") : Collections.emptyMap();
for (Map.Entry<Object, Object> entry : droppedMessages.entrySet())
{
- out.printf("%-20s%10s%n", entry.getKey(), entry.getValue());
+ out.printf("%-20s%10s", entry.getKey(), entry.getValue());
+ if (waitLatencies.containsKey(entry.getKey()))
+ {
+ double[] latencies = waitLatencies.get(entry.getKey());
+ out.printf("%18.2f%18.2f%18.2f%18.2f", latencies[0], latencies[2], latencies[4], latencies[6]);
+ }
+ else
+ {
+ out.printf("%18s%18s%18s%18s", "N/A", "N/A", "N/A", "N/A");
+ }
+
+ out.printf("%n");
}
}
}
diff --git a/src/java/org/apache/cassandra/tracing/ExpiredTraceState.java b/src/java/org/apache/cassandra/tracing/ExpiredTraceState.java
index 9230d38..bf95080 100644
--- a/src/java/org/apache/cassandra/tracing/ExpiredTraceState.java
+++ b/src/java/org/apache/cassandra/tracing/ExpiredTraceState.java
@@ -29,7 +29,7 @@
ExpiredTraceState(TraceState delegate)
{
- super(FBUtilities.getBroadcastAddress(), delegate.sessionId, delegate.traceType);
+ super(FBUtilities.getBroadcastAddressAndPort(), delegate.sessionId, delegate.traceType);
this.delegate = delegate;
}
diff --git a/src/java/org/apache/cassandra/tracing/TraceKeyspace.java b/src/java/org/apache/cassandra/tracing/TraceKeyspace.java
index 0d7c4f1..8c6d8c8 100644
--- a/src/java/org/apache/cassandra/tracing/TraceKeyspace.java
+++ b/src/java/org/apache/cassandra/tracing/TraceKeyspace.java
@@ -21,17 +21,22 @@
import java.nio.ByteBuffer;
import java.util.*;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.SchemaConstants;
+import org.apache.cassandra.cql3.statements.schema.CreateTableStatement;
import org.apache.cassandra.db.Mutation;
-import org.apache.cassandra.db.rows.Row;
import org.apache.cassandra.db.partitions.PartitionUpdate;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.gms.Gossiper;
import org.apache.cassandra.schema.KeyspaceMetadata;
import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.SchemaConstants;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.schema.Tables;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.UUIDGen;
+import static java.lang.String.format;
+
public final class TraceKeyspace
{
private TraceKeyspace()
@@ -47,18 +52,19 @@
* will ever start; see the note below for why this is necessary; actual change in 3.0:
* removed default ttl, reduced bloom filter fp chance from 0.1 to 0.01.
* gen 1577836800000001: (pre-)adds coordinator_port column to sessions and source_port column to events in 3.0, 3.11, 4.0
+ * gen 1577836800000002: compression chunk length reduced to 16KiB, memtable_flush_period_in_ms now unset on all tables in 4.0
*
* * Until CASSANDRA-6016 (Oct 13, 2.0.2) and in all of 1.2, we used to create system_traces keyspace and
* tables in the same way that we created the purely local 'system' keyspace - using current time on node bounce
* (+1). For new definitions to take, we need to bump the generation further than that.
*/
- public static final long GENERATION = 1577836800000001L;
+ public static final long GENERATION = 1577836800000002L;
public static final String SESSIONS = "sessions";
public static final String EVENTS = "events";
- private static final CFMetaData Sessions =
- compile(SESSIONS,
+ private static final TableMetadata Sessions =
+ parse(SESSIONS,
"tracing sessions",
"CREATE TABLE %s ("
+ "session_id uuid,"
@@ -72,8 +78,8 @@
+ "started_at timestamp,"
+ "PRIMARY KEY ((session_id)))");
- private static final CFMetaData Events =
- compile(EVENTS,
+ private static final TableMetadata Events =
+ parse(EVENTS,
"tracing events",
"CREATE TABLE %s ("
+ "session_id uuid,"
@@ -85,10 +91,13 @@
+ "thread text,"
+ "PRIMARY KEY ((session_id), event_id))");
- private static CFMetaData compile(String name, String description, String schema)
+ private static TableMetadata parse(String table, String description, String cql)
{
- return CFMetaData.compile(String.format(schema, name), SchemaConstants.TRACE_KEYSPACE_NAME)
- .comment(description);
+ return CreateTableStatement.parse(format(cql, table), SchemaConstants.TRACE_KEYSPACE_NAME)
+ .id(TableId.forSystemTable(SchemaConstants.TRACE_KEYSPACE_NAME, table))
+ .gcGraceSeconds(0)
+ .comment(description)
+ .build();
}
public static KeyspaceMetadata metadata()
@@ -105,14 +114,16 @@
int ttl)
{
PartitionUpdate.SimpleBuilder builder = PartitionUpdate.simpleBuilder(Sessions, sessionId);
- builder.row()
- .ttl(ttl)
- .add("client", client)
- .add("coordinator", FBUtilities.getBroadcastAddress())
- .add("request", request)
- .add("started_at", new Date(startedAt))
- .add("command", command)
- .appendAll("parameters", parameters);
+ Row.SimpleBuilder rb = builder.row();
+ rb.ttl(ttl)
+ .add("client", client)
+ .add("coordinator", FBUtilities.getBroadcastAddressAndPort().address);
+ if (!Gossiper.instance.haveMajorVersion3Nodes())
+ rb.add("coordinator_port", FBUtilities.getBroadcastAddressAndPort().port);
+ rb.add("request", request)
+ .add("started_at", new Date(startedAt))
+ .add("command", command)
+ .appendAll("parameters", parameters);
return builder.buildAsMutation();
}
@@ -133,8 +144,10 @@
.ttl(ttl);
rowBuilder.add("activity", message)
- .add("source", FBUtilities.getBroadcastAddress())
- .add("thread", threadName);
+ .add("source", FBUtilities.getBroadcastAddressAndPort().address);
+ if (!Gossiper.instance.haveMajorVersion3Nodes())
+ rowBuilder.add("source_port", FBUtilities.getBroadcastAddressAndPort().port);
+ rowBuilder.add("thread", threadName);
if (elapsed >= 0)
rowBuilder.add("source_elapsed", elapsed);
diff --git a/src/java/org/apache/cassandra/tracing/TraceState.java b/src/java/org/apache/cassandra/tracing/TraceState.java
index b4eff6b..1e0813c 100644
--- a/src/java/org/apache/cassandra/tracing/TraceState.java
+++ b/src/java/org/apache/cassandra/tracing/TraceState.java
@@ -17,7 +17,6 @@
*/
package org.apache.cassandra.tracing;
-import java.net.InetAddress;
import java.nio.ByteBuffer;
import java.util.List;
import java.util.UUID;
@@ -28,6 +27,7 @@
import com.google.common.base.Stopwatch;
import org.slf4j.helpers.MessageFormatter;
+import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.progress.ProgressEvent;
import org.apache.cassandra.utils.progress.ProgressEventNotifier;
@@ -40,7 +40,7 @@
public abstract class TraceState implements ProgressEventNotifier
{
public final UUID sessionId;
- public final InetAddress coordinator;
+ public final InetAddressAndPort coordinator;
public final Stopwatch watch;
public final ByteBuffer sessionIdBytes;
public final Tracing.TraceType traceType;
@@ -63,7 +63,7 @@
// See CASSANDRA-7626 for more details.
private final AtomicInteger references = new AtomicInteger(1);
- protected TraceState(InetAddress coordinator, UUID sessionId, Tracing.TraceType traceType)
+ protected TraceState(InetAddressAndPort coordinator, UUID sessionId, Tracing.TraceType traceType)
{
assert coordinator != null;
assert sessionId != null;
@@ -161,7 +161,7 @@
trace(MessageFormatter.format(format, arg1, arg2).getMessage());
}
- public void trace(String format, Object[] args)
+ public void trace(String format, Object... args)
{
trace(MessageFormatter.arrayFormat(format, args).getMessage());
}
diff --git a/src/java/org/apache/cassandra/tracing/TraceStateImpl.java b/src/java/org/apache/cassandra/tracing/TraceStateImpl.java
index 349000a..48f193c 100644
--- a/src/java/org/apache/cassandra/tracing/TraceStateImpl.java
+++ b/src/java/org/apache/cassandra/tracing/TraceStateImpl.java
@@ -17,7 +17,6 @@
*/
package org.apache.cassandra.tracing;
-import java.net.InetAddress;
import java.util.Collections;
import java.util.Set;
import java.util.UUID;
@@ -32,10 +31,10 @@
import org.slf4j.LoggerFactory;
import org.apache.cassandra.concurrent.Stage;
-import org.apache.cassandra.concurrent.StageManager;
import org.apache.cassandra.db.ConsistencyLevel;
import org.apache.cassandra.db.Mutation;
import org.apache.cassandra.exceptions.OverloadedException;
+import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.service.StorageProxy;
import org.apache.cassandra.utils.JVMStabilityInspector;
import org.apache.cassandra.utils.WrappedRunnable;
@@ -54,7 +53,7 @@
private final Set<Future<?>> pendingFutures = ConcurrentHashMap.newKeySet();
- public TraceStateImpl(InetAddress coordinator, UUID sessionId, Tracing.TraceType traceType)
+ public TraceStateImpl(InetAddressAndPort coordinator, UUID sessionId, Tracing.TraceType traceType)
{
super(coordinator, sessionId, traceType);
}
@@ -108,7 +107,7 @@
{
mutateWithCatch(mutation);
}
- }, StageManager.getStage(Stage.TRACING));
+ }, Stage.TRACING.executor());
boolean ret = pendingFutures.add(fut);
if (!ret)
diff --git a/src/java/org/apache/cassandra/tracing/Tracing.java b/src/java/org/apache/cassandra/tracing/Tracing.java
index 33e1967..311685b 100644
--- a/src/java/org/apache/cassandra/tracing/Tracing.java
+++ b/src/java/org/apache/cassandra/tracing/Tracing.java
@@ -19,6 +19,7 @@
*/
package org.apache.cassandra.tracing;
+import java.io.IOException;
import java.net.InetAddress;
import java.nio.ByteBuffer;
import java.util.Collections;
@@ -27,7 +28,6 @@
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
-import com.google.common.collect.ImmutableMap;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -35,8 +35,12 @@
import org.apache.cassandra.concurrent.ExecutorLocal;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.marshal.TimeUUIDType;
-import org.apache.cassandra.net.MessageIn;
-import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.io.IVersionedSerializer;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.net.Message;
+import org.apache.cassandra.net.ParamType;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.JVMStabilityInspector;
import org.apache.cassandra.utils.UUIDGen;
@@ -48,8 +52,23 @@
*/
public abstract class Tracing implements ExecutorLocal<TraceState>
{
- public static final String TRACE_HEADER = "TraceSession";
- public static final String TRACE_TYPE = "TraceType";
+ public static final IVersionedSerializer<TraceType> traceTypeSerializer = new IVersionedSerializer<TraceType>()
+ {
+ public void serialize(TraceType traceType, DataOutputPlus out, int version) throws IOException
+ {
+ out.write((byte)traceType.ordinal());
+ }
+
+ public TraceType deserialize(DataInputPlus in, int version) throws IOException
+ {
+ return TraceType.deserialize(in.readByte());
+ }
+
+ public long serializedSize(TraceType traceType, int version)
+ {
+ return 1;
+ }
+ };
public enum TraceType
{
@@ -83,7 +102,7 @@
protected static final Logger logger = LoggerFactory.getLogger(Tracing.class);
- private final InetAddress localAddress = FBUtilities.getLocalAddress();
+ private final InetAddressAndPort localAddress = FBUtilities.getLocalAddressAndPort();
private final FastThreadLocal<TraceState> state = new FastThreadLocal<>();
@@ -224,49 +243,76 @@
/**
* Determines the tracing context from a message. Does NOT set the threadlocal state.
*
- * @param message The internode message
+ * @param header The internode message header
*/
- public TraceState initializeFromMessage(final MessageIn<?> message)
+ public TraceState initializeFromMessage(final Message.Header header)
{
- final byte[] sessionBytes = message.parameters.get(TRACE_HEADER);
-
- if (sessionBytes == null)
+ final UUID sessionId = header.traceSession();
+ if (sessionId == null)
return null;
- assert sessionBytes.length == 16;
- UUID sessionId = UUIDGen.getUUID(ByteBuffer.wrap(sessionBytes));
TraceState ts = get(sessionId);
if (ts != null && ts.acquireReference())
return ts;
- byte[] tmpBytes;
- TraceType traceType = TraceType.QUERY;
- if ((tmpBytes = message.parameters.get(TRACE_TYPE)) != null)
- traceType = TraceType.deserialize(tmpBytes[0]);
+ TraceType traceType = header.traceType();
- if (message.verb == MessagingService.Verb.REQUEST_RESPONSE)
+ if (header.verb.isResponse())
{
// received a message for a session we've already closed out. see CASSANDRA-5668
- return new ExpiredTraceState(newTraceState(message.from, sessionId, traceType));
+ return new ExpiredTraceState(newTraceState(header.from, sessionId, traceType));
}
else
{
- ts = newTraceState(message.from, sessionId, traceType);
+ ts = newTraceState(header.from, sessionId, traceType);
sessions.put(sessionId, ts);
return ts;
}
}
- public Map<String, byte[]> getTraceHeaders()
+ /**
+ * Record any tracing data, if enabled on this message.
+ */
+ public void traceOutgoingMessage(Message<?> message, int serializedSize, InetAddressAndPort sendTo)
+ {
+ try
+ {
+ final UUID sessionId = message.traceSession();
+ if (sessionId == null)
+ return;
+
+ String logMessage = String.format("Sending %s message to %s message size %d bytes", message.verb(), sendTo,
+ serializedSize);
+
+ TraceState state = get(sessionId);
+ if (state == null) // session may have already finished; see CASSANDRA-5668
+ {
+ TraceType traceType = message.traceType();
+ trace(ByteBuffer.wrap(UUIDGen.decompose(sessionId)), logMessage, traceType.getTTL());
+ }
+ else
+ {
+ state.trace(logMessage);
+ if (message.verb().isResponse())
+ doneWithNonLocalSession(state);
+ }
+ }
+ catch (Exception e)
+ {
+ logger.warn("failed to capture the tracing info for an outbound message to {}, ignoring", sendTo, e);
+ }
+ }
+
+ public Map<ParamType, Object> addTraceHeaders(Map<ParamType, Object> addToMutable)
{
assert isTracing();
- return ImmutableMap.of(
- TRACE_HEADER, UUIDGen.decompose(Tracing.instance.getSessionId()),
- TRACE_TYPE, new byte[] { Tracing.TraceType.serialize(Tracing.instance.getTraceType()) });
+ addToMutable.put(ParamType.TRACE_SESSION, Tracing.instance.getSessionId());
+ addToMutable.put(ParamType.TRACE_TYPE, Tracing.instance.getTraceType());
+ return addToMutable;
}
- protected abstract TraceState newTraceState(InetAddress coordinator, UUID sessionId, Tracing.TraceType traceType);
+ protected abstract TraceState newTraceState(InetAddressAndPort coordinator, UUID sessionId, Tracing.TraceType traceType);
// repair just gets a varargs method since it's so heavyweight anyway
public static void traceRepair(String format, Object... args)
@@ -316,8 +362,7 @@
}
/**
- * Called from {@link org.apache.cassandra.net.OutboundTcpConnection} for non-local traces (traces
- * that are not initiated by local node == coordinator).
+ * Called for non-local traces (traces that are not initiated by local node == coordinator).
*/
public abstract void trace(ByteBuffer sessionId, String message, int ttl);
}
diff --git a/src/java/org/apache/cassandra/tracing/TracingImpl.java b/src/java/org/apache/cassandra/tracing/TracingImpl.java
index d774abb..c786fa2 100644
--- a/src/java/org/apache/cassandra/tracing/TracingImpl.java
+++ b/src/java/org/apache/cassandra/tracing/TracingImpl.java
@@ -25,7 +25,7 @@
import java.util.UUID;
import org.apache.cassandra.concurrent.Stage;
-import org.apache.cassandra.concurrent.StageManager;
+import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.utils.WrappedRunnable;
@@ -93,20 +93,19 @@
}
@Override
- protected TraceState newTraceState(InetAddress coordinator, UUID sessionId, TraceType traceType)
+ protected TraceState newTraceState(InetAddressAndPort coordinator, UUID sessionId, TraceType traceType)
{
return new TraceStateImpl(coordinator, sessionId, traceType);
}
/**
- * Called from {@link org.apache.cassandra.net.OutboundTcpConnection} for non-local traces (traces
- * that are not initiated by local node == coordinator).
+ * Called for non-local traces (traces that are not initiated by local node == coordinator).
*/
public void trace(final ByteBuffer sessionId, final String message, final int ttl)
{
final String threadName = Thread.currentThread().getName();
- StageManager.getStage(Stage.TRACING).execute(new WrappedRunnable()
+ Stage.TRACING.execute(new WrappedRunnable()
{
public void runMayThrow()
{
diff --git a/src/java/org/apache/cassandra/transport/CBUtil.java b/src/java/org/apache/cassandra/transport/CBUtil.java
index 66e5e73..6e0c8ff 100644
--- a/src/java/org/apache/cassandra/transport/CBUtil.java
+++ b/src/java/org/apache/cassandra/transport/CBUtil.java
@@ -23,9 +23,9 @@
import java.nio.ByteBuffer;
import java.nio.CharBuffer;
import java.nio.charset.CharacterCodingException;
-import java.nio.charset.Charset;
import java.nio.charset.CharsetDecoder;
import java.nio.charset.CoderResult;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
@@ -38,7 +38,6 @@
import io.netty.buffer.ByteBufUtil;
import io.netty.buffer.PooledByteBufAllocator;
import io.netty.buffer.UnpooledByteBufAllocator;
-import io.netty.util.CharsetUtil;
import io.netty.util.concurrent.FastThreadLocal;
import org.apache.cassandra.config.Config;
import org.apache.cassandra.db.ConsistencyLevel;
@@ -58,13 +57,14 @@
{
public static final boolean USE_HEAP_ALLOCATOR = Boolean.getBoolean(Config.PROPERTY_PREFIX + "netty_use_heap_allocator");
public static final ByteBufAllocator allocator = USE_HEAP_ALLOCATOR ? new UnpooledByteBufAllocator(false) : new PooledByteBufAllocator(true);
+ private static final int UUID_SIZE = 16;
private final static FastThreadLocal<CharsetDecoder> TL_UTF8_DECODER = new FastThreadLocal<CharsetDecoder>()
{
@Override
protected CharsetDecoder initialValue()
{
- return Charset.forName("UTF-8").newDecoder();
+ return StandardCharsets.UTF_8.newDecoder();
}
};
@@ -136,12 +136,22 @@
}
}
+ /**
+ * Write US-ASCII strings. It does not work if containing any char > 0x007F (127)
+ * @param str, satisfies {@link org.apache.cassandra.db.marshal.AsciiType},
+ * i.e. seven-bit ASCII, a.k.a. ISO646-US
+ */
+ public static void writeAsciiString(String str, ByteBuf cb)
+ {
+ cb.writeShort(str.length());
+ ByteBufUtil.writeAscii(cb, str);
+ }
+
public static void writeString(String str, ByteBuf cb)
{
- int writerIndex = cb.writerIndex();
- cb.writeShort(0);
- int lengthBytes = ByteBufUtil.writeUtf8(cb, str);
- cb.setShort(writerIndex, lengthBytes);
+ int length = TypeSizes.encodedUTF8Length(str);
+ cb.writeShort(length);
+ ByteBufUtil.reserveAndWriteUtf8(cb, str, length);
}
public static int sizeOfString(String str)
@@ -149,6 +159,16 @@
return 2 + TypeSizes.encodedUTF8Length(str);
}
+ /**
+ * Returns the ecoding size of a US-ASCII string. It does not work if containing any char > 0x007F (127)
+ * @param str, satisfies {@link org.apache.cassandra.db.marshal.AsciiType},
+ * i.e. seven-bit ASCII, a.k.a. ISO646-US
+ */
+ public static int sizeOfAsciiString(String str)
+ {
+ return 2 + str.length();
+ }
+
public static String readLongString(ByteBuf cb)
{
try
@@ -164,14 +184,14 @@
public static void writeLongString(String str, ByteBuf cb)
{
- byte[] bytes = str.getBytes(CharsetUtil.UTF_8);
- cb.writeInt(bytes.length);
- cb.writeBytes(bytes);
+ int length = TypeSizes.encodedUTF8Length(str);
+ cb.writeInt(length);
+ ByteBufUtil.reserveAndWriteUtf8(cb, str, length);
}
public static int sizeOfLongString(String str)
{
- return 4 + str.getBytes(CharsetUtil.UTF_8).length;
+ return 4 + TypeSizes.encodedUTF8Length(str);
}
public static byte[] readBytes(ByteBuf cb)
@@ -264,19 +284,21 @@
public static <T extends Enum<T>> void writeEnumValue(T enumValue, ByteBuf cb)
{
- writeString(enumValue.toString(), cb);
+ // UTF-8 (non-ascii) literals can be used for as a valid identifier in Java. It is possible for an enum to be named using those characters.
+ // There is no such occurence in the code base.
+ writeAsciiString(enumValue.toString(), cb);
}
public static <T extends Enum<T>> int sizeOfEnumValue(T enumValue)
{
- return sizeOfString(enumValue.toString());
+ return sizeOfAsciiString(enumValue.toString());
}
public static UUID readUUID(ByteBuf cb)
{
- byte[] bytes = new byte[16];
- cb.readBytes(bytes);
- return UUIDGen.getUUID(ByteBuffer.wrap(bytes));
+ ByteBuffer buffer = cb.nioBuffer(cb.readerIndex(), UUID_SIZE);
+ cb.skipBytes(buffer.remaining());
+ return UUIDGen.getUUID(buffer);
}
public static void writeUUID(UUID uuid, ByteBuf cb)
@@ -286,7 +308,7 @@
public static int sizeOfUUID(UUID uuid)
{
- return 16;
+ return UUID_SIZE;
}
public static List<String> readStringList(ByteBuf cb)
@@ -386,9 +408,19 @@
int length = cb.readInt();
if (length < 0)
return null;
- ByteBuf slice = cb.readSlice(length);
- return ByteBuffer.wrap(readRawBytes(slice));
+ return ByteBuffer.wrap(readRawBytes(cb, length));
+ }
+
+ public static ByteBuffer readValueNoCopy(ByteBuf cb)
+ {
+ int length = cb.readInt();
+ if (length < 0)
+ return null;
+
+ ByteBuffer buffer = cb.nioBuffer(cb.readerIndex(), length);
+ cb.skipBytes(length);
+ return buffer;
}
public static ByteBuffer readBoundValue(ByteBuf cb, ProtocolVersion protocolVersion)
@@ -405,9 +437,7 @@
else
throw new ProtocolException("Invalid ByteBuf length " + length);
}
- ByteBuf slice = cb.readSlice(length);
-
- return ByteBuffer.wrap(readRawBytes(slice));
+ return ByteBuffer.wrap(readRawBytes(cb, length));
}
public static void writeValue(byte[] bytes, ByteBuf cb)
@@ -560,9 +590,20 @@
*/
public static byte[] readRawBytes(ByteBuf cb)
{
- byte[] bytes = new byte[cb.readableBytes()];
+ return readRawBytes(cb, cb.readableBytes());
+ }
+
+ private static byte[] readRawBytes(ByteBuf cb, int length)
+ {
+ byte[] bytes = new byte[length];
cb.readBytes(bytes);
return bytes;
}
+ public static int readUnsignedShort(ByteBuf buf)
+ {
+ int ch1 = buf.readByte() & 0xFF;
+ int ch2 = buf.readByte() & 0xFF;
+ return (ch1 << 8) + (ch2);
+ }
}
diff --git a/src/java/org/apache/cassandra/transport/Client.java b/src/java/org/apache/cassandra/transport/Client.java
index 368b1d7..ec86579 100644
--- a/src/java/org/apache/cassandra/transport/Client.java
+++ b/src/java/org/apache/cassandra/transport/Client.java
@@ -28,24 +28,29 @@
import org.apache.cassandra.auth.PasswordAuthenticator;
import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.config.EncryptionOptions;
import org.apache.cassandra.cql3.QueryOptions;
import org.apache.cassandra.db.ConsistencyLevel;
import org.apache.cassandra.db.marshal.Int32Type;
import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.transport.frame.checksum.ChecksummingTransformer;
+import org.apache.cassandra.transport.frame.compress.CompressingTransformer;
+import org.apache.cassandra.transport.frame.compress.Compressor;
+import org.apache.cassandra.transport.frame.compress.LZ4Compressor;
+import org.apache.cassandra.transport.frame.compress.SnappyCompressor;
import org.apache.cassandra.transport.messages.*;
+import org.apache.cassandra.utils.ChecksumType;
import org.apache.cassandra.utils.Hex;
import org.apache.cassandra.utils.JVMStabilityInspector;
import org.apache.cassandra.utils.MD5Digest;
-import static org.apache.cassandra.config.EncryptionOptions.ClientEncryptionOptions;
-
public class Client extends SimpleClient
{
private final SimpleEventHandler eventHandler = new SimpleEventHandler();
- public Client(String host, int port, ProtocolVersion version, ClientEncryptionOptions encryptionOptions)
+ public Client(String host, int port, ProtocolVersion version, EncryptionOptions encryptionOptions)
{
- super(host, port, version, encryptionOptions);
+ super(host, port, version, version.isBeta(), encryptionOptions);
setEventHandler(eventHandler);
}
@@ -106,15 +111,56 @@
{
Map<String, String> options = new HashMap<String, String>();
options.put(StartupMessage.CQL_VERSION, "3.0.0");
+ Compressor compressor = null;
+ ChecksumType checksumType = null;
while (iter.hasNext())
{
- String next = iter.next();
- if (next.toLowerCase().equals("snappy"))
+ String next = iter.next().toLowerCase();
+ switch (next)
{
- options.put(StartupMessage.COMPRESSION, "snappy");
- connection.setCompressor(FrameCompressor.SnappyCompressor.instance);
+ case "snappy": {
+ if (options.containsKey(StartupMessage.COMPRESSION))
+ throw new RuntimeException("Multiple compression types supplied");
+ options.put(StartupMessage.COMPRESSION, "snappy");
+ compressor = SnappyCompressor.INSTANCE;
+ break;
+ }
+ case "lz4": {
+ if (options.containsKey(StartupMessage.COMPRESSION))
+ throw new RuntimeException("Multiple compression types supplied");
+ options.put(StartupMessage.COMPRESSION, "lz4");
+ compressor = LZ4Compressor.INSTANCE;
+ break;
+ }
+ case "crc32": {
+ if (options.containsKey(StartupMessage.CHECKSUM))
+ throw new RuntimeException("Multiple checksum types supplied");
+ options.put(StartupMessage.CHECKSUM, ChecksumType.CRC32.name());
+ checksumType = ChecksumType.CRC32;
+ break;
+ }
+ case "adler32": {
+ if (options.containsKey(StartupMessage.CHECKSUM))
+ throw new RuntimeException("Multiple checksum types supplied");
+ options.put(StartupMessage.CHECKSUM, ChecksumType.ADLER32.name());
+ checksumType = ChecksumType.ADLER32;
+ break;
+ }
}
}
+
+ if (checksumType == null)
+ {
+ if (compressor != null)
+ {
+ connection.setTransformer(CompressingTransformer.getTransformer(compressor));
+ }
+ }
+ else
+ {
+ connection.setTransformer(ChecksummingTransformer.getTransformer(checksumType, compressor));
+ }
+
return new StartupMessage(options);
}
else if (msgType.equals("QUERY"))
@@ -136,18 +182,20 @@
return null;
}
}
- return new QueryMessage(query, QueryOptions.create(ConsistencyLevel.ONE, Collections.<ByteBuffer>emptyList(), false, pageSize, null, null, version));
+ return new QueryMessage(query, QueryOptions.create(ConsistencyLevel.ONE, Collections.<ByteBuffer>emptyList(), false, pageSize, null, null, version, null));
}
else if (msgType.equals("PREPARE"))
{
String query = line.substring(8);
- return new PrepareMessage(query);
+ return new PrepareMessage(query, null);
}
else if (msgType.equals("EXECUTE"))
{
try
{
- byte[] id = Hex.hexToBytes(iter.next());
+ byte[] preparedStatementId = Hex.hexToBytes(iter.next());
+ byte[] resultMetadataId = Hex.hexToBytes(iter.next());
+
List<ByteBuffer> values = new ArrayList<ByteBuffer>();
while(iter.hasNext())
{
@@ -164,7 +212,7 @@
}
values.add(bb);
}
- return new ExecuteMessage(MD5Digest.wrap(id), QueryOptions.forInternalCalls(ConsistencyLevel.ONE, values));
+ return new ExecuteMessage(MD5Digest.wrap(preparedStatementId), MD5Digest.wrap(resultMetadataId), QueryOptions.forInternalCalls(ConsistencyLevel.ONE, values));
}
catch (Exception e)
{
@@ -175,13 +223,6 @@
{
return new OptionsMessage();
}
- else if (msgType.equals("CREDENTIALS"))
- {
- System.err.println("[WARN] CREDENTIALS command is deprecated, use AUTHENTICATE instead");
- CredentialsMessage msg = new CredentialsMessage();
- msg.credentials.putAll(readCredentials(iter));
- return msg;
- }
else if (msgType.equals("AUTHENTICATE"))
{
Map<String, String> credentials = readCredentials(iter);
@@ -251,9 +292,9 @@
// Parse options.
String host = args[0];
int port = Integer.parseInt(args[1]);
- ProtocolVersion version = args.length == 3 ? ProtocolVersion.decode(Integer.parseInt(args[2]), ProtocolVersionLimit.SERVER_DEFAULT) : ProtocolVersion.CURRENT;
+ ProtocolVersion version = args.length == 3 ? ProtocolVersion.decode(Integer.parseInt(args[2]), DatabaseDescriptor.getNativeTransportAllowOlderProtocols()) : ProtocolVersion.CURRENT;
- ClientEncryptionOptions encryptionOptions = new ClientEncryptionOptions();
+ EncryptionOptions encryptionOptions = new EncryptionOptions();
System.out.println("CQL binary protocol console " + host + "@" + port + " using native protocol version " + version);
new Client(host, port, version, encryptionOptions).run();
diff --git a/src/java/org/apache/cassandra/transport/ClientStat.java b/src/java/org/apache/cassandra/transport/ClientStat.java
new file mode 100644
index 0000000..7e78597
--- /dev/null
+++ b/src/java/org/apache/cassandra/transport/ClientStat.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.transport;
+
+import java.net.InetAddress;
+import java.util.Map;
+
+import com.google.common.collect.ImmutableMap;
+
+public final class ClientStat
+{
+ public static final String INET_ADDRESS = "inetAddress";
+ public static final String PROTOCOL_VERSION = "protocolVersion";
+ public static final String LAST_SEEN_TIME = "lastSeenTime";
+
+ final InetAddress remoteAddress;
+ final ProtocolVersion protocolVersion;
+ final long lastSeenTime;
+
+ ClientStat(InetAddress remoteAddress, ProtocolVersion protocolVersion, long lastSeenTime)
+ {
+ this.remoteAddress = remoteAddress;
+ this.lastSeenTime = lastSeenTime;
+ this.protocolVersion = protocolVersion;
+ }
+
+ @Override
+ public String toString()
+ {
+ return String.format("ClientStat{%s, %s, %d}", remoteAddress, protocolVersion, lastSeenTime);
+ }
+
+ public Map<String, String> asMap()
+ {
+ return ImmutableMap.<String, String>builder()
+ .put(INET_ADDRESS, remoteAddress.toString())
+ .put(PROTOCOL_VERSION, protocolVersion.toString())
+ .put(LAST_SEEN_TIME, String.valueOf(lastSeenTime))
+ .build();
+ }
+}
diff --git a/src/java/org/apache/cassandra/transport/ConfiguredLimit.java b/src/java/org/apache/cassandra/transport/ConfiguredLimit.java
deleted file mode 100644
index 16d3867..0000000
--- a/src/java/org/apache/cassandra/transport/ConfiguredLimit.java
+++ /dev/null
@@ -1,112 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.transport;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.db.SystemKeyspace;
-import org.apache.cassandra.utils.CassandraVersion;
-
-public abstract class ConfiguredLimit implements ProtocolVersionLimit
-{
- private static final Logger logger = LoggerFactory.getLogger(ConfiguredLimit.class);
- static final String DISABLE_MAX_PROTOCOL_AUTO_OVERRIDE = "cassandra.disable_max_protocol_auto_override";
- static final CassandraVersion MIN_VERSION_FOR_V4 = new CassandraVersion("3.0.0");
-
- public abstract ProtocolVersion getMaxVersion();
- public abstract void updateMaxSupportedVersion();
-
- public static ConfiguredLimit newLimit()
- {
- if (Boolean.getBoolean(DISABLE_MAX_PROTOCOL_AUTO_OVERRIDE))
- return new StaticLimit(ProtocolVersion.MAX_SUPPORTED_VERSION);
-
- int fromConfig = DatabaseDescriptor.getNativeProtocolMaxVersionOverride();
- return fromConfig != Integer.MIN_VALUE
- ? new StaticLimit(ProtocolVersion.decode(fromConfig, ProtocolVersionLimit.SERVER_DEFAULT))
- : new DynamicLimit(ProtocolVersion.MAX_SUPPORTED_VERSION);
- }
-
- private static class StaticLimit extends ConfiguredLimit
- {
- private final ProtocolVersion maxVersion;
- private StaticLimit(ProtocolVersion maxVersion)
- {
- this.maxVersion = maxVersion;
- logger.info("Native transport max negotiable version statically limited to {}", maxVersion);
- }
-
- public ProtocolVersion getMaxVersion()
- {
- return maxVersion;
- }
-
- public void updateMaxSupportedVersion()
- {
- // statically configured, so this is a no-op
- }
- }
-
- private static class DynamicLimit extends ConfiguredLimit
- {
- private volatile ProtocolVersion maxVersion;
- private DynamicLimit(ProtocolVersion initialLimit)
- {
- maxVersion = initialLimit;
- maybeUpdateVersion(true);
- }
-
- public ProtocolVersion getMaxVersion()
- {
- return maxVersion;
- }
-
- public void updateMaxSupportedVersion()
- {
- maybeUpdateVersion(false);
- }
-
- private void maybeUpdateVersion(boolean allowLowering)
- {
- boolean enforceV3Cap = SystemKeyspace.loadPeerVersions()
- .values()
- .stream()
- .anyMatch(v -> v.compareTo(MIN_VERSION_FOR_V4) < 0);
-
- if (!enforceV3Cap)
- {
- maxVersion = ProtocolVersion.MAX_SUPPORTED_VERSION;
- return;
- }
-
- if (ProtocolVersion.V3.isSmallerThan(maxVersion) && !allowLowering)
- {
- logger.info("Detected peers which do not fully support protocol V4, but V4 was previously negotiable. " +
- "Not enforcing cap as this can cause issues for older client versions. After the next " +
- "restart the server will apply the cap");
- return;
- }
-
- logger.info("Detected peers which do not fully support protocol V4. Capping max negotiable version to V3");
- maxVersion = ProtocolVersion.V3;
- }
- }
-}
diff --git a/src/java/org/apache/cassandra/transport/ConnectedClient.java b/src/java/org/apache/cassandra/transport/ConnectedClient.java
new file mode 100644
index 0000000..ca100f2
--- /dev/null
+++ b/src/java/org/apache/cassandra/transport/ConnectedClient.java
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.transport;
+
+import java.net.InetSocketAddress;
+import java.util.Map;
+import java.util.Optional;
+
+import com.google.common.collect.ImmutableMap;
+
+import io.netty.handler.ssl.SslHandler;
+import org.apache.cassandra.auth.AuthenticatedUser;
+import org.apache.cassandra.service.ClientState;
+
+public final class ConnectedClient
+{
+ public static final String ADDRESS = "address";
+ public static final String USER = "user";
+ public static final String VERSION = "version";
+ public static final String DRIVER_NAME = "driverName";
+ public static final String DRIVER_VERSION = "driverVersion";
+ public static final String REQUESTS = "requests";
+ public static final String KEYSPACE = "keyspace";
+ public static final String SSL = "ssl";
+ public static final String CIPHER = "cipher";
+ public static final String PROTOCOL = "protocol";
+
+ private static final String UNDEFINED = "undefined";
+
+ private final ServerConnection connection;
+
+ ConnectedClient(ServerConnection connection)
+ {
+ this.connection = connection;
+ }
+
+ public ConnectionStage stage()
+ {
+ return connection.stage();
+ }
+
+ public InetSocketAddress remoteAddress()
+ {
+ return state().getRemoteAddress();
+ }
+
+ public Optional<String> username()
+ {
+ AuthenticatedUser user = state().getUser();
+
+ return null != user
+ ? Optional.of(user.getName())
+ : Optional.empty();
+ }
+
+ public int protocolVersion()
+ {
+ return connection.getVersion().asInt();
+ }
+
+ public Optional<String> driverName()
+ {
+ return state().getDriverName();
+ }
+
+ public Optional<String> driverVersion()
+ {
+ return state().getDriverVersion();
+ }
+
+ public long requestCount()
+ {
+ return connection.requests.getCount();
+ }
+
+ public Optional<String> keyspace()
+ {
+ return Optional.ofNullable(state().getRawKeyspace());
+ }
+
+ public boolean sslEnabled()
+ {
+ return null != sslHandler();
+ }
+
+ public Optional<String> sslCipherSuite()
+ {
+ SslHandler sslHandler = sslHandler();
+
+ return null != sslHandler
+ ? Optional.of(sslHandler.engine().getSession().getCipherSuite())
+ : Optional.empty();
+ }
+
+ public Optional<String> sslProtocol()
+ {
+ SslHandler sslHandler = sslHandler();
+
+ return null != sslHandler
+ ? Optional.of(sslHandler.engine().getSession().getProtocol())
+ : Optional.empty();
+ }
+
+ private ClientState state()
+ {
+ return connection.getClientState();
+ }
+
+ private SslHandler sslHandler()
+ {
+ return connection.channel().pipeline().get(SslHandler.class);
+ }
+
+ public Map<String, String> asMap()
+ {
+ return ImmutableMap.<String, String>builder()
+ .put(ADDRESS, remoteAddress().toString())
+ .put(USER, username().orElse(UNDEFINED))
+ .put(VERSION, String.valueOf(protocolVersion()))
+ .put(DRIVER_NAME, driverName().orElse(UNDEFINED))
+ .put(DRIVER_VERSION, driverVersion().orElse(UNDEFINED))
+ .put(REQUESTS, String.valueOf(requestCount()))
+ .put(KEYSPACE, keyspace().orElse(""))
+ .put(SSL, Boolean.toString(sslEnabled()))
+ .put(CIPHER, sslCipherSuite().orElse(UNDEFINED))
+ .put(PROTOCOL, sslProtocol().orElse(UNDEFINED))
+ .build();
+ }
+}
diff --git a/src/java/org/apache/cassandra/transport/Connection.java b/src/java/org/apache/cassandra/transport/Connection.java
index 7e17f46..b7f5b17 100644
--- a/src/java/org/apache/cassandra/transport/Connection.java
+++ b/src/java/org/apache/cassandra/transport/Connection.java
@@ -19,6 +19,7 @@
import io.netty.channel.Channel;
import io.netty.util.AttributeKey;
+import org.apache.cassandra.transport.frame.FrameBodyTransformer;
public class Connection
{
@@ -28,7 +29,7 @@
private final ProtocolVersion version;
private final Tracker tracker;
- private volatile FrameCompressor frameCompressor;
+ private volatile FrameBodyTransformer transformer;
private boolean throwOnOverload;
public Connection(Channel channel, ProtocolVersion version, Tracker tracker)
@@ -40,14 +41,14 @@
tracker.addConnection(channel, this);
}
- public void setCompressor(FrameCompressor compressor)
+ public void setTransformer(FrameBodyTransformer transformer)
{
- this.frameCompressor = compressor;
+ this.transformer = transformer;
}
- public FrameCompressor getCompressor()
+ public FrameBodyTransformer getTransformer()
{
- return frameCompressor;
+ return transformer;
}
public void setThrowOnOverload(boolean throwOnOverload)
diff --git a/src/java/org/apache/cassandra/transport/ConnectionLimitHandler.java b/src/java/org/apache/cassandra/transport/ConnectionLimitHandler.java
index 7bcf280..2269036 100644
--- a/src/java/org/apache/cassandra/transport/ConnectionLimitHandler.java
+++ b/src/java/org/apache/cassandra/transport/ConnectionLimitHandler.java
@@ -22,6 +22,8 @@
import io.netty.channel.ChannelHandlerContext;
import io.netty.channel.ChannelInboundHandlerAdapter;
import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.utils.NoSpamLogger;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -29,6 +31,7 @@
import java.net.InetSocketAddress;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
@@ -40,6 +43,8 @@
final class ConnectionLimitHandler extends ChannelInboundHandlerAdapter
{
private static final Logger logger = LoggerFactory.getLogger(ConnectionLimitHandler.class);
+ private static final NoSpamLogger noSpamLogger = NoSpamLogger.getLogger(logger, 1L, TimeUnit.MINUTES);
+
private final ConcurrentMap<InetAddress, AtomicLong> connectionsPerClient = new ConcurrentHashMap<>();
private final AtomicLong counter = new AtomicLong(0);
@@ -56,7 +61,7 @@
if (count > limit)
{
// The decrement will be done in channelClosed(...)
- logger.warn("Exceeded maximum native connection limit of {} by using {} connections", limit, count);
+ noSpamLogger.error("Exceeded maximum native connection limit of {} by using {} connections (see native_transport_max_concurrent_connections in cassandra.yaml)", limit, count);
ctx.close();
}
else
@@ -80,7 +85,7 @@
if (perIpCount.incrementAndGet() > perIpLimit)
{
// The decrement will be done in channelClosed(...)
- logger.warn("Exceeded maximum native connection limit per ip of {} by using {} connections", perIpLimit, perIpCount);
+ noSpamLogger.error("Exceeded maximum native connection limit per ip of {} by using {} connections (see native_transport_max_concurrent_connections_per_ip)", perIpLimit, perIpCount);
ctx.close();
return;
}
diff --git a/src/java/org/apache/cassandra/transport/ConnectionStage.java b/src/java/org/apache/cassandra/transport/ConnectionStage.java
new file mode 100644
index 0000000..128411d
--- /dev/null
+++ b/src/java/org/apache/cassandra/transport/ConnectionStage.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.transport;
+
+public enum ConnectionStage
+{
+ ESTABLISHED, AUTHENTICATING, READY
+}
diff --git a/src/java/org/apache/cassandra/transport/DataType.java b/src/java/org/apache/cassandra/transport/DataType.java
index 6456b74..9d45e8a 100644
--- a/src/java/org/apache/cassandra/transport/DataType.java
+++ b/src/java/org/apache/cassandra/transport/DataType.java
@@ -160,12 +160,12 @@
break;
case UDT:
UserType udt = (UserType)value;
- CBUtil.writeString(udt.keyspace, cb);
- CBUtil.writeString(UTF8Type.instance.compose(udt.name), cb);
+ CBUtil.writeAsciiString(udt.keyspace, cb);
+ CBUtil.writeAsciiString(UTF8Type.instance.compose(udt.name), cb);
cb.writeShort(udt.size());
for (int i = 0; i < udt.size(); i++)
{
- CBUtil.writeString(udt.fieldName(i).toString(), cb);
+ CBUtil.writeAsciiString(udt.fieldName(i).toString(), cb);
codec.writeOne(DataType.fromType(udt.fieldType(i), version), cb, version);
}
break;
@@ -200,12 +200,12 @@
case UDT:
UserType udt = (UserType)value;
int size = 0;
- size += CBUtil.sizeOfString(udt.keyspace);
- size += CBUtil.sizeOfString(UTF8Type.instance.compose(udt.name));
+ size += CBUtil.sizeOfAsciiString(udt.keyspace);
+ size += CBUtil.sizeOfAsciiString(UTF8Type.instance.compose(udt.name));
size += 2;
for (int i = 0; i < udt.size(); i++)
{
- size += CBUtil.sizeOfString(udt.fieldName(i).toString());
+ size += CBUtil.sizeOfAsciiString(udt.fieldName(i).toString());
size += codec.oneSerializedSize(DataType.fromType(udt.fieldType(i), version), version);
}
return size;
diff --git a/src/java/org/apache/cassandra/transport/Event.java b/src/java/org/apache/cassandra/transport/Event.java
index ed77e59..c62a73f 100644
--- a/src/java/org/apache/cassandra/transport/Event.java
+++ b/src/java/org/apache/cassandra/transport/Event.java
@@ -23,7 +23,11 @@
import java.util.List;
import com.google.common.base.Objects;
+
import io.netty.buffer.ByteBuf;
+import org.apache.cassandra.cql3.functions.UDAggregate;
+import org.apache.cassandra.cql3.functions.UDFunction;
+import org.apache.cassandra.locator.InetAddressAndPort;
public abstract class Event
{
@@ -86,9 +90,9 @@
{
public final InetSocketAddress node;
- public InetAddress nodeAddress()
+ public InetAddressAndPort nodeAddressAndPort()
{
- return node.getAddress();
+ return InetAddressAndPort.getByAddressOverrideDefaults(node.getAddress(), node.getPort());
}
private NodeEvent(Type type, InetSocketAddress node)
@@ -110,19 +114,19 @@
this.change = change;
}
- public static TopologyChange newNode(InetAddress host, int port)
+ public static TopologyChange newNode(InetAddressAndPort address)
{
- return new TopologyChange(Change.NEW_NODE, new InetSocketAddress(host, port));
+ return new TopologyChange(Change.NEW_NODE, new InetSocketAddress(address.address, address.port));
}
- public static TopologyChange removedNode(InetAddress host, int port)
+ public static TopologyChange removedNode(InetAddressAndPort address)
{
- return new TopologyChange(Change.REMOVED_NODE, new InetSocketAddress(host, port));
+ return new TopologyChange(Change.REMOVED_NODE, new InetSocketAddress(address.address, address.port));
}
- public static TopologyChange movedNode(InetAddress host, int port)
+ public static TopologyChange movedNode(InetAddressAndPort address)
{
- return new TopologyChange(Change.MOVED_NODE, new InetSocketAddress(host, port));
+ return new TopologyChange(Change.MOVED_NODE, new InetSocketAddress(address.address, address.port));
}
// Assumes the type has already been deserialized
@@ -181,14 +185,14 @@
this.status = status;
}
- public static StatusChange nodeUp(InetAddress host, int port)
+ public static StatusChange nodeUp(InetAddressAndPort address)
{
- return new StatusChange(Status.UP, new InetSocketAddress(host, port));
+ return new StatusChange(Status.UP, new InetSocketAddress(address.address, address.port));
}
- public static StatusChange nodeDown(InetAddress host, int port)
+ public static StatusChange nodeDown(InetAddressAndPort address)
{
- return new StatusChange(Status.DOWN, new InetSocketAddress(host, port));
+ return new StatusChange(Status.DOWN, new InetSocketAddress(address.address, address.port));
}
// Assumes the type has already been deserialized
@@ -267,6 +271,16 @@
this(change, Target.KEYSPACE, keyspace, null);
}
+ public static SchemaChange forFunction(Change change, UDFunction function)
+ {
+ return new SchemaChange(change, Target.FUNCTION, function.name().keyspace, function.name().name, function.argumentsList());
+ }
+
+ public static SchemaChange forAggregate(Change change, UDAggregate aggregate)
+ {
+ return new SchemaChange(change, Target.AGGREGATE, aggregate.name().keyspace, aggregate.name().name, aggregate.argumentsList());
+ }
+
// Assumes the type has already been deserialized
public static SchemaChange deserializeEvent(ByteBuf cb, ProtocolVersion version)
{
@@ -299,8 +313,8 @@
// available since protocol version 4
CBUtil.writeEnumValue(change, dest);
CBUtil.writeEnumValue(target, dest);
- CBUtil.writeString(keyspace, dest);
- CBUtil.writeString(name, dest);
+ CBUtil.writeAsciiString(keyspace, dest);
+ CBUtil.writeAsciiString(name, dest);
CBUtil.writeStringList(argTypes, dest);
}
else
@@ -309,8 +323,8 @@
CBUtil.writeEnumValue(Change.UPDATED, dest);
if (version.isGreaterOrEqualTo(ProtocolVersion.V3))
CBUtil.writeEnumValue(Target.KEYSPACE, dest);
- CBUtil.writeString(keyspace, dest);
- CBUtil.writeString("", dest);
+ CBUtil.writeAsciiString(keyspace, dest);
+ CBUtil.writeAsciiString("", dest);
}
return;
}
@@ -319,9 +333,9 @@
{
CBUtil.writeEnumValue(change, dest);
CBUtil.writeEnumValue(target, dest);
- CBUtil.writeString(keyspace, dest);
+ CBUtil.writeAsciiString(keyspace, dest);
if (target != Target.KEYSPACE)
- CBUtil.writeString(name, dest);
+ CBUtil.writeAsciiString(name, dest);
}
else
{
@@ -330,14 +344,14 @@
// For the v1/v2 protocol, we have no way to represent type changes, so we simply say the keyspace
// was updated. See CASSANDRA-7617.
CBUtil.writeEnumValue(Change.UPDATED, dest);
- CBUtil.writeString(keyspace, dest);
- CBUtil.writeString("", dest);
+ CBUtil.writeAsciiString(keyspace, dest);
+ CBUtil.writeAsciiString("", dest);
}
else
{
CBUtil.writeEnumValue(change, dest);
- CBUtil.writeString(keyspace, dest);
- CBUtil.writeString(target == Target.KEYSPACE ? "" : name, dest);
+ CBUtil.writeAsciiString(keyspace, dest);
+ CBUtil.writeAsciiString(target == Target.KEYSPACE ? "" : name, dest);
}
}
}
@@ -349,26 +363,26 @@
if (version.isGreaterOrEqualTo(ProtocolVersion.V4))
return CBUtil.sizeOfEnumValue(change)
+ CBUtil.sizeOfEnumValue(target)
- + CBUtil.sizeOfString(keyspace)
- + CBUtil.sizeOfString(name)
+ + CBUtil.sizeOfAsciiString(keyspace)
+ + CBUtil.sizeOfAsciiString(name)
+ CBUtil.sizeOfStringList(argTypes);
if (version.isGreaterOrEqualTo(ProtocolVersion.V3))
return CBUtil.sizeOfEnumValue(Change.UPDATED)
+ CBUtil.sizeOfEnumValue(Target.KEYSPACE)
- + CBUtil.sizeOfString(keyspace);
+ + CBUtil.sizeOfAsciiString(keyspace);
return CBUtil.sizeOfEnumValue(Change.UPDATED)
- + CBUtil.sizeOfString(keyspace)
- + CBUtil.sizeOfString("");
+ + CBUtil.sizeOfAsciiString(keyspace)
+ + CBUtil.sizeOfAsciiString("");
}
if (version.isGreaterOrEqualTo(ProtocolVersion.V3))
{
int size = CBUtil.sizeOfEnumValue(change)
+ CBUtil.sizeOfEnumValue(target)
- + CBUtil.sizeOfString(keyspace);
+ + CBUtil.sizeOfAsciiString(keyspace);
if (target != Target.KEYSPACE)
- size += CBUtil.sizeOfString(name);
+ size += CBUtil.sizeOfAsciiString(name);
return size;
}
@@ -377,12 +391,12 @@
if (target == Target.TYPE)
{
return CBUtil.sizeOfEnumValue(Change.UPDATED)
- + CBUtil.sizeOfString(keyspace)
- + CBUtil.sizeOfString("");
+ + CBUtil.sizeOfAsciiString(keyspace)
+ + CBUtil.sizeOfAsciiString("");
}
return CBUtil.sizeOfEnumValue(change)
- + CBUtil.sizeOfString(keyspace)
- + CBUtil.sizeOfString(target == Target.KEYSPACE ? "" : name);
+ + CBUtil.sizeOfAsciiString(keyspace)
+ + CBUtil.sizeOfAsciiString(target == Target.KEYSPACE ? "" : name);
}
}
diff --git a/src/java/org/apache/cassandra/transport/Frame.java b/src/java/org/apache/cassandra/transport/Frame.java
index e603e6c..b597cc2 100644
--- a/src/java/org/apache/cassandra/transport/Frame.java
+++ b/src/java/org/apache/cassandra/transport/Frame.java
@@ -22,6 +22,8 @@
import java.util.EnumSet;
import java.util.List;
+import com.google.common.annotations.VisibleForTesting;
+
import io.netty.buffer.ByteBuf;
import io.netty.channel.*;
import io.netty.handler.codec.ByteToMessageDecoder;
@@ -30,6 +32,8 @@
import io.netty.util.Attribute;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.metrics.ClientRequestSizeMetrics;
+import org.apache.cassandra.transport.frame.FrameBodyTransformer;
import org.apache.cassandra.transport.messages.ErrorMessage;
public class Frame
@@ -102,7 +106,8 @@
TRACING,
CUSTOM_PAYLOAD,
WARNING,
- USE_BETA;
+ USE_BETA,
+ CHECKSUMMED;
private static final Flag[] ALL_VALUES = values();
@@ -142,16 +147,14 @@
private int tooLongStreamId;
private final Connection.Factory factory;
- private final ProtocolVersionLimit versionCap;
- public Decoder(Connection.Factory factory, ProtocolVersionLimit versionCap)
+ public Decoder(Connection.Factory factory)
{
this.factory = factory;
- this.versionCap = versionCap;
}
- @Override
- protected void decode(ChannelHandlerContext ctx, ByteBuf buffer, List<Object> results)
+ @VisibleForTesting
+ Frame decodeFrame(ByteBuf buffer)
throws Exception
{
if (discardingTooLongFrame)
@@ -160,12 +163,12 @@
// If we have discarded everything, throw the exception
if (bytesToDiscard <= 0)
fail();
- return;
+ return null;
}
int readableBytes = buffer.readableBytes();
if (readableBytes == 0)
- return;
+ return null;
int idx = buffer.readerIndex();
@@ -174,11 +177,11 @@
int firstByte = buffer.getByte(idx++);
Message.Direction direction = Message.Direction.extractFromVersion(firstByte);
int versionNum = firstByte & PROTOCOL_VERSION_MASK;
- ProtocolVersion version = ProtocolVersion.decode(versionNum, versionCap);
+ ProtocolVersion version = ProtocolVersion.decode(versionNum, DatabaseDescriptor.getNativeTransportAllowOlderProtocols());
// Wait until we have the complete header
if (readableBytes < Header.LENGTH)
- return;
+ return null;
int flags = buffer.getByte(idx++);
EnumSet<Header.Flag> decodedFlags = Header.Flag.deserialize(flags);
@@ -214,11 +217,14 @@
bytesToDiscard = discard(buffer, frameLength);
if (bytesToDiscard <= 0)
fail();
- return;
+ return null;
}
if (buffer.readableBytes() < frameLength)
- return;
+ return null;
+
+ ClientRequestSizeMetrics.totalBytesRead.inc(frameLength);
+ ClientRequestSizeMetrics.bytesRecievedPerFrame.update(frameLength);
// extract body
ByteBuf body = buffer.slice(idx, (int) bodyLength);
@@ -227,24 +233,33 @@
idx += bodyLength;
buffer.readerIndex(idx);
+ return new Frame(new Header(version, decodedFlags, streamId, type, bodyLength), body);
+ }
+
+ @Override
+ protected void decode(ChannelHandlerContext ctx, ByteBuf buffer, List<Object> results)
+ throws Exception
+ {
+ Frame frame = decodeFrame(buffer);
+ if (frame == null) return;
+
Attribute<Connection> attrConn = ctx.channel().attr(Connection.attributeKey);
Connection connection = attrConn.get();
if (connection == null)
{
// First message seen on this channel, attach the connection object
- connection = factory.newConnection(ctx.channel(), version);
+ connection = factory.newConnection(ctx.channel(), frame.header.version);
attrConn.set(connection);
}
- else if (connection.getVersion() != version)
+ else if (connection.getVersion() != frame.header.version)
{
throw ErrorMessage.wrap(
new ProtocolException(String.format(
"Invalid message version. Got %s but previous messages on this connection had version %s",
- version, connection.getVersion())),
- streamId);
+ frame.header.version, connection.getVersion())),
+ frame.header.streamId);
}
-
- results.add(new Frame(new Header(version, decodedFlags, streamId, type, bodyLength), body));
+ results.add(frame);
}
private void fail()
@@ -288,60 +303,80 @@
header.writeByte(type.opcode);
header.writeInt(frame.body.readableBytes());
+ int messageSize = header.readableBytes() + frame.body.readableBytes();
+ ClientRequestSizeMetrics.totalBytesWritten.inc(messageSize);
+ ClientRequestSizeMetrics.bytesTransmittedPerFrame.update(messageSize);
+
results.add(header);
results.add(frame.body);
}
}
@ChannelHandler.Sharable
- public static class Decompressor extends MessageToMessageDecoder<Frame>
+ public static class InboundBodyTransformer extends MessageToMessageDecoder<Frame>
{
public void decode(ChannelHandlerContext ctx, Frame frame, List<Object> results)
throws IOException
{
Connection connection = ctx.channel().attr(Connection.attributeKey).get();
- if (!frame.header.flags.contains(Header.Flag.COMPRESSED) || connection == null)
+ if ((!frame.header.flags.contains(Header.Flag.COMPRESSED) && !frame.header.flags.contains(Header.Flag.CHECKSUMMED)) || connection == null)
{
results.add(frame);
return;
}
- FrameCompressor compressor = connection.getCompressor();
- if (compressor == null)
+ FrameBodyTransformer transformer = connection.getTransformer();
+ if (transformer == null)
{
results.add(frame);
return;
}
- results.add(compressor.decompress(frame));
+ try
+ {
+ results.add(frame.with(transformer.transformInbound(frame.body, frame.header.flags)));
+ }
+ finally
+ {
+ // release the old frame
+ frame.release();
+ }
}
}
@ChannelHandler.Sharable
- public static class Compressor extends MessageToMessageEncoder<Frame>
+ public static class OutboundBodyTransformer extends MessageToMessageEncoder<Frame>
{
public void encode(ChannelHandlerContext ctx, Frame frame, List<Object> results)
throws IOException
{
Connection connection = ctx.channel().attr(Connection.attributeKey).get();
- // Never compress STARTUP messages
+ // Never transform STARTUP messages
if (frame.header.type == Message.Type.STARTUP || connection == null)
{
results.add(frame);
return;
}
- FrameCompressor compressor = connection.getCompressor();
- if (compressor == null)
+ FrameBodyTransformer transformer = connection.getTransformer();
+ if (transformer == null)
{
results.add(frame);
return;
}
- frame.header.flags.add(Header.Flag.COMPRESSED);
- results.add(compressor.compress(frame));
+ try
+ {
+ results.add(frame.with(transformer.transformOutbound(frame.body)));
+ frame.header.flags.addAll(transformer.getOutboundHeaderFlags());
+ }
+ finally
+ {
+ // release the old frame
+ frame.release();
+ }
}
}
}
diff --git a/src/java/org/apache/cassandra/transport/FrameCompressor.java b/src/java/org/apache/cassandra/transport/FrameCompressor.java
deleted file mode 100644
index 01c0c31..0000000
--- a/src/java/org/apache/cassandra/transport/FrameCompressor.java
+++ /dev/null
@@ -1,211 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.transport;
-
-import java.io.IOException;
-
-import io.netty.buffer.ByteBuf;
-import org.xerial.snappy.Snappy;
-import org.xerial.snappy.SnappyError;
-
-import net.jpountz.lz4.LZ4Factory;
-
-import org.apache.cassandra.utils.JVMStabilityInspector;
-
-public interface FrameCompressor
-{
- public Frame compress(Frame frame) throws IOException;
- public Frame decompress(Frame frame) throws IOException;
-
- /*
- * TODO: We can probably do more efficient, like by avoiding copy.
- * Also, we don't reuse ICompressor because the API doesn't expose enough.
- */
- public static class SnappyCompressor implements FrameCompressor
- {
- public static final SnappyCompressor instance;
- static
- {
- SnappyCompressor i;
- try
- {
- i = new SnappyCompressor();
- }
- catch (Exception e)
- {
- JVMStabilityInspector.inspectThrowable(e);
- i = null;
- }
- catch (NoClassDefFoundError | SnappyError | UnsatisfiedLinkError e)
- {
- i = null;
- }
- instance = i;
- }
-
- private SnappyCompressor()
- {
- // this would throw java.lang.NoClassDefFoundError if Snappy class
- // wasn't found at runtime which should be processed by the calling method
- Snappy.getNativeLibraryVersion();
- }
-
- public Frame compress(Frame frame) throws IOException
- {
- byte[] input = CBUtil.readRawBytes(frame.body);
- ByteBuf output = CBUtil.allocator.heapBuffer(Snappy.maxCompressedLength(input.length));
-
- try
- {
- int written = Snappy.compress(input, 0, input.length, output.array(), output.arrayOffset());
- output.writerIndex(written);
- }
- catch (final Throwable e)
- {
- output.release();
- throw e;
- }
- finally
- {
- //release the old frame
- frame.release();
- }
-
- return frame.with(output);
- }
-
- public Frame decompress(Frame frame) throws IOException
- {
- byte[] input = CBUtil.readRawBytes(frame.body);
-
- if (!Snappy.isValidCompressedBuffer(input, 0, input.length))
- throw new ProtocolException("Provided frame does not appear to be Snappy compressed");
-
- ByteBuf output = CBUtil.allocator.heapBuffer(Snappy.uncompressedLength(input));
-
- try
- {
- int size = Snappy.uncompress(input, 0, input.length, output.array(), output.arrayOffset());
- output.writerIndex(size);
- }
- catch (final Throwable e)
- {
- output.release();
- throw e;
- }
- finally
- {
- //release the old frame
- frame.release();
- }
-
- return frame.with(output);
- }
- }
-
- /*
- * This is very close to the ICompressor implementation, and in particular
- * it also layout the uncompressed size at the beginning of the message to
- * make uncompression faster, but contrarly to the ICompressor, that length
- * is written in big-endian. The native protocol is entirely big-endian, so
- * it feels like putting little-endian here would be a annoying trap for
- * client writer.
- */
- public static class LZ4Compressor implements FrameCompressor
- {
- public static final LZ4Compressor instance = new LZ4Compressor();
-
- private static final int INTEGER_BYTES = 4;
- private final net.jpountz.lz4.LZ4Compressor compressor;
- private final net.jpountz.lz4.LZ4Decompressor decompressor;
-
- private LZ4Compressor()
- {
- final LZ4Factory lz4Factory = LZ4Factory.fastestInstance();
- compressor = lz4Factory.fastCompressor();
- decompressor = lz4Factory.decompressor();
- }
-
- public Frame compress(Frame frame) throws IOException
- {
- byte[] input = CBUtil.readRawBytes(frame.body);
-
- int maxCompressedLength = compressor.maxCompressedLength(input.length);
- ByteBuf outputBuf = CBUtil.allocator.heapBuffer(INTEGER_BYTES + maxCompressedLength);
-
- byte[] output = outputBuf.array();
- int outputOffset = outputBuf.arrayOffset();
-
- output[outputOffset + 0] = (byte) (input.length >>> 24);
- output[outputOffset + 1] = (byte) (input.length >>> 16);
- output[outputOffset + 2] = (byte) (input.length >>> 8);
- output[outputOffset + 3] = (byte) (input.length);
-
- try
- {
- int written = compressor.compress(input, 0, input.length, output, outputOffset + INTEGER_BYTES, maxCompressedLength);
- outputBuf.writerIndex(INTEGER_BYTES + written);
-
- return frame.with(outputBuf);
- }
- catch (final Throwable e)
- {
- outputBuf.release();
- throw e;
- }
- finally
- {
- //release the old frame
- frame.release();
- }
- }
-
- public Frame decompress(Frame frame) throws IOException
- {
- byte[] input = CBUtil.readRawBytes(frame.body);
-
- int uncompressedLength = ((input[0] & 0xFF) << 24)
- | ((input[1] & 0xFF) << 16)
- | ((input[2] & 0xFF) << 8)
- | ((input[3] & 0xFF));
-
- ByteBuf output = CBUtil.allocator.heapBuffer(uncompressedLength);
-
- try
- {
- int read = decompressor.decompress(input, INTEGER_BYTES, output.array(), output.arrayOffset(), uncompressedLength);
- if (read != input.length - INTEGER_BYTES)
- throw new IOException("Compressed lengths mismatch");
-
- output.writerIndex(uncompressedLength);
-
- return frame.with(output);
- }
- catch (final Throwable e)
- {
- output.release();
- throw e;
- }
- finally
- {
- //release the old frame
- frame.release();
- }
- }
- }
-}
diff --git a/src/java/org/apache/cassandra/transport/Message.java b/src/java/org/apache/cassandra/transport/Message.java
index f899022..d43a015 100644
--- a/src/java/org/apache/cassandra/transport/Message.java
+++ b/src/java/org/apache/cassandra/transport/Message.java
@@ -48,9 +48,12 @@
import org.apache.cassandra.metrics.ClientMetrics;
import org.apache.cassandra.net.ResourceLimits;
import org.apache.cassandra.service.ClientWarn;
+import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.tracing.Tracing;
import org.apache.cassandra.transport.messages.*;
import org.apache.cassandra.service.QueryState;
import org.apache.cassandra.utils.JVMStabilityInspector;
+import org.apache.cassandra.utils.UUIDGen;
import static org.apache.cassandra.concurrent.SharedExecutorPool.SHARED;
@@ -95,7 +98,7 @@
STARTUP (1, Direction.REQUEST, StartupMessage.codec),
READY (2, Direction.RESPONSE, ReadyMessage.codec),
AUTHENTICATE (3, Direction.RESPONSE, AuthenticateMessage.codec),
- CREDENTIALS (4, Direction.REQUEST, CredentialsMessage.codec),
+ CREDENTIALS (4, Direction.REQUEST, UnsupportedMessageCodec.instance),
OPTIONS (5, Direction.REQUEST, OptionsMessage.codec),
SUPPORTED (6, Direction.RESPONSE, SupportedMessage.codec),
QUERY (7, Direction.REQUEST, QueryMessage.codec),
@@ -207,7 +210,7 @@
public static abstract class Request extends Message
{
- protected boolean tracingRequested;
+ private boolean tracingRequested;
protected Request(Type type)
{
@@ -217,14 +220,56 @@
throw new IllegalArgumentException();
}
- public abstract Response execute(QueryState queryState, long queryStartNanoTime);
-
- public void setTracingRequested()
+ protected boolean isTraceable()
{
- this.tracingRequested = true;
+ return false;
}
- public boolean isTracingRequested()
+ protected abstract Response execute(QueryState queryState, long queryStartNanoTime, boolean traceRequest);
+
+ final Response execute(QueryState queryState, long queryStartNanoTime)
+ {
+ boolean shouldTrace = false;
+ UUID tracingSessionId = null;
+
+ if (isTraceable())
+ {
+ if (isTracingRequested())
+ {
+ shouldTrace = true;
+ tracingSessionId = UUIDGen.getTimeUUID();
+ Tracing.instance.newSession(tracingSessionId, getCustomPayload());
+ }
+ else if (StorageService.instance.shouldTraceProbablistically())
+ {
+ shouldTrace = true;
+ Tracing.instance.newSession(getCustomPayload());
+ }
+ }
+
+ Response response;
+ try
+ {
+ response = execute(queryState, queryStartNanoTime, shouldTrace);
+ }
+ finally
+ {
+ if (shouldTrace)
+ Tracing.instance.stopSession();
+ }
+
+ if (isTraceable() && isTracingRequested())
+ response.setTracingId(tracingSessionId);
+
+ return response;
+ }
+
+ void setTracingRequested()
+ {
+ tracingRequested = true;
+ }
+
+ boolean isTracingRequested()
{
return tracingRequested;
}
@@ -243,18 +288,18 @@
throw new IllegalArgumentException();
}
- public Message setTracingId(UUID tracingId)
+ Message setTracingId(UUID tracingId)
{
this.tracingId = tracingId;
return this;
}
- public UUID getTracingId()
+ UUID getTracingId()
{
return tracingId;
}
- public Message setWarnings(List<String> warnings)
+ Message setWarnings(List<String> warnings)
{
this.warnings = warnings;
return this;
@@ -322,19 +367,11 @@
@ChannelHandler.Sharable
public static class ProtocolEncoder extends MessageToMessageEncoder<Message>
{
- private final ProtocolVersionLimit versionCap;
-
- ProtocolEncoder(ProtocolVersionLimit versionCap)
- {
- this.versionCap = versionCap;
- }
-
public void encode(ChannelHandlerContext ctx, Message message, List results)
{
Connection connection = ctx.channel().attr(Connection.attributeKey).get();
// The only case the connection can be null is when we send the initial STARTUP message (client side thus)
- ProtocolVersion version = connection == null ? versionCap.getMaxVersion() : connection.getVersion();
-
+ ProtocolVersion version = connection == null ? ProtocolVersion.CURRENT : connection.getVersion();
EnumSet<Frame.Header.Flag> flags = EnumSet.noneOf(Frame.Header.Flag.class);
Codec<Message> codec = (Codec<Message>)message.type.codec;
@@ -425,6 +462,7 @@
public static class Dispatcher extends SimpleChannelInboundHandler<Request>
{
private static final LocalAwareExecutorService requestExecutor = SHARED.newExecutor(DatabaseDescriptor.getNativeTransportMaxThreads(),
+ DatabaseDescriptor::setNativeTransportMaxThreads,
Integer.MAX_VALUE,
"transport",
"Native-Transport-Requests");
@@ -680,9 +718,10 @@
if (connection.getVersion().isGreaterOrEqualTo(ProtocolVersion.V4))
ClientWarn.instance.captureWarnings();
- QueryState qstate = connection.validateNewMessage(request.type, connection.getVersion(), request.getStreamId());
+ QueryState qstate = connection.validateNewMessage(request.type, connection.getVersion());
logger.trace("Received: {}, v={}", request, connection.getVersion());
+ connection.requests.inc();
response = request.execute(qstate, queryStartNanoTime);
response.setStreamId(request.getStreamId());
response.setWarnings(ClientWarn.instance.getWarnings());
@@ -800,7 +839,9 @@
message = "Unexpected exception during request; channel = <unprintable>";
}
- if (!alwaysLogAtError && exception instanceof IOException)
+ // netty wraps SSL errors in a CodecExcpetion
+ boolean isIOException = exception instanceof IOException || (exception.getCause() instanceof IOException);
+ if (!alwaysLogAtError && isIOException)
{
String errorMessage = exception.getMessage();
boolean logAtTrace = false;
diff --git a/src/java/org/apache/cassandra/transport/ProtocolVersion.java b/src/java/org/apache/cassandra/transport/ProtocolVersion.java
index 05fafc2..5c8c299 100644
--- a/src/java/org/apache/cassandra/transport/ProtocolVersion.java
+++ b/src/java/org/apache/cassandra/transport/ProtocolVersion.java
@@ -84,9 +84,18 @@
return ret;
}
- public static ProtocolVersion decode(int versionNum, ProtocolVersionLimit ceiling)
+ public static List<ProtocolVersion> supportedVersionsStartingWith(ProtocolVersion smallestVersion)
{
- ProtocolVersion ret = versionNum >= MIN_SUPPORTED_VERSION.num && versionNum <= ceiling.getMaxVersion().num
+ ArrayList<ProtocolVersion> versions = new ArrayList<>(SUPPORTED_VERSIONS.length);
+ for (ProtocolVersion version : SUPPORTED_VERSIONS)
+ if (version.isGreaterOrEqualTo(smallestVersion))
+ versions.add(version);
+ return versions;
+ }
+
+ public static ProtocolVersion decode(int versionNum, boolean allowOlderProtocols)
+ {
+ ProtocolVersion ret = versionNum >= MIN_SUPPORTED_VERSION.num && versionNum <= MAX_SUPPORTED_VERSION.num
? SUPPORTED_VERSIONS[versionNum - MIN_SUPPORTED_VERSION.num]
: null;
@@ -95,16 +104,19 @@
// if this is not a supported version check the old versions
for (ProtocolVersion version : UNSUPPORTED)
{
- // if it is an old version that is no longer supported this ensures that we reply
+ // if it is an old version that is no longer supported this ensures that we respond
// with that same version
if (version.num == versionNum)
throw new ProtocolException(ProtocolVersion.invalidVersionMessage(versionNum), version);
}
- // If the version is invalid reply with the highest version that we support
- throw new ProtocolException(invalidVersionMessage(versionNum), ceiling.getMaxVersion());
+ // If the version is invalid response with the highest version that we support
+ throw new ProtocolException(invalidVersionMessage(versionNum), MAX_SUPPORTED_VERSION);
}
+ if (!allowOlderProtocols && ret.isSmallerThan(CURRENT))
+ throw new ProtocolException(String.format("Rejecting Protocol Version %s < %s.", ret, ProtocolVersion.CURRENT));
+
return ret;
}
@@ -124,6 +136,11 @@
return num;
}
+ public boolean supportsChecksums()
+ {
+ return num >= V5.asInt();
+ }
+
@Override
public String toString()
{
diff --git a/src/java/org/apache/cassandra/transport/ProtocolVersionLimit.java b/src/java/org/apache/cassandra/transport/ProtocolVersionLimit.java
deleted file mode 100644
index 9738a19..0000000
--- a/src/java/org/apache/cassandra/transport/ProtocolVersionLimit.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.transport;
-
-@FunctionalInterface
-public interface ProtocolVersionLimit
-{
- public ProtocolVersion getMaxVersion();
-
- public static final ProtocolVersionLimit SERVER_DEFAULT = () -> ProtocolVersion.MAX_SUPPORTED_VERSION;
-}
diff --git a/src/java/org/apache/cassandra/transport/ProtocolVersionTracker.java b/src/java/org/apache/cassandra/transport/ProtocolVersionTracker.java
new file mode 100644
index 0000000..f289377
--- /dev/null
+++ b/src/java/org/apache/cassandra/transport/ProtocolVersionTracker.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.transport;
+
+import java.net.InetAddress;
+import java.util.ArrayList;
+import java.util.EnumMap;
+import java.util.List;
+
+import com.github.benmanes.caffeine.cache.Cache;
+import com.github.benmanes.caffeine.cache.Caffeine;
+import com.github.benmanes.caffeine.cache.LoadingCache;
+
+/**
+ * This class tracks the last 100 connections per protocol version
+ */
+public class ProtocolVersionTracker
+{
+ private static final int DEFAULT_MAX_CAPACITY = 100;
+
+ private final EnumMap<ProtocolVersion, LoadingCache<InetAddress, Long>> clientsByProtocolVersion;
+
+ ProtocolVersionTracker()
+ {
+ this(DEFAULT_MAX_CAPACITY);
+ }
+
+ private ProtocolVersionTracker(int capacity)
+ {
+ clientsByProtocolVersion = new EnumMap<>(ProtocolVersion.class);
+
+ for (ProtocolVersion version : ProtocolVersion.values())
+ {
+ clientsByProtocolVersion.put(version, Caffeine.newBuilder().maximumSize(capacity)
+ .build(key -> System.currentTimeMillis()));
+ }
+ }
+
+ void addConnection(InetAddress addr, ProtocolVersion version)
+ {
+ clientsByProtocolVersion.get(version).put(addr, System.currentTimeMillis());
+ }
+
+ List<ClientStat> getAll()
+ {
+ List<ClientStat> result = new ArrayList<>();
+
+ clientsByProtocolVersion.forEach((version, cache) ->
+ cache.asMap().forEach((address, lastSeenTime) -> result.add(new ClientStat(address, version, lastSeenTime))));
+
+ return result;
+ }
+
+ List<ClientStat> getAll(ProtocolVersion version)
+ {
+ List<ClientStat> result = new ArrayList<>();
+
+ clientsByProtocolVersion.get(version).asMap().forEach((address, lastSeenTime) ->
+ result.add(new ClientStat(address, version, lastSeenTime)));
+
+ return result;
+ }
+
+ public void clear()
+ {
+ clientsByProtocolVersion.values().forEach(Cache::invalidateAll);
+ }
+}
diff --git a/src/java/org/apache/cassandra/transport/Server.java b/src/java/org/apache/cassandra/transport/Server.java
index ced764f..69c87ee 100644
--- a/src/java/org/apache/cassandra/transport/Server.java
+++ b/src/java/org/apache/cassandra/transport/Server.java
@@ -23,10 +23,9 @@
import java.net.UnknownHostException;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.atomic.AtomicBoolean;
-import javax.net.ssl.SSLContext;
-import javax.net.ssl.SSLEngine;
+import java.util.concurrent.TimeUnit;
import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import org.slf4j.Logger;
@@ -34,6 +33,7 @@
import io.netty.bootstrap.ServerBootstrap;
import io.netty.buffer.ByteBuf;
+import io.netty.buffer.ByteBufAllocator;
import io.netty.channel.*;
import io.netty.channel.epoll.EpollEventLoopGroup;
import io.netty.channel.epoll.EpollServerSocketChannel;
@@ -42,16 +42,24 @@
import io.netty.channel.nio.NioEventLoopGroup;
import io.netty.channel.socket.nio.NioServerSocketChannel;
import io.netty.handler.codec.ByteToMessageDecoder;
+import io.netty.handler.ssl.SslContext;
import io.netty.handler.ssl.SslHandler;
+import io.netty.handler.timeout.IdleState;
+import io.netty.handler.timeout.IdleStateEvent;
+import io.netty.handler.timeout.IdleStateHandler;
import io.netty.util.Version;
import io.netty.util.concurrent.EventExecutor;
import io.netty.util.concurrent.GlobalEventExecutor;
import io.netty.util.internal.logging.InternalLoggerFactory;
import io.netty.util.internal.logging.Slf4JLoggerFactory;
+import org.apache.cassandra.auth.AuthenticatedUser;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.config.EncryptionOptions;
import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.net.ResourceLimits;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.SchemaChangeListener;
import org.apache.cassandra.security.SSLFactory;
import org.apache.cassandra.service.*;
import org.apache.cassandra.transport.messages.EventMessage;
@@ -82,14 +90,11 @@
private final AtomicBoolean isRunning = new AtomicBoolean(false);
private EventLoopGroup workerGroup;
- private final ProtocolVersionLimit protocolVersionLimit;
private Server (Builder builder)
{
this.socket = builder.getSocket();
this.useSSL = builder.useSSL;
- this.protocolVersionLimit = builder.getProtocolVersionLimit();
-
if (builder.workerGroup != null)
{
workerGroup = builder.workerGroup;
@@ -103,7 +108,7 @@
}
EventNotifier notifier = new EventNotifier(this);
StorageService.instance.register(notifier);
- MigrationManager.instance.register(notifier);
+ Schema.instance.registerListener(notifier);
}
public void stop()
@@ -136,7 +141,7 @@
if (this.useSSL)
{
- final EncryptionOptions.ClientEncryptionOptions clientEnc = DatabaseDescriptor.getClientEncryptionOptions();
+ final EncryptionOptions clientEnc = DatabaseDescriptor.getNativeProtocolEncryptionOptions();
if (clientEnc.optional)
{
@@ -160,15 +165,44 @@
ChannelFuture bindFuture = bootstrap.bind(socket);
if (!bindFuture.awaitUninterruptibly().isSuccess())
- throw new IllegalStateException(String.format("Failed to bind port %d on %s.", socket.getPort(), socket.getAddress().getHostAddress()));
+ throw new IllegalStateException(String.format("Failed to bind port %d on %s.", socket.getPort(), socket.getAddress().getHostAddress()),
+ bindFuture.cause());
connectionTracker.allChannels.add(bindFuture.channel());
isRunning.set(true);
}
- public int getConnectedClients()
+ public int countConnectedClients()
{
- return connectionTracker.getConnectedClients();
+ return connectionTracker.countConnectedClients();
+ }
+
+ public Map<String, Integer> countConnectedClientsByUser()
+ {
+ return connectionTracker.countConnectedClientsByUser();
+ }
+
+ public List<ConnectedClient> getConnectedClients()
+ {
+ List<ConnectedClient> result = new ArrayList<>();
+ for (Channel c : connectionTracker.allChannels)
+ {
+ Connection conn = c.attr(Connection.attributeKey).get();
+ if (conn instanceof ServerConnection)
+ result.add(new ConnectedClient((ServerConnection) conn));
+ }
+ return result;
+ }
+
+ public List<ClientStat> recentClientStats()
+ {
+ return connectionTracker.protocolVersionTracker.getAll();
+ }
+
+ @Override
+ public void clearConnectionHistory()
+ {
+ connectionTracker.protocolVersionTracker.clear();
}
private void close()
@@ -187,7 +221,6 @@
private InetAddress hostAddr;
private int port = -1;
private InetSocketAddress socket;
- private ProtocolVersionLimit versionLimit;
public Builder withSSL(boolean useSSL)
{
@@ -215,19 +248,6 @@
return this;
}
- public Builder withProtocolVersionLimit(ProtocolVersionLimit limit)
- {
- this.versionLimit = limit;
- return this;
- }
-
- ProtocolVersionLimit getProtocolVersionLimit()
- {
- if (versionLimit == null)
- throw new IllegalArgumentException("Missing protocol version limiter");
- return versionLimit;
- }
-
public Server build()
{
return new Server(this);
@@ -255,6 +275,7 @@
// TODO: should we be using the GlobalEventExecutor or defining our own?
public final ChannelGroup allChannels = new DefaultChannelGroup(GlobalEventExecutor.INSTANCE);
private final EnumMap<Event.Type, ChannelGroup> groups = new EnumMap<>(Event.Type.class);
+ private final ProtocolVersionTracker protocolVersionTracker = new ProtocolVersionTracker();
public ConnectionTracker()
{
@@ -265,6 +286,9 @@
public void addConnection(Channel ch, Connection connection)
{
allChannels.add(ch);
+
+ if (ch.remoteAddress() instanceof InetSocketAddress)
+ protocolVersionTracker.addConnection(((InetSocketAddress) ch.remoteAddress()).getAddress(), connection.getVersion());
}
public void register(Event.Type type, Channel ch)
@@ -277,12 +301,12 @@
groups.get(event.type).writeAndFlush(new EventMessage(event));
}
- public void closeAll()
+ void closeAll()
{
allChannels.close().awaitUninterruptibly();
}
- public int getConnectedClients()
+ int countConnectedClients()
{
/*
- When server is running: allChannels contains all clients' connections (channels)
@@ -291,6 +315,24 @@
*/
return allChannels.size() != 0 ? allChannels.size() - 1 : 0;
}
+
+ Map<String, Integer> countConnectedClientsByUser()
+ {
+ Map<String, Integer> result = new HashMap<>();
+ for (Channel c : allChannels)
+ {
+ Connection connection = c.attr(Connection.attributeKey).get();
+ if (connection instanceof ServerConnection)
+ {
+ ServerConnection conn = (ServerConnection) connection;
+ AuthenticatedUser user = conn.getClientState().getUser();
+ String name = (null != user) ? user.getName() : null;
+ result.put(name, result.getOrDefault(name, 0) + 1);
+ }
+ }
+ return result;
+ }
+
}
// global inflight payload across all channels across all endpoints
@@ -324,6 +366,34 @@
}
}
+ public static long getGlobalLimit()
+ {
+ return DatabaseDescriptor.getNativeTransportMaxConcurrentRequestsInBytes();
+ }
+
+ public static void setGlobalLimit(long newLimit)
+ {
+ DatabaseDescriptor.setNativeTransportMaxConcurrentRequestsInBytes(newLimit);
+ long existingLimit = globalRequestPayloadInFlight.setLimit(DatabaseDescriptor.getNativeTransportMaxConcurrentRequestsInBytes());
+
+ logger.info("Changed native_max_transport_requests_in_bytes from {} to {}", existingLimit, newLimit);
+ }
+
+ public static long getEndpointLimit()
+ {
+ return DatabaseDescriptor.getNativeTransportMaxConcurrentRequestsInBytesPerIp();
+ }
+
+ public static void setEndpointLimit(long newLimit)
+ {
+ long existingLimit = DatabaseDescriptor.getNativeTransportMaxConcurrentRequestsInBytesPerIp();
+ DatabaseDescriptor.setNativeTransportMaxConcurrentRequestsInBytesPerIp(newLimit); // ensure new trackers get the new limit
+ for (EndpointPayloadTracker tracker : requestPayloadInFlightPerEndpoint.values())
+ existingLimit = tracker.endpointAndGlobalPayloadsInFlight.endpoint().setLimit(newLimit);
+
+ logger.info("Changed native_max_transport_requests_in_bytes_per_ip from {} to {}", existingLimit, newLimit);
+ }
+
private boolean acquire()
{
return 0 < refCount.updateAndGet(i -> i < 0 ? i : i + 1);
@@ -340,8 +410,9 @@
{
// Stateless handlers
private static final Message.ProtocolDecoder messageDecoder = new Message.ProtocolDecoder();
- private static final Frame.Decompressor frameDecompressor = new Frame.Decompressor();
- private static final Frame.Compressor frameCompressor = new Frame.Compressor();
+ private static final Message.ProtocolEncoder messageEncoder = new Message.ProtocolEncoder();
+ private static final Frame.InboundBodyTransformer inboundFrameTransformer = new Frame.InboundBodyTransformer();
+ private static final Frame.OutboundBodyTransformer outboundFrameTransformer = new Frame.OutboundBodyTransformer();
private static final Frame.Encoder frameEncoder = new Frame.Encoder();
private static final Message.ExceptionHandler exceptionHandler = new Message.ExceptionHandler();
private static final ConnectionLimitHandler connectionLimitHandler = new ConnectionLimitHandler();
@@ -365,16 +436,30 @@
pipeline.addFirst("connectionLimitHandler", connectionLimitHandler);
}
+ long idleTimeout = DatabaseDescriptor.nativeTransportIdleTimeout();
+ if (idleTimeout > 0)
+ {
+ pipeline.addLast("idleStateHandler", new IdleStateHandler(false, 0, 0, idleTimeout, TimeUnit.MILLISECONDS)
+ {
+ @Override
+ protected void channelIdle(ChannelHandlerContext ctx, IdleStateEvent evt)
+ {
+ logger.info("Closing client connection {} after timeout of {}ms", channel.remoteAddress(), idleTimeout);
+ ctx.close();
+ }
+ });
+ }
+
//pipeline.addLast("debug", new LoggingHandler());
- pipeline.addLast("frameDecoder", new Frame.Decoder(server.connectionFactory, server.protocolVersionLimit));
+ pipeline.addLast("frameDecoder", new Frame.Decoder(server.connectionFactory));
pipeline.addLast("frameEncoder", frameEncoder);
- pipeline.addLast("frameDecompressor", frameDecompressor);
- pipeline.addLast("frameCompressor", frameCompressor);
+ pipeline.addLast("inboundFrameTransformer", inboundFrameTransformer);
+ pipeline.addLast("outboundFrameTransformer", outboundFrameTransformer);
pipeline.addLast("messageDecoder", messageDecoder);
- pipeline.addLast("messageEncoder", new Message.ProtocolEncoder(server.protocolVersionLimit));
+ pipeline.addLast("messageEncoder", messageEncoder);
pipeline.addLast("executor", new Message.Dispatcher(DatabaseDescriptor.useNativeTransportLegacyFlusher(),
EndpointPayloadTracker.get(((InetSocketAddress) channel.remoteAddress()).getAddress())));
@@ -391,31 +476,18 @@
protected abstract static class AbstractSecureIntializer extends Initializer
{
- private final SSLContext sslContext;
private final EncryptionOptions encryptionOptions;
protected AbstractSecureIntializer(Server server, EncryptionOptions encryptionOptions)
{
super(server);
this.encryptionOptions = encryptionOptions;
- try
- {
- this.sslContext = SSLFactory.createSSLContext(encryptionOptions, encryptionOptions.require_client_auth);
- }
- catch (IOException e)
- {
- throw new RuntimeException("Failed to setup secure pipeline", e);
- }
}
- protected final SslHandler createSslHandler()
+ protected final SslHandler createSslHandler(ByteBufAllocator allocator) throws IOException
{
- SSLEngine sslEngine = sslContext.createSSLEngine();
- sslEngine.setUseClientMode(false);
- String[] suites = SSLFactory.filterCipherSuites(sslEngine.getSupportedCipherSuites(), encryptionOptions.cipher_suites);
- sslEngine.setEnabledCipherSuites(suites);
- sslEngine.setNeedClientAuth(encryptionOptions.require_client_auth);
- return new SslHandler(sslEngine);
+ SslContext sslContext = SSLFactory.getOrCreateSslContext(encryptionOptions, encryptionOptions.require_client_auth, SSLFactory.SocketType.SERVER);
+ return sslContext.newHandler(allocator);
}
}
@@ -444,7 +516,7 @@
{
// Connection uses SSL/TLS, replace the detection handler with a SslHandler and so use
// encryption.
- SslHandler sslHandler = createSslHandler();
+ SslHandler sslHandler = createSslHandler(channel.alloc());
channelHandlerContext.pipeline().replace(this, "ssl", sslHandler);
}
else
@@ -467,7 +539,7 @@
protected void initChannel(Channel channel) throws Exception
{
- SslHandler sslHandler = createSslHandler();
+ SslHandler sslHandler = createSslHandler(channel.alloc());
super.initChannel(channel);
channel.pipeline().addFirst("ssl", sslHandler);
}
@@ -507,68 +579,49 @@
}
}
- private static class EventNotifier extends MigrationListener implements IEndpointLifecycleSubscriber
+ private static class EventNotifier extends SchemaChangeListener implements IEndpointLifecycleSubscriber
{
private final Server server;
// We keep track of the latest status change events we have sent to avoid sending duplicates
// since StorageService may send duplicate notifications (CASSANDRA-7816, CASSANDRA-8236, CASSANDRA-9156)
- private final Map<InetAddress, LatestEvent> latestEvents = new ConcurrentHashMap<>();
+ private final Map<InetAddressAndPort, LatestEvent> latestEvents = new ConcurrentHashMap<>();
// We also want to delay delivering a NEW_NODE notification until the new node has set its RPC ready
// state. This tracks the endpoints which have joined, but not yet signalled they're ready for clients
- private final Set<InetAddress> endpointsPendingJoinedNotification = ConcurrentHashMap.newKeySet();
-
-
- private static final InetAddress bindAll;
- static
- {
- try
- {
- bindAll = InetAddress.getByAddress(new byte[4]);
- }
- catch (UnknownHostException e)
- {
- throw new AssertionError(e);
- }
- }
+ private final Set<InetAddressAndPort> endpointsPendingJoinedNotification = ConcurrentHashMap.newKeySet();
private EventNotifier(Server server)
{
this.server = server;
}
- private InetAddress getRpcAddress(InetAddress endpoint)
+ private InetAddressAndPort getNativeAddress(InetAddressAndPort endpoint)
{
try
{
- InetAddress rpcAddress = InetAddress.getByName(StorageService.instance.getRpcaddress(endpoint));
- // If rpcAddress == 0.0.0.0 (i.e. bound on all addresses), returning that is not very helpful,
- // so return the internal address (which is ok since "we're bound on all addresses").
- // Note that after all nodes are running a version that includes CASSANDRA-5899, rpcAddress should
- // never be 0.0.0.0, so this can eventually be removed.
- return rpcAddress.equals(bindAll) ? endpoint : rpcAddress;
+ return InetAddressAndPort.getByName(StorageService.instance.getNativeaddress(endpoint, true));
}
catch (UnknownHostException e)
{
// That should not happen, so log an error, but return the
// endpoint address since there's a good change this is right
logger.error("Problem retrieving RPC address for {}", endpoint, e);
- return endpoint;
+ return InetAddressAndPort.getByAddressOverrideDefaults(endpoint.address, DatabaseDescriptor.getNativeTransportPort());
}
}
- private void send(InetAddress endpoint, Event.NodeEvent event)
+ private void send(InetAddressAndPort endpoint, Event.NodeEvent event)
{
if (logger.isTraceEnabled())
- logger.trace("Sending event for endpoint {}, rpc address {}", endpoint, event.nodeAddress());
+ logger.trace("Sending event for endpoint {}, rpc address {}", endpoint, event.nodeAddressAndPort());
// If the endpoint is not the local node, extract the node address
// and if it is the same as our own RPC broadcast address (which defaults to the rcp address)
// then don't send the notification. This covers the case of rpc_address set to "localhost",
// which is not useful to any driver and in fact may cauase serious problems to some drivers,
// see CASSANDRA-10052
- if (!endpoint.equals(FBUtilities.getBroadcastAddress()) &&
- event.nodeAddress().equals(FBUtilities.getBroadcastRpcAddress()))
+ if (!endpoint.equals(FBUtilities.getBroadcastAddressAndPort()) &&
+ event.nodeAddressAndPort().equals(FBUtilities.getBroadcastNativeAddressAndPort()))
return;
send(event);
@@ -579,38 +632,38 @@
server.connectionTracker.send(event);
}
- public void onJoinCluster(InetAddress endpoint)
+ public void onJoinCluster(InetAddressAndPort endpoint)
{
if (!StorageService.instance.isRpcReady(endpoint))
endpointsPendingJoinedNotification.add(endpoint);
else
- onTopologyChange(endpoint, Event.TopologyChange.newNode(getRpcAddress(endpoint), server.socket.getPort()));
+ onTopologyChange(endpoint, Event.TopologyChange.newNode(getNativeAddress(endpoint)));
}
- public void onLeaveCluster(InetAddress endpoint)
+ public void onLeaveCluster(InetAddressAndPort endpoint)
{
- onTopologyChange(endpoint, Event.TopologyChange.removedNode(getRpcAddress(endpoint), server.socket.getPort()));
+ onTopologyChange(endpoint, Event.TopologyChange.removedNode(getNativeAddress(endpoint)));
}
- public void onMove(InetAddress endpoint)
+ public void onMove(InetAddressAndPort endpoint)
{
- onTopologyChange(endpoint, Event.TopologyChange.movedNode(getRpcAddress(endpoint), server.socket.getPort()));
+ onTopologyChange(endpoint, Event.TopologyChange.movedNode(getNativeAddress(endpoint)));
}
- public void onUp(InetAddress endpoint)
+ public void onUp(InetAddressAndPort endpoint)
{
if (endpointsPendingJoinedNotification.remove(endpoint))
onJoinCluster(endpoint);
- onStatusChange(endpoint, Event.StatusChange.nodeUp(getRpcAddress(endpoint), server.socket.getPort()));
+ onStatusChange(endpoint, Event.StatusChange.nodeUp(getNativeAddress(endpoint)));
}
- public void onDown(InetAddress endpoint)
+ public void onDown(InetAddressAndPort endpoint)
{
- onStatusChange(endpoint, Event.StatusChange.nodeDown(getRpcAddress(endpoint), server.socket.getPort()));
+ onStatusChange(endpoint, Event.StatusChange.nodeDown(getNativeAddress(endpoint)));
}
- private void onTopologyChange(InetAddress endpoint, Event.TopologyChange event)
+ private void onTopologyChange(InetAddressAndPort endpoint, Event.TopologyChange event)
{
if (logger.isTraceEnabled())
logger.trace("Topology changed event : {}, {}", endpoint, event.change);
@@ -624,7 +677,7 @@
}
}
- private void onStatusChange(InetAddress endpoint, Event.StatusChange event)
+ private void onStatusChange(InetAddressAndPort endpoint, Event.StatusChange event)
{
if (logger.isTraceEnabled())
logger.trace("Status changed event : {}, {}", endpoint, event.status);
@@ -643,12 +696,12 @@
send(new Event.SchemaChange(Event.SchemaChange.Change.CREATED, ksName));
}
- public void onCreateColumnFamily(String ksName, String cfName)
+ public void onCreateTable(String ksName, String cfName)
{
send(new Event.SchemaChange(Event.SchemaChange.Change.CREATED, Event.SchemaChange.Target.TABLE, ksName, cfName));
}
- public void onCreateUserType(String ksName, String typeName)
+ public void onCreateType(String ksName, String typeName)
{
send(new Event.SchemaChange(Event.SchemaChange.Change.CREATED, Event.SchemaChange.Target.TYPE, ksName, typeName));
}
@@ -665,28 +718,28 @@
ksName, aggregateName, AbstractType.asCQLTypeStringList(argTypes)));
}
- public void onUpdateKeyspace(String ksName)
+ public void onAlterKeyspace(String ksName)
{
send(new Event.SchemaChange(Event.SchemaChange.Change.UPDATED, ksName));
}
- public void onUpdateColumnFamily(String ksName, String cfName, boolean affectsStatements)
+ public void onAlterTable(String ksName, String cfName, boolean affectsStatements)
{
send(new Event.SchemaChange(Event.SchemaChange.Change.UPDATED, Event.SchemaChange.Target.TABLE, ksName, cfName));
}
- public void onUpdateUserType(String ksName, String typeName)
+ public void onAlterType(String ksName, String typeName)
{
send(new Event.SchemaChange(Event.SchemaChange.Change.UPDATED, Event.SchemaChange.Target.TYPE, ksName, typeName));
}
- public void onUpdateFunction(String ksName, String functionName, List<AbstractType<?>> argTypes)
+ public void onAlterFunction(String ksName, String functionName, List<AbstractType<?>> argTypes)
{
send(new Event.SchemaChange(Event.SchemaChange.Change.UPDATED, Event.SchemaChange.Target.FUNCTION,
ksName, functionName, AbstractType.asCQLTypeStringList(argTypes)));
}
- public void onUpdateAggregate(String ksName, String aggregateName, List<AbstractType<?>> argTypes)
+ public void onAlterAggregate(String ksName, String aggregateName, List<AbstractType<?>> argTypes)
{
send(new Event.SchemaChange(Event.SchemaChange.Change.UPDATED, Event.SchemaChange.Target.AGGREGATE,
ksName, aggregateName, AbstractType.asCQLTypeStringList(argTypes)));
@@ -697,12 +750,12 @@
send(new Event.SchemaChange(Event.SchemaChange.Change.DROPPED, ksName));
}
- public void onDropColumnFamily(String ksName, String cfName)
+ public void onDropTable(String ksName, String cfName)
{
send(new Event.SchemaChange(Event.SchemaChange.Change.DROPPED, Event.SchemaChange.Target.TABLE, ksName, cfName));
}
- public void onDropUserType(String ksName, String typeName)
+ public void onDropType(String ksName, String typeName)
{
send(new Event.SchemaChange(Event.SchemaChange.Change.DROPPED, Event.SchemaChange.Target.TYPE, ksName, typeName));
}
diff --git a/src/java/org/apache/cassandra/transport/ServerConnection.java b/src/java/org/apache/cassandra/transport/ServerConnection.java
index 9374ca0..de8a02a 100644
--- a/src/java/org/apache/cassandra/transport/ServerConnection.java
+++ b/src/java/org/apache/cassandra/transport/ServerConnection.java
@@ -17,10 +17,15 @@
*/
package org.apache.cassandra.transport;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
+import javax.net.ssl.SSLPeerUnverifiedException;
+import javax.security.cert.X509Certificate;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import io.netty.channel.Channel;
+import com.codahale.metrics.Counter;
+import io.netty.handler.ssl.SslHandler;
import org.apache.cassandra.auth.IAuthenticator;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.service.ClientState;
@@ -28,43 +33,40 @@
public class ServerConnection extends Connection
{
- private enum State { UNINITIALIZED, AUTHENTICATION, READY }
+ private static final Logger logger = LoggerFactory.getLogger(ServerConnection.class);
private volatile IAuthenticator.SaslNegotiator saslNegotiator;
private final ClientState clientState;
- private volatile State state;
+ private volatile ConnectionStage stage;
+ public final Counter requests = new Counter();
- private final ConcurrentMap<Integer, QueryState> queryStates = new ConcurrentHashMap<>();
-
- public ServerConnection(Channel channel, ProtocolVersion version, Connection.Tracker tracker)
+ ServerConnection(Channel channel, ProtocolVersion version, Connection.Tracker tracker)
{
super(channel, version, tracker);
- this.clientState = ClientState.forExternalCalls(channel.remoteAddress());
- this.state = State.UNINITIALIZED;
+
+ clientState = ClientState.forExternalCalls(channel.remoteAddress());
+ stage = ConnectionStage.ESTABLISHED;
}
- private QueryState getQueryState(int streamId)
+ public ClientState getClientState()
{
- QueryState qState = queryStates.get(streamId);
- if (qState == null)
- {
- // In theory we shouldn't get any race here, but it never hurts to be careful
- QueryState newState = new QueryState(clientState);
- if ((qState = queryStates.putIfAbsent(streamId, newState)) == null)
- qState = newState;
- }
- return qState;
+ return clientState;
}
- public QueryState validateNewMessage(Message.Type type, ProtocolVersion version, int streamId)
+ ConnectionStage stage()
{
- switch (state)
+ return stage;
+ }
+
+ QueryState validateNewMessage(Message.Type type, ProtocolVersion version)
+ {
+ switch (stage)
{
- case UNINITIALIZED:
+ case ESTABLISHED:
if (type != Message.Type.STARTUP && type != Message.Type.OPTIONS)
throw new ProtocolException(String.format("Unexpected message %s, expecting STARTUP or OPTIONS", type));
break;
- case AUTHENTICATION:
+ case AUTHENTICATING:
// Support both SASL auth from protocol v2 and the older style Credentials auth from v1
if (type != Message.Type.AUTH_RESPONSE && type != Message.Type.CREDENTIALS)
throw new ProtocolException(String.format("Unexpected message %s, expecting %s", type, version == ProtocolVersion.V1 ? "CREDENTIALS" : "SASL_RESPONSE"));
@@ -76,29 +78,30 @@
default:
throw new AssertionError();
}
- return getQueryState(streamId);
+
+ return new QueryState(clientState);
}
- public void applyStateTransition(Message.Type requestType, Message.Type responseType)
+ void applyStateTransition(Message.Type requestType, Message.Type responseType)
{
- switch (state)
+ switch (stage)
{
- case UNINITIALIZED:
+ case ESTABLISHED:
if (requestType == Message.Type.STARTUP)
{
if (responseType == Message.Type.AUTHENTICATE)
- state = State.AUTHENTICATION;
+ stage = ConnectionStage.AUTHENTICATING;
else if (responseType == Message.Type.READY)
- state = State.READY;
+ stage = ConnectionStage.READY;
}
break;
- case AUTHENTICATION:
+ case AUTHENTICATING:
// Support both SASL auth from protocol v2 and the older style Credentials auth from v1
assert requestType == Message.Type.AUTH_RESPONSE || requestType == Message.Type.CREDENTIALS;
if (responseType == Message.Type.READY || responseType == Message.Type.AUTH_SUCCESS)
{
- state = State.READY;
+ stage = ConnectionStage.READY;
// we won't use the authenticator again, null it so that it can be GC'd
saslNegotiator = null;
}
@@ -113,7 +116,30 @@
public IAuthenticator.SaslNegotiator getSaslNegotiator(QueryState queryState)
{
if (saslNegotiator == null)
- saslNegotiator = DatabaseDescriptor.getAuthenticator().newSaslNegotiator(queryState.getClientAddress());
+ saslNegotiator = DatabaseDescriptor.getAuthenticator()
+ .newSaslNegotiator(queryState.getClientAddress(), certificates());
return saslNegotiator;
}
+
+ private X509Certificate[] certificates()
+ {
+ SslHandler sslHandler = (SslHandler) channel().pipeline()
+ .get("ssl");
+ X509Certificate[] certificates = null;
+
+ if (sslHandler != null)
+ {
+ try
+ {
+ certificates = sslHandler.engine()
+ .getSession()
+ .getPeerCertificateChain();
+ }
+ catch (SSLPeerUnverifiedException e)
+ {
+ logger.error("Failed to get peer certificates for peer {}", channel().remoteAddress(), e);
+ }
+ }
+ return certificates;
+ }
}
diff --git a/src/java/org/apache/cassandra/transport/SimpleClient.java b/src/java/org/apache/cassandra/transport/SimpleClient.java
index 1e6ea64..ef3c1db 100644
--- a/src/java/org/apache/cassandra/transport/SimpleClient.java
+++ b/src/java/org/apache/cassandra/transport/SimpleClient.java
@@ -28,8 +28,6 @@
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.SynchronousQueue;
-import javax.net.ssl.SSLContext;
-import javax.net.ssl.SSLEngine;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -40,12 +38,17 @@
import io.netty.channel.ChannelOption;
import io.netty.channel.SimpleChannelInboundHandler;
import io.netty.channel.nio.NioEventLoopGroup;
+import io.netty.handler.ssl.SslContext;
import io.netty.util.internal.logging.InternalLoggerFactory;
import io.netty.util.internal.logging.Slf4JLoggerFactory;
+import org.apache.cassandra.config.EncryptionOptions;
import org.apache.cassandra.cql3.QueryOptions;
import org.apache.cassandra.db.ConsistencyLevel;
import org.apache.cassandra.security.SSLFactory;
-import org.apache.cassandra.transport.messages.CredentialsMessage;
+import org.apache.cassandra.transport.frame.checksum.ChecksummingTransformer;
+import org.apache.cassandra.transport.frame.compress.CompressingTransformer;
+import org.apache.cassandra.transport.frame.compress.Compressor;
+import org.apache.cassandra.transport.frame.compress.LZ4Compressor;
import org.apache.cassandra.transport.messages.ErrorMessage;
import org.apache.cassandra.transport.messages.EventMessage;
import org.apache.cassandra.transport.messages.ExecuteMessage;
@@ -53,13 +56,11 @@
import org.apache.cassandra.transport.messages.QueryMessage;
import org.apache.cassandra.transport.messages.ResultMessage;
import org.apache.cassandra.transport.messages.StartupMessage;
-import org.apache.cassandra.utils.MD5Digest;
import io.netty.channel.Channel;
import io.netty.channel.ChannelFuture;
import io.netty.channel.ChannelHandlerContext;
import io.netty.channel.ChannelPipeline;
-import io.netty.handler.ssl.SslHandler;
-import static org.apache.cassandra.config.EncryptionOptions.ClientEncryptionOptions;
+import org.apache.cassandra.utils.ChecksumType;
public class SimpleClient implements Closeable
{
@@ -71,7 +72,7 @@
private static final Logger logger = LoggerFactory.getLogger(SimpleClient.class);
public final String host;
public final int port;
- private final ClientEncryptionOptions encryptionOptions;
+ private final EncryptionOptions encryptionOptions;
protected final ResponseHandler responseHandler = new ResponseHandler();
protected final Connection.Tracker tracker = new ConnectionTracker();
@@ -90,22 +91,22 @@
}
};
- public SimpleClient(String host, int port, ProtocolVersion version, ClientEncryptionOptions encryptionOptions)
+ public SimpleClient(String host, int port, ProtocolVersion version, EncryptionOptions encryptionOptions)
{
this(host, port, version, false, encryptionOptions);
}
- public SimpleClient(String host, int port, ClientEncryptionOptions encryptionOptions)
+ public SimpleClient(String host, int port, EncryptionOptions encryptionOptions)
{
this(host, port, ProtocolVersion.CURRENT, encryptionOptions);
}
public SimpleClient(String host, int port, ProtocolVersion version)
{
- this(host, port, version, new ClientEncryptionOptions());
+ this(host, port, version, new EncryptionOptions());
}
- public SimpleClient(String host, int port, ProtocolVersion version, boolean useBeta, ClientEncryptionOptions encryptionOptions)
+ public SimpleClient(String host, int port, ProtocolVersion version, boolean useBeta, EncryptionOptions encryptionOptions)
{
this.host = host;
this.port = port;
@@ -118,31 +119,39 @@
public SimpleClient(String host, int port)
{
- this(host, port, new ClientEncryptionOptions());
+ this(host, port, new EncryptionOptions());
}
- public void connect(boolean useCompression) throws IOException
+ public SimpleClient connect(boolean useCompression, boolean useChecksums) throws IOException
{
- connect(useCompression, false);
+ return connect(useCompression, useChecksums, false);
}
- public void connect(boolean useCompression, boolean throwOnOverload) throws IOException
+ public SimpleClient connect(boolean useCompression, boolean useChecksums, boolean throwOnOverload) throws IOException
{
establishConnection();
Map<String, String> options = new HashMap<>();
options.put(StartupMessage.CQL_VERSION, "3.0.0");
-
if (throwOnOverload)
options.put(StartupMessage.THROW_ON_OVERLOAD, "1");
connection.setThrowOnOverload(throwOnOverload);
- if (useCompression)
+ if (useChecksums)
{
- options.put(StartupMessage.COMPRESSION, "snappy");
- connection.setCompressor(FrameCompressor.SnappyCompressor.instance);
+ Compressor compressor = useCompression ? LZ4Compressor.INSTANCE : null;
+ connection.setTransformer(ChecksummingTransformer.getTransformer(ChecksumType.CRC32, compressor));
+ options.put(StartupMessage.CHECKSUM, "crc32");
+ options.put(StartupMessage.COMPRESSION, "lz4");
}
+ else if (useCompression)
+ {
+ connection.setTransformer(CompressingTransformer.getTransformer(LZ4Compressor.INSTANCE));
+ options.put(StartupMessage.COMPRESSION, "lz4");
+ }
+
execute(new StartupMessage(options));
+ return this;
}
public void setEventHandler(EventHandler eventHandler)
@@ -159,7 +168,7 @@
.option(ChannelOption.TCP_NODELAY, true);
// Configure the pipeline factory.
- if(encryptionOptions.enabled)
+ if(encryptionOptions.isEnabled())
{
bootstrap.handler(new SecureInitializer());
}
@@ -178,13 +187,6 @@
}
}
- public void login(Map<String, String> credentials)
- {
- CredentialsMessage msg = new CredentialsMessage();
- msg.credentials.putAll(credentials);
- execute(msg);
- }
-
public ResultMessage execute(String query, ConsistencyLevel consistency)
{
return execute(query, Collections.<ByteBuffer>emptyList(), consistency);
@@ -199,14 +201,14 @@
public ResultMessage.Prepared prepare(String query)
{
- Message.Response msg = execute(new PrepareMessage(query));
+ Message.Response msg = execute(new PrepareMessage(query, null));
assert msg instanceof ResultMessage.Prepared;
return (ResultMessage.Prepared)msg;
}
- public ResultMessage executePrepared(byte[] statementId, List<ByteBuffer> values, ConsistencyLevel consistency)
+ public ResultMessage executePrepared(ResultMessage.Prepared prepared, List<ByteBuffer> values, ConsistencyLevel consistency)
{
- Message.Response msg = execute(new ExecuteMessage(MD5Digest.wrap(statementId), QueryOptions.forInternalCalls(consistency, values)));
+ Message.Response msg = execute(new ExecuteMessage(prepared.statementId, prepared.resultMetadataId, QueryOptions.forInternalCalls(consistency, values)));
assert msg instanceof ResultMessage;
return (ResultMessage)msg;
}
@@ -259,9 +261,9 @@
// Stateless handlers
private static final Message.ProtocolDecoder messageDecoder = new Message.ProtocolDecoder();
- private static final Message.ProtocolEncoder messageEncoder = new Message.ProtocolEncoder(ProtocolVersionLimit.SERVER_DEFAULT);
- private static final Frame.Decompressor frameDecompressor = new Frame.Decompressor();
- private static final Frame.Compressor frameCompressor = new Frame.Compressor();
+ private static final Message.ProtocolEncoder messageEncoder = new Message.ProtocolEncoder();
+ private static final Frame.InboundBodyTransformer inboundFrameTransformer = new Frame.InboundBodyTransformer();
+ private static final Frame.OutboundBodyTransformer outboundFrameTransformer = new Frame.OutboundBodyTransformer();
private static final Frame.Encoder frameEncoder = new Frame.Encoder();
private static class ConnectionTracker implements Connection.Tracker
@@ -282,11 +284,11 @@
channel.attr(Connection.attributeKey).set(connection);
ChannelPipeline pipeline = channel.pipeline();
- pipeline.addLast("frameDecoder", new Frame.Decoder(connectionFactory, ProtocolVersionLimit.SERVER_DEFAULT));
+ pipeline.addLast("frameDecoder", new Frame.Decoder(connectionFactory));
pipeline.addLast("frameEncoder", frameEncoder);
- pipeline.addLast("frameDecompressor", frameDecompressor);
- pipeline.addLast("frameCompressor", frameCompressor);
+ pipeline.addLast("inboundFrameTransformer", inboundFrameTransformer);
+ pipeline.addLast("outboundFrameTransformer", outboundFrameTransformer);
pipeline.addLast("messageDecoder", messageDecoder);
pipeline.addLast("messageEncoder", messageEncoder);
@@ -297,21 +299,12 @@
private class SecureInitializer extends Initializer
{
- private final SSLContext sslContext;
-
- public SecureInitializer() throws IOException
- {
- this.sslContext = SSLFactory.createSSLContext(encryptionOptions, true);
- }
-
protected void initChannel(Channel channel) throws Exception
{
super.initChannel(channel);
- SSLEngine sslEngine = sslContext.createSSLEngine();
- sslEngine.setUseClientMode(true);
- String[] suites = SSLFactory.filterCipherSuites(sslEngine.getSupportedCipherSuites(), encryptionOptions.cipher_suites);
- sslEngine.setEnabledCipherSuites(suites);
- channel.pipeline().addFirst("ssl", new SslHandler(sslEngine));
+ SslContext sslContext = SSLFactory.getOrCreateSslContext(encryptionOptions, encryptionOptions.require_client_auth,
+ SSLFactory.SocketType.CLIENT);
+ channel.pipeline().addFirst("ssl", sslContext.newHandler(channel.alloc()));
}
}
diff --git a/src/java/org/apache/cassandra/transport/frame/FrameBodyTransformer.java b/src/java/org/apache/cassandra/transport/frame/FrameBodyTransformer.java
new file mode 100644
index 0000000..0a6b22f
--- /dev/null
+++ b/src/java/org/apache/cassandra/transport/frame/FrameBodyTransformer.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.transport.frame;
+
+import java.io.IOException;
+import java.util.EnumSet;
+
+import io.netty.buffer.ByteBuf;
+import org.apache.cassandra.transport.Frame;
+
+public interface FrameBodyTransformer
+{
+ /**
+ * Accepts the input buffer representing the frame body of an incoming message and applies a transformation.
+ * Example transformations include decompression and recombining checksummed chunks into a single, serialized
+ * message body.
+ * @param inputBuf the frame body from an inbound message
+ * @return the new frame body bytes
+ * @throws IOException if the transformation failed for any reason
+ */
+ ByteBuf transformInbound(ByteBuf inputBuf, EnumSet<Frame.Header.Flag> flags) throws IOException;
+
+ /**
+ * Accepts an input buffer representing the frame body of an outbound message and applies a transformation.
+ * Example transformations include compression and splitting into checksummed chunks.
+
+ * @param inputBuf the frame body from an outgoing message
+ * @return the new frame body bytes
+ * @throws IOException if the transformation failed for any reason
+ */
+ ByteBuf transformOutbound(ByteBuf inputBuf) throws IOException;
+
+ /**
+ * Returns an EnumSet of the flags that should be added to the header for any message whose frame body has been
+ * modified by the transformer. E.g. it may add perform chunking & checksumming to the frame body,
+ * compress it, or both.
+ * @return EnumSet containing the header flags to set on messages transformed
+ */
+ EnumSet<Frame.Header.Flag> getOutboundHeaderFlags();
+
+}
diff --git a/src/java/org/apache/cassandra/transport/frame/checksum/ChecksummingTransformer.java b/src/java/org/apache/cassandra/transport/frame/checksum/ChecksummingTransformer.java
new file mode 100644
index 0000000..b2aeac9
--- /dev/null
+++ b/src/java/org/apache/cassandra/transport/frame/checksum/ChecksummingTransformer.java
@@ -0,0 +1,361 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.transport.frame.checksum;
+
+import java.io.IOException;
+import java.util.EnumSet;
+
+import com.google.common.collect.ImmutableTable;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.Unpooled;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.transport.Frame;
+import org.apache.cassandra.transport.ProtocolException;
+import org.apache.cassandra.transport.frame.FrameBodyTransformer;
+import org.apache.cassandra.transport.frame.compress.Compressor;
+import org.apache.cassandra.transport.frame.compress.LZ4Compressor;
+import org.apache.cassandra.transport.frame.compress.SnappyCompressor;
+import org.apache.cassandra.utils.ChecksumType;
+
+import static org.apache.cassandra.transport.CBUtil.readUnsignedShort;
+
+/**
+ * Provides a format that implements chunking and checksumming logic
+ * that maybe used in conjunction with a frame Compressor if required
+ * <p>
+ * <strong>1.1. Checksummed/Compression Serialized Format</strong>
+ * <p>
+ * <pre>
+ * {@code
+ * 1 1 1 1 1 1 1 1 1 1 2 2 2 2 2 2 2 2 2 2 3 3
+ * 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ * | Number of Compressed Chunks | Compressed Length (e1) /
+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ * / Compressed Length cont. (e1) | Uncompressed Length (e1) /
+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ * | Uncompressed Length cont. (e1)| Checksum of Lengths (e1) |
+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ * | Checksum of Lengths cont. (e1)| Compressed Bytes (e1) +//
+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ * | Checksum (e1) ||
+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ * | Compressed Length (e2) |
+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ * | Uncompressed Length (e2) |
+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ * | Checksum of Lengths (e2) |
+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ * | Compressed Bytes (e2) +//
+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ * | Checksum (e2) ||
+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ * | Compressed Length (en) |
+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ * | Uncompressed Length (en) |
+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ * | Checksum of Lengths (en) |
+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ * | Compressed Bytes (en) +//
+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ * | Checksum (en) ||
+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ * }
+ * </pre>
+ * <p>
+ * <p>
+ * <strong>1.2. Checksum Compression Description</strong>
+ * <p>
+ * The entire payload is broken into n chunks each with a pair of checksums:
+ * <ul>
+ * <li>[int]: compressed length of serialized bytes for this chunk (e.g. the length post compression)
+ * <li>[int]: expected length of the decompressed bytes (e.g. the length after decompression)
+ * <li>[int]: digest of decompressed and compressed length components above
+ * <li>[k bytes]: compressed payload for this chunk
+ * <li>[int]: digest of the decompressed result of the payload above for this chunk
+ * </ul>
+ * <p>
+ */
+public class ChecksummingTransformer implements FrameBodyTransformer
+{
+ private static final Logger logger = LoggerFactory.getLogger(ChecksummingTransformer.class);
+
+ private static final EnumSet<Frame.Header.Flag> CHECKSUMS_ONLY = EnumSet.of(Frame.Header.Flag.CHECKSUMMED);
+ private static final EnumSet<Frame.Header.Flag> CHECKSUMS_AND_COMPRESSION = EnumSet.of(Frame.Header.Flag.CHECKSUMMED, Frame.Header.Flag.COMPRESSED);
+
+ private static final int CHUNK_HEADER_OVERHEAD = Integer.BYTES + Integer.BYTES + Integer.BYTES + Integer.BYTES;
+
+ private static final ChecksummingTransformer CRC32_NO_COMPRESSION = new ChecksummingTransformer(ChecksumType.CRC32, null);
+ private static final ChecksummingTransformer ADLER32_NO_COMPRESSION = new ChecksummingTransformer(ChecksumType.ADLER32, null);
+ private static final ImmutableTable<ChecksumType, Compressor, ChecksummingTransformer> transformers;
+ static
+ {
+ ImmutableTable.Builder<ChecksumType, Compressor, ChecksummingTransformer> builder = ImmutableTable.builder();
+ builder.put(ChecksumType.CRC32, LZ4Compressor.INSTANCE, new ChecksummingTransformer(ChecksumType.CRC32, LZ4Compressor.INSTANCE));
+ builder.put(ChecksumType.CRC32, SnappyCompressor.INSTANCE, new ChecksummingTransformer(ChecksumType.CRC32, SnappyCompressor.INSTANCE));
+ builder.put(ChecksumType.ADLER32, LZ4Compressor.INSTANCE, new ChecksummingTransformer(ChecksumType.ADLER32, LZ4Compressor.INSTANCE));
+ builder.put(ChecksumType.ADLER32, SnappyCompressor.INSTANCE, new ChecksummingTransformer(ChecksumType.ADLER32, SnappyCompressor.INSTANCE));
+ transformers = builder.build();
+ }
+
+ private final int blockSize;
+ private final Compressor compressor;
+ private final ChecksumType checksum;
+
+ public static ChecksummingTransformer getTransformer(ChecksumType checksumType, Compressor compressor)
+ {
+ ChecksummingTransformer transformer = compressor == null
+ ? checksumType == ChecksumType.CRC32 ? CRC32_NO_COMPRESSION : ADLER32_NO_COMPRESSION
+ : transformers.get(checksumType, compressor);
+
+ if (transformer == null)
+ {
+ logger.warn("Invalid compression/checksum options supplied. %s / %s", checksumType, compressor.getClass().getName());
+ throw new RuntimeException("Invalid compression / checksum options supplied");
+ }
+
+ return transformer;
+ }
+
+ ChecksummingTransformer(ChecksumType checksumType, Compressor compressor)
+ {
+ this(checksumType, DatabaseDescriptor.getNativeTransportFrameBlockSize(), compressor);
+ }
+
+ ChecksummingTransformer(ChecksumType checksumType, int blockSize, Compressor compressor)
+ {
+ this.checksum = checksumType;
+ this.blockSize = blockSize;
+ this.compressor = compressor;
+ }
+
+ public EnumSet<Frame.Header.Flag> getOutboundHeaderFlags()
+ {
+ return null == compressor ? CHECKSUMS_ONLY : CHECKSUMS_AND_COMPRESSION;
+ }
+
+ public ByteBuf transformOutbound(ByteBuf inputBuf)
+ {
+ // be pessimistic about life and assume the compressed output will be the same size as the input bytes
+ int maxTotalCompressedLength = maxCompressedLength(inputBuf.readableBytes());
+ int expectedChunks = (int) Math.ceil((double) maxTotalCompressedLength / blockSize);
+ int expectedMaxSerializedLength = Short.BYTES + (expectedChunks * CHUNK_HEADER_OVERHEAD) + maxTotalCompressedLength;
+ byte[] retBuf = new byte[expectedMaxSerializedLength];
+ ByteBuf ret = Unpooled.wrappedBuffer(retBuf);
+ ret.writerIndex(0);
+ ret.readerIndex(0);
+
+ // write out bogus short to start with as we'll encode one at the end
+ // when we finalize the number of compressed chunks to expect and this
+ // sets the writer index correctly for starting the first chunk
+ ret.writeShort((short) 0);
+
+ byte[] inBuf = new byte[blockSize];
+ byte[] outBuf = new byte[maxCompressedLength(blockSize)];
+ byte[] chunkLengths = new byte[8];
+
+ int numCompressedChunks = 0;
+ int readableBytes;
+ int lengthsChecksum;
+ while ((readableBytes = inputBuf.readableBytes()) > 0)
+ {
+ int lengthToRead = Math.min(blockSize, readableBytes);
+ inputBuf.readBytes(inBuf, 0, lengthToRead);
+ int uncompressedChunkChecksum = (int) checksum.of(inBuf, 0, lengthToRead);
+ int compressedSize = maybeCompress(inBuf, lengthToRead, outBuf);
+
+ if (compressedSize < lengthToRead)
+ {
+ // there was some benefit to compression so write out the compressed
+ // and uncompressed sizes of the chunk
+ ret.writeInt(compressedSize);
+ ret.writeInt(lengthToRead);
+ putInt(compressedSize, chunkLengths, 0);
+ }
+ else
+ {
+ // if no compression was possible, there's no need to write two lengths, so
+ // just write the size of the original content (or block size), with its
+ // sign flipped to signal no compression.
+ ret.writeInt(-lengthToRead);
+ putInt(-lengthToRead, chunkLengths, 0);
+ }
+
+ putInt(lengthToRead, chunkLengths, 4);
+
+ // calculate the checksum of the compressed and decompressed lengths
+ // protect us against a bogus length causing potential havoc on deserialization
+ lengthsChecksum = (int) checksum.of(chunkLengths, 0, chunkLengths.length);
+ ret.writeInt(lengthsChecksum);
+
+ // figure out how many actual bytes we're going to write and make sure we have capacity
+ int toWrite = Math.min(compressedSize, lengthToRead);
+ if (ret.writableBytes() < (CHUNK_HEADER_OVERHEAD + toWrite))
+ {
+ // this really shouldn't ever happen -- it means we either mis-calculated the number of chunks we
+ // expected to create, we gave some input to the compressor that caused the output to be much
+ // larger than the input.. or some other edge condition. Regardless -- resize if necessary.
+ byte[] resizedRetBuf = new byte[(retBuf.length + (CHUNK_HEADER_OVERHEAD + toWrite)) * 3 / 2];
+ System.arraycopy(retBuf, 0, resizedRetBuf, 0, retBuf.length);
+ retBuf = resizedRetBuf;
+ ByteBuf resizedRetByteBuf = Unpooled.wrappedBuffer(retBuf);
+ resizedRetByteBuf.writerIndex(ret.writerIndex());
+ ret = resizedRetByteBuf;
+ }
+
+ // write the bytes, either compressed or uncompressed
+ if (compressedSize < lengthToRead)
+ ret.writeBytes(outBuf, 0, toWrite); // compressed
+ else
+ ret.writeBytes(inBuf, 0, toWrite); // uncompressed
+
+ // checksum of the uncompressed chunk
+ ret.writeInt(uncompressedChunkChecksum);
+
+ numCompressedChunks++;
+ }
+
+ // now update the number of chunks
+ ret.setShort(0, (short) numCompressedChunks);
+ return ret;
+ }
+
+ public ByteBuf transformInbound(ByteBuf inputBuf, EnumSet<Frame.Header.Flag> flags)
+ {
+ int numChunks = readUnsignedShort(inputBuf);
+
+ int currentPosition = 0;
+ int decompressedLength;
+ int lengthsChecksum;
+
+ byte[] buf = null;
+ byte[] retBuf = new byte[inputBuf.readableBytes()];
+ byte[] chunkLengths = new byte[8];
+ for (int i = 0; i < numChunks; i++)
+ {
+ int compressedLength = inputBuf.readInt();
+ // if the input was actually compressed, then the writer should have written a decompressed
+ // length. If not, then we can infer that the compressed length has had its sign bit flipped
+ // and can derive the decompressed length from that
+ decompressedLength = compressedLength >= 0 ? inputBuf.readInt() : Math.abs(compressedLength);
+
+ putInt(compressedLength, chunkLengths, 0);
+ putInt(decompressedLength, chunkLengths, 4);
+ lengthsChecksum = inputBuf.readInt();
+ // calculate checksum on lengths (decompressed and compressed) and make sure it matches
+ int calculatedLengthsChecksum = (int) checksum.of(chunkLengths, 0, chunkLengths.length);
+ if (lengthsChecksum != calculatedLengthsChecksum)
+ {
+ throw new ProtocolException(String.format("Checksum invalid on chunk bytes lengths. Deserialized compressed " +
+ "length: %d decompressed length: %d. %d != %d", compressedLength,
+ decompressedLength, lengthsChecksum, calculatedLengthsChecksum));
+ }
+
+ // do we have enough space in the decompression buffer?
+ if (currentPosition + decompressedLength > retBuf.length)
+ {
+ byte[] resizedBuf = new byte[retBuf.length + decompressedLength * 3 / 2];
+ System.arraycopy(retBuf, 0, resizedBuf, 0, retBuf.length);
+ retBuf = resizedBuf;
+ }
+
+ // now we've validated the lengths checksum, we can abs the compressed length
+ // to figure out the actual number of bytes we're going to read
+ int toRead = Math.abs(compressedLength);
+ if (buf == null || buf.length < toRead)
+ buf = new byte[toRead];
+
+ // get the (possibly) compressed bytes for this chunk
+ inputBuf.readBytes(buf, 0, toRead);
+
+ // decompress using the original compressed length, so it's a no-op if that's < 0
+ byte[] decompressedChunk = maybeDecompress(buf, compressedLength, decompressedLength, flags);
+
+ // add the decompressed bytes into the ret buf
+ System.arraycopy(decompressedChunk, 0, retBuf, currentPosition, decompressedLength);
+ currentPosition += decompressedLength;
+
+ // get the checksum of the original source bytes and compare against what we read
+ int expectedDecompressedChecksum = inputBuf.readInt();
+ int calculatedDecompressedChecksum = (int) checksum.of(decompressedChunk, 0, decompressedLength);
+ if (expectedDecompressedChecksum != calculatedDecompressedChecksum)
+ {
+ throw new ProtocolException("Decompressed checksum for chunk does not match expected checksum");
+ }
+ }
+
+ ByteBuf ret = Unpooled.wrappedBuffer(retBuf, 0, currentPosition);
+ ret.writerIndex(currentPosition);
+ return ret;
+ }
+
+ private int maxCompressedLength(int uncompressedLength)
+ {
+ return null == compressor ? uncompressedLength : compressor.maxCompressedLength(uncompressedLength);
+
+ }
+
+ private int maybeCompress(byte[] input, int length, byte[] output)
+ {
+ if (null == compressor)
+ {
+ System.arraycopy(input, 0, output, 0, length);
+ return length;
+ }
+
+ try
+ {
+ return compressor.compress(input, 0, length, output, 0);
+ }
+ catch (IOException e)
+ {
+ logger.info("IO error during compression of frame body chunk", e);
+ throw new ProtocolException("Error compressing frame body chunk");
+ }
+ }
+
+ private byte[] maybeDecompress(byte[] input, int length, int expectedLength, EnumSet<Frame.Header.Flag> flags)
+ {
+ if (null == compressor || !flags.contains(Frame.Header.Flag.COMPRESSED) || length < 0)
+ return input;
+
+ try
+ {
+ return compressor.decompress(input, 0, length, expectedLength);
+ }
+ catch (IOException e)
+ {
+ logger.info("IO error during decompression of frame body chunk", e);
+ throw new ProtocolException("Error decompressing frame body chunk");
+ }
+ }
+
+ private void putInt(int val, byte[] dest, int offset)
+ {
+ dest[offset] = (byte) (val >>> 24);
+ dest[offset + 1] = (byte) (val >>> 16);
+ dest[offset + 2] = (byte) (val >>> 8);
+ dest[offset + 3] = (byte) (val);
+ }
+
+}
diff --git a/src/java/org/apache/cassandra/transport/frame/compress/CompressingTransformer.java b/src/java/org/apache/cassandra/transport/frame/compress/CompressingTransformer.java
new file mode 100644
index 0000000..db99edf
--- /dev/null
+++ b/src/java/org/apache/cassandra/transport/frame/compress/CompressingTransformer.java
@@ -0,0 +1,164 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.transport.frame.compress;
+
+import java.io.IOException;
+import java.util.EnumSet;
+
+import io.netty.buffer.ByteBuf;
+import org.apache.cassandra.transport.CBUtil;
+import org.apache.cassandra.transport.Frame;
+import org.apache.cassandra.transport.ProtocolException;
+import org.apache.cassandra.transport.frame.FrameBodyTransformer;
+
+public abstract class CompressingTransformer implements FrameBodyTransformer
+{
+ private static final CompressingTransformer LZ4 = new LZ4();
+ private static final CompressingTransformer SNAPPY = new Snappy();
+
+ private static final EnumSet<Frame.Header.Flag> headerFlags = EnumSet.of(Frame.Header.Flag.COMPRESSED);
+
+ public static final CompressingTransformer getTransformer(Compressor compressor)
+ {
+ if (compressor instanceof LZ4Compressor)
+ return LZ4;
+
+ if (compressor instanceof SnappyCompressor)
+ {
+ if (SnappyCompressor.INSTANCE == null)
+ throw new ProtocolException("This instance does not support Snappy compression");
+
+ return SNAPPY;
+ }
+
+ throw new ProtocolException("Unsupported compression implementation: " + compressor.getClass().getCanonicalName());
+ }
+
+ CompressingTransformer() {}
+
+ public EnumSet<Frame.Header.Flag> getOutboundHeaderFlags()
+ {
+ return headerFlags;
+ }
+
+ public ByteBuf transformInbound(ByteBuf inputBuf, EnumSet<Frame.Header.Flag> flags) throws IOException
+ {
+ return transformInbound(inputBuf);
+ }
+
+ abstract ByteBuf transformInbound(ByteBuf inputBuf) throws IOException;
+
+ // Simple LZ4 encoding prefixes the compressed bytes with the
+ // length of the uncompressed bytes. This length is explicitly big-endian
+ // as the native protocol is entirely big-endian, so it feels like putting
+ // little-endian here would be a annoying trap for client writer
+ private static class LZ4 extends CompressingTransformer
+ {
+ public ByteBuf transformOutbound(ByteBuf inputBuf) throws IOException
+ {
+ byte[] input = CBUtil.readRawBytes(inputBuf);
+ int maxCompressedLength = LZ4Compressor.INSTANCE.maxCompressedLength(input.length);
+ ByteBuf outputBuf = CBUtil.allocator.heapBuffer(Integer.BYTES + maxCompressedLength);
+ byte[] output = outputBuf.array();
+ int outputOffset = outputBuf.arrayOffset();
+ output[outputOffset] = (byte) (input.length >>> 24);
+ output[outputOffset + 1] = (byte) (input.length >>> 16);
+ output[outputOffset + 2] = (byte) (input.length >>> 8);
+ output[outputOffset + 3] = (byte) (input.length);
+ try
+ {
+ int written = LZ4Compressor.INSTANCE.compress(input, 0, input.length, output, Integer.BYTES + outputOffset);
+ outputBuf.writerIndex(Integer.BYTES + written);
+ return outputBuf;
+ }
+ catch (IOException e)
+ {
+ outputBuf.release();
+ throw e;
+ }
+ }
+
+ ByteBuf transformInbound(ByteBuf inputBuf) throws IOException
+ {
+ byte[] input = CBUtil.readRawBytes(inputBuf);
+ int uncompressedLength = ((input[0] & 0xFF) << 24)
+ | ((input[1] & 0xFF) << 16)
+ | ((input[2] & 0xFF) << 8)
+ | ((input[3] & 0xFF));
+ ByteBuf outputBuf = CBUtil.allocator.heapBuffer(uncompressedLength);
+ try
+ {
+ outputBuf.writeBytes(LZ4Compressor.INSTANCE.decompress(input,
+ Integer.BYTES,
+ input.length - Integer.BYTES,
+ uncompressedLength));
+ return outputBuf;
+ }
+ catch (IOException e)
+ {
+ outputBuf.release();
+ throw e;
+ }
+ }
+ }
+
+ // Simple Snappy encoding simply writes the compressed bytes, without the preceding length
+ private static class Snappy extends CompressingTransformer
+ {
+ public ByteBuf transformOutbound(ByteBuf inputBuf) throws IOException
+ {
+ byte[] input = CBUtil.readRawBytes(inputBuf);
+ int uncompressedLength = input.length;
+ int maxCompressedLength = SnappyCompressor.INSTANCE.maxCompressedLength(uncompressedLength);
+ ByteBuf outputBuf = CBUtil.allocator.heapBuffer(maxCompressedLength);
+ try
+ {
+ int written = SnappyCompressor.INSTANCE.compress(input,
+ 0,
+ uncompressedLength,
+ outputBuf.array(),
+ outputBuf.arrayOffset());
+ outputBuf.writerIndex(written);
+ return outputBuf;
+ }
+ catch (IOException e)
+ {
+ outputBuf.release();
+ throw e;
+ }
+ }
+
+ ByteBuf transformInbound(ByteBuf inputBuf) throws IOException
+ {
+ byte[] input = CBUtil.readRawBytes(inputBuf);
+ int uncompressedLength = org.xerial.snappy.Snappy.uncompressedLength(input);
+ ByteBuf outputBuf = CBUtil.allocator.heapBuffer(uncompressedLength);
+ try
+ {
+ outputBuf.writeBytes(SnappyCompressor.INSTANCE.decompress(input, 0, input.length, uncompressedLength));
+ return outputBuf;
+ }
+ catch (IOException e)
+ {
+ outputBuf.release();
+ throw e;
+ }
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/transport/frame/compress/Compressor.java b/src/java/org/apache/cassandra/transport/frame/compress/Compressor.java
new file mode 100644
index 0000000..e458bdb
--- /dev/null
+++ b/src/java/org/apache/cassandra/transport/frame/compress/Compressor.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.transport.frame.compress;
+
+import java.io.IOException;
+
+/**
+ * Analogous to {@link org.apache.cassandra.io.compress.ICompressor}, but different enough that
+ * it's worth specializing:
+ * <ul>
+ * <li>disk IO is mostly oriented around ByteBuffers, whereas with Frames raw byte arrays are
+ * primarily used </li>
+ * <li>our LZ4 compression format is opionated about the endianness of the preceding length
+ * bytes, big for protocol, little for disk</li>
+ * <li>ICompressor doesn't make it easy to pre-allocate the output buffer/array</li>
+ * </ul>
+ *
+ * In future it may be worth revisiting to unify the interfaces.
+ */
+public interface Compressor
+{
+ /**
+ * @param length the decompressed length being compressed
+ * @return the maximum length output possible for an input of the provided length
+ */
+ int maxCompressedLength(int length);
+
+ /**
+ * @param src the input bytes to be compressed
+ * @param srcOffset the offset to start compressing src from
+ * @param length the total number of bytes from srcOffset to pass to the compressor implementation
+ * @param dest the output buffer to write the compressed bytes to
+ * @param destOffset the offset into the dest buffer to start writing the compressed bytes
+ * @return the length of resulting compressed bytes written into the dest buffer
+ * @throws IOException if the compression implementation failed while compressing the input bytes
+ */
+ int compress(byte[] src, int srcOffset, int length, byte[] dest, int destOffset) throws IOException;
+
+ /**
+ * @param src the compressed bytes to be decompressed
+ * @param expectedDecompressedLength the expected length the input bytes will decompress to
+ * @return a byte[] containing the resuling decompressed bytes
+ * @throws IOException thrown if the compression implementation failed to decompress the provided input bytes
+ */
+ byte[] decompress(byte[] src, int srcOffset, int length, int expectedDecompressedLength) throws IOException;
+}
diff --git a/src/java/org/apache/cassandra/transport/frame/compress/LZ4Compressor.java b/src/java/org/apache/cassandra/transport/frame/compress/LZ4Compressor.java
new file mode 100644
index 0000000..88633c6
--- /dev/null
+++ b/src/java/org/apache/cassandra/transport/frame/compress/LZ4Compressor.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.transport.frame.compress;
+
+import java.io.IOException;
+
+import net.jpountz.lz4.LZ4Factory;
+import net.jpountz.lz4.LZ4SafeDecompressor;
+
+public class LZ4Compressor implements Compressor
+{
+ public static final LZ4Compressor INSTANCE = new LZ4Compressor();
+
+ private final net.jpountz.lz4.LZ4Compressor compressor;
+ private final LZ4SafeDecompressor decompressor;
+
+ private LZ4Compressor()
+ {
+ final LZ4Factory lz4Factory = LZ4Factory.fastestInstance();
+ compressor = lz4Factory.fastCompressor();
+ decompressor = lz4Factory.safeDecompressor();
+ }
+
+ public int maxCompressedLength(int length)
+ {
+ return compressor.maxCompressedLength(length);
+ }
+
+ public int compress(byte[] src, int srcOffset, int length, byte[] dest, int destOffset) throws IOException
+ {
+ try
+ {
+ return compressor.compress(src, srcOffset, length, dest, destOffset);
+ }
+ catch (Throwable t)
+ {
+ throw new IOException("Error caught during LZ4 compression", t);
+ }
+ }
+
+ public byte[] decompress(byte[] src, int offset, int length, int expectedDecompressedLength) throws IOException
+ {
+ try
+ {
+ byte[] decompressed = new byte[expectedDecompressedLength];
+ decompressor.decompress(src, offset, length, decompressed, 0, expectedDecompressedLength);
+ return decompressed;
+ }
+ catch (Throwable t)
+ {
+ throw new IOException("Error caught during LZ4 decompression", t);
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/transport/frame/compress/SnappyCompressor.java b/src/java/org/apache/cassandra/transport/frame/compress/SnappyCompressor.java
new file mode 100644
index 0000000..27ea4c3
--- /dev/null
+++ b/src/java/org/apache/cassandra/transport/frame/compress/SnappyCompressor.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.transport.frame.compress;
+
+import java.io.IOException;
+
+import org.apache.cassandra.utils.JVMStabilityInspector;
+import org.xerial.snappy.Snappy;
+import org.xerial.snappy.SnappyError;
+
+public class SnappyCompressor implements Compressor
+{
+ public static final SnappyCompressor INSTANCE;
+ static
+ {
+ SnappyCompressor i;
+ try
+ {
+ i = new SnappyCompressor();
+ }
+ catch (Exception e)
+ {
+ JVMStabilityInspector.inspectThrowable(e);
+ i = null;
+ }
+ catch (NoClassDefFoundError | SnappyError | UnsatisfiedLinkError e)
+ {
+ i = null;
+ }
+ INSTANCE = i;
+ }
+
+ private SnappyCompressor()
+ {
+ // this would throw java.lang.NoClassDefFoundError if Snappy class
+ // wasn't found at runtime which should be processed by the calling method
+ Snappy.getNativeLibraryVersion();
+ }
+
+ @Override
+ public int maxCompressedLength(int length)
+ {
+ return Snappy.maxCompressedLength(length);
+ }
+
+ @Override
+ public int compress(byte[] src, int srcOffset, int length, byte[] dest, int destOffset) throws IOException
+ {
+ return Snappy.compress(src, 0, src.length, dest, destOffset);
+ }
+
+ @Override
+ public byte[] decompress(byte[] src, int offset, int length, int expectedDecompressedLength) throws IOException
+ {
+ if (!Snappy.isValidCompressedBuffer(src, 0, length))
+ throw new IOException("Provided frame does not appear to be Snappy compressed");
+
+ int uncompressedLength = Snappy.uncompressedLength(src);
+ byte[] output = new byte[uncompressedLength];
+ Snappy.uncompress(src, offset, length, output, 0);
+ return output;
+ }
+}
diff --git a/src/java/org/apache/cassandra/transport/messages/AuthResponse.java b/src/java/org/apache/cassandra/transport/messages/AuthResponse.java
index 332b024..81002c8 100644
--- a/src/java/org/apache/cassandra/transport/messages/AuthResponse.java
+++ b/src/java/org/apache/cassandra/transport/messages/AuthResponse.java
@@ -20,10 +20,11 @@
import java.nio.ByteBuffer;
import io.netty.buffer.ByteBuf;
+import org.apache.cassandra.auth.AuthEvents;
import org.apache.cassandra.auth.AuthenticatedUser;
import org.apache.cassandra.auth.IAuthenticator;
import org.apache.cassandra.exceptions.AuthenticationException;
-import org.apache.cassandra.metrics.AuthMetrics;
+import org.apache.cassandra.metrics.ClientMetrics;
import org.apache.cassandra.service.QueryState;
import org.apache.cassandra.transport.*;
@@ -68,7 +69,7 @@
}
@Override
- public Response execute(QueryState queryState, long queryStartNanoTime)
+ protected Response execute(QueryState queryState, long queryStartNanoTime, boolean traceRequest)
{
try
{
@@ -78,7 +79,8 @@
{
AuthenticatedUser user = negotiator.getAuthenticatedUser();
queryState.getClientState().login(user);
- AuthMetrics.instance.markSuccess();
+ ClientMetrics.instance.markAuthSuccess();
+ AuthEvents.instance.notifyAuthSuccess(queryState);
// authentication is complete, send a ready message to the client
return new AuthSuccess(challenge);
}
@@ -89,7 +91,8 @@
}
catch (AuthenticationException e)
{
- AuthMetrics.instance.markFailure();
+ ClientMetrics.instance.markAuthFailure();
+ AuthEvents.instance.notifyAuthFailure(queryState, e);
return ErrorMessage.fromException(e);
}
}
diff --git a/src/java/org/apache/cassandra/transport/messages/AuthenticateMessage.java b/src/java/org/apache/cassandra/transport/messages/AuthenticateMessage.java
index 1261083..ee3f3fa 100644
--- a/src/java/org/apache/cassandra/transport/messages/AuthenticateMessage.java
+++ b/src/java/org/apache/cassandra/transport/messages/AuthenticateMessage.java
@@ -38,12 +38,13 @@
public void encode(AuthenticateMessage msg, ByteBuf dest, ProtocolVersion version)
{
- CBUtil.writeString(msg.authenticator, dest);
+ // Safe to skip. `msg.authenticator` is a FQCN string. All characters are ASCII encoded.
+ CBUtil.writeAsciiString(msg.authenticator, dest);
}
public int encodedSize(AuthenticateMessage msg, ProtocolVersion version)
{
- return CBUtil.sizeOfString(msg.authenticator);
+ return CBUtil.sizeOfAsciiString(msg.authenticator);
}
};
diff --git a/src/java/org/apache/cassandra/transport/messages/BatchMessage.java b/src/java/org/apache/cassandra/transport/messages/BatchMessage.java
index 0be027f..e760960 100644
--- a/src/java/org/apache/cassandra/transport/messages/BatchMessage.java
+++ b/src/java/org/apache/cassandra/transport/messages/BatchMessage.java
@@ -20,24 +20,31 @@
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
-import java.util.UUID;
import com.google.common.collect.ImmutableMap;
-import io.netty.buffer.ByteBuf;
-import org.apache.cassandra.cql3.*;
+import io.netty.buffer.ByteBuf;
+import org.apache.cassandra.cql3.Attributes;
+import org.apache.cassandra.cql3.BatchQueryOptions;
+import org.apache.cassandra.cql3.CQLStatement;
+import org.apache.cassandra.cql3.QueryEvents;
+import org.apache.cassandra.cql3.QueryHandler;
+import org.apache.cassandra.cql3.QueryOptions;
+import org.apache.cassandra.cql3.QueryProcessor;
+import org.apache.cassandra.cql3.VariableSpecifications;
import org.apache.cassandra.cql3.statements.BatchStatement;
import org.apache.cassandra.cql3.statements.ModificationStatement;
-import org.apache.cassandra.cql3.statements.ParsedStatement;
import org.apache.cassandra.exceptions.InvalidRequestException;
import org.apache.cassandra.exceptions.PreparedQueryNotFoundException;
import org.apache.cassandra.service.ClientState;
import org.apache.cassandra.service.QueryState;
import org.apache.cassandra.tracing.Tracing;
-import org.apache.cassandra.transport.*;
+import org.apache.cassandra.transport.CBUtil;
+import org.apache.cassandra.transport.Message;
+import org.apache.cassandra.transport.ProtocolException;
+import org.apache.cassandra.transport.ProtocolVersion;
import org.apache.cassandra.utils.JVMStabilityInspector;
import org.apache.cassandra.utils.MD5Digest;
-import org.apache.cassandra.utils.UUIDGen;
public class BatchMessage extends Message.Request
{
@@ -147,52 +154,44 @@
this.options = options;
}
- public Message.Response execute(QueryState state, long queryStartNanoTime)
+ @Override
+ protected boolean isTraceable()
{
+ return true;
+ }
+
+ @Override
+ protected Message.Response execute(QueryState state, long queryStartNanoTime, boolean traceRequest)
+ {
+ List<QueryHandler.Prepared> prepared = null;
try
{
- UUID tracingId = null;
- if (isTracingRequested())
- {
- tracingId = UUIDGen.getTimeUUID();
- state.prepareTracingSession(tracingId);
- }
-
- if (state.traceNextQuery())
- {
- state.createTracingSession(getCustomPayload());
-
- ImmutableMap.Builder<String, String> builder = ImmutableMap.builder();
- if(options.getConsistency() != null)
- builder.put("consistency_level", options.getConsistency().name());
- if(options.getSerialConsistency() != null)
- builder.put("serial_consistency_level", options.getSerialConsistency().name());
-
- // TODO we don't have [typed] access to CQL bind variables here. CASSANDRA-4560 is open to add support.
- Tracing.instance.begin("Execute batch of CQL3 queries", state.getClientAddress(), builder.build());
- }
+ if (traceRequest)
+ traceQuery(state);
QueryHandler handler = ClientState.getCQLQueryHandler();
- List<ParsedStatement.Prepared> prepared = new ArrayList<>(queryOrIdList.size());
+ prepared = new ArrayList<>(queryOrIdList.size());
for (int i = 0; i < queryOrIdList.size(); i++)
{
Object query = queryOrIdList.get(i);
- ParsedStatement.Prepared p;
+ CQLStatement statement;
+ QueryHandler.Prepared p;
if (query instanceof String)
{
- p = QueryProcessor.parseStatement((String)query, state);
+ statement = QueryProcessor.parseStatement((String)query, state.getClientState().cloneWithKeyspaceIfSet(options.getKeyspace()));
+ p = new QueryHandler.Prepared(statement, (String) query);
}
else
{
p = handler.getPrepared((MD5Digest)query);
- if (p == null)
+ if (null == p)
throw new PreparedQueryNotFoundException((MD5Digest)query);
}
List<ByteBuffer> queryValues = values.get(i);
- if (queryValues.size() != p.statement.getBoundTerms())
+ if (queryValues.size() != p.statement.getBindVariables().size())
throw new InvalidRequestException(String.format("There were %d markers(?) in CQL but %d bound variables",
- p.statement.getBoundTerms(),
+ p.statement.getBindVariables().size(),
queryValues.size()));
prepared.add(p);
@@ -200,36 +199,48 @@
BatchQueryOptions batchOptions = BatchQueryOptions.withPerStatementVariables(options, values, queryOrIdList);
List<ModificationStatement> statements = new ArrayList<>(prepared.size());
+ List<String> queries = QueryEvents.instance.hasListeners() ? new ArrayList<>(prepared.size()) : null;
for (int i = 0; i < prepared.size(); i++)
{
- ParsedStatement.Prepared p = prepared.get(i);
- batchOptions.prepareStatement(i, p.boundNames);
+ CQLStatement statement = prepared.get(i).statement;
+ if (queries != null)
+ queries.add(prepared.get(i).rawCQLStatement);
+ batchOptions.prepareStatement(i, statement.getBindVariables());
- if (!(p.statement instanceof ModificationStatement))
+ if (!(statement instanceof ModificationStatement))
throw new InvalidRequestException("Invalid statement in batch: only UPDATE, INSERT and DELETE statements are allowed.");
- statements.add((ModificationStatement)p.statement);
+ statements.add((ModificationStatement) statement);
}
// Note: It's ok at this point to pass a bogus value for the number of bound terms in the BatchState ctor
// (and no value would be really correct, so we prefer passing a clearly wrong one).
- BatchStatement batch = new BatchStatement(-1, batchType, statements, Attributes.none());
+ BatchStatement batch = new BatchStatement(batchType, VariableSpecifications.empty(), statements, Attributes.none());
+
+ long queryTime = System.currentTimeMillis();
Message.Response response = handler.processBatch(batch, state, batchOptions, getCustomPayload(), queryStartNanoTime);
-
- if (tracingId != null)
- response.setTracingId(tracingId);
-
+ if (queries != null)
+ QueryEvents.instance.notifyBatchSuccess(batchType, statements, queries, values, options, state, queryTime, response);
return response;
}
catch (Exception e)
{
+ QueryEvents.instance.notifyBatchFailure(prepared, batchType, queryOrIdList, values, options, state, e);
JVMStabilityInspector.inspectThrowable(e);
return ErrorMessage.fromException(e);
}
- finally
- {
- Tracing.instance.stopSession();
- }
+ }
+
+ private void traceQuery(QueryState state)
+ {
+ ImmutableMap.Builder<String, String> builder = ImmutableMap.builder();
+ if (options.getConsistency() != null)
+ builder.put("consistency_level", options.getConsistency().name());
+ if (options.getSerialConsistency() != null)
+ builder.put("serial_consistency_level", options.getSerialConsistency().name());
+
+ // TODO we don't have [typed] access to CQL bind variables here. CASSANDRA-4560 is open to add support.
+ Tracing.instance.begin("Execute batch of CQL3 queries", state.getClientAddress(), builder.build());
}
@Override
diff --git a/src/java/org/apache/cassandra/transport/messages/CredentialsMessage.java b/src/java/org/apache/cassandra/transport/messages/CredentialsMessage.java
deleted file mode 100644
index 764d992..0000000
--- a/src/java/org/apache/cassandra/transport/messages/CredentialsMessage.java
+++ /dev/null
@@ -1,97 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.transport.messages;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import io.netty.buffer.ByteBuf;
-import org.apache.cassandra.auth.AuthenticatedUser;
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.exceptions.AuthenticationException;
-import org.apache.cassandra.metrics.AuthMetrics;
-import org.apache.cassandra.service.QueryState;
-import org.apache.cassandra.transport.CBUtil;
-import org.apache.cassandra.transport.Message;
-import org.apache.cassandra.transport.ProtocolException;
-import org.apache.cassandra.transport.ProtocolVersion;
-
-/**
- * Message to indicate that the server is ready to receive requests.
- */
-public class CredentialsMessage extends Message.Request
-{
- public static final Message.Codec<CredentialsMessage> codec = new Message.Codec<CredentialsMessage>()
- {
- public CredentialsMessage decode(ByteBuf body, ProtocolVersion version)
- {
- if (version.isGreaterThan(ProtocolVersion.V1))
- throw new ProtocolException("Legacy credentials authentication is not supported in " +
- "protocol versions > 1. Please use SASL authentication via a SaslResponse message");
-
- Map<String, String> credentials = CBUtil.readStringMap(body);
- return new CredentialsMessage(credentials);
- }
-
- public void encode(CredentialsMessage msg, ByteBuf dest, ProtocolVersion version)
- {
- CBUtil.writeStringMap(msg.credentials, dest);
- }
-
- public int encodedSize(CredentialsMessage msg, ProtocolVersion version)
- {
- return CBUtil.sizeOfStringMap(msg.credentials);
- }
- };
-
- public final Map<String, String> credentials;
-
- public CredentialsMessage()
- {
- this(new HashMap<String, String>());
- }
-
- private CredentialsMessage(Map<String, String> credentials)
- {
- super(Message.Type.CREDENTIALS);
- this.credentials = credentials;
- }
-
- public Message.Response execute(QueryState state, long queryStartNanoTime)
- {
- try
- {
- AuthenticatedUser user = DatabaseDescriptor.getAuthenticator().legacyAuthenticate(credentials);
- state.getClientState().login(user);
- AuthMetrics.instance.markSuccess();
- }
- catch (AuthenticationException e)
- {
- AuthMetrics.instance.markFailure();
- return ErrorMessage.fromException(e);
- }
-
- return new ReadyMessage();
- }
-
- @Override
- public String toString()
- {
- return "CREDENTIALS";
- }
-}
diff --git a/src/java/org/apache/cassandra/transport/messages/ErrorMessage.java b/src/java/org/apache/cassandra/transport/messages/ErrorMessage.java
index ac4b3dc..14b4ac4 100644
--- a/src/java/org/apache/cassandra/transport/messages/ErrorMessage.java
+++ b/src/java/org/apache/cassandra/transport/messages/ErrorMessage.java
@@ -18,14 +18,15 @@
package org.apache.cassandra.transport.messages;
import java.net.InetAddress;
+import java.util.Collections;
import java.util.List;
import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
import io.netty.buffer.ByteBuf;
import io.netty.handler.codec.CodecException;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Predicate;
+import com.google.common.collect.ImmutableMap;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -33,6 +34,7 @@
import org.apache.cassandra.db.ConsistencyLevel;
import org.apache.cassandra.db.WriteType;
import org.apache.cassandra.exceptions.*;
+import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.transport.*;
import org.apache.cassandra.utils.MD5Digest;
@@ -67,7 +69,7 @@
ConsistencyLevel cl = CBUtil.readConsistencyLevel(body);
int required = body.readInt();
int alive = body.readInt();
- te = new UnavailableException(cl, required, alive);
+ te = UnavailableException.create(cl, required, alive);
}
break;
case OVERLOADED:
@@ -88,15 +90,21 @@
// The number of failures is also present in protocol v5, but used instead to specify the size of the failure map
int failure = body.readInt();
- Map<InetAddress, RequestFailureReason> failureReasonByEndpoint = new ConcurrentHashMap<>();
+ Map<InetAddressAndPort, RequestFailureReason> failureReasonByEndpoint;
if (version.isGreaterOrEqualTo(ProtocolVersion.V5))
{
+ ImmutableMap.Builder<InetAddressAndPort, RequestFailureReason> builder = ImmutableMap.builderWithExpectedSize(failure);
for (int i = 0; i < failure; i++)
{
InetAddress endpoint = CBUtil.readInetAddr(body);
RequestFailureReason failureReason = RequestFailureReason.fromCode(body.readUnsignedShort());
- failureReasonByEndpoint.put(endpoint, failureReason);
+ builder.put(InetAddressAndPort.getByAddress(endpoint), failureReason);
}
+ failureReasonByEndpoint = builder.build();
+ }
+ else
+ {
+ failureReasonByEndpoint = Collections.emptyMap();
}
if (code == ExceptionCode.WRITE_FAILURE)
@@ -113,20 +121,30 @@
break;
case WRITE_TIMEOUT:
case READ_TIMEOUT:
- ConsistencyLevel cl = CBUtil.readConsistencyLevel(body);
- int received = body.readInt();
- int blockFor = body.readInt();
- if (code == ExceptionCode.WRITE_TIMEOUT)
{
- WriteType writeType = Enum.valueOf(WriteType.class, CBUtil.readString(body));
- te = new WriteTimeoutException(writeType, cl, received, blockFor);
+ ConsistencyLevel cl = CBUtil.readConsistencyLevel(body);
+ int received = body.readInt();
+ int blockFor = body.readInt();
+ if (code == ExceptionCode.WRITE_TIMEOUT)
+ {
+ WriteType writeType = Enum.valueOf(WriteType.class, CBUtil.readString(body));
+ if (version.isGreaterOrEqualTo(ProtocolVersion.V5) && writeType == WriteType.CAS)
+ {
+ int contentions = body.readShort();
+ te = new CasWriteTimeoutException(writeType, cl, received, blockFor, contentions);
+ }
+ else
+ {
+ te = new WriteTimeoutException(writeType, cl, received, blockFor);
+ }
+ }
+ else
+ {
+ byte dataPresent = body.readByte();
+ te = new ReadTimeoutException(cl, received, blockFor, dataPresent != 0);
+ }
+ break;
}
- else
- {
- byte dataPresent = body.readByte();
- te = new ReadTimeoutException(cl, received, blockFor, dataPresent != 0);
- }
- break;
case FUNCTION_FAILURE:
String fKeyspace = CBUtil.readString(body);
String fName = CBUtil.readString(body);
@@ -151,6 +169,9 @@
case CONFIG_ERROR:
te = new ConfigurationException(msg);
break;
+ case CDC_WRITE_FAILURE:
+ te = new CDCWriteException(msg);
+ break;
case ALREADY_EXISTS:
String ksName = CBUtil.readString(body);
String cfName = CBUtil.readString(body);
@@ -159,6 +180,14 @@
else
te = new AlreadyExistsException(ksName, cfName);
break;
+ case CAS_WRITE_UNKNOWN:
+ assert version.isGreaterOrEqualTo(ProtocolVersion.V5);
+
+ ConsistencyLevel cl = CBUtil.readConsistencyLevel(body);
+ int received = body.readInt();
+ int blockFor = body.readInt();
+ te = new CasWriteUnknownResultException(cl, received, blockFor);
+ break;
}
return new ErrorMessage(te);
}
@@ -181,8 +210,7 @@
case WRITE_FAILURE:
case READ_FAILURE:
{
- RequestFailureException rfe = (RequestFailureException)err;
- boolean isWrite = err.code() == ExceptionCode.WRITE_FAILURE;
+ RequestFailureException rfe = (RequestFailureException) err;
CBUtil.writeConsistencyLevel(rfe.consistency, dest);
dest.writeInt(rfe.received);
@@ -192,36 +220,42 @@
if (version.isGreaterOrEqualTo(ProtocolVersion.V5))
{
- for (Map.Entry<InetAddress, RequestFailureReason> entry : rfe.failureReasonByEndpoint.entrySet())
+ for (Map.Entry<InetAddressAndPort, RequestFailureReason> entry : rfe.failureReasonByEndpoint.entrySet())
{
- CBUtil.writeInetAddr(entry.getKey(), dest);
+ CBUtil.writeInetAddr(entry.getKey().address, dest);
dest.writeShort(entry.getValue().code);
}
}
- if (isWrite)
- CBUtil.writeString(((WriteFailureException)rfe).writeType.toString(), dest);
+ if (err.code() == ExceptionCode.WRITE_FAILURE)
+ CBUtil.writeAsciiString(((WriteFailureException) rfe).writeType.toString(), dest);
else
- dest.writeByte((byte)(((ReadFailureException)rfe).dataPresent ? 1 : 0));
+ dest.writeByte((byte) (((ReadFailureException) rfe).dataPresent ? 1 : 0));
+ break;
}
- break;
case WRITE_TIMEOUT:
case READ_TIMEOUT:
RequestTimeoutException rte = (RequestTimeoutException)err;
- boolean isWrite = err.code() == ExceptionCode.WRITE_TIMEOUT;
CBUtil.writeConsistencyLevel(rte.consistency, dest);
dest.writeInt(rte.received);
dest.writeInt(rte.blockFor);
- if (isWrite)
- CBUtil.writeString(((WriteTimeoutException)rte).writeType.toString(), dest);
+ if (err.code() == ExceptionCode.WRITE_TIMEOUT)
+ {
+ CBUtil.writeAsciiString(((WriteTimeoutException)rte).writeType.toString(), dest);
+ // CasWriteTimeoutException already implies protocol V5, but double check to be safe.
+ if (version.isGreaterOrEqualTo(ProtocolVersion.V5) && rte instanceof CasWriteTimeoutException)
+ dest.writeShort(((CasWriteTimeoutException)rte).contentions);
+ }
else
+ {
dest.writeByte((byte)(((ReadTimeoutException)rte).dataPresent ? 1 : 0));
+ }
break;
case FUNCTION_FAILURE:
FunctionExecutionException fee = (FunctionExecutionException)msg.error;
- CBUtil.writeString(fee.functionName.keyspace, dest);
- CBUtil.writeString(fee.functionName.name, dest);
+ CBUtil.writeAsciiString(fee.functionName.keyspace, dest);
+ CBUtil.writeAsciiString(fee.functionName.name, dest);
CBUtil.writeStringList(fee.argTypes, dest);
break;
case UNPREPARED:
@@ -230,15 +264,21 @@
break;
case ALREADY_EXISTS:
AlreadyExistsException aee = (AlreadyExistsException)err;
- CBUtil.writeString(aee.ksName, dest);
- CBUtil.writeString(aee.cfName, dest);
+ CBUtil.writeAsciiString(aee.ksName, dest);
+ CBUtil.writeAsciiString(aee.cfName, dest);
break;
+ case CAS_WRITE_UNKNOWN:
+ assert version.isGreaterOrEqualTo(ProtocolVersion.V5);
+ CasWriteUnknownResultException cwue = (CasWriteUnknownResultException)err;
+ CBUtil.writeConsistencyLevel(cwue.consistency, dest);
+ dest.writeInt(cwue.received);
+ dest.writeInt(cwue.blockFor);
}
}
public int encodedSize(ErrorMessage msg, ProtocolVersion version)
{
- final TransportException err = getBackwardsCompatibleException(msg, version);
+ TransportException err = getBackwardsCompatibleException(msg, version);
String errorString = err.getMessage() == null ? "" : err.getMessage();
int size = 4 + CBUtil.sizeOfString(errorString);
switch (err.code())
@@ -251,15 +291,18 @@
case READ_FAILURE:
{
RequestFailureException rfe = (RequestFailureException)err;
- boolean isWrite = err.code() == ExceptionCode.WRITE_FAILURE;
+
size += CBUtil.sizeOfConsistencyLevel(rfe.consistency) + 4 + 4 + 4;
- size += isWrite ? CBUtil.sizeOfString(((WriteFailureException)rfe).writeType.toString()) : 1;
+ if (err.code() == ExceptionCode.WRITE_FAILURE)
+ size += CBUtil.sizeOfAsciiString(((WriteFailureException)rfe).writeType.toString());
+ else
+ size += 1;
if (version.isGreaterOrEqualTo(ProtocolVersion.V5))
{
- for (Map.Entry<InetAddress, RequestFailureReason> entry : rfe.failureReasonByEndpoint.entrySet())
+ for (Map.Entry<InetAddressAndPort, RequestFailureReason> entry : rfe.failureReasonByEndpoint.entrySet())
{
- size += CBUtil.sizeOfInetAddr(entry.getKey());
+ size += CBUtil.sizeOfInetAddr(entry.getKey().address);
size += 2; // RequestFailureReason code
}
}
@@ -270,12 +313,19 @@
RequestTimeoutException rte = (RequestTimeoutException)err;
boolean isWrite = err.code() == ExceptionCode.WRITE_TIMEOUT;
size += CBUtil.sizeOfConsistencyLevel(rte.consistency) + 8;
- size += isWrite ? CBUtil.sizeOfString(((WriteTimeoutException)rte).writeType.toString()) : 1;
+ if (isWrite)
+ size += CBUtil.sizeOfAsciiString(((WriteTimeoutException)rte).writeType.toString());
+ else
+ size += 1;
+
+ // CasWriteTimeoutException already implies protocol V5, but double check to be safe.
+ if (isWrite && version.isGreaterOrEqualTo(ProtocolVersion.V5) && rte instanceof CasWriteTimeoutException)
+ size += 2; // CasWriteTimeoutException appends a short for contentions occured.
break;
case FUNCTION_FAILURE:
FunctionExecutionException fee = (FunctionExecutionException)msg.error;
- size += CBUtil.sizeOfString(fee.functionName.keyspace);
- size += CBUtil.sizeOfString(fee.functionName.name);
+ size += CBUtil.sizeOfAsciiString(fee.functionName.keyspace);
+ size += CBUtil.sizeOfAsciiString(fee.functionName.name);
size += CBUtil.sizeOfStringList(fee.argTypes);
break;
case UNPREPARED:
@@ -284,8 +334,13 @@
break;
case ALREADY_EXISTS:
AlreadyExistsException aee = (AlreadyExistsException)err;
- size += CBUtil.sizeOfString(aee.ksName);
- size += CBUtil.sizeOfString(aee.cfName);
+ size += CBUtil.sizeOfAsciiString(aee.ksName);
+ size += CBUtil.sizeOfAsciiString(aee.cfName);
+ break;
+ case CAS_WRITE_UNKNOWN:
+ assert version.isGreaterOrEqualTo(ProtocolVersion.V5);
+ CasWriteUnknownResultException cwue = (CasWriteUnknownResultException)err;
+ size += CBUtil.sizeOfConsistencyLevel(cwue.consistency) + 4 + 4; // receivedFor: 4, blockFor: 4
break;
}
return size;
@@ -305,10 +360,28 @@
WriteFailureException wfe = (WriteFailureException) msg.error;
return new WriteTimeoutException(wfe.writeType, wfe.consistency, wfe.received, wfe.blockFor);
case FUNCTION_FAILURE:
+ case CDC_WRITE_FAILURE:
return new InvalidRequestException(msg.toString());
}
}
+ if (version.isSmallerThan(ProtocolVersion.V5))
+ {
+ switch (msg.error.code())
+ {
+ case WRITE_TIMEOUT:
+ if (msg.error instanceof CasWriteTimeoutException)
+ {
+ CasWriteTimeoutException cwte = (CasWriteTimeoutException) msg.error;
+ return new WriteTimeoutException(WriteType.CAS, cwte.consistency, cwte.received, cwte.blockFor);
+ }
+ break;
+ case CAS_WRITE_UNKNOWN:
+ CasWriteUnknownResultException cwue = (CasWriteUnknownResultException) msg.error;
+ return new WriteTimeoutException(WriteType.CAS, cwue.consistency, cwue.received, cwue.blockFor);
+ }
+ }
+
return msg.error;
}
@@ -371,7 +444,7 @@
if (e instanceof ProtocolException)
{
// if the driver attempted to connect with a protocol version not supported then
- // reply with the appropiate version, see ProtocolVersion.decode()
+ // respond with the appropiate version, see ProtocolVersion.decode()
ProtocolVersion forcedProtocolVersion = ((ProtocolException) e).getForcedProtocolVersion();
if (forcedProtocolVersion != null)
message.forcedProtocolVersion = forcedProtocolVersion;
diff --git a/src/java/org/apache/cassandra/transport/messages/ExecuteMessage.java b/src/java/org/apache/cassandra/transport/messages/ExecuteMessage.java
index d881e63..3b98996 100644
--- a/src/java/org/apache/cassandra/transport/messages/ExecuteMessage.java
+++ b/src/java/org/apache/cassandra/transport/messages/ExecuteMessage.java
@@ -17,24 +17,25 @@
*/
package org.apache.cassandra.transport.messages;
-import java.util.UUID;
-
import com.google.common.collect.ImmutableMap;
-import io.netty.buffer.ByteBuf;
+import io.netty.buffer.ByteBuf;
import org.apache.cassandra.cql3.CQLStatement;
import org.apache.cassandra.cql3.ColumnSpecification;
+import org.apache.cassandra.cql3.QueryEvents;
import org.apache.cassandra.cql3.QueryHandler;
import org.apache.cassandra.cql3.QueryOptions;
-import org.apache.cassandra.cql3.statements.ParsedStatement;
+import org.apache.cassandra.cql3.ResultSet;
import org.apache.cassandra.exceptions.PreparedQueryNotFoundException;
import org.apache.cassandra.service.ClientState;
import org.apache.cassandra.service.QueryState;
import org.apache.cassandra.tracing.Tracing;
-import org.apache.cassandra.transport.*;
+import org.apache.cassandra.transport.CBUtil;
+import org.apache.cassandra.transport.Message;
+import org.apache.cassandra.transport.ProtocolException;
+import org.apache.cassandra.transport.ProtocolVersion;
import org.apache.cassandra.utils.JVMStabilityInspector;
import org.apache.cassandra.utils.MD5Digest;
-import org.apache.cassandra.utils.UUIDGen;
public class ExecuteMessage extends Message.Request
{
@@ -42,13 +43,22 @@
{
public ExecuteMessage decode(ByteBuf body, ProtocolVersion version)
{
- byte[] id = CBUtil.readBytes(body);
- return new ExecuteMessage(MD5Digest.wrap(id), QueryOptions.codec.decode(body, version));
+ MD5Digest statementId = MD5Digest.wrap(CBUtil.readBytes(body));
+
+ MD5Digest resultMetadataId = null;
+ if (version.isGreaterOrEqualTo(ProtocolVersion.V5))
+ resultMetadataId = MD5Digest.wrap(CBUtil.readBytes(body));
+
+ return new ExecuteMessage(statementId, resultMetadataId, QueryOptions.codec.decode(body, version));
}
public void encode(ExecuteMessage msg, ByteBuf dest, ProtocolVersion version)
{
CBUtil.writeBytes(msg.statementId.bytes, dest);
+
+ if (version.isGreaterOrEqualTo(ProtocolVersion.V5))
+ CBUtil.writeBytes(msg.resultMetadataId.bytes, dest);
+
if (version == ProtocolVersion.V1)
{
CBUtil.writeValueList(msg.options.getValues(), dest);
@@ -64,6 +74,10 @@
{
int size = 0;
size += CBUtil.sizeOfBytes(msg.statementId.bytes);
+
+ if (version.isGreaterOrEqualTo(ProtocolVersion.V5))
+ size += CBUtil.sizeOfBytes(msg.resultMetadataId.bytes);
+
if (version == ProtocolVersion.V1)
{
size += CBUtil.sizeOfValueList(msg.options.getValues());
@@ -78,94 +92,124 @@
};
public final MD5Digest statementId;
+ public final MD5Digest resultMetadataId;
public final QueryOptions options;
- public ExecuteMessage(MD5Digest statementId, QueryOptions options)
+ public ExecuteMessage(MD5Digest statementId, MD5Digest resultMetadataId, QueryOptions options)
{
super(Message.Type.EXECUTE);
this.statementId = statementId;
this.options = options;
+ this.resultMetadataId = resultMetadataId;
}
- public Message.Response execute(QueryState state, long queryStartNanoTime)
+ @Override
+ protected boolean isTraceable()
{
+ return true;
+ }
+
+ @Override
+ protected Message.Response execute(QueryState state, long queryStartNanoTime, boolean traceRequest)
+ {
+ QueryHandler.Prepared prepared = null;
try
{
QueryHandler handler = ClientState.getCQLQueryHandler();
- ParsedStatement.Prepared prepared = handler.getPrepared(statementId);
+ prepared = handler.getPrepared(statementId);
if (prepared == null)
throw new PreparedQueryNotFoundException(statementId);
- options.prepare(prepared.boundNames);
CQLStatement statement = prepared.statement;
+ options.prepare(statement.getBindVariables());
if (options.getPageSize() == 0)
throw new ProtocolException("The page size cannot be 0");
- UUID tracingId = null;
- if (isTracingRequested())
- {
- tracingId = UUIDGen.getTimeUUID();
- state.prepareTracingSession(tracingId);
- }
-
- if (state.traceNextQuery())
- {
- state.createTracingSession(getCustomPayload());
-
- ImmutableMap.Builder<String, String> builder = ImmutableMap.builder();
- if (options.getPageSize() > 0)
- builder.put("page_size", Integer.toString(options.getPageSize()));
- if(options.getConsistency() != null)
- builder.put("consistency_level", options.getConsistency().name());
- if(options.getSerialConsistency() != null)
- builder.put("serial_consistency_level", options.getSerialConsistency().name());
- builder.put("query", prepared.rawCQLStatement);
-
- for(int i=0;i<prepared.boundNames.size();i++)
- {
- ColumnSpecification cs = prepared.boundNames.get(i);
- String boundName = cs.name.toString();
- String boundValue = cs.type.asCQL3Type().toCQLLiteral(options.getValues().get(i), options.getProtocolVersion());
- if ( boundValue.length() > 1000 )
- {
- boundValue = boundValue.substring(0, 1000) + "...'";
- }
-
- //Here we prefix boundName with the index to avoid possible collission in builder keys due to
- //having multiple boundValues for the same variable
- builder.put("bound_var_" + Integer.toString(i) + "_" + boundName, boundValue);
- }
-
- Tracing.instance.begin("Execute CQL3 prepared query", state.getClientAddress(), builder.build());
- }
+ if (traceRequest)
+ traceQuery(state, prepared);
// Some custom QueryHandlers are interested by the bound names. We provide them this information
// by wrapping the QueryOptions.
- QueryOptions queryOptions = QueryOptions.addColumnSpecifications(options, prepared.boundNames);
- Message.Response response = handler.processPrepared(statement, state, queryOptions, getCustomPayload(), queryStartNanoTime);
- if (options.skipMetadata() && response instanceof ResultMessage.Rows)
- ((ResultMessage.Rows)response).result.metadata.setSkipMetadata();
+ QueryOptions queryOptions = QueryOptions.addColumnSpecifications(options, prepared.statement.getBindVariables());
- if (tracingId != null)
- response.setTracingId(tracingId);
+ long requestStartTime = System.currentTimeMillis();
+
+ Message.Response response = handler.processPrepared(statement, state, queryOptions, getCustomPayload(), queryStartNanoTime);
+
+ QueryEvents.instance.notifyExecuteSuccess(prepared.statement, prepared.rawCQLStatement, options, state, requestStartTime, response);
+
+ if (response instanceof ResultMessage.Rows)
+ {
+ ResultMessage.Rows rows = (ResultMessage.Rows) response;
+
+ ResultSet.ResultMetadata resultMetadata = rows.result.metadata;
+
+ if (options.getProtocolVersion().isGreaterOrEqualTo(ProtocolVersion.V5))
+ {
+ // For LWTs, always send a resultset metadata but avoid setting a metadata changed flag. This way
+ // Client will always receive fresh metadata, but will avoid caching and reusing it. See CASSANDRA-13992
+ // for details.
+ if (!statement.hasConditions())
+ {
+ // Starting with V5 we can rely on the result metadata id coming with execute message in order to
+ // check if there was a change, comparing it with metadata that's about to be returned to client.
+ if (!resultMetadata.getResultMetadataId().equals(resultMetadataId))
+ resultMetadata.setMetadataChanged();
+ else if (options.skipMetadata())
+ resultMetadata.setSkipMetadata();
+ }
+ }
+ else
+ {
+ // Pre-V5 code has to rely on the difference between the metadata in the prepared message cache
+ // and compare it with the metadata to be returned to client.
+ if (options.skipMetadata() && prepared.resultMetadataId.equals(resultMetadata.getResultMetadataId()))
+ resultMetadata.setSkipMetadata();
+ }
+ }
return response;
}
catch (Exception e)
{
+ QueryEvents.instance.notifyExecuteFailure(prepared, options, state, e);
JVMStabilityInspector.inspectThrowable(e);
return ErrorMessage.fromException(e);
}
- finally
+ }
+
+ private void traceQuery(QueryState state, QueryHandler.Prepared prepared)
+ {
+ ImmutableMap.Builder<String, String> builder = ImmutableMap.builder();
+ if (options.getPageSize() > 0)
+ builder.put("page_size", Integer.toString(options.getPageSize()));
+ if (options.getConsistency() != null)
+ builder.put("consistency_level", options.getConsistency().name());
+ if (options.getSerialConsistency() != null)
+ builder.put("serial_consistency_level", options.getSerialConsistency().name());
+
+ builder.put("query", prepared.rawCQLStatement);
+
+ for (int i = 0; i < prepared.statement.getBindVariables().size(); i++)
{
- Tracing.instance.stopSession();
+ ColumnSpecification cs = prepared.statement.getBindVariables().get(i);
+ String boundName = cs.name.toString();
+ String boundValue = cs.type.asCQL3Type().toCQLLiteral(options.getValues().get(i), options.getProtocolVersion());
+ if (boundValue.length() > 1000)
+ boundValue = boundValue.substring(0, 1000) + "...'";
+
+ //Here we prefix boundName with the index to avoid possible collission in builder keys due to
+ //having multiple boundValues for the same variable
+ builder.put("bound_var_" + i + '_' + boundName, boundValue);
}
+
+ Tracing.instance.begin("Execute CQL3 prepared query", state.getClientAddress(), builder.build());
}
@Override
public String toString()
{
- return "EXECUTE " + statementId + " with " + options.getValues().size() + " values at consistency " + options.getConsistency();
+ return String.format("EXECUTE %s with %d values at consistency %s", statementId, options.getValues().size(), options.getConsistency());
}
}
diff --git a/src/java/org/apache/cassandra/transport/messages/OptionsMessage.java b/src/java/org/apache/cassandra/transport/messages/OptionsMessage.java
index 914ccb1..a29145a 100644
--- a/src/java/org/apache/cassandra/transport/messages/OptionsMessage.java
+++ b/src/java/org/apache/cassandra/transport/messages/OptionsMessage.java
@@ -26,9 +26,10 @@
import org.apache.cassandra.cql3.QueryProcessor;
import org.apache.cassandra.service.QueryState;
-import org.apache.cassandra.transport.FrameCompressor;
import org.apache.cassandra.transport.Message;
import org.apache.cassandra.transport.ProtocolVersion;
+import org.apache.cassandra.transport.frame.compress.SnappyCompressor;
+import org.apache.cassandra.utils.ChecksumType;
/**
* Message to indicate that the server is ready to receive requests.
@@ -57,22 +58,32 @@
super(Message.Type.OPTIONS);
}
- public Message.Response execute(QueryState state, long queryStartNanoTime)
+ @Override
+ protected Message.Response execute(QueryState state, long queryStartNanoTime, boolean traceRequest)
{
- List<String> cqlVersions = new ArrayList<String>();
+ List<String> cqlVersions = new ArrayList<>();
cqlVersions.add(QueryProcessor.CQL_VERSION.toString());
- List<String> compressions = new ArrayList<String>();
- if (FrameCompressor.SnappyCompressor.instance != null)
+ List<String> compressions = new ArrayList<>();
+ if (SnappyCompressor.INSTANCE != null)
compressions.add("snappy");
// LZ4 is always available since worst case scenario it default to a pure JAVA implem.
compressions.add("lz4");
- Map<String, List<String>> supported = new HashMap<String, List<String>>();
+ Map<String, List<String>> supported = new HashMap<>();
supported.put(StartupMessage.CQL_VERSION, cqlVersions);
supported.put(StartupMessage.COMPRESSION, compressions);
supported.put(StartupMessage.PROTOCOL_VERSIONS, ProtocolVersion.supportedVersions());
+ if (connection.getVersion().supportsChecksums())
+ {
+ ChecksumType[] types = ChecksumType.values();
+ List<String> checksumImpls = new ArrayList<>(types.length);
+ for (ChecksumType type : types)
+ checksumImpls.add(type.toString());
+ supported.put(StartupMessage.CHECKSUM, checksumImpls);
+ }
+
return new SupportedMessage(supported);
}
diff --git a/src/java/org/apache/cassandra/transport/messages/PrepareMessage.java b/src/java/org/apache/cassandra/transport/messages/PrepareMessage.java
index 04d2966..fa77c68 100644
--- a/src/java/org/apache/cassandra/transport/messages/PrepareMessage.java
+++ b/src/java/org/apache/cassandra/transport/messages/PrepareMessage.java
@@ -17,17 +17,20 @@
*/
package org.apache.cassandra.transport.messages;
-import java.util.UUID;
+import java.util.function.Supplier;
import com.google.common.collect.ImmutableMap;
-import io.netty.buffer.ByteBuf;
+import io.netty.buffer.ByteBuf;
+import org.apache.cassandra.cql3.QueryEvents;
+import org.apache.cassandra.cql3.QueryHandler;
import org.apache.cassandra.service.ClientState;
import org.apache.cassandra.service.QueryState;
import org.apache.cassandra.tracing.Tracing;
-import org.apache.cassandra.transport.*;
+import org.apache.cassandra.transport.CBUtil;
+import org.apache.cassandra.transport.Message;
+import org.apache.cassandra.transport.ProtocolVersion;
import org.apache.cassandra.utils.JVMStabilityInspector;
-import org.apache.cassandra.utils.UUIDGen;
public class PrepareMessage extends Message.Request
{
@@ -36,61 +39,88 @@
public PrepareMessage decode(ByteBuf body, ProtocolVersion version)
{
String query = CBUtil.readLongString(body);
- return new PrepareMessage(query);
+ String keyspace = null;
+ if (version.isGreaterOrEqualTo(ProtocolVersion.V5)) {
+ // If flags grows, we may want to consider creating a PrepareOptions class with an internal codec
+ // class that handles flags and options of the prepare message. Since there's only one right now,
+ // we just take care of business here.
+
+ int flags = (int)body.readUnsignedInt();
+ if ((flags & 0x1) == 0x1)
+ keyspace = CBUtil.readString(body);
+ }
+ return new PrepareMessage(query, keyspace);
}
public void encode(PrepareMessage msg, ByteBuf dest, ProtocolVersion version)
{
CBUtil.writeLongString(msg.query, dest);
+ if (version.isGreaterOrEqualTo(ProtocolVersion.V5))
+ {
+ // If we have no keyspace, write out a 0-valued flag field.
+ if (msg.keyspace == null)
+ dest.writeInt(0x0);
+ else {
+ dest.writeInt(0x1);
+ CBUtil.writeAsciiString(msg.keyspace, dest);
+ }
+ }
}
public int encodedSize(PrepareMessage msg, ProtocolVersion version)
{
- return CBUtil.sizeOfLongString(msg.query);
+ int size = CBUtil.sizeOfLongString(msg.query);
+ if (version.isGreaterOrEqualTo(ProtocolVersion.V5))
+ {
+ // We always emit a flags int
+ size += 4;
+
+ // If we have a keyspace, we'd write it out. Otherwise, we'd write nothing.
+ size += msg.keyspace == null
+ ? 0
+ : CBUtil.sizeOfAsciiString(msg.keyspace);
+ }
+ return size;
}
};
private final String query;
+ private final String keyspace;
- public PrepareMessage(String query)
+ public PrepareMessage(String query, String keyspace)
{
super(Message.Type.PREPARE);
this.query = query;
+ this.keyspace = keyspace;
}
- public Message.Response execute(QueryState state, long queryStartNanoTime)
+ @Override
+ protected boolean isTraceable()
+ {
+ return true;
+ }
+
+ @Override
+ protected Message.Response execute(QueryState state, long queryStartNanoTime, boolean traceRequest)
{
try
{
- UUID tracingId = null;
- if (isTracingRequested())
- {
- tracingId = UUIDGen.getTimeUUID();
- state.prepareTracingSession(tracingId);
- }
-
- if (state.traceNextQuery())
- {
- state.createTracingSession(getCustomPayload());
+ if (traceRequest)
Tracing.instance.begin("Preparing CQL3 query", state.getClientAddress(), ImmutableMap.of("query", query));
- }
- Message.Response response = ClientState.getCQLQueryHandler().prepare(query, state, getCustomPayload());
-
- if (tracingId != null)
- response.setTracingId(tracingId);
-
+ ClientState clientState = state.getClientState().cloneWithKeyspaceIfSet(keyspace);
+ QueryHandler queryHandler = ClientState.getCQLQueryHandler();
+ long queryTime = System.currentTimeMillis();
+ ResultMessage.Prepared response = queryHandler.prepare(query, clientState, getCustomPayload());
+ QueryEvents.instance.notifyPrepareSuccess(() -> queryHandler.getPrepared(response.statementId), query, state, queryTime, response);
return response;
}
catch (Exception e)
{
+ QueryEvents.instance.notifyPrepareFailure(null, query, state, e);
JVMStabilityInspector.inspectThrowable(e);
return ErrorMessage.fromException(e);
}
- finally
- {
- Tracing.instance.stopSession();
- }
}
@Override
diff --git a/src/java/org/apache/cassandra/transport/messages/QueryMessage.java b/src/java/org/apache/cassandra/transport/messages/QueryMessage.java
index 4c761dd..71d7c73 100644
--- a/src/java/org/apache/cassandra/transport/messages/QueryMessage.java
+++ b/src/java/org/apache/cassandra/transport/messages/QueryMessage.java
@@ -17,11 +17,12 @@
*/
package org.apache.cassandra.transport.messages;
-import java.util.UUID;
-
import com.google.common.collect.ImmutableMap;
import io.netty.buffer.ByteBuf;
+import org.apache.cassandra.cql3.CQLStatement;
+import org.apache.cassandra.cql3.QueryEvents;
+import org.apache.cassandra.cql3.QueryHandler;
import org.apache.cassandra.cql3.QueryOptions;
import org.apache.cassandra.exceptions.RequestExecutionException;
import org.apache.cassandra.exceptions.RequestValidationException;
@@ -33,7 +34,6 @@
import org.apache.cassandra.transport.ProtocolException;
import org.apache.cassandra.transport.ProtocolVersion;
import org.apache.cassandra.utils.JVMStabilityInspector;
-import org.apache.cassandra.utils.UUIDGen;
/**
* A CQL query
@@ -83,61 +83,63 @@
this.options = options;
}
- public Message.Response execute(QueryState state, long queryStartNanoTime)
+ @Override
+ protected boolean isTraceable()
{
+ return true;
+ }
+
+ @Override
+ protected Message.Response execute(QueryState state, long queryStartNanoTime, boolean traceRequest)
+ {
+ CQLStatement statement = null;
try
{
if (options.getPageSize() == 0)
throw new ProtocolException("The page size cannot be 0");
- UUID tracingId = null;
- if (isTracingRequested())
- {
- tracingId = UUIDGen.getTimeUUID();
- state.prepareTracingSession(tracingId);
- }
+ if (traceRequest)
+ traceQuery(state);
- if (state.traceNextQuery())
- {
- state.createTracingSession(getCustomPayload());
+ long queryStartTime = System.currentTimeMillis();
- ImmutableMap.Builder<String, String> builder = ImmutableMap.builder();
- builder.put("query", query);
- if (options.getPageSize() > 0)
- builder.put("page_size", Integer.toString(options.getPageSize()));
- if(options.getConsistency() != null)
- builder.put("consistency_level", options.getConsistency().name());
- if(options.getSerialConsistency() != null)
- builder.put("serial_consistency_level", options.getSerialConsistency().name());
+ QueryHandler queryHandler = ClientState.getCQLQueryHandler();
+ statement = queryHandler.parse(query, state, options);
+ Message.Response response = queryHandler.process(statement, state, options, getCustomPayload(), queryStartNanoTime);
+ QueryEvents.instance.notifyQuerySuccess(statement, query, options, state, queryStartTime, response);
- Tracing.instance.begin("Execute CQL3 query", state.getClientAddress(), builder.build());
- }
-
- Message.Response response = ClientState.getCQLQueryHandler().process(query, state, options, getCustomPayload(), queryStartNanoTime);
if (options.skipMetadata() && response instanceof ResultMessage.Rows)
((ResultMessage.Rows)response).result.metadata.setSkipMetadata();
- if (tracingId != null)
- response.setTracingId(tracingId);
-
return response;
}
catch (Exception e)
{
+ QueryEvents.instance.notifyQueryFailure(statement, query, options, state, e);
JVMStabilityInspector.inspectThrowable(e);
if (!((e instanceof RequestValidationException) || (e instanceof RequestExecutionException)))
logger.error("Unexpected error during query", e);
return ErrorMessage.fromException(e);
}
- finally
- {
- Tracing.instance.stopSession();
- }
+ }
+
+ private void traceQuery(QueryState state)
+ {
+ ImmutableMap.Builder<String, String> builder = ImmutableMap.builder();
+ builder.put("query", query);
+ if (options.getPageSize() > 0)
+ builder.put("page_size", Integer.toString(options.getPageSize()));
+ if (options.getConsistency() != null)
+ builder.put("consistency_level", options.getConsistency().name());
+ if (options.getSerialConsistency() != null)
+ builder.put("serial_consistency_level", options.getSerialConsistency().name());
+
+ Tracing.instance.begin("Execute CQL3 query", state.getClientAddress(), builder.build());
}
@Override
public String toString()
{
- return "QUERY " + query + "[pageSize = " + options.getPageSize() + "]";
+ return String.format("QUERY %s [pageSize = %d]", query, options.getPageSize());
}
}
diff --git a/src/java/org/apache/cassandra/transport/messages/RegisterMessage.java b/src/java/org/apache/cassandra/transport/messages/RegisterMessage.java
index 2356dae..f8eb55e 100644
--- a/src/java/org/apache/cassandra/transport/messages/RegisterMessage.java
+++ b/src/java/org/apache/cassandra/transport/messages/RegisterMessage.java
@@ -62,7 +62,8 @@
this.eventTypes = eventTypes;
}
- public Response execute(QueryState state, long queryStartNanoTime)
+ @Override
+ protected Response execute(QueryState state, long queryStartNanoTime, boolean traceRequest)
{
assert connection instanceof ServerConnection;
Connection.Tracker tracker = connection.getTracker();
diff --git a/src/java/org/apache/cassandra/transport/messages/ResultMessage.java b/src/java/org/apache/cassandra/transport/messages/ResultMessage.java
index 05a1276..a8d8dae 100644
--- a/src/java/org/apache/cassandra/transport/messages/ResultMessage.java
+++ b/src/java/org/apache/cassandra/transport/messages/ResultMessage.java
@@ -17,19 +17,13 @@
*/
package org.apache.cassandra.transport.messages;
-import java.util.*;
+
+import com.google.common.annotations.VisibleForTesting;
import io.netty.buffer.ByteBuf;
-import org.apache.cassandra.cql3.ColumnSpecification;
-import org.apache.cassandra.cql3.CQLStatement;
import org.apache.cassandra.cql3.ResultSet;
-import org.apache.cassandra.cql3.statements.SelectStatement;
-import org.apache.cassandra.cql3.statements.ParsedStatement;
import org.apache.cassandra.transport.*;
-import org.apache.cassandra.thrift.CqlPreparedResult;
-import org.apache.cassandra.thrift.CqlResult;
-import org.apache.cassandra.thrift.CqlResultType;
import org.apache.cassandra.utils.MD5Digest;
public abstract class ResultMessage extends Message.Response
@@ -56,12 +50,12 @@
public enum Kind
{
- VOID (1, Void.subcodec),
- ROWS (2, Rows.subcodec),
- SET_KEYSPACE (3, SetKeyspace.subcodec),
- PREPARED (4, Prepared.subcodec),
- SCHEMA_CHANGE(5, SchemaChange.subcodec);
+ VOID (1, Void.subcodec),
+ ROWS (2, Rows.subcodec),
+ SET_KEYSPACE (3, SetKeyspace.subcodec),
+ PREPARED (4, Prepared.subcodec),
+ SCHEMA_CHANGE (5, SchemaChange.subcodec);
public final int id;
public final Message.Codec<ResultMessage> subcodec;
@@ -103,8 +97,6 @@
this.kind = kind;
}
- public abstract CqlResult toThriftResult();
-
public static class Void extends ResultMessage
{
// Even though we have no specific information here, don't make a
@@ -132,11 +124,6 @@
}
};
- public CqlResult toThriftResult()
- {
- return new CqlResult(CqlResultType.VOID);
- }
-
@Override
public String toString()
{
@@ -165,21 +152,16 @@
public void encode(ResultMessage msg, ByteBuf dest, ProtocolVersion version)
{
assert msg instanceof SetKeyspace;
- CBUtil.writeString(((SetKeyspace)msg).keyspace, dest);
+ CBUtil.writeAsciiString(((SetKeyspace)msg).keyspace, dest);
}
public int encodedSize(ResultMessage msg, ProtocolVersion version)
{
assert msg instanceof SetKeyspace;
- return CBUtil.sizeOfString(((SetKeyspace)msg).keyspace);
+ return CBUtil.sizeOfAsciiString(((SetKeyspace)msg).keyspace);
}
};
- public CqlResult toThriftResult()
- {
- return new CqlResult(CqlResultType.VOID);
- }
-
@Override
public String toString()
{
@@ -219,11 +201,6 @@
this.result = result;
}
- public CqlResult toThriftResult()
- {
- return result.toThriftResult();
- }
-
@Override
public String toString()
{
@@ -238,13 +215,16 @@
public ResultMessage decode(ByteBuf body, ProtocolVersion version)
{
MD5Digest id = MD5Digest.wrap(CBUtil.readBytes(body));
+ MD5Digest resultMetadataId = null;
+ if (version.isGreaterOrEqualTo(ProtocolVersion.V5))
+ resultMetadataId = MD5Digest.wrap(CBUtil.readBytes(body));
ResultSet.PreparedMetadata metadata = ResultSet.PreparedMetadata.codec.decode(body, version);
ResultSet.ResultMetadata resultMetadata = ResultSet.ResultMetadata.EMPTY;
if (version.isGreaterThan(ProtocolVersion.V1))
resultMetadata = ResultSet.ResultMetadata.codec.decode(body, version);
- return new Prepared(id, -1, metadata, resultMetadata);
+ return new Prepared(id, resultMetadataId, metadata, resultMetadata);
}
public void encode(ResultMessage msg, ByteBuf dest, ProtocolVersion version)
@@ -254,6 +234,9 @@
assert prepared.statementId != null;
CBUtil.writeBytes(prepared.statementId.bytes, dest);
+ if (version.isGreaterOrEqualTo(ProtocolVersion.V5))
+ CBUtil.writeBytes(prepared.resultMetadataId.bytes, dest);
+
ResultSet.PreparedMetadata.codec.encode(prepared.metadata, dest, version);
if (version.isGreaterThan(ProtocolVersion.V1))
ResultSet.ResultMetadata.codec.encode(prepared.resultMetadata, dest, version);
@@ -267,6 +250,8 @@
int size = 0;
size += CBUtil.sizeOfBytes(prepared.statementId.bytes);
+ if (version.isGreaterOrEqualTo(ProtocolVersion.V5))
+ size += CBUtil.sizeOfBytes(prepared.resultMetadataId.bytes);
size += ResultSet.PreparedMetadata.codec.encodedSize(prepared.metadata, version);
if (version.isGreaterThan(ProtocolVersion.V1))
size += ResultSet.ResultMetadata.codec.encodedSize(prepared.resultMetadata, version);
@@ -275,6 +260,7 @@
};
public final MD5Digest statementId;
+ public final MD5Digest resultMetadataId;
/** Describes the variables to be bound in the prepared statement */
public final ResultSet.PreparedMetadata metadata;
@@ -282,51 +268,19 @@
/** Describes the results of executing this prepared statement */
public final ResultSet.ResultMetadata resultMetadata;
- // statement id for CQL-over-thrift compatibility. The binary protocol ignore that.
- private final int thriftStatementId;
-
- public Prepared(MD5Digest statementId, ParsedStatement.Prepared prepared)
- {
- this(statementId, -1, new ResultSet.PreparedMetadata(prepared.boundNames, prepared.partitionKeyBindIndexes), extractResultMetadata(prepared.statement));
- }
-
- public static Prepared forThrift(int statementId, List<ColumnSpecification> names)
- {
- return new Prepared(null, statementId, new ResultSet.PreparedMetadata(names, null), ResultSet.ResultMetadata.EMPTY);
- }
-
- private Prepared(MD5Digest statementId, int thriftStatementId, ResultSet.PreparedMetadata metadata, ResultSet.ResultMetadata resultMetadata)
+ public Prepared(MD5Digest statementId, MD5Digest resultMetadataId, ResultSet.PreparedMetadata metadata, ResultSet.ResultMetadata resultMetadata)
{
super(Kind.PREPARED);
this.statementId = statementId;
- this.thriftStatementId = thriftStatementId;
+ this.resultMetadataId = resultMetadataId;
this.metadata = metadata;
this.resultMetadata = resultMetadata;
}
- private static ResultSet.ResultMetadata extractResultMetadata(CQLStatement statement)
+ @VisibleForTesting
+ public Prepared withResultMetadata(ResultSet.ResultMetadata resultMetadata)
{
- if (!(statement instanceof SelectStatement))
- return ResultSet.ResultMetadata.EMPTY;
-
- return ((SelectStatement)statement).getResultMetadata();
- }
-
- public CqlResult toThriftResult()
- {
- throw new UnsupportedOperationException();
- }
-
- public CqlPreparedResult toThriftPreparedResult()
- {
- List<String> namesString = new ArrayList<String>(metadata.names.size());
- List<String> typesString = new ArrayList<String>(metadata.names.size());
- for (ColumnSpecification name : metadata.names)
- {
- namesString.add(name.toString());
- typesString.add(name.type.toString());
- }
- return new CqlPreparedResult(thriftStatementId, metadata.names.size()).setVariable_types(typesString).setVariable_names(namesString);
+ return new Prepared(statementId, resultMetadata.getResultMetadataId(), metadata, resultMetadata);
}
@Override
@@ -368,11 +322,6 @@
}
};
- public CqlResult toThriftResult()
- {
- return new CqlResult(CqlResultType.VOID);
- }
-
@Override
public String toString()
{
diff --git a/src/java/org/apache/cassandra/transport/messages/StartupMessage.java b/src/java/org/apache/cassandra/transport/messages/StartupMessage.java
index 8b4b0a4..ee2b34e 100644
--- a/src/java/org/apache/cassandra/transport/messages/StartupMessage.java
+++ b/src/java/org/apache/cassandra/transport/messages/StartupMessage.java
@@ -23,9 +23,16 @@
import io.netty.buffer.ByteBuf;
import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.service.ClientState;
import org.apache.cassandra.service.QueryState;
import org.apache.cassandra.transport.*;
+import org.apache.cassandra.transport.frame.checksum.ChecksummingTransformer;
+import org.apache.cassandra.transport.frame.compress.CompressingTransformer;
+import org.apache.cassandra.transport.frame.compress.Compressor;
+import org.apache.cassandra.transport.frame.compress.LZ4Compressor;
+import org.apache.cassandra.transport.frame.compress.SnappyCompressor;
import org.apache.cassandra.utils.CassandraVersion;
+import org.apache.cassandra.utils.ChecksumType;
/**
* The initial message of the protocol.
@@ -36,7 +43,9 @@
public static final String CQL_VERSION = "CQL_VERSION";
public static final String COMPRESSION = "COMPRESSION";
public static final String PROTOCOL_VERSIONS = "PROTOCOL_VERSIONS";
- public static final String NO_COMPACT = "NO_COMPACT";
+ public static final String DRIVER_NAME = "DRIVER_NAME";
+ public static final String DRIVER_VERSION = "DRIVER_VERSION";
+ public static final String CHECKSUM = "CONTENT_CHECKSUM";
public static final String THROW_ON_OVERLOAD = "THROW_ON_OVERLOAD";
public static final Message.Codec<StartupMessage> codec = new Message.Codec<StartupMessage>()
@@ -65,7 +74,8 @@
this.options = options;
}
- public Message.Response execute(QueryState state, long queryStartNanoTime)
+ @Override
+ protected Message.Response execute(QueryState state, long queryStartNanoTime, boolean traceRequest)
{
String cqlVersion = options.get(CQL_VERSION);
if (cqlVersion == null)
@@ -81,30 +91,30 @@
throw new ProtocolException(e.getMessage());
}
- if (options.containsKey(COMPRESSION))
+ ChecksumType checksumType = getChecksumType();
+ Compressor compressor = getCompressor();
+
+ if (null != checksumType)
{
- String compression = options.get(COMPRESSION).toLowerCase();
- if (compression.equals("snappy"))
- {
- if (FrameCompressor.SnappyCompressor.instance == null)
- throw new ProtocolException("This instance does not support Snappy compression");
- connection.setCompressor(FrameCompressor.SnappyCompressor.instance);
- }
- else if (compression.equals("lz4"))
- {
- connection.setCompressor(FrameCompressor.LZ4Compressor.instance);
- }
- else
- {
- throw new ProtocolException(String.format("Unknown compression algorithm: %s", compression));
- }
+ if (!connection.getVersion().supportsChecksums())
+ throw new ProtocolException(String.format("Invalid message flag. Protocol version %s does not support frame body checksums", connection.getVersion().toString()));
+ connection.setTransformer(ChecksummingTransformer.getTransformer(checksumType, compressor));
+ }
+ else if (null != compressor)
+ {
+ connection.setTransformer(CompressingTransformer.getTransformer(compressor));
}
- if (options.containsKey(NO_COMPACT) && Boolean.parseBoolean(options.get(NO_COMPACT)))
- state.getClientState().setNoCompactMode();
-
connection.setThrowOnOverload("1".equals(options.get(THROW_ON_OVERLOAD)));
+ ClientState clientState = state.getClientState();
+ String driverName = options.get(DRIVER_NAME);
+ if (null != driverName)
+ {
+ clientState.setDriverName(driverName);
+ clientState.setDriverVersion(options.get(DRIVER_VERSION));
+ }
+
if (DatabaseDescriptor.getAuthenticator().requireAuthentication())
return new AuthenticateMessage(DatabaseDescriptor.getAuthenticator().getClass().getName());
else
@@ -119,6 +129,42 @@
return newMap;
}
+ private ChecksumType getChecksumType() throws ProtocolException
+ {
+ String name = options.get(CHECKSUM);
+ try
+ {
+ return name != null ? ChecksumType.valueOf(name.toUpperCase()) : null;
+ }
+ catch (IllegalArgumentException e)
+ {
+ throw new ProtocolException(String.format("Requested checksum type %s is not known or supported by " +
+ "this version of Cassandra", name));
+ }
+ }
+
+ private Compressor getCompressor() throws ProtocolException
+ {
+ String name = options.get(COMPRESSION);
+ if (null == name)
+ return null;
+
+ switch (name.toLowerCase())
+ {
+ case "snappy":
+ {
+ if (SnappyCompressor.INSTANCE == null)
+ throw new ProtocolException("This instance does not support Snappy compression");
+
+ return SnappyCompressor.INSTANCE;
+ }
+ case "lz4":
+ return LZ4Compressor.INSTANCE;
+ default:
+ throw new ProtocolException(String.format("Unknown compression algorithm: %s", name));
+ }
+ }
+
@Override
public String toString()
{
diff --git a/src/java/org/apache/cassandra/transport/messages/UnsupportedMessageCodec.java b/src/java/org/apache/cassandra/transport/messages/UnsupportedMessageCodec.java
new file mode 100644
index 0000000..563e5d6
--- /dev/null
+++ b/src/java/org/apache/cassandra/transport/messages/UnsupportedMessageCodec.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.transport.messages;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import io.netty.buffer.ByteBuf;
+import org.apache.cassandra.transport.Message;
+import org.apache.cassandra.transport.ProtocolException;
+import org.apache.cassandra.transport.ProtocolVersion;
+
+/**
+ * Catch-all codec for any unsupported legacy messages.
+ */
+public class UnsupportedMessageCodec <T extends Message> implements Message.Codec<T>
+{
+ public final static UnsupportedMessageCodec instance = new UnsupportedMessageCodec();
+
+ private static final Logger logger = LoggerFactory.getLogger(UnsupportedMessageCodec.class);
+
+ public T decode(ByteBuf body, ProtocolVersion version)
+ {
+ if (ProtocolVersion.SUPPORTED.contains(version))
+ {
+ logger.error("Received invalid message for supported protocol version {}", version);
+ }
+ throw new ProtocolException("Unsupported message");
+ }
+
+ public void encode(T t, ByteBuf dest, ProtocolVersion version)
+ {
+ throw new ProtocolException("Unsupported message");
+ }
+
+ public int encodedSize(T t, ProtocolVersion version)
+ {
+ return 0;
+ }
+}
diff --git a/src/java/org/apache/cassandra/triggers/CustomClassLoader.java b/src/java/org/apache/cassandra/triggers/CustomClassLoader.java
index 32a987f..cb0918c 100644
--- a/src/java/org/apache/cassandra/triggers/CustomClassLoader.java
+++ b/src/java/org/apache/cassandra/triggers/CustomClassLoader.java
@@ -35,6 +35,8 @@
import com.google.common.io.Files;
+import org.apache.cassandra.io.util.FileUtils;
+
/**
* Custom class loader will load the classes from the class path, CCL will load
* the classes from the the URL first, if it cannot find the required class it
@@ -76,7 +78,7 @@
};
for (File inputJar : dir.listFiles(filter))
{
- File lib = new File(System.getProperty("java.io.tmpdir"), "lib");
+ File lib = new File(FileUtils.getTempDir(), "lib");
if (!lib.exists())
{
lib.mkdir();
@@ -84,7 +86,7 @@
}
try
{
- File out = File.createTempFile("cassandra-", ".jar", lib);
+ File out = FileUtils.createTempFile("cassandra-", ".jar", lib);
out.deleteOnExit();
logger.info("Loading new jar {}", inputJar.getAbsolutePath());
Files.copy(inputJar, out);
diff --git a/src/java/org/apache/cassandra/triggers/TriggerExecutor.java b/src/java/org/apache/cassandra/triggers/TriggerExecutor.java
index 0354fde..295003f 100644
--- a/src/java/org/apache/cassandra/triggers/TriggerExecutor.java
+++ b/src/java/org/apache/cassandra/triggers/TriggerExecutor.java
@@ -33,6 +33,7 @@
import org.apache.cassandra.db.partitions.PartitionUpdate;
import org.apache.cassandra.exceptions.CassandraException;
import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.schema.TriggerMetadata;
import org.apache.cassandra.schema.Triggers;
import org.apache.cassandra.utils.FBUtilities;
@@ -86,7 +87,7 @@
if (intermediate == null || intermediate.isEmpty())
return updates;
- List<PartitionUpdate> augmented = validateForSinglePartition(updates.metadata().cfId,
+ List<PartitionUpdate> augmented = validateForSinglePartition(updates.metadata().id,
updates.partitionKey(),
intermediate);
// concatenate augmented and origin
@@ -162,9 +163,9 @@
return merged;
}
- private List<PartitionUpdate> validateForSinglePartition(UUID cfId,
- DecoratedKey key,
- Collection<Mutation> tmutations)
+ private List<PartitionUpdate> validateForSinglePartition(TableId tableId,
+ DecoratedKey key,
+ Collection<Mutation> tmutations)
throws InvalidRequestException
{
validate(tmutations);
@@ -174,7 +175,7 @@
List<PartitionUpdate> updates = Lists.newArrayList(Iterables.getOnlyElement(tmutations).getPartitionUpdates());
if (updates.size() > 1)
throw new InvalidRequestException("The updates generated by triggers are not all for the same partition");
- validateSamePartition(cfId, key, Iterables.getOnlyElement(updates));
+ validateSamePartition(tableId, key, Iterables.getOnlyElement(updates));
return updates;
}
@@ -183,20 +184,20 @@
{
for (PartitionUpdate update : mutation.getPartitionUpdates())
{
- validateSamePartition(cfId, key, update);
+ validateSamePartition(tableId, key, update);
updates.add(update);
}
}
return updates;
}
- private void validateSamePartition(UUID cfId, DecoratedKey key, PartitionUpdate update)
+ private void validateSamePartition(TableId tableId, DecoratedKey key, PartitionUpdate update)
throws InvalidRequestException
{
if (!key.equals(update.partitionKey()))
throw new InvalidRequestException("Partition key of additional mutation does not match primary update key");
- if (!cfId.equals(update.metadata().cfId))
+ if (!tableId.equals(update.metadata().id))
throw new InvalidRequestException("table of additional mutation does not match primary update table");
}
@@ -216,7 +217,7 @@
*/
private List<Mutation> executeInternal(PartitionUpdate update)
{
- Triggers triggers = update.metadata().getTriggers();
+ Triggers triggers = update.metadata().triggers;
if (triggers.isEmpty())
return null;
List<Mutation> tmutations = Lists.newLinkedList();
@@ -243,7 +244,7 @@
}
catch (Exception ex)
{
- throw new RuntimeException(String.format("Exception while executing trigger on table with ID: %s", update.metadata().cfId), ex);
+ throw new RuntimeException(String.format("Exception while executing trigger on table with ID: %s", update.metadata().id), ex);
}
finally
{
@@ -251,11 +252,11 @@
}
}
- public synchronized ITrigger loadTriggerInstance(String triggerName) throws Exception
+ public synchronized ITrigger loadTriggerInstance(String triggerClass) throws Exception
{
// double check.
- if (cachedTriggers.get(triggerName) != null)
- return cachedTriggers.get(triggerName);
- return (ITrigger) customClassLoader.loadClass(triggerName).getConstructor().newInstance();
+ if (cachedTriggers.get(triggerClass) != null)
+ return cachedTriggers.get(triggerClass);
+ return (ITrigger) customClassLoader.loadClass(triggerClass).getConstructor().newInstance();
}
}
diff --git a/src/java/org/apache/cassandra/utils/AbstractIterator.java b/src/java/org/apache/cassandra/utils/AbstractIterator.java
index dd3d73c..7dd32b8 100644
--- a/src/java/org/apache/cassandra/utils/AbstractIterator.java
+++ b/src/java/org/apache/cassandra/utils/AbstractIterator.java
@@ -23,7 +23,7 @@
import com.google.common.collect.PeekingIterator;
-public abstract class AbstractIterator<V> implements Iterator<V>, PeekingIterator<V>
+public abstract class AbstractIterator<V> implements Iterator<V>, PeekingIterator<V>, CloseableIterator<V>
{
private static enum State { MUST_FETCH, HAS_NEXT, DONE, FAILED }
@@ -80,4 +80,9 @@
{
throw new UnsupportedOperationException();
}
+
+ public void close()
+ {
+ //no-op
+ }
}
diff --git a/src/java/org/apache/cassandra/utils/ApproximateTime.java b/src/java/org/apache/cassandra/utils/ApproximateTime.java
new file mode 100644
index 0000000..32b6e44
--- /dev/null
+++ b/src/java/org/apache/cassandra/utils/ApproximateTime.java
@@ -0,0 +1,192 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.utils;
+
+import java.util.concurrent.CancellationException;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.concurrent.ScheduledExecutors;
+import org.apache.cassandra.config.Config;
+
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+import static org.apache.cassandra.utils.ApproximateTime.Measurement.ALMOST_NOW;
+import static org.apache.cassandra.utils.ApproximateTime.Measurement.ALMOST_SAME_TIME;
+
+/**
+ * This class provides approximate time utilities:
+ * - An imprecise nanoTime (monotonic) and currentTimeMillis (non-monotonic), that are faster than their regular counterparts
+ * They have a configured approximate precision (default of 10ms), which is the cadence they will be updated if the system is healthy
+ * - A mechanism for converting between nanoTime and currentTimeMillis measurements.
+ * These conversions may have drifted, and they offer no absolute guarantees on precision
+ */
+public class ApproximateTime
+{
+ private static final Logger logger = LoggerFactory.getLogger(ApproximateTime.class);
+ private static final int ALMOST_NOW_UPDATE_INTERVAL_MS = Math.max(1, Integer.parseInt(System.getProperty(Config.PROPERTY_PREFIX + "approximate_time_precision_ms", "2")));
+ private static final String CONVERSION_UPDATE_INTERVAL_PROPERTY = Config.PROPERTY_PREFIX + "NANOTIMETOMILLIS_TIMESTAMP_UPDATE_INTERVAL";
+ private static final long ALMOST_SAME_TIME_UPDATE_INTERVAL_MS = Long.getLong(CONVERSION_UPDATE_INTERVAL_PROPERTY, 10000);
+
+ public static class AlmostSameTime
+ {
+ final long millis;
+ final long nanos;
+ final long error; // maximum error of millis measurement (in nanos)
+
+ private AlmostSameTime(long millis, long nanos, long error)
+ {
+ this.millis = millis;
+ this.nanos = nanos;
+ this.error = error;
+ }
+
+ public long toCurrentTimeMillis(long nanoTime)
+ {
+ return millis + TimeUnit.NANOSECONDS.toMillis(nanoTime - nanos);
+ }
+
+ public long toNanoTime(long currentTimeMillis)
+ {
+ return nanos + MILLISECONDS.toNanos(currentTimeMillis - millis);
+ }
+ }
+
+ public enum Measurement { ALMOST_NOW, ALMOST_SAME_TIME }
+
+ private static volatile Future<?> almostNowUpdater;
+ private static volatile Future<?> almostSameTimeUpdater;
+
+ private static volatile long almostNowMillis;
+ private static volatile long almostNowNanos;
+
+ private static volatile AlmostSameTime almostSameTime = new AlmostSameTime(0L, 0L, Long.MAX_VALUE);
+ private static double failedAlmostSameTimeUpdateModifier = 1.0;
+
+ private static final Runnable refreshAlmostNow = () -> {
+ almostNowMillis = System.currentTimeMillis();
+ almostNowNanos = System.nanoTime();
+ };
+
+ private static final Runnable refreshAlmostSameTime = () -> {
+ final int tries = 3;
+ long[] samples = new long[2 * tries + 1];
+ samples[0] = System.nanoTime();
+ for (int i = 1 ; i < samples.length ; i += 2)
+ {
+ samples[i] = System.currentTimeMillis();
+ samples[i + 1] = System.nanoTime();
+ }
+
+ int best = 1;
+ // take sample with minimum delta between calls
+ for (int i = 3 ; i < samples.length - 1 ; i += 2)
+ {
+ if ((samples[i+1] - samples[i-1]) < (samples[best+1]-samples[best-1]))
+ best = i;
+ }
+
+ long millis = samples[best];
+ long nanos = (samples[best+1] / 2) + (samples[best-1] / 2);
+ long error = (samples[best+1] / 2) - (samples[best-1] / 2);
+
+ AlmostSameTime prev = almostSameTime;
+ AlmostSameTime next = new AlmostSameTime(millis, nanos, error);
+
+ if (next.error > prev.error && next.error > prev.error * failedAlmostSameTimeUpdateModifier)
+ {
+ failedAlmostSameTimeUpdateModifier *= 1.1;
+ return;
+ }
+
+ failedAlmostSameTimeUpdateModifier = 1.0;
+ almostSameTime = next;
+ };
+
+ static
+ {
+ start(ALMOST_NOW);
+ start(ALMOST_SAME_TIME);
+ }
+
+ public static synchronized void stop(Measurement measurement)
+ {
+ switch (measurement)
+ {
+ case ALMOST_NOW:
+ almostNowUpdater.cancel(true);
+ try { almostNowUpdater.get(); } catch (Throwable t) { }
+ almostNowUpdater = null;
+ break;
+ case ALMOST_SAME_TIME:
+ almostSameTimeUpdater.cancel(true);
+ try { almostSameTimeUpdater.get(); } catch (Throwable t) { }
+ almostSameTimeUpdater = null;
+ break;
+ }
+ }
+
+ public static synchronized void start(Measurement measurement)
+ {
+ switch (measurement)
+ {
+ case ALMOST_NOW:
+ if (almostNowUpdater != null)
+ throw new IllegalStateException("Already running");
+ refreshAlmostNow.run();
+ logger.info("Scheduling approximate time-check task with a precision of {} milliseconds", ALMOST_NOW_UPDATE_INTERVAL_MS);
+ almostNowUpdater = ScheduledExecutors.scheduledFastTasks.scheduleWithFixedDelay(refreshAlmostNow, ALMOST_NOW_UPDATE_INTERVAL_MS, ALMOST_NOW_UPDATE_INTERVAL_MS, MILLISECONDS);
+ break;
+ case ALMOST_SAME_TIME:
+ if (almostSameTimeUpdater != null)
+ throw new IllegalStateException("Already running");
+ refreshAlmostSameTime.run();
+ logger.info("Scheduling approximate time conversion task with an interval of {} milliseconds", ALMOST_SAME_TIME_UPDATE_INTERVAL_MS);
+ almostSameTimeUpdater = ScheduledExecutors.scheduledFastTasks.scheduleWithFixedDelay(refreshAlmostSameTime, ALMOST_SAME_TIME_UPDATE_INTERVAL_MS, ALMOST_SAME_TIME_UPDATE_INTERVAL_MS, MILLISECONDS);
+ break;
+ }
+ }
+
+
+ /**
+ * Request an immediate refresh; this shouldn't generally be invoked, except perhaps by tests
+ */
+ @VisibleForTesting
+ public static synchronized void refresh(Measurement measurement)
+ {
+ stop(measurement);
+ start(measurement);
+ }
+
+ /** no guarantees about relationship to nanoTime; non-monotonic (tracks currentTimeMillis as closely as possible) */
+ public static long currentTimeMillis()
+ {
+ return almostNowMillis;
+ }
+
+ /** no guarantees about relationship to currentTimeMillis; monotonic */
+ public static long nanoTime()
+ {
+ return almostNowNanos;
+ }
+}
diff --git a/src/java/org/apache/cassandra/utils/Architecture.java b/src/java/org/apache/cassandra/utils/Architecture.java
index aa00ca1..3e9f579 100644
--- a/src/java/org/apache/cassandra/utils/Architecture.java
+++ b/src/java/org/apache/cassandra/utils/Architecture.java
@@ -26,15 +26,16 @@
public final class Architecture
{
- // Note that s390x & aarch64 architecture are not officially supported and adding it here is only done out of convenience
- // for those that want to run C* on this architecture at their own risk (see #11214 & #13326)
+ // Note that s390x, aarch64, & ppc64le architectures are not officially supported and adding them here is only done out
+ // of convenience for those that want to run C* on these architectures at their own risk (see #11214, #13326, & #13615)
private static final Set<String> UNALIGNED_ARCH = Collections.unmodifiableSet(Sets.newHashSet(
- "i386",
- "x86",
- "amd64",
- "x86_64",
- "s390x",
- "aarch64"
+ "i386",
+ "x86",
+ "amd64",
+ "x86_64",
+ "s390x",
+ "aarch64",
+ "ppc64le"
));
public static final boolean IS_UNALIGNED = UNALIGNED_ARCH.contains(System.getProperty("os.arch"));
diff --git a/src/java/org/apache/cassandra/utils/BatchRemoveIterator.java b/src/java/org/apache/cassandra/utils/BatchRemoveIterator.java
deleted file mode 100644
index 4377426..0000000
--- a/src/java/org/apache/cassandra/utils/BatchRemoveIterator.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.utils;
-
-import java.util.Iterator;
-
-/**
- * Iterator that allows us to more efficiently remove many items
- */
-public interface BatchRemoveIterator<T> extends Iterator<T>
-{
- /**
- * Commits the remove operations in this batch iterator. After this no more
- * deletes can be made. Any further calls to remove() or commit() will throw IllegalStateException.
- */
- void commit();
-}
diff --git a/src/java/org/apache/cassandra/utils/BiLongAccumulator.java b/src/java/org/apache/cassandra/utils/BiLongAccumulator.java
new file mode 100644
index 0000000..2c3d6b5
--- /dev/null
+++ b/src/java/org/apache/cassandra/utils/BiLongAccumulator.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.utils;
+
+public interface BiLongAccumulator<T, A>
+{
+ long apply(T obj, A arguemnt, long v);
+}
diff --git a/src/java/org/apache/cassandra/utils/BloomFilter.java b/src/java/org/apache/cassandra/utils/BloomFilter.java
index 4ff07b7..bf48d43 100644
--- a/src/java/org/apache/cassandra/utils/BloomFilter.java
+++ b/src/java/org/apache/cassandra/utils/BloomFilter.java
@@ -37,18 +37,12 @@
public final IBitSet bitset;
public final int hashCount;
- /**
- * CASSANDRA-8413: 3.0 (inverted) bloom filters have no 'static' bits caused by using the same upper bits
- * for both bloom filter and token distribution.
- */
- public final boolean oldBfHashOrder;
- BloomFilter(int hashCount, IBitSet bitset, boolean oldBfHashOrder)
+ BloomFilter(int hashCount, IBitSet bitset)
{
super(bitset);
this.hashCount = hashCount;
this.bitset = bitset;
- this.oldBfHashOrder = oldBfHashOrder;
}
private BloomFilter(BloomFilter copy)
@@ -56,7 +50,6 @@
super(copy);
this.hashCount = copy.hashCount;
this.bitset = copy.bitset;
- this.oldBfHashOrder = copy.oldBfHashOrder;
}
public long serializedSize()
@@ -66,7 +59,7 @@
// Murmur is faster than an SHA-based approach and provides as-good collision
// resistance. The combinatorial generation approach described in
- // http://www.eecs.harvard.edu/~kirsch/pubs/bbbf/esa06.pdf
+ // https://www.eecs.harvard.edu/~michaelm/postscripts/tr-02-05.pdf
// does prove to work in actual tests, and is obviously faster
// than performing further iterations of murmur.
@@ -101,13 +94,6 @@
@Inline
private void setIndexes(long base, long inc, int count, long max, long[] results)
{
- if (oldBfHashOrder)
- {
- long x = inc;
- inc = base;
- base = x;
- }
-
for (int i = 0; i < count; i++)
{
results[i] = FBUtilities.abs(base % max);
@@ -155,7 +141,7 @@
public String toString()
{
- return "BloomFilter[hashCount=" + hashCount + ";oldBfHashOrder=" + oldBfHashOrder + ";capacity=" + bitset.capacity() + ']';
+ return "BloomFilter[hashCount=" + hashCount + ";capacity=" + bitset.capacity() + ']';
}
public void addTo(Ref.IdentityCollection identities)
diff --git a/src/java/org/apache/cassandra/utils/BloomFilterSerializer.java b/src/java/org/apache/cassandra/utils/BloomFilterSerializer.java
index 6f57fc8..d3c08b5 100644
--- a/src/java/org/apache/cassandra/utils/BloomFilterSerializer.java
+++ b/src/java/org/apache/cassandra/utils/BloomFilterSerializer.java
@@ -17,16 +17,15 @@
*/
package org.apache.cassandra.utils;
-import java.io.DataInput;
+import java.io.DataInputStream;
import java.io.IOException;
import org.apache.cassandra.db.TypeSizes;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.utils.obs.IBitSet;
import org.apache.cassandra.utils.obs.OffHeapBitSet;
-import org.apache.cassandra.utils.obs.OpenBitSet;
-final class BloomFilterSerializer
+public final class BloomFilterSerializer
{
private BloomFilterSerializer()
{
@@ -38,18 +37,13 @@
bf.bitset.serialize(out);
}
- public static BloomFilter deserialize(DataInput in, boolean oldBfHashOrder) throws IOException
- {
- return deserialize(in, false, oldBfHashOrder);
- }
-
@SuppressWarnings("resource")
- public static BloomFilter deserialize(DataInput in, boolean offheap, boolean oldBfHashOrder) throws IOException
+ public static BloomFilter deserialize(DataInputStream in, boolean oldBfFormat) throws IOException
{
int hashes = in.readInt();
- IBitSet bs = offheap ? OffHeapBitSet.deserialize(in) : OpenBitSet.deserialize(in);
+ IBitSet bs = OffHeapBitSet.deserialize(in, oldBfFormat);
- return new BloomFilter(hashes, bs, oldBfHashOrder);
+ return new BloomFilter(hashes, bs);
}
/**
diff --git a/src/java/org/apache/cassandra/utils/ByteBufferUtil.java b/src/java/org/apache/cassandra/utils/ByteBufferUtil.java
index 41d5247..ff3fb3d 100644
--- a/src/java/org/apache/cassandra/utils/ByteBufferUtil.java
+++ b/src/java/org/apache/cassandra/utils/ByteBufferUtil.java
@@ -101,6 +101,16 @@
return FastByteOperations.compareUnsigned(o1, o2, 0, o2.length);
}
+ public static int compare(ByteBuffer o1, int s1, int l1, byte[] o2)
+ {
+ return FastByteOperations.compareUnsigned(o1, s1, l1, o2, 0, o2.length);
+ }
+
+ public static int compare(byte[] o1, ByteBuffer o2, int s2, int l2)
+ {
+ return FastByteOperations.compareUnsigned(o1, 0, o1.length, o2, s2, l2);
+ }
+
/**
* Decode a String representation.
* This method assumes that the encoding charset is UTF_8.
@@ -161,16 +171,25 @@
*/
public static byte[] getArray(ByteBuffer buffer)
{
- int length = buffer.remaining();
+ return getArray(buffer, buffer.position(), buffer.remaining());
+ }
+
+ /**
+ * You should almost never use this. Instead, use the write* methods to avoid copies.
+ */
+ public static byte[] getArray(ByteBuffer buffer, int position, int length)
+ {
if (buffer.hasArray())
{
- int boff = buffer.arrayOffset() + buffer.position();
+ int boff = buffer.arrayOffset() + position;
return Arrays.copyOfRange(buffer.array(), boff, boff + length);
}
+
// else, DirectByteBuffer.get() is the fastest route
byte[] bytes = new byte[length];
- buffer.duplicate().get(bytes);
-
+ ByteBuffer dup = buffer.duplicate();
+ dup.position(position).limit(position + length);
+ dup.get(bytes);
return bytes;
}
@@ -255,14 +274,14 @@
return clone;
}
- public static void arrayCopy(ByteBuffer src, int srcPos, byte[] dst, int dstPos, int length)
+ public static void copyBytes(ByteBuffer src, int srcPos, byte[] dst, int dstPos, int length)
{
FastByteOperations.copy(src, srcPos, dst, dstPos, length);
}
/**
* Transfer bytes from one ByteBuffer to another.
- * This function acts as System.arrayCopy() but for ByteBuffers.
+ * This function acts as System.arrayCopy() but for ByteBuffers, and operates safely on direct memory.
*
* @param src the source ByteBuffer
* @param srcPos starting position in the source ByteBuffer
@@ -270,7 +289,7 @@
* @param dstPos starting position in the destination ByteBuffer
* @param length the number of bytes to copy
*/
- public static void arrayCopy(ByteBuffer src, int srcPos, ByteBuffer dst, int dstPos, int length)
+ public static void copyBytes(ByteBuffer src, int srcPos, ByteBuffer dst, int dstPos, int length)
{
FastByteOperations.copy(src, srcPos, dst, dstPos, length);
}
@@ -278,12 +297,37 @@
public static int put(ByteBuffer src, ByteBuffer trg)
{
int length = Math.min(src.remaining(), trg.remaining());
- arrayCopy(src, src.position(), trg, trg.position(), length);
+ copyBytes(src, src.position(), trg, trg.position(), length);
trg.position(trg.position() + length);
src.position(src.position() + length);
return length;
}
+ public static void writeZeroes(ByteBuffer dest, int count)
+ {
+ if (count >= 8)
+ {
+ // align
+ while ((dest.position() & 0x7) != 0)
+ {
+ dest.put((byte) 0);
+ --count;
+ }
+ }
+ // write aligned longs
+ while (count >= 8)
+ {
+ dest.putLong(0L);
+ count -= 8;
+ }
+ // finish up
+ while (count > 0)
+ {
+ dest.put((byte) 0);
+ --count;
+ }
+ }
+
public static void writeWithLength(ByteBuffer bytes, DataOutputPlus out) throws IOException
{
out.writeInt(bytes.remaining());
@@ -411,6 +455,15 @@
return bytes;
}
+ public static byte[] readBytesWithLength(DataInput in) throws IOException
+ {
+ int length = in.readInt();
+ if (length < 0)
+ throw new IOException("Corrupt (negative) value length encountered");
+
+ return readBytes(in, length);
+ }
+
/**
* Convert a byte buffer to an integer.
* Does not change the byte buffer position.
@@ -435,6 +488,18 @@
return bytes.getShort(bytes.position());
}
+ /**
+ * Convert a byte buffer to a short.
+ * Does not change the byte buffer position.
+ *
+ * @param bytes byte buffer to convert to byte
+ * @return byte representation of the byte buffer
+ */
+ public static byte toByte(ByteBuffer bytes)
+ {
+ return bytes.get(bytes.position());
+ }
+
public static long toLong(ByteBuffer bytes)
{
return bytes.getLong(bytes.position());
@@ -585,6 +650,7 @@
assert bytes1.limit() >= offset1 + length : "The first byte array isn't long enough for the specified offset and length.";
assert bytes2.limit() >= offset2 + length : "The second byte array isn't long enough for the specified offset and length.";
+
for (int i = 0; i < length; i++)
{
byte byte1 = bytes1.get(offset1 + i);
diff --git a/src/java/org/apache/cassandra/utils/CassandraVersion.java b/src/java/org/apache/cassandra/utils/CassandraVersion.java
index bf9fe6a..aed0fe7 100644
--- a/src/java/org/apache/cassandra/utils/CassandraVersion.java
+++ b/src/java/org/apache/cassandra/utils/CassandraVersion.java
@@ -118,11 +118,6 @@
return compareIdentifiers(build, other.build, -1);
}
- public boolean is30()
- {
- return major == 3 && minor == 0;
- }
-
/**
* Returns a version that is backward compatible with this version amongst a list
* of provided version, or null if none can be found.
diff --git a/src/java/org/apache/cassandra/utils/ChecksumType.java b/src/java/org/apache/cassandra/utils/ChecksumType.java
index 413a171..fa920aa 100644
--- a/src/java/org/apache/cassandra/utils/ChecksumType.java
+++ b/src/java/org/apache/cassandra/utils/ChecksumType.java
@@ -26,7 +26,7 @@
public enum ChecksumType
{
- Adler32
+ ADLER32
{
@Override
diff --git a/src/java/org/apache/cassandra/utils/Clock.java b/src/java/org/apache/cassandra/utils/Clock.java
deleted file mode 100644
index eb9822c..0000000
--- a/src/java/org/apache/cassandra/utils/Clock.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.utils;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Wrapper around time related functions that are either implemented by using the default JVM calls
- * or by using a custom implementation for testing purposes.
- *
- * See {@link #instance} for how to use a custom implementation.
- *
- * Please note that {@link java.time.Clock} wasn't used, as it would not be possible to provide an
- * implementation for {@link #nanoTime()} with the exact same properties of {@link System#nanoTime()}.
- */
-public class Clock
-{
- private static final Logger logger = LoggerFactory.getLogger(Clock.class);
-
- /**
- * Static singleton object that will be instanciated by default with a system clock
- * implementation. Set <code>cassandra.clock</code> system property to a FQCN to use a
- * different implementation instead.
- */
- public static Clock instance;
-
- static
- {
- String sclock = System.getProperty("cassandra.clock");
- if (sclock == null)
- {
- instance = new Clock();
- }
- else
- {
- try
- {
- logger.debug("Using custom clock implementation: {}", sclock);
- instance = (Clock) Class.forName(sclock).newInstance();
- }
- catch (Exception e)
- {
- logger.error(e.getMessage(), e);
- }
- }
- }
-
- /**
- * @see System#nanoTime()
- */
- public long nanoTime()
- {
- return System.nanoTime();
- }
-
- /**
- * @see System#currentTimeMillis()
- */
- public long currentTimeMillis()
- {
- return System.currentTimeMillis();
- }
-
-}
diff --git a/src/java/org/apache/cassandra/utils/CoalescingStrategies.java b/src/java/org/apache/cassandra/utils/CoalescingStrategies.java
deleted file mode 100644
index 9f3b118..0000000
--- a/src/java/org/apache/cassandra/utils/CoalescingStrategies.java
+++ /dev/null
@@ -1,572 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.utils;
-
-import org.apache.cassandra.concurrent.NamedThreadFactory;
-import org.apache.cassandra.config.Config;
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.io.util.FileUtils;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.RandomAccessFile;
-import java.lang.reflect.Constructor;
-import java.nio.ByteBuffer;
-import java.nio.channels.FileChannel.MapMode;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.List;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.locks.LockSupport;
-import java.util.Locale;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-
-public class CoalescingStrategies
-{
- static protected final Logger logger = LoggerFactory.getLogger(CoalescingStrategies.class);
-
- /*
- * Log debug information at info level about what the average is and when coalescing is enabled/disabled
- */
- private static final String DEBUG_COALESCING_PROPERTY = Config.PROPERTY_PREFIX + "coalescing_debug";
- private static final boolean DEBUG_COALESCING = Boolean.getBoolean(DEBUG_COALESCING_PROPERTY);
-
- private static final String DEBUG_COALESCING_PATH_PROPERTY = Config.PROPERTY_PREFIX + "coalescing_debug_path";
- private static final String DEBUG_COALESCING_PATH = System.getProperty(DEBUG_COALESCING_PATH_PROPERTY, "/tmp/coleascing_debug");
-
- static
- {
- if (DEBUG_COALESCING)
- {
- File directory = new File(DEBUG_COALESCING_PATH);
-
- if (directory.exists())
- FileUtils.deleteRecursive(directory);
-
- if (!directory.mkdirs())
- throw new ExceptionInInitializerError("Couldn't create log dir");
- }
- }
-
- @VisibleForTesting
- interface Clock
- {
- long nanoTime();
- }
-
- @VisibleForTesting
- static Clock CLOCK = new Clock()
- {
- public long nanoTime()
- {
- return System.nanoTime();
- }
- };
-
- public static interface Coalescable
- {
- long timestampNanos();
- }
-
- @VisibleForTesting
- static void parkLoop(long nanos)
- {
- long now = System.nanoTime();
- final long timer = now + nanos;
- // We shouldn't loop if it's within a few % of the target sleep time if on a second iteration.
- // See CASSANDRA-8692.
- final long limit = timer - nanos / 16;
- do
- {
- LockSupport.parkNanos(timer - now);
- now = System.nanoTime();
- }
- while (now < limit);
- }
-
- private static boolean maybeSleep(int messages, long averageGap, long maxCoalesceWindow, Parker parker)
- {
- // Do not sleep if there are still items in the backlog (CASSANDRA-13090).
- if (messages >= DatabaseDescriptor.getOtcCoalescingEnoughCoalescedMessages())
- return false;
-
- // only sleep if we can expect to double the number of messages we're sending in the time interval
- long sleep = messages * averageGap;
- if (sleep <= 0 || sleep > maxCoalesceWindow)
- return false;
-
- // assume we receive as many messages as we expect; apply the same logic to the future batch:
- // expect twice as many messages to consider sleeping for "another" interval; this basically translates
- // to doubling our sleep period until we exceed our max sleep window
- while (sleep * 2 < maxCoalesceWindow)
- sleep *= 2;
- parker.park(sleep);
- return true;
- }
-
- public static abstract class CoalescingStrategy
- {
- protected final Parker parker;
- protected final Logger logger;
- protected volatile boolean shouldLogAverage = false;
- protected final ByteBuffer logBuffer;
- private RandomAccessFile ras;
- private final String displayName;
-
- protected CoalescingStrategy(Parker parker, Logger logger, String displayName)
- {
- this.parker = parker;
- this.logger = logger;
- this.displayName = displayName;
- if (DEBUG_COALESCING)
- {
- NamedThreadFactory.createThread(() ->
- {
- while (true)
- {
- try
- {
- Thread.sleep(5000);
- }
- catch (InterruptedException e)
- {
- throw new AssertionError();
- }
- shouldLogAverage = true;
- }
- }, displayName + " debug thread").start();
- }
- RandomAccessFile rasTemp = null;
- ByteBuffer logBufferTemp = null;
- if (DEBUG_COALESCING)
- {
- try
- {
- File outFile = File.createTempFile("coalescing_" + this.displayName + "_", ".log", new File(DEBUG_COALESCING_PATH));
- rasTemp = new RandomAccessFile(outFile, "rw");
- logBufferTemp = ras.getChannel().map(MapMode.READ_WRITE, 0, Integer.MAX_VALUE);
- logBufferTemp.putLong(0);
- }
- catch (Exception e)
- {
- logger.error("Unable to create output file for debugging coalescing", e);
- }
- }
- ras = rasTemp;
- logBuffer = logBufferTemp;
- }
-
- /*
- * If debugging is enabled log to the logger the current average gap calculation result.
- */
- final protected void debugGap(long averageGap)
- {
- if (DEBUG_COALESCING && shouldLogAverage)
- {
- shouldLogAverage = false;
- logger.info("{} gap {}μs", this, TimeUnit.NANOSECONDS.toMicros(averageGap));
- }
- }
-
- /*
- * If debugging is enabled log the provided nanotime timestamp to a file.
- */
- final protected void debugTimestamp(long timestamp)
- {
- if(DEBUG_COALESCING && logBuffer != null)
- {
- logBuffer.putLong(0, logBuffer.getLong(0) + 1);
- logBuffer.putLong(timestamp);
- }
- }
-
- /*
- * If debugging is enabled log the timestamps of all the items in the provided collection
- * to a file.
- */
- final protected <C extends Coalescable> void debugTimestamps(Collection<C> coalescables)
- {
- if (DEBUG_COALESCING)
- {
- for (C coalescable : coalescables)
- {
- debugTimestamp(coalescable.timestampNanos());
- }
- }
- }
-
- /**
- * Drain from the input blocking queue to the output list up to maxItems elements.
- *
- * The coalescing strategy may choose to park the current thread if it thinks it will
- * be able to produce an output list with more elements.
- *
- * @param input Blocking queue to retrieve elements from
- * @param out Output list to place retrieved elements in. Must be empty.
- * @param maxItems Maximum number of elements to place in the output list
- */
- public <C extends Coalescable> void coalesce(BlockingQueue<C> input, List<C> out, int maxItems) throws InterruptedException
- {
- Preconditions.checkArgument(out.isEmpty(), "out list should be empty");
- coalesceInternal(input, out, maxItems);
- }
-
- protected abstract <C extends Coalescable> void coalesceInternal(BlockingQueue<C> input, List<C> out, int maxItems) throws InterruptedException;
-
- }
-
- @VisibleForTesting
- interface Parker
- {
- void park(long nanos);
- }
-
- private static final Parker PARKER = new Parker()
- {
- @Override
- public void park(long nanos)
- {
- parkLoop(nanos);
- }
- };
-
- @VisibleForTesting
- static class TimeHorizonMovingAverageCoalescingStrategy extends CoalescingStrategy
- {
- // for now we'll just use 64ms per bucket; this can be made configurable, but results in ~1s for 16 samples
- private static final int INDEX_SHIFT = 26;
- private static final long BUCKET_INTERVAL = 1L << 26;
- private static final int BUCKET_COUNT = 16;
- private static final long INTERVAL = BUCKET_INTERVAL * BUCKET_COUNT;
- private static final long MEASURED_INTERVAL = BUCKET_INTERVAL * (BUCKET_COUNT - 1);
-
- // the minimum timestamp we will now accept updates for; only moves forwards, never backwards
- private long epoch = CLOCK.nanoTime();
- // the buckets, each following on from epoch; the measurements run from ix(epoch) to ix(epoch - 1)
- // ix(epoch-1) is a partial result, that is never actually part of the calculation, and most updates
- // are expected to hit this bucket
- private final int samples[] = new int[BUCKET_COUNT];
- private long sum = 0;
- private final long maxCoalesceWindow;
-
- public TimeHorizonMovingAverageCoalescingStrategy(int maxCoalesceWindow, Parker parker, Logger logger, String displayName)
- {
- super(parker, logger, displayName);
- this.maxCoalesceWindow = TimeUnit.MICROSECONDS.toNanos(maxCoalesceWindow);
- sum = 0;
- }
-
- private void logSample(long nanos)
- {
- debugTimestamp(nanos);
- long epoch = this.epoch;
- long delta = nanos - epoch;
- if (delta < 0)
- // have to simply ignore, but would be a bit crazy to get such reordering
- return;
-
- if (delta > INTERVAL)
- epoch = rollepoch(delta, epoch, nanos);
-
- int ix = ix(nanos);
- samples[ix]++;
-
- // if we've updated an old bucket, we need to update the sum to match
- if (ix != ix(epoch - 1))
- sum++;
- }
-
- private long averageGap()
- {
- if (sum == 0)
- return Integer.MAX_VALUE;
- return MEASURED_INTERVAL / sum;
- }
-
- // this sample extends past the end of the range we cover, so rollover
- private long rollepoch(long delta, long epoch, long nanos)
- {
- if (delta > 2 * INTERVAL)
- {
- // this sample is more than twice our interval ahead, so just clear our counters completely
- epoch = epoch(nanos);
- sum = 0;
- Arrays.fill(samples, 0);
- }
- else
- {
- // ix(epoch - 1) => last index; this is our partial result bucket, so we add this to the sum
- sum += samples[ix(epoch - 1)];
- // then we roll forwards, clearing buckets, until our interval covers the new sample time
- while (epoch + INTERVAL < nanos)
- {
- int index = ix(epoch);
- sum -= samples[index];
- samples[index] = 0;
- epoch += BUCKET_INTERVAL;
- }
- }
- // store the new epoch
- this.epoch = epoch;
- return epoch;
- }
-
- private long epoch(long latestNanos)
- {
- return (latestNanos - MEASURED_INTERVAL) & ~(BUCKET_INTERVAL - 1);
- }
-
- private int ix(long nanos)
- {
- return (int) ((nanos >>> INDEX_SHIFT) & 15);
- }
-
- @Override
- protected <C extends Coalescable> void coalesceInternal(BlockingQueue<C> input, List<C> out, int maxItems) throws InterruptedException
- {
- if (input.drainTo(out, maxItems) == 0)
- {
- out.add(input.take());
- input.drainTo(out, maxItems - out.size());
- }
-
- for (Coalescable qm : out)
- logSample(qm.timestampNanos());
-
- long averageGap = averageGap();
- debugGap(averageGap);
-
- int count = out.size();
- if (maybeSleep(count, averageGap, maxCoalesceWindow, parker))
- {
- input.drainTo(out, maxItems - out.size());
- int prevCount = count;
- count = out.size();
- for (int i = prevCount; i < count; i++)
- logSample(out.get(i).timestampNanos());
- }
- }
-
- @Override
- public String toString()
- {
- return "Time horizon moving average";
- }
- }
-
- /*
- * Start coalescing by sleeping if the moving average is < the requested window.
- * The actual time spent waiting to coalesce will be the min( window, moving average * 2)
- * The actual amount of time spent waiting can be greater then the window. For instance
- * observed time spent coalescing was 400 microseconds with the window set to 200 in one benchmark.
- */
- @VisibleForTesting
- static class MovingAverageCoalescingStrategy extends CoalescingStrategy
- {
- private final int samples[] = new int[16];
- private long lastSample = 0;
- private int index = 0;
- private long sum = 0;
-
- private final long maxCoalesceWindow;
-
- public MovingAverageCoalescingStrategy(int maxCoalesceWindow, Parker parker, Logger logger, String displayName)
- {
- super(parker, logger, displayName);
- this.maxCoalesceWindow = TimeUnit.MICROSECONDS.toNanos(maxCoalesceWindow);
- for (int ii = 0; ii < samples.length; ii++)
- samples[ii] = Integer.MAX_VALUE;
- sum = Integer.MAX_VALUE * (long)samples.length;
- }
-
- private long logSample(int value)
- {
- sum -= samples[index];
- sum += value;
- samples[index] = value;
- index++;
- index = index & ((1 << 4) - 1);
- return sum / 16;
- }
-
- private long notifyOfSample(long sample)
- {
- debugTimestamp(sample);
- if (sample > lastSample)
- {
- final int delta = (int)(Math.min(Integer.MAX_VALUE, sample - lastSample));
- lastSample = sample;
- return logSample(delta);
- }
- else
- {
- return logSample(1);
- }
- }
-
- @Override
- protected <C extends Coalescable> void coalesceInternal(BlockingQueue<C> input, List<C> out, int maxItems) throws InterruptedException
- {
- if (input.drainTo(out, maxItems) == 0)
- {
- out.add(input.take());
- input.drainTo(out, maxItems - out.size());
- }
-
- long average = notifyOfSample(out.get(0).timestampNanos());
- debugGap(average);
-
- if (maybeSleep(out.size(), average, maxCoalesceWindow, parker)) {
- input.drainTo(out, maxItems - out.size());
- }
-
- for (int ii = 1; ii < out.size(); ii++)
- notifyOfSample(out.get(ii).timestampNanos());
- }
-
- @Override
- public String toString()
- {
- return "Moving average";
- }
- }
-
- /*
- * A fixed strategy as a backup in case MovingAverage or TimeHorizongMovingAverage fails in some scenario
- */
- @VisibleForTesting
- static class FixedCoalescingStrategy extends CoalescingStrategy
- {
- private final long coalesceWindow;
-
- public FixedCoalescingStrategy(int coalesceWindowMicros, Parker parker, Logger logger, String displayName)
- {
- super(parker, logger, displayName);
- coalesceWindow = TimeUnit.MICROSECONDS.toNanos(coalesceWindowMicros);
- }
-
- @Override
- protected <C extends Coalescable> void coalesceInternal(BlockingQueue<C> input, List<C> out, int maxItems) throws InterruptedException
- {
- int enough = DatabaseDescriptor.getOtcCoalescingEnoughCoalescedMessages();
-
- if (input.drainTo(out, maxItems) == 0)
- {
- out.add(input.take());
- input.drainTo(out, maxItems - out.size());
- if (out.size() < enough) {
- parker.park(coalesceWindow);
- input.drainTo(out, maxItems - out.size());
- }
- }
- debugTimestamps(out);
- }
-
- @Override
- public String toString()
- {
- return "Fixed";
- }
- }
-
- /*
- * A coalesscing strategy that just returns all currently available elements
- */
- @VisibleForTesting
- static class DisabledCoalescingStrategy extends CoalescingStrategy
- {
-
- public DisabledCoalescingStrategy(int coalesceWindowMicros, Parker parker, Logger logger, String displayName)
- {
- super(parker, logger, displayName);
- }
-
- @Override
- protected <C extends Coalescable> void coalesceInternal(BlockingQueue<C> input, List<C> out, int maxItems) throws InterruptedException
- {
- if (input.drainTo(out, maxItems) == 0)
- {
- out.add(input.take());
- input.drainTo(out, maxItems - 1);
- }
- debugTimestamps(out);
- }
-
- @Override
- public String toString()
- {
- return "Disabled";
- }
- }
-
- @VisibleForTesting
- static CoalescingStrategy newCoalescingStrategy(String strategy,
- int coalesceWindow,
- Parker parker,
- Logger logger,
- String displayName)
- {
- String classname = null;
- String strategyCleaned = strategy.trim().toUpperCase(Locale.ENGLISH);
- switch(strategyCleaned)
- {
- case "MOVINGAVERAGE":
- classname = MovingAverageCoalescingStrategy.class.getName();
- break;
- case "FIXED":
- classname = FixedCoalescingStrategy.class.getName();
- break;
- case "TIMEHORIZON":
- classname = TimeHorizonMovingAverageCoalescingStrategy.class.getName();
- break;
- case "DISABLED":
- classname = DisabledCoalescingStrategy.class.getName();
- break;
- default:
- classname = strategy;
- }
-
- try
- {
- Class<?> clazz = Class.forName(classname);
-
- if (!CoalescingStrategy.class.isAssignableFrom(clazz))
- {
- throw new RuntimeException(classname + " is not an instance of CoalescingStrategy");
- }
-
- Constructor<?> constructor = clazz.getConstructor(int.class, Parker.class, Logger.class, String.class);
-
- return (CoalescingStrategy)constructor.newInstance(coalesceWindow, parker, logger, displayName);
- }
- catch (Exception e)
- {
- throw new RuntimeException(e);
- }
- }
-
- public static CoalescingStrategy newCoalescingStrategy(String strategy, int coalesceWindow, Logger logger, String displayName)
- {
- return newCoalescingStrategy(strategy, coalesceWindow, PARKER, logger, displayName);
- }
-}
diff --git a/src/java/org/apache/cassandra/utils/Collectors3.java b/src/java/org/apache/cassandra/utils/Collectors3.java
new file mode 100644
index 0000000..f8f262e
--- /dev/null
+++ b/src/java/org/apache/cassandra/utils/Collectors3.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.utils;
+
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collector;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+
+/**
+ * Some extra Collector implementations.
+ *
+ * Named Collectors3 just in case Guava ever makes a Collectors2
+ */
+public class Collectors3
+{
+ private static final Collector.Characteristics[] LIST_CHARACTERISTICS = new Collector.Characteristics[] { };
+ public static <T> Collector<T, ?, List<T>> toImmutableList()
+ {
+ return Collector.of(ImmutableList.Builder<T>::new,
+ ImmutableList.Builder<T>::add,
+ (l, r) -> l.addAll(r.build()),
+ ImmutableList.Builder<T>::build,
+ LIST_CHARACTERISTICS);
+ }
+
+ private static final Collector.Characteristics[] SET_CHARACTERISTICS = new Collector.Characteristics[] { Collector.Characteristics.UNORDERED };
+ public static <T> Collector<T, ?, Set<T>> toImmutableSet()
+ {
+ return Collector.of(ImmutableSet.Builder<T>::new,
+ ImmutableSet.Builder<T>::add,
+ (l, r) -> l.addAll(r.build()),
+ ImmutableSet.Builder<T>::build,
+ SET_CHARACTERISTICS);
+ }
+}
diff --git a/src/java/org/apache/cassandra/utils/DiagnosticSnapshotService.java b/src/java/org/apache/cassandra/utils/DiagnosticSnapshotService.java
index 5c48412..d1f33ed 100644
--- a/src/java/org/apache/cassandra/utils/DiagnosticSnapshotService.java
+++ b/src/java/org/apache/cassandra/utils/DiagnosticSnapshotService.java
@@ -21,7 +21,6 @@
import java.net.InetAddress;
import java.time.LocalDate;
import java.time.format.DateTimeFormatter;
-import java.util.UUID;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicLong;
@@ -31,10 +30,14 @@
import org.slf4j.LoggerFactory;
import org.apache.cassandra.concurrent.NamedThreadFactory;
-import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.db.*;
-import org.apache.cassandra.net.MessageOut;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.net.Message;
import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.net.Verb;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.schema.TableMetadata;
+import org.hsqldb.Table;
/**
* Provides a means to take snapshots when triggered by anomalous events or when the breaking of invariants is
@@ -63,6 +66,7 @@
public static final DiagnosticSnapshotService instance =
new DiagnosticSnapshotService(Executors.newSingleThreadExecutor(new NamedThreadFactory("DiagnosticSnapshot")));
+ public static final String REPAIRED_DATA_MISMATCH_SNAPSHOT_PREFIX = "RepairedDataMismatch-";
public static final String DUPLICATE_ROWS_DETECTED_SNAPSHOT_PREFIX = "DuplicateRows-";
private final Executor executor;
@@ -78,19 +82,25 @@
// Overridable via system property for testing.
private static final long SNAPSHOT_INTERVAL_NANOS = TimeUnit.MINUTES.toNanos(1);
private static final DateTimeFormatter DATE_FORMAT = DateTimeFormatter.BASIC_ISO_DATE;
- private final ConcurrentHashMap<UUID, AtomicLong> lastSnapshotTimes = new ConcurrentHashMap<>();
+ private final ConcurrentHashMap<TableId, AtomicLong> lastSnapshotTimes = new ConcurrentHashMap<>();
- public static void duplicateRows(CFMetaData metadata, Iterable<InetAddress> replicas)
+ public static void duplicateRows(TableMetadata metadata, Iterable<InetAddressAndPort> replicas)
{
instance.maybeTriggerSnapshot(metadata, DUPLICATE_ROWS_DETECTED_SNAPSHOT_PREFIX, replicas);
}
- public static boolean isDiagnosticSnapshotRequest(SnapshotCommand command)
+ public static void repairedDataMismatch(TableMetadata metadata, Iterable<InetAddressAndPort> replicas)
{
- return command.snapshot_name.startsWith(DUPLICATE_ROWS_DETECTED_SNAPSHOT_PREFIX);
+ instance.maybeTriggerSnapshot(metadata, REPAIRED_DATA_MISMATCH_SNAPSHOT_PREFIX, replicas);
}
- public static void snapshot(SnapshotCommand command, InetAddress initiator)
+ public static boolean isDiagnosticSnapshotRequest(SnapshotCommand command)
+ {
+ return command.snapshot_name.startsWith(REPAIRED_DATA_MISMATCH_SNAPSHOT_PREFIX)
+ || command.snapshot_name.startsWith(DUPLICATE_ROWS_DETECTED_SNAPSHOT_PREFIX);
+ }
+
+ public static void snapshot(SnapshotCommand command, InetAddressAndPort initiator)
{
Preconditions.checkArgument(isDiagnosticSnapshotRequest(command));
instance.maybeSnapshot(command, initiator);
@@ -107,20 +117,21 @@
ExecutorUtils.shutdownNowAndWait(timeout, unit, executor);
}
- private void maybeTriggerSnapshot(CFMetaData metadata, String prefix, Iterable<InetAddress> endpoints)
+ private void maybeTriggerSnapshot(TableMetadata metadata, String prefix, Iterable<InetAddressAndPort> endpoints)
{
long now = System.nanoTime();
- AtomicLong cached = lastSnapshotTimes.computeIfAbsent(metadata.cfId, u -> new AtomicLong(0));
+ AtomicLong cached = lastSnapshotTimes.computeIfAbsent(metadata.id, u -> new AtomicLong(0));
long last = cached.get();
long interval = Long.getLong("cassandra.diagnostic_snapshot_interval_nanos", SNAPSHOT_INTERVAL_NANOS);
if (now - last > interval && cached.compareAndSet(last, now))
{
- MessageOut<?> msg = new SnapshotCommand(metadata.ksName,
- metadata.cfName,
- getSnapshotName(prefix),
- false).createMessage();
- for (InetAddress replica : endpoints)
- MessagingService.instance().sendOneWay(msg, replica);
+ Message<SnapshotCommand> msg = Message.out(Verb.SNAPSHOT_REQ,
+ new SnapshotCommand(metadata.keyspace,
+ metadata.name,
+ getSnapshotName(prefix),
+ false));
+ for (InetAddressAndPort replica : endpoints)
+ MessagingService.instance().send(msg, replica);
}
else
{
@@ -128,7 +139,7 @@
}
}
- private void maybeSnapshot(SnapshotCommand command, InetAddress initiator)
+ private void maybeSnapshot(SnapshotCommand command, InetAddressAndPort initiator)
{
executor.execute(new DiagnosticSnapshotTask(command, initiator));
}
@@ -136,9 +147,9 @@
private static class DiagnosticSnapshotTask implements Runnable
{
final SnapshotCommand command;
- final InetAddress from;
+ final InetAddressAndPort from;
- DiagnosticSnapshotTask(SnapshotCommand command, InetAddress from)
+ DiagnosticSnapshotTask(SnapshotCommand command, InetAddressAndPort from)
{
this.command = command;
this.from = from;
diff --git a/src/java/org/apache/cassandra/utils/EstimatedHistogram.java b/src/java/org/apache/cassandra/utils/EstimatedHistogram.java
index 0914a58..f25cc1e 100644
--- a/src/java/org/apache/cassandra/utils/EstimatedHistogram.java
+++ b/src/java/org/apache/cassandra/utils/EstimatedHistogram.java
@@ -112,11 +112,7 @@
return bucketOffsets;
}
- /**
- * Increments the count of the bucket closest to n, rounding UP.
- * @param n
- */
- public void add(long n)
+ private int findIndex(long n)
{
int index = Arrays.binarySearch(bucketOffsets, n);
if (index < 0)
@@ -124,8 +120,25 @@
// inexact match, take the first bucket higher than n
index = -index - 1;
}
- // else exact match; we're good
- buckets.incrementAndGet(index);
+ return index;
+ }
+
+ /**
+ * Increments the count of the bucket closest to n, rounding UP.
+ * @param n
+ */
+ public void add(long n)
+ {
+ buckets.incrementAndGet(findIndex(n));
+ }
+
+ /**
+ * Increments the count of the bucket closest to n, rounding UP by delta
+ * @param n
+ */
+ public void add(long n, long delta)
+ {
+ buckets.addAndGet(findIndex(n), delta);
}
/**
diff --git a/src/java/org/apache/cassandra/utils/ExpiringMap.java b/src/java/org/apache/cassandra/utils/ExpiringMap.java
deleted file mode 100644
index a6895c5..0000000
--- a/src/java/org/apache/cassandra/utils/ExpiringMap.java
+++ /dev/null
@@ -1,191 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.utils;
-
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
-
-import com.google.common.base.Function;
-import com.google.common.util.concurrent.Uninterruptibles;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.cassandra.concurrent.DebuggableScheduledThreadPoolExecutor;
-
-public class ExpiringMap<K, V>
-{
- private static final Logger logger = LoggerFactory.getLogger(ExpiringMap.class);
- private volatile boolean shutdown;
-
- public static class CacheableObject<T>
- {
- public final T value;
- public final long timeout;
- private final long createdAt;
-
- private CacheableObject(T value, long timeout)
- {
- assert value != null;
- this.value = value;
- this.timeout = timeout;
- this.createdAt = Clock.instance.nanoTime();
- }
-
- private boolean isReadyToDieAt(long atNano)
- {
- return atNano - createdAt > TimeUnit.MILLISECONDS.toNanos(timeout);
- }
- }
-
- // if we use more ExpiringMaps we may want to add multiple threads to this executor
- private static final ScheduledExecutorService service = new DebuggableScheduledThreadPoolExecutor("EXPIRING-MAP-REAPER");
-
- private final ConcurrentMap<K, CacheableObject<V>> cache = new ConcurrentHashMap<K, CacheableObject<V>>();
- private final long defaultExpiration;
-
- public ExpiringMap(long defaultExpiration)
- {
- this(defaultExpiration, null);
- }
-
- /**
- *
- * @param defaultExpiration the TTL for objects in the cache in milliseconds
- */
- public ExpiringMap(long defaultExpiration, final Function<Pair<K,CacheableObject<V>>, ?> postExpireHook)
- {
- this.defaultExpiration = defaultExpiration;
-
- if (defaultExpiration <= 0)
- {
- throw new IllegalArgumentException("Argument specified must be a positive number");
- }
-
- Runnable runnable = new Runnable()
- {
- public void run()
- {
- long start = Clock.instance.nanoTime();
- int n = 0;
- for (Map.Entry<K, CacheableObject<V>> entry : cache.entrySet())
- {
- if (entry.getValue().isReadyToDieAt(start))
- {
- if (cache.remove(entry.getKey()) != null)
- {
- n++;
- if (postExpireHook != null)
- postExpireHook.apply(Pair.create(entry.getKey(), entry.getValue()));
- }
- }
- }
- logger.trace("Expired {} entries", n);
- }
- };
- service.scheduleWithFixedDelay(runnable, defaultExpiration / 2, defaultExpiration / 2, TimeUnit.MILLISECONDS);
- }
-
- public boolean shutdownBlocking()
- {
- service.shutdown();
- try
- {
- return service.awaitTermination(defaultExpiration * 2, TimeUnit.MILLISECONDS);
- }
- catch (InterruptedException e)
- {
- throw new AssertionError(e);
- }
- }
-
- public void reset()
- {
- shutdown = false;
- clear();
- }
-
- public void clear()
- {
- cache.clear();
- }
-
- public V put(K key, V value)
- {
- return put(key, value, this.defaultExpiration);
- }
-
- public V put(K key, V value, long timeout)
- {
- if (shutdown)
- {
- // StorageProxy isn't equipped to deal with "I'm nominally alive, but I can't send any messages out."
- // So we'll just sit on this thread until the rest of the server shutdown completes.
- //
- // See comments in CustomTThreadPoolServer.serve, CASSANDRA-3335, and CASSANDRA-3727.
- Uninterruptibles.sleepUninterruptibly(Long.MAX_VALUE, TimeUnit.NANOSECONDS);
- }
- CacheableObject<V> previous = cache.put(key, new CacheableObject<V>(value, timeout));
- return (previous == null) ? null : previous.value;
- }
-
- public V get(K key)
- {
- CacheableObject<V> co = cache.get(key);
- return co == null ? null : co.value;
- }
-
- public V remove(K key)
- {
- CacheableObject<V> co = cache.remove(key);
- return co == null ? null : co.value;
- }
-
- /**
- * @return System.nanoTime() when key was put into the map.
- */
- public long getAge(K key)
- {
- CacheableObject<V> co = cache.get(key);
- return co == null ? 0 : co.createdAt;
- }
-
- public int size()
- {
- return cache.size();
- }
-
- public boolean containsKey(K key)
- {
- return cache.containsKey(key);
- }
-
- public boolean isEmpty()
- {
- return cache.isEmpty();
- }
-
- public Set<K> keySet()
- {
- return cache.keySet();
- }
-}
diff --git a/src/java/org/apache/cassandra/utils/FBUtilities.java b/src/java/org/apache/cassandra/utils/FBUtilities.java
index e5dc2be..8e000d5 100644
--- a/src/java/org/apache/cassandra/utils/FBUtilities.java
+++ b/src/java/org/apache/cassandra/utils/FBUtilities.java
@@ -26,6 +26,9 @@
import java.security.NoSuchAlgorithmException;
import java.util.*;
import java.util.concurrent.*;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+import java.util.stream.Stream;
import java.util.zip.CRC32;
import java.util.zip.Checksum;
@@ -36,16 +39,17 @@
import com.google.common.base.Joiner;
import com.google.common.base.Strings;
import com.google.common.util.concurrent.Uninterruptibles;
-
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.databind.ObjectMapper;
-import io.netty.util.concurrent.FastThreadLocal;
+import org.apache.cassandra.auth.AllowAllNetworkAuthorizer;
+import org.apache.cassandra.audit.IAuditLogger;
import org.apache.cassandra.auth.IAuthenticator;
import org.apache.cassandra.auth.IAuthorizer;
+import org.apache.cassandra.auth.INetworkAuthorizer;
import org.apache.cassandra.auth.IRoleManager;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.DecoratedKey;
@@ -65,10 +69,15 @@
import org.apache.cassandra.io.util.DataOutputBufferFixed;
import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.locator.InetAddressAndPort;
-import org.apache.cassandra.net.AsyncOneResponse;
+
public class FBUtilities
{
+ static
+ {
+ preventIllegalAccessWarnings();
+ }
+
private static final Logger logger = LoggerFactory.getLogger(FBUtilities.class);
private static final ObjectMapper jsonMapper = new ObjectMapper(new JsonFactory());
@@ -84,7 +93,10 @@
private static volatile InetAddress localInetAddress;
private static volatile InetAddress broadcastInetAddress;
- private static volatile InetAddress broadcastRpcAddress;
+ private static volatile InetAddress broadcastNativeAddress;
+ private static volatile InetAddressAndPort broadcastNativeAddressAndPort;
+ private static volatile InetAddressAndPort broadcastInetAddressAndPort;
+ private static volatile InetAddressAndPort localInetAddressAndPort;
private static volatile String previousReleaseVersionString;
@@ -97,24 +109,8 @@
return Runtime.getRuntime().availableProcessors();
}
- private static final FastThreadLocal<MessageDigest> localMD5Digest = new FastThreadLocal<MessageDigest>()
- {
- @Override
- protected MessageDigest initialValue()
- {
- return newMessageDigest("MD5");
- }
- };
-
public static final int MAX_UNSIGNED_SHORT = 0xFFFF;
- public static MessageDigest threadLocalMD5Digest()
- {
- MessageDigest md = localMD5Digest.get();
- md.reset();
- return md;
- }
-
public static MessageDigest newMessageDigest(String algorithm)
{
try
@@ -128,68 +124,142 @@
}
/**
- * Please use getBroadcastAddress instead. You need this only when you have to listen/connect.
+ * Please use getJustBroadcastAddress instead. You need this only when you have to listen/connect. It's also missing
+ * the port you should be using. 99% of code doesn't want this.
*/
- public static InetAddress getLocalAddress()
+ public static InetAddress getJustLocalAddress()
{
if (localInetAddress == null)
- try
+ {
+ if (DatabaseDescriptor.getListenAddress() == null)
{
- localInetAddress = DatabaseDescriptor.getListenAddress() == null
- ? InetAddress.getLocalHost()
- : DatabaseDescriptor.getListenAddress();
+ try
+ {
+ localInetAddress = InetAddress.getLocalHost();
+ logger.info("InetAddress.getLocalHost() was used to resolve listen_address to {}, double check this is "
+ + "correct. Please check your node's config and set the listen_address in cassandra.yaml accordingly if applicable.",
+ localInetAddress);
+ }
+ catch(UnknownHostException e)
+ {
+ logger.info("InetAddress.getLocalHost() could not resolve the address for the hostname ({}), please "
+ + "check your node's config and set the listen_address in cassandra.yaml. Falling back to {}",
+ e,
+ InetAddress.getLoopbackAddress());
+ // CASSANDRA-15901 fallback for misconfigured nodes
+ localInetAddress = InetAddress.getLoopbackAddress();
+ }
}
- catch (UnknownHostException e)
- {
- throw new RuntimeException(e);
- }
+ else
+ localInetAddress = DatabaseDescriptor.getListenAddress();
+ }
return localInetAddress;
}
- public static InetAddress getBroadcastAddress()
+ /**
+ * The address and port to listen on for intra-cluster storage traffic (not client). Use this to get the correct
+ * stuff to listen on for intra-cluster communication.
+ */
+ public static InetAddressAndPort getLocalAddressAndPort()
+ {
+ if (localInetAddressAndPort == null)
+ {
+ if(DatabaseDescriptor.getRawConfig() == null)
+ {
+ localInetAddressAndPort = InetAddressAndPort.getByAddress(getJustLocalAddress());
+ }
+ else
+ {
+ localInetAddressAndPort = InetAddressAndPort.getByAddressOverrideDefaults(getJustLocalAddress(),
+ DatabaseDescriptor.getStoragePort());
+ }
+ }
+ return localInetAddressAndPort;
+ }
+
+ /**
+ * Retrieve just the broadcast address but not the port. This is almost always the wrong thing to be using because
+ * it's ambiguous since you need the address and port to identify a node. You want getBroadcastAddressAndPort
+ */
+ public static InetAddress getJustBroadcastAddress()
{
if (broadcastInetAddress == null)
broadcastInetAddress = DatabaseDescriptor.getBroadcastAddress() == null
- ? getLocalAddress()
+ ? getJustLocalAddress()
: DatabaseDescriptor.getBroadcastAddress();
return broadcastInetAddress;
}
/**
+ * Get the broadcast address and port for intra-cluster storage traffic. This the address to advertise that uniquely
+ * identifies the node and is reachable from everywhere. This is the one you want unless you are trying to connect
+ * to the local address specifically.
+ */
+ public static InetAddressAndPort getBroadcastAddressAndPort()
+ {
+ if (broadcastInetAddressAndPort == null)
+ {
+ if(DatabaseDescriptor.getRawConfig() == null)
+ {
+ broadcastInetAddressAndPort = InetAddressAndPort.getByAddress(getJustBroadcastAddress());
+ }
+ else
+ {
+ broadcastInetAddressAndPort = InetAddressAndPort.getByAddressOverrideDefaults(getJustBroadcastAddress(),
+ DatabaseDescriptor.getStoragePort());
+ }
+ }
+ return broadcastInetAddressAndPort;
+ }
+
+ /**
* <b>THIS IS FOR TESTING ONLY!!</b>
*/
- @VisibleForTesting
public static void setBroadcastInetAddress(InetAddress addr)
{
broadcastInetAddress = addr;
+ broadcastInetAddressAndPort = InetAddressAndPort.getByAddress(broadcastInetAddress);
}
- public static InetAddress getBroadcastRpcAddress()
+ /**
+ * <b>THIS IS FOR TESTING ONLY!!</b>
+ */
+ public static void setBroadcastInetAddressAndPort(InetAddressAndPort addr)
{
- if (broadcastRpcAddress == null)
- broadcastRpcAddress = DatabaseDescriptor.getBroadcastRpcAddress() == null
+ broadcastInetAddress = addr.address;
+ broadcastInetAddressAndPort = addr;
+ }
+
+ /**
+ * This returns the address that is bound to for the native protocol for communicating with clients. This is ambiguous
+ * because it doesn't include the port and it's almost always the wrong thing to be using you want getBroadcastNativeAddressAndPort
+ */
+ public static InetAddress getJustBroadcastNativeAddress()
+ {
+ if (broadcastNativeAddress == null)
+ broadcastNativeAddress = DatabaseDescriptor.getBroadcastRpcAddress() == null
? DatabaseDescriptor.getRpcAddress()
: DatabaseDescriptor.getBroadcastRpcAddress();
- return broadcastRpcAddress;
+ return broadcastNativeAddress;
}
- public static Collection<InetAddress> getAllLocalAddresses()
+ /**
+ * This returns the address that is bound to for the native protocol for communicating with clients. This is almost
+ * always what you need to identify a node and how to connect to it as a client.
+ */
+ public static InetAddressAndPort getBroadcastNativeAddressAndPort()
{
- Set<InetAddress> localAddresses = new HashSet<InetAddress>();
- try
- {
- Enumeration<NetworkInterface> nets = NetworkInterface.getNetworkInterfaces();
- if (nets != null)
+ if (broadcastNativeAddressAndPort == null)
+ if(DatabaseDescriptor.getRawConfig() == null)
{
- while (nets.hasMoreElements())
- localAddresses.addAll(Collections.list(nets.nextElement().getInetAddresses()));
+ broadcastNativeAddressAndPort = InetAddressAndPort.getByAddress(getJustBroadcastNativeAddress());
}
- }
- catch (SocketException e)
- {
- throw new AssertionError(e);
- }
- return localAddresses;
+ else
+ {
+ broadcastNativeAddressAndPort = InetAddressAndPort.getByAddressOverrideDefaults(getJustBroadcastNativeAddress(),
+ DatabaseDescriptor.getNativeTransportPort());
+ }
+ return broadcastNativeAddressAndPort;
}
public static String getNetworkInterface(InetAddress localAddress)
@@ -253,30 +323,6 @@
return compareUnsigned(bytes1, bytes2, 0, 0, bytes1.length, bytes2.length);
}
- /**
- * @return The bitwise XOR of the inputs. The output will be the same length as the
- * longer input, but if either input is null, the output will be null.
- */
- public static byte[] xor(byte[] left, byte[] right)
- {
- if (left == null || right == null)
- return null;
- if (left.length > right.length)
- {
- byte[] swap = left;
- left = right;
- right = swap;
- }
-
- // left.length is now <= right.length
- byte[] out = Arrays.copyOf(right, right.length);
- for (int i = 0; i < left.length; i++)
- {
- out[i] = (byte)((left[i] & 0xFF) ^ (right[i] & 0xFF));
- }
- return out;
- }
-
public static void sortSampledKeys(List<DecoratedKey> keys, Range<Token> range)
{
if (range.left.compareTo(range.right) >= 0)
@@ -447,12 +493,6 @@
}
}
- public static void waitOnFutures(List<AsyncOneResponse> results, long ms) throws TimeoutException
- {
- for (AsyncOneResponse result : results)
- result.get(ms, TimeUnit.MILLISECONDS);
- }
-
public static <T> Future<? extends T> waitOnFirstFuture(Iterable<? extends Future<? extends T>> futures)
{
return waitOnFirstFuture(futures, 100);
@@ -488,6 +528,81 @@
Uninterruptibles.sleepUninterruptibly(delay, TimeUnit.MILLISECONDS);
}
}
+
+ /**
+ * Returns a new {@link Future} wrapping the given list of futures and returning a list of their results.
+ */
+ public static Future<List> allOf(Collection<Future> futures)
+ {
+ if (futures.isEmpty())
+ return CompletableFuture.completedFuture(null);
+
+ return new Future<List>()
+ {
+ @Override
+ @SuppressWarnings("unchecked")
+ public List get() throws InterruptedException, ExecutionException
+ {
+ List result = new ArrayList<>(futures.size());
+ for (Future current : futures)
+ {
+ result.add(current.get());
+ }
+ return result;
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public List get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException
+ {
+ List result = new ArrayList<>(futures.size());
+ long deadline = System.nanoTime() + TimeUnit.NANOSECONDS.convert(timeout, unit);
+ for (Future current : futures)
+ {
+ long remaining = deadline - System.nanoTime();
+ if (remaining <= 0)
+ throw new TimeoutException();
+
+ result.add(current.get(remaining, TimeUnit.NANOSECONDS));
+ }
+ return result;
+ }
+
+ @Override
+ public boolean cancel(boolean mayInterruptIfRunning)
+ {
+ for (Future current : futures)
+ {
+ if (!current.cancel(mayInterruptIfRunning))
+ return false;
+ }
+ return true;
+ }
+
+ @Override
+ public boolean isCancelled()
+ {
+ for (Future current : futures)
+ {
+ if (!current.isCancelled())
+ return false;
+ }
+ return true;
+ }
+
+ @Override
+ public boolean isDone()
+ {
+ for (Future current : futures)
+ {
+ if (!current.isDone())
+ return false;
+ }
+ return true;
+ }
+ };
+ }
+
/**
* Create a new instance of a partitioner defined in an SSTable Descriptor
* @param desc Descriptor of an sstable
@@ -543,6 +658,51 @@
return FBUtilities.construct(className, "role manager");
}
+ public static INetworkAuthorizer newNetworkAuthorizer(String className)
+ {
+ if (className == null)
+ {
+ return new AllowAllNetworkAuthorizer();
+ }
+ if (!className.contains("."))
+ {
+ className = "org.apache.cassandra.auth." + className;
+ }
+ return FBUtilities.construct(className, "network authorizer");
+ }
+
+ public static IAuditLogger newAuditLogger(String className, Map<String, String> parameters) throws ConfigurationException
+ {
+ if (!className.contains("."))
+ className = "org.apache.cassandra.audit." + className;
+
+ try
+ {
+ Class<?> auditLoggerClass = Class.forName(className);
+ return (IAuditLogger) auditLoggerClass.getConstructor(Map.class).newInstance(parameters);
+ }
+ catch (Exception ex)
+ {
+ throw new ConfigurationException("Unable to create instance of IAuditLogger.", ex);
+ }
+ }
+
+ public static boolean isAuditLoggerClassExists(String className)
+ {
+ if (!className.contains("."))
+ className = "org.apache.cassandra.audit." + className;
+
+ try
+ {
+ FBUtilities.classForName(className, "Audit logger");
+ }
+ catch (ConfigurationException e)
+ {
+ return false;
+ }
+ return true;
+ }
+
/**
* @return The Class for the given name.
* @param classname Fully qualified classname.
@@ -881,42 +1041,6 @@
return historyDir;
}
- public static void updateWithShort(MessageDigest digest, int val)
- {
- digest.update((byte) ((val >> 8) & 0xFF));
- digest.update((byte) (val & 0xFF));
- }
-
- public static void updateWithByte(MessageDigest digest, int val)
- {
- digest.update((byte) (val & 0xFF));
- }
-
- public static void updateWithInt(MessageDigest digest, int val)
- {
- digest.update((byte) ((val >>> 24) & 0xFF));
- digest.update((byte) ((val >>> 16) & 0xFF));
- digest.update((byte) ((val >>> 8) & 0xFF));
- digest.update((byte) ((val >>> 0) & 0xFF));
- }
-
- public static void updateWithLong(MessageDigest digest, long val)
- {
- digest.update((byte) ((val >>> 56) & 0xFF));
- digest.update((byte) ((val >>> 48) & 0xFF));
- digest.update((byte) ((val >>> 40) & 0xFF));
- digest.update((byte) ((val >>> 32) & 0xFF));
- digest.update((byte) ((val >>> 24) & 0xFF));
- digest.update((byte) ((val >>> 16) & 0xFF));
- digest.update((byte) ((val >>> 8) & 0xFF));
- digest.update((byte) ((val >>> 0) & 0xFF));
- }
-
- public static void updateWithBoolean(MessageDigest digest, boolean val)
- {
- updateWithByte(digest, val ? 0 : 1);
- }
-
public static void closeAll(Collection<? extends AutoCloseable> l) throws Exception
{
Exception toThrow = null;
@@ -972,10 +1096,36 @@
}
@VisibleForTesting
- protected static void reset()
+ public static void reset()
{
localInetAddress = null;
+ localInetAddressAndPort = null;
broadcastInetAddress = null;
- broadcastRpcAddress = null;
+ broadcastInetAddressAndPort = null;
+ broadcastNativeAddress = null;
+ }
+
+ /**
+ * Hack to prevent the ugly "illegal access" warnings in Java 11+ like the following.
+ */
+ public static void preventIllegalAccessWarnings()
+ {
+ // Example "annoying" trace:
+ // WARNING: An illegal reflective access operation has occurred
+ // WARNING: Illegal reflective access by io.netty.util.internal.ReflectionUtil (file:...)
+ // WARNING: Please consider reporting this to the maintainers of io.netty.util.internal.ReflectionUtil
+ // WARNING: Use --illegal-access=warn to enable warnings of further illegal reflective access operations
+ // WARNING: All illegal access operations will be denied in a future release
+ try
+ {
+ Class<?> c = Class.forName("jdk.internal.module.IllegalAccessLogger");
+ Field f = c.getDeclaredField("logger");
+ f.setAccessible(true);
+ f.set(null, null);
+ }
+ catch (Exception e)
+ {
+ // ignore
+ }
}
}
diff --git a/src/java/org/apache/cassandra/utils/FastByteOperations.java b/src/java/org/apache/cassandra/utils/FastByteOperations.java
index 6581736..060dee5 100644
--- a/src/java/org/apache/cassandra/utils/FastByteOperations.java
+++ b/src/java/org/apache/cassandra/utils/FastByteOperations.java
@@ -55,6 +55,16 @@
return -BestHolder.BEST.compare(b2, b1, s1, l1);
}
+ public static int compareUnsigned(ByteBuffer b1, int s1, int l1, byte[] b2, int s2, int l2)
+ {
+ return BestHolder.BEST.compare(b1, s1, l1, b2, s2, l2);
+ }
+
+ public static int compareUnsigned(byte[] b1, int s1, int l1, ByteBuffer b2, int s2, int l2)
+ {
+ return -BestHolder.BEST.compare(b2, s2, l2, b1, s1, l1);
+ }
+
public static int compareUnsigned(ByteBuffer b1, ByteBuffer b2)
{
return BestHolder.BEST.compare(b1, b2);
@@ -77,6 +87,8 @@
abstract public int compare(ByteBuffer buffer1, byte[] buffer2, int offset2, int length2);
+ abstract public int compare(ByteBuffer buffer1, int offset1, int length1, byte[] buffer2, int offset2, int length2);
+
abstract public int compare(ByteBuffer buffer1, ByteBuffer buffer2);
abstract public void copy(ByteBuffer src, int srcPosition, byte[] trg, int trgPosition, int length);
@@ -187,25 +199,24 @@
public int compare(ByteBuffer buffer1, byte[] buffer2, int offset2, int length2)
{
+ return compare(buffer1, buffer1.position(), buffer1.remaining(), buffer2, offset2, length2);
+ }
+
+ public int compare(ByteBuffer buffer1, int position1, int length1, byte[] buffer2, int offset2, int length2)
+ {
Object obj1;
long offset1;
if (buffer1.hasArray())
{
obj1 = buffer1.array();
- offset1 = BYTE_ARRAY_BASE_OFFSET + buffer1.arrayOffset();
+ offset1 = BYTE_ARRAY_BASE_OFFSET + buffer1.arrayOffset() + position1;
}
else
{
obj1 = null;
- offset1 = theUnsafe.getLong(buffer1, DIRECT_BUFFER_ADDRESS_OFFSET);
+ offset1 = theUnsafe.getLong(buffer1, DIRECT_BUFFER_ADDRESS_OFFSET) + position1;
}
- int length1;
- {
- int position = buffer1.position();
- int limit = buffer1.limit();
- length1 = limit - position;
- offset1 += position;
- }
+
return compareTo(obj1, offset1, length1, buffer2, BYTE_ARRAY_BASE_OFFSET + offset2, length2);
}
@@ -397,11 +408,28 @@
return length1 - length2;
}
+ public int compare(ByteBuffer buffer1, int position1, int length1, byte[] buffer2, int offset2, int length2)
+ {
+ if (buffer1.hasArray())
+ return compare(buffer1.array(), buffer1.arrayOffset() + position1, length1, buffer2, offset2, length2);
+
+ if (position1 != buffer1.position())
+ {
+ buffer1 = buffer1.duplicate();
+ buffer1.position(position1);
+ }
+
+ return compare(buffer1, ByteBuffer.wrap(buffer2, offset2, length2));
+ }
+
public int compare(ByteBuffer buffer1, byte[] buffer2, int offset2, int length2)
{
if (buffer1.hasArray())
+ {
return compare(buffer1.array(), buffer1.arrayOffset() + buffer1.position(), buffer1.remaining(),
buffer2, offset2, length2);
+ }
+
return compare(buffer1, ByteBuffer.wrap(buffer2, offset2, length2));
}
diff --git a/src/java/org/apache/cassandra/utils/FilterFactory.java b/src/java/org/apache/cassandra/utils/FilterFactory.java
index ddcf1bb..4cf0cbf 100644
--- a/src/java/org/apache/cassandra/utils/FilterFactory.java
+++ b/src/java/org/apache/cassandra/utils/FilterFactory.java
@@ -17,16 +17,11 @@
*/
package org.apache.cassandra.utils;
-import java.io.DataInput;
-import java.io.IOException;
-
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.utils.obs.IBitSet;
import org.apache.cassandra.utils.obs.OffHeapBitSet;
-import org.apache.cassandra.utils.obs.OpenBitSet;
public class FilterFactory
{
@@ -35,21 +30,11 @@
private static final Logger logger = LoggerFactory.getLogger(FilterFactory.class);
private static final long BITSET_EXCESS = 20;
- public static void serialize(IFilter bf, DataOutputPlus output) throws IOException
- {
- BloomFilterSerializer.serialize((BloomFilter) bf, output);
- }
-
- public static IFilter deserialize(DataInput input, boolean offheap, boolean oldBfHashOrder) throws IOException
- {
- return BloomFilterSerializer.deserialize(input, offheap, oldBfHashOrder);
- }
-
/**
* @return A BloomFilter with the lowest practical false positive
* probability for the given number of elements.
*/
- public static IFilter getFilter(long numElements, int targetBucketsPerElem, boolean offheap, boolean oldBfHashOrder)
+ public static IFilter getFilter(long numElements, int targetBucketsPerElem)
{
int maxBucketsPerElement = Math.max(1, BloomCalculations.maxBucketsPerElement(numElements));
int bucketsPerElement = Math.min(targetBucketsPerElem, maxBucketsPerElement);
@@ -58,7 +43,7 @@
logger.warn("Cannot provide an optimal BloomFilter for {} elements ({}/{} buckets per element).", numElements, bucketsPerElement, targetBucketsPerElem);
}
BloomCalculations.BloomSpecification spec = BloomCalculations.computeBloomSpec(bucketsPerElement);
- return createFilter(spec.K, numElements, spec.bucketsPerElement, offheap, oldBfHashOrder);
+ return createFilter(spec.K, numElements, spec.bucketsPerElement);
}
/**
@@ -68,21 +53,21 @@
* Asserts that the given probability can be satisfied using this
* filter.
*/
- public static IFilter getFilter(long numElements, double maxFalsePosProbability, boolean offheap, boolean oldBfHashOrder)
+ public static IFilter getFilter(long numElements, double maxFalsePosProbability)
{
assert maxFalsePosProbability <= 1.0 : "Invalid probability";
if (maxFalsePosProbability == 1.0)
return new AlwaysPresentFilter();
int bucketsPerElement = BloomCalculations.maxBucketsPerElement(numElements);
BloomCalculations.BloomSpecification spec = BloomCalculations.computeBloomSpec(bucketsPerElement, maxFalsePosProbability);
- return createFilter(spec.K, numElements, spec.bucketsPerElement, offheap, oldBfHashOrder);
+ return createFilter(spec.K, numElements, spec.bucketsPerElement);
}
@SuppressWarnings("resource")
- private static IFilter createFilter(int hash, long numElements, int bucketsPer, boolean offheap, boolean oldBfHashOrder)
+ private static IFilter createFilter(int hash, long numElements, int bucketsPer)
{
long numBits = (numElements * bucketsPer) + BITSET_EXCESS;
- IBitSet bitset = offheap ? new OffHeapBitSet(numBits) : new OpenBitSet(numBits);
- return new BloomFilter(hash, bitset, oldBfHashOrder);
+ IBitSet bitset = new OffHeapBitSet(numBits);
+ return new BloomFilter(hash, bitset);
}
}
diff --git a/src/java/org/apache/cassandra/utils/GuidGenerator.java b/src/java/org/apache/cassandra/utils/GuidGenerator.java
index 9742353..2b3af01 100644
--- a/src/java/org/apache/cassandra/utils/GuidGenerator.java
+++ b/src/java/org/apache/cassandra/utils/GuidGenerator.java
@@ -37,7 +37,7 @@
long secureInitializer = mySecureRand.nextLong();
myRand = new Random(secureInitializer);
try {
- s_id = FBUtilities.getLocalAddress().toString();
+ s_id = FBUtilities.getLocalAddressAndPort().toString();
}
catch (RuntimeException e) {
throw new AssertionError(e);
@@ -83,7 +83,7 @@
.append(Long.toString(rand));
String valueBeforeMD5 = sbValueBeforeMD5.toString();
- return ByteBuffer.wrap(FBUtilities.threadLocalMD5Digest().digest(valueBeforeMD5.getBytes()));
+ return ByteBuffer.wrap(MD5Digest.threadLocalMD5Digest().digest(valueBeforeMD5.getBytes()));
}
public static ByteBuffer guidAsBytes()
diff --git a/src/java/org/apache/cassandra/utils/IntegerInterval.java b/src/java/org/apache/cassandra/utils/IntegerInterval.java
index f600136..b26ac45 100644
--- a/src/java/org/apache/cassandra/utils/IntegerInterval.java
+++ b/src/java/org/apache/cassandra/utils/IntegerInterval.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.cassandra.utils;
import java.util.*;
diff --git a/src/java/org/apache/cassandra/utils/JMXServerUtils.java b/src/java/org/apache/cassandra/utils/JMXServerUtils.java
index 48d02f7..1f79a33 100644
--- a/src/java/org/apache/cassandra/utils/JMXServerUtils.java
+++ b/src/java/org/apache/cassandra/utils/JMXServerUtils.java
@@ -19,7 +19,10 @@
package org.apache.cassandra.utils;
import java.io.IOException;
+import java.lang.invoke.MethodHandle;
+import java.lang.invoke.MethodHandles;
import java.lang.management.ManagementFactory;
+import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationHandler;
import java.lang.reflect.Proxy;
import java.net.Inet6Address;
@@ -44,12 +47,12 @@
import javax.rmi.ssl.SslRMIServerSocketFactory;
import javax.security.auth.Subject;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableMap;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.sun.jmx.remote.security.JMXPluggableAuthenticator;
import org.apache.cassandra.auth.jmx.AuthenticationProxy;
public class JMXServerUtils
@@ -61,7 +64,8 @@
* inaccessable.
*/
@SuppressWarnings("resource")
- public static JMXConnectorServer createJMXServer(int port, boolean local)
+ @VisibleForTesting
+ public static JMXConnectorServer createJMXServer(int port, String hostname, boolean local)
throws IOException
{
Map<String, Object> env = new HashMap<>();
@@ -118,7 +122,7 @@
(RMIClientSocketFactory) env.get(RMIConnectorServer.RMI_CLIENT_SOCKET_FACTORY_ATTRIBUTE),
(RMIServerSocketFactory) env.get(RMIConnectorServer.RMI_SERVER_SOCKET_FACTORY_ATTRIBUTE),
env);
- JMXServiceURL serviceURL = new JMXServiceURL("rmi", null, rmiPort);
+ JMXServiceURL serviceURL = new JMXServiceURL("rmi", hostname, rmiPort);
RMIConnectorServer jmxServer = new RMIConnectorServer(serviceURL, env, server, ManagementFactory.getPlatformMBeanServer());
// If a custom authz proxy was created, attach it to the server now.
@@ -131,6 +135,12 @@
return jmxServer;
}
+ @SuppressWarnings("resource")
+ public static JMXConnectorServer createJMXServer(int port, boolean local) throws IOException
+ {
+ return createJMXServer(port, null, local);
+ }
+
private static Map<String, Object> configureJmxAuthentication()
{
Map<String, Object> env = new HashMap<>();
@@ -239,7 +249,7 @@
String hostName;
if (serverAddress == null)
{
- hostName = FBUtilities.getBroadcastAddress() instanceof Inet6Address ? "[::]" : "0.0.0.0";
+ hostName = FBUtilities.getJustBroadcastAddress() instanceof Inet6Address ? "[::]" : "0.0.0.0";
}
else
{
@@ -254,18 +264,34 @@
private static void logJmxSslConfig(SslRMIServerSocketFactory serverFactory)
{
- logger.debug("JMX SSL configuration. { protocols: [{}], cipher_suites: [{}], require_client_auth: {} }",
- serverFactory.getEnabledProtocols() == null
- ? "'JVM defaults'"
- : Arrays.stream(serverFactory.getEnabledProtocols()).collect(Collectors.joining("','", "'", "'")),
- serverFactory.getEnabledCipherSuites() == null
- ? "'JVM defaults'"
- : Arrays.stream(serverFactory.getEnabledCipherSuites()).collect(Collectors.joining("','", "'", "'")),
- serverFactory.getNeedClientAuth());
+ if (logger.isDebugEnabled())
+ logger.debug("JMX SSL configuration. { protocols: [{}], cipher_suites: [{}], require_client_auth: {} }",
+ serverFactory.getEnabledProtocols() == null
+ ? "'JVM defaults'"
+ : Arrays.stream(serverFactory.getEnabledProtocols()).collect(Collectors.joining("','", "'", "'")),
+ serverFactory.getEnabledCipherSuites() == null
+ ? "'JVM defaults'"
+ : Arrays.stream(serverFactory.getEnabledCipherSuites()).collect(Collectors.joining("','", "'", "'")),
+ serverFactory.getNeedClientAuth());
}
private static class JMXPluggableAuthenticatorWrapper implements JMXAuthenticator
{
+ private static final MethodHandle ctorHandle;
+ static
+ {
+ try
+ {
+ Class c = Class.forName("com.sun.jmx.remote.security.JMXPluggableAuthenticator");
+ Constructor ctor = c.getDeclaredConstructor(Map.class);
+ ctorHandle = MethodHandles.lookup().unreflectConstructor(ctor);
+ }
+ catch (Exception e)
+ {
+ throw new RuntimeException(e);
+ }
+ }
+
final Map<?, ?> env;
private JMXPluggableAuthenticatorWrapper(Map<?, ?> env)
{
@@ -274,8 +300,15 @@
public Subject authenticate(Object credentials)
{
- JMXPluggableAuthenticator authenticator = new JMXPluggableAuthenticator(env);
- return authenticator.authenticate(credentials);
+ try
+ {
+ JMXAuthenticator authenticator = (JMXAuthenticator) ctorHandle.invoke(env);
+ return authenticator.authenticate(credentials);
+ }
+ catch (Throwable e)
+ {
+ throw new RuntimeException(e);
+ }
}
}
diff --git a/src/java/org/apache/cassandra/utils/JVMStabilityInspector.java b/src/java/org/apache/cassandra/utils/JVMStabilityInspector.java
index e058ae2..64403e7 100644
--- a/src/java/org/apache/cassandra/utils/JVMStabilityInspector.java
+++ b/src/java/org/apache/cassandra/utils/JVMStabilityInspector.java
@@ -45,6 +45,9 @@
private static Object lock = new Object();
private static boolean printingHeapHistogram;
+ // It is used for unit test
+ public static OnKillHook killerHook;
+
private JVMStabilityInspector() {}
/**
@@ -53,7 +56,12 @@
* @param t
* The Throwable to check for server-stop conditions
*/
- public static void inspectThrowable(Throwable t)
+ public static void inspectThrowable(Throwable t) throws OutOfMemoryError
+ {
+ inspectThrowable(t, true);
+ }
+
+ public static void inspectThrowable(Throwable t, boolean propagateOutOfMemory) throws OutOfMemoryError
{
boolean isUnstable = false;
if (t instanceof OutOfMemoryError)
@@ -76,6 +84,9 @@
StorageService.instance.removeShutdownHook();
// We let the JVM handle the error. The startup checks should have warned the user if it did not configure
// the JVM behavior in case of OOM (CASSANDRA-13006).
+ if (!propagateOutOfMemory)
+ return;
+
throw (OutOfMemoryError) t;
}
@@ -161,11 +172,26 @@
t.printStackTrace(System.err);
logger.error("JVM state determined to be unstable. Exiting forcefully due to:", t);
}
- if (killing.compareAndSet(false, true))
+
+ boolean doExit = killerHook != null ? killerHook.execute(t) : true;
+
+ if (doExit && killing.compareAndSet(false, true))
{
StorageService.instance.removeShutdownHook();
System.exit(100);
}
}
}
+
+ /**
+ * This class is usually used to avoid JVM exit when running junit tests.
+ */
+ public interface OnKillHook
+ {
+ /**
+ *
+ * @return False will skip exit
+ */
+ boolean execute(Throwable t);
+ }
}
diff --git a/src/java/org/apache/cassandra/utils/LongAccumulator.java b/src/java/org/apache/cassandra/utils/LongAccumulator.java
new file mode 100644
index 0000000..fe3c195
--- /dev/null
+++ b/src/java/org/apache/cassandra/utils/LongAccumulator.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.utils;
+
+public interface LongAccumulator<T>
+{
+ long apply(T obj, long v);
+}
diff --git a/src/java/org/apache/cassandra/utils/MD5Digest.java b/src/java/org/apache/cassandra/utils/MD5Digest.java
index 4e736dc..d542991 100644
--- a/src/java/org/apache/cassandra/utils/MD5Digest.java
+++ b/src/java/org/apache/cassandra/utils/MD5Digest.java
@@ -19,9 +19,9 @@
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
+import java.security.MessageDigest;
import java.util.Arrays;
-
/**
* The result of the computation of an MD5 digest.
*
@@ -32,6 +32,32 @@
*/
public class MD5Digest
{
+ /**
+ * In the interest not breaking things, we're consciously keeping this single remaining instance
+ * of MessageDigest around for usage by GuidGenerator (which is only ever used by RandomPartitioner)
+ * and some client native transport methods, where we're tied to the usage of MD5 in the protocol.
+ * As RandomPartitioner will always be MD5 and cannot be changed, we can switch over all our
+ * other digest usage to Guava's Hasher to make switching the hashing function used during message
+ * digests etc possible, but not regress on performance or bugs in RandomPartitioner's usage of
+ * MD5 and MessageDigest.
+ */
+ private static final ThreadLocal<MessageDigest> localMD5Digest = new ThreadLocal<MessageDigest>()
+ {
+ @Override
+ protected MessageDigest initialValue()
+ {
+ return FBUtilities.newMessageDigest("MD5");
+ }
+
+ @Override
+ public MessageDigest get()
+ {
+ MessageDigest digest = super.get();
+ digest.reset();
+ return digest;
+ }
+ };
+
public final byte[] bytes;
private final int hashCode;
@@ -48,7 +74,7 @@
public static MD5Digest compute(byte[] toHash)
{
- return new MD5Digest(FBUtilities.threadLocalMD5Digest().digest(toHash));
+ return new MD5Digest(localMD5Digest.get().digest(toHash));
}
public static MD5Digest compute(String toHash)
@@ -82,4 +108,9 @@
{
return Hex.bytesToHex(bytes);
}
+
+ public static MessageDigest threadLocalMD5Digest()
+ {
+ return localMD5Digest.get();
+ }
}
diff --git a/src/java/org/apache/cassandra/utils/MergeIterator.java b/src/java/org/apache/cassandra/utils/MergeIterator.java
index c9e445b..6713dd0 100644
--- a/src/java/org/apache/cassandra/utils/MergeIterator.java
+++ b/src/java/org/apache/cassandra/utils/MergeIterator.java
@@ -31,6 +31,7 @@
this.reducer = reducer;
}
+ @SuppressWarnings("resource")
public static <In, Out> MergeIterator<In, Out> get(List<? extends Iterator<In>> sources,
Comparator<? super In> comparator,
Reducer<In, Out> reducer)
@@ -51,8 +52,9 @@
public void close()
{
- for (Iterator<In> iterator : this.iterators)
+ for (int i=0, isize=iterators.size(); i<isize; i++)
{
+ Iterator<In> iterator = iterators.get(i);
try
{
if (iterator instanceof AutoCloseable)
diff --git a/src/java/org/apache/cassandra/utils/MerkleTree.java b/src/java/org/apache/cassandra/utils/MerkleTree.java
index 9572a27..1b92555 100644
--- a/src/java/org/apache/cassandra/utils/MerkleTree.java
+++ b/src/java/org/apache/cassandra/utils/MerkleTree.java
@@ -19,26 +19,35 @@
import java.io.DataInput;
import java.io.IOException;
-import java.io.Serializable;
+import java.nio.ByteBuffer;
import java.util.*;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.collect.PeekingIterator;
+import com.google.common.primitives.Ints;
+import com.google.common.primitives.Shorts;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.cassandra.db.TypeSizes;
+import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.dht.IPartitioner;
-import org.apache.cassandra.dht.IPartitionerDependentSerializer;
+import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.dht.RandomPartitioner;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.io.IVersionedSerializer;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
-import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.utils.concurrent.Ref;
+import org.apache.cassandra.utils.memory.MemoryUtil;
+
+import static java.lang.String.format;
+import static org.apache.cassandra.db.TypeSizes.sizeof;
+import static org.apache.cassandra.utils.ByteBufferUtil.compare;
+import static org.apache.cassandra.utils.MerkleTree.Difference.*;
/**
* A MerkleTree implemented as a binary tree.
@@ -60,85 +69,34 @@
* If two MerkleTrees have the same hashdepth, they represent a perfect tree
* of the same depth, and can always be compared, regardless of size or splits.
*/
-public class MerkleTree implements Serializable
+public class MerkleTree
{
- private static Logger logger = LoggerFactory.getLogger(MerkleTree.class);
+ private static final Logger logger = LoggerFactory.getLogger(MerkleTree.class);
- public static final MerkleTreeSerializer serializer = new MerkleTreeSerializer();
- private static final long serialVersionUID = 2L;
+ private static final int HASH_SIZE = 32; // 2xMM3_128 = 32 bytes.
+ private static final byte[] EMPTY_HASH = new byte[HASH_SIZE];
+
+ /*
+ * Thread-local byte array, large enough to host 32B of digest or MM3/Random partitoners' tokens
+ */
+ private static final ThreadLocal<byte[]> byteArray = ThreadLocal.withInitial(() -> new byte[HASH_SIZE]);
+
+ private static byte[] getTempArray(int minimumSize)
+ {
+ return minimumSize <= HASH_SIZE ? byteArray.get() : new byte[minimumSize];
+ }
public static final byte RECOMMENDED_DEPTH = Byte.MAX_VALUE - 1;
- public static final int CONSISTENT = 0;
- public static final int FULLY_INCONSISTENT = 1;
- public static final int PARTIALLY_INCONSISTENT = 2;
- private static final byte[] EMPTY_HASH = new byte[0];
-
- public final byte hashdepth;
+ private final int hashdepth;
/** The top level range that this MerkleTree covers. */
- public final Range<Token> fullRange;
+ final Range<Token> fullRange;
private final IPartitioner partitioner;
private long maxsize;
private long size;
- private Hashable root;
-
- public static class MerkleTreeSerializer implements IVersionedSerializer<MerkleTree>
- {
- public void serialize(MerkleTree mt, DataOutputPlus out, int version) throws IOException
- {
- out.writeByte(mt.hashdepth);
- out.writeLong(mt.maxsize);
- out.writeLong(mt.size);
- out.writeUTF(mt.partitioner.getClass().getCanonicalName());
- // full range
- Token.serializer.serialize(mt.fullRange.left, out, version);
- Token.serializer.serialize(mt.fullRange.right, out, version);
- Hashable.serializer.serialize(mt.root, out, version);
- }
-
- public MerkleTree deserialize(DataInputPlus in, int version) throws IOException
- {
- byte hashdepth = in.readByte();
- long maxsize = in.readLong();
- long size = in.readLong();
- IPartitioner partitioner;
- try
- {
- partitioner = FBUtilities.newPartitioner(in.readUTF());
- }
- catch (ConfigurationException e)
- {
- throw new IOException(e);
- }
-
- // full range
- Token left = Token.serializer.deserialize(in, partitioner, version);
- Token right = Token.serializer.deserialize(in, partitioner, version);
- Range<Token> fullRange = new Range<>(left, right);
-
- MerkleTree mt = new MerkleTree(partitioner, fullRange, hashdepth, maxsize);
- mt.size = size;
- mt.root = Hashable.serializer.deserialize(in, partitioner, version);
- return mt;
- }
-
- public long serializedSize(MerkleTree mt, int version)
- {
- long size = 1 // mt.hashdepth
- + TypeSizes.sizeof(mt.maxsize)
- + TypeSizes.sizeof(mt.size)
- + TypeSizes.sizeof(mt.partitioner.getClass().getCanonicalName());
-
- // full range
- size += Token.serializer.serializedSize(mt.fullRange.left, version);
- size += Token.serializer.serializedSize(mt.fullRange.right, version);
-
- size += Hashable.serializer.serializedSize(mt.root, version);
- return size;
- }
- }
+ private Node root;
/**
* @param partitioner The partitioner in use.
@@ -147,59 +105,67 @@
* of the key space covered by each subrange of a fully populated tree.
* @param maxsize The maximum number of subranges in the tree.
*/
- public MerkleTree(IPartitioner partitioner, Range<Token> range, byte hashdepth, long maxsize)
+ public MerkleTree(IPartitioner partitioner, Range<Token> range, int hashdepth, long maxsize)
+ {
+ this(new OnHeapLeaf(), partitioner, range, hashdepth, maxsize, 1);
+ }
+
+ /**
+ * @param partitioner The partitioner in use.
+ * @param range the range this tree covers
+ * @param hashdepth The maximum depth of the tree. 100/(2^depth) is the %
+ * of the key space covered by each subrange of a fully populated tree.
+ * @param maxsize The maximum number of subranges in the tree.
+ * @param size The size of the tree. Typically 1, unless deserilized from an existing tree
+ */
+ private MerkleTree(Node root, IPartitioner partitioner, Range<Token> range, int hashdepth, long maxsize, long size)
{
assert hashdepth < Byte.MAX_VALUE;
+
+ this.root = root;
this.fullRange = Preconditions.checkNotNull(range);
this.partitioner = Preconditions.checkNotNull(partitioner);
this.hashdepth = hashdepth;
this.maxsize = maxsize;
-
- size = 1;
- root = new Leaf(null);
- }
-
-
- static byte inc(byte in)
- {
- assert in < Byte.MAX_VALUE;
- return (byte)(in + 1);
+ this.size = size;
}
/**
* Initializes this tree by splitting it until hashdepth is reached,
* or until an additional level of splits would violate maxsize.
*
- * NB: Replaces all nodes in the tree.
+ * NB: Replaces all nodes in the tree, and always builds on the heap
*/
public void init()
{
// determine the depth to which we can safely split the tree
- byte sizedepth = (byte)(Math.log10(maxsize) / Math.log10(2));
- byte depth = (byte)Math.min(sizedepth, hashdepth);
+ int sizedepth = (int) (Math.log10(maxsize) / Math.log10(2));
+ int depth = Math.min(sizedepth, hashdepth);
- root = initHelper(fullRange.left, fullRange.right, (byte)0, depth);
- size = (long)Math.pow(2, depth);
+ root = initHelper(fullRange.left, fullRange.right, 0, depth);
+ size = (long) Math.pow(2, depth);
}
- private Hashable initHelper(Token left, Token right, byte depth, byte max)
+ private OnHeapNode initHelper(Token left, Token right, int depth, int max)
{
if (depth == max)
// we've reached the leaves
- return new Leaf();
+ return new OnHeapLeaf();
Token midpoint = partitioner.midpoint(left, right);
if (midpoint.equals(left) || midpoint.equals(right))
- return new Leaf();
+ return new OnHeapLeaf();
- Hashable lchild = initHelper(left, midpoint, inc(depth), max);
- Hashable rchild = initHelper(midpoint, right, inc(depth), max);
- return new Inner(midpoint, lchild, rchild);
+ OnHeapNode leftChild = initHelper(left, midpoint, depth + 1, max);
+ OnHeapNode rightChild = initHelper(midpoint, right, depth + 1, max);
+ return new OnHeapInner(midpoint, leftChild, rightChild);
}
- Hashable root()
+ public void release()
{
- return root;
+ if (root instanceof OffHeapNode)
+ ((OffHeapNode) root).release();
+ root = null;
}
public IPartitioner partitioner()
@@ -234,20 +200,21 @@
public static List<TreeRange> difference(MerkleTree ltree, MerkleTree rtree)
{
if (!ltree.fullRange.equals(rtree.fullRange))
- throw new IllegalArgumentException("Difference only make sense on tree covering the same range (but " + ltree.fullRange + " != " + rtree.fullRange + ")");
+ throw new IllegalArgumentException("Difference only make sense on tree covering the same range (but " + ltree.fullRange + " != " + rtree.fullRange + ')');
+
+ // ensure on-heap trees' inner node hashes have been computed
+ ltree.fillInnerHashes();
+ rtree.fillInnerHashes();
List<TreeRange> diff = new ArrayList<>();
- TreeDifference active = new TreeDifference(ltree.fullRange.left, ltree.fullRange.right, (byte)0);
+ TreeRange active = new TreeRange(ltree.fullRange.left, ltree.fullRange.right, 0);
- Hashable lnode = ltree.find(active);
- Hashable rnode = rtree.find(active);
- byte[] lhash = lnode.hash();
- byte[] rhash = rnode.hash();
- active.setSize(lnode.sizeOfRange(), rnode.sizeOfRange());
+ Node lnode = ltree.root;
+ Node rnode = rtree.root;
- if (lhash != null && rhash != null && !Arrays.equals(lhash, rhash))
+ if (lnode.hashesDiffer(rnode))
{
- if(lnode instanceof Leaf || rnode instanceof Leaf)
+ if (lnode instanceof Leaf || rnode instanceof Leaf)
{
logger.debug("Digest mismatch detected among leaf nodes {}, {}", lnode, rnode);
diff.add(active);
@@ -262,20 +229,20 @@
}
}
}
- else if (lhash == null || rhash == null)
- diff.add(active);
+
return diff;
}
+ enum Difference { CONSISTENT, FULLY_INCONSISTENT, PARTIALLY_INCONSISTENT }
+
/**
- * TODO: This function could be optimized into a depth first traversal of
- * the two trees in parallel.
+ * TODO: This function could be optimized into a depth first traversal of the two trees in parallel.
*
* Takes two trees and a range for which they have hashes, but are inconsistent.
* @return FULLY_INCONSISTENT if active is inconsistent, PARTIALLY_INCONSISTENT if only a subrange is inconsistent.
*/
@VisibleForTesting
- static int differenceHelper(MerkleTree ltree, MerkleTree rtree, List<TreeRange> diff, TreeRange active)
+ static Difference differenceHelper(MerkleTree ltree, MerkleTree rtree, List<TreeRange> diff, TreeRange active)
{
if (active.depth == Byte.MAX_VALUE)
return CONSISTENT;
@@ -290,51 +257,46 @@
return FULLY_INCONSISTENT;
}
- TreeDifference left = new TreeDifference(active.left, midpoint, inc(active.depth));
- TreeDifference right = new TreeDifference(midpoint, active.right, inc(active.depth));
+ TreeRange left = new TreeRange(active.left, midpoint, active.depth + 1);
+ TreeRange right = new TreeRange(midpoint, active.right, active.depth + 1);
logger.debug("({}) Hashing sub-ranges [{}, {}] for {} divided by midpoint {}", active.depth, left, right, active, midpoint);
- byte[] lhash, rhash;
- Hashable lnode, rnode;
+ Node lnode, rnode;
// see if we should recurse left
lnode = ltree.find(left);
rnode = rtree.find(left);
- lhash = lnode.hash();
- rhash = rnode.hash();
- left.setSize(lnode.sizeOfRange(), rnode.sizeOfRange());
- left.setRows(lnode.rowsInRange(), rnode.rowsInRange());
- int ldiff = CONSISTENT;
- boolean lreso = lhash != null && rhash != null;
- if (lreso && !Arrays.equals(lhash, rhash))
+ Difference ldiff = CONSISTENT;
+ if (null != lnode && null != rnode && lnode.hashesDiffer(rnode))
{
logger.debug("({}) Inconsistent digest on left sub-range {}: [{}, {}]", active.depth, left, lnode, rnode);
- if (lnode instanceof Leaf) ldiff = FULLY_INCONSISTENT;
- else ldiff = differenceHelper(ltree, rtree, diff, left);
+
+ if (lnode instanceof Leaf)
+ ldiff = FULLY_INCONSISTENT;
+ else
+ ldiff = differenceHelper(ltree, rtree, diff, left);
}
- else if (!lreso)
+ else if (null == lnode || null == rnode)
{
- logger.debug("({}) Left sub-range fully inconsistent {}", active.depth, right);
+ logger.debug("({}) Left sub-range fully inconsistent {}", active.depth, left);
ldiff = FULLY_INCONSISTENT;
}
// see if we should recurse right
lnode = ltree.find(right);
rnode = rtree.find(right);
- lhash = lnode.hash();
- rhash = rnode.hash();
- right.setSize(lnode.sizeOfRange(), rnode.sizeOfRange());
- right.setRows(lnode.rowsInRange(), rnode.rowsInRange());
- int rdiff = CONSISTENT;
- boolean rreso = lhash != null && rhash != null;
- if (rreso && !Arrays.equals(lhash, rhash))
+ Difference rdiff = CONSISTENT;
+ if (null != lnode && null != rnode && lnode.hashesDiffer(rnode))
{
logger.debug("({}) Inconsistent digest on right sub-range {}: [{}, {}]", active.depth, right, lnode, rnode);
- if (rnode instanceof Leaf) rdiff = FULLY_INCONSISTENT;
- else rdiff = differenceHelper(ltree, rtree, diff, right);
+
+ if (rnode instanceof Leaf)
+ rdiff = FULLY_INCONSISTENT;
+ else
+ rdiff = differenceHelper(ltree, rtree, diff, right);
}
- else if (!rreso)
+ else if (null == lnode || null == rnode)
{
logger.debug("({}) Right sub-range fully inconsistent {}", active.depth, right);
rdiff = FULLY_INCONSISTENT;
@@ -363,133 +325,70 @@
}
/**
- * For testing purposes.
- * Gets the smallest range containing the token.
+ * Exceptions that stop recursion early when we are sure that no answer
+ * can be found.
*/
- public TreeRange get(Token t)
+ static abstract class StopRecursion extends Exception
{
- return getHelper(root, fullRange.left, fullRange.right, (byte)0, t);
- }
-
- TreeRange getHelper(Hashable hashable, Token pleft, Token pright, byte depth, Token t)
- {
- while (true)
- {
- if (hashable instanceof Leaf)
- {
- // we've reached a hash: wrap it up and deliver it
- return new TreeRange(this, pleft, pright, depth, hashable);
- }
- // else: node.
-
- Inner node = (Inner) hashable;
- depth = inc(depth);
- if (Range.contains(pleft, node.token, t))
- { // left child contains token
- hashable = node.lchild;
- pright = node.token;
- }
- else
- { // else: right child contains token
- hashable = node.rchild;
- pleft = node.token;
- }
- }
+ static class TooDeep extends StopRecursion {}
+ static class BadRange extends StopRecursion {}
}
/**
- * Invalidates the ranges containing the given token.
- * Useful for testing.
- */
- public void invalidate(Token t)
- {
- invalidateHelper(root, fullRange.left, t);
- }
-
- private void invalidateHelper(Hashable hashable, Token pleft, Token t)
- {
- hashable.hash(null);
- if (hashable instanceof Leaf)
- return;
- // else: node.
-
- Inner node = (Inner)hashable;
- if (Range.contains(pleft, node.token, t))
- // left child contains token
- invalidateHelper(node.lchild, pleft, t);
- else
- // right child contains token
- invalidateHelper(node.rchild, node.token, t);
- }
-
- /**
- * Hash the given range in the tree. The range must have been generated
- * with recursive applications of partitioner.midpoint().
- *
- * NB: Currently does not support wrapping ranges that do not end with
- * partitioner.getMinimumToken().
- *
- * @return Null if any subrange of the range is invalid, or if the exact
- * range cannot be calculated using this tree.
- */
- public byte[] hash(Range<Token> range)
- {
- return find(range).hash();
- }
-
- /**
- * Find the {@link Hashable} node that matches the given {@code range}.
+ * Find the {@link Node} node that matches the given {@code range}.
*
* @param range Range to find
- * @return {@link Hashable} found. If nothing found, return {@link Leaf} with null hash.
+ * @return {@link Node} found. If nothing found, return {@code null}
*/
- private Hashable find(Range<Token> range)
+ @VisibleForTesting
+ private Node find(Range<Token> range)
{
try
{
- return findHelper(root, new Range<Token>(fullRange.left, fullRange.right), range);
+ return findHelper(root, fullRange, range);
}
catch (StopRecursion e)
{
- return new Leaf();
+ return null;
}
}
/**
* @throws StopRecursion If no match could be found for the range.
*/
- private Hashable findHelper(Hashable current, Range<Token> activeRange, Range<Token> find) throws StopRecursion
+ private Node findHelper(Node current, Range<Token> activeRange, Range<Token> find) throws StopRecursion
{
while (true)
{
if (current instanceof Leaf)
{
if (!find.contains(activeRange))
- // we are not fully contained in this range!
- throw new StopRecursion.BadRange();
+ throw new StopRecursion.BadRange(); // we are not fully contained in this range!
+
return current;
}
- // else: node.
- Inner node = (Inner) current;
- Range<Token> leftRange = new Range<>(activeRange.left, node.token);
- Range<Token> rightRange = new Range<>(node.token, activeRange.right);
+ assert current instanceof Inner;
+ Inner inner = (Inner) current;
- if (find.contains(activeRange))
- // this node is fully contained in the range
- return node.calc();
+ if (find.contains(activeRange)) // this node is fully contained in the range
+ return inner.fillInnerHashes();
+
+ Token midpoint = inner.token();
+ Range<Token> leftRange = new Range<>(activeRange.left, midpoint);
+ Range<Token> rightRange = new Range<>(midpoint, activeRange.right);
// else: one of our children contains the range
- if (leftRange.contains(find))
- { // left child contains/matches the range
- current = node.lchild;
+ if (leftRange.contains(find)) // left child contains/matches the range
+ {
activeRange = leftRange;
+ current = inner.left();
}
- else if (rightRange.contains(find))
- { // right child contains/matches the range
- current = node.rchild;
+ else if (rightRange.contains(find)) // right child contains/matches the range
+ {
activeRange = rightRange;
+ current = inner.right();
}
else
{
@@ -507,12 +406,12 @@
*/
public boolean split(Token t)
{
- if (!(size < maxsize))
+ if (size >= maxsize)
return false;
try
{
- root = splitHelper(root, fullRange.left, fullRange.right, (byte)0, t);
+ root = splitHelper(root, fullRange.left, fullRange.right, 0, t);
}
catch (StopRecursion.TooDeep e)
{
@@ -521,12 +420,12 @@
return true;
}
- private Hashable splitHelper(Hashable hashable, Token pleft, Token pright, byte depth, Token t) throws StopRecursion.TooDeep
+ private OnHeapNode splitHelper(Node node, Token pleft, Token pright, int depth, Token t) throws StopRecursion.TooDeep
{
if (depth >= hashdepth)
throw new StopRecursion.TooDeep();
- if (hashable instanceof Leaf)
+ if (node instanceof Leaf)
{
Token midpoint = partitioner.midpoint(pleft, pright);
@@ -537,47 +436,47 @@
// split
size++;
- return new Inner(midpoint, new Leaf(), new Leaf());
+ return new OnHeapInner(midpoint, new OnHeapLeaf(), new OnHeapLeaf());
}
// else: node.
// recurse on the matching child
- Inner node = (Inner)hashable;
+ assert node instanceof OnHeapInner;
+ OnHeapInner inner = (OnHeapInner) node;
- if (Range.contains(pleft, node.token, t))
- // left child contains token
- node.lchild(splitHelper(node.lchild, pleft, node.token, inc(depth), t));
- else
- // else: right child contains token
- node.rchild(splitHelper(node.rchild, node.token, pright, inc(depth), t));
- return node;
+ if (Range.contains(pleft, inner.token(), t)) // left child contains token
+ inner.left(splitHelper(inner.left(), pleft, inner.token(), depth + 1, t));
+ else // else: right child contains token
+ inner.right(splitHelper(inner.right(), inner.token(), pright, depth + 1, t));
+
+ return inner;
}
/**
* Returns a lazy iterator of invalid TreeRanges that need to be filled
* in order to make the given Range valid.
*/
- public TreeRangeIterator invalids()
+ TreeRangeIterator rangeIterator()
{
return new TreeRangeIterator(this);
}
- public EstimatedHistogram histogramOfRowSizePerLeaf()
+ EstimatedHistogram histogramOfRowSizePerLeaf()
{
HistogramBuilder histbuild = new HistogramBuilder();
for (TreeRange range : new TreeRangeIterator(this))
{
- histbuild.add(range.hashable.sizeOfRange);
+ histbuild.add(range.node.sizeOfRange());
}
return histbuild.buildWithStdevRangesAroundMean();
}
- public EstimatedHistogram histogramOfRowCountPerLeaf()
+ EstimatedHistogram histogramOfRowCountPerLeaf()
{
HistogramBuilder histbuild = new HistogramBuilder();
for (TreeRange range : new TreeRangeIterator(this))
{
- histbuild.add(range.hashable.rowsInRange);
+ histbuild.add(range.node.partitionsInRange());
}
return histbuild.buildWithStdevRangesAroundMean();
}
@@ -587,7 +486,7 @@
long count = 0;
for (TreeRange range : new TreeRangeIterator(this))
{
- count += range.hashable.rowsInRange;
+ count += range.node.partitionsInRange();
}
return count;
}
@@ -598,61 +497,23 @@
StringBuilder buff = new StringBuilder();
buff.append("#<MerkleTree root=");
root.toString(buff, 8);
- buff.append(">");
+ buff.append('>');
return buff.toString();
}
- public static class TreeDifference extends TreeRange
+ @Override
+ public boolean equals(Object other)
{
- private static final long serialVersionUID = 6363654174549968183L;
+ if (!(other instanceof MerkleTree))
+ return false;
+ MerkleTree that = (MerkleTree) other;
- private long sizeOnLeft;
- private long sizeOnRight;
- private long rowsOnLeft;
- private long rowsOnRight;
-
- void setSize(long sizeOnLeft, long sizeOnRight)
- {
- this.sizeOnLeft = sizeOnLeft;
- this.sizeOnRight = sizeOnRight;
- }
-
- void setRows(long rowsOnLeft, long rowsOnRight)
- {
- this.rowsOnLeft = rowsOnLeft;
- this.rowsOnRight = rowsOnRight;
- }
-
- public long sizeOnLeft()
- {
- return sizeOnLeft;
- }
-
- public long sizeOnRight()
- {
- return sizeOnRight;
- }
-
- public long rowsOnLeft()
- {
- return rowsOnLeft;
- }
-
- public long rowsOnRight()
- {
- return rowsOnRight;
- }
-
- public TreeDifference(Token left, Token right, byte depth)
- {
- super(null, left, right, depth, null);
- }
-
- public long totalRows()
- {
- return rowsOnLeft + rowsOnRight;
- }
-
+ return this.root.equals(that.root)
+ && this.fullRange.equals(that.fullRange)
+ && this.partitioner == that.partitioner
+ && this.hashdepth == that.hashdepth
+ && this.maxsize == that.maxsize
+ && this.size == that.size;
}
/**
@@ -665,28 +526,27 @@
*/
public static class TreeRange extends Range<Token>
{
- public static final long serialVersionUID = 1L;
private final MerkleTree tree;
- public final byte depth;
- private final Hashable hashable;
+ public final int depth;
+ private final Node node;
- TreeRange(MerkleTree tree, Token left, Token right, byte depth, Hashable hashable)
+ TreeRange(MerkleTree tree, Token left, Token right, int depth, Node node)
{
super(left, right);
this.tree = tree;
this.depth = depth;
- this.hashable = hashable;
+ this.node = node;
+ }
+
+ TreeRange(Token left, Token right, int depth)
+ {
+ this(null, left, right, depth, null);
}
public void hash(byte[] hash)
{
assert tree != null : "Not intended for modification!";
- hashable.hash(hash);
- }
-
- public byte[] hash()
- {
- return hashable.hash();
+ node.hash(hash);
}
/**
@@ -694,33 +554,26 @@
*/
public void addHash(RowHash entry)
{
- assert tree != null : "Not intended for modification!";
- assert hashable instanceof Leaf;
-
- hashable.addHash(entry.hash, entry.size);
+ addHash(entry.hash, entry.size);
}
- public void ensureHashInitialised()
+ void addHash(byte[] hash, long partitionSize)
{
assert tree != null : "Not intended for modification!";
- assert hashable instanceof Leaf;
- if (hashable.hash == null)
- hashable.hash = EMPTY_HASH;
+ assert node instanceof OnHeapLeaf;
+ ((OnHeapLeaf) node).addHash(hash, partitionSize);
}
public void addAll(Iterator<RowHash> entries)
{
- while (entries.hasNext())
- addHash(entries.next());
+ while (entries.hasNext()) addHash(entries.next());
}
@Override
public String toString()
{
- StringBuilder buff = new StringBuilder("#<TreeRange ");
- buff.append(super.toString()).append(" depth=").append(depth);
- return buff.append(">").toString();
+ return "#<TreeRange " + super.toString() + " depth=" + depth + '>';
}
}
@@ -741,8 +594,8 @@
TreeRangeIterator(MerkleTree tree)
{
- tovisit = new ArrayDeque<TreeRange>();
- tovisit.add(new TreeRange(tree, tree.fullRange.left, tree.fullRange.right, (byte)0, tree.root));
+ tovisit = new ArrayDeque<>();
+ tovisit.add(new TreeRange(tree, tree.fullRange.left, tree.fullRange.right, 0, tree.root));
this.tree = tree;
}
@@ -757,7 +610,7 @@
{
TreeRange active = tovisit.pop();
- if (active.hashable instanceof Leaf)
+ if (active.node instanceof Leaf)
{
// found a leaf invalid range
if (active.isWrapAround() && !tovisit.isEmpty())
@@ -766,9 +619,9 @@
return active;
}
- Inner node = (Inner)active.hashable;
- TreeRange left = new TreeRange(tree, active.left, node.token, inc(active.depth), node.lchild);
- TreeRange right = new TreeRange(tree, node.token, active.right, inc(active.depth), node.rchild);
+ Inner node = (Inner)active.node;
+ TreeRange left = new TreeRange(tree, active.left, node.token(), active.depth + 1, node.left());
+ TreeRange right = new TreeRange(tree, node.token(), active.right, active.depth + 1, node.right());
if (right.isWrapAround())
{
@@ -793,149 +646,355 @@
}
/**
- * An inner node in the MerkleTree. Inners can contain cached hash values, which
- * are the binary hash of their two children.
+ * Hash value representing a row, to be used to pass hashes to the MerkleTree.
+ * The byte[] hash value should contain a digest of the key and value of the row
+ * created using a very strong hash function.
*/
- static class Inner extends Hashable
+ public static class RowHash
{
- public static final long serialVersionUID = 1L;
- static final byte IDENT = 2;
public final Token token;
- private Hashable lchild;
- private Hashable rchild;
+ public final byte[] hash;
+ public final long size;
- private static final InnerSerializer serializer = new InnerSerializer();
-
- /**
- * Constructs an Inner with the given token and children, and a null hash.
- */
- public Inner(Token token, Hashable lchild, Hashable rchild)
+ public RowHash(Token token, byte[] hash, long size)
{
- super(null);
this.token = token;
- this.lchild = lchild;
- this.rchild = rchild;
- }
-
- public Hashable lchild()
- {
- return lchild;
- }
-
- public Hashable rchild()
- {
- return rchild;
- }
-
- public void lchild(Hashable child)
- {
- lchild = child;
- }
-
- public void rchild(Hashable child)
- {
- rchild = child;
- }
-
- Hashable calc()
- {
- if (hash == null)
- {
- // hash and size haven't been calculated; calc children then compute
- Hashable lnode = lchild.calc();
- Hashable rnode = rchild.calc();
- // cache the computed value
- hash(lnode.hash, rnode.hash);
- sizeOfRange = lnode.sizeOfRange + rnode.sizeOfRange;
- rowsInRange = lnode.rowsInRange + rnode.rowsInRange;
- }
- return this;
- }
-
- /**
- * Recursive toString.
- */
- public void toString(StringBuilder buff, int maxdepth)
- {
- buff.append("#<").append(getClass().getSimpleName());
- buff.append(" ").append(token);
- buff.append(" hash=").append(Hashable.toString(hash()));
- buff.append(" children=[");
- if (maxdepth < 1)
- {
- buff.append("#");
- }
- else
- {
- if (lchild == null)
- buff.append("null");
- else
- lchild.toString(buff, maxdepth-1);
- buff.append(" ");
- if (rchild == null)
- buff.append("null");
- else
- rchild.toString(buff, maxdepth-1);
- }
- buff.append("]>");
+ this.hash = hash;
+ this.size = size;
}
@Override
public String toString()
{
- StringBuilder buff = new StringBuilder();
- toString(buff, 1);
- return buff.toString();
+ return "#<RowHash " + token + ' ' + (hash == null ? "null" : Hex.bytesToHex(hash)) + " @ " + size + " bytes>";
+ }
+ }
+
+ public void serialize(DataOutputPlus out, int version) throws IOException
+ {
+ out.writeByte(hashdepth);
+ out.writeLong(maxsize);
+ out.writeLong(size);
+ out.writeUTF(partitioner.getClass().getCanonicalName());
+ Token.serializer.serialize(fullRange.left, out, version);
+ Token.serializer.serialize(fullRange.right, out, version);
+ root.serialize(out, version);
+ }
+
+ public long serializedSize(int version)
+ {
+ long size = 1 // mt.hashdepth
+ + sizeof(maxsize)
+ + sizeof(this.size)
+ + sizeof(partitioner.getClass().getCanonicalName());
+ size += Token.serializer.serializedSize(fullRange.left, version);
+ size += Token.serializer.serializedSize(fullRange.right, version);
+ size += root.serializedSize(version);
+ return size;
+ }
+
+ public static MerkleTree deserialize(DataInputPlus in, int version) throws IOException
+ {
+ return deserialize(in, DatabaseDescriptor.useOffheapMerkleTrees(), version);
+ }
+
+ public static MerkleTree deserialize(DataInputPlus in, boolean offHeapRequested, int version) throws IOException
+ {
+ int hashDepth = in.readByte();
+ long maxSize = in.readLong();
+ int innerNodeCount = Ints.checkedCast(in.readLong());
+
+ IPartitioner partitioner;
+ try
+ {
+ partitioner = FBUtilities.newPartitioner(in.readUTF());
+ }
+ catch (ConfigurationException e)
+ {
+ throw new IOException(e);
}
- private static class InnerSerializer implements IPartitionerDependentSerializer<Inner>
+ Token left = Token.serializer.deserialize(in, partitioner, version);
+ Token right = Token.serializer.deserialize(in, partitioner, version);
+ Range<Token> fullRange = new Range<>(left, right);
+ Node root = deserializeTree(in, partitioner, innerNodeCount, offHeapRequested, version);
+ return new MerkleTree(root, partitioner, fullRange, hashDepth, maxSize, innerNodeCount);
+ }
+
+ private static boolean shouldUseOffHeapTrees(IPartitioner partitioner, boolean offHeapRequested)
+ {
+ boolean offHeapSupported = partitioner instanceof Murmur3Partitioner || partitioner instanceof RandomPartitioner;
+
+ if (offHeapRequested && !offHeapSupported && !warnedOnce)
{
- public void serialize(Inner inner, DataOutputPlus out, int version) throws IOException
+ logger.warn("Configuration requests off-heap merkle trees, but partitioner does not support it. Ignoring.");
+ warnedOnce = true;
+ }
+
+ return offHeapRequested && offHeapSupported;
+ }
+ private static boolean warnedOnce;
+
+ private static ByteBuffer allocate(int innerNodeCount, IPartitioner partitioner)
+ {
+ int size = offHeapBufferSize(innerNodeCount, partitioner);
+ logger.debug("Allocating direct buffer of size {} for an off-heap merkle tree", size);
+ ByteBuffer buffer = ByteBuffer.allocateDirect(size);
+ if (Ref.DEBUG_ENABLED)
+ MemoryUtil.setAttachment(buffer, new Ref<>(null, null));
+ return buffer;
+ }
+
+ private static Node deserializeTree(DataInputPlus in, IPartitioner partitioner, int innerNodeCount, boolean offHeapRequested, int version) throws IOException
+ {
+ return shouldUseOffHeapTrees(partitioner, offHeapRequested)
+ ? deserializeOffHeap(in, partitioner, innerNodeCount, version)
+ : OnHeapNode.deserialize(in, partitioner, version);
+ }
+
+ /*
+ * Coordinating multiple trees from multiple replicas can get expensive.
+ * On the deserialization path, we know in advance what the tree looks like,
+ * So we can pre-size an offheap buffer and deserialize into that.
+ */
+ MerkleTree tryMoveOffHeap() throws IOException
+ {
+ return root instanceof OnHeapNode && shouldUseOffHeapTrees(partitioner, DatabaseDescriptor.useOffheapMerkleTrees())
+ ? moveOffHeap()
+ : this;
+ }
+
+ @VisibleForTesting
+ MerkleTree moveOffHeap() throws IOException
+ {
+ assert root instanceof OnHeapNode;
+ root.fillInnerHashes(); // ensure on-heap trees' inner node hashes have been computed
+ ByteBuffer buffer = allocate(Ints.checkedCast(size), partitioner);
+ int pointer = ((OnHeapNode) root).serializeOffHeap(buffer, partitioner);
+ OffHeapNode newRoot = fromPointer(pointer, buffer, partitioner);
+ return new MerkleTree(newRoot, partitioner, fullRange, hashdepth, maxsize, size);
+ }
+
+ private static OffHeapNode deserializeOffHeap(DataInputPlus in, IPartitioner partitioner, int innerNodeCount, int version) throws IOException
+ {
+ ByteBuffer buffer = allocate(innerNodeCount, partitioner);
+ int pointer = OffHeapNode.deserialize(in, buffer, partitioner, version);
+ return fromPointer(pointer, buffer, partitioner);
+ }
+
+ private static OffHeapNode fromPointer(int pointer, ByteBuffer buffer, IPartitioner partitioner)
+ {
+ return pointer >= 0 ? new OffHeapInner(buffer, pointer, partitioner) : new OffHeapLeaf(buffer, ~pointer);
+ }
+
+ private static int offHeapBufferSize(int innerNodeCount, IPartitioner partitioner)
+ {
+ return innerNodeCount * OffHeapInner.maxOffHeapSize(partitioner) + (innerNodeCount + 1) * OffHeapLeaf.maxOffHeapSize();
+ }
+
+ interface Node
+ {
+ byte[] hash();
+
+ boolean hasEmptyHash();
+
+ void hash(byte[] hash);
+
+ boolean hashesDiffer(Node other);
+
+ default Node fillInnerHashes()
+ {
+ return this;
+ }
+
+ default long sizeOfRange()
+ {
+ return 0;
+ }
+
+ default long partitionsInRange()
+ {
+ return 0;
+ }
+
+ void serialize(DataOutputPlus out, int version) throws IOException;
+ int serializedSize(int version);
+
+ void toString(StringBuilder buff, int maxdepth);
+
+ static String toString(byte[] hash)
+ {
+ return hash == null
+ ? "null"
+ : '[' + Hex.bytesToHex(hash) + ']';
+ }
+
+ boolean equals(Node node);
+ }
+
+ static abstract class OnHeapNode implements Node
+ {
+ long sizeOfRange;
+ long partitionsInRange;
+
+ protected byte[] hash;
+
+ OnHeapNode(byte[] hash)
+ {
+ if (hash == null)
+ throw new IllegalArgumentException();
+
+ this.hash = hash;
+ }
+
+ public byte[] hash()
+ {
+ return hash;
+ }
+
+ public boolean hasEmptyHash()
+ {
+ //noinspection ArrayEquality
+ return hash == EMPTY_HASH;
+ }
+
+ public void hash(byte[] hash)
+ {
+ if (hash == null)
+ throw new IllegalArgumentException();
+
+ this.hash = hash;
+ }
+
+ public boolean hashesDiffer(Node other)
+ {
+ return other instanceof OnHeapNode
+ ? hashesDiffer( (OnHeapNode) other)
+ : hashesDiffer((OffHeapNode) other);
+ }
+
+ private boolean hashesDiffer(OnHeapNode other)
+ {
+ return !Arrays.equals(hash(), other.hash());
+ }
+
+ private boolean hashesDiffer(OffHeapNode other)
+ {
+ return compare(hash(), other.buffer(), other.hashBytesOffset(), HASH_SIZE) != 0;
+ }
+
+ @Override
+ public long sizeOfRange()
+ {
+ return sizeOfRange;
+ }
+
+ @Override
+ public long partitionsInRange()
+ {
+ return partitionsInRange;
+ }
+
+ static OnHeapNode deserialize(DataInputPlus in, IPartitioner p, int version) throws IOException
+ {
+ byte ident = in.readByte();
+
+ switch (ident)
{
- if (version < MessagingService.VERSION_30)
- {
- if (inner.hash == null)
- out.writeInt(-1);
- else
- {
- out.writeInt(inner.hash.length);
- out.write(inner.hash);
- }
- }
- Token.serializer.serialize(inner.token, out, version);
- Hashable.serializer.serialize(inner.lchild, out, version);
- Hashable.serializer.serialize(inner.rchild, out, version);
+ case Inner.IDENT:
+ return OnHeapInner.deserializeWithoutIdent(in, p, version);
+ case Leaf.IDENT:
+ return OnHeapLeaf.deserializeWithoutIdent(in);
+ default:
+ throw new IOException("Unexpected node type: " + ident);
}
+ }
- public Inner deserialize(DataInput in, IPartitioner p, int version) throws IOException
+ abstract int serializeOffHeap(ByteBuffer buffer, IPartitioner p) throws IOException;
+ }
+
+ static abstract class OffHeapNode implements Node
+ {
+ protected final ByteBuffer buffer;
+ protected final int offset;
+
+ OffHeapNode(ByteBuffer buffer, int offset)
+ {
+ this.buffer = buffer;
+ this.offset = offset;
+ }
+
+ ByteBuffer buffer()
+ {
+ return buffer;
+ }
+
+ public byte[] hash()
+ {
+ final int position = buffer.position();
+ buffer.position(hashBytesOffset());
+ byte[] array = new byte[HASH_SIZE];
+ buffer.get(array);
+ buffer.position(position);
+ return array;
+ }
+
+ public boolean hasEmptyHash()
+ {
+ return compare(buffer(), hashBytesOffset(), HASH_SIZE, EMPTY_HASH) == 0;
+ }
+
+ public void hash(byte[] hash)
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ public boolean hashesDiffer(Node other)
+ {
+ return other instanceof OnHeapNode
+ ? hashesDiffer((OnHeapNode) other)
+ : hashesDiffer((OffHeapNode) other);
+ }
+
+ private boolean hashesDiffer(OnHeapNode other)
+ {
+ return compare(buffer(), hashBytesOffset(), HASH_SIZE, other.hash()) != 0;
+ }
+
+ private boolean hashesDiffer(OffHeapNode other)
+ {
+ int thisOffset = hashBytesOffset();
+ int otherOffset = other.hashBytesOffset();
+
+ for (int i = 0; i < HASH_SIZE; i += 8)
+ if (buffer().getLong(thisOffset + i) != other.buffer().getLong(otherOffset + i))
+ return true;
+
+ return false;
+ }
+
+ void release()
+ {
+ Object attachment = MemoryUtil.getAttachment(buffer);
+ if (attachment instanceof Ref)
+ ((Ref) attachment).release();
+ FileUtils.clean(buffer);
+ }
+
+ abstract int hashBytesOffset();
+
+ static int deserialize(DataInputPlus in, ByteBuffer buffer, IPartitioner p, int version) throws IOException
+ {
+ byte ident = in.readByte();
+
+ switch (ident)
{
- if (version < MessagingService.VERSION_30)
- {
- int hashLen = in.readInt();
- byte[] hash = hashLen >= 0 ? new byte[hashLen] : null;
- if (hash != null)
- in.readFully(hash);
- }
- Token token = Token.serializer.deserialize(in, p, version);
- Hashable lchild = Hashable.serializer.deserialize(in, p, version);
- Hashable rchild = Hashable.serializer.deserialize(in, p, version);
- return new Inner(token, lchild, rchild);
- }
-
- public long serializedSize(Inner inner, int version)
- {
- long size = 0;
- if (version < MessagingService.VERSION_30)
- {
- size += inner.hash == null
- ? TypeSizes.sizeof(-1)
- : TypeSizes.sizeof(inner.hash().length) + inner.hash().length;
- }
-
- size += Token.serializer.serializedSize(inner.token, version)
- + Hashable.serializer.serializedSize(inner.lchild, version)
- + Hashable.serializer.serializedSize(inner.rchild, version);
- return size;
+ case Inner.IDENT:
+ return OffHeapInner.deserializeWithoutIdent(in, buffer, p, version);
+ case Leaf.IDENT:
+ return OffHeapLeaf.deserializeWithoutIdent(in, buffer);
+ default:
+ throw new IOException("Unexpected node type: " + ident);
}
}
}
@@ -949,245 +1008,646 @@
* tree extending below the Leaf is generated in memory, but only the root
* is stored in the Leaf.
*/
- static class Leaf extends Hashable
+ interface Leaf extends Node
{
- public static final long serialVersionUID = 1L;
static final byte IDENT = 1;
- private static final LeafSerializer serializer = new LeafSerializer();
- /**
- * Constructs a null hash.
- */
- public Leaf()
+ default void serialize(DataOutputPlus out, int version) throws IOException
{
- super(null);
+ byte[] hash = hash();
+ assert hash.length == HASH_SIZE;
+
+ out.writeByte(Leaf.IDENT);
+
+ if (!hasEmptyHash())
+ {
+ out.writeByte(HASH_SIZE);
+ out.write(hash);
+ }
+ else
+ {
+ out.writeByte(0);
+ }
}
- public Leaf(byte[] hash)
+ default int serializedSize(int version)
{
- super(hash);
+ return 2 + (hasEmptyHash() ? 0 : HASH_SIZE);
}
- public void toString(StringBuilder buff, int maxdepth)
+ default void toString(StringBuilder buff, int maxdepth)
{
buff.append(toString());
}
- @Override
- public String toString()
+ default boolean equals(Node other)
{
- return "#<Leaf " + Hashable.toString(hash()) + ">";
- }
-
- private static class LeafSerializer implements IPartitionerDependentSerializer<Leaf>
- {
- public void serialize(Leaf leaf, DataOutputPlus out, int version) throws IOException
- {
- if (leaf.hash == null)
- {
- if (version < MessagingService.VERSION_30)
- out.writeInt(-1);
- else
- out.writeByte(-1);
- }
- else
- {
- if (version < MessagingService.VERSION_30)
- out.writeInt(leaf.hash.length);
- else
- out.writeByte(leaf.hash.length);
- out.write(leaf.hash);
- }
- }
-
- public Leaf deserialize(DataInput in, IPartitioner p, int version) throws IOException
- {
- int hashLen = version < MessagingService.VERSION_30 ? in.readInt() : in.readByte();
- byte[] hash = hashLen < 0 ? null : new byte[hashLen];
- if (hash != null)
- in.readFully(hash);
- return new Leaf(hash);
- }
-
- public long serializedSize(Leaf leaf, int version)
- {
- long size = version < MessagingService.VERSION_30 ? TypeSizes.sizeof(1) : 1;
- if (leaf.hash != null)
- {
- size += leaf.hash().length;
- }
- return size;
- }
+ return other instanceof Leaf && !hashesDiffer(other);
}
}
- /**
- * Hash value representing a row, to be used to pass hashes to the MerkleTree.
- * The byte[] hash value should contain a digest of the key and value of the row
- * created using a very strong hash function.
- */
- public static class RowHash
+ static class OnHeapLeaf extends OnHeapNode implements Leaf
{
- public final Token token;
- public final byte[] hash;
- public final long size;
- public RowHash(Token token, byte[] hash, long size)
+ OnHeapLeaf()
{
- this.token = token;
- this.hash = hash;
- this.size = size;
+ super(EMPTY_HASH);
}
- @Override
- public String toString()
+ OnHeapLeaf(byte[] hash)
{
- return "#<RowHash " + token + " " + Hashable.toString(hash) + " @ " + size + " bytes>";
- }
- }
-
- /**
- * Abstract class containing hashing logic, and containing a single hash field.
- */
- static abstract class Hashable implements Serializable
- {
- private static final long serialVersionUID = 1L;
- private static final IPartitionerDependentSerializer<Hashable> serializer = new HashableSerializer();
-
- protected byte[] hash;
- protected long sizeOfRange;
- protected long rowsInRange;
-
- protected Hashable(byte[] hash)
- {
- this.hash = hash;
- }
-
- public byte[] hash()
- {
- return hash;
- }
-
- public long sizeOfRange()
- {
- return sizeOfRange;
- }
-
- public long rowsInRange()
- {
- return rowsInRange;
- }
-
- void hash(byte[] hash)
- {
- this.hash = hash;
- }
-
- Hashable calc()
- {
- return this;
- }
-
- /**
- * Sets the value of this hash to binaryHash of its children.
- * @param lefthash Hash of left child.
- * @param righthash Hash of right child.
- */
- void hash(byte[] lefthash, byte[] righthash)
- {
- hash = binaryHash(lefthash, righthash);
+ super(hash);
}
/**
* Mixes the given value into our hash. If our hash is null,
* our hash will become the given value.
*/
- void addHash(byte[] righthash, long sizeOfRow)
+ void addHash(byte[] partitionHash, long partitionSize)
{
- if (hash == null)
- hash = righthash;
+ if (hasEmptyHash())
+ hash(partitionHash);
else
- hash = binaryHash(hash, righthash);
- this.sizeOfRange += sizeOfRow;
- this.rowsInRange += 1;
+ xorOntoLeft(hash, partitionHash);
+
+ sizeOfRange += partitionSize;
+ partitionsInRange += 1;
}
- /**
- * The primitive with which all hashing should be accomplished: hashes
- * a left and right value together.
- */
- static byte[] binaryHash(final byte[] left, final byte[] right)
+ static OnHeapLeaf deserializeWithoutIdent(DataInputPlus in) throws IOException
{
- return FBUtilities.xor(left, right);
+ int size = in.readByte();
+ switch (size)
+ {
+ case HASH_SIZE:
+ byte[] hash = new byte[HASH_SIZE];
+ in.readFully(hash);
+ return new OnHeapLeaf(hash);
+ case 0:
+ return new OnHeapLeaf();
+ default:
+ throw new IllegalStateException(format("Hash of size %d encountered, expecting %d or %d", size, HASH_SIZE, 0));
+ }
}
- public abstract void toString(StringBuilder buff, int maxdepth);
-
- public static String toString(byte[] hash)
+ int serializeOffHeap(ByteBuffer buffer, IPartitioner p)
{
- if (hash == null)
- return "null";
- return "[" + Hex.bytesToHex(hash) + "]";
+ if (buffer.remaining() < OffHeapLeaf.maxOffHeapSize())
+ throw new IllegalStateException("Insufficient remaining bytes to deserialize a Leaf node off-heap");
+
+ if (hash.length != HASH_SIZE)
+ throw new IllegalArgumentException("Hash of unexpected size when serializing a Leaf off-heap: " + hash.length);
+
+ final int position = buffer.position();
+ buffer.put(hash);
+ return ~position;
}
- private static class HashableSerializer implements IPartitionerDependentSerializer<Hashable>
+ @Override
+ public String toString()
{
- public void serialize(Hashable h, DataOutputPlus out, int version) throws IOException
+ return "#<OnHeapLeaf " + Node.toString(hash()) + '>';
+ }
+ }
+
+ static class OffHeapLeaf extends OffHeapNode implements Leaf
+ {
+ static final int HASH_BYTES_OFFSET = 0;
+
+ OffHeapLeaf(ByteBuffer buffer, int offset)
+ {
+ super(buffer, offset);
+ }
+
+ public int hashBytesOffset()
+ {
+ return offset + HASH_BYTES_OFFSET;
+ }
+
+ static int deserializeWithoutIdent(DataInput in, ByteBuffer buffer) throws IOException
+ {
+ if (buffer.remaining() < maxOffHeapSize())
+ throw new IllegalStateException("Insufficient remaining bytes to deserialize a Leaf node off-heap");
+
+ final int position = buffer.position();
+
+ int hashLength = in.readByte();
+ if (hashLength > 0)
{
- if (h instanceof Inner)
- {
- out.writeByte(Inner.IDENT);
- Inner.serializer.serialize((Inner)h, out, version);
- }
- else if (h instanceof Leaf)
- {
- out.writeByte(Leaf.IDENT);
- Leaf.serializer.serialize((Leaf) h, out, version);
- }
- else
- throw new IOException("Unexpected Hashable: " + h.getClass().getCanonicalName());
+ if (hashLength != HASH_SIZE)
+ throw new IllegalStateException("Hash of unexpected size when deserializing an off-heap Leaf node: " + hashLength);
+
+ byte[] hashBytes = getTempArray(HASH_SIZE);
+ in.readFully(hashBytes, 0, HASH_SIZE);
+ buffer.put(hashBytes, 0, HASH_SIZE);
+ }
+ else
+ {
+ buffer.put(EMPTY_HASH, 0, HASH_SIZE);
}
- public Hashable deserialize(DataInput in, IPartitioner p, int version) throws IOException
- {
- byte ident = in.readByte();
- if (Inner.IDENT == ident)
- return Inner.serializer.deserialize(in, p, version);
- else if (Leaf.IDENT == ident)
- return Leaf.serializer.deserialize(in, p, version);
- else
- throw new IOException("Unexpected Hashable: " + ident);
- }
+ return ~position;
+ }
- public long serializedSize(Hashable h, int version)
- {
- if (h instanceof Inner)
- return 1 + Inner.serializer.serializedSize((Inner) h, version);
- else if (h instanceof Leaf)
- return 1 + Leaf.serializer.serializedSize((Leaf) h, version);
- throw new AssertionError(h.getClass());
- }
+ static int maxOffHeapSize()
+ {
+ return HASH_SIZE;
+ }
+
+ @Override
+ public String toString()
+ {
+ return "#<OffHeapLeaf " + Node.toString(hash()) + '>';
}
}
/**
- * Exceptions that stop recursion early when we are sure that no answer
- * can be found.
+ * An inner node in the MerkleTree. Inners can contain cached hash values, which
+ * are the binary hash of their two children.
*/
- static abstract class StopRecursion extends Exception
+ interface Inner extends Node
{
- static class BadRange extends StopRecursion
+ static final byte IDENT = 2;
+
+ public Token token();
+
+ public Node left();
+ public Node right();
+
+ default void serialize(DataOutputPlus out, int version) throws IOException
{
- public BadRange(){ super(); }
+ out.writeByte(Inner.IDENT);
+ Token.serializer.serialize(token(), out, version);
+ left().serialize(out, version);
+ right().serialize(out, version);
}
- static class InvalidHash extends StopRecursion
+ default int serializedSize(int version)
{
- public InvalidHash(){ super(); }
+ return 1
+ + (int) Token.serializer.serializedSize(token(), version)
+ + left().serializedSize(version)
+ + right().serializedSize(version);
}
- static class TooDeep extends StopRecursion
+ default void toString(StringBuilder buff, int maxdepth)
{
- public TooDeep(){ super(); }
+ buff.append("#<").append(getClass().getSimpleName())
+ .append(' ').append(token())
+ .append(" hash=").append(Node.toString(hash()))
+ .append(" children=[");
+
+ if (maxdepth < 1)
+ {
+ buff.append('#');
+ }
+ else
+ {
+ Node left = left();
+ if (left == null)
+ buff.append("null");
+ else
+ left.toString(buff, maxdepth - 1);
+
+ buff.append(' ');
+
+ Node right = right();
+ if (right == null)
+ buff.append("null");
+ else
+ right.toString(buff, maxdepth - 1);
+ }
+
+ buff.append("]>");
}
+
+ default boolean equals(Node other)
+ {
+ if (!(other instanceof Inner))
+ return false;
+ Inner that = (Inner) other;
+ return !hashesDiffer(other) && this.left().equals(that.left()) && this.right().equals(that.right());
+ }
+
+ default void unsafeInvalidate()
+ {
+ }
+ }
+
+ static class OnHeapInner extends OnHeapNode implements Inner
+ {
+ private final Token token;
+
+ private OnHeapNode left;
+ private OnHeapNode right;
+
+ private boolean computed;
+
+ OnHeapInner(Token token, OnHeapNode left, OnHeapNode right)
+ {
+ super(EMPTY_HASH);
+
+ this.token = token;
+ this.left = left;
+ this.right = right;
+ }
+
+ public Token token()
+ {
+ return token;
+ }
+
+ public OnHeapNode left()
+ {
+ return left;
+ }
+
+ public OnHeapNode right()
+ {
+ return right;
+ }
+
+ void left(OnHeapNode child)
+ {
+ left = child;
+ }
+
+ void right(OnHeapNode child)
+ {
+ right = child;
+ }
+
+ @Override
+ public Node fillInnerHashes()
+ {
+ if (!computed) // hash and size haven't been calculated; compute children then compute this
+ {
+ left.fillInnerHashes();
+ right.fillInnerHashes();
+
+ if (!left.hasEmptyHash() && !right.hasEmptyHash())
+ hash = xor(left.hash(), right.hash());
+ else if (left.hasEmptyHash())
+ hash = right.hash();
+ else if (right.hasEmptyHash())
+ hash = left.hash();
+
+ sizeOfRange = left.sizeOfRange() + right.sizeOfRange();
+ partitionsInRange = left.partitionsInRange() + right.partitionsInRange();
+
+ computed = true;
+ }
+
+ return this;
+ }
+
+ static OnHeapInner deserializeWithoutIdent(DataInputPlus in, IPartitioner p, int version) throws IOException
+ {
+ Token token = Token.serializer.deserialize(in, p, version);
+ OnHeapNode left = OnHeapNode.deserialize(in, p, version);
+ OnHeapNode right = OnHeapNode.deserialize(in, p, version);
+ return new OnHeapInner(token, left, right);
+ }
+
+ int serializeOffHeap(ByteBuffer buffer, IPartitioner partitioner) throws IOException
+ {
+ if (buffer.remaining() < OffHeapInner.maxOffHeapSize(partitioner))
+ throw new IllegalStateException("Insufficient remaining bytes to deserialize Inner node off-heap");
+
+ final int offset = buffer.position();
+
+ int tokenSize = partitioner.getTokenFactory().byteSize(token);
+ buffer.putShort(offset + OffHeapInner.TOKEN_LENGTH_OFFSET, Shorts.checkedCast(tokenSize));
+ buffer.position(offset + OffHeapInner.TOKEN_BYTES_OFFSET);
+ partitioner.getTokenFactory().serialize(token, buffer);
+
+ int leftPointer = left.serializeOffHeap(buffer, partitioner);
+ int rightPointer = right.serializeOffHeap(buffer, partitioner);
+
+ buffer.putInt(offset + OffHeapInner.LEFT_CHILD_POINTER_OFFSET, leftPointer);
+ buffer.putInt(offset + OffHeapInner.RIGHT_CHILD_POINTER_OFFSET, rightPointer);
+
+ int leftHashOffset = OffHeapInner.hashBytesOffset(leftPointer);
+ int rightHashOffset = OffHeapInner.hashBytesOffset(rightPointer);
+
+ for (int i = 0; i < HASH_SIZE; i += 8)
+ {
+ buffer.putLong(offset + OffHeapInner.HASH_BYTES_OFFSET + i,
+ buffer.getLong(leftHashOffset + i) ^ buffer.getLong(rightHashOffset + i));
+ }
+
+ return offset;
+ }
+
+ @Override
+ public String toString()
+ {
+ StringBuilder buff = new StringBuilder();
+ toString(buff, 1);
+ return buff.toString();
+ }
+
+ @Override
+ public void unsafeInvalidate()
+ {
+ computed = false;
+ }
+ }
+
+ static class OffHeapInner extends OffHeapNode implements Inner
+ {
+ /**
+ * All we want to keep here is just a pointer to the start of the Inner leaf in the
+ * direct buffer. From there, we'll be able to deserialize the following, in this order:
+ *
+ * 1. pointer to left child (int)
+ * 2. pointer to right child (int)
+ * 3. hash bytes (space allocated as HASH_MAX_SIZE)
+ * 4. token length (short)
+ * 5. token bytes (variable length)
+ */
+ static final int LEFT_CHILD_POINTER_OFFSET = 0;
+ static final int RIGHT_CHILD_POINTER_OFFSET = 4;
+ static final int HASH_BYTES_OFFSET = 8;
+ static final int TOKEN_LENGTH_OFFSET = 8 + HASH_SIZE;
+ static final int TOKEN_BYTES_OFFSET = TOKEN_LENGTH_OFFSET + 2;
+
+ private final IPartitioner partitioner;
+
+ OffHeapInner(ByteBuffer buffer, int offset, IPartitioner partitioner)
+ {
+ super(buffer, offset);
+ this.partitioner = partitioner;
+ }
+
+ public Token token()
+ {
+ int length = buffer.getShort(offset + TOKEN_LENGTH_OFFSET);
+ return partitioner.getTokenFactory().fromByteBuffer(buffer, offset + TOKEN_BYTES_OFFSET, length);
+ }
+
+ public Node left()
+ {
+ return child(LEFT_CHILD_POINTER_OFFSET);
+ }
+
+ public Node right()
+ {
+ return child(RIGHT_CHILD_POINTER_OFFSET);
+ }
+
+ private Node child(int childOffset)
+ {
+ int pointer = buffer.getInt(offset + childOffset);
+ return pointer >= 0 ? new OffHeapInner(buffer, pointer, partitioner) : new OffHeapLeaf(buffer, ~pointer);
+ }
+
+ public int hashBytesOffset()
+ {
+ return offset + HASH_BYTES_OFFSET;
+ }
+
+ static int deserializeWithoutIdent(DataInputPlus in, ByteBuffer buffer, IPartitioner partitioner, int version) throws IOException
+ {
+ if (buffer.remaining() < maxOffHeapSize(partitioner))
+ throw new IllegalStateException("Insufficient remaining bytes to deserialize Inner node off-heap");
+
+ final int offset = buffer.position();
+
+ int tokenSize = Token.serializer.deserializeSize(in);
+ byte[] tokenBytes = getTempArray(tokenSize);
+ in.readFully(tokenBytes, 0, tokenSize);
+
+ buffer.putShort(offset + OffHeapInner.TOKEN_LENGTH_OFFSET, Shorts.checkedCast(tokenSize));
+ buffer.position(offset + OffHeapInner.TOKEN_BYTES_OFFSET);
+ buffer.put(tokenBytes, 0, tokenSize);
+
+ int leftPointer = OffHeapNode.deserialize(in, buffer, partitioner, version);
+ int rightPointer = OffHeapNode.deserialize(in, buffer, partitioner, version);
+
+ buffer.putInt(offset + OffHeapInner.LEFT_CHILD_POINTER_OFFSET, leftPointer);
+ buffer.putInt(offset + OffHeapInner.RIGHT_CHILD_POINTER_OFFSET, rightPointer);
+
+ int leftHashOffset = hashBytesOffset(leftPointer);
+ int rightHashOffset = hashBytesOffset(rightPointer);
+
+ for (int i = 0; i < HASH_SIZE; i += 8)
+ {
+ buffer.putLong(offset + OffHeapInner.HASH_BYTES_OFFSET + i,
+ buffer.getLong(leftHashOffset + i) ^ buffer.getLong(rightHashOffset + i));
+ }
+
+ return offset;
+ }
+
+ static int maxOffHeapSize(IPartitioner partitioner)
+ {
+ return 4 // left pointer
+ + 4 // right pointer
+ + HASH_SIZE
+ + 2 + partitioner.getMaxTokenSize();
+ }
+
+ static int hashBytesOffset(int pointer)
+ {
+ return pointer >= 0 ? pointer + OffHeapInner.HASH_BYTES_OFFSET : ~pointer + OffHeapLeaf.HASH_BYTES_OFFSET;
+ }
+
+ @Override
+ public String toString()
+ {
+ StringBuilder buff = new StringBuilder();
+ toString(buff, 1);
+ return buff.toString();
+ }
+ }
+
+ /**
+ * @return The bitwise XOR of the inputs.
+ */
+ static byte[] xor(byte[] left, byte[] right)
+ {
+ assert left.length == right.length;
+
+ byte[] out = Arrays.copyOf(right, right.length);
+ for (int i = 0; i < left.length; i++)
+ out[i] = (byte)((left[i] & 0xFF) ^ (right[i] & 0xFF));
+ return out;
+ }
+
+ /**
+ * Bitwise XOR of the inputs, in place on the left array.
+ */
+ private static void xorOntoLeft(byte[] left, byte[] right)
+ {
+ assert left.length == right.length;
+
+ for (int i = 0; i < left.length; i++)
+ left[i] = (byte) ((left[i] & 0xFF) ^ (right[i] & 0xFF));
+ }
+
+ /**
+ * Estimate the allowable depth while keeping the resulting heap usage of this tree under the provided
+ * number of bytes. This is important for ensuring that we do not allocate overly large trees that could
+ * OOM the JVM and cause instability.
+ *
+ * Calculated using the following logic:
+ *
+ * Let T = size of a tree of depth n
+ *
+ * T = #leafs * sizeof(leaf) + #inner * sizeof(inner)
+ * T = 2^n * L + 2^n - 1 * I
+ *
+ * T = 2^n * L + 2^n * I - I;
+ *
+ * So to solve for n given sizeof(tree_n) T:
+ *
+ * n = floor(log_2((T + I) / (L + I))
+ *
+ * @param numBytes: The number of bytes to fit the tree within
+ * @param bytesPerHash: The number of bytes stored in a leaf node, for example 2 * murmur128 will be 256 bits
+ * or 32 bytes
+ * @return the estimated depth that will fit within the provided number of bytes
+ */
+ public static int estimatedMaxDepthForBytes(IPartitioner partitioner, long numBytes, int bytesPerHash)
+ {
+ byte[] hashLeft = new byte[bytesPerHash];
+ byte[] hashRigth = new byte[bytesPerHash];
+ OnHeapLeaf left = new OnHeapLeaf(hashLeft);
+ OnHeapLeaf right = new OnHeapLeaf(hashRigth);
+ Inner inner = new OnHeapInner(partitioner.getMinimumToken(), left, right);
+ inner.fillInnerHashes();
+
+ // Some partioners have variable token sizes, try to estimate as close as we can by using the same
+ // heap estimate as the memtables use.
+ long innerTokenSize = ObjectSizes.measureDeep(partitioner.getMinimumToken());
+ long realInnerTokenSize = partitioner.getMinimumToken().getHeapSize();
+
+ long sizeOfLeaf = ObjectSizes.measureDeep(left);
+ long sizeOfInner = ObjectSizes.measureDeep(inner) -
+ (ObjectSizes.measureDeep(left) + ObjectSizes.measureDeep(right) + innerTokenSize) +
+ realInnerTokenSize;
+
+ long adjustedBytes = Math.max(1, (numBytes + sizeOfInner) / (sizeOfLeaf + sizeOfInner));
+ return Math.max(1, (int) Math.floor(Math.log(adjustedBytes) / Math.log(2)));
+ }
+
+ /*
+ * Test-only methods.
+ */
+
+ /**
+ * Invalidates the ranges containing the given token.
+ * Useful for testing.
+ */
+ @VisibleForTesting
+ void unsafeInvalidate(Token t)
+ {
+ unsafeInvalidateHelper(root, fullRange.left, t);
+ }
+
+ private void unsafeInvalidateHelper(Node node, Token pleft, Token t)
+ {
+ node.hash(EMPTY_HASH);
+
+ if (node instanceof Leaf)
+ return;
+
+ assert node instanceof Inner;
+ Inner inner = (Inner) node;
+ inner.unsafeInvalidate();
+
+ if (Range.contains(pleft, inner.token(), t))
+ unsafeInvalidateHelper(inner.left(), pleft, t); // left child contains token
+ else
+ unsafeInvalidateHelper(inner.right(), inner.token(), t); // right child contains token
+ }
+
+ /**
+ * Hash the given range in the tree. The range must have been generated
+ * with recursive applications of partitioner.midpoint().
+ *
+ * NB: Currently does not support wrapping ranges that do not end with
+ * partitioner.getMinimumToken().
+ *
+ * @return {@link #EMPTY_HASH} if any subrange of the range is invalid, or if the exact
+ * range cannot be calculated using this tree.
+ */
+ @VisibleForTesting
+ byte[] hash(Range<Token> range)
+ {
+ return find(range).hash();
+ }
+
+ interface Consumer<E extends Exception>
+ {
+ void accept(Node node) throws E;
+ }
+
+ @VisibleForTesting
+ <E extends Exception> boolean ifHashesRange(Range<Token> range, Consumer<E> consumer) throws E
+ {
+ try
+ {
+ Node node = findHelper(root, new Range<>(fullRange.left, fullRange.right), range);
+ boolean hasHash = !node.hasEmptyHash();
+ if (hasHash)
+ consumer.accept(node);
+ return hasHash;
+ }
+ catch (StopRecursion e)
+ {
+ return false;
+ }
+ }
+
+ @VisibleForTesting
+ boolean hashesRange(Range<Token> range)
+ {
+ return ifHashesRange(range, n -> {});
+ }
+
+ /**
+ * For testing purposes.
+ * Gets the smallest range containing the token.
+ */
+ @VisibleForTesting
+ public TreeRange get(Token t)
+ {
+ return getHelper(root, fullRange.left, fullRange.right, t);
+ }
+
+ private TreeRange getHelper(Node node, Token pleft, Token pright, Token t)
+ {
+ int depth = 0;
+
+ while (true)
+ {
+ if (node instanceof Leaf)
+ {
+ // we've reached a hash: wrap it up and deliver it
+ return new TreeRange(this, pleft, pright, depth, node);
+ }
+
+ assert node instanceof Inner;
+ Inner inner = (Inner) node;
+
+ if (Range.contains(pleft, inner.token(), t)) // left child contains token
+ {
+ pright = inner.token();
+ node = inner.left();
+ }
+ else // right child contains token
+ {
+ pleft = inner.token();
+ node = inner.right();
+ }
+
+ depth++;
+ }
+ }
+
+ private void fillInnerHashes()
+ {
+ root.fillInnerHashes();
}
}
diff --git a/src/java/org/apache/cassandra/utils/MerkleTrees.java b/src/java/org/apache/cassandra/utils/MerkleTrees.java
index d2a8058..0043fe0 100644
--- a/src/java/org/apache/cassandra/utils/MerkleTrees.java
+++ b/src/java/org/apache/cassandra/utils/MerkleTrees.java
@@ -44,9 +44,9 @@
{
public static final MerkleTreesSerializer serializer = new MerkleTreesSerializer();
- private Map<Range<Token>, MerkleTree> merkleTrees = new TreeMap<>(new TokenRangeComparator());
+ private final Map<Range<Token>, MerkleTree> merkleTrees = new TreeMap<>(new TokenRangeComparator());
- private IPartitioner partitioner;
+ private final IPartitioner partitioner;
/**
* Creates empty MerkleTrees object.
@@ -143,6 +143,15 @@
}
/**
+ * Dereference all merkle trees and release direct memory for all off-heap trees.
+ */
+ public void release()
+ {
+ merkleTrees.values().forEach(MerkleTree::release);
+ merkleTrees.clear();
+ }
+
+ /**
* Init a selected MerkleTree with an even tree distribution.
*
* @param range
@@ -171,7 +180,7 @@
@VisibleForTesting
public void invalidate(Token t)
{
- getMerkleTree(t).invalidate(t);
+ getMerkleTree(t).unsafeInvalidate(t);
}
/**
@@ -247,11 +256,11 @@
}
/**
- * Get an iterator for all the invalids generated by the MerkleTrees.
+ * Get an iterator for all the iterator generated by the MerkleTrees.
*
* @return
*/
- public TreeRangeIterator invalids()
+ public TreeRangeIterator rangeIterator()
{
return new TreeRangeIterator();
}
@@ -285,30 +294,20 @@
@VisibleForTesting
public byte[] hash(Range<Token> range)
{
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- boolean hashed = false;
-
- try
+ try (ByteArrayOutputStream baos = new ByteArrayOutputStream())
{
- for (Range<Token> rt : merkleTrees.keySet())
- {
- if (rt.intersects(range))
- {
- byte[] bytes = merkleTrees.get(rt).hash(range);
- if (bytes != null)
- {
- baos.write(bytes);
- hashed = true;
- }
- }
- }
+ boolean hashed = false;
+
+ for (Map.Entry<Range<Token>, MerkleTree> entry : merkleTrees.entrySet())
+ if (entry.getKey().intersects(range))
+ hashed |= entry.getValue().ifHashesRange(range, n -> baos.write(n.hash()));
+
+ return hashed ? baos.toByteArray() : null;
}
catch (IOException e)
{
throw new RuntimeException("Unable to append merkle tree hash to result");
}
-
- return hashed ? baos.toByteArray() : null;
}
/**
@@ -354,7 +353,7 @@
{
if (it.hasNext())
{
- current = it.next().invalids();
+ current = it.next().rangeIterator();
return current.next();
}
@@ -369,6 +368,17 @@
}
/**
+ * @return a new {@link MerkleTrees} instance with all trees moved off heap.
+ */
+ public MerkleTrees tryMoveOffHeap() throws IOException
+ {
+ Map<Range<Token>, MerkleTree> movedTrees = new TreeMap<>(new TokenRangeComparator());
+ for (Map.Entry<Range<Token>, MerkleTree> entry : merkleTrees.entrySet())
+ movedTrees.put(entry.getKey(), entry.getValue().tryMoveOffHeap());
+ return new MerkleTrees(partitioner, movedTrees.values());
+ }
+
+ /**
* Get the differences between the two sets of MerkleTrees.
*
* @param ltree
@@ -379,9 +389,7 @@
{
List<Range<Token>> differences = new ArrayList<>();
for (MerkleTree tree : ltree.merkleTrees.values())
- {
differences.addAll(MerkleTree.difference(tree, rtree.getMerkleTree(tree.fullRange)));
- }
return differences;
}
@@ -392,7 +400,7 @@
out.writeInt(trees.merkleTrees.size());
for (MerkleTree tree : trees.merkleTrees.values())
{
- MerkleTree.serializer.serialize(tree, out, version);
+ tree.serialize(out, version);
}
}
@@ -405,7 +413,7 @@
{
for (int i = 0; i < nTrees; i++)
{
- MerkleTree tree = MerkleTree.serializer.deserialize(in, version);
+ MerkleTree tree = MerkleTree.deserialize(in, version);
trees.add(tree);
if (partitioner == null)
@@ -425,7 +433,7 @@
long size = TypeSizes.sizeof(trees.merkleTrees.size());
for (MerkleTree tree : trees.merkleTrees.values())
{
- size += MerkleTree.serializer.serializedSize(tree, version);
+ size += tree.serializedSize(version);
}
return size;
}
diff --git a/src/java/org/apache/cassandra/utils/MonotonicClock.java b/src/java/org/apache/cassandra/utils/MonotonicClock.java
new file mode 100644
index 0000000..5a1aa3c
--- /dev/null
+++ b/src/java/org/apache/cassandra/utils/MonotonicClock.java
@@ -0,0 +1,346 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.utils;
+
+import java.lang.reflect.Constructor;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.function.LongSupplier;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.concurrent.ScheduledExecutors;
+import org.apache.cassandra.config.Config;
+
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+
+/**
+ * Wrapper around time related functions that are either implemented by using the default JVM calls
+ * or by using a custom implementation for testing purposes.
+ *
+ * See {@link #preciseTime} for how to use a custom implementation.
+ *
+ * Please note that {@link java.time.Clock} wasn't used, as it would not be possible to provide an
+ * implementation for {@link #now()} with the exact same properties of {@link System#nanoTime()}.
+ */
+public interface MonotonicClock
+{
+ /**
+ * Static singleton object that will be instantiated by default with a system clock
+ * implementation. Set <code>cassandra.clock</code> system property to a FQCN to use a
+ * different implementation instead.
+ */
+ public static final MonotonicClock preciseTime = Defaults.precise();
+ public static final MonotonicClock approxTime = Defaults.approx(preciseTime);
+
+ /**
+ * @see System#nanoTime()
+ *
+ * Provides a monotonic time that can be compared with any other such value produced by the same clock
+ * since the application started only; these times cannot be persisted or serialized to other nodes.
+ *
+ * Nanosecond precision.
+ */
+ public long now();
+
+ /**
+ * @return nanoseconds of potential error
+ */
+ public long error();
+
+ public MonotonicClockTranslation translate();
+
+ public boolean isAfter(long instant);
+ public boolean isAfter(long now, long instant);
+
+ static class Defaults
+ {
+ private static final Logger logger = LoggerFactory.getLogger(MonotonicClock.class);
+
+ private static MonotonicClock precise()
+ {
+ String sclock = System.getProperty("cassandra.clock");
+ if (sclock == null)
+ sclock = System.getProperty("cassandra.monotonic_clock.precise");
+
+ if (sclock != null)
+ {
+ try
+ {
+ logger.debug("Using custom clock implementation: {}", sclock);
+ return (MonotonicClock) Class.forName(sclock).newInstance();
+ }
+ catch (Exception e)
+ {
+ logger.error(e.getMessage(), e);
+ }
+ }
+
+ return new SystemClock();
+ }
+
+ private static MonotonicClock approx(MonotonicClock precise)
+ {
+ String sclock = System.getProperty("cassandra.monotonic_clock.approx");
+ if (sclock != null)
+ {
+ try
+ {
+ logger.debug("Using custom clock implementation: {}", sclock);
+ Class<? extends MonotonicClock> clazz = (Class<? extends MonotonicClock>) Class.forName(sclock);
+
+ if (SystemClock.class.equals(clazz) && SystemClock.class.equals(precise.getClass()))
+ return precise;
+
+ try
+ {
+ Constructor<? extends MonotonicClock> withPrecise = clazz.getConstructor(MonotonicClock.class);
+ return withPrecise.newInstance(precise);
+ }
+ catch (NoSuchMethodException nme)
+ {
+ }
+
+ return clazz.newInstance();
+ }
+ catch (Exception e)
+ {
+ logger.error(e.getMessage(), e);
+ }
+ }
+
+ return new SampledClock(precise);
+ }
+ }
+
+ static abstract class AbstractEpochSamplingClock implements MonotonicClock
+ {
+ private static final Logger logger = LoggerFactory.getLogger(AbstractEpochSamplingClock.class);
+ private static final String UPDATE_INTERVAL_PROPERTY = Config.PROPERTY_PREFIX + "NANOTIMETOMILLIS_TIMESTAMP_UPDATE_INTERVAL";
+ private static final long UPDATE_INTERVAL_MS = Long.getLong(UPDATE_INTERVAL_PROPERTY, 10000);
+
+ private static class AlmostSameTime implements MonotonicClockTranslation
+ {
+ final long millisSinceEpoch;
+ final long monotonicNanos;
+ final long error; // maximum error of millis measurement (in nanos)
+
+ private AlmostSameTime(long millisSinceEpoch, long monotonicNanos, long errorNanos)
+ {
+ this.millisSinceEpoch = millisSinceEpoch;
+ this.monotonicNanos = monotonicNanos;
+ this.error = errorNanos;
+ }
+
+ public long fromMillisSinceEpoch(long currentTimeMillis)
+ {
+ return monotonicNanos + MILLISECONDS.toNanos(currentTimeMillis - millisSinceEpoch);
+ }
+
+ public long toMillisSinceEpoch(long nanoTime)
+ {
+ return millisSinceEpoch + TimeUnit.NANOSECONDS.toMillis(nanoTime - monotonicNanos);
+ }
+
+ public long error()
+ {
+ return error;
+ }
+ }
+
+ final LongSupplier millisSinceEpoch;
+
+ private volatile AlmostSameTime almostSameTime = new AlmostSameTime(0L, 0L, Long.MAX_VALUE);
+ private Future<?> almostSameTimeUpdater;
+ private static double failedAlmostSameTimeUpdateModifier = 1.0;
+
+ AbstractEpochSamplingClock(LongSupplier millisSinceEpoch)
+ {
+ this.millisSinceEpoch = millisSinceEpoch;
+ resumeEpochSampling();
+ }
+
+ public MonotonicClockTranslation translate()
+ {
+ return almostSameTime;
+ }
+
+ public synchronized void pauseEpochSampling()
+ {
+ if (almostSameTimeUpdater == null)
+ return;
+
+ almostSameTimeUpdater.cancel(true);
+ try { almostSameTimeUpdater.get(); } catch (Throwable t) { }
+ almostSameTimeUpdater = null;
+ }
+
+ public synchronized void resumeEpochSampling()
+ {
+ if (almostSameTimeUpdater != null)
+ throw new IllegalStateException("Already running");
+ updateAlmostSameTime();
+ logger.info("Scheduling approximate time conversion task with an interval of {} milliseconds", UPDATE_INTERVAL_MS);
+ almostSameTimeUpdater = ScheduledExecutors.scheduledFastTasks.scheduleWithFixedDelay(this::updateAlmostSameTime, UPDATE_INTERVAL_MS, UPDATE_INTERVAL_MS, MILLISECONDS);
+ }
+
+ private void updateAlmostSameTime()
+ {
+ final int tries = 3;
+ long[] samples = new long[2 * tries + 1];
+ samples[0] = System.nanoTime();
+ for (int i = 1 ; i < samples.length ; i += 2)
+ {
+ samples[i] = millisSinceEpoch.getAsLong();
+ samples[i + 1] = now();
+ }
+
+ int best = 1;
+ // take sample with minimum delta between calls
+ for (int i = 3 ; i < samples.length - 1 ; i += 2)
+ {
+ if ((samples[i+1] - samples[i-1]) < (samples[best+1]-samples[best-1]))
+ best = i;
+ }
+
+ long millis = samples[best];
+ long nanos = (samples[best+1] / 2) + (samples[best-1] / 2);
+ long error = (samples[best+1] / 2) - (samples[best-1] / 2);
+
+ AlmostSameTime prev = almostSameTime;
+ AlmostSameTime next = new AlmostSameTime(millis, nanos, error);
+
+ if (next.error > prev.error && next.error > prev.error * failedAlmostSameTimeUpdateModifier)
+ {
+ failedAlmostSameTimeUpdateModifier *= 1.1;
+ return;
+ }
+
+ failedAlmostSameTimeUpdateModifier = 1.0;
+ almostSameTime = next;
+ }
+ }
+
+ public static class SystemClock extends AbstractEpochSamplingClock
+ {
+ private SystemClock()
+ {
+ super(System::currentTimeMillis);
+ }
+
+ @Override
+ public long now()
+ {
+ return System.nanoTime();
+ }
+
+ @Override
+ public long error()
+ {
+ return 1;
+ }
+
+ @Override
+ public boolean isAfter(long instant)
+ {
+ return now() > instant;
+ }
+
+ @Override
+ public boolean isAfter(long now, long instant)
+ {
+ return now > instant;
+ }
+ }
+
+ public static class SampledClock implements MonotonicClock
+ {
+ private static final Logger logger = LoggerFactory.getLogger(SampledClock.class);
+ private static final int UPDATE_INTERVAL_MS = Math.max(1, Integer.parseInt(System.getProperty(Config.PROPERTY_PREFIX + "approximate_time_precision_ms", "2")));
+ private static final long ERROR_NANOS = MILLISECONDS.toNanos(UPDATE_INTERVAL_MS);
+
+ private final MonotonicClock precise;
+
+ private volatile long almostNow;
+ private Future<?> almostNowUpdater;
+
+ public SampledClock(MonotonicClock precise)
+ {
+ this.precise = precise;
+ resumeNowSampling();
+ }
+
+ @Override
+ public long now()
+ {
+ return almostNow;
+ }
+
+ @Override
+ public long error()
+ {
+ return ERROR_NANOS;
+ }
+
+ @Override
+ public MonotonicClockTranslation translate()
+ {
+ return precise.translate();
+ }
+
+ @Override
+ public boolean isAfter(long instant)
+ {
+ return isAfter(almostNow, instant);
+ }
+
+ @Override
+ public boolean isAfter(long now, long instant)
+ {
+ return now - ERROR_NANOS > instant;
+ }
+
+ public synchronized void pauseNowSampling()
+ {
+ if (almostNowUpdater == null)
+ return;
+
+ almostNowUpdater.cancel(true);
+ try { almostNowUpdater.get(); } catch (Throwable t) { }
+ almostNowUpdater = null;
+ }
+
+ public synchronized void resumeNowSampling()
+ {
+ if (almostNowUpdater != null)
+ throw new IllegalStateException("Already running");
+
+ almostNow = precise.now();
+ logger.info("Scheduling approximate time-check task with a precision of {} milliseconds", UPDATE_INTERVAL_MS);
+ almostNowUpdater = ScheduledExecutors.scheduledFastTasks.scheduleWithFixedDelay(() -> almostNow = precise.now(), UPDATE_INTERVAL_MS, UPDATE_INTERVAL_MS, MILLISECONDS);
+ }
+
+ public synchronized void refreshNow()
+ {
+ pauseNowSampling();
+ resumeNowSampling();
+ }
+ }
+
+}
diff --git a/src/java/org/apache/cassandra/utils/MonotonicClockTranslation.java b/src/java/org/apache/cassandra/utils/MonotonicClockTranslation.java
new file mode 100644
index 0000000..f7f83e4
--- /dev/null
+++ b/src/java/org/apache/cassandra/utils/MonotonicClockTranslation.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.utils;
+
+public interface MonotonicClockTranslation
+{
+ /** accepts millis since epoch, returns nanoTime in the related clock */
+ public long fromMillisSinceEpoch(long currentTimeMillis);
+ /** accepts nanoTime in the related MonotinicClock, returns millis since epoch */
+ public long toMillisSinceEpoch(long nanoTime);
+ /** Nanoseconds of probable error in the translation */
+ public long error();
+}
diff --git a/src/java/org/apache/cassandra/utils/Mx4jTool.java b/src/java/org/apache/cassandra/utils/Mx4jTool.java
index 77a6013..eda6354 100644
--- a/src/java/org/apache/cassandra/utils/Mx4jTool.java
+++ b/src/java/org/apache/cassandra/utils/Mx4jTool.java
@@ -19,6 +19,7 @@
import javax.management.ObjectName;
+import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -74,17 +75,18 @@
private static String getAddress()
{
- return System.getProperty("mx4jaddress", FBUtilities.getBroadcastAddress().getHostAddress());
+ String sAddress = System.getProperty("mx4jaddress");
+ if (StringUtils.isEmpty(sAddress))
+ sAddress = FBUtilities.getBroadcastAddressAndPort().address.getHostAddress();
+ return sAddress;
}
private static int getPort()
{
int port = 8081;
String sPort = System.getProperty("mx4jport");
- if (sPort != null && !sPort.equals(""))
- {
+ if (StringUtils.isNotEmpty(sPort))
port = Integer.parseInt(sPort);
- }
return port;
}
}
diff --git a/src/java/org/apache/cassandra/utils/NanoTimeToCurrentTimeMillis.java b/src/java/org/apache/cassandra/utils/NanoTimeToCurrentTimeMillis.java
deleted file mode 100644
index 5aafbe5..0000000
--- a/src/java/org/apache/cassandra/utils/NanoTimeToCurrentTimeMillis.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.utils;
-
-import java.util.concurrent.TimeUnit;
-
-import org.apache.cassandra.concurrent.ScheduledExecutors;
-import org.apache.cassandra.config.Config;
-
-/*
- * Convert from nanotime to non-monotonic current time millis. Beware of weaker ordering guarantees.
- */
-public class NanoTimeToCurrentTimeMillis
-{
- /*
- * How often to pull a new timestamp from the system.
- */
- private static final String TIMESTAMP_UPDATE_INTERVAL_PROPERTY = Config.PROPERTY_PREFIX + "NANOTIMETOMILLIS_TIMESTAMP_UPDATE_INTERVAL";
- private static final long TIMESTAMP_UPDATE_INTERVAL = Long.getLong(TIMESTAMP_UPDATE_INTERVAL_PROPERTY, 10000);
-
- private static volatile long TIMESTAMP_BASE[] = new long[] { System.currentTimeMillis(), System.nanoTime() };
-
- /*
- * System.currentTimeMillis() is 25 nanoseconds. This is 2 nanoseconds (maybe) according to JMH.
- * Faster than calling both currentTimeMillis() and nanoTime().
- *
- * There is also the issue of how scalable nanoTime() and currentTimeMillis() are which is a moving target.
- *
- * These timestamps don't order with System.currentTimeMillis() because currentTimeMillis() can tick over
- * before this one does. I have seen it behind by as much as 2ms on Linux and 25ms on Windows.
- */
- public static long convert(long nanoTime)
- {
- final long timestampBase[] = TIMESTAMP_BASE;
- return timestampBase[0] + TimeUnit.NANOSECONDS.toMillis(nanoTime - timestampBase[1]);
- }
-
- public static void updateNow()
- {
- ScheduledExecutors.scheduledFastTasks.submit(NanoTimeToCurrentTimeMillis::updateTimestampBase);
- }
-
- static
- {
- ScheduledExecutors.scheduledFastTasks.scheduleWithFixedDelay(NanoTimeToCurrentTimeMillis::updateTimestampBase,
- TIMESTAMP_UPDATE_INTERVAL,
- TIMESTAMP_UPDATE_INTERVAL,
- TimeUnit.MILLISECONDS);
- }
-
- private static void updateTimestampBase()
- {
- TIMESTAMP_BASE = new long[] {
- Math.max(TIMESTAMP_BASE[0], System.currentTimeMillis()),
- Math.max(TIMESTAMP_BASE[1], System.nanoTime()) };
- }
-}
diff --git a/src/java/org/apache/cassandra/utils/NativeLibrary.java b/src/java/org/apache/cassandra/utils/NativeLibrary.java
index 7015a44..8bcd6f6 100644
--- a/src/java/org/apache/cassandra/utils/NativeLibrary.java
+++ b/src/java/org/apache/cassandra/utils/NativeLibrary.java
@@ -29,7 +29,6 @@
import org.slf4j.LoggerFactory;
import com.sun.jna.LastErrorException;
-import sun.nio.ch.FileChannelImpl;
import org.apache.cassandra.io.FSWriteError;
@@ -51,7 +50,7 @@
OTHER;
}
- private static final OSType osType;
+ public static final OSType osType;
private static final int MCL_CURRENT;
private static final int MCL_FUTURE;
@@ -80,7 +79,14 @@
static
{
FILE_DESCRIPTOR_FD_FIELD = FBUtilities.getProtectedField(FileDescriptor.class, "fd");
- FILE_CHANNEL_FD_FIELD = FBUtilities.getProtectedField(FileChannelImpl.class, "fd");
+ try
+ {
+ FILE_CHANNEL_FD_FIELD = FBUtilities.getProtectedField(Class.forName("sun.nio.ch.FileChannelImpl"), "fd");
+ }
+ catch (ClassNotFoundException e)
+ {
+ throw new RuntimeException(e);
+ }
// detect the OS type the JVM is running on and then set the CLibraryWrapper
// instance to a compatable implementation of CLibraryWrapper for that OS type
@@ -128,11 +134,15 @@
private static OSType getOsType()
{
String osName = System.getProperty("os.name").toLowerCase();
- if (osName.contains("mac"))
+ if (osName.contains("linux"))
+ return LINUX;
+ else if (osName.contains("mac"))
return MAC;
else if (osName.contains("windows"))
return WINDOWS;
- else if (osName.contains("aix"))
+
+ logger.warn("the current operating system, {}, is unsupported by cassandra", osName);
+ if (osName.contains("aix"))
return AIX;
else
// fall back to the Linux impl for all unknown OS types until otherwise implicitly supported as needed
@@ -188,7 +198,7 @@
{
logger.warn("Unable to lock JVM memory (ENOMEM)."
+ " This can result in part of the JVM being swapped out, especially with mmapped I/O enabled."
- + " Increase RLIMIT_MEMLOCK or run Cassandra as root.");
+ + " Increase RLIMIT_MEMLOCK.");
}
else if (osType != MAC)
{
@@ -327,7 +337,7 @@
if (!(e instanceof LastErrorException))
throw e;
- String errMsg = String.format("fsync(%s) failed, errno %s", fd, errno(e));
+ String errMsg = String.format("fsync(%s) failed, errno (%s) %s", fd, errno(e), e.getMessage());
logger.warn(errMsg);
throw new FSWriteError(e, errMsg);
}
diff --git a/src/java/org/apache/cassandra/utils/NativeSSTableLoaderClient.java b/src/java/org/apache/cassandra/utils/NativeSSTableLoaderClient.java
index 1f10d2b..9763d7e 100644
--- a/src/java/org/apache/cassandra/utils/NativeSSTableLoaderClient.java
+++ b/src/java/org/apache/cassandra/utils/NativeSSTableLoaderClient.java
@@ -19,52 +19,56 @@
import java.net.InetAddress;
import java.nio.ByteBuffer;
+import java.net.InetSocketAddress;
+import java.net.UnknownHostException;
import java.util.*;
-import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
import com.datastax.driver.core.*;
-import org.apache.cassandra.config.ColumnDefinition;
-import org.apache.cassandra.config.ColumnDefinition.ClusteringOrder;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.SchemaConstants;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.schema.*;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.ColumnMetadata.ClusteringOrder;
import org.apache.cassandra.cql3.ColumnIdentifier;
import org.apache.cassandra.db.marshal.*;
import org.apache.cassandra.dht.*;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.dht.Token.TokenFactory;
import org.apache.cassandra.io.sstable.SSTableLoader;
+import org.apache.cassandra.schema.TableMetadata;
+
import org.apache.cassandra.schema.CQLTypeParser;
import org.apache.cassandra.schema.SchemaKeyspace;
import org.apache.cassandra.schema.Types;
public class NativeSSTableLoaderClient extends SSTableLoader.Client
{
- protected final Map<String, CFMetaData> tables;
- private final Collection<InetAddress> hosts;
- private final int port;
+ protected final Map<String, TableMetadataRef> tables;
+ private final Collection<InetSocketAddress> hosts;
+ private final int storagePort;
private final AuthProvider authProvider;
private final SSLOptions sslOptions;
-
- public NativeSSTableLoaderClient(Collection<InetAddress> hosts, int port, String username, String password, SSLOptions sslOptions)
+ public NativeSSTableLoaderClient(Collection<InetSocketAddress> hosts, int storagePort, String username, String password, SSLOptions sslOptions)
{
- this(hosts, port, new PlainTextAuthProvider(username, password), sslOptions);
+ this(hosts, storagePort, new PlainTextAuthProvider(username, password), sslOptions);
}
- public NativeSSTableLoaderClient(Collection<InetAddress> hosts, int port, AuthProvider authProvider, SSLOptions sslOptions)
+ public NativeSSTableLoaderClient(Collection<InetSocketAddress> hosts, int storagePort, AuthProvider authProvider, SSLOptions sslOptions)
{
super();
this.tables = new HashMap<>();
this.hosts = hosts;
- this.port = port;
this.authProvider = authProvider;
this.sslOptions = sslOptions;
+ this.storagePort = storagePort;
}
public void init(String keyspace)
{
- Cluster.Builder builder = Cluster.builder().addContactPoints(hosts).withPort(port);
+ Cluster.Builder builder = Cluster.builder().addContactPointsWithPorts(hosts).allowBetaProtocolVersion();
+
if (sslOptions != null)
builder.withSSL(sslOptions);
if (authProvider != null)
@@ -86,26 +90,35 @@
Range<Token> range = new Range<>(tokenFactory.fromString(tokenRange.getStart().getValue().toString()),
tokenFactory.fromString(tokenRange.getEnd().getValue().toString()));
for (Host endpoint : endpoints)
- addRangeForEndpoint(range, endpoint.getBroadcastAddress());
+ {
+ int broadcastPort = endpoint.getBroadcastSocketAddress().getPort();
+ // use port from broadcast address if set.
+ int portToUse = broadcastPort != 0 ? broadcastPort : storagePort;
+ addRangeForEndpoint(range, InetAddressAndPort.getByNameOverrideDefaults(endpoint.getAddress().getHostAddress(), portToUse));
+ }
}
Types types = fetchTypes(keyspace, session);
tables.putAll(fetchTables(keyspace, session, partitioner, types));
- // We only need the CFMetaData for the views, so we only load that.
+ // We only need the TableMetadata for the views, so we only load that.
tables.putAll(fetchViews(keyspace, session, partitioner, types));
}
+ catch (UnknownHostException e)
+ {
+ throw new RuntimeException(e);
+ }
}
- public CFMetaData getTableMetadata(String tableName)
+ public TableMetadataRef getTableMetadata(String tableName)
{
return tables.get(tableName);
}
@Override
- public void setTableMetadata(CFMetaData cfm)
+ public void setTableMetadata(TableMetadataRef cfm)
{
- tables.put(cfm.cfName, cfm);
+ tables.put(cfm.name, cfm);
}
private static Types fetchTypes(String keyspace, Session session)
@@ -132,9 +145,9 @@
* Note: It is not safe for this class to use static methods from SchemaKeyspace (static final fields are ok)
* as that triggers initialization of the class, which fails in client mode.
*/
- private static Map<String, CFMetaData> fetchTables(String keyspace, Session session, IPartitioner partitioner, Types types)
+ private static Map<String, TableMetadataRef> fetchTables(String keyspace, Session session, IPartitioner partitioner, Types types)
{
- Map<String, CFMetaData> tables = new HashMap<>();
+ Map<String, TableMetadataRef> tables = new HashMap<>();
String query = String.format("SELECT * FROM %s.%s WHERE keyspace_name = ?", SchemaConstants.SCHEMA_KEYSPACE_NAME, SchemaKeyspace.TABLES);
for (Row row : session.execute(query, keyspace))
@@ -146,12 +159,9 @@
return tables;
}
- /*
- * In the case where we are creating View CFMetaDatas, we
- */
- private static Map<String, CFMetaData> fetchViews(String keyspace, Session session, IPartitioner partitioner, Types types)
+ private static Map<String, TableMetadataRef> fetchViews(String keyspace, Session session, IPartitioner partitioner, Types types)
{
- Map<String, CFMetaData> tables = new HashMap<>();
+ Map<String, TableMetadataRef> tables = new HashMap<>();
String query = String.format("SELECT * FROM %s.%s WHERE keyspace_name = ?", SchemaConstants.SCHEMA_KEYSPACE_NAME, SchemaKeyspace.VIEWS);
for (Row row : session.execute(query, keyspace))
@@ -163,56 +173,42 @@
return tables;
}
- private static CFMetaData createTableMetadata(String keyspace,
- Session session,
- IPartitioner partitioner,
- boolean isView,
- Row row,
- String name,
- Types types)
+ private static TableMetadataRef createTableMetadata(String keyspace,
+ Session session,
+ IPartitioner partitioner,
+ boolean isView,
+ Row row,
+ String name,
+ Types types)
{
- UUID id = row.getUUID("id");
- Set<CFMetaData.Flag> flags = isView ? Collections.emptySet() : CFMetaData.flagsFromStrings(row.getSet("flags", String.class));
+ TableMetadata.Builder builder = TableMetadata.builder(keyspace, name, TableId.fromUUID(row.getUUID("id")))
+ .partitioner(partitioner);
- boolean isSuper = flags.contains(CFMetaData.Flag.SUPER);
- boolean isCounter = flags.contains(CFMetaData.Flag.COUNTER);
- boolean isDense = flags.contains(CFMetaData.Flag.DENSE);
- boolean isCompound = isView || flags.contains(CFMetaData.Flag.COMPOUND);
+ if (!isView)
+ builder.flags(TableMetadata.Flag.fromStringSet(row.getSet("flags", String.class)));
String columnsQuery = String.format("SELECT * FROM %s.%s WHERE keyspace_name = ? AND table_name = ?",
SchemaConstants.SCHEMA_KEYSPACE_NAME,
SchemaKeyspace.COLUMNS);
- List<ColumnDefinition> defs = new ArrayList<>();
for (Row colRow : session.execute(columnsQuery, keyspace, name))
- defs.add(createDefinitionFromRow(colRow, keyspace, name, types));
-
- CFMetaData metadata = CFMetaData.create(keyspace,
- name,
- id,
- isDense,
- isCompound,
- isSuper,
- isCounter,
- isView,
- defs,
- partitioner);
+ builder.addColumn(createDefinitionFromRow(colRow, keyspace, name, types));
String droppedColumnsQuery = String.format("SELECT * FROM %s.%s WHERE keyspace_name = ? AND table_name = ?",
SchemaConstants.SCHEMA_KEYSPACE_NAME,
SchemaKeyspace.DROPPED_COLUMNS);
- Map<ByteBuffer, CFMetaData.DroppedColumn> droppedColumns = new HashMap<>();
+ Map<ByteBuffer, DroppedColumn> droppedColumns = new HashMap<>();
for (Row colRow : session.execute(droppedColumnsQuery, keyspace, name))
{
- CFMetaData.DroppedColumn droppedColumn = createDroppedColumnFromRow(colRow, keyspace);
- droppedColumns.put(UTF8Type.instance.decompose(droppedColumn.name), droppedColumn);
+ DroppedColumn droppedColumn = createDroppedColumnFromRow(colRow, keyspace, name);
+ droppedColumns.put(droppedColumn.column.name.bytes, droppedColumn);
}
- metadata.droppedColumns(droppedColumns);
+ builder.droppedColumns(droppedColumns);
- return metadata;
+ return TableMetadataRef.forOfflineTools(builder.build());
}
- private static ColumnDefinition createDefinitionFromRow(Row row, String keyspace, String table, Types types)
+ private static ColumnMetadata createDefinitionFromRow(Row row, String keyspace, String table, Types types)
{
ClusteringOrder order = ClusteringOrder.valueOf(row.getString("clustering_order").toUpperCase());
AbstractType<?> type = CQLTypeParser.parse(keyspace, row.getString("type"), types);
@@ -222,17 +218,17 @@
ColumnIdentifier name = new ColumnIdentifier(row.getBytes("column_name_bytes"), row.getString("column_name"));
int position = row.getInt("position");
- ColumnDefinition.Kind kind = ColumnDefinition.Kind.valueOf(row.getString("kind").toUpperCase());
- return new ColumnDefinition(keyspace, table, name, type, position, kind);
+ org.apache.cassandra.schema.ColumnMetadata.Kind kind = ColumnMetadata.Kind.valueOf(row.getString("kind").toUpperCase());
+ return new ColumnMetadata(keyspace, table, name, type, position, kind);
}
- private static CFMetaData.DroppedColumn createDroppedColumnFromRow(Row row, String keyspace)
+ private static DroppedColumn createDroppedColumnFromRow(Row row, String keyspace, String table)
{
String name = row.getString("column_name");
- ColumnDefinition.Kind kind =
- row.isNull("kind") ? null : ColumnDefinition.Kind.valueOf(row.getString("kind").toUpperCase());
AbstractType<?> type = CQLTypeParser.parse(keyspace, row.getString("type"), Types.none());
- long droppedTime = TimeUnit.MILLISECONDS.toMicros(row.getTimestamp("dropped_time").getTime());
- return new CFMetaData.DroppedColumn(name, kind, type, droppedTime);
+ ColumnMetadata.Kind kind = ColumnMetadata.Kind.valueOf(row.getString("kind").toUpperCase());
+ ColumnMetadata column = new ColumnMetadata(keyspace, table, ColumnIdentifier.getInterned(name, true), type, ColumnMetadata.NO_POSITION, kind);
+ long droppedTime = row.getTimestamp("dropped_time").getTime();
+ return new DroppedColumn(column, droppedTime);
}
}
diff --git a/src/java/org/apache/cassandra/utils/ObjectSizes.java b/src/java/org/apache/cassandra/utils/ObjectSizes.java
index dea2bac..fa02ba7 100644
--- a/src/java/org/apache/cassandra/utils/ObjectSizes.java
+++ b/src/java/org/apache/cassandra/utils/ObjectSizes.java
@@ -138,6 +138,7 @@
* @param str String to calculate memory size of
* @return Total in-memory size of the String
*/
+ //@TODO hard coding this to 2 isn't necessarily correct in Java 11
public static long sizeOf(String str)
{
return STRING_EMPTY_SIZE + sizeOfArray(str.length(), 2);
diff --git a/src/java/org/apache/cassandra/utils/Pair.java b/src/java/org/apache/cassandra/utils/Pair.java
index ea8b8fc..cb09529 100644
--- a/src/java/org/apache/cassandra/utils/Pair.java
+++ b/src/java/org/apache/cassandra/utils/Pair.java
@@ -53,6 +53,18 @@
return "(" + left + "," + right + ")";
}
+ //For functional interfaces
+ public T1 left()
+ {
+ return left;
+ }
+
+ //For functional interfaces
+ public T2 right()
+ {
+ return right;
+ }
+
public static <X, Y> Pair<X, Y> create(X x, Y y)
{
return new Pair<X, Y>(x, y);
diff --git a/src/java/org/apache/cassandra/utils/SlidingTimeRate.java b/src/java/org/apache/cassandra/utils/SlidingTimeRate.java
index 3053a05..5e3936e 100644
--- a/src/java/org/apache/cassandra/utils/SlidingTimeRate.java
+++ b/src/java/org/apache/cassandra/utils/SlidingTimeRate.java
@@ -42,9 +42,10 @@
/**
* Creates a sliding rate whose time window is of the given size, with the given precision and time unit.
- * <br/>
+ * <p>
* The precision defines how accurate the rate computation is, as it will be computed over window size +/-
* precision.
+ * </p>
*/
public SlidingTimeRate(TimeSource timeSource, long size, long precision, TimeUnit unit)
{
diff --git a/src/java/org/apache/cassandra/utils/SortedBiMultiValMap.java b/src/java/org/apache/cassandra/utils/SortedBiMultiValMap.java
index e8bcee1..44ac0a0 100644
--- a/src/java/org/apache/cassandra/utils/SortedBiMultiValMap.java
+++ b/src/java/org/apache/cassandra/utils/SortedBiMultiValMap.java
@@ -18,7 +18,6 @@
package org.apache.cassandra.utils;
import java.util.Collection;
-import java.util.Comparator;
import java.util.SortedMap;
import java.util.TreeMap;
@@ -27,9 +26,6 @@
public class SortedBiMultiValMap<K, V> extends BiMultiValMap<K, V>
{
- @SuppressWarnings("unchecked")
- private static final Comparator DEFAULT_COMPARATOR = (o1, o2) -> ((Comparable) o1).compareTo(o2);
-
protected SortedBiMultiValMap(SortedMap<K, V> forwardMap, SortedSetMultimap<V, K> reverseMap)
{
super(forwardMap, reverseMap);
@@ -40,41 +36,15 @@
return new SortedBiMultiValMap<K, V>(new TreeMap<K,V>(), TreeMultimap.<V, K>create());
}
- public static <K, V> SortedBiMultiValMap<K, V> create(Comparator<K> keyComparator, Comparator<V> valueComparator)
- {
- if (keyComparator == null)
- keyComparator = defaultComparator();
- if (valueComparator == null)
- valueComparator = defaultComparator();
- return new SortedBiMultiValMap<K, V>(new TreeMap<K,V>(keyComparator), TreeMultimap.<V, K>create(valueComparator, keyComparator));
- }
-
public static <K extends Comparable<K>, V extends Comparable<V>> SortedBiMultiValMap<K, V> create(BiMultiValMap<K, V> map)
{
SortedBiMultiValMap<K, V> newMap = SortedBiMultiValMap.<K,V>create();
- copy(map, newMap);
- return newMap;
- }
-
- public static <K, V> SortedBiMultiValMap<K, V> create(BiMultiValMap<K, V> map, Comparator<K> keyComparator, Comparator<V> valueComparator)
- {
- SortedBiMultiValMap<K, V> newMap = create(keyComparator, valueComparator);
- copy(map, newMap);
- return newMap;
- }
-
- private static <K, V> void copy(BiMultiValMap<K, V> map, BiMultiValMap<K, V> newMap)
- {
newMap.forwardMap.putAll(map.forwardMap);
// Put each individual TreeSet instead of Multimap#putAll(Multimap) to get linear complexity
// See CASSANDRA-14660
for (Entry<V, Collection<K>> entry : map.inverse().asMap().entrySet())
newMap.reverseMap.putAll(entry.getKey(), entry.getValue());
+ return newMap;
}
- @SuppressWarnings("unchecked")
- private static <T> Comparator<T> defaultComparator()
- {
- return DEFAULT_COMPARATOR;
- }
}
diff --git a/src/java/org/apache/cassandra/utils/StatusLogger.java b/src/java/org/apache/cassandra/utils/StatusLogger.java
index c33190b..dcb1135 100644
--- a/src/java/org/apache/cassandra/utils/StatusLogger.java
+++ b/src/java/org/apache/cassandra/utils/StatusLogger.java
@@ -17,13 +17,12 @@
*/
package org.apache.cassandra.utils;
-import java.lang.management.ManagementFactory;
-import java.util.Map;
-import javax.management.*;
+import java.util.concurrent.locks.ReentrantLock;
import org.apache.cassandra.cache.*;
-
+import org.apache.cassandra.metrics.CassandraMetricsRegistry;
import org.apache.cassandra.metrics.ThreadPoolMetrics;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -37,24 +36,43 @@
public class StatusLogger
{
private static final Logger logger = LoggerFactory.getLogger(StatusLogger.class);
-
+ private static final ReentrantLock busyMonitor = new ReentrantLock();
public static void log()
{
- MBeanServer server = ManagementFactory.getPlatformMBeanServer();
-
- // everything from o.a.c.concurrent
- logger.info(String.format("%-25s%10s%10s%15s%10s%18s", "Pool Name", "Active", "Pending", "Completed", "Blocked", "All Time Blocked"));
-
- for (Map.Entry<String, String> tpool : ThreadPoolMetrics.getJmxThreadPools(server).entries())
+ // avoid logging more than once at the same time. throw away any attempts to log concurrently, as it would be
+ // confusing and noisy for operators - and don't bother logging again, immediately as it'll just be the same data
+ if (busyMonitor.tryLock())
{
- logger.info(String.format("%-25s%10s%10s%15s%10s%18s%n",
- tpool.getValue(),
- ThreadPoolMetrics.getJmxMetric(server, tpool.getKey(), tpool.getValue(), "ActiveTasks"),
- ThreadPoolMetrics.getJmxMetric(server, tpool.getKey(), tpool.getValue(), "PendingTasks"),
- ThreadPoolMetrics.getJmxMetric(server, tpool.getKey(), tpool.getValue(), "CompletedTasks"),
- ThreadPoolMetrics.getJmxMetric(server, tpool.getKey(), tpool.getValue(), "CurrentlyBlockedTasks"),
- ThreadPoolMetrics.getJmxMetric(server, tpool.getKey(), tpool.getValue(), "TotalBlockedTasks")));
+ try
+ {
+ logStatus();
+ }
+ finally
+ {
+ busyMonitor.unlock();
+ }
+ }
+ else
+ {
+ logger.trace("StatusLogger is busy");
+ }
+ }
+
+ private static void logStatus()
+ {
+ // everything from o.a.c.concurrent
+ logger.info(String.format("%-28s%10s%10s%15s%10s%18s", "Pool Name", "Active", "Pending", "Completed", "Blocked", "All Time Blocked"));
+
+ for (ThreadPoolMetrics tpool : CassandraMetricsRegistry.Metrics.allThreadPoolMetrics())
+ {
+ logger.info(String.format("%-28s%10s%10s%15s%10s%18s",
+ tpool.poolName,
+ tpool.activeTasks.getValue(),
+ tpool.pendingTasks.getValue(),
+ tpool.completedTasks.getValue(),
+ tpool.currentBlocked.getCount(),
+ tpool.totalBlocked.getCount()));
}
// one offs
diff --git a/src/java/org/apache/cassandra/utils/StreamingHistogram.java b/src/java/org/apache/cassandra/utils/StreamingHistogram.java
deleted file mode 100644
index df49d8d..0000000
--- a/src/java/org/apache/cassandra/utils/StreamingHistogram.java
+++ /dev/null
@@ -1,337 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.utils;
-
-import java.io.IOException;
-import java.util.*;
-
-import com.google.common.base.Objects;
-
-import org.apache.cassandra.db.TypeSizes;
-import org.apache.cassandra.io.ISerializer;
-import org.apache.cassandra.io.sstable.SSTable;
-import org.apache.cassandra.io.util.DataInputPlus;
-import org.apache.cassandra.io.util.DataOutputPlus;
-
-/**
- * Histogram that can be constructed from streaming of data.
- *
- * The algorithm is taken from following paper:
- * Yael Ben-Haim and Elad Tom-Tov, "A Streaming Parallel Decision Tree Algorithm" (2010)
- * http://jmlr.csail.mit.edu/papers/volume11/ben-haim10a/ben-haim10a.pdf
- */
-public class StreamingHistogram
-{
- public static final StreamingHistogramSerializer serializer = new StreamingHistogramSerializer();
-
- // TreeMap to hold bins of histogram.
- // The key is a numeric type so we can avoid boxing/unboxing streams of different key types
- // The value is a unboxed long array always of length == 1
- // Serialized Histograms always writes with double keys for backwards compatibility
- private final TreeMap<Number, long[]> bin;
-
- // maximum bin size for this histogram
- private final int maxBinSize;
-
-
- /**
- * Creates a new histogram with max bin size of maxBinSize
- * @param maxBinSize maximum number of bins this histogram can have
- * @param source the existing bins in map form
- */
- private StreamingHistogram(int maxBinSize, Map<Number, long[]> source)
- {
- this.maxBinSize = maxBinSize;
- this.bin = new TreeMap<>((o1, o2) -> {
- if (o1.getClass().equals(o2.getClass()))
- return ((Comparable)o1).compareTo(o2);
- else
- return Double.compare(o1.doubleValue(), o2.doubleValue());
- });
- for (Map.Entry<Number, long[]> entry : source.entrySet())
- this.bin.put(entry.getKey(), new long[]{entry.getValue()[0]});
- }
-
- /**
- * Calculates estimated number of points in interval [-inf,b].
- *
- * @param b upper bound of a interval to calculate sum
- * @return estimated number of points in a interval [-inf,b].
- */
- public double sum(double b)
- {
- double sum = 0;
- // find the points pi, pnext which satisfy pi <= b < pnext
- Map.Entry<Number, long[]> pnext = bin.higherEntry(b);
- if (pnext == null)
- {
- // if b is greater than any key in this histogram,
- // just count all appearance and return
- for (long[] value : bin.values())
- sum += value[0];
- }
- else
- {
- Map.Entry<Number, long[]> pi = bin.floorEntry(b);
- if (pi == null)
- return 0;
- // calculate estimated count mb for point b
- double weight = (b - pi.getKey().doubleValue()) / (pnext.getKey().doubleValue() - pi.getKey().doubleValue());
- double mb = pi.getValue()[0] + (pnext.getValue()[0] - pi.getValue()[0]) * weight;
- sum += (pi.getValue()[0] + mb) * weight / 2;
-
- sum += pi.getValue()[0] / 2.0;
- for (long[] value : bin.headMap(pi.getKey(), false).values())
- sum += value[0];
- }
- return sum;
- }
-
- public Map<Number, long[]> getAsMap()
- {
- return Collections.unmodifiableMap(bin);
- }
-
- public static class StreamingHistogramBuilder
- {
- // TreeMap to hold bins of histogram.
- // The key is a numeric type so we can avoid boxing/unboxing streams of different key types
- // The value is a unboxed long array always of length == 1
- // Serialized Histograms always writes with double keys for backwards compatibility
- private final TreeMap<Number, long[]> bin;
-
- // Keep a second, larger buffer to spool data in, before finalizing it into `bin`
- private final TreeMap<Number, long[]> spool;
-
- // maximum bin size for this histogram
- private final int maxBinSize;
-
- // maximum size of the spool
- private final int maxSpoolSize;
-
- // voluntarily give up resolution for speed
- private final int roundSeconds;
-
- /**
- * Creates a new histogram with max bin size of maxBinSize
- * @param maxBinSize maximum number of bins this histogram can have
- */
- public StreamingHistogramBuilder(int maxBinSize, int maxSpoolSize, int roundSeconds)
- {
- this.maxBinSize = maxBinSize;
- this.maxSpoolSize = maxSpoolSize;
- this.roundSeconds = roundSeconds;
- bin = new TreeMap<>((o1, o2) -> {
- if (o1.getClass().equals(o2.getClass()))
- return ((Comparable)o1).compareTo(o2);
- else
- return Double.compare(o1.doubleValue(), o2.doubleValue());
- });
- spool = new TreeMap<>((o1, o2) -> {
- if (o1.getClass().equals(o2.getClass()))
- return ((Comparable)o1).compareTo(o2);
- else
- return Double.compare(o1.doubleValue(), o2.doubleValue());
- });
-
- }
-
- public StreamingHistogram build()
- {
- flushHistogram();
- return new StreamingHistogram(maxBinSize, bin);
- }
-
- /**
- * Adds new point p to this histogram.
- * @param p
- */
- public void update(Number p)
- {
- update(p, 1L);
- }
-
- /**
- * Adds new point p with value m to this histogram.
- * @param p
- * @param m
- */
- public void update(Number p, long m)
- {
- Number d = p.longValue() % this.roundSeconds;
- if (d.longValue() > 0)
- p =p.longValue() + (this.roundSeconds - d.longValue());
-
- long[] mi = spool.get(p);
- if (mi != null)
- {
- // we found the same p so increment that counter
- mi[0] += m;
- }
- else
- {
- mi = new long[]{m};
- spool.put(p, mi);
- }
-
- // If spool has overflowed, compact it
- if(spool.size() > maxSpoolSize)
- flushHistogram();
- }
-
-
-
- /**
- * Drain the temporary spool into the final bins
- */
- public void flushHistogram()
- {
- if (spool.size() > 0)
- {
- long[] spoolValue;
- long[] binValue;
-
- // Iterate over the spool, copying the value into the primary bin map
- // and compacting that map as necessary
- for (Map.Entry<Number, long[]> entry : spool.entrySet())
- {
- Number key = entry.getKey();
- spoolValue = entry.getValue();
- binValue = bin.get(key);
-
- // If this value is already in the final histogram bins
- // Simply increment and update, otherwise, insert a new long[1] value
- if(binValue != null)
- {
- binValue[0] += spoolValue[0];
- bin.put(key, binValue);
- }
- else
- {
- bin.put(key, new long[]{spoolValue[0]});
- }
-
- if (bin.size() > maxBinSize)
- {
- // find points p1, p2 which have smallest difference
- Iterator<Number> keys = bin.keySet().iterator();
- double p1 = keys.next().doubleValue();
- double p2 = keys.next().doubleValue();
- double smallestDiff = p2 - p1;
- double q1 = p1, q2 = p2;
- while (keys.hasNext())
- {
- p1 = p2;
- p2 = keys.next().doubleValue();
- double diff = p2 - p1;
- if (diff < smallestDiff)
- {
- smallestDiff = diff;
- q1 = p1;
- q2 = p2;
- }
- }
- // merge those two
- long[] a1 = bin.remove(q1);
- long[] a2 = bin.remove(q2);
- long k1 = a1[0];
- long k2 = a2[0];
-
- a1[0] += k2;
- bin.put((q1 * k1 + q2 * k2) / (k1 + k2), a1);
-
- }
- }
- spool.clear();
- }
- }
-
- /**
- * Merges given histogram with this histogram.
- *
- * @param other histogram to merge
- */
- public void merge(StreamingHistogram other)
- {
- if (other == null)
- return;
-
- for (Map.Entry<Number, long[]> entry : other.getAsMap().entrySet())
- update(entry.getKey(), entry.getValue()[0]);
- }
- }
-
- public static class StreamingHistogramSerializer implements ISerializer<StreamingHistogram>
- {
- public void serialize(StreamingHistogram histogram, DataOutputPlus out) throws IOException
- {
- out.writeInt(histogram.maxBinSize);
- Map<Number, long[]> entries = histogram.getAsMap();
- out.writeInt(entries.size());
- for (Map.Entry<Number, long[]> entry : entries.entrySet())
- {
- out.writeDouble(entry.getKey().doubleValue());
- out.writeLong(entry.getValue()[0]);
- }
- }
-
- public StreamingHistogram deserialize(DataInputPlus in) throws IOException
- {
- int maxBinSize = in.readInt();
- int size = in.readInt();
- Map<Number, long[]> tmp = new HashMap<>(size);
- for (int i = 0; i < size; i++)
- {
- tmp.put(in.readDouble(), new long[]{in.readLong()});
- }
-
- return new StreamingHistogram(maxBinSize, tmp);
- }
-
- public long serializedSize(StreamingHistogram histogram)
- {
- long size = TypeSizes.sizeof(histogram.maxBinSize);
- Map<Number, long[]> entries = histogram.getAsMap();
- size += TypeSizes.sizeof(entries.size());
- // size of entries = size * (8(double) + 8(long))
- size += entries.size() * (8L + 8L);
- return size;
- }
- }
-
- @Override
- public boolean equals(Object o)
- {
- if (this == o)
- return true;
-
- if (!(o instanceof StreamingHistogram))
- return false;
-
- StreamingHistogram that = (StreamingHistogram) o;
- return maxBinSize == that.maxBinSize &&
- bin.equals(that.bin);
- }
-
- @Override
- public int hashCode()
- {
- return Objects.hashCode(bin.hashCode(), maxBinSize);
- }
-
-}
diff --git a/src/java/org/apache/cassandra/utils/SyncUtil.java b/src/java/org/apache/cassandra/utils/SyncUtil.java
index 64d64cf..1917e8b 100644
--- a/src/java/org/apache/cassandra/utils/SyncUtil.java
+++ b/src/java/org/apache/cassandra/utils/SyncUtil.java
@@ -28,8 +28,11 @@
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.cassandra.config.Config;
+import org.apache.cassandra.service.CassandraDaemon;
import com.google.common.base.Preconditions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/*
* A wrapper around various mechanisms for syncing files that makes it possible it intercept
@@ -38,12 +41,14 @@
*/
public class SyncUtil
{
- public static boolean SKIP_SYNC = Boolean.getBoolean(Config.PROPERTY_PREFIX + "skip_sync");
+ public static final boolean SKIP_SYNC;
private static final Field mbbFDField;
private static final Field fdClosedField;
private static final Field fdUseCountField;
+ private static final Logger logger = LoggerFactory.getLogger(SyncUtil.class);
+
static
{
Field mbbFDFieldTemp = null;
@@ -80,6 +85,15 @@
{
}
fdUseCountField = fdUseCountTemp;
+
+ //If skipping syncing is requested by any means then skip them.
+ boolean skipSyncProperty = Boolean.getBoolean(Config.PROPERTY_PREFIX + "skip_sync");
+ boolean skipSyncEnv = Boolean.valueOf(System.getenv().getOrDefault("CASSANDRA_SKIP_SYNC", "false"));
+ SKIP_SYNC = skipSyncProperty || skipSyncEnv;
+ if (SKIP_SYNC)
+ {
+ logger.info("Skip fsync enabled due to property {} and environment {}", skipSyncProperty, skipSyncEnv);
+ }
}
public static MappedByteBuffer force(MappedByteBuffer buf)
diff --git a/src/java/org/apache/cassandra/utils/Throwables.java b/src/java/org/apache/cassandra/utils/Throwables.java
index 5ad9686..f727b5a 100644
--- a/src/java/org/apache/cassandra/utils/Throwables.java
+++ b/src/java/org/apache/cassandra/utils/Throwables.java
@@ -20,9 +20,13 @@
import java.io.File;
import java.io.IOException;
+import java.lang.reflect.InvocationTargetException;
import java.util.Arrays;
import java.util.Iterator;
import java.util.Optional;
+import java.util.concurrent.CompletionException;
+import java.util.concurrent.ExecutionException;
+import java.util.function.Predicate;
import java.util.stream.Stream;
import org.apache.cassandra.io.FSReadError;
@@ -37,6 +41,11 @@
void perform() throws E;
}
+ public static boolean isCausedBy(Throwable t, Predicate<Throwable> cause)
+ {
+ return cause.test(t) || (t.getCause() != null && cause.test(t.getCause()));
+ }
+
public static <T extends Throwable> T merge(T existingFail, T newFail)
{
if (existingFail == null)
@@ -75,9 +84,15 @@
}
@SafeVarargs
+ public static <E extends Exception> void maybeFail(DiscreteAction<? extends E> ... actions)
+ {
+ maybeFail(Throwables.perform(null, Stream.of(actions)));
+ }
+
+ @SafeVarargs
public static <E extends Exception> void perform(DiscreteAction<? extends E> ... actions) throws E
{
- perform(Stream.of(actions));
+ Throwables.<E>perform(Stream.of(actions));
}
public static <E extends Exception> void perform(Stream<? extends DiscreteAction<? extends E>> stream, DiscreteAction<? extends E> ... extra) throws E
@@ -88,7 +103,7 @@
@SuppressWarnings("unchecked")
public static <E extends Exception> void perform(Stream<DiscreteAction<? extends E>> actions) throws E
{
- Throwable fail = perform((Throwable) null, actions);
+ Throwable fail = perform(null, actions);
if (failIfCanCast(fail, null))
throw (E) fail;
}
@@ -181,4 +196,56 @@
}
return Optional.empty();
}
+
+ /**
+ * If the provided throwable is a "wrapping" exception (see below), return the cause of that throwable, otherwise
+ * return its argument untouched.
+ * <p>
+ * We call a "wrapping" exception in the context of that method an exception whose only purpose is to wrap another
+ * exception, and currently this method recognize only 2 exception as "wrapping" ones: {@link ExecutionException}
+ * and {@link CompletionException}.
+ */
+ public static Throwable unwrapped(Throwable t)
+ {
+ Throwable unwrapped = t;
+ while (unwrapped instanceof CompletionException ||
+ unwrapped instanceof ExecutionException ||
+ unwrapped instanceof InvocationTargetException)
+ unwrapped = unwrapped.getCause();
+
+ // I don't think it make sense for those 2 exception classes to ever be used with null causes, but no point
+ // in failing here if this happen. We still wrap the original exception if that happen so we get a sign
+ // that the assumption of this method is wrong.
+ return unwrapped == null
+ ? new RuntimeException("Got wrapping exception not wrapping anything", t)
+ : unwrapped;
+ }
+
+ /**
+ * If the provided exception is unchecked, return it directly, otherwise wrap it into a {@link RuntimeException}
+ * to make it unchecked.
+ */
+ public static RuntimeException unchecked(Throwable t)
+ {
+ return t instanceof RuntimeException ? (RuntimeException)t : new RuntimeException(t);
+ }
+
+ /**
+ * throw the exception as a unchecked exception, wrapping if a checked exception, else rethroing as is.
+ */
+ public static RuntimeException throwAsUncheckedException(Throwable t)
+ {
+ if (t instanceof Error)
+ throw (Error) t;
+ throw unchecked(t);
+ }
+
+ /**
+ * A shortcut for {@code unchecked(unwrapped(t))}. This is called "cleaned" because this basically removes the annoying
+ * cruft surrounding an exception :).
+ */
+ public static RuntimeException cleaned(Throwable t)
+ {
+ return unchecked(unwrapped(t));
+ }
}
diff --git a/src/java/org/apache/cassandra/utils/TopKSampler.java b/src/java/org/apache/cassandra/utils/TopKSampler.java
deleted file mode 100644
index 37fcb60..0000000
--- a/src/java/org/apache/cassandra/utils/TopKSampler.java
+++ /dev/null
@@ -1,139 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.utils;
-
-import java.io.Serializable;
-import java.util.*;
-import java.util.concurrent.*;
-
-import org.apache.cassandra.concurrent.*;
-import org.slf4j.*;
-
-import com.clearspring.analytics.stream.*;
-import com.clearspring.analytics.stream.cardinality.HyperLogLogPlus;
-import com.google.common.annotations.VisibleForTesting;
-
-public class TopKSampler<T>
-{
- private static final Logger logger = LoggerFactory.getLogger(TopKSampler.class);
- private volatile boolean enabled = false;
-
- @VisibleForTesting
- static final ThreadPoolExecutor samplerExecutor = new JMXEnabledThreadPoolExecutor(1, 1,
- TimeUnit.SECONDS,
- new LinkedBlockingQueue<Runnable>(),
- new NamedThreadFactory("Sampler"),
- "internal");
-
- private StreamSummary<T> summary;
- @VisibleForTesting
- HyperLogLogPlus hll;
-
- /**
- * Start to record samples
- *
- * @param capacity
- * Number of sample items to keep in memory, the lower this is
- * the less accurate results are. For best results use value
- * close to cardinality, but understand the memory trade offs.
- */
- public synchronized void beginSampling(int capacity)
- {
- if (!enabled)
- {
- summary = new StreamSummary<T>(capacity);
- hll = new HyperLogLogPlus(14);
- enabled = true;
- }
- }
-
- /**
- * Call to stop collecting samples, and gather the results
- * @param count Number of most frequent items to return
- */
- public synchronized SamplerResult<T> finishSampling(int count)
- {
- List<Counter<T>> results = Collections.EMPTY_LIST;
- long cardinality = 0;
- if (enabled)
- {
- enabled = false;
- results = summary.topK(count);
- cardinality = hll.cardinality();
- }
- return new SamplerResult<T>(results, cardinality);
- }
-
- public void addSample(T item)
- {
- addSample(item, item.hashCode(), 1);
- }
-
- /**
- * Adds a sample to statistics collection. This method is non-blocking and will
- * use the "Sampler" thread pool to record results if the sampler is enabled. If not
- * sampling this is a NOOP
- */
- public void addSample(final T item, final long hash, final int value)
- {
- if (enabled)
- {
- final Object lock = this;
- samplerExecutor.execute(new Runnable()
- {
- public void run()
- {
- // samplerExecutor is single threaded but still need
- // synchronization against jmx calls to finishSampling
- synchronized (lock)
- {
- if (enabled)
- {
- try
- {
- summary.offer(item, value);
- hll.offerHashed(hash);
- } catch (Exception e)
- {
- logger.trace("Failure to offer sample", e);
- }
- }
- }
- }
- });
- }
- }
-
- /**
- * Represents the cardinality and the topK ranked items collected during a
- * sample period
- */
- public static class SamplerResult<S> implements Serializable
- {
- public final List<Counter<S>> topK;
- public final long cardinality;
-
- public SamplerResult(List<Counter<S>> topK, long cardinality)
- {
- this.topK = topK;
- this.cardinality = cardinality;
- }
- }
-
-}
-
diff --git a/src/java/org/apache/cassandra/utils/UUIDGen.java b/src/java/org/apache/cassandra/utils/UUIDGen.java
index 8fac816..c83e292 100644
--- a/src/java/org/apache/cassandra/utils/UUIDGen.java
+++ b/src/java/org/apache/cassandra/utils/UUIDGen.java
@@ -18,19 +18,31 @@
package org.apache.cassandra.utils;
import java.net.InetAddress;
+import java.net.NetworkInterface;
+import java.net.SocketException;
import java.nio.ByteBuffer;
-import java.security.MessageDigest;
-import java.security.NoSuchAlgorithmException;
import java.security.SecureRandom;
import java.util.Collection;
+import java.util.Collections;
+import java.util.Enumeration;
+import java.util.HashSet;
+import java.util.List;
import java.util.Random;
+import java.util.Set;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.TimeUnit;
+import java.util.function.Function;
+import java.util.stream.Collectors;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.hash.Hasher;
+import com.google.common.hash.Hashing;
import com.google.common.primitives.Ints;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.locator.InetAddressAndPort;
+
/**
* The goods are here: www.ietf.org/rfc/rfc4122.txt.
*/
@@ -40,6 +52,8 @@
private static final long START_EPOCH = -12219292800000L;
private static final long clockSeqAndNode = makeClockSeqAndNode();
+ public static final int UUID_LEN = 16;
+
/*
* The min and max possible lsb for a UUID.
* Note that his is not 0 and all 1's because Cassandra TimeUUIDType
@@ -106,10 +120,10 @@
}
/**
- * Similar to {@link getTimeUUIDFromMicros}, but randomize (using SecureRandom) the clock and sequence.
+ * Similar to {@link #getTimeUUIDFromMicros}, but randomize (using SecureRandom) the clock and sequence.
* <p>
* If you can guarantee that the {@code whenInMicros} argument is unique (for this JVM instance) for
- * every call, then you should prefer {@link getTimeUUIDFromMicros} which is faster. If you can't
+ * every call, then you should prefer {@link #getTimeUUIDFromMicros} which is faster. If you can't
* guarantee this however, this method will ensure the returned UUID are still unique (accross calls)
* through randomization.
*
@@ -143,7 +157,7 @@
public static ByteBuffer toByteBuffer(UUID uuid)
{
- ByteBuffer buffer = ByteBuffer.allocate(16);
+ ByteBuffer buffer = ByteBuffer.allocate(UUID_LEN);
buffer.putLong(uuid.getMostSignificantBits());
buffer.putLong(uuid.getLeastSignificantBits());
buffer.flip();
@@ -354,12 +368,12 @@
* The spec says that one option is to take as many source that identify
* this node as possible and hash them together. That's what we do here by
* gathering all the ip of this host.
- * Note that FBUtilities.getBroadcastAddress() should be enough to uniquely
+ * Note that FBUtilities.getJustBroadcastAddress() should be enough to uniquely
* identify the node *in the cluster* but it triggers DatabaseDescriptor
* instanciation and the UUID generator is used in Stress for instance,
* where we don't want to require the yaml.
*/
- Collection<InetAddress> localAddresses = FBUtilities.getAllLocalAddresses();
+ Collection<InetAddressAndPort> localAddresses = getAllLocalAddresses();
if (localAddresses.isEmpty())
throw new RuntimeException("Cannot generate the node component of the UUID because cannot retrieve any IP addresses.");
@@ -375,32 +389,83 @@
return node | 0x0000010000000000L;
}
- private static byte[] hash(Collection<InetAddress> data)
+ private static byte[] hash(Collection<InetAddressAndPort> data)
{
+ // Identify the host.
+ Hasher hasher = Hashing.md5().newHasher();
+ for(InetAddressAndPort addr : data)
+ {
+ hasher.putBytes(addr.addressBytes);
+ hasher.putInt(addr.port);
+ }
+
+ // Identify the process on the load: we use both the PID and class loader hash.
+ long pid = NativeLibrary.getProcessID();
+ if (pid < 0)
+ pid = new Random(System.currentTimeMillis()).nextLong();
+ updateWithLong(hasher, pid);
+
+ ClassLoader loader = UUIDGen.class.getClassLoader();
+ int loaderId = loader != null ? System.identityHashCode(loader) : 0;
+ updateWithInt(hasher, loaderId);
+
+ return hasher.hash().asBytes();
+ }
+
+ private static void updateWithInt(Hasher hasher, int val)
+ {
+ hasher.putByte((byte) ((val >>> 24) & 0xFF));
+ hasher.putByte((byte) ((val >>> 16) & 0xFF));
+ hasher.putByte((byte) ((val >>> 8) & 0xFF));
+ hasher.putByte((byte) ((val >>> 0) & 0xFF));
+ }
+
+ public static void updateWithLong(Hasher hasher, long val)
+ {
+ hasher.putByte((byte) ((val >>> 56) & 0xFF));
+ hasher.putByte((byte) ((val >>> 48) & 0xFF));
+ hasher.putByte((byte) ((val >>> 40) & 0xFF));
+ hasher.putByte((byte) ((val >>> 32) & 0xFF));
+ hasher.putByte((byte) ((val >>> 24) & 0xFF));
+ hasher.putByte((byte) ((val >>> 16) & 0xFF));
+ hasher.putByte((byte) ((val >>> 8) & 0xFF));
+ hasher.putByte((byte) ((val >>> 0) & 0xFF));
+ }
+
+ /**
+ * Helper function used exclusively by UUIDGen to create
+ **/
+ public static Collection<InetAddressAndPort> getAllLocalAddresses()
+ {
+ Set<InetAddressAndPort> localAddresses = new HashSet<>();
try
{
- // Identify the host.
- MessageDigest messageDigest = MessageDigest.getInstance("MD5");
- for(InetAddress addr : data)
- messageDigest.update(addr.getAddress());
-
- // Identify the process on the load: we use both the PID and class loader hash.
- long pid = NativeLibrary.getProcessID();
- if (pid < 0)
- pid = new Random(System.currentTimeMillis()).nextLong();
- FBUtilities.updateWithLong(messageDigest, pid);
-
- ClassLoader loader = UUIDGen.class.getClassLoader();
- int loaderId = loader != null ? System.identityHashCode(loader) : 0;
- FBUtilities.updateWithInt(messageDigest, loaderId);
-
- return messageDigest.digest();
+ Enumeration<NetworkInterface> nets = NetworkInterface.getNetworkInterfaces();
+ if (nets != null)
+ {
+ while (nets.hasMoreElements())
+ {
+ Function<InetAddress, InetAddressAndPort> converter =
+ address -> InetAddressAndPort.getByAddressOverrideDefaults(address, 0);
+ List<InetAddressAndPort> addresses =
+ Collections.list(nets.nextElement().getInetAddresses()).stream().map(converter).collect(Collectors.toList());
+ localAddresses.addAll(addresses);
+ }
+ }
}
- catch (NoSuchAlgorithmException nsae)
+ catch (SocketException e)
{
- throw new RuntimeException("MD5 digest algorithm is not available", nsae);
+ throw new AssertionError(e);
}
+ if (DatabaseDescriptor.isDaemonInitialized())
+ {
+ localAddresses.add(FBUtilities.getBroadcastAddressAndPort());
+ localAddresses.add(FBUtilities.getBroadcastNativeAddressAndPort());
+ localAddresses.add(FBUtilities.getLocalAddressAndPort());
+ }
+ return localAddresses;
}
+
}
// for the curious, here is how I generated START_EPOCH
diff --git a/src/java/org/apache/cassandra/utils/WrappedInt.java b/src/java/org/apache/cassandra/utils/WrappedInt.java
deleted file mode 100644
index a106575..0000000
--- a/src/java/org/apache/cassandra/utils/WrappedInt.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.utils;
-
-/**
- * Simple wrapper for native int type
- */
-public class WrappedInt
-{
- private int value;
-
- public WrappedInt(int initial)
- {
- this.value = initial;
- }
-
- public int get()
- {
- return value;
- }
-
- public void set(int value)
- {
- this.value = value;
- }
-
- public void increment()
- {
- ++value;
- }
-
- public void decrement()
- {
- --value;
- }
-}
diff --git a/src/java/org/apache/cassandra/utils/binlog/BinLog.java b/src/java/org/apache/cassandra/utils/binlog/BinLog.java
new file mode 100644
index 0000000..7f91761
--- /dev/null
+++ b/src/java/org/apache/cassandra/utils/binlog/BinLog.java
@@ -0,0 +1,479 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.utils.binlog;
+
+import java.io.File;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Strings;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import net.openhft.chronicle.queue.ChronicleQueue;
+import net.openhft.chronicle.queue.ChronicleQueueBuilder;
+import net.openhft.chronicle.queue.ExcerptAppender;
+import net.openhft.chronicle.queue.RollCycle;
+import net.openhft.chronicle.queue.RollCycles;
+import net.openhft.chronicle.wire.WireOut;
+import net.openhft.chronicle.wire.WriteMarshallable;
+import org.apache.cassandra.concurrent.NamedThreadFactory;
+import org.apache.cassandra.io.FSError;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.utils.NoSpamLogger;
+import org.apache.cassandra.utils.Throwables;
+import org.apache.cassandra.utils.concurrent.WeightedQueue;
+
+/**
+ * Bin log is a is quick and dirty binary log that is kind of a NIH version of binary logging with a traditional logging
+ * framework. It's goal is good enough performance, predictable footprint, simplicity in terms of implementation and configuration
+ * and most importantly minimal impact on producers of log records.
+ *
+ * Performance safety is accomplished by feeding items to the binary log using a weighted queue and dropping records if the binary log falls
+ * sufficiently far behind.
+ *
+ * Simplicity and good enough perforamance is achieved by using a single log writing thread as well as Chronicle Queue
+ * to handle writing the log, making it available for readers, as well as log rolling.
+ *
+ */
+public class BinLog implements Runnable
+{
+ private static final Logger logger = LoggerFactory.getLogger(BinLog.class);
+ private static final NoSpamLogger noSpamLogger = NoSpamLogger.getLogger(logger, 1, TimeUnit.MINUTES);
+ private static final NoSpamLogger.NoSpamLogStatement droppedSamplesStatement = noSpamLogger.getStatement("Dropped {} binary log samples", 1, TimeUnit.MINUTES);
+
+ public final Path path;
+
+ public static final String VERSION = "version";
+ public static final String TYPE = "type";
+
+ private ChronicleQueue queue;
+ private ExcerptAppender appender;
+ @VisibleForTesting
+ Thread binLogThread = new NamedThreadFactory("Binary Log thread").newThread(this);
+ final WeightedQueue<ReleaseableWriteMarshallable> sampleQueue;
+ private final BinLogArchiver archiver;
+ private final boolean blocking;
+
+ private final AtomicLong droppedSamplesSinceLastLog = new AtomicLong();
+
+ /*
+ This set contains all the paths we are currently logging to, it is used to make sure
+ we don't start writing audit and full query logs to the same path.
+ */
+ private static final Set<Path> currentPaths = Collections.synchronizedSet(new HashSet<>());
+
+ private static final ReleaseableWriteMarshallable NO_OP = new ReleaseableWriteMarshallable()
+ {
+ @Override
+ protected long version()
+ {
+ return 0;
+ }
+
+ @Override
+ protected String type()
+ {
+ return "no-op";
+ }
+
+ @Override
+ public void writeMarshallablePayload(WireOut wire)
+ {
+ }
+
+ @Override
+ public void release()
+ {
+ }
+ };
+
+ private volatile boolean shouldContinue = true;
+
+ /**
+ * @param path Path to store the BinLog. Can't be shared with anything else.
+ * @param rollCycle How often to roll the log file so it can potentially be deleted
+ * @param maxQueueWeight Maximum weight of in memory queue for records waiting to be written to the file before blocking or dropping
+ */
+ private BinLog(Path path, RollCycle rollCycle, int maxQueueWeight, BinLogArchiver archiver, boolean blocking)
+ {
+ Preconditions.checkNotNull(path, "path was null");
+ Preconditions.checkNotNull(rollCycle, "rollCycle was null");
+ Preconditions.checkArgument(maxQueueWeight > 0, "maxQueueWeight must be > 0");
+ ChronicleQueueBuilder builder = ChronicleQueueBuilder.single(path.toFile());
+ builder.rollCycle(rollCycle);
+
+ sampleQueue = new WeightedQueue<>(maxQueueWeight);
+ this.archiver = archiver;
+ builder.storeFileListener(this.archiver);
+ queue = builder.build();
+ appender = queue.acquireAppender();
+ this.blocking = blocking;
+ this.path = path;
+ }
+
+ /**
+ * Start the consumer thread that writes log records. Can only be done once.
+ */
+ @VisibleForTesting
+ void start()
+ {
+ if (!shouldContinue)
+ {
+ throw new IllegalStateException("Can't reuse stopped BinLog");
+ }
+ binLogThread.start();
+ }
+
+ /**
+ * Stop the consumer thread that writes log records. Can be called multiple times.
+ * @throws InterruptedException
+ */
+ public synchronized void stop() throws InterruptedException
+ {
+ if (!shouldContinue)
+ {
+ return;
+ }
+
+ shouldContinue = false;
+ sampleQueue.put(NO_OP);
+ binLogThread.join();
+ appender = null;
+ queue = null;
+ archiver.stop();
+ currentPaths.remove(path);
+ }
+
+ /**
+ * Offer a record to the log. If the in memory queue is full the record will be dropped and offer will return false.
+ * @param record The record to write to the log
+ * @return true if the record was queued and false otherwise
+ */
+ public boolean offer(ReleaseableWriteMarshallable record)
+ {
+ if (!shouldContinue)
+ {
+ return false;
+ }
+
+ return sampleQueue.offer(record);
+ }
+
+ /**
+ * Put a record into the log. If the in memory queue is full the putting thread will be blocked until there is space or it is interrupted.
+ * @param record The record to write to the log
+ * @throws InterruptedException
+ */
+ public void put(ReleaseableWriteMarshallable record) throws InterruptedException
+ {
+ if (!shouldContinue)
+ {
+ return;
+ }
+
+ //Resolve potential deadlock at shutdown when queue is full
+ while (shouldContinue)
+ {
+ if (sampleQueue.offer(record, 1, TimeUnit.SECONDS))
+ {
+ return;
+ }
+ }
+ }
+
+ private void processTasks(List<ReleaseableWriteMarshallable> tasks)
+ {
+ for (int ii = 0; ii < tasks.size(); ii++)
+ {
+ WriteMarshallable t = tasks.get(ii);
+ //Don't write an empty document
+ if (t == NO_OP)
+ {
+ continue;
+ }
+
+ appender.writeDocument(t);
+ }
+ }
+
+ @Override
+ public void run()
+ {
+ List<ReleaseableWriteMarshallable> tasks = new ArrayList<>(16);
+ while (shouldContinue)
+ {
+ try
+ {
+ tasks.clear();
+ ReleaseableWriteMarshallable task = sampleQueue.take();
+ tasks.add(task);
+ sampleQueue.drainTo(tasks, 15);
+
+ processTasks(tasks);
+ }
+ catch (Throwable t)
+ {
+ logger.error("Unexpected exception in binary log thread", t);
+ }
+ finally
+ {
+ for (int ii = 0; ii < tasks.size(); ii++)
+ {
+ tasks.get(ii).release();
+ }
+ }
+ }
+
+ //Clean up the buffers on thread exit, finalization will check again once this
+ //is no longer reachable ensuring there are no stragglers in the queue.
+ finalize();
+ }
+
+
+ /**
+ * There is a race where we might not release a buffer, going to let finalization
+ * catch it since it shouldn't happen to a lot of buffers. Only test code would run
+ * into it anyways.
+ */
+ @Override
+ public void finalize()
+ {
+ ReleaseableWriteMarshallable toRelease;
+ while (((toRelease = sampleQueue.poll()) != null))
+ {
+ toRelease.release();
+ }
+ }
+
+ // todo: refactor to helper class?
+ public void logRecord(ReleaseableWriteMarshallable record)
+ {
+ boolean putInQueue = false;
+ try
+ {
+ if (blocking)
+ {
+ try
+ {
+ put(record);
+ putInQueue = true;
+ }
+ catch (InterruptedException e)
+ {
+ throw new RuntimeException(e);
+ }
+ }
+ else
+ {
+ if (!offer(record))
+ {
+ logDroppedSample();
+ }
+ else
+ {
+ putInQueue = true;
+ }
+ }
+ }
+ finally
+ {
+ if (!putInQueue)
+ {
+ record.release();
+ }
+ }
+ }
+
+ /**
+ * This is potentially lossy, but it's not super critical as we will always generally know
+ * when this is happening and roughly how bad it is.
+ */
+ private void logDroppedSample()
+ {
+ droppedSamplesSinceLastLog.incrementAndGet();
+ if (droppedSamplesStatement.warn(new Object[] {droppedSamplesSinceLastLog.get()}))
+ {
+ droppedSamplesSinceLastLog.set(0);
+ }
+ }
+
+
+ public abstract static class ReleaseableWriteMarshallable implements WriteMarshallable
+ {
+ @Override
+ public final void writeMarshallable(WireOut wire)
+ {
+ wire.write(VERSION).int16(version());
+ wire.write(TYPE).text(type());
+
+ writeMarshallablePayload(wire);
+ }
+
+ protected abstract long version();
+
+ protected abstract String type();
+
+ protected abstract void writeMarshallablePayload(WireOut wire);
+
+ public abstract void release();
+ }
+
+ public static class Builder
+ {
+ private Path path;
+ private String rollCycle;
+ private int maxQueueWeight;
+ private long maxLogSize;
+ private String archiveCommand;
+ private int maxArchiveRetries;
+ private boolean blocking;
+
+ public Builder path(Path path)
+ {
+ Preconditions.checkNotNull(path, "path was null");
+ File pathAsFile = path.toFile();
+ //Exists and is a directory or can be created
+ Preconditions.checkArgument((pathAsFile.exists() && pathAsFile.isDirectory()) || (!pathAsFile.exists() && pathAsFile.mkdirs()), "path exists and is not a directory or couldn't be created");
+ Preconditions.checkArgument(pathAsFile.canRead() && pathAsFile.canWrite() && pathAsFile.canExecute(), "path is not readable, writable, and executable");
+ this.path = path;
+ return this;
+ }
+
+ public Builder rollCycle(String rollCycle)
+ {
+ Preconditions.checkNotNull(rollCycle, "rollCycle was null");
+ rollCycle = rollCycle.toUpperCase();
+ Preconditions.checkNotNull(RollCycles.valueOf(rollCycle), "unrecognized roll cycle");
+ this.rollCycle = rollCycle;
+ return this;
+ }
+
+ public Builder maxQueueWeight(int maxQueueWeight)
+ {
+ Preconditions.checkArgument(maxQueueWeight > 0, "maxQueueWeight must be > 0");
+ this.maxQueueWeight = maxQueueWeight;
+ return this;
+ }
+
+ public Builder maxLogSize(long maxLogSize)
+ {
+ Preconditions.checkArgument(maxLogSize > 0, "maxLogSize must be > 0");
+ this.maxLogSize = maxLogSize;
+ return this;
+ }
+
+ public Builder archiveCommand(String archiveCommand)
+ {
+ this.archiveCommand = archiveCommand;
+ return this;
+ }
+
+ public Builder maxArchiveRetries(int maxArchiveRetries)
+ {
+ this.maxArchiveRetries = maxArchiveRetries;
+ return this;
+ }
+
+ public Builder blocking(boolean blocking)
+ {
+ this.blocking = blocking;
+ return this;
+ }
+
+
+ public BinLog build(boolean cleanDirectory)
+ {
+ logger.info("Attempting to configure bin log: Path: {} Roll cycle: {} Blocking: {} Max queue weight: {} Max log size:{} Archive command: {}", path, rollCycle, blocking, maxQueueWeight, maxLogSize, archiveCommand);
+ synchronized (currentPaths)
+ {
+ if (currentPaths.contains(path))
+ throw new IllegalStateException("Already logging to " + path);
+ currentPaths.add(path);
+ }
+ try
+ {
+ // create the archiver before cleaning directories - ExternalArchiver will try to archive any existing file.
+ BinLogArchiver archiver = Strings.isNullOrEmpty(archiveCommand) ? new DeletingArchiver(maxLogSize) : new ExternalArchiver(archiveCommand, path, maxArchiveRetries);
+ if (cleanDirectory)
+ {
+ logger.info("Cleaning directory: {} as requested", path);
+ if (path.toFile().exists())
+ {
+ Throwable error = cleanDirectory(path.toFile(), null);
+ if (error != null)
+ {
+ throw new RuntimeException(error);
+ }
+ }
+ }
+ BinLog binlog = new BinLog(path, RollCycles.valueOf(rollCycle), maxQueueWeight, archiver, blocking);
+ binlog.start();
+ return binlog;
+ }
+ catch (Exception e)
+ {
+ currentPaths.remove(path);
+ throw e;
+ }
+ }
+ }
+
+ public static Throwable cleanDirectory(File directory, Throwable accumulate)
+ {
+ if (!directory.exists())
+ {
+ return Throwables.merge(accumulate, new RuntimeException(String.format("%s does not exists", directory)));
+ }
+ if (!directory.isDirectory())
+ {
+ return Throwables.merge(accumulate, new RuntimeException(String.format("%s is not a directory", directory)));
+ }
+ for (File f : directory.listFiles())
+ {
+ accumulate = deleteRecursively(f, accumulate);
+ }
+ if (accumulate instanceof FSError)
+ {
+ FileUtils.handleFSError((FSError)accumulate);
+ }
+ return accumulate;
+ }
+
+ private static Throwable deleteRecursively(File fileOrDirectory, Throwable accumulate)
+ {
+ if (fileOrDirectory.isDirectory())
+ {
+ for (File f : fileOrDirectory.listFiles())
+ {
+ accumulate = FileUtils.deleteWithConfirm(f, accumulate);
+ }
+ }
+ return FileUtils.deleteWithConfirm(fileOrDirectory, accumulate);
+ }
+}
diff --git a/src/java/org/apache/cassandra/utils/binlog/BinLogArchiver.java b/src/java/org/apache/cassandra/utils/binlog/BinLogArchiver.java
new file mode 100644
index 0000000..9a6f0bc
--- /dev/null
+++ b/src/java/org/apache/cassandra/utils/binlog/BinLogArchiver.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.utils.binlog;
+
+import java.io.File;
+
+import net.openhft.chronicle.queue.impl.StoreFileListener;
+
+public interface BinLogArchiver extends StoreFileListener
+{
+ public void onReleased(int cycle, File file);
+ public void stop();
+}
diff --git a/src/java/org/apache/cassandra/utils/binlog/BinLogOptions.java b/src/java/org/apache/cassandra/utils/binlog/BinLogOptions.java
new file mode 100644
index 0000000..8005ca3
--- /dev/null
+++ b/src/java/org/apache/cassandra/utils/binlog/BinLogOptions.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.utils.binlog;
+
+import org.apache.commons.lang3.StringUtils;
+
+public class BinLogOptions
+{
+ public String archive_command = StringUtils.EMPTY;
+ /**
+ * How often to roll BinLog segments so they can potentially be reclaimed. Available options are:
+ * MINUTELY, HOURLY, DAILY, LARGE_DAILY, XLARGE_DAILY, HUGE_DAILY.
+ * For more options, refer: net.openhft.chronicle.queue.RollCycles
+ */
+ public String roll_cycle = "HOURLY";
+ /**
+ * Indicates if the BinLog should block if the it falls behind or should drop bin log records.
+ * Default is set to true so that BinLog records wont be lost
+ */
+ public boolean block = true;
+
+ /**
+ * Maximum weight of in memory queue for records waiting to be written to the binlog file
+ * before blocking or dropping the log records. For advanced configurations
+ */
+ public int max_queue_weight = 256 * 1024 * 1024;
+
+ /**
+ * Maximum size of the rolled files to retain on disk before deleting the oldest file. For advanced configurations.
+ */
+ public long max_log_size = 16L * 1024L * 1024L * 1024L;
+
+ /**
+ * Limit the number of times to retry a command.
+ */
+ public int max_archive_retries = 10;
+}
diff --git a/src/java/org/apache/cassandra/utils/binlog/DeletingArchiver.java b/src/java/org/apache/cassandra/utils/binlog/DeletingArchiver.java
new file mode 100644
index 0000000..3bdbb8f
--- /dev/null
+++ b/src/java/org/apache/cassandra/utils/binlog/DeletingArchiver.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.utils.binlog;
+
+import java.io.File;
+import java.util.Queue;
+import java.util.concurrent.ConcurrentLinkedQueue;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class DeletingArchiver implements BinLogArchiver
+{
+ private static final Logger logger = LoggerFactory.getLogger(DeletingArchiver.class);
+ /**
+ * The files in the chronicle queue that have already rolled
+ */
+ private final Queue<File> chronicleStoreFiles = new ConcurrentLinkedQueue<>();
+ private final long maxLogSize;
+ /**
+ * The number of bytes in store files that have already rolled
+ */
+ private long bytesInStoreFiles;
+
+ public DeletingArchiver(long maxLogSize)
+ {
+ Preconditions.checkArgument(maxLogSize > 0, "maxLogSize must be > 0");
+ this.maxLogSize = maxLogSize;
+ }
+
+ /**
+ * Track store files as they are added and their storage impact. Delete them if over storage limit.
+ * @param cycle
+ * @param file
+ */
+ public synchronized void onReleased(int cycle, File file)
+ {
+ chronicleStoreFiles.offer(file);
+ //This isn't accurate because the files are sparse, but it's at least pessimistic
+ bytesInStoreFiles += file.length();
+ logger.debug("Chronicle store file {} rolled file size {}", file.getPath(), file.length());
+ while (bytesInStoreFiles > maxLogSize & !chronicleStoreFiles.isEmpty())
+ {
+ File toDelete = chronicleStoreFiles.poll();
+ long toDeleteLength = toDelete.length();
+ if (!toDelete.delete())
+ {
+ logger.error("Failed to delete chronicle store file: {} store file size: {} bytes in store files: {}. " +
+ "You will need to clean this up manually or reset full query logging.",
+ toDelete.getPath(), toDeleteLength, bytesInStoreFiles);
+ }
+ else
+ {
+ bytesInStoreFiles -= toDeleteLength;
+ logger.info("Deleted chronicle store file: {} store file size: {} bytes in store files: {} max log size: {}.",
+ file.getPath(), toDeleteLength, bytesInStoreFiles, maxLogSize);
+ }
+ }
+ }
+
+ @VisibleForTesting
+ long getBytesInStoreFiles()
+ {
+ return bytesInStoreFiles;
+ }
+
+ public void stop()
+ {
+ }
+}
diff --git a/src/java/org/apache/cassandra/utils/binlog/ExternalArchiver.java b/src/java/org/apache/cassandra/utils/binlog/ExternalArchiver.java
new file mode 100644
index 0000000..e53c5b0
--- /dev/null
+++ b/src/java/org/apache/cassandra/utils/binlog/ExternalArchiver.java
@@ -0,0 +1,206 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.utils.binlog;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.concurrent.DelayQueue;
+import java.util.concurrent.Delayed;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.primitives.Longs;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import net.openhft.chronicle.queue.impl.single.SingleChronicleQueue;
+import org.apache.cassandra.concurrent.NamedThreadFactory;
+import org.apache.cassandra.utils.FBUtilities;
+
+/**
+ * Archives binary log files immediately when they are rolled using a configure archive command.
+ *
+ * The archive command should be "/path/to/script.sh %path" where %path will be replaced with the file to be archived
+ */
+public class ExternalArchiver implements BinLogArchiver
+{
+ private static final Logger logger = LoggerFactory.getLogger(ExternalArchiver.class);
+ // used to replace %path with the actual file to archive when calling the archive command
+ private static final Pattern PATH = Pattern.compile("%path");
+ private static final long DEFAULT_RETRY_DELAY_MS = TimeUnit.MILLISECONDS.convert(5, TimeUnit.MINUTES);
+
+ /**
+ * use a DelayQueue to simplify retries - we want first tries to be executed immediately and retries should wait DEFAULT_RETRY_DELAY_MS
+ */
+ private final DelayQueue<DelayFile> archiveQueue = new DelayQueue<>();
+ private final String archiveCommand;
+ private final ExecutorService executor = Executors.newSingleThreadExecutor(new NamedThreadFactory("BinLogArchiver"));
+ private final Path path;
+ /**
+ * for testing, to be able to make sure that the command is executed
+ */
+ private final ExecCommand commandExecutor;
+ private volatile boolean shouldContinue = true;
+
+ public ExternalArchiver(String archiveCommand, Path path, int maxArchiveRetries)
+ {
+ this(archiveCommand, path, DEFAULT_RETRY_DELAY_MS, maxArchiveRetries, ExternalArchiver::exec);
+ }
+
+ @VisibleForTesting
+ ExternalArchiver(String archiveCommand, Path path, long retryDelayMs, int maxRetries, ExecCommand command)
+ {
+ this.archiveCommand = archiveCommand;
+ this.commandExecutor = command;
+ // if there are any .cq4 files in path, archive them on startup - this handles any leftover files from crashes etc
+ archiveExisting(path);
+ this.path = path;
+
+ executor.execute(() -> {
+ while (shouldContinue)
+ {
+ DelayFile toArchive = null;
+ try
+ {
+ toArchive = archiveQueue.poll(100, TimeUnit.MILLISECONDS);
+ if (toArchive != null)
+ archiveFile(toArchive.file);
+ }
+ catch (Throwable t)
+ {
+ if (toArchive != null)
+ {
+
+ if (toArchive.retries < maxRetries)
+ {
+ logger.error("Got error archiving {}, retrying in {} minutes", toArchive.file, TimeUnit.MINUTES.convert(retryDelayMs, TimeUnit.MILLISECONDS), t);
+ archiveQueue.add(new DelayFile(toArchive.file, retryDelayMs, TimeUnit.MILLISECONDS, toArchive.retries + 1));
+ }
+ else
+ {
+ logger.error("Max retries {} reached for {}, leaving on disk", toArchive.retries, toArchive.file, t);
+ }
+ }
+ else
+ logger.error("Got error waiting for files to archive", t);
+ }
+ }
+ logger.debug("Exiting archiver thread");
+ });
+ }
+
+ public void onReleased(int cycle, File file)
+ {
+ logger.debug("BinLog file released: {}", file);
+ archiveQueue.add(new DelayFile(file, 0, TimeUnit.MILLISECONDS, 0));
+ }
+
+ /**
+ * Stops the archiver thread and tries to archive all existing files
+ *
+ * this handles the case where a user explicitly disables full/audit log and would expect all log files to be archived
+ * rolled or not
+ */
+ public void stop()
+ {
+ shouldContinue = false;
+ try
+ {
+ // wait for the archiver thread to stop;
+ executor.submit(() -> {}).get();
+ // and try to archive all remaining files before exiting
+ archiveExisting(path);
+ }
+ catch (InterruptedException | ExecutionException e)
+ {
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * Iterates over all files in path, executing the archive command for each.
+ */
+ private void archiveExisting(Path path)
+ {
+ if (path == null)
+ return;
+ for (File f : path.toFile().listFiles((f) -> f.isFile() && f.getName().endsWith(SingleChronicleQueue.SUFFIX)))
+ {
+ try
+ {
+ logger.debug("Archiving existing file {}", f);
+ archiveFile(f);
+ }
+ catch (IOException e)
+ {
+ logger.error("Got error archiving existing file {}", f, e);
+ }
+ }
+ }
+
+ private void archiveFile(File f) throws IOException
+ {
+ String cmd = PATH.matcher(archiveCommand).replaceAll(Matcher.quoteReplacement(f.getAbsolutePath()));
+ logger.debug("Executing archive command: {}", cmd);
+ commandExecutor.exec(cmd);
+ }
+
+ static void exec(String command) throws IOException
+ {
+ ProcessBuilder pb = new ProcessBuilder(command.split(" "));
+ pb.redirectErrorStream(true);
+ FBUtilities.exec(pb);
+ }
+
+ private static class DelayFile implements Delayed
+ {
+ public final File file;
+ private final long delayTime;
+ private final int retries;
+
+ public DelayFile(File file, long delay, TimeUnit delayUnit, int retries)
+ {
+ this.file = file;
+ this.delayTime = System.currentTimeMillis() + TimeUnit.MILLISECONDS.convert(delay, delayUnit);
+ this.retries = retries;
+ }
+ public long getDelay(TimeUnit unit)
+ {
+ return unit.convert(delayTime - System.currentTimeMillis(), TimeUnit.MILLISECONDS);
+ }
+
+ public int compareTo(Delayed o)
+ {
+ DelayFile other = (DelayFile)o;
+ return Longs.compare(delayTime, other.delayTime);
+ }
+ }
+
+ interface ExecCommand
+ {
+ public void exec(String command) throws IOException;
+ }
+}
diff --git a/src/java/org/apache/cassandra/utils/btree/BTree.java b/src/java/org/apache/cassandra/utils/btree/BTree.java
index a6c9826..6c546ac 100644
--- a/src/java/org/apache/cassandra/utils/btree/BTree.java
+++ b/src/java/org/apache/cassandra/utils/btree/BTree.java
@@ -19,14 +19,18 @@
package org.apache.cassandra.utils.btree;
import java.util.*;
+import java.util.function.BiConsumer;
import java.util.function.Consumer;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
import com.google.common.base.Predicate;
import com.google.common.collect.Iterators;
import com.google.common.collect.Ordering;
+import org.apache.cassandra.utils.BiLongAccumulator;
+import org.apache.cassandra.utils.LongAccumulator;
import org.apache.cassandra.utils.ObjectSizes;
import static com.google.common.collect.Iterables.concat;
@@ -34,6 +38,7 @@
import static com.google.common.collect.Iterables.transform;
import static java.lang.Math.max;
import static java.lang.Math.min;
+import static java.util.Comparator.naturalOrder;
public class BTree
{
@@ -57,18 +62,42 @@
// The maximum fan factor used for B-Trees
static final int FAN_SHIFT;
+
+ // The maximun tree size for certain heigth of tree
+ static final int[] TREE_SIZE;
+
+ // NB we encode Path indexes as Bytes, so this needs to be less than Byte.MAX_VALUE / 2
+ static final int FAN_FACTOR;
+
+ static final int MAX_TREE_SIZE = Integer.MAX_VALUE;
+
static
{
- int fanfactor = 32;
- if (System.getProperty("cassandra.btree.fanfactor") != null)
- fanfactor = Integer.parseInt(System.getProperty("cassandra.btree.fanfactor"));
+ int fanfactor = Integer.parseInt(System.getProperty("cassandra.btree.fanfactor", "32"));
+ assert fanfactor >= 2 : "the minimal btree fanfactor is 2";
int shift = 1;
while (1 << shift < fanfactor)
shift += 1;
+
FAN_SHIFT = shift;
+ FAN_FACTOR = 1 << FAN_SHIFT;
+
+ // For current FAN_FACTOR, calculate the maximum height of the tree we could build
+ int maxHeight = 0;
+ for (long maxSize = 0; maxSize < MAX_TREE_SIZE; maxHeight++)
+ // each tree node could have (FAN_FACTOR + 1) children,
+ // plus current node could have FAN_FACTOR number of values
+ maxSize = maxSize * (FAN_FACTOR + 1) + FAN_FACTOR;
+
+ TREE_SIZE = new int[maxHeight];
+
+ TREE_SIZE[0] = FAN_FACTOR;
+ for (int i = 1; i < maxHeight - 1; i++)
+ TREE_SIZE[i] = TREE_SIZE[i - 1] * (FAN_FACTOR + 1) + FAN_FACTOR;
+
+ TREE_SIZE[maxHeight - 1] = MAX_TREE_SIZE;
}
- // NB we encode Path indexes as Bytes, so this needs to be less than Byte.MAX_VALUE / 2
- static final int FAN_FACTOR = 1 << FAN_SHIFT;
+
static final int MINIMAL_NODE_SIZE = FAN_FACTOR >> 1;
@@ -87,6 +116,38 @@
public static Dir desc(boolean desc) { return desc ? DESC : ASC; }
}
+ /**
+ * Enables methods to consume the contents of iterators, collections, or arrays without duplicating code or
+ * allocating intermediate objects. Instead of taking an argument that implements an interface, a method takes
+ * an opaque object as the input, and a singleton helper object it uses as an intermediary to access it's contents.
+ * The purpose of doing things this way is to avoid memory allocations on hot paths.
+ */
+ private interface IteratingFunction<T>
+ {
+ /**
+ * Returns the next object at the given index. This method must be called with sequentially increasing index
+ * values, starting at 0, and must only be called once per index value. The results of calling this method
+ * without following these rules are undefined.
+ */
+ <K> K nextAt(T input, int idx);
+ }
+
+ private static final IteratingFunction<Iterator> ITERATOR_FUNCTION = new IteratingFunction<Iterator>()
+ {
+ public <K> K nextAt(Iterator input, int idx)
+ {
+ return (K) input.next();
+ }
+ };
+
+ private static final IteratingFunction<Object[]> ARRAY_FUNCTION = new IteratingFunction<Object[]>()
+ {
+ public <K> K nextAt(Object[] input, int idx)
+ {
+ return (K) input[idx];
+ }
+ };
+
public static Object[] empty()
{
return EMPTY_LEAF;
@@ -99,12 +160,7 @@
public static <C, K extends C, V extends C> Object[] build(Collection<K> source, UpdateFunction<K, V> updateF)
{
- return buildInternal(source, source.size(), updateF);
- }
-
- public static <C, K extends C, V extends C> Object[] build(Iterable<K> source, UpdateFunction<K, V> updateF)
- {
- return buildInternal(source, -1, updateF);
+ return buildInternal(source.iterator(), ITERATOR_FUNCTION, source.size(), updateF);
}
/**
@@ -118,35 +174,84 @@
{
if (size < 0)
throw new IllegalArgumentException(Integer.toString(size));
- return buildInternal(source, size, updateF);
+ return buildInternal(source.iterator(), ITERATOR_FUNCTION, size, updateF);
}
- /**
- * As build(), except:
- * @param size < 0 if size is unknown
- */
- private static <C, K extends C, V extends C> Object[] buildInternal(Iterable<K> source, int size, UpdateFunction<K, V> updateF)
+ public static <C, K extends C, V extends C> Object[] build(Object[] source, int size, UpdateFunction<K, V> updateF)
{
- if ((size >= 0) & (size < FAN_FACTOR))
+ if (size < 0)
+ throw new IllegalArgumentException(Integer.toString(size));
+ return buildInternal(source, ARRAY_FUNCTION, size, updateF);
+ }
+
+ private static <C, K extends C, V extends C, S> Object[] buildLeaf(S source, IteratingFunction<S> iterFunc, int size, int startIdx, UpdateFunction<K, V> updateF)
+ {
+ V[] values = (V[]) new Object[size | 1];
+
+ int idx = startIdx;
+ for (int i = 0; i < size; i++)
{
- if (size == 0)
- return EMPTY_LEAF;
- // pad to odd length to match contract that all leaf nodes are odd
- V[] values = (V[]) new Object[size | 1];
- {
- int i = 0;
- for (K k : source)
- values[i++] = updateF.apply(k);
- }
- if (updateF != UpdateFunction.noOp())
- updateF.allocated(ObjectSizes.sizeOfArray(values));
- return values;
+ K k = iterFunc.nextAt(source, idx);
+ values[i] = updateF.apply(k);
+ idx++;
+ }
+ if (updateF != UpdateFunction.<K>noOp())
+ updateF.allocated(ObjectSizes.sizeOfArray(values));
+ return values;
+ }
+
+ private static <C, K extends C, V extends C, S> Object[] buildInternal(S source, IteratingFunction<S> iterFunc, int size, int level, int startIdx, UpdateFunction<K, V> updateF)
+ {
+ assert size > 0;
+ assert level >= 0;
+ if (level == 0)
+ return buildLeaf(source, iterFunc, size, startIdx, updateF);
+
+ // calcuate child num: (size - (childNum - 1)) / maxChildSize <= childNum
+ int childNum = size / (TREE_SIZE[level - 1] + 1) + 1;
+
+ V[] values = (V[]) new Object[childNum * 2];
+ if (updateF != UpdateFunction.<K>noOp())
+ updateF.allocated(ObjectSizes.sizeOfArray(values));
+
+ int[] indexOffsets = new int[childNum];
+ int childPos = childNum - 1;
+
+ int index = 0;
+ for (int i = 0; i < childNum - 1; i++)
+ {
+ // Calculate the next childSize by splitting the remaining values to the remaining child nodes.
+ // The performance could be improved by pre-compute the childSize (see #9989 comments).
+ int childSize = (size - index) / (childNum - i);
+ // Build the tree with inorder traversal
+ values[childPos + i] = (V) buildInternal(source, iterFunc, childSize, level - 1, startIdx + index, updateF);
+ index += childSize;
+ indexOffsets[i] = index;
+
+ K k = iterFunc.nextAt(source, startIdx + index);
+ values[i] = updateF.apply(k);
+ index++;
}
- TreeBuilder builder = TreeBuilder.newInstance();
- Object[] btree = builder.build(source, updateF, size);
+ values[childPos + childNum - 1] = (V) buildInternal(source, iterFunc, size - index, level - 1, startIdx + index, updateF);
+ indexOffsets[childNum - 1] = size;
- return btree;
+ values[childPos + childNum] = (V) indexOffsets;
+
+ return values;
+ }
+
+ private static <C, K extends C, V extends C, S> Object[] buildInternal(S source, IteratingFunction<S> iterFunc, int size, UpdateFunction<K, V> updateF)
+ {
+ assert size >= 0;
+ if (size == 0)
+ return EMPTY_LEAF;
+
+ // find out the height of the tree
+ int level = 0;
+ while (size > TREE_SIZE[level])
+ level++;
+ return buildInternal(source, iterFunc, size, level, 0, updateF);
}
public static <C, K extends C, V extends C> Object[] update(Object[] btree,
@@ -201,12 +306,14 @@
public static <V> Iterator<V> iterator(Object[] btree, Dir dir)
{
- return new BTreeSearchIterator<>(btree, null, dir);
+ return isLeaf(btree) ? new LeafBTreeSearchIterator<>(btree, null, dir)
+ : new FullBTreeSearchIterator<>(btree, null, dir);
}
public static <V> Iterator<V> iterator(Object[] btree, int lb, int ub, Dir dir)
{
- return new BTreeSearchIterator<>(btree, null, dir, lb, ub);
+ return isLeaf(btree) ? new LeafBTreeSearchIterator<>(btree, null, dir, lb, ub)
+ : new FullBTreeSearchIterator<>(btree, null, dir, lb, ub);
}
public static <V> Iterable<V> iterable(Object[] btree)
@@ -234,7 +341,8 @@
*/
public static <K, V> BTreeSearchIterator<K, V> slice(Object[] btree, Comparator<? super K> comparator, Dir dir)
{
- return new BTreeSearchIterator<>(btree, comparator, dir);
+ return isLeaf(btree) ? new LeafBTreeSearchIterator<>(btree, comparator, dir)
+ : new FullBTreeSearchIterator<>(btree, comparator, dir);
}
/**
@@ -251,6 +359,20 @@
}
/**
+ * @param btree the tree to iterate over
+ * @param comparator the comparator that defines the ordering over the items in the tree
+ * @param startIndex the start index of the range to return, inclusive
+ * @param endIndex the end index of the range to return, inclusive
+ * @param dir if false, the iterator will start at the last item and move backwards
+ * @return an Iterator over the defined sub-range of the tree
+ */
+ public static <K, V extends K> BTreeSearchIterator<K, V> slice(Object[] btree, Comparator<? super K> comparator, int startIndex, int endIndex, Dir dir)
+ {
+ return isLeaf(btree) ? new LeafBTreeSearchIterator<>(btree, comparator, dir, startIndex, endIndex)
+ : new FullBTreeSearchIterator<>(btree, comparator, dir, startIndex, endIndex);
+ }
+
+ /**
* @param btree the tree to iterate over
* @param comparator the comparator that defines the ordering over the items in the tree
* @param start low bound of the range
@@ -270,7 +392,8 @@
end == null ? Integer.MAX_VALUE
: endInclusive ? floorIndex(btree, comparator, end)
: lowerIndex(btree, comparator, end));
- return new BTreeSearchIterator<>(btree, comparator, dir, inclusiveLowerBound, inclusiveUpperBound);
+ return isLeaf(btree) ? new LeafBTreeSearchIterator<>(btree, comparator, dir, inclusiveLowerBound, inclusiveUpperBound)
+ : new FullBTreeSearchIterator<>(btree, comparator, dir, inclusiveLowerBound, inclusiveUpperBound);
}
/**
@@ -572,7 +695,7 @@
}
// returns true if the provided node is a leaf, false if it is a branch
- static boolean isLeaf(Object[] node)
+ public static boolean isLeaf(Object[] node)
{
return (node.length & 1) == 1;
}
@@ -674,7 +797,11 @@
remainder = filter(transform(remainder, function), (x) -> x != null);
Iterable<V> build = concat(head, remainder);
- return buildInternal(build, -1, UpdateFunction.<V>noOp());
+ Builder<V> builder = builder((Comparator<? super V>) naturalOrder());
+ builder.auto(false);
+ for (V v : build)
+ builder.add(v);
+ return builder.build();
}
private static <V> Object[] transformAndFilter(Object[] btree, FiltrationTracker<V> function)
@@ -1089,7 +1216,7 @@
{
if (auto)
autoEnforce();
- return BTree.build(Arrays.asList(values).subList(0, count), UpdateFunction.noOp());
+ return BTree.build(values, count, UpdateFunction.noOp());
}
}
@@ -1162,37 +1289,19 @@
return compare(cmp, previous, max) < 0;
}
- /**
- * Simple method to walk the btree forwards or reversed and apply a function to each element
- *
- * Public method
- *
- */
- public static <V> void apply(Object[] btree, Consumer<V> function, boolean reversed)
+ private static <V, A> void applyValue(V value, BiConsumer<A, V> function, A argument)
{
- if (reversed)
- applyReverse(btree, function, null);
- else
- applyForwards(btree, function, null);
+ function.accept(argument, value);
}
- /**
- * Simple method to walk the btree forwards or reversed and apply a function till a stop condition is reached
- *
- * Public method
- *
- */
- public static <V> void apply(Object[] btree, Consumer<V> function, Predicate<V> stopCondition, boolean reversed)
+ public static <V, A> void applyLeaf(Object[] btree, BiConsumer<A, V> function, A argument)
{
- if (reversed)
- applyReverse(btree, function, stopCondition);
- else
- applyForwards(btree, function, stopCondition);
+ Preconditions.checkArgument(isLeaf(btree));
+ int limit = getLeafKeyEnd(btree);
+ for (int i=0; i<limit; i++)
+ applyValue((V) btree[i], function, argument);
}
-
-
-
/**
* Simple method to walk the btree forwards and apply a function till a stop condition is reached
*
@@ -1200,89 +1309,143 @@
*
* @param btree
* @param function
- * @param stopCondition
*/
- private static <V> boolean applyForwards(Object[] btree, Consumer<V> function, Predicate<V> stopCondition)
+ public static <V, A> void apply(Object[] btree, BiConsumer<A, V> function, A argument)
{
- boolean isLeaf = isLeaf(btree);
- int childOffset = isLeaf ? Integer.MAX_VALUE : getChildStart(btree);
- int limit = isLeaf ? getLeafKeyEnd(btree) : btree.length - 1;
- for (int i = 0 ; i < limit ; i++)
+ if (isLeaf(btree))
{
- // we want to visit in iteration order, so we visit our key nodes inbetween our children
- int idx = isLeaf ? i : (i / 2) + (i % 2 == 0 ? childOffset : 0);
- Object current = btree[idx];
- if (idx < childOffset)
- {
- V castedCurrent = (V) current;
- if (stopCondition != null && stopCondition.apply(castedCurrent))
- return true;
-
- function.accept(castedCurrent);
- }
- else
- {
- if (applyForwards((Object[]) current, function, stopCondition))
- return true;
- }
+ applyLeaf(btree, function, argument);
+ return;
}
- return false;
+ int childOffset = getChildStart(btree);
+ int limit = btree.length - 1 - childOffset;
+ for (int i = 0 ; i < limit ; i++)
+ {
+
+ apply((Object[]) btree[childOffset + i], function, argument);
+
+ if (i < childOffset)
+ applyValue((V) btree[i], function, argument);
+ }
}
/**
- * Simple method to walk the btree in reverse and apply a function till a stop condition is reached
+ * Simple method to walk the btree forwards and apply a function till a stop condition is reached
*
* Private method
*
* @param btree
* @param function
- * @param stopCondition
*/
- private static <V> boolean applyReverse(Object[] btree, Consumer<V> function, Predicate<V> stopCondition)
+ public static <V> void apply(Object[] btree, Consumer<V> function)
{
- boolean isLeaf = isLeaf(btree);
- int childOffset = isLeaf ? 0 : getChildStart(btree);
- int limit = isLeaf ? getLeafKeyEnd(btree) : btree.length - 1;
- for (int i = limit - 1, visited = 0; i >= 0 ; i--, visited++)
+ BTree.<V, Consumer<V>>apply(btree, Consumer::accept, function);
+ }
+
+ private static <V> int find(Object[] btree, V from, Comparator<V> comparator)
+ {
+ // find the start index in iteration order
+ Preconditions.checkNotNull(comparator);
+ int keyEnd = getKeyEnd(btree);
+ return Arrays.binarySearch((V[]) btree, 0, keyEnd, from, comparator);
+ }
+
+ private static boolean isStopSentinel(long v)
+ {
+ return v == Long.MAX_VALUE;
+ }
+
+ private static <V, A> long accumulateLeaf(Object[] btree, BiLongAccumulator<A, V> accumulator, A arg, Comparator<V> comparator, V from, long initialValue)
+ {
+ Preconditions.checkArgument(isLeaf(btree));
+ long value = initialValue;
+ int limit = getLeafKeyEnd(btree);
+
+ int startIdx = 0;
+ if (from != null)
{
- int idx = i;
+ int i = find(btree, from, comparator);
+ boolean isExact = i >= 0;
+ startIdx = isExact ? i : (-1 - i);
+ }
- // we want to visit in reverse iteration order, so we visit our children nodes inbetween our keys
- if (!isLeaf)
+ for (int i = startIdx; i < limit; i++)
+ {
+ value = accumulator.apply(arg, (V) btree[i], value);
+
+ if (isStopSentinel(value))
+ break;
+ }
+ return value;
+ }
+
+ /**
+ * Walk the btree and accumulate a long value using the supplied accumulator function. Iteration will stop if the
+ * accumulator function returns the sentinel values Long.MIN_VALUE or Long.MAX_VALUE
+ *
+ * If the optional from argument is not null, iteration will start from that value (or the one after it's insertion
+ * point if an exact match isn't found)
+ */
+ public static <V, A> long accumulate(Object[] btree, BiLongAccumulator<A, V> accumulator, A arg, Comparator<V> comparator, V from, long initialValue)
+ {
+ if (isLeaf(btree))
+ return accumulateLeaf(btree, accumulator, arg, comparator, from, initialValue);
+
+ long value = initialValue;
+ int childOffset = getChildStart(btree);
+
+ int startChild = 0;
+ if (from != null)
+ {
+ int i = find(btree, from, comparator);
+ boolean isExact = i >= 0;
+
+ startChild = isExact ? i + 1 : -1 - i;
+
+ if (isExact)
{
- int typeOffset = visited / 2;
-
- if (i % 2 == 0)
- {
- // This is a child branch. Since children are in the second half of the array, we must
- // adjust for the key's we've visited along the way
- idx += typeOffset;
- }
- else
- {
- // This is a key. Since the keys are in the first half of the array and we are iterating
- // in reverse we subtract the childOffset and adjust for children we've walked so far
- idx = i - childOffset + typeOffset;
- }
- }
-
- Object current = btree[idx];
- if (isLeaf || idx < childOffset)
- {
- V castedCurrent = (V) current;
- if (stopCondition != null && stopCondition.apply(castedCurrent))
- return true;
-
- function.accept(castedCurrent);
- }
- else
- {
- if (applyReverse((Object[]) current, function, stopCondition))
- return true;
+ value = accumulator.apply(arg, (V) btree[i], value);
+ if (isStopSentinel(value))
+ return value;
+ from = null;
}
}
- return false;
+ int limit = btree.length - 1 - childOffset;
+ for (int i=startChild; i<limit; i++)
+ {
+ value = accumulate((Object[]) btree[childOffset + i], accumulator, arg, comparator, from, value);
+
+ if (isStopSentinel(value))
+ break;
+
+ if (i < childOffset)
+ {
+ value = accumulator.apply(arg, (V) btree[i], value);
+ // stop if a sentinel stop value was returned
+ if (isStopSentinel(value))
+ break;
+ }
+
+ if (from != null)
+ from = null;
+ }
+ return value;
+ }
+
+ public static <V> long accumulate(Object[] btree, LongAccumulator<V> accumulator, Comparator<V> comparator, V from, long initialValue)
+ {
+ return accumulate(btree, LongAccumulator::apply, accumulator, comparator, from, initialValue);
+ }
+
+ public static <V> long accumulate(Object[] btree, LongAccumulator<V> accumulator, long initialValue)
+ {
+ return accumulate(btree, accumulator, null, null, initialValue);
+ }
+
+ public static <V, A> long accumulate(Object[] btree, BiLongAccumulator<A, V> accumulator, A arg, long initialValue)
+ {
+ return accumulate(btree, accumulator, arg, null, null, initialValue);
}
}
diff --git a/src/java/org/apache/cassandra/utils/btree/BTreeSearchIterator.java b/src/java/org/apache/cassandra/utils/btree/BTreeSearchIterator.java
index ec16a8e..2ad7f40 100644
--- a/src/java/org/apache/cassandra/utils/btree/BTreeSearchIterator.java
+++ b/src/java/org/apache/cassandra/utils/btree/BTreeSearchIterator.java
@@ -18,146 +18,15 @@
*/
package org.apache.cassandra.utils.btree;
-import java.util.Comparator;
import java.util.Iterator;
-import java.util.NoSuchElementException;
import org.apache.cassandra.utils.IndexedSearchIterator;
-import static org.apache.cassandra.utils.btree.BTree.size;
-public class BTreeSearchIterator<K, V> extends TreeCursor<K> implements IndexedSearchIterator<K, V>, Iterator<V>
+public interface BTreeSearchIterator<K, V> extends IndexedSearchIterator<K, V>, Iterator<V>
{
- private final boolean forwards;
-
- // for simplicity, we just always use the index feature of the btree to maintain our bounds within the tree,
- // whether or not they are constrained
- private int index;
- private byte state;
- private final int lowerBound, upperBound; // inclusive
-
- private static final int MIDDLE = 0; // only "exists" as an absence of other states
- private static final int ON_ITEM = 1; // may only co-exist with LAST (or MIDDLE, which is 0)
- private static final int BEFORE_FIRST = 2; // may not coexist with any other state
- private static final int LAST = 4; // may co-exist with ON_ITEM, in which case we are also at END
- private static final int END = 5; // equal to LAST | ON_ITEM
-
- public BTreeSearchIterator(Object[] btree, Comparator<? super K> comparator, BTree.Dir dir)
- {
- this(btree, comparator, dir, 0, size(btree)-1);
- }
-
- BTreeSearchIterator(Object[] btree, Comparator<? super K> comparator, BTree.Dir dir, int lowerBound, int upperBound)
- {
- super(comparator, btree);
- this.forwards = dir == BTree.Dir.ASC;
- this.lowerBound = lowerBound;
- this.upperBound = upperBound;
- rewind();
- }
-
- /**
- * @return 0 if we are on the last item, 1 if we are past the last item, and -1 if we are before it
- */
- private int compareToLast(int idx)
- {
- return forwards ? idx - upperBound : lowerBound - idx;
- }
-
- private int compareToFirst(int idx)
- {
- return forwards ? idx - lowerBound : upperBound - idx;
- }
-
- public boolean hasNext()
- {
- return state != END;
- }
-
- public V next()
- {
- switch (state)
- {
- case ON_ITEM:
- if (compareToLast(index = moveOne(forwards)) >= 0)
- state = END;
- break;
- case BEFORE_FIRST:
- seekTo(index = forwards ? lowerBound : upperBound);
- state = (byte) (upperBound == lowerBound ? LAST : MIDDLE);
- case LAST:
- case MIDDLE:
- state |= ON_ITEM;
- break;
- default:
- throw new NoSuchElementException();
- }
-
- return current();
- }
-
- public V next(K target)
- {
- if (!hasNext())
- return null;
-
- int state = this.state;
- boolean found = seekTo(target, forwards, (state & (ON_ITEM | BEFORE_FIRST)) != 0);
- int index = cur.globalIndex();
-
- V next = null;
- if (state == BEFORE_FIRST && compareToFirst(index) < 0)
- return null;
-
- int compareToLast = compareToLast(index);
- if ((compareToLast <= 0))
- {
- state = compareToLast < 0 ? MIDDLE : LAST;
- if (found)
- {
- state |= ON_ITEM;
- next = (V) currentValue();
- }
- }
- else state = END;
-
- this.state = (byte) state;
- this.index = index;
- return next;
- }
-
/**
* Reset this Iterator to its starting position
*/
- public void rewind()
- {
- if (upperBound < lowerBound)
- {
- state = (byte) END;
- }
- else
- {
- // we don't move into the tree until the first request is made, so we know where to go
- reset(forwards);
- state = (byte) BEFORE_FIRST;
- }
- }
-
- private void checkOnItem()
- {
- if ((state & ON_ITEM) != ON_ITEM)
- throw new NoSuchElementException();
- }
-
- public V current()
- {
- checkOnItem();
- return (V) currentValue();
- }
-
- public int indexOfCurrent()
- {
- checkOnItem();
- return compareToFirst(index);
- }
+ public void rewind();
}
diff --git a/src/java/org/apache/cassandra/utils/btree/BTreeSet.java b/src/java/org/apache/cassandra/utils/btree/BTreeSet.java
index a59e481..1a28d78 100644
--- a/src/java/org/apache/cassandra/utils/btree/BTreeSet.java
+++ b/src/java/org/apache/cassandra/utils/btree/BTreeSet.java
@@ -27,6 +27,7 @@
import static org.apache.cassandra.utils.btree.BTree.findIndex;
+
public class BTreeSet<V> implements NavigableSet<V>, List<V>
{
protected final Comparator<? super V> comparator;
@@ -361,7 +362,7 @@
@Override
protected BTreeSearchIterator<V, V> slice(Dir dir)
{
- return new BTreeSearchIterator<>(tree, comparator, dir, lowerBound, upperBound);
+ return BTree.slice(tree, comparator, lowerBound, upperBound, dir);
}
@Override
diff --git a/src/java/org/apache/cassandra/utils/btree/FullBTreeSearchIterator.java b/src/java/org/apache/cassandra/utils/btree/FullBTreeSearchIterator.java
new file mode 100644
index 0000000..c19d447
--- /dev/null
+++ b/src/java/org/apache/cassandra/utils/btree/FullBTreeSearchIterator.java
@@ -0,0 +1,159 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.utils.btree;
+
+import static org.apache.cassandra.utils.btree.BTree.size;
+
+import java.util.Comparator;
+import java.util.NoSuchElementException;
+
+public class FullBTreeSearchIterator<K, V> extends TreeCursor<K> implements BTreeSearchIterator<K, V>
+{
+ private final boolean forwards;
+
+ // for simplicity, we just always use the index feature of the btree to maintain our bounds within the tree,
+ // whether or not they are constrained
+ private int index;
+ private byte state;
+ private final int lowerBound, upperBound; // inclusive
+
+ private static final int MIDDLE = 0; // only "exists" as an absence of other states
+ private static final int ON_ITEM = 1; // may only co-exist with LAST (or MIDDLE, which is 0)
+ private static final int BEFORE_FIRST = 2; // may not coexist with any other state
+ private static final int LAST = 4; // may co-exist with ON_ITEM, in which case we are also at END
+ private static final int END = 5; // equal to LAST | ON_ITEM
+
+ public FullBTreeSearchIterator(Object[] btree, Comparator<? super K> comparator, BTree.Dir dir)
+ {
+ this(btree, comparator, dir, 0, size(btree)-1);
+ }
+
+ FullBTreeSearchIterator(Object[] btree, Comparator<? super K> comparator, BTree.Dir dir, int lowerBound, int upperBound)
+ {
+ super(comparator, btree);
+ this.forwards = dir == BTree.Dir.ASC;
+ this.lowerBound = lowerBound;
+ this.upperBound = upperBound;
+ rewind();
+ }
+
+ /**
+ * @return 0 if we are on the last item, 1 if we are past the last item, and -1 if we are before it
+ */
+ private int compareToLast(int idx)
+ {
+ return forwards ? idx - upperBound : lowerBound - idx;
+ }
+
+ private int compareToFirst(int idx)
+ {
+ return forwards ? idx - lowerBound : upperBound - idx;
+ }
+
+ public boolean hasNext()
+ {
+ return state != END;
+ }
+
+ public V next()
+ {
+ switch (state)
+ {
+ case ON_ITEM:
+ if (compareToLast(index = moveOne(forwards)) >= 0)
+ state = END;
+ break;
+ case BEFORE_FIRST:
+ seekTo(index = forwards ? lowerBound : upperBound);
+ state = (byte) (upperBound == lowerBound ? LAST : MIDDLE);
+ case LAST:
+ case MIDDLE:
+ state |= ON_ITEM;
+ break;
+ default:
+ throw new NoSuchElementException();
+ }
+
+ return current();
+ }
+
+ public V next(K target)
+ {
+ if (!hasNext())
+ return null;
+
+ int state = this.state;
+ boolean found = seekTo(target, forwards, (state & (ON_ITEM | BEFORE_FIRST)) != 0);
+ int index = cur.globalIndex();
+
+ V next = null;
+ if (state == BEFORE_FIRST && compareToFirst(index) < 0)
+ return null;
+
+ int compareToLast = compareToLast(index);
+ if ((compareToLast <= 0))
+ {
+ state = compareToLast < 0 ? MIDDLE : LAST;
+ if (found)
+ {
+ state |= ON_ITEM;
+ next = (V) currentValue();
+ }
+ }
+ else state = END;
+
+ this.state = (byte) state;
+ this.index = index;
+ return next;
+ }
+
+ /**
+ * Reset this Iterator to its starting position
+ */
+ public void rewind()
+ {
+ if (upperBound < lowerBound)
+ {
+ state = (byte) END;
+ }
+ else
+ {
+ // we don't move into the tree until the first request is made, so we know where to go
+ reset(forwards);
+ state = (byte) BEFORE_FIRST;
+ }
+ }
+
+ private void checkOnItem()
+ {
+ if ((state & ON_ITEM) != ON_ITEM)
+ throw new NoSuchElementException();
+ }
+
+ public V current()
+ {
+ checkOnItem();
+ return (V) currentValue();
+ }
+
+ public int indexOfCurrent()
+ {
+ checkOnItem();
+ return compareToFirst(index);
+ }
+}
diff --git a/src/java/org/apache/cassandra/utils/btree/LeafBTreeSearchIterator.java b/src/java/org/apache/cassandra/utils/btree/LeafBTreeSearchIterator.java
new file mode 100644
index 0000000..29aed4be
--- /dev/null
+++ b/src/java/org/apache/cassandra/utils/btree/LeafBTreeSearchIterator.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.utils.btree;
+
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.NoSuchElementException;
+
+import static org.apache.cassandra.utils.btree.BTree.size;
+
+public class LeafBTreeSearchIterator<K, V> implements BTreeSearchIterator<K, V>
+{
+ private final boolean forwards;
+ private final K[] keys;
+ private final Comparator<? super K> comparator;
+ private int nextPos;
+ private final int lowerBound, upperBound; // inclusive
+ private boolean hasNext;
+ private boolean hasCurrent;
+
+ public LeafBTreeSearchIterator(Object[] btree, Comparator<? super K> comparator, BTree.Dir dir)
+ {
+ this(btree, comparator, dir, 0, size(btree) -1);
+ }
+
+ LeafBTreeSearchIterator(Object[] btree, Comparator<? super K> comparator, BTree.Dir dir, int lowerBound, int upperBound)
+ {
+ this.keys = (K[]) btree;
+ this.forwards = dir == BTree.Dir.ASC;
+ this.comparator = comparator;
+ this.lowerBound = lowerBound;
+ this.upperBound = upperBound;
+ rewind();
+ }
+
+ public void rewind()
+ {
+ nextPos = forwards ? lowerBound : upperBound;
+ hasNext = nextPos >= lowerBound && nextPos <= upperBound;
+ }
+
+ public V next()
+ {
+ if (!hasNext)
+ throw new NoSuchElementException();
+ final V elem = (V) keys[nextPos];
+ nextPos += forwards ? 1 : -1;
+ hasNext = nextPos >= lowerBound && nextPos <= upperBound;
+ hasCurrent = true;
+ return elem;
+ }
+
+ public boolean hasNext()
+ {
+ return hasNext;
+ }
+
+ private int searchNext(K key)
+ {
+ int lb = forwards ? nextPos : lowerBound; // inclusive
+ int ub = forwards ? upperBound : nextPos; // inclusive
+
+ return Arrays.binarySearch(keys, lb, ub + 1, key, comparator);
+ }
+
+ private void updateHasNext()
+ {
+ hasNext = nextPos >= lowerBound && nextPos <= upperBound;
+ }
+
+ public V next(K key)
+ {
+ if (!hasNext)
+ return null;
+ V result = null;
+
+ // first check the current position in case of sequential access
+ if (comparator.compare(key, keys[nextPos]) == 0)
+ {
+ hasCurrent = true;
+ result = (V) keys[nextPos];
+ nextPos += forwards ? 1 : -1;
+ }
+ updateHasNext();
+
+ if (result != null || !hasNext)
+ return result;
+
+ // otherwise search against the remaining values
+ int find = searchNext(key);
+ if (find >= 0)
+ {
+ hasCurrent = true;
+ result = (V) keys[find];
+ nextPos = find + (forwards ? 1 : -1);
+ }
+ else
+ {
+ nextPos = (forwards ? -1 : -2) - find;
+ hasCurrent = false;
+ }
+ updateHasNext();
+ return result;
+ }
+
+ public V current()
+ {
+ if (!hasCurrent)
+ throw new NoSuchElementException();
+ int current = forwards ? nextPos - 1 : nextPos + 1;
+ return (V) keys[current];
+ }
+
+ public int indexOfCurrent()
+ {
+ if (!hasCurrent)
+ throw new NoSuchElementException();
+ int current = forwards ? nextPos - 1 : nextPos + 1;
+ return forwards ? current - lowerBound : upperBound - current;
+ }
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/utils/btree/TreeBuilder.java b/src/java/org/apache/cassandra/utils/btree/TreeBuilder.java
index f42de0f..128ff6a 100644
--- a/src/java/org/apache/cassandra/utils/btree/TreeBuilder.java
+++ b/src/java/org/apache/cassandra/utils/btree/TreeBuilder.java
@@ -22,8 +22,6 @@
import io.netty.util.Recycler;
-import static org.apache.cassandra.utils.btree.BTree.EMPTY_LEAF;
-import static org.apache.cassandra.utils.btree.BTree.FAN_SHIFT;
import static org.apache.cassandra.utils.btree.BTree.POSITIVE_INFINITY;
/**
@@ -116,31 +114,7 @@
Object[] r = current.toNode();
current.clear();
- builderRecycler.recycle(this, recycleHandle);
-
- return r;
- }
-
- public <C, K extends C, V extends C> Object[] build(Iterable<K> source, UpdateFunction<K, V> updateF, int size)
- {
- assert updateF != null;
-
- NodeBuilder current = rootBuilder;
- // we descend only to avoid wasting memory; in update() we will often descend into existing trees
- // so here we want to descend also, so we don't have lg max(N) depth in both directions
- while ((size >>= FAN_SHIFT) > 0)
- current = current.ensureChild();
-
- current.reset(EMPTY_LEAF, POSITIVE_INFINITY, updateF, null);
- for (K key : source)
- current.addNewKey(key);
-
- current = current.ascendToRoot();
-
- Object[] r = current.toNode();
- current.clear();
-
- builderRecycler.recycle(this, recycleHandle);
+ recycleHandle.recycle(this);
return r;
}
diff --git a/src/java/org/apache/cassandra/utils/concurrent/Accumulator.java b/src/java/org/apache/cassandra/utils/concurrent/Accumulator.java
index ca9bb09..7adb33b 100644
--- a/src/java/org/apache/cassandra/utils/concurrent/Accumulator.java
+++ b/src/java/org/apache/cassandra/utils/concurrent/Accumulator.java
@@ -18,6 +18,8 @@
*/
package org.apache.cassandra.utils.concurrent;
+import java.util.AbstractCollection;
+import java.util.Collection;
import java.util.Arrays;
import java.util.Iterator;
import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
@@ -28,7 +30,7 @@
*
* @param <E>
*/
-public class Accumulator<E> implements Iterable<E>
+public class Accumulator<E>
{
private volatile int nextIndex;
private volatile int presentCount;
@@ -106,7 +108,7 @@
return values.length;
}
- public Iterator<E> iterator()
+ private Iterator<E> iterator(int count)
{
return new Iterator<E>()
{
@@ -114,7 +116,7 @@
public boolean hasNext()
{
- return p < presentCount;
+ return p < count;
}
public E next()
@@ -137,6 +139,25 @@
return (E) values[i];
}
+ public Collection<E> snapshot()
+ {
+ int count = presentCount;
+ return new AbstractCollection<E>()
+ {
+ @Override
+ public Iterator<E> iterator()
+ {
+ return Accumulator.this.iterator(count);
+ }
+
+ @Override
+ public int size()
+ {
+ return count;
+ }
+ };
+ }
+
/**
* Removes all of the elements from this accumulator.
*
diff --git a/src/java/org/apache/cassandra/utils/concurrent/Locks.java b/src/java/org/apache/cassandra/utils/concurrent/Locks.java
deleted file mode 100644
index dcbfe63..0000000
--- a/src/java/org/apache/cassandra/utils/concurrent/Locks.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.utils.concurrent;
-
-import sun.misc.Unsafe;
-
-import java.lang.reflect.Field;
-
-public class Locks
-{
- static final Unsafe unsafe;
-
- static
- {
- try
- {
- Field field = sun.misc.Unsafe.class.getDeclaredField("theUnsafe");
- field.setAccessible(true);
- unsafe = (sun.misc.Unsafe) field.get(null);
- }
- catch (Exception e)
- {
- throw new AssertionError(e);
- }
- }
-
- // enters the object's monitor IF UNSAFE IS PRESENT. If it isn't, this is a no-op.
- public static void monitorEnterUnsafe(Object object)
- {
- if (unsafe != null)
- unsafe.monitorEnter(object);
- }
-
- public static void monitorExitUnsafe(Object object)
- {
- if (unsafe != null)
- unsafe.monitorExit(object);
- }
-}
diff --git a/src/java/org/apache/cassandra/utils/concurrent/Ref.java b/src/java/org/apache/cassandra/utils/concurrent/Ref.java
index 5b6c3d6..a3733474 100644
--- a/src/java/org/apache/cassandra/utils/concurrent/Ref.java
+++ b/src/java/org/apache/cassandra/utils/concurrent/Ref.java
@@ -51,7 +51,6 @@
import org.cliffc.high_scale_lib.NonBlockingHashMap;
import static java.util.Collections.emptyList;
-import org.apache.cassandra.concurrent.InfiniteLoopExecutor.InterruptibleRunnable;
import static org.apache.cassandra.utils.ExecutorUtils.awaitTermination;
import static org.apache.cassandra.utils.ExecutorUtils.shutdownNow;
@@ -685,7 +684,7 @@
this.candidates.retainAll(candidates);
if (!this.candidates.isEmpty())
{
- List<String> names = new ArrayList<>();
+ List<String> names = new ArrayList<>(this.candidates.size());
for (Tidy tidy : this.candidates)
names.add(tidy.name());
logger.warn("Strong reference leak candidates detected: {}", names);
diff --git a/src/java/org/apache/cassandra/utils/concurrent/SimpleCondition.java b/src/java/org/apache/cassandra/utils/concurrent/SimpleCondition.java
index 57614e0..0ff9018 100644
--- a/src/java/org/apache/cassandra/utils/concurrent/SimpleCondition.java
+++ b/src/java/org/apache/cassandra/utils/concurrent/SimpleCondition.java
@@ -48,10 +48,15 @@
public boolean await(long time, TimeUnit unit) throws InterruptedException
{
- if (isSignaled())
- return true;
long start = System.nanoTime();
long until = start + unit.toNanos(time);
+ return awaitUntil(until);
+ }
+
+ public boolean awaitUntil(long deadlineNanos) throws InterruptedException
+ {
+ if (isSignaled())
+ return true;
if (waiting == null)
waitingUpdater.compareAndSet(this, null, new WaitQueue());
WaitQueue.Signal s = waiting.register();
@@ -60,7 +65,7 @@
s.cancel();
return true;
}
- return s.awaitUntil(until) || isSignaled();
+ return s.awaitUntil(deadlineNanos) || isSignaled();
}
public void signal()
diff --git a/src/java/org/apache/cassandra/utils/concurrent/WaitQueue.java b/src/java/org/apache/cassandra/utils/concurrent/WaitQueue.java
index 5b453b0..3647623 100644
--- a/src/java/org/apache/cassandra/utils/concurrent/WaitQueue.java
+++ b/src/java/org/apache/cassandra/utils/concurrent/WaitQueue.java
@@ -263,6 +263,15 @@
* @throws InterruptedException
*/
public boolean awaitUntil(long nanos) throws InterruptedException;
+
+ /**
+ * Wait until signalled, or the provided time is reached, or the thread is interrupted. If signalled,
+ * isSignalled() will be true on exit, and the method will return true; if timedout, the method will return
+ * false and isCancelled() will be true
+ * @param nanos System.nanoTime() to wait until
+ * @return true if signalled, false if timed out
+ */
+ public boolean awaitUntilUninterruptibly(long nanos);
}
/**
@@ -306,6 +315,17 @@
return checkAndClear();
}
+ public boolean awaitUntilUninterruptibly(long until)
+ {
+ long now;
+ while (until > (now = System.nanoTime()) && !isSignalled())
+ {
+ long delta = until - now;
+ LockSupport.parkNanos(delta);
+ }
+ return checkAndClear();
+ }
+
private void checkInterrupted() throws InterruptedException
{
if (Thread.interrupted())
diff --git a/src/java/org/apache/cassandra/utils/concurrent/WeightedQueue.java b/src/java/org/apache/cassandra/utils/concurrent/WeightedQueue.java
new file mode 100644
index 0000000..3a6505e
--- /dev/null
+++ b/src/java/org/apache/cassandra/utils/concurrent/WeightedQueue.java
@@ -0,0 +1,333 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.utils.concurrent;
+
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.Objects;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.TimeUnit;
+
+import com.google.common.base.Preconditions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Weighted queue is a wrapper around any blocking queue that turns it into a blocking weighted queue. The queue
+ * will weigh each element being added and removed. Adding to the queue is blocked if adding would violate
+ * the weight bound.
+ *
+ * If an element weighs in at larger than the capacity of the queue then exactly one such element will be allowed
+ * into the queue at a time.
+ *
+ * If the weight of an object changes after it is added you are going to have a bad time. Checking weight should be
+ * cheap so memoize expensive to compute weights. If weight throws that can also result in leaked permits so it's
+ * always a good idea to memoize weight so it doesn't throw.
+ *
+ * In the interests of not writing unit tests for methods no one uses there is a lot of UnsupportedOperationException.
+ * If you need them then add them and add proper unit tests to WeightedQueueTest. "Good" tests. 100% coverage including
+ * exception paths and resource leaks.
+ **/
+public class WeightedQueue<T> implements BlockingQueue<T>
+{
+ private static final Logger logger = LoggerFactory.getLogger(WeightedQueue.class);
+ public static final Weigher NATURAL_WEIGHER = (Weigher<Object>) weighable ->
+ {
+ if (weighable instanceof Weighable)
+ {
+ return ((Weighable)weighable).weight();
+ }
+ return 1;
+ };
+
+ private final Weigher<T> weigher;
+ private final BlockingQueue<T> queue;
+ private final int maxWeight;
+ final Semaphore availableWeight;
+
+ public boolean add(T e)
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ public boolean offer(T t)
+ {
+ Preconditions.checkNotNull(t);
+ boolean acquired = tryAcquireWeight(t);
+ if (acquired)
+ {
+ boolean offered = false;
+ try
+ {
+ offered = queue.offer(t);
+ return offered;
+ }
+ finally
+ {
+ if (!offered)
+ {
+ releaseWeight(t);
+ }
+ }
+ }
+ return false;
+ }
+
+ public T remove()
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ public T poll()
+ {
+ T retval = queue.poll();
+ releaseWeight(retval);
+ return retval;
+ }
+
+ public T element()
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ public T peek()
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ public void put(T t) throws InterruptedException
+ {
+ Preconditions.checkNotNull(t);
+ acquireWeight(t, 0, null);
+ boolean put = false;
+ try
+ {
+ queue.put(t);
+ put = true;
+ }
+ finally
+ {
+ if (!put)
+ {
+ releaseWeight(t);
+ }
+ }
+ }
+
+ public boolean offer(T t, long timeout, TimeUnit unit) throws InterruptedException
+ {
+ Preconditions.checkNotNull(t);
+ Preconditions.checkNotNull(unit);
+ boolean acquired = acquireWeight(t, timeout, unit);
+ if (acquired)
+ {
+ boolean offered = false;
+ try
+ {
+ offered = queue.offer(t, timeout, unit);
+ return offered;
+ }
+ finally
+ {
+ if (!offered)
+ {
+ releaseWeight(t);
+ }
+ }
+ }
+ return false;
+ }
+
+ public T take() throws InterruptedException
+ {
+ T retval = queue.take();
+ releaseWeight(retval);
+ return retval;
+ }
+
+ public T poll(long timeout, TimeUnit unit) throws InterruptedException
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ public int remainingCapacity()
+ {
+ throw new UnsupportedOperationException("Seems like a bad idea");
+ }
+
+ public boolean remove(Object o)
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ public boolean containsAll(Collection<?> c)
+ {
+ throw new UnsupportedOperationException("Seems like a bad idea");
+ }
+
+ public boolean addAll(Collection<? extends T> c)
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ public boolean removeAll(Collection<?> c)
+ {
+ throw new UnsupportedOperationException("Seems like a bad idea");
+ }
+
+ public boolean retainAll(Collection<?> c)
+ {
+ throw new UnsupportedOperationException("Seems like a bad idea");
+ }
+
+ public void clear()
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ public int size()
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ public boolean isEmpty()
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ public boolean contains(Object o)
+ {
+ throw new UnsupportedOperationException("Seems like a bad idea");
+ }
+
+ public Iterator<T> iterator()
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ public Object[] toArray()
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ public <T1> T1[] toArray(T1[] a)
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ public int drainTo(Collection<? super T> c)
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ public int drainTo(Collection<? super T> c, int maxElements)
+ {
+ int count = 0;
+ T o;
+ while(count < maxElements && (o = poll()) != null)
+ {
+ c.add(o);
+ count++;
+ }
+ return count;
+ }
+
+ public interface Weigher<T>
+ {
+ int weigh(T weighable);
+ }
+
+ public interface Weighable
+ {
+ int weight();
+ }
+
+ public WeightedQueue(int maxWeight)
+ {
+ this(maxWeight, new LinkedBlockingQueue<T>(), NATURAL_WEIGHER);
+ }
+
+ public WeightedQueue(int maxWeight, BlockingQueue<T> queue, Weigher<T> weigher)
+ {
+ Preconditions.checkNotNull(queue);
+ Preconditions.checkNotNull(weigher);
+ Preconditions.checkArgument(maxWeight > 0);
+ this.maxWeight = maxWeight;
+ this.queue = queue;
+ this.weigher = weigher;
+ availableWeight = new Semaphore(maxWeight);
+ }
+
+ boolean acquireWeight(T weighable, long timeout, TimeUnit unit) throws InterruptedException
+ {
+ int weight = weigher.weigh(weighable);
+ if (weight < 1)
+ {
+ throw new IllegalArgumentException(String.format("Weighable: \"%s\" had illegal weight %d", Objects.toString(weighable), weight));
+ }
+
+ //Allow exactly one overweight element
+ weight = Math.min(maxWeight, weight);
+
+ if (unit != null)
+ {
+ return availableWeight.tryAcquire(weight, timeout, unit);
+ }
+ else
+ {
+ availableWeight.acquire(weight);
+ return true;
+ }
+ }
+
+ boolean tryAcquireWeight(T weighable)
+ {
+ int weight = weigher.weigh(weighable);
+ if (weight < 1)
+ {
+ throw new IllegalArgumentException(String.format("Weighable: \"%s\" had illegal weight %d", Objects.toString(weighable), weight));
+ }
+
+ //Allow exactly one overweight element
+ weight = Math.min(maxWeight, weight);
+
+ return availableWeight.tryAcquire(weight);
+ }
+
+ void releaseWeight(T weighable)
+ {
+ if (weighable == null)
+ {
+ return;
+ }
+
+ int weight = weigher.weigh(weighable);
+ if (weight < 1)
+ {
+ throw new IllegalArgumentException(String.format("Weighable: \"%s\" had illegal weight %d", Objects.toString(weighable), weight));
+ }
+
+ //Allow exactly one overweight element
+ weight = Math.min(maxWeight, weight);
+
+ availableWeight.release(weight);
+ }
+}
diff --git a/src/java/org/apache/cassandra/utils/logging/LogbackLoggingSupport.java b/src/java/org/apache/cassandra/utils/logging/LogbackLoggingSupport.java
index 3229460..7249caf 100644
--- a/src/java/org/apache/cassandra/utils/logging/LogbackLoggingSupport.java
+++ b/src/java/org/apache/cassandra/utils/logging/LogbackLoggingSupport.java
@@ -48,6 +48,8 @@
// To work around this, a custom ReconfigureOnChangeFilter is installed, that simply
// prevents this configuration file check and possible reload of the configuration,
// while executing sandboxed UDF code.
+ //
+ // NOTE: this is obsolte with logback versions (at least since 1.2.3)
Logger logbackLogger = (Logger) LoggerFactory.getLogger(ThreadAwareSecurityManager.class);
LoggerContext ctx = logbackLogger.getLoggerContext();
@@ -121,6 +123,9 @@
/**
* The purpose of this class is to prevent logback from checking for config file change,
* if the current thread is executing a sandboxed thread to avoid {@link AccessControlException}s.
+ *
+ * This is obsolete with logback versions that replaced {@link ReconfigureOnChangeFilter}
+ * with {@link ch.qos.logback.classic.joran.ReconfigureOnChangeTask} (at least logback since 1.2.3).
*/
private static class SMAwareReconfigureOnChangeFilter extends ReconfigureOnChangeFilter
{
diff --git a/src/java/org/apache/cassandra/utils/logging/LoggingSupportFactory.java b/src/java/org/apache/cassandra/utils/logging/LoggingSupportFactory.java
index 3e7adab..575caca 100644
--- a/src/java/org/apache/cassandra/utils/logging/LoggingSupportFactory.java
+++ b/src/java/org/apache/cassandra/utils/logging/LoggingSupportFactory.java
@@ -34,7 +34,7 @@
{
loggingSupport = new NoOpFallbackLoggingSupport();
logger.warn("You are using Cassandra with an unsupported deployment. The intended logging implementation library logback is not used by slf4j. Detected slf4j logger factory: {}. "
- + "You will not be able to dynamically manage log levels via JMX and may have performance or other issues.", loggerFactoryClass);
+ + "You will not be able to dynamically manage log levels via JMX and may have performance or other issues.", loggerFactoryClass);
}
}
return loggingSupport;
diff --git a/src/java/org/apache/cassandra/utils/memory/BufferPool.java b/src/java/org/apache/cassandra/utils/memory/BufferPool.java
index e91c9e2..c0686c4 100644
--- a/src/java/org/apache/cassandra/utils/memory/BufferPool.java
+++ b/src/java/org/apache/cassandra/utils/memory/BufferPool.java
@@ -21,13 +21,21 @@
import java.lang.ref.PhantomReference;
import java.lang.ref.ReferenceQueue;
import java.nio.ByteBuffer;
-import java.util.Arrays;
+import java.nio.ByteOrder;
+import java.util.ArrayDeque;
+import java.util.Collections;
import java.util.Queue;
+import java.util.Set;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicLongFieldUpdater;
+import java.util.function.BiPredicate;
+import java.util.function.Consumer;
+import java.util.function.Supplier;
import com.google.common.annotations.VisibleForTesting;
+
+import net.nicoulaj.compilecommand.annotations.Inline;
import org.apache.cassandra.concurrent.InfiniteLoopExecutor;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -37,32 +45,36 @@
import org.apache.cassandra.io.compress.BufferType;
import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.metrics.BufferPoolMetrics;
-import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.NoSpamLogger;
import org.apache.cassandra.utils.concurrent.Ref;
-import static org.apache.cassandra.utils.ExecutorUtils.awaitTermination;
-import static org.apache.cassandra.utils.ExecutorUtils.shutdownNow;
+import static com.google.common.collect.ImmutableList.of;
+import static org.apache.cassandra.utils.ExecutorUtils.*;
+import static org.apache.cassandra.utils.FBUtilities.prettyPrintMemory;
+import static org.apache.cassandra.utils.memory.MemoryUtil.isExactlyDirect;
/**
* A pool of ByteBuffers that can be recycled.
+ *
+ * TODO: document the semantics of this class carefully
+ * Notably: we do not automatically release from the local pool any chunk that has been incompletely allocated from
*/
public class BufferPool
{
- /** The size of a page aligned buffer, 64KiB */
- public static final int CHUNK_SIZE = 64 << 10;
+ /** The size of a page aligned buffer, 128KiB */
+ public static final int NORMAL_CHUNK_SIZE = 128 << 10;
+ public static final int NORMAL_ALLOCATION_UNIT = NORMAL_CHUNK_SIZE / 64;
+ public static final int TINY_CHUNK_SIZE = NORMAL_ALLOCATION_UNIT;
+ public static final int TINY_ALLOCATION_UNIT = TINY_CHUNK_SIZE / 64;
+ public static final int TINY_ALLOCATION_LIMIT = TINY_CHUNK_SIZE / 2;
+ private final static BufferPoolMetrics metrics = new BufferPoolMetrics();
+
+ // TODO: this should not be using FileCacheSizeInMB
@VisibleForTesting
public static long MEMORY_USAGE_THRESHOLD = DatabaseDescriptor.getFileCacheSizeInMB() * 1024L * 1024L;
- @VisibleForTesting
- public static boolean ALLOCATE_ON_HEAP_WHEN_EXAHUSTED = DatabaseDescriptor.getBufferPoolUseHeapIfExhausted();
-
- @VisibleForTesting
- public static boolean DISABLED = Boolean.parseBoolean(System.getProperty("cassandra.test.disable_buffer_pool", "false"));
-
- @VisibleForTesting
- public static boolean DEBUG = false;
+ private static Debug debug;
private static final Logger logger = LoggerFactory.getLogger(BufferPool.class);
private static final NoSpamLogger noSpamLogger = NoSpamLogger.getLogger(logger, 15L, TimeUnit.MINUTES);
@@ -79,110 +91,68 @@
{
return new LocalPool();
}
- };
- public static ByteBuffer get(int size)
- {
- if (DISABLED)
- return allocate(size, ALLOCATE_ON_HEAP_WHEN_EXAHUSTED);
- else
- return takeFromPool(size, ALLOCATE_ON_HEAP_WHEN_EXAHUSTED);
- }
+ protected void onRemoval(LocalPool value)
+ {
+ value.release();
+ }
+ };
public static ByteBuffer get(int size, BufferType bufferType)
{
- boolean direct = bufferType == BufferType.OFF_HEAP;
- if (DISABLED || !direct)
- return allocate(size, !direct);
+ if (bufferType == BufferType.ON_HEAP)
+ return allocate(size, bufferType);
else
- return takeFromPool(size, !direct);
+ return localPool.get().get(size);
+ }
+
+ public static ByteBuffer getAtLeast(int size, BufferType bufferType)
+ {
+ if (bufferType == BufferType.ON_HEAP)
+ return allocate(size, bufferType);
+ else
+ return localPool.get().getAtLeast(size);
}
/** Unlike the get methods, this will return null if the pool is exhausted */
public static ByteBuffer tryGet(int size)
{
- if (DISABLED)
- return allocate(size, ALLOCATE_ON_HEAP_WHEN_EXAHUSTED);
- else
- return maybeTakeFromPool(size, ALLOCATE_ON_HEAP_WHEN_EXAHUSTED);
+ return localPool.get().tryGet(size, false);
}
- private static ByteBuffer allocate(int size, boolean onHeap)
+ public static ByteBuffer tryGetAtLeast(int size)
{
- return onHeap
+ return localPool.get().tryGet(size, true);
+ }
+
+ private static ByteBuffer allocate(int size, BufferType bufferType)
+ {
+ return bufferType == BufferType.ON_HEAP
? ByteBuffer.allocate(size)
: ByteBuffer.allocateDirect(size);
}
- private static ByteBuffer takeFromPool(int size, boolean allocateOnHeapWhenExhausted)
- {
- ByteBuffer ret = maybeTakeFromPool(size, allocateOnHeapWhenExhausted);
- if (ret != null)
- return ret;
-
- if (logger.isTraceEnabled())
- logger.trace("Requested buffer size {} has been allocated directly due to lack of capacity", FBUtilities.prettyPrintMemory(size));
-
- return localPool.get().allocate(size, allocateOnHeapWhenExhausted);
- }
-
- private static ByteBuffer maybeTakeFromPool(int size, boolean allocateOnHeapWhenExhausted)
- {
- if (size < 0)
- throw new IllegalArgumentException("Size must be positive (" + size + ")");
-
- if (size == 0)
- return EMPTY_BUFFER;
-
- if (size > CHUNK_SIZE)
- {
- if (logger.isTraceEnabled())
- logger.trace("Requested buffer size {} is bigger than {}, allocating directly",
- FBUtilities.prettyPrintMemory(size),
- FBUtilities.prettyPrintMemory(CHUNK_SIZE));
-
- return localPool.get().allocate(size, allocateOnHeapWhenExhausted);
- }
-
- return localPool.get().get(size);
- }
-
public static void put(ByteBuffer buffer)
{
- if (!(DISABLED || buffer.hasArray()))
+ if (isExactlyDirect(buffer))
localPool.get().put(buffer);
}
- /** This is not thread safe and should only be used for unit testing. */
- @VisibleForTesting
- static void reset()
+ public static void putUnusedPortion(ByteBuffer buffer)
{
- localPool.get().reset();
- globalPool.reset();
- }
-
- @VisibleForTesting
- static Chunk currentChunk()
- {
- return localPool.get().chunks[0];
- }
-
- @VisibleForTesting
- static int numChunks()
- {
- int ret = 0;
- for (Chunk chunk : localPool.get().chunks)
+ if (isExactlyDirect(buffer))
{
- if (chunk != null)
- ret++;
+ LocalPool pool = localPool.get();
+ if (buffer.limit() > 0)
+ pool.putUnusedPortion(buffer);
+ else
+ pool.put(buffer);
}
- return ret;
}
- @VisibleForTesting
- static void assertAllRecycled()
+ public static void setRecycleWhenFreeForCurrentThread(boolean recycleWhenFree)
{
- globalPool.debug.check();
+ localPool.get().recycleWhenFree(recycleWhenFree);
}
public static long sizeInBytes()
@@ -190,24 +160,20 @@
return globalPool.sizeInBytes();
}
- static final class Debug
+ interface Debug
{
- long recycleRound = 1;
- final Queue<Chunk> allChunks = new ConcurrentLinkedQueue<>();
- void register(Chunk chunk)
- {
- allChunks.add(chunk);
- }
- void recycle(Chunk chunk)
- {
- chunk.lastRecycled = recycleRound;
- }
- void check()
- {
- for (Chunk chunk : allChunks)
- assert chunk.lastRecycled == recycleRound;
- recycleRound++;
- }
+ void registerNormal(Chunk chunk);
+ void recycleNormal(Chunk oldVersion, Chunk newVersion);
+ }
+
+ public static void debug(Debug setDebug)
+ {
+ debug = setDebug;
+ }
+
+ interface Recycler
+ {
+ void recycle(Chunk chunk);
}
/**
@@ -217,26 +183,21 @@
*
* This class is shared by multiple thread local pools and must be thread-safe.
*/
- static final class GlobalPool
+ static final class GlobalPool implements Supplier<Chunk>, Recycler
{
- /** The size of a bigger chunk, 1-mbit, must be a multiple of CHUNK_SIZE */
- static final int MACRO_CHUNK_SIZE = 1 << 20;
+ /** The size of a bigger chunk, 1 MiB, must be a multiple of NORMAL_CHUNK_SIZE */
+ static final int MACRO_CHUNK_SIZE = 64 * NORMAL_CHUNK_SIZE;
static
{
- assert Integer.bitCount(CHUNK_SIZE) == 1; // must be a power of 2
+ assert Integer.bitCount(NORMAL_CHUNK_SIZE) == 1; // must be a power of 2
assert Integer.bitCount(MACRO_CHUNK_SIZE) == 1; // must be a power of 2
- assert MACRO_CHUNK_SIZE % CHUNK_SIZE == 0; // must be a multiple
+ assert MACRO_CHUNK_SIZE % NORMAL_CHUNK_SIZE == 0; // must be a multiple
- if (DISABLED)
- logger.info("Global buffer pool is disabled, allocating {}", ALLOCATE_ON_HEAP_WHEN_EXAHUSTED ? "on heap" : "off heap");
- else
- logger.info("Global buffer pool is enabled, when pool is exhausted (max is {}) it will allocate {}",
- FBUtilities.prettyPrintMemory(MEMORY_USAGE_THRESHOLD),
- ALLOCATE_ON_HEAP_WHEN_EXAHUSTED ? "on heap" : "off heap");
+ logger.info("Global buffer pool limit is {}",
+ prettyPrintMemory(MEMORY_USAGE_THRESHOLD));
}
- private final Debug debug = new Debug();
private final Queue<Chunk> macroChunks = new ConcurrentLinkedQueue<>();
// TODO (future): it would be preferable to use a CLStack to improve cache occupancy; it would also be preferable to use "CoreLocal" storage
private final Queue<Chunk> chunks = new ConcurrentLinkedQueue<>();
@@ -245,32 +206,36 @@
/** Return a chunk, the caller will take owership of the parent chunk. */
public Chunk get()
{
- while (true)
- {
- Chunk chunk = chunks.poll();
- if (chunk != null)
- return chunk;
+ Chunk chunk = chunks.poll();
+ if (chunk != null)
+ return chunk;
- if (!allocateMoreChunks())
- // give it one last attempt, in case someone else allocated before us
- return chunks.poll();
- }
+ chunk = allocateMoreChunks();
+ if (chunk != null)
+ return chunk;
+
+ // another thread may have just allocated last macro chunk, so make one final attempt before returning null
+ return chunks.poll();
}
/**
* This method might be called by multiple threads and that's fine if we add more
* than one chunk at the same time as long as we don't exceed the MEMORY_USAGE_THRESHOLD.
*/
- private boolean allocateMoreChunks()
+ private Chunk allocateMoreChunks()
{
while (true)
{
long cur = memoryUsage.get();
if (cur + MACRO_CHUNK_SIZE > MEMORY_USAGE_THRESHOLD)
{
- noSpamLogger.info("Maximum memory usage reached ({}), cannot allocate chunk of {}",
- MEMORY_USAGE_THRESHOLD, MACRO_CHUNK_SIZE);
- return false;
+ if (MEMORY_USAGE_THRESHOLD > 0)
+ {
+ noSpamLogger.info("Maximum memory usage reached ({}), cannot allocate chunk of {}",
+ prettyPrintMemory(MEMORY_USAGE_THRESHOLD),
+ prettyPrintMemory(MACRO_CHUNK_SIZE));
+ }
+ return null;
}
if (memoryUsage.compareAndSet(cur, cur + MACRO_CHUNK_SIZE))
break;
@@ -280,33 +245,41 @@
Chunk chunk;
try
{
- chunk = new Chunk(allocateDirectAligned(MACRO_CHUNK_SIZE));
+ chunk = new Chunk(null, allocateDirectAligned(MACRO_CHUNK_SIZE));
}
catch (OutOfMemoryError oom)
{
noSpamLogger.error("Buffer pool failed to allocate chunk of {}, current size {} ({}). " +
"Attempting to continue; buffers will be allocated in on-heap memory which can degrade performance. " +
"Make sure direct memory size (-XX:MaxDirectMemorySize) is large enough to accommodate off-heap memtables and caches.",
- MACRO_CHUNK_SIZE, sizeInBytes(), oom.toString());
- return false;
+ prettyPrintMemory(MACRO_CHUNK_SIZE),
+ prettyPrintMemory(sizeInBytes()),
+ oom.toString());
+ return null;
}
chunk.acquire(null);
macroChunks.add(chunk);
- for (int i = 0 ; i < MACRO_CHUNK_SIZE ; i += CHUNK_SIZE)
- {
- Chunk add = new Chunk(chunk.get(CHUNK_SIZE));
- chunks.add(add);
- if (DEBUG)
- debug.register(add);
- }
- return true;
+ final Chunk callerChunk = new Chunk(this, chunk.get(NORMAL_CHUNK_SIZE));
+ if (debug != null)
+ debug.registerNormal(callerChunk);
+ for (int i = NORMAL_CHUNK_SIZE; i < MACRO_CHUNK_SIZE; i += NORMAL_CHUNK_SIZE)
+ {
+ Chunk add = new Chunk(this, chunk.get(NORMAL_CHUNK_SIZE));
+ chunks.add(add);
+ if (debug != null)
+ debug.registerNormal(add);
+ }
+ return callerChunk;
}
public void recycle(Chunk chunk)
{
- chunks.add(chunk);
+ Chunk recycleAs = new Chunk(chunk);
+ if (debug != null)
+ debug.recycleNormal(chunk, recycleAs);
+ chunks.add(recycleAs);
}
public long sizeInBytes()
@@ -316,25 +289,21 @@
/** This is not thread safe and should only be used for unit testing. */
@VisibleForTesting
- void reset()
+ void unsafeFree()
{
while (!chunks.isEmpty())
- chunks.poll().reset();
+ chunks.poll().unsafeFree();
while (!macroChunks.isEmpty())
- macroChunks.poll().reset();
+ macroChunks.poll().unsafeFree();
memoryUsage.set(0);
}
}
- /**
- * A thread local class that grabs chunks from the global pool for this thread allocations.
- * Only one thread can do the allocations but multiple threads can release the allocations.
- */
- static final class LocalPool
+ private static class MicroQueueOfChunks
{
- private final static BufferPoolMetrics metrics = new BufferPoolMetrics();
+
// a microqueue of Chunks:
// * if any are null, they are at the end;
// * new Chunks are added to the last null index
@@ -342,17 +311,430 @@
// * this results in a queue that will typically be visited in ascending order of available space, so that
// small allocations preferentially slice from the Chunks with the smallest space available to furnish them
// WARNING: if we ever change the size of this, we must update removeFromLocalQueue, and addChunk
- private final Chunk[] chunks = new Chunk[3];
- private byte chunkCount = 0;
+ private Chunk chunk0, chunk1, chunk2;
+ private int count;
+
+ // add a new chunk, if necessary evicting the chunk with the least available memory (returning the evicted chunk)
+ private Chunk add(Chunk chunk)
+ {
+ switch (count)
+ {
+ case 0:
+ chunk0 = chunk;
+ count = 1;
+ break;
+ case 1:
+ chunk1 = chunk;
+ count = 2;
+ break;
+ case 2:
+ chunk2 = chunk;
+ count = 3;
+ break;
+ case 3:
+ {
+ Chunk release;
+ int chunk0Free = chunk0.freeSlotCount();
+ int chunk1Free = chunk1.freeSlotCount();
+ int chunk2Free = chunk2.freeSlotCount();
+ if (chunk0Free < chunk1Free)
+ {
+ if (chunk0Free < chunk2Free)
+ {
+ release = chunk0;
+ chunk0 = chunk;
+ }
+ else
+ {
+ release = chunk2;
+ chunk2 = chunk;
+ }
+ }
+ else
+ {
+ if (chunk1Free < chunk2Free)
+ {
+ release = chunk1;
+ chunk1 = chunk;
+ }
+ else
+ {
+ release = chunk2;
+ chunk2 = chunk;
+ }
+ }
+ return release;
+ }
+ default:
+ throw new IllegalStateException();
+ }
+ return null;
+ }
+
+ private void remove(Chunk chunk)
+ {
+ // since we only have three elements in the queue, it is clearer, easier and faster to just hard code the options
+ if (chunk0 == chunk)
+ { // remove first by shifting back second two
+ chunk0 = chunk1;
+ chunk1 = chunk2;
+ }
+ else if (chunk1 == chunk)
+ { // remove second by shifting back last
+ chunk1 = chunk2;
+ }
+ else if (chunk2 != chunk)
+ {
+ return;
+ }
+ // whatever we do, the last element must be null
+ chunk2 = null;
+ --count;
+ }
+
+ ByteBuffer get(int size, boolean sizeIsLowerBound, ByteBuffer reuse)
+ {
+ ByteBuffer buffer;
+ if (null != chunk0)
+ {
+ if (null != (buffer = chunk0.get(size, sizeIsLowerBound, reuse)))
+ return buffer;
+ if (null != chunk1)
+ {
+ if (null != (buffer = chunk1.get(size, sizeIsLowerBound, reuse)))
+ return buffer;
+ if (null != chunk2 && null != (buffer = chunk2.get(size, sizeIsLowerBound, reuse)))
+ return buffer;
+ }
+ }
+ return null;
+ }
+
+ private void forEach(Consumer<Chunk> consumer)
+ {
+ forEach(consumer, count, chunk0, chunk1, chunk2);
+ }
+
+ private void clearForEach(Consumer<Chunk> consumer)
+ {
+ Chunk chunk0 = this.chunk0, chunk1 = this.chunk1, chunk2 = this.chunk2;
+ this.chunk0 = this.chunk1 = this.chunk2 = null;
+ forEach(consumer, count, chunk0, chunk1, chunk2);
+ count = 0;
+ }
+
+ private static void forEach(Consumer<Chunk> consumer, int count, Chunk chunk0, Chunk chunk1, Chunk chunk2)
+ {
+ switch (count)
+ {
+ case 3:
+ consumer.accept(chunk2);
+ case 2:
+ consumer.accept(chunk1);
+ case 1:
+ consumer.accept(chunk0);
+ }
+ }
+
+ private <T> void removeIf(BiPredicate<Chunk, T> predicate, T value)
+ {
+ // do not release matching chunks before we move null chunks to the back of the queue;
+ // because, with current buffer release from another thread, "chunk#release()" may eventually come back to
+ // "removeIf" causing NPE as null chunks are not at the back of the queue.
+ Chunk toRelease0 = null, toRelease1 = null, toRelease2 = null;
+
+ try
+ {
+ switch (count)
+ {
+ case 3:
+ if (predicate.test(chunk2, value))
+ {
+ --count;
+ toRelease2 = chunk2;
+ chunk2 = null;
+ }
+ case 2:
+ if (predicate.test(chunk1, value))
+ {
+ --count;
+ toRelease1 = chunk1;
+ chunk1 = null;
+ }
+ case 1:
+ if (predicate.test(chunk0, value))
+ {
+ --count;
+ toRelease0 = chunk0;
+ chunk0 = null;
+ }
+ break;
+ case 0:
+ return;
+ }
+ switch (count)
+ {
+ case 2:
+ // Find the only null item, and shift non-null so that null is at chunk2
+ if (chunk0 == null)
+ {
+ chunk0 = chunk1;
+ chunk1 = chunk2;
+ chunk2 = null;
+ }
+ else if (chunk1 == null)
+ {
+ chunk1 = chunk2;
+ chunk2 = null;
+ }
+ break;
+ case 1:
+ // Find the only non-null item, and shift it to chunk0
+ if (chunk1 != null)
+ {
+ chunk0 = chunk1;
+ chunk1 = null;
+ }
+ else if (chunk2 != null)
+ {
+ chunk0 = chunk2;
+ chunk2 = null;
+ }
+ break;
+ }
+ }
+ finally
+ {
+ if (toRelease0 != null)
+ toRelease0.release();
+
+ if (toRelease1 != null)
+ toRelease1.release();
+
+ if (toRelease2 != null)
+ toRelease2.release();
+ }
+ }
+
+ private void release()
+ {
+ clearForEach(Chunk::release);
+ }
+
+ private void unsafeRecycle()
+ {
+ clearForEach(Chunk::unsafeRecycle);
+ }
+ }
+
+ /**
+ * A thread local class that grabs chunks from the global pool for this thread allocations.
+ * Only one thread can do the allocations but multiple threads can release the allocations.
+ */
+ public static final class LocalPool implements Recycler
+ {
+ private final Queue<ByteBuffer> reuseObjects;
+ private final Supplier<Chunk> parent;
+ private final LocalPoolRef leakRef;
+
+ private final MicroQueueOfChunks chunks = new MicroQueueOfChunks();
+ /**
+ * If we are on outer LocalPool, whose chunks are == NORMAL_CHUNK_SIZE, we may service allocation requests
+ * for buffers much smaller than
+ */
+ private LocalPool tinyPool;
+ private final int tinyLimit;
+ private boolean recycleWhenFree = true;
public LocalPool()
{
- localPoolReferences.add(new LocalPoolRef(this, localPoolRefQueue));
+ this.parent = globalPool;
+ this.tinyLimit = TINY_ALLOCATION_LIMIT;
+ this.reuseObjects = new ArrayDeque<>();
+ localPoolReferences.add(leakRef = new LocalPoolRef(this, localPoolRefQueue));
}
- private Chunk addChunkFromGlobalPool()
+ /**
+ * Invoked by an existing LocalPool, to create a child pool
+ */
+ private LocalPool(LocalPool parent)
{
- Chunk chunk = globalPool.get();
+ this.parent = () -> {
+ ByteBuffer buffer = parent.tryGetInternal(TINY_CHUNK_SIZE, false);
+ if (buffer == null)
+ return null;
+ return new Chunk(parent, buffer);
+ };
+ this.tinyLimit = 0; // we only currently permit one layer of nesting (which brings us down to 32 byte allocations, so is plenty)
+ this.reuseObjects = parent.reuseObjects; // we share the same ByteBuffer object reuse pool, as we both have the same exclusive access to it
+ localPoolReferences.add(leakRef = new LocalPoolRef(this, localPoolRefQueue));
+ }
+
+ private LocalPool tinyPool()
+ {
+ if (tinyPool == null)
+ tinyPool = new LocalPool(this).recycleWhenFree(recycleWhenFree);
+ return tinyPool;
+ }
+
+ public void put(ByteBuffer buffer)
+ {
+ Chunk chunk = Chunk.getParentChunk(buffer);
+ if (chunk == null)
+ FileUtils.clean(buffer);
+ else
+ put(buffer, chunk);
+ }
+
+ public void put(ByteBuffer buffer, Chunk chunk)
+ {
+ LocalPool owner = chunk.owner;
+ if (owner != null && owner == tinyPool)
+ {
+ tinyPool.put(buffer, chunk);
+ return;
+ }
+
+ // ask the free method to take exclusive ownership of the act of recycling
+ // if we are either: already not owned by anyone, or owned by ourselves
+ long free = chunk.free(buffer, owner == null || (owner == this && recycleWhenFree));
+ if (free == 0L)
+ {
+ // 0L => we own recycling responsibility, so must recycle;
+ // if we are the owner, we must remove the Chunk from our local queue
+ if (owner == this)
+ remove(chunk);
+ chunk.recycle();
+ }
+ else if (((free == -1L) && owner != this) && chunk.owner == null)
+ {
+ // although we try to take recycle ownership cheaply, it is not always possible to do so if the owner is racing to unset.
+ // we must also check after completely freeing if the owner has since been unset, and try to recycle
+ chunk.tryRecycle();
+ }
+
+ if (owner == this)
+ {
+ MemoryUtil.setAttachment(buffer, null);
+ MemoryUtil.setDirectByteBuffer(buffer, 0, 0);
+ reuseObjects.add(buffer);
+ }
+ }
+
+ public void putUnusedPortion(ByteBuffer buffer)
+ {
+ Chunk chunk = Chunk.getParentChunk(buffer);
+ if (chunk == null)
+ return;
+
+ chunk.freeUnusedPortion(buffer);
+ }
+
+ public ByteBuffer get(int size)
+ {
+ return get(size, false);
+ }
+
+ public ByteBuffer getAtLeast(int size)
+ {
+ return get(size, true);
+ }
+
+ private ByteBuffer get(int size, boolean sizeIsLowerBound)
+ {
+ ByteBuffer ret = tryGet(size, sizeIsLowerBound);
+ if (ret != null)
+ return ret;
+
+ if (size > NORMAL_CHUNK_SIZE)
+ {
+ if (logger.isTraceEnabled())
+ logger.trace("Requested buffer size {} is bigger than {}; allocating directly",
+ prettyPrintMemory(size),
+ prettyPrintMemory(NORMAL_CHUNK_SIZE));
+ }
+ else
+ {
+ if (logger.isTraceEnabled())
+ logger.trace("Requested buffer size {} has been allocated directly due to lack of capacity", prettyPrintMemory(size));
+ }
+
+ metrics.misses.mark();
+ return allocate(size, BufferType.OFF_HEAP);
+ }
+
+ public ByteBuffer tryGet(int size)
+ {
+ return tryGet(size, false);
+ }
+
+ public ByteBuffer tryGetAtLeast(int size)
+ {
+ return tryGet(size, true);
+ }
+
+ private ByteBuffer tryGet(int size, boolean sizeIsLowerBound)
+ {
+ LocalPool pool = this;
+ if (size <= tinyLimit)
+ {
+ if (size <= 0)
+ {
+ if (size == 0)
+ return EMPTY_BUFFER;
+ throw new IllegalArgumentException("Size must be non-negative (" + size + ')');
+ }
+
+ pool = tinyPool();
+ }
+ else if (size > NORMAL_CHUNK_SIZE)
+ {
+ return null;
+ }
+
+ return pool.tryGetInternal(size, sizeIsLowerBound);
+ }
+
+ @Inline
+ private ByteBuffer tryGetInternal(int size, boolean sizeIsLowerBound)
+ {
+ ByteBuffer reuse = this.reuseObjects.poll();
+ ByteBuffer buffer = chunks.get(size, sizeIsLowerBound, reuse);
+ if (buffer != null)
+ return buffer;
+
+ // else ask the global pool
+ Chunk chunk = addChunkFromParent();
+ if (chunk != null)
+ {
+ ByteBuffer result = chunk.get(size, sizeIsLowerBound, reuse);
+ if (result != null)
+ return result;
+ }
+
+ if (reuse != null)
+ this.reuseObjects.add(reuse);
+ return null;
+ }
+
+ // recycle
+ public void recycle(Chunk chunk)
+ {
+ ByteBuffer buffer = chunk.slab;
+ Chunk parentChunk = Chunk.getParentChunk(buffer);
+ put(buffer, parentChunk);
+ }
+
+ private void remove(Chunk chunk)
+ {
+ chunks.remove(chunk);
+ if (tinyPool != null)
+ tinyPool.chunks.removeIf((child, parent) -> Chunk.getParentChunk(child.slab) == parent, chunk);
+ }
+
+ private Chunk addChunkFromParent()
+ {
+ Chunk chunk = parent.get();
if (chunk == null)
return null;
@@ -363,118 +745,43 @@
private void addChunk(Chunk chunk)
{
chunk.acquire(this);
-
- if (chunkCount < 3)
+ Chunk evict = chunks.add(chunk);
+ if (evict != null)
{
- chunks[chunkCount++] = chunk;
- return;
- }
-
- int smallestChunkIdx = 0;
- if (chunks[1].free() < chunks[0].free())
- smallestChunkIdx = 1;
- if (chunks[2].free() < chunks[smallestChunkIdx].free())
- smallestChunkIdx = 2;
-
- chunks[smallestChunkIdx].release();
- if (smallestChunkIdx != 2)
- chunks[smallestChunkIdx] = chunks[2];
- chunks[2] = chunk;
- }
-
- public ByteBuffer get(int size)
- {
- for (Chunk chunk : chunks)
- { // first see if our own chunks can serve this buffer
- if (chunk == null)
- break;
-
- ByteBuffer buffer = chunk.get(size);
- if (buffer != null)
- return buffer;
- }
-
- // else ask the global pool
- Chunk chunk = addChunkFromGlobalPool();
- if (chunk != null)
- return chunk.get(size);
-
- return null;
- }
-
- private ByteBuffer allocate(int size, boolean onHeap)
- {
- metrics.misses.mark();
- return BufferPool.allocate(size, onHeap);
- }
-
- public void put(ByteBuffer buffer)
- {
- Chunk chunk = Chunk.getParentChunk(buffer);
- if (chunk == null)
- {
- FileUtils.clean(buffer);
- return;
- }
-
- LocalPool owner = chunk.owner;
- // ask the free method to take exclusive ownership of the act of recycling
- // if we are either: already not owned by anyone, or owned by ourselves
- long free = chunk.free(buffer, owner == null | owner == this);
- if (free == 0L)
- {
- // 0L => we own recycling responsibility, so must recycle;
- chunk.recycle();
- // if we are also the owner, we must remove the Chunk from our local queue
- if (owner == this)
- removeFromLocalQueue(chunk);
- }
- else if (((free == -1L) && owner != this) && chunk.owner == null)
- {
- // although we try to take recycle ownership cheaply, it is not always possible to do so if the owner is racing to unset.
- // we must also check after completely freeing if the owner has since been unset, and try to recycle
- chunk.tryRecycle();
+ if (tinyPool != null)
+ tinyPool.chunks.removeIf((child, parent) -> Chunk.getParentChunk(child.slab) == parent, evict);
+ evict.release();
}
}
- private void removeFromLocalQueue(Chunk chunk)
+ public void release()
{
- // since we only have three elements in the queue, it is clearer, easier and faster to just hard code the options
- if (chunks[0] == chunk)
- { // remove first by shifting back second two
- chunks[0] = chunks[1];
- chunks[1] = chunks[2];
- }
- else if (chunks[1] == chunk)
- { // remove second by shifting back last
- chunks[1] = chunks[2];
- }
- else assert chunks[2] == chunk;
- // whatever we do, the last element myst be null
- chunks[2] = null;
- chunkCount--;
+ chunks.release();
+ reuseObjects.clear();
+ localPoolReferences.remove(leakRef);
+ leakRef.clear();
+ if (tinyPool != null)
+ tinyPool.release();
}
@VisibleForTesting
- void reset()
+ void unsafeRecycle()
{
- chunkCount = 0;
- for (int i = 0; i < chunks.length; i++)
- {
- if (chunks[i] != null)
- {
- chunks[i].owner = null;
- chunks[i].freeSlots = 0L;
- chunks[i].recycle();
- chunks[i] = null;
- }
- }
+ chunks.unsafeRecycle();
+ }
+
+ public LocalPool recycleWhenFree(boolean recycleWhenFree)
+ {
+ this.recycleWhenFree = recycleWhenFree;
+ if (tinyPool != null)
+ tinyPool.recycleWhenFree = recycleWhenFree;
+ return this;
}
}
- private static final class LocalPoolRef extends PhantomReference<LocalPool>
+ private static final class LocalPoolRef extends PhantomReference<LocalPool>
{
- private final Chunk[] chunks;
+ private final MicroQueueOfChunks chunks;
public LocalPoolRef(LocalPool localPool, ReferenceQueue<? super LocalPool> q)
{
super(localPool, q);
@@ -483,18 +790,11 @@
public void release()
{
- for (int i = 0 ; i < chunks.length ; i++)
- {
- if (chunks[i] != null)
- {
- chunks[i].release();
- chunks[i] = null;
- }
- }
+ chunks.release();
}
}
- private static final ConcurrentLinkedQueue<LocalPoolRef> localPoolReferences = new ConcurrentLinkedQueue<>();
+ private static final Set<LocalPoolRef> localPoolReferences = Collections.newSetFromMap(new ConcurrentHashMap<>());
private static final ReferenceQueue<Object> localPoolRefQueue = new ReferenceQueue<>();
private static final InfiniteLoopExecutor EXEC = new InfiniteLoopExecutor("LocalPool-Cleaner", BufferPool::cleanupOneReference).start();
@@ -562,8 +862,10 @@
// if this is set, it means the chunk may not be recycled because we may still allocate from it;
// if it has been unset the local pool has finished with it, and it may be recycled
private volatile LocalPool owner;
- private long lastRecycled;
- private final Chunk original;
+ private final Recycler recycler;
+
+ @VisibleForTesting
+ Object debugAttachment;
Chunk(Chunk recycle)
{
@@ -572,14 +874,13 @@
this.baseAddress = recycle.baseAddress;
this.shift = recycle.shift;
this.freeSlots = -1L;
- this.original = recycle.original;
- if (DEBUG)
- globalPool.debug.recycle(original);
+ this.recycler = recycle.recycler;
}
- Chunk(ByteBuffer slab)
+ Chunk(Recycler recycler, ByteBuffer slab)
{
- assert !slab.hasArray();
+ assert MemoryUtil.isExactlyDirect(slab);
+ this.recycler = recycler;
this.slab = slab;
this.baseAddress = MemoryUtil.getAddress(slab);
@@ -588,7 +889,6 @@
this.shift = 31 & (Integer.numberOfTrailingZeros(slab.capacity() / 64));
// -1 means all free whilst 0 means all in use
this.freeSlots = slab.capacity() == 0 ? 0L : -1L;
- this.original = DEBUG ? this : null;
}
/**
@@ -622,7 +922,7 @@
void recycle()
{
assert freeSlots == 0L;
- globalPool.recycle(new Chunk(this));
+ recycler.recycle(this);
}
/**
@@ -643,14 +943,12 @@
return null;
}
- ByteBuffer setAttachment(ByteBuffer buffer)
+ void setAttachment(ByteBuffer buffer)
{
if (Ref.DEBUG_ENABLED)
MemoryUtil.setAttachment(buffer, new Ref<>(this, null));
else
MemoryUtil.setAttachment(buffer, this);
-
- return buffer;
}
boolean releaseAttachment(ByteBuffer buffer)
@@ -659,23 +957,13 @@
if (attachment == null)
return false;
- if (attachment instanceof Ref)
+ if (Ref.DEBUG_ENABLED)
((Ref<Chunk>) attachment).release();
return true;
}
@VisibleForTesting
- void reset()
- {
- Chunk parent = getParentChunk(slab);
- if (parent != null)
- parent.free(slab, false);
- else
- FileUtils.clean(slab);
- }
-
- @VisibleForTesting
long setFreeSlots(long val)
{
long ret = freeSlots;
@@ -704,15 +992,27 @@
return Long.bitCount(freeSlots) * unit();
}
+ int freeSlotCount()
+ {
+ return Long.bitCount(freeSlots);
+ }
+
+ ByteBuffer get(int size)
+ {
+ return get(size, false, null);
+ }
+
/**
* Return the next available slice of this size. If
* we have exceeded the capacity we return null.
*/
- ByteBuffer get(int size)
+ ByteBuffer get(int size, boolean sizeIsLowerBound, ByteBuffer into)
{
// how many multiples of our units is the size?
// we add (unit - 1), so that when we divide by unit (>>> shift), we effectively round up
int slotCount = (size - 1 + unit()) >>> shift;
+ if (sizeIsLowerBound)
+ size = slotCount << shift;
// if we require more than 64 slots, we cannot possibly accommodate the allocation
if (slotCount > 64)
@@ -776,17 +1076,18 @@
// make sure no other thread has cleared the candidate bits
assert ((candidate & cur) == candidate);
}
- return get(index << shift, size);
+ return set(index << shift, size, into);
}
}
}
- private ByteBuffer get(int offset, int size)
+ private ByteBuffer set(int offset, int size, ByteBuffer into)
{
- slab.limit(offset + size);
- slab.position(offset);
-
- return setAttachment(slab.slice());
+ if (into == null)
+ into = MemoryUtil.getHollowDirectByteBuffer(ByteOrder.BIG_ENDIAN);
+ MemoryUtil.sliceDirectByteBuffer(slab, into, offset, size);
+ setAttachment(into);
+ return into;
}
/**
@@ -808,25 +1109,16 @@
if (!releaseAttachment(buffer))
return 1L;
- long address = MemoryUtil.getAddress(buffer);
- assert (address >= baseAddress) & (address <= baseAddress + capacity());
-
- int position = (int)(address - baseAddress);
int size = roundUp(buffer.capacity());
+ long address = MemoryUtil.getAddress(buffer);
+ assert (address >= baseAddress) & (address + size <= baseAddress + capacity());
- position >>= shift;
+ int position = ((int)(address - baseAddress)) >> shift;
+
int slotCount = size >> shift;
-
- long slotBits = (1L << slotCount) - 1;
+ long slotBits = 0xffffffffffffffffL >>> (64 - slotCount);
long shiftedSlotBits = (slotBits << position);
- if (slotCount == 64)
- {
- assert size == capacity();
- assert position == 0;
- shiftedSlotBits = -1L;
- }
-
long next;
while (true)
{
@@ -840,20 +1132,72 @@
}
}
+ void freeUnusedPortion(ByteBuffer buffer)
+ {
+ int size = roundUp(buffer.limit());
+ int capacity = roundUp(buffer.capacity());
+ if (size == capacity)
+ return;
+
+ long address = MemoryUtil.getAddress(buffer);
+ assert (address >= baseAddress) & (address + size <= baseAddress + capacity());
+
+ // free any spare slots above the size we are using
+ int position = ((int)(address + size - baseAddress)) >> shift;
+ int slotCount = (capacity - size) >> shift;
+
+ long slotBits = 0xffffffffffffffffL >>> (64 - slotCount);
+ long shiftedSlotBits = (slotBits << position);
+
+ long next;
+ while (true)
+ {
+ long cur = freeSlots;
+ next = cur | shiftedSlotBits;
+ assert next == (cur ^ shiftedSlotBits); // ensure no double free
+ if (freeSlotsUpdater.compareAndSet(this, cur, next))
+ break;
+ }
+ MemoryUtil.setByteBufferCapacity(buffer, size);
+ }
+
@Override
public String toString()
{
return String.format("[slab %s, slots bitmap %s, capacity %d, free %d]", slab, Long.toBinaryString(freeSlots), capacity(), free());
}
+
+ @VisibleForTesting
+ void unsafeFree()
+ {
+ Chunk parent = getParentChunk(slab);
+ if (parent != null)
+ parent.free(slab, false);
+ else
+ FileUtils.clean(slab);
+ }
+
+ static void unsafeRecycle(Chunk chunk)
+ {
+ if (chunk != null)
+ {
+ chunk.owner = null;
+ chunk.freeSlots = 0L;
+ chunk.recycle();
+ }
+ }
}
@VisibleForTesting
- public static int roundUpNormal(int size)
+ public static int roundUp(int size)
{
- return roundUp(size, CHUNK_SIZE / 64);
+ if (size <= TINY_ALLOCATION_LIMIT)
+ return roundUp(size, TINY_ALLOCATION_UNIT);
+ return roundUp(size, NORMAL_ALLOCATION_UNIT);
}
- private static int roundUp(int size, int unit)
+ @VisibleForTesting
+ public static int roundUp(int size, int unit)
{
int mask = unit - 1;
return (size + mask) & ~mask;
@@ -862,7 +1206,47 @@
@VisibleForTesting
public static void shutdownLocalCleaner(long timeout, TimeUnit unit) throws InterruptedException, TimeoutException
{
- shutdownNow(Arrays.asList(EXEC));
- awaitTermination(timeout, unit, Arrays.asList(EXEC));
+ shutdownNow(of(EXEC));
+ awaitTermination(timeout, unit, of(EXEC));
}
+
+ public static long unsafeGetBytesInUse()
+ {
+ long totalMemory = globalPool.memoryUsage.get();
+ class L { long v; }
+ final L availableMemory = new L();
+ for (Chunk chunk : globalPool.chunks)
+ {
+ availableMemory.v += chunk.capacity();
+ }
+ for (LocalPoolRef ref : localPoolReferences)
+ {
+ ref.chunks.forEach(chunk -> availableMemory.v += chunk.free());
+ }
+ return totalMemory - availableMemory.v;
+ }
+
+ /** This is not thread safe and should only be used for unit testing. */
+ @VisibleForTesting
+ static void unsafeReset()
+ {
+ localPool.get().unsafeRecycle();
+ globalPool.unsafeFree();
+ }
+
+ @VisibleForTesting
+ static Chunk unsafeCurrentChunk()
+ {
+ return localPool.get().chunks.chunk0;
+ }
+
+ @VisibleForTesting
+ static int unsafeNumChunks()
+ {
+ LocalPool pool = localPool.get();
+ return (pool.chunks.chunk0 != null ? 1 : 0)
+ + (pool.chunks.chunk1 != null ? 1 : 0)
+ + (pool.chunks.chunk2 != null ? 1 : 0);
+ }
+
}
diff --git a/src/java/org/apache/cassandra/utils/memory/MemoryUtil.java b/src/java/org/apache/cassandra/utils/memory/MemoryUtil.java
index 6c2e6fd..e194962 100644
--- a/src/java/org/apache/cassandra/utils/memory/MemoryUtil.java
+++ b/src/java/org/apache/cassandra/utils/memory/MemoryUtil.java
@@ -27,7 +27,6 @@
import org.apache.cassandra.utils.Architecture;
import sun.misc.Unsafe;
-import sun.nio.ch.DirectBuffer;
public abstract class MemoryUtil
{
@@ -158,7 +157,7 @@
public static ByteBuffer getByteBuffer(long address, int length, ByteOrder order)
{
ByteBuffer instance = getHollowDirectByteBuffer(order);
- setByteBuffer(instance, address, length);
+ setDirectByteBuffer(instance, address, length);
return instance;
}
@@ -197,11 +196,9 @@
return instance;
}
- public static void setByteBuffer(ByteBuffer instance, long address, int length)
+ public static boolean isExactlyDirect(ByteBuffer buffer)
{
- unsafe.putLong(instance, DIRECT_BYTE_BUFFER_ADDRESS_OFFSET, address);
- unsafe.putInt(instance, DIRECT_BYTE_BUFFER_CAPACITY_OFFSET, length);
- unsafe.putInt(instance, DIRECT_BYTE_BUFFER_LIMIT_OFFSET, length);
+ return buffer.getClass() == DIRECT_BYTE_BUFFER_CLASS;
}
public static Object getAttachment(ByteBuffer instance)
@@ -226,6 +223,26 @@
return hollowBuffer;
}
+ public static ByteBuffer sliceDirectByteBuffer(ByteBuffer source, ByteBuffer hollowBuffer, int offset, int length)
+ {
+ assert source.getClass() == DIRECT_BYTE_BUFFER_CLASS || source.getClass() == RO_DIRECT_BYTE_BUFFER_CLASS;
+ setDirectByteBuffer(hollowBuffer, offset + unsafe.getLong(source, DIRECT_BYTE_BUFFER_ADDRESS_OFFSET), length);
+ return hollowBuffer;
+ }
+
+ public static void setDirectByteBuffer(ByteBuffer instance, long address, int length)
+ {
+ unsafe.putLong(instance, DIRECT_BYTE_BUFFER_ADDRESS_OFFSET, address);
+ unsafe.putInt(instance, DIRECT_BYTE_BUFFER_POSITION_OFFSET, 0);
+ unsafe.putInt(instance, DIRECT_BYTE_BUFFER_CAPACITY_OFFSET, length);
+ unsafe.putInt(instance, DIRECT_BYTE_BUFFER_LIMIT_OFFSET, length);
+ }
+
+ public static void setByteBufferCapacity(ByteBuffer instance, int capacity)
+ {
+ unsafe.putInt(instance, DIRECT_BYTE_BUFFER_CAPACITY_OFFSET, capacity);
+ }
+
public static long getLongByByte(long address)
{
if (BIG_ENDIAN)
@@ -337,7 +354,7 @@
return;
if (buffer.isDirect())
- setBytes(((DirectBuffer)buffer).address() + start, address, count);
+ setBytes(getAddress(buffer) + start, address, count);
else
setBytes(address, buffer.array(), buffer.arrayOffset() + start, count);
}
diff --git a/src/java/org/apache/cassandra/utils/memory/MemtablePool.java b/src/java/org/apache/cassandra/utils/memory/MemtablePool.java
index bd17f78..5ef023f 100644
--- a/src/java/org/apache/cassandra/utils/memory/MemtablePool.java
+++ b/src/java/org/apache/cassandra/utils/memory/MemtablePool.java
@@ -74,7 +74,6 @@
ExecutorUtils.shutdownNowAndWait(timeout, unit, cleaner);
}
-
public abstract MemtableAllocator newAllocator();
/**
diff --git a/src/java/org/apache/cassandra/utils/memory/SlabAllocator.java b/src/java/org/apache/cassandra/utils/memory/SlabAllocator.java
index f72a2c3..538cd3f 100644
--- a/src/java/org/apache/cassandra/utils/memory/SlabAllocator.java
+++ b/src/java/org/apache/cassandra/utils/memory/SlabAllocator.java
@@ -26,9 +26,9 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.concurrent.OpOrder;
-import sun.nio.ch.DirectBuffer;
/**
+ * The SlabAllocator is a bump-the-pointer allocator that allocates
@@ -116,7 +116,7 @@
public void setDiscarded()
{
for (Region region : offHeapRegions)
- ((DirectBuffer) region.data).cleaner().clean();
+ FileUtils.clean(region.data);
super.setDiscarded();
}
diff --git a/src/java/org/apache/cassandra/utils/obs/IBitSet.java b/src/java/org/apache/cassandra/utils/obs/IBitSet.java
index 15ff361..b262cf5 100644
--- a/src/java/org/apache/cassandra/utils/obs/IBitSet.java
+++ b/src/java/org/apache/cassandra/utils/obs/IBitSet.java
@@ -18,9 +18,9 @@
package org.apache.cassandra.utils.obs;
import java.io.Closeable;
-import java.io.DataOutput;
import java.io.IOException;
+import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.utils.concurrent.Ref;
public interface IBitSet extends Closeable
@@ -44,7 +44,7 @@
*/
public void clear(long index);
- public void serialize(DataOutput out) throws IOException;
+ public void serialize(DataOutputPlus out) throws IOException;
public long serializedSize();
diff --git a/src/java/org/apache/cassandra/utils/obs/OffHeapBitSet.java b/src/java/org/apache/cassandra/utils/obs/OffHeapBitSet.java
index 8593a11..486ec38 100644
--- a/src/java/org/apache/cassandra/utils/obs/OffHeapBitSet.java
+++ b/src/java/org/apache/cassandra/utils/obs/OffHeapBitSet.java
@@ -18,11 +18,17 @@
package org.apache.cassandra.utils.obs;
import java.io.DataInput;
+import java.io.DataInputStream;
import java.io.DataOutput;
import java.io.IOException;
+import com.google.common.annotations.VisibleForTesting;
+
import org.apache.cassandra.db.TypeSizes;
+import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.io.util.Memory;
+import org.apache.cassandra.io.util.MemoryOutputStream;
+import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.concurrent.Ref;
/**
@@ -35,8 +41,8 @@
public OffHeapBitSet(long numBits)
{
- // OpenBitSet.bits2words calculation is there for backward compatibility.
- long wordCount = OpenBitSet.bits2words(numBits);
+ /** returns the number of 64 bit words it would take to hold numBits */
+ long wordCount = (((numBits - 1) >>> 6) + 1);
if (wordCount > Integer.MAX_VALUE)
throw new UnsupportedOperationException("Bloom filter size is > 16GB, reduce the bloom_filter_fp_chance");
try
@@ -109,19 +115,26 @@
bytes.setMemory(0, bytes.size(), (byte) 0);
}
- public void serialize(DataOutput out) throws IOException
+ public void serialize(DataOutputPlus out) throws IOException
{
out.writeInt((int) (bytes.size() / 8));
- for (long i = 0; i < bytes.size();)
+ out.write(bytes, 0, bytes.size());
+ }
+
+ @VisibleForTesting
+ public void serializeOldBfFormat(DataOutputPlus out) throws IOException
+ {
+ out.writeInt((int) (bytes.size() / 8));
+ for (long i = 0; i < bytes.size(); )
{
long value = ((bytes.getByte(i++) & 0xff) << 0)
- + ((bytes.getByte(i++) & 0xff) << 8)
- + ((bytes.getByte(i++) & 0xff) << 16)
- + ((long) (bytes.getByte(i++) & 0xff) << 24)
- + ((long) (bytes.getByte(i++) & 0xff) << 32)
- + ((long) (bytes.getByte(i++) & 0xff) << 40)
- + ((long) (bytes.getByte(i++) & 0xff) << 48)
- + ((long) bytes.getByte(i++) << 56);
+ + ((bytes.getByte(i++) & 0xff) << 8)
+ + ((bytes.getByte(i++) & 0xff) << 16)
+ + ((long) (bytes.getByte(i++) & 0xff) << 24)
+ + ((long) (bytes.getByte(i++) & 0xff) << 32)
+ + ((long) (bytes.getByte(i++) & 0xff) << 40)
+ + ((long) (bytes.getByte(i++) & 0xff) << 48)
+ + ((long) bytes.getByte(i++) << 56);
out.writeLong(value);
}
}
@@ -132,21 +145,28 @@
}
@SuppressWarnings("resource")
- public static OffHeapBitSet deserialize(DataInput in) throws IOException
+ public static OffHeapBitSet deserialize(DataInputStream in, boolean oldBfFormat) throws IOException
{
long byteCount = in.readInt() * 8L;
Memory memory = Memory.allocate(byteCount);
- for (long i = 0; i < byteCount;)
+ if (oldBfFormat)
{
- long v = in.readLong();
- memory.setByte(i++, (byte) (v >>> 0));
- memory.setByte(i++, (byte) (v >>> 8));
- memory.setByte(i++, (byte) (v >>> 16));
- memory.setByte(i++, (byte) (v >>> 24));
- memory.setByte(i++, (byte) (v >>> 32));
- memory.setByte(i++, (byte) (v >>> 40));
- memory.setByte(i++, (byte) (v >>> 48));
- memory.setByte(i++, (byte) (v >>> 56));
+ for (long i = 0; i < byteCount; )
+ {
+ long v = in.readLong();
+ memory.setByte(i++, (byte) (v >>> 0));
+ memory.setByte(i++, (byte) (v >>> 8));
+ memory.setByte(i++, (byte) (v >>> 16));
+ memory.setByte(i++, (byte) (v >>> 24));
+ memory.setByte(i++, (byte) (v >>> 32));
+ memory.setByte(i++, (byte) (v >>> 40));
+ memory.setByte(i++, (byte) (v >>> 48));
+ memory.setByte(i++, (byte) (v >>> 56));
+ }
+ }
+ else
+ {
+ FBUtilities.copy(in, new MemoryOutputStream(memory), byteCount);
}
return new OffHeapBitSet(memory);
}
diff --git a/src/java/org/apache/cassandra/utils/obs/OpenBitSet.java b/src/java/org/apache/cassandra/utils/obs/OpenBitSet.java
deleted file mode 100644
index a21729a..0000000
--- a/src/java/org/apache/cassandra/utils/obs/OpenBitSet.java
+++ /dev/null
@@ -1,494 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.utils.obs;
-
-import java.util.Arrays;
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
-import org.apache.cassandra.db.TypeSizes;
-import org.apache.cassandra.utils.concurrent.Ref;
-
-/**
- * <p>
- * An "open" BitSet implementation that allows direct access to the arrays of words
- * storing the bits. Derived from Lucene's OpenBitSet, but with a paged backing array
- * (see bits delaration, below).
- * </p>
- * <p>
- * Unlike java.util.bitset, the fact that bits are packed into an array of longs
- * is part of the interface. This allows efficient implementation of other algorithms
- * by someone other than the author. It also allows one to efficiently implement
- * alternate serialization or interchange formats.
- * </p>
- * <p>
- * <code>OpenBitSet</code> is faster than <code>java.util.BitSet</code> in most operations
- * and *much* faster at calculating cardinality of sets and results of set operations.
- * It can also handle sets of larger cardinality (up to 64 * 2**32-1)
- * </p>
- * <p>
- * The goals of <code>OpenBitSet</code> are the fastest implementation possible, and
- * maximum code reuse. Extra safety and encapsulation
- * may always be built on top, but if that's built in, the cost can never be removed (and
- * hence people re-implement their own version in order to get better performance).
- * If you want a "safe", totally encapsulated (and slower and limited) BitSet
- * class, use <code>java.util.BitSet</code>.
- * </p>
- */
-
-public class OpenBitSet implements IBitSet
-{
- /**
- * We break the bitset up into multiple arrays to avoid promotion failure caused by attempting to allocate
- * large, contiguous arrays (CASSANDRA-2466). All sub-arrays but the last are uniformly PAGE_SIZE words;
- * to avoid waste in small bloom filters (of which Cassandra has many: one per row) the last sub-array
- * is sized to exactly the remaining number of words required to achieve the desired set size (CASSANDRA-3618).
- */
- private final long[][] bits;
- private int wlen; // number of words (elements) used in the array
- private final int pageCount;
- private static final int PAGE_SIZE = 4096;
-
- /**
- * Constructs an OpenBitSet large enough to hold numBits.
- * @param numBits
- */
- public OpenBitSet(long numBits)
- {
- wlen = (int) bits2words(numBits);
- int lastPageSize = wlen % PAGE_SIZE;
- int fullPageCount = wlen / PAGE_SIZE;
- pageCount = fullPageCount + (lastPageSize == 0 ? 0 : 1);
-
- bits = new long[pageCount][];
-
- for (int i = 0; i < fullPageCount; ++i)
- bits[i] = new long[PAGE_SIZE];
-
- if (lastPageSize != 0)
- bits[bits.length - 1] = new long[lastPageSize];
- }
-
- public OpenBitSet()
- {
- this(64);
- }
-
- /**
- * @return the pageSize
- */
- public int getPageSize()
- {
- return PAGE_SIZE;
- }
-
- public int getPageCount()
- {
- return pageCount;
- }
-
- public long[] getPage(int pageIdx)
- {
- return bits[pageIdx];
- }
-
- /** Returns the current capacity in bits (1 greater than the index of the last bit) */
- public long capacity() { return ((long)wlen) << 6; }
-
- @Override
- public long offHeapSize()
- {
- return 0;
- }
-
- public void addTo(Ref.IdentityCollection identities)
- {
- }
-
- /**
- * Returns the current capacity of this set. Included for
- * compatibility. This is *not* equal to {@link #cardinality}
- */
- public long size()
- {
- return capacity();
- }
-
- // @Override -- not until Java 1.6
- public long length()
- {
- return capacity();
- }
-
- /** Returns true if there are no set bits */
- public boolean isEmpty() { return cardinality()==0; }
-
-
- /** Expert: gets the number of longs in the array that are in use */
- public int getNumWords() { return wlen; }
-
-
- /**
- * Returns true or false for the specified bit index.
- * The index should be less than the OpenBitSet size
- */
- public boolean get(int index)
- {
- int i = index >> 6; // div 64
- // signed shift will keep a negative index and force an
- // array-index-out-of-bounds-exception, removing the need for an explicit check.
- int bit = index & 0x3f; // mod 64
- long bitmask = 1L << bit;
- // TODO perfectionist one can implement this using bit operations
- return (bits[i / PAGE_SIZE][i % PAGE_SIZE ] & bitmask) != 0;
- }
-
- /**
- * Returns true or false for the specified bit index.
- * The index should be less than the OpenBitSet size.
- */
- public boolean get(long index)
- {
- int i = (int)(index >> 6); // div 64
- int bit = (int)index & 0x3f; // mod 64
- long bitmask = 1L << bit;
- // TODO perfectionist one can implement this using bit operations
- return (bits[i / PAGE_SIZE][i % PAGE_SIZE ] & bitmask) != 0;
- }
-
- /**
- * Sets the bit at the specified index.
- * The index should be less than the OpenBitSet size.
- */
- public void set(long index)
- {
- int wordNum = (int)(index >> 6);
- int bit = (int)index & 0x3f;
- long bitmask = 1L << bit;
- bits[ wordNum / PAGE_SIZE ][ wordNum % PAGE_SIZE ] |= bitmask;
- }
-
- /**
- * Sets the bit at the specified index.
- * The index should be less than the OpenBitSet size.
- */
- public void set(int index)
- {
- int wordNum = index >> 6; // div 64
- int bit = index & 0x3f; // mod 64
- long bitmask = 1L << bit;
- bits[ wordNum / PAGE_SIZE ][ wordNum % PAGE_SIZE ] |= bitmask;
- }
-
- /**
- * clears a bit.
- * The index should be less than the OpenBitSet size.
- */
- public void clear(int index)
- {
- int wordNum = index >> 6;
- int bit = index & 0x03f;
- long bitmask = 1L << bit;
- bits[wordNum / PAGE_SIZE][wordNum % PAGE_SIZE] &= ~bitmask;
- // hmmm, it takes one more instruction to clear than it does to set... any
- // way to work around this? If there were only 63 bits per word, we could
- // use a right shift of 10111111...111 in binary to position the 0 in the
- // correct place (using sign extension).
- // Could also use Long.rotateRight() or rotateLeft() *if* they were converted
- // by the JVM into a native instruction.
- // bits[word] &= Long.rotateLeft(0xfffffffe,bit);
- }
-
- /**
- * clears a bit.
- * The index should be less than the OpenBitSet size.
- */
- public void clear(long index)
- {
- int wordNum = (int)(index >> 6); // div 64
- int bit = (int)index & 0x3f; // mod 64
- long bitmask = 1L << bit;
- bits[wordNum / PAGE_SIZE][wordNum % PAGE_SIZE] &= ~bitmask;
- }
-
- /**
- * Clears a range of bits. Clearing past the end does not change the size of the set.
- *
- * @param startIndex lower index
- * @param endIndex one-past the last bit to clear
- */
- public void clear(int startIndex, int endIndex)
- {
- if (endIndex <= startIndex) return;
-
- int startWord = (startIndex>>6);
- if (startWord >= wlen) return;
-
- // since endIndex is one past the end, this is index of the last
- // word to be changed.
- int endWord = ((endIndex-1)>>6);
-
- long startmask = -1L << startIndex;
- long endmask = -1L >>> -endIndex; // 64-(endIndex&0x3f) is the same as -endIndex due to wrap
-
- // invert masks since we are clearing
- startmask = ~startmask;
- endmask = ~endmask;
-
- if (startWord == endWord)
- {
- bits[startWord / PAGE_SIZE][startWord % PAGE_SIZE] &= (startmask | endmask);
- return;
- }
-
-
- bits[startWord / PAGE_SIZE][startWord % PAGE_SIZE] &= startmask;
-
- int middle = Math.min(wlen, endWord);
- if (startWord / PAGE_SIZE == middle / PAGE_SIZE)
- {
- Arrays.fill(bits[startWord/PAGE_SIZE], (startWord+1) % PAGE_SIZE, middle % PAGE_SIZE, 0L);
- } else
- {
- while (++startWord<middle)
- bits[startWord / PAGE_SIZE][startWord % PAGE_SIZE] = 0L;
- }
- if (endWord < wlen)
- {
- bits[endWord / PAGE_SIZE][endWord % PAGE_SIZE] &= endmask;
- }
- }
-
-
- /** Clears a range of bits. Clearing past the end does not change the size of the set.
- *
- * @param startIndex lower index
- * @param endIndex one-past the last bit to clear
- */
- public void clear(long startIndex, long endIndex)
- {
- if (endIndex <= startIndex) return;
-
- int startWord = (int)(startIndex>>6);
- if (startWord >= wlen) return;
-
- // since endIndex is one past the end, this is index of the last
- // word to be changed.
- int endWord = (int)((endIndex-1)>>6);
-
- long startmask = -1L << startIndex;
- long endmask = -1L >>> -endIndex; // 64-(endIndex&0x3f) is the same as -endIndex due to wrap
-
- // invert masks since we are clearing
- startmask = ~startmask;
- endmask = ~endmask;
-
- if (startWord == endWord)
-{
- bits[startWord / PAGE_SIZE][startWord % PAGE_SIZE] &= (startmask | endmask);
- return;
- }
-
- bits[startWord / PAGE_SIZE][startWord % PAGE_SIZE] &= startmask;
-
- int middle = Math.min(wlen, endWord);
- if (startWord / PAGE_SIZE == middle / PAGE_SIZE)
- {
- Arrays.fill(bits[startWord/PAGE_SIZE], (startWord+1) % PAGE_SIZE, middle % PAGE_SIZE, 0L);
- } else
- {
- while (++startWord<middle)
- bits[startWord / PAGE_SIZE][startWord % PAGE_SIZE] = 0L;
- }
- if (endWord < wlen)
- {
- bits[endWord / PAGE_SIZE][endWord % PAGE_SIZE] &= endmask;
- }
- }
-
- /** @return the number of set bits */
- public long cardinality()
- {
- long bitCount = 0L;
- for (int i=getPageCount();i-->0;)
- bitCount+=BitUtil.pop_array(bits[i],0,wlen);
-
- return bitCount;
- }
-
- /** this = this AND other */
- public void intersect(OpenBitSet other)
- {
- int newLen= Math.min(this.wlen,other.wlen);
- long[][] thisArr = this.bits;
- long[][] otherArr = other.bits;
- int thisPageSize = PAGE_SIZE;
- int otherPageSize = OpenBitSet.PAGE_SIZE;
- // testing against zero can be more efficient
- int pos=newLen;
- while(--pos>=0)
- {
- thisArr[pos / thisPageSize][ pos % thisPageSize] &= otherArr[pos / otherPageSize][pos % otherPageSize];
- }
-
- if (this.wlen > newLen)
- {
- // fill zeros from the new shorter length to the old length
- for (pos=wlen;pos-->newLen;)
- thisArr[pos / thisPageSize][ pos % thisPageSize] =0;
- }
- this.wlen = newLen;
- }
-
- // some BitSet compatability methods
-
- //** see {@link intersect} */
- public void and(OpenBitSet other)
- {
- intersect(other);
- }
-
- /** Lowers numWords, the number of words in use,
- * by checking for trailing zero words.
- */
- public void trimTrailingZeros()
- {
- int idx = wlen-1;
- while (idx>=0 && bits[idx / PAGE_SIZE][idx % PAGE_SIZE]==0) idx--;
- wlen = idx+1;
- }
-
- /** returns the number of 64 bit words it would take to hold numBits */
- public static long bits2words(long numBits)
- {
- return (((numBits-1)>>>6)+1);
- }
-
- /** returns true if both sets have the same bits set */
- @Override
- public boolean equals(Object o)
- {
- if (this == o) return true;
- if (!(o instanceof OpenBitSet)) return false;
- OpenBitSet a;
- OpenBitSet b = (OpenBitSet)o;
- // make a the larger set.
- if (b.wlen > this.wlen)
- {
- a = b; b=this;
- }
- else
- {
- a=this;
- }
-
- int aPageSize = OpenBitSet.PAGE_SIZE;
- int bPageSize = OpenBitSet.PAGE_SIZE;
-
- // check for any set bits out of the range of b
- for (int i=a.wlen-1; i>=b.wlen; i--)
- {
- if (a.bits[i/aPageSize][i % aPageSize]!=0) return false;
- }
-
- for (int i=b.wlen-1; i>=0; i--)
- {
- if (a.bits[i/aPageSize][i % aPageSize] != b.bits[i/bPageSize][i % bPageSize]) return false;
- }
-
- return true;
- }
-
-
- @Override
- public int hashCode()
- {
- // Start with a zero hash and use a mix that results in zero if the input is zero.
- // This effectively truncates trailing zeros without an explicit check.
- long h = 0;
- for (int i = wlen; --i>=0;)
- {
- h ^= bits[i / PAGE_SIZE][i % PAGE_SIZE];
- h = (h << 1) | (h >>> 63); // rotate left
- }
- // fold leftmost bits into right and add a constant to prevent
- // empty sets from returning 0, which is too common.
- return (int)((h>>32) ^ h) + 0x98761234;
- }
-
- public void close()
- {
- // noop, let GC do the cleanup.
- }
-
- public void serialize(DataOutput out) throws IOException
- {
- int bitLength = getNumWords();
- int pageSize = getPageSize();
- int pageCount = getPageCount();
-
- out.writeInt(bitLength);
- for (int p = 0; p < pageCount; p++)
- {
- long[] bits = getPage(p);
- for (int i = 0; i < pageSize && bitLength-- > 0; i++)
- {
- out.writeLong(bits[i]);
- }
- }
-}
-
- public long serializedSize()
- {
- int bitLength = getNumWords();
- int pageSize = getPageSize();
- int pageCount = getPageCount();
-
- long size = TypeSizes.sizeof(bitLength); // length
- for (int p = 0; p < pageCount; p++)
- {
- long[] bits = getPage(p);
- for (int i = 0; i < pageSize && bitLength-- > 0; i++)
- size += TypeSizes.sizeof(bits[i]); // bucket
- }
- return size;
- }
-
- public void clear()
- {
- clear(0, capacity());
- }
-
- public static OpenBitSet deserialize(DataInput in) throws IOException
- {
- long bitLength = in.readInt();
-
- OpenBitSet bs = new OpenBitSet(bitLength << 6);
- int pageSize = bs.getPageSize();
- int pageCount = bs.getPageCount();
-
- for (int p = 0; p < pageCount; p++)
- {
- long[] bits = bs.getPage(p);
- for (int i = 0; i < pageSize && bitLength-- > 0; i++)
- bits[i] = in.readLong();
- }
- return bs;
- }
-}
diff --git a/src/java/org/apache/cassandra/utils/progress/jmx/JMXBroadcastExecutor.java b/src/java/org/apache/cassandra/utils/progress/jmx/JMXBroadcastExecutor.java
new file mode 100644
index 0000000..f28609c
--- /dev/null
+++ b/src/java/org/apache/cassandra/utils/progress/jmx/JMXBroadcastExecutor.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.utils.progress.jmx;
+
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+import org.apache.cassandra.concurrent.NamedThreadFactory;
+
+/**
+ * Holds dedicated executor for JMX event handling. Events will internally queued by ArrayNotificationBuffer,
+ * synchronized by an exclusive write lock, which makes a shared single thread executor desirable.
+ */
+public final class JMXBroadcastExecutor
+{
+
+ private JMXBroadcastExecutor() { }
+
+ public final static ExecutorService executor = Executors.newSingleThreadExecutor(new NamedThreadFactory("JMX"));
+
+}
diff --git a/src/java/org/apache/cassandra/utils/progress/jmx/LegacyJMXProgressSupport.java b/src/java/org/apache/cassandra/utils/progress/jmx/LegacyJMXProgressSupport.java
deleted file mode 100644
index 3caae38..0000000
--- a/src/java/org/apache/cassandra/utils/progress/jmx/LegacyJMXProgressSupport.java
+++ /dev/null
@@ -1,108 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.utils.progress.jmx;
-
-import java.util.Optional;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.regex.Pattern;
-import javax.management.Notification;
-import javax.management.NotificationBroadcasterSupport;
-import javax.management.ObjectName;
-
-import org.apache.cassandra.utils.progress.ProgressEvent;
-import org.apache.cassandra.utils.progress.ProgressListener;
-
-import static org.apache.cassandra.service.ActiveRepairService.Status;
-
-/**
- * ProgressListener that translates ProgressEvent to legacy JMX Notification message (backward compatibility support)
- */
-public class LegacyJMXProgressSupport implements ProgressListener
-{
- protected static final Pattern SESSION_FAILED_MATCHER = Pattern.compile("Repair session .* for range .* failed with error .*|Repair command .* failed with error .*");
- protected static final Pattern SESSION_SUCCESS_MATCHER = Pattern.compile("Repair session .* for range .* finished");
-
- private final AtomicLong notificationSerialNumber = new AtomicLong();
- private final String jmxObjectName;
-
- private final NotificationBroadcasterSupport broadcaster;
-
- public LegacyJMXProgressSupport(NotificationBroadcasterSupport broadcaster,
- String jmxObjectName)
- {
- this.broadcaster = broadcaster;
- this.jmxObjectName = jmxObjectName;
- }
-
- @Override
- public void progress(String tag, ProgressEvent event)
- {
- if (tag.startsWith("repair:"))
- {
- Optional<int[]> legacyUserData = getLegacyUserdata(tag, event);
- if (legacyUserData.isPresent())
- {
- Notification jmxNotification = new Notification("repair", jmxObjectName, notificationSerialNumber.incrementAndGet(), event.getMessage());
- jmxNotification.setUserData(legacyUserData.get());
- broadcaster.sendNotification(jmxNotification);
- }
- }
- }
-
- protected static Optional<int[]> getLegacyUserdata(String tag, ProgressEvent event)
- {
- Optional<Status> status = getStatus(event);
- if (status.isPresent())
- {
- int[] result = new int[2];
- result[0] = getCmd(tag);
- result[1] = status.get().ordinal();
- return Optional.of(result);
- }
- return Optional.empty();
- }
-
- protected static Optional<Status> getStatus(ProgressEvent event)
- {
- switch (event.getType())
- {
- case START:
- return Optional.of(Status.STARTED);
- case COMPLETE:
- return Optional.of(Status.FINISHED);
- case ERROR:
- case PROGRESS:
- if (SESSION_FAILED_MATCHER.matcher(event.getMessage()).matches())
- {
- return Optional.of(Status.SESSION_FAILED);
- }
- else if (SESSION_SUCCESS_MATCHER.matcher(event.getMessage()).matches())
- {
- return Optional.of(Status.SESSION_SUCCESS);
- }
- }
-
- return Optional.empty();
- }
-
- protected static int getCmd(String tag)
- {
- return Integer.parseInt(tag.split(":")[1]);
- }
-}
diff --git a/src/java/org/apache/cassandra/utils/streamhist/HistogramDataConsumer.java b/src/java/org/apache/cassandra/utils/streamhist/HistogramDataConsumer.java
new file mode 100755
index 0000000..274e7d5
--- /dev/null
+++ b/src/java/org/apache/cassandra/utils/streamhist/HistogramDataConsumer.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.utils.streamhist;
+
+/**
+ * This interface exists to avoid boxing primitive ints to Integers (otherwise <i>{@link java.util.function.BiConsumer}<Integer, Integer></i> would have been sufficient).
+ */
+public interface HistogramDataConsumer<T extends Exception>
+{
+ void consume(int point, int value) throws T;
+}
diff --git a/src/java/org/apache/cassandra/utils/streamhist/StreamingTombstoneHistogramBuilder.java b/src/java/org/apache/cassandra/utils/streamhist/StreamingTombstoneHistogramBuilder.java
new file mode 100755
index 0000000..eda88bc
--- /dev/null
+++ b/src/java/org/apache/cassandra/utils/streamhist/StreamingTombstoneHistogramBuilder.java
@@ -0,0 +1,541 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.utils.streamhist;
+
+import java.math.RoundingMode;
+import java.util.Arrays;
+import java.util.stream.Collectors;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.math.IntMath;
+
+import org.apache.cassandra.db.rows.Cell;
+
+/**
+ * Histogram that can be constructed from streaming of data.
+ *
+ * Histogram used to retrieve the number of droppable tombstones for example via
+ * {@link org.apache.cassandra.io.sstable.format.SSTableReader#getDroppableTombstonesBefore(int)}.
+ * <p>
+ * When an sstable is written (or streamed), this histogram-builder receives the "local deletion timestamp"
+ * as an {@code int} via {@link #update(int)}. Negative values are not supported.
+ * <p>
+ * Algorithm: Histogram is represented as collection of {point, weight} pairs. When new point <i>p</i> with weight <i>m</i> is added:
+ * <ol>
+ * <li>If point <i>p</i> is already exists in collection, add <i>m</i> to recorded value of point <i>p</i> </li>
+ * <li>If there is no point <i>p</i> in the collection, add point <i>p</i> with weight <i>m</i> </li>
+ * <li>If point was added and collection size became lorger than maxBinSize:</li>
+ * </ol>
+ *
+ * <ol type="a">
+ * <li>Find nearest points <i>p1</i> and <i>p2</i> in the collection </li>
+ * <li>Replace theese two points with one weighted point <i>p3 = (p1*m1+p2*m2)/(p1+p2)</i></li>
+ * </ol>
+ *
+ * <p>
+ * There are some optimization to make histogram builder faster:
+ * <ol>
+ * <li>Spool: big map that saves from excessively merging of small bin. This map can contains up to maxSpoolSize points and accumulate weight from same points.
+ * For example, if spoolSize=100, binSize=10 and there are only 50 different points. it will be only 40 merges regardless how many points will be added.</li>
+ * <li>Spool is organized as open-addressing primitive hash map where odd elements are points and event elements are values.
+ * Spool can not resize => when number of collisions became bigger than threashold or size became large that <i>array_size/2</i> Spool is drained to bin</li>
+ * <li>Bin is organized as sorted arrays. It reduces garbage collection pressure and allows to find elements in log(binSize) time via binary search</li>
+ * <li>To use existing Arrays.binarySearch <i></>{point, values}</i> in bin pairs is packed in one long</li>
+ * </ol>
+ * <p>
+ * The original algorithm is taken from following paper:
+ * Yael Ben-Haim and Elad Tom-Tov, "A Streaming Parallel Decision Tree Algorithm" (2010)
+ * http://jmlr.csail.mit.edu/papers/volume11/ben-haim10a/ben-haim10a.pdf
+ */
+public class StreamingTombstoneHistogramBuilder
+{
+ // Buffer with point-value pair
+ private final DataHolder bin;
+
+ // Keep a second, larger buffer to spool data in, before finalizing it into `bin`
+ private final Spool spool;
+
+ // voluntarily give up resolution for speed
+ private final int roundSeconds;
+
+ public StreamingTombstoneHistogramBuilder(int maxBinSize, int maxSpoolSize, int roundSeconds)
+ {
+ assert maxBinSize > 0 && maxSpoolSize >= 0 && roundSeconds > 0: "Invalid arguments: maxBinSize:" + maxBinSize + " maxSpoolSize:" + maxSpoolSize + " delta:" + roundSeconds;
+
+ this.roundSeconds = roundSeconds;
+ this.bin = new DataHolder(maxBinSize + 1, roundSeconds);
+ this.spool = new Spool(maxSpoolSize);
+ }
+
+ /**
+ * Adds new point to this histogram with a default value of 1.
+ *
+ * @param point the point to be added
+ */
+ public void update(int point)
+ {
+ update(point, 1);
+ }
+
+ /**
+ * Adds new point {@param point} with value {@param value} to this histogram.
+ */
+ public void update(int point, int value)
+ {
+ point = ceilKey(point, roundSeconds);
+
+ if (spool.capacity > 0)
+ {
+ if (!spool.tryAddOrAccumulate(point, value))
+ {
+ flushHistogram();
+ final boolean success = spool.tryAddOrAccumulate(point, value);
+ assert success : "Can not add value to spool"; // after spool flushing we should always be able to insert new value
+ }
+ }
+ else
+ {
+ flushValue(point, value);
+ }
+ }
+
+ /**
+ * Drain the temporary spool into the final bins
+ */
+ public void flushHistogram()
+ {
+ spool.forEach(this::flushValue);
+ spool.clear();
+ }
+
+ private void flushValue(int key, int spoolValue)
+ {
+ bin.addValue(key, spoolValue);
+
+ if (bin.isFull())
+ {
+ bin.mergeNearestPoints();
+ }
+ }
+
+ /**
+ * Creates a 'finished' snapshot of the current state of the historgram, but leaves this builder instance
+ * open for subsequent additions to the histograms. Basically, this allows us to have some degree of sanity
+ * wrt sstable early open.
+ */
+ public TombstoneHistogram build()
+ {
+ flushHistogram();
+ return new TombstoneHistogram(bin);
+ }
+
+ /**
+ * An ordered collection of histogram buckets, each entry in the collection represents a pair (bucket, count).
+ * Once the collection is full it merges the closest buckets using a weighted approach see {@link #mergeNearestPoints()}.
+ */
+ static class DataHolder
+ {
+ private static final long EMPTY = Long.MAX_VALUE;
+ private final long[] data;
+ private final int roundSeconds;
+
+ DataHolder(int maxCapacity, int roundSeconds)
+ {
+ data = new long[maxCapacity];
+ Arrays.fill(data, EMPTY);
+ this.roundSeconds = roundSeconds;
+ }
+
+ DataHolder(DataHolder holder)
+ {
+ data = Arrays.copyOf(holder.data, holder.data.length);
+ roundSeconds = holder.roundSeconds;
+ }
+
+ @VisibleForTesting
+ int getValue(int point)
+ {
+ long key = wrap(point, 0);
+ int index = Arrays.binarySearch(data, key);
+ if (index < 0)
+ index = -index - 1;
+ if (index >= data.length)
+ return -1; // not-found sentinel
+ if (unwrapPoint(data[index]) != point)
+ return -2; // not-found sentinel
+ return unwrapValue(data[index]);
+ }
+
+ /**
+ * Adds value {@code delta} to the point {@code point}.
+ *
+ * @return {@code true} if inserted, {@code false} if accumulated
+ */
+ boolean addValue(int point, int delta)
+ {
+ long key = wrap(point, 0);
+ int index = Arrays.binarySearch(data, key);
+ if (index < 0)
+ {
+ index = -index - 1;
+ assert (index < data.length) : "No more space in array";
+
+ if (unwrapPoint(data[index]) != point) //ok, someone else at this point, let's shift array and insert
+ {
+ assert (data[data.length - 1] == EMPTY) : "No more space in array";
+
+ System.arraycopy(data, index, data, index + 1, data.length - index - 1);
+
+ data[index] = wrap(point, delta);
+ return true;
+ }
+ else
+ {
+ data[index] = wrap(point, (long) unwrapValue(data[index]) + delta);
+ }
+ }
+ else
+ {
+ data[index] = wrap(point, (long) unwrapValue(data[index]) + delta);
+ }
+
+ return false;
+ }
+
+ /**
+ * Finds nearest points <i>p1</i> and <i>p2</i> in the collection
+ * Replaces theese two points with one weighted point <i>p3 = (p1*m1+p2*m2)/(p1+p2)
+ */
+ @VisibleForTesting
+ void mergeNearestPoints()
+ {
+ assert isFull() : "DataHolder must be full in order to merge two points";
+
+ final int[] smallestDifference = findPointPairWithSmallestDistance();
+
+ final int point1 = smallestDifference[0];
+ final int point2 = smallestDifference[1];
+
+ long key = wrap(point1, 0);
+ int index = Arrays.binarySearch(data, key);
+ if (index < 0)
+ {
+ index = -index - 1;
+ assert (index < data.length) : "Not found in array";
+ assert (unwrapPoint(data[index]) == point1) : "Not found in array";
+ }
+
+ long value1 = unwrapValue(data[index]);
+ long value2 = unwrapValue(data[index + 1]);
+
+ assert (unwrapPoint(data[index + 1]) == point2) : "point2 should follow point1";
+
+ long sum = value1 + value2;
+
+ //let's evaluate in long values to handle overflow in multiplication
+ int newPoint = saturatingCastToInt((point1 * value1 + point2 * value2) / sum);
+ newPoint = ceilKey(newPoint, roundSeconds);
+ data[index] = wrap(newPoint, saturatingCastToInt(sum));
+
+ System.arraycopy(data, index + 2, data, index + 1, data.length - index - 2);
+ data[data.length - 1] = EMPTY;
+ }
+
+ private int[] findPointPairWithSmallestDistance()
+ {
+ assert isFull(): "The DataHolder must be full in order to find the closest pair of points";
+
+ int point1 = 0;
+ int point2 = Integer.MAX_VALUE;
+
+ for (int i = 0; i < data.length - 1; i++)
+ {
+ int pointA = unwrapPoint(data[i]);
+ int pointB = unwrapPoint(data[i + 1]);
+
+ assert pointB > pointA : "DataHolder not sorted, p2(" + pointB +") < p1(" + pointA + ") for " + this;
+
+ if (point2 - point1 > pointB - pointA)
+ {
+ point1 = pointA;
+ point2 = pointB;
+ }
+ }
+
+ return new int[]{point1, point2};
+ }
+
+ private int[] unwrap(long key)
+ {
+ final int point = unwrapPoint(key);
+ final int value = unwrapValue(key);
+ return new int[]{ point, value };
+ }
+
+ private int unwrapPoint(long key)
+ {
+ return (int) (key >> 32);
+ }
+
+ private int unwrapValue(long key)
+ {
+ return (int) (key & 0xFF_FF_FF_FFL);
+ }
+
+ private long wrap(int point, long value)
+ {
+ assert point >= 0 : "Invalid argument: point:" + point;
+ return (((long) point) << 32) | saturatingCastToInt(value);
+ }
+
+ public String toString()
+ {
+ return Arrays.stream(data).filter(x -> x != EMPTY).mapToObj(this::unwrap).map(Arrays::toString).collect(Collectors.joining());
+ }
+
+ public boolean isFull()
+ {
+ return data[data.length - 1] != EMPTY;
+ }
+
+ public <E extends Exception> void forEach(HistogramDataConsumer<E> histogramDataConsumer) throws E
+ {
+ for (long datum : data)
+ {
+ if (datum == EMPTY)
+ {
+ break;
+ }
+
+ histogramDataConsumer.consume(unwrapPoint(datum), unwrapValue(datum));
+ }
+ }
+
+ public int size()
+ {
+ int[] accumulator = new int[1];
+ forEach((point, value) -> accumulator[0]++);
+ return accumulator[0];
+ }
+
+ public double sum(int b)
+ {
+ double sum = 0;
+
+ for (int i = 0; i < data.length; i++)
+ {
+ long pointAndValue = data[i];
+ if (pointAndValue == EMPTY)
+ {
+ break;
+ }
+ final int point = unwrapPoint(pointAndValue);
+ final int value = unwrapValue(pointAndValue);
+ if (point > b)
+ {
+ if (i == 0)
+ { // no prev point
+ return 0;
+ }
+ else
+ {
+ final int prevPoint = unwrapPoint(data[i - 1]);
+ final int prevValue = unwrapValue(data[i - 1]);
+ // calculate estimated count mb for point b
+ double weight = (b - prevPoint) / (double) (point - prevPoint);
+ double mb = prevValue + (value - prevValue) * weight;
+ sum -= prevValue;
+ sum += (prevValue + mb) * weight / 2;
+ sum += prevValue / 2.0;
+ return sum;
+ }
+ }
+ else
+ {
+ sum += value;
+ }
+ }
+ return sum;
+ }
+
+ @Override
+ public int hashCode()
+ {
+ return Arrays.hashCode(data);
+ }
+
+ @Override
+ public boolean equals(Object o)
+ {
+ if (!(o instanceof DataHolder))
+ return false;
+
+ final DataHolder other = ((DataHolder) o);
+
+ if (this.size()!=other.size())
+ return false;
+
+ for (int i=0; i<size(); i++)
+ {
+ if (data[i]!=other.data[i])
+ {
+ return false;
+ }
+ }
+ return true;
+ }
+ }
+
+ /**
+ * This class is a specialized open addressing HashMap that uses int as keys and int as values.
+ * This is an optimization to avoid allocating objects.
+ * In order for this class to work correctly it should have a power of 2 capacity.
+ * This last invariant is taken care of during construction.
+ */
+ static class Spool
+ {
+ final int[] points;
+ final int[] values;
+
+ final int capacity;
+ int size;
+
+ Spool(int requestedCapacity)
+ {
+ if (requestedCapacity < 0)
+ throw new IllegalArgumentException("Illegal capacity " + requestedCapacity);
+
+ this.capacity = getPowerOfTwoCapacity(requestedCapacity);
+
+ // x2 because we want no more than two reprobes on average when _capacity_ entries will be written
+ points = new int[capacity * 2];
+ values = new int[capacity * 2];
+ clear();
+ }
+
+ private int getPowerOfTwoCapacity(int requestedCapacity)
+ {
+ //for spool we need power-of-two cells
+ return requestedCapacity == 0 ? 0 : IntMath.pow(2, IntMath.log2(requestedCapacity, RoundingMode.CEILING));
+ }
+
+ void clear()
+ {
+ Arrays.fill(points, -1);
+ size = 0;
+ }
+
+ boolean tryAddOrAccumulate(int point, int delta)
+ {
+ if (size > capacity)
+ {
+ return false;
+ }
+
+ final int cell = (capacity - 1) & hash(point);
+
+ // We use linear scanning. I think cluster of 100 elements is large enough to give up.
+ for (int attempt = 0; attempt < 100; attempt++)
+ {
+ if (tryCell(cell + attempt, point, delta))
+ return true;
+ }
+ return false;
+ }
+
+ private int hash(int i)
+ {
+ long largePrime = 948701839L;
+ return (int) (i * largePrime);
+ }
+
+ <E extends Exception> void forEach(HistogramDataConsumer<E> consumer) throws E
+ {
+ for (int i = 0; i < points.length; i++)
+ {
+ if (points[i] != -1)
+ {
+ consumer.consume(points[i], values[i]);
+ }
+ }
+ }
+
+ private boolean tryCell(int cell, int point, int delta)
+ {
+ assert cell >= 0 && point >= 0 && delta >= 0 : "Invalid arguments: cell:" + cell + " point:" + point + " delta:" + delta;
+
+ cell = cell % points.length;
+ if (points[cell] == -1)
+ {
+ points[cell] = point;
+ values[cell] = delta;
+ size++;
+ return true;
+ }
+ if (points[cell] == point)
+ {
+ values[cell] = saturatingCastToInt((long) values[cell] + (long) delta);
+ return true;
+ }
+ return false;
+ }
+
+ public String toString()
+ {
+ StringBuilder sb = new StringBuilder();
+ sb.append('[');
+ for (int i = 0; i < points.length; i++)
+ {
+ if (points[i] == -1)
+ continue;
+ if (sb.length() > 1)
+ sb.append(", ");
+ sb.append('[').append(points[i]).append(',').append(values[i]).append(']');
+ }
+ sb.append(']');
+ return sb.toString();
+ }
+ }
+
+ private static int ceilKey(int point, int bucketSize)
+ {
+ int delta = point % bucketSize;
+
+ if (delta == 0)
+ return point;
+
+ return saturatingCastToMaxDeletionTime((long) point + (long) bucketSize - (long) delta);
+ }
+
+ public static int saturatingCastToInt(long value)
+ {
+ return (int) (value > Integer.MAX_VALUE ? Integer.MAX_VALUE : value);
+ }
+
+ /**
+ * Cast to an int with maximum value of {@code Cell.MAX_DELETION_TIME} to avoid representing values that
+ * aren't a tombstone
+ */
+ public static int saturatingCastToMaxDeletionTime(long value)
+ {
+ return (value < 0L || value > Cell.MAX_DELETION_TIME)
+ ? Cell.MAX_DELETION_TIME
+ : (int) value;
+ }
+}
diff --git a/src/java/org/apache/cassandra/utils/streamhist/TombstoneHistogram.java b/src/java/org/apache/cassandra/utils/streamhist/TombstoneHistogram.java
new file mode 100755
index 0000000..5f2787b
--- /dev/null
+++ b/src/java/org/apache/cassandra/utils/streamhist/TombstoneHistogram.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.utils.streamhist;
+
+import java.io.IOException;
+
+import org.apache.cassandra.db.TypeSizes;
+import org.apache.cassandra.io.ISerializer;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.utils.streamhist.StreamingTombstoneHistogramBuilder.DataHolder;
+
+/**
+ * A snapshot or finished histrogram of tombstones for a sstable, as generated from {@link StreamingTombstoneHistogramBuilder}.
+ */
+public class TombstoneHistogram
+{
+ public static final HistogramSerializer serializer = new HistogramSerializer();
+
+ // Buffer with point-value pair
+ private final DataHolder bin;
+
+ /**
+ * Creates a new histogram with max bin size of maxBinSize
+ */
+ TombstoneHistogram(DataHolder holder)
+ {
+ bin = new DataHolder(holder);
+ }
+
+ public static TombstoneHistogram createDefault()
+ {
+ return new TombstoneHistogram(new DataHolder(0, 1));
+ }
+
+ /**
+ * Calculates estimated number of points in interval [-inf,b].
+ *
+ * @param b upper bound of a interval to calculate sum
+ * @return estimated number of points in a interval [-inf,b].
+ */
+ public double sum(double b)
+ {
+ return bin.sum((int) b);
+ }
+
+ public int size()
+ {
+ return this.bin.size();
+ }
+
+ public <E extends Exception> void forEach(HistogramDataConsumer<E> histogramDataConsumer) throws E
+ {
+ this.bin.forEach(histogramDataConsumer);
+ }
+
+ public static class HistogramSerializer implements ISerializer<TombstoneHistogram>
+ {
+ public void serialize(TombstoneHistogram histogram, DataOutputPlus out) throws IOException
+ {
+ final int size = histogram.size();
+ final int maxBinSize = size; // we write this for legacy reasons
+ out.writeInt(maxBinSize);
+ out.writeInt(size);
+ histogram.forEach((point, value) ->
+ {
+ out.writeDouble((double) point);
+ out.writeLong((long) value);
+ });
+ }
+
+ public TombstoneHistogram deserialize(DataInputPlus in) throws IOException
+ {
+ in.readInt(); // max bin size
+ int size = in.readInt();
+ DataHolder dataHolder = new DataHolder(size, 1);
+ for (int i = 0; i < size; i++)
+ {
+ // Already serialized sstable metadata may contain negative deletion-time values (see CASSANDRA-14092).
+ // Just do a "safe cast" and it should be good. For safety, also do that for the 'value' (tombstone count).
+ int localDeletionTime = StreamingTombstoneHistogramBuilder.saturatingCastToMaxDeletionTime((long) in.readDouble());
+ int count = StreamingTombstoneHistogramBuilder.saturatingCastToInt(in.readLong());
+
+ dataHolder.addValue(localDeletionTime, count);
+ }
+
+ return new TombstoneHistogram(dataHolder);
+ }
+
+ public long serializedSize(TombstoneHistogram histogram)
+ {
+ int maxBinSize = 0;
+ long size = TypeSizes.sizeof(maxBinSize);
+ final int histSize = histogram.size();
+ size += TypeSizes.sizeof(histSize);
+ // size of entries = size * (8(double) + 8(long))
+ size += histSize * (8L + 8L);
+ return size;
+ }
+ }
+
+ @Override
+ public boolean equals(Object o)
+ {
+ if (this == o)
+ return true;
+
+ if (!(o instanceof TombstoneHistogram))
+ return false;
+
+ TombstoneHistogram that = (TombstoneHistogram) o;
+ return bin.equals(that.bin);
+ }
+
+ @Override
+ public int hashCode()
+ {
+ return bin.hashCode();
+ }
+}
diff --git a/src/java/org/apache/cassandra/utils/vint/VIntCoding.java b/src/java/org/apache/cassandra/utils/vint/VIntCoding.java
index ac5a74e..6961d9f 100644
--- a/src/java/org/apache/cassandra/utils/vint/VIntCoding.java
+++ b/src/java/org/apache/cassandra/utils/vint/VIntCoding.java
@@ -53,6 +53,7 @@
import io.netty.util.concurrent.FastThreadLocal;
import net.nicoulaj.compilecommand.annotations.Inline;
+import org.apache.cassandra.io.util.DataInputPlus;
/**
* Borrows idea from
@@ -60,6 +61,7 @@
*/
public class VIntCoding
{
+ public static final int MAX_SIZE = 10;
public static long readUnsignedVInt(DataInput input) throws IOException
{
@@ -81,6 +83,13 @@
return retval;
}
+ public static void skipUnsignedVInt(DataInputPlus input) throws IOException
+ {
+ int firstByte = input.readByte();
+ if (firstByte < 0)
+ input.skipBytesFully(numberOfExtraBytesToRead(firstByte));
+ }
+
/**
* Note this method is the same as {@link #readUnsignedVInt(DataInput)},
* except that we do *not* block if there are not enough bytes in the buffer
@@ -94,7 +103,6 @@
{
return getUnsignedVInt(input, readerIndex, input.limit());
}
-
public static long getUnsignedVInt(ByteBuffer input, int readerIndex, int readerLimit)
{
if (readerIndex >= readerLimit)
@@ -121,6 +129,24 @@
return retval;
}
+ /**
+ * Computes size of an unsigned vint that starts at readerIndex of the provided ByteBuf.
+ *
+ * @return -1 if there are not enough bytes in the input to calculate the size; else, the vint unsigned value size in bytes.
+ */
+ public static int computeUnsignedVIntSize(ByteBuffer input, int readerIndex)
+ {
+ return computeUnsignedVIntSize(input, readerIndex, input.limit());
+ }
+ public static int computeUnsignedVIntSize(ByteBuffer input, int readerIndex, int readerLimit)
+ {
+ if (readerIndex >= readerLimit)
+ return -1;
+
+ int firstByte = input.get(readerIndex);
+ return 1 + ((firstByte >= 0) ? 0 : numberOfExtraBytesToRead(firstByte));
+ }
+
public static long readVInt(DataInput input) throws IOException
{
return decodeZigZag64(readUnsignedVInt(input));
@@ -156,6 +182,7 @@
}
};
+ @Inline
public static void writeUnsignedVInt(long value, DataOutput output) throws IOException
{
int size = VIntCoding.computeUnsignedVIntSize(value);
@@ -165,24 +192,47 @@
return;
}
- output.write(VIntCoding.encodeVInt(value, size), 0, size);
+ output.write(VIntCoding.encodeUnsignedVInt(value, size), 0, size);
}
@Inline
- public static byte[] encodeVInt(long value, int size)
+ public static void writeUnsignedVInt(long value, ByteBuffer output)
{
- byte encodingSpace[] = encodingBuffer.get();
- int extraBytes = size - 1;
-
- for (int i = extraBytes ; i >= 0; --i)
+ int size = VIntCoding.computeUnsignedVIntSize(value);
+ if (size == 1)
{
- encodingSpace[i] = (byte) value;
- value >>= 8;
+ output.put((byte) value);
+ return;
}
- encodingSpace[0] |= VIntCoding.encodeExtraBytesToRead(extraBytes);
+
+ output.put(VIntCoding.encodeUnsignedVInt(value, size), 0, size);
+ }
+
+ /**
+ * @return a TEMPORARY THREAD LOCAL BUFFER containing the encoded bytes of the value
+ * This byte[] must be discarded by the caller immediately, and synchronously
+ */
+ @Inline
+ private static byte[] encodeUnsignedVInt(long value, int size)
+ {
+ byte[] encodingSpace = encodingBuffer.get();
+ encodeUnsignedVInt(value, size, encodingSpace);
return encodingSpace;
}
+ @Inline
+ private static void encodeUnsignedVInt(long value, int size, byte[] encodeInto)
+ {
+ int extraBytes = size - 1;
+ for (int i = extraBytes ; i >= 0; --i)
+ {
+ encodeInto[i] = (byte) value;
+ value >>= 8;
+ }
+ encodeInto[0] |= VIntCoding.encodeExtraBytesToRead(extraBytes);
+ }
+
+ @Inline
public static void writeVInt(long value, DataOutput output) throws IOException
{
writeUnsignedVInt(encodeZigZag64(value), output);
diff --git a/src/jdkoverride/java/util/zip/CRC32.class b/src/jdkoverride/java/util/zip/CRC32.class
deleted file mode 100644
index 546199e..0000000
--- a/src/jdkoverride/java/util/zip/CRC32.class
+++ /dev/null
Binary files differ
diff --git a/src/resources/org/apache/cassandra/cql3/functions/JavaSourceUDF.txt b/src/resources/org/apache/cassandra/cql3/functions/JavaSourceUDF.txt
index 5dafd98..fd9ca36 100644
--- a/src/resources/org/apache/cassandra/cql3/functions/JavaSourceUDF.txt
+++ b/src/resources/org/apache/cassandra/cql3/functions/JavaSourceUDF.txt
@@ -7,9 +7,9 @@
import org.apache.cassandra.cql3.functions.UDFContext;
import org.apache.cassandra.transport.ProtocolVersion;
-import com.datastax.driver.core.TypeCodec;
-import com.datastax.driver.core.TupleValue;
-import com.datastax.driver.core.UDTValue;
+import org.apache.cassandra.cql3.functions.types.TypeCodec;
+import org.apache.cassandra.cql3.functions.types.TupleValue;
+import org.apache.cassandra.cql3.functions.types.UDTValue;
public final class #class_name# extends JavaUDF
{
diff --git a/test/burn/org/apache/cassandra/net/BytesInFlightController.java b/test/burn/org/apache/cassandra/net/BytesInFlightController.java
new file mode 100644
index 0000000..edd9a7e
--- /dev/null
+++ b/test/burn/org/apache/cassandra/net/BytesInFlightController.java
@@ -0,0 +1,174 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.net;
+
+import java.util.Map;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.ConcurrentSkipListMap;
+import java.util.concurrent.atomic.AtomicLongFieldUpdater;
+import java.util.concurrent.locks.LockSupport;
+import java.util.function.IntConsumer;
+
+import org.apache.cassandra.utils.Pair;
+
+public class BytesInFlightController
+{
+ private static final AtomicLongFieldUpdater<BytesInFlightController> sentBytesUpdater = AtomicLongFieldUpdater.newUpdater(BytesInFlightController.class, "sentBytes");
+ private static final AtomicLongFieldUpdater<BytesInFlightController> receivedBytesUpdater = AtomicLongFieldUpdater.newUpdater(BytesInFlightController.class, "receivedBytes");
+
+ private volatile long minimumInFlightBytes, maximumInFlightBytes;
+ private volatile long sentBytes;
+ private volatile long receivedBytes;
+ private final ConcurrentLinkedQueue<Pair<Integer, IntConsumer>> deferredBytes = new ConcurrentLinkedQueue<>();
+ private final ConcurrentSkipListMap<Long, Thread> waitingToSend = new ConcurrentSkipListMap<>();
+
+ BytesInFlightController(long maximumInFlightBytes)
+ {
+ this.maximumInFlightBytes = maximumInFlightBytes;
+ }
+
+ public void send(long bytes) throws InterruptedException
+ {
+ long sentBytes = sentBytesUpdater.getAndAdd(this, bytes);
+ maybeProcessDeferred();
+ if ((sentBytes - receivedBytes) >= maximumInFlightBytes)
+ {
+ long waitUntilReceived = sentBytes - maximumInFlightBytes;
+ // overlap shouldn't occur, but cannot guarantee it when we modify maximumInFlightBytes
+ Thread prev = waitingToSend.putIfAbsent(waitUntilReceived, Thread.currentThread());
+ while (prev != null)
+ prev = waitingToSend.putIfAbsent(++waitUntilReceived, Thread.currentThread());
+
+ boolean isInterrupted;
+ while (!(isInterrupted = Thread.currentThread().isInterrupted())
+ && waitUntilReceived - receivedBytes >= 0
+ && waitingToSend.get(waitUntilReceived) != null)
+ {
+ LockSupport.park();
+ }
+ waitingToSend.remove(waitUntilReceived);
+
+ if (isInterrupted)
+ throw new InterruptedException();
+ }
+ }
+
+ public long minimumInFlightBytes() { return minimumInFlightBytes; }
+ public long maximumInFlightBytes() { return maximumInFlightBytes; }
+
+ void adjust(int predictedSentBytes, int actualSentBytes)
+ {
+ receivedBytesUpdater.addAndGet(this, predictedSentBytes - actualSentBytes);
+ if (predictedSentBytes > actualSentBytes) wakeupSenders();
+ else maybeProcessDeferred();
+ }
+
+ public long inFlight()
+ {
+ return sentBytes - receivedBytes;
+ }
+
+ public void fail(int bytes)
+ {
+ receivedBytesUpdater.addAndGet(this, bytes);
+ wakeupSenders();
+ }
+
+ public void process(int bytes, IntConsumer releaseBytes)
+ {
+ while (true)
+ {
+ long sent = sentBytes;
+ long received = receivedBytes;
+ long newReceived = received + bytes;
+ if (sent - newReceived <= minimumInFlightBytes)
+ {
+ deferredBytes.add(Pair.create(bytes, releaseBytes));
+ break;
+ }
+ if (receivedBytesUpdater.compareAndSet(this, received, newReceived))
+ {
+ releaseBytes.accept(bytes);
+ break;
+ }
+ }
+ maybeProcessDeferred();
+ wakeupSenders();
+ }
+
+ void setInFlightByteBounds(long minimumInFlightBytes, long maximumInFlightBytes)
+ {
+ this.minimumInFlightBytes = minimumInFlightBytes;
+ this.maximumInFlightBytes = maximumInFlightBytes;
+ maybeProcessDeferred();
+ }
+
+ // unlike the rest of the class, this method does not handle wrap-around of sent/received;
+ // since this shouldn't happen it's no big deal, but maybe for absurdly long runs it might.
+ // if so, fix it.
+ private void wakeupSenders()
+ {
+ Map.Entry<Long, Thread> next;
+ while (null != (next = waitingToSend.firstEntry()))
+ {
+ if (next.getKey() - receivedBytes >= 0)
+ break;
+
+ if (waitingToSend.remove(next.getKey(), next.getValue()))
+ LockSupport.unpark(next.getValue());
+ }
+ }
+
+ private void maybeProcessDeferred()
+ {
+ while (true)
+ {
+ long sent = sentBytes;
+ long received = receivedBytes;
+ if (sent - received <= minimumInFlightBytes)
+ break;
+
+ Pair<Integer, IntConsumer> next = deferredBytes.poll();
+ if (next == null)
+ break;
+
+ int receive = next.left;
+ IntConsumer callbacks = next.right;
+ while (true)
+ {
+ long newReceived = received + receive;
+ if (receivedBytesUpdater.compareAndSet(this, received, newReceived))
+ {
+ callbacks.accept(receive);
+ wakeupSenders();
+ break;
+ }
+
+ sent = sentBytes;
+ received = receivedBytes;
+ if (sent - received <= minimumInFlightBytes)
+ {
+ deferredBytes.add(next);
+ break; // continues with outer loop to maybe process it if minimumInFlightBytes has changed meanwhile
+ }
+ }
+ }
+ }
+
+}
diff --git a/test/burn/org/apache/cassandra/net/Connection.java b/test/burn/org/apache/cassandra/net/Connection.java
new file mode 100644
index 0000000..c74c0ae
--- /dev/null
+++ b/test/burn/org/apache/cassandra/net/Connection.java
@@ -0,0 +1,397 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.net;
+
+import java.io.IOException;
+import java.nio.channels.ClosedChannelException;
+import java.util.concurrent.Executor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.net.Verifier.Destiny;
+
+import static org.apache.cassandra.net.MessagingService.VERSION_40;
+import static org.apache.cassandra.net.MessagingService.current_version;
+import static org.apache.cassandra.utils.ExecutorUtils.runWithThreadName;
+import static org.apache.cassandra.utils.MonotonicClock.approxTime;
+
+public class Connection implements InboundMessageCallbacks, OutboundMessageCallbacks, OutboundDebugCallbacks
+{
+ static class IntentionalIOException extends IOException {}
+ static class IntentionalRuntimeException extends RuntimeException {}
+
+ final InetAddressAndPort sender;
+ final InetAddressAndPort recipient;
+ final BytesInFlightController controller;
+ final InboundMessageHandlers inbound;
+ final OutboundConnection outbound;
+ final OutboundConnectionSettings outboundTemplate;
+ final Verifier verifier;
+ final MessageGenerator sendGenerator;
+ final String linkId;
+ final long minId;
+ final long maxId;
+ final AtomicInteger isSending = new AtomicInteger();
+ private volatile Runnable onSync;
+ final Lock managementLock = new ReentrantLock();
+
+ private final AtomicLong nextSendId = new AtomicLong();
+
+ Connection(InetAddressAndPort sender, InetAddressAndPort recipient, ConnectionType type,
+ InboundMessageHandlers inbound,
+ OutboundConnectionSettings outboundTemplate, ResourceLimits.EndpointAndGlobal reserveCapacityInBytes,
+ MessageGenerator generator,
+ long minId, long maxId)
+ {
+ this.sender = sender;
+ this.recipient = recipient;
+ this.controller = new BytesInFlightController(1 << 20);
+ this.sendGenerator = generator.copy();
+ this.minId = minId;
+ this.maxId = maxId;
+ this.nextSendId.set(minId);
+ this.linkId = sender.toString(false) + "->" + recipient.toString(false) + "-" + type;
+ this.outboundTemplate = outboundTemplate.toEndpoint(recipient)
+ .withFrom(sender)
+ .withCallbacks(this)
+ .withDebugCallbacks(this);
+ this.inbound = inbound;
+ this.outbound = new OutboundConnection(type, this.outboundTemplate, reserveCapacityInBytes);
+ this.verifier = new Verifier(controller, outbound, inbound);
+ }
+
+ void startVerifier(Runnable onFailure, Executor executor, long deadlineNanos)
+ {
+ executor.execute(runWithThreadName(() -> verifier.run(onFailure, deadlineNanos), "Verify-" + linkId));
+ }
+
+ boolean isSending()
+ {
+ return isSending.get() > 0;
+ }
+
+ boolean registerSender()
+ {
+ return isSending.updateAndGet(i -> i < 0 ? i : i + 1) > 0;
+ }
+
+ void unregisterSender()
+ {
+ if (isSending.updateAndGet(i -> i < 0 ? i + 1 : i - 1) == -1)
+ {
+ Runnable onSync = this.onSync;
+ this.onSync = null;
+ verifier.onSync(() -> {
+ onSync.run();
+ isSending.set(0);
+ });
+ }
+ }
+
+ boolean setInFlightByteBounds(long minBytes, long maxBytes)
+ {
+ if (managementLock.tryLock())
+ {
+ try
+ {
+ if (isSending.get() >= 0)
+ {
+ controller.setInFlightByteBounds(minBytes, maxBytes);
+ return true;
+ }
+ }
+ finally
+ {
+ managementLock.unlock();
+ }
+ }
+ return false;
+ }
+
+ void sync(Runnable onCompletion)
+ {
+ managementLock.lock();
+ try
+ {
+ assert onSync == null;
+ assert isSending.get() >= 0;
+ isSending.updateAndGet(i -> -2 -i);
+ long previousMin = controller.minimumInFlightBytes();
+ long previousMax = controller.maximumInFlightBytes();
+ controller.setInFlightByteBounds(0, Long.MAX_VALUE);
+ onSync = () -> {
+ long inFlight = controller.inFlight();
+ if (inFlight != 0)
+ verifier.logFailure("%s has %d bytes in flight, but connection is idle", linkId, inFlight);
+ controller.setInFlightByteBounds(previousMin, previousMax);
+ onCompletion.run();
+ };
+ unregisterSender();
+ }
+ finally
+ {
+ managementLock.unlock();
+ }
+ }
+
+ void sendOne() throws InterruptedException
+ {
+ long id = nextSendId.getAndUpdate(i -> i == maxId ? minId : i + 1);
+ try
+ {
+ Destiny destiny = Destiny.SUCCEED;
+ byte realDestiny = 0;
+ Message<?> msg;
+ synchronized (sendGenerator)
+ {
+ if (0 == sendGenerator.uniformInt(1 << 10))
+ {
+ // abnormal destiny
+ realDestiny = (byte) (1 + sendGenerator.uniformInt(6));
+ destiny = realDestiny <= 3 ? Destiny.FAIL_TO_SERIALIZE : Destiny.FAIL_TO_DESERIALIZE;
+ }
+ msg = sendGenerator.generate(id, realDestiny);
+ }
+
+ controller.send(msg.serializedSize(current_version));
+ Verifier.EnqueueMessageEvent e = verifier.onEnqueue(msg, destiny);
+ outbound.enqueue(msg);
+ e.complete(verifier);
+ }
+ catch (ClosedChannelException e)
+ {
+ // TODO: make this a tested, not illegal, state
+ throw new IllegalStateException(e);
+ }
+ }
+
+ void reconnectWith(OutboundConnectionSettings template)
+ {
+ outbound.reconnectWith(template);
+ }
+
+ void serialize(long id, byte[] payload, DataOutputPlus out, int messagingVersion) throws IOException
+ {
+ verifier.onSerialize(id, messagingVersion);
+ int firstWrite = payload.length, remainder = 0;
+ boolean willFail = false;
+ if (outbound.type() != ConnectionType.LARGE_MESSAGES || messagingVersion >= VERSION_40)
+ {
+ // We cannot (with Netty) know how many bytes make it to the network as any partially written block
+ // will be failed despite having partially succeeded. So to support this behaviour here, we would
+ // need to accept either outcome, in which case what is the point?
+ // TODO: it would be nice to fix this, still
+ willFail = outbound.type() != ConnectionType.LARGE_MESSAGES;
+ byte info = MessageGenerator.getInfo(payload);
+ switch (info)
+ {
+ case 1:
+ switch ((int) (id & 1))
+ {
+ case 0: throw new IntentionalIOException();
+ case 1: throw new IntentionalRuntimeException();
+ }
+ break;
+ case 2:
+ willFail = true;
+ firstWrite -= (int)id % payload.length;
+ break;
+ case 3:
+ willFail = true;
+ remainder = (int)id & 65535;
+ break;
+ }
+ }
+
+ MessageGenerator.writeLength(payload, out, messagingVersion);
+ out.write(payload, 0, firstWrite);
+ while (remainder > 0)
+ {
+ out.write(payload, 0, Math.min(remainder, payload.length));
+ remainder -= payload.length;
+ }
+ if (!willFail)
+ verifier.onFinishSerializeLarge(id);
+ }
+
+ byte[] deserialize(MessageGenerator.Header header, DataInputPlus in, int messagingVersion) throws IOException
+ {
+ verifier.onDeserialize(header.id, messagingVersion);
+ int length = header.length;
+ switch (header.info)
+ {
+ case 4:
+ switch ((int) (header.id & 1))
+ {
+ case 0: throw new IntentionalIOException();
+ case 1: throw new IntentionalRuntimeException();
+ }
+ break;
+ case 5: {
+ length -= (int)header.id % header.length;
+ break;
+ }
+ case 6: {
+ length += (int)header.id & 65535;
+ break;
+ }
+ }
+ byte[] result = header.read(in, Math.min(header.length, length), messagingVersion);
+ if (length > header.length)
+ {
+ length -= header.length;
+ while (length >= 8)
+ {
+ in.readLong();
+ length -= 8;
+ }
+ while (length-- > 0)
+ in.readByte();
+ }
+ return result;
+ }
+
+ public void process(Message message)
+ {
+ verifier.process(message);
+ }
+
+ public void onHeaderArrived(int messageSize, Message.Header header, long timeElapsed, TimeUnit unit)
+ {
+ }
+
+ public void onArrived(int messageSize, Message.Header header, long timeElapsed, TimeUnit unit)
+ {
+ verifier.onArrived(header.id, messageSize);
+ }
+
+ public void onArrivedExpired(int messageSize, Message.Header header, boolean wasCorrupt, long timeElapsed, TimeUnit timeUnit)
+ {
+ controller.fail(messageSize);
+ verifier.onArrivedExpired(header.id, messageSize, wasCorrupt, timeElapsed, timeUnit);
+ }
+
+ public void onArrivedCorrupt(int messageSize, Message.Header header, long timeElapsed, TimeUnit unit)
+ {
+ controller.fail(messageSize);
+ verifier.onFailedDeserialize(header.id, messageSize);
+ }
+
+ public void onClosedBeforeArrival(int messageSize, Message.Header header, int bytesReceived, boolean wasCorrupt, boolean wasExpired)
+ {
+ controller.fail(messageSize);
+ verifier.onClosedBeforeArrival(header.id, messageSize);
+ }
+
+ public void onFailedDeserialize(int messageSize, Message.Header header, Throwable t)
+ {
+ controller.fail(messageSize);
+ verifier.onFailedDeserialize(header.id, messageSize);
+ }
+
+ public void onDispatched(int messageSize, Message.Header header)
+ {
+ }
+
+ public void onExecuting(int messageSize, Message.Header header, long timeElapsed, TimeUnit unit)
+ {
+ }
+
+ public void onProcessed(int messageSize, Message.Header header)
+ {
+ }
+
+ public void onExpired(int messageSize, Message.Header header, long timeElapsed, TimeUnit timeUnit)
+ {
+ controller.fail(messageSize);
+ verifier.onProcessExpired(header.id, messageSize, timeElapsed, timeUnit);
+ }
+
+ public void onExecuted(int messageSize, Message.Header header, long timeElapsed, TimeUnit unit)
+ {
+ }
+
+ InboundCounters inboundCounters()
+ {
+ return inbound.countersFor(outbound.type());
+ }
+
+ public void onSendSmallFrame(int messageCount, int payloadSizeInBytes)
+ {
+ verifier.onSendFrame(messageCount, payloadSizeInBytes);
+ }
+
+ public void onSentSmallFrame(int messageCount, int payloadSizeInBytes)
+ {
+ verifier.onSentFrame(messageCount, payloadSizeInBytes);
+ }
+
+ public void onFailedSmallFrame(int messageCount, int payloadSizeInBytes)
+ {
+ controller.fail(payloadSizeInBytes);
+ verifier.onFailedFrame(messageCount, payloadSizeInBytes);
+ }
+
+ public void onConnect(int messagingVersion, OutboundConnectionSettings settings)
+ {
+ verifier.onConnectOutbound(messagingVersion, settings);
+ }
+
+ public void onConnectInbound(int messagingVersion, InboundMessageHandler handler)
+ {
+ verifier.onConnectInbound(messagingVersion, handler);
+ }
+
+ public void onOverloaded(Message<?> message, InetAddressAndPort peer)
+ {
+ controller.fail(message.serializedSize(current_version));
+ verifier.onOverloaded(message.id());
+ }
+
+ public void onExpired(Message<?> message, InetAddressAndPort peer)
+ {
+ controller.fail(message.serializedSize(current_version));
+ verifier.onExpiredBeforeSend(message.id(), message.serializedSize(current_version), approxTime.now() - message.createdAtNanos(), TimeUnit.NANOSECONDS);
+ }
+
+ public void onFailedSerialize(Message<?> message, InetAddressAndPort peer, int messagingVersion, int bytesWrittenToNetwork, Throwable failure)
+ {
+ if (bytesWrittenToNetwork == 0)
+ controller.fail(message.serializedSize(messagingVersion));
+ verifier.onFailedSerialize(message.id(), bytesWrittenToNetwork, failure);
+ }
+
+ public void onDiscardOnClose(Message<?> message, InetAddressAndPort peer)
+ {
+ controller.fail(message.serializedSize(current_version));
+ verifier.onFailedClosing(message.id());
+ }
+
+ public String toString()
+ {
+ return linkId;
+ }
+}
+
diff --git a/test/burn/org/apache/cassandra/net/ConnectionBurnTest.java b/test/burn/org/apache/cassandra/net/ConnectionBurnTest.java
new file mode 100644
index 0000000..a421f3e
--- /dev/null
+++ b/test/burn/org/apache/cassandra/net/ConnectionBurnTest.java
@@ -0,0 +1,675 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.net;
+
+import java.io.IOException;
+import java.net.UnknownHostException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.function.BiConsumer;
+import java.util.function.Consumer;
+import java.util.function.IntConsumer;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.util.concurrent.Uninterruptibles;
+
+import org.junit.BeforeClass;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import io.netty.channel.Channel;
+import net.openhft.chronicle.core.util.ThrowingBiConsumer;
+import net.openhft.chronicle.core.util.ThrowingRunnable;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.commitlog.CommitLog;
+import org.apache.cassandra.io.IVersionedSerializer;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.net.MessageGenerator.UniformPayloadGenerator;
+import org.apache.cassandra.utils.ExecutorUtils;
+import org.apache.cassandra.utils.MonotonicClock;
+import org.apache.cassandra.utils.memory.BufferPool;
+
+import static java.lang.Math.min;
+import static org.apache.cassandra.net.MessagingService.current_version;
+import static org.apache.cassandra.net.ConnectionType.LARGE_MESSAGES;
+import static org.apache.cassandra.utils.MonotonicClock.approxTime;
+import static org.apache.cassandra.utils.MonotonicClock.preciseTime;
+
+public class ConnectionBurnTest
+{
+ private static final Logger logger = LoggerFactory.getLogger(ConnectionBurnTest.class);
+
+ static
+ {
+ // stop updating ALMOST_SAME_TIME so that we get consistent message expiration times
+ ((MonotonicClock.AbstractEpochSamplingClock) preciseTime).pauseEpochSampling();
+ DatabaseDescriptor.daemonInitialization();
+ DatabaseDescriptor.setCrossNodeTimeout(true);
+ }
+
+ static class NoGlobalInboundMetrics implements InboundMessageHandlers.GlobalMetricCallbacks
+ {
+ static final NoGlobalInboundMetrics instance = new NoGlobalInboundMetrics();
+ public LatencyConsumer internodeLatencyRecorder(InetAddressAndPort to)
+ {
+ return (timeElapsed, timeUnit) -> {};
+ }
+ public void recordInternalLatency(Verb verb, long timeElapsed, TimeUnit timeUnit) {}
+ public void recordInternodeDroppedMessage(Verb verb, long timeElapsed, TimeUnit timeUnit) {}
+ }
+
+ static class Inbound
+ {
+ final Map<InetAddressAndPort, Map<InetAddressAndPort, InboundMessageHandlers>> handlersByRecipientThenSender;
+ final InboundSockets sockets;
+
+ Inbound(List<InetAddressAndPort> endpoints, GlobalInboundSettings settings, Test test)
+ {
+ final InboundMessageHandlers.GlobalResourceLimits globalInboundLimits = new InboundMessageHandlers.GlobalResourceLimits(new ResourceLimits.Concurrent(settings.globalReserveLimit));
+ Map<InetAddressAndPort, Map<InetAddressAndPort, InboundMessageHandlers>> handlersByRecipientThenSender = new HashMap<>();
+ List<InboundConnectionSettings> bind = new ArrayList<>();
+ for (InetAddressAndPort recipient : endpoints)
+ {
+ Map<InetAddressAndPort, InboundMessageHandlers> handlersBySender = new HashMap<>();
+ for (InetAddressAndPort sender : endpoints)
+ handlersBySender.put(sender, new InboundMessageHandlers(recipient, sender, settings.queueCapacity, settings.endpointReserveLimit, globalInboundLimits, NoGlobalInboundMetrics.instance, test, test));
+
+ handlersByRecipientThenSender.put(recipient, handlersBySender);
+ bind.add(settings.template.withHandlers(handlersBySender::get).withBindAddress(recipient));
+ }
+ this.sockets = new InboundSockets(bind);
+ this.handlersByRecipientThenSender = handlersByRecipientThenSender;
+ }
+ }
+
+ private static class ConnectionKey
+ {
+ final InetAddressAndPort from;
+ final InetAddressAndPort to;
+ final ConnectionType type;
+
+ private ConnectionKey(InetAddressAndPort from, InetAddressAndPort to, ConnectionType type)
+ {
+ this.from = from;
+ this.to = to;
+ this.type = type;
+ }
+
+ public boolean equals(Object o)
+ {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ ConnectionKey that = (ConnectionKey) o;
+ return Objects.equals(from, that.from) &&
+ Objects.equals(to, that.to) &&
+ type == that.type;
+ }
+
+ public int hashCode()
+ {
+ return Objects.hash(from, to, type);
+ }
+ }
+
+ private static class Test implements InboundMessageHandlers.HandlerProvider, InboundMessageHandlers.MessageConsumer
+ {
+ private final IVersionedSerializer<byte[]> serializer = new IVersionedSerializer<byte[]>()
+ {
+ public void serialize(byte[] payload, DataOutputPlus out, int version) throws IOException
+ {
+ long id = MessageGenerator.getId(payload);
+ forId(id).serialize(id, payload, out, version);
+ }
+
+ public byte[] deserialize(DataInputPlus in, int version) throws IOException
+ {
+ MessageGenerator.Header header = MessageGenerator.readHeader(in, version);
+ return forId(header.id).deserialize(header, in, version);
+ }
+
+ public long serializedSize(byte[] payload, int version)
+ {
+ return MessageGenerator.serializedSize(payload, version);
+ }
+ };
+
+ static class Builder
+ {
+ long time;
+ TimeUnit timeUnit;
+ int endpoints;
+ MessageGenerators generators;
+ OutboundConnectionSettings outbound;
+ GlobalInboundSettings inbound;
+ public Builder time(long time, TimeUnit timeUnit) { this.time = time; this.timeUnit = timeUnit; return this; }
+ public Builder endpoints(int endpoints) { this.endpoints = endpoints; return this; }
+ public Builder inbound(GlobalInboundSettings inbound) { this.inbound = inbound; return this; }
+ public Builder outbound(OutboundConnectionSettings outbound) { this.outbound = outbound; return this; }
+ public Builder generators(MessageGenerators generators) { this.generators = generators; return this; }
+ Test build() { return new Test(endpoints, generators, inbound, outbound, timeUnit.toNanos(time)); }
+ }
+
+ static Builder builder() { return new Builder(); }
+
+ private static final int messageIdsPerConnection = 1 << 20;
+
+ final long runForNanos;
+ final int version;
+ final List<InetAddressAndPort> endpoints;
+ final Inbound inbound;
+ final Connection[] connections;
+ final long[] connectionMessageIds;
+ final ExecutorService executor = Executors.newCachedThreadPool();
+ final Map<ConnectionKey, Connection> connectionLookup = new HashMap<>();
+
+ private Test(int simulateEndpoints, MessageGenerators messageGenerators, GlobalInboundSettings inboundSettings, OutboundConnectionSettings outboundTemplate, long runForNanos)
+ {
+ this.endpoints = endpoints(simulateEndpoints);
+ this.inbound = new Inbound(endpoints, inboundSettings, this);
+ this.connections = new Connection[endpoints.size() * endpoints.size() * 3];
+ this.connectionMessageIds = new long[connections.length];
+ this.version = outboundTemplate.acceptVersions == null ? current_version : outboundTemplate.acceptVersions.max;
+ this.runForNanos = runForNanos;
+
+ int i = 0;
+ long minId = 0, maxId = messageIdsPerConnection - 1;
+ for (InetAddressAndPort recipient : endpoints)
+ {
+ for (InetAddressAndPort sender : endpoints)
+ {
+ InboundMessageHandlers inboundHandlers = inbound.handlersByRecipientThenSender.get(recipient).get(sender);
+ OutboundConnectionSettings template = outboundTemplate.withDefaultReserveLimits();
+ ResourceLimits.Limit reserveEndpointCapacityInBytes = new ResourceLimits.Concurrent(template.applicationSendQueueReserveEndpointCapacityInBytes);
+ ResourceLimits.EndpointAndGlobal reserveCapacityInBytes = new ResourceLimits.EndpointAndGlobal(reserveEndpointCapacityInBytes, template.applicationSendQueueReserveGlobalCapacityInBytes);
+ for (ConnectionType type : ConnectionType.MESSAGING_TYPES)
+ {
+ Connection connection = new Connection(sender, recipient, type, inboundHandlers, template, reserveCapacityInBytes, messageGenerators.get(type), minId, maxId);
+ this.connections[i] = connection;
+ this.connectionMessageIds[i] = minId;
+ connectionLookup.put(new ConnectionKey(sender, recipient, type), connection);
+ minId = maxId + 1;
+ maxId += messageIdsPerConnection;
+ ++i;
+ }
+ }
+ }
+ }
+
+ Connection forId(long messageId)
+ {
+ int i = Arrays.binarySearch(connectionMessageIds, messageId);
+ if (i < 0) i = -2 -i;
+ Connection connection = connections[i];
+ assert connection.minId <= messageId && connection.maxId >= messageId;
+ return connection;
+ }
+
+ List<Connection> getConnections(InetAddressAndPort endpoint, boolean inbound)
+ {
+ List<Connection> result = new ArrayList<>();
+ for (ConnectionType type : ConnectionType.MESSAGING_TYPES)
+ {
+ for (InetAddressAndPort other : endpoints)
+ {
+ result.add(connectionLookup.get(inbound ? new ConnectionKey(other, endpoint, type)
+ : new ConnectionKey(endpoint, other, type)));
+ }
+ }
+ result.forEach(c -> {assert endpoint.equals(inbound ? c.recipient : c.sender); });
+ return result;
+ }
+
+ /**
+ * Test connections with broken messages, live in-flight bytes updates, reconnect
+ */
+ public void run() throws ExecutionException, InterruptedException, NoSuchFieldException, IllegalAccessException, TimeoutException
+ {
+ Reporters reporters = new Reporters(endpoints, connections);
+ try
+ {
+ long deadline = System.nanoTime() + runForNanos;
+ Verb._TEST_2.unsafeSetHandler(() -> message -> {});
+ Verb._TEST_2.unsafeSetSerializer(() -> serializer);
+ inbound.sockets.open().get();
+
+ CountDownLatch failed = new CountDownLatch(1);
+ for (Connection connection : connections)
+ connection.startVerifier(failed::countDown, executor, deadline);
+
+ for (int i = 0 ; i < 2 * connections.length ; ++i)
+ {
+ executor.execute(() -> {
+ String threadName = Thread.currentThread().getName();
+ try
+ {
+ ThreadLocalRandom random = ThreadLocalRandom.current();
+ while (approxTime.now() < deadline && !Thread.currentThread().isInterrupted())
+ {
+ Connection connection = connections[random.nextInt(connections.length)];
+ if (!connection.registerSender())
+ continue;
+
+ try
+ {
+ Thread.currentThread().setName("Generate-" + connection.linkId);
+ int count = 0;
+ switch (random.nextInt() & 3)
+ {
+ case 0: count = random.nextInt(100, 200); break;
+ case 1: count = random.nextInt(200, 1000); break;
+ case 2: count = random.nextInt(1000, 2000); break;
+ case 3: count = random.nextInt(2000, 10000); break;
+ }
+
+ if (connection.outbound.type() == LARGE_MESSAGES)
+ count /= 2;
+
+ while (connection.isSending()
+ && count-- > 0
+ && approxTime.now() < deadline
+ && !Thread.currentThread().isInterrupted())
+ connection.sendOne();
+ }
+ finally
+ {
+ Thread.currentThread().setName(threadName);
+ connection.unregisterSender();
+ }
+ }
+ }
+ catch (Throwable t)
+ {
+ if (t instanceof InterruptedException)
+ return;
+ logger.error("Unexpected exception", t);
+ failed.countDown();
+ }
+ });
+ }
+
+ executor.execute(() -> {
+ Thread.currentThread().setName("Test-SetInFlight");
+ ThreadLocalRandom random = ThreadLocalRandom.current();
+ List<Connection> connections = new ArrayList<>(Arrays.asList(this.connections));
+ while (!Thread.currentThread().isInterrupted())
+ {
+ Collections.shuffle(connections);
+ int total = random.nextInt(1 << 20, 128 << 20);
+ for (int i = connections.size() - 1; i >= 1 ; --i)
+ {
+ int average = total / (i + 1);
+ int max = random.nextInt(1, min(2 * average, total - 2));
+ int min = random.nextInt(0, max);
+ connections.get(i).setInFlightByteBounds(min, max);
+ total -= max;
+ }
+ // note that setInFlightByteBounds might not
+ connections.get(0).setInFlightByteBounds(random.nextInt(0, total), total);
+ Uninterruptibles.sleepUninterruptibly(1L, TimeUnit.SECONDS);
+ }
+ });
+
+ // TODO: slowly modify the pattern of interrupts, from often to infrequent
+ executor.execute(() -> {
+ Thread.currentThread().setName("Test-Reconnect");
+ ThreadLocalRandom random = ThreadLocalRandom.current();
+ while (deadline > System.nanoTime())
+ {
+ try
+ {
+ Thread.sleep(random.nextInt(60000));
+ }
+ catch (InterruptedException e)
+ {
+ break;
+ }
+ Connection connection = connections[random.nextInt(connections.length)];
+ OutboundConnectionSettings template = connection.outboundTemplate;
+ template = ConnectionTest.SETTINGS.get(random.nextInt(ConnectionTest.SETTINGS.size()))
+ .outbound.apply(template);
+ connection.reconnectWith(template);
+ }
+ });
+
+ executor.execute(() -> {
+ Thread.currentThread().setName("Test-Sync");
+ ThreadLocalRandom random = ThreadLocalRandom.current();
+ BiConsumer<InetAddressAndPort, List<Connection>> checkStoppedTo = (to, from) -> {
+ InboundMessageHandlers handlers = from.get(0).inbound;
+ long using = handlers.usingCapacity();
+ long usingReserve = handlers.usingEndpointReserveCapacity();
+ if (using != 0 || usingReserve != 0)
+ {
+ String message = to + " inbound using %d capacity and %d reserve; should be zero";
+ from.get(0).verifier.logFailure(message, using, usingReserve);
+ }
+ };
+ BiConsumer<InetAddressAndPort, List<Connection>> checkStoppedFrom = (from, to) -> {
+ long using = to.stream().map(c -> c.outbound).mapToLong(OutboundConnection::pendingBytes).sum();
+ long usingReserve = to.get(0).outbound.unsafeGetEndpointReserveLimits().using();
+ if (using != 0 || usingReserve != 0)
+ {
+ String message = from + " outbound using %d capacity and %d reserve; should be zero";
+ to.get(0).verifier.logFailure(message, using, usingReserve);
+ }
+ };
+ ThrowingBiConsumer<List<Connection>, ThrowingRunnable<InterruptedException>, InterruptedException> sync =
+ (connections, exec) -> {
+ logger.info("Syncing connections: {}", connections);
+ final CountDownLatch ready = new CountDownLatch(connections.size());
+ final CountDownLatch done = new CountDownLatch(1);
+ for (Connection connection : connections)
+ {
+ connection.sync(() -> {
+ ready.countDown();
+ try { done.await(); }
+ catch (InterruptedException e) { Thread.interrupted(); }
+ });
+ }
+ ready.await();
+ try
+ {
+ exec.run();
+ }
+ finally
+ {
+ done.countDown();
+ }
+ logger.info("Sync'd connections: {}", connections);
+ };
+
+ int count = 0;
+ while (deadline > System.nanoTime())
+ {
+
+ try
+ {
+ Thread.sleep(random.nextInt(10000));
+
+ if (++count % 10 == 0)
+// {
+// boolean checkInbound = random.nextBoolean();
+// BiConsumer<InetAddressAndPort, List<Connection>> verifier = checkInbound ? checkStoppedTo : checkStoppedFrom;
+// InetAddressAndPort endpoint = endpoints.get(random.nextInt(endpoints.size()));
+// List<Connection> connections = getConnections(endpoint, checkInbound);
+// sync.accept(connections, () -> verifier.accept(endpoint, connections));
+// }
+// else if (count % 100 == 0)
+ {
+ sync.accept(ImmutableList.copyOf(connections), () -> {
+
+ for (InetAddressAndPort endpoint : endpoints)
+ {
+ checkStoppedTo .accept(endpoint, getConnections(endpoint, true ));
+ checkStoppedFrom.accept(endpoint, getConnections(endpoint, false));
+ }
+ long inUse = BufferPool.unsafeGetBytesInUse();
+ if (inUse > 0)
+ {
+// try
+// {
+// ManagementFactory.getPlatformMXBean(HotSpotDiagnosticMXBean.class).dumpHeap("/Users/belliottsmith/code/cassandra/cassandra/leak.hprof", true);
+// }
+// catch (IOException e)
+// {
+// throw new RuntimeException(e);
+// }
+ connections[0].verifier.logFailure("Using %d bytes of BufferPool, but all connections are idle", inUse);
+ }
+ });
+ }
+ else
+ {
+ CountDownLatch latch = new CountDownLatch(1);
+ Connection connection = connections[random.nextInt(connections.length)];
+ connection.sync(latch::countDown);
+ latch.await();
+ }
+ }
+ catch (InterruptedException e)
+ {
+ break;
+ }
+ }
+ });
+
+ while (deadline > System.nanoTime() && failed.getCount() > 0)
+ {
+ reporters.update();
+ reporters.print();
+ Uninterruptibles.awaitUninterruptibly(failed, 30L, TimeUnit.SECONDS);
+ }
+
+ executor.shutdownNow();
+ ExecutorUtils.awaitTermination(1L, TimeUnit.MINUTES, executor);
+ }
+ finally
+ {
+ reporters.update();
+ reporters.print();
+
+ inbound.sockets.close().get();
+ new FutureCombiner(Arrays.stream(connections)
+ .map(c -> c.outbound.close(false))
+ .collect(Collectors.toList()))
+ .get();
+ }
+ }
+
+ class WrappedInboundCallbacks implements InboundMessageCallbacks
+ {
+ private final InboundMessageCallbacks wrapped;
+
+ WrappedInboundCallbacks(InboundMessageCallbacks wrapped)
+ {
+ this.wrapped = wrapped;
+ }
+
+ public void onHeaderArrived(int messageSize, Message.Header header, long timeElapsed, TimeUnit unit)
+ {
+ forId(header.id).onHeaderArrived(messageSize, header, timeElapsed, unit);
+ wrapped.onHeaderArrived(messageSize, header, timeElapsed, unit);
+ }
+
+ public void onArrived(int messageSize, Message.Header header, long timeElapsed, TimeUnit unit)
+ {
+ forId(header.id).onArrived(messageSize, header, timeElapsed, unit);
+ wrapped.onArrived(messageSize, header, timeElapsed, unit);
+ }
+
+ public void onArrivedExpired(int messageSize, Message.Header header, boolean wasCorrupt, long timeElapsed, TimeUnit unit)
+ {
+ forId(header.id).onArrivedExpired(messageSize, header, wasCorrupt, timeElapsed, unit);
+ wrapped.onArrivedExpired(messageSize, header, wasCorrupt, timeElapsed, unit);
+ }
+
+ public void onArrivedCorrupt(int messageSize, Message.Header header, long timeElapsed, TimeUnit unit)
+ {
+ forId(header.id).onArrivedCorrupt(messageSize, header, timeElapsed, unit);
+ wrapped.onArrivedCorrupt(messageSize, header, timeElapsed, unit);
+ }
+
+ public void onClosedBeforeArrival(int messageSize, Message.Header header, int bytesReceived, boolean wasCorrupt, boolean wasExpired)
+ {
+ forId(header.id).onClosedBeforeArrival(messageSize, header, bytesReceived, wasCorrupt, wasExpired);
+ wrapped.onClosedBeforeArrival(messageSize, header, bytesReceived, wasCorrupt, wasExpired);
+ }
+
+ public void onFailedDeserialize(int messageSize, Message.Header header, Throwable t)
+ {
+ forId(header.id).onFailedDeserialize(messageSize, header, t);
+ wrapped.onFailedDeserialize(messageSize, header, t);
+ }
+
+ public void onDispatched(int messageSize, Message.Header header)
+ {
+ forId(header.id).onDispatched(messageSize, header);
+ wrapped.onDispatched(messageSize, header);
+ }
+
+ public void onExecuting(int messageSize, Message.Header header, long timeElapsed, TimeUnit unit)
+ {
+ forId(header.id).onExecuting(messageSize, header, timeElapsed, unit);
+ wrapped.onExecuting(messageSize, header, timeElapsed, unit);
+ }
+
+ public void onProcessed(int messageSize, Message.Header header)
+ {
+ forId(header.id).onProcessed(messageSize, header);
+ wrapped.onProcessed(messageSize, header);
+ }
+
+ public void onExpired(int messageSize, Message.Header header, long timeElapsed, TimeUnit unit)
+ {
+ forId(header.id).onExpired(messageSize, header, timeElapsed, unit);
+ wrapped.onExpired(messageSize, header, timeElapsed, unit);
+ }
+
+ public void onExecuted(int messageSize, Message.Header header, long timeElapsed, TimeUnit unit)
+ {
+ forId(header.id).onExecuted(messageSize, header, timeElapsed, unit);
+ wrapped.onExecuted(messageSize, header, timeElapsed, unit);
+ }
+ }
+
+ public void fail(Message.Header header, Throwable failure)
+ {
+// forId(header.id).verifier.logFailure("Unexpected failure", failure);
+ }
+
+ public void accept(Message<?> message)
+ {
+ forId(message.id()).process(message);
+ }
+
+ public InboundMessageCallbacks wrap(InboundMessageCallbacks wrap)
+ {
+ return new WrappedInboundCallbacks(wrap);
+ }
+
+ public InboundMessageHandler provide(
+ FrameDecoder decoder,
+ ConnectionType type,
+ Channel channel,
+ InetAddressAndPort self,
+ InetAddressAndPort peer,
+ int version,
+ int largeMessageThreshold,
+ int queueCapacity,
+ ResourceLimits.Limit endpointReserveCapacity,
+ ResourceLimits.Limit globalReserveCapacity,
+ InboundMessageHandler.WaitQueue endpointWaitQueue,
+ InboundMessageHandler.WaitQueue globalWaitQueue,
+ InboundMessageHandler.OnHandlerClosed onClosed,
+ InboundMessageCallbacks callbacks,
+ Consumer<Message<?>> messageSink
+ )
+ {
+ return new InboundMessageHandler(decoder, type, channel, self, peer, version, largeMessageThreshold, queueCapacity, endpointReserveCapacity, globalReserveCapacity, endpointWaitQueue, globalWaitQueue, onClosed, wrap(callbacks), messageSink)
+ {
+ final IntConsumer releaseCapacity = size -> super.releaseProcessedCapacity(size, null);
+ protected void releaseProcessedCapacity(int bytes, Message.Header header)
+ {
+ forId(header.id).controller.process(bytes, releaseCapacity);
+ }
+ };
+ }
+ }
+
+ static List<InetAddressAndPort> endpoints(int count)
+ {
+ return IntStream.rangeClosed(1, count)
+ .mapToObj(ConnectionBurnTest::endpoint)
+ .collect(Collectors.toList());
+ }
+
+ private static InetAddressAndPort endpoint(int i)
+ {
+ try
+ {
+ return InetAddressAndPort.getByName("127.0.0." + i);
+ }
+ catch (UnknownHostException e)
+ {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private void test(GlobalInboundSettings inbound, OutboundConnectionSettings outbound) throws ExecutionException, InterruptedException, NoSuchFieldException, IllegalAccessException, TimeoutException
+ {
+ MessageGenerator small = new UniformPayloadGenerator(0, 1, (1 << 15));
+ MessageGenerator large = new UniformPayloadGenerator(0, 1, (1 << 16) + (1 << 15));
+ MessageGenerators generators = new MessageGenerators(small, large);
+ outbound = outbound.withApplicationSendQueueCapacityInBytes(1 << 18)
+ .withApplicationReserveSendQueueCapacityInBytes(1 << 30, new ResourceLimits.Concurrent(Integer.MAX_VALUE));
+
+ Test.builder()
+ .generators(generators)
+ .endpoints(4)
+ .inbound(inbound)
+ .outbound(outbound)
+ // change the following for a longer burn
+ .time(2L, TimeUnit.MINUTES)
+ .build().run();
+ }
+
+ public static void main(String[] args) throws ExecutionException, InterruptedException, NoSuchFieldException, IllegalAccessException, TimeoutException
+ {
+ setup();
+ new ConnectionBurnTest().test();
+ }
+
+ @BeforeClass
+ public static void setup()
+ {
+ // since CASSANDRA-15295, commitlog needs to be manually started.
+ CommitLog.instance.start();
+ }
+
+ @org.junit.Test
+ public void test() throws ExecutionException, InterruptedException, NoSuchFieldException, IllegalAccessException, TimeoutException
+ {
+ GlobalInboundSettings inboundSettings = new GlobalInboundSettings()
+ .withQueueCapacity(1 << 18)
+ .withEndpointReserveLimit(1 << 20)
+ .withGlobalReserveLimit(1 << 21)
+ .withTemplate(new InboundConnectionSettings()
+ .withEncryption(ConnectionTest.encryptionOptions));
+
+ test(inboundSettings, new OutboundConnectionSettings(null)
+ .withTcpUserTimeoutInMS(0));
+ MessagingService.instance().socketFactory.shutdownNow();
+ }
+}
diff --git a/test/burn/org/apache/cassandra/net/GlobalInboundSettings.java b/test/burn/org/apache/cassandra/net/GlobalInboundSettings.java
new file mode 100644
index 0000000..9b23041
--- /dev/null
+++ b/test/burn/org/apache/cassandra/net/GlobalInboundSettings.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.net;
+
+class GlobalInboundSettings
+{
+ final int queueCapacity;
+ final long endpointReserveLimit;
+ final long globalReserveLimit;
+ final InboundConnectionSettings template;
+
+ GlobalInboundSettings()
+ {
+ this(0, 0, 0, null);
+ }
+
+ GlobalInboundSettings(int queueCapacity, long endpointReserveLimit, long globalReserveLimit, InboundConnectionSettings template)
+ {
+ this.queueCapacity = queueCapacity;
+ this.endpointReserveLimit = endpointReserveLimit;
+ this.globalReserveLimit = globalReserveLimit;
+ this.template = template;
+ }
+
+ GlobalInboundSettings withQueueCapacity(int queueCapacity)
+ {
+ return new GlobalInboundSettings(queueCapacity, endpointReserveLimit, globalReserveLimit, template);
+ }
+ GlobalInboundSettings withEndpointReserveLimit(int endpointReserveLimit)
+ {
+ return new GlobalInboundSettings(queueCapacity, endpointReserveLimit, globalReserveLimit, template);
+ }
+ GlobalInboundSettings withGlobalReserveLimit(int globalReserveLimit)
+ {
+ return new GlobalInboundSettings(queueCapacity, endpointReserveLimit, globalReserveLimit, template);
+ }
+ GlobalInboundSettings withTemplate(InboundConnectionSettings template)
+ {
+ return new GlobalInboundSettings(queueCapacity, endpointReserveLimit, globalReserveLimit, template);
+ }
+}
\ No newline at end of file
diff --git a/test/burn/org/apache/cassandra/net/LogbackFilter.java b/test/burn/org/apache/cassandra/net/LogbackFilter.java
new file mode 100644
index 0000000..94aa2f9
--- /dev/null
+++ b/test/burn/org/apache/cassandra/net/LogbackFilter.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.net;
+
+import java.io.EOFException;
+import java.nio.BufferOverflowException;
+import java.util.Set;
+import java.util.regex.Pattern;
+
+import com.google.common.collect.ImmutableSet;
+
+import ch.qos.logback.classic.spi.IThrowableProxy;
+import ch.qos.logback.classic.spi.LoggingEvent;
+import ch.qos.logback.core.filter.Filter;
+import ch.qos.logback.core.spi.FilterReply;
+
+public class LogbackFilter extends Filter
+{
+ private static final Pattern ignore = Pattern.compile("(successfully connected|connection established), version =");
+
+ public FilterReply decide(Object o)
+ {
+ if (!(o instanceof LoggingEvent))
+ return FilterReply.NEUTRAL;
+
+ LoggingEvent e = (LoggingEvent) o;
+// if (ignore.matcher(e.getMessage()).find())
+// return FilterReply.DENY;
+
+ IThrowableProxy t = e.getThrowableProxy();
+ if (t == null)
+ return FilterReply.NEUTRAL;
+
+ if (!isIntentional(t))
+ return FilterReply.NEUTRAL;
+
+// logger.info("Filtered exception {}: {}", t.getClassName(), t.getMessage());
+ return FilterReply.DENY;
+ }
+
+ private static final Set<String> intentional = ImmutableSet.of(
+ Connection.IntentionalIOException.class.getName(),
+ Connection.IntentionalRuntimeException.class.getName(),
+ InvalidSerializedSizeException.class.getName(),
+ BufferOverflowException.class.getName(),
+ EOFException.class.getName()
+ );
+
+ public static boolean isIntentional(IThrowableProxy t)
+ {
+ while (true)
+ {
+ if (intentional.contains(t.getClassName()))
+ return true;
+
+ if (null == t.getCause())
+ return false;
+
+ t = t.getCause();
+ }
+ }
+
+
+}
diff --git a/test/burn/org/apache/cassandra/net/MessageGenerator.java b/test/burn/org/apache/cassandra/net/MessageGenerator.java
new file mode 100644
index 0000000..3c03689
--- /dev/null
+++ b/test/burn/org/apache/cassandra/net/MessageGenerator.java
@@ -0,0 +1,190 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.net;
+
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.utils.vint.VIntCoding;
+import sun.misc.Unsafe;
+
+import static org.apache.cassandra.net.MessagingService.VERSION_40;
+import static org.apache.cassandra.utils.MonotonicClock.approxTime;
+
+abstract class MessageGenerator
+{
+ final long seed;
+ final Random random;
+
+ private MessageGenerator(long seed)
+ {
+ this.seed = seed;
+ this.random = new Random();
+ }
+
+ Message.Builder<Object> builder(long id)
+ {
+ random.setSeed(id ^ seed);
+ long now = approxTime.now();
+
+ int expiresInMillis;
+ int expiryMask = random.nextInt();
+ if (0 == (expiryMask & 0xffff)) expiresInMillis = 2;
+ else if (0 == (expiryMask & 0xfff)) expiresInMillis = 10;
+ else if (0 == (expiryMask & 0xff)) expiresInMillis = 100;
+ else if (0 == (expiryMask & 0xf)) expiresInMillis = 1000;
+ else expiresInMillis = 60 * 1000;
+
+ long expiresInNanos = TimeUnit.MILLISECONDS.toNanos((expiresInMillis / 2) + random.nextInt(expiresInMillis / 2));
+
+ return Message.builder(Verb._TEST_2, null)
+ .withId(id)
+ .withCreatedAt(now)
+ .withExpiresAt(now + expiresInNanos); // don't expire for now
+ }
+
+ public int uniformInt(int limit)
+ {
+ return random.nextInt(limit);
+ }
+
+ // generate a Message<?> with the provided id and with both id and info encoded in its payload
+ abstract Message<?> generate(long id, byte info);
+ abstract MessageGenerator copy();
+
+ static final class UniformPayloadGenerator extends MessageGenerator
+ {
+ final int minSize;
+ final int maxSize;
+ final byte[] fillWithBytes;
+ UniformPayloadGenerator(long seed, int minSize, int maxSize)
+ {
+ super(seed);
+ this.minSize = Math.max(9, minSize);
+ this.maxSize = Math.max(9, maxSize);
+ this.fillWithBytes = new byte[32];
+ random.setSeed(seed);
+ random.nextBytes(fillWithBytes);
+ }
+
+ Message<?> generate(long id, byte info)
+ {
+ Message.Builder<Object> builder = builder(id);
+ byte[] payload = new byte[minSize + random.nextInt(maxSize - minSize)];
+ ByteBuffer wrapped = ByteBuffer.wrap(payload);
+ setId(payload, id);
+ payload[8] = info;
+ wrapped.position(9);
+ while (wrapped.hasRemaining())
+ wrapped.put(fillWithBytes, 0, Math.min(fillWithBytes.length, wrapped.remaining()));
+ builder.withPayload(payload);
+ return builder.build();
+ }
+
+ MessageGenerator copy()
+ {
+ return new UniformPayloadGenerator(seed, minSize, maxSize);
+ }
+ }
+
+ static long getId(byte[] payload)
+ {
+ return unsafe.getLong(payload, BYTE_ARRAY_BASE_OFFSET);
+ }
+ static byte getInfo(byte[] payload)
+ {
+ return payload[8];
+ }
+ private static void setId(byte[] payload, long id)
+ {
+ unsafe.putLong(payload, BYTE_ARRAY_BASE_OFFSET, id);
+ }
+
+ static class Header
+ {
+ public final int length;
+ public final long id;
+ public final byte info;
+
+ Header(int length, long id, byte info)
+ {
+ this.length = length;
+ this.id = id;
+ this.info = info;
+ }
+
+ public byte[] read(DataInputPlus in, int length, int messagingVersion) throws IOException
+ {
+ byte[] result = new byte[Math.max(9, length)];
+ setId(result, id);
+ result[8] = info;
+ in.readFully(result, 9, Math.max(0, length - 9));
+ return result;
+ }
+ }
+
+ static Header readHeader(DataInputPlus in, int messagingVersion) throws IOException
+ {
+ int length = messagingVersion < VERSION_40
+ ? in.readInt()
+ : (int) in.readUnsignedVInt();
+ long id = in.readLong();
+ if (ByteOrder.nativeOrder() == ByteOrder.LITTLE_ENDIAN)
+ id = Long.reverseBytes(id);
+ byte info = in.readByte();
+ return new Header(length, id, info);
+ }
+
+ static void writeLength(byte[] payload, DataOutputPlus out, int messagingVersion) throws IOException
+ {
+ if (messagingVersion < VERSION_40)
+ out.writeInt(payload.length);
+ else
+ out.writeUnsignedVInt(payload.length);
+ }
+
+ static long serializedSize(byte[] payload, int messagingVersion)
+ {
+ return payload.length + (messagingVersion < VERSION_40 ? 4 : VIntCoding.computeUnsignedVIntSize(payload.length));
+ }
+
+ private static final Unsafe unsafe;
+ static
+ {
+ try
+ {
+ Field field = sun.misc.Unsafe.class.getDeclaredField("theUnsafe");
+ field.setAccessible(true);
+ unsafe = (sun.misc.Unsafe) field.get(null);
+ }
+ catch (Exception e)
+ {
+ throw new AssertionError(e);
+ }
+ }
+ private static final long BYTE_ARRAY_BASE_OFFSET = unsafe.arrayBaseOffset(byte[].class);
+
+}
+
diff --git a/test/burn/org/apache/cassandra/net/MessageGenerators.java b/test/burn/org/apache/cassandra/net/MessageGenerators.java
new file mode 100644
index 0000000..92aab3a
--- /dev/null
+++ b/test/burn/org/apache/cassandra/net/MessageGenerators.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.net;
+
+final class MessageGenerators
+{
+ final MessageGenerator small;
+ final MessageGenerator large;
+
+ MessageGenerators(MessageGenerator small, MessageGenerator large)
+ {
+ this.small = small;
+ this.large = large;
+ }
+
+ MessageGenerator get(ConnectionType type)
+ {
+ switch (type)
+ {
+ case SMALL_MESSAGES:
+ case URGENT_MESSAGES:
+ return small;
+ case LARGE_MESSAGES:
+ return large;
+ default:
+ throw new IllegalStateException();
+ }
+ }
+}
diff --git a/test/burn/org/apache/cassandra/net/Reporters.java b/test/burn/org/apache/cassandra/net/Reporters.java
new file mode 100644
index 0000000..9ab4643
--- /dev/null
+++ b/test/burn/org/apache/cassandra/net/Reporters.java
@@ -0,0 +1,322 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.net;
+
+import java.util.Arrays;
+import java.util.BitSet;
+import java.util.Collection;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import java.util.function.LongFunction;
+import java.util.function.ToLongFunction;
+
+import com.google.common.collect.ImmutableList;
+
+import org.apache.cassandra.locator.InetAddressAndPort;
+
+class Reporters
+{
+ final Collection<InetAddressAndPort> endpoints;
+ final Connection[] connections;
+ final List<Reporter> reporters;
+ final long start = System.nanoTime();
+
+ Reporters(Collection<InetAddressAndPort> endpoints, Connection[] connections)
+ {
+ this.endpoints = endpoints;
+ this.connections = connections;
+ this.reporters = ImmutableList.of(
+ outboundReporter (true, "Outbound Throughput", OutboundConnection::sentBytes, Reporters::prettyPrintMemory),
+ inboundReporter (true, "Inbound Throughput", InboundCounters::processedBytes, Reporters::prettyPrintMemory),
+
+ outboundReporter (false, "Outbound Pending Bytes", OutboundConnection::pendingBytes, Reporters::prettyPrintMemory),
+ reporter (false, "Inbound Pending Bytes", c -> c.inbound.usingCapacity(), Reporters::prettyPrintMemory),
+
+ outboundReporter (true, "Outbound Expirations", OutboundConnection::expiredCount, Long::toString),
+ inboundReporter (true, "Inbound Expirations", InboundCounters::expiredCount, Long::toString),
+
+ outboundReporter (true, "Outbound Errors", OutboundConnection::errorCount, Long::toString),
+ inboundReporter (true, "Inbound Errors", InboundCounters::errorCount, Long::toString),
+
+ outboundReporter (true, "Outbound Connection Attempts", OutboundConnection::connectionAttempts, Long::toString)
+ );
+ }
+
+ void update()
+ {
+ for (Reporter reporter : reporters)
+ reporter.update();
+ }
+
+ void print()
+ {
+ System.out.println("==" + prettyPrintElapsed(System.nanoTime() - start) + "==\n");
+
+ for (Reporter reporter : reporters)
+ {
+ reporter.print();
+ }
+ }
+
+ private Reporter outboundReporter(boolean accumulates, String name, ToLongFunction<OutboundConnection> get, LongFunction<String> printer)
+ {
+ return new Reporter(accumulates, name, (conn) -> get.applyAsLong(conn.outbound), printer);
+ }
+
+ private Reporter inboundReporter(boolean accumulates, String name, ToLongFunction<InboundCounters> get, LongFunction<String> printer)
+ {
+ return new Reporter(accumulates, name, (conn) -> get.applyAsLong(conn.inboundCounters()), printer);
+ }
+
+ private Reporter reporter(boolean accumulates, String name, ToLongFunction<Connection> get, LongFunction<String> printer)
+ {
+ return new Reporter(accumulates, name, get, printer);
+ }
+
+ class Reporter
+ {
+ boolean accumulates;
+ final String name;
+ final ToLongFunction<Connection> get;
+ final LongFunction<String> print;
+ final long[][] previousValue;
+ final long[] columnTotals = new long[1 + endpoints.size() * 3];
+ final Table table;
+
+ Reporter(boolean accumulates, String name, ToLongFunction<Connection> get, LongFunction<String> print)
+ {
+ this.accumulates = accumulates;
+ this.name = name;
+ this.get = get;
+ this.print = print;
+
+ previousValue = accumulates ? new long[endpoints.size()][endpoints.size() * 3] : null;
+
+ String[] rowNames = new String[endpoints.size() + 1];
+ for (int row = 0 ; row < endpoints.size() ; ++row)
+ {
+ rowNames[row] = Integer.toString(1 + row);
+ }
+ rowNames[rowNames.length - 1] = "Total";
+
+ String[] columnNames = new String[endpoints.size() * 3 + 1];
+ for (int column = 0 ; column < endpoints.size() * 3 ; column += 3)
+ {
+ String endpoint = Integer.toString(1 + column / 3);
+ columnNames[ column] = endpoint + ".Urgent";
+ columnNames[1 + column] = endpoint + ".Small";
+ columnNames[2 + column] = endpoint + ".Large";
+ }
+ columnNames[columnNames.length - 1] = "Total";
+
+ table = new Table(rowNames, columnNames, "Recipient");
+ }
+
+ public void update()
+ {
+ Arrays.fill(columnTotals, 0);
+ int row = 0, connection = 0;
+ for (InetAddressAndPort recipient : endpoints)
+ {
+ int column = 0;
+ long rowTotal = 0;
+ for (InetAddressAndPort sender : endpoints)
+ {
+ for (ConnectionType type : ConnectionType.MESSAGING_TYPES)
+ {
+ assert recipient.equals(connections[connection].recipient);
+ assert sender.equals(connections[connection].sender);
+ assert type == connections[connection].outbound.type();
+
+ long cur = get.applyAsLong(connections[connection]);
+ long value;
+ if (accumulates)
+ {
+ long prev = previousValue[row][column];
+ previousValue[row][column] = cur;
+ value = cur - prev;
+ }
+ else
+ {
+ value = cur;
+ }
+ table.set(row, column, print.apply(value));
+ columnTotals[column] += value;
+ rowTotal += value;
+ ++column;
+ ++connection;
+ }
+ }
+ columnTotals[column] += rowTotal;
+ table.set(row, column, print.apply(rowTotal));
+ table.displayRow(row, rowTotal > 0);
+ ++row;
+ }
+
+ boolean displayTotalRow = false;
+ for (int column = 0 ; column < columnTotals.length ; ++column)
+ {
+ table.set(endpoints.size(), column, print.apply(columnTotals[column]));
+ table.displayColumn(column, columnTotals[column] > 0);
+ displayTotalRow |= columnTotals[column] > 0;
+ }
+ table.displayRow(endpoints.size(), displayTotalRow);
+ }
+
+ public void print()
+ {
+ table.print("===" + name + "===");
+ }
+ }
+
+ private static class Table
+ {
+ final String[][] print;
+ final int[] width;
+ final BitSet rowMask = new BitSet();
+ final BitSet columnMask = new BitSet();
+
+ public Table(String[] rowNames, String[] columnNames, String rowNameHeader)
+ {
+ print = new String[rowNames.length + 1][columnNames.length + 1];
+ width = new int[columnNames.length + 1];
+ print[0][0] = rowNameHeader;
+ for (int i = 0 ; i < columnNames.length ; ++i)
+ print[0][1 + i] = columnNames[i];
+ for (int i = 0 ; i < rowNames.length ; ++i)
+ print[1 + i][0] = rowNames[i];
+ }
+
+ void set(int row, int column, String value)
+ {
+ print[row + 1][column + 1] = value;
+ }
+
+ void displayRow(int row, boolean display)
+ {
+ rowMask.set(row, display);
+ }
+
+ void displayColumn(int column, boolean display)
+ {
+ columnMask.set(column, display);
+ }
+
+ void print(String heading)
+ {
+ if (rowMask.isEmpty() && columnMask.isEmpty())
+ return;
+
+ System.out.println(heading + '\n');
+
+ Arrays.fill(width, 0);
+ for (int row = 0 ; row < print.length ; ++row)
+ {
+ for (int column = 0 ; column < width.length ; ++column)
+ {
+ width[column] = Math.max(width[column], print[row][column].length());
+ }
+ }
+
+ for (int row = 0 ; row < print.length ; ++row)
+ {
+// if (row > 0 && !rowMask.get(row - 1))
+// continue;
+
+ StringBuilder builder = new StringBuilder();
+ for (int column = 0 ; column < width.length ; ++column)
+ {
+// if (column > 0 && !columnMask.get(column - 1))
+// continue;
+
+ String s = print[row][column];
+ int pad = width[column] - s.length();
+ for (int i = 0 ; i < pad ; ++i)
+ builder.append(' ');
+ builder.append(s);
+ builder.append(" ");
+ }
+ System.out.println(builder.toString());
+ }
+ System.out.println();
+ }
+ }
+
+ private static final class OneTimeUnit
+ {
+ final TimeUnit unit;
+ final String symbol;
+ final long nanos;
+
+ private OneTimeUnit(TimeUnit unit, String symbol)
+ {
+ this.unit = unit;
+ this.symbol = symbol;
+ this.nanos = unit.toNanos(1L);
+ }
+ }
+
+ private static final List<OneTimeUnit> prettyPrintElapsed = ImmutableList.of(
+ new OneTimeUnit(TimeUnit.DAYS, "d"),
+ new OneTimeUnit(TimeUnit.HOURS, "h"),
+ new OneTimeUnit(TimeUnit.MINUTES, "m"),
+ new OneTimeUnit(TimeUnit.SECONDS, "s"),
+ new OneTimeUnit(TimeUnit.MILLISECONDS, "ms"),
+ new OneTimeUnit(TimeUnit.MICROSECONDS, "us"),
+ new OneTimeUnit(TimeUnit.NANOSECONDS, "ns")
+ );
+
+ private static String prettyPrintElapsed(long nanos)
+ {
+ if (nanos == 0)
+ return "0ns";
+
+ int count = 0;
+ StringBuilder builder = new StringBuilder();
+ for (OneTimeUnit unit : prettyPrintElapsed)
+ {
+ if (count == 2)
+ break;
+
+ if (nanos >= unit.nanos)
+ {
+ if (count > 0)
+ builder.append(' ');
+ long inUnit = unit.unit.convert(nanos, TimeUnit.NANOSECONDS);
+ nanos -= unit.unit.toNanos(inUnit);
+ builder.append(inUnit);
+ builder.append(unit.symbol);
+ ++count;
+ } else if (count > 0)
+ ++count;
+ }
+
+ return builder.toString();
+ }
+
+ static String prettyPrintMemory(long size)
+ {
+ if (size >= 1000 * 1000 * 1000)
+ return String.format("%.0fG", size / (double) (1 << 30));
+ if (size >= 1000 * 1000)
+ return String.format("%.0fM", size / (double) (1 << 20));
+ return String.format("%.0fK", size / (double) (1 << 10));
+ }
+}
+
diff --git a/test/burn/org/apache/cassandra/net/Verifier.java b/test/burn/org/apache/cassandra/net/Verifier.java
new file mode 100644
index 0000000..219e613
--- /dev/null
+++ b/test/burn/org/apache/cassandra/net/Verifier.java
@@ -0,0 +1,1639 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.net;
+
+import java.nio.BufferOverflowException;
+import java.util.Arrays;
+import java.util.Map;
+import java.util.concurrent.ConcurrentSkipListMap;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReferenceArray;
+import java.util.concurrent.locks.LockSupport;
+import java.util.function.Consumer;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.carrotsearch.hppc.LongObjectHashMap;
+import com.carrotsearch.hppc.predicates.LongObjectPredicate;
+import com.carrotsearch.hppc.procedures.LongObjectProcedure;
+import com.carrotsearch.hppc.procedures.LongProcedure;
+import org.apache.cassandra.net.Verifier.ExpiredMessageEvent.ExpirationType;
+import org.apache.cassandra.utils.concurrent.WaitQueue;
+
+import static java.util.concurrent.TimeUnit.*;
+import static java.util.concurrent.TimeUnit.NANOSECONDS;
+import static org.apache.cassandra.net.MessagingService.VERSION_40;
+import static org.apache.cassandra.net.MessagingService.current_version;
+import static org.apache.cassandra.net.ConnectionType.LARGE_MESSAGES;
+import static org.apache.cassandra.net.OutboundConnection.LargeMessageDelivery.DEFAULT_BUFFER_SIZE;
+import static org.apache.cassandra.net.OutboundConnections.LARGE_MESSAGE_THRESHOLD;
+import static org.apache.cassandra.net.Verifier.EventCategory.OTHER;
+import static org.apache.cassandra.net.Verifier.EventCategory.RECEIVE;
+import static org.apache.cassandra.net.Verifier.EventCategory.SEND;
+import static org.apache.cassandra.net.Verifier.EventType.ARRIVE;
+import static org.apache.cassandra.net.Verifier.EventType.CLOSED_BEFORE_ARRIVAL;
+import static org.apache.cassandra.net.Verifier.EventType.DESERIALIZE;
+import static org.apache.cassandra.net.Verifier.EventType.ENQUEUE;
+import static org.apache.cassandra.net.Verifier.EventType.FAILED_CLOSING;
+import static org.apache.cassandra.net.Verifier.EventType.FAILED_DESERIALIZE;
+import static org.apache.cassandra.net.Verifier.EventType.FAILED_EXPIRED_ON_SEND;
+import static org.apache.cassandra.net.Verifier.EventType.FAILED_EXPIRED_ON_RECEIVE;
+import static org.apache.cassandra.net.Verifier.EventType.FAILED_FRAME;
+import static org.apache.cassandra.net.Verifier.EventType.FAILED_OVERLOADED;
+import static org.apache.cassandra.net.Verifier.EventType.FAILED_SERIALIZE;
+import static org.apache.cassandra.net.Verifier.EventType.FINISH_SERIALIZE_LARGE;
+import static org.apache.cassandra.net.Verifier.EventType.PROCESS;
+import static org.apache.cassandra.net.Verifier.EventType.SEND_FRAME;
+import static org.apache.cassandra.net.Verifier.EventType.SENT_FRAME;
+import static org.apache.cassandra.net.Verifier.EventType.SERIALIZE;
+import static org.apache.cassandra.net.Verifier.ExpiredMessageEvent.ExpirationType.ON_SENT;
+import static org.apache.cassandra.utils.MonotonicClock.approxTime;
+
+/**
+ * This class is a single-threaded verifier monitoring a single link, with events supplied by inbound and outbound threads
+ *
+ * By making verification single threaded, it is easier to reason about (and complex enough as is), but also permits
+ * a dedicated thread to monitor timeliness of events, e.g. elapsed time between a given SEND and its corresponding RECEIVE
+ *
+ * TODO: timeliness of events
+ * TODO: periodically stop all activity to/from a given endpoint, until it stops (and verify queues all empty, counters all accurate)
+ * TODO: integrate with proxy that corrupts frames
+ * TODO: test _OutboundConnection_ close
+ */
+@SuppressWarnings("WeakerAccess")
+public class Verifier
+{
+ private static final Logger logger = LoggerFactory.getLogger(Verifier.class);
+
+ public enum Destiny
+ {
+ SUCCEED,
+ FAIL_TO_SERIALIZE,
+ FAIL_TO_DESERIALIZE,
+ }
+
+ enum EventCategory
+ {
+ SEND, RECEIVE, OTHER
+ }
+
+ enum EventType
+ {
+ FAILED_OVERLOADED(SEND),
+ ENQUEUE(SEND),
+ FAILED_EXPIRED_ON_SEND(SEND),
+ SERIALIZE(SEND),
+ FAILED_SERIALIZE(SEND),
+ FINISH_SERIALIZE_LARGE(SEND),
+ SEND_FRAME(SEND),
+ FAILED_FRAME(SEND),
+ SENT_FRAME(SEND),
+ ARRIVE(RECEIVE),
+ FAILED_EXPIRED_ON_RECEIVE(RECEIVE),
+ DESERIALIZE(RECEIVE),
+ CLOSED_BEFORE_ARRIVAL(RECEIVE),
+ FAILED_DESERIALIZE(RECEIVE),
+ PROCESS(RECEIVE),
+
+ FAILED_CLOSING(SEND),
+
+ CONNECT_OUTBOUND(OTHER),
+ SYNC(OTHER), // the connection will stop sending messages, and promptly process any waiting inbound messages
+ CONTROLLER_UPDATE(OTHER);
+
+ final EventCategory category;
+
+ EventType(EventCategory category)
+ {
+ this.category = category;
+ }
+ }
+
+ public static class Event
+ {
+ final EventType type;
+ Event(EventType type)
+ {
+ this.type = type;
+ }
+ }
+
+ static class SimpleEvent extends Event
+ {
+ final long at;
+ SimpleEvent(EventType type, long at)
+ {
+ super(type);
+ this.at = at;
+ }
+ public String toString() { return type.toString(); }
+ }
+
+ static class BoundedEvent extends Event
+ {
+ final long start;
+ volatile long end;
+ BoundedEvent(EventType type, long start)
+ {
+ super(type);
+ this.start = start;
+ }
+ public void complete(Verifier verifier)
+ {
+ end = verifier.sequenceId.getAndIncrement();
+ verifier.events.put(end, this);
+ }
+ }
+
+ static class SimpleMessageEvent extends SimpleEvent
+ {
+ final long messageId;
+ SimpleMessageEvent(EventType type, long at, long messageId)
+ {
+ super(type, at);
+ this.messageId = messageId;
+ }
+ }
+
+ static class BoundedMessageEvent extends BoundedEvent
+ {
+ final long messageId;
+ BoundedMessageEvent(EventType type, long start, long messageId)
+ {
+ super(type, start);
+ this.messageId = messageId;
+ }
+ }
+
+ static class EnqueueMessageEvent extends BoundedMessageEvent
+ {
+ final Message<?> message;
+ final Destiny destiny;
+ EnqueueMessageEvent(EventType type, long start, Message<?> message, Destiny destiny)
+ {
+ super(type, start, message.id());
+ this.message = message;
+ this.destiny = destiny;
+ }
+ public String toString() { return String.format("%s{%s}", type, destiny); }
+ }
+
+ static class SerializeMessageEvent extends SimpleMessageEvent
+ {
+ final int messagingVersion;
+ SerializeMessageEvent(EventType type, long at, long messageId, int messagingVersion)
+ {
+ super(type, at, messageId);
+ this.messagingVersion = messagingVersion;
+ }
+ public String toString() { return String.format("%s{ver=%d}", type, messagingVersion); }
+ }
+
+ static class SimpleMessageEventWithSize extends SimpleMessageEvent
+ {
+ final int messageSize;
+ SimpleMessageEventWithSize(EventType type, long at, long messageId, int messageSize)
+ {
+ super(type, at, messageId);
+ this.messageSize = messageSize;
+ }
+ public String toString() { return String.format("%s{size=%d}", type, messageSize); }
+ }
+
+ static class FailedSerializeEvent extends SimpleMessageEvent
+ {
+ final int bytesWrittenToNetwork;
+ final Throwable failure;
+ FailedSerializeEvent(long at, long messageId, int bytesWrittenToNetwork, Throwable failure)
+ {
+ super(FAILED_SERIALIZE, at, messageId);
+ this.bytesWrittenToNetwork = bytesWrittenToNetwork;
+ this.failure = failure;
+ }
+ public String toString() { return String.format("FAILED_SERIALIZE{written=%d, failure=%s}", bytesWrittenToNetwork, failure); }
+ }
+
+ static class ExpiredMessageEvent extends SimpleMessageEvent
+ {
+ enum ExpirationType {ON_SENT, ON_ARRIVED, ON_PROCESSED }
+ final int messageSize;
+ final long timeElapsed;
+ final TimeUnit timeUnit;
+ final ExpirationType expirationType;
+ ExpiredMessageEvent(long at, long messageId, int messageSize, long timeElapsed, TimeUnit timeUnit, ExpirationType expirationType)
+ {
+ super(expirationType == ON_SENT ? FAILED_EXPIRED_ON_SEND : FAILED_EXPIRED_ON_RECEIVE, at, messageId);
+ this.messageSize = messageSize;
+ this.timeElapsed = timeElapsed;
+ this.timeUnit = timeUnit;
+ this.expirationType = expirationType;
+ }
+ public String toString() { return String.format("EXPIRED_%s{size=%d,elapsed=%d,unit=%s}", expirationType, messageSize, timeElapsed, timeUnit); }
+ }
+
+ static class FrameEvent extends SimpleEvent
+ {
+ final int messageCount;
+ final int payloadSizeInBytes;
+ FrameEvent(EventType type, long at, int messageCount, int payloadSizeInBytes)
+ {
+ super(type, at);
+ this.messageCount = messageCount;
+ this.payloadSizeInBytes = payloadSizeInBytes;
+ }
+ }
+
+ static class ProcessMessageEvent extends SimpleMessageEvent
+ {
+ final Message<?> message;
+ ProcessMessageEvent(long at, Message<?> message)
+ {
+ super(PROCESS, at, message.id());
+ this.message = message;
+ }
+ }
+
+ EnqueueMessageEvent onEnqueue(Message<?> message, Destiny destiny)
+ {
+ EnqueueMessageEvent enqueue = new EnqueueMessageEvent(ENQUEUE, nextId(), message, destiny);
+ events.put(enqueue.start, enqueue);
+ return enqueue;
+ }
+ void onOverloaded(long messageId)
+ {
+ long at = nextId();
+ events.put(at, new SimpleMessageEvent(FAILED_OVERLOADED, at, messageId));
+ }
+ void onFailedClosing(long messageId)
+ {
+ long at = nextId();
+ events.put(at, new SimpleMessageEvent(FAILED_CLOSING, at, messageId));
+ }
+ void onSerialize(long messageId, int messagingVersion)
+ {
+ long at = nextId();
+ events.put(at, new SerializeMessageEvent(SERIALIZE, at, messageId, messagingVersion));
+ }
+ void onFinishSerializeLarge(long messageId)
+ {
+ long at = nextId();
+ events.put(at, new SimpleMessageEvent(FINISH_SERIALIZE_LARGE, at, messageId));
+ }
+ void onFailedSerialize(long messageId, int bytesWrittenToNetwork, Throwable failure)
+ {
+ long at = nextId();
+ events.put(at, new FailedSerializeEvent(at, messageId, bytesWrittenToNetwork, failure));
+ }
+ void onExpiredBeforeSend(long messageId, int messageSize, long timeElapsed, TimeUnit timeUnit)
+ {
+ onExpired(messageId, messageSize, timeElapsed, timeUnit, ON_SENT);
+ }
+ void onSendFrame(int messageCount, int payloadSizeInBytes)
+ {
+ long at = nextId();
+ events.put(at, new FrameEvent(SEND_FRAME, at, messageCount, payloadSizeInBytes));
+ }
+ void onSentFrame(int messageCount, int payloadSizeInBytes)
+ {
+ long at = nextId();
+ events.put(at, new FrameEvent(SENT_FRAME, at, messageCount, payloadSizeInBytes));
+ }
+ void onFailedFrame(int messageCount, int payloadSizeInBytes)
+ {
+ long at = nextId();
+ events.put(at, new FrameEvent(FAILED_FRAME, at, messageCount, payloadSizeInBytes));
+ }
+ void onArrived(long messageId, int messageSize)
+ {
+ long at = nextId();
+ events.put(at, new SimpleMessageEventWithSize(ARRIVE, at, messageId, messageSize));
+ }
+ void onArrivedExpired(long messageId, int messageSize, boolean wasCorrupt, long timeElapsed, TimeUnit timeUnit)
+ {
+ onExpired(messageId, messageSize, timeElapsed, timeUnit, ExpirationType.ON_ARRIVED);
+ }
+ void onDeserialize(long messageId, int messagingVersion)
+ {
+ long at = nextId();
+ events.put(at, new SerializeMessageEvent(DESERIALIZE, at, messageId, messagingVersion));
+ }
+ void onClosedBeforeArrival(long messageId, int messageSize)
+ {
+ long at = nextId();
+ events.put(at, new SimpleMessageEventWithSize(CLOSED_BEFORE_ARRIVAL, at, messageId, messageSize));
+ }
+ void onFailedDeserialize(long messageId, int messageSize)
+ {
+ long at = nextId();
+ events.put(at, new SimpleMessageEventWithSize(FAILED_DESERIALIZE, at, messageId, messageSize));
+ }
+ void process(Message<?> message)
+ {
+ long at = nextId();
+ events.put(at, new ProcessMessageEvent(at, message));
+ }
+ void onProcessExpired(long messageId, int messageSize, long timeElapsed, TimeUnit timeUnit)
+ {
+ onExpired(messageId, messageSize, timeElapsed, timeUnit, ExpirationType.ON_PROCESSED);
+ }
+ private void onExpired(long messageId, int messageSize, long timeElapsed, TimeUnit timeUnit, ExpirationType expirationType)
+ {
+ long at = nextId();
+ events.put(at, new ExpiredMessageEvent(at, messageId, messageSize, timeElapsed, timeUnit, expirationType));
+ }
+
+
+
+ static class ConnectOutboundEvent extends SimpleEvent
+ {
+ final int messagingVersion;
+ final OutboundConnectionSettings settings;
+ ConnectOutboundEvent(long at, int messagingVersion, OutboundConnectionSettings settings)
+ {
+ super(EventType.CONNECT_OUTBOUND, at);
+ this.messagingVersion = messagingVersion;
+ this.settings = settings;
+ }
+ }
+
+ // TODO: do we need this?
+ static class ConnectInboundEvent extends SimpleEvent
+ {
+ final int messagingVersion;
+ final InboundMessageHandler handler;
+ ConnectInboundEvent(long at, int messagingVersion, InboundMessageHandler handler)
+ {
+ super(EventType.CONNECT_OUTBOUND, at);
+ this.messagingVersion = messagingVersion;
+ this.handler = handler;
+ }
+ }
+
+ static class SyncEvent extends SimpleEvent
+ {
+ final Runnable onCompletion;
+ SyncEvent(long at, Runnable onCompletion)
+ {
+ super(EventType.SYNC, at);
+ this.onCompletion = onCompletion;
+ }
+ }
+
+ static class ControllerEvent extends BoundedEvent
+ {
+ final long minimumBytesInFlight;
+ final long maximumBytesInFlight;
+ ControllerEvent(long start, long minimumBytesInFlight, long maximumBytesInFlight)
+ {
+ super(EventType.CONTROLLER_UPDATE, start);
+ this.minimumBytesInFlight = minimumBytesInFlight;
+ this.maximumBytesInFlight = maximumBytesInFlight;
+ }
+ }
+
+ void onSync(Runnable onCompletion)
+ {
+ SyncEvent connect = new SyncEvent(nextId(), onCompletion);
+ events.put(connect.at, connect);
+ }
+
+ void onConnectOutbound(int messagingVersion, OutboundConnectionSettings settings)
+ {
+ ConnectOutboundEvent connect = new ConnectOutboundEvent(nextId(), messagingVersion, settings);
+ events.put(connect.at, connect);
+ }
+
+ void onConnectInbound(int messagingVersion, InboundMessageHandler handler)
+ {
+ ConnectInboundEvent connect = new ConnectInboundEvent(nextId(), messagingVersion, handler);
+ events.put(connect.at, connect);
+ }
+
+ private final BytesInFlightController controller;
+ private final AtomicLong sequenceId = new AtomicLong();
+ private final EventSequence events = new EventSequence();
+ private final InboundMessageHandlers inbound;
+ private final OutboundConnection outbound;
+
+ Verifier(BytesInFlightController controller, OutboundConnection outbound, InboundMessageHandlers inbound)
+ {
+ this.controller = controller;
+ this.inbound = inbound;
+ this.outbound = outbound;
+ }
+
+ private long nextId()
+ {
+ return sequenceId.getAndIncrement();
+ }
+
+ public void logFailure(String message, Object ... params)
+ {
+ fail(message, params);
+ }
+
+ private void fail(String message, Object ... params)
+ {
+ logger.error("{}", String.format(message, params));
+ logger.error("Connection: {}", currentConnection);
+ }
+
+ private void fail(String message, Throwable t, Object ... params)
+ {
+ logger.error("{}", String.format(message, params), t);
+ logger.error("Connection: {}", currentConnection);
+ }
+
+ private void failinfo(String message, Object ... params)
+ {
+ logger.error("{}", String.format(message, params));
+ }
+
+ private static class MessageState
+ {
+ final Message<?> message;
+ final Destiny destiny;
+ int messagingVersion;
+ // set initially to message.expiresAtNanos, but if at serialization time we use
+ // an older messaging version we may not be able to serialize expiration
+ long expiresAtNanos;
+ long enqueueStart, enqueueEnd, serialize, arrive, deserialize;
+ boolean processOnEventLoop, processOutOfOrder;
+ Event sendState, receiveState;
+ long lastUpdateAt;
+ long lastUpdateNanos;
+ ConnectionState sentOn;
+ boolean doneSend, doneReceive;
+
+ int messageSize()
+ {
+ return message.serializedSize(messagingVersion);
+ }
+
+ MessageState(Message<?> message, Destiny destiny, long enqueueStart)
+ {
+ this.message = message;
+ this.destiny = destiny;
+ this.enqueueStart = enqueueStart;
+ this.expiresAtNanos = message.expiresAtNanos();
+ }
+
+ void update(SimpleEvent event, long now)
+ {
+ update(event, event.at, now);
+ }
+ void update(Event event, long at, long now)
+ {
+ lastUpdateAt = at;
+ lastUpdateNanos = now;
+ switch (event.type.category)
+ {
+ case SEND:
+ sendState = event;
+ break;
+ case RECEIVE:
+ receiveState = event;
+ break;
+ default: throw new IllegalStateException();
+ }
+ }
+
+ boolean is(EventType type)
+ {
+ switch (type.category)
+ {
+ case SEND: return sendState != null && sendState.type == type;
+ case RECEIVE: return receiveState != null && receiveState.type == type;
+ default: return false;
+ }
+ }
+
+ boolean is(EventType type1, EventType type2)
+ {
+ return is(type1) || is(type2);
+ }
+
+ boolean is(EventType type1, EventType type2, EventType type3)
+ {
+ return is(type1) || is(type2) || is(type3);
+ }
+
+ void require(EventType event, Verifier verifier, EventType type)
+ {
+ if (!is(type))
+ verifier.fail("Invalid state at %s for %s: expected %s", event, this, type);
+ }
+
+ void require(EventType event, Verifier verifier, EventType type1, EventType type2)
+ {
+ if (!is(type1) && !is(type2))
+ verifier.fail("Invalid state at %s for %s: expected %s or %s", event, this, type1, type2);
+ }
+
+ void require(EventType event, Verifier verifier, EventType type1, EventType type2, EventType type3)
+ {
+ if (!is(type1) && !is(type2) && !is(type3))
+ verifier.fail("Invalid state %s for %s: expected %s, %s or %s", event, this, type1, type2, type3);
+ }
+
+ public String toString()
+ {
+ return String.format("{id:%d, state:[%s,%s], upd:%d, ver:%d, enqueue:[%d,%d], ser:%d, arr:%d, deser:%d, expires:%d, sentOn: %d}",
+ message.id(), sendState, receiveState, lastUpdateAt, messagingVersion, enqueueStart, enqueueEnd, serialize, arrive, deserialize, approxTime.translate().toMillisSinceEpoch(expiresAtNanos), sentOn == null ? -1 : sentOn.connectionId);
+ }
+ }
+
+ private final LongObjectHashMap<MessageState> messages = new LongObjectHashMap<>();
+
+ // messages start here, but may enter in a haphazard (non-sequential) fashion;
+ // ENQUEUE_START, ENQUEUE_END both take place here, with the latter imposing bounds on the out-of-order appearance of messages.
+ // note that ENQUEUE_END - being concurrent - may not appear before the message's lifespan has completely ended.
+ private final Queue<MessageState> enqueueing = new Queue<>();
+
+ static final class ConnectionState
+ {
+ final long connectionId;
+ final int messagingVersion;
+ // Strict message order will then be determined at serialization time, since this happens on a single thread.
+ // The order in which messages arrive here determines the order they will arrive on the other node.
+ // must follow either ENQUEUE_START or ENQUEUE_END
+ final Queue<MessageState> serializing = new Queue<>();
+
+ // Messages sent on the small connection will all be sent in frames; this is a concurrent operation,
+ // so only the sendingFrame MUST be encountered before any future events -
+ // large connections skip this step and goes straight to arriving
+ // we consult the queues in reverse order in arriving, as it is acceptable to find our frame in any of these queues
+ final FramesInFlight framesInFlight = new FramesInFlight(); // unknown if the messages will arrive, accept either
+
+ // for large messages OR < VERSION_40, arriving can occur BEFORE serializing completes successfully
+ // OR a frame is fully serialized
+ final Queue<MessageState> arriving = new Queue<>();
+
+ final Queue<MessageState> deserializingOnEventLoop = new Queue<>(),
+ deserializingOffEventLoop = new Queue<>();
+
+ InboundMessageHandler inbound;
+
+ // TODO
+ long sentCount, sentBytes;
+ long receivedCount, receivedBytes;
+
+ ConnectionState(long connectionId, int messagingVersion)
+ {
+ this.connectionId = connectionId;
+ this.messagingVersion = messagingVersion;
+ }
+
+ public String toString()
+ {
+ return String.format("{id: %d, ver: %d, ser: %d, inFlight: %s, arriving: %d, deserOn: %d, deserOff: %d}",
+ connectionId, messagingVersion, serializing.size(), framesInFlight, arriving.size(), deserializingOnEventLoop.size(), deserializingOffEventLoop.size());
+ }
+ }
+
+ private final Queue<MessageState> processingOutOfOrder = new Queue<>();
+
+ private SyncEvent sync;
+ private long nextMessageId = 0;
+ private long now;
+ private long connectionCounter;
+ private ConnectionState currentConnection = new ConnectionState(connectionCounter++, current_version);
+
+ private long outboundSentCount, outboundSentBytes;
+ private long outboundSubmittedCount;
+ private long outboundOverloadedCount, outboundOverloadedBytes;
+ private long outboundExpiredCount, outboundExpiredBytes;
+ private long outboundErrorCount, outboundErrorBytes;
+
+ public void run(Runnable onFailure, long deadlineNanos)
+ {
+ try
+ {
+ long lastEventAt = approxTime.now();
+ while ((now = approxTime.now()) < deadlineNanos)
+ {
+ Event next = events.await(nextMessageId, 100L, MILLISECONDS);
+ if (next == null)
+ {
+ // decide if we have any messages waiting too long to proceed
+ while (!processingOutOfOrder.isEmpty())
+ {
+ MessageState m = processingOutOfOrder.get(0);
+ if (now - m.lastUpdateNanos > SECONDS.toNanos(10L))
+ {
+ fail("Unreasonably long period spent waiting for out-of-order deser/delivery of received message %d", m.message.id());
+ MessageState v = maybeRemove(m.message.id(), PROCESS);
+ controller.fail(v.message.serializedSize(v.messagingVersion == 0 ? current_version : v.messagingVersion));
+ processingOutOfOrder.remove(0);
+ }
+ else break;
+ }
+
+ if (sync != null)
+ {
+ // if we have waited 100ms since beginning a sync, with no events, and ANY of our queues are
+ // non-empty, something is probably wrong; however, let's give ourselves a little bit longer
+
+ boolean done =
+ currentConnection.serializing.isEmpty()
+ && currentConnection.arriving.isEmpty()
+ && currentConnection.deserializingOnEventLoop.isEmpty()
+ && currentConnection.deserializingOffEventLoop.isEmpty()
+ && currentConnection.framesInFlight.isEmpty()
+ && enqueueing.isEmpty()
+ && processingOutOfOrder.isEmpty()
+ && messages.isEmpty()
+ && controller.inFlight() == 0;
+
+ //outbound.pendingCount() > 0 ? 5L : 2L
+ if (!done && now - lastEventAt > SECONDS.toNanos(5L))
+ {
+ // TODO: even 2s or 5s are unreasonable periods of time without _any_ movement on a message waiting to arrive
+ // this seems to happen regularly on MacOS, but we should confirm this does not happen on Linux
+ fail("Unreasonably long period spent waiting for sync (%dms)", NANOSECONDS.toMillis(now - lastEventAt));
+ messages.<LongObjectProcedure<MessageState>>forEach((k, v) -> {
+ failinfo("%s", v);
+ controller.fail(v.message.serializedSize(v.messagingVersion == 0 ? current_version : v.messagingVersion));
+ });
+ currentConnection.serializing.clear();
+ currentConnection.arriving.clear();
+ currentConnection.deserializingOnEventLoop.clear();
+ currentConnection.deserializingOffEventLoop.clear();
+ enqueueing.clear();
+ processingOutOfOrder.clear();
+ messages.clear();
+ while (!currentConnection.framesInFlight.isEmpty())
+ currentConnection.framesInFlight.poll();
+ done = true;
+ }
+
+ if (done)
+ {
+ ConnectionUtils.check(outbound)
+ .pending(0, 0)
+ .error(outboundErrorCount, outboundErrorBytes)
+ .submitted(outboundSubmittedCount)
+ .expired(outboundExpiredCount, outboundExpiredBytes)
+ .overload(outboundOverloadedCount, outboundOverloadedBytes)
+ .sent(outboundSentCount, outboundSentBytes)
+ .check((message, expect, actual) -> fail("%s: expect %d, actual %d", message, expect, actual));
+
+ sync.onCompletion.run();
+ sync = null;
+ }
+ }
+ continue;
+ }
+ events.clear(nextMessageId); // TODO: simplify collection if we end up using it exclusively as a queue, as we are now
+ lastEventAt = now;
+
+ switch (next.type)
+ {
+ case ENQUEUE:
+ {
+ MessageState m;
+ EnqueueMessageEvent e = (EnqueueMessageEvent) next;
+ assert nextMessageId == e.start || nextMessageId == e.end;
+ assert e.message != null;
+ if (nextMessageId == e.start)
+ {
+ if (sync != null)
+ fail("Sync in progress - there should be no messages beginning to enqueue");
+
+ m = new MessageState(e.message, e.destiny, e.start);
+ messages.put(e.messageId, m);
+ enqueueing.add(m);
+ m.update(e, e.start, now);
+ }
+ else
+ {
+ // warning: enqueueEnd can occur at any time in the future, since it's a different thread;
+ // it could be arbitrarily paused, long enough even for the messsage to be fully processed
+ m = messages.get(e.messageId);
+ if (m != null)
+ m.enqueueEnd = e.end;
+ outboundSubmittedCount += 1;
+ }
+ break;
+ }
+ case FAILED_OVERLOADED:
+ {
+ // TODO: verify that we could have exceeded our memory limits
+ SimpleMessageEvent e = (SimpleMessageEvent) next;
+ assert nextMessageId == e.at;
+ MessageState m = remove(e.messageId, enqueueing, messages);
+ m.require(FAILED_OVERLOADED, this, ENQUEUE);
+ outboundOverloadedBytes += m.message.serializedSize(current_version);
+ outboundOverloadedCount += 1;
+ break;
+ }
+ case FAILED_CLOSING:
+ {
+ // TODO: verify if this is acceptable due to e.g. inbound refusing to process for long enough
+ SimpleMessageEvent e = (SimpleMessageEvent) next;
+ assert nextMessageId == e.at;
+ MessageState m = messages.remove(e.messageId); // definitely cannot have been sent (in theory)
+ enqueueing.remove(m);
+ m.require(FAILED_CLOSING, this, ENQUEUE);
+ fail("Invalid discard of %d: connection was closing for too long", m.message.id());
+ break;
+ }
+ case SERIALIZE:
+ {
+ // serialize happens serially, so we can compress the asynchronicity of the above enqueue
+ // into a linear sequence of events we expect to occur on arrival
+ SerializeMessageEvent e = (SerializeMessageEvent) next;
+ assert nextMessageId == e.at;
+ MessageState m = get(e);
+ assert m.is(ENQUEUE);
+ m.serialize = e.at;
+ m.messagingVersion = e.messagingVersion;
+ if (current_version != e.messagingVersion)
+ controller.adjust(m.message.serializedSize(current_version), m.message.serializedSize(e.messagingVersion));
+
+ m.processOnEventLoop = willProcessOnEventLoop(outbound.type(), m.message, e.messagingVersion);
+ m.expiresAtNanos = expiresAtNanos(m.message, e.messagingVersion);
+ int mi = enqueueing.indexOf(m);
+ for (int i = 0 ; i < mi ; ++i)
+ {
+ MessageState pm = enqueueing.get(i);
+ if (pm.enqueueEnd != 0 && pm.enqueueEnd < m.enqueueStart)
+ {
+ fail("Invalid order of events: %s enqueued strictly before %s, but serialized after",
+ pm, m);
+ }
+ }
+ enqueueing.remove(mi);
+ m.sentOn = currentConnection;
+ currentConnection.serializing.add(m);
+ m.update(e, now);
+ break;
+ }
+ case FINISH_SERIALIZE_LARGE:
+ {
+ // serialize happens serially, so we can compress the asynchronicity of the above enqueue
+ // into a linear sequence of events we expect to occur on arrival
+ SimpleMessageEvent e = (SimpleMessageEvent) next;
+ assert nextMessageId == e.at;
+ MessageState m = maybeRemove(e);
+ outboundSentBytes += m.messageSize();
+ outboundSentCount += 1;
+ m.sentOn.serializing.remove(m);
+ m.update(e, now);
+ break;
+ }
+ case FAILED_SERIALIZE:
+ {
+ FailedSerializeEvent e = (FailedSerializeEvent) next;
+ assert nextMessageId == e.at;
+ MessageState m = maybeRemove(e);
+
+ if (outbound.type() == LARGE_MESSAGES)
+ assert e.failure instanceof InvalidSerializedSizeException || e.failure instanceof Connection.IntentionalIOException || e.failure instanceof Connection.IntentionalRuntimeException;
+ else
+ assert e.failure instanceof InvalidSerializedSizeException || e.failure instanceof Connection.IntentionalIOException || e.failure instanceof Connection.IntentionalRuntimeException || e.failure instanceof BufferOverflowException;
+
+ if (e.bytesWrittenToNetwork == 0) // TODO: use header size
+ messages.remove(m.message.id());
+
+ InvalidSerializedSizeException ex;
+ if (outbound.type() != LARGE_MESSAGES
+ || !(e.failure instanceof InvalidSerializedSizeException)
+ || ((ex = (InvalidSerializedSizeException) e.failure).expectedSize <= DEFAULT_BUFFER_SIZE && ex.actualSizeAtLeast <= DEFAULT_BUFFER_SIZE)
+ || (ex.expectedSize > DEFAULT_BUFFER_SIZE && ex.actualSizeAtLeast < DEFAULT_BUFFER_SIZE))
+ {
+ assert e.bytesWrittenToNetwork == 0;
+ }
+
+ m.require(FAILED_SERIALIZE, this, SERIALIZE);
+ m.sentOn.serializing.remove(m);
+ if (m.destiny != Destiny.FAIL_TO_SERIALIZE)
+ fail("%s failed to serialize, but its destiny was to %s", m, m.destiny);
+ outboundErrorBytes += m.messageSize();
+ outboundErrorCount += 1;
+ m.update(e, now);
+ break;
+ }
+ case SEND_FRAME:
+ {
+ FrameEvent e = (FrameEvent) next;
+ assert nextMessageId == e.at;
+ int size = 0;
+ Frame frame = new Frame();
+ MessageState first = currentConnection.serializing.get(0);
+ int messagingVersion = first.messagingVersion;
+ for (int i = 0 ; i < e.messageCount ; ++i)
+ {
+ MessageState m = currentConnection.serializing.get(i);
+ size += m.message.serializedSize(m.messagingVersion);
+ if (m.messagingVersion != messagingVersion)
+ {
+ fail("Invalid sequence of events: %s encoded to same frame as %s",
+ m, first);
+ }
+
+ frame.add(m);
+ m.update(e, now);
+ assert !m.doneSend;
+ m.doneSend = true;
+ if (m.doneReceive)
+ messages.remove(m.message.id());
+ }
+ frame.payloadSizeInBytes = e.payloadSizeInBytes;
+ frame.messageCount = e.messageCount;
+ frame.messagingVersion = messagingVersion;
+ currentConnection.framesInFlight.add(frame);
+ currentConnection.serializing.removeFirst(e.messageCount);
+ if (e.payloadSizeInBytes != size)
+ fail("Invalid frame payload size with %s: expected %d, actual %d", first, size, e.payloadSizeInBytes);
+ break;
+ }
+ case SENT_FRAME:
+ {
+ Frame frame = currentConnection.framesInFlight.supplySendStatus(Frame.Status.SUCCESS);
+ frame.forEach(m -> m.update((SimpleEvent) next, now));
+
+ outboundSentBytes += frame.payloadSizeInBytes;
+ outboundSentCount += frame.messageCount;
+ break;
+ }
+ case FAILED_FRAME:
+ {
+ // TODO: is it possible for this to be signalled AFTER our reconnect event? probably, in which case this will be wrong
+ // TODO: verify that this was expected
+ Frame frame = currentConnection.framesInFlight.supplySendStatus(Frame.Status.FAILED);
+ frame.forEach(m -> m.update((SimpleEvent) next, now));
+ if (frame.messagingVersion >= VERSION_40)
+ {
+ // the contents cannot be delivered without the whole frame arriving, so clear the contents now
+ clear(frame, messages);
+ currentConnection.framesInFlight.remove(frame);
+ }
+ outboundErrorBytes += frame.payloadSizeInBytes;
+ outboundErrorCount += frame.messageCount;
+ break;
+ }
+ case ARRIVE:
+ {
+ SimpleMessageEventWithSize e = (SimpleMessageEventWithSize) next;
+ assert nextMessageId == e.at;
+ MessageState m = get(e);
+
+ m.arrive = e.at;
+ if (e.messageSize != m.messageSize())
+ fail("onArrived with invalid size for %s: %d vs %d", m, e.messageSize, m.messageSize());
+
+ if (outbound.type() == LARGE_MESSAGES)
+ {
+ m.require(ARRIVE, this, SERIALIZE, FAILED_SERIALIZE, FINISH_SERIALIZE_LARGE);
+ }
+ else
+ {
+ if (!m.is(SEND_FRAME, SENT_FRAME))
+ {
+ fail("Invalid order of events: %s arrived before being sent in a frame", m);
+ break;
+ }
+
+ int fi = -1, mi = -1;
+ while (fi + 1 < m.sentOn.framesInFlight.size() && mi < 0)
+ mi = m.sentOn.framesInFlight.get(++fi).indexOf(m);
+
+ if (fi == m.sentOn.framesInFlight.size())
+ {
+ fail("Invalid state: %s, but no frame in flight was found to contain it", m);
+ break;
+ }
+
+ if (fi > 0)
+ {
+ // we have skipped over some frames, meaning these have either failed (and we know it)
+ // or we have not yet heard about them and they have presumably failed, or something
+ // has gone wrong
+ fail("BEGIN: Successfully sent frames were not delivered");
+ for (int i = 0 ; i < fi ; ++i)
+ {
+ Frame skip = m.sentOn.framesInFlight.get(i);
+ skip.receiveStatus = Frame.Status.FAILED;
+ if (skip.sendStatus == Frame.Status.SUCCESS)
+ {
+ failinfo("Frame %s", skip);
+ for (int j = 0 ; j < skip.size() ; ++j)
+ failinfo("Containing: %s", skip.get(j));
+ }
+ clear(skip, messages);
+ }
+ m.sentOn.framesInFlight.removeFirst(fi);
+ failinfo("END: Successfully sent frames were not delivered");
+ }
+
+ Frame frame = m.sentOn.framesInFlight.get(0);
+ for (int i = 0; i < mi; ++i)
+ fail("Invalid order of events: %s serialized strictly before %s, but arrived after", frame.get(i), m);
+
+ frame.remove(mi);
+ if (frame.isEmpty())
+ m.sentOn.framesInFlight.poll();
+ }
+ m.sentOn.arriving.add(m);
+ m.update(e, now);
+ break;
+ }
+ case DESERIALIZE:
+ {
+ // deserialize may happen in parallel for large messages, but in sequence for small messages
+ // we currently require that this event be issued before any possible error is thrown
+ SimpleMessageEvent e = (SimpleMessageEvent) next;
+ assert nextMessageId == e.at;
+ MessageState m = get(e);
+ m.require(DESERIALIZE, this, ARRIVE);
+ m.deserialize = e.at;
+ // deserialize may be off-loaded, so we can only impose meaningful ordering constraints
+ // on those messages we know to have been processed on the event loop
+ int mi = m.sentOn.arriving.indexOf(m);
+ if (m.processOnEventLoop)
+ {
+ for (int i = 0 ; i < mi ; ++i)
+ {
+ MessageState pm = m.sentOn.arriving.get(i);
+ if (pm.processOnEventLoop)
+ {
+ fail("Invalid order of events: %d (%d, %d) arrived strictly before %d (%d, %d), but deserialized after",
+ pm.message.id(), pm.arrive, pm.deserialize, m.message.id(), m.arrive, m.deserialize);
+ }
+ }
+ m.sentOn.deserializingOnEventLoop.add(m);
+ }
+ else
+ {
+ m.sentOn.deserializingOffEventLoop.add(m);
+ }
+ m.sentOn.arriving.remove(mi);
+ m.update(e, now);
+ break;
+ }
+ case CLOSED_BEFORE_ARRIVAL:
+ {
+ SimpleMessageEventWithSize e = (SimpleMessageEventWithSize) next;
+ assert nextMessageId == e.at;
+ MessageState m = maybeRemove(e);
+
+ if (e.messageSize != m.messageSize())
+ fail("onClosedBeforeArrival has invalid size for %s: %d vs %d", m, e.messageSize, m.messageSize());
+
+ m.sentOn.deserializingOffEventLoop.remove(m);
+ if (m.destiny == Destiny.FAIL_TO_SERIALIZE && outbound.type() == LARGE_MESSAGES)
+ break;
+ fail("%s closed before arrival, but its destiny was to %s", m, m.destiny);
+ break;
+ }
+ case FAILED_DESERIALIZE:
+ {
+ SimpleMessageEventWithSize e = (SimpleMessageEventWithSize) next;
+ assert nextMessageId == e.at;
+ MessageState m = maybeRemove(e);
+
+ if (e.messageSize != m.messageSize())
+ fail("onFailedDeserialize has invalid size for %s: %d vs %d", m, e.messageSize, m.messageSize());
+ m.require(FAILED_DESERIALIZE, this, ARRIVE, DESERIALIZE);
+ (m.processOnEventLoop ? m.sentOn.deserializingOnEventLoop : m.sentOn.deserializingOffEventLoop).remove(m);
+ switch (m.destiny)
+ {
+ case FAIL_TO_DESERIALIZE:
+ break;
+ case FAIL_TO_SERIALIZE:
+ if (outbound.type() == LARGE_MESSAGES)
+ break;
+ default:
+ fail("%s failed to deserialize, but its destiny was to %s", m, m.destiny);
+ }
+ break;
+ }
+ case PROCESS:
+ {
+ ProcessMessageEvent e = (ProcessMessageEvent) next;
+ assert nextMessageId == e.at;
+ MessageState m = maybeRemove(e);
+
+ m.require(PROCESS, this, DESERIALIZE);
+ if (!Arrays.equals((byte[]) e.message.payload, (byte[]) m.message.payload))
+ {
+ fail("Invalid message payload for %d: %s supplied by processor, but %s implied by original message and messaging version",
+ e.messageId, Arrays.toString((byte[]) e.message.payload), Arrays.toString((byte[]) m.message.payload));
+ }
+
+ if (m.processOutOfOrder)
+ {
+ assert !m.processOnEventLoop; // will have already been reported small (processOnEventLoop) messages
+ processingOutOfOrder.remove(m);
+ }
+ else if (m.processOnEventLoop)
+ {
+ // we can expect that processing happens sequentially in this case, more specifically
+ // we can actually expect that this event will occur _immediately_ after the deserialize event
+ // so that we have exactly one mess
+ // c
+ int mi = m.sentOn.deserializingOnEventLoop.indexOf(m);
+ for (int i = 0 ; i < mi ; ++i)
+ {
+ MessageState pm = m.sentOn.deserializingOnEventLoop.get(i);
+ fail("Invalid order of events: %s deserialized strictly before %s, but processed after",
+ pm, m);
+ }
+ clearFirst(mi, m.sentOn.deserializingOnEventLoop, messages);
+ m.sentOn.deserializingOnEventLoop.poll();
+ }
+ else
+ {
+ int mi = m.sentOn.deserializingOffEventLoop.indexOf(m);
+ // process may be off-loaded, so we can only impose meaningful ordering constraints
+ // on those messages we know to have been processed on the event loop
+ for (int i = 0 ; i < mi ; ++i)
+ {
+ MessageState pm = m.sentOn.deserializingOffEventLoop.get(i);
+ pm.processOutOfOrder = true;
+ processingOutOfOrder.add(pm);
+ }
+ m.sentOn.deserializingOffEventLoop.removeFirst(mi + 1);
+ }
+ // this message has been fully validated
+ break;
+ }
+ case FAILED_EXPIRED_ON_SEND:
+ case FAILED_EXPIRED_ON_RECEIVE:
+ {
+ ExpiredMessageEvent e = (ExpiredMessageEvent) next;
+ assert nextMessageId == e.at;
+ MessageState m;
+ switch (e.expirationType)
+ {
+ case ON_SENT:
+ {
+ m = messages.remove(e.messageId);
+ m.require(e.type, this, ENQUEUE);
+ outboundExpiredBytes += m.message.serializedSize(current_version);
+ outboundExpiredCount += 1;
+ messages.remove(m.message.id());
+ break;
+ }
+ case ON_ARRIVED:
+ m = maybeRemove(e);
+ if (!m.is(ARRIVE))
+ {
+ if (outbound.type() != LARGE_MESSAGES) m.require(e.type, this, SEND_FRAME, SENT_FRAME, FAILED_FRAME);
+ else m.require(e.type, this, SERIALIZE, FAILED_SERIALIZE, FINISH_SERIALIZE_LARGE);
+ }
+ break;
+ case ON_PROCESSED:
+ m = maybeRemove(e);
+ m.require(e.type, this, DESERIALIZE);
+ break;
+ default:
+ throw new IllegalStateException();
+ }
+
+ now = System.nanoTime();
+ if (m.expiresAtNanos > now)
+ {
+ // we fix the conversion AlmostSameTime for an entire run, which should suffice to guarantee these comparisons
+ fail("Invalid expiry of %d: expiry should occur in %dms; event believes %dms have elapsed, and %dms have actually elapsed", m.message.id(),
+ NANOSECONDS.toMillis(m.expiresAtNanos - m.message.createdAtNanos()),
+ e.timeUnit.toMillis(e.timeElapsed),
+ NANOSECONDS.toMillis(now - m.message.createdAtNanos()));
+ }
+
+ switch (e.expirationType)
+ {
+ case ON_SENT:
+ enqueueing.remove(m);
+ break;
+ case ON_ARRIVED:
+ if (m.is(ARRIVE))
+ m.sentOn.arriving.remove(m);
+ switch (m.sendState.type)
+ {
+ case SEND_FRAME:
+ case SENT_FRAME:
+ case FAILED_FRAME:
+ // TODO: this should be robust to re-ordering; should perhaps extract a common method
+ m.sentOn.framesInFlight.get(0).remove(m);
+ if (m.sentOn.framesInFlight.get(0).isEmpty())
+ m.sentOn.framesInFlight.poll();
+ break;
+ }
+ break;
+ case ON_PROCESSED:
+ (m.processOnEventLoop ? m.sentOn.deserializingOnEventLoop : m.sentOn.deserializingOffEventLoop).remove(m);
+ break;
+ }
+
+ if (m.messagingVersion != 0 && e.messageSize != m.messageSize())
+ fail("onExpired %s with invalid size for %s: %d vs %d", e.expirationType, m, e.messageSize, m.messageSize());
+
+ break;
+ }
+ case CONTROLLER_UPDATE:
+ {
+ break;
+ }
+ case CONNECT_OUTBOUND:
+ {
+ ConnectOutboundEvent e = (ConnectOutboundEvent) next;
+ currentConnection = new ConnectionState(connectionCounter++, e.messagingVersion);
+ break;
+ }
+ case SYNC:
+ {
+ sync = (SyncEvent) next;
+ break;
+ }
+ default:
+ throw new IllegalStateException();
+ }
+ ++nextMessageId;
+ }
+ }
+ catch (InterruptedException e)
+ {
+ }
+ catch (Throwable t)
+ {
+ logger.error("Unexpected error:", t);
+ onFailure.run();
+ }
+ }
+
+ private MessageState get(SimpleMessageEvent onEvent)
+ {
+ MessageState m = messages.get(onEvent.messageId);
+ if (m == null)
+ throw new IllegalStateException("Missing " + onEvent + ": " + onEvent.messageId);
+ return m;
+ }
+ private MessageState maybeRemove(SimpleMessageEvent onEvent)
+ {
+ return maybeRemove(onEvent.messageId, onEvent.type, onEvent);
+ }
+ private MessageState maybeRemove(long messageId, EventType onEvent)
+ {
+ return maybeRemove(messageId, onEvent, onEvent);
+ }
+ private MessageState maybeRemove(long messageId, EventType onEvent, Object id)
+ {
+ MessageState m = messages.get(messageId);
+ if (m == null)
+ throw new IllegalStateException("Missing " + id + ": " + messageId);
+ switch (onEvent.category)
+ {
+ case SEND:
+ if (m.doneSend)
+ fail("%s already doneSend %s", onEvent, m);
+ m.doneSend = true;
+ if (m.doneReceive) messages.remove(messageId);
+ break;
+ case RECEIVE:
+ if (m.doneReceive)
+ fail("%s already doneReceive %s", onEvent, m);
+ m.doneReceive = true;
+ if (m.doneSend) messages.remove(messageId);
+ }
+ return m;
+ }
+
+
+ private static class Frame extends Queue<MessageState>
+ {
+ enum Status { SUCCESS, FAILED, UNKNOWN }
+ Status sendStatus = Status.UNKNOWN, receiveStatus = Status.UNKNOWN;
+ int messagingVersion;
+ int messageCount;
+ int payloadSizeInBytes;
+
+ public String toString()
+ {
+ return String.format("{count:%d, size:%d, version:%d, send:%s, receive:%s}",
+ messageCount, payloadSizeInBytes, messagingVersion, sendStatus, receiveStatus);
+ }
+ }
+
+ private static MessageState remove(long messageId, Queue<MessageState> queue, LongObjectHashMap<MessageState> lookup)
+ {
+ MessageState m = lookup.remove(messageId);
+ queue.remove(m);
+ return m;
+ }
+
+ private static void clearFirst(int count, Queue<MessageState> queue, LongObjectHashMap<MessageState> lookup)
+ {
+ if (count > 0)
+ {
+ for (int i = 0 ; i < count ; ++i)
+ lookup.remove(queue.get(i).message.id());
+ queue.removeFirst(count);
+ }
+ }
+
+ private static void clear(Queue<MessageState> queue, LongObjectHashMap<MessageState> lookup)
+ {
+ if (!queue.isEmpty())
+ clearFirst(queue.size(), queue, lookup);
+ }
+
+ private static class EventSequence
+ {
+ static final int CHUNK_SIZE = 1 << 10;
+ static class Chunk extends AtomicReferenceArray<Event>
+ {
+ final long sequenceId;
+ int removed = 0;
+ Chunk(long sequenceId)
+ {
+ super(CHUNK_SIZE);
+ this.sequenceId = sequenceId;
+ }
+ Event get(long sequenceId)
+ {
+ return get((int)(sequenceId - this.sequenceId));
+ }
+ void set(long sequenceId, Event event)
+ {
+ lazySet((int)(sequenceId - this.sequenceId), event);
+ }
+ }
+
+ // we use a concurrent skip list to permit efficient searching, even if we always append
+ final ConcurrentSkipListMap<Long, Chunk> chunkList = new ConcurrentSkipListMap<>();
+ final WaitQueue writerWaiting = new WaitQueue();
+
+ volatile Chunk writerChunk = new Chunk(0);
+ Chunk readerChunk = writerChunk;
+
+ long readerWaitingFor;
+ volatile Thread readerWaiting;
+
+ EventSequence()
+ {
+ chunkList.put(0L, writerChunk);
+ }
+
+ public void put(long sequenceId, Event event)
+ {
+ long chunkSequenceId = sequenceId & -CHUNK_SIZE;
+ Chunk chunk = writerChunk;
+ if (chunk.sequenceId != chunkSequenceId)
+ {
+ try
+ {
+ chunk = ensureChunk(chunkSequenceId);
+ }
+ catch (InterruptedException e)
+ {
+ throw new RuntimeException(e);
+ }
+ }
+
+ chunk.set(sequenceId, event);
+
+ Thread wake = readerWaiting;
+ long wakeIf = readerWaitingFor; // we are guarded by the above volatile read
+ if (wake != null && wakeIf == sequenceId)
+ LockSupport.unpark(wake);
+ }
+
+ Chunk ensureChunk(long chunkSequenceId) throws InterruptedException
+ {
+ Chunk chunk = chunkList.get(chunkSequenceId);
+ if (chunk == null)
+ {
+ Map.Entry<Long, Chunk> e;
+ while ( null != (e = chunkList.firstEntry()) && chunkSequenceId - e.getKey() > 1 << 12)
+ {
+ WaitQueue.Signal signal = writerWaiting.register();
+ if (null != (e = chunkList.firstEntry()) && chunkSequenceId - e.getKey() > 1 << 12)
+ signal.await();
+ else
+ signal.cancel();
+ }
+ chunk = chunkList.get(chunkSequenceId);
+ if (chunk == null)
+ {
+ synchronized (this)
+ {
+ chunk = chunkList.get(chunkSequenceId);
+ if (chunk == null)
+ chunkList.put(chunkSequenceId, chunk = new Chunk(chunkSequenceId));
+ }
+ }
+ }
+ return chunk;
+ }
+
+ Chunk readerChunk(long readerId) throws InterruptedException
+ {
+ long chunkSequenceId = readerId & -CHUNK_SIZE;
+ if (readerChunk.sequenceId != chunkSequenceId)
+ readerChunk = ensureChunk(chunkSequenceId);
+ return readerChunk;
+ }
+
+ public Event await(long id, long timeout, TimeUnit unit) throws InterruptedException
+ {
+ return await(id, System.nanoTime() + unit.toNanos(timeout));
+ }
+
+ public Event await(long id, long deadlineNanos) throws InterruptedException
+ {
+ Chunk chunk = readerChunk(id);
+ Event result = chunk.get(id);
+ if (result != null)
+ return result;
+
+ readerWaitingFor = id;
+ readerWaiting = Thread.currentThread();
+ while (null == (result = chunk.get(id)))
+ {
+ long waitNanos = deadlineNanos - System.nanoTime();
+ if (waitNanos <= 0)
+ return null;
+ LockSupport.parkNanos(waitNanos);
+ if (Thread.interrupted())
+ throw new InterruptedException();
+ }
+ readerWaitingFor = -1;
+ readerWaiting = null;
+ return result;
+ }
+
+ public Event find(long sequenceId)
+ {
+ long chunkSequenceId = sequenceId & -CHUNK_SIZE;
+ Chunk chunk = readerChunk;
+ if (chunk.sequenceId != chunkSequenceId)
+ {
+ chunk = writerChunk;
+ if (chunk.sequenceId != chunkSequenceId)
+ chunk = chunkList.get(chunkSequenceId);
+ }
+ return chunk.get(sequenceId);
+ }
+
+ public void clear(long sequenceId)
+ {
+ long chunkSequenceId = sequenceId & -CHUNK_SIZE;
+ Chunk chunk = chunkList.get(chunkSequenceId);
+ chunk.set(sequenceId, null);
+ if (++chunk.removed == CHUNK_SIZE)
+ {
+ chunkList.remove(chunkSequenceId);
+ writerWaiting.signalAll();
+ }
+ }
+ }
+
+ static class Queue<T>
+ {
+ private Object[] items = new Object[10];
+ private int begin, end;
+
+ int size()
+ {
+ return end - begin;
+ }
+
+ T get(int i)
+ {
+ //noinspection unchecked
+ return (T) items[i + begin];
+ }
+
+ int indexOf(T item)
+ {
+ for (int i = begin ; i < end ; ++i)
+ {
+ if (item == items[i])
+ return i - begin;
+ }
+ return -1;
+ }
+
+ void remove(T item)
+ {
+ int i = indexOf(item);
+ if (i >= 0)
+ remove(i);
+ }
+
+ void remove(int i)
+ {
+ i += begin;
+ assert i < end;
+
+ if (i == begin || i + 1 == end)
+ {
+ items[i] = null;
+ if (begin + 1 == end) begin = end = 0;
+ else if (i == begin) ++begin;
+ else --end;
+ }
+ else if (i - begin < end - i)
+ {
+ System.arraycopy(items, begin, items, begin + 1, i - begin);
+ items[begin++] = null;
+ }
+ else
+ {
+ System.arraycopy(items, i + 1, items, i, (end - 1) - i);
+ items[--end] = null;
+ }
+ }
+
+ void add(T item)
+ {
+ if (end == items.length)
+ {
+ Object[] src = items;
+ Object[] trg;
+ if (end - begin < src.length / 2) trg = src;
+ else trg = new Object[src.length * 2];
+ System.arraycopy(src, begin, trg, 0, end - begin);
+ end -= begin;
+ begin = 0;
+ items = trg;
+ }
+ items[end++] = item;
+ }
+
+ void clear()
+ {
+ Arrays.fill(items, begin, end, null);
+ begin = end = 0;
+ }
+
+ void removeFirst(int count)
+ {
+ Arrays.fill(items, begin, begin + count, null);
+ begin += count;
+ if (begin == end)
+ begin = end = 0;
+ }
+
+ T poll()
+ {
+ if (begin == end)
+ return null;
+ //noinspection unchecked
+ T result = (T) items[begin];
+ items[begin++] = null;
+ if (begin == end)
+ begin = end = 0;
+ return result;
+ }
+
+ void forEach(Consumer<T> consumer)
+ {
+ for (int i = 0 ; i < size() ; ++i)
+ consumer.accept(get(i));
+ }
+
+ boolean isEmpty()
+ {
+ return begin == end;
+ }
+
+ public String toString()
+ {
+ StringBuilder result = new StringBuilder();
+ result.append('[');
+ toString(result);
+ result.append(']');
+ return result.toString();
+ }
+
+ void toString(StringBuilder out)
+ {
+ for (int i = 0 ; i < size() ; ++i)
+ {
+ if (i > 0) out.append(", ");
+ out.append(get(i));
+ }
+ }
+ }
+
+
+
+ static class FramesInFlight
+ {
+ // this may be negative, indicating we have processed a frame whose status we did not know at the time
+ // TODO: we should verify the status of these frames by logging the inferred status and verifying it matches
+ final Queue<Frame> inFlight = new Queue<>();
+ final Queue<Frame> retiredWithoutStatus = new Queue<>();
+ private int withStatus;
+
+ Frame supplySendStatus(Frame.Status status)
+ {
+ Frame frame;
+ if (withStatus >= 0) frame = inFlight.get(withStatus);
+ else frame = retiredWithoutStatus.poll();
+ assert frame.sendStatus == Frame.Status.UNKNOWN;
+ frame.sendStatus = status;
+ ++withStatus;
+ return frame;
+ }
+
+ boolean isEmpty()
+ {
+ return inFlight.isEmpty();
+ }
+
+ int size()
+ {
+ return inFlight.size();
+ }
+
+ Frame get(int i)
+ {
+ return inFlight.get(i);
+ }
+
+ void add(Frame frame)
+ {
+ assert frame.sendStatus == Frame.Status.UNKNOWN;
+ inFlight.add(frame);
+ }
+
+ void remove(Frame frame)
+ {
+ int i = inFlight.indexOf(frame);
+ if (i > 0) throw new IllegalStateException();
+ if (i == 0) poll();
+ }
+
+ void removeFirst(int count)
+ {
+ while (count-- > 0)
+ poll();
+ }
+
+ Frame poll()
+ {
+ Frame frame = inFlight.poll();
+ if (--withStatus < 0)
+ {
+ assert frame.sendStatus == Frame.Status.UNKNOWN;
+ retiredWithoutStatus.add(frame);
+ }
+ else
+ assert frame.sendStatus != Frame.Status.UNKNOWN;
+ return frame;
+ }
+
+ public String toString()
+ {
+ StringBuilder result = new StringBuilder();
+ result.append("[withStatus=");
+ result.append(withStatus);
+ result.append("; ");
+ inFlight.toString(result);
+ result.append("; ");
+ retiredWithoutStatus.toString(result);
+ result.append(']');
+ return result.toString();
+ }
+ }
+
+ private static boolean willProcessOnEventLoop(ConnectionType type, Message<?> message, int messagingVersion)
+ {
+ int size = message.serializedSize(messagingVersion);
+ if (type == ConnectionType.SMALL_MESSAGES && messagingVersion >= VERSION_40)
+ return size <= LARGE_MESSAGE_THRESHOLD;
+ else if (messagingVersion >= VERSION_40)
+ return size <= DEFAULT_BUFFER_SIZE;
+ else
+ return size <= LARGE_MESSAGE_THRESHOLD;
+ }
+
+ private static long expiresAtNanos(Message<?> message, int messagingVersion)
+ {
+ return messagingVersion < VERSION_40 ? message.verb().expiresAtNanos(message.createdAtNanos())
+ : message.expiresAtNanos();
+ }
+
+}
diff --git a/test/burn/org/apache/cassandra/utils/LongBTreeTest.java b/test/burn/org/apache/cassandra/utils/LongBTreeTest.java
index c052015..d21f7b3 100644
--- a/test/burn/org/apache/cassandra/utils/LongBTreeTest.java
+++ b/test/burn/org/apache/cassandra/utils/LongBTreeTest.java
@@ -61,6 +61,10 @@
private static int perThreadTrees = 100;
private static int minTreeSize = 4;
private static int maxTreeSize = 10000;
+ private static float generateTreeByUpdateChance = 0.8f;
+ private static float generateTreeByCopyChance = 0.1f;
+ private static float generateTreeByBuilderChance = 0.1f;
+ private static float generateTreeTotalChance = generateTreeByUpdateChance + generateTreeByCopyChance + generateTreeByBuilderChance;
private static int threads = DEBUG ? 1 : Runtime.getRuntime().availableProcessors() * 8;
private static final MetricRegistry metrics = new MetricRegistry();
private static final Timer BTREE_TIMER = metrics.timer(MetricRegistry.name(BTree.class, "BTREE"));
@@ -80,8 +84,12 @@
public void testSearchIterator() throws InterruptedException
{
final int perTreeSelections = 100;
- testRandomSelection(perThreadTrees, perTreeSelections,
- (test) -> {
+ testRandomSelection(perThreadTrees, perTreeSelections, testSearchIteratorFactory());
+ }
+
+ private BTreeTestFactory testSearchIteratorFactory()
+ {
+ return (test) -> {
IndexedSearchIterator<Integer, Integer> iter1 = test.testAsSet.iterator();
IndexedSearchIterator<Integer, Integer> iter2 = test.testAsList.iterator();
return (key) ->
@@ -110,45 +118,52 @@
else
Assert.assertNull(iter2.next(key));
};
- });
+ };
}
@Test
public void testInequalityLookups() throws InterruptedException
{
final int perTreeSelections = 2;
- testRandomSelectionOfSet(perThreadTrees, perTreeSelections,
- (test, canonical) -> {
- if (!canonical.isEmpty() || !test.isEmpty())
- {
- Assert.assertEquals(canonical.isEmpty(), test.isEmpty());
- Assert.assertEquals(canonical.first(), test.first());
- Assert.assertEquals(canonical.last(), test.last());
- }
- return (key) ->
- {
- Assert.assertEquals(test.ceiling(key), canonical.ceiling(key));
- Assert.assertEquals(test.higher(key), canonical.higher(key));
- Assert.assertEquals(test.floor(key), canonical.floor(key));
- Assert.assertEquals(test.lower(key), canonical.lower(key));
- };
- });
+ testRandomSelectionOfSet(perThreadTrees, perTreeSelections, testInequalityLookupsFactory());
+ }
+
+ private BTreeSetTestFactory testInequalityLookupsFactory()
+ {
+ return (test, canonical) -> {
+ if (!canonical.isEmpty() || !test.isEmpty())
+ {
+ Assert.assertEquals(canonical.isEmpty(), test.isEmpty());
+ Assert.assertEquals(canonical.first(), test.first());
+ Assert.assertEquals(canonical.last(), test.last());
+ }
+ return (key) ->
+ {
+ Assert.assertEquals(test.ceiling(key), canonical.ceiling(key));
+ Assert.assertEquals(test.higher(key), canonical.higher(key));
+ Assert.assertEquals(test.floor(key), canonical.floor(key));
+ Assert.assertEquals(test.lower(key), canonical.lower(key));
+ };
+ };
}
@Test
public void testListIndexes() throws InterruptedException
{
- testRandomSelectionOfList(perThreadTrees, 4,
- (test, canonical, cmp) ->
- (key) ->
- {
- int javaIndex = Collections.binarySearch(canonical, key, cmp);
- int btreeIndex = test.indexOf(key);
- Assert.assertEquals(javaIndex, btreeIndex);
- if (javaIndex >= 0)
- Assert.assertEquals(canonical.get(javaIndex), test.get(btreeIndex));
- }
- );
+ testRandomSelectionOfList(perThreadTrees, 4, testListIndexesFactory());
+ }
+
+ private BTreeListTestFactory testListIndexesFactory()
+ {
+ return (test, canonical, cmp) ->
+ (key) ->
+ {
+ int javaIndex = Collections.binarySearch(canonical, key, cmp);
+ int btreeIndex = test.indexOf(key);
+ Assert.assertEquals(javaIndex, btreeIndex);
+ if (javaIndex >= 0)
+ Assert.assertEquals(canonical.get(javaIndex), test.get(btreeIndex));
+ };
}
@Test
@@ -264,13 +279,23 @@
void testOne(Integer value);
}
+ private void run(BTreeTestFactory testRun, RandomSelection selection)
+ {
+ TestEachKey testEachKey = testRun.get(selection);
+ for (Integer key : selection.testKeys)
+ testEachKey.testOne(key);
+ }
+
+ private void run(BTreeSetTestFactory testRun, RandomSelection selection)
+ {
+ TestEachKey testEachKey = testRun.get(selection.testAsSet, selection.canonicalSet);
+ for (Integer key : selection.testKeys)
+ testEachKey.testOne(key);
+ }
+
private void testRandomSelection(int perThreadTrees, int perTreeSelections, BTreeTestFactory testRun) throws InterruptedException
{
- testRandomSelection(perThreadTrees, perTreeSelections, (selection) -> {
- TestEachKey testEachKey = testRun.get(selection);
- for (Integer key : selection.testKeys)
- testEachKey.testOne(key);
- });
+ testRandomSelection(perThreadTrees, perTreeSelections, (RandomSelection selection) -> run(testRun, selection));
}
private void testRandomSelection(int perThreadTrees, int perTreeSelections, Consumer<RandomSelection> testRun) throws InterruptedException
@@ -287,29 +312,29 @@
final long totalCount = threads * perThreadTrees * perTreeSelections;
for (int t = 0 ; t < threads ; t++)
{
- Runnable runnable = new Runnable()
+ Runnable runnable = () ->
{
- public void run()
+ try
{
- try
+ for (int i = 0 ; i < perThreadTrees ; i++)
{
- for (int i = 0 ; i < perThreadTrees ; i++)
+ // not easy to usefully log seed, as run tests in parallel; need to really pass through to exceptions
+ long seed = ThreadLocalRandom.current().nextLong();
+ Random random = new Random(seed);
+ RandomTree tree = randomTree(minTreeSize, maxTreeSize, random);
+ for (int j = 0 ; j < perTreeSelections ; j++)
{
- RandomTree tree = randomTree(minTreeSize, maxTreeSize);
- for (int j = 0 ; j < perTreeSelections ; j++)
- {
- testRun.accept(tree.select(narrow, mixInNotPresentItems, permitReversal));
- count.incrementAndGet();
- }
+ testRun.accept(tree.select(narrow, mixInNotPresentItems, permitReversal));
+ count.incrementAndGet();
}
}
- catch (Throwable t)
- {
- errors.incrementAndGet();
- t.printStackTrace();
- }
- latch.countDown();
}
+ catch (Throwable t1)
+ {
+ errors.incrementAndGet();
+ t1.printStackTrace();
+ }
+ latch.countDown();
};
MODIFY.execute(runnable);
}
@@ -347,18 +372,19 @@
private static class RandomTree
{
+ final Random random;
final NavigableSet<Integer> canonical;
final BTreeSet<Integer> test;
- private RandomTree(NavigableSet<Integer> canonical, BTreeSet<Integer> test)
+ private RandomTree(NavigableSet<Integer> canonical, BTreeSet<Integer> test, Random random)
{
this.canonical = canonical;
this.test = test;
+ this.random = random;
}
RandomSelection select(boolean narrow, boolean mixInNotPresentItems, boolean permitReversal)
{
- ThreadLocalRandom random = ThreadLocalRandom.current();
NavigableSet<Integer> canonicalSet = this.canonical;
BTreeSet<Integer> testAsSet = this.test;
List<Integer> canonicalList = new ArrayList<>(canonicalSet);
@@ -368,7 +394,7 @@
Assert.assertEquals(canonicalList.size(), testAsList.size());
// sometimes select keys first, so we cover full range
- List<Integer> allKeys = randomKeys(canonical, mixInNotPresentItems);
+ List<Integer> allKeys = randomKeys(canonical, mixInNotPresentItems, random);
List<Integer> keys = allKeys;
int narrowCount = random.nextInt(3);
@@ -391,7 +417,7 @@
if (useLb)
{
- lbKeyIndex = random.nextInt(0, indexRange - 1);
+ lbKeyIndex = random.nextInt(indexRange - 1);
Integer candidate = keys.get(lbKeyIndex);
if (useLb = (candidate > lbKey && candidate <= ubKey))
{
@@ -404,7 +430,8 @@
}
if (useUb)
{
- ubKeyIndex = random.nextInt(Math.max(lbKeyIndex, keys.size() - indexRange), keys.size() - 1);
+ int lb = Math.max(lbKeyIndex, keys.size() - indexRange);
+ ubKeyIndex = random.nextInt(keys.size() - (1 + lb)) + lb;
Integer candidate = keys.get(ubKeyIndex);
if (useUb = (candidate < ubKey && candidate >= lbKey))
{
@@ -468,31 +495,53 @@
}
}
- private static RandomTree randomTree(int minSize, int maxSize)
+ private static RandomTree randomTree(int minSize, int maxSize, Random random)
{
// perform most of our tree constructions via update, as this is more efficient; since every run uses this
// we test builder disproportionately more often than if it had its own test anyway
- return ThreadLocalRandom.current().nextFloat() < 0.95 ? randomTreeByUpdate(minSize, maxSize)
- : randomTreeByBuilder(minSize, maxSize);
+ int maxIntegerValue = random.nextInt(Integer.MAX_VALUE - 1) + 1;
+ float f = random.nextFloat() / generateTreeTotalChance;
+ f -= generateTreeByUpdateChance;
+ if (f < 0)
+ return randomTreeByUpdate(minSize, maxSize, maxIntegerValue, random);
+ f -= generateTreeByCopyChance;
+ if (f < 0)
+ return randomTreeByCopy(minSize, maxSize, maxIntegerValue, random);
+ return randomTreeByBuilder(minSize, maxSize, maxIntegerValue, random);
}
- private static RandomTree randomTreeByUpdate(int minSize, int maxSize)
+ private static RandomTree randomTreeByCopy(int minSize, int maxSize, int maxIntegerValue, Random random)
{
assert minSize > 3;
TreeSet<Integer> canonical = new TreeSet<>();
- ThreadLocalRandom random = ThreadLocalRandom.current();
- int targetSize = random.nextInt(minSize, maxSize);
- int maxModificationSize = random.nextInt(2, targetSize);
+ int targetSize = random.nextInt(maxSize - minSize) + minSize;
+ int curSize = 0;
+ while (curSize < targetSize)
+ {
+ Integer next = random.nextInt(maxIntegerValue);
+ if (canonical.add(next))
+ ++curSize;
+ }
+ return new RandomTree(canonical, BTreeSet.<Integer>wrap(BTree.build(canonical, UpdateFunction.noOp()), naturalOrder()), random);
+ }
+
+ private static RandomTree randomTreeByUpdate(int minSize, int maxSize, int maxIntegerValue, Random random)
+ {
+ assert minSize > 3;
+ TreeSet<Integer> canonical = new TreeSet<>();
+
+ int targetSize = random.nextInt(maxSize - minSize) + minSize;
+ int maxModificationSize = random.nextInt(targetSize - 2) + 2;
Object[] accmumulate = BTree.empty();
int curSize = 0;
while (curSize < targetSize)
{
- int nextSize = maxModificationSize == 1 ? 1 : random.nextInt(1, maxModificationSize);
+ int nextSize = maxModificationSize == 1 ? 1 : random.nextInt(maxModificationSize - 1) + 1;
TreeSet<Integer> build = new TreeSet<>();
for (int i = 0 ; i < nextSize ; i++)
{
- Integer next = random.nextInt();
+ Integer next = random.nextInt(maxIntegerValue);
build.add(next);
canonical.add(next);
}
@@ -500,16 +549,15 @@
curSize += nextSize;
maxModificationSize = Math.min(maxModificationSize, targetSize - curSize);
}
- return new RandomTree(canonical, BTreeSet.<Integer>wrap(accmumulate, naturalOrder()));
+ return new RandomTree(canonical, BTreeSet.<Integer>wrap(accmumulate, naturalOrder()), random);
}
- private static RandomTree randomTreeByBuilder(int minSize, int maxSize)
+ private static RandomTree randomTreeByBuilder(int minSize, int maxSize, int maxIntegerValue, Random random)
{
assert minSize > 3;
- ThreadLocalRandom random = ThreadLocalRandom.current();
BTree.Builder<Integer> builder = BTree.builder(naturalOrder());
- int targetSize = random.nextInt(minSize, maxSize);
+ int targetSize = random.nextInt(maxSize - minSize) + minSize;
int maxModificationSize = (int) Math.sqrt(targetSize);
TreeSet<Integer> canonical = new TreeSet<>();
@@ -519,7 +567,7 @@
List<Integer> shuffled = new ArrayList<>();
while (curSize < targetSize)
{
- int nextSize = maxModificationSize <= 1 ? 1 : random.nextInt(1, maxModificationSize);
+ int nextSize = maxModificationSize <= 1 ? 1 : random.nextInt(maxModificationSize - 1) + 1;
// leave a random selection of previous values
(random.nextBoolean() ? ordered.headSet(random.nextInt()) : ordered.tailSet(random.nextInt())).clear();
@@ -527,7 +575,7 @@
for (int i = 0 ; i < nextSize ; i++)
{
- Integer next = random.nextInt();
+ Integer next = random.nextInt(maxIntegerValue);
ordered.add(next);
shuffled.add(next);
canonical.add(next);
@@ -558,16 +606,15 @@
BTreeSet<Integer> btree = BTreeSet.<Integer>wrap(builder.build(), naturalOrder());
Assert.assertEquals(canonical.size(), btree.size());
- return new RandomTree(canonical, btree);
+ return new RandomTree(canonical, btree, random);
}
// select a random subset of the keys, with an optional random population of keys inbetween those that are present
// return a value with the search position
- private static List<Integer> randomKeys(Iterable<Integer> canonical, boolean mixInNotPresentItems)
+ private static List<Integer> randomKeys(Iterable<Integer> canonical, boolean mixInNotPresentItems, Random random)
{
- ThreadLocalRandom rnd = ThreadLocalRandom.current();
- boolean useFake = mixInNotPresentItems && rnd.nextBoolean();
- final float fakeRatio = rnd.nextFloat();
+ boolean useFake = mixInNotPresentItems && random.nextBoolean();
+ final float fakeRatio = random.nextFloat();
List<Integer> results = new ArrayList<>();
Long fakeLb = (long) Integer.MIN_VALUE, fakeUb = null;
Integer max = null;
@@ -575,7 +622,7 @@
{
if ( !useFake
|| (fakeUb == null ? v - 1 : fakeUb) <= fakeLb + 1
- || rnd.nextFloat() < fakeRatio)
+ || random.nextFloat() < fakeRatio)
{
// if we cannot safely construct a fake value, or our randomizer says not to, we emit the next real value
results.add(v);
@@ -597,13 +644,32 @@
}
if (useFake && max != null && max < Integer.MAX_VALUE)
results.add(max + 1);
- final float useChance = rnd.nextFloat();
- return Lists.newArrayList(filter(results, (x) -> rnd.nextFloat() < useChance));
+ final float useChance = random.nextFloat();
+ return Lists.newArrayList(filter(results, (x) -> random.nextFloat() < useChance));
}
/************************** TEST MUTATION ********************************************/
@Test
+ public void testBuildNewTree()
+ {
+ int max = 10000;
+ final List<Integer> list = new ArrayList<>(max);
+ final NavigableSet<Integer> set = new TreeSet<>();
+ BTreeSetTestFactory test = testInequalityLookupsFactory();
+ for (int i = 0 ; i < max ; ++i)
+ {
+ list.add(i);
+ set.add(i);
+ Object[] tree = BTree.build(list, UpdateFunction.noOp());
+ Assert.assertTrue(BTree.isWellFormed(tree, Comparator.naturalOrder()));
+ BTreeSet<Integer> btree = new BTreeSet<>(tree, Comparator.naturalOrder());
+ RandomSelection selection = new RandomSelection(list, set, btree, list, btree, Comparator.naturalOrder());
+ run(test, selection);
+ }
+ }
+
+ @Test
public void testOversizedMiddleInsert()
{
TreeSet<Integer> canon = new TreeSet<>();
diff --git a/test/burn/org/apache/cassandra/utils/memory/LongBufferPoolTest.java b/test/burn/org/apache/cassandra/utils/memory/LongBufferPoolTest.java
index 66abe5a..c8368dd 100644
--- a/test/burn/org/apache/cassandra/utils/memory/LongBufferPoolTest.java
+++ b/test/burn/org/apache/cassandra/utils/memory/LongBufferPoolTest.java
@@ -27,11 +27,15 @@
import java.util.concurrent.atomic.AtomicBoolean;
import com.google.common.util.concurrent.Uninterruptibles;
+import org.junit.BeforeClass;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.cassandra.concurrent.NamedThreadFactory;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.io.compress.BufferType;
import org.apache.cassandra.utils.DynamicList;
import static org.junit.Assert.*;
@@ -66,6 +70,48 @@
private static final int STDEV_BUFFER_SIZE = 10 << 10; // picked to ensure exceeding buffer size is rare, but occurs
private static final DateFormat DATE_FORMAT = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss");
+ static final class Debug implements BufferPool.Debug
+ {
+ static class DebugChunk
+ {
+ volatile long lastRecycled;
+ static DebugChunk get(BufferPool.Chunk chunk)
+ {
+ if (chunk.debugAttachment == null)
+ chunk.debugAttachment = new DebugChunk();
+ return (DebugChunk) chunk.debugAttachment;
+ }
+ }
+ long recycleRound = 1;
+ final List<BufferPool.Chunk> normalChunks = new ArrayList<>();
+ final List<BufferPool.Chunk> tinyChunks = new ArrayList<>();
+ public synchronized void registerNormal(BufferPool.Chunk chunk)
+ {
+ chunk.debugAttachment = new DebugChunk();
+ normalChunks.add(chunk);
+ }
+ public void recycleNormal(BufferPool.Chunk oldVersion, BufferPool.Chunk newVersion)
+ {
+ newVersion.debugAttachment = oldVersion.debugAttachment;
+ DebugChunk.get(oldVersion).lastRecycled = recycleRound;
+ }
+ public synchronized void check()
+ {
+// for (BufferPool.Chunk chunk : tinyChunks)
+// assert DebugChunk.get(chunk).lastRecycled == recycleRound;
+ for (BufferPool.Chunk chunk : normalChunks)
+ assert DebugChunk.get(chunk).lastRecycled == recycleRound;
+ tinyChunks.clear(); // they don't survive a recycleRound
+ recycleRound++;
+ }
+ }
+
+ @BeforeClass
+ public static void setup() throws Exception
+ {
+ DatabaseDescriptor.daemonInitialization();
+ }
+
@Test
public void testAllocate() throws InterruptedException, ExecutionException
{
@@ -125,7 +171,7 @@
makingProgress = new AtomicBoolean[threadCount];
burnFreed = new AtomicBoolean(false);
freedAllMemory = new AtomicBoolean[threadCount];
- executorService = Executors.newFixedThreadPool(threadCount + 2);
+ executorService = Executors.newFixedThreadPool(threadCount + 2, new NamedThreadFactory("test"));
threadResultFuture = new ArrayList<>(threadCount);
for (int i = 0; i < sharedRecycle.length; i++)
@@ -141,7 +187,7 @@
// using their own algorithm the targetSize should be poolSize / targetSizeQuanta.
//
// This should divide double the poolSize across the working threads,
- // plus CHUNK_SIZE for thread0 and 1/10 poolSize for the burn producer/consumer pair.
+ // plus NORMAL_CHUNK_SIZE for thread0 and 1/10 poolSize for the burn producer/consumer pair.
targetSizeQuanta = 2 * poolSize / sum1toN(threadCount - 1);
}
@@ -201,7 +247,8 @@
long prevPoolSize = BufferPool.MEMORY_USAGE_THRESHOLD;
logger.info("Overriding configured BufferPool.MEMORY_USAGE_THRESHOLD={} and enabling BufferPool.DEBUG", poolSize);
BufferPool.MEMORY_USAGE_THRESHOLD = poolSize;
- BufferPool.DEBUG = true;
+ Debug debug = new Debug();
+ BufferPool.debug(debug);
TestEnvironment testEnv = new TestEnvironment(threadCount, duration, poolSize);
@@ -222,7 +269,7 @@
for (AtomicBoolean freedMemory : testEnv.freedAllMemory)
allFreed = allFreed && freedMemory.getAndSet(false);
if (allFreed)
- BufferPool.assertAllRecycled();
+ debug.check();
else
logger.info("All threads did not free all memory in this time slot - skipping buffer recycle check");
}
@@ -242,7 +289,7 @@
logger.info("Reverting BufferPool.MEMORY_USAGE_THRESHOLD={}", prevPoolSize);
BufferPool.MEMORY_USAGE_THRESHOLD = prevPoolSize;
- BufferPool.DEBUG = false;
+ BufferPool.debug(null);
testEnv.assertCheckedThreadsSucceeded();
@@ -254,7 +301,7 @@
{
return testEnv.executorService.submit(new TestUntil(testEnv.until)
{
- final int targetSize = threadIdx == 0 ? BufferPool.CHUNK_SIZE : testEnv.targetSizeQuanta * threadIdx;
+ final int targetSize = threadIdx == 0 ? BufferPool.NORMAL_CHUNK_SIZE : testEnv.targetSizeQuanta * threadIdx;
final SPSCQueue<BufferCheck> shareFrom = testEnv.sharedRecycle[threadIdx];
final DynamicList<BufferCheck> checks = new DynamicList<>((int) Math.max(1, targetSize / (1 << 10)));
final SPSCQueue<BufferCheck> shareTo = testEnv.sharedRecycle[(threadIdx + 1) % testEnv.threadCount];
@@ -271,7 +318,6 @@
void testOne() throws Exception
{
-
long currentTargetSize = (rand.nextInt(testEnv.poolSize / 1024) == 0 || !testEnv.freedAllMemory[threadIdx].get()) ? 0 : targetSize;
int spinCount = 0;
while (totalSize > currentTargetSize - freeingSize)
@@ -301,8 +347,8 @@
checks.remove(check.listnode);
check.validate();
- size = BufferPool.roundUpNormal(check.buffer.capacity());
- if (size > BufferPool.CHUNK_SIZE)
+ size = BufferPool.roundUp(check.buffer.capacity());
+ if (size > BufferPool.NORMAL_CHUNK_SIZE)
size = 0;
// either share to free, or free immediately
@@ -326,9 +372,9 @@
// allocate a new buffer
size = (int) Math.max(1, AVG_BUFFER_SIZE + (STDEV_BUFFER_SIZE * rand.nextGaussian()));
- if (size <= BufferPool.CHUNK_SIZE)
+ if (size <= BufferPool.NORMAL_CHUNK_SIZE)
{
- totalSize += BufferPool.roundUpNormal(size);
+ totalSize += BufferPool.roundUp(size);
allocate(size);
}
else if (rand.nextBoolean())
@@ -341,10 +387,10 @@
while (totalSize < testEnv.poolSize)
{
size = (int) Math.max(1, AVG_BUFFER_SIZE + (STDEV_BUFFER_SIZE * rand.nextGaussian()));
- if (size <= BufferPool.CHUNK_SIZE)
+ if (size <= BufferPool.NORMAL_CHUNK_SIZE)
{
allocate(size);
- totalSize += BufferPool.roundUpNormal(size);
+ totalSize += BufferPool.roundUp(size);
}
}
}
@@ -379,7 +425,7 @@
BufferCheck allocate(int size)
{
- ByteBuffer buffer = BufferPool.get(size);
+ ByteBuffer buffer = BufferPool.get(size, BufferType.OFF_HEAP);
assertNotNull(buffer);
BufferCheck check = new BufferCheck(buffer, rand.nextLong());
assertEquals(size, buffer.capacity());
@@ -435,7 +481,7 @@
final ThreadLocalRandom rand = ThreadLocalRandom.current();
void testOne() throws Exception
{
- if (count * BufferPool.CHUNK_SIZE >= testEnv.poolSize / 10)
+ if (count * BufferPool.NORMAL_CHUNK_SIZE >= testEnv.poolSize / 10)
{
if (burn.exhausted)
{
@@ -448,7 +494,8 @@
return;
}
- ByteBuffer buffer = BufferPool.tryGet(BufferPool.CHUNK_SIZE);
+ ByteBuffer buffer = rand.nextInt(4) < 1 ? BufferPool.tryGet(BufferPool.NORMAL_CHUNK_SIZE)
+ : BufferPool.tryGet(BufferPool.TINY_ALLOCATION_LIMIT);
if (buffer == null)
{
Thread.yield();
@@ -515,7 +562,7 @@
{
logger.error("Got exception {}, current chunk {}",
ex.getMessage(),
- BufferPool.currentChunk());
+ BufferPool.unsafeCurrentChunk());
ex.printStackTrace();
return false;
}
@@ -523,7 +570,7 @@
{
logger.error("Got throwable {}, current chunk {}",
tr.getMessage(),
- BufferPool.currentChunk());
+ BufferPool.unsafeCurrentChunk());
tr.printStackTrace();
return false;
}
@@ -539,6 +586,7 @@
{
try
{
+ LongBufferPoolTest.setup();
new LongBufferPoolTest().testAllocate(Runtime.getRuntime().availableProcessors(),
TimeUnit.HOURS.toNanos(2L), 16 << 20);
System.exit(0);
@@ -546,6 +594,7 @@
catch (Throwable tr)
{
System.out.println(String.format("Test failed - %s", tr.getMessage()));
+ tr.printStackTrace();
System.exit(1); // Force exit so that non-daemon threads like REQUEST-SCHEDULER do not hang the process on failure
}
}
diff --git a/test/conf/cassandra-murmur.yaml b/test/conf/cassandra-murmur.yaml
index a4b25ba..3c263a5 100644
--- a/test/conf/cassandra-murmur.yaml
+++ b/test/conf/cassandra-murmur.yaml
@@ -13,8 +13,7 @@
hints_directory: build/test/cassandra/hints
partitioner: org.apache.cassandra.dht.Murmur3Partitioner
listen_address: 127.0.0.1
-storage_port: 7010
-rpc_port: 9170
+storage_port: 7012
start_native_transport: true
native_transport_port: 9042
column_index_size_in_kb: 4
@@ -25,11 +24,9 @@
seed_provider:
- class_name: org.apache.cassandra.locator.SimpleSeedProvider
parameters:
- - seeds: "127.0.0.1"
+ - seeds: "127.0.0.1:7012"
endpoint_snitch: org.apache.cassandra.locator.SimpleSnitch
dynamic_snitch: true
-request_scheduler: org.apache.cassandra.scheduler.RoundRobinScheduler
-request_scheduler_id: keyspace
server_encryption_options:
internode_encryption: none
keystore: conf/.keystore
@@ -43,3 +40,5 @@
row_cache_size_in_mb: 16
enable_user_defined_functions: true
enable_scripted_user_defined_functions: true
+enable_sasi_indexes: true
+enable_materialized_views: true
diff --git a/test/conf/cassandra-rackdc.properties b/test/conf/cassandra-rackdc.properties
index be2e7d2..742def3 100644
--- a/test/conf/cassandra-rackdc.properties
+++ b/test/conf/cassandra-rackdc.properties
@@ -22,3 +22,15 @@
# Add a suffix to a datacenter name. Used by the Ec2Snitch and Ec2MultiRegionSnitch
# to append a string to the EC2 region name.
#dc_suffix=
+
+# Datacenter and rack naming convention used by the Ec2Snitch and Ec2MultiRegionSnitch.
+# Options are:
+# legacy : datacenter name is the part of the availability zone name preceding the last "-"
+# when the zone ends in -1 and includes the number if not -1. Rack is the portion of
+# the availability zone name following the last "-".
+# Examples: us-west-1a => dc: us-west, rack: 1a; us-west-2b => dc: us-west-2, rack: 2b;
+# standard : datacenter name is the standard AWS region name, including the number. rack name is the
+# region plus the availability zone letter.
+# Examples: us-west-1a => dc: us-west-1, rack: us-west-1a; us-west-2b => dc: us-west-2, rack: us-west-2b;
+# default: standard
+ec2_naming_scheme=standard
diff --git a/test/conf/cassandra-seeds.yaml b/test/conf/cassandra-seeds.yaml
index 02d25d2..f3279ae 100644
--- a/test/conf/cassandra-seeds.yaml
+++ b/test/conf/cassandra-seeds.yaml
@@ -14,7 +14,7 @@
hints_directory: build/test/cassandra/hints
partitioner: org.apache.cassandra.dht.ByteOrderedPartitioner
listen_address: 127.0.0.1
-storage_port: 7010
+storage_port: 7012
start_native_transport: true
native_transport_port: 9042
column_index_size_in_kb: 4
diff --git a/test/conf/cassandra.yaml b/test/conf/cassandra.yaml
index 96ca9a0..89b7ff1 100644
--- a/test/conf/cassandra.yaml
+++ b/test/conf/cassandra.yaml
@@ -9,13 +9,15 @@
commitlog_sync_batch_window_in_ms: 1.0
commitlog_segment_size_in_mb: 5
commitlog_directory: build/test/cassandra/commitlog
+# commitlog_compression:
+# - class_name: LZ4Compressor
cdc_raw_directory: build/test/cassandra/cdc_raw
cdc_enabled: false
hints_directory: build/test/cassandra/hints
partitioner: org.apache.cassandra.dht.ByteOrderedPartitioner
listen_address: 127.0.0.1
-storage_port: 7010
-rpc_port: 9170
+storage_port: 7012
+ssl_storage_port: 7011
start_native_transport: true
native_transport_port: 9042
column_index_size_in_kb: 4
@@ -26,11 +28,9 @@
seed_provider:
- class_name: org.apache.cassandra.locator.SimpleSeedProvider
parameters:
- - seeds: "127.0.0.1"
+ - seeds: "127.0.0.1:7012"
endpoint_snitch: org.apache.cassandra.locator.SimpleSnitch
dynamic_snitch: true
-request_scheduler: org.apache.cassandra.scheduler.RoundRobinScheduler
-request_scheduler_id: keyspace
server_encryption_options:
internode_encryption: none
keystore: conf/.keystore
@@ -45,3 +45,8 @@
enable_user_defined_functions: true
enable_scripted_user_defined_functions: true
prepared_statements_cache_size_mb: 1
+corrupted_tombstone_strategy: exception
+stream_entire_sstables: true
+stream_throughput_outbound_megabits_per_sec: 200000000
+enable_sasi_indexes: true
+enable_materialized_views: true
diff --git a/test/conf/cassandra_ssl_test.keystore b/test/conf/cassandra_ssl_test.keystore
new file mode 100644
index 0000000..8b2b218
--- /dev/null
+++ b/test/conf/cassandra_ssl_test.keystore
Binary files differ
diff --git a/test/conf/cassandra_ssl_test.truststore b/test/conf/cassandra_ssl_test.truststore
new file mode 100644
index 0000000..49cf332
--- /dev/null
+++ b/test/conf/cassandra_ssl_test.truststore
Binary files differ
diff --git a/test/conf/cdc.yaml b/test/conf/cdc.yaml
index f79930a..8fb9427 100644
--- a/test/conf/cdc.yaml
+++ b/test/conf/cdc.yaml
@@ -1 +1,4 @@
cdc_enabled: true
+# Compression enabled since uncompressed + cdc isn't compatible w/Windows
+commitlog_compression:
+ - class_name: LZ4Compressor
diff --git a/test/conf/commitlog_compression.yaml b/test/conf/commitlog_compression_LZ4.yaml
similarity index 100%
rename from test/conf/commitlog_compression.yaml
rename to test/conf/commitlog_compression_LZ4.yaml
diff --git a/test/conf/commitlog_compression_Zstd.yaml b/test/conf/commitlog_compression_Zstd.yaml
new file mode 100644
index 0000000..0c440ae
--- /dev/null
+++ b/test/conf/commitlog_compression_Zstd.yaml
@@ -0,0 +1,2 @@
+commitlog_compression:
+ - class_name: ZstdCompressor
diff --git a/test/conf/logback-burntest.xml b/test/conf/logback-burntest.xml
new file mode 100644
index 0000000..e1e48a9
--- /dev/null
+++ b/test/conf/logback-burntest.xml
@@ -0,0 +1,66 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<configuration debug="false" scan="true" scanPeriod="60 seconds">
+ <define name="instance_id" class="org.apache.cassandra.distributed.impl.InstanceIDDefiner" />
+
+ <!-- Shutdown hook ensures that async appender flushes -->
+ <shutdownHook class="ch.qos.logback.core.hook.DelayingShutdownHook"/>
+
+ <appender name="FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
+
+ <file>./build/test/logs/${cassandra.testtag}/TEST-${suitename}.log</file>
+ <rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
+ <fileNamePattern>./build/test/logs/${cassandra.testtag}/TEST-${suitename}.log.%i.gz</fileNamePattern>
+ <minIndex>1</minIndex>
+ <maxIndex>20</maxIndex>
+ </rollingPolicy>
+
+ <triggeringPolicy class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
+ <maxFileSize>20MB</maxFileSize>
+ </triggeringPolicy>
+
+ <encoder>
+ <pattern>%-5level [%thread] ${instance_id} %date{ISO8601} %msg%n</pattern>
+ </encoder>
+ <immediateFlush>false</immediateFlush>
+ </appender>
+
+ <appender name="ASYNCFILE" class="ch.qos.logback.classic.AsyncAppender">
+ <discardingThreshold>0</discardingThreshold>
+ <maxFlushTime>0</maxFlushTime>
+ <queueSize>1024</queueSize>
+ <appender-ref ref="FILE"/>
+ <filter class="org.apache.cassandra.net.LogbackFilter"/>
+ </appender>
+
+ <appender name="STDOUT" target="System.out" class="ch.qos.logback.core.ConsoleAppender">
+ <encoder>
+ <pattern>%-5level [%thread] ${instance_id} %date{ISO8601} %F:%L - %msg%n</pattern>
+ </encoder>
+ <filter class="org.apache.cassandra.net.LogbackFilter"/>
+ </appender>
+
+ <logger name="org.apache.hadoop" level="WARN"/>
+ <logger name="io.netty.handler.ssl.SslHandler" level="WARN"/>
+
+ <root level="DEBUG">
+ <appender-ref ref="ASYNCFILE" />
+ <appender-ref ref="STDOUT" />
+ </root>
+</configuration>
diff --git a/test/conf/logback-dtest.xml b/test/conf/logback-dtest.xml
index 4282fee..370e1e5 100644
--- a/test/conf/logback-dtest.xml
+++ b/test/conf/logback-dtest.xml
@@ -60,15 +60,6 @@
<appender name="INSTANCESTDOUT" target="System.out" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
- <pattern>%-5level %date{HH:mm:ss,SSS} %msg%n</pattern>
- </encoder>
- <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
- <level>WARN</level>
- </filter>
- </appender>
-
- <appender name="INSTANCESTDOUT" target="System.out" class="ch.qos.logback.core.ConsoleAppender">
- <encoder>
<pattern>%-5level [%thread] ${instance_id} %date{ISO8601} %F:%L - %msg%n</pattern>
</encoder>
<filter class="ch.qos.logback.classic.filter.ThresholdFilter">
diff --git a/test/conf/logback-test.xml b/test/conf/logback-test.xml
index eb8ee57..3e3349f 100644
--- a/test/conf/logback-test.xml
+++ b/test/conf/logback-test.xml
@@ -17,7 +17,7 @@
under the License.
-->
-<configuration debug="false" scan="true">
+<configuration debug="false" scan="true" scanPeriod="60 seconds">
<!-- Shutdown hook ensures that async appender flushes -->
<shutdownHook class="ch.qos.logback.core.hook.DelayingShutdownHook"/>
diff --git a/test/conf/unit-test-conf/test-native-port.yaml b/test/conf/unit-test-conf/test-native-port.yaml
index a90f100..b46525f 100644
--- a/test/conf/unit-test-conf/test-native-port.yaml
+++ b/test/conf/unit-test-conf/test-native-port.yaml
@@ -9,34 +9,34 @@
commitlog_sync_batch_window_in_ms: 1.0
commitlog_segment_size_in_mb: 5
commitlog_directory: build/test/cassandra/commitlog
+# commitlog_compression:
+# - class_name: LZ4Compressor
cdc_raw_directory: build/test/cassandra/cdc_raw
cdc_enabled: false
hints_directory: build/test/cassandra/hints
partitioner: org.apache.cassandra.dht.ByteOrderedPartitioner
listen_address: 127.0.0.1
storage_port: 7010
-rpc_port: 9170
+ssl_storage_port: 7011
start_native_transport: true
native_transport_port_ssl: 9142
column_index_size_in_kb: 4
saved_caches_directory: build/test/cassandra/saved_caches
data_file_directories:
- - build/test/cassandra/data
+- build/test/cassandra/data
disk_access_mode: mmap
seed_provider:
- - class_name: org.apache.cassandra.locator.SimpleSeedProvider
- parameters:
- - seeds: "127.0.0.1"
+- class_name: org.apache.cassandra.locator.SimpleSeedProvider
+ parameters:
+ - seeds: "127.0.0.1:7010"
endpoint_snitch: org.apache.cassandra.locator.SimpleSnitch
dynamic_snitch: true
-request_scheduler: org.apache.cassandra.scheduler.RoundRobinScheduler
-request_scheduler_id: keyspace
server_encryption_options:
- internode_encryption: none
- keystore: conf/.keystore
- keystore_password: cassandra
- truststore: conf/.truststore
- truststore_password: cassandra
+ internode_encryption: none
+ keystore: conf/.keystore
+ keystore_password: cassandra
+ truststore: conf/.truststore
+ truststore_password: cassandra
incremental_backups: true
concurrent_compactors: 4
compaction_throughput_mb_per_sec: 0
@@ -45,6 +45,10 @@
enable_user_defined_functions: true
enable_scripted_user_defined_functions: true
prepared_statements_cache_size_mb: 1
+corrupted_tombstone_strategy: exception
+stream_entire_sstables: true
+stream_throughput_outbound_megabits_per_sec: 200000000
+
client_encryption_options:
enabled: true
# If enabled and optional is set to true encrypted and unencrypted connections are handled.
diff --git a/test/data/CASSANDRA-15313/lz4-jvm-crash-failure.txt b/test/data/CASSANDRA-15313/lz4-jvm-crash-failure.txt
new file mode 100644
index 0000000..0f64d84
--- /dev/null
+++ b/test/data/CASSANDRA-15313/lz4-jvm-crash-failure.txt
@@ -0,0 +1 @@
+656465656364656365656464636364636563646563646365636563646465656564656363646364656565646563636463656465646464646565656365646365636564646465636563656464636563656363646364646465656363636364646364636464656464646565646363656563636365646363646365636563646464646463656564646365656565656365646465656463656564646563646365656563656465656465646463636563656364646364656565656464636563656363646363656363656563636464656365646364656565636463656465646565646563646363636464656563646565646464646563656565636365656465656463656464646465636564636565646365656563656465646463636463646364636564646364646565656363636564646563636465636363646464646363656465636564636564656364646364636563636563656363646364656463656463636463646365646364646463636363656463656563656564646463656363636363656565646464646563656365656464656465656363646565636565636463656563646564646363646565636364656363636564656364656363636563636464646365636564636365636565636463646465656565646364636564656365646363636563656563646563636365656363656463646465656564646564636563636365656563656565656465656463636365646465646563636564646563636463636464656363656365636565656465646365646465636465656565636563656465656365636565656365636463636563636363646363656563636464656364636365636464656465656363656463646364636363636464636463656464656563636363646565646465646464646563646365636463646564656365636365646363656463636464636365656563636563636565656463646463636463636464656363636364636564646363656364646364636364646465636365646464636465646364646464656565656465646563636363636463636363656365656563656464646564636563656464646564656363636363646463646365656464646363656564646464636465646463636364636463656464636564646565656564656565636365636465646465646463636463656363656365656465636463636565646464636464656364636364636364646565636465636363636465636365656363646563646563636564636564656464646564656565646365636464636464656463656565636564656465646363646364656465656363646565646565646365646564646463636364656464646565636463636464636363646565636365636565646563636363646563646565646364656463656465646463636564656365646363646564636363646463656365656363656363646565636465636364656464646464656365646363656463646563646564636463636565656363656463656563646365646364636564646465636365656463646563646363636364636564636565656365636465656565646465656463656564646365656465636565646364646364636565646365636365636363646363656365646465656564646363646464636564636564636365646363646465646363656464636365636364646465646365656364656365636364646565636565656463636363636365636464646364646365636564636563646464636363646464636465646463656564636363636565636364656364636563656364656363636363636363636365656364656363636465646563646363636464656464656364646463646563646365656463656365636465636465646564636464636364636364646463656463656465656563646364656464636563646365656463636465646363656563656363636364656465646363656565646363636565656363646465646465656364646565656463656365646564656465636364646563646363646465656465646363636565636563636364646464646364636364656563656365656465636465656564636565636563646364636465636464646463656465656464636365636464636364656364656464656365646363646365646565656463656565636465646563656565646463636564656464636363636463636364656464646364636363636465646363646563656465636463646463636465656563656363646563646564656363646565646363656365646464636563656565646463656464656365646463656565646365646465646465656565636564636564636563656565656464636564646565636363656363636463646363656365646564636563646563646363636565646463646563656365636565656465646365646365636364636565646464636564646465656363656365656365636464636565636465636365636365646463636363636563656364636365646565656565656563636464636364656463656465656463646364646463646365656363636565656364636364656465636364636463646564646565646465656564636364636364656463646463636364636464656565646364656564636464636565636464636464656464636465656465636365646565646364636465636363636365636363656564636363656464646463636363646364636465636564636365646363636364646363646464636565646463646364656364656465646463656565636464646563646463656365636463656363636463636463656465646464636365646364656465636564636364636463636463636563636464646465646564656365646564636463646365646465636464656465646363656563636564656365636464646464636565656563646363656363656564646364656565656363656564646363646564636463646364646563636465636365646365656365646365656565636564646363656563636563656364636363646464646564656564636363636565636365646565636563656365636365646464656563656563656363656363656465656363656465636563656563646465656463636463656564656463656563656464646464646564656365636365656365656565646564636464656465636563636364636463636363646565656364656365656465636463636563646565656364636564646364656464646563636565646463646463656464636563656365656565646463656364636564636563636464656364646363636564636565656563656464636463636463636564646565656463646364636565646365646465646364656463656564646364646563646464646565646463636563636565656563656364656565646563656563646464646463636564636363656563646465646565656465636465646463656365656565636463646363646364636464646563636463656364656363656465646363646364656465646365646364656363646463656465656463636563646365636565656564636564646363636463636464646365646364656564656363656563636564636465636465656565636363656563646563646364646465646564636465636364656363646463636363656363656564636365636464646464646463636365636364656563656464656564636363646465656363636565646463656563656464656464656565656365646564656363636365646364656363646364646363656463646564636364646363636463646463656564646365636463646363646363636465636365656563636563656563656363656563656563656364656564636364656564646463646365656565636563656364646465646465636563636365636363656465646365646364646463636365656365656463636464636465656565636465636465656364656363646463656363636565656365646363656564646363646365646564656465636365636463646464656364646363656363656364636564636464636363646364636564636365636365656465636363646465656365636464646365646465646463636464636363636463656463636364646465636365636565646463646463646364636564636564656564646364656563636463656464636364646564636363636565656364636365636363656463636465646364636563636563646464636464636465646565656463636364636363646463636565636465646363636565646564636363636363636464656464656363656565656365656563636364656364656463646364636463656363636363636465636565646363646363636465646363656365646365636364656364656463656464636465656463646463656563656463646363636463636464646365646365636564636365636565646364636365636364656365636464646365636563636563656363636365646565636464646563646563646365656365646364656565646564636463636564636465636364646363646465646465646463646365646363656465646564656563646563656565636565646364636365656365646364636365646564646364636465636563656563646365656465656364636464636465646463646463646365636365656564656563636363636563646365636564646463636465656363656363656363636563636465656365636564636465646365646365656564656465656363656463646464636465636565656563646564646565656464646365636563636363636364636363636564646464646464636564656363656365636563636564656363646563646464636565646563656364656363646364636363656365636464656464646463636364636463636465656564646365636563636464646463656563636463656363646465656564636565646563636463646565656364636365646364656463636563636565636463636464636365656464636564636563636464656565656465656465636464656464646363646565656363636565656365636563636465636565646463646363636464646565646365646563636463636565636465636565636463646563646465636463636465646465636564646365646464656464656364646563656365646563656463636565656363646363646565636464656363646465636563636463656464646364646463636364636563656563646364646365636463636463636363646463646563656463646464656563656364636363636464656363656565636465636364636464636463656464646464636563656363646364646463646463636464646565636464646465646463636564636365656563646465636564636365646364646464636464646463646565646565656463636365656363656563656364636465636364656464636363646364646363636563656463636564646563656463646364646465636565656564656563646464646365646363656365656464646465646364636563646364646563646464646563646365646563656464656565656365646463636565646464646463646463656365646464646365656364656363636564646364636364646463646363656563656364636463656363636465656365636463656364656464636364636363646465636364656565636363636364656565646565636463656565636465636463656464646363656564646565656565646363656563646365636363636363646365646464656364656564646465656363656564656465636465656463636465646363646463656564636363636463646365636563656465656364636463656465646565646463646563656365646363646464636564646463646563646464636463646564636365646363636364656364656565646463646564646363656564636463646464636463636365646563636465656364636364646365656565636563636564656565646365656365646463656564646565636363636465636465656363636365636464656463636464656565656464656565636363636464646463656465656565656463646565646365656563636464646563636465646363646365636565656365646565636563656465656364636564656464636564656365656563646563656364636464656365636363636564646563646364656363636363646465636363646563656363646564656465646465656565636463656563636465646463636363646363646364646564656365656563656464636365636363656565636564656565636563656563636363646463656563656365636565646565646364646365646465636363646565646363636464646463656465636463656363646463656563636563636464636563636463646365646365646565656463636364646465646565656463636464636363636563646364656563656365646463656465646365636464646365656564636363656365656564656565636463656364656463646463636463656464646463646464646465656364636465636563636564636565636363656464646565636465636365636363636365636363636364636465656364646563656563646463636564646364636464636563636365646564636365646563646465636565636565636565636563636465656364656565636365646563636363646464636463656464656564656463636465646465636363646463646464636564646564636363636564656463636563636464646463656565646463656563656563656564656464656464646564656464636364636365656465646363656465646465636564646363656464636363636565636565636363636465656565646464656564646563656465636363656363636365656365636365636465636465636463656363646363656465656365636564656364636464656563646365656365636364636364636565656464656464656465656464656565636364656463636563636363656365656464636563656565646463646465656565646564656363646464636465646363656564636365646464636363636464656364636365656564656565646563656363656563656563656463636463656563656363656464636365646363656464646365646465646363646364636463636564636563656364636364636563636563636463636364636563656364636564646465656364656564646465646463656565636563646565656563656363646563646463656463636365636463636364646363656563656365636564646363636364646465656463656564646365646364636463636565656465646564636365656363656564646564636463636365656565656564646563636363646363656365636465656364656463656464646363636465646363636563636364646565656363636563646365646564656365646464656564646365636464656463646365656563646365636565646363636465656465646465636463656365656563636464636564656363646464646565636365636463636365656463636364646363646564646564636463646364646565646463656365656465636365646363646363636364656365636465636364646363636564656365636465656364656364656465646563636365646565656363656464646565636364646465646363636365646463646463646365646463646364646363646364646464646464646464636563646365636465636463636363636364646364636365636464656564636463646465636563656463656564656464636365656565636464636563636363656364646464636463656364646365646565646363656363656365646563656364656463636464656365646563656464656564646564646564636563636464646363656463636363656564646364656464646563646565646563636365646364646463656563636363656463636363636365636563646364636464656463646463636364656465646364646564656464636363646365636464656464636465656565646363636365636363636364646465646463656564636363656565636364636364636565656564646365646463646365636564636565636363646463656464636364646565636363646465646563636564646565636363656565646563636563656464636563636464656563656465656565636365636365646364646363636364656364656563636363646564636565646564656464646364636564646364636364646564656364636465656463636463656365636465656564646464646463646564646465646464656464646463646365656364636563656463646463656563656565636465656563636363636465656364636563656363656464636364646564646564636463646364656563636563656363636564646364646565636564636563646465636563646563636464656364656364656365636563636563636563636464656464646365646564636463656565646365636465636563656465656464646464656364646464636363646465646465636464656363646363656465656464656563656563636465636564636464646563636363636365646464636364656464656463656564656563646465636565656465656463656564646364636464646363636365636364646364656364656563656464646363656365646363636363636363646364646463636465656563646465636565646365656464656364646465646363656463646363636563656563656565636563636364636364646563646465636565636565636464656364656364646463646463636463646465656564636464636465646563646364656363656364646363646365656464636465646563656365646463636565646465646364646465646363646464646463646363656565646463646363636563656363656565656564656364656365646565636563656564636464636565646464636565656363646363636565656464656363656563646563646464636363646465636465656465636464646363646363656565646563646363646465636363656564646365646365636564646565646363636363656364656465656364646365646564646363656465656364636563646365646365636565656563656565636563636564656464656365636565656464656363636563656465646563656364656564656364636365646363636464656364656364646565646464656563636365656365636563646364636465646564656563636463646564656563636465636464636363636463656464636363636463646365646565646363636564636363656364656463636465636565656365656363636365636563646463646464656464636463636465656563636363646564646563646365646564636464636464656564646565646565656465636363636565636464636565636465646565646365636363656565636565636464636363646564656564656464656563646363636363646565656363646563646464636564636563636563656363656463646365636564646363656563646365646463646565636365636364636465656564636465646365636563646365646464656364636364646363656565656365636565636563656565656463646365646464656463656363646364636365636565656565636565656464636363646563636365636363636363636365636365636463646564636563646564656465646463636464636564636563656564646365656363646364656364646564656465656365656363646463636563656563656463636465636465636463636563636365646365656563646364636465636365656364656363646565636465656565636363646563646365636463656564646364646463636464656365646564636564636565656365636364646363636564656463656363646363646363636563636565636465646364636363646465646363646364636563636463656465636464656465636364656565646464636563636363636363636564636464646565636463636363636364636465646463656364656464636565646363636365646465636564636464636563646565656463636465636463636365646365636464636363646564646464656365636363646364636365636563646363646364646563646364646463636463646465636365636364646464646465646363646465636565636365636364656463656364646364646465656464656364656364636563646464656363636464656365636364636364636564636365646463656464646364636365636564646364646565636463646364636464646564636564646563636365636565646363636364636363646364636365636465636363646463636564656563656365646464646564656363636565656364646463656563636365646364646565636465636563646463646564656465656364656364656464656464636364646364646364636465656363646363646464646365656464636365636463646465646363656565646464636465646464656563636465656465656365656465646363646565646365636465656465656363646364636565646363636365646565656365646564646365656465646563646564656365636463636563656365656565636563646465646563646465646563646463636465636464646563646564646564636563656463636363636463636365636564646363656464646465646565636364646363636564656364656365636564636465646563636463646365636364646463646363646563646363656463656465646563656465656565646563646563656563656364636565656563636363656363656563646365656365646365646363636363646464656563636464636364636364646465646363636363636463646565636464646365646364636365646464656465646463656365646465646565646365646464646563636364656564656564636365636463646563636465636563656365656465636565656565656464656463646465636465646463636365636463636363636363656365656564656465636363656363646565656565646364646465656364636564646465636565636565636363636563646364636464636463636565656563646365656363656463646463636465656364636465636465646463656565646463656565636564636464646364636565646565636564656564656563646363656363656364636465646563656563656564636463646464646464646365646463656363656364656363656363636364646464646364656465646563636465636464646363656463646465656363656563636464646565646463636364636565646563646563636363656563656464636565646465656564646565656464656463656563656464656463636363646463636365636464636363636564646565646365656565646363646465646364636465636463636565646563656364656465656564636365636364656364646565646365646365646565656463656364636565636364646364656463646464656464636563636363636565646465656465646465646464646365646363646363656464656465646364646364636365646365636364636564656363646564656565636563656464656365656465646465636463646464636463636365636465646564646364636363636563636464646465636364656365636565656464636564636563636463636565646463656463636364636465656465656565636363646464646465636563646563636463656365636563646363646363636364656364636364646365646463636463636364646365646564656564646565656564646365656464636365636464646563656563656565636365656364636365656563636364636364656463636464636463656365646463636364656363646365656563636463636363646463636563646464646364656563646565636563656365656363636463646363646364646364646464646563636363646465636564656564646564646465656563636463646565636563646563656563656465636363636564636565656563656364636565646363646464656464646563646365656465646563656564646565646563646365636364636463646364636464656565646565656463636465656564656365656365636364656564636363646564646463636364646565656463636564636364646565656365636564656463636463636464656465656563646363636363646464656563646564636464646563636564636564656565646564656363636363646463656464656563656563656564636363646463636363656363646365646464656564656464656565646464646464636465636363646464636464636365636463646364656464646364636463656465646364656564646463646464646363646465646364646563656365646363636563636565656464656565636465636565656465636364656463646365636564636464646365636365636365646563636563636463656565646563656565646564656565636363636465636465646565646463646565636464636464646563646565656464656465636464636565656564646463646463656563656464636365656564646463646464636565656365656565656364646464636365636465636464656363646563656464656463636364636563646563636363636363656364636563636463646365656465656563646365636563656364656564636365656463646363656463646364646365646363646363636564646363636564636565656363656364656564636564656464646364636464646364636365636463656463636463646564646363646365636363646463656565656365656563646564636565636465646563656364656363656365656563636365646565656364656565656563636563636464646364636364636365636463636564636363636564646364646463636563636563636363636365636365636563646563636465636364646565656563656365636464636363646465636463646464646365656365646364646365656463656563636464636365656365656563656465656463646463646464646363646363636563646564656565646565656364646563656563636465636364646563656365646363646363646564636365636363646463636465636364656465636464646463636365656465646363636365636464636563636365636365636363636563646465636465656465656363636364636464656363636363656363636565656464636465656465636464656565656364636465656563636565656464636364646565636365656463656365646365636364656564656563646565656463636565636564636463636465636364656365646364646465656365656465646365656565636465656363646464646365656464636564636465636364646365656464656465636363636565636565636565636464636565636364646465656563636564636565646464636563636363646464646364646564656465646365636565646565646563656564656564646463656465656364636564656565636463636463646465636363656464636564656363656564636563656565636465656563656563636365636564646465636365656365646464656465636464646565646364636464646563646563646364636365636463636564656463656564656463636463646463646563646563656565646565646363646464656563636363656564636464646463646464646564656464656564656565636465656365636565636464646365646565636465646564656463646564646563656363656363656565646363636364646563646464646563646363646465636363656363636365656565646563636465656363646465646363636463636463646363646463636365656464646564656363656463636563636365656463636365656565646564636365636463656465636364656564646463656565656363646363656365646563636463636564656365646464646465646463646563656565646563656364646365646365636365636565636464636563636565646364656563636363646364646363646565656465646464656564636464646464646463646563646463646363646365646365646463656465656565646565646464636465636463656564646364656563636463646463646364656363646563636465646363656564636464636565646464656463656465636464656564656563646465646565646464656563636464656365636465656565646563646565656464656463656465656564636465646365656465646464646564656563646563656564656365646465656564656463656463646563646465636463636364656564656464656463636364646565646364646463656465656564636464646563656365656464636465656464656565656365646563636565656563636363636464646365636564646364636465636463636465646364656363656365636465636363656565656565646565646564656365646565656364636464636365656465656463636564646564656364646464656563636465646464636364636464656465646563646565646564636363656364646363646565656563646564656365656564636563646465656563656364636464636365656463656465636363636464636465656363636365646463656565656363656363646463636463646564656565656563636564636565656365656463636364636365636563656364656565636465656465636464646465656363646563646363646464656364656564646565656364636365636564646364636363636363636365636465636563646463636563646364646564656465636565636365656463636363636363646364636464646363646465646563636464636564656563636563636565646563646563646464646465646465656463636364636365646565646464646464636463646364656463636563656364656565646564636364636364656563646365646363646565646563636365636363646563646565636364636564636564656564656565646563636365646363646464656565656565636565636465646563636463646365646364636363646464636564646365646363656464646363636364646465636565656463636564646465656365636363646363646565646564636564636363646464636465636365636465636565636363636563636464646464636363636565636465636463656365636363656564656464656563656564646363656563646464646563656563646365646565646364656364646465656563636365656364636565636563646363656363636364636364636464646565646463636565646465636463636364636463646363646365636464656363646564636565636365646565646363656363646565636464656463656363636563646563656364646464656463646364636465656363636364636565636364656564656463646364656364646363636463646365636465646565646563646463656364636365656565656563656563656463646364656464636463636364646563656365636364636363646564646564636365646565656564656564656464656465636364656365656463646365636563636563636464636563646565636565636464646563636565646463636463646565636365656364646463636364656563636563656365646464646464636363656365656565646363646364636365646363636465646565656363656463636363636564656563656363646465656563646463636363656465656363646463636565646565656465646563656364636565636365646363636464646364656363646365636364656363636363636463636564636364656465636363646564656564636563636565646464636365656365656463636463636363646364636463656463656565636363646363656563646563636463636364646363656463636363636564656565636565646463656363636364636565636464646365646564656463656365636363636565656364656464646564656465636563646463656465656363636365636464656465656564636564656463646565656463636365646463636365656463636465636464656463636364636564636465646565656364646363646564656464636463656563646464656363656463656564646565636364636463636364646364636465656364646365636465656565636465656363636465646364636363636365656365636363656365656363646465656463656563646464646363656565636363636464646465656465636363646365646364636365656365656563636365656463646564636564656364636364656363636565636564636563646463646564636464646465656565646363636363636463636463656464636463656564646365636363646364646564636464656463636563646364656463646565656364656364646463656364646564636565656464656363656564656363656365636563646365656464656365636464646463646563656363636465656465646563656364636563636464646364636564646365656365646363636464636564636464636564636464636364636363636563656564636464636363636365656464656365636364636365656465646565646564636463636465646563646465636365646463646564656463636363646365636365636564646465636463636463646463636564636564656463656363636365636364656365636563636364656563656464636365636564646565636365636463646365656564656364636463656564646563656364636563636363636364646464646363636365646463646363656365646563636363656463646463646564656364656465636463646365646464646563646464636564636363636465636463656463636364656563636463656463636465646565656463656563636563636565656564636564646364646563636563656464646463656364656464636565656363656365646464656565636464636363636365646464636363656364646563656565646363656464646363646565646465636465646563656465646465646563646363656464636363656463636363636563656465636364656363636363636464646564636363656363646363646365646565636463636364636563636465636365646463636463636364646364636364636565646363656364646365646464656565656563636364656563656363656564636465646564636464656363656365646464656564646465636565646564646463656464636463656365646565636364636365656365636363656565646565646464646565656463656464636363636565656365636463656465656464656465656563656464646364636563656364646463656363636563646363636363636363656364656364636565646365656465636464656465646363636464646463656463656463636463646565656465656364636564636563636363636463656364636364636364646563656465656564636365636464646564646463646565636565656364656465646464656463656564636465636464656365636565656563646563646565636465636564636564646365646363656463636465636365636564656565636363636364636463646364646563636565656565636465646363646565636364656465656363636363646563636463636565656364646363656464636564646465646365656364636363636363656465636465636563646364656363656365646565646464636564636464656463656364656565656363636364636464656564636564656564656365656564656563656563646464636363636564636365636364656365636463646364646464656464646563636463646463646365636465636364656465656565636564636564636565636363656564636465636364646464646563646564636465646463646363636463646464636465646563636565636464646464646465646464646364646564636364636464646464656363656363636365646363646464646364646365646464646365636465646565656364646464646564636465636364636363656363646565646465646363656465656563646463636364646464636565656465636464656363656563636363656364646465646565636565636564636463656465656563656364636563646464646465656564636365636363646465636563646363646363656363636365646364636465656463636464646464656563636463636364656364656565646565656563636565656564636465656365646563646565646364656563656363636363656365646364656464636464656365646364646465636565636465636365636565646563636465656563646365646464656465646464646563656563656364656465636564646464636363656564656464636365636563636465636463636565636565656363646363636363656463656363656463646465646363656363646464636565646464656463636364636365636464656365646364646364636363636563646365636464636463636463646364646465646563636564636564656565636565656363656563636564636364656364636463636365646364636465636565636465646365656363636364656463656464646365636363646365636463636364636563656464636564646364656464646564656564656565636564656464646363646465656463646465636565656465656564636364636464646564646463646565636464636563636363656464636465656363656563646564646563636365646365656464656364646365656564656564656564636365636564636464636565646564636564656464646464656463636563656363636464656363656563636563636565646463636464646565636363646364656365636363656563636463646465636463656365656564646464656364636465646465636563636565656465636565636463646465636365636563636564646463656464646364656463646365656464646563656565646563646465646365646465646564646363656364636465656565656364646563656464636563646365656365646564636463636364636463646563646465656564646364656565646465636363646564646464636565636565636565646363636364636363656363646465646363656463656563636364636464656563656563646563646565636465656364656363636463636565636364656565656365636364646363636564646364646465636564636565656463636565646563646365646364636364646363636564646563656363656464646364646363646365646565646465646364656364656563656364636464656464646565636565656364646364656365646364656563636364636363656363636363636563656563636463646365646564646464636364656365636363656565646363636464656363636365646564656363646364636363646463646564646464636365656565656564646465656465646563646363636563656363646464636363646364636465646565656563636365646364636465646464636365646563636464646363656465656365646364646465656565646563636365636464656565636465656564646363646563646464656365656363636463636463636464656563636563646564646565656464636563646563646465656364646464646465636463646463636463656465636364646463656363656463646464636565636565646463656365636365646565656364656464636363636563636364636565656564646465636463636363656464636564636365636564646365636464646365656563636363636463656563646363646463636565646565646363646464636565646564656363636563636364646465646463636365646464646463636464656365636365656365636464656465646364636564636565636565636565646463656564636563646464636563636463646363646364636465636363646464646563646365646463656364636363636465656563636564656563656464636463636463636364646464646363656563636464636465656364646465656365656364656564646465656565656565656563636565646364656564646365646363646364636365636565656363656565636465656464636565646465656464646564636565646465656363646364636463656365646365656465646563646463636365646563646463656564656463646464646465636464636463646464636365656365656564656464646365636564656363636564646365646363636364656365646463656365656563636563636364636363646465636363636563636364656463656564656563636364646363636463646464656365656365636463656463636463656364656564636463656464646464646363656464646564656464656464636465636565636365656564646363656565646365646563656464636463646363646465636364646565636465646563646563636563646464646464656363656364636363656464636563636465656564646565636463656563646364656463646564636564656463636463636363656563636363636465656464636465636564646464656563636463656565646565646365646363656564636365636364636363636364656565656563646564636564636365646364646564656565646365636463636364656363646465656465656563636464656365646363636465636465656364646465656363656564646463636464656365636565656465646363646365636363656565656463656363656465646365636563636365656564636464646363646563636564636465646365656563636564656463656564646464656364646563636364636565646463636465656563636563656465636564636363656463646564646465656363646463656364636363636464656565636564656364636363656363646365656464646364656565656364646463646363656564636563636565656564636565656563646363646564646464656363636464646563636563656365656363646364636565646363656365656464636465636565646565636464656465646365646463656364656363636365646464646365636463656463636463656365646464656365636563636564636563646363656465636565636464656565656565656463656464656463656365646465636465646563636365636465636465646463636464656463656365646563636365646464646565656464646365636465656464646463656563656464656464646363646565636363646564646364646564646563636565646465636365636565646564656364636364636365656564636563646563656463646464646364636363646563646565646363636564656364646464646363656565656563646464636464636365656564656464636364656463636564646463646563646564636564656365656363646364646363656565646564636363636364656565656464656565636464646465646565646465656563636464646365656565646565636363636465646363646365636364646464656563646464646564656564646464636465636364636464656364656563656463636464636364646565636464646363636365656365656463646365656463646364636464646464656463656464636363656364646363636564656364656363636364656465646365656363636464646363656463646365656464636464646363646365636564646364656565636564656565656564646564636463646364636364646365646564656564656564646363646365636463636564646563636565646363636364646364636563656363636364646463656365646465646364646364656363636563656464646463636463636564656365646363656365646365646465656563646563636464656364646364636565656564656563646463646463636563646364646364656363656364646364636363646364656363636365656463656465636563636365656364656363636364656363636563636464646365636363646365646363646465636563636363646364656564646363656363636365636463646365636565636465656365646365656365636563636464636463656464656363656563646563656463636463646463636365656563656465646365656463656464636563636363636565636365656464636563636365636563636363636563646563646465646463636464656463656464646464646563656363636465646565636465636563636463636565656365656465656463656463646563636564636365636464656363646564646464656465656364636365646464656363636564636463636563656464656463636565656465656564646365636565646564656463636363636365656364656563656563636463646463656365656365636563646363656565636363656465656363636363646565646565636363646365656465636564656565636465656365646564646364646463636564646564646464646463646364636563636364636363656363636563646465656565656564646464646563656463656365636565656363656463636464656564656365646463656563656364646564646563646565636463646464646563656464656464636364646365646464636363646563646564646565636363636364656465646363636563656464636564646363636463656464656464656564646563656363636563656364656364656363656363656464636565636564636465656564636363646565636464646565636465656363636463646565636563646463656565636464646564646365646364636564646364646365636464646563656363646363656463636563646464636363636463646464656363646465636564656465656463646464656363656363656564636465656364656464656464646564636364636364636565646463646563656564656564636465646364646363636465656364656563636563656564656464646363636363656364636463656564646464656564646564636465656365636564646564656464656563636563636463636363636565636563646464656464656365646364656564636364656364636465656565656463646565656365636563656363656565656365646365646563656463656363656465646363646463636465636363656463656464646364636364656364646565646463646564636465656465656563636464656564636563646463636564636363656565656465636364636463646464636363656364646463636365636463646463636464646363636363656564636363646465636464656464636464656464656464656463656364646565636363636364656464646563636364646564656565636464636464636463646463636463656364656563636564636563646564636564656364636364636564646564636364646365646364636564636563656565646463656464656564656565646364646563636364636463636364646363646564656365656464646465636365636564636464646363646465656564636363646365646364646363646464656565636563636364646563656365636363636364656365636365636363656564656564636364646365646464646564656463656465656364646363646363646463636464636463646564656463646463636365636364656565656565646365636564646364656365656463656363646563656364636463646564646365636464646465646465636563656364646464636563636563636364636563636563636565656364656565646363656464646463636364636364636465646565636363636363656363636465646563636565636465646364656463636363646365636463636364656364636365646465646464646363646563656464646564646363646364646465656364636564646563636363656563636563646463636363656564656564656465636365656564656365646465656564636463636463646465646565646465656364636364646365636564636364656563636364646564646463646364636365636363646365656363646363646565656363656364636463656363646464656563646563636365636565636364636365636564646565656365646464636364656465646565646363646363636465646563646365636463656563656563646563646463656565636563646565636564636363646465646465646464646363656363636464636363646365636364646563636465656463636465646364636365656563646565636364636464636364636564646565636365656463636363636465636364656363656565636463636364636364656363646365646464656364656563646365646463656365646365646364656565656363656464636364656364636363636464636364656465646365636364636563656564656563636463636564646363636563646465636463656563636563636563646363646563646365636564656563636465646464636465646364646563656564636563636365636365656563636363646463656564636464646363646564646464646464636563646565636464636563646563656563656565656565646563656363636565656364636564656565636364636364656363636463656563646465646564636465656563656365646563656465646564636565636465656464656565636363656365646463636463636363636565636464646563646464646364646463646564656564636363636563646363636563636565636564646464656363646464636563656463636464636364646463656463636364646363656364656365656464636564656564646363646464656363656463646363636564636465656464636364636365636365636564636465636564646364656465636363636363646364636363656363646363656364636465636564646463656364646364636464656465646364646565646463646363636463656363636364646564656363656365656463656363646464656363636563636364646563636464656363636463656565646364646464656465636364636365636363656463646364636563656364636365636564636363636365646564656365656464646364646564636464636465656464646365636363646364646564646463646464636463636365636565636363646363646564636363656463656564646564636464656365656565636364656363656364646564656364636565646365636363656364636365636463646565646464646465656465636564646564656565636365646564646364656364636465636563646364656364656564636564656465646465646564656465646364646365636564646564656565646363656565636464646564646564636565646464646365646365636465636365646564646363636565636563646364656563646565646464646463656464636564646563636564636363646365656363656464636363636564646463646463656365656464636364656464646365636363656464636563646463636464636365636565646463646565656563646564636363636463646563646463656365636363646463656365656365636365646564636465656465646365636464636364646463646364636564636363636464636364646565656463656464636365656565656363656365636563646463646364656464636364646363646463656465656564656363636363636465656463656365656465646564646364646464646565636363636563636465636564646365646565636465636364656564636363636364656465636564656364636364636464646565636564636464646564646465636563656563646565656363636363636363656565656463656364636363656465636364646365636364656465636564656364656565656365656464646365636465646565636363646365636365636463636464656563636365646365636564636565646463656463656364646564636564646363656365656563636363636363646364646564656564636363656565646565656464646465636363636565636564636465636563636465646563646364656563636563656363656363636463636465646464636364636564636364656464636365636563646363656364656565646564646563656465646565646465636464636564656565646465656464636463636565646563636563656363636564646564636565646463656365656564656363646565636563646364656465646463656565646565636464636464636365646364646365646365646464656463636364656363646363636365636465636363636563646565656465656363656365636563646464636465636565636563646464646563636563646365646364636465636464636364636565636363636364656364656365636363636463636563636463636565656463636563656563656363646463656563636365656465656565656563646563636364646565656565656364636464656363656365636463646465656365656565636363646465646465636563646563646365636563636363656564636565636464646565646563656563636565646364656465646464646564656365636464636364646364636364646565656464636364646463656563656464656565656464636565656463636363656365636463636364636364656563636465656363646563656464646465646364636563656364636464646465636363656563646364636465646363636565656365636364646365656564646564656465636365646463656363636565636564636564636464656463636464656363646464646564646363636364636563656365636363636365636365636363636363646364636363646463656565656364636465636465656364636364656564656463636364656464636463656563636563646563646464646364646465646464646563646464656463646465646464656465656564636464656365646363646465646464646363656463636464636465646564656363646465656364656364656563656463656465656363636364636464656564646463656364636463646465656364646563656463656364656464636464646563646565646363646465646364646365646564646365656363646565656563656365636364636563646465656465636364656563636365636465636363636363636363656563646563646463656563656564636364646364656565646365636465636365646463636464656363636564646364636364636564636463646363646364636463656464656364646364646365646365656565636464656463636363636363636363646364646463636565636564636364636465636563646463656363646565646564656464646465646463646363656565646565656465636363656363646564636364656463636364636563636365646565636463636365656465646463646365646365636465636564656565636464646564636463636365636364646463646563656363636364636465636464656365656464646464646463636565646464646363646465646363656464636563646363656364636363646365636365636565636364646365646363656565636464646563646465636565646565646464636364646464646365636465646565656363646464656464656364636363646364656463646564646563636463636465656365636463646563636363656365636564636464636465646363636565646563646365636565646563646564646565656564656564646365646565636365636464656463656465656464656564656465646364646363646563636565636365636464636364636563656565646363646465646463636464636365636463656365636363656465646364636563646563656563656464646464636365646563656465646363636563646364636364656464636363636363636464646463656463636564636364656463636464646464656465646365656563646463646563646363656463636464636565656565646465646463636465646565636365656363656565646364636365656464646564636365636463636364656364646465636464646464646463656463646564646465636363656464636465656363656564646564636363636464646565646364656563636364636364656363646464636563636465646564656564646564636365646365646463656563636363656363656563646363636364636565646564636565636564656464636464646563636463646465636465656565656465636463656464656463646564646563636563646465636364636365646364656465656363646364636363646365636365636565656465656464636563646563636565636363656464656464656363656364656363656565636364636365656465636465646365636464646463646564636365656363646464656364646364636363656363636565646563636465656565656565636364636463646565646365646464656365656363646464636563636363636565646363656565646564656563646365646564636564656464656363636563646465646563656564636465646563646463646465656363646363646565646363636563656364646564646463636463646365636564636464636563646564636564646363636363636365646363636464636565656563656363656564636465636363656364636565646564646563636563636565646465636365636563646563636565636563636564636364636563646364646465656565646563636463646464636563656365656464646363656363636564646464656565656564646465636564636564656363636463636363656563656565656463656363656463636564646365646563646363656563656463636565656364646564636563656364656465636365646364656364636563656465656464636364636565646463646563646464636365656565636563656463646364656564646463646464646565656364636364646363636564656363656564646365646564656463656364656464646465646565646565636363646364636564636564656564656363646563636363656565656364636363656363646564656464646565656464656563656364636363656464636465656365636463636564646565636563646364646363636365656463656564656364646464646465636365656563646464656464646563656564646564656463636564636364656464636563636365646465646565656463656365636564636463636464636364656565646563646563636464656463636565646564636364656464656464646464636464656464636564656564636464636363656364636565636365656463646365636364656465656465646463646465656365646464646563656565656563646465646565646564646465636563636563646465636465646365646565656464656464646365656563636363646365636463646464656463646464656463656363656364646563646564646365636565636465636563636363656464646364636363656464656365656565646463636464646464646363636365636364656363656464646465656464636563646363646564656464656364646464636365636463636364636363646565646363656564636464646465656364646464646364636365646463646463656363636364646563656464636463656363646564646465646464636563656463636365636465656463656465646565636464636565656463636563656365656464646364646463646465646363656364656364636464636365636564646464636464656364646564636365646564636464646365656465656465636364646464636364636464636465656565636464636463636564646565636363646363636565636564636365656363636565636364646464646365656364646363636363636363636465646565636365646563636463656365636364646465636465656364656563656364656365656363636463636563636465646464656364636463646463636565636465646363646565636465656465656363656463656563656463636365636563636565646363646465646565636464656465656365646463636463646364656465636463656364636464646564646364656464646463646563656565646463656365656564636563636463656364646564636465646363656565636463636464636563636364636563646464646363636364656564636364646565636563656565656363656565656463646364656365656565636563646563636464656363636365646563646565656463656363646565646363656565656564656464656565636463636564656365646464646365636464656464636363656565656464636464656363656563656464646464646565656365656365656465636564656365636463656564636465636564636565636465636364636563646363646563636464636363656363646465636463636364656365656364656564646464636365656464636464656565646564646364636564656363656463656565656563636364656564656365636563636565656563636465656363646465636565656364636563656365646364656363656365656363636363646565656565636365656464656564646564636463636464636465656364636364646564636465646363636464656363636363636463656564646363656364646363646563636464646464636563636565646363636364656364656564636563656565636464646363636565646465636465636363646363646564646563646563646563656364646563636465636363656463646464656364656364636564656564646363656564636364636365646563636364646563656465656464636463646565646463646365636565636364656363646563636363646464646464646365646464646464636464636465636464646565636365656363636363646563656565656565656463646464646563646463646465646364656563656365646365656463636464636564656565636365636465636364646564646465636465656463656363656364646563636464646364646364646363636565646465636565636363636465646464646365656365646465656463646364636464656563656465646563636363656463646365636463646464646565656464646464656565656563636364636364656364656465646364656364636463646563646363646563656465646363636363646565646464646563646565646364636563656464636363656364656464636464636465646465646463636563656565646464656363636365646464636363656363636465646365636363646463636363646464636365656564646364646563636363656463656465646563636563656363646465636563636563646565656464646364646464636363656563636465636565656365646465636463636463646565646563646464646365656464656563656463646463646464656564636465646465656563656365636463656563656564646464656365646364656364646464646365656563656463636463636463636464636565656365646564656365656363636463646363646364636564656565656464646564636365656465656364656363646364636465656465636363656364656463646365646364636565636365646463646464656464646364646363636463646563636564636365656363636363656465636464656365646363646363656363636463656464636365636465656463636465656365636365646464646563646563636563646563636365636464636464646363656464646563636364646563646463636565646365646463646365646564636564646463646363656463646564636564656365636563636464656463646463656564646363646463646364636464636365636363656563636463636464636565656565636364646364656364636465656363656565656463646365646464636363646364656463636464636365636463636465636463636564646564656364646363646565646563636364636363656563646465656364646464636463646565636363646563656363656364636365646564646564636465646565646364646464636463646465656463656363636465636364646464656563646465656365656363636465636563646363656463656563636565636363656564656565646563656564656563656463646565646564646365646364646564646365646464656463656363636465636465656463656565646363656565656363636564646463646463636465656565636365636364636465656565656464646564636463656464656464646465636363656565646565636563646564636363636463656464646364646564646365636564646365636363636463656363636365656464656363646564646565636463656565656363646463636364646564636365646564636364636563636464646464636563646363646365646364636365656365646464656463656463656465636465656563636463646365646363636364646363656363636564656363656364636565646565656564646364636465656565656363646565656564636463656363646465656363656464646364656465646465636363636365636464656465656563646465636563636564636365646564636463636363646363636464636565636564656465656464646564646464656363656365656564636363646465646365656365656465636565646463646564636364656363636465636464646565656363656463656364646563636463656364646563636464646464656364646364646364646363656365636565656564636464646363656565636363656564656464656364656564646463636365656464636563656464636363636565656563656565636563656565656565636565656563636365636364636565656565656563636564656364656464646563656365656364636565646565656563636463636565646565646564646565656463646364656464636563656564636463636363636365656363646565656463636463646363636464636565636464656463656365636365646365656565656564656564636363656364646565636564636564656365636464656563636365656363636565646565646463656565656563656463646465636363646363656463656365636363636465656465656563646365636464646464636363656364636465646463636365646564656564656465656364646464646463636463656465636463656463636363636463646463636365646364636365646464656565646563646364656364636363656563636565636564646463636365646363646463636365646364646464646465646565636463646564636364656564646365656565636563646564636464646563646465636363646365656365656464636364636563636463656463636363656364656364636365636464646463656563656563646564646465656465656364656565636463636465656365636364646464646563646564636465656565636364646464636565656563646364646464656563656365646564636464636365636563636365656565636463646563636565636565656564646565636363646463636365636565656563636363656465646565636563646463656563656564636363636564646363646563656565636465646464646565636465636363656363656564656565656364656363636363656365636363646563656364656465656364646464636464656363646464656364656364636463646465646565646565636564656463656364656564636364646363656363646564656364656565646364636564636565646463636463646363656364646465646565646363656563646564656565636565656364646563656363646464636463656464646463656565656464636364636565636463636464636563646463656565636463636465636464656463656363656465646565646363646465636365656464656565636563636564636364646563636363636365636465636363656364656464636463636464656363656564646364656565656463636364656564646465646564646464646565636464636363636364636565636365656363636564656463656364656363646363646364656565646563636464646363636465646364636464636465646363636464646565636363646563656565656363646565656564656364636565636363646563656363636464636363636463636563646465656565646565636565656365646363636465636565656364656463636364646363656363646363636464646565636565656563646465646463636465646565646463646565646363636563646564646564656364656365656564636564646564656363656563636465636365656464646463636464656564636365656365636463646365636463646363646463646365636463656564646364646365656464636365646563636463636363636364636364656465646365646363656464656364646363636463656563646364656364636464656363636365656363636564636363636465636465636565656564656464636465646565636563646464656463656463656365656565636464656564646464656564646565636564656564646365646363646463636565656464636363656363656363656365646463656564656563636465646563656563636363636365646463636364656565636563656565646564656565636563636363646563656363636563636564646364656463646464656564656464646365636364646365636563636564656364656365646463646563636464636365636564646463646563656365656363656464646363646465636565636563636464646463656463646463656564656363636365646564636364646564646365636364646363636463646365646565646363646364656565656463656365646365636563656363656565646365636563646364656364656465646464646564636465656365636364636463656365646563636465636464636464636364656563646364656565636365656563656564646364636363656364646465656363646564646563636564656464636563646363656564636363646564656365656565646465636363656563656464646464646365656465646564646463656563646363646564656563646565636465656363636463636364646363636465656364636565636465656563636365656463636564656465636464646463646365656364656465646465656465656465656364636563646565636464636563636464646464646365646464646463656563636464636365636463636463636463646563656465656463656364636565656464646463646463646465646464646563656465646463636365636363646465636563646463656565636563636565656564646365646365656564636463636364656363656365656363656463636463636363656463646564636565636565656363646565656564636465646563646563646463636463646365636565636363636465636463656465636565656565646365636363656364656365636363636365646564656465646364646465646464656365646563656463646563636365636563656463646565646464646565646564636464646464646565646465636464646365656363656463656363636463646364646463656563646363646465636463636563656564646464656463636563636563646465636465646363656464656463646363636363636564656463636465636565636463636363646463656565656563646565646464646563656463636565646564656465656463656564636363636563646364636463636465636463636564656463646564646463636464656463636564656464646465636463646463636363636364636465636564656565656463636563646465636463636463656365636464646564636364656563636463646463636364636563646565646463646565656465656563636364646463656565636363636365636565636465656565656565636463646464656464656463636564656464636365636465646564646365636464646463636365636463646565646364656465636464646463656563646565636365636465656363636464646363656463656464636563656565636363646465646363656565636463636564646464656564646563656365636465636365646363656464636565636563656463636563636363646464656363636465636565636464646563646464636365636364646463646563636365656464646364656564636564636564636563636465656363636565636365656464646565656465636464656563656564646565656563636364636565636565656464636565646363646465656564656565646364646564646464646565656363646565646363646463656363636563646464646365646465646564636464656463636464636463646463656465656364646464636565656364646564636364636563656465636364636465656364636365636563636464636463656365646465646363646564656563636365636364656363646563636465636564646365646363656464646363646564656563646463636464646563636464636364646364636564656563646365636463636363636564636363636365646363636463656564636364636364636565646564636365636464646364636464656365636365636563656565646365656563646365636463636465636364636563646463636465646463646365636463656364646365646565656563646563636365636364636463656565646564646364636564656564646364646463636363656563646363656365656563646564646464646465636563656363646465636563636563636365646365646463636564636464636565636563636564636364636363656565646565646463646565656464646463646365656364646365656463636464636363656465636363636363656464656463656363646364656465646563646463646364646363636464656563636564646464636563646463636465646563636465656464636563636365656464646364646365646363656464636365636464656365646363656363646463656563646465646364636564646463636465636563646363646465656564636363646464646363646464636464636565656365646465646363636464646565636465646464636464656463636364646465656363646563656364636565636365646464636464636464656564656363636463636564646563636565656563656565636463636563656465636363656565646564656463646363636564656565636464636363656365636463646565646464646364636563636465656564636364646365646463636463636563636463646564646564646365636564636563656363636365656363656565636365656563636565636365636365646463636364636464656364646363636564656463636363636365646463646465656365646363636463656363656564646463636464636364656465646565646363656465656463646564656365636564636363636464646365646363656463636465656564656463656564636564656564646464656565656363656464646365646563636564656564656463646364636465636564646565636365636363636465646564636365656363646564646363636463656465636464646563636364636364646363656565636463646465656565646365646364656564656363646563656464636563656364646464656563646564656363656364656464636363636564636364646564656563646365646563656365656363636464646465636463656365646465636365636465646564636364636463636563636363636363656365646364646464656464646365636363646465636464646563646465636363636464656465636465636363636363656363646463636365656463656363656364636365646364646464636565656365636465636465636465646363636363636563646364636565646363646564656564656463656465656563656465646363646465646463656464656463636364646465656364656465646463636565656363656464636363636364656463656364656564636565656364656565656465636364636463636363656365636564656463656463646364636363646365636463636464646363636465636364656564656563636363646464636364656565636364656364646364646565636364656563636565646564656363636363646365636365656364646364636563646363636465646363646465646465646363636564636565656564656363656365656563636464646463656363656463636365646365656363646463656365646564636364636363646463646564656364636464636363646364646565656563636563656463656463656464656465646463636365656365646563636463656365636365656465646463656464656365656464636565636564646463636563656564636463646364646563636564646365636364646464646563656465646363636565636363656565646465636563656564636364646363646465636363656465636363656363636364636365646563646564646464636563656363646364636563656364636564656464636465636563656365646364656364646365646464656364646565646363646564646564646465646364636565636563646464636463646363646463636363646463656463636464636464636565656563646365646565636465636563656563646464636464636563636564636363636465656465646363636463646364636465636365646563646364656565636365646563646465646563636464646464646465636365656364656463656463646464636463646465636463646364646363646463636465636364636464646364636364656465636464636365656563656363656464646364646565636563656465636465636563636564656364656464636564656463646565636465656365656564656464636364636363646563656365656563656463636465646465646563656564646565646565646463636563656363656364636564656564636363656565646563636464646364656463636464646565636465656565646563646363646464646463656463656464636365646565636564636463636364646465656464636365646365646465656363636563656363636365656563656464656563636363656565656463636364646565646365656564636563636363646565636364656463646365636563646564646463646464656364636363646363636365656364636563636564646365656463636363636563646565656463656565636465656363636563656464636563636365636565636364646564636463636463636564636463646564636364646564646465656564656363656564636465646365646365636463646563646365646563636465656465636363646463646564636363636364646365656564636565656464656565646365636365636365636364646564636364656463656564646465656363656465656364656364646363656365656365636463656364656364636463636365636363656465636364646363656364636563646565656564646564656364636465656363636465646364646464656464636563656365646364656565646364646363636465656565646465646563646364656463636463636365636564646465656465646364646365646363656464636464656564646465636464656565636463646364636565636365636363646563636364646563636564636565656565646563646565636563636363646563646365646564656564636563636565636465636364646563646364636365656563656463646564636565636464656464636563636363646464636565636364636364646464646463646563656363636463656363636363636364646463656465656463656363636463636564646465636564646464636464636563636364656564656565646564656463656465646564646463636465636364636464656565656365646363636465646365636465646463636465636564636364646565656465646464656563646365646563636363646364656365656363646464656565646364636465636365646365646463656564636564636464656463656464636564636363646564656563656465636464636565636363636463646463636464636364636464636463636464636365646365636463646465636364636364656465646363656564636465646463646364656464656364656363656463636565636364636563656363656364636463636463636364646465656364646463656563646463656465636463646365656463646365636363646464646365646365636363646465656465656363636564646465656563636463656363656365646564656564656364636565636564646364656565646463636463656464646563636464636463636365646564636464646464636463646565646463656463656565636465656564656463636364646365646464656464656465646564636465646465646563656363636564636563646464646563636465636363636364636465656365636564636363636363636465636564636563636364636565646363656465646364646463656364646363646363636364636565636565636565636363646465636465646564636363646364646563656363646463656565656563636365636565646365656365646463636464636564636564646563636365636565636363646564636464646465636365636464656464646363636463646363646463636464656363636565646363636564636365636464636363656463636363646364646463646563646463646464636563646563646564636565656364636565636564636365636364656363646364636363636364636363646363636465636464646563656463656364656463636365656464646364646363656564656565636563636465636463636564646564636464656363646565646464656463636463646464636465656463636565636563636564636563636564646563646364646564636563656365646363636363656565656364646464656363646363646363656465636565646565656463636564636463646365636565646463646564646363646563656464646365646363656464646365646563636465646564646464646364656363646563636364656465656463656465646365636563636465656364646463646565636463636363646363636465656465656464636565646464636364646465656365646465636565636463636463656463646365636364646564656364636465646465636463646364636464656364636563656463636464636463636463636565636464636363646563636565646365636463636465656365656465636464656364656363646465636463656364646465636363656463636563636565656363656464646564636363656365636463646564646365636563656465656364646564656563636363636563646365656564646365636465656463646363656363656565646365636463646564646564636465636463646464636563636364636464646563656464636464636565636365636363646464636465646464656463636364636464636365646463656365636363646364636563656464646363656563636465656564636564636363636565636364646563656564656563636564656364656363656565646464646465636365646363656563646463656563646565636463646464636563646463656364636563656463656365646463636564656564646565646364636364646464646563656464656463636363646363656565646465656365656565636365656365636363636565636564656563646465636565636565636563656565646464656565636364636365656463656363646365656463656463656364646564646463646565636565646465636364636363636364656464656364636565656364656364646563656363636565636563656565656564646565656463636465636364646564646364636365646564656565656364656365656364636364656465656465656465646564656463656564656464646464636465646564636465656463656465636464636565636465656563636464646363636463636463656463646363656463636465636363656465646464656363636563656464656463636565636563636364646565636463646364636464646365636564646565636365656465656463636463646363656465636563636565656363636364656465646364636563636464646464656565656364636565636363636465656565646364636465646364636564656465636364636563656464656564636363656565656563656365646465656363636363656564646464636363646365636563646364656565656363636464636363646464656363636564646464636463636363646465636463646563656363646464646464646564656565646365656564636363656363656565646563636464636465646465656364646365636563636463656563656564636465636564656364656563646464636464656464656464646363646363636564636465656364636563636365636364656565636464636463656564646565636364636364656365646565656563636465636465636463636564656365646565656463646463636365636464636363636363656463656465646463636564636364646363646365636464646463636364646363636364646565656563636564636365636363646563646365656565646463646563646365646363636565656563646464646564636465636464636564646564646364636465656565636363656464656363636363646563646464656465656565636365636363646565656463636463636463656363656363656365646364656464636365636464646563656363646465636363646464656365656464636463646465646464656464646563656363636364646463636565646364636365636463646364636564636563636463636365646565646364656464646563656365646564656365656463646364656465636564636563656364646363656365646363656363656363656365646465656465636564636365636365656563646363646364656564656564636564646463636463656364656365646565646565656465646463656363636365656465656563636364646563636563656563636365636564656364656363636463656364636565636463656563656364636564656365656563646465646365636465656465646465636564656565636463636563646364646565646463646564646363646464636564646464636465646565636365636465636463636365636465636363636365656465636463646363636364636364646463656564646365656564656564636463636364656363656463636563656464636565646564656564656463656563636464636463656364636364656363646565656464656464636365646365636564656564636563656363646563646465656363656563656465656363636364656563636363636363656564656364656564646464656565636564636365636363646463636365636463656364646464636464656464636565656465646564656464656463646564636365636364636463636464636363636465646565656563656365646463656364636364646464656363646364646364656565636564656464656565646465646365646364656365656364646363656563656364636464656564636465636463646464656563656565656363636364636363656363646363636364646464656563656463646563646464656565656463646563646464646365636363636365636365636564656465656364636363646463636563636563656363636464656464636463646563656363646465636464646563646365656365646364646563646365636364636364656465636463656465636364636363646465646463646363636564656463656563636363656463646363656363646365646465646364636563656364636365646565646363656465656363646364656465646363656463636464646565646364636563646365646465656364646365636563646463646464656364656565636363636365636363646565646464646463646464646365636564646464636465636464636463646363636465646565646365646365636563646363656464636363636563646363656565646363636563636365636565656465656463656364636365646564646563646365646363636364646564656364646563656365646564646363636563646464636363646563646363636563656465646464656365636464656563646465656564656463646363636365656464646363636463656565656563646565636463656463646365636363636364636363646364646463656465656563646364656465656463646364656464646563656464636363636465656363646564656465636365656563656463646464636464636363636563646363646364646363636464646464656564656365636565636364646564656565636363646463636563656563656365636464646463646363646563656364646463646565656563636464646563646564656365636364636363656364646465636565656465646565636464656564656363646565646563656564636563646363656565636365636363646364636365656363636563646464636463636364656364646463636365656563636463656364656363656365636565656463636365656564646563656464656465636364636364636364636563646463646465646465656563656465646464636364656365646463656465656365656464636464646364636563646564636463636364646364656465636365646564646465656364646463646365646365636465656563656363646465636365646365646365646565636463656365656464636563636365656364656365646565646564656464646364636365646463656463646365646364636563656565656563636564646465636365646465656363636364636465646364636563636563656564646363646564646363656363656565646564636564656463656465646565646463656465646563646463656364656565646364646564636364636563646363656364646564636563656464656364656364646464636365636464646465636563636363636364656463636364646563636364636364646364656464636364636465646463656565636563656463636563656363646564656465636364646565646465646565636464636364646464646363656465656365636464646465646563646565646363636565656563646363646564646463636463646564636565656565656565656465656465636364636564646365636563656365636464656363646563636365636565656365656364646363636364646564646365646463656465646364656563656565656365656365636365656563656364636463646463646565656563656363646364646364656363656465636565636464636463646464646564656464646364656365646564656463656365656563656363636363636365656365656363646464646363646364656464636364636365656363656563636463646363636365656463656364646363656365656563656464636364646465646464636464636563646463656364656364646364646364636365646563646363656365646463636363646364636563646365656364656564636463636463636564636463636563656463646565636364646364636363636464656464636365636565656464646565646463636565636363636365636465646464636463646463656464656363636463636364656363646465636464646565636564636564656363656464656465646564646563636365646465656464646365656564656364646564636464656364646564636464656465636465636363646465656563646363646364656363656463656563656363656364656565646565636463646464656364656363646363636565656365646463646464656564636563636464636463656464636564636563646365636363636565656463656363646364656464646464636363636463636465646363636463636464636464646363646363656565636464646465636563646365656463646463646365636564646563646464656363636363656464656564646363646463636564636364646363646565636563646365636564636565656364656565636565636564636565636465656463656363656563646363636363646463656563636465656465646464656365656464656365646563656365646563636363646565636365646363656464636365636564636464636563656564656364636463646364636363646364646365656564656465646563646363646364636464646365646465656464656465656364646564646363656464656465636464646365636365656365636563636563636365656563656365636463656363656463646563656464646365636364636365656563636364636463656565656365646564656465646565646564636365656463636465636363656563636563636364646463646563656365646465646363646464656363646565636563636464656563656564646363656365636464636564656463636365646464656363656365636363636564636564646463636463656365636365636565636363656365646363636363656363656463646565646465636465656365646364636364646365646464656365636565646465636363656464656463656363656365656363656564656365636564636563636463656463646463636364646463636465636465656564646463636564636464646464636463656364636564636363646365636464636463636364656463636365636365636564636463636565636463646565636563636463636465636564646565646365656363656564646365646463646564646565646464656463636364656465646463656364656463656465646363656463656563636464636363636465636364646564636563636563656564656563646465646565646564656464646463646463656563636363636363656563636465646363636564656465656365636363646364636464656465636463656565656463646564636364656364656363656463656565656463646363656564646564656465646463646565636463636364636364636463636363646364636364646463646463636364636465646564636364656363656364636564656364656463646465646464636564646565646363636463646563656364646463636563656563636364636563646464656464656364646464636363656563646563656565636364636565656364646465656564656363636365656563656464646365656465636464636464656465656464636365646465646465656564656463636364646565656465636563636563636565646463646563656463656564636463646363656464656564646463636563646364646565636463636565656464636464646365656364636563656563656463636365646464646365646464636564646464646565656363646565646563656563636564646363636463656365636365636464636363636463636564646463656365636564646365646365646563646364656363636364636365636463636363646565656365656465636463636464636365636564656465656363646465656364636563636464636564636565656363656565636365656565646363636364656463636465646463636364636364656364656365646565646364636563656363646464646465656563656364656364636565636564636365656363646565646364636364646364646463636465646364646465646363636464656564636364636365636464646465636463656365636365656563636365636564636463646465646563636465656463636463656564636563656464656464636464636564656463656463646464646364636463656563636363636563636464656463636363646365646563656464646363656465646364646564636563646364656463636463656563646363646364636365636565656563636464636565646465656563656563656365656563646365636365646363636563636363656365636563636563656364656365646363636364656563656565656364646365646364646364646463656364646464636465646365646463656463656463636465656364656363656463646363656364646564646365646465656363636465636465636563636463656565656465636463646364636464636563656465636563646565636364656564636363636565646563656364646465656363656565656565636463656363646463656563656563646563636465656465656565646465646463636463636463646364646364656363656565636463646464646364656564646564636465646363646564636563636364656465636464636363636364656363656463656565656564656365656563646364636365646564636464656465646364646464646463646364656464656465636564636365656565656363656465656364636565636565646564656365636365656364636363636463656364656363656363636363636563656463636363636564636364646464646463646564636565636565656465636364646563646565646563636564636363646563636465646363646463646363646565646363646563646364646465646463646364656563646563646465656465636363636465646363656465646365646464636365646563646365646465646365656463636464646464646463636465646464646563646365656365656364646563646564656363656465646365656563646463656564656364656363646464646563656563646565646563636363646465656363636465656463656464656563636365646465636563646365636365656364636364656365656365646565636465656465636363636564646565646564656463646365636364646463656364646363636465656365656564656565656465646464636464636464646364636464636365636365636363646565656564656565636365656364636463636363636564646463656365636465656463646563646563636465646363646365646464656464636363646465656565636463656465656365646363636564656464656463656564656465656565636463646464636565636363646464646465656364656463636365636365646364636364656464656364656464636363656364636563636364646365636363646565636465636463656563636465636363636565636465636565656464646363646365656465646364636363656463636463646363646563646464636464656363656565636363636564646563636463636564646463646363636363646463646465646465636464636363646464636364646564636564636565636564646465646364646465646363636464656465646363656563636564646365656565656563646465646364656565636465646563656564636463636464636363656364636564656365656565636365636563636365636363656565646565646464636363656463646563656463656363636364646364656365636464646463646564656463636465656364636464636363656463646465646364646563656464636363646465636363656363656563646564656564656564646364636464636465656565656365646465636363636363656363656365656365656465656563636563656465636363656364646364636365636464646564656465636563636463646564636465636565646363636563646565646365646565636365636465656564646363646463636365636365646565656365636563636465656364636363636363646363656464656365636463636563636464646564656564646465636364646564646464656463636563636564636465636565636464646463636465646363646564646465646565646464646365646463646463646465656563646363636364636364646563646364636464636465656564636565656563636365636465636363636465636365646465636365636563656463656564646464646565636463656464646565656563646564636365646565636563646565636565646365636363656465636464646463636465646565646465646364656465636365656364636364656564636365656463636564656364646463636463636464646464646363636364656564646363656363636363646464656563656363636464656464636363636464646464656565646563646365636465646464636364656363636365656465636465646565646465636363636363646365646463656565656363646563646364646563636365636465646463656563656364636363636564646463636465636365656563636463656365636364636565636365656365636463636365656463656363646563656565656364656565656463646364636463636363636565656363636365636564656365646464656563646563646564646363646564656364656465636465656364656564656563646364656463646363646563656565636363636563656563646563636565646463656565656363636564656464636363636563646563656563646364646363646465646364646364656464656565636364636564636365636364636463656365656464646565636464636363656465636365646563636465656363636464656463636464646565656563636465656464646465656565636364646464646563656363646563646464646564636364656564656464646465646363646363656463656365636465646463636463656365656565646565646364656564636565656363636563656463646363646565646465656563656464636563646363656364636465646363656363636364636365646465646565646365656363636363636364646365636565646365656563636364646465636363646564646464646363656363636364636463636464656364646365646464646564656365636465646463646465656364646364636363646363636465636565646464656363656363656463656464646564656563656563656363636563636564656564636563656465656564636363656465656364636563646364646565636563636363656563646564656365636465656364646564636364656363646365656365656364636363636565636465656364656363646465646563636365656363656363656364646463656565646563656564646364636465636463646365636463646564636363636565646563656464656365646364656463646463636463656363636464646564636365636463636563656564656365636463646363636364656365636564636465636363656365636564656365646564646363636365656365636563636363636463656365636364656365646464646363646464646565656363646464636363656564636465646363636363636364646565646363646565656464636563636565646465636363646365636363656365646465636363636464646365656363646365636363656464656563646464636463636364636463636463636564636563656563646365636463656465656364636564646565646363636365636463656363636363656365656363646564656365636363636565646463646463636464636563656363636465646565656363646463656465646565636363646363656365656363636463636364646563636463646464646363656363636363646465656365636565636463636565646565646363636363636465636465636564646464636563656463636463636364646565646565636563636464646364636564636563656464646464646463636464656363646565646364636565646563656563636563636565636363656565636463656463656465656364646464636564646565646563636365656465636365636364636464656363656465656463656565656363636563646464636464646365656563646464656565646565636463646464656564636465656564646564646364646463646363646464636563656465636463646363636563656365656563636365656364636365646465646464646463656365636565656364656464636363656463656464636463636463636365646463636465636464636563656565656365646563636463646365646363656463646365656463646364646463636365636563636565656365646563656364636463656465656564656465656464646463656464636364646565646565646564646465636365636464636463646365656463656464636365636564656363646563636364646564646463646364646565656365656363656564656563636564646364646565636364646364636364656363646463646463636363656363656364646363646465646363656365656365656465636563656563636464636363636465646363656465636363656564636464646363636465656563656363646465656463636463656565636565636563646463656365636363636464646364656463656465636564646364646364656465646463656565646465656463636563646463636365656565636564636365646565636364636364646365636365636463636363656563656364656463646564636365646464646463636364646364646564646463636463646364656364656563636365656463646365636363656563646564656464636364656563656365656563636563636363646363646464656465646563656564646463656564646565646463656365656465656564656564646563656563646564646564636364656463636563656464636563636365646364636463636563636365636363646465646564636364646365656465656364636365656465636563646363636463656564656564656365656565656364636365656363636463636563636465636363646563646464636365656464656363656564656465636564636464656563636465636465646464656363646364646465636563646464646365646563646564656565636365636363636364656463636465636363636563646363646465656364636365656564646364656563646465646565646363646365646365646563646565636563646363646465646365656463636363646464656464646364656463636365636565646363636463636463636363636465646565646364646363656564636565656363636564656465636565656365646563656563646364656464636564646564656364646364646365656563646565646563656364646364636563646365646363636364656565636563646365636464636563636564636364636464646363636564646563656365646464636463656463636363646363656465656563656363636563636465656465636363636463636465656363656364636365636563636465646364656565646564636463636365636565646465646464636464636365656563646565656364636463656364646464636364646363646564656465636563656364646563636363636363656363636564646463646364656364636463636464636363636363646463636463636565656564636463636365636465636564636464636464646563646363646365636564656365636365636564636464646564656365656463646564646464656464636364646363646363636463646465636364636463656464636365646565636363656463636363646563656365636565636363646564646363636365656463656565646565656563646365646365636364656563646463636565646564636565656364656563646563656563636565636465636563646565636565646364636463636464636465636364656364636364656564646465636563646464646363636464646464646465656565646465656563656464636364636464636563636365656465636464646365636563656365656465636564656564656564646563636563656363656565656365646464636463646363636563646464656365656465646465636463636464636363646364656363636364636363646463636363636563636563636565656565656465646363646365646563646565656464636564656464646464656465646565656364646465656465656564646363646463646365646563646363656464656564646365636563646463636364656464636365636563636564646564646363656363646364636563646365656565656464646365646564656463646364646365636365636365636565646563656565656465656564636463636364656464636464636564646463656365656564646464636363636564656363646365636365636363656565636564646365656465636563656363656463636464656363646565646364636363646564656364656565646563636465656463636363636464636463636365646465656563656364656364656365656565636363646564656565646365636565636564656565636564636365646464646363636563636364636463636563646364656564656363646465656464656564656463656464636565656565656563636464646464646564636564636465646363646565636564636464656564656364656564646464656463636564646365636364636464656463636365646363636564646565656463646564646563646564636564636463646563646464656563636563656465656364656463646463636365646463636565636365646565656364636564656563646464636365646365646365646465636565646463656363636565636364646564646565646365656464646365646565656563636363636363656365656465636364636364646563636565656464646365646364646465636463636563646465656465646564656364646365656564656563656564646565646365656365636364636364646465656364646365646364636363636565656464656564656464636563656465646465656363636565636465656565646364646363656464646364636363656564636364656364646363656364646563636464636463656463656563656463636365646364636464656363656464646365636564656464656564656363656365636464646363646365656464656463656364636363636465656463656463646465636365656563656365646463646363646464646365656365646563646563656363646464636365636464646564656364646464636564636363646465656364636464646565646464636463646463636563646463636565646465636463636565646365636464656464646463656563656364646363636465646465656464646564636564636565636564656463656465636464656563646365646364646564646365656363646464636564656464656363636364656465656364636365636364636365656363646365646364636364646563636564646365656563656363636564636565656365636364656465636563656564656565636564636465656465636463636365656363636365646563646363646465646463656365656364646365656364656363636465636363646364646364646564636465636465646563656565656563636463636365636564636463646363646365646565656563636364656464646564636564636465656464656364656365636364656464636363646364636364656465646365636564636364646464656464646465646364646363636464656365636464656363656364646365656464656365636463646465646364636364646465656463636364646564636365656463636465636364656465646364646463636363636364656564656364646563636565646363656464656564656363656365656364646463646465656563646363646463636564636565636465656365646364636363636364636564636365636363636564656565656463646463636363646563656564636565636465656364646363636463646463656465646464656565636563646565636364646565646363636364636364636564656364636564646463636363646565636465636364656565646363656563656363636563636464636565636363646563646364656463646565656464646464636363636363636564646463646565636364636463636365646465656365636565636563646364656464636363656465656565636565636464646565656365656564656364636364636363656565656463656465656365656563656365656364636565646365646464656563646563646463636563636464636465646463636464646463646365656565636464646563636565656465636564656463646465646564636464656463636464656464656364636563656563646463646364636363646564636563636464636464656463656465656364636463636465636465646365646565636564656364656565656563646365656464636565646364646465646364646464656563656564646463646464636364656464646464636364646463656363646363646563646564646464656365656463646564636464646464636365646365636564646464646363636563646465646464656464656365636564646565656463636565646565636364646463636363636463646563646563656563656563656463636464656563646563636563656464636465636564656465636365646563636363646465646565646364656463636564646465656364646463646364646565656563636465656463646565636565646364656565636363646464656464656463646365646565646563636565646365656365646463636563646465656364636564656564636563646365656464656565656463636364656565656364636465656464646564636465656464636365636363646464646563656563646464656564646565646365636563646564636364636464646465646465656564636364646465656365636563656365636364656465646463656565636565646363636465656365636564636564636465656363636463656564656363636464646463636564646465636465656465636364636564636564656564656565646363636563636464646464656465656563646465636365656363656564656465646464656564646463656464636464656563656364636564636363636363636363636365656565646564656463646463646465646564636365656363656564656363656463646365636465636465636564656465636363636463656565636563656463656365636365656564636365656564646365646563636363656363656364656565656363656564646363656365656563656365656463636563646464656464636464646564656463656364636463636464646464646464656565646365636563646465646464646465646463646463636363656464636565646363656565636363636564646363646465646563656563636565646365656364636563636364646364646365656563656465636465636563636364656365636365646565636365636463636564646564656364646363636365636465636563656564656464636465636563646365636563646463646365636363656565646363646563656364636465636365656463636365636563656364646365646564636365636364646364636463646465656364646565656563646564656363646564646363646565656364656364656565646565656463636563646365636463646363636564636363636565656363646564646565636465646363636464636563656364656364656565646363636564656565636465646364636564656564656564646463646364656565636364656565636465636565646365636564656564636365636565646563636463636365636563646363646463656364646563646563646563636563646563636363636465636464646364656465636365646463636364656363646365636564636364636465646563646363646564636465636363646364656564646564656365646465646365636564636465646563656463636464636565656463636363636563646464656465636463636564636563646365646463656464646464636464636365656564646565636363656364656564656564646464646564646363656365646465646465656364646363636563646364656363646464656364656464646463646465656464636564646565646565656563646364636565656465636363636563656565646364656564636363636364636463636563646463636464656564636465646464646463656563636464656564646465656365656465646563646563636565646465646563646465656363636464646365636463656465646363636563636563646565656563646565636363656563656364646563656563646465646363646565656363656363646564646565656463656365636363656564656365636564656464636465636563646364656465646563646363656464636565646464646463656363656563656364656465646564656364656363656363636365646463656564636365636363646464656465636365636563656465646563636365656563646465636365646363646563646463646465636364636364636563656464656464646563636364636363646565646464636565646463636464636365646364646563646363636363636465636465656365656563646564656364636564636563636463656365656365636363636565646365656465636465636364646563646364656565636563656563656464636363656364636365636563656363646464656564636565646364646363656463636465636564636365636464646464646563646363646464636463656564646564656464656465656464646363656565646464636565636563636563646565646463656565646564656563656465646464656363636464656564636365636465636564646365636365646463656364636364636463636464646564656363636564646463656464656363646364646465656463636364636364646463636565656465636365656564646464646564636363646563636364646464656363636565646565646565656464636465656464636364646364636463646364636364646463656465656563646363646465646463646464656565656563636364656563646364646363646463646463646464646563636365646365646464656563636365646464656464656565656564646565636564636463656365656464656565636564646464646363646463646564646364656463656365656364656365646565656564656363636363646465646563646364656565636364646563646364646465636564646463646564656564636565646465646363646465656563646465636363656363636365656363656564656464636564646563646463636364636564636465636465636564636363636365646363656365646464636463656365656363656563646364656465656564636364646563636364646565636563636464646563636365636564636364656463636364656464646563636463646364646565636465656463646565646463636463636565636363646463646365646463636465646363656363646463646565656365646563636363636565656363636365646465646363646565646365636563636363656465646463636365636464656563656465656363636464656563636364646363656365656364656363656365646365656463646465656364636463636363646363656365646463646465656365646564646363646463646363636364636564646363656463646564636563656565636564656564636365636364646465646564646463646463656463656464646463646463636565636363656465646463656564656563636465636363656465646464636465656464636464636465636465636465656363656364636565656365636365636365636363656564656465656564646365646565646465646465636565656563636564646463656364646463656364656463656565636365656565656464636365636365636565656365656364656463636364646563636565636364636563656463656365656563646563636364646564636465656463636565636363636363636563636363636364636563646365646565636564646565656565636565646563646463656363646564636364646563646563656365646563656365636565636563636565656364636364646365656365636465636463636365646463656364656363656464636565646564656463636563646463656364646563636464656363656465656565656463646363656364636564636463636563636464646563646563656564636364636464646364646465636464636463646464636364646563646563636463646364656463636363646463656565646363646364646365636465656464656465636364636363636363646465636363636363646564636465636363646463636465636464636363646463646565636564656565646564656364646463636364636563646463646464636363646364656565636563646364636465656565646365646464636464656364636565656565656463656463656564636463656363636565636463636564636564656364646365646364646464636365646464646563646565656363636563636365656465636365656565646364636465646563656364636563636564646563646464646364636465636365636364646565636565656565646465646464656565636563656365636365646363656564656565656365656363646463646464646365636464636364636564656463646565656364646465636363646563656563646364646363636363646465646463656465646363636464636464636465636365636563646363656564656465636365646365636464656363656463636463646564636465636364636563646363636465656565636564646565646564636364646463656365636563656564636364656464636464636465646465656463636363646563646565636564646464636363656464656364656363646463636463636463636563646565656463636564656463646364636565646363636365636465646565636363656463636364646564656365636563656465656464636363646565656464646563646364646564636363656364656465636363656563636363636565656563636363646363656464636465646563646365656365646563636463656464646464656363656363636563636363656463636363656365646364656365656463636465656465656363636464656465656463656364636364636364636364656364636365656565646565646365656365636365646564646465636365636365646463656365656564656363636564646464636463646464656564636565636463636463646465636463646563636563656565656364646463646563636465636365646365656364636363656365646463646363636463656465636465656465636465656365636463646465646365646463646363636463646564656465636363636363636464646464646564636365656563646565636364636563656465646465636563636565636565636563656563636465656365646564636365636364636465636565646465646564656363636363636565656564646365636363646365656463656464636464636364656363646463656563646464636365656363646564636463636565656364656564646563646463636465646463656463636464636364656565636364636565646565646463656364646363656364646463646364656465646365646463646564646565656565646365656365656365636565656364636364636563646363646363646464646463656365636563656565656364636364636364636363636565646565646363646564636565656563646465656563656363636464636464646364656463636563646564656464636563656563656464646365656464636465636563646363646465656365636463646364646465636365636565636463656364656464636465646464656363656565646563656364636364636565646563636565636363646364656365636564646364646363646365636465636464646464656364656463636563636463636563646465656563646564636465636364646364636563646465636464656364646364656364646464636564656465646565646465646565636563636464646565646465646565636563656365646364636564636463646365656564656363636364636465646463636563656465646564656564636363636463656464646563636463646463656363656563656563656365636564656364646464646363636464646463636563656363636464636364656464656363636564656565656464636563656563636564636365656363646564646364646465646563636464656463636465636363656363636363656463656364646463636363646465656365646563646363646364636363646364656563636463636364656565636363656565656363636363656364656465636563636365656464636463636564636365656365636363656363646464636364636364636564656564636463656465646365636363646365646563636364636365646463646563656465646363636363646465656563656564656363636565636565636565636363636465646364646363646465656463646363636365646464646365656463656565636363656365646364646365656365636465656365646363646565656464656563636463636563636364656364656565636363636363636565636465636563646463656464646563646464646364636465636463636365646365656464646365646365656563656563636365656363656563646365646364636364656563656364656365656463646364646464646564636565636565656564646363646565636465646363636463656463636363636463646463646565636463646464656463636365646563636464646365656363636463656465646465646463656463636565646465646563656365636363636565636364636563646363646363646564646565636464636465636564636463656463636564656464636363646463656564646564656363646563636363636464656465656364636464646463646365646464636564656364636565646365656565656364646463646463656563636465646365636463656564646563636564656363636363636465646564636363656463646364636364656463656465636563656464646364646563636365656563656563646465636463656463636363656363636363646365636564646463656563656463636565636564636565656365646464646563656563646464646364656365646365636365646565656463646563656465656365646465646363636365656465636464646564656564646463636365636464636465656563636363656365646563646563636565636363636563656465646364636364656364646564636463636463656364656463646464646563656465656464646365646463636564636463636464636364636563636363646364636565656365646363636363656465656365656365646465636463636563636364656464636363656363636363646565636465636463636363636363636463646565656564646364646363646463656465646565656563636465656364646365656465656363636465656563646463646465646465646563636364656363646364636463646564646564646564636363646564636464636465636364656464646565636465636465656365656464656364636465636565656563656365646563646464636565636464636463656364636364646363646364656365636464636465646463646465636364646564636463656465646463646464646364636365646565646365636565636564646465646564656564646465646363656565656363646565646465656465646564646364636463646463646365646364656463636464656463646465636565646363656563646463646564656364636364656564636565656363656363646565656563646364656365636464636564636365646463646565646563646365656465646563646363646363646563636463646364656464636564636363646364656564636565636465656364636464636363646565646465646363636363656363636565636364646363636563646365646465646564656464656564646465636563636465636363646563646464636565656364656565636564636464646363636565636464636365646565656364646364646365656364646463646365636364646563646564646563636363656464646363636365636464646464636363636464646563636565636363636365646564646363636465646564656563656465646463636564656463656464646564636565636464656363646563636364636465656565636565636565646564656363646464646463636465636364636363636364656563636363636563656363656564636364656365636463656564636365656564646463646363636363656463646364646563656365646363646463646363656465646564646465656464646363636565656463636363636464646364646365646363646563646465646364646465656365636564636565656463656464646463636563646565636565656365636364646565636463646464636463636463656365646363636564646463656465646564646364656364656363646363636465646464636364636564636363646563656465646363646365636363636363636363636563636564636564636363646363646364646464636464656565636365646363656565636365656464656365636565646564646563636463656365656365646364646465636564636563656464656564636365636465636365646465656565636564646563656565636463636465646365636563646563656465636463636463656565656364646563646364656364636365656564656365656465646565636564646364656363636463646565646563636463656363656563646563636564646363646364636463656363636465636363656464646363636463636565656463636363646563656563646563636365636364636365656565636364656463646364656365636364646565646463646464646465636465636464646364636465646563656464656565636365636465646363636564646463656465636465656563656363636464636563646463646465656364646363646463656363646464646565636565656363636363636564656464646563656563646563636365656564636563656564636364636464656363646365656563656565646363636564636364646564636564646563646564646364646365636363656363656363656465636463646464656465646365636563656463646364646365646363656365646363646563636464646363646363656363636465646563646563636565646365636464636563646364646565646363646365636564656564656565636563656363636565656565646465636363636363636565636565646564646464656365646463636565656563646463656363646363656365646463646363656465636563646564646464636363656463646565656564646463646365636565636363636463646363656365656365656465636364646465646364656563646564656565646365656365646464646463636564636563646464636464656465656363636464656565636365636564656463656464646465636465656464636465636463636463646463636464656465656563656364646463656565646364646465656565646465646465646363656464646565636463656363646363646364646364646363646563646365656563656465646463646363636465646565636364636563656363646463646565656565646365656464656365656364656364636564646465636363656563656465646365656564656564646465636564636564646563646563656563646465656465646363656565656465636463656364646465636463646464656465656563656363646563656364636565646465636565636565636465656463656364636564646563636465636464656463656364646464636363646563636565636465646364646564656564656564636365656465656565656464636464656463646365636363636565636563646564646563646463636564636564646563646564646363656363646363646365646463646364636363656564636463646363646364646364656563656464646463656564646564646364646365636365646363646564656564656364656363646463656364656563646363636463646563646464646365646564656364656364636563646465656565646463636464636563646563656365636463656564646564636564636365646463646363656365656565656365636364656465636564646364646365656464636463656364636465636463646463636563656465656563656365646463646563636463656564636463656563636563656465646464646363656365656363636363656463636563636363656363646565636464636363656565646464656465636363646565636365636563646463646463656464636364656463656364636464656465656464656363636363656365656565636564636364646463656364636365636364646564646464646364636465646565636463656464656363646363636363656464636565656464656363656565656363646363636463636563636563636564636464656465656464646464656463636563636363646465636465646465656564636364636565636463636363656363646464646563636463656363636564636563636465656464646365656564656365646465646465646364656565646365646563646365636464636365636364646563646463656364646565656463646363656563656463646364646464646563636464656463656564646465656365636463636465636463656363636463656465646564646363656363636563656364636463656364636363636564636364646365656463646563646464636364656463646563636365656464656363646564646364646464646364636364646463656463636363636363636563656465636363636363636465656363646463656364636364656365636463636465646364646365646364656365636365656465636465636365636365646364636464646563636363636564646365646365656563646365636463636465656464646364656363646565636465656563636565656365636465636464636464656364636364636565656465636563636563656365656364636365646465636364636564646565636564646363636363636565656465646463656365646565646565636564656565646464636365646463646463646463656564636465636364656364636363636465646464656463636365636363656365636464646364656465636565636564636365656364646563656465636465656563656365636465646463656365656465636364646365646365636463646363656363636464646363646564656464646463646564646365646463646363636464636363646463656565656463656364636463636565636464656463636564656463646463646364656364656565636364656564636363646564656465646564636463656464656563646563636564636465636463636564636363636463636564636563646463646363656564656463636364646463656563636364636464646563656564656464656563636463656464636564656564636364636463646564646463656563656365646564636464646363636364636365636565646365656365656563636563636565646365646563636365636464636363656363646564636363656365636363656363646465646465656565646363656364636364656463646363656564646364646364656564646363636364656465656563646465646465636564636563636564646365646465636563646464656463636364656564636365656363646364656363656565636463636563646463656364636564656363646463656465636564656565646365636564636563646465656464656565646365656363636365646363636363656463636564636364656463646563646564636364646465656365646364656365646363646465636564636465636563646364646363636463636363636465646563646463656464646463656464636364646463646564636564636565646365646363646465656365656363656464656564656464656463656365636363636563656564636463656564656365636565646463656565636365646465656565636365636465656364656565636563656563636363646463636563636563636364636463636465656364646364656563636564656463656565636364656363656363636463646463646364646465646364636364646563646464656464656463636563656363646564636465636465646464656363636463636463636464636465646365636465636463656363636564646365636363636564646563656365636565636465636363656463636363646563636565636563646364646564636463646364636563646463646363636564636363636364656564656364656465646464646364646565656364656364636564636364646563636464636565656565636564656464646365646563656465656363646564636364636363646363646365636364656464636365646463656364636464656564656365656465646563636365656363636363656364636363646464656563656365656464656463636363646464646563636363636564656463646364646363636364656463656363656364646565646565646464636463646363646564646363656564656465656364636464646365656363646363646464636563636563656465646364656464636463646564646464646364636364656364646365656564636463656563656364636365636365636464646364656565656563636464656563646365656564656363646363656464636363656563656365646465656465636563636565656365636465636564656464646565636363656464656363656463646564656363646563646365646464636463656563646464646463656365636364656563646365636463656365646565646463636464656564656564656564636364646363656463636463656565656463656364646563646363636465656563656363656565646464656465646565636365646364646463646465636365636463636565636463636464656464646464636363636565636365656465656364636363646464636463656465656363656563646364636563636563656364636363636365636464656564636564636563646564656465646465656365636464656463656565656464656363656364636364636363636564656463646465656564646565656463646364636364656563636563646363636365646364656363656363636565656363646565646564636363656365646365656365636464636564636364646463646364656565636363636464646564636563656365646464656365636465636464656465636463636365656363656363646364636365656364646364646563656565636365636565646463656563656464636465636365656364646365656465646565646364636565656565636465646463646364636363656565636463656365656463656563636463646365656465636464636465656363656564636365636363636363636464636364636365646364646565636363636465636564656563636564656565646563656363636363656565636565636565636363636564656365636465646563636564646565646365646365646464636365646363646463656463656365656564636464636465656565646363636563646563646363646464656465646565636364636363646565656465656563656465646563656364636563636465646463636463636364646464656563646365636564646565646365656463646463646563656465646563636565636565656564646364656563646363636463646363646563656364636464636464636565646364636365656565656563646463646464646465636565636365646563646363656465636364656363646365656363656464646564656563636365636363646363646563636464656464636565646465636365656364636364646565646463646465636365636564646465636463646364646563636464636464656365646463636363636363656464646463636563646463646563656365646463636364656563636363656564656363636364656365636464636465646363636464636463646363646565646563636465636364636365646364636363636364636563646563656364656563656364656365646465636564656564636463646563656563636365636464636463636365646465636365646565636365636463636363656464646465656365646563646365636463656464636563636363656365656463646363646463656565646365636365636464636364636463656565646565646565636565656363656363646563646464646464646563656563636364646364636463646363656565646464646364636464636364656365656565656565636564636363636364656364656463636363636465636463656563636564656464636465656463646464656464646463636565646363646365656364646364636364646563646364656365656364636465636363656464646465636363636564656365636364636363636363656363636363636364656365656465646364646463656564636563636564646365656364636563656464646364636563646364636363636365646463646363636465636465636364636465636465646565656465656565636563636463646364636564636464656465646365656564646565646365636365636565656364646365656465646364636564656363636365636465636565656363636365656463636363636465636463656563646564646563656465646365656363636564646463646363646464656463636563656565636364636464646363646365656465646363636365646564656365656465646564656363646465646464656465636364656564636363646565636364646565656364636364646564646565646464636465656364656363636364646365636365636563636364656565656565656463656465636465656563636365636564646365636463656465646465656465656363656564646463646465656363656465656364636563646465636463636363636564656365636465636365636564656364636565636465636565646463646364656463636464636364646463636564656565656363636564636565656363646464646563636464636363646464646365646564656464646563636464656564636564646565636564636364636464646463646565656564646363646563636365646563656465646364646464656564636563646564646465636564656564656364646564646564646463656564636363636465636463636364646464646465636463646365636565646565646563636463636363636564646363656365646564636363646365636365636563636365646465646563646363656364636563656464656464656465656564656364656465646363636363646463636365636363646565656364636463646565636563646364646363646364656365656463636363636365636563646563636465636464656463646365646565656564636465636563656365636365636463636565646363646465636363636464636563646563646363636465656465656465656465636465636464656365646563646365646363656365646365636564656464636564646465636565656465656465656563646563636465636464646365646465656463656565636363636463636365646563646363656363646364646563656365656463656363646565636365646565636363636365636364646463646464636565656364656463636464636364636565636365646363636363636463646364646365636463656363656465656565636364636363646364636365636563656564646465636564646463646365656363636464636364646364646565646463646564656464636463656564636364646363636465656564656363646564636463646363656465656365636364636364636365646463636565646365656565636564646363636564636564646565636365636465656363636563656564636564636363656465656464646363656464656364646363656463636363636565636563656464656363636565646365656464656365656563656365656563646463646365656563646564636464646563636364656464646465636563656464656363656365656563656565636465636365636564656563636363656363646563636565636464636464636564636565646363636565656364646564646364646563636463656565636564646563646564656363646563656563646463636565636364636363656464646564656463656563636365656465646363656563636363636363646363636364646464636363656363636365636364656363656464656363636365646463646464646364636365646364646464656564646565656463656565646464646564646564636464636563636563656563636365636364646364656365636365656463656365636563646464636565646364646464646564646565646363636363636564656365656464636364636564646463636465656363646464636364646563656363646463646464656365636563636565636365636365646465636464656365656465636365656465646564636563656365656565636363636463656563656463646565656363646563656465636463646464636565646564646465636465656364656463636463646563656565636364646564636563656564656365636364636565646565636464656463636463646564646463646364656365646363646464646363646365646564646463636364646363646363656564636563646564646365656465636464636465636464646465646365646464646365636563646365646465636564636364656463636463656463636563636364646365646564636364646364656464636464636463656463646464656364646464636464646363656564646564636365646365636563656465656465636565656363656364656364636463646464636464656464636365646365636463636465656563636563646565636464646465636565636565646365646564646463656363636463636563656464636565636563636363656565636564656563646364646563646565646565656363656363646365646464646565636363646464656565646464656463656564656464656364656463646564646363646364636464646365646564636363656564656463656364646465646565636363646365636363636563646464646465636563636464636565646365656363636364636464646563656364656464656564646464636364636365646364656463656464646464656464636563646363656563656564636364636363646365636564636463646563646464636364646364636564646565636463636565656365646565636465656565646364646464636564646563656364656463656463636564636563646464646365636563646463636365636463636464656563656464646365646463636465646365646463636464656565656363656364656563636465636565646464646365656364646565646565636464636565636363656564656565646464636465646363646564636465646465656564636463656464656365656364636564656365656563646465656565656563646464636565656364636564656364636563636363646465636364636365636464636365636564636563646463656464636465636464636465636465646363656363636564646365656565646363646363656564656563636465656463646564636363636463636464656465646465646565636365636463646464646464646365636365636565646463646364656563656365656565636563656365636463646365646363656365656563646363636463656465646464636463656465656363636363636363636463646363656464646363636363646563636464656563656563636363656365636563636463636565656365636464636464646465656565646363636364646564636565636564636463656464636463646365656363656565636563656364636364636465636463636563646365656464646565656364656364656365646365646363656363636563636463656565636364636364646363636464656463646363656463656463636363636465636363636463646464646564646464656465656463656465646563656463656365656365646465636365646465636564656564636565656564636565646464636364646563636364636463646465636564656464646563656463656364636463636565646563656364636463646465646563656363636565646564646465656565656563646465636565636364646563656364656564646565646563656464646365656565646364636463636463636565656563656563636564656463646465656365656464636463636565646465636364636463656464636563656565656365636463646365636565656463646465646563656364646363636464636364656363636363646363656464646364636464636563646565636563636465646565656465656464636363636563656464646365636363656564636464656563636364656565636365656564636464656363656364646563646464656563656464646564636563656463656463636563646563656565656364636565636463646563646363646463656464636463636563646365646465646363656363636565646464656464636364636363636563636364656564636364646565646464646565646464646364636563646364646464646365636364656363636465636364656563636463636363656464656564646463646365656463636463636364646465656365636463646364646464646564636365636465656464656363656463636365646463646465636365646365656564636465646365636565656365656363646364646563646563636565646565656464636463636365656363656563636364636565636463636565656463636463646463636363636565636365646564646563656563636565646365656363636364636365636563656363646565636463636463646465656564656565636464646563636363636565656563636365636565646464656465646365646464636363636363656563656364636363636363646363656365646364636364656563636363636565646463636564656563636364656365646464656564636464656464636563636364646564636465656364636363636564646364656365636564656463646363646564636565646363636364646363656365656365636564656463656565656365636565636563636464636563656565646463636463636365656563656465636465636565636363636463656364656565646564636563656565636565656465636565656465636363656563636565656563636364646565646464656364646464656465646364636363636363656363656565646465636564636463646565636564646565646564636465646463656365656363646464656565656464636463646363656564636463636365636363636465636563636463636565636564656564636364646363636365646465646563656564636463636364656563636463636564636465656565646465636464636465646363636565646464646565646564656463656563646463646465636364646465636463636363636463636463636363646564656464636563636364646563656364646464636563636563656563636363656465656465646465656565646563656465656365646464636364646363646363646365646365636564636563646363656565646463646464636365636364656363646465636465636563656464656463636463636365656563636365636365636564646464636463636463646365656463636565656464656464636465656464646363646363636565656365636365656563636563656365646364646365656565656463646564646364646363636563656363636464646365636463656364646363636364636364656364636364646364656363646564656364646463656364656365656463656465656464636465636465636465636464646363636365646563636563636363656364646565636363636464636363636463636464636363636565646465646563646563656463636565656464646364636364646464656465646564646363636465646564646465646464656464646465646364656463646463636563636363656564656365646365636363636364656463646563636563646365636365646463636463636464656464656563636364646365656465646464636365656465636364656464656365646564656364646463656364646365646364646465656363636463636565636564636363636465656365656463636463646363656363646464656463636365656563636465656563646363646463636363646365636463656363646564646364636363636364656363636463656563636363656364646463656365636364646463636363636564646364636364636564646363636463636463646565656564656565646563636563656463646563636464656363656364656464656465656365636463646464656363636565636563646363636363646363636463656465656564656563656463646565636565646563636463656563656364636464646563646564646563656564656364636564646464636463646364656363656465636564646464646364636364656563656564646565646564636565636463636365646565636463636365656363636564646363646365636464656363646564646463636464656565656564646564656465646463636364646365646364656564646364646564636365656363656565636563646363646463656364656463636463646364646463656464646364656365636364646564646564656463636465636563636565656363656363646363636564646563646364646564636365656365636564646365636564646465636365656364646565646464636363646565636464656364646363656565646363656365636463646364636563656564636564636364646564636365636465636463656563636363646465636464636565656364636565656563646564656463636464646365636463656564646464646563636365636364636563646465646464646463656364636463646463636565646465656564636463646363646363636363656463636363656464646363636465646565636564636363656364636464636364656563646563646563656465656465636563646463636463646365646563646463646563656564646365656465636365636464636465656363636365636463656564656463656365646465636563646565646563656464636463656563656565636364646464656465636464656364636463646363636364656563656363656564656464646563656465636564656364656363636363646363646465656564656365636564656364646465656465636364656363646363636365656465636365646463646363636464656365656563636464646365636363636464656564646364636364636465656565636463646565656365636465636364646363646565636464636564656563656365646363636363646563656365656364636463646565646365656464656365646564636565636363656465636463636465646563646463636564646463656565656564646365646464656363656363656463656465646563636564646363636465636364646563636363646363646565636365646465636465636365656363646364636364636464636563636465646365636363656565636364656463656465636464636463646363646365656563646365636363656565646464646363636463646363646365656365636465636563646364646463656463646365656564646464636363646565656363636464646363646563636465656363636364646565656365636464636463636464646465656565636364646365656564646365656463646363646363636365656564636463636563636365646364656364636563636463636365656365646464646564656563636364636565656465656464636564636465636463646364646465656565646363656464656364636464656365656463656465636365636465656365646565656564646465636464646465636564636364636464636563646463636463636465656564656465636463636363656563636363656463636463646565656465656564636563646464636363646363656465636565646465646365636563646363656363656365646363656563636464636364646564636563646463636564656463656365646465636463656563656365656564646564646465646464656365646365656463656363656465656465636363636563636365656463656463636364636364646363636364636463656363646364636564636465656463656363656363646364636464646463636364636463636563636464656563656363656564646465636565636364646363656464646363656364656565636363636364646564636364646565636564646364636565646464636564646364646364656364646564646464646464656363656363656465636363646465636463656463636564636464656363646363646563656364656564646465656465646464656464656464636565646563636464646563656364636463656364646365636564636563636365656563646464646563636465636563646565656563636363646363646563636565656463636564646564636365636565656464636563656564646463646565646463646463636564656465636365636465636563656364656565646463636363646363636465636465646463646564656564656565656464636563646365646565636465636363636564636463656363636563656465646363636363636563636465646563636565636564646363656563656564656365646465656364646365646465636564636564636363636565636463656465656564636465656564636564646363656563656463646564646465636464636463636464636465636363636565656463636565646465646564636463636463646465636464636463646463636563636364646564656365646365646464636564656464646463636463646563646564656365636363636565646365646363636364646363656564636364656465646563656363646464646463656464636463636565636365646364636464656463646564656564636365646563646363636365646463636365656363646564656464636365656464646365656363646565656464636563656563636465656563636464656564646563646364656465636463636563656564646363656464636464656563656363636565646365646365656565656465636563646565646465636464636364646365636565636465646564646363656563646364646364656464646463636463646563636464646564636365636364656363646364656463636463636463656364646365646464656464646363646363636465646365636465656363646463646365636465636464646565646364646465636365646464646464636564646463636564646564656563656364636463636464636364636463636464646464646365656563656564656365636465656365636463646465646465656363636464646565656364656464646463656565656563636364656463636464656363656464646563646365656363646364656564656465646463646565636563656464636364636464636563636363636565636464656364636365656465656464636365656564656563656364646363636464656463646565656463646464636365656365656464636463656563636564646563656363646563646463646364646463656565656365636563636464636564636464636465646564636565656564646564646365646364636564636563656565636463646364646464646565636365656463646364646463646463646564646463646365656565646363636364656465646563636363656463636563636363636463646563646465636464646463656464636565656564636563646464646363646363646565646464646463656464656564646563636463636364646564646364656564636563646463636364656465636363636364646463656365646365636364646365636563646464646364656564646563646365656563636365636463636564646465656365646564646564646465646564636564646464636564646463656465656564656464646365646565646463636563646464646464646465656463656364656563636363656564656363636363636564636365636463646365636464646365656564636563636465656363646463656565636563636565656464656364646464656364636363636463636364636463646564646463656364656365646465636464646464656565656564656363636464636464656364636463636464636564646464656564646464656465646564636563656564656564646565646465646563656563646463646363656463646465656464646565636464636463656463636564656363636463646563636464646463646363656364656564646565636365656563646364636564636365636564656365646464656463646564646363636563646464636465656364636363646364646465636563646563656465646364656563656565636463636363656564636463646465656365636365646464646464636363656363646465636565656565656363646563646565656564646464636563646564656563656563636463646564646465636363656563646564656365636464646565646363636465646463646564636465636565646365646463636365656464656564636363636465656464636363656463636565656565636363636463636563646563646464646363656463646364656565636365636464656363636564636565636363646364636365656463636465656364646363656563646463636465646564646364636565656564646465636365636465656365636563656565636563646463646565646463656465656565646464646465646463656564636464646363656563636564636465646564636463636565656364646563646365636364636565656364656365656563656463646565636365646463656463656563656463636365656364636463646463636465646465646364636364656465636365656363636364636563636363636564656365636365636365636363636363646564636365656563656364636563656364636564656565646363636364646364636465646464636364656365636363656463656465646565656365656364656565636463646363646464646464656364636363646563646465636363636463656563656363636564636564656363636365636565636463646464646565646363656465646465646364656465636463646363656364656364656363636463636465656464656365646563656564656565636365656364636364646564636563636465636465646365646565636565646565646563636465646364636463646563656563636463636465636564656365656365636363636365636363656564636563636463656464646464656563646363646363636364646365646363656564636364656563656563656365636365646563646365636565646363636563646563646563636364646465656564636564636464636363636364636563636364656564646563646365656465636365636464646365636565646563636465646564636363636565646363646364646364636464646564636565646564636463656465646463636464656565636563646464656464636563646465646563636363646363636365636364636563656465636564656464646565656565646465646565636463656363646363636565656463636365646565646363656364636465646364646364636463656465636564646563646463646365656464646465656463646563656463656363636364636564636363656464656364636463656365636563646464636465636363636363656563646564646563636465646365636363636364636465646465636365636364646365636465646464636364646363646465636363636363656364636563656363636365646363646364646565636464636364646563636563636363646563646463636363646465646563636365656465636563646363656365636463656463636564636464646365636464636363636565636464636463646565656463656365636363636365636465636464656464646363646465656464636363656464646463636365636464636464636564636564656463646463656564646365646565636565636564646463636564656364646365636565636363636464656565656465656464646465656564646363636565636364646463636463636363636363636365656565646363636465646563656463636564636563646363636463646563646365646365656563656364656363636463656465646465656463636465656363636365656364636365646564636364646464636364646365646363656365656363636364656364656465636563646464656464636564656463636563646465656565636565636364636364646463636565636363646565636363656463646564656564656465636364646564636464636364636464646565656563636464636465636463646365636565646565636465656364636565636564646564636364636364636363646464636564636563646565656363656364636563646363656363656465646363656563656463646563636465656465656363656564656563656465646463656563636463646363636465636563646465636464636463636563636465636363646563656564636465636565636565636363646365646463646563656564656464636463656463656563636563656564636363646363636365646364656463656365646364636463646564656365646564656564646464646465656563636465646564636564656363656564646465656463646563656465636563656563646565656563636464656463656363656364656464636365656465636363646364646365646465646564656364656463656464656564636565656364646463646364656465656565646563656363646464646365636363656465636463656365636565656363636363646564656565656563636365636365636463636465636363646365656563636564646563636363636463656364646364636565656465636465656564636364656465656565646565636365646464656565636563656465636465656565636464646563656564646464646463656364636464656564656464646464636463656365646364636363636363656463636365656365636363656564636563656365636463646564646463646564636464636563656465636463636563636363656365656563636564646463646563636465656364636365636363636363656463636363646364636365656363646364636565656465646465646463646365636565636364636464656464636565636464636364646463636364656464636365636364656563656463636364646364656465646463656464646464646565646365636564636565646364636364636565636563636364646365656363656463656364636565656465636565636465646563656464636463636365636464636564646363646365636365656565646463636465646564636364656463656464636364646463646563646363656465636563636565646365656565636365646564646465636563646463656363636565636463636464636565636464656463656463646365646365636363656365656363656563636563656363646464636363656463646564656563636464636563636364636464646364646563646565656363646463636564636565646364646464646463656364656563646363636463636363646464656563636465646363636464646564646465646565646565646364636565636565636563656364646364636563646363636463656364636564646565646565646464656465646563636364636363656363656364636363656463646565636564636364646464656563636463646465656364656363636565646364656463636463636465656564636463636564636564636563656465646363636465636565636463656465636365646563636465646563636563646365646564636464636563636563646565636465646463646363656464636564636364636565636365636564636364636463636363646363646465636363656364656364636364656564646463636563646364636465646365636464646363636565636465646365636565636364646564646363636363636563636563636565646465636364636465656464656565636364646465646465636365646563646364656463636363636565636465646564636364646464646465636365646463646565636363656463656365646463646463636364636364656365646563636364656365656465636364656363636364656465636463636365646363656365636365646363636363656465656563656563646564656563646463656464636363636363646463646365656364646563646463646363646565646564646465646365656363636565636364656365636365636565636363656565646465646565646464636465646364636463636565646564646464656364636565646464646463656465636465636463646563646565646564656465656363646563636364636565646363656463646564636365646463656363646564646565656565636565656364646463636565646563656463636364656463636563646564656565646565656364636364646365656365656363656364636364646365636363646465636364656464646363656463636363656363636463656363636465656363646563646364656565656364656564646464646564636363646365636364656364636364646465646563636564646463636463646564656465636464636464646464646465656463636463646363636465646464656463636564636563656563646465636463656563656363656564656463646563646464656463636365656563656363656464636564646565656464636463636464646465656464646565646565656463656363636363646565656463636463646364656364636363636563646463636364656465656365656565646363646565646465646564646465656563656564636564656563656364646463636463656364646563636364656364636464646565656363646364656564636565636465656464636565646563646565646463656563656364646464646564646364636364656363646363646465656463636465646464636464656363646564646365636365636563636364636363636564646363646463656463656463656363646363646364656363646363636565656564656463646563646365646564656564656465656564656363636463656463646465636464636564636464646463646363646563646363656363646463636363636465636463646363656463656464646563656564636365656464646363636364636563646565656364636563636464636564646363636364646464656563646365636465636464656365636564636463656564646465636463656364636464656463636363656464636463656563646363656364636565636363646464656463656565636465656364656564636463636565646563636464656565636465636365636464656365636463636465656564646463646463646563646565646563646365636363646463636363646365646365646463646463636365636464646463636563656463636464636563636363646564656363646465636565656363646563656465636465646565636365646563656365656465636463646465656464656565646463656365636565656364646364656464656364646365636465656463656563646563646463646564656463636363656563656463646563656463646564636464636563646565646564656464656363636563636563636564636363646563636564656363656365636364656364646564646464656463636565656463646464656365646464636463656464636464656564656463646364646365646363646564636564646565646563636563656463656463646563636363636564636565636564636463646465636364636464636365656563636464646465656565656365646564646464646464656463646565656463636463636565636463636563656563656565656365636364636464636463646363636463646363636365646364646463636565656563636363636365636563646565656364636465646463656364656463656363656463636363636464646364656363646363656363656363636563646564636564636563646363636464656363636364646464656463656465646565636564646365636365646563656463646363636563646365656365636463646464636463636364656364656363646363646363636563646565646464656365636464646465646364656565656463656365646465656565656465656565656565656363646565636565656364646363656465636565636563656563646463656465646464646464646564646563656564656363656563636364646564656564636565656563636465636563636364656365636464656463656364636563646365636363656365656363656363636365656363636464636464646363646463636564656564656563636565646364656565646364656564646464656365646363656465646363636563636365656563656363646465656364636563656565636465636563636365636565646565636463656564636364646564656564656563656565646464636365646363636364646464636563646464646465646564656463646363646463636363636365646365636564646363656563636563636565636563646563636465656364646465656563636363636363636465656363656364636364646463636565656563636564636363646564646463656463656363636463656464646564656463646364646465656564646364636364636365646465656365656465646363636363646463656364656564636364656564646363636364656565656564646363656565656563646363656363646363636565636364636464656465636365636563656363636364646365636464636463646563636563636465656565656363656464636364646365636565636464636365636564656365646364656563636565636363646364636364636464656465636463646564636465636463656564646364636465636363656463636365656463646465656364636465656563656563656364636363656363656363636365656364656364636565636465636363646364646365636365646364656364646365656363636464656564636565656364656363656563646463636364636565656463656563646465636365636464646564656565656363646363636363646565646464656465656565636363656463646365656563656564636563636363646364636364656364656463656365646363656563646464646363636563656565656363646464646564656363656565646465656363656363636564636463656463636464636363636465656463636563656464646363636564636364656364656463656364636463646364636463636564646565636465656363636563656464646563636364646463646464636365636563636564646363646363636563636363646564656365656364646463656563656465656363646565646463656365656365656365646364646463656463646363656364646364646365656564646363636463656564636363636365646463646465656465656463636565636364656564656363646364656463656465656563636565636363656463656563656464636464656564636563646564636463646563656463646464636463646364636365636364636565636563636564646464646364656564646564646563646465646363646364636463656363646365646563656565646564636564646364656564636563636463646363636365636463636363646563656463646564636563656565646564646564656565636565636365656363636364636363646465636463636463636465646463656363636465646563636563636563656463646565636363636463636363636463656465646564636463646565636365646563646364646363656565656363656565636363646463656463656463646465636365636465646563656563656365656463636563636464656563656365636463646365636563636465656464646465636365646565646464646463636465656363656465656364646363636463646565646465656563656365646564646465646563656365636463656564636364636363656363636363646564646565656363636363636363656565646465636563656364636463646365646364646564646563636364636563636463646563636565646363656365656463656364646363646464646563646463636564646565636464656365656464656363636363636363656464636365656365636365636363656563636364636463656464636565646364646563636364646364636364636464656364636364646465646364646364656363646563646464656564656364646565636563646564656363646465656365646463636363646564656563636465636363656364646363646464646565646465656465656565636564636464636563656365636563656363636564636463656564656465656564656563636463636463656464646463636565636463656365636565646363646563636365646364656365636364636564656563636463656464656365636564646364636363646465646465646563646364636464656363636365656365646464656364636365646363636463646463636463646463636364636363636463646465646365656365646363636364656364636563646563636464656564646363646465646463656363636365646464646563656564656563636364636365656463646564646565636364646463646565646464646365636565636363656563646465636563636365656365636465646565636364656565646464636563656464636363646563646364656563636563646563646563636564656565656565636465646465646463656464636365646563656465656563636465646364656565636464656463656363646565636364646363646363636463646564646563646364636463656363646364636465636564636363646563646465636565646463656365646465646464636365646365646563656463656565646364636565636565646363636363656363646463636364636464636563656365656564636563646465646464636365646564656364656463636363636465656565636365636363646463656463656564636464636464656465636463636365646465636564646365636364656363636363646463646563656365646563636363646364636363636365656563656364636565646465656364656563656464646564646363636564636364646565646565636364636565646563646364636365646363646564646464636463636565636363656364656464656363656463656363656565656465636564656363646465656564636365656464656364656563636563656565636364636365636365636463656463636565656563656365656565636563656565656365656465636465636363646364646465646563656563656465636465636564636363656364646565656363656364646565656463646364636565636563636565656465636463646564646364636464656565656363646365646563656364646364646565636363636365646365646363656565646463656465646563636463636364636365646565656564646563646365636465656365636564636464636464656364646565636463636564636463636364646365636565636564636365636563646565646364636463656564636464656365646463656563646365646465646363656464656465656365636563646464636464656564656365636463636565636565646564636365646563656364636563656463656363656464646565656464636565636464656563636363636463636463656364656565656563646564636365656564636365636563636365646465636365646563646564656364656563646365646365656565656464656363636365636363646463656564636464636465646564636464656465656463656465646364656363636365646363636364636364646364656364636565646363656464646365636364646464636363636564636564656365646463636464646363646363636463656465656563636363656565646364646364656364646463636363636364636365636464656465646563646564636365656563656564636365636364656365636464636463646363636365636565636464656464646463636563646463646465646363636565636564646464656565656364646363646363636565636565656463646365646563636463636564646463636365636565636463636565656363646465646365656365646563636364646365636564656365656465656365656464656563656365656564656365646465646563646465656365646365646464646564636464636564646465636564656564656564636364656363636563636365656363636465646463656363656365656465646563656465646365656564636464646363656565646563656465656563646464656363636565656465636465636465636465656563636464646363636364646465636565656564636565656463646363636464656564656563636463646464646564656565636564656464646365636363656464636563656363646365636564636363646365646463636563636365636463646365656364646565656363656365656364646465656464636563636565646564656563656464636563646563636463646464646465646463656564636465636363636364636365656365656364646465656464636565656563636463646363636565656565656363636363636565636565646564636565646564656465656365636464636465636564636565646363646365646463646563646564636364656465646563636363636464636564656465646465656465636564636563656463636463646563646365656464646365656365636463646563656465636563656365646463656463636564646364636363646365636365656464636463636564656465636365646464646365646364646365656564646463656563636363656364656463636365636365636464636463636463646563646463646364636364646565636465646363646365656463636465636365646365646465656463646565636564636464646463636363636363636564636363646363636364656365636464656565646463646465656463636363636364636464646565636363636463656365646464656464656565636364636563636363646565646465656563646464656363646565656363636463646363636364656565646464646464636565656363636464636565656463636464656464636365636365646565646565636563646463636363636363646364636464656465656363646464656363636364646365646465646465636563656365646565656563646565646563646363646463656365646563636464646563636464636465656464656564656564636463656564636363646463656465646364636565656363636363646463646563656363656463636463636564636564656465656465656563636364636363646464636363636363636365656463656365636563656565656465636564636564636563636465656465636363636364646564636463636564646364656564636565656463656464656463656365646465646365636363646563656463636465636465656563646364646363656465646363656365636465656565656364636464646564636463656465646564656364636464656564636364636565646465636365646365636564656564636463636564656464636464636464646464646464636365656465646465646463646465656465646463636363646364656564656363646363656463646564636365636464636363656465636464656364646563646563656363636363646565656565656364656465656464646464656565646463646563636563636564636364646565656364656565656363646364636463636565646565656364656363656565636364636364636463646563636465636565636464636463646564636465646463646364646365636563636364656564656563636464646464636563656565636363656363646563646565656565636563656363646363656564656565646363636565646564656563656464636565646364636565656364636464646565646565656364646463636565656563636363656363636563646364656563636363646463646464656565636563646463636364636563636463636563656565636564646363646363636365646364636464646464636464656363636363646363636463656465636463646364656564656465636563656364636364656565646565656364646365646463656364656463636365646364646565656464656363646363646563656564646364636563646363636563646365656464656463646464646365656464646363656565646563646465636364656464656463646564646465636463636363656564656364636463656465636564646464636363636365656463656465636465636365636365636565656464636365636564636364636363646565636364656463636364656465636565636365646563656363656364646464646565636463636465636563646463656364636563636364636364636565656563646364646463656563656465636363636564656363636465656465636463646363646564636465656365646564646463636565646563636465656464646465646563636564656564636463636463636464636365636464646564656363646363656463656563636465646563656365636564636363656464646563646365656464636464646563656464646463646565636563636364636565636465636363646463656363646564656364656365656565656463646563636464656363636365656363636563636463656363656463656564656565656465646465636564646364636363646563656465656464646564656564646565656363636365636363646565656463646564636563636564636465646465656465636364636365636565656364646363646563636465646465636564656465656363636363646463636564636363636365646565636464646564646365636563636463636464656465636463636565646465636563636565656463646363656465636464656565636365636563646465646563646365656364656463656465656463656464636564656463646364636464636365656363646565656464646563656565646363636565656563646365646364636365636365656563656563646563636464656364656463636464656463656563656563646364636463646464656365656565656363646465656563646563636465656464656465656563636364636563646563636564636564656464636464656563646565636463646565646464656565636464646563656364636465646464636464656365636364646363656563646465646363636365656364646364656464636464646565656465636363646563646464656364656364636464636365656365656464636363646463636463646464636464646563656463656565636364656564636563646563646364656365656463656565636465636363636464656463656463636563636463656564636364646363636363646363656563656364636463636463636364636363636463646463636465646463646465656463656463646563656364636464656565646563636565656464636363646463656364636564646464646464636365636364656365646563636464646363636565656564636563656563646365656364656565636563636363646564646464656563636465646565646464646565646364656464636563636463656563646463636465646465656464636365646564636565646464646564656563656365636465656564646363636363636364636364636365646563636565646563656465656363646364656463636363656365636565646563646364636463646363646364646563646464656563656363656363636365646465636465646463646465656464656564636564636463636364636365636365636463646363636363636564656564636464636463656365636364656463636463656364646364636463636464646464656363646365646464636363656565646563636564646365656363656565636463656465646465646564656363656464656365656364656365636365636365646463646363636563636365656465646365636564656465646463636564656563646465646363656463636463636465636463656463646563646564646464646464636565636563636464656463636364636564646564646563656463646464646464646465656563636364656363646465636364646365636464656564656464646463656565646465636564636363636464656563656463646363636565656563656465656364656464646464646464656565656464646365636463656563636463646365636564636465646564636565656363636464656563646363636563646365656565646463646464646363636365636564656364656364646364656365636565646465646364656563656464646364646563636563656363646365636563636565646565646465656465656564656564646365636365636464656564646363636564646464636464646563656463646565656564646563646465646465636365646364636565646565646465656464646464646365636563656463636364636363646564636364636365646564646565636564646463656363636363636365656563636464646363646363646465656363646465646464656364636464636463636363646464646563646563656563636463646564656563656464636363646564646463656563656563636363646363646365646464646465646564646464636365656563646364656464636364636464636365636564636463646363636565646365656465656463636363656364646565646465656563636565646365646465646465656564636465656364646565646563656464646365656564656565636464646465646565646564636364636464646364636565656365636465646364636465656363656463636564646364656464646465636464636463636563636563636365656563636463646564636464646364656563646565656465656565656565646565656565636563646465646563656463636465636365646363656463636363636364636464636563646464636363646363646464636463636464636463636364656463636465656463646363646364646465636565646364656463636463646463646365656564636464656565646564636563646563656463656464636465636465656364646464636564636365646464646363646365646463646565656565646564636463646465636364636363646365656565656565636463636565646564636365636564656464646365656364646563636563656365646564636463646563646363656463656365646564656563656463646463646563656364636363646564646363636364636365646365646364646563636563646564636365656465636463636363646465656364646565646364656564636464656564656563656363646363636563646364636464656464646363656363646364636464636463646464636463636564646365636464646365656565636364656564646563636364636363636463656464656464636563636364656364656564646564646563646363646364646364636465656564636364636364646563636465636364646463636365646465646464656463636365636565636363646463646464646465636363656363646563636365646563656363646564646363646363646464646563636464646465636464646365636365656564656564636565636464636463656563646364646363646365656464646565646463646565656364646363646363656365656564646365646364636463646365636464656363646563646565636463646565646465636464656363646565646363656463656365646564636364646564656363646365656463646464636464636365646564646363646463636364646463636463646464636565656463656563656364636363646565656463636564656463646463656563636464656363636563646465636464636565646463646465656464646363646463656565636363636363646465646363646363636564636565656563656365656565656363636565656563636465636365656564636365636364636565646365646364636564636363646365636363646363636365656365656465656363636363656363646363636464636364636564656564636463656563646363636563646365656564646365646464656465636364656364636463656365636463636563636364636465656464656563646464636463646363656365656463656464636364656365636563656564646564646465646365646564646464646465656363646464646463656363636563646563646364646564656364636365646365636565646563646564656363636464656464636465656563646564646365656463636464636464636465656463636465636364646363646463636363646563636564636565656463656465646364646364636365636464636464656465656465656463646563656364636464646563636363656364656364656363656564646363646364636463636464656565636365646565636363656364646365656464636364636563656565636464656564646365656563636363636563636563646364656463636464636363646363636363656564636463636464636564636464656364646365646465656365656364656563636363646363646365646565656464646363636563656464646363656363636564646464646364646563636464636365656565636364636364646365646363656364646464636563656463636465646564656565636364656364656365646363656564646365646365636565646365646364636463656364646465636565636565636563636363656365656364646463656465646565656464656464656365646364636565656564646563636365656363636465646565646563636463656463636463646363636563636564636463636464636364656465646463636563636564636363646465646363636465656364636565636565636563646365646465656363656363646464636463636464656463636464646365656365646564646564646364636365646363636365646464646565656565646563656564656363646363636565636463636464636363636365656465636463646363646564656565636563656564636465636565656565656363636364636364656563636463646563646465636463646563636563636565646464646364646365656363646364656464636363646365646363656365656565646365656463636463636565646563636363636363646364646363636365646365656364656364656563646464636463636464636563636563646564656363646565656564646364646565646464646465636464636465636465636363646364656364636365656563656364636363646564636365656365656364656464646363646365636565646463656565646365646464646565646565636563636564646564646463656465636463636365636463636363646564646565636364636564636365636563636363656364636565646563646364636465636564646465636363636365656463656464646565636363636465656363646565646363646464636365636465646563646563636563636565646463636364636365646364656463646364646363646563646565646465646365656365646465656363656465646565646465656364636465656565646563646363636363646363646363636564646465636465656464646563636465646564636364656563636365636563636365656463646564636363646463646564656564636363636564636365646363656363646564646464636365636564656563656465656464656564656364646363646363646363646364636364636365646363636464656563646364656365646464646363636565646364646364646465646465646364646565636564656565646364646463636363636465646463656364636463656363636464646565656465646463646364646364646363656365656363656563646563656463646363636465646364656364636363656464636464646464636563636364646465656363656363636565636365636365656564656464656564646363646465656565636465656563636565636465636464636465636563646565636464636563636464656465646565636464656565636365636564646363636464646463646565656465646364636364646463646564656565636365646364636363656564636364646363656363636365646365646463656464636464656363656465656465656364656564656365636363646463636363656364656563656463656365646565636464656464656363636565656463656464636464656563636465646465636363656365646565636465636363656365656563636564646565636364656364636364656364636463646565656465646364636363636464636365646465636563656564646364636564636563656365636463656463636464646563636565656563636363656365646464646464656363636364646363656463636465636564636365636563646564656464656364656364656364646363636465646365636565636463656564656363636464636565646363656464636364656465646463656463656363636365636363646364656364656565656465646363646563656364636564656564636463646364646465636565636365646365656464646564646463646363646363636564646465646563646464646363646365646563656364636464636565636564646465646364646365636465636564656564636365636465656464656464656463636463646563636565636565636465636364636365656463656365636363646563656565646465656463646363636563646365636564646364656463646363656464646363636365656463656464646363646363636464656364636564636464646463646463646564656363636564636564636364636363656465646564646463656564636565656465646563636564636464646364656364656363646363636465656363636463636564636465656465636563656365646563656463656365646565636563656565656465636465636563636363636365656463646563636565636564646565636465646565646363656363656463656365656564636465656464646463646563656465656564636564656363646363646564646564656564646465656363646564636564656464636363646464646564636365646463646565646463646365636464636564636363646364656465656363656364646564636465656565656365656465636365646364656565646465636365656564656363636363636463646563636564656363646565646463656563636563656364636364636463656364646564636364656563636365636363646463646364656364646464656363646364656464656364646463636565636564646364636465636365646564646563646365636463656463636463656363636463656365646363656365646563656363646365646463646565636463646365656365636363636365646364636464646564636364656564656563646463656463656465636365656463646363646564636364646364636365656364646465646364636564636465636564636363646564636463646465636364636465646465656364646363656463656363636463646565656565646365646364656563646365646565656463656465636563646564646463646463656564656565636563646564636363646365656364656465636564636364646465636564646465646463646465646464656465636363636364646564656465636363646563646464636563646365646364636365646363656464646563636465656365646565656364646365656363636563656565636465636365646565656464636563646463656365636365636365656565656363646464646463656563656563656464636463646463636465636564656565656363656363636365646564636364636565656463636463656365656464636363636365656564636363656564636364646564656465656364636365656564656364646563636463656464656465636364656363656464636364656465636463646464656364636363636364646364656563656463636364636563656564636564636464656363656565636564636464646465656465636465636565656363656565656364656365646465656363636463636563636464656363636464646463646363636563656363646464636364636564656364656364646364646363636463656463656465656365646363656363656464636563636365646365646464636565646464636565646363636465636363636363636364636463636364656364646463646364646564656365636464656364646363636363636365646363636363646564656464656464646465656365636364656365656365646463656365646563656364656563646363656564646463656365646565656464646364656464656464646464646364636464636564656365636565646463646463646463656564636565646465656465646464656564636463636365646363656565636364636464656363656465636363646465646564636564636563646465656465656465656365656463636563636564656463646563646563656565636365656564636363646464646463646463646465646563646364646364656565636464646464636463636364636463646463646463646563656463646365646464636563636464646363636463646364646564656365646463636564656464636365636564636564636465656463646365636565656464656463636465646565636363646563636365646563656365656465636464656564636365656365636564636463636464636563636564636564646563646564656364646563636363646565636363636464646463636463646363646365656464656363646365636365656463646564636365646365656463656564656563656364636564656364646365656463656364636463656365646565646565646364656464646465646363656364646364646465636465636564636463646465656365636464636463636564646465636363636465636363636465636563636365636363656465656363656365656463636465636363646464636463636564636563636363636564636363636563636463636365636465646363646463656563656563636563636565636463656563646363636364646363646563636365636565656363646565636364646365646465646364656364636463646365636563646363636365636465656365656563646565656364646563646363646365636465646563646363636363636363646363656565646363636365656563636363646563656364646464646365656365636463646464656565656365636365636563646365636363656465656365646463656363656565646563636564636563636564646364636465636364646563656363636564636464646563646464656365646463656565646565656363656363646364636464636564646464656564646464656465656364636465646363636563656364636463646563656364656564656465656565656563646565636363656463656364636463646363636463646365646563656365636565656364656363646563656364636464636564656463646364636363636365646363646565646463656365646464636463636363646364656463636363636364656564646363656564636565656363636363636564646364636565646465646364636563636363656464646463646465646564656465636563656464636564636465646463646363636565646365636464646563656565656563646363636565656565636564636364646563656463646464646363646363656563636563636464656565636565646365656464656564646563646364646364656465656363636563636363636563656565646464636364656463636463636364656463656463656365636363646563636463656363636365636564656365656364636364656364636565646463656463646364656565646463636563646564646464636365636365636563656465636563646363646464656463636364646363646564636465646364636563656365646363656563636365636464656463656464636463646463656463646463636364646364646564636465656464646365636364636565636564636564646364636364646564636463646465646463656364656363636463646565646365636564636464656364636365636564656464656565646363636464636565656563636465656564656365646565636565656364646363636465656465646365646463656465636563636364636565656563646364636564636565656564656364656465646364636464656565636464646463636365656363636463646464656464636365646463646564656463636365656565646463656563656564646465636564656563656463636363656565636464656563656463646364656365646463636363646363646563636564646365656463646464636563646365656463646464636463646463646464646364636565636364636364656564646565656465656363646363636363646463636364656563636563646563656563636364636565656363656464646564646565646364656363636563646463636365646365656464646465636365646365656563646365636565636364636564656565656565646363646564646565636564656563646363636463636464646464656564656565636565636364646463646464636363636364646363636463636464656463646465646465646364646365636563636463646463656564646365656364646565636464636365656364646463636563646364646364646464646565646363646365636563646463656464656463646465646365636465646564636364646365636364636563646365646364636463636563646464656364646365656363646364656464636563636464646364646463646465636565656464636364646363636463636365646465656563656463636564646363636563636563646363656464636463646464636464656564636565656364646463656563656364636563646363656563636365656464636564646564636564646463656363656363646364636564646465656363646564646564656463646463636464646363646364646464636565636464646465656363646364646465656563646363656563636563656463646463646365656364646564636563656465646365636563646363656365646363636464646465646363636563646565656463636364656563656465636565646465636363656463636564656464646565636465646464646363636465656465656465636365636363656465656563646465636564656365656564636364646463636464636365646563646365646564646364636563656463646465636564656364656463636564636365656465636363636563636365656464636465636364636465656363636564636565636564656364646565646465646563636465636463646363656563646563646365636464646364646465636564646463636463656363636465646564636464636465656464656364656463646363646564656365656365646564646464636463646364636465646563646364636365656465656463636363636364646563656363646364636364656365636563656563656364636463646565636363636363656464656565656365646465636363636563656364656363646563656364646465636363636364646364636565646463656463646364636563646365636564636365646464646465646564646364636564646464636363646464646465656565656363656563636463636364636564656364656463646365656565646365636563656463636463656364636364646365636465656563646464636365636563636364656465636463646363656365656565636464656465646465656463636364636464656365656464646463636364656364636363656364636364646464656364636563646363646463636365636465636464646465656463646563646464646365646464636565636564636363636465646463646565656465646463656363656363646564656564646465646365636465646363636563656363646563656463636364646563636463636564646464636365646364656463656563636365636363636565636365646464656464636465656363646563636464636565656464646363656564636564646364636365656363636364636564636365646364646365636564636465656364656564656463646463646363646463656463646365636464646564646363646463646563636464636464646564636363636564656365636564656465636465656364656463656564636364646564646564636465656563636465656565636463646563636563636463656463656464636565656464656363636364646365646464636363656364656563656463656365636564646364646363656564656363646463646565636464636464656365656463646364636465656464656565646363636365636464646365656463636465646465636465636565656563646463636463646463646565636465636365646363636565636363636365656565646365656565656364636363646564656364636463636565636565656564636463636363636565636564636465646464656365656564656565636564646364646465636365646563656564636364636564646464636463636365646363646363636564656365656463656563656464636465656363656563646564636463636465646364646564656563646363656364656463636364636565636563646463636364636564646463636563636565636363656365636464656365636363646464646564646565646565646464646563636364636465636364636363636564636363656465656463646465646564636564646463636364646365636565656563656564646465646364636565646364636365646363636365656465656464656463636464646363636463656464646364636365656363636465636364636365636563656564656365646563656565646363646363656363636565656365656565656465656564636363656563636563636465636363656463646464656464636464636363636464636565636363646463636363636365636563636363646463656465636364646364646363656365656465646463636363636463656564636365656463646465636364656465636563656463656564636365646464656565636464646564656364656464636563656363656464656564656465656365646363646363656565636564646464656364636564656463656564656565636365636363656463656565636364636364646465656464636363646464636465656464646463656463636463636363646363656563636363656465636363646465636464636463636564646463656464646364646464636464636564656563646463646464656565656464646464636564636563656364646464646464636363646365636463656363656465636463636364636363656364656563656564636563646364646464646464636564646565636564656563656564646365646565656563656563646463636363646465646363636365636363636464646564656564646464636563646363646564636563646565656465646564636365646465656565636463636464646563646364646463646365656563646465636565646363636364656565646564636563636563636565646363656564656565656463646463646464636464636363656364636564636464646463636464636563656364646365636563656464656364646363656463636463636463636563646563636564636465656365656463656463656464646463636564636465636363646563636564636365636563646464636463646363636563646363636565646463656465646465636364636463636363636463646564656364636563656564646364646365646463646463646363636564656563636565646463636365636363646464656563636465636363636563656463656565656465636363646364656564636463636563656464636365656565646464636563656363656563636563636463646365646363656464636363646565636564636463646365656563646364646363636464656463656363646563646365656365646463646565646564636563636365636464636463646564636564636363656565646465656363656463646563656565656464656464636564636463636364646365636463656363656565656463656363646564656564636365646563636363646564656465636565646463636365646365646363636565636364636363656365636564656565646464656564656565646563646463646465636465646364636364636363656563636465646363636464656463646365636365636564636463656365636464646564636363636363636363656563636364656464656363656564646563636463656563636365646364646463656363636564656364646465656564646563636564636463656363656564636565636364636464636465656465656465656465656563636464636364636565656565636364646363646463646364646463646465646464636563646463636363656363636363636365646564646464646365636363656364636364656563646365646564646463636564656565636363636365656365636465636363646365636363656364646365636464656365656465646463636565656364636463656463646363656364636463656464636363636564636463636564646363656365636363636564636364646563636365646463646463636363646363636363636464656363636464646463636563636363656565646564636463636463656365646363636463656365636465636464646364646463636464636364636563656563636363636363646563636564636465636364646564636363646363656563636364636464636364636563636363636363656365656464636364646363636564636563646563636363646464656365656363636363636463636563646364636465636363646365646464656364646463646363636564646363656365656465636564636363656363636463636464656463636363636565636463636364646363636365656365646363646563636363636363636363656365636563646363646563636463636364656363646464636363656363636563646364636365636563646363656465646365656364656464656364646365646364646365656565636365636363646363656363636465656563646363656464656564636363656365656563656363636364636563646563636363636363646363636563656563656463636363646365636563636364646364636364646363636464646365636365646563636365636565646365636363636363656465646364646363636464636364636363636565656563646565636363636464656564646365636364636363656364636563646563656363656365646465656363656363636465636364646563636463646364636363646365636563636463636363646364656565646563636363636364636364636365636363646363636464656363646363636364636363636363636464656364636463656363636365636464646364646365636364646464636363636464636364636465636464636563646563646563636464656565656563636463636463646363646363646363636364656363636463656363636363636364636363646464636365636464646464636363646363646364646563636564656363656363636463656563646464636565636463646364636363646363646364636363646363636463636564636464636365636464656363636365656563636363656364646363646365636363646464656364636463636464636564636364636363636563636363656564636364636365646363646364636464636363636363656463646364656564636363656464646463636363656364646564636565646565636365656465656463646364646364656365636564636563646363656463636463646464636463646463646363636465636565646563636465636364636463646363656464636463656463636465636365636465646463636464646363636365636563636363636363656564656365636365656363646563636464656463646363636363646564636464636363646364656563646463636365636364636364636364636365636464636363646365636563646364636363636464636463636363656365636365646563646563656564636563646463636563636563656365646465646365646563646363656365636365636365646465636563636364646563636564636464656464636363636563636364636363656463636363646465636363636364646465646565646463636565646563636464656465636364636563636363646363646463646463636563636563656565636363646464656364636363646563636564646363656363636364656463646463636363636363656464636364646463636365646464646563646463636463646564636564636565636363656464656363636364636364646365646563646365636463656564656365636365656563646364646364636463646365636463636363636464636563636463656463636363656363636363636565646563636563636363656365636363636363636363646463646463646363636464636563656563656563636365636463636363656563636463636565656563636565656364656364646363656463636364656363656364636565636364636363636565636563636365636365646465656364636364646363656365646464636563636464636464636365636465646363656363656565656365636363636364656565656463656564636363636563636564636365646363656363636464636363636363636363636363646365646465636365636365646564636364656365636564636365656363646463656465646463636564636463636463646363636463646563646363636463646464646364636365656365656565636463646363636363646465646365636364636564656365636563636463646364646364636364656363636364646364636563636463636463636465636363656363646364656363656463636365656363646365636464646363636364636565636365656464646363656464646363656363636564636365636364636464636364636363636463636565636565656464646463646565646564656364646365636464646463636363656564636563636565646563636364636363646463636364636564636363636565656363656363636365636363646363636365656364646465646363636364636365636364636363646363646463636463636565636365646363636363656363636563656364656364636563636463656563636364636564646563646565656463656365646464636365656563636465636365646463636565656363646363646563636463646363636463636465636363656563636363646363656563656365656363646364636363646363646365636365646463636463636464656565646364646363636363656563656364636363636363636563656563636463636363656365636464646563636363646464636364636563636563656363656365636363636363636565636563656365656565656463636463636365656363646363646364656563636563646363636564646363636563646363646564636365636463636364656363636465636563636464656565636564636463656363646364656364646465636564656363636563656364636363656363636463646463636563636463656563646363636363646363636463636563646463646464636463656363656364636365636463656463636563636464636463636463636465646363636365646363636364636363646364636363636364636565636365636365656564636363636463636464636463646464646463646563636365636465646565656365656363656464656463636463646363636363656364636564636564636565636564646463636463656463636563636563636565636464656363636464656563656364636463656563646363646364656465636464636363636463636563646463636464656364636364636464636364636463636563646364656364636364656463636363636465656363636364646365656563636465646563646363636563646363646365636564636463656363636364636364646364636463646565646465636463636365636365636363656465636463636465636365636463636563646563636365646464636463636365636365646363636363636465646364636363646463656363656363646563656365656463656464646363636363656565656563636364646463636464636464636565646464636463636464656365646364646564636563636365636364646563656363646464656563636464636464636365656363636364636363636364636363636463646463646363636463636564636363656463636464636463656363646463646364636464636463636364646465656464656464656465646364656365646363646363646365646363656463646463636565646463646365636463636363656464636564636363636563636463636364656363636463636365636463646363646363646465646365636463646365636365646363636463636363646365636365656365656464656365646363636463636463656363646364636563656463646463656363636363636565636364636363636363646363636363646464656464646563636563646363646363636464636563646363656565656363646565646364646463656463656364656364656465636364656364646563636363636365656564656564636365636563656564646565646364656363636464636463656364656464636463656363646464636363636364646364636363646365636463656463636363646364646463656464646563636463636465646565646565636463656563646363636364636563636564646363636563646363636564646564636463636563636563636364656363636365636364636565636464636363636563646564636365636463646563636363636463656563646364636363656363636565656363636563636363636465636364656464656365636363636365636363656463636563646565636463646363636363646363656364646465656464656463636464646563636564646363646363636464646463656363636363636363636363636363636364636365656363656363656363636363636465636563646363646364646364636564636465656364646365646363646363636463656364656365646363636364636363636365636563656463636363636363646563636365636363646364656465646563636363636364636363636565636463646363636464656465636363646465636363636364646463636363656363646463636463656364656363636365646365636365656465636363636565656563636565636464656363656365656365636565646565646363636464636365656365636463656364636364636465646563656363636365636363646563646465656363636365636563646364636365636363646363646564656364636363646364636365636363646364636565636364636563636364646364656465636564636565656363656463646564636365656465646363636365636565646363646463636564636463636365636463636364656564636363646363646363636563636564636563636464636364646463636463636464636564646564636563646365636363636363646464656563646464636364646363656363636463636363636465636464646365636464636363656465646363656463636365656363646463646364636364646363646463636463636565636563636363636363656564636363636364636464636363656363636464636463646364646364636363656465646363646364646363656564656365646464646464636363646363636364656463636365636363646363656365656363646464646363636463636365636363656364636365646563636564656463636363636363656463656364636463636365636363646363646363656363636464636365646363656363636363646364646464636564636463656464636363646363636363636463636363636363636363636364636364636363656364636363646365636463656463636464656363636565636464636365636565646564636364656463656364636464646365636463636364636363656363636563636465646564636564636363636363636563656463656463636364636363646364656463646364636465636365656363646365646463636363636563646463656463636365636464636363656463656364646363636465636363646563636363636465656363656565636363656563636365656364636365636465646363636363636463646564636363646365656563636463646563636365636363646365656464636463636363636563636365636565646463636364656363636463636463636463656464636463656463656365656363656364636363646463636563636363636363636365656465636364646364636363636363636463656364646364636465646365636363646363636365636463646464646563636565636563636464636465646464646363636463656363656365636364646365646564656364636563636564646463656563636565636463636364656363636564636564646564646465636364636464636564656363646463636565656463656464646363636565646464646363646364636564656363636563656563636365646363636364646364636363646464656463646464656363656464646363656564646563646363646363646363646565636365636364646364636363646464636465636465646463636364636364646463656363636564636365636364636363646363646364656365646464646365636565636465636365636364636563636364646564646563636365636363636565636563636463656364646464646365646564656364636365636565636363636363646363656363636363636365636364646363636363636364636465636364636365646464636364636464646365656364636363656363656463656365656563636563636465656363656364636465636363656363636363646464636365646465636463636563656364656363636363636365646365636463646563646363636363656463636363636563636565646363636463646364656563636564646363636463636365636364636363636463646563636365656464636364636363656563656564656364636364646363636465636363636363646563636465646463636563646363636463636363636463636464646363636564636363656365656365656465636363646364636364636363636365656564636463656464636563656464656364656363636363656463646563656565636563646365636564636463656363656363636564636563646465636564656463646363656364636363646365646363636365636464636365646364636465656364636364636364636363636365636565636363656563656463656363656465636364646465656463636365656564636464646464646564646365636563656363636364636464636463636465656563646363646363636364646363656563656363656464646464636463646363656465646364646363636463636365636364636365656363636563636465636364636364636365636564646364646364656564636463656563656364636463646363636465646363636463636363636363636464636363636463636564636364636464656363636464646365646465636364636463636365646563646465646365646364656563656364646364656363646364636465636363646363656563656363636463636365656363656563636464646564646365636564646564646465636565646364636363646364656363656565636463646365646463646364636363636363636464636363636463636464636464636365636363636363636364636463656463636563656365636363636465646363636364656365656563646464636364646564646364636364636563636365656563656463656564646365646564636363656363656363636463636463656365636363636463636363646463646364646565636463646463646363636365656364646463636364646363636363636363636463656365646563636363656464636565656363636463656565636464636364636563636365646365636364656463656365636364656364636365636564646364646363646364636365636464636364656465646365636465636363636463656565656463636564636565636363646464646463636364646564636463636463656463636564656363656463646365646364636365656365636563636364646464636363636463646463636465646463636363636364656565646563636365636464656364656365636563636564636463636365656364636364646464646365636465646363636364636565646363646364646363636365656563656363656564656464656364636464636365646363636365636564636564636363636565636463646463636563636365636464656563636564656364636463646363646463656363646565636565636363636463646563636464636563646365646365656363646364636564636463636365656364636463636364646463636363636363636465636463656363646364636464646363636364636463636463646463656365656464646563656463646363636563636465636464636463636563656363646364636363636363656364636565636365636463636463656463656465656363636363646563636363636363636565646363656563636363636563646563646363646563646563636363656465636565636365646363656365646363636563656363656364646363646463636365636563656363646364636563656363656463636363646363636565646364646363646564646363636363646464646364656364636363636363636463646563636564636365646363656364646363656563656565636564636563636363636363646363656464646563636363636363636563636363646364636364646463636463636464656364636464636363636363656563636363656364636563636363636463646363636565636364646365656364656364646365646465636363656364636363636364656563646463636463636565636363636363646364646564636465636463646363646463636364656463646463636464656464636365656364646365656465636463636464636463646563656564646364636363646365636464646465656363636463636365646363646364636463636365656365636363646564646565636364636463636463646364656564636363646463646363636364636365656364636564636463656365636465656563636363636365656363656364636463636463646463646463636563646363636565636365646363646463636363656365636463636465656563646363646463646364636563646563646465636363646365646464636364636363636464636563646365636464636363636463636365656464636564656563636563656364646564656363656364636565636464646565636363646364636363636463656563646365646365636564636364646464646465636463636465656565636563636463646463636563636364636363636563656565636365656564646465656463646565646363646363646564636563636363636465636465636464636364646363646364646363656364636363636363636463636463636363646364656463646563646364636464646364636463646363636564636464636463656363636464636465636364636463636463636464656363646365646363636364636363636463636363636363646363636363656363636363636464636363636363656465656364656563656365656365636463646363646364636464646563636563646365656564636363636363636363656365636465636563656363636363646465656365636363636363636364656364636364636365646463636464636363636363656565646363646364646465656365656463656364646363646364636563636364636363646364636465636363656365656364636363636364636364636565656364636463636563656363636464646363636364636463636363656363636363636563646463656363636565656465636365646463636363636363636363646363636365656363636363656365636463636463646365656364656365636564636365646363646464656464646363636365636464636464656363656363646363646365636463646363636464646363646363636463636364636363636463656465636364636565636364636363636465646364656565636363646465636564646363636363646364636563636465646365646365636465646363636365636363646464636563656365646363636463636463636363656364636463636463646363636364656365646465646563646363646463636363646363656463656365656563646364656463636463646564656363636563636564656565656364636563656464636563636464646364656365636464636365636364636363656564636364646363636363656564646365646463646363636565636363656463636463656365646565646364656365656564636563646363636363646363656363636464636363646463636363646565636465656565636463636363646363656364636463656363656463636363636364646464656563656363636363636365636364636464636363656464656563636363646363656464656363646564646363636464636563636363656463636363646364636363656565636564636364636463636463646563646464656363636365646364636365636563656463656463636363636564636364646563656363656565646363636365646464636463656465636363636463636463646363656464636363636364636564636463656564646364646364646563636463646563636363646563656365646463636465646363636463656363636363646465656363636563636364636563636363646363646364636363636564646364646365656363636564656463646363646363636364646364636565636563656563646463646363636363636364646363656463636363646565646363646365646363646364636563656463656465646563636464636564636563636463636365636364636364636465646363656363646363656464646564636364636363656363636363656464646363656363636365636364646563646363636364656564656465636564636564646564636464656463646465646364636463646463656563636463656463656364646364646463636365636364636363636464636464636365636464636363636363636464636565636363646564636363656463636563636363636464636363656364646564656363636465656463646463636365646363636364656465636364656464646463636363646563646363646463646363636463646364646364636564636365636463636463636563646364636563646463636365656465656464636465636365636465636463656365636363636365656364646364636463646364636363646363656363656364636463636363646363646564646464656364656464636364636463636363636365646563636463636363646465636564646364636364646464646364636363656463656565656364646463646363636565636563656365636465646463646363646364636364656463636364636363646363656463636365636465636363646563636565646465636364656363636365636365656463636364636363656463656365636364636464646364636463656364636563646363636564656463656363646364646563646363656563646363636564646565636363646363636363646464636465636363646365636363636363656365646363646363636363656563636363656364656364656464656365636563646363646465646363636563636463646363646363656363656465656363636563656363636363636363636364636564646463636365646363646464636564656564656365636363656565636363636364646564636363636363636364636463636563646363636464646365646364646564646463636464656363646463636463636464636565636565636364646364636363636364636464646363656463636463646363636464636363646464636363636363656565646564636363646363656363656363636463636464636565656463656364646465646563646364636365656363636463636365636363656365646363636465656563636463656565636464656465636465646465636363646463636563636464636464656363636364636463646363636563646465636365636365636463656364636364656563656364636563636463646464656363636463636465646363636465636465656463646363636364636365646465636463646363646465636565636464646463636465646463636363636363656464646363636465636364636365636363656563636464636563636463636363636565646365636563656364646563656564636363636363646363656465636363646464646363636463636463646364636364656364656565636363646365646563656563636465656363646363636563646464636464646565656364636363646564656363636363636464646465636363636463636363636464656464656363636365656565636363656363636364646364636363656563636363646364656465636463636563646563656364636564656363636463646363636564636563636463636465636364636565646463646563636563636363646564636363636564646363656365636365636465636563646364636465656463636563636364636363636364636363636563646564636364636363636363636463636564646463636464656364646464646364646363646363636365636564656463636365656365636364656365636564636363636364656464636364636463636364656563646364646464636363646564636364646365656363636365636563656465636463646365636363636464636365636463646463636363636565636463646463656464636365656363656363636564656464636463636363636363646364646363656364656564636463636363656465636563636565656464656363636364636563646364636364656563636364636463636364646363656463646363636465636364636563646463636564636563636364636463636463636363636363646465636463636464636565636363646364636363636364656465636465636363636463636365636364636363636365636463636464646463636363646563646363646365646364646363636363636463646464656363656363636463646463646363636563646464646363646364636365636463646363636363646563636365636363636363636364646464636363636363636365646463636363636465656363646465646463656363636363636463646463636463656464636364646463656365636463656365656363646364656564636363636363636465646563636363636363636463646465636363646363636564646563656363656363636463636364656363636465636465646365646565636464636464656364636463636364636464636564636465636465646363636563646365636564656363656565636564656564636363646463636464646463636465636363646363656363636363646464646364646363636463656363636363636363656465656463636463646465636465636364636463636363636364636363656363636465646363646365646364656365646563656363646363646363646565656364636363636563636563636365646563646365636564636364636363636363646465646463646463646363636563636463656363636364636364646363636463646363656363646463636363656365636464646363636565646563646563646464656464636563646463656363636363656363636365646363636363636363646363646463636363656363636363656463646563636364636465646563636365646363636364646365646364656463636364636365646463636563646463646565636463636564646363636463656365646363656363646363636463636463636463646364646464646363636363636465636363636363636363636563636364636465636464636463636463636464636365646363636565636364656463646564646464656363646365636463636563656463636563646463656564636363636463636564636563636464636465646363646363636363646363636463636463656464646365636564646365656463646463656365636364636564636464646364656363636464636564646465636564636365636363656564646563636464636465636364636463636464646565636363646364636465656363656364636364636565636465646365646464646463656364646463646365636463636364656364636364636363636364646463636365636363646463636463656364646464656463636563636363636465636564636565656363636463646464646564646364646365636364656364636563636365636364656464656363646563636363636565636463656464636565656363636564636363636365636565656363656363636363636365636364636363636363646565656563636564656463646365636365656365656364646363636465636563646364646564646465636363656463656563636364646564656563636563636463636563656463646465636365656363636365636463636565646563636364636363656463636463656363636363636464656464636363636363636563646563646363636563646364636363636363636563656463646465636363636564656365646563636463636463656363656364636363646463646464656564636365636463636464656465646564656363656563636564656363636464636363646563656464656463646363636363636363636363646563636564636364646363636563636463636363636363646463636463646364636463656363636365636365636364646464656364636364636364636463646464636463646364636563636363646365646364636363656363636564646563656464636464646463656363636363636364636363646463646364636464646363636463646564656363636364636364636363646365636363646464636463636364636363646465636465656565636463636365636365636364656563636464636564646363656563656364636363656363656563636463656564646563636563636463636463636465636465636464636463646463636563646365636364636364636363636363646464636364636363656364636365636364646363646363636365636563636363646464646564636363636464636463636463646563636363656363646363636563636563656565646363646463636363656364646364636463656364636463636365646463636465656364636464646463636363636365636363646365646565636365646364636365636463636465656463656564636363656363656563656363656563656565636563646564636465646465636364636463636364636364646463646363656464646363636463646464636563656563656463636563636364636464636564636365636363636563646465656563636463636463646565636365646365646465646564636363646363646363636365646463636363636365656363646364636463646464636364646564646564646363636363636365656465656365636363656363656363636464636365636364636365636563646364646463646363636565636364636463646364646565636465636363646564636565646463646364636464636363656363636365636563646363636564646465636364636363636464636363636463646564646364636563636363636364646364636363656364636463646563656363636363646464636365646364636363636564636364636463636464646364646464636363636364656363636363646464646564636464656364636464646364646363636465636363646464656363646563636464646463636563646564656464646364646365646363646363636565636363636563636565636465636564636464646364656565636363646565646365636565636363636363646363636464636363636563646463646363646464656565656464636365656564636463656363656463656364656565646365656365636363646363636363636564656363656363656465636364636365656364656465656465646465636463646464636363636363636363656363636364636463636563646463636365636363636463646465646463646465636564636563646363636363656465646465636365656565636363646463646563636465636563636364636363646463636563646564646564646564636565636365656465636464656365636563636364646364646564636463646564636364636363656365646363636564636565646463636363656363656565656364636564636364646364636565636363646564636363636463636463646464656365636463636463636365636363636463636463636365646463636363636463636365636564646563656364656365636463636465646365636364646463646463636364656465636363656464656463636364636465646565656463636364656464646364636564636465656464646363646463636363646363646365636565656364646365646363646364646365636463636465656363636565636465636363656465646465636463636563636463636363636464656565646364646365656365636364636363656363656563636363646364656363636463636564656565646565636465636363656564656464656364646364646565636363636564636364646363646364646363656365656564656463636465636564646563646563656363646365636363656365636363656463636365646364646463636363646463636363646365636564646365636563636364636365636363656565636365656364646463636464636463646363656563636365656564656463636563636363636364636363636365636463646563636364636363656363636363636365646563656363636464636463656563636464636365636365636365656363656364636363646363656365646363646363646364646363636464636363636365636565636464636563636365636563646565636465656364636463636363646565636364636363636363636365636463636363656464636363656363646463656364636363636363636364646465636365636563636365656363646363656364656463636565646363656564636363636465646363646463646465646463646363636363636364656365636465636365636465656465636465636365636465636365656463656463636364636365636363636363646365636563636564636564646364646363656463636364636464646464636564636363656563656565646363636465636463646363646465636364636565636563636463656364646463656463636363656363646363636363636363656463646365646364646364646363646364636365646363636463656363646465646363646564646363656363646565636364656464646363636364636363636363656564656563636563636363636364636565656364636363646363636363636463646363636463636563646363646463646365636363636465636363636563646364656364636363636364656364656363646563656464636363636363646363656363656365656465636364646463636363656463646563646463646464646363636365636463646364636364636563636364636363656565636363656365656563646365636463636363656363636363656363636364636564636364656363646364646364646464656464636363646363636365636363636363636463636364636463636363646364656463636463636463636465656463656463636364636463656364636463636463636563636364636364646565636563636365636464646365646563646463646463636364646365656363636463636363636463636363636465646463656465656363636363636363646363636364646364646364656363636564636364656363636364636363656364636364636464646463646363636364636564656464656563646363646365636363636563646365646463646363636363636463646363636463636564646365646364646564646363636364636464636463636463646563636563646364636363646463656364636363656563656463656364636563636463636363656363656364636563656364636563656564646563636363646563636463646464646364636363636465636564646365636463636365636365636563646363636363636364636364646464646365656363656464636463656365646564636363656365656363656463636463656463656465646564646565646464646463646363636363646563636563646464636563656563646363636363636365636564646364636363656363636565636564646365636563646363646463656563636363636464636464636363646464646364646364646363636463656464656563636363636364636363646563656364636364636565656463636564646365656463656364656363636363636464636463636363646463636464656365636565656463636465636463656363636563646463656365656365636563636463646463636465646463636365646563636563636464636365636564636463636364636365656563636363636463636564636363636463636365636364636564646364636463646365636465646363646365636363656565636363646565656363646564656464646465646363636564636365636563636364636464646463656363646464646365646364646365636364636364636563636363646365636564636464646363636363636463656365636465656465636364646363656565636463646563656363636563656563646463656364636365636463636364636364636364656364656363636363656564646365636364656564636363646463636563636463646564636364636363656463636365656363646463646565636565636564636464656563656365656363656463646364636464636465636363656565636563646563636563656363646465656564656365636465636363636565636364636465636363636464646563656364646465646463636463636365656363636565636364646464646364636464636365656465636363646463656465636363636363656363636364646363636464646463636463656464656463646563636365636363636463646464636364646463646564656464646365636363646364646463656463646564636463636364656465636363636563636464636365656463646365636463636363636465646363646365646565646363656365656563636463636364636364636564656363646363636463636564656463646464636363646565636465636363646564646363636365656365636363636463636463646363646563636463646363646465636363646364636364646363636363656465656465636364636364636363636364636564636463646465636363646463656563636363656364656363636563636465646564656463656363636565636463636363656463636365646363656563646364636564646363656363646365636563646463636365656364656563636463636363636363646563636564646563646563636563646363646365636365636563646365656465646464646363636563646564646564646564636464646364636463636565636563656464636463636463646363636364636563646363656563646563636563636363646363636463636363636565656463636563646563636363636465646563636563636363646364656363636363656464636363656363656363656364636364646363636463636463656363646564646565656564656563636365636363646365656565646364636564636364636364646363646564656363636363636363636365636365636563646464656563656563656364646363656564636465636365646363636364636363636564636365646363646364656363646463636565646563656364646564636365656463646463656363636365656363646365646363646563656363646465636363656363646363636364656465636463636465646363646464656364656463636363646363656364656365636364646464636363636363636563656463646364636564636363636363636364636463646363646363636465636365636464656364656364636564646363636363646364636463646564636564636563636363656363636363636465656364646563636364636363646363636565636363646364636465636465656464646563646365646365636365636465636364636364636365656463636364646565646564656365636463646364636364636363636565656363656363636363646464656463636365636464636364636365656464646363636564636463636363636364656364656564636465636364656463636363656563636364636363636465656364636364636363636563646564636464656363636464646364646364646463636465636565636463656365656563636363646465656564646363636565636363636565646563636365656463636463656363636363636364646463636364636565656463646364646364636463646363636365646463656465636365636364646464656363636463656364656465636463636564636463636563656564646463636464646363646364636465636463646563636364656363646463646563636565636363656464636363636365636365646365656364636364636563646563646364656564636563646363636463636364636563636463646363656564636364656364656365646565636563646565636363646364646363636363656564636363636363636363646463636365646563636464646365646363646463646363636563646463656463656563636365646463646364646463646564646564656464636363636463656464636463656563636563636364646564656565636465656364656463646363636563636464656364636565636563656463656365636563636364646364636563646364636563646364636365646465656363646563636363636563636463656464646364646563636463656363636363636364646564656365646363636363636563636463646363646565646463646463646365636363636563656465646464636563636465646363656363636463636363636463636363636465636563636464656363636365656364646363636463646564636564636464656463646363636363636463636564656565656363636363656363656463636364636364646363646364646364636363656563646363646565636363636363656363646463636364646463656364636363636463646563636464636364636564656365646363646364656463636465636463636564636365646365646363636363656464636363636363636363636464646363646565646363636465646363656363646363636564636363646463646363636463646563636363636564636363636364656364636365636363636365636363636564656464636564636565636463636365636564646363646463646563636463636363636364646564636365656364656364656464656564656363656565646365636463636363656365636365656463656564656463646463636563656363646564636464646463646363656563646463646363656464636463646563646464646463646465636364636363636563646365636363646563636464656363636363646463646364636364646564646365636564636364656463636364656363636564646365636563636363636563636564636363636465656463636364646463636563636463636563646463636465636565646365636365636364636364636463636365636564636563656465656365636464646463646363646365646363656565636363646463656363636363656364646463656464646565656563636464636363636364646364636564646363646365656463636364646363646563636364646363646364636564646364656564636463646564646363636364646364646363636463646363636365646563636363656464636463656364646463636465636463656463636565656363636464656364656364636364646463646363646364636463636365646463646363646463636363636463656363646463656363636563636363646363646564656365636464636463646363656465636363636564646364636364636563636463636363636365656363636564646365656464656364636363646363636465636364636363656463656463656363636464636364646363636465636363636365646564636363636563636364636463636364646365656364656365646463636363636463636364636463646363646465656564636363646463656365656464636564636563636364636363636465636364636563656363656463646364646364646363636464636564656363636363636465646363656563656363646564636463636363636363646465656463656364636363636465646363636364636363646563656463636365656464636463646364646363636364636464636363636563636364636363646364646363646463636464646564646363656364646363656364656364636364636365636365636363636563646564636464636565656565636363646564646364656463646365636363646364636564636365636463646363636465646563636563636364636463636563636563646563636565646364636363636563646363646363656565656364646563636463656465636363636464636563636465636463646365636565636363636463646364636364636365636464636365646463646364646363636563636563656563636363636363636563646465646364636463646363636365656465636563636363636363636463656465636565646563656463636464636464656364636363636365636363636563636463646363636364656363636464636463656363656463636364636363656365646364636363636463636363656563646463646463636463636364636565636364636464646363646465636365646365636363656563646563636363646363636363636463636463636364646465646364636563636365656463636363656363646563636364646364646363656363646363636463636465656465656364646464636363656363636364646365646364636464636463636563636463646363656465636364656364646465636563646364646464656463636363636363636564646463646363636563636565646364636364646463636565646363636365646364646364656364656363636563646363636464636565656363636365646564636363636365646365656364636563636363636463636564636564646363646363646363636363646363636363646363636363636363636364656463636463636365656465646565656464656363636465636365656464646564636364636363636463636364636564636364646565646463636563636464636363636365636564636363636363646465656363656364636364636564636363646364636564636364636364656363636563646564646463646564646363636463636365646365646364636564656363636465636364636563646464656363636465636365646463656364636564646365646363646364636463636363636363636463636563636465636563656363646365646363636563646365656465636565656363656463646363636363636363656465636465656465636565646364636463636463636463646465636365636365636363646365646363656564656365636363646464636364656363656364636363646363636465656463656363646463656365646463636363636363646363646463636463646365646463636363636463636465636364646365646564636465636363656463646463636363636363646463646464636363656363646363636463646563646464646363646363636363636563636363636464636563656563636363636365636365636464646364636365636363636563636364656464636564656364646463636363646364646563636365656463646364656365646363656365656464656463636363636363636463646465646365656363636363646364646463636363656364636463636363636365646465636363646363646364636563636365646364636463646363656363636464636363646564646563656563646365636364646463636463636563646365656363646463656463656463636463656563646465636364646363646465646563656365646365646364646363646364646564656463636363646463646564636465656563646364636363646465646464646363636365646465646364646364646363656464636465656364636363646464656464636363656363646365646465646563636563656563636563636363656564636365636363636463646363636464656364636565646363636563636464646563646365636563636363636363646363646463646363656363656563646563636464656464636565636364636564636364636363656465636563636565636465636364646365636564636463646363656563646464636563646365656363636465636463636364636465636364636363646463656564656565646463636563636565636565656364656464646364636363656363636364636364636464636365636363646363656364646363646363636465636363656363646364636363636363646564656364636363636464646463656563636463636464636365656363636465656363636364656563636364636365646364646465636363636363656564636465636463656463636563636564636563646363646363636463636364656565636363636564656363646563636364636364636563656563636363636363646464636565656363636563636364646365636363656363656363646365636363646364656363656363636465646464636463636363656363646363636364636363636363646463636364646363636363646363646463636363636364636563656365636563636363636365646365636464656563646564636363646363646365636364646563636465656363656363656463646463636463646363636363656363656363656463636365656463636363656563656363656363656363636463646463656563636464636363636364656364656564656363646363636363636365636365656363646564656565636564656363646363636465656363636563636363636565656363636565636464636563636565646564646564636564656464636363636364646365656563646463636563646363636564646364656365636363636363636565636363656363646365646465656364636563646463656363636363646463646463636363656364636463636365636464636465636563646364636564636365646563636364656463646363656563636364646463636363636363646564646363636363636564636364656565646463656465656465656564656364636364636365656563636464636465636363636363636565636463646364636463636463656364656364636365636364646363656363636563656463636364636363636564646563636363636363656364646363636465636363636563636363656464636463646365646463636563656363636363646364636365636365656363646365636363636564656564646365636563646563646363656364656363646563646363646463656363636365636563636463636465636363636463646465636365636465646464646563646463646363646463636364646363636363646463646363646364656364656463636363636363656563646363636363656463656364646463656363636563646563656363656363636363646363646564636564636363646464636363656464646363636365636365636463636464646364656363636565646363646564636365636364646463636563646464636363636363646364636464656364646363636464656364636563636565656463636365636463636563646363636363656363646464636563636363636364636463656365636364636463646363636563636363636364646565646363636363636365636363636364636463636463636464636365636463636365646365656363646463656463636563636563636363636564646563636463656563646363636463646365636364646464636463636363646463646564646364646464636363646563646465636564656463636565636363656463636563656364636563636565636363636365636464656564656564636364636463646563636464656363646465636365636363636363636364646364636465636363656563646463636463636363636364636463636564636364636563636563636565636565636463636464656564636563656365656563636363656563636365646564636363636463636564646364646363636365646364636364636363636465646365636564656463646463636365636565646464636364636463636365646463636464656364636565646465646563636563636364646463646365646565646464636464646565646464636464636564646364636363636463646364646463646365636463636463636563656363636563636363646464656463656563636364646363646463636363646363646463656564636363646363636464636363646364656363646364656465636363636363646565636464636364646363656364636563646363646463636465656463656464646363656464656364656363646364636363636463636365636363636564656364656464636464646463646364636364636363656364636564636364656363646563636463636364636565656363656363656463656363636563636465636464646365636465636364636363656365646363646363636363636363646363636363636364636465636463646463656363656363646364636363656463646364636463656565646363636465636363656365636363636363656365646364636363646463636365636563646464636563656363656363646464636464646463636363646363636363646465636463646563646363636465656563636365636364646364636563656363656465656463656563656363656363656363636463636363656363656363636363636563656565656464646463646564656563636365656563656364636363656365646464636465646365646363646363636563636464636465646464646465646363646364636363636465646463636365636463636463636463636463636363656465636464656364636364646564636565656364636365656364636363656465646365656365656364646364636465636363636565646563636463656563656363636564646365636364636364636365646464636464656363646363636565636463636463646565636363656363646463636363636364636465636365636463636565636363646563636463656363636364646463646463646465656365656363646363656463636364646565646463636463656463646465636363636463656365646465656364636564636363646364646564646563656364656563656363646564656465636563636465656363636365656363646564656363656363646365636363636365636365646363636365636363636363636364636365636464636364656463646463636364636464636363646364656464656464646363636463636364636363646363636563646563636363636463636364656363646363656364646563636563636363636564636464636463646363636565646364656563636363656363636363656363646565636463636463636565636364636365636463636365646363646365646364636364636363646464636563636364636463636364646363636563646464646564646565646363646363656564646464646363646365646463636463636465636465636563636363636465636363636363636463636563646364636363636363646463646363656465646364636463636463636364656463636363636463636464636363656365636363636363636463646364656363636363636463646463636463646465636565656365656464646363646363646363636463646363636565636465636364646365636463646364636363636565646463636563656363656364636364646563636564646465636563636363656463656363646463636463636463646565646463636365636364636363636364646363636364636463656365646563636464646464636563646464636365646364636463636565636365636563636364636463646464636464636364636363636464646464646463656463636363636365636364646363636365646365656363646464656365646363636363656363636363636363636363636464636564636364636365646563646364636465636563636465636564656465636365646564636365636363636463636365656363656565636464646363646564646363636364636363656363636365636463646463646564636365636565646563646363636363636463636364636563646363656363656363636364646463656363636563646363636563656363656463636365656363646363646364636365646463646464636365636463646463656363636563646464646463656364646464656563636363656464636363636363646365636565646465646464636364656363636463656365646564636363636464636463636363646365636363646363636363636463636465646363636363656564656565646463646365636363636563646564636365636364636563636463636365656564636365656464646364656564636364646463636364656564636463636365636563636365646363636463646563646363636363636463636363636563636364636363656465636365636563646363636363636465656563636464656565646363636363636364656463656565636363646563636465636364636363656363636363646564646463636365646463636363646563656563636365636563656564656563646465646364636464636363636564646364636365636364656465636363646564636363636363656564636463646364646563636563636364646363636563646563636364656464656464636363656364646565646363646463656464636565646365656364636464646364636565636563636563636563636364646563636363636563636363636365656365646363636563636463636463646563646363656563636463646463636363636363636364656563636363636463636365636363656363636465646563636463636363646463646364636363656563636365636364636363636463636363636363636463636364636363636563636464646465636363636363656363646563656365636465656364646364636464636565636363636363656363656563646464636364636363656363656563636363636364636364646563656363656465636363636363636363636363646564646464646364636463636363656363646363646364636364646363636364636363656363646363636463636363656564656363636365656464636465636363636365636565646563656465636363636365646364656363646564636363656563636463636564636363636463646565636363646363656463646363636363646363646364646563636364636363636365656363656563636564646565646465636464636363646463646463656365656364636363636463656463646364656363636363646363636463646365646363656363636365636464636363636563636563656363636564636465636463656364636563656365636363646364636463636565656563636365646563646363646564636365656363656364646365636365636464636565646365646364636364646363656363636465636563656563636363636464646563656464636363636363636565646365646465646364636463636564636564636563656363656464656363636363636364656564646363646364646563636364636565646564646463636365646565646463636465656565636364646564636463656565656365636365656363636363656363656363646363646364656463636363636463656365646364636463646364636463656363646364636464636363646363646363636363656564646463646364636363636463636464646363636563646363636365646563636365646463656363646564656463636364636464636363636563636463656463636363636463646363646364636564646464636565636365636363656365636364646463636365646463646364636363646364636364656563646365646363636465636363636363656365636363636463636363636564636465636363646565646464646463636363636364636365646563656363646563646363636565656463636564636364636463636364636463636363656363636363636364656364646363646365646564636365646365636363636464656563636365646563636365646363656563636364636365636565646465656363656463636465636565646465636363636464646463646463656363636363636364636365636564646563646363646463636365636464646563646463636465646463636365636565646563636363636364656365656463636363636363636565636363656363646363646364636363636563636564646364636363646363656565636363656465636465636464646364636464636363636364636565636563636363636363636364636363656564636463636463646565646363636363656563656565646365656463636363636364646564636363646363636464646363636564636465656363636364656465636363646463656463656564656465646363646463636563646564656564636464636564636363646563636565646363636364636465646363646363646564636463636463636364646464656563636363656564636463636463636463656563656363636463646463656565636363636363636365636364636363636363656563636365646365636463636563646364656364636464636463636364646365646364636463646363656363646563636363656463646463636365636363636365646563636565656464636365636465636365636564636464636564636463636563656565646363656364636363636464636363656563646364636463636464656465636463646363646363636365636463646363646365656363636363646363646463656464656464646565656365636463656463656363636463636563636365636363636363636363656365636464636363636463636363636463636363636364636363656465636464656363646563646563636563646564656465656364636463646464636463646364636363636465656364636463636363636463636363636363646365646463646565636464636563636465656363636363646464646463656463656365646464646563636363646463646464656363636563636464656365646364636364656363646363636565646363646363636363646363656365656365636363646363646463636463646364656365646363636563636465636565636464636463646463656565656464636464646464636563636363646463646463636464636463636464636364636464646363646565636363636563636564636363636463636363636363636563636363636465646363636564646365636363646463636465646363646363636565646363636465636363646465646565646363656364636365646464636463636363636465636365636365646464646464636364656564646365646565636363636363646364636463636465636364646363646563636463636564636463636365646464656363636364636563646364656463646563636363636365636365636464636363646463646565656365656363636563636363656364636364646363656363636364636363656463656365656363636364646565636564646365646364656364646463636465646365636364646364656364646464636464646363656364646563646465646364636363636363656463656363646565656565646365656364636365646364636363636365656363646363636363646363636563636464636564656564646364636564646363636363636565636364636464636364636464646363636563656565646363636563646563656565636365656463636463656563656565656565636464636363646363656463636363636563636364636363636464636465636464636365636463646364656365646363636364636564636563656363636464636363636565636463656563636463636364646363656364646564656363646363636563646463636363636365646363636463636363656365636363636564636365636363656464636463636463636363646363656364646563656564636463636365656463656365636363646363646365646463636363636563636363646363646363636363646363646365636363646363656364646563656364636563636363656364656464636363636464656465636363646363646564636463636363646365646364636463656463656463656363656365656464636463656464656364656363636565636363646363636363636365646563636363646463646463656363646364636363636464646465636464646463636363636365636365636365636364636363656365636363636363636365656464646363636465636563646363636365636463636364636564636363646565636365656365636564636563636363646363636365636364646363656565636564656365636363646364636565636463636363656363656364636463646563636565646464636565636363646365636364646363636564636363636463636563646463636363636565636363646563656563656463656563656565636565636465636465636364646564656363636363656565646363636565636463656365636364656364656363636563656463636563636363636364636363646364636463636465656464646363636364636464646365636465656365656365656464646564636365646465636465656563636363636463636363636364636463636563656363636463646363646363636363636565646363636465636564636363636464636365636364646364646363646364636463636365646364636563656363656463656364636563636363636565636363646363646563636464636464636363636464636464646564636463636363636464656365636563636363636563656365636565636365646363646563656463656563636563646463636363656463636363636363656363636465656365646565636363636365646464656563656363636363636463646563656563636565636563636463636363646464636363636463636463646465636463656564636364636364636563636563636465636464646363636364656363656465636464636463646365656563656465636363636364636563646463646363636363646363636365656463646363656363656365646363646363636463656364636463646465636563636564636364656363636363636565636465636463646465646365646363636463636363646563636365656564646563636364636363656364636464646364636363636465636464656364636464636565656365636363636563656563656565636364646364636363636363636563646365646363636563636563646363636463636363636364636363636563656365636463636564636463636363656365636364646364636563646564646463636464636365656564636564646363646463646463656463646364636364656364636363636363636564646363636364636363636564656363636363646363636563646463656463656364636464636363636365636463646564636363636564646563646364656463646364646363646564656363636364636363636363636463636365646365636565646364636364656364646363656565656564636363646465636363636564636464636363656565636463646463636363646364636363636363636363646563636363636464636365636364656464636465636364636363656463656365646465646365656564636565636364636363646463636463646564636363646464636465636363646464656364646363646463636463636365646363656464656464636363646464646363636363656363636465656564646563636365636363636363636465656363636563636365656363646364646363646365636463636464656365636364646363636363656563636463656463636364656365646363656464636564636463636363636364656463646363636464636463636365656463646563646563656364656363656363646564656364656563646464646463636364656463636465656564656463636463656463646565636464636565636364636363636463636463636465636565636363636565656364636364656463646363656363636464646365656463636463646463636565636564646365636363656365646563646363646464636363636364636363636364656465646463636363636564636464646464636565656464646364646363636363646363636464656465636365656563646364636463636564656363636364636363656363656565656564646465646464636363656563656563646365656363656363636464656565636364646563646363636463646563656465646363636365636365636465646565636464636363636364646364656364656563636364646464636465636463656463656365646464636364636364656465636363656363656463636363636563636563636365646363646363656465636363636365636364636363636364646463646363636565646363636563636465636364646363656363656563656564636363636463636363636364646365636365636464636563636363636364636364636363656364636565656363636364646463636363656565636463636363636565636363656364636463646464646463656563636363646364656563646465646363636563636365646363636364636363636365636463636364656363646465636463646364646364646365646463656565636364656363646563636363636363636365636365636465646464636364636464636465636363636464636363656364636363636563656364636565646363636363656365636364656565636363636364646363646463646364636365656363636564636463646365646363636564656563636565646563646363656565636464636364636363646364636364636363636465656363646464636463656365636364656363636363636464646365656463636563646363656364636364636363636365636464656463646365656465656364636463646365636363636365656363646365636365636363646563636464636364636464636364636365636463656363636563636364636364636365646363636365636364646563636364636364636564656465636564636463636363656364646364646465636363636464646364646364646564636363636363636464636363636364646363636363656564656365636365646564636363636364646364646564656464636365636363646363646363646364636463646563636465646464656365656563646363636563636563636363636365656463636364636363636563646564646563636363646363636465636365636365636364636564636563656465646363636363646363636464636364646463636564636364636464656563636364636364636465646364656464636364636363636365646363636363636364636363646363646363636463656363646364646463656363636365636563636364636363636364636364636563646363636365636364636365656363636563636365646363646365656465656463656365636464636464646363646365656563636363636563636464636363646463646463656465646365636464646364656365636464656465646365636364646463646365646363636363646364636363646565636363656363656464636463646563636463636564656364636464636463636365646463646365636565656465636563646563656564636464656563636363636464656565656564636363656463636465636365636363636363656463646564656563646363656463636363656365636463636365656363656363646464636464656463646363646363636363636463636365656363636565656363636464656365636463636363636563656364656364636364636364646465636465636364646564646565646464656363646563636465656365636364636463656564636463636465636363636464636463646563646364636564636464656463636363646463656463636463646465636364646363636464656464636363646463636563636365646463636565636465646364636464646363646363636463656364636364656464636364636565636463646564656363636364636364656364636363646563646564646465636363656565646363636363656465646363636363636463646365636363656365656363636563636364636563636465636363636365636363646364636365656563656363646463646465646464656563636363646463636363656364636563636565636465636564646563636365656364646463636364656363656363656465656563646364646364646363646465636363646563636463636465646464636365636564636563656364646463656363656365646363646363646563636363656565636463656364646563646364636364646363646465656563636464636363636565636563646464646365646363636463646363636463646363656365646564636565636365636464646364646364636365636464646363656363646463646365636563656363646365636463656364636363646365636363636363636464646363646363656464646363656364656365656565646463636563646463636364636365656364636565646363636364656464636464646565646463646465646563646465656563636563636364636365636365646463636363646363636365646364646464656563656364656365636464646364656463636363636363656565636364636565636363656363656463646463636464646463636364646363656563636365636463636463636565656463636565646363636363656464636564636463636563646365656364636365646564636463636364636464636465636365636363636365636363636363636363646365646363636365656364636365656463636465646465636464646564636363636565636363656364646463636364646564656365636563636364636364656565646364656363636364646363656363656364656363636464636365636363636363636363636364636563636364646364636565646364636563636463636465636363656563656363636565636363636363646363646363636564646563646364656465646365636363656363646363646363646563646464636364636363636363646465636464646363636464636363656363656563646363636363636364636365636363636364656564656363646463636563646365636463646565656364636363636463636364636363636363636363656363636363646364636363646564656363656565646363656363656563636365646364656364636464636465646463636465636465636364656464646563646363656363646364636565656464636463636365636463636365656463656463656564646463636563636364636563646363636364636363646563636363636364636365636363656364636363636465636463646363636365636363646463656364636365636363636363636564636565636363646364636565646363656363656463656565636363636363636363656363636565636363636363636363636364646463636363636563646563636364656363636363636365636363646563646463646363636464646363636464636565646363636463636365656463636464636465636363646363636363646364656364636363656565656364636463636365636364636363646463636364656363646463646463656364656363656463636364636464656464636363636363636363636565646363636564636365656563636463636465636365636463646563646363646565636363636364636463636463636564636565646363636465656363636464646463636463636364636563646464656463636365656463636363636363646563646463636363646465636363636464646563636463636464656363636363636363656564646364636363636363636565656564656463646463636363656364636464636363636464646563636565636463646464656365636363646363636563636364656364646363636363646365656363636464636363636563646465636363636564636463646564656363636564646363636363646364646465656365636465636465636564656363656463656464636565646364656563636364636363636363646464636364636365656364646365656365656364636363636463636563636365636363646463646364646365656463646365636365636464656464636464656463636365636363636464636564636363636464646365636463636363646563646363636464636564636463656563656364636363636564646564646465636464636463636365646364646363636463656363636565656565656365646464636363636563636363636465636463646363636363646363636365636365646365646363636364646463636365636364646363636465636363636563636563636565636464646364636364636563646563656363636463646364636364646363656463646363636463636365646463646363646464636363646363656563646363636563656364646565636463636363656565646565656464646364636463656564636363636365656563636363636465636365636464646564636365646464656564646464636364636463646564636464646463656363636363656464656363656364636363636565636363656363646364656363636463636364636364646463646564636364646364636364636463636363636364656463636363636363636365636364656363636465656363636463636563646565636464636465646364646464636565646563646464656364646364656563636365636563636563636363636464636363656364636564646463636364646563636363636463636364636363656364646563656364636465636363636363646363646364636463636463636363646563656365636363636364646365646465646364636365636564636464646463636364646465636563636364636565636365636563636363656364646463636363636563656364636363636363646365646565656463636365636363656363656365656363646564646363636463636465636365636363646363636363656364636565646363636363636463636363636365636363656363646365636565656564636363636465646563636563636565656563656363646364646365646465646463636465636363636465656463636363646364636463636465636363636464656365646363646363636563636364656363636363636364646465646564646563636565646363636463636365656563646463646363636365636365636363636364636363636464646563636463636563636363636563636464656565636563646463646363656364636563636465636363636363636363636363636563636464646563646464656563646563646365636363636465646363636363646364636364636564636564656363636363646465656364636564656464656465646465656365646465646364636365636363636363636363646463646363636365646563636363636563636364646364646464646364636463646363646363646364636463636363656464636465636563656465636364636363636363636465656365636463646363636463636463636463636363656363656563646365636463636464636463646463636363636463656463636465656463646363636365656363646363656364646363636363646463636463636364656363636463656364646364636364646363656363636363646464636463646464636363646365656364656365636363636363636365646363636463636365646463646363646465656365636363636563646564636464646464636563636464646364646563646464646463636364636463656465636364636363646464656364636463656563656465636565646463636364646363636364656564656563636463656564636365646364656463636563636363636363636463646363636364646463636465646465636563636464636563646365646363656365646363646364656363656563636364646363636463656464656363646363636365646363636465646463656363636463646363646364656363646364636463656564636363646563656363636463636564636464646464636563646364636365636464656364636364636363656465636564646464636363656563636463656363636463646363646463646564636363646363646564646564636365636363636563656464646563636363656363656463656363636465636465636563656464636365636465636565636363636565636463656463636363656463636363636563646564646563636365656563656463656363646363656365636363636363646365636464636463636563636365636365636464656363646365636563656465656363656364636565646463646565636563656563646463636563656363656365656365646565656563636463646565636363656563646363646564636365636563636563636564646364636465646464636363646563636464646463636464656363646364646363646464656463656363646465646365636465636464636563646363656363636464636363646364636465636465636365646463646465636465656463636463636463636364636464636463656565636463656365656565636464636464636363636363656364646563646363636363636363646563636364656464646364646564636464656363636464656463636464646363636564646365646364656565636365656363636363646463636564646465636463646563636365636463656464636464646464636563636563646364646463656465636463646363656363636363636363636363656363656364636363656563636463636363636363636364636364646363636563636364656564636363636463646365636365656463636565636364636364636464646565646463646564636365646465646463646563636364636464646365636365656365636363656563646465656365646463636563636363646463646563646565636463636463636463646465636364646463636363636363656465636564636463636363636364636363636563656463636363656465646364656365636364656464646363636463636463646363646464646563636363636564636363656363656464646465636563646363646365656563636364656363636363636364656365646464646464636363636363636365646363636563636364646463646363656364646365646363636464646563636563636365656564636365636563656364636565636364636365636463646363636365646463636564646463646363636364646365636463656363636465646464636464646364646563636364636364656463646563636363646465636564656464656364646363636364646364636363656563636363646563636565646364636363636363636363646365656363636365656363656364646563656463636364636563636463646463636365646565636564646363656465636565646463646363656364656364636463636363636465636563656365636363656563646564636364636364656365636364636365646365656464646363636363636364636363636463636364636364646363646363636565636364646564646463636563646465656364656463646565656364636363656463656465636563656364636463636565646464636363636363636563656363636363646363646363636365636465636364636365636364656365656364636464636463636365646363656363636465656364646364636364636363636563636364636463656364636565636364636463636363646463636465646364656563636363646563636363636363646365656363656363646363646463636363646364636365636464646363636463646565636364636463636363646363636463656363636463656365646564636564636364656564636364636564656363646364636464656363636463646364636465646564636364636463646363656365646365636365636363646364636463646363636363656463636563636363656564646364656463636364646364636365656363656463636563636464636465636465646464646364636465646465636464656365636364656365636363656364636465636464656464636363636465646363646463636564646464646464636463636563646564636365646365636465636364636363636364646364636365636463646463636363656464656464636463636363636564636463636563646563636364646363656563656463656464646365636363646463656364636463636363636564646363636363636365646364636364636463646365656465636365636363636464636365646364636365656363636364636364646564656563646465656365646464636563656464656363646363636465636364636363636363656363636363656565646565636464636464646363646363656465646363646364646363646365646464636463646363636363636465636465646363636563636363636365636465636363646465646563656363656363636563646463646364636563636464656463636363646463656563646565656563636365636363636463646463636364646464656463646463646363636364636363656565646564636463656465636363656364636365636365636563636363656363646463636363646363636364656563636363646364646564636363656563636363646564656363646464646365636363636364656364656363636463636464646463636363636563636363636363656563636463646463646363636363646564646363656364646563656464636363656363636563636363646464656464636563646364656464636364646565636463646363656364646365636463646364656364636363636564636464636563636363656464636364656363636363636365636365656364636364656363656363636363656364636365636365636563636463646563656564636363656565646364646363636363636463656365636563636363656463646363636563636363656463656465636464636363636363656364646363636463656363636463636463646563636565636364636464656364636563656364636363636365636365636464656463636564636463636465636363636464656463656364636364636463656364636365636363636364636565646365636463636364636564636365636464646565636363636463656363656363656563656363646465656364646364636564636363646364636563656465636463646363646364636563636363636463636464636565636365656564656365636363646363656363646464636463636463636363636365646364656364636564636565636564636564646365636364636465636365646464636363656463646464646464646564636363646463636564636363656465636565636564646364636364646363636463636365646363636464636363656464646365636464636463636465636564646363636365636563646365646463636365656463636363646363646363636464636565636365636563636465636363636364646363636365646565636365646364646565636363656363646363636465636465636364636364636464646364656465646563636363636364636564646564636563636364656464636563636465636363636463656363636464636365656363636463656365646565636365636463636363636463646463636563646365636464646464646364636465646565646464656465636563646563636365636463636465646363656365646365646463636364646464636463646364636464636563636363636363636464636363636563636565656364656363636363656565636363656464646363646564636363646365646463656565646363646364656363656363656365636365636363636363646464646463636464636463636464656464646464636363636365646363636363636564646363636363656363636363636363636563636363656563656464646563646364646363636363646464636463646363656363636365646363636363636363656465646363656465646464636563656364636363636364646464646564656363636463656364656363636363636463646565636563636563646463636465646364646563636363636363636563656364636563646363636363636464636565636565636464636364656564636364646363636464636464636463646365656565646463646465636365646563646365646363636363636364636363636465636563636363636564656364636464656364646365636363636364656464636463636563636364636563646463636364636463636363636363646365656363636363656363636563646464636463636463636364646363636565646464636365636465646363656465646465636363636363636464656364646363656363636363656464656563636565636363656464636464646363636365656363636463636563636363636363636463636364646463636565646564656363636363636363636463656364646364646564636363646363636364646563636463636364636563656363636363646363636463636363636564636563636463636363636465636364646464636363636463636364656363636364656563646464636364636463636465646363636464636364636364636364636363636465636464646363636564636565636363656363636463636463636563636463646363646363636363656363636363646464636463646365646365636365656363636363636365636463636463646564656364636364656363646364656363636564656364636465646363656363656464656563636365636464656363656563636363636363656365636363636563646365646363636363636563646363656363646363646363636363636363636365646564646363636365656563636365636365656563636463636363636563646465646365636565656365656563646463636563646464636363636364636363646363636564646365636364636364656565646364656363656563636364656563636565636364636363656563656463656563646365636365636363636463646363646363636363636363646363656365646365646463636463656463636365646464636465646363636363636364646563646563646563636563636363636364636364656363636364636564636465656464646364646364646363656465646363636363646463646363646564656563636463646563636465636364636464656363636464636365656365636463636364636463636565636563656563636465646363646363636563636464646463636363656463656363636365636364636564636464646463636363636365646465646364646363636565656463646363636364636364636364636363646464636363636564636364636565656365636464646563656365636463636463636565656364656364636363656365656463656364646563636365636464636363636363636363636363636363656364646463636563636363646364656364646363656564646464636464636363636364646463636365636363636365636365646463636363636363646363636463656463656563636363636365636564636365636364636563646365636463636364656363656464636365636363656465636363646364646363636364636465646364636364656464646563656365646463636465636564646565656564656363646363656363656363636365636363636363636363636563646363636365646463636365636363656364646464636563646565636363636363656364636564636465636363636563656465646464656564646463636363636365656565636463636463636364656463656364636363636565636363646563656363636363636365636364656563636365656463636364636464636363656463636363656363656465636364656363646464636363636463646363656463636365646365636363636463636463646363636364636363646465656463636564636564636363636563646463636563636364656463636363656563636463646563656363636364646564646364636463636563656364656463636365636364636564636464636565636364636463636364646363656563636363646463636563646364656563646364636563636464636365636363646464636364636563636363636363636464636464636463646363636563636363646563636363656464646463656365646363636465636563636363636364636364636563636463636465656363656463636563636564646563646463656463656464636365656463646463636365646563636465636564656363656364646565636364656465646363636563636463656363636465636463636464646363646563636463636363636563646364656565646365636364636563646563636465636363636563646563646364656363656463656564636464646364636465646464656364636365646565636463636363636463636364646363646463636363656564636464636565656463636563636465636363636363636465636365636563646364636463646464636563646463656464636463646365636563646465656363636365656564656365636363636365646363656565636363656564636463636563656365636365646364636465636364656363636364646465656364636463636363656463636364646365636364636563656565656463646564636364656465636363636563646464636363636564636364646364646463636363636363636363656365656565656563646364636363636564636363656364656464656465636463636363646463636365636464656364636363656364636464646364636363636363656363636564636363656363636565636463636464646363636363636463636463646564646363636464646464656365646365636465636463636563636465636365646563646364636463656463646364656363646565636364656463646363656463646364636365656363636564646363646365636563656563636565636464656364656363646364656464646364646363636363656565646363636364636363656464636363636563636563656364656364636565636363636363646563636464636365656463646363636363636463636363656564636363646363636363636365646563656363636364646564636563656563646463636563636365636464646563646464646565646365636364656563636463646365656364656565636464646363646565636463636363636363636364646463636564636564636365636363646364656363636363646363646364646463646363646564656564636363636565636363636363656463646563646365646363636363636564636363656463656363636364646563656364646564636465646363646363656364636364646464646363646364646365636465656363646563646365636363656464646365636363636365636363636363646464646363636464646364656365636363656365646364636463656465656465636363636363656563656364636363636365656564656563636364636465656364636364636564636463636363646564636563646365656364646363656363636364636363636465636363646463636363636564656463656363636463646364636465656463636463636464636464656363636464636564656363656364636564646364646463656365636563636363636464656363636365636363646365636365636565636463636563636364636364636363636365636565656563636465636464636363656465646364656465636463646565636363636364636463636363636463656464636363646365636563636464656464636363646365636365646465636363636363636463636363636563636464636363656463636563636363646363646363656363646463646363656463646464636463636363636363636364636363636365656464636363636463646563636563636364646364646363646363646363636363636363646464656363646365656465636465656465636363646363636463656364636464646364656465656364646363656365636464636363656363636365656363636364656365636565656363656463636465646364636563636565646563636363636365636563646363636463646464646363646364646364636563636563636364636363656363636464656364636465656363636363646563636364656464636363636365646563656364656564636465646463636363646365636463636565636364636364656363656363656365656563636464646363646463646464646563636363636565656363636363646365636365656363646364636363636363636364646563646364636363656364656363656363646363636563636464656464656564636365646363636364636364656464636563636463646463656364646465636563636365646364636364636565646364636465636563636463636363656364636364656363636363646363656363636365636364646463636363636365656365656365646363646363646563646463646363636365636364656363656364656563656363636463646564656364636363636565656364636364636364636364646363656365656565656564646463656563636364636563636365646463646464656363646365636364636363636364646363636564656363636565656464636363646463636364636564636563636563636364636564656363656463636464636363656363646363636364646463636464646463636465646465656564636464636365646363636363646363656563636564636365646564646363656363636465636364636465636365656564656465636364656564636364646365636363636563646563636363636464656363636463636563636465646365656364636465646464636363656365646465636464636465636363636364646464636365646465636463646564636464636365636363636363646565636363646463646363636463656363636364646563636465646563656364646465636465636363656463636563636364656463646465646365636463636365636563646363636363636363636363636363646363646463646363636564636365646464656364636463656364636463636463656363636563646363656363656463656363656363646364636463636363636363636364646563646364636564656363646563636364636363636364646363656363636364646364656465646563636364636364636464646363636364636365636465656563656364636364636563646363646363636465646463646563636464646363646463636365646363636363656364656463646464636463646463636563646363646563636564656463636464636464646364636464636463636463646363656365646364646363646365636464646363646365646563646563636464636464646463636564636364636365636463646464636463646365636464646464656464636563636364636363636565656563636465646565646363636365636363636463636364636363636564636563646463636365636463636363646363656463646363656364646365656464636365646465636365636364636464636463636364636564636365636363636463636463646365656363636563636364636463636465636464646463646463636464646363656364656364646364636364636463636465646363636563636364646363636463636363636365646463636364636463636365656364656463636565636364636363656565646463636364656565646563636463636463636463636363636365656565646465646564636363636465636565636363636465646563636463646363636563636464636563656364656364646464646565646463636463646463636363636364646363656364646363636364636365656563636364646463636364636363646465636363646363636565636564636564636565636563636563656363636463656463656363646464646563656365656363656363636363636365636364636363636465636465646363636365656364656365646364656563646363656464656363646364646463636363636465636363636564636464636465646363656464656363656564636563646464656463646464636464636564656463636363646363646364646365636463636464656463636463636464646565636364636365656463636363636463646564646363656364636365656465656463636563646364636365646365656563636363636463656463636463646364636565646564656363646363646464636364636465636363656563636365636364636463636363646463636363636563656363646464656463656365656465636463656365646363646365656363636563656463656363646563636563656364636464636363656563646563636463656363656363656365636463646565636563646463636363646564656563646464656363636363636365646565656563636463636364646364646463636363656364656465636363646464646363646463656565636565636465656464636363646364636363646463646563636563656365656363656365656563656365656363646565636363646463646563636365656363636465646564636465646465636365656464636364646363636364656363646363636564636563646463636363656563636563646364646365636564646364636365636363656363646363656463636363636564636364636363646464646364646465636565636363656364636365656363646563636363636363646365636564646364636464656365646363646463636363636365646465646364656564636363646363636365636564646363646363656463636365646363646465636365636463646563636563646365636565636563646363646363636565636363656563646365646464636563646364636363636463646463636563646363646363656564636363646564646563636363646364636464656363646363636563646463636565646463656363656564646463646363646463636365646464646364636363636363656363636564636563636365636464646463656364636565646363656463656463646363636463636363646563636364636464646465636365636365636363636364646564636363656463646363646363656463636364646463636363646363646465656365646465656363646463656364646363636363646563636364656363656363646363656564646364656365636464636564636365636363636365656565636365646363636463636565646363636365656463636465636365636363656363636464636363656365636463636464636564656363646364646463636363636564656364646463646563636563636464656365656464656463646363656363636363646463646365656364656363646364636363646365646363636565646363656364646364656463636364636465636463646564656364636463656464636563646564656364636363646363636365646363656564636363636363636563636465646363646463636463646364646463636563646364636465636463646363636364646364646563646363636364646463636564636463646463656363636563636365646464646564656565636365636363656463656464636364656463646363646464656463646364646363636365636563656563636363646364636364636365646463636364636363636363636463636463636363636363636363646364636363656565656363656565646365636363656364646563636363636364646364656565656363636365636363636564646463636564656464636363646364656463646364656364646364636565656563636565646364636563646563656463636364646463656363636364656364646563656565656363636365636563646363636365636364656463656365636364636464656363636464636365656564656563646464656563656363636363646463636465636564656464646563656365656363646364636565636363656364646464656365636364646363636564636565656364636365636464636363646464636464636464636363646463636365636463646365656365636464636363636463636464636364656365646564656565636563646564636463656565656464646364636363656463656563636364636463656364636464636364646464636363636363636364656363646363646464636463636563656363636464636464636364656364656563636463636363646463636363636463636363656464636363656465636365656463656464646364636363636363636363636364636364646463646363636563636363646364646563636565636365646464656365646464656463646564656564636465636364656363646363636564646465656363636365646363646364656363636465646363636363636564646465656365646363646464646364636563636363646363636363646365646363656363656564646465656365646563636463636563636363646365636363636464636563646563646465636563656465636363636363646363636563636363636463636564636363636363636464636364646364646463636463646465646563636564636564656464656363636563656464636363636365636463636365636363646463636363646364636564646363646364656463636363636364636563646464636365646364636464636363646363656363656363636364646564636465636363636463636463636363646465636564636464646363646564636463646463656363646463636463636465656563636364636365636464636565636363656564636563646465646364636463636565636563646363646363636563646564656363646465636465636564646364646364636364656363646364656563646363646465636564656464646363636365646463636364656363636565636563656463636364636364636365646564646563656365636365646364646365656564656364636364636463646563636365636363636365636563636363636363636465636363656565636463656364636363646564636363646463636363646364656363636464636365646464636364636563636363636363646563656365636563646464656463646464646364656463646363636363646363636364646464636363636464646463636365636364656463636463656563646364636565636363636463646464656463656464646364636363646563646463646564636463646565636363636364636365656363636564636363636465636464636363646465656363656465636565636364636564646463636363646565646563636564636363636564636363646363646465646364636363636364636365636365636464636363656463646364636363646363636363636464636365636364636463656363636564636365636365646363656363636365636464636363636363636563636364636565636364656463646464636365636564646363636364646463636463636563656465636364646563636463636564646463636365636365646365646464646464636563656563636363636364646363646463636363646363636363656363636365636365646563636364636563646464646363636365636563636363636365656363636364636463636564636364656364636463636463646363636365646364636463646363656563656563636363636364636465636364656563636364656363646464656363636463646363636463646363646363636363646363656363636363636364646363636363656563646364636365646465646464636363646563656363636365636463656363636363656563636463636463636363636563646464646363646363636364646563636364656363656363636563646363636463656363646563646363636464636363646564656564636465656463646363636365646363656465636464636363636464636565636364646563636363636565636364646565636365656463636463636363646463636563646363656463636363646363656364636564646363636565656464636363656364636363656463636563636363656363646565636563636363636363646363646365656363636564636363636363636363636363656365646565636565646463646363636564646465636564636365646463636364636464646364656563636464636363646364656363636363646364636363636364656463636364656463646363656364646364636563646363646463636464646363636463636463636465636363646465636563636363646364636463656364646364636364636363646464636563656363646463656364636464646463636563656365646563636363636365656565636565656463646463636464636364636363636363636363636565636364646465646363636464636563636363636364646365636364636464646364646364646464636464646365636463656464656363636465636364656365646564646463656365656363656364636364636464656365646563656364656565636465646363646363636463656363636363636563636364646564646363646564656363656463636563646464636363646365656365656463646563656363646364656364636564636365636363636564636464646363636563636463646565636463636364636563646463646363636464646563636363656463636364646365636563656565636464646464646463636463646364646564636363646463656363636464656463636465656365656563646363646363636463646365646363646364636363656364656365646364636364646464636363636565636565646464636364636365656363636365636463646465656365656463656464656465656463636564636365646364646365656464636464636363646365656564656463646365656363646365636365646364636464646363636363636363636463656364636463656364646463636563656364646365636463656564636563636364636365636465636563656365646465656563646364656464646364656463656363636563636363646564646563656464636563646364656363636363636363656363656363646563646463636365636563636463646365656464646363636363656364646465636463636563646463646365636463636363636363636364646463656464636463636464646364636363636363646364646464646364636464636463646464636463636563636365636564636363656463646563646464646465656364656365636363636463636364646365636565636364636464656365636364636465636463646563636363636465646363636463656563636363636365636363656465656363636465646363636364636363636463636363636563636363636464636465646463656465646463636463636363636563656464636564636365636364636363636365636465646465656464636363656363646465636563636363636363636563646565646364656365636464656465656565636463656464636365636364636464636563656363646563656563636365656364636563636363636563636365636463636364636465646565656463656465636565646463646564636464636464636564656463656365646463636365636365646464636563636363656464636365656364646463646365656363636464636363636363656364636463656365636563656464646365646464636364636463636363636564656365636363646363656463636363646363636463646364636464636363646363636563636463636363636365636363656463656464656364656463646563656364646464636464636363636365646363656563636363636364656365636363656363636463636465636465636363636563656363636365636565636363656464656465646363636463636463656364636565636564656464636363656463636464646463656564646463636563636464646463646364636363636363656363656363636363656463636464656463636465636363636363636564646364636463656365636465636364646365656564656363656463636565646463636363646463636364636564636563636365646363636363636563636565656464636363656364636363646465636564646463646365636564656364646465656564646364656363656365646464656464636463646463646365656363636464636363636365656563656563636365646463656563656464636564636363646463656464646364636365636364656363636463656465646463636365656365636465636364636365656363636465646365636364646563636364656363636365656563646563636463646363656364636364646363646564646363636364636363636364656463636363636364636565636364636463636363646364636463636463636363646363646563646365636463656564636565636463656563656463656463636364656463646363646364656563646563646365636363636364636463636563636364646363636364636463646363636463656363636365636364646463656464656365646364636464646363656564656463646365636464636365636363636464636363636464656464636563646363656463646364636463656364656363646363656365656364646364646363646563656463636563636563636564656365646363636564636363646364656564636463636463656363646463636365656463636464636365646463636463656564636563636363636364646364656365656465636565646363656463646463636563656363636463656363656563636465646464646364636363656363636563646464636563636364646364636463636363636464636563636364636465646464646364636564636465636464656463646564636565636464656364656563636363656463636363636463636363646363646363646463646363646563636363636463636365636465636465646565656365656364646364636463636465646563636463646565636364656364656563646363636563656363656464636363636364636364656364636363646463636363656464646463636464636364636463646463636363646365636463636364636564646365636463646363636364636363656463646464656463636564636463636464656463656465656364636564636364636363636363636363646563656363636363636364636463646365656463646463636465636563636564636364646364636563636463646363636363656363646363656364646363656563646564636363656365636363636463636363646364646363636364636363646563636363636563636564636463636365646365636463636564636363636464636363636463636565656364646564636364646363636463646563646564636565656565636363636465636564656465646563636364636563636565656565656365636363636363646463636365636363656463636364646563646463636364636363636364656364646364636564636364636365656364656563636363636363636464636563636563636364636363646365636564636364646465636364636463636364636363656363636564636364646364636363636365636363656365646563646363636563646363646565636463636363636464646363646563646565656363636465656364646363646364646363636463636463646364636465656463646363646363656463636463646463636364646364636363636363636563636363636565636364636563636363646363636365646463646463636364656363636365646563636364636364636465636564646365636365636364636365636463636463636363636363636563636363646365636463636364636365636463646564636363646363636463656463636463636365656364646363656463636364636565636364636565636463636363636364646565636365636563636463636463646363636364636563646563636363656463656365636564636565636364636363656365636364636364636463636463636363636563636563646364656363656363636365636363636365636464636464646364646463656463646463646564636364656363636363636363646365636563646564656563636563636563646463636563646564636464636364646363646363646463646363646363656363636463636363636564636365636364636364656465636565646363636465636465636363636463636364636563636364656463646363656465636565636363636363646464636465646563636364646463636364636365656363646464636363636463656363636464646563636563646364636465646365646363636364646363656364656564636563656363656464646563636365656364636463646465656363636465646364636364646464656463636364646363636364636563646465636464636463646363636363636463636465636363646364646363646565656363646363646364646364636363636464646464636563636364656564636563636465646464636563636463646563636363656463646463636364656464636363636565646363656563646563636363636463646464646464636463636363636465636464656363636363656563636465636364646363636363636363646465656363636363636565636363636563636363646464636363646364656464646365636364646363636363636363636563636463636363636364656465636465656363646363636363636465646565636563636563636463636463636464636565636363636463636363656363636364636465636565656363656364636363636563636465646464636463636365636564636464646365656363646363636365656565636364656564636463646363656365636365646463656363656363636565636564646364646363646463646464636563646565636365636465636565646363636465646464636464636463646363636563636463656465646363636563646363646364656565646363636463636564636563636465646365636563646364656564636363636463636564656564656364656364656465646463636465646563646464636363646464636563646464636463636364646365646365646464656463646463656363646363636563656563646364656463636363636463636364636564636363636363646365636463646363656363646363646465636564636563646565646463646364636563636565636365646365636463636465656363646464656363646364656563636364636363656363636565636463646564636563646463646463636363636364646563656365636364656463646364656364646365646364656463646364636365656463656564636363656565636364656564646563646465646363646364636363656564636363636364646464636465636463656363646363636363636364636464646463646363636364636364636364646465646463656463646463646363646463646364656363636365646465646364636464636364646365636464636363636363656364636563636363646563646464636363636363636363646463636365636363636363646565636465656363656364636463646563636363636564646363636364636363656365636463646364656464646564656363636363646463636563646363636464636363646463646365646463656465656363646564656465646365636365636464646363656363636364646565646364636464646564646464646363636363636363636365636464636564656364656563646563636364646365646363636563636563646365656364636363656464636365646463636363636463646463656365646563636363636363636464646363636565636364646464646363646563636365656463636364656365646465646465636363646565656363636563636563636364646363636463636363636363636465646364646363636463656365636363636365636563656363636363646463646565636363636363646465646564636464656463636363646464656363656364636565646463656363636364636363636364636363656365636363646563636463646365656464636563656465636563646463636565656365646465636364636363646463636363636364636364646364646564656363656563646363636463636463636363646364646363646563636464636363636364636363656363656465636363636364656363646363656363646563636464636363636363636464636363646565636363636564656564646363636363656463646563636564636464646364636365656463646363636463646364656363636565656563646563656463636363656465646363636564636364656464636365636363656463656564656463636363646364646464636365646364636363636365656463636465636363646363656365636565636365636365636563636363646563636363636363646565656365636363646365646365636363636565646363646364636464636563636563656463636363636563646464636464656463656563636364646563636363656364656363636464656364636363656363636464636565636363656365636464646364636363656563636365646365636564646363636363646363636363636364636563636364646563656364646363636463646363646364646365656363646365636463636465646563646365636363636564656363636364656564636463636463636363656363636363656364636463636365636564656364646365636464656463656463636463656463646463646364646463636464636564656564636364636364636464656364656464636563656565656364656364636565636365646363646363636363636563656465646563636464656564636363636363646364646365646364656363636465636563636565646365646464636463646464636365656365656465646363636563646364636364646563656363656365646363636364646364636363646363656364646363656363636364646464636363656363646364636364636363656465646363646363656563656364636364636464656363656363636363656363636464636365646365636465656463636363636463646363656365636463646464646363636363646365636463656363636563636363656465646364656363636464646563636364636465656463636363636363656563646364646363636364636365636365636463656363636364656364636463646364636365636564636365636364656563636563646563636563636363636464656363656363656565656363636464636465636463636364636465636563646363636463636463646363646365636363656464636363656564656464636463636364636365646363636565646465636465656363636364636363636564636464636364646364636363656363646363636563646365656363636563656464636364646364636363656363636564636363636365636363636463636364636564646363636463656364646363646463636364636464636463636564656364636463636465646365656363636363636365636364636364636364636364646364636465636364636563646363636363656364636364636563646465636565636465636364646565636463646365656363636463636363636364636363636464656363646463636363636563636363646565636464646463646364636363646363646463636363636563636363636363646463636563656563636463646365636364636365656463646465636363646363636464656363636563646563636463646565646463656363636364656464646363646365636565636364636363646365636364646463636465636464636463636463636363646363636363646564636364656563636363636465646365646363636563636463636363636364636465636363636564636364656365636463636363656364656565636365646563636564646463656465656364656364636563646364646363646365646363636564636563636363656565646463646563636565656365656563646463656363646564636465646464646564636465636565636565636565636363646463636564636465646363636365636563656464646364636463656363646564636463636363636363636364636364636463636463646363646563646565636363656363636363636464636464636363646365656363636364636364646365636363646363656563656363646365636563646563636363636365646563636463656463656363636364646363656464636364636363636364646565646364636464636463636363646563636564646563636365636363636363646363636364656363636364646465636363636564636465636464646464636463636564636465656463646363636363646465636363646363636465646364636463656363636363656465656463636563636565636464646564636364636463646363636365646365656563636363636364636364646465656463646365646463656463636463636563636463636364646463636463656563636363636564646364636463636463636365656363636464636463636563636565636365656363656465656564656463636364636364646565636364636465656565636363646365636463646363636463636365656465636364636363656463636463636463656363636565656363636563636363646363636365636363636363646363636564636363636464646363636463656363636563636363646364656363636565636363646565656365636564636464646465646565656363646363636563656463646363636463636464656464646563636363656364636364656464656463636463636565656363636363646365656565646463646563636365646564646365636564636364646364636465636465656463636564636463636563636363636463656363636465656563646363646363636464636363656365656563646364646563636365636463636563656365636564656463646564656564646363646463636364636363646363636363636364636363636364636565656563636365646364636463636363636563636465646364636564636363656565646563646363636364636364656564646463636563636563656565646364636464636564656465656464646465646363656365646563656563636563636563646564636563636463636464636465636463646365636363636464636363636365646363656464646463646565646563636464646564656465646364646565656463656463636564646563636364636364646363656463656364636464646365636364636565646463656563656563656364636563636563646563656463646563646364646365646563646363646365636364636363656364636463656363636563636565636365646465636465646364636464656564646363646364646564656363636463636463646363656464656364636365636364646463656363646463636464636463646363646363636463636365636463656465646463646464636463636464646465646363656464636363636364656465646463646464656364636365646563636564636563646564656365656365656563636363636565636463636464636464646465646464636464636464656363636363636364646464636365636563646563636365636363636365646464636564656563646565636563636464636363636464656364636465636363636563656465636363636463636464636363646364646563656563636463636563656363646363636463636363656363656364656463636365656365656363646365636363636463656365646365646363636363646364656463656364646463636563646364636364636565636363646363636565636363636465636563636364636363656463656363636463646463636363636465636465646364646365636463636565636365636365656363636563656365656364636365636364646364636364656565636363636363636463636363636365646363636363636365636365656465646363636563646364646363636563646563646564636364636463636363656364636363636363656565636364636363636363646565636365646464636364646464636363636363656463646464646465646364636363646563646463636365656465656363636363646563636363636364656364636464636363636463646364656565636563636363656365636563646363656363656364636365656363636465646464646364636464646463636563646464656363636563636563646365646565636563636463636563636564656463636463646363656465636363636464636463636364636365636364656564656363646563636463636363636365636463636364636563636464636563646364646463636564636563636364636464646364636363656463656363656363636363636364636563636364636563656463636463636563656365646365636365656363636563636363636365646363636363656364656363636465646565646364636563656364636563646363636464656465636364646464656363636464646364636463636564636563656463656365636363636363636365646563646463636363636563646363636363646364656364646463646463646365636464656363646564636363656363646365656463636364636464656365646363636363636363636364656364636365636363646465646364656464656363636365646363646364646363656563636363636464636564646365646464636364636364646363646365656363656363636564646464636463636565636363646365646465656363636365636563636364656364656363636564646463656463636563636363646364636364636363636363656463646364636363656563646363636565646363656363656464646465646464636463646463636364656365656564636363656363636564646363646365636365646464636363636363646363636363656463656465636364656465636363636364636365646363646363636463636363646364656365646464656364656464646363636463646365656364646464636465636365646564656463636564646363636363636463646364636465656363636365646563646563646364636364636463656363656363656463636363646364646564636363646463636565636364646563646365636363656363646465636365636363636364636365656363646363636363646465646563656565656565656463646463636465646463636464636365636363646464636464646365636364646564646463636364656365646364636363636363656563636365656364636463636363656565636365636463636365636464646464656564636365636564636364646365636565636363636465636363636563646363646464636363656364656363636363636363636564636464636363636464646563636563646465636564636563636563636565636363636363646563636363636463636363636464636363636363656363646363636365656563636464646363646565646465656563636363636363636363636364636364656365636363636564636564636563646364656363636364636364636564656465656464646363636563636363646563636363636464646564636365636465646364636363636363636564656464656465636564636463646563636563646463636565636363636363636563636363636463636464646463636463646364636365656563636363656365656364636363636364656564636565656363636364636564636363656465646563636465636563646563636364646563656563646363646463636363636564656464646363646363636364656363636465656564636464636465646565656463636465656363656363656364646363646363636463656563656364646464646363636464636364636365636563646363636465656563646365636363646363646465636363646463656463646463656563646364656363646565656463646463636464636363636365646564636365636563656363636464656364636363646463646465636563646364636363636363646365646363646363656563636363646365646364646563636463636364636463636463636364646365646363646463656464636563646463636563656565636363636563656364646563656465636363646363656463636363646463636564636364656463656365656563646464636364656365656363646364646463636463636363636363636364636363646463636363636463646463656365636363646563636563636364636463646363636363636365636463636463646464636363636464656363636365656563656463636363656364636363646463636565656363656363656364636463646363656363646563636563656363636363646563636563636463656363636365636364636463636463636365646363636365636564636363636564636363636565646363636563646364636364636464636563636365656563646363636364636565646365646365636363636464636563646464646464646363646364656365656363636365636463636365636363636563656363636364646464646365646464646365636365656363646463636465636365636364636363656563656363646563636463646463636363636363646565636363656363646364636465656464636363636564646364636563636463656463646363636465636464636563636463646365646365656464636365636564656363636463636464636364636365646563636363636363646563636363656363636563646363656465646463646363636363656363656565646564646464636364656363636364646563656365646463646563656364656565646563636464636365636363646364656464656363656464646363636465646564636464636363636364646464646365636565636365646363646463646364636564646463646465636563646563636463646364636563636363646463636363636564646364636465636365636465636365636365656564656364636464636365636363646463636363656464636363646363646363646363636363636365646363656363636464636463646364656465636365636364636365636565636364656365646463636464646364636363636364636464636363646363636363636464656564646365636463636463656364636364636363636465636363636464636365636463646364646363646565636364636563656463636464656563656565636564646565646363636463636465656363646363646464646365636364656363636463636563656463636364656365656363646363646563656464646464656364636365636363646363646463656463646364636463646365656564656363636563636563646464656563646465636464636463636463646363636363656363636365656363646365636564656565656463636363656563636364636363636464646564646363636363646363646363646464636563656365636363636464636365646563636463636363656364646565636364656364636563646463636363656465656463636564656363656565636464636363636364646464656363656463646464636563646363656463656364646563656365646364646363656463646363646563636563636364656563646363656565636363656364636463656364636463646565646565646363636464656463636464636364646363636565646463656364646565636565646465656463646464646365646464636463636463646463646565636365636363656365646363636363636564636563636465656365656565646364636463636365636365656364636465646363636363636364636363646365656363636364636465656463646364636365646565636365636365646564636364656363656564636465656363636564646463636364636363656464656365636363656463636364636364636363636363636364646364646465636363636365656463636364636363656564646363646464646563636363636563646363646365636363636465646364636364646464636463656563636564636463656363636363646563646363656363636364646464656363636563646463636364636364656563636363656565636563646363656363646365636463636363636364646465636463656464636464636563636563636563636363646364646463656463656463636364636364656365656463636363636365646363636364656564636363646463636464646464656563636363656364646364636363636363636363656364636363646363636563636465656463656365636363636364636463646465636563636563656463636463636464636365656363646563636363636463656564636464636463636563636363646365656365636464656463636363636363636364656563636465646364636563636463636565636464656563646465636463646365636363646464636365636365636464636465656463636364636364636563636565636465636364656463636563646365656363646363656565656564636564656363646364646363656463636463636463636465636463646463636564646364656565656365636464636564646463646463636464636363656465636363646363646465636465646364656363636465636463636563636363646363636464646564636563656363656364646565636364646563636363656363636463636463636465636564636363646563656365646464646363636364646464636464646364656564646363646463656463646563646463636465656563636364646463636564636363646365636364636464656363636363646364656563656363636363636463656464646565646365646463656565636563636364636363636463636363636564656364636365636365636364636564646365636363646365636564636463646365636563656364636363656364636464656365656463636364636563636364646565656463656463656565646464636564636364636363646463636363646565646464656564646463656464656465646563656363646464636363636363636464646363656365656464636464636565636364636365646564646363636364636565636363656463636465636364636365656363636363636464636364636463656563646363636463646564646365646565636465636463646363646463646564636564636465636463656363646364646464636563646363636563636364656363656463636463636463646563656365636363646365636563646564646364656364656364646363636363646364646363636463636363636563636564656363646464646364646364636565646463636364656364636364646465636565636364636565646465656364636363636363646363646563636364636363636465636463636463636364636363656365636364636464636463636363636464636464636463636365636464636463646365656364636463636463636363646364646563656364636563636465636363656363636364636563636464636464656363636363646464636365636363636463636563636365636363636365656363656365636563636364636363636464636463636564646364646364656365646365646463636365636363636563656364636463636363646463636463636363656464656563656464636363646463636564636364636364656463656463636464656363636563656364636365636364646563646465636564636463646464636364656363646363656463636363656564636463646364646365656363636563656463656563646463636463636365646563636364636563636363656364646364636363636364636364636564636363636564656365636363656363636364646365656364656364646364646364636365636464636363636463636364636563656463646365636364656464636363646563656363646563636365636563636564636365636363646363646463636364636564646464636563656463636363636365656563636563646363636363636365646363656365636363636465636363646363636363636363636365636363646364656363646563636365636563646564636463636365636363646364646363636564646363636363646463636463636463656464646464656363636363646463636363636364636363636463646365636364656364636363636364646465656363646364656564636363636563636563636464636365636564636463636464646365636363646364656463656563636464646464636363636365656365646363636364646365646363646365636563636363656565656563636463636464646363636364656464636563656363646564646363636363636563636363636364646363636364636465636364636363636363656564636364636363636464636363636464636465636365646465646464656563636563646564646363656365656463646363656563636465636465636365636364646365636364636363636563636464656565636564636563646463646363656464646363636363656364636563636565656563636363646364646564636365636363636364646363636564646363656463636363656363636463656363656363636465656465656363646463636365646363656363646464646563656564646363636565636363646364636564646365646563656363656365646363656463646463636563646563646464636465636463636463636464646365636463636565636363656563636565636465646363646465636465656364636465656363636465656364646364636464646364636464636563636365646564636364636363636365656364646564646364656363646363636363636563636364656464636364636364656363646563646565656464636463636463636365646364656364656365656363636465636463646463636364656365636365636363646563636565636464646363636564646564646364656463636364636465636464636463636563636363636363656465646365636364656465636364636364646564636463646363636563636463646464636363656363646363656465656463636464646463646563636463636364636464646363646364646363636565636363636563636365636565646363646565646364636363646363636564646564636463656463636363636465636563656564656463636363646463656365646363636465646463646365646364636363646464636363656563636464656363646363646564656564646463636464646364636363646363646464636564636363646463646463656564646463636363646563646465636564656363656363636365646363636363656364636565636364656464636363646463636463646364636363636463646464636565636364636463636363656363646363646363636365636365646564636563646364656564636363636464636464636363656365636565646365656363646464656363636563656463646563646564656464646363646563636564646465646564636463636463636563656364656463656363636363636363636364656463636463636363656564636463636364656465646364646363636565636363636363636364636364636563646463636463646463636363656464656464656463636464636363636364636363636463656464636364656465656463656565636563636363636364656463636365636363636363636564636464636363656564646463636363636463636363636365646563636464636463646363636363656363656464646363636365636365636463636563646363656463636563636463636563636463636563656363646364636364636363656363656463636364646564636363636363636363646365646363656365636463656364656563646364636463646363636365636363646565636563636363646464646363636463636363636463656565636463636364636565646564636364646364636363646363656363636563636363636364656365636563646563636363636564636364636363656364656364636463656365636364636363636563646363636463636564656565656464636563656365656363636363636363656364646365656365636364636464646563646365646363636564646464646363646465636363636363636565636363636363656363636364646564636564656363636563636463636363636565646364646363636363636464636363646363646364646363636364656563656465656463636364656463646363646365656463656363646363656363646464636363646363636364636363636363656463646563646464636465636463646563636363636363636363646463656565646463636563646465636363636363646363646364636463656563646564646463636464646463646363646363636363636363646463646363636465636363636563656563646364656364656363656363636363636564636363636563636563656363656365646565646365656363646364656464636463656464636363636365636463636363656363646464636363646365636463636365636363646464636364656464636464636565636363636364636463646363646365636563646564636363636364636364646463636365636365636365636363646363656364646565636465656565636463646465636463636463646463656564646464636463646364646463646563646363636365636363656363636364636363656565656563636464656364646463646364646563646364646463646364636363636363636365646365636364636364636363636365636563636363636363636464636365636364636363646363636363646465656363656364656364636365646363636365636463646363646564636364636363636463636564636363636564656365646363646365656364636363646463656364636363636563636364656564636364646463636565656464636364636363656464636465636564656463636365636363656363636364636364646465636364636465636363636464636365636363646363636464646465636363646363636464646363656363646364646463636364636363646465636363646363656364636563646363636363646363656365636364636363656563636364656363646363636364636363646363636465656463636365646563656364646363636363646364636463656365646363636565636463646565636363636364636563646363636563656563636564646363636364636365656465656465646364646363636364656563636364646364646463636365656463646363636363636363636463636463636464636565656463656365646363646464636363646463636364636364656563636463636463656463636464646364636564636564636464656464646564636364636364646463636465636364636363636563656564636363646564656563656365646563656564636365646463636364646563636365646464636365636564636363636363636363646465636565636463646565646364656463656565646363636563656464636363646463636465636563636365636363636363646463656563646363636563636565636363636463636463636363636565656463656365656363636363636463636364646363656563636464656465636363656563646363636564636364646363646363656365636565646363646465656563636363636564636363636563636363656463636363646363656463646464646363656463636463646565636364636363636363636463646363636365646563636463636563656463636364656364646465656464636365636365636363656564646365646563636363636364646363656365636365646464646365636463636563646463636465636365646364636365636364656563646364656364646463636465646463646363636564636464646363646464636363656565636463636563636463636363636364636564656563646363646363656563656564656364646363636363646364636364636363636564656365656463636563646463636563636563636363636365636363636364656563636364646465636563646363636363636465636365636465636364636463636365646365636364636365656365646465636563636364636363636465656365636364646363636363636365646363636464636464636463646464636365646464636563636564636464636463636363636363636464636363636464646363646463646363646363656563646563656364636365656364636363636365656363646363636363636463636365656363636363636463656363636563636464646464636363636363656365636565636464636563636363646563646463636563636463636363656364646363636464636463646363636563646463636364636563636365636463656565646463636365636364636364636463636363636365646463636363636365646363646465646463646364636565646465636363636463646365656463636363636363636464636365646564646463636364656365636465646463646365656364646363636364646363656564656363636364636463646464636365636363636363656364636365636463656363656463646363636363646563636564656465646363646463646364656463636364636363636363656463636364656464636465636363646464646564636363636363636463646563636563656363656363656363656363646363656365656564636363656365646363636563646365636365646463656463636363636464656363636365636464636563636464636365636363646364636563646363636364636364646564646464646463636363636563636564646363636363646563636564646363646565656464636363636565636364646365636363646563656363656465636563636463656464636465636363636364636363636465636464636565636565646364656563646365636465656464636463636563636564636563646364656363656363636363636363636363646364656463636463656463646363646464636463636563636363656365646465636364636363656363646463646565646564636463646463636563636364636363646463656463646563636363636465636463636364636564646363646364646365636363646363636463636363646365656363646364656464646463646463656365636565646364646464636363646563636363646364646463646363646463636363646563636564636364656464636465636465646565656365656364646465636363646563656465646364636364636565656565636563656365646364656563656363636365636464646365656363636564646463636463636363646563636365636565656363646363636364646364636363656364656365636565636563636365656464636465646564636364636363636365636463636363636564646363636365646565646563636364636365646463646565646463636363646565636363636365636363646363636464636564636363636363646363636463636464636363636564656563656463636463646363646464646363636364636363636464646463656464636363636364636363646365636463636564646565656365636363646365646363636463636464646463646364636364646463656363636464636364636363636464656463636564656363636463646464646363636463636464646565646564656463646363636363656565656463656363656365636363636463646365636363636465636464636463636464656363646464656465636463656364656363656363646365656363646363636463646463636563646465636363636465636565636563656463636363646463636464646363636365646363646464646564646363636563646363646565636563656463636363636363656363636463646563646463636464656465646363636463636364636364656363646563636463636363636364656463636364636363636463656363646363646363656463636363636363656363656465646363636463636365636365646365636563636365636363646364636363636364646465636363636365636563656365646565636364656563636463646563656363636364646465646463636564636563646465656464636365636364636463636564636563636364636365646363636465656365646365636363636363656364656463646563656364646463636365656463656363656564646563636363636364636463636464646363636565636363636363646463656363656363636464636363656564646364636363636564656465656463646563656363636564646563646365636564646363636363646464636463636463656563636463656365636463636364636365636463656363646463646363646465656564656563636363646365656363646563636563636363636364636364636365646463656463636464636365636463646563636564656363636463636463636563636463636564656363636563646365646563646463636364656365636465656463636564636564656363636465636563646563646365636463636464656363636363646463636363636363636565656565646563636365646364636464656463656465636364636365636463646564636464636363646563646364646565646463636364656364636364656463656363656465636465636463656365656464656564636364636363646463646363636363646363636365656563636563636563636563646563646463636464646364636365646465636465636463656363656363636363636465636363636463646565656464656363646363646463656363636363636365656463636463656363636464636465636463656464646565636364656365636365636364646463656363646463636463636363646465656365656563646365636565636365636464636363656365636363636365636463636463656563636363656364636563646363636363636363656463656465636364646565656363636463636363656364636365636365636463636563636365636363636363636464636365636363656465646564656363636563656364636363636364636363656363646563636364636363636564656364636363646565646365636463636463656564636363636463646363636365656464636563636364636364636363656364636363636564646463656363636463646464636465636363636364636363636363636364636463646565636363636364636364636463636365636363636563636363636364646363636464646564646364656565636363646365636363636563636563646463646363646564656364646364636463636565636363646463636365636365646363636563646565656363646363656564646564656564636565636463656464636363646363646363646363636365636564636364636563636565646365656363636463636364656465636363646364646564636463636463646365636565656463636464646563646364646363636563636465656363646565656365656463636364646365656563636365636565636363636564636365646563646364646365636364656364636463656364646363636363636563656463656464656365636463636363636463656463646563636364636464646365656364656463656363656563646363636463636563646563656363636464636563636563636363636563636565636364636363646363656363646363636363636364636364636563646363646363636463636464636363656364656464636463656365636363656563636363646564636364636465636365646365636363656463636463656364636363636364656565636564646463636563636364636365656565656463636464656563656464646364656463646364646465636363636363636364646463646565656363656363636363656465646463636365646363656363636363636363646363636365656363656464656463636564636564656565646363636365656363636363636563636363636365636363636363636463636363636365656563636563656564636363646363646363636364646364646465636363646363646363646363636363636364646563636464646363656465636564656364636565656565656464656365636365636363636363656563646563646464646363646564646563656464646363646564636564636463636463646563636465646365636465636364636463646363646563636364636563646363636563656365656463656464636364646464646364646363656463636564656464636463636363636363636563636364656464636464646563636365636563656463636463636363636563646563636364646365636463636363636363656363646463646564646463656363636465656463636465646463636365636563636364656563646364646463646564636463636563646365636465646365636363636364636464646364656564636364636365636563636363636563656363656463636463636364646463636364636363636463646564656363636463636363646363646363646465646365656363646564646363646364646365636363636463636364646365636463636364646464636363636463656363636363636365636365636463646364636363636564656363636363646363646463636565646463636364636363646465636463656363636563636464636363636563646463646464636363636563656364636464636463646363636363656364636465646364656563656363646365636464636463636364656463646363636364646563636363656464636463636464636463646365636363646365646465636363636364636363646364656363656364636363646463656363636565656364646363656364636463636563636563636464636364636364656363656363636365656363636363646363636363636563656464646463636364656363636465636465646363636563646464646463636365656365636464646464636464636363636363636363656563636363646363646463656363636363636563636363656363646364656364636463636463636363646564636363636363636363636563636563636563636363636563646364646363636363656464636363646363656564636363636363646364636463656363656363636363636464636463636564636363636563646363636365646463636463656463636363646365646365636463646465646465656363636465646463636463636363646365636563656364636365656463656363636363646364636364646463636363636364646563646363656464646363636365646363646463636365636464656364636464646565636363646365656363636463636364656363646365636563636364636464636363656463636363646563636363636364636563636364646363636463646363656365636463656563656463656363636364656365636363636363646363646563646365636464646364646563636365646465636465636365646563656363646463646563636363636564636564636363646463646563646363646464656464636565646363646363646363656364646565636363636363646563646564656463656464646365636563636563646365636364636363656565646363656463636363636463636465636363636363636363646463636463656364636365646464656363656463636564636363636563646363646363646464636363646463636464636465636365646465656363646563636563646363656564636363646363636464656365636564646365646463646564656464636364636463636364636363656363636363636463646564636564656365636465636463646563636363636464636363646365646563646464656363636564636565656563636363656463636363656463646464646365646463636365656363646364636463646364636465646463636463646564656364656563636463646364646464636464646363656563646365646364656463636365646463656464656363636364636563636563636465646364656464656364656463636363646363636364636363636463636363636365646363636363646464646564656565646463636363636565636564636363636563636565636565636563646463636364646363636363636463636565646365636463636463646463636363636363646463636463636463636363636465636364656363636365636563636365656363646563646463656364646364646363636364636464636365646465646363656465636563646463636365636463656364646563646363636463636563646364636364636563636563656363636364636364636363636363636565656463646364636563636365636463636463636464636363636465636364636563646364636465636363636364646463646463646363636465646363656365646365656565646365646464636363646464656365646563636564646364646363656465646563636363636364636365636463636463646364646365636565636563646364636463646365636465636465656463646463646463636364636363646563636463656563646563646563636363646464636363646363656364636364646465646465636363656463636563636363646365636564636465636465646463656364646364656363646463646563636363646563636363636364636464636363646364646364636363636365646463636364656463636363636365636465636463636364646364656563636363646364646365636363636565646464636464646563636363656463636363656464656364636463636363656363636464646564656363636464646563636363636365636565636363636463646364636363656365646363636463636363636363646364656363636364636465646463636363646563656363656363636564636365636465636365646563636364646563656465656563636463636364636464636464636563636565656364646464636563636463656463636565656365636463636364646365656565636563656564636363646464636465636363646563656563646364646363656564656463636364636563646565636463656464636563646365636364636364656364636363636463646363646564636365636463636463636463636564646463646565636364656363636363636364636363646563636363646463636563636363636464656363636464646465636363636365646363656363636363646364636363636363656565636363656563646563636463656363656364646365646464636363646463636563636564646564646364636364646463636365646364646463646464636465636463656464646363656365636565646365636365646364636565646363636463636363646363646365646463636463646365656564636363636464656464636364636363636365646363646563646364656365656365646365656364636563646363636564636365636565636464636365646364636365636464636363636363636465636464646563636563656463646564646363636363636563636463636365646363636363636365636563646365636363636465646363636363636363656464636363636564646465636464656364636463636464636465636465636363646365636564636363656365646464646365636464636464646563636364636463646463656564646564646364636364636464636564636363656465656463646563636364656465636363656463636463636364646464636363636464646464636463656364636564646564646563636464656363656363636363636565646364656364636363636363646363636363646563646364636563636464656363646563636563636465656364636464636464636564636563646565646463636464636363646365636563646364636465636363646465646363646564656465636563636463646363656563636564636465636463646363636365636365646365646365636463636365636365656363636463636463636464636465656363636563646363656465636465636364636463646364656365656463636465636463636463636364636365636565656564656363636365636463636363636465636563646364656463646363636364636363636463636465636363636565636364656563636465656465646364636363636464646364636364656363636463646463636463636363636463636465636363636364636363636363636565656363636563646364646363646463646364656564646363636363656464646364636363636463636364646363656464636563636364656465636565656463636463656363656363636364656363636363636463636463646365646363646363636464656464656463636463636364636463646365646465646463636563646563646463636365636363646564636363636363636563636463646364646463656363636365636463656463636465636363656363646363636463636365656563656365656365646463656564646465636363636463636363656464646465636564636364636363636363636463636363646465636563636563646363636363656563656563646564636363636365636363636364656364656463636463636564656464636365636463636365636465646365656363636465646564636565646364646464636563636463636465636363656364656463636364636363646564636363656363636365656363636464636364636363636364636463646363636564636463656465636463636364636463646563656364646563656563636563656365646363636563656364636464656463656365636464646563636365636363656364636463636364636463636563646463636365646563636463656363646363646365636365646464636364646464636364636364646464636565646364636464636363636464646363646563636463636464646563636464656463636363646363656565636363636365646364636464656563636363636363636563636364636564636363636363646463656565656463656564646364646464656464646364646463646363636563636464656465636363636363636365656464636564646463636363656463636363646363646363646463636564646363646363656563656363656563636364636363636363636463656364656463646465636365636463636365646363646463646463636565636365636565656563646364636363636463636565646363656363636365656365656363636363636564646365636363656464636365646464636364656465646365656564646465636364636365636365636563636365656464646463636365646464656365656365636463646364636365636363636564636365646363656363656563636565656363646563636565646465646363636363636463636463646563646563636364636363646464636563636363636363656563656564636563636364646463656364646364656463636464646363656563636463636463636465636363636363656463636464656363636564636564636563636364636363656364656363646365646465656463646363656363636463636365636564636365636464636463646563646363636463646364646364646464636465646463636563656364646365656563646365656365656463646364636463656465656363636365646363646563636363636463636365636364636463656364656463636363636563636463656563656364636363636365646364636364636364646564656363646364656363656563636364646463636463646364636365656365656363656463636563636563636364656463636564646463636565636363636363636565636464646363656363636464636464656463636363656565646364656363636565646565646364636563636364646465646363656363636363646365646363646364646565636365646464636564636563636463636563636363636363646465646465656365636463646364636464636365636563636365636463656565636565636464656365656363636364646465646464656364636363646563646564656363646365656563636464656365636564636365636364646363646365636464646464646464646465646364646365636464636364646563636463636365656463646564636364656365636364636363656363636363646463636365636364636464656364636465646363636465646363656463656563636364646463656565636364656364636563646364646565656564646565646365656363646563636364636363646464636565656563636565646363646564636464636563636563656363656563656465656364646463646465636464656363636365656463636463636363646464656465636463656365636465646363636463636364636563636363636465636463646464636463636564636363646363636563656365636364656565656463646364636364636465646563656365636463636463636363636464636364636363636363646364656463636365636564636565636563656564646463636464636464646363636465636363636463636463646464646563636363636363656363656364636563656563636363646363656565636365636364656363656363646563646465656463636364646563636363646463656463656365646363646463636464656364656565646563636364636363636463656463636465636363646465636563646365656363656563636464636364636364646563646463636463636463636564656365636464646363636563636365636363636463636465656365646463656363646363636464636363646465636364636364636364646465636563636364636365636365636465646463636565646463636363656465636363656463646463636365646465636363636364636563636565656363656463646364646463636464636363636463646463646363656563636465656364646363636363636364656364636465636363646365656365646365646463636363636365636363636364636364636563656465646364636364636564636365636363636463646365636465646363656463656463636363636364636465646463636464656463656363636463646365646365656364636364636464646364636363656363636463636365636365636463636363636363636363656364646364636364646465646364636563656365646364656463636463636463636363636363636363636363646565646363656363636563636463636563646363636463656465646463636363636364636364636363646363636365636363636464646563656364636564646464656363656363636363656564636365646363636363636463656364636363656563646464636564636363656363646363646463646365636364656463636463646463656563656463636464636563646363646365636565646564636463656364656563636463646563636564636364636363646364656363636363636364656463636463636563636463646463656463636364636364646464656464646363636364636463636364646463656465636364646363646565636364636364656563656465636364636364636365636563646363636363636364646363636365636563636463636563646563636463636364656363646464636465636363656463636563656363636464636564636363636363636364646365656363636564636465656365646364636465636464636364656563646463636364646463636363646364636365636365646365646463636464636364636563646563646364636364646463636563646464636364646363636363636465636364636364646363636564646363636564636463636363636365636563656363636463646363636463636363646363646463636563636365656363646463646365656463646463646364636363636363646564636563636365636363646363636363636563636563646463636363646463636363636463636564656463656363656364646464646365636363646363656464636363636464636365656365636564646365636563636463656363636363646463636465636364646464636463636364636463636465636364656363656463636465646464656565636365636363636363656564646363646363636564646363636365636363636564636365656363646564656363656363636563656463646364636363646463646363646463656465656364636363636363656465636363646563646364656365636363636465656365636465636363646365636363646364656365636563636365636463636464656563636364636363636365656465646363646564636464636464636564636463646363646563636363636363636465646365646363636463636565636363656464636465636363646363646464646364636363656364656365646464656565636463636464636464656365636364656563656465636463636563656564646463636563636463636363636463636463636365646464636364636465636564636363656464636363636363636463656464636365646463636563636363646464636564636565646363636463646563656364636563636364646463636463636464656363636364636365656364636363636364646363636463656463636363636363656363636363636364636463636364636365656364656464646564636465656364646465646465646564636463646363636364656364646564656463656565636365646364656365636364636564636364656564656563636365656563636364636363636364636565636363646365656363646363646365656365646365646463636365636363656364636564646464636463636363646363636463656565636564636364636365636363646363636363636563636363636364636363656364636364646463636465636363646364646563646364646563646363636563636363656364636465636364636565656363646465656363646564636463646564646564646463646364636363636363656363656564636363656363646363656365656363646465656564646463656363636563636363636465636364646363636563636364636564636565636463656563646364646564636363636363656565656363656363646564646363646363636363646363646463636564646363656564656363656364656465636363636464636363656463636363646563656465636363656365656463656565646364646365646363646565646365636464646364636363646365636564636463636563646365636363646463656565636563636564636364636563636363636464636363646364636463656463636363656465646363636463636363636563646463636364636363646363636365636364646563656363636364636463636363636564636565636564656465636363656365636365636463656363656363636463646365646364636363656563646464656463646463636463646565636463636465636364646465636463656363636563646565646464636363656363646463636365656464646363646465646363646563656465646364646364646365636464646363656464636364646563646365646463646363636363646465636563636363636365646463636464646464636363646365646365636464646363636563646363636463636464656364646563636363636564656363646465646363646363656464636464656365656365656464656364636363636363636363636565636365646463636563656463656363656363646363656363656365636363656363636563636563646464636363646363656563636364636463656465636363646363646464646363636365646563646564646363646365636363636463636363646363646463636564636463636363656365636463636464636563656363646365656363636364636364636463636565636365646563636463636463646565646363636464636465646465636364636364636363656364646465656563636363636365656563636563646363636463636564636564646365636363646464646463646363646364656364656465646365646363646363636363636363636363636564646364646363646565636365636363636363636364636365636563636463646463656363646464646463656565656363636365646564636363646465656363646463636363656364636463646465646563646563656363636363636563636463656363646563646364656465656363636463636464646363636363636364656464636364656563636364646363646363646565656564636365646363636464646564646364636363646564636363636464646363656563646363636563656365636363636365636363636363636365646363636363636464636365656463646564646363636363636364656365636365656364646464656564646363646365656565646363636363636364636465636363636364636363636565636564656463646465656363636564636365636363636563656564636463636363636463636365646463636363646363656565646365646563636465636564636363636565646463646363636364646463636463646465646463636365646364636563646563636363636364646363636364636363656465636363636564636464636463636563646363636363636363636563636465636364636463636364656565636363636363636464636363656465646464636564656364656364636565646463656465646365646563646365656563656363636363636565636365636463636463656363646563646363646364636364636565636363656463636563636363636563646365656463646463636365646463636463646365646463656564646363646365636365646463646564656464646363646364646364656464636563656365656365636365656365646563656463636563656465636363646364656564646564656463636364636364646463636365646363636565646565636363646365636463636463636464646365636563636364646464646564636363646463646365636563646363656363646563646563656363646363656463646465636364636463636563636363636363636463636363636563656363636463636363656363656365636363646564646363646465636363646363636563646364636363636365636465636365636364636364656363636363636463656363636463646363636363636363636365656363636364656363636365646365646363636564636564646363646563646464646363636363646563656464646363636365636464636465636363656364636363636363656364636363636365646363646463656563656364656364646563646363646363636463636463636363636363636363636463636364636363656363646365646364636565646363636463636363646563656563636565646463636563636365646365636463646464656563636463646363636564656463646364646565636463646363656464646365646364656363646463636565646464636563636563656563656363636364636465636464636363636463646365636563636363646363636363646364636363656364656363636465636563636363656365636463636363646365656465636363646463656463636363656564646464636464636564656364646463636364636464656465636363656363636364636463656463656464636463656363646364646363656463646463636563636363636563636364656563636363646564656363646364656363646363646364656463636363636363646463646364636463636463636363636463636564646363646564636464646363636465646464636363636363656564656365646363656363636364656363646463636364636365636364636364636565636465636463636363646463636364636464636565636363636365636365646363636364636563646563646363646364636463646563646365646563646563656364646563636364656364636463636565636565636464636563646463656465636363656363646364656363636364656464646365646563656565646365646363636363636363636363636364636364636365646364656365656364646463646364636364646364646464636365646363656363636365646463636565636363646565636363656563646563646463656363646564636364646365636363646365636565636363646565636465646363636463636364636463646363636363636463646463656364646363636363646464636363636465636564636363646364636363636364636363646363646364646365636364636363636364656363636363636363636363636563636464646364636363646363636363636465656564646465646363636363636364646463636563636365656465636464646464656363636363636363646463646465636363646463646463646363636363636464646463636364646363636365636364636565636463636564646364656363636564646463656363656563636563656363636363636463646363636464636364636363636463646463636363636364646464636463646465636365636563636564646564646363646563636465646365656463636363656365646363636363656363636363636463646363646464636363656364636565636464646564636564656363656563646463636365646365646464636363646463636563656364656565656463656463636464636363636465646363656363636364636464646363656363646365636363646363636363646565636463656463656464636563646563646363646364636563646464636364636464636464636363646365636364646363646565636365636363636363646463636463636365636364646363636563646365646564636364636464636563646463636363656363656465646563656365636363636363656565636365636363646463656363636565656363656363636464646364646463636363646563636363646564636563636464636465646564636364636363656363656463636364646363646363646563636465656463636564656463646363636363656463636365646464646364646464656363636464636364646463646365636464656463656364646364636365636363656364636463636464636364646463646364636363646465646463636364636564636363646363636363636365656365646464636363636363656463636465636365656563636464646563636464636364646363646563656364646365636363636463646364636365636463656464646363636363656363656363636363636463656364636363636564636563656463636465636363636563636364656365636464636463636364636463636463656363656363656365646465656464656364656465636365656464636363636464636465636363636365646363636465636364646564636564636564656363636464646363636365636464636565646363636363636463646363636364636463656463636365636363636463646465636563636463646363636463636363636464636465646363636363636363656463636363646563646463646364636365646365636565636463646364636363646463646563646463646563636563646564656364646363636364636365646364646465636463636364656563656363646463636363656463646364636363636364636463646563646363636365646364636363646463636364646364636363636363656465636565636363656363656364656365636364646365646465636463636365646363646564656363636563646364646365646365646365646363636364656564636363636363646363636563646463636364646363636465636365646465636363646365646365636365636363656464636463636365636365646464646364656463656365636465636365636364636365646365656563646365646363646364636364636464656363636565636365636363636463656465646363636464636465636563646465646363636365656363656363636364646464636364636464636563636463636564636363646363636364656363636363646365636365636363646464636565646464636464646564646565646363656465656365656364636464646464646563636363646464646363656565636364636363636464636363636463636363656363646463656363656463646464646563636464656563656565656365646365636464656364646363636365636363636564636365636463646464636565646563656364636564636564646363636563636463646463646363636363636463646563646464636464646364646365646464636563646463656564636363636365636565656363646364636463646363636565646364636364636363636364636364636464636463636363636565656465646363646363636364646465636565656363636563636364646463656363656563636463646563636564646465646464636363646463656463636365636463636364656363646365636564636364646565636465636365636365656465656564636564636464636465646365646465646463656363646365636464636464636364636363636564636464636564646565636364646363636363656464656365646363636463646463646563636463656363636563646363636564636363646563656563636363656463636464636365636363646465636563646564636563636365646363646463636363646463646364636463636363646464656463636363636363636365646564646563646563636464656463646463636465636364646563646363646363636565656364636364646364636365636363636565636463656463636365636464636465646365646363656363636463646363656363636365636364656363656365646363636364636365656564636465636563656565636363636464636364636363636365636363636563646563636463636463646365646564636364636465636363636363636365656365636565636563636565636364646465636463636364636364636564646565636363636463646365646363656365656463636364636464636363636363636464646465636565636463636565636363656365636463636463646365656363636564646365646365636364636365646364646364636563656563636463656464636463646563636364656363636464656363646363656463646365636364646463636563636565646464656364636564646363646363656363646463636363656365636463636363636563636363646363636364636363636365636463656563646364646365646564646363656465636465646364636565636364646365646364636463646363656563646563636363636463636564646364636363646363636563636565646364636363636465636364636365636563636463636464656563646363656464636363656363636363636363636464636565636363636565656365646564656464646363636563646363636364636363636365656364636565646563646363636363636365636364636364646464646363646364636363636364636463646363636563656364656564636464636363646363656464636363646364646363656364636464636363656465636363636365646564636363636365656364646463646463646364656363636463636564636364656363636563636463646365646363656365636464636565646464656363636563656465636363636465656464646565646563646365636363646564646463646363646363636463656363646364646463636464636365646363646465636564656365636363636365646463656464636364636365646364646463636363646364656363636563636564656363636365646363636363636363646363636363656463636465646365636564646364656564636463646363636563656463636563656464636363656365646463646364646563656465646463646564656564636465636363636363636363636463636464636463636565636364636363636463656463646363636364636363636363656363636363636364646364646363646364646565656564646565636563636563656465656464636364636363656363646564636364656463636463636363656564646363636563636565636365646464636563656365636465646463646363656465656565636363636564636564636363656463646364636363636364636463636465646363636365656363646363636464656564636363636365636363636463646363636463646364656564656365636363636463636363646363646364636364636463646363646564636363646364636363636464636364646464636465656463646563646463656463636463636365656564656363646464636364646464656563646465636463636364636563636464636365646465646365636463636365636463636565636563636465636363636365636363636563646364646363656363646563656364636563646364656365646464646565636363636463656363636365646563646365646563646364646363656564646364636363646363636464656363636565636363636364636363646363646463636363636563636363636364646464656364636364656365636563636465636364636565646464636363646564636464636365656365636465646365636363646363636363656364636363636563646464636564636364636363636564636364646363646365636365656563636365656363646364646365636363636363656563656363636464646465636563646365636564646364636364646363646463656363646565646363646365636365646465646364646365636564656363636465636364636363656363636464636464656563646363656365636363646563636465636364646564636365636464636463656363646363656465646363646364636564636365636363636463656363636364646364656563656363636463636363636463656363656363636363636363636564656364636565646465636563646363646363646364636363646363636363656363636363636464656563656363636365636364646364636464636363646365646364646364646363646364636464656364656365636463636565636363636364636363636465636464656464656465646464646363656463636364656465646365646464636564636363656464646464636363636563636464656463656464636364656465656364646565636364636363656563636563646463636463636465646363656563636463646565646363636365646363636364636365646464656363636563646563636364636363646463636563636363656365656363636463656464656363646363646364636364646563656364656464646465646364656463636564636363636363656563646564636463636463646563636364646365646463656363636563636363646365656364656364636564656363656363636364646365636564636463656364656464636363646463636363636364636365656565636365636363646564656365636365646364656563636363656364636365636363636564646465646463636364636363636363646563636463656464636465656363636363646364646465656364646463656465636563656564656564646463656363636564656463656565656365636463656365636463646464646463646463636563646363636365636365646463636464646363656464636365646364636465636463646363646464636465636363656363636364636464636363636464636564636364646563636364636364636464636364636563646364636364636465656364636563636363646364636363636465636364636364646563636364636465636364646463656463646464636463636363636463656363636565646363636364646363656365636365646465656365636463656363636564646364656565636363646463636363636363636364636565656363636363646465636363646565636463636564636563636465636363646364636364646563636463636364646363646364636363646464636363646463636464636363646363636463636363636563656363636565636463636363636363636463656365646364636365636463636463636463646463636365656363646365636465636363656363656363646564636563636363656364656364636464656464636463646363636463656363646363646465636564636363656365636563646463636565646465646463656363656563636364636364646465656365636563636563646364636364636464656363656463636463656463646363636364636364656563636563636364636365646363636364646365636565636363646465646365636363636464636363636365636363636364636463656463636363636563636363636463636363656363636364636363646363636364646463636564656363656563646365646364636463646363656363646463636363636563636363636565636464646463636364646565656363656564636363636565646364636364636564636463636463656463656365636563636364636463636565636365636365646363656363636365636365636464636363646364656365636364646564636463656363636563636464646363636463656363636464646563646363646563636363636464646465646563646363656365636363636565646363646363636564636363646563646364646363636463646363646363636364636365636364646464656463636464656463656563636365636464646563646363636463636464656463656364636563656464636365646365636364636363656363636463646565646464636463636364636363646365636463656463656464636363636463646463636365636363636563646365656464636463656363646365636363636363636364636563656564656465636363636365646563636363636463636364636563636565636363636563656364646363646363636365636364656464656464636463656563646565636565636363636363636563636463656463636363636363646365636364636565636363646463636463656563646363636465646463636365636365636364646463646363646363636363656365636365656563636363636363636464636363636364636364636363646464646463636565636463636563646364646463656463636364656364636563656363656364656363636363636464636363636463646364636463636463636363636363646563646365646465636564656363636464656364636364646465636364646564636363646465646565646364656363656365636364656365636463646463656463636464636464636363636363656463646563656563636365636463646365636363636564636364636463646463646463656364636565636364636463636363646563656363646364636463646464636463636363646365656363646365636563636563636463646365656465656364636364636363636563636564636564656463646463636363636363636363646363646564656364636364636363656363656564636364636365636364646364636465646363656365636363656465636363646364656465636563656363636365636365646363636563656463646464636364656365656463646463656365646363636565656465646564646363636363646465636363656363636363636363646363636364646465636364656563646363636363636363646365646463656464636363646563636364636365656464646364636364636363646463646363636363646464636364646363636364636364636363636364646363636565636363646563656563636364636564636364636363646363656365636564656365636363636363646364636363656363656365636563646465636463636563646465636364636363656463656563636363646463646365656363656363636463646563636363636364646363656565656564656564646363636564646563646364656364646565646363646364646363656363636364636365656364636463636363656365636363646563636363656363636464636363636363646564636463636464656465646365636463636363656363636363636464636363636563656564636464646364636365646363636464656464646565636563636363646363646363646363646363636365636464656463636363656464656364656563636364646363646364636364636364646363636363636364636363646363636464646563636464656463656563636463656363636463636363636463646365656363636563646363636464646463636364636464646463636564636363636364656463636564646363646463646563636364646365656365646363656363636463636363646463636563656564646363656563636364636363646363646364636363636364656363646464636364646363636365646463646365646564646563646365656363636464636465636363636463636463636363656465646564656364646565636565636463636564636464656564646365656363636463646463656363646364636563636365636563656465656363656563636364636363636465636363636363646363636363656464636364646365636363656564646565656364646363646463646363636364646565656465656363646563636365636364646363646464636363636364646363636363636364636364636365636363636463636363636363636364636364636564656465646363636463646365636465646365656363636463636363656465636565656563646365636365646363636463646364636463636463656363646563656363636364646365656363636363636364646364636463646565646365636364636363656364656564646365636563656463656363636463656363636363656364636363656364646563646363646563646563656363636364636463646564636363646365646363646564656363636563636463636363636363636365636463656563636463636364636465646363646564656565646464656363636364656364636463636463636563646363646363656463656364646563656563656564636465656363656363636563636364646564636363646363656364636364656364636363646463636363646363656364646463636565656363636363646365636565646465656364646563636363656563656365636363646565646563656465636464656563636465636364656363636363636365636463646363646464656463646464636363636363656563646363656364636363646365646365656565646563636565636365636365646563646563636463636364656465656364636464636464646363636463636365656363636463636363636565646363636365656564636563636563646463636363656363636464636563636363656364636364636364636563656363646365636363646465636563636364646364656364646565636465636463656563636364646565646363646364656363636365646363636364636563636364636463656363646465636365646563656363636465646464646363636563646364636363656463646564646465646463656464636363656363636363646464636463646565636464646463646463636463646463636365636364636463636563646463636563636565636363646564636464636363646365646565636564636363646364636464636365656363636564636564656463636363656364636464656364636465636364646364636364636363636364636465656464636363646464636565646464636363636364646364636463656465636565636363636363636465656364646565646364656563636463636464646363636363636365636363636363636363636363646463646365646464636464636363636563646563656365636463636565656463646463646363656463636465646463636463636363636365646365646363656464646364636364636464646364646563646363656463636563636364636365646365646363646364636363656464656363636363646363636464656563656563636563646363656363636465636564636363636464646363646564636363636364636363636364646363636363636563636363636364636363636363636463636365636465636563636363636363646363656363656363636563646365636363636463636563646364636365656364646363636463636464656364656363636363636365646465636363646565656564636563646565646365646464646465646465656364636364656463646365646365636363636463656364656363636364636463636363636565636463636363636364636365636363636463646363636463636363636563636564646564636463636463636463636463646465646363646364656364646463636363646363636463636563646363646563656464636463636563656364656463656363646563636463646465636463636363636465636463636363656563656465656363646565636464656363636365636565636463636564636363636364646365636364646563636465656363646563646363636363636363646464656365646463636565636364656563646563646363656363646563656364636363646364646363656365636465636363636463656464656365656365636463646365636363636463636365656363636464636363636363636365636363646463636364656363636563656364636563656463636464646363656363636364636364636465656465636464656365636565636363636463646463656463656463636363656463646563636463646565656564646464636563636363636365646563656463636465636465636364656563636463636464656363656463656363636563636565646364636564636363636465646564636365636363646563656465636363636363636364646465656564646563646463656364636463636363646465636363636363646363636563646364656364656463636363656463656365656563636463636364636363646463646563646563656563636563636364656463636363636564636465636363656363646364646563636563656363636363656363636564636364636565656364656564636465646463646563646463636363636365646363656565656463646464646363636363636364636463636463656365636463636363646464636565646365636463636463636364656364636565646565636364646364636364646365646365646465636364636363636363636363646363646463646363646365656363656364636365646364636565646465636563636563656464636464636464636363636363646363636364656463656563646363636363636463636363656463656564636363636363646363636463656563646365636364636463636463636464636463636464636365646364656365656363646363636465656563656463636465636563646564656463656363636364636363656363636464636563646565636463646363656364636464656565656565656463656364636363656363636364646365656363646463636563636364636365646463656563646364636463636463646564656363636463656563636563636564646364636564646365636364646365636364656564646464636365646463636464636465636463636365646464636363646363656464636363636364656365646565636565636363636363636363636563646463656465656364646364636363636363646364646463646363646465636465636464656365636363646465646363646365636364636364646465636363636464636364656363656564646563656564646563646465636363636363646364646364636364656365656463656463656463646563646364636363656563656365656363646565646463636364646563636564636363636363656364636564656464636463656465636464636364636364636363636364646564636563636463636365636364636365646463636364646463646463656565636463646563636363636363636363636364646465636364636463656364646363656363636465636464636464636364636365646363636364636364646563636363646363636363646365636363656363636365646463636564646563636365636365646464636363646363656464636465636364656364656464646563636563636364646365646364646564646363646563656463636463646363636363646463646364656363646563636363646363656463636364646363646364636465636363646463656363656364636363636565656463636363636365636464646364636563636363636463636563636365656463636464636363636364646364656363656564636565656363646464656365656365636563646363636364636363636363636563646364646363646363636465646363636563636363646564656463636465656464646465636563636364636463646463636364656465636364656363636463636465636365646463636364636364636363636365636364646363656365646464646563646363656565636463646365636563636365636563636363636464646563636363636363656364636363656365636465656563656463646363636363636363646365646363636363636363636463636463646364636363646465636365656464646565636365636363636363636563636563646565646564636363636365636365646463636463636464636363636363646365636363636464636363636465636365646363636464656365656463636365636363656564656365636465636364656363656363636564636363636363646363636564656465646564636364646565636363636363636364636463646464636363636563646365646363636365636565636365646363656364646363636365636363656464656364656363646563656563646363656364646463636364636363636363636463646463636365636365656365636463636463646364646363636364636463646363636464636364636363656363636464646363636564656363636363636465636463646363646464646363636464646465656363656464646363646463636464636563656463636363636563636364636363646363636365636363636363636464636464636363656363636464646364636463646464646463636363636364636363636363636464646365636364656565646564636363646463636463656363646364646563636464636363636363646563636364656564646465656363636563636365636465636365636464646463636363636363636363646365646363636464636464636364636463656463636563636364636464646364656363636363656563636365636463636363636563646364636465636363636363636363636464646465636365636565646565636364656463636564636364636465646364656363636464636365636363646363636363636363636563646563636563636563636363656363656464636463656463656364636364656365646463646363636363646363646564646365656565636363636563646563636564636464636463636463636364646365636363636363656464646465636465636564656564656363636363646363646464636464636563636363636364636565646564656464636563636464636365656565646563656565656363636463636463646465636564656364636363646363636565656364636363636465646464646365636365636363636563636463656363646365646364646363646363656465646564646363636363636464636563636464646463636564656365636363636463636463646363636564636464636364636465636464656363636465636464646364636363656363646463646364636464636365646463656364646363636365646565636364636363646365646363636363636564636564636364636565656464636463636564636463656563656363646363646465636465646464646365646365636464636363646565636563636463656365636363636364646364636363656363656363636464646465656464636364636465636365636365636365646363646363636463656363656464636464656563646364636464646565636563646364636565636364656365636465636463656464646564646363636363646463646363636463646364646563636364636364646363636365656364656363656564646364636465646363636364636565656563646565636364646563636464646464646563636363636363646565656463646464636564636363636465646363646364636565636563636363636365636364636563636463636365636563656564636365646363636465636363636564656565646365656464646565646363636364636464646564646364646363636363656565656563636465646365636365646464646364636363636365636364636563646364636365636363656364636365646365636365656365636465646464646365646563646465646363636563636365636463636464656463656463646365646563646363636364646464636365646365646463636463636564646365646364646463636364636463656364636564646463636463646563646464636363636564656564646464636363646364636363656363636363636563636363636364646464656465656464636463646463636364636564636363646364636364636363646365636364636365636363646564646363656563636463636463646463656363636364636363636464646565656364636364636563646464646364646363646464636464636563636465636464656363636563656464646564636563636563656463636563636564656465636363636364636364656564646365646364656563646365636463656363636364646363656364636363646364646364646363656363656463636463646463636365646564636364636465636364636463656563636463646363636363636463646363636365646563636364636463646564636365636364646563646564636364636564646363636363636363646364646363656364636463646363636363636463636363646564646364656363646464636565646564636563636464656363646463656563646363636565636365636365636464646463636363636363656463656364636364636463646363636563646464646564656563636463646463636564636464656463646365646363636363636463656563646564656364636463636565646363656464636364656364656364636364636563656363636363646363646363656564646363636364656565636363636463636463646465636463656463646464656563636364636365636564636464656363646365646363636364636364636363646364636563656564656364636563636364646363636464646365656363636465636363636364636464636364656363636465636364656564646464636363636363636464646364656363646363636463646464656363636364646463636363636363646363646564656364646364636365636463636564636564636364636365636464646364636364656365636463636365636563656563656564636463656363636464656363636363656463636563646564646463636363636565636464646564656363636565646363636564646464636465636365636363636565636364656464646365656364656565636564656363646365656465656363636463646365636364636364646365656465646563656363646364646563636364646563636365636363636363646363646364656463636564636563646464646363636365636465646563636465656363636363636563636364636465656364636465636363656363636463656465636363636563636365636464646464646463656464646363646363636363646364636563636363656365656563636363636463656363636464646465636363646564656563656365656465636564646463656363646565636363646364636563636364636563656363636364646363636365636363646363656463636564656563646363636464636463636463646364636363646463646463636463636463646563646363646563636563646365636563636364656463636463646465636565636364646564646363646565636564646565646563636365656463646465636465636364646364636564656365636464656463636364636363646363636363646563656463646363636363656364646364636563646563656463636363656463636363636463636464636563636364646563656364636364636465656363636464636364636365646463636563636565656463636463646564636463636363656365636364646363636363636463656365646363636464636364646363636463636565646365636564646363636563646364636563646564636563646563656464636365636363636563656563636363646565636363646365646364636563636364636363656464636365656363646363636363646463636365646563646364646363646563636564636364646464646565656363636565646363636365636463636463636363636465636365636363636364636363656363646563636463656465636564646463646563646364656365636363636463636365656465636363656463636465646563636363646463636363636363636363636363646365636363656465646363646464636564636565656564636463646464636563636363636563646365636564646363656364656464646463646363646364636363656363646464646364656363656564656465656563656363636464646364636363636563646365636363636364636465636363636364656363636565636363646463656363636464646363636363636463636363656465636364636465636563646464656365646364636463636463656564646463646364656363646364636363656365636463636565636563636363646364636363636463636563636364636364636563646365636463646364646363636365656363646465636364636464636363646363646565646364636363636364646364646364636363656464646464656365646563636363636363646363636365636463636365636365646563636363646564656463656463636463636563646364656463656364636463656364656463636363646365636565646363646365646364646463646363636365656563636563636363656565636363646363656464646365646465636563636363636363636363646464636363636363646363636364646563646463656463636465646365646363656365636365646364656363656463656463636363646363656465636364636463636563636463636364656464636363636363646564636564636365656365636364646465646364646363636364646365636563636363636563636464636463636464636463636364636463656563646563636564636363636464636363646463646363636363636464656463636365636463646565636563636564646363636363636363646364636463636463636363656363636465646564656364636363646364646364636363656363636364636363656364636364636364646364646463636363636365656363636464646463646365636465656564636463636363656364646464636364636363636364636464646463646463636363646564636363636364636465636363646364656363656363636565656563656563646364636364636464646463656364636364636463636464636363636363636364636563636363636563656563636363636563646563636463636364656363656365636363646363636364636364656564656363636365636565646365636463636365636465636463636564646565656363636363656463636363636364646363646365656363646363636464636465636464636365636563646365646364656365646364636463646364636364656464656465636364646465636463636464636365646363656363636365636364646365646463646463656365656364656364646563646365636364636365646363656463636364646564636465656363636364646363636363636464636363656363636565656565636465656463636363636565636363636363636463636364646364656363656463636363656363636564646363636463656363636564636464636463656364656363646565656463636363646464656463646364636363636363636463636563646463636364636364646364636363646565646564656363636565656563636364636465646565636364656465636363656465646363636363636363636364646364636364646364656365656463636463656364636563656363646465636463636464656363656364636365646365646563636463636365636365636364646563636363646364636463656365636563636463636563646363636363656365646463636565656365646364636464636563656463646363656565656363636563636363646463636463646363636563656564636363656363636364656564656463656363646363636363646363656564636363656463646563636363636463656363656363636363656463636565646363636463646363646563636564646363656365636363636364646364646564646565636464656363636563646364646365636364636465636364646363646563646363656463656363656364656364646463636363646563656464636365636463636365636365636363656365636363636463636463646363656364656563656363636463646363636464656563636363636363646363636363646463656363636365646363636365636365646363656464646364636363636563646363636364636363636464646465636465636464656563636463636365646363646464656463656565646463656464656363636563636363636563636565636363636463636365646563636464636363636364636563646364636464656463656363636363636363646563656465636564636363646465636463636565646364646464646364646364656463636363636564646364656564656363636363656563636363636463646365646565636363656363646563636463646464636363636563656363646363636363636363636463636463646563656563646363656363656563656363636465636365636364646564636363646363636464636463636465636464656363646364636363656363656563646363636563646364646463656563646463636364646363636364636365636463636363646364636465636564646364636364656564646363656464646565656363636364646464636564656364636363646365646363636364636464646563646363656363636464646363636364636463636363656364636565656363646465656363636463646565646364646363636564646363636363636365636363646363646463636363646364656565636563646365646364646364636465656465636363636363636465646463646463636364636363636363636463636364646363656564646364636363636363656364636463646365656364636563656365646563646463656363636363636364646364646363636463656363636365636363656363636564646465646565636365636365636363636363636465636365636463646464646363636563646364646465636564636363656363636363636364646364636463636363656563646564646563636363646465636364636464636365646363656365646463636465636463636563646465636363646463636463636365646463656364636364636365646563646465646565636563656465656363646363656463656363646364636363636564636363636565636363636363646565636363646563636363646363656563636365646363636564646463656363636363656464646365656563646465636363636365636363656463646364646565636364636464636364656363636564636364636363656363636463646465636465636364656565636365636464636363636563636464636563646464646365636363656363646364656463646464656564646363656563636363656363656564646463636363636363636463636463636363656363636463636465636564656363646363636363646563656463636365636463636465646463646564636465646365646364636464646563646464636463636363656563656463646365636463636463656363646463636563636363656463636363636563646363636363636463636363656563636464636465646363646364636564656564636564636563636465636363646364646565656363656365646463646463656363636363646363646564656363656364646565656365646563636364636363646363636365656464636365656363656365636364636564656363636364646463656564636464636563636563646364636363636564636363646465636364636363646565646363646364636464646465646463646364636363656363646363636363656365656363636465636364636563656363636364636365636363646463646364636563636363636563636464636364636363646365656565646463636364636365646365636463636365646364646563656363636463646563636364646365636364636563646465656463656364636363636364636364636363636363646463656463636365656563636365656564646363656364636363646465636565636465656363636364636365636365646365636463646563646365636563646564636563636365646464656363656563636564646463636463646363646363636364646563636465636463636463636363646363646463636365646564636564636463636365636564636563636365636465636363646564646364636363636364636464656365636463646365646364636363636463636564656465656363636363646565656463646363636363646564636464636363636363656464646363636565636363656364646563636464636365646463646364636464636563656563656365636463646363636463646365646463636563636465656363646463656365636465636465636363646464636365636365636363646563636363646364636465636463636465636563636563656463636364636365656463646364636464636565656364646565636363636463636463646363646363646363636564636465646463636365636563656363646365656363636363636365656463646363636565636363636464646563636565646463646363656363646564646565636563646465636363636364636364656565636363636365636564636463646563656364636465646563646463636363656563636363636464656463636563636364636463636463636363636463646363636463636363636363656363636363636464636463656364646365636563636463636464636564656363636563646364636464646365636363636365656563646563636365636363636363636363646465646364636363656363636364646465636564646365636463636463636364636365636565636363636564656363636365656463636363656365636363656364646563636363646364636364656564646363636563646364646464636364636563636465636463656363656463636563636463656463656464636363646365636364646365656563646564636363646464656563646363656364636365646465656563636364656363636464646364636563636463656463656463646563646463646464636365636565646564636365656363636564636363656365636365636363636363636363656364636564656564656363636465646364646365636364646365636463646565636363646363636463636363656563636364636363646465656364636565656465646463656463636365636364636563646463636564656364646564636363656564656464646363636465646363656465646563636563636463656464636465656463656363636565636363656464636364656464646363646564636465636363636364646563636463646463636463636364646365636563636563636564636364636363636464636365646363636363636364636364636464636364656364636563656463636465646464656563636463646465646363636465636365636565646363646364636463646364636363636463646464656365636364646364656364636364636463636364656363656464656464636463636565636465656363636565636463646563656363646465636363636364636363636363646365646363656364636363636363636363636565636365646365646363636464636463636564636464646463636364646565646565636465656363636365656364656363646365646363646564656363636463656363646563646563656364636464636463636563646363636464636463646563636564636463646463646364646463636365656363636465656363646364656563656364646363636363646363646364636465646563636465646364656364636363636363656364646363636564656564646364636563646464636363636363636364646563636364656563646363656563636564636564656363636363656463656463636463656365636463636363646364636563636365656365636364656363636364636464636563636464646464636565656464636364636463656365656465636563636365636363636363636364656363636363646364646463646364646565646464646363636463646365636363636363646463646364646364646463656563636563636563646564636363636463646363636463656564646564636364636363656363646365636364656365636463656463636463646364646463636365646463656363646563656565646363636565636465636363646365636363656363646365636564646463636363636364636365636363636364636463656364636364646365646363636565636563636363636363636364636463636465646365656363636563636363646365646463656464656565636463656465636363646464646464646464646563636364656563636565646363646364656564636363636465646463636464636363636464636463636365646364656364646565656363646363636363636365636564646363636465636564636463636364636364636363636563636364636363636363636463656364636363646364646363636463636364636564636363656463656363636363636364646363636464656465646363656364646464656364636563646465656563646565656363636564636565656463646563656464636363656464646363636363636364656365656565656365646363646363646364636563646564636465656565646363636363646564656363636363656465646563636364636463656364656463646365656363646363636364636564636463636364636565636464646463646365636463646563646364646363646563636463636364656363646464636363646363636363636463636364656363636563636364636365656463636363656464636365636563646363656363636363646365636564646363636565646365636363646463636463636363636363636463646565646563636363636463656365656363636363646365646563636364646364636463636365646364656363646363656464646363636363636563656364646363636365646364636365646463636563646563646463636463646364646363646463636565646463636565646464656563636463636463646463646363636363636464636464646365636563636363646464656363636464656465656364636363636365656363636563636463656464646365646365636363646563636365636363636363636363636364646365646363646565636563656463636565636365636463646463656564656365646463646563646565636463656463636363646364636565636363636463646465636364646365646465636563656365646463636364636364646363656463646565636363656364656565636464636463656463636564636563636365636464636363636363646463636464646565636364636464656364636463636463636463636465636365636564656563656464636564646365646564656463636463636363636463656363646564636465636563636363646564636365636464646463656563656363656363636464646363636463636363646363646365656563646465636364656365636363636463646363636364656364636363656463646463646563636363636463636465636463636463636463646464656463646564646564656563636464646565636563656363636363636364636563636365646363636464636363646363636363636363636363636563656465656365646365656363636463646463636363636363636363646363636364656363656465646364656463636363656365636563646364636464646565656463646364636363656463656464636365646563646463636563636363656463636463656363636563636363636564636564646363636464646564656364636364636463636464636565656363636565656363646363646364656364636364636465636565636563646364646363656363646564636363636463636365636563636363636363636563636363636564636363646563636363646464636363646463646463636365636465636363636365646365656464636363656363646465636364656463646463656364656564636365636364636463646363636563636463646565636363646363646365636365656463656465636463646464656363636364636564636465636365636365636565636463636465636365656463656364636464646364636363646363646363656364656363636564636565636363646563646565646365646363656363656564646465646463656363646364636464646463636564646363636465646363656364646563646363646565636463636363636463656363636363636363636363636464656564646365646564636365636464646365656363656365636365656365656564636364636563636364646564636563656365656563656364646464656463646363636463646363656364636363636363656564656364636563646564636365646463646463646563636363656363646464646364656565636363646563656464646365656463656365656363646363636465636463656364646363636365646463656564646364646363636564636565636364646463656365636363656463646563636563636463646364636364646364636565636463656363636464646363636365636565646363636463636463656464636363636364656365636363636563636565646363646363636563636364636363636364636563636363636363636463636364646464636364636363646365636364636363646365646365646463646363636463646363656565636364636365636363646564656363636465646463656364636363656563636463656463636363656563646363636464646363656363646363656563646364636564636465646563656563636565636463636364636363636464636364636463636564646563656363656365636363646565646463636463656363636565646364646363636564656363656465656364636564636365656364656365636565646364636463636364646563636364656363656363656363636363636363656563646563636563636564656365656465636363636563636363636363636463636363656565656363636363646364646464636565646463646363646363636464636463656563656363656464656363646363636363656464636563646564636365636364656365636364636563636363636463656465656563656563656465636363636465636463646364636365636465636463646364636363656364636563656464636364636564636364636465636364636363656364636564636363646464646364656363636464656364636565646365636363636363636364656364636463636463636565636363636564636363636364646464636364636363646464656363656464656464646363656364636364636463646464636363656463656563636363646464636564636363656363656363646363656363636465636363656463636363646363636364636365636364636363636365656563636365636563636363636463636365656363636563636463646363636563636365636364646565636463656364656365646563656365636363636363646365656563636365646463656364636363646463656563656463656365646464636463636464636465636364656564646463636463636463656363636363636465636463656363636463656465636364636364656363646364636463636563656563646463636464646463636363646563636364656463636363656563646363636363636363636465636463636463656363646363656365656363646365636363646463646463636363636363656364656564656364646363636363636363636563656365656564636363636363656363636364646464656463646465636463656463636364656564636365636365636463636465646365646363636564636563646364636463636465636463656363646563636464636365656363656365636363636463636563636363646563656463636363636365646565636364636363656363636364646463656564656463656364656364636365656463636364636463646363646465636463636463636463656364636364646564656365636463636364636463636363646363646463646564636365656364646363636364636464656563656565636364656364636365646363656364656465656563636363636463646463636363646563656463636363646364636363636564636364646363646363636363646363656363636464636463636565646365636463646363656563636563646563656564646363636463646365636565656363646363636363646463636363646564656364656364646364636363656464636463636564636364646464636465636363636363636364636463636465636564636363636365656365636364636363636464656463646563636364636365646363646463636363636465646363636563646565646464636365646563646463646363646463636364656564646463646364636465636364636364636363656464636365636364636363646463636463636563646364636564646563636463646363636463636365656364636364636364636364646365646564636364636364656565636363656463656465636363636363656465656463636465646363656363646365646463636363636463636464636364636363636463646563656464636565646563636364646465646464636363656463636565636463646464636565646465656363636464656563636363646464636564656463636365646463646464636563636564636564656365636363656463646564636364636364656563636363636465636463636564636463636565636363656565636465636363646563656363636364636464636363656364636463636464636463656363646363636363636463636363636363636463656363636465646363646365636363656363646463646564636463636463656564636463646363636464636363646463636463656465636563656363636563646565646363646365636464636363646363656563636363636363636363646463656364646565656464636463636563646565636363636463636564656564646365636363656363646363636463646363646365636464636565646363646463646465646365656565636463646464646365636463636463646564636363646463646365636365636363646464636463656464636363636565656465636464636365636364646363636463636363636463646464646365646364636464636563646464656463636363636363636463656463636363646563636363636364646463656365636463636564636563646363656563636564646463646364636363656364646364656363636463636463636464636364646363636564656564646464646463636564656365636364636363636463656463636563636364636463636363636364656563646365656363636363636363636563656363636364636364646563636563646363656364636365636463636565636364636363636364636364646465656564636463636363636363636464656463636363646563636463656365636563636465656365636563656563636565636365646365656465636365636363636363636463646564656564656565646463646463636465636363636363646363656465636564636563636364646363636363646363646463636363636364636564636463636364636364636363636563656465636563636563656564636463636363646565636363646463636364656365646363656563646364636363636564646363656363656364636363656463636465656365656365646463636564636464636465656463656563646363636463646365636364636364636463636365636463656464646364636365636364636563636365636363636564636464636463636564646365636565636563636465636363636365636564656365646364636363636363636465646463646464636363636364636464636463656364656363656363646463656364656464646464646363656463656364636563636563636364636363656465636363646364636463646564656563636465646563656364636463636363646363636363656563656465636464636363646564646565656464646364636564636364636363636363636363636363636363636363636363656364646463636364656564636363636563646363656363636363636363636565636463636363646364636463656564636365636363646563636363636563646363636563646465636363646363646363656365636463656564636363656363646563636563636563636463636364656564636564646563636464646563646564656363636364636564636363656465656463656563636363636463646364636365636464656363636564646564636563656463646363636365646463636563636463636565636363636363636364636563636463646363646463636564636563656463646563656364656464646363656463636563656364656563636464636363646564636463646363636364656363646563636363636465656464636565646463636363646365636363636465656465636365656563646463636363646364636465646363636364636364636563636365636563656563646463636364646363636364636363646463636563646364636565636464636364636364656363636363656565656565646464646463636465656365636363636364646565636364656365656363656365646365636363636464636364646363636463636463636563656463656563636463646364656363646465636364656364636563656363636563636363636564646463656364656463636365636363636464636364656463636563636365646365636365636465656564636364636464646463656563646563636363636365656564656363636363646364656463636364636463636363636563636363656363636464636463636364636463636363636563636563636565646363636564646363636465636363656363636363636363636465636563646364636364636463646463636563636364646363636364646565636565646364636463656563646564646463636363646463636564636364646364656365656463656364646564646363646364636363646465646564636565636463646463646364656463636365636463646463656565636364636563636463646364636563636363636365646563636565646365636463636365636465656363636563656465636563646564646364656365656365656465636364636363636463646463656563636463646364636465636463656364646563656363636363636363636464656565636463646565656563656365636465636364646564646463656363636463656563636565636364636565646463636465636364646365646563656363656363636463646365636364636564656364636463636565656563636365656565636464636363646363636363646463656563646363646464636464646563646363656564646463636363636364636363636563646563636363656364636365656463646363646363646364646363646364636565646563636564636565636363656364656464636365646465656363646565656363646363636365636363656564656364656464636364636363656364646563636463646463636463636364636364636564636363636363636565646363656363636463656363656364656464656565636565636363636464636463646363636363646565656563646363636364656364646365656363636365636363646365636365636365636364636465636565656363656463636463636564646363646465636363656463646363636463636464656464646464656364636464636463656363636465636363636463646563636464646365636563646564646463646563636365656363636564646464636464636464636564636363636363636363656363636564636364636464656563636564656364636363636365636364646363636464636464636364636363636364656364646463636363646363636463636464636463656364636565636564656565646463636364646564646364646363636465656365656465646564646364646364646463636365636365646463636363636465636465656365636364656463636464636565646564656363636365646363636364636465646464636363656464646364636365656365636363636365646363646363646365646365656363636363646363646364636463636364636463646364636563656363646363656363656364636363636463636463656363636363656463656563656363636365646563636363636463656364636565636364636363646364656464636363636565656463646364636363656463636363646363646365636364636363636363656464636465646364636365656463636363636365636363656364656465646564646363636364646363646464656563646365636563646464636363636363636364636363636563646363636565636365636563646464636563636463656363636463636363646364636463636463636363646363636364636465656563646364636365636363656364636465636363636363656464636364636365636563636363636364656363646463636463656463646364636364646365656363636363646365646463646464636364636364636363636363656463646363636363636463646365636363646463656463636564646363656363636464636363646365636564636364656563636564636463646463636463636363636363636465636463646365636363636564636363646365646364636463636465636363636363636465636463636363656364646363656464656463636364656465636464646465646564636464636563636563656465656365646463636365636363636363646364636363636463646364636363636564636463656463656364646363636465636563636563646464656364656364646363646363636363646363646363646463636365646363646365636564636363646464646363636463636465636363656565636363646363646563656364636463656564636563636364636464636463656363646463646464646363636465656564656564646464636464636465636363656363646564636465656365636463646463656464646563646565636363636365646565656365636463646564636564656363656463656464636363636363646363656563636463646363646463656363636464636563646363636463636563636564636363656365656464636365646463646363636565636464656464636464646363636364636565646365646365656365656565636363646363646563636364636364636364656364636363646364636365636463636363636563646563636365646363646363656563636365646463646463636463656365636463636464646463636363656463636364656563646563636464636465636363646365636364656464636364636364646563646463656464656365646564646564636463656364636463636564656365636463636564636363636363646363636464636365636563636463656365636463636363636364636363636463636365636363636364646463636365646363636364636363636363636363636363646363646363636364636365646363636363636463636365656564656365636464646363636365656563636464646365646464636563636464656365646364656463646463636365636465646463656563656363636363636463646363646565646563656363656464646363646464646563636463636565646364636464646463646463656364646564656463656563636363656464656363636463636363636463646363646363636365646563646363636563656363646463646363636564646363636564636363646463646363636363646364636364656363646563636365636363646463656463646364656365656365646363636363636563636363646363646463636363646464636463646365636364646363636463646563646364646364636364646363636363636365636464656364636565636563636463656363636463636564636365636563636363646365646363636563636363656363646363646563636363636465656463646564636463656564636365636463636563636363636363636363646364636465636363636465636363646363636363656563636363636463646364636463636363636463636363656563656363636365646565646364656363656364646563636465646363636465636365636363636464646364636563656463656463636463646365636463636464656465636363636365636364636464636364656364656464636464636364636463656363636363656365646465636363636563636363646563646364636364656363656364636564636463646465646365656364646363656563636363636463636465656565646463646563656465636363636363656564656365656363636464636365636565656563656563636365646464636465636363636564636363656464646364636565646463636363656363646563656365646463636565636564636463636365646463646364656364646564656463636364636464656363636463636363656565656365656363636364636363636463646364656565656563656465656365646563636364636363656564636464656363636465636364636364656363646363646464656464636463656363636464656463636363646363636565636463646465656363646463636463636465646363636363636363646365656363636363636463646464636463646464656365656464646365656463656364656363646365636365656363636363636365656364656363636365636364636363636365656565646365646463636464656363636465636464646464656564646563646364636564646464656364636365656365636563636563646463656463636564636364646563636363636464636363646464646365656563656365636363656464656463636365646364636365636465646463636463646365646363636365636363636363646365646363636364646563646364636464636463656463636463646563646564646463636365636363636365636364656363636463646464656363646364636363656465646463646363636363636565646364656363636363656363636464656363636363636465656463636464636464636564636465636363636463646364636463636563636563656565636563636463636563646563636465646365636563636465636564636365636365636463646363636365636363636363636363636364636463636563646565656463636364656464646463636364646464646363636465636464636464636463646364646463636363636363646563636465636365656563636363636363636463656365636463636464636363636365636563636463646464636365646463636563646363636363636363656363646465636365646363646564636465646563636563636464656363656365646463636563636463636564646563646363656365646465646465636565646563656363636363636464646365656365636364646364636363646365636364636365646363646365636364636363656365636463646563636364636364636464656565636365646363656463656364636363646563636463646364636463656463636464646563636364656364646363636465636364656363636563636364636364646463636563636563646363646463656465636563636363656363636364636365636563646363646364646363636463656564636464636365636563636365656464656565636363636464646564646365646363636363636565646463636563636463646464646363636463656363636364636563646464646465646364636564646364636363636365636363636463636563646464636563636564646463656564646363656463636564636364656563636365636563636465636363646365646363636363646363636364646463636563636563636463646365646463636365636365656365656464646563646563656363636363646363636465636363646363636365636363636364636463636364636564656364646363636363636363656363636464636363636364646363636463636464636465636363636563646463636564656463646365656363656364646363636464636364636465636363656463656464646363636363646363636365636563656565646363636364646364636363656364636463636363646365636363656363646463636363636563646563636363646463646363636463656463646364636363636363646463636364656563636463636364646463656363636463636363656463636363646363646463646363636364636463636464636365656463646364646363636363636364636464636364636464636464656363656363636365646563646463656463636363656463646363646563656365636463646463656363636563636464646363636365636363636464636463636563636465656365636564636463636365656463636363636563646363636463636364646463636363646364636563636364636564636364636365636465656363656363646565636463646363646363636565646364636363646463656365636363656364646465636364636363646363636363646463636464636565646363636564636364646465646464646563636365636563636565636463636465646463656463636564646365646364636463656363656465646464636365636464636364636364636364636363636365646463646465636364636363636563646363656463636363636565646365656363646563646463636463656563656364646363646464646364636564656564656464646463636565636463636364636364636364636464636564636363636463636564636363636463636364636565636364656364636563646563646465636463636565646363646563656563636364646365646363646364656365646363656565636463636463656363656363636463646465646364646465636563646463656564646565636563656464646463636364646363636363646464636563636363636365636464636363656564636464656364636363636363636465636364636363646563646563636463636364636363646365636464646365636364636464646365646364646464636365656364646363636363636464646364656465636463636364636364646563646364646364636365636363636364636364646364646363656364646564646363636363636365656364636365636563646363636364636464656363636563646463646463646463636464636363656463656365636463636563636565636563636363636365636463646563656363646364646363636565636363656365646565646364646364636365636364636364636463636564636363636364636563636365646563656565636365636564636465636365636565636563636463636563646363636364636364656464646363656464646365636365646564636564636563636363636465646463636563656365636363646465636365636364636463656364636363656365646564636364656363656365656363656463656363656563636365656364636363636365646464656363636364636563646363646464636463636564636563636464636464646364636463656363646365646463646464646565636463646563636563636363636365636363646563656563636564646463636363646564646363646564636463646365636365636365636364646465646465656363636563636463656463636464636565656363636363636463636363636465656464636463636364636364636364646464636563656563636465636365656365646364636464636365636364646465636363636563636563636365636463636365636463646363656364636364636363636363636363636463636463636564646464656563646563646363636465646363646363636363636363636364636563636364636463646363656463646364656463656463646363646563646364646365636365646563646564656363636363646563636364636364646464636363656463656464636363646364636363646465656364636365656363656365636363636363646563646365636563646363656464656564656364646563636463636463636363656463656564636464656363636463636463636364636565636363636464656563656463636463636363636563636463636364646465646465656363646363646365656464636363636563646365636364636363636363636463636363636463646465646565636363646365646363646365636463636365656364646363636364636563636365636465656464636364646463636363636364636563646364636363656363656564656364636363636364656463636364646465646365636363646463636565646463656363646564636563646464646364636465636565656565636463646464646363636563636564646363656365636465656363646365636363636563646563656564646463636463636363636365656465656363646363656464636463636465646363656363646463636364636463636463636464646463646363656363636365636565656464636563636563646363656363646563636365656363656565636463646365646363636563656364636363636564646364656463636563656365646465636563636464636363636563636364636463656363636365636364646364656563646565636464636363636363636365636463646364646365646464636363646363636464636363636463646364636563656463656363636363636364656363636563646463656565646364636365646363636364636463656563636363646464656365656365636364636363636464646363636565656465646364646564636564646364646464636465656363636363636463646364656365636364636364656364646364646463646464656363636564636465636563646465636363656463636563636364656363656364636364636464646363636363646365636464646364646563646565636464646363646564646365636563656363646463646363646363646564636363646364646464636465636365636564656463636363636364646365646363636564636464646363646363636565656465636463636365646364636463636563636465646364636364656364656463646364646565636365636363656364636463656463636363646563636464656364656464636364646363646563646364646364646463646365646363646464636363656365636463656563646363646363636465656365636465636563646463636465656363646365636565656463636363656463646465656363636363636363656364656564646363646463646464636365646363636364636465646463636365636363646564636363646564646565636363636463636363636463656363656564656565646463636363636364636363636464656363636363636364636563636463646463656463646564646363646365636364646363656463636564636464636365656363636464646565636464636363646563636363646564646364636363636464636563636463636463636365636563646563636463646363656364646563636365646363646463646463636463636465656463636365646363636565656363636364636463646363636463656365636565636563636365656464646565636364636465636463656463646464636363656463636364636363636365656564636363636564646363636463656465656464636363656565646565636363656363636365656465646463636463636564656564636464636365636363636363646565656463646365636563636364656364636465636363636563636364636365636365636365636565656564636463636364636464646564646565646465636463646363646363636563646363656463646465646364636364636363646565636463646564656363656465636565636564636364636363646563656363646363646565646364656564636364636465636563646364636465636563646563656363636563646364646565636563646363636364636464646363636363636363636464656464636363646364656465646463656365656363646464636563656563656364656364636464646564646565646563636463636363656465636463646564646364636463636563656463646364636365636463636363636465636565636363646563636565656364646465656364636563636563636364646463646564636563636363636563646563636364636464656365636365636463636364646464636363646363636563646363636463636564636365636363636365636464636463656365636365636363636563636363656563636363646564646363656463646364636465656565656363636363646364636363636564636565646463636364636364656364636363656564656363636463636463636464636365636563646365646364656363636463636364656365646363636463636363636565636364646364636464656364636563636564656463656563636565656463656464636364636363656363656563636563636364646365646563636363636363636363646364646365636464636363636564646364656363636464636364636463646464646365656363646464646463656364636363636363636363636364646564646463646463636365646563656363636363636364656363656563636364646364656365636363646464656464656465636363656363636464636565636463636365636463646564656365636463636363646363646463636563636364656563656364636363636463656363656465656363646364636363636363636565656463636364636564636363636463656363646365646565656565636363656564636465646364636363636363646363636365636464646465646463636463646364646365636565636565646363646363636363656365646365636363636464636363636365656363636563636463656564636364636365646363646364636564646363636563636363646365636463636364636364636363646464636464636364646363636463646463636364656364646463636363636365656363646463636564646563646364636364646465636465636363646363636365636363636463636463636363656563636363636565636465646464656465646364646463646463636363656363636465646363646364636463636363636364636464656364636364646365656363636464646463646463636365646363636465646365636464636364646463656364636363636363656363636363636363636364646463636565636463646363636363636364646363646363646563656363636363646464636364636363636363656463646565646463646363636363636364646463646363656364646563636364636363636365656363646563656364636363646364656563656563636363646463636363636363656463636563646364646464656363646464646465646365636465636364646363656463646564646365646365636365636465636465646565636464656364636363636363646464636564646564636465646465636363636365656364636564636364636463646464646364636365646364646363636563636565656364636363656465636463656563646363656363636364636363646363646464646363636464636363636363636364646463636563636363636363636464636364646364656465636365636565646363646363646363636563656365656365636563646565646564646564646363646363636463636363656464646564636564646363656564636363636465636463636365636364636563636563636364636364646363646363636565636363636363646363656364636563656464646465656364636463636564646564646363656563636364636363636564646363636364636363646463636364636464646464646564636463636363656365636463636465656365636464656363636464636365636365656365636364646463656364656364646463656363656365646463646363636463636364656463646465636365646363656363656464656365656363636364636364636363646363636464636463646465646363636464636365636463636363646565646364656364646363636363646365636363636364656463646463636465636563636563636365636365636463636464636365636363636365636363636464636463636463656363636463636463636563636363636464646364656363646564646364656363636363636364656463636363636365636363646464646563646364646463646363636364636465636364646465636365636363636363656464646463646364636464636363636463646563636463636463656463636565646364646363646563646364646465646365636365646565656464636363636363646464636363636364646363646465656363636564656363636365636364636363636465636465646464646465636463636363636364636463636463646564646363646363646364636464636463656564646464646464636464636364636365646365636363656563636364656363636463646463636463656363656364636365636363646365636564646565646363646364656364646465656365646563646463656464636363646365636463646364646363636563656363636363636565656464646463646563636363636363636364636364636463636365636363636364646364636565636365646465646364636564646363636363646564636365636364636363646364656363636365656565636363636364656365656365636463656365636463636464656363646563636364636565646365646564646363646464646364636364656463646365636364656363636564646363656465646563636365636363646364636563646364646463656465656364636463656465636465636563646365656364656363646363656364646365636363656363646363646363656363646463636365636564636464646464636563656363636463636363656564646364656463646565636463636363656364636464636364636563636463656364646463636464636463636364656363646564636563636565656465656365656364636463636463636563636363636464636564656363646463636364636364636564646564636364636564636463646365646363636563646365636364636364636463646563646563636363636563656364636363656463646363636564656465636563656464636364646465636364646463636463636463636364636463646563636363646364636365636463636563656463636563636364646563656463656465636363636364636363646463636463646364636465636363636364636563636365636363636464636363646565646363656463646463636563636563646563636363646564646565636363636563636363636565636563636363636464646363656565636363636463656363636363636463636463646465646564636363656463636364636364656364656463646464656563656363656363636363636364656363636464636363646364636564646363636465636363636364636364656563636365636565646364636463636363646363646563636464646463646563636464636365646463656464636363656463636363636363656465636364636364636465636464646564656564636363656363636465636464646463646463656463646465646463636464636363656463636463646565636563656363636565636365636363636363636464636364646563646364656465636563656363646363656463646463636364646363646364636465636365636364656563636463636564636363646564646363636563646463656363636364646363636463636363656363646365636365636364646365646363636564646363636363646363636363636365656564636363636364656465636565636563636563646364636364636364636363636365646364646364636464636365646463636363636563636363646465636363636563636463656465636463636365636363636364636463636463636363646363646565656463636363636463646463636363656565646463636464636564636364636565636363656463646363656363636363636363636363636363636565636464636464636564646363646464636465636365636365636364656364646465636363656363636465636464636363636364636464636363636365636463646465646364636563636364636363636364636363636463636364636363646365656364636563636363646565636564636464656463636563636365646563656465636364646464636464636363646565656463636363646563636363636363646463636463646363656364636363646464636363636465646364636564656563636564646365656563636365636564636565646463646363646364636463646463636465636565656363636363636365646464636563646465636463656563646564636364636363646465636563646463636464656464646565646365636365636463636463646363646464636363636564636363636363636364656364636364646364646463656363636363646363656363646363656364636363636463636363636464656465636364636365646363636363646465646463646363636565636465646463656364646463656364656563646563646564636464636563636464636465636365636365646563646364636363636363636364646565656465636463636463636563646563656563636563656364646365646463646363636365646564656464646464656365636464636365656563636364646563656363636463646465636363646363656465636363636364646363636365636465636463636463646463646364646565636365636363636364636363656365636464636563656363636564636365636464636365656563636464636363636465656463656564646364646364656363636563636364636463646463636463636363646463636364646365636464656565636363656463646363646363656564636363646365636363656463646465636465636363646463636364646463656565636364636364656463636363636363656364656465636563636363646463636364636464636365646365646363636364646563636463636363656363636565646363646565646363646363646463636464646363656463656463636363656563646464656463636564656463646565646363636364636463636465646463646363636564646363636463646464646363646563636463646565636364656563636464636463646463636363646563636364656463646365636564646364646463636365646563636364636465646463656464656363636463646563636363636363636464656363656563636464636464656463636563636364636365636365636363656465646363646564636565636365646465636364636464646365636464636465636365656463636564636364636563656363636363656363646365646364636365636464656564636363646364636563656363636465636363636364656363636464636363636365636364636363646463646463636363636463636363636363636564636364636465646564636464636364636464646564646364646465636365636463636565646363656365656463656563636464636364646365656363636364656465636363646363636563636464636463656363636365646565636364646464636463646363636365636365636564636364646463646363646363656465636363656565636463646564646463636364656365636365646464646465646465636463636363636363636364636364656363636363646563646563656565636364656365656364636563636463646463646565636365646365656363646463656363656364656464646465636363646364636463636363636564636363646465636363656363656363646565636363636363636365636364646363656363646365636463646364636363646565636363636465656364656364646365636363636363656563646563636463646464646564646364636363656365646365636563636463636563646365636565656564656563636363646464636563656365656563636463656363636565636464646363646563636463636365636364636363636364636363636465636463646364636365636363636365636364656563656364636364656364646363656563636364636364636565646364646463636464646463646364646364646364656363636364636364636364636363636464636565656463636363636463636365636565636364656365636363636365636565646363646364636364636563646563636564646364656565636463636563656565656363656565646463636565646564646563636365656563636565656563646365636565656363646363646463656363646363646364636365656563636363656364636563656364646364636463636364636363636563656365636363646363636465646463636563636463636465636365636563646364646463646464636363636563636464636564646363646363656463636463636363656365636464646564636363656463646363636464636463636363656465636563656364656564646565636563636364656565636563636463656364636364636363636363646365636364636465636363636363636465656364636363646363656363636364636465636464636463646363636364636363636364636364636464636365656363656365636463646463656364646365636363636364636463636463646363636363656364656363646564636364646563646365636563656365656463656363636564636463636364656464646464636364636365636363646363656464646464636563636564636563656465646363636363656563646364636363646363636463636363636363636365646365636463636363656363656563636363636365646363646564646463636463656364656563646364646364646564636565636365636363636463636364656363656464646565636464636365636565646464636463656464636463656463636463646363646363636363656563646463636363636365646465646363636365636364636363636364646463646563636565636363646465636363656465636365636364646465636564646364636464636564636563636365636564646564636365646363656464646563636464656465646365646563636364636465636363636365636463636463656363646363656363636463636463636363636364656563636363636364636564636364656564636363636563646463636464646364636565636365646465656564636463646564636463646363636363646564646463646365646463636463656463636365636364636364656363636363646363646365636565636363646365636563646364636564656463636363636563656363636565646363636465646564636463656464636363646563636563636463656363636464646563636464646463646364646465636463636463636563646364656464656364656463636564646363636463636464636365646363646365656563636364656464636363636565656363656463636563636364656363636364646363656363656363636364646463636363636365636463646364636464636364636364636363636364636364636363636363646363646563646464646365656563646463636563656363656363656463636363656365636564636563636363656563656365636363636364656363636563636363646365636363656564646464636363646365636565656364646363646365646364636564656363646363646364646364636364646464636363646465636464646363636565646363656365656565656364646363636363656464636363646563656363636363636563656565646465656363646565636363656363656364636365636463646364636363636363656364636364636363656365656464656363656363636364636365636463646563636365636464646363636565636564636563636363636364646365636364636363636364636563636363646463656463656463636463636565656563646364636463636364636463636363656463656365636465646465656363636564656464636363636464636463646563636363636364636364646464636463636364636365646363646363636364636365636563636365636363656365636365646563636465646463646363646463636564646464656564636463646364636464646564646363636565656364656363646365656564636363656364636364636563636364636363646363636365636563646363646363646565646363636365646564646363656464636463636565656365646465636364646463656364656564636463636363656363636365636464656364656563646565636365656465656463646464646563656364656563636463636363636364646363636364636564646364636363656563656363646464636363636365646564636364636363646563636565636364656365636364646463656364636463636464646465656363636463636464636563656563636463646363636563646464656463636363636365636365636463636463636364636564656465636363636363636463636463656464646364636363636463636364656464636463656465646363636363646363646563656363636363656463636463636464656365636363636364636363646464656365636563636363636463636363636463646464636364656464646565646463636465656363646364636363656465646363636463636363636364636564646364646363636364656364656565636363646363636363646363636364646364636463656463636563636363636365636363646364636363656463656464636563646365636463636364636563656564646563636463646365646363636463636465636465636564636463636564636464636465656563646465646565636365636563636363636463636463636463646565646463656563636565646463656565636463636363636363656363636463636364636363636563636363636464636365636363636465656364636563636464646465646365636363636364646463636363636364636463636464646363636564636363656364646465636563656563656363636365636365636363646365636364636363636464656563636363646363636463636363636464636463656464646364656463636564646463656363636364646365646463646463656363636365656564636363656564636463646363636363636564656363636365636563636563646465636464636363646564636363636463636364636365636363636363636563646465656363646563656365636465636463636365636463636365636464646363636364636364636363656563646364636464636364646464646564646465656563646365636463656365646563656363646464636363636363636365646364646563646463636363636563656563646564636464636363636563646465636365636364636464636464646463636363656364646363636564656563636363636363636565646564636363636463646363636363646564646363656363646364636365646364656363646365646363636365636364656565636565636464636463646465636463636563636465636563636363636363636565646363656464636463656365646363636463656464646563636563636465636463646465646363656564646364656564636365656363636565636365636563636563636463656363636464636363636365646463646465636563656363656365636464646463656463636563636364656365636464636463646464636364646564656364656364636463636564636464646564636365636463656463636364636363646364656365636363636364636363656364636363656463646463636364636364636364646363636563636463646364656363636464636363636363646364636563636363646365646463646564636364656363646463646364636563646565646564646363656365646563636463636363636563646464646565636565646363636563636364636465636363646464636465636463636363656463656563636563636364636463636363636465656465646564636363646364656463656363636363636563646364636363656463636564636363636464636465636465656464636364636563636565646363656463656463656465656463636563656464636365656363636463636363636363636463656363636364646465636365636563636464646465636364636563646363636464646363636563636464636565646464646364646364646364636463636363646563656363636363656364646563656364636563636465636364646363636365636563656365636565636464636363636363636564646363646365636464646365646363646363636363656363656365636365646463646363636364636463646363656364646563646363636364646363646364646363636364646463636363636365656363656463636363636363646565636364636463646365646464656363636363636363636363646363636365636363636465646363646363636463646563646563636363636363636364646363636564656463646464636463656364646465636365636363636363646363636465636363656363636563656564636565646365636564656464636463646363656363646365636363656464636363656464656364656363636463646363636363636363636364636463646363636365636363656564636365636464636364636363636363636463646363656565646363646363646363646464656465636363636563636565636563636365636365636364646364636563656364656465656463656363636364656365636363646564646365636363646363646364646365646463656563656364656364636464636463656563646363656464656465636465656365636463646363636365646365636363646463636464636364636564636364646463636365656363636365636364636364636563636363636365646463646563636565646363656563636364636363646363656363636465636365656463636464656363646363646364646564646364636364646363636563636363636563636364646463646564636563656564636365636363636363646463646365646565636563636363636463636463636365636464646563646563636563636363636363636564656464646563636364636364636463656363636565656563636464636564636463636364636364636364636365656363636464636364636363636363646465656563636463636365636363646563636364656363646365636363636463646564646365646363646363646364636365636363636464636564646363636464646464636364646364656563646464646463656364646465636365646363636363646363636364646363646364656464636464646363656363646463646463636565636564636464646363636463636465646365636363636463636563636563646363656363656464636565636364646365646363636463656363646563636364636363636564646363636365646363656463646563646364646364636564636463636363636563656365646363636463636363646364646363646363636563636363656465636363636463636465646365646364636363656363636363636364646463636363636363656363646363656464656364656365636364636363656463656364646463646564636365636364636565636563646565636365636364636463636464646365636363646563636364646564636463656563636463646365656364636363636465636463646365636363646363636463646364636463636563656563636363646364646365636564656363636365636563636364646564636363636563646364656464656365656363646565636564646364656364646363646463636365636464636463636465646565656365636364646564656365636563636365656363646364636465646364636364636365656363636363636464656463636363636464646363636363646365636463646463646363636463636363636365636363636463636463636465636363646363636565636363656464636564646564636363656465646463646364636563656565636563636464636465646464636363646363646364656365636363646365636464646364636463646563656465636465636365636364646465646364636463636563636363636364636563636463646365636365636363636363656365636365646463646364656463656463636464646363646365636564646364636364646363646564646464656563636363636363636363636363636464646565636363636463636464656365646363636365636364656363636565636563636465656364636464636365646565656564636363636563636363646363656565636363636564656363656363646463646465636465646463636464636363636463656363646364636463646463656365636365656565636363656565646363636563636463646364646365636565636363636364656463656463656563656363636464636364646565656365656464636363646363646363646363646463636464636365646365656465646363656563646365646563656364636463636364636365636563636363646364636364646564646363636365646363636363656363656363636363636564646363656364656563646565656364636565636365636464646463636363636363636363646465656563646363636464646363656363636465636364656365636364646363646365646564646364636364636565636363636363656564646363636363656463646364636463636463646365656363636563636563656364656565636363636363646363656464636464646464636363636363636363636563636363656363646363636464656563656565646564636564636363656363646465646563636564656563646365636563636363646464636464636565636365636363646363636363636365636463646563646364636365646364636363656363636363636364646363656363636465636465646465656463636364636364636364636564646463646363646363636463636463646363656363636363636464656463646463636363656463646365636363656364636363656363636463636363636363636563646365636364636564646464636565656563656465636463636463636463636464646365636364646365646364636564636464646463656463636364636363636364636364646363646564656563646563646363636563636363646363646363646563636463656563636363646363636364636463636463646463646565656364646363646363636365656363636565656365656464636563646464636364636364636364636364656464656565646363636364646363656363646364636363636463646563636363646563656365636365646563636563636363646363646363646564656563656463636364656463646363646363646365636564636463636464636463646365646463646363656563656463636564646465636464636463646365646363656564646363646563656464636365646363656363636363656363646363636563636565646363636363636363636364656563636565656365646563656363636365636463636465636363636463636563636463636365636363636464636364646463646365636465656363636563646565646365636563656464636563656465656365636563636463656365646563646363636463656363636363656363646463646565646363646364646564646564636364646363636363646363646465656363636363636365636563646363636363656464636365646564646365636364636363636564646564656465636464646363646565636465636564636463646363656363636464646365636464636364636363646364646464636364646364646465636363646463646464646464636563636464636364636363656364656365656363636365646364656463656363636563636363636563636464636465636364646565636563656463636463636363646363636363656363646363636364636364656463646363636464646464646464636364656363656565646463646563646363636563646365646363636363636463636363636365636363636363656363636364636364656464656464636364636463636363646363636463636564646363636564646563636463636463636463656365636464656463636463656465646363636563646363636564656564636463636364636465656463636463636564646464646364656465636365646363646564636363636363636563646363656464636363636563636463636463656363636363636563636463636565656364656463636564656563646463656463636563656364656363636464636463636363646363636563636364636463636364656463636363636563646563656363636363636565656365656564646565646365636365646463646365646364646563636363646363636363636465656563646463636363636463646463636465636563636563656364636463636364636563646365636363636465646364646563646365636465646465646365636364636465636463646563636465636463636365646564636563656464636363636363646365636364636464646363656463636464636464636563656565636363636563646564646365656363636363656463656363646365646465646363636363656365656464646364636563636363646363636465646563636365646463656363646464646365646364656463636364656463656465636363636363646463646565646363636464656563646365646463636365636463656463636363636563646363656565636464636465636363636365636463636364646365646365656364636563656463636464636364646565646365656364646464636365646465636463636463646363636364656463646463646464636463646565636363646364636363636464636565636365646463656364636464636463656363636365636364636364656364646363646464636365656364646363656464636463646463636564636363656363636565636564636464636364636363646465636464636463636564636363636563656364636563636464636364656464636564646565636363636363656364636364636363636364636363636363646463636365646363636463636465656363636363636364646363636364636563636465636363646364636363656465646463646563636365636563636363646463656365636563636364646463636363656563636363646363636363646364646464656363636464636465636364636364646564656563656365656463636365656364636364636363636364636363636364656363636464646365646364636365636365646364636565656363636463636565646563636363646464636464656463646564656365636363646364636363656563636463636364636364636465646463636363646363636363636363636463636363636463656365656363646565636364646563656563636363636564646563646363636564646363636363646364636563636363646364636365636363646364636464646364636365636464636563656565636564646363646563636465656363636465646364656363636363646363656365656463636463656464636364636364636363636365646364636463646363636463646565656465646563636364656463636363656363646365656564656364636365656465646563656563656363656365646365646463646463646363636363646363636365646363636363646365636564636465656363646463646364636365636565636564636463656363646363646364636464656463646363636563636365646464636364636363656363656363656364636364646363636363636465656363646364636563636463646364656365646363646363646464636464656564636565636565636463636465636364646363646363656364656464636564646364646363636363636363636364646363636364636364636365636364646463636363646364636365636363646465636463646465636364636564646363656463636363636364636364646463646365656363636364636563636363646364636563656365636563636364636463636465636364636564636463646463636363636363646464646363646565636363636564636364646363636563636363656363636564636364636463656365636364646363646464636565656363646365636464646364646564646565636364646563636363646463646465646363646363656365636363636364636363636564636364656465646364656564636364656464646364636563656365636364646463636563646563636463646363656365636363656363636464646363646464636465646463636364636463646464636363646365646564646363636363656463636363636563636463656363636565646363646365646464636363646363636565646363646364636365636364646563646363636563636463646363636363656363646563636463656463646565636465636464636464636564636464636365656464646364656565636464646363646463636365636564636463646463636365636465646563646563656364646363656564636365646563646463636465656465656365636363636364646364656463656363656364636363636363656563636365636363636463656363636563636563636563636363646463636463636563636363656364636363646363636363636463636364646463636465646465656463636363636463656463656564636463636463656365646363636364636363646363646363636363636364636365646364646564646463646364646463636564646363636363656563646464656564656363636465636464636463636463636564646363636365636365636563646364646363636364636463656465636364646363646363646363646363656463636364646363656365636363646465636463646565636364656563646365656363656563636365636363656364656464636363646363636363646363656464646563646364656563636363656564636464646463646363636364636565636565636464636363656364636363636365636465646363636563636563646464636563656565656465636564636565656464636364646363636363646363656364636364636365646463646464646365636565636365636363636465636564636565656463656363636364646564656463646363636463656363646563656364636465646363646563636363636464636465636365636463646564646365656463656363636363646363646463636363656463636363636363656364636363646464646463636363646363656365636464646364636464636563646465636464636463636364636564636563656565636363636565636465636364646363646463646463636563636363636364656565646365636463636363656364636363636364646363646564656564656363636364656364636565636564636363656564636563646463656365656363636563636463646463636363646363636364636564636463636363636363636363636364656564636563646363636464636564646563636463636364646364636363656363636463636465636463656564636363636364646565636464636464636363636464636464636463636564636365636564636364636465636363656364656464636363636464656363636363636365646363636563646363656565636363636464636464636363646364636564656364636364636463636363636365636364656363656363636365636364636365636564646363636363646363646363636363636364646363636365646365656464636464636463636363636363646364636363636363646463646364636363646363636563636564656463646364636363656465636363646565636463646465636363636463646363636363636364646363636463636463646364646464646363636563636563636364636463646463636563656365656463656563646463636363636465636364646563636364636563636364636564636364636364646365636463636363656564636364656565646363636463646463636363656465656563636463636463636364636465646363636564646364636364636365646563636364636465636463636363636564636364636563636363636465646365636363636364656365636564656364636463656463636464646365636563636363636365636365636364656363646363636464646363646563646565656363636464636563656464656363646363646364636363636363636565636464636364636363636363646363636364636465636465656364636463656364636463636363636363636365656364656564646365656565636364636563646363656363656364646365646365646364636363646364636463656464646363646363646364656364656364636465636363636363656563656463636364656565646363636463636363636363636465636364656564656465656363636563656463636363636364636363636364636363646363636463636365636364646364636464636363636563646365636463646363636363636365636464636363636565636363656363636563646364636463646364646563646365636465646363636464656563636563636463636365656363646564646365656365636465636364636463656464656563646363636363646365636364636363646364656364656364636364646464636365636363646563636363636364656363636363646363646565636363656564646464636563656363636363636565646563636463636465636363656363646363656463646364646363636364636365656465656365646364656564636463656363646364636463656363636564636563646465636363636563636565636364636463656363636365646464636364636565636464646363636563646363646363636363636465656363656463656465636563636364636365656465636565656365636363636463636463636363656463636364646464656463636465636363646563636463656363646465636563656363636363636564656465646565656363636465636363646363636364636363656363646363656463646363646365636464656363646365636565636363646464646364656463636364636563646365656363656363636564656364636365636365636364636363636363636363646563636364656363636363636363636465636563636363646363636365636364636364636364646564636565646363636464636463656463636564636365636363636365646363646563656564636463636363636465636364636465646364636365636564636465636564646564646564646464636465656465646464646565636464646563646363646463636363646363656363656464646365656463646563636564646363646563646564636365636563636363636463646363636465636563646363636363636365636363636363646365636363636364646565646463636465646565646365656365636565636463636464636365636363636365636464636465656564636364646465636363646363656564636364656363646463656363656464646463636463656363646463636563636563656363636563646563636465646564636363656464636363656565656363636364656363646465636463636464656563656363646463656364646563636463636363646463646365646365646363636365656563646465646363636565646363636463646363646364636363646364636363636363656463636563636464646465646365636363636464646563646364656463636363636463636363636363646565646565636563646563646465646465646464636463656363636563646463636363636464656464636463636363636463646464656563636365636365636364656563636564636363646563636465636364636565636363636364656565656464646364656365646363636563646564636363636363636363646564656563646463646463636463636363636365636363646463646363636365636565646365636365646463636563656363636363636364646363636564646365656365646563646464636563636463656365656364636464636465656363646363636363646364656363656563636365656364636365636363636363636463656465636463656564636563636563636463636363636363646363636463656565646363656463646465656563636363636564646363636463636365656363636565636363646363646364636463636463636464636363646363636563656365656365636365646365656563646464646365636465656365636363646564636363636363646365646363636564636565636363646463646364636363636464636464656463646365646363656463646563646464636363636463636463646465656463636563636364636464646465656563656563656464636463646365646363636363636363636463636464656464636563656363656363646565646465646363656364636465636464636564656564646463636364646363636364636364646364656363646364636364636364636563646365636363636464646563656363636565636563646365636365646465636565636363636365646463646363636364636363656363656465636364646365646363646463636463636364656463656363636365636363636563656564636565646463646363636363646464636364636365636565636364646364636363636363646363636565636463646563636365656464636564646363656365656363646363636463636463636463646464636363656463636365636565656565636363636363636363636463646463636364646464656565636363636363636364646364646363636464646364636363636363636363656565636363636363636463636363636464636463656363646565656564646464646364646363636563656464646563636364636363656463636365656463636363636464636464656363636364636464646563636364646365636365636363656365636564636363656365636463646563656464646364636563636363636463636363646363636564646563636365656564656364636463646363636463636463636465636363636464636364636463636465646363656465656463656563636465636363636363646364646363656364636563636363636363646564646463636363656465646365646463636363636363646364636365636364636564636564636365636363646363636365636564636363636365646463636463656363656364656363636364646563636464636363636363656363646464636365646464646365636364636363636464646463646563636464646365646363636363646364646363656465656363646564636463656364646363636464646363656364636565656364656363646364636465656363636363646365646463646464636363636363656464646365636364636563636564656363656363656564636363636465636465656463646363636364636463656465636363636363636465636464646363636464646465656563646463636563636365656465636563636465636363646463656364636364636564636564636465636363636465656364636463656463636364646363636364636363636364636463646563636463646363656365646464636463646463656564636464636464646464636464636463646565656464636565636564656364656365636363636365646363636365636563656364636363646565646363656363646363656364636563646565636464636365656364646364636463656364636463656564646464656363636363646364646564656364646563646463636365656563646364636463636463636563636363646365646363656465656364636464646463646465636463646565646465646463646463636563636465656363636363656363636363646565656465636363656463646463636463636363636363636363636364646363646364646565656363636363646364646463646365646364636364646363636565636363646364636363656563646465636465636565646364656463636564646365646563636363636363656365656563636464636363646364636363646463646463636365646465646363636563636465646364636364646564656363636364636364656563656365636463656464636463636363636463636464656365656363636464646363656363636363636363636464636563646564646565646364636463656363636464656564636563636463646364636363656365646364646363646365646563656563636363646363636364636363636465636463636365656365656364646363646363636364646363636465646364656463636565646564656564656464636364636363636563636463636565656365636365646464636363636364656564636363636364636363656363656363656464636463656365636464636364636463636465636363636464646365656363636363656463656364636365636365656363636464646463636563636365636363636363636365636564646464636365636465636564656563656464646563646565656563646363646465646563646563636464636463646364636564646463646463646364636563636363656363636463636563656363646563636563646363636364636563646463636463636363646465636363646464646363656464636465646364636363636463636464646364646563656363636363646465636465636365636363646364636563636563656463646363646363636465646363646564636363636363656464646363646463646464646363636365636463656363636363656464636463636364636563656363646364636364646364656365636463646365646463636565656363656363636563636365636563636565636464636565646363646365636364636363656563636365636464646363646463646364636364636365636364636465646463636365636363656363656364636463636365636363636364636363646563656364636363636565656563646364636365646564656464656365636563636363656463656363646364646365636364636463636365646463636463636363646363656364646363646363636365656365646363646365636363646563656563656363636363646365636563636463646364646364656464636364656463656364636363636363636565636463636363646363636564656465646565656563656464636364636463636463636564646463636365636363636364646363646363646565636363636365656463636465646363636363646363636564656364636563636363636364636364636363646365646563636364656364636463646464636365646363646464656365646564646463636364636563656363636364646563636463636465636465636363636565636363656364656463646363636463636363636363636363636564636363656464646465636364646365646464646363636364636363656463636363636363636364656463646464636563636464646463636364636363636363636464646363636565646363636464656564636365636563636363636365636363646463646363656465646563646363636564636563636364636464646363656363646364646365646463636564646465636565636363636563646465636365646365636363646365636463636364636463636463656363656363636464636563646463636363636363636363646563646363636563646363656465646563636564656564656365646364636564636363636363646365636465636463636563656463646563636463636563646363636363636364646463636363646463646564636365656465656364636363636365636363636564636364646363636463636363636364646465656363646365656365636563636364656463636563636365646464656365656464646463646363636364646364636465636364646363636363636563656563656365636364636363636363636365646363656363636465636464646563636363646363636463646464656363646364646363636463646463636365636564636364646565636363646463656364636463636463636463656463636563636365636464636463646463636564636364656564656465656363636563636363646364646464636564646364646463656564636363636565656463646463636464636563636365636463646363646363646464646363656564646464636363636563636363646364636563656463636363636565646464636364646363636464646463656465646364636564656363636363646365656363646364636365636363636363636463636463636365636363646463636463636563636364636365636363646365636363636465646463656563656565636364636365646564646363636565636363646363656365636364636563636365646563656363636463636365646463646564646565656363636363646463636564636364636564636463636363636364646465656364646363636363636564656364656463656365636463636464636363656463636563636564656363636563636364636365636465646365646464656363656463646364646464656365646363636463646563636564646465656564636564636365636364646364646363636563636363636565636363646463636563656464636564636365636365646363646363636463636463636463646363636463636363636364646363636463636463646565646364636365646363636463646465656463636563636364656465646364656463646463636365636463636363636463636363636464636463646363636463636565636363656365646363646464636363646363636363646364636363656464636464646564656563636363646363646565656363656365636463656363636363636463646465656364656464636364636463636364646463636463656365646365646463636464646363636563636564646363656565646463636365636563646364636363646565656363636363636364636363636463646364646464656563636363646363636563646365656564636364656363646364636563656363656363636365656363656363636463636464636365646564636463636364636464656363656563646465646365646464646464656363656564636464636463636364646363656363636363656364646363656363636363636363636463646564646363636565646363636363636464646563656463646463646564656563636563636463656563636363656565656363646463636464646363636564656364656563636363646563656363646565656565646564656364646363656363646563636463636465636463636463646463646465636363636464656563646464646465636463656363656465636564636363646563656364656365636465636364646564636363656365656364636464646465636564636563636363636363646363636364636464646463636363656564636363646365646364646465636564636364646564656363656463636364646464636563636363646363636364636365656363636364656363636365646465636363636363646463656365646363636363636465636563636563646564656463646565636465646463646364636564646363646363656564636463636464636363646564636464646364656363646565636363636564646363646364636363636363646463636365646563636364636563646364646363646463636464646363636465636563636363636463636364636463636564646363636463636563656364656565656463656463646365636364646463646363636565636465636365636463636463636363646463646563646363646565636363636363636463646465656363656363646363656365636565636464636363656363636463646463646463646364636364636463636365636463656564636363636365636364636564646564646565646465646364646363646464656363636564636363636363656365636463636564636363636364646463636563646365656564636563636464636464636564636463636363636363636464656364636365646563656363636565646364636363656364646364646364636365636563646364636365646365636465636463636365636363636564646363636563646463646565656365636463656564656465636364646463646363636463646364656563646363656364636464636363636564646364646364656363636364636363656563646364646363636465656463656363636564646464636463646365646365646463636464646463636465636563636365646363636463646364646364636363636363636363636363646465656364636364636463636564656465646464636365656463646463636364636364656363636363636464636364656363656463656364646363636365636563636463636465636364656363636364636364646463636463646564636363636463646364656364636363656463656464656365646363636365636564636464636464636564656363636463646363636464646464646465646365646364636365636463636465636364646363656465646463646563636565636364636364646565646363656563646365636565656563636363636464636363646565656565656464636464656363636363646363646563636465636463646463636563646465656463636363646364646465636563656464646365636363646363636364656463636464646363656464656363636463656364646363656465656363656463636364636363636363646563636563636463646363636364646463636364656364646363636464646565636365646363636365646363636463646464656364636363646363646464656365636363636365656464636365636563646364636563656463636463656363636563656365656463656363636364646363656364646363656365646363646365646363636463646463646364646363636363646363656364636363646463646464656463636463636363656363636365656465646565646364646363636363636563636363646363636464636465656465636363636363646363656464656564636363646363636463656363636365636363636365636363636563646365646364646364646363636463636365636364656363636364646463656365636563636464636363636564656465646463646564636465646565646463636465646363636364636563656364636363656463636463646365636465636563636365636463636364636563636363656364636463636364636364636364646565636563656364656463646363656563636464636363646364656463636465656464656565636363636463656563646465636364646363636465646463646463636463636565646364636363636363636364656363636563646465636464636363656463656365646564646364646464646364636363646363636364636364636363646463636565646364636564636363636563646463646363656563646563636565636363656365636365636563646563636364646363646563636364656463646465646365636463636363656564636365646565636564646364636363656364636564646363636464656363646363656463656363646363636364636364646464646563656465636365646363636465636364636563636364636563646363656565636564636363636365636565656465646463656464636364636363646464636363646364636363656363646465636364656363636363636363656464636463646364636364656565656363636365636365636364646463636365636364656464636363636365656364636463636364656464646464636363636564636363636465636365646365636563636465656365636463636464656564636363636363636465636463636463656364636463636463636364636364646364656463646463656365656363636365656564646463636363656363636464646463656465646465656364646363636564636564646463636463656563646565646563636364646363636363636364636365646463636363636565636463646364646363646363656563636563636463646564646463646363636364646365656364636365656364646464636463636364636364656363656365636363636364646365656563636564636465656363636563656365636464636364636464636363646563656365646365636364636463636463646363636363636464636363656565636365636365636463636363636363636365636363636363636563646365656363646363636563656363636563636563636565646463656464646363646464656463656365636564636363656365646363636363636364636363636365636463636363646563636365646363656364636464646364636563646463636365646364636363656363636563636464656363656364636365636365656363636364656364646463636363646563646364636363646463646365636365656364646563646463636364646464646463656564636365636464636363646364646363646465656364646365636563656465636564636363656463636364646364636363646463646363656364636463646463636565656563646363656463636363646363656563656364656464636363646563636364646363636365636363636364646365636464656563636365636565646463636565636363646364636363636363636564636563636363656364646565646363656363646565646363636363656463646363636563636363636363636363636563656363646363646465636365646364636365646565636563636465636464636363636364636365636463636363636363636363646365636364636363646363636565646564636463636364636463636465636565646463656364656565656363656363646463636564636365636465636364646364636364646365656564636365636564636463646563646563656465636363646565636363636365636563656465646363636465636364636465656565646363636463646363646464636564636363656563656363636363636365636563636565636363636563636563656363636364656364646563656363636364646365646464636364646364636563636363656363636363636563636463636563646363646364636365646364646565636365656363646563636463646363636363646565636464636465636463646365656363636463656363636364656463636364636465656364646363646363656364636563656465636465656463646563636363646363656564656465656364646364636363636463646465636464656463636364656363636465636364656463636565656363636364646464646365636363636365656564646465636563636363636365636365636563646463646365656363636363636563656563646364636563636364646465646463636364636563646563646563656463636464646565636365646363646565646364646363646463656465636363656465646365646363636363636364656363636463646465646463646565636365636364636465646363646363636363646464656364636363636363636365646363656364636365656465636364656464656363656363636363656364646365656464646365636563656563636465656463636363646463646364636465636465636365636564636364636564636364646464636363636365656563636363656363646363656464646563636364636365646565656363656563656365636463656363636365636463656463656563656365636464636465636363636363636563636465646463646463656465636363636363636464656564646364646563656363646564636565646565656465636365636465656465636464636463636564636463646364636364636363636463646564636363636564656464636365636464656463636363636363636463656563656363646565636463636363646365636363636365646563646363656563636464636363636363636365636365656463636363636364636463636363636365636463646463646563646364656465636363636563636365636363656563636363656565636363636563636364636363646363646365636564636363646363656363646363636363636563636463636363646465646463646365646363646563636463636463636365636463636563656365656364656365636363636563636463646463636563646463636363636564646463656565646363636364656364636363636463646364636364636564636564636363646365656363636364636363646465656365646464656365656363636463646363646465636363656463656563646363656463636465636364656563636563656564636365636363646465636364636563636464636364636564646365646365636463656463646465636565636364636365636364636365646364636363646465636365656563656465636463636364656563646564636364656363646363636463636563646364636463646363656564646464646565636463656365636364636363656565636363636463636564646464646363646363636363636363646363656563646464636363636464656564646465656563646364646563636363636365636564636463636363656363656363636463636463636464636465636464646363636364636565656563636564636365636363646463636564646364656365636363636364636364646465656463636363656364636464636363646464636363646464636464636363646564636463646364636364646363636364656564646365636563636364636463656363646564656363636465636364646363636464646363636365636365636463646363656463656363646565636463646363636364636364646463636563636363636463636563636363646464636364636463636465636363636363636565656363646563636563656465646463646363636463636365636463646363646565656365636464636363646364636463646564646363636565656363636363636363656363636363636363656363636563636565656364636465656364636363646564646364636365636363656365636463636463646365636463636363636364656463636364636463656363656364656363636564646564636363656363636564636363636563656363636564646463636564656365636363636363636563646563636364656563636363636365636463646363646463646463636464656364636463636565636363646363636563646463656363656363656365656363636365636463636464636465646363636563636563636363636464636364646463636563646464646563636363636563646365636563656563646463646463636564646463656363636364636464636363636565636364646363636363636464646565656464636564636365656363646363646365656364636463636465636363636563646363636463646364656463636464636363646364636363636364646363656463636364636363636364636365636565646465656363646365646463636363646364636363636463656464646363636464646364636563636363656363656463636365656363636364646463656364646463636363636465636363636465636463656564646563636363636363656364646564646363646365656565656363646464636465636363646364636563646363646363636365646564636363646363646365646363656463646363646363646363646364636463636364656363646363646364646564646364636465656464636463656365636463656363636563636463656565646563636563636365636464636364646463656563636563646363636463636463646463646364646463636363636363636563636363636364636563636363646563636564656365646465636464636565646463636363646363636364636463636363636563636363646363636463636363636363636563646363656463646463646364636363636464636465636563636563646363646563656365636364646363636565646463646363646463636465646463636464636563636364646564636363636365656463656365656464656464656463656363636463656564646464636365636464636463636363636563636463636463656364636564646363636463636463646565636363656564636364636364636563636563636565636363636463646365656363636363636563636563636465636463636563636563636564656465646363636465636463646464636564656464636565636363636363636363646364646563646465636364656463636465636465656463646463636465636363656363646363646564636564646563636363636563656364656463636464636463656563636363646464646363646464646463656363646465646363646363636564656363656363646363646463656364636463636563656463646563636563636463656363636363636463656463656363636363646463636364656464636563636364646563646364656563646465636363636563656364636464646563646363636563656463646464636363636364636365646365636363646363636464646463656363656463636363656464646365636365636363656363656363636465646463646363646365656463636363636463636363636364636364636365646364636364646365636365636365656363636565636465646463646363656463636364636363636564636363636463636565636363636565656363636564636465636365636363656564636363646363656363636364656464636363646364636464646563636365636364646565646363656463656363646465646363636564636363636364636365636365646363656563656564636363636564656365646363646365646563636363646463636463636365646363636464656563656364646363656563646364636363646563646465636364656463656564636365636563646364656563646563636563646463636564636364656363646463636364656363636465636364636363646363636364636563646463656363646465636364636465636565636363656465636463636564636463636463636365656365636563636363656363636364646564646563656465646563636363656465636363636463636363636363656363646563646465656465646563656363646363646463636463636563636365656363636363636563646463636463646363636463646363646363636363646463646463646363646364636464636363636464646465636363656464636363636565636363636364636364636363636464646463656364646363636364636363656363646464636363646464646364636463656463636364636563636463646463636464636364646464636365646563646465636564646464646365646364656465656463646563656563646564646363636363636464636364636363636563656463636565636563636364656364636565636365636565646364656364636464636464636565656363636565636464636565636564636565656363636365656363636463636364646364656565636464656463656364646363646363646364636364646564646465646563656465646463636563636463646365656363636363636365636464636464656363646363656463656364646364646564636363656565656363646365656563636464636365636564656364636363636365636364636565636363656364636364656363646363636464646464646363636563656363656465636463636363646564656465646363646363656364646363656364636464646564636563646564636465636564646463656563636363656465656464656364646365656363636363646463636564656363636563636363646463636463636564636363636463646463656365656463646363646463636565636565636363636463646363636463656364636363656563646565646363656363646364636464656463636363636563636463656363656464636463636363646363646463636364636464636564646363636563636564646463636364646364656363646464636464636465636363646464636365636365636565636364646564636363656364636565636363646463636363646364636463636363656464636463636463636563656464646464646465636565656463646565656564636464656463636363646463646464646363646364636563646364636564636363636465636363656465636365636363646463636364636565646564636564646364636363646465636463646365636364636364646463646365656563636365646363636363646464646563656364636464636363656463646463636364636363636563646464636363636563636365656363636363636465636464656363636364656564636563656363636365646563636363656363636563636465656463656364636363646364636463656563636364636464636365656363636363656363636365656365636463636463636365646464636563656365656463646564636565646363656463636363636364656563646363646463646365656463636463646363646363646464656564636563646363656363646363636465636465646363656363636363636363656364636364636363636363646465636363646364646463656563656563636563636363636363646463636363656463636465636365656565636363636363636363636465636365636363636464646363656363646364646363636564636363636464636365636463636464656565636464636465636364636364636365636563646565656464636363656363656464656364636365636365646363646363646363636365636463656465646565636364636364646363636364636463636563636363636465646365636365646563636564636365646563636464636563646363636363656564656463636363636364636365646363636565646364646363646364646564636563636363656563646363636363636364646365636364646464636463636363646363636565636365636463636463656364646563646563636365656463646363646564656363636363636363646363656465656463636463656365636463636365646564636363646463646363636363646365636363656363646463636364636364636463656363636563636464646363656563656465636364646465636365636565656565646463656363636363636565636463656365646363646363646363636364656563636364656463636465656563636363636465646363646464646563636363646363636563656565636464656364636365636364646463636363656363646363646463636563656365636463646365656563636365636363656365636463656363646364656465636364656364636364656363656365646365646364646565636463646563656364636365636364636365636563636565656464636563656463656363636565646464646365646363636463656463646565636365636465636465636364646363656363636364636464646563656463636365646565656563646564636363636564646365646565646463646365636365646563636463636364646463656363656463636363636363636363636364646364646463636364636563636364636464636365636463636464636364636463646364656463646564636563636363636465636363656463646364636363636564636365656463646365656364646563636363646363636563636465636463646364646364636463636363636463636464636363656364646365646363636465636364636363636464646565656363656463636363646363636363646463636364646564636363656363636464636364646363646363636365656463636364646565636364656363636363636563656363636364656463646463656363636365646463636565646364646565656464646364646464646463636364656565656363636365646365636564636363636363636363656363646363656365646463656564646364656363656363656563636363656564646463636365646365636563656363636364636463646463636365656365636363646564646463646363646463636563636365646463646564656363646464646363636463646464636363656463636363636564646364636365636365646363636363636363636563646465646364646364656564646564646563656363656463636563636364646364646563636463656464656363646363656563636565636565646563636363656365636565636464636564636564636365636564656364636363636364646365646365636364636563656365636363646465646363636564636463636363636363636363636564636463636363646364636363636363636364656563636363656463656365636365636464636363646365636463636363636363636363656365646465656365646464636464636465636365656563656463636565646563636463636565646363646365656364636363646364636364636365636464636363636364636463656364636463646563646364656563646465636463646364636363636563636364636463636564656363636363656463656563646363656364656364636363646365646565646465636563656363636363646563646464636364636463636564636463646363636363636563636463656463646364646465636363636563636364646363636363646364656363646465636563636463646563646363656365646365636463636364636364646364656364646365646364656563656363646565636363646463646365656563636364656364636363656363636363636364636365636563636463636463636563636464636363636364646563636564646464636465636364636465656564646363636363646464636463636363656363646463636364636365636363656463636364656563656464636363646463646463656563636463636565636563636363636363636363636363656364646365656464656364656363636465636463636464646564656463636363636364656463636565646363636563656465636465646364646363636563656363656363636365636364636563646363636363636465636363636564656465656363656563646563636465636363646363636363636365656364656465636464636364656463656565656363636363656463636363636363646465656465656363646563646465646365636363646363656365636363636563646463636364656465656465656565646363636363636464646563656363636363646563646363636363656364646563646564636363636363636563636564646364646464656564656563636563646564636363636364646464636363636364636364636364636464646365636364636463646463656364656563636564646365636465656364656465636363636365646565656563636464636563636463646563646363646363636364636364656363646364636563636465636365636464636565636363646564656364656463636464656464636564656363636364646463636364656363636464636363636363636564636363636363656564656564636363636363636363636563646564636363636563646565636364636364656463656463636365656363656363636365636563646563656364656364656463656364636463636364656563646463646364636363636363636363646365636464636564636365636564656463646365636563656464646365636564656363636363656563636363636563636564656365646363646464646363636363646363636565636463636564636465636364636363636565636463646564636363636563656463636364636464636463636463646463636363636564636365646363646463656364636363656364636464656563636364636465646364636563656365636363636464656463636363636565656365646463646364636463636363636363636364646563656563636364646364636364636364646364636564636363656563646363636463656464646363646564656464656363646363636564646463656364636365636463656465636563646463656365656364636363636363636363636363656565646365636464646363636565636363646363636363656463636563646563636363636365636363636463656563636364656563636363656464636363636363656463636363636563636364636364636364636365646563646364636463646563636463656564636563656364636365636363636365646365646465636364646363636365636463636465636363646464636564656364656363636365636364636363636364646563636365636363636563646363636463656463636364646463656565636463646363636363636365646464656563636363646364636365656464646363656463646364636363636363636364636363646365636364636564656363636363656364636365656364636364636463636464636363636363636463636364646465636564636563636365656365646463656464656564636463656364636565636363636563636363646364636565656363656364656363636364636363636564636565636464646363646465646463656463646365636363656464636465636365656363636465636364636363636363656365636363636365656365636564656363636463646364656363646363636365646364636363646465646463646563636365636465636563646564636363636463636464636365656465646463636363636363636463636364636364636564636463636463656364636465636464646364646363636363636364656363646364646563656363636363646565636365636463656363636564656363646365656363656463656363636463636463636365656364646465636564646463646463636363636363636365646363656364636364656363636365656465646463636363636364656565636465646563646363636464646365636463656464636563646563656463646363646363636363646463636463656363656364636563646464656363636363646364636463636363636565636463636364656363646363646464656363646563636464656563646363636463636564646364636365636465646363646464646464656363636563636363636364656465656363646364636363646363636364646565636563636365636364636365656363646363646464636363636363646364656564636463636364656463656463646364636464636363636563636463656464656363656564646465636365636563636463636363656364636363636464656364636465636364636364636463636464636365636465646563646364656363656464656563646363636564646464646564636564656365636463636363636364646463636363636464646365646363636564656363646465636463656363636464646364636364636365656364656363636565646463646363636565636463636565636363636563636365636364646365636464656364646363636564646364636364646363636363636363656463636365646463646363636364636563646363646463656563646564656364646464636363636563646363656563636363636465646363636465636464636563636363646365636464636564636363636363646464656364646363656563656465646363636465646365656563636465646363636363636463636465636563636563636565636363656565646365636565636364656363646363656563636364636463656363646463656363646363636464636363646563646365656363636563636364656363636463656563636564636464646563636465636363636363646464636363646363646463636463636364636363646463646364656565646364636364646363646363646364636364636363636364636363646463636365636463646364656464636464636463656365636363636363636365636463656363636464636564636563636365656563636364636364656363636364636463646563636364636363636363636363646465636363636365636464636563646363636463656463636564636563656464636563656563646365636564646365636563636463646364636363656465656365646563636463636365636365636363636363646463636463646563656363636464636563656463646363636365636363646363636563636365636363646364636564646564636363646364636363636363646363636364656363636364636463636563656464656363636364636363656363636463636364636465636464636365636363646463646564636564646364646364636463646563636463656464656364656364656363656463656563646463636463646363646363636363636364636463646363646565656364636363636564636363636464636365636364636363656463646364656464646464636463636363636364646364636364636364656463646364646363636465656464636365636363636365636363636464646363636463636564636364636464636363646463646463646565636363636363636363646363646363636363656364646365646465636365636565636363636364636365636363636463636463656363646363636363656463656463646364646364636365636463636464636363646463656564636365646563636464636364656364656565656463656465646464636363636564646365646563656564646363636463636365656363636463646563646363656563636565636563646463636564636564646365636463656364646563636363656465636363656364636463646365646363646463636365636363636565636464636463636363636464646365636465636563646565656564646363636463656463656363646365646365636363636464636463636464636363656363636563636565646365636563646364636363636463646564636364636563636365646563636363646463646363656563646463646563656363636565636465636464646365656464636363646365646363656364656365646464646363636463646364636363636465656364636364636564646363646363656364636364646363656363656364636363656363656564646363636563636564646564656364646464636563656464646364636363656463646364636363636563636363636563636363646465646363636363656363646363646364636363646465636363656463636565646365636363656364646363636363636464656463636563636563646363656363646465636563656465636363636463636363656363646563636364646363636364636464636364636363656465636364646463636363636363636464646463656364636363656563646464656565636564636464636564636463636363646364636563636563636563636364636363636565636363646365646363636564646364636364636463636363646463646363636465636463636365636364646365646463656363636463636464636565636463646364636564656464636364636564656465656363656363656463636564646463636363636363646365646463656364636364636363636363636465646563646364646363636363636563656364636564656465636565636364646565636464636364636365646463636563636365656363636365636363646363656363646364636465646363646364636363636565646365636564646465636363636364656463636465636565646563646564636363636465646563636364646563646364646464656565636464656463636363656563636364636363636463636463646363656563646363636365646363656364646464656465636464636465646365646364646365636364656464636363646564636365646563636564636463636564636364636364636363636363646365636365656365656365636363636364636365646363636365636365646365646465646363636363636463656563636363636463646463636363656563646363656364656363646363656363656363646464636364646565636364646363656365646364636563636364646463646463636364636465636365646563646365636365646463636563636563646464646364636463636563636563656564636563656363646463656365636464636465636365646463646365646363646365646465636463646363646363636564656364636563636463636364636465636363646563656363646363646363646464636365636564636364646563646363636364636463636464636363636365646365656364636363636564636463646363656465636463646363646464656365636365636463636463636463646563646364636463656364636364646363656365636364656364636363646363636364656463636463636364656365646364656463656363656465646364656363636564656364656363636463636465636364646364636563646463636363636363656463656463636363636363636363636364656363656564636463646564636463646363636364636463636564636363636363636463646365646363636463636364636565636464646363636563646563636363656363636365636564636363636365656364656464636464636463646463646564636464636463636563636363636363656564636463646563656464636564636363636363656365636363636364636363636463636564646465646364636464656463646364636465636363646363646363646465636365636564656463636365656463646363636364636364646363646364636464636564636364656465646363636363646563646564646565636365636563636363636363636563636465646364656365646364636464656465646465656363646363636363636463646463646563646363636563636363636363656364646364646365636365646364636363636364636365646463656463656363636364646563636363646363656363656465636365636363646363636363636364636563646463656565646565656564646363636363636465636463646565636464636363646363656363646464646463646364646363636463636363636363636465646364646463656465636363656464646465636363656565636364656563636463646364636364636364656363636364656364636564646563646363636564646563646363636363656463636464656363636463636365636364636365636363656363636464646463636363636365636463636463636463636564646465636463636364656463636363656463636564646364636363636463656365636464636463656565646363646365636464636365636464656363646363636363646364636464646365656363646364636563636564636363636464636564636365656463646363636364636564656363636464636463636363646565636364656564646464656464656363646564646564656463636563636365646365646463636364636463636365636463656463646364636364646563636464636464636363636563636365636463656363646463646563656363636363636564656364656364646365646363636463656365656463636363636363656463656363636364656363656464636365646363636364636463636464636464656465636363636363646564636463636364656363636364636363646363656364656364646464656363646364646365636465636363636463636464636563646564656364646365656363646465646364646563636465646364656365636365636465636363636464646363636364636463636463636363656364646465636463646363656564636363656565646365636464646365656365636365646363
diff --git a/test/data/invalid-legacy-sstables/Keyspace1/cf_with_duplicates_2_0/lb-1-big-CompressionInfo.db b/test/data/invalid-legacy-sstables/Keyspace1/cf_with_duplicates_2_0/lb-1-big-CompressionInfo.db
deleted file mode 100644
index 307eeb3..0000000
--- a/test/data/invalid-legacy-sstables/Keyspace1/cf_with_duplicates_2_0/lb-1-big-CompressionInfo.db
+++ /dev/null
Binary files differ
diff --git a/test/data/invalid-legacy-sstables/Keyspace1/cf_with_duplicates_2_0/lb-1-big-Data.db b/test/data/invalid-legacy-sstables/Keyspace1/cf_with_duplicates_2_0/lb-1-big-Data.db
deleted file mode 100644
index 175a5b6..0000000
--- a/test/data/invalid-legacy-sstables/Keyspace1/cf_with_duplicates_2_0/lb-1-big-Data.db
+++ /dev/null
Binary files differ
diff --git a/test/data/invalid-legacy-sstables/Keyspace1/cf_with_duplicates_2_0/lb-1-big-Digest.adler32 b/test/data/invalid-legacy-sstables/Keyspace1/cf_with_duplicates_2_0/lb-1-big-Digest.adler32
deleted file mode 100644
index ad624d2..0000000
--- a/test/data/invalid-legacy-sstables/Keyspace1/cf_with_duplicates_2_0/lb-1-big-Digest.adler32
+++ /dev/null
@@ -1 +0,0 @@
-408097082
\ No newline at end of file
diff --git a/test/data/invalid-legacy-sstables/Keyspace1/cf_with_duplicates_2_0/lb-1-big-Filter.db b/test/data/invalid-legacy-sstables/Keyspace1/cf_with_duplicates_2_0/lb-1-big-Filter.db
deleted file mode 100644
index 00a88b4..0000000
--- a/test/data/invalid-legacy-sstables/Keyspace1/cf_with_duplicates_2_0/lb-1-big-Filter.db
+++ /dev/null
Binary files differ
diff --git a/test/data/invalid-legacy-sstables/Keyspace1/cf_with_duplicates_2_0/lb-1-big-Index.db b/test/data/invalid-legacy-sstables/Keyspace1/cf_with_duplicates_2_0/lb-1-big-Index.db
deleted file mode 100644
index c3b42d8..0000000
--- a/test/data/invalid-legacy-sstables/Keyspace1/cf_with_duplicates_2_0/lb-1-big-Index.db
+++ /dev/null
Binary files differ
diff --git a/test/data/invalid-legacy-sstables/Keyspace1/cf_with_duplicates_2_0/lb-1-big-Statistics.db b/test/data/invalid-legacy-sstables/Keyspace1/cf_with_duplicates_2_0/lb-1-big-Statistics.db
deleted file mode 100644
index 056cf17..0000000
--- a/test/data/invalid-legacy-sstables/Keyspace1/cf_with_duplicates_2_0/lb-1-big-Statistics.db
+++ /dev/null
Binary files differ
diff --git a/test/data/invalid-legacy-sstables/Keyspace1/cf_with_duplicates_2_0/lb-1-big-Summary.db b/test/data/invalid-legacy-sstables/Keyspace1/cf_with_duplicates_2_0/lb-1-big-Summary.db
deleted file mode 100644
index 453753f..0000000
--- a/test/data/invalid-legacy-sstables/Keyspace1/cf_with_duplicates_2_0/lb-1-big-Summary.db
+++ /dev/null
Binary files differ
diff --git a/test/data/invalid-legacy-sstables/Keyspace1/cf_with_duplicates_2_0/lb-1-big-TOC.txt b/test/data/invalid-legacy-sstables/Keyspace1/cf_with_duplicates_2_0/lb-1-big-TOC.txt
deleted file mode 100644
index ceb1dab..0000000
--- a/test/data/invalid-legacy-sstables/Keyspace1/cf_with_duplicates_2_0/lb-1-big-TOC.txt
+++ /dev/null
@@ -1,8 +0,0 @@
-CompressionInfo.db
-Digest.adler32
-TOC.txt
-Filter.db
-Data.db
-Index.db
-Statistics.db
-Summary.db
diff --git a/test/data/legacy-commitlog/2.0/CommitLog-3-1431528750790.log b/test/data/legacy-commitlog/2.0/CommitLog-3-1431528750790.log
deleted file mode 100644
index 3301331..0000000
--- a/test/data/legacy-commitlog/2.0/CommitLog-3-1431528750790.log
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-commitlog/2.0/CommitLog-3-1431528750791.log b/test/data/legacy-commitlog/2.0/CommitLog-3-1431528750791.log
deleted file mode 100644
index 04314d6..0000000
--- a/test/data/legacy-commitlog/2.0/CommitLog-3-1431528750791.log
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-commitlog/2.0/CommitLog-3-1431528750792.log b/test/data/legacy-commitlog/2.0/CommitLog-3-1431528750792.log
deleted file mode 100644
index a9af9e4..0000000
--- a/test/data/legacy-commitlog/2.0/CommitLog-3-1431528750792.log
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-commitlog/2.0/CommitLog-3-1431528750793.log b/test/data/legacy-commitlog/2.0/CommitLog-3-1431528750793.log
deleted file mode 100644
index 3301331..0000000
--- a/test/data/legacy-commitlog/2.0/CommitLog-3-1431528750793.log
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-commitlog/2.0/hash.txt b/test/data/legacy-commitlog/2.0/hash.txt
deleted file mode 100644
index 4bbec02..0000000
--- a/test/data/legacy-commitlog/2.0/hash.txt
+++ /dev/null
@@ -1,3 +0,0 @@
-cfid = 4d331c44-f018-302b-91c2-2dcf94c4bfad
-cells = 9724
-hash = -682777064
diff --git a/test/data/legacy-commitlog/2.1/CommitLog-4-1431529069529.log b/test/data/legacy-commitlog/2.1/CommitLog-4-1431529069529.log
deleted file mode 100644
index 60064ee..0000000
--- a/test/data/legacy-commitlog/2.1/CommitLog-4-1431529069529.log
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-commitlog/2.1/CommitLog-4-1431529069530.log b/test/data/legacy-commitlog/2.1/CommitLog-4-1431529069530.log
deleted file mode 100644
index fdf7071..0000000
--- a/test/data/legacy-commitlog/2.1/CommitLog-4-1431529069530.log
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-commitlog/2.1/hash.txt b/test/data/legacy-commitlog/2.1/hash.txt
deleted file mode 100644
index f05cf97..0000000
--- a/test/data/legacy-commitlog/2.1/hash.txt
+++ /dev/null
@@ -1,3 +0,0 @@
-cfid = 6c622920-f980-11e4-b8a0-e7d448d5e26d
-cells = 5165
-hash = -1915888171
diff --git a/test/data/legacy-commitlog/2.2-lz4-bitrot/CommitLog-5-1438186885380.log b/test/data/legacy-commitlog/2.2-lz4-bitrot/CommitLog-5-1438186885380.log
deleted file mode 100644
index d248d59..0000000
--- a/test/data/legacy-commitlog/2.2-lz4-bitrot/CommitLog-5-1438186885380.log
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-commitlog/2.2-lz4-bitrot/hash.txt b/test/data/legacy-commitlog/2.2-lz4-bitrot/hash.txt
deleted file mode 100644
index c4d8fe7..0000000
--- a/test/data/legacy-commitlog/2.2-lz4-bitrot/hash.txt
+++ /dev/null
@@ -1,6 +0,0 @@
-#CommitLog bitrot test, version 2.2.0-SNAPSHOT
-#This is a copy of 2.2-lz4 with some overwritten bytes.
-#Replaying this should result in an error which can be overridden.
-cells=6051
-hash=-170208326
-cfid=dc32ce20-360d-11e5-826c-afadad37221d
diff --git a/test/data/legacy-commitlog/2.2-lz4-bitrot2/CommitLog-5-1438186885380.log b/test/data/legacy-commitlog/2.2-lz4-bitrot2/CommitLog-5-1438186885380.log
deleted file mode 100644
index 083d65c..0000000
--- a/test/data/legacy-commitlog/2.2-lz4-bitrot2/CommitLog-5-1438186885380.log
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-commitlog/2.2-lz4-bitrot2/hash.txt b/test/data/legacy-commitlog/2.2-lz4-bitrot2/hash.txt
deleted file mode 100644
index c49dda0..0000000
--- a/test/data/legacy-commitlog/2.2-lz4-bitrot2/hash.txt
+++ /dev/null
@@ -1,6 +0,0 @@
-#CommitLog upgrade test, version 2.2.0-SNAPSHOT
-#This is a copy of 2.2-lz4 with some overwritten bytes.
-#Replaying this should result in an error which can be overridden.
-cells=6037
-hash=-1312748407
-cfid=dc32ce20-360d-11e5-826c-afadad37221d
diff --git a/test/data/legacy-commitlog/2.2-lz4-truncated/CommitLog-5-1438186885380.log b/test/data/legacy-commitlog/2.2-lz4-truncated/CommitLog-5-1438186885380.log
deleted file mode 100644
index 939d408..0000000
--- a/test/data/legacy-commitlog/2.2-lz4-truncated/CommitLog-5-1438186885380.log
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-commitlog/2.2-lz4-truncated/hash.txt b/test/data/legacy-commitlog/2.2-lz4-truncated/hash.txt
deleted file mode 100644
index ce7f600..0000000
--- a/test/data/legacy-commitlog/2.2-lz4-truncated/hash.txt
+++ /dev/null
@@ -1,5 +0,0 @@
-#Truncated CommitLog test.
-#This is a copy of 2.2-lz4 with the last 50 bytes deleted.
-cells=6037
-hash=-889057729
-cfid=dc32ce20-360d-11e5-826c-afadad37221d
diff --git a/test/data/legacy-commitlog/2.2-lz4/CommitLog-5-1438186885380.log b/test/data/legacy-commitlog/2.2-lz4/CommitLog-5-1438186885380.log
deleted file mode 100644
index b98304a..0000000
--- a/test/data/legacy-commitlog/2.2-lz4/CommitLog-5-1438186885380.log
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-commitlog/2.2-lz4/CommitLog-5-1438186885381.log b/test/data/legacy-commitlog/2.2-lz4/CommitLog-5-1438186885381.log
deleted file mode 100644
index adac94f..0000000
--- a/test/data/legacy-commitlog/2.2-lz4/CommitLog-5-1438186885381.log
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-commitlog/2.2-lz4/hash.txt b/test/data/legacy-commitlog/2.2-lz4/hash.txt
deleted file mode 100644
index 20aa6e5..0000000
--- a/test/data/legacy-commitlog/2.2-lz4/hash.txt
+++ /dev/null
@@ -1,5 +0,0 @@
-#CommitLog upgrade test, version 2.2.0-SNAPSHOT
-#Wed Jul 29 19:21:31 EEST 2015
-cells=6052
-hash=1274136076
-cfid=dc32ce20-360d-11e5-826c-afadad37221d
diff --git a/test/data/legacy-commitlog/2.2-snappy/CommitLog-5-1438186915514.log b/test/data/legacy-commitlog/2.2-snappy/CommitLog-5-1438186915514.log
deleted file mode 100644
index e69dfb7..0000000
--- a/test/data/legacy-commitlog/2.2-snappy/CommitLog-5-1438186915514.log
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-commitlog/2.2-snappy/CommitLog-5-1438186915515.log b/test/data/legacy-commitlog/2.2-snappy/CommitLog-5-1438186915515.log
deleted file mode 100644
index 3e06675..0000000
--- a/test/data/legacy-commitlog/2.2-snappy/CommitLog-5-1438186915515.log
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-commitlog/2.2-snappy/hash.txt b/test/data/legacy-commitlog/2.2-snappy/hash.txt
deleted file mode 100644
index f3dd72e..0000000
--- a/test/data/legacy-commitlog/2.2-snappy/hash.txt
+++ /dev/null
@@ -1,5 +0,0 @@
-#CommitLog upgrade test, version 2.2.0-SNAPSHOT
-#Wed Jul 29 19:22:01 EEST 2015
-cells=6051
-hash=881633109
-cfid=ee2fe860-360d-11e5-951c-afadad37221d
diff --git a/test/data/legacy-commitlog/2.2/CommitLog-5-1438186815314.log b/test/data/legacy-commitlog/2.2/CommitLog-5-1438186815314.log
deleted file mode 100644
index 5032519..0000000
--- a/test/data/legacy-commitlog/2.2/CommitLog-5-1438186815314.log
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-commitlog/2.2/CommitLog-5-1438186815315.log b/test/data/legacy-commitlog/2.2/CommitLog-5-1438186815315.log
deleted file mode 100644
index 34a02fe..0000000
--- a/test/data/legacy-commitlog/2.2/CommitLog-5-1438186815315.log
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-commitlog/2.2/hash.txt b/test/data/legacy-commitlog/2.2/hash.txt
deleted file mode 100644
index 64f9dbb..0000000
--- a/test/data/legacy-commitlog/2.2/hash.txt
+++ /dev/null
@@ -1,5 +0,0 @@
-#CommitLog upgrade test, version 2.2.0-SNAPSHOT
-#Wed Jul 29 19:20:21 EEST 2015
-cells=6366
-hash=-802535821
-cfid=b28a7000-360d-11e5-ae92-afadad37221d
diff --git a/test/data/legacy-sstables/jb/Keyspace1/Keyspace1-Standard1-jb-0-CRC.db b/test/data/legacy-sstables/jb/Keyspace1/Keyspace1-Standard1-jb-0-CRC.db
deleted file mode 100644
index 0b6dab4..0000000
--- a/test/data/legacy-sstables/jb/Keyspace1/Keyspace1-Standard1-jb-0-CRC.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/jb/Keyspace1/Keyspace1-Standard1-jb-0-Data.db b/test/data/legacy-sstables/jb/Keyspace1/Keyspace1-Standard1-jb-0-Data.db
deleted file mode 100644
index 7d9407e..0000000
--- a/test/data/legacy-sstables/jb/Keyspace1/Keyspace1-Standard1-jb-0-Data.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/jb/Keyspace1/Keyspace1-Standard1-jb-0-Digest.sha1 b/test/data/legacy-sstables/jb/Keyspace1/Keyspace1-Standard1-jb-0-Digest.sha1
deleted file mode 100644
index 963bd9b..0000000
--- a/test/data/legacy-sstables/jb/Keyspace1/Keyspace1-Standard1-jb-0-Digest.sha1
+++ /dev/null
@@ -1 +0,0 @@
-4a9f1896a599e4b3ff5d19600901de1a0b851bc1 Keyspace1-Standard1-jb-0-Data.db
\ No newline at end of file
diff --git a/test/data/legacy-sstables/jb/Keyspace1/Keyspace1-Standard1-jb-0-Filter.db b/test/data/legacy-sstables/jb/Keyspace1/Keyspace1-Standard1-jb-0-Filter.db
deleted file mode 100644
index a3a807c..0000000
--- a/test/data/legacy-sstables/jb/Keyspace1/Keyspace1-Standard1-jb-0-Filter.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/jb/Keyspace1/Keyspace1-Standard1-jb-0-Index.db b/test/data/legacy-sstables/jb/Keyspace1/Keyspace1-Standard1-jb-0-Index.db
deleted file mode 100644
index ee9f5fb..0000000
--- a/test/data/legacy-sstables/jb/Keyspace1/Keyspace1-Standard1-jb-0-Index.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/jb/Keyspace1/Keyspace1-Standard1-jb-0-Statistics.db b/test/data/legacy-sstables/jb/Keyspace1/Keyspace1-Standard1-jb-0-Statistics.db
deleted file mode 100644
index daec1c3..0000000
--- a/test/data/legacy-sstables/jb/Keyspace1/Keyspace1-Standard1-jb-0-Statistics.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/jb/Keyspace1/Keyspace1-Standard1-jb-0-Summary.db b/test/data/legacy-sstables/jb/Keyspace1/Keyspace1-Standard1-jb-0-Summary.db
deleted file mode 100644
index 1fbe040..0000000
--- a/test/data/legacy-sstables/jb/Keyspace1/Keyspace1-Standard1-jb-0-Summary.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/jb/Keyspace1/Keyspace1-Standard1-jb-0-TOC.txt b/test/data/legacy-sstables/jb/Keyspace1/Keyspace1-Standard1-jb-0-TOC.txt
deleted file mode 100644
index d3aa557..0000000
--- a/test/data/legacy-sstables/jb/Keyspace1/Keyspace1-Standard1-jb-0-TOC.txt
+++ /dev/null
@@ -1,8 +0,0 @@
-Index.db
-TOC.txt
-Summary.db
-Filter.db
-Statistics.db
-Data.db
-CRC.db
-Digest.sha1
diff --git a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_clust/legacy_tables-legacy_jb_clust-jb-1-CompressionInfo.db b/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_clust/legacy_tables-legacy_jb_clust-jb-1-CompressionInfo.db
deleted file mode 100644
index 6d49922..0000000
--- a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_clust/legacy_tables-legacy_jb_clust-jb-1-CompressionInfo.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_clust/legacy_tables-legacy_jb_clust-jb-1-Data.db b/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_clust/legacy_tables-legacy_jb_clust-jb-1-Data.db
deleted file mode 100644
index 326498b..0000000
--- a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_clust/legacy_tables-legacy_jb_clust-jb-1-Data.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_clust/legacy_tables-legacy_jb_clust-jb-1-Filter.db b/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_clust/legacy_tables-legacy_jb_clust-jb-1-Filter.db
deleted file mode 100644
index c3cb27c..0000000
--- a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_clust/legacy_tables-legacy_jb_clust-jb-1-Filter.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_clust/legacy_tables-legacy_jb_clust-jb-1-Index.db b/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_clust/legacy_tables-legacy_jb_clust-jb-1-Index.db
deleted file mode 100644
index 44b89c4..0000000
--- a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_clust/legacy_tables-legacy_jb_clust-jb-1-Index.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_clust/legacy_tables-legacy_jb_clust-jb-1-Statistics.db b/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_clust/legacy_tables-legacy_jb_clust-jb-1-Statistics.db
deleted file mode 100644
index a9a404a..0000000
--- a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_clust/legacy_tables-legacy_jb_clust-jb-1-Statistics.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_clust/legacy_tables-legacy_jb_clust-jb-1-Summary.db b/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_clust/legacy_tables-legacy_jb_clust-jb-1-Summary.db
deleted file mode 100644
index 266c494..0000000
--- a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_clust/legacy_tables-legacy_jb_clust-jb-1-Summary.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_clust/legacy_tables-legacy_jb_clust-jb-1-TOC.txt b/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_clust/legacy_tables-legacy_jb_clust-jb-1-TOC.txt
deleted file mode 100644
index abc3147..0000000
--- a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_clust/legacy_tables-legacy_jb_clust-jb-1-TOC.txt
+++ /dev/null
@@ -1,7 +0,0 @@
-CompressionInfo.db
-Statistics.db
-Filter.db
-Data.db
-TOC.txt
-Index.db
-Summary.db
diff --git a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_clust_compact/legacy_tables-legacy_jb_clust_compact-jb-1-CompressionInfo.db b/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_clust_compact/legacy_tables-legacy_jb_clust_compact-jb-1-CompressionInfo.db
deleted file mode 100644
index 5eddda7..0000000
--- a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_clust_compact/legacy_tables-legacy_jb_clust_compact-jb-1-CompressionInfo.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_clust_compact/legacy_tables-legacy_jb_clust_compact-jb-1-Data.db b/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_clust_compact/legacy_tables-legacy_jb_clust_compact-jb-1-Data.db
deleted file mode 100644
index 61ef270..0000000
--- a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_clust_compact/legacy_tables-legacy_jb_clust_compact-jb-1-Data.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_clust_compact/legacy_tables-legacy_jb_clust_compact-jb-1-Filter.db b/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_clust_compact/legacy_tables-legacy_jb_clust_compact-jb-1-Filter.db
deleted file mode 100644
index c3cb27c..0000000
--- a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_clust_compact/legacy_tables-legacy_jb_clust_compact-jb-1-Filter.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_clust_compact/legacy_tables-legacy_jb_clust_compact-jb-1-Index.db b/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_clust_compact/legacy_tables-legacy_jb_clust_compact-jb-1-Index.db
deleted file mode 100644
index 9e18f8e..0000000
--- a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_clust_compact/legacy_tables-legacy_jb_clust_compact-jb-1-Index.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_clust_compact/legacy_tables-legacy_jb_clust_compact-jb-1-Statistics.db b/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_clust_compact/legacy_tables-legacy_jb_clust_compact-jb-1-Statistics.db
deleted file mode 100644
index ab83acc..0000000
--- a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_clust_compact/legacy_tables-legacy_jb_clust_compact-jb-1-Statistics.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_clust_compact/legacy_tables-legacy_jb_clust_compact-jb-1-Summary.db b/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_clust_compact/legacy_tables-legacy_jb_clust_compact-jb-1-Summary.db
deleted file mode 100644
index 896a529..0000000
--- a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_clust_compact/legacy_tables-legacy_jb_clust_compact-jb-1-Summary.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_clust_compact/legacy_tables-legacy_jb_clust_compact-jb-1-TOC.txt b/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_clust_compact/legacy_tables-legacy_jb_clust_compact-jb-1-TOC.txt
deleted file mode 100644
index b67360a..0000000
--- a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_clust_compact/legacy_tables-legacy_jb_clust_compact-jb-1-TOC.txt
+++ /dev/null
@@ -1,7 +0,0 @@
-Data.db
-CompressionInfo.db
-Index.db
-Summary.db
-TOC.txt
-Statistics.db
-Filter.db
diff --git a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_clust_counter/legacy_tables-legacy_jb_clust_counter-jb-1-CompressionInfo.db b/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_clust_counter/legacy_tables-legacy_jb_clust_counter-jb-1-CompressionInfo.db
deleted file mode 100644
index fe2e257..0000000
--- a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_clust_counter/legacy_tables-legacy_jb_clust_counter-jb-1-CompressionInfo.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_clust_counter/legacy_tables-legacy_jb_clust_counter-jb-1-Data.db b/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_clust_counter/legacy_tables-legacy_jb_clust_counter-jb-1-Data.db
deleted file mode 100644
index 12c8fdc..0000000
--- a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_clust_counter/legacy_tables-legacy_jb_clust_counter-jb-1-Data.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_clust_counter/legacy_tables-legacy_jb_clust_counter-jb-1-Filter.db b/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_clust_counter/legacy_tables-legacy_jb_clust_counter-jb-1-Filter.db
deleted file mode 100644
index c3cb27c..0000000
--- a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_clust_counter/legacy_tables-legacy_jb_clust_counter-jb-1-Filter.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_clust_counter/legacy_tables-legacy_jb_clust_counter-jb-1-Index.db b/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_clust_counter/legacy_tables-legacy_jb_clust_counter-jb-1-Index.db
deleted file mode 100644
index 51ddf91..0000000
--- a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_clust_counter/legacy_tables-legacy_jb_clust_counter-jb-1-Index.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_clust_counter/legacy_tables-legacy_jb_clust_counter-jb-1-Statistics.db b/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_clust_counter/legacy_tables-legacy_jb_clust_counter-jb-1-Statistics.db
deleted file mode 100644
index a5eff40..0000000
--- a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_clust_counter/legacy_tables-legacy_jb_clust_counter-jb-1-Statistics.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_clust_counter/legacy_tables-legacy_jb_clust_counter-jb-1-Summary.db b/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_clust_counter/legacy_tables-legacy_jb_clust_counter-jb-1-Summary.db
deleted file mode 100644
index 750a780..0000000
--- a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_clust_counter/legacy_tables-legacy_jb_clust_counter-jb-1-Summary.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_clust_counter/legacy_tables-legacy_jb_clust_counter-jb-1-TOC.txt b/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_clust_counter/legacy_tables-legacy_jb_clust_counter-jb-1-TOC.txt
deleted file mode 100644
index abc3147..0000000
--- a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_clust_counter/legacy_tables-legacy_jb_clust_counter-jb-1-TOC.txt
+++ /dev/null
@@ -1,7 +0,0 @@
-CompressionInfo.db
-Statistics.db
-Filter.db
-Data.db
-TOC.txt
-Index.db
-Summary.db
diff --git a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_clust_counter_compact/legacy_tables-legacy_jb_clust_counter_compact-jb-1-CompressionInfo.db b/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_clust_counter_compact/legacy_tables-legacy_jb_clust_counter_compact-jb-1-CompressionInfo.db
deleted file mode 100644
index 34d459d..0000000
--- a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_clust_counter_compact/legacy_tables-legacy_jb_clust_counter_compact-jb-1-CompressionInfo.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_clust_counter_compact/legacy_tables-legacy_jb_clust_counter_compact-jb-1-Data.db b/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_clust_counter_compact/legacy_tables-legacy_jb_clust_counter_compact-jb-1-Data.db
deleted file mode 100644
index b511d30..0000000
--- a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_clust_counter_compact/legacy_tables-legacy_jb_clust_counter_compact-jb-1-Data.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_clust_counter_compact/legacy_tables-legacy_jb_clust_counter_compact-jb-1-Filter.db b/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_clust_counter_compact/legacy_tables-legacy_jb_clust_counter_compact-jb-1-Filter.db
deleted file mode 100644
index c3cb27c..0000000
--- a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_clust_counter_compact/legacy_tables-legacy_jb_clust_counter_compact-jb-1-Filter.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_clust_counter_compact/legacy_tables-legacy_jb_clust_counter_compact-jb-1-Index.db b/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_clust_counter_compact/legacy_tables-legacy_jb_clust_counter_compact-jb-1-Index.db
deleted file mode 100644
index 10df1e8..0000000
--- a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_clust_counter_compact/legacy_tables-legacy_jb_clust_counter_compact-jb-1-Index.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_clust_counter_compact/legacy_tables-legacy_jb_clust_counter_compact-jb-1-Statistics.db b/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_clust_counter_compact/legacy_tables-legacy_jb_clust_counter_compact-jb-1-Statistics.db
deleted file mode 100644
index aa3c757..0000000
--- a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_clust_counter_compact/legacy_tables-legacy_jb_clust_counter_compact-jb-1-Statistics.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_clust_counter_compact/legacy_tables-legacy_jb_clust_counter_compact-jb-1-Summary.db b/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_clust_counter_compact/legacy_tables-legacy_jb_clust_counter_compact-jb-1-Summary.db
deleted file mode 100644
index 896a529..0000000
--- a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_clust_counter_compact/legacy_tables-legacy_jb_clust_counter_compact-jb-1-Summary.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_clust_counter_compact/legacy_tables-legacy_jb_clust_counter_compact-jb-1-TOC.txt b/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_clust_counter_compact/legacy_tables-legacy_jb_clust_counter_compact-jb-1-TOC.txt
deleted file mode 100644
index b67360a..0000000
--- a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_clust_counter_compact/legacy_tables-legacy_jb_clust_counter_compact-jb-1-TOC.txt
+++ /dev/null
@@ -1,7 +0,0 @@
-Data.db
-CompressionInfo.db
-Index.db
-Summary.db
-TOC.txt
-Statistics.db
-Filter.db
diff --git a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_simple/legacy_tables-legacy_jb_simple-jb-1-CompressionInfo.db b/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_simple/legacy_tables-legacy_jb_simple-jb-1-CompressionInfo.db
deleted file mode 100644
index c80e64c..0000000
--- a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_simple/legacy_tables-legacy_jb_simple-jb-1-CompressionInfo.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_simple/legacy_tables-legacy_jb_simple-jb-1-Data.db b/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_simple/legacy_tables-legacy_jb_simple-jb-1-Data.db
deleted file mode 100644
index 401fe93..0000000
--- a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_simple/legacy_tables-legacy_jb_simple-jb-1-Data.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_simple/legacy_tables-legacy_jb_simple-jb-1-Filter.db b/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_simple/legacy_tables-legacy_jb_simple-jb-1-Filter.db
deleted file mode 100644
index c3cb27c..0000000
--- a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_simple/legacy_tables-legacy_jb_simple-jb-1-Filter.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_simple/legacy_tables-legacy_jb_simple-jb-1-Index.db b/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_simple/legacy_tables-legacy_jb_simple-jb-1-Index.db
deleted file mode 100644
index f0717e0..0000000
--- a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_simple/legacy_tables-legacy_jb_simple-jb-1-Index.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_simple/legacy_tables-legacy_jb_simple-jb-1-Statistics.db b/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_simple/legacy_tables-legacy_jb_simple-jb-1-Statistics.db
deleted file mode 100644
index a2bcfaf..0000000
--- a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_simple/legacy_tables-legacy_jb_simple-jb-1-Statistics.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_simple/legacy_tables-legacy_jb_simple-jb-1-Summary.db b/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_simple/legacy_tables-legacy_jb_simple-jb-1-Summary.db
deleted file mode 100644
index af5e781..0000000
--- a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_simple/legacy_tables-legacy_jb_simple-jb-1-Summary.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_simple/legacy_tables-legacy_jb_simple-jb-1-TOC.txt b/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_simple/legacy_tables-legacy_jb_simple-jb-1-TOC.txt
deleted file mode 100644
index abc3147..0000000
--- a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_simple/legacy_tables-legacy_jb_simple-jb-1-TOC.txt
+++ /dev/null
@@ -1,7 +0,0 @@
-CompressionInfo.db
-Statistics.db
-Filter.db
-Data.db
-TOC.txt
-Index.db
-Summary.db
diff --git a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_simple_compact/legacy_tables-legacy_jb_simple_compact-jb-1-CompressionInfo.db b/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_simple_compact/legacy_tables-legacy_jb_simple_compact-jb-1-CompressionInfo.db
deleted file mode 100644
index d530b73..0000000
--- a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_simple_compact/legacy_tables-legacy_jb_simple_compact-jb-1-CompressionInfo.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_simple_compact/legacy_tables-legacy_jb_simple_compact-jb-1-Data.db b/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_simple_compact/legacy_tables-legacy_jb_simple_compact-jb-1-Data.db
deleted file mode 100644
index c7e8586..0000000
--- a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_simple_compact/legacy_tables-legacy_jb_simple_compact-jb-1-Data.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_simple_compact/legacy_tables-legacy_jb_simple_compact-jb-1-Filter.db b/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_simple_compact/legacy_tables-legacy_jb_simple_compact-jb-1-Filter.db
deleted file mode 100644
index c3cb27c..0000000
--- a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_simple_compact/legacy_tables-legacy_jb_simple_compact-jb-1-Filter.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_simple_compact/legacy_tables-legacy_jb_simple_compact-jb-1-Index.db b/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_simple_compact/legacy_tables-legacy_jb_simple_compact-jb-1-Index.db
deleted file mode 100644
index d2ec218..0000000
--- a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_simple_compact/legacy_tables-legacy_jb_simple_compact-jb-1-Index.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_simple_compact/legacy_tables-legacy_jb_simple_compact-jb-1-Statistics.db b/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_simple_compact/legacy_tables-legacy_jb_simple_compact-jb-1-Statistics.db
deleted file mode 100644
index 792e733..0000000
--- a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_simple_compact/legacy_tables-legacy_jb_simple_compact-jb-1-Statistics.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_simple_compact/legacy_tables-legacy_jb_simple_compact-jb-1-Summary.db b/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_simple_compact/legacy_tables-legacy_jb_simple_compact-jb-1-Summary.db
deleted file mode 100644
index af5e781..0000000
--- a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_simple_compact/legacy_tables-legacy_jb_simple_compact-jb-1-Summary.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_simple_compact/legacy_tables-legacy_jb_simple_compact-jb-1-TOC.txt b/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_simple_compact/legacy_tables-legacy_jb_simple_compact-jb-1-TOC.txt
deleted file mode 100644
index b67360a..0000000
--- a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_simple_compact/legacy_tables-legacy_jb_simple_compact-jb-1-TOC.txt
+++ /dev/null
@@ -1,7 +0,0 @@
-Data.db
-CompressionInfo.db
-Index.db
-Summary.db
-TOC.txt
-Statistics.db
-Filter.db
diff --git a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_simple_counter/legacy_tables-legacy_jb_simple_counter-jb-1-CompressionInfo.db b/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_simple_counter/legacy_tables-legacy_jb_simple_counter-jb-1-CompressionInfo.db
deleted file mode 100644
index 9c3416e..0000000
--- a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_simple_counter/legacy_tables-legacy_jb_simple_counter-jb-1-CompressionInfo.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_simple_counter/legacy_tables-legacy_jb_simple_counter-jb-1-Data.db b/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_simple_counter/legacy_tables-legacy_jb_simple_counter-jb-1-Data.db
deleted file mode 100644
index b72f790..0000000
--- a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_simple_counter/legacy_tables-legacy_jb_simple_counter-jb-1-Data.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_simple_counter/legacy_tables-legacy_jb_simple_counter-jb-1-Filter.db b/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_simple_counter/legacy_tables-legacy_jb_simple_counter-jb-1-Filter.db
deleted file mode 100644
index c3cb27c..0000000
--- a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_simple_counter/legacy_tables-legacy_jb_simple_counter-jb-1-Filter.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_simple_counter/legacy_tables-legacy_jb_simple_counter-jb-1-Index.db b/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_simple_counter/legacy_tables-legacy_jb_simple_counter-jb-1-Index.db
deleted file mode 100644
index 932936c..0000000
--- a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_simple_counter/legacy_tables-legacy_jb_simple_counter-jb-1-Index.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_simple_counter/legacy_tables-legacy_jb_simple_counter-jb-1-Statistics.db b/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_simple_counter/legacy_tables-legacy_jb_simple_counter-jb-1-Statistics.db
deleted file mode 100644
index 6baf1de..0000000
--- a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_simple_counter/legacy_tables-legacy_jb_simple_counter-jb-1-Statistics.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_simple_counter/legacy_tables-legacy_jb_simple_counter-jb-1-Summary.db b/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_simple_counter/legacy_tables-legacy_jb_simple_counter-jb-1-Summary.db
deleted file mode 100644
index af5e781..0000000
--- a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_simple_counter/legacy_tables-legacy_jb_simple_counter-jb-1-Summary.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_simple_counter/legacy_tables-legacy_jb_simple_counter-jb-1-TOC.txt b/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_simple_counter/legacy_tables-legacy_jb_simple_counter-jb-1-TOC.txt
deleted file mode 100644
index abc3147..0000000
--- a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_simple_counter/legacy_tables-legacy_jb_simple_counter-jb-1-TOC.txt
+++ /dev/null
@@ -1,7 +0,0 @@
-CompressionInfo.db
-Statistics.db
-Filter.db
-Data.db
-TOC.txt
-Index.db
-Summary.db
diff --git a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_simple_counter_compact/legacy_tables-legacy_jb_simple_counter_compact-jb-1-CompressionInfo.db b/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_simple_counter_compact/legacy_tables-legacy_jb_simple_counter_compact-jb-1-CompressionInfo.db
deleted file mode 100644
index 01c5478..0000000
--- a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_simple_counter_compact/legacy_tables-legacy_jb_simple_counter_compact-jb-1-CompressionInfo.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_simple_counter_compact/legacy_tables-legacy_jb_simple_counter_compact-jb-1-Data.db b/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_simple_counter_compact/legacy_tables-legacy_jb_simple_counter_compact-jb-1-Data.db
deleted file mode 100644
index f545b04..0000000
--- a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_simple_counter_compact/legacy_tables-legacy_jb_simple_counter_compact-jb-1-Data.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_simple_counter_compact/legacy_tables-legacy_jb_simple_counter_compact-jb-1-Filter.db b/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_simple_counter_compact/legacy_tables-legacy_jb_simple_counter_compact-jb-1-Filter.db
deleted file mode 100644
index c3cb27c..0000000
--- a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_simple_counter_compact/legacy_tables-legacy_jb_simple_counter_compact-jb-1-Filter.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_simple_counter_compact/legacy_tables-legacy_jb_simple_counter_compact-jb-1-Index.db b/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_simple_counter_compact/legacy_tables-legacy_jb_simple_counter_compact-jb-1-Index.db
deleted file mode 100644
index 48c153c..0000000
--- a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_simple_counter_compact/legacy_tables-legacy_jb_simple_counter_compact-jb-1-Index.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_simple_counter_compact/legacy_tables-legacy_jb_simple_counter_compact-jb-1-Statistics.db b/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_simple_counter_compact/legacy_tables-legacy_jb_simple_counter_compact-jb-1-Statistics.db
deleted file mode 100644
index 8657050..0000000
--- a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_simple_counter_compact/legacy_tables-legacy_jb_simple_counter_compact-jb-1-Statistics.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_simple_counter_compact/legacy_tables-legacy_jb_simple_counter_compact-jb-1-Summary.db b/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_simple_counter_compact/legacy_tables-legacy_jb_simple_counter_compact-jb-1-Summary.db
deleted file mode 100644
index af5e781..0000000
--- a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_simple_counter_compact/legacy_tables-legacy_jb_simple_counter_compact-jb-1-Summary.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_simple_counter_compact/legacy_tables-legacy_jb_simple_counter_compact-jb-1-TOC.txt b/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_simple_counter_compact/legacy_tables-legacy_jb_simple_counter_compact-jb-1-TOC.txt
deleted file mode 100644
index b67360a..0000000
--- a/test/data/legacy-sstables/jb/legacy_tables/legacy_jb_simple_counter_compact/legacy_tables-legacy_jb_simple_counter_compact-jb-1-TOC.txt
+++ /dev/null
@@ -1,7 +0,0 @@
-Data.db
-CompressionInfo.db
-Index.db
-Summary.db
-TOC.txt
-Statistics.db
-Filter.db
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14766/legacy_tables-legacy_ka_14766-ka-1-CompressionInfo.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14766/legacy_tables-legacy_ka_14766-ka-1-CompressionInfo.db
deleted file mode 100644
index b5b5246..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14766/legacy_tables-legacy_ka_14766-ka-1-CompressionInfo.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14766/legacy_tables-legacy_ka_14766-ka-1-Data.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14766/legacy_tables-legacy_ka_14766-ka-1-Data.db
deleted file mode 100644
index 18cf478..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14766/legacy_tables-legacy_ka_14766-ka-1-Data.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14766/legacy_tables-legacy_ka_14766-ka-1-Digest.sha1 b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14766/legacy_tables-legacy_ka_14766-ka-1-Digest.sha1
deleted file mode 100644
index f37a2b3..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14766/legacy_tables-legacy_ka_14766-ka-1-Digest.sha1
+++ /dev/null
@@ -1 +0,0 @@
-1576541413
\ No newline at end of file
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14766/legacy_tables-legacy_ka_14766-ka-1-Filter.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14766/legacy_tables-legacy_ka_14766-ka-1-Filter.db
deleted file mode 100644
index 7a31048..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14766/legacy_tables-legacy_ka_14766-ka-1-Filter.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14766/legacy_tables-legacy_ka_14766-ka-1-Index.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14766/legacy_tables-legacy_ka_14766-ka-1-Index.db
deleted file mode 100644
index 5e4995c..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14766/legacy_tables-legacy_ka_14766-ka-1-Index.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14766/legacy_tables-legacy_ka_14766-ka-1-Statistics.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14766/legacy_tables-legacy_ka_14766-ka-1-Statistics.db
deleted file mode 100644
index d4b0526..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14766/legacy_tables-legacy_ka_14766-ka-1-Statistics.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14766/legacy_tables-legacy_ka_14766-ka-1-Summary.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14766/legacy_tables-legacy_ka_14766-ka-1-Summary.db
deleted file mode 100644
index 38cc933..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14766/legacy_tables-legacy_ka_14766-ka-1-Summary.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14766/legacy_tables-legacy_ka_14766-ka-1-TOC.txt b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14766/legacy_tables-legacy_ka_14766-ka-1-TOC.txt
deleted file mode 100644
index db5ac46..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14766/legacy_tables-legacy_ka_14766-ka-1-TOC.txt
+++ /dev/null
@@ -1,8 +0,0 @@
-Data.db
-TOC.txt
-Digest.sha1
-Filter.db
-Statistics.db
-CompressionInfo.db
-Summary.db
-Index.db
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14803/legacy_tables-legacy_ka_14803-ka-1-CompressionInfo.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14803/legacy_tables-legacy_ka_14803-ka-1-CompressionInfo.db
deleted file mode 100644
index bb15937..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14803/legacy_tables-legacy_ka_14803-ka-1-CompressionInfo.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14803/legacy_tables-legacy_ka_14803-ka-1-Data.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14803/legacy_tables-legacy_ka_14803-ka-1-Data.db
deleted file mode 100644
index 9f946ab..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14803/legacy_tables-legacy_ka_14803-ka-1-Data.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14803/legacy_tables-legacy_ka_14803-ka-1-Digest.sha1 b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14803/legacy_tables-legacy_ka_14803-ka-1-Digest.sha1
deleted file mode 100644
index ec58891..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14803/legacy_tables-legacy_ka_14803-ka-1-Digest.sha1
+++ /dev/null
@@ -1 +0,0 @@
-2454867855
\ No newline at end of file
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14803/legacy_tables-legacy_ka_14803-ka-1-Filter.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14803/legacy_tables-legacy_ka_14803-ka-1-Filter.db
deleted file mode 100644
index 606783d..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14803/legacy_tables-legacy_ka_14803-ka-1-Filter.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14803/legacy_tables-legacy_ka_14803-ka-1-Index.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14803/legacy_tables-legacy_ka_14803-ka-1-Index.db
deleted file mode 100644
index bcf40a1..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14803/legacy_tables-legacy_ka_14803-ka-1-Index.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14803/legacy_tables-legacy_ka_14803-ka-1-Statistics.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14803/legacy_tables-legacy_ka_14803-ka-1-Statistics.db
deleted file mode 100644
index d30baa5..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14803/legacy_tables-legacy_ka_14803-ka-1-Statistics.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14803/legacy_tables-legacy_ka_14803-ka-1-Summary.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14803/legacy_tables-legacy_ka_14803-ka-1-Summary.db
deleted file mode 100644
index a4d9a6e..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14803/legacy_tables-legacy_ka_14803-ka-1-Summary.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14803/legacy_tables-legacy_ka_14803-ka-1-TOC.txt b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14803/legacy_tables-legacy_ka_14803-ka-1-TOC.txt
deleted file mode 100644
index 141f12c..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14803/legacy_tables-legacy_ka_14803-ka-1-TOC.txt
+++ /dev/null
@@ -1,8 +0,0 @@
-Summary.db
-Data.db
-Index.db
-Digest.sha1
-CompressionInfo.db
-TOC.txt
-Filter.db
-Statistics.db
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14873/legacy_tables-legacy_ka_14873-ka-1-CompressionInfo.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14873/legacy_tables-legacy_ka_14873-ka-1-CompressionInfo.db
deleted file mode 100644
index 4a87419..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14873/legacy_tables-legacy_ka_14873-ka-1-CompressionInfo.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14873/legacy_tables-legacy_ka_14873-ka-1-Data.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14873/legacy_tables-legacy_ka_14873-ka-1-Data.db
deleted file mode 100644
index 007cc50..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14873/legacy_tables-legacy_ka_14873-ka-1-Data.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14873/legacy_tables-legacy_ka_14873-ka-1-Digest.sha1 b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14873/legacy_tables-legacy_ka_14873-ka-1-Digest.sha1
deleted file mode 100644
index 71e6242..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14873/legacy_tables-legacy_ka_14873-ka-1-Digest.sha1
+++ /dev/null
@@ -1 +0,0 @@
-4060752841
\ No newline at end of file
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14873/legacy_tables-legacy_ka_14873-ka-1-Filter.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14873/legacy_tables-legacy_ka_14873-ka-1-Filter.db
deleted file mode 100644
index 7a31048..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14873/legacy_tables-legacy_ka_14873-ka-1-Filter.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14873/legacy_tables-legacy_ka_14873-ka-1-Index.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14873/legacy_tables-legacy_ka_14873-ka-1-Index.db
deleted file mode 100644
index 7245332..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14873/legacy_tables-legacy_ka_14873-ka-1-Index.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14873/legacy_tables-legacy_ka_14873-ka-1-Statistics.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14873/legacy_tables-legacy_ka_14873-ka-1-Statistics.db
deleted file mode 100644
index f4b26ee0..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14873/legacy_tables-legacy_ka_14873-ka-1-Statistics.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14873/legacy_tables-legacy_ka_14873-ka-1-Summary.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14873/legacy_tables-legacy_ka_14873-ka-1-Summary.db
deleted file mode 100644
index c1784f4..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14873/legacy_tables-legacy_ka_14873-ka-1-Summary.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14873/legacy_tables-legacy_ka_14873-ka-1-TOC.txt b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14873/legacy_tables-legacy_ka_14873-ka-1-TOC.txt
deleted file mode 100644
index db5ac46..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14873/legacy_tables-legacy_ka_14873-ka-1-TOC.txt
+++ /dev/null
@@ -1,8 +0,0 @@
-Data.db
-TOC.txt
-Digest.sha1
-Filter.db
-Statistics.db
-CompressionInfo.db
-Summary.db
-Index.db
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14912/legacy_tables-legacy_ka_14912-ka-1-CompressionInfo.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14912/legacy_tables-legacy_ka_14912-ka-1-CompressionInfo.db
deleted file mode 100644
index cf8c97a..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14912/legacy_tables-legacy_ka_14912-ka-1-CompressionInfo.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14912/legacy_tables-legacy_ka_14912-ka-1-Data.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14912/legacy_tables-legacy_ka_14912-ka-1-Data.db
deleted file mode 100644
index 19c7d79..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14912/legacy_tables-legacy_ka_14912-ka-1-Data.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14912/legacy_tables-legacy_ka_14912-ka-1-Digest.sha1 b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14912/legacy_tables-legacy_ka_14912-ka-1-Digest.sha1
deleted file mode 100644
index 66d3a1c..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14912/legacy_tables-legacy_ka_14912-ka-1-Digest.sha1
+++ /dev/null
@@ -1 +0,0 @@
-2565739962
\ No newline at end of file
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14912/legacy_tables-legacy_ka_14912-ka-1-Filter.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14912/legacy_tables-legacy_ka_14912-ka-1-Filter.db
deleted file mode 100644
index 1b7fa17..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14912/legacy_tables-legacy_ka_14912-ka-1-Filter.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14912/legacy_tables-legacy_ka_14912-ka-1-Index.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14912/legacy_tables-legacy_ka_14912-ka-1-Index.db
deleted file mode 100644
index a34ee93..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14912/legacy_tables-legacy_ka_14912-ka-1-Index.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14912/legacy_tables-legacy_ka_14912-ka-1-Statistics.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14912/legacy_tables-legacy_ka_14912-ka-1-Statistics.db
deleted file mode 100644
index 405c3e3..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14912/legacy_tables-legacy_ka_14912-ka-1-Statistics.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14912/legacy_tables-legacy_ka_14912-ka-1-Summary.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14912/legacy_tables-legacy_ka_14912-ka-1-Summary.db
deleted file mode 100644
index 9756785..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14912/legacy_tables-legacy_ka_14912-ka-1-Summary.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14912/legacy_tables-legacy_ka_14912-ka-1-TOC.txt b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14912/legacy_tables-legacy_ka_14912-ka-1-TOC.txt
deleted file mode 100644
index 7c351d8..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_14912/legacy_tables-legacy_ka_14912-ka-1-TOC.txt
+++ /dev/null
@@ -1,8 +0,0 @@
-TOC.txt
-Filter.db
-Index.db
-Summary.db
-Data.db
-CompressionInfo.db
-Digest.sha1
-Statistics.db
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_15081/legacy_tables-legacy_ka_15081-ka-1-CompressionInfo.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_15081/legacy_tables-legacy_ka_15081-ka-1-CompressionInfo.db
deleted file mode 100644
index 3793e50..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_15081/legacy_tables-legacy_ka_15081-ka-1-CompressionInfo.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_15081/legacy_tables-legacy_ka_15081-ka-1-Data.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_15081/legacy_tables-legacy_ka_15081-ka-1-Data.db
deleted file mode 100644
index 94c6f93..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_15081/legacy_tables-legacy_ka_15081-ka-1-Data.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_15081/legacy_tables-legacy_ka_15081-ka-1-Digest.sha1 b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_15081/legacy_tables-legacy_ka_15081-ka-1-Digest.sha1
deleted file mode 100644
index 60bd60d..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_15081/legacy_tables-legacy_ka_15081-ka-1-Digest.sha1
+++ /dev/null
@@ -1 +0,0 @@
-718738748
\ No newline at end of file
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_15081/legacy_tables-legacy_ka_15081-ka-1-Filter.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_15081/legacy_tables-legacy_ka_15081-ka-1-Filter.db
deleted file mode 100644
index 00a88b4..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_15081/legacy_tables-legacy_ka_15081-ka-1-Filter.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_15081/legacy_tables-legacy_ka_15081-ka-1-Index.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_15081/legacy_tables-legacy_ka_15081-ka-1-Index.db
deleted file mode 100644
index c3b42d8..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_15081/legacy_tables-legacy_ka_15081-ka-1-Index.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_15081/legacy_tables-legacy_ka_15081-ka-1-Statistics.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_15081/legacy_tables-legacy_ka_15081-ka-1-Statistics.db
deleted file mode 100644
index d708358..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_15081/legacy_tables-legacy_ka_15081-ka-1-Statistics.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_15081/legacy_tables-legacy_ka_15081-ka-1-Summary.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_15081/legacy_tables-legacy_ka_15081-ka-1-Summary.db
deleted file mode 100644
index 6bfc8aa..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_15081/legacy_tables-legacy_ka_15081-ka-1-Summary.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_15081/legacy_tables-legacy_ka_15081-ka-1-TOC.txt b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_15081/legacy_tables-legacy_ka_15081-ka-1-TOC.txt
deleted file mode 100644
index 5ece1a1..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_15081/legacy_tables-legacy_ka_15081-ka-1-TOC.txt
+++ /dev/null
@@ -1,8 +0,0 @@
-Digest.sha1
-Data.db
-Statistics.db
-Summary.db
-Index.db
-TOC.txt
-Filter.db
-CompressionInfo.db
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust/legacy_tables-legacy_ka_clust-ka-1-CompressionInfo.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust/legacy_tables-legacy_ka_clust-ka-1-CompressionInfo.db
deleted file mode 100644
index 69a8355..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust/legacy_tables-legacy_ka_clust-ka-1-CompressionInfo.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust/legacy_tables-legacy_ka_clust-ka-1-Data.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust/legacy_tables-legacy_ka_clust-ka-1-Data.db
deleted file mode 100644
index 7acbf92..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust/legacy_tables-legacy_ka_clust-ka-1-Data.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust/legacy_tables-legacy_ka_clust-ka-1-Digest.sha1 b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust/legacy_tables-legacy_ka_clust-ka-1-Digest.sha1
deleted file mode 100644
index fef7106..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust/legacy_tables-legacy_ka_clust-ka-1-Digest.sha1
+++ /dev/null
@@ -1 +0,0 @@
-4293822635
\ No newline at end of file
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust/legacy_tables-legacy_ka_clust-ka-1-Filter.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust/legacy_tables-legacy_ka_clust-ka-1-Filter.db
deleted file mode 100644
index c3cb27c..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust/legacy_tables-legacy_ka_clust-ka-1-Filter.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust/legacy_tables-legacy_ka_clust-ka-1-Index.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust/legacy_tables-legacy_ka_clust-ka-1-Index.db
deleted file mode 100644
index 44b89c4..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust/legacy_tables-legacy_ka_clust-ka-1-Index.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust/legacy_tables-legacy_ka_clust-ka-1-Statistics.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust/legacy_tables-legacy_ka_clust-ka-1-Statistics.db
deleted file mode 100644
index 5f07da5..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust/legacy_tables-legacy_ka_clust-ka-1-Statistics.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust/legacy_tables-legacy_ka_clust-ka-1-Summary.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust/legacy_tables-legacy_ka_clust-ka-1-Summary.db
deleted file mode 100644
index 35b5e22..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust/legacy_tables-legacy_ka_clust-ka-1-Summary.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust/legacy_tables-legacy_ka_clust-ka-1-TOC.txt b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust/legacy_tables-legacy_ka_clust-ka-1-TOC.txt
deleted file mode 100644
index 7be41d8..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust/legacy_tables-legacy_ka_clust-ka-1-TOC.txt
+++ /dev/null
@@ -1,8 +0,0 @@
-Index.db
-Digest.sha1
-CompressionInfo.db
-Data.db
-Statistics.db
-Summary.db
-TOC.txt
-Filter.db
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust_compact/legacy_tables-legacy_ka_clust_compact-ka-1-CompressionInfo.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust_compact/legacy_tables-legacy_ka_clust_compact-ka-1-CompressionInfo.db
deleted file mode 100644
index 654094e..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust_compact/legacy_tables-legacy_ka_clust_compact-ka-1-CompressionInfo.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust_compact/legacy_tables-legacy_ka_clust_compact-ka-1-Data.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust_compact/legacy_tables-legacy_ka_clust_compact-ka-1-Data.db
deleted file mode 100644
index 4c87e07..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust_compact/legacy_tables-legacy_ka_clust_compact-ka-1-Data.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust_compact/legacy_tables-legacy_ka_clust_compact-ka-1-Digest.sha1 b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust_compact/legacy_tables-legacy_ka_clust_compact-ka-1-Digest.sha1
deleted file mode 100644
index 4690757..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust_compact/legacy_tables-legacy_ka_clust_compact-ka-1-Digest.sha1
+++ /dev/null
@@ -1 +0,0 @@
-1331331706
\ No newline at end of file
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust_compact/legacy_tables-legacy_ka_clust_compact-ka-1-Filter.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust_compact/legacy_tables-legacy_ka_clust_compact-ka-1-Filter.db
deleted file mode 100644
index c3cb27c..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust_compact/legacy_tables-legacy_ka_clust_compact-ka-1-Filter.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust_compact/legacy_tables-legacy_ka_clust_compact-ka-1-Index.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust_compact/legacy_tables-legacy_ka_clust_compact-ka-1-Index.db
deleted file mode 100644
index 9e18f8e..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust_compact/legacy_tables-legacy_ka_clust_compact-ka-1-Index.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust_compact/legacy_tables-legacy_ka_clust_compact-ka-1-Statistics.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust_compact/legacy_tables-legacy_ka_clust_compact-ka-1-Statistics.db
deleted file mode 100644
index ab55258..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust_compact/legacy_tables-legacy_ka_clust_compact-ka-1-Statistics.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust_compact/legacy_tables-legacy_ka_clust_compact-ka-1-Summary.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust_compact/legacy_tables-legacy_ka_clust_compact-ka-1-Summary.db
deleted file mode 100644
index 774cbd1..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust_compact/legacy_tables-legacy_ka_clust_compact-ka-1-Summary.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust_compact/legacy_tables-legacy_ka_clust_compact-ka-1-TOC.txt b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust_compact/legacy_tables-legacy_ka_clust_compact-ka-1-TOC.txt
deleted file mode 100644
index 7f7fe79..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust_compact/legacy_tables-legacy_ka_clust_compact-ka-1-TOC.txt
+++ /dev/null
@@ -1,8 +0,0 @@
-Filter.db
-TOC.txt
-Statistics.db
-Summary.db
-Index.db
-Data.db
-Digest.sha1
-CompressionInfo.db
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust_counter/legacy_tables-legacy_ka_clust_counter-ka-1-CompressionInfo.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust_counter/legacy_tables-legacy_ka_clust_counter-ka-1-CompressionInfo.db
deleted file mode 100644
index 3c7291c..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust_counter/legacy_tables-legacy_ka_clust_counter-ka-1-CompressionInfo.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust_counter/legacy_tables-legacy_ka_clust_counter-ka-1-Data.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust_counter/legacy_tables-legacy_ka_clust_counter-ka-1-Data.db
deleted file mode 100644
index 3566e5a..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust_counter/legacy_tables-legacy_ka_clust_counter-ka-1-Data.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust_counter/legacy_tables-legacy_ka_clust_counter-ka-1-Digest.sha1 b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust_counter/legacy_tables-legacy_ka_clust_counter-ka-1-Digest.sha1
deleted file mode 100644
index a679541..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust_counter/legacy_tables-legacy_ka_clust_counter-ka-1-Digest.sha1
+++ /dev/null
@@ -1 +0,0 @@
-2539906592
\ No newline at end of file
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust_counter/legacy_tables-legacy_ka_clust_counter-ka-1-Filter.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust_counter/legacy_tables-legacy_ka_clust_counter-ka-1-Filter.db
deleted file mode 100644
index c3cb27c..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust_counter/legacy_tables-legacy_ka_clust_counter-ka-1-Filter.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust_counter/legacy_tables-legacy_ka_clust_counter-ka-1-Index.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust_counter/legacy_tables-legacy_ka_clust_counter-ka-1-Index.db
deleted file mode 100644
index 51ddf91..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust_counter/legacy_tables-legacy_ka_clust_counter-ka-1-Index.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust_counter/legacy_tables-legacy_ka_clust_counter-ka-1-Statistics.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust_counter/legacy_tables-legacy_ka_clust_counter-ka-1-Statistics.db
deleted file mode 100644
index 36e9dc2..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust_counter/legacy_tables-legacy_ka_clust_counter-ka-1-Statistics.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust_counter/legacy_tables-legacy_ka_clust_counter-ka-1-Summary.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust_counter/legacy_tables-legacy_ka_clust_counter-ka-1-Summary.db
deleted file mode 100644
index 35b5e22..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust_counter/legacy_tables-legacy_ka_clust_counter-ka-1-Summary.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust_counter/legacy_tables-legacy_ka_clust_counter-ka-1-TOC.txt b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust_counter/legacy_tables-legacy_ka_clust_counter-ka-1-TOC.txt
deleted file mode 100644
index 7be41d8..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust_counter/legacy_tables-legacy_ka_clust_counter-ka-1-TOC.txt
+++ /dev/null
@@ -1,8 +0,0 @@
-Index.db
-Digest.sha1
-CompressionInfo.db
-Data.db
-Statistics.db
-Summary.db
-TOC.txt
-Filter.db
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust_counter_compact/legacy_tables-legacy_ka_clust_counter_compact-ka-1-CompressionInfo.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust_counter_compact/legacy_tables-legacy_ka_clust_counter_compact-ka-1-CompressionInfo.db
deleted file mode 100644
index e3b71a4..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust_counter_compact/legacy_tables-legacy_ka_clust_counter_compact-ka-1-CompressionInfo.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust_counter_compact/legacy_tables-legacy_ka_clust_counter_compact-ka-1-Data.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust_counter_compact/legacy_tables-legacy_ka_clust_counter_compact-ka-1-Data.db
deleted file mode 100644
index 90d42a5..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust_counter_compact/legacy_tables-legacy_ka_clust_counter_compact-ka-1-Data.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust_counter_compact/legacy_tables-legacy_ka_clust_counter_compact-ka-1-Digest.sha1 b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust_counter_compact/legacy_tables-legacy_ka_clust_counter_compact-ka-1-Digest.sha1
deleted file mode 100644
index 52e6552..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust_counter_compact/legacy_tables-legacy_ka_clust_counter_compact-ka-1-Digest.sha1
+++ /dev/null
@@ -1 +0,0 @@
-2793875907
\ No newline at end of file
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust_counter_compact/legacy_tables-legacy_ka_clust_counter_compact-ka-1-Filter.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust_counter_compact/legacy_tables-legacy_ka_clust_counter_compact-ka-1-Filter.db
deleted file mode 100644
index c3cb27c..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust_counter_compact/legacy_tables-legacy_ka_clust_counter_compact-ka-1-Filter.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust_counter_compact/legacy_tables-legacy_ka_clust_counter_compact-ka-1-Index.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust_counter_compact/legacy_tables-legacy_ka_clust_counter_compact-ka-1-Index.db
deleted file mode 100644
index 10df1e8..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust_counter_compact/legacy_tables-legacy_ka_clust_counter_compact-ka-1-Index.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust_counter_compact/legacy_tables-legacy_ka_clust_counter_compact-ka-1-Statistics.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust_counter_compact/legacy_tables-legacy_ka_clust_counter_compact-ka-1-Statistics.db
deleted file mode 100644
index 8360ed5..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust_counter_compact/legacy_tables-legacy_ka_clust_counter_compact-ka-1-Statistics.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust_counter_compact/legacy_tables-legacy_ka_clust_counter_compact-ka-1-Summary.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust_counter_compact/legacy_tables-legacy_ka_clust_counter_compact-ka-1-Summary.db
deleted file mode 100644
index 774cbd1..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust_counter_compact/legacy_tables-legacy_ka_clust_counter_compact-ka-1-Summary.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust_counter_compact/legacy_tables-legacy_ka_clust_counter_compact-ka-1-TOC.txt b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust_counter_compact/legacy_tables-legacy_ka_clust_counter_compact-ka-1-TOC.txt
deleted file mode 100644
index 7f7fe79..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust_counter_compact/legacy_tables-legacy_ka_clust_counter_compact-ka-1-TOC.txt
+++ /dev/null
@@ -1,8 +0,0 @@
-Filter.db
-TOC.txt
-Statistics.db
-Summary.db
-Index.db
-Data.db
-Digest.sha1
-CompressionInfo.db
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_compacted_multi_block_rt/legacy_tables-legacy_ka_compacted_multi_block_rt-ka-4-CompressionInfo.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_compacted_multi_block_rt/legacy_tables-legacy_ka_compacted_multi_block_rt-ka-4-CompressionInfo.db
deleted file mode 100644
index d320406..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_compacted_multi_block_rt/legacy_tables-legacy_ka_compacted_multi_block_rt-ka-4-CompressionInfo.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_compacted_multi_block_rt/legacy_tables-legacy_ka_compacted_multi_block_rt-ka-4-Data.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_compacted_multi_block_rt/legacy_tables-legacy_ka_compacted_multi_block_rt-ka-4-Data.db
deleted file mode 100644
index 775b68c..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_compacted_multi_block_rt/legacy_tables-legacy_ka_compacted_multi_block_rt-ka-4-Data.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_compacted_multi_block_rt/legacy_tables-legacy_ka_compacted_multi_block_rt-ka-4-Digest.sha1 b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_compacted_multi_block_rt/legacy_tables-legacy_ka_compacted_multi_block_rt-ka-4-Digest.sha1
deleted file mode 100644
index 63993fc..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_compacted_multi_block_rt/legacy_tables-legacy_ka_compacted_multi_block_rt-ka-4-Digest.sha1
+++ /dev/null
@@ -1 +0,0 @@
-3417730863
\ No newline at end of file
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_compacted_multi_block_rt/legacy_tables-legacy_ka_compacted_multi_block_rt-ka-4-Filter.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_compacted_multi_block_rt/legacy_tables-legacy_ka_compacted_multi_block_rt-ka-4-Filter.db
deleted file mode 100644
index aa97e86..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_compacted_multi_block_rt/legacy_tables-legacy_ka_compacted_multi_block_rt-ka-4-Filter.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_compacted_multi_block_rt/legacy_tables-legacy_ka_compacted_multi_block_rt-ka-4-Index.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_compacted_multi_block_rt/legacy_tables-legacy_ka_compacted_multi_block_rt-ka-4-Index.db
deleted file mode 100644
index f425226..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_compacted_multi_block_rt/legacy_tables-legacy_ka_compacted_multi_block_rt-ka-4-Index.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_compacted_multi_block_rt/legacy_tables-legacy_ka_compacted_multi_block_rt-ka-4-Statistics.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_compacted_multi_block_rt/legacy_tables-legacy_ka_compacted_multi_block_rt-ka-4-Statistics.db
deleted file mode 100644
index 2580202..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_compacted_multi_block_rt/legacy_tables-legacy_ka_compacted_multi_block_rt-ka-4-Statistics.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_compacted_multi_block_rt/legacy_tables-legacy_ka_compacted_multi_block_rt-ka-4-Summary.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_compacted_multi_block_rt/legacy_tables-legacy_ka_compacted_multi_block_rt-ka-4-Summary.db
deleted file mode 100644
index c85b4a8..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_compacted_multi_block_rt/legacy_tables-legacy_ka_compacted_multi_block_rt-ka-4-Summary.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_compacted_multi_block_rt/legacy_tables-legacy_ka_compacted_multi_block_rt-ka-4-TOC.txt b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_compacted_multi_block_rt/legacy_tables-legacy_ka_compacted_multi_block_rt-ka-4-TOC.txt
deleted file mode 100644
index 3fc5eec..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_compacted_multi_block_rt/legacy_tables-legacy_ka_compacted_multi_block_rt-ka-4-TOC.txt
+++ /dev/null
@@ -1,8 +0,0 @@
-Summary.db
-Digest.sha1
-CompressionInfo.db
-TOC.txt
-Filter.db
-Data.db
-Index.db
-Statistics.db
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_cql_created_dense_table_with_bytes/legacy_tables-legacy_ka_cql_created_dense_table_with_bytes-ka-1-CompressionInfo.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_cql_created_dense_table_with_bytes/legacy_tables-legacy_ka_cql_created_dense_table_with_bytes-ka-1-CompressionInfo.db
deleted file mode 100644
index 01bde10..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_cql_created_dense_table_with_bytes/legacy_tables-legacy_ka_cql_created_dense_table_with_bytes-ka-1-CompressionInfo.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_cql_created_dense_table_with_bytes/legacy_tables-legacy_ka_cql_created_dense_table_with_bytes-ka-1-Data.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_cql_created_dense_table_with_bytes/legacy_tables-legacy_ka_cql_created_dense_table_with_bytes-ka-1-Data.db
deleted file mode 100644
index 4c891d2..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_cql_created_dense_table_with_bytes/legacy_tables-legacy_ka_cql_created_dense_table_with_bytes-ka-1-Data.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_cql_created_dense_table_with_bytes/legacy_tables-legacy_ka_cql_created_dense_table_with_bytes-ka-1-Digest.sha1 b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_cql_created_dense_table_with_bytes/legacy_tables-legacy_ka_cql_created_dense_table_with_bytes-ka-1-Digest.sha1
deleted file mode 100644
index e71840b..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_cql_created_dense_table_with_bytes/legacy_tables-legacy_ka_cql_created_dense_table_with_bytes-ka-1-Digest.sha1
+++ /dev/null
@@ -1 +0,0 @@
-3389985016
\ No newline at end of file
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_cql_created_dense_table_with_bytes/legacy_tables-legacy_ka_cql_created_dense_table_with_bytes-ka-1-Filter.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_cql_created_dense_table_with_bytes/legacy_tables-legacy_ka_cql_created_dense_table_with_bytes-ka-1-Filter.db
deleted file mode 100644
index b6728a1..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_cql_created_dense_table_with_bytes/legacy_tables-legacy_ka_cql_created_dense_table_with_bytes-ka-1-Filter.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_cql_created_dense_table_with_bytes/legacy_tables-legacy_ka_cql_created_dense_table_with_bytes-ka-1-Index.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_cql_created_dense_table_with_bytes/legacy_tables-legacy_ka_cql_created_dense_table_with_bytes-ka-1-Index.db
deleted file mode 100644
index 64e12cd..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_cql_created_dense_table_with_bytes/legacy_tables-legacy_ka_cql_created_dense_table_with_bytes-ka-1-Index.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_cql_created_dense_table_with_bytes/legacy_tables-legacy_ka_cql_created_dense_table_with_bytes-ka-1-Statistics.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_cql_created_dense_table_with_bytes/legacy_tables-legacy_ka_cql_created_dense_table_with_bytes-ka-1-Statistics.db
deleted file mode 100644
index 1361f7c..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_cql_created_dense_table_with_bytes/legacy_tables-legacy_ka_cql_created_dense_table_with_bytes-ka-1-Statistics.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_cql_created_dense_table_with_bytes/legacy_tables-legacy_ka_cql_created_dense_table_with_bytes-ka-1-Summary.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_cql_created_dense_table_with_bytes/legacy_tables-legacy_ka_cql_created_dense_table_with_bytes-ka-1-Summary.db
deleted file mode 100644
index 76791c7..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_cql_created_dense_table_with_bytes/legacy_tables-legacy_ka_cql_created_dense_table_with_bytes-ka-1-Summary.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_cql_created_dense_table_with_bytes/legacy_tables-legacy_ka_cql_created_dense_table_with_bytes-ka-1-TOC.txt b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_cql_created_dense_table_with_bytes/legacy_tables-legacy_ka_cql_created_dense_table_with_bytes-ka-1-TOC.txt
deleted file mode 100644
index 402e1ab..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_cql_created_dense_table_with_bytes/legacy_tables-legacy_ka_cql_created_dense_table_with_bytes-ka-1-TOC.txt
+++ /dev/null
@@ -1,8 +0,0 @@
-Digest.sha1
-CompressionInfo.db
-Summary.db
-Statistics.db
-Data.db
-Index.db
-TOC.txt
-Filter.db
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_cql_created_dense_table_with_int/legacy_tables-legacy_ka_cql_created_dense_table_with_int-ka-1-CompressionInfo.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_cql_created_dense_table_with_int/legacy_tables-legacy_ka_cql_created_dense_table_with_int-ka-1-CompressionInfo.db
deleted file mode 100644
index 6f36650..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_cql_created_dense_table_with_int/legacy_tables-legacy_ka_cql_created_dense_table_with_int-ka-1-CompressionInfo.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_cql_created_dense_table_with_int/legacy_tables-legacy_ka_cql_created_dense_table_with_int-ka-1-Data.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_cql_created_dense_table_with_int/legacy_tables-legacy_ka_cql_created_dense_table_with_int-ka-1-Data.db
deleted file mode 100644
index bdad431..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_cql_created_dense_table_with_int/legacy_tables-legacy_ka_cql_created_dense_table_with_int-ka-1-Data.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_cql_created_dense_table_with_int/legacy_tables-legacy_ka_cql_created_dense_table_with_int-ka-1-Digest.sha1 b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_cql_created_dense_table_with_int/legacy_tables-legacy_ka_cql_created_dense_table_with_int-ka-1-Digest.sha1
deleted file mode 100644
index f9e4b9c..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_cql_created_dense_table_with_int/legacy_tables-legacy_ka_cql_created_dense_table_with_int-ka-1-Digest.sha1
+++ /dev/null
@@ -1 +0,0 @@
-1334250623
\ No newline at end of file
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_cql_created_dense_table_with_int/legacy_tables-legacy_ka_cql_created_dense_table_with_int-ka-1-Filter.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_cql_created_dense_table_with_int/legacy_tables-legacy_ka_cql_created_dense_table_with_int-ka-1-Filter.db
deleted file mode 100644
index b6728a1..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_cql_created_dense_table_with_int/legacy_tables-legacy_ka_cql_created_dense_table_with_int-ka-1-Filter.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_cql_created_dense_table_with_int/legacy_tables-legacy_ka_cql_created_dense_table_with_int-ka-1-Index.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_cql_created_dense_table_with_int/legacy_tables-legacy_ka_cql_created_dense_table_with_int-ka-1-Index.db
deleted file mode 100644
index 64e12cd..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_cql_created_dense_table_with_int/legacy_tables-legacy_ka_cql_created_dense_table_with_int-ka-1-Index.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_cql_created_dense_table_with_int/legacy_tables-legacy_ka_cql_created_dense_table_with_int-ka-1-Statistics.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_cql_created_dense_table_with_int/legacy_tables-legacy_ka_cql_created_dense_table_with_int-ka-1-Statistics.db
deleted file mode 100644
index 13dc64a..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_cql_created_dense_table_with_int/legacy_tables-legacy_ka_cql_created_dense_table_with_int-ka-1-Statistics.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_cql_created_dense_table_with_int/legacy_tables-legacy_ka_cql_created_dense_table_with_int-ka-1-Summary.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_cql_created_dense_table_with_int/legacy_tables-legacy_ka_cql_created_dense_table_with_int-ka-1-Summary.db
deleted file mode 100644
index 76791c7..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_cql_created_dense_table_with_int/legacy_tables-legacy_ka_cql_created_dense_table_with_int-ka-1-Summary.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_cql_created_dense_table_with_int/legacy_tables-legacy_ka_cql_created_dense_table_with_int-ka-1-TOC.txt b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_cql_created_dense_table_with_int/legacy_tables-legacy_ka_cql_created_dense_table_with_int-ka-1-TOC.txt
deleted file mode 100644
index 402e1ab..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_cql_created_dense_table_with_int/legacy_tables-legacy_ka_cql_created_dense_table_with_int-ka-1-TOC.txt
+++ /dev/null
@@ -1,8 +0,0 @@
-Digest.sha1
-CompressionInfo.db
-Summary.db
-Statistics.db
-Data.db
-Index.db
-TOC.txt
-Filter.db
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_flushed_multi_block_rt/legacy_tables-legacy_ka_flushed_multi_block_rt-ka-1-CompressionInfo.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_flushed_multi_block_rt/legacy_tables-legacy_ka_flushed_multi_block_rt-ka-1-CompressionInfo.db
deleted file mode 100644
index 2336902..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_flushed_multi_block_rt/legacy_tables-legacy_ka_flushed_multi_block_rt-ka-1-CompressionInfo.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_flushed_multi_block_rt/legacy_tables-legacy_ka_flushed_multi_block_rt-ka-1-Data.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_flushed_multi_block_rt/legacy_tables-legacy_ka_flushed_multi_block_rt-ka-1-Data.db
deleted file mode 100644
index e7a9fd7..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_flushed_multi_block_rt/legacy_tables-legacy_ka_flushed_multi_block_rt-ka-1-Data.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_flushed_multi_block_rt/legacy_tables-legacy_ka_flushed_multi_block_rt-ka-1-Digest.sha1 b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_flushed_multi_block_rt/legacy_tables-legacy_ka_flushed_multi_block_rt-ka-1-Digest.sha1
deleted file mode 100644
index bfe4bc3..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_flushed_multi_block_rt/legacy_tables-legacy_ka_flushed_multi_block_rt-ka-1-Digest.sha1
+++ /dev/null
@@ -1 +0,0 @@
-3995406674
\ No newline at end of file
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_flushed_multi_block_rt/legacy_tables-legacy_ka_flushed_multi_block_rt-ka-1-Filter.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_flushed_multi_block_rt/legacy_tables-legacy_ka_flushed_multi_block_rt-ka-1-Filter.db
deleted file mode 100644
index 606783d..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_flushed_multi_block_rt/legacy_tables-legacy_ka_flushed_multi_block_rt-ka-1-Filter.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_flushed_multi_block_rt/legacy_tables-legacy_ka_flushed_multi_block_rt-ka-1-Index.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_flushed_multi_block_rt/legacy_tables-legacy_ka_flushed_multi_block_rt-ka-1-Index.db
deleted file mode 100644
index 1faa378..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_flushed_multi_block_rt/legacy_tables-legacy_ka_flushed_multi_block_rt-ka-1-Index.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_flushed_multi_block_rt/legacy_tables-legacy_ka_flushed_multi_block_rt-ka-1-Statistics.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_flushed_multi_block_rt/legacy_tables-legacy_ka_flushed_multi_block_rt-ka-1-Statistics.db
deleted file mode 100644
index 0070c96..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_flushed_multi_block_rt/legacy_tables-legacy_ka_flushed_multi_block_rt-ka-1-Statistics.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_flushed_multi_block_rt/legacy_tables-legacy_ka_flushed_multi_block_rt-ka-1-Summary.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_flushed_multi_block_rt/legacy_tables-legacy_ka_flushed_multi_block_rt-ka-1-Summary.db
deleted file mode 100644
index d2adbfa..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_flushed_multi_block_rt/legacy_tables-legacy_ka_flushed_multi_block_rt-ka-1-Summary.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_flushed_multi_block_rt/legacy_tables-legacy_ka_flushed_multi_block_rt-ka-1-TOC.txt b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_flushed_multi_block_rt/legacy_tables-legacy_ka_flushed_multi_block_rt-ka-1-TOC.txt
deleted file mode 100644
index 3fc5eec..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_flushed_multi_block_rt/legacy_tables-legacy_ka_flushed_multi_block_rt-ka-1-TOC.txt
+++ /dev/null
@@ -1,8 +0,0 @@
-Summary.db
-Digest.sha1
-CompressionInfo.db
-TOC.txt
-Filter.db
-Data.db
-Index.db
-Statistics.db
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-CompressionInfo.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-CompressionInfo.db
deleted file mode 100644
index ecd3ddb..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-CompressionInfo.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-Data.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-Data.db
deleted file mode 100644
index d1e4e2f..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-Data.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-Digest.sha1 b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-Digest.sha1
deleted file mode 100644
index bce117c..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-Digest.sha1
+++ /dev/null
@@ -1 +0,0 @@
-76435450
\ No newline at end of file
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-Filter.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-Filter.db
deleted file mode 100644
index 00a88b4..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-Filter.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-Index.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-Index.db
deleted file mode 100644
index 9ba4894..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-Index.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-Statistics.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-Statistics.db
deleted file mode 100644
index a57d32b..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-Statistics.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-Summary.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-Summary.db
deleted file mode 100644
index d60d8f4..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-Summary.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-TOC.txt b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-TOC.txt
deleted file mode 100644
index 25fc863..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-TOC.txt
+++ /dev/null
@@ -1,8 +0,0 @@
-Data.db
-Summary.db
-TOC.txt
-Statistics.db
-Digest.sha1
-Filter.db
-Index.db
-CompressionInfo.db
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-CompressionInfo.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-CompressionInfo.db
deleted file mode 100644
index 09c4cfa..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-CompressionInfo.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-Data.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-Data.db
deleted file mode 100644
index 40ee3c6..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-Data.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-Digest.sha1 b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-Digest.sha1
deleted file mode 100644
index 55ac08c..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-Digest.sha1
+++ /dev/null
@@ -1 +0,0 @@
-3851004816
\ No newline at end of file
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-Filter.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-Filter.db
deleted file mode 100644
index 00a88b4..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-Filter.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-Index.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-Index.db
deleted file mode 100644
index fb6ceed..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-Index.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-Statistics.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-Statistics.db
deleted file mode 100644
index b08f500..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-Statistics.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-Summary.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-Summary.db
deleted file mode 100644
index d60d8f4..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-Summary.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-TOC.txt b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-TOC.txt
deleted file mode 100644
index 6865eca..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-TOC.txt
+++ /dev/null
@@ -1,8 +0,0 @@
-CompressionInfo.db
-Summary.db
-Data.db
-Index.db
-Statistics.db
-TOC.txt
-Digest.sha1
-Filter.db
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_repeated_rt/Keyspace1-legacy_ka_repeated_rt-ka-1-CompressionInfo.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_repeated_rt/Keyspace1-legacy_ka_repeated_rt-ka-1-CompressionInfo.db
deleted file mode 100644
index 9a33154..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_repeated_rt/Keyspace1-legacy_ka_repeated_rt-ka-1-CompressionInfo.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_repeated_rt/Keyspace1-legacy_ka_repeated_rt-ka-1-Data.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_repeated_rt/Keyspace1-legacy_ka_repeated_rt-ka-1-Data.db
deleted file mode 100644
index 80a7c46..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_repeated_rt/Keyspace1-legacy_ka_repeated_rt-ka-1-Data.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_repeated_rt/Keyspace1-legacy_ka_repeated_rt-ka-1-Digest.sha1 b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_repeated_rt/Keyspace1-legacy_ka_repeated_rt-ka-1-Digest.sha1
deleted file mode 100644
index de07755..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_repeated_rt/Keyspace1-legacy_ka_repeated_rt-ka-1-Digest.sha1
+++ /dev/null
@@ -1 +0,0 @@
-1973536272
\ No newline at end of file
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_repeated_rt/Keyspace1-legacy_ka_repeated_rt-ka-1-Filter.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_repeated_rt/Keyspace1-legacy_ka_repeated_rt-ka-1-Filter.db
deleted file mode 100644
index dfcab1f..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_repeated_rt/Keyspace1-legacy_ka_repeated_rt-ka-1-Filter.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_repeated_rt/Keyspace1-legacy_ka_repeated_rt-ka-1-Index.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_repeated_rt/Keyspace1-legacy_ka_repeated_rt-ka-1-Index.db
deleted file mode 100644
index 9fefd10..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_repeated_rt/Keyspace1-legacy_ka_repeated_rt-ka-1-Index.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_repeated_rt/Keyspace1-legacy_ka_repeated_rt-ka-1-Statistics.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_repeated_rt/Keyspace1-legacy_ka_repeated_rt-ka-1-Statistics.db
deleted file mode 100644
index 77c6233..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_repeated_rt/Keyspace1-legacy_ka_repeated_rt-ka-1-Statistics.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_repeated_rt/Keyspace1-legacy_ka_repeated_rt-ka-1-Summary.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_repeated_rt/Keyspace1-legacy_ka_repeated_rt-ka-1-Summary.db
deleted file mode 100644
index 0c15fd4..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_repeated_rt/Keyspace1-legacy_ka_repeated_rt-ka-1-Summary.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_repeated_rt/Keyspace1-legacy_ka_repeated_rt-ka-1-TOC.txt b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_repeated_rt/Keyspace1-legacy_ka_repeated_rt-ka-1-TOC.txt
deleted file mode 100644
index a78243a..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_repeated_rt/Keyspace1-legacy_ka_repeated_rt-ka-1-TOC.txt
+++ /dev/null
@@ -1,8 +0,0 @@
-Data.db
-TOC.txt
-Digest.sha1
-Index.db
-CompressionInfo.db
-Filter.db
-Summary.db
-Statistics.db
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple/legacy_tables-legacy_ka_simple-ka-1-CompressionInfo.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple/legacy_tables-legacy_ka_simple-ka-1-CompressionInfo.db
deleted file mode 100644
index c80e64c..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple/legacy_tables-legacy_ka_simple-ka-1-CompressionInfo.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple/legacy_tables-legacy_ka_simple-ka-1-Data.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple/legacy_tables-legacy_ka_simple-ka-1-Data.db
deleted file mode 100644
index b29a26a..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple/legacy_tables-legacy_ka_simple-ka-1-Data.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple/legacy_tables-legacy_ka_simple-ka-1-Digest.sha1 b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple/legacy_tables-legacy_ka_simple-ka-1-Digest.sha1
deleted file mode 100644
index c889c8d..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple/legacy_tables-legacy_ka_simple-ka-1-Digest.sha1
+++ /dev/null
@@ -1 +0,0 @@
-2802392853
\ No newline at end of file
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple/legacy_tables-legacy_ka_simple-ka-1-Filter.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple/legacy_tables-legacy_ka_simple-ka-1-Filter.db
deleted file mode 100644
index c3cb27c..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple/legacy_tables-legacy_ka_simple-ka-1-Filter.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple/legacy_tables-legacy_ka_simple-ka-1-Index.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple/legacy_tables-legacy_ka_simple-ka-1-Index.db
deleted file mode 100644
index f0717e0..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple/legacy_tables-legacy_ka_simple-ka-1-Index.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple/legacy_tables-legacy_ka_simple-ka-1-Statistics.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple/legacy_tables-legacy_ka_simple-ka-1-Statistics.db
deleted file mode 100644
index 2af5467..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple/legacy_tables-legacy_ka_simple-ka-1-Statistics.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple/legacy_tables-legacy_ka_simple-ka-1-Summary.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple/legacy_tables-legacy_ka_simple-ka-1-Summary.db
deleted file mode 100644
index 35b5e22..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple/legacy_tables-legacy_ka_simple-ka-1-Summary.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple/legacy_tables-legacy_ka_simple-ka-1-TOC.txt b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple/legacy_tables-legacy_ka_simple-ka-1-TOC.txt
deleted file mode 100644
index 7be41d8..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple/legacy_tables-legacy_ka_simple-ka-1-TOC.txt
+++ /dev/null
@@ -1,8 +0,0 @@
-Index.db
-Digest.sha1
-CompressionInfo.db
-Data.db
-Statistics.db
-Summary.db
-TOC.txt
-Filter.db
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple_compact/legacy_tables-legacy_ka_simple_compact-ka-1-CompressionInfo.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple_compact/legacy_tables-legacy_ka_simple_compact-ka-1-CompressionInfo.db
deleted file mode 100644
index d530b73..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple_compact/legacy_tables-legacy_ka_simple_compact-ka-1-CompressionInfo.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple_compact/legacy_tables-legacy_ka_simple_compact-ka-1-Data.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple_compact/legacy_tables-legacy_ka_simple_compact-ka-1-Data.db
deleted file mode 100644
index 6a38c52..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple_compact/legacy_tables-legacy_ka_simple_compact-ka-1-Data.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple_compact/legacy_tables-legacy_ka_simple_compact-ka-1-Digest.sha1 b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple_compact/legacy_tables-legacy_ka_simple_compact-ka-1-Digest.sha1
deleted file mode 100644
index be8e5fb..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple_compact/legacy_tables-legacy_ka_simple_compact-ka-1-Digest.sha1
+++ /dev/null
@@ -1 +0,0 @@
-606280675
\ No newline at end of file
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple_compact/legacy_tables-legacy_ka_simple_compact-ka-1-Filter.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple_compact/legacy_tables-legacy_ka_simple_compact-ka-1-Filter.db
deleted file mode 100644
index c3cb27c..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple_compact/legacy_tables-legacy_ka_simple_compact-ka-1-Filter.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple_compact/legacy_tables-legacy_ka_simple_compact-ka-1-Index.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple_compact/legacy_tables-legacy_ka_simple_compact-ka-1-Index.db
deleted file mode 100644
index d2ec218..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple_compact/legacy_tables-legacy_ka_simple_compact-ka-1-Index.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple_compact/legacy_tables-legacy_ka_simple_compact-ka-1-Statistics.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple_compact/legacy_tables-legacy_ka_simple_compact-ka-1-Statistics.db
deleted file mode 100644
index e3fd855..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple_compact/legacy_tables-legacy_ka_simple_compact-ka-1-Statistics.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple_compact/legacy_tables-legacy_ka_simple_compact-ka-1-Summary.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple_compact/legacy_tables-legacy_ka_simple_compact-ka-1-Summary.db
deleted file mode 100644
index af8ad8b..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple_compact/legacy_tables-legacy_ka_simple_compact-ka-1-Summary.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple_compact/legacy_tables-legacy_ka_simple_compact-ka-1-TOC.txt b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple_compact/legacy_tables-legacy_ka_simple_compact-ka-1-TOC.txt
deleted file mode 100644
index 7f7fe79..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple_compact/legacy_tables-legacy_ka_simple_compact-ka-1-TOC.txt
+++ /dev/null
@@ -1,8 +0,0 @@
-Filter.db
-TOC.txt
-Statistics.db
-Summary.db
-Index.db
-Data.db
-Digest.sha1
-CompressionInfo.db
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple_counter/legacy_tables-legacy_ka_simple_counter-ka-1-CompressionInfo.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple_counter/legacy_tables-legacy_ka_simple_counter-ka-1-CompressionInfo.db
deleted file mode 100644
index 9c3416e..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple_counter/legacy_tables-legacy_ka_simple_counter-ka-1-CompressionInfo.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple_counter/legacy_tables-legacy_ka_simple_counter-ka-1-Data.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple_counter/legacy_tables-legacy_ka_simple_counter-ka-1-Data.db
deleted file mode 100644
index 1aee64c..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple_counter/legacy_tables-legacy_ka_simple_counter-ka-1-Data.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple_counter/legacy_tables-legacy_ka_simple_counter-ka-1-Digest.sha1 b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple_counter/legacy_tables-legacy_ka_simple_counter-ka-1-Digest.sha1
deleted file mode 100644
index 3da96e6..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple_counter/legacy_tables-legacy_ka_simple_counter-ka-1-Digest.sha1
+++ /dev/null
@@ -1 +0,0 @@
-3671794375
\ No newline at end of file
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple_counter/legacy_tables-legacy_ka_simple_counter-ka-1-Filter.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple_counter/legacy_tables-legacy_ka_simple_counter-ka-1-Filter.db
deleted file mode 100644
index c3cb27c..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple_counter/legacy_tables-legacy_ka_simple_counter-ka-1-Filter.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple_counter/legacy_tables-legacy_ka_simple_counter-ka-1-Index.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple_counter/legacy_tables-legacy_ka_simple_counter-ka-1-Index.db
deleted file mode 100644
index 932936c..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple_counter/legacy_tables-legacy_ka_simple_counter-ka-1-Index.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple_counter/legacy_tables-legacy_ka_simple_counter-ka-1-Statistics.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple_counter/legacy_tables-legacy_ka_simple_counter-ka-1-Statistics.db
deleted file mode 100644
index fa74e4b..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple_counter/legacy_tables-legacy_ka_simple_counter-ka-1-Statistics.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple_counter/legacy_tables-legacy_ka_simple_counter-ka-1-Summary.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple_counter/legacy_tables-legacy_ka_simple_counter-ka-1-Summary.db
deleted file mode 100644
index 35b5e22..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple_counter/legacy_tables-legacy_ka_simple_counter-ka-1-Summary.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple_counter/legacy_tables-legacy_ka_simple_counter-ka-1-TOC.txt b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple_counter/legacy_tables-legacy_ka_simple_counter-ka-1-TOC.txt
deleted file mode 100644
index 7be41d8..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple_counter/legacy_tables-legacy_ka_simple_counter-ka-1-TOC.txt
+++ /dev/null
@@ -1,8 +0,0 @@
-Index.db
-Digest.sha1
-CompressionInfo.db
-Data.db
-Statistics.db
-Summary.db
-TOC.txt
-Filter.db
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple_counter_compact/legacy_tables-legacy_ka_simple_counter_compact-ka-1-CompressionInfo.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple_counter_compact/legacy_tables-legacy_ka_simple_counter_compact-ka-1-CompressionInfo.db
deleted file mode 100644
index 01c5478..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple_counter_compact/legacy_tables-legacy_ka_simple_counter_compact-ka-1-CompressionInfo.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple_counter_compact/legacy_tables-legacy_ka_simple_counter_compact-ka-1-Data.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple_counter_compact/legacy_tables-legacy_ka_simple_counter_compact-ka-1-Data.db
deleted file mode 100644
index 5f4a7db..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple_counter_compact/legacy_tables-legacy_ka_simple_counter_compact-ka-1-Data.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple_counter_compact/legacy_tables-legacy_ka_simple_counter_compact-ka-1-Digest.sha1 b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple_counter_compact/legacy_tables-legacy_ka_simple_counter_compact-ka-1-Digest.sha1
deleted file mode 100644
index a71f766..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple_counter_compact/legacy_tables-legacy_ka_simple_counter_compact-ka-1-Digest.sha1
+++ /dev/null
@@ -1 +0,0 @@
-616768162
\ No newline at end of file
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple_counter_compact/legacy_tables-legacy_ka_simple_counter_compact-ka-1-Filter.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple_counter_compact/legacy_tables-legacy_ka_simple_counter_compact-ka-1-Filter.db
deleted file mode 100644
index c3cb27c..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple_counter_compact/legacy_tables-legacy_ka_simple_counter_compact-ka-1-Filter.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple_counter_compact/legacy_tables-legacy_ka_simple_counter_compact-ka-1-Index.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple_counter_compact/legacy_tables-legacy_ka_simple_counter_compact-ka-1-Index.db
deleted file mode 100644
index 48c153c..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple_counter_compact/legacy_tables-legacy_ka_simple_counter_compact-ka-1-Index.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple_counter_compact/legacy_tables-legacy_ka_simple_counter_compact-ka-1-Statistics.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple_counter_compact/legacy_tables-legacy_ka_simple_counter_compact-ka-1-Statistics.db
deleted file mode 100644
index 4a6e940..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple_counter_compact/legacy_tables-legacy_ka_simple_counter_compact-ka-1-Statistics.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple_counter_compact/legacy_tables-legacy_ka_simple_counter_compact-ka-1-Summary.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple_counter_compact/legacy_tables-legacy_ka_simple_counter_compact-ka-1-Summary.db
deleted file mode 100644
index af8ad8b..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple_counter_compact/legacy_tables-legacy_ka_simple_counter_compact-ka-1-Summary.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple_counter_compact/legacy_tables-legacy_ka_simple_counter_compact-ka-1-TOC.txt b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple_counter_compact/legacy_tables-legacy_ka_simple_counter_compact-ka-1-TOC.txt
deleted file mode 100644
index 7f7fe79..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple_counter_compact/legacy_tables-legacy_ka_simple_counter_compact-ka-1-TOC.txt
+++ /dev/null
@@ -1,8 +0,0 @@
-Filter.db
-TOC.txt
-Statistics.db
-Summary.db
-Index.db
-Data.db
-Digest.sha1
-CompressionInfo.db
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names/legacy_tables-legacy_ka_with_illegal_cell_names-ka-1-CRC.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names/legacy_tables-legacy_ka_with_illegal_cell_names-ka-1-CRC.db
deleted file mode 100644
index ee733ee..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names/legacy_tables-legacy_ka_with_illegal_cell_names-ka-1-CRC.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names/legacy_tables-legacy_ka_with_illegal_cell_names-ka-1-Data.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names/legacy_tables-legacy_ka_with_illegal_cell_names-ka-1-Data.db
deleted file mode 100644
index 6cf2e4c..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names/legacy_tables-legacy_ka_with_illegal_cell_names-ka-1-Data.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names/legacy_tables-legacy_ka_with_illegal_cell_names-ka-1-Digest.sha1 b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names/legacy_tables-legacy_ka_with_illegal_cell_names-ka-1-Digest.sha1
deleted file mode 100644
index f419fd2..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names/legacy_tables-legacy_ka_with_illegal_cell_names-ka-1-Digest.sha1
+++ /dev/null
@@ -1 +0,0 @@
-3673239127
\ No newline at end of file
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names/legacy_tables-legacy_ka_with_illegal_cell_names-ka-1-Filter.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names/legacy_tables-legacy_ka_with_illegal_cell_names-ka-1-Filter.db
deleted file mode 100644
index f8e53be..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names/legacy_tables-legacy_ka_with_illegal_cell_names-ka-1-Filter.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names/legacy_tables-legacy_ka_with_illegal_cell_names-ka-1-Index.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names/legacy_tables-legacy_ka_with_illegal_cell_names-ka-1-Index.db
deleted file mode 100644
index d6d8130..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names/legacy_tables-legacy_ka_with_illegal_cell_names-ka-1-Index.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names/legacy_tables-legacy_ka_with_illegal_cell_names-ka-1-Statistics.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names/legacy_tables-legacy_ka_with_illegal_cell_names-ka-1-Statistics.db
deleted file mode 100644
index 281b3da..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names/legacy_tables-legacy_ka_with_illegal_cell_names-ka-1-Statistics.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names/legacy_tables-legacy_ka_with_illegal_cell_names-ka-1-Summary.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names/legacy_tables-legacy_ka_with_illegal_cell_names-ka-1-Summary.db
deleted file mode 100644
index f2a5cd5..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names/legacy_tables-legacy_ka_with_illegal_cell_names-ka-1-Summary.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names/legacy_tables-legacy_ka_with_illegal_cell_names-ka-1-TOC.txt b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names/legacy_tables-legacy_ka_with_illegal_cell_names-ka-1-TOC.txt
deleted file mode 100644
index 497e06b..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names/legacy_tables-legacy_ka_with_illegal_cell_names-ka-1-TOC.txt
+++ /dev/null
@@ -1,8 +0,0 @@
-Statistics.db
-CRC.db
-Data.db
-TOC.txt
-Filter.db
-Index.db
-Digest.sha1
-Summary.db
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names/legacy_tables-legacy_ka_with_illegal_cell_names-ka-2-CompressionInfo.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names/legacy_tables-legacy_ka_with_illegal_cell_names-ka-2-CompressionInfo.db
deleted file mode 100644
index 26a0dbe..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names/legacy_tables-legacy_ka_with_illegal_cell_names-ka-2-CompressionInfo.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names/legacy_tables-legacy_ka_with_illegal_cell_names-ka-2-Data.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names/legacy_tables-legacy_ka_with_illegal_cell_names-ka-2-Data.db
deleted file mode 100644
index c805f7d..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names/legacy_tables-legacy_ka_with_illegal_cell_names-ka-2-Data.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names/legacy_tables-legacy_ka_with_illegal_cell_names-ka-2-Digest.sha1 b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names/legacy_tables-legacy_ka_with_illegal_cell_names-ka-2-Digest.sha1
deleted file mode 100644
index 0c696fb..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names/legacy_tables-legacy_ka_with_illegal_cell_names-ka-2-Digest.sha1
+++ /dev/null
@@ -1 +0,0 @@
-2529627719
\ No newline at end of file
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names/legacy_tables-legacy_ka_with_illegal_cell_names-ka-2-Filter.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names/legacy_tables-legacy_ka_with_illegal_cell_names-ka-2-Filter.db
deleted file mode 100644
index 5543328..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names/legacy_tables-legacy_ka_with_illegal_cell_names-ka-2-Filter.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names/legacy_tables-legacy_ka_with_illegal_cell_names-ka-2-Index.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names/legacy_tables-legacy_ka_with_illegal_cell_names-ka-2-Index.db
deleted file mode 100644
index fbdd950..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names/legacy_tables-legacy_ka_with_illegal_cell_names-ka-2-Index.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names/legacy_tables-legacy_ka_with_illegal_cell_names-ka-2-Statistics.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names/legacy_tables-legacy_ka_with_illegal_cell_names-ka-2-Statistics.db
deleted file mode 100644
index 0e471d4..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names/legacy_tables-legacy_ka_with_illegal_cell_names-ka-2-Statistics.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names/legacy_tables-legacy_ka_with_illegal_cell_names-ka-2-TOC.txt b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names/legacy_tables-legacy_ka_with_illegal_cell_names-ka-2-TOC.txt
deleted file mode 100644
index 1222811..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names/legacy_tables-legacy_ka_with_illegal_cell_names-ka-2-TOC.txt
+++ /dev/null
@@ -1,8 +0,0 @@
-Digest.sha1
-Data.db
-Filter.db
-Summary.db
-Index.db
-TOC.txt
-CompressionInfo.db
-Statistics.db
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names_2/legacy_tables-legacy_ka_with_illegal_cell_names_2-ka-1-CompressionInfo.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names_2/legacy_tables-legacy_ka_with_illegal_cell_names_2-ka-1-CompressionInfo.db
deleted file mode 100644
index 908f3b1..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names_2/legacy_tables-legacy_ka_with_illegal_cell_names_2-ka-1-CompressionInfo.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names_2/legacy_tables-legacy_ka_with_illegal_cell_names_2-ka-1-Data.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names_2/legacy_tables-legacy_ka_with_illegal_cell_names_2-ka-1-Data.db
deleted file mode 100644
index 33b88a0..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names_2/legacy_tables-legacy_ka_with_illegal_cell_names_2-ka-1-Data.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names_2/legacy_tables-legacy_ka_with_illegal_cell_names_2-ka-1-Digest.sha1 b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names_2/legacy_tables-legacy_ka_with_illegal_cell_names_2-ka-1-Digest.sha1
deleted file mode 100644
index 20deb5b..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names_2/legacy_tables-legacy_ka_with_illegal_cell_names_2-ka-1-Digest.sha1
+++ /dev/null
@@ -1 +0,0 @@
-3340111295
\ No newline at end of file
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names_2/legacy_tables-legacy_ka_with_illegal_cell_names_2-ka-1-Filter.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names_2/legacy_tables-legacy_ka_with_illegal_cell_names_2-ka-1-Filter.db
deleted file mode 100644
index 5543328..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names_2/legacy_tables-legacy_ka_with_illegal_cell_names_2-ka-1-Filter.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names_2/legacy_tables-legacy_ka_with_illegal_cell_names_2-ka-1-Index.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names_2/legacy_tables-legacy_ka_with_illegal_cell_names_2-ka-1-Index.db
deleted file mode 100644
index fbdd950..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names_2/legacy_tables-legacy_ka_with_illegal_cell_names_2-ka-1-Index.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names_2/legacy_tables-legacy_ka_with_illegal_cell_names_2-ka-1-Statistics.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names_2/legacy_tables-legacy_ka_with_illegal_cell_names_2-ka-1-Statistics.db
deleted file mode 100644
index f83575c..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names_2/legacy_tables-legacy_ka_with_illegal_cell_names_2-ka-1-Statistics.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names_2/legacy_tables-legacy_ka_with_illegal_cell_names_2-ka-1-Summary.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names_2/legacy_tables-legacy_ka_with_illegal_cell_names_2-ka-1-Summary.db
deleted file mode 100644
index 9b90005..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names_2/legacy_tables-legacy_ka_with_illegal_cell_names_2-ka-1-Summary.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names_2/legacy_tables-legacy_ka_with_illegal_cell_names_2-ka-1-TOC.txt b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names_2/legacy_tables-legacy_ka_with_illegal_cell_names_2-ka-1-TOC.txt
deleted file mode 100644
index 8d621be..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names_2/legacy_tables-legacy_ka_with_illegal_cell_names_2-ka-1-TOC.txt
+++ /dev/null
@@ -1,8 +0,0 @@
-Data.db
-TOC.txt
-Filter.db
-Summary.db
-CompressionInfo.db
-Statistics.db
-Digest.sha1
-Index.db
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names_indexed/legacy_tables-legacy_ka_with_illegal_cell_names_indexed-ka-1-CRC.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names_indexed/legacy_tables-legacy_ka_with_illegal_cell_names_indexed-ka-1-CRC.db
deleted file mode 100644
index 82ca06a..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names_indexed/legacy_tables-legacy_ka_with_illegal_cell_names_indexed-ka-1-CRC.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names_indexed/legacy_tables-legacy_ka_with_illegal_cell_names_indexed-ka-1-Data.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names_indexed/legacy_tables-legacy_ka_with_illegal_cell_names_indexed-ka-1-Data.db
deleted file mode 100644
index 269a739..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names_indexed/legacy_tables-legacy_ka_with_illegal_cell_names_indexed-ka-1-Data.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names_indexed/legacy_tables-legacy_ka_with_illegal_cell_names_indexed-ka-1-Digest.sha1 b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names_indexed/legacy_tables-legacy_ka_with_illegal_cell_names_indexed-ka-1-Digest.sha1
deleted file mode 100644
index 7c85191..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names_indexed/legacy_tables-legacy_ka_with_illegal_cell_names_indexed-ka-1-Digest.sha1
+++ /dev/null
@@ -1 +0,0 @@
-1999183849
\ No newline at end of file
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names_indexed/legacy_tables-legacy_ka_with_illegal_cell_names_indexed-ka-1-Filter.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names_indexed/legacy_tables-legacy_ka_with_illegal_cell_names_indexed-ka-1-Filter.db
deleted file mode 100644
index f3f7da5..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names_indexed/legacy_tables-legacy_ka_with_illegal_cell_names_indexed-ka-1-Filter.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names_indexed/legacy_tables-legacy_ka_with_illegal_cell_names_indexed-ka-1-Index.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names_indexed/legacy_tables-legacy_ka_with_illegal_cell_names_indexed-ka-1-Index.db
deleted file mode 100644
index bff0123..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names_indexed/legacy_tables-legacy_ka_with_illegal_cell_names_indexed-ka-1-Index.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names_indexed/legacy_tables-legacy_ka_with_illegal_cell_names_indexed-ka-1-Statistics.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names_indexed/legacy_tables-legacy_ka_with_illegal_cell_names_indexed-ka-1-Statistics.db
deleted file mode 100644
index febb2be..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names_indexed/legacy_tables-legacy_ka_with_illegal_cell_names_indexed-ka-1-Statistics.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names_indexed/legacy_tables-legacy_ka_with_illegal_cell_names_indexed-ka-1-TOC.txt b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names_indexed/legacy_tables-legacy_ka_with_illegal_cell_names_indexed-ka-1-TOC.txt
deleted file mode 100644
index c360dbf..0000000
--- a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_with_illegal_cell_names_indexed/legacy_tables-legacy_ka_with_illegal_cell_names_indexed-ka-1-TOC.txt
+++ /dev/null
@@ -1,8 +0,0 @@
-CRC.db
-Statistics.db
-TOC.txt
-Data.db
-Index.db
-Summary.db
-Digest.sha1
-Filter.db
diff --git a/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust/la-1-big-CompressionInfo.db b/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust/la-1-big-CompressionInfo.db
deleted file mode 100644
index 13701c4..0000000
--- a/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust/la-1-big-CompressionInfo.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust/la-1-big-Data.db b/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust/la-1-big-Data.db
deleted file mode 100644
index f04344a..0000000
--- a/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust/la-1-big-Data.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust/la-1-big-Digest.adler32 b/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust/la-1-big-Digest.adler32
deleted file mode 100644
index d6157b2..0000000
--- a/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust/la-1-big-Digest.adler32
+++ /dev/null
@@ -1 +0,0 @@
-1633775217
\ No newline at end of file
diff --git a/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust/la-1-big-Filter.db b/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust/la-1-big-Filter.db
deleted file mode 100644
index c3cb27c..0000000
--- a/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust/la-1-big-Filter.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust/la-1-big-Index.db b/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust/la-1-big-Index.db
deleted file mode 100644
index 44b89c4..0000000
--- a/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust/la-1-big-Index.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust/la-1-big-Statistics.db b/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust/la-1-big-Statistics.db
deleted file mode 100644
index a54d94d..0000000
--- a/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust/la-1-big-Statistics.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust/la-1-big-Summary.db b/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust/la-1-big-Summary.db
deleted file mode 100644
index 35b5e22..0000000
--- a/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust/la-1-big-Summary.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust/la-1-big-TOC.txt b/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust/la-1-big-TOC.txt
deleted file mode 100644
index dec3a3f..0000000
--- a/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust/la-1-big-TOC.txt
+++ /dev/null
@@ -1,8 +0,0 @@
-CompressionInfo.db
-Digest.adler32
-Filter.db
-Summary.db
-Data.db
-Statistics.db
-TOC.txt
-Index.db
diff --git a/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust_compact/la-1-big-CompressionInfo.db b/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust_compact/la-1-big-CompressionInfo.db
deleted file mode 100644
index 2a72f70..0000000
--- a/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust_compact/la-1-big-CompressionInfo.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust_compact/la-1-big-Data.db b/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust_compact/la-1-big-Data.db
deleted file mode 100644
index 6bc08d2..0000000
--- a/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust_compact/la-1-big-Data.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust_compact/la-1-big-Digest.adler32 b/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust_compact/la-1-big-Digest.adler32
deleted file mode 100644
index 943dd1e..0000000
--- a/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust_compact/la-1-big-Digest.adler32
+++ /dev/null
@@ -1 +0,0 @@
-1372047449
\ No newline at end of file
diff --git a/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust_compact/la-1-big-Filter.db b/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust_compact/la-1-big-Filter.db
deleted file mode 100644
index c3cb27c..0000000
--- a/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust_compact/la-1-big-Filter.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust_compact/la-1-big-Index.db b/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust_compact/la-1-big-Index.db
deleted file mode 100644
index 9e18f8e..0000000
--- a/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust_compact/la-1-big-Index.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust_compact/la-1-big-Statistics.db b/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust_compact/la-1-big-Statistics.db
deleted file mode 100644
index b2fd408..0000000
--- a/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust_compact/la-1-big-Statistics.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust_compact/la-1-big-Summary.db b/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust_compact/la-1-big-Summary.db
deleted file mode 100644
index 6cd998f..0000000
--- a/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust_compact/la-1-big-Summary.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust_compact/la-1-big-TOC.txt b/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust_compact/la-1-big-TOC.txt
deleted file mode 100644
index 0aef810..0000000
--- a/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust_compact/la-1-big-TOC.txt
+++ /dev/null
@@ -1,8 +0,0 @@
-TOC.txt
-Statistics.db
-Digest.adler32
-CompressionInfo.db
-Summary.db
-Data.db
-Filter.db
-Index.db
diff --git a/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust_counter/la-1-big-CompressionInfo.db b/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust_counter/la-1-big-CompressionInfo.db
deleted file mode 100644
index 0bdb82a..0000000
--- a/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust_counter/la-1-big-CompressionInfo.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust_counter/la-1-big-Data.db b/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust_counter/la-1-big-Data.db
deleted file mode 100644
index 76d4cbc..0000000
--- a/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust_counter/la-1-big-Data.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust_counter/la-1-big-Digest.adler32 b/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust_counter/la-1-big-Digest.adler32
deleted file mode 100644
index e704111..0000000
--- a/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust_counter/la-1-big-Digest.adler32
+++ /dev/null
@@ -1 +0,0 @@
-287946299
\ No newline at end of file
diff --git a/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust_counter/la-1-big-Filter.db b/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust_counter/la-1-big-Filter.db
deleted file mode 100644
index c3cb27c..0000000
--- a/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust_counter/la-1-big-Filter.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust_counter/la-1-big-Index.db b/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust_counter/la-1-big-Index.db
deleted file mode 100644
index 51ddf91..0000000
--- a/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust_counter/la-1-big-Index.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust_counter/la-1-big-Statistics.db b/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust_counter/la-1-big-Statistics.db
deleted file mode 100644
index b6ad155..0000000
--- a/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust_counter/la-1-big-Statistics.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust_counter/la-1-big-Summary.db b/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust_counter/la-1-big-Summary.db
deleted file mode 100644
index 35b5e22..0000000
--- a/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust_counter/la-1-big-Summary.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust_counter/la-1-big-TOC.txt b/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust_counter/la-1-big-TOC.txt
deleted file mode 100644
index dec3a3f..0000000
--- a/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust_counter/la-1-big-TOC.txt
+++ /dev/null
@@ -1,8 +0,0 @@
-CompressionInfo.db
-Digest.adler32
-Filter.db
-Summary.db
-Data.db
-Statistics.db
-TOC.txt
-Index.db
diff --git a/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust_counter_compact/la-1-big-CompressionInfo.db b/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust_counter_compact/la-1-big-CompressionInfo.db
deleted file mode 100644
index d4dec70..0000000
--- a/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust_counter_compact/la-1-big-CompressionInfo.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust_counter_compact/la-1-big-Data.db b/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust_counter_compact/la-1-big-Data.db
deleted file mode 100644
index 63ee721..0000000
--- a/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust_counter_compact/la-1-big-Data.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust_counter_compact/la-1-big-Digest.adler32 b/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust_counter_compact/la-1-big-Digest.adler32
deleted file mode 100644
index 577407e..0000000
--- a/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust_counter_compact/la-1-big-Digest.adler32
+++ /dev/null
@@ -1 +0,0 @@
-2583914481
\ No newline at end of file
diff --git a/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust_counter_compact/la-1-big-Filter.db b/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust_counter_compact/la-1-big-Filter.db
deleted file mode 100644
index c3cb27c..0000000
--- a/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust_counter_compact/la-1-big-Filter.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust_counter_compact/la-1-big-Index.db b/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust_counter_compact/la-1-big-Index.db
deleted file mode 100644
index 10df1e8..0000000
--- a/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust_counter_compact/la-1-big-Index.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust_counter_compact/la-1-big-Statistics.db b/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust_counter_compact/la-1-big-Statistics.db
deleted file mode 100644
index 2bfc59d..0000000
--- a/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust_counter_compact/la-1-big-Statistics.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust_counter_compact/la-1-big-Summary.db b/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust_counter_compact/la-1-big-Summary.db
deleted file mode 100644
index 6cd998f..0000000
--- a/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust_counter_compact/la-1-big-Summary.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust_counter_compact/la-1-big-TOC.txt b/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust_counter_compact/la-1-big-TOC.txt
deleted file mode 100644
index 0aef810..0000000
--- a/test/data/legacy-sstables/la/legacy_tables/legacy_la_clust_counter_compact/la-1-big-TOC.txt
+++ /dev/null
@@ -1,8 +0,0 @@
-TOC.txt
-Statistics.db
-Digest.adler32
-CompressionInfo.db
-Summary.db
-Data.db
-Filter.db
-Index.db
diff --git a/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple/la-1-big-CompressionInfo.db b/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple/la-1-big-CompressionInfo.db
deleted file mode 100644
index c80e64c..0000000
--- a/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple/la-1-big-CompressionInfo.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple/la-1-big-Data.db b/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple/la-1-big-Data.db
deleted file mode 100644
index ae136f5..0000000
--- a/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple/la-1-big-Data.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple/la-1-big-Digest.adler32 b/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple/la-1-big-Digest.adler32
deleted file mode 100644
index dacf8ac..0000000
--- a/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple/la-1-big-Digest.adler32
+++ /dev/null
@@ -1 +0,0 @@
-4239203875
\ No newline at end of file
diff --git a/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple/la-1-big-Filter.db b/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple/la-1-big-Filter.db
deleted file mode 100644
index c3cb27c..0000000
--- a/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple/la-1-big-Filter.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple/la-1-big-Index.db b/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple/la-1-big-Index.db
deleted file mode 100644
index f0717e0..0000000
--- a/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple/la-1-big-Index.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple/la-1-big-Statistics.db b/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple/la-1-big-Statistics.db
deleted file mode 100644
index 49b9275..0000000
--- a/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple/la-1-big-Statistics.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple/la-1-big-Summary.db b/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple/la-1-big-Summary.db
deleted file mode 100644
index 35b5e22..0000000
--- a/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple/la-1-big-Summary.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple/la-1-big-TOC.txt b/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple/la-1-big-TOC.txt
deleted file mode 100644
index dec3a3f..0000000
--- a/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple/la-1-big-TOC.txt
+++ /dev/null
@@ -1,8 +0,0 @@
-CompressionInfo.db
-Digest.adler32
-Filter.db
-Summary.db
-Data.db
-Statistics.db
-TOC.txt
-Index.db
diff --git a/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple_compact/la-1-big-CompressionInfo.db b/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple_compact/la-1-big-CompressionInfo.db
deleted file mode 100644
index d530b73..0000000
--- a/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple_compact/la-1-big-CompressionInfo.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple_compact/la-1-big-Data.db b/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple_compact/la-1-big-Data.db
deleted file mode 100644
index 2e912a1..0000000
--- a/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple_compact/la-1-big-Data.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple_compact/la-1-big-Digest.adler32 b/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple_compact/la-1-big-Digest.adler32
deleted file mode 100644
index c07a57f..0000000
--- a/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple_compact/la-1-big-Digest.adler32
+++ /dev/null
@@ -1 +0,0 @@
-278403976
\ No newline at end of file
diff --git a/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple_compact/la-1-big-Filter.db b/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple_compact/la-1-big-Filter.db
deleted file mode 100644
index c3cb27c..0000000
--- a/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple_compact/la-1-big-Filter.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple_compact/la-1-big-Index.db b/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple_compact/la-1-big-Index.db
deleted file mode 100644
index d2ec218..0000000
--- a/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple_compact/la-1-big-Index.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple_compact/la-1-big-Statistics.db b/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple_compact/la-1-big-Statistics.db
deleted file mode 100644
index a81e03e..0000000
--- a/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple_compact/la-1-big-Statistics.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple_compact/la-1-big-Summary.db b/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple_compact/la-1-big-Summary.db
deleted file mode 100644
index 6cd998f..0000000
--- a/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple_compact/la-1-big-Summary.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple_compact/la-1-big-TOC.txt b/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple_compact/la-1-big-TOC.txt
deleted file mode 100644
index 0aef810..0000000
--- a/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple_compact/la-1-big-TOC.txt
+++ /dev/null
@@ -1,8 +0,0 @@
-TOC.txt
-Statistics.db
-Digest.adler32
-CompressionInfo.db
-Summary.db
-Data.db
-Filter.db
-Index.db
diff --git a/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple_counter/la-1-big-CompressionInfo.db b/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple_counter/la-1-big-CompressionInfo.db
deleted file mode 100644
index 9c3416e..0000000
--- a/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple_counter/la-1-big-CompressionInfo.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple_counter/la-1-big-Data.db b/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple_counter/la-1-big-Data.db
deleted file mode 100644
index 010bd1a..0000000
--- a/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple_counter/la-1-big-Data.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple_counter/la-1-big-Digest.adler32 b/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple_counter/la-1-big-Digest.adler32
deleted file mode 100644
index 562547a..0000000
--- a/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple_counter/la-1-big-Digest.adler32
+++ /dev/null
@@ -1 +0,0 @@
-590029692
\ No newline at end of file
diff --git a/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple_counter/la-1-big-Filter.db b/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple_counter/la-1-big-Filter.db
deleted file mode 100644
index c3cb27c..0000000
--- a/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple_counter/la-1-big-Filter.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple_counter/la-1-big-Index.db b/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple_counter/la-1-big-Index.db
deleted file mode 100644
index 932936c..0000000
--- a/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple_counter/la-1-big-Index.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple_counter/la-1-big-Statistics.db b/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple_counter/la-1-big-Statistics.db
deleted file mode 100644
index 525a4b1..0000000
--- a/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple_counter/la-1-big-Statistics.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple_counter/la-1-big-Summary.db b/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple_counter/la-1-big-Summary.db
deleted file mode 100644
index 35b5e22..0000000
--- a/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple_counter/la-1-big-Summary.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple_counter/la-1-big-TOC.txt b/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple_counter/la-1-big-TOC.txt
deleted file mode 100644
index dec3a3f..0000000
--- a/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple_counter/la-1-big-TOC.txt
+++ /dev/null
@@ -1,8 +0,0 @@
-CompressionInfo.db
-Digest.adler32
-Filter.db
-Summary.db
-Data.db
-Statistics.db
-TOC.txt
-Index.db
diff --git a/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple_counter_compact/la-1-big-CompressionInfo.db b/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple_counter_compact/la-1-big-CompressionInfo.db
deleted file mode 100644
index 01c5478..0000000
--- a/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple_counter_compact/la-1-big-CompressionInfo.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple_counter_compact/la-1-big-Data.db b/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple_counter_compact/la-1-big-Data.db
deleted file mode 100644
index 323ff37..0000000
--- a/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple_counter_compact/la-1-big-Data.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple_counter_compact/la-1-big-Digest.adler32 b/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple_counter_compact/la-1-big-Digest.adler32
deleted file mode 100644
index 92237e7..0000000
--- a/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple_counter_compact/la-1-big-Digest.adler32
+++ /dev/null
@@ -1 +0,0 @@
-2048991053
\ No newline at end of file
diff --git a/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple_counter_compact/la-1-big-Filter.db b/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple_counter_compact/la-1-big-Filter.db
deleted file mode 100644
index c3cb27c..0000000
--- a/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple_counter_compact/la-1-big-Filter.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple_counter_compact/la-1-big-Index.db b/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple_counter_compact/la-1-big-Index.db
deleted file mode 100644
index 48c153c..0000000
--- a/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple_counter_compact/la-1-big-Index.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple_counter_compact/la-1-big-Statistics.db b/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple_counter_compact/la-1-big-Statistics.db
deleted file mode 100644
index 37324a7..0000000
--- a/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple_counter_compact/la-1-big-Statistics.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple_counter_compact/la-1-big-Summary.db b/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple_counter_compact/la-1-big-Summary.db
deleted file mode 100644
index 6cd998f..0000000
--- a/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple_counter_compact/la-1-big-Summary.db
+++ /dev/null
Binary files differ
diff --git a/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple_counter_compact/la-1-big-TOC.txt b/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple_counter_compact/la-1-big-TOC.txt
deleted file mode 100644
index 0aef810..0000000
--- a/test/data/legacy-sstables/la/legacy_tables/legacy_la_simple_counter_compact/la-1-big-TOC.txt
+++ /dev/null
@@ -1,8 +0,0 @@
-TOC.txt
-Statistics.db
-Digest.adler32
-CompressionInfo.db
-Summary.db
-Data.db
-Filter.db
-Index.db
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-CompressionInfo.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-CompressionInfo.db
new file mode 100644
index 0000000..8fad34f
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-CompressionInfo.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Data.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Data.db
new file mode 100644
index 0000000..ae35335
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Data.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Digest.crc32 b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Digest.crc32
new file mode 100644
index 0000000..8a92f3c
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Digest.crc32
@@ -0,0 +1 @@
+2977407251
\ No newline at end of file
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Filter.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Filter.db
new file mode 100644
index 0000000..8868e5c
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Filter.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Index.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Index.db
new file mode 100644
index 0000000..d50fdeb
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Index.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Statistics.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Statistics.db
new file mode 100644
index 0000000..7341864
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Statistics.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Summary.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Summary.db
new file mode 100644
index 0000000..9b24e04
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Summary.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-TOC.txt b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-TOC.txt
new file mode 100644
index 0000000..b03b283
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-TOC.txt
@@ -0,0 +1,8 @@
+Filter.db
+Digest.crc32
+Index.db
+TOC.txt
+Summary.db
+Statistics.db
+CompressionInfo.db
+Data.db
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-bti-CompressionInfo.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-bti-CompressionInfo.db
new file mode 100644
index 0000000..c96fb7d
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-bti-CompressionInfo.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-bti-Data.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-bti-Data.db
new file mode 100644
index 0000000..5ecab70
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-bti-Data.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-bti-Digest.crc32 b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-bti-Digest.crc32
new file mode 100644
index 0000000..e94e369
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-bti-Digest.crc32
@@ -0,0 +1 @@
+2666613329
\ No newline at end of file
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-bti-Filter.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-bti-Filter.db
new file mode 100644
index 0000000..2e1d5d2
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-bti-Filter.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-bti-Partitions.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-bti-Partitions.db
new file mode 100644
index 0000000..aded0e1
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-bti-Partitions.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-bti-Rows.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-bti-Rows.db
new file mode 100644
index 0000000..44803b4
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-bti-Rows.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-bti-Statistics.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-bti-Statistics.db
new file mode 100644
index 0000000..9e8a4b1
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-bti-Statistics.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-bti-TOC.txt b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-bti-TOC.txt
new file mode 100644
index 0000000..c20f4a8
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-bti-TOC.txt
@@ -0,0 +1,8 @@
+Data.db
+Filter.db
+Statistics.db
+CompressionInfo.db
+Partitions.db
+TOC.txt
+Rows.db
+Digest.crc32
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-CompressionInfo.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-CompressionInfo.db
new file mode 100644
index 0000000..6d3ac8e
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-CompressionInfo.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-Data.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-Data.db
new file mode 100644
index 0000000..fd6aebb
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-Data.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-Digest.crc32 b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-Digest.crc32
new file mode 100644
index 0000000..239b372
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-Digest.crc32
@@ -0,0 +1 @@
+3704038982
\ No newline at end of file
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-Filter.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-Filter.db
new file mode 100644
index 0000000..2e1d5d2
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-Filter.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-Index.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-Index.db
new file mode 100644
index 0000000..2e20951
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-Index.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-Statistics.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-Statistics.db
new file mode 100644
index 0000000..cd90137
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-Statistics.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-Summary.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-Summary.db
new file mode 100644
index 0000000..9b24e04
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-Summary.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-TOC.txt b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-TOC.txt
new file mode 100644
index 0000000..734a80d
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-TOC.txt
@@ -0,0 +1,8 @@
+Index.db
+TOC.txt
+Filter.db
+CompressionInfo.db
+Summary.db
+Data.db
+Statistics.db
+Digest.crc32
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-bti-CompressionInfo.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-bti-CompressionInfo.db
new file mode 100644
index 0000000..9c013f7
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-bti-CompressionInfo.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-bti-Data.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-bti-Data.db
new file mode 100644
index 0000000..1f54033
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-bti-Data.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-bti-Digest.crc32 b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-bti-Digest.crc32
new file mode 100644
index 0000000..e57848f
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-bti-Digest.crc32
@@ -0,0 +1 @@
+874495544
\ No newline at end of file
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-bti-Filter.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-bti-Filter.db
new file mode 100644
index 0000000..2e1d5d2
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-bti-Filter.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-bti-Partitions.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-bti-Partitions.db
new file mode 100644
index 0000000..aded0e1
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-bti-Partitions.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-bti-Rows.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-bti-Rows.db
new file mode 100644
index 0000000..6992a68
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-bti-Rows.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-bti-Statistics.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-bti-Statistics.db
new file mode 100644
index 0000000..05a8251
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-bti-Statistics.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-bti-TOC.txt b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-bti-TOC.txt
new file mode 100644
index 0000000..c20f4a8
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-bti-TOC.txt
@@ -0,0 +1,8 @@
+Data.db
+Filter.db
+Statistics.db
+CompressionInfo.db
+Partitions.db
+TOC.txt
+Rows.db
+Digest.crc32
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-CompressionInfo.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-CompressionInfo.db
new file mode 100644
index 0000000..f0a1cfb
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-CompressionInfo.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Data.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Data.db
new file mode 100644
index 0000000..b487fe8
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Data.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Digest.crc32 b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Digest.crc32
new file mode 100644
index 0000000..ca286e0
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Digest.crc32
@@ -0,0 +1 @@
+2759187708
\ No newline at end of file
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Filter.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Filter.db
new file mode 100644
index 0000000..8868e5c
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Filter.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Index.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Index.db
new file mode 100644
index 0000000..c981a22
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Index.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Statistics.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Statistics.db
new file mode 100644
index 0000000..33fccc9
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Statistics.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Summary.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Summary.db
new file mode 100644
index 0000000..9b24e04
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Summary.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-TOC.txt b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-TOC.txt
new file mode 100644
index 0000000..b03b283
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-TOC.txt
@@ -0,0 +1,8 @@
+Filter.db
+Digest.crc32
+Index.db
+TOC.txt
+Summary.db
+Statistics.db
+CompressionInfo.db
+Data.db
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-bti-CompressionInfo.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-bti-CompressionInfo.db
new file mode 100644
index 0000000..a8ab572
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-bti-CompressionInfo.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-bti-Data.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-bti-Data.db
new file mode 100644
index 0000000..3632c25
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-bti-Data.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-bti-Digest.crc32 b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-bti-Digest.crc32
new file mode 100644
index 0000000..a199ec3
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-bti-Digest.crc32
@@ -0,0 +1 @@
+2912620103
\ No newline at end of file
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-bti-Filter.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-bti-Filter.db
new file mode 100644
index 0000000..2e1d5d2
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-bti-Filter.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-bti-Partitions.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-bti-Partitions.db
new file mode 100644
index 0000000..aded0e1
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-bti-Partitions.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-bti-Rows.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-bti-Rows.db
new file mode 100644
index 0000000..b0aea85
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-bti-Rows.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-bti-Statistics.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-bti-Statistics.db
new file mode 100644
index 0000000..3b51c1b
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-bti-Statistics.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-bti-TOC.txt b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-bti-TOC.txt
new file mode 100644
index 0000000..c20f4a8
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-bti-TOC.txt
@@ -0,0 +1,8 @@
+Data.db
+Filter.db
+Statistics.db
+CompressionInfo.db
+Partitions.db
+TOC.txt
+Rows.db
+Digest.crc32
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-CompressionInfo.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-CompressionInfo.db
new file mode 100644
index 0000000..1b1f2da
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-CompressionInfo.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-Data.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-Data.db
new file mode 100644
index 0000000..1976981
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-Data.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-Digest.crc32 b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-Digest.crc32
new file mode 100644
index 0000000..22ed33b
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-Digest.crc32
@@ -0,0 +1 @@
+2742231118
\ No newline at end of file
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-Filter.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-Filter.db
new file mode 100644
index 0000000..2e1d5d2
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-Filter.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-Index.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-Index.db
new file mode 100644
index 0000000..022bc3d
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-Index.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-Statistics.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-Statistics.db
new file mode 100644
index 0000000..d393159
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-Statistics.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-Summary.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-Summary.db
new file mode 100644
index 0000000..9b24e04
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-Summary.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-TOC.txt b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-TOC.txt
new file mode 100644
index 0000000..734a80d
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-TOC.txt
@@ -0,0 +1,8 @@
+Index.db
+TOC.txt
+Filter.db
+CompressionInfo.db
+Summary.db
+Data.db
+Statistics.db
+Digest.crc32
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-bti-CompressionInfo.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-bti-CompressionInfo.db
new file mode 100644
index 0000000..b8a1dac
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-bti-CompressionInfo.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-bti-Data.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-bti-Data.db
new file mode 100644
index 0000000..f1c5c62
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-bti-Data.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-bti-Digest.crc32 b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-bti-Digest.crc32
new file mode 100644
index 0000000..7f68c2c
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-bti-Digest.crc32
@@ -0,0 +1 @@
+1995270006
\ No newline at end of file
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-bti-Filter.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-bti-Filter.db
new file mode 100644
index 0000000..2e1d5d2
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-bti-Filter.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-bti-Partitions.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-bti-Partitions.db
new file mode 100644
index 0000000..aded0e1
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-bti-Partitions.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-bti-Rows.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-bti-Rows.db
new file mode 100644
index 0000000..e63ee20
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-bti-Rows.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-bti-Statistics.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-bti-Statistics.db
new file mode 100644
index 0000000..e4fcc94
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-bti-Statistics.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-bti-TOC.txt b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-bti-TOC.txt
new file mode 100644
index 0000000..c20f4a8
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-bti-TOC.txt
@@ -0,0 +1,8 @@
+Data.db
+Filter.db
+Statistics.db
+CompressionInfo.db
+Partitions.db
+TOC.txt
+Rows.db
+Digest.crc32
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-CompressionInfo.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-CompressionInfo.db
new file mode 100644
index 0000000..fc38a25
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-CompressionInfo.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Data.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Data.db
new file mode 100644
index 0000000..11219d0
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Data.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Digest.crc32 b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Digest.crc32
new file mode 100644
index 0000000..985d6dc
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Digest.crc32
@@ -0,0 +1 @@
+462858821
\ No newline at end of file
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Filter.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Filter.db
new file mode 100644
index 0000000..8868e5c
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Filter.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Index.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Index.db
new file mode 100644
index 0000000..b3094bf
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Index.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Statistics.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Statistics.db
new file mode 100644
index 0000000..3c68ac5
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Statistics.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Summary.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Summary.db
new file mode 100644
index 0000000..9b24e04
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Summary.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-TOC.txt b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-TOC.txt
new file mode 100644
index 0000000..b03b283
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-TOC.txt
@@ -0,0 +1,8 @@
+Filter.db
+Digest.crc32
+Index.db
+TOC.txt
+Summary.db
+Statistics.db
+CompressionInfo.db
+Data.db
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-bti-CompressionInfo.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-bti-CompressionInfo.db
new file mode 100644
index 0000000..0b7faea
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-bti-CompressionInfo.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-bti-Data.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-bti-Data.db
new file mode 100644
index 0000000..277996b
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-bti-Data.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-bti-Digest.crc32 b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-bti-Digest.crc32
new file mode 100644
index 0000000..654f52b
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-bti-Digest.crc32
@@ -0,0 +1 @@
+4102718625
\ No newline at end of file
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-bti-Filter.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-bti-Filter.db
new file mode 100644
index 0000000..2e1d5d2
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-bti-Filter.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-bti-Partitions.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-bti-Partitions.db
new file mode 100644
index 0000000..f297888
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-bti-Partitions.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-bti-Rows.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-bti-Rows.db
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-bti-Rows.db
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-bti-Statistics.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-bti-Statistics.db
new file mode 100644
index 0000000..62b5d3f
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-bti-Statistics.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-bti-TOC.txt b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-bti-TOC.txt
new file mode 100644
index 0000000..c20f4a8
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-bti-TOC.txt
@@ -0,0 +1,8 @@
+Data.db
+Filter.db
+Statistics.db
+CompressionInfo.db
+Partitions.db
+TOC.txt
+Rows.db
+Digest.crc32
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-big-CompressionInfo.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-big-CompressionInfo.db
new file mode 100644
index 0000000..1c738aa
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-big-CompressionInfo.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-big-Data.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-big-Data.db
new file mode 100644
index 0000000..b9b836f
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-big-Data.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-big-Digest.crc32 b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-big-Digest.crc32
new file mode 100644
index 0000000..669bbcf
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-big-Digest.crc32
@@ -0,0 +1 @@
+1732739494
\ No newline at end of file
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-big-Filter.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-big-Filter.db
new file mode 100644
index 0000000..2e1d5d2
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-big-Filter.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-big-Index.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-big-Index.db
new file mode 100644
index 0000000..56f29df
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-big-Index.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-big-Statistics.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-big-Statistics.db
new file mode 100644
index 0000000..b112e58
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-big-Statistics.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-big-Summary.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-big-Summary.db
new file mode 100644
index 0000000..9b24e04
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-big-Summary.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-big-TOC.txt b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-big-TOC.txt
new file mode 100644
index 0000000..734a80d
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-big-TOC.txt
@@ -0,0 +1,8 @@
+Index.db
+TOC.txt
+Filter.db
+CompressionInfo.db
+Summary.db
+Data.db
+Statistics.db
+Digest.crc32
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-bti-CompressionInfo.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-bti-CompressionInfo.db
new file mode 100644
index 0000000..adb7fc4
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-bti-CompressionInfo.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-bti-Data.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-bti-Data.db
new file mode 100644
index 0000000..68f29ba
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-bti-Data.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-bti-Digest.crc32 b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-bti-Digest.crc32
new file mode 100644
index 0000000..bf77552
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-bti-Digest.crc32
@@ -0,0 +1 @@
+380992464
\ No newline at end of file
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-bti-Filter.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-bti-Filter.db
new file mode 100644
index 0000000..2e1d5d2
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-bti-Filter.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-bti-Partitions.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-bti-Partitions.db
new file mode 100644
index 0000000..e4f9ea9
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-bti-Partitions.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-bti-Rows.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-bti-Rows.db
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-bti-Rows.db
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-bti-Statistics.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-bti-Statistics.db
new file mode 100644
index 0000000..b853cfb
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-bti-Statistics.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-bti-TOC.txt b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-bti-TOC.txt
new file mode 100644
index 0000000..c20f4a8
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-bti-TOC.txt
@@ -0,0 +1,8 @@
+Data.db
+Filter.db
+Statistics.db
+CompressionInfo.db
+Partitions.db
+TOC.txt
+Rows.db
+Digest.crc32
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-CompressionInfo.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-CompressionInfo.db
new file mode 100644
index 0000000..e2860e1
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-CompressionInfo.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Data.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Data.db
new file mode 100644
index 0000000..620cdf2
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Data.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Digest.crc32 b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Digest.crc32
new file mode 100644
index 0000000..bc5f671
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Digest.crc32
@@ -0,0 +1 @@
+3987542254
\ No newline at end of file
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Filter.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Filter.db
new file mode 100644
index 0000000..8868e5c
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Filter.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Index.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Index.db
new file mode 100644
index 0000000..59e65ca
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Index.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Statistics.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Statistics.db
new file mode 100644
index 0000000..689bec8
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Statistics.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Summary.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Summary.db
new file mode 100644
index 0000000..9b24e04
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Summary.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-TOC.txt b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-TOC.txt
new file mode 100644
index 0000000..b03b283
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-TOC.txt
@@ -0,0 +1,8 @@
+Filter.db
+Digest.crc32
+Index.db
+TOC.txt
+Summary.db
+Statistics.db
+CompressionInfo.db
+Data.db
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-bti-CompressionInfo.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-bti-CompressionInfo.db
new file mode 100644
index 0000000..0d9c077
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-bti-CompressionInfo.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-bti-Data.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-bti-Data.db
new file mode 100644
index 0000000..1489bab
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-bti-Data.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-bti-Digest.crc32 b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-bti-Digest.crc32
new file mode 100644
index 0000000..a804901
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-bti-Digest.crc32
@@ -0,0 +1 @@
+163579974
\ No newline at end of file
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-bti-Filter.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-bti-Filter.db
new file mode 100644
index 0000000..2e1d5d2
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-bti-Filter.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-bti-Partitions.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-bti-Partitions.db
new file mode 100644
index 0000000..1eed5ad
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-bti-Partitions.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-bti-Rows.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-bti-Rows.db
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-bti-Rows.db
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-bti-Statistics.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-bti-Statistics.db
new file mode 100644
index 0000000..171655c
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-bti-Statistics.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-bti-TOC.txt b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-bti-TOC.txt
new file mode 100644
index 0000000..c20f4a8
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-bti-TOC.txt
@@ -0,0 +1,8 @@
+Data.db
+Filter.db
+Statistics.db
+CompressionInfo.db
+Partitions.db
+TOC.txt
+Rows.db
+Digest.crc32
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-big-CompressionInfo.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-big-CompressionInfo.db
new file mode 100644
index 0000000..1237cc7
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-big-CompressionInfo.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-big-Data.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-big-Data.db
new file mode 100644
index 0000000..366315f
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-big-Data.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-big-Digest.crc32 b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-big-Digest.crc32
new file mode 100644
index 0000000..839639c
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-big-Digest.crc32
@@ -0,0 +1 @@
+138332031
\ No newline at end of file
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-big-Filter.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-big-Filter.db
new file mode 100644
index 0000000..2e1d5d2
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-big-Filter.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-big-Index.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-big-Index.db
new file mode 100644
index 0000000..d094f73
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-big-Index.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-big-Statistics.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-big-Statistics.db
new file mode 100644
index 0000000..613f2ad
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-big-Statistics.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-big-Summary.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-big-Summary.db
new file mode 100644
index 0000000..9b24e04
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-big-Summary.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-big-TOC.txt b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-big-TOC.txt
new file mode 100644
index 0000000..734a80d
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-big-TOC.txt
@@ -0,0 +1,8 @@
+Index.db
+TOC.txt
+Filter.db
+CompressionInfo.db
+Summary.db
+Data.db
+Statistics.db
+Digest.crc32
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-bti-CompressionInfo.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-bti-CompressionInfo.db
new file mode 100644
index 0000000..56c95a8
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-bti-CompressionInfo.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-bti-Data.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-bti-Data.db
new file mode 100644
index 0000000..2977f11
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-bti-Data.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-bti-Digest.crc32 b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-bti-Digest.crc32
new file mode 100644
index 0000000..02bf600
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-bti-Digest.crc32
@@ -0,0 +1 @@
+1528982319
\ No newline at end of file
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-bti-Filter.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-bti-Filter.db
new file mode 100644
index 0000000..2e1d5d2
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-bti-Filter.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-bti-Partitions.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-bti-Partitions.db
new file mode 100644
index 0000000..05e27b4
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-bti-Partitions.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-bti-Rows.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-bti-Rows.db
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-bti-Rows.db
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-bti-Statistics.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-bti-Statistics.db
new file mode 100644
index 0000000..08d8f3e
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-bti-Statistics.db
Binary files differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-bti-TOC.txt b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-bti-TOC.txt
new file mode 100644
index 0000000..c20f4a8
--- /dev/null
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-bti-TOC.txt
@@ -0,0 +1,8 @@
+Data.db
+Filter.db
+Statistics.db
+CompressionInfo.db
+Partitions.db
+TOC.txt
+Rows.db
+Digest.crc32
diff --git a/test/data/migration-sstables/2.1/system/compactions_in_progress-55080ab05d9c388690a4acb25fe1f77b/system-compactions_in_progress-ka-1-CompressionInfo.db b/test/data/migration-sstables/2.1/system/compactions_in_progress-55080ab05d9c388690a4acb25fe1f77b/system-compactions_in_progress-ka-1-CompressionInfo.db
deleted file mode 100644
index d9446df..0000000
--- a/test/data/migration-sstables/2.1/system/compactions_in_progress-55080ab05d9c388690a4acb25fe1f77b/system-compactions_in_progress-ka-1-CompressionInfo.db
+++ /dev/null
Binary files differ
diff --git a/test/data/migration-sstables/2.1/system/compactions_in_progress-55080ab05d9c388690a4acb25fe1f77b/system-compactions_in_progress-ka-1-Data.db b/test/data/migration-sstables/2.1/system/compactions_in_progress-55080ab05d9c388690a4acb25fe1f77b/system-compactions_in_progress-ka-1-Data.db
deleted file mode 100644
index f7b696d..0000000
--- a/test/data/migration-sstables/2.1/system/compactions_in_progress-55080ab05d9c388690a4acb25fe1f77b/system-compactions_in_progress-ka-1-Data.db
+++ /dev/null
Binary files differ
diff --git a/test/data/migration-sstables/2.1/system/compactions_in_progress-55080ab05d9c388690a4acb25fe1f77b/system-compactions_in_progress-ka-1-Digest.sha1 b/test/data/migration-sstables/2.1/system/compactions_in_progress-55080ab05d9c388690a4acb25fe1f77b/system-compactions_in_progress-ka-1-Digest.sha1
deleted file mode 100644
index 55756dd..0000000
--- a/test/data/migration-sstables/2.1/system/compactions_in_progress-55080ab05d9c388690a4acb25fe1f77b/system-compactions_in_progress-ka-1-Digest.sha1
+++ /dev/null
@@ -1 +0,0 @@
-3043896114
\ No newline at end of file
diff --git a/test/data/migration-sstables/2.1/system/compactions_in_progress-55080ab05d9c388690a4acb25fe1f77b/system-compactions_in_progress-ka-1-Filter.db b/test/data/migration-sstables/2.1/system/compactions_in_progress-55080ab05d9c388690a4acb25fe1f77b/system-compactions_in_progress-ka-1-Filter.db
deleted file mode 100644
index 3015f10..0000000
--- a/test/data/migration-sstables/2.1/system/compactions_in_progress-55080ab05d9c388690a4acb25fe1f77b/system-compactions_in_progress-ka-1-Filter.db
+++ /dev/null
Binary files differ
diff --git a/test/data/migration-sstables/2.1/system/compactions_in_progress-55080ab05d9c388690a4acb25fe1f77b/system-compactions_in_progress-ka-1-Index.db b/test/data/migration-sstables/2.1/system/compactions_in_progress-55080ab05d9c388690a4acb25fe1f77b/system-compactions_in_progress-ka-1-Index.db
deleted file mode 100644
index c8b59fb..0000000
--- a/test/data/migration-sstables/2.1/system/compactions_in_progress-55080ab05d9c388690a4acb25fe1f77b/system-compactions_in_progress-ka-1-Index.db
+++ /dev/null
Binary files differ
diff --git a/test/data/migration-sstables/2.1/system/compactions_in_progress-55080ab05d9c388690a4acb25fe1f77b/system-compactions_in_progress-ka-1-Statistics.db b/test/data/migration-sstables/2.1/system/compactions_in_progress-55080ab05d9c388690a4acb25fe1f77b/system-compactions_in_progress-ka-1-Statistics.db
deleted file mode 100644
index 8535f6a..0000000
--- a/test/data/migration-sstables/2.1/system/compactions_in_progress-55080ab05d9c388690a4acb25fe1f77b/system-compactions_in_progress-ka-1-Statistics.db
+++ /dev/null
Binary files differ
diff --git a/test/data/migration-sstables/2.1/system/compactions_in_progress-55080ab05d9c388690a4acb25fe1f77b/system-compactions_in_progress-ka-1-Summary.db b/test/data/migration-sstables/2.1/system/compactions_in_progress-55080ab05d9c388690a4acb25fe1f77b/system-compactions_in_progress-ka-1-Summary.db
deleted file mode 100644
index d9ce8c2..0000000
--- a/test/data/migration-sstables/2.1/system/compactions_in_progress-55080ab05d9c388690a4acb25fe1f77b/system-compactions_in_progress-ka-1-Summary.db
+++ /dev/null
Binary files differ
diff --git a/test/data/migration-sstables/2.1/system/compactions_in_progress-55080ab05d9c388690a4acb25fe1f77b/system-compactions_in_progress-ka-1-TOC.txt b/test/data/migration-sstables/2.1/system/compactions_in_progress-55080ab05d9c388690a4acb25fe1f77b/system-compactions_in_progress-ka-1-TOC.txt
deleted file mode 100644
index 7dc8930..0000000
--- a/test/data/migration-sstables/2.1/system/compactions_in_progress-55080ab05d9c388690a4acb25fe1f77b/system-compactions_in_progress-ka-1-TOC.txt
+++ /dev/null
@@ -1,8 +0,0 @@
-Data.db
-TOC.txt
-Filter.db
-Statistics.db
-Summary.db
-Index.db
-Digest.sha1
-CompressionInfo.db
diff --git a/test/data/migration-sstables/2.1/test/foo-0094ac203e7411e59149ef9f87394ca6/test-foo-ka-3-CompressionInfo.db b/test/data/migration-sstables/2.1/test/foo-0094ac203e7411e59149ef9f87394ca6/test-foo-ka-3-CompressionInfo.db
deleted file mode 100644
index b867db8..0000000
--- a/test/data/migration-sstables/2.1/test/foo-0094ac203e7411e59149ef9f87394ca6/test-foo-ka-3-CompressionInfo.db
+++ /dev/null
Binary files differ
diff --git a/test/data/migration-sstables/2.1/test/foo-0094ac203e7411e59149ef9f87394ca6/test-foo-ka-3-Data.db b/test/data/migration-sstables/2.1/test/foo-0094ac203e7411e59149ef9f87394ca6/test-foo-ka-3-Data.db
deleted file mode 100644
index f14d86d..0000000
--- a/test/data/migration-sstables/2.1/test/foo-0094ac203e7411e59149ef9f87394ca6/test-foo-ka-3-Data.db
+++ /dev/null
Binary files differ
diff --git a/test/data/migration-sstables/2.1/test/foo-0094ac203e7411e59149ef9f87394ca6/test-foo-ka-3-Digest.sha1 b/test/data/migration-sstables/2.1/test/foo-0094ac203e7411e59149ef9f87394ca6/test-foo-ka-3-Digest.sha1
deleted file mode 100644
index 2f4daa9..0000000
--- a/test/data/migration-sstables/2.1/test/foo-0094ac203e7411e59149ef9f87394ca6/test-foo-ka-3-Digest.sha1
+++ /dev/null
@@ -1 +0,0 @@
-4283441474
\ No newline at end of file
diff --git a/test/data/migration-sstables/2.1/test/foo-0094ac203e7411e59149ef9f87394ca6/test-foo-ka-3-Filter.db b/test/data/migration-sstables/2.1/test/foo-0094ac203e7411e59149ef9f87394ca6/test-foo-ka-3-Filter.db
deleted file mode 100644
index a5bdd8e..0000000
--- a/test/data/migration-sstables/2.1/test/foo-0094ac203e7411e59149ef9f87394ca6/test-foo-ka-3-Filter.db
+++ /dev/null
Binary files differ
diff --git a/test/data/migration-sstables/2.1/test/foo-0094ac203e7411e59149ef9f87394ca6/test-foo-ka-3-Index.db b/test/data/migration-sstables/2.1/test/foo-0094ac203e7411e59149ef9f87394ca6/test-foo-ka-3-Index.db
deleted file mode 100644
index 5d71315..0000000
--- a/test/data/migration-sstables/2.1/test/foo-0094ac203e7411e59149ef9f87394ca6/test-foo-ka-3-Index.db
+++ /dev/null
Binary files differ
diff --git a/test/data/migration-sstables/2.1/test/foo-0094ac203e7411e59149ef9f87394ca6/test-foo-ka-3-Statistics.db b/test/data/migration-sstables/2.1/test/foo-0094ac203e7411e59149ef9f87394ca6/test-foo-ka-3-Statistics.db
deleted file mode 100644
index aeb2bb8..0000000
--- a/test/data/migration-sstables/2.1/test/foo-0094ac203e7411e59149ef9f87394ca6/test-foo-ka-3-Statistics.db
+++ /dev/null
Binary files differ
diff --git a/test/data/migration-sstables/2.1/test/foo-0094ac203e7411e59149ef9f87394ca6/test-foo-ka-3-Summary.db b/test/data/migration-sstables/2.1/test/foo-0094ac203e7411e59149ef9f87394ca6/test-foo-ka-3-Summary.db
deleted file mode 100644
index 602ec06..0000000
--- a/test/data/migration-sstables/2.1/test/foo-0094ac203e7411e59149ef9f87394ca6/test-foo-ka-3-Summary.db
+++ /dev/null
Binary files differ
diff --git a/test/data/migration-sstables/2.1/test/foo-0094ac203e7411e59149ef9f87394ca6/test-foo-ka-3-TOC.txt b/test/data/migration-sstables/2.1/test/foo-0094ac203e7411e59149ef9f87394ca6/test-foo-ka-3-TOC.txt
deleted file mode 100644
index 7dc8930..0000000
--- a/test/data/migration-sstables/2.1/test/foo-0094ac203e7411e59149ef9f87394ca6/test-foo-ka-3-TOC.txt
+++ /dev/null
@@ -1,8 +0,0 @@
-Data.db
-TOC.txt
-Filter.db
-Statistics.db
-Summary.db
-Index.db
-Digest.sha1
-CompressionInfo.db
diff --git a/test/data/migration-sstables/2.1/test/foo-0094ac203e7411e59149ef9f87394ca6/test-foo-tmp-ka-4-Data.db b/test/data/migration-sstables/2.1/test/foo-0094ac203e7411e59149ef9f87394ca6/test-foo-tmp-ka-4-Data.db
deleted file mode 100644
index f14d86d..0000000
--- a/test/data/migration-sstables/2.1/test/foo-0094ac203e7411e59149ef9f87394ca6/test-foo-tmp-ka-4-Data.db
+++ /dev/null
Binary files differ
diff --git a/test/data/migration-sstables/2.1/test/foo-0094ac203e7411e59149ef9f87394ca6/test-foo-tmp-ka-4-Index.db b/test/data/migration-sstables/2.1/test/foo-0094ac203e7411e59149ef9f87394ca6/test-foo-tmp-ka-4-Index.db
deleted file mode 100644
index 5d71315..0000000
--- a/test/data/migration-sstables/2.1/test/foo-0094ac203e7411e59149ef9f87394ca6/test-foo-tmp-ka-4-Index.db
+++ /dev/null
Binary files differ
diff --git a/test/data/migration-sstables/2.1/test/foo-0094ac203e7411e59149ef9f87394ca6/test-foo-tmplink-ka-4-Data.db b/test/data/migration-sstables/2.1/test/foo-0094ac203e7411e59149ef9f87394ca6/test-foo-tmplink-ka-4-Data.db
deleted file mode 100644
index f14d86d..0000000
--- a/test/data/migration-sstables/2.1/test/foo-0094ac203e7411e59149ef9f87394ca6/test-foo-tmplink-ka-4-Data.db
+++ /dev/null
Binary files differ
diff --git a/test/data/migration-sstables/2.1/test/foo-0094ac203e7411e59149ef9f87394ca6/test-foo-tmplink-ka-4-Index.db b/test/data/migration-sstables/2.1/test/foo-0094ac203e7411e59149ef9f87394ca6/test-foo-tmplink-ka-4-Index.db
deleted file mode 100644
index 5d71315..0000000
--- a/test/data/migration-sstables/2.1/test/foo-0094ac203e7411e59149ef9f87394ca6/test-foo-tmplink-ka-4-Index.db
+++ /dev/null
Binary files differ
diff --git a/test/data/migration-sstables/2.2/keyspace1/test-dfcc85801bc811e5aa694b06169f4ffa/la-1-big-CompressionInfo.db b/test/data/migration-sstables/2.2/keyspace1/test-dfcc85801bc811e5aa694b06169f4ffa/la-1-big-CompressionInfo.db
deleted file mode 100644
index f7a81f0..0000000
--- a/test/data/migration-sstables/2.2/keyspace1/test-dfcc85801bc811e5aa694b06169f4ffa/la-1-big-CompressionInfo.db
+++ /dev/null
Binary files differ
diff --git a/test/data/migration-sstables/2.2/keyspace1/test-dfcc85801bc811e5aa694b06169f4ffa/la-1-big-Data.db b/test/data/migration-sstables/2.2/keyspace1/test-dfcc85801bc811e5aa694b06169f4ffa/la-1-big-Data.db
deleted file mode 100644
index 2d5e60a..0000000
--- a/test/data/migration-sstables/2.2/keyspace1/test-dfcc85801bc811e5aa694b06169f4ffa/la-1-big-Data.db
+++ /dev/null
Binary files differ
diff --git a/test/data/migration-sstables/2.2/keyspace1/test-dfcc85801bc811e5aa694b06169f4ffa/la-1-big-Digest.adler32 b/test/data/migration-sstables/2.2/keyspace1/test-dfcc85801bc811e5aa694b06169f4ffa/la-1-big-Digest.adler32
deleted file mode 100644
index deffbd1..0000000
--- a/test/data/migration-sstables/2.2/keyspace1/test-dfcc85801bc811e5aa694b06169f4ffa/la-1-big-Digest.adler32
+++ /dev/null
@@ -1 +0,0 @@
-2055934203
\ No newline at end of file
diff --git a/test/data/migration-sstables/2.2/keyspace1/test-dfcc85801bc811e5aa694b06169f4ffa/la-1-big-Filter.db b/test/data/migration-sstables/2.2/keyspace1/test-dfcc85801bc811e5aa694b06169f4ffa/la-1-big-Filter.db
deleted file mode 100644
index a749417..0000000
--- a/test/data/migration-sstables/2.2/keyspace1/test-dfcc85801bc811e5aa694b06169f4ffa/la-1-big-Filter.db
+++ /dev/null
Binary files differ
diff --git a/test/data/migration-sstables/2.2/keyspace1/test-dfcc85801bc811e5aa694b06169f4ffa/la-1-big-Index.db b/test/data/migration-sstables/2.2/keyspace1/test-dfcc85801bc811e5aa694b06169f4ffa/la-1-big-Index.db
deleted file mode 100644
index d3923ab..0000000
--- a/test/data/migration-sstables/2.2/keyspace1/test-dfcc85801bc811e5aa694b06169f4ffa/la-1-big-Index.db
+++ /dev/null
Binary files differ
diff --git a/test/data/migration-sstables/2.2/keyspace1/test-dfcc85801bc811e5aa694b06169f4ffa/la-1-big-Statistics.db b/test/data/migration-sstables/2.2/keyspace1/test-dfcc85801bc811e5aa694b06169f4ffa/la-1-big-Statistics.db
deleted file mode 100644
index 664bfa5..0000000
--- a/test/data/migration-sstables/2.2/keyspace1/test-dfcc85801bc811e5aa694b06169f4ffa/la-1-big-Statistics.db
+++ /dev/null
Binary files differ
diff --git a/test/data/migration-sstables/2.2/keyspace1/test-dfcc85801bc811e5aa694b06169f4ffa/la-1-big-Summary.db b/test/data/migration-sstables/2.2/keyspace1/test-dfcc85801bc811e5aa694b06169f4ffa/la-1-big-Summary.db
deleted file mode 100644
index a74f96f..0000000
--- a/test/data/migration-sstables/2.2/keyspace1/test-dfcc85801bc811e5aa694b06169f4ffa/la-1-big-Summary.db
+++ /dev/null
Binary files differ
diff --git a/test/data/migration-sstables/2.2/keyspace1/test-dfcc85801bc811e5aa694b06169f4ffa/la-1-big-TOC.txt b/test/data/migration-sstables/2.2/keyspace1/test-dfcc85801bc811e5aa694b06169f4ffa/la-1-big-TOC.txt
deleted file mode 100644
index 92dc9fe..0000000
--- a/test/data/migration-sstables/2.2/keyspace1/test-dfcc85801bc811e5aa694b06169f4ffa/la-1-big-TOC.txt
+++ /dev/null
@@ -1,8 +0,0 @@
-Statistics.db
-Summary.db
-TOC.txt
-Filter.db
-Data.db
-CompressionInfo.db
-Digest.adler32
-Index.db
diff --git a/test/data/migration-sstables/2.2/keyspace1/test-dfcc85801bc811e5aa694b06169f4ffa/tmp-la-2-big-Data.db b/test/data/migration-sstables/2.2/keyspace1/test-dfcc85801bc811e5aa694b06169f4ffa/tmp-la-2-big-Data.db
deleted file mode 100644
index 2d5e60a..0000000
--- a/test/data/migration-sstables/2.2/keyspace1/test-dfcc85801bc811e5aa694b06169f4ffa/tmp-la-2-big-Data.db
+++ /dev/null
Binary files differ
diff --git a/test/data/migration-sstables/2.2/keyspace1/test-dfcc85801bc811e5aa694b06169f4ffa/tmp-la-2-big-Index.db b/test/data/migration-sstables/2.2/keyspace1/test-dfcc85801bc811e5aa694b06169f4ffa/tmp-la-2-big-Index.db
deleted file mode 100644
index d3923ab..0000000
--- a/test/data/migration-sstables/2.2/keyspace1/test-dfcc85801bc811e5aa694b06169f4ffa/tmp-la-2-big-Index.db
+++ /dev/null
Binary files differ
diff --git a/test/data/migration-sstables/2.2/keyspace1/test-dfcc85801bc811e5aa694b06169f4ffa/tmp-lb-3-big-Data.db b/test/data/migration-sstables/2.2/keyspace1/test-dfcc85801bc811e5aa694b06169f4ffa/tmp-lb-3-big-Data.db
deleted file mode 100644
index 2d5e60a..0000000
--- a/test/data/migration-sstables/2.2/keyspace1/test-dfcc85801bc811e5aa694b06169f4ffa/tmp-lb-3-big-Data.db
+++ /dev/null
Binary files differ
diff --git a/test/data/migration-sstables/2.2/keyspace1/test-dfcc85801bc811e5aa694b06169f4ffa/tmp-lb-3-big-Index.db b/test/data/migration-sstables/2.2/keyspace1/test-dfcc85801bc811e5aa694b06169f4ffa/tmp-lb-3-big-Index.db
deleted file mode 100644
index d3923ab..0000000
--- a/test/data/migration-sstables/2.2/keyspace1/test-dfcc85801bc811e5aa694b06169f4ffa/tmp-lb-3-big-Index.db
+++ /dev/null
Binary files differ
diff --git a/test/data/migration-sstables/2.2/keyspace1/test-dfcc85801bc811e5aa694b06169f4ffa/tmplink-la-2-big-Data.db b/test/data/migration-sstables/2.2/keyspace1/test-dfcc85801bc811e5aa694b06169f4ffa/tmplink-la-2-big-Data.db
deleted file mode 100644
index 2d5e60a..0000000
--- a/test/data/migration-sstables/2.2/keyspace1/test-dfcc85801bc811e5aa694b06169f4ffa/tmplink-la-2-big-Data.db
+++ /dev/null
Binary files differ
diff --git a/test/data/migration-sstables/2.2/keyspace1/test-dfcc85801bc811e5aa694b06169f4ffa/tmplink-la-2-big-Index.db b/test/data/migration-sstables/2.2/keyspace1/test-dfcc85801bc811e5aa694b06169f4ffa/tmplink-la-2-big-Index.db
deleted file mode 100644
index d3923ab..0000000
--- a/test/data/migration-sstables/2.2/keyspace1/test-dfcc85801bc811e5aa694b06169f4ffa/tmplink-la-2-big-Index.db
+++ /dev/null
Binary files differ
diff --git a/test/data/migration-sstables/2.2/system/compactions_in_progress-55080ab05d9c388690a4acb25fe1f77b/snapshots/1435108403246-compactions_in_progress/manifest.json b/test/data/migration-sstables/2.2/system/compactions_in_progress-55080ab05d9c388690a4acb25fe1f77b/snapshots/1435108403246-compactions_in_progress/manifest.json
deleted file mode 100644
index d5fdb4f..0000000
--- a/test/data/migration-sstables/2.2/system/compactions_in_progress-55080ab05d9c388690a4acb25fe1f77b/snapshots/1435108403246-compactions_in_progress/manifest.json
+++ /dev/null
@@ -1 +0,0 @@
-{"files":[]}
diff --git a/test/data/migration-sstables/2.2/system/compactions_in_progress-55080ab05d9c388690a4acb25fe1f77b/snapshots/1435298241281-upgrade-3.0.0-SNAPSHOT-2.2.0-rc1-SNAPSHOT/manifest.json b/test/data/migration-sstables/2.2/system/compactions_in_progress-55080ab05d9c388690a4acb25fe1f77b/snapshots/1435298241281-upgrade-3.0.0-SNAPSHOT-2.2.0-rc1-SNAPSHOT/manifest.json
deleted file mode 100644
index d5fdb4f..0000000
--- a/test/data/migration-sstables/2.2/system/compactions_in_progress-55080ab05d9c388690a4acb25fe1f77b/snapshots/1435298241281-upgrade-3.0.0-SNAPSHOT-2.2.0-rc1-SNAPSHOT/manifest.json
+++ /dev/null
@@ -1 +0,0 @@
-{"files":[]}
diff --git a/test/data/migration-sstables/2.2/system/compactions_in_progress-55080ab05d9c388690a4acb25fe1f77b/snapshots/1435298241532-compactions_in_progress/manifest.json b/test/data/migration-sstables/2.2/system/compactions_in_progress-55080ab05d9c388690a4acb25fe1f77b/snapshots/1435298241532-compactions_in_progress/manifest.json
deleted file mode 100644
index d5fdb4f..0000000
--- a/test/data/migration-sstables/2.2/system/compactions_in_progress-55080ab05d9c388690a4acb25fe1f77b/snapshots/1435298241532-compactions_in_progress/manifest.json
+++ /dev/null
@@ -1 +0,0 @@
-{"files":[]}
diff --git a/test/data/migration-sstables/ic/Keyspace1/legacyleveled/Keyspace1-legacyleveled-ic-0-Data.db b/test/data/migration-sstables/ic/Keyspace1/legacyleveled/Keyspace1-legacyleveled-ic-0-Data.db
deleted file mode 100644
index 98d3f41..0000000
--- a/test/data/migration-sstables/ic/Keyspace1/legacyleveled/Keyspace1-legacyleveled-ic-0-Data.db
+++ /dev/null
Binary files differ
diff --git a/test/data/migration-sstables/ic/Keyspace1/legacyleveled/Keyspace1-legacyleveled-ic-0-Digest.sha1 b/test/data/migration-sstables/ic/Keyspace1/legacyleveled/Keyspace1-legacyleveled-ic-0-Digest.sha1
deleted file mode 100644
index 470b056..0000000
--- a/test/data/migration-sstables/ic/Keyspace1/legacyleveled/Keyspace1-legacyleveled-ic-0-Digest.sha1
+++ /dev/null
@@ -1 +0,0 @@
-9ee805b905aa147afe14d4f37f5ed3be3af53c72 Keyspace1-legacyleveled-ic-0-Data.db
\ No newline at end of file
diff --git a/test/data/migration-sstables/ic/Keyspace1/legacyleveled/Keyspace1-legacyleveled-ic-0-Filter.db b/test/data/migration-sstables/ic/Keyspace1/legacyleveled/Keyspace1-legacyleveled-ic-0-Filter.db
deleted file mode 100644
index c63729b..0000000
--- a/test/data/migration-sstables/ic/Keyspace1/legacyleveled/Keyspace1-legacyleveled-ic-0-Filter.db
+++ /dev/null
Binary files differ
diff --git a/test/data/migration-sstables/ic/Keyspace1/legacyleveled/Keyspace1-legacyleveled-ic-0-Index.db b/test/data/migration-sstables/ic/Keyspace1/legacyleveled/Keyspace1-legacyleveled-ic-0-Index.db
deleted file mode 100644
index 6603018..0000000
--- a/test/data/migration-sstables/ic/Keyspace1/legacyleveled/Keyspace1-legacyleveled-ic-0-Index.db
+++ /dev/null
Binary files differ
diff --git a/test/data/migration-sstables/ic/Keyspace1/legacyleveled/Keyspace1-legacyleveled-ic-0-Statistics.db b/test/data/migration-sstables/ic/Keyspace1/legacyleveled/Keyspace1-legacyleveled-ic-0-Statistics.db
deleted file mode 100644
index 5ed9ce0..0000000
--- a/test/data/migration-sstables/ic/Keyspace1/legacyleveled/Keyspace1-legacyleveled-ic-0-Statistics.db
+++ /dev/null
Binary files differ
diff --git a/test/data/migration-sstables/ic/Keyspace1/legacyleveled/Keyspace1-legacyleveled-ic-0-Summary.db b/test/data/migration-sstables/ic/Keyspace1/legacyleveled/Keyspace1-legacyleveled-ic-0-Summary.db
deleted file mode 100644
index c1c8fd8..0000000
--- a/test/data/migration-sstables/ic/Keyspace1/legacyleveled/Keyspace1-legacyleveled-ic-0-Summary.db
+++ /dev/null
Binary files differ
diff --git a/test/data/migration-sstables/ic/Keyspace1/legacyleveled/Keyspace1-legacyleveled-ic-0-TOC.txt b/test/data/migration-sstables/ic/Keyspace1/legacyleveled/Keyspace1-legacyleveled-ic-0-TOC.txt
deleted file mode 100644
index 6baaf14..0000000
--- a/test/data/migration-sstables/ic/Keyspace1/legacyleveled/Keyspace1-legacyleveled-ic-0-TOC.txt
+++ /dev/null
@@ -1,7 +0,0 @@
-Filter.db
-Summary.db
-Data.db
-Digest.sha1
-Index.db
-TOC.txt
-Statistics.db
diff --git a/test/data/migration-sstables/ic/Keyspace1/legacyleveled/Keyspace1-legacyleveled-ic-1-Data.db b/test/data/migration-sstables/ic/Keyspace1/legacyleveled/Keyspace1-legacyleveled-ic-1-Data.db
deleted file mode 100644
index 98d3f41..0000000
--- a/test/data/migration-sstables/ic/Keyspace1/legacyleveled/Keyspace1-legacyleveled-ic-1-Data.db
+++ /dev/null
Binary files differ
diff --git a/test/data/migration-sstables/ic/Keyspace1/legacyleveled/Keyspace1-legacyleveled-ic-1-Digest.sha1 b/test/data/migration-sstables/ic/Keyspace1/legacyleveled/Keyspace1-legacyleveled-ic-1-Digest.sha1
deleted file mode 100644
index d8db723..0000000
--- a/test/data/migration-sstables/ic/Keyspace1/legacyleveled/Keyspace1-legacyleveled-ic-1-Digest.sha1
+++ /dev/null
@@ -1 +0,0 @@
-9ee805b905aa147afe14d4f37f5ed3be3af53c72 Keyspace1-legacyleveled-ic-1-Data.db
\ No newline at end of file
diff --git a/test/data/migration-sstables/ic/Keyspace1/legacyleveled/Keyspace1-legacyleveled-ic-1-Filter.db b/test/data/migration-sstables/ic/Keyspace1/legacyleveled/Keyspace1-legacyleveled-ic-1-Filter.db
deleted file mode 100644
index c63729b..0000000
--- a/test/data/migration-sstables/ic/Keyspace1/legacyleveled/Keyspace1-legacyleveled-ic-1-Filter.db
+++ /dev/null
Binary files differ
diff --git a/test/data/migration-sstables/ic/Keyspace1/legacyleveled/Keyspace1-legacyleveled-ic-1-Index.db b/test/data/migration-sstables/ic/Keyspace1/legacyleveled/Keyspace1-legacyleveled-ic-1-Index.db
deleted file mode 100644
index 6603018..0000000
--- a/test/data/migration-sstables/ic/Keyspace1/legacyleveled/Keyspace1-legacyleveled-ic-1-Index.db
+++ /dev/null
Binary files differ
diff --git a/test/data/migration-sstables/ic/Keyspace1/legacyleveled/Keyspace1-legacyleveled-ic-1-Statistics.db b/test/data/migration-sstables/ic/Keyspace1/legacyleveled/Keyspace1-legacyleveled-ic-1-Statistics.db
deleted file mode 100644
index 5ed9ce0..0000000
--- a/test/data/migration-sstables/ic/Keyspace1/legacyleveled/Keyspace1-legacyleveled-ic-1-Statistics.db
+++ /dev/null
Binary files differ
diff --git a/test/data/migration-sstables/ic/Keyspace1/legacyleveled/Keyspace1-legacyleveled-ic-1-Summary.db b/test/data/migration-sstables/ic/Keyspace1/legacyleveled/Keyspace1-legacyleveled-ic-1-Summary.db
deleted file mode 100644
index c1c8fd8..0000000
--- a/test/data/migration-sstables/ic/Keyspace1/legacyleveled/Keyspace1-legacyleveled-ic-1-Summary.db
+++ /dev/null
Binary files differ
diff --git a/test/data/migration-sstables/ic/Keyspace1/legacyleveled/Keyspace1-legacyleveled-ic-1-TOC.txt b/test/data/migration-sstables/ic/Keyspace1/legacyleveled/Keyspace1-legacyleveled-ic-1-TOC.txt
deleted file mode 100644
index 6baaf14..0000000
--- a/test/data/migration-sstables/ic/Keyspace1/legacyleveled/Keyspace1-legacyleveled-ic-1-TOC.txt
+++ /dev/null
@@ -1,7 +0,0 @@
-Filter.db
-Summary.db
-Data.db
-Digest.sha1
-Index.db
-TOC.txt
-Statistics.db
diff --git a/test/data/migration-sstables/ic/Keyspace1/legacyleveled/Keyspace1-legacyleveled-ic-2-Data.db b/test/data/migration-sstables/ic/Keyspace1/legacyleveled/Keyspace1-legacyleveled-ic-2-Data.db
deleted file mode 100644
index 98d3f41..0000000
--- a/test/data/migration-sstables/ic/Keyspace1/legacyleveled/Keyspace1-legacyleveled-ic-2-Data.db
+++ /dev/null
Binary files differ
diff --git a/test/data/migration-sstables/ic/Keyspace1/legacyleveled/Keyspace1-legacyleveled-ic-2-Digest.sha1 b/test/data/migration-sstables/ic/Keyspace1/legacyleveled/Keyspace1-legacyleveled-ic-2-Digest.sha1
deleted file mode 100644
index 31da1c4..0000000
--- a/test/data/migration-sstables/ic/Keyspace1/legacyleveled/Keyspace1-legacyleveled-ic-2-Digest.sha1
+++ /dev/null
@@ -1 +0,0 @@
-9ee805b905aa147afe14d4f37f5ed3be3af53c72 Keyspace1-legacyleveled-ic-2-Data.db
\ No newline at end of file
diff --git a/test/data/migration-sstables/ic/Keyspace1/legacyleveled/Keyspace1-legacyleveled-ic-2-Filter.db b/test/data/migration-sstables/ic/Keyspace1/legacyleveled/Keyspace1-legacyleveled-ic-2-Filter.db
deleted file mode 100644
index c63729b..0000000
--- a/test/data/migration-sstables/ic/Keyspace1/legacyleveled/Keyspace1-legacyleveled-ic-2-Filter.db
+++ /dev/null
Binary files differ
diff --git a/test/data/migration-sstables/ic/Keyspace1/legacyleveled/Keyspace1-legacyleveled-ic-2-Index.db b/test/data/migration-sstables/ic/Keyspace1/legacyleveled/Keyspace1-legacyleveled-ic-2-Index.db
deleted file mode 100644
index 6603018..0000000
--- a/test/data/migration-sstables/ic/Keyspace1/legacyleveled/Keyspace1-legacyleveled-ic-2-Index.db
+++ /dev/null
Binary files differ
diff --git a/test/data/migration-sstables/ic/Keyspace1/legacyleveled/Keyspace1-legacyleveled-ic-2-Statistics.db b/test/data/migration-sstables/ic/Keyspace1/legacyleveled/Keyspace1-legacyleveled-ic-2-Statistics.db
deleted file mode 100644
index 5ed9ce0..0000000
--- a/test/data/migration-sstables/ic/Keyspace1/legacyleveled/Keyspace1-legacyleveled-ic-2-Statistics.db
+++ /dev/null
Binary files differ
diff --git a/test/data/migration-sstables/ic/Keyspace1/legacyleveled/Keyspace1-legacyleveled-ic-2-Summary.db b/test/data/migration-sstables/ic/Keyspace1/legacyleveled/Keyspace1-legacyleveled-ic-2-Summary.db
deleted file mode 100644
index c1c8fd8..0000000
--- a/test/data/migration-sstables/ic/Keyspace1/legacyleveled/Keyspace1-legacyleveled-ic-2-Summary.db
+++ /dev/null
Binary files differ
diff --git a/test/data/migration-sstables/ic/Keyspace1/legacyleveled/Keyspace1-legacyleveled-ic-2-TOC.txt b/test/data/migration-sstables/ic/Keyspace1/legacyleveled/Keyspace1-legacyleveled-ic-2-TOC.txt
deleted file mode 100644
index 6baaf14..0000000
--- a/test/data/migration-sstables/ic/Keyspace1/legacyleveled/Keyspace1-legacyleveled-ic-2-TOC.txt
+++ /dev/null
@@ -1,7 +0,0 @@
-Filter.db
-Summary.db
-Data.db
-Digest.sha1
-Index.db
-TOC.txt
-Statistics.db
diff --git a/test/data/migration-sstables/ic/Keyspace1/legacyleveled/legacyleveled.json b/test/data/migration-sstables/ic/Keyspace1/legacyleveled/legacyleveled.json
deleted file mode 100644
index 1fc9c01..0000000
--- a/test/data/migration-sstables/ic/Keyspace1/legacyleveled/legacyleveled.json
+++ /dev/null
@@ -1,27 +0,0 @@
-{
- "generations" : [ {
- "generation" : 0,
- "members" : [ 0 ]
- }, {
- "generation" : 1,
- "members" : [ 1 ]
- }, {
- "generation" : 2,
- "members" : [ 2 ]
- }, {
- "generation" : 3,
- "members" : [ ]
- }, {
- "generation" : 4,
- "members" : [ ]
- }, {
- "generation" : 5,
- "members" : [ ]
- }, {
- "generation" : 6,
- "members" : [ ]
- }, {
- "generation" : 7,
- "members" : [ ]
- } ]
-}
\ No newline at end of file
diff --git a/test/data/serialization/4.0/gms.EndpointState.bin b/test/data/serialization/4.0/gms.EndpointState.bin
new file mode 100644
index 0000000..17fc088
--- /dev/null
+++ b/test/data/serialization/4.0/gms.EndpointState.bin
Binary files differ
diff --git a/test/data/serialization/4.0/gms.Gossip.bin b/test/data/serialization/4.0/gms.Gossip.bin
new file mode 100644
index 0000000..2fbd5d4
--- /dev/null
+++ b/test/data/serialization/4.0/gms.Gossip.bin
Binary files differ
diff --git a/test/data/serialization/4.0/service.SyncComplete.bin b/test/data/serialization/4.0/service.SyncComplete.bin
new file mode 100644
index 0000000..4e8caa6
--- /dev/null
+++ b/test/data/serialization/4.0/service.SyncComplete.bin
Binary files differ
diff --git a/test/data/serialization/4.0/service.SyncRequest.bin b/test/data/serialization/4.0/service.SyncRequest.bin
new file mode 100644
index 0000000..b0cc44e
--- /dev/null
+++ b/test/data/serialization/4.0/service.SyncRequest.bin
Binary files differ
diff --git a/test/data/serialization/4.0/service.ValidationComplete.bin b/test/data/serialization/4.0/service.ValidationComplete.bin
new file mode 100644
index 0000000..7402c9e
--- /dev/null
+++ b/test/data/serialization/4.0/service.ValidationComplete.bin
Binary files differ
diff --git a/test/data/serialization/4.0/service.ValidationRequest.bin b/test/data/serialization/4.0/service.ValidationRequest.bin
new file mode 100644
index 0000000..fa4a913
--- /dev/null
+++ b/test/data/serialization/4.0/service.ValidationRequest.bin
Binary files differ
diff --git a/test/data/serialization/4.0/utils.BloomFilter1000.bin b/test/data/serialization/4.0/utils.BloomFilter1000.bin
new file mode 100644
index 0000000..b1bfe99
--- /dev/null
+++ b/test/data/serialization/4.0/utils.BloomFilter1000.bin
Binary files differ
diff --git a/test/data/serialization/4.0/utils.EstimatedHistogram.bin b/test/data/serialization/4.0/utils.EstimatedHistogram.bin
new file mode 100644
index 0000000..e878eda
--- /dev/null
+++ b/test/data/serialization/4.0/utils.EstimatedHistogram.bin
Binary files differ
diff --git a/test/distributed/org/apache/cassandra/distributed/Cluster.java b/test/distributed/org/apache/cassandra/distributed/Cluster.java
index 95ead50..e8e6041 100644
--- a/test/distributed/org/apache/cassandra/distributed/Cluster.java
+++ b/test/distributed/org/apache/cassandra/distributed/Cluster.java
@@ -22,8 +22,8 @@
import java.util.function.Consumer;
import org.apache.cassandra.distributed.api.IInstanceConfig;
-import org.apache.cassandra.distributed.api.IInvokableInstance;
import org.apache.cassandra.distributed.impl.AbstractCluster;
+import org.apache.cassandra.distributed.api.IInvokableInstance;
import org.apache.cassandra.distributed.shared.AbstractBuilder;
import org.apache.cassandra.distributed.shared.Versions;
diff --git a/test/distributed/org/apache/cassandra/distributed/UpgradeableCluster.java b/test/distributed/org/apache/cassandra/distributed/UpgradeableCluster.java
index bde5d4e..532c1b1 100644
--- a/test/distributed/org/apache/cassandra/distributed/UpgradeableCluster.java
+++ b/test/distributed/org/apache/cassandra/distributed/UpgradeableCluster.java
@@ -80,3 +80,4 @@
}
}
}
+
diff --git a/test/distributed/org/apache/cassandra/distributed/impl/AbstractCluster.java b/test/distributed/org/apache/cassandra/distributed/impl/AbstractCluster.java
index 3cb8dac..a28c935 100644
--- a/test/distributed/org/apache/cassandra/distributed/impl/AbstractCluster.java
+++ b/test/distributed/org/apache/cassandra/distributed/impl/AbstractCluster.java
@@ -33,6 +33,7 @@
import java.util.function.BiConsumer;
import java.util.function.Consumer;
import java.util.stream.Collectors;
+import java.util.stream.IntStream;
import java.util.stream.Stream;
import com.google.common.collect.Sets;
@@ -57,16 +58,17 @@
import org.apache.cassandra.distributed.api.IUpgradeableInstance;
import org.apache.cassandra.distributed.api.NodeToolResult;
import org.apache.cassandra.distributed.api.TokenSupplier;
-import org.apache.cassandra.distributed.shared.AbstractBuilder;
import org.apache.cassandra.distributed.shared.InstanceClassLoader;
import org.apache.cassandra.distributed.shared.MessageFilters;
import org.apache.cassandra.distributed.shared.NetworkTopology;
import org.apache.cassandra.distributed.shared.Versions;
import org.apache.cassandra.io.util.FileUtils;
-import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.net.Verb;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.concurrent.SimpleCondition;
+import static org.apache.cassandra.distributed.shared.NetworkTopology.addressAndPort;
+
/**
* AbstractCluster creates, initializes and manages Cassandra instances ({@link Instance}.
*
@@ -117,10 +119,31 @@
// mutated by user-facing API
private final MessageFilters filters;
+ private final INodeProvisionStrategy.Strategy nodeProvisionStrategy;
private final BiConsumer<ClassLoader, Integer> instanceInitializer;
-
private volatile Thread.UncaughtExceptionHandler previousHandler = null;
+ /**
+ * Common builder, add methods that are applicable to both Cluster and Upgradable cluster here.
+ */
+ public static abstract class AbstractBuilder<I extends IInstance, C extends ICluster, B extends AbstractBuilder<I, C, B>>
+ extends org.apache.cassandra.distributed.shared.AbstractBuilder<I, C, B>
+ {
+ private INodeProvisionStrategy.Strategy nodeProvisionStrategy = INodeProvisionStrategy.Strategy.MultipleNetworkInterfaces;
+
+ public AbstractBuilder(Factory<I, C, B> factory)
+ {
+ super(factory);
+ }
+
+ public B withNodeProvisionStrategy(INodeProvisionStrategy.Strategy nodeProvisionStrategy)
+ {
+ this.nodeProvisionStrategy = nodeProvisionStrategy;
+ return (B) this;
+ }
+ }
+
+
protected class Wrapper extends DelegatingInvokableInstance implements IUpgradeableInstance
{
private final int generation;
@@ -157,8 +180,8 @@
ClassLoader classLoader = new InstanceClassLoader(generation, config.num(), version.classpath, sharedClassLoader);
if (instanceInitializer != null)
instanceInitializer.accept(classLoader, config.num());
- return Instance.transferAdhoc((SerializableBiFunction<IInstanceConfig, ClassLoader, IInvokableInstance>)Instance::new, classLoader)
- .apply(config, classLoader);
+ return Instance.transferAdhoc((SerializableBiFunction<IInstanceConfig, ClassLoader, Instance>)Instance::new, classLoader)
+ .apply(config.forVersion(version.major), classLoader);
}
public IInstanceConfig config()
@@ -174,9 +197,16 @@
@Override
public synchronized void startup()
{
+ startup(AbstractCluster.this);
+ }
+
+ public synchronized void startup(ICluster cluster)
+ {
+ if (cluster != AbstractCluster.this)
+ throw new IllegalArgumentException("Only the owning cluster can be used for startup");
if (!isShutdown)
throw new IllegalStateException();
- delegateForStartup().startup(AbstractCluster.this);
+ delegateForStartup().startup(cluster);
isShutdown = false;
updateMessagingVersions();
}
@@ -262,6 +292,7 @@
this.nodeIdTopology = builder.getNodeIdTopology();
this.configUpdater = builder.getConfigUpdater();
this.broadcastPort = builder.getBroadcastPort();
+ this.nodeProvisionStrategy = builder.nodeProvisionStrategy;
this.instances = new ArrayList<>();
this.instanceMap = new HashMap<>();
this.initialVersion = builder.getVersion();
@@ -290,20 +321,30 @@
private InstanceConfig createInstanceConfig(int nodeNum)
{
- String ipPrefix = "127.0." + subnet + ".";
- String seedIp = ipPrefix + "1";
- String ipAddress = ipPrefix + nodeNum;
+ INodeProvisionStrategy provisionStrategy = nodeProvisionStrategy.create(subnet);
long token = tokenSupplier.token(nodeNum);
-
- NetworkTopology topology = NetworkTopology.build(ipPrefix, broadcastPort, nodeIdTopology);
-
- InstanceConfig config = InstanceConfig.generate(nodeNum, ipAddress, topology, root, String.valueOf(token), seedIp);
+ NetworkTopology topology = buildNetworkTopology(provisionStrategy, nodeIdTopology);
+ InstanceConfig config = InstanceConfig.generate(nodeNum, provisionStrategy, topology, root, Long.toString(token));
if (configUpdater != null)
configUpdater.accept(config);
return config;
}
+ public static NetworkTopology buildNetworkTopology(INodeProvisionStrategy provisionStrategy,
+ Map<Integer, NetworkTopology.DcAndRack> nodeIdTopology)
+ {
+ NetworkTopology topology = NetworkTopology.build("", 0, Collections.emptyMap());
+
+ IntStream.rangeClosed(1, nodeIdTopology.size()).forEach(nodeId -> {
+ InetSocketAddress addressAndPort = addressAndPort(provisionStrategy.ipAddress(nodeId), provisionStrategy.storagePort(nodeId));
+ NetworkTopology.DcAndRack dcAndRack = nodeIdTopology.get(nodeId);
+ topology.put(addressAndPort, dcAndRack);
+ });
+ return topology;
+ }
+
+
protected abstract I newInstanceWrapper(int generation, Versions.Version version, IInstanceConfig config);
protected I newInstanceWrapperInternal(int generation, Versions.Version version, IInstanceConfig config)
@@ -408,11 +449,11 @@
return filters;
}
- public IMessageFilters.Builder verbs(MessagingService.Verb... verbs)
+ public IMessageFilters.Builder verbs(Verb... verbs)
{
int[] ids = new int[verbs.length];
for (int i = 0; i < verbs.length; ++i)
- ids[i] = verbs[i].ordinal();
+ ids[i] = verbs[i].id;
return filters.verbs(ids);
}
@@ -613,7 +654,7 @@
InstanceClassLoader cl = (InstanceClassLoader) thread.getContextClassLoader();
get(cl.getInstanceId()).uncaughtException(thread, error);
}
-
+
@Override
public void close()
{
@@ -631,9 +672,21 @@
Thread.setDefaultUncaughtExceptionHandler(previousHandler);
previousHandler = null;
+ //checkForThreadLeaks();
//withThreadLeakCheck(futures);
}
+ private void checkForThreadLeaks()
+ {
+ //This is an alternate version of the thread leak check that just checks to see if any threads are still alive
+ // with the context classloader.
+ Set<Thread> threadSet = Thread.getAllStackTraces().keySet();
+ threadSet.stream().filter(t->t.getContextClassLoader() instanceof InstanceClassLoader).forEach(t->{
+ t.setContextClassLoader(null);
+ throw new RuntimeException("Unterminated thread detected " + t.getName() + " in group " + t.getThreadGroup().getName());
+ });
+ }
+
// We do not want this check to run every time until we fix problems with tread stops
private void withThreadLeakCheck(List<Future<?>> futures)
{
diff --git a/test/distributed/org/apache/cassandra/distributed/impl/Coordinator.java b/test/distributed/org/apache/cassandra/distributed/impl/Coordinator.java
index 329fa37..2ee209d 100644
--- a/test/distributed/org/apache/cassandra/distributed/impl/Coordinator.java
+++ b/test/distributed/org/apache/cassandra/distributed/impl/Coordinator.java
@@ -84,7 +84,7 @@
private SimpleQueryResult executeInternal(String query, ConsistencyLevel consistencyLevelOrigin, Object[] boundValues)
{
ClientState clientState = makeFakeClientState();
- CQLStatement prepared = QueryProcessor.getStatement(query, clientState).statement;
+ CQLStatement prepared = QueryProcessor.getStatement(query, clientState);
List<ByteBuffer> boundBBValues = new ArrayList<>();
ConsistencyLevel consistencyLevel = ConsistencyLevel.valueOf(consistencyLevelOrigin.name());
for (Object boundValue : boundValues)
@@ -98,7 +98,8 @@
Integer.MAX_VALUE,
null,
null,
- ProtocolVersion.CURRENT),
+ ProtocolVersion.V4,
+ null),
System.nanoTime());
return RowUtil.toQueryResult(res);
@@ -123,7 +124,7 @@
return instance.sync(() -> {
ClientState clientState = makeFakeClientState();
ConsistencyLevel consistencyLevel = ConsistencyLevel.valueOf(consistencyLevelOrigin.name());
- CQLStatement prepared = QueryProcessor.getStatement(query, clientState).statement;
+ CQLStatement prepared = QueryProcessor.getStatement(query, clientState);
List<ByteBuffer> boundBBValues = new ArrayList<>();
for (Object boundValue : boundValues)
{
@@ -141,7 +142,8 @@
pageSize,
null,
null,
- ProtocolVersion.CURRENT),
+ ProtocolVersion.CURRENT,
+ selectStatement.keyspace()),
FBUtilities.nowInSeconds())
.getPager(null, ProtocolVersion.CURRENT);
@@ -166,8 +168,8 @@
}).call();
}
- private static ClientState makeFakeClientState()
+ private static final ClientState makeFakeClientState()
{
- return ClientState.forExternalCalls(new InetSocketAddress(FBUtilities.getLocalAddress(), 9042));
+ return ClientState.forExternalCalls(new InetSocketAddress(FBUtilities.getJustLocalAddress(), 9042));
}
}
diff --git a/test/distributed/org/apache/cassandra/distributed/impl/DistributedTestSnitch.java b/test/distributed/org/apache/cassandra/distributed/impl/DistributedTestSnitch.java
index 8652bbc..e671f4d 100644
--- a/test/distributed/org/apache/cassandra/distributed/impl/DistributedTestSnitch.java
+++ b/test/distributed/org/apache/cassandra/distributed/impl/DistributedTestSnitch.java
@@ -20,7 +20,6 @@
import java.net.InetAddress;
import java.net.InetSocketAddress;
-import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
@@ -98,19 +97,11 @@
if (current != null)
return current;
- EndpointState epState = Gossiper.instance.getEndpointStateForEndpoint(endpoint.address);
+ EndpointState epState = Gossiper.instance.getEndpointStateForEndpoint(endpoint);
if (epState == null || epState.getApplicationState(state) == null)
{
if (savedEndpoints == null)
- {
- savedEndpoints = new HashMap<>();
- int storage_port = Config.getOverrideLoadConfig().get().storage_port;
- for (Map.Entry<InetAddress, Map<String, String>> entry : SystemKeyspace.loadDcRackInfo().entrySet())
- {
- savedEndpoints.put(InetAddressAndPort.getByAddressOverrideDefaults(endpoint.address, storage_port),
- entry.getValue());
- }
- }
+ savedEndpoints = SystemKeyspace.loadDcRackInfo();
if (savedEndpoints.containsKey(endpoint))
return savedEndpoints.get(endpoint).get("data_center");
@@ -129,8 +120,9 @@
{
super.gossiperStarting();
-
+ Gossiper.instance.addLocalApplicationState(ApplicationState.INTERNAL_ADDRESS_AND_PORT,
+ StorageService.instance.valueFactory.internalAddressAndPort(FBUtilities.getLocalAddressAndPort()));
Gossiper.instance.addLocalApplicationState(ApplicationState.INTERNAL_IP,
- StorageService.instance.valueFactory.internalIP(FBUtilities.getLocalAddress().getHostAddress()));
+ StorageService.instance.valueFactory.internalIP(FBUtilities.getJustLocalAddress().getHostAddress()));
}
}
diff --git a/test/distributed/org/apache/cassandra/distributed/impl/INodeProvisionStrategy.java b/test/distributed/org/apache/cassandra/distributed/impl/INodeProvisionStrategy.java
new file mode 100644
index 0000000..32f82c0
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/impl/INodeProvisionStrategy.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.impl;
+
+public interface INodeProvisionStrategy
+{
+
+ public enum Strategy
+ {
+ OneNetworkInterface
+ {
+ INodeProvisionStrategy create(int subnet) {
+ return new INodeProvisionStrategy()
+ {
+ public String seedIp()
+ {
+ return "127.0." + subnet + ".1";
+ }
+
+ public int seedPort()
+ {
+ return 7012;
+ }
+
+ public String ipAddress(int nodeNum)
+ {
+ return "127.0." + subnet + ".1";
+ }
+
+ public int storagePort(int nodeNum)
+ {
+ return 7011 + nodeNum;
+ }
+
+ public int nativeTransportPort(int nodeNum)
+ {
+ return 9041 + nodeNum;
+ }
+ };
+ }
+ },
+ MultipleNetworkInterfaces
+ {
+ INodeProvisionStrategy create(int subnet) {
+ String ipPrefix = "127.0." + subnet + ".";
+ return new INodeProvisionStrategy()
+ {
+ public String seedIp()
+ {
+ return ipPrefix + "1";
+ }
+
+ public int seedPort()
+ {
+ return 7012;
+ }
+
+ public String ipAddress(int nodeNum)
+ {
+ return ipPrefix + nodeNum;
+ }
+
+ public int storagePort(int nodeNum)
+ {
+ return 7012;
+ }
+
+ public int nativeTransportPort(int nodeNum)
+ {
+ return 9042;
+ }
+ };
+ }
+ };
+ abstract INodeProvisionStrategy create(int subnet);
+ }
+
+ abstract String seedIp();
+ abstract int seedPort();
+ abstract String ipAddress(int nodeNum);
+ abstract int storagePort(int nodeNum);
+ abstract int nativeTransportPort(int nodeNum);
+}
diff --git a/test/distributed/org/apache/cassandra/distributed/impl/IUpgradeableInstance.java b/test/distributed/org/apache/cassandra/distributed/impl/IUpgradeableInstance.java
deleted file mode 100644
index d42e799..0000000
--- a/test/distributed/org/apache/cassandra/distributed/impl/IUpgradeableInstance.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.distributed.impl;
-
-import org.apache.cassandra.distributed.api.IInstance;
-import org.apache.cassandra.distributed.shared.Versions;
-
-// this lives outside the api package so that we do not have to worry about inter-version compatibility
-public interface IUpgradeableInstance extends IInstance
-{
- // only to be invoked while the node is shutdown!
- public void setVersion(Versions.Version version);
-}
diff --git a/test/distributed/org/apache/cassandra/distributed/impl/Instance.java b/test/distributed/org/apache/cassandra/distributed/impl/Instance.java
index 6066ecf..989bf6e 100644
--- a/test/distributed/org/apache/cassandra/distributed/impl/Instance.java
+++ b/test/distributed/org/apache/cassandra/distributed/impl/Instance.java
@@ -20,9 +20,7 @@
import java.io.File;
import java.io.IOException;
-import java.net.InetAddress;
import java.net.InetSocketAddress;
-import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
@@ -34,37 +32,37 @@
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import java.util.concurrent.TimeoutException;
-import java.util.function.BiConsumer;
import java.util.function.Function;
import javax.management.ListenerNotFoundException;
import javax.management.Notification;
import javax.management.NotificationListener;
+import com.google.common.annotations.VisibleForTesting;
+
+import io.netty.util.concurrent.GlobalEventExecutor;
import org.apache.cassandra.batchlog.BatchlogManager;
+import org.apache.cassandra.concurrent.ExecutorLocals;
import org.apache.cassandra.concurrent.ScheduledExecutors;
import org.apache.cassandra.concurrent.SharedExecutorPool;
-import org.apache.cassandra.concurrent.StageManager;
+import org.apache.cassandra.concurrent.Stage;
import org.apache.cassandra.config.Config;
import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.config.Schema;
-import org.apache.cassandra.config.SchemaConstants;
import org.apache.cassandra.cql3.CQLStatement;
+import org.apache.cassandra.cql3.QueryHandler;
import org.apache.cassandra.cql3.QueryOptions;
import org.apache.cassandra.cql3.QueryProcessor;
-import org.apache.cassandra.cql3.statements.ParsedStatement;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.db.Memtable;
import org.apache.cassandra.db.SystemKeyspace;
+import org.apache.cassandra.db.SystemKeyspaceMigrator40;
import org.apache.cassandra.db.commitlog.CommitLog;
import org.apache.cassandra.db.compaction.CompactionManager;
-import org.apache.cassandra.db.monitoring.ApproximateTime;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.distributed.api.ICluster;
import org.apache.cassandra.distributed.api.ICoordinator;
-import org.apache.cassandra.distributed.api.IInstance;
import org.apache.cassandra.distributed.api.IInstanceConfig;
import org.apache.cassandra.distributed.api.IInvokableInstance;
import org.apache.cassandra.distributed.api.IListen;
@@ -73,7 +71,6 @@
import org.apache.cassandra.distributed.api.SimpleQueryResult;
import org.apache.cassandra.distributed.mock.nodetool.InternalNodeProbe;
import org.apache.cassandra.distributed.mock.nodetool.InternalNodeProbeFactory;
-import org.apache.cassandra.distributed.shared.NetworkTopology;
import org.apache.cassandra.gms.ApplicationState;
import org.apache.cassandra.gms.Gossiper;
import org.apache.cassandra.gms.VersionedValue;
@@ -83,19 +80,23 @@
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.util.DataInputBuffer;
import org.apache.cassandra.io.util.DataOutputBuffer;
-import org.apache.cassandra.net.IMessageSink;
-import org.apache.cassandra.net.MessageIn;
-import org.apache.cassandra.net.MessageOut;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.net.Message;
import org.apache.cassandra.net.MessagingService;
-import org.apache.cassandra.schema.LegacySchemaMigrator;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.SchemaConstants;
+import org.apache.cassandra.service.ActiveRepairService;
import org.apache.cassandra.service.CassandraDaemon;
import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.service.DefaultFSErrorHandler;
import org.apache.cassandra.service.PendingRangeCalculatorService;
import org.apache.cassandra.service.QueryState;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.service.StorageServiceMBean;
-import org.apache.cassandra.streaming.StreamCoordinator;
-import org.apache.cassandra.streaming.StreamSession;
+import org.apache.cassandra.streaming.StreamReceiveTask;
+import org.apache.cassandra.streaming.StreamTransferTask;
+import org.apache.cassandra.streaming.async.StreamingInboundHandler;
import org.apache.cassandra.tools.NodeTool;
import org.apache.cassandra.tracing.TraceState;
import org.apache.cassandra.tracing.Tracing;
@@ -103,15 +104,18 @@
import org.apache.cassandra.utils.DiagnosticSnapshotService;
import org.apache.cassandra.utils.ExecutorUtils;
import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.JVMStabilityInspector;
import org.apache.cassandra.utils.Throwables;
-import org.apache.cassandra.utils.UUIDGen;
import org.apache.cassandra.utils.concurrent.Ref;
import org.apache.cassandra.utils.memory.BufferPool;
+import org.apache.cassandra.utils.progress.jmx.JMXBroadcastExecutor;
import static java.util.concurrent.TimeUnit.MINUTES;
import static org.apache.cassandra.distributed.api.Feature.GOSSIP;
import static org.apache.cassandra.distributed.api.Feature.NATIVE_PROTOCOL;
import static org.apache.cassandra.distributed.api.Feature.NETWORK;
+import static org.apache.cassandra.distributed.impl.DistributedTestSnitch.fromCassandraInetAddressAndPort;
+import static org.apache.cassandra.distributed.impl.DistributedTestSnitch.toCassandraInetAddressAndPort;
public class Instance extends IsolatedExecutor implements IInvokableInstance
{
@@ -131,12 +135,17 @@
super("node" + config.num(), classLoader);
this.config = config;
InstanceIDDefiner.setInstanceId(config.num());
- FBUtilities.setBroadcastInetAddress(config.broadcastAddress().getAddress());
+ FBUtilities.setBroadcastInetAddressAndPort(InetAddressAndPort.getByAddressOverrideDefaults(config.broadcastAddress().getAddress(),
+ config.broadcastAddress().getPort()));
// Set the config at instance creation, possibly before startup() has run on all other instances.
// setMessagingVersions below will call runOnInstance which will instantiate
// the MessagingService and dependencies preventing later changes to network parameters.
Config.setOverrideLoadConfig(() -> loadConfig(config));
+
+ // Enable streaming inbound handler tracking so they can be closed properly without leaking
+ // the blocking IO thread.
+ StreamingInboundHandler.trackInboundHandlers();
}
@Override
@@ -163,10 +172,9 @@
public SimpleQueryResult executeInternalWithResult(String query, Object... args)
{
return sync(() -> {
- ParsedStatement.Prepared prepared = QueryProcessor.prepareInternal(query);
- ResultMessage result = prepared.statement.executeInternal(QueryProcessor.internalQueryState(),
- QueryProcessor.makeInternalOptions(prepared, args));
-
+ QueryHandler.Prepared prepared = QueryProcessor.prepareInternal(query);
+ ResultMessage result = prepared.statement.executeLocally(QueryProcessor.internalQueryState(),
+ QueryProcessor.makeInternalOptions(prepared.statement, args));
return RowUtil.toQueryResult(result);
}).call();
}
@@ -195,15 +203,14 @@
sync(() -> {
try
{
- ClientState state = ClientState.forInternalCalls();
- state.setKeyspace(SchemaConstants.SYSTEM_KEYSPACE_NAME);
+ ClientState state = ClientState.forInternalCalls(SchemaConstants.SYSTEM_KEYSPACE_NAME);
QueryState queryState = new QueryState(state);
- CQLStatement statement = QueryProcessor.parseStatement(query, queryState).statement;
+ CQLStatement statement = QueryProcessor.parseStatement(query, queryState.getClientState());
statement.validate(state);
QueryOptions options = QueryOptions.forInternalCalls(Collections.emptyList());
- statement.executeInternal(queryState, options);
+ statement.executeLocally(queryState, options);
}
catch (Exception e)
{
@@ -212,214 +219,85 @@
}).run();
}
- private void registerMockMessaging(ICluster<IInstance> cluster)
+ private void registerMockMessaging(ICluster cluster)
{
- BiConsumer<InetSocketAddress, IMessage> deliverToInstance = (to, message) -> cluster.get(to).receiveMessage(message);
- BiConsumer<InetSocketAddress, IMessage> deliverToInstanceIfNotFiltered = (to, message) -> {
- int fromNum = config().num();
- int toNum = cluster.get(to).config().num();
-
- if (cluster.filters().permitOutbound(fromNum, toNum, message)
- && cluster.filters().permitInbound(fromNum, toNum, message))
- deliverToInstance.accept(to, message);
- };
-
- Map<InetAddress, InetSocketAddress> addressAndPortMap = new HashMap<>();
- cluster.stream().forEach(instance -> {
- InetSocketAddress addressAndPort = instance.broadcastAddress();
- if (!addressAndPort.equals(instance.config().broadcastAddress()))
- throw new IllegalStateException("addressAndPort mismatch: " + addressAndPort + " vs " + instance.config().broadcastAddress());
- InetSocketAddress prev = addressAndPortMap.put(addressAndPort.getAddress(),
- addressAndPort);
- if (null != prev)
- throw new IllegalStateException("This version of Cassandra does not support multiple nodes with the same InetAddress: " + addressAndPort + " vs " + prev);
- });
-
- MessagingService.instance().addMessageSink(new MessageDeliverySink(deliverToInstanceIfNotFiltered, addressAndPortMap::get));
- }
-
- // unnecessary if registerMockMessaging used
- private void registerFilters(ICluster cluster)
- {
- IInstance instance = this;
- MessagingService.instance().addMessageSink(new IMessageSink()
- {
- public boolean allowOutgoingMessage(MessageOut message, int id, InetAddress toAddress)
- {
- // Port is not passed in, so take a best guess at the destination port from this instance
- IInstance to = cluster.get(NetworkTopology.addressAndPort(toAddress,
- instance.config().broadcastAddress().getPort()));
- int fromNum = config().num();
- int toNum = to.config().num();
- return cluster.filters().permitOutbound(fromNum, toNum, serializeMessage(message, id,
- broadcastAddress(),
- to.config().broadcastAddress()));
- }
-
- public boolean allowIncomingMessage(MessageIn message, int id)
- {
- // Port is not passed in, so take a best guess at the destination port from this instance
- IInstance from = cluster.get(NetworkTopology.addressAndPort(message.from,
- instance.config().broadcastAddress().getPort()));
- int fromNum = from.config().num();
- int toNum = config().num();
-
-
- IMessage msg = serializeMessage(message, id, from.broadcastAddress(), broadcastAddress());
-
- return cluster.filters().permitInbound(fromNum, toNum, msg);
- }
- });
- }
-
- public static IMessage serializeMessage(MessageOut messageOut, int id, InetSocketAddress from, InetSocketAddress to)
- {
- try (DataOutputBuffer out = new DataOutputBuffer(1024))
- {
- int version = MessagingService.instance().getVersion(to.getAddress());
-
- out.writeInt(MessagingService.PROTOCOL_MAGIC);
- out.writeInt(id);
- long timestamp = System.currentTimeMillis();
- out.writeInt((int) timestamp);
- messageOut.serialize(out, version);
- return new MessageImpl(messageOut.verb.ordinal(), out.toByteArray(), id, version, from);
- }
- catch (IOException e)
- {
- throw new RuntimeException(e);
- }
- }
-
- public static IMessage serializeMessage(MessageIn messageIn, int id, InetSocketAddress from, InetSocketAddress to)
- {
- try (DataOutputBuffer out = new DataOutputBuffer(1024))
- {
- int version = MessagingService.instance().getVersion(to.getAddress());
-
- out.writeInt(MessagingService.PROTOCOL_MAGIC);
- out.writeInt(id);
- long timestamp = System.currentTimeMillis();
- out.writeInt((int) timestamp);
-
- MessageOut.serialize(out,
- from.getAddress(),
- messageIn.verb,
- messageIn.parameters,
- messageIn.payload,
- version);
-
- return new MessageImpl(messageIn.verb.ordinal(), out.toByteArray(), id, version, from);
- }
- catch (IOException e)
- {
- throw new RuntimeException(e);
- }
- }
-
- private class MessageDeliverySink implements IMessageSink
- {
- private final BiConsumer<InetSocketAddress, IMessage> deliver;
- private final Function<InetAddress, InetSocketAddress> lookupAddressAndPort;
-
- MessageDeliverySink(BiConsumer<InetSocketAddress, IMessage> deliver,
- Function<InetAddress, InetSocketAddress> lookupAddressAndPort)
- {
- this.deliver = deliver;
- this.lookupAddressAndPort = lookupAddressAndPort;
- }
-
- public boolean allowOutgoingMessage(MessageOut messageOut, int id, InetAddress to)
- {
- InetSocketAddress from = broadcastAddress();
- assert from.equals(lookupAddressAndPort.apply(messageOut.from));
-
- // Tracing logic - similar to org.apache.cassandra.net.OutboundTcpConnection.writeConnected
- byte[] sessionBytes = (byte[]) messageOut.parameters.get(Tracing.TRACE_HEADER);
- if (sessionBytes != null)
- {
- UUID sessionId = UUIDGen.getUUID(ByteBuffer.wrap(sessionBytes));
- TraceState state = Tracing.instance.get(sessionId);
- String message = String.format("Sending %s message to %s", messageOut.verb, to);
- // session may have already finished; see CASSANDRA-5668
- if (state == null)
- {
- byte[] traceTypeBytes = (byte[]) messageOut.parameters.get(Tracing.TRACE_TYPE);
- Tracing.TraceType traceType = traceTypeBytes == null ? Tracing.TraceType.QUERY : Tracing.TraceType.deserialize(traceTypeBytes[0]);
- Tracing.instance.trace(ByteBuffer.wrap(sessionBytes), message, traceType.getTTL());
- }
- else
- {
- state.trace(message);
- if (messageOut.verb == MessagingService.Verb.REQUEST_RESPONSE)
- Tracing.instance.doneWithNonLocalSession(state);
- }
- }
-
- InetSocketAddress toFull = lookupAddressAndPort.apply(to);
- deliver.accept(toFull,
- serializeMessage(messageOut, id, broadcastAddress(), toFull));
-
+ MessagingService.instance().outboundSink.add((message, to) -> {
+ InetSocketAddress toAddr = fromCassandraInetAddressAndPort(to);
+ cluster.get(toAddr).receiveMessage(serializeMessage(message.from(), to, message));
return false;
- }
+ });
+ }
- public boolean allowIncomingMessage(MessageIn message, int id)
+ private void registerInboundFilter(ICluster cluster)
+ {
+ MessagingService.instance().inboundSink.add(message -> {
+ IMessage serialized = serializeMessage(message.from(), toCassandraInetAddressAndPort(broadcastAddress()), message);
+ int fromNum = cluster.get(serialized.from()).config().num();
+ int toNum = config.num(); // since this instance is reciving the message, to will always be this instance
+ return cluster.filters().permitInbound(fromNum, toNum, serialized);
+ });
+ }
+
+ private void registerOutboundFilter(ICluster cluster)
+ {
+ MessagingService.instance().outboundSink.add((message, to) -> {
+ IMessage serialzied = serializeMessage(message.from(), to, message);
+ int fromNum = config.num(); // since this instance is sending the message, from will always be this instance
+ int toNum = cluster.get(fromCassandraInetAddressAndPort(to)).config().num();
+ return cluster.filters().permitOutbound(fromNum, toNum, serialzied);
+ });
+ }
+
+ public void uncaughtException(Thread thread, Throwable throwable)
+ {
+ sync(CassandraDaemon::uncaughtException).accept(thread, throwable);
+ }
+
+ private static IMessage serializeMessage(InetAddressAndPort from, InetAddressAndPort to, Message<?> messageOut)
+ {
+ try (DataOutputBuffer out = new DataOutputBuffer(1024))
{
- // we can filter to our heart's content on the outgoing message; no need to worry about incoming
- return true;
+ int version = MessagingService.instance().versions.get(to);
+ Message.serializer.serialize(messageOut, out, version);
+ return new MessageImpl(messageOut.verb().id, out.toByteArray(), messageOut.id(), version, fromCassandraInetAddressAndPort(from));
+ }
+ catch (IOException e)
+ {
+ throw new RuntimeException(e);
}
}
- public static MessageIn<Object> deserializeMessage(IMessage imessage)
+ @VisibleForTesting
+ public static Message<?> deserializeMessage(IMessage message)
{
- // Based on org.apache.cassandra.net.IncomingTcpConnection.receiveMessage
- try (DataInputBuffer input = new DataInputBuffer(imessage.bytes()))
+ try (DataInputBuffer in = new DataInputBuffer(message.bytes()))
{
- int version = imessage.version();
- if (version > MessagingService.current_version)
- {
- throw new IllegalStateException(String.format("Received message version %d but current version is %d",
- version,
- MessagingService.current_version));
- }
-
- MessagingService.validateMagic(input.readInt());
- int id;
- if (version < MessagingService.VERSION_20)
- id = Integer.parseInt(input.readUTF());
- else
- id = input.readInt();
- long currentTime = ApproximateTime.currentTimeMillis();
- return MessageIn.read(input, version, id, MessageIn.readConstructionTime(imessage.from().getAddress(), input, currentTime));
+ return Message.serializer.deserialize(in, toCassandraInetAddressAndPort(message.from()), message.version());
}
catch (Throwable t)
{
- throw new RuntimeException(t);
+ throw new RuntimeException("Can not deserialize message " + message, t);
}
}
- public void receiveMessage(IMessage imessage)
+ @Override
+ public void receiveMessage(IMessage message)
{
sync(() -> {
- // Based on org.apache.cassandra.net.IncomingTcpConnection.receiveMessage
- try
+ if (message.version() > MessagingService.current_version)
{
- MessageIn message = deserializeMessage(imessage);
- if (message == null)
- {
- // callback expired; nothing to do
- return;
- }
- if (message.version <= MessagingService.current_version)
- {
- MessagingService.instance().receive(message, imessage.id());
- }
- // else ignore message
+ throw new IllegalStateException(String.format("Node%d received message version %d but current version is %d",
+ this.config.num(),
+ message.version(),
+ MessagingService.current_version));
}
- catch (Throwable t)
- {
- throw new RuntimeException("Exception occurred on node " + broadcastAddress(), t);
- }
+
+ Message<?> messageIn = deserializeMessage(message);
+ Message.Header header = messageIn.header;
+ TraceState state = Tracing.instance.initializeFromMessage(header);
+ if (state != null) state.trace("{} message received from {}", header.verb, header.from);
+ header.verb.stage.execute(() -> MessagingService.instance().inboundSink.accept(messageIn),
+ ExecutorLocals.create(state));
}).run();
}
@@ -428,9 +306,10 @@
return callsOnInstance(() -> MessagingService.current_version).call();
}
+ @Override
public void setMessagingVersion(InetSocketAddress endpoint, int version)
{
- runOnInstance(() -> MessagingService.instance().setVersion(endpoint.getAddress(), version));
+ MessagingService.instance().versions.set(toCassandraInetAddressAndPort(endpoint), version);
}
public void flush(String keyspace)
@@ -458,18 +337,30 @@
sync(() -> {
try
{
+ FileUtils.setFSErrorHandler(new DefaultFSErrorHandler());
+
+ if (config.has(GOSSIP))
+ {
+ // TODO: hacky
+ System.setProperty("cassandra.ring_delay_ms", "5000");
+ System.setProperty("cassandra.consistent.rangemovement", "false");
+ System.setProperty("cassandra.consistent.simultaneousmoves.allow", "true");
+ }
+
mkdirs();
- assert config.networkTopology().contains(config.broadcastAddress());
+ assert config.networkTopology().contains(config.broadcastAddress()) : String.format("Network topology %s doesn't contain the address %s",
+ config.networkTopology(), config.broadcastAddress());
DistributedTestSnitch.assign(config.networkTopology());
DatabaseDescriptor.daemonInitialization();
DatabaseDescriptor.createAllDirectories();
+ CommitLog.instance.start();
- // We need to persist this as soon as possible after startup checks.
+ // We need to persist this as soon as possible after startup checks.
// This should be the first write to SystemKeyspace (CASSANDRA-11742)
SystemKeyspace.persistLocalMetadata();
- LegacySchemaMigrator.migrate();
+ SystemKeyspaceMigrator40.migrate();
try
{
@@ -495,7 +386,6 @@
if (config.has(NETWORK))
{
- registerFilters(cluster);
MessagingService.instance().listen();
}
else
@@ -505,12 +395,17 @@
// -- not sure what that means? SocketFactory.instance.getClass();
registerMockMessaging(cluster);
}
+ registerInboundFilter(cluster);
+ registerOutboundFilter(cluster);
+
+ JVMStabilityInspector.replaceKiller(new InstanceKiller());
// TODO: this is more than just gossip
if (config.has(GOSSIP))
{
StorageService.instance.initServer();
StorageService.instance.removeShutdownHook();
+ Gossiper.waitToSettle();
}
else
{
@@ -523,15 +418,19 @@
if (config.has(NATIVE_PROTOCOL))
{
+ // Start up virtual table support
+ CassandraDaemon.getInstanceForTesting().setupVirtualKeyspaces();
+
CassandraDaemon.getInstanceForTesting().initializeNativeTransport();
CassandraDaemon.getInstanceForTesting().startNativeTransport();
StorageService.instance.setRpcReady(true);
}
- if (!FBUtilities.getBroadcastAddress().equals(broadcastAddress().getAddress()))
- throw new IllegalStateException();
- if (DatabaseDescriptor.getStoragePort() != broadcastAddress().getPort())
- throw new IllegalStateException();
+ if (!FBUtilities.getBroadcastAddressAndPort().address.equals(broadcastAddress().getAddress()) ||
+ FBUtilities.getBroadcastAddressAndPort().port != broadcastAddress().getPort())
+ throw new IllegalStateException(String.format("%s != %s", FBUtilities.getBroadcastAddressAndPort(), broadcastAddress()));
+
+ ActiveRepairService.instance.start();
}
catch (Throwable t)
{
@@ -551,7 +450,7 @@
new File(dir).mkdirs();
}
- private static Config loadConfig(IInstanceConfig overrides)
+ private Config loadConfig(IInstanceConfig overrides)
{
Config config = new Config();
overrides.propagate(config, mapper);
@@ -584,28 +483,32 @@
for (int i = 0; i < tokens.size(); i++)
{
InetSocketAddress ep = hosts.get(i);
+ InetAddressAndPort addressAndPort = toCassandraInetAddressAndPort(ep);
UUID hostId = hostIds.get(i);
Token token = tokens.get(i);
Gossiper.runInGossipStageBlocking(() -> {
- Gossiper.instance.initializeNodeUnsafe(ep.getAddress(), hostId, 1);
- Gossiper.instance.injectApplicationState(ep.getAddress(),
+ Gossiper.instance.initializeNodeUnsafe(addressAndPort, hostId, 1);
+ Gossiper.instance.injectApplicationState(addressAndPort,
ApplicationState.TOKENS,
new VersionedValue.VersionedValueFactory(partitioner).tokens(Collections.singleton(token)));
- storageService.onChange(ep.getAddress(),
+ storageService.onChange(addressAndPort,
+ ApplicationState.STATUS_WITH_PORT,
+ new VersionedValue.VersionedValueFactory(partitioner).normal(Collections.singleton(token)));
+ storageService.onChange(addressAndPort,
ApplicationState.STATUS,
new VersionedValue.VersionedValueFactory(partitioner).normal(Collections.singleton(token)));
- Gossiper.instance.realMarkAlive(ep.getAddress(), Gossiper.instance.getEndpointStateForEndpoint(ep.getAddress()));
+ Gossiper.instance.realMarkAlive(addressAndPort, Gossiper.instance.getEndpointStateForEndpoint(addressAndPort));
});
int messagingVersion = cluster.get(ep).isShutdown()
? MessagingService.current_version
: Math.min(MessagingService.current_version, cluster.get(ep).getMessagingVersion());
- MessagingService.instance().setVersion(ep.getAddress(), messagingVersion);
+ MessagingService.instance().versions.set(addressAndPort, messagingVersion);
}
// check that all nodes are in token metadata
for (int i = 0; i < tokens.size(); ++i)
- assert storageService.getTokenMetadata().isMember(hosts.get(i).getAddress());
+ assert storageService.getTokenMetadata().isMember(toCassandraInetAddressAndPort(hosts.get(i)));
}
catch (Throwable e) // UnknownHostException
{
@@ -621,9 +524,6 @@
@Override
public Future<Void> shutdown(boolean graceful)
{
- if (!graceful)
- MessagingService.instance().shutdown(false);
-
Future<?> future = async((ExecutorService executor) -> {
Throwable error = null;
@@ -641,28 +541,34 @@
CompactionManager.instance::forceShutdown,
() -> BatchlogManager.instance.shutdownAndWait(1L, MINUTES),
HintsService.instance::shutdownBlocking,
- () -> StreamCoordinator.shutdownAndWait(1L, MINUTES),
- () -> StreamSession.shutdownAndWait(1L, MINUTES),
+ StreamingInboundHandler::shutdown,
+ () -> StreamReceiveTask.shutdownAndWait(1L, MINUTES),
+ () -> StreamTransferTask.shutdownAndWait(1L, MINUTES),
() -> SecondaryIndexManager.shutdownAndWait(1L, MINUTES),
() -> IndexSummaryManager.instance.shutdownAndWait(1L, MINUTES),
() -> ColumnFamilyStore.shutdownExecutorsAndWait(1L, MINUTES),
- () -> PendingRangeCalculatorService.instance.shutdownExecutor(1L, MINUTES),
+ () -> PendingRangeCalculatorService.instance.shutdownAndWait(1L, MINUTES),
() -> BufferPool.shutdownLocalCleaner(1L, MINUTES),
() -> Ref.shutdownReferenceReaper(1L, MINUTES),
() -> Memtable.MEMORY_POOL.shutdownAndWait(1L, MINUTES),
- () -> SSTableReader.shutdownBlocking(1L, MINUTES),
- () -> DiagnosticSnapshotService.instance.shutdownAndWait(1L, MINUTES)
- );
- error = parallelRun(error, executor,
+ () -> DiagnosticSnapshotService.instance.shutdownAndWait(1L, MINUTES),
() -> ScheduledExecutors.shutdownAndWait(1L, MINUTES),
- MessagingService.instance()::shutdown
+ () -> SSTableReader.shutdownBlocking(1L, MINUTES),
+ () -> shutdownAndWait(Collections.singletonList(ActiveRepairService.repairCommandExecutor())),
+ () -> ScheduledExecutors.shutdownAndWait(1L, MINUTES)
+ );
+
+ error = parallelRun(error, executor,
+ CommitLog.instance::shutdownBlocking,
+ () -> MessagingService.instance().shutdown(1L, MINUTES, false, true)
);
error = parallelRun(error, executor,
- () -> StageManager.shutdownAndWait(1L, MINUTES),
+ () -> GlobalEventExecutor.INSTANCE.awaitInactivity(1l, MINUTES),
+ () -> Stage.shutdownAndWait(1L, MINUTES),
() -> SharedExecutorPool.SHARED.shutdownAndWait(1L, MINUTES)
);
error = parallelRun(error, executor,
- CommitLog.instance::shutdownBlocking
+ () -> shutdownAndWait(Collections.singletonList(JMXBroadcastExecutor.executor))
);
Throwables.maybeFail(error);
@@ -744,12 +650,6 @@
}
}
- public void uncaughtException(Thread thread, Throwable throwable)
- {
- System.out.println(String.format("Exception %s occurred on thread %s", throwable.getMessage(), thread.getName()));
- throwable.printStackTrace();
- }
-
public long killAttempts()
{
return callOnInstance(InstanceKiller::getKillAttempts);
@@ -793,4 +693,4 @@
}
return accumulate;
}
-}
+}
\ No newline at end of file
diff --git a/test/distributed/org/apache/cassandra/distributed/impl/InstanceConfig.java b/test/distributed/org/apache/cassandra/distributed/impl/InstanceConfig.java
index 212fcc4..a8ed918 100644
--- a/test/distributed/org/apache/cassandra/distributed/impl/InstanceConfig.java
+++ b/test/distributed/org/apache/cassandra/distributed/impl/InstanceConfig.java
@@ -66,12 +66,15 @@
String broadcast_rpc_address,
String rpc_address,
String seedIp,
+ int seedPort,
String saved_caches_directory,
String[] data_file_directories,
String commitlog_directory,
String hints_directory,
String cdc_raw_directory,
- String initial_token)
+ String initial_token,
+ int storage_port,
+ int native_transport_port)
{
this.num = num;
this.networkTopology = networkTopology;
@@ -97,10 +100,13 @@
.set("concurrent_compactors", 1)
.set("memtable_heap_space_in_mb", 10)
.set("commitlog_sync", "batch")
- .set("storage_port", 7012)
+ .set("storage_port", storage_port)
+ .set("native_transport_port", native_transport_port)
.set("endpoint_snitch", DistributedTestSnitch.class.getName())
.set("seed_provider", new ParameterizedClass(SimpleSeedProvider.class.getName(),
- Collections.singletonMap("seeds", seedIp)))
+ Collections.singletonMap("seeds", seedIp + ":" + seedPort)))
+ // required settings for dtest functionality
+ .set("diagnostic_events_enabled", true)
.set("auto_bootstrap", false)
// capacities that are based on `totalMemory` that should be fixed size
.set("index_summary_capacity_in_mb", 50l)
@@ -261,21 +267,28 @@
return (String)params.get(name);
}
- public static InstanceConfig generate(int nodeNum, String ipAddress, NetworkTopology networkTopology, File root, String token, String seedIp)
+ public static InstanceConfig generate(int nodeNum,
+ INodeProvisionStrategy provisionStrategy,
+ NetworkTopology networkTopology,
+ File root,
+ String token)
{
return new InstanceConfig(nodeNum,
networkTopology,
- ipAddress,
- ipAddress,
- ipAddress,
- ipAddress,
- seedIp,
+ provisionStrategy.ipAddress(nodeNum),
+ provisionStrategy.ipAddress(nodeNum),
+ provisionStrategy.ipAddress(nodeNum),
+ provisionStrategy.ipAddress(nodeNum),
+ provisionStrategy.seedIp(),
+ provisionStrategy.seedPort(),
String.format("%s/node%d/saved_caches", root, nodeNum),
new String[] { String.format("%s/node%d/data", root, nodeNum) },
String.format("%s/node%d/commitlog", root, nodeNum),
String.format("%s/node%d/hints", root, nodeNum),
String.format("%s/node%d/cdc", root, nodeNum),
- token);
+ token,
+ provisionStrategy.storagePort(nodeNum),
+ provisionStrategy.nativeTransportPort(nodeNum));
}
public InstanceConfig forVersion(Versions.Major major)
diff --git a/test/distributed/org/apache/cassandra/distributed/impl/IsolatedExecutor.java b/test/distributed/org/apache/cassandra/distributed/impl/IsolatedExecutor.java
index fc31fdf..0d8f96f 100644
--- a/test/distributed/org/apache/cassandra/distributed/impl/IsolatedExecutor.java
+++ b/test/distributed/org/apache/cassandra/distributed/impl/IsolatedExecutor.java
@@ -47,6 +47,7 @@
import org.apache.cassandra.concurrent.NamedThreadFactory;
import org.apache.cassandra.distributed.api.IIsolatedExecutor;
import org.apache.cassandra.utils.ExecutorUtils;
+import org.apache.cassandra.utils.Throwables;
public class IsolatedExecutor implements IIsolatedExecutor
{
@@ -65,7 +66,7 @@
public Future<Void> shutdown()
{
- isolatedExecutor.shutdown();
+ isolatedExecutor.shutdownNow();
/* Use a thread pool with a core pool size of zero to terminate the thread as soon as possible
** so the instance class loader can be garbage collected. Uses a custom thread factory
@@ -202,11 +203,12 @@
}
catch (InterruptedException e)
{
- throw new RuntimeException(e);
+ Thread.currentThread().interrupt();
+ throw Throwables.throwAsUncheckedException(e);
}
catch (ExecutionException e)
{
- throw new RuntimeException(e.getCause());
+ throw Throwables.throwAsUncheckedException(e.getCause());
}
}
diff --git a/test/distributed/org/apache/cassandra/distributed/impl/Listen.java b/test/distributed/org/apache/cassandra/distributed/impl/Listen.java
index 9d9beea..e37c4f7 100644
--- a/test/distributed/org/apache/cassandra/distributed/impl/Listen.java
+++ b/test/distributed/org/apache/cassandra/distributed/impl/Listen.java
@@ -18,12 +18,12 @@
package org.apache.cassandra.distributed.impl;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.locks.LockSupport;
-import java.util.function.Supplier;
+import java.util.function.Consumer;
+import org.apache.cassandra.diag.DiagnosticEventService;
import org.apache.cassandra.distributed.api.IListen;
+import org.apache.cassandra.gms.GossiperEvent;
+import org.apache.cassandra.schema.SchemaEvent;
public class Listen implements IListen
{
@@ -36,31 +36,15 @@
public Cancel schema(Runnable onChange)
{
- return start(onChange, instance::schemaVersion);
+ Consumer<SchemaEvent> consumer = event -> onChange.run();
+ DiagnosticEventService.instance().subscribe(SchemaEvent.class, SchemaEvent.SchemaEventType.VERSION_UPDATED, consumer);
+ return () -> DiagnosticEventService.instance().unsubscribe(SchemaEvent.class, consumer);
}
public Cancel liveMembers(Runnable onChange)
{
- return start(onChange, instance::liveMemberCount);
- }
-
- protected <T> Cancel start(Runnable onChange, Supplier<T> valueSupplier) {
- AtomicBoolean cancel = new AtomicBoolean(false);
- instance.isolatedExecutor.execute(() -> {
- T prev = valueSupplier.get();
- while (true)
- {
- if (cancel.get())
- return;
-
- LockSupport.parkNanos(TimeUnit.MILLISECONDS.toNanos(10L));
-
- T cur = valueSupplier.get();
- if (!prev.equals(cur))
- onChange.run();
- prev = cur;
- }
- });
- return () -> cancel.set(true);
+ Consumer<GossiperEvent> consumer = event -> onChange.run();
+ DiagnosticEventService.instance().subscribe(GossiperEvent.class, GossiperEvent.GossiperEventType.REAL_MARKED_ALIVE, consumer);
+ return () -> DiagnosticEventService.instance().unsubscribe(GossiperEvent.class, consumer);
}
}
diff --git a/test/distributed/org/apache/cassandra/distributed/mock/nodetool/InternalNodeProbe.java b/test/distributed/org/apache/cassandra/distributed/mock/nodetool/InternalNodeProbe.java
index b1ff9fc..9ab264e 100644
--- a/test/distributed/org/apache/cassandra/distributed/mock/nodetool/InternalNodeProbe.java
+++ b/test/distributed/org/apache/cassandra/distributed/mock/nodetool/InternalNodeProbe.java
@@ -18,11 +18,9 @@
package org.apache.cassandra.distributed.mock.nodetool;
-import java.io.IOException;
import java.lang.management.ManagementFactory;
import java.util.Iterator;
import java.util.Map;
-
import javax.management.ListenerNotFoundException;
import com.google.common.collect.Multimap;
@@ -41,6 +39,7 @@
import org.apache.cassandra.locator.EndpointSnitchInfoMBean;
import org.apache.cassandra.metrics.CassandraMetricsRegistry;
import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.service.ActiveRepairService;
import org.apache.cassandra.service.CacheService;
import org.apache.cassandra.service.CacheServiceMBean;
import org.apache.cassandra.service.GCInspector;
@@ -67,7 +66,6 @@
mbeanServerConn = null;
jmxc = null;
-
if (withNotifications)
{
ssProxy = StorageService.instance;
@@ -88,8 +86,6 @@
}
ssProxy = mock;
}
-
- ssProxy = StorageService.instance;
msProxy = MessagingService.instance();
streamProxy = StreamManager.instance;
compactionProxy = CompactionManager.instance;
@@ -100,21 +96,25 @@
gcProxy = new GCInspector();
gossProxy = Gossiper.instance;
bmProxy = BatchlogManager.instance;
+ arsProxy = ActiveRepairService.instance;
memProxy = ManagementFactory.getMemoryMXBean();
runtimeProxy = ManagementFactory.getRuntimeMXBean();
}
- public void close() throws IOException
+ @Override
+ public void close()
{
// nothing to close. no-op
}
+ @Override
// overrides all the methods referenced mbeanServerConn/jmxc in super
public EndpointSnitchInfoMBean getEndpointSnitchInfoProxy()
{
return new EndpointSnitchInfo();
}
+ @Override
public DynamicEndpointSnitchMBean getDynamicEndpointSnitchInfoProxy()
{
return (DynamicEndpointSnitchMBean) DatabaseDescriptor.createEndpointSnitch(true, DatabaseDescriptor.getRawConfig().endpoint_snitch);
@@ -125,57 +125,68 @@
return cacheService;
}
+ @Override
public ColumnFamilyStoreMBean getCfsProxy(String ks, String cf)
{
return Keyspace.open(ks).getColumnFamilyStore(cf);
}
// The below methods are only used by the commands (i.e. Info, TableHistogram, TableStats, etc.) that display informations. Not useful for dtest, so disable it.
+ @Override
public Object getCacheMetric(String cacheType, String metricName)
{
throw new UnsupportedOperationException();
}
+ @Override
public Iterator<Map.Entry<String, ColumnFamilyStoreMBean>> getColumnFamilyStoreMBeanProxies()
{
throw new UnsupportedOperationException();
}
+ @Override
public Multimap<String, String> getThreadPools()
{
throw new UnsupportedOperationException();
}
+ @Override
public Object getThreadPoolMetric(String pathName, String poolName, String metricName)
{
throw new UnsupportedOperationException();
}
+ @Override
public Object getColumnFamilyMetric(String ks, String cf, String metricName)
{
throw new UnsupportedOperationException();
}
+ @Override
public CassandraMetricsRegistry.JmxTimerMBean getProxyMetric(String scope)
{
throw new UnsupportedOperationException();
}
+ @Override
public CassandraMetricsRegistry.JmxTimerMBean getMessagingQueueWaitMetrics(String verb)
{
throw new UnsupportedOperationException();
}
+ @Override
public Object getCompactionMetric(String metricName)
{
throw new UnsupportedOperationException();
}
+ @Override
public Object getClientMetric(String metricName)
{
throw new UnsupportedOperationException();
}
+ @Override
public long getStorageMetric(String metricName)
{
throw new UnsupportedOperationException();
diff --git a/test/distributed/org/apache/cassandra/distributed/mock/nodetool/InternalNodeProbeFactory.java b/test/distributed/org/apache/cassandra/distributed/mock/nodetool/InternalNodeProbeFactory.java
index e7734da..1904aa7 100644
--- a/test/distributed/org/apache/cassandra/distributed/mock/nodetool/InternalNodeProbeFactory.java
+++ b/test/distributed/org/apache/cassandra/distributed/mock/nodetool/InternalNodeProbeFactory.java
@@ -21,9 +21,9 @@
import java.io.IOException;
import org.apache.cassandra.tools.NodeProbe;
-import org.apache.cassandra.tools.NodeProbeFactory;
+import org.apache.cassandra.tools.INodeProbeFactory;
-public class InternalNodeProbeFactory extends NodeProbeFactory
+public class InternalNodeProbeFactory implements INodeProbeFactory
{
private final boolean withNotifications;
diff --git a/test/distributed/org/apache/cassandra/distributed/shared/RepairResult.java b/test/distributed/org/apache/cassandra/distributed/shared/RepairResult.java
new file mode 100644
index 0000000..7be381a
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/shared/RepairResult.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.shared;
+
+public class RepairResult
+{
+ public final boolean success;
+ public final boolean wasInconsistent;
+
+ public RepairResult(boolean success, boolean wasInconsistent)
+ {
+ this.success = success;
+ this.wasInconsistent = wasInconsistent;
+ }
+}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/CasWriteTest.java b/test/distributed/org/apache/cassandra/distributed/test/CasWriteTest.java
new file mode 100644
index 0000000..81b52f7
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/test/CasWriteTest.java
@@ -0,0 +1,294 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Consumer;
+import java.util.function.Function;
+import java.util.function.Supplier;
+
+import com.google.common.util.concurrent.Uninterruptibles;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.distributed.Cluster;
+import org.apache.cassandra.distributed.api.ConsistencyLevel;
+import org.apache.cassandra.distributed.api.ICluster;
+import org.apache.cassandra.distributed.shared.InstanceClassLoader;
+import org.apache.cassandra.exceptions.CasWriteTimeoutException;
+import org.apache.cassandra.exceptions.CasWriteUnknownResultException;
+import org.apache.cassandra.net.Verb;
+import org.apache.cassandra.utils.FBUtilities;
+import org.hamcrest.BaseMatcher;
+import org.hamcrest.Description;
+
+import static org.hamcrest.CoreMatchers.containsString;
+import static org.apache.cassandra.distributed.shared.AssertUtils.*;
+
+// TODO: this test should be removed after running in-jvm dtests is set up via the shared API repository
+public class CasWriteTest extends TestBaseImpl
+{
+ // Sharing the same cluster to boost test speed. Using a pkGen to make sure queries has distinct pk value for paxos instances.
+ private static ICluster cluster;
+ private static final AtomicInteger pkGen = new AtomicInteger(1_000); // preserve any pk values less than 1000 for manual queries.
+ private static final Logger logger = LoggerFactory.getLogger(CasWriteTest.class);
+
+ @Rule
+ public ExpectedException thrown = ExpectedException.none();
+
+ @BeforeClass
+ public static void setupCluster() throws Throwable
+ {
+ cluster = init(Cluster.build().withNodes(3).start());
+ cluster.schemaChange("CREATE TABLE " + KEYSPACE + ".tbl (pk int, ck int, v int, PRIMARY KEY (pk, ck))");
+ }
+
+ @AfterClass
+ public static void close() throws Exception
+ {
+ cluster.close();
+ cluster = null;
+ }
+
+ @Before @After
+ public void resetFilters()
+ {
+ cluster.filters().reset();
+ }
+
+ @Test
+ public void testCasWriteSuccessWithNoContention()
+ {
+ cluster.coordinator(1).execute("INSERT INTO " + KEYSPACE + ".tbl (pk, ck, v) VALUES (1, 1, 1) IF NOT EXISTS",
+ ConsistencyLevel.QUORUM);
+ assertRows(cluster.coordinator(1).execute("SELECT * FROM " + KEYSPACE + ".tbl WHERE pk = 1",
+ ConsistencyLevel.QUORUM),
+ row(1, 1, 1));
+
+ cluster.coordinator(1).execute("UPDATE " + KEYSPACE + ".tbl SET v = 2 WHERE pk = 1 AND ck = 1 IF v = 1",
+ ConsistencyLevel.QUORUM);
+ assertRows(cluster.coordinator(1).execute("SELECT * FROM " + KEYSPACE + ".tbl WHERE pk = 1",
+ ConsistencyLevel.QUORUM),
+ row(1, 1, 2));
+ }
+
+ @Test
+ public void testCasWriteTimeoutAtPreparePhase_ReqLost()
+ {
+ expectCasWriteTimeout();
+ cluster.filters().verbs(Verb.PAXOS_PREPARE_REQ.id).from(1).to(2, 3).drop().on(); // drop the internode messages to acceptors
+ cluster.coordinator(1).execute(mkUniqueCasInsertQuery(1), ConsistencyLevel.QUORUM);
+ }
+
+ @Test
+ public void testCasWriteTimeoutAtPreparePhase_RspLost()
+ {
+ expectCasWriteTimeout();
+ cluster.filters().verbs(Verb.PAXOS_PREPARE_RSP.id).from(2, 3).to(1).drop().on(); // drop the internode messages to acceptors
+ cluster.coordinator(1).execute(mkUniqueCasInsertQuery(1), ConsistencyLevel.QUORUM);
+ }
+
+ @Test
+ public void testCasWriteTimeoutAtProposePhase_ReqLost()
+ {
+ expectCasWriteTimeout();
+ cluster.filters().verbs(Verb.PAXOS_PROPOSE_REQ.id).from(1).to(2, 3).drop().on();
+ cluster.coordinator(1).execute(mkUniqueCasInsertQuery(1), ConsistencyLevel.QUORUM);
+ }
+
+ @Test
+ public void testCasWriteTimeoutAtProposePhase_RspLost()
+ {
+ expectCasWriteTimeout();
+ cluster.filters().verbs(Verb.PAXOS_PROPOSE_RSP.id).from(2, 3).to(1).drop().on();
+ cluster.coordinator(1).execute(mkUniqueCasInsertQuery(1), ConsistencyLevel.QUORUM);
+ }
+
+ @Test
+ public void testCasWriteTimeoutAtCommitPhase_ReqLost()
+ {
+ expectCasWriteTimeout();
+ cluster.filters().verbs(Verb.PAXOS_COMMIT_REQ.id).from(1).to(2, 3).drop().on();
+ cluster.coordinator(1).execute(mkUniqueCasInsertQuery(1), ConsistencyLevel.QUORUM);
+ }
+
+ @Test
+ public void testCasWriteTimeoutAtCommitPhase_RspLost()
+ {
+ expectCasWriteTimeout();
+ cluster.filters().verbs(Verb.PAXOS_COMMIT_RSP.id).from(2, 3).to(1).drop().on();
+ cluster.coordinator(1).execute(mkUniqueCasInsertQuery(1), ConsistencyLevel.QUORUM);
+ }
+
+
+
+ @Test
+ public void casWriteContentionTimeoutTest() throws InterruptedException
+ {
+ testWithContention(101,
+ Arrays.asList(1, 3),
+ c -> {
+ c.filters().reset();
+ c.filters().verbs(Verb.PAXOS_PREPARE_REQ.id).from(1).to(3).drop();
+ c.filters().verbs(Verb.PAXOS_PROPOSE_REQ.id).from(1).to(2).drop();
+ },
+ failure ->
+ failure.get() != null &&
+ failure.get()
+ .getClass().getCanonicalName()
+ .equals(CasWriteTimeoutException.class.getCanonicalName()),
+ "Expecting cause to be CasWriteTimeoutException");
+ }
+
+ private void testWithContention(int testUid,
+ List<Integer> contendingNodes,
+ Consumer<ICluster> setupForEachRound,
+ Function<AtomicReference<Throwable>, Boolean> expectedException,
+ String assertHintMessage) throws InterruptedException
+ {
+ assert contendingNodes.size() == 2;
+ AtomicInteger curPk = new AtomicInteger(1);
+ ExecutorService es = Executors.newFixedThreadPool(3);
+ AtomicReference<Throwable> failure = new AtomicReference<>();
+ Supplier<Boolean> hasExpectedException = () -> expectedException.apply(failure);
+ while (!hasExpectedException.get())
+ {
+ failure.set(null);
+ setupForEachRound.accept(cluster);
+
+ List<Future<?>> futures = new ArrayList<>();
+ CountDownLatch latch = new CountDownLatch(3);
+ contendingNodes.forEach(nodeId -> {
+ String query = mkCasInsertQuery((a) -> curPk.get(), testUid, nodeId);
+ futures.add(es.submit(() -> {
+ try
+ {
+ latch.countDown();
+ latch.await(1, TimeUnit.SECONDS); // help threads start at approximately same time
+ cluster.coordinator(nodeId).execute(query, ConsistencyLevel.QUORUM);
+ }
+ catch (Throwable t)
+ {
+ failure.set(t);
+ }
+ }));
+ });
+
+ FBUtilities.waitOnFutures(futures);
+ curPk.incrementAndGet();
+ }
+
+ es.shutdownNow();
+ es.awaitTermination(1, TimeUnit.MINUTES);
+ Assert.assertTrue(assertHintMessage, hasExpectedException.get());
+ }
+
+ private void expectCasWriteTimeout()
+ {
+ thrown.expect(new BaseMatcher<Throwable>()
+ {
+ public boolean matches(Object item)
+ {
+ return InstanceClassLoader.wasLoadedByAnInstanceClassLoader(item.getClass());
+ }
+
+ public void describeTo(Description description)
+ {
+ description.appendText("Cause should be loaded by InstanceClassLoader");
+ }
+ });
+ // unable to assert on class becuase the exception thrown was loaded by a differnet classloader, InstanceClassLoader
+ // therefor asserts the FQCN name present in the message as a workaround
+ thrown.expect(new BaseMatcher<Throwable>()
+ {
+ public boolean matches(Object item)
+ {
+ return item.getClass().getCanonicalName().equals(CasWriteTimeoutException.class.getCanonicalName());
+ }
+
+ public void describeTo(Description description)
+ {
+ description.appendText("Class was expected to be " + CasWriteTimeoutException.class.getCanonicalName() + " but was not");
+ }
+ });
+ thrown.expectMessage(containsString("CAS operation timed out"));
+ }
+
+ @Test
+ public void testWriteUnknownResult()
+ {
+ cluster.filters().reset();
+ int pk = pkGen.getAndIncrement();
+ CountDownLatch ready = new CountDownLatch(1);
+ cluster.filters().verbs(Verb.PAXOS_PROPOSE_REQ.id).from(1).to(2, 3).messagesMatching((from, to, msg) -> {
+ if (to == 2)
+ {
+ // Inject a single CAS request in-between prepare and propose phases
+ cluster.coordinator(2).execute(mkCasInsertQuery((a) -> pk, 1, 2),
+ ConsistencyLevel.QUORUM);
+ ready.countDown();
+ } else {
+ Uninterruptibles.awaitUninterruptibly(ready);
+ }
+ return false;
+ }).drop();
+
+ try
+ {
+ cluster.coordinator(1).execute(mkCasInsertQuery((a) -> pk, 1, 1), ConsistencyLevel.QUORUM);
+ }
+ catch (Throwable t)
+ {
+ Assert.assertEquals("Expecting cause to be CasWriteUnknownResultException",
+ CasWriteUnknownResultException.class.getCanonicalName(), t.getClass().getCanonicalName());
+ return;
+ }
+ Assert.fail("Expecting test to throw a CasWriteUnknownResultException");
+ }
+
+ // every invokation returns a query with an unique pk
+ private String mkUniqueCasInsertQuery(int v)
+ {
+ return mkCasInsertQuery(AtomicInteger::getAndIncrement, 1, v);
+ }
+
+ private String mkCasInsertQuery(Function<AtomicInteger, Integer> pkFunc, int ck, int v)
+ {
+ String query = String.format("INSERT INTO %s.tbl (pk, ck, v) VALUES (%d, %d, %d) IF NOT EXISTS", KEYSPACE, pkFunc.apply(pkGen), ck, v);
+ logger.info("Generated query: " + query);
+ return query;
+ }
+}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/DistributedRepairUtils.java b/test/distributed/org/apache/cassandra/distributed/test/DistributedRepairUtils.java
new file mode 100644
index 0000000..023d02c
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/test/DistributedRepairUtils.java
@@ -0,0 +1,208 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test;
+
+import java.util.Collections;
+import java.util.Set;
+import java.util.function.Consumer;
+
+import com.google.common.collect.ImmutableSet;
+import org.apache.commons.lang3.ArrayUtils;
+import org.junit.Assert;
+
+import org.apache.cassandra.distributed.api.ConsistencyLevel;
+import org.apache.cassandra.distributed.api.IInvokableInstance;
+import org.apache.cassandra.distributed.api.NodeToolResult;
+import org.apache.cassandra.distributed.api.QueryResult;
+import org.apache.cassandra.distributed.api.Row;
+import org.apache.cassandra.distributed.impl.AbstractCluster;
+import org.apache.cassandra.metrics.StorageMetrics;
+
+import static org.apache.cassandra.utils.Retry.retryWithBackoffBlocking;
+
+public final class DistributedRepairUtils
+{
+ public static final int DEFAULT_COORDINATOR = 1;
+
+ private DistributedRepairUtils()
+ {
+
+ }
+
+ public static NodeToolResult repair(AbstractCluster<?> cluster, RepairType repairType, boolean withNotifications, String... args) {
+ return repair(cluster, DEFAULT_COORDINATOR, repairType, withNotifications, args);
+ }
+
+ public static NodeToolResult repair(AbstractCluster<?> cluster, int node, RepairType repairType, boolean withNotifications, String... args) {
+ args = repairType.append(args);
+ args = ArrayUtils.addAll(new String[] { "repair" }, args);
+ return cluster.get(node).nodetoolResult(withNotifications, args);
+ }
+
+ public static <I extends IInvokableInstance, C extends AbstractCluster<I>> long getRepairExceptions(C cluster)
+ {
+ return getRepairExceptions(cluster, DEFAULT_COORDINATOR);
+ }
+
+ public static <I extends IInvokableInstance, C extends AbstractCluster<I>> long getRepairExceptions(C cluster, int node)
+ {
+ return cluster.get(node).callOnInstance(() -> StorageMetrics.repairExceptions.getCount());
+ }
+
+ public static QueryResult queryParentRepairHistory(AbstractCluster<?> cluster, String ks, String table)
+ {
+ return queryParentRepairHistory(cluster, DEFAULT_COORDINATOR, ks, table);
+ }
+
+ public static QueryResult queryParentRepairHistory(AbstractCluster<?> cluster, int coordinator, String ks, String table)
+ {
+ // This is kinda brittle since the caller never gets the ID and can't ask for the ID; it needs to infer the id
+ // this logic makes the assumption the ks/table pairs are unique (should be or else create should fail) so any
+ // repair for that pair will be the repair id
+ Set<String> tableNames = table == null? Collections.emptySet() : ImmutableSet.of(table);
+
+ QueryResult rs = retryWithBackoffBlocking(10, () -> cluster.coordinator(coordinator)
+ .executeWithResult("SELECT * FROM system_distributed.parent_repair_history", ConsistencyLevel.QUORUM)
+ .filter(row -> ks.equals(row.getString("keyspace_name")))
+ .filter(row -> tableNames.equals(row.getSet("columnfamily_names"))));
+ return rs;
+ }
+
+ public static void assertParentRepairNotExist(AbstractCluster<?> cluster, String ks, String table)
+ {
+ assertParentRepairNotExist(cluster, DEFAULT_COORDINATOR, ks, table);
+ }
+
+ public static void assertParentRepairNotExist(AbstractCluster<?> cluster, int coordinator, String ks, String table)
+ {
+ QueryResult rs = queryParentRepairHistory(cluster, coordinator, ks, table);
+ Assert.assertFalse("No repairs should be found but at least one found", rs.hasNext());
+ }
+
+ public static void assertParentRepairNotExist(AbstractCluster<?> cluster, String ks)
+ {
+ assertParentRepairNotExist(cluster, DEFAULT_COORDINATOR, ks);
+ }
+
+ public static void assertParentRepairNotExist(AbstractCluster<?> cluster, int coordinator, String ks)
+ {
+ QueryResult rs = queryParentRepairHistory(cluster, coordinator, ks, null);
+ Assert.assertFalse("No repairs should be found but at least one found", rs.hasNext());
+ }
+
+ public static void assertParentRepairSuccess(AbstractCluster<?> cluster, String ks, String table)
+ {
+ assertParentRepairSuccess(cluster, DEFAULT_COORDINATOR, ks, table);
+ }
+
+ public static void assertParentRepairSuccess(AbstractCluster<?> cluster, int coordinator, String ks, String table)
+ {
+ QueryResult rs = queryParentRepairHistory(cluster, coordinator, ks, table);
+ validateExistingParentRepair(rs, row -> {
+ // check completed
+ Assert.assertNotNull("finished_at not found, the repair is not complete?", row.getTimestamp("finished_at"));
+
+ // check not failed (aka success)
+ Assert.assertNull("Exception found", row.getString("exception_stacktrace"));
+ Assert.assertNull("Exception found", row.getString("exception_message"));
+ });
+ }
+
+ public static void assertParentRepairFailedWithMessageContains(AbstractCluster<?> cluster, String ks, String table, String message)
+ {
+ assertParentRepairFailedWithMessageContains(cluster, DEFAULT_COORDINATOR, ks, table, message);
+ }
+
+ public static void assertParentRepairFailedWithMessageContains(AbstractCluster<?> cluster, int coordinator, String ks, String table, String message)
+ {
+ QueryResult rs = queryParentRepairHistory(cluster, coordinator, ks, table);
+ validateExistingParentRepair(rs, row -> {
+ // check completed
+ Assert.assertNotNull("finished_at not found, the repair is not complete?", row.getTimestamp("finished_at"));
+
+ // check failed
+ Assert.assertNotNull("Exception not found", row.getString("exception_stacktrace"));
+ String exceptionMessage = row.getString("exception_message");
+ Assert.assertNotNull("Exception not found", exceptionMessage);
+
+ Assert.assertTrue("Unable to locate message '" + message + "' in repair error message: " + exceptionMessage, exceptionMessage.contains(message));
+ });
+ }
+
+ private static void validateExistingParentRepair(QueryResult rs, Consumer<Row> fn)
+ {
+ Assert.assertTrue("No rows found", rs.hasNext());
+ Row row = rs.next();
+
+ Assert.assertNotNull("parent_id (which is the primary key) was null", row.getUUID("parent_id"));
+
+ fn.accept(row);
+
+ // make sure no other records found
+ Assert.assertFalse("Only one repair expected, but found more than one", rs.hasNext());
+ }
+
+ public enum RepairType {
+ FULL {
+ public String[] append(String... args)
+ {
+ return ArrayUtils.add(args, "--full");
+ }
+ },
+ INCREMENTAL {
+ public String[] append(String... args)
+ {
+ // incremental is the default
+ return args;
+ }
+ },
+ PREVIEW {
+ public String[] append(String... args)
+ {
+ return ArrayUtils.addAll(args, "--preview");
+ }
+ };
+
+ public abstract String[] append(String... args);
+ }
+
+ public enum RepairParallelism {
+ SEQUENTIAL {
+ public String[] append(String... args)
+ {
+ return ArrayUtils.add(args, "--sequential");
+ }
+ },
+ PARALLEL {
+ public String[] append(String... args)
+ {
+ // default is to be parallel
+ return args;
+ }
+ },
+ DATACENTER_AWARE {
+ public String[] append(String... args)
+ {
+ return ArrayUtils.add(args, "--dc-parallel");
+ }
+ };
+
+ public abstract String[] append(String... args);
+ }
+}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/ExecUtil.java b/test/distributed/org/apache/cassandra/distributed/test/ExecUtil.java
new file mode 100644
index 0000000..b9fbd5c
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/test/ExecUtil.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test;
+
+import java.io.Serializable;
+
+import org.apache.cassandra.distributed.api.IIsolatedExecutor;
+
+public class ExecUtil
+{
+
+ public interface ThrowingSerializableRunnable<T extends Throwable> extends Serializable
+ {
+ public void run() throws T;
+ }
+
+ public static <T extends Throwable> IIsolatedExecutor.SerializableRunnable rethrow(ThrowingSerializableRunnable<T> run)
+ {
+ return () -> {
+ try
+ {
+ run.run();
+ }
+ catch (RuntimeException | Error t)
+ {
+ throw t;
+ }
+ catch (Throwable t)
+ {
+ throw new RuntimeException(t);
+ }
+ };
+ }
+
+}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/FailingRepairTest.java b/test/distributed/org/apache/cassandra/distributed/test/FailingRepairTest.java
new file mode 100644
index 0000000..bad6d87
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/test/FailingRepairTest.java
@@ -0,0 +1,348 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import com.google.common.util.concurrent.Uninterruptibles;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.distributed.Cluster;
+import org.apache.cassandra.distributed.api.ConsistencyLevel;
+import org.apache.cassandra.db.DataRange;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.db.filter.ColumnFilter;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.distributed.api.Feature;
+import org.apache.cassandra.distributed.api.ICluster;
+import org.apache.cassandra.distributed.api.IIsolatedExecutor.SerializableRunnable;
+import org.apache.cassandra.distributed.api.IInvokableInstance;
+import org.apache.cassandra.distributed.impl.InstanceKiller;
+import org.apache.cassandra.io.sstable.CorruptSSTableException;
+import org.apache.cassandra.io.sstable.ISSTableScanner;
+import org.apache.cassandra.io.sstable.format.ForwardingSSTableReader;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.format.SSTableReadsListener;
+import org.apache.cassandra.io.util.ChannelProxy;
+import org.apache.cassandra.net.Verb;
+import org.apache.cassandra.repair.RepairParallelism;
+import org.apache.cassandra.repair.messages.RepairOption;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.service.ActiveRepairService.ParentRepairStatus;
+import org.apache.cassandra.service.StorageService;
+
+@RunWith(Parameterized.class)
+public class FailingRepairTest extends TestBaseImpl implements Serializable
+{
+ private static ICluster<IInvokableInstance> CLUSTER;
+
+ private final Verb messageType;
+ private final RepairParallelism parallelism;
+ private final boolean withTracing;
+ private final SerializableRunnable setup;
+
+ public FailingRepairTest(Verb messageType, RepairParallelism parallelism, boolean withTracing, SerializableRunnable setup)
+ {
+ this.messageType = messageType;
+ this.parallelism = parallelism;
+ this.withTracing = withTracing;
+ this.setup = setup;
+ }
+
+ @Parameters(name = "{0}/{1}/{2}")
+ public static Collection<Object[]> messages()
+ {
+ List<Object[]> tests = new ArrayList<>();
+ for (RepairParallelism parallelism : RepairParallelism.values())
+ {
+ for (Boolean withTracing : Arrays.asList(Boolean.TRUE, Boolean.FALSE))
+ {
+ tests.add(new Object[]{ Verb.VALIDATION_REQ, parallelism, withTracing, failingReaders(Verb.VALIDATION_REQ, parallelism, withTracing) });
+ }
+ }
+ return tests;
+ }
+
+ private static SerializableRunnable failingReaders(Verb type, RepairParallelism parallelism, boolean withTracing)
+ {
+ return () -> {
+ String cfName = getCfName(type, parallelism, withTracing);
+ ColumnFamilyStore cf = Keyspace.open(KEYSPACE).getColumnFamilyStore(cfName);
+ cf.forceBlockingFlush();
+ Set<SSTableReader> remove = cf.getLiveSSTables();
+ Set<SSTableReader> replace = new HashSet<>();
+ if (type == Verb.VALIDATION_REQ)
+ {
+ for (SSTableReader r : remove)
+ replace.add(new FailingSSTableReader(r));
+ }
+ else
+ {
+ throw new UnsupportedOperationException("verb: " + type);
+ }
+ cf.getTracker().removeUnsafe(remove);
+ cf.addSSTables(replace);
+ };
+ }
+
+ private static String getCfName(Verb type, RepairParallelism parallelism, boolean withTracing)
+ {
+ return type.name().toLowerCase() + "_" + parallelism.name().toLowerCase() + "_" + withTracing;
+ }
+
+ @BeforeClass
+ public static void setupCluster() throws IOException
+ {
+ // streaming requires networking ATM
+ // streaming also requires gossip or isn't setup properly
+ CLUSTER = init(Cluster.build()
+ .withNodes(2)
+ .withConfig(c -> c.with(Feature.NETWORK)
+ .with(Feature.GOSSIP)
+ .set("disk_failure_policy", "die"))
+ .start());
+ }
+
+ @AfterClass
+ public static void teardownCluster() throws Exception
+ {
+ if (CLUSTER != null)
+ CLUSTER.close();
+ }
+
+ @Before
+ public void cleanupState()
+ {
+ for (int i = 1; i <= CLUSTER.size(); i++)
+ CLUSTER.get(i).runOnInstance(InstanceKiller::clear);
+ }
+
+ @Test(timeout = 10 * 60 * 1000)
+ public void testFailingMessage() throws IOException
+ {
+ final int replica = 1;
+ final int coordinator = 2;
+ String tableName = getCfName(messageType, parallelism, withTracing);
+ String fqtn = KEYSPACE + "." + tableName;
+
+ CLUSTER.schemaChange("CREATE TABLE " + fqtn + " (k INT, PRIMARY KEY (k))");
+
+ // create data which will NOT conflict
+ int lhsOffset = 10;
+ int rhsOffset = 20;
+ int limit = rhsOffset + (rhsOffset - lhsOffset);
+
+ // setup data which is consistent on both sides
+ for (int i = 0; i < lhsOffset; i++)
+ CLUSTER.coordinator(replica)
+ .execute("INSERT INTO " + fqtn + " (k) VALUES (?)", ConsistencyLevel.ALL, i);
+
+ // create data on LHS which does NOT exist in RHS
+ for (int i = lhsOffset; i < rhsOffset; i++)
+ CLUSTER.get(replica).executeInternal("INSERT INTO " + fqtn + " (k) VALUES (?)", i);
+
+ // create data on RHS which does NOT exist in LHS
+ for (int i = rhsOffset; i < limit; i++)
+ CLUSTER.get(coordinator).executeInternal("INSERT INTO " + fqtn + " (k) VALUES (?)", i);
+
+ // at this point, the two nodes should be out of sync, so confirm missing data
+ // node 1
+ Object[][] node1Records = toRows(IntStream.range(0, rhsOffset));
+ Object[][] node1Actuals = toNaturalOrder(CLUSTER.get(replica).executeInternal("SELECT k FROM " + fqtn));
+ Assert.assertArrayEquals(node1Records, node1Actuals);
+
+ // node 2
+ Object[][] node2Records = toRows(IntStream.concat(IntStream.range(0, lhsOffset), IntStream.range(rhsOffset, limit)));
+ Object[][] node2Actuals = toNaturalOrder(CLUSTER.get(coordinator).executeInternal("SELECT k FROM " + fqtn));
+ Assert.assertArrayEquals(node2Records, node2Actuals);
+
+ // Inject the failure
+ CLUSTER.get(replica).runOnInstance(() -> setup.run());
+
+ // run a repair which is expected to fail
+ List<String> repairStatus = CLUSTER.get(coordinator).callOnInstance(() -> {
+ // need all ranges on the host
+ String ranges = StorageService.instance.getLocalAndPendingRanges(KEYSPACE).stream()
+ .map(r -> r.left + ":" + r.right)
+ .collect(Collectors.joining(","));
+ Map<String, String> args = new HashMap<String, String>()
+ {{
+ put(RepairOption.PARALLELISM_KEY, parallelism.getName());
+ put(RepairOption.PRIMARY_RANGE_KEY, "false");
+ put(RepairOption.INCREMENTAL_KEY, "false");
+ put(RepairOption.TRACE_KEY, Boolean.toString(withTracing));
+ put(RepairOption.PULL_REPAIR_KEY, "false");
+ put(RepairOption.FORCE_REPAIR_KEY, "false");
+ put(RepairOption.RANGES_KEY, ranges);
+ put(RepairOption.COLUMNFAMILIES_KEY, tableName);
+ }};
+ int cmd = StorageService.instance.repairAsync(KEYSPACE, args);
+ Assert.assertFalse("repair return status was 0, expected non-zero return status, 0 indicates repair not submitted", cmd == 0);
+ List<String> status;
+ do
+ {
+ Uninterruptibles.sleepUninterruptibly(100, TimeUnit.MILLISECONDS);
+ status = StorageService.instance.getParentRepairStatus(cmd);
+ } while (status == null || status.get(0).equals(ParentRepairStatus.IN_PROGRESS.name()));
+
+ return status;
+ });
+ Assert.assertEquals(repairStatus.toString(), ParentRepairStatus.FAILED, ParentRepairStatus.valueOf(repairStatus.get(0)));
+
+ // its possible that the coordinator gets the message that the replica failed before the replica completes
+ // shutting down; this then means that isKilled could be updated after the fact
+ IInvokableInstance replicaInstance = CLUSTER.get(replica);
+ while (replicaInstance.killAttempts() <= 0)
+ Uninterruptibles.sleepUninterruptibly(50, TimeUnit.MILLISECONDS);
+
+ Assert.assertEquals("replica should be killed", 1, replicaInstance.killAttempts());
+ Assert.assertEquals("coordinator should not be killed", 0, CLUSTER.get(coordinator).killAttempts());
+ }
+
+ private static Object[][] toNaturalOrder(Object[][] actuals)
+ {
+ // data is returned in token order, so rather than try to be fancy and order expected in token order
+ // convert it to natural
+ int[] values = new int[actuals.length];
+ for (int i = 0; i < values.length; i++)
+ values[i] = (Integer) actuals[i][0];
+ Arrays.sort(values);
+ return toRows(IntStream.of(values));
+ }
+
+ private static Object[][] toRows(IntStream values)
+ {
+ return values
+ .mapToObj(v -> new Object[]{ v })
+ .toArray(Object[][]::new);
+ }
+
+ private static final class FailingSSTableReader extends ForwardingSSTableReader
+ {
+
+ private FailingSSTableReader(SSTableReader delegate)
+ {
+ super(delegate);
+ }
+
+ public ISSTableScanner getScanner()
+ {
+ return new FailingISSTableScanner();
+ }
+
+ public ISSTableScanner getScanner(Collection<Range<Token>> ranges)
+ {
+ return new FailingISSTableScanner();
+ }
+
+ public ISSTableScanner getScanner(Iterator<AbstractBounds<PartitionPosition>> rangeIterator)
+ {
+ return new FailingISSTableScanner();
+ }
+
+ public ISSTableScanner getScanner(ColumnFilter columns, DataRange dataRange, SSTableReadsListener listener)
+ {
+ return new FailingISSTableScanner();
+ }
+
+ public ChannelProxy getDataChannel()
+ {
+ throw new RuntimeException();
+ }
+
+ public String toString()
+ {
+ return "FailingSSTableReader[" + super.toString() + "]";
+ }
+ }
+
+ private static final class FailingISSTableScanner implements ISSTableScanner
+ {
+ public long getLengthInBytes()
+ {
+ return 0;
+ }
+
+ public long getCompressedLengthInBytes()
+ {
+ return 0;
+ }
+
+ public long getCurrentPosition()
+ {
+ return 0;
+ }
+
+ public long getBytesScanned()
+ {
+ return 0;
+ }
+
+ public Set<SSTableReader> getBackingSSTables()
+ {
+ return Collections.emptySet();
+ }
+
+ public TableMetadata metadata()
+ {
+ return null;
+ }
+
+ public void close()
+ {
+
+ }
+
+ public boolean hasNext()
+ {
+ throw new CorruptSSTableException(new IOException("Test commands it"), "mahahahaha!");
+ }
+
+ public UnfilteredRowIterator next()
+ {
+ throw new CorruptSSTableException(new IOException("Test commands it"), "mahahahaha!");
+ }
+ }
+}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/FullRepairCoordinatorFastTest.java b/test/distributed/org/apache/cassandra/distributed/test/FullRepairCoordinatorFastTest.java
new file mode 100644
index 0000000..e380985
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/test/FullRepairCoordinatorFastTest.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test;
+
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import org.apache.cassandra.distributed.test.DistributedRepairUtils.RepairParallelism;
+import org.apache.cassandra.distributed.test.DistributedRepairUtils.RepairType;
+
+@RunWith(Parameterized.class)
+public class FullRepairCoordinatorFastTest extends RepairCoordinatorFast
+{
+ public FullRepairCoordinatorFastTest(RepairParallelism parallelism, boolean withNotifications)
+ {
+ super(RepairType.FULL, parallelism, withNotifications);
+ }
+}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/FullRepairCoordinatorNeighbourDownTest.java b/test/distributed/org/apache/cassandra/distributed/test/FullRepairCoordinatorNeighbourDownTest.java
new file mode 100644
index 0000000..1053925
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/test/FullRepairCoordinatorNeighbourDownTest.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test;
+
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import org.apache.cassandra.distributed.test.DistributedRepairUtils.RepairParallelism;
+import org.apache.cassandra.distributed.test.DistributedRepairUtils.RepairType;
+
+@RunWith(Parameterized.class)
+public class FullRepairCoordinatorNeighbourDownTest extends RepairCoordinatorNeighbourDown
+{
+ public FullRepairCoordinatorNeighbourDownTest(RepairParallelism parallelism, boolean withNotifications)
+ {
+ super(RepairType.FULL, parallelism, withNotifications);
+ }
+}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/FullRepairCoordinatorTimeoutTest.java b/test/distributed/org/apache/cassandra/distributed/test/FullRepairCoordinatorTimeoutTest.java
new file mode 100644
index 0000000..d91cb5d
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/test/FullRepairCoordinatorTimeoutTest.java
@@ -0,0 +1,16 @@
+package org.apache.cassandra.distributed.test;
+
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import org.apache.cassandra.distributed.test.DistributedRepairUtils.RepairParallelism;
+import org.apache.cassandra.distributed.test.DistributedRepairUtils.RepairType;
+
+@RunWith(Parameterized.class)
+public class FullRepairCoordinatorTimeoutTest extends RepairCoordinatorTimeout
+{
+ public FullRepairCoordinatorTimeoutTest(RepairParallelism parallelism, boolean withNotifications)
+ {
+ super(RepairType.FULL, parallelism, withNotifications);
+ }
+}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/GossipTest.java b/test/distributed/org/apache/cassandra/distributed/test/GossipTest.java
deleted file mode 100644
index 83c62c8..0000000
--- a/test/distributed/org/apache/cassandra/distributed/test/GossipTest.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.distributed.test;
-
-import java.net.InetAddress;
-import java.util.Collection;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.locks.LockSupport;
-import java.util.stream.Collectors;
-
-import com.google.common.collect.Iterables;
-import org.junit.Assert;
-import org.junit.Test;
-
-import org.apache.cassandra.dht.Token;
-import org.apache.cassandra.distributed.Cluster;
-import org.apache.cassandra.gms.ApplicationState;
-import org.apache.cassandra.gms.EndpointState;
-import org.apache.cassandra.gms.Gossiper;
-import org.apache.cassandra.service.StorageService;
-import org.apache.cassandra.utils.FBUtilities;
-
-import static org.apache.cassandra.distributed.api.Feature.GOSSIP;
-import static org.apache.cassandra.distributed.api.Feature.NETWORK;
-
-public class GossipTest extends TestBaseImpl
-{
-
- @Test
- public void nodeDownDuringMove() throws Throwable
- {
- int liveCount = 1;
- System.setProperty("cassandra.ring_delay_ms", "5000"); // down from 30s default
- System.setProperty("cassandra.consistent.rangemovement", "false");
- System.setProperty("cassandra.consistent.simultaneousmoves.allow", "true");
- try (Cluster cluster = Cluster.build(2 + liveCount)
- .withConfig(config -> config.with(NETWORK).with(GOSSIP))
- .createWithoutStarting())
- {
- int fail = liveCount + 1;
- int late = fail + 1;
- for (int i = 1 ; i <= liveCount ; ++i)
- cluster.get(i).startup();
- cluster.get(fail).startup();
- Collection<String> expectTokens = cluster.get(fail).callsOnInstance(() ->
- StorageService.instance.getTokenMetadata().getTokens(FBUtilities.getBroadcastAddress())
- .stream().map(Object::toString).collect(Collectors.toList())
- ).call();
-
- InetAddress failAddress = cluster.get(fail).broadcastAddress().getAddress();
- // wait for NORMAL state
- for (int i = 1 ; i <= liveCount ; ++i)
- {
- cluster.get(i).acceptsOnInstance((InetAddress endpoint) -> {
- EndpointState ep;
- while (null == (ep = Gossiper.instance.getEndpointStateForEndpoint(endpoint))
- || ep.getApplicationState(ApplicationState.STATUS) == null
- || !ep.getApplicationState(ApplicationState.STATUS).value.startsWith("NORMAL"))
- LockSupport.parkNanos(TimeUnit.MILLISECONDS.toNanos(10L));
- }).accept(failAddress);
- }
-
- // set ourselves to MOVING, and wait for it to propagate
- cluster.get(fail).runOnInstance(() -> {
-
- Token token = Iterables.getFirst(StorageService.instance.getTokenMetadata().getTokens(FBUtilities.getBroadcastAddress()), null);
- Gossiper.instance.addLocalApplicationState(ApplicationState.STATUS, StorageService.instance.valueFactory.moving(token));
- });
-
- for (int i = 1 ; i <= liveCount ; ++i)
- {
- cluster.get(i).acceptsOnInstance((InetAddress endpoint) -> {
- EndpointState ep;
- while (null == (ep = Gossiper.instance.getEndpointStateForEndpoint(endpoint))
- || (ep.getApplicationState(ApplicationState.STATUS) == null
- || !ep.getApplicationState(ApplicationState.STATUS).value.startsWith("MOVING")))
- LockSupport.parkNanos(TimeUnit.MILLISECONDS.toNanos(10L));
- }).accept(failAddress);
- }
-
- cluster.get(fail).shutdown(false).get();
- cluster.get(late).startup();
- cluster.get(late).acceptsOnInstance((InetAddress endpoint) -> {
- EndpointState ep;
- while (null == (ep = Gossiper.instance.getEndpointStateForEndpoint(endpoint))
- || !ep.getApplicationState(ApplicationState.STATUS).value.startsWith("MOVING"))
- LockSupport.parkNanos(TimeUnit.MILLISECONDS.toNanos(10L));
- }).accept(failAddress);
-
- Collection<String> tokens = cluster.get(late).appliesOnInstance((InetAddress endpoint) ->
- StorageService.instance.getTokenMetadata().getTokens(failAddress)
- .stream().map(Object::toString).collect(Collectors.toList())
- ).apply(failAddress);
-
- Assert.assertEquals(expectTokens, tokens);
- }
- }
-
-}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/IncrementalRepairCoordinatorFastTest.java b/test/distributed/org/apache/cassandra/distributed/test/IncrementalRepairCoordinatorFastTest.java
new file mode 100644
index 0000000..7a4c98e
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/test/IncrementalRepairCoordinatorFastTest.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test;
+
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import org.apache.cassandra.distributed.test.DistributedRepairUtils.RepairParallelism;
+import org.apache.cassandra.distributed.test.DistributedRepairUtils.RepairType;
+
+@RunWith(Parameterized.class)
+public class IncrementalRepairCoordinatorFastTest extends RepairCoordinatorFast
+{
+ public IncrementalRepairCoordinatorFastTest(RepairParallelism parallelism, boolean withNotifications)
+ {
+ super(RepairType.INCREMENTAL, parallelism, withNotifications);
+ }
+}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/IncrementalRepairCoordinatorNeighbourDownTest.java b/test/distributed/org/apache/cassandra/distributed/test/IncrementalRepairCoordinatorNeighbourDownTest.java
new file mode 100644
index 0000000..af17567
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/test/IncrementalRepairCoordinatorNeighbourDownTest.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test;
+
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import org.apache.cassandra.distributed.test.DistributedRepairUtils.RepairParallelism;
+import org.apache.cassandra.distributed.test.DistributedRepairUtils.RepairType;
+
+@RunWith(Parameterized.class)
+public class IncrementalRepairCoordinatorNeighbourDownTest extends RepairCoordinatorNeighbourDown
+{
+ public IncrementalRepairCoordinatorNeighbourDownTest(RepairParallelism parallelism, boolean withNotifications)
+ {
+ super(RepairType.INCREMENTAL, parallelism, withNotifications);
+ }
+}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/IncrementalRepairCoordinatorTimeoutTest.java b/test/distributed/org/apache/cassandra/distributed/test/IncrementalRepairCoordinatorTimeoutTest.java
new file mode 100644
index 0000000..0fdae57
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/test/IncrementalRepairCoordinatorTimeoutTest.java
@@ -0,0 +1,16 @@
+package org.apache.cassandra.distributed.test;
+
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import org.apache.cassandra.distributed.test.DistributedRepairUtils.RepairParallelism;
+import org.apache.cassandra.distributed.test.DistributedRepairUtils.RepairType;
+
+@RunWith(Parameterized.class)
+public class IncrementalRepairCoordinatorTimeoutTest extends RepairCoordinatorTimeout
+{
+ public IncrementalRepairCoordinatorTimeoutTest(RepairParallelism parallelism, boolean withNotifications)
+ {
+ super(RepairType.INCREMENTAL, parallelism, withNotifications);
+ }
+}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/LargeColumnTest.java b/test/distributed/org/apache/cassandra/distributed/test/LargeColumnTest.java
new file mode 100644
index 0000000..0a81359
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/test/LargeColumnTest.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test;
+
+import java.util.Random;
+import java.util.concurrent.ThreadLocalRandom;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.distributed.api.ConsistencyLevel;
+import org.apache.cassandra.distributed.api.ICluster;
+
+import static java.util.concurrent.TimeUnit.SECONDS;
+
+// TODO: this test should be removed after running in-jvm dtests is set up via the shared API repository
+public class LargeColumnTest extends TestBaseImpl
+{
+ private static final Logger logger = LoggerFactory.getLogger(LargeColumnTest.class);
+
+ private static String str(int length, Random random, long seed)
+ {
+ random.setSeed(seed);
+ char[] chars = new char[length];
+ int i = 0;
+ int s = 0;
+ long v = 0;
+ while (i < length)
+ {
+ if (s == 0)
+ {
+ v = random.nextLong();
+ s = 8;
+ }
+ chars[i] = (char) (((v & 127) + 32) & 127);
+ v >>= 8;
+ --s;
+ ++i;
+ }
+ return new String(chars);
+ }
+
+ private void testLargeColumns(int nodes, int columnSize, int rowCount) throws Throwable
+ {
+ Random random = new Random();
+ long seed = ThreadLocalRandom.current().nextLong();
+ logger.info("Using seed {}", seed);
+
+ try (ICluster cluster = init(builder()
+ .withNodes(nodes)
+ .withConfig(config ->
+ config.set("commitlog_segment_size_in_mb", (columnSize * 3) >> 20)
+ .set("internode_application_send_queue_reserve_endpoint_capacity_in_bytes", columnSize * 2)
+ .set("internode_application_send_queue_reserve_global_capacity_in_bytes", columnSize * 3)
+ .set("write_request_timeout_in_ms", SECONDS.toMillis(30L))
+ .set("read_request_timeout_in_ms", SECONDS.toMillis(30L))
+ .set("memtable_heap_space_in_mb", 1024)
+ )
+ .start()))
+ {
+ cluster.schemaChange(String.format("CREATE TABLE %s.cf (k int, c text, PRIMARY KEY (k))", KEYSPACE));
+
+ for (int i = 0; i < rowCount; ++i)
+ cluster.coordinator(1).execute(String.format("INSERT INTO %s.cf (k, c) VALUES (?, ?);", KEYSPACE), ConsistencyLevel.ALL, i, str(columnSize, random, seed | i));
+
+ for (int i = 0; i < rowCount; ++i)
+ {
+ Object[][] results = cluster.coordinator(1).execute(String.format("SELECT k, c FROM %s.cf WHERE k = ?;", KEYSPACE), ConsistencyLevel.ALL, i);
+ Assert.assertTrue(str(columnSize, random, seed | i).equals(results[0][1]));
+ }
+ }
+ }
+
+ @Test
+ public void test() throws Throwable
+ {
+ testLargeColumns(2, 16 << 20, 5);
+ }
+}
\ No newline at end of file
diff --git a/test/distributed/org/apache/cassandra/distributed/test/MessageFiltersTest.java b/test/distributed/org/apache/cassandra/distributed/test/MessageFiltersTest.java
index bb8d7fb..bd09891 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/MessageFiltersTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/MessageFiltersTest.java
@@ -22,20 +22,28 @@
import java.util.Arrays;
import java.util.HashSet;
import java.util.Set;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
+import com.google.common.collect.Sets;
import org.junit.Assert;
import org.junit.Test;
import org.apache.cassandra.distributed.Cluster;
import org.apache.cassandra.distributed.api.ConsistencyLevel;
+import org.apache.cassandra.distributed.api.ICluster;
+import org.apache.cassandra.distributed.api.IInvokableInstance;
import org.apache.cassandra.distributed.api.IIsolatedExecutor;
import org.apache.cassandra.distributed.api.IMessage;
import org.apache.cassandra.distributed.api.IMessageFilters;
import org.apache.cassandra.distributed.impl.Instance;
import org.apache.cassandra.distributed.shared.MessageFilters;
-import org.apache.cassandra.net.MessageIn;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.net.Message;
import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.net.NoPayload;
+import org.apache.cassandra.net.Verb;
public class MessageFiltersTest extends TestBaseImpl
{
@@ -58,10 +66,9 @@
private static void simpleFiltersTest(boolean inbound)
{
- int VERB1 = MessagingService.Verb.READ.ordinal();
- int VERB2 = MessagingService.Verb.REQUEST_RESPONSE.ordinal();
- int VERB3 = MessagingService.Verb.READ_REPAIR.ordinal();
-
+ int VERB1 = Verb.READ_REQ.id;
+ int VERB2 = Verb.READ_RSP.id;
+ int VERB3 = Verb.READ_REPAIR_REQ.id;
int i1 = 1;
int i2 = 2;
int i3 = 3;
@@ -133,15 +140,20 @@
public int id() { return 0; }
public int version() { return 0; }
public InetSocketAddress from() { return null; }
+ public int fromPort()
+ {
+ return 0;
+ }
};
}
+
@Test
public void testFilters() throws Throwable
{
String read = "SELECT * FROM " + KEYSPACE + ".tbl";
String write = "INSERT INTO " + KEYSPACE + ".tbl (pk, ck, v) VALUES (1, 1, 1)";
- try (Cluster cluster = Cluster.create(2))
+ try (ICluster cluster = builder().withNodes(2).start())
{
cluster.schemaChange("CREATE KEYSPACE " + KEYSPACE + " WITH replication = {'class': 'SimpleStrategy', 'replication_factor': " + cluster.size() + "};");
cluster.schemaChange("CREATE TABLE " + KEYSPACE + ".tbl (pk int, ck int, v int, PRIMARY KEY (pk, ck))");
@@ -155,7 +167,7 @@
cluster.filters().reset();
// Reads are going to timeout only when 1 serves as a coordinator
- cluster.verbs(MessagingService.Verb.RANGE_SLICE).from(1).to(2).drop();
+ cluster.filters().verbs(Verb.RANGE_REQ.id).from(1).to(2).drop();
assertTimeOut(() -> cluster.coordinator(1).execute(read, ConsistencyLevel.ALL));
cluster.coordinator(2).execute(read, ConsistencyLevel.ALL);
@@ -171,42 +183,84 @@
String read = "SELECT * FROM " + KEYSPACE + ".tbl";
String write = "INSERT INTO " + KEYSPACE + ".tbl (pk, ck, v) VALUES (1, 1, 1)";
- try (Cluster cluster = Cluster.create(2))
+ try (ICluster<IInvokableInstance> cluster = builder().withNodes(2).start())
{
cluster.schemaChange("CREATE KEYSPACE " + KEYSPACE + " WITH replication = {'class': 'SimpleStrategy', 'replication_factor': " + cluster.size() + "};");
cluster.schemaChange("CREATE TABLE " + KEYSPACE + ".tbl (pk int, ck int, v int, PRIMARY KEY (pk, ck))");
AtomicInteger counter = new AtomicInteger();
- Set<Integer> verbs = new HashSet<>(Arrays.asList(MessagingService.Verb.RANGE_SLICE.ordinal(),
- MessagingService.Verb.MUTATION.ordinal()));
+ Set<Integer> verbs = Sets.newHashSet(Arrays.asList(Verb.RANGE_REQ.id,
+ Verb.RANGE_RSP.id,
+ Verb.MUTATION_REQ.id,
+ Verb.MUTATION_RSP.id));
- // Reads and writes are going to time out in both directions
- IMessageFilters.Filter filter = cluster.filters()
- .allVerbs()
- .from(1)
- .to(2)
- .messagesMatching((from, to, msg) -> {
- // Decode and verify message on instance; return the result back here
- Integer id = cluster.get(1).callsOnInstance((IIsolatedExecutor.SerializableCallable<Integer>) () -> {
- MessageIn decoded = Instance.deserializeMessage(msg);
- if (decoded != null)
- return (Integer) decoded.verb.ordinal();
- return -1;
- }).call();
- if (id > 0)
+ for (boolean inbound : Arrays.asList(true, false))
+ {
+ counter.set(0);
+ // Reads and writes are going to time out in both directions
+ IMessageFilters.Filter filter = cluster.filters()
+ .allVerbs()
+ .inbound(inbound)
+ .from(1)
+ .to(2)
+ .messagesMatching((from, to, msg) -> {
+ // Decode and verify message on instance; return the result back here
+ Integer id = cluster.get(1).callsOnInstance((IIsolatedExecutor.SerializableCallable<Integer>) () -> {
+ Message decoded = Instance.deserializeMessage(msg);
+ return (Integer) decoded.verb().id;
+ }).call();
Assert.assertTrue(verbs.contains(id));
- counter.incrementAndGet();
- return false;
- }).drop();
+ counter.incrementAndGet();
+ return false;
+ }).drop();
- for (int i : new int[]{ 1, 2 })
- cluster.coordinator(i).execute(read, ConsistencyLevel.ALL);
- for (int i : new int[]{ 1, 2 })
- cluster.coordinator(i).execute(write, ConsistencyLevel.ALL);
+ for (int i : new int[]{ 1, 2 })
+ cluster.coordinator(i).execute(read, ConsistencyLevel.ALL);
+ for (int i : new int[]{ 1, 2 })
+ cluster.coordinator(i).execute(write, ConsistencyLevel.ALL);
- filter.off();
- Assert.assertEquals(4, counter.get());
+ filter.off();
+ Assert.assertEquals(4, counter.get());
+ }
+ }
+ }
+
+ @Test
+ public void outboundBeforeInbound() throws Throwable
+ {
+ try (Cluster cluster = Cluster.create(2))
+ {
+ InetAddressAndPort other = InetAddressAndPort.getByAddressOverrideDefaults(cluster.get(2).broadcastAddress().getAddress(),
+ cluster.get(2).broadcastAddress().getPort());
+ CountDownLatch waitForIt = new CountDownLatch(1);
+ Set<Integer> outboundMessagesSeen = new HashSet<>();
+ Set<Integer> inboundMessagesSeen = new HashSet<>();
+ AtomicBoolean outboundAfterInbound = new AtomicBoolean(false);
+ cluster.filters().outbound().verbs(Verb.ECHO_REQ.id, Verb.ECHO_RSP.id).messagesMatching((from, to, msg) -> {
+ outboundMessagesSeen.add(msg.verb());
+ if (inboundMessagesSeen.contains(msg.verb()))
+ outboundAfterInbound.set(true);
+ return false;
+ }).drop(); // drop is confusing since I am not dropping, im just listening...
+ cluster.filters().inbound().verbs(Verb.ECHO_REQ.id, Verb.ECHO_RSP.id).messagesMatching((from, to, msg) -> {
+ inboundMessagesSeen.add(msg.verb());
+ return false;
+ }).drop(); // drop is confusing since I am not dropping, im just listening...
+ cluster.filters().inbound().verbs(Verb.ECHO_RSP.id).messagesMatching((from, to, msg) -> {
+ waitForIt.countDown();
+ return false;
+ }).drop(); // drop is confusing since I am not dropping, im just listening...
+ cluster.get(1).runOnInstance(() -> {
+ MessagingService.instance().send(Message.out(Verb.ECHO_REQ, NoPayload.noPayload), other);
+ });
+
+ waitForIt.await();
+
+ Assert.assertEquals(outboundMessagesSeen, inboundMessagesSeen);
+ // since both are equal, only need to confirm the size of one
+ Assert.assertEquals(2, outboundMessagesSeen.size());
+ Assert.assertFalse("outbound message saw after inbound", outboundAfterInbound.get());
}
}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/MessageForwardingTest.java b/test/distributed/org/apache/cassandra/distributed/test/MessageForwardingTest.java
index 61ccb5f..153d7de 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/MessageForwardingTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/MessageForwardingTest.java
@@ -25,14 +25,15 @@
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.Future;
+import java.util.stream.Collectors;
import java.util.stream.IntStream;
import java.util.stream.Stream;
import org.junit.Assert;
import org.junit.Test;
-import org.apache.cassandra.distributed.Cluster;
import org.apache.cassandra.distributed.api.ConsistencyLevel;
+import org.apache.cassandra.distributed.Cluster;
import org.apache.cassandra.distributed.impl.IsolatedExecutor;
import org.apache.cassandra.distributed.impl.TracingUtil;
import org.apache.cassandra.utils.UUIDGen;
@@ -44,28 +45,30 @@
{
String originalTraceTimeout = TracingUtil.setWaitForTracingEventTimeoutSecs("1");
final int numInserts = 100;
- Map<InetAddress,Integer> commitCounts = new HashMap<>();
+ Map<InetAddress, Integer> forwardFromCounts = new HashMap<>();
+ Map<InetAddress, Integer> commitCounts = new HashMap<>();
- try (Cluster cluster = init(Cluster.build()
- .withDC("dc0", 1)
- .withDC("dc1", 3)
- .start()))
+ try (Cluster cluster = (Cluster) init(builder()
+ .withDC("dc0", 1)
+ .withDC("dc1", 3)
+ .start()))
{
cluster.schemaChange("CREATE TABLE " + KEYSPACE + ".tbl (pk int, ck int, v text, PRIMARY KEY (pk, ck))");
cluster.forEach(instance -> commitCounts.put(instance.broadcastAddress().getAddress(), 0));
final UUID sessionId = UUIDGen.getTimeUUID();
- Stream<Future<Object[][]>> inserts = IntStream.range(0, numInserts).mapToObj((idx) ->
- cluster.coordinator(1).asyncExecuteWithTracing(sessionId,
- "INSERT INTO " + KEYSPACE + ".tbl(pk,ck,v) VALUES (1, 1, 'x')",
- ConsistencyLevel.ALL)
- );
+ Stream<Future<Object[][]>> inserts = IntStream.range(0, numInserts).mapToObj((idx) -> {
+ return cluster.coordinator(1).asyncExecuteWithTracing(sessionId,
+ "INSERT INTO " + KEYSPACE + ".tbl(pk,ck,v) VALUES (1, 1, 'x')",
+ ConsistencyLevel.ALL);
+ });
// Wait for each of the futures to complete before checking the traces, don't care
// about the result so
//noinspection ResultOfMethodCallIgnored
- inserts.map(IsolatedExecutor::waitOn).count();
+ inserts.map(IsolatedExecutor::waitOn).collect(Collectors.toList());
+ cluster.stream("dc1").forEach(instance -> forwardFromCounts.put(instance.broadcastAddress().getAddress(), 0));
cluster.forEach(instance -> commitCounts.put(instance.broadcastAddress().getAddress(), 0));
List<TracingUtil.TraceEntry> traces = TracingUtil.getTrace(cluster, sessionId, ConsistencyLevel.ALL);
traces.forEach(traceEntry -> {
@@ -73,8 +76,16 @@
{
commitCounts.compute(traceEntry.source, (k, v) -> (v != null ? v : 0) + 1);
}
+ else if (traceEntry.activity.contains("Enqueuing forwarded write to "))
+ {
+ forwardFromCounts.compute(traceEntry.source, (k, v) -> (v != null ? v : 0) + 1);
+ }
});
+ // Check that each node in dc1 was the forwarder at least once. There is a (1/3)^numInserts chance
+ // that the same node will be picked, but the odds of that are ~2e-48.
+ forwardFromCounts.forEach((source, count) -> Assert.assertTrue(source + " should have been randomized to forward messages", count > 0));
+
// Check that each node received the forwarded messages once (and only once)
commitCounts.forEach((source, count) ->
Assert.assertEquals(source + " appending to commitlog traces",
@@ -89,4 +100,4 @@
TracingUtil.setWaitForTracingEventTimeoutSecs(originalTraceTimeout);
}
}
-}
+}
\ No newline at end of file
diff --git a/test/distributed/org/apache/cassandra/distributed/test/NodeToolTest.java b/test/distributed/org/apache/cassandra/distributed/test/NodeToolTest.java
index 1d78152..1a1bdc7 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/NodeToolTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/NodeToolTest.java
@@ -20,7 +20,7 @@
import org.junit.Test;
-import org.apache.cassandra.distributed.Cluster;
+import org.apache.cassandra.distributed.api.ICluster;
import static org.junit.Assert.assertEquals;
@@ -29,7 +29,7 @@
@Test
public void test() throws Throwable
{
- try (Cluster cluster = init(Cluster.create(1)))
+ try (ICluster cluster = init(builder().withNodes(1).start()))
{
assertEquals(0, cluster.get(1).nodetool("help"));
assertEquals(0, cluster.get(1).nodetool("flush"));
diff --git a/test/distributed/org/apache/cassandra/distributed/test/PreviewRepairCoordinatorFastTest.java b/test/distributed/org/apache/cassandra/distributed/test/PreviewRepairCoordinatorFastTest.java
new file mode 100644
index 0000000..bafef05
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/test/PreviewRepairCoordinatorFastTest.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test;
+
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import org.apache.cassandra.distributed.test.DistributedRepairUtils.RepairParallelism;
+import org.apache.cassandra.distributed.test.DistributedRepairUtils.RepairType;
+
+@RunWith(Parameterized.class)
+public class PreviewRepairCoordinatorFastTest extends RepairCoordinatorFast
+{
+ public PreviewRepairCoordinatorFastTest(RepairParallelism parallelism, boolean withNotifications)
+ {
+ super(RepairType.PREVIEW, parallelism, withNotifications);
+ }
+}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/PreviewRepairCoordinatorNeighbourDownTest.java b/test/distributed/org/apache/cassandra/distributed/test/PreviewRepairCoordinatorNeighbourDownTest.java
new file mode 100644
index 0000000..1926f9b
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/test/PreviewRepairCoordinatorNeighbourDownTest.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test;
+
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import org.apache.cassandra.distributed.test.DistributedRepairUtils.RepairParallelism;
+import org.apache.cassandra.distributed.test.DistributedRepairUtils.RepairType;
+
+@RunWith(Parameterized.class)
+public class PreviewRepairCoordinatorNeighbourDownTest extends RepairCoordinatorNeighbourDown
+{
+ public PreviewRepairCoordinatorNeighbourDownTest(RepairParallelism parallelism, boolean withNotifications)
+ {
+ super(RepairType.PREVIEW, parallelism, withNotifications);
+ }
+}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/PreviewRepairCoordinatorTimeoutTest.java b/test/distributed/org/apache/cassandra/distributed/test/PreviewRepairCoordinatorTimeoutTest.java
new file mode 100644
index 0000000..8b90909
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/test/PreviewRepairCoordinatorTimeoutTest.java
@@ -0,0 +1,16 @@
+package org.apache.cassandra.distributed.test;
+
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import org.apache.cassandra.distributed.test.DistributedRepairUtils.RepairParallelism;
+import org.apache.cassandra.distributed.test.DistributedRepairUtils.RepairType;
+
+@RunWith(Parameterized.class)
+public class PreviewRepairCoordinatorTimeoutTest extends RepairCoordinatorTimeout
+{
+ public PreviewRepairCoordinatorTimeoutTest(RepairParallelism parallelism, boolean withNotifications)
+ {
+ super(RepairType.PREVIEW, parallelism, withNotifications);
+ }
+}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/PreviewRepairTest.java b/test/distributed/org/apache/cassandra/distributed/test/PreviewRepairTest.java
new file mode 100644
index 0000000..7c306c0
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/test/PreviewRepairTest.java
@@ -0,0 +1,387 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.util.concurrent.Uninterruptibles;
+import org.junit.Test;
+
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.distributed.Cluster;
+import org.apache.cassandra.distributed.api.ConsistencyLevel;
+import org.apache.cassandra.distributed.api.ICoordinator;
+import org.apache.cassandra.distributed.api.IInvokableInstance;
+import org.apache.cassandra.distributed.api.IIsolatedExecutor;
+import org.apache.cassandra.distributed.api.IMessage;
+import org.apache.cassandra.distributed.api.IMessageFilters;
+import org.apache.cassandra.distributed.shared.RepairResult;
+import org.apache.cassandra.net.Verb;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.repair.RepairParallelism;
+import org.apache.cassandra.repair.messages.RepairOption;
+import org.apache.cassandra.service.ActiveRepairService;
+import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.streaming.PreviewKind;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.concurrent.SimpleCondition;
+import org.apache.cassandra.utils.progress.ProgressEventType;
+
+import static org.apache.cassandra.distributed.api.Feature.GOSSIP;
+import static org.apache.cassandra.distributed.api.Feature.NETWORK;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class PreviewRepairTest extends TestBaseImpl
+{
+ /**
+ * makes sure that the repaired sstables are not matching on the two
+ * nodes by disabling autocompaction on node2 and then running an
+ * incremental repair
+ */
+ @Test
+ public void testWithMismatchingPending() throws Throwable
+ {
+ try(Cluster cluster = init(Cluster.build(2).withConfig(config -> config.with(GOSSIP).with(NETWORK)).start()))
+ {
+ cluster.schemaChange("create table " + KEYSPACE + ".tbl (id int primary key, t int)");
+ insert(cluster.coordinator(1), 0, 100);
+ cluster.forEach((node) -> node.flush(KEYSPACE));
+ cluster.get(1).callOnInstance(repair(options(false)));
+ insert(cluster.coordinator(1), 100, 100);
+ cluster.forEach((node) -> node.flush(KEYSPACE));
+
+ // make sure that all sstables have moved to repaired by triggering a compaction
+ // also disables autocompaction on the nodes
+ cluster.forEach((node) -> node.runOnInstance(() -> {
+ ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore("tbl");
+ FBUtilities.waitOnFutures(CompactionManager.instance.submitBackground(cfs));
+ cfs.disableAutoCompaction();
+ }));
+ cluster.get(1).callOnInstance(repair(options(false)));
+ // now re-enable autocompaction on node1, this moves the sstables for the new repair to repaired
+ cluster.get(1).runOnInstance(() -> {
+ ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore("tbl");
+ cfs.enableAutoCompaction();
+ FBUtilities.waitOnFutures(CompactionManager.instance.submitBackground(cfs));
+ });
+
+ //IR and Preview repair can't run concurrently. In case the test is flaky, please check CASSANDRA-15685
+ Thread.sleep(1000);
+
+ RepairResult rs = cluster.get(1).callOnInstance(repair(options(true)));
+ assertTrue(rs.success); // preview repair should succeed
+ assertFalse(rs.wasInconsistent); // and we should see no mismatches
+ }
+ }
+
+ /**
+ * another case where the repaired datasets could mismatch is if an incremental repair finishes just as the preview
+ * repair is starting up.
+ *
+ * This tests this case:
+ * 1. we start a preview repair
+ * 2. pause the validation requests from node1 -> node2
+ * 3. node1 starts its validation
+ * 4. run an incremental repair which completes fine
+ * 5. node2 resumes its validation
+ *
+ * Now we will include sstables from the second incremental repair on node2 but not on node1
+ * This should fail since we fail any preview repair which is ongoing when an incremental repair finishes (step 4 above)
+ */
+ @Test
+ public void testFinishingIncRepairDuringPreview() throws IOException, InterruptedException, ExecutionException
+ {
+ ExecutorService es = Executors.newSingleThreadExecutor();
+ try(Cluster cluster = init(Cluster.build(2).withConfig(config -> config.with(GOSSIP).with(NETWORK)).start()))
+ {
+ cluster.schemaChange("create table " + KEYSPACE + ".tbl (id int primary key, t int)");
+
+ insert(cluster.coordinator(1), 0, 100);
+ cluster.forEach((node) -> node.flush(KEYSPACE));
+ cluster.get(1).callOnInstance(repair(options(false)));
+
+ insert(cluster.coordinator(1), 100, 100);
+ cluster.forEach((node) -> node.flush(KEYSPACE));
+
+ SimpleCondition continuePreviewRepair = new SimpleCondition();
+ DelayMessageFilter filter = new DelayMessageFilter(continuePreviewRepair);
+ // this pauses the validation request sent from node1 to node2 until we have run a full inc repair below
+ cluster.filters().outbound().verbs(Verb.VALIDATION_REQ.id).from(1).to(2).messagesMatching(filter).drop();
+
+ Future<RepairResult> rsFuture = es.submit(() -> cluster.get(1).callOnInstance(repair(options(true))));
+ Thread.sleep(1000);
+ // this needs to finish before the preview repair is unpaused on node2
+ cluster.get(1).callOnInstance(repair(options(false)));
+ continuePreviewRepair.signalAll();
+ RepairResult rs = rsFuture.get();
+ assertFalse(rs.success); // preview repair should have failed
+ assertFalse(rs.wasInconsistent); // and no mismatches should have been reported
+ }
+ finally
+ {
+ es.shutdown();
+ }
+ }
+
+ /**
+ * Same as testFinishingIncRepairDuringPreview but the previewed range does not intersect the incremental repair
+ * so both preview and incremental repair should finish fine (without any mismatches)
+ */
+ @Test
+ public void testFinishingNonIntersectingIncRepairDuringPreview() throws IOException, InterruptedException, ExecutionException
+ {
+ ExecutorService es = Executors.newSingleThreadExecutor();
+ try(Cluster cluster = init(Cluster.build(2).withConfig(config -> config.with(GOSSIP).with(NETWORK)).start()))
+ {
+ cluster.schemaChange("create table " + KEYSPACE + ".tbl (id int primary key, t int)");
+
+ insert(cluster.coordinator(1), 0, 100);
+ cluster.forEach((node) -> node.flush(KEYSPACE));
+ assertTrue(cluster.get(1).callOnInstance(repair(options(false))).success);
+
+ insert(cluster.coordinator(1), 100, 100);
+ cluster.forEach((node) -> node.flush(KEYSPACE));
+
+ // pause preview repair validation messages on node2 until node1 has finished
+ SimpleCondition continuePreviewRepair = new SimpleCondition();
+ DelayMessageFilter filter = new DelayMessageFilter(continuePreviewRepair);
+ cluster.filters().outbound().verbs(Verb.VALIDATION_REQ.id).from(1).to(2).messagesMatching(filter).drop();
+
+ // get local ranges to repair two separate ranges:
+ List<String> localRanges = cluster.get(1).callOnInstance(() -> {
+ List<String> res = new ArrayList<>();
+ for (Range<Token> r : StorageService.instance.getLocalReplicas(KEYSPACE).ranges())
+ res.add(r.left.getTokenValue()+ ":"+ r.right.getTokenValue());
+ return res;
+ });
+
+ assertEquals(2, localRanges.size());
+ Future<RepairResult> repairStatusFuture = es.submit(() -> cluster.get(1).callOnInstance(repair(options(true, localRanges.get(0)))));
+ Thread.sleep(1000); // wait for node1 to start validation compaction
+ // this needs to finish before the preview repair is unpaused on node2
+ assertTrue(cluster.get(1).callOnInstance(repair(options(false, localRanges.get(1)))).success);
+
+ continuePreviewRepair.signalAll();
+ RepairResult rs = repairStatusFuture.get();
+ assertTrue(rs.success); // repair should succeed
+ assertFalse(rs.wasInconsistent); // and no mismatches
+ }
+ finally
+ {
+ es.shutdown();
+ }
+ }
+
+ @Test
+ public void snapshotTest() throws IOException, InterruptedException
+ {
+ try(Cluster cluster = init(Cluster.build(3).withConfig(config ->
+ config.set("snapshot_on_repaired_data_mismatch", true)
+ .with(GOSSIP)
+ .with(NETWORK))
+ .start()))
+ {
+ cluster.schemaChange("create table " + KEYSPACE + ".tbl (id int primary key, t int)");
+ cluster.schemaChange("create table " + KEYSPACE + ".tbl2 (id int primary key, t int)");
+ Thread.sleep(1000);
+
+ // populate 2 tables
+ insert(cluster.coordinator(1), 0, 100, "tbl");
+ insert(cluster.coordinator(1), 0, 100, "tbl2");
+ cluster.forEach((n) -> n.flush(KEYSPACE));
+
+ // make sure everything is marked repaired
+ cluster.get(1).callOnInstance(repair(options(false)));
+ waitMarkedRepaired(cluster);
+ // make node2 mismatch
+ unmarkRepaired(cluster.get(2), "tbl");
+ verifySnapshots(cluster, "tbl", true);
+ verifySnapshots(cluster, "tbl2", true);
+
+ AtomicInteger snapshotMessageCounter = new AtomicInteger();
+ cluster.filters().verbs(Verb.SNAPSHOT_REQ.id).messagesMatching((from, to, message) -> {
+ snapshotMessageCounter.incrementAndGet();
+ return false;
+ }).drop();
+ cluster.get(1).callOnInstance(repair(options(true)));
+ verifySnapshots(cluster, "tbl", false);
+ // tbl2 should not have a mismatch, so the snapshots should be empty here
+ verifySnapshots(cluster, "tbl2", true);
+ assertEquals(3, snapshotMessageCounter.get());
+
+ // and make sure that we don't try to snapshot again
+ snapshotMessageCounter.set(0);
+ cluster.get(3).callOnInstance(repair(options(true)));
+ assertEquals(0, snapshotMessageCounter.get());
+ }
+ }
+
+ private void waitMarkedRepaired(Cluster cluster)
+ {
+ cluster.forEach(node -> node.runOnInstance(() -> {
+ for (String table : Arrays.asList("tbl", "tbl2"))
+ {
+ ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(table);
+ while (true)
+ {
+ if (cfs.getLiveSSTables().stream().allMatch(SSTableReader::isRepaired))
+ return;
+ Uninterruptibles.sleepUninterruptibly(100, TimeUnit.MILLISECONDS);
+ }
+ }
+ }));
+ }
+
+ private void unmarkRepaired(IInvokableInstance instance, String table)
+ {
+ instance.runOnInstance(() -> {
+ ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(table);
+ try
+ {
+ cfs.getCompactionStrategyManager().mutateRepaired(cfs.getLiveSSTables(), ActiveRepairService.UNREPAIRED_SSTABLE, null, false);
+ }
+ catch (IOException e)
+ {
+ throw new RuntimeException(e);
+ }
+ });
+ }
+
+ private void verifySnapshots(Cluster cluster, String table, boolean shouldBeEmpty)
+ {
+ cluster.forEach(node -> node.runOnInstance(() -> {
+ ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(table);
+ if(shouldBeEmpty)
+ {
+ assertTrue(cfs.getSnapshotDetails().isEmpty());
+ }
+ else
+ {
+ while (cfs.getSnapshotDetails().isEmpty())
+ Uninterruptibles.sleepUninterruptibly(100, TimeUnit.MILLISECONDS);
+ }
+ }));
+ }
+
+ static class DelayMessageFilter implements IMessageFilters.Matcher
+ {
+ private final SimpleCondition condition;
+ private final AtomicBoolean waitForRepair = new AtomicBoolean(true);
+
+ public DelayMessageFilter(SimpleCondition condition)
+ {
+ this.condition = condition;
+ }
+ public boolean matches(int from, int to, IMessage message)
+ {
+ try
+ {
+ // only the first validation req should be delayed:
+ if (waitForRepair.compareAndSet(true, false))
+ condition.await();
+ }
+ catch (Exception e)
+ {
+ throw new RuntimeException(e);
+ }
+ return false; // don't drop the message
+ }
+ }
+
+ private static void insert(ICoordinator coordinator, int start, int count)
+ {
+ insert(coordinator, start, count, "tbl");
+ }
+
+ static void insert(ICoordinator coordinator, int start, int count, String table)
+ {
+ for (int i = start; i < start + count; i++)
+ coordinator.execute("insert into " + KEYSPACE + "." + table + " (id, t) values (?, ?)", ConsistencyLevel.ALL, i, i);
+ }
+
+ /**
+ * returns a pair with [repair success, was inconsistent]
+ */
+ private static IIsolatedExecutor.SerializableCallable<RepairResult> repair(Map<String, String> options)
+ {
+ return () -> {
+ SimpleCondition await = new SimpleCondition();
+ AtomicBoolean success = new AtomicBoolean(true);
+ AtomicBoolean wasInconsistent = new AtomicBoolean(false);
+ StorageService.instance.repair(KEYSPACE, options, ImmutableList.of((tag, event) -> {
+ if (event.getType() == ProgressEventType.ERROR)
+ {
+ success.set(false);
+ await.signalAll();
+ }
+ else if (event.getType() == ProgressEventType.NOTIFICATION && event.getMessage().contains("Repaired data is inconsistent"))
+ {
+ wasInconsistent.set(true);
+ }
+ else if (event.getType() == ProgressEventType.COMPLETE)
+ await.signalAll();
+ }));
+ try
+ {
+ await.await(1, TimeUnit.MINUTES);
+ }
+ catch (InterruptedException e)
+ {
+ throw new RuntimeException(e);
+ }
+ return new RepairResult(success.get(), wasInconsistent.get());
+ };
+ }
+
+ private static Map<String, String> options(boolean preview)
+ {
+ Map<String, String> config = new HashMap<>();
+ config.put(RepairOption.INCREMENTAL_KEY, "true");
+ config.put(RepairOption.PARALLELISM_KEY, RepairParallelism.PARALLEL.toString());
+ if (preview)
+ config.put(RepairOption.PREVIEW, PreviewKind.REPAIRED.toString());
+ return config;
+ }
+
+ private static Map<String, String> options(boolean preview, String range)
+ {
+ Map<String, String> options = options(preview);
+ options.put(RepairOption.RANGES_KEY, range);
+ return options;
+ }
+}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/QueryReplayerEndToEndTest.java b/test/distributed/org/apache/cassandra/distributed/test/QueryReplayerEndToEndTest.java
new file mode 100644
index 0000000..9202cb4
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/test/QueryReplayerEndToEndTest.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import org.apache.cassandra.cql3.QueryOptions;
+import org.apache.cassandra.distributed.api.ConsistencyLevel;
+import org.apache.cassandra.distributed.api.Feature;
+import org.apache.cassandra.distributed.api.ICluster;
+import org.apache.cassandra.distributed.api.IInvokableInstance;
+import org.apache.cassandra.fqltool.FQLQuery;
+import org.apache.cassandra.fqltool.QueryReplayer;
+
+public class QueryReplayerEndToEndTest extends TestBaseImpl
+{
+ private final AtomicLong queryStartTimeGenerator = new AtomicLong(1000);
+ private final AtomicInteger ckGenerator = new AtomicInteger(1);
+
+ @Test
+ public void testReplayAndCloseMultipleTimes() throws Throwable
+ {
+ try (ICluster<IInvokableInstance> cluster = init(builder().withNodes(3)
+ .withConfig(conf -> conf.with(Feature.NATIVE_PROTOCOL, Feature.GOSSIP, Feature.NETWORK))
+ .start()))
+ {
+ cluster.schemaChange("CREATE TABLE " + KEYSPACE + ".tbl (pk int, ck int, v int, PRIMARY KEY (pk, ck))");
+ List<String> hosts = cluster.stream()
+ .map(i -> i.config().broadcastAddress().getAddress().getHostAddress())
+ .collect(Collectors.toList());
+
+ final int queriesCount = 3;
+ // replay for the first time, it should pass
+ replayAndClose(Collections.singletonList(makeFQLQueries(queriesCount)), hosts);
+ // replay for the second time, it should pass too
+ // however, if the cached sessions are not released, the second replay will reused the closed sessions from previous replay and fail to insert
+ replayAndClose(Collections.singletonList(makeFQLQueries(queriesCount)), hosts);
+ Object[][] result = cluster.coordinator(1)
+ .execute("SELECT * FROM " + KEYSPACE + ".tbl WHERE pk = ?",
+ ConsistencyLevel.QUORUM, 1);
+ Assert.assertEquals(String.format("Expecting to see %d rows since it replayed twice and each with %d queries", queriesCount * 2, queriesCount),
+ queriesCount * 2, result.length);
+ }
+ }
+
+ private void replayAndClose(List<List<FQLQuery>> allFqlQueries, List<String> hosts) throws IOException
+ {
+ List<Predicate<FQLQuery>> allowAll = Collections.singletonList(fqlQuery -> true);
+ try (QueryReplayer queryReplayer = new QueryReplayer(allFqlQueries.iterator(), hosts, null, allowAll, null))
+ {
+ queryReplayer.replay();
+ }
+ }
+
+ // generate a new list of FQLQuery for each invocation
+ private List<FQLQuery> makeFQLQueries(int n)
+ {
+ return IntStream.range(0, n)
+ .boxed()
+ .map(i -> new FQLQuery.Single(KEYSPACE,
+ QueryOptions.DEFAULT.getProtocolVersion().asInt(),
+ QueryOptions.DEFAULT, queryStartTimeGenerator.incrementAndGet(),
+ 2222,
+ 3333,
+ String.format("INSERT INTO %s.tbl (pk, ck, v) VALUES (1, %d, %d)", KEYSPACE, ckGenerator.incrementAndGet(), i),
+ Collections.emptyList()))
+ .collect(Collectors.toList());
+ }
+}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/ReadRepairTest.java b/test/distributed/org/apache/cassandra/distributed/test/ReadRepairTest.java
index 3f50bd4..f0c82b8 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/ReadRepairTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/ReadRepairTest.java
@@ -18,15 +18,115 @@
package org.apache.cassandra.distributed.test;
+import java.net.InetSocketAddress;
import java.util.Iterator;
+import java.util.List;
import org.junit.Test;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.dht.Token;
import org.apache.cassandra.distributed.Cluster;
import org.apache.cassandra.distributed.api.ConsistencyLevel;
+import org.apache.cassandra.distributed.api.ICluster;
+import org.apache.cassandra.distributed.shared.NetworkTopology;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.service.PendingRangeCalculatorService;
+import org.apache.cassandra.service.StorageService;
+
+import static org.apache.cassandra.net.Verb.READ_REPAIR_REQ;
+import static org.apache.cassandra.net.Verb.READ_REQ;
+import static org.apache.cassandra.distributed.shared.AssertUtils.*;
public class ReadRepairTest extends TestBaseImpl
{
+
+ @Test
+ public void readRepairTest() throws Throwable
+ {
+ try (ICluster cluster = init(builder().withNodes(3).start()))
+ {
+ cluster.schemaChange("CREATE TABLE " + KEYSPACE + ".tbl (pk int, ck int, v int, PRIMARY KEY (pk, ck)) WITH read_repair='blocking'");
+
+ cluster.get(1).executeInternal("INSERT INTO " + KEYSPACE + ".tbl (pk, ck, v) VALUES (1, 1, 1)");
+ cluster.get(2).executeInternal("INSERT INTO " + KEYSPACE + ".tbl (pk, ck, v) VALUES (1, 1, 1)");
+
+ assertRows(cluster.get(3).executeInternal("SELECT * FROM " + KEYSPACE + ".tbl WHERE pk = 1"));
+
+ assertRows(cluster.coordinator(1).execute("SELECT * FROM " + KEYSPACE + ".tbl WHERE pk = 1",
+ ConsistencyLevel.ALL),
+ row(1, 1, 1));
+
+ // Verify that data got repaired to the third node
+ assertRows(cluster.get(3).executeInternal("SELECT * FROM " + KEYSPACE + ".tbl WHERE pk = 1"),
+ row(1, 1, 1));
+ }
+ }
+
+ @Test
+ public void failingReadRepairTest() throws Throwable
+ {
+ try (ICluster cluster = init(builder().withNodes(3).start()))
+ {
+ cluster.schemaChange("CREATE TABLE " + KEYSPACE + ".tbl (pk int, ck int, v int, PRIMARY KEY (pk, ck)) WITH read_repair='blocking'");
+
+ for (int i = 1 ; i <= 2 ; ++i)
+ cluster.get(i).executeInternal("INSERT INTO " + KEYSPACE + ".tbl (pk, ck, v) VALUES (1, 1, 1)");
+
+ assertRows(cluster.get(3).executeInternal("SELECT * FROM " + KEYSPACE + ".tbl WHERE pk = 1"));
+
+ cluster.filters().verbs(READ_REPAIR_REQ.id).to(3).drop();
+ assertRows(cluster.coordinator(1).execute("SELECT * FROM " + KEYSPACE + ".tbl WHERE pk = 1",
+ ConsistencyLevel.QUORUM),
+ row(1, 1, 1));
+
+ // Data was not repaired
+ assertRows(cluster.get(3).executeInternal("SELECT * FROM " + KEYSPACE + ".tbl WHERE pk = 1"));
+ }
+ }
+
+ @Test
+ public void movingTokenReadRepairTest() throws Throwable
+ {
+ try (Cluster cluster = (Cluster) init(Cluster.create(4), 3))
+ {
+ List<Token> tokens = cluster.tokens();
+
+ cluster.schemaChange("CREATE TABLE " + KEYSPACE + ".tbl (pk int, ck int, v int, PRIMARY KEY (pk, ck)) WITH read_repair='blocking'");
+
+ int i = 0;
+ while (true)
+ {
+ Token t = Murmur3Partitioner.instance.getToken(Int32Type.instance.decompose(i));
+ if (t.compareTo(tokens.get(2 - 1)) < 0 && t.compareTo(tokens.get(1 - 1)) > 0)
+ break;
+ ++i;
+ }
+
+ // write only to #4
+ cluster.get(4).executeInternal("INSERT INTO " + KEYSPACE + ".tbl (pk, ck, v) VALUES (?, 1, 1)", i);
+ // mark #2 as leaving in #4
+ cluster.get(4).acceptsOnInstance((InetSocketAddress endpoint) -> {
+ StorageService.instance.getTokenMetadata().addLeavingEndpoint(InetAddressAndPort.getByAddressOverrideDefaults(endpoint.getAddress(), endpoint.getPort()));
+ PendingRangeCalculatorService.instance.update();
+ PendingRangeCalculatorService.instance.blockUntilFinished();
+ }).accept(cluster.get(2).broadcastAddress());
+
+ // prevent #4 from reading or writing to #3, so our QUORUM must contain #2 and #4
+ // since #1 is taking over the range, this means any read-repair must make it to #1 as well
+ cluster.filters().verbs(READ_REQ.ordinal()).from(4).to(3).drop();
+ cluster.filters().verbs(READ_REPAIR_REQ.ordinal()).from(4).to(3).drop();
+ assertRows(cluster.coordinator(4).execute("SELECT * FROM " + KEYSPACE + ".tbl WHERE pk = ?",
+ ConsistencyLevel.ALL, i),
+ row(i, 1, 1));
+
+ // verify that #1 receives the write
+ assertRows(cluster.get(1).executeInternal("SELECT * FROM " + KEYSPACE + ".tbl WHERE pk = ?", i),
+ row(i, 1, 1));
+ }
+ }
+
@Test
public void emptyRangeTombstones1() throws Throwable
{
@@ -40,11 +140,11 @@
cluster.get(1).executeInternal("DELETE FROM distributed_test_keyspace.tbl WHERE key=? AND column1>? AND column1<?;",
"test", Integer.MIN_VALUE, Integer.MAX_VALUE);
cluster.coordinator(2).execute("SELECT * FROM distributed_test_keyspace.tbl WHERE key = ? and column1 > ? and column1 <= ?",
- ConsistencyLevel.ALL,
- "test", 10, 10);
+ ConsistencyLevel.ALL,
+ "test", 10, 10);
cluster.coordinator(2).execute("SELECT * FROM distributed_test_keyspace.tbl WHERE key = ? and column1 > ? and column1 <= ?",
- ConsistencyLevel.ALL,
- "test", 11, 11);
+ ConsistencyLevel.ALL,
+ "test", 11, 11);
cluster.get(2).executeInternal("DELETE FROM distributed_test_keyspace.tbl WHERE key=? AND column1>? AND column1<?;",
"test", Integer.MIN_VALUE, Integer.MAX_VALUE);
}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/RepairCoordinatorBase.java b/test/distributed/org/apache/cassandra/distributed/test/RepairCoordinatorBase.java
new file mode 100644
index 0000000..fc058db
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/test/RepairCoordinatorBase.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.runners.Parameterized;
+
+import org.apache.cassandra.distributed.Cluster;
+import org.apache.cassandra.distributed.api.Feature;
+import org.apache.cassandra.distributed.api.NodeToolResult;
+import org.apache.cassandra.distributed.test.DistributedRepairUtils.RepairParallelism;
+import org.apache.cassandra.distributed.test.DistributedRepairUtils.RepairType;
+
+public class RepairCoordinatorBase extends TestBaseImpl
+{
+ protected static Cluster CLUSTER;
+
+ protected final RepairType repairType;
+ protected final RepairParallelism parallelism;
+ protected final boolean withNotifications;
+
+ public RepairCoordinatorBase(RepairType repairType, RepairParallelism parallelism, boolean withNotifications)
+ {
+ this.repairType = repairType;
+ this.parallelism = parallelism;
+ this.withNotifications = withNotifications;
+ }
+
+ @Parameterized.Parameters(name = "{0}/{1}")
+ public static Collection<Object[]> testsWithoutType()
+ {
+ List<Object[]> tests = new ArrayList<>();
+ for (RepairParallelism p : RepairParallelism.values())
+ {
+ tests.add(new Object[] { p, true });
+ tests.add(new Object[] { p, false });
+ }
+ return tests;
+ }
+
+ @BeforeClass
+ public static void before()
+ {
+ // This only works because the way CI works
+ // In CI a new JVM is spun up for each test file, so this doesn't have to worry about another test file
+ // getting this set first
+ System.setProperty("cassandra.nodetool.jmx_notification_poll_interval_seconds", "1");
+ }
+
+ @BeforeClass
+ public static void setupCluster() throws IOException
+ {
+ // streaming requires networking ATM
+ // streaming also requires gossip or isn't setup properly
+ CLUSTER = init(Cluster.build(2)
+ .withConfig(c -> c.with(Feature.NETWORK)
+ .with(Feature.GOSSIP))
+ .start());
+ }
+
+ @AfterClass
+ public static void teardownCluster()
+ {
+ if (CLUSTER != null)
+ CLUSTER.close();
+ }
+
+ protected String tableName(String prefix) {
+ return prefix + "_" + postfix();
+ }
+
+ protected String postfix()
+ {
+ return repairType.name().toLowerCase() + "_" + parallelism.name().toLowerCase() + "_" + withNotifications;
+ }
+
+ protected NodeToolResult repair(int node, String... args) {
+ return DistributedRepairUtils.repair(CLUSTER, node, repairType, withNotifications, parallelism.append(args));
+ }
+}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/RepairCoordinatorFailingMessageTest.java b/test/distributed/org/apache/cassandra/distributed/test/RepairCoordinatorFailingMessageTest.java
new file mode 100644
index 0000000..0d04649
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/test/RepairCoordinatorFailingMessageTest.java
@@ -0,0 +1,186 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+import org.junit.AfterClass;
+import org.junit.Assume;
+import org.junit.BeforeClass;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.distributed.Cluster;
+import org.apache.cassandra.distributed.api.Feature;
+import org.apache.cassandra.distributed.api.IMessageFilters;
+import org.apache.cassandra.distributed.api.NodeToolResult;
+import org.apache.cassandra.distributed.test.DistributedRepairUtils.RepairType;
+import org.apache.cassandra.net.Verb;
+
+import static java.lang.String.format;
+import static org.apache.cassandra.distributed.api.IMessageFilters.Matcher.of;
+
+@RunWith(Parameterized.class)
+@Ignore("Until CASSANDRA-15566 is in these tests all time out")
+public class RepairCoordinatorFailingMessageTest extends TestBaseImpl implements Serializable
+{
+ private static Cluster CLUSTER;
+
+ private final RepairType repairType;
+ private final boolean withNotifications;
+
+ public RepairCoordinatorFailingMessageTest(RepairType repairType, boolean withNotifications)
+ {
+ this.repairType = repairType;
+ this.withNotifications = withNotifications;
+ }
+
+ @Parameterized.Parameters(name = "{0}/{1}")
+ public static Collection<Object[]> messages()
+ {
+ List<Object[]> tests = new ArrayList<>();
+ for (RepairType type : RepairType.values())
+ {
+ tests.add(new Object[] { type, true });
+ tests.add(new Object[] { type, false });
+ }
+ return tests;
+ }
+
+ @BeforeClass
+ public static void before()
+ {
+ DatabaseDescriptor.clientInitialization();
+ }
+
+ @BeforeClass
+ public static void setupCluster() throws IOException
+ {
+ // streaming requires networking ATM
+ // streaming also requires gossip or isn't setup properly
+ CLUSTER = init(Cluster.build(3) // set to 3 so streaming hits non-local case
+ .withConfig(c -> c.with(Feature.NETWORK)
+ .with(Feature.GOSSIP))
+ .start());
+ }
+
+ @AfterClass
+ public static void teardownCluster()
+ {
+ if (CLUSTER != null)
+ CLUSTER.close();
+ }
+
+ private String tableName(String prefix) {
+ return prefix + "_" + postfix() + "_" + withNotifications;
+ }
+
+ private String postfix()
+ {
+ return repairType.name().toLowerCase();
+ }
+
+ private NodeToolResult repair(int node, String... args) {
+ return DistributedRepairUtils.repair(CLUSTER, node, repairType, withNotifications, args);
+ }
+
+ @Test(timeout = 1 * 60 * 1000)
+ public void prepareIrFailure()
+ {
+ Assume.assumeTrue("The Verb.PREPARE_CONSISTENT_REQ is only for incremental, so disable in non-incremental", repairType == RepairType.INCREMENTAL);
+ // Wait, isn't this copy paste of RepairCoordinatorTest::prepareFailure? NO!
+ // Incremental repair sends the PREPARE message the same way full does, but then after it does it sends
+ // a consistent prepare message... and that one doesn't handle errors...
+ CLUSTER.schemaChange("CREATE TABLE " + KEYSPACE + ".prepareirfailure (key text, value text, PRIMARY KEY (key))");
+ IMessageFilters.Filter filter = CLUSTER.verbs(Verb.PREPARE_CONSISTENT_REQ).messagesMatching(of(m -> {
+ throw new RuntimeException("prepare fail");
+ })).drop();
+ try
+ {
+ NodeToolResult result = repair(1, KEYSPACE, "prepareirfailure");
+ result.asserts()
+ .failure()
+ .errorContains("error prepare fail")
+ .notificationContains(NodeToolResult.ProgressEventType.ERROR, "error prepare fail")
+ .notificationContains(NodeToolResult.ProgressEventType.COMPLETE, "finished with error");
+ }
+ finally
+ {
+ filter.off();
+ }
+ }
+
+ //TODO failure reply murkle tree
+ //TODO failure reply murkle tree IR
+
+ @Test(timeout = 1 * 60 * 1000)
+ public void validationFailure()
+ {
+ String table = tableName("validationfailure");
+ CLUSTER.schemaChange(format("CREATE TABLE %s.%s (key text, value text, PRIMARY KEY (key))", KEYSPACE, table));
+ IMessageFilters.Filter filter = CLUSTER.verbs(Verb.VALIDATION_REQ).messagesMatching(of(m -> {
+ throw new RuntimeException("validation fail");
+ })).drop();
+ try
+ {
+ NodeToolResult result = repair(1, KEYSPACE, table);
+ result.asserts()
+ .failure()
+ .errorContains("Some repair failed")
+ .notificationContains(NodeToolResult.ProgressEventType.ERROR, "Some repair failed")
+ .notificationContains(NodeToolResult.ProgressEventType.COMPLETE, "finished with error");
+ }
+ finally
+ {
+ filter.off();
+ }
+ }
+
+ @Test(timeout = 1 * 60 * 1000)
+ public void streamFailure()
+ {
+ String table = tableName("streamfailure");
+ CLUSTER.schemaChange(format("CREATE TABLE %s.%s (key text, value text, PRIMARY KEY (key))", KEYSPACE, table));
+ // there needs to be a difference to cause streaming to happen, so add to one node
+ CLUSTER.get(2).executeInternal(format("INSERT INTO %s.%s (key) VALUES (?)", KEYSPACE, table), "some data");
+ IMessageFilters.Filter filter = CLUSTER.verbs(Verb.SYNC_REQ).messagesMatching(of(m -> {
+ throw new RuntimeException("stream fail");
+ })).drop();
+ try
+ {
+ NodeToolResult result = repair(1, KEYSPACE, table);
+ result.asserts()
+ .failure()
+ .errorContains("Some repair failed")
+ .notificationContains(NodeToolResult.ProgressEventType.ERROR, "Some repair failed")
+ .notificationContains(NodeToolResult.ProgressEventType.COMPLETE, "finished with error");
+ }
+ finally
+ {
+ filter.off();
+ }
+ }
+}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/RepairCoordinatorFast.java b/test/distributed/org/apache/cassandra/distributed/test/RepairCoordinatorFast.java
new file mode 100644
index 0000000..0e156da
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/test/RepairCoordinatorFast.java
@@ -0,0 +1,408 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test;
+
+import java.time.Duration;
+import java.util.Set;
+
+import com.google.common.collect.Iterables;
+import org.junit.Assert;
+import org.junit.Assume;
+import org.junit.Test;
+
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.distributed.api.ConsistencyLevel;
+import org.apache.cassandra.distributed.api.IMessageFilters;
+import org.apache.cassandra.distributed.api.LongTokenRange;
+import org.apache.cassandra.distributed.api.NodeToolResult;
+import org.apache.cassandra.distributed.api.NodeToolResult.ProgressEventType;
+import org.apache.cassandra.distributed.test.DistributedRepairUtils.RepairParallelism;
+import org.apache.cassandra.distributed.test.DistributedRepairUtils.RepairType;
+import org.apache.cassandra.net.Verb;
+import org.apache.cassandra.service.StorageService;
+
+import static java.lang.String.format;
+import static org.apache.cassandra.distributed.api.IMessageFilters.Matcher.of;
+import static org.apache.cassandra.distributed.test.DistributedRepairUtils.assertParentRepairFailedWithMessageContains;
+import static org.apache.cassandra.distributed.test.DistributedRepairUtils.assertParentRepairNotExist;
+import static org.apache.cassandra.distributed.test.DistributedRepairUtils.assertParentRepairSuccess;
+import static org.apache.cassandra.distributed.test.DistributedRepairUtils.getRepairExceptions;
+import static org.apache.cassandra.utils.AssertUtil.assertTimeoutPreemptively;
+
+public abstract class RepairCoordinatorFast extends RepairCoordinatorBase
+{
+ public RepairCoordinatorFast(RepairType repairType, RepairParallelism parallelism, boolean withNotifications)
+ {
+ super(repairType, parallelism, withNotifications);
+ }
+
+ @Test
+ public void simple() {
+ String table = tableName("simple");
+ assertTimeoutPreemptively(Duration.ofMinutes(1), () -> {
+ CLUSTER.schemaChange(format("CREATE TABLE %s.%s (key text, PRIMARY KEY (key))", KEYSPACE, table));
+ CLUSTER.coordinator(1).execute(format("INSERT INTO %s.%s (key) VALUES (?)", KEYSPACE, table), ConsistencyLevel.ANY, "some text");
+
+ long repairExceptions = getRepairExceptions(CLUSTER, 2);
+ NodeToolResult result = repair(2, KEYSPACE, table);
+ result.asserts().success();
+ if (withNotifications)
+ {
+ result.asserts()
+ .notificationContains(ProgressEventType.START, "Starting repair command")
+ .notificationContains(ProgressEventType.START, "repairing keyspace " + KEYSPACE + " with repair options")
+ .notificationContains(ProgressEventType.SUCCESS, repairType != RepairType.PREVIEW ? "Repair completed successfully": "Repair preview completed successfully")
+ .notificationContains(ProgressEventType.COMPLETE, "finished");
+ }
+
+ if (repairType != RepairType.PREVIEW)
+ {
+ assertParentRepairSuccess(CLUSTER, KEYSPACE, table);
+ }
+ else
+ {
+ assertParentRepairNotExist(CLUSTER, KEYSPACE, table);
+ }
+
+ Assert.assertEquals(repairExceptions, getRepairExceptions(CLUSTER, 2));
+ });
+ }
+
+ @Test
+ public void missingKeyspace()
+ {
+ assertTimeoutPreemptively(Duration.ofMinutes(1), () -> {
+ // as of this moment the check is done in nodetool so the JMX notifications are not imporant
+ // nor is the history stored
+ long repairExceptions = getRepairExceptions(CLUSTER, 2);
+ NodeToolResult result = repair(2, "doesnotexist");
+ result.asserts()
+ .failure()
+ .errorContains("Keyspace [doesnotexist] does not exist.");
+
+ Assert.assertEquals(repairExceptions, getRepairExceptions(CLUSTER, 2));
+
+ assertParentRepairNotExist(CLUSTER, "doesnotexist");
+ });
+ }
+
+ @Test
+ public void missingTable()
+ {
+ assertTimeoutPreemptively(Duration.ofMinutes(1), () -> {
+ long repairExceptions = getRepairExceptions(CLUSTER, 2);
+ String tableName = tableName("doesnotexist");
+ NodeToolResult result = repair(2, KEYSPACE, tableName);
+ result.asserts()
+ .failure();
+ if (withNotifications)
+ {
+ result.asserts()
+ .errorContains("Unknown keyspace/cf pair (distributed_test_keyspace." + tableName + ")")
+ // Start notification is ignored since this is checked during setup (aka before start)
+ .notificationContains(ProgressEventType.ERROR, "failed with error Unknown keyspace/cf pair (distributed_test_keyspace." + tableName + ")")
+ .notificationContains(ProgressEventType.COMPLETE, "finished with error");
+ }
+
+ assertParentRepairNotExist(CLUSTER, KEYSPACE, "doesnotexist");
+
+ Assert.assertEquals(repairExceptions + 1, getRepairExceptions(CLUSTER, 2));
+ });
+ }
+
+ @Test
+ public void noTablesToRepair()
+ {
+ // index CF currently don't support repair, so they get dropped when listed
+ // this is done in this test to cause the keyspace to have 0 tables to repair, which causes repair to no-op
+ // early and skip.
+ String table = tableName("withindex");
+ assertTimeoutPreemptively(Duration.ofMinutes(1), () -> {
+ CLUSTER.schemaChange(format("CREATE TABLE %s.%s (key text, value text, PRIMARY KEY (key))", KEYSPACE, table));
+ CLUSTER.schemaChange(format("CREATE INDEX value_%s ON %s.%s (value)", postfix(), KEYSPACE, table));
+
+ long repairExceptions = getRepairExceptions(CLUSTER, 2);
+ // if CF has a . in it, it is assumed to be a 2i which rejects repairs
+ NodeToolResult result = repair(2, KEYSPACE, table + ".value");
+ result.asserts().success();
+ if (withNotifications)
+ {
+ result.asserts()
+ .notificationContains("Empty keyspace")
+ .notificationContains("skipping repair: " + KEYSPACE)
+ // Start notification is ignored since this is checked during setup (aka before start)
+ .notificationContains(ProgressEventType.SUCCESS, "Empty keyspace") // will fail since success isn't returned; only complete
+ .notificationContains(ProgressEventType.COMPLETE, "finished"); // will fail since it doesn't do this
+ }
+
+ assertParentRepairNotExist(CLUSTER, KEYSPACE, table + ".value");
+
+ // this is actually a SKIP and not a FAILURE, so shouldn't increment
+ Assert.assertEquals(repairExceptions, getRepairExceptions(CLUSTER, 2));
+ });
+ }
+
+ @Test
+ public void intersectingRange()
+ {
+ // this test exists to show that this case will cause repair to finish; success or failure isn't imporant
+ // if repair is enhanced to allow intersecting ranges w/ local then this test will fail saying that we expected
+ // repair to fail but it didn't, this would be fine and this test should be updated to reflect the new
+ // semantic
+ String table = tableName("intersectingrange");
+ assertTimeoutPreemptively(Duration.ofMinutes(1), () -> {
+ CLUSTER.schemaChange(format("CREATE TABLE %s.%s (key text, value text, PRIMARY KEY (key))", KEYSPACE, table));
+
+ //TODO dtest api for this?
+ LongTokenRange tokenRange = CLUSTER.get(2).callOnInstance(() -> {
+ Set<Range<Token>> ranges = StorageService.instance.getLocalReplicas(KEYSPACE).ranges();
+ Range<Token> range = Iterables.getFirst(ranges, null);
+ long left = (long) range.left.getTokenValue();
+ long right = (long) range.right.getTokenValue();
+ return new LongTokenRange(left, right);
+ });
+ LongTokenRange intersectingRange = new LongTokenRange(tokenRange.maxInclusive - 7, tokenRange.maxInclusive + 7);
+
+ long repairExceptions = getRepairExceptions(CLUSTER, 2);
+ NodeToolResult result = repair(2, KEYSPACE, table,
+ "--start-token", Long.toString(intersectingRange.minExclusive),
+ "--end-token", Long.toString(intersectingRange.maxInclusive));
+ result.asserts()
+ .failure()
+ .errorContains("Requested range " + intersectingRange + " intersects a local range (" + tokenRange + ") but is not fully contained in one");
+ if (withNotifications)
+ {
+ result.asserts()
+ .notificationContains(ProgressEventType.START, "Starting repair command")
+ .notificationContains(ProgressEventType.START, "repairing keyspace " + KEYSPACE + " with repair options")
+ .notificationContains(ProgressEventType.ERROR, "Requested range " + intersectingRange + " intersects a local range (" + tokenRange + ") but is not fully contained in one")
+ .notificationContains(ProgressEventType.COMPLETE, "finished with error");
+ }
+
+ assertParentRepairNotExist(CLUSTER, KEYSPACE, table);
+
+ Assert.assertEquals(repairExceptions + 1, getRepairExceptions(CLUSTER, 2));
+ });
+ }
+
+ @Test
+ public void unknownHost()
+ {
+ String table = tableName("unknownhost");
+ assertTimeoutPreemptively(Duration.ofMinutes(1), () -> {
+ CLUSTER.schemaChange(format("CREATE TABLE %s.%s (key text, value text, PRIMARY KEY (key))", KEYSPACE, table));
+
+ long repairExceptions = getRepairExceptions(CLUSTER, 2);
+ NodeToolResult result = repair(2, KEYSPACE, table, "--in-hosts", "thisreally.should.not.exist.apache.org");
+ result.asserts()
+ .failure()
+ .errorContains("Unknown host specified thisreally.should.not.exist.apache.org");
+ if (withNotifications)
+ {
+ result.asserts()
+ .notificationContains(ProgressEventType.START, "Starting repair command")
+ .notificationContains(ProgressEventType.START, "repairing keyspace " + KEYSPACE + " with repair options")
+ .notificationContains(ProgressEventType.ERROR, "Unknown host specified thisreally.should.not.exist.apache.org")
+ .notificationContains(ProgressEventType.COMPLETE, "finished with error");
+ }
+
+ assertParentRepairNotExist(CLUSTER, KEYSPACE, table);
+
+ Assert.assertEquals(repairExceptions + 1, getRepairExceptions(CLUSTER, 2));
+ });
+ }
+
+ @Test
+ public void desiredHostNotCoordinator()
+ {
+ // current limitation is that the coordinator must be apart of the repair, so as long as that exists this test
+ // verifies that the validation logic will termniate the repair properly
+ String table = tableName("desiredhostnotcoordinator");
+ assertTimeoutPreemptively(Duration.ofMinutes(1), () -> {
+ CLUSTER.schemaChange(format("CREATE TABLE %s.%s (key text, value text, PRIMARY KEY (key))", KEYSPACE, table));
+
+ long repairExceptions = getRepairExceptions(CLUSTER, 2);
+ NodeToolResult result = repair(2, KEYSPACE, table, "--in-hosts", "localhost");
+ result.asserts()
+ .failure()
+ .errorContains("The current host must be part of the repair");
+ if (withNotifications)
+ {
+ result.asserts()
+ .notificationContains(ProgressEventType.START, "Starting repair command")
+ .notificationContains(ProgressEventType.START, "repairing keyspace " + KEYSPACE + " with repair options")
+ .notificationContains(ProgressEventType.ERROR, "The current host must be part of the repair")
+ .notificationContains(ProgressEventType.COMPLETE, "finished with error");
+ }
+
+ assertParentRepairNotExist(CLUSTER, KEYSPACE, table);
+
+ Assert.assertEquals(repairExceptions + 1, getRepairExceptions(CLUSTER, 2));
+ });
+ }
+
+ @Test
+ public void onlyCoordinator()
+ {
+ // this is very similar to ::desiredHostNotCoordinator but has the difference that the only host to do repair
+ // is the coordinator
+ String table = tableName("onlycoordinator");
+ assertTimeoutPreemptively(Duration.ofMinutes(1), () -> {
+ CLUSTER.schemaChange(format("CREATE TABLE %s.%s (key text, value text, PRIMARY KEY (key))", KEYSPACE, table));
+
+ long repairExceptions = getRepairExceptions(CLUSTER, 2);
+ NodeToolResult result = repair(1, KEYSPACE, table, "--in-hosts", "localhost");
+ result.asserts()
+ .failure()
+ .errorContains("Specified hosts [localhost] do not share range");
+ if (withNotifications)
+ {
+ result.asserts()
+ .notificationContains(ProgressEventType.START, "Starting repair command")
+ .notificationContains(ProgressEventType.START, "repairing keyspace " + KEYSPACE + " with repair options")
+ .notificationContains(ProgressEventType.ERROR, "Specified hosts [localhost] do not share range")
+ .notificationContains(ProgressEventType.COMPLETE, "finished with error");
+ }
+
+ assertParentRepairNotExist(CLUSTER, KEYSPACE, table);
+
+ //TODO should this be marked as fail to match others? Should they not be marked?
+ Assert.assertEquals(repairExceptions, getRepairExceptions(CLUSTER, 2));
+ });
+ }
+
+ @Test
+ public void replicationFactorOne()
+ {
+ // In the case of rf=1 repair fails to create a cmd handle so node tool exists early
+ String table = tableName("one");
+ assertTimeoutPreemptively(Duration.ofMinutes(1), () -> {
+ // since cluster is shared and this test gets called multiple times, need "IF NOT EXISTS" so the second+ attempt
+ // does not fail
+ CLUSTER.schemaChange("CREATE KEYSPACE IF NOT EXISTS replicationfactor WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1};");
+ CLUSTER.schemaChange(format("CREATE TABLE replicationfactor.%s (key text, value text, PRIMARY KEY (key))", table));
+
+ long repairExceptions = getRepairExceptions(CLUSTER, 1);
+ NodeToolResult result = repair(1, "replicationfactor", table);
+ result.asserts()
+ .success();
+
+ assertParentRepairNotExist(CLUSTER, KEYSPACE, table);
+
+ Assert.assertEquals(repairExceptions, getRepairExceptions(CLUSTER, 1));
+ });
+ }
+
+ @Test
+ public void prepareFailure()
+ {
+ String table = tableName("preparefailure");
+ assertTimeoutPreemptively(Duration.ofMinutes(1), () -> {
+ CLUSTER.schemaChange(format("CREATE TABLE %s.%s (key text, value text, PRIMARY KEY (key))", KEYSPACE, table));
+ IMessageFilters.Filter filter = CLUSTER.verbs(Verb.PREPARE_MSG).messagesMatching(of(m -> {
+ throw new RuntimeException("prepare fail");
+ })).drop();
+ try
+ {
+ long repairExceptions = getRepairExceptions(CLUSTER, 1);
+ NodeToolResult result = repair(1, KEYSPACE, table);
+ result.asserts()
+ .failure()
+ .errorContains("Got negative replies from endpoints");
+ if (withNotifications)
+ {
+ result.asserts()
+ .notificationContains(ProgressEventType.START, "Starting repair command")
+ .notificationContains(ProgressEventType.START, "repairing keyspace " + KEYSPACE + " with repair options")
+ .notificationContains(ProgressEventType.ERROR, "Got negative replies from endpoints")
+ .notificationContains(ProgressEventType.COMPLETE, "finished with error");
+ }
+
+ Assert.assertEquals(repairExceptions + 1, getRepairExceptions(CLUSTER, 1));
+ if (repairType != RepairType.PREVIEW)
+ {
+ assertParentRepairFailedWithMessageContains(CLUSTER, KEYSPACE, table, "Got negative replies from endpoints");
+ }
+ else
+ {
+ assertParentRepairNotExist(CLUSTER, KEYSPACE, table);
+ }
+ }
+ finally
+ {
+ filter.off();
+ }
+ });
+ }
+
+ @Test
+ public void snapshotFailure()
+ {
+ Assume.assumeFalse("incremental does not do snapshot", repairType == RepairType.INCREMENTAL);
+ Assume.assumeFalse("Parallel repair does not perform snapshots", parallelism == RepairParallelism.PARALLEL);
+
+ String table = tableName("snapshotfailure");
+ assertTimeoutPreemptively(Duration.ofMinutes(1), () -> {
+ CLUSTER.schemaChange(format("CREATE TABLE %s.%s (key text, value text, PRIMARY KEY (key))", KEYSPACE, table));
+ IMessageFilters.Filter filter = CLUSTER.verbs(Verb.SNAPSHOT_MSG).messagesMatching(of(m -> {
+ throw new RuntimeException("snapshot fail");
+ })).drop();
+ try
+ {
+ long repairExceptions = getRepairExceptions(CLUSTER, 1);
+ NodeToolResult result = repair(1, KEYSPACE, table);
+ result.asserts()
+ .failure();
+ // Right now coordination doesn't propgate the first exception, so we only know "there exists a issue".
+ // With notifications on nodetool will see the error then complete, so the cmd state (what nodetool
+ // polls on) is ignored. With notifications off or dropped, the poll await fails and queries cmd
+ // state, and that will have the below error.
+ // NOTE: this isn't desireable, would be good to propgate
+ // TODO replace with errorContainsAny once dtest api updated
+ Throwable error = result.getError();
+ Assert.assertNotNull("Error was null", error);
+ if (!(error.getMessage().contains("Could not create snapshot") || error.getMessage().contains("Some repair failed")))
+ throw new AssertionError("Unexpected error, expected to contain 'Could not create snapshot' or 'Some repair failed'", error);
+ if (withNotifications)
+ {
+ result.asserts()
+ .notificationContains(ProgressEventType.START, "Starting repair command")
+ .notificationContains(ProgressEventType.START, "repairing keyspace " + KEYSPACE + " with repair options")
+ .notificationContains(ProgressEventType.ERROR, "Could not create snapshot ")
+ .notificationContains(ProgressEventType.COMPLETE, "finished with error");
+ }
+
+ Assert.assertEquals(repairExceptions + 1, getRepairExceptions(CLUSTER, 1));
+ if (repairType != RepairType.PREVIEW)
+ {
+ assertParentRepairFailedWithMessageContains(CLUSTER, KEYSPACE, table, "Could not create snapshot");
+ }
+ else
+ {
+ assertParentRepairNotExist(CLUSTER, KEYSPACE, table);
+ }
+ }
+ finally
+ {
+ filter.off();
+ }
+ });
+ }
+}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/RepairCoordinatorNeighbourDown.java b/test/distributed/org/apache/cassandra/distributed/test/RepairCoordinatorNeighbourDown.java
new file mode 100644
index 0000000..dd6e2c4
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/test/RepairCoordinatorNeighbourDown.java
@@ -0,0 +1,199 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test;
+
+import java.net.UnknownHostException;
+import java.time.Duration;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+
+import com.google.common.util.concurrent.Uninterruptibles;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import org.apache.cassandra.distributed.api.NodeToolResult;
+import org.apache.cassandra.distributed.test.DistributedRepairUtils.RepairParallelism;
+import org.apache.cassandra.distributed.test.DistributedRepairUtils.RepairType;
+import org.apache.cassandra.gms.FailureDetector;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.net.Verb;
+import org.apache.cassandra.utils.FBUtilities;
+
+import static java.lang.String.format;
+import static org.apache.cassandra.distributed.api.IMessageFilters.Matcher.of;
+import static org.apache.cassandra.distributed.test.DistributedRepairUtils.assertParentRepairFailedWithMessageContains;
+import static org.apache.cassandra.distributed.test.DistributedRepairUtils.assertParentRepairNotExist;
+import static org.apache.cassandra.distributed.test.DistributedRepairUtils.getRepairExceptions;
+import static org.apache.cassandra.utils.AssertUtil.assertTimeoutPreemptively;
+
+public abstract class RepairCoordinatorNeighbourDown extends RepairCoordinatorBase
+{
+ public RepairCoordinatorNeighbourDown(RepairType repairType, RepairParallelism parallelism, boolean withNotifications)
+ {
+ super(repairType, parallelism, withNotifications);
+ }
+
+ @Before
+ public void beforeTest()
+ {
+ CLUSTER.filters().reset();
+ CLUSTER.forEach(i -> {
+ try
+ {
+ i.startup();
+ }
+ catch (IllegalStateException e)
+ {
+ // ignore, node wasn't down
+ }
+ });
+ }
+
+ @Test
+ public void neighbourDown()
+ {
+ String table = tableName("neighbourdown");
+ assertTimeoutPreemptively(Duration.ofMinutes(1), () -> {
+ CLUSTER.schemaChange(format("CREATE TABLE %s.%s (key text, value text, PRIMARY KEY (key))", KEYSPACE, table));
+ String downNodeAddress = CLUSTER.get(2).callOnInstance(() -> FBUtilities.getBroadcastAddressAndPort().toString());
+ Future<Void> shutdownFuture = CLUSTER.get(2).shutdown();
+ try
+ {
+ // wait for the node to stop
+ shutdownFuture.get();
+ // wait for the failure detector to detect this
+ CLUSTER.get(1).runOnInstance(() -> {
+ InetAddressAndPort neighbor;
+ try
+ {
+ neighbor = InetAddressAndPort.getByName(downNodeAddress);
+ }
+ catch (UnknownHostException e)
+ {
+ throw new RuntimeException(e);
+ }
+ while (FailureDetector.instance.isAlive(neighbor))
+ Uninterruptibles.sleepUninterruptibly(500, TimeUnit.MILLISECONDS);
+ });
+
+ long repairExceptions = getRepairExceptions(CLUSTER, 1);
+ NodeToolResult result = repair(1, KEYSPACE, table);
+ result.asserts()
+ .failure()
+ .errorContains("Endpoint not alive");
+ if (withNotifications)
+ {
+ result.asserts()
+ .notificationContains(NodeToolResult.ProgressEventType.START, "Starting repair command")
+ .notificationContains(NodeToolResult.ProgressEventType.START, "repairing keyspace " + KEYSPACE + " with repair options")
+ .notificationContains(NodeToolResult.ProgressEventType.ERROR, "Endpoint not alive")
+ .notificationContains(NodeToolResult.ProgressEventType.COMPLETE, "finished with error");
+ }
+
+ Assert.assertEquals(repairExceptions + 1, getRepairExceptions(CLUSTER, 1));
+ }
+ finally
+ {
+ CLUSTER.get(2).startup();
+ }
+
+ // make sure to call outside of the try/finally so the node is up so we can actually query
+ if (repairType != RepairType.PREVIEW)
+ {
+ assertParentRepairFailedWithMessageContains(CLUSTER, KEYSPACE, table, "Endpoint not alive");
+ }
+ else
+ {
+ assertParentRepairNotExist(CLUSTER, KEYSPACE, table);
+ }
+ });
+ }
+
+ @Test
+ public void validationParticipentCrashesAndComesBack()
+ {
+ // Test what happens when a participant restarts in the middle of validation
+ // Currently this isn't recoverable but could be.
+ // TODO since this is a real restart, how would I test "long pause"? Can't send SIGSTOP since same procress
+ String table = tableName("validationparticipentcrashesandcomesback");
+ assertTimeoutPreemptively(Duration.ofMinutes(1), () -> {
+ CLUSTER.schemaChange(format("CREATE TABLE %s.%s (key text, value text, PRIMARY KEY (key))", KEYSPACE, table));
+ AtomicReference<Future<Void>> participantShutdown = new AtomicReference<>();
+ CLUSTER.verbs(Verb.VALIDATION_REQ).to(2).messagesMatching(of(m -> {
+ // the nice thing about this is that this lambda is "capturing" and not "transfer", what this means is that
+ // this lambda isn't serialized and any object held isn't copied.
+ participantShutdown.set(CLUSTER.get(2).shutdown());
+ return true; // drop it so this node doesn't reply before shutdown.
+ })).drop();
+ // since nodetool is blocking, need to handle participantShutdown in the background
+ CompletableFuture<Void> recovered = CompletableFuture.runAsync(() -> {
+ try {
+ while (participantShutdown.get() == null) {
+ // event not happened, wait for it
+ TimeUnit.MILLISECONDS.sleep(100);
+ }
+ Future<Void> f = participantShutdown.get();
+ f.get(); // wait for shutdown to complete
+ CLUSTER.get(2).startup();
+ } catch (Exception e) {
+ if (e instanceof RuntimeException) {
+ throw (RuntimeException) e;
+ }
+ throw new RuntimeException(e);
+ }
+ });
+
+ long repairExceptions = getRepairExceptions(CLUSTER, 1);
+ NodeToolResult result = repair(1, KEYSPACE, table);
+ recovered.join(); // if recovery didn't happen then the results are not what are being tested, so block here first
+ result.asserts()
+ .failure();
+ if (withNotifications)
+ {
+ result.asserts()
+ .errorContains("Endpoint 127.0.0.2:7012 died")
+ .notificationContains(NodeToolResult.ProgressEventType.ERROR, "Endpoint 127.0.0.2:7012 died")
+ .notificationContains(NodeToolResult.ProgressEventType.COMPLETE, "finished with error");
+ }
+ else
+ {
+ // Right now coordination doesn't propgate the first exception, so we only know "there exists a issue".
+ // With notifications on nodetool will see the error then complete, so the cmd state (what nodetool
+ // polls on) is ignored. With notifications off, the poll await fails and queries cmd state, and that
+ // will have the below error.
+ // NOTE: this isn't desireable, would be good to propgate
+ result.asserts()
+ .errorContains("Some repair failed");
+ }
+
+ Assert.assertEquals(repairExceptions + 1, getRepairExceptions(CLUSTER, 1));
+ if (repairType != RepairType.PREVIEW)
+ {
+ assertParentRepairFailedWithMessageContains(CLUSTER, KEYSPACE, table, "Endpoint 127.0.0.2:7012 died");
+ }
+ else
+ {
+ assertParentRepairNotExist(CLUSTER, KEYSPACE, table);
+ }
+ });
+ }
+}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/RepairCoordinatorTimeout.java b/test/distributed/org/apache/cassandra/distributed/test/RepairCoordinatorTimeout.java
new file mode 100644
index 0000000..f523396
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/test/RepairCoordinatorTimeout.java
@@ -0,0 +1,67 @@
+package org.apache.cassandra.distributed.test;
+
+import java.time.Duration;
+
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import org.apache.cassandra.distributed.api.NodeToolResult;
+import org.apache.cassandra.distributed.test.DistributedRepairUtils.RepairParallelism;
+import org.apache.cassandra.distributed.test.DistributedRepairUtils.RepairType;
+import org.apache.cassandra.net.Verb;
+
+import static java.lang.String.format;
+import static org.apache.cassandra.distributed.test.DistributedRepairUtils.assertParentRepairFailedWithMessageContains;
+import static org.apache.cassandra.distributed.test.DistributedRepairUtils.assertParentRepairNotExist;
+import static org.apache.cassandra.distributed.test.DistributedRepairUtils.getRepairExceptions;
+import static org.apache.cassandra.utils.AssertUtil.assertTimeoutPreemptively;
+
+public abstract class RepairCoordinatorTimeout extends RepairCoordinatorBase
+{
+ public RepairCoordinatorTimeout(RepairType repairType, RepairParallelism parallelism, boolean withNotifications)
+ {
+ super(repairType, parallelism, withNotifications);
+ }
+
+ @Before
+ public void beforeTest()
+ {
+ CLUSTER.filters().reset();
+ }
+
+ @Test
+ public void prepareRPCTimeout()
+ {
+ String table = tableName("preparerpctimeout");
+ assertTimeoutPreemptively(Duration.ofMinutes(1), () -> {
+ CLUSTER.schemaChange(format("CREATE TABLE %s.%s (key text, value text, PRIMARY KEY (key))", KEYSPACE, table));
+ CLUSTER.verbs(Verb.PREPARE_MSG).drop();
+
+ long repairExceptions = getRepairExceptions(CLUSTER, 1);
+ NodeToolResult result = repair(1, KEYSPACE, table);
+ result.asserts()
+ .failure()
+ .errorContains("Did not get replies from all endpoints.");
+ if (withNotifications)
+ {
+ result.asserts()
+ .notificationContains(NodeToolResult.ProgressEventType.START, "Starting repair command")
+ .notificationContains(NodeToolResult.ProgressEventType.START, "repairing keyspace " + KEYSPACE + " with repair options")
+ .notificationContains(NodeToolResult.ProgressEventType.ERROR, "Did not get replies from all endpoints.")
+ .notificationContains(NodeToolResult.ProgressEventType.COMPLETE, "finished with error");
+ }
+
+ if (repairType != RepairType.PREVIEW)
+ {
+ assertParentRepairFailedWithMessageContains(CLUSTER, KEYSPACE, table, "Did not get replies from all endpoints.");
+ }
+ else
+ {
+ assertParentRepairNotExist(CLUSTER, KEYSPACE, table);
+ }
+
+ Assert.assertEquals(repairExceptions + 1, getRepairExceptions(CLUSTER, 1));
+ });
+ }
+}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/RepairDigestTrackingTest.java b/test/distributed/org/apache/cassandra/distributed/test/RepairDigestTrackingTest.java
new file mode 100644
index 0000000..308702a
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/test/RepairDigestTrackingTest.java
@@ -0,0 +1,477 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test;
+
+import java.io.IOException;
+import java.time.LocalDate;
+import java.time.format.DateTimeFormatter;
+import java.util.EnumSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.IntStream;
+import java.util.stream.Stream;
+
+import com.google.common.util.concurrent.Uninterruptibles;
+import org.junit.Assert;
+import org.junit.Test;
+
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.distributed.Cluster;
+import org.apache.cassandra.distributed.api.ConsistencyLevel;
+import org.apache.cassandra.distributed.api.IInvokableInstance;
+import org.apache.cassandra.distributed.api.IIsolatedExecutor;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.metadata.MetadataComponent;
+import org.apache.cassandra.io.sstable.metadata.MetadataType;
+import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
+import org.apache.cassandra.service.ActiveRepairService;
+import org.apache.cassandra.service.SnapshotVerbHandler;
+import org.apache.cassandra.service.StorageProxy;
+import org.apache.cassandra.utils.DiagnosticSnapshotService;
+
+import static org.apache.cassandra.distributed.shared.AssertUtils.assertRows;
+import static org.junit.Assert.fail;
+
+public class RepairDigestTrackingTest extends TestBaseImpl
+{
+ private static final String TABLE = "tbl";
+ private static final String KS_TABLE = KEYSPACE + "." + TABLE;
+
+ @Test
+ public void testInconsistenciesFound() throws Throwable
+ {
+ try (Cluster cluster = (Cluster) init(builder().withNodes(2).start()))
+ {
+
+ cluster.get(1).runOnInstance(() -> {
+ StorageProxy.instance.enableRepairedDataTrackingForRangeReads();
+ });
+
+ cluster.schemaChange("CREATE TABLE " + KS_TABLE+ " (k INT, c INT, v INT, PRIMARY KEY (k,c)) with read_repair='NONE'");
+ for (int i = 0; i < 10; i++)
+ {
+ cluster.coordinator(1).execute("INSERT INTO " + KS_TABLE + " (k, c, v) VALUES (?, ?, ?)",
+ ConsistencyLevel.ALL,
+ i, i, i);
+ }
+ cluster.forEach(i -> i.flush(KEYSPACE));
+
+ for (int i = 10; i < 20; i++)
+ {
+ cluster.coordinator(1).execute("INSERT INTO " + KS_TABLE + " (k, c, v) VALUES (?, ?, ?)",
+ ConsistencyLevel.ALL,
+ i, i, i);
+ }
+ cluster.forEach(i -> i.flush(KEYSPACE));
+ cluster.forEach(i -> i.runOnInstance(assertNotRepaired()));
+
+ // mark everything on node 2 repaired
+ cluster.get(2).runOnInstance(markAllRepaired());
+ cluster.get(2).runOnInstance(assertRepaired());
+
+ // insert more data on node1 to generate an initial mismatch
+ cluster.get(1).executeInternal("INSERT INTO " + KS_TABLE + " (k, c, v) VALUES (?, ?, ?)", 5, 5, 55);
+ cluster.get(1).runOnInstance(assertNotRepaired());
+
+ long ccBefore = getConfirmedInconsistencies(cluster.get(1));
+ cluster.coordinator(1).execute("SELECT * FROM " + KS_TABLE, ConsistencyLevel.ALL);
+ long ccAfter = getConfirmedInconsistencies(cluster.get(1));
+ Assert.assertEquals("confirmed count should differ by 1 after range read", ccBefore + 1, ccAfter);
+ }
+ }
+
+ @Test
+ public void testPurgeableTombstonesAreIgnored() throws Throwable
+ {
+ try (Cluster cluster = (Cluster) init(builder().withNodes(2).start()))
+ {
+ cluster.get(1).runOnInstance(() -> {
+ StorageProxy.instance.enableRepairedDataTrackingForRangeReads();
+ });
+
+ cluster.schemaChange("CREATE TABLE " + KS_TABLE + " (k INT, c INT, v1 INT, v2 INT, PRIMARY KEY (k,c)) WITH gc_grace_seconds=0");
+ // on node1 only insert some tombstones, then flush
+ for (int i = 0; i < 10; i++)
+ {
+ cluster.get(1).executeInternal("DELETE v1 FROM " + KS_TABLE + " USING TIMESTAMP 0 WHERE k=? and c=? ", i, i);
+ }
+ cluster.get(1).flush(KEYSPACE);
+
+ // insert data on both nodes and flush
+ for (int i = 0; i < 10; i++)
+ {
+ cluster.coordinator(1).execute("INSERT INTO " + KS_TABLE + " (k, c, v2) VALUES (?, ?, ?) USING TIMESTAMP 1",
+ ConsistencyLevel.ALL,
+ i, i, i);
+ }
+ cluster.forEach(i -> i.flush(KEYSPACE));
+
+ // nothing is repaired yet
+ cluster.forEach(i -> i.runOnInstance(assertNotRepaired()));
+ // mark everything repaired
+ cluster.forEach(i -> i.runOnInstance(markAllRepaired()));
+ cluster.forEach(i -> i.runOnInstance(assertRepaired()));
+
+ // now overwrite on node2 only to generate digest mismatches, but don't flush so the repaired dataset is not affected
+ for (int i = 0; i < 10; i++)
+ {
+ cluster.get(2).executeInternal("INSERT INTO " + KS_TABLE + " (k, c, v2) VALUES (?, ?, ?) USING TIMESTAMP 2", i, i, i * 2);
+ }
+
+ long ccBefore = getConfirmedInconsistencies(cluster.get(1));
+ // Unfortunately we need to sleep here to ensure that nowInSec > the local deletion time of the tombstones
+ TimeUnit.SECONDS.sleep(2);
+ cluster.coordinator(1).execute("SELECT * FROM " + KS_TABLE, ConsistencyLevel.ALL);
+ long ccAfter = getConfirmedInconsistencies(cluster.get(1));
+
+ Assert.assertEquals("No repaired data inconsistencies should be detected", ccBefore, ccAfter);
+ }
+ }
+
+ @Test
+ public void testSnapshottingOnInconsistency() throws Throwable
+ {
+ try (Cluster cluster = init(Cluster.create(2)))
+ {
+ cluster.get(1).runOnInstance(() -> {
+ StorageProxy.instance.enableRepairedDataTrackingForPartitionReads();
+ });
+
+ cluster.schemaChange("CREATE TABLE " + KS_TABLE + " (k INT, c INT, v INT, PRIMARY KEY (k,c))");
+ for (int i = 0; i < 10; i++)
+ {
+ cluster.coordinator(1).execute("INSERT INTO " + KS_TABLE + " (k, c, v) VALUES (0, ?, ?)",
+ ConsistencyLevel.ALL, i, i);
+ }
+ cluster.forEach(c -> c.flush(KEYSPACE));
+
+ for (int i = 10; i < 20; i++)
+ {
+ cluster.coordinator(1).execute("INSERT INTO " + KS_TABLE + " (k, c, v) VALUES (0, ?, ?)",
+ ConsistencyLevel.ALL, i, i);
+ }
+ cluster.forEach(c -> c.flush(KEYSPACE));
+ cluster.forEach(i -> i.runOnInstance(assertNotRepaired()));
+ // Mark everything repaired on node2
+ cluster.get(2).runOnInstance(markAllRepaired());
+ cluster.get(2).runOnInstance(assertRepaired());
+
+ // now overwrite on node1 only to generate digest mismatches
+ cluster.get(1).executeInternal("INSERT INTO " + KS_TABLE + " (k, c, v) VALUES (0, ?, ?)", 5, 55);
+ cluster.get(1).runOnInstance(assertNotRepaired());
+
+ // Execute a partition read and assert inconsistency is detected (as nothing is repaired on node1)
+ long ccBefore = getConfirmedInconsistencies(cluster.get(1));
+ cluster.coordinator(1).execute("SELECT * FROM " + KS_TABLE + " WHERE k=0", ConsistencyLevel.ALL);
+ long ccAfter = getConfirmedInconsistencies(cluster.get(1));
+ Assert.assertEquals("confirmed count should increment by 1 after each partition read", ccBefore + 1, ccAfter);
+
+ String snapshotName = DiagnosticSnapshotService.getSnapshotName(DiagnosticSnapshotService.REPAIRED_DATA_MISMATCH_SNAPSHOT_PREFIX);
+
+ cluster.forEach(i -> i.runOnInstance(assertSnapshotNotPresent(snapshotName)));
+
+ // re-introduce a mismatch, enable snapshotting and try again
+ cluster.get(1).executeInternal("INSERT INTO " + KS_TABLE + " (k, c, v) VALUES (0, ?, ?)", 5, 555);
+ cluster.get(1).runOnInstance(() -> {
+ StorageProxy.instance.enableSnapshotOnRepairedDataMismatch();
+ });
+
+ cluster.coordinator(1).execute("SELECT * FROM " + KS_TABLE + " WHERE k=0", ConsistencyLevel.ALL);
+ ccAfter = getConfirmedInconsistencies(cluster.get(1));
+ Assert.assertEquals("confirmed count should increment by 1 after each partition read", ccBefore + 2, ccAfter);
+
+ cluster.forEach(i -> i.runOnInstance(assertSnapshotPresent(snapshotName)));
+ }
+ }
+
+ @Test
+ public void testRepairedReadCountNormalizationWithInitialUnderread() throws Throwable
+ {
+ // Asserts that the amount of repaired data read for digest generation is consistent
+ // across replicas where one has to read less repaired data to satisfy the original
+ // limits of the read request.
+ try (Cluster cluster = init(Cluster.create(2)))
+ {
+
+ cluster.get(1).runOnInstance(() -> {
+ StorageProxy.instance.enableRepairedDataTrackingForRangeReads();
+ StorageProxy.instance.enableRepairedDataTrackingForPartitionReads();
+ });
+
+ cluster.schemaChange("CREATE TABLE " + KS_TABLE + " (k INT, c INT, v1 INT, PRIMARY KEY (k,c)) " +
+ "WITH CLUSTERING ORDER BY (c DESC)");
+
+ // insert data on both nodes and flush
+ for (int i=0; i<20; i++)
+ {
+ cluster.coordinator(1).execute("INSERT INTO " + KS_TABLE + " (k, c, v1) VALUES (0, ?, ?) USING TIMESTAMP 0",
+ ConsistencyLevel.ALL, i, i);
+ cluster.coordinator(1).execute("INSERT INTO " + KS_TABLE + " (k, c, v1) VALUES (1, ?, ?) USING TIMESTAMP 1",
+ ConsistencyLevel.ALL, i, i);
+ }
+ cluster.forEach(c -> c.flush(KEYSPACE));
+ // nothing is repaired yet
+ cluster.forEach(i -> i.runOnInstance(assertNotRepaired()));
+ // mark everything repaired
+ cluster.forEach(i -> i.runOnInstance(markAllRepaired()));
+ cluster.forEach(i -> i.runOnInstance(assertRepaired()));
+
+ // Add some unrepaired data to both nodes
+ for (int i=20; i<30; i++)
+ {
+ cluster.coordinator(1).execute("INSERT INTO " + KS_TABLE + " (k, c, v1) VALUES (1, ?, ?) USING TIMESTAMP 1",
+ ConsistencyLevel.ALL, i, i);
+ }
+ // And some more unrepaired data to node2 only. This causes node2 to read less repaired data than node1
+ // when satisfying the limits of the read. So node2 needs to overread more repaired data than node1 when
+ // calculating the repaired data digest.
+ cluster.get(2).executeInternal("INSERT INTO " + KS_TABLE + " (k, c, v1) VALUES (1, ?, ?) USING TIMESTAMP 1", 30, 30);
+
+ // Verify single partition read
+ long ccBefore = getConfirmedInconsistencies(cluster.get(1));
+ assertRows(cluster.coordinator(1).execute("SELECT * FROM " + KS_TABLE + " WHERE k=1 LIMIT 20", ConsistencyLevel.ALL),
+ rows(1, 30, 11));
+ long ccAfterPartitionRead = getConfirmedInconsistencies(cluster.get(1));
+
+ // Recreate a mismatch in unrepaired data and verify partition range read
+ cluster.get(2).executeInternal("INSERT INTO " + KS_TABLE + " (k, c, v1) VALUES (1, ?, ?)", 31, 31);
+ assertRows(cluster.coordinator(1).execute("SELECT * FROM " + KS_TABLE + " LIMIT 30", ConsistencyLevel.ALL),
+ rows(1, 31, 2));
+ long ccAfterRangeRead = getConfirmedInconsistencies(cluster.get(1));
+
+ if (ccAfterPartitionRead != ccAfterRangeRead)
+ if (ccAfterPartitionRead != ccBefore)
+ fail("Both range and partition reads reported data inconsistencies but none were expected");
+ else
+ fail("Reported inconsistency during range read but none were expected");
+ else if (ccAfterPartitionRead != ccBefore)
+ fail("Reported inconsistency during partition read but none were expected");
+ }
+ }
+
+ @Test
+ public void testRepairedReadCountNormalizationWithInitialOverread() throws Throwable
+ {
+ // Asserts that the amount of repaired data read for digest generation is consistent
+ // across replicas where one has to read more repaired data to satisfy the original
+ // limits of the read request.
+ try (Cluster cluster = init(Cluster.create(2)))
+ {
+
+ cluster.get(1).runOnInstance(() -> {
+ StorageProxy.instance.enableRepairedDataTrackingForRangeReads();
+ StorageProxy.instance.enableRepairedDataTrackingForPartitionReads();
+ });
+
+ cluster.schemaChange("CREATE TABLE " + KS_TABLE + " (k INT, c INT, v1 INT, PRIMARY KEY (k,c)) " +
+ "WITH CLUSTERING ORDER BY (c DESC)");
+
+ // insert data on both nodes and flush
+ for (int i=0; i<10; i++)
+ {
+ cluster.coordinator(1).execute("INSERT INTO " + KS_TABLE + " (k, c, v1) VALUES (0, ?, ?) USING TIMESTAMP 0",
+ ConsistencyLevel.ALL, i, i);
+ cluster.coordinator(1).execute("INSERT INTO " + KS_TABLE + " (k, c, v1) VALUES (1, ?, ?) USING TIMESTAMP 1",
+ ConsistencyLevel.ALL, i, i);
+ }
+ cluster.forEach(c -> c.flush(KEYSPACE));
+ // nothing is repaired yet
+ cluster.forEach(i -> i.runOnInstance(assertNotRepaired()));
+ // mark everything repaired
+ cluster.forEach(i -> i.runOnInstance(markAllRepaired()));
+ cluster.forEach(i -> i.runOnInstance(assertRepaired()));
+
+ // Add some unrepaired data to both nodes
+ for (int i=10; i<13; i++)
+ {
+ cluster.coordinator(1).execute("INSERT INTO " + KS_TABLE + " (k, c, v1) VALUES (0, ?, ?) USING TIMESTAMP 1",
+ ConsistencyLevel.ALL, i, i);
+ cluster.coordinator(1).execute("INSERT INTO " + KS_TABLE + " (k, c, v1) VALUES (1, ?, ?) USING TIMESTAMP 1",
+ ConsistencyLevel.ALL, i, i);
+ }
+ cluster.forEach(c -> c.flush(KEYSPACE));
+ // And some row deletions on node2 only which cover data in the repaired set
+ // This will cause node2 to read more repaired data in satisfying the limit of the read request
+ // so it should overread less than node1 (in fact, it should not overread at all) in order to
+ // calculate the repaired data digest.
+ for (int i=7; i<10; i++)
+ {
+ cluster.get(2).executeInternal("DELETE FROM " + KS_TABLE + " USING TIMESTAMP 2 WHERE k = 0 AND c = ?", i);
+ cluster.get(2).executeInternal("DELETE FROM " + KS_TABLE + " USING TIMESTAMP 2 WHERE k = 1 AND c = ?", i);
+ }
+
+ // Verify single partition read
+ long ccBefore = getConfirmedInconsistencies(cluster.get(1));
+ assertRows(cluster.coordinator(1).execute("SELECT * FROM " + KS_TABLE + " WHERE k=0 LIMIT 5", ConsistencyLevel.ALL),
+ rows(rows(0, 12, 10), rows(0, 6, 5)));
+ long ccAfterPartitionRead = getConfirmedInconsistencies(cluster.get(1));
+
+ // Verify partition range read
+ assertRows(cluster.coordinator(1).execute("SELECT * FROM " + KS_TABLE + " LIMIT 11", ConsistencyLevel.ALL),
+ rows(rows(1, 12, 10), rows(1, 6, 0), rows(0, 12, 12)));
+ long ccAfterRangeRead = getConfirmedInconsistencies(cluster.get(1));
+
+ if (ccAfterPartitionRead != ccAfterRangeRead)
+ if (ccAfterPartitionRead != ccBefore)
+ fail("Both range and partition reads reported data inconsistencies but none were expected");
+ else
+ fail("Reported inconsistency during range read but none were expected");
+ else if (ccAfterPartitionRead != ccBefore)
+ fail("Reported inconsistency during partition read but none were expected");
+ }
+ }
+
+ private Object[][] rows(Object[][] head, Object[][]...tail)
+ {
+ return Stream.concat(Stream.of(head),
+ Stream.of(tail).flatMap(Stream::of))
+ .toArray(Object[][]::new);
+ }
+
+ private Object[][] rows(int partitionKey, int start, int end)
+ {
+ if (start == end)
+ return new Object[][] { new Object[] { partitionKey, start, end } };
+
+ IntStream clusterings = start > end
+ ? IntStream.range(end -1, start).map(i -> start - i + end - 1)
+ : IntStream.range(start, end);
+
+ return clusterings.mapToObj(i -> new Object[] {partitionKey, i, i}).toArray(Object[][]::new);
+ }
+
+ private IIsolatedExecutor.SerializableRunnable assertNotRepaired()
+ {
+ return () ->
+ {
+ try
+ {
+ Iterator<SSTableReader> sstables = Keyspace.open(KEYSPACE)
+ .getColumnFamilyStore(TABLE)
+ .getLiveSSTables()
+ .iterator();
+ while (sstables.hasNext())
+ {
+ SSTableReader sstable = sstables.next();
+ Descriptor descriptor = sstable.descriptor;
+ Map<MetadataType, MetadataComponent> metadata = descriptor.getMetadataSerializer()
+ .deserialize(descriptor, EnumSet.of(MetadataType.STATS));
+
+ StatsMetadata stats = (StatsMetadata) metadata.get(MetadataType.STATS);
+ Assert.assertEquals("repaired at is set for sstable: " + descriptor,
+ stats.repairedAt,
+ ActiveRepairService.UNREPAIRED_SSTABLE);
+ }
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ };
+ }
+
+ private IIsolatedExecutor.SerializableRunnable markAllRepaired()
+ {
+ return () ->
+ {
+ try
+ {
+ Iterator<SSTableReader> sstables = Keyspace.open(KEYSPACE)
+ .getColumnFamilyStore(TABLE)
+ .getLiveSSTables()
+ .iterator();
+ while (sstables.hasNext())
+ {
+ SSTableReader sstable = sstables.next();
+ Descriptor descriptor = sstable.descriptor;
+ descriptor.getMetadataSerializer()
+ .mutateRepairMetadata(descriptor, System.currentTimeMillis(), null, false);
+ sstable.reloadSSTableMetadata();
+ }
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ };
+ }
+
+ private IIsolatedExecutor.SerializableRunnable assertRepaired()
+ {
+ return () ->
+ {
+ try
+ {
+ Iterator<SSTableReader> sstables = Keyspace.open(KEYSPACE)
+ .getColumnFamilyStore(TABLE)
+ .getLiveSSTables()
+ .iterator();
+ while (sstables.hasNext())
+ {
+ SSTableReader sstable = sstables.next();
+ Descriptor descriptor = sstable.descriptor;
+ Map<MetadataType, MetadataComponent> metadata = descriptor.getMetadataSerializer()
+ .deserialize(descriptor, EnumSet.of(MetadataType.STATS));
+
+ StatsMetadata stats = (StatsMetadata) metadata.get(MetadataType.STATS);
+ Assert.assertTrue("repaired at is not set for sstable: " + descriptor, stats.repairedAt > 0);
+ }
+ }
+ catch (IOException e)
+ {
+ throw new RuntimeException(e);
+ }
+ };
+ }
+
+ private IInvokableInstance.SerializableRunnable assertSnapshotPresent(String snapshotName)
+ {
+ return () ->
+ {
+ // snapshots are taken asynchronously, this is crude but it gives it a chance to happen
+ int attempts = 100;
+ ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(TABLE);
+
+ while (cfs.getSnapshotDetails().isEmpty())
+ {
+ Uninterruptibles.sleepUninterruptibly(100, TimeUnit.MILLISECONDS);
+ if (attempts-- < 0)
+ throw new AssertionError(String.format("Snapshot %s not found for for %s", snapshotName, KS_TABLE));
+ }
+ };
+ }
+
+ private IInvokableInstance.SerializableRunnable assertSnapshotNotPresent(String snapshotName)
+ {
+ return () ->
+ {
+ ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(TABLE);
+ Assert.assertFalse(cfs.snapshotExists(snapshotName));
+ };
+ }
+
+ private long getConfirmedInconsistencies(IInvokableInstance instance)
+ {
+ return instance.callOnInstance(() -> Keyspace.open(KEYSPACE)
+ .getColumnFamilyStore(TABLE)
+ .metric
+ .confirmedRepairedInconsistencies
+ .table
+ .getCount());
+ }
+}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/RepairTest.java b/test/distributed/org/apache/cassandra/distributed/test/RepairTest.java
new file mode 100644
index 0000000..f37a3d8
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/test/RepairTest.java
@@ -0,0 +1,185 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Map;
+import java.util.function.Consumer;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.distributed.Cluster;
+import org.apache.cassandra.distributed.api.ICluster;
+import org.apache.cassandra.distributed.api.IInstanceConfig;
+import org.apache.cassandra.distributed.api.IInvokableInstance;
+import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.utils.concurrent.SimpleCondition;
+import org.apache.cassandra.utils.progress.ProgressEventType;
+
+import static java.util.concurrent.TimeUnit.MINUTES;
+import static org.apache.cassandra.distributed.test.ExecUtil.rethrow;
+import static org.apache.cassandra.distributed.api.Feature.GOSSIP;
+import static org.apache.cassandra.distributed.api.Feature.NETWORK;
+import static org.apache.cassandra.distributed.shared.AssertUtils.*;
+
+public class RepairTest extends TestBaseImpl
+{
+ private static final String insert = withKeyspace("INSERT INTO %s.test (k, c1, c2) VALUES (?, 'value1', 'value2');");
+ private static final String query = withKeyspace("SELECT k, c1, c2 FROM %s.test WHERE k = ?;");
+
+ private static ICluster<IInvokableInstance> cluster;
+
+ private static void insert(ICluster<IInvokableInstance> cluster, int start, int end, int ... nodes)
+ {
+ for (int i = start ; i < end ; ++i)
+ for (int node : nodes)
+ cluster.get(node).executeInternal(insert, Integer.toString(i));
+ }
+
+ private static void verify(ICluster<IInvokableInstance> cluster, int start, int end, int ... nodes)
+ {
+ for (int i = start ; i < end ; ++i)
+ {
+ for (int node = 1 ; node <= cluster.size() ; ++node)
+ {
+ Object[][] rows = cluster.get(node).executeInternal(query, Integer.toString(i));
+ if (Arrays.binarySearch(nodes, node) >= 0)
+ assertRows(rows, new Object[] { Integer.toString(i), "value1", "value2" });
+ else
+ assertRows(rows);
+ }
+ }
+ }
+
+ private static void flush(ICluster<IInvokableInstance> cluster, int ... nodes)
+ {
+ for (int node : nodes)
+ cluster.get(node).runOnInstance(rethrow(() -> StorageService.instance.forceKeyspaceFlush(KEYSPACE)));
+ }
+
+ private static ICluster create(Consumer<IInstanceConfig> configModifier) throws IOException
+ {
+ configModifier = configModifier.andThen(
+ config -> config.set("hinted_handoff_enabled", false)
+ .set("commitlog_sync_batch_window_in_ms", 5)
+ .with(NETWORK)
+ .with(GOSSIP)
+ );
+
+ return init(Cluster.build().withNodes(3).withConfig(configModifier).start());
+ }
+
+ private void repair(ICluster<IInvokableInstance> cluster, Map<String, String> options)
+ {
+ cluster.get(1).runOnInstance(rethrow(() -> {
+ SimpleCondition await = new SimpleCondition();
+ StorageService.instance.repair(KEYSPACE, options, ImmutableList.of((tag, event) -> {
+ if (event.getType() == ProgressEventType.COMPLETE)
+ await.signalAll();
+ })).right.get();
+ await.await(1L, MINUTES);
+ }));
+ }
+
+ void populate(ICluster<IInvokableInstance> cluster, String compression) throws Exception
+ {
+ try
+ {
+ cluster.schemaChange(withKeyspace("DROP TABLE IF EXISTS %s.test;"));
+ cluster.schemaChange(withKeyspace("CREATE TABLE %s.test (k text, c1 text, c2 text, PRIMARY KEY (k)) WITH compression = " + compression));
+
+ insert(cluster, 0, 1000, 1, 2, 3);
+ flush(cluster, 1);
+ insert(cluster, 1000, 1001, 1, 2);
+ insert(cluster, 1001, 2001, 1, 2, 3);
+ flush(cluster, 1, 2, 3);
+
+ verify(cluster, 0, 1000, 1, 2, 3);
+ verify(cluster, 1000, 1001, 1, 2);
+ verify(cluster, 1001, 2001, 1, 2, 3);
+ }
+ catch (Throwable t)
+ {
+ cluster.close();
+ throw t;
+ }
+
+ }
+
+ void repair(ICluster<IInvokableInstance> cluster, boolean sequential, String compression) throws Exception
+ {
+ populate(cluster, compression);
+ repair(cluster, ImmutableMap.of("parallelism", sequential ? "sequential" : "parallel"));
+ verify(cluster, 0, 2001, 1, 2, 3);
+ }
+
+ @BeforeClass
+ public static void setupCluster() throws IOException
+ {
+ cluster = create(config -> {});
+ }
+
+ @AfterClass
+ public static void closeCluster() throws Exception
+ {
+ if (cluster != null)
+ cluster.close();
+ }
+
+ @Test
+ public void testSequentialRepairWithDefaultCompression() throws Exception
+ {
+ repair(cluster, true, "{'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}");
+ }
+
+ @Test
+ public void testParallelRepairWithDefaultCompression() throws Exception
+ {
+ repair(cluster, false, "{'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}");
+ }
+
+ @Test
+ public void testSequentialRepairWithMinCompressRatio() throws Exception
+ {
+ repair(cluster, true, "{'class': 'org.apache.cassandra.io.compress.LZ4Compressor', 'min_compress_ratio': 4.0}");
+ }
+
+ @Test
+ public void testParallelRepairWithMinCompressRatio() throws Exception
+ {
+ repair(cluster, false, "{'class': 'org.apache.cassandra.io.compress.LZ4Compressor', 'min_compress_ratio': 4.0}");
+ }
+
+ @Test
+ public void testSequentialRepairWithoutCompression() throws Exception
+ {
+ repair(cluster, true, "{'enabled': false}");
+ }
+
+ @Test
+ public void testParallelRepairWithoutCompression() throws Exception
+ {
+ repair(cluster, false, "{'enabled': false}");
+ }
+}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/ResourceLeakTest.java b/test/distributed/org/apache/cassandra/distributed/test/ResourceLeakTest.java
index 1c4850a..5430800 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/ResourceLeakTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/ResourceLeakTest.java
@@ -33,12 +33,9 @@
import org.junit.Test;
import com.sun.management.HotSpotDiagnosticMXBean;
-import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.distributed.Cluster;
import org.apache.cassandra.distributed.api.ConsistencyLevel;
import org.apache.cassandra.distributed.api.IInstanceConfig;
-import org.apache.cassandra.gms.Gossiper;
-import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.SigarLibrary;
import static org.apache.cassandra.distributed.api.Feature.GOSSIP;
@@ -145,16 +142,13 @@
for (int loop = 0; loop < numTestLoops; loop++)
{
System.out.println(String.format("========== Starting loop %03d ========", loop));
- try (Cluster cluster = Cluster.build(numClusterNodes).withConfig(updater).start())
+ try (Cluster cluster = (Cluster) builder().withNodes(numClusterNodes).withConfig(updater).start())
{
- if (cluster.get(1).config().has(GOSSIP)) // Wait for gossip to settle on the seed node
- cluster.get(1).runOnInstance(() -> Gossiper.waitToSettle());
-
init(cluster);
String tableName = "tbl" + loop;
cluster.schemaChange("CREATE TABLE " + KEYSPACE + "." + tableName + " (pk int, ck int, v int, PRIMARY KEY (pk, ck))");
cluster.coordinator(1).execute("INSERT INTO " + KEYSPACE + "." + tableName + "(pk,ck,v) VALUES (0,0,0)", ConsistencyLevel.ALL);
- cluster.get(1).callOnInstance(() -> FBUtilities.waitOnFutures(Keyspace.open(KEYSPACE).flush()));
+ cluster.get(1).flush(KEYSPACE);
if (dumpEveryLoop)
{
dumpResources(String.format("loop%03d", loop));
diff --git a/test/distributed/org/apache/cassandra/distributed/test/SharedClusterTestBase.java b/test/distributed/org/apache/cassandra/distributed/test/SharedClusterTestBase.java
deleted file mode 100644
index c502af2..0000000
--- a/test/distributed/org/apache/cassandra/distributed/test/SharedClusterTestBase.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.distributed.test;
-
-import java.io.IOException;
-
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-
-import org.apache.cassandra.distributed.Cluster;
-import org.apache.cassandra.distributed.api.ICluster;
-
-public class SharedClusterTestBase extends TestBaseImpl
-{
- protected static ICluster cluster;
-
- @BeforeClass
- public static void before() throws IOException
- {
- cluster = init(Cluster.build().withNodes(3).start());
- }
-
- @AfterClass
- public static void after() throws Exception
- {
- cluster.close();
- }
-
- @After
- public void afterEach()
- {
- cluster.schemaChange("DROP KEYSPACE IF EXISTS " + KEYSPACE);
- init(cluster);
- }
-}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/SimpleReadWriteTest.java b/test/distributed/org/apache/cassandra/distributed/test/SimpleReadWriteTest.java
index 75e5ba9..a547c76 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/SimpleReadWriteTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/SimpleReadWriteTest.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.cassandra.distributed.test;
import java.util.Set;
@@ -5,207 +23,325 @@
import org.junit.Assert;
import org.junit.Test;
+import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.distributed.Cluster;
import org.apache.cassandra.distributed.api.ConsistencyLevel;
import org.apache.cassandra.distributed.api.ICluster;
import org.apache.cassandra.distributed.api.IInvokableInstance;
+import org.apache.cassandra.metrics.ReadRepairMetrics;
import org.apache.cassandra.io.sstable.format.SSTableReader;
-import static org.junit.Assert.assertEquals;
-
+import static org.apache.cassandra.distributed.api.Feature.NETWORK;
import static org.apache.cassandra.distributed.shared.AssertUtils.*;
+import static org.apache.cassandra.net.OutboundConnections.LARGE_MESSAGE_THRESHOLD;
+import static org.apache.cassandra.net.Verb.READ_REPAIR_REQ;
+import static org.apache.cassandra.net.Verb.READ_REPAIR_RSP;
+import static org.junit.Assert.fail;
// TODO: this test should be removed after running in-jvm dtests is set up via the shared API repository
-public class SimpleReadWriteTest extends SharedClusterTestBase
+public class SimpleReadWriteTest extends TestBaseImpl
{
@Test
public void coordinatorReadTest() throws Throwable
{
- cluster.schemaChange("CREATE TABLE " + KEYSPACE + ".tbl (pk int, ck int, v int, PRIMARY KEY (pk, ck))");
+ try (ICluster cluster = init(builder().withNodes(3).start()))
+ {
+ cluster.schemaChange("CREATE TABLE " + KEYSPACE + ".tbl (pk int, ck int, v int, PRIMARY KEY (pk, ck)) WITH read_repair='none'");
- cluster.get(1).executeInternal("INSERT INTO " + KEYSPACE + ".tbl (pk, ck, v) VALUES (1, 1, 1)");
- cluster.get(2).executeInternal("INSERT INTO " + KEYSPACE + ".tbl (pk, ck, v) VALUES (1, 2, 2)");
- cluster.get(3).executeInternal("INSERT INTO " + KEYSPACE + ".tbl (pk, ck, v) VALUES (1, 3, 3)");
+ cluster.get(1).executeInternal("INSERT INTO " + KEYSPACE + ".tbl (pk, ck, v) VALUES (1, 1, 1)");
+ cluster.get(2).executeInternal("INSERT INTO " + KEYSPACE + ".tbl (pk, ck, v) VALUES (1, 2, 2)");
+ cluster.get(3).executeInternal("INSERT INTO " + KEYSPACE + ".tbl (pk, ck, v) VALUES (1, 3, 3)");
- assertRows(cluster.coordinator(1).execute("SELECT * FROM " + KEYSPACE + ".tbl WHERE pk = ?",
- ConsistencyLevel.ALL,
- 1),
- row(1, 1, 1),
- row(1, 2, 2),
- row(1, 3, 3));
+ assertRows(cluster.coordinator(1).execute("SELECT * FROM " + KEYSPACE + ".tbl WHERE pk = ?",
+ ConsistencyLevel.ALL,
+ 1),
+ row(1, 1, 1),
+ row(1, 2, 2),
+ row(1, 3, 3));
+ }
}
@Test
public void largeMessageTest() throws Throwable
{
- int largeMessageThreshold = 1024 * 64;
- cluster.schemaChange("CREATE TABLE " + KEYSPACE + ".tbl (pk int, ck int, v text, PRIMARY KEY (pk, ck))");
- StringBuilder builder = new StringBuilder();
- for (int i = 0; i < largeMessageThreshold; i++)
- builder.append('a');
- String s = builder.toString();
- cluster.coordinator(1).execute("INSERT INTO " + KEYSPACE + ".tbl (pk, ck, v) VALUES (1, 1, ?)",
- ConsistencyLevel.ALL,
- s);
- assertRows(cluster.coordinator(1).execute("SELECT * FROM " + KEYSPACE + ".tbl WHERE pk = ?",
- ConsistencyLevel.ALL,
- 1),
- row(1, 1, s));
+ try (ICluster cluster = init(builder().withNodes(2).start()))
+ {
+ cluster.schemaChange("CREATE TABLE " + KEYSPACE + ".tbl (pk int, ck int, v text, PRIMARY KEY (pk, ck))");
+ StringBuilder builder = new StringBuilder();
+ for (int i = 0; i < LARGE_MESSAGE_THRESHOLD ; i++)
+ builder.append('a');
+ String s = builder.toString();
+ cluster.coordinator(1).execute("INSERT INTO " + KEYSPACE + ".tbl (pk, ck, v) VALUES (1, 1, ?)",
+ ConsistencyLevel.ALL,
+ s);
+ assertRows(cluster.coordinator(1).execute("SELECT * FROM " + KEYSPACE + ".tbl WHERE pk = ?",
+ ConsistencyLevel.ALL,
+ 1),
+ row(1, 1, s));
+ }
}
@Test
public void coordinatorWriteTest() throws Throwable
{
- cluster.schemaChange("CREATE TABLE " + KEYSPACE + ".tbl (pk int, ck int, v int, PRIMARY KEY (pk, ck))");
-
- cluster.coordinator(1).execute("INSERT INTO " + KEYSPACE + ".tbl (pk, ck, v) VALUES (1, 1, 1)",
- ConsistencyLevel.QUORUM);
-
- for (int i = 0; i < 3; i++)
+ try (ICluster cluster = init(builder().withNodes(3).start()))
{
- assertRows(cluster.get(1).executeInternal("SELECT * FROM " + KEYSPACE + ".tbl WHERE pk = 1"),
+ cluster.schemaChange("CREATE TABLE " + KEYSPACE + ".tbl (pk int, ck int, v int, PRIMARY KEY (pk, ck)) WITH read_repair='none'");
+
+ cluster.coordinator(1).execute("INSERT INTO " + KEYSPACE + ".tbl (pk, ck, v) VALUES (1, 1, 1)",
+ ConsistencyLevel.QUORUM);
+
+ for (int i = 0; i < 3; i++)
+ {
+ assertRows(cluster.get(1).executeInternal("SELECT * FROM " + KEYSPACE + ".tbl WHERE pk = 1"),
+ row(1, 1, 1));
+ }
+
+ assertRows(cluster.coordinator(1).execute("SELECT * FROM " + KEYSPACE + ".tbl WHERE pk = 1",
+ ConsistencyLevel.QUORUM),
row(1, 1, 1));
}
-
- assertRows(cluster.coordinator(1).execute("SELECT * FROM " + KEYSPACE + ".tbl WHERE pk = 1",
- ConsistencyLevel.QUORUM),
- row(1, 1, 1));
}
@Test
public void readRepairTest() throws Throwable
{
- cluster.schemaChange("CREATE TABLE " + KEYSPACE + ".tbl (pk int, ck int, v int, PRIMARY KEY (pk, ck))");
+ try (ICluster cluster = init(builder().withNodes(3).start()))
+ {
+ cluster.schemaChange("CREATE TABLE " + KEYSPACE + ".tbl (pk int, ck int, v int, PRIMARY KEY (pk, ck)) WITH read_repair='blocking'");
- cluster.get(1).executeInternal("INSERT INTO " + KEYSPACE + ".tbl (pk, ck, v) VALUES (1, 1, 1)");
- cluster.get(2).executeInternal("INSERT INTO " + KEYSPACE + ".tbl (pk, ck, v) VALUES (1, 1, 1)");
+ cluster.get(1).executeInternal("INSERT INTO " + KEYSPACE + ".tbl (pk, ck, v) VALUES (1, 1, 1)");
+ cluster.get(2).executeInternal("INSERT INTO " + KEYSPACE + ".tbl (pk, ck, v) VALUES (1, 1, 1)");
- assertRows(cluster.get(3).executeInternal("SELECT * FROM " + KEYSPACE + ".tbl WHERE pk = 1"));
+ assertRows(cluster.get(3).executeInternal("SELECT * FROM " + KEYSPACE + ".tbl WHERE pk = 1"));
- assertRows(cluster.coordinator(1).execute("SELECT * FROM " + KEYSPACE + ".tbl WHERE pk = 1",
- ConsistencyLevel.ALL), // ensure node3 in preflist
- row(1, 1, 1));
+ assertRows(cluster.coordinator(1).execute("SELECT * FROM " + KEYSPACE + ".tbl WHERE pk = 1",
+ ConsistencyLevel.ALL), // ensure node3 in preflist
+ row(1, 1, 1));
- // Verify that data got repaired to the third node
- assertRows(cluster.get(3).executeInternal("SELECT * FROM " + KEYSPACE + ".tbl WHERE pk = 1"),
- row(1, 1, 1));
+ // Verify that data got repaired to the third node
+ assertRows(cluster.get(3).executeInternal("SELECT * FROM " + KEYSPACE + ".tbl WHERE pk = 1"),
+ row(1, 1, 1));
+ }
+ }
+
+ @Test
+ public void readRepairTimeoutTest() throws Throwable
+ {
+ final long reducedReadTimeout = 3000L;
+ try (Cluster cluster = (Cluster) init(builder().withNodes(3).start()))
+ {
+ cluster.forEach(i -> i.runOnInstance(() -> DatabaseDescriptor.setReadRpcTimeout(reducedReadTimeout)));
+ cluster.schemaChange("CREATE TABLE " + KEYSPACE + ".tbl (pk int, ck int, v int, PRIMARY KEY (pk, ck)) WITH read_repair='blocking'");
+ cluster.get(1).executeInternal("INSERT INTO " + KEYSPACE + ".tbl (pk, ck, v) VALUES (1, 1, 1)");
+ cluster.get(2).executeInternal("INSERT INTO " + KEYSPACE + ".tbl (pk, ck, v) VALUES (1, 1, 1)");
+ assertRows(cluster.get(3).executeInternal("SELECT * FROM " + KEYSPACE + ".tbl WHERE pk = 1"));
+ cluster.verbs(READ_REPAIR_RSP).to(1).drop();
+ final long start = System.currentTimeMillis();
+ try
+ {
+ cluster.coordinator(1).execute("SELECT * FROM " + KEYSPACE + ".tbl WHERE pk = 1", ConsistencyLevel.ALL);
+ fail("Read timeout expected but it did not occur");
+ }
+ catch (Exception ex)
+ {
+ // the containing exception class was loaded by another class loader. Comparing the message as a workaround to assert the exception
+ Assert.assertTrue(ex.getClass().toString().contains("ReadTimeoutException"));
+ long actualTimeTaken = System.currentTimeMillis() - start;
+ long magicDelayAmount = 100L; // it might not be the best way to check if the time taken is around the timeout value.
+ // Due to the delays, the actual time taken from client perspective is slighly more than the timeout value
+ Assert.assertTrue(actualTimeTaken > reducedReadTimeout);
+ // But it should not exceed too much
+ Assert.assertTrue(actualTimeTaken < reducedReadTimeout + magicDelayAmount);
+ assertRows(cluster.get(3).executeInternal("SELECT * FROM " + KEYSPACE + ".tbl WHERE pk = 1"),
+ row(1, 1, 1)); // the partition happened when the repaired node sending back ack. The mutation should be in fact applied.
+ }
+ }
+ }
+
+ @Test
+ public void failingReadRepairTest() throws Throwable
+ {
+ // This test makes a explicit assumption about which nodes are read from; that 1 and 2 will be "contacts", and that 3 will be ignored.
+ // This is a implementation detail of org.apache.cassandra.locator.ReplicaPlans#contactForRead and
+ // org.apache.cassandra.locator.AbstractReplicationStrategy.getNaturalReplicasForToken that may change
+ // in a future release; when that happens this test could start to fail but should only fail with the explicit
+ // check that detects this behavior has changed.
+ // see CASSANDRA-15507
+ try (Cluster cluster = init(Cluster.create(3)))
+ {
+ cluster.schemaChange("CREATE TABLE " + KEYSPACE + ".tbl (pk int, ck int, v int, PRIMARY KEY (pk, ck)) WITH read_repair='blocking'");
+
+ // nodes 1 and 3 are identical
+ cluster.get(1).executeInternal("INSERT INTO " + KEYSPACE + ".tbl (pk, ck, v) VALUES (1, 1, 1) USING TIMESTAMP 43");
+ cluster.get(3).executeInternal("INSERT INTO " + KEYSPACE + ".tbl (pk, ck, v) VALUES (1, 1, 1) USING TIMESTAMP 43");
+
+ // node 2 is different because of the timestamp; a repair is needed
+ cluster.get(2).executeInternal("INSERT INTO " + KEYSPACE + ".tbl (pk, ck, v) VALUES (1, 1, 1) USING TIMESTAMP 42");
+
+ // 2 is out of date so needs to be repaired. This will make sure that the repair does not happen on the node
+ // which will trigger the coordinator to write to node 3
+ cluster.verbs(READ_REPAIR_REQ).to(2).drop();
+
+ // save the state of the counters so its known if the contacts list changed
+ long readRepairRequestsBefore = cluster.get(1).callOnInstance(() -> Keyspace.open(KEYSPACE).getColumnFamilyStore("tbl").metric.readRepairRequests.getCount());
+ long speculatedWriteBefore = cluster.get(1).callOnInstance(() -> ReadRepairMetrics.speculatedWrite.getCount());
+
+ Object[][] rows = cluster.coordinator(1)
+ .execute("SELECT pk, ck, v, WRITETIME(v) FROM " + KEYSPACE + ".tbl WHERE pk = 1", ConsistencyLevel.QUORUM);
+
+ // make sure to check counters first, so can detect if read repair executed as expected
+ long readRepairRequestsAfter = cluster.get(1).callOnInstance(() -> Keyspace.open(KEYSPACE).getColumnFamilyStore("tbl").metric.readRepairRequests.getCount());
+ long speculatedWriteAfter = cluster.get(1).callOnInstance(() -> ReadRepairMetrics.speculatedWrite.getCount());
+
+ // defensive checks to make sure the nodes selected are the ones expected
+ Assert.assertEquals("number of read repairs after query does not match expected; its possible the nodes involved with the query did not match expected", readRepairRequestsBefore + 1, readRepairRequestsAfter);
+ Assert.assertEquals("number of read repairs speculated writes after query does not match expected; its possible the nodes involved with the query did not match expected", speculatedWriteBefore + 1, speculatedWriteAfter);
+
+ // 1 has newer data than 2 so its write timestamp will be used for the result
+ assertRows(rows, row(1, 1, 1, 43L));
+
+ // cheack each node's local state
+ // 1 and 3 should match quorum response
+ assertRows(cluster.get(1).executeInternal("SELECT pk, ck, v, WRITETIME(v) FROM " + KEYSPACE + ".tbl WHERE pk = 1"), row(1, 1, 1, 43L));
+ assertRows(cluster.get(3).executeInternal("SELECT pk, ck, v, WRITETIME(v) FROM " + KEYSPACE + ".tbl WHERE pk = 1"), row(1, 1, 1, 43L));
+
+ // 2 was not repaired (message was dropped), so still has old data
+ assertRows(cluster.get(2).executeInternal("SELECT pk, ck, v, WRITETIME(v) FROM " + KEYSPACE + ".tbl WHERE pk = 1"), row(1, 1, 1, 42L));
+ }
}
@Test
public void writeWithSchemaDisagreement() throws Throwable
{
- cluster.schemaChange("CREATE TABLE " + KEYSPACE + ".tbl (pk int, ck int, v1 int, PRIMARY KEY (pk, ck))");
-
- cluster.get(1).executeInternal("INSERT INTO " + KEYSPACE + ".tbl (pk, ck, v1) VALUES (1, 1, 1)");
- cluster.get(2).executeInternal("INSERT INTO " + KEYSPACE + ".tbl (pk, ck, v1) VALUES (1, 1, 1)");
- cluster.get(3).executeInternal("INSERT INTO " + KEYSPACE + ".tbl (pk, ck, v1) VALUES (1, 1, 1)");
-
- // Introduce schema disagreement
- cluster.schemaChange("ALTER TABLE " + KEYSPACE + ".tbl ADD v2 int", 1);
-
- Exception thrown = null;
- try
+ try (ICluster cluster = init(builder().withNodes(3).withConfig(config -> config.with(NETWORK)).start()))
{
- cluster.coordinator(1).execute("INSERT INTO " + KEYSPACE + ".tbl (pk, ck, v1, v2) VALUES (2, 2, 2, 2)",
- ConsistencyLevel.QUORUM);
- }
- catch (RuntimeException e)
- {
- thrown = e;
- }
+ cluster.schemaChange("CREATE TABLE " + KEYSPACE + ".tbl (pk int, ck int, v1 int, PRIMARY KEY (pk, ck))");
- Assert.assertTrue(thrown.getMessage().contains("Exception occurred on node"));
- Assert.assertTrue(thrown.getCause().getCause().getCause().getMessage().contains("Unknown column v2 during deserialization"));
+ cluster.get(1).executeInternal("INSERT INTO " + KEYSPACE + ".tbl (pk, ck, v1) VALUES (1, 1, 1)");
+ cluster.get(2).executeInternal("INSERT INTO " + KEYSPACE + ".tbl (pk, ck, v1) VALUES (1, 1, 1)");
+ cluster.get(3).executeInternal("INSERT INTO " + KEYSPACE + ".tbl (pk, ck, v1) VALUES (1, 1, 1)");
+
+ // Introduce schema disagreement
+ cluster.schemaChange("ALTER TABLE " + KEYSPACE + ".tbl ADD v2 int", 1);
+
+ try
+ {
+ cluster.coordinator(1).execute("INSERT INTO " + KEYSPACE + ".tbl (pk, ck, v1, v2) VALUES (2, 2, 2, 2)",
+ ConsistencyLevel.QUORUM);
+ fail("Should have failed because of schema disagreement.");
+ }
+ catch (Exception e)
+ {
+ // for some reason, we get weird errors when trying to check class directly
+ // I suppose it has to do with some classloader manipulation going on
+ Assert.assertTrue(e.getClass().toString().contains("WriteFailureException"));
+ // we may see 1 or 2 failures in here, because of the fail-fast behavior of AbstractWriteResponseHandler
+ Assert.assertTrue(e.getMessage().contains("INCOMPATIBLE_SCHEMA from 127.0.0.2")
+ || e.getMessage().contains("INCOMPATIBLE_SCHEMA from 127.0.0.3"));
+
+ }
+ }
}
@Test
public void readWithSchemaDisagreement() throws Throwable
{
- cluster.schemaChange("CREATE TABLE " + KEYSPACE + ".tbl (pk int, ck int, v1 int, PRIMARY KEY (pk, ck))");
-
- cluster.get(1).executeInternal("INSERT INTO " + KEYSPACE + ".tbl (pk, ck, v1) VALUES (1, 1, 1)");
- cluster.get(2).executeInternal("INSERT INTO " + KEYSPACE + ".tbl (pk, ck, v1) VALUES (1, 1, 1)");
- cluster.get(3).executeInternal("INSERT INTO " + KEYSPACE + ".tbl (pk, ck, v1) VALUES (1, 1, 1)");
-
- // Introduce schema disagreement
- cluster.schemaChange("ALTER TABLE " + KEYSPACE + ".tbl ADD v2 int", 1);
-
- Exception thrown = null;
- try
+ try (ICluster cluster = init(builder().withNodes(3).withConfig(config -> config.with(NETWORK)).start()))
{
- assertRows(cluster.coordinator(1).execute("SELECT * FROM " + KEYSPACE + ".tbl WHERE pk = 1",
- ConsistencyLevel.ALL),
- row(1, 1, 1, null));
- }
- catch (Exception e)
- {
- thrown = e;
- }
+ cluster.schemaChange("CREATE TABLE " + KEYSPACE + ".tbl (pk int, ck int, v1 int, PRIMARY KEY (pk, ck))");
- Assert.assertTrue(thrown.getMessage().contains("Exception occurred on node"));
- Assert.assertTrue(thrown.getCause().getCause().getCause().getMessage().contains("Unknown column v2 during deserialization"));
+ cluster.get(1).executeInternal("INSERT INTO " + KEYSPACE + ".tbl (pk, ck, v1) VALUES (1, 1, 1)");
+ cluster.get(2).executeInternal("INSERT INTO " + KEYSPACE + ".tbl (pk, ck, v1) VALUES (1, 1, 1)");
+ cluster.get(3).executeInternal("INSERT INTO " + KEYSPACE + ".tbl (pk, ck, v1) VALUES (1, 1, 1)");
+
+ // Introduce schema disagreement
+ cluster.schemaChange("ALTER TABLE " + KEYSPACE + ".tbl ADD v2 int", 1);
+
+ try
+ {
+ cluster.coordinator(1).execute("SELECT * FROM " + KEYSPACE + ".tbl WHERE pk = 1", ConsistencyLevel.ALL);
+ fail("Should have failed because of schema disagreement.");
+ }
+ catch (Exception e)
+ {
+ // for some reason, we get weird errors when trying to check class directly
+ // I suppose it has to do with some classloader manipulation going on
+ Assert.assertTrue(e.getClass().toString().contains("ReadFailureException"));
+ // we may see 1 or 2 failures in here, because of the fail-fast behavior of ReadCallback
+ Assert.assertTrue(e.getMessage().contains("INCOMPATIBLE_SCHEMA from 127.0.0.2")
+ || e.getMessage().contains("INCOMPATIBLE_SCHEMA from 127.0.0.3"));
+ }
+
+ }
}
@Test
public void simplePagedReadsTest() throws Throwable
{
-
- cluster.schemaChange("CREATE TABLE " + KEYSPACE + ".tbl (pk int, ck int, v int, PRIMARY KEY (pk, ck))");
-
- int size = 100;
- Object[][] results = new Object[size][];
- for (int i = 0; i < size; i++)
+ try (ICluster cluster = init(builder().withNodes(3).start()))
{
- cluster.coordinator(1).execute("INSERT INTO " + KEYSPACE + ".tbl (pk, ck, v) VALUES (1, ?, ?)",
- ConsistencyLevel.QUORUM,
- i, i);
- results[i] = new Object[]{ 1, i, i };
- }
+ cluster.schemaChange("CREATE TABLE " + KEYSPACE + ".tbl (pk int, ck int, v int, PRIMARY KEY (pk, ck))");
- // Make sure paged read returns same results with different page sizes
- for (int pageSize : new int[]{ 1, 2, 3, 5, 10, 20, 50 })
- {
- assertRows(cluster.coordinator(1).executeWithPaging("SELECT * FROM " + KEYSPACE + ".tbl",
- ConsistencyLevel.QUORUM,
- pageSize),
- results);
+ int size = 100;
+ Object[][] results = new Object[size][];
+ for (int i = 0; i < size; i++)
+ {
+ cluster.coordinator(1).execute("INSERT INTO " + KEYSPACE + ".tbl (pk, ck, v) VALUES (1, ?, ?)",
+ ConsistencyLevel.QUORUM,
+ i, i);
+ results[i] = new Object[] { 1, i, i};
+ }
+
+ // Make sure paged read returns same results with different page sizes
+ for (int pageSize : new int[] { 1, 2, 3, 5, 10, 20, 50})
+ {
+ assertRows(cluster.coordinator(1).executeWithPaging("SELECT * FROM " + KEYSPACE + ".tbl",
+ ConsistencyLevel.QUORUM,
+ pageSize),
+ results);
+ }
}
}
@Test
public void pagingWithRepairTest() throws Throwable
{
- cluster.schemaChange("CREATE TABLE " + KEYSPACE + ".tbl (pk int, ck int, v int, PRIMARY KEY (pk, ck))");
-
- int size = 100;
- Object[][] results = new Object[size][];
- for (int i = 0; i < size; i++)
+ try (ICluster cluster = init(builder().withNodes(3).start()))
{
- // Make sure that data lands on different nodes and not coordinator
- cluster.get(i % 2 == 0 ? 2 : 3).executeInternal("INSERT INTO " + KEYSPACE + ".tbl (pk, ck, v) VALUES (1, ?, ?)",
- i, i);
+ cluster.schemaChange("CREATE TABLE " + KEYSPACE + ".tbl (pk int, ck int, v int, PRIMARY KEY (pk, ck))");
- results[i] = new Object[]{ 1, i, i };
- }
+ int size = 100;
+ Object[][] results = new Object[size][];
+ for (int i = 0; i < size; i++)
+ {
+ // Make sure that data lands on different nodes and not coordinator
+ cluster.get(i % 2 == 0 ? 2 : 3).executeInternal("INSERT INTO " + KEYSPACE + ".tbl (pk, ck, v) VALUES (1, ?, ?)",
+ i, i);
- // Make sure paged read returns same results with different page sizes
- for (int pageSize : new int[]{ 1, 2, 3, 5, 10, 20, 50 })
- {
- assertRows(cluster.coordinator(1).executeWithPaging("SELECT * FROM " + KEYSPACE + ".tbl",
- ConsistencyLevel.ALL,
- pageSize),
+ results[i] = new Object[] { 1, i, i};
+ }
+
+ // Make sure paged read returns same results with different page sizes
+ for (int pageSize : new int[] { 1, 2, 3, 5, 10, 20, 50})
+ {
+ assertRows(cluster.coordinator(1).executeWithPaging("SELECT * FROM " + KEYSPACE + ".tbl",
+ ConsistencyLevel.ALL,
+ pageSize),
+ results);
+ }
+
+ assertRows(cluster.get(1).executeInternal("SELECT * FROM " + KEYSPACE + ".tbl"),
results);
}
-
- assertRows(cluster.get(1).executeInternal("SELECT * FROM " + KEYSPACE + ".tbl"),
- results);
}
@Test
public void pagingTests() throws Throwable
{
- try (ICluster singleNode = init(builder().withNodes(1).withSubnet(1).start()))
+ try (ICluster cluster = init(builder().withNodes(3).start());
+ ICluster singleNode = init(builder().withNodes(1).withSubnet(1).start()))
{
cluster.schemaChange("CREATE TABLE " + KEYSPACE + ".tbl (pk int, ck int, v int, PRIMARY KEY (pk, ck))");
singleNode.schemaChange("CREATE TABLE " + KEYSPACE + ".tbl (pk int, ck int, v int, PRIMARY KEY (pk, ck))");
@@ -223,22 +359,22 @@
}
}
- int[] pageSizes = new int[]{ 1, 2, 3, 5, 10, 20, 50 };
- String[] statements = new String[]{ "SELECT * FROM " + KEYSPACE + ".tbl WHERE pk = 1 AND ck > 5",
- "SELECT * FROM " + KEYSPACE + ".tbl WHERE pk = 1 AND ck >= 5",
- "SELECT * FROM " + KEYSPACE + ".tbl WHERE pk = 1 AND ck > 5 AND ck <= 10",
- "SELECT * FROM " + KEYSPACE + ".tbl WHERE pk = 1 AND ck > 5 LIMIT 3",
- "SELECT * FROM " + KEYSPACE + ".tbl WHERE pk = 1 AND ck >= 5 LIMIT 2",
- "SELECT * FROM " + KEYSPACE + ".tbl WHERE pk = 1 AND ck > 5 AND ck <= 10 LIMIT 2",
- "SELECT * FROM " + KEYSPACE + ".tbl WHERE pk = 1 AND ck > 5 ORDER BY ck DESC",
- "SELECT * FROM " + KEYSPACE + ".tbl WHERE pk = 1 AND ck >= 5 ORDER BY ck DESC",
- "SELECT * FROM " + KEYSPACE + ".tbl WHERE pk = 1 AND ck > 5 AND ck <= 10 ORDER BY ck DESC",
- "SELECT * FROM " + KEYSPACE + ".tbl WHERE pk = 1 AND ck > 5 ORDER BY ck DESC LIMIT 3",
- "SELECT * FROM " + KEYSPACE + ".tbl WHERE pk = 1 AND ck >= 5 ORDER BY ck DESC LIMIT 2",
- "SELECT * FROM " + KEYSPACE + ".tbl WHERE pk = 1 AND ck > 5 AND ck <= 10 ORDER BY ck DESC LIMIT 2",
- "SELECT DISTINCT pk FROM " + KEYSPACE + ".tbl LIMIT 3",
- "SELECT DISTINCT pk FROM " + KEYSPACE + ".tbl WHERE pk IN (3,5,8,10)",
- "SELECT DISTINCT pk FROM " + KEYSPACE + ".tbl WHERE pk IN (3,5,8,10) LIMIT 2"
+ int[] pageSizes = new int[] { 1, 2, 3, 5, 10, 20, 50};
+ String[] statements = new String [] {"SELECT * FROM " + KEYSPACE + ".tbl WHERE pk = 1 AND ck > 5",
+ "SELECT * FROM " + KEYSPACE + ".tbl WHERE pk = 1 AND ck >= 5",
+ "SELECT * FROM " + KEYSPACE + ".tbl WHERE pk = 1 AND ck > 5 AND ck <= 10",
+ "SELECT * FROM " + KEYSPACE + ".tbl WHERE pk = 1 AND ck > 5 LIMIT 3",
+ "SELECT * FROM " + KEYSPACE + ".tbl WHERE pk = 1 AND ck >= 5 LIMIT 2",
+ "SELECT * FROM " + KEYSPACE + ".tbl WHERE pk = 1 AND ck > 5 AND ck <= 10 LIMIT 2",
+ "SELECT * FROM " + KEYSPACE + ".tbl WHERE pk = 1 AND ck > 5 ORDER BY ck DESC",
+ "SELECT * FROM " + KEYSPACE + ".tbl WHERE pk = 1 AND ck >= 5 ORDER BY ck DESC",
+ "SELECT * FROM " + KEYSPACE + ".tbl WHERE pk = 1 AND ck > 5 AND ck <= 10 ORDER BY ck DESC",
+ "SELECT * FROM " + KEYSPACE + ".tbl WHERE pk = 1 AND ck > 5 ORDER BY ck DESC LIMIT 3",
+ "SELECT * FROM " + KEYSPACE + ".tbl WHERE pk = 1 AND ck >= 5 ORDER BY ck DESC LIMIT 2",
+ "SELECT * FROM " + KEYSPACE + ".tbl WHERE pk = 1 AND ck > 5 AND ck <= 10 ORDER BY ck DESC LIMIT 2",
+ "SELECT DISTINCT pk FROM " + KEYSPACE + ".tbl LIMIT 3",
+ "SELECT DISTINCT pk FROM " + KEYSPACE + ".tbl WHERE pk IN (3,5,8,10)",
+ "SELECT DISTINCT pk FROM " + KEYSPACE + ".tbl WHERE pk IN (3,5,8,10) LIMIT 2"
};
for (String statement : statements)
{
@@ -246,31 +382,35 @@
{
assertRows(cluster.coordinator(1)
.executeWithPaging(statement,
- ConsistencyLevel.QUORUM, pageSize),
+ ConsistencyLevel.QUORUM, pageSize),
singleNode.coordinator(1)
.executeWithPaging(statement,
- ConsistencyLevel.QUORUM, Integer.MAX_VALUE));
+ ConsistencyLevel.QUORUM, Integer.MAX_VALUE));
}
}
+
}
}
@Test
public void metricsCountQueriesTest() throws Throwable
{
- cluster.schemaChange("CREATE TABLE " + KEYSPACE + ".tbl (pk int, ck int, v int, PRIMARY KEY (pk, ck))");
- for (int i = 0; i < 100; i++)
- cluster.coordinator(1).execute("INSERT INTO " + KEYSPACE + ".tbl (pk, ck, v) VALUES (?,?,?)", ConsistencyLevel.ALL, i, i, i);
+ try (ICluster<IInvokableInstance> cluster = init(Cluster.create(2)))
+ {
+ cluster.schemaChange("CREATE TABLE " + KEYSPACE + ".tbl (pk int, ck int, v int, PRIMARY KEY (pk, ck))");
+ for (int i = 0; i < 100; i++)
+ cluster.coordinator(1).execute("INSERT INTO "+KEYSPACE+".tbl (pk, ck, v) VALUES (?,?,?)", ConsistencyLevel.ALL, i, i, i);
- long readCount1 = readCount((IInvokableInstance) cluster.get(1));
- long readCount2 = readCount((IInvokableInstance) cluster.get(2));
- for (int i = 0; i < 100; i++)
- cluster.coordinator(1).execute("SELECT * FROM " + KEYSPACE + ".tbl WHERE pk = ? and ck = ?", ConsistencyLevel.ALL, i, i);
+ long readCount1 = readCount(cluster.get(1));
+ long readCount2 = readCount(cluster.get(2));
+ for (int i = 0; i < 100; i++)
+ cluster.coordinator(1).execute("SELECT * FROM "+KEYSPACE+".tbl WHERE pk = ? and ck = ?", ConsistencyLevel.ALL, i, i);
- readCount1 = readCount((IInvokableInstance) cluster.get(1)) - readCount1;
- readCount2 = readCount((IInvokableInstance) cluster.get(2)) - readCount2;
- assertEquals(readCount1, readCount2);
- assertEquals(100, readCount1);
+ readCount1 = readCount(cluster.get(1)) - readCount1;
+ readCount2 = readCount(cluster.get(2)) - readCount2;
+ Assert.assertEquals(readCount1, readCount2);
+ Assert.assertEquals(100, readCount1);
+ }
}
@@ -290,8 +430,9 @@
Set<SSTableReader> sstables = Keyspace.open(KEYSPACE)
.getColumnFamilyStore("tbl")
.getLiveSSTables();
- assertEquals(1, sstables.size());
- assertEquals(1, sstables.iterator().next().getMinTimestamp());
+ assertEquals("Expected a single sstable, but found " + sstables.size(), 1, sstables.size());
+ long minTimestamp = sstables.iterator().next().getMinTimestamp();
+ assertEquals("Expected min timestamp of 1, but was " + minTimestamp, 1, minTimestamp);
});
// on node 2, add a row for the deleted partition with an older timestamp than the deletion so it should be shadowed
@@ -301,7 +442,7 @@
Object[][] rows = cluster.coordinator(1)
.execute("SELECT * FROM " + KEYSPACE + ".tbl WHERE pk=0 AND ck > 5",
ConsistencyLevel.ALL);
- assertEquals(0, rows.length);
+ assertEquals("Expected 0 rows, but found " + rows.length, 0, rows.length);
}
}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/StreamingTest.java b/test/distributed/org/apache/cassandra/distributed/test/StreamingTest.java
new file mode 100644
index 0000000..956f21e
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/test/StreamingTest.java
@@ -0,0 +1,182 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test;
+
+import java.io.Serializable;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Queue;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.junit.Assert;
+import org.junit.Test;
+
+import org.apache.cassandra.distributed.Cluster;
+import org.apache.cassandra.distributed.api.IInvokableInstance;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.streaming.StreamSession;
+import org.apache.cassandra.streaming.messages.StreamMessage;
+
+import static org.apache.cassandra.distributed.api.Feature.NETWORK;
+import static org.apache.cassandra.streaming.StreamSession.State.PREPARING;
+import static org.apache.cassandra.streaming.StreamSession.State.STREAMING;
+import static org.apache.cassandra.streaming.StreamSession.State.WAIT_COMPLETE;
+import static org.apache.cassandra.streaming.messages.StreamMessage.Type.PREPARE_ACK;
+import static org.apache.cassandra.streaming.messages.StreamMessage.Type.PREPARE_SYN;
+import static org.apache.cassandra.streaming.messages.StreamMessage.Type.PREPARE_SYNACK;
+import static org.apache.cassandra.streaming.messages.StreamMessage.Type.RECEIVED;
+import static org.apache.cassandra.streaming.messages.StreamMessage.Type.STREAM;
+import static org.apache.cassandra.streaming.messages.StreamMessage.Type.STREAM_INIT;
+
+public class StreamingTest extends TestBaseImpl
+{
+
+ private void testStreaming(int nodes, int replicationFactor, int rowCount, String compactionStrategy) throws Throwable
+ {
+ try (Cluster cluster = (Cluster) builder().withNodes(nodes).withConfig(config -> config.with(NETWORK)).start())
+ {
+ cluster.schemaChange("CREATE KEYSPACE " + KEYSPACE + " WITH replication = {'class': 'SimpleStrategy', 'replication_factor': " + replicationFactor + "};");
+ cluster.schemaChange(String.format("CREATE TABLE %s.cf (k text, c1 text, c2 text, PRIMARY KEY (k)) WITH compaction = {'class': '%s', 'enabled': 'true'}", KEYSPACE, compactionStrategy));
+
+ for (int i = 0 ; i < rowCount ; ++i)
+ {
+ for (int n = 1 ; n < nodes ; ++n)
+ cluster.get(n).executeInternal(String.format("INSERT INTO %s.cf (k, c1, c2) VALUES (?, 'value1', 'value2');", KEYSPACE), Integer.toString(i));
+ }
+
+ cluster.get(nodes).executeInternal("TRUNCATE system.available_ranges;");
+ {
+ Object[][] results = cluster.get(nodes).executeInternal(String.format("SELECT k, c1, c2 FROM %s.cf;", KEYSPACE));
+ Assert.assertEquals(0, results.length);
+ }
+
+ // collect message and state
+ registerSink(cluster, nodes);
+
+ cluster.get(nodes).runOnInstance(() -> StorageService.instance.rebuild(null, KEYSPACE, null, null));
+ {
+ Object[][] results = cluster.get(nodes).executeInternal(String.format("SELECT k, c1, c2 FROM %s.cf;", KEYSPACE));
+ Assert.assertEquals(1000, results.length);
+ Arrays.sort(results, Comparator.comparingInt(a -> Integer.parseInt((String) a[0])));
+ for (int i = 0 ; i < results.length ; ++i)
+ {
+ Assert.assertEquals(Integer.toString(i), results[i][0]);
+ Assert.assertEquals("value1", results[i][1]);
+ Assert.assertEquals("value2", results[i][2]);
+ }
+ }
+ }
+ }
+
+ @Test
+ public void test() throws Throwable
+ {
+ testStreaming(2, 2, 1000, "LeveledCompactionStrategy");
+ }
+
+ public static void registerSink(Cluster cluster, int initiatorNodeId)
+ {
+ IInvokableInstance initiatorNode = cluster.get(initiatorNodeId);
+ InetSocketAddress initiator = initiatorNode.broadcastAddress();
+ MessageStateSinkImpl initiatorSink = new MessageStateSinkImpl();
+
+ for (int node = 1; node <= cluster.size(); node++)
+ {
+ if (initiatorNodeId == node)
+ continue;
+
+ IInvokableInstance followerNode = cluster.get(node);
+ InetSocketAddress follower = followerNode.broadcastAddress();
+
+ // verify on initiator's stream session
+ initiatorSink.messages(follower, Arrays.asList(PREPARE_SYNACK, STREAM, StreamMessage.Type.COMPLETE));
+ initiatorSink.states(follower, Arrays.asList(PREPARING, STREAMING, WAIT_COMPLETE, StreamSession.State.COMPLETE));
+
+ // verify on follower's stream session
+ MessageStateSinkImpl followerSink = new MessageStateSinkImpl();
+ followerSink.messages(initiator, Arrays.asList(STREAM_INIT, PREPARE_SYN, PREPARE_ACK, RECEIVED));
+ followerSink.states(initiator, Arrays.asList(PREPARING, STREAMING, StreamSession.State.COMPLETE));
+ followerNode.runOnInstance(() -> StreamSession.sink = followerSink);
+ }
+
+ cluster.get(initiatorNodeId).runOnInstance(() -> StreamSession.sink = initiatorSink);
+ }
+
+ @VisibleForTesting
+ public static class MessageStateSinkImpl implements StreamSession.MessageStateSink, Serializable
+ {
+ // use enum ordinal instead of enum to walk around inter-jvm class loader issue, only classes defined in
+ // InstanceClassLoader#sharedClassNames are shareable between server jvm and test jvm
+ public final Map<InetAddress, Queue<Integer>> messageSink = new ConcurrentHashMap<>();
+ public final Map<InetAddress, Queue<Integer>> stateTransitions = new ConcurrentHashMap<>();
+
+ public void messages(InetSocketAddress peer, List<StreamMessage.Type> messages)
+ {
+ messageSink.put(peer.getAddress(), messages.stream().map(Enum::ordinal).collect(Collectors.toCollection(LinkedList::new)));
+ }
+
+ public void states(InetSocketAddress peer, List<StreamSession.State> states)
+ {
+ stateTransitions.put(peer.getAddress(), states.stream().map(Enum::ordinal).collect(Collectors.toCollection(LinkedList::new)));
+ }
+
+ @Override
+ public void recordState(InetAddressAndPort from, StreamSession.State state)
+ {
+ Queue<Integer> states = stateTransitions.get(from.address);
+ if (states.peek() == null)
+ Assert.fail("Unexpected state " + state);
+
+ int expected = states.poll();
+ Assert.assertEquals(StreamSession.State.values()[expected], state);
+ }
+
+ @Override
+ public void recordMessage(InetAddressAndPort from, StreamMessage.Type message)
+ {
+ if (message == StreamMessage.Type.KEEP_ALIVE)
+ return;
+
+ Queue<Integer> messages = messageSink.get(from.address);
+ if (messages.peek() == null)
+ Assert.fail("Unexpected message " + message);
+
+ int expected = messages.poll();
+ Assert.assertEquals(StreamMessage.Type.values()[expected], message);
+ }
+
+ @Override
+ public void onClose(InetAddressAndPort from)
+ {
+ Queue<Integer> states = stateTransitions.get(from.address);
+ Assert.assertTrue("Missing states: " + states, states.isEmpty());
+
+ Queue<Integer> messages = messageSink.get(from.address);
+ Assert.assertTrue("Missing messages: " + messages, messages.isEmpty());
+ }
+ }
+}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/TableEstimatesTest.java b/test/distributed/org/apache/cassandra/distributed/test/TableEstimatesTest.java
new file mode 100644
index 0000000..0130f28
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/test/TableEstimatesTest.java
@@ -0,0 +1,84 @@
+package org.apache.cassandra.distributed.test;
+
+import java.io.IOException;
+
+import org.junit.AfterClass;
+import org.junit.Assume;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.distributed.Cluster;
+import org.apache.cassandra.distributed.api.ConsistencyLevel;
+import org.apache.cassandra.distributed.api.IInstance;
+import org.apache.cassandra.distributed.api.QueryResult;
+import org.assertj.core.api.Assertions;
+
+public class TableEstimatesTest extends TestBaseImpl
+{
+ private static Cluster CLUSTER;
+
+ @BeforeClass
+ public static void setupCluster() throws IOException
+ {
+ CLUSTER = init(Cluster.build(1).start());
+ }
+
+ @AfterClass
+ public static void teardownCluster()
+ {
+ if (CLUSTER != null)
+ CLUSTER.close();
+ }
+
+ /**
+ * Replaces Python Dtest: nodetool_test.py#test_refresh_size_estimates_clears_invalid_entries
+ */
+ @Test
+ public void refreshTableEstimatesClearsInvalidEntries()
+ {
+ String table_estimatesInsert = "INSERT INTO system.table_estimates (keyspace_name, table_name, range_type, range_start, range_end, mean_partition_size, partitions_count) VALUES (?, ?, ?, ?, ?, ?, ?)";
+ IInstance node = CLUSTER.get(1);
+
+ try
+ {
+ node.executeInternal(table_estimatesInsert, "system_auth", "bad_table", "local_primary", "-5", "5", 0L, 0L);
+ node.executeInternal(table_estimatesInsert, "bad_keyspace", "bad_table", "local_primary", "-5", "5", 0L, 0L);
+ }
+ catch (Exception e)
+ {
+ // to make this test portable (with the intent to extract out), handle the case where the table_estimates isn't defined
+ Assertions.assertThat(e.getClass().getCanonicalName()).isEqualTo("org.apache.cassandra.exceptions.InvalidRequestException");
+ Assertions.assertThat(e).hasMessageContaining("does not exist");
+ Assume.assumeTrue("system.table_estimates not present", false);
+ }
+
+ node.nodetoolResult("refreshsizeestimates").asserts().success();
+
+ QueryResult qr = CLUSTER.coordinator(1).executeWithResult("SELECT * FROM system.table_estimates WHERE keyspace_name=? AND table_name=?", ConsistencyLevel.ONE, "system_auth", "bad_table");
+ Assertions.assertThat(qr).isExhausted();
+
+ qr = CLUSTER.coordinator(1).executeWithResult("SELECT * FROM system.table_estimates WHERE keyspace_name=?", ConsistencyLevel.ONE, "bad_keyspace");
+ Assertions.assertThat(qr).isExhausted();
+ }
+
+ /**
+ * Replaces Python Dtest: nodetool_test.py#test_refresh_size_estimates_clears_invalid_entries
+ */
+ @Test
+ public void refreshSizeEstimatesClearsInvalidEntries()
+ {
+ String size_estimatesInsert = "INSERT INTO system.size_estimates (keyspace_name, table_name, range_start, range_end, mean_partition_size, partitions_count) VALUES (?, ?, ?, ?, ?, ?)";
+ IInstance node = CLUSTER.get(1);
+
+ node.executeInternal(size_estimatesInsert, "system_auth", "bad_table", "-5", "5", 0L, 0L);
+ node.executeInternal(size_estimatesInsert, "bad_keyspace", "bad_table", "-5", "5", 0L, 0L);
+
+ node.nodetoolResult("refreshsizeestimates").asserts().success();
+
+ QueryResult qr = CLUSTER.coordinator(1).executeWithResult("SELECT * FROM system.size_estimates WHERE keyspace_name=? AND table_name=?", ConsistencyLevel.ONE, "system_auth", "bad_table");
+ Assertions.assertThat(qr).isExhausted();
+
+ qr = CLUSTER.coordinator(1).executeWithResult("SELECT * FROM system.size_estimates WHERE keyspace_name=?", ConsistencyLevel.ONE, "bad_keyspace");
+ Assertions.assertThat(qr).isExhausted();
+ }
+}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/TopologyChangeTest.java b/test/distributed/org/apache/cassandra/distributed/test/TopologyChangeTest.java
new file mode 100644
index 0000000..f766775
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/test/TopologyChangeTest.java
@@ -0,0 +1,198 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+import java.util.Objects;
+import java.util.concurrent.TimeUnit;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import com.datastax.driver.core.Host;
+import com.datastax.driver.core.Session;
+import org.apache.cassandra.distributed.Cluster;
+import org.apache.cassandra.distributed.api.IInvokableInstance;
+import org.apache.cassandra.distributed.impl.INodeProvisionStrategy.Strategy;
+import org.apache.cassandra.distributed.test.TopologyChangeTest.EventStateListener.Event;
+
+import static org.apache.cassandra.distributed.api.Feature.GOSSIP;
+import static org.apache.cassandra.distributed.api.Feature.NATIVE_PROTOCOL;
+import static org.apache.cassandra.distributed.api.Feature.NETWORK;
+import static org.apache.cassandra.distributed.impl.INodeProvisionStrategy.Strategy.MultipleNetworkInterfaces;
+import static org.apache.cassandra.distributed.impl.INodeProvisionStrategy.Strategy.OneNetworkInterface;
+import static org.apache.cassandra.distributed.test.TopologyChangeTest.EventStateListener.EventType.Down;
+import static org.apache.cassandra.distributed.test.TopologyChangeTest.EventStateListener.EventType.Remove;
+import static org.apache.cassandra.distributed.test.TopologyChangeTest.EventStateListener.EventType.Up;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.awaitility.Awaitility.await;
+
+@RunWith(Parameterized.class)
+public class TopologyChangeTest extends TestBaseImpl
+{
+ static class EventStateListener implements Host.StateListener
+ {
+ enum EventType
+ {
+ Add,
+ Up,
+ Down,
+ Remove,
+ }
+
+ static class Event
+ {
+ String host;
+ EventType type;
+
+ Event(EventType type, Host host)
+ {
+ this.type = type;
+ this.host = host.getBroadcastSocketAddress().toString();
+ }
+
+ public Event(EventType type, IInvokableInstance iInvokableInstance)
+ {
+ this.type = type;
+ this.host = iInvokableInstance.broadcastAddress().toString();
+ }
+
+
+ public String toString()
+ {
+ return "Event{" +
+ "host='" + host + '\'' +
+ ", type=" + type +
+ '}';
+ }
+
+ public boolean equals(Object o)
+ {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ Event event = (Event) o;
+ return Objects.equals(host, event.host) &&
+ type == event.type;
+ }
+
+ public int hashCode()
+ {
+ return Objects.hash(host, type);
+ }
+ }
+
+ private List<Event> events = new ArrayList<>();
+
+ public void onAdd(Host host)
+ {
+ events.add(new Event(EventType.Add, host));
+ }
+
+ public void onUp(Host host)
+ {
+ events.add(new Event(Up, host));
+ }
+
+ public void onDown(Host host)
+ {
+ events.add(new Event(EventType.Down, host));
+ }
+
+ public void onRemove(Host host)
+ {
+ events.add(new Event(Remove, host));
+ }
+
+ public void onRegister(com.datastax.driver.core.Cluster cluster)
+ {
+ }
+
+ public void onUnregister(com.datastax.driver.core.Cluster cluster)
+ {
+ }
+ }
+
+ @Parameterized.Parameter(0)
+ public Strategy strategy;
+
+ @Parameterized.Parameters(name = "{index}: provision strategy={0}")
+ public static Collection<Strategy[]> data()
+ {
+ return Arrays.asList(new Strategy[][]{ { MultipleNetworkInterfaces },
+ { OneNetworkInterface }
+ });
+ }
+
+ @Test
+ public void testDecommission() throws Throwable
+ {
+ try (Cluster control = init(Cluster.build().withNodes(3).withNodeProvisionStrategy(strategy).withConfig(
+ config -> {
+ config.with(GOSSIP, NETWORK, NATIVE_PROTOCOL);
+ }).start()))
+ {
+ final com.datastax.driver.core.Cluster cluster = com.datastax.driver.core.Cluster.builder().addContactPoint("127.0.0.1").build();
+ Session session = cluster.connect();
+ EventStateListener eventStateListener = new EventStateListener();
+ session.getCluster().register(eventStateListener);
+ control.get(3).nodetool("disablebinary");
+ control.get(3).nodetool("decommission", "-f");
+ await().atMost(5, TimeUnit.SECONDS)
+ .untilAsserted(() -> Assert.assertEquals(2, cluster.getMetadata().getAllHosts().size()));
+ assertThat(eventStateListener.events).containsExactly(new Event(Remove, control.get(3)));
+ session.close();
+ cluster.close();
+ }
+ }
+
+ @Test
+ public void testRestartNode() throws Throwable
+ {
+ try (Cluster control = init(Cluster.build().withNodes(3).withNodeProvisionStrategy(strategy).withConfig(
+ config -> {
+ config.with(GOSSIP, NETWORK, NATIVE_PROTOCOL);
+ }).start()))
+ {
+ final com.datastax.driver.core.Cluster cluster = com.datastax.driver.core.Cluster.builder().addContactPoint("127.0.0.1").build();
+ Session session = cluster.connect();
+ EventStateListener eventStateListener = new EventStateListener();
+ session.getCluster().register(eventStateListener);
+
+ control.get(3).shutdown().get();
+ await().atMost(5, TimeUnit.SECONDS)
+ .untilAsserted(() -> Assert.assertEquals(2, cluster.getMetadata().getAllHosts().stream().filter(h -> h.isUp()).count()));
+
+ control.get(3).startup();
+ await().atMost(30, TimeUnit.SECONDS)
+ .untilAsserted(() -> Assert.assertEquals(3, cluster.getMetadata().getAllHosts().stream().filter(h -> h.isUp()).count()));
+
+ assertThat(eventStateListener.events).containsExactly(new Event(Down, control.get(3)),
+ new Event(Up, control.get(3)));
+
+ session.close();
+ cluster.close();
+ }
+ }
+}
+
diff --git a/test/distributed/org/apache/cassandra/distributed/upgrade/CompactStorage2to3UpgradeTest.java b/test/distributed/org/apache/cassandra/distributed/upgrade/CompactStorage2to3UpgradeTest.java
deleted file mode 100644
index f138861..0000000
--- a/test/distributed/org/apache/cassandra/distributed/upgrade/CompactStorage2to3UpgradeTest.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.distributed.upgrade;
-
-import org.junit.Test;
-
-import org.apache.cassandra.distributed.api.ConsistencyLevel;
-import org.apache.cassandra.distributed.api.ICoordinator;
-import org.apache.cassandra.distributed.shared.DistributedTestBase;
-import org.apache.cassandra.distributed.shared.Versions;
-import static org.apache.cassandra.distributed.shared.AssertUtils.*;
-
-public class CompactStorage2to3UpgradeTest extends UpgradeTestBase
-{
- @Test
- public void multiColumn() throws Throwable
- {
- new TestCase()
- .upgrade(Versions.Major.v22, Versions.Major.v30)
- .setup(cluster -> {
- assert cluster.size() == 3;
- int rf = cluster.size() - 1;
- assert rf == 2;
- cluster.schemaChange("CREATE KEYSPACE ks WITH replication = {'class': 'SimpleStrategy', 'replication_factor': " + (cluster.size() - 1) + "};");
- cluster.schemaChange("CREATE TABLE ks.tbl (pk int, v1 int, v2 text, PRIMARY KEY (pk)) WITH COMPACT STORAGE");
- ICoordinator coordinator = cluster.coordinator(1);
- // these shouldn't be replicated by the 3rd node
- coordinator.execute("INSERT INTO ks.tbl (pk, v1, v2) VALUES (3, 3, '3')", ConsistencyLevel.ALL);
- coordinator.execute("INSERT INTO ks.tbl (pk, v1, v2) VALUES (9, 9, '9')", ConsistencyLevel.ALL);
- for (int i = 0; i < cluster.size(); i++)
- {
- int nodeNum = i + 1;
- System.out.println(String.format("****** node %s: %s", nodeNum, cluster.get(nodeNum).config()));
- }
- })
- .runAfterNodeUpgrade(((cluster, node) -> {
- if (node != 2)
- return;
-
- Object[][] rows = cluster.coordinator(3).execute("SELECT * FROM ks.tbl LIMIT 2", ConsistencyLevel.ALL);
- Object[][] expected = {
- row(9, 9, "9"),
- row(3, 3, "3")
- };
- assertRows(rows, expected);
- })).run();
- }
-
- @Test
- public void singleColumn() throws Throwable
- {
- new TestCase()
- .upgrade(Versions.Major.v22, Versions.Major.v30)
- .setup(cluster -> {
- assert cluster.size() == 3;
- int rf = cluster.size() - 1;
- assert rf == 2;
- cluster.schemaChange("CREATE KEYSPACE ks WITH replication = {'class': 'SimpleStrategy', 'replication_factor': " + (cluster.size() - 1) + "};");
- cluster.schemaChange("CREATE TABLE ks.tbl (pk int, v int, PRIMARY KEY (pk)) WITH COMPACT STORAGE");
- ICoordinator coordinator = cluster.coordinator(1);
- // these shouldn't be replicated by the 3rd node
- coordinator.execute("INSERT INTO ks.tbl (pk, v) VALUES (3, 3)", ConsistencyLevel.ALL);
- coordinator.execute("INSERT INTO ks.tbl (pk, v) VALUES (9, 9)", ConsistencyLevel.ALL);
- for (int i = 0; i < cluster.size(); i++)
- {
- int nodeNum = i + 1;
- System.out.println(String.format("****** node %s: %s", nodeNum, cluster.get(nodeNum).config()));
- }
- })
- .runAfterNodeUpgrade(((cluster, node) -> {
-
- if (node < 2)
- return;
-
- Object[][] rows = cluster.coordinator(3).execute("SELECT * FROM ks.tbl LIMIT 2", ConsistencyLevel.ALL);
- Object[][] expected = {
- row(9, 9),
- row(3, 3)
- };
- assertRows(rows, expected);
- })).run();
- }
-}
\ No newline at end of file
diff --git a/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeRangeTombstoneTest.java b/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeRangeTombstoneTest.java
deleted file mode 100644
index e4b3a17..0000000
--- a/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeRangeTombstoneTest.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.distributed.upgrade;
-
-import org.junit.Test;
-
-import org.apache.cassandra.distributed.api.ConsistencyLevel;
-import org.apache.cassandra.distributed.shared.DistributedTestBase;
-import org.apache.cassandra.distributed.shared.Versions;
-
-import static java.lang.String.format;
-import static org.apache.cassandra.distributed.shared.AssertUtils.assertRows;
-import static org.apache.cassandra.distributed.shared.AssertUtils.row;
-
-/**
- * Tests related to the handle of range tombstones during 2.x to 3.x upgrades.
- */
-public class MixedModeRangeTombstoneTest extends UpgradeTestBase
-{
- /**
- * Tests the interaction of range tombstones covering multiple rows and collection tombsones within the covered
- * rows.
- *
- * <p>This test reproduces the issue of CASSANDRA-15805.
- */
- @Test
- public void multiRowsRangeTombstoneAndCollectionTombstoneInteractionTest() throws Throwable {
- String tableName = DistributedTestBase.KEYSPACE + ".t";
- String schema = "CREATE TABLE " + tableName + " (" +
- " k int," +
- " c1 text," +
- " c2 text," +
- " a text," +
- " b set<text>," +
- " c text," +
- " PRIMARY KEY((k), c1, c2)" +
- " )";
-
-
- new TestCase()
- .nodes(2)
- .upgrade(Versions.Major.v22, Versions.Major.v30)
- .setup(cluster -> {
- cluster.schemaChange(schema);
- cluster.coordinator(1).execute(format("DELETE FROM %s USING TIMESTAMP 1 WHERE k = 0 AND c1 = 'A'", tableName), ConsistencyLevel.ALL);
- cluster.coordinator(1).execute(format("INSERT INTO %s(k, c1, c2, a, b, c) VALUES (0, 'A', 'X', 'foo', {'whatever'}, 'bar') USING TIMESTAMP 2", tableName), ConsistencyLevel.ALL);
- cluster.coordinator(1).execute(format("DELETE b FROM %s USING TIMESTAMP 3 WHERE k = 0 AND c1 = 'A' and c2 = 'X'", tableName), ConsistencyLevel.ALL);
- cluster.get(1).flush(DistributedTestBase.KEYSPACE);
- cluster.get(2).flush(DistributedTestBase.KEYSPACE);
- })
- .runAfterNodeUpgrade((cluster, node) -> {
- assertRows(cluster.coordinator(node).execute(format("SELECT * FROM %s", tableName), ConsistencyLevel.ALL),
- row(0, "A", "X", "foo", null, "bar"));
- })
- .run();
- }
-}
diff --git a/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeReadRepairTest.java b/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeReadRepairTest.java
index e9391e0..cc50053 100644
--- a/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeReadRepairTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeReadRepairTest.java
@@ -18,19 +18,12 @@
package org.apache.cassandra.distributed.upgrade;
-import java.util.Arrays;
-import java.util.Iterator;
-
-import com.google.common.collect.Iterators;
import org.junit.Test;
-import org.apache.cassandra.distributed.UpgradeableCluster;
import org.apache.cassandra.distributed.api.ConsistencyLevel;
import org.apache.cassandra.distributed.shared.DistributedTestBase;
import org.apache.cassandra.distributed.shared.Versions;
-import static org.junit.Assert.fail;
-
public class MixedModeReadRepairTest extends UpgradeTestBase
{
@Test
@@ -56,82 +49,4 @@
.runAfterClusterUpgrade((cluster) -> cluster.get(2).forceCompact(DistributedTestBase.KEYSPACE, "tbl"))
.run();
}
-
- @Test
- public void mixedModeReadRepairDuplicateRows() throws Throwable
- {
- final String[] workload1 = new String[]
- {
- "DELETE FROM " + DistributedTestBase.KEYSPACE + ".tbl USING TIMESTAMP 1 WHERE pk = 1 AND ck = 2;",
- "INSERT INTO " + DistributedTestBase.KEYSPACE + ".tbl (pk, ck, v) VALUES (1, 1, {'a':'b'}) USING TIMESTAMP 3;",
- "INSERT INTO " + DistributedTestBase.KEYSPACE + ".tbl (pk, ck, v) VALUES (1, 2, {'c':'d'}) USING TIMESTAMP 3;",
- "INSERT INTO " + DistributedTestBase.KEYSPACE + ".tbl (pk, ck, v) VALUES (1, 3, {'e':'f'}) USING TIMESTAMP 3;",
- };
-
- final String[] workload2 = new String[]
- {
- "INSERT INTO " + DistributedTestBase.KEYSPACE + ".tbl (pk, ck, v) VALUES (1, 2, {'g':'h'}) USING TIMESTAMP 5;",
- };
-
- new TestCase()
- .nodes(2)
- .upgrade(Versions.Major.v22, Versions.Major.v30)
- .setup((cluster) ->
- {
- cluster.schemaChange("CREATE TABLE " + DistributedTestBase.KEYSPACE + ".tbl (pk int, ck int, v map<text, text>, PRIMARY KEY (pk, ck));");
- })
- .runAfterNodeUpgrade((cluster, node) ->
- {
- if (node == 2)
- return;
-
- // now node1 is 3.0 and node2 is 2.2
- for (int i = 0; i < workload1.length; i++ )
- cluster.coordinator(2).execute(workload1[i], ConsistencyLevel.QUORUM);
-
- cluster.get(1).flush(KEYSPACE);
- cluster.get(2).flush(KEYSPACE);
-
- validate(cluster, 2, false);
-
- for (int i = 0; i < workload2.length; i++ )
- cluster.coordinator(2).execute(workload2[i], ConsistencyLevel.QUORUM);
-
- cluster.get(1).flush(KEYSPACE);
- cluster.get(2).flush(KEYSPACE);
-
- validate(cluster, 1, true);
- })
- .run();
- }
-
- private void validate(UpgradeableCluster cluster, int nodeid, boolean local)
- {
- String query = "SELECT * FROM " + KEYSPACE + ".tbl";
-
- Iterator<Object[]> iter = local
- ? Iterators.forArray(cluster.get(nodeid).executeInternal(query))
- : cluster.coordinator(nodeid).executeWithPaging(query, ConsistencyLevel.ALL, 2);
-
- Object[] prevRow = null;
- Object prevClustering = null;
-
- while (iter.hasNext())
- {
- Object[] row = iter.next();
- Object clustering = row[1];
-
- if (clustering.equals(prevClustering))
- {
- fail(String.format("Duplicate rows on node %d in %s mode: \n%s\n%s",
- nodeid,
- local ? "local" : "distributed",
- Arrays.toString(prevRow),
- Arrays.toString(row)));
- }
-
- prevRow = row;
- prevClustering = clustering;
- }
- }
}
diff --git a/test/distributed/org/apache/cassandra/distributed/upgrade/UpgradeTest.java b/test/distributed/org/apache/cassandra/distributed/upgrade/UpgradeTest.java
index 5970992..88dd0f9 100644
--- a/test/distributed/org/apache/cassandra/distributed/upgrade/UpgradeTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/upgrade/UpgradeTest.java
@@ -18,15 +18,10 @@
package org.apache.cassandra.distributed.upgrade;
-import java.util.Iterator;
-
-import com.google.common.collect.Iterators;
import org.junit.Test;
import org.apache.cassandra.distributed.api.ConsistencyLevel;
import org.apache.cassandra.distributed.shared.Versions;
-
-import junit.framework.Assert;
import static org.apache.cassandra.distributed.shared.AssertUtils.*;
public class UpgradeTest extends UpgradeTestBase
@@ -37,6 +32,7 @@
{
new TestCase()
.upgrade(Versions.Major.v22, Versions.Major.v30, Versions.Major.v3X)
+ .upgrade(Versions.Major.v30, Versions.Major.v3X, Versions.Major.v4)
.setup((cluster) -> {
cluster.schemaChange("CREATE TABLE " + KEYSPACE + ".tbl (pk int, ck int, v int, PRIMARY KEY (pk, ck))");
@@ -46,44 +42,11 @@
})
.runAfterClusterUpgrade((cluster) -> {
assertRows(cluster.coordinator(1).execute("SELECT * FROM " + KEYSPACE + ".tbl WHERE pk = ?",
- ConsistencyLevel.ALL,
- 1),
- row(1, 1, 1),
- row(1, 2, 2),
- row(1, 3, 3));
- }).run();
- }
-
- @Test
- public void mixedModePagingTest() throws Throwable
- {
- new TestCase()
- .upgrade(Versions.Major.v22, Versions.Major.v30)
- .nodes(2)
- .nodesToUpgrade(2)
- .setup((cluster) -> {
- cluster.schemaChange("ALTER KEYSPACE " + KEYSPACE + " WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1}");
- cluster.schemaChange("CREATE TABLE " + KEYSPACE + ".tbl (pk int, ck int, v int, PRIMARY KEY (pk, ck)) with compact storage");
- for (int i = 0; i < 100; i++)
- for (int j = 0; j < 200; j++)
- cluster.coordinator(2).execute("INSERT INTO " + KEYSPACE + ".tbl (pk, ck, v) VALUES (?, ?, 1)", ConsistencyLevel.ALL, i, j);
- cluster.forEach((i) -> i.flush(KEYSPACE));
- for (int i = 0; i < 100; i++)
- for (int j = 10; j < 30; j++)
- cluster.coordinator(2).execute("DELETE FROM " + KEYSPACE + ".tbl where pk=? and ck=?", ConsistencyLevel.ALL, i, j);
- cluster.forEach((i) -> i.flush(KEYSPACE));
- })
- .runAfterClusterUpgrade((cluster) -> {
- for (int i = 0; i < 100; i++)
- {
- for (int pageSize = 10; pageSize < 100; pageSize++)
- {
- Iterator<Object[]> res = cluster.coordinator(1).executeWithPaging("SELECT * FROM " + KEYSPACE + ".tbl WHERE pk = ?",
- ConsistencyLevel.ALL,
- pageSize, i);
- Assert.assertEquals(180, Iterators.size(res));
- }
- }
+ ConsistencyLevel.ALL,
+ 1),
+ row(1, 1, 1),
+ row(1, 2, 2),
+ row(1, 3, 3));
}).run();
}
}
\ No newline at end of file
diff --git a/test/distributed/org/apache/cassandra/distributed/upgrade/UpgradeTestBase.java b/test/distributed/org/apache/cassandra/distributed/upgrade/UpgradeTestBase.java
index 4f0c700..7b1f6f6 100644
--- a/test/distributed/org/apache/cassandra/distributed/upgrade/UpgradeTestBase.java
+++ b/test/distributed/org/apache/cassandra/distributed/upgrade/UpgradeTestBase.java
@@ -31,7 +31,6 @@
import org.apache.cassandra.distributed.UpgradeableCluster;
import org.apache.cassandra.distributed.api.ICluster;
import org.apache.cassandra.distributed.api.IInstanceConfig;
-import org.apache.cassandra.distributed.api.IUpgradeableInstance;
import org.apache.cassandra.distributed.impl.Instance;
import org.apache.cassandra.distributed.shared.DistributedTestBase;
import org.apache.cassandra.distributed.shared.Versions;
@@ -173,7 +172,7 @@
for (Version version : upgrade.upgrade)
{
- for (int n=1; n<=nodesToUpgrade.size(); n++)
+ for (int n : nodesToUpgrade)
{
cluster.get(n).shutdown().get();
cluster.get(n).setVersion(version);
@@ -197,4 +196,4 @@
}
}
-}
\ No newline at end of file
+}
diff --git a/test/distributed/org/apache/cassandra/distributed/util/PyDtest.java b/test/distributed/org/apache/cassandra/distributed/util/PyDtest.java
new file mode 100644
index 0000000..3b2425f
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/util/PyDtest.java
@@ -0,0 +1,186 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.util;
+
+import java.util.Arrays;
+import java.util.List;
+
+public class PyDtest
+{
+
+ public static class CreateCf
+ {
+ final String keyspace;
+ final String name;
+ String primaryKey, clustering, keyType, speculativeRetry, compression, validation, compactionStrategy;
+ Float readRepair;
+ Integer gcGrace;
+ List<String> columns;
+ Boolean compactStorage;
+
+ public CreateCf(String keyspace, String name)
+ {
+ this.keyspace = keyspace;
+ this.name = name;
+ }
+
+ public CreateCf withPrimaryKey(String primaryKey)
+ {
+ this.primaryKey = primaryKey;
+ return this;
+ }
+
+ public CreateCf withClustering(String clustering)
+ {
+ this.clustering = clustering;
+ return this;
+ }
+
+ public CreateCf withKeyType(String keyType)
+ {
+ this.keyType = keyType;
+ return this;
+ }
+
+ public CreateCf withSpeculativeRetry(String speculativeRetry)
+ {
+ this.speculativeRetry = speculativeRetry;
+ return this;
+ }
+
+ public CreateCf withCompression(String compression)
+ {
+ this.compression = compression;
+ return this;
+ }
+
+ public CreateCf withValidation(String validation)
+ {
+ this.validation = validation;
+ return this;
+ }
+
+ public CreateCf withCompactionStrategy(String compactionStrategy)
+ {
+ this.compactionStrategy = compactionStrategy;
+ return this;
+ }
+
+ public CreateCf withReadRepair(Float readRepair)
+ {
+ this.readRepair = readRepair;
+ return this;
+ }
+
+ public CreateCf withGcGrace(Integer gcGrace)
+ {
+ this.gcGrace = gcGrace;
+ return this;
+ }
+
+ public CreateCf withColumns(List<String> columns)
+ {
+ this.columns = columns;
+ return this;
+ }
+
+ public CreateCf withColumns(String ... columns)
+ {
+ this.columns = Arrays.asList(columns);
+ return this;
+ }
+
+ public CreateCf withCompactStorage(Boolean compactStorage)
+ {
+ this.compactStorage = compactStorage;
+ return this;
+ }
+
+ public String build()
+ {
+ if (keyspace == null)
+ throw new IllegalArgumentException();
+ if (name == null)
+ throw new IllegalArgumentException();
+ if (keyType == null)
+ keyType = "varchar";
+ if (validation == null)
+ validation = "UTF8Type";
+ if (compactionStrategy == null)
+ compactionStrategy = "SizeTieredCompactionStrategy";
+ if (compactStorage == null)
+ compactStorage = false;
+
+
+ String compaction_fragment = String.format("compaction = {'class': '%s', 'enabled': 'true'}", compactionStrategy);
+
+ String query;
+ String additional_columns = "";
+ if (columns == null)
+ {
+ query = String.format("CREATE COLUMNFAMILY %s.%s (key %s, c varchar, v varchar, PRIMARY KEY(key, c)) WITH comment=\'test cf\'", keyspace, name, keyType);
+ }
+ else
+ {
+ for (String pair : columns)
+ {
+ String[] split = pair.split(":");
+ String key = split[0];
+ String type = split[1];
+ additional_columns += ", " + key + " " + type;
+ }
+
+ if (primaryKey != null)
+ query = String.format("CREATE COLUMNFAMILY %s.%s (key %s%s, PRIMARY KEY(%s)) WITH comment=\'test cf\'", keyspace, name, keyType, additional_columns, primaryKey);
+ else
+ query = String.format("CREATE COLUMNFAMILY %s.%s (key %s PRIMARY KEY%s) WITH comment=\'test cf\'", keyspace, name, keyType, additional_columns);
+ }
+
+
+ if (compaction_fragment != null)
+ query += " AND " + compaction_fragment;
+
+ if (clustering != null)
+ query += String.format(" AND CLUSTERING ORDER BY (%s)", clustering);
+
+ if (compression != null)
+ query += String.format(" AND compression = { \'sstable_compression\': \'%sCompressor\' }", compression);
+ else
+ query += " AND compression = {}";
+
+ if (readRepair != null)
+ query += String.format(" AND read_repair_chance=%f AND dclocal_read_repair_chance=%f", readRepair, readRepair);
+ if (gcGrace != null)
+ query += String.format(" AND gc_grace_seconds=%d", gcGrace);
+ if (speculativeRetry != null)
+ query += String.format(" AND speculative_retry=\'%s\'", speculativeRetry);
+
+ if (compactStorage != null && compactStorage)
+ query += " AND COMPACT STORAGE";
+
+ return query;
+ }
+ }
+
+ public static CreateCf createCf(String keyspace, String name)
+ {
+ return new CreateCf(keyspace, name);
+ }
+
+}
diff --git a/test/distributed/org/apache/cassandra/io/sstable/format/ForwardingSSTableReader.java b/test/distributed/org/apache/cassandra/io/sstable/format/ForwardingSSTableReader.java
new file mode 100644
index 0000000..40adc9c
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/io/sstable/format/ForwardingSSTableReader.java
@@ -0,0 +1,679 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.sstable.format;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.UUID;
+
+import com.google.common.util.concurrent.RateLimiter;
+
+import org.apache.cassandra.cache.InstrumentingCache;
+import org.apache.cassandra.cache.KeyCacheKey;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DataRange;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.db.RowIndexEntry;
+import org.apache.cassandra.db.Slices;
+import org.apache.cassandra.db.filter.ColumnFilter;
+import org.apache.cassandra.db.rows.EncodingStats;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.dht.IPartitioner;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.io.compress.CompressionMetadata;
+import org.apache.cassandra.io.sstable.Component;
+import org.apache.cassandra.io.sstable.ISSTableScanner;
+import org.apache.cassandra.io.sstable.SSTable;
+import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
+import org.apache.cassandra.io.util.ChannelProxy;
+import org.apache.cassandra.io.util.FileDataInput;
+import org.apache.cassandra.io.util.FileHandle;
+import org.apache.cassandra.io.util.RandomAccessReader;
+import org.apache.cassandra.metrics.RestorableMeter;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.schema.TableMetadataRef;
+import org.apache.cassandra.utils.EstimatedHistogram;
+import org.apache.cassandra.utils.IFilter;
+import org.apache.cassandra.utils.concurrent.Ref;
+
+public abstract class ForwardingSSTableReader extends SSTableReader
+{
+ private final SSTableReader delegate;
+ private final Ref<SSTableReader> selfRef;
+
+ public ForwardingSSTableReader(SSTableReader delegate)
+ {
+ super(delegate.descriptor, SSTable.componentsFor(delegate.descriptor),
+ TableMetadataRef.forOfflineTools(delegate.metadata()), delegate.maxDataAge, delegate.getSSTableMetadata(),
+ delegate.openReason, delegate.header);
+ this.delegate = delegate;
+ this.first = delegate.first;
+ this.last = delegate.last;
+ this.selfRef = new Ref<>(this, new Tidy()
+ {
+ public void tidy() throws Exception
+ {
+ Ref<SSTableReader> ref = delegate.tryRef();
+ if (ref != null)
+ ref.release();
+ }
+
+ public String name()
+ {
+ return descriptor.toString();
+ }
+ });
+ }
+
+ protected RowIndexEntry getPosition(PartitionPosition key, Operator op, boolean updateCacheAndStats, boolean permitMatchPastLast, SSTableReadsListener listener)
+ {
+ return delegate.getPosition(key, op, updateCacheAndStats, permitMatchPastLast, listener);
+ }
+
+ public UnfilteredRowIterator iterator(DecoratedKey key, Slices slices, ColumnFilter selectedColumns, boolean reversed, SSTableReadsListener listener)
+ {
+ return delegate.iterator(key, slices, selectedColumns, reversed, listener);
+ }
+
+ public UnfilteredRowIterator iterator(FileDataInput file, DecoratedKey key, RowIndexEntry indexEntry, Slices slices, ColumnFilter selectedColumns, boolean reversed)
+ {
+ return delegate.iterator(file, key, indexEntry, slices, selectedColumns, reversed);
+ }
+
+ public UnfilteredRowIterator simpleIterator(FileDataInput file, DecoratedKey key, RowIndexEntry indexEntry, boolean tombstoneOnly)
+ {
+ return delegate.simpleIterator(file, key, indexEntry, tombstoneOnly);
+ }
+
+ public ISSTableScanner getScanner()
+ {
+ return delegate.getScanner();
+ }
+
+ public ISSTableScanner getScanner(Collection<Range<Token>> ranges)
+ {
+ return delegate.getScanner(ranges);
+ }
+
+ public ISSTableScanner getScanner(Iterator<AbstractBounds<PartitionPosition>> rangeIterator)
+ {
+ return delegate.getScanner(rangeIterator);
+ }
+
+ public ISSTableScanner getScanner(ColumnFilter columns, DataRange dataRange, SSTableReadsListener listener)
+ {
+ return delegate.getScanner(columns, dataRange, listener);
+ }
+
+ public void setupOnline()
+ {
+ delegate.setupOnline();
+ }
+
+ public String getFilename()
+ {
+ return delegate.getFilename();
+ }
+
+ public boolean equals(Object that)
+ {
+ return delegate.equals(that);
+ }
+
+ public int hashCode()
+ {
+ return delegate.hashCode();
+ }
+
+ public boolean loadSummary()
+ {
+ return delegate.loadSummary();
+ }
+
+ public void saveSummary()
+ {
+ delegate.saveSummary();
+ }
+
+ public void saveBloomFilter()
+ {
+ delegate.saveBloomFilter();
+ }
+
+ public void setReplaced()
+ {
+ delegate.setReplaced();
+ }
+
+ public boolean isReplaced()
+ {
+ return delegate.isReplaced();
+ }
+
+ public void runOnClose(Runnable runOnClose)
+ {
+ delegate.runOnClose(runOnClose);
+ }
+
+ public SSTableReader cloneWithRestoredStart(DecoratedKey restoredStart)
+ {
+ return delegate.cloneWithRestoredStart(restoredStart);
+ }
+
+ public SSTableReader cloneWithNewStart(DecoratedKey newStart, Runnable runOnClose)
+ {
+ return delegate.cloneWithNewStart(newStart, runOnClose);
+ }
+
+ public SSTableReader cloneWithNewSummarySamplingLevel(ColumnFamilyStore parent, int samplingLevel) throws IOException
+ {
+ return delegate.cloneWithNewSummarySamplingLevel(parent, samplingLevel);
+ }
+
+ public RestorableMeter getReadMeter()
+ {
+ return delegate.getReadMeter();
+ }
+
+ public int getIndexSummarySamplingLevel()
+ {
+ return delegate.getIndexSummarySamplingLevel();
+ }
+
+ public long getIndexSummaryOffHeapSize()
+ {
+ return delegate.getIndexSummaryOffHeapSize();
+ }
+
+ public int getMinIndexInterval()
+ {
+ return delegate.getMinIndexInterval();
+ }
+
+ public double getEffectiveIndexInterval()
+ {
+ return delegate.getEffectiveIndexInterval();
+ }
+
+ public void releaseSummary()
+ {
+ delegate.releaseSummary();
+ }
+
+ public long getIndexScanPosition(PartitionPosition key)
+ {
+ return delegate.getIndexScanPosition(key);
+ }
+
+ public CompressionMetadata getCompressionMetadata()
+ {
+ return delegate.getCompressionMetadata();
+ }
+
+ public long getCompressionMetadataOffHeapSize()
+ {
+ return delegate.getCompressionMetadataOffHeapSize();
+ }
+
+ public void forceFilterFailures()
+ {
+ delegate.forceFilterFailures();
+ }
+
+ public IFilter getBloomFilter()
+ {
+ return delegate.getBloomFilter();
+ }
+
+ public long getBloomFilterSerializedSize()
+ {
+ return delegate.getBloomFilterSerializedSize();
+ }
+
+ public long getBloomFilterOffHeapSize()
+ {
+ return delegate.getBloomFilterOffHeapSize();
+ }
+
+ public long estimatedKeys()
+ {
+ return delegate.estimatedKeys();
+ }
+
+ public long estimatedKeysForRanges(Collection<Range<Token>> ranges)
+ {
+ return delegate.estimatedKeysForRanges(ranges);
+ }
+
+ public int getIndexSummarySize()
+ {
+ return delegate.getIndexSummarySize();
+ }
+
+ public int getMaxIndexSummarySize()
+ {
+ return delegate.getMaxIndexSummarySize();
+ }
+
+ public byte[] getIndexSummaryKey(int index)
+ {
+ return delegate.getIndexSummaryKey(index);
+ }
+
+ public Iterable<DecoratedKey> getKeySamples(Range<Token> range)
+ {
+ return delegate.getKeySamples(range);
+ }
+
+ public List<PartitionPositionBounds> getPositionsForRanges(Collection<Range<Token>> ranges)
+ {
+ return delegate.getPositionsForRanges(ranges);
+ }
+
+ public KeyCacheKey getCacheKey(DecoratedKey key)
+ {
+ return delegate.getCacheKey(key);
+ }
+
+ public void cacheKey(DecoratedKey key, RowIndexEntry info)
+ {
+ delegate.cacheKey(key, info);
+ }
+
+ public RowIndexEntry getCachedPosition(DecoratedKey key, boolean updateStats)
+ {
+ return delegate.getCachedPosition(key, updateStats);
+ }
+
+ protected RowIndexEntry getCachedPosition(KeyCacheKey unifiedKey, boolean updateStats)
+ {
+ return delegate.getCachedPosition(unifiedKey, updateStats);
+ }
+
+ public boolean isKeyCacheEnabled()
+ {
+ return delegate.isKeyCacheEnabled();
+ }
+
+ public DecoratedKey firstKeyBeyond(PartitionPosition token)
+ {
+ return delegate.firstKeyBeyond(token);
+ }
+
+ public long uncompressedLength()
+ {
+ return delegate.uncompressedLength();
+ }
+
+ public long onDiskLength()
+ {
+ return delegate.onDiskLength();
+ }
+
+ public double getCrcCheckChance()
+ {
+ return delegate.getCrcCheckChance();
+ }
+
+ public void setCrcCheckChance(double crcCheckChance)
+ {
+ delegate.setCrcCheckChance(crcCheckChance);
+ }
+
+ public void markObsolete(Runnable tidier)
+ {
+ delegate.markObsolete(tidier);
+ }
+
+ public boolean isMarkedCompacted()
+ {
+ return delegate.isMarkedCompacted();
+ }
+
+ public void markSuspect()
+ {
+ delegate.markSuspect();
+ }
+
+ public void unmarkSuspect()
+ {
+ delegate.unmarkSuspect();
+ }
+
+ public boolean isMarkedSuspect()
+ {
+ return delegate.isMarkedSuspect();
+ }
+
+ public ISSTableScanner getScanner(Range<Token> range)
+ {
+ return delegate.getScanner(range);
+ }
+
+ public FileDataInput getFileDataInput(long position)
+ {
+ return delegate.getFileDataInput(position);
+ }
+
+ public boolean newSince(long age)
+ {
+ return delegate.newSince(age);
+ }
+
+ public void createLinks(String snapshotDirectoryPath)
+ {
+ delegate.createLinks(snapshotDirectoryPath);
+ }
+
+ public boolean isRepaired()
+ {
+ return delegate.isRepaired();
+ }
+
+ public DecoratedKey keyAt(long indexPosition) throws IOException
+ {
+ return delegate.keyAt(indexPosition);
+ }
+
+ public boolean isPendingRepair()
+ {
+ return delegate.isPendingRepair();
+ }
+
+ public UUID getPendingRepair()
+ {
+ return delegate.getPendingRepair();
+ }
+
+ public long getRepairedAt()
+ {
+ return delegate.getRepairedAt();
+ }
+
+ public boolean isTransient()
+ {
+ return delegate.isTransient();
+ }
+
+ public boolean intersects(Collection<Range<Token>> ranges)
+ {
+ return delegate.intersects(ranges);
+ }
+
+ public long getBloomFilterFalsePositiveCount()
+ {
+ return delegate.getBloomFilterFalsePositiveCount();
+ }
+
+ public long getRecentBloomFilterFalsePositiveCount()
+ {
+ return delegate.getRecentBloomFilterFalsePositiveCount();
+ }
+
+ public long getBloomFilterTruePositiveCount()
+ {
+ return delegate.getBloomFilterTruePositiveCount();
+ }
+
+ public long getRecentBloomFilterTruePositiveCount()
+ {
+ return delegate.getRecentBloomFilterTruePositiveCount();
+ }
+
+ public InstrumentingCache<KeyCacheKey, RowIndexEntry> getKeyCache()
+ {
+ return delegate.getKeyCache();
+ }
+
+ public EstimatedHistogram getEstimatedPartitionSize()
+ {
+ return delegate.getEstimatedPartitionSize();
+ }
+
+ public EstimatedHistogram getEstimatedCellPerPartitionCount()
+ {
+ return delegate.getEstimatedCellPerPartitionCount();
+ }
+
+ public double getEstimatedDroppableTombstoneRatio(int gcBefore)
+ {
+ return delegate.getEstimatedDroppableTombstoneRatio(gcBefore);
+ }
+
+ public double getDroppableTombstonesBefore(int gcBefore)
+ {
+ return delegate.getDroppableTombstonesBefore(gcBefore);
+ }
+
+ public double getCompressionRatio()
+ {
+ return delegate.getCompressionRatio();
+ }
+
+ public long getMinTimestamp()
+ {
+ return delegate.getMinTimestamp();
+ }
+
+ public long getMaxTimestamp()
+ {
+ return delegate.getMaxTimestamp();
+ }
+
+ public int getMinLocalDeletionTime()
+ {
+ return delegate.getMinLocalDeletionTime();
+ }
+
+ public int getMaxLocalDeletionTime()
+ {
+ return delegate.getMaxLocalDeletionTime();
+ }
+
+ public boolean mayHaveTombstones()
+ {
+ return delegate.mayHaveTombstones();
+ }
+
+ public int getMinTTL()
+ {
+ return delegate.getMinTTL();
+ }
+
+ public int getMaxTTL()
+ {
+ return delegate.getMaxTTL();
+ }
+
+ public long getTotalColumnsSet()
+ {
+ return delegate.getTotalColumnsSet();
+ }
+
+ public long getTotalRows()
+ {
+ return delegate.getTotalRows();
+ }
+
+ public int getAvgColumnSetPerRow()
+ {
+ return delegate.getAvgColumnSetPerRow();
+ }
+
+ public int getSSTableLevel()
+ {
+ return delegate.getSSTableLevel();
+ }
+
+ public void reloadSSTableMetadata() throws IOException
+ {
+ delegate.reloadSSTableMetadata();
+ }
+
+ public StatsMetadata getSSTableMetadata()
+ {
+ return delegate.getSSTableMetadata();
+ }
+
+ public RandomAccessReader openDataReader(RateLimiter limiter)
+ {
+ return delegate.openDataReader(limiter);
+ }
+
+ public RandomAccessReader openDataReader()
+ {
+ return delegate.openDataReader();
+ }
+
+ public RandomAccessReader openIndexReader()
+ {
+ return delegate.openIndexReader();
+ }
+
+ public ChannelProxy getIndexChannel()
+ {
+ return delegate.getIndexChannel();
+ }
+
+ public FileHandle getIndexFile()
+ {
+ return delegate.getIndexFile();
+ }
+
+ public long getCreationTimeFor(Component component)
+ {
+ return delegate.getCreationTimeFor(component);
+ }
+
+ public long getKeyCacheHit()
+ {
+ return delegate.getKeyCacheHit();
+ }
+
+ public long getKeyCacheRequest()
+ {
+ return delegate.getKeyCacheRequest();
+ }
+
+ public void incrementReadCount()
+ {
+ delegate.incrementReadCount();
+ }
+
+ public EncodingStats stats()
+ {
+ return delegate.stats();
+ }
+
+ public Ref<SSTableReader> tryRef()
+ {
+ return selfRef.tryRef();
+ }
+
+ public Ref<SSTableReader> selfRef()
+ {
+ return selfRef;
+ }
+
+ public Ref<SSTableReader> ref()
+ {
+ return selfRef.ref();
+ }
+
+ void setup(boolean trackHotness)
+ {
+ delegate.setup(trackHotness);
+ }
+
+ public void overrideReadMeter(RestorableMeter readMeter)
+ {
+ delegate.overrideReadMeter(readMeter);
+ }
+
+ public void addTo(Ref.IdentityCollection identities)
+ {
+ delegate.addTo(identities);
+ }
+
+ public TableMetadata metadata()
+ {
+ return delegate.metadata();
+ }
+
+ public IPartitioner getPartitioner()
+ {
+ return delegate.getPartitioner();
+ }
+
+ public DecoratedKey decorateKey(ByteBuffer key)
+ {
+ return delegate.decorateKey(key);
+ }
+
+ public String getIndexFilename()
+ {
+ return delegate.getIndexFilename();
+ }
+
+ public String getColumnFamilyName()
+ {
+ return delegate.getColumnFamilyName();
+ }
+
+ public String getKeyspaceName()
+ {
+ return delegate.getKeyspaceName();
+ }
+
+ public List<String> getAllFilePaths()
+ {
+ return delegate.getAllFilePaths();
+ }
+
+// protected long estimateRowsFromIndex(RandomAccessReader ifile) throws IOException
+// {
+// return delegate.estimateRowsFromIndex(ifile);
+// }
+
+ public long bytesOnDisk()
+ {
+ return delegate.bytesOnDisk();
+ }
+
+ public String toString()
+ {
+ return delegate.toString();
+ }
+
+ public AbstractBounds<Token> getBounds()
+ {
+ return delegate.getBounds();
+ }
+
+ public ChannelProxy getDataChannel()
+ {
+ return delegate.getDataChannel();
+ }
+}
diff --git a/test/long/org/apache/cassandra/cql3/CachingBench.java b/test/long/org/apache/cassandra/cql3/CachingBench.java
index 25f746b..0a6657f 100644
--- a/test/long/org/apache/cassandra/cql3/CachingBench.java
+++ b/test/long/org/apache/cassandra/cql3/CachingBench.java
@@ -33,7 +33,7 @@
import org.junit.BeforeClass;
import org.junit.Test;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.cassandra.config.Config.CommitLogSync;
import org.apache.cassandra.config.Config.DiskAccessMode;
import org.apache.cassandra.cache.ChunkCache;
@@ -340,7 +340,7 @@
int countRows(ColumnFamilyStore cfs)
{
- boolean enforceStrictLiveness = cfs.metadata.enforceStrictLiveness();
+ boolean enforceStrictLiveness = cfs.metadata().enforceStrictLiveness();
int nowInSec = FBUtilities.nowInSeconds();
return count(cfs, x -> x.isRow() && ((Row) x).hasLiveData(nowInSec, enforceStrictLiveness));
}
diff --git a/test/long/org/apache/cassandra/cql3/CorruptionTest.java b/test/long/org/apache/cassandra/cql3/CorruptionTest.java
index 43cf5e0..f2ed36a 100644
--- a/test/long/org/apache/cassandra/cql3/CorruptionTest.java
+++ b/test/long/org/apache/cassandra/cql3/CorruptionTest.java
@@ -37,7 +37,7 @@
import com.datastax.driver.core.utils.Bytes;
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.config.Schema;
+import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.service.EmbeddedCassandraService;
diff --git a/test/long/org/apache/cassandra/cql3/GcCompactionBench.java b/test/long/org/apache/cassandra/cql3/GcCompactionBench.java
index 84c0384..01abb63 100644
--- a/test/long/org/apache/cassandra/cql3/GcCompactionBench.java
+++ b/test/long/org/apache/cassandra/cql3/GcCompactionBench.java
@@ -33,7 +33,7 @@
import org.junit.BeforeClass;
import org.junit.Test;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.cassandra.config.Config.CommitLogSync;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.ColumnFamilyStore;
@@ -339,7 +339,7 @@
int countRows(ColumnFamilyStore cfs)
{
- boolean enforceStrictLiveness = cfs.metadata.enforceStrictLiveness();
+ boolean enforceStrictLiveness = cfs.metadata().enforceStrictLiveness();
int nowInSec = FBUtilities.nowInSeconds();
return count(cfs, x -> x.isRow() && ((Row) x).hasLiveData(nowInSec, enforceStrictLiveness));
}
diff --git a/test/long/org/apache/cassandra/cql3/ViewLongTest.java b/test/long/org/apache/cassandra/cql3/ViewLongTest.java
index ddf62e9..7102649 100644
--- a/test/long/org/apache/cassandra/cql3/ViewLongTest.java
+++ b/test/long/org/apache/cassandra/cql3/ViewLongTest.java
@@ -38,7 +38,6 @@
import org.apache.cassandra.transport.ProtocolVersion;
import org.apache.cassandra.concurrent.SEPExecutor;
import org.apache.cassandra.concurrent.Stage;
-import org.apache.cassandra.concurrent.StageManager;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.WrappedRunnable;
@@ -135,7 +134,7 @@
for (int i = 0; i < writers * insertsPerWriter; i++)
{
- if (executeNet(protocolVersion, "SELECT COUNT(*) FROM system.batchlog").one().getLong(0) == 0)
+ if (executeNet(protocolVersion, "SELECT COUNT(*) FROM system.batches").one().getLong(0) == 0)
break;
try
{
@@ -407,8 +406,8 @@
private void updateViewWithFlush(String query, boolean flush, Object... params) throws Throwable
{
executeNet(protocolVersion, query, params);
- while (!(((SEPExecutor) StageManager.getStage(Stage.VIEW_MUTATION)).getPendingTasks() == 0
- && ((SEPExecutor) StageManager.getStage(Stage.VIEW_MUTATION)).getActiveCount() == 0))
+ while (!(((SEPExecutor) Stage.VIEW_MUTATION.executor()).getPendingTaskCount() == 0
+ && ((SEPExecutor) Stage.VIEW_MUTATION.executor()).getActiveTaskCount() == 0))
{
Thread.sleep(1);
}
diff --git a/test/long/org/apache/cassandra/db/commitlog/CommitLogStressTest.java b/test/long/org/apache/cassandra/db/commitlog/CommitLogStressTest.java
index 2162d85..7fadac4 100644
--- a/test/long/org/apache/cassandra/db/commitlog/CommitLogStressTest.java
+++ b/test/long/org/apache/cassandra/db/commitlog/CommitLogStressTest.java
@@ -51,6 +51,7 @@
import org.apache.cassandra.io.compress.SnappyCompressor;
import org.apache.cassandra.io.util.DataInputBuffer;
import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.security.EncryptionContext;
import org.apache.cassandra.security.EncryptionContextGenerator;
@@ -214,7 +215,7 @@
}
verifySizes(commitLog);
- commitLog.discardCompletedSegments(Schema.instance.getCFMetaData("Keyspace1", "Standard1").cfId,
+ commitLog.discardCompletedSegments(Schema.instance.getTableMetadata("Keyspace1", "Standard1").id,
CommitLogPosition.NONE, discardedPos);
threads.clear();
@@ -396,7 +397,7 @@
rl.acquire();
ByteBuffer key = randomBytes(16, rand);
- UpdateBuilder builder = UpdateBuilder.create(Schema.instance.getCFMetaData("Keyspace1", "Standard1"), Util.dk(key));
+ UpdateBuilder builder = UpdateBuilder.create(Schema.instance.getTableMetadata("Keyspace1", "Standard1"), Util.dk(key));
for (int ii = 0; ii < numCells; ii++)
{
int sz = randomSize ? rand.nextInt(cellSize) : cellSize;
@@ -447,7 +448,7 @@
{
mutation = Mutation.serializer.deserialize(bufIn,
desc.getMessagingVersion(),
- SerializationHelper.Flag.LOCAL);
+ DeserializationHelper.Flag.LOCAL);
}
catch (IOException e)
{
diff --git a/test/long/org/apache/cassandra/db/commitlog/GroupCommitLogStressTest.java b/test/long/org/apache/cassandra/db/commitlog/GroupCommitLogStressTest.java
new file mode 100644
index 0000000..e3fa961
--- /dev/null
+++ b/test/long/org/apache/cassandra/db/commitlog/GroupCommitLogStressTest.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.commitlog;
+
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import org.apache.cassandra.config.Config;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.config.ParameterizedClass;
+import org.apache.cassandra.security.EncryptionContext;
+
+@RunWith(Parameterized.class)
+public class GroupCommitLogStressTest extends CommitLogStressTest
+{
+ public GroupCommitLogStressTest(ParameterizedClass commitLogCompression, EncryptionContext encryptionContext)
+ {
+ super(commitLogCompression, encryptionContext);
+ DatabaseDescriptor.setCommitLogSync(Config.CommitLogSync.group);
+ DatabaseDescriptor.setCommitLogSyncGroupWindow(1);
+ }
+}
diff --git a/test/long/org/apache/cassandra/db/compaction/LongCompactionsTest.java b/test/long/org/apache/cassandra/db/compaction/LongCompactionsTest.java
index d684e11..fe8cdc2 100644
--- a/test/long/org/apache/cassandra/db/compaction/LongCompactionsTest.java
+++ b/test/long/org/apache/cassandra/db/compaction/LongCompactionsTest.java
@@ -28,7 +28,7 @@
import org.apache.cassandra.UpdateBuilder;
import org.apache.cassandra.SchemaLoader;
-import org.apache.cassandra.config.Schema;
+import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.Util;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.db.*;
@@ -55,7 +55,7 @@
SchemaLoader.createKeyspace(KEYSPACE1,
KeyspaceParams.simple(1),
SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD)
- .compaction(CompactionParams.scts(compactionOptions)));
+ .compaction(CompactionParams.stcs(compactionOptions)));
}
@Before
@@ -108,7 +108,7 @@
{
String key = String.valueOf(j);
// last sstable has highest timestamps
- UpdateBuilder builder = UpdateBuilder.create(store.metadata, String.valueOf(j))
+ UpdateBuilder builder = UpdateBuilder.create(store.metadata(), String.valueOf(j))
.withTimestamp(k);
for (int i = 0; i < rowsPerPartition; i++)
builder.newRow(String.valueOf(i)).add("val", String.valueOf(i));
@@ -123,11 +123,11 @@
Thread.sleep(1000);
long start = System.nanoTime();
- final int gcBefore = (int) (System.currentTimeMillis() / 1000) - Schema.instance.getCFMetaData(KEYSPACE1, "Standard1").params.gcGraceSeconds;
+ final int gcBefore = (int) (System.currentTimeMillis() / 1000) - Schema.instance.getTableMetadata(KEYSPACE1, "Standard1").params.gcGraceSeconds;
try (LifecycleTransaction txn = store.getTracker().tryModify(sstables, OperationType.COMPACTION))
{
assert txn != null : "Cannot markCompacting all sstables";
- new CompactionTask(store, txn, gcBefore).execute(null);
+ new CompactionTask(store, txn, gcBefore).execute(ActiveCompactionsTracker.NOOP);
}
System.out.println(String.format("%s: sstables=%d rowsper=%d colsper=%d: %d ms",
this.getClass().getName(),
@@ -146,7 +146,7 @@
cfs.clearUnsafe();
final int ROWS_PER_SSTABLE = 10;
- final int SSTABLES = cfs.metadata.params.minIndexInterval * 3 / ROWS_PER_SSTABLE;
+ final int SSTABLES = cfs.metadata().params.minIndexInterval * 3 / ROWS_PER_SSTABLE;
// disable compaction while flushing
cfs.disableAutoCompaction();
@@ -158,7 +158,7 @@
DecoratedKey key = Util.dk(String.valueOf(i % 2));
long timestamp = j * ROWS_PER_SSTABLE + i;
maxTimestampExpected = Math.max(timestamp, maxTimestampExpected);
- UpdateBuilder.create(cfs.metadata, key)
+ UpdateBuilder.create(cfs.metadata(), key)
.withTimestamp(timestamp)
.newRow(String.valueOf(i / 2)).add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER)
.apply();
diff --git a/test/long/org/apache/cassandra/db/compaction/LongLeveledCompactionStrategyTest.java b/test/long/org/apache/cassandra/db/compaction/LongLeveledCompactionStrategyTest.java
index 0d54173..f8f94a0 100644
--- a/test/long/org/apache/cassandra/db/compaction/LongLeveledCompactionStrategyTest.java
+++ b/test/long/org/apache/cassandra/db/compaction/LongLeveledCompactionStrategyTest.java
@@ -20,6 +20,7 @@
import java.nio.ByteBuffer;
import java.util.*;
import java.util.concurrent.*;
+import java.util.stream.Collectors;
import com.google.common.collect.Lists;
@@ -38,8 +39,11 @@
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.schema.CompactionParams;
import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.service.ActiveRepairService;
+import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.FBUtilities;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
public class LongLeveledCompactionStrategyTest
@@ -75,22 +79,7 @@
ByteBuffer value = ByteBuffer.wrap(new byte[100 * 1024]); // 100 KB value, make it easy to have multiple files
- // Enough data to have a level 1 and 2
- int rows = 128;
- int columns = 10;
-
- // Adds enough data to trigger multiple sstable per level
- for (int r = 0; r < rows; r++)
- {
- DecoratedKey key = Util.dk(String.valueOf(r));
- UpdateBuilder builder = UpdateBuilder.create(store.metadata, key);
- for (int c = 0; c < columns; c++)
- builder.newRow("column" + c).add("val", value);
-
- Mutation rm = new Mutation(builder.build());
- rm.apply();
- store.forceBlockingFlush();
- }
+ populateSSTables(store);
// Execute LCS in parallel
ExecutorService executor = new ThreadPoolExecutor(4, 4,
@@ -108,7 +97,7 @@
{
public void run()
{
- nextTask.execute(null);
+ nextTask.execute(ActiveCompactionsTracker.NOOP);
}
});
}
@@ -153,22 +142,8 @@
ColumnFamilyStore store = keyspace.getColumnFamilyStore(CF_STANDARDLVL2);
ByteBuffer value = ByteBuffer.wrap(new byte[100 * 1024]); // 100 KB value, make it easy to have multiple files
- // Enough data to have a level 1 and 2
- int rows = 128;
- int columns = 10;
+ populateSSTables(store);
- // Adds enough data to trigger multiple sstable per level
- for (int r = 0; r < rows; r++)
- {
- DecoratedKey key = Util.dk(String.valueOf(r));
- UpdateBuilder builder = UpdateBuilder.create(store.metadata, key);
- for (int c = 0; c < columns; c++)
- builder.newRow("column" + c).add("val", value);
-
- Mutation rm = new Mutation(builder.build());
- rm.apply();
- store.forceBlockingFlush();
- }
LeveledCompactionStrategyTest.waitForLeveling(store);
store.disableAutoCompaction();
CompactionStrategyManager mgr = store.getCompactionStrategyManager();
@@ -180,7 +155,7 @@
for (int r = 0; r < 10; r++)
{
DecoratedKey key = Util.dk(String.valueOf(r));
- UpdateBuilder builder = UpdateBuilder.create(store.metadata, key);
+ UpdateBuilder builder = UpdateBuilder.create(store.metadata(), key);
for (int c = 0; c < 10; c++)
builder.newRow("column" + c).add("val", value);
@@ -229,4 +204,66 @@
}
+
+ @Test
+ public void testRepairStatusChanges() throws Exception
+ {
+ String ksname = KEYSPACE1;
+ String cfname = "StandardLeveled";
+ Keyspace keyspace = Keyspace.open(ksname);
+ ColumnFamilyStore store = keyspace.getColumnFamilyStore(cfname);
+ store.disableAutoCompaction();
+
+ CompactionStrategyManager mgr = store.getCompactionStrategyManager();
+ LeveledCompactionStrategy repaired = (LeveledCompactionStrategy) mgr.getStrategies().get(0).get(0);
+ LeveledCompactionStrategy unrepaired = (LeveledCompactionStrategy) mgr.getStrategies().get(1).get(0);
+
+ // populate repaired sstables
+ populateSSTables(store);
+ assertTrue(repaired.getSSTables().isEmpty());
+ assertFalse(unrepaired.getSSTables().isEmpty());
+ mgr.mutateRepaired(store.getLiveSSTables(), FBUtilities.nowInSeconds(), null, false);
+ assertFalse(repaired.getSSTables().isEmpty());
+ assertTrue(unrepaired.getSSTables().isEmpty());
+
+ // populate unrepaired sstables
+ populateSSTables(store);
+ assertFalse(repaired.getSSTables().isEmpty());
+ assertFalse(unrepaired.getSSTables().isEmpty());
+
+ // compact them into upper levels
+ store.forceMajorCompaction();
+ assertFalse(repaired.getSSTables().isEmpty());
+ assertFalse(unrepaired.getSSTables().isEmpty());
+
+ // mark unrepair
+ mgr.mutateRepaired(store.getLiveSSTables().stream().filter(s -> s.isRepaired()).collect(Collectors.toList()),
+ ActiveRepairService.UNREPAIRED_SSTABLE,
+ null,
+ false);
+ assertTrue(repaired.getSSTables().isEmpty());
+ assertFalse(unrepaired.getSSTables().isEmpty());
+ }
+
+ private void populateSSTables(ColumnFamilyStore store)
+ {
+ ByteBuffer value = ByteBuffer.wrap(new byte[100 * 1024]); // 100 KB value, make it easy to have multiple files
+
+ // Enough data to have a level 1 and 2
+ int rows = 128;
+ int columns = 10;
+
+ // Adds enough data to trigger multiple sstable per level
+ for (int r = 0; r < rows; r++)
+ {
+ DecoratedKey key = Util.dk(String.valueOf(r));
+ UpdateBuilder builder = UpdateBuilder.create(store.metadata(), key);
+ for (int c = 0; c < columns; c++)
+ builder.newRow("column" + c).add("val", value);
+
+ Mutation rm = new Mutation(builder.build());
+ rm.apply();
+ store.forceBlockingFlush();
+ }
+ }
}
diff --git a/test/long/org/apache/cassandra/dht/tokenallocator/AbstractReplicationAwareTokenAllocatorTest.java b/test/long/org/apache/cassandra/dht/tokenallocator/AbstractReplicationAwareTokenAllocatorTest.java
index eb79f12..5f9aa31 100644
--- a/test/long/org/apache/cassandra/dht/tokenallocator/AbstractReplicationAwareTokenAllocatorTest.java
+++ b/test/long/org/apache/cassandra/dht/tokenallocator/AbstractReplicationAwareTokenAllocatorTest.java
@@ -523,12 +523,12 @@
SummaryStatistics unitStat = new SummaryStatistics();
for (Map.Entry<Unit, Double> en : ownership.entrySet())
unitStat.addValue(en.getValue() * inverseAverage / t.unitToTokens.get(en.getKey()).size());
- su.update(unitStat);
+ su.update(unitStat, t.unitCount());
SummaryStatistics tokenStat = new SummaryStatistics();
for (Token tok : t.sortedTokens.keySet())
tokenStat.addValue(replicatedTokenOwnership(tok, t.sortedTokens, t.strategy) * inverseAverage);
- st.update(tokenStat);
+ st.update(tokenStat, t.unitCount());
if (print)
{
diff --git a/test/long/org/apache/cassandra/dht/tokenallocator/NoReplicationTokenAllocatorTest.java b/test/long/org/apache/cassandra/dht/tokenallocator/NoReplicationTokenAllocatorTest.java
index c53f788..ee38a28 100644
--- a/test/long/org/apache/cassandra/dht/tokenallocator/NoReplicationTokenAllocatorTest.java
+++ b/test/long/org/apache/cassandra/dht/tokenallocator/NoReplicationTokenAllocatorTest.java
@@ -20,7 +20,6 @@
import java.util.Collections;
import java.util.List;
-import java.util.Map;
import java.util.NavigableMap;
import java.util.PriorityQueue;
import java.util.Random;
@@ -29,7 +28,7 @@
import org.apache.commons.math3.stat.descriptive.SummaryStatistics;
import org.junit.Test;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.Murmur3Partitioner;
import org.apache.cassandra.dht.RandomPartitioner;
@@ -184,7 +183,7 @@
{
unitStat.addValue(wu.weight * size / t.tokensInUnits.get(wu.value.unit).size());
}
- su.update(unitStat);
+ su.update(unitStat, t.sortedUnits.size());
SummaryStatistics tokenStat = new SummaryStatistics();
for (PriorityQueue<TokenAllocatorBase.Weighted<TokenAllocatorBase.TokenInfo>> tokens : t.tokensInUnits.values())
@@ -194,7 +193,7 @@
tokenStat.addValue(token.weight);
}
}
- st.update(tokenStat);
+ st.update(tokenStat, t.sortedUnits.size());
if (print)
{
diff --git a/test/long/org/apache/cassandra/dht/tokenallocator/RandomReplicationAwareTokenAllocatorTest.java b/test/long/org/apache/cassandra/dht/tokenallocator/RandomReplicationAwareTokenAllocatorTest.java
index bd94442..6a2d59e 100644
--- a/test/long/org/apache/cassandra/dht/tokenallocator/RandomReplicationAwareTokenAllocatorTest.java
+++ b/test/long/org/apache/cassandra/dht/tokenallocator/RandomReplicationAwareTokenAllocatorTest.java
@@ -41,13 +41,6 @@
@Test
public void testNewClusterr()
{
- Util.flakyTest(this::flakyTestNewCluster,
- 3,
- "It tends to fail sometimes due to the random selection of the tokens in the first few nodes.");
- }
-
- private void flakyTestNewCluster()
- {
testNewCluster(new RandomPartitioner(), MAX_VNODE_COUNT);
}
diff --git a/test/long/org/apache/cassandra/dht/tokenallocator/TokenAllocatorTestBase.java b/test/long/org/apache/cassandra/dht/tokenallocator/TokenAllocatorTestBase.java
index 8612ac1..8722426 100644
--- a/test/long/org/apache/cassandra/dht/tokenallocator/TokenAllocatorTestBase.java
+++ b/test/long/org/apache/cassandra/dht/tokenallocator/TokenAllocatorTestBase.java
@@ -25,6 +25,7 @@
import org.apache.commons.math3.stat.descriptive.SummaryStatistics;
+import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.Token;
@@ -36,6 +37,11 @@
protected static final int TARGET_CLUSTER_SIZE = 250;
protected static final int MAX_VNODE_COUNT = 64;
+ public TokenAllocatorTestBase()
+ {
+ DatabaseDescriptor.clientInitialization();
+ }
+
interface TestReplicationStrategy extends ReplicationStrategy<Unit>
{
void addUnit(Unit n);
@@ -124,19 +130,34 @@
class Summary
{
double min = 1;
+ int minAt = -1;
double max = 1;
+ int maxAt = - 1;
double stddev = 0;
+ int stddevAt = -1;
- void update(SummaryStatistics stat)
+ void update(SummaryStatistics stat, int point)
{
- min = Math.min(min, stat.getMin());
- max = Math.max(max, stat.getMax());
- stddev = Math.max(stddev, stat.getStandardDeviation());
+ if (stat.getMin() <= min)
+ {
+ min = Math.min(min, stat.getMin());
+ minAt = point;
+ }
+ if (stat.getMax() >= max)
+ {
+ max = Math.max(max, stat.getMax());
+ maxAt = point;
+ }
+ if (stat.getStandardDeviation() >= stddev)
+ {
+ stddev = Math.max(stddev, stat.getStandardDeviation());
+ stddevAt = point;
+ }
}
public String toString()
{
- return String.format("max %.2f min %.2f stddev %.4f", max, min, stddev);
+ return String.format("max %.4f @%d min %.4f @%d stddev %.4f @%d", max, maxAt, min, minAt, stddev, stddevAt);
}
}
diff --git a/test/long/org/apache/cassandra/hints/HintsWriteThenReadTest.java b/test/long/org/apache/cassandra/hints/HintsWriteThenReadTest.java
index fd880cb..a905f9a 100644
--- a/test/long/org/apache/cassandra/hints/HintsWriteThenReadTest.java
+++ b/test/long/org/apache/cassandra/hints/HintsWriteThenReadTest.java
@@ -31,8 +31,8 @@
import org.junit.Test;
import org.apache.cassandra.SchemaLoader;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.Schema;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.db.Mutation;
import org.apache.cassandra.db.RowUpdateBuilder;
import org.apache.cassandra.db.rows.Cell;
@@ -167,7 +167,7 @@
private static Mutation createMutation(int index, long timestamp)
{
- CFMetaData table = Schema.instance.getCFMetaData(KEYSPACE, TABLE);
+ TableMetadata table = Schema.instance.getTableMetadata(KEYSPACE, TABLE);
return new RowUpdateBuilder(table, timestamp, bytes(index))
.clustering(bytes(index))
.add("val", bytes(index))
diff --git a/test/long/org/apache/cassandra/io/compress/CompressorPerformance.java b/test/long/org/apache/cassandra/io/compress/CompressorPerformance.java
index e703839..b3cdaa1 100644
--- a/test/long/org/apache/cassandra/io/compress/CompressorPerformance.java
+++ b/test/long/org/apache/cassandra/io/compress/CompressorPerformance.java
@@ -35,7 +35,9 @@
SnappyCompressor.instance, // warm up
DeflateCompressor.instance,
LZ4Compressor.create(Collections.emptyMap()),
- SnappyCompressor.instance
+ SnappyCompressor.instance,
+ ZstdCompressor.getOrCreate(ZstdCompressor.FAST_COMPRESSION_LEVEL),
+ ZstdCompressor.getOrCreate(ZstdCompressor.DEFAULT_COMPRESSION_LEVEL)
})
{
for (BufferType in: BufferType.values())
@@ -70,10 +72,15 @@
int count = 100;
long time = System.nanoTime();
+ long uncompressedBytes = 0;
+ long compressedBytes = 0;
for (int i=0; i<count; ++i)
{
output.clear();
compressor.compress(dataSource, output);
+ uncompressedBytes += dataSource.limit();
+ compressedBytes += output.position();
+
// Make sure not optimized away.
checksum += output.get(ThreadLocalRandom.current().nextInt(output.position()));
dataSource.rewind();
@@ -93,7 +100,7 @@
input.rewind();
}
long timed = System.nanoTime() - time;
- System.out.format("Compressor %s %s->%s compress %.3f ns/b %.3f mb/s uncompress %.3f ns/b %.3f mb/s.%s\n",
+ System.out.format("Compressor %s %s->%s compress %.3f ns/b %.3f mb/s uncompress %.3f ns/b %.3f mb/s ratio %.2f:1.%s\n",
compressor.getClass().getSimpleName(),
in,
out,
@@ -101,6 +108,7 @@
Math.scalb(1.0e9, -20) * count * len / timec,
1.0 * timed / (count * len),
Math.scalb(1.0e9, -20) * count * len / timed,
+ ((double) uncompressedBytes) / ((double) compressedBytes),
checksum == 0 ? " " : "");
}
diff --git a/test/long/org/apache/cassandra/io/sstable/CQLSSTableWriterLongTest.java b/test/long/org/apache/cassandra/io/sstable/CQLSSTableWriterLongTest.java
index 9674ca3..a6f428a 100644
--- a/test/long/org/apache/cassandra/io/sstable/CQLSSTableWriterLongTest.java
+++ b/test/long/org/apache/cassandra/io/sstable/CQLSSTableWriterLongTest.java
@@ -25,11 +25,9 @@
import com.google.common.io.Files;
-import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
import org.apache.cassandra.SchemaLoader;
-import org.apache.cassandra.config.Config;
import org.apache.cassandra.service.StorageService;
public class CQLSSTableWriterLongTest
diff --git a/test/long/org/apache/cassandra/locator/DynamicEndpointSnitchLongTest.java b/test/long/org/apache/cassandra/locator/DynamicEndpointSnitchLongTest.java
index 35bf5b4..2e27738 100644
--- a/test/long/org/apache/cassandra/locator/DynamicEndpointSnitchLongTest.java
+++ b/test/long/org/apache/cassandra/locator/DynamicEndpointSnitchLongTest.java
@@ -20,7 +20,6 @@
package org.apache.cassandra.locator;
import java.io.IOException;
-import java.net.InetAddress;
import java.util.*;
import org.junit.Test;
@@ -31,6 +30,8 @@
import org.apache.cassandra.utils.FBUtilities;
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+
public class DynamicEndpointSnitchLongTest
{
static
@@ -53,21 +54,23 @@
StorageService.instance.unsafeInitialize();
SimpleSnitch ss = new SimpleSnitch();
DynamicEndpointSnitch dsnitch = new DynamicEndpointSnitch(ss, String.valueOf(ss.hashCode()));
- InetAddress self = FBUtilities.getBroadcastAddress();
+ InetAddressAndPort self = FBUtilities.getBroadcastAddressAndPort();
- List<InetAddress> hosts = new ArrayList<>();
+ EndpointsForRange.Builder replicasBuilder = EndpointsForRange.builder(ReplicaUtils.FULL_RANGE);
// We want a big list of hosts so sorting takes time, making it much more likely to reproduce the
// problem we're looking for.
for (int i = 0; i < 100; i++)
for (int j = 0; j < 256; j++)
- hosts.add(InetAddress.getByAddress(new byte[]{127, 0, (byte)i, (byte)j}));
+ replicasBuilder.add(ReplicaUtils.full(InetAddressAndPort.getByAddress(new byte[]{ 127, 0, (byte)i, (byte)j})));
- ScoreUpdater updater = new ScoreUpdater(dsnitch, hosts);
+ EndpointsForRange replicas = replicasBuilder.build();
+
+ ScoreUpdater updater = new ScoreUpdater(dsnitch, replicas);
updater.start();
- List<InetAddress> result = null;
+ EndpointsForRange result = replicas;
for (int i = 0; i < ITERATIONS; i++)
- result = dsnitch.getSortedListByProximity(self, hosts);
+ result = dsnitch.sortedByProximity(self, result);
updater.stopped = true;
updater.join();
@@ -85,10 +88,10 @@
public volatile boolean stopped;
private final DynamicEndpointSnitch dsnitch;
- private final List<InetAddress> hosts;
+ private final EndpointsForRange hosts;
private final Random random = new Random();
- public ScoreUpdater(DynamicEndpointSnitch dsnitch, List<InetAddress> hosts)
+ public ScoreUpdater(DynamicEndpointSnitch dsnitch, EndpointsForRange hosts)
{
this.dsnitch = dsnitch;
this.hosts = hosts;
@@ -98,9 +101,9 @@
{
while (!stopped)
{
- InetAddress host = hosts.get(random.nextInt(hosts.size()));
+ Replica host = hosts.get(random.nextInt(hosts.size()));
int score = random.nextInt(SCORE_RANGE);
- dsnitch.receiveTiming(host, score);
+ dsnitch.receiveTiming(host.endpoint(), score, MILLISECONDS);
}
}
}
diff --git a/test/long/org/apache/cassandra/streaming/LongStreamingTest.java b/test/long/org/apache/cassandra/streaming/LongStreamingTest.java
index 1340224..e37045a 100644
--- a/test/long/org/apache/cassandra/streaming/LongStreamingTest.java
+++ b/test/long/org/apache/cassandra/streaming/LongStreamingTest.java
@@ -24,22 +24,22 @@
import java.util.concurrent.TimeUnit;
import com.google.common.io.Files;
-import org.junit.AfterClass;
+import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
import org.apache.cassandra.SchemaLoader;
-import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.config.Schema;
import org.apache.cassandra.cql3.QueryProcessor;
import org.apache.cassandra.cql3.UntypedResultSet;
import org.apache.cassandra.db.Keyspace;
-import org.apache.cassandra.dht.Range;
-import org.apache.cassandra.dht.Token;
import org.apache.cassandra.exceptions.InvalidRequestException;
import org.apache.cassandra.io.sstable.CQLSSTableWriter;
import org.apache.cassandra.io.sstable.SSTableLoader;
+import org.apache.cassandra.locator.Replica;
+import org.apache.cassandra.schema.CompressionParams;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.TableMetadataRef;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.OutputHandler;
@@ -63,26 +63,42 @@
}
@Test
- public void testCompressedStream() throws InvalidRequestException, IOException, ExecutionException, InterruptedException
+ public void testSstableCompressionStreaming() throws InterruptedException, ExecutionException, IOException
{
- String KS = "cql_keyspace";
+ testStream(true);
+ }
+
+ @Test
+ public void testStreamCompressionStreaming() throws InterruptedException, ExecutionException, IOException
+ {
+ testStream(false);
+ }
+
+ private void testStream(boolean useSstableCompression) throws InvalidRequestException, IOException, ExecutionException, InterruptedException
+ {
+ String KS = useSstableCompression ? "sstable_compression_ks" : "stream_compression_ks";
String TABLE = "table1";
File tempdir = Files.createTempDir();
File dataDir = new File(tempdir.getAbsolutePath() + File.separator + KS + File.separator + TABLE);
assert dataDir.mkdirs();
- String schema = "CREATE TABLE cql_keyspace.table1 ("
+ String schema = "CREATE TABLE " + KS + '.' + TABLE + " ("
+ " k int PRIMARY KEY,"
+ " v1 text,"
+ " v2 int"
- + ");";// with compression = {};";
- String insert = "INSERT INTO cql_keyspace.table1 (k, v1, v2) VALUES (?, ?, ?)";
+ + ") with compression = " + (useSstableCompression ? "{'class': 'LZ4Compressor'};" : "{};");
+ String insert = "INSERT INTO " + KS + '.' + TABLE + " (k, v1, v2) VALUES (?, ?, ?)";
CQLSSTableWriter writer = CQLSSTableWriter.builder()
.sorted()
.inDirectory(dataDir)
.forTable(schema)
.using(insert).build();
+
+ CompressionParams compressionParams = Keyspace.open(KS).getColumnFamilyStore(TABLE).metadata().params.compression;
+ Assert.assertEquals(useSstableCompression, compressionParams.isEnabled());
+
+
long start = System.nanoTime();
for (int i = 0; i < 10_000_000; i++)
@@ -104,15 +120,15 @@
private String ks;
public void init(String keyspace)
{
- for (Range<Token> range : StorageService.instance.getLocalRanges("cql_keyspace"))
- addRangeForEndpoint(range, FBUtilities.getBroadcastAddress());
+ for (Replica range : StorageService.instance.getLocalReplicas(KS))
+ addRangeForEndpoint(range.range(), FBUtilities.getBroadcastAddressAndPort());
this.ks = keyspace;
}
- public CFMetaData getTableMetadata(String cfName)
+ public TableMetadataRef getTableMetadata(String cfName)
{
- return Schema.instance.getCFMetaData(ks, cfName);
+ return Schema.instance.getTableMetadataRef(ks, cfName);
}
}, new OutputHandler.SystemOutput(false, false));
@@ -131,15 +147,15 @@
private String ks;
public void init(String keyspace)
{
- for (Range<Token> range : StorageService.instance.getLocalRanges("cql_keyspace"))
- addRangeForEndpoint(range, FBUtilities.getBroadcastAddress());
+ for (Replica range : StorageService.instance.getLocalReplicas(KS))
+ addRangeForEndpoint(range.range(), FBUtilities.getBroadcastAddressAndPort());
this.ks = keyspace;
}
- public CFMetaData getTableMetadata(String cfName)
+ public TableMetadataRef getTableMetadata(String cfName)
{
- return Schema.instance.getCFMetaData(ks, cfName);
+ return Schema.instance.getTableMetadataRef(ks, cfName);
}
}, new OutputHandler.SystemOutput(false, false));
@@ -161,7 +177,7 @@
millis / 1000d,
(dataSize * 2 / (1 << 20) / (millis / 1000d)) * 8));
- UntypedResultSet rs = QueryProcessor.executeInternal("SELECT * FROM cql_keyspace.table1 limit 100;");
+ UntypedResultSet rs = QueryProcessor.executeInternal("SELECT * FROM " + KS + '.' + TABLE + " limit 100;");
assertEquals(100, rs.size());
}
}
diff --git a/test/long/org/apache/cassandra/utils/LongBitSetTest.java b/test/long/org/apache/cassandra/utils/LongBitSetTest.java
deleted file mode 100644
index f20a4f8..0000000
--- a/test/long/org/apache/cassandra/utils/LongBitSetTest.java
+++ /dev/null
@@ -1,134 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.utils;
-
-import java.util.Random;
-import java.util.concurrent.TimeUnit;
-
-import org.junit.Assert;
-
-import org.apache.cassandra.utils.obs.OffHeapBitSet;
-import org.apache.cassandra.utils.obs.OpenBitSet;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class LongBitSetTest
-{
- private static final Logger logger = LoggerFactory.getLogger(LongBitSetTest.class);
- private static final Random random = new Random();
-
- public void populateRandom(OffHeapBitSet offbs, OpenBitSet obs, long index)
- {
- if (random.nextBoolean())
- {
- offbs.set(index);
- obs.set(index);
- }
- }
-
- public void compare(OffHeapBitSet offbs, OpenBitSet obs, long index)
- {
- if (offbs.get(index) != obs.get(index))
- throw new RuntimeException();
- Assert.assertEquals(offbs.get(index), obs.get(index));
- }
-
- @Test
- public void testBitSetOperations()
- {
- long size_to_test = Integer.MAX_VALUE / 40;
- long size_and_excess = size_to_test + 20;
- OffHeapBitSet offbs = new OffHeapBitSet(size_and_excess);
- OpenBitSet obs = new OpenBitSet(size_and_excess);
- for (long i = 0; i < size_to_test; i++)
- populateRandom(offbs, obs, i);
-
- for (long i = 0; i < size_to_test; i++)
- compare(offbs, obs, i);
- }
-
- @Test
- public void timeit()
- {
- long size_to_test = Integer.MAX_VALUE / 10; // about 214 million
- long size_and_excess = size_to_test + 20;
-
- OpenBitSet obs = new OpenBitSet(size_and_excess);
- OffHeapBitSet offbs = new OffHeapBitSet(size_and_excess);
- logger.info("||Open BS set's|Open BS get's|Open BS clear's|Offheap BS set's|Offheap BS get's|Offheap BS clear's|");
- // System.out.println("||Open BS set's|Open BS get's|Open BS clear's|Offheap BS set's|Offheap BS get's|Offheap BS clear's|");
- loopOnce(obs, offbs, size_to_test);
- }
-
- public void loopOnce(OpenBitSet obs, OffHeapBitSet offbs, long size_to_test)
- {
- StringBuffer buffer = new StringBuffer();
- // start off fresh.
- System.gc();
- long start = System.nanoTime();
- for (long i = 0; i < size_to_test; i++)
- obs.set(i);
- buffer.append("||").append(TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start));
-
- start = System.nanoTime();
- for (long i = 0; i < size_to_test; i++)
- obs.get(i);
- buffer.append("|").append(TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start));
-
- start = System.nanoTime();
- for (long i = 0; i < size_to_test; i++)
- obs.clear(i);
- buffer.append("|").append(TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start));
-
- System.gc();
- start = System.nanoTime();
- for (long i = 0; i < size_to_test; i++)
- offbs.set(i);
- buffer.append("|").append(TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start));
-
- start = System.nanoTime();
- for (long i = 0; i < size_to_test; i++)
- offbs.get(i);
- buffer.append("|").append(TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start));
-
- start = System.nanoTime();
- for (long i = 0; i < size_to_test; i++)
- offbs.clear(i);
- buffer.append("|").append(TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start)).append("|");
- logger.info(buffer.toString());
- // System.out.println(buffer.toString());
- }
-
- /**
- * Just to make sure JIT doesn't come on our way
- */
- @Test
- // @Ignore
- public void loopIt()
- {
- long size_to_test = Integer.MAX_VALUE / 10; // about 214 million
- long size_and_excess = size_to_test + 20;
-
- OpenBitSet obs = new OpenBitSet(size_and_excess);
- OffHeapBitSet offbs = new OffHeapBitSet(size_and_excess);
- for (int i = 0; i < 10; i++)
- // 10 times to do approx 2B keys each.
- loopOnce(obs, offbs, size_to_test);
- }
-}
diff --git a/test/long/org/apache/cassandra/utils/LongBloomFilterTest.java b/test/long/org/apache/cassandra/utils/LongBloomFilterTest.java
index c50296d..d998e4d 100644
--- a/test/long/org/apache/cassandra/utils/LongBloomFilterTest.java
+++ b/test/long/org/apache/cassandra/utils/LongBloomFilterTest.java
@@ -38,33 +38,23 @@
@Test
public void testBigInt()
{
- testBigInt(false);
- testBigInt(true);
- }
- private static void testBigInt(boolean oldBfHashOrder)
- {
int size = 10 * 1000 * 1000;
- IFilter bf = getFilter(size, FilterTestHelper.spec.bucketsPerElement, false, oldBfHashOrder);
+ IFilter bf = getFilter(size, FilterTestHelper.spec.bucketsPerElement);
double fp = testFalsePositives(bf,
new KeyGenerator.IntGenerator(size),
new KeyGenerator.IntGenerator(size, size * 2));
- logger.info("Bloom filter false positive for oldBfHashOrder={}: {}", oldBfHashOrder, fp);
+ logger.info("Bloom filter false positive: {}", fp);
}
@Test
public void testBigRandom()
{
- testBigRandom(false);
- testBigRandom(true);
- }
- private static void testBigRandom(boolean oldBfHashOrder)
- {
int size = 10 * 1000 * 1000;
- IFilter bf = getFilter(size, FilterTestHelper.spec.bucketsPerElement, false, oldBfHashOrder);
+ IFilter bf = getFilter(size, FilterTestHelper.spec.bucketsPerElement);
double fp = testFalsePositives(bf,
new KeyGenerator.RandomStringGenerator(new Random().nextInt(), size),
new KeyGenerator.RandomStringGenerator(new Random().nextInt(), size));
- logger.info("Bloom filter false positive for oldBfHashOrder={}: {}", oldBfHashOrder, fp);
+ logger.info("Bloom filter false positive: {}", fp);
}
/**
@@ -73,26 +63,21 @@
@Test
public void testConstrained()
{
- testConstrained(false);
- testConstrained(true);
- }
- private static void testConstrained(boolean oldBfHashOrder)
- {
int size = 10 * 1000 * 1000;
- try (IFilter bf = getFilter(size, 0.01, false, oldBfHashOrder))
+ try (IFilter bf = getFilter(size, 0.01))
{
double fp = testFalsePositives(bf,
new KeyGenerator.IntGenerator(size),
new KeyGenerator.IntGenerator(size, size * 2));
- logger.info("Bloom filter false positive for oldBfHashOrder={}: {}", oldBfHashOrder, fp);
+ logger.info("Bloom filter false positive: {}", fp);
}
}
- private static void testConstrained(double targetFp, int elements, boolean oldBfHashOrder, int staticBitCount, long ... staticBits)
+ private static void testConstrained(double targetFp, int elements, int staticBitCount, long ... staticBits)
{
for (long bits : staticBits)
{
- try (IFilter bf = getFilter(elements, targetFp, false, oldBfHashOrder);)
+ try (IFilter bf = getFilter(elements, targetFp))
{
SequentialHashGenerator gen = new SequentialHashGenerator(staticBitCount, bits);
long[] hash = new long[2];
@@ -131,23 +116,17 @@
@Test
public void testBffp()
{
- bffp(false);
- bffp(true);
- }
-
- private static void bffp(boolean flipInputs)
- {
- System.out.println("Bloom filter false posiitive with flipInputs=" + flipInputs);
+ System.out.println("Bloom filter false posiitive");
long[] staticBits = staticBits(4, 0);
- testConstrained(0.01d, 10 << 20, flipInputs, 0, staticBits);
- testConstrained(0.01d, 1 << 20, flipInputs, 6, staticBits);
- testConstrained(0.01d, 10 << 20, flipInputs, 6, staticBits);
- testConstrained(0.01d, 1 << 19, flipInputs, 10, staticBits);
- testConstrained(0.01d, 1 << 20, flipInputs, 10, staticBits);
- testConstrained(0.01d, 10 << 20, flipInputs, 10, staticBits);
- testConstrained(0.1d, 10 << 20, flipInputs, 0, staticBits);
- testConstrained(0.1d, 10 << 20, flipInputs, 8, staticBits);
- testConstrained(0.1d, 10 << 20, flipInputs, 10, staticBits);
+ testConstrained(0.01d, 10 << 20, 0, staticBits);
+ testConstrained(0.01d, 1 << 20, 6, staticBits);
+ testConstrained(0.01d, 10 << 20, 6, staticBits);
+ testConstrained(0.01d, 1 << 19, 10, staticBits);
+ testConstrained(0.01d, 1 << 20, 10, staticBits);
+ testConstrained(0.01d, 10 << 20, 10, staticBits);
+ testConstrained(0.1d, 10 << 20, 0, staticBits);
+ testConstrained(0.1d, 10 << 20, 8, staticBits);
+ testConstrained(0.1d, 10 << 20, 10, staticBits);
}
static long[] staticBits(int random, long ... fixed)
@@ -180,13 +159,8 @@
@Test
public void timeit()
{
- timeit(false);
- timeit(true);
- }
- private static void timeit(boolean oldBfHashOrder)
- {
int size = 300 * FilterTestHelper.ELEMENTS;
- IFilter bf = getFilter(size, FilterTestHelper.spec.bucketsPerElement, false, oldBfHashOrder);
+ IFilter bf = getFilter(size, FilterTestHelper.spec.bucketsPerElement);
double sumfp = 0;
for (int i = 0; i < 10; i++)
{
@@ -196,6 +170,6 @@
bf.clear();
}
- logger.info("Bloom filter mean false positive for oldBfHashOrder={}: {}", oldBfHashOrder, sumfp / 10);
+ logger.info("Bloom filter mean false positive: {}", sumfp / 10);
}
}
diff --git a/test/memory/org/apache/cassandra/db/compaction/CompactionAllocationTest.java b/test/memory/org/apache/cassandra/db/compaction/CompactionAllocationTest.java
new file mode 100644
index 0000000..a58303d
--- /dev/null
+++ b/test/memory/org/apache/cassandra/db/compaction/CompactionAllocationTest.java
@@ -0,0 +1,767 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+import java.io.IOException;
+import java.lang.management.ManagementFactory;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.Set;
+
+import com.google.common.base.Joiner;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.common.primitives.Ints;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.monitoring.runtime.instrumentation.AllocationRecorder;
+import com.google.monitoring.runtime.instrumentation.Sampler;
+import com.sun.management.ThreadMXBean;
+import org.apache.cassandra.OrderedJUnit4ClassRunner;
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.cql3.QueryOptions;
+import org.apache.cassandra.cql3.QueryProcessor;
+import org.apache.cassandra.cql3.statements.SelectStatement;
+import org.apache.cassandra.cql3.statements.schema.CreateTableStatement;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.ReadExecutionController;
+import org.apache.cassandra.db.ReadQuery;
+import org.apache.cassandra.db.filter.ColumnFilter;
+import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
+import org.apache.cassandra.db.partitions.UnfilteredPartitionIterators;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.util.DataOutputBuffer;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.io.util.UnbufferedDataOutputStreamPlus;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.service.QueryState;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.ObjectSizes;
+
+@RunWith(OrderedJUnit4ClassRunner.class)
+public class CompactionAllocationTest
+{
+ private static final Logger logger = LoggerFactory.getLogger(CompactionAllocationTest.class);
+ private static final ThreadMXBean threadMX = (ThreadMXBean) ManagementFactory.getThreadMXBean();
+
+ private static final boolean AGENT_MEASUREMENT = true;
+
+ private static final boolean PROFILING_READS = false;
+ private static final boolean PROFILING_COMPACTION = false;
+ private static final boolean PROFILING = PROFILING_READS || PROFILING_COMPACTION;
+ private static final List<String> summaries = new ArrayList<>();
+
+ private static class CompactionSummary
+ {
+ final Measurement measurement;
+ final int numPartitions;
+ final int numRows;
+
+ public CompactionSummary(Measurement measurement, int numPartitions, int numRows)
+ {
+ this.measurement = measurement;
+ this.numPartitions = numPartitions;
+ this.numRows = numRows;
+ }
+
+ List<String> cells()
+ {
+ long b = measurement.bytes();
+ return Lists.newArrayList(Long.toString(b), Long.toString(b/numPartitions), Long.toString(b/numRows));
+ }
+
+ static final List<String> HEADERS = Lists.newArrayList("bytes", "/p", "/r");
+ static final List<String> EMPTY = Lists.newArrayList("n/a", "n/a", "n/a");
+ }
+
+ private static class ReadSummary
+ {
+ final Measurement measurement;
+ final int numReads;
+
+ public ReadSummary(Measurement measurement, int numReads)
+ {
+ this.measurement = measurement;
+ this.numReads = numReads;
+ }
+
+ List<String> cells()
+ {
+ long b = measurement.bytes();
+ return Lists.newArrayList(Long.toString(b), Long.toString(b/numReads));
+ }
+ static final List<String> HEADERS = Lists.newArrayList("bytes", "/rd");
+ static final List<String> EMPTY = Lists.newArrayList("n/a", "n/a");
+ }
+
+ private static final Map<String, CompactionSummary> compactionSummaries = new HashMap<>();
+ private static final Map<String, ReadSummary> readSummaries = new HashMap<>();
+
+ /*
+ add to jvm args:
+ -javaagent:${build.dir}/lib/jars/java-allocation-instrumenter-${allocation-instrumenter.version}.jar
+ */
+
+ private static final long MIN_OBJECTS_ALLOCATED;
+ private static final long MIN_BYTES_ALLOCATED;
+
+ static
+ {
+ if (AGENT_MEASUREMENT)
+ {
+ AgentMeasurement measurement = new AgentMeasurement();
+ measurement.start();
+ measurement.stop();
+ MIN_OBJECTS_ALLOCATED = measurement.objectsAllocated;
+ MIN_BYTES_ALLOCATED = measurement.bytesAllocated;
+ }
+ else
+ {
+ MIN_OBJECTS_ALLOCATED = 0;
+ MIN_BYTES_ALLOCATED = 0;
+ logger.warn("{} is using the ThreadMXBean to measure memory usage, this is less accurate than the allocation instrumenter agent", CompactionAllocationTest.class.getSimpleName());
+ logger.warn("If you're running this in your IDE, add the following jvm arg: " +
+ "-javaagent:<build.dir>/lib/jars/java-allocation-instrumenter-<allocation-instrumenter.version>.jar " +
+ "(and replace <> with appropriate values from build.xml)");
+ }
+ }
+
+ @BeforeClass
+ public static void setupClass() throws Throwable
+ {
+ SchemaLoader.prepareServer();
+ SchemaLoader.startGossiper();
+ testTinyPartitions("warmup", 9, maybeInflate(300), true);
+ }
+
+ @AfterClass
+ public static void afterClass()
+ {
+
+ logger.info("SUMMARIES:");
+ for (String summary : summaries)
+ logger.info(summary);
+
+
+ List<List<String>> groups = new ArrayList<>();
+ groups.add(Lists.newArrayList("tinyNonOverlapping3",
+ "tinyNonOverlapping9",
+ "tinyOverlapping3",
+ "tinyOverlapping9"));
+ groups.add(Lists.newArrayList("mediumNonOverlappingPartitions3",
+ "mediumNonOverlappingPartitions9",
+ "mediumOverlappingPartitions3",
+ "mediumOverlappingPartitions9",
+ "mediumPartitionsOverlappingRows3",
+ "mediumPartitionsOverlappingRows9"));
+ groups.add(Lists.newArrayList("wideNonOverlappingPartitions3",
+ "wideNonOverlappingPartitions9",
+ "wideOverlappingPartitions3",
+ "wideOverlappingPartitions9",
+ "widePartitionsOverlappingRows9",
+ "widePartitionsOverlappingRows3"));
+
+ Map<String, List<String>> fullRows = new HashMap<>();
+ for (String workload : Iterables.concat(groups))
+ {
+ CompactionSummary cs = compactionSummaries.get(workload);
+ ReadSummary rs = readSummaries.get(workload);
+ fullRows.put(workload, Lists.newArrayList(Iterables.concat(cs != null ? cs.cells() : CompactionSummary.EMPTY,
+ rs != null ? rs.cells() : ReadSummary.EMPTY)));
+ }
+ logger.info("");
+ logger.info("TAB DELIMITED:");
+ String header = Joiner.on('\t').join(Iterables.concat(CompactionSummary.HEADERS, ReadSummary.HEADERS));
+ for (List<String> group: groups)
+ {
+ logger.info(Joiner.on('\t').join(group));
+ logger.info(header);
+ logger.info(Joiner.on('\t').join(Iterables.concat(Iterables.transform(group, g -> fullRows.getOrDefault(g, Collections.emptyList())))));
+ }
+ }
+
+ private static int maybeInflate(int base, int inflate)
+ {
+ return PROFILING ? base * inflate : base;
+ }
+
+ private static int maybeInflate(int base)
+ {
+ return maybeInflate(base, 3);
+ }
+
+ private interface Workload
+ {
+ void setup();
+ ColumnFamilyStore getCfs();
+ String name();
+ List<Runnable> getReads();
+ }
+
+ private static Measurement createMeasurement()
+ {
+ return AGENT_MEASUREMENT ? new AgentMeasurement() : new MXMeasurement();
+ }
+
+ private interface Measurement
+ {
+ void start();
+
+ void stop();
+
+ long cpu();
+
+ long bytes();
+
+ long objects();
+
+ default String prettyBytes()
+ {
+ return FBUtilities.prettyPrintMemory(bytes());
+ }
+
+ }
+
+ public static class AgentMeasurement implements Measurement, Sampler
+ {
+ long objectsAllocated = 0;
+ long bytesAllocated = 0;
+
+ private final long threadID = Thread.currentThread().getId();
+
+ public void sampleAllocation(int count, String desc, Object newObj, long bytes)
+ {
+ if (Thread.currentThread().getId() != threadID)
+ return;
+ objectsAllocated++;
+ bytesAllocated += bytes;
+ }
+
+ public void start()
+ {
+ AllocationRecorder.addSampler(this);
+ }
+
+ public void stop()
+ {
+ AllocationRecorder.removeSampler(this);
+ if (bytesAllocated == 0)
+ logger.warn("no allocations recorded, make sure junit is run with -javaagent:${build.dir}/lib/jars/java-allocation-instrumenter-${allocation-instrumenter.version}.jar");
+ }
+
+ public long cpu()
+ {
+ return 0;
+ }
+
+ public long objects()
+ {
+ return objectsAllocated - MIN_OBJECTS_ALLOCATED;
+ }
+
+ public long bytes()
+ {
+ return bytesAllocated - MIN_BYTES_ALLOCATED;
+ }
+ }
+
+ public static class MXMeasurement implements Measurement
+ {
+ private final Thread thread = Thread.currentThread();
+
+ private class Point
+ {
+ long bytes;
+ long cpu;
+
+ void capture()
+ {
+ bytes = threadMX.getThreadAllocatedBytes(thread.getId());
+ cpu = threadMX.getThreadCpuTime(thread.getId());
+ }
+ }
+
+ private final Point start = new Point();
+ private final Point stop = new Point();
+
+ public void start()
+ {
+ start.capture();
+ }
+
+ public void stop()
+ {
+ stop.capture();
+ }
+
+ public long cpu()
+ {
+ return stop.cpu - start.cpu;
+ }
+
+ public long bytes()
+ {
+ return stop.bytes - start.bytes;
+ }
+
+ public long objects()
+ {
+ return 0;
+ }
+ }
+
+ @Test
+ public void allocMeasuring()
+ {
+ long size = ObjectSizes.measure(5);
+ int numAlloc = 1000;
+
+ Measurement measurement = createMeasurement();
+ measurement.start();
+ for (int i=0; i<numAlloc; i++)
+ new Integer(i);
+
+ measurement.stop();
+ logger.info(" ** {}", measurement.prettyBytes());
+ logger.info(" ** expected {}", size * numAlloc);
+ }
+
+ private static void measure(Workload workload) throws Throwable
+ {
+ workload.setup();
+
+ Measurement readSampler = createMeasurement();
+ Measurement compactionSampler = createMeasurement();
+
+ String readSummary = "SKIPPED";
+ if (!PROFILING_COMPACTION)
+ {
+ List<Runnable> reads = workload.getReads();
+ readSampler.start();
+ if (PROFILING_READS && !workload.name().equals("warmup"))
+ {
+ logger.info(">>> Start profiling");
+ Thread.sleep(10000);
+ }
+ for (int i=0; i<reads.size(); i++)
+ reads.get(i).run();
+ Thread.sleep(1000);
+ if (PROFILING_READS && !workload.name().equals("warmup"))
+ {
+ logger.info(">>> Stop profiling");
+ Thread.sleep(10000);
+ }
+ readSampler.stop();
+
+ readSummary = String.format("%s bytes, %s /read, %s cpu", readSampler.bytes(), readSampler.bytes()/reads.size(), readSampler.cpu());
+ readSummaries.put(workload.name(), new ReadSummary(readSampler, reads.size()));
+ }
+
+ ColumnFamilyStore cfs = workload.getCfs();
+ ActiveCompactions active = new ActiveCompactions();
+ Set<SSTableReader> sstables = cfs.getLiveSSTables();
+
+ CompactionTasks tasks = cfs.getCompactionStrategyManager()
+ .getUserDefinedTasks(sstables, FBUtilities.nowInSeconds());
+ Assert.assertFalse(tasks.isEmpty());
+
+ String compactionSummary = "SKIPPED";
+ if (!PROFILING_READS)
+ {
+ compactionSampler.start();
+ if (PROFILING_COMPACTION && !workload.name().equals("warmup"))
+ {
+ logger.info(">>> Start profiling");
+ Thread.sleep(10000);
+ }
+ for (AbstractCompactionTask task : tasks)
+ task.execute(active);
+ Thread.sleep(1000);
+ if (PROFILING_COMPACTION && !workload.name().equals("warmup"))
+ {
+ logger.info(">>> Stop profiling");
+ Thread.sleep(10000);
+ }
+ compactionSampler.stop();
+
+ Assert.assertEquals(1, cfs.getLiveSSTables().size());
+ int numPartitions = Ints.checkedCast(Iterables.getOnlyElement(cfs.getLiveSSTables()).getSSTableMetadata().estimatedPartitionSize.count());
+ int numRows = Ints.checkedCast(Iterables.getOnlyElement(cfs.getLiveSSTables()).getSSTableMetadata().totalRows);
+
+ compactionSummary = String.format("%s bytes, %s /partition, %s /row, %s cpu", compactionSampler.bytes(), compactionSampler.bytes()/numPartitions, compactionSampler.bytes()/numRows, compactionSampler.cpu());
+ compactionSummaries.put(workload.name(), new CompactionSummary(compactionSampler, numPartitions, numRows));
+ }
+
+ cfs.truncateBlocking();
+
+ logger.info("***");
+ logger.info("*** {} reads summary", workload.name());
+ logger.info(readSummary);
+ logger.info("*** {} compaction summary", workload.name());
+ logger.info(compactionSummary);
+ if (!workload.name().equals("warmup"))
+ {
+ summaries.add(workload.name() + " reads summary: " + readSummary);
+ summaries.add(workload.name() + " compaction summary: " + compactionSummary);
+ }
+ Thread.sleep(1000); // avoid losing report when running in IDE
+ }
+
+ private static final DataOutputPlus NOOP_OUT = new UnbufferedDataOutputStreamPlus()
+ {
+ public void write(byte[] buffer, int offset, int count) throws IOException {}
+
+ public void write(int oneByte) throws IOException {}
+ };
+
+ private static void runQuery(ReadQuery query, TableMetadata metadata)
+ {
+ try (ReadExecutionController executionController = query.executionController();
+ UnfilteredPartitionIterator partitions = query.executeLocally(executionController))
+ {
+ UnfilteredPartitionIterators.serializerForIntraNode().serialize(partitions, ColumnFilter.all(metadata), NOOP_OUT, MessagingService.current_version);
+ }
+ catch (IOException e)
+ {
+ throw new AssertionError(e);
+ }
+ }
+
+ private static void testTinyPartitions(String name, int numSSTable, int sstablePartitions, boolean overlap) throws Throwable
+ {
+ String ksname = "ks_" + name.toLowerCase();
+
+ SchemaLoader.createKeyspace(ksname, KeyspaceParams.simple(1),
+ CreateTableStatement.parse("CREATE TABLE tbl (k INT PRIMARY KEY, v INT)", ksname).build());
+
+ ColumnFamilyStore cfs = Schema.instance.getColumnFamilyStoreInstance(Schema.instance.getTableMetadata(ksname, "tbl").id);
+ Assert.assertNotNull(cfs);
+ cfs.disableAutoCompaction();
+ List<Runnable> reads = new ArrayList<>(numSSTable * (overlap ? 1 : sstablePartitions));
+
+ measure(new Workload()
+ {
+ public void setup()
+ {
+ cfs.disableAutoCompaction();
+ String insert = String.format("INSERT INTO %s.%s (k, v) VALUES (?,?)", ksname, "tbl");
+ String read = String.format("SELECT * FROM %s.%s WHERE k = ?", ksname, "tbl");
+ SelectStatement select = (SelectStatement) QueryProcessor.parseStatement(read).prepare(ClientState.forInternalCalls());
+ QueryState queryState = QueryState.forInternalCalls();
+ for (int f=0; f<numSSTable; f++)
+ {
+ for (int p = 0; p < sstablePartitions; p++)
+ {
+ int key = overlap ? p : (f * sstablePartitions) + p;
+ QueryProcessor.executeInternal(insert, key, key);
+ if (!overlap || f == 0)
+ {
+ QueryOptions options = QueryProcessor.makeInternalOptions(select, new Object[]{f});
+ ReadQuery query = select.getQuery(options, queryState.getNowInSeconds());
+ reads.add(() -> runQuery(query, cfs.metadata.get()));
+ }
+ }
+ cfs.forceBlockingFlush();
+ }
+
+ Assert.assertEquals(numSSTable, cfs.getLiveSSTables().size());
+ }
+
+ public List<Runnable> getReads()
+ {
+ return reads;
+ }
+
+ public ColumnFamilyStore getCfs()
+ {
+ return cfs;
+ }
+
+ public String name()
+ {
+ return name;
+ }
+ });
+ }
+
+ @Test
+ public void tinyNonOverlapping3() throws Throwable
+ {
+ testTinyPartitions("tinyNonOverlapping3", 3, maybeInflate(900, 6), false);
+ }
+
+ @Test
+ public void tinyNonOverlapping9() throws Throwable
+ {
+ testTinyPartitions("tinyNonOverlapping9", 9, maybeInflate(300, 6), false);
+ }
+
+ @Test
+ public void tinyOverlapping3() throws Throwable
+ {
+ testTinyPartitions("tinyOverlapping3", 3, maybeInflate(900, 6), true);
+ }
+
+ @Test
+ public void tinyOverlapping9() throws Throwable
+ {
+ testTinyPartitions("tinyOverlapping9", 9, maybeInflate(300, 6), true);
+ }
+
+ private static final Random globalRandom = new Random();
+ private static final Random localRandom = new Random();
+
+ public static String makeRandomString(int length)
+ {
+ return makeRandomString(length, -1);
+
+ }
+
+ public static String makeRandomString(int length, int seed)
+ {
+ Random r;
+ if (seed < 0)
+ {
+ r = globalRandom;
+ }
+ else
+ {
+ r = localRandom;
+ r.setSeed(seed);
+ }
+
+ char[] chars = new char[length];
+ for (int i = 0; i < length; ++i)
+ chars[i] = (char) ('a' + r.nextInt('z' - 'a' + 1));
+ return new String(chars);
+ }
+
+ private static void testMediumPartitions(String name, int numSSTable, int sstablePartitions, boolean overlap, boolean overlapCK) throws Throwable
+ {
+ String ksname = "ks_" + name.toLowerCase();
+
+ SchemaLoader.createKeyspace(ksname, KeyspaceParams.simple(1),
+ CreateTableStatement.parse("CREATE TABLE tbl (k text, c text, v1 text, v2 text, v3 text, v4 text, PRIMARY KEY (k, c))", ksname).build());
+
+ ColumnFamilyStore cfs = Schema.instance.getColumnFamilyStoreInstance(Schema.instance.getTableMetadata(ksname, "tbl").id);
+ Assert.assertNotNull(cfs);
+ cfs.disableAutoCompaction();
+ int rowsPerPartition = 200;
+ List<Runnable> reads = new ArrayList<>(numSSTable * (overlap ? 1 : sstablePartitions));
+ measure(new Workload()
+ {
+ public void setup()
+ {
+ cfs.disableAutoCompaction();
+ String insert = String.format("INSERT INTO %s.%s (k, c, v1, v2, v3, v4) VALUES (?, ?, ?, ?, ?, ?)", ksname, "tbl");
+ String read = String.format("SELECT * FROM %s.%s WHERE k = ?", ksname, "tbl");
+ SelectStatement select = (SelectStatement) QueryProcessor.parseStatement(read).prepare(ClientState.forInternalCalls());
+ QueryState queryState = QueryState.forInternalCalls();
+ for (int f=0; f<numSSTable; f++)
+ {
+ for (int p = 0; p < sstablePartitions; p++)
+ {
+ String key = String.format("%08d", overlap ? p : (f * sstablePartitions) + p);
+ for (int r=0; r<rowsPerPartition; r++)
+ {
+ QueryProcessor.executeInternal(insert, key, makeRandomString(6, overlapCK ? r : -1),
+ makeRandomString(8), makeRandomString(8),
+ makeRandomString(8), makeRandomString(8));
+
+ }
+ if (!overlap || f == 0)
+ {
+ QueryOptions options = QueryProcessor.makeInternalOptions(select, new Object[]{key});
+ ReadQuery query = select.getQuery(options, queryState.getNowInSeconds());
+ reads.add(() -> runQuery(query, cfs.metadata.get()));
+ }
+ }
+ cfs.forceBlockingFlush();
+ }
+
+ Assert.assertEquals(numSSTable, cfs.getLiveSSTables().size());
+ }
+
+ public ColumnFamilyStore getCfs()
+ {
+ return cfs;
+ }
+
+ public List<Runnable> getReads()
+ {
+ return reads;
+ }
+
+ public String name()
+ {
+ return name;
+ }
+ });
+ }
+
+ @Test
+ public void mediumNonOverlappingPartitions3() throws Throwable
+ {
+ testMediumPartitions("mediumNonOverlappingPartitions3", 3, maybeInflate(60), false, false);
+ }
+
+ @Test
+ public void mediumNonOverlappingPartitions9() throws Throwable
+ {
+ testMediumPartitions("mediumNonOverlappingPartitions9", 9, maybeInflate(20), false, false);
+ }
+
+ @Test
+ public void mediumOverlappingPartitions3() throws Throwable
+ {
+ testMediumPartitions("mediumOverlappingPartitions3", 3, maybeInflate(60), true, false);
+ }
+
+ @Test
+ public void mediumOverlappingPartitions9() throws Throwable
+ {
+ testMediumPartitions("mediumOverlappingPartitions9", 9, maybeInflate(20), true, false);
+ }
+
+ @Test
+ public void mediumPartitionsOverlappingRows3() throws Throwable
+ {
+ testMediumPartitions("mediumPartitionsOverlappingRows3", 3, maybeInflate(60), true, true);
+ }
+
+ @Test
+ public void mediumPartitionsOverlappingRows9() throws Throwable
+ {
+ testMediumPartitions("mediumPartitionsOverlappingRows9", 9, maybeInflate(20), true, true);
+ }
+
+ private static void testWidePartitions(String name, int numSSTable, int sstablePartitions, boolean overlap, boolean overlapCK) throws Throwable
+ {
+ String ksname = "ks_" + name.toLowerCase();
+
+ SchemaLoader.createKeyspace(ksname, KeyspaceParams.simple(1),
+ CreateTableStatement.parse("CREATE TABLE tbl (k text, c text, v1 text, v2 text, v3 text, v4 text, PRIMARY KEY (k, c))", ksname).build());
+
+ ColumnFamilyStore cfs = Schema.instance.getColumnFamilyStoreInstance(Schema.instance.getTableMetadata(ksname, "tbl").id);
+ Assert.assertNotNull(cfs);
+ cfs.disableAutoCompaction();
+ int rowWidth = 100;
+ int rowsPerPartition = 1000;
+ List<Runnable> reads = new ArrayList<>(numSSTable * (overlap ? 1 : sstablePartitions));
+
+ measure(new Workload()
+ {
+ public void setup()
+ {
+ cfs.disableAutoCompaction();
+ String insert = String.format("INSERT INTO %s.%s (k, c, v1, v2, v3, v4) VALUES (?, ?, ?, ?, ?, ?)", ksname, "tbl");
+ String read = String.format("SELECT * FROM %s.%s WHERE k = ?", ksname, "tbl");
+ SelectStatement select = (SelectStatement) QueryProcessor.parseStatement(read).prepare(ClientState.forInternalCalls());
+ QueryState queryState = QueryState.forInternalCalls();
+ for (int f=0; f<numSSTable; f++)
+ {
+ for (int p = 0; p < sstablePartitions; p++)
+ {
+ String key = String.format("%08d", overlap ? p : (f * sstablePartitions) + p);
+ for (int r=0; r<rowsPerPartition; r++)
+ {
+ QueryProcessor.executeInternal(insert , key, makeRandomString(6, overlapCK ? r : -1),
+ makeRandomString(rowWidth>>2), makeRandomString(rowWidth>>2),
+ makeRandomString(rowWidth>>2), makeRandomString(rowWidth>>2));
+ }
+ if (!overlap || f == 0)
+ {
+ QueryOptions options = QueryProcessor.makeInternalOptions(select, new Object[]{key});
+ ReadQuery query = select.getQuery(options, queryState.getNowInSeconds());
+ reads.add(() -> runQuery(query, cfs.metadata.get()));
+ }
+ }
+ cfs.forceBlockingFlush();
+ }
+
+ Assert.assertEquals(numSSTable, cfs.getLiveSSTables().size());
+ }
+
+ public ColumnFamilyStore getCfs()
+ {
+ return cfs;
+ }
+
+ public List<Runnable> getReads()
+ {
+ return reads;
+ }
+
+ public String name()
+ {
+ return name;
+ }
+ });
+ }
+
+ @Test
+ public void wideNonOverlappingPartitions3() throws Throwable
+ {
+ testWidePartitions("wideNonOverlappingPartitions3", 3, maybeInflate(24), false, false);
+ }
+
+ @Test
+ public void wideNonOverlappingPartitions9() throws Throwable
+ {
+ testWidePartitions("wideNonOverlappingPartitions9", 9, maybeInflate(8), false, false);
+ }
+
+ @Test
+ public void wideOverlappingPartitions3() throws Throwable
+ {
+ testWidePartitions("wideOverlappingPartitions3", 3, maybeInflate(24), true, false);
+ }
+
+ @Test
+ public void wideOverlappingPartitions9() throws Throwable
+ {
+ testWidePartitions("wideOverlappingPartitions9", 9, maybeInflate(8), true, false);
+ }
+
+ @Test
+ public void widePartitionsOverlappingRows9() throws Throwable
+ {
+ testWidePartitions("widePartitionsOverlappingRows9", 9, maybeInflate(8), true, true);
+ }
+
+ @Test
+ public void widePartitionsOverlappingRows3() throws Throwable
+ {
+ testWidePartitions("widePartitionsOverlappingRows3", 3, maybeInflate(24), true, true);
+ }
+}
diff --git a/test/microbench/org/apache/cassandra/test/microbench/AutoBoxingBench.java b/test/microbench/org/apache/cassandra/test/microbench/AutoBoxingBench.java
new file mode 100644
index 0000000..fd6df39
--- /dev/null
+++ b/test/microbench/org/apache/cassandra/test/microbench/AutoBoxingBench.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.test.microbench;
+
+import java.util.concurrent.TimeUnit;
+import java.util.function.BooleanSupplier;
+import java.util.function.IntSupplier;
+import java.util.function.Supplier;
+
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.annotations.Warmup;
+
+@BenchmarkMode(Mode.SampleTime)
+@OutputTimeUnit(TimeUnit.NANOSECONDS)
+@Warmup(iterations = 3, time = 1)
+@Measurement(iterations = 6, time = 20)
+@Fork(value = 1,jvmArgsAppend = { "-Xmx256M", "-Djmh.executor=CUSTOM", "-Djmh.executor.class=org.apache.cassandra.test.microbench.FastThreadExecutor"})
+@Threads(4) // make sure this matches the number of _physical_cores_
+@State(Scope.Benchmark)
+public class AutoBoxingBench
+{
+
+ @Benchmark
+ public boolean booleanFromBooleanSupplier()
+ {
+ BooleanSupplier bs = () -> true;
+ return bs.getAsBoolean();
+ }
+
+ @Benchmark
+ public boolean booleanFromPlainSupplier()
+ {
+ Supplier<Boolean> bs = () -> true;
+ return bs.get();
+ }
+
+ @Benchmark
+ public int intFromIntSupplier()
+ {
+ IntSupplier bs = () -> 42;
+ return bs.getAsInt();
+ }
+
+ @Benchmark
+ public int intFromPlainSupplier()
+ {
+ Supplier<Integer> bs = () -> 42;
+ return bs.get();
+ }
+}
diff --git a/test/microbench/org/apache/cassandra/test/microbench/BTreeBuildBench.java b/test/microbench/org/apache/cassandra/test/microbench/BTreeBuildBench.java
index 0d89ebb..f9ab004 100644
--- a/test/microbench/org/apache/cassandra/test/microbench/BTreeBuildBench.java
+++ b/test/microbench/org/apache/cassandra/test/microbench/BTreeBuildBench.java
@@ -93,4 +93,10 @@
Object[] btree = builder.build();
return BTree.size(btree);
}
+
+ @Benchmark
+ public int buildTreeTest()
+ {
+ return buildTree(data);
+ }
}
diff --git a/test/microbench/org/apache/cassandra/test/microbench/BTreeSearchIteratorBench.java b/test/microbench/org/apache/cassandra/test/microbench/BTreeSearchIteratorBench.java
new file mode 100644
index 0000000..400b297
--- /dev/null
+++ b/test/microbench/org/apache/cassandra/test/microbench/BTreeSearchIteratorBench.java
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.test.microbench;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.cassandra.utils.btree.BTreeSearchIterator;
+import org.apache.cassandra.utils.btree.BTree;
+import org.apache.cassandra.utils.btree.BTree.Dir;
+import org.apache.cassandra.utils.btree.FullBTreeSearchIterator;
+import org.apache.cassandra.utils.btree.LeafBTreeSearchIterator;
+import org.apache.cassandra.utils.btree.UpdateFunction;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OperationsPerInvocation;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.annotations.Warmup;
+
+@BenchmarkMode(Mode.SampleTime)
+@OutputTimeUnit(TimeUnit.NANOSECONDS)
+@Warmup(iterations = 1, time = 1, timeUnit = TimeUnit.SECONDS)
+@Measurement(iterations = 2, time = 4, timeUnit = TimeUnit.SECONDS)
+@Fork(value = 2)
+@Threads(1)
+@State(Scope.Benchmark)
+public class BTreeSearchIteratorBench
+{
+ private final int btreeSize = 32;
+
+ @Param({"TREE", "LEAF"})
+ private String iteratorType;
+
+ @Param({ "0", "1", "2", "3", "16", "17", "18", "19",
+ "24", "25", "26", "30", "31" })
+ private int targetIdx;
+
+ private final int cellSize = 1000;
+
+ @Param({"ASC", "DESC"})
+ private String dirParam;
+
+ private Dir dir;
+ private Object[] btree;
+ private ArrayList<String> data;
+ private ArrayList<String> nonExistData;
+
+ private static ArrayList<String> seq(int count, int minCellSize)
+ {
+ ArrayList<String> ret = new ArrayList<>();
+ for (int i = 0 ; i < count ; i++)
+ {
+ StringBuilder sb = new StringBuilder();
+ while (sb.length() < minCellSize)
+ {
+ String uuid = UUID.randomUUID().toString();
+ sb.append(uuid);
+ }
+ ret.add(sb.toString());
+ }
+ Collections.sort(ret);
+ return ret;
+ }
+
+ private static final Comparator<String> CMP = new Comparator<String>()
+ {
+ public int compare(String s1, String s2)
+ {
+ return s1.compareTo(s2);
+ }
+ };
+
+ @Setup(Level.Trial)
+ public void setup() throws Throwable
+ {
+ data = seq(btreeSize, cellSize);
+ nonExistData = new ArrayList<>();
+ btree = BTree.build(data, UpdateFunction.noOp());
+ for (String d : data)
+ {
+ nonExistData.add(d.substring(0, d.length() - 1) + "!");
+ }
+ dir = Dir.valueOf(dirParam);
+
+ }
+
+ @Benchmark
+ public void searchFound()
+ {
+ BTreeSearchIterator<String, String> iter = getIterator();
+ String val = iter.next(data.get(targetIdx));
+ assert(val != null);
+ }
+
+ private BTreeSearchIterator<String,String> getIterator()
+ {
+ switch (iteratorType)
+ {
+ case "LEAF":
+ return new LeafBTreeSearchIterator<>(btree, CMP, dir);
+ case "TREE":
+ return new FullBTreeSearchIterator<>(btree, CMP, dir);
+ default:
+ throw new IllegalArgumentException("unknown btree iterator type: " + iteratorType);
+ }
+ }
+
+ @Benchmark
+ public void searchNotFound()
+ {
+ BTreeSearchIterator<String, String> iter = getIterator();
+ String val = iter.next(nonExistData.get(targetIdx));
+ assert(val == null);
+ }
+}
diff --git a/test/microbench/org/apache/cassandra/test/microbench/BloomFilterSerializerBench.java b/test/microbench/org/apache/cassandra/test/microbench/BloomFilterSerializerBench.java
new file mode 100644
index 0000000..9222811
--- /dev/null
+++ b/test/microbench/org/apache/cassandra/test/microbench/BloomFilterSerializerBench.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.test.microbench;
+
+import java.io.DataInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.cassandra.db.BufferDecoratedKey;
+import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.io.util.BufferedDataOutputStreamPlus;
+import org.apache.cassandra.io.util.DataOutputStreamPlus;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.utils.BloomFilter;
+import org.apache.cassandra.utils.BloomFilterSerializer;
+import org.apache.cassandra.utils.FilterFactory;
+import org.apache.cassandra.utils.IFilter;
+import org.apache.cassandra.utils.SerializationsTest;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.MICROSECONDS)
+@Warmup(iterations = 1, time = 1, timeUnit = TimeUnit.SECONDS)
+@Measurement(iterations = 2, time = 4, timeUnit = TimeUnit.SECONDS)
+@Fork(value = 2)
+@State(Scope.Benchmark)
+public class BloomFilterSerializerBench
+{
+
+ @Param({"1", "10", "100", "1024"})
+ private long numElemsInK;
+
+ @Param({"true", "false"})
+ public boolean oldBfFormat;
+
+ static final IFilter.FilterKey wrap(ByteBuffer buf)
+ {
+ return new BufferDecoratedKey(new Murmur3Partitioner.LongToken(0L), buf);
+ }
+
+ private ByteBuffer testVal = ByteBuffer.wrap(new byte[] { 0, 1});
+
+ @Benchmark
+ public void serializationTest() throws IOException
+ {
+ File file = FileUtils.createTempFile("bloomFilterTest-", ".dat");
+ try
+ {
+ BloomFilter filter = (BloomFilter) FilterFactory.getFilter(numElemsInK * 1024, 0.01d);
+ filter.add(wrap(testVal));
+ DataOutputStreamPlus out = new BufferedDataOutputStreamPlus(new FileOutputStream(file));
+ if (oldBfFormat)
+ SerializationsTest.serializeOldBfFormat(filter, out);
+ else
+ BloomFilterSerializer.serialize(filter, out);
+ out.close();
+ filter.close();
+
+ DataInputStream in = new DataInputStream(new FileInputStream(file));
+ BloomFilter filter2 = BloomFilterSerializer.deserialize(in, oldBfFormat);
+ FileUtils.closeQuietly(in);
+ filter2.close();
+ }
+ finally
+ {
+ file.delete();
+ }
+ }
+
+}
diff --git a/test/microbench/org/apache/cassandra/test/microbench/ChecksummingTransformerLz4.java b/test/microbench/org/apache/cassandra/test/microbench/ChecksummingTransformerLz4.java
new file mode 100644
index 0000000..7ac62fe
--- /dev/null
+++ b/test/microbench/org/apache/cassandra/test/microbench/ChecksummingTransformerLz4.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.test.microbench;
+
+import java.io.BufferedReader;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
+import java.util.EnumSet;
+import java.util.concurrent.TimeUnit;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.ByteBufUtil;
+import io.netty.buffer.Unpooled;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.transport.Frame;
+import org.apache.cassandra.transport.frame.checksum.ChecksummingTransformer;
+import org.apache.cassandra.transport.frame.compress.LZ4Compressor;
+import org.apache.cassandra.utils.ChecksumType;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+
+@Warmup(iterations = 4, time = 1)
+@Measurement(iterations = 8, time = 2)
+@Fork(value = 2)
+@State(Scope.Benchmark)
+public class ChecksummingTransformerLz4
+{
+ private static final EnumSet<Frame.Header.Flag> FLAGS = EnumSet.of(Frame.Header.Flag.COMPRESSED, Frame.Header.Flag.CHECKSUMMED);
+
+ static {
+ DatabaseDescriptor.clientInitialization();
+ }
+
+ private final ChecksummingTransformer transformer = ChecksummingTransformer.getTransformer(ChecksumType.CRC32, LZ4Compressor.INSTANCE);
+ private ByteBuf smallEnglishASCIICompressed;
+ private ByteBuf smallEnglishUtf8Compressed;
+ private ByteBuf largeBlobCompressed;
+
+ @Setup
+ public void setup() throws IOException
+ {
+ byte[] smallEnglishASCII = "this is small".getBytes(StandardCharsets.US_ASCII);
+ this.smallEnglishASCIICompressed = transformer.transformOutbound(Unpooled.wrappedBuffer(smallEnglishASCII));
+ byte[] smallEnglishUtf8 = "this is small".getBytes(StandardCharsets.UTF_8);
+ this.smallEnglishUtf8Compressed = transformer.transformOutbound(Unpooled.wrappedBuffer(smallEnglishUtf8));
+
+ String failureHex;
+ try (BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream("test/data/CASSANDRA-15313/lz4-jvm-crash-failure.txt"), StandardCharsets.UTF_8))) {
+ failureHex = reader.readLine().trim();
+ }
+ byte[] failure = ByteBufUtil.decodeHexDump(failureHex);
+ this.largeBlobCompressed = transformer.transformOutbound(Unpooled.wrappedBuffer(failure));
+ }
+
+ @Benchmark
+ public ByteBuf decompresSsmallEnglishASCII() {
+ smallEnglishASCIICompressed.readerIndex(0);
+ return transformer.transformInbound(smallEnglishASCIICompressed, FLAGS);
+ }
+
+ @Benchmark
+ public ByteBuf decompresSsmallEnglishUtf8() {
+ smallEnglishUtf8Compressed.readerIndex(0);
+ return transformer.transformInbound(smallEnglishUtf8Compressed, FLAGS);
+ }
+
+ @Benchmark
+ public ByteBuf decompresLargeBlob() {
+ largeBlobCompressed.readerIndex(0);
+ return transformer.transformInbound(largeBlobCompressed, FLAGS);
+ }
+}
diff --git a/test/microbench/org/apache/cassandra/test/microbench/CompactionBench.java b/test/microbench/org/apache/cassandra/test/microbench/CompactionBench.java
index d8dfd66..41220a2 100644
--- a/test/microbench/org/apache/cassandra/test/microbench/CompactionBench.java
+++ b/test/microbench/org/apache/cassandra/test/microbench/CompactionBench.java
@@ -21,51 +21,16 @@
import java.io.File;
import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.Collection;
import java.util.List;
import java.util.concurrent.*;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
-import com.google.common.util.concurrent.Uninterruptibles;
-
-import org.apache.cassandra.UpdateBuilder;
-import org.apache.cassandra.concurrent.StageManager;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.Config;
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.config.Schema;
import org.apache.cassandra.cql3.CQLTester;
-import org.apache.cassandra.cql3.statements.ParsedStatement;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Directories;
import org.apache.cassandra.db.Keyspace;
-import org.apache.cassandra.db.Mutation;
-import org.apache.cassandra.db.compaction.CompactionManager;
-import org.apache.cassandra.dht.Murmur3Partitioner;
import org.apache.cassandra.io.sstable.Descriptor;
-import org.apache.cassandra.io.util.DataInputBuffer;
-import org.apache.cassandra.io.util.DataOutputBuffer;
-import org.apache.cassandra.io.util.DataOutputBufferFixed;
import org.apache.cassandra.io.util.FileUtils;
-import org.apache.cassandra.net.MessageIn;
-import org.apache.cassandra.net.MessageOut;
-import org.apache.cassandra.net.MessagingService;
-import org.apache.cassandra.schema.KeyspaceMetadata;
-import org.apache.cassandra.schema.KeyspaceParams;
-import org.apache.cassandra.service.CassandraDaemon;
-import org.apache.cassandra.service.StorageService;
-import org.apache.cassandra.transport.messages.ResultMessage;
-import org.apache.cassandra.utils.FBUtilities;
-import org.apache.hadoop.util.bloom.Key;
import org.openjdk.jmh.annotations.*;
-import org.openjdk.jmh.profile.StackProfiler;
-import org.openjdk.jmh.results.Result;
-import org.openjdk.jmh.results.RunResult;
-import org.openjdk.jmh.runner.Runner;
-import org.openjdk.jmh.runner.options.Options;
-import org.openjdk.jmh.runner.options.OptionsBuilder;
@BenchmarkMode(Mode.AverageTime)
@OutputTimeUnit(TimeUnit.MILLISECONDS)
diff --git a/test/microbench/org/apache/cassandra/test/microbench/DiagnosticEventPersistenceBench.java b/test/microbench/org/apache/cassandra/test/microbench/DiagnosticEventPersistenceBench.java
new file mode 100644
index 0000000..5ddf0c5
--- /dev/null
+++ b/test/microbench/org/apache/cassandra/test/microbench/DiagnosticEventPersistenceBench.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.test.microbench;
+
+import java.util.concurrent.TimeUnit;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.config.OverrideConfigurationLoader;
+import org.apache.cassandra.diag.DiagnosticEvent;
+import org.apache.cassandra.diag.DiagnosticEventPersistence;
+import org.apache.cassandra.diag.DiagnosticEventService;
+import org.apache.cassandra.test.microbench.DiagnosticEventServiceBench.DummyEvent;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.annotations.Warmup;
+
+@BenchmarkMode(Mode.All)
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+@Warmup(iterations = 4, time = 1, timeUnit = TimeUnit.SECONDS)
+@Measurement(iterations = 8, time = 2, timeUnit = TimeUnit.SECONDS)
+@Fork(value = 2)
+@Threads(4)
+@State(Scope.Benchmark)
+public class DiagnosticEventPersistenceBench
+{
+ private DiagnosticEventService service = DiagnosticEventService.instance();
+ private DiagnosticEventPersistence persistence = DiagnosticEventPersistence.instance();
+ private DiagnosticEvent event = new DummyEvent();
+
+ @Setup
+ public void setup()
+ {
+ OverrideConfigurationLoader.override((config) -> {
+ config.diagnostic_events_enabled = true;
+ });
+ DatabaseDescriptor.daemonInitialization();
+
+ service.cleanup();
+
+ // make persistence subscribe to and store events
+ persistence.enableEventPersistence(DummyEvent.class.getName());
+ }
+
+ @Benchmark
+ public void persistEvents()
+ {
+ service.publish(event);
+ }
+}
diff --git a/test/microbench/org/apache/cassandra/test/microbench/DiagnosticEventServiceBench.java b/test/microbench/org/apache/cassandra/test/microbench/DiagnosticEventServiceBench.java
new file mode 100644
index 0000000..351138e
--- /dev/null
+++ b/test/microbench/org/apache/cassandra/test/microbench/DiagnosticEventServiceBench.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.test.microbench;
+
+import java.io.Serializable;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Consumer;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.config.OverrideConfigurationLoader;
+import org.apache.cassandra.diag.DiagnosticEvent;
+import org.apache.cassandra.diag.DiagnosticEventService;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.annotations.Warmup;
+
+@BenchmarkMode(Mode.Throughput)
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+@Warmup(iterations = 4, time = 1, timeUnit = TimeUnit.SECONDS)
+@Measurement(iterations = 8, time = 2, timeUnit = TimeUnit.SECONDS)
+@Fork(value = 2)
+@Threads(4)
+@State(Scope.Benchmark)
+public class DiagnosticEventServiceBench
+{
+ private DiagnosticEventService service = DiagnosticEventService.instance();
+ private DiagnosticEvent event = new DummyEvent();
+
+ @Param({ "0", "1", "12" })
+ private int subscribers = 0;
+
+ @Setup
+ public void setup()
+ {
+ OverrideConfigurationLoader.override((config) -> {
+ config.diagnostic_events_enabled = true;
+ });
+ DatabaseDescriptor.daemonInitialization();
+
+ service.cleanup();
+
+ for (int i = 0; i < subscribers; i++)
+ {
+ service.subscribe(DummyEvent.class, new Consumer<DummyEvent>()
+ {
+ public void accept(DummyEvent dummyEvent)
+ {
+ // No-op
+ }
+ });
+ }
+ }
+
+ @Benchmark
+ public void publishEvents()
+ {
+ service.publish(event);
+ }
+
+ final static class DummyEvent extends DiagnosticEvent
+ {
+ public Enum<?> getType()
+ {
+ return null;
+ }
+
+ public Object getSource()
+ {
+ return null;
+ }
+
+ public Map<String, Serializable> toMap()
+ {
+ return null;
+ }
+ }
+}
diff --git a/test/microbench/org/apache/cassandra/test/microbench/HashingBench.java b/test/microbench/org/apache/cassandra/test/microbench/HashingBench.java
new file mode 100644
index 0000000..b6c81a6
--- /dev/null
+++ b/test/microbench/org/apache/cassandra/test/microbench/HashingBench.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.test.microbench;
+
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+
+import com.google.common.hash.Hasher;
+import com.google.common.hash.Hashing;
+
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.annotations.Warmup;
+
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.NANOSECONDS)
+@Warmup(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS)
+@Measurement(iterations = 5, time = 2, timeUnit = TimeUnit.SECONDS)
+@Fork(value = 1,jvmArgsAppend = { "-Xmx512M", "-Djmh.executor=CUSTOM", "-Djmh.executor.class=org.apache.cassandra.test.microbench.FastThreadExecutor"})
+@Threads(4) // make sure this matches the number of _physical_cores_
+@State(Scope.Benchmark)
+public class HashingBench
+{
+ private static final Random random = new Random(12345678);
+
+ private static final MessageDigest messageDigest;
+ static
+ {
+ try
+ {
+ messageDigest = MessageDigest.getInstance("MD5");
+ }
+ catch (NoSuchAlgorithmException nsae)
+ {
+ throw new RuntimeException("MD5 digest algorithm is not available", nsae);
+ }
+ }
+
+
+ // intentionally not on power-of-2 values
+ @Param({ "31", "131", "517", "2041" })
+ private int bufferSize;
+
+ private byte[] array;
+
+ @Setup
+ public void setup() throws NoSuchAlgorithmException
+ {
+ array = new byte[bufferSize];
+ random.nextBytes(array);
+ }
+
+ @Benchmark
+ public byte[] benchMessageDigestMD5()
+ {
+ try
+ {
+ MessageDigest clone = (MessageDigest) messageDigest.clone();
+ clone.update(array);
+ return clone.digest();
+ }
+ catch (CloneNotSupportedException e)
+ {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Benchmark
+ public byte[] benchHasherMD5()
+ {
+ Hasher md5Hasher = Hashing.md5().newHasher();
+ md5Hasher.putBytes(array);
+ return md5Hasher.hash().asBytes();
+ }
+
+ @Benchmark
+ public byte[] benchHasherMurmur3_128()
+ {
+ Hasher murmur3_128Hasher = Hashing.murmur3_128().newHasher();
+ murmur3_128Hasher.putBytes(array);
+ return murmur3_128Hasher.hash().asBytes();
+ }
+}
diff --git a/test/microbench/org/apache/cassandra/test/microbench/LatencyTrackingBench.java b/test/microbench/org/apache/cassandra/test/microbench/LatencyTrackingBench.java
new file mode 100644
index 0000000..28e0da7
--- /dev/null
+++ b/test/microbench/org/apache/cassandra/test/microbench/LatencyTrackingBench.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.test.microbench;
+
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+
+import com.codahale.metrics.Histogram;
+import com.codahale.metrics.Timer;
+import org.apache.cassandra.metrics.CassandraMetricsRegistry;
+import org.apache.cassandra.metrics.ClearableHistogram;
+import org.apache.cassandra.metrics.DecayingEstimatedHistogramReservoir;
+import org.apache.cassandra.metrics.LatencyMetrics;
+import org.apache.cassandra.metrics.LatencyMetricsTest;
+import org.apache.cassandra.metrics.MetricNameFactory;
+import org.apache.cassandra.metrics.TableMetrics;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.CompilerControl;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OperationsPerInvocation;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.infra.Blackhole;
+
+@BenchmarkMode(Mode.Throughput)
+@OutputTimeUnit(TimeUnit.SECONDS)
+@Warmup(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS)
+@Measurement(iterations = 5, time = 2, timeUnit = TimeUnit.SECONDS)
+@Fork(value = 1,jvmArgsAppend = { "-Xmx512M", "-Djmh.executor=CUSTOM", "-Djmh.executor.class=org.apache.cassandra.test.microbench.FastThreadExecutor"})
+@Threads(4) // make sure this matches the number of _physical_cores_
+@State(Scope.Benchmark)
+public class LatencyTrackingBench
+{
+ private LatencyMetrics metrics;
+ private LatencyMetrics parent;
+ private LatencyMetrics grandParent;
+ private DecayingEstimatedHistogramReservoir dehr;
+ private final MetricNameFactory factory = new BenchMetricsNameFactory();
+ private long[] values = new long[1024];
+
+ class BenchMetricsNameFactory implements MetricNameFactory
+ {
+
+ @Override
+ public CassandraMetricsRegistry.MetricName createMetricName(String metricName)
+ {
+ return new CassandraMetricsRegistry.MetricName(BenchMetricsNameFactory.class, metricName);
+ }
+ }
+
+ @Setup(Level.Iteration)
+ public void setup()
+ {
+ parent = new LatencyMetrics("test", "testCF");
+ grandParent = new LatencyMetrics("test", "testCF");
+
+ // Replicates behavior from ColumnFamilyStore metrics
+ metrics = new LatencyMetrics(factory, "testCF", parent, grandParent);
+ dehr = new DecayingEstimatedHistogramReservoir(false);
+ for(int i = 0; i < 1024; i++)
+ {
+ values[i] = TimeUnit.MICROSECONDS.toNanos(ThreadLocalRandom.current().nextLong(346));
+ }
+ }
+
+ @Setup(Level.Invocation)
+ public void reset()
+ {
+ dehr = new DecayingEstimatedHistogramReservoir(false);
+ metrics.release();
+ metrics = new LatencyMetrics(factory, "testCF", parent, grandParent);
+ }
+
+ @Benchmark
+ @OperationsPerInvocation(1024)
+ public void benchLatencyMetricsWrite()
+ {
+ for(int i = 0; i < values.length; i++)
+ {
+ metrics.addNano(values[i]);
+ }
+ }
+
+ @Benchmark
+ @OperationsPerInvocation(1024)
+ public void benchInsertToDEHR(Blackhole bh)
+ {
+ for(int i = 0; i < values.length; i++)
+ {
+ dehr.update(values[i]);
+ }
+ bh.consume(dehr);
+ }
+}
diff --git a/test/microbench/org/apache/cassandra/test/microbench/MessageOutBench.java b/test/microbench/org/apache/cassandra/test/microbench/MessageOutBench.java
new file mode 100644
index 0000000..4ab607f
--- /dev/null
+++ b/test/microbench/org/apache/cassandra/test/microbench/MessageOutBench.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.test.microbench;
+
+import java.io.IOException;
+import java.util.EnumMap;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+
+import com.google.common.net.InetAddresses;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.Unpooled;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.io.util.DataInputBuffer;
+import org.apache.cassandra.io.util.DataOutputBuffer;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.net.Message;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.net.NoPayload;
+import org.apache.cassandra.net.ParamType;
+import org.apache.cassandra.utils.UUIDGen;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+
+import static org.apache.cassandra.net.Verb.ECHO_REQ;
+
+@State(Scope.Thread)
+@Warmup(iterations = 4, time = 1, timeUnit = TimeUnit.SECONDS)
+@Measurement(iterations = 8, time = 4, timeUnit = TimeUnit.SECONDS)
+@Fork(value = 1,jvmArgsAppend = "-Xmx512M")
+@OutputTimeUnit(TimeUnit.NANOSECONDS)
+@BenchmarkMode(Mode.SampleTime)
+public class MessageOutBench
+{
+ @Param({ "true", "false" })
+ private boolean withParams;
+
+ private Message msgOut;
+ private ByteBuf buf;
+ private InetAddressAndPort addr;
+
+ @Setup
+ public void setup()
+ {
+ DatabaseDescriptor.daemonInitialization();
+
+ UUID uuid = UUIDGen.getTimeUUID();
+ Map<ParamType, Object> parameters = new EnumMap<>(ParamType.class);
+
+ if (withParams)
+ {
+ parameters.put(ParamType.TRACE_SESSION, uuid);
+ }
+
+ addr = InetAddressAndPort.getByAddress(InetAddresses.forString("127.0.73.101"));
+ msgOut = Message.builder(ECHO_REQ, NoPayload.noPayload)
+ .from(addr)
+ .build();
+ buf = Unpooled.buffer(1024, 1024); // 1k should be enough for everybody!
+ }
+
+ @Benchmark
+ public int serialize40() throws Exception
+ {
+ return serialize(MessagingService.VERSION_40);
+ }
+
+ private int serialize(int messagingVersion) throws IOException
+ {
+ try (DataOutputBuffer out = new DataOutputBuffer())
+ {
+ Message.serializer.serialize(Message.builder(msgOut).withCreatedAt(System.nanoTime()).withId(42).build(),
+ out, messagingVersion);
+ DataInputBuffer in = new DataInputBuffer(out.buffer(), false);
+ Message.serializer.deserialize(in, addr, messagingVersion);
+ return msgOut.serializedSize(messagingVersion);
+ }
+ }
+
+ @Benchmark
+ public int serializePre40() throws Exception
+ {
+ return serialize(MessagingService.VERSION_30);
+ }
+}
diff --git a/test/microbench/org/apache/cassandra/test/microbench/MutationBench.java b/test/microbench/org/apache/cassandra/test/microbench/MutationBench.java
index 8c177cf..41d6aab 100644
--- a/test/microbench/org/apache/cassandra/test/microbench/MutationBench.java
+++ b/test/microbench/org/apache/cassandra/test/microbench/MutationBench.java
@@ -25,17 +25,15 @@
import java.util.concurrent.*;
import org.apache.cassandra.UpdateBuilder;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.Config;
+import org.apache.cassandra.cql3.statements.schema.CreateTableStatement;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.config.Schema;
+import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.db.Mutation;
import org.apache.cassandra.dht.Murmur3Partitioner;
import org.apache.cassandra.io.util.DataInputBuffer;
import org.apache.cassandra.io.util.DataOutputBuffer;
import org.apache.cassandra.io.util.DataOutputBufferFixed;
-import org.apache.cassandra.net.MessageIn;
-import org.apache.cassandra.net.MessageOut;
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.schema.KeyspaceMetadata;
import org.apache.cassandra.schema.KeyspaceParams;
@@ -73,7 +71,6 @@
static String keyspace = "keyspace1";
private Mutation mutation;
- private MessageOut<Mutation> messageOut;
private ByteBuffer buffer;
private DataOutputBuffer outputBuffer;
@@ -83,7 +80,7 @@
@State(Scope.Thread)
public static class ThreadState
{
- MessageIn<Mutation> in;
+ Mutation in;
int counter = 0;
}
@@ -91,31 +88,30 @@
public void setup() throws IOException
{
Schema.instance.load(KeyspaceMetadata.create(keyspace, KeyspaceParams.simple(1)));
- KeyspaceMetadata ksm = Schema.instance.getKSMetaData(keyspace);
- CFMetaData metadata = CFMetaData.compile("CREATE TABLE userpics " +
- "( userid bigint," +
- "picid bigint," +
- "commentid bigint, " +
- "PRIMARY KEY(userid, picid))", keyspace);
+ KeyspaceMetadata ksm = Schema.instance.getKeyspaceMetadata(keyspace);
+ TableMetadata metadata =
+ CreateTableStatement.parse("CREATE TABLE userpics " +
+ "( userid bigint," +
+ "picid bigint," +
+ "commentid bigint, " +
+ "PRIMARY KEY(userid, picid))", keyspace)
+ .build();
- Schema.instance.load(metadata);
- Schema.instance.setKeyspaceMetadata(ksm.withSwapped(ksm.tables.with(metadata)));
-
+ Schema.instance.load(ksm.withSwapped(ksm.tables.with(metadata)));
mutation = (Mutation)UpdateBuilder.create(metadata, 1L).newRow(1L).add("commentid", 32L).makeMutation();
- messageOut = mutation.createMessage();
- buffer = ByteBuffer.allocate(messageOut.serializedSize(MessagingService.current_version));
+ buffer = ByteBuffer.allocate(mutation.serializedSize(MessagingService.current_version));
outputBuffer = new DataOutputBufferFixed(buffer);
inputBuffer = new DataInputBuffer(buffer, false);
- messageOut.serialize(outputBuffer, MessagingService.current_version);
+ Mutation.serializer.serialize(mutation, outputBuffer, MessagingService.current_version);
}
@Benchmark
public void serialize(ThreadState state) throws IOException
{
buffer.rewind();
- messageOut.serialize(outputBuffer, MessagingService.current_version);
+ Mutation.serializer.serialize(mutation, outputBuffer, MessagingService.current_version);
state.counter++;
}
@@ -123,7 +119,7 @@
public void deserialize(ThreadState state) throws IOException
{
buffer.rewind();
- state.in = MessageIn.read(inputBuffer, MessagingService.current_version, 0);
+ state.in = Mutation.serializer.deserialize(inputBuffer, MessagingService.current_version);
state.counter++;
}
diff --git a/test/microbench/org/apache/cassandra/test/microbench/PendingRangesBench.java b/test/microbench/org/apache/cassandra/test/microbench/PendingRangesBench.java
index 9ec1aa6..73a2b71 100644
--- a/test/microbench/org/apache/cassandra/test/microbench/PendingRangesBench.java
+++ b/test/microbench/org/apache/cassandra/test/microbench/PendingRangesBench.java
@@ -21,18 +21,22 @@
package org.apache.cassandra.test.microbench;
import com.google.common.collect.HashMultimap;
+import com.google.common.collect.Lists;
import com.google.common.collect.Multimap;
import org.apache.cassandra.dht.RandomPartitioner;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.locator.PendingRangeMaps;
+import org.apache.cassandra.locator.Replica;
+import org.apache.cassandra.locator.ReplicaUtils;
import org.openjdk.jmh.annotations.*;
import org.openjdk.jmh.infra.Blackhole;
-import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.Collection;
import java.util.HashSet;
+import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ThreadLocalRandom;
@@ -50,7 +54,7 @@
PendingRangeMaps pendingRangeMaps;
int maxToken = 256 * 100;
- Multimap<Range<Token>, InetAddress> oldPendingRanges;
+ Multimap<Range<Token>, Replica> oldPendingRanges;
private Range<Token> genRange(String left, String right)
{
@@ -63,15 +67,17 @@
pendingRangeMaps = new PendingRangeMaps();
oldPendingRanges = HashMultimap.create();
- InetAddress[] addresses = {InetAddress.getByName("127.0.0.1"), InetAddress.getByName("127.0.0.2")};
+ List<InetAddressAndPort> endpoints = Lists.newArrayList(InetAddressAndPort.getByName("127.0.0.1"),
+ InetAddressAndPort.getByName("127.0.0.2"));
for (int i = 0; i < maxToken; i++)
{
for (int j = 0; j < ThreadLocalRandom.current().nextInt(2); j ++)
{
Range<Token> range = genRange(Integer.toString(i * 10 + 5), Integer.toString(i * 10 + 15));
- pendingRangeMaps.addPendingRange(range, addresses[j]);
- oldPendingRanges.put(range, addresses[j]);
+ Replica replica = Replica.fullReplica(endpoints.get(j), range);
+ pendingRangeMaps.addPendingRange(range, replica);
+ oldPendingRanges.put(range, replica);
}
}
@@ -79,8 +85,9 @@
for (int j = 0; j < ThreadLocalRandom.current().nextInt(2); j ++)
{
Range<Token> range = genRange(Integer.toString(maxToken * 10 + 5), Integer.toString(5));
- pendingRangeMaps.addPendingRange(range, addresses[j]);
- oldPendingRanges.put(range, addresses[j]);
+ Replica replica = Replica.fullReplica(endpoints.get(j), range);
+ pendingRangeMaps.addPendingRange(range, replica);
+ oldPendingRanges.put(range, replica);
}
}
@@ -97,13 +104,13 @@
{
int randomToken = ThreadLocalRandom.current().nextInt(maxToken * 10 + 5);
Token searchToken = new RandomPartitioner.BigIntegerToken(Integer.toString(randomToken));
- Set<InetAddress> endpoints = new HashSet<>();
- for (Map.Entry<Range<Token>, Collection<InetAddress>> entry : oldPendingRanges.asMap().entrySet())
+ Set<Replica> replicas = new HashSet<>();
+ for (Map.Entry<Range<Token>, Collection<Replica>> entry : oldPendingRanges.asMap().entrySet())
{
if (entry.getKey().contains(searchToken))
- endpoints.addAll(entry.getValue());
+ replicas.addAll(entry.getValue());
}
- bh.consume(endpoints);
+ bh.consume(replicas);
}
}
diff --git a/test/microbench/org/apache/cassandra/test/microbench/PreaggregatedByteBufsBench.java b/test/microbench/org/apache/cassandra/test/microbench/PreaggregatedByteBufsBench.java
new file mode 100644
index 0000000..9971cc5
--- /dev/null
+++ b/test/microbench/org/apache/cassandra/test/microbench/PreaggregatedByteBufsBench.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.test.microbench;
+
+import java.util.concurrent.TimeUnit;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.channel.embedded.EmbeddedChannel;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+
+@State(Scope.Thread)
+@Warmup(iterations = 4, time = 1, timeUnit = TimeUnit.SECONDS)
+@Measurement(iterations = 8, time = 4, timeUnit = TimeUnit.SECONDS)
+@Fork(value = 1,jvmArgsAppend = "-Xmx512M")
+@OutputTimeUnit(TimeUnit.NANOSECONDS)
+@BenchmarkMode(Mode.SampleTime)
+public class PreaggregatedByteBufsBench
+{
+ @Param({ "1000", "2500", "5000", "10000", "20000", "40000"})
+ private int len;
+
+ private static final int subBufferCount = 64;
+
+ private EmbeddedChannel channel;
+
+ @Setup
+ public void setUp()
+ {
+ channel = new EmbeddedChannel();
+ }
+
+ @Benchmark
+ public boolean oneBigBuf()
+ {
+ boolean success = true;
+ try
+ {
+ ByteBuf buf = channel.alloc().directBuffer(len);
+ buf.writerIndex(len);
+ channel.writeAndFlush(buf);
+ }
+ catch (Exception e)
+ {
+ success = false;
+ }
+ finally
+ {
+ channel.releaseOutbound();
+ }
+
+ return success;
+ }
+
+ @Benchmark
+ public boolean chunkedBuf()
+ {
+ boolean success = true;
+ try
+ {
+ int chunkLen = len / subBufferCount;
+
+ for (int i = 0; i < subBufferCount; i++)
+ {
+ ByteBuf buf = channel.alloc().directBuffer(chunkLen);
+ buf.writerIndex(chunkLen);
+ channel.write(buf);
+ }
+ channel.flush();
+ }
+ catch (Exception e)
+ {
+ success = false;
+ }
+ finally
+ {
+ channel.releaseOutbound();
+ }
+
+ return success;
+ }
+}
diff --git a/test/microbench/org/apache/cassandra/test/microbench/ReadWriteTest.java b/test/microbench/org/apache/cassandra/test/microbench/ReadWriteTest.java
index 89973fd..066c289 100644
--- a/test/microbench/org/apache/cassandra/test/microbench/ReadWriteTest.java
+++ b/test/microbench/org/apache/cassandra/test/microbench/ReadWriteTest.java
@@ -19,44 +19,13 @@
package org.apache.cassandra.test.microbench;
-import java.io.File;
import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.Collection;
-import java.util.List;
import java.util.concurrent.*;
-import org.apache.cassandra.UpdateBuilder;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.Config;
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.config.Schema;
import org.apache.cassandra.cql3.CQLTester;
-import org.apache.cassandra.cql3.statements.ParsedStatement;
import org.apache.cassandra.db.ColumnFamilyStore;
-import org.apache.cassandra.db.Directories;
import org.apache.cassandra.db.Keyspace;
-import org.apache.cassandra.db.Mutation;
-import org.apache.cassandra.dht.Murmur3Partitioner;
-import org.apache.cassandra.io.util.DataInputBuffer;
-import org.apache.cassandra.io.util.DataOutputBuffer;
-import org.apache.cassandra.io.util.DataOutputBufferFixed;
-import org.apache.cassandra.io.util.FileUtils;
-import org.apache.cassandra.net.MessageIn;
-import org.apache.cassandra.net.MessageOut;
-import org.apache.cassandra.net.MessagingService;
-import org.apache.cassandra.schema.KeyspaceMetadata;
-import org.apache.cassandra.schema.KeyspaceParams;
-import org.apache.cassandra.service.CassandraDaemon;
-import org.apache.cassandra.service.StorageService;
-import org.apache.cassandra.transport.messages.ResultMessage;
import org.openjdk.jmh.annotations.*;
-import org.openjdk.jmh.profile.StackProfiler;
-import org.openjdk.jmh.results.Result;
-import org.openjdk.jmh.results.RunResult;
-import org.openjdk.jmh.runner.Runner;
-import org.openjdk.jmh.runner.options.Options;
-import org.openjdk.jmh.runner.options.OptionsBuilder;
@BenchmarkMode(Mode.Throughput)
@OutputTimeUnit(TimeUnit.MILLISECONDS)
diff --git a/test/microbench/org/apache/cassandra/test/microbench/Sample.java b/test/microbench/org/apache/cassandra/test/microbench/Sample.java
index 1f149c0..52f7c3e 100644
--- a/test/microbench/org/apache/cassandra/test/microbench/Sample.java
+++ b/test/microbench/org/apache/cassandra/test/microbench/Sample.java
@@ -20,7 +20,7 @@
import net.jpountz.lz4.LZ4Compressor;
import net.jpountz.lz4.LZ4Factory;
-import net.jpountz.lz4.LZ4FastDecompressor;
+import net.jpountz.lz4.LZ4SafeDecompressor;
import org.openjdk.jmh.annotations.*;
import org.xerial.snappy.Snappy;
@@ -56,7 +56,7 @@
private byte[][] snappyBytes;
private byte[][] rawBytes;
- private LZ4FastDecompressor lz4Decompressor = LZ4Factory.fastestInstance().fastDecompressor();
+ private LZ4SafeDecompressor lz4Decompressor = LZ4Factory.fastestInstance().safeDecompressor();
private LZ4Compressor lz4Compressor = LZ4Factory.fastestInstance().fastCompressor();
diff --git a/test/microbench/org/apache/cassandra/test/microbench/StreamingHistogramBench.java b/test/microbench/org/apache/cassandra/test/microbench/StreamingHistogramBench.java
deleted file mode 100644
index c1ecf6d..0000000
--- a/test/microbench/org/apache/cassandra/test/microbench/StreamingHistogramBench.java
+++ /dev/null
@@ -1,403 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.test.microbench;
-
-
-import java.io.IOException;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.TimeUnit;
-import java.util.Random;
-
-import org.apache.cassandra.utils.StreamingHistogram;
-import org.openjdk.jmh.annotations.*;
-
-@BenchmarkMode(Mode.AverageTime)
-@OutputTimeUnit(TimeUnit.MILLISECONDS)
-@Warmup(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS)
-@Measurement(iterations = 5, time = 2, timeUnit = TimeUnit.SECONDS)
-@Fork(value = 1)
-@Threads(1)
-@State(Scope.Benchmark)
-public class StreamingHistogramBench
-{
- StreamingHistogram.StreamingHistogramBuilder streamingHistogram;
- StreamingHistogram.StreamingHistogramBuilder newStreamingHistogram;
- StreamingHistogram.StreamingHistogramBuilder newStreamingHistogram2;
- StreamingHistogram.StreamingHistogramBuilder newStreamingHistogram3;
- StreamingHistogram.StreamingHistogramBuilder newStreamingHistogram4;
- StreamingHistogram.StreamingHistogramBuilder newStreamingHistogram5;
- StreamingHistogram.StreamingHistogramBuilder newStreamingHistogram6;
- StreamingHistogram.StreamingHistogramBuilder streamingHistogram60;
- StreamingHistogram.StreamingHistogramBuilder newStreamingHistogram60;
- StreamingHistogram.StreamingHistogramBuilder newStreamingHistogram100x60;
-
- StreamingHistogram.StreamingHistogramBuilder narrowstreamingHistogram;
- StreamingHistogram.StreamingHistogramBuilder narrownewStreamingHistogram;
- StreamingHistogram.StreamingHistogramBuilder narrownewStreamingHistogram2;
- StreamingHistogram.StreamingHistogramBuilder narrownewStreamingHistogram3;
- StreamingHistogram.StreamingHistogramBuilder narrownewStreamingHistogram4;
- StreamingHistogram.StreamingHistogramBuilder narrownewStreamingHistogram5;
- StreamingHistogram.StreamingHistogramBuilder narrownewStreamingHistogram6;
- StreamingHistogram.StreamingHistogramBuilder narrownewStreamingHistogram60;
- StreamingHistogram.StreamingHistogramBuilder narrowstreamingHistogram60;
- StreamingHistogram.StreamingHistogramBuilder narrownewStreamingHistogram100x60;
-
- StreamingHistogram.StreamingHistogramBuilder sparsestreamingHistogram;
- StreamingHistogram.StreamingHistogramBuilder sparsenewStreamingHistogram;
- StreamingHistogram.StreamingHistogramBuilder sparsenewStreamingHistogram2;
- StreamingHistogram.StreamingHistogramBuilder sparsenewStreamingHistogram3;
- StreamingHistogram.StreamingHistogramBuilder sparsenewStreamingHistogram4;
- StreamingHistogram.StreamingHistogramBuilder sparsenewStreamingHistogram5;
- StreamingHistogram.StreamingHistogramBuilder sparsenewStreamingHistogram6;
- StreamingHistogram.StreamingHistogramBuilder sparsestreamingHistogram60;
- StreamingHistogram.StreamingHistogramBuilder sparsenewStreamingHistogram60;
- StreamingHistogram.StreamingHistogramBuilder sparsenewStreamingHistogram100x60;
-
- static int[] ttls = new int[10000000];
- static int[] narrowttls = new int[10000000];
- static int[] sparsettls = new int[10000000];
- static
- {
- Random random = new Random();
- for(int i = 0 ; i < 10000000; i++)
- {
- // Seconds in a day
- ttls[i] = random.nextInt(86400);
- // Seconds in 3 hours
- narrowttls[i] = random.nextInt(14400);
- // Seconds in a minute
- sparsettls[i] = random.nextInt(60);
- }
- }
-
- @Setup(Level.Trial)
- public void setup() throws Throwable
- {
-
- streamingHistogram = new StreamingHistogram.StreamingHistogramBuilder(100, 0, 1);
- newStreamingHistogram = new StreamingHistogram.StreamingHistogramBuilder(100, 1000, 1);
- newStreamingHistogram2 = new StreamingHistogram.StreamingHistogramBuilder(100, 10000, 1);
- newStreamingHistogram3 = new StreamingHistogram.StreamingHistogramBuilder(100, 100000, 1);
- newStreamingHistogram4 = new StreamingHistogram.StreamingHistogramBuilder(50, 100000, 1);
- newStreamingHistogram5 = new StreamingHistogram.StreamingHistogramBuilder(50, 10000,1 );
- newStreamingHistogram6 = new StreamingHistogram.StreamingHistogramBuilder(100, 1000000, 1);
- streamingHistogram60 = new StreamingHistogram.StreamingHistogramBuilder(100, 0, 60);
- newStreamingHistogram60 = new StreamingHistogram.StreamingHistogramBuilder(100, 100000, 60);
- newStreamingHistogram100x60 = new StreamingHistogram.StreamingHistogramBuilder(100, 10000, 60);
-
- narrowstreamingHistogram = new StreamingHistogram.StreamingHistogramBuilder(100, 0, 1);
- narrownewStreamingHistogram = new StreamingHistogram.StreamingHistogramBuilder(100, 1000, 1);
- narrownewStreamingHistogram2 = new StreamingHistogram.StreamingHistogramBuilder(100, 10000, 1);
- narrownewStreamingHistogram3 = new StreamingHistogram.StreamingHistogramBuilder(100, 100000, 1);
- narrownewStreamingHistogram4 = new StreamingHistogram.StreamingHistogramBuilder(50, 100000, 1);
- narrownewStreamingHistogram5 = new StreamingHistogram.StreamingHistogramBuilder(50, 10000, 1);
- narrownewStreamingHistogram6 = new StreamingHistogram.StreamingHistogramBuilder(100, 1000000, 1);
- narrowstreamingHistogram60 = new StreamingHistogram.StreamingHistogramBuilder(100, 0, 60);
- narrownewStreamingHistogram60 = new StreamingHistogram.StreamingHistogramBuilder(100, 100000, 60);
- narrownewStreamingHistogram100x60 = new StreamingHistogram.StreamingHistogramBuilder(100, 10000, 60);
-
-
- sparsestreamingHistogram = new StreamingHistogram.StreamingHistogramBuilder(100, 0, 1);
- sparsenewStreamingHistogram = new StreamingHistogram.StreamingHistogramBuilder(100, 1000, 1);
- sparsenewStreamingHistogram2 = new StreamingHistogram.StreamingHistogramBuilder(100, 10000, 1);
- sparsenewStreamingHistogram3 = new StreamingHistogram.StreamingHistogramBuilder(100, 100000, 1);
- sparsenewStreamingHistogram4 = new StreamingHistogram.StreamingHistogramBuilder(50, 100000, 1);
- sparsenewStreamingHistogram5 = new StreamingHistogram.StreamingHistogramBuilder(50, 10000, 1);
- sparsenewStreamingHistogram6 = new StreamingHistogram.StreamingHistogramBuilder(100, 1000000, 1);
- sparsestreamingHistogram60 = new StreamingHistogram.StreamingHistogramBuilder(100, 0, 60);
- sparsenewStreamingHistogram60 = new StreamingHistogram.StreamingHistogramBuilder(100, 100000, 60);
- sparsenewStreamingHistogram100x60 = new StreamingHistogram.StreamingHistogramBuilder(100, 10000, 60);
-
- }
-
- @TearDown(Level.Trial)
- public void teardown() throws IOException, ExecutionException, InterruptedException
- {
-
- }
-
- @Benchmark
- public void existingSH() throws Throwable
- {
- for(int i = 0 ; i < ttls.length; i++)
- streamingHistogram.update(ttls[i]);
- streamingHistogram.build();
- }
-
- @Benchmark
- public void newSH10x() throws Throwable
- {
- for(int i = 0 ; i < ttls.length; i++)
- newStreamingHistogram.update(ttls[i]);
- newStreamingHistogram.build();
-
- }
-
- @Benchmark
- public void newSH100x() throws Throwable
- {
- for(int i = 0 ; i < ttls.length; i++)
- newStreamingHistogram2.update(ttls[i]);
- newStreamingHistogram2.build();
-
- }
-
- @Benchmark
- public void newSH1000x() throws Throwable
- {
- for(int i = 0 ; i < ttls.length; i++)
- newStreamingHistogram3.update(ttls[i]);
- newStreamingHistogram3.build();
-
- }
-
- @Benchmark
- public void newSH10000x() throws Throwable
- {
- for(int i = 0 ; i < ttls.length; i++)
- newStreamingHistogram6.update(ttls[i]);
- newStreamingHistogram6.build();
-
- }
-
-
- @Benchmark
- public void newSH50and1000() throws Throwable
- {
- for(int i = 0 ; i < ttls.length; i++)
- newStreamingHistogram4.update(ttls[i]);
- newStreamingHistogram4.build();
-
- }
-
- @Benchmark
- public void newSH50and100x() throws Throwable
- {
- for(int i = 0 ; i < ttls.length; i++)
- newStreamingHistogram5.update(ttls[i]);
- newStreamingHistogram5.build();
-
- }
-
- @Benchmark
- public void streaminghistogram60s() throws Throwable
- {
- for(int i = 0 ; i < ttls.length; i++)
- streamingHistogram60.update(sparsettls[i]);
- streamingHistogram60.build();
-
- }
-
- @Benchmark
- public void newstreaminghistogram1000x60s() throws Throwable
- {
- for(int i = 0 ; i < ttls.length; i++)
- newStreamingHistogram60.update(sparsettls[i]);
- newStreamingHistogram60.build();
- }
-
- @Benchmark
- public void newstreaminghistogram100x60s() throws Throwable
- {
- for(int i = 0 ; i < ttls.length; i++)
- newStreamingHistogram100x60.update(sparsettls[i]);
- newStreamingHistogram100x60.build();
- }
-
-
- @Benchmark
- public void narrowexistingSH() throws Throwable
- {
- for(int i = 0 ; i < ttls.length; i++)
- narrowstreamingHistogram.update(narrowttls[i]);
- narrowstreamingHistogram.build();
- }
-
- @Benchmark
- public void narrownewSH10x() throws Throwable
- {
- for(int i = 0 ; i < ttls.length; i++)
- narrownewStreamingHistogram.update(narrowttls[i]);
- narrownewStreamingHistogram.build();
-
- }
-
- @Benchmark
- public void narrownewSH100x() throws Throwable
- {
- for(int i = 0 ; i < ttls.length; i++)
- narrownewStreamingHistogram2.update(narrowttls[i]);
- narrownewStreamingHistogram2.build();
-
- }
-
- @Benchmark
- public void narrownewSH1000x() throws Throwable
- {
- for(int i = 0 ; i < ttls.length; i++)
- narrownewStreamingHistogram3.update(narrowttls[i]);
- narrownewStreamingHistogram3.build();
-
- }
-
- @Benchmark
- public void narrownewSH10000x() throws Throwable
- {
- for(int i = 0 ; i < ttls.length; i++)
- narrownewStreamingHistogram6.update(ttls[i]);
- narrownewStreamingHistogram6.build();
-
- }
-
-
- @Benchmark
- public void narrownewSH50and1000x() throws Throwable
- {
- for(int i = 0 ; i < ttls.length; i++)
- narrownewStreamingHistogram4.update(narrowttls[i]);
- narrownewStreamingHistogram4.build();
-
- }
-
- @Benchmark
- public void narrownewSH50and100x() throws Throwable
- {
- for(int i = 0 ; i < ttls.length; i++)
- narrownewStreamingHistogram5.update(narrowttls[i]);
- narrownewStreamingHistogram5.build();
-
- }
-
- @Benchmark
- public void narrowstreaminghistogram60s() throws Throwable
- {
- for(int i = 0 ; i < ttls.length; i++)
- narrowstreamingHistogram60.update(sparsettls[i]);
- narrowstreamingHistogram60.build();
-
- }
-
- @Benchmark
- public void narrownewstreaminghistogram1000x60s() throws Throwable
- {
- for(int i = 0 ; i < ttls.length; i++)
- narrownewStreamingHistogram60.update(sparsettls[i]);
- narrownewStreamingHistogram60.build();
-
- }
-
- @Benchmark
- public void narrownewstreaminghistogram100x60s() throws Throwable
- {
- for(int i = 0 ; i < ttls.length; i++)
- narrownewStreamingHistogram100x60.update(sparsettls[i]);
- narrownewStreamingHistogram100x60.build();
-
- }
-
-
- @Benchmark
- public void sparseexistingSH() throws Throwable
- {
- for(int i = 0 ; i < ttls.length; i++)
- sparsestreamingHistogram.update(sparsettls[i]);
- sparsestreamingHistogram.build();
- }
-
- @Benchmark
- public void sparsenewSH10x() throws Throwable
- {
- for(int i = 0 ; i < ttls.length; i++)
- sparsenewStreamingHistogram.update(sparsettls[i]);
- sparsenewStreamingHistogram.build();
-
- }
-
- @Benchmark
- public void sparsenewSH100x() throws Throwable
- {
- for(int i = 0 ; i < ttls.length; i++)
- sparsenewStreamingHistogram2.update(sparsettls[i]);
- sparsenewStreamingHistogram2.build();
-
- }
-
- @Benchmark
- public void sparsenewSH1000x() throws Throwable
- {
- for(int i = 0 ; i < ttls.length; i++)
- sparsenewStreamingHistogram3.update(sparsettls[i]);
- sparsenewStreamingHistogram3.build();
-
- }
-
- @Benchmark
- public void sparsenewSH10000x() throws Throwable
- {
- for(int i = 0 ; i < ttls.length; i++)
- sparsenewStreamingHistogram6.update(ttls[i]);
- sparsenewStreamingHistogram6.build();
- }
-
-
- @Benchmark
- public void sparsenewSH50and1000x() throws Throwable
- {
- for(int i = 0 ; i < ttls.length; i++)
- sparsenewStreamingHistogram4.update(sparsettls[i]);
- sparsenewStreamingHistogram4.build();
-
- }
-
- @Benchmark
- public void sparsenewSH50and100x() throws Throwable
- {
- for(int i = 0 ; i < ttls.length; i++)
- sparsenewStreamingHistogram5.update(sparsettls[i]);
- sparsenewStreamingHistogram5.build();
-
- }
-
- @Benchmark
- public void sparsestreaminghistogram60s() throws Throwable
- {
- for(int i = 0 ; i < ttls.length; i++)
- sparsestreamingHistogram60.update(sparsettls[i]);
- sparsestreamingHistogram60.build();
-
- }
-
- @Benchmark
- public void sparsenewstreaminghistogram1000x60() throws Throwable
- {
- for(int i = 0 ; i < ttls.length; i++)
- sparsenewStreamingHistogram60.update(sparsettls[i]);
- sparsenewStreamingHistogram60.build();
-
- }
-
- @Benchmark
- public void sparsenewstreaminghistogram100x60() throws Throwable
- {
- for(int i = 0 ; i < ttls.length; i++)
- sparsenewStreamingHistogram100x60.update(sparsettls[i]);
- sparsenewStreamingHistogram100x60.build();
-
- }
-}
diff --git a/test/microbench/org/apache/cassandra/test/microbench/StreamingTombstoneHistogramBuilderBench.java b/test/microbench/org/apache/cassandra/test/microbench/StreamingTombstoneHistogramBuilderBench.java
new file mode 100755
index 0000000..f3df8c1
--- /dev/null
+++ b/test/microbench/org/apache/cassandra/test/microbench/StreamingTombstoneHistogramBuilderBench.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.test.microbench;
+
+
+import java.util.concurrent.TimeUnit;
+import java.util.Random;
+
+import org.apache.cassandra.utils.streamhist.StreamingTombstoneHistogramBuilder;
+import org.openjdk.jmh.annotations.*;
+import org.openjdk.jmh.profile.*;
+import org.openjdk.jmh.runner.*;
+import org.openjdk.jmh.runner.options.*;
+
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+@Warmup(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS)
+@Measurement(iterations = 5, time = 2, timeUnit = TimeUnit.SECONDS)
+@Fork(value = 1)
+@Threads(1)
+@State(Scope.Benchmark)
+public class StreamingTombstoneHistogramBuilderBench
+{
+
+ static int[] secondInMonth = new int[10000000];
+ static int[] secondInDay = new int[10000000];
+ static int[] secondIn3Hour = new int[10000000];
+ static int[] secondInMin = new int[10000000];
+
+ static
+ {
+ final int now = (int) (System.currentTimeMillis() / 1000L);
+ Random random = new Random();
+ for(int i = 0 ; i < 10000000; i++)
+ {
+ // Seconds in a month
+ secondInMonth[i] = now + random.nextInt(3600 * 24 * 30);
+ // Seconds in a day
+ secondInDay[i] = now + random.nextInt(3600 * 24);
+ // Seconds in 3 hours
+ secondIn3Hour[i] = now + random.nextInt(3600 * 3);
+ // Seconds in a minute
+ secondInMin[i] = now + random.nextInt(60);
+ }
+ }
+
+ @Param({ "secondInMonth", "secondInDay", "secondIn3Hour", "secondInMin" })
+ String a_workLoad;
+
+ @Param({ "0", "1000", "10000", "100000" })
+ int b_spoolSize;
+
+ @Benchmark
+ public void test()
+ {
+ StreamingTombstoneHistogramBuilder histogram = new StreamingTombstoneHistogramBuilder(100, b_spoolSize, 1);
+ int[] data = selectWorkload(a_workLoad);
+
+ for (int time : data)
+ {
+ histogram.update(time);
+ }
+
+ histogram.flushHistogram();
+ }
+
+ private int[] selectWorkload(String workLoad)
+ {
+ switch (workLoad)
+ {
+ case "secondInMonth":
+ return secondInMonth;
+ case "secondInDay":
+ return secondInDay;
+ case "secondIn3Hour":
+ return secondIn3Hour;
+ case "secondInMin":
+ return secondInMin;
+ default:
+ throw new IllegalArgumentException("Invalid workload type: " + workLoad);
+ }
+ }
+
+
+ public static void main(String[] args) throws Exception
+ {
+ Options opt = new OptionsBuilder()
+ .include(StreamingTombstoneHistogramBuilderBench.class.getSimpleName())
+ .warmupIterations(3)
+ .measurementIterations(10)
+ .addProfiler(GCProfiler.class)
+ .threads(1)
+ .forks(1)
+ .build();
+ new Runner(opt).run();
+ }
+}
diff --git a/test/microbench/org/apache/cassandra/test/microbench/StringsEncodeBench.java b/test/microbench/org/apache/cassandra/test/microbench/StringsEncodeBench.java
new file mode 100644
index 0000000..5a1ccba
--- /dev/null
+++ b/test/microbench/org/apache/cassandra/test/microbench/StringsEncodeBench.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.test.microbench;
+
+import java.util.concurrent.TimeUnit;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.ByteBufUtil;
+import io.netty.buffer.Unpooled;
+import org.apache.cassandra.db.TypeSizes;
+import org.apache.cassandra.transport.CBUtil;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.NANOSECONDS)
+@Warmup(iterations = 3, time = 1)
+@Measurement(iterations = 6, time = 20)
+@Fork(value = 1,jvmArgsAppend = { "-Xmx256M", "-Djmh.executor=CUSTOM", "-Djmh.executor.class=org.apache.cassandra.test.microbench.FastThreadExecutor"})
+@State(Scope.Benchmark)
+public class StringsEncodeBench
+{
+ private String shortText = "abcdefghijk";
+ private String longText = "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.";
+ private int shortTextEncodeSize = CBUtil.sizeOfString(shortText);
+ private int longTextEncodeSize = CBUtil.sizeOfString(longText);
+
+ @Benchmark
+ public int writeShortText() // expecting resize
+ {
+ ByteBuf cb = Unpooled.buffer(shortTextEncodeSize);
+ cb.writeShort(0); // field for str length
+ return ByteBufUtil.writeUtf8(cb, shortText);
+ }
+
+ @Benchmark
+ public int writeShortTextWithExactSize() // no resize
+ {
+ ByteBuf cb = Unpooled.buffer(shortTextEncodeSize);
+ int size = TypeSizes.encodedUTF8Length(shortText);
+ cb.writeShort(size); // field for str length
+ return ByteBufUtil.reserveAndWriteUtf8(cb, shortText, size);
+ }
+
+ @Benchmark
+ public int writeShortTextWithExactSizeSkipCalc() // no resize; from the encodeSize, we already know the amount of bytes required.
+ {
+ ByteBuf cb = Unpooled.buffer(shortTextEncodeSize);
+ cb.writeShort(0); // field for str length
+ int size = cb.capacity() - cb.writerIndex(); // leverage the pre-calculated encodeSize
+ return ByteBufUtil.reserveAndWriteUtf8(cb, shortText, size);
+ }
+
+ @Benchmark
+ public void writeShortTextAsASCII()
+ {
+ ByteBuf cb = Unpooled.buffer(shortTextEncodeSize);
+ CBUtil.writeAsciiString(shortText, cb);
+ }
+
+ @Benchmark
+ public int writeLongText() // expecting resize
+ {
+ ByteBuf cb = Unpooled.buffer(longTextEncodeSize);
+ cb.writeShort(0); // field for str length
+ return ByteBufUtil.writeUtf8(cb, longText);
+ }
+
+ @Benchmark
+ public int writeLongTextWithExactSize() // no resize
+ {
+ ByteBuf cb = Unpooled.buffer(longTextEncodeSize);
+ int size = TypeSizes.encodedUTF8Length(longText);
+ cb.writeShort(size); // field for str length
+ return ByteBufUtil.reserveAndWriteUtf8(cb, longText, size);
+ }
+
+ @Benchmark
+ public int writeLongTextWithExactSizeSkipCalc() // no resize
+ {
+ ByteBuf cb = Unpooled.buffer(longTextEncodeSize);
+ cb.writeShort(0); // field for str length
+ int size = cb.capacity() - cb.writerIndex(); // leverage the pre-calculated encodeSize
+ return ByteBufUtil.reserveAndWriteUtf8(cb, longText, size);
+ }
+
+ @Benchmark
+ public void writeLongTextAsASCII()
+ {
+ ByteBuf cb = Unpooled.buffer(longTextEncodeSize);
+ CBUtil.writeAsciiString(longText, cb);
+ }
+
+ @Benchmark
+ public int sizeOfString()
+ {
+ return CBUtil.sizeOfString(longText);
+ }
+
+ @Benchmark
+ public int sizeOfAsciiString()
+ {
+ return CBUtil.sizeOfAsciiString(longText);
+ }
+}
diff --git a/test/microbench/org/apache/cassandra/test/microbench/ZeroCopyStreamingBenchmark.java b/test/microbench/org/apache/cassandra/test/microbench/ZeroCopyStreamingBenchmark.java
new file mode 100644
index 0000000..8ecf6cb
--- /dev/null
+++ b/test/microbench/org/apache/cassandra/test/microbench/ZeroCopyStreamingBenchmark.java
@@ -0,0 +1,329 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.test.microbench;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.WritableByteChannel;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.channel.ChannelOutboundHandlerAdapter;
+import io.netty.channel.ChannelPromise;
+import io.netty.channel.DefaultFileRegion;
+import io.netty.channel.embedded.EmbeddedChannel;
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.RowUpdateBuilder;
+import org.apache.cassandra.db.commitlog.CommitLog;
+import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.db.streaming.CassandraEntireSSTableStreamReader;
+import org.apache.cassandra.db.streaming.CassandraEntireSSTableStreamWriter;
+import org.apache.cassandra.db.streaming.CassandraOutgoingFile;
+import org.apache.cassandra.db.streaming.CassandraStreamHeader;
+import org.apache.cassandra.db.streaming.CassandraStreamReader;
+import org.apache.cassandra.db.streaming.CassandraStreamWriter;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.io.sstable.SSTableMultiWriter;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.net.AsyncStreamingInputPlus;
+import org.apache.cassandra.net.AsyncStreamingOutputPlus;
+import org.apache.cassandra.schema.CachingParams;
+import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.streaming.DefaultConnectionFactory;
+import org.apache.cassandra.streaming.PreviewKind;
+import org.apache.cassandra.streaming.SessionInfo;
+import org.apache.cassandra.streaming.StreamCoordinator;
+import org.apache.cassandra.streaming.StreamEventHandler;
+import org.apache.cassandra.streaming.StreamOperation;
+import org.apache.cassandra.streaming.StreamResultFuture;
+import org.apache.cassandra.streaming.StreamSession;
+import org.apache.cassandra.streaming.StreamSummary;
+import org.apache.cassandra.streaming.messages.StreamMessageHeader;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.FBUtilities;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.annotations.Warmup;
+
+/**
+ * Please ensure that this benchmark is run with stream_throughput_outbound_megabits_per_sec set to a
+ * really high value otherwise, throttling will kick in and the results will not be meaningful.
+ */
+@Warmup(iterations = 1, time = 1, timeUnit = TimeUnit.SECONDS)
+@Measurement(iterations = 10, time = 1, timeUnit = TimeUnit.SECONDS)
+@Fork(value = 1)
+@Threads(1)
+public class ZeroCopyStreamingBenchmark
+{
+ static final int STREAM_SIZE = 50 * 1024 * 1024;
+
+ @State(Scope.Thread)
+ public static class BenchmarkState
+ {
+ public static final String KEYSPACE = "ZeroCopyStreamingBenchmark";
+ public static final String CF_STANDARD = "Standard1";
+ public static final String CF_INDEXED = "Indexed1";
+ public static final String CF_STANDARDLOWINDEXINTERVAL = "StandardLowIndexInterval";
+
+ private static SSTableReader sstable;
+ private static ColumnFamilyStore store;
+ private StreamSession session;
+ private CassandraEntireSSTableStreamWriter blockStreamWriter;
+ private ByteBuf serializedBlockStream;
+ private InetAddressAndPort peer = FBUtilities.getBroadcastAddressAndPort();
+ private CassandraEntireSSTableStreamReader blockStreamReader;
+ private CassandraStreamWriter partialStreamWriter;
+ private CassandraStreamReader partialStreamReader;
+ private ByteBuf serializedPartialStream;
+
+ @Setup
+ public void setupBenchmark() throws IOException
+ {
+ Keyspace keyspace = setupSchemaAndKeySpace();
+ store = keyspace.getColumnFamilyStore("Standard1");
+ generateData();
+
+ sstable = store.getLiveSSTables().iterator().next();
+ session = setupStreamingSessionForTest();
+ blockStreamWriter = new CassandraEntireSSTableStreamWriter(sstable, session, CassandraOutgoingFile.getComponentManifest(sstable));
+
+ CapturingNettyChannel blockStreamCaptureChannel = new CapturingNettyChannel(STREAM_SIZE);
+ AsyncStreamingOutputPlus out = new AsyncStreamingOutputPlus(blockStreamCaptureChannel);
+ blockStreamWriter.write(out);
+ serializedBlockStream = blockStreamCaptureChannel.getSerializedStream();
+ out.close();
+
+ session.prepareReceiving(new StreamSummary(sstable.metadata().id, 1, serializedBlockStream.readableBytes()));
+
+ CassandraStreamHeader entireSSTableStreamHeader =
+ CassandraStreamHeader.builder()
+ .withSSTableFormat(sstable.descriptor.formatType)
+ .withSSTableVersion(sstable.descriptor.version)
+ .withSSTableLevel(0)
+ .withEstimatedKeys(sstable.estimatedKeys())
+ .withSections(Collections.emptyList())
+ .withSerializationHeader(sstable.header.toComponent())
+ .withComponentManifest(CassandraOutgoingFile.getComponentManifest(sstable))
+ .isEntireSSTable(true)
+ .withFirstKey(sstable.first)
+ .withTableId(sstable.metadata().id)
+ .build();
+
+ blockStreamReader = new CassandraEntireSSTableStreamReader(new StreamMessageHeader(sstable.metadata().id,
+ peer, session.planId(), false,
+ 0, 0, 0,
+ null), entireSSTableStreamHeader, session);
+
+ List<Range<Token>> requestedRanges = Arrays.asList(new Range<>(sstable.first.minValue().getToken(), sstable.last.getToken()));
+ partialStreamWriter = new CassandraStreamWriter(sstable, sstable.getPositionsForRanges(requestedRanges), session);
+
+ CapturingNettyChannel partialStreamChannel = new CapturingNettyChannel(STREAM_SIZE);
+ partialStreamWriter.write(new AsyncStreamingOutputPlus(partialStreamChannel));
+ serializedPartialStream = partialStreamChannel.getSerializedStream();
+
+ CassandraStreamHeader partialSSTableStreamHeader =
+ CassandraStreamHeader.builder()
+ .withSSTableFormat(sstable.descriptor.formatType)
+ .withSSTableVersion(sstable.descriptor.version)
+ .withSSTableLevel(0)
+ .withEstimatedKeys(sstable.estimatedKeys())
+ .withSections(sstable.getPositionsForRanges(requestedRanges))
+ .withSerializationHeader(sstable.header.toComponent())
+ .withTableId(sstable.metadata().id)
+ .build();
+
+ partialStreamReader = new CassandraStreamReader(new StreamMessageHeader(sstable.metadata().id,
+ peer, session.planId(), false,
+ 0, 0, 0,
+ null),
+ partialSSTableStreamHeader, session);
+ }
+
+ private Keyspace setupSchemaAndKeySpace()
+ {
+ SchemaLoader.prepareServer();
+ SchemaLoader.createKeyspace(KEYSPACE,
+ KeyspaceParams.simple(1),
+ SchemaLoader.standardCFMD(KEYSPACE, CF_STANDARD),
+ SchemaLoader.compositeIndexCFMD(KEYSPACE, CF_INDEXED, true),
+ SchemaLoader.standardCFMD(KEYSPACE, CF_STANDARDLOWINDEXINTERVAL)
+ .minIndexInterval(8)
+ .maxIndexInterval(256)
+ .caching(CachingParams.CACHE_NOTHING));
+
+ return Keyspace.open(KEYSPACE);
+ }
+
+ private void generateData()
+ {
+ // insert data and compact to a single sstable
+ CompactionManager.instance.disableAutoCompaction();
+ for (int j = 0; j < 1_000_000; j++)
+ {
+ new RowUpdateBuilder(store.metadata(), j, String.valueOf(j))
+ .clustering("0")
+ .add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER)
+ .build()
+ .applyUnsafe();
+ }
+ store.forceBlockingFlush();
+ CompactionManager.instance.performMaximal(store, false);
+ }
+
+ @TearDown
+ public void tearDown() throws IOException
+ {
+ SchemaLoader.cleanupAndLeaveDirs();
+ CommitLog.instance.stopUnsafe(true);
+ }
+
+ private StreamSession setupStreamingSessionForTest()
+ {
+ StreamCoordinator streamCoordinator = new StreamCoordinator(StreamOperation.BOOTSTRAP, 1, new DefaultConnectionFactory(), false, false, null, PreviewKind.NONE);
+ StreamResultFuture future = StreamResultFuture.createInitiator(UUID.randomUUID(), StreamOperation.BOOTSTRAP, Collections.<StreamEventHandler>emptyList(), streamCoordinator);
+
+ InetAddressAndPort peer = FBUtilities.getBroadcastAddressAndPort();
+ streamCoordinator.addSessionInfo(new SessionInfo(peer, 0, peer, Collections.emptyList(), Collections.emptyList(), StreamSession.State.INITIALIZED));
+
+ StreamSession session = streamCoordinator.getOrCreateNextSession(peer);
+ session.init(future);
+ return session;
+ }
+ }
+
+ @Benchmark
+ @BenchmarkMode(Mode.Throughput)
+ public void blockStreamWriter(BenchmarkState state) throws Exception
+ {
+ EmbeddedChannel channel = createMockNettyChannel();
+ AsyncStreamingOutputPlus out = new AsyncStreamingOutputPlus(channel);
+ state.blockStreamWriter.write(out);
+ out.close();
+ channel.finishAndReleaseAll();
+ }
+
+ @Benchmark
+ @BenchmarkMode(Mode.Throughput)
+ public void blockStreamReader(BenchmarkState state) throws Exception
+ {
+ EmbeddedChannel channel = createMockNettyChannel();
+ AsyncStreamingInputPlus in = new AsyncStreamingInputPlus(channel);
+ in.append(state.serializedBlockStream.retainedDuplicate());
+ SSTableMultiWriter sstableWriter = state.blockStreamReader.read(in);
+ Collection<SSTableReader> newSstables = sstableWriter.finished();
+ in.close();
+ channel.finishAndReleaseAll();
+ }
+
+ @Benchmark
+ @BenchmarkMode(Mode.Throughput)
+ public void partialStreamWriter(BenchmarkState state) throws Exception
+ {
+ EmbeddedChannel channel = createMockNettyChannel();
+ AsyncStreamingOutputPlus out = new AsyncStreamingOutputPlus(channel);
+ state.partialStreamWriter.write(out);
+ out.close();
+ channel.finishAndReleaseAll();
+ }
+
+ @Benchmark
+ @BenchmarkMode(Mode.Throughput)
+ public void partialStreamReader(BenchmarkState state) throws Exception
+ {
+ EmbeddedChannel channel = createMockNettyChannel();
+ AsyncStreamingInputPlus in = new AsyncStreamingInputPlus(channel);
+ in.append(state.serializedPartialStream.retainedDuplicate());
+ SSTableMultiWriter sstableWriter = state.partialStreamReader.read(in);
+ Collection<SSTableReader> newSstables = sstableWriter.finished();
+ in.close();
+ channel.finishAndReleaseAll();
+ }
+
+ private EmbeddedChannel createMockNettyChannel()
+ {
+ EmbeddedChannel channel = new EmbeddedChannel();
+ channel.config().setWriteBufferHighWaterMark(STREAM_SIZE); // avoid blocking
+ return channel;
+ }
+
+ private static class CapturingNettyChannel extends EmbeddedChannel
+ {
+ private final ByteBuf serializedStream;
+ private final WritableByteChannel proxyWBC = new WritableByteChannel()
+ {
+ public int write(ByteBuffer src) throws IOException
+ {
+ int rem = src.remaining();
+ serializedStream.writeBytes(src);
+ return rem;
+ }
+
+ public boolean isOpen()
+ {
+ return true;
+ }
+
+ public void close() throws IOException
+ {
+ }
+ };
+
+ public CapturingNettyChannel(int capacity)
+ {
+ this.serializedStream = alloc().buffer(capacity);
+ this.pipeline().addLast(new ChannelOutboundHandlerAdapter()
+ {
+ @Override
+ public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise promise) throws Exception
+ {
+ if (msg instanceof ByteBuf)
+ serializedStream.writeBytes((ByteBuf) msg);
+ else if (msg instanceof ByteBuffer)
+ serializedStream.writeBytes((ByteBuffer) msg);
+ else if (msg instanceof DefaultFileRegion)
+ ((DefaultFileRegion) msg).transferTo(proxyWBC, 0);
+ }
+ });
+ config().setWriteBufferHighWaterMark(capacity);
+ }
+
+ public ByteBuf getSerializedStream()
+ {
+ return serializedStream.copy();
+ }
+ }
+}
diff --git a/test/resources/functions/configure_cassandra.sh b/test/resources/functions/configure_cassandra.sh
index 3464653..38ff098 100644
--- a/test/resources/functions/configure_cassandra.sh
+++ b/test/resources/functions/configure_cassandra.sh
@@ -45,40 +45,25 @@
;;
esac
- OH_SIX_CONFIG="/etc/cassandra/conf/storage-conf.xml"
-
- if [[ -e "$OH_SIX_CONFIG" ]] ; then
- config_file=$OH_SIX_CONFIG
- seeds=""
+ config_file="/etc/cassandra/conf/cassandra.yaml"
+ if [[ "x"`grep -e '^seeds:' $config_file` == "x" ]]; then
+ seeds="$1" # 08 format seeds
+ shift
for server in "$@"; do
- seeds="${seeds}<Seed>${server}</Seed>"
+ seeds="${seeds},${server}"
done
-
- #TODO set replication
- sed -i -e "s|<Seed>127.0.0.1</Seed>|$seeds|" $config_file
- sed -i -e "s|<ListenAddress>localhost</ListenAddress>|<ListenAddress>$PRIVATE_SELF_HOST</ListenAddress>|" $config_file
- sed -i -e "s|<ThriftAddress>localhost</ThriftAddress>|<ThriftAddress>$PUBLIC_SELF_HOST</ThriftAddress>|" $config_file
+ sed -i -e "s|- seeds: \"127.0.0.1\"|- seeds: \"${seeds}\"|" $config_file
else
- config_file="/etc/cassandra/conf/cassandra.yaml"
- if [[ "x"`grep -e '^seeds:' $config_file` == "x" ]]; then
- seeds="$1" # 08 format seeds
- shift
- for server in "$@"; do
- seeds="${seeds},${server}"
- done
- sed -i -e "s|- seeds: \"127.0.0.1\"|- seeds: \"${seeds}\"|" $config_file
- else
- seeds="" # 07 format seeds
- for server in "$@"; do
- seeds="${seeds}\n - ${server}"
- done
- sed -i -e "/^seeds:/,/^/d" $config_file ; echo -e "seeds:${seeds}" >> $config_file
- fi
-
- sed -i -e "s|listen_address: localhost|listen_address: $PRIVATE_SELF_HOST|" $config_file
- sed -i -e "s|rpc_address: localhost|rpc_address: $PUBLIC_SELF_HOST|" $config_file
+ seeds="" # 07 format seeds
+ for server in "$@"; do
+ seeds="${seeds}\n - ${server}"
+ done
+ sed -i -e "/^seeds:/,/^/d" $config_file ; echo -e "seeds:${seeds}" >> $config_file
fi
+ sed -i -e "s|listen_address: localhost|listen_address: $PRIVATE_SELF_HOST|" $config_file
+ sed -i -e "s|rpc_address: localhost|rpc_address: $PUBLIC_SELF_HOST|" $config_file
+
# Now that it's configured, start Cassandra
nohup /etc/rc.local &
diff --git a/test/unit/org/apache/cassandra/AbstractSerializationsTester.java b/test/unit/org/apache/cassandra/AbstractSerializationsTester.java
index 3a1f348..3611f0e 100644
--- a/test/unit/org/apache/cassandra/AbstractSerializationsTester.java
+++ b/test/unit/org/apache/cassandra/AbstractSerializationsTester.java
@@ -36,16 +36,11 @@
public class AbstractSerializationsTester
{
- protected static final String CUR_VER = System.getProperty("cassandra.version", "3.0");
+ protected static final String CUR_VER = System.getProperty("cassandra.version", "4.0");
protected static final Map<String, Integer> VERSION_MAP = new HashMap<String, Integer> ()
{{
- put("0.7", 1);
- put("1.0", 3);
- put("1.2", MessagingService.VERSION_12);
- put("2.0", MessagingService.VERSION_20);
- put("2.1", MessagingService.VERSION_21);
- put("2.2", MessagingService.VERSION_22);
put("3.0", MessagingService.VERSION_30);
+ put("4.0", MessagingService.VERSION_40);
}};
protected static final boolean EXECUTE_WRITES = Boolean.getBoolean("cassandra.test-serialization-writes");
diff --git a/test/unit/org/apache/cassandra/CassandraIsolatedJunit4ClassRunner.java b/test/unit/org/apache/cassandra/CassandraIsolatedJunit4ClassRunner.java
new file mode 100644
index 0000000..90e3896
--- /dev/null
+++ b/test/unit/org/apache/cassandra/CassandraIsolatedJunit4ClassRunner.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra;
+
+import java.io.IOException;
+import java.net.URLClassLoader;
+import java.util.function.Predicate;
+
+import org.junit.runners.BlockJUnit4ClassRunner;
+import org.junit.runners.model.InitializationError;
+
+import org.apache.cassandra.distributed.impl.AbstractCluster;
+import org.apache.cassandra.distributed.shared.Versions;
+import org.apache.cassandra.utils.FBUtilities;
+
+/**
+ *
+ * This class is usually used to test singletons. It ensure singletons can be unique in each test case.
+ *
+ */
+public class CassandraIsolatedJunit4ClassRunner extends BlockJUnit4ClassRunner
+{
+
+ private static final Predicate<String> isolatedPackage = name ->
+ name.startsWith("org.apache.cassandra.") ||
+ // YAML could not be shared because
+ // org.apache.cassandra.config.Config is loaded by org.yaml.snakeyaml.YAML
+ name.startsWith("org.yaml.snakeyaml.");
+
+
+ /**
+ * Creates a CassandraIsolatedJunit4ClassRunner to run {@code klass}
+ *
+ * @param clazz
+ * @throws InitializationError if the test class is malformed.
+ */
+ public CassandraIsolatedJunit4ClassRunner(Class<?> clazz) throws InitializationError
+ {
+ super(createClassLoader(clazz));
+ }
+
+ private static Class<?> createClassLoader(Class<?> clazz) throws InitializationError {
+ try {
+ ClassLoader testClassLoader = new CassandraIsolatedClassLoader();
+ return Class.forName(clazz.getName(), true, testClassLoader);
+ } catch (ClassNotFoundException e) {
+ throw new InitializationError(e);
+ }
+ }
+
+ public static class CassandraIsolatedClassLoader extends URLClassLoader
+ {
+ public CassandraIsolatedClassLoader()
+ {
+ super(AbstractCluster.CURRENT_VERSION.classpath);
+ }
+
+ @Override
+ public Class<?> loadClass(String name) throws ClassNotFoundException
+ {
+
+ if (isolatedPackage.test(name))
+ {
+ synchronized (getClassLoadingLock(name))
+ {
+ // First, check if the class has already been loaded
+ Class<?> c = findLoadedClass(name);
+
+ if (c == null)
+ c = findClass(name);
+
+ return c;
+ }
+ }
+ else
+ {
+ return super.loadClass(name);
+ }
+ }
+
+ protected void finalize()
+ {
+ try
+ {
+ close();
+ }
+ catch (IOException e)
+ {
+ e.printStackTrace();
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/MockSchema.java b/test/unit/org/apache/cassandra/MockSchema.java
deleted file mode 100644
index 804bccb..0000000
--- a/test/unit/org/apache/cassandra/MockSchema.java
+++ /dev/null
@@ -1,187 +0,0 @@
-/*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*/
-package org.apache.cassandra;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.RandomAccessFile;
-import java.util.*;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import com.google.common.collect.ImmutableSet;
-
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.db.*;
-import org.apache.cassandra.db.marshal.UTF8Type;
-import org.apache.cassandra.dht.Murmur3Partitioner;
-import org.apache.cassandra.io.sstable.Component;
-import org.apache.cassandra.io.sstable.Descriptor;
-import org.apache.cassandra.io.sstable.IndexSummary;
-import org.apache.cassandra.io.sstable.format.SSTableFormat;
-import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
-import org.apache.cassandra.io.sstable.metadata.MetadataType;
-import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
-import org.apache.cassandra.io.util.FileUtils;
-import org.apache.cassandra.io.util.Memory;
-import org.apache.cassandra.io.util.FileHandle;
-import org.apache.cassandra.schema.CachingParams;
-import org.apache.cassandra.schema.KeyspaceMetadata;
-import org.apache.cassandra.schema.KeyspaceParams;
-import org.apache.cassandra.utils.AlwaysPresentFilter;
-import org.apache.cassandra.utils.ByteBufferUtil;
-
-public class MockSchema
-{
- static
- {
- Memory offsets = Memory.allocate(4);
- offsets.setInt(0, 0);
- indexSummary = new IndexSummary(Murmur3Partitioner.instance, offsets, 0, Memory.allocate(4), 0, 0, 0, 1);
- }
- private static final AtomicInteger id = new AtomicInteger();
- public static final Keyspace ks = Keyspace.mockKS(KeyspaceMetadata.create("mockks", KeyspaceParams.simpleTransient(1)));
-
- public static final IndexSummary indexSummary;
- private static final FileHandle RANDOM_ACCESS_READER_FACTORY = new FileHandle.Builder(temp("mocksegmentedfile").getAbsolutePath()).complete();
-
- public static Memtable memtable(ColumnFamilyStore cfs)
- {
- return new Memtable(cfs.metadata);
- }
-
- public static SSTableReader sstable(int generation, ColumnFamilyStore cfs)
- {
- return sstable(generation, false, cfs);
- }
-
- public static SSTableReader sstable(int generation, boolean keepRef, ColumnFamilyStore cfs)
- {
- return sstable(generation, 0, keepRef, cfs);
- }
-
- public static SSTableReader sstable(int generation, int size, ColumnFamilyStore cfs)
- {
- return sstable(generation, size, false, cfs);
- }
-
- public static SSTableReader sstable(int generation, int size, boolean keepRef, ColumnFamilyStore cfs)
- {
- Descriptor descriptor = new Descriptor(cfs.getDirectories().getDirectoryForNewSSTables(),
- cfs.keyspace.getName(),
- cfs.getColumnFamilyName(),
- generation, SSTableFormat.Type.BIG);
- Set<Component> components = ImmutableSet.of(Component.DATA, Component.PRIMARY_INDEX, Component.FILTER, Component.TOC);
- for (Component component : components)
- {
- File file = new File(descriptor.filenameFor(component));
- try
- {
- file.createNewFile();
- }
- catch (IOException e)
- {
- }
- }
- if (size > 0)
- {
- try
- {
- File file = new File(descriptor.filenameFor(Component.DATA));
- try (RandomAccessFile raf = new RandomAccessFile(file, "rw"))
- {
- raf.setLength(size);
- }
- }
- catch (IOException e)
- {
- throw new RuntimeException(e);
- }
- }
- SerializationHeader header = SerializationHeader.make(cfs.metadata, Collections.emptyList());
- StatsMetadata metadata = (StatsMetadata) new MetadataCollector(cfs.metadata.comparator)
- .finalizeMetadata(cfs.metadata.partitioner.getClass().getCanonicalName(), 0.01f, -1, header)
- .get(MetadataType.STATS);
- SSTableReader reader = SSTableReader.internalOpen(descriptor, components, cfs.metadata,
- RANDOM_ACCESS_READER_FACTORY.sharedCopy(), RANDOM_ACCESS_READER_FACTORY.sharedCopy(), indexSummary.sharedCopy(),
- new AlwaysPresentFilter(), 1L, metadata, SSTableReader.OpenReason.NORMAL, header);
- reader.first = reader.last = readerBounds(generation);
- if (!keepRef)
- reader.selfRef().release();
- return reader;
- }
-
- public static ColumnFamilyStore newCFS()
- {
- return newCFS(ks.getName());
- }
-
- public static ColumnFamilyStore newCFS(String ksname)
- {
- String cfname = "mockcf" + (id.incrementAndGet());
- CFMetaData metadata = newCFMetaData(ksname, cfname);
- return new ColumnFamilyStore(ks, cfname, 0, metadata, new Directories(metadata), false, false, false);
- }
-
- public static CFMetaData newCFMetaData(String ksname, String cfname)
- {
- CFMetaData metadata = CFMetaData.Builder.create(ksname, cfname)
- .addPartitionKey("key", UTF8Type.instance)
- .addClusteringColumn("col", UTF8Type.instance)
- .addRegularColumn("value", UTF8Type.instance)
- .withPartitioner(Murmur3Partitioner.instance)
- .build();
- metadata.caching(CachingParams.CACHE_NOTHING);
- return metadata;
- }
-
- public static BufferDecoratedKey readerBounds(int generation)
- {
- return new BufferDecoratedKey(new Murmur3Partitioner.LongToken(generation), ByteBufferUtil.EMPTY_BYTE_BUFFER);
- }
-
- private static File temp(String id)
- {
- try
- {
- File file = File.createTempFile(id, "tmp");
- file.deleteOnExit();
- return file;
- }
- catch (IOException e)
- {
- throw new RuntimeException(e);
- }
- }
-
- public static void cleanup()
- {
- // clean up data directory which are stored as data directory/keyspace/data files
- for (String dirName : DatabaseDescriptor.getAllDataFileLocations())
- {
- File dir = new File(dirName);
- if (!dir.exists())
- continue;
- String[] children = dir.list();
- for (String child : children)
- FileUtils.deleteRecursive(new File(dir, child));
- }
- }
-}
diff --git a/test/unit/org/apache/cassandra/OffsetAwareConfigurationLoader.java b/test/unit/org/apache/cassandra/OffsetAwareConfigurationLoader.java
index 0047f48..23138b0 100644
--- a/test/unit/org/apache/cassandra/OffsetAwareConfigurationLoader.java
+++ b/test/unit/org/apache/cassandra/OffsetAwareConfigurationLoader.java
@@ -20,8 +20,15 @@
import org.apache.cassandra.config.Config;
import org.apache.cassandra.config.YamlConfigurationLoader;
import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.locator.InetAddressAndPort;
import java.io.File;
+import java.net.Inet6Address;
+import java.net.UnknownHostException;
+import java.util.Arrays;
+import java.util.stream.Collectors;
+
+import com.google.common.base.Joiner;
public class OffsetAwareConfigurationLoader extends YamlConfigurationLoader
@@ -49,9 +56,34 @@
String sep = File.pathSeparator;
- config.rpc_port += offset;
config.native_transport_port += offset;
config.storage_port += offset;
+ config.ssl_storage_port += offset;
+
+ //Rewrite the seed ports string
+ String[] hosts = config.seed_provider.parameters.get("seeds").split(",", -1);
+ String rewrittenSeeds = Joiner.on(", ").join(Arrays.stream(hosts).map(host -> {
+ StringBuilder sb = new StringBuilder();
+ try
+ {
+ InetAddressAndPort address = InetAddressAndPort.getByName(host.trim());
+ if (address.address instanceof Inet6Address)
+ {
+ sb.append('[').append(address.address.getHostAddress()).append(']');
+ }
+ else
+ {
+ sb.append(address.address.getHostAddress());
+ }
+ sb.append(':').append(address.port + offset);
+ return sb.toString();
+ }
+ catch (UnknownHostException e)
+ {
+ throw new ConfigurationException("Error in OffsetAwareConfigurationLoader reworking seed list", e);
+ }
+ }).collect(Collectors.toList()));
+ config.seed_provider.parameters.put("seeds", rewrittenSeeds);
config.commitlog_directory += sep + offset;
config.saved_caches_directory += sep + offset;
diff --git a/test/unit/org/apache/cassandra/SchemaLoader.java b/test/unit/org/apache/cassandra/SchemaLoader.java
index 822ee67..50a06e1 100644
--- a/test/unit/org/apache/cassandra/SchemaLoader.java
+++ b/test/unit/org/apache/cassandra/SchemaLoader.java
@@ -21,28 +21,35 @@
import java.io.IOException;
import java.util.*;
-import org.apache.cassandra.dht.Murmur3Partitioner;
-import org.apache.cassandra.index.sasi.SASIIndex;
-import org.apache.cassandra.index.sasi.disk.OnDiskIndexBuilder;
-import org.junit.After;
-import org.junit.BeforeClass;
-
+import org.apache.cassandra.auth.AuthKeyspace;
+import org.apache.cassandra.auth.AuthSchemaChangeListener;
+import org.apache.cassandra.auth.IAuthenticator;
+import org.apache.cassandra.auth.IAuthorizer;
+import org.apache.cassandra.auth.INetworkAuthorizer;
+import org.apache.cassandra.auth.IRoleManager;
import org.apache.cassandra.config.*;
import org.apache.cassandra.cql3.CQLTester;
import org.apache.cassandra.cql3.ColumnIdentifier;
-import org.apache.cassandra.cql3.statements.IndexTarget;
+import org.apache.cassandra.cql3.statements.schema.CreateTableStatement;
+import org.apache.cassandra.cql3.statements.schema.IndexTarget;
import org.apache.cassandra.db.RowUpdateBuilder;
import org.apache.cassandra.db.commitlog.CommitLog;
import org.apache.cassandra.db.marshal.*;
+import org.apache.cassandra.dht.Murmur3Partitioner;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.gms.Gossiper;
import org.apache.cassandra.index.StubIndex;
+import org.apache.cassandra.index.sasi.SASIIndex;
+import org.apache.cassandra.index.sasi.disk.OnDiskIndexBuilder;
import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.schema.*;
-import org.apache.cassandra.service.MigrationManager;
+import org.apache.cassandra.schema.MigrationManager;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FBUtilities;
+import org.junit.After;
+import org.junit.BeforeClass;
+
public class SchemaLoader
{
@BeforeClass
@@ -95,6 +102,8 @@
String ks_nocommit = testName + "NoCommitlogSpace";
String ks_prsi = testName + "PerRowSecondaryIndex";
String ks_cql = testName + "cql_keyspace";
+ String ks_cql_replicated = testName + "cql_keyspace_replicated";
+ String ks_with_transient = testName + "ks_with_transient";
AbstractType bytes = BytesType.instance;
@@ -119,44 +128,31 @@
KeyspaceParams.simple(1),
Tables.of(
// Column Families
- standardCFMD(ks1, "Standard1").compaction(CompactionParams.scts(compactionOptions)),
- standardCFMD(ks1, "Standard2"),
- standardCFMD(ks1, "Standard3"),
- standardCFMD(ks1, "Standard4"),
- standardCFMD(ks1, "StandardGCGS0").gcGraceSeconds(0),
- standardCFMD(ks1, "StandardLong1"),
- standardCFMD(ks1, "StandardLong2"),
- //CFMetaData.Builder.create(ks1, "ValuesWithQuotes").build(),
- superCFMD(ks1, "Super1", LongType.instance),
- superCFMD(ks1, "Super2", LongType.instance),
- superCFMD(ks1, "Super3", LongType.instance),
- superCFMD(ks1, "Super4", UTF8Type.instance),
- superCFMD(ks1, "Super5", bytes),
- superCFMD(ks1, "Super6", LexicalUUIDType.instance, UTF8Type.instance),
- keysIndexCFMD(ks1, "Indexed1", true),
- keysIndexCFMD(ks1, "Indexed2", false),
- //CFMetaData.Builder.create(ks1, "StandardInteger1").withColumnNameComparator(IntegerType.instance).build(),
- //CFMetaData.Builder.create(ks1, "StandardLong3").withColumnNameComparator(IntegerType.instance).build(),
- //CFMetaData.Builder.create(ks1, "Counter1", false, false, true).build(),
- //CFMetaData.Builder.create(ks1, "SuperCounter1", false, false, true, true).build(),
- superCFMD(ks1, "SuperDirectGC", BytesType.instance).gcGraceSeconds(0),
-// jdbcCFMD(ks1, "JdbcInteger", IntegerType.instance).addColumnDefinition(integerColumn(ks1, "JdbcInteger")),
- jdbcCFMD(ks1, "JdbcUtf8", UTF8Type.instance).addColumnDefinition(utf8Column(ks1, "JdbcUtf8")),
- jdbcCFMD(ks1, "JdbcLong", LongType.instance),
- jdbcCFMD(ks1, "JdbcBytes", bytes),
- jdbcCFMD(ks1, "JdbcAscii", AsciiType.instance),
- //CFMetaData.Builder.create(ks1, "StandardComposite", false, true, false).withColumnNameComparator(composite).build(),
- //CFMetaData.Builder.create(ks1, "StandardComposite2", false, true, false).withColumnNameComparator(compositeMaxMin).build(),
- //CFMetaData.Builder.create(ks1, "StandardDynamicComposite", false, true, false).withColumnNameComparator(dynamicComposite).build(),
- standardCFMD(ks1, "StandardLeveled").compaction(CompactionParams.lcs(leveledOptions)),
- standardCFMD(ks1, "legacyleveled").compaction(CompactionParams.lcs(leveledOptions)),
+ standardCFMD(ks1, "Standard1").compaction(CompactionParams.stcs(compactionOptions)).build(),
+ standardCFMD(ks1, "Standard2").build(),
+ standardCFMD(ks1, "Standard3").build(),
+ standardCFMD(ks1, "Standard4").build(),
+ standardCFMD(ks1, "StandardGCGS0").gcGraceSeconds(0).build(),
+ standardCFMD(ks1, "StandardLong1").build(),
+ standardCFMD(ks1, "StandardLong2").build(),
+ superCFMD(ks1, "Super1", LongType.instance).build(),
+ superCFMD(ks1, "Super2", LongType.instance).build(),
+ superCFMD(ks1, "Super3", LongType.instance).build(),
+ superCFMD(ks1, "Super4", UTF8Type.instance).build(),
+ superCFMD(ks1, "Super5", bytes).build(),
+ superCFMD(ks1, "Super6", LexicalUUIDType.instance, UTF8Type.instance).build(),
+ keysIndexCFMD(ks1, "Indexed1", true).build(),
+ keysIndexCFMD(ks1, "Indexed2", false).build(),
+ superCFMD(ks1, "SuperDirectGC", BytesType.instance).gcGraceSeconds(0).build(),
+ jdbcCFMD(ks1, "JdbcUtf8", UTF8Type.instance).addColumn(utf8Column(ks1, "JdbcUtf8")).build(),
+ jdbcCFMD(ks1, "JdbcLong", LongType.instance).build(),
+ jdbcCFMD(ks1, "JdbcBytes", bytes).build(),
+ jdbcCFMD(ks1, "JdbcAscii", AsciiType.instance).build(),
+ standardCFMD(ks1, "StandardLeveled").compaction(CompactionParams.lcs(leveledOptions)).build(),
+ standardCFMD(ks1, "legacyleveled").compaction(CompactionParams.lcs(leveledOptions)).build(),
standardCFMD(ks1, "StandardLowIndexInterval").minIndexInterval(8)
.maxIndexInterval(256)
- .caching(CachingParams.CACHE_NOTHING)
- //CFMetaData.Builder.create(ks1, "UUIDKeys").addPartitionKey("key",UUIDType.instance).build(),
- //CFMetaData.Builder.create(ks1, "MixedTypes").withColumnNameComparator(LongType.instance).addPartitionKey("key", UUIDType.instance).build(),
- //CFMetaData.Builder.create(ks1, "MixedTypesComposite", false, true, false).withColumnNameComparator(composite).addPartitionKey("key", composite).build(),
- //CFMetaData.Builder.create(ks1, "AsciiKeys").addPartitionKey("key", AsciiType.instance).build()
+ .caching(CachingParams.CACHE_NOTHING).build()
)));
// Keyspace 2
@@ -164,118 +160,123 @@
KeyspaceParams.simple(1),
Tables.of(
// Column Families
- standardCFMD(ks2, "Standard1"),
- standardCFMD(ks2, "Standard3"),
- superCFMD(ks2, "Super3", bytes),
- superCFMD(ks2, "Super4", TimeUUIDType.instance),
- keysIndexCFMD(ks2, "Indexed1", true),
- compositeIndexCFMD(ks2, "Indexed2", true),
- compositeIndexCFMD(ks2, "Indexed3", true).gcGraceSeconds(0))));
+ standardCFMD(ks2, "Standard1").build(),
+ standardCFMD(ks2, "Standard3").build(),
+ superCFMD(ks2, "Super3", bytes).build(),
+ superCFMD(ks2, "Super4", TimeUUIDType.instance).build(),
+ keysIndexCFMD(ks2, "Indexed1", true).build(),
+ compositeIndexCFMD(ks2, "Indexed2", true).build(),
+ compositeIndexCFMD(ks2, "Indexed3", true).gcGraceSeconds(0).build())));
// Keyspace 3
schema.add(KeyspaceMetadata.create(ks3,
KeyspaceParams.simple(5),
Tables.of(
- standardCFMD(ks3, "Standard1"),
- keysIndexCFMD(ks3, "Indexed1", true))));
+ standardCFMD(ks3, "Standard1").build(),
+ keysIndexCFMD(ks3, "Indexed1", true).build())));
// Keyspace 4
schema.add(KeyspaceMetadata.create(ks4,
KeyspaceParams.simple(3),
Tables.of(
- standardCFMD(ks4, "Standard1"),
- standardCFMD(ks4, "Standard3"),
- superCFMD(ks4, "Super3", bytes),
- superCFMD(ks4, "Super4", TimeUUIDType.instance),
- superCFMD(ks4, "Super5", TimeUUIDType.instance, BytesType.instance))));
+ standardCFMD(ks4, "Standard1").build(),
+ standardCFMD(ks4, "Standard3").build(),
+ superCFMD(ks4, "Super3", bytes).build(),
+ superCFMD(ks4, "Super4", TimeUUIDType.instance).build(),
+ superCFMD(ks4, "Super5", TimeUUIDType.instance, BytesType.instance).build())));
// Keyspace 5
schema.add(KeyspaceMetadata.create(ks5,
KeyspaceParams.simple(2),
- Tables.of(standardCFMD(ks5, "Standard1"))));
+ Tables.of(standardCFMD(ks5, "Standard1").build())));
// Keyspace 6
schema.add(KeyspaceMetadata.create(ks6,
KeyspaceParams.simple(1),
- Tables.of(keysIndexCFMD(ks6, "Indexed1", true))));
+ Tables.of(keysIndexCFMD(ks6, "Indexed1", true).build())));
// Keyspace 7
schema.add(KeyspaceMetadata.create(ks7,
KeyspaceParams.simple(1),
- Tables.of(customIndexCFMD(ks7, "Indexed1"))));
+ Tables.of(customIndexCFMD(ks7, "Indexed1").build())));
// KeyCacheSpace
schema.add(KeyspaceMetadata.create(ks_kcs,
KeyspaceParams.simple(1),
Tables.of(
- standardCFMD(ks_kcs, "Standard1"),
- standardCFMD(ks_kcs, "Standard2"),
- standardCFMD(ks_kcs, "Standard3"))));
+ standardCFMD(ks_kcs, "Standard1").build(),
+ standardCFMD(ks_kcs, "Standard2").build(),
+ standardCFMD(ks_kcs, "Standard3").build())));
// RowCacheSpace
schema.add(KeyspaceMetadata.create(ks_rcs,
KeyspaceParams.simple(1),
Tables.of(
- standardCFMD(ks_rcs, "CFWithoutCache").caching(CachingParams.CACHE_NOTHING),
- standardCFMD(ks_rcs, "CachedCF").caching(CachingParams.CACHE_EVERYTHING),
- standardCFMD(ks_rcs, "CachedNoClustering", 1, IntegerType.instance, IntegerType.instance, null).caching(CachingParams.CACHE_EVERYTHING),
- standardCFMD(ks_rcs, "CachedIntCF").
- caching(new CachingParams(true, 100)))));
-
- // CounterCacheSpace
- /*schema.add(KeyspaceMetadata.testMetadata(ks_ccs,
- simple,
- opts_rf1,
- CFMetaData.Builder.create(ks_ccs, "Counter1", false, false, true).build(),
- CFMetaData.Builder.create(ks_ccs, "Counter1", false, false, true).build()));*/
+ standardCFMD(ks_rcs, "CFWithoutCache").caching(CachingParams.CACHE_NOTHING).build(),
+ standardCFMD(ks_rcs, "CachedCF").caching(CachingParams.CACHE_EVERYTHING).build(),
+ standardCFMD(ks_rcs, "CachedNoClustering", 1, IntegerType.instance, IntegerType.instance, null).caching(CachingParams.CACHE_EVERYTHING).build(),
+ standardCFMD(ks_rcs, "CachedIntCF").caching(new CachingParams(true, 100)).build())));
schema.add(KeyspaceMetadata.create(ks_nocommit, KeyspaceParams.simpleTransient(1), Tables.of(
- standardCFMD(ks_nocommit, "Standard1"))));
+ standardCFMD(ks_nocommit, "Standard1").build())));
+ String simpleTable = "CREATE TABLE table1 ("
+ + "k int PRIMARY KEY,"
+ + "v1 text,"
+ + "v2 int"
+ + ")";
// CQLKeyspace
schema.add(KeyspaceMetadata.create(ks_cql, KeyspaceParams.simple(1), Tables.of(
- // Column Families
- CFMetaData.compile("CREATE TABLE table1 ("
- + "k int PRIMARY KEY,"
- + "v1 text,"
- + "v2 int"
- + ")", ks_cql),
+ // Column Families
+ CreateTableStatement.parse(simpleTable, ks_cql).build(),
- CFMetaData.compile("CREATE TABLE table2 ("
- + "k text,"
- + "c text,"
- + "v text,"
- + "PRIMARY KEY (k, c))", ks_cql),
- CFMetaData.compile("CREATE TABLE foo ("
- + "bar text, "
- + "baz text, "
- + "qux text, "
- + "PRIMARY KEY(bar, baz) ) "
- + "WITH COMPACT STORAGE", ks_cql),
- CFMetaData.compile("CREATE TABLE foofoo ("
- + "bar text, "
- + "baz text, "
- + "qux text, "
- + "quz text, "
- + "foo text, "
- + "PRIMARY KEY((bar, baz), qux, quz) ) "
- + "WITH COMPACT STORAGE", ks_cql)
+ CreateTableStatement.parse("CREATE TABLE table2 ("
+ + "k text,"
+ + "c text,"
+ + "v text,"
+ + "PRIMARY KEY (k, c))", ks_cql)
+ .build()
)));
- if (DatabaseDescriptor.getPartitioner() instanceof Murmur3Partitioner)
- schema.add(KeyspaceMetadata.create("sasi", KeyspaceParams.simpleTransient(1), Tables.of(sasiCFMD("sasi", "test_cf"), clusteringSASICFMD("sasi", "clustering_test_cf"))));
+ schema.add(KeyspaceMetadata.create(ks_cql_replicated, KeyspaceParams.simple(3),
+ Tables.of(CreateTableStatement.parse(simpleTable, ks_cql_replicated).build())));
- if (Boolean.parseBoolean(System.getProperty("cassandra.test.compression", "false")))
- useCompression(schema);
+ schema.add(KeyspaceMetadata.create(ks_with_transient, KeyspaceParams.simple("3/1"),
+ Tables.of(CreateTableStatement.parse(simpleTable, ks_with_transient).build())));
+
+ if (DatabaseDescriptor.getPartitioner() instanceof Murmur3Partitioner)
+ {
+ schema.add(KeyspaceMetadata.create("sasi",
+ KeyspaceParams.simpleTransient(1),
+ Tables.of(sasiCFMD("sasi", "test_cf").build(),
+ clusteringSASICFMD("sasi", "clustering_test_cf").build())));
+ }
// if you're messing with low-level sstable stuff, it can be useful to inject the schema directly
// Schema.instance.load(schemaDefinition());
for (KeyspaceMetadata ksm : schema)
MigrationManager.announceNewKeyspace(ksm, false);
+
+ if (Boolean.parseBoolean(System.getProperty("cassandra.test.compression", "false")))
+ useCompression(schema);
}
- public static void createKeyspace(String name, KeyspaceParams params, CFMetaData... tables)
+ public static void createKeyspace(String name, KeyspaceParams params)
+ {
+ MigrationManager.announceNewKeyspace(KeyspaceMetadata.create(name, params, Tables.of()), true);
+ }
+
+ public static void createKeyspace(String name, KeyspaceParams params, TableMetadata.Builder... builders)
+ {
+ Tables.Builder tables = Tables.builder();
+ for (TableMetadata.Builder builder : builders)
+ tables.add(builder.build());
+
+ MigrationManager.announceNewKeyspace(KeyspaceMetadata.create(name, params, tables.build()), true);
+ }
+
+ public static void createKeyspace(String name, KeyspaceParams params, TableMetadata... tables)
{
MigrationManager.announceNewKeyspace(KeyspaceMetadata.create(name, params, Tables.of(tables)), true);
}
@@ -285,418 +286,426 @@
MigrationManager.announceNewKeyspace(KeyspaceMetadata.create(name, params, tables, Views.none(), types, Functions.none()), true);
}
- public static ColumnDefinition integerColumn(String ksName, String cfName)
+ public static void setupAuth(IRoleManager roleManager, IAuthenticator authenticator, IAuthorizer authorizer, INetworkAuthorizer networkAuthorizer)
{
- return new ColumnDefinition(ksName,
- cfName,
- ColumnIdentifier.getInterned(IntegerType.instance.fromString("42"), IntegerType.instance),
- UTF8Type.instance,
- ColumnDefinition.NO_POSITION,
- ColumnDefinition.Kind.REGULAR);
+ DatabaseDescriptor.setRoleManager(roleManager);
+ DatabaseDescriptor.setAuthenticator(authenticator);
+ DatabaseDescriptor.setAuthorizer(authorizer);
+ DatabaseDescriptor.setNetworkAuthorizer(networkAuthorizer);
+ MigrationManager.announceNewKeyspace(AuthKeyspace.metadata(), true);
+ DatabaseDescriptor.getRoleManager().setup();
+ DatabaseDescriptor.getAuthenticator().setup();
+ DatabaseDescriptor.getAuthorizer().setup();
+ DatabaseDescriptor.getNetworkAuthorizer().setup();
+ Schema.instance.registerListener(new AuthSchemaChangeListener());
}
- public static ColumnDefinition utf8Column(String ksName, String cfName)
+ public static ColumnMetadata integerColumn(String ksName, String cfName)
{
- return new ColumnDefinition(ksName,
- cfName,
- ColumnIdentifier.getInterned("fortytwo", true),
- UTF8Type.instance,
- ColumnDefinition.NO_POSITION,
- ColumnDefinition.Kind.REGULAR);
+ return new ColumnMetadata(ksName,
+ cfName,
+ ColumnIdentifier.getInterned(IntegerType.instance.fromString("42"), IntegerType.instance),
+ UTF8Type.instance,
+ ColumnMetadata.NO_POSITION,
+ ColumnMetadata.Kind.REGULAR);
}
- public static CFMetaData perRowIndexedCFMD(String ksName, String cfName)
+ public static ColumnMetadata utf8Column(String ksName, String cfName)
{
- final Map<String, String> indexOptions = Collections.singletonMap(
- IndexTarget.CUSTOM_INDEX_OPTION_NAME,
- StubIndex.class.getName());
+ return new ColumnMetadata(ksName,
+ cfName,
+ ColumnIdentifier.getInterned("fortytwo", true),
+ UTF8Type.instance,
+ ColumnMetadata.NO_POSITION,
+ ColumnMetadata.Kind.REGULAR);
+ }
- CFMetaData cfm = CFMetaData.Builder.create(ksName, cfName)
- .addPartitionKey("key", AsciiType.instance)
- .build();
+ public static TableMetadata perRowIndexedCFMD(String ksName, String cfName)
+ {
+ ColumnMetadata indexedColumn = ColumnMetadata.regularColumn(ksName, cfName, "indexed", AsciiType.instance);
- ColumnDefinition indexedColumn = ColumnDefinition.regularDef(ksName, cfName, "indexed", AsciiType.instance);
- cfm.addOrReplaceColumnDefinition(indexedColumn);
+ TableMetadata.Builder builder =
+ TableMetadata.builder(ksName, cfName)
+ .addPartitionKeyColumn("key", AsciiType.instance)
+ .addColumn(indexedColumn);
- cfm.indexes(
- cfm.getIndexes()
- .with(IndexMetadata.fromIndexTargets(cfm,
- Collections.singletonList(new IndexTarget(indexedColumn.name,
- IndexTarget.Type.VALUES)),
- "indexe1",
- IndexMetadata.Kind.CUSTOM,
- indexOptions)));
- return cfm;
+ final Map<String, String> indexOptions = Collections.singletonMap(IndexTarget.CUSTOM_INDEX_OPTION_NAME, StubIndex.class.getName());
+ builder.indexes(Indexes.of(IndexMetadata.fromIndexTargets(
+ Collections.singletonList(new IndexTarget(indexedColumn.name,
+ IndexTarget.Type.VALUES)),
+ "indexe1",
+ IndexMetadata.Kind.CUSTOM,
+ indexOptions)));
+
+ return builder.build();
}
private static void useCompression(List<KeyspaceMetadata> schema)
{
for (KeyspaceMetadata ksm : schema)
- for (CFMetaData cfm : ksm.tablesAndViews())
- cfm.compression(CompressionParams.snappy());
+ for (TableMetadata cfm : ksm.tablesAndViews())
+ MigrationManager.announceTableUpdate(cfm.unbuild().compression(CompressionParams.snappy()).build(), true);
}
- public static CFMetaData counterCFMD(String ksName, String cfName)
+ public static TableMetadata.Builder counterCFMD(String ksName, String cfName)
{
- return CFMetaData.Builder.create(ksName, cfName, false, true, true)
- .addPartitionKey("key", AsciiType.instance)
- .addClusteringColumn("name", AsciiType.instance)
- .addRegularColumn("val", CounterColumnType.instance)
- .addRegularColumn("val2", CounterColumnType.instance)
- .build()
- .compression(getCompressionParameters());
+ return TableMetadata.builder(ksName, cfName)
+ .isCounter(true)
+ .addPartitionKeyColumn("key", AsciiType.instance)
+ .addClusteringColumn("name", AsciiType.instance)
+ .addRegularColumn("val", CounterColumnType.instance)
+ .addRegularColumn("val2", CounterColumnType.instance)
+ .compression(getCompressionParameters());
}
- public static CFMetaData standardCFMD(String ksName, String cfName)
+ public static TableMetadata.Builder standardCFMD(String ksName, String cfName)
{
return standardCFMD(ksName, cfName, 1, AsciiType.instance);
}
- public static CFMetaData standardCFMD(String ksName, String cfName, int columnCount, AbstractType<?> keyType)
+ public static TableMetadata.Builder standardCFMD(String ksName, String cfName, int columnCount, AbstractType<?> keyType)
{
return standardCFMD(ksName, cfName, columnCount, keyType, AsciiType.instance);
}
- public static CFMetaData standardCFMD(String ksName, String cfName, int columnCount, AbstractType<?> keyType, AbstractType<?> valType)
+ public static TableMetadata.Builder standardCFMD(String ksName, String cfName, int columnCount, AbstractType<?> keyType, AbstractType<?> valType)
{
return standardCFMD(ksName, cfName, columnCount, keyType, valType, AsciiType.instance);
}
- public static CFMetaData standardCFMD(String ksName, String cfName, int columnCount, AbstractType<?> keyType, AbstractType<?> valType, AbstractType<?> clusteringType)
+ public static TableMetadata.Builder standardCFMD(String ksName, String cfName, int columnCount, AbstractType<?> keyType, AbstractType<?> valType, AbstractType<?> clusteringType)
{
- CFMetaData.Builder builder;
- builder = CFMetaData.Builder.create(ksName, cfName)
- .addPartitionKey("key", keyType)
- .addRegularColumn("val", valType);
+ TableMetadata.Builder builder =
+ TableMetadata.builder(ksName, cfName)
+ .addPartitionKeyColumn("key", keyType)
+ .addRegularColumn("val", valType)
+ .compression(getCompressionParameters());
- if(clusteringType != null)
- builder = builder.addClusteringColumn("name", clusteringType);
+ if (clusteringType != null)
+ builder.addClusteringColumn("name", clusteringType);
for (int i = 0; i < columnCount; i++)
builder.addRegularColumn("val" + i, AsciiType.instance);
- return builder.build()
- .compression(getCompressionParameters());
+ return builder;
}
- public static CFMetaData staticCFMD(String ksName, String cfName)
+ public static TableMetadata.Builder staticCFMD(String ksName, String cfName)
{
- return CFMetaData.Builder.create(ksName, cfName)
- .addPartitionKey("key", AsciiType.instance)
+ return TableMetadata.builder(ksName, cfName)
+ .addPartitionKeyColumn("key", AsciiType.instance)
.addClusteringColumn("cols", AsciiType.instance)
.addStaticColumn("val", AsciiType.instance)
- .addRegularColumn("val2", AsciiType.instance)
- .build();
+ .addRegularColumn("val2", AsciiType.instance);
}
- public static CFMetaData denseCFMD(String ksName, String cfName)
+ public static TableMetadata.Builder denseCFMD(String ksName, String cfName)
{
return denseCFMD(ksName, cfName, AsciiType.instance);
}
- public static CFMetaData denseCFMD(String ksName, String cfName, AbstractType cc)
+ public static TableMetadata.Builder denseCFMD(String ksName, String cfName, AbstractType cc)
{
return denseCFMD(ksName, cfName, cc, null);
}
- public static CFMetaData denseCFMD(String ksName, String cfName, AbstractType cc, AbstractType subcc)
+ public static TableMetadata.Builder denseCFMD(String ksName, String cfName, AbstractType cc, AbstractType subcc)
{
AbstractType comp = cc;
if (subcc != null)
comp = CompositeType.getInstance(Arrays.asList(new AbstractType<?>[]{cc, subcc}));
- return CFMetaData.Builder.createDense(ksName, cfName, subcc != null, false)
- .addPartitionKey("key", AsciiType.instance)
- .addClusteringColumn("cols", comp)
- .addRegularColumn("val", AsciiType.instance)
- .build()
- .compression(getCompressionParameters());
+ return TableMetadata.builder(ksName, cfName)
+ .isDense(true)
+ .isCompound(subcc != null)
+ .addPartitionKeyColumn("key", AsciiType.instance)
+ .addClusteringColumn("cols", comp)
+ .addRegularColumn("val", AsciiType.instance)
+ .compression(getCompressionParameters());
}
- public static CFMetaData superCFMD(String ksName, String cfName, AbstractType subcc)
+ // TODO: Fix superCFMD failing on legacy table creation. Seems to be applying composite comparator to partition key
+ public static TableMetadata.Builder superCFMD(String ksName, String cfName, AbstractType subcc)
{
return superCFMD(ksName, cfName, BytesType.instance, subcc);
}
-
- public static CFMetaData superCFMD(String ksName, String cfName, AbstractType cc, AbstractType subcc)
+ public static TableMetadata.Builder superCFMD(String ksName, String cfName, AbstractType cc, AbstractType subcc)
{
- return CFMetaData.Builder.createSuper(ksName, cfName, false)
- .addPartitionKey("key", BytesType.instance)
- .addClusteringColumn("column1", cc)
- .addRegularColumn("", MapType.getInstance(AsciiType.instance, subcc, true))
- .build();
+ return superCFMD(ksName, cfName, "cols", cc, subcc);
+ }
+ public static TableMetadata.Builder superCFMD(String ksName, String cfName, String ccName, AbstractType cc, AbstractType subcc)
+ {
+ return standardCFMD(ksName, cfName);
}
- public static CFMetaData compositeIndexCFMD(String ksName, String cfName, boolean withRegularIndex) throws ConfigurationException
+ public static TableMetadata.Builder compositeIndexCFMD(String ksName, String cfName, boolean withRegularIndex) throws ConfigurationException
{
return compositeIndexCFMD(ksName, cfName, withRegularIndex, false);
}
- public static CFMetaData compositeIndexCFMD(String ksName, String cfName, boolean withRegularIndex, boolean withStaticIndex) throws ConfigurationException
+ public static TableMetadata.Builder compositeIndexCFMD(String ksName, String cfName, boolean withRegularIndex, boolean withStaticIndex) throws ConfigurationException
{
- CFMetaData cfm = CFMetaData.Builder.create(ksName, cfName)
- .addPartitionKey("key", AsciiType.instance)
- .addClusteringColumn("c1", AsciiType.instance)
- .addRegularColumn("birthdate", LongType.instance)
- .addRegularColumn("notbirthdate", LongType.instance)
- .addStaticColumn("static", LongType.instance)
- .build();
+ // the withIndex flag exists to allow tests index creation
+ // on existing columns
+ TableMetadata.Builder builder =
+ TableMetadata.builder(ksName, cfName)
+ .addPartitionKeyColumn("key", AsciiType.instance)
+ .addClusteringColumn("c1", AsciiType.instance)
+ .addRegularColumn("birthdate", LongType.instance)
+ .addRegularColumn("notbirthdate", LongType.instance)
+ .addStaticColumn("static", LongType.instance)
+ .compression(getCompressionParameters());
+
+ Indexes.Builder indexes = Indexes.builder();
if (withRegularIndex)
{
- cfm.indexes(
- cfm.getIndexes()
- .with(IndexMetadata.fromIndexTargets(cfm,
- Collections.singletonList(
- new IndexTarget(new ColumnIdentifier("birthdate", true),
- IndexTarget.Type.VALUES)),
- "birthdate_key_index",
- IndexMetadata.Kind.COMPOSITES,
- Collections.EMPTY_MAP)));
+ indexes.add(IndexMetadata.fromIndexTargets(
+ Collections.singletonList(
+ new IndexTarget(new ColumnIdentifier("birthdate", true),
+ IndexTarget.Type.VALUES)),
+ cfName + "_birthdate_key_index",
+ IndexMetadata.Kind.COMPOSITES,
+ Collections.EMPTY_MAP));
}
if (withStaticIndex)
{
- cfm.indexes(
- cfm.getIndexes()
- .with(IndexMetadata.fromIndexTargets(cfm,
- Collections.singletonList(
- new IndexTarget(new ColumnIdentifier("static", true),
- IndexTarget.Type.VALUES)),
- "static_index",
- IndexMetadata.Kind.COMPOSITES,
- Collections.EMPTY_MAP)));
+ indexes.add(IndexMetadata.fromIndexTargets(
+ Collections.singletonList(
+ new IndexTarget(new ColumnIdentifier("static", true),
+ IndexTarget.Type.VALUES)),
+ cfName + "_static_index",
+ IndexMetadata.Kind.COMPOSITES,
+ Collections.EMPTY_MAP));
}
- return cfm.compression(getCompressionParameters());
+ return builder.indexes(indexes.build());
}
- public static CFMetaData compositeMultipleIndexCFMD(String ksName, String cfName) throws ConfigurationException
+ public static TableMetadata.Builder compositeMultipleIndexCFMD(String ksName, String cfName) throws ConfigurationException
{
- // the withIndex flag exists to allow tests index creation
- // on existing columns
- CFMetaData cfm = CFMetaData.Builder.create(ksName, cfName)
- .addPartitionKey("key", AsciiType.instance)
- .addClusteringColumn("c1", AsciiType.instance)
- .addRegularColumn("birthdate", LongType.instance)
- .addRegularColumn("notbirthdate", LongType.instance)
- .build();
-
- cfm.indexes(
- cfm.getIndexes()
- .with(IndexMetadata.fromIndexTargets(cfm,
- Collections.singletonList(
- new IndexTarget(new ColumnIdentifier("birthdate", true),
- IndexTarget.Type.VALUES)),
- "birthdate_key_index",
- IndexMetadata.Kind.COMPOSITES,
- Collections.EMPTY_MAP))
- .with(IndexMetadata.fromIndexTargets(cfm,
- Collections.singletonList(
- new IndexTarget(new ColumnIdentifier("notbirthdate", true),
- IndexTarget.Type.VALUES)),
- "notbirthdate_key_index",
- IndexMetadata.Kind.COMPOSITES,
- Collections.EMPTY_MAP))
- );
+ TableMetadata.Builder builder = TableMetadata.builder(ksName, cfName)
+ .addPartitionKeyColumn("key", AsciiType.instance)
+ .addClusteringColumn("c1", AsciiType.instance)
+ .addRegularColumn("birthdate", LongType.instance)
+ .addRegularColumn("notbirthdate", LongType.instance)
+ .compression(getCompressionParameters());
- return cfm.compression(getCompressionParameters());
+ Indexes.Builder indexes = Indexes.builder();
+
+ indexes.add(IndexMetadata.fromIndexTargets(Collections.singletonList(
+ new IndexTarget(new ColumnIdentifier("birthdate", true),
+ IndexTarget.Type.VALUES)),
+ "birthdate_key_index",
+ IndexMetadata.Kind.COMPOSITES,
+ Collections.EMPTY_MAP));
+ indexes.add(IndexMetadata.fromIndexTargets(Collections.singletonList(
+ new IndexTarget(new ColumnIdentifier("notbirthdate", true),
+ IndexTarget.Type.VALUES)),
+ "notbirthdate_key_index",
+ IndexMetadata.Kind.COMPOSITES,
+ Collections.EMPTY_MAP));
+
+
+ return builder.indexes(indexes.build());
}
- public static CFMetaData keysIndexCFMD(String ksName, String cfName, boolean withIndex) throws ConfigurationException
+ public static TableMetadata.Builder keysIndexCFMD(String ksName, String cfName, boolean withIndex)
{
- CFMetaData cfm = CFMetaData.Builder.createDense(ksName, cfName, false, false)
- .addPartitionKey("key", AsciiType.instance)
- .addClusteringColumn("c1", AsciiType.instance)
- .addStaticColumn("birthdate", LongType.instance)
- .addStaticColumn("notbirthdate", LongType.instance)
- .addRegularColumn("value", LongType.instance)
- .build();
+ TableMetadata.Builder builder =
+ TableMetadata.builder(ksName, cfName)
+ .isCompound(false)
+ .isDense(true)
+ .addPartitionKeyColumn("key", AsciiType.instance)
+ .addClusteringColumn("c1", AsciiType.instance)
+ .addStaticColumn("birthdate", LongType.instance)
+ .addStaticColumn("notbirthdate", LongType.instance)
+ .addRegularColumn("value", LongType.instance)
+ .compression(getCompressionParameters());
if (withIndex)
- cfm.indexes(
- cfm.getIndexes()
- .with(IndexMetadata.fromIndexTargets(cfm,
- Collections.singletonList(
- new IndexTarget(new ColumnIdentifier("birthdate", true),
- IndexTarget.Type.VALUES)),
- "birthdate_composite_index",
- IndexMetadata.Kind.KEYS,
- Collections.EMPTY_MAP)));
+ {
+ IndexMetadata index =
+ IndexMetadata.fromIndexTargets(
+ Collections.singletonList(new IndexTarget(new ColumnIdentifier("birthdate", true),
+ IndexTarget.Type.VALUES)),
+ cfName + "_birthdate_composite_index",
+ IndexMetadata.Kind.KEYS,
+ Collections.EMPTY_MAP);
+ builder.indexes(Indexes.builder().add(index).build());
+ }
-
- return cfm.compression(getCompressionParameters());
+ return builder;
}
- public static CFMetaData customIndexCFMD(String ksName, String cfName) throws ConfigurationException
+ public static TableMetadata.Builder customIndexCFMD(String ksName, String cfName)
{
- CFMetaData cfm = CFMetaData.Builder.createDense(ksName, cfName, false, false)
- .addPartitionKey("key", AsciiType.instance)
- .addClusteringColumn("c1", AsciiType.instance)
- .addRegularColumn("value", LongType.instance)
- .build();
+ TableMetadata.Builder builder =
+ TableMetadata.builder(ksName, cfName)
+ .isCompound(false)
+ .isDense(true)
+ .addPartitionKeyColumn("key", AsciiType.instance)
+ .addClusteringColumn("c1", AsciiType.instance)
+ .addRegularColumn("value", LongType.instance)
+ .compression(getCompressionParameters());
- cfm.indexes(
- cfm.getIndexes()
- .with(IndexMetadata.fromIndexTargets(cfm,
- Collections.singletonList(
- new IndexTarget(new ColumnIdentifier("value", true),
- IndexTarget.Type.VALUES)),
- "value_index",
- IndexMetadata.Kind.CUSTOM,
- Collections.singletonMap(
- IndexTarget.CUSTOM_INDEX_OPTION_NAME,
- StubIndex.class.getName()))));
+ IndexMetadata index =
+ IndexMetadata.fromIndexTargets(
+ Collections.singletonList(new IndexTarget(new ColumnIdentifier("value", true), IndexTarget.Type.VALUES)),
+ cfName + "_value_index",
+ IndexMetadata.Kind.CUSTOM,
+ Collections.singletonMap(IndexTarget.CUSTOM_INDEX_OPTION_NAME, StubIndex.class.getName()));
+ builder.indexes(Indexes.of(index));
- return cfm.compression(getCompressionParameters());
+ return builder;
}
- public static CFMetaData jdbcCFMD(String ksName, String cfName, AbstractType comp)
+ public static TableMetadata.Builder jdbcCFMD(String ksName, String cfName, AbstractType comp)
{
- return CFMetaData.Builder.create(ksName, cfName).addPartitionKey("key", BytesType.instance)
- .build()
- .compression(getCompressionParameters());
+ return TableMetadata.builder(ksName, cfName)
+ .addPartitionKeyColumn("key", BytesType.instance)
+ .compression(getCompressionParameters());
}
- public static CFMetaData sasiCFMD(String ksName, String cfName)
+ public static TableMetadata.Builder sasiCFMD(String ksName, String cfName)
{
- CFMetaData cfm = CFMetaData.Builder.create(ksName, cfName)
- .addPartitionKey("id", UTF8Type.instance)
- .addRegularColumn("first_name", UTF8Type.instance)
- .addRegularColumn("last_name", UTF8Type.instance)
- .addRegularColumn("age", Int32Type.instance)
- .addRegularColumn("height", Int32Type.instance)
- .addRegularColumn("timestamp", LongType.instance)
- .addRegularColumn("address", UTF8Type.instance)
- .addRegularColumn("score", DoubleType.instance)
- .addRegularColumn("comment", UTF8Type.instance)
- .addRegularColumn("comment_suffix_split", UTF8Type.instance)
- .addRegularColumn("/output/full-name/", UTF8Type.instance)
- .addRegularColumn("/data/output/id", UTF8Type.instance)
- .addRegularColumn("first_name_prefix", UTF8Type.instance)
- .build();
+ TableMetadata.Builder builder =
+ TableMetadata.builder(ksName, cfName)
+ .addPartitionKeyColumn("id", UTF8Type.instance)
+ .addRegularColumn("first_name", UTF8Type.instance)
+ .addRegularColumn("last_name", UTF8Type.instance)
+ .addRegularColumn("age", Int32Type.instance)
+ .addRegularColumn("height", Int32Type.instance)
+ .addRegularColumn("timestamp", LongType.instance)
+ .addRegularColumn("address", UTF8Type.instance)
+ .addRegularColumn("score", DoubleType.instance)
+ .addRegularColumn("comment", UTF8Type.instance)
+ .addRegularColumn("comment_suffix_split", UTF8Type.instance)
+ .addRegularColumn("/output/full-name/", UTF8Type.instance)
+ .addRegularColumn("/data/output/id", UTF8Type.instance)
+ .addRegularColumn("first_name_prefix", UTF8Type.instance);
- cfm.indexes(cfm.getIndexes()
- .with(IndexMetadata.fromSchemaMetadata("first_name", IndexMetadata.Kind.CUSTOM, new HashMap<String, String>()
- {{
- put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, SASIIndex.class.getName());
- put(IndexTarget.TARGET_OPTION_NAME, "first_name");
- put("mode", OnDiskIndexBuilder.Mode.CONTAINS.toString());
- }}))
- .with(IndexMetadata.fromSchemaMetadata("last_name", IndexMetadata.Kind.CUSTOM, new HashMap<String, String>()
- {{
- put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, SASIIndex.class.getName());
- put(IndexTarget.TARGET_OPTION_NAME, "last_name");
- put("mode", OnDiskIndexBuilder.Mode.CONTAINS.toString());
- }}))
- .with(IndexMetadata.fromSchemaMetadata("age", IndexMetadata.Kind.CUSTOM, new HashMap<String, String>()
- {{
- put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, SASIIndex.class.getName());
- put(IndexTarget.TARGET_OPTION_NAME, "age");
+ Indexes.Builder indexes = Indexes.builder();
- }}))
- .with(IndexMetadata.fromSchemaMetadata("timestamp", IndexMetadata.Kind.CUSTOM, new HashMap<String, String>()
- {{
- put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, SASIIndex.class.getName());
- put(IndexTarget.TARGET_OPTION_NAME, "timestamp");
- put("mode", OnDiskIndexBuilder.Mode.SPARSE.toString());
+ indexes.add(IndexMetadata.fromSchemaMetadata(cfName + "_first_name", IndexMetadata.Kind.CUSTOM, new HashMap<String, String>()
+ {{
+ put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, SASIIndex.class.getName());
+ put(IndexTarget.TARGET_OPTION_NAME, "first_name");
+ put("mode", OnDiskIndexBuilder.Mode.CONTAINS.toString());
+ }}))
+ .add(IndexMetadata.fromSchemaMetadata(cfName + "_last_name", IndexMetadata.Kind.CUSTOM, new HashMap<String, String>()
+ {{
+ put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, SASIIndex.class.getName());
+ put(IndexTarget.TARGET_OPTION_NAME, "last_name");
+ put("mode", OnDiskIndexBuilder.Mode.CONTAINS.toString());
+ }}))
+ .add(IndexMetadata.fromSchemaMetadata(cfName + "_age", IndexMetadata.Kind.CUSTOM, new HashMap<String, String>()
+ {{
+ put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, SASIIndex.class.getName());
+ put(IndexTarget.TARGET_OPTION_NAME, "age");
+ }}))
+ .add(IndexMetadata.fromSchemaMetadata(cfName + "_timestamp", IndexMetadata.Kind.CUSTOM, new HashMap<String, String>()
+ {{
+ put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, SASIIndex.class.getName());
+ put(IndexTarget.TARGET_OPTION_NAME, "timestamp");
+ put("mode", OnDiskIndexBuilder.Mode.SPARSE.toString());
- }}))
- .with(IndexMetadata.fromSchemaMetadata("address", IndexMetadata.Kind.CUSTOM, new HashMap<String, String>()
- {{
- put("analyzer_class", "org.apache.cassandra.index.sasi.analyzer.NonTokenizingAnalyzer");
- put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, SASIIndex.class.getName());
- put(IndexTarget.TARGET_OPTION_NAME, "address");
- put("mode", OnDiskIndexBuilder.Mode.PREFIX.toString());
- put("case_sensitive", "false");
- }}))
- .with(IndexMetadata.fromSchemaMetadata("score", IndexMetadata.Kind.CUSTOM, new HashMap<String, String>()
- {{
- put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, SASIIndex.class.getName());
- put(IndexTarget.TARGET_OPTION_NAME, "score");
- }}))
- .with(IndexMetadata.fromSchemaMetadata("comment", IndexMetadata.Kind.CUSTOM, new HashMap<String, String>()
- {{
- put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, SASIIndex.class.getName());
- put(IndexTarget.TARGET_OPTION_NAME, "comment");
- put("mode", OnDiskIndexBuilder.Mode.CONTAINS.toString());
- put("analyzed", "true");
- }}))
- .with(IndexMetadata.fromSchemaMetadata("comment_suffix_split", IndexMetadata.Kind.CUSTOM, new HashMap<String, String>()
- {{
- put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, SASIIndex.class.getName());
- put(IndexTarget.TARGET_OPTION_NAME, "comment_suffix_split");
- put("mode", OnDiskIndexBuilder.Mode.CONTAINS.toString());
- put("analyzed", "false");
- }}))
- .with(IndexMetadata.fromSchemaMetadata("output_full_name", IndexMetadata.Kind.CUSTOM, new HashMap<String, String>()
- {{
- put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, SASIIndex.class.getName());
- put(IndexTarget.TARGET_OPTION_NAME, "/output/full-name/");
- put("analyzed", "true");
- put("analyzer_class", "org.apache.cassandra.index.sasi.analyzer.NonTokenizingAnalyzer");
- put("case_sensitive", "false");
- }}))
- .with(IndexMetadata.fromSchemaMetadata("data_output_id", IndexMetadata.Kind.CUSTOM, new HashMap<String, String>()
- {{
- put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, SASIIndex.class.getName());
- put(IndexTarget.TARGET_OPTION_NAME, "/data/output/id");
- put("mode", OnDiskIndexBuilder.Mode.CONTAINS.toString());
- }}))
- .with(IndexMetadata.fromSchemaMetadata("first_name_prefix", IndexMetadata.Kind.CUSTOM, new HashMap<String, String>()
- {{
- put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, SASIIndex.class.getName());
- put(IndexTarget.TARGET_OPTION_NAME, "first_name_prefix");
- put("analyzed", "true");
- put("tokenization_normalize_lowercase", "true");
- }})));
+ }}))
+ .add(IndexMetadata.fromSchemaMetadata(cfName + "_address", IndexMetadata.Kind.CUSTOM, new HashMap<String, String>()
+ {{
+ put("analyzer_class", "org.apache.cassandra.index.sasi.analyzer.NonTokenizingAnalyzer");
+ put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, SASIIndex.class.getName());
+ put(IndexTarget.TARGET_OPTION_NAME, "address");
+ put("mode", OnDiskIndexBuilder.Mode.PREFIX.toString());
+ put("case_sensitive", "false");
+ }}))
+ .add(IndexMetadata.fromSchemaMetadata(cfName + "_score", IndexMetadata.Kind.CUSTOM, new HashMap<String, String>()
+ {{
+ put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, SASIIndex.class.getName());
+ put(IndexTarget.TARGET_OPTION_NAME, "score");
+ }}))
+ .add(IndexMetadata.fromSchemaMetadata(cfName + "_comment", IndexMetadata.Kind.CUSTOM, new HashMap<String, String>()
+ {{
+ put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, SASIIndex.class.getName());
+ put(IndexTarget.TARGET_OPTION_NAME, "comment");
+ put("mode", OnDiskIndexBuilder.Mode.CONTAINS.toString());
+ put("analyzed", "true");
+ }}))
+ .add(IndexMetadata.fromSchemaMetadata(cfName + "_comment_suffix_split", IndexMetadata.Kind.CUSTOM, new HashMap<String, String>()
+ {{
+ put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, SASIIndex.class.getName());
+ put(IndexTarget.TARGET_OPTION_NAME, "comment_suffix_split");
+ put("mode", OnDiskIndexBuilder.Mode.CONTAINS.toString());
+ put("analyzed", "false");
+ }}))
+ .add(IndexMetadata.fromSchemaMetadata(cfName + "_output_full_name", IndexMetadata.Kind.CUSTOM, new HashMap<String, String>()
+ {{
+ put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, SASIIndex.class.getName());
+ put(IndexTarget.TARGET_OPTION_NAME, "/output/full-name/");
+ put("analyzed", "true");
+ put("analyzer_class", "org.apache.cassandra.index.sasi.analyzer.NonTokenizingAnalyzer");
+ put("case_sensitive", "false");
+ }}))
+ .add(IndexMetadata.fromSchemaMetadata(cfName + "_data_output_id", IndexMetadata.Kind.CUSTOM, new HashMap<String, String>()
+ {{
+ put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, SASIIndex.class.getName());
+ put(IndexTarget.TARGET_OPTION_NAME, "/data/output/id");
+ put("mode", OnDiskIndexBuilder.Mode.CONTAINS.toString());
+ }}))
+ .add(IndexMetadata.fromSchemaMetadata(cfName + "_first_name_prefix", IndexMetadata.Kind.CUSTOM, new HashMap<String, String>()
+ {{
+ put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, SASIIndex.class.getName());
+ put(IndexTarget.TARGET_OPTION_NAME, "first_name_prefix");
+ put("analyzed", "true");
+ put("tokenization_normalize_lowercase", "true");
+ }}));
- return cfm;
- }
+ return builder.indexes(indexes.build());
+}
- public static CFMetaData clusteringSASICFMD(String ksName, String cfName)
+public static TableMetadata.Builder clusteringSASICFMD(String ksName, String cfName)
+{
+ return clusteringSASICFMD(ksName, cfName, "location", "age", "height", "score");
+}
+
+ public static TableMetadata.Builder clusteringSASICFMD(String ksName, String cfName, String...indexedColumns)
{
- return clusteringSASICFMD(ksName, cfName, "location", "age", "height", "score");
- }
-
- public static CFMetaData clusteringSASICFMD(String ksName, String cfName, String...indexedColumns)
- {
- CFMetaData cfm = CFMetaData.Builder.create(ksName, cfName)
- .addPartitionKey("name", UTF8Type.instance)
- .addClusteringColumn("location", UTF8Type.instance)
- .addClusteringColumn("age", Int32Type.instance)
- .addRegularColumn("height", Int32Type.instance)
- .addRegularColumn("score", DoubleType.instance)
- .addStaticColumn("nickname", UTF8Type.instance)
- .build();
-
- Indexes indexes = cfm.getIndexes();
+ Indexes.Builder indexes = Indexes.builder();
for (String indexedColumn : indexedColumns)
{
- indexes = indexes.with(IndexMetadata.fromSchemaMetadata(indexedColumn, IndexMetadata.Kind.CUSTOM, new HashMap<String, String>()
+ indexes.add(IndexMetadata.fromSchemaMetadata(cfName + "_" + indexedColumn, IndexMetadata.Kind.CUSTOM, new HashMap<String, String>()
{{
put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, SASIIndex.class.getName());
put(IndexTarget.TARGET_OPTION_NAME, indexedColumn);
put("mode", OnDiskIndexBuilder.Mode.PREFIX.toString());
}}));
}
- cfm.indexes(indexes);
- return cfm;
+
+ return TableMetadata.builder(ksName, cfName)
+ .addPartitionKeyColumn("name", UTF8Type.instance)
+ .addClusteringColumn("location", UTF8Type.instance)
+ .addClusteringColumn("age", Int32Type.instance)
+ .addRegularColumn("height", Int32Type.instance)
+ .addRegularColumn("score", DoubleType.instance)
+ .addStaticColumn("nickname", UTF8Type.instance)
+ .indexes(indexes.build());
}
- public static CFMetaData staticSASICFMD(String ksName, String cfName)
+ public static TableMetadata.Builder staticSASICFMD(String ksName, String cfName)
{
- CFMetaData cfm = CFMetaData.Builder.create(ksName, cfName)
- .addPartitionKey("sensor_id", Int32Type.instance)
- .addStaticColumn("sensor_type", UTF8Type.instance)
- .addClusteringColumn("date", LongType.instance)
- .addRegularColumn("value", DoubleType.instance)
- .addRegularColumn("variance", Int32Type.instance)
- .build();
+ TableMetadata.Builder builder =
+ TableMetadata.builder(ksName, cfName)
+ .addPartitionKeyColumn("sensor_id", Int32Type.instance)
+ .addStaticColumn("sensor_type", UTF8Type.instance)
+ .addClusteringColumn("date", LongType.instance)
+ .addRegularColumn("value", DoubleType.instance)
+ .addRegularColumn("variance", Int32Type.instance);
- Indexes indexes = cfm.getIndexes();
- indexes = indexes.with(IndexMetadata.fromSchemaMetadata("sensor_type", IndexMetadata.Kind.CUSTOM, new HashMap<String, String>()
+ Indexes.Builder indexes = Indexes.builder();
+
+ indexes.add(IndexMetadata.fromSchemaMetadata(cfName + "_sensor_type", IndexMetadata.Kind.CUSTOM, new HashMap<String, String>()
{{
put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, SASIIndex.class.getName());
put(IndexTarget.TARGET_OPTION_NAME, "sensor_type");
@@ -705,34 +714,34 @@
put("case_sensitive", "false");
}}));
- indexes = indexes.with(IndexMetadata.fromSchemaMetadata("value", IndexMetadata.Kind.CUSTOM, new HashMap<String, String>()
+ indexes.add(IndexMetadata.fromSchemaMetadata(cfName + "_value", IndexMetadata.Kind.CUSTOM, new HashMap<String, String>()
{{
put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, SASIIndex.class.getName());
put(IndexTarget.TARGET_OPTION_NAME, "value");
put("mode", OnDiskIndexBuilder.Mode.PREFIX.toString());
}}));
- indexes = indexes.with(IndexMetadata.fromSchemaMetadata("variance", IndexMetadata.Kind.CUSTOM, new HashMap<String, String>()
+ indexes.add(IndexMetadata.fromSchemaMetadata(cfName + "_variance", IndexMetadata.Kind.CUSTOM, new HashMap<String, String>()
{{
put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, SASIIndex.class.getName());
put(IndexTarget.TARGET_OPTION_NAME, "variance");
put("mode", OnDiskIndexBuilder.Mode.PREFIX.toString());
}}));
- cfm.indexes(indexes);
- return cfm;
+ return builder.indexes(indexes.build());
}
- public static CFMetaData fullTextSearchSASICFMD(String ksName, String cfName)
+ public static TableMetadata.Builder fullTextSearchSASICFMD(String ksName, String cfName)
{
- CFMetaData cfm = CFMetaData.Builder.create(ksName, cfName)
- .addPartitionKey("song_id", UUIDType.instance)
- .addRegularColumn("title", UTF8Type.instance)
- .addRegularColumn("artist", UTF8Type.instance)
- .build();
+ TableMetadata.Builder builder =
+ TableMetadata.builder(ksName, cfName)
+ .addPartitionKeyColumn("song_id", UUIDType.instance)
+ .addRegularColumn("title", UTF8Type.instance)
+ .addRegularColumn("artist", UTF8Type.instance);
- Indexes indexes = cfm.getIndexes();
- indexes = indexes.with(IndexMetadata.fromSchemaMetadata("title", IndexMetadata.Kind.CUSTOM, new HashMap<String, String>()
+ Indexes.Builder indexes = Indexes.builder();
+
+ indexes.add(IndexMetadata.fromSchemaMetadata(cfName + "_title", IndexMetadata.Kind.CUSTOM, new HashMap<String, String>()
{{
put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, SASIIndex.class.getName());
put(IndexTarget.TARGET_OPTION_NAME, "title");
@@ -744,7 +753,7 @@
put("tokenization_normalize_lowercase", "true");
}}));
- indexes = indexes.with(IndexMetadata.fromSchemaMetadata("artist", IndexMetadata.Kind.CUSTOM, new HashMap<String, String>()
+ indexes.add(IndexMetadata.fromSchemaMetadata(cfName + "_artist", IndexMetadata.Kind.CUSTOM, new HashMap<String, String>()
{{
put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, SASIIndex.class.getName());
put(IndexTarget.TARGET_OPTION_NAME, "artist");
@@ -754,8 +763,7 @@
}}));
- cfm.indexes(indexes);
- return cfm;
+ return builder.indexes(indexes.build());
}
public static CompressionParams getCompressionParameters()
@@ -766,7 +774,7 @@
public static CompressionParams getCompressionParameters(Integer chunkSize)
{
if (Boolean.parseBoolean(System.getProperty("cassandra.test.compression", "false")))
- return CompressionParams.snappy(chunkSize);
+ return chunkSize != null ? CompressionParams.snappy(chunkSize) : CompressionParams.snappy();
return CompressionParams.noCompression();
}
@@ -818,7 +826,7 @@
public static void insertData(String keyspace, String columnFamily, int offset, int numberOfRows)
{
- CFMetaData cfm = Schema.instance.getCFMetaData(keyspace, columnFamily);
+ TableMetadata cfm = Schema.instance.getTableMetadata(keyspace, columnFamily);
for (int i = offset; i < offset + numberOfRows; i++)
{
diff --git a/test/unit/org/apache/cassandra/UpdateBuilder.java b/test/unit/org/apache/cassandra/UpdateBuilder.java
index 9fcda15..0f3c918 100644
--- a/test/unit/org/apache/cassandra/UpdateBuilder.java
+++ b/test/unit/org/apache/cassandra/UpdateBuilder.java
@@ -17,13 +17,10 @@
*/
package org.apache.cassandra;
-import java.nio.ByteBuffer;
-
-import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.rows.*;
import org.apache.cassandra.db.partitions.*;
-import org.apache.cassandra.utils.FBUtilities;
/**
@@ -42,7 +39,7 @@
this.updateBuilder = updateBuilder;
}
- public static UpdateBuilder create(CFMetaData metadata, Object... partitionKey)
+ public static UpdateBuilder create(TableMetadata metadata, Object... partitionKey)
{
return new UpdateBuilder(PartitionUpdate.simpleBuilder(metadata, partitionKey));
}
diff --git a/test/unit/org/apache/cassandra/Util.java b/test/unit/org/apache/cassandra/Util.java
index fa24167..7880276 100644
--- a/test/unit/org/apache/cassandra/Util.java
+++ b/test/unit/org/apache/cassandra/Util.java
@@ -21,26 +21,36 @@
import java.io.Closeable;
import java.io.EOFException;
+import java.io.File;
import java.io.IOError;
-import java.net.InetAddress;
import java.net.UnknownHostException;
import java.nio.ByteBuffer;
import java.util.*;
import java.util.concurrent.Callable;
import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.Supplier;
+import java.util.stream.Collectors;
import com.google.common.base.Function;
import com.google.common.base.Preconditions;
+import com.google.common.collect.Iterables;
import com.google.common.collect.Iterators;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
+import afu.org.checkerframework.checker.oigj.qual.O;
+import org.apache.cassandra.db.compaction.ActiveCompactionsTracker;
+import org.apache.cassandra.db.compaction.CompactionTasks;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.ReplicaCollection;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.ColumnIdentifier;
@@ -71,7 +81,9 @@
import org.apache.cassandra.utils.FBUtilities;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
public class Util
@@ -110,13 +122,6 @@
return PartitionPosition.ForKey.get(ByteBufferUtil.bytes(key), partitioner);
}
- public static Cell getRegularCell(CFMetaData metadata, Row row, String name)
- {
- ColumnDefinition column = metadata.getColumnDefinition(ByteBufferUtil.bytes(name));
- assert column != null;
- return row.getCell(column);
- }
-
public static Clustering clustering(ClusteringComparator comparator, Object... o)
{
return comparator.make(o);
@@ -179,12 +184,12 @@
{
IMutation first = mutations.get(0);
String keyspaceName = first.getKeyspaceName();
- UUID cfid = first.getColumnFamilyIds().iterator().next();
+ TableId tableId = first.getTableIds().iterator().next();
for (Mutation rm : mutations)
rm.applyUnsafe();
- ColumnFamilyStore store = Keyspace.open(keyspaceName).getColumnFamilyStore(cfid);
+ ColumnFamilyStore store = Keyspace.open(keyspaceName).getColumnFamilyStore(tableId);
store.forceBlockingFlush();
return store;
}
@@ -198,7 +203,7 @@
* Creates initial set of nodes and tokens. Nodes are added to StorageService as 'normal'
*/
public static void createInitialRing(StorageService ss, IPartitioner partitioner, List<Token> endpointTokens,
- List<Token> keyTokens, List<InetAddress> hosts, List<UUID> hostIds, int howMany)
+ List<Token> keyTokens, List<InetAddressAndPort> hosts, List<UUID> hostIds, int howMany)
throws UnknownHostException
{
// Expand pool of host IDs as necessary
@@ -216,10 +221,13 @@
for (int i=0; i<endpointTokens.size(); i++)
{
- InetAddress ep = InetAddress.getByName("127.0.0." + String.valueOf(i + 1));
+ InetAddressAndPort ep = InetAddressAndPort.getByName("127.0.0." + String.valueOf(i + 1));
Gossiper.instance.initializeNodeUnsafe(ep, hostIds.get(i), 1);
Gossiper.instance.injectApplicationState(ep, ApplicationState.TOKENS, new VersionedValue.VersionedValueFactory(partitioner).tokens(Collections.singleton(endpointTokens.get(i))));
ss.onChange(ep,
+ ApplicationState.STATUS_WITH_PORT,
+ new VersionedValue.VersionedValueFactory(partitioner).normal(Collections.singleton(endpointTokens.get(i))));
+ ss.onChange(ep,
ApplicationState.STATUS,
new VersionedValue.VersionedValueFactory(partitioner).normal(Collections.singleton(endpointTokens.get(i))));
hosts.add(ep);
@@ -241,9 +249,11 @@
public static void compact(ColumnFamilyStore cfs, Collection<SSTableReader> sstables)
{
int gcBefore = cfs.gcBefore(FBUtilities.nowInSeconds());
- List<AbstractCompactionTask> tasks = cfs.getCompactionStrategyManager().getUserDefinedTasks(sstables, gcBefore);
- for (AbstractCompactionTask task : tasks)
- task.execute(null);
+ try (CompactionTasks tasks = cfs.getCompactionStrategyManager().getUserDefinedTasks(sstables, gcBefore))
+ {
+ for (AbstractCompactionTask task : tasks)
+ task.execute(ActiveCompactionsTracker.NOOP);
+ }
}
public static void expectEOF(Callable<?> callable)
@@ -270,7 +280,7 @@
public static AbstractReadCommandBuilder.SinglePartitionBuilder cmd(ColumnFamilyStore cfs, Object... partitionKey)
{
- return new AbstractReadCommandBuilder.SinglePartitionBuilder(cfs, makeKey(cfs.metadata, partitionKey));
+ return new AbstractReadCommandBuilder.SinglePartitionBuilder(cfs, makeKey(cfs.metadata(), partitionKey));
}
public static AbstractReadCommandBuilder.PartitionRangeBuilder cmd(ColumnFamilyStore cfs)
@@ -278,13 +288,13 @@
return new AbstractReadCommandBuilder.PartitionRangeBuilder(cfs);
}
- static DecoratedKey makeKey(CFMetaData metadata, Object... partitionKey)
+ static DecoratedKey makeKey(TableMetadata metadata, Object... partitionKey)
{
if (partitionKey.length == 1 && partitionKey[0] instanceof DecoratedKey)
return (DecoratedKey)partitionKey[0];
- ByteBuffer key = CFMetaData.serializePartitionKey(metadata.getKeyValidatorAsClusteringComparator().make(partitionKey));
- return metadata.decorateKey(key);
+ ByteBuffer key = metadata.partitionKeyAsClusteringComparator().make(partitionKey).serializeAsPartitionKey();
+ return metadata.partitioner.decorateKey(key);
}
public static void assertEmptyUnfiltered(ReadCommand command)
@@ -296,7 +306,7 @@
{
try (UnfilteredRowIterator partition = iterator.next())
{
- throw new AssertionError("Expected no results for query " + command.toCQLString() + " but got key " + command.metadata().getKeyValidator().getString(partition.partitionKey().getKey()));
+ throw new AssertionError("Expected no results for query " + command.toCQLString() + " but got key " + command.metadata().partitionKeyType.getString(partition.partitionKey().getKey()));
}
}
}
@@ -311,7 +321,7 @@
{
try (RowIterator partition = iterator.next())
{
- throw new AssertionError("Expected no results for query " + command.toCQLString() + " but got key " + command.metadata().getKeyValidator().getString(partition.partitionKey().getKey()));
+ throw new AssertionError("Expected no results for query " + command.toCQLString() + " but got key " + command.metadata().partitionKeyType.getString(partition.partitionKey().getKey()));
}
}
}
@@ -423,7 +433,7 @@
public static Cell cell(ColumnFamilyStore cfs, Row row, String columnName)
{
- ColumnDefinition def = cfs.metadata.getColumnDefinition(ByteBufferUtil.bytes(columnName));
+ ColumnMetadata def = cfs.metadata().getColumn(ByteBufferUtil.bytes(columnName));
assert def != null;
return row.getCell(def);
}
@@ -436,7 +446,7 @@
public static void assertCellValue(Object value, ColumnFamilyStore cfs, Row row, String columnName)
{
Cell cell = cell(cfs, row, columnName);
- assert cell != null : "Row " + row.toString(cfs.metadata) + " has no cell for " + columnName;
+ assert cell != null : "Row " + row.toString(cfs.metadata()) + " has no cell for " + columnName;
assertEquals(value, cell.column().type.compose(cell.value()));
}
@@ -449,6 +459,14 @@
}
}
+ public static void consume(UnfilteredPartitionIterator iterator)
+ {
+ while (iterator.hasNext())
+ {
+ consume(iterator.next());
+ }
+ }
+
public static int size(PartitionIterator iter)
{
int size = 0;
@@ -460,15 +478,6 @@
return size;
}
- public static CBuilder getCBuilderForCFM(CFMetaData cfm)
- {
- List<ColumnDefinition> clusteringColumns = cfm.clusteringColumns();
- List<AbstractType<?>> types = new ArrayList<>(clusteringColumns.size());
- for (ColumnDefinition def : clusteringColumns)
- types.add(def.type);
- return CBuilder.create(new ClusteringComparator(types));
- }
-
public static boolean equal(UnfilteredRowIterator a, UnfilteredRowIterator b)
{
return Objects.equals(a.columns(), b.columns())
@@ -490,14 +499,23 @@
&& Iterators.elementsEqual(a, b);
}
+ public static boolean sameContent(RowIterator a, RowIterator b)
+ {
+ return Objects.equals(a.metadata(), b.metadata())
+ && Objects.equals(a.isReverseOrder(), b.isReverseOrder())
+ && Objects.equals(a.partitionKey(), b.partitionKey())
+ && Objects.equals(a.staticRow(), b.staticRow())
+ && Iterators.elementsEqual(a, b);
+ }
+
public static boolean sameContent(Mutation a, Mutation b)
{
- if (!a.key().equals(b.key()) || !a.getColumnFamilyIds().equals(b.getColumnFamilyIds()))
+ if (!a.key().equals(b.key()) || !a.getTableIds().equals(b.getTableIds()))
return false;
- for (UUID cfId : a.getColumnFamilyIds())
+ for (PartitionUpdate update : a.getPartitionUpdates())
{
- if (!sameContent(a.getPartitionUpdate(cfId).unfilteredIterator(), b.getPartitionUpdate(cfId).unfilteredIterator()))
+ if (!sameContent(update.unfilteredIterator(), b.getPartitionUpdate(update.metadata()).unfilteredIterator()))
return false;
}
return true;
@@ -521,9 +539,9 @@
StringUtils.join(expectedColumnNames, ","));
}
- public static void assertColumn(CFMetaData cfm, Row row, String name, String value, long timestamp)
+ public static void assertColumn(TableMetadata cfm, Row row, String name, String value, long timestamp)
{
- Cell cell = row.getCell(cfm.getColumnDefinition(new ColumnIdentifier(name, true)));
+ Cell cell = row.getCell(cfm.getColumn(new ColumnIdentifier(name, true)));
assertColumn(cell, value, timestamp);
}
@@ -534,7 +552,7 @@
assertEquals(timestamp, cell.timestamp());
}
- public static void assertClustering(CFMetaData cfm, Row row, Object... clusteringValue)
+ public static void assertClustering(TableMetadata cfm, Row row, Object... clusteringValue)
{
assertEquals(row.clustering().size(), clusteringValue.length);
assertEquals(0, cfm.comparator.compare(row.clustering(), cfm.comparator.make(clusteringValue)));
@@ -563,16 +581,24 @@
}
}
- public static void spinAssertEquals(Object expected, Supplier<Object> s, int timeoutInSeconds)
+ public static void spinAssertEquals(Object expected, Supplier<Object> actualSupplier, int timeoutInSeconds)
{
- long start = System.currentTimeMillis();
- while (System.currentTimeMillis() < start + (1000 * timeoutInSeconds))
+ spinAssertEquals(null, expected, actualSupplier, timeoutInSeconds, TimeUnit.SECONDS);
+ }
+
+ public static <T> void spinAssertEquals(String message, T expected, Supplier<? extends T> actualSupplier, long timeout, TimeUnit timeUnit)
+ {
+ long startNano = System.nanoTime();
+ long expireAtNano = startNano + timeUnit.toNanos(timeout);
+ T actual = null;
+ while (System.nanoTime() < expireAtNano)
{
- if (s.get().equals(expected))
+ actual = actualSupplier.get();
+ if (actual.equals(expected))
break;
Thread.yield();
}
- assertEquals(expected, s.get());
+ assertEquals(message, expected, actual);
}
public static void joinThread(Thread thread) throws InterruptedException
@@ -647,12 +673,12 @@
{
Iterator<Unfiltered> content;
- public UnfilteredSource(CFMetaData cfm, DecoratedKey partitionKey, Row staticRow, Iterator<Unfiltered> content)
+ public UnfilteredSource(TableMetadata metadata, DecoratedKey partitionKey, Row staticRow, Iterator<Unfiltered> content)
{
- super(cfm,
+ super(metadata,
partitionKey,
DeletionTime.LIVE,
- cfm.partitionColumns(),
+ metadata.regularAndStaticColumns(),
staticRow != null ? staticRow : Rows.EMPTY_STATIC_ROW,
false,
EncodingStats.NO_STATS);
@@ -697,19 +723,54 @@
public static PagingState makeSomePagingState(ProtocolVersion protocolVersion, int remainingInPartition)
{
- CFMetaData metadata = CFMetaData.Builder.create("ks", "tbl")
- .addPartitionKey("k", AsciiType.instance)
- .addClusteringColumn("c1", AsciiType.instance)
- .addClusteringColumn("c1", Int32Type.instance)
- .addRegularColumn("myCol", AsciiType.instance)
- .build();
+ TableMetadata metadata =
+ TableMetadata.builder("ks", "tbl")
+ .addPartitionKeyColumn("k", AsciiType.instance)
+ .addClusteringColumn("c1", AsciiType.instance)
+ .addClusteringColumn("c2", Int32Type.instance)
+ .addRegularColumn("myCol", AsciiType.instance)
+ .build();
ByteBuffer pk = ByteBufferUtil.bytes("someKey");
- ColumnDefinition def = metadata.getColumnDefinition(new ColumnIdentifier("myCol", false));
+ ColumnMetadata def = metadata.getColumn(new ColumnIdentifier("myCol", false));
Clustering c = Clustering.make(ByteBufferUtil.bytes("c1"), ByteBufferUtil.bytes(42));
Row row = BTreeRow.singleCellRow(c, BufferCell.live(def, 0, ByteBufferUtil.EMPTY_BYTE_BUFFER));
PagingState.RowMark mark = PagingState.RowMark.create(metadata, row, protocolVersion);
return new PagingState(pk, mark, 10, remainingInPartition);
}
+
+ public static void assertRCEquals(ReplicaCollection<?> a, ReplicaCollection<?> b)
+ {
+ assertTrue(a + " not equal to " + b, Iterables.elementsEqual(a, b));
+ }
+
+ public static void assertNotRCEquals(ReplicaCollection<?> a, ReplicaCollection<?> b)
+ {
+ assertFalse(a + " equal to " + b, Iterables.elementsEqual(a, b));
+ }
+
+ /**
+ * Makes sure that the sstables on disk are the same ones as the cfs live sstables (that they have the same generation)
+ */
+ public static void assertOnDiskState(ColumnFamilyStore cfs, int expectedSSTableCount)
+ {
+ LifecycleTransaction.waitForDeletions();
+ assertEquals(expectedSSTableCount, cfs.getLiveSSTables().size());
+ Set<Integer> liveGenerations = cfs.getLiveSSTables().stream().map(sstable -> sstable.descriptor.generation).collect(Collectors.toSet());
+ int fileCount = 0;
+ for (File f : cfs.getDirectories().getCFDirectories())
+ {
+ for (File sst : f.listFiles())
+ {
+ if (sst.getName().contains("Data"))
+ {
+ Descriptor d = Descriptor.fromFilename(sst.getAbsolutePath());
+ assertTrue(liveGenerations.contains(d.generation));
+ fileCount++;
+ }
+ }
+ }
+ assertEquals(expectedSSTableCount, fileCount);
+ }
}
diff --git a/test/unit/org/apache/cassandra/audit/AuditLogFilterTest.java b/test/unit/org/apache/cassandra/audit/AuditLogFilterTest.java
new file mode 100644
index 0000000..8054f90
--- /dev/null
+++ b/test/unit/org/apache/cassandra/audit/AuditLogFilterTest.java
@@ -0,0 +1,189 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.audit;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import static org.apache.cassandra.audit.AuditLogFilter.isFiltered;
+
+public class AuditLogFilterTest
+{
+ @Test
+ public void isFiltered_IncludeSetOnly()
+ {
+ Set<String> includeSet = new HashSet<>();
+ includeSet.add("a");
+ includeSet.add("b");
+ includeSet.add("c");
+
+ Set<String> excludeSet = new HashSet<>();
+
+ Assert.assertFalse(isFiltered("a", includeSet, excludeSet));
+ Assert.assertFalse(isFiltered("b", includeSet, excludeSet));
+ Assert.assertFalse(isFiltered("c", includeSet, excludeSet));
+ Assert.assertTrue(isFiltered("d", includeSet, excludeSet));
+ Assert.assertTrue(isFiltered("e", includeSet, excludeSet));
+ }
+
+ @Test
+ public void isFiltered_ExcludeSetOnly()
+ {
+ Set<String> includeSet = new HashSet<>();
+
+ Set<String> excludeSet = new HashSet<>();
+ excludeSet.add("a");
+ excludeSet.add("b");
+ excludeSet.add("c");
+
+ Assert.assertTrue(isFiltered("a", includeSet, excludeSet));
+ Assert.assertTrue(isFiltered("b", includeSet, excludeSet));
+ Assert.assertTrue(isFiltered("c", includeSet, excludeSet));
+ Assert.assertFalse(isFiltered("d", includeSet, excludeSet));
+ Assert.assertFalse(isFiltered("e", includeSet, excludeSet));
+ }
+
+ @Test
+ public void isFiltered_MutualExclusive()
+ {
+ Set<String> includeSet = new HashSet<>();
+ includeSet.add("a");
+ includeSet.add("b");
+ includeSet.add("c");
+
+ Set<String> excludeSet = new HashSet<>();
+ excludeSet.add("a");
+
+ Assert.assertTrue(isFiltered("a", includeSet, excludeSet));
+ Assert.assertFalse(isFiltered("b", includeSet, excludeSet));
+ Assert.assertFalse(isFiltered("c", includeSet, excludeSet));
+ Assert.assertTrue(isFiltered("e", includeSet, excludeSet));
+ }
+
+ @Test
+ public void isFiltered_MutualInclusive()
+ {
+ Set<String> includeSet = new HashSet<>();
+ includeSet.add("a");
+ includeSet.add("b");
+
+ Set<String> excludeSet = new HashSet<>();
+ excludeSet.add("c");
+ excludeSet.add("d");
+
+ Assert.assertFalse(isFiltered("a", includeSet, excludeSet));
+ Assert.assertFalse(isFiltered("b", includeSet, excludeSet));
+ Assert.assertTrue(isFiltered("c", includeSet, excludeSet));
+ Assert.assertTrue(isFiltered("d", includeSet, excludeSet));
+ Assert.assertTrue(isFiltered("e", includeSet, excludeSet));
+ Assert.assertTrue(isFiltered("f", includeSet, excludeSet));
+ }
+
+ @Test
+ public void isFiltered_UnSpecifiedInput()
+ {
+ Set<String> includeSet = new HashSet<>();
+ includeSet.add("a");
+ includeSet.add("b");
+ includeSet.add("c");
+
+ Set<String> excludeSet = new HashSet<>();
+ excludeSet.add("a");
+
+ Assert.assertTrue(isFiltered("a", includeSet, excludeSet));
+ Assert.assertFalse(isFiltered("b", includeSet, excludeSet));
+ Assert.assertFalse(isFiltered("c", includeSet, excludeSet));
+ Assert.assertTrue(isFiltered("d", includeSet, excludeSet));
+ Assert.assertTrue(isFiltered("e", includeSet, excludeSet));
+ }
+
+ @Test
+ public void isFiltered_SpecifiedInput()
+ {
+ Set<String> includeSet = new HashSet<>();
+ includeSet.add("a");
+ includeSet.add("b");
+ includeSet.add("c");
+
+ Set<String> excludeSet = new HashSet<>();
+ excludeSet.add("a");
+
+ Assert.assertTrue(isFiltered("a", includeSet, excludeSet));
+ Assert.assertFalse(isFiltered("b", includeSet, excludeSet));
+ Assert.assertFalse(isFiltered("c", includeSet, excludeSet));
+ }
+
+ @Test
+ public void isFiltered_FilteredInput_EmptyInclude()
+ {
+ Set<String> includeSet = new HashSet<>();
+ Set<String> excludeSet = new HashSet<>();
+ excludeSet.add("a");
+
+ Assert.assertTrue(isFiltered("a", includeSet, excludeSet));
+ Assert.assertFalse(isFiltered("b", includeSet, excludeSet));
+ }
+
+ @Test
+ public void isFiltered_FilteredInput_EmptyExclude()
+ {
+ Set<String> includeSet = new HashSet<>();
+ includeSet.add("a");
+ includeSet.add("b");
+ includeSet.add("c");
+
+ Set<String> excludeSet = new HashSet<>();
+
+ Assert.assertFalse(isFiltered("a", includeSet, excludeSet));
+ Assert.assertFalse(isFiltered("b", includeSet, excludeSet));
+ Assert.assertFalse(isFiltered("c", includeSet, excludeSet));
+ Assert.assertTrue(isFiltered("e", includeSet, excludeSet));
+ }
+
+ @Test
+ public void isFiltered_EmptyInputs()
+ {
+ Set<String> includeSet = new HashSet<>();
+ Set<String> excludeSet = new HashSet<>();
+
+ Assert.assertFalse(isFiltered("a", includeSet, excludeSet));
+ Assert.assertFalse(isFiltered("e", includeSet, excludeSet));
+ }
+
+ @Test
+ public void isFiltered_NullInputs()
+ {
+ Set<String> includeSet = new HashSet<>();
+ Set<String> excludeSet = new HashSet<>();
+ Assert.assertFalse(isFiltered(null, includeSet, excludeSet));
+
+ includeSet.add("a");
+ includeSet.add("b");
+ includeSet.add("c");
+ Assert.assertTrue(isFiltered(null, includeSet, excludeSet));
+
+ includeSet = new HashSet<>();
+ excludeSet.add("a");
+ excludeSet.add("b");
+ Assert.assertFalse(isFiltered(null, includeSet, excludeSet));
+ }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/audit/AuditLoggerAuthTest.java b/test/unit/org/apache/cassandra/audit/AuditLoggerAuthTest.java
new file mode 100644
index 0000000..bbd8561
--- /dev/null
+++ b/test/unit/org/apache/cassandra/audit/AuditLoggerAuthTest.java
@@ -0,0 +1,292 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.audit;
+
+import java.net.InetAddress;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Queue;
+
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+import com.datastax.driver.core.Cluster;
+import com.datastax.driver.core.Session;
+import com.datastax.driver.core.exceptions.AuthenticationException;
+import com.datastax.driver.core.exceptions.UnauthorizedException;
+import org.apache.cassandra.OrderedJUnit4ClassRunner;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.config.OverrideConfigurationLoader;
+import org.apache.cassandra.config.ParameterizedClass;
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.service.EmbeddedCassandraService;
+
+import static org.hamcrest.CoreMatchers.containsString;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * AuditLoggerAuthTest class is responsible for covering test cases for Authenticated user (LOGIN) audits.
+ * Non authenticated tests are covered in {@link AuditLoggerTest}
+ */
+
+@RunWith(OrderedJUnit4ClassRunner.class)
+public class AuditLoggerAuthTest
+{
+ private static EmbeddedCassandraService embedded;
+
+ private static final String TEST_USER = "testuser";
+ private static final String TEST_ROLE = "testrole";
+ private static final String TEST_PW = "testpassword";
+ private static final String CASS_USER = "cassandra";
+ private static final String CASS_PW = "cassandra";
+
+ @BeforeClass
+ public static void setup() throws Exception
+ {
+ OverrideConfigurationLoader.override((config) -> {
+ config.authenticator = "PasswordAuthenticator";
+ config.role_manager = "CassandraRoleManager";
+ config.authorizer = "CassandraAuthorizer";
+ config.audit_logging_options.enabled = true;
+ config.audit_logging_options.logger = new ParameterizedClass("InMemoryAuditLogger", null);
+ });
+ CQLTester.prepareServer();
+
+ System.setProperty("cassandra.superuser_setup_delay_ms", "0");
+ embedded = new EmbeddedCassandraService();
+ embedded.start();
+
+ executeWithCredentials(
+ Arrays.asList(getCreateRoleCql(TEST_USER, true, false),
+ getCreateRoleCql("testuser_nologin", false, false),
+ "CREATE KEYSPACE testks WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'}",
+ "CREATE TABLE testks.table1 (key text PRIMARY KEY, col1 int, col2 int)"),
+ "cassandra", "cassandra", null);
+ }
+
+ @AfterClass
+ public static void shutdown()
+ {
+ embedded.stop();
+ }
+
+ @Before
+ public void clearInMemoryLogger()
+ {
+ getInMemAuditLogger().clear();
+ }
+
+ @Test
+ public void testCqlLoginAuditing() throws Throwable
+ {
+ executeWithCredentials(Collections.emptyList(), TEST_USER, "wrongpassword",
+ AuditLogEntryType.LOGIN_ERROR);
+ assertEquals(0, getInMemAuditLogger().size());
+ clearInMemoryLogger();
+
+ executeWithCredentials(Collections.emptyList(), "wronguser", TEST_PW, AuditLogEntryType.LOGIN_ERROR);
+ assertEquals(0, getInMemAuditLogger().size());
+ clearInMemoryLogger();
+
+ executeWithCredentials(Collections.emptyList(), "testuser_nologin",
+ TEST_PW, AuditLogEntryType.LOGIN_ERROR);
+ assertEquals(0, getInMemAuditLogger().size());
+ clearInMemoryLogger();
+
+ executeWithCredentials(Collections.emptyList(), TEST_USER, TEST_PW, AuditLogEntryType.LOGIN_SUCCESS);
+ assertEquals(0, getInMemAuditLogger().size());
+ clearInMemoryLogger();
+ }
+
+ @Test
+ public void testCqlCreateRoleAuditing()
+ {
+ createTestRole();
+ }
+
+ @Test
+ public void testCqlALTERRoleAuditing()
+ {
+ createTestRole();
+ String cql = "ALTER ROLE " + TEST_ROLE + " WITH PASSWORD = 'foo_bar'";
+ executeWithCredentials(Arrays.asList(cql), CASS_USER, CASS_PW, AuditLogEntryType.LOGIN_SUCCESS);
+ assertTrue(getInMemAuditLogger().size() > 0);
+ AuditLogEntry logEntry = getInMemAuditLogger().poll();
+ assertLogEntry(logEntry, AuditLogEntryType.ALTER_ROLE, cql, CASS_USER);
+ assertEquals(0, getInMemAuditLogger().size());
+ }
+
+ @Test
+ public void testCqlDROPRoleAuditing()
+ {
+ createTestRole();
+ String cql = "DROP ROLE " + TEST_ROLE;
+ executeWithCredentials(Arrays.asList(cql), CASS_USER, CASS_PW, AuditLogEntryType.LOGIN_SUCCESS);
+ assertTrue(getInMemAuditLogger().size() > 0);
+ AuditLogEntry logEntry = getInMemAuditLogger().poll();
+ assertLogEntry(logEntry, AuditLogEntryType.DROP_ROLE, cql, CASS_USER);
+ assertEquals(0, getInMemAuditLogger().size());
+ }
+
+ @Test
+ public void testCqlLISTROLESAuditing()
+ {
+ String cql = "LIST ROLES";
+ executeWithCredentials(Arrays.asList(cql), CASS_USER, CASS_PW, AuditLogEntryType.LOGIN_SUCCESS);
+ assertTrue(getInMemAuditLogger().size() > 0);
+ AuditLogEntry logEntry = getInMemAuditLogger().poll();
+ assertLogEntry(logEntry, AuditLogEntryType.LIST_ROLES, cql, CASS_USER);
+ }
+
+ @Test
+ public void testCqlLISTPERMISSIONSAuditing()
+ {
+ String cql = "LIST ALL";
+ executeWithCredentials(Arrays.asList(cql), CASS_USER, CASS_PW, AuditLogEntryType.LOGIN_SUCCESS);
+ assertTrue(getInMemAuditLogger().size() > 0);
+ AuditLogEntry logEntry = getInMemAuditLogger().poll();
+ assertLogEntry(logEntry, AuditLogEntryType.LIST_PERMISSIONS, cql, CASS_USER);
+ }
+
+ @Test
+ public void testCqlGRANTAuditing()
+ {
+ createTestRole();
+ String cql = "GRANT SELECT ON ALL KEYSPACES TO " + TEST_ROLE;
+ executeWithCredentials(Arrays.asList(cql), CASS_USER, CASS_PW, AuditLogEntryType.LOGIN_SUCCESS);
+ assertTrue(getInMemAuditLogger().size() > 0);
+ AuditLogEntry logEntry = getInMemAuditLogger().poll();
+ assertLogEntry(logEntry, AuditLogEntryType.GRANT, cql, CASS_USER);
+ }
+
+ @Test
+ public void testCqlREVOKEAuditing()
+ {
+ createTestRole();
+ String cql = "REVOKE ALTER ON ALL KEYSPACES FROM " + TEST_ROLE;
+ executeWithCredentials(Arrays.asList(cql), CASS_USER, CASS_PW, AuditLogEntryType.LOGIN_SUCCESS);
+ assertTrue(getInMemAuditLogger().size() > 0);
+ AuditLogEntry logEntry = getInMemAuditLogger().poll();
+ assertLogEntry(logEntry, AuditLogEntryType.REVOKE, cql, CASS_USER);
+ }
+
+ @Test
+ public void testUNAUTHORIZED_ATTEMPTAuditing()
+ {
+ createTestRole();
+ String cql = "ALTER ROLE " + TEST_ROLE + " WITH superuser = true";
+ executeWithCredentials(Arrays.asList(cql), TEST_USER, TEST_PW, AuditLogEntryType.LOGIN_SUCCESS);
+ assertTrue(getInMemAuditLogger().size() > 0);
+ AuditLogEntry logEntry = getInMemAuditLogger().poll();
+ assertLogEntry(logEntry, AuditLogEntryType.UNAUTHORIZED_ATTEMPT, cql, TEST_USER);
+ assertEquals(0, getInMemAuditLogger().size());
+ }
+
+ /**
+ * Helper methods
+ */
+
+ private static void executeWithCredentials(List<String> queries, String username, String password,
+ AuditLogEntryType expectedType)
+ {
+ boolean authFailed = false;
+ try (Cluster cluster = Cluster.builder().addContactPoints(InetAddress.getLoopbackAddress())
+ .withoutJMXReporting()
+ .withCredentials(username, password)
+ .withPort(DatabaseDescriptor.getNativeTransportPort()).build())
+ {
+ try (Session session = cluster.connect())
+ {
+ for (String query : queries)
+ session.execute(query);
+ }
+ catch (AuthenticationException e)
+ {
+ authFailed = true;
+ }
+ catch (UnauthorizedException ue)
+ {
+ //no-op, taken care by caller
+ }
+ }
+
+ if (expectedType != null)
+ {
+ assertTrue(getInMemAuditLogger().size() > 0);
+ AuditLogEntry logEntry = getInMemAuditLogger().poll();
+
+ assertEquals(expectedType, logEntry.getType());
+ assertTrue(!authFailed || logEntry.getType() == AuditLogEntryType.LOGIN_ERROR);
+ assertSource(logEntry, username);
+
+ // drain all remaining login related events, as there's no specification how connections and login attempts
+ // should be handled by the driver, so we can't assert a fixed number of login events
+ getInMemAuditLogger()
+ .removeIf(auditLogEntry -> auditLogEntry.getType() == AuditLogEntryType.LOGIN_ERROR
+ || auditLogEntry.getType() == AuditLogEntryType.LOGIN_SUCCESS);
+ }
+ }
+
+ private static Queue<AuditLogEntry> getInMemAuditLogger()
+ {
+ return ((InMemoryAuditLogger) AuditLogManager.instance.getLogger()).inMemQueue;
+ }
+
+ private static void assertLogEntry(AuditLogEntry logEntry, AuditLogEntryType type, String cql, String username)
+ {
+ assertSource(logEntry, username);
+ assertNotEquals(0, logEntry.getTimestamp());
+ assertEquals(type, logEntry.getType());
+ if (null != cql && !cql.isEmpty())
+ {
+ assertThat(logEntry.getOperation(), containsString(cql));
+ }
+ }
+
+ private static void assertSource(AuditLogEntry logEntry, String username)
+ {
+ assertEquals(InetAddressAndPort.getLoopbackAddress().address, logEntry.getSource().address);
+ assertTrue(logEntry.getSource().port > 0);
+ if (logEntry.getType() != AuditLogEntryType.LOGIN_ERROR)
+ assertEquals(username, logEntry.getUser());
+ }
+
+ private static String getCreateRoleCql(String role, boolean login, boolean superUser)
+ {
+ return String.format("CREATE ROLE IF NOT EXISTS %s WITH LOGIN = %s AND SUPERUSER = %s AND PASSWORD = '%s'",
+ role, login, superUser, TEST_PW);
+ }
+
+ private static void createTestRole()
+ {
+ String createTestRoleCQL = getCreateRoleCql(TEST_ROLE, true, false);
+ executeWithCredentials(Arrays.asList(createTestRoleCQL), CASS_USER, CASS_PW, AuditLogEntryType.LOGIN_SUCCESS);
+ assertTrue(getInMemAuditLogger().size() > 0);
+ AuditLogEntry logEntry = getInMemAuditLogger().poll();
+ assertLogEntry(logEntry, AuditLogEntryType.CREATE_ROLE, createTestRoleCQL, CASS_USER);
+ assertEquals(0, getInMemAuditLogger().size());
+ }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/audit/AuditLoggerTest.java b/test/unit/org/apache/cassandra/audit/AuditLoggerTest.java
new file mode 100644
index 0000000..ac0170f
--- /dev/null
+++ b/test/unit/org/apache/cassandra/audit/AuditLoggerTest.java
@@ -0,0 +1,819 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.audit;
+
+import org.junit.After;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Map;
+
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import com.datastax.driver.core.BatchStatement;
+import com.datastax.driver.core.PreparedStatement;
+import com.datastax.driver.core.ResultSet;
+import com.datastax.driver.core.Session;
+import com.datastax.driver.core.exceptions.NoHostAvailableException;
+import com.datastax.driver.core.exceptions.SyntaxError;
+import net.openhft.chronicle.queue.RollCycles;
+import org.apache.cassandra.auth.AuthEvents;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.config.ParameterizedClass;
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.cql3.QueryEvents;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.service.StorageService;
+
+import static org.hamcrest.CoreMatchers.containsString;
+import static org.hamcrest.CoreMatchers.instanceOf;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+/**
+ * AuditLoggerTest is responsible for covering the test cases for Audit Logging CASSANDRA-12151 functionality.
+ * Authenticated user audit (LOGIN) tests are segregated from unauthenticated user audit tests.
+ */
+public class AuditLoggerTest extends CQLTester
+{
+ @BeforeClass
+ public static void setUp()
+ {
+ AuditLogOptions options = new AuditLogOptions();
+ options.enabled = true;
+ options.logger = new ParameterizedClass("InMemoryAuditLogger", null);
+ DatabaseDescriptor.setAuditLoggingOptions(options);
+ requireNetwork();
+ }
+
+ @Before
+ public void beforeTestMethod()
+ {
+ AuditLogOptions options = new AuditLogOptions();
+ enableAuditLogOptions(options);
+ }
+
+ @After
+ public void afterTestMethod()
+ {
+ disableAuditLogOptions();
+ }
+
+ private void enableAuditLogOptions(AuditLogOptions options)
+ {
+ String loggerName = "InMemoryAuditLogger";
+ String includedKeyspaces = options.included_keyspaces;
+ String excludedKeyspaces = options.excluded_keyspaces;
+ String includedCategories = options.included_categories;
+ String excludedCategories = options.excluded_categories;
+ String includedUsers = options.included_users;
+ String excludedUsers = options.excluded_users;
+
+ StorageService.instance.enableAuditLog(loggerName, null, includedKeyspaces, excludedKeyspaces, includedCategories, excludedCategories, includedUsers, excludedUsers);
+ }
+
+ private void disableAuditLogOptions()
+ {
+ StorageService.instance.disableAuditLog();
+ }
+
+ @Test
+ public void testAuditLogFilters() throws Throwable
+ {
+ createTable("CREATE TABLE %s (id int primary key, v1 text, v2 text)");
+ execute("INSERT INTO %s (id, v1, v2) VALUES (?, ?, ?)", 1, "Apache", "Cassandra");
+ execute("INSERT INTO %s (id, v1, v2) VALUES (?, ?, ?)", 2, "trace", "test");
+
+ AuditLogOptions options = new AuditLogOptions();
+ options.excluded_keyspaces += ',' + KEYSPACE;
+ enableAuditLogOptions(options);
+
+ String cql = "SELECT id, v1, v2 FROM " + KEYSPACE + '.' + currentTable() + " WHERE id = ?";
+ ResultSet rs = executeAndAssertNoAuditLog(cql, 1);
+ assertEquals(1, rs.all().size());
+
+ options = new AuditLogOptions();
+ options.included_keyspaces = KEYSPACE;
+ enableAuditLogOptions(options);
+
+ cql = "SELECT id, v1, v2 FROM " + KEYSPACE + '.' + currentTable() + " WHERE id = ?";
+ rs = executeAndAssertWithPrepare(cql, AuditLogEntryType.SELECT, 1);
+ assertEquals(1, rs.all().size());
+
+ options = new AuditLogOptions();
+ options.included_keyspaces = KEYSPACE;
+ options.excluded_keyspaces += ',' + KEYSPACE;
+ enableAuditLogOptions(options);
+
+ cql = "SELECT id, v1, v2 FROM " + KEYSPACE + '.' + currentTable() + " WHERE id = ?";
+ rs = executeAndAssertNoAuditLog(cql, 1);
+ assertEquals(1, rs.all().size());
+
+ options = new AuditLogOptions();
+ enableAuditLogOptions(options);
+
+ cql = "SELECT id, v1, v2 FROM " + KEYSPACE + '.' + currentTable() + " WHERE id = ?";
+ rs = executeAndAssertWithPrepare(cql, AuditLogEntryType.SELECT, 1);
+ assertEquals(1, rs.all().size());
+ }
+
+ @Test
+ public void testAuditLogFiltersTransitions() throws Throwable
+ {
+ createTable("CREATE TABLE %s (id int primary key, v1 text, v2 text)");
+ execute("INSERT INTO %s (id, v1, v2) VALUES (?, ?, ?)", 1, "Apache", "Cassandra");
+ execute("INSERT INTO %s (id, v1, v2) VALUES (?, ?, ?)", 2, "trace", "test");
+
+ AuditLogOptions options = new AuditLogOptions();
+ options.excluded_keyspaces += ',' + KEYSPACE;
+ enableAuditLogOptions(options);
+
+ String cql = "SELECT id, v1, v2 FROM " + KEYSPACE + '.' + currentTable() + " WHERE id = ?";
+ ResultSet rs = executeAndAssertNoAuditLog(cql, 1);
+ assertEquals(1, rs.all().size());
+ assertEquals(1, QueryEvents.instance.listenerCount());
+ assertEquals(1, AuthEvents.instance.listenerCount());
+ disableAuditLogOptions();
+ assertEquals(0, QueryEvents.instance.listenerCount());
+ assertEquals(0, AuthEvents.instance.listenerCount());
+
+ cql = "SELECT id, v1, v2 FROM " + KEYSPACE + '.' + currentTable() + " WHERE id = ?";
+ rs = executeAndAssertDisableAuditLog(cql, 1);
+ assertEquals(1, rs.all().size());
+
+ options = new AuditLogOptions();
+ options.included_keyspaces = KEYSPACE;
+ options.excluded_keyspaces += ',' + KEYSPACE;
+ enableAuditLogOptions(options);
+
+ cql = "SELECT id, v1, v2 FROM " + KEYSPACE + '.' + currentTable() + " WHERE id = ?";
+ rs = executeAndAssertNoAuditLog(cql, 1);
+ assertEquals(1, rs.all().size());
+
+ disableAuditLogOptions();
+
+ cql = "SELECT id, v1, v2 FROM " + KEYSPACE + '.' + currentTable() + " WHERE id = ?";
+ rs = executeAndAssertDisableAuditLog(cql, 1);
+ assertEquals(1, rs.all().size());
+ }
+
+ @Test
+ public void testAuditLogExceptions()
+ {
+ AuditLogOptions options = new AuditLogOptions();
+ options.excluded_keyspaces += ',' + KEYSPACE;
+ enableAuditLogOptions(options);
+ Assert.assertTrue(AuditLogManager.instance.isEnabled());
+ }
+
+ @Test
+ public void testAuditLogFilterIncludeExclude() throws Throwable
+ {
+ createTable("CREATE TABLE %s (id int primary key, v1 text, v2 text)");
+ String tbl1 = currentTable();
+ execute("INSERT INTO %s (id, v1, v2) VALUES (?, ?, ?)", 1, "Apache", "Cassandra");
+ execute("INSERT INTO %s (id, v1, v2) VALUES (?, ?, ?)", 2, "trace", "test");
+
+ AuditLogOptions options = new AuditLogOptions();
+ options.excluded_categories = "QUERY";
+ options.included_categories = "QUERY,DML,PREPARE";
+ enableAuditLogOptions(options);
+
+ //QUERY - Should be filtered, part of excluded categories,
+ String cql = "SELECT id, v1, v2 FROM " + KEYSPACE + '.' + currentTable() + " WHERE id = 1";
+ Session session = sessionNet();
+ ResultSet rs = session.execute(cql);
+
+ assertEquals(0, ((InMemoryAuditLogger) AuditLogManager.instance.getLogger()).inMemQueue.size());
+ assertEquals(1, rs.all().size());
+
+ //DML - Should not be filtered, part of included categories
+ cql = "INSERT INTO " + KEYSPACE + '.' + currentTable() + " (id, v1, v2) VALUES (?, ?, ?)";
+ executeAndAssertWithPrepare(cql, AuditLogEntryType.UPDATE, 1, "insert_audit", "test");
+
+ //DDL - Should be filtered, not part of included categories
+ cql = "ALTER TABLE " + KEYSPACE + '.' + currentTable() + " ADD v3 text";
+ session = sessionNet();
+ rs = session.execute(cql);
+ assertEquals(0, ((InMemoryAuditLogger) AuditLogManager.instance.getLogger()).inMemQueue.size());
+ }
+
+ @Test
+ public void testCqlSelectAuditing() throws Throwable
+ {
+ createTable("CREATE TABLE %s (id int primary key, v1 text, v2 text)");
+ execute("INSERT INTO %s (id, v1, v2) VALUES (?, ?, ?)", 1, "Apache", "Cassandra");
+ execute("INSERT INTO %s (id, v1, v2) VALUES (?, ?, ?)", 2, "trace", "test");
+
+ String cql = "SELECT id, v1, v2 FROM " + KEYSPACE + '.' + currentTable() + " WHERE id = ?";
+ ResultSet rs = executeAndAssertWithPrepare(cql, AuditLogEntryType.SELECT, 1);
+
+ assertEquals(1, rs.all().size());
+ }
+
+ @Test
+ public void testCqlInsertAuditing() throws Throwable
+ {
+ createTable("CREATE TABLE %s (id int primary key, v1 text, v2 text)");
+
+ String cql = "INSERT INTO " + KEYSPACE + '.' + currentTable() + " (id, v1, v2) VALUES (?, ?, ?)";
+ executeAndAssertWithPrepare(cql, AuditLogEntryType.UPDATE, 1, "insert_audit", "test");
+ }
+
+ @Test
+ public void testCqlUpdateAuditing() throws Throwable
+ {
+ createTable("CREATE TABLE %s (id int primary key, v1 text, v2 text)");
+ execute("INSERT INTO %s (id, v1, v2) VALUES (?, ?, ?)", 1, "Apache", "Cassandra");
+ execute("INSERT INTO %s (id, v1, v2) VALUES (?, ?, ?)", 2, "trace", "test");
+
+ String cql = "UPDATE " + KEYSPACE + '.' + currentTable() + " SET v1 = 'ApacheCassandra' WHERE id = 1";
+ executeAndAssert(cql, AuditLogEntryType.UPDATE);
+
+ cql = "UPDATE " + KEYSPACE + '.' + currentTable() + " SET v1 = ? WHERE id = ?";
+ executeAndAssertWithPrepare(cql, AuditLogEntryType.UPDATE, "AuditingTest", 2);
+ }
+
+ @Test
+ public void testCqlDeleteAuditing() throws Throwable
+ {
+ createTable("CREATE TABLE %s (id int primary key, v1 text, v2 text)");
+ execute("INSERT INTO %s (id, v1, v2) VALUES (?, ?, ?)", 1, "Apache", "Cassandra");
+ execute("INSERT INTO %s (id, v1, v2) VALUES (?, ?, ?)", 2, "trace", "test");
+
+ String cql = "DELETE FROM " + KEYSPACE + '.' + currentTable() + " WHERE id = ?";
+ executeAndAssertWithPrepare(cql, AuditLogEntryType.DELETE, 1);
+ }
+
+ @Test
+ public void testCqlTruncateAuditing() throws Throwable
+ {
+ createTable("CREATE TABLE %s (id int primary key, v1 text, v2 text)");
+ execute("INSERT INTO %s (id, v1, v2) VALUES (?, ?, ?)", 1, "Apache", "Cassandra");
+ execute("INSERT INTO %s (id, v1, v2) VALUES (?, ?, ?)", 2, "trace", "test");
+
+ String cql = "TRUNCATE TABLE " + KEYSPACE + '.' + currentTable();
+ executeAndAssertWithPrepare(cql, AuditLogEntryType.TRUNCATE);
+ }
+
+ @Test
+ public void testCqlBatchAuditing() throws Throwable
+ {
+ createTable("CREATE TABLE %s (id int primary key, v1 text, v2 text)");
+
+ Session session = sessionNet();
+
+ BatchStatement batchStatement = new BatchStatement();
+
+ String cqlInsert = "INSERT INTO " + KEYSPACE + "." + currentTable() + " (id, v1, v2) VALUES (?, ?, ?)";
+ PreparedStatement prep = session.prepare(cqlInsert);
+ AuditLogEntry logEntry = ((InMemoryAuditLogger) AuditLogManager.instance.getLogger()).inMemQueue.poll();
+ assertLogEntry(cqlInsert, AuditLogEntryType.PREPARE_STATEMENT, logEntry, false);
+
+ batchStatement.add(prep.bind(1, "Apapche", "Cassandra"));
+ batchStatement.add(prep.bind(2, "Apapche1", "Cassandra1"));
+
+ String cqlUpdate = "UPDATE " + KEYSPACE + "." + currentTable() + " SET v1 = ? WHERE id = ?";
+ prep = session.prepare(cqlUpdate);
+ logEntry = ((InMemoryAuditLogger) AuditLogManager.instance.getLogger()).inMemQueue.poll();
+ assertLogEntry(cqlUpdate, AuditLogEntryType.PREPARE_STATEMENT, logEntry, false);
+
+ batchStatement.add(prep.bind("Apache Cassandra", 1));
+
+ String cqlDelete = "DELETE FROM " + KEYSPACE + "." + currentTable() + " WHERE id = ?";
+ prep = session.prepare(cqlDelete);
+ logEntry = ((InMemoryAuditLogger) AuditLogManager.instance.getLogger()).inMemQueue.poll();
+ assertLogEntry(cqlDelete, AuditLogEntryType.PREPARE_STATEMENT, logEntry, false);
+
+ batchStatement.add(prep.bind(1));
+
+ ResultSet rs = session.execute(batchStatement);
+
+ assertEquals(5, ((InMemoryAuditLogger) AuditLogManager.instance.getLogger()).inMemQueue.size());
+ logEntry = ((InMemoryAuditLogger) AuditLogManager.instance.getLogger()).inMemQueue.poll();
+
+ assertEquals(AuditLogEntryType.BATCH, logEntry.getType());
+ assertTrue(logEntry.getOperation().contains("BatchId"));
+ assertNotEquals(0, logEntry.getTimestamp());
+
+ logEntry = ((InMemoryAuditLogger) AuditLogManager.instance.getLogger()).inMemQueue.poll();
+ assertLogEntry(cqlInsert, AuditLogEntryType.UPDATE, logEntry, false);
+
+ logEntry = ((InMemoryAuditLogger) AuditLogManager.instance.getLogger()).inMemQueue.poll();
+ assertLogEntry(cqlInsert, AuditLogEntryType.UPDATE, logEntry, false);
+
+ logEntry = ((InMemoryAuditLogger) AuditLogManager.instance.getLogger()).inMemQueue.poll();
+ assertLogEntry(cqlUpdate, AuditLogEntryType.UPDATE, logEntry, false);
+
+ logEntry = ((InMemoryAuditLogger) AuditLogManager.instance.getLogger()).inMemQueue.poll();
+ assertLogEntry(cqlDelete, AuditLogEntryType.DELETE, logEntry, false);
+
+ int size = rs.all().size();
+
+ assertEquals(0, size);
+ }
+
+ @Test
+ public void testCqlBatch_MultipleTablesAuditing()
+ {
+ createTable("CREATE TABLE %s (id int primary key, v1 text, v2 text)");
+ String table1 = currentTable();
+
+ Session session = sessionNet();
+
+ BatchStatement batchStatement = new BatchStatement();
+
+ String cqlInsert1 = "INSERT INTO " + KEYSPACE + "." + table1 + " (id, v1, v2) VALUES (?, ?, ?)";
+ PreparedStatement prep = session.prepare(cqlInsert1);
+ AuditLogEntry logEntry = ((InMemoryAuditLogger) AuditLogManager.instance.getLogger()).inMemQueue.poll();
+ assertLogEntry(cqlInsert1, AuditLogEntryType.PREPARE_STATEMENT, logEntry, false);
+
+ batchStatement.add(prep.bind(1, "Apapche", "Cassandra"));
+
+ createTable("CREATE TABLE %s (id int primary key, v1 text, v2 text)");
+ String table2 = currentTable();
+
+ String cqlInsert2 = "INSERT INTO " + KEYSPACE + "." + table2 + " (id, v1, v2) VALUES (?, ?, ?)";
+ prep = session.prepare(cqlInsert2);
+ logEntry = ((InMemoryAuditLogger) AuditLogManager.instance.getLogger()).inMemQueue.poll();
+ assertLogEntry(cqlInsert2, AuditLogEntryType.PREPARE_STATEMENT, logEntry, false);
+
+ batchStatement.add(prep.bind(1, "Apapche", "Cassandra"));
+
+ createKeyspace("CREATE KEYSPACE %s WITH replication={ 'class' : 'SimpleStrategy', 'replication_factor' : 1 }");
+ String ks2 = currentKeyspace();
+
+ createTable(ks2, "CREATE TABLE %s (id int primary key, v1 text, v2 text)");
+ String table3 = currentTable();
+
+ String cqlInsert3 = "INSERT INTO " + ks2 + "." + table3 + " (id, v1, v2) VALUES (?, ?, ?)";
+ prep = session.prepare(cqlInsert3);
+ logEntry = ((InMemoryAuditLogger) AuditLogManager.instance.getLogger()).inMemQueue.poll();
+ assertLogEntry(cqlInsert3, AuditLogEntryType.PREPARE_STATEMENT, logEntry, false, ks2);
+
+ batchStatement.add(prep.bind(1, "Apapche", "Cassandra"));
+
+ ResultSet rs = session.execute(batchStatement);
+
+ assertEquals(4, ((InMemoryAuditLogger) AuditLogManager.instance.getLogger()).inMemQueue.size());
+ logEntry = ((InMemoryAuditLogger) AuditLogManager.instance.getLogger()).inMemQueue.poll();
+
+ logEntry = ((InMemoryAuditLogger) AuditLogManager.instance.getLogger()).inMemQueue.poll();
+ assertLogEntry(cqlInsert1, table1, AuditLogEntryType.UPDATE, logEntry, false, KEYSPACE);
+
+ logEntry = ((InMemoryAuditLogger) AuditLogManager.instance.getLogger()).inMemQueue.poll();
+ assertLogEntry(cqlInsert2, table2, AuditLogEntryType.UPDATE, logEntry, false, KEYSPACE);
+
+ logEntry = ((InMemoryAuditLogger) AuditLogManager.instance.getLogger()).inMemQueue.poll();
+ assertLogEntry(cqlInsert3, table3, AuditLogEntryType.UPDATE, logEntry, false, ks2);
+
+ int size = rs.all().size();
+
+ assertEquals(0, size);
+ }
+
+ @Test
+ public void testCqlKeyspaceAuditing() throws Throwable
+ {
+ createTable("CREATE TABLE %s (id int primary key, v1 text, v2 text)");
+
+ String cql = "CREATE KEYSPACE " + createKeyspaceName() + " WITH REPLICATION = { 'class' : 'SimpleStrategy', 'replication_factor' : 2} ";
+ executeAndAssert(cql, AuditLogEntryType.CREATE_KEYSPACE, true, currentKeyspace());
+
+ cql = "CREATE KEYSPACE IF NOT EXISTS " + createKeyspaceName() + " WITH REPLICATION = { 'class' : 'SimpleStrategy', 'replication_factor' : 2} ";
+ executeAndAssert(cql, AuditLogEntryType.CREATE_KEYSPACE, true, currentKeyspace());
+
+ cql = "ALTER KEYSPACE " + currentKeyspace() + " WITH REPLICATION = { 'class' : 'SimpleStrategy', 'replication_factor' : 2} ";
+ executeAndAssert(cql, AuditLogEntryType.ALTER_KEYSPACE, true, currentKeyspace());
+
+ cql = "DROP KEYSPACE " + currentKeyspace();
+ executeAndAssert(cql, AuditLogEntryType.DROP_KEYSPACE, true, currentKeyspace());
+ }
+
+ @Test
+ public void testCqlTableAuditing() throws Throwable
+ {
+ String cql = "CREATE TABLE " + KEYSPACE + "." + createTableName() + " (id int primary key, v1 text, v2 text)";
+ executeAndAssert(cql, AuditLogEntryType.CREATE_TABLE);
+
+ cql = "CREATE TABLE IF NOT EXISTS " + KEYSPACE + "." + createTableName() + " (id int primary key, v1 text, v2 text)";
+ executeAndAssert(cql, AuditLogEntryType.CREATE_TABLE);
+
+ cql = "ALTER TABLE " + KEYSPACE + "." + currentTable() + " ADD v3 text";
+ executeAndAssert(cql, AuditLogEntryType.ALTER_TABLE);
+
+ cql = "DROP TABLE " + KEYSPACE + "." + currentTable();
+ executeAndAssert(cql, AuditLogEntryType.DROP_TABLE);
+ }
+
+ @Test
+ public void testCqlMVAuditing() throws Throwable
+ {
+ createTable("CREATE TABLE %s (id int primary key, v1 text, v2 text)");
+ execute("INSERT INTO %s (id, v1, v2) VALUES (?, ?, ?)", 1, "Apache", "Cassandra");
+ execute("INSERT INTO %s (id, v1, v2) VALUES (?, ?, ?)", 2, "trace", "test");
+
+ String tblName = currentTable();
+ String cql = "CREATE MATERIALIZED VIEW " + KEYSPACE + "." + createTableName() + " AS SELECT id,v1 FROM " + KEYSPACE + "." + tblName + " WHERE id IS NOT NULL AND v1 IS NOT NULL PRIMARY KEY ( id, v1 ) ";
+ executeAndAssert(cql, AuditLogEntryType.CREATE_VIEW);
+
+ cql = "CREATE MATERIALIZED VIEW IF NOT EXISTS " + KEYSPACE + "." + currentTable() + " AS SELECT id,v1 FROM " + KEYSPACE + "." + tblName + " WHERE id IS NOT NULL AND v1 IS NOT NULL PRIMARY KEY ( id, v1 ) ";
+ executeAndAssert(cql, AuditLogEntryType.CREATE_VIEW);
+
+ cql = "ALTER MATERIALIZED VIEW " + KEYSPACE + "." + currentTable() + " WITH caching = { 'keys' : 'NONE' };";
+ executeAndAssert(cql, AuditLogEntryType.ALTER_VIEW);
+
+ cql = "DROP MATERIALIZED VIEW " + KEYSPACE + "." + currentTable();
+ executeAndAssert(cql, AuditLogEntryType.DROP_VIEW);
+ }
+
+ @Test
+ public void testCqlTypeAuditing() throws Throwable
+ {
+ createTable("CREATE TABLE %s (id int primary key, v1 text, v2 text)");
+
+ String tblName = createTableName();
+
+ String cql = "CREATE TYPE " + KEYSPACE + "." + tblName + " (id int, v1 text, v2 text)";
+ executeAndAssert(cql, AuditLogEntryType.CREATE_TYPE);
+
+ cql = "CREATE TYPE IF NOT EXISTS " + KEYSPACE + "." + tblName + " (id int, v1 text, v2 text)";
+ executeAndAssert(cql, AuditLogEntryType.CREATE_TYPE);
+
+ cql = "ALTER TYPE " + KEYSPACE + "." + tblName + " ADD v3 int";
+ executeAndAssert(cql, AuditLogEntryType.ALTER_TYPE);
+
+ cql = "ALTER TYPE " + KEYSPACE + "." + tblName + " RENAME v3 TO v4";
+ executeAndAssert(cql, AuditLogEntryType.ALTER_TYPE);
+
+ cql = "DROP TYPE " + KEYSPACE + "." + tblName;
+ executeAndAssert(cql, AuditLogEntryType.DROP_TYPE);
+
+ cql = "DROP TYPE IF EXISTS " + KEYSPACE + "." + tblName;
+ executeAndAssert(cql, AuditLogEntryType.DROP_TYPE);
+ }
+
+ @Test
+ public void testCqlIndexAuditing() throws Throwable
+ {
+ createTable("CREATE TABLE %s (id int primary key, v1 text, v2 text)");
+
+ String tblName = currentTable();
+
+ String indexName = createTableName();
+
+ String cql = "CREATE INDEX " + indexName + " ON " + KEYSPACE + "." + tblName + " (v1)";
+ executeAndAssert(cql, AuditLogEntryType.CREATE_INDEX);
+
+ cql = "DROP INDEX " + KEYSPACE + "." + indexName;
+ executeAndAssert(cql, AuditLogEntryType.DROP_INDEX);
+ }
+
+ @Test
+ public void testCqlFunctionAuditing() throws Throwable
+ {
+ String tblName = createTableName();
+
+ String cql = "CREATE FUNCTION IF NOT EXISTS " + KEYSPACE + "." + tblName + " (column TEXT,num int) RETURNS NULL ON NULL INPUT RETURNS text LANGUAGE javascript AS $$ column.substring(0,num) $$";
+ executeAndAssert(cql, AuditLogEntryType.CREATE_FUNCTION);
+
+ cql = "DROP FUNCTION " + KEYSPACE + "." + tblName;
+ executeAndAssert(cql, AuditLogEntryType.DROP_FUNCTION);
+ }
+
+ @Test
+ public void testCqlTriggerAuditing() throws Throwable
+ {
+ createTable("CREATE TABLE %s (id int primary key, v1 text, v2 text)");
+
+ String tblName = currentTable();
+ String triggerName = createTableName();
+
+ String cql = "DROP TRIGGER IF EXISTS " + triggerName + " ON " + KEYSPACE + "." + tblName;
+ executeAndAssert(cql, AuditLogEntryType.DROP_TRIGGER);
+ }
+
+ @Test
+ public void testCqlAggregateAuditing() throws Throwable
+ {
+ String aggName = createTableName();
+ String cql = "DROP AGGREGATE IF EXISTS " + KEYSPACE + "." + aggName;
+ executeAndAssert(cql, AuditLogEntryType.DROP_AGGREGATE);
+ }
+
+ @Test
+ public void testCqlQuerySyntaxError()
+ {
+ String cql = "INSERT INTO " + KEYSPACE + '.' + currentTable() + "1 (id, v1, v2) VALUES (1, 'insert_audit, 'test')";
+ try
+ {
+ createTable("CREATE TABLE %s (id int primary key, v1 text, v2 text)");
+ Session session = sessionNet();
+ ResultSet rs = session.execute(cql);
+ Assert.fail("should not succeed");
+ }
+ catch (SyntaxError e)
+ {
+ // nop
+ }
+
+ AuditLogEntry logEntry = ((InMemoryAuditLogger) AuditLogManager.instance.getLogger()).inMemQueue.poll();
+ assertLogEntry(logEntry, cql);
+ assertEquals(0, ((InMemoryAuditLogger) AuditLogManager.instance.getLogger()).inMemQueue.size());
+ }
+
+ @Test
+ public void testCqlSelectQuerySyntaxError()
+ {
+ createTable("CREATE TABLE %s (id int primary key, v1 text, v2 text)");
+ String cql = "SELECT * FROM " + KEYSPACE + '.' + currentTable() + " LIMIT 2w";
+
+ try
+ {
+ Session session = sessionNet();
+ ResultSet rs = session.execute(cql);
+ Assert.fail("should not succeed");
+ }
+ catch (SyntaxError e)
+ {
+ // nop
+ }
+
+ AuditLogEntry logEntry = ((InMemoryAuditLogger) AuditLogManager.instance.getLogger()).inMemQueue.poll();
+ assertLogEntry(logEntry, cql);
+ assertEquals(0, ((InMemoryAuditLogger) AuditLogManager.instance.getLogger()).inMemQueue.size());
+ }
+
+ @Test
+ public void testCqlPrepareQueryError()
+ {
+ createTable("CREATE TABLE %s (id int primary key, v1 text, v2 text)");
+ String cql = "INSERT INTO " + KEYSPACE + '.' + currentTable() + " (id, v1, v2) VALUES (?,?,?)";
+ try
+ {
+ Session session = sessionNet();
+
+ PreparedStatement pstmt = session.prepare(cql);
+ AuditLogEntry logEntry = ((InMemoryAuditLogger) AuditLogManager.instance.getLogger()).inMemQueue.poll();
+ assertLogEntry(cql, AuditLogEntryType.PREPARE_STATEMENT, logEntry, false);
+
+ dropTable("DROP TABLE %s");
+ ResultSet rs = session.execute(pstmt.bind(1, "insert_audit", "test"));
+ Assert.fail("should not succeed");
+ }
+ catch (NoHostAvailableException e)
+ {
+ // nop
+ }
+
+ AuditLogEntry logEntry = ((InMemoryAuditLogger) AuditLogManager.instance.getLogger()).inMemQueue.poll();
+ assertLogEntry(logEntry, null);
+ logEntry = ((InMemoryAuditLogger) AuditLogManager.instance.getLogger()).inMemQueue.poll();
+ assertLogEntry(logEntry, cql);
+ assertEquals(0, ((InMemoryAuditLogger) AuditLogManager.instance.getLogger()).inMemQueue.size());
+ }
+
+ @Test
+ public void testCqlPrepareQuerySyntaxError()
+ {
+ String cql = "INSERT INTO " + KEYSPACE + '.' + "foo" + "(id, v1, v2) VALES (?,?,?)";
+ try
+ {
+ createTable("CREATE TABLE %s (id int primary key, v1 text, v2 text)");
+ Session session = sessionNet();
+ PreparedStatement pstmt = session.prepare(cql);
+ ResultSet rs = session.execute(pstmt.bind(1, "insert_audit", "test"));
+ Assert.fail("should not succeed");
+ }
+ catch (SyntaxError e)
+ {
+ // nop
+ }
+ AuditLogEntry logEntry = ((InMemoryAuditLogger) AuditLogManager.instance.getLogger()).inMemQueue.poll();
+ assertLogEntry(logEntry, cql);
+ assertEquals(0, ((InMemoryAuditLogger) AuditLogManager.instance.getLogger()).inMemQueue.size());
+ }
+
+ @Test
+ public void testIncludeSystemKeyspaces() throws Throwable
+ {
+ AuditLogOptions options = new AuditLogOptions();
+ options.included_categories = "QUERY,DML,PREPARE";
+ options.excluded_keyspaces = "system_schema,system_virtual_schema";
+ enableAuditLogOptions(options);
+
+ Session session = sessionNet();
+ String cql = "SELECT * FROM system.local limit 2";
+ ResultSet rs = session.execute(cql);
+
+ assertEquals (1,((InMemoryAuditLogger) AuditLogManager.instance.getLogger()).inMemQueue.size());
+ AuditLogEntry logEntry = ((InMemoryAuditLogger) AuditLogManager.instance.getLogger()).inMemQueue.poll();
+ assertLogEntry(cql, "local",AuditLogEntryType.SELECT,logEntry,false, "system");
+ assertEquals (0,((InMemoryAuditLogger) AuditLogManager.instance.getLogger()).inMemQueue.size());
+ }
+
+ @Test
+ public void testExcludeSystemKeyspaces() throws Throwable
+ {
+ AuditLogOptions options = new AuditLogOptions();
+ options.included_categories = "QUERY,DML,PREPARE";
+ options.excluded_keyspaces = "system,system_schema,system_virtual_schema";
+ enableAuditLogOptions(options);
+
+ Session session = sessionNet();
+ String cql = "SELECT * FROM system.local limit 2";
+ ResultSet rs = session.execute(cql);
+
+ assertEquals (0,((InMemoryAuditLogger) AuditLogManager.instance.getLogger()).inMemQueue.size());
+ }
+
+ @Test
+ public void testEnableDisable() throws IOException
+ {
+ disableAuditLogOptions();
+ assertEquals(0, QueryEvents.instance.listenerCount());
+ assertEquals(0, AuthEvents.instance.listenerCount());
+ enableAuditLogOptions(new AuditLogOptions());
+ assertEquals(1, QueryEvents.instance.listenerCount());
+ assertEquals(1, AuthEvents.instance.listenerCount());
+
+ Path p = Files.createTempDirectory("fql");
+ StorageService.instance.enableFullQueryLogger(p.toString(), RollCycles.HOURLY.toString(), false, 1000, 1000, null, 0);
+ assertEquals(2, QueryEvents.instance.listenerCount());
+ assertEquals(1, AuthEvents.instance.listenerCount()); // fql not listening to auth events
+ StorageService.instance.resetFullQueryLogger();
+ assertEquals(1, QueryEvents.instance.listenerCount());
+ assertEquals(1, AuthEvents.instance.listenerCount());
+ disableAuditLogOptions();
+
+ assertEquals(0, QueryEvents.instance.listenerCount());
+ assertEquals(0, AuthEvents.instance.listenerCount());
+ }
+
+ @Test
+ public void testConflictingPaths()
+ {
+ disableAuditLogOptions();
+ AuditLogOptions options = new AuditLogOptions();
+ DatabaseDescriptor.setAuditLoggingOptions(options);
+ StorageService.instance.enableAuditLog(null, null, options.included_keyspaces, options.excluded_keyspaces, options.included_categories, options.excluded_categories, options.included_users, options.excluded_users);
+ try
+ {
+ assertEquals(1, QueryEvents.instance.listenerCount());
+ assertEquals(1, AuthEvents.instance.listenerCount());
+ StorageService.instance.enableFullQueryLogger(options.audit_logs_dir, RollCycles.HOURLY.toString(), false, 1000, 1000, null, 0);
+ fail("Conflicting directories - should throw exception");
+ }
+ catch (IllegalStateException e)
+ {
+ // ok
+ }
+ assertEquals(1, QueryEvents.instance.listenerCount());
+ assertEquals(1, AuthEvents.instance.listenerCount());
+ }
+
+
+ @Test
+ public void testConflictingPathsFQLFirst()
+ {
+ disableAuditLogOptions();
+ AuditLogOptions options = new AuditLogOptions();
+ DatabaseDescriptor.setAuditLoggingOptions(options);
+ StorageService.instance.enableFullQueryLogger(options.audit_logs_dir, RollCycles.HOURLY.toString(), false, 1000, 1000, null, 0);
+ try
+ {
+ assertEquals(1, QueryEvents.instance.listenerCount());
+ assertEquals(0, AuthEvents.instance.listenerCount());
+ StorageService.instance.enableAuditLog(null, null, options.included_keyspaces, options.excluded_keyspaces, options.included_categories, options.excluded_categories, options.included_users, options.excluded_users);
+ fail("Conflicting directories - should throw exception");
+ }
+ catch (ConfigurationException e)
+ {
+ // ok
+ }
+ assertEquals(1, QueryEvents.instance.listenerCount());
+ assertEquals(0, AuthEvents.instance.listenerCount());
+ }
+
+ /**
+ * Helper methods for Audit Log CQL Testing
+ */
+
+ private ResultSet executeAndAssert(String cql, AuditLogEntryType type) throws Throwable
+ {
+ return executeAndAssert(cql, type, false, KEYSPACE);
+ }
+
+ private ResultSet executeAndAssert(String cql, AuditLogEntryType type, boolean isTableNull, String keyspace) throws Throwable
+ {
+ Session session = sessionNet();
+
+ ResultSet rs = session.execute(cql);
+
+ AuditLogEntry logEntry1 = ((InMemoryAuditLogger) AuditLogManager.instance.getLogger()).inMemQueue.poll();
+ assertLogEntry(cql, type, logEntry1, isTableNull, keyspace);
+
+ assertEquals(0, ((InMemoryAuditLogger) AuditLogManager.instance.getLogger()).inMemQueue.size());
+ return rs;
+ }
+
+ private ResultSet executeAndAssertWithPrepare(String cql, AuditLogEntryType exceuteType, Object... bindValues) throws Throwable
+ {
+ return executeAndAssertWithPrepare(cql, exceuteType, false, bindValues);
+ }
+
+ private ResultSet executeAndAssertWithPrepare(String cql, AuditLogEntryType executeType, boolean isTableNull, Object... bindValues) throws Throwable
+ {
+ Session session = sessionNet();
+
+ PreparedStatement pstmt = session.prepare(cql);
+ ResultSet rs = session.execute(pstmt.bind(bindValues));
+
+ AuditLogEntry logEntry1 = ((InMemoryAuditLogger) AuditLogManager.instance.getLogger()).inMemQueue.poll();
+ assertLogEntry(cql, AuditLogEntryType.PREPARE_STATEMENT, logEntry1, isTableNull);
+
+ AuditLogEntry logEntry2 = ((InMemoryAuditLogger) AuditLogManager.instance.getLogger()).inMemQueue.poll();
+ assertLogEntry(cql, executeType, logEntry2, isTableNull);
+
+ assertEquals(0, ((InMemoryAuditLogger) AuditLogManager.instance.getLogger()).inMemQueue.size());
+ return rs;
+ }
+
+ private ResultSet executeAndAssertNoAuditLog(String cql, Object... bindValues)
+ {
+ Session session = sessionNet();
+
+ PreparedStatement pstmt = session.prepare(cql);
+ ResultSet rs = session.execute(pstmt.bind(bindValues));
+
+ assertEquals(0, ((InMemoryAuditLogger) AuditLogManager.instance.getLogger()).inMemQueue.size());
+ return rs;
+ }
+
+ private ResultSet executeAndAssertDisableAuditLog(String cql, Object... bindValues)
+ {
+ Session session = sessionNet();
+
+ PreparedStatement pstmt = session.prepare(cql);
+ ResultSet rs = session.execute(pstmt.bind(bindValues));
+
+ assertThat(AuditLogManager.instance.getLogger(),instanceOf(NoOpAuditLogger.class));
+ return rs;
+ }
+
+ private void assertLogEntry(String cql, AuditLogEntryType type, AuditLogEntry actual, boolean isTableNull)
+ {
+ assertLogEntry(cql, type, actual, isTableNull, KEYSPACE);
+ }
+
+ private void assertLogEntry(String cql, AuditLogEntryType type, AuditLogEntry actual, boolean isTableNull, String keyspace)
+ {
+ assertLogEntry(cql, currentTable(), type, actual, isTableNull, keyspace);
+ }
+
+ private void assertLogEntry(String cql, String table, AuditLogEntryType type, AuditLogEntry actual, boolean isTableNull, String keyspace)
+ {
+ assertEquals(keyspace, actual.getKeyspace());
+ if (!isTableNull)
+ {
+ assertEquals(table, actual.getScope());
+ }
+ assertEquals(type, actual.getType());
+ assertEquals(cql, actual.getOperation());
+ assertNotEquals(0,actual.getTimestamp());
+ }
+
+ private void assertLogEntry(AuditLogEntry logEntry, String cql)
+ {
+ assertNull(logEntry.getKeyspace());
+ assertNull(logEntry.getScope());
+ assertNotEquals(0,logEntry.getTimestamp());
+ assertEquals(AuditLogEntryType.REQUEST_FAILURE, logEntry.getType());
+ if (null != cql && !cql.isEmpty())
+ {
+ assertThat(logEntry.getOperation(), containsString(cql));
+ }
+ }
+}
diff --git a/test/unit/org/apache/cassandra/audit/BinAuditLoggerTest.java b/test/unit/org/apache/cassandra/audit/BinAuditLoggerTest.java
new file mode 100644
index 0000000..a450cbf
--- /dev/null
+++ b/test/unit/org/apache/cassandra/audit/BinAuditLoggerTest.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.audit;
+
+import java.nio.file.Path;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import com.datastax.driver.core.PreparedStatement;
+import com.datastax.driver.core.ResultSet;
+import com.datastax.driver.core.Session;
+import net.openhft.chronicle.queue.ChronicleQueue;
+import net.openhft.chronicle.queue.ChronicleQueueBuilder;
+import net.openhft.chronicle.queue.ExcerptTailer;
+import net.openhft.chronicle.queue.RollCycles;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.config.ParameterizedClass;
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.utils.binlog.BinLogTest;
+
+import static org.hamcrest.core.StringContains.containsString;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
+
+public class BinAuditLoggerTest extends CQLTester
+{
+ private static Path tempDir;
+
+ @BeforeClass
+ public static void setUp() throws Exception
+ {
+ tempDir = BinLogTest.tempDir();
+
+ AuditLogOptions options = new AuditLogOptions();
+ options.enabled = true;
+ options.logger = new ParameterizedClass("BinAuditLogger", null);
+ options.roll_cycle = "TEST_SECONDLY";
+ options.audit_logs_dir = tempDir.toString();
+ DatabaseDescriptor.setAuditLoggingOptions(options);
+ AuditLogManager.instance.enable(DatabaseDescriptor.getAuditLoggingOptions());
+ requireNetwork();
+ }
+
+ @Test
+ public void testSelectRoundTripQuery() throws Throwable
+ {
+ createTable("CREATE TABLE %s (id int primary key, v1 text, v2 text)");
+ execute("INSERT INTO %s (id, v1, v2) VALUES (?, ?, ?)", 1, "Apache", "Cassandra");
+ execute("INSERT INTO %s (id, v1, v2) VALUES (?, ?, ?)", 2, "trace", "test");
+
+ String cql = "SELECT id, v1, v2 FROM " + KEYSPACE + '.' + currentTable() + " WHERE id = ?";
+
+ Session session = sessionNet();
+
+ PreparedStatement pstmt = session.prepare(cql);
+ ResultSet rs = session.execute(pstmt.bind(1));
+
+ assertEquals(1, rs.all().size());
+ try (ChronicleQueue queue = ChronicleQueueBuilder.single(tempDir.toFile()).rollCycle(RollCycles.TEST_SECONDLY).build())
+ {
+ ExcerptTailer tailer = queue.createTailer();
+ assertTrue(tailer.readDocument(wire -> {
+ assertEquals(0L, wire.read("version").int16());
+ assertEquals("audit", wire.read("type").text());
+ assertThat(wire.read("message").text(), containsString(AuditLogEntryType.PREPARE_STATEMENT.toString()));
+ }));
+
+ assertTrue(tailer.readDocument(wire -> {
+ assertEquals(0L, wire.read("version").int16());
+ assertEquals("audit", wire.read("type").text());
+ assertThat(wire.read("message").text(), containsString(AuditLogEntryType.SELECT.toString()));
+ }));
+ assertFalse(tailer.readDocument(wire -> {
+ }));
+ }
+ }
+}
diff --git a/test/unit/org/apache/cassandra/audit/InMemoryAuditLogger.java b/test/unit/org/apache/cassandra/audit/InMemoryAuditLogger.java
new file mode 100644
index 0000000..f9a4038
--- /dev/null
+++ b/test/unit/org/apache/cassandra/audit/InMemoryAuditLogger.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.audit;
+
+import java.util.LinkedList;
+import java.util.Map;
+import java.util.Queue;
+
+public class InMemoryAuditLogger implements IAuditLogger
+{
+ final Queue<AuditLogEntry> inMemQueue = new LinkedList<>();
+ private boolean enabled = true;
+
+ public InMemoryAuditLogger(Map<String, String> params)
+ {
+
+ }
+
+ @Override
+ public boolean isEnabled()
+ {
+ return enabled;
+ }
+
+ @Override
+ public void log(AuditLogEntry logMessage)
+ {
+ inMemQueue.offer(logMessage);
+ }
+
+ @Override
+ public void stop()
+ {
+ enabled = false;
+ inMemQueue.clear();
+ }
+}
diff --git a/test/unit/org/apache/cassandra/auth/AuthCacheTest.java b/test/unit/org/apache/cassandra/auth/AuthCacheTest.java
index 0030603..217821e 100644
--- a/test/unit/org/apache/cassandra/auth/AuthCacheTest.java
+++ b/test/unit/org/apache/cassandra/auth/AuthCacheTest.java
@@ -17,9 +17,10 @@
*/
package org.apache.cassandra.auth;
-import java.util.function.Consumer;
+import java.util.function.BooleanSupplier;
import java.util.function.Function;
-import java.util.function.Supplier;
+import java.util.function.IntConsumer;
+import java.util.function.IntSupplier;
import org.junit.Test;
@@ -27,6 +28,9 @@
import org.apache.cassandra.exceptions.UnavailableException;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
public class AuthCacheTest
{
@@ -110,12 +114,79 @@
assertEquals(2, loadCounter);
}
+ @Test
+ public void testCacheLoaderIsCalledAfterReset()
+ {
+ TestCache<String, Integer> authCache = new TestCache<>(this::countingLoader, this::setValidity, () -> validity, () -> isCacheEnabled);
+ authCache.get("10");
+
+ authCache.cache = null;
+ int result = authCache.get("10");
+
+ assertEquals(10, result);
+ assertEquals(2, loadCounter);
+ }
+
+ @Test
+ public void testThatZeroValidityTurnOffCaching()
+ {
+ setValidity(0);
+ TestCache<String, Integer> authCache = new TestCache<>(this::countingLoader, this::setValidity, () -> validity, () -> isCacheEnabled);
+ authCache.get("10");
+ int result = authCache.get("10");
+
+ assertNull(authCache.cache);
+ assertEquals(10, result);
+ assertEquals(2, loadCounter);
+ }
+
+ @Test
+ public void testThatRaisingValidityTurnOnCaching()
+ {
+ setValidity(0);
+ TestCache<String, Integer> authCache = new TestCache<>(this::countingLoader, this::setValidity, () -> validity, () -> isCacheEnabled);
+
+ authCache.setValidity(2000);
+ authCache.cache = authCache.initCache(null);
+
+ assertNotNull(authCache.cache);
+ }
+
+ @Test
+ public void testDisableCache()
+ {
+ isCacheEnabled = false;
+ TestCache<String, Integer> authCache = new TestCache<>(this::countingLoader, this::setValidity, () -> validity, () -> isCacheEnabled);
+
+ assertNull(authCache.cache);
+ }
+
+ @Test
+ public void testDynamicallyEnableCache()
+ {
+ isCacheEnabled = false;
+ TestCache<String, Integer> authCache = new TestCache<>(this::countingLoader, this::setValidity, () -> validity, () -> isCacheEnabled);
+
+ isCacheEnabled = true;
+ authCache.cache = authCache.initCache(null);
+
+ assertNotNull(authCache.cache);
+ }
+
+ @Test
+ public void testDefaultPolicies()
+ {
+ TestCache<String, Integer> authCache = new TestCache<>(this::countingLoader, this::setValidity, () -> validity, () -> isCacheEnabled);
+
+ assertTrue(authCache.cache.policy().expireAfterWrite().isPresent());
+ assertTrue(authCache.cache.policy().refreshAfterWrite().isPresent());
+ assertTrue(authCache.cache.policy().eviction().isPresent());
+ }
+
@Test(expected = UnavailableException.class)
public void testCassandraExceptionPassThroughWhenCacheEnabled()
{
- TestCache<String, Integer> cache = new TestCache<>(s -> {
- throw new UnavailableException(ConsistencyLevel.QUORUM, 3, 1);
- }, this::setValidity, () -> validity, () -> isCacheEnabled);
+ TestCache<String, Integer> cache = new TestCache<>(s -> { throw UnavailableException.create(ConsistencyLevel.QUORUM, 3, 1); }, this::setValidity, () -> validity, () -> isCacheEnabled);
cache.get("expect-exception");
}
@@ -124,9 +195,7 @@
public void testCassandraExceptionPassThroughWhenCacheDisable()
{
isCacheEnabled = false;
- TestCache<String, Integer> cache = new TestCache<>(s -> {
- throw new UnavailableException(ConsistencyLevel.QUORUM, 3, 1);
- }, this::setValidity, () -> validity, () -> isCacheEnabled);
+ TestCache<String, Integer> cache = new TestCache<>(s -> { throw UnavailableException.create(ConsistencyLevel.QUORUM, 3, 1); }, this::setValidity, () -> validity, () -> isCacheEnabled);
cache.get("expect-exception");
}
@@ -146,16 +215,14 @@
{
private static int nameCounter = 0; // Allow us to create many instances of cache with same name prefix
- TestCache(Function<K, V> loadFunction, Consumer<Integer> setValidityDelegate, Supplier<Integer> getValidityDelegate, Supplier<Boolean> cacheEnabledDelegate)
+ TestCache(Function<K, V> loadFunction, IntConsumer setValidityDelegate, IntSupplier getValidityDelegate, BooleanSupplier cacheEnabledDelegate)
{
super("TestCache" + nameCounter++,
setValidityDelegate,
getValidityDelegate,
- (updateInterval) -> {
- },
+ (updateInterval) -> {},
() -> 1000,
- (maxEntries) -> {
- },
+ (maxEntries) -> {},
() -> 10,
loadFunction,
cacheEnabledDelegate);
diff --git a/test/unit/org/apache/cassandra/auth/CassandraNetworkAuthorizerTest.java b/test/unit/org/apache/cassandra/auth/CassandraNetworkAuthorizerTest.java
new file mode 100644
index 0000000..2e57173
--- /dev/null
+++ b/test/unit/org/apache/cassandra/auth/CassandraNetworkAuthorizerTest.java
@@ -0,0 +1,258 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.auth;
+
+import java.util.Set;
+
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Sets;
+import org.apache.commons.lang.RandomStringUtils;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.CQLStatement;
+import org.apache.cassandra.cql3.QueryOptions;
+import org.apache.cassandra.cql3.QueryProcessor;
+import org.apache.cassandra.cql3.UntypedResultSet;
+import org.apache.cassandra.cql3.statements.AlterRoleStatement;
+import org.apache.cassandra.cql3.statements.AuthenticationStatement;
+import org.apache.cassandra.cql3.statements.BatchStatement;
+import org.apache.cassandra.cql3.statements.CreateRoleStatement;
+import org.apache.cassandra.cql3.statements.DropRoleStatement;
+import org.apache.cassandra.cql3.statements.SelectStatement;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.exceptions.RequestExecutionException;
+import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.service.QueryState;
+import org.apache.cassandra.transport.messages.ResultMessage;
+
+import static org.apache.cassandra.auth.AuthKeyspace.NETWORK_PERMISSIONS;
+import static org.apache.cassandra.auth.RoleTestUtils.LocalCassandraRoleManager;
+import static org.apache.cassandra.schema.SchemaConstants.AUTH_KEYSPACE_NAME;
+import static org.apache.cassandra.auth.RoleTestUtils.getReadCount;
+
+public class CassandraNetworkAuthorizerTest
+{
+ private static class LocalCassandraAuthorizer extends CassandraAuthorizer
+ {
+ ResultMessage.Rows select(SelectStatement statement, QueryOptions options)
+ {
+ return statement.executeLocally(QueryState.forInternalCalls(), options);
+ }
+
+ UntypedResultSet process(String query) throws RequestExecutionException
+ {
+ return QueryProcessor.executeInternal(query);
+ }
+
+ @Override
+ void processBatch(BatchStatement statement)
+ {
+ statement.executeLocally(QueryState.forInternalCalls(), QueryOptions.DEFAULT);
+ }
+ }
+
+ private static class LocalCassandraNetworkAuthorizer extends CassandraNetworkAuthorizer
+ {
+ ResultMessage.Rows select(SelectStatement statement, QueryOptions options)
+ {
+ return statement.executeLocally(QueryState.forInternalCalls(), options);
+ }
+
+ void process(String query)
+ {
+ QueryProcessor.executeInternal(query);
+ }
+ }
+
+ private static void setupSuperUser()
+ {
+ QueryProcessor.executeInternal(String.format("INSERT INTO %s.%s (role, is_superuser, can_login, salted_hash) "
+ + "VALUES ('%s', true, true, '%s')",
+ AUTH_KEYSPACE_NAME,
+ AuthKeyspace.ROLES,
+ CassandraRoleManager.DEFAULT_SUPERUSER_NAME,
+ "xxx"));
+ }
+
+ @BeforeClass
+ public static void defineSchema() throws ConfigurationException
+ {
+ SchemaLoader.prepareServer();
+ SchemaLoader.setupAuth(new LocalCassandraRoleManager(),
+ new PasswordAuthenticator(),
+ new LocalCassandraAuthorizer(),
+ new LocalCassandraNetworkAuthorizer());
+ setupSuperUser();
+ // not strictly necessary to init the cache here, but better to be explicit
+ Roles.initRolesCache(DatabaseDescriptor.getRoleManager(), () -> true);
+ }
+
+ @Before
+ public void clear()
+ {
+ Keyspace.open(AUTH_KEYSPACE_NAME).getColumnFamilyStore(NETWORK_PERMISSIONS).truncateBlocking();
+ }
+
+ private static void assertNoDcPermRow(String username)
+ {
+ String query = String.format("SELECT dcs FROM %s.%s WHERE role = '%s'",
+ AUTH_KEYSPACE_NAME,
+ NETWORK_PERMISSIONS,
+ RoleResource.role(username).getName());
+ UntypedResultSet results = QueryProcessor.executeInternal(query);
+ Assert.assertTrue(results.isEmpty());
+ }
+
+ private static void assertDcPermRow(String username, String... dcs)
+ {
+ Set<String> expected = Sets.newHashSet(dcs);
+ String query = String.format("SELECT dcs FROM %s.%s WHERE role = '%s'",
+ AUTH_KEYSPACE_NAME,
+ NETWORK_PERMISSIONS,
+ RoleResource.role(username).getName());
+ UntypedResultSet results = QueryProcessor.executeInternal(query);
+ UntypedResultSet.Row row = Iterables.getOnlyElement(results);
+ Set<String> actual = row.getFrozenSet("dcs", UTF8Type.instance);
+ Assert.assertEquals(expected, actual);
+ }
+
+ private static String createName()
+ {
+ return RandomStringUtils.randomAlphabetic(8).toLowerCase();
+ }
+
+ private static ClientState getClientState()
+ {
+ ClientState state = ClientState.forInternalCalls();
+ state.login(new AuthenticatedUser(CassandraRoleManager.DEFAULT_SUPERUSER_NAME));
+ return state;
+ }
+
+ private static void auth(String query, Object... args)
+ {
+ CQLStatement statement = QueryProcessor.parseStatement(String.format(query, args)).prepare(ClientState.forInternalCalls());
+ assert statement instanceof CreateRoleStatement
+ || statement instanceof AlterRoleStatement
+ || statement instanceof DropRoleStatement;
+ AuthenticationStatement authStmt = (AuthenticationStatement) statement;
+
+ // invalidate roles cache so that any changes to the underlying roles are picked up
+ Roles.clearCache();
+ authStmt.execute(getClientState());
+ }
+
+ private static DCPermissions dcPerms(String username)
+ {
+ AuthenticatedUser user = new AuthenticatedUser(username);
+ return DatabaseDescriptor.getNetworkAuthorizer().authorize(user.getPrimaryRole());
+ }
+
+ @Test
+ public void create()
+ {
+ String username = createName();
+
+ // user should implicitly have access to all datacenters
+ assertNoDcPermRow(username);
+ auth("CREATE ROLE %s WITH password = 'password' AND LOGIN = true AND ACCESS TO DATACENTERS {'dc1', 'dc2'}", username);
+ Assert.assertEquals(DCPermissions.subset("dc1", "dc2"), dcPerms(username));
+ assertDcPermRow(username, "dc1", "dc2");
+ }
+
+ @Test
+ public void alter()
+ {
+
+ String username = createName();
+
+ assertNoDcPermRow(username);
+ // user should implicitly have access to all datacenters
+ auth("CREATE ROLE %s WITH password = 'password' AND LOGIN = true", username);
+ Assert.assertEquals(DCPermissions.all(), dcPerms(username));
+ assertDcPermRow(username);
+
+ // unless explicitly restricted
+ auth("ALTER ROLE %s WITH ACCESS TO DATACENTERS {'dc1', 'dc2'}", username);
+ Assert.assertEquals(DCPermissions.subset("dc1", "dc2"), dcPerms(username));
+ assertDcPermRow(username, "dc1", "dc2");
+
+ auth("ALTER ROLE %s WITH ACCESS TO DATACENTERS {'dc1'}", username);
+ Assert.assertEquals(DCPermissions.subset("dc1"), dcPerms(username));
+ assertDcPermRow(username, "dc1");
+
+ auth("ALTER ROLE %s WITH ACCESS TO ALL DATACENTERS", username);
+ Assert.assertEquals(DCPermissions.all(), dcPerms(username));
+ assertDcPermRow(username);
+ }
+
+ @Test
+ public void drop()
+ {
+ String username = createName();
+
+ assertNoDcPermRow(username);
+ // user should implicitly have access to all datacenters
+ auth("CREATE ROLE %s WITH password = 'password' AND LOGIN = true AND ACCESS TO DATACENTERS {'dc1'}", username);
+ assertDcPermRow(username, "dc1");
+
+ auth("DROP ROLE %s", username);
+ assertNoDcPermRow(username);
+ }
+
+ @Test
+ public void superUser()
+ {
+ String username = createName();
+ auth("CREATE ROLE %s WITH password = 'password' AND LOGIN = true AND ACCESS TO DATACENTERS {'dc1'}", username);
+ Assert.assertEquals(DCPermissions.subset("dc1"), dcPerms(username));
+ assertDcPermRow(username, "dc1");
+
+ // clear the roles cache to lose the (non-)superuser status for the user
+ Roles.clearCache();
+ auth("ALTER ROLE %s WITH superuser = true", username);
+ Assert.assertEquals(DCPermissions.all(), dcPerms(username));
+ }
+
+ @Test
+ public void cantLogin()
+ {
+ String username = createName();
+ auth("CREATE ROLE %s", username);
+ Assert.assertEquals(DCPermissions.none(), dcPerms(username));
+ }
+
+ @Test
+ public void getLoginPrivilegeFromRolesCache() throws Exception
+ {
+ String username = createName();
+ auth("CREATE ROLE %s", username);
+ long readCount = getReadCount();
+ dcPerms(username);
+ Assert.assertEquals(++readCount, getReadCount());
+ dcPerms(username);
+ Assert.assertEquals(readCount, getReadCount());
+ }
+}
diff --git a/test/unit/org/apache/cassandra/auth/CassandraRoleManagerTest.java b/test/unit/org/apache/cassandra/auth/CassandraRoleManagerTest.java
new file mode 100644
index 0000000..6583c49
--- /dev/null
+++ b/test/unit/org/apache/cassandra/auth/CassandraRoleManagerTest.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.auth;
+
+import java.util.Set;
+
+import com.google.common.collect.Iterables;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.SchemaConstants;
+import org.apache.cassandra.schema.TableMetadata;
+
+import static org.apache.cassandra.auth.RoleTestUtils.*;
+import static org.junit.Assert.assertEquals;
+
+public class CassandraRoleManagerTest
+{
+
+ @BeforeClass
+ public static void setupClass()
+ {
+ SchemaLoader.prepareServer();
+ // create the system_auth keyspace so the IRoleManager can function as normal
+ SchemaLoader.createKeyspace(SchemaConstants.AUTH_KEYSPACE_NAME,
+ KeyspaceParams.simple(1),
+ Iterables.toArray(AuthKeyspace.metadata().tables, TableMetadata.class));
+ }
+
+ @Test
+ public void getGrantedRolesImplMinimizesReads()
+ {
+ // IRoleManager::getRoleDetails was not in the initial API, so a default impl
+ // was added which uses the existing methods on IRoleManager as primitive to
+ // construct the Role objects. While this will work for any IRoleManager impl
+ // it is inefficient, so CassandraRoleManager has its own implementation which
+ // collects all of the necessary info with a single query for each granted role.
+ // This just tests that that is the case, i.e. we perform 1 read per role in the
+ // transitive set of granted roles
+ IRoleManager roleManager = new LocalCassandraRoleManager();
+ roleManager.setup();
+ for (RoleResource r : ALL_ROLES)
+ roleManager.createRole(AuthenticatedUser.ANONYMOUS_USER, r, new RoleOptions());
+
+ // simple role with no grants
+ fetchRolesAndCheckReadCount(roleManager, ROLE_A);
+ // single level of grants
+ grantRolesTo(roleManager, ROLE_A, ROLE_B, ROLE_C);
+ fetchRolesAndCheckReadCount(roleManager, ROLE_A);
+
+ // multi level role hierarchy
+ grantRolesTo(roleManager, ROLE_B, ROLE_B_1, ROLE_B_2, ROLE_B_3);
+ grantRolesTo(roleManager, ROLE_C, ROLE_C_1, ROLE_C_2, ROLE_C_3);
+ fetchRolesAndCheckReadCount(roleManager, ROLE_A);
+
+ // Check that when granted roles appear multiple times in parallel levels of the hierarchy, we don't
+ // do redundant reads. E.g. here role_b_1, role_b_2 and role_b3 are granted to both role_b and role_c
+ // but we only want to actually read them once
+ grantRolesTo(roleManager, ROLE_C, ROLE_B_1, ROLE_B_2, ROLE_B_3);
+ fetchRolesAndCheckReadCount(roleManager, ROLE_A);
+ }
+
+ private void fetchRolesAndCheckReadCount(IRoleManager roleManager, RoleResource primaryRole)
+ {
+ long before = getReadCount();
+ Set<Role> granted = roleManager.getRoleDetails(primaryRole);
+ long after = getReadCount();
+ assertEquals(granted.size(), after - before);
+ }
+}
diff --git a/test/unit/org/apache/cassandra/auth/PasswordAuthenticatorTest.java b/test/unit/org/apache/cassandra/auth/PasswordAuthenticatorTest.java
index 0dd75eb..5dd4ab5 100644
--- a/test/unit/org/apache/cassandra/auth/PasswordAuthenticatorTest.java
+++ b/test/unit/org/apache/cassandra/auth/PasswordAuthenticatorTest.java
@@ -26,13 +26,14 @@
import org.junit.Test;
import com.datastax.driver.core.Authenticator;
+import com.datastax.driver.core.EndPoint;
import com.datastax.driver.core.PlainTextAuthProvider;
import org.apache.cassandra.SchemaLoader;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.SchemaConstants;
import org.apache.cassandra.cql3.CQLTester;
import org.apache.cassandra.exceptions.AuthenticationException;
import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.SchemaConstants;
+import org.apache.cassandra.schema.TableMetadata;
import static org.apache.cassandra.auth.CassandraRoleManager.*;
import static org.apache.cassandra.auth.PasswordAuthenticator.*;
@@ -121,7 +122,7 @@
{
SaslNegotiator negotiator = authenticator.newSaslNegotiator(null);
Authenticator clientAuthenticator = (new PlainTextAuthProvider(username, password))
- .newAuthenticator(null, null);
+ .newAuthenticator((EndPoint) null, null);
negotiator.evaluateResponse(clientAuthenticator.initialResponse());
negotiator.getAuthenticatedUser();
@@ -132,7 +133,7 @@
{
SchemaLoader.createKeyspace(SchemaConstants.AUTH_KEYSPACE_NAME,
KeyspaceParams.simple(1),
- Iterables.toArray(AuthKeyspace.metadata().tables, CFMetaData.class));
+ Iterables.toArray(AuthKeyspace.metadata().tables, TableMetadata.class));
authenticator.setup();
}
diff --git a/test/unit/org/apache/cassandra/auth/RoleTestUtils.java b/test/unit/org/apache/cassandra/auth/RoleTestUtils.java
new file mode 100644
index 0000000..e2d1006
--- /dev/null
+++ b/test/unit/org/apache/cassandra/auth/RoleTestUtils.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.auth;
+
+import java.util.concurrent.Callable;
+
+import org.apache.cassandra.cql3.QueryOptions;
+import org.apache.cassandra.cql3.QueryProcessor;
+import org.apache.cassandra.cql3.UntypedResultSet;
+import org.apache.cassandra.cql3.statements.SelectStatement;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.ConsistencyLevel;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.schema.SchemaConstants;
+import org.apache.cassandra.service.QueryState;
+import org.apache.cassandra.transport.messages.ResultMessage;
+
+
+public class RoleTestUtils
+{
+
+ public static final RoleResource ROLE_A = RoleResource.role("role_a");
+ public static final RoleResource ROLE_B = RoleResource.role("role_b");
+ public static final RoleResource ROLE_B_1 = RoleResource.role("role_b_1");
+ public static final RoleResource ROLE_B_2 = RoleResource.role("role_b_2");
+ public static final RoleResource ROLE_B_3 = RoleResource.role("role_b_3");
+ public static final RoleResource ROLE_C = RoleResource.role("role_c");
+ public static final RoleResource ROLE_C_1 = RoleResource.role("role_c_1");
+ public static final RoleResource ROLE_C_2 = RoleResource.role("role_c_2");
+ public static final RoleResource ROLE_C_3 = RoleResource.role("role_c_3");
+ public static final RoleResource[] ALL_ROLES = new RoleResource[] {ROLE_A,
+ ROLE_B, ROLE_B_1, ROLE_B_2, ROLE_B_3,
+ ROLE_C, ROLE_C_1, ROLE_C_2, ROLE_C_3};
+ /**
+ * This just extends the internal IRoleManager implementation to ensure that
+ * all access to underlying tables is made via
+ * QueryProcessor.executeOnceInternal/CQLStatement.executeInternal and not
+ * StorageProxy so that it can be used in unit tests.
+ */
+ public static class LocalCassandraRoleManager extends CassandraRoleManager
+ {
+ ResultMessage.Rows select(SelectStatement statement, QueryOptions options)
+ {
+ return statement.executeLocally(QueryState.forInternalCalls(), options);
+ }
+
+ UntypedResultSet process(String query, ConsistencyLevel consistencyLevel)
+ {
+ return QueryProcessor.executeInternal(query);
+ }
+
+ protected void scheduleSetupTask(final Callable<Void> setupTask)
+ {
+ // skip data migration or setting up default role for tests
+ }
+ }
+
+ public static void grantRolesTo(IRoleManager roleManager, RoleResource grantee, RoleResource...granted)
+ {
+ for(RoleResource toGrant : granted)
+ roleManager.grantRole(AuthenticatedUser.ANONYMOUS_USER, toGrant, grantee);
+ }
+
+ public static long getReadCount()
+ {
+ ColumnFamilyStore rolesTable = Keyspace.open(SchemaConstants.AUTH_KEYSPACE_NAME).getColumnFamilyStore(AuthKeyspace.ROLES);
+ return rolesTable.metric.readLatency.latency.getCount();
+ }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/auth/RolesTest.java b/test/unit/org/apache/cassandra/auth/RolesTest.java
new file mode 100644
index 0000000..94322a7
--- /dev/null
+++ b/test/unit/org/apache/cassandra/auth/RolesTest.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.auth;
+
+import java.util.Set;
+
+import com.google.common.collect.Iterables;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.SchemaConstants;
+import org.apache.cassandra.schema.TableMetadata;
+
+import static org.apache.cassandra.auth.RoleTestUtils.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class RolesTest
+{
+
+ @BeforeClass
+ public static void setupClass()
+ {
+ SchemaLoader.prepareServer();
+ // create the system_auth keyspace so the IRoleManager can function as normal
+ SchemaLoader.createKeyspace(SchemaConstants.AUTH_KEYSPACE_NAME,
+ KeyspaceParams.simple(1),
+ Iterables.toArray(AuthKeyspace.metadata().tables, TableMetadata.class));
+
+ IRoleManager roleManager = new LocalCassandraRoleManager();
+ roleManager.setup();
+ Roles.initRolesCache(roleManager, () -> true);
+ for (RoleResource role : ALL_ROLES)
+ roleManager.createRole(AuthenticatedUser.ANONYMOUS_USER, role, new RoleOptions());
+ grantRolesTo(roleManager, ROLE_A, ROLE_B, ROLE_C);
+ }
+
+ @Test
+ public void superuserStatusIsCached()
+ {
+ boolean hasSuper = Roles.hasSuperuserStatus(ROLE_A);
+ long count = getReadCount();
+
+ assertEquals(hasSuper, Roles.hasSuperuserStatus(ROLE_A));
+ assertEquals(count, getReadCount());
+ }
+
+ @Test
+ public void loginPrivilegeIsCached()
+ {
+ boolean canLogin = Roles.canLogin(ROLE_A);
+ long count = getReadCount();
+
+ assertEquals(canLogin, Roles.canLogin(ROLE_A));
+ assertEquals(count, getReadCount());
+ }
+
+ @Test
+ public void grantedRoleDetailsAreCached()
+ {
+ Iterable<Role> granted = Roles.getRoleDetails(ROLE_A);
+ long count = getReadCount();
+
+ assertTrue(Iterables.elementsEqual(granted, Roles.getRoleDetails(ROLE_A)));
+ assertEquals(count, getReadCount());
+ }
+
+ @Test
+ public void grantedRoleResourcesAreCached()
+ {
+ Set<RoleResource> granted = Roles.getRoles(ROLE_A);
+ long count = getReadCount();
+
+ assertEquals(granted, Roles.getRoles(ROLE_A));
+ assertEquals(count, getReadCount());
+ }
+}
diff --git a/test/unit/org/apache/cassandra/auth/jmx/AuthorizationProxyTest.java b/test/unit/org/apache/cassandra/auth/jmx/AuthorizationProxyTest.java
index e68ef20..85b1f7f 100644
--- a/test/unit/org/apache/cassandra/auth/jmx/AuthorizationProxyTest.java
+++ b/test/unit/org/apache/cassandra/auth/jmx/AuthorizationProxyTest.java
@@ -20,8 +20,9 @@
import java.util.*;
import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.BooleanSupplier;
import java.util.function.Function;
-import java.util.function.Supplier;
+import java.util.function.Predicate;
import java.util.stream.Collectors;
import javax.management.MalformedObjectNameException;
import javax.management.ObjectName;
@@ -495,9 +496,9 @@
{
Function<RoleResource, Set<PermissionDetails>> getPermissions;
Function<ObjectName, Set<ObjectName>> queryNames;
- Function<RoleResource, Boolean> isSuperuser;
- Supplier<Boolean> isAuthzRequired;
- Supplier<Boolean> isAuthSetupComplete = () -> true;
+ Predicate<RoleResource> isSuperuser;
+ BooleanSupplier isAuthzRequired;
+ BooleanSupplier isAuthSetupComplete = () -> true;
AuthorizationProxy build()
{
@@ -532,19 +533,19 @@
return this;
}
- ProxyBuilder isSuperuser(Function<RoleResource, Boolean> f)
+ ProxyBuilder isSuperuser(Predicate<RoleResource> f)
{
isSuperuser = f;
return this;
}
- ProxyBuilder isAuthzRequired(Supplier<Boolean> s)
+ ProxyBuilder isAuthzRequired(BooleanSupplier s)
{
isAuthzRequired = s;
return this;
}
- ProxyBuilder isAuthSetupComplete(Supplier<Boolean> s)
+ ProxyBuilder isAuthSetupComplete(BooleanSupplier s)
{
isAuthSetupComplete = s;
return this;
@@ -562,17 +563,17 @@
this.queryNames = f;
}
- void setIsSuperuser(Function<RoleResource, Boolean> f)
+ void setIsSuperuser(Predicate<RoleResource> f)
{
this.isSuperuser = f;
}
- void setIsAuthzRequired(Supplier<Boolean> s)
+ void setIsAuthzRequired(BooleanSupplier s)
{
this.isAuthzRequired = s;
}
- void setIsAuthSetupComplete(Supplier<Boolean> s)
+ void setIsAuthSetupComplete(BooleanSupplier s)
{
this.isAuthSetupComplete = s;
}
diff --git a/test/unit/org/apache/cassandra/auth/jmx/JMXAuthTest.java b/test/unit/org/apache/cassandra/auth/jmx/JMXAuthTest.java
index 08a89df..3bc28a9 100644
--- a/test/unit/org/apache/cassandra/auth/jmx/JMXAuthTest.java
+++ b/test/unit/org/apache/cassandra/auth/jmx/JMXAuthTest.java
@@ -63,7 +63,6 @@
@BeforeClass
public static void setupClass() throws Exception
{
- DatabaseDescriptor.daemonInitialization();
setupAuthorizer();
setupJMXServer();
}
@@ -91,7 +90,7 @@
System.setProperty("java.security.auth.login.config", config);
System.setProperty("cassandra.jmx.remote.login.config", "TestLogin");
System.setProperty("cassandra.jmx.authorizer", NoSuperUserAuthorizationProxy.class.getName());
- jmxServer = JMXServerUtils.createJMXServer(9999, true);
+ jmxServer = JMXServerUtils.createJMXServer(9999, "localhost", true);
jmxServer.start();
JMXServiceURL jmxUrl = new JMXServiceURL("service:jmx:rmi:///jndi/rmi://localhost:9999/jmxrmi");
diff --git a/test/unit/org/apache/cassandra/batchlog/BatchTest.java b/test/unit/org/apache/cassandra/batchlog/BatchTest.java
deleted file mode 100644
index 4e64ec6..0000000
--- a/test/unit/org/apache/cassandra/batchlog/BatchTest.java
+++ /dev/null
@@ -1,168 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.batchlog;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-import java.util.UUID;
-
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import org.apache.cassandra.SchemaLoader;
-import org.apache.cassandra.Util;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.db.Keyspace;
-import org.apache.cassandra.db.Mutation;
-import org.apache.cassandra.db.RowUpdateBuilder;
-import org.apache.cassandra.db.marshal.BytesType;
-import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.io.util.DataInputBuffer;
-import org.apache.cassandra.io.util.DataInputPlus;
-import org.apache.cassandra.io.util.DataOutputBuffer;
-import org.apache.cassandra.net.MessagingService;
-import org.apache.cassandra.schema.KeyspaceParams;
-import org.apache.cassandra.utils.FBUtilities;
-import org.apache.cassandra.utils.UUIDGen;
-
-import static org.apache.cassandra.utils.ByteBufferUtil.bytes;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-public class BatchTest
-{
- private static final String KEYSPACE = "BatchRequestTest";
- private static final String CF_STANDARD = "Standard";
-
- @BeforeClass
- public static void defineSchema() throws ConfigurationException
- {
- SchemaLoader.prepareServer();
- SchemaLoader.createKeyspace(KEYSPACE,
- KeyspaceParams.simple(1),
- SchemaLoader.standardCFMD(KEYSPACE, CF_STANDARD, 1, BytesType.instance));
- }
-
- @Test
- public void testSerialization() throws IOException
- {
- CFMetaData cfm = Keyspace.open(KEYSPACE).getColumnFamilyStore(CF_STANDARD).metadata;
-
- long now = FBUtilities.timestampMicros();
- int version = MessagingService.current_version;
- UUID uuid = UUIDGen.getTimeUUID();
-
- List<Mutation> mutations = new ArrayList<>(10);
- for (int i = 0; i < 10; i++)
- {
- mutations.add(new RowUpdateBuilder(cfm, FBUtilities.timestampMicros(), bytes(i))
- .clustering("name" + i)
- .add("val", "val" + i)
- .build());
- }
-
- Batch batch1 = Batch.createLocal(uuid, now, mutations);
- assertEquals(uuid, batch1.id);
- assertEquals(now, batch1.creationTime);
- assertEquals(mutations, batch1.decodedMutations);
-
- DataOutputBuffer out = new DataOutputBuffer();
- Batch.serializer.serialize(batch1, out, version);
-
- assertEquals(out.getLength(), Batch.serializer.serializedSize(batch1, version));
-
- DataInputPlus dis = new DataInputBuffer(out.getData());
- Batch batch2 = Batch.serializer.deserialize(dis, version);
-
- assertEquals(batch1.id, batch2.id);
- assertEquals(batch1.creationTime, batch2.creationTime);
- assertEquals(batch1.decodedMutations.size(), batch2.encodedMutations.size());
-
- Iterator<Mutation> it1 = batch1.decodedMutations.iterator();
- Iterator<ByteBuffer> it2 = batch2.encodedMutations.iterator();
- while (it1.hasNext())
- {
- try (DataInputBuffer in = new DataInputBuffer(it2.next().array()))
- {
- assertEquals(it1.next().toString(), Mutation.serializer.deserialize(in, version).toString());
- }
- }
- }
-
- /**
- * This is just to test decodeMutations() when deserializing,
- * since Batch will never be serialized at a version 2.2.
- * @throws IOException
- */
- @Test
- public void testSerializationNonCurrentVersion() throws IOException
- {
- CFMetaData cfm = Keyspace.open(KEYSPACE).getColumnFamilyStore(CF_STANDARD).metadata;
-
- long now = FBUtilities.timestampMicros();
- int version = MessagingService.VERSION_22;
- UUID uuid = UUIDGen.getTimeUUID();
-
- List<Mutation> mutations = new ArrayList<>(10);
- for (int i = 0; i < 10; i++)
- {
- mutations.add(new RowUpdateBuilder(cfm, FBUtilities.timestampMicros(), bytes(i))
- .clustering("name" + i)
- .add("val", "val" + i)
- .build());
- }
-
- Batch batch1 = Batch.createLocal(uuid, now, mutations);
- assertEquals(uuid, batch1.id);
- assertEquals(now, batch1.creationTime);
- assertEquals(mutations, batch1.decodedMutations);
-
- DataOutputBuffer out = new DataOutputBuffer();
- Batch.serializer.serialize(batch1, out, version);
-
- assertEquals(out.getLength(), Batch.serializer.serializedSize(batch1, version));
-
- DataInputPlus dis = new DataInputBuffer(out.getData());
- Batch batch2 = Batch.serializer.deserialize(dis, version);
-
- assertEquals(batch1.id, batch2.id);
- assertEquals(batch1.creationTime, batch2.creationTime);
- assertEquals(batch1.decodedMutations.size(), batch2.decodedMutations.size());
-
- Iterator<Mutation> it1 = batch1.decodedMutations.iterator();
- Iterator<Mutation> it2 = batch2.decodedMutations.iterator();
- while (it1.hasNext())
- {
- // We can't simply test the equality of both mutation string representation, that is do:
- // assertEquals(it1.next().toString(), it2.next().toString());
- // because when deserializing from the old format, the returned iterator will always have it's 'columns()'
- // method return all the table columns (no matter what's the actual content), and the table contains a
- // 'val0' column we're not setting in that test.
- //
- // And it's actually not easy to fix legacy deserialization as we'd need to know which columns are actually
- // set upfront, which would require use to iterate over the whole content first, which would be costly. And
- // as the result of 'columns()' is only meant as a superset of the columns in the iterator, we don't bother.
- Mutation mut1 = it1.next();
- Mutation mut2 = it2.next();
- assertTrue(mut1 + " != " + mut2, Util.sameContent(mut1, mut2));
- }
- }
-}
diff --git a/test/unit/org/apache/cassandra/batchlog/BatchlogEndpointFilterTest.java b/test/unit/org/apache/cassandra/batchlog/BatchlogEndpointFilterTest.java
index 7db1cfa..c2b9fc9 100644
--- a/test/unit/org/apache/cassandra/batchlog/BatchlogEndpointFilterTest.java
+++ b/test/unit/org/apache/cassandra/batchlog/BatchlogEndpointFilterTest.java
@@ -17,11 +17,9 @@
*/
package org.apache.cassandra.batchlog;
-import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.Collection;
import java.util.Collections;
-import java.util.HashSet;
import java.util.List;
import com.google.common.collect.ImmutableMultimap;
@@ -29,8 +27,13 @@
import org.junit.Test;
import org.junit.matchers.JUnitMatchers;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.ReplicaPlan;
+import org.apache.cassandra.locator.ReplicaPlans;
+
import static org.hamcrest.CoreMatchers.is;
import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
public class BatchlogEndpointFilterTest
{
@@ -39,106 +42,88 @@
@Test
public void shouldSelect2hostsFromNonLocalRacks() throws UnknownHostException
{
- Multimap<String, InetAddress> endpoints = ImmutableMultimap.<String, InetAddress> builder()
- .put(LOCAL, InetAddress.getByName("0"))
- .put(LOCAL, InetAddress.getByName("00"))
- .put("1", InetAddress.getByName("1"))
- .put("1", InetAddress.getByName("11"))
- .put("2", InetAddress.getByName("2"))
- .put("2", InetAddress.getByName("22"))
+ Multimap<String, InetAddressAndPort> endpoints = ImmutableMultimap.<String, InetAddressAndPort> builder()
+ .put(LOCAL, InetAddressAndPort.getByName("0"))
+ .put(LOCAL, InetAddressAndPort.getByName("00"))
+ .put("1", InetAddressAndPort.getByName("1"))
+ .put("1", InetAddressAndPort.getByName("11"))
+ .put("2", InetAddressAndPort.getByName("2"))
+ .put("2", InetAddressAndPort.getByName("22"))
.build();
- Collection<InetAddress> result = new TestEndpointFilter(LOCAL, endpoints).filter();
+ Collection<InetAddressAndPort> result = filterBatchlogEndpoints(endpoints);
assertThat(result.size(), is(2));
- assertThat(result, JUnitMatchers.hasItem(InetAddress.getByName("11")));
- assertThat(result, JUnitMatchers.hasItem(InetAddress.getByName("22")));
+ assertTrue(result.contains(InetAddressAndPort.getByName("11")));
+ assertTrue(result.contains(InetAddressAndPort.getByName("22")));
}
@Test
public void shouldSelectHostFromLocal() throws UnknownHostException
{
- Multimap<String, InetAddress> endpoints = ImmutableMultimap.<String, InetAddress> builder()
- .put(LOCAL, InetAddress.getByName("0"))
- .put(LOCAL, InetAddress.getByName("00"))
- .put("1", InetAddress.getByName("1"))
+ Multimap<String, InetAddressAndPort> endpoints = ImmutableMultimap.<String, InetAddressAndPort> builder()
+ .put(LOCAL, InetAddressAndPort.getByName("0"))
+ .put(LOCAL, InetAddressAndPort.getByName("00"))
+ .put("1", InetAddressAndPort.getByName("1"))
.build();
- Collection<InetAddress> result = new TestEndpointFilter(LOCAL, endpoints).filter();
+ Collection<InetAddressAndPort> result = filterBatchlogEndpoints(endpoints);
assertThat(result.size(), is(2));
- assertThat(result, JUnitMatchers.hasItem(InetAddress.getByName("1")));
- assertThat(result, JUnitMatchers.hasItem(InetAddress.getByName("0")));
+ assertTrue(result.contains(InetAddressAndPort.getByName("1")));
+ assertTrue(result.contains(InetAddressAndPort.getByName("0")));
}
@Test
public void shouldReturnAsIsIfNoEnoughEndpoints() throws UnknownHostException
{
- Multimap<String, InetAddress> endpoints = ImmutableMultimap.<String, InetAddress> builder()
- .put(LOCAL, InetAddress.getByName("0"))
+ Multimap<String, InetAddressAndPort> endpoints = ImmutableMultimap.<String, InetAddressAndPort> builder()
+ .put(LOCAL, InetAddressAndPort.getByName("0"))
.build();
- Collection<InetAddress> result = new TestEndpointFilter(LOCAL, endpoints).filter();
+ Collection<InetAddressAndPort> result = filterBatchlogEndpoints(endpoints);
assertThat(result.size(), is(1));
- assertThat(result, JUnitMatchers.hasItem(InetAddress.getByName("0")));
+ assertTrue(result.contains(InetAddressAndPort.getByName("0")));
}
@Test
public void shouldSelectTwoRandomHostsFromSingleOtherRack() throws UnknownHostException
{
- Multimap<String, InetAddress> endpoints = ImmutableMultimap.<String, InetAddress> builder()
- .put(LOCAL, InetAddress.getByName("0"))
- .put(LOCAL, InetAddress.getByName("00"))
- .put("1", InetAddress.getByName("1"))
- .put("1", InetAddress.getByName("11"))
- .put("1", InetAddress.getByName("111"))
+ Multimap<String, InetAddressAndPort> endpoints = ImmutableMultimap.<String, InetAddressAndPort> builder()
+ .put(LOCAL, InetAddressAndPort.getByName("0"))
+ .put(LOCAL, InetAddressAndPort.getByName("00"))
+ .put("1", InetAddressAndPort.getByName("1"))
+ .put("1", InetAddressAndPort.getByName("11"))
+ .put("1", InetAddressAndPort.getByName("111"))
.build();
- Collection<InetAddress> result = new TestEndpointFilter(LOCAL, endpoints).filter();
+ Collection<InetAddressAndPort> result = filterBatchlogEndpoints(endpoints);
// result should be the last two non-local replicas
// (Collections.shuffle has been replaced with Collections.reverse for testing)
assertThat(result.size(), is(2));
- assertThat(result, JUnitMatchers.hasItem(InetAddress.getByName("11")));
- assertThat(result, JUnitMatchers.hasItem(InetAddress.getByName("111")));
+ assertTrue(result.contains(InetAddressAndPort.getByName("11")));
+ assertTrue(result.contains(InetAddressAndPort.getByName("111")));
}
@Test
public void shouldSelectTwoRandomHostsFromSingleRack() throws UnknownHostException
{
- Multimap<String, InetAddress> endpoints = ImmutableMultimap.<String, InetAddress> builder()
- .put(LOCAL, InetAddress.getByName("1"))
- .put(LOCAL, InetAddress.getByName("11"))
- .put(LOCAL, InetAddress.getByName("111"))
- .put(LOCAL, InetAddress.getByName("1111"))
+ Multimap<String, InetAddressAndPort> endpoints = ImmutableMultimap.<String, InetAddressAndPort> builder()
+ .put(LOCAL, InetAddressAndPort.getByName("1"))
+ .put(LOCAL, InetAddressAndPort.getByName("11"))
+ .put(LOCAL, InetAddressAndPort.getByName("111"))
+ .put(LOCAL, InetAddressAndPort.getByName("1111"))
.build();
- Collection<InetAddress> result = new TestEndpointFilter(LOCAL, endpoints).filter();
+ Collection<InetAddressAndPort> result = filterBatchlogEndpoints(endpoints);
// result should be the last two non-local replicas
// (Collections.shuffle has been replaced with Collections.reverse for testing)
assertThat(result.size(), is(2));
- assertThat(result, JUnitMatchers.hasItem(InetAddress.getByName("111")));
- assertThat(result, JUnitMatchers.hasItem(InetAddress.getByName("1111")));
+ assertTrue(result.contains(InetAddressAndPort.getByName("111")));
+ assertTrue(result.contains(InetAddressAndPort.getByName("1111")));
}
- private static class TestEndpointFilter extends BatchlogManager.EndpointFilter
+ private Collection<InetAddressAndPort> filterBatchlogEndpoints(Multimap<String, InetAddressAndPort> endpoints)
{
- TestEndpointFilter(String localRack, Multimap<String, InetAddress> endpoints)
- {
- super(localRack, endpoints);
- }
-
- @Override
- protected boolean isValid(InetAddress input)
- {
- // We will use always alive non-localhost endpoints
- return true;
- }
-
- @Override
- protected int getRandomInt(int bound)
- {
- // We don't need random behavior here
- return bound - 1;
- }
-
- @Override
- protected void shuffle(List<?> list)
- {
- // We don't need random behavior here
- Collections.reverse(list);
- }
+ return ReplicaPlans.filterBatchlogEndpoints(LOCAL, endpoints,
+ // Reverse instead of shuffle
+ Collections::reverse,
+ // Always alive
+ (addr) -> true,
+ // Always pick the last
+ (size) -> size - 1);
}
}
diff --git a/test/unit/org/apache/cassandra/batchlog/BatchlogManagerTest.java b/test/unit/org/apache/cassandra/batchlog/BatchlogManagerTest.java
index f192bcf..361759f 100644
--- a/test/unit/org/apache/cassandra/batchlog/BatchlogManagerTest.java
+++ b/test/unit/org/apache/cassandra/batchlog/BatchlogManagerTest.java
@@ -18,10 +18,8 @@
package org.apache.cassandra.batchlog;
import java.io.IOException;
-import java.net.InetAddress;
import java.util.*;
import java.util.concurrent.ExecutionException;
-import java.util.concurrent.TimeUnit;
import com.google.common.collect.Lists;
@@ -30,10 +28,11 @@
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.Util;
import org.apache.cassandra.Util.PartitionerSwitcher;
-import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.config.Schema;
-import org.apache.cassandra.config.SchemaConstants;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.SchemaConstants;
import org.apache.cassandra.cql3.UntypedResultSet;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.DecoratedKey;
@@ -49,13 +48,13 @@
import org.apache.cassandra.dht.Murmur3Partitioner;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.locator.TokenMetadata;
-import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.schema.KeyspaceParams;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.UUIDGen;
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
import static org.apache.cassandra.cql3.QueryProcessor.executeInternal;
import static org.junit.Assert.*;
@@ -96,18 +95,17 @@
public void setUp() throws Exception
{
TokenMetadata metadata = StorageService.instance.getTokenMetadata();
- InetAddress localhost = InetAddress.getByName("127.0.0.1");
+ InetAddressAndPort localhost = InetAddressAndPort.getByName("127.0.0.1");
metadata.updateNormalToken(Util.token("A"), localhost);
metadata.updateHostId(UUIDGen.getTimeUUID(), localhost);
Keyspace.open(SchemaConstants.SYSTEM_KEYSPACE_NAME).getColumnFamilyStore(SystemKeyspace.BATCHES).truncateBlocking();
- Keyspace.open(SchemaConstants.SYSTEM_KEYSPACE_NAME).getColumnFamilyStore(SystemKeyspace.LEGACY_BATCHLOG).truncateBlocking();
}
@Test
public void testDelete()
{
ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD1);
- CFMetaData cfm = cfs.metadata;
+ TableMetadata cfm = cfs.metadata();
new RowUpdateBuilder(cfm, FBUtilities.timestampMicros(), ByteBufferUtil.bytes("1234"))
.clustering("c")
.add("val", "val" + 1234)
@@ -129,24 +127,13 @@
}
@Test
- public void testReplay() throws Exception
- {
- testReplay(false);
- }
-
- @Test
- public void testLegacyReplay() throws Exception
- {
- testReplay(true);
- }
-
@SuppressWarnings("deprecation")
- private static void testReplay(boolean legacy) throws Exception
+ public void testReplay() throws Exception
{
long initialAllBatches = BatchlogManager.instance.countAllBatches();
long initialReplayedBatches = BatchlogManager.instance.getTotalBatchesReplayed();
- CFMetaData cfm = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD1).metadata;
+ TableMetadata cfm = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD1).metadata();
// Generate 1000 mutations (100 batches of 10 mutations each) and put them all into the batchlog.
// Half batches (50) ready to be replayed, half not.
@@ -165,16 +152,7 @@
? (System.currentTimeMillis() - BatchlogManager.getBatchlogTimeout())
: (System.currentTimeMillis() + BatchlogManager.getBatchlogTimeout());
- if (legacy)
- LegacyBatchlogMigrator.store(Batch.createLocal(UUIDGen.getTimeUUID(timestamp, i), timestamp * 1000, mutations), MessagingService.current_version);
- else
- BatchlogManager.store(Batch.createLocal(UUIDGen.getTimeUUID(timestamp, i), timestamp * 1000, mutations));
- }
-
- if (legacy)
- {
- Keyspace.open(SchemaConstants.SYSTEM_KEYSPACE_NAME).getColumnFamilyStore(SystemKeyspace.LEGACY_BATCHLOG).forceBlockingFlush();
- LegacyBatchlogMigrator.migrate();
+ BatchlogManager.store(Batch.createLocal(UUIDGen.getTimeUUID(timestamp, i), timestamp * 1000, mutations));
}
// Flush the batchlog to disk (see CASSANDRA-6822).
@@ -226,8 +204,8 @@
@Test
public void testTruncatedReplay() throws InterruptedException, ExecutionException
{
- CFMetaData cf2 = Schema.instance.getCFMetaData(KEYSPACE1, CF_STANDARD2);
- CFMetaData cf3 = Schema.instance.getCFMetaData(KEYSPACE1, CF_STANDARD3);
+ TableMetadata cf2 = Schema.instance.getTableMetadata(KEYSPACE1, CF_STANDARD2);
+ TableMetadata cf3 = Schema.instance.getTableMetadata(KEYSPACE1, CF_STANDARD3);
// Generate 2000 mutations (1000 batchlog entries) and put them all into the batchlog.
// Each batchlog entry with a mutation for Standard2 and Standard3.
// In the middle of the process, 'truncate' Standard2.
@@ -295,110 +273,12 @@
}
@Test
- @SuppressWarnings("deprecation")
- public void testConversion() throws Exception
- {
- long initialAllBatches = BatchlogManager.instance.countAllBatches();
- long initialReplayedBatches = BatchlogManager.instance.getTotalBatchesReplayed();
- CFMetaData cfm = Schema.instance.getCFMetaData(KEYSPACE1, CF_STANDARD4);
-
- // Generate 1400 version 2.0 mutations and put them all into the batchlog.
- // Half ready to be replayed, half not.
- for (int i = 0; i < 1400; i++)
- {
- Mutation mutation = new RowUpdateBuilder(cfm, FBUtilities.timestampMicros(), ByteBufferUtil.bytes(i))
- .clustering("name" + i)
- .add("val", "val" + i)
- .build();
-
- long timestamp = i < 700
- ? (System.currentTimeMillis() - BatchlogManager.getBatchlogTimeout())
- : (System.currentTimeMillis() + BatchlogManager.getBatchlogTimeout());
-
-
- Mutation batchMutation = LegacyBatchlogMigrator.getStoreMutation(Batch.createLocal(UUIDGen.getTimeUUID(timestamp, i),
- TimeUnit.MILLISECONDS.toMicros(timestamp),
- Collections.singleton(mutation)),
- MessagingService.VERSION_20);
- assertTrue(LegacyBatchlogMigrator.isLegacyBatchlogMutation(batchMutation));
- LegacyBatchlogMigrator.handleLegacyMutation(batchMutation);
- }
-
- // Mix in 100 current version mutations, 50 ready for replay.
- for (int i = 1400; i < 1500; i++)
- {
- Mutation mutation = new RowUpdateBuilder(cfm, FBUtilities.timestampMicros(), ByteBufferUtil.bytes(i))
- .clustering("name" + i)
- .add("val", "val" + i)
- .build();
-
- long timestamp = i < 1450
- ? (System.currentTimeMillis() - BatchlogManager.getBatchlogTimeout())
- : (System.currentTimeMillis() + BatchlogManager.getBatchlogTimeout());
-
-
- BatchlogManager.store(Batch.createLocal(UUIDGen.getTimeUUID(timestamp, i),
- FBUtilities.timestampMicros(),
- Collections.singleton(mutation)));
- }
-
- // Flush the batchlog to disk (see CASSANDRA-6822).
- Keyspace.open(SchemaConstants.SYSTEM_KEYSPACE_NAME).getColumnFamilyStore(SystemKeyspace.BATCHES).forceBlockingFlush();
-
- assertEquals(1500, BatchlogManager.instance.countAllBatches() - initialAllBatches);
- assertEquals(0, BatchlogManager.instance.getTotalBatchesReplayed() - initialReplayedBatches);
-
- UntypedResultSet result = executeInternal(String.format("SELECT count(*) FROM \"%s\".\"%s\"", SchemaConstants.SYSTEM_KEYSPACE_NAME, SystemKeyspace.LEGACY_BATCHLOG));
- assertNotNull(result);
- assertEquals("Count in blog legacy", 0, result.one().getLong("count"));
- result = executeInternal(String.format("SELECT count(*) FROM \"%s\".\"%s\"", SchemaConstants.SYSTEM_KEYSPACE_NAME, SystemKeyspace.BATCHES));
- assertNotNull(result);
- assertEquals("Count in blog", 1500, result.one().getLong("count"));
-
- // Force batchlog replay and wait for it to complete.
- BatchlogManager.instance.performInitialReplay();
-
- // Ensure that the first half, and only the first half, got replayed.
- assertEquals(750, BatchlogManager.instance.countAllBatches() - initialAllBatches);
- assertEquals(750, BatchlogManager.instance.getTotalBatchesReplayed() - initialReplayedBatches);
-
- for (int i = 0; i < 1500; i++)
- {
- result = executeInternal(String.format("SELECT * FROM \"%s\".\"%s\" WHERE key = intAsBlob(%d)", KEYSPACE1, CF_STANDARD4, i));
- assertNotNull(result);
- if (i < 700 || i >= 1400 && i < 1450)
- {
- assertEquals(ByteBufferUtil.bytes(i), result.one().getBytes("key"));
- assertEquals("name" + i, result.one().getString("name"));
- assertEquals("val" + i, result.one().getString("val"));
- }
- else
- {
- assertTrue("Present at " + i, result.isEmpty());
- }
- }
-
- // Ensure that no stray mutations got somehow applied.
- result = executeInternal(String.format("SELECT count(*) FROM \"%s\".\"%s\"", KEYSPACE1, CF_STANDARD4));
- assertNotNull(result);
- assertEquals(750, result.one().getLong("count"));
-
- // Ensure batchlog is left as expected.
- result = executeInternal(String.format("SELECT count(*) FROM \"%s\".\"%s\"", SchemaConstants.SYSTEM_KEYSPACE_NAME, SystemKeyspace.BATCHES));
- assertNotNull(result);
- assertEquals("Count in blog after initial replay", 750, result.one().getLong("count"));
- result = executeInternal(String.format("SELECT count(*) FROM \"%s\".\"%s\"", SchemaConstants.SYSTEM_KEYSPACE_NAME, SystemKeyspace.LEGACY_BATCHLOG));
- assertNotNull(result);
- assertEquals("Count in blog legacy after initial replay ", 0, result.one().getLong("count"));
- }
-
- @Test
public void testAddBatch() throws IOException
{
long initialAllBatches = BatchlogManager.instance.countAllBatches();
- CFMetaData cfm = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD5).metadata;
+ TableMetadata cfm = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD5).metadata();
- long timestamp = (System.currentTimeMillis() - DatabaseDescriptor.getWriteRpcTimeout() * 2) * 1000;
+ long timestamp = (System.currentTimeMillis() - DatabaseDescriptor.getWriteRpcTimeout(MILLISECONDS) * 2) * 1000;
UUID uuid = UUIDGen.getTimeUUID();
// Add a batch with 10 mutations
@@ -428,9 +308,9 @@
public void testRemoveBatch()
{
long initialAllBatches = BatchlogManager.instance.countAllBatches();
- CFMetaData cfm = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD5).metadata;
+ TableMetadata cfm = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD5).metadata();
- long timestamp = (System.currentTimeMillis() - DatabaseDescriptor.getWriteRpcTimeout() * 2) * 1000;
+ long timestamp = (System.currentTimeMillis() - DatabaseDescriptor.getWriteRpcTimeout(MILLISECONDS) * 2) * 1000;
UUID uuid = UUIDGen.getTimeUUID();
// Add a batch with 10 mutations
@@ -465,14 +345,14 @@
@Test
public void testReplayWithNoPeers() throws Exception
{
- StorageService.instance.getTokenMetadata().removeEndpoint(InetAddress.getByName("127.0.0.1"));
+ StorageService.instance.getTokenMetadata().removeEndpoint(InetAddressAndPort.getByName("127.0.0.1"));
long initialAllBatches = BatchlogManager.instance.countAllBatches();
long initialReplayedBatches = BatchlogManager.instance.getTotalBatchesReplayed();
- CFMetaData cfm = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD1).metadata;
+ TableMetadata cfm = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD1).metadata();
- long timestamp = (System.currentTimeMillis() - DatabaseDescriptor.getWriteRpcTimeout() * 2) * 1000;
+ long timestamp = (System.currentTimeMillis() - DatabaseDescriptor.getWriteRpcTimeout(MILLISECONDS) * 2) * 1000;
UUID uuid = UUIDGen.getTimeUUID();
// Add a batch with 10 mutations
diff --git a/test/unit/org/apache/cassandra/batchlog/BatchlogTest.java b/test/unit/org/apache/cassandra/batchlog/BatchlogTest.java
new file mode 100644
index 0000000..8fa4afc
--- /dev/null
+++ b/test/unit/org/apache/cassandra/batchlog/BatchlogTest.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.batchlog;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.UUID;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.Mutation;
+import org.apache.cassandra.db.RowUpdateBuilder;
+import org.apache.cassandra.db.marshal.BytesType;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.io.util.DataInputBuffer;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputBuffer;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.UUIDGen;
+
+import static org.apache.cassandra.utils.ByteBufferUtil.bytes;
+import static org.junit.Assert.assertEquals;
+
+public class BatchlogTest
+{
+ private static final String KEYSPACE = "BatchRequestTest";
+ private static final String CF_STANDARD = "Standard";
+
+ @BeforeClass
+ public static void defineSchema() throws ConfigurationException
+ {
+ SchemaLoader.prepareServer();
+ SchemaLoader.createKeyspace(KEYSPACE,
+ KeyspaceParams.simple(1),
+ SchemaLoader.standardCFMD(KEYSPACE, CF_STANDARD, 1, BytesType.instance));
+ }
+
+ @Test
+ public void testSerialization() throws IOException
+ {
+ TableMetadata cfm = Keyspace.open(KEYSPACE).getColumnFamilyStore(CF_STANDARD).metadata();
+
+ long now = FBUtilities.timestampMicros();
+ int version = MessagingService.current_version;
+ UUID uuid = UUIDGen.getTimeUUID();
+
+ List<Mutation> mutations = new ArrayList<>(10);
+ for (int i = 0; i < 10; i++)
+ {
+ mutations.add(new RowUpdateBuilder(cfm, FBUtilities.timestampMicros(), bytes(i))
+ .clustering("name" + i)
+ .add("val", "val" + i)
+ .build());
+ }
+
+ Batch batch1 = Batch.createLocal(uuid, now, mutations);
+ assertEquals(uuid, batch1.id);
+ assertEquals(now, batch1.creationTime);
+ assertEquals(mutations, batch1.decodedMutations);
+
+ DataOutputBuffer out = new DataOutputBuffer();
+ Batch.serializer.serialize(batch1, out, version);
+
+ assertEquals(out.getLength(), Batch.serializer.serializedSize(batch1, version));
+
+ DataInputPlus dis = new DataInputBuffer(out.getData());
+ Batch batch2 = Batch.serializer.deserialize(dis, version);
+
+ assertEquals(batch1.id, batch2.id);
+ assertEquals(batch1.creationTime, batch2.creationTime);
+ assertEquals(batch1.decodedMutations.size(), batch2.encodedMutations.size());
+
+ Iterator<Mutation> it1 = batch1.decodedMutations.iterator();
+ Iterator<ByteBuffer> it2 = batch2.encodedMutations.iterator();
+ while (it1.hasNext())
+ {
+ try (DataInputBuffer in = new DataInputBuffer(it2.next().array()))
+ {
+ assertEquals(it1.next().toString(), Mutation.serializer.deserialize(in, version).toString());
+ }
+ }
+ }
+}
diff --git a/test/unit/org/apache/cassandra/cache/AutoSavingCacheTest.java b/test/unit/org/apache/cassandra/cache/AutoSavingCacheTest.java
index c952470..bb5129a 100644
--- a/test/unit/org/apache/cassandra/cache/AutoSavingCacheTest.java
+++ b/test/unit/org/apache/cassandra/cache/AutoSavingCacheTest.java
@@ -17,8 +17,8 @@
*/
package org.apache.cassandra.cache;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.marshal.AsciiType;
@@ -45,10 +45,9 @@
SchemaLoader.prepareServer();
SchemaLoader.createKeyspace(KEYSPACE1,
KeyspaceParams.simple(1),
- CFMetaData.Builder.create(KEYSPACE1, CF_STANDARD1)
- .addPartitionKey("pKey", AsciiType.instance)
- .addRegularColumn("col1", AsciiType.instance)
- .build());
+ TableMetadata.builder(KEYSPACE1, CF_STANDARD1)
+ .addPartitionKeyColumn("pKey", AsciiType.instance)
+ .addRegularColumn("col1", AsciiType.instance));
}
@Test
@@ -71,8 +70,8 @@
cfs.truncateBlocking();
for (int i = 0; i < 2; i++)
{
- ColumnDefinition colDef = ColumnDefinition.regularDef(cfs.metadata, ByteBufferUtil.bytes("col1"), AsciiType.instance);
- RowUpdateBuilder rowBuilder = new RowUpdateBuilder(cfs.metadata, System.currentTimeMillis(), "key1");
+ ColumnMetadata colDef = ColumnMetadata.regularColumn(cfs.metadata(), ByteBufferUtil.bytes("col1"), AsciiType.instance);
+ RowUpdateBuilder rowBuilder = new RowUpdateBuilder(cfs.metadata(), System.currentTimeMillis(), "key1");
rowBuilder.add(colDef, "val1");
rowBuilder.build().apply();
cfs.forceBlockingFlush();
@@ -95,6 +94,6 @@
// then load saved
keyCache.loadSavedAsync().get();
for (SSTableReader sstable : cfs.getLiveSSTables())
- Assert.assertNotNull(keyCache.get(new KeyCacheKey(cfs.metadata.ksAndCFName, sstable.descriptor, ByteBufferUtil.bytes("key1"))));
+ Assert.assertNotNull(keyCache.get(new KeyCacheKey(cfs.metadata(), sstable.descriptor, ByteBufferUtil.bytes("key1"))));
}
}
diff --git a/test/unit/org/apache/cassandra/cache/CacheProviderTest.java b/test/unit/org/apache/cassandra/cache/CacheProviderTest.java
index eca124f..7ed8a60 100644
--- a/test/unit/org/apache/cassandra/cache/CacheProviderTest.java
+++ b/test/unit/org/apache/cassandra/cache/CacheProviderTest.java
@@ -19,34 +19,38 @@
package org.apache.cassandra.cache;
import java.nio.ByteBuffer;
-import java.security.MessageDigest;
-import java.security.NoSuchAlgorithmException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
import org.junit.BeforeClass;
import org.junit.Test;
-import static org.junit.Assert.*;
-
-import java.util.ArrayList;
-import java.util.List;
-
-
+import com.github.benmanes.caffeine.cache.Weigher;
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.concurrent.NamedThreadFactory;
-import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.utils.Pair;
-
-import com.googlecode.concurrentlinkedhashmap.Weighers;
-
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.db.*;
-import org.apache.cassandra.db.rows.*;
+import org.apache.cassandra.db.Digest;
+import org.apache.cassandra.db.RowUpdateBuilder;
import org.apache.cassandra.db.marshal.AsciiType;
-import org.apache.cassandra.db.partitions.*;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.db.partitions.CachedBTreePartition;
+import org.apache.cassandra.db.partitions.PartitionUpdate;
+import org.apache.cassandra.db.rows.UnfilteredRowIterators;
+import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.schema.IndexMetadata;
+import org.apache.cassandra.schema.Indexes;
import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.utils.FBUtilities;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNotSame;
+import static org.junit.Assert.assertTrue;
+
public class CacheProviderTest
{
MeasureableString key1 = new MeasureableString("key1");
@@ -58,20 +62,20 @@
private static final String KEYSPACE1 = "CacheProviderTest1";
private static final String CF_STANDARD1 = "Standard1";
- private static CFMetaData cfm;
+ private static TableMetadata cfm;
@BeforeClass
public static void defineSchema() throws ConfigurationException
{
SchemaLoader.prepareServer();
- cfm = CFMetaData.Builder.create(KEYSPACE1, CF_STANDARD1)
- .addPartitionKey("pKey", AsciiType.instance)
- .addRegularColumn("col1", AsciiType.instance)
- .build();
- SchemaLoader.createKeyspace(KEYSPACE1,
- KeyspaceParams.simple(1),
- cfm);
+ cfm =
+ TableMetadata.builder(KEYSPACE1, CF_STANDARD1)
+ .addPartitionKeyColumn("pKey", AsciiType.instance)
+ .addRegularColumn("col1", AsciiType.instance)
+ .build();
+
+ SchemaLoader.createKeyspace(KEYSPACE1, KeyspaceParams.simple(1), cfm);
}
private CachedBTreePartition createPartition()
@@ -100,18 +104,11 @@
private void assertDigests(IRowCacheEntry one, CachedBTreePartition two)
{
assertTrue(one instanceof CachedBTreePartition);
- try
- {
- MessageDigest d1 = MessageDigest.getInstance("MD5");
- MessageDigest d2 = MessageDigest.getInstance("MD5");
- UnfilteredRowIterators.digest(null, ((CachedBTreePartition) one).unfilteredIterator(), d1, MessagingService.current_version);
- UnfilteredRowIterators.digest(null, ((CachedBTreePartition) two).unfilteredIterator(), d2, MessagingService.current_version);
- assertTrue(MessageDigest.isEqual(d1.digest(), d2.digest()));
- }
- catch (NoSuchAlgorithmException e)
- {
- throw new RuntimeException(e);
- }
+ Digest d1 = Digest.forReadResponse();
+ Digest d2 = Digest.forReadResponse();
+ UnfilteredRowIterators.digest(((CachedBTreePartition) one).unfilteredIterator(), d1, MessagingService.current_version);
+ UnfilteredRowIterators.digest(((CachedBTreePartition) two).unfilteredIterator(), d2, MessagingService.current_version);
+ assertArrayEquals(d1.digest(), d2.digest());
}
private void concurrentCase(final CachedBTreePartition partition, final ICache<MeasureableString, IRowCacheEntry> cache) throws InterruptedException
@@ -147,7 +144,8 @@
@Test
public void testSerializingCache() throws InterruptedException
{
- ICache<MeasureableString, IRowCacheEntry> cache = SerializingCache.create(CAPACITY, Weighers.<RefCountedMemory>singleton(), new SerializingCacheProvider.RowCacheSerializer());
+ ICache<MeasureableString, IRowCacheEntry> cache = SerializingCache.create(CAPACITY,
+ Weigher.singletonWeigher(), new SerializingCacheProvider.RowCacheSerializer());
CachedBTreePartition partition = createPartition();
simpleCase(partition, cache);
concurrentCase(partition, cache);
@@ -156,16 +154,44 @@
@Test
public void testKeys()
{
- Pair<String, String> ksAndCFName = Pair.create(KEYSPACE1, CF_STANDARD1);
+ TableId id1 = TableId.generate();
byte[] b1 = {1, 2, 3, 4};
- RowCacheKey key1 = new RowCacheKey(ksAndCFName, ByteBuffer.wrap(b1));
+ RowCacheKey key1 = new RowCacheKey(id1, null, ByteBuffer.wrap(b1));
+ TableId id2 = TableId.fromString(id1.toString());
byte[] b2 = {1, 2, 3, 4};
- RowCacheKey key2 = new RowCacheKey(ksAndCFName, ByteBuffer.wrap(b2));
+ RowCacheKey key2 = new RowCacheKey(id2, null, ByteBuffer.wrap(b2));
assertEquals(key1, key2);
assertEquals(key1.hashCode(), key2.hashCode());
+ TableMetadata tm = TableMetadata.builder("ks", "tab", id1)
+ .addPartitionKeyColumn("pk", UTF8Type.instance)
+ .build();
+
+ assertTrue(key1.sameTable(tm));
+
byte[] b3 = {1, 2, 3, 5};
- RowCacheKey key3 = new RowCacheKey(ksAndCFName, ByteBuffer.wrap(b3));
+ RowCacheKey key3 = new RowCacheKey(id1, null, ByteBuffer.wrap(b3));
+ assertNotSame(key1, key3);
+ assertNotSame(key1.hashCode(), key3.hashCode());
+
+ // with index name
+
+ key1 = new RowCacheKey(id1, "indexFoo", ByteBuffer.wrap(b1));
+ assertNotSame(key1, key2);
+ assertNotSame(key1.hashCode(), key2.hashCode());
+
+ key2 = new RowCacheKey(id2, "indexFoo", ByteBuffer.wrap(b2));
+ assertEquals(key1, key2);
+ assertEquals(key1.hashCode(), key2.hashCode());
+
+ tm = TableMetadata.builder("ks", "tab.indexFoo", id1)
+ .kind(TableMetadata.Kind.INDEX)
+ .addPartitionKeyColumn("pk", UTF8Type.instance)
+ .indexes(Indexes.of(IndexMetadata.fromSchemaMetadata("indexFoo", IndexMetadata.Kind.KEYS, Collections.emptyMap())))
+ .build();
+ assertTrue(key1.sameTable(tm));
+
+ key3 = new RowCacheKey(id1, "indexFoo", ByteBuffer.wrap(b3));
assertNotSame(key1, key3);
assertNotSame(key1.hashCode(), key3.hashCode());
}
diff --git a/test/unit/org/apache/cassandra/client/TestRingCache.java b/test/unit/org/apache/cassandra/client/TestRingCache.java
deleted file mode 100644
index 51bf566..0000000
--- a/test/unit/org/apache/cassandra/client/TestRingCache.java
+++ /dev/null
@@ -1,121 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.client;
-
-import java.net.InetAddress;
-import java.nio.ByteBuffer;
-import java.util.Collection;
-
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.hadoop.ConfigHelper;
-import org.apache.cassandra.thrift.Cassandra;
-import org.apache.cassandra.thrift.Column;
-import org.apache.cassandra.thrift.ColumnParent;
-import org.apache.cassandra.thrift.ColumnPath;
-import org.apache.cassandra.thrift.ConsistencyLevel;
-import org.apache.cassandra.utils.ByteBufferUtil;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.thrift.protocol.TBinaryProtocol;
-import org.apache.thrift.transport.TFramedTransport;
-import org.apache.thrift.transport.TSocket;
-
-
-/**
- * Sample code that uses RingCache in the client.
- */
-public class TestRingCache
-{
- private RingCache ringCache;
- private Cassandra.Client thriftClient;
- private Configuration conf;
-
- public TestRingCache(String keyspace)
- {
- ConfigHelper.setOutputColumnFamily(conf, keyspace, "Standard1");
- ringCache = new RingCache(conf);
- }
-
- private void setup(String server, int port) throws Exception
- {
- /* Establish a thrift connection to the cassandra instance */
- TSocket socket = new TSocket(server, port);
- System.out.println(" connected to " + server + ":" + port + ".");
- TBinaryProtocol binaryProtocol = new TBinaryProtocol(new TFramedTransport(socket));
- Cassandra.Client cassandraClient = new Cassandra.Client(binaryProtocol);
- socket.open();
- thriftClient = cassandraClient;
- String seed = DatabaseDescriptor.getSeeds().iterator().next().getHostAddress();
- conf = new Configuration();
- ConfigHelper.setOutputPartitioner(conf, DatabaseDescriptor.getPartitioner().getClass().getName());
- ConfigHelper.setOutputInitialAddress(conf, seed);
- ConfigHelper.setOutputRpcPort(conf, Integer.toString(DatabaseDescriptor.getRpcPort()));
-
- }
-
- /**
- * usage: java -cp <configpath> org.apache.cassandra.client.TestRingCache [keyspace row-id-prefix row-id-int]
- * to test a single keyspace/row, use the parameters. row-id-prefix and row-id-int are appended together to form a
- * single row id. If you supply now parameters, 'Keyspace1' is assumed and will check 9 rows ('row1' through 'row9').
- * @param args
- * @throws Exception
- */
- public static void main(String[] args) throws Throwable
- {
- int minRow;
- int maxRow;
- String rowPrefix, keyspace = "Keyspace1";
-
- if (args.length > 0)
- {
- keyspace = args[0];
- rowPrefix = args[1];
- minRow = Integer.parseInt(args[2]);
- maxRow = minRow + 1;
- }
- else
- {
- minRow = 1;
- maxRow = 10;
- rowPrefix = "row";
- }
-
- TestRingCache tester = new TestRingCache(keyspace);
-
- for (int nRows = minRow; nRows < maxRow; nRows++)
- {
- ByteBuffer row = ByteBufferUtil.bytes((rowPrefix + nRows));
- ColumnPath col = new ColumnPath("Standard1").setSuper_column((ByteBuffer)null).setColumn("col1".getBytes());
- ColumnParent parent = new ColumnParent("Standard1").setSuper_column((ByteBuffer)null);
-
- Collection<InetAddress> endpoints = tester.ringCache.getEndpoint(row);
- InetAddress firstEndpoint = endpoints.iterator().next();
- System.out.printf("hosts with key %s : %s; choose %s%n",
- new String(row.array()), StringUtils.join(endpoints, ","), firstEndpoint);
-
- // now, read the row back directly from the host owning the row locally
- tester.setup(firstEndpoint.getHostAddress(), DatabaseDescriptor.getRpcPort());
- tester.thriftClient.set_keyspace(keyspace);
- tester.thriftClient.insert(row, parent, new Column(ByteBufferUtil.bytes("col1")).setValue(ByteBufferUtil.bytes("val1")).setTimestamp(1), ConsistencyLevel.ONE);
- Column column = tester.thriftClient.get(row, col, ConsistencyLevel.ONE).column;
- System.out.println("read row " + new String(row.array()) + " " + new String(column.name.array()) + ":" + new String(column.value.array()) + ":" + column.timestamp);
- }
-
- System.exit(1);
- }
-}
diff --git a/test/unit/org/apache/cassandra/concurrent/DebuggableScheduledThreadPoolExecutorTest.java b/test/unit/org/apache/cassandra/concurrent/DebuggableScheduledThreadPoolExecutorTest.java
index bf78d65..1aac470 100644
--- a/test/unit/org/apache/cassandra/concurrent/DebuggableScheduledThreadPoolExecutorTest.java
+++ b/test/unit/org/apache/cassandra/concurrent/DebuggableScheduledThreadPoolExecutorTest.java
@@ -29,7 +29,7 @@
import org.junit.BeforeClass;
import org.junit.Test;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.cassandra.service.EmbeddedCassandraService;
import org.apache.cassandra.service.StorageService;
diff --git a/test/unit/org/apache/cassandra/concurrent/DebuggableThreadPoolExecutorTest.java b/test/unit/org/apache/cassandra/concurrent/DebuggableThreadPoolExecutorTest.java
index 9276248..58200c9 100644
--- a/test/unit/org/apache/cassandra/concurrent/DebuggableThreadPoolExecutorTest.java
+++ b/test/unit/org/apache/cassandra/concurrent/DebuggableThreadPoolExecutorTest.java
@@ -21,13 +21,25 @@
*/
+import java.util.UUID;
+import java.util.concurrent.CountDownLatch;
import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.RunnableFuture;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import com.google.common.base.Throwables;
+import com.google.common.net.InetAddresses;
+import com.google.common.util.concurrent.ListenableFutureTask;
+import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.tracing.TraceState;
+import org.apache.cassandra.tracing.TraceStateImpl;
+import org.apache.cassandra.tracing.Tracing;
import org.apache.cassandra.utils.WrappedRunnable;
public class DebuggableThreadPoolExecutorTest
@@ -65,4 +77,153 @@
long delta = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start);
assert delta >= 9 * 50 : delta;
}
+
+ @Test
+ public void testExecuteFutureTaskWhileTracing()
+ {
+ LinkedBlockingQueue<Runnable> q = new LinkedBlockingQueue<Runnable>(1);
+ DebuggableThreadPoolExecutor executor = new DebuggableThreadPoolExecutor(1,
+ Integer.MAX_VALUE,
+ TimeUnit.MILLISECONDS,
+ q,
+ new NamedThreadFactory("TEST"));
+ Runnable test = () -> executor.execute(failingTask());
+ try
+ {
+ // make sure the non-tracing case works
+ Throwable cause = catchUncaughtExceptions(test);
+ Assert.assertEquals(DebuggingThrowsException.class, cause.getClass());
+
+ // tracing should have the same semantics
+ cause = catchUncaughtExceptions(() -> withTracing(test));
+ Assert.assertEquals(DebuggingThrowsException.class, cause.getClass());
+ }
+ finally
+ {
+ executor.shutdown();
+ }
+ }
+
+ @Test
+ public void testSubmitFutureTaskWhileTracing()
+ {
+ LinkedBlockingQueue<Runnable> q = new LinkedBlockingQueue<Runnable>(1);
+ DebuggableThreadPoolExecutor executor = new DebuggableThreadPoolExecutor(1,
+ Integer.MAX_VALUE,
+ TimeUnit.MILLISECONDS,
+ q,
+ new NamedThreadFactory("TEST"));
+ FailingRunnable test = () -> executor.submit(failingTask()).get();
+ try
+ {
+ // make sure the non-tracing case works
+ Throwable cause = catchUncaughtExceptions(test);
+ Assert.assertEquals(DebuggingThrowsException.class, cause.getClass());
+
+ // tracing should have the same semantics
+ cause = catchUncaughtExceptions(() -> withTracing(test));
+ Assert.assertEquals(DebuggingThrowsException.class, cause.getClass());
+ }
+ finally
+ {
+ executor.shutdown();
+ }
+ }
+
+ @Test
+ public void testSubmitWithResultFutureTaskWhileTracing()
+ {
+ LinkedBlockingQueue<Runnable> q = new LinkedBlockingQueue<Runnable>(1);
+ DebuggableThreadPoolExecutor executor = new DebuggableThreadPoolExecutor(1,
+ Integer.MAX_VALUE,
+ TimeUnit.MILLISECONDS,
+ q,
+ new NamedThreadFactory("TEST"));
+ FailingRunnable test = () -> executor.submit(failingTask(), 42).get();
+ try
+ {
+ Throwable cause = catchUncaughtExceptions(test);
+ Assert.assertEquals(DebuggingThrowsException.class, cause.getClass());
+ cause = catchUncaughtExceptions(() -> withTracing(test));
+ Assert.assertEquals(DebuggingThrowsException.class, cause.getClass());
+ }
+ finally
+ {
+ executor.shutdown();
+ }
+ }
+
+ private static void withTracing(Runnable fn)
+ {
+ TraceState state = Tracing.instance.get();
+ try {
+ Tracing.instance.set(new TraceStateImpl(InetAddressAndPort.getByAddress(InetAddresses.forString("127.0.0.1")), UUID.randomUUID(), Tracing.TraceType.NONE));
+ fn.run();
+ }
+ finally
+ {
+ Tracing.instance.set(state);
+ }
+ }
+
+ private static Throwable catchUncaughtExceptions(Runnable fn)
+ {
+ Thread.UncaughtExceptionHandler defaultHandler = Thread.getDefaultUncaughtExceptionHandler();
+ try
+ {
+ AtomicReference<Throwable> ref = new AtomicReference<>(null);
+ CountDownLatch latch = new CountDownLatch(1);
+ Thread.setDefaultUncaughtExceptionHandler((thread, cause) -> {
+ ref.set(cause);
+ latch.countDown();
+ });
+ fn.run();
+ try
+ {
+ latch.await(30, TimeUnit.SECONDS);
+ }
+ catch (InterruptedException e)
+ {
+ throw new AssertionError(e);
+ }
+ return ref.get();
+ }
+ finally
+ {
+ Thread.setDefaultUncaughtExceptionHandler(defaultHandler);
+ }
+ }
+
+ private static String failingFunction()
+ {
+ throw new DebuggingThrowsException();
+ }
+
+ private static RunnableFuture<String> failingTask()
+ {
+ return ListenableFutureTask.create(DebuggableThreadPoolExecutorTest::failingFunction);
+ }
+
+ private static final class DebuggingThrowsException extends RuntimeException {
+
+ }
+
+ // REVIEWER : I know this is the same as WrappedRunnable, but that doesn't support lambda...
+ private interface FailingRunnable extends Runnable
+ {
+ void doRun() throws Throwable;
+
+ default void run()
+ {
+ try
+ {
+ doRun();
+ }
+ catch (Throwable t)
+ {
+ Throwables.throwIfUnchecked(t);
+ throw new RuntimeException(t);
+ }
+ }
+ }
}
diff --git a/test/unit/org/apache/cassandra/concurrent/SEPExecutorTest.java b/test/unit/org/apache/cassandra/concurrent/SEPExecutorTest.java
index 9702e8f..9a2d52d 100644
--- a/test/unit/org/apache/cassandra/concurrent/SEPExecutorTest.java
+++ b/test/unit/org/apache/cassandra/concurrent/SEPExecutorTest.java
@@ -22,24 +22,23 @@
import java.io.PrintStream;
import java.util.Arrays;
import java.util.concurrent.ExecutorService;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
import org.junit.Assert;
-import org.junit.BeforeClass;
import org.junit.Test;
-import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.utils.FBUtilities;
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+import static java.util.concurrent.TimeUnit.MINUTES;
public class SEPExecutorTest
{
- @BeforeClass
- public static void beforeClass()
- {
- DatabaseDescriptor.daemonInitialization();
- }
-
@Test
public void shutdownTest() throws Throwable
{
@@ -77,4 +76,126 @@
}
}
}
+
+ @Test
+ public void changingMaxWorkersMeetsConcurrencyGoalsTest() throws InterruptedException, TimeoutException
+ {
+ // Number of busy worker threads to run and gum things up. Chosen to be
+ // between the low and high max pool size so the test exercises resizing
+ // under a number of different conditions.
+ final int numBusyWorkers = 2;
+ SharedExecutorPool sharedPool = new SharedExecutorPool("ChangingMaxWorkersMeetsConcurrencyGoalsTest");
+ final AtomicInteger notifiedMaxPoolSize = new AtomicInteger();
+
+ LocalAwareExecutorService executor = sharedPool.newExecutor(0, notifiedMaxPoolSize::set, 4, "internal", "resizetest");
+
+ // Keep feeding the executor work while resizing
+ // so it stays under load.
+ AtomicBoolean stayBusy = new AtomicBoolean(true);
+ Semaphore busyWorkerPermits = new Semaphore(numBusyWorkers);
+ Thread makeBusy = new Thread(() -> {
+ while (stayBusy.get() == true)
+ {
+ try
+ {
+ if (busyWorkerPermits.tryAcquire(1, MILLISECONDS)) {
+ executor.execute(new BusyWork(busyWorkerPermits));
+ }
+ }
+ catch (InterruptedException e)
+ {
+ // ignore, will either stop looping if done or retry the lock
+ }
+ }
+ });
+
+ makeBusy.start();
+ try
+ {
+ for (int repeat = 0; repeat < 1000; repeat++)
+ {
+ assertMaxTaskConcurrency(executor, 1);
+ Assert.assertEquals(1, notifiedMaxPoolSize.get());
+
+ assertMaxTaskConcurrency(executor, 2);
+ Assert.assertEquals(2, notifiedMaxPoolSize.get());
+
+ assertMaxTaskConcurrency(executor, 1);
+ Assert.assertEquals(1, notifiedMaxPoolSize.get());
+
+ assertMaxTaskConcurrency(executor, 3);
+ Assert.assertEquals(3, notifiedMaxPoolSize.get());
+
+ executor.setMaximumPoolSize(0);
+ Assert.assertEquals(0, notifiedMaxPoolSize.get());
+
+ assertMaxTaskConcurrency(executor, 4);
+ Assert.assertEquals(4, notifiedMaxPoolSize.get());
+ }
+ }
+ finally
+ {
+ stayBusy.set(false);
+ makeBusy.join(TimeUnit.SECONDS.toMillis(5));
+ Assert.assertFalse("makeBusy thread should have checked stayBusy and exited",
+ makeBusy.isAlive());
+ sharedPool.shutdownAndWait(1L, MINUTES);
+ }
+ }
+
+ static class LatchWaiter implements Runnable
+ {
+ CountDownLatch latch;
+ long timeout;
+ TimeUnit unit;
+
+ public LatchWaiter(CountDownLatch latch, long timeout, TimeUnit unit)
+ {
+ this.latch = latch;
+ this.timeout = timeout;
+ this.unit = unit;
+ }
+
+ public void run()
+ {
+ latch.countDown();
+ try
+ {
+ latch.await(timeout, unit); // block until all the latch waiters have run, now at desired concurrency
+ }
+ catch (InterruptedException e)
+ {
+ Assert.fail("interrupted: " + e);
+ }
+ }
+ }
+
+ static class BusyWork implements Runnable
+ {
+ private Semaphore busyWorkers;
+
+ public BusyWork(Semaphore busyWorkers)
+ {
+ this.busyWorkers = busyWorkers;
+ }
+
+ public void run()
+ {
+ busyWorkers.release();
+ }
+ }
+
+ void assertMaxTaskConcurrency(LocalAwareExecutorService executor, int concurrency) throws InterruptedException
+ {
+ executor.setMaximumPoolSize(concurrency);
+
+ CountDownLatch concurrencyGoal = new CountDownLatch(concurrency);
+ for (int i = 0; i < concurrency; i++)
+ {
+ executor.execute(new LatchWaiter(concurrencyGoal, 5L, TimeUnit.SECONDS));
+ }
+ // Will return true if all of the LatchWaiters count down before the timeout
+ Assert.assertEquals("Test tasks did not hit max concurrency goal",
+ true, concurrencyGoal.await(3L, TimeUnit.SECONDS));
+ }
}
diff --git a/test/unit/org/apache/cassandra/config/CFMetaDataTest.java b/test/unit/org/apache/cassandra/config/CFMetaDataTest.java
deleted file mode 100644
index 78b372e..0000000
--- a/test/unit/org/apache/cassandra/config/CFMetaDataTest.java
+++ /dev/null
@@ -1,265 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.cassandra.config;
-
-import java.util.*;
-
-import org.apache.cassandra.SchemaLoader;
-import org.apache.cassandra.cql3.QueryProcessor;
-import org.apache.cassandra.cql3.UntypedResultSet;
-import org.apache.cassandra.db.ColumnFamilyStore;
-import org.apache.cassandra.db.Keyspace;
-import org.apache.cassandra.db.Mutation;
-import org.apache.cassandra.db.marshal.*;
-import org.apache.cassandra.db.partitions.PartitionUpdate;
-import org.apache.cassandra.db.rows.UnfilteredRowIterators;
-import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.schema.*;
-import org.apache.cassandra.thrift.CfDef;
-import org.apache.cassandra.thrift.ColumnDef;
-import org.apache.cassandra.thrift.IndexType;
-import org.apache.cassandra.thrift.ThriftConversion;
-import org.apache.cassandra.utils.ByteBufferUtil;
-import org.apache.cassandra.utils.FBUtilities;
-
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
-public class CFMetaDataTest
-{
- private static final String KEYSPACE1 = "CFMetaDataTest1";
- private static final String CF_STANDARD1 = "Standard1";
-
- private static List<ColumnDef> columnDefs = new ArrayList<ColumnDef>();
-
- static
- {
- columnDefs.add(new ColumnDef(ByteBufferUtil.bytes("col1"), AsciiType.class.getCanonicalName())
- .setIndex_name("col1Index")
- .setIndex_type(IndexType.KEYS));
-
- columnDefs.add(new ColumnDef(ByteBufferUtil.bytes("col2"), UTF8Type.class.getCanonicalName())
- .setIndex_name("col2Index")
- .setIndex_type(IndexType.KEYS));
-
- Map<String, String> customIndexOptions = new HashMap<>();
- customIndexOptions.put("option1", "value1");
- customIndexOptions.put("option2", "value2");
- columnDefs.add(new ColumnDef(ByteBufferUtil.bytes("col3"), Int32Type.class.getCanonicalName())
- .setIndex_name("col3Index")
- .setIndex_type(IndexType.CUSTOM)
- .setIndex_options(customIndexOptions));
- }
-
- @BeforeClass
- public static void defineSchema() throws ConfigurationException
- {
- SchemaLoader.prepareServer();
- SchemaLoader.createKeyspace(KEYSPACE1,
- KeyspaceParams.simple(1),
- SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD1));
- }
-
- @Test
- public void testThriftConversion() throws Exception
- {
- CfDef cfDef = new CfDef().setDefault_validation_class(AsciiType.class.getCanonicalName())
- .setComment("Test comment")
- .setColumn_metadata(columnDefs)
- .setKeyspace(KEYSPACE1)
- .setName(CF_STANDARD1);
-
- // convert Thrift to CFMetaData
- CFMetaData cfMetaData = ThriftConversion.fromThrift(cfDef);
-
- CfDef thriftCfDef = new CfDef();
- thriftCfDef.keyspace = KEYSPACE1;
- thriftCfDef.name = CF_STANDARD1;
- thriftCfDef.default_validation_class = cfDef.default_validation_class;
- thriftCfDef.comment = cfDef.comment;
- thriftCfDef.column_metadata = new ArrayList<>();
- for (ColumnDef columnDef : columnDefs)
- {
- ColumnDef c = new ColumnDef();
- c.name = ByteBufferUtil.clone(columnDef.name);
- c.validation_class = columnDef.getValidation_class();
- c.index_name = columnDef.getIndex_name();
- c.index_type = columnDef.getIndex_type();
- if (columnDef.isSetIndex_options())
- c.setIndex_options(columnDef.getIndex_options());
- thriftCfDef.column_metadata.add(c);
- }
-
- CfDef converted = ThriftConversion.toThrift(cfMetaData);
-
- assertEquals(thriftCfDef.keyspace, converted.keyspace);
- assertEquals(thriftCfDef.name, converted.name);
- assertEquals(thriftCfDef.default_validation_class, converted.default_validation_class);
- assertEquals(thriftCfDef.comment, converted.comment);
- assertEquals(new HashSet<>(thriftCfDef.column_metadata), new HashSet<>(converted.column_metadata));
- }
-
- @Test
- public void testConversionsInverses() throws Exception
- {
- for (String keyspaceName : Schema.instance.getNonSystemKeyspaces())
- {
- for (ColumnFamilyStore cfs : Keyspace.open(keyspaceName).getColumnFamilyStores())
- {
- CFMetaData cfm = cfs.metadata;
- if (!cfm.isThriftCompatible())
- continue;
-
- checkInverses(cfm);
-
- // Testing with compression to catch #3558
- CFMetaData withCompression = cfm.copy();
- withCompression.compression(CompressionParams.snappy(32768));
- checkInverses(withCompression);
- }
- }
- }
-
- private void checkInverses(CFMetaData cfm) throws Exception
- {
- KeyspaceMetadata keyspace = Schema.instance.getKSMetaData(cfm.ksName);
-
- // Test thrift conversion
- CFMetaData before = cfm;
- CFMetaData after = ThriftConversion.fromThriftForUpdate(ThriftConversion.toThrift(before), before);
- assert before.equals(after) : String.format("%n%s%n!=%n%s", before, after);
-
- // Test schema conversion
- Mutation rm = SchemaKeyspace.makeCreateTableMutation(keyspace, cfm, FBUtilities.timestampMicros()).build();
- PartitionUpdate cfU = rm.getPartitionUpdate(Schema.instance.getId(SchemaConstants.SCHEMA_KEYSPACE_NAME, SchemaKeyspace.TABLES));
- PartitionUpdate cdU = rm.getPartitionUpdate(Schema.instance.getId(SchemaConstants.SCHEMA_KEYSPACE_NAME, SchemaKeyspace.COLUMNS));
-
- UntypedResultSet.Row tableRow = QueryProcessor.resultify(String.format("SELECT * FROM %s.%s", SchemaConstants.SCHEMA_KEYSPACE_NAME, SchemaKeyspace.TABLES),
- UnfilteredRowIterators.filter(cfU.unfilteredIterator(), FBUtilities.nowInSeconds()))
- .one();
- TableParams params = SchemaKeyspace.createTableParamsFromRow(tableRow);
-
- UntypedResultSet columnsRows = QueryProcessor.resultify(String.format("SELECT * FROM %s.%s", SchemaConstants.SCHEMA_KEYSPACE_NAME, SchemaKeyspace.COLUMNS),
- UnfilteredRowIterators.filter(cdU.unfilteredIterator(), FBUtilities.nowInSeconds()));
- Set<ColumnDefinition> columns = new HashSet<>();
- for (UntypedResultSet.Row row : columnsRows)
- columns.add(SchemaKeyspace.createColumnFromRow(row, Types.none()));
-
- assertEquals(cfm.params, params);
- assertEquals(new HashSet<>(cfm.allColumns()), columns);
- }
-
- @Test
- public void testIsNameValidPositive()
- {
- assertTrue(CFMetaData.isNameValid("abcdefghijklmnopqrstuvwxyz"));
- assertTrue(CFMetaData.isNameValid("ABCDEFGHIJKLMNOPQRSTUVWXYZ"));
- assertTrue(CFMetaData.isNameValid("_01234567890"));
- }
-
- @Test
- public void testIsNameValidNegative()
- {
- assertFalse(CFMetaData.isNameValid(null));
- assertFalse(CFMetaData.isNameValid(""));
- assertFalse(CFMetaData.isNameValid(" "));
- assertFalse(CFMetaData.isNameValid("@"));
- assertFalse(CFMetaData.isNameValid("!"));
- }
-
- private static Set<String> primitiveTypes = new HashSet<String>(Arrays.asList(new String[] { "ascii", "bigint", "blob", "boolean", "date",
- "duration", "decimal", "double", "float",
- "inet", "int", "smallint", "text", "time",
- "timestamp", "timeuuid", "tinyint", "uuid",
- "varchar", "varint" }));
-
- @Test
- public void typeCompatibilityTest() throws Throwable
- {
- Map<String, Set<String>> compatibilityMap = new HashMap<>();
- compatibilityMap.put("bigint", new HashSet<>(Arrays.asList(new String[] {"timestamp"})));
- compatibilityMap.put("blob", new HashSet<>(Arrays.asList(new String[] {"ascii", "bigint", "boolean", "date", "decimal", "double", "duration",
- "float", "inet", "int", "smallint", "text", "time", "timestamp",
- "timeuuid", "tinyint", "uuid", "varchar", "varint"})));
- compatibilityMap.put("date", new HashSet<>(Arrays.asList(new String[] {"int"})));
- compatibilityMap.put("time", new HashSet<>(Arrays.asList(new String[] {"bigint"})));
- compatibilityMap.put("text", new HashSet<>(Arrays.asList(new String[] {"ascii", "varchar"})));
- compatibilityMap.put("timestamp", new HashSet<>(Arrays.asList(new String[] {"bigint"})));
- compatibilityMap.put("varchar", new HashSet<>(Arrays.asList(new String[] {"ascii", "text"})));
- compatibilityMap.put("varint", new HashSet<>(Arrays.asList(new String[] {"bigint", "int", "timestamp"})));
- compatibilityMap.put("uuid", new HashSet<>(Arrays.asList(new String[] {"timeuuid"})));
-
- for (String sourceTypeString: primitiveTypes)
- {
- AbstractType sourceType = CQLTypeParser.parse("KEYSPACE", sourceTypeString, Types.none());
- for (String destinationTypeString: primitiveTypes)
- {
- AbstractType destinationType = CQLTypeParser.parse("KEYSPACE", destinationTypeString, Types.none());
-
- if (compatibilityMap.get(destinationTypeString) != null &&
- compatibilityMap.get(destinationTypeString).contains(sourceTypeString) ||
- sourceTypeString.equals(destinationTypeString))
- {
- assertTrue(sourceTypeString + " should be compatible with " + destinationTypeString,
- destinationType.isValueCompatibleWith(sourceType));
- }
- else
- {
- assertFalse(sourceTypeString + " should not be compatible with " + destinationTypeString,
- destinationType.isValueCompatibleWith(sourceType));
- }
- }
- }
- }
-
- @Test
- public void clusteringColumnTypeCompatibilityTest() throws Throwable
- {
- Map<String, Set<String>> compatibilityMap = new HashMap<>();
- compatibilityMap.put("blob", new HashSet<>(Arrays.asList(new String[] {"ascii", "text", "varchar"})));
- compatibilityMap.put("text", new HashSet<>(Arrays.asList(new String[] {"ascii", "varchar"})));
- compatibilityMap.put("varchar", new HashSet<>(Arrays.asList(new String[] {"ascii", "text" })));
-
- for (String sourceTypeString: primitiveTypes)
- {
- AbstractType sourceType = CQLTypeParser.parse("KEYSPACE", sourceTypeString, Types.none());
- for (String destinationTypeString: primitiveTypes)
- {
- AbstractType destinationType = CQLTypeParser.parse("KEYSPACE", destinationTypeString, Types.none());
-
- if (compatibilityMap.get(destinationTypeString) != null &&
- compatibilityMap.get(destinationTypeString).contains(sourceTypeString) ||
- sourceTypeString.equals(destinationTypeString))
- {
- assertTrue(sourceTypeString + " should be compatible with " + destinationTypeString,
- destinationType.isCompatibleWith(sourceType));
- }
- else
- {
- assertFalse(sourceTypeString + " should not be compatible with " + destinationTypeString,
- destinationType.isCompatibleWith(sourceType));
- }
- }
- }
- }
-}
diff --git a/test/unit/org/apache/cassandra/config/ColumnDefinitionTest.java b/test/unit/org/apache/cassandra/config/ColumnDefinitionTest.java
deleted file mode 100644
index 1e8e704..0000000
--- a/test/unit/org/apache/cassandra/config/ColumnDefinitionTest.java
+++ /dev/null
@@ -1,62 +0,0 @@
-package org.apache.cassandra.config;
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import org.apache.cassandra.db.marshal.*;
-import org.apache.cassandra.thrift.ThriftConversion;
-import org.apache.cassandra.utils.ByteBufferUtil;
-
-public class ColumnDefinitionTest
-{
- @BeforeClass
- public static void setupDD()
- {
- DatabaseDescriptor.daemonInitialization();
- }
-
- @Test
- public void testSerializeDeserialize() throws Exception
- {
- CFMetaData cfm = CFMetaData.Builder.create("ks", "cf", true, false, false)
- .addPartitionKey("pkey", AsciiType.instance)
- .addClusteringColumn("name", AsciiType.instance)
- .addRegularColumn("val", AsciiType.instance)
- .build();
-
- ColumnDefinition cd0 = ColumnDefinition.staticDef(cfm, ByteBufferUtil.bytes("TestColumnDefinitionName0"), BytesType.instance);
- ColumnDefinition cd1 = ColumnDefinition.staticDef(cfm, ByteBufferUtil.bytes("TestColumnDefinition1"), LongType.instance);
-
- testSerializeDeserialize(cfm, cd0);
- testSerializeDeserialize(cfm, cd1);
- }
-
- protected void testSerializeDeserialize(CFMetaData cfm, ColumnDefinition cd) throws Exception
- {
- ColumnDefinition newCd = ThriftConversion.fromThrift(cfm.ksName, cfm.cfName, cfm.comparator.subtype(0), null, ThriftConversion.toThrift(cfm, cd));
- Assert.assertNotSame(cd, newCd);
- Assert.assertEquals(cd.hashCode(), newCd.hashCode());
- Assert.assertEquals(cd, newCd);
- }
-}
diff --git a/test/unit/org/apache/cassandra/config/DatabaseDescriptorRefTest.java b/test/unit/org/apache/cassandra/config/DatabaseDescriptorRefTest.java
index ce59c01..c43d622 100644
--- a/test/unit/org/apache/cassandra/config/DatabaseDescriptorRefTest.java
+++ b/test/unit/org/apache/cassandra/config/DatabaseDescriptorRefTest.java
@@ -54,38 +54,57 @@
public class DatabaseDescriptorRefTest
{
static final String[] validClasses = {
+ "org.apache.cassandra.audit.AuditLogOptions",
+ "org.apache.cassandra.audit.BinAuditLogger",
+ "org.apache.cassandra.audit.BinLogAuditLogger",
+ "org.apache.cassandra.audit.IAuditLogger",
+ "org.apache.cassandra.auth.AllowAllInternodeAuthenticator",
"org.apache.cassandra.auth.IInternodeAuthenticator",
"org.apache.cassandra.auth.IAuthenticator",
"org.apache.cassandra.auth.IAuthorizer",
"org.apache.cassandra.auth.IRoleManager",
+ "org.apache.cassandra.auth.INetworkAuthorizer",
"org.apache.cassandra.config.DatabaseDescriptor",
"org.apache.cassandra.config.ConfigurationLoader",
"org.apache.cassandra.config.Config",
"org.apache.cassandra.config.Config$1",
- "org.apache.cassandra.config.Config$RequestSchedulerId",
"org.apache.cassandra.config.Config$CommitLogSync",
+ "org.apache.cassandra.config.Config$CommitFailurePolicy",
"org.apache.cassandra.config.Config$DiskAccessMode",
"org.apache.cassandra.config.Config$DiskFailurePolicy",
- "org.apache.cassandra.config.Config$CommitFailurePolicy",
"org.apache.cassandra.config.Config$DiskOptimizationStrategy",
+ "org.apache.cassandra.config.Config$FlushCompression",
"org.apache.cassandra.config.Config$InternodeCompression",
"org.apache.cassandra.config.Config$MemtableAllocationType",
+ "org.apache.cassandra.config.Config$RepairCommandPoolFullStrategy",
"org.apache.cassandra.config.Config$UserFunctionTimeoutPolicy",
- "org.apache.cassandra.config.RequestSchedulerOptions",
+ "org.apache.cassandra.config.Config$CorruptedTombstoneStrategy",
+ "org.apache.cassandra.config.DatabaseDescriptor$ByteUnit",
"org.apache.cassandra.config.ParameterizedClass",
"org.apache.cassandra.config.EncryptionOptions",
"org.apache.cassandra.config.EncryptionOptions$ClientEncryptionOptions",
"org.apache.cassandra.config.EncryptionOptions$ServerEncryptionOptions",
"org.apache.cassandra.config.EncryptionOptions$ServerEncryptionOptions$InternodeEncryption",
+ "org.apache.cassandra.config.EncryptionOptions$ServerEncryptionOptions$OutgoingEncryptedPortSource",
"org.apache.cassandra.config.YamlConfigurationLoader",
"org.apache.cassandra.config.YamlConfigurationLoader$PropertiesChecker",
"org.apache.cassandra.config.YamlConfigurationLoader$PropertiesChecker$1",
"org.apache.cassandra.config.YamlConfigurationLoader$CustomConstructor",
"org.apache.cassandra.config.TransparentDataEncryptionOptions",
+ "org.apache.cassandra.db.ConsistencyLevel",
+ "org.apache.cassandra.db.commitlog.CommitLogSegmentManagerFactory",
+ "org.apache.cassandra.db.commitlog.DefaultCommitLogSegmentMgrFactory",
+ "org.apache.cassandra.db.commitlog.AbstractCommitLogSegmentManager",
+ "org.apache.cassandra.db.commitlog.CommitLogSegmentManagerCDC",
+ "org.apache.cassandra.db.commitlog.CommitLogSegmentManagerStandard",
+ "org.apache.cassandra.db.commitlog.CommitLog",
+ "org.apache.cassandra.db.commitlog.CommitLogMBean",
"org.apache.cassandra.dht.IPartitioner",
- "org.apache.cassandra.distributed.impl.InstanceClassLoader",
+ "org.apache.cassandra.distributed.api.IInstance",
+ "org.apache.cassandra.distributed.api.IIsolatedExecutor",
+ "org.apache.cassandra.distributed.shared.InstanceClassLoader",
"org.apache.cassandra.distributed.impl.InstanceConfig",
- "org.apache.cassandra.distributed.impl.InvokableInstance",
+ "org.apache.cassandra.distributed.api.IInvokableInstance",
"org.apache.cassandra.distributed.impl.InvokableInstance$CallableNoExcept",
"org.apache.cassandra.distributed.impl.InvokableInstance$InstanceFunction",
"org.apache.cassandra.distributed.impl.InvokableInstance$SerializableBiConsumer",
@@ -97,14 +116,18 @@
"org.apache.cassandra.distributed.impl.InvokableInstance$SerializableTriFunction",
"org.apache.cassandra.distributed.impl.InvokableInstance$TriFunction",
"org.apache.cassandra.distributed.impl.Message",
+ "org.apache.cassandra.distributed.impl.NetworkTopology",
"org.apache.cassandra.exceptions.ConfigurationException",
"org.apache.cassandra.exceptions.RequestValidationException",
"org.apache.cassandra.exceptions.CassandraException",
"org.apache.cassandra.exceptions.TransportException",
+ "org.apache.cassandra.fql.FullQueryLogger",
+ "org.apache.cassandra.fql.FullQueryLoggerOptions",
"org.apache.cassandra.locator.IEndpointSnitch",
"org.apache.cassandra.io.FSWriteError",
"org.apache.cassandra.io.FSError",
"org.apache.cassandra.io.compress.ICompressor",
+ "org.apache.cassandra.io.compress.ICompressor$Uses",
"org.apache.cassandra.io.compress.LZ4Compressor",
"org.apache.cassandra.io.sstable.metadata.MetadataType",
"org.apache.cassandra.io.util.BufferedDataOutputStreamPlus",
@@ -114,12 +137,13 @@
"org.apache.cassandra.io.util.DataOutputPlus",
"org.apache.cassandra.io.util.DiskOptimizationStrategy",
"org.apache.cassandra.io.util.SpinningDiskOptimizationStrategy",
+ "org.apache.cassandra.locator.Replica",
"org.apache.cassandra.locator.SimpleSeedProvider",
"org.apache.cassandra.locator.SeedProvider",
"org.apache.cassandra.net.BackPressureStrategy",
- "org.apache.cassandra.scheduler.IRequestScheduler",
"org.apache.cassandra.security.EncryptionContext",
"org.apache.cassandra.service.CacheService$CacheType",
+ "org.apache.cassandra.utils.binlog.BinLogOptions",
"org.apache.cassandra.utils.FBUtilities",
"org.apache.cassandra.utils.FBUtilities$1",
"org.apache.cassandra.utils.CloseableIterator",
@@ -140,6 +164,9 @@
"org.apache.cassandra.config.EncryptionOptions$ServerEncryptionOptionsCustomizer",
"org.apache.cassandra.ConsoleAppenderBeanInfo",
"org.apache.cassandra.ConsoleAppenderCustomizer",
+ "org.apache.cassandra.locator.InetAddressAndPort",
+ "org.apache.cassandra.cql3.statements.schema.AlterKeyspaceStatement",
+ "org.apache.cassandra.cql3.statements.schema.CreateKeyspaceStatement"
};
static final Set<String> checkedClasses = new HashSet<>(Arrays.asList(validClasses));
@@ -173,6 +200,13 @@
protected Class<?> findClass(String name) throws ClassNotFoundException
{
+ if (name.startsWith("java."))
+ // Java 11 does not allow a "custom" class loader (i.e. user code)
+ // to define classes in protected packages (like java, java.sql, etc).
+ // Therefore we have to delegate the call to the delegate class loader
+ // itself.
+ return delegate.loadClass(name);
+
Class<?> cls = classMap.get(name);
if (cls != null)
return cls;
@@ -187,7 +221,15 @@
URL url = delegate.getResource(name.replace('.', '/') + ".class");
if (url == null)
- throw new ClassNotFoundException(name);
+ {
+ // For Java 11: system class files are not readable via getResource(), so
+ // try it this way
+ cls = Class.forName(name, false, delegate);
+ classMap.put(name, cls);
+ return cls;
+ }
+
+ // Java8 way + all non-system class files
try (InputStream in = url.openConnection().getInputStream())
{
ByteArrayOutputStream os = new ByteArrayOutputStream();
@@ -215,7 +257,6 @@
for (String methodName : new String[]{
"clientInitialization",
"applyAddressConfig",
- "applyThriftHSHA",
"applyInitialTokens",
// no seed provider in default configuration for clients
// "applySeedProvider",
@@ -225,17 +266,16 @@
// "applySnitch",
"applyEncryptionContext",
// starts "REQUEST-SCHEDULER" thread via RoundRobinScheduler
- // "applyRequestScheduler",
})
{
Method method = cDatabaseDescriptor.getDeclaredMethod(methodName);
method.invoke(null);
if ("clientInitialization".equals(methodName) &&
- threadCount + 1 == threads.getThreadCount())
+ threadCount + 2 == threads.getThreadCount())
{
- // ignore the "AsyncAppender-Worker-ASYNC" thread
- threadCount++;
+ // ignore the "AsyncAppender-Worker-ASYNC" and "logback-1" threads
+ threadCount = threadCount + 2;
}
if (threadCount != threads.getThreadCount())
diff --git a/test/unit/org/apache/cassandra/config/DatabaseDescriptorTest.java b/test/unit/org/apache/cassandra/config/DatabaseDescriptorTest.java
index 4788289..2992a60 100644
--- a/test/unit/org/apache/cassandra/config/DatabaseDescriptorTest.java
+++ b/test/unit/org/apache/cassandra/config/DatabaseDescriptorTest.java
@@ -18,7 +18,6 @@
*/
package org.apache.cassandra.config;
-import java.io.IOException;
import java.net.Inet4Address;
import java.net.Inet6Address;
import java.net.InetAddress;
@@ -27,27 +26,21 @@
import java.util.Collection;
import java.util.Enumeration;
+
+import com.google.common.base.Throwables;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.runner.RunWith;
+
import org.apache.cassandra.OrderedJUnit4ClassRunner;
-import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.exceptions.InvalidRequestException;
-import org.apache.cassandra.gms.Gossiper;
-import org.apache.cassandra.schema.KeyspaceMetadata;
-import org.apache.cassandra.schema.KeyspaceParams;
-import org.apache.cassandra.service.MigrationManager;
-import org.apache.cassandra.thrift.ThriftConversion;
+import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.fail;
-
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
@RunWith(OrderedJUnit4ClassRunner.class)
public class DatabaseDescriptorTest
@@ -58,72 +51,8 @@
DatabaseDescriptor.daemonInitialization();
}
- @Test
- public void testCFMetaDataSerialization() throws ConfigurationException, InvalidRequestException
- {
- // test serialization of all defined test CFs.
- for (String keyspaceName : Schema.instance.getNonSystemKeyspaces())
- {
- for (CFMetaData cfm : Schema.instance.getTablesAndViews(keyspaceName))
- {
- CFMetaData cfmDupe = ThriftConversion.fromThrift(ThriftConversion.toThrift(cfm));
- assertNotNull(cfmDupe);
- assertEquals(cfm, cfmDupe);
- }
- }
- }
-
- @Test
- public void testKSMetaDataSerialization() throws ConfigurationException
- {
- for (String ks : Schema.instance.getNonSystemKeyspaces())
- {
- // Not testing round-trip on the KsDef via serDe() because maps
- KeyspaceMetadata ksm = Schema.instance.getKSMetaData(ks);
- KeyspaceMetadata ksmDupe = ThriftConversion.fromThrift(ThriftConversion.toThrift(ksm));
- assertNotNull(ksmDupe);
- assertEquals(ksm, ksmDupe);
- }
- }
-
// this came as a result of CASSANDRA-995
@Test
- public void testTransKsMigration() throws ConfigurationException, IOException
- {
- SchemaLoader.cleanupAndLeaveDirs();
- Schema.instance.loadFromDisk();
- assertEquals(0, Schema.instance.getNonSystemKeyspaces().size());
-
- Gossiper.instance.start((int)(System.currentTimeMillis() / 1000));
- Keyspace.setInitialized();
-
- try
- {
- // add a few.
- MigrationManager.announceNewKeyspace(KeyspaceMetadata.create("ks0", KeyspaceParams.simple(3)));
- MigrationManager.announceNewKeyspace(KeyspaceMetadata.create("ks1", KeyspaceParams.simple(3)));
-
- assertNotNull(Schema.instance.getKSMetaData("ks0"));
- assertNotNull(Schema.instance.getKSMetaData("ks1"));
-
- Schema.instance.clearKeyspaceMetadata(Schema.instance.getKSMetaData("ks0"));
- Schema.instance.clearKeyspaceMetadata(Schema.instance.getKSMetaData("ks1"));
-
- assertNull(Schema.instance.getKSMetaData("ks0"));
- assertNull(Schema.instance.getKSMetaData("ks1"));
-
- Schema.instance.loadFromDisk();
-
- assertNotNull(Schema.instance.getKSMetaData("ks0"));
- assertNotNull(Schema.instance.getKSMetaData("ks1"));
- }
- finally
- {
- Gossiper.instance.stop();
- }
- }
-
- @Test
public void testConfigurationLoader() throws Exception
{
// By default, we should load from the yaml
@@ -279,12 +208,196 @@
}
@Test
+ public void testInvalidPartition() throws Exception
+ {
+ Config testConfig = DatabaseDescriptor.loadConfig();
+ testConfig.partitioner = "ThisDoesNotExist";
+
+ try
+ {
+ DatabaseDescriptor.applyPartitioner(testConfig);
+ Assert.fail("Partition does not exist, so should fail");
+ }
+ catch (ConfigurationException e)
+ {
+ Assert.assertEquals("Invalid partitioner class ThisDoesNotExist", e.getMessage());
+ Throwable cause = Throwables.getRootCause(e);
+ Assert.assertNotNull("Unable to find root cause why partitioner was rejected", cause);
+ // this is a bit implementation specific, so free to change; mostly here to make sure reason isn't lost
+ Assert.assertEquals(ClassNotFoundException.class, cause.getClass());
+ Assert.assertEquals("org.apache.cassandra.dht.ThisDoesNotExist", cause.getMessage());
+ }
+ }
+
+ @Test
+ public void testInvalidPartitionPropertyOverride() throws Exception
+ {
+ String key = Config.PROPERTY_PREFIX + "partitioner";
+ String previous = System.getProperty(key);
+ try
+ {
+ System.setProperty(key, "ThisDoesNotExist");
+ Config testConfig = DatabaseDescriptor.loadConfig();
+ testConfig.partitioner = "Murmur3Partitioner";
+
+ try
+ {
+ DatabaseDescriptor.applyPartitioner(testConfig);
+ Assert.fail("Partition does not exist, so should fail");
+ }
+ catch (ConfigurationException e)
+ {
+ Assert.assertEquals("Invalid partitioner class ThisDoesNotExist", e.getMessage());
+ Throwable cause = Throwables.getRootCause(e);
+ Assert.assertNotNull("Unable to find root cause why partitioner was rejected", cause);
+ // this is a bit implementation specific, so free to change; mostly here to make sure reason isn't lost
+ Assert.assertEquals(ClassNotFoundException.class, cause.getClass());
+ Assert.assertEquals("org.apache.cassandra.dht.ThisDoesNotExist", cause.getMessage());
+ }
+ }
+ finally
+ {
+ if (previous == null)
+ {
+ System.getProperties().remove(key);
+ }
+ else
+ {
+ System.setProperty(key, previous);
+ }
+ }
+ }
+
+ @Test
public void testTokensFromString()
{
assertTrue(DatabaseDescriptor.tokensFromString(null).isEmpty());
Collection<String> tokens = DatabaseDescriptor.tokensFromString(" a,b ,c , d, f,g,h");
assertEquals(7, tokens.size());
- assertTrue(tokens.containsAll(Arrays.asList(new String[]{ "a", "b", "c", "d", "f", "g", "h" })));
+ assertTrue(tokens.containsAll(Arrays.asList(new String[]{"a", "b", "c", "d", "f", "g", "h"})));
+ }
+
+ @Test
+ public void testExceptionsForInvalidConfigValues() {
+ try
+ {
+ DatabaseDescriptor.setColumnIndexCacheSize(-1);
+ fail("Should have received a ConfigurationException column_index_cache_size_in_kb = -1");
+ }
+ catch (ConfigurationException ignored) { }
+ Assert.assertEquals(2048, DatabaseDescriptor.getColumnIndexCacheSize());
+
+ try
+ {
+ DatabaseDescriptor.setColumnIndexCacheSize(2 * 1024 * 1024);
+ fail("Should have received a ConfigurationException column_index_cache_size_in_kb = 2GiB");
+ }
+ catch (ConfigurationException ignored) { }
+ Assert.assertEquals(2048, DatabaseDescriptor.getColumnIndexCacheSize());
+
+ try
+ {
+ DatabaseDescriptor.setColumnIndexSize(-1);
+ fail("Should have received a ConfigurationException column_index_size_in_kb = -1");
+ }
+ catch (ConfigurationException ignored) { }
+ Assert.assertEquals(4096, DatabaseDescriptor.getColumnIndexSize());
+
+ try
+ {
+ DatabaseDescriptor.setColumnIndexSize(2 * 1024 * 1024);
+ fail("Should have received a ConfigurationException column_index_size_in_kb = 2GiB");
+ }
+ catch (ConfigurationException ignored) { }
+ Assert.assertEquals(4096, DatabaseDescriptor.getColumnIndexSize());
+
+ try
+ {
+ DatabaseDescriptor.setBatchSizeWarnThresholdInKB(-1);
+ fail("Should have received a ConfigurationException batch_size_warn_threshold_in_kb = -1");
+ }
+ catch (ConfigurationException ignored) { }
+ Assert.assertEquals(5120, DatabaseDescriptor.getBatchSizeWarnThreshold());
+
+ try
+ {
+ DatabaseDescriptor.setBatchSizeWarnThresholdInKB(2 * 1024 * 1024);
+ fail("Should have received a ConfigurationException batch_size_warn_threshold_in_kb = 2GiB");
+ }
+ catch (ConfigurationException ignored) { }
+ Assert.assertEquals(4096, DatabaseDescriptor.getColumnIndexSize());
+ }
+
+ @Test
+ public void testLowestAcceptableTimeouts() throws ConfigurationException
+ {
+ Config testConfig = new Config();
+ testConfig.read_request_timeout_in_ms = DatabaseDescriptor.LOWEST_ACCEPTED_TIMEOUT + 1;
+ testConfig.range_request_timeout_in_ms = DatabaseDescriptor.LOWEST_ACCEPTED_TIMEOUT + 1;
+ testConfig.write_request_timeout_in_ms = DatabaseDescriptor.LOWEST_ACCEPTED_TIMEOUT + 1;
+ testConfig.truncate_request_timeout_in_ms = DatabaseDescriptor.LOWEST_ACCEPTED_TIMEOUT + 1;
+ testConfig.cas_contention_timeout_in_ms = DatabaseDescriptor.LOWEST_ACCEPTED_TIMEOUT + 1;
+ testConfig.counter_write_request_timeout_in_ms = DatabaseDescriptor.LOWEST_ACCEPTED_TIMEOUT + 1;
+ testConfig.request_timeout_in_ms = DatabaseDescriptor.LOWEST_ACCEPTED_TIMEOUT + 1;
+
+ assertTrue(testConfig.read_request_timeout_in_ms > DatabaseDescriptor.LOWEST_ACCEPTED_TIMEOUT);
+ assertTrue(testConfig.range_request_timeout_in_ms > DatabaseDescriptor.LOWEST_ACCEPTED_TIMEOUT);
+ assertTrue(testConfig.write_request_timeout_in_ms > DatabaseDescriptor.LOWEST_ACCEPTED_TIMEOUT);
+ assertTrue(testConfig.truncate_request_timeout_in_ms > DatabaseDescriptor.LOWEST_ACCEPTED_TIMEOUT);
+ assertTrue(testConfig.cas_contention_timeout_in_ms > DatabaseDescriptor.LOWEST_ACCEPTED_TIMEOUT);
+ assertTrue(testConfig.counter_write_request_timeout_in_ms > DatabaseDescriptor.LOWEST_ACCEPTED_TIMEOUT);
+ assertTrue(testConfig.request_timeout_in_ms > DatabaseDescriptor.LOWEST_ACCEPTED_TIMEOUT);
+
+ //set less than Lowest acceptable value
+ testConfig.read_request_timeout_in_ms = DatabaseDescriptor.LOWEST_ACCEPTED_TIMEOUT - 1;
+ testConfig.range_request_timeout_in_ms = DatabaseDescriptor.LOWEST_ACCEPTED_TIMEOUT - 1;
+ testConfig.write_request_timeout_in_ms = DatabaseDescriptor.LOWEST_ACCEPTED_TIMEOUT - 1;
+ testConfig.truncate_request_timeout_in_ms = DatabaseDescriptor.LOWEST_ACCEPTED_TIMEOUT - 1;
+ testConfig.cas_contention_timeout_in_ms = DatabaseDescriptor.LOWEST_ACCEPTED_TIMEOUT - 1;
+ testConfig.counter_write_request_timeout_in_ms = DatabaseDescriptor.LOWEST_ACCEPTED_TIMEOUT - 1;
+ testConfig.request_timeout_in_ms = DatabaseDescriptor.LOWEST_ACCEPTED_TIMEOUT - 1;
+
+ DatabaseDescriptor.checkForLowestAcceptedTimeouts(testConfig);
+
+ assertTrue(testConfig.read_request_timeout_in_ms == DatabaseDescriptor.LOWEST_ACCEPTED_TIMEOUT);
+ assertTrue(testConfig.range_request_timeout_in_ms == DatabaseDescriptor.LOWEST_ACCEPTED_TIMEOUT);
+ assertTrue(testConfig.write_request_timeout_in_ms == DatabaseDescriptor.LOWEST_ACCEPTED_TIMEOUT);
+ assertTrue(testConfig.truncate_request_timeout_in_ms == DatabaseDescriptor.LOWEST_ACCEPTED_TIMEOUT);
+ assertTrue(testConfig.cas_contention_timeout_in_ms == DatabaseDescriptor.LOWEST_ACCEPTED_TIMEOUT);
+ assertTrue(testConfig.counter_write_request_timeout_in_ms == DatabaseDescriptor.LOWEST_ACCEPTED_TIMEOUT);
+ assertTrue(testConfig.request_timeout_in_ms == DatabaseDescriptor.LOWEST_ACCEPTED_TIMEOUT);
+ }
+
+ @Test
+ public void testRepairSessionMemorySizeToggles()
+ {
+ int previousSize = DatabaseDescriptor.getRepairSessionSpaceInMegabytes();
+ try
+ {
+ Assert.assertEquals((Runtime.getRuntime().maxMemory() / (1024 * 1024) / 16),
+ DatabaseDescriptor.getRepairSessionSpaceInMegabytes());
+
+ int targetSize = (int) (Runtime.getRuntime().maxMemory() / (1024 * 1024) / 4) + 1;
+
+ DatabaseDescriptor.setRepairSessionSpaceInMegabytes(targetSize);
+ Assert.assertEquals(targetSize, DatabaseDescriptor.getRepairSessionSpaceInMegabytes());
+
+ DatabaseDescriptor.setRepairSessionSpaceInMegabytes(10);
+ Assert.assertEquals(10, DatabaseDescriptor.getRepairSessionSpaceInMegabytes());
+
+ try
+ {
+ DatabaseDescriptor.setRepairSessionSpaceInMegabytes(0);
+ fail("Should have received a ConfigurationException for depth of 9");
+ }
+ catch (ConfigurationException ignored) { }
+
+ Assert.assertEquals(10, DatabaseDescriptor.getRepairSessionSpaceInMegabytes());
+ }
+ finally
+ {
+ DatabaseDescriptor.setRepairSessionSpaceInMegabytes(previousSize);
+ }
}
@Test
@@ -293,7 +406,7 @@
int previousDepth = DatabaseDescriptor.getRepairSessionMaxTreeDepth();
try
{
- Assert.assertEquals(18, DatabaseDescriptor.getRepairSessionMaxTreeDepth());
+ Assert.assertEquals(20, DatabaseDescriptor.getRepairSessionMaxTreeDepth());
DatabaseDescriptor.setRepairSessionMaxTreeDepth(10);
Assert.assertEquals(10, DatabaseDescriptor.getRepairSessionMaxTreeDepth());
@@ -321,4 +434,72 @@
DatabaseDescriptor.setRepairSessionMaxTreeDepth(previousDepth);
}
}
+
+ @Test
+ public void testCalculateDefaultSpaceInMB()
+ {
+ // check prefered size is used for a small storage volume
+ int preferredInMB = 667;
+ int numerator = 2;
+ int denominator = 3;
+ int spaceInBytes = 999 * 1024 * 1024;
+
+ assertEquals(666, // total size is less than preferred, so return lower limit
+ DatabaseDescriptor.calculateDefaultSpaceInMB("type", "/path", "setting_name", preferredInMB, spaceInBytes, numerator, denominator));
+
+ // check preferred size is used for a small storage volume
+ preferredInMB = 100;
+ numerator = 1;
+ denominator = 3;
+ spaceInBytes = 999 * 1024 * 1024;
+
+ assertEquals(100, // total size is more than preferred so keep the configured limit
+ DatabaseDescriptor.calculateDefaultSpaceInMB("type", "/path", "setting_name", preferredInMB, spaceInBytes, numerator, denominator));
+ }
+
+ @Test
+ public void testConcurrentValidations()
+ {
+ Config conf = new Config();
+ conf.concurrent_compactors = 8;
+ // if concurrent_validations is < 1 (including being unset) it should default to concurrent_compactors
+ assertThat(conf.concurrent_validations).isLessThan(1);
+ DatabaseDescriptor.applyConcurrentValidations(conf);
+ assertThat(conf.concurrent_validations).isEqualTo(conf.concurrent_compactors);
+
+ // otherwise, it must be <= concurrent_compactors
+ conf.concurrent_validations = conf.concurrent_compactors + 1;
+ try
+ {
+ DatabaseDescriptor.applyConcurrentValidations(conf);
+ fail("Expected exception");
+ }
+ catch (ConfigurationException e)
+ {
+ assertThat(e.getMessage()).isEqualTo("To set concurrent_validations > concurrent_compactors, " +
+ "set the system property cassandra.allow_unlimited_concurrent_validations=true");
+ }
+
+ // unless we disable that check (done with a system property at startup or via JMX)
+ DatabaseDescriptor.allowUnlimitedConcurrentValidations = true;
+ conf.concurrent_validations = conf.concurrent_compactors + 1;
+ DatabaseDescriptor.applyConcurrentValidations(conf);
+ assertThat(conf.concurrent_validations).isEqualTo(conf.concurrent_compactors + 1);
+ }
+
+ @Test
+ public void testRepairCommandPoolSize()
+ {
+ Config conf = new Config();
+ conf.concurrent_validations = 3;
+ // if repair_command_pool_size is < 1 (including being unset) it should default to concurrent_validations
+ assertThat(conf.repair_command_pool_size).isLessThan(1);
+ DatabaseDescriptor.applyRepairCommandPoolSize(conf);
+ assertThat(conf.repair_command_pool_size).isEqualTo(conf.concurrent_validations);
+
+ // but it can be overridden
+ conf.repair_command_pool_size = conf.concurrent_validations + 1;
+ DatabaseDescriptor.applyRepairCommandPoolSize(conf);
+ assertThat(conf.repair_command_pool_size).isEqualTo(conf.concurrent_validations + 1);
+ }
}
diff --git a/test/unit/org/apache/cassandra/config/OverrideConfigurationLoader.java b/test/unit/org/apache/cassandra/config/OverrideConfigurationLoader.java
new file mode 100644
index 0000000..e0a5576
--- /dev/null
+++ b/test/unit/org/apache/cassandra/config/OverrideConfigurationLoader.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.config;
+
+import java.util.function.Consumer;
+import java.util.function.Function;
+
+import org.apache.cassandra.exceptions.ConfigurationException;
+
+/**
+ * Helper class for programmatically overriding individual config values before DatabaseDescriptor is bootstrapped.
+ */
+public class OverrideConfigurationLoader implements ConfigurationLoader
+{
+
+ private static Consumer<Config> configModifier;
+
+ public Config loadConfig() throws ConfigurationException
+ {
+ YamlConfigurationLoader loader = new YamlConfigurationLoader();
+ Config config = loader.loadConfig();
+ configModifier.accept(config);
+ return config;
+ }
+
+ public static void override(Consumer<Config> modifier)
+ {
+ System.setProperty(Config.PROPERTY_PREFIX + "config.loader", OverrideConfigurationLoader.class.getName());
+ configModifier = modifier;
+ }
+}
diff --git a/test/unit/org/apache/cassandra/cql3/CQLTester.java b/test/unit/org/apache/cassandra/cql3/CQLTester.java
index e545e9f..619fdad 100644
--- a/test/unit/org/apache/cassandra/cql3/CQLTester.java
+++ b/test/unit/org/apache/cassandra/cql3/CQLTester.java
@@ -23,15 +23,19 @@
import java.math.BigInteger;
import java.net.InetAddress;
import java.net.ServerSocket;
+import java.net.UnknownHostException;
import java.nio.ByteBuffer;
import java.util.*;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
import java.util.stream.Collectors;
import com.google.common.base.Objects;
+import com.google.common.base.Strings;
import com.google.common.collect.ImmutableSet;
import org.junit.*;
import org.slf4j.Logger;
@@ -42,14 +46,19 @@
import com.datastax.driver.core.ResultSet;
import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.audit.AuditLogManager;
import org.apache.cassandra.concurrent.ScheduledExecutors;
-import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.db.virtual.VirtualKeyspaceRegistry;
+import org.apache.cassandra.db.virtual.VirtualSchemaKeyspace;
+import org.apache.cassandra.index.SecondaryIndexManager;
+import org.apache.cassandra.config.EncryptionOptions;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.Replica;
+import org.apache.cassandra.locator.TokenMetadata;
import org.apache.cassandra.metrics.ClientMetrics;
+import org.apache.cassandra.schema.*;
import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.config.Schema;
-import org.apache.cassandra.config.SchemaConstants;
import org.apache.cassandra.cql3.functions.FunctionName;
-import org.apache.cassandra.cql3.statements.ParsedStatement;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.commitlog.CommitLog;
import org.apache.cassandra.db.marshal.*;
@@ -58,14 +67,16 @@
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.exceptions.SyntaxException;
import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.locator.AbstractEndpointSnitch;
+import org.apache.cassandra.schema.IndexMetadata;
+import org.apache.cassandra.schema.KeyspaceMetadata;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.serializers.TypeSerializer;
import org.apache.cassandra.service.ClientState;
import org.apache.cassandra.service.QueryState;
import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.transport.*;
import org.apache.cassandra.transport.ProtocolVersion;
-import org.apache.cassandra.transport.ConfiguredLimit;
-import org.apache.cassandra.transport.Event;
-import org.apache.cassandra.transport.Server;
import org.apache.cassandra.transport.messages.ResultMessage;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FBUtilities;
@@ -87,18 +98,29 @@
protected static final long ROW_CACHE_SIZE_IN_MB = Integer.valueOf(System.getProperty("cassandra.test.row_cache_size_in_mb", "0"));
private static final AtomicInteger seqNumber = new AtomicInteger();
protected static final ByteBuffer TOO_BIG = ByteBuffer.allocate(FBUtilities.MAX_UNSIGNED_SHORT + 1024);
+ public static final String DATA_CENTER = "datacenter1";
+ public static final String DATA_CENTER_REMOTE = "datacenter2";
+ public static final String RACK1 = "rack1";
private static org.apache.cassandra.transport.Server server;
protected static final int nativePort;
protected static final InetAddress nativeAddr;
+ protected static final Set<InetAddressAndPort> remoteAddrs = new HashSet<>();
private static final Map<ProtocolVersion, Cluster> clusters = new HashMap<>();
- private static final Map<ProtocolVersion, Session> sessions = new HashMap<>();
- protected static ConfiguredLimit protocolVersionLimit;
+ protected static final Map<ProtocolVersion, Session> sessions = new HashMap<>();
private static boolean isServerPrepared = false;
public static final List<ProtocolVersion> PROTOCOL_VERSIONS = new ArrayList<>(ProtocolVersion.SUPPORTED.size());
+ private static final String CREATE_INDEX_NAME_REGEX = "(\\s*(\\w*|\"\\w*\")\\s*)";
+ private static final String CREATE_INDEX_REGEX = String.format("\\A\\s*CREATE(?:\\s+CUSTOM)?\\s+INDEX" +
+ "(?:\\s+IF\\s+NOT\\s+EXISTS)?\\s*" +
+ "%s?\\s*ON\\s+(%<s\\.)?%<s\\s*" +
+ "(\\((?:\\s*\\w+\\s*\\()?%<s\\))?",
+ CREATE_INDEX_NAME_REGEX);
+ private static final Pattern CREATE_INDEX_PATTERN = Pattern.compile(CREATE_INDEX_REGEX, Pattern.CASE_INSENSITIVE);
+
/** Return the current server version if supported by the driver, else
* the latest that is supported.
*
@@ -130,6 +152,18 @@
nativeAddr = InetAddress.getLoopbackAddress();
+ // Register an EndpointSnitch which returns fixed values for test.
+ DatabaseDescriptor.setEndpointSnitch(new AbstractEndpointSnitch()
+ {
+ @Override public String getRack(InetAddressAndPort endpoint) { return RACK1; }
+ @Override public String getDatacenter(InetAddressAndPort endpoint) {
+ if (remoteAddrs.contains(endpoint))
+ return DATA_CENTER_REMOTE;
+ return DATA_CENTER;
+ }
+ @Override public int compareEndpoints(InetAddressAndPort target, Replica a1, Replica a2) { return 0; }
+ });
+
try
{
try (ServerSocket serverSocket = new ServerSocket(0))
@@ -168,6 +202,8 @@
return;
DatabaseDescriptor.daemonInitialization();
+ DatabaseDescriptor.setTransientReplicationEnabledUnsafe(true);
+ CommitLog.instance.start();
// Cleanup first
try
@@ -180,6 +216,15 @@
throw new RuntimeException(e);
}
+ try {
+ remoteAddrs.add(InetAddressAndPort.getByName("127.0.0.4"));
+ }
+ catch (UnknownHostException e)
+ {
+ logger.error("Failed to lookup host");
+ throw new RuntimeException(e);
+ }
+
Thread.setDefaultUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler()
{
public void uncaughtException(Thread t, Throwable e)
@@ -191,12 +236,15 @@
ThreadAwareSecurityManager.install();
Keyspace.setInitialized();
+ SystemKeyspace.persistLocalMetadata();
+ AuditLogManager.instance.initialize();
isServerPrepared = true;
}
public static void cleanupAndLeaveDirs() throws IOException
{
// We need to stop and unmap all CLS instances prior to cleanup() or we'll get failures on Windows.
+ CommitLog.instance.start();
CommitLog.instance.stopUnsafe(true);
mkdirs();
cleanup();
@@ -252,7 +300,6 @@
{
if (ROW_CACHE_SIZE_IN_MB > 0)
DatabaseDescriptor.setRowCacheSizeInMB(ROW_CACHE_SIZE_IN_MB);
-
StorageService.instance.setPartitionerUnsafe(Murmur3Partitioner.instance);
// Once per-JVM is enough
@@ -274,6 +321,9 @@
// statements are not cached but re-prepared every time). So we clear the cache between test files to avoid accumulating too much.
if (reusePrepared)
QueryProcessor.clearInternalStatementsCache();
+
+ TokenMetadata metadata = StorageService.instance.getTokenMetadata();
+ metadata.clearUnsafe();
}
@Before
@@ -356,43 +406,13 @@
if (server != null)
return;
- prepareNetwork();
- initializeNetwork();
- }
-
- protected static void prepareNetwork()
- {
SystemKeyspace.finishStartup();
+ VirtualKeyspaceRegistry.instance.register(VirtualSchemaKeyspace.instance);
+
StorageService.instance.initServer();
SchemaLoader.startGossiper();
- }
- protected static void reinitializeNetwork()
- {
- if (server != null && server.isRunning())
- {
- server.stop();
- server = null;
- }
- List<CloseFuture> futures = new ArrayList<>();
- for (Cluster cluster : clusters.values())
- futures.add(cluster.closeAsync());
- for (Session session : sessions.values())
- futures.add(session.closeAsync());
- FBUtilities.waitOnFutures(futures);
- clusters.clear();
- sessions.clear();
-
- initializeNetwork();
- }
-
- private static void initializeNetwork()
- {
- protocolVersionLimit = ConfiguredLimit.newLimit();
- server = new Server.Builder().withHost(nativeAddr)
- .withPort(nativePort)
- .withProtocolVersionLimit(protocolVersionLimit)
- .build();
+ server = new Server.Builder().withHost(nativeAddr).withPort(nativePort).build();
ClientMetrics.instance.init(Collections.singleton(server));
server.start();
@@ -401,15 +421,18 @@
if (clusters.containsKey(version))
continue;
- if (version.isGreaterThan(protocolVersionLimit.getMaxVersion()))
- continue;
+ Cluster.Builder builder = Cluster.builder()
+ .withoutJMXReporting()
+ .addContactPoints(nativeAddr)
+ .withClusterName("Test Cluster")
+ .withPort(nativePort);
- Cluster cluster = Cluster.builder()
- .addContactPoints(nativeAddr)
- .withClusterName("Test Cluster-" + version.name())
- .withPort(nativePort)
- .withProtocolVersion(com.datastax.driver.core.ProtocolVersion.fromInt(version.asInt()))
- .build();
+ if (version.isBeta())
+ builder = builder.allowBetaProtocolVersion();
+ else
+ builder = builder.withProtocolVersion(com.datastax.driver.core.ProtocolVersion.fromInt(version.asInt()));
+
+ Cluster cluster = builder.build();
clusters.put(version, cluster);
sessions.put(version, cluster.connect());
@@ -417,14 +440,6 @@
}
}
- protected void updateMaxNegotiableProtocolVersion()
- {
- if (protocolVersionLimit == null)
- throw new IllegalStateException("Native transport server has not been initialized");
-
- protocolVersionLimit.updateMaxSupportedVersion();
- }
-
protected void dropPerTestKeyspace() throws Throwable
{
execute(String.format("DROP KEYSPACE IF EXISTS %s", KEYSPACE_PER_TEST));
@@ -560,6 +575,13 @@
return tables.get(tables.size() - 1);
}
+ protected String currentKeyspace()
+ {
+ if (keyspaces.isEmpty())
+ return null;
+ return keyspaces.get(keyspaces.size() - 1);
+ }
+
protected ByteBuffer unset()
{
return ByteBufferUtil.UNSET_BYTE_BUFFER;
@@ -582,7 +604,7 @@
protected String createType(String query)
{
- String typeName = "type_" + seqNumber.getAndIncrement();
+ String typeName = String.format("type_%02d", seqNumber.getAndIncrement());
String fullQuery = String.format(query, KEYSPACE + "." + typeName);
types.add(typeName);
logger.info(fullQuery);
@@ -592,7 +614,7 @@
protected String createFunction(String keyspace, String argTypes, String query) throws Throwable
{
- String functionName = keyspace + ".function_" + seqNumber.getAndIncrement();
+ String functionName = String.format("%s.function_%02d", keyspace, seqNumber.getAndIncrement());
createFunctionOverload(functionName, argTypes, query);
return functionName;
}
@@ -607,7 +629,7 @@
protected String createAggregate(String keyspace, String argTypes, String query) throws Throwable
{
- String aggregateName = keyspace + "." + "aggregate_" + seqNumber.getAndIncrement();
+ String aggregateName = String.format("%s.aggregate_%02d", keyspace, seqNumber.getAndIncrement());
createAggregateOverload(aggregateName, argTypes, query);
return aggregateName;
}
@@ -629,9 +651,23 @@
return currentKeyspace;
}
+ protected void alterKeyspace(String query)
+ {
+ String fullQuery = String.format(query, currentKeyspace());
+ logger.info(fullQuery);
+ schemaChange(fullQuery);
+ }
+
+ protected void alterKeyspaceMayThrow(String query) throws Throwable
+ {
+ String fullQuery = String.format(query, currentKeyspace());
+ logger.info(fullQuery);
+ QueryProcessor.executeOnceInternal(fullQuery);
+ }
+
protected String createKeyspaceName()
{
- String currentKeyspace = "keyspace_" + seqNumber.getAndIncrement();
+ String currentKeyspace = String.format("keyspace_%02d", seqNumber.getAndIncrement());
keyspaces.add(currentKeyspace);
return currentKeyspace;
}
@@ -652,7 +688,7 @@
protected String createTableName()
{
- String currentTable = "table_" + seqNumber.getAndIncrement();
+ String currentTable = String.format("table_%02d", seqNumber.getAndIncrement());
tables.add(currentTable);
return currentTable;
}
@@ -690,15 +726,46 @@
schemaChange(formattedQuery);
}
- protected void createIndex(String query)
+ protected String createIndex(String query)
{
- createFormattedIndex(formatQuery(query));
+ String formattedQuery = formatQuery(query);
+ return createFormattedIndex(formattedQuery);
}
- protected void createFormattedIndex(String formattedQuery)
+ protected String createFormattedIndex(String formattedQuery)
{
logger.info(formattedQuery);
+ String indexName = getCreateIndexName(formattedQuery);
schemaChange(formattedQuery);
+ return indexName;
+ }
+
+ protected static String getCreateIndexName(String formattedQuery)
+ {
+ Matcher matcher = CREATE_INDEX_PATTERN.matcher(formattedQuery);
+ if (!matcher.find())
+ throw new IllegalArgumentException("Expected valid create index query but found: " + formattedQuery);
+
+ String index = matcher.group(2);
+ if (!Strings.isNullOrEmpty(index))
+ return index;
+
+ String keyspace = matcher.group(5);
+ if (Strings.isNullOrEmpty(keyspace))
+ throw new IllegalArgumentException("Keyspace name should be specified: " + formattedQuery);
+
+ String table = matcher.group(7);
+ if (Strings.isNullOrEmpty(table))
+ throw new IllegalArgumentException("Table name should be specified: " + formattedQuery);
+
+ String column = matcher.group(9);
+
+ String baseName = Strings.isNullOrEmpty(column)
+ ? IndexMetadata.generateDefaultIndexName(table)
+ : IndexMetadata.generateDefaultIndexName(table, new ColumnIdentifier(column, true));
+
+ KeyspaceMetadata ks = Schema.instance.getKeyspaceMetadata(keyspace);
+ return ks.findAvailableIndexName(baseName);
}
/**
@@ -731,6 +798,38 @@
return indexCreated;
}
+ /**
+ * Index creation is asynchronous, this method waits until the specified index hasn't any building task running.
+ * <p>
+ * This method differs from {@link #waitForIndex(String, String, String)} in that it doesn't require the index to be
+ * fully nor successfully built, so it can be used to wait for failing index builds.
+ *
+ * @param keyspace the index keyspace name
+ * @param indexName the index name
+ * @return {@code true} if the index build tasks have finished in 5 seconds, {@code false} otherwise
+ */
+ protected boolean waitForIndexBuilds(String keyspace, String indexName) throws InterruptedException
+ {
+ long start = System.currentTimeMillis();
+ SecondaryIndexManager indexManager = getCurrentColumnFamilyStore(keyspace).indexManager;
+
+ while (true)
+ {
+ if (!indexManager.isIndexBuilding(indexName))
+ {
+ return true;
+ }
+ else if (System.currentTimeMillis() - start > 5000)
+ {
+ return false;
+ }
+ else
+ {
+ Thread.sleep(10);
+ }
+ }
+ }
+
protected void createIndexMayThrow(String query) throws Throwable
{
String fullQuery = formatQuery(query);
@@ -762,16 +861,15 @@
{
try
{
- ClientState state = ClientState.forInternalCalls();
- state.setKeyspace(SchemaConstants.SYSTEM_KEYSPACE_NAME);
+ ClientState state = ClientState.forInternalCalls(SchemaConstants.SYSTEM_KEYSPACE_NAME);
QueryState queryState = new QueryState(state);
- ParsedStatement.Prepared prepared = QueryProcessor.parseStatement(query, queryState);
- prepared.statement.validate(state);
+ CQLStatement statement = QueryProcessor.parseStatement(query, queryState.getClientState());
+ statement.validate(state);
QueryOptions options = QueryOptions.forInternalCalls(Collections.<ByteBuffer>emptyList());
- lastSchemaChangeResult = prepared.statement.executeInternal(queryState, options);
+ lastSchemaChangeResult = statement.executeLocally(queryState, options);
}
catch (Exception e)
{
@@ -780,9 +878,9 @@
}
}
- protected CFMetaData currentTableMetadata()
+ protected TableMetadata currentTableMetadata()
{
- return Schema.instance.getCFMetaData(KEYSPACE, currentTable());
+ return Schema.instance.getTableMetadata(KEYSPACE, currentTable());
}
protected com.datastax.driver.core.ResultSet executeNet(ProtocolVersion protocolVersion, String query, Object... values) throws Throwable
@@ -790,6 +888,21 @@
return sessionNet(protocolVersion).execute(formatQuery(query), values);
}
+ protected com.datastax.driver.core.ResultSet executeNet(String query, Object... values) throws Throwable
+ {
+ return sessionNet().execute(formatQuery(query), values);
+ }
+
+ protected com.datastax.driver.core.ResultSet executeNet(ProtocolVersion protocolVersion, Statement statement)
+ {
+ return sessionNet(protocolVersion).execute(statement);
+ }
+
+ protected com.datastax.driver.core.ResultSet executeNetWithPaging(ProtocolVersion version, String query, int pageSize) throws Throwable
+ {
+ return sessionNet(version).execute(new SimpleStatement(formatQuery(query)).setFetchSize(pageSize));
+ }
+
protected com.datastax.driver.core.ResultSet executeNetWithPaging(String query, int pageSize) throws Throwable
{
return sessionNet().execute(new SimpleStatement(formatQuery(query)).setFetchSize(pageSize));
@@ -807,6 +920,16 @@
return sessions.get(protocolVersion);
}
+ protected SimpleClient newSimpleClient(ProtocolVersion version, boolean compression, boolean checksums, boolean isOverloadedException) throws IOException
+ {
+ return new SimpleClient(nativeAddr.getHostAddress(), nativePort, version, version.isBeta(), new EncryptionOptions()).connect(compression, checksums, isOverloadedException);
+ }
+
+ protected SimpleClient newSimpleClient(ProtocolVersion version, boolean compression, boolean checksums) throws IOException
+ {
+ return newSimpleClient(version, compression, checksums, false);
+ }
+
protected String formatQuery(String query)
{
return formatQuery(KEYSPACE, query);
@@ -820,7 +943,7 @@
protected ResultMessage.Prepared prepare(String query) throws Throwable
{
- return QueryProcessor.prepare(formatQuery(query), ClientState.forInternalCalls(), false);
+ return QueryProcessor.prepare(formatQuery(query), ClientState.forInternalCalls());
}
protected UntypedResultSet execute(String query, Object... values) throws Throwable
@@ -961,16 +1084,18 @@
ByteBuffer expectedByteValue = makeByteBuffer(expected == null ? null : expected[j], column.type);
ByteBuffer actualValue = actual.getBytes(column.name.toString());
+ if (expectedByteValue != null)
+ expectedByteValue = expectedByteValue.duplicate();
if (!Objects.equal(expectedByteValue, actualValue))
{
Object actualValueDecoded = actualValue == null ? null : column.type.getSerializer().deserialize(actualValue);
- if (!Objects.equal(expected[j], actualValueDecoded))
+ if (!Objects.equal(expected != null ? expected[j] : null, actualValueDecoded))
Assert.fail(String.format("Invalid value for row %d column %d (%s of type %s), expected <%s> but got <%s>",
i,
j,
column.name,
column.type.asCQL3Type(),
- formatValue(expectedByteValue, column.type),
+ formatValue(expectedByteValue != null ? expectedByteValue.duplicate() : null, column.type),
formatValue(actualValue, column.type)));
}
}
@@ -1520,7 +1645,7 @@
return s;
}
- private static ByteBuffer makeByteBuffer(Object value, AbstractType type)
+ protected static ByteBuffer makeByteBuffer(Object value, AbstractType type)
{
if (value == null)
return null;
@@ -1551,7 +1676,7 @@
return type.getString(bb);
}
- protected Object tuple(Object...values)
+ protected TupleValue tuple(Object...values)
{
return new TupleValue(values);
}
@@ -1586,7 +1711,7 @@
protected Object map(Object...values)
{
if (values.length % 2 != 0)
- throw new IllegalArgumentException();
+ throw new IllegalArgumentException("Invalid number of arguments, got " + values.length);
int size = values.length / 2;
Map m = new LinkedHashMap(size);
@@ -1595,7 +1720,7 @@
return m;
}
- protected com.datastax.driver.core.TupleType tupleTypeOf(ProtocolVersion protocolVersion, DataType...types)
+ protected com.datastax.driver.core.TupleType tupleTypeOf(ProtocolVersion protocolVersion, com.datastax.driver.core.DataType...types)
{
requireNetwork();
return clusters.get(protocolVersion).getMetadata().newTupleType(types);
diff --git a/test/unit/org/apache/cassandra/cql3/ColumnConditionTest.java b/test/unit/org/apache/cassandra/cql3/ColumnConditionTest.java
deleted file mode 100644
index ca0c182..0000000
--- a/test/unit/org/apache/cassandra/cql3/ColumnConditionTest.java
+++ /dev/null
@@ -1,589 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.cql3;
-
-import java.nio.ByteBuffer;
-import java.util.*;
-
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.db.rows.BufferCell;
-import org.apache.cassandra.db.rows.Cell;
-import org.apache.cassandra.db.rows.CellPath;
-import org.apache.cassandra.db.marshal.*;
-import org.apache.cassandra.exceptions.InvalidRequestException;
-import org.apache.cassandra.serializers.Int32Serializer;
-import org.apache.cassandra.utils.ByteBufferUtil;
-import org.apache.cassandra.utils.UUIDGen;
-
-import static org.junit.Assert.*;
-
-public class ColumnConditionTest
-{
- private static final CellPath LIST_PATH = CellPath.create(ByteBuffer.wrap(UUIDGen.getTimeUUIDBytes()));
-
- public static final ByteBuffer ZERO = Int32Type.instance.fromString("0");
- public static final ByteBuffer ONE = Int32Type.instance.fromString("1");
- public static final ByteBuffer TWO = Int32Type.instance.fromString("2");
-
- public static final ByteBuffer A = AsciiType.instance.fromString("a");
- public static final ByteBuffer B = AsciiType.instance.fromString("b");
-
- @BeforeClass
- public static void setupDD()
- {
- DatabaseDescriptor.daemonInitialization();
- }
-
- private static boolean isSatisfiedBy(ColumnCondition.Bound bound, ByteBuffer conditionValue, ByteBuffer columnValue) throws InvalidRequestException
- {
- Cell cell = null;
- if (columnValue != null)
- {
- ColumnDefinition definition = ColumnDefinition.regularDef("ks", "cf", "c", ListType.getInstance(Int32Type.instance, true));
- cell = testCell(definition, columnValue, LIST_PATH);
- }
- return bound.isSatisfiedByValue(conditionValue, cell, Int32Type.instance, bound.operator);
- }
-
- private static Cell testCell(ColumnDefinition column, ByteBuffer value, CellPath path)
- {
- return new BufferCell(column, 0L, Cell.NO_TTL, Cell.NO_DELETION_TIME, value, path);
- }
-
- private static void assertThrowsIRE(ColumnCondition.Bound bound, ByteBuffer conditionValue, ByteBuffer columnValue)
- {
- try
- {
- isSatisfiedBy(bound, conditionValue, columnValue);
- fail("Expected InvalidRequestException was not thrown");
- } catch (InvalidRequestException e) { }
- }
-
- @Test
- public void testSimpleBoundIsSatisfiedByValue() throws InvalidRequestException
- {
- ColumnDefinition definition = ColumnDefinition.regularDef("ks", "cf", "c", ListType.getInstance(Int32Type.instance, true));
-
- // EQ
- ColumnCondition condition = ColumnCondition.condition(definition, new Constants.Value(ONE), Operator.EQ);
- ColumnCondition.Bound bound = condition.bind(QueryOptions.DEFAULT);
- assertTrue(isSatisfiedBy(bound, ONE, ONE));
- assertFalse(isSatisfiedBy(bound, ZERO, ONE));
- assertFalse(isSatisfiedBy(bound, TWO, ONE));
- assertFalse(isSatisfiedBy(bound, ByteBufferUtil.EMPTY_BYTE_BUFFER, ONE));
- assertFalse(isSatisfiedBy(bound, ONE, ByteBufferUtil.EMPTY_BYTE_BUFFER));
- assertTrue(isSatisfiedBy(bound, ByteBufferUtil.EMPTY_BYTE_BUFFER, ByteBufferUtil.EMPTY_BYTE_BUFFER));
- assertTrue(isSatisfiedBy(bound, null, null));
- assertFalse(isSatisfiedBy(bound, ONE, null));
- assertFalse(isSatisfiedBy(bound, null, ONE));
-
- // NEQ
- condition = ColumnCondition.condition(definition, new Constants.Value(ONE), Operator.NEQ);
- bound = condition.bind(QueryOptions.DEFAULT);
- assertFalse(isSatisfiedBy(bound, ONE, ONE));
- assertTrue(isSatisfiedBy(bound, ZERO, ONE));
- assertTrue(isSatisfiedBy(bound, TWO, ONE));
- assertTrue(isSatisfiedBy(bound, ByteBufferUtil.EMPTY_BYTE_BUFFER, ONE));
- assertTrue(isSatisfiedBy(bound, ONE, ByteBufferUtil.EMPTY_BYTE_BUFFER));
- assertFalse(isSatisfiedBy(bound, ByteBufferUtil.EMPTY_BYTE_BUFFER, ByteBufferUtil.EMPTY_BYTE_BUFFER));
- assertFalse(isSatisfiedBy(bound, null, null));
- assertTrue(isSatisfiedBy(bound, ONE, null));
- assertTrue(isSatisfiedBy(bound, null, ONE));
-
- // LT
- condition = ColumnCondition.condition(definition, new Constants.Value(ONE), Operator.LT);
- bound = condition.bind(QueryOptions.DEFAULT);
- assertFalse(isSatisfiedBy(bound, ONE, ONE));
- assertFalse(isSatisfiedBy(bound, ZERO, ONE));
- assertTrue(isSatisfiedBy(bound, TWO, ONE));
- assertFalse(isSatisfiedBy(bound, ByteBufferUtil.EMPTY_BYTE_BUFFER, ONE));
- assertTrue(isSatisfiedBy(bound, ONE, ByteBufferUtil.EMPTY_BYTE_BUFFER));
- assertFalse(isSatisfiedBy(bound, ByteBufferUtil.EMPTY_BYTE_BUFFER, ByteBufferUtil.EMPTY_BYTE_BUFFER));
- assertThrowsIRE(bound, null, ONE);
- assertFalse(isSatisfiedBy(bound, ONE, null));
-
- // LTE
- condition = ColumnCondition.condition(definition, new Constants.Value(ONE), Operator.LTE);
- bound = condition.bind(QueryOptions.DEFAULT);
- assertTrue(isSatisfiedBy(bound, ONE, ONE));
- assertFalse(isSatisfiedBy(bound, ZERO, ONE));
- assertTrue(isSatisfiedBy(bound, TWO, ONE));
- assertFalse(isSatisfiedBy(bound, ByteBufferUtil.EMPTY_BYTE_BUFFER, ONE));
- assertTrue(isSatisfiedBy(bound, ONE, ByteBufferUtil.EMPTY_BYTE_BUFFER));
- assertTrue(isSatisfiedBy(bound, ByteBufferUtil.EMPTY_BYTE_BUFFER, ByteBufferUtil.EMPTY_BYTE_BUFFER));
- assertThrowsIRE(bound, null, ONE);
- assertFalse(isSatisfiedBy(bound, ONE, null));
-
- // GT
- condition = ColumnCondition.condition(definition, new Constants.Value(ONE), Operator.GT);
- bound = condition.bind(QueryOptions.DEFAULT);
- assertFalse(isSatisfiedBy(bound, ONE, ONE));
- assertTrue(isSatisfiedBy(bound, ZERO, ONE));
- assertFalse(isSatisfiedBy(bound, TWO, ONE));
- assertTrue(isSatisfiedBy(bound, ByteBufferUtil.EMPTY_BYTE_BUFFER, ONE));
- assertFalse(isSatisfiedBy(bound, ONE, ByteBufferUtil.EMPTY_BYTE_BUFFER));
- assertFalse(isSatisfiedBy(bound, ByteBufferUtil.EMPTY_BYTE_BUFFER, ByteBufferUtil.EMPTY_BYTE_BUFFER));
- assertThrowsIRE(bound, null, ONE);
- assertFalse(isSatisfiedBy(bound, ONE, null));
-
- // GT
- condition = ColumnCondition.condition(definition, new Constants.Value(ONE), Operator.GTE);
- bound = condition.bind(QueryOptions.DEFAULT);
- assertTrue(isSatisfiedBy(bound, ONE, ONE));
- assertTrue(isSatisfiedBy(bound, ZERO, ONE));
- assertFalse(isSatisfiedBy(bound, TWO, ONE));
- assertTrue(isSatisfiedBy(bound, ByteBufferUtil.EMPTY_BYTE_BUFFER, ONE));
- assertFalse(isSatisfiedBy(bound, ONE, ByteBufferUtil.EMPTY_BYTE_BUFFER));
- assertTrue(isSatisfiedBy(bound, ByteBufferUtil.EMPTY_BYTE_BUFFER, ByteBufferUtil.EMPTY_BYTE_BUFFER));
- assertThrowsIRE(bound, null, ONE);
- assertFalse(isSatisfiedBy(bound, ONE, null));
- }
-
- private static List<ByteBuffer> list(ByteBuffer... values)
- {
- return Arrays.asList(values);
- }
-
- private static boolean listAppliesTo(ColumnCondition.CollectionBound bound, List<ByteBuffer> conditionValues, List<ByteBuffer> columnValues)
- {
- CFMetaData cfm = CFMetaData.compile("create table foo(a int PRIMARY KEY, b int, c list<int>)", "ks");
- Map<ByteBuffer, CollectionType> typeMap = new HashMap<>();
- typeMap.put(ByteBufferUtil.bytes("c"), ListType.getInstance(Int32Type.instance, true));
-
- ColumnDefinition definition = ColumnDefinition.regularDef(cfm, ByteBufferUtil.bytes("c"), ListType.getInstance(Int32Type.instance, true));
-
- List<Cell> cells = new ArrayList<>(columnValues.size());
- if (columnValues != null)
- {
- for (int i = 0; i < columnValues.size(); i++)
- {
- ByteBuffer key = Int32Serializer.instance.serialize(i);
- ByteBuffer value = columnValues.get(i);
- cells.add(testCell(definition, value, CellPath.create(key)));
- };
- }
-
- return bound.listAppliesTo(ListType.getInstance(Int32Type.instance, true), cells == null ? null : cells.iterator(), conditionValues, bound.operator);
- }
-
- @Test
- // sets use the same check as lists
- public void testListCollectionBoundAppliesTo() throws InvalidRequestException
- {
- ColumnDefinition definition = ColumnDefinition.regularDef("ks", "cf", "c", ListType.getInstance(Int32Type.instance, true));
-
- // EQ
- ColumnCondition condition = ColumnCondition.condition(definition, new Lists.Value(Arrays.asList(ONE)), Operator.EQ);
- ColumnCondition.CollectionBound bound = (ColumnCondition.CollectionBound) condition.bind(QueryOptions.DEFAULT);
- assertTrue(listAppliesTo(bound, list(ONE), list(ONE)));
- assertTrue(listAppliesTo(bound, list(), list()));
- assertFalse(listAppliesTo(bound, list(ZERO), list(ONE)));
- assertFalse(listAppliesTo(bound, list(ONE), list(ZERO)));
- assertFalse(listAppliesTo(bound, list(ONE), list(ONE, ONE)));
- assertFalse(listAppliesTo(bound, list(ONE, ONE), list(ONE)));
- assertFalse(listAppliesTo(bound, list(), list(ONE)));
- assertFalse(listAppliesTo(bound, list(ONE), list()));
-
- assertFalse(listAppliesTo(bound, list(ByteBufferUtil.EMPTY_BYTE_BUFFER), list(ONE)));
- assertFalse(listAppliesTo(bound, list(ONE), list(ByteBufferUtil.EMPTY_BYTE_BUFFER)));
- assertTrue(listAppliesTo(bound, list(ByteBufferUtil.EMPTY_BYTE_BUFFER), list(ByteBufferUtil.EMPTY_BYTE_BUFFER)));
-
- // NEQ
- condition = ColumnCondition.condition(definition, new Lists.Value(Arrays.asList(ONE)), Operator.NEQ);
- bound = (ColumnCondition.CollectionBound) condition.bind(QueryOptions.DEFAULT);
- assertFalse(listAppliesTo(bound, list(ONE), list(ONE)));
- assertFalse(listAppliesTo(bound, list(), list()));
- assertTrue(listAppliesTo(bound, list(ZERO), list(ONE)));
- assertTrue(listAppliesTo(bound, list(ONE), list(ZERO)));
- assertTrue(listAppliesTo(bound, list(ONE), list(ONE, ONE)));
- assertTrue(listAppliesTo(bound, list(ONE, ONE), list(ONE)));
- assertTrue(listAppliesTo(bound, list(), list(ONE)));
- assertTrue(listAppliesTo(bound, list(ONE), list()));
-
- assertTrue(listAppliesTo(bound, list(ByteBufferUtil.EMPTY_BYTE_BUFFER), list(ONE)));
- assertTrue(listAppliesTo(bound, list(ONE), list(ByteBufferUtil.EMPTY_BYTE_BUFFER)));
- assertFalse(listAppliesTo(bound, list(ByteBufferUtil.EMPTY_BYTE_BUFFER), list(ByteBufferUtil.EMPTY_BYTE_BUFFER)));
-
- // LT
- condition = ColumnCondition.condition(definition, new Lists.Value(Arrays.asList(ONE)), Operator.LT);
- bound = (ColumnCondition.CollectionBound) condition.bind(QueryOptions.DEFAULT);
- assertFalse(listAppliesTo(bound, list(ONE), list(ONE)));
- assertFalse(listAppliesTo(bound, list(), list()));
- assertFalse(listAppliesTo(bound, list(ZERO), list(ONE)));
- assertTrue(listAppliesTo(bound, list(ONE), list(ZERO)));
- assertFalse(listAppliesTo(bound, list(ONE), list(ONE, ONE)));
- assertTrue(listAppliesTo(bound, list(ONE, ONE), list(ONE)));
- assertFalse(listAppliesTo(bound, list(), list(ONE)));
- assertTrue(listAppliesTo(bound, list(ONE), list()));
-
- assertFalse(listAppliesTo(bound, list(ByteBufferUtil.EMPTY_BYTE_BUFFER), list(ONE)));
- assertTrue(listAppliesTo(bound, list(ONE), list(ByteBufferUtil.EMPTY_BYTE_BUFFER)));
- assertFalse(listAppliesTo(bound, list(ByteBufferUtil.EMPTY_BYTE_BUFFER), list(ByteBufferUtil.EMPTY_BYTE_BUFFER)));
-
- // LTE
- condition = ColumnCondition.condition(definition, new Lists.Value(Arrays.asList(ONE)), Operator.LTE);
- bound = (ColumnCondition.CollectionBound) condition.bind(QueryOptions.DEFAULT);
- assertTrue(listAppliesTo(bound, list(ONE), list(ONE)));
- assertTrue(listAppliesTo(bound, list(), list()));
- assertFalse(listAppliesTo(bound, list(ZERO), list(ONE)));
- assertTrue(listAppliesTo(bound, list(ONE), list(ZERO)));
- assertFalse(listAppliesTo(bound, list(ONE), list(ONE, ONE)));
- assertTrue(listAppliesTo(bound, list(ONE, ONE), list(ONE)));
- assertFalse(listAppliesTo(bound, list(), list(ONE)));
- assertTrue(listAppliesTo(bound, list(ONE), list()));
-
- assertFalse(listAppliesTo(bound, list(ByteBufferUtil.EMPTY_BYTE_BUFFER), list(ONE)));
- assertTrue(listAppliesTo(bound, list(ONE), list(ByteBufferUtil.EMPTY_BYTE_BUFFER)));
- assertTrue(listAppliesTo(bound, list(ByteBufferUtil.EMPTY_BYTE_BUFFER), list(ByteBufferUtil.EMPTY_BYTE_BUFFER)));
-
- // GT
- condition = ColumnCondition.condition(definition, new Lists.Value(Arrays.asList(ONE)), Operator.GT);
- bound = (ColumnCondition.CollectionBound) condition.bind(QueryOptions.DEFAULT);
- assertFalse(listAppliesTo(bound, list(ONE), list(ONE)));
- assertFalse(listAppliesTo(bound, list(), list()));
- assertTrue(listAppliesTo(bound, list(ZERO), list(ONE)));
- assertFalse(listAppliesTo(bound, list(ONE), list(ZERO)));
- assertTrue(listAppliesTo(bound, list(ONE), list(ONE, ONE)));
- assertFalse(listAppliesTo(bound, list(ONE, ONE), list(ONE)));
- assertTrue(listAppliesTo(bound, list(), list(ONE)));
- assertFalse(listAppliesTo(bound, list(ONE), list()));
-
- assertTrue(listAppliesTo(bound, list(ByteBufferUtil.EMPTY_BYTE_BUFFER), list(ONE)));
- assertFalse(listAppliesTo(bound, list(ONE), list(ByteBufferUtil.EMPTY_BYTE_BUFFER)));
- assertFalse(listAppliesTo(bound, list(ByteBufferUtil.EMPTY_BYTE_BUFFER), list(ByteBufferUtil.EMPTY_BYTE_BUFFER)));
-
- // GTE
- condition = ColumnCondition.condition(definition, new Lists.Value(Arrays.asList(ONE)), Operator.GTE);
- bound = (ColumnCondition.CollectionBound) condition.bind(QueryOptions.DEFAULT);
- assertTrue(listAppliesTo(bound, list(ONE), list(ONE)));
- assertTrue(listAppliesTo(bound, list(), list()));
- assertTrue(listAppliesTo(bound, list(ZERO), list(ONE)));
- assertFalse(listAppliesTo(bound, list(ONE), list(ZERO)));
- assertTrue(listAppliesTo(bound, list(ONE), list(ONE, ONE)));
- assertFalse(listAppliesTo(bound, list(ONE, ONE), list(ONE)));
- assertTrue(listAppliesTo(bound, list(), list(ONE)));
- assertFalse(listAppliesTo(bound, list(ONE), list()));
-
- assertTrue(listAppliesTo(bound, list(ByteBufferUtil.EMPTY_BYTE_BUFFER), list(ONE)));
- assertFalse(listAppliesTo(bound, list(ONE), list(ByteBufferUtil.EMPTY_BYTE_BUFFER)));
- assertTrue(listAppliesTo(bound, list(ByteBufferUtil.EMPTY_BYTE_BUFFER), list(ByteBufferUtil.EMPTY_BYTE_BUFFER)));
- }
-
- private static SortedSet<ByteBuffer> set(ByteBuffer... values)
- {
- SortedSet<ByteBuffer> results = new TreeSet<>(Int32Type.instance);
- results.addAll(Arrays.asList(values));
- return results;
- }
-
- private static boolean setAppliesTo(ColumnCondition.CollectionBound bound, Set<ByteBuffer> conditionValues, List<ByteBuffer> columnValues)
- {
- CFMetaData cfm = CFMetaData.compile("create table foo(a int PRIMARY KEY, b int, c set<int>)", "ks");
- Map<ByteBuffer, CollectionType> typeMap = new HashMap<>();
- typeMap.put(ByteBufferUtil.bytes("c"), SetType.getInstance(Int32Type.instance, true));
- ColumnDefinition definition = ColumnDefinition.regularDef(cfm, ByteBufferUtil.bytes("c"), SetType.getInstance(Int32Type.instance, true));
-
- List<Cell> cells = new ArrayList<>(columnValues.size());
- if (columnValues != null)
- {
- for (int i = 0; i < columnValues.size(); i++)
- {
- ByteBuffer key = columnValues.get(i);
- cells.add(testCell(definition, ByteBufferUtil.EMPTY_BYTE_BUFFER, CellPath.create(key)));
- };
- }
-
- return bound.setAppliesTo(SetType.getInstance(Int32Type.instance, true), cells == null ? null : cells.iterator(), conditionValues, bound.operator);
- }
-
- @Test
- public void testSetCollectionBoundAppliesTo() throws InvalidRequestException
- {
- ColumnDefinition definition = ColumnDefinition.regularDef("ks", "cf", "c", ListType.getInstance(Int32Type.instance, true));
-
- // EQ
- ColumnCondition condition = ColumnCondition.condition(definition, new Sets.Value(set(ONE)), Operator.EQ);
- ColumnCondition.CollectionBound bound = (ColumnCondition.CollectionBound) condition.bind(QueryOptions.DEFAULT);
- assertTrue(setAppliesTo(bound, set(ONE), list(ONE)));
- assertTrue(setAppliesTo(bound, set(), list()));
- assertFalse(setAppliesTo(bound, set(ZERO), list(ONE)));
- assertFalse(setAppliesTo(bound, set(ONE), list(ZERO)));
- assertFalse(setAppliesTo(bound, set(ONE), list(ONE, TWO)));
- assertFalse(setAppliesTo(bound, set(ONE, TWO), list(ONE)));
- assertFalse(setAppliesTo(bound, set(), list(ONE)));
- assertFalse(setAppliesTo(bound, set(ONE), list()));
-
- assertFalse(setAppliesTo(bound, set(ByteBufferUtil.EMPTY_BYTE_BUFFER), list(ONE)));
- assertFalse(setAppliesTo(bound, set(ONE), list(ByteBufferUtil.EMPTY_BYTE_BUFFER)));
- assertTrue(setAppliesTo(bound, set(ByteBufferUtil.EMPTY_BYTE_BUFFER), list(ByteBufferUtil.EMPTY_BYTE_BUFFER)));
-
- // NEQ
- condition = ColumnCondition.condition(definition, new Sets.Value(set(ONE)), Operator.NEQ);
- bound = (ColumnCondition.CollectionBound) condition.bind(QueryOptions.DEFAULT);
- assertFalse(setAppliesTo(bound, set(ONE), list(ONE)));
- assertFalse(setAppliesTo(bound, set(), list()));
- assertTrue(setAppliesTo(bound, set(ZERO), list(ONE)));
- assertTrue(setAppliesTo(bound, set(ONE), list(ZERO)));
- assertTrue(setAppliesTo(bound, set(ONE), list(ONE, TWO)));
- assertTrue(setAppliesTo(bound, set(ONE, TWO), list(ONE)));
- assertTrue(setAppliesTo(bound, set(), list(ONE)));
- assertTrue(setAppliesTo(bound, set(ONE), list()));
-
- assertTrue(setAppliesTo(bound, set(ByteBufferUtil.EMPTY_BYTE_BUFFER), list(ONE)));
- assertTrue(setAppliesTo(bound, set(ONE), list(ByteBufferUtil.EMPTY_BYTE_BUFFER)));
- assertFalse(setAppliesTo(bound, set(ByteBufferUtil.EMPTY_BYTE_BUFFER), list(ByteBufferUtil.EMPTY_BYTE_BUFFER)));
-
- // LT
- condition = ColumnCondition.condition(definition, new Lists.Value(Arrays.asList(ONE)), Operator.LT);
- bound = (ColumnCondition.CollectionBound) condition.bind(QueryOptions.DEFAULT);
- assertFalse(setAppliesTo(bound, set(ONE), list(ONE)));
- assertFalse(setAppliesTo(bound, set(), list()));
- assertFalse(setAppliesTo(bound, set(ZERO), list(ONE)));
- assertTrue(setAppliesTo(bound, set(ONE), list(ZERO)));
- assertFalse(setAppliesTo(bound, set(ONE), list(ONE, TWO)));
- assertTrue(setAppliesTo(bound, set(ONE, TWO), list(ONE)));
- assertFalse(setAppliesTo(bound, set(), list(ONE)));
- assertTrue(setAppliesTo(bound, set(ONE), list()));
-
- assertFalse(setAppliesTo(bound, set(ByteBufferUtil.EMPTY_BYTE_BUFFER), list(ONE)));
- assertTrue(setAppliesTo(bound, set(ONE), list(ByteBufferUtil.EMPTY_BYTE_BUFFER)));
- assertFalse(setAppliesTo(bound, set(ByteBufferUtil.EMPTY_BYTE_BUFFER), list(ByteBufferUtil.EMPTY_BYTE_BUFFER)));
-
- // LTE
- condition = ColumnCondition.condition(definition, new Lists.Value(Arrays.asList(ONE)), Operator.LTE);
- bound = (ColumnCondition.CollectionBound) condition.bind(QueryOptions.DEFAULT);
- assertTrue(setAppliesTo(bound, set(ONE), list(ONE)));
- assertTrue(setAppliesTo(bound, set(), list()));
- assertFalse(setAppliesTo(bound, set(ZERO), list(ONE)));
- assertTrue(setAppliesTo(bound, set(ONE), list(ZERO)));
- assertFalse(setAppliesTo(bound, set(ONE), list(ONE, TWO)));
- assertTrue(setAppliesTo(bound, set(ONE, TWO), list(ONE)));
- assertFalse(setAppliesTo(bound, set(), list(ONE)));
- assertTrue(setAppliesTo(bound, set(ONE), list()));
-
- assertFalse(setAppliesTo(bound, set(ByteBufferUtil.EMPTY_BYTE_BUFFER), list(ONE)));
- assertTrue(setAppliesTo(bound, set(ONE), list(ByteBufferUtil.EMPTY_BYTE_BUFFER)));
- assertTrue(setAppliesTo(bound, set(ByteBufferUtil.EMPTY_BYTE_BUFFER), list(ByteBufferUtil.EMPTY_BYTE_BUFFER)));
-
- // GT
- condition = ColumnCondition.condition(definition, new Lists.Value(Arrays.asList(ONE)), Operator.GT);
- bound = (ColumnCondition.CollectionBound) condition.bind(QueryOptions.DEFAULT);
- assertFalse(setAppliesTo(bound, set(ONE), list(ONE)));
- assertFalse(setAppliesTo(bound, set(), list()));
- assertTrue(setAppliesTo(bound, set(ZERO), list(ONE)));
- assertFalse(setAppliesTo(bound, set(ONE), list(ZERO)));
- assertTrue(setAppliesTo(bound, set(ONE), list(ONE, TWO)));
- assertFalse(setAppliesTo(bound, set(ONE, TWO), list(ONE)));
- assertTrue(setAppliesTo(bound, set(), list(ONE)));
- assertFalse(setAppliesTo(bound, set(ONE), list()));
-
- assertTrue(setAppliesTo(bound, set(ByteBufferUtil.EMPTY_BYTE_BUFFER), list(ONE)));
- assertFalse(setAppliesTo(bound, set(ONE), list(ByteBufferUtil.EMPTY_BYTE_BUFFER)));
- assertFalse(setAppliesTo(bound, set(ByteBufferUtil.EMPTY_BYTE_BUFFER), list(ByteBufferUtil.EMPTY_BYTE_BUFFER)));
-
- // GTE
- condition = ColumnCondition.condition(definition, new Lists.Value(Arrays.asList(ONE)), Operator.GTE);
- bound = (ColumnCondition.CollectionBound) condition.bind(QueryOptions.DEFAULT);
- assertTrue(setAppliesTo(bound, set(ONE), list(ONE)));
- assertTrue(setAppliesTo(bound, set(), list()));
- assertTrue(setAppliesTo(bound, set(ZERO), list(ONE)));
- assertFalse(setAppliesTo(bound, set(ONE), list(ZERO)));
- assertTrue(setAppliesTo(bound, set(ONE), list(ONE, TWO)));
- assertFalse(setAppliesTo(bound, set(ONE, TWO), list(ONE)));
- assertTrue(setAppliesTo(bound, set(), list(ONE)));
- assertFalse(setAppliesTo(bound, set(ONE), list()));
-
- assertTrue(setAppliesTo(bound, set(ByteBufferUtil.EMPTY_BYTE_BUFFER), list(ONE)));
- assertFalse(setAppliesTo(bound, set(ONE), list(ByteBufferUtil.EMPTY_BYTE_BUFFER)));
- assertTrue(setAppliesTo(bound, set(ByteBufferUtil.EMPTY_BYTE_BUFFER), list(ByteBufferUtil.EMPTY_BYTE_BUFFER)));
- }
-
- // values should be a list of key, value, key, value, ...
- private static Map<ByteBuffer, ByteBuffer> map(ByteBuffer... values)
- {
- Map<ByteBuffer, ByteBuffer> map = new TreeMap<>();
- for (int i = 0; i < values.length; i += 2)
- map.put(values[i], values[i + 1]);
-
- return map;
- }
-
- private static boolean mapAppliesTo(ColumnCondition.CollectionBound bound, Map<ByteBuffer, ByteBuffer> conditionValues, Map<ByteBuffer, ByteBuffer> columnValues)
- {
- CFMetaData cfm = CFMetaData.compile("create table foo(a int PRIMARY KEY, b map<int, int>)", "ks");
- Map<ByteBuffer, CollectionType> typeMap = new HashMap<>();
- typeMap.put(ByteBufferUtil.bytes("b"), MapType.getInstance(Int32Type.instance, Int32Type.instance, true));
- ColumnDefinition definition = ColumnDefinition.regularDef(cfm, ByteBufferUtil.bytes("b"), MapType.getInstance(Int32Type.instance, Int32Type.instance, true));
-
- List<Cell> cells = new ArrayList<>(columnValues.size());
- if (columnValues != null)
- {
- for (Map.Entry<ByteBuffer, ByteBuffer> entry : columnValues.entrySet())
- cells.add(testCell(definition, entry.getValue(), CellPath.create(entry.getKey())));
- }
-
- return bound.mapAppliesTo(MapType.getInstance(Int32Type.instance, Int32Type.instance, true), cells.iterator(), conditionValues, bound.operator);
- }
-
- @Test
- public void testMapCollectionBoundIsSatisfiedByValue() throws InvalidRequestException
- {
- ColumnDefinition definition = ColumnDefinition.regularDef("ks", "cf", "c", ListType.getInstance(Int32Type.instance, true));
-
- Map<ByteBuffer, ByteBuffer> placeholderMap = new TreeMap<>();
- placeholderMap.put(ONE, ONE);
- Maps.Value placeholder = new Maps.Value(placeholderMap);
-
- // EQ
- ColumnCondition condition = ColumnCondition.condition(definition, placeholder, Operator.EQ);
- ColumnCondition.CollectionBound bound = (ColumnCondition.CollectionBound) condition.bind(QueryOptions.DEFAULT);
-
- assertTrue(mapAppliesTo(bound, map(ONE, ONE), map(ONE, ONE)));
- assertTrue(mapAppliesTo(bound, map(), map()));
- assertFalse(mapAppliesTo(bound, map(ZERO, ONE), map(ONE, ONE)));
- assertFalse(mapAppliesTo(bound, map(ONE, ONE), map(ZERO, ONE)));
- assertFalse(mapAppliesTo(bound, map(ONE, ZERO), map(ONE, ONE)));
- assertFalse(mapAppliesTo(bound, map(ONE, ONE), map(ONE, ZERO)));
- assertFalse(mapAppliesTo(bound, map(ONE, ONE), map(ONE, ONE, TWO, ONE)));
- assertFalse(mapAppliesTo(bound, map(ONE, ONE, TWO, ONE), map(ONE, ONE)));
- assertFalse(mapAppliesTo(bound, map(), map(ONE, ONE)));
- assertFalse(mapAppliesTo(bound, map(ONE, ONE), map()));
-
- assertFalse(mapAppliesTo(bound, map(ByteBufferUtil.EMPTY_BYTE_BUFFER, ONE), map(ONE, ONE)));
- assertFalse(mapAppliesTo(bound, map(ONE, ONE), map(ByteBufferUtil.EMPTY_BYTE_BUFFER, ONE)));
- assertFalse(mapAppliesTo(bound, map(ONE, ByteBufferUtil.EMPTY_BYTE_BUFFER), map(ONE, ONE)));
- assertFalse(mapAppliesTo(bound, map(ONE, ONE), map(ONE, ByteBufferUtil.EMPTY_BYTE_BUFFER)));
- assertTrue(mapAppliesTo(bound, map(ByteBufferUtil.EMPTY_BYTE_BUFFER, ONE), map(ByteBufferUtil.EMPTY_BYTE_BUFFER, ONE)));
- assertTrue(mapAppliesTo(bound, map(ONE, ByteBufferUtil.EMPTY_BYTE_BUFFER), map(ONE, ByteBufferUtil.EMPTY_BYTE_BUFFER)));
-
- // NEQ
- condition = ColumnCondition.condition(definition, placeholder, Operator.NEQ);
- bound = (ColumnCondition.CollectionBound) condition.bind(QueryOptions.DEFAULT);
-
- assertFalse(mapAppliesTo(bound, map(ONE, ONE), map(ONE, ONE)));
- assertFalse(mapAppliesTo(bound, map(), map()));
- assertTrue(mapAppliesTo(bound, map(ZERO, ONE), map(ONE, ONE)));
- assertTrue(mapAppliesTo(bound, map(ONE, ONE), map(ZERO, ONE)));
- assertTrue(mapAppliesTo(bound, map(ONE, ZERO), map(ONE, ONE)));
- assertTrue(mapAppliesTo(bound, map(ONE, ONE), map(ONE, ZERO)));
- assertTrue(mapAppliesTo(bound, map(ONE, ONE), map(ONE, ONE, TWO, ONE)));
- assertTrue(mapAppliesTo(bound, map(ONE, ONE, TWO, ONE), map(ONE, ONE)));
- assertTrue(mapAppliesTo(bound, map(), map(ONE, ONE)));
- assertTrue(mapAppliesTo(bound, map(ONE, ONE), map()));
-
- assertTrue(mapAppliesTo(bound, map(ByteBufferUtil.EMPTY_BYTE_BUFFER, ONE), map(ONE, ONE)));
- assertTrue(mapAppliesTo(bound, map(ONE, ONE), map(ByteBufferUtil.EMPTY_BYTE_BUFFER, ONE)));
- assertTrue(mapAppliesTo(bound, map(ONE, ByteBufferUtil.EMPTY_BYTE_BUFFER), map(ONE, ONE)));
- assertTrue(mapAppliesTo(bound, map(ONE, ONE), map(ONE, ByteBufferUtil.EMPTY_BYTE_BUFFER)));
- assertFalse(mapAppliesTo(bound, map(ByteBufferUtil.EMPTY_BYTE_BUFFER, ONE), map(ByteBufferUtil.EMPTY_BYTE_BUFFER, ONE)));
- assertFalse(mapAppliesTo(bound, map(ONE, ByteBufferUtil.EMPTY_BYTE_BUFFER), map(ONE, ByteBufferUtil.EMPTY_BYTE_BUFFER)));
-
- // LT
- condition = ColumnCondition.condition(definition, placeholder, Operator.LT);
- bound = (ColumnCondition.CollectionBound) condition.bind(QueryOptions.DEFAULT);
-
- assertFalse(mapAppliesTo(bound, map(ONE, ONE), map(ONE, ONE)));
- assertFalse(mapAppliesTo(bound, map(), map()));
- assertFalse(mapAppliesTo(bound, map(ZERO, ONE), map(ONE, ONE)));
- assertTrue(mapAppliesTo(bound, map(ONE, ONE), map(ZERO, ONE)));
- assertFalse(mapAppliesTo(bound, map(ONE, ZERO), map(ONE, ONE)));
- assertTrue(mapAppliesTo(bound, map(ONE, ONE), map(ONE, ZERO)));
- assertFalse(mapAppliesTo(bound, map(ONE, ONE), map(ONE, ONE, TWO, ONE)));
- assertTrue(mapAppliesTo(bound, map(ONE, ONE, TWO, ONE), map(ONE, ONE)));
- assertFalse(mapAppliesTo(bound, map(), map(ONE, ONE)));
- assertTrue(mapAppliesTo(bound, map(ONE, ONE), map()));
-
- assertFalse(mapAppliesTo(bound, map(ByteBufferUtil.EMPTY_BYTE_BUFFER, ONE), map(ONE, ONE)));
- assertTrue(mapAppliesTo(bound, map(ONE, ONE), map(ByteBufferUtil.EMPTY_BYTE_BUFFER, ONE)));
- assertFalse(mapAppliesTo(bound, map(ONE, ByteBufferUtil.EMPTY_BYTE_BUFFER), map(ONE, ONE)));
- assertTrue(mapAppliesTo(bound, map(ONE, ONE), map(ONE, ByteBufferUtil.EMPTY_BYTE_BUFFER)));
- assertFalse(mapAppliesTo(bound, map(ByteBufferUtil.EMPTY_BYTE_BUFFER, ONE), map(ByteBufferUtil.EMPTY_BYTE_BUFFER, ONE)));
- assertFalse(mapAppliesTo(bound, map(ONE, ByteBufferUtil.EMPTY_BYTE_BUFFER), map(ONE, ByteBufferUtil.EMPTY_BYTE_BUFFER)));
-
- // LTE
- condition = ColumnCondition.condition(definition, placeholder, Operator.LTE);
- bound = (ColumnCondition.CollectionBound) condition.bind(QueryOptions.DEFAULT);
-
- assertTrue(mapAppliesTo(bound, map(ONE, ONE), map(ONE, ONE)));
- assertTrue(mapAppliesTo(bound, map(), map()));
- assertFalse(mapAppliesTo(bound, map(ZERO, ONE), map(ONE, ONE)));
- assertTrue(mapAppliesTo(bound, map(ONE, ONE), map(ZERO, ONE)));
- assertFalse(mapAppliesTo(bound, map(ONE, ZERO), map(ONE, ONE)));
- assertTrue(mapAppliesTo(bound, map(ONE, ONE), map(ONE, ZERO)));
- assertFalse(mapAppliesTo(bound, map(ONE, ONE), map(ONE, ONE, TWO, ONE)));
- assertTrue(mapAppliesTo(bound, map(ONE, ONE, TWO, ONE), map(ONE, ONE)));
- assertFalse(mapAppliesTo(bound, map(), map(ONE, ONE)));
- assertTrue(mapAppliesTo(bound, map(ONE, ONE), map()));
-
- assertFalse(mapAppliesTo(bound, map(ByteBufferUtil.EMPTY_BYTE_BUFFER, ONE), map(ONE, ONE)));
- assertTrue(mapAppliesTo(bound, map(ONE, ONE), map(ByteBufferUtil.EMPTY_BYTE_BUFFER, ONE)));
- assertFalse(mapAppliesTo(bound, map(ONE, ByteBufferUtil.EMPTY_BYTE_BUFFER), map(ONE, ONE)));
- assertTrue(mapAppliesTo(bound, map(ONE, ONE), map(ONE, ByteBufferUtil.EMPTY_BYTE_BUFFER)));
- assertTrue(mapAppliesTo(bound, map(ByteBufferUtil.EMPTY_BYTE_BUFFER, ONE), map(ByteBufferUtil.EMPTY_BYTE_BUFFER, ONE)));
- assertTrue(mapAppliesTo(bound, map(ONE, ByteBufferUtil.EMPTY_BYTE_BUFFER), map(ONE, ByteBufferUtil.EMPTY_BYTE_BUFFER)));
-
- // GT
- condition = ColumnCondition.condition(definition, placeholder, Operator.GT);
- bound = (ColumnCondition.CollectionBound) condition.bind(QueryOptions.DEFAULT);
-
- assertFalse(mapAppliesTo(bound, map(ONE, ONE), map(ONE, ONE)));
- assertFalse(mapAppliesTo(bound, map(), map()));
- assertTrue(mapAppliesTo(bound, map(ZERO, ONE), map(ONE, ONE)));
- assertFalse(mapAppliesTo(bound, map(ONE, ONE), map(ZERO, ONE)));
- assertTrue(mapAppliesTo(bound, map(ONE, ZERO), map(ONE, ONE)));
- assertFalse(mapAppliesTo(bound, map(ONE, ONE), map(ONE, ZERO)));
- assertTrue(mapAppliesTo(bound, map(ONE, ONE), map(ONE, ONE, TWO, ONE)));
- assertFalse(mapAppliesTo(bound, map(ONE, ONE, TWO, ONE), map(ONE, ONE)));
- assertTrue(mapAppliesTo(bound, map(), map(ONE, ONE)));
- assertFalse(mapAppliesTo(bound, map(ONE, ONE), map()));
-
- assertTrue(mapAppliesTo(bound, map(ByteBufferUtil.EMPTY_BYTE_BUFFER, ONE), map(ONE, ONE)));
- assertFalse(mapAppliesTo(bound, map(ONE, ONE), map(ByteBufferUtil.EMPTY_BYTE_BUFFER, ONE)));
- assertTrue(mapAppliesTo(bound, map(ONE, ByteBufferUtil.EMPTY_BYTE_BUFFER), map(ONE, ONE)));
- assertFalse(mapAppliesTo(bound, map(ONE, ONE), map(ONE, ByteBufferUtil.EMPTY_BYTE_BUFFER)));
- assertFalse(mapAppliesTo(bound, map(ByteBufferUtil.EMPTY_BYTE_BUFFER, ONE), map(ByteBufferUtil.EMPTY_BYTE_BUFFER, ONE)));
- assertFalse(mapAppliesTo(bound, map(ONE, ByteBufferUtil.EMPTY_BYTE_BUFFER), map(ONE, ByteBufferUtil.EMPTY_BYTE_BUFFER)));
-
- // GTE
- condition = ColumnCondition.condition(definition, placeholder, Operator.GTE);
- bound = (ColumnCondition.CollectionBound) condition.bind(QueryOptions.DEFAULT);
-
- assertTrue(mapAppliesTo(bound, map(ONE, ONE), map(ONE, ONE)));
- assertTrue(mapAppliesTo(bound, map(), map()));
- assertTrue(mapAppliesTo(bound, map(ZERO, ONE), map(ONE, ONE)));
- assertFalse(mapAppliesTo(bound, map(ONE, ONE), map(ZERO, ONE)));
- assertTrue(mapAppliesTo(bound, map(ONE, ZERO), map(ONE, ONE)));
- assertFalse(mapAppliesTo(bound, map(ONE, ONE), map(ONE, ZERO)));
- assertTrue(mapAppliesTo(bound, map(ONE, ONE), map(ONE, ONE, TWO, ONE)));
- assertFalse(mapAppliesTo(bound, map(ONE, ONE, TWO, ONE), map(ONE, ONE)));
- assertTrue(mapAppliesTo(bound, map(), map(ONE, ONE)));
- assertFalse(mapAppliesTo(bound, map(ONE, ONE), map()));
-
- assertTrue(mapAppliesTo(bound, map(ByteBufferUtil.EMPTY_BYTE_BUFFER, ONE), map(ONE, ONE)));
- assertFalse(mapAppliesTo(bound, map(ONE, ONE), map(ByteBufferUtil.EMPTY_BYTE_BUFFER, ONE)));
- assertTrue(mapAppliesTo(bound, map(ONE, ByteBufferUtil.EMPTY_BYTE_BUFFER), map(ONE, ONE)));
- assertFalse(mapAppliesTo(bound, map(ONE, ONE), map(ONE, ByteBufferUtil.EMPTY_BYTE_BUFFER)));
- assertTrue(mapAppliesTo(bound, map(ByteBufferUtil.EMPTY_BYTE_BUFFER, ONE), map(ByteBufferUtil.EMPTY_BYTE_BUFFER, ONE)));
- assertTrue(mapAppliesTo(bound, map(ONE, ByteBufferUtil.EMPTY_BYTE_BUFFER), map(ONE, ByteBufferUtil.EMPTY_BYTE_BUFFER)));
- }
-}
diff --git a/test/unit/org/apache/cassandra/cql3/ColumnIdentifierTest.java b/test/unit/org/apache/cassandra/cql3/ColumnIdentifierTest.java
index ea483c4..c37d82a 100644
--- a/test/unit/org/apache/cassandra/cql3/ColumnIdentifierTest.java
+++ b/test/unit/org/apache/cassandra/cql3/ColumnIdentifierTest.java
@@ -23,7 +23,7 @@
import org.junit.Test;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.db.marshal.BytesType;
import org.apache.cassandra.db.marshal.UTF8Type;
diff --git a/test/unit/org/apache/cassandra/cql3/CustomNowInSecondsTest.java b/test/unit/org/apache/cassandra/cql3/CustomNowInSecondsTest.java
new file mode 100644
index 0000000..983acfa
--- /dev/null
+++ b/test/unit/org/apache/cassandra/cql3/CustomNowInSecondsTest.java
@@ -0,0 +1,213 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import com.google.common.collect.ImmutableList;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.cql3.statements.BatchStatement;
+import org.apache.cassandra.cql3.statements.ModificationStatement;
+import org.apache.cassandra.db.ConsistencyLevel;
+import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.service.QueryState;
+import org.apache.cassandra.transport.ProtocolVersion;
+import org.apache.cassandra.transport.messages.ResultMessage;
+
+import static java.lang.String.format;
+import static org.junit.Assert.assertEquals;
+
+public class CustomNowInSecondsTest extends CQLTester
+{
+ @BeforeClass
+ public static void setUpClass()
+ {
+ prepareServer();
+ requireNetwork();
+ }
+
+ @Test
+ public void testSelectQuery()
+ {
+ testSelectQuery(false);
+ testSelectQuery(true);
+ }
+
+ private void testSelectQuery(boolean prepared)
+ {
+ int day = 86400;
+
+ String ks = createKeyspace("CREATE KEYSPACE %s WITH replication={ 'class' : 'SimpleStrategy', 'replication_factor' : 1 }");
+ String tbl = createTable(ks, "CREATE TABLE %s (id int primary key, val int)");
+
+ // insert a row with TTL = 1 day.
+ executeModify(format("INSERT INTO %s.%s (id, val) VALUES (0, 0) USING TTL %d", ks, tbl, day), Integer.MIN_VALUE, prepared);
+
+ int now = (int) (System.currentTimeMillis() / 1000);
+
+ // execute a SELECT query without overriding nowInSeconds - make sure we observe one row.
+ assertEquals(1, executeSelect(format("SELECT * FROM %s.%s", ks, tbl), Integer.MIN_VALUE, prepared).size());
+
+ // execute a SELECT query with nowInSeconds set to [now + 1 day + 1], when the row should have expired.
+ assertEquals(0, executeSelect(format("SELECT * FROM %s.%s", ks, tbl), now + day + 1, prepared).size());
+ }
+
+ @Test
+ public void testModifyQuery()
+ {
+ testModifyQuery(false);
+ testModifyQuery(true);
+ }
+
+ private void testModifyQuery(boolean prepared)
+ {
+ int now = (int) (System.currentTimeMillis() / 1000);
+ int day = 86400;
+
+ String ks = createKeyspace("CREATE KEYSPACE %s WITH replication={ 'class' : 'SimpleStrategy', 'replication_factor' : 1 }");
+ String tbl = createTable(ks, "CREATE TABLE %s (id int primary key, val int)");
+
+ // execute an INSERT query with now set to [now + 1 day], with ttl = 1, making its effective ttl = 1 day + 1.
+ executeModify(format("INSERT INTO %s.%s (id, val) VALUES (0, 0) USING TTL %d", ks, tbl, 1), now + day, prepared);
+
+ // verify that despite TTL having passed (if not for nowInSeconds override) the row is still there.
+ assertEquals(1, executeSelect(format("SELECT * FROM %s.%s", ks, tbl), now + 1, prepared).size());
+
+ // jump in time by one day, make sure the row expired
+ assertEquals(0, executeSelect(format("SELECT * FROM %s.%s", ks, tbl), now + day + 1, prepared).size());
+ }
+
+ @Test
+ public void testBatchQuery()
+ {
+ testBatchQuery(false);
+ testBatchQuery(true);
+ }
+
+ private void testBatchQuery(boolean prepared)
+ {
+ int now = (int) (System.currentTimeMillis() / 1000);
+ int day = 86400;
+
+ String ks = createKeyspace("CREATE KEYSPACE %s WITH replication={ 'class' : 'SimpleStrategy', 'replication_factor' : 1 }");
+ String tbl = createTable(ks, "CREATE TABLE %s (id int primary key, val int)");
+
+ // execute an BATCH query with now set to [now + 1 day], with ttl = 1, making its effective ttl = 1 day + 1.
+ String batch = format("BEGIN BATCH " +
+ "INSERT INTO %s.%s (id, val) VALUES (0, 0) USING TTL %d; " +
+ "INSERT INTO %s.%s (id, val) VALUES (1, 1) USING TTL %d; " +
+ "APPLY BATCH;",
+ ks, tbl, 1,
+ ks, tbl, 1);
+ executeModify(batch, now + day, prepared);
+
+ // verify that despite TTL having passed at now + 1 the rows are still there.
+ assertEquals(2, executeSelect(format("SELECT * FROM %s.%s", ks, tbl), now + 1, prepared).size());
+
+ // jump in time by one day, make sure the row expired.
+ assertEquals(0, executeSelect(format("SELECT * FROM %s.%s", ks, tbl), now + day + 1, prepared).size());
+ }
+
+ @Test
+ public void testBatchMessage()
+ {
+ // test BatchMessage path
+
+ int now = (int) (System.currentTimeMillis() / 1000);
+ int day = 86400;
+
+ String ks = createKeyspace("CREATE KEYSPACE %s WITH replication={ 'class' : 'SimpleStrategy', 'replication_factor' : 1 }");
+ String tbl = createTable(ks, "CREATE TABLE %s (id int primary key, val int)");
+
+ List<String> queries = ImmutableList.of(
+ format("INSERT INTO %s.%s (id, val) VALUES (0, 0) USING TTL %d;", ks, tbl, 1),
+ format("INSERT INTO %s.%s (id, val) VALUES (1, 1) USING TTL %d;", ks, tbl, 1)
+ );
+
+ ClientState cs = ClientState.forInternalCalls();
+ QueryState qs = new QueryState(cs);
+
+ List<ModificationStatement> statements = new ArrayList<>(queries.size());
+ for (String query : queries)
+ statements.add((ModificationStatement) QueryProcessor.parseStatement(query, cs));
+
+ BatchStatement batch =
+ new BatchStatement(BatchStatement.Type.UNLOGGED, VariableSpecifications.empty(), statements, Attributes.none());
+
+ // execute an BATCH message with now set to [now + 1 day], with ttl = 1, making its effective ttl = 1 day + 1.
+ QueryProcessor.instance.processBatch(batch, qs, batchQueryOptions(now + day), Collections.emptyMap(), System.nanoTime());
+
+ // verify that despite TTL having passed at now + 1 the rows are still there.
+ assertEquals(2, executeSelect(format("SELECT * FROM %s.%s", ks, tbl), now + 1, false).size());
+
+ // jump in time by one day, make sure the row expired.
+ assertEquals(0, executeSelect(format("SELECT * FROM %s.%s", ks, tbl), now + day + 1, false).size());
+ }
+
+ private static ResultSet executeSelect(String query, int nowInSeconds, boolean prepared)
+ {
+ ResultMessage message = execute(query, nowInSeconds, prepared);
+ return ((ResultMessage.Rows) message).result;
+ }
+
+ private static void executeModify(String query, int nowInSeconds, boolean prepared)
+ {
+ execute(query, nowInSeconds, prepared);
+ }
+
+ // prepared = false tests QueryMessage path, prepared = true tests ExecuteMessage path
+ private static ResultMessage execute(String query, int nowInSeconds, boolean prepared)
+ {
+ ClientState cs = ClientState.forInternalCalls();
+ QueryState qs = new QueryState(cs);
+
+ if (prepared)
+ {
+ CQLStatement statement = QueryProcessor.parseStatement(query, cs);
+ return QueryProcessor.instance.processPrepared(statement, qs, queryOptions(nowInSeconds), Collections.emptyMap(), System.nanoTime());
+ }
+ else
+ {
+ CQLStatement statement = QueryProcessor.instance.parse(query, qs, queryOptions(nowInSeconds));
+ return QueryProcessor.instance.process(statement, qs, queryOptions(nowInSeconds), Collections.emptyMap(), System.nanoTime());
+ }
+ }
+
+ private static QueryOptions queryOptions(int nowInSeconds)
+ {
+ return QueryOptions.create(ConsistencyLevel.ONE,
+ Collections.emptyList(),
+ false,
+ Integer.MAX_VALUE,
+ null,
+ null,
+ ProtocolVersion.CURRENT,
+ null,
+ Long.MIN_VALUE,
+ nowInSeconds);
+ }
+
+ private static BatchQueryOptions batchQueryOptions(int nowInSeconds)
+ {
+ return BatchQueryOptions.withoutPerStatementVariables(queryOptions(nowInSeconds));
+ }
+}
diff --git a/test/unit/org/apache/cassandra/cql3/DistinctQueryPagingTest.java b/test/unit/org/apache/cassandra/cql3/DistinctQueryPagingTest.java
index 6f0477d..61ac66b 100644
--- a/test/unit/org/apache/cassandra/cql3/DistinctQueryPagingTest.java
+++ b/test/unit/org/apache/cassandra/cql3/DistinctQueryPagingTest.java
@@ -28,7 +28,7 @@
@Test
public void testSelectDistinct() throws Throwable
{
- // Test a regular(CQL3) table.
+ // Test a regular (CQL3) table.
createTable("CREATE TABLE %s (pk0 int, pk1 int, ck0 int, val int, PRIMARY KEY((pk0, pk1), ck0))");
for (int i = 0; i < 3; i++)
@@ -48,37 +48,6 @@
// Test selection validation.
assertInvalidMessage("queries must request all the partition key columns", "SELECT DISTINCT pk0 FROM %s");
assertInvalidMessage("queries must only request partition key columns", "SELECT DISTINCT pk0, pk1, ck0 FROM %s");
-
- //Test a 'compact storage' table.
- createTable("CREATE TABLE %s (pk0 int, pk1 int, val int, PRIMARY KEY((pk0, pk1))) WITH COMPACT STORAGE");
-
- for (int i = 0; i < 3; i++)
- execute("INSERT INTO %s (pk0, pk1, val) VALUES (?, ?, ?)", i, i, i);
-
- assertRows(execute("SELECT DISTINCT pk0, pk1 FROM %s LIMIT 1"),
- row(0, 0));
-
- assertRows(execute("SELECT DISTINCT pk0, pk1 FROM %s LIMIT 3"),
- row(0, 0),
- row(2, 2),
- row(1, 1));
-
- // Test a 'wide row' thrift table.
- createTable("CREATE TABLE %s (pk int, name text, val int, PRIMARY KEY(pk, name)) WITH COMPACT STORAGE");
-
- for (int i = 0; i < 3; i++)
- {
- execute("INSERT INTO %s (pk, name, val) VALUES (?, 'name0', 0)", i);
- execute("INSERT INTO %s (pk, name, val) VALUES (?, 'name1', 1)", i);
- }
-
- assertRows(execute("SELECT DISTINCT pk FROM %s LIMIT 1"),
- row(1));
-
- assertRows(execute("SELECT DISTINCT pk FROM %s LIMIT 3"),
- row(1),
- row(0),
- row(2));
}
/**
diff --git a/test/unit/org/apache/cassandra/cql3/DurationTest.java b/test/unit/org/apache/cassandra/cql3/DurationTest.java
index b8f4400..ef031c4 100644
--- a/test/unit/org/apache/cassandra/cql3/DurationTest.java
+++ b/test/unit/org/apache/cassandra/cql3/DurationTest.java
@@ -18,6 +18,14 @@
*/
package org.apache.cassandra.cql3;
+import java.text.ParsePosition;
+import java.text.SimpleDateFormat;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.TimeZone;
+
+import org.apache.commons.lang3.time.DateUtils;
+
import org.junit.Assert;
import org.junit.Test;
@@ -99,11 +107,79 @@
assertInvalidDuration("P0002-00-20", "Unable to convert 'P0002-00-20' to a duration");
}
+ @Test
+ public void testAddTo()
+ {
+ assertEquals(toMillis("2016-09-21T00:00:00"), Duration.from("0m").addTo(toMillis("2016-09-21T00:00:00")));
+ assertEquals(toMillis("2016-09-21T00:00:00"), Duration.from("10us").addTo(toMillis("2016-09-21T00:00:00")));
+ assertEquals(toMillis("2016-09-21T00:10:00"), Duration.from("10m").addTo(toMillis("2016-09-21T00:00:00")));
+ assertEquals(toMillis("2016-09-21T01:30:00"), Duration.from("90m").addTo(toMillis("2016-09-21T00:00:00")));
+ assertEquals(toMillis("2016-09-21T02:10:00"), Duration.from("2h10m").addTo(toMillis("2016-09-21T00:00:00")));
+ assertEquals(toMillis("2016-09-23T00:10:00"), Duration.from("2d10m").addTo(toMillis("2016-09-21T00:00:00")));
+ assertEquals(toMillis("2016-09-24T01:00:00"), Duration.from("2d25h").addTo(toMillis("2016-09-21T00:00:00")));
+ assertEquals(toMillis("2016-10-21T00:00:00"), Duration.from("1mo").addTo(toMillis("2016-09-21T00:00:00")));
+ assertEquals(toMillis("2017-11-21T00:00:00"), Duration.from("14mo").addTo(toMillis("2016-09-21T00:00:00")));
+ assertEquals(toMillis("2017-02-28T00:00:00"), Duration.from("12mo").addTo(toMillis("2016-02-29T00:00:00")));
+ }
+
+ @Test
+ public void testAddToWithNegativeDurations()
+ {
+ assertEquals(toMillis("2016-09-21T00:00:00"), Duration.from("-0m").addTo(toMillis("2016-09-21T00:00:00")));
+ assertEquals(toMillis("2016-09-21T00:00:00"), Duration.from("-10us").addTo(toMillis("2016-09-21T00:00:00")));
+ assertEquals(toMillis("2016-09-20T23:50:00"), Duration.from("-10m").addTo(toMillis("2016-09-21T00:00:00")));
+ assertEquals(toMillis("2016-09-20T22:30:00"), Duration.from("-90m").addTo(toMillis("2016-09-21T00:00:00")));
+ assertEquals(toMillis("2016-09-20T21:50:00"), Duration.from("-2h10m").addTo(toMillis("2016-09-21T00:00:00")));
+ assertEquals(toMillis("2016-09-18T23:50:00"), Duration.from("-2d10m").addTo(toMillis("2016-09-21T00:00:00")));
+ assertEquals(toMillis("2016-09-17T23:00:00"), Duration.from("-2d25h").addTo(toMillis("2016-09-21T00:00:00")));
+ assertEquals(toMillis("2016-08-21T00:00:00"), Duration.from("-1mo").addTo(toMillis("2016-09-21T00:00:00")));
+ assertEquals(toMillis("2015-07-21T00:00:00"), Duration.from("-14mo").addTo(toMillis("2016-09-21T00:00:00")));
+ assertEquals(toMillis("2015-02-28T00:00:00"), Duration.from("-12mo").addTo(toMillis("2016-02-29T00:00:00")));
+ }
+
+ @Test
+ public void testSubstractFrom()
+ {
+ assertEquals(toMillis("2016-09-21T00:00:00"), Duration.from("0m").substractFrom(toMillis("2016-09-21T00:00:00")));
+ assertEquals(toMillis("2016-09-21T00:00:00"), Duration.from("10us").substractFrom(toMillis("2016-09-21T00:00:00")));
+ assertEquals(toMillis("2016-09-20T23:50:00"), Duration.from("10m").substractFrom(toMillis("2016-09-21T00:00:00")));
+ assertEquals(toMillis("2016-09-20T22:30:00"), Duration.from("90m").substractFrom(toMillis("2016-09-21T00:00:00")));
+ assertEquals(toMillis("2016-09-20T21:50:00"), Duration.from("2h10m").substractFrom(toMillis("2016-09-21T00:00:00")));
+ assertEquals(toMillis("2016-09-18T23:50:00"), Duration.from("2d10m").substractFrom(toMillis("2016-09-21T00:00:00")));
+ assertEquals(toMillis("2016-09-17T23:00:00"), Duration.from("2d25h").substractFrom(toMillis("2016-09-21T00:00:00")));
+ assertEquals(toMillis("2016-08-21T00:00:00"), Duration.from("1mo").substractFrom(toMillis("2016-09-21T00:00:00")));
+ assertEquals(toMillis("2015-07-21T00:00:00"), Duration.from("14mo").substractFrom(toMillis("2016-09-21T00:00:00")));
+ assertEquals(toMillis("2015-02-28T00:00:00"), Duration.from("12mo").substractFrom(toMillis("2016-02-29T00:00:00")));
+ }
+
+ @Test
+ public void testSubstractWithNegativeDurations()
+ {
+ assertEquals(toMillis("2016-09-21T00:00:00"), Duration.from("-0m").substractFrom(toMillis("2016-09-21T00:00:00")));
+ assertEquals(toMillis("2016-09-21T00:00:00"), Duration.from("-10us").substractFrom(toMillis("2016-09-21T00:00:00")));
+ assertEquals(toMillis("2016-09-21T00:10:00"), Duration.from("-10m").substractFrom(toMillis("2016-09-21T00:00:00")));
+ assertEquals(toMillis("2016-09-21T01:30:00"), Duration.from("-90m").substractFrom(toMillis("2016-09-21T00:00:00")));
+ assertEquals(toMillis("2016-09-21T02:10:00"), Duration.from("-2h10m").substractFrom(toMillis("2016-09-21T00:00:00")));
+ assertEquals(toMillis("2016-09-23T00:10:00"), Duration.from("-2d10m").substractFrom(toMillis("2016-09-21T00:00:00")));
+ assertEquals(toMillis("2016-09-24T01:00:00"), Duration.from("-2d25h").substractFrom(toMillis("2016-09-21T00:00:00")));
+ assertEquals(toMillis("2016-10-21T00:00:00"), Duration.from("-1mo").substractFrom(toMillis("2016-09-21T00:00:00")));
+ assertEquals(toMillis("2017-11-21T00:00:00"), Duration.from("-14mo").substractFrom(toMillis("2016-09-21T00:00:00")));
+ assertEquals(toMillis("2017-02-28T00:00:00"), Duration.from("-12mo").substractFrom(toMillis("2016-02-29T00:00:00")));
+ }
+
+ private long toMillis(String timeAsString)
+ {
+ SimpleDateFormat parser = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
+ parser.setTimeZone(TimeZone.getTimeZone("UTC"));
+ Date date = parser.parse(timeAsString, new ParsePosition(0));
+ return DateUtils.truncate(date, Calendar.SECOND).getTime();
+ }
+
public void assertInvalidDuration(String duration, String expectedErrorMessage)
{
try
{
- System.out.println(Duration.from(duration));
+ Duration.from(duration);
Assert.fail();
}
catch (InvalidRequestException e)
diff --git a/test/unit/org/apache/cassandra/cql3/GcCompactionTest.java b/test/unit/org/apache/cassandra/cql3/GcCompactionTest.java
index 3af5dee..2fc07eb 100644
--- a/test/unit/org/apache/cassandra/cql3/GcCompactionTest.java
+++ b/test/unit/org/apache/cassandra/cql3/GcCompactionTest.java
@@ -25,7 +25,7 @@
import java.util.Collections;
import java.util.HashSet;
import java.util.Set;
-import java.util.function.Function;
+import java.util.function.ToIntFunction;
import com.google.common.collect.Iterables;
import org.junit.Test;
@@ -410,7 +410,7 @@
int countRows(SSTableReader reader)
{
- boolean enforceStrictLiveness = reader.metadata.enforceStrictLiveness();
+ boolean enforceStrictLiveness = reader.metadata().enforceStrictLiveness();
int nowInSec = FBUtilities.nowInSeconds();
return count(reader, x -> x.isRow() && ((Row) x).hasLiveData(nowInSec, enforceStrictLiveness) ? 1 : 0, x -> 0);
}
@@ -438,7 +438,7 @@
return ccd.cellsCount();
}
- int count(SSTableReader reader, Function<Unfiltered, Integer> predicate, Function<UnfilteredRowIterator, Integer> partitionPredicate)
+ int count(SSTableReader reader, ToIntFunction<Unfiltered> predicate, ToIntFunction<UnfilteredRowIterator> partitionPredicate)
{
int instances = 0;
try (ISSTableScanner partitions = reader.getScanner())
@@ -447,11 +447,11 @@
{
try (UnfilteredRowIterator iter = partitions.next())
{
- instances += partitionPredicate.apply(iter);
+ instances += partitionPredicate.applyAsInt(iter);
while (iter.hasNext())
{
Unfiltered atom = iter.next();
- instances += predicate.apply(atom);
+ instances += predicate.applyAsInt(atom);
}
}
}
diff --git a/test/unit/org/apache/cassandra/cql3/KeyCacheCqlTest.java b/test/unit/org/apache/cassandra/cql3/KeyCacheCqlTest.java
index 32e6aec..b76cc78 100644
--- a/test/unit/org/apache/cassandra/cql3/KeyCacheCqlTest.java
+++ b/test/unit/org/apache/cassandra/cql3/KeyCacheCqlTest.java
@@ -30,20 +30,19 @@
import org.apache.cassandra.cache.KeyCacheKey;
import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.config.Schema;
+import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.index.Index;
import org.apache.cassandra.metrics.CacheMetrics;
import org.apache.cassandra.metrics.CassandraMetricsRegistry;
import org.apache.cassandra.schema.CachingParams;
+import org.apache.cassandra.schema.TableMetadataRef;
import org.apache.cassandra.service.CacheService;
import org.apache.cassandra.service.StorageService;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.assertNull;
-import org.apache.cassandra.utils.Pair;
public class KeyCacheCqlTest extends CQLTester
@@ -120,9 +119,9 @@
}
@Override
- protected void createIndex(String query)
+ protected String createIndex(String query)
{
- createFormattedIndex(formatQuery(KEYSPACE_PER_TEST, query));
+ return createFormattedIndex(formatQuery(KEYSPACE_PER_TEST, query));
}
@Override
@@ -247,8 +246,8 @@
String table = createTable("CREATE TABLE %s ("
+ commonColumnsDef
+ "PRIMARY KEY ((part_key_a, part_key_b),clust_key_a,clust_key_b,clust_key_c))");
- createIndex("CREATE INDEX some_index ON %s (col_int)");
- insertData(table, "some_index", true);
+ String indexName = createIndex("CREATE INDEX ON %s (col_int)");
+ insertData(table, indexName, true);
clearCache();
CacheMetrics metrics = CacheService.instance.keyCache.getMetrics();
@@ -301,11 +300,6 @@
assertEquals(500, result.size());
}
- //Test Schema.getColumnFamilyStoreIncludingIndexes, several null check paths
- //are defensive and unreachable
- assertNull(Schema.instance.getColumnFamilyStoreIncludingIndexes(Pair.create("foo", "bar")));
- assertNull(Schema.instance.getColumnFamilyStoreIncludingIndexes(Pair.create(KEYSPACE_PER_TEST, "bar")));
-
dropTable("DROP TABLE %s");
Schema.instance.updateVersion();
@@ -337,8 +331,8 @@
String table = createTable("CREATE TABLE %s ("
+ commonColumnsDef
+ "PRIMARY KEY ((part_key_a, part_key_b),clust_key_a,clust_key_b,clust_key_c))");
- createIndex("CREATE INDEX some_index ON %s (col_int)");
- insertData(table, "some_index", true);
+ String indexName = createIndex("CREATE INDEX ON %s (col_int)");
+ insertData(table, indexName, true);
clearCache();
CacheMetrics metrics = CacheService.instance.keyCache.getMetrics();
@@ -386,8 +380,9 @@
while(iter.hasNext())
{
KeyCacheKey key = iter.next();
- Assert.assertFalse(key.ksAndCFName.left.equals("KEYSPACE_PER_TEST"));
- Assert.assertFalse(key.ksAndCFName.right.startsWith(table));
+ TableMetadataRef tableMetadataRef = Schema.instance.getTableMetadataRef(key.tableId);
+ Assert.assertFalse(tableMetadataRef.keyspace.equals("KEYSPACE_PER_TEST"));
+ Assert.assertFalse(tableMetadataRef.name.startsWith(table));
}
}
diff --git a/test/unit/org/apache/cassandra/cql3/LargeCompactValueTest.java b/test/unit/org/apache/cassandra/cql3/LargeCompactValueTest.java
deleted file mode 100644
index 93b16ce..0000000
--- a/test/unit/org/apache/cassandra/cql3/LargeCompactValueTest.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.cql3;
-
-import java.nio.ByteBuffer;
-
-import org.junit.Before;
-import org.junit.Test;
-
-public class LargeCompactValueTest extends CQLTester
-{
- @Before
- public void before()
- {
- createTable("CREATE TABLE %s (key TEXT, column TEXT, value BLOB, PRIMARY KEY (key, column)) WITH COMPACT STORAGE");
- }
-
- @Test
- public void testInsertAndQuery() throws Throwable
- {
- ByteBuffer largeBytes = ByteBuffer.wrap(new byte[100000]);
- execute("INSERT INTO %s (key, column, value) VALUES (?, ?, ?)", "test", "a", largeBytes);
- ByteBuffer smallBytes = ByteBuffer.wrap(new byte[10]);
- execute("INSERT INTO %s (key, column, value) VALUES (?, ?, ?)", "test", "c", smallBytes);
-
- flush();
-
- assertRows(execute("SELECT column FROM %s WHERE key = ? AND column IN (?, ?, ?)", "test", "c", "a", "b"),
- row("a"),
- row("c"));
- }
-}
diff --git a/test/unit/org/apache/cassandra/cql3/ListsTest.java b/test/unit/org/apache/cassandra/cql3/ListsTest.java
index 07623a2..1155619 100644
--- a/test/unit/org/apache/cassandra/cql3/ListsTest.java
+++ b/test/unit/org/apache/cassandra/cql3/ListsTest.java
@@ -28,15 +28,16 @@
import org.junit.Assert;
import org.junit.Test;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
import org.apache.cassandra.cql3.Lists.PrecisionTime;
import org.apache.cassandra.db.Clustering;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.rows.Cell;
import org.apache.cassandra.db.rows.Row;
import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.UUIDGen;
public class ListsTest extends CQLTester
@@ -132,15 +133,16 @@
private void testPrepender_execute(List<ByteBuffer> terms)
{
createTable("CREATE TABLE %s (k int PRIMARY KEY, l list<text>)");
- CFMetaData metaData = currentTableMetadata();
+ TableMetadata metaData = currentTableMetadata();
- ColumnDefinition columnDefinition = metaData.getColumnDefinition(ByteBufferUtil.bytes("l"));
+ ColumnMetadata columnMetadata = metaData.getColumn(ByteBufferUtil.bytes("l"));
Term term = new Lists.Value(terms);
- Lists.Prepender prepender = new Lists.Prepender(columnDefinition, term);
+ Lists.Prepender prepender = new Lists.Prepender(columnMetadata, term);
ByteBuffer keyBuf = ByteBufferUtil.bytes("key");
DecoratedKey key = Murmur3Partitioner.instance.decorateKey(keyBuf);
- UpdateParameters parameters = new UpdateParameters(metaData, null, null, System.currentTimeMillis(), 1000, Collections.emptyMap());
+ UpdateParameters parameters =
+ new UpdateParameters(metaData, null, QueryOptions.DEFAULT, System.currentTimeMillis(), FBUtilities.nowInSeconds(), 1000, Collections.emptyMap());
Clustering clustering = Clustering.make(ByteBufferUtil.bytes(1));
parameters.newRow(clustering);
prepender.execute(key, parameters);
diff --git a/test/unit/org/apache/cassandra/cql3/OutOfSpaceTest.java b/test/unit/org/apache/cassandra/cql3/OutOfSpaceTest.java
index 24efc5e..b4fe0f5 100644
--- a/test/unit/org/apache/cassandra/cql3/OutOfSpaceTest.java
+++ b/test/unit/org/apache/cassandra/cql3/OutOfSpaceTest.java
@@ -20,7 +20,6 @@
import static junit.framework.Assert.fail;
import java.io.Closeable;
-import java.util.UUID;
import java.util.concurrent.ExecutionException;
import org.junit.Assert;
@@ -34,6 +33,7 @@
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.gms.Gossiper;
import org.apache.cassandra.io.FSWriteError;
+import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.utils.JVMStabilityInspector;
import org.apache.cassandra.utils.KillerForTests;
@@ -125,9 +125,9 @@
}
// Make sure commit log wasn't discarded.
- UUID cfid = currentTableMetadata().cfId;
+ TableId tableId = currentTableMetadata().id;
for (CommitLogSegment segment : CommitLog.instance.segmentManager.getActiveSegments())
- if (segment.getDirtyCFIDs().contains(cfid))
+ if (segment.getDirtyTableIds().contains(tableId))
return;
fail("Expected commit log to remain dirty for the affected table.");
}
diff --git a/test/unit/org/apache/cassandra/cql3/PagingTest.java b/test/unit/org/apache/cassandra/cql3/PagingTest.java
index a054d01..50bba0e 100644
--- a/test/unit/org/apache/cassandra/cql3/PagingTest.java
+++ b/test/unit/org/apache/cassandra/cql3/PagingTest.java
@@ -35,8 +35,7 @@
import org.apache.cassandra.dht.Murmur3Partitioner;
import org.apache.cassandra.dht.Murmur3Partitioner.LongToken;
-import org.apache.cassandra.locator.AbstractEndpointSnitch;
-import org.apache.cassandra.locator.IEndpointSnitch;
+import org.apache.cassandra.locator.*;
import org.apache.cassandra.service.EmbeddedCassandraService;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.FBUtilities;
@@ -104,30 +103,30 @@
IEndpointSnitch snitch = new AbstractEndpointSnitch()
{
private IEndpointSnitch oldSnitch = DatabaseDescriptor.getEndpointSnitch();
- public int compareEndpoints(InetAddress target, InetAddress a1, InetAddress a2)
+ public int compareEndpoints(InetAddressAndPort target, Replica a1, Replica a2)
{
return oldSnitch.compareEndpoints(target, a1, a2);
}
- public String getRack(InetAddress endpoint)
+ public String getRack(InetAddressAndPort endpoint)
{
return oldSnitch.getRack(endpoint);
}
- public String getDatacenter(InetAddress endpoint)
+ public String getDatacenter(InetAddressAndPort endpoint)
{
return oldSnitch.getDatacenter(endpoint);
}
@Override
- public boolean isWorthMergingForRangeQuery(List<InetAddress> merged, List<InetAddress> l1, List<InetAddress> l2)
+ public boolean isWorthMergingForRangeQuery(ReplicaCollection merged, ReplicaCollection l1, ReplicaCollection l2)
{
return false;
}
};
DatabaseDescriptor.setEndpointSnitch(snitch);
StorageService.instance.getTokenMetadata().clearUnsafe();
- StorageService.instance.getTokenMetadata().updateNormalToken(new LongToken(5097162189738624638L), FBUtilities.getBroadcastAddress());
+ StorageService.instance.getTokenMetadata().updateNormalToken(new LongToken(5097162189738624638L), FBUtilities.getBroadcastAddressAndPort());
session.execute(createTableStatement);
for (int i = 0; i < 110; i++)
diff --git a/test/unit/org/apache/cassandra/cql3/PreparedStatementsTest.java b/test/unit/org/apache/cassandra/cql3/PreparedStatementsTest.java
index e01b812..11df055 100644
--- a/test/unit/org/apache/cassandra/cql3/PreparedStatementsTest.java
+++ b/test/unit/org/apache/cassandra/cql3/PreparedStatementsTest.java
@@ -17,63 +17,51 @@
*/
package org.apache.cassandra.cql3;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
+import java.nio.ByteBuffer;
+import java.util.*;
+import java.util.stream.Collectors;
+
+import org.junit.Before;
import org.junit.Test;
import com.datastax.driver.core.Cluster;
import com.datastax.driver.core.PreparedStatement;
+import com.datastax.driver.core.ResultSet;
import com.datastax.driver.core.Session;
import com.datastax.driver.core.exceptions.SyntaxError;
-import org.apache.cassandra.SchemaLoader;
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.config.Schema;
+import org.apache.cassandra.db.ConsistencyLevel;
+import org.apache.cassandra.exceptions.PreparedQueryNotFoundException;
import org.apache.cassandra.index.StubIndex;
-import org.apache.cassandra.service.EmbeddedCassandraService;
+import org.apache.cassandra.serializers.BooleanSerializer;
+import org.apache.cassandra.serializers.Int32Serializer;
+import org.apache.cassandra.transport.ProtocolVersion;
+import org.apache.cassandra.transport.SimpleClient;
+import org.apache.cassandra.transport.messages.ResultMessage;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
-public class PreparedStatementsTest extends SchemaLoader
+public class PreparedStatementsTest extends CQLTester
{
- private static Cluster cluster;
- private static Session session;
-
private static final String KEYSPACE = "prepared_stmt_cleanup";
private static final String createKsStatement = "CREATE KEYSPACE " + KEYSPACE +
" WITH REPLICATION = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 };";
private static final String dropKsStatement = "DROP KEYSPACE IF EXISTS " + KEYSPACE;
- @BeforeClass
- public static void setup() throws Exception
+ @Before
+ public void setup()
{
- Schema.instance.clear();
-
- EmbeddedCassandraService cassandra = new EmbeddedCassandraService();
- cassandra.start();
-
- // Currently the native server start method return before the server is fully binded to the socket, so we need
- // to wait slightly before trying to connect to it. We should fix this but in the meantime using a sleep.
- Thread.sleep(500);
-
- cluster = Cluster.builder().addContactPoint("127.0.0.1")
- .withPort(DatabaseDescriptor.getNativeTransportPort())
- .build();
- session = cluster.connect();
-
- session.execute(dropKsStatement);
- session.execute(createKsStatement);
- }
-
- @AfterClass
- public static void tearDown() throws Exception
- {
- cluster.close();
+ requireNetwork();
}
@Test
public void testInvalidatePreparedStatementsOnDrop()
{
+ Session session = sessions.get(ProtocolVersion.V5);
+ session.execute(dropKsStatement);
+ session.execute(createKsStatement);
+
String createTableStatement = "CREATE TABLE IF NOT EXISTS " + KEYSPACE + ".qp_cleanup (id int PRIMARY KEY, cid int, val text);";
String dropTableStatement = "DROP TABLE IF EXISTS " + KEYSPACE + ".qp_cleanup;";
@@ -101,15 +89,128 @@
}
@Test
+ public void testInvalidatePreparedStatementOnAlterV5()
+ {
+ testInvalidatePreparedStatementOnAlter(ProtocolVersion.V5, true);
+ }
+
+ @Test
+ public void testInvalidatePreparedStatementOnAlterV4()
+ {
+ testInvalidatePreparedStatementOnAlter(ProtocolVersion.V4, false);
+ }
+
+ private void testInvalidatePreparedStatementOnAlter(ProtocolVersion version, boolean supportsMetadataChange)
+ {
+ Session session = sessions.get(version);
+ String createTableStatement = "CREATE TABLE IF NOT EXISTS " + KEYSPACE + ".qp_cleanup (a int PRIMARY KEY, b int, c int);";
+ String alterTableStatement = "ALTER TABLE " + KEYSPACE + ".qp_cleanup ADD d int;";
+
+ session.execute(dropKsStatement);
+ session.execute(createKsStatement);
+ session.execute(createTableStatement);
+
+ PreparedStatement preparedSelect = session.prepare("SELECT * FROM " + KEYSPACE + ".qp_cleanup");
+ session.execute("INSERT INTO " + KEYSPACE + ".qp_cleanup (a, b, c) VALUES (?, ?, ?);",
+ 1, 2, 3);
+ session.execute("INSERT INTO " + KEYSPACE + ".qp_cleanup (a, b, c) VALUES (?, ?, ?);",
+ 2, 3, 4);
+
+ assertRowsNet(session.execute(preparedSelect.bind()),
+ row(1, 2, 3),
+ row(2, 3, 4));
+
+ session.execute(alterTableStatement);
+ session.execute("INSERT INTO " + KEYSPACE + ".qp_cleanup (a, b, c, d) VALUES (?, ?, ?, ?);",
+ 3, 4, 5, 6);
+
+ ResultSet rs;
+ if (supportsMetadataChange)
+ {
+ rs = session.execute(preparedSelect.bind());
+ assertRowsNet(version,
+ rs,
+ row(1, 2, 3, null),
+ row(2, 3, 4, null),
+ row(3, 4, 5, 6));
+ assertEquals(rs.getColumnDefinitions().size(), 4);
+ }
+ else
+ {
+ rs = session.execute(preparedSelect.bind());
+ assertRowsNet(rs,
+ row(1, 2, 3),
+ row(2, 3, 4),
+ row(3, 4, 5));
+ assertEquals(rs.getColumnDefinitions().size(), 3);
+ }
+
+ session.execute(dropKsStatement);
+ }
+
+ @Test
+ public void testInvalidatePreparedStatementOnAlterUnchangedMetadataV4()
+ {
+ testInvalidatePreparedStatementOnAlterUnchangedMetadata(ProtocolVersion.V4);
+ }
+
+ @Test
+ public void testInvalidatePreparedStatementOnAlterUnchangedMetadataV5()
+ {
+ testInvalidatePreparedStatementOnAlterUnchangedMetadata(ProtocolVersion.V5);
+ }
+
+ private void testInvalidatePreparedStatementOnAlterUnchangedMetadata(ProtocolVersion version)
+ {
+ Session session = sessions.get(version);
+ String createTableStatement = "CREATE TABLE IF NOT EXISTS " + KEYSPACE + ".qp_cleanup (a int PRIMARY KEY, b int, c int);";
+ String alterTableStatement = "ALTER TABLE " + KEYSPACE + ".qp_cleanup ADD d int;";
+
+ session.execute(dropKsStatement);
+ session.execute(createKsStatement);
+ session.execute(createTableStatement);
+
+ PreparedStatement preparedSelect = session.prepare("SELECT a, b, c FROM " + KEYSPACE + ".qp_cleanup");
+ session.execute("INSERT INTO " + KEYSPACE + ".qp_cleanup (a, b, c) VALUES (?, ?, ?);",
+ 1, 2, 3);
+ session.execute("INSERT INTO " + KEYSPACE + ".qp_cleanup (a, b, c) VALUES (?, ?, ?);",
+ 2, 3, 4);
+
+ ResultSet rs = session.execute(preparedSelect.bind());
+
+ assertRowsNet(rs,
+ row(1, 2, 3),
+ row(2, 3, 4));
+ assertEquals(rs.getColumnDefinitions().size(), 3);
+
+ session.execute(alterTableStatement);
+ session.execute("INSERT INTO " + KEYSPACE + ".qp_cleanup (a, b, c, d) VALUES (?, ?, ?, ?);",
+ 3, 4, 5, 6);
+
+ rs = session.execute(preparedSelect.bind());
+ assertRowsNet(rs,
+ row(1, 2, 3),
+ row(2, 3, 4),
+ row(3, 4, 5));
+ assertEquals(rs.getColumnDefinitions().size(), 3);
+
+ session.execute(dropKsStatement);
+ }
+
+ @Test
public void testStatementRePreparationOnReconnect()
{
+ Session session = sessions.get(ProtocolVersion.V5);
+ session.execute("USE " + keyspace());
+
session.execute(dropKsStatement);
session.execute(createKsStatement);
- session.execute("CREATE TABLE IF NOT EXISTS " + KEYSPACE + ".qp_test (id int PRIMARY KEY, cid int, val text);");
+ createTable("CREATE TABLE %s (id int PRIMARY KEY, cid int, val text);");
- String insertCQL = "INSERT INTO " + KEYSPACE + ".qp_test (id, cid, val) VALUES (?, ?, ?)";
- String selectCQL = "Select * from " + KEYSPACE + ".qp_test where id = ?";
+
+ String insertCQL = "INSERT INTO " + currentTable() + " (id, cid, val) VALUES (?, ?, ?)";
+ String selectCQL = "Select * from " + currentTable() + " where id = ?";
PreparedStatement preparedInsert = session.prepare(insertCQL);
PreparedStatement preparedSelect = session.prepare(selectCQL);
@@ -117,23 +218,31 @@
session.execute(preparedInsert.bind(1, 1, "value"));
assertEquals(1, session.execute(preparedSelect.bind(1)).all().size());
- cluster.close();
+ try (Cluster newCluster = Cluster.builder()
+ .addContactPoints(nativeAddr)
+ .withClusterName("Test Cluster")
+ .withPort(nativePort)
+ .withoutJMXReporting()
+ .allowBetaProtocolVersion()
+ .build())
+ {
+ try (Session newSession = newCluster.connect())
+ {
+ newSession.execute("USE " + keyspace());
+ preparedInsert = newSession.prepare(insertCQL);
+ preparedSelect = newSession.prepare(selectCQL);
+ session.execute(preparedInsert.bind(1, 1, "value"));
- cluster = Cluster.builder().addContactPoint("127.0.0.1")
- .withPort(DatabaseDescriptor.getNativeTransportPort())
- .build();
- session = cluster.connect();
-
- preparedInsert = session.prepare(insertCQL);
- preparedSelect = session.prepare(selectCQL);
- session.execute(preparedInsert.bind(1, 1, "value"));
-
- assertEquals(1, session.execute(preparedSelect.bind(1)).all().size());
+ assertEquals(1, session.execute(preparedSelect.bind(1)).all().size());
+ }
+ }
}
@Test
public void prepareAndExecuteWithCustomExpressions() throws Throwable
{
+ Session session = sessions.get(ProtocolVersion.V5);
+
session.execute(dropKsStatement);
session.execute(createKsStatement);
String table = "custom_expr_test";
@@ -163,4 +272,313 @@
assertEquals("Bind variables cannot be used for index names", e.getMessage());
}
}
+
+ @Test
+ public void testMetadataFlagsWithLWTs() throws Throwable
+ {
+ // Verify the behavior of CASSANDRA-10786 (result metadata IDs) on the protocol level.
+ // Tests are against an LWT statement and a "regular" SELECT statement.
+ // The fundamental difference between a SELECT and an LWT statement is that the result metadata
+ // of an LWT can change between invocations - therefore we always return the resultset metadata
+ // for LWTs. For "normal" SELECTs, the resultset metadata can only change when DDLs happen
+ // (aka the famous prepared 'SELECT * FROM ks.tab' stops working after the schema of that table
+ // changes). In those cases, the Result.Rows message contains a METADATA_CHANGED flag to tell
+ // clients that the cached metadata for this statement has changed and is included in the result,
+ // whereas the resultset metadata is omitted, if the metadata ID sent with the EXECUTE message
+ // matches the one for the (current) schema.
+ // Note: this test does not cover all aspects of 10786 (yet) - it was intended to test the
+ // changes for CASSANDRA-13992.
+
+ createTable("CREATE TABLE %s (pk int, v1 int, v2 int, PRIMARY KEY (pk))");
+ execute("INSERT INTO %s (pk, v1, v2) VALUES (1,1,1)");
+
+ try (SimpleClient simpleClient = newSimpleClient(ProtocolVersion.BETA.orElse(ProtocolVersion.CURRENT), false, false))
+ {
+ ResultMessage.Prepared prepUpdate = simpleClient.prepare(String.format("UPDATE %s.%s SET v1 = ?, v2 = ? WHERE pk = 1 IF v1 = ?",
+ keyspace(), currentTable()));
+ ResultMessage.Prepared prepSelect = simpleClient.prepare(String.format("SELECT * FROM %s.%s WHERE pk = ?",
+ keyspace(), currentTable()));
+
+ // This is a _successful_ LWT update
+ verifyMetadataFlagsWithLWTsUpdate(simpleClient,
+ prepUpdate,
+ Arrays.asList(Int32Serializer.instance.serialize(10),
+ Int32Serializer.instance.serialize(20),
+ Int32Serializer.instance.serialize(1)),
+ Arrays.asList("[applied]"),
+ Arrays.asList(BooleanSerializer.instance.serialize(true)));
+
+ prepSelect = verifyMetadataFlagsWithLWTsSelect(simpleClient,
+ prepSelect,
+ Arrays.asList("pk", "v1", "v2"),
+ Arrays.asList(Int32Serializer.instance.serialize(1),
+ Int32Serializer.instance.serialize(10),
+ Int32Serializer.instance.serialize(20)),
+ EnumSet.of(org.apache.cassandra.cql3.ResultSet.Flag.GLOBAL_TABLES_SPEC));
+
+ // This is an _unsuccessful_ LWT update (as the condition fails)
+ verifyMetadataFlagsWithLWTsUpdate(simpleClient,
+ prepUpdate,
+ Arrays.asList(Int32Serializer.instance.serialize(10),
+ Int32Serializer.instance.serialize(20),
+ Int32Serializer.instance.serialize(1)),
+ Arrays.asList("[applied]", "v1"),
+ Arrays.asList(BooleanSerializer.instance.serialize(false),
+ Int32Serializer.instance.serialize(10)));
+
+ prepSelect = verifyMetadataFlagsWithLWTsSelect(simpleClient,
+ prepSelect,
+ Arrays.asList("pk", "v1", "v2"),
+ Arrays.asList(Int32Serializer.instance.serialize(1),
+ Int32Serializer.instance.serialize(10),
+ Int32Serializer.instance.serialize(20)),
+ EnumSet.of(org.apache.cassandra.cql3.ResultSet.Flag.GLOBAL_TABLES_SPEC));
+
+ // force a schema change on that table
+ simpleClient.execute(String.format("ALTER TABLE %s.%s ADD v3 int",
+ keyspace(), currentTable()),
+ ConsistencyLevel.LOCAL_ONE);
+
+ try
+ {
+ simpleClient.executePrepared(prepUpdate,
+ Arrays.asList(Int32Serializer.instance.serialize(1),
+ Int32Serializer.instance.serialize(30),
+ Int32Serializer.instance.serialize(10)),
+ ConsistencyLevel.LOCAL_ONE);
+ fail();
+ }
+ catch (RuntimeException re)
+ {
+ assertTrue(re.getCause() instanceof PreparedQueryNotFoundException);
+ // the prepared statement has been removed from the pstmt cache, need to re-prepare it
+ // only prepare the statement on the server side but don't set the variable
+ simpleClient.prepare(String.format("UPDATE %s.%s SET v1 = ?, v2 = ? WHERE pk = 1 IF v1 = ?",
+ keyspace(), currentTable()));
+ }
+ try
+ {
+ simpleClient.executePrepared(prepSelect,
+ Arrays.asList(Int32Serializer.instance.serialize(1)),
+ ConsistencyLevel.LOCAL_ONE);
+ fail();
+ }
+ catch (RuntimeException re)
+ {
+ assertTrue(re.getCause() instanceof PreparedQueryNotFoundException);
+ // the prepared statement has been removed from the pstmt cache, need to re-prepare it
+ // only prepare the statement on the server side but don't set the variable
+ simpleClient.prepare(String.format("SELECT * FROM %s.%s WHERE pk = ?",
+ keyspace(), currentTable()));
+ }
+
+ // This is a _successful_ LWT update
+ verifyMetadataFlagsWithLWTsUpdate(simpleClient,
+ prepUpdate,
+ Arrays.asList(Int32Serializer.instance.serialize(1),
+ Int32Serializer.instance.serialize(30),
+ Int32Serializer.instance.serialize(10)),
+ Arrays.asList("[applied]"),
+ Arrays.asList(BooleanSerializer.instance.serialize(true)));
+
+ // Re-assign prepSelect here, as the resultset metadata changed to submit the updated
+ // resultset-metadata-ID in the next SELECT. This behavior does not apply to LWT statements.
+ prepSelect = verifyMetadataFlagsWithLWTsSelect(simpleClient,
+ prepSelect,
+ Arrays.asList("pk", "v1", "v2", "v3"),
+ Arrays.asList(Int32Serializer.instance.serialize(1),
+ Int32Serializer.instance.serialize(1),
+ Int32Serializer.instance.serialize(30),
+ null),
+ EnumSet.of(org.apache.cassandra.cql3.ResultSet.Flag.GLOBAL_TABLES_SPEC,
+ org.apache.cassandra.cql3.ResultSet.Flag.METADATA_CHANGED));
+
+ // This is an _unsuccessful_ LWT update (as the condition fails)
+ verifyMetadataFlagsWithLWTsUpdate(simpleClient,
+ prepUpdate,
+ Arrays.asList(Int32Serializer.instance.serialize(1),
+ Int32Serializer.instance.serialize(30),
+ Int32Serializer.instance.serialize(10)),
+ Arrays.asList("[applied]", "v1"),
+ Arrays.asList(BooleanSerializer.instance.serialize(false),
+ Int32Serializer.instance.serialize(1)));
+
+ verifyMetadataFlagsWithLWTsSelect(simpleClient,
+ prepSelect,
+ Arrays.asList("pk", "v1", "v2", "v3"),
+ Arrays.asList(Int32Serializer.instance.serialize(1),
+ Int32Serializer.instance.serialize(1),
+ Int32Serializer.instance.serialize(30),
+ null),
+ EnumSet.of(org.apache.cassandra.cql3.ResultSet.Flag.GLOBAL_TABLES_SPEC));
+ }
+ }
+
+ private ResultMessage.Prepared verifyMetadataFlagsWithLWTsSelect(SimpleClient simpleClient,
+ ResultMessage.Prepared prepSelect,
+ List<String> columnNames,
+ List<ByteBuffer> expectedRow,
+ EnumSet<org.apache.cassandra.cql3.ResultSet.Flag> expectedFlags)
+ {
+ ResultMessage result = simpleClient.executePrepared(prepSelect,
+ Collections.singletonList(Int32Serializer.instance.serialize(1)),
+ ConsistencyLevel.LOCAL_ONE);
+ ResultMessage.Rows rows = (ResultMessage.Rows) result;
+ EnumSet<org.apache.cassandra.cql3.ResultSet.Flag> resultFlags = rows.result.metadata.getFlags();
+ assertEquals(expectedFlags,
+ resultFlags);
+ assertEquals(columnNames.size(),
+ rows.result.metadata.getColumnCount());
+ assertEquals(columnNames,
+ rows.result.metadata.names.stream().map(cs -> cs.name.toString()).collect(Collectors.toList()));
+ assertEquals(1,
+ rows.result.size());
+ assertEquals(expectedRow,
+ rows.result.rows.get(0));
+
+ if (resultFlags.contains(org.apache.cassandra.cql3.ResultSet.Flag.METADATA_CHANGED))
+ prepSelect = prepSelect.withResultMetadata(rows.result.metadata);
+ return prepSelect;
+ }
+
+ private void verifyMetadataFlagsWithLWTsUpdate(SimpleClient simpleClient,
+ ResultMessage.Prepared prepUpdate,
+ List<ByteBuffer> params,
+ List<String> columnNames,
+ List<ByteBuffer> expectedRow)
+ {
+ ResultMessage result = simpleClient.executePrepared(prepUpdate,
+ params,
+ ConsistencyLevel.LOCAL_ONE);
+ ResultMessage.Rows rows = (ResultMessage.Rows) result;
+ EnumSet<org.apache.cassandra.cql3.ResultSet.Flag> resultFlags = rows.result.metadata.getFlags();
+ assertEquals(EnumSet.of(org.apache.cassandra.cql3.ResultSet.Flag.GLOBAL_TABLES_SPEC),
+ resultFlags);
+ assertEquals(columnNames.size(),
+ rows.result.metadata.getColumnCount());
+ assertEquals(columnNames,
+ rows.result.metadata.names.stream().map(cs -> cs.name.toString()).collect(Collectors.toList()));
+ assertEquals(1,
+ rows.result.size());
+ assertEquals(expectedRow,
+ rows.result.rows.get(0));
+ }
+
+ @Test
+ public void testPrepareWithLWT() throws Throwable
+ {
+ testPrepareWithLWT(ProtocolVersion.V4);
+ testPrepareWithLWT(ProtocolVersion.V5);
+ }
+
+ private void testPrepareWithLWT(ProtocolVersion version) throws Throwable
+ {
+ Session session = sessionNet(version);
+ session.execute("USE " + keyspace());
+ createTable("CREATE TABLE %s (pk int, v1 int, v2 int, PRIMARY KEY (pk))");
+
+ PreparedStatement prepared1 = session.prepare(String.format("UPDATE %s SET v1 = ?, v2 = ? WHERE pk = 1 IF v1 = ?", currentTable()));
+ PreparedStatement prepared2 = session.prepare(String.format("INSERT INTO %s (pk, v1, v2) VALUES (?, 200, 300) IF NOT EXISTS", currentTable()));
+ execute("INSERT INTO %s (pk, v1, v2) VALUES (1,1,1)");
+ execute("INSERT INTO %s (pk, v1, v2) VALUES (2,2,2)");
+
+ ResultSet rs;
+
+ rs = session.execute(prepared1.bind(10, 20, 1));
+ assertRowsNet(rs,
+ row(true));
+ assertEquals(rs.getColumnDefinitions().size(), 1);
+
+ rs = session.execute(prepared1.bind(100, 200, 1));
+ assertRowsNet(rs,
+ row(false, 10));
+ assertEquals(rs.getColumnDefinitions().size(), 2);
+
+ rs = session.execute(prepared1.bind(30, 40, 10));
+ assertRowsNet(rs,
+ row(true));
+ assertEquals(rs.getColumnDefinitions().size(), 1);
+
+ // Try executing the same message once again
+ rs = session.execute(prepared1.bind(100, 200, 1));
+ assertRowsNet(rs,
+ row(false, 30));
+ assertEquals(rs.getColumnDefinitions().size(), 2);
+
+ rs = session.execute(prepared2.bind(1));
+ assertRowsNet(rs,
+ row(false, 1, 30, 40));
+ assertEquals(rs.getColumnDefinitions().size(), 4);
+
+ alterTable("ALTER TABLE %s ADD v3 int;");
+
+ rs = session.execute(prepared2.bind(1));
+ assertRowsNet(rs,
+ row(false, 1, 30, 40, null));
+ assertEquals(rs.getColumnDefinitions().size(), 5);
+
+ rs = session.execute(prepared2.bind(20));
+ assertRowsNet(rs,
+ row(true));
+ assertEquals(rs.getColumnDefinitions().size(), 1);
+
+ rs = session.execute(prepared2.bind(20));
+ assertRowsNet(rs,
+ row(false, 20, 200, 300, null));
+ assertEquals(rs.getColumnDefinitions().size(), 5);
+ }
+
+ @Test
+ public void testPrepareWithBatchLWT() throws Throwable
+ {
+ testPrepareWithBatchLWT(ProtocolVersion.V4);
+ testPrepareWithBatchLWT(ProtocolVersion.V5);
+ }
+
+ private void testPrepareWithBatchLWT(ProtocolVersion version) throws Throwable
+ {
+ Session session = sessionNet(version);
+ session.execute("USE " + keyspace());
+ createTable("CREATE TABLE %s (pk int, v1 int, v2 int, PRIMARY KEY (pk))");
+
+ PreparedStatement prepared1 = session.prepare("BEGIN BATCH " +
+ "UPDATE " + currentTable() + " SET v1 = ? WHERE pk = 1 IF v1 = ?;" +
+ "UPDATE " + currentTable() + " SET v2 = ? WHERE pk = 1 IF v2 = ?;" +
+ "APPLY BATCH;");
+ PreparedStatement prepared2 = session.prepare("BEGIN BATCH " +
+ "INSERT INTO " + currentTable() + " (pk, v1, v2) VALUES (1, 200, 300) IF NOT EXISTS;" +
+ "APPLY BATCH");
+ execute("INSERT INTO %s (pk, v1, v2) VALUES (1,1,1)");
+ execute("INSERT INTO %s (pk, v1, v2) VALUES (2,2,2)");
+
+ com.datastax.driver.core.ResultSet rs;
+
+ rs = session.execute(prepared1.bind(10, 1, 20, 1));
+ assertRowsNet(rs,
+ row(true));
+ assertEquals(rs.getColumnDefinitions().size(), 1);
+
+ rs = session.execute(prepared1.bind(100, 1, 200, 1));
+ assertRowsNet(rs,
+ row(false, 1, 10, 20));
+ assertEquals(rs.getColumnDefinitions().size(), 4);
+
+ // Try executing the same message once again
+ rs = session.execute(prepared1.bind(100, 1, 200, 1));
+ assertRowsNet(rs,
+ row(false, 1, 10, 20));
+ assertEquals(rs.getColumnDefinitions().size(), 4);
+
+ rs = session.execute(prepared2.bind());
+ assertRowsNet(rs,
+ row(false, 1, 10, 20));
+ assertEquals(rs.getColumnDefinitions().size(), 4);
+
+ alterTable("ALTER TABLE %s ADD v3 int;");
+
+ rs = session.execute(prepared2.bind());
+ assertRowsNet(rs,
+ row(false, 1, 10, 20, null));
+ assertEquals(rs.getColumnDefinitions().size(), 5);
+ }
}
diff --git a/test/unit/org/apache/cassandra/cql3/PstmtPersistenceTest.java b/test/unit/org/apache/cassandra/cql3/PstmtPersistenceTest.java
index e7adc8e..eca6c20 100644
--- a/test/unit/org/apache/cassandra/cql3/PstmtPersistenceTest.java
+++ b/test/unit/org/apache/cassandra/cql3/PstmtPersistenceTest.java
@@ -22,15 +22,14 @@
import java.util.Collections;
import java.util.List;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
-import junit.framework.Assert;
-import org.apache.cassandra.config.SchemaConstants;
-import org.apache.cassandra.cql3.statements.ParsedStatement;
import org.apache.cassandra.db.SystemKeyspace;
import org.apache.cassandra.db.marshal.Int32Type;
import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.schema.SchemaConstants;
import org.apache.cassandra.schema.SchemaKeyspace;
import org.apache.cassandra.service.ClientState;
import org.apache.cassandra.service.QueryState;
@@ -78,14 +77,14 @@
assertEquals(5, stmtIds.size());
assertEquals(5, QueryProcessor.preparedStatementsCount());
- Assert.assertEquals(5, numberOfStatementsOnDisk());
+ assertEquals(5, numberOfStatementsOnDisk());
QueryHandler handler = ClientState.getCQLQueryHandler();
validatePstmts(stmtIds, handler);
// clear prepared statements cache
QueryProcessor.clearPreparedStatements(true);
- Assert.assertEquals(0, QueryProcessor.preparedStatementsCount());
+ assertEquals(0, QueryProcessor.preparedStatementsCount());
for (MD5Digest stmtId : stmtIds)
Assert.assertNull(handler.getPrepared(stmtId));
@@ -99,7 +98,7 @@
for (UntypedResultSet.Row row : QueryProcessor.executeOnceInternal(queryAll))
{
MD5Digest digest = MD5Digest.wrap(ByteBufferUtil.getArray(row.getBytes("prepared_id")));
- ParsedStatement.Prepared prepared = QueryProcessor.instance.getPrepared(digest);
+ QueryProcessor.Prepared prepared = QueryProcessor.instance.getPrepared(digest);
Assert.assertNotNull(prepared);
}
@@ -128,8 +127,8 @@
private static void validatePstmt(QueryHandler handler, MD5Digest stmtId, QueryOptions options)
{
- ParsedStatement.Prepared prepared = handler.getPrepared(stmtId);
- assertNotNull(prepared);
+ QueryProcessor.Prepared prepared = handler.getPrepared(stmtId);
+ Assert.assertNotNull(prepared);
handler.processPrepared(prepared.statement, QueryState.forInternalCalls(), options, Collections.emptyMap(), System.nanoTime());
}
@@ -186,6 +185,6 @@
private MD5Digest prepareStatement(String stmt, String keyspace, String table, ClientState clientState)
{
- return QueryProcessor.prepare(String.format(stmt, keyspace + "." + table), clientState, false).statementId;
+ return QueryProcessor.prepare(String.format(stmt, keyspace + "." + table), clientState).statementId;
}
}
diff --git a/test/unit/org/apache/cassandra/cql3/QueryEventsTest.java b/test/unit/org/apache/cassandra/cql3/QueryEventsTest.java
new file mode 100644
index 0000000..dd1cb45
--- /dev/null
+++ b/test/unit/org/apache/cassandra/cql3/QueryEventsTest.java
@@ -0,0 +1,352 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cql3;
+
+import java.nio.ByteBuffer;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import javax.annotation.Nullable;
+
+import org.junit.Test;
+
+import com.datastax.driver.core.PreparedStatement;
+import com.datastax.driver.core.ResultSet;
+import com.datastax.driver.core.Session;
+import com.datastax.driver.core.SimpleStatement;
+import com.datastax.driver.core.Statement;
+import org.apache.cassandra.cql3.statements.BatchStatement;
+import org.apache.cassandra.cql3.statements.ModificationStatement;
+import org.apache.cassandra.cql3.statements.SelectStatement;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.service.QueryState;
+import org.apache.cassandra.transport.Message;
+import org.apache.cassandra.transport.messages.ResultMessage;
+import org.apache.cassandra.utils.ByteBufferUtil;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import static com.google.common.collect.Lists.newArrayList;
+
+public class QueryEventsTest extends CQLTester
+{
+ @Test
+ public void queryTest() throws Throwable
+ {
+ createTable("create table %s (id int primary key, v int)");
+ MockListener listener = new MockListener(getCurrentColumnFamilyStore());
+ QueryEvents.instance.registerListener(listener);
+ String query = formatQuery("insert into %s (id, v) values (1, 1)");
+ executeNet(query);
+ listener.verify("querySuccess", 1);
+ assertEquals(query, listener.query);
+ assertTrue(listener.statement instanceof ModificationStatement);
+
+ query = formatQuery("select * from %s where id=1");
+ executeNet(query);
+ listener.verify("querySuccess", 2);
+ assertEquals(query, listener.query);
+ assertTrue(listener.statement instanceof SelectStatement);
+
+ query = formatQuery("select xyz from %s where id=1");
+ Exception expectedException = null;
+ try
+ {
+ executeNet(query);
+ fail("Query should fail");
+ }
+ catch (Exception e)
+ {
+ expectedException = e;
+ }
+ listener.verify(newArrayList("querySuccess", "queryFailure"), newArrayList(2, 1));
+ assertEquals(query, listener.query);
+ assertNotNull(listener.e);
+ assertNotNull(expectedException);
+ }
+
+ @Test
+ public void prepareExecuteTest()
+ {
+ createTable("create table %s (id int primary key, v int)");
+ MockListener listener = new MockListener(getCurrentColumnFamilyStore());
+ QueryEvents.instance.registerListener(listener);
+ Session session = sessionNet();
+ String query = formatQuery("select * from %s where id = 1");
+ PreparedStatement ps = session.prepare(query);
+ listener.verify("prepareSuccess", 1);
+ assertEquals(query, listener.query);
+ assertTrue(listener.statement instanceof SelectStatement);
+ Statement s = ps.bind();
+ session.execute(s);
+ listener.verify(newArrayList("prepareSuccess", "executeSuccess"), newArrayList(1, 1));
+
+ QueryProcessor.clearPreparedStatements(false);
+ s = ps.bind();
+ session.execute(s); // this re-prepares the query!!
+ listener.verify(newArrayList("prepareSuccess", "executeSuccess", "executeFailure"), newArrayList(2, 2, 1));
+
+ query = formatQuery("select abcdef from %s where id = 1");
+ Exception expectedException = null;
+ try
+ {
+ session.prepare(query);
+ fail("should fail");
+ }
+ catch (Exception e)
+ {
+ expectedException = e;
+ }
+ listener.verify(newArrayList("prepareSuccess", "prepareFailure", "executeSuccess", "executeFailure"), newArrayList(2, 1, 2, 1));
+ assertNotNull(listener.e);
+ assertNotNull(expectedException);
+ }
+
+ @Test
+ public void batchTest()
+ {
+ createTable("create table %s (id int primary key, v int)");
+ BatchMockListener listener = new BatchMockListener(getCurrentColumnFamilyStore());
+ QueryEvents.instance.registerListener(listener);
+ Session session = sessionNet();
+ com.datastax.driver.core.BatchStatement batch = new com.datastax.driver.core.BatchStatement(com.datastax.driver.core.BatchStatement.Type.UNLOGGED);
+ String q1 = formatQuery("insert into %s (id, v) values (?, ?)");
+ PreparedStatement ps = session.prepare(q1);
+ batch.add(ps.bind(1,1));
+ batch.add(ps.bind(2,2));
+ String q2 = formatQuery("insert into %s (id, v) values (1,1)");
+ batch.add(new SimpleStatement(formatQuery("insert into %s (id, v) values (1,1)")));
+ session.execute(batch);
+
+ listener.verify(newArrayList("prepareSuccess", "batchSuccess"), newArrayList(1, 1));
+ assertEquals(3, listener.queries.size());
+ assertEquals(BatchStatement.Type.UNLOGGED, listener.batchType);
+ assertEquals(newArrayList(q1, q1, q2), listener.queries);
+ assertEquals(newArrayList(newArrayList(ByteBufferUtil.bytes(1), ByteBufferUtil.bytes(1)),
+ newArrayList(ByteBufferUtil.bytes(2), ByteBufferUtil.bytes(2)),
+ newArrayList(newArrayList())), listener.values);
+
+ batch.add(new SimpleStatement("insert into abc.def (id, v) values (1,1)"));
+ try
+ {
+ session.execute(batch);
+ fail("Batch should fail");
+ }
+ catch (Exception e)
+ {
+ // ok
+ }
+ listener.verify(newArrayList("prepareSuccess", "batchSuccess", "batchFailure"), newArrayList(1, 1, 1));
+ assertEquals(3, listener.queries.size());
+ assertEquals(BatchStatement.Type.UNLOGGED, listener.batchType);
+ assertEquals(newArrayList(q1, q1, q2), listener.queries);
+ assertEquals(newArrayList(newArrayList(ByteBufferUtil.bytes(1), ByteBufferUtil.bytes(1)),
+ newArrayList(ByteBufferUtil.bytes(2), ByteBufferUtil.bytes(2)),
+ newArrayList(newArrayList()),
+ newArrayList(newArrayList())), listener.values);
+ }
+
+ @Test
+ public void errorInListenerTest() throws Throwable
+ {
+ createTable("create table %s (id int primary key, v int)");
+ QueryEvents.Listener listener = new QueryEvents.Listener()
+ {
+ @Override
+ public void querySuccess(CQLStatement statement, String query, QueryOptions options, QueryState state, long queryTime, Message.Response response)
+ {
+ throw new AssertionError("whoo");
+ }
+
+ @Override
+ public void queryFailure(@Nullable CQLStatement statement, String query, QueryOptions options, QueryState state, Exception cause)
+ {
+ throw new AssertionError("whee");
+ }
+ };
+
+ QueryEvents.instance.registerListener(listener);
+
+ executeNet("insert into %s (id, v) values (2,2)");
+ ResultSet rs = executeNet("select * from %s");
+ assertEquals(2, rs.one().getInt("id"));
+
+ // record the exception without the throwing listener:
+ QueryEvents.instance.unregisterListener(listener);
+ Exception expected = null;
+ try
+ {
+ executeNet("select blabla from %s");
+ fail("Query should throw");
+ }
+ catch (Exception e)
+ {
+ expected = e;
+ }
+
+ QueryEvents.instance.registerListener(listener);
+ // and with the listener:
+ try
+ {
+ executeNet("select blabla from %s");
+ fail("Query should throw");
+ }
+ catch (Exception e)
+ {
+ // make sure we throw the same exception even if the listener throws;
+ assertSame(expected.getClass(), e.getClass());
+ assertEquals(expected.getMessage(), e.getMessage());
+ }
+
+
+ }
+
+ private static class MockListener implements QueryEvents.Listener
+ {
+ private final String tableName;
+ private Map<String, Integer> callCounts = new HashMap<>();
+ private String query;
+ private CQLStatement statement;
+ private long start = System.currentTimeMillis();
+ long queryTime;
+ private Exception e;
+
+
+ MockListener(ColumnFamilyStore currentColumnFamilyStore)
+ {
+ tableName = currentColumnFamilyStore.getTableName();
+ }
+
+ public void querySuccess(CQLStatement statement, String query, QueryOptions options, QueryState state, long queryTime, Message.Response response)
+ {
+ if (query.contains(tableName))
+ {
+ inc("querySuccess");
+ assertNotNull(query);
+ this.query = query;
+ assertNotNull(statement);
+ this.statement = statement;
+ this.queryTime = queryTime;
+ }
+ }
+
+ public void queryFailure(@Nullable CQLStatement statement, String query, QueryOptions options, QueryState state, Exception cause)
+ {
+
+ if (query.contains(tableName))
+ {
+ inc("queryFailure");
+ assertNotNull(query);
+ this.query = query;
+ this.statement = statement;
+ e = cause;
+ }
+ }
+
+ public void executeSuccess(CQLStatement statement, String query, QueryOptions options, QueryState state, long queryTime, Message.Response response)
+ {
+ if (query.contains(tableName))
+ inc("executeSuccess");
+ }
+ public void executeFailure(@Nullable CQLStatement statement, @Nullable String query, QueryOptions options, QueryState state, Exception cause)
+ {
+ inc("executeFailure");
+ e = cause;
+ }
+
+
+ public void prepareSuccess(CQLStatement statement, String query, QueryState state, long queryTime, ResultMessage.Prepared response)
+ {
+ if (query.contains(tableName))
+ {
+ inc("prepareSuccess");
+ assertNotNull(query);
+ this.query = query;
+ this.statement = statement;
+ this.queryTime = queryTime;
+ }
+ }
+ public void prepareFailure(@Nullable CQLStatement statement, String query, QueryState state, Exception cause)
+ {
+ if (query.contains(tableName))
+ {
+ inc("prepareFailure");
+ assertNotNull(query);
+ assertNull(statement);
+ this.query = query;
+ }
+ }
+
+ void inc(String key)
+ {
+ callCounts.compute(key, (k, v) -> v == null ? 1 : v + 1);
+ }
+
+ private void verify(String key, int expected)
+ {
+ verify(Collections.singletonList(key), Collections.singletonList(expected));
+ }
+
+ void verify(List<String> keys, List<Integer> expected)
+ {
+ assertEquals("key count not equal: "+keys+" : "+callCounts, keys.size(), callCounts.size());
+ assertTrue(queryTime >= start && queryTime <= System.currentTimeMillis());
+ for (int i = 0; i < keys.size(); i++)
+ {
+ assertEquals("expected count mismatch for "+keys.get(i), (int)expected.get(i), (int) callCounts.get(keys.get(i)));
+ }
+ }
+ }
+
+ private static class BatchMockListener extends MockListener
+ {
+ private List<String> queries;
+ private List<List<ByteBuffer>> values;
+ private BatchStatement.Type batchType;
+
+ BatchMockListener(ColumnFamilyStore currentColumnFamilyStore)
+ {
+ super(currentColumnFamilyStore);
+ }
+
+ public void batchSuccess(BatchStatement.Type batchType, List<? extends CQLStatement> statements, List<String> queries, List<List<ByteBuffer>> values, QueryOptions options, QueryState state, long queryTime, Message.Response response)
+ {
+ inc("batchSuccess");
+ this.queries = queries;
+ this.values = values;
+ this.batchType = batchType;
+ this.queryTime = queryTime;
+ }
+
+ public void batchFailure(BatchStatement.Type batchType, List<? extends CQLStatement> statements, List<String> queries, List<List<ByteBuffer>> values, QueryOptions options, QueryState state, Exception cause)
+ {
+ inc("batchFailure");
+ this.queries = queries;
+ this.values = values;
+ this.batchType = batchType;
+ }
+
+ }
+}
diff --git a/test/unit/org/apache/cassandra/cql3/ReservedKeywordsTest.java b/test/unit/org/apache/cassandra/cql3/ReservedKeywordsTest.java
index aaf9824..111897e 100644
--- a/test/unit/org/apache/cassandra/cql3/ReservedKeywordsTest.java
+++ b/test/unit/org/apache/cassandra/cql3/ReservedKeywordsTest.java
@@ -20,7 +20,7 @@
import org.junit.Test;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.cassandra.exceptions.SyntaxException;
public class ReservedKeywordsTest
diff --git a/test/unit/org/apache/cassandra/cql3/SerializationMirrorTest.java b/test/unit/org/apache/cassandra/cql3/SerializationMirrorTest.java
index 3268784..ed49d70 100644
--- a/test/unit/org/apache/cassandra/cql3/SerializationMirrorTest.java
+++ b/test/unit/org/apache/cassandra/cql3/SerializationMirrorTest.java
@@ -24,7 +24,7 @@
import org.junit.Test;
-import junit.framework.Assert;
+import org.junit.Assert;
public class SerializationMirrorTest extends CQLTester
{
diff --git a/test/unit/org/apache/cassandra/cql3/SimpleQueryTest.java b/test/unit/org/apache/cassandra/cql3/SimpleQueryTest.java
index 62fe5a1..0c89f9b 100644
--- a/test/unit/org/apache/cassandra/cql3/SimpleQueryTest.java
+++ b/test/unit/org/apache/cassandra/cql3/SimpleQueryTest.java
@@ -22,31 +22,6 @@
public class SimpleQueryTest extends CQLTester
{
@Test
- public void testStaticCompactTables() throws Throwable
- {
- createTable("CREATE TABLE %s (k text PRIMARY KEY, v1 int, v2 text) WITH COMPACT STORAGE");
-
- execute("INSERT INTO %s (k, v1, v2) values (?, ?, ?)", "first", 1, "value1");
- execute("INSERT INTO %s (k, v1, v2) values (?, ?, ?)", "second", 2, "value2");
- execute("INSERT INTO %s (k, v1, v2) values (?, ?, ?)", "third", 3, "value3");
-
- assertRows(execute("SELECT * FROM %s WHERE k = ?", "first"),
- row("first", 1, "value1")
- );
-
- assertRows(execute("SELECT v2 FROM %s WHERE k = ?", "second"),
- row("value2")
- );
-
- // Murmur3 order
- assertRows(execute("SELECT * FROM %s"),
- row("third", 3, "value3"),
- row("second", 2, "value2"),
- row("first", 1, "value1")
- );
- }
-
- @Test
public void testDynamicCompactTables() throws Throwable
{
createTable("CREATE TABLE %s (k text, t int, v text, PRIMARY KEY (k, t));");
@@ -502,26 +477,6 @@
}
@Test
- public void testCompactStorageUpdateWithNull() throws Throwable
- {
- createTable("CREATE TABLE %s (partitionKey int," +
- "clustering_1 int," +
- "value int," +
- " PRIMARY KEY (partitionKey, clustering_1)) WITH COMPACT STORAGE");
-
- execute("INSERT INTO %s (partitionKey, clustering_1, value) VALUES (0, 0, 0)");
- execute("INSERT INTO %s (partitionKey, clustering_1, value) VALUES (0, 1, 1)");
-
- flush();
-
- execute("UPDATE %s SET value = ? WHERE partitionKey = ? AND clustering_1 = ?", null, 0, 0);
-
- assertRows(execute("SELECT * FROM %s WHERE partitionKey = ? AND (clustering_1) IN ((?), (?))", 0, 0, 1),
- row(0, 1, 1)
- );
- }
-
- @Test
public void test2ndaryIndexBug() throws Throwable
{
createTable("CREATE TABLE %s (k int, c1 int, c2 int, v int, PRIMARY KEY(k, c1, c2))");
diff --git a/test/unit/org/apache/cassandra/cql3/ThriftCompatibilityTest.java b/test/unit/org/apache/cassandra/cql3/ThriftCompatibilityTest.java
deleted file mode 100644
index 521c0a0..0000000
--- a/test/unit/org/apache/cassandra/cql3/ThriftCompatibilityTest.java
+++ /dev/null
@@ -1,110 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.cql3;
-
-import java.util.Arrays;
-import java.util.Collections;
-
-import org.junit.Test;
-
-import org.apache.cassandra.SchemaLoader;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.Schema;
-import org.apache.cassandra.db.marshal.BytesType;
-import org.apache.cassandra.db.marshal.Int32Type;
-import org.apache.cassandra.db.marshal.UTF8Type;
-import org.apache.cassandra.schema.KeyspaceParams;
-import org.apache.cassandra.service.MigrationManager;
-import org.apache.cassandra.thrift.CfDef;
-import org.apache.cassandra.thrift.ColumnDef;
-import org.apache.cassandra.thrift.ThriftConversion;
-import org.apache.cassandra.utils.ByteBufferUtil;
-
-import static junit.framework.Assert.assertFalse;
-import static junit.framework.Assert.assertTrue;
-import static org.junit.Assert.assertEquals;
-import static org.apache.cassandra.utils.ByteBufferUtil.bytes;
-
-public class ThriftCompatibilityTest extends SchemaLoader
-{
- @Test // test for CASSANDRA-8178
- public void testNonTextComparator() throws Throwable
- {
- ColumnDef column = new ColumnDef();
- column.setName(bytes(42))
- .setValidation_class(UTF8Type.instance.toString());
-
- CfDef cf = new CfDef("thriftcompat", "JdbcInteger");
- cf.setColumn_type("Standard")
- .setComparator_type(Int32Type.instance.toString())
- .setDefault_validation_class(UTF8Type.instance.toString())
- .setKey_validation_class(BytesType.instance.toString())
- .setColumn_metadata(Collections.singletonList(column));
-
- SchemaLoader.createKeyspace("thriftcompat", KeyspaceParams.simple(1), ThriftConversion.fromThrift(cf));
-
- // the comparator is IntegerType, and there is a column named 42 with a UTF8Type validation type
- execute("INSERT INTO \"thriftcompat\".\"JdbcInteger\" (key, \"42\") VALUES (0x00000001, 'abc')");
- execute("UPDATE \"thriftcompat\".\"JdbcInteger\" SET \"42\" = 'abc' WHERE key = 0x00000001");
- execute("DELETE \"42\" FROM \"thriftcompat\".\"JdbcInteger\" WHERE key = 0x00000000");
- UntypedResultSet results = execute("SELECT key, \"42\" FROM \"thriftcompat\".\"JdbcInteger\"");
- assertEquals(1, results.size());
- UntypedResultSet.Row row = results.iterator().next();
- assertEquals(ByteBufferUtil.bytes(1), row.getBytes("key"));
- assertEquals("abc", row.getString("42"));
- }
-
- @Test // test for CASSANDRA-9867
- public void testDropCompactStaticColumn()
- {
- ColumnDef column1 = new ColumnDef();
- column1.setName(bytes(42))
- .setValidation_class(UTF8Type.instance.toString());
-
- ColumnDef column2 = new ColumnDef();
- column2.setName(bytes(25))
- .setValidation_class(UTF8Type.instance.toString());
-
- CfDef cf = new CfDef("thriftks", "staticcompact");
- cf.setColumn_type("Standard")
- .setComparator_type(Int32Type.instance.toString())
- .setDefault_validation_class(UTF8Type.instance.toString())
- .setKey_validation_class(BytesType.instance.toString())
- .setColumn_metadata(Arrays.asList(column1, column2));
-
- SchemaLoader.createKeyspace("thriftks", KeyspaceParams.simple(1), ThriftConversion.fromThrift(cf));
- CFMetaData cfm = Schema.instance.getCFMetaData("thriftks", "staticcompact");
-
- // assert the both columns are in the metadata
- assertTrue(cfm.getColumnMetadata().containsKey(bytes(42)));
- assertTrue(cfm.getColumnMetadata().containsKey(bytes(25)));
-
- // remove column2
- cf.setColumn_metadata(Collections.singletonList(column1));
- MigrationManager.announceColumnFamilyUpdate(ThriftConversion.fromThriftForUpdate(cf, cfm), true);
-
- // assert that it's gone from metadata
- assertTrue(cfm.getColumnMetadata().containsKey(bytes(42)));
- assertFalse(cfm.getColumnMetadata().containsKey(bytes(25)));
- }
-
- private static UntypedResultSet execute(String query)
- {
- return QueryProcessor.executeInternal(query);
- }
-}
diff --git a/test/unit/org/apache/cassandra/cql3/ViewComplexTest.java b/test/unit/org/apache/cassandra/cql3/ViewComplexTest.java
index 99f0097..490756f 100644
--- a/test/unit/org/apache/cassandra/cql3/ViewComplexTest.java
+++ b/test/unit/org/apache/cassandra/cql3/ViewComplexTest.java
@@ -17,7 +17,6 @@
import org.apache.cassandra.concurrent.SEPExecutor;
import org.apache.cassandra.concurrent.Stage;
-import org.apache.cassandra.concurrent.StageManager;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.db.compaction.CompactionManager;
@@ -41,6 +40,7 @@
{
requireNetwork();
}
+
@Before
public void begin()
{
@@ -70,8 +70,8 @@
private void updateViewWithFlush(String query, boolean flush, Object... params) throws Throwable
{
executeNet(protocolVersion, query, params);
- while (!(((SEPExecutor) StageManager.getStage(Stage.VIEW_MUTATION)).getPendingTasks() == 0
- && ((SEPExecutor) StageManager.getStage(Stage.VIEW_MUTATION)).getActiveCount() == 0))
+ while (!(((SEPExecutor) Stage.VIEW_MUTATION.executor()).getPendingTaskCount() == 0
+ && ((SEPExecutor) Stage.VIEW_MUTATION.executor()).getActiveTaskCount() == 0))
{
Thread.sleep(1);
}
@@ -79,7 +79,7 @@
Keyspace.open(keyspace()).flush();
}
- // for now, unselected column cannot be fully supported, SEE CASSANDRA-13826
+ // for now, unselected column cannot be fully supported, SEE CASSANDRA-11500
@Ignore
@Test
public void testPartialDeleteUnselectedColumn() throws Throwable
@@ -143,7 +143,7 @@
executeNet(protocolVersion, "USE " + keyspace());
createTable("CREATE TABLE %s (k int, c int, a int, b int, e int, f int, PRIMARY KEY (k, c))");
createView("mv",
- "CREATE MATERIALIZED VIEW %s AS SELECT a, b FROM %%s WHERE k IS NOT NULL AND c IS NOT NULL PRIMARY KEY (k,c)");
+ "CREATE MATERIALIZED VIEW %s AS SELECT a, b, c, k FROM %%s WHERE k IS NOT NULL AND c IS NOT NULL PRIMARY KEY (k,c)");
Keyspace ks = Keyspace.open(keyspace());
ks.getColumnFamilyStore("mv").disableAutoCompaction();
@@ -400,7 +400,7 @@
assertRowsIgnoringOrder(execute("SELECT * from %s WHERE c = ? AND p = ?", 0, 0), row(0, 0, null, 1));
assertRowsIgnoringOrder(execute("SELECT * from mv WHERE c = ? AND p = ?", 0, 0), row(0, 0));
- assertInvalidMessage(String.format("Cannot drop column v2 on base table %s with materialized views.", baseTable), "ALTER TABLE %s DROP v2");
+ assertInvalidMessage(String.format("Cannot drop column v2 on base table %s with materialized views", baseTable), "ALTER TABLE %s DROP v2");
// // drop unselected base column, unselected metadata should be removed, thus view row is dead
// updateView("ALTER TABLE %s DROP v2");
// assertRowsIgnoringOrder(execute("SELECT * from %s WHERE c = ? AND p = ?", 0, 0));
@@ -427,7 +427,7 @@
executeNet(protocolVersion, "USE " + keyspace());
String baseTable = createTable("CREATE TABLE %s (k int, c int, a int, b int, l list<int>, s set<int>, m map<int,int>, PRIMARY KEY (k, c))");
createView("mv",
- "CREATE MATERIALIZED VIEW %s AS SELECT a, b FROM %%s WHERE k IS NOT NULL AND c IS NOT NULL PRIMARY KEY (c, k)");
+ "CREATE MATERIALIZED VIEW %s AS SELECT a, b, c, k FROM %%s WHERE k IS NOT NULL AND c IS NOT NULL PRIMARY KEY (c, k)");
Keyspace ks = Keyspace.open(keyspace());
ks.getColumnFamilyStore("mv").disableAutoCompaction();
@@ -461,7 +461,7 @@
assertRowsIgnoringOrder(execute("SELECT k,c,a,b from %s"), row(1, 1, null, null));
assertRowsIgnoringOrder(execute("SELECT * from mv"), row(1, 1, null, null));
- assertInvalidMessage(String.format("Cannot drop column m on base table %s with materialized views.", baseTable), "ALTER TABLE %s DROP m");
+ assertInvalidMessage(String.format("Cannot drop column m on base table %s with materialized views", baseTable), "ALTER TABLE %s DROP m");
// executeNet(protocolVersion, "ALTER TABLE %s DROP m");
// ks.getColumnFamilyStore("mv").forceMajorCompaction();
// assertRowsIgnoringOrder(execute("SELECT k,c,a,b from %s WHERE k = 1 AND c = 1"));
@@ -850,10 +850,10 @@
for (String view : Arrays.asList("mv1", "mv2"))
{
// paging
- assertEquals(1, executeNetWithPaging(String.format("SELECT k,a,b FROM %s limit 1", view), 1).all().size());
- assertEquals(2, executeNetWithPaging(String.format("SELECT k,a,b FROM %s limit 2", view), 1).all().size());
- assertEquals(2, executeNetWithPaging(String.format("SELECT k,a,b FROM %s", view), 1).all().size());
- assertRowsNet(executeNetWithPaging(String.format("SELECT k,a,b FROM %s ", view), 1),
+ assertEquals(1, executeNetWithPaging(protocolVersion, String.format("SELECT k,a,b FROM %s limit 1", view), 1).all().size());
+ assertEquals(2, executeNetWithPaging(protocolVersion, String.format("SELECT k,a,b FROM %s limit 2", view), 1).all().size());
+ assertEquals(2, executeNetWithPaging(protocolVersion, String.format("SELECT k,a,b FROM %s", view), 1).all().size());
+ assertRowsNet(protocolVersion, executeNetWithPaging(protocolVersion, String.format("SELECT k,a,b FROM %s ", view), 1),
row(50, 50, 50),
row(100, 100, 100));
// limit
@@ -930,7 +930,7 @@
assertRowsIgnoringOrder(execute("SELECT k,a,b from mv"), row(1, 1, 2));
assertRowsIgnoringOrder(execute("SELECT k,a,b from %s"), row(1, 1, 2));
- assertInvalidMessage(String.format("Cannot drop column a on base table %s with materialized views.", baseTable), "ALTER TABLE %s DROP a");
+ assertInvalidMessage(String.format("Cannot drop column a on base table %s with materialized views", baseTable), "ALTER TABLE %s DROP a");
}
@Test
@@ -1272,7 +1272,7 @@
// all selected
"CREATE MATERIALIZED VIEW %s AS SELECT * FROM %%s WHERE a IS NOT NULL AND b IS NOT NULL PRIMARY KEY (a,b)",
// unselected e,f
- "CREATE MATERIALIZED VIEW %s AS SELECT c,d FROM %%s WHERE a IS NOT NULL AND b IS NOT NULL PRIMARY KEY (a,b)",
+ "CREATE MATERIALIZED VIEW %s AS SELECT a,b,c,d FROM %%s WHERE a IS NOT NULL AND b IS NOT NULL PRIMARY KEY (a,b)",
// no selected
"CREATE MATERIALIZED VIEW %s AS SELECT a,b FROM %%s WHERE a IS NOT NULL AND b IS NOT NULL PRIMARY KEY (a,b)",
// all selected, re-order keys
@@ -1403,5 +1403,4 @@
}
}
}
-
}
diff --git a/test/unit/org/apache/cassandra/cql3/ViewFilteringTest.java b/test/unit/org/apache/cassandra/cql3/ViewFilteringTest.java
index 6803230..46488af 100644
--- a/test/unit/org/apache/cassandra/cql3/ViewFilteringTest.java
+++ b/test/unit/org/apache/cassandra/cql3/ViewFilteringTest.java
@@ -28,14 +28,10 @@
import org.junit.Test;
import com.datastax.driver.core.exceptions.InvalidQueryException;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.cassandra.concurrent.SEPExecutor;
import org.apache.cassandra.concurrent.Stage;
-import org.apache.cassandra.concurrent.StageManager;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
-import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.db.SystemKeyspace;
import org.apache.cassandra.transport.ProtocolVersion;
@@ -83,8 +79,8 @@
private void updateView(String query, Object... params) throws Throwable
{
executeNet(protocolVersion, query, params);
- while (!(((SEPExecutor) StageManager.getStage(Stage.VIEW_MUTATION)).getPendingTasks() == 0
- && ((SEPExecutor) StageManager.getStage(Stage.VIEW_MUTATION)).getActiveCount() == 0))
+ while (!(((SEPExecutor) Stage.VIEW_MUTATION.executor()).getPendingTaskCount() == 0
+ && ((SEPExecutor) Stage.VIEW_MUTATION.executor()).getActiveTaskCount() == 0))
{
Thread.sleep(1);
}
@@ -96,13 +92,7 @@
views.remove(name);
}
- private static void waitForView(String keyspace, String view) throws InterruptedException
- {
- while (!SystemKeyspace.isViewBuilt(keyspace, view))
- Thread.sleep(10);
- }
-
- // TODO will revise the non-pk filter condition in MV, see CASSANDRA-13826
+ // TODO will revise the non-pk filter condition in MV, see CASSANDRA-11500
@Ignore
@Test
public void testViewFilteringWithFlush() throws Throwable
@@ -110,7 +100,7 @@
testViewFiltering(true);
}
- // TODO will revise the non-pk filter condition in MV, see CASSANDRA-13826
+ // TODO will revise the non-pk filter condition in MV, see CASSANDRA-11500
@Ignore
@Test
public void testViewFilteringWithoutFlush() throws Throwable
@@ -326,7 +316,7 @@
dropTable("DROP TABLE %s");
}
- // TODO will revise the non-pk filter condition in MV, see CASSANDRA-13826
+ // TODO will revise the non-pk filter condition in MV, see CASSANDRA-11500
@Ignore
@Test
public void testMVFilteringWithComplexColumn() throws Throwable
@@ -782,6 +772,15 @@
}
}
+
+
+
+ private static void waitForView(String keyspace, String view) throws InterruptedException
+ {
+ while (!SystemKeyspace.isViewBuilt(keyspace, view))
+ Thread.sleep(10);
+ }
+
@Test
public void testPartitionKeyRestrictions() throws Throwable
{
@@ -1753,61 +1752,61 @@
"udtval";
createTable(
- "CREATE TABLE %s (" +
- "asciival ascii, " +
- "bigintval bigint, " +
- "blobval blob, " +
- "booleanval boolean, " +
- "dateval date, " +
- "decimalval decimal, " +
- "doubleval double, " +
- "floatval float, " +
- "inetval inet, " +
- "intval int, " +
- "textval text, " +
- "timeval time, " +
- "timestampval timestamp, " +
- "timeuuidval timeuuid, " +
- "uuidval uuid," +
- "varcharval varchar, " +
- "varintval varint, " +
- "frozenlistval frozen<list<int>>, " +
- "frozensetval frozen<set<uuid>>, " +
- "frozenmapval frozen<map<ascii, int>>," +
- "tupleval frozen<tuple<int, ascii, uuid>>," +
- "udtval frozen<" + myType + ">, " +
- "PRIMARY KEY (" + columnNames + "))");
+ "CREATE TABLE %s (" +
+ "asciival ascii, " +
+ "bigintval bigint, " +
+ "blobval blob, " +
+ "booleanval boolean, " +
+ "dateval date, " +
+ "decimalval decimal, " +
+ "doubleval double, " +
+ "floatval float, " +
+ "inetval inet, " +
+ "intval int, " +
+ "textval text, " +
+ "timeval time, " +
+ "timestampval timestamp, " +
+ "timeuuidval timeuuid, " +
+ "uuidval uuid," +
+ "varcharval varchar, " +
+ "varintval varint, " +
+ "frozenlistval frozen<list<int>>, " +
+ "frozensetval frozen<set<uuid>>, " +
+ "frozenmapval frozen<map<ascii, int>>," +
+ "tupleval frozen<tuple<int, ascii, uuid>>," +
+ "udtval frozen<" + myType + ">, " +
+ "PRIMARY KEY (" + columnNames + "))");
execute("USE " + keyspace());
executeNet(protocolVersion, "USE " + keyspace());
createView(
- "mv_test",
- "CREATE MATERIALIZED VIEW %s AS SELECT * FROM %%s WHERE " +
- "asciival = 'abc' AND " +
- "bigintval = 123 AND " +
- "blobval = 0xfeed AND " +
- "booleanval = true AND " +
- "dateval = '1987-03-23' AND " +
- "decimalval = 123.123 AND " +
- "doubleval = 123.123 AND " +
- "floatval = 123.123 AND " +
- "inetval = '127.0.0.1' AND " +
- "intval = 123 AND " +
- "textval = 'abc' AND " +
- "timeval = '07:35:07.000111222' AND " +
- "timestampval = 123123123 AND " +
- "timeuuidval = 6BDDC89A-5644-11E4-97FC-56847AFE9799 AND " +
- "uuidval = 6BDDC89A-5644-11E4-97FC-56847AFE9799 AND " +
- "varcharval = 'abc' AND " +
- "varintval = 123123123 AND " +
- "frozenlistval = [1, 2, 3] AND " +
- "frozensetval = {6BDDC89A-5644-11E4-97FC-56847AFE9799} AND " +
- "frozenmapval = {'a': 1, 'b': 2} AND " +
- "tupleval = (1, 'foobar', 6BDDC89A-5644-11E4-97FC-56847AFE9799) AND " +
- "udtval = {a: 1, b: 6BDDC89A-5644-11E4-97FC-56847AFE9799, c: {'foo', 'bar'}} " +
- "PRIMARY KEY (" + columnNames + ")");
+ "mv_test",
+ "CREATE MATERIALIZED VIEW %s AS SELECT * FROM %%s WHERE " +
+ "asciival = 'abc' AND " +
+ "bigintval = 123 AND " +
+ "blobval = 0xfeed AND " +
+ "booleanval = true AND " +
+ "dateval = '1987-03-23' AND " +
+ "decimalval = 123.123 AND " +
+ "doubleval = 123.123 AND " +
+ "floatval = 123.123 AND " +
+ "inetval = '127.0.0.1' AND " +
+ "intval = 123 AND " +
+ "textval = 'abc' AND " +
+ "timeval = '07:35:07.000111222' AND " +
+ "timestampval = 123123123 AND " +
+ "timeuuidval = 6BDDC89A-5644-11E4-97FC-56847AFE9799 AND " +
+ "uuidval = 6BDDC89A-5644-11E4-97FC-56847AFE9799 AND " +
+ "varcharval = 'abc' AND " +
+ "varintval = 123123123 AND " +
+ "frozenlistval = [1, 2, 3] AND " +
+ "frozensetval = {6BDDC89A-5644-11E4-97FC-56847AFE9799} AND " +
+ "frozenmapval = {'a': 1, 'b': 2} AND " +
+ "tupleval = (1, 'foobar', 6BDDC89A-5644-11E4-97FC-56847AFE9799) AND " +
+ "udtval = {a: 1, b: 6BDDC89A-5644-11E4-97FC-56847AFE9799, c: {'foo', 'bar'}} " +
+ "PRIMARY KEY (" + columnNames + ")");
execute("INSERT INTO %s (" + columnNames + ") VALUES (" +
"'abc'," +
@@ -1848,7 +1847,7 @@
executeNet(protocolVersion, "USE " + keyspace());
try {
- createView("mv_test", "CREATE MATERIALIZED VIEW %s AS SELECT * FROM %%s WHERE b IS NOT NULL AND c IS NOT NULL AND d = 1 PRIMARY KEY (a, b, c)");
+ createView("mv_test", "CREATE MATERIALIZED VIEW %s AS SELECT * FROM %%s WHERE a IS NOT NULL AND b IS NOT NULL AND c IS NOT NULL AND d = 1 PRIMARY KEY (a, b, c)");
dropView("mv_test");
} catch(Exception e) {
throw new RuntimeException("MV creation with non primary column restrictions failed.", e);
@@ -1881,77 +1880,77 @@
Thread.sleep(10);
assertRowsIgnoringOrder(execute("SELECT a, b, c, d FROM mv_test"),
- row(0, 0, 1, 0),
- row(0, 1, 1, 0),
- row(1, 0, 1, 0),
- row(1, 1, 1, 0)
+ row(0, 0, 1, 0),
+ row(0, 1, 1, 0),
+ row(1, 0, 1, 0),
+ row(1, 1, 1, 0)
);
// insert new rows that do not match the filter
execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 2, 0, 0, 0);
execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 2, 1, 2, 0);
assertRowsIgnoringOrder(execute("SELECT a, b, c, d FROM mv_test"),
- row(0, 0, 1, 0),
- row(0, 1, 1, 0),
- row(1, 0, 1, 0),
- row(1, 1, 1, 0)
+ row(0, 0, 1, 0),
+ row(0, 1, 1, 0),
+ row(1, 0, 1, 0),
+ row(1, 1, 1, 0)
);
// insert new row that does match the filter
execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 1, 2, 1, 0);
assertRowsIgnoringOrder(execute("SELECT a, b, c, d FROM mv_test"),
- row(0, 0, 1, 0),
- row(0, 1, 1, 0),
- row(1, 0, 1, 0),
- row(1, 1, 1, 0),
- row(1, 2, 1, 0)
+ row(0, 0, 1, 0),
+ row(0, 1, 1, 0),
+ row(1, 0, 1, 0),
+ row(1, 1, 1, 0),
+ row(1, 2, 1, 0)
);
// update rows that don't match the filter
execute("UPDATE %s SET d = ? WHERE a = ? AND b = ?", 2, 2, 0);
execute("UPDATE %s SET d = ? WHERE a = ? AND b = ?", 1, 2, 1);
assertRowsIgnoringOrder(execute("SELECT a, b, c, d FROM mv_test"),
- row(0, 0, 1, 0),
- row(0, 1, 1, 0),
- row(1, 0, 1, 0),
- row(1, 1, 1, 0),
- row(1, 2, 1, 0)
+ row(0, 0, 1, 0),
+ row(0, 1, 1, 0),
+ row(1, 0, 1, 0),
+ row(1, 1, 1, 0),
+ row(1, 2, 1, 0)
);
// update a row that does match the filter
execute("UPDATE %s SET d = ? WHERE a = ? AND b = ?", 1, 1, 0);
assertRowsIgnoringOrder(execute("SELECT a, b, c, d FROM mv_test"),
- row(0, 0, 1, 0),
- row(0, 1, 1, 0),
- row(1, 0, 1, 1),
- row(1, 1, 1, 0),
- row(1, 2, 1, 0)
+ row(0, 0, 1, 0),
+ row(0, 1, 1, 0),
+ row(1, 0, 1, 1),
+ row(1, 1, 1, 0),
+ row(1, 2, 1, 0)
);
// delete rows that don't match the filter
execute("DELETE FROM %s WHERE a = ? AND b = ?", 2, 0);
assertRowsIgnoringOrder(execute("SELECT a, b, c, d FROM mv_test"),
- row(0, 0, 1, 0),
- row(0, 1, 1, 0),
- row(1, 0, 1, 1),
- row(1, 1, 1, 0),
- row(1, 2, 1, 0)
+ row(0, 0, 1, 0),
+ row(0, 1, 1, 0),
+ row(1, 0, 1, 1),
+ row(1, 1, 1, 0),
+ row(1, 2, 1, 0)
);
// delete a row that does match the filter
execute("DELETE FROM %s WHERE a = ? AND b = ?", 1, 2);
assertRowsIgnoringOrder(execute("SELECT a, b, c, d FROM mv_test"),
- row(0, 0, 1, 0),
- row(0, 1, 1, 0),
- row(1, 0, 1, 1),
- row(1, 1, 1, 0)
+ row(0, 0, 1, 0),
+ row(0, 1, 1, 0),
+ row(1, 0, 1, 1),
+ row(1, 1, 1, 0)
);
// delete a partition that matches the filter
execute("DELETE FROM %s WHERE a = ?", 1);
assertRowsIgnoringOrder(execute("SELECT a, b, c, d FROM mv_test"),
- row(0, 0, 1, 0),
- row(0, 1, 1, 0)
+ row(0, 0, 1, 0),
+ row(0, 1, 1, 0)
);
dropView("mv_test");
@@ -2039,6 +2038,7 @@
//Tombstone c
executeNet(protocolVersion, "DELETE FROM %s WHERE a = ? and b = ?", 0, 0);
+ assertRowsIgnoringOrder(execute("SELECT d from mv"));
assertRows(execute("SELECT d from mv"));
//Add back without D
@@ -2058,18 +2058,18 @@
// delete with timestamp 0 (which should only delete d)
executeNet(protocolVersion, "DELETE FROM %s USING TIMESTAMP 0 WHERE a = ? AND b = ?", 1, 0);
assertRows(execute("SELECT a, b, c, d, e from mv WHERE c = ? and a = ? and b = ?", 1, 1, 0),
- row(1, 0, 1, null, 0)
+ row(1, 0, 1, null, 0)
);
executeNet(protocolVersion, "UPDATE %s USING TIMESTAMP 2 SET c = ? WHERE a = ? AND b = ?", 1, 1, 1);
executeNet(protocolVersion, "UPDATE %s USING TIMESTAMP 3 SET c = ? WHERE a = ? AND b = ?", 1, 1, 0);
assertRows(execute("SELECT a, b, c, d, e from mv WHERE c = ? and a = ? and b = ?", 1, 1, 0),
- row(1, 0, 1, null, 0)
+ row(1, 0, 1, null, 0)
);
executeNet(protocolVersion, "UPDATE %s USING TIMESTAMP 3 SET d = ? WHERE a = ? AND b = ?", 0, 1, 0);
assertRows(execute("SELECT a, b, c, d, e from mv WHERE c = ? and a = ? and b = ?", 1, 1, 0),
- row(1, 0, 1, 0, 0)
+ row(1, 0, 1, 0, 0)
);
}
@@ -2079,9 +2079,9 @@
// Regression test for CASSANDRA-10910
createTable("CREATE TABLE %s (" +
- "k int PRIMARY KEY, " +
- "c int, " +
- "val int)");
+ "k int PRIMARY KEY, " +
+ "c int, " +
+ "val int)");
execute("USE " + keyspace());
executeNet(protocolVersion, "USE " + keyspace());
@@ -2107,10 +2107,10 @@
public void testOldTimestampsWithRestrictions() throws Throwable
{
createTable("CREATE TABLE %s (" +
- "k int, " +
- "c int, " +
- "val text, " + "" +
- "PRIMARY KEY(k, c))");
+ "k int, " +
+ "c int, " +
+ "val text, " + "" +
+ "PRIMARY KEY(k, c))");
execute("USE " + keyspace());
executeNet(protocolVersion, "USE " + keyspace());
diff --git a/test/unit/org/apache/cassandra/cql3/ViewSchemaTest.java b/test/unit/org/apache/cassandra/cql3/ViewSchemaTest.java
index c83d96d..2dcfe32 100644
--- a/test/unit/org/apache/cassandra/cql3/ViewSchemaTest.java
+++ b/test/unit/org/apache/cassandra/cql3/ViewSchemaTest.java
@@ -29,14 +29,13 @@
import java.util.List;
import java.util.UUID;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.cassandra.concurrent.SEPExecutor;
import org.apache.cassandra.concurrent.Stage;
-import org.apache.cassandra.concurrent.StageManager;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
-import org.apache.cassandra.config.Schema;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.db.SystemKeyspace;
import org.apache.cassandra.serializers.SimpleDateSerializer;
import org.apache.cassandra.serializers.TimeSerializer;
@@ -84,8 +83,8 @@
private void updateView(String query, Object... params) throws Throwable
{
executeNet(protocolVersion, query, params);
- while (!(((SEPExecutor) StageManager.getStage(Stage.VIEW_MUTATION)).getPendingTasks() == 0
- && ((SEPExecutor) StageManager.getStage(Stage.VIEW_MUTATION)).getActiveCount() == 0))
+ while (!(((SEPExecutor) Stage.VIEW_MUTATION.executor()).getPendingTaskCount() == 0
+ && ((SEPExecutor) Stage.VIEW_MUTATION.executor()).getActiveTaskCount() == 0))
{
Thread.sleep(1);
}
@@ -176,8 +175,8 @@
//Test alter add
executeNet(protocolVersion, "ALTER TABLE %s ADD foo text");
- CFMetaData metadata = Schema.instance.getCFMetaData(keyspace(), "mv1_test");
- Assert.assertNotNull(metadata.getColumnDefinition(ByteBufferUtil.bytes("foo")));
+ TableMetadata metadata = Schema.instance.getTableMetadata(keyspace(), "mv1_test");
+ Assert.assertNotNull(metadata.getColumn(ByteBufferUtil.bytes("foo")));
updateView("INSERT INTO %s(k,asciival,bigintval,foo)VALUES(?,?,?,?)", 0, "foo", 1L, "bar");
assertRows(execute("SELECT foo from %s"), row("bar"));
@@ -186,8 +185,8 @@
executeNet(protocolVersion, "ALTER TABLE %s RENAME asciival TO bar");
assertRows(execute("SELECT bar from %s"), row("foo"));
- metadata = Schema.instance.getCFMetaData(keyspace(), "mv1_test");
- Assert.assertNotNull(metadata.getColumnDefinition(ByteBufferUtil.bytes("bar")));
+ metadata = Schema.instance.getTableMetadata(keyspace(), "mv1_test");
+ Assert.assertNotNull(metadata.getColumn(ByteBufferUtil.bytes("bar")));
}
@@ -286,12 +285,12 @@
"tupleval frozen<tuple<int, ascii, uuid>>," +
"udtval frozen<" + myType + ">)");
- CFMetaData metadata = currentTableMetadata();
+ TableMetadata metadata = currentTableMetadata();
execute("USE " + keyspace());
executeNet(protocolVersion, "USE " + keyspace());
- for (ColumnDefinition def : new HashSet<>(metadata.allColumns()))
+ for (ColumnMetadata def : new HashSet<>(metadata.columns()))
{
try
{
@@ -672,7 +671,7 @@
executeNet(protocolVersion, "USE " + keyspace());
createView(keyspace() + ".mv1",
- "CREATE MATERIALIZED VIEW %s AS SELECT * FROM %%s WHERE b IS NOT NULL AND c IS NOT NULL PRIMARY KEY (a, b, c)");
+ "CREATE MATERIALIZED VIEW %s AS SELECT * FROM %%s WHERE a IS NOT NULL AND b IS NOT NULL AND c IS NOT NULL PRIMARY KEY (a, b, c)");
try
{
@@ -681,7 +680,7 @@
}
catch (InvalidQueryException e)
{
- Assert.assertEquals("Cannot use DROP TABLE on Materialized View", e.getMessage());
+ Assert.assertEquals("Cannot use DROP TABLE on a materialized view. Please use DROP MATERIALIZED VIEW instead.", e.getMessage());
}
}
@@ -694,7 +693,7 @@
executeNet(protocolVersion, "USE " + keyspace());
- assertInvalidMessage("Non-primary key columns cannot be restricted in the SELECT statement used for materialized view creation",
+ assertInvalidMessage("Non-primary key columns can only be restricted with 'IS NOT NULL'",
"CREATE MATERIALIZED VIEW " + keyspace() + ".mv AS SELECT * FROM %s "
+ "WHERE b IS NOT NULL AND c IS NOT NULL AND a IS NOT NULL "
+ "AND d = 1 PRIMARY KEY (c, b, a)");
diff --git a/test/unit/org/apache/cassandra/cql3/ViewTest.java b/test/unit/org/apache/cassandra/cql3/ViewTest.java
index 9978f1d..29f2be5 100644
--- a/test/unit/org/apache/cassandra/cql3/ViewTest.java
+++ b/test/unit/org/apache/cassandra/cql3/ViewTest.java
@@ -18,6 +18,8 @@
package org.apache.cassandra.cql3;
+import static org.junit.Assert.*;
+
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
@@ -25,7 +27,7 @@
import java.util.concurrent.TimeUnit;
import com.google.common.util.concurrent.Uninterruptibles;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.junit.After;
import org.junit.Before;
import org.junit.BeforeClass;
@@ -34,28 +36,20 @@
import com.datastax.driver.core.ResultSet;
import com.datastax.driver.core.Row;
import com.datastax.driver.core.exceptions.InvalidQueryException;
-import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.concurrent.SEPExecutor;
import org.apache.cassandra.concurrent.Stage;
-import org.apache.cassandra.concurrent.StageManager;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.view.View;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.db.SystemKeyspace;
import org.apache.cassandra.db.compaction.CompactionManager;
-import org.apache.cassandra.db.marshal.AsciiType;
-import org.apache.cassandra.db.view.View;
-import org.apache.cassandra.exceptions.SyntaxException;
-import org.apache.cassandra.schema.KeyspaceParams;
import org.apache.cassandra.service.ClientWarn;
import org.apache.cassandra.transport.ProtocolVersion;
import org.apache.cassandra.utils.FBUtilities;
-import static junit.framework.Assert.fail;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
public class ViewTest extends CQLTester
{
@@ -91,8 +85,8 @@
private void updateView(String query, Object... params) throws Throwable
{
executeNet(protocolVersion, query, params);
- while (!(((SEPExecutor) StageManager.getStage(Stage.VIEW_MUTATION)).getPendingTasks() == 0
- && ((SEPExecutor) StageManager.getStage(Stage.VIEW_MUTATION)).getActiveCount() == 0))
+ while (!(((SEPExecutor) Stage.VIEW_MUTATION.executor()).getPendingTaskCount() == 0
+ && ((SEPExecutor) Stage.VIEW_MUTATION.executor()).getActiveTaskCount() == 0))
{
Thread.sleep(1);
}
@@ -101,8 +95,8 @@
@Test
public void testNonExistingOnes() throws Throwable
{
- assertInvalidMessage("Cannot drop non existing materialized view", "DROP MATERIALIZED VIEW " + KEYSPACE + ".view_does_not_exist");
- assertInvalidMessage("Cannot drop non existing materialized view", "DROP MATERIALIZED VIEW keyspace_does_not_exist.view_does_not_exist");
+ assertInvalidMessage(String.format("Materialized view '%s.view_does_not_exist' doesn't exist", KEYSPACE), "DROP MATERIALIZED VIEW " + KEYSPACE + ".view_does_not_exist");
+ assertInvalidMessage("Materialized view 'keyspace_does_not_exist.view_does_not_exist' doesn't exist", "DROP MATERIALIZED VIEW keyspace_does_not_exist.view_does_not_exist");
execute("DROP MATERIALIZED VIEW IF EXISTS " + KEYSPACE + ".view_does_not_exist");
execute("DROP MATERIALIZED VIEW IF EXISTS keyspace_does_not_exist.view_does_not_exist");
@@ -164,7 +158,7 @@
execute("USE " + keyspace());
executeNet(protocolVersion, "USE " + keyspace());
- createView("view1", "CREATE MATERIALIZED VIEW view1 AS SELECT k1 FROM %%s WHERE k1 IS NOT NULL AND c1 IS NOT NULL AND val IS NOT NULL PRIMARY KEY (val, k1, c1)");
+ createView("view1", "CREATE MATERIALIZED VIEW view1 AS SELECT k1, c1, val FROM %%s WHERE k1 IS NOT NULL AND c1 IS NOT NULL AND val IS NOT NULL PRIMARY KEY (val, k1, c1)");
updateView("INSERT INTO %s (k1, c1, val) VALUES (1, 2, 200)");
updateView("INSERT INTO %s (k1, c1, val) VALUES (1, 3, 300)");
@@ -178,7 +172,6 @@
Assert.assertEquals(0, execute("select * from view1").size());
}
-
@Test
public void createMvWithUnrestrictedPKParts() throws Throwable
{
@@ -187,7 +180,7 @@
execute("USE " + keyspace());
executeNet(protocolVersion, "USE " + keyspace());
- createView("view1", "CREATE MATERIALIZED VIEW view1 AS SELECT k1 FROM %%s WHERE k1 IS NOT NULL AND c1 IS NOT NULL AND val IS NOT NULL PRIMARY KEY (val, k1, c1)");
+ createView("view1", "CREATE MATERIALIZED VIEW view1 AS SELECT val, k1, c1 FROM %%s WHERE k1 IS NOT NULL AND c1 IS NOT NULL AND val IS NOT NULL PRIMARY KEY (val, k1, c1)");
}
@@ -199,7 +192,7 @@
execute("USE " + keyspace());
executeNet(protocolVersion, "USE " + keyspace());
- createView("view1", "CREATE MATERIALIZED VIEW view1 AS SELECT k1 FROM %%s WHERE k1 IS NOT NULL AND c1 IS NOT NULL AND val IS NOT NULL PRIMARY KEY (val, k1, c1)");
+ createView("view1", "CREATE MATERIALIZED VIEW view1 AS SELECT k1, c1, val FROM %%s WHERE k1 IS NOT NULL AND c1 IS NOT NULL AND val IS NOT NULL PRIMARY KEY (val, k1, c1)");
updateView("INSERT INTO %s (k1, c1, val) VALUES (1, 2, 200)");
updateView("INSERT INTO %s (k1, c1, val) VALUES (1, 3, 300)");
@@ -229,7 +222,7 @@
try
{
createView("mv_test", "CREATE MATERIALIZED VIEW %s AS SELECT * FROM %%s");
- fail("Should fail if no primary key is filtered as NOT NULL");
+ Assert.fail("Should fail if no primary key is filtered as NOT NULL");
}
catch (Exception e)
{
@@ -239,7 +232,7 @@
try
{
createView("mv_test", "CREATE MATERIALIZED VIEW %s AS SELECT * FROM %%s WHERE bigintval IS NOT NULL AND asciival IS NOT NULL PRIMARY KEY (bigintval, k, asciival)");
- fail("Should fail if compound primary is not completely filtered as NOT NULL");
+ Assert.fail("Should fail if compound primary is not completely filtered as NOT NULL");
}
catch (Exception e)
{
@@ -255,14 +248,21 @@
try
{
createView("mv_test", "CREATE MATERIALIZED VIEW %s AS SELECT * FROM %%s");
- fail("Should fail if no primary key is filtered as NOT NULL");
+ Assert.fail("Should fail if no primary key is filtered as NOT NULL");
}
catch (Exception e)
{
}
- // Can omit "k IS NOT NULL" because we have a sinlge partition key
- createView("mv_test", "CREATE MATERIALIZED VIEW %s AS SELECT * FROM %%s WHERE bigintval IS NOT NULL AND asciival IS NOT NULL PRIMARY KEY (bigintval, k, asciival)");
+ // Must still include both even when the partition key is composite
+ try
+ {
+ createView("mv_test", "CREATE MATERIALIZED VIEW %s AS SELECT * FROM %%s WHERE bigintval IS NOT NULL AND asciival IS NOT NULL PRIMARY KEY (bigintval, k, asciival)");
+ Assert.fail("Should fail if compound primary is not completely filtered as NOT NULL");
+ }
+ catch (Exception e)
+ {
+ }
}
@Test
@@ -281,7 +281,7 @@
try
{
createView("mv_static", "CREATE MATERIALIZED VIEW %%s AS SELECT * FROM %s WHERE sval IS NOT NULL AND k IS NOT NULL AND c IS NOT NULL PRIMARY KEY (sval,k,c)");
- fail("Use of static column in a MV primary key should fail");
+ Assert.fail("Use of static column in a MV primary key should fail");
}
catch (InvalidQueryException e)
{
@@ -290,7 +290,7 @@
try
{
createView("mv_static", "CREATE MATERIALIZED VIEW %%s AS SELECT val, sval FROM %s WHERE val IS NOT NULL AND k IS NOT NULL AND c IS NOT NULL PRIMARY KEY (val, k, c)");
- fail("Explicit select of static column in MV should fail");
+ Assert.fail("Explicit select of static column in MV should fail");
}
catch (InvalidQueryException e)
{
@@ -299,7 +299,7 @@
try
{
createView("mv_static", "CREATE MATERIALIZED VIEW %s AS SELECT * FROM %%s WHERE val IS NOT NULL AND k IS NOT NULL AND c IS NOT NULL PRIMARY KEY (val,k,c)");
- fail("Implicit select of static column in MV should fail");
+ Assert.fail("Implicit select of static column in MV should fail");
}
catch (InvalidQueryException e)
{
@@ -403,7 +403,7 @@
try
{
createView("mv_counter", "CREATE MATERIALIZED VIEW %s AS SELECT * FROM %%s WHERE count IS NOT NULL AND k IS NOT NULL PRIMARY KEY (count,k)");
- fail("MV on counter should fail");
+ Assert.fail("MV on counter should fail");
}
catch (InvalidQueryException e)
{
@@ -411,29 +411,6 @@
}
@Test
- public void testSuperCoumn() throws Throwable
- {
- String keyspace = createKeyspaceName();
- String table = createTableName();
- SchemaLoader.createKeyspace(keyspace,
- KeyspaceParams.simple(1),
- SchemaLoader.superCFMD(keyspace, table, AsciiType.instance, AsciiType.instance));
-
- execute("USE " + keyspace);
- executeNet(protocolVersion, "USE " + keyspace);
-
- try
- {
- createView("mv_super_column", "CREATE MATERIALIZED VIEW %s AS SELECT * FROM " + keyspace + "." + table + " WHERE key IS NOT NULL AND column1 IS NOT NULL PRIMARY KEY (key,column1)");
- fail("MV on SuperColumn table should fail");
- }
- catch (InvalidQueryException e)
- {
- assertEquals("Materialized views are not supported on SuperColumn tables", e.getMessage());
- }
- }
-
- @Test
public void testDurationsTable() throws Throwable
{
createTable("CREATE TABLE %s (" +
@@ -446,11 +423,11 @@
try
{
createView("mv_duration", "CREATE MATERIALIZED VIEW %s AS SELECT * FROM %%s WHERE result IS NOT NULL AND k IS NOT NULL PRIMARY KEY (result,k)");
- fail("MV on duration should fail");
+ Assert.fail("MV on duration should fail");
}
catch (InvalidQueryException e)
{
- Assert.assertEquals("Cannot use Duration column 'result' in PRIMARY KEY of materialized view", e.getMessage());
+ Assert.assertEquals("duration type is not supported for PRIMARY KEY column 'result'", e.getMessage());
}
}
@@ -481,9 +458,6 @@
executeNet(protocolVersion, "INSERT INTO %s (a, b, c, d) VALUES (0, 0, 1, 0) USING TIMESTAMP 0");
assertRows(execute("SELECT d from mv WHERE c = ? and a = ? and b = ?", 1, 0, 0), row(0));
- if (flush)
- FBUtilities.waitOnFutures(ks.flush());
-
//update c's timestamp TS=2
executeNet(protocolVersion, "UPDATE %s USING TIMESTAMP 2 SET c = ? WHERE a = ? and b = ? ", 1, 0, 0);
assertRows(execute("SELECT d from mv WHERE c = ? and a = ? and b = ?", 1, 0, 0), row(0));
@@ -723,12 +697,12 @@
"bigintval bigint, " +
"PRIMARY KEY((k, asciival)))");
- CFMetaData metadata = currentTableMetadata();
+ TableMetadata metadata = currentTableMetadata();
execute("USE " + keyspace());
executeNet(protocolVersion, "USE " + keyspace());
- for (ColumnDefinition def : new HashSet<>(metadata.allColumns()))
+ for (ColumnMetadata def : new HashSet<>(metadata.columns()))
{
try
{
@@ -738,12 +712,12 @@
createView("mv1_" + def.name, query);
if (def.type.isMultiCell())
- fail("MV on a multicell should fail " + def);
+ Assert.fail("MV on a multicell should fail " + def);
}
catch (InvalidQueryException e)
{
if (!def.type.isMultiCell() && !def.isPartitionKey())
- fail("MV creation failed on " + def);
+ Assert.fail("MV creation failed on " + def);
}
@@ -755,12 +729,12 @@
createView("mv2_" + def.name, query);
if (def.type.isMultiCell())
- fail("MV on a multicell should fail " + def);
+ Assert.fail("MV on a multicell should fail " + def);
}
catch (InvalidQueryException e)
{
if (!def.type.isMultiCell() && !def.isPartitionKey())
- fail("MV creation failed on " + def);
+ Assert.fail("MV creation failed on " + def);
}
try
@@ -770,12 +744,12 @@
createView("mv3_" + def.name, query);
if (def.type.isMultiCell())
- fail("MV on a multicell should fail " + def);
+ Assert.fail("MV on a multicell should fail " + def);
}
catch (InvalidQueryException e)
{
if (!def.type.isMultiCell() && !def.isPartitionKey())
- fail("MV creation failed on " + def);
+ Assert.fail("MV creation failed on " + def);
}
@@ -785,7 +759,7 @@
+ (def.name.toString().equals("asciival") ? "" : "AND asciival IS NOT NULL ") + "PRIMARY KEY ((" + def.name + ", k), asciival)";
createView("mv3_" + def.name, query);
- fail("Should fail on duplicate name");
+ Assert.fail("Should fail on duplicate name");
}
catch (Exception e)
{
@@ -795,8 +769,8 @@
{
String query = "CREATE MATERIALIZED VIEW %s AS SELECT * FROM %%s WHERE " + def.name + " IS NOT NULL AND k IS NOT NULL "
+ (def.name.toString().equals("asciival") ? "" : "AND asciival IS NOT NULL ") + "PRIMARY KEY ((" + def.name + ", k), nonexistentcolumn)";
- createView("mv3_" + def.name, query);
- fail("Should fail with unknown base column");
+ createView("mv4_" + def.name, query);
+ Assert.fail("Should fail with unknown base column");
}
catch (InvalidQueryException e)
{
@@ -1058,10 +1032,10 @@
executeNet(protocolVersion, "USE " + keyspace());
- createView("mv1", "CREATE MATERIALIZED VIEW %s AS SELECT * FROM %%s WHERE b IS NOT NULL AND c IS NOT NULL PRIMARY KEY (a, b, c) WITH CLUSTERING ORDER BY (b DESC)");
- createView("mv2", "CREATE MATERIALIZED VIEW %s AS SELECT * FROM %%s WHERE b IS NOT NULL AND c IS NOT NULL PRIMARY KEY (a, c, b) WITH CLUSTERING ORDER BY (c ASC)");
- createView("mv3", "CREATE MATERIALIZED VIEW %s AS SELECT * FROM %%s WHERE b IS NOT NULL AND c IS NOT NULL PRIMARY KEY (a, b, c)");
- createView("mv4", "CREATE MATERIALIZED VIEW %s AS SELECT * FROM %%s WHERE b IS NOT NULL AND c IS NOT NULL PRIMARY KEY (a, c, b) WITH CLUSTERING ORDER BY (c DESC)");
+ createView("mv1", "CREATE MATERIALIZED VIEW %s AS SELECT * FROM %%s WHERE a IS NOT NULL AND b IS NOT NULL AND c IS NOT NULL PRIMARY KEY (a, b, c) WITH CLUSTERING ORDER BY (b DESC, c ASC)");
+ createView("mv2", "CREATE MATERIALIZED VIEW %s AS SELECT * FROM %%s WHERE a IS NOT NULL AND b IS NOT NULL AND c IS NOT NULL PRIMARY KEY (a, c, b) WITH CLUSTERING ORDER BY (c ASC, b ASC)");
+ createView("mv3", "CREATE MATERIALIZED VIEW %s AS SELECT * FROM %%s WHERE a IS NOT NULL AND b IS NOT NULL AND c IS NOT NULL PRIMARY KEY (a, b, c)");
+ createView("mv4", "CREATE MATERIALIZED VIEW %s AS SELECT * FROM %%s WHERE a IS NOT NULL AND b IS NOT NULL AND c IS NOT NULL PRIMARY KEY (a, c, b) WITH CLUSTERING ORDER BY (c DESC, b ASC)");
updateView("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 1, 1, 1, 1);
updateView("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 1, 2, 2, 2);
@@ -1097,7 +1071,7 @@
executeNet(protocolVersion, "USE " + keyspace());
- createView("mv1", "CREATE MATERIALIZED VIEW %s AS SELECT * FROM %%s WHERE b IS NOT NULL PRIMARY KEY (b, a)");
+ createView("mv1", "CREATE MATERIALIZED VIEW %s AS SELECT * FROM %%s WHERE a IS NOT NULL AND b IS NOT NULL PRIMARY KEY (b, a)");
updateView("INSERT INTO %s (a, b) VALUES (?, ?)", 1, 1);
updateView("INSERT INTO %s (a, b) VALUES (?, ?)", 1, 2);
@@ -1129,7 +1103,7 @@
executeNet(protocolVersion, "USE " + keyspace());
// Cannot use SELECT *, as those are always handled by the includeAll shortcut in View.updateAffectsView
- createView("mv1", "CREATE MATERIALIZED VIEW %s AS SELECT a, b FROM %%s WHERE b IS NOT NULL PRIMARY KEY (b, a)");
+ createView("mv1", "CREATE MATERIALIZED VIEW %s AS SELECT a, b FROM %%s WHERE a IS NOT NULL AND b IS NOT NULL PRIMARY KEY (b, a)");
updateView("INSERT INTO %s (a, b) VALUES (?, ?)", 1, 1);
@@ -1218,7 +1192,7 @@
"PRIMARY KEY (a))");
executeNet(protocolVersion, "USE " + keyspace());
- createView("mvmap", "CREATE MATERIALIZED VIEW %s AS SELECT a, b FROM %%s WHERE b IS NOT NULL PRIMARY KEY (b, a)");
+ createView("mvmap", "CREATE MATERIALIZED VIEW %s AS SELECT a, b FROM %%s WHERE a IS NOT NULL AND b IS NOT NULL PRIMARY KEY (b, a)");
updateView("INSERT INTO %s (a, b) VALUES (?, ?)", 0, 0);
ResultSet mvRows = executeNet(protocolVersion, "SELECT a, b FROM mvmap WHERE b = ?", 0);
@@ -1247,7 +1221,7 @@
try
{
createView("mv_de", "CREATE MATERIALIZED VIEW %s AS SELECT * FROM %%s WHERE a IS NOT NULL AND b IS NOT NULL AND c IS NOT NULL AND d IS NOT NULL AND e IS NOT NULL PRIMARY KEY ((d, a), b, e, c)");
- fail("Should have rejected a query including multiple non-primary key base columns");
+ Assert.fail("Should have rejected a query including multiple non-primary key base columns");
}
catch (Exception e)
{
@@ -1256,11 +1230,12 @@
try
{
createView("mv_de", "CREATE MATERIALIZED VIEW %s AS SELECT * FROM %%s WHERE a IS NOT NULL AND b IS NOT NULL AND c IS NOT NULL AND d IS NOT NULL AND e IS NOT NULL PRIMARY KEY ((a, b), c, d, e)");
- fail("Should have rejected a query including multiple non-primary key base columns");
+ Assert.fail("Should have rejected a query including multiple non-primary key base columns");
}
catch (Exception e)
{
}
+
}
@Test
@@ -1315,9 +1290,9 @@
public void testCreateMvWithTTL() throws Throwable
{
createTable("CREATE TABLE %s (" +
- "k int PRIMARY KEY, " +
- "c int, " +
- "val int) WITH default_time_to_live = 60");
+ "k int PRIMARY KEY, " +
+ "c int, " +
+ "val int) WITH default_time_to_live = 60");
execute("USE " + keyspace());
executeNet(protocolVersion, "USE " + keyspace());
@@ -1326,7 +1301,7 @@
try
{
createView("mv_ttl1", "CREATE MATERIALIZED VIEW %s AS SELECT * FROM %%s WHERE k IS NOT NULL AND c IS NOT NULL PRIMARY KEY (k,c) WITH default_time_to_live = 30");
- fail("Should fail if TTL is provided for materialized view");
+ Assert.fail("Should fail if TTL is provided for materialized view");
}
catch (Exception e)
{
@@ -1347,15 +1322,14 @@
try
{
executeNet(protocolVersion, "ALTER MATERIALIZED VIEW %s WITH default_time_to_live = 30");
- fail("Should fail if TTL is provided while altering materialized view");
+ Assert.fail("Should fail if TTL is provided while altering materialized view");
}
catch (Exception e)
{
}
}
- @Test
- public void testViewBuilderResume() throws Throwable
+ private void testViewBuilderResume(int concurrentViewBuilders) throws Throwable
{
createTable("CREATE TABLE %s (" +
"k int, " +
@@ -1366,6 +1340,7 @@
execute("USE " + keyspace());
executeNet(protocolVersion, "USE " + keyspace());
+ CompactionManager.instance.setConcurrentViewBuilders(concurrentViewBuilders);
CompactionManager.instance.setCoreCompactorThreads(1);
CompactionManager.instance.setMaximumCompactorThreads(1);
ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
@@ -1391,37 +1366,35 @@
cfs.forceBlockingFlush();
- createView("mv_test", "CREATE MATERIALIZED VIEW %s AS SELECT * FROM %%s WHERE val IS NOT NULL AND k IS NOT NULL AND c IS NOT NULL PRIMARY KEY (val,k,c)");
+ String viewName1 = "mv_test_" + concurrentViewBuilders;
+ createView(viewName1, "CREATE MATERIALIZED VIEW %s AS SELECT * FROM %%s WHERE val IS NOT NULL AND k IS NOT NULL AND c IS NOT NULL PRIMARY KEY (val,k,c)");
cfs.enableAutoCompaction();
List<Future<?>> futures = CompactionManager.instance.submitBackground(cfs);
+ String viewName2 = viewName1 + "_2";
//Force a second MV on the same base table, which will restart the first MV builder...
- createView("mv_test2", "CREATE MATERIALIZED VIEW %s AS SELECT val, k, c FROM %%s WHERE val IS NOT NULL AND k IS NOT NULL AND c IS NOT NULL PRIMARY KEY (val,k,c)");
+ createView(viewName2, "CREATE MATERIALIZED VIEW %s AS SELECT val, k, c FROM %%s WHERE val IS NOT NULL AND k IS NOT NULL AND c IS NOT NULL PRIMARY KEY (val,k,c)");
//Compact the base table
FBUtilities.waitOnFutures(futures);
- while (!SystemKeyspace.isViewBuilt(keyspace(), "mv_test"))
+ while (!SystemKeyspace.isViewBuilt(keyspace(), viewName1))
Uninterruptibles.sleepUninterruptibly(1, TimeUnit.SECONDS);
- assertRows(execute("SELECT count(*) FROM mv_test"), row(1024L));
+ assertRows(execute("SELECT count(*) FROM " + viewName1), row(1024L));
}
-
- @Test(expected = SyntaxException.class)
- public void emptyViewNameTest() throws Throwable
+ @Test
+ public void testViewBuilderResume() throws Throwable
{
- execute("CREATE MATERIALIZED VIEW \"\" AS SELECT a, b FROM tbl WHERE b IS NOT NULL PRIMARY KEY (b, a)");
+ for (int i = 1; i <= 8; i *= 2)
+ {
+ testViewBuilderResume(i);
+ }
}
- @Test(expected = SyntaxException.class)
- public void emptyBaseTableNameTest() throws Throwable
- {
- execute("CREATE MATERIALIZED VIEW myview AS SELECT a, b FROM \"\" WHERE b IS NOT NULL PRIMARY KEY (b, a)");
- }
-
/**
* Tests that a client warning is issued on materialized view creation.
*/
@@ -1433,7 +1406,7 @@
ClientWarn.instance.captureWarnings();
String viewName = keyspace() + ".warning_view";
execute("CREATE MATERIALIZED VIEW " + viewName +
- " AS SELECT v FROM %s WHERE k IS NOT NULL AND v IS NOT NULL PRIMARY KEY (v, k)");
+ " AS SELECT * FROM %s WHERE k IS NOT NULL AND v IS NOT NULL PRIMARY KEY (v, k)");
views.add(viewName);
List<String> warnings = ClientWarn.instance.getWarnings();
@@ -1457,7 +1430,7 @@
{
DatabaseDescriptor.setEnableMaterializedViews(false);
createView("view1", "CREATE MATERIALIZED VIEW %s AS SELECT v FROM %%s WHERE k IS NOT NULL AND v IS NOT NULL PRIMARY KEY (v, k)");
- fail("Should not be able to create a materialized view if they are disabled");
+ Assert.fail("Should not be able to create a materialized view if they are disabled");
}
catch (Throwable e)
{
@@ -1469,22 +1442,4 @@
DatabaseDescriptor.setEnableMaterializedViews(enableMaterializedViews);
}
}
-
- @Test
- public void viewOnCompactTableTest() throws Throwable
- {
- createTable("CREATE TABLE %s (a int, b int, v int, PRIMARY KEY (a, b)) WITH COMPACT STORAGE");
- executeNet(protocolVersion, "USE " + keyspace());
- try
- {
- createView("mv",
- "CREATE MATERIALIZED VIEW %s AS SELECT a, b, value FROM %%s WHERE b IS NOT NULL PRIMARY KEY (b, a)");
- fail("Should have thrown an exception");
- }
- catch (Throwable t)
- {
- Assert.assertEquals("Undefined column name value",
- t.getMessage());
- }
- }
-}
\ No newline at end of file
+}
diff --git a/test/unit/org/apache/cassandra/cql3/conditions/ColumnConditionTest.java b/test/unit/org/apache/cassandra/cql3/conditions/ColumnConditionTest.java
new file mode 100644
index 0000000..0035fa1
--- /dev/null
+++ b/test/unit/org/apache/cassandra/cql3/conditions/ColumnConditionTest.java
@@ -0,0 +1,555 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.conditions;
+
+import java.nio.ByteBuffer;
+import java.util.*;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import org.apache.cassandra.cql3.*;
+import org.apache.cassandra.db.Clustering;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.marshal.ListType;
+import org.apache.cassandra.db.marshal.MapType;
+import org.apache.cassandra.db.marshal.SetType;
+import org.apache.cassandra.db.rows.*;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.serializers.TimeUUIDSerializer;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.UUIDGen;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import static org.apache.cassandra.cql3.Operator.*;
+import static org.apache.cassandra.utils.ByteBufferUtil.EMPTY_BYTE_BUFFER;
+
+
+public class ColumnConditionTest
+{
+ public static final ByteBuffer ZERO = Int32Type.instance.fromString("0");
+ public static final ByteBuffer ONE = Int32Type.instance.fromString("1");
+ public static final ByteBuffer TWO = Int32Type.instance.fromString("2");
+
+ private static Row newRow(ColumnMetadata definition, ByteBuffer value)
+ {
+ BufferCell cell = new BufferCell(definition, 0L, Cell.NO_TTL, Cell.NO_DELETION_TIME, value, null);
+ return BTreeRow.singleCellRow(Clustering.EMPTY, cell);
+ }
+
+ private static Row newRow(ColumnMetadata definition, List<ByteBuffer> values)
+ {
+ Row.Builder builder = BTreeRow.sortedBuilder();
+ builder.newRow(Clustering.EMPTY);
+ long now = System.currentTimeMillis();
+ if (values != null)
+ {
+ for (int i = 0, m = values.size(); i < m; i++)
+ {
+ UUID uuid = UUIDGen.getTimeUUID(now, i);
+ ByteBuffer key = TimeUUIDSerializer.instance.serialize(uuid);
+ ByteBuffer value = values.get(i);
+ BufferCell cell = new BufferCell(definition,
+ 0L,
+ Cell.NO_TTL,
+ Cell.NO_DELETION_TIME,
+ value,
+ CellPath.create(key));
+ builder.addCell(cell);
+ }
+ }
+ return builder.build();
+ }
+
+ private static Row newRow(ColumnMetadata definition, SortedSet<ByteBuffer> values)
+ {
+ Row.Builder builder = BTreeRow.sortedBuilder();
+ builder.newRow(Clustering.EMPTY);
+ if (values != null)
+ {
+ for (ByteBuffer value : values)
+ {
+ BufferCell cell = new BufferCell(definition,
+ 0L,
+ Cell.NO_TTL,
+ Cell.NO_DELETION_TIME,
+ ByteBufferUtil.EMPTY_BYTE_BUFFER,
+ CellPath.create(value));
+ builder.addCell(cell);
+ }
+ }
+ return builder.build();
+ }
+
+ private static Row newRow(ColumnMetadata definition, Map<ByteBuffer, ByteBuffer> values)
+ {
+ Row.Builder builder = BTreeRow.sortedBuilder();
+ builder.newRow(Clustering.EMPTY);
+ if (values != null)
+ {
+ for (Map.Entry<ByteBuffer, ByteBuffer> entry : values.entrySet())
+ {
+ BufferCell cell = new BufferCell(definition,
+ 0L,
+ Cell.NO_TTL,
+ Cell.NO_DELETION_TIME,
+ entry.getValue(),
+ CellPath.create(entry.getKey()));
+ builder.addCell(cell);
+ }
+ }
+ return builder.build();
+ }
+
+ private static boolean conditionApplies(ByteBuffer rowValue, Operator op, ByteBuffer conditionValue)
+ {
+ ColumnMetadata definition = ColumnMetadata.regularColumn("ks", "cf", "c", Int32Type.instance);
+ ColumnCondition condition = ColumnCondition.condition(definition, op, Terms.of(new Constants.Value(conditionValue)));
+ ColumnCondition.Bound bound = condition.bind(QueryOptions.DEFAULT);
+ return bound.appliesTo(newRow(definition, rowValue));
+ }
+
+ private static boolean conditionApplies(List<ByteBuffer> rowValue, Operator op, List<ByteBuffer> conditionValue)
+ {
+ ColumnMetadata definition = ColumnMetadata.regularColumn("ks", "cf", "c", ListType.getInstance(Int32Type.instance, true));
+ ColumnCondition condition = ColumnCondition.condition(definition, op, Terms.of(new Lists.Value(conditionValue)));
+ ColumnCondition.Bound bound = condition.bind(QueryOptions.DEFAULT);
+ return bound.appliesTo(newRow(definition, rowValue));
+ }
+
+ private static boolean conditionApplies(SortedSet<ByteBuffer> rowValue, Operator op, SortedSet<ByteBuffer> conditionValue)
+ {
+ ColumnMetadata definition = ColumnMetadata.regularColumn("ks", "cf", "c", SetType.getInstance(Int32Type.instance, true));
+ ColumnCondition condition = ColumnCondition.condition(definition, op, Terms.of(new Sets.Value(conditionValue)));
+ ColumnCondition.Bound bound = condition.bind(QueryOptions.DEFAULT);
+ return bound.appliesTo(newRow(definition, rowValue));
+ }
+
+ private static boolean conditionApplies(Map<ByteBuffer, ByteBuffer> rowValue, Operator op, Map<ByteBuffer, ByteBuffer> conditionValue)
+ {
+ ColumnMetadata definition = ColumnMetadata.regularColumn("ks", "cf", "c", MapType.getInstance(Int32Type.instance, Int32Type.instance, true));
+ ColumnCondition condition = ColumnCondition.condition(definition, op, Terms.of(new Maps.Value(conditionValue)));
+ ColumnCondition.Bound bound = condition.bind(QueryOptions.DEFAULT);
+ return bound.appliesTo(newRow(definition, rowValue));
+ }
+
+ @FunctionalInterface
+ public interface CheckedFunction {
+ void apply();
+ }
+
+ private static void assertThrowsIRE(CheckedFunction runnable, String errorMessage)
+ {
+ try
+ {
+ runnable.apply();
+ fail("Expected InvalidRequestException was not thrown");
+ } catch (InvalidRequestException e)
+ {
+ Assert.assertTrue("Expected error message to contain '" + errorMessage + "', but got '" + e.getMessage() + "'",
+ e.getMessage().contains(errorMessage));
+ }
+ }
+
+ @Test
+ public void testSimpleBoundIsSatisfiedByValue() throws InvalidRequestException
+ {
+ // EQ
+ assertTrue(conditionApplies(ONE, EQ, ONE));
+ assertFalse(conditionApplies(TWO, EQ, ONE));
+ assertFalse(conditionApplies(ONE, EQ, TWO));
+ assertFalse(conditionApplies(ONE, EQ, EMPTY_BYTE_BUFFER));
+ assertFalse(conditionApplies(EMPTY_BYTE_BUFFER, EQ, ONE));
+ assertTrue(conditionApplies(EMPTY_BYTE_BUFFER, EQ, EMPTY_BYTE_BUFFER));
+ assertFalse(conditionApplies(ONE, EQ, null));
+ assertFalse(conditionApplies(null, EQ, ONE));
+ assertTrue(conditionApplies((ByteBuffer) null, EQ, (ByteBuffer) null));
+
+ // NEQ
+ assertFalse(conditionApplies(ONE, NEQ, ONE));
+ assertTrue(conditionApplies(TWO, NEQ, ONE));
+ assertTrue(conditionApplies(ONE, NEQ, TWO));
+ assertTrue(conditionApplies(ONE, NEQ, EMPTY_BYTE_BUFFER));
+ assertTrue(conditionApplies(EMPTY_BYTE_BUFFER, NEQ, ONE));
+ assertFalse(conditionApplies(EMPTY_BYTE_BUFFER, NEQ, EMPTY_BYTE_BUFFER));
+ assertTrue(conditionApplies(ONE, NEQ, null));
+ assertTrue(conditionApplies(null, NEQ, ONE));
+ assertFalse(conditionApplies((ByteBuffer) null, NEQ, (ByteBuffer) null));
+
+ // LT
+ assertFalse(conditionApplies(ONE, LT, ONE));
+ assertFalse(conditionApplies(TWO, LT, ONE));
+ assertTrue(conditionApplies(ONE, LT, TWO));
+ assertFalse(conditionApplies(ONE, LT, EMPTY_BYTE_BUFFER));
+ assertTrue(conditionApplies(EMPTY_BYTE_BUFFER, LT, ONE));
+ assertFalse(conditionApplies(EMPTY_BYTE_BUFFER, LT, EMPTY_BYTE_BUFFER));
+ assertThrowsIRE(() -> conditionApplies(ONE, LT, null), "Invalid comparison with null for operator \"<\"");
+ assertFalse(conditionApplies(null, LT, ONE));
+
+ // LTE
+ assertTrue(conditionApplies(ONE, LTE, ONE));
+ assertFalse(conditionApplies(TWO, LTE, ONE));
+ assertTrue(conditionApplies(ONE, LTE, TWO));
+ assertFalse(conditionApplies(ONE, LTE, EMPTY_BYTE_BUFFER));
+ assertTrue(conditionApplies(EMPTY_BYTE_BUFFER, LTE, ONE));
+ assertTrue(conditionApplies(EMPTY_BYTE_BUFFER, LTE, EMPTY_BYTE_BUFFER));
+ assertThrowsIRE(() -> conditionApplies(ONE, LTE, null), "Invalid comparison with null for operator \"<=\"");
+ assertFalse(conditionApplies(null, LTE, ONE));
+
+ // GT
+ assertFalse(conditionApplies(ONE, GT, ONE));
+ assertTrue(conditionApplies(TWO, GT, ONE));
+ assertFalse(conditionApplies(ONE, GT, TWO));
+ assertTrue(conditionApplies(ONE, GT, EMPTY_BYTE_BUFFER));
+ assertFalse(conditionApplies(EMPTY_BYTE_BUFFER, GT, ONE));
+ assertFalse(conditionApplies(EMPTY_BYTE_BUFFER, GT, EMPTY_BYTE_BUFFER));
+ assertThrowsIRE(() -> conditionApplies(ONE, GT, null), "Invalid comparison with null for operator \">\"");
+ assertFalse(conditionApplies(null, GT, ONE));
+
+ // GTE
+ assertTrue(conditionApplies(ONE, GTE, ONE));
+ assertTrue(conditionApplies(TWO, GTE, ONE));
+ assertFalse(conditionApplies(ONE, GTE, TWO));
+ assertTrue(conditionApplies(ONE, GTE, EMPTY_BYTE_BUFFER));
+ assertFalse(conditionApplies(EMPTY_BYTE_BUFFER, GTE, ONE));
+ assertTrue(conditionApplies(EMPTY_BYTE_BUFFER, GTE, EMPTY_BYTE_BUFFER));
+ assertThrowsIRE(() -> conditionApplies(ONE, GTE, null), "Invalid comparison with null for operator \">=\"");
+ assertFalse(conditionApplies(null, GTE, ONE));
+ }
+
+ private static List<ByteBuffer> list(ByteBuffer... values)
+ {
+ return Arrays.asList(values);
+ }
+
+ @Test
+ // sets use the same check as lists
+ public void testListCollectionBoundAppliesTo() throws InvalidRequestException
+ {
+ // EQ
+ assertTrue(conditionApplies(list(ONE), EQ, list(ONE)));
+ assertTrue(conditionApplies(list(), EQ, list()));
+ assertFalse(conditionApplies(list(ONE), EQ, list(ZERO)));
+ assertFalse(conditionApplies(list(ZERO), EQ, list(ONE)));
+ assertFalse(conditionApplies(list(ONE, ONE), EQ, list(ONE)));
+ assertFalse(conditionApplies(list(ONE), EQ, list(ONE, ONE)));
+ assertFalse(conditionApplies(list(ONE), EQ, list()));
+ assertFalse(conditionApplies(list(), EQ, list(ONE)));
+
+ assertFalse(conditionApplies(list(ONE), EQ, list(ByteBufferUtil.EMPTY_BYTE_BUFFER)));
+ assertFalse(conditionApplies(list(ByteBufferUtil.EMPTY_BYTE_BUFFER), EQ, list(ONE)));
+ assertTrue(conditionApplies(list(ByteBufferUtil.EMPTY_BYTE_BUFFER), EQ, list(ByteBufferUtil.EMPTY_BYTE_BUFFER)));
+
+ // NEQ
+ assertFalse(conditionApplies(list(ONE), NEQ, list(ONE)));
+ assertFalse(conditionApplies(list(), NEQ, list()));
+ assertTrue(conditionApplies(list(ONE), NEQ, list(ZERO)));
+ assertTrue(conditionApplies(list(ZERO), NEQ, list(ONE)));
+ assertTrue(conditionApplies(list(ONE, ONE), NEQ, list(ONE)));
+ assertTrue(conditionApplies(list(ONE), NEQ, list(ONE, ONE)));
+ assertTrue(conditionApplies(list(ONE), NEQ, list()));
+ assertTrue(conditionApplies(list(), NEQ, list(ONE)));
+
+ assertTrue(conditionApplies(list(ONE), NEQ, list(ByteBufferUtil.EMPTY_BYTE_BUFFER)));
+ assertTrue(conditionApplies(list(ByteBufferUtil.EMPTY_BYTE_BUFFER), NEQ, list(ONE)));
+ assertFalse(conditionApplies(list(ByteBufferUtil.EMPTY_BYTE_BUFFER), NEQ, list(ByteBufferUtil.EMPTY_BYTE_BUFFER)));
+
+ // LT
+ assertFalse(conditionApplies(list(ONE), LT, list(ONE)));
+ assertFalse(conditionApplies(list(), LT, list()));
+ assertFalse(conditionApplies(list(ONE), LT, list(ZERO)));
+ assertTrue(conditionApplies(list(ZERO), LT, list(ONE)));
+ assertFalse(conditionApplies(list(ONE, ONE), LT, list(ONE)));
+ assertTrue(conditionApplies(list(ONE), LT, list(ONE, ONE)));
+ assertFalse(conditionApplies(list(ONE), LT, list()));
+ assertTrue(conditionApplies(list(), LT, list(ONE)));
+
+ assertFalse(conditionApplies(list(ONE), LT, list(ByteBufferUtil.EMPTY_BYTE_BUFFER)));
+ assertTrue(conditionApplies(list(ByteBufferUtil.EMPTY_BYTE_BUFFER), LT, list(ONE)));
+ assertFalse(conditionApplies(list(ByteBufferUtil.EMPTY_BYTE_BUFFER), LT, list(ByteBufferUtil.EMPTY_BYTE_BUFFER)));
+
+ // LTE
+ assertTrue(conditionApplies(list(ONE), LTE, list(ONE)));
+ assertTrue(conditionApplies(list(), LTE, list()));
+ assertFalse(conditionApplies(list(ONE), LTE, list(ZERO)));
+ assertTrue(conditionApplies(list(ZERO), LTE, list(ONE)));
+ assertFalse(conditionApplies(list(ONE, ONE), LTE, list(ONE)));
+ assertTrue(conditionApplies(list(ONE), LTE, list(ONE, ONE)));
+ assertFalse(conditionApplies(list(ONE), LTE, list()));
+ assertTrue(conditionApplies(list(), LTE, list(ONE)));
+
+ assertFalse(conditionApplies(list(ONE), LTE, list(ByteBufferUtil.EMPTY_BYTE_BUFFER)));
+ assertTrue(conditionApplies(list(ByteBufferUtil.EMPTY_BYTE_BUFFER), LTE, list(ONE)));
+ assertTrue(conditionApplies(list(ByteBufferUtil.EMPTY_BYTE_BUFFER), LTE, list(ByteBufferUtil.EMPTY_BYTE_BUFFER)));
+
+ // GT
+ assertFalse(conditionApplies(list(ONE), GT, list(ONE)));
+ assertFalse(conditionApplies(list(), GT, list()));
+ assertTrue(conditionApplies(list(ONE), GT, list(ZERO)));
+ assertFalse(conditionApplies(list(ZERO), GT, list(ONE)));
+ assertTrue(conditionApplies(list(ONE, ONE), GT, list(ONE)));
+ assertFalse(conditionApplies(list(ONE), GT, list(ONE, ONE)));
+ assertTrue(conditionApplies(list(ONE), GT, list()));
+ assertFalse(conditionApplies(list(), GT, list(ONE)));
+
+ assertTrue(conditionApplies(list(ONE), GT, list(ByteBufferUtil.EMPTY_BYTE_BUFFER)));
+ assertFalse(conditionApplies(list(ByteBufferUtil.EMPTY_BYTE_BUFFER), GT, list(ONE)));
+ assertFalse(conditionApplies(list(ByteBufferUtil.EMPTY_BYTE_BUFFER), GT, list(ByteBufferUtil.EMPTY_BYTE_BUFFER)));
+
+ // GTE
+ assertTrue(conditionApplies(list(ONE), GTE, list(ONE)));
+ assertTrue(conditionApplies(list(), GTE, list()));
+ assertTrue(conditionApplies(list(ONE), GTE, list(ZERO)));
+ assertFalse(conditionApplies(list(ZERO), GTE, list(ONE)));
+ assertTrue(conditionApplies(list(ONE, ONE), GTE, list(ONE)));
+ assertFalse(conditionApplies(list(ONE), GTE, list(ONE, ONE)));
+ assertTrue(conditionApplies(list(ONE), GTE, list()));
+ assertFalse(conditionApplies(list(), GTE, list(ONE)));
+
+ assertTrue(conditionApplies(list(ONE), GTE, list(ByteBufferUtil.EMPTY_BYTE_BUFFER)));
+ assertFalse(conditionApplies(list(ByteBufferUtil.EMPTY_BYTE_BUFFER), GTE, list(ONE)));
+ assertTrue(conditionApplies(list(ByteBufferUtil.EMPTY_BYTE_BUFFER), GTE, list(ByteBufferUtil.EMPTY_BYTE_BUFFER)));
+ }
+
+ private static SortedSet<ByteBuffer> set(ByteBuffer... values)
+ {
+ SortedSet<ByteBuffer> results = new TreeSet<>(Int32Type.instance);
+ results.addAll(Arrays.asList(values));
+ return results;
+ }
+
+ @Test
+ public void testSetCollectionBoundAppliesTo() throws InvalidRequestException
+ {
+ // EQ
+ assertTrue(conditionApplies(set(ONE), EQ, set(ONE)));
+ assertTrue(conditionApplies(set(), EQ, set()));
+ assertFalse(conditionApplies(set(ONE), EQ, set(ZERO)));
+ assertFalse(conditionApplies(set(ZERO), EQ, set(ONE)));
+ assertFalse(conditionApplies(set(ONE, TWO), EQ, set(ONE)));
+ assertFalse(conditionApplies(set(ONE), EQ, set(ONE, TWO)));
+ assertFalse(conditionApplies(set(ONE), EQ, set()));
+ assertFalse(conditionApplies(set(), EQ, set(ONE)));
+
+ assertFalse(conditionApplies(set(ONE), EQ, set(ByteBufferUtil.EMPTY_BYTE_BUFFER)));
+ assertFalse(conditionApplies(set(ByteBufferUtil.EMPTY_BYTE_BUFFER), EQ, set(ONE)));
+ assertTrue(conditionApplies(set(ByteBufferUtil.EMPTY_BYTE_BUFFER), EQ, set(ByteBufferUtil.EMPTY_BYTE_BUFFER)));
+
+ // NEQ
+ assertFalse(conditionApplies(set(ONE), NEQ, set(ONE)));
+ assertFalse(conditionApplies(set(), NEQ, set()));
+ assertTrue(conditionApplies(set(ONE), NEQ, set(ZERO)));
+ assertTrue(conditionApplies(set(ZERO), NEQ, set(ONE)));
+ assertTrue(conditionApplies(set(ONE, TWO), NEQ, set(ONE)));
+ assertTrue(conditionApplies(set(ONE), NEQ, set(ONE, TWO)));
+ assertTrue(conditionApplies(set(ONE), NEQ, set()));
+ assertTrue(conditionApplies(set(), NEQ, set(ONE)));
+
+ assertTrue(conditionApplies(set(ONE), NEQ, set(ByteBufferUtil.EMPTY_BYTE_BUFFER)));
+ assertTrue(conditionApplies(set(ByteBufferUtil.EMPTY_BYTE_BUFFER), NEQ, set(ONE)));
+ assertFalse(conditionApplies(set(ByteBufferUtil.EMPTY_BYTE_BUFFER), NEQ, set(ByteBufferUtil.EMPTY_BYTE_BUFFER)));
+
+ // LT
+ assertFalse(conditionApplies(set(ONE), LT, set(ONE)));
+ assertFalse(conditionApplies(set(), LT, set()));
+ assertFalse(conditionApplies(set(ONE), LT, set(ZERO)));
+ assertTrue(conditionApplies(set(ZERO), LT, set(ONE)));
+ assertFalse(conditionApplies(set(ONE, TWO), LT, set(ONE)));
+ assertTrue(conditionApplies(set(ONE), LT, set(ONE, TWO)));
+ assertFalse(conditionApplies(set(ONE), LT, set()));
+ assertTrue(conditionApplies(set(), LT, set(ONE)));
+
+ assertFalse(conditionApplies(set(ONE), LT, set(ByteBufferUtil.EMPTY_BYTE_BUFFER)));
+ assertTrue(conditionApplies(set(ByteBufferUtil.EMPTY_BYTE_BUFFER), LT, set(ONE)));
+ assertFalse(conditionApplies(set(ByteBufferUtil.EMPTY_BYTE_BUFFER), LT, set(ByteBufferUtil.EMPTY_BYTE_BUFFER)));
+
+ // LTE
+ assertTrue(conditionApplies(set(ONE), LTE, set(ONE)));
+ assertTrue(conditionApplies(set(), LTE, set()));
+ assertFalse(conditionApplies(set(ONE), LTE, set(ZERO)));
+ assertTrue(conditionApplies(set(ZERO), LTE, set(ONE)));
+ assertFalse(conditionApplies(set(ONE, TWO), LTE, set(ONE)));
+ assertTrue(conditionApplies(set(ONE), LTE, set(ONE, TWO)));
+ assertFalse(conditionApplies(set(ONE), LTE, set()));
+ assertTrue(conditionApplies(set(), LTE, set(ONE)));
+
+ assertFalse(conditionApplies(set(ONE), LTE, set(ByteBufferUtil.EMPTY_BYTE_BUFFER)));
+ assertTrue(conditionApplies(set(ByteBufferUtil.EMPTY_BYTE_BUFFER), LTE, set(ONE)));
+ assertTrue(conditionApplies(set(ByteBufferUtil.EMPTY_BYTE_BUFFER), LTE, set(ByteBufferUtil.EMPTY_BYTE_BUFFER)));
+
+ // GT
+ assertFalse(conditionApplies(set(ONE), GT, set(ONE)));
+ assertFalse(conditionApplies(set(), GT, set()));
+ assertTrue(conditionApplies(set(ONE), GT, set(ZERO)));
+ assertFalse(conditionApplies(set(ZERO), GT, set(ONE)));
+ assertTrue(conditionApplies(set(ONE, TWO), GT, set(ONE)));
+ assertFalse(conditionApplies(set(ONE), GT, set(ONE, TWO)));
+ assertTrue(conditionApplies(set(ONE), GT, set()));
+ assertFalse(conditionApplies(set(), GT, set(ONE)));
+
+ assertTrue(conditionApplies(set(ONE), GT, set(ByteBufferUtil.EMPTY_BYTE_BUFFER)));
+ assertFalse(conditionApplies(set(ByteBufferUtil.EMPTY_BYTE_BUFFER), GT, set(ONE)));
+ assertFalse(conditionApplies(set(ByteBufferUtil.EMPTY_BYTE_BUFFER), GT, set(ByteBufferUtil.EMPTY_BYTE_BUFFER)));
+
+ // GTE
+ assertTrue(conditionApplies(set(ONE), GTE, set(ONE)));
+ assertTrue(conditionApplies(set(), GTE, set()));
+ assertTrue(conditionApplies(set(ONE), GTE, set(ZERO)));
+ assertFalse(conditionApplies(set(ZERO), GTE, set(ONE)));
+ assertTrue(conditionApplies(set(ONE, TWO), GTE, set(ONE)));
+ assertFalse(conditionApplies(set(ONE), GTE, set(ONE, TWO)));
+ assertTrue(conditionApplies(set(ONE), GTE, set()));
+ assertFalse(conditionApplies(set(), GTE, set(ONE)));
+
+ assertTrue(conditionApplies(set(ONE), GTE, set(ByteBufferUtil.EMPTY_BYTE_BUFFER)));
+ assertFalse(conditionApplies(set(ByteBufferUtil.EMPTY_BYTE_BUFFER), GTE, set(ONE)));
+ assertTrue(conditionApplies(set(ByteBufferUtil.EMPTY_BYTE_BUFFER), GTE, set(ByteBufferUtil.EMPTY_BYTE_BUFFER)));
+ }
+
+ // values should be a list of key, value, key, value, ...
+ private static Map<ByteBuffer, ByteBuffer> map(ByteBuffer... values)
+ {
+ Map<ByteBuffer, ByteBuffer> map = new TreeMap<>();
+ for (int i = 0; i < values.length; i += 2)
+ map.put(values[i], values[i + 1]);
+
+ return map;
+ }
+
+ @Test
+ public void testMapCollectionBoundIsSatisfiedByValue() throws InvalidRequestException
+ {
+ // EQ
+ assertTrue(conditionApplies(map(ONE, ONE), EQ, map(ONE, ONE)));
+ assertTrue(conditionApplies(map(), EQ, map()));
+ assertFalse(conditionApplies(map(ONE, ONE), EQ, map(ZERO, ONE)));
+ assertFalse(conditionApplies(map(ZERO, ONE), EQ, map(ONE, ONE)));
+ assertFalse(conditionApplies(map(ONE, ONE), EQ, map(ONE, ZERO)));
+ assertFalse(conditionApplies(map(ONE, ZERO), EQ, map(ONE, ONE)));
+ assertFalse(conditionApplies(map(ONE, ONE, TWO, ONE), EQ, map(ONE, ONE)));
+ assertFalse(conditionApplies(map(ONE, ONE), EQ, map(ONE, ONE, TWO, ONE)));
+ assertFalse(conditionApplies(map(ONE, ONE), EQ, map()));
+ assertFalse(conditionApplies(map(), EQ, map(ONE, ONE)));
+
+ assertFalse(conditionApplies(map(ONE, ONE), EQ, map(ByteBufferUtil.EMPTY_BYTE_BUFFER, ONE)));
+ assertFalse(conditionApplies(map(ByteBufferUtil.EMPTY_BYTE_BUFFER, ONE), EQ, map(ONE, ONE)));
+ assertFalse(conditionApplies(map(ONE, ONE), EQ, map(ONE, ByteBufferUtil.EMPTY_BYTE_BUFFER)));
+ assertFalse(conditionApplies(map(ONE, ByteBufferUtil.EMPTY_BYTE_BUFFER), EQ, map(ONE, ONE)));
+ assertTrue(conditionApplies(map(ByteBufferUtil.EMPTY_BYTE_BUFFER, ONE), EQ, map(ByteBufferUtil.EMPTY_BYTE_BUFFER, ONE)));
+ assertTrue(conditionApplies(map(ONE, ByteBufferUtil.EMPTY_BYTE_BUFFER), EQ, map(ONE, ByteBufferUtil.EMPTY_BYTE_BUFFER)));
+
+ // NEQ
+ assertFalse(conditionApplies(map(ONE, ONE), NEQ, map(ONE, ONE)));
+ assertFalse(conditionApplies(map(), NEQ, map()));
+ assertTrue(conditionApplies(map(ONE, ONE), NEQ, map(ZERO, ONE)));
+ assertTrue(conditionApplies(map(ZERO, ONE), NEQ, map(ONE, ONE)));
+ assertTrue(conditionApplies(map(ONE, ONE), NEQ, map(ONE, ZERO)));
+ assertTrue(conditionApplies(map(ONE, ZERO), NEQ, map(ONE, ONE)));
+ assertTrue(conditionApplies(map(ONE, ONE, TWO, ONE), NEQ, map(ONE, ONE)));
+ assertTrue(conditionApplies(map(ONE, ONE), NEQ, map(ONE, ONE, TWO, ONE)));
+ assertTrue(conditionApplies(map(ONE, ONE), NEQ, map()));
+ assertTrue(conditionApplies(map(), NEQ, map(ONE, ONE)));
+
+ assertTrue(conditionApplies(map(ONE, ONE), NEQ, map(ByteBufferUtil.EMPTY_BYTE_BUFFER, ONE)));
+ assertTrue(conditionApplies(map(ByteBufferUtil.EMPTY_BYTE_BUFFER, ONE), NEQ, map(ONE, ONE)));
+ assertTrue(conditionApplies(map(ONE, ONE), NEQ, map(ONE, ByteBufferUtil.EMPTY_BYTE_BUFFER)));
+ assertTrue(conditionApplies(map(ONE, ByteBufferUtil.EMPTY_BYTE_BUFFER), NEQ, map(ONE, ONE)));
+ assertFalse(conditionApplies(map(ByteBufferUtil.EMPTY_BYTE_BUFFER, ONE), NEQ, map(ByteBufferUtil.EMPTY_BYTE_BUFFER, ONE)));
+ assertFalse(conditionApplies(map(ONE, ByteBufferUtil.EMPTY_BYTE_BUFFER), NEQ, map(ONE, ByteBufferUtil.EMPTY_BYTE_BUFFER)));
+
+ // LT
+ assertFalse(conditionApplies(map(ONE, ONE), LT, map(ONE, ONE)));
+ assertFalse(conditionApplies(map(), LT, map()));
+ assertFalse(conditionApplies(map(ONE, ONE), LT, map(ZERO, ONE)));
+ assertTrue(conditionApplies(map(ZERO, ONE), LT, map(ONE, ONE)));
+ assertFalse(conditionApplies(map(ONE, ONE), LT, map(ONE, ZERO)));
+ assertTrue(conditionApplies(map(ONE, ZERO), LT, map(ONE, ONE)));
+ assertFalse(conditionApplies(map(ONE, ONE, TWO, ONE), LT, map(ONE, ONE)));
+ assertTrue(conditionApplies(map(ONE, ONE), LT, map(ONE, ONE, TWO, ONE)));
+ assertFalse(conditionApplies(map(ONE, ONE), LT, map()));
+ assertTrue(conditionApplies(map(), LT, map(ONE, ONE)));
+
+ assertFalse(conditionApplies(map(ONE, ONE), LT, map(ByteBufferUtil.EMPTY_BYTE_BUFFER, ONE)));
+ assertTrue(conditionApplies(map(ByteBufferUtil.EMPTY_BYTE_BUFFER, ONE), LT, map(ONE, ONE)));
+ assertFalse(conditionApplies(map(ONE, ONE), LT, map(ONE, ByteBufferUtil.EMPTY_BYTE_BUFFER)));
+ assertTrue(conditionApplies(map(ONE, ByteBufferUtil.EMPTY_BYTE_BUFFER), LT, map(ONE, ONE)));
+ assertFalse(conditionApplies(map(ByteBufferUtil.EMPTY_BYTE_BUFFER, ONE), LT, map(ByteBufferUtil.EMPTY_BYTE_BUFFER, ONE)));
+ assertFalse(conditionApplies(map(ONE, ByteBufferUtil.EMPTY_BYTE_BUFFER), LT, map(ONE, ByteBufferUtil.EMPTY_BYTE_BUFFER)));
+
+ // LTE
+ assertTrue(conditionApplies(map(ONE, ONE), LTE, map(ONE, ONE)));
+ assertTrue(conditionApplies(map(), LTE, map()));
+ assertFalse(conditionApplies(map(ONE, ONE), LTE, map(ZERO, ONE)));
+ assertTrue(conditionApplies(map(ZERO, ONE), LTE, map(ONE, ONE)));
+ assertFalse(conditionApplies(map(ONE, ONE), LTE, map(ONE, ZERO)));
+ assertTrue(conditionApplies(map(ONE, ZERO), LTE, map(ONE, ONE)));
+ assertFalse(conditionApplies(map(ONE, ONE, TWO, ONE), LTE, map(ONE, ONE)));
+ assertTrue(conditionApplies(map(ONE, ONE), LTE, map(ONE, ONE, TWO, ONE)));
+ assertFalse(conditionApplies(map(ONE, ONE), LTE, map()));
+ assertTrue(conditionApplies(map(), LTE, map(ONE, ONE)));
+
+ assertFalse(conditionApplies(map(ONE, ONE), LTE, map(ByteBufferUtil.EMPTY_BYTE_BUFFER, ONE)));
+ assertTrue(conditionApplies(map(ByteBufferUtil.EMPTY_BYTE_BUFFER, ONE), LTE, map(ONE, ONE)));
+ assertFalse(conditionApplies(map(ONE, ONE), LTE, map(ONE, ByteBufferUtil.EMPTY_BYTE_BUFFER)));
+ assertTrue(conditionApplies(map(ONE, ByteBufferUtil.EMPTY_BYTE_BUFFER), LTE, map(ONE, ONE)));
+ assertTrue(conditionApplies(map(ByteBufferUtil.EMPTY_BYTE_BUFFER, ONE), LTE, map(ByteBufferUtil.EMPTY_BYTE_BUFFER, ONE)));
+ assertTrue(conditionApplies(map(ONE, ByteBufferUtil.EMPTY_BYTE_BUFFER), LTE, map(ONE, ByteBufferUtil.EMPTY_BYTE_BUFFER)));
+
+ // GT
+ assertFalse(conditionApplies(map(ONE, ONE), GT, map(ONE, ONE)));
+ assertFalse(conditionApplies(map(), GT, map()));
+ assertTrue(conditionApplies(map(ONE, ONE), GT, map(ZERO, ONE)));
+ assertFalse(conditionApplies(map(ZERO, ONE), GT, map(ONE, ONE)));
+ assertTrue(conditionApplies(map(ONE, ONE), GT, map(ONE, ZERO)));
+ assertFalse(conditionApplies(map(ONE, ZERO), GT, map(ONE, ONE)));
+ assertTrue(conditionApplies(map(ONE, ONE, TWO, ONE), GT, map(ONE, ONE)));
+ assertFalse(conditionApplies(map(ONE, ONE), GT, map(ONE, ONE, TWO, ONE)));
+ assertTrue(conditionApplies(map(ONE, ONE), GT, map()));
+ assertFalse(conditionApplies(map(), GT, map(ONE, ONE)));
+
+ assertTrue(conditionApplies(map(ONE, ONE), GT, map(ByteBufferUtil.EMPTY_BYTE_BUFFER, ONE)));
+ assertFalse(conditionApplies(map(ByteBufferUtil.EMPTY_BYTE_BUFFER, ONE), GT, map(ONE, ONE)));
+ assertTrue(conditionApplies(map(ONE, ONE), GT, map(ONE, ByteBufferUtil.EMPTY_BYTE_BUFFER)));
+ assertFalse(conditionApplies(map(ONE, ByteBufferUtil.EMPTY_BYTE_BUFFER), GT, map(ONE, ONE)));
+ assertFalse(conditionApplies(map(ByteBufferUtil.EMPTY_BYTE_BUFFER, ONE), GT, map(ByteBufferUtil.EMPTY_BYTE_BUFFER, ONE)));
+ assertFalse(conditionApplies(map(ONE, ByteBufferUtil.EMPTY_BYTE_BUFFER), GT, map(ONE, ByteBufferUtil.EMPTY_BYTE_BUFFER)));
+
+ // GTE
+ assertTrue(conditionApplies(map(ONE, ONE), GTE, map(ONE, ONE)));
+ assertTrue(conditionApplies(map(), GTE, map()));
+ assertTrue(conditionApplies(map(ONE, ONE), GTE, map(ZERO, ONE)));
+ assertFalse(conditionApplies(map(ZERO, ONE), GTE, map(ONE, ONE)));
+ assertTrue(conditionApplies(map(ONE, ONE), GTE, map(ONE, ZERO)));
+ assertFalse(conditionApplies(map(ONE, ZERO), GTE, map(ONE, ONE)));
+ assertTrue(conditionApplies(map(ONE, ONE, TWO, ONE), GTE, map(ONE, ONE)));
+ assertFalse(conditionApplies(map(ONE, ONE), GTE, map(ONE, ONE, TWO, ONE)));
+ assertTrue(conditionApplies(map(ONE, ONE), GTE, map()));
+ assertFalse(conditionApplies(map(), GTE, map(ONE, ONE)));
+
+ assertTrue(conditionApplies(map(ONE, ONE), GTE, map(ByteBufferUtil.EMPTY_BYTE_BUFFER, ONE)));
+ assertFalse(conditionApplies(map(ByteBufferUtil.EMPTY_BYTE_BUFFER, ONE), GTE, map(ONE, ONE)));
+ assertTrue(conditionApplies(map(ONE, ONE), GTE, map(ONE, ByteBufferUtil.EMPTY_BYTE_BUFFER)));
+ assertFalse(conditionApplies(map(ONE, ByteBufferUtil.EMPTY_BYTE_BUFFER), GTE, map(ONE, ONE)));
+ assertTrue(conditionApplies(map(ByteBufferUtil.EMPTY_BYTE_BUFFER, ONE), GTE, map(ByteBufferUtil.EMPTY_BYTE_BUFFER, ONE)));
+ assertTrue(conditionApplies(map(ONE, ByteBufferUtil.EMPTY_BYTE_BUFFER), GTE, map(ONE, ByteBufferUtil.EMPTY_BYTE_BUFFER)));
+ }
+}
diff --git a/test/unit/org/apache/cassandra/cql3/functions/CastFctsTest.java b/test/unit/org/apache/cassandra/cql3/functions/CastFctsTest.java
index 2ffd8b4..ee6c69f 100644
--- a/test/unit/org/apache/cassandra/cql3/functions/CastFctsTest.java
+++ b/test/unit/org/apache/cassandra/cql3/functions/CastFctsTest.java
@@ -19,14 +19,17 @@
import java.math.BigDecimal;
import java.math.BigInteger;
+import java.time.LocalDate;
+import java.time.LocalTime;
+import java.time.ZoneOffset;
+import java.time.ZonedDateTime;
+import java.time.format.DateTimeFormatter;
import java.util.Date;
import org.apache.cassandra.cql3.CQLTester;
import org.apache.cassandra.serializers.SimpleDateSerializer;
import org.apache.cassandra.utils.UUIDGen;
-import org.joda.time.DateTime;
-import org.joda.time.DateTimeZone;
-import org.joda.time.format.DateTimeFormat;
+
import org.junit.Test;
public class CastFctsTest extends CQLTester
@@ -217,25 +220,23 @@
{
createTable("CREATE TABLE %s (a timeuuid primary key, b timestamp, c date, d time)");
- DateTime dateTime = DateTimeFormat.forPattern("yyyy-MM-dd hh:mm:ss")
- .withZone(DateTimeZone.UTC)
- .parseDateTime("2015-05-21 11:03:02");
+ final String yearMonthDay = "2015-05-21";
+ final LocalDate localDate = LocalDate.of(2015, 5, 21);
+ ZonedDateTime date = localDate.atStartOfDay(ZoneOffset.UTC);
- DateTime date = DateTimeFormat.forPattern("yyyy-MM-dd")
- .withZone(DateTimeZone.UTC)
- .parseDateTime("2015-05-21");
+ ZonedDateTime dateTime = ZonedDateTime.of(localDate, LocalTime.of(11,3,2), ZoneOffset.UTC);
- long timeInMillis = dateTime.getMillis();
+ long timeInMillis = dateTime.toInstant().toEpochMilli();
- execute("INSERT INTO %s (a, b, c, d) VALUES (?, '2015-05-21 11:03:02+00', '2015-05-21', '11:03:02')",
+ execute("INSERT INTO %s (a, b, c, d) VALUES (?, '" + yearMonthDay + " 11:03:02+00', '2015-05-21', '11:03:02')",
UUIDGen.getTimeUUID(timeInMillis));
assertRows(execute("SELECT CAST(a AS timestamp), " +
"CAST(b AS timestamp), " +
"CAST(c AS timestamp) FROM %s"),
- row(new Date(dateTime.getMillis()), new Date(dateTime.getMillis()), new Date(date.getMillis())));
+ row(Date.from(dateTime.toInstant()), Date.from(dateTime.toInstant()), Date.from(date.toInstant())));
- int timeInMillisToDay = SimpleDateSerializer.timeInMillisToDay(date.getMillis());
+ int timeInMillisToDay = SimpleDateSerializer.timeInMillisToDay(date.toInstant().toEpochMilli());
assertRows(execute("SELECT CAST(a AS date), " +
"CAST(b AS date), " +
"CAST(c AS date) FROM %s"),
@@ -244,7 +245,7 @@
assertRows(execute("SELECT CAST(b AS text), " +
"CAST(c AS text), " +
"CAST(d AS text) FROM %s"),
- row("2015-05-21T11:03:02.000Z", "2015-05-21", "11:03:02.000000000"));
+ row(yearMonthDay + "T11:03:02.000Z", yearMonthDay, "11:03:02.000000000"));
}
@Test
diff --git a/test/unit/org/apache/cassandra/cql3/functions/OperationFctsTest.java b/test/unit/org/apache/cassandra/cql3/functions/OperationFctsTest.java
new file mode 100644
index 0000000..c8ee935
--- /dev/null
+++ b/test/unit/org/apache/cassandra/cql3/functions/OperationFctsTest.java
@@ -0,0 +1,864 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.functions;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.util.Date;
+
+import org.junit.Test;
+
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.cql3.UntypedResultSet;
+import org.apache.cassandra.exceptions.OperationExecutionException;
+import org.apache.cassandra.serializers.SimpleDateSerializer;
+import org.apache.cassandra.serializers.TimestampSerializer;
+
+public class OperationFctsTest extends CQLTester
+{
+ @Test
+ public void testSingleOperations() throws Throwable
+ {
+ createTable("CREATE TABLE %s (a tinyint, b smallint, c int, d bigint, e float, f double, g varint, h decimal, PRIMARY KEY(a, b, c))");
+ execute("INSERT INTO %S (a, b, c, d, e, f, g, h) VALUES (1, 2, 3, 4, 5.5, 6.5, 7, 8.5)");
+
+ // Test additions
+ assertColumnNames(execute("SELECT a + a, b + a, c + a, d + a, e + a, f + a, g + a, h + a FROM %s WHERE a = 1 AND b = 2 AND c = 1 + 2"),
+ "a + a", "b + a", "c + a", "d + a", "e + a", "f + a", "g + a", "h + a");
+
+ assertRows(execute("SELECT a + a, b + a, c + a, d + a, e + a, f + a, g + a, h + a FROM %s WHERE a = 1 AND b = 2 AND c = 1 + 2"),
+ row((byte) 2, (short) 3, 4, 5L, 6.5F, 7.5, BigInteger.valueOf(8), BigDecimal.valueOf(9.5)));
+
+ assertRows(execute("SELECT a + b, b + b, c + b, d + b, e + b, f + b, g + b, h + b FROM %s WHERE a = 1 AND b = 2 AND c = 1 + 2"),
+ row((short) 3, (short) 4, 5, 6L, 7.5F, 8.5, BigInteger.valueOf(9), BigDecimal.valueOf(10.5)));
+
+ assertRows(execute("SELECT a + c, b + c, c + c, d + c, e + c, f + c, g + c, h + c FROM %s WHERE a = 1 AND b = 2 AND c = 1 + 2"),
+ row(4, 5, 6, 7L, 8.5F, 9.5, BigInteger.valueOf(10), BigDecimal.valueOf(11.5)));
+
+ assertRows(execute("SELECT a + d, b + d, c + d, d + d, e + d, f + d, g + d, h + d FROM %s WHERE a = 1 AND b = 2 AND c = 1 + 2"),
+ row(5L, 6L, 7L, 8L, 9.5, 10.5, BigInteger.valueOf(11), BigDecimal.valueOf(12.5)));
+
+ assertRows(execute("SELECT a + e, b + e, c + e, d + e, e + e, f + e, g + e, h + e FROM %s WHERE a = 1 AND b = 2 AND c = 1 + 2"),
+ row(6.5F, 7.5F, 8.5F, 9.5, 11.0F, 12.0, BigDecimal.valueOf(12.5), BigDecimal.valueOf(14.0)));
+
+ assertRows(execute("SELECT a + f, b + f, c + f, d + f, e + f, f + f, g + f, h + f FROM %s WHERE a = 1 AND b = 2 AND c = 1 + 2"),
+ row(7.5, 8.5, 9.5, 10.5, 12.0, 13.0, BigDecimal.valueOf(13.5), BigDecimal.valueOf(15.0)));
+
+ assertRows(execute("SELECT a + g, b + g, c + g, d + g, e + g, f + g, g + g, h + g FROM %s WHERE a = 1 AND b = 2 AND c = 1 + 2"),
+ row(BigInteger.valueOf(8),
+ BigInteger.valueOf(9),
+ BigInteger.valueOf(10),
+ BigInteger.valueOf(11),
+ BigDecimal.valueOf(12.5),
+ BigDecimal.valueOf(13.5),
+ BigInteger.valueOf(14),
+ BigDecimal.valueOf(15.5)));
+
+ assertRows(execute("SELECT a + h, b + h, c + h, d + h, e + h, f + h, g + h, h + h FROM %s WHERE a = 1 AND b = 2 AND c = 1 + 2"),
+ row(BigDecimal.valueOf(9.5),
+ BigDecimal.valueOf(10.5),
+ BigDecimal.valueOf(11.5),
+ BigDecimal.valueOf(12.5),
+ BigDecimal.valueOf(14.0),
+ BigDecimal.valueOf(15.0),
+ BigDecimal.valueOf(15.5),
+ BigDecimal.valueOf(17.0)));
+
+ // Test substractions
+
+ assertColumnNames(execute("SELECT a - a, b - a, c - a, d - a, e - a, f - a, g - a, h - a FROM %s WHERE a = 1 AND b = 2 AND c = 4 - 1"),
+ "a - a", "b - a", "c - a", "d - a", "e - a", "f - a", "g - a", "h - a");
+
+ assertRows(execute("SELECT a - a, b - a, c - a, d - a, e - a, f - a, g - a, h - a FROM %s WHERE a = 1 AND b = 2 AND c = 4 - 1"),
+ row((byte) 0, (short) 1, 2, 3L, 4.5F, 5.5, BigInteger.valueOf(6), BigDecimal.valueOf(7.5)));
+
+ assertRows(execute("SELECT a - b, b - b, c - b, d - b, e - b, f - b, g - b, h - b FROM %s WHERE a = 1 AND b = 2 AND c = 4 - 1"),
+ row((short) -1, (short) 0, 1, 2L, 3.5F, 4.5, BigInteger.valueOf(5), BigDecimal.valueOf(6.5)));
+
+ assertRows(execute("SELECT a - c, b - c, c - c, d - c, e - c, f - c, g - c, h - c FROM %s WHERE a = 1 AND b = 2 AND c = 4 - 1"),
+ row(-2, -1, 0, 1L, 2.5F, 3.5, BigInteger.valueOf(4), BigDecimal.valueOf(5.5)));
+
+ assertRows(execute("SELECT a - d, b - d, c - d, d - d, e - d, f - d, g - d, h - d FROM %s WHERE a = 1 AND b = 2 AND c = 4 - 1"),
+ row(-3L, -2L, -1L, 0L, 1.5, 2.5, BigInteger.valueOf(3), BigDecimal.valueOf(4.5)));
+
+ assertRows(execute("SELECT a - e, b - e, c - e, d - e, e - e, f - e, g - e, h - e FROM %s WHERE a = 1 AND b = 2 AND c = 4 - 1"),
+ row(-4.5F, -3.5F, -2.5F, -1.5, 0.0F, 1.0, BigDecimal.valueOf(1.5), BigDecimal.valueOf(3.0)));
+
+ assertRows(execute("SELECT a - f, b - f, c - f, d - f, e - f, f - f, g - f, h - f FROM %s WHERE a = 1 AND b = 2 AND c = 4 - 1"),
+ row(-5.5, -4.5, -3.5, -2.5, -1.0, 0.0, BigDecimal.valueOf(0.5), BigDecimal.valueOf(2.0)));
+
+ assertRows(execute("SELECT a - g, b - g, c - g, d - g, e - g, f - g, g - g, h - g FROM %s WHERE a = 1 AND b = 2 AND c = 4 - 1"),
+ row(BigInteger.valueOf(-6),
+ BigInteger.valueOf(-5),
+ BigInteger.valueOf(-4),
+ BigInteger.valueOf(-3),
+ BigDecimal.valueOf(-1.5),
+ BigDecimal.valueOf(-0.5),
+ BigInteger.valueOf(0),
+ BigDecimal.valueOf(1.5)));
+
+ assertRows(execute("SELECT a - h, b - h, c - h, d - h, e - h, f - h, g - h, h - h FROM %s WHERE a = 1 AND b = 2 AND c = 4 - 1"),
+ row(BigDecimal.valueOf(-7.5),
+ BigDecimal.valueOf(-6.5),
+ BigDecimal.valueOf(-5.5),
+ BigDecimal.valueOf(-4.5),
+ BigDecimal.valueOf(-3.0),
+ BigDecimal.valueOf(-2.0),
+ BigDecimal.valueOf(-1.5),
+ BigDecimal.valueOf(0.0)));
+
+ // Test multiplications
+
+ assertColumnNames(execute("SELECT a * a, b * a, c * a, d * a, e * a, f * a, g * a, h * a FROM %s WHERE a = 1 AND b = 2 AND c = 3 * 1"),
+ "a * a", "b * a", "c * a", "d * a", "e * a", "f * a", "g * a", "h * a");
+
+ assertRows(execute("SELECT a * a, b * a, c * a, d * a, e * a, f * a, g * a, h * a FROM %s WHERE a = 1 AND b = 2 AND c = 3 * 1"),
+ row((byte) 1, (short) 2, 3, 4L, 5.5F, 6.5, BigInteger.valueOf(7), new BigDecimal("8.50")));
+
+ assertRows(execute("SELECT a * b, b * b, c * b, d * b, e * b, f * b, g * b, h * b FROM %s WHERE a = 1 AND b = 2 AND c = 3 * 1"),
+ row((short) 2, (short) 4, 6, 8L, 11.0F, 13.0, BigInteger.valueOf(14), new BigDecimal("17.00")));
+
+ assertRows(execute("SELECT a * c, b * c, c * c, d * c, e * c, f * c, g * c, h * c FROM %s WHERE a = 1 AND b = 2 AND c = 3 * 1"),
+ row(3, 6, 9, 12L, 16.5F, 19.5, BigInteger.valueOf(21), new BigDecimal("25.50")));
+
+ assertRows(execute("SELECT a * d, b * d, c * d, d * d, e * d, f * d, g * d, h * d FROM %s WHERE a = 1 AND b = 2 AND c = 3 * 1"),
+ row(4L, 8L, 12L, 16L, 22.0, 26.0, BigInteger.valueOf(28), new BigDecimal("34.00")));
+
+ assertRows(execute("SELECT a * e, b * e, c * e, d * e, e * e, f * e, g * e, h * e FROM %s WHERE a = 1 AND b = 2 AND c = 3 * 1"),
+ row(5.5F, 11.0F, 16.5F, 22.0, 30.25F, 35.75, new BigDecimal("38.5"), new BigDecimal("46.75")));
+
+ assertRows(execute("SELECT a * f, b * f, c * f, d * f, e * f, f * f, g * f, h * f FROM %s WHERE a = 1 AND b = 2 AND c = 3 * 1"),
+ row(6.5, 13.0, 19.5, 26.0, 35.75, 42.25, new BigDecimal("45.5"), BigDecimal.valueOf(55.25)));
+
+ assertRows(execute("SELECT a * g, b * g, c * g, d * g, e * g, f * g, g * g, h * g FROM %s WHERE a = 1 AND b = 2 AND c = 3 * 1"),
+ row(BigInteger.valueOf(7),
+ BigInteger.valueOf(14),
+ BigInteger.valueOf(21),
+ BigInteger.valueOf(28),
+ new BigDecimal("38.5"),
+ new BigDecimal("45.5"),
+ BigInteger.valueOf(49),
+ new BigDecimal("59.5")));
+
+ assertRows(execute("SELECT a * h, b * h, c * h, d * h, e * h, f * h, g * h, h * h FROM %s WHERE a = 1 AND b = 2 AND c = 3 * 1"),
+ row(new BigDecimal("8.50"),
+ new BigDecimal("17.00"),
+ new BigDecimal("25.50"),
+ new BigDecimal("34.00"),
+ new BigDecimal("46.75"),
+ new BigDecimal("55.25"),
+ new BigDecimal("59.5"),
+ new BigDecimal("72.25")));
+
+ // Test divisions
+
+ assertColumnNames(execute("SELECT a / a, b / a, c / a, d / a, e / a, f / a, g / a, h / a FROM %s WHERE a = 1 AND b = 2 AND c = 3 / 1"),
+ "a / a", "b / a", "c / a", "d / a", "e / a", "f / a", "g / a", "h / a");
+
+ assertRows(execute("SELECT a / a, b / a, c / a, d / a, e / a, f / a, g / a, h / a FROM %s WHERE a = 1 AND b = 2 AND c = 3 / 1"),
+ row((byte) 1, (short) 2, 3, 4L, 5.5F, 6.5, BigInteger.valueOf(7), new BigDecimal("8.5")));
+
+ assertRows(execute("SELECT a / b, b / b, c / b, d / b, e / b, f / b, g / b, h / b FROM %s WHERE a = 1 AND b = 2 AND c = 3 / 1"),
+ row((short) 0, (short) 1, 1, 2L, 2.75F, 3.25, BigInteger.valueOf(3), new BigDecimal("4.25")));
+
+ assertRows(execute("SELECT a / c, b / c, c / c, d / c, e / c, f / c, g / c, h / c FROM %s WHERE a = 1 AND b = 2 AND c = 3 / 1"),
+ row(0, 0, 1, 1L, 1.8333334F, 2.1666666666666665, BigInteger.valueOf(2), new BigDecimal("2.83333333333333333333333333333333")));
+
+ assertRows(execute("SELECT a / d, b / d, c / d, d / d, e / d, f / d, g / d, h / d FROM %s WHERE a = 1 AND b = 2 AND c = 3 / 1"),
+ row(0L, 0L, 0L, 1L, 1.375, 1.625, BigInteger.valueOf(1), new BigDecimal("2.125")));
+
+ assertRows(execute("SELECT a / e, b / e, c / e, d / e, e / e, f / e, g / e, h / e FROM %s WHERE a = 1 AND b = 2 AND c = 3 / 1"),
+ row(0.18181819F, 0.36363637F, 0.54545456F, 0.7272727272727273, 1.0F, 1.1818181818181819, new BigDecimal("1.27272727272727272727272727272727"), new BigDecimal("1.54545454545454545454545454545455")));
+
+ assertRows(execute("SELECT a / f, b / f, c / f, d / f, e / f, f / f, g / f, h / f FROM %s WHERE a = 1 AND b = 2 AND c = 3 / 1"),
+ row(0.15384615384615385, 0.3076923076923077, 0.46153846153846156, 0.6153846153846154, 0.8461538461538461, 1.0, new BigDecimal("1.07692307692307692307692307692308"), new BigDecimal("1.30769230769230769230769230769231")));
+
+ assertRows(execute("SELECT a / g, b / g, c / g, d / g, e / g, f / g, g / g, h / g FROM %s WHERE a = 1 AND b = 2 AND c = 3 / 1"),
+ row(BigInteger.valueOf(0),
+ BigInteger.valueOf(0),
+ BigInteger.valueOf(0),
+ BigInteger.valueOf(0),
+ new BigDecimal("0.78571428571428571428571428571429"),
+ new BigDecimal("0.92857142857142857142857142857143"),
+ BigInteger.valueOf(1),
+ new BigDecimal("1.21428571428571428571428571428571")));
+
+ assertRows(execute("SELECT a / h, b / h, c / h, d / h, e / h, f / h, g / h, h / h FROM %s WHERE a = 1 AND b = 2 AND c = 3 / 1"),
+ row(new BigDecimal("0.11764705882352941176470588235294"),
+ new BigDecimal("0.23529411764705882352941176470588"),
+ new BigDecimal("0.35294117647058823529411764705882"),
+ new BigDecimal("0.47058823529411764705882352941176"),
+ new BigDecimal("0.64705882352941176470588235294118"),
+ new BigDecimal("0.76470588235294117647058823529412"),
+ new BigDecimal("0.82352941176470588235294117647059"),
+ new BigDecimal("1")));
+
+ // Test modulo operations
+
+ assertColumnNames(execute("SELECT a %% a, b %% a, c %% a, d %% a, e %% a, f %% a, g %% a, h %% a FROM %s WHERE a = 1 AND b = 2 AND c = 23 %% 5"),
+ "a % a", "b % a", "c % a", "d % a", "e % a", "f % a", "g % a", "h % a");
+
+ assertRows(execute("SELECT a %% a, b %% a, c %% a, d %% a, e %% a, f %% a, g %% a, h %% a FROM %s WHERE a = 1 AND b = 2 AND c = 23 %% 5"),
+ row((byte) 0, (short) 0, 0, 0L, 0.5F, 0.5, BigInteger.valueOf(0), new BigDecimal("0.5")));
+
+ assertRows(execute("SELECT a %% b, b %% b, c %% b, d %% b, e %% b, f %% b, g %% b, h %% b FROM %s WHERE a = 1 AND b = 2 AND c = 23 %% 5"),
+ row((short) 1, (short) 0, 1, 0L, 1.5F, 0.5, BigInteger.valueOf(1), new BigDecimal("0.5")));
+
+ assertRows(execute("SELECT a %% c, b %% c, c %% c, d %% c, e %% c, f %% c, g %% c, h %% c FROM %s WHERE a = 1 AND b = 2 AND c = 23 %% 5"),
+ row(1, 2, 0, 1L, 2.5F, 0.5, BigInteger.valueOf(1), new BigDecimal("2.5")));
+
+ assertRows(execute("SELECT a %% d, b %% d, c %% d, d %% d, e %% d, f %% d, g %% d, h %% d FROM %s WHERE a = 1 AND b = 2 AND c = 23 %% 5"),
+ row(1L, 2L, 3L, 0L, 1.5, 2.5, BigInteger.valueOf(3), new BigDecimal("0.5")));
+
+ assertRows(execute("SELECT a %% e, b %% e, c %% e, d %% e, e %% e, f %% e, g %% e, h %% e FROM %s WHERE a = 1 AND b = 2 AND c = 23 %% 5"),
+ row(1.0F, 2.0F, 3.0F, 4.0, 0.0F, 1.0, new BigDecimal("1.5"), new BigDecimal("3.0")));
+
+ assertRows(execute("SELECT a %% f, b %% f, c %% f, d %% f, e %% f, f %% f, g %% f, h %% f FROM %s WHERE a = 1 AND b = 2 AND c = 23 %% 5"),
+ row(1.0, 2.0, 3.0, 4.0, 5.5, 0.0, new BigDecimal("0.5"), new BigDecimal("2.0")));
+
+ assertRows(execute("SELECT a %% g, b %% g, c %% g, d %% g, e %% g, f %% g, g %% g, h %% g FROM %s WHERE a = 1 AND b = 2 AND c = 23 %% 5"),
+ row(BigInteger.valueOf(1),
+ BigInteger.valueOf(2),
+ BigInteger.valueOf(3),
+ BigInteger.valueOf(4),
+ new BigDecimal("5.5"),
+ new BigDecimal("6.5"),
+ BigInteger.valueOf(0),
+ new BigDecimal("1.5")));
+
+ assertRows(execute("SELECT a %% h, b %% h, c %% h, d %% h, e %% h, f %% h, g %% h, h %% h FROM %s WHERE a = 1 AND b = 2 AND c = 23 %% 5"),
+ row(new BigDecimal("1.0"),
+ new BigDecimal("2.0"),
+ new BigDecimal("3.0"),
+ new BigDecimal("4.0"),
+ new BigDecimal("5.5"),
+ new BigDecimal("6.5"),
+ new BigDecimal("7"),
+ new BigDecimal("0.0")));
+
+ // Test negation
+
+ assertColumnNames(execute("SELECT -a, -b, -c, -d, -e, -f, -g, -h FROM %s WHERE a = 1 AND b = 2"),
+ "-a", "-b", "-c", "-d", "-e", "-f", "-g", "-h");
+
+ assertRows(execute("SELECT -a, -b, -c, -d, -e, -f, -g, -h FROM %s WHERE a = 1 AND b = 2"),
+ row((byte) -1, (short) -2, -3, -4L, -5.5F, -6.5, BigInteger.valueOf(-7), new BigDecimal("-8.5")));
+
+ // Test with null
+ execute("UPDATE %s SET d = ? WHERE a = ? AND b = ? AND c = ?", null, (byte) 1, (short) 2, 3);
+ assertRows(execute("SELECT a + d, b + d, c + d, d + d, e + d, f + d, g + d, h + d FROM %s WHERE a = 1 AND b = 2"),
+ row(null, null, null, null, null, null, null, null));
+ }
+
+ @Test
+ public void testModuloWithDecimals() throws Throwable
+ {
+ createTable("CREATE TABLE %s (numerator decimal, dec_mod decimal, int_mod int, bigint_mod bigint, PRIMARY KEY((numerator, dec_mod)))");
+ execute("INSERT INTO %s (numerator, dec_mod, int_mod, bigint_mod) VALUES (123456789112345678921234567893123456, 2, 2, 2)");
+
+ assertRows(execute("SELECT numerator %% dec_mod, numerator %% int_mod, numerator %% bigint_mod from %s"),
+ row(new BigDecimal("0"), new BigDecimal("0.0"), new BigDecimal("0.0")));
+ }
+
+ @Test
+ public void testSingleOperationsWithLiterals() throws Throwable
+ {
+ createTable("CREATE TABLE %s (pk int, c1 tinyint, c2 smallint, v text, PRIMARY KEY(pk, c1, c2))");
+ execute("INSERT INTO %S (pk, c1, c2, v) VALUES (2, 2, 2, 'test')");
+
+ // There is only one function outputing tinyint
+ assertRows(execute("SELECT * FROM %s WHERE pk = 2 AND c1 = 1 + 1"),
+ row(2, (byte) 2, (short) 2, "test"));
+
+ // As the operation can only be a sum between tinyints the expected type is tinyint
+ assertInvalidMessage("Expected 1 byte for a tinyint (4)",
+ "SELECT * FROM %s WHERE pk = 2 AND c1 = 1 + ?", 1);
+
+ assertRows(execute("SELECT * FROM %s WHERE pk = 2 AND c1 = 1 + ?", (byte) 1),
+ row(2, (byte) 2, (short) 2, "test"));
+
+ assertRows(execute("SELECT * FROM %s WHERE pk = 1 + 1 AND c1 = 2"),
+ row(2, (byte) 2, (short) 2, "test"));
+
+ assertRows(execute("SELECT * FROM %s WHERE pk = 2 AND c1 = 2 AND c2 = 1 + 1"),
+ row(2, (byte) 2, (short) 2, "test"));
+
+ assertRows(execute("SELECT * FROM %s WHERE pk = 2 AND c1 = 2 AND c2 = 1 * (1 + 1)"),
+ row(2, (byte) 2, (short) 2, "test"));
+
+ // tinyint, smallint and int could be used there so we need to disambiguate
+ assertInvalidMessage("Ambiguous '+' operation with args ? and 1: use type casts to disambiguate",
+ "SELECT * FROM %s WHERE pk = ? + 1 AND c1 = 2", 1);
+
+ assertInvalidMessage("Ambiguous '+' operation with args ? and 1: use type casts to disambiguate",
+ "SELECT * FROM %s WHERE pk = 2 AND c1 = 2 AND c2 = 1 * (? + 1)", 1);
+
+ assertRows(execute("SELECT 1 + 1, v FROM %s WHERE pk = 2 AND c1 = 2"),
+ row(2, "test"));
+
+ // As the output type is unknown the ? type cannot be determined
+ assertInvalidMessage("Ambiguous '+' operation with args 1 and ?: use type casts to disambiguate",
+ "SELECT 1 + ?, v FROM %s WHERE pk = 2 AND c1 = 2", 1);
+
+ // As the prefered type for the constants is int, the returned type will be int
+ assertRows(execute("SELECT 100 + 50, v FROM %s WHERE pk = 2 AND c1 = 2"),
+ row(150, "test"));
+
+ // As the output type is unknown the ? type cannot be determined
+ assertInvalidMessage("Ambiguous '+' operation with args ? and 50: use type casts to disambiguate",
+ "SELECT ? + 50, v FROM %s WHERE pk = 2 AND c1 = 2", 100);
+
+ createTable("CREATE TABLE %s (a tinyint, b smallint, c int, d bigint, e float, f double, g varint, h decimal, PRIMARY KEY(a, b))"
+ + " WITH CLUSTERING ORDER BY (b DESC)"); // Make sure we test with ReversedTypes
+ execute("INSERT INTO %S (a, b, c, d, e, f, g, h) VALUES (1, 2, 3, 4, 5.5, 6.5, 7, 8.5)");
+
+ // Test additions
+ assertColumnNames(execute("SELECT a + 1, b + 1, c + 1, d + 1, e + 1, f + 1, g + 1, h + 1 FROM %s WHERE a = 1 AND b = 2"),
+ "a + 1", "b + 1", "c + 1", "d + 1", "e + 1", "f + 1", "g + 1", "h + 1");
+
+ assertRows(execute("SELECT a + 1, b + 1, c + 1, d + 1, e + 1, f + 1, g + 1, h + 1 FROM %s WHERE a = 1 AND b = 2"),
+ row(2, 3, 4, 5L, 6.5F, 7.5, BigInteger.valueOf(8), BigDecimal.valueOf(9.5)));
+
+ assertRows(execute("SELECT 2 + a, 2 + b, 2 + c, 2 + d, 2 + e, 2 + f, 2 + g, 2 + h FROM %s WHERE a = 1 AND b = 2"),
+ row(3, 4, 5, 6L, 7.5F, 8.5, BigInteger.valueOf(9), BigDecimal.valueOf(10.5)));
+
+ long bigInt = Integer.MAX_VALUE + 10L;
+
+ assertRows(execute("SELECT a + " + bigInt + ","
+ + " b + " + bigInt + ","
+ + " c + " + bigInt + ","
+ + " d + " + bigInt + ","
+ + " e + " + bigInt + ","
+ + " f + " + bigInt + ","
+ + " g + " + bigInt + ","
+ + " h + " + bigInt + " FROM %s WHERE a = 1 AND b = 2"),
+ row(1L + bigInt,
+ 2L + bigInt,
+ 3L + bigInt,
+ 4L + bigInt,
+ 5.5 + bigInt,
+ 6.5 + bigInt,
+ BigInteger.valueOf(bigInt + 7),
+ BigDecimal.valueOf(bigInt + 8.5)));
+
+ assertRows(execute("SELECT a + 5.5, b + 5.5, c + 5.5, d + 5.5, e + 5.5, f + 5.5, g + 5.5, h + 5.5 FROM %s WHERE a = 1 AND b = 2"),
+ row(6.5, 7.5, 8.5, 9.5, 11.0, 12.0, BigDecimal.valueOf(12.5), BigDecimal.valueOf(14.0)));
+
+ assertRows(execute("SELECT a + 6.5, b + 6.5, c + 6.5, d + 6.5, e + 6.5, f + 6.5, g + 6.5, h + 6.5 FROM %s WHERE a = 1 AND b = 2"),
+ row(7.5, 8.5, 9.5, 10.5, 12.0, 13.0, BigDecimal.valueOf(13.5), BigDecimal.valueOf(15.0)));
+
+ // Test substractions
+
+ assertColumnNames(execute("SELECT a - 1, b - 1, c - 1, d - 1, e - 1, f - 1, g - 1, h - 1 FROM %s WHERE a = 1 AND b = 2"),
+ "a - 1", "b - 1", "c - 1", "d - 1", "e - 1", "f - 1", "g - 1", "h - 1");
+
+ assertRows(execute("SELECT a - 1, b - 1, c - 1, d - 1, e - 1, f - 1, g - 1, h - 1 FROM %s WHERE a = 1 AND b = 2"),
+ row(0, 1, 2, 3L, 4.5F, 5.5, BigInteger.valueOf(6), BigDecimal.valueOf(7.5)));
+
+ assertRows(execute("SELECT a - 2, b - 2, c - 2, d - 2, e - 2, f - 2, g - 2, h - 2 FROM %s WHERE a = 1 AND b = 2"),
+ row(-1, 0, 1, 2L, 3.5F, 4.5, BigInteger.valueOf(5), BigDecimal.valueOf(6.5)));
+
+ assertRows(execute("SELECT a - 3, b - 3, 3 - 3, d - 3, e - 3, f - 3, g - 3, h - 3 FROM %s WHERE a = 1 AND b = 2"),
+ row(-2, -1, 0, 1L, 2.5F, 3.5, BigInteger.valueOf(4), BigDecimal.valueOf(5.5)));
+
+ assertRows(execute("SELECT a - " + bigInt + ","
+ + " b - " + bigInt + ","
+ + " c - " + bigInt + ","
+ + " d - " + bigInt + ","
+ + " e - " + bigInt + ","
+ + " f - " + bigInt + ","
+ + " g - " + bigInt + ","
+ + " h - " + bigInt + " FROM %s WHERE a = 1 AND b = 2"),
+ row(1L - bigInt,
+ 2L - bigInt,
+ 3L - bigInt,
+ 4L - bigInt,
+ 5.5 - bigInt,
+ 6.5 - bigInt,
+ BigInteger.valueOf(7 - bigInt),
+ BigDecimal.valueOf(8.5 - bigInt)));
+
+ assertRows(execute("SELECT a - 5.5, b - 5.5, c - 5.5, d - 5.5, e - 5.5, f - 5.5, g - 5.5, h - 5.5 FROM %s WHERE a = 1 AND b = 2"),
+ row(-4.5, -3.5, -2.5, -1.5, 0.0, 1.0, BigDecimal.valueOf(1.5), BigDecimal.valueOf(3.0)));
+
+ assertRows(execute("SELECT a - 6.5, b - 6.5, c - 6.5, d - 6.5, e - 6.5, f - 6.5, g - 6.5, h - 6.5 FROM %s WHERE a = 1 AND b = 2"),
+ row(-5.5, -4.5, -3.5, -2.5, -1.0, 0.0, BigDecimal.valueOf(0.5), BigDecimal.valueOf(2.0)));
+
+ // Test multiplications
+
+ assertColumnNames(execute("SELECT a * 1, b * 1, c * 1, d * 1, e * 1, f * 1, g * 1, h * 1 FROM %s WHERE a = 1 AND b = 2"),
+ "a * 1", "b * 1", "c * 1", "d * 1", "e * 1", "f * 1", "g * 1", "h * 1");
+
+ assertRows(execute("SELECT a * 1, b * 1, c * 1, d * 1, e * 1, f * 1, g * 1, h * 1 FROM %s WHERE a = 1 AND b = 2"),
+ row(1, 2, 3, 4L, 5.5F, 6.5, BigInteger.valueOf(7), new BigDecimal("8.50")));
+
+ assertRows(execute("SELECT a * 2, b * 2, c * 2, d * 2, e * 2, f * 2, g * 2, h * 2 FROM %s WHERE a = 1 AND b = 2"),
+ row(2, 4, 6, 8L, 11.0F, 13.0, BigInteger.valueOf(14), new BigDecimal("17.00")));
+
+ assertRows(execute("SELECT a * 3, b * 3, c * 3, d * 3, e * 3, f * 3, g * 3, h * 3 FROM %s WHERE a = 1 AND b = 2"),
+ row(3, 6, 9, 12L, 16.5F, 19.5, BigInteger.valueOf(21), new BigDecimal("25.50")));
+
+ assertRows(execute("SELECT a * " + bigInt + ","
+ + " b * " + bigInt + ","
+ + " c * " + bigInt + ","
+ + " d * " + bigInt + ","
+ + " e * " + bigInt + ","
+ + " f * " + bigInt + ","
+ + " g * " + bigInt + ","
+ + " h * " + bigInt + " FROM %s WHERE a = 1 AND b = 2"),
+ row(1L * bigInt,
+ 2L * bigInt,
+ 3L * bigInt,
+ 4L * bigInt,
+ 5.5 * bigInt,
+ 6.5 * bigInt,
+ BigInteger.valueOf(7 * bigInt),
+ BigDecimal.valueOf(8.5 * bigInt)));
+
+ assertRows(execute("SELECT a * 5.5, b * 5.5, c * 5.5, d * 5.5, e * 5.5, f * 5.5, g * 5.5, h * 5.5 FROM %s WHERE a = 1 AND b = 2"),
+ row(5.5, 11.0, 16.5, 22.0, 30.25, 35.75, new BigDecimal("38.5"), new BigDecimal("46.75")));
+
+ assertRows(execute("SELECT a * 6.5, b * 6.5, c * 6.5, d * 6.5, e * 6.5, 6.5 * f, g * 6.5, h * 6.5 FROM %s WHERE a = 1 AND b = 2"),
+ row(6.5, 13.0, 19.5, 26.0, 35.75, 42.25, new BigDecimal("45.5"), BigDecimal.valueOf(55.25)));
+
+ // Test divisions
+
+ assertColumnNames(execute("SELECT a / 1, b / 1, c / 1, d / 1, e / 1, f / 1, g / 1, h / 1 FROM %s WHERE a = 1 AND b = 2"),
+ "a / 1", "b / 1", "c / 1", "d / 1", "e / 1", "f / 1", "g / 1", "h / 1");
+
+ assertRows(execute("SELECT a / 1, b / 1, c / 1, d / 1, e / 1, f / 1, g / 1, h / 1 FROM %s WHERE a = 1 AND b = 2"),
+ row(1, 2, 3, 4L, 5.5F, 6.5, BigInteger.valueOf(7), new BigDecimal("8.5")));
+
+ assertRows(execute("SELECT a / 2, b / 2, c / 2, d / 2, e / 2, f / 2, g / 2, h / 2 FROM %s WHERE a = 1 AND b = 2"),
+ row(0, 1, 1, 2L, 2.75F, 3.25, BigInteger.valueOf(3), new BigDecimal("4.25")));
+
+ assertRows(execute("SELECT a / 3, b / 3, c / 3, d / 3, e / 3, f / 3, g / 3, h / 3 FROM %s WHERE a = 1 AND b = 2"),
+ row(0, 0, 1, 1L, 1.8333334F, 2.1666666666666665, BigInteger.valueOf(2), new BigDecimal("2.83333333333333333333333333333333")));
+
+ assertRows(execute("SELECT a / " + bigInt + ","
+ + " b / " + bigInt + ","
+ + " c / " + bigInt + ","
+ + " d / " + bigInt + ","
+ + " e / " + bigInt + ","
+ + " f / " + bigInt + ","
+ + " g / " + bigInt + " FROM %s WHERE a = 1 AND b = 2"),
+ row(1L / bigInt,
+ 2L / bigInt,
+ 3L / bigInt,
+ 4L / bigInt,
+ 5.5 / bigInt,
+ 6.5 / bigInt,
+ BigInteger.valueOf(7).divide(BigInteger.valueOf(bigInt))));
+
+ assertRows(execute("SELECT a / 5.5, b / 5.5, c / 5.5, d / 5.5, e / 5.5, f / 5.5, g / 5.5, h / 5.5 FROM %s WHERE a = 1 AND b = 2"),
+ row(0.18181818181818182, 0.36363636363636365, 0.5454545454545454, 0.7272727272727273, 1.0, 1.1818181818181819, new BigDecimal("1.27272727272727272727272727272727"), new BigDecimal("1.54545454545454545454545454545455")));
+
+ assertRows(execute("SELECT a / 6.5, b / 6.5, c / 6.5, d / 6.5, e / 6.5, f / 6.5, g / 6.5, h / 6.5 FROM %s WHERE a = 1 AND b = 2"),
+ row(0.15384615384615385, 0.3076923076923077, 0.46153846153846156, 0.6153846153846154, 0.8461538461538461, 1.0, new BigDecimal("1.07692307692307692307692307692308"), new BigDecimal("1.30769230769230769230769230769231")));
+
+ // Test modulo operations
+
+ assertColumnNames(execute("SELECT a %% 1, b %% 1, c %% 1, d %% 1, e %% 1, f %% 1, g %% 1, h %% 1 FROM %s WHERE a = 1 AND b = 2"),
+ "a % 1", "b % 1", "c % 1", "d % 1", "e % 1", "f % 1", "g % 1", "h % 1");
+
+ assertRows(execute("SELECT a %% 1, b %% 1, c %% 1, d %% 1, e %% 1, f %% 1, g %% 1, h %% 1 FROM %s WHERE a = 1 AND b = 2"),
+ row(0, 0, 0, 0L, 0.5F, 0.5, BigInteger.valueOf(0), new BigDecimal("0.5")));
+
+ assertRows(execute("SELECT a %% 2, b %% 2, c %% 2, d %% 2, e %% 2, f %% 2, g %% 2, h %% 2 FROM %s WHERE a = 1 AND b = 2"),
+ row(1, 0, 1, 0L, 1.5F, 0.5, BigInteger.valueOf(1), new BigDecimal("0.5")));
+
+ assertRows(execute("SELECT a %% 3, b %% 3, c %% 3, d %% 3, e %% 3, f %% 3, g %% 3, h %% 3 FROM %s WHERE a = 1 AND b = 2"),
+ row(1, 2, 0, 1L, 2.5F, 0.5, BigInteger.valueOf(1), new BigDecimal("2.5")));
+
+ assertRows(execute("SELECT a %% " + bigInt + ","
+ + " b %% " + bigInt + ","
+ + " c %% " + bigInt + ","
+ + " d %% " + bigInt + ","
+ + " e %% " + bigInt + ","
+ + " f %% " + bigInt + ","
+ + " g %% " + bigInt + ","
+ + " h %% " + bigInt + " FROM %s WHERE a = 1 AND b = 2"),
+ row(1L % bigInt,
+ 2L % bigInt,
+ 3L % bigInt,
+ 4L % bigInt,
+ 5.5 % bigInt,
+ 6.5 % bigInt,
+ BigInteger.valueOf(7 % bigInt),
+ BigDecimal.valueOf(8.5 % bigInt)));
+
+ assertRows(execute("SELECT a %% 5.5, b %% 5.5, c %% 5.5, d %% 5.5, e %% 5.5, f %% 5.5, g %% 5.5, h %% 5.5 FROM %s WHERE a = 1 AND b = 2"),
+ row(1.0, 2.0, 3.0, 4.0, 0.0, 1.0, new BigDecimal("1.5"), new BigDecimal("3.0")));
+
+ assertRows(execute("SELECT a %% 6.5, b %% 6.5, c %% 6.5, d %% 6.5, e %% 6.5, f %% 6.5, g %% 6.5, h %% 6.5 FROM %s WHERE a = 1 AND b = 2"),
+ row(1.0, 2.0, 3.0, 4.0, 5.5, 0.0, new BigDecimal("0.5"), new BigDecimal("2.0")));
+
+ assertRows(execute("SELECT a, b, 1 + 1, 2 - 1, 2 * 2, 2 / 1 , 2 %% 1, (int) -1 FROM %s WHERE a = 1 AND b = 2"),
+ row((byte) 1, (short) 2, 2, 1, 4, 2, 0, -1));
+ }
+
+ @Test
+ public void testDivisionWithDecimals() throws Throwable
+ {
+ createTable("CREATE TABLE %s (numerator decimal, denominator decimal, PRIMARY KEY((numerator, denominator)))");
+ execute("INSERT INTO %s (numerator, denominator) VALUES (8.5, 200000000000000000000000000000000000)");
+ execute("INSERT INTO %s (numerator, denominator) VALUES (10000, 3)");
+
+ assertRows(execute("SELECT numerator / denominator from %s"),
+ row(new BigDecimal("0.0000000000000000000000000000000000425")),
+ row(new BigDecimal("3333.33333333333333333333333333333333")));
+ }
+
+ @Test
+ public void testWithCounters() throws Throwable
+ {
+ createTable("CREATE TABLE %s (a int PRIMARY KEY, b counter)");
+ execute("UPDATE %s SET b = b + 1 WHERE a = 1");
+ execute("UPDATE %s SET b = b + 1 WHERE a = 1");
+ assertRows(execute("SELECT b FROM %s WHERE a = 1"), row(2L));
+
+ assertRows(execute("SELECT b + (tinyint) 1,"
+ + " b + (smallint) 1,"
+ + " b + 1,"
+ + " b + (bigint) 1,"
+ + " b + (float) 1.5,"
+ + " b + 1.5,"
+ + " b + (varint) 1,"
+ + " b + (decimal) 1.5,"
+ + " b + b FROM %s WHERE a = 1"),
+ row(3L, 3L, 3L, 3L, 3.5, 3.5, BigInteger.valueOf(3), new BigDecimal("3.5"), 4L));
+
+ assertRows(execute("SELECT b - (tinyint) 1,"
+ + " b - (smallint) 1,"
+ + " b - 1,"
+ + " b - (bigint) 1,"
+ + " b - (float) 1.5,"
+ + " b - 1.5,"
+ + " b - (varint) 1,"
+ + " b - (decimal) 1.5,"
+ + " b - b FROM %s WHERE a = 1"),
+ row(1L, 1L, 1L, 1L, 0.5, 0.5, BigInteger.valueOf(1), new BigDecimal("0.5"), 0L));
+
+ assertRows(execute("SELECT b * (tinyint) 1,"
+ + " b * (smallint) 1,"
+ + " b * 1,"
+ + " b * (bigint) 1,"
+ + " b * (float) 1.5,"
+ + " b * 1.5,"
+ + " b * (varint) 1,"
+ + " b * (decimal) 1.5,"
+ + " b * b FROM %s WHERE a = 1"),
+ row(2L, 2L, 2L, 2L, 3.0, 3.0, BigInteger.valueOf(2), new BigDecimal("3.00"), 4L));
+
+ assertRows(execute("SELECT b / (tinyint) 1,"
+ + " b / (smallint) 1,"
+ + " b / 1,"
+ + " b / (bigint) 1,"
+ + " b / (float) 0.5,"
+ + " b / 0.5,"
+ + " b / (varint) 1,"
+ + " b / (decimal) 0.5,"
+ + " b / b FROM %s WHERE a = 1"),
+ row(2L, 2L, 2L, 2L, 4.0, 4.0, BigInteger.valueOf(2), new BigDecimal("4"), 1L));
+
+ assertRows(execute("SELECT b %% (tinyint) 1,"
+ + " b %% (smallint) 1,"
+ + " b %% 1,"
+ + " b %% (bigint) 1,"
+ + " b %% (float) 0.5,"
+ + " b %% 0.5,"
+ + " b %% (varint) 1,"
+ + " b %% (decimal) 0.5,"
+ + " b %% b FROM %s WHERE a = 1"),
+ row(0L, 0L, 0L, 0L, 0.0, 0.0, BigInteger.valueOf(0), new BigDecimal("0.0"), 0L));
+
+ assertRows(execute("SELECT -b FROM %s WHERE a = 1"), row(-2L));
+ }
+
+ @Test
+ public void testPrecedenceAndParentheses() throws Throwable
+ {
+ createTable("CREATE TABLE %s (a int, b int, c int, d int, PRIMARY KEY(a, b))");
+ execute("INSERT INTO %S (a, b, c, d) VALUES (2, 5, 25, 4)");
+
+ UntypedResultSet rs = execute("SELECT a - c / b + d FROM %s");
+ assertColumnNames(rs, "a - c / b + d");
+ assertRows(rs, row(1));
+
+ rs = execute("SELECT (c - b) / a + d FROM %s");
+ assertColumnNames(rs, "(c - b) / a + d");
+ assertRows(rs, row(14));
+
+ rs = execute("SELECT c / a / b FROM %s");
+ assertColumnNames(rs, "c / a / b");
+ assertRows(rs, row(2));
+
+ rs = execute("SELECT c / b / d FROM %s");
+ assertColumnNames(rs, "c / b / d");
+ assertRows(rs, row(1));
+
+ rs = execute("SELECT (c - a) %% d / a FROM %s");
+ assertColumnNames(rs, "(c - a) % d / a");
+ assertRows(rs, row(1));
+
+ rs = execute("SELECT (c - a) %% d / a + d FROM %s");
+ assertColumnNames(rs, "(c - a) % d / a + d");
+ assertRows(rs, row(5));
+
+ rs = execute("SELECT -(c - a) %% d / a + d FROM %s");
+ assertColumnNames(rs, "-(c - a) % d / a + d");
+ assertRows(rs, row(3));
+
+ rs = execute("SELECT (-c - a) %% d / a + d FROM %s");
+ assertColumnNames(rs, "(-c - a) % d / a + d");
+ assertRows(rs, row(3));
+
+ rs = execute("SELECT c - a %% d / a + d FROM %s");
+ assertColumnNames(rs, "c - a % d / a + d");
+ assertRows(rs, row(28));
+
+ rs = execute("SELECT (int)((c - a) %% d / (a + d)) FROM %s");
+ assertColumnNames(rs, "(int)((c - a) % d / (a + d))");
+ assertRows(rs, row(0));
+
+ // test with aliases
+ rs = execute("SELECT (int)((c - a) %% d / (a + d)) as result FROM %s");
+ assertColumnNames(rs, "result");
+ assertRows(rs, row(0));
+
+ rs = execute("SELECT c / a / b as divisions FROM %s");
+ assertColumnNames(rs, "divisions");
+ assertRows(rs, row(2));
+
+ assertRows(execute("SELECT * FROM %s WHERE a = ? AND b = (int) ? / 2 - 5", 2, 20),
+ row(2, 5, 25, 4));
+
+ assertRows(execute("SELECT * FROM %s WHERE a = ? AND b = (int) ? / (2 + 2)", 2, 20),
+ row(2, 5, 25, 4));
+ }
+
+ @Test
+ public void testWithDivisionByZero() throws Throwable
+ {
+ createTable("CREATE TABLE %s (a tinyint, b smallint, c int, d bigint, e float, f double, g varint, h decimal, PRIMARY KEY(a, b))");
+ execute("INSERT INTO %S (a, b, c, d, e, f, g, h) VALUES (0, 2, 3, 4, 5.5, 6.5, 7, 8.5)");
+
+ assertInvalidThrowMessage("the operation 'tinyint / tinyint' failed: / by zero",
+ OperationExecutionException.class,
+ "SELECT a / a FROM %s WHERE a = 0 AND b = 2");
+
+ assertInvalidThrowMessage("the operation 'smallint / tinyint' failed: / by zero",
+ OperationExecutionException.class,
+ "SELECT b / a FROM %s WHERE a = 0 AND b = 2");
+
+ assertInvalidThrowMessage("the operation 'int / tinyint' failed: / by zero",
+ OperationExecutionException.class,
+ "SELECT c / a FROM %s WHERE a = 0 AND b = 2");
+
+ assertInvalidThrowMessage("the operation 'bigint / tinyint' failed: / by zero",
+ OperationExecutionException.class,
+ "SELECT d / a FROM %s WHERE a = 0 AND b = 2");
+
+ assertInvalidThrowMessage("the operation 'smallint / smallint' failed: / by zero",
+ OperationExecutionException.class,
+ "SELECT a FROM %s WHERE a = 0 AND b = 10/0");
+
+ assertRows(execute("SELECT e / a FROM %s WHERE a = 0 AND b = 2"), row(Float.POSITIVE_INFINITY));
+ assertRows(execute("SELECT f / a FROM %s WHERE a = 0 AND b = 2"), row(Double.POSITIVE_INFINITY));
+
+ assertInvalidThrowMessage("the operation 'varint / tinyint' failed: BigInteger divide by zero",
+ OperationExecutionException.class,
+ "SELECT g / a FROM %s WHERE a = 0 AND b = 2");
+
+ assertInvalidThrowMessage("the operation 'decimal / tinyint' failed: BigInteger divide by zero",
+ OperationExecutionException.class,
+ "SELECT h / a FROM %s WHERE a = 0 AND b = 2");
+ }
+
+ @Test
+ public void testWithNanAndInfinity() throws Throwable
+ {
+ createTable("CREATE TABLE %s (a int PRIMARY KEY, b double, c decimal)");
+ assertInvalidMessage("Ambiguous '+' operation with args ? and 1: use type casts to disambiguate",
+ "INSERT INTO %S (a, b, c) VALUES (? + 1, ?, ?)", 0, Double.NaN, BigDecimal.valueOf(1));
+
+ execute("INSERT INTO %S (a, b, c) VALUES ((int) ? + 1, -?, ?)", 0, Double.NaN, BigDecimal.valueOf(1));
+
+ assertRows(execute("SELECT * FROM %s"), row(1, Double.NaN, BigDecimal.valueOf(1)));
+
+ assertRows(execute("SELECT a + NAN, b + 1 FROM %s"), row(Double.NaN, Double.NaN));
+ assertInvalidThrowMessage("the operation 'decimal + double' failed: A NaN cannot be converted into a decimal",
+ OperationExecutionException.class,
+ "SELECT c + NAN FROM %s");
+
+ assertRows(execute("SELECT a + (float) NAN, b + 1 FROM %s"), row(Float.NaN, Double.NaN));
+ assertInvalidThrowMessage("the operation 'decimal + float' failed: A NaN cannot be converted into a decimal",
+ OperationExecutionException.class,
+ "SELECT c + (float) NAN FROM %s");
+
+ execute("INSERT INTO %S (a, b, c) VALUES (?, ?, ?)", 1, Double.POSITIVE_INFINITY, BigDecimal.valueOf(1));
+ assertRows(execute("SELECT * FROM %s"), row(1, Double.POSITIVE_INFINITY, BigDecimal.valueOf(1)));
+
+ assertRows(execute("SELECT a + INFINITY, b + 1 FROM %s"), row(Double.POSITIVE_INFINITY, Double.POSITIVE_INFINITY));
+ assertInvalidThrowMessage("the operation 'decimal + double' failed: An infinite number cannot be converted into a decimal",
+ OperationExecutionException.class,
+ "SELECT c + INFINITY FROM %s");
+
+ assertRows(execute("SELECT a + (float) INFINITY, b + 1 FROM %s"), row(Float.POSITIVE_INFINITY, Double.POSITIVE_INFINITY));
+ assertInvalidThrowMessage("the operation 'decimal + float' failed: An infinite number cannot be converted into a decimal",
+ OperationExecutionException.class,
+ "SELECT c + (float) INFINITY FROM %s");
+
+ execute("INSERT INTO %S (a, b, c) VALUES (?, ?, ?)", 1, Double.NEGATIVE_INFINITY, BigDecimal.valueOf(1));
+ assertRows(execute("SELECT * FROM %s"), row(1, Double.NEGATIVE_INFINITY, BigDecimal.valueOf(1)));
+
+ assertRows(execute("SELECT a + -INFINITY, b + 1 FROM %s"), row(Double.NEGATIVE_INFINITY, Double.NEGATIVE_INFINITY));
+ assertInvalidThrowMessage("the operation 'decimal + double' failed: An infinite number cannot be converted into a decimal",
+ OperationExecutionException.class,
+ "SELECT c + -INFINITY FROM %s");
+
+ assertRows(execute("SELECT a + (float) -INFINITY, b + 1 FROM %s"), row(Float.NEGATIVE_INFINITY, Double.NEGATIVE_INFINITY));
+ assertInvalidThrowMessage("the operation 'decimal + float' failed: An infinite number cannot be converted into a decimal",
+ OperationExecutionException.class,
+ "SELECT c + (float) -INFINITY FROM %s");
+ }
+
+ @Test
+ public void testInvalidTypes() throws Throwable
+ {
+ createTable("CREATE TABLE %s (a int PRIMARY KEY, b boolean, c text)");
+ execute("INSERT INTO %S (a, b, c) VALUES (?, ?, ?)", 1, true, "test");
+
+ assertInvalidMessage("the '+' operation is not supported between a and b", "SELECT a + b FROM %s");
+ assertInvalidMessage("the '+' operation is not supported between b and c", "SELECT b + c FROM %s");
+ assertInvalidMessage("the '+' operation is not supported between b and 1", "SELECT b + 1 FROM %s");
+ assertInvalidMessage("the '+' operation is not supported between 1 and b", "SELECT 1 + b FROM %s");
+ assertInvalidMessage("the '+' operation is not supported between b and NaN", "SELECT b + NaN FROM %s");
+ assertInvalidMessage("the '/' operation is not supported between a and b", "SELECT a / b FROM %s");
+ assertInvalidMessage("the '/' operation is not supported between b and c", "SELECT b / c FROM %s");
+ assertInvalidMessage("the '/' operation is not supported between b and 1", "SELECT b / 1 FROM %s");
+ assertInvalidMessage("the '/' operation is not supported between NaN and b", "SELECT NaN / b FROM %s");
+ assertInvalidMessage("the '/' operation is not supported between -Infinity and b", "SELECT -Infinity / b FROM %s");
+ }
+
+ @Test
+ public void testOverflow() throws Throwable
+ {
+ createTable("CREATE TABLE %s (a int PRIMARY KEY, b tinyint, c smallint)");
+ execute("INSERT INTO %S (a, b, c) VALUES (?, ?, ?)", 1, (byte) 1, (short) 1);
+ assertRows(execute("SELECT a + (int) ?, b + (tinyint) ?, c + (smallint) ? FROM %s", 1, (byte) 1, (short) 1),
+ row(2, (byte) 2,(short) 2));
+ assertRows(execute("SELECT a + (int) ?, b + (tinyint) ?, c + (smallint) ? FROM %s", Integer.MAX_VALUE, Byte.MAX_VALUE, Short.MAX_VALUE),
+ row(Integer.MIN_VALUE, Byte.MIN_VALUE, Short.MIN_VALUE));
+ }
+
+ @Test
+ public void testOperationsWithDuration() throws Throwable
+ {
+ // Test with timestamp type.
+ createTable("CREATE TABLE %s (pk int, time timestamp, v int, primary key (pk, time))");
+
+ execute("INSERT INTO %s (pk, time, v) VALUES (1, '2016-09-27 16:10:00 UTC', 1)");
+ execute("INSERT INTO %s (pk, time, v) VALUES (1, '2016-09-27 16:12:00 UTC', 2)");
+ execute("INSERT INTO %s (pk, time, v) VALUES (1, '2016-09-27 16:14:00 UTC', 3)");
+ execute("INSERT INTO %s (pk, time, v) VALUES (1, '2016-09-27 16:15:00 UTC', 4)");
+ execute("INSERT INTO %s (pk, time, v) VALUES (1, '2016-09-27 16:21:00 UTC', 5)");
+ execute("INSERT INTO %s (pk, time, v) VALUES (1, '2016-09-27 16:22:00 UTC', 6)");
+
+ assertRows(execute("SELECT * FROM %s WHERE pk = 1 AND time > ? - 5m", toTimestamp("2016-09-27 16:20:00 UTC")),
+ row(1, toTimestamp("2016-09-27 16:21:00 UTC"), 5),
+ row(1, toTimestamp("2016-09-27 16:22:00 UTC"), 6));
+
+ assertRows(execute("SELECT * FROM %s WHERE pk = 1 AND time >= ? - 10m", toTimestamp("2016-09-27 16:25:00 UTC")),
+ row(1, toTimestamp("2016-09-27 16:15:00 UTC"), 4),
+ row(1, toTimestamp("2016-09-27 16:21:00 UTC"), 5),
+ row(1, toTimestamp("2016-09-27 16:22:00 UTC"), 6));
+
+ assertRows(execute("SELECT * FROM %s WHERE pk = 1 AND time >= ? + 5m", toTimestamp("2016-09-27 16:15:00 UTC")),
+ row(1, toTimestamp("2016-09-27 16:21:00 UTC"), 5),
+ row(1, toTimestamp("2016-09-27 16:22:00 UTC"), 6));
+
+ assertRows(execute("SELECT time - 10m FROM %s WHERE pk = 1"),
+ row(toTimestamp("2016-09-27 16:00:00 UTC")),
+ row(toTimestamp("2016-09-27 16:02:00 UTC")),
+ row(toTimestamp("2016-09-27 16:04:00 UTC")),
+ row(toTimestamp("2016-09-27 16:05:00 UTC")),
+ row(toTimestamp("2016-09-27 16:11:00 UTC")),
+ row(toTimestamp("2016-09-27 16:12:00 UTC")));
+
+ assertInvalidMessage("the '%' operation is not supported between time and 10m",
+ "SELECT time %% 10m FROM %s WHERE pk = 1");
+ assertInvalidMessage("the '*' operation is not supported between time and 10m",
+ "SELECT time * 10m FROM %s WHERE pk = 1");
+ assertInvalidMessage("the '/' operation is not supported between time and 10m",
+ "SELECT time / 10m FROM %s WHERE pk = 1");
+ assertInvalidMessage("the operation 'timestamp - duration' failed: The duration must have a millisecond precision. Was: 10us",
+ "SELECT * FROM %s WHERE pk = 1 AND time > ? - 10us", toTimestamp("2016-09-27 16:15:00 UTC"));
+
+ // Test with date type.
+ createTable("CREATE TABLE %s (pk int, time date, v int, primary key (pk, time))");
+
+ execute("INSERT INTO %s (pk, time, v) VALUES (1, '2016-09-27', 1)");
+ execute("INSERT INTO %s (pk, time, v) VALUES (1, '2016-09-28', 2)");
+ execute("INSERT INTO %s (pk, time, v) VALUES (1, '2016-09-29', 3)");
+ execute("INSERT INTO %s (pk, time, v) VALUES (1, '2016-09-30', 4)");
+ execute("INSERT INTO %s (pk, time, v) VALUES (1, '2016-10-01', 5)");
+ execute("INSERT INTO %s (pk, time, v) VALUES (1, '2016-10-04', 6)");
+
+ assertRows(execute("SELECT * FROM %s WHERE pk = 1 AND time > ? - 5d", toDate("2016-10-04")),
+ row(1, toDate("2016-09-30"), 4),
+ row(1, toDate("2016-10-01"), 5),
+ row(1, toDate("2016-10-04"), 6));
+
+ assertRows(execute("SELECT * FROM %s WHERE pk = 1 AND time > ? - 6d", toDate("2016-10-04")),
+ row(1, toDate("2016-09-29"), 3),
+ row(1, toDate("2016-09-30"), 4),
+ row(1, toDate("2016-10-01"), 5),
+ row(1, toDate("2016-10-04"), 6));
+
+ assertRows(execute("SELECT * FROM %s WHERE pk = 1 AND time >= ? + 1d", toDate("2016-10-01")),
+ row(1, toDate("2016-10-04"), 6));
+
+ assertRows(execute("SELECT time - 2d FROM %s WHERE pk = 1"),
+ row(toDate("2016-09-25")),
+ row(toDate("2016-09-26")),
+ row(toDate("2016-09-27")),
+ row(toDate("2016-09-28")),
+ row(toDate("2016-09-29")),
+ row(toDate("2016-10-02")));
+
+ assertInvalidMessage("the '%' operation is not supported between time and 10m",
+ "SELECT time %% 10m FROM %s WHERE pk = 1");
+ assertInvalidMessage("the '*' operation is not supported between time and 10m",
+ "SELECT time * 10m FROM %s WHERE pk = 1");
+ assertInvalidMessage("the '/' operation is not supported between time and 10m",
+ "SELECT time / 10m FROM %s WHERE pk = 1");
+ assertInvalidMessage("the operation 'date - duration' failed: The duration must have a day precision. Was: 10m",
+ "SELECT * FROM %s WHERE pk = 1 AND time > ? - 10m", toDate("2016-10-04"));
+ }
+
+ private Date toTimestamp(String timestampAsString)
+ {
+ return new Date(TimestampSerializer.dateStringToTimestamp(timestampAsString));
+ }
+
+ private int toDate(String dateAsString)
+ {
+ return SimpleDateSerializer.dateStringToDays(dateAsString);
+ }
+}
diff --git a/test/unit/org/apache/cassandra/cql3/functions/TimeFctsTest.java b/test/unit/org/apache/cassandra/cql3/functions/TimeFctsTest.java
index 5b9737f..b0a4bb9 100644
--- a/test/unit/org/apache/cassandra/cql3/functions/TimeFctsTest.java
+++ b/test/unit/org/apache/cassandra/cql3/functions/TimeFctsTest.java
@@ -18,7 +18,10 @@
package org.apache.cassandra.cql3.functions;
import java.nio.ByteBuffer;
-import java.util.Arrays;
+import java.time.*;
+import java.time.format.DateTimeFormatter;
+import java.util.Collections;
+import java.util.Date;
import java.util.List;
import org.junit.Test;
@@ -30,24 +33,28 @@
import org.apache.cassandra.transport.ProtocolVersion;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.UUIDGen;
-import org.joda.time.DateTime;
-import org.joda.time.DateTimeZone;
-import org.joda.time.format.DateTimeFormat;
+import static org.apache.cassandra.cql3.functions.TimeFcts.*;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
public class TimeFctsTest
{
+ private static final LocalDate LOCAL_DATE = LocalDate.of(2019, 8, 3);
+
+ private static final ZonedDateTime DATE = LOCAL_DATE.atStartOfDay(ZoneOffset.UTC);
+ private static final LocalTime LOCAL_TIME = LocalTime.of(11, 3, 2);
+ private static final ZonedDateTime DATE_TIME =
+ ZonedDateTime.of(LOCAL_DATE, LOCAL_TIME, ZoneOffset.UTC);
+ private static final String DATE_STRING = DATE.format(DateTimeFormatter.ISO_LOCAL_DATE);
+ private static final String DATE_TIME_STRING =
+ DATE_STRING + " " + LOCAL_TIME.format(DateTimeFormatter.ISO_LOCAL_TIME);
+
@Test
public void testMinTimeUuid()
{
- DateTime dateTime = DateTimeFormat.forPattern("yyyy-MM-dd hh:mm:ss")
- .withZone(DateTimeZone.UTC)
- .parseDateTime("2015-05-21 11:03:02");
-
- long timeInMillis = dateTime.getMillis();
- ByteBuffer input = TimestampType.instance.fromString("2015-05-21 11:03:02+00");
+ long timeInMillis = DATE_TIME.toInstant().toEpochMilli();
+ ByteBuffer input = TimestampType.instance.fromString(DATE_TIME_STRING + "+00");
ByteBuffer output = executeFunction(TimeFcts.minTimeuuidFct, input);
assertEquals(UUIDGen.minTimeUUID(timeInMillis), TimeUUIDType.instance.compose(output));
}
@@ -55,12 +62,8 @@
@Test
public void testMaxTimeUuid()
{
- DateTime dateTime = DateTimeFormat.forPattern("yyyy-MM-dd hh:mm:ss")
- .withZone(DateTimeZone.UTC)
- .parseDateTime("2015-05-21 11:03:02");
-
- long timeInMillis = dateTime.getMillis();
- ByteBuffer input = TimestampType.instance.fromString("2015-05-21 11:03:02+00");
+ long timeInMillis = DATE_TIME.toInstant().toEpochMilli();
+ ByteBuffer input = TimestampType.instance.fromString(DATE_TIME_STRING + "+00");
ByteBuffer output = executeFunction(TimeFcts.maxTimeuuidFct, input);
assertEquals(UUIDGen.maxTimeUUID(timeInMillis), TimeUUIDType.instance.compose(output));
}
@@ -68,37 +71,26 @@
@Test
public void testDateOf()
{
- DateTime dateTime = DateTimeFormat.forPattern("yyyy-MM-dd hh:mm:ss")
- .withZone(DateTimeZone.UTC)
- .parseDateTime("2015-05-21 11:03:02");
- long timeInMillis = dateTime.getMillis();
+ long timeInMillis = DATE_TIME.toInstant().toEpochMilli();
ByteBuffer input = ByteBuffer.wrap(UUIDGen.getTimeUUIDBytes(timeInMillis, 0));
ByteBuffer output = executeFunction(TimeFcts.dateOfFct, input);
- assertEquals(dateTime.toDate(), TimestampType.instance.compose(output));
+ assertEquals(Date.from(DATE_TIME.toInstant()), TimestampType.instance.compose(output));
}
@Test
public void testTimeUuidToTimestamp()
{
- DateTime dateTime = DateTimeFormat.forPattern("yyyy-MM-dd hh:mm:ss")
- .withZone(DateTimeZone.UTC)
- .parseDateTime("2015-05-21 11:03:02");
-
- long timeInMillis = dateTime.getMillis();
+ long timeInMillis = DATE_TIME.toInstant().toEpochMilli();
ByteBuffer input = ByteBuffer.wrap(UUIDGen.getTimeUUIDBytes(timeInMillis, 0));
- ByteBuffer output = executeFunction(TimeFcts.timeUuidToTimestamp, input);
- assertEquals(dateTime.toDate(), TimestampType.instance.compose(output));
+ ByteBuffer output = executeFunction(toTimestamp(TimeUUIDType.instance), input);
+ assertEquals(Date.from(DATE_TIME.toInstant()), TimestampType.instance.compose(output));
}
@Test
public void testUnixTimestampOfFct()
{
- DateTime dateTime = DateTimeFormat.forPattern("yyyy-MM-dd hh:mm:ss")
- .withZone(DateTimeZone.UTC)
- .parseDateTime("2015-05-21 11:03:02");
-
- long timeInMillis = dateTime.getMillis();
+ long timeInMillis = DATE_TIME.toInstant().toEpochMilli();
ByteBuffer input = ByteBuffer.wrap(UUIDGen.getTimeUUIDBytes(timeInMillis, 0));
ByteBuffer output = executeFunction(TimeFcts.unixTimestampOfFct, input);
assertEquals(timeInMillis, LongType.instance.compose(output).longValue());
@@ -107,31 +99,20 @@
@Test
public void testTimeUuidToUnixTimestamp()
{
- DateTime dateTime = DateTimeFormat.forPattern("yyyy-MM-dd hh:mm:ss")
- .withZone(DateTimeZone.UTC)
- .parseDateTime("2015-05-21 11:03:02");
-
- long timeInMillis = dateTime.getMillis();
+ long timeInMillis = DATE_TIME.toInstant().toEpochMilli();
ByteBuffer input = ByteBuffer.wrap(UUIDGen.getTimeUUIDBytes(timeInMillis, 0));
- ByteBuffer output = executeFunction(TimeFcts.timeUuidToUnixTimestamp, input);
+ ByteBuffer output = executeFunction(toUnixTimestamp(TimeUUIDType.instance), input);
assertEquals(timeInMillis, LongType.instance.compose(output).longValue());
}
@Test
public void testTimeUuidToDate()
{
- DateTime dateTime = DateTimeFormat.forPattern("yyyy-MM-dd hh:mm:ss")
- .withZone(DateTimeZone.UTC)
- .parseDateTime("2015-05-21 11:03:02");
-
- long timeInMillis = dateTime.getMillis();
+ long timeInMillis = DATE_TIME.toInstant().toEpochMilli();
ByteBuffer input = ByteBuffer.wrap(UUIDGen.getTimeUUIDBytes(timeInMillis, 0));
- ByteBuffer output = executeFunction(TimeFcts.timeUuidtoDate, input);
+ ByteBuffer output = executeFunction(toDate(TimeUUIDType.instance), input);
- long expectedTime = DateTimeFormat.forPattern("yyyy-MM-dd")
- .withZone(DateTimeZone.UTC)
- .parseDateTime("2015-05-21")
- .getMillis();
+ long expectedTime = DATE.toInstant().toEpochMilli();
assertEquals(expectedTime, SimpleDateType.instance.toTimeInMillis(output));
}
@@ -139,68 +120,52 @@
@Test
public void testDateToTimestamp()
{
- DateTime dateTime = DateTimeFormat.forPattern("yyyy-MM-dd")
- .withZone(DateTimeZone.UTC)
- .parseDateTime("2015-05-21");
-
- ByteBuffer input = SimpleDateType.instance.fromString("2015-05-21");
- ByteBuffer output = executeFunction(TimeFcts.dateToTimestamp, input);
- assertEquals(dateTime.toDate(), TimestampType.instance.compose(output));
+ ByteBuffer input = SimpleDateType.instance.fromString(DATE_STRING);
+ ByteBuffer output = executeFunction(toTimestamp(SimpleDateType.instance), input);
+ assertEquals(Date.from(DATE.toInstant()), TimestampType.instance.compose(output));
}
@Test
public void testDateToUnixTimestamp()
{
- DateTime dateTime = DateTimeFormat.forPattern("yyyy-MM-dd")
- .withZone(DateTimeZone.UTC)
- .parseDateTime("2015-05-21");
-
- ByteBuffer input = SimpleDateType.instance.fromString("2015-05-21");
- ByteBuffer output = executeFunction(TimeFcts.dateToUnixTimestamp, input);
- assertEquals(dateTime.getMillis(), LongType.instance.compose(output).longValue());
+ ByteBuffer input = SimpleDateType.instance.fromString(DATE_STRING);
+ ByteBuffer output = executeFunction(toUnixTimestamp(SimpleDateType.instance), input);
+ assertEquals(DATE.toInstant().toEpochMilli(), LongType.instance.compose(output).longValue());
}
@Test
public void testTimestampToDate()
{
- DateTime dateTime = DateTimeFormat.forPattern("yyyy-MM-dd")
- .withZone(DateTimeZone.UTC)
- .parseDateTime("2015-05-21");
-
- ByteBuffer input = TimestampType.instance.fromString("2015-05-21 11:03:02+00");
- ByteBuffer output = executeFunction(TimeFcts.timestampToDate, input);
- assertEquals(dateTime.getMillis(), SimpleDateType.instance.toTimeInMillis(output));
+ ByteBuffer input = TimestampType.instance.fromString(DATE_TIME_STRING + "+00");
+ ByteBuffer output = executeFunction(toDate(TimestampType.instance), input);
+ assertEquals(DATE.toInstant().toEpochMilli(), SimpleDateType.instance.toTimeInMillis(output));
}
@Test
public void testTimestampToDateWithEmptyInput()
{
- ByteBuffer output = executeFunction(TimeFcts.timestampToDate, ByteBufferUtil.EMPTY_BYTE_BUFFER);
+ ByteBuffer output = executeFunction(toDate(TimestampType.instance), ByteBufferUtil.EMPTY_BYTE_BUFFER);
assertNull(output);
}
@Test
public void testTimestampToUnixTimestamp()
{
- DateTime dateTime = DateTimeFormat.forPattern("yyyy-MM-dd hh:mm:ss")
- .withZone(DateTimeZone.UTC)
- .parseDateTime("2015-05-21 11:03:02");
-
- ByteBuffer input = TimestampType.instance.decompose(dateTime.toDate());
- ByteBuffer output = executeFunction(TimeFcts.timestampToUnixTimestamp, input);
- assertEquals(dateTime.getMillis(), LongType.instance.compose(output).longValue());
+ ByteBuffer input = TimestampType.instance.decompose(Date.from(DATE_TIME.toInstant()));
+ ByteBuffer output = executeFunction(toUnixTimestamp(TimestampType.instance), input);
+ assertEquals(DATE_TIME.toInstant().toEpochMilli(), LongType.instance.compose(output).longValue());
}
@Test
public void testTimestampToUnixTimestampWithEmptyInput()
{
- ByteBuffer output = executeFunction(TimeFcts.timestampToUnixTimestamp, ByteBufferUtil.EMPTY_BYTE_BUFFER);
+ ByteBuffer output = executeFunction(TimeFcts.toUnixTimestamp(TimestampType.instance), ByteBufferUtil.EMPTY_BYTE_BUFFER);
assertNull(output);
}
private static ByteBuffer executeFunction(Function function, ByteBuffer input)
{
- List<ByteBuffer> params = Arrays.asList(input);
+ List<ByteBuffer> params = Collections.singletonList(input);
return ((ScalarFunction) function).execute(ProtocolVersion.CURRENT, params);
}
}
diff --git a/test/unit/org/apache/cassandra/cql3/restrictions/ClusteringColumnRestrictionsTest.java b/test/unit/org/apache/cassandra/cql3/restrictions/ClusteringColumnRestrictionsTest.java
index 83c00d0..35adff3 100644
--- a/test/unit/org/apache/cassandra/cql3/restrictions/ClusteringColumnRestrictionsTest.java
+++ b/test/unit/org/apache/cassandra/cql3/restrictions/ClusteringColumnRestrictionsTest.java
@@ -24,8 +24,8 @@
import org.junit.BeforeClass;
import org.junit.Test;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.*;
import org.apache.cassandra.cql3.Term.MultiItemTerminal;
@@ -52,9 +52,9 @@
@Test
public void testBoundsAsClusteringWithNoRestrictions()
{
- CFMetaData cfMetaData = newCFMetaData(Sort.ASC);
+ TableMetadata tableMetadata = newTableMetadata(Sort.ASC);
- ClusteringColumnRestrictions restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ ClusteringColumnRestrictions restrictions = new ClusteringColumnRestrictions(tableMetadata);
SortedSet<ClusteringBound> bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
assertEquals(1, bounds.size());
@@ -71,12 +71,12 @@
@Test
public void testBoundsAsClusteringWithOneEqRestrictionsAndOneClusteringColumn()
{
- CFMetaData cfMetaData = newCFMetaData(Sort.ASC);
+ TableMetadata tableMetadata = newTableMetadata(Sort.ASC);
ByteBuffer clustering_0 = ByteBufferUtil.bytes(1);
- Restriction eq = newSingleEq(cfMetaData, 0, clustering_0);
+ Restriction eq = newSingleEq(tableMetadata, 0, clustering_0);
- ClusteringColumnRestrictions restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ ClusteringColumnRestrictions restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(eq);
SortedSet<ClusteringBound> bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
@@ -94,12 +94,12 @@
@Test
public void testBoundsAsClusteringWithOneEqRestrictionsAndTwoClusteringColumns()
{
- CFMetaData cfMetaData = newCFMetaData(Sort.ASC, Sort.ASC);
+ TableMetadata tableMetadata = newTableMetadata(Sort.ASC, Sort.ASC);
ByteBuffer clustering_0 = ByteBufferUtil.bytes(1);
- Restriction eq = newSingleEq(cfMetaData, 0, clustering_0);
+ Restriction eq = newSingleEq(tableMetadata, 0, clustering_0);
- ClusteringColumnRestrictions restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ ClusteringColumnRestrictions restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(eq);
SortedSet<ClusteringBound> bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
@@ -121,11 +121,11 @@
ByteBuffer value2 = ByteBufferUtil.bytes(2);
ByteBuffer value3 = ByteBufferUtil.bytes(3);
- CFMetaData cfMetaData = newCFMetaData(Sort.ASC, Sort.ASC);
+ TableMetadata tableMetadata = newTableMetadata(Sort.ASC, Sort.ASC);
- Restriction in = newSingleIN(cfMetaData, 0, value1, value2, value3);
+ Restriction in = newSingleIN(tableMetadata, 0, value1, value2, value3);
- ClusteringColumnRestrictions restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ ClusteringColumnRestrictions restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(in);
SortedSet<ClusteringBound> bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
@@ -147,13 +147,13 @@
@Test
public void testBoundsAsClusteringWithSliceRestrictionsAndOneClusteringColumn()
{
- CFMetaData cfMetaData = newCFMetaData(Sort.ASC, Sort.ASC);
+ TableMetadata tableMetadata = newTableMetadata(Sort.ASC, Sort.ASC);
ByteBuffer value1 = ByteBufferUtil.bytes(1);
ByteBuffer value2 = ByteBufferUtil.bytes(2);
- Restriction slice = newSingleSlice(cfMetaData, 0, Bound.START, false, value1);
- ClusteringColumnRestrictions restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ Restriction slice = newSingleSlice(tableMetadata, 0, Bound.START, false, value1);
+ ClusteringColumnRestrictions restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(slice);
SortedSet<ClusteringBound> bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
@@ -164,8 +164,8 @@
assertEquals(1, bounds.size());
assertEmptyEnd(get(bounds, 0));
- slice = newSingleSlice(cfMetaData, 0, Bound.START, true, value1);
- restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ slice = newSingleSlice(tableMetadata, 0, Bound.START, true, value1);
+ restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(slice);
bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
@@ -176,8 +176,8 @@
assertEquals(1, bounds.size());
assertEmptyEnd(get(bounds, 0));
- slice = newSingleSlice(cfMetaData, 0, Bound.END, true, value1);
- restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ slice = newSingleSlice(tableMetadata, 0, Bound.END, true, value1);
+ restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(slice);
bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
@@ -188,8 +188,8 @@
assertEquals(1, bounds.size());
assertEndBound(get(bounds, 0), true, value1);
- slice = newSingleSlice(cfMetaData, 0, Bound.END, false, value1);
- restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ slice = newSingleSlice(tableMetadata, 0, Bound.END, false, value1);
+ restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(slice);
bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
@@ -200,9 +200,9 @@
assertEquals(1, bounds.size());
assertEndBound(get(bounds, 0), false, value1);
- slice = newSingleSlice(cfMetaData, 0, Bound.START, false, value1);
- Restriction slice2 = newSingleSlice(cfMetaData, 0, Bound.END, false, value2);
- restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ slice = newSingleSlice(tableMetadata, 0, Bound.START, false, value1);
+ Restriction slice2 = newSingleSlice(tableMetadata, 0, Bound.END, false, value2);
+ restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(slice).mergeWith(slice2);
bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
@@ -213,9 +213,9 @@
assertEquals(1, bounds.size());
assertEndBound(get(bounds, 0), false, value2);
- slice = newSingleSlice(cfMetaData, 0, Bound.START, true, value1);
- slice2 = newSingleSlice(cfMetaData, 0, Bound.END, true, value2);
- restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ slice = newSingleSlice(tableMetadata, 0, Bound.START, true, value1);
+ slice2 = newSingleSlice(tableMetadata, 0, Bound.END, true, value2);
+ restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(slice).mergeWith(slice2);
bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
@@ -233,13 +233,13 @@
@Test
public void testBoundsAsClusteringWithSliceRestrictionsAndOneDescendingClusteringColumn()
{
- CFMetaData cfMetaData = newCFMetaData(Sort.DESC, Sort.DESC);
+ TableMetadata tableMetadata = newTableMetadata(Sort.DESC, Sort.DESC);
ByteBuffer value1 = ByteBufferUtil.bytes(1);
ByteBuffer value2 = ByteBufferUtil.bytes(2);
- Restriction slice = newSingleSlice(cfMetaData, 0, Bound.START, false, value1);
- ClusteringColumnRestrictions restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ Restriction slice = newSingleSlice(tableMetadata, 0, Bound.START, false, value1);
+ ClusteringColumnRestrictions restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(slice);
SortedSet<ClusteringBound> bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
@@ -250,8 +250,8 @@
assertEquals(1, bounds.size());
assertEndBound(get(bounds, 0), false, value1);
- slice = newSingleSlice(cfMetaData, 0, Bound.START, true, value1);
- restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ slice = newSingleSlice(tableMetadata, 0, Bound.START, true, value1);
+ restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(slice);
bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
@@ -262,8 +262,8 @@
assertEquals(1, bounds.size());
assertEndBound(get(bounds, 0), true, value1);
- slice = newSingleSlice(cfMetaData, 0, Bound.END, true, value1);
- restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ slice = newSingleSlice(tableMetadata, 0, Bound.END, true, value1);
+ restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(slice);
bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
@@ -274,8 +274,8 @@
assertEquals(1, bounds.size());
assertEmptyEnd(get(bounds, 0));
- slice = newSingleSlice(cfMetaData, 0, Bound.END, false, value1);
- restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ slice = newSingleSlice(tableMetadata, 0, Bound.END, false, value1);
+ restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(slice);
bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
@@ -286,9 +286,9 @@
assertEquals(1, bounds.size());
assertEmptyEnd(get(bounds, 0));
- slice = newSingleSlice(cfMetaData, 0, Bound.START, false, value1);
- Restriction slice2 = newSingleSlice(cfMetaData, 0, Bound.END, false, value2);
- restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ slice = newSingleSlice(tableMetadata, 0, Bound.START, false, value1);
+ Restriction slice2 = newSingleSlice(tableMetadata, 0, Bound.END, false, value2);
+ restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(slice).mergeWith(slice2);
bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
@@ -299,9 +299,9 @@
assertEquals(1, bounds.size());
assertEndBound(get(bounds, 0), false, value1);
- slice = newSingleSlice(cfMetaData, 0, Bound.START, true, value1);
- slice2 = newSingleSlice(cfMetaData, 0, Bound.END, true, value2);
- restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ slice = newSingleSlice(tableMetadata, 0, Bound.START, true, value1);
+ slice2 = newSingleSlice(tableMetadata, 0, Bound.END, true, value2);
+ restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(slice).mergeWith(slice2);
bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
@@ -319,14 +319,14 @@
@Test
public void testBoundsAsClusteringWithEqAndInRestrictions()
{
- CFMetaData cfMetaData = newCFMetaData(Sort.ASC, Sort.ASC);
+ TableMetadata tableMetadata = newTableMetadata(Sort.ASC, Sort.ASC);
ByteBuffer value1 = ByteBufferUtil.bytes(1);
ByteBuffer value2 = ByteBufferUtil.bytes(2);
ByteBuffer value3 = ByteBufferUtil.bytes(3);
- Restriction eq = newSingleEq(cfMetaData, 0, value1);
- Restriction in = newSingleIN(cfMetaData, 1, value1, value2, value3);
- ClusteringColumnRestrictions restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ Restriction eq = newSingleEq(tableMetadata, 0, value1);
+ Restriction in = newSingleIN(tableMetadata, 1, value1, value2, value3);
+ ClusteringColumnRestrictions restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(eq).mergeWith(in);
SortedSet<ClusteringBound> bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
@@ -348,16 +348,16 @@
@Test
public void testBoundsAsClusteringWithEqAndSliceRestrictions()
{
- CFMetaData cfMetaData = newCFMetaData(Sort.ASC, Sort.ASC);
+ TableMetadata tableMetadata = newTableMetadata(Sort.ASC, Sort.ASC);
ByteBuffer value1 = ByteBufferUtil.bytes(1);
ByteBuffer value2 = ByteBufferUtil.bytes(2);
ByteBuffer value3 = ByteBufferUtil.bytes(3);
- Restriction eq = newSingleEq(cfMetaData, 0, value3);
+ Restriction eq = newSingleEq(tableMetadata, 0, value3);
- Restriction slice = newSingleSlice(cfMetaData, 1, Bound.START, false, value1);
- ClusteringColumnRestrictions restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ Restriction slice = newSingleSlice(tableMetadata, 1, Bound.START, false, value1);
+ ClusteringColumnRestrictions restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(eq).mergeWith(slice);
SortedSet<ClusteringBound> bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
@@ -368,8 +368,8 @@
assertEquals(1, bounds.size());
assertEndBound(get(bounds, 0), true, value3);
- slice = newSingleSlice(cfMetaData, 1, Bound.START, true, value1);
- restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ slice = newSingleSlice(tableMetadata, 1, Bound.START, true, value1);
+ restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(eq).mergeWith(slice);
bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
@@ -380,8 +380,8 @@
assertEquals(1, bounds.size());
assertEndBound(get(bounds, 0), true, value3);
- slice = newSingleSlice(cfMetaData, 1, Bound.END, true, value1);
- restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ slice = newSingleSlice(tableMetadata, 1, Bound.END, true, value1);
+ restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(eq).mergeWith(slice);
bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
@@ -392,8 +392,8 @@
assertEquals(1, bounds.size());
assertEndBound(get(bounds, 0), true, value3, value1);
- slice = newSingleSlice(cfMetaData, 1, Bound.END, false, value1);
- restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ slice = newSingleSlice(tableMetadata, 1, Bound.END, false, value1);
+ restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(eq).mergeWith(slice);
bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
@@ -404,9 +404,9 @@
assertEquals(1, bounds.size());
assertEndBound(get(bounds, 0), false, value3, value1);
- slice = newSingleSlice(cfMetaData, 1, Bound.START, false, value1);
- Restriction slice2 = newSingleSlice(cfMetaData, 1, Bound.END, false, value2);
- restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ slice = newSingleSlice(tableMetadata, 1, Bound.START, false, value1);
+ Restriction slice2 = newSingleSlice(tableMetadata, 1, Bound.END, false, value2);
+ restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(eq).mergeWith(slice).mergeWith(slice2);
bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
@@ -417,9 +417,9 @@
assertEquals(1, bounds.size());
assertEndBound(get(bounds, 0), false, value3, value2);
- slice = newSingleSlice(cfMetaData, 1, Bound.START, true, value1);
- slice2 = newSingleSlice(cfMetaData, 1, Bound.END, true, value2);
- restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ slice = newSingleSlice(tableMetadata, 1, Bound.START, true, value1);
+ slice2 = newSingleSlice(tableMetadata, 1, Bound.END, true, value2);
+ restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(eq).mergeWith(slice).mergeWith(slice2);
bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
@@ -437,12 +437,12 @@
@Test
public void testBoundsAsClusteringWithMultiEqRestrictions()
{
- CFMetaData cfMetaData = newCFMetaData(Sort.ASC, Sort.ASC);
+ TableMetadata tableMetadata = newTableMetadata(Sort.ASC, Sort.ASC);
ByteBuffer value1 = ByteBufferUtil.bytes(1);
ByteBuffer value2 = ByteBufferUtil.bytes(2);
- Restriction eq = newMultiEq(cfMetaData, 0, value1, value2);
- ClusteringColumnRestrictions restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ Restriction eq = newMultiEq(tableMetadata, 0, value1, value2);
+ ClusteringColumnRestrictions restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(eq);
SortedSet<ClusteringBound> bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
@@ -460,13 +460,13 @@
@Test
public void testBoundsAsClusteringWithMultiInRestrictions()
{
- CFMetaData cfMetaData = newCFMetaData(Sort.ASC, Sort.ASC);
+ TableMetadata tableMetadata = newTableMetadata(Sort.ASC, Sort.ASC);
ByteBuffer value1 = ByteBufferUtil.bytes(1);
ByteBuffer value2 = ByteBufferUtil.bytes(2);
ByteBuffer value3 = ByteBufferUtil.bytes(3);
- Restriction in = newMultiIN(cfMetaData, 0, asList(value1, value2), asList(value2, value3));
- ClusteringColumnRestrictions restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ Restriction in = newMultiIN(tableMetadata, 0, asList(value1, value2), asList(value2, value3));
+ ClusteringColumnRestrictions restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(in);
SortedSet<ClusteringBound> bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
@@ -486,14 +486,14 @@
@Test
public void testBoundsAsClusteringWithMultiSliceRestrictionsWithOneClusteringColumn()
{
- CFMetaData cfMetaData = newCFMetaData(Sort.ASC);
+ TableMetadata tableMetadata = newTableMetadata(Sort.ASC);
ByteBuffer value1 = ByteBufferUtil.bytes(1);
ByteBuffer value2 = ByteBufferUtil.bytes(2);
- Restriction slice = newMultiSlice(cfMetaData, 0, Bound.START, false, value1);
- ClusteringColumnRestrictions restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ Restriction slice = newMultiSlice(tableMetadata, 0, Bound.START, false, value1);
+ ClusteringColumnRestrictions restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(slice);
SortedSet<ClusteringBound> bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
@@ -504,8 +504,8 @@
assertEquals(1, bounds.size());
assertEmptyEnd(get(bounds, 0));
- slice = newMultiSlice(cfMetaData, 0, Bound.START, true, value1);
- restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ slice = newMultiSlice(tableMetadata, 0, Bound.START, true, value1);
+ restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(slice);
bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
@@ -516,8 +516,8 @@
assertEquals(1, bounds.size());
assertEmptyEnd(get(bounds, 0));
- slice = newMultiSlice(cfMetaData, 0, Bound.END, true, value1);
- restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ slice = newMultiSlice(tableMetadata, 0, Bound.END, true, value1);
+ restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(slice);
bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
@@ -528,8 +528,8 @@
assertEquals(1, bounds.size());
assertEndBound(get(bounds, 0), true, value1);
- slice = newMultiSlice(cfMetaData, 0, Bound.END, false, value1);
- restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ slice = newMultiSlice(tableMetadata, 0, Bound.END, false, value1);
+ restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(slice);
bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
@@ -540,9 +540,9 @@
assertEquals(1, bounds.size());
assertEndBound(get(bounds, 0), false, value1);
- slice = newMultiSlice(cfMetaData, 0, Bound.START, false, value1);
- Restriction slice2 = newMultiSlice(cfMetaData, 0, Bound.END, false, value2);
- restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ slice = newMultiSlice(tableMetadata, 0, Bound.START, false, value1);
+ Restriction slice2 = newMultiSlice(tableMetadata, 0, Bound.END, false, value2);
+ restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(slice).mergeWith(slice2);
bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
@@ -553,9 +553,9 @@
assertEquals(1, bounds.size());
assertEndBound(get(bounds, 0), false, value2);
- slice = newMultiSlice(cfMetaData, 0, Bound.START, true, value1);
- slice2 = newMultiSlice(cfMetaData, 0, Bound.END, true, value2);
- restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ slice = newMultiSlice(tableMetadata, 0, Bound.START, true, value1);
+ slice2 = newMultiSlice(tableMetadata, 0, Bound.END, true, value2);
+ restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(slice).mergeWith(slice2);
bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
@@ -574,13 +574,13 @@
@Test
public void testBoundsAsClusteringWithMultiSliceRestrictionsWithOneDescendingClusteringColumn()
{
- CFMetaData cfMetaData = newCFMetaData(Sort.DESC);
+ TableMetadata tableMetadata = newTableMetadata(Sort.DESC);
ByteBuffer value1 = ByteBufferUtil.bytes(1);
ByteBuffer value2 = ByteBufferUtil.bytes(2);
- Restriction slice = newMultiSlice(cfMetaData, 0, Bound.START, false, value1);
- ClusteringColumnRestrictions restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ Restriction slice = newMultiSlice(tableMetadata, 0, Bound.START, false, value1);
+ ClusteringColumnRestrictions restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(slice);
SortedSet<ClusteringBound> bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
@@ -591,8 +591,8 @@
assertEquals(1, bounds.size());
assertEndBound(get(bounds, 0), false, value1);
- slice = newMultiSlice(cfMetaData, 0, Bound.START, true, value1);
- restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ slice = newMultiSlice(tableMetadata, 0, Bound.START, true, value1);
+ restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(slice);
bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
@@ -603,8 +603,8 @@
assertEquals(1, bounds.size());
assertEndBound(get(bounds, 0), true, value1);
- slice = newMultiSlice(cfMetaData, 0, Bound.END, true, value1);
- restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ slice = newMultiSlice(tableMetadata, 0, Bound.END, true, value1);
+ restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(slice);
bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
@@ -615,8 +615,8 @@
assertEquals(1, bounds.size());
assertEmptyEnd(get(bounds, 0));
- slice = newMultiSlice(cfMetaData, 0, Bound.END, false, value1);
- restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ slice = newMultiSlice(tableMetadata, 0, Bound.END, false, value1);
+ restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(slice);
bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
@@ -627,9 +627,9 @@
assertEquals(1, bounds.size());
assertEmptyEnd(get(bounds, 0));
- slice = newMultiSlice(cfMetaData, 0, Bound.START, false, value1);
- Restriction slice2 = newMultiSlice(cfMetaData, 0, Bound.END, false, value2);
- restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ slice = newMultiSlice(tableMetadata, 0, Bound.START, false, value1);
+ Restriction slice2 = newMultiSlice(tableMetadata, 0, Bound.END, false, value2);
+ restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(slice).mergeWith(slice2);
bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
@@ -640,9 +640,9 @@
assertEquals(1, bounds.size());
assertEndBound(get(bounds, 0), false, value1);
- slice = newMultiSlice(cfMetaData, 0, Bound.START, true, value1);
- slice2 = newMultiSlice(cfMetaData, 0, Bound.END, true, value2);
- restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ slice = newMultiSlice(tableMetadata, 0, Bound.START, true, value1);
+ slice2 = newMultiSlice(tableMetadata, 0, Bound.END, true, value2);
+ restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(slice).mergeWith(slice2);
bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
@@ -660,14 +660,14 @@
@Test
public void testBoundsAsClusteringWithMultiSliceRestrictionsWithTwoClusteringColumn()
{
- CFMetaData cfMetaData = newCFMetaData(Sort.ASC, Sort.ASC);
+ TableMetadata tableMetadata = newTableMetadata(Sort.ASC, Sort.ASC);
ByteBuffer value1 = ByteBufferUtil.bytes(1);
ByteBuffer value2 = ByteBufferUtil.bytes(2);
// (clustering_0, clustering1) > (1, 2)
- Restriction slice = newMultiSlice(cfMetaData, 0, Bound.START, false, value1, value2);
- ClusteringColumnRestrictions restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ Restriction slice = newMultiSlice(tableMetadata, 0, Bound.START, false, value1, value2);
+ ClusteringColumnRestrictions restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(slice);
SortedSet<ClusteringBound> bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
@@ -679,8 +679,8 @@
assertEmptyEnd(get(bounds, 0));
// (clustering_0, clustering1) >= (1, 2)
- slice = newMultiSlice(cfMetaData, 0, Bound.START, true, value1, value2);
- restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ slice = newMultiSlice(tableMetadata, 0, Bound.START, true, value1, value2);
+ restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(slice);
bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
@@ -692,8 +692,8 @@
assertEmptyEnd(get(bounds, 0));
// (clustering_0, clustering1) <= (1, 2)
- slice = newMultiSlice(cfMetaData, 0, Bound.END, true, value1, value2);
- restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ slice = newMultiSlice(tableMetadata, 0, Bound.END, true, value1, value2);
+ restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(slice);
bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
@@ -705,8 +705,8 @@
assertEndBound(get(bounds, 0), true, value1, value2);
// (clustering_0, clustering1) < (1, 2)
- slice = newMultiSlice(cfMetaData, 0, Bound.END, false, value1, value2);
- restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ slice = newMultiSlice(tableMetadata, 0, Bound.END, false, value1, value2);
+ restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(slice);
bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
@@ -718,9 +718,9 @@
assertEndBound(get(bounds, 0), false, value1, value2);
// (clustering_0, clustering1) > (1, 2) AND (clustering_0) < (2)
- slice = newMultiSlice(cfMetaData, 0, Bound.START, false, value1, value2);
- Restriction slice2 = newMultiSlice(cfMetaData, 0, Bound.END, false, value2);
- restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ slice = newMultiSlice(tableMetadata, 0, Bound.START, false, value1, value2);
+ Restriction slice2 = newMultiSlice(tableMetadata, 0, Bound.END, false, value2);
+ restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(slice).mergeWith(slice2);
bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
@@ -732,9 +732,9 @@
assertEndBound(get(bounds, 0), false, value2);
// (clustering_0, clustering1) >= (1, 2) AND (clustering_0, clustering1) <= (2, 1)
- slice = newMultiSlice(cfMetaData, 0, Bound.START, true, value1, value2);
- slice2 = newMultiSlice(cfMetaData, 0, Bound.END, true, value2, value1);
- restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ slice = newMultiSlice(tableMetadata, 0, Bound.START, true, value1, value2);
+ slice2 = newMultiSlice(tableMetadata, 0, Bound.END, true, value2, value1);
+ restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(slice).mergeWith(slice2);
bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
@@ -752,14 +752,14 @@
@Test
public void testBoundsAsClusteringWithMultiSliceRestrictionsWithTwoDescendingClusteringColumns()
{
- CFMetaData cfMetaData = newCFMetaData(Sort.DESC, Sort.DESC);
+ TableMetadata tableMetadata = newTableMetadata(Sort.DESC, Sort.DESC);
ByteBuffer value1 = ByteBufferUtil.bytes(1);
ByteBuffer value2 = ByteBufferUtil.bytes(2);
// (clustering_0, clustering1) > (1, 2)
- Restriction slice = newMultiSlice(cfMetaData, 0, Bound.START, false, value1, value2);
- ClusteringColumnRestrictions restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ Restriction slice = newMultiSlice(tableMetadata, 0, Bound.START, false, value1, value2);
+ ClusteringColumnRestrictions restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(slice);
SortedSet<ClusteringBound> bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
@@ -771,8 +771,8 @@
assertEndBound(get(bounds, 0), false, value1, value2);
// (clustering_0, clustering1) >= (1, 2)
- slice = newMultiSlice(cfMetaData, 0, Bound.START, true, value1, value2);
- restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ slice = newMultiSlice(tableMetadata, 0, Bound.START, true, value1, value2);
+ restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(slice);
bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
@@ -784,8 +784,8 @@
assertEndBound(get(bounds, 0), true, value1, value2);
// (clustering_0, clustering1) <= (1, 2)
- slice = newMultiSlice(cfMetaData, 0, Bound.END, true, value1, value2);
- restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ slice = newMultiSlice(tableMetadata, 0, Bound.END, true, value1, value2);
+ restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(slice);
bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
@@ -797,8 +797,8 @@
assertEmptyEnd(get(bounds, 0));
// (clustering_0, clustering1) < (1, 2)
- slice = newMultiSlice(cfMetaData, 0, Bound.END, false, value1, value2);
- restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ slice = newMultiSlice(tableMetadata, 0, Bound.END, false, value1, value2);
+ restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(slice);
bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
@@ -811,9 +811,9 @@
// (clustering_0, clustering1) > (1, 2) AND (clustering_0) < (2)
- slice = newMultiSlice(cfMetaData, 0, Bound.START, false, value1, value2);
- Restriction slice2 = newMultiSlice(cfMetaData, 0, Bound.END, false, value2);
- restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ slice = newMultiSlice(tableMetadata, 0, Bound.START, false, value1, value2);
+ Restriction slice2 = newMultiSlice(tableMetadata, 0, Bound.END, false, value2);
+ restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(slice).mergeWith(slice2);
bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
@@ -825,9 +825,9 @@
assertEndBound(get(bounds, 0), false, value1, value2);
// (clustering_0, clustering1) >= (1, 2) AND (clustering_0, clustering1) <= (2, 1)
- slice = newMultiSlice(cfMetaData, 0, Bound.START, true, value1, value2);
- slice2 = newMultiSlice(cfMetaData, 0, Bound.END, true, value2, value1);
- restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ slice = newMultiSlice(tableMetadata, 0, Bound.START, true, value1, value2);
+ slice2 = newMultiSlice(tableMetadata, 0, Bound.END, true, value2, value1);
+ restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(slice).mergeWith(slice2);
bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
@@ -846,14 +846,14 @@
@Test
public void testBoundsAsClusteringWithMultiSliceRestrictionsWithOneDescendingAndOneAscendingClusteringColumns()
{
- CFMetaData cfMetaData = newCFMetaData(Sort.DESC, Sort.ASC);
+ TableMetadata tableMetadata = newTableMetadata(Sort.DESC, Sort.ASC);
ByteBuffer value1 = ByteBufferUtil.bytes(1);
ByteBuffer value2 = ByteBufferUtil.bytes(2);
// (clustering_0, clustering1) > (1, 2)
- Restriction slice = newMultiSlice(cfMetaData, 0, Bound.START, false, value1, value2);
- ClusteringColumnRestrictions restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ Restriction slice = newMultiSlice(tableMetadata, 0, Bound.START, false, value1, value2);
+ ClusteringColumnRestrictions restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(slice);
SortedSet<ClusteringBound> bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
@@ -867,8 +867,8 @@
assertEndBound(get(bounds, 1), true, value1);
// (clustering_0, clustering1) >= (1, 2)
- slice = newMultiSlice(cfMetaData, 0, Bound.START, true, value1, value2);
- restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ slice = newMultiSlice(tableMetadata, 0, Bound.START, true, value1, value2);
+ restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(slice);
bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
@@ -882,8 +882,8 @@
assertEndBound(get(bounds, 1), true, value1);
// (clustering_0, clustering1) <= (1, 2)
- slice = newMultiSlice(cfMetaData, 0, Bound.END, true, value1, value2);
- restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ slice = newMultiSlice(tableMetadata, 0, Bound.END, true, value1, value2);
+ restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(slice);
bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
@@ -897,8 +897,8 @@
assertEmptyEnd(get(bounds, 1));
// (clustering_0, clustering1) < (1, 2)
- slice = newMultiSlice(cfMetaData, 0, Bound.END, false, value1, value2);
- restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ slice = newMultiSlice(tableMetadata, 0, Bound.END, false, value1, value2);
+ restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(slice);
bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
@@ -912,9 +912,9 @@
assertEmptyEnd(get(bounds, 1));
// (clustering_0, clustering1) > (1, 2) AND (clustering_0) < (2)
- slice = newMultiSlice(cfMetaData, 0, Bound.START, false, value1, value2);
- Restriction slice2 = newMultiSlice(cfMetaData, 0, Bound.END, false, value2);
- restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ slice = newMultiSlice(tableMetadata, 0, Bound.START, false, value1, value2);
+ Restriction slice2 = newMultiSlice(tableMetadata, 0, Bound.END, false, value2);
+ restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(slice).mergeWith(slice2);
bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
@@ -928,9 +928,9 @@
assertEndBound(get(bounds, 1), true, value1);
// (clustering_0) > (1) AND (clustering_0, clustering1) < (2, 1)
- slice = newMultiSlice(cfMetaData, 0, Bound.START, false, value1);
- slice2 = newMultiSlice(cfMetaData, 0, Bound.END, false, value2, value1);
- restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ slice = newMultiSlice(tableMetadata, 0, Bound.START, false, value1);
+ slice2 = newMultiSlice(tableMetadata, 0, Bound.END, false, value2, value1);
+ restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(slice).mergeWith(slice2);
bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
@@ -944,9 +944,9 @@
assertEndBound(get(bounds, 1), false, value1);
// (clustering_0, clustering1) >= (1, 2) AND (clustering_0, clustering1) <= (2, 1)
- slice = newMultiSlice(cfMetaData, 0, Bound.START, true, value1, value2);
- slice2 = newMultiSlice(cfMetaData, 0, Bound.END, true, value2, value1);
- restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ slice = newMultiSlice(tableMetadata, 0, Bound.START, true, value1, value2);
+ slice2 = newMultiSlice(tableMetadata, 0, Bound.END, true, value2, value1);
+ restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(slice).mergeWith(slice2);
bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
@@ -969,14 +969,14 @@
@Test
public void testBoundsAsClusteringWithMultiSliceRestrictionsWithOneAscendingAndOneDescendingClusteringColumns()
{
- CFMetaData cfMetaData = newCFMetaData(Sort.ASC, Sort.DESC);
+ TableMetadata tableMetadata = newTableMetadata(Sort.ASC, Sort.DESC);
ByteBuffer value1 = ByteBufferUtil.bytes(1);
ByteBuffer value2 = ByteBufferUtil.bytes(2);
// (clustering_0, clustering1) > (1, 2)
- Restriction slice = newMultiSlice(cfMetaData, 0, Bound.START, false, value1, value2);
- ClusteringColumnRestrictions restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ Restriction slice = newMultiSlice(tableMetadata, 0, Bound.START, false, value1, value2);
+ ClusteringColumnRestrictions restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(slice);
SortedSet<ClusteringBound> bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
@@ -990,8 +990,8 @@
assertEmptyEnd(get(bounds, 1));
// (clustering_0, clustering1) >= (1, 2)
- slice = newMultiSlice(cfMetaData, 0, Bound.START, true, value1, value2);
- restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ slice = newMultiSlice(tableMetadata, 0, Bound.START, true, value1, value2);
+ restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(slice);
bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
@@ -1005,8 +1005,8 @@
assertEmptyEnd(get(bounds, 1));
// (clustering_0, clustering1) <= (1, 2)
- slice = newMultiSlice(cfMetaData, 0, Bound.END, true, value1, value2);
- restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ slice = newMultiSlice(tableMetadata, 0, Bound.END, true, value1, value2);
+ restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(slice);
bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
@@ -1020,8 +1020,8 @@
assertEndBound(get(bounds, 1), true, value1);
// (clustering_0, clustering1) < (1, 2)
- slice = newMultiSlice(cfMetaData, 0, Bound.END, false, value1, value2);
- restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ slice = newMultiSlice(tableMetadata, 0, Bound.END, false, value1, value2);
+ restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(slice);
bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
@@ -1035,9 +1035,9 @@
assertEndBound(get(bounds, 1), true, value1);
// (clustering_0, clustering1) > (1, 2) AND (clustering_0) < (2)
- slice = newMultiSlice(cfMetaData, 0, Bound.START, false, value1, value2);
- Restriction slice2 = newMultiSlice(cfMetaData, 0, Bound.END, false, value2);
- restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ slice = newMultiSlice(tableMetadata, 0, Bound.START, false, value1, value2);
+ Restriction slice2 = newMultiSlice(tableMetadata, 0, Bound.END, false, value2);
+ restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(slice).mergeWith(slice2);
bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
@@ -1051,9 +1051,9 @@
assertEndBound(get(bounds, 1), false, value2);
// (clustering_0, clustering1) >= (1, 2) AND (clustering_0, clustering1) <= (2, 1)
- slice = newMultiSlice(cfMetaData, 0, Bound.START, true, value1, value2);
- slice2 = newMultiSlice(cfMetaData, 0, Bound.END, true, value2, value1);
- restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ slice = newMultiSlice(tableMetadata, 0, Bound.START, true, value1, value2);
+ slice2 = newMultiSlice(tableMetadata, 0, Bound.END, true, value2, value1);
+ restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(slice).mergeWith(slice2);
bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
@@ -1076,7 +1076,7 @@
@Test
public void testBoundsAsClusteringWithMultiSliceRestrictionsWithTwoAscendingAndTwoDescendingClusteringColumns()
{
- CFMetaData cfMetaData = newCFMetaData(Sort.ASC, Sort.ASC, Sort.DESC, Sort.DESC);
+ TableMetadata tableMetadata = newTableMetadata(Sort.ASC, Sort.ASC, Sort.DESC, Sort.DESC);
ByteBuffer value1 = ByteBufferUtil.bytes(1);
ByteBuffer value2 = ByteBufferUtil.bytes(2);
@@ -1084,8 +1084,8 @@
ByteBuffer value4 = ByteBufferUtil.bytes(4);
// (clustering_0, clustering1, clustering_2, clustering_3) > (1, 2, 3, 4)
- Restriction slice = newMultiSlice(cfMetaData, 0, Bound.START, false, value1, value2, value3, value4);
- ClusteringColumnRestrictions restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ Restriction slice = newMultiSlice(tableMetadata, 0, Bound.START, false, value1, value2, value3, value4);
+ ClusteringColumnRestrictions restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(slice);
SortedSet<ClusteringBound> bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
@@ -1099,9 +1099,9 @@
assertEmptyEnd(get(bounds, 1));
// clustering_0 = 1 AND (clustering_1, clustering_2, clustering_3) > (2, 3, 4)
- Restriction eq = newSingleEq(cfMetaData, 0, value1);
- slice = newMultiSlice(cfMetaData, 1, Bound.START, false, value2, value3, value4);
- restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ Restriction eq = newSingleEq(tableMetadata, 0, value1);
+ slice = newMultiSlice(tableMetadata, 1, Bound.START, false, value2, value3, value4);
+ restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(slice);
restrictions = restrictions.mergeWith(eq);
@@ -1116,9 +1116,9 @@
assertEndBound(get(bounds, 1), true, value1);
// clustering_0 IN (1, 2) AND (clustering_1, clustering_2, clustering_3) > (2, 3, 4)
- Restriction in = newSingleIN(cfMetaData, 0, value1, value2);
- slice = newMultiSlice(cfMetaData, 1, Bound.START, false, value2, value3, value4);
- restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ Restriction in = newSingleIN(tableMetadata, 0, value1, value2);
+ slice = newMultiSlice(tableMetadata, 1, Bound.START, false, value2, value3, value4);
+ restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(slice);
restrictions = restrictions.mergeWith(in);
@@ -1137,8 +1137,8 @@
assertEndBound(get(bounds, 3), true, value2);
// (clustering_0, clustering1) >= (1, 2)
- slice = newMultiSlice(cfMetaData, 0, Bound.START, true, value1, value2);
- restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ slice = newMultiSlice(tableMetadata, 0, Bound.START, true, value1, value2);
+ restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(slice);
bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
@@ -1150,8 +1150,8 @@
assertEmptyEnd(get(bounds, 0));
// (clustering_0, clustering1, clustering_2, clustering_3) >= (1, 2, 3, 4)
- slice = newMultiSlice(cfMetaData, 0, Bound.START, true, value1, value2, value3, value4);
- restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ slice = newMultiSlice(tableMetadata, 0, Bound.START, true, value1, value2, value3, value4);
+ restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(slice);
bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
@@ -1165,8 +1165,8 @@
assertEmptyEnd(get(bounds, 1));
// (clustering_0, clustering1, clustering_2, clustering_3) <= (1, 2, 3, 4)
- slice = newMultiSlice(cfMetaData, 0, Bound.END, true, value1, value2, value3, value4);
- restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ slice = newMultiSlice(tableMetadata, 0, Bound.END, true, value1, value2, value3, value4);
+ restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(slice);
bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
@@ -1180,8 +1180,8 @@
assertEndBound(get(bounds, 1), true, value1, value2);
// (clustering_0, clustering1, clustering_2, clustering_3) < (1, 2, 3, 4)
- slice = newMultiSlice(cfMetaData, 0, Bound.END, false, value1, value2, value3, value4);
- restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ slice = newMultiSlice(tableMetadata, 0, Bound.END, false, value1, value2, value3, value4);
+ restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(slice);
bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
@@ -1195,9 +1195,9 @@
assertEndBound(get(bounds, 1), true, value1, value2);
// (clustering_0, clustering1, clustering_2, clustering_3) > (1, 2, 3, 4) AND (clustering_0, clustering_1) < (2, 3)
- slice = newMultiSlice(cfMetaData, 0, Bound.START, false, value1, value2, value3, value4);
- Restriction slice2 = newMultiSlice(cfMetaData, 0, Bound.END, false, value2, value3);
- restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ slice = newMultiSlice(tableMetadata, 0, Bound.START, false, value1, value2, value3, value4);
+ Restriction slice2 = newMultiSlice(tableMetadata, 0, Bound.END, false, value2, value3);
+ restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(slice).mergeWith(slice2);
bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
@@ -1211,9 +1211,9 @@
assertEndBound(get(bounds, 1), false, value2, value3);
// (clustering_0, clustering1, clustering_2, clustering_3) >= (1, 2, 3, 4) AND (clustering_0, clustering1, clustering_2, clustering_3) <= (4, 3, 2, 1)
- slice = newMultiSlice(cfMetaData, 0, Bound.START, true, value1, value2, value3, value4);
- slice2 = newMultiSlice(cfMetaData, 0, Bound.END, true, value4, value3, value2, value1);
- restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ slice = newMultiSlice(tableMetadata, 0, Bound.START, true, value1, value2, value3, value4);
+ slice2 = newMultiSlice(tableMetadata, 0, Bound.END, true, value4, value3, value2, value1);
+ restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(slice).mergeWith(slice2);
bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
@@ -1236,7 +1236,7 @@
@Test
public void testBoundsAsClusteringWithMultiSliceRestrictionsWithAscendingDescendingColumnMix()
{
- CFMetaData cfMetaData = newCFMetaData(Sort.ASC, Sort.DESC, Sort.ASC, Sort.DESC);
+ TableMetadata tableMetadata = newTableMetadata(Sort.ASC, Sort.DESC, Sort.ASC, Sort.DESC);
ByteBuffer value1 = ByteBufferUtil.bytes(1);
ByteBuffer value2 = ByteBufferUtil.bytes(2);
@@ -1244,8 +1244,8 @@
ByteBuffer value4 = ByteBufferUtil.bytes(4);
// (clustering_0, clustering1, clustering_2, clustering_3) > (1, 2, 3, 4)
- Restriction slice = newMultiSlice(cfMetaData, 0, Bound.START, false, value1, value2, value3, value4);
- ClusteringColumnRestrictions restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ Restriction slice = newMultiSlice(tableMetadata, 0, Bound.START, false, value1, value2, value3, value4);
+ ClusteringColumnRestrictions restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(slice);
SortedSet<ClusteringBound> bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
@@ -1264,9 +1264,9 @@
assertEmptyEnd(get(bounds, 3));
// clustering_0 = 1 AND (clustering_1, clustering_2, clustering_3) > (2, 3, 4)
- Restriction eq = newSingleEq(cfMetaData, 0, value1);
- slice = newMultiSlice(cfMetaData, 1, Bound.START, false, value2, value3, value4);
- restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ Restriction eq = newSingleEq(tableMetadata, 0, value1);
+ slice = newMultiSlice(tableMetadata, 1, Bound.START, false, value2, value3, value4);
+ restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(slice);
restrictions = restrictions.mergeWith(eq);
@@ -1283,8 +1283,8 @@
assertEndBound(get(bounds, 2), true, value1, value2);
// (clustering_0, clustering1) >= (1, 2)
- slice = newMultiSlice(cfMetaData, 0, Bound.START, true, value1, value2);
- restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ slice = newMultiSlice(tableMetadata, 0, Bound.START, true, value1, value2);
+ restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(slice);
bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
@@ -1298,8 +1298,8 @@
assertEmptyEnd(get(bounds, 1));
// (clustering_0, clustering1, clustering_2, clustering_3) >= (1, 2, 3, 4)
- slice = newMultiSlice(cfMetaData, 0, Bound.START, true, value1, value2, value3, value4);
- restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ slice = newMultiSlice(tableMetadata, 0, Bound.START, true, value1, value2, value3, value4);
+ restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(slice);
bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
@@ -1317,8 +1317,8 @@
assertEmptyEnd(get(bounds, 3));
// (clustering_0, clustering1, clustering_2, clustering_3) <= (1, 2, 3, 4)
- slice = newMultiSlice(cfMetaData, 0, Bound.END, true, value1, value2, value3, value4);
- restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ slice = newMultiSlice(tableMetadata, 0, Bound.END, true, value1, value2, value3, value4);
+ restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(slice);
bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
@@ -1336,8 +1336,8 @@
assertEndBound(get(bounds, 3), true, value1);
// (clustering_0, clustering1, clustering_2, clustering_3) < (1, 2, 3, 4)
- slice = newMultiSlice(cfMetaData, 0, Bound.END, false, value1, value2, value3, value4);
- restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ slice = newMultiSlice(tableMetadata, 0, Bound.END, false, value1, value2, value3, value4);
+ restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(slice);
bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
@@ -1355,9 +1355,9 @@
assertEndBound(get(bounds, 3), true, value1);
// (clustering_0, clustering1, clustering_2, clustering_3) > (1, 2, 3, 4) AND (clustering_0, clustering_1) < (2, 3)
- slice = newMultiSlice(cfMetaData, 0, Bound.START, false, value1, value2, value3, value4);
- Restriction slice2 = newMultiSlice(cfMetaData, 0, Bound.END, false, value2, value3);
- restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ slice = newMultiSlice(tableMetadata, 0, Bound.START, false, value1, value2, value3, value4);
+ Restriction slice2 = newMultiSlice(tableMetadata, 0, Bound.END, false, value2, value3);
+ restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(slice).mergeWith(slice2);
bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
@@ -1377,9 +1377,9 @@
assertEndBound(get(bounds, 4), true, value2);
// (clustering_0, clustering1, clustering_2, clustering_3) >= (1, 2, 3, 4) AND (clustering_0, clustering1, clustering_2, clustering_3) <= (4, 3, 2, 1)
- slice = newMultiSlice(cfMetaData, 0, Bound.START, true, value1, value2, value3, value4);
- slice2 = newMultiSlice(cfMetaData, 0, Bound.END, true, value4, value3, value2, value1);
- restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ slice = newMultiSlice(tableMetadata, 0, Bound.START, true, value1, value2, value3, value4);
+ slice2 = newMultiSlice(tableMetadata, 0, Bound.END, true, value4, value3, value2, value1);
+ restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(slice).mergeWith(slice2);
bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
@@ -1409,7 +1409,7 @@
@Test
public void testBoundsAsClusteringWithSingleEqAndMultiEqRestrictions()
{
- CFMetaData cfMetaData = newCFMetaData(Sort.ASC, Sort.ASC, Sort.ASC, Sort.ASC);
+ TableMetadata tableMetadata = newTableMetadata(Sort.ASC, Sort.ASC, Sort.ASC, Sort.ASC);
ByteBuffer value1 = ByteBufferUtil.bytes(1);
ByteBuffer value2 = ByteBufferUtil.bytes(2);
@@ -1417,9 +1417,9 @@
ByteBuffer value4 = ByteBufferUtil.bytes(4);
// clustering_0 = 1 AND (clustering_1, clustering_2) = (2, 3)
- Restriction singleEq = newSingleEq(cfMetaData, 0, value1);
- Restriction multiEq = newMultiEq(cfMetaData, 1, value2, value3);
- ClusteringColumnRestrictions restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ Restriction singleEq = newSingleEq(tableMetadata, 0, value1);
+ Restriction multiEq = newMultiEq(tableMetadata, 1, value2, value3);
+ ClusteringColumnRestrictions restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(singleEq).mergeWith(multiEq);
SortedSet<ClusteringBound> bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
@@ -1431,10 +1431,10 @@
assertEndBound(get(bounds, 0), true, value1, value2, value3);
// clustering_0 = 1 AND clustering_1 = 2 AND (clustering_2, clustering_3) = (3, 4)
- singleEq = newSingleEq(cfMetaData, 0, value1);
- Restriction singleEq2 = newSingleEq(cfMetaData, 1, value2);
- multiEq = newMultiEq(cfMetaData, 2, value3, value4);
- restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ singleEq = newSingleEq(tableMetadata, 0, value1);
+ Restriction singleEq2 = newSingleEq(tableMetadata, 1, value2);
+ multiEq = newMultiEq(tableMetadata, 2, value3, value4);
+ restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(singleEq).mergeWith(singleEq2).mergeWith(multiEq);
bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
@@ -1446,9 +1446,9 @@
assertEndBound(get(bounds, 0), true, value1, value2, value3, value4);
// (clustering_0, clustering_1) = (1, 2) AND clustering_2 = 3
- singleEq = newSingleEq(cfMetaData, 2, value3);
- multiEq = newMultiEq(cfMetaData, 0, value1, value2);
- restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ singleEq = newSingleEq(tableMetadata, 2, value3);
+ multiEq = newMultiEq(tableMetadata, 0, value1, value2);
+ restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(singleEq).mergeWith(multiEq);
bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
@@ -1460,10 +1460,10 @@
assertEndBound(get(bounds, 0), true, value1, value2, value3);
// clustering_0 = 1 AND (clustering_1, clustering_2) = (2, 3) AND clustering_3 = 4
- singleEq = newSingleEq(cfMetaData, 0, value1);
- singleEq2 = newSingleEq(cfMetaData, 3, value4);
- multiEq = newMultiEq(cfMetaData, 1, value2, value3);
- restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ singleEq = newSingleEq(tableMetadata, 0, value1);
+ singleEq2 = newSingleEq(tableMetadata, 3, value4);
+ multiEq = newMultiEq(tableMetadata, 1, value2, value3);
+ restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(singleEq).mergeWith(multiEq).mergeWith(singleEq2);
bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
@@ -1481,7 +1481,7 @@
@Test
public void testBoundsAsClusteringWithSingleEqAndMultiINRestrictions()
{
- CFMetaData cfMetaData = newCFMetaData(Sort.ASC, Sort.ASC, Sort.ASC, Sort.ASC);
+ TableMetadata tableMetadata = newTableMetadata(Sort.ASC, Sort.ASC, Sort.ASC, Sort.ASC);
ByteBuffer value1 = ByteBufferUtil.bytes(1);
ByteBuffer value2 = ByteBufferUtil.bytes(2);
@@ -1490,9 +1490,9 @@
ByteBuffer value5 = ByteBufferUtil.bytes(5);
// clustering_0 = 1 AND (clustering_1, clustering_2) IN ((2, 3), (4, 5))
- Restriction singleEq = newSingleEq(cfMetaData, 0, value1);
- Restriction multiIN = newMultiIN(cfMetaData, 1, asList(value2, value3), asList(value4, value5));
- ClusteringColumnRestrictions restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ Restriction singleEq = newSingleEq(tableMetadata, 0, value1);
+ Restriction multiIN = newMultiIN(tableMetadata, 1, asList(value2, value3), asList(value4, value5));
+ ClusteringColumnRestrictions restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(singleEq).mergeWith(multiIN);
SortedSet<ClusteringBound> bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
@@ -1506,9 +1506,9 @@
assertEndBound(get(bounds, 1), true, value1, value4, value5);
// clustering_0 = 1 AND (clustering_1, clustering_2) IN ((2, 3))
- singleEq = newSingleEq(cfMetaData, 0, value1);
- multiIN = newMultiIN(cfMetaData, 1, asList(value2, value3));
- restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ singleEq = newSingleEq(tableMetadata, 0, value1);
+ multiIN = newMultiIN(tableMetadata, 1, asList(value2, value3));
+ restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(multiIN).mergeWith(singleEq);
bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
@@ -1520,10 +1520,10 @@
assertEndBound(get(bounds, 0), true, value1, value2, value3);
// clustering_0 = 1 AND clustering_1 = 5 AND (clustering_2, clustering_3) IN ((2, 3), (4, 5))
- singleEq = newSingleEq(cfMetaData, 0, value1);
- Restriction singleEq2 = newSingleEq(cfMetaData, 1, value5);
- multiIN = newMultiIN(cfMetaData, 2, asList(value2, value3), asList(value4, value5));
- restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ singleEq = newSingleEq(tableMetadata, 0, value1);
+ Restriction singleEq2 = newSingleEq(tableMetadata, 1, value5);
+ multiIN = newMultiIN(tableMetadata, 2, asList(value2, value3), asList(value4, value5));
+ restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(singleEq).mergeWith(multiIN).mergeWith(singleEq2);
bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
@@ -1544,7 +1544,7 @@
@Test
public void testBoundsAsClusteringWithSingleEqAndSliceRestrictions()
{
- CFMetaData cfMetaData = newCFMetaData(Sort.ASC, Sort.ASC, Sort.ASC);
+ TableMetadata tableMetadata = newTableMetadata(Sort.ASC, Sort.ASC, Sort.ASC);
ByteBuffer value1 = ByteBufferUtil.bytes(1);
ByteBuffer value2 = ByteBufferUtil.bytes(2);
@@ -1553,9 +1553,9 @@
ByteBuffer value5 = ByteBufferUtil.bytes(5);
// clustering_0 = 1 AND (clustering_1, clustering_2) > (2, 3)
- Restriction singleEq = newSingleEq(cfMetaData, 0, value1);
- Restriction multiSlice = newMultiSlice(cfMetaData, 1, Bound.START, false, value2, value3);
- ClusteringColumnRestrictions restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ Restriction singleEq = newSingleEq(tableMetadata, 0, value1);
+ Restriction multiSlice = newMultiSlice(tableMetadata, 1, Bound.START, false, value2, value3);
+ ClusteringColumnRestrictions restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(singleEq).mergeWith(multiSlice);
SortedSet<ClusteringBound> bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
@@ -1567,10 +1567,10 @@
assertEndBound(get(bounds, 0), true, value1);
// clustering_0 = 1 AND (clustering_1, clustering_2) > (2, 3) AND (clustering_1) < (4)
- singleEq = newSingleEq(cfMetaData, 0, value1);
- multiSlice = newMultiSlice(cfMetaData, 1, Bound.START, false, value2, value3);
- Restriction multiSlice2 = newMultiSlice(cfMetaData, 1, Bound.END, false, value4);
- restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ singleEq = newSingleEq(tableMetadata, 0, value1);
+ multiSlice = newMultiSlice(tableMetadata, 1, Bound.START, false, value2, value3);
+ Restriction multiSlice2 = newMultiSlice(tableMetadata, 1, Bound.END, false, value4);
+ restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(multiSlice2).mergeWith(singleEq).mergeWith(multiSlice);
bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
@@ -1582,10 +1582,10 @@
assertEndBound(get(bounds, 0), false, value1, value4);
// clustering_0 = 1 AND (clustering_1, clustering_2) => (2, 3) AND (clustering_1, clustering_2) <= (4, 5)
- singleEq = newSingleEq(cfMetaData, 0, value1);
- multiSlice = newMultiSlice(cfMetaData, 1, Bound.START, true, value2, value3);
- multiSlice2 = newMultiSlice(cfMetaData, 1, Bound.END, true, value4, value5);
- restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ singleEq = newSingleEq(tableMetadata, 0, value1);
+ multiSlice = newMultiSlice(tableMetadata, 1, Bound.START, true, value2, value3);
+ multiSlice2 = newMultiSlice(tableMetadata, 1, Bound.END, true, value4, value5);
+ restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(multiSlice2).mergeWith(singleEq).mergeWith(multiSlice);
bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
@@ -1604,16 +1604,16 @@
@Test
public void testBoundsAsClusteringWithMultiEqAndSingleSliceRestrictions()
{
- CFMetaData cfMetaData = newCFMetaData(Sort.ASC, Sort.ASC, Sort.ASC);
+ TableMetadata tableMetadata = newTableMetadata(Sort.ASC, Sort.ASC, Sort.ASC);
ByteBuffer value1 = ByteBufferUtil.bytes(1);
ByteBuffer value2 = ByteBufferUtil.bytes(2);
ByteBuffer value3 = ByteBufferUtil.bytes(3);
// (clustering_0, clustering_1) = (1, 2) AND clustering_2 > 3
- Restriction multiEq = newMultiEq(cfMetaData, 0, value1, value2);
- Restriction singleSlice = newSingleSlice(cfMetaData, 2, Bound.START, false, value3);
- ClusteringColumnRestrictions restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ Restriction multiEq = newMultiEq(tableMetadata, 0, value1, value2);
+ Restriction singleSlice = newSingleSlice(tableMetadata, 2, Bound.START, false, value3);
+ ClusteringColumnRestrictions restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(multiEq).mergeWith(singleSlice);
SortedSet<ClusteringBound> bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
@@ -1628,7 +1628,7 @@
@Test
public void testBoundsAsClusteringWithSeveralMultiColumnRestrictions()
{
- CFMetaData cfMetaData = newCFMetaData(Sort.ASC, Sort.ASC, Sort.ASC, Sort.ASC);
+ TableMetadata tableMetadata = newTableMetadata(Sort.ASC, Sort.ASC, Sort.ASC, Sort.ASC);
ByteBuffer value1 = ByteBufferUtil.bytes(1);
ByteBuffer value2 = ByteBufferUtil.bytes(2);
@@ -1637,9 +1637,9 @@
ByteBuffer value5 = ByteBufferUtil.bytes(5);
// (clustering_0, clustering_1) = (1, 2) AND (clustering_2, clustering_3) > (3, 4)
- Restriction multiEq = newMultiEq(cfMetaData, 0, value1, value2);
- Restriction multiSlice = newMultiSlice(cfMetaData, 2, Bound.START, false, value3, value4);
- ClusteringColumnRestrictions restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ Restriction multiEq = newMultiEq(tableMetadata, 0, value1, value2);
+ Restriction multiSlice = newMultiSlice(tableMetadata, 2, Bound.START, false, value3, value4);
+ ClusteringColumnRestrictions restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(multiEq).mergeWith(multiSlice);
SortedSet<ClusteringBound> bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
@@ -1651,9 +1651,9 @@
assertEndBound(get(bounds, 0), true, value1, value2);
// (clustering_0, clustering_1) = (1, 2) AND (clustering_2, clustering_3) IN ((3, 4), (4, 5))
- multiEq = newMultiEq(cfMetaData, 0, value1, value2);
- Restriction multiIN = newMultiIN(cfMetaData, 2, asList(value3, value4), asList(value4, value5));
- restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ multiEq = newMultiEq(tableMetadata, 0, value1, value2);
+ Restriction multiIN = newMultiIN(tableMetadata, 2, asList(value3, value4), asList(value4, value5));
+ restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(multiEq).mergeWith(multiIN);
bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
@@ -1667,9 +1667,9 @@
assertEndBound(get(bounds, 1), true, value1, value2, value4, value5);
// (clustering_0, clustering_1) = (1, 2) AND (clustering_2, clustering_3) = (3, 4)
- multiEq = newMultiEq(cfMetaData, 0, value1, value2);
- Restriction multiEq2 = newMultiEq(cfMetaData, 2, value3, value4);
- restrictions = new ClusteringColumnRestrictions(cfMetaData);
+ multiEq = newMultiEq(tableMetadata, 0, value1, value2);
+ Restriction multiEq2 = newMultiEq(tableMetadata, 2, value3, value4);
+ restrictions = new ClusteringColumnRestrictions(tableMetadata);
restrictions = restrictions.mergeWith(multiEq).mergeWith(multiEq2);
bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
@@ -1741,21 +1741,16 @@
}
}
- /**
- * Creates a new <code>CFMetaData</code> instance.
- *
- * @param numberOfClusteringColumns the number of clustering column
- * @return a new <code>CFMetaData</code> instance
- */
- private static CFMetaData newCFMetaData(Sort... sorts)
+ private static TableMetadata newTableMetadata(Sort... sorts)
{
List<AbstractType<?>> types = new ArrayList<>();
for (Sort sort : sorts)
types.add(sort == Sort.ASC ? Int32Type.instance : ReversedType.getInstance(Int32Type.instance));
- CFMetaData.Builder builder = CFMetaData.Builder.create("keyspace", "test")
- .addPartitionKey("partition_key", Int32Type.instance);
+ TableMetadata.Builder builder =
+ TableMetadata.builder("keyspace", "test")
+ .addPartitionKeyColumn("partition_key", Int32Type.instance);
for (int i = 0; i < sorts.length; i++)
builder.addClusteringColumn("clustering_" + i, types.get(i));
@@ -1766,116 +1761,116 @@
/**
* Creates a new <code>SingleColumnRestriction.EQ</code> instance for the specified clustering column.
*
- * @param cfMetaData the column family meta data
+ * @param tableMetadata the column family meta data
* @param index the clustering column index
* @param value the equality value
* @return a new <code>SingleColumnRestriction.EQ</code> instance for the specified clustering column
*/
- private static Restriction newSingleEq(CFMetaData cfMetaData, int index, ByteBuffer value)
+ private static Restriction newSingleEq(TableMetadata tableMetadata, int index, ByteBuffer value)
{
- ColumnDefinition columnDef = getClusteringColumnDefinition(cfMetaData, index);
+ ColumnMetadata columnDef = getClusteringColumnDefinition(tableMetadata, index);
return new SingleColumnRestriction.EQRestriction(columnDef, toTerm(value));
}
/**
* Creates a new <code>MultiColumnRestriction.EQ</code> instance for the specified clustering column.
*
- * @param cfMetaData the column family meta data
- * @param index the clustering column index
- * @param value the equality value
+ * @param tableMetadata the column family meta data
+ * @param firstIndex the clustering column index
+ * @param values the equality value
* @return a new <code>MultiColumnRestriction.EQ</code> instance for the specified clustering column
*/
- private static Restriction newMultiEq(CFMetaData cfMetaData, int firstIndex, ByteBuffer... values)
+ private static Restriction newMultiEq(TableMetadata tableMetadata, int firstIndex, ByteBuffer... values)
{
- List<ColumnDefinition> columnDefinitions = new ArrayList<>();
+ List<ColumnMetadata> columnMetadatas = new ArrayList<>();
for (int i = 0; i < values.length; i++)
{
- columnDefinitions.add(getClusteringColumnDefinition(cfMetaData, firstIndex + i));
+ columnMetadatas.add(getClusteringColumnDefinition(tableMetadata, firstIndex + i));
}
- return new MultiColumnRestriction.EQRestriction(columnDefinitions, toMultiItemTerminal(values));
+ return new MultiColumnRestriction.EQRestriction(columnMetadatas, toMultiItemTerminal(values));
}
/**
* Creates a new <code>MultiColumnRestriction.IN</code> instance for the specified clustering column.
*
- * @param cfMetaData the column family meta data
+ * @param tableMetadata the column family meta data
* @param firstIndex the index of the first clustering column
* @param values the in values
* @return a new <code>MultiColumnRestriction.IN</code> instance for the specified clustering column
*/
@SafeVarargs
- private static Restriction newMultiIN(CFMetaData cfMetaData, int firstIndex, List<ByteBuffer>... values)
+ private static Restriction newMultiIN(TableMetadata tableMetadata, int firstIndex, List<ByteBuffer>... values)
{
- List<ColumnDefinition> columnDefinitions = new ArrayList<>();
+ List<ColumnMetadata> columnMetadatas = new ArrayList<>();
List<Term> terms = new ArrayList<>();
for (int i = 0; i < values.length; i++)
{
- columnDefinitions.add(getClusteringColumnDefinition(cfMetaData, firstIndex + i));
+ columnMetadatas.add(getClusteringColumnDefinition(tableMetadata, firstIndex + i));
terms.add(toMultiItemTerminal(values[i].toArray(new ByteBuffer[0])));
}
- return new MultiColumnRestriction.InRestrictionWithValues(columnDefinitions, terms);
+ return new MultiColumnRestriction.InRestrictionWithValues(columnMetadatas, terms);
}
/**
* Creates a new <code>SingleColumnRestriction.IN</code> instance for the specified clustering column.
*
- * @param cfMetaData the column family meta data
+ * @param tableMetadata the column family meta data
* @param index the clustering column index
* @param values the in values
* @return a new <code>SingleColumnRestriction.IN</code> instance for the specified clustering column
*/
- private static Restriction newSingleIN(CFMetaData cfMetaData, int index, ByteBuffer... values)
+ private static Restriction newSingleIN(TableMetadata tableMetadata, int index, ByteBuffer... values)
{
- ColumnDefinition columnDef = getClusteringColumnDefinition(cfMetaData, index);
+ ColumnMetadata columnDef = getClusteringColumnDefinition(tableMetadata, index);
return new SingleColumnRestriction.InRestrictionWithValues(columnDef, toTerms(values));
}
/**
- * Returns the clustering <code>ColumnDefinition</code> for the specified position.
+ * Returns the clustering <code>ColumnMetadata</code> for the specified position.
*
- * @param cfMetaData the column family meta data
+ * @param tableMetadata the column family meta data
* @param index the clustering column index
- * @return the clustering <code>ColumnDefinition</code> for the specified position.
+ * @return the clustering <code>ColumnMetadata</code> for the specified position.
*/
- private static ColumnDefinition getClusteringColumnDefinition(CFMetaData cfMetaData, int index)
+ private static ColumnMetadata getClusteringColumnDefinition(TableMetadata tableMetadata, int index)
{
- return cfMetaData.clusteringColumns().get(index);
+ return tableMetadata.clusteringColumns().get(index);
}
/**
* Creates a new <code>SingleColumnRestriction.Slice</code> instance for the specified clustering column.
*
- * @param cfMetaData the column family meta data
+ * @param tableMetadata the column family meta data
* @param index the clustering column index
* @param bound the slice bound
* @param inclusive <code>true</code> if the bound is inclusive
* @param value the bound value
* @return a new <code>SingleColumnRestriction.Slice</code> instance for the specified clustering column
*/
- private static Restriction newSingleSlice(CFMetaData cfMetaData, int index, Bound bound, boolean inclusive, ByteBuffer value)
+ private static Restriction newSingleSlice(TableMetadata tableMetadata, int index, Bound bound, boolean inclusive, ByteBuffer value)
{
- ColumnDefinition columnDef = getClusteringColumnDefinition(cfMetaData, index);
+ ColumnMetadata columnDef = getClusteringColumnDefinition(tableMetadata, index);
return new SingleColumnRestriction.SliceRestriction(columnDef, bound, inclusive, toTerm(value));
}
/**
* Creates a new <code>SingleColumnRestriction.Slice</code> instance for the specified clustering column.
*
- * @param cfMetaData the column family meta data
- * @param index the clustering column index
+ * @param tableMetadata the column family meta data
+ * @param firstIndex the clustering column index
* @param bound the slice bound
* @param inclusive <code>true</code> if the bound is inclusive
- * @param value the bound value
+ * @param values the bound value
* @return a new <code>SingleColumnRestriction.Slice</code> instance for the specified clustering column
*/
- private static Restriction newMultiSlice(CFMetaData cfMetaData, int firstIndex, Bound bound, boolean inclusive, ByteBuffer... values)
+ private static Restriction newMultiSlice(TableMetadata tableMetadata, int firstIndex, Bound bound, boolean inclusive, ByteBuffer... values)
{
- List<ColumnDefinition> columnDefinitions = new ArrayList<>();
+ List<ColumnMetadata> columnMetadatas = new ArrayList<>();
for (int i = 0; i < values.length; i++)
{
- columnDefinitions.add(getClusteringColumnDefinition(cfMetaData, i + firstIndex));
+ columnMetadatas.add(getClusteringColumnDefinition(tableMetadata, i + firstIndex));
}
- return new MultiColumnRestriction.SliceRestriction(columnDefinitions, bound, inclusive, toMultiItemTerminal(values));
+ return new MultiColumnRestriction.SliceRestriction(columnMetadatas, bound, inclusive, toMultiItemTerminal(values));
}
/**
diff --git a/test/unit/org/apache/cassandra/cql3/selection/SelectionColumnMappingTest.java b/test/unit/org/apache/cassandra/cql3/selection/SelectionColumnMappingTest.java
index ece2d1d..d6de5ff 100644
--- a/test/unit/org/apache/cassandra/cql3/selection/SelectionColumnMappingTest.java
+++ b/test/unit/org/apache/cassandra/cql3/selection/SelectionColumnMappingTest.java
@@ -26,9 +26,9 @@
import org.junit.BeforeClass;
import org.junit.Test;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.config.Schema;
+import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.cql3.*;
import org.apache.cassandra.cql3.statements.SelectStatement;
import org.apache.cassandra.db.marshal.*;
@@ -39,12 +39,13 @@
import org.apache.cassandra.service.QueryState;
import org.apache.cassandra.utils.ByteBufferUtil;
+import static java.util.Arrays.asList;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
public class SelectionColumnMappingTest extends CQLTester
{
- private static final ColumnDefinition NULL_DEF = null;
+ private static final ColumnMetadata NULL_DEF = null;
String tableName;
String typeName;
UserType userType;
@@ -70,7 +71,7 @@
" v1 int," +
" v2 ascii," +
" v3 frozen<" + typeName + ">)");
- userType = Schema.instance.getKSMetaData(KEYSPACE).types.get(ByteBufferUtil.bytes(typeName)).get().freeze();
+ userType = Schema.instance.getKeyspaceMetadata(KEYSPACE).types.get(ByteBufferUtil.bytes(typeName)).get().freeze();
functionName = createFunction(KEYSPACE, "int, ascii",
"CREATE FUNCTION %s (i int, a ascii) " +
"CALLED ON NULL INPUT " +
@@ -102,6 +103,14 @@
testMixedColumnTypes();
testMultipleUnaliasedSelectionOfSameColumn();
testUserDefinedAggregate();
+ testListLitteral();
+ testEmptyListLitteral();
+ testSetLitteral();
+ testEmptySetLitteral();
+ testMapLitteral();
+ testEmptyMapLitteral();
+ testUDTLitteral();
+ testTupleLitteral();
}
@Test
@@ -115,13 +124,13 @@
// we don't use verify like with the other tests because this query will produce no results
SelectStatement statement = getSelect("SELECT token(a,b) FROM %s");
verifyColumnMapping(expected, statement);
- statement.executeInternal(QueryState.forInternalCalls(), QueryOptions.DEFAULT);
+ statement.executeLocally(QueryState.forInternalCalls(), QueryOptions.DEFAULT);
}
private void testSimpleTypes() throws Throwable
{
// simple column identifiers without aliases are represented in
- // ResultSet.Metadata by the underlying ColumnDefinition
+ // ResultSet.Metadata by the underlying ColumnMetadata
ColumnSpecification kSpec = columnSpecification("k", Int32Type.instance);
ColumnSpecification v1Spec = columnSpecification("v1", Int32Type.instance);
ColumnSpecification v2Spec = columnSpecification("v2", AsciiType.instance);
@@ -135,12 +144,12 @@
private void testWildcard() throws Throwable
{
- // Wildcard select represents each column in the table with a ColumnDefinition
+ // Wildcard select represents each column in the table with a ColumnMetadata
// in the ResultSet metadata
- ColumnDefinition kSpec = columnDefinition("k");
- ColumnDefinition v1Spec = columnDefinition("v1");
- ColumnDefinition v2Spec = columnDefinition("v2");
- ColumnDefinition v3Spec = columnDefinition("v3");
+ ColumnMetadata kSpec = columnDefinition("k");
+ ColumnMetadata v1Spec = columnDefinition("v1");
+ ColumnMetadata v2Spec = columnDefinition("v2");
+ ColumnMetadata v3Spec = columnDefinition("v3");
SelectionColumnMapping expected = SelectionColumnMapping.newMapping()
.addMapping(kSpec, columnDefinition("k"))
.addMapping(v1Spec, columnDefinition("v1"))
@@ -153,7 +162,7 @@
private void testSimpleTypesWithAliases() throws Throwable
{
// simple column identifiers with aliases are represented in ResultSet.Metadata
- // by a ColumnSpecification based on the underlying ColumnDefinition
+ // by a ColumnSpecification based on the underlying ColumnMetadata
ColumnSpecification kSpec = columnSpecification("k_alias", Int32Type.instance);
ColumnSpecification v1Spec = columnSpecification("v1_alias", Int32Type.instance);
ColumnSpecification v2Spec = columnSpecification("v2_alias", AsciiType.instance);
@@ -397,7 +406,7 @@
private void testMultipleUnaliasedSelectionOfSameColumn() throws Throwable
{
// simple column identifiers without aliases are represented in
- // ResultSet.Metadata by the underlying ColumnDefinition
+ // ResultSet.Metadata by the underlying ColumnMetadata
SelectionColumns expected = SelectionColumnMapping.newMapping()
.addMapping(columnSpecification("v1", Int32Type.instance),
columnDefinition("v1"))
@@ -407,6 +416,91 @@
verify(expected, "SELECT v1, v1 FROM %s");
}
+ private void testListLitteral() throws Throwable
+ {
+ ColumnSpecification listSpec = columnSpecification("[k, v1]", ListType.getInstance(Int32Type.instance, false));
+ SelectionColumnMapping expected = SelectionColumnMapping.newMapping()
+ .addMapping(listSpec, asList(columnDefinition("k"),
+ columnDefinition("v1")));
+
+ verify(expected, "SELECT [k, v1] FROM %s");
+ }
+
+ private void testEmptyListLitteral() throws Throwable
+ {
+ ColumnSpecification listSpec = columnSpecification("(list<int>)[]", ListType.getInstance(Int32Type.instance, false));
+ SelectionColumnMapping expected = SelectionColumnMapping.newMapping()
+ .addMapping(listSpec, (ColumnMetadata) null);
+
+ verify(expected, "SELECT (list<int>)[] FROM %s");
+ }
+
+ private void testSetLitteral() throws Throwable
+ {
+ ColumnSpecification setSpec = columnSpecification("{k, v1}", SetType.getInstance(Int32Type.instance, false));
+ SelectionColumnMapping expected = SelectionColumnMapping.newMapping()
+ .addMapping(setSpec, asList(columnDefinition("k"),
+ columnDefinition("v1")));
+
+ verify(expected, "SELECT {k, v1} FROM %s");
+ }
+
+ private void testEmptySetLitteral() throws Throwable
+ {
+ ColumnSpecification setSpec = columnSpecification("(set<int>){}", SetType.getInstance(Int32Type.instance, false));
+ SelectionColumnMapping expected = SelectionColumnMapping.newMapping()
+ .addMapping(setSpec, (ColumnMetadata) null);
+
+ verify(expected, "SELECT (set<int>){} FROM %s");
+ }
+
+ private void testMapLitteral() throws Throwable
+ {
+ ColumnSpecification mapSpec = columnSpecification("(map<text, int>){'min': system.min(v1), 'max': system.max(v1)}", MapType.getInstance(UTF8Type.instance, Int32Type.instance, false));
+ SelectionColumnMapping expected = SelectionColumnMapping.newMapping()
+ .addMapping(mapSpec, asList(columnDefinition("v1")));
+
+ verify(expected, "SELECT (map<text, int>){'min': min(v1), 'max': max(v1)} FROM %s");
+ }
+
+ private void testEmptyMapLitteral() throws Throwable
+ {
+ ColumnSpecification mapSpec = columnSpecification("(map<text, int>){}", MapType.getInstance(UTF8Type.instance, Int32Type.instance, false));
+ SelectionColumnMapping expected = SelectionColumnMapping.newMapping()
+ .addMapping(mapSpec, (ColumnMetadata) null);
+
+ verify(expected, "SELECT (map<text, int>){} FROM %s");
+ }
+
+ private void testUDTLitteral() throws Throwable
+ {
+ UserType type = new UserType(KEYSPACE, ByteBufferUtil.bytes(typeName),
+ asList(FieldIdentifier.forUnquoted("f1"),
+ FieldIdentifier.forUnquoted("f2")),
+ asList(Int32Type.instance,
+ UTF8Type.instance),
+ false);
+
+ ColumnSpecification spec = columnSpecification("(" + KEYSPACE + "." + typeName + "){f1: v1, f2: v2}", type);
+ SelectionColumnMapping expected = SelectionColumnMapping.newMapping()
+ .addMapping(spec, asList(columnDefinition("v1"),
+ columnDefinition("v2")));
+
+ verify(expected, "SELECT ("+ typeName + "){f1: v1, f2: v2} FROM %s");
+ }
+
+ private void testTupleLitteral() throws Throwable
+ {
+ TupleType type = new TupleType(asList(Int32Type.instance, UTF8Type.instance));
+
+ ColumnSpecification setSpec = columnSpecification("(tuple<int, text>)(v1, v2)", type);
+ SelectionColumnMapping expected = SelectionColumnMapping.newMapping()
+ .addMapping(setSpec, asList(columnDefinition("v1"),
+ columnDefinition("v2")));
+
+ verify(expected, "SELECT (tuple<int, text>)(v1, v2) FROM %s");
+ }
+
private void testMixedColumnTypes() throws Throwable
{
ColumnSpecification kSpec = columnSpecification("k_alias", Int32Type.instance);
@@ -463,7 +557,7 @@
" LANGUAGE javascript" +
" AS 'a*a'");
- ColumnDefinition v1 = columnDefinition("v1");
+ ColumnMetadata v1 = columnDefinition("v1");
SelectionColumns expected = SelectionColumnMapping.newMapping()
.addMapping(columnSpecification(aFunc + "(v1)",
Int32Type.instance),
@@ -487,8 +581,8 @@
private void checkExecution(SelectStatement statement, List<ColumnSpecification> expectedResultColumns)
throws RequestExecutionException, RequestValidationException
{
- UntypedResultSet rs = UntypedResultSet.create(statement.executeInternal(QueryState.forInternalCalls(),
- QueryOptions.DEFAULT).result);
+ UntypedResultSet rs = UntypedResultSet.create(statement.executeLocally(QueryState.forInternalCalls(),
+ QueryOptions.DEFAULT).result);
assertEquals(expectedResultColumns, rs.one().getColumns());
}
@@ -496,7 +590,7 @@
private SelectStatement getSelect(String query) throws RequestValidationException
{
CQLStatement statement = QueryProcessor.getStatement(String.format(query, KEYSPACE + "." + tableName),
- ClientState.forInternalCalls()).statement;
+ ClientState.forInternalCalls());
assertTrue(statement instanceof SelectStatement);
return (SelectStatement)statement;
}
@@ -506,18 +600,17 @@
assertEquals(expected, select.getSelection().getColumnMapping());
}
- private Iterable<ColumnDefinition> columnDefinitions(String...names)
+ private Iterable<ColumnMetadata> columnDefinitions(String...names)
{
- List<ColumnDefinition> defs = new ArrayList<>();
+ List<ColumnMetadata> defs = new ArrayList<>();
for (String n : names)
defs.add(columnDefinition(n));
return defs;
}
- private ColumnDefinition columnDefinition(String name)
+ private ColumnMetadata columnDefinition(String name)
{
- return Schema.instance.getCFMetaData(KEYSPACE, tableName)
- .getColumnDefinition(new ColumnIdentifier(name, true));
+ return Schema.instance.getTableMetadata(KEYSPACE, tableName).getColumn(new ColumnIdentifier(name, true));
}
diff --git a/test/unit/org/apache/cassandra/cql3/selection/TermSelectionTest.java b/test/unit/org/apache/cassandra/cql3/selection/TermSelectionTest.java
index a07f8f9..fb46809 100644
--- a/test/unit/org/apache/cassandra/cql3/selection/TermSelectionTest.java
+++ b/test/unit/org/apache/cassandra/cql3/selection/TermSelectionTest.java
@@ -43,20 +43,26 @@
@Test
public void testSelectLiteral() throws Throwable
{
+ long timestampInMicros = System.currentTimeMillis() * 1000;
createTable("CREATE TABLE %s (pk int, ck int, t text, PRIMARY KEY (pk, ck) )");
- execute("INSERT INTO %s (pk, ck, t) VALUES (1, 1, 'one')");
- execute("INSERT INTO %s (pk, ck, t) VALUES (1, 2, 'two')");
- execute("INSERT INTO %s (pk, ck, t) VALUES (1, 3, 'three')");
+ execute("INSERT INTO %s (pk, ck, t) VALUES (?, ?, ?) USING TIMESTAMP ?", 1, 1, "one", timestampInMicros);
+ execute("INSERT INTO %s (pk, ck, t) VALUES (?, ?, ?) USING TIMESTAMP ?", 1, 2, "two", timestampInMicros);
+ execute("INSERT INTO %s (pk, ck, t) VALUES (?, ?, ?) USING TIMESTAMP ?", 1, 3, "three", timestampInMicros);
assertInvalidMessage("Cannot infer type for term", "SELECT ck, t, 'a const' FROM %s");
assertConstantResult(execute("SELECT ck, t, (text)'a const' FROM %s"), "a const");
assertInvalidMessage("Cannot infer type for term", "SELECT ck, t, 42 FROM %s");
- assertConstantResult(execute("SELECT ck, t, (int)42 FROM %s"), 42);
+ assertConstantResult(execute("SELECT ck, t, (smallint)42 FROM %s"), (short) 42);
assertInvalidMessage("Cannot infer type for term", "SELECT ck, t, (1, 'foo') FROM %s");
assertConstantResult(execute("SELECT ck, t, (tuple<int, text>)(1, 'foo') FROM %s"), tuple(1, "foo"));
+ assertInvalidMessage("Cannot infer type for term ((1)) in selection clause", "SELECT ck, t, ((1)) FROM %s");
+ // We cannot differentiate a tuple containing a tuple from a tuple between parentheses.
+ assertInvalidMessage("Cannot infer type for term ((tuple<int>)(1))", "SELECT ck, t, ((tuple<int>)(1)) FROM %s");
+ assertConstantResult(execute("SELECT ck, t, (tuple<tuple<int>>)((1)) FROM %s"), tuple(tuple(1)));
+
assertInvalidMessage("Cannot infer type for term", "SELECT ck, t, [1, 2, 3] FROM %s");
assertConstantResult(execute("SELECT ck, t, (list<int>)[1, 2, 3] FROM %s"), list(1, 2, 3));
@@ -66,11 +72,343 @@
assertInvalidMessage("Cannot infer type for term", "SELECT ck, t, {1: 'foo', 2: 'bar', 3: 'baz'} FROM %s");
assertConstantResult(execute("SELECT ck, t, (map<int, text>){1: 'foo', 2: 'bar', 3: 'baz'} FROM %s"), map(1, "foo", 2, "bar", 3, "baz"));
+ assertInvalidMessage("Cannot infer type for term", "SELECT ck, t, {} FROM %s");
+ assertConstantResult(execute("SELECT ck, t, (map<int, text>){} FROM %s"), map());
+ assertConstantResult(execute("SELECT ck, t, (set<int>){} FROM %s"), set());
+
assertColumnNames(execute("SELECT ck, t, (int)42, (int)43 FROM %s"), "ck", "t", "(int)42", "(int)43");
assertRows(execute("SELECT ck, t, (int) 42, (int) 43 FROM %s"),
row(1, "one", 42, 43),
row(2, "two", 42, 43),
row(3, "three", 42, 43));
+
+ assertRows(execute("SELECT min(ck), max(ck), [min(ck), max(ck)] FROM %s"), row(1, 3, list(1, 3)));
+ assertRows(execute("SELECT [min(ck), max(ck)] FROM %s"), row(list(1, 3)));
+ assertRows(execute("SELECT {min(ck), max(ck)} FROM %s"), row(set(1, 3)));
+
+ // We need to use a cast to differentiate between a map and an UDT
+ assertInvalidMessage("Cannot infer type for term {'min': system.min(ck), 'max': system.max(ck)}",
+ "SELECT {'min' : min(ck), 'max' : max(ck)} FROM %s");
+ assertRows(execute("SELECT (map<text, int>){'min' : min(ck), 'max' : max(ck)} FROM %s"), row(map("min", 1, "max", 3)));
+
+ assertRows(execute("SELECT [1, min(ck), max(ck)] FROM %s"), row(list(1, 1, 3)));
+ assertRows(execute("SELECT {1, min(ck), max(ck)} FROM %s"), row(set(1, 1, 3)));
+ assertRows(execute("SELECT (map<text, int>) {'litteral' : 1, 'min' : min(ck), 'max' : max(ck)} FROM %s"), row(map("litteral", 1, "min", 1, "max", 3)));
+
+ // Test List nested within Lists
+ assertRows(execute("SELECT [[], [min(ck), max(ck)]] FROM %s"),
+ row(list(list(), list(1, 3))));
+ assertRows(execute("SELECT [[], [CAST(pk AS BIGINT), CAST(ck AS BIGINT), WRITETIME(t)]] FROM %s"),
+ row(list(list(), list(1L, 1L, timestampInMicros))),
+ row(list(list(), list(1L, 2L, timestampInMicros))),
+ row(list(list(), list(1L, 3L, timestampInMicros))));
+ assertRows(execute("SELECT [[min(ck)], [max(ck)]] FROM %s"),
+ row(list(list(1), list(3))));
+ assertRows(execute("SELECT [[min(ck)], ([max(ck)])] FROM %s"),
+ row(list(list(1), list(3))));
+ assertRows(execute("SELECT [[pk], [ck]] FROM %s"),
+ row(list(list(1), list(1))),
+ row(list(list(1), list(2))),
+ row(list(list(1), list(3))));
+ assertRows(execute("SELECT [[pk], [ck]] FROM %s WHERE pk = 1 ORDER BY ck DESC"),
+ row(list(list(1), list(3))),
+ row(list(list(1), list(2))),
+ row(list(list(1), list(1))));
+
+ // Test Sets nested within Lists
+ assertRows(execute("SELECT [{}, {min(ck), max(ck)}] FROM %s"),
+ row(list(set(), set(1, 3))));
+ assertRows(execute("SELECT [{}, {CAST(pk AS BIGINT), CAST(ck AS BIGINT), WRITETIME(t)}] FROM %s"),
+ row(list(set(), set(1L, 1L, timestampInMicros))),
+ row(list(set(), set(1L, 2L, timestampInMicros))),
+ row(list(set(), set(1L, 3L, timestampInMicros))));
+ assertRows(execute("SELECT [{min(ck)}, {max(ck)}] FROM %s"),
+ row(list(set(1), set(3))));
+ assertRows(execute("SELECT [{min(ck)}, ({max(ck)})] FROM %s"),
+ row(list(set(1), set(3))));
+ assertRows(execute("SELECT [{pk}, {ck}] FROM %s"),
+ row(list(set(1), set(1))),
+ row(list(set(1), set(2))),
+ row(list(set(1), set(3))));
+ assertRows(execute("SELECT [{pk}, {ck}] FROM %s WHERE pk = 1 ORDER BY ck DESC"),
+ row(list(set(1), set(3))),
+ row(list(set(1), set(2))),
+ row(list(set(1), set(1))));
+
+ // Test Maps nested within Lists
+ assertRows(execute("SELECT [{}, (map<text, int>){'min' : min(ck), 'max' : max(ck)}] FROM %s"),
+ row(list(map(), map("min", 1, "max", 3))));
+ assertRows(execute("SELECT [{}, (map<text, bigint>){'pk' : CAST(pk AS BIGINT), 'ck' : CAST(ck AS BIGINT), 'writetime' : WRITETIME(t)}] FROM %s"),
+ row(list(map(), map("pk", 1L, "ck", 1L, "writetime", timestampInMicros))),
+ row(list(map(), map("pk", 1L, "ck", 2L, "writetime", timestampInMicros))),
+ row(list(map(), map("pk", 1L, "ck", 3L, "writetime", timestampInMicros))));
+ assertRows(execute("SELECT [{}, (map<text, int>){'pk' : pk, 'ck' : ck}] FROM %s WHERE pk = 1 ORDER BY ck DESC"),
+ row(list(map(), map("pk", 1, "ck", 3))),
+ row(list(map(), map("pk", 1, "ck", 2))),
+ row(list(map(), map("pk", 1, "ck", 1))));
+
+ // Test Tuples nested within Lists
+ assertRows(execute("SELECT [(pk, ck, WRITETIME(t))] FROM %s"),
+ row(list(tuple(1, 1, timestampInMicros))),
+ row(list(tuple(1, 2, timestampInMicros))),
+ row(list(tuple(1, 3, timestampInMicros))));
+ assertRows(execute("SELECT [(min(ck), max(ck))] FROM %s"),
+ row(list(tuple(1, 3))));
+ assertRows(execute("SELECT [(CAST(pk AS BIGINT), CAST(ck AS BIGINT)), (t, WRITETIME(t))] FROM %s"),
+ row(list(tuple(1L, 1L), tuple("one", timestampInMicros))),
+ row(list(tuple(1L, 2L), tuple("two", timestampInMicros))),
+ row(list(tuple(1L, 3L), tuple("three", timestampInMicros))));
+
+ // Test UDTs nested within Lists
+ String type = createType("CREATE TYPE %s(a int, b int, c bigint)");
+ assertRows(execute("SELECT [(" + type + "){a : min(ck), b: max(ck)}] FROM %s"),
+ row(list(userType("a", 1, "b", 3, "c", null))));
+ assertRows(execute("SELECT [(" + type + "){a : pk, b : ck, c : WRITETIME(t)}] FROM %s"),
+ row(list(userType("a", 1, "b", 1, "c", timestampInMicros))),
+ row(list(userType("a", 1, "b", 2, "c", timestampInMicros))),
+ row(list(userType("a", 1, "b", 3, "c", timestampInMicros))));
+ assertRows(execute("SELECT [(" + type + "){a : pk, b : ck, c : WRITETIME(t)}] FROM %s WHERE pk = 1 ORDER BY ck DESC"),
+ row(list(userType("a", 1, "b", 3, "c", timestampInMicros))),
+ row(list(userType("a", 1, "b", 2, "c", timestampInMicros))),
+ row(list(userType("a", 1, "b", 1, "c", timestampInMicros))));
+
+ // Test Lists nested within Sets
+ assertRows(execute("SELECT {[], [min(ck), max(ck)]} FROM %s"),
+ row(set(list(), list(1, 3))));
+ assertRows(execute("SELECT {[], [pk, ck]} FROM %s LIMIT 2"),
+ row(set(list(), list(1, 1))),
+ row(set(list(), list(1, 2))));
+ assertRows(execute("SELECT {[], [pk, ck]} FROM %s WHERE pk = 1 ORDER BY ck DESC LIMIT 2"),
+ row(set(list(), list(1, 3))),
+ row(set(list(), list(1, 2))));
+ assertRows(execute("SELECT {[min(ck)], ([max(ck)])} FROM %s"),
+ row(set(list(1), list(3))));
+ assertRows(execute("SELECT {[pk], ([ck])} FROM %s"),
+ row(set(list(1), list(1))),
+ row(set(list(1), list(2))),
+ row(set(list(1), list(3))));
+ assertRows(execute("SELECT {([min(ck)]), [max(ck)]} FROM %s"),
+ row(set(list(1), list(3))));
+
+ // Test Sets nested within Sets
+ assertRows(execute("SELECT {{}, {min(ck), max(ck)}} FROM %s"),
+ row(set(set(), set(1, 3))));
+ assertRows(execute("SELECT {{}, {pk, ck}} FROM %s LIMIT 2"),
+ row(set(set(), set(1, 1))),
+ row(set(set(), set(1, 2))));
+ assertRows(execute("SELECT {{}, {pk, ck}} FROM %s WHERE pk = 1 ORDER BY ck DESC LIMIT 2"),
+ row(set(set(), set(1, 3))),
+ row(set(set(), set(1, 2))));
+ assertRows(execute("SELECT {{min(ck)}, ({max(ck)})} FROM %s"),
+ row(set(set(1), set(3))));
+ assertRows(execute("SELECT {{pk}, ({ck})} FROM %s"),
+ row(set(set(1), set(1))),
+ row(set(set(1), set(2))),
+ row(set(set(1), set(3))));
+ assertRows(execute("SELECT {({min(ck)}), {max(ck)}} FROM %s"),
+ row(set(set(1), set(3))));
+
+ // Test Maps nested within Sets
+ assertRows(execute("SELECT {{}, (map<text, int>){'min' : min(ck), 'max' : max(ck)}} FROM %s"),
+ row(set(map(), map("min", 1, "max", 3))));
+ assertRows(execute("SELECT {{}, (map<text, int>){'pk' : pk, 'ck' : ck}} FROM %s"),
+ row(set(map(), map("pk", 1, "ck", 1))),
+ row(set(map(), map("pk", 1, "ck", 2))),
+ row(set(map(), map("pk", 1, "ck", 3))));
+
+ // Test Tuples nested within Sets
+ assertRows(execute("SELECT {(pk, ck, WRITETIME(t))} FROM %s"),
+ row(set(tuple(1, 1, timestampInMicros))),
+ row(set(tuple(1, 2, timestampInMicros))),
+ row(set(tuple(1, 3, timestampInMicros))));
+ assertRows(execute("SELECT {(min(ck), max(ck))} FROM %s"),
+ row(set(tuple(1, 3))));
+
+ // Test UDTs nested within Sets
+ assertRows(execute("SELECT {(" + type + "){a : min(ck), b: max(ck)}} FROM %s"),
+ row(set(userType("a", 1, "b", 3, "c", null))));
+ assertRows(execute("SELECT {(" + type + "){a : pk, b : ck, c : WRITETIME(t)}} FROM %s"),
+ row(set(userType("a", 1, "b", 1, "c", timestampInMicros))),
+ row(set(userType("a", 1, "b", 2, "c", timestampInMicros))),
+ row(set(userType("a", 1, "b", 3, "c", timestampInMicros))));
+ assertRows(execute("SELECT {(" + type + "){a : pk, b : ck, c : WRITETIME(t)}} FROM %s WHERE pk = 1 ORDER BY ck DESC"),
+ row(set(userType("a", 1, "b", 3, "c", timestampInMicros))),
+ row(set(userType("a", 1, "b", 2, "c", timestampInMicros))),
+ row(set(userType("a", 1, "b", 1, "c", timestampInMicros))));
+
+ // Test Lists nested within Maps
+ assertRows(execute("SELECT (map<frozen<list<int>>, frozen<list<int>>>){[min(ck)]:[max(ck)]} FROM %s"),
+ row(map(list(1), list(3))));
+ assertRows(execute("SELECT (map<frozen<list<int>>, frozen<list<int>>>){[pk]: [ck]} FROM %s"),
+ row(map(list(1), list(1))),
+ row(map(list(1), list(2))),
+ row(map(list(1), list(3))));
+
+ // Test Sets nested within Maps
+ assertRows(execute("SELECT (map<frozen<set<int>>, frozen<set<int>>>){{min(ck)} : {max(ck)}} FROM %s"),
+ row(map(set(1), set(3))));
+ assertRows(execute("SELECT (map<frozen<set<int>>, frozen<set<int>>>){{pk} : {ck}} FROM %s"),
+ row(map(set(1), set(1))),
+ row(map(set(1), set(2))),
+ row(map(set(1), set(3))));
+
+ // Test Maps nested within Maps
+ assertRows(execute("SELECT (map<frozen<map<text, int>>, frozen<map<text, int>>>){{'min' : min(ck)} : {'max' : max(ck)}} FROM %s"),
+ row(map(map("min", 1), map("max", 3))));
+ assertRows(execute("SELECT (map<frozen<map<text, int>>, frozen<map<text, int>>>){{'pk' : pk} : {'ck' : ck}} FROM %s"),
+ row(map(map("pk", 1), map("ck", 1))),
+ row(map(map("pk", 1), map("ck", 2))),
+ row(map(map("pk", 1), map("ck", 3))));
+
+ // Test Tuples nested within Maps
+ assertRows(execute("SELECT (map<frozen<tuple<int, int>>, frozen<tuple<bigint>>>){(pk, ck) : (WRITETIME(t))} FROM %s"),
+ row(map(tuple(1, 1), tuple(timestampInMicros))),
+ row(map(tuple(1, 2), tuple(timestampInMicros))),
+ row(map(tuple(1, 3), tuple(timestampInMicros))));
+ assertRows(execute("SELECT (map<frozen<tuple<int>> , frozen<tuple<int>>>){(min(ck)) : (max(ck))} FROM %s"),
+ row(map(tuple(1), tuple(3))));
+
+ // Test UDTs nested within Maps
+ assertRows(execute("SELECT (map<int, frozen<" + type + ">>){ck : {a : min(ck), b: max(ck)}} FROM %s"),
+ row(map(1, userType("a", 1, "b", 3, "c", null))));
+ assertRows(execute("SELECT (map<int, frozen<" + type + ">>){ck : {a : pk, b : ck, c : WRITETIME(t)}} FROM %s"),
+ row(map(1, userType("a", 1, "b", 1, "c", timestampInMicros))),
+ row(map(2, userType("a", 1, "b", 2, "c", timestampInMicros))),
+ row(map(3, userType("a", 1, "b", 3, "c", timestampInMicros))));
+ assertRows(execute("SELECT (map<int, frozen<" + type + ">>){ck : {a : pk, b : ck, c : WRITETIME(t)}} FROM %s WHERE pk = 1 ORDER BY ck DESC"),
+ row(map(3, userType("a", 1, "b", 3, "c", timestampInMicros))),
+ row(map(2, userType("a", 1, "b", 2, "c", timestampInMicros))),
+ row(map(1, userType("a", 1, "b", 1, "c", timestampInMicros))));
+
+ // Test Lists nested within Tuples
+ assertRows(execute("SELECT ([min(ck)], [max(ck)]) FROM %s"),
+ row(tuple(list(1), list(3))));
+ assertRows(execute("SELECT ([pk], [ck]) FROM %s"),
+ row(tuple(list(1), list(1))),
+ row(tuple(list(1), list(2))),
+ row(tuple(list(1), list(3))));
+
+ // Test Sets nested within Tuples
+ assertRows(execute("SELECT ({min(ck)}, {max(ck)}) FROM %s"),
+ row(tuple(set(1), set(3))));
+ assertRows(execute("SELECT ({pk}, {ck}) FROM %s"),
+ row(tuple(set(1), set(1))),
+ row(tuple(set(1), set(2))),
+ row(tuple(set(1), set(3))));
+
+ // Test Maps nested within Tuples
+ assertRows(execute("SELECT ((map<text, int>){'min' : min(ck)}, (map<text, int>){'max' : max(ck)}) FROM %s"),
+ row(tuple(map("min", 1), map("max", 3))));
+ assertRows(execute("SELECT ((map<text, int>){'pk' : pk}, (map<text, int>){'ck' : ck}) FROM %s"),
+ row(tuple(map("pk", 1), map("ck", 1))),
+ row(tuple(map("pk", 1), map("ck", 2))),
+ row(tuple(map("pk", 1), map("ck", 3))));
+
+ // Test Tuples nested within Tuples
+ assertRows(execute("SELECT (tuple<tuple<int, int, bigint>>)((pk, ck, WRITETIME(t))) FROM %s"),
+ row(tuple(tuple(1, 1, timestampInMicros))),
+ row(tuple(tuple(1, 2, timestampInMicros))),
+ row(tuple(tuple(1, 3, timestampInMicros))));
+ assertRows(execute("SELECT (tuple<tuple<int, int, bigint>>)((min(ck), max(ck))) FROM %s"),
+ row(tuple(tuple(1, 3))));
+
+ assertRows(execute("SELECT ((t, WRITETIME(t)), (CAST(pk AS BIGINT), CAST(ck AS BIGINT))) FROM %s"),
+ row(tuple(tuple("one", timestampInMicros), tuple(1L, 1L))),
+ row(tuple(tuple("two", timestampInMicros), tuple(1L, 2L))),
+ row(tuple(tuple("three", timestampInMicros), tuple(1L, 3L))));
+
+ // Test UDTs nested within Tuples
+ assertRows(execute("SELECT (tuple<" + type + ">)({a : min(ck), b: max(ck)}) FROM %s"),
+ row(tuple(userType("a", 1, "b", 3, "c", null))));
+ assertRows(execute("SELECT (tuple<" + type + ">)({a : pk, b : ck, c : WRITETIME(t)}) FROM %s"),
+ row(tuple(userType("a", 1, "b", 1, "c", timestampInMicros))),
+ row(tuple(userType("a", 1, "b", 2, "c", timestampInMicros))),
+ row(tuple(userType("a", 1, "b", 3, "c", timestampInMicros))));
+ assertRows(execute("SELECT (tuple<" + type + ">)({a : pk, b : ck, c : WRITETIME(t)}) FROM %s WHERE pk = 1 ORDER BY ck DESC"),
+ row(tuple(userType("a", 1, "b", 3, "c", timestampInMicros))),
+ row(tuple(userType("a", 1, "b", 2, "c", timestampInMicros))),
+ row(tuple(userType("a", 1, "b", 1, "c", timestampInMicros))));
+
+ // Test Lists nested within UDTs
+ String containerType = createType("CREATE TYPE %s(l list<int>)");
+ assertRows(execute("SELECT (" + containerType + "){l : [min(ck), max(ck)]} FROM %s"),
+ row(userType("l", list(1, 3))));
+ assertRows(execute("SELECT (" + containerType + "){l : [pk, ck]} FROM %s"),
+ row(userType("l", list(1, 1))),
+ row(userType("l", list(1, 2))),
+ row(userType("l", list(1, 3))));
+
+ // Test Sets nested within UDTs
+ containerType = createType("CREATE TYPE %s(s set<int>)");
+ assertRows(execute("SELECT (" + containerType + "){s : {min(ck), max(ck)}} FROM %s"),
+ row(userType("s", set(1, 3))));
+ assertRows(execute("SELECT (" + containerType + "){s : {pk, ck}} FROM %s"),
+ row(userType("s", set(1))),
+ row(userType("s", set(1, 2))),
+ row(userType("s", set(1, 3))));
+
+ // Test Maps nested within UDTs
+ containerType = createType("CREATE TYPE %s(m map<text, int>)");
+ assertRows(execute("SELECT (" + containerType + "){m : {'min' : min(ck), 'max' : max(ck)}} FROM %s"),
+ row(userType("m", map("min", 1, "max", 3))));
+ assertRows(execute("SELECT (" + containerType + "){m : {'pk' : pk, 'ck' : ck}} FROM %s"),
+ row(userType("m", map("pk", 1, "ck", 1))),
+ row(userType("m", map("pk", 1, "ck", 2))),
+ row(userType("m", map("pk", 1, "ck", 3))));
+
+ // Test Tuples nested within UDTs
+ containerType = createType("CREATE TYPE %s(t tuple<int, int>, w tuple<bigint>)");
+ assertRows(execute("SELECT (" + containerType + "){t : (pk, ck), w : (WRITETIME(t))} FROM %s"),
+ row(userType("t", tuple(1, 1), "w", tuple(timestampInMicros))),
+ row(userType("t", tuple(1, 2), "w", tuple(timestampInMicros))),
+ row(userType("t", tuple(1, 3), "w", tuple(timestampInMicros))));
+
+ // Test UDTs nested within Maps
+ containerType = createType("CREATE TYPE %s(t frozen<" + type + ">)");
+ assertRows(execute("SELECT (" + containerType + "){t : {a : min(ck), b: max(ck)}} FROM %s"),
+ row(userType("t", userType("a", 1, "b", 3, "c", null))));
+ assertRows(execute("SELECT (" + containerType + "){t : {a : pk, b : ck, c : WRITETIME(t)}} FROM %s"),
+ row(userType("t", userType("a", 1, "b", 1, "c", timestampInMicros))),
+ row(userType("t", userType("a", 1, "b", 2, "c", timestampInMicros))),
+ row(userType("t", userType("a", 1, "b", 3, "c", timestampInMicros))));
+ assertRows(execute("SELECT (" + containerType + "){t : {a : pk, b : ck, c : WRITETIME(t)}} FROM %s WHERE pk = 1 ORDER BY ck DESC"),
+ row(userType("t", userType("a", 1, "b", 3, "c", timestampInMicros))),
+ row(userType("t", userType("a", 1, "b", 2, "c", timestampInMicros))),
+ row(userType("t", userType("a", 1, "b", 1, "c", timestampInMicros))));
+
+
+ // Test Litteral Set with Duration elements
+ assertInvalidMessage("Durations are not allowed inside sets: set<duration>",
+ "SELECT pk, ck, (set<duration>){2d, 1mo} FROM %s");
+
+ assertInvalidMessage("Invalid field selection: system.min(ck) of type int is not a user type",
+ "SELECT min(ck).min FROM %s");
+ assertInvalidMessage("Invalid field selection: (map<text, int>){'min': system.min(ck), 'max': system.max(ck)} of type frozen<map<text, int>> is not a user type",
+ "SELECT (map<text, int>) {'min' : min(ck), 'max' : max(ck)}.min FROM %s");
+ }
+
+ @Test
+ public void testCollectionLiteralsWithDurations() throws Throwable
+ {
+ createTable("CREATE TABLE %s (pk int, ck int, d1 duration, d2 duration, PRIMARY KEY (pk, ck) )");
+ execute("INSERT INTO %s (pk, ck, d1, d2) VALUES (1, 1, 15h, 13h)");
+ execute("INSERT INTO %s (pk, ck, d1, d2) VALUES (1, 2, 10h, 12h)");
+ execute("INSERT INTO %s (pk, ck, d1, d2) VALUES (1, 3, 11h, 13h)");
+
+ assertRows(execute("SELECT [d1, d2] FROM %s"),
+ row(list(Duration.from("15h"), Duration.from("13h"))),
+ row(list(Duration.from("10h"), Duration.from("12h"))),
+ row(list(Duration.from("11h"), Duration.from("13h"))));
+
+ assertInvalidMessage("Durations are not allowed inside sets: frozen<set<duration>>", "SELECT {d1, d2} FROM %s");
+
+ assertRows(execute("SELECT (map<int, duration>){ck : d1} FROM %s"),
+ row(map(1, Duration.from("15h"))),
+ row(map(2, Duration.from("10h"))),
+ row(map(3, Duration.from("11h"))));
+
+ assertInvalidMessage("Durations are not allowed as map keys: map<duration, int>",
+ "SELECT (map<duration, int>){d1 : ck, d2 :ck} FROM %s");
}
@Test
@@ -81,11 +419,41 @@
execute("INSERT INTO %s(k, v) VALUES (?, ?)", 0, userType("a", 3, "b", "foo"));
- assertInvalidMessage("Cannot infer type for term", "SELECT k, v, { a: 4, b: 'bar'} FROM %s");
+ assertInvalidMessage("Cannot infer type for term", "SELECT { a: 4, b: 'bar'} FROM %s");
assertRows(execute("SELECT k, v, (" + type + "){ a: 4, b: 'bar'} FROM %s"),
row(0, userType("a", 3, "b", "foo"), userType("a", 4, "b", "bar"))
);
+
+ assertRows(execute("SELECT k, v, (" + type + ")({ a: 4, b: 'bar'}) FROM %s"),
+ row(0, userType("a", 3, "b", "foo"), userType("a", 4, "b", "bar"))
+ );
+
+ assertRows(execute("SELECT k, v, ((" + type + "){ a: 4, b: 'bar'}).a FROM %s"),
+ row(0, userType("a", 3, "b", "foo"), 4)
+ );
+
+ assertRows(execute("SELECT k, v, (" + type + "){ a: 4, b: 'bar'}.a FROM %s"),
+ row(0, userType("a", 3, "b", "foo"), 4)
+ );
+
+ assertInvalidMessage("Cannot infer type for term", "SELECT { a: 4} FROM %s");
+
+ assertRows(execute("SELECT k, v, (" + type + "){ a: 4} FROM %s"),
+ row(0, userType("a", 3, "b", "foo"), userType("a", 4, "b", null))
+ );
+
+ assertRows(execute("SELECT k, v, (" + type + "){ b: 'bar'} FROM %s"),
+ row(0, userType("a", 3, "b", "foo"), userType("a", null, "b", "bar"))
+ );
+
+ execute("INSERT INTO %s(k, v) VALUES (?, ?)", 1, userType("a", 5, "b", "foo"));
+ assertRows(execute("SELECT (" + type + "){ a: max(v.a) , b: 'max'} FROM %s"),
+ row(userType("a", 5, "b", "max"))
+ );
+ assertRows(execute("SELECT (" + type + "){ a: min(v.a) , b: 'min'} FROM %s"),
+ row(userType("a", 3, "b", "min"))
+ );
}
@Test
@@ -221,11 +589,11 @@
createFunctionOverload(fAmbiguousFunc1, "int,int",
"CREATE FUNCTION %s (val1 int, val2 int) " +
"CALLED ON NULL INPUT " +
- "RETURNS bigint " +
+ "RETURNS int " +
"LANGUAGE java\n" +
- "AS 'return (long)Math.max(val1, val2);';");
- assertInvalidMessage("Ambiguous call to function cql_test_keyspace.function_",
- "SELECT pk, " + fAmbiguousFunc1 + "(valInt, 100) FROM %s");
+ "AS 'return Math.max(val1, val2);';");
+ assertRows(execute("SELECT pk, " + fAmbiguousFunc1 + "(valInt, 100) FROM %s"),
+ row(1, 100));
}
@Test
diff --git a/test/unit/org/apache/cassandra/cql3/statements/AlterRoleStatementTest.java b/test/unit/org/apache/cassandra/cql3/statements/AlterRoleStatementTest.java
new file mode 100644
index 0000000..883b4f5
--- /dev/null
+++ b/test/unit/org/apache/cassandra/cql3/statements/AlterRoleStatementTest.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cql3.statements;
+
+import com.google.common.collect.Sets;
+import org.junit.Assert;
+import org.junit.Test;
+
+import org.apache.cassandra.auth.DCPermissions;
+import org.apache.cassandra.cql3.CQLStatement;
+import org.apache.cassandra.cql3.QueryProcessor;
+
+public class AlterRoleStatementTest
+{
+ private static AlterRoleStatement parse(String query)
+ {
+ CQLStatement.Raw stmt = QueryProcessor.parseStatement(query);
+ Assert.assertTrue(stmt instanceof AlterRoleStatement);
+ return (AlterRoleStatement) stmt;
+ }
+
+ private static DCPermissions dcPerms(String query)
+ {
+ return parse(query).dcPermissions;
+ }
+
+ @Test
+ public void dcsNotSpecified() throws Exception
+ {
+ Assert.assertNull(dcPerms("ALTER ROLE r1 WITH PASSWORD = 'password'"));
+ }
+
+ @Test
+ public void dcsAllSpecified() throws Exception
+ {
+ DCPermissions dcPerms = dcPerms("ALTER ROLE r1 WITH ACCESS TO ALL DATACENTERS");
+ Assert.assertNotNull(dcPerms);
+ Assert.assertFalse(dcPerms.restrictsAccess());
+ }
+
+ @Test
+ public void singleDc() throws Exception
+ {
+ DCPermissions dcPerms = dcPerms("ALTER ROLE r1 WITH ACCESS TO DATACENTERS {'dc1'}");
+ Assert.assertNotNull(dcPerms);
+ Assert.assertTrue(dcPerms.restrictsAccess());
+ Assert.assertEquals(Sets.newHashSet("dc1"), dcPerms.allowedDCs());
+ }
+
+ @Test
+ public void multiDcs() throws Exception
+ {
+ DCPermissions dcPerms = dcPerms("ALTER ROLE r1 WITH ACCESS TO DATACENTERS {'dc1', 'dc2'}");
+ Assert.assertNotNull(dcPerms);
+ Assert.assertTrue(dcPerms.restrictsAccess());
+ Assert.assertEquals(Sets.newHashSet("dc1", "dc2"), dcPerms.allowedDCs());
+ }
+}
diff --git a/test/unit/org/apache/cassandra/cql3/statements/CreateRoleStatementTest.java b/test/unit/org/apache/cassandra/cql3/statements/CreateRoleStatementTest.java
new file mode 100644
index 0000000..7a23da2
--- /dev/null
+++ b/test/unit/org/apache/cassandra/cql3/statements/CreateRoleStatementTest.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cql3.statements;
+
+import com.google.common.collect.Sets;
+import org.junit.Assert;
+import org.junit.Test;
+
+import org.apache.cassandra.auth.DCPermissions;
+import org.apache.cassandra.cql3.CQLStatement;
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.cql3.QueryProcessor;
+
+public class CreateRoleStatementTest extends CQLTester
+{
+
+ private static CreateRoleStatement parse(String query)
+ {
+ CQLStatement.Raw stmt = QueryProcessor.parseStatement(query);
+ Assert.assertTrue(stmt instanceof CreateRoleStatement);
+ return (CreateRoleStatement) stmt;
+ }
+
+ private static DCPermissions dcPerms(String query)
+ {
+ return parse(query).dcPermissions;
+ }
+
+ @Test
+ public void allDcsImplicit() throws Exception
+ {
+ Assert.assertFalse(dcPerms("CREATE ROLE role").restrictsAccess());
+ }
+
+ @Test
+ public void allDcsExplicit() throws Exception
+ {
+ Assert.assertFalse(dcPerms("CREATE ROLE role WITH ACCESS TO ALL DATACENTERS").restrictsAccess());
+ }
+
+ @Test
+ public void singleDc() throws Exception
+ {
+ DCPermissions perms = dcPerms("CREATE ROLE role WITH ACCESS TO DATACENTERS {'dc1'}");
+ Assert.assertTrue(perms.restrictsAccess());
+ Assert.assertEquals(Sets.newHashSet("dc1"), perms.allowedDCs());
+ }
+
+ @Test
+ public void multiDcs() throws Exception
+ {
+ DCPermissions perms = dcPerms("CREATE ROLE role WITH ACCESS TO DATACENTERS {'dc1', 'dc2'}");
+ Assert.assertTrue(perms.restrictsAccess());
+ Assert.assertEquals(Sets.newHashSet("dc1", "dc2"), perms.allowedDCs());
+
+ }
+}
diff --git a/test/unit/org/apache/cassandra/cql3/statements/CreateUserStatementTest.java b/test/unit/org/apache/cassandra/cql3/statements/CreateUserStatementTest.java
new file mode 100644
index 0000000..51c61bb
--- /dev/null
+++ b/test/unit/org/apache/cassandra/cql3/statements/CreateUserStatementTest.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cql3.statements;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import org.apache.cassandra.auth.DCPermissions;
+import org.apache.cassandra.cql3.CQLStatement;
+import org.apache.cassandra.cql3.QueryProcessor;
+
+public class CreateUserStatementTest
+{
+ private static CreateRoleStatement parse(String query)
+ {
+ CQLStatement.Raw stmt = QueryProcessor.parseStatement(query);
+ Assert.assertTrue(stmt instanceof CreateRoleStatement);
+ return (CreateRoleStatement) stmt;
+ }
+
+ private static DCPermissions dcPerms(String query)
+ {
+ return parse(query).dcPermissions;
+ }
+
+ @Test
+ public void allDcsImplicit() throws Exception
+ {
+ Assert.assertFalse(dcPerms("CREATE USER u1").restrictsAccess());
+ }
+}
diff --git a/test/unit/org/apache/cassandra/cql3/statements/DescribeStatementTest.java b/test/unit/org/apache/cassandra/cql3/statements/DescribeStatementTest.java
new file mode 100644
index 0000000..0b6d8a3
--- /dev/null
+++ b/test/unit/org/apache/cassandra/cql3/statements/DescribeStatementTest.java
@@ -0,0 +1,815 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.statements;
+
+import java.util.Iterator;
+import java.util.Optional;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+
+import org.apache.commons.lang3.ArrayUtils;
+
+import org.junit.Test;
+
+import com.datastax.driver.core.ResultSet;
+import com.datastax.driver.core.Row;
+import com.datastax.driver.core.SimpleStatement;
+import com.datastax.driver.core.exceptions.InvalidQueryException;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.TokenMetadata;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.transport.ProtocolVersion;
+
+import static java.lang.String.format;
+import static org.apache.cassandra.schema.SchemaConstants.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+public class DescribeStatementTest extends CQLTester
+{
+ @Test
+ public void testSchemaChangeDuringPaging()
+ {
+ SimpleStatement stmt = new SimpleStatement("DESCRIBE KEYSPACES");
+ stmt.setFetchSize(1);
+ ResultSet rs = executeNet(ProtocolVersion.CURRENT, stmt);
+ Iterator<Row> iter = rs.iterator();
+ assertTrue(iter.hasNext());
+ iter.next();
+
+ createKeyspace("CREATE KEYSPACE %s WITH REPLICATION = {'class' : 'SimpleStrategy', 'replication_factor' : 1};");
+
+ try
+ {
+ iter.next();
+ fail("Expected InvalidQueryException");
+ }
+ catch (InvalidQueryException e)
+ {
+ assertEquals(DescribeStatement.SCHEMA_CHANGED_WHILE_PAGING_MESSAGE, e.getMessage());
+ }
+ }
+
+ @Test
+ public void testDescribeFunctionAndAggregate() throws Throwable
+ {
+ String fNonOverloaded = createFunction(KEYSPACE,
+ "",
+ "CREATE OR REPLACE FUNCTION %s() " +
+ "CALLED ON NULL INPUT " +
+ "RETURNS int " +
+ "LANGUAGE java " +
+ "AS 'throw new RuntimeException();';");
+
+ String fOverloaded = createFunction(KEYSPACE,
+ "int, ascii",
+ "CREATE FUNCTION %s (input int, other_in ascii) " +
+ "RETURNS NULL ON NULL INPUT " +
+ "RETURNS text " +
+ "LANGUAGE java " +
+ "AS 'return \"Hello World\";'");
+ createFunctionOverload(fOverloaded,
+ "text, ascii",
+ "CREATE FUNCTION %s (input text, other_in ascii) " +
+ "RETURNS NULL ON NULL INPUT " +
+ "RETURNS text " +
+ "LANGUAGE java " +
+ "AS 'return \"Hello World\";'");
+
+ for (String describeKeyword : new String[]{"DESCRIBE", "DESC"})
+ {
+ assertRowsNet(executeDescribeNet(describeKeyword + " FUNCTION " + fNonOverloaded),
+ row(KEYSPACE,
+ "function",
+ shortFunctionName(fNonOverloaded) + "()",
+ "CREATE FUNCTION " + fNonOverloaded + "()\n" +
+ " CALLED ON NULL INPUT\n" +
+ " RETURNS int\n" +
+ " LANGUAGE java\n" +
+ " AS $$throw new RuntimeException();$$;"));
+
+ assertRowsNet(executeDescribeNet(describeKeyword + " FUNCTION " + fOverloaded),
+ row(KEYSPACE,
+ "function",
+ shortFunctionName(fOverloaded) + "(int, ascii)",
+ "CREATE FUNCTION " + fOverloaded + "(input int, other_in ascii)\n" +
+ " RETURNS NULL ON NULL INPUT\n" +
+ " RETURNS text\n" +
+ " LANGUAGE java\n" +
+ " AS $$return \"Hello World\";$$;"),
+ row(KEYSPACE,
+ "function",
+ shortFunctionName(fOverloaded) + "(text, ascii)",
+ "CREATE FUNCTION " + fOverloaded + "(input text, other_in ascii)\n" +
+ " RETURNS NULL ON NULL INPUT\n" +
+ " RETURNS text\n" +
+ " LANGUAGE java\n" +
+ " AS $$return \"Hello World\";$$;"));
+
+ assertRowsNet(executeDescribeNet(describeKeyword + " FUNCTIONS"),
+ row(KEYSPACE,
+ "function",
+ shortFunctionName(fNonOverloaded) + "()"),
+ row(KEYSPACE,
+ "function",
+ shortFunctionName(fOverloaded) + "(int, ascii)"),
+ row(KEYSPACE,
+ "function",
+ shortFunctionName(fOverloaded) + "(text, ascii)"));
+ }
+
+ String fIntState = createFunction(KEYSPACE,
+ "int, int",
+ "CREATE FUNCTION %s (state int, add_to int) " +
+ "CALLED ON NULL INPUT " +
+ "RETURNS int " +
+ "LANGUAGE java " +
+ "AS 'return state + add_to;'");
+ String fFinal = createFunction(KEYSPACE,
+ "int",
+ "CREATE FUNCTION %s (state int) " +
+ "RETURNS NULL ON NULL INPUT " +
+ "RETURNS int " +
+ "LANGUAGE java " +
+ "AS 'return state;'");
+
+ String aNonDeterministic = createAggregate(KEYSPACE,
+ "int",
+ format("CREATE AGGREGATE %%s(int) " +
+ "SFUNC %s " +
+ "STYPE int " +
+ "INITCOND 42",
+ shortFunctionName(fIntState)));
+ String aDeterministic = createAggregate(KEYSPACE,
+ "int",
+ format("CREATE AGGREGATE %%s(int) " +
+ "SFUNC %s " +
+ "STYPE int " +
+ "FINALFUNC %s ",
+ shortFunctionName(fIntState),
+ shortFunctionName(fFinal)));
+
+ for (String describeKeyword : new String[]{"DESCRIBE", "DESC"})
+ {
+ assertRowsNet(executeDescribeNet(describeKeyword + " AGGREGATE " + aNonDeterministic),
+ row(KEYSPACE,
+ "aggregate",
+ shortFunctionName(aNonDeterministic) + "(int)",
+ "CREATE AGGREGATE " + aNonDeterministic + "(int)\n" +
+ " SFUNC " + shortFunctionName(fIntState) + "\n" +
+ " STYPE int\n" +
+ " INITCOND 42;"));
+ assertRowsNet(executeDescribeNet(describeKeyword + " AGGREGATE " + aDeterministic),
+ row(KEYSPACE,
+ "aggregate",
+ shortFunctionName(aDeterministic) + "(int)",
+ "CREATE AGGREGATE " + aDeterministic + "(int)\n" +
+ " SFUNC " + shortFunctionName(fIntState) + "\n" +
+ " STYPE int\n" +
+ " FINALFUNC " + shortFunctionName(fFinal) + ";"));
+ assertRowsNet(executeDescribeNet(describeKeyword + " AGGREGATES"),
+ row(KEYSPACE,
+ "aggregate",
+ shortFunctionName(aNonDeterministic) + "(int)"),
+ row(KEYSPACE,
+ "aggregate",
+ shortFunctionName(aDeterministic) + "(int)"));
+ }
+ }
+
+ @Test
+ public void testDescribeFunctionWithTuples() throws Throwable
+ {
+ String function = createFunction(KEYSPACE,
+ "tuple<int>, list<frozen<tuple<int, text>>>, tuple<frozen<tuple<int, text>>, text>",
+ "CREATE OR REPLACE FUNCTION %s(t tuple<int>, l list<frozen<tuple<int, text>>>, nt tuple<frozen<tuple<int, text>>, text>) " +
+ "CALLED ON NULL INPUT " +
+ "RETURNS tuple<int, text> " +
+ "LANGUAGE java " +
+ "AS 'throw new RuntimeException();';");
+
+ assertRowsNet(executeDescribeNet("DESCRIBE FUNCTION " + function),
+ row(KEYSPACE,
+ "function",
+ shortFunctionName(function) + "(tuple<int>, list<frozen<tuple<int, text>>>, tuple<frozen<tuple<int, text>>, text>)",
+ "CREATE FUNCTION " + function + "(t tuple<int>, l list<frozen<tuple<int, text>>>, nt tuple<frozen<tuple<int, text>>, text>)\n" +
+ " CALLED ON NULL INPUT\n" +
+ " RETURNS tuple<int, text>\n" +
+ " LANGUAGE java\n" +
+ " AS $$throw new RuntimeException();$$;"));
+ }
+
+ @Test
+ public void testDescribeMaterializedView() throws Throwable
+ {
+ assertRowsNet(executeDescribeNet("DESCRIBE ONLY KEYSPACE system_virtual_schema;"),
+ row("system_virtual_schema",
+ "keyspace",
+ "system_virtual_schema",
+ "/*\n" +
+ "Warning: Keyspace system_virtual_schema is a virtual keyspace and cannot be recreated with CQL.\n" +
+ "Structure, for reference:\n" +
+ "VIRTUAL KEYSPACE system_virtual_schema;\n" +
+ "*/"));
+
+ assertRowsNet(executeDescribeNet("DESCRIBE TABLE system_virtual_schema.columns;"),
+ row("system_virtual_schema",
+ "table",
+ "columns",
+ "/*\n" +
+ "Warning: Table system_virtual_schema.columns is a virtual table and cannot be recreated with CQL.\n" +
+ "Structure, for reference:\n" +
+ "VIRTUAL TABLE system_virtual_schema.columns (\n" +
+ " keyspace_name text,\n" +
+ " table_name text,\n" +
+ " column_name text,\n" +
+ " clustering_order text,\n" +
+ " column_name_bytes blob,\n" +
+ " kind text,\n" +
+ " position int,\n" +
+ " type text,\n" +
+ " PRIMARY KEY (keyspace_name, table_name, column_name)\n" +
+ ") WITH CLUSTERING ORDER BY (table_name ASC, column_name ASC)\n" +
+ " AND comment = 'virtual column definitions';\n" +
+ "*/"));
+ }
+
+ @Test
+ public void testDescribe() throws Throwable
+ {
+ try
+ {
+ execute("CREATE KEYSPACE test WITH REPLICATION = {'class' : 'SimpleStrategy', 'replication_factor' : 1};");
+ execute("CREATE TABLE test.users ( userid text PRIMARY KEY, firstname text, lastname text, age int);");
+ execute("CREATE INDEX myindex ON test.users (age);");
+ execute("CREATE TABLE test.\"Test\" (id int, col int, val text, PRIMARY KEY(id, col));");
+ execute("CREATE INDEX ON test.\"Test\" (col);");
+ execute("CREATE INDEX ON test.\"Test\" (val)");
+ execute("CREATE TABLE test.users_mv (username varchar, password varchar, gender varchar, session_token varchar, " +
+ "state varchar, birth_year bigint, PRIMARY KEY (username));");
+ execute("CREATE MATERIALIZED VIEW test.users_by_state AS SELECT * FROM test.users_mv " +
+ "WHERE STATE IS NOT NULL AND username IS NOT NULL PRIMARY KEY (state, username)");
+ execute(allTypesTable());
+
+ // Test describe schema
+
+ Object[][] testSchemaOutput = rows(
+ row(KEYSPACE, "keyspace", KEYSPACE,
+ "CREATE KEYSPACE " + KEYSPACE +
+ " WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'}" +
+ " AND durable_writes = true;"),
+ row(KEYSPACE_PER_TEST, "keyspace", KEYSPACE_PER_TEST,
+ "CREATE KEYSPACE " + KEYSPACE_PER_TEST +
+ " WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'}" +
+ " AND durable_writes = true;"),
+ row("test", "keyspace", "test", keyspaceOutput()),
+ row("test", "table", "has_all_types", allTypesTable()),
+ row("test", "table", "\"Test\"", testTableOutput()),
+ row("test", "index", "\"Test_col_idx\"", indexOutput("\"Test_col_idx\"", "\"Test\"", "col")),
+ row("test", "index", "\"Test_val_idx\"", indexOutput("\"Test_val_idx\"", "\"Test\"", "val")),
+ row("test", "table", "users", userTableOutput()),
+ row("test", "index", "myindex", indexOutput("myindex", "users", "age")),
+ row("test", "table", "users_mv", usersMvTableOutput()),
+ row("test", "materialized_view", "users_by_state", usersByStateMvOutput()));
+
+ assertRowsNet(executeDescribeNet("DESCRIBE SCHEMA"), testSchemaOutput);
+ assertRowsNet(executeDescribeNet("DESC SCHEMA"), testSchemaOutput);
+
+ // Test describe keyspaces/keyspace
+
+ Object[][] testKeyspacesOutput = rows(row(KEYSPACE, "keyspace", KEYSPACE),
+ row(KEYSPACE_PER_TEST, "keyspace", KEYSPACE_PER_TEST),
+ row(SYSTEM_KEYSPACE_NAME, "keyspace", SYSTEM_KEYSPACE_NAME),
+ row(AUTH_KEYSPACE_NAME, "keyspace", AUTH_KEYSPACE_NAME),
+ row(DISTRIBUTED_KEYSPACE_NAME, "keyspace", DISTRIBUTED_KEYSPACE_NAME),
+ row(SCHEMA_KEYSPACE_NAME, "keyspace", SCHEMA_KEYSPACE_NAME),
+ row(TRACE_KEYSPACE_NAME, "keyspace", TRACE_KEYSPACE_NAME),
+ row(VIRTUAL_SCHEMA, "keyspace", VIRTUAL_SCHEMA),
+ row("test", "keyspace", "test"));
+
+ for (String describeKeyword : new String[]{"DESCRIBE", "DESC"})
+ {
+ assertRowsNet(executeDescribeNet(describeKeyword + " KEYSPACES"), testKeyspacesOutput);
+ assertRowsNet(executeDescribeNet("test", describeKeyword + " KEYSPACES"), testKeyspacesOutput);
+
+ assertRowsNet(executeDescribeNet(describeKeyword + " ONLY KEYSPACE test"),
+ row("test", "keyspace", "test", keyspaceOutput()));
+ }
+
+ Object[][] testKeyspaceOutput = rows(row("test", "keyspace", "test", keyspaceOutput()),
+ row("test", "table", "has_all_types", allTypesTable()),
+ row("test", "table", "\"Test\"", testTableOutput()),
+ row("test", "index", "\"Test_col_idx\"", indexOutput("\"Test_col_idx\"", "\"Test\"", "col")),
+ row("test", "index", "\"Test_val_idx\"", indexOutput("\"Test_val_idx\"", "\"Test\"", "val")),
+ row("test", "table", "users", userTableOutput()),
+ row("test", "index", "myindex", indexOutput("myindex", "users", "age")),
+ row("test", "table", "users_mv", usersMvTableOutput()),
+ row("test", "materialized_view", "users_by_state", usersByStateMvOutput()));
+
+ for (String describeKeyword : new String[]{"DESCRIBE", "DESC"})
+ {
+ assertRowsNet(executeDescribeNet(describeKeyword + " KEYSPACE test"), testKeyspaceOutput);
+ assertRowsNet(executeDescribeNet(describeKeyword + " test"), testKeyspaceOutput);
+
+ describeError(describeKeyword + " test2", "'test2' not found in keyspaces");
+ }
+
+ // Test describe tables/table
+ for (String cmd : new String[]{"describe TABLES", "DESC tables"})
+ assertRowsNet(executeDescribeNet("test", cmd),
+ row("test", "table", "has_all_types"),
+ row("test", "table", "\"Test\""),
+ row("test", "table", "users"),
+ row("test", "table", "users_mv"));
+
+ testDescribeTable("test", "has_all_types", row("test", "table", "has_all_types", allTypesTable()));
+
+ testDescribeTable("test", "\"Test\"",
+ row("test", "table", "\"Test\"", testTableOutput()),
+ row("test", "index", "\"Test_col_idx\"", indexOutput("\"Test_col_idx\"", "\"Test\"", "col")),
+ row("test", "index", "\"Test_val_idx\"", indexOutput("\"Test_val_idx\"", "\"Test\"", "val")));
+
+ testDescribeTable("test", "users", row("test", "table", "users", userTableOutput()),
+ row("test", "index", "myindex", indexOutput("myindex", "users", "age")));
+
+ describeError("test", "DESCRIBE users2", "'users2' not found in keyspace 'test'");
+ describeError("DESCRIBE test.users2", "'users2' not found in keyspace 'test'");
+
+ // Test describe index
+
+ testDescribeIndex("test", "myindex", row("test", "index", "myindex", indexOutput("myindex", "users", "age")));
+ testDescribeIndex("test", "\"Test_col_idx\"", row("test", "index", "\"Test_col_idx\"", indexOutput("\"Test_col_idx\"", "\"Test\"", "col")));
+ testDescribeIndex("test", "\"Test_val_idx\"", row("test", "index", "\"Test_val_idx\"", indexOutput("\"Test_val_idx\"", "\"Test\"", "val")));
+
+ describeError("DESCRIBE test.myindex2", "'myindex2' not found in keyspace 'test'");
+ describeError("test", "DESCRIBE myindex2", "'myindex2' not found in keyspace 'test'");
+
+ // Test describe materialized view
+
+ testDescribeMaterializedView("test", "users_by_state", row("test", "materialized_view", "users_by_state", usersByStateMvOutput()));
+ }
+ finally
+ {
+ execute("DROP KEYSPACE IF EXISTS test");
+ }
+ }
+
+ private void testDescribeTable(String keyspace, String table, Object[]... rows) throws Throwable
+ {
+ for (String describeKeyword : new String[]{"describe", "desc"})
+ {
+ for (String cmd : new String[]{describeKeyword + " table " + keyspace + "." + table,
+ describeKeyword + " columnfamily " + keyspace + "." + table,
+ describeKeyword + " " + keyspace + "." + table})
+ {
+ assertRowsNet(executeDescribeNet(cmd), rows);
+ }
+
+ for (String cmd : new String[]{describeKeyword + " table " + table,
+ describeKeyword + " columnfamily " + table,
+ describeKeyword + " " + table})
+ {
+ assertRowsNet(executeDescribeNet(keyspace, cmd), rows);
+ }
+ }
+ }
+
+ private void testDescribeIndex(String keyspace, String index, Object[]... rows) throws Throwable
+ {
+ for (String describeKeyword : new String[]{"describe", "desc"})
+ {
+ for (String cmd : new String[]{describeKeyword + " index " + keyspace + "." + index,
+ describeKeyword + " " + keyspace + "." + index})
+ {
+ assertRowsNet(executeDescribeNet(cmd), rows);
+ }
+
+ for (String cmd : new String[]{describeKeyword + " index " + index,
+ describeKeyword + " " + index})
+ {
+ assertRowsNet(executeDescribeNet(keyspace, cmd), rows);
+ }
+ }
+ }
+
+ private void testDescribeMaterializedView(String keyspace, String view, Object[]... rows) throws Throwable
+ {
+ for (String describeKeyword : new String[]{"describe", "desc"})
+ {
+ for (String cmd : new String[]{describeKeyword + " materialized view " + keyspace + "." + view,
+ describeKeyword + " " + keyspace + "." + view})
+ {
+ assertRowsNet(executeDescribeNet(cmd), rows);
+ }
+
+ for (String cmd : new String[]{describeKeyword + " materialized view " + view,
+ describeKeyword + " " + view})
+ {
+ assertRowsNet(executeDescribeNet(keyspace, cmd), rows);
+ }
+ }
+ }
+
+ @Test
+ public void testDescribeCluster() throws Throwable
+ {
+ for (String describeKeyword : new String[]{"DESCRIBE", "DESC"})
+ {
+ assertRowsNet(executeDescribeNet(describeKeyword + " CLUSTER"),
+ row("Test Cluster",
+ "ByteOrderedPartitioner",
+ DatabaseDescriptor.getEndpointSnitch().getClass().getName()));
+
+ assertRowsNet(executeDescribeNet("system_virtual_schema", describeKeyword + " CLUSTER"),
+ row("Test Cluster",
+ "ByteOrderedPartitioner",
+ DatabaseDescriptor.getEndpointSnitch().getClass().getName()));
+ }
+
+ TokenMetadata tokenMetadata = StorageService.instance.getTokenMetadata();
+ Token token = tokenMetadata.sortedTokens().get(0);
+ InetAddressAndPort addressAndPort = tokenMetadata.getAllEndpoints().iterator().next();
+
+ assertRowsNet(executeDescribeNet(KEYSPACE, "DESCRIBE CLUSTER"),
+ row("Test Cluster",
+ "ByteOrderedPartitioner",
+ DatabaseDescriptor.getEndpointSnitch().getClass().getName(),
+ ImmutableMap.of(token.toString(), ImmutableList.of(addressAndPort.toString()))));
+ }
+
+ @Test
+ public void testDescribeTableWithInternals() throws Throwable
+ {
+ String table = createTable("CREATE TABLE %s (pk1 text, pk2 int, c int, s decimal static, v1 text, v2 int, v3 int, PRIMARY KEY ((pk1, pk2), c ))");
+
+ TableId id = Schema.instance.getTableMetadata(KEYSPACE, table).id;
+
+ String tableCreateStatement = "CREATE TABLE " + KEYSPACE + "." + table + " (\n" +
+ " pk1 text,\n" +
+ " pk2 int,\n" +
+ " c int,\n" +
+ " s decimal static,\n" +
+ " v1 text,\n" +
+ " v2 int,\n" +
+ " v3 int,\n" +
+ " PRIMARY KEY ((pk1, pk2), c)\n" +
+ ") WITH ID = " + id + "\n" +
+ " AND CLUSTERING ORDER BY (c ASC)\n" +
+ " AND " + tableParametersCql();
+
+ assertRowsNet(executeDescribeNet("DESCRIBE TABLE " + KEYSPACE + "." + table + " WITH INTERNALS"),
+ row(KEYSPACE,
+ "table",
+ table,
+ tableCreateStatement));
+
+ String dropStatement = "ALTER TABLE " + KEYSPACE + "." + table + " DROP v3 USING TIMESTAMP 1589286942065000;";
+
+ execute(dropStatement);
+
+ assertRowsNet(executeDescribeNet("DESCRIBE TABLE " + KEYSPACE + "." + table + " WITH INTERNALS"),
+ row(KEYSPACE,
+ "table",
+ table,
+ tableCreateStatement + "\n" +
+ dropStatement));
+
+ String addStatement = "ALTER TABLE " + KEYSPACE + "." + table + " ADD v3 int;";
+
+ execute(addStatement);
+
+ assertRowsNet(executeDescribeNet("DESCRIBE TABLE " + KEYSPACE + "." + table + " WITH INTERNALS"),
+ row(KEYSPACE,
+ "table",
+ table,
+ tableCreateStatement + "\n" +
+ dropStatement + "\n" +
+ addStatement));
+ }
+
+ @Test
+ public void testPrimaryKeyPositionWithAndWithoutInternals() throws Throwable
+ {
+ String table = createTable("CREATE TABLE %s (pk text, v1 text, v2 int, v3 int, PRIMARY KEY (pk))");
+
+ TableId id = Schema.instance.getTableMetadata(KEYSPACE, table).id;
+
+ String tableCreateStatement = "CREATE TABLE " + KEYSPACE + "." + table + " (\n" +
+ " pk text PRIMARY KEY,\n" +
+ " v1 text,\n" +
+ " v2 int,\n" +
+ " v3 int\n" +
+ ") WITH ID = " + id + "\n" +
+ " AND " + tableParametersCql();
+
+ assertRowsNet(executeDescribeNet("DESCRIBE TABLE " + KEYSPACE + "." + table + " WITH INTERNALS"),
+ row(KEYSPACE,
+ "table",
+ table,
+ tableCreateStatement));
+
+ String dropStatement = "ALTER TABLE " + KEYSPACE + "." + table + " DROP v3 USING TIMESTAMP 1589286942065000;";
+
+ execute(dropStatement);
+
+ assertRowsNet(executeDescribeNet("DESCRIBE TABLE " + KEYSPACE + "." + table + " WITH INTERNALS"),
+ row(KEYSPACE,
+ "table",
+ table,
+ tableCreateStatement + "\n" +
+ dropStatement));
+
+ String tableCreateStatementWithoutDroppedColumn = "CREATE TABLE " + KEYSPACE + "." + table + " (\n" +
+ " pk text PRIMARY KEY,\n" +
+ " v1 text,\n" +
+ " v2 int\n" +
+ ") WITH " + tableParametersCql();
+
+ assertRowsNet(executeDescribeNet("DESCRIBE TABLE " + KEYSPACE + "." + table),
+ row(KEYSPACE,
+ "table",
+ table,
+ tableCreateStatementWithoutDroppedColumn));
+ }
+
+
+ @Test
+ public void testDescribeMissingKeyspace() throws Throwable
+ {
+ describeError("DESCRIBE TABLE foop",
+ "No keyspace specified and no current keyspace");
+ describeError("DESCRIBE MATERIALIZED VIEW foop",
+ "No keyspace specified and no current keyspace");
+ describeError("DESCRIBE INDEX foop",
+ "No keyspace specified and no current keyspace");
+ describeError("DESCRIBE TYPE foop",
+ "No keyspace specified and no current keyspace");
+ describeError("DESCRIBE FUNCTION foop",
+ "No keyspace specified and no current keyspace");
+ describeError("DESCRIBE AGGREGATE foop",
+ "No keyspace specified and no current keyspace");
+ }
+
+ @Test
+ public void testDescribeNotFound() throws Throwable
+ {
+ describeError(format("DESCRIBE AGGREGATE %s.%s", KEYSPACE, "aggr_foo"),
+ format("User defined aggregate '%s' not found in '%s'", "aggr_foo", KEYSPACE));
+
+ describeError(format("DESCRIBE FUNCTION %s.%s", KEYSPACE, "func_foo"),
+ format("User defined function '%s' not found in '%s'", "func_foo", KEYSPACE));
+
+ describeError(format("DESCRIBE %s.%s", KEYSPACE, "func_foo"),
+ format("'%s' not found in keyspace '%s'", "func_foo", KEYSPACE));
+
+ describeError(format("DESCRIBE %s", "foo"),
+ format("'%s' not found in keyspaces", "foo"));
+ }
+
+ @Test
+ public void testDescribeTypes() throws Throwable
+ {
+ String type1 = createType("CREATE TYPE %s (a int)");
+ String type2 = createType("CREATE TYPE %s (x text, y text)");
+ String type3 = createType("CREATE TYPE %s (a text, b frozen<" + type2 + ">)");
+ execute("ALTER TYPE " + KEYSPACE + "." + type1 + " ADD b frozen<" + type3 + ">");
+
+ try
+ {
+ assertRowsNet(executeDescribeNet(KEYSPACE, "DESCRIBE TYPES"),
+ row(KEYSPACE, "type", type1),
+ row(KEYSPACE, "type", type2),
+ row(KEYSPACE, "type", type3));
+
+ assertRowsNet(executeDescribeNet(KEYSPACE, "DESCRIBE TYPE " + type2),
+ row(KEYSPACE, "type", type2, "CREATE TYPE " + KEYSPACE + "." + type2 + " (\n" +
+ " x text,\n" +
+ " y text\n" +
+ ");"));
+ assertRowsNet(executeDescribeNet(KEYSPACE, "DESCRIBE TYPE " + type1),
+ row(KEYSPACE, "type", type1, "CREATE TYPE " + KEYSPACE + "." + type1 + " (\n" +
+ " a int,\n" +
+ " b frozen<" + type3 + ">\n" +
+ ");"));
+
+ assertRowsNet(executeDescribeNet(KEYSPACE, "DESCRIBE KEYSPACE " + KEYSPACE),
+ row(KEYSPACE, "keyspace", KEYSPACE, "CREATE KEYSPACE " + KEYSPACE +
+ " WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'}" +
+ " AND durable_writes = true;"),
+ row(KEYSPACE, "type", type2, "CREATE TYPE " + KEYSPACE + "." + type2 + " (\n" +
+ " x text,\n" +
+ " y text\n" +
+ ");"),
+ row(KEYSPACE, "type", type3, "CREATE TYPE " + KEYSPACE + "." + type3 + " (\n" +
+ " a text,\n" +
+ " b frozen<" + type2 + ">\n" +
+ ");"),
+ row(KEYSPACE, "type", type1, "CREATE TYPE " + KEYSPACE + "." + type1 + " (\n" +
+ " a int,\n" +
+ " b frozen<" + type3 + ">\n" +
+ ");"));
+ }
+ finally
+ {
+ execute("DROP TYPE " + KEYSPACE + "." + type1);
+ execute("DROP TYPE " + KEYSPACE + "." + type3);
+ execute("DROP TYPE " + KEYSPACE + "." + type2);
+ }
+ }
+
+ /**
+ * Tests for the error reported in CASSANDRA-9064 by:
+ *
+ * - creating the table described in the bug report, using LCS,
+ * - DESCRIBE-ing that table via cqlsh, then DROPping it,
+ * - running the output of the DESCRIBE statement as a CREATE TABLE statement, and
+ * - inserting a value into the table.
+ *
+ * The final two steps of the test should not fall down. If one does, that
+ * indicates the output of DESCRIBE is not a correct CREATE TABLE statement.
+ */
+ @Test
+ public void testDescribeRoundtrip() throws Throwable
+ {
+ for (String withInternals : new String[]{"", " WITH INTERNALS"})
+ {
+ String table = createTable("CREATE TABLE %s (key int PRIMARY KEY) WITH compaction = {'class': 'LeveledCompactionStrategy'}");
+
+ String output = executeDescribeNet(KEYSPACE, "DESCRIBE TABLE " + table + withInternals).all().get(0).getString("create_statement");
+
+ execute("DROP TABLE %s");
+
+ executeNet(output);
+
+ String output2 = executeDescribeNet(KEYSPACE, "DESCRIBE TABLE " + table + withInternals).all().get(0).getString("create_statement");
+ assertEquals(output, output2);
+
+ execute("INSERT INTO %s (key) VALUES (1)");
+ }
+ }
+
+ private static String allTypesTable()
+ {
+ return "CREATE TABLE test.has_all_types (\n" +
+ " num int PRIMARY KEY,\n" +
+ " asciicol ascii,\n" +
+ " bigintcol bigint,\n" +
+ " blobcol blob,\n" +
+ " booleancol boolean,\n" +
+ " decimalcol decimal,\n" +
+ " doublecol double,\n" +
+ " durationcol duration,\n" +
+ " floatcol float,\n" +
+ " frozenlistcol frozen<list<text>>,\n" +
+ " frozenmapcol frozen<map<timestamp, timeuuid>>,\n" +
+ " frozensetcol frozen<set<bigint>>,\n" +
+ " intcol int,\n" +
+ " smallintcol smallint,\n" +
+ " textcol text,\n" +
+ " timestampcol timestamp,\n" +
+ " tinyintcol tinyint,\n" +
+ " tuplecol frozen<tuple<text, int, frozen<tuple<timestamp>>>>,\n" +
+ " uuidcol uuid,\n" +
+ " varcharcol text,\n" +
+ " varintcol varint,\n" +
+ " listcol list<decimal>,\n" +
+ " mapcol map<timestamp, timeuuid>,\n" +
+ " setcol set<tinyint>\n" +
+ ") WITH " + tableParametersCql();
+ }
+
+ private static String usersByStateMvOutput()
+ {
+ return "CREATE MATERIALIZED VIEW test.users_by_state AS\n" +
+ " SELECT *\n" +
+ " FROM test.users_mv\n" +
+ " WHERE state IS NOT NULL AND username IS NOT NULL\n" +
+ " PRIMARY KEY (state, username)\n" +
+ " WITH CLUSTERING ORDER BY (username ASC)\n" +
+ " AND " + tableParametersCql();
+ }
+
+ private static String indexOutput(String index, String table, String col)
+ {
+ return format("CREATE INDEX %s ON %s.%s (%s);", index, "test", table, col);
+ }
+
+ private static String usersMvTableOutput()
+ {
+ return "CREATE TABLE test.users_mv (\n" +
+ " username text PRIMARY KEY,\n" +
+ " birth_year bigint,\n" +
+ " gender text,\n" +
+ " password text,\n" +
+ " session_token text,\n" +
+ " state text\n" +
+ ") WITH " + tableParametersCql();
+ }
+
+ private static String userTableOutput()
+ {
+ return "CREATE TABLE test.users (\n" +
+ " userid text PRIMARY KEY,\n" +
+ " age int,\n" +
+ " firstname text,\n" +
+ " lastname text\n" +
+ ") WITH " + tableParametersCql();
+ }
+
+ private static String testTableOutput()
+ {
+ return "CREATE TABLE test.\"Test\" (\n" +
+ " id int,\n" +
+ " col int,\n" +
+ " val text,\n" +
+ " PRIMARY KEY (id, col)\n" +
+ ") WITH CLUSTERING ORDER BY (col ASC)\n" +
+ " AND " + tableParametersCql();
+ }
+
+ private static String tableParametersCql()
+ {
+ return "additional_write_policy = '99p'\n" +
+ " AND bloom_filter_fp_chance = 0.01\n" +
+ " AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}\n" +
+ " AND cdc = false\n" +
+ " AND comment = ''\n" +
+ " AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}\n" +
+ " AND compression = {'chunk_length_in_kb': '16', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}\n" +
+ " AND crc_check_chance = 1.0\n" +
+ " AND default_time_to_live = 0\n" +
+ " AND extensions = {}\n" +
+ " AND gc_grace_seconds = 864000\n" +
+ " AND max_index_interval = 2048\n" +
+ " AND memtable_flush_period_in_ms = 0\n" +
+ " AND min_index_interval = 128\n" +
+ " AND read_repair = 'BLOCKING'\n" +
+ " AND speculative_retry = '99p';";
+ }
+
+ private static String keyspaceOutput()
+ {
+ return "CREATE KEYSPACE test WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'} AND durable_writes = true;";
+ }
+
+ private void describeError(String cql, String msg) throws Throwable
+ {
+ describeError(null, cql, msg);
+ }
+
+ private void describeError(String useKs, String cql, String msg) throws Throwable
+ {
+ assertInvalidThrowMessage(Optional.of(getProtocolVersion(useKs)), msg, InvalidQueryException.class, cql, ArrayUtils.EMPTY_OBJECT_ARRAY);
+ }
+
+ private ResultSet executeDescribeNet(String cql) throws Throwable
+ {
+ return executeDescribeNet(null, cql);
+ }
+
+ private ResultSet executeDescribeNet(String useKs, String cql) throws Throwable
+ {
+ return executeNetWithPaging(getProtocolVersion(useKs), cql, 3);
+ }
+
+ private ProtocolVersion getProtocolVersion(String useKs) throws Throwable
+ {
+ // We're using a trick here to distinguish driver sessions with a "USE keyspace" and without:
+ // As different ProtocolVersions use different driver instances, we use different ProtocolVersions
+ // for the with and without "USE keyspace" cases.
+
+ ProtocolVersion v = useKs != null ? ProtocolVersion.CURRENT : ProtocolVersion.V5;
+
+ if (useKs != null)
+ executeNet(v, "USE " + useKs);
+ return v;
+ }
+
+ private static Object[][] rows(Object[]... rows)
+ {
+ return rows;
+ }
+}
diff --git a/test/unit/org/apache/cassandra/cql3/statements/SelectStatementTest.java b/test/unit/org/apache/cassandra/cql3/statements/SelectStatementTest.java
new file mode 100644
index 0000000..5856bce
--- /dev/null
+++ b/test/unit/org/apache/cassandra/cql3/statements/SelectStatementTest.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cql3.statements;
+
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.CQLStatement;
+import org.apache.cassandra.cql3.QueryOptions;
+import org.apache.cassandra.cql3.QueryProcessor;
+import org.apache.cassandra.db.Slices;
+import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.service.ClientState;
+
+public class SelectStatementTest
+{
+
+ private static final String KEYSPACE = "ks";
+
+ @BeforeClass
+ public static void setupClass()
+ {
+ DatabaseDescriptor.daemonInitialization();
+ SchemaLoader.prepareServer();
+ SchemaLoader.createKeyspace(KEYSPACE, KeyspaceParams.simple(1));
+ }
+
+ private static SelectStatement parseSelect(String query)
+ {
+ CQLStatement stmt = QueryProcessor.parseStatement(query).prepare(ClientState.forInternalCalls());
+ assert stmt instanceof SelectStatement;
+ return (SelectStatement) stmt;
+ }
+
+ @Test
+ public void testNonsensicalBounds()
+ {
+ QueryProcessor.executeOnceInternal("CREATE TABLE ks.tbl (k int, c int, v int, primary key (k, c))");
+ QueryProcessor.executeOnceInternal("INSERT INTO ks.tbl (k, c, v) VALUES (100, 10, 0)");
+ Assert.assertEquals(Slices.NONE, parseSelect("SELECT * FROM ks.tbl WHERE k=100 AND c > 10 AND c <= 10").makeSlices(QueryOptions.DEFAULT));
+ Assert.assertEquals(Slices.NONE, parseSelect("SELECT * FROM ks.tbl WHERE k=100 AND c < 10 AND c >= 10").makeSlices(QueryOptions.DEFAULT));
+ Assert.assertEquals(Slices.NONE, parseSelect("SELECT * FROM ks.tbl WHERE k=100 AND c < 10 AND c > 10").makeSlices(QueryOptions.DEFAULT));
+ }
+}
diff --git a/test/unit/org/apache/cassandra/cql3/validation/ThriftIllegalColumnsTest.java b/test/unit/org/apache/cassandra/cql3/validation/ThriftIllegalColumnsTest.java
deleted file mode 100644
index 2d922e0..0000000
--- a/test/unit/org/apache/cassandra/cql3/validation/ThriftIllegalColumnsTest.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.cql3.validation;
-
-import java.nio.ByteBuffer;
-
-import org.junit.Test;
-
-import org.apache.cassandra.cql3.validation.operations.ThriftCQLTester;
-import org.apache.cassandra.db.marshal.CompositeType;
-import org.apache.cassandra.thrift.Cassandra;
-import org.apache.cassandra.thrift.Column;
-import org.apache.cassandra.thrift.ColumnParent;
-import org.apache.cassandra.thrift.InvalidRequestException;
-import org.apache.cassandra.utils.ByteBufferUtil;
-
-import static org.apache.cassandra.thrift.ConsistencyLevel.ONE;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-
-public class ThriftIllegalColumnsTest extends ThriftCQLTester
-{
- final String NON_COMPACT_TABLE = "t1";
- final String COMPACT_TABLE = "t2";
-
- @Test
- public void testNonCompactUpdateWithPrimaryKeyColumnName() throws Throwable
- {
- Cassandra.Client client = getClient();
- client.set_keyspace(KEYSPACE);
- String table = createTable(KEYSPACE, "CREATE TABLE %s (k int, c1 int, c2 int, v int, PRIMARY KEY (k, c1, c2))");
-
- // A cell name which represents a primary key column
- ByteBuffer badCellName = CompositeType.build(ByteBufferUtil.bytes(0), ByteBufferUtil.bytes(0), ByteBufferUtil.bytes("c1"));
- // A cell name which represents a regular column
- ByteBuffer goodCellName = CompositeType.build(ByteBufferUtil.bytes(0), ByteBufferUtil.bytes(0), ByteBufferUtil.bytes("v"));
-
- ColumnParent parent = new ColumnParent(table);
- ByteBuffer key = ByteBufferUtil.bytes(0);
- Column column = new Column();
- column.setName(badCellName);
- column.setValue(ByteBufferUtil.bytes(999));
- column.setTimestamp(System.currentTimeMillis());
-
- try
- {
- client.insert(key, parent, column, ONE);
- fail("Expected exception");
- } catch (InvalidRequestException e) {
- assertEquals("Cannot add primary key column c1 to partition update", e.getWhy());
- }
-
- column.setName(goodCellName);
- client.insert(key, parent, column, ONE);
- assertRows(execute("SELECT v from %s WHERE k = 0"), row(999));
- }
-
- @Test
- public void testThriftCompactUpdateWithPrimaryKeyColumnName() throws Throwable
- {
- Cassandra.Client client = getClient();
- client.set_keyspace(KEYSPACE);
- String table = createTable(KEYSPACE, "CREATE TABLE %s (k int, v int, PRIMARY KEY (k)) WITH COMPACT STORAGE");
-
- // A cell name which represents a primary key column
- ByteBuffer badCellName = ByteBufferUtil.bytes("k");
- // A cell name which represents a regular column
- ByteBuffer goodCellName = ByteBufferUtil.bytes("v");
-
- ColumnParent parent = new ColumnParent(table);
- ByteBuffer key = ByteBufferUtil.bytes(0);
- Column column = new Column();
- column.setName(badCellName);
- column.setValue(ByteBufferUtil.bytes(999));
- column.setTimestamp(System.currentTimeMillis());
- // if the table is compact, a cell name which appears to reference a primary
- // key column is treated as a dynamic column and so the update is allowed
- client.insert(key, parent, column, ONE);
-
- column.setName(goodCellName);
- client.insert(key, parent, column, ONE);
- assertRows(execute("SELECT v from %s where k=0"), row(999));
- }
-}
diff --git a/test/unit/org/apache/cassandra/cql3/validation/ThriftIntegrationTest.java b/test/unit/org/apache/cassandra/cql3/validation/ThriftIntegrationTest.java
deleted file mode 100644
index 7e89228..0000000
--- a/test/unit/org/apache/cassandra/cql3/validation/ThriftIntegrationTest.java
+++ /dev/null
@@ -1,942 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.cql3.validation;
-
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-import org.apache.cassandra.cql3.ColumnIdentifier;
-import org.apache.cassandra.cql3.UntypedResultSet;
-import org.apache.cassandra.cql3.validation.operations.ThriftCQLTester;
-import org.apache.cassandra.db.Keyspace;
-import org.apache.cassandra.db.marshal.AsciiType;
-import org.apache.cassandra.db.marshal.CounterColumnType;
-import org.apache.cassandra.db.marshal.Int32Type;
-import org.apache.cassandra.db.marshal.LongType;
-import org.apache.cassandra.locator.SimpleStrategy;
-import org.apache.cassandra.service.StorageService;
-import org.apache.cassandra.thrift.Cassandra;
-import org.apache.cassandra.thrift.CfDef;
-import org.apache.cassandra.thrift.Column;
-import org.apache.cassandra.thrift.ColumnDef;
-import org.apache.cassandra.thrift.ColumnOrSuperColumn;
-import org.apache.cassandra.thrift.ColumnParent;
-import org.apache.cassandra.thrift.ColumnPath;
-import org.apache.cassandra.thrift.CounterColumn;
-import org.apache.cassandra.thrift.CounterSuperColumn;
-import org.apache.cassandra.thrift.Deletion;
-import org.apache.cassandra.thrift.KsDef;
-import org.apache.cassandra.thrift.Mutation;
-import org.apache.cassandra.thrift.SlicePredicate;
-import org.apache.cassandra.thrift.SliceRange;
-import org.apache.cassandra.thrift.SuperColumn;
-import org.apache.cassandra.utils.ByteBufferUtil;
-import org.apache.cassandra.utils.FBUtilities;
-
-import static org.apache.cassandra.thrift.ConsistencyLevel.ONE;
-import static org.junit.Assert.assertEquals;
-
-public class ThriftIntegrationTest extends ThriftCQLTester
-{
- final static AtomicInteger seqNumber = new AtomicInteger();
- final String KEYSPACE = "thrift_compact_table_with_supercolumns_test_" + seqNumber.incrementAndGet();
-
- @Before
- public void setupSuperColumnFamily() throws Throwable
- {
- StorageService.instance.setRpcReady(true);
-
- final String denseTableName = createTableName();
- final String sparseTableName = currentSparseTable();
- final String counterTableName = currentCounterTable();
-
- CfDef cfDef = new CfDef().setColumn_type("Super")
- .setSubcomparator_type(Int32Type.instance.toString())
- .setComparator_type(AsciiType.instance.toString())
- .setDefault_validation_class(AsciiType.instance.toString())
- .setKey_validation_class(AsciiType.instance.toString())
- .setKeyspace(KEYSPACE)
- .setName(denseTableName);
-
- CfDef sparseCfDef = new CfDef().setColumn_type("Super")
- .setComparator_type(AsciiType.instance.toString())
- .setSubcomparator_type(AsciiType.instance.toString())
- .setKey_validation_class(AsciiType.instance.toString())
- .setColumn_metadata(Arrays.asList(new ColumnDef(ByteBufferUtil.bytes("col1"), LongType.instance.toString()),
- new ColumnDef(ByteBufferUtil.bytes("col2"), LongType.instance.toString())))
- .setKeyspace(KEYSPACE)
- .setName(sparseTableName);
-
- CfDef counterCfDef = new CfDef().setColumn_type("Super")
- .setSubcomparator_type(AsciiType.instance.toString())
- .setComparator_type(AsciiType.instance.toString())
- .setDefault_validation_class(CounterColumnType.instance.toString())
- .setKey_validation_class(AsciiType.instance.toString())
- .setKeyspace(KEYSPACE)
- .setName(counterTableName);
-
- KsDef ksDef = new KsDef(KEYSPACE,
- SimpleStrategy.class.getName(),
- Arrays.asList(cfDef, sparseCfDef, counterCfDef));
-
- ksDef.setStrategy_options(Collections.singletonMap("replication_factor", "1"));
-
- Cassandra.Client client = getClient();
- client.system_add_keyspace(ksDef);
- client.set_keyspace(KEYSPACE);
- }
-
- @After
- public void tearDown() throws Throwable
- {
- getClient().send_system_drop_keyspace(KEYSPACE);
- }
-
- @Test
- public void testCounterTableReads() throws Throwable
- {
- populateCounterTable();
- beforeAndAfterFlush(this::testCounterTableReadsInternal);
- }
-
- private void testCounterTableReadsInternal() throws Throwable
- {
- UntypedResultSet resultSet = execute(String.format("select * from %s.%s", KEYSPACE, currentCounterTable()));
- assertRows(resultSet,
- row("key1", "ck1", "counter1", 10L),
- row("key1", "ck1", "counter2", 5L),
- row("key2", "ck1", "counter1", 10L),
- row("key2", "ck1", "counter2", 5L));
- }
-
- @Test
- public void testCounterTableThriftUpdates() throws Throwable
- {
- populateCounterTable();
-
- Cassandra.Client client = getClient();
- Mutation mutation = new Mutation();
- ColumnOrSuperColumn csoc = new ColumnOrSuperColumn();
- csoc.setCounter_super_column(new CounterSuperColumn(ByteBufferUtil.bytes("ck1"),
- Arrays.asList(new CounterColumn(ByteBufferUtil.bytes("counter1"), 1))));
- mutation.setColumn_or_supercolumn(csoc);
-
- Mutation mutation2 = new Mutation();
- ColumnOrSuperColumn csoc2 = new ColumnOrSuperColumn();
- csoc2.setCounter_super_column(new CounterSuperColumn(ByteBufferUtil.bytes("ck1"),
- Arrays.asList(new CounterColumn(ByteBufferUtil.bytes("counter1"), 100))));
- mutation2.setColumn_or_supercolumn(csoc2);
- client.batch_mutate(Collections.singletonMap(ByteBufferUtil.bytes("key1"),
- Collections.singletonMap(currentCounterTable(), Arrays.asList(mutation))),
- ONE);
- client.batch_mutate(Collections.singletonMap(ByteBufferUtil.bytes("key2"),
- Collections.singletonMap(currentCounterTable(), Arrays.asList(mutation2))),
- ONE);
-
- beforeAndAfterFlush(() -> {
- UntypedResultSet resultSet = execute(String.format("select * from %s.%s", KEYSPACE, currentCounterTable()));
- assertRows(resultSet,
- row("key1", "ck1", "counter1", 11L),
- row("key1", "ck1", "counter2", 5L),
- row("key2", "ck1", "counter1", 110L),
- row("key2", "ck1", "counter2", 5L));
- });
- }
-
- @Test
- public void testCounterTableCqlUpdates() throws Throwable
- {
- populateCounterTable();
-
- execute(String.format("UPDATE %s.%s set value = value + 1 WHERE key = ? AND column1 = ? AND column2 = ?", KEYSPACE, currentCounterTable()),
- "key1", "ck1", "counter1");
- execute(String.format("UPDATE %s.%s set value = value + 100 WHERE key = 'key2' AND column1 = 'ck1' AND column2 = 'counter1'", KEYSPACE, currentCounterTable()));
-
- execute(String.format("UPDATE %s.%s set value = value - ? WHERE key = 'key1' AND column1 = 'ck1' AND column2 = 'counter2'", KEYSPACE, currentCounterTable()), 2L);
- execute(String.format("UPDATE %s.%s set value = value - ? WHERE key = 'key2' AND column1 = 'ck1' AND column2 = 'counter2'", KEYSPACE, currentCounterTable()), 100L);
-
- beforeAndAfterFlush(() -> {
- UntypedResultSet resultSet = execute(String.format("select * from %s.%s", KEYSPACE, currentCounterTable()));
- assertRows(resultSet,
- row("key1", "ck1", "counter1", 11L),
- row("key1", "ck1", "counter2", 3L),
- row("key2", "ck1", "counter1", 110L),
- row("key2", "ck1", "counter2", -95L));
- });
- }
-
- @Test
- public void testCounterTableCqlDeletes() throws Throwable
- {
- populateCounterTable();
-
- assertRows(execute(String.format("select * from %s.%s", KEYSPACE, currentCounterTable())),
- row("key1", "ck1", "counter1", 10L),
- row("key1", "ck1", "counter2", 5L),
- row("key2", "ck1", "counter1", 10L),
- row("key2", "ck1", "counter2", 5L));
-
- execute(String.format("DELETE value FROM %s.%s WHERE key = ? AND column1 = ? AND column2 = ?", KEYSPACE, currentCounterTable()),
- "key1", "ck1", "counter1");
-
- assertRows(execute(String.format("select * from %s.%s", KEYSPACE, currentCounterTable())),
- row("key1", "ck1", "counter2", 5L),
- row("key2", "ck1", "counter1", 10L),
- row("key2", "ck1", "counter2", 5L));
-
- execute(String.format("DELETE FROM %s.%s WHERE key = ? AND column1 = ?", KEYSPACE, currentCounterTable()),
- "key1", "ck1");
-
- assertRows(execute(String.format("select * from %s.%s", KEYSPACE, currentCounterTable())),
- row("key2", "ck1", "counter1", 10L),
- row("key2", "ck1", "counter2", 5L));
-
- execute(String.format("DELETE FROM %s.%s WHERE key = ?", KEYSPACE, currentCounterTable()),
- "key2");
-
- assertEmpty(execute(String.format("select * from %s.%s", KEYSPACE, currentCounterTable())));
- }
-
- @Test
- public void testDenseTableAlter() throws Throwable
- {
- populateDenseTable();
-
- alterTable(String.format("ALTER TABLE %s.%s RENAME column1 TO renamed_column1", KEYSPACE, currentDenseTable()));
- alterTable(String.format("ALTER TABLE %s.%s RENAME column2 TO renamed_column2", KEYSPACE, currentDenseTable()));
- alterTable(String.format("ALTER TABLE %s.%s RENAME key TO renamed_key", KEYSPACE, currentDenseTable()));
- alterTable(String.format("ALTER TABLE %s.%s RENAME value TO renamed_value", KEYSPACE, currentDenseTable()));
-
- beforeAndAfterFlush(() -> {
- UntypedResultSet resultSet = execute(String.format("select * from %s.%s", KEYSPACE, currentDenseTable()));
- assertEquals("renamed_key", resultSet.metadata().get(0).name.toString());
- assertEquals("renamed_column1", resultSet.metadata().get(1).name.toString());
- assertEquals("renamed_column2", resultSet.metadata().get(2).name.toString());
- assertEquals("renamed_value", resultSet.metadata().get(3).name.toString());
- assertRows(resultSet,
- row("key1", "val1", 1, "value1"),
- row("key1", "val1", 2, "value2"),
- row("key1", "val2", 4, "value4"),
- row("key1", "val2", 5, "value5"),
- row("key2", "val1", 1, "value1"),
- row("key2", "val1", 2, "value2"),
- row("key2", "val2", 4, "value4"),
- row("key2", "val2", 5, "value5"));
- });
- }
-
- @Test
- public void testDenseTableReads() throws Throwable
- {
- populateDenseTable();
- beforeAndAfterFlush(this::testDenseTableReadsInternal);
- }
-
- private void testDenseTableReadsInternal() throws Throwable
- {
- UntypedResultSet resultSet = execute(String.format("select * from %s.%s", KEYSPACE, currentDenseTable()));
- assertEquals("key", resultSet.metadata().get(0).name.toString());
- assertEquals("column1", resultSet.metadata().get(1).name.toString());
- assertEquals("column2", resultSet.metadata().get(2).name.toString());
- assertEquals("value", resultSet.metadata().get(3).name.toString());
-
-
- assertRows(resultSet,
- row("key1", "val1", 1, "value1"),
- row("key1", "val1", 2, "value2"),
- row("key1", "val2", 4, "value4"),
- row("key1", "val2", 5, "value5"),
- row("key2", "val1", 1, "value1"),
- row("key2", "val1", 2, "value2"),
- row("key2", "val2", 4, "value4"),
- row("key2", "val2", 5, "value5"));
-
- assertRows(execute(String.format("select * from %s.%s LIMIT 5", KEYSPACE, currentDenseTable())),
- row("key1", "val1", 1, "value1"),
- row("key1", "val1", 2, "value2"),
- row("key1", "val2", 4, "value4"),
- row("key1", "val2", 5, "value5"),
- row("key2", "val1", 1, "value1"));
-
- assertRows(execute(String.format("select value, column2, column1, key from %s.%s", KEYSPACE, currentDenseTable())),
- row("value1", 1, "val1", "key1"),
- row("value2", 2, "val1", "key1"),
- row("value4", 4, "val2", "key1"),
- row("value5", 5, "val2", "key1"),
- row("value1", 1, "val1", "key2"),
- row("value2", 2, "val1", "key2"),
- row("value4", 4, "val2", "key2"),
- row("value5", 5, "val2", "key2"));
-
- assertRows(execute(String.format("select * from %s.%s WHERE key = ? AND column1 = ?", KEYSPACE, currentDenseTable()), "key1", "val2"),
- row("key1", "val2", 4, "value4"),
- row("key1", "val2", 5, "value5"));
-
- assertRows(execute(String.format("select * from %s.%s where key IN ('key1', 'key2') and column1 = 'val1' and column2 = 2", KEYSPACE, currentDenseTable())),
- row("key1", "val1", 2, "value2"),
- row("key2", "val1", 2, "value2"));
- assertRows(execute(String.format("select * from %s.%s where key IN ('key1', 'key2') and column1 = 'val1' and column2 > 1", KEYSPACE, currentDenseTable())),
- row("key1", "val1", 2, "value2"),
- row("key2", "val1", 2, "value2"));
- assertRows(execute(String.format("select * from %s.%s where key IN ('key1', 'key2') and column1 = 'val1' and column2 >= 2", KEYSPACE, currentDenseTable())),
- row("key1", "val1", 2, "value2"),
- row("key2", "val1", 2, "value2"));
- assertEmpty(execute(String.format("select * from %s.%s where key IN ('key1', 'key2') and column1 = 'val1' and column2 > 2", KEYSPACE, currentDenseTable())));
-
- assertRows(execute(String.format("select column2, key from %s.%s WHERE key = ? AND column1 = ? and column2 = 5", KEYSPACE, currentDenseTable()), "key1", "val2"),
- row(5, "key1"));
- assertRows(execute(String.format("select * from %s.%s WHERE key = ? AND column1 = ? and column2 >= ?", KEYSPACE, currentDenseTable()), "key1", "val2", 5),
- row("key1", "val2", 5, "value5"));
- assertRows(execute(String.format("select * from %s.%s WHERE key = ? AND column1 = ? and column2 > ?", KEYSPACE, currentDenseTable()), "key1", "val2", 4),
- row("key1", "val2", 5, "value5"));
- assertRows(execute(String.format("select * from %s.%s WHERE key = ? AND column1 = ? and column2 < ?", KEYSPACE, currentDenseTable()), "key1", "val2", 5),
- row("key1", "val2", 4, "value4"));
- assertRows(execute(String.format("select * from %s.%s WHERE key = ? AND column1 = ? and column2 <= ?", KEYSPACE, currentDenseTable()), "key1", "val2", 5),
- row("key1", "val2", 4, "value4"),
- row("key1", "val2", 5, "value5"));
-
- assertRows(execute(String.format("select * from %s.%s where key = 'key1' and column1 in ('val1', 'val2') and column2 IN (1, 4)", KEYSPACE, currentDenseTable())),
- row("key1", "val1", 1, "value1"),
- row("key1", "val2", 4, "value4"));
-
- assertRows(execute(String.format("select * from %s.%s where key = 'key1' and column1 in ('val1', 'val2')", KEYSPACE, currentDenseTable())),
- row("key1", "val1", 1, "value1"),
- row("key1", "val1", 2, "value2"),
- row("key1", "val2", 4, "value4"),
- row("key1", "val2", 5, "value5"));
-
- assertRows(execute(String.format("select * from %s.%s where key = 'key1' and column1 in ('val1', 'val2') and column2 = 1", KEYSPACE, currentDenseTable())),
- row("key1", "val1", 1, "value1"));
-
- assertRows(execute(String.format("select * from %s.%s where key = 'key1' and (column1, column2) = ('val2', 4)", KEYSPACE, currentDenseTable())),
- row("key1", "val2", 4, "value4"));
-
- assertRows(execute(String.format("select * from %s.%s where key = 'key1' and (column1, column2) >= ('val2', 4)", KEYSPACE, currentDenseTable())),
- row("key1", "val2", 4, "value4"),
- row("key1", "val2", 5, "value5"));
-
- assertRows(execute(String.format("select * from %s.%s where key = 'key1' and (column1, column2) > ('val1', 1)", KEYSPACE, currentDenseTable())),
- row("key1", "val1", 2, "value2"),
- row("key1", "val2", 4, "value4"),
- row("key1", "val2", 5, "value5"));
-
- assertRows(execute(String.format("select * from %s.%s where key = 'key1' and (column1, column2) > ('val2', 1)", KEYSPACE, currentDenseTable())),
- row("key1", "val2", 4, "value4"),
- row("key1", "val2", 5, "value5"));
-
- resultSet = execute(String.format("select key as a, column1 as b, column2 as c, value as d " +
- "from %s.%s WHERE key = ? AND column1 = ?", KEYSPACE, currentDenseTable()), "key1", "val2");
- assertRows(resultSet,
- row("key1", "val2", 4, "value4"),
- row("key1", "val2", 5, "value5"));
- assertEquals(resultSet.metadata().get(2).type, Int32Type.instance);
- assertEquals(resultSet.metadata().get(3).type, AsciiType.instance);
-
- assertRows(execute(String.format("select column2, value from %s.%s WHERE key = ? AND column1 = ?", KEYSPACE, currentDenseTable()), "key1", "val2"),
- row(4, "value4"),
- row(5, "value5"));
-
- assertRows(execute(String.format("select column1, value from %s.%s WHERE key = ? AND column1 = ?", KEYSPACE, currentDenseTable()), "key2", "val1"),
- row("val1", "value1"),
- row("val1", "value2"));
-
- assertInvalidMessage("Secondary indexes are not supported on COMPACT STORAGE tables that have clustering columns",
- String.format("CREATE INDEX ON %s.%s (column2)", KEYSPACE, currentDenseTable()));
- assertInvalidMessage("Secondary indexes are not supported on COMPACT STORAGE tables that have clustering columns",
- String.format("CREATE INDEX ON %s.%s (value)", KEYSPACE, currentDenseTable()));
-
- assertRows(execute(String.format("SELECT JSON * FROM %s.%s WHERE key = ? AND column1 = ?", KEYSPACE, currentDenseTable()), "key1", "val2"),
- row("{\"key\": \"key1\", \"column1\": \"val2\", \"column2\": 4, \"value\": \"value4\"}"),
- row("{\"key\": \"key1\", \"column1\": \"val2\", \"column2\": 5, \"value\": \"value5\"}"));
- }
-
- @Test
- public void testDenseTablePartialCqlInserts() throws Throwable
- {
- assertInvalidMessage("Column value is mandatory for SuperColumn tables",
- String.format("INSERT INTO %s.%s (key, column1, column2) VALUES ('key1', 'val1', 1)", KEYSPACE, currentDenseTable()));
-
- // That's slightly different from 2.X, since null map keys are not allowed
- assertInvalidMessage("Column key is mandatory for SuperColumn tables",
- String.format("INSERT INTO %s.%s (key, column1, value) VALUES ('key1', 'val1', 'value1')", KEYSPACE, currentDenseTable()));
-
- execute(String.format("INSERT INTO %s.%s (key, column1, column2, value) VALUES ('key1', 'val1', 1, NULL)", KEYSPACE, currentDenseTable()));
- execute(String.format("INSERT INTO %s.%s (key, column1, column2, value) VALUES ('key1', 'val1', 1, ?)", KEYSPACE, currentDenseTable()), unset());
- assertEmpty(execute(String.format("select * from %s.%s", KEYSPACE, currentDenseTable())));
- }
-
- @Test
- public void testDenseTableCqlInserts() throws Throwable
- {
- Cassandra.Client client = getClient();
-
- execute(String.format("INSERT INTO %s.%s (key, column1, column2, value) VALUES (?, ?, ?, ?)", KEYSPACE, currentDenseTable()),
- "key1", "val1", 1, "value1");
- execute(String.format("INSERT INTO %s.%s (key, column1, column2, value) VALUES (?, ?, 2, ?)", KEYSPACE, currentDenseTable()),
- "key1", "val1", "value2");
- execute(String.format("INSERT INTO %s.%s (key, column1, column2, value) VALUES ('key1', 'val2', 4, 'value4')", KEYSPACE, currentDenseTable()));
- execute(String.format("INSERT INTO %s.%s (key, column1, column2, value) VALUES ('key1', 'val2', 5, 'value5')", KEYSPACE, currentDenseTable()));
-
- execute(String.format("INSERT INTO %s.%s (key, column1, column2, value) VALUES ('key2', 'val1', 1, 'value1')", KEYSPACE, currentDenseTable()));
- execute(String.format("INSERT INTO %s.%s (key, column1, column2, value) VALUES ('key2', 'val1', 2, 'value2')", KEYSPACE, currentDenseTable()));
- execute(String.format("INSERT INTO %s.%s (key, column1, column2, value) VALUES ('key2', 'val2', 4, 'value4')", KEYSPACE, currentDenseTable()));
- execute(String.format("INSERT INTO %s.%s (key, column1, column2, value) VALUES ('key2', 'val2', 5, 'value5')", KEYSPACE, currentDenseTable()));
-
- ColumnPath path = new ColumnPath(currentDenseTable());
- path.setSuper_column(ByteBufferUtil.bytes("val1"));
-
- ColumnOrSuperColumn cosc = client.get(ByteBufferUtil.bytes("key1"), path, ONE);
- assertEquals(cosc.getSuper_column().columns.get(0).name, ByteBufferUtil.bytes(1));
- assertEquals(cosc.getSuper_column().columns.get(0).value, ByteBufferUtil.bytes("value1"));
- assertEquals(cosc.getSuper_column().columns.get(1).name, ByteBufferUtil.bytes(2));
- assertEquals(cosc.getSuper_column().columns.get(1).value, ByteBufferUtil.bytes("value2"));
- }
-
- @Test
- public void testDenseTableCqlUpdates() throws Throwable
- {
- assertInvalidMessage("Column key is mandatory for SuperColumn tables",
- String.format("UPDATE %s.%s SET column2 = 1, value = 'value1' WHERE key = 'key1' AND column1 = 'val1'", KEYSPACE, currentDenseTable()));
- assertInvalidMessage("Column `column2` of type `int` found in SET part",
- String.format("UPDATE %s.%s SET column2 = 1, value = 'value1' WHERE key = 'key1' AND column1 = 'val1' AND column2 = 1", KEYSPACE, currentDenseTable()));
- assertInvalidMessage("Some clustering keys are missing: column1",
- String.format("UPDATE %s.%s SET value = 'value1' WHERE key = 'key1' AND column2 = 1", KEYSPACE, currentDenseTable()));
-
- execute(String.format("UPDATE %s.%s SET value = 'value1' WHERE key = 'key1' AND column1 = 'val1' AND column2 = 1", KEYSPACE, currentDenseTable()));
- execute(String.format("UPDATE %s.%s SET value = 'value2' WHERE key = 'key1' AND column1 = 'val1' AND column2 = 2", KEYSPACE, currentDenseTable()));
-
- execute(String.format("UPDATE %s.%s SET value = ? WHERE key = ? AND column1 = ? AND column2 = ?", KEYSPACE, currentDenseTable()),
- "value1", "key2", "val2", 1);
- execute(String.format("UPDATE %s.%s SET value = 'value2' WHERE key = 'key2' AND column1 = ? AND column2 = ?", KEYSPACE, currentDenseTable()),
- "val2", 2);
-
- Cassandra.Client client = getClient();
- ColumnPath path = new ColumnPath(currentDenseTable());
- path.setSuper_column(ByteBufferUtil.bytes("val1"));
-
- ColumnOrSuperColumn cosc = client.get(ByteBufferUtil.bytes("key1"), path, ONE);
- assertEquals(cosc.getSuper_column().columns.get(0).name, ByteBufferUtil.bytes(1));
- assertEquals(cosc.getSuper_column().columns.get(0).value, ByteBufferUtil.bytes("value1"));
- assertEquals(cosc.getSuper_column().columns.get(1).name, ByteBufferUtil.bytes(2));
- assertEquals(cosc.getSuper_column().columns.get(1).value, ByteBufferUtil.bytes("value2"));
-
- path = new ColumnPath(currentDenseTable());
- path.setSuper_column(ByteBufferUtil.bytes("val2"));
-
- cosc = client.get(ByteBufferUtil.bytes("key2"), path, ONE);
- assertEquals(cosc.getSuper_column().columns.get(0).name, ByteBufferUtil.bytes(1));
- assertEquals(cosc.getSuper_column().columns.get(0).value, ByteBufferUtil.bytes("value1"));
- assertEquals(cosc.getSuper_column().columns.get(1).name, ByteBufferUtil.bytes(2));
- assertEquals(cosc.getSuper_column().columns.get(1).value, ByteBufferUtil.bytes("value2"));
- }
-
-
- @Test
- public void testDenseTableCqlDeletes() throws Throwable
- {
- execute(String.format("INSERT INTO %s.%s (key, column1, column2, value) VALUES ('key1', 'val1', 1, 'value1')", KEYSPACE, currentDenseTable()));
- execute(String.format("INSERT INTO %s.%s (key, column1, column2, value) VALUES ('key1', 'val1', 2, 'value2')", KEYSPACE, currentDenseTable()));
- execute(String.format("INSERT INTO %s.%s (key, column1, column2, value) VALUES ('key1', 'val2', 4, 'value4')", KEYSPACE, currentDenseTable()));
- execute(String.format("INSERT INTO %s.%s (key, column1, column2, value) VALUES ('key1', 'val2', 5, 'value5')", KEYSPACE, currentDenseTable()));
-
- execute(String.format("INSERT INTO %s.%s (key, column1, column2, value) VALUES ('key2', 'val1', 1, 'value1')", KEYSPACE, currentDenseTable()));
- execute(String.format("INSERT INTO %s.%s (key, column1, column2, value) VALUES ('key2', 'val1', 2, 'value2')", KEYSPACE, currentDenseTable()));
- execute(String.format("INSERT INTO %s.%s (key, column1, column2, value) VALUES ('key2', 'val2', 4, 'value4')", KEYSPACE, currentDenseTable()));
- execute(String.format("INSERT INTO %s.%s (key, column1, column2, value) VALUES ('key2', 'val2', 5, 'value5')", KEYSPACE, currentDenseTable()));
-
- execute(String.format("DELETE FROM %s.%s WHERE key = 'key1' AND column1 = 'val2' AND column2 = 5", KEYSPACE, currentDenseTable()));
- assertRows(execute(String.format("SELECT * FROM %s.%s WHERE key = 'key1'", KEYSPACE, currentDenseTable())),
- row("key1", "val1", 1, "value1"),
- row("key1", "val1", 2, "value2"),
- row("key1", "val2", 4, "value4"));
- execute(String.format("DELETE FROM %s.%s WHERE key = 'key1' AND column1 = 'val2'", KEYSPACE, currentDenseTable()));
- assertRows(execute(String.format("SELECT * FROM %s.%s WHERE key = 'key1'", KEYSPACE, currentDenseTable())),
- row("key1", "val1", 1, "value1"),
- row("key1", "val1", 2, "value2"));
- execute(String.format("DELETE FROM %s.%s WHERE key = 'key1'", KEYSPACE, currentDenseTable()));
- assertEmpty(execute(String.format("SELECT * FROM %s.%s WHERE key = 'key1'", KEYSPACE, currentDenseTable())));
-
- Cassandra.Client client = getClient();
-
- Mutation mutation1 = new Mutation();
- SlicePredicate slicePredicate = new SlicePredicate();
- slicePredicate.setSlice_range(new SliceRange(ByteBufferUtil.bytes("val1"), ByteBufferUtil.bytes("val1"), false, 1));
- Deletion deletion1 = new Deletion();
- deletion1.setTimestamp(FBUtilities.timestampMicros());
- deletion1.setPredicate(slicePredicate);
- mutation1.setDeletion(deletion1);
- client.batch_mutate(Collections.singletonMap(ByteBufferUtil.bytes("key2"),
- Collections.singletonMap(currentDenseTable(), Arrays.asList(mutation1))),
- ONE);
- assertRows(execute(String.format("SELECT * FROM %s.%s WHERE key = 'key2'", KEYSPACE, currentDenseTable())),
- row("key2", "val2", 4, "value4"),
- row("key2", "val2", 5, "value5"));
-
- Mutation mutation2 = new Mutation();
- Deletion deletion2 = new Deletion();
- deletion2.setTimestamp(FBUtilities.timestampMicros());
- deletion2.setSuper_column(ByteBufferUtil.bytes("val2"));
- mutation2.setDeletion(deletion2);
- client.batch_mutate(Collections.singletonMap(ByteBufferUtil.bytes("key2"),
- Collections.singletonMap(currentDenseTable(), Arrays.asList(mutation2))),
- ONE);
-
- assertEmpty(execute(String.format("SELECT * FROM %s.%s WHERE key = 'key2'", KEYSPACE, currentDenseTable())));
-
- execute(String.format("INSERT INTO %s.%s (key, column1, column2, value) VALUES ('key1', 'val1', 1, 'value1')", KEYSPACE, currentDenseTable()));
- execute(String.format("INSERT INTO %s.%s (key, column1, column2, value) VALUES ('key2', 'val1', 1, 'value1')", KEYSPACE, currentDenseTable()));
- execute(String.format("INSERT INTO %s.%s (key, column1, column2, value) VALUES ('key3', 'val1', 1, 'value1')", KEYSPACE, currentDenseTable()));
-
- execute(String.format("DELETE FROM %s.%s WHERE key IN ('key1', 'key2')", KEYSPACE, currentDenseTable()));
- assertRows(execute(String.format("SELECT * FROM %s.%s", KEYSPACE, currentDenseTable())),
- row("key3", "val1", 1, "value1"));
-
- assertInvalidMessage("Multi-column relations cannot be used in WHERE clauses for UPDATE and DELETE statements",
- String.format("DELETE FROM %s.%s WHERE key = 'key3' AND (column1, column2) = ('val1', 1)", KEYSPACE, currentDenseTable()));
-
- assertInvalidMessage("Token relations cannot be used in WHERE clauses for UPDATE and DELETE statements: token(key) > token('key3')",
- String.format("DELETE FROM %s.%s WHERE token(key) > token('key3')", KEYSPACE, currentDenseTable()));
- }
-
- @Test
- public void testSparseTableAlter() throws Throwable
- {
- populateSparseTable();
-
- alterTable(String.format("ALTER TABLE %s.%s RENAME column1 TO renamed_column1", KEYSPACE, currentSparseTable()));
- alterTable(String.format("ALTER TABLE %s.%s RENAME key TO renamed_key", KEYSPACE, currentSparseTable()));
- assertInvalidMessage("Cannot rename non PRIMARY KEY part col1",
- String.format("ALTER TABLE %s.%s RENAME col1 TO renamed_col1", KEYSPACE, currentSparseTable()));
- assertInvalidMessage("Cannot rename non PRIMARY KEY part col2",
- String.format("ALTER TABLE %s.%s RENAME col2 TO renamed_col2", KEYSPACE, currentSparseTable()));
- assertInvalidMessage("Cannot rename unknown column column2 in keyspace",
- String.format("ALTER TABLE %s.%s RENAME column2 TO renamed_column2", KEYSPACE, currentSparseTable()));
- assertInvalidMessage("Cannot rename unknown column value in keyspace",
- String.format("ALTER TABLE %s.%s RENAME value TO renamed_value", KEYSPACE, currentSparseTable()));
-
-
- UntypedResultSet resultSet = execute(String.format("select * from %s.%s", KEYSPACE, currentSparseTable()));
- assertEquals("renamed_key", resultSet.metadata().get(0).name.toString());
- assertEquals("renamed_column1", resultSet.metadata().get(1).name.toString());
- assertEquals("col1", resultSet.metadata().get(2).name.toString());
- assertEquals("col2", resultSet.metadata().get(3).name.toString());
-
- assertRows(resultSet,
- row("key1", "val1", 3L, 4L),
- row("key1", "val2", 3L, 4L),
- row("key2", "val1", 3L, 4L),
- row("key2", "val2", 3L, 4L));
- }
-
- @Test
- public void testSparseTableCqlReads() throws Throwable
- {
- populateSparseTable();
- beforeAndAfterFlush(this::testSparseTableCqlReadsInternal);
- }
-
- private void testSparseTableCqlReadsInternal() throws Throwable
- {
- UntypedResultSet resultSet = execute(String.format("select * from %s.%s", KEYSPACE, currentSparseTable()));
- assertEquals("key", resultSet.metadata().get(0).name.toString());
- assertEquals("column1", resultSet.metadata().get(1).name.toString());
- assertEquals("col1", resultSet.metadata().get(2).name.toString());
- assertEquals("col2", resultSet.metadata().get(3).name.toString());
-
- assertRows(resultSet,
- row("key1", "val1", 3L, 4L),
- row("key1", "val2", 3L, 4L),
- row("key2", "val1", 3L, 4L),
- row("key2", "val2", 3L, 4L));
-
- assertRows(execute(String.format("select col1, col2, column1, key from %s.%s", KEYSPACE, currentSparseTable())),
- row(3L, 4L, "val1", "key1"),
- row(3L, 4L, "val2", "key1"),
- row(3L, 4L, "val1", "key2"),
- row(3L, 4L, "val2", "key2"));
-
- assertInvalidMessage("Undefined column name value",
- String.format("select value from %s.%s", KEYSPACE, currentSparseTable()));
-
- assertRows(execute(String.format("select * from %s.%s WHERE key = ? AND column1 = ?", KEYSPACE, currentSparseTable()), "key1", "val2"),
- row("key1", "val2", 3L, 4L));
-
- resultSet = execute(String.format("select col1 as a, col2 as b, column1 as c, key as d from %s.%s WHERE key = ? AND column1 = ?", KEYSPACE, currentSparseTable()), "key1", "val2");
- assertRows(resultSet,
- row(3L, 4L, "val2", "key1"));
- assertEquals(resultSet.metadata().get(0).name, ColumnIdentifier.getInterned("a", true));
- assertEquals(resultSet.metadata().get(1).name, ColumnIdentifier.getInterned("b", true));
- assertEquals(resultSet.metadata().get(2).name, ColumnIdentifier.getInterned("c", true));
- assertEquals(resultSet.metadata().get(3).name, ColumnIdentifier.getInterned("d", true));
-
- assertRows(execute(String.format("select col1, col2 from %s.%s WHERE key = ? AND column1 = ?", KEYSPACE, currentSparseTable()), "key1", "val2"),
- row(3L, 4L));
-
- assertInvalidMessage("Secondary indexes are not supported on COMPACT STORAGE tables that have clustering columns",
- String.format("CREATE INDEX ON %s.%s (column1)", KEYSPACE, currentSparseTable()));
- assertInvalidMessage("Secondary indexes are not supported on COMPACT STORAGE tables that have clustering columns",
- String.format("CREATE INDEX ON %s.%s (col1)", KEYSPACE, currentSparseTable()));
-
- assertRows(execute(String.format("SELECT JSON * FROM %s.%s WHERE key = ? AND column1 = ?", KEYSPACE, currentSparseTable()), "key1", "val2"),
- row("{\"key\": \"key1\", \"column1\": \"val2\", \"col1\": 3, \"col2\": 4}"));
- }
-
- @Test
- public void testSparseTableCqlInserts() throws Throwable
- {
- execute(String.format("insert into %s.%s (key, column1, col1, col2) values ('key1', 'val1', 1, 2)", KEYSPACE, currentSparseTable()));
- execute(String.format("insert into %s.%s (key, column1, col1, col2) values ('key1', 'val2', 3, 4)", KEYSPACE, currentSparseTable()));
- execute(String.format("insert into %s.%s (key, column1, col1, col2) values ('key2', 'val1', 5, 6)", KEYSPACE, currentSparseTable()));
- execute(String.format("insert into %s.%s (key, column1, col1, col2) values ('key2', 'val2', 7, 8)", KEYSPACE, currentSparseTable()));
-
- assertRows(execute(String.format("SELECT * FROM %s.%s", KEYSPACE, currentSparseTable())),
- row("key1", "val1", 1L, 2L),
- row("key1", "val2", 3L, 4L),
- row("key2", "val1", 5L, 6L),
- row("key2", "val2", 7L, 8L));
-
- execute(String.format("truncate %s.%s", KEYSPACE, currentSparseTable()));
-
- execute(String.format("insert into %s.%s (key, column1) values ('key1', 'val1')", KEYSPACE, currentSparseTable()));
- assertRows(execute(String.format("select * from %s.%s", KEYSPACE, currentSparseTable())));
-
- execute(String.format("insert into %s.%s (key, column1, col1) values ('key1', 'val1', 1)", KEYSPACE, currentSparseTable()));
- execute(String.format("insert into %s.%s (key, column1, col2) values ('key1', 'val1', 2)", KEYSPACE, currentSparseTable()));
- assertRows(execute(String.format("select * from %s.%s", KEYSPACE, currentSparseTable())),
- row("key1", "val1", 1L, 2L));
-
- Cassandra.Client client = getClient();
- ColumnPath path = new ColumnPath(currentSparseTable());
- path.setSuper_column(ByteBufferUtil.bytes("val1"));
-
- ColumnOrSuperColumn cosc = client.get(ByteBufferUtil.bytes("key1"), path, ONE);
- assertEquals(cosc.getSuper_column().columns.get(0).value, ByteBufferUtil.bytes(1L));
- assertEquals(cosc.getSuper_column().columns.get(0).name, ByteBufferUtil.bytes("col1"));
- assertEquals(cosc.getSuper_column().columns.get(1).value, ByteBufferUtil.bytes(2L));
- assertEquals(cosc.getSuper_column().columns.get(1).name, ByteBufferUtil.bytes("col2"));
- }
-
- @Test
- public void testSparseTableCqlUpdates() throws Throwable
- {
- execute(String.format("UPDATE %s.%s set col1 = 1, col2 = 2 WHERE key = 'key1' AND column1 = 'val1'", KEYSPACE, currentSparseTable()));
- execute(String.format("UPDATE %s.%s set col1 = 3, col2 = 4 WHERE key = 'key1' AND column1 = 'val2'", KEYSPACE, currentSparseTable()));
- execute(String.format("UPDATE %s.%s set col1 = 5, col2 = 6 WHERE key = 'key2' AND column1 = 'val1'", KEYSPACE, currentSparseTable()));
- execute(String.format("UPDATE %s.%s set col1 = 7, col2 = 8 WHERE key = 'key2' AND column1 = 'val2'", KEYSPACE, currentSparseTable()));
-
- assertRows(execute(String.format("SELECT * FROM %s.%s", KEYSPACE, currentSparseTable())),
- row("key1", "val1", 1L, 2L),
- row("key1", "val2", 3L, 4L),
- row("key2", "val1", 5L, 6L),
- row("key2", "val2", 7L, 8L));
-
- Cassandra.Client client = getClient();
- ColumnPath path = new ColumnPath(currentSparseTable());
- path.setSuper_column(ByteBufferUtil.bytes("val1"));
-
- ColumnOrSuperColumn cosc = client.get(ByteBufferUtil.bytes("key1"), path, ONE);
- assertEquals(cosc.getSuper_column().columns.get(0).value, ByteBufferUtil.bytes(1L));
- assertEquals(cosc.getSuper_column().columns.get(0).name, ByteBufferUtil.bytes("col1"));
- assertEquals(cosc.getSuper_column().columns.get(1).value, ByteBufferUtil.bytes(2L));
- assertEquals(cosc.getSuper_column().columns.get(1).name, ByteBufferUtil.bytes("col2"));
- }
-
- @Test
- public void testSparseTableCqlDeletes() throws Throwable
- {
- execute(String.format("insert into %s.%s (key, column1, col1, col2) values ('key1', 'val1', 1, 2)", KEYSPACE, currentSparseTable()));
- execute(String.format("insert into %s.%s (key, column1, col1, col2) values ('key1', 'val2', 3, 4)", KEYSPACE, currentSparseTable()));
- execute(String.format("insert into %s.%s (key, column1, col1, col2) values ('key2', 'val1', 5, 6)", KEYSPACE, currentSparseTable()));
- execute(String.format("insert into %s.%s (key, column1, col1, col2) values ('key2', 'val2', 7, 8)", KEYSPACE, currentSparseTable()));
-
- execute(String.format("DELETE col1 FROM %s.%s WHERE key = 'key1' AND column1 = 'val1'", KEYSPACE, currentSparseTable()));
-
- assertRows(execute(String.format("SELECT * FROM %s.%s", KEYSPACE, currentSparseTable())),
- row("key1", "val1", null, 2L),
- row("key1", "val2", 3L, 4L),
- row("key2", "val1", 5L, 6L),
- row("key2", "val2", 7L, 8L));
-
- execute(String.format("DELETE FROM %s.%s WHERE key = 'key1' AND column1 = 'val2'", KEYSPACE, currentSparseTable()));
-
- assertRows(execute(String.format("SELECT * FROM %s.%s", KEYSPACE, currentSparseTable())),
- row("key1", "val1", null, 2L),
- row("key2", "val1", 5L, 6L),
- row("key2", "val2", 7L, 8L));
-
- execute(String.format("DELETE FROM %s.%s WHERE key = 'key2'", KEYSPACE, currentSparseTable()));
-
- assertRows(execute(String.format("SELECT * FROM %s.%s", KEYSPACE, currentSparseTable())),
- row("key1", "val1", null, 2L));
- }
-
- @Test
- public void testInsertJson() throws Throwable
- {
- execute(String.format("INSERT INTO %s.%s JSON ?", KEYSPACE, currentDenseTable()),
- "{\"key\": \"key5\", \"column1\": \"val2\", \"column2\": 4, \"value\": \"value4\"}");
- assertRows(execute(String.format("SELECT * FROM %s.%s", KEYSPACE, currentDenseTable())),
- row("key5", "val2", 4, "value4"));
-
- execute(String.format("INSERT INTO %s.%s JSON ?", KEYSPACE, currentSparseTable()),
- "{\"key\": \"key1\", \"column1\": \"val1\", \"col1\": 1, \"col2\": 2}");
- assertRows(execute(String.format("SELECT * FROM %s.%s", KEYSPACE, currentSparseTable())),
- row("key1", "val1", 1L, 2L));
- }
-
- @Test
- public void testFiltering() throws Throwable
- {
- assertInvalidMessage("Filtering is not supported on SuperColumn tables",
- String.format("select * from %s.%s WHERE value = ?", KEYSPACE, currentDenseTable()),
- "value5");
- assertInvalidMessage("Filtering is not supported on SuperColumn tables",
- String.format("select * from %s.%s WHERE value = ? ALLOW FILTERING", KEYSPACE, currentDenseTable()),
- "value5");
- assertInvalidMessage("Filtering is not supported on SuperColumn tables",
- String.format("SELECT * FROM %s.%s WHERE value = 'value2' ALLOW FILTERING", KEYSPACE, currentDenseTable()));
- assertInvalidMessage("Filtering is not supported on SuperColumn tables",
- String.format("SELECT * FROM %s.%s WHERE column2 = 1 ALLOW FILTERING", KEYSPACE, currentDenseTable()));
- }
-
- @Test
- public void testLwt() throws Throwable
- {
- assertRows(execute(String.format("INSERT INTO %s.%s (key, column1, column2, value) VALUES (?, ?, ?, ?) IF NOT EXISTS", KEYSPACE, currentDenseTable()),
- "key1", "val1", 1, "value1"),
- row(true));
- assertRows(execute(String.format("SELECT * FROM %s.%s", KEYSPACE, currentDenseTable())),
- row("key1", "val1", 1, "value1"));
- assertRows(execute(String.format("INSERT INTO %s.%s (key, column1, column2, value) VALUES (?, ?, ?, ?) IF NOT EXISTS", KEYSPACE, currentDenseTable()),
- "key1", "val1", 1, "value1"),
- row(false, "key1", "val1", 1, "value1"));
-
- // in 2.2 this query was a no-op
- assertInvalidMessage("Lightweight transactions on SuperColumn tables are only supported with supplied SuperColumn key",
- String.format("UPDATE %s.%s SET value = 'changed' WHERE key = ? AND column1 = ? IF value = ?", KEYSPACE, currentDenseTable()));
-
- assertRows(execute(String.format("UPDATE %s.%s SET value = 'changed' WHERE key = ? AND column1 = ? AND column2 = ? IF value = ?", KEYSPACE, currentDenseTable()),
- "key1", "val1", 1, "value1"),
- row(true));
- assertRows(execute(String.format("SELECT * FROM %s.%s", KEYSPACE, currentDenseTable())),
- row("key1", "val1", 1, "changed"));
- assertRows(execute(String.format("UPDATE %s.%s SET value = 'changed' WHERE key = ? AND column1 = ? AND column2 = ? IF value = ?", KEYSPACE, currentDenseTable()),
- "key1", "val1", 1, "value1"),
- row(false, "changed"));
-
- assertRows(execute(String.format("UPDATE %s.%s SET value = 'changed2' WHERE key = ? AND column1 = ? AND column2 = ? IF value > ?", KEYSPACE, currentDenseTable()),
- "key1", "val1", 1, "a"),
- row(true));
- assertRows(execute(String.format("SELECT * FROM %s.%s", KEYSPACE, currentDenseTable())),
- row("key1", "val1", 1, "changed2"));
- assertRows(execute(String.format("UPDATE %s.%s SET value = 'changed2' WHERE key = ? AND column1 = ? AND column2 = ? IF value < ?", KEYSPACE, currentDenseTable()),
- "key1", "val1", 1, "a"),
- row(false, "changed2"));
-
- assertInvalidMessage("PRIMARY KEY column 'column2' cannot have IF conditions",
- String.format("UPDATE %s.%s SET value = 'changed2' WHERE key = ? AND column1 = ? AND column2 = ? IF value > ? AND column2 = ?", KEYSPACE, currentDenseTable()));
-
- assertInvalidMessage("Lightweight transactions on SuperColumn tables are only supported with supplied SuperColumn key",
- String.format("UPDATE %s.%s SET value = 'changed2' WHERE key = ? AND column1 = ? IF value > ?", KEYSPACE, currentDenseTable()));
-
- execute(String.format("DELETE FROM %s.%s WHERE key = 'key1' AND column1 = 'val1' AND column2 = 1 IF EXISTS", KEYSPACE, currentDenseTable()));
- assertEmpty(execute(String.format("SELECT * FROM %s.%s", KEYSPACE, currentDenseTable())));
-
- execute(String.format("INSERT INTO %s.%s (key, column1, column2, value) VALUES (?, ?, ?, ?)", KEYSPACE, currentDenseTable()),
- "key1", "val1", 1, "value1");
-
- assertRows(execute(String.format("DELETE FROM %s.%s WHERE key = 'key1' AND column1 = 'val1' AND column2 = 1 IF value = 'value1'", KEYSPACE, currentDenseTable())),
- row(true));
- assertEmpty(execute(String.format("SELECT * FROM %s.%s", KEYSPACE, currentDenseTable())));
-
- assertRows(execute(String.format("DELETE FROM %s.%s WHERE key = 'key1' AND column1 = 'val1' AND column2 = 1 IF value = 'value1'", KEYSPACE, currentDenseTable())),
- row(false));
- assertEmpty(execute(String.format("SELECT * FROM %s.%s", KEYSPACE, currentDenseTable())));
- }
-
- @Test
- public void testCqlAggregateFunctions() throws Throwable
- {
- populateDenseTable();
- populateSparseTable();
-
- assertRows(execute(String.format("select count(*) from %s.%s", KEYSPACE, currentDenseTable())),
- row(8L));
- assertRows(execute(String.format("select count(*) from %s.%s", KEYSPACE, currentSparseTable())),
- row(4L));
-
- assertRows(execute(String.format("select count(*) from %s.%s where key = ? AND column1 = ?", KEYSPACE, currentDenseTable()), "key1", "val1"),
- row(2L));
- assertRows(execute(String.format("select count(*) from %s.%s where key = ? AND column1 = ?", KEYSPACE, currentSparseTable()), "key1", "val1"),
- row(1L));
- assertRows(execute(String.format("select count(*) from %s.%s where key = ?", KEYSPACE, currentSparseTable()), "key1"),
- row(2L));
-
- assertRows(execute(String.format("select max(value) from %s.%s", KEYSPACE, currentDenseTable())),
- row("value5"));
- assertRows(execute(String.format("select max(col1) from %s.%s", KEYSPACE, currentSparseTable())),
- row(3L));
-
- assertRows(execute(String.format("select avg(column2) from %s.%s", KEYSPACE, currentDenseTable())),
- row(3));
- assertRows(execute(String.format("select avg(col1) from %s.%s", KEYSPACE, currentSparseTable())),
- row(3L));
- }
-
- private void populateDenseTable() throws Throwable
- {
- Cassandra.Client client = getClient();
-
- Mutation mutation = new Mutation();
- ColumnOrSuperColumn csoc = new ColumnOrSuperColumn();
- csoc.setSuper_column(getSuperColumnForInsert(ByteBufferUtil.bytes("val1"),
- Arrays.asList(getColumnForInsert(ByteBufferUtil.bytes(1), ByteBufferUtil.bytes("value1")),
- getColumnForInsert(ByteBufferUtil.bytes(2), ByteBufferUtil.bytes("value2")))));
- mutation.setColumn_or_supercolumn(csoc);
-
- Mutation mutation2 = new Mutation();
- ColumnOrSuperColumn csoc2 = new ColumnOrSuperColumn();
- csoc2.setSuper_column(getSuperColumnForInsert(ByteBufferUtil.bytes("val2"),
- Arrays.asList(getColumnForInsert(ByteBufferUtil.bytes(4), ByteBufferUtil.bytes("value4")),
- getColumnForInsert(ByteBufferUtil.bytes(5), ByteBufferUtil.bytes("value5")))));
- mutation2.setColumn_or_supercolumn(csoc2);
-
- client.batch_mutate(Collections.singletonMap(ByteBufferUtil.bytes("key1"),
- Collections.singletonMap(currentDenseTable(), Arrays.asList(mutation, mutation2))),
- ONE);
-
- client.batch_mutate(Collections.singletonMap(ByteBufferUtil.bytes("key2"),
- Collections.singletonMap(currentDenseTable(), Arrays.asList(mutation, mutation2))),
- ONE);
- }
-
- private void populateSparseTable() throws Throwable
- {
- Cassandra.Client client = getClient();
-
- Mutation mutation = new Mutation();
- ColumnOrSuperColumn csoc = new ColumnOrSuperColumn();
- csoc.setSuper_column(getSuperColumnForInsert(ByteBufferUtil.bytes("val1"),
- Arrays.asList(getColumnForInsert(ByteBufferUtil.bytes("value1"), ByteBufferUtil.bytes(1L)),
- getColumnForInsert(ByteBufferUtil.bytes("value2"), ByteBufferUtil.bytes(2L)),
- getColumnForInsert(ByteBufferUtil.bytes("col1"), ByteBufferUtil.bytes(3L)),
- getColumnForInsert(ByteBufferUtil.bytes("col2"), ByteBufferUtil.bytes(4L)))));
- mutation.setColumn_or_supercolumn(csoc);
-
- Mutation mutation2 = new Mutation();
- ColumnOrSuperColumn csoc2 = new ColumnOrSuperColumn();
- csoc2.setSuper_column(getSuperColumnForInsert(ByteBufferUtil.bytes("val2"),
- Arrays.asList(getColumnForInsert(ByteBufferUtil.bytes("value1"), ByteBufferUtil.bytes(1L)),
- getColumnForInsert(ByteBufferUtil.bytes("value2"), ByteBufferUtil.bytes(2L)),
- getColumnForInsert(ByteBufferUtil.bytes("col1"), ByteBufferUtil.bytes(3L)),
- getColumnForInsert(ByteBufferUtil.bytes("col2"), ByteBufferUtil.bytes(4L)))));
- mutation2.setColumn_or_supercolumn(csoc2);
-
- client.batch_mutate(Collections.singletonMap(ByteBufferUtil.bytes("key1"),
- Collections.singletonMap(currentSparseTable(), Arrays.asList(mutation, mutation2))),
- ONE);
-
- client.batch_mutate(Collections.singletonMap(ByteBufferUtil.bytes("key2"),
- Collections.singletonMap(currentSparseTable(), Arrays.asList(mutation, mutation2))),
- ONE);
- }
-
- private void populateCounterTable() throws Throwable
- {
- Cassandra.Client client = getClient();
-
- ColumnParent cp = new ColumnParent(currentCounterTable());
- cp.setSuper_column(ByteBufferUtil.bytes("ck1"));
- client.add(ByteBufferUtil.bytes("key1"),
- cp,
- new CounterColumn(ByteBufferUtil.bytes("counter1"), 10L),
- ONE);
- cp = new ColumnParent(currentCounterTable());
- cp.setSuper_column(ByteBufferUtil.bytes("ck1"));
- client.add(ByteBufferUtil.bytes("key1"),
- cp,
- new CounterColumn(ByteBufferUtil.bytes("counter2"), 5L),
- ONE);
- cp = new ColumnParent(currentCounterTable());
- cp.setSuper_column(ByteBufferUtil.bytes("ck1"));
- client.add(ByteBufferUtil.bytes("key2"),
- cp,
- new CounterColumn(ByteBufferUtil.bytes("counter1"), 10L),
- ONE);
- cp = new ColumnParent(currentCounterTable());
- cp.setSuper_column(ByteBufferUtil.bytes("ck1"));
- client.add(ByteBufferUtil.bytes("key2"),
- cp,
- new CounterColumn(ByteBufferUtil.bytes("counter2"), 5L),
- ONE);
- }
-
- private String currentCounterTable()
- {
- return currentTable() + "_counter";
- }
-
- private String currentSparseTable()
- {
- return currentTable() + "_sparse";
- }
-
- private String currentDenseTable()
- {
- return currentTable();
- }
-
- private Column getColumnForInsert(ByteBuffer columnName, ByteBuffer value)
- {
- Column column = new Column();
- column.setName(columnName);
- column.setValue(value);
- column.setTimestamp(System.currentTimeMillis());
- return column;
- }
-
- private SuperColumn getSuperColumnForInsert(ByteBuffer columnName, List<Column> columns)
- {
- SuperColumn column = new SuperColumn();
- column.setName(columnName);
- for (Column c : columns)
- column.addToColumns(c);
- return column;
- }
-
- public void beforeAndAfterFlush(CheckedFunction runnable) throws Throwable
- {
- runnable.apply();
- flushAll();
- runnable.apply();
- }
-
- private void flushAll()
- {
- for (String cfName : new String[]{ currentTable(), currentSparseTable(), currentCounterTable() })
- Keyspace.open(KEYSPACE).getColumnFamilyStore(cfName);
- }
-}
diff --git a/test/unit/org/apache/cassandra/cql3/validation/entities/CollectionsTest.java b/test/unit/org/apache/cassandra/cql3/validation/entities/CollectionsTest.java
index 5c4e935..54dd0c4 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/entities/CollectionsTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/entities/CollectionsTest.java
@@ -21,7 +21,14 @@
import org.junit.Test;
+import com.datastax.driver.core.ColumnDefinitions;
+import com.datastax.driver.core.DataType;
+import com.datastax.driver.core.ResultSet;
+import com.datastax.driver.core.Session;
import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.cql3.ColumnSpecification;
+import org.apache.cassandra.cql3.UntypedResultSet;
+import org.apache.cassandra.db.marshal.UTF8Type;
import org.apache.cassandra.utils.FBUtilities;
import static org.junit.Assert.assertEquals;
@@ -115,13 +122,14 @@
);
execute("UPDATE %s SET s += ? WHERE k = 0", set("v5"));
- execute("UPDATE %s SET s += ? WHERE k = 0", set("v6"));
+ execute("UPDATE %s SET s += {'v6'} WHERE k = 0");
assertRows(execute("SELECT s FROM %s WHERE k = 0"),
row(set("v5", "v6", "v7"))
);
- execute("UPDATE %s SET s -= ? WHERE k = 0", set("v6", "v5"));
+ execute("UPDATE %s SET s -= ? WHERE k = 0", set("v5"));
+ execute("UPDATE %s SET s -= {'v6'} WHERE k = 0");
assertRows(execute("SELECT s FROM %s WHERE k = 0"),
row(set("v7"))
@@ -144,6 +152,15 @@
{
createTable("CREATE TABLE %s (k int PRIMARY KEY, m map<text, int>)");
+ assertInvalidMessage("Value for a map addition has to be a map, but was: '{1}'",
+ "UPDATE %s SET m = m + {1} WHERE k = 0;");
+ assertInvalidMessage("Not enough bytes to read a map",
+ "UPDATE %s SET m += ? WHERE k = 0", set("v1"));
+ assertInvalidMessage("Value for a map substraction has to be a set, but was: '{'v1': 1}'",
+ "UPDATE %s SET m = m - {'v1': 1} WHERE k = 0", map("v1", 1));
+ assertInvalidMessage("Unexpected extraneous bytes after set value",
+ "UPDATE %s SET m -= ? WHERE k = 0", map("v1", 1));
+
execute("INSERT INTO %s(k, m) VALUES (0, ?)", map("v1", 1, "v2", 2));
assertRows(execute("SELECT m FROM %s WHERE k = 0"),
@@ -193,6 +210,18 @@
row(map("v5", 5, "v6", 6))
);
+ execute("UPDATE %s SET m += {'v7': 7} WHERE k = 0");
+
+ assertRows(execute("SELECT m FROM %s WHERE k = 0"),
+ row(map("v5", 5, "v6", 6, "v7", 7))
+ );
+
+ execute("UPDATE %s SET m -= {'v7'} WHERE k = 0");
+
+ assertRows(execute("SELECT m FROM %s WHERE k = 0"),
+ row(map("v5", 5, "v6", 6))
+ );
+
execute("DELETE m[?] FROM %s WHERE k = 0", "v6");
assertRows(execute("SELECT m FROM %s WHERE k = 0"),
@@ -276,6 +305,10 @@
execute("UPDATE %s SET l = l - ? WHERE k=0", list("v11"));
assertRows(execute("SELECT l FROM %s WHERE k = 0"), row((Object) null));
+
+ execute("UPDATE %s SET l = l + ? WHERE k = 0", list("v1", "v2", "v1", "v2", "v1", "v2"));
+ execute("UPDATE %s SET l = l - ? WHERE k=0", list("v1", "v2"));
+ assertRows(execute("SELECT l FROM %s WHERE k = 0"), row((Object) null));
}
@Test
@@ -543,16 +576,6 @@
}
/**
- * Migrated from cql_tests.py:TestCQL.collection_compact_test()
- */
- @Test
- public void testCompactCollections() throws Throwable
- {
- String tableName = KEYSPACE + "." + createTableName();
- assertInvalid(String.format("CREATE TABLE %s (user ascii PRIMARY KEY, mails list < text >) WITH COMPACT STORAGE;", tableName));
- }
-
- /**
* Migrated from cql_tests.py:TestCQL.collection_function_test()
*/
@Test
@@ -564,17 +587,54 @@
assertInvalid("SELECT writetime(l) FROM %s WHERE k = 0");
}
- /**
- * Migrated from cql_tests.py:TestCQL.bug_5376()
- */
@Test
- public void testInClauseWithCollections() throws Throwable
+ public void testInRestrictionWithCollection() throws Throwable
{
- createTable("CREATE TABLE %s (key text, c bigint, v text, x set < text >, PRIMARY KEY(key, c) )");
+ for (boolean frozen : new boolean[]{true, false})
+ {
+ createTable(frozen ? "CREATE TABLE %s (a int, b int, c int, d frozen<list<int>>, e frozen<map<int, int>>, f frozen<set<int>>, PRIMARY KEY (a, b, c))"
+ : "CREATE TABLE %s (a int, b int, c int, d list<int>, e map<int, int>, f set<int>, PRIMARY KEY (a, b, c))");
- assertInvalid("select * from %s where key = 'foo' and c in (1,3,4)");
+ execute("INSERT INTO %s (a, b, c, d, e, f) VALUES (1, 1, 1, [1, 2], {1: 2}, {1, 2})");
+ execute("INSERT INTO %s (a, b, c, d, e, f) VALUES (1, 1, 2, [1, 3], {1: 3}, {1, 3})");
+ execute("INSERT INTO %s (a, b, c, d, e, f) VALUES (1, 1, 3, [1, 4], {1: 4}, {1, 4})");
+ execute("INSERT INTO %s (a, b, c, d, e, f) VALUES (1, 2, 3, [1, 3], {1: 3}, {1, 3})");
+ execute("INSERT INTO %s (a, b, c, d, e, f) VALUES (1, 2, 4, [1, 3], {1: 3}, {1, 3})");
+ execute("INSERT INTO %s (a, b, c, d, e, f) VALUES (2, 1, 1, [1, 2], {2: 2}, {1, 2})");
+
+ beforeAndAfterFlush(() -> {
+ assertRows(execute("SELECT * FROM %s WHERE a in (1,2)"),
+ row(1, 1, 1, list(1, 2), map(1, 2), set(1, 2)),
+ row(1, 1, 2, list(1, 3), map(1, 3), set(1, 3)),
+ row(1, 1, 3, list(1, 4), map(1, 4), set(1, 4)),
+ row(1, 2, 3, list(1, 3), map(1, 3), set(1, 3)),
+ row(1, 2, 4, list(1, 3), map(1, 3), set(1, 3)),
+ row(2, 1, 1, list(1, 2), map(2, 2), set(1, 2)));
+
+ assertRows(execute("SELECT * FROM %s WHERE a = 1 AND b IN (1,2)"),
+ row(1, 1, 1, list(1, 2), map(1, 2), set(1, 2)),
+ row(1, 1, 2, list(1, 3), map(1, 3), set(1, 3)),
+ row(1, 1, 3, list(1, 4), map(1, 4), set(1, 4)),
+ row(1, 2, 3, list(1, 3), map(1, 3), set(1, 3)),
+ row(1, 2, 4, list(1, 3), map(1, 3), set(1, 3)));
+
+ assertRows(execute("SELECT * FROM %s WHERE a = 1 AND b = 1 AND c in (1,2)"),
+ row(1, 1, 1, list(1, 2), map(1, 2), set(1, 2)),
+ row(1, 1, 2, list(1, 3), map(1, 3), set(1, 3)));
+
+ assertRows(execute("SELECT * FROM %s WHERE a = 1 AND b IN (1, 2) AND c in (1,2,3)"),
+ row(1, 1, 1, list(1, 2), map(1, 2), set(1, 2)),
+ row(1, 1, 2, list(1, 3), map(1, 3), set(1, 3)),
+ row(1, 1, 3, list(1, 4), map(1, 4), set(1, 4)),
+ row(1, 2, 3, list(1, 3), map(1, 3), set(1, 3)));
+
+ assertRows(execute("SELECT * FROM %s WHERE a = 1 AND b IN (1, 2) AND c in (1,2,3) AND d CONTAINS 4 ALLOW FILTERING"),
+ row(1, 1, 3, list(1, 4), map(1, 4), set(1, 4)));
+ });
+ }
}
+
/**
* Test for bug #5795,
* migrated from cql_tests.py:TestCQL.nonpure_function_collection_test()
@@ -622,13 +682,13 @@
}
@Test
- public void testDropAndReaddFrozenCollection() throws Throwable
+ public void testDropAndReaddDroppedCollection() throws Throwable
{
- createTable("create table %s (k int primary key, v frozen<set<text>>, x int)");
+ createTable("create table %s (k int primary key, v set<text>, x int)");
execute("insert into %s (k, v) VALUES (0, {'fffffffff'})");
flush();
execute("alter table %s drop v");
- assertInvalid("alter table %s add v frozen<set<int>>");
+ execute("alter table %s add v set<text>");
}
@Test
@@ -1037,6 +1097,790 @@
}
@Test
+ public void testMapOperation() throws Throwable
+ {
+ createTable("CREATE TABLE %s (k int, c int, l text, " +
+ "m map<text, text>, " +
+ "fm frozen<map<text, text>>, " +
+ "sm map<text, text> STATIC, " +
+ "fsm frozen<map<text, text>> STATIC, " +
+ "o int, PRIMARY KEY (k, c))");
+
+ execute("INSERT INTO %s(k, c, l, m, fm, sm, fsm, o) VALUES (0, 0, 'foobar', ?, ?, ?, ?, 42)",
+ map("22", "value22", "333", "value333"),
+ map("1", "fvalue1", "22", "fvalue22", "333", "fvalue333"),
+ map("22", "svalue22", "333", "svalue333"),
+ map("1", "fsvalue1", "22", "fsvalue22", "333", "fsvalue333"));
+
+ execute("INSERT INTO %s(k, c, l, m, fm, sm, fsm, o) VALUES (2, 0, 'row2', ?, ?, ?, ?, 88)",
+ map("22", "2value22", "333", "2value333"),
+ map("1", "2fvalue1", "22", "2fvalue22", "333", "2fvalue333"),
+ map("22", "2svalue22", "333", "2svalue333"),
+ map("1", "2fsvalue1", "22", "2fsvalue22", "333", "2fsvalue333"));
+
+ flush();
+
+ execute("UPDATE %s SET m = m + ? WHERE k = 0 AND c = 0",
+ map("1", "value1"));
+
+ execute("UPDATE %s SET sm = sm + ? WHERE k = 0",
+ map("1", "svalue1"));
+
+ flush();
+
+ assertRows(execute("SELECT m['22'] FROM %s WHERE k = 0 AND c = 0"),
+ row("value22")
+ );
+ assertRows(execute("SELECT m['1'], m['22'], m['333'] FROM %s WHERE k = 0 AND c = 0"),
+ row("value1", "value22", "value333")
+ );
+ assertRows(execute("SELECT m['2'..'3'] FROM %s WHERE k = 0 AND c = 0"),
+ row(map("22", "value22"))
+ );
+
+ execute("INSERT INTO %s(k, c, l, m, fm, o) VALUES (0, 1, 'foobar', ?, ?, 42)",
+ map("1", "value1_2", "333", "value333_2"),
+ map("1", "fvalue1_2", "333", "fvalue333_2"));
+
+ assertRows(execute("SELECT c, m['1'], fm['1'] FROM %s WHERE k = 0"),
+ row(0, "value1", "fvalue1"),
+ row(1, "value1_2", "fvalue1_2")
+ );
+ assertRows(execute("SELECT c, sm['1'], fsm['1'] FROM %s WHERE k = 0"),
+ row(0, "svalue1", "fsvalue1"),
+ row(1, "svalue1", "fsvalue1")
+ );
+
+ assertRows(execute("SELECT c, m['1'], fm['1'] FROM %s WHERE k = 0 AND c = 0"),
+ row(0, "value1", "fvalue1")
+ );
+
+ assertRows(execute("SELECT c, m['1'], fm['1'] FROM %s WHERE k = 0"),
+ row(0, "value1", "fvalue1"),
+ row(1, "value1_2", "fvalue1_2")
+ );
+
+ assertColumnNames(execute("SELECT k, l, m['1'] as mx, o FROM %s WHERE k = 0"),
+ "k", "l", "mx", "o");
+ assertColumnNames(execute("SELECT k, l, m['1'], o FROM %s WHERE k = 0"),
+ "k", "l", "m['1']", "o");
+
+ assertRows(execute("SELECT k, l, m['22'], o FROM %s WHERE k = 0"),
+ row(0, "foobar", "value22", 42),
+ row(0, "foobar", null, 42)
+ );
+ assertColumnNames(execute("SELECT k, l, m['22'], o FROM %s WHERE k = 0"),
+ "k", "l", "m['22']", "o");
+
+ assertRows(execute("SELECT k, l, m['333'], o FROM %s WHERE k = 0"),
+ row(0, "foobar", "value333", 42),
+ row(0, "foobar", "value333_2", 42)
+ );
+
+ assertRows(execute("SELECT k, l, m['foobar'], o FROM %s WHERE k = 0"),
+ row(0, "foobar", null, 42),
+ row(0, "foobar", null, 42)
+ );
+
+ assertRows(execute("SELECT k, l, m['1'..'22'], o FROM %s WHERE k = 0"),
+ row(0, "foobar", map("1", "value1",
+ "22", "value22"), 42),
+ row(0, "foobar", map("1", "value1_2"), 42)
+ );
+
+ assertRows(execute("SELECT k, l, m[''..'23'], o FROM %s WHERE k = 0"),
+ row(0, "foobar", map("1", "value1",
+ "22", "value22"), 42),
+ row(0, "foobar", map("1", "value1_2"), 42)
+ );
+ assertColumnNames(execute("SELECT k, l, m[''..'23'], o FROM %s WHERE k = 0"),
+ "k", "l", "m[''..'23']", "o");
+
+ assertRows(execute("SELECT k, l, m['2'..'3'], o FROM %s WHERE k = 0"),
+ row(0, "foobar", map("22", "value22"), 42),
+ row(0, "foobar", null, 42)
+ );
+
+ assertRows(execute("SELECT k, l, m['22'..], o FROM %s WHERE k = 0"),
+ row(0, "foobar", map("22", "value22",
+ "333", "value333"), 42),
+ row(0, "foobar", map("333", "value333_2"), 42)
+ );
+
+ assertRows(execute("SELECT k, l, m[..'22'], o FROM %s WHERE k = 0"),
+ row(0, "foobar", map("1", "value1",
+ "22", "value22"), 42),
+ row(0, "foobar", map("1", "value1_2"), 42)
+ );
+
+ assertRows(execute("SELECT k, l, m, o FROM %s WHERE k = 0"),
+ row(0, "foobar", map("1", "value1",
+ "22", "value22",
+ "333", "value333"), 42),
+ row(0, "foobar", map("1", "value1_2",
+ "333", "value333_2"), 42)
+ );
+
+ assertRows(execute("SELECT k, l, m, m as m2, o FROM %s WHERE k = 0"),
+ row(0, "foobar", map("1", "value1",
+ "22", "value22",
+ "333", "value333"),
+ map("1", "value1",
+ "22", "value22",
+ "333", "value333"), 42),
+ row(0, "foobar", map("1", "value1_2",
+ "333", "value333_2"),
+ map("1", "value1_2",
+ "333", "value333_2"), 42)
+ );
+
+ // with UDF as slice arg
+
+ String f = createFunction(KEYSPACE, "text",
+ "CREATE FUNCTION %s(arg text) " +
+ "CALLED ON NULL INPUT " +
+ "RETURNS TEXT " +
+ "LANGUAGE java AS 'return arg;'");
+
+ assertRows(execute("SELECT k, c, l, m[" + f +"('1').." + f +"('22')], o FROM %s WHERE k = 0"),
+ row(0, 0, "foobar", map("1", "value1",
+ "22", "value22"), 42),
+ row(0, 1, "foobar", map("1", "value1_2"), 42)
+ );
+
+ assertRows(execute("SELECT k, c, l, m[" + f +"(?).." + f +"(?)], o FROM %s WHERE k = 0", "1", "22"),
+ row(0, 0, "foobar", map("1", "value1",
+ "22", "value22"), 42),
+ row(0, 1, "foobar", map("1", "value1_2"), 42)
+ );
+
+ // with UDF taking a map
+
+ f = createFunction(KEYSPACE, "map<text,text>",
+ "CREATE FUNCTION %s(m text) " +
+ "CALLED ON NULL INPUT " +
+ "RETURNS TEXT " +
+ "LANGUAGE java AS $$return m;$$");
+
+ assertRows(execute("SELECT k, c, " + f + "(m['1']) FROM %s WHERE k = 0"),
+ row(0, 0, "value1"),
+ row(0, 1, "value1_2"));
+
+ // with UDF taking multiple cols
+
+ f = createFunction(KEYSPACE, "map<text,text>,map<text,text>,int,int",
+ "CREATE FUNCTION %s(m1 map<text,text>, m2 text, k int, c int) " +
+ "CALLED ON NULL INPUT " +
+ "RETURNS TEXT " +
+ "LANGUAGE java AS $$return m1.get(\"1\") + ':' + m2 + ':' + k + ':' + c;$$");
+
+ assertRows(execute("SELECT " + f + "(m, m['1'], k, c) FROM %s WHERE k = 0"),
+ row("value1:value1:0:0"),
+ row("value1_2:value1_2:0:1"));
+
+ // with nested UDF + aggregation and multiple cols
+
+ f = createFunction(KEYSPACE, "int,int",
+ "CREATE FUNCTION %s(k int, c int) " +
+ "CALLED ON NULL INPUT " +
+ "RETURNS int " +
+ "LANGUAGE java AS $$return k + c;$$");
+
+ assertColumnNames(execute("SELECT max(" + f + "(k, c)) as sel1, max(" + f + "(k, c)) FROM %s WHERE k = 0"),
+ "sel1", "system.max(" + f + "(k, c))");
+ assertRows(execute("SELECT max(" + f + "(k, c)) as sel1, max(" + f + "(k, c)) FROM %s WHERE k = 0"),
+ row(1, 1));
+
+ assertColumnNames(execute("SELECT max(" + f + "(k, c)) as sel1, max(" + f + "(k, c)) FROM %s"),
+ "sel1", "system.max(" + f + "(k, c))");
+ assertRows(execute("SELECT max(" + f + "(k, c)) as sel1, max(" + f + "(k, c)) FROM %s"),
+ row(2, 2));
+
+ // prepared parameters
+
+ assertRows(execute("SELECT c, m[?], fm[?] FROM %s WHERE k = 0", "1", "1"),
+ row(0, "value1", "fvalue1"),
+ row(1, "value1_2", "fvalue1_2")
+ );
+ assertRows(execute("SELECT c, sm[?], fsm[?] FROM %s WHERE k = 0", "1", "1"),
+ row(0, "svalue1", "fsvalue1"),
+ row(1, "svalue1", "fsvalue1")
+ );
+ assertRows(execute("SELECT k, l, m[?..?], o FROM %s WHERE k = 0", "1", "22"),
+ row(0, "foobar", map("1", "value1",
+ "22", "value22"), 42),
+ row(0, "foobar", map("1", "value1_2"), 42)
+ );
+ }
+
+ @Test
+ public void testMapOperationWithIntKey() throws Throwable
+ {
+ // used type "int" as map key intentionally since CQL parsing relies on "BigInteger"
+
+ createTable("CREATE TABLE %s (k int, c int, l text, " +
+ "m map<int, text>, " +
+ "fm frozen<map<int, text>>, " +
+ "sm map<int, text> STATIC, " +
+ "fsm frozen<map<int, text>> STATIC, " +
+ "o int, PRIMARY KEY (k, c))");
+
+ execute("INSERT INTO %s(k, c, l, m, fm, sm, fsm, o) VALUES (0, 0, 'foobar', ?, ?, ?, ?, 42)",
+ map(22, "value22", 333, "value333"),
+ map(1, "fvalue1", 22, "fvalue22", 333, "fvalue333"),
+ map(22, "svalue22", 333, "svalue333"),
+ map(1, "fsvalue1", 22, "fsvalue22", 333, "fsvalue333"));
+
+ execute("INSERT INTO %s(k, c, l, m, fm, sm, fsm, o) VALUES (2, 0, 'row2', ?, ?, ?, ?, 88)",
+ map(22, "2value22", 333, "2value333"),
+ map(1, "2fvalue1", 22, "2fvalue22", 333, "2fvalue333"),
+ map(22, "2svalue22", 333, "2svalue333"),
+ map(1, "2fsvalue1", 22, "2fsvalue22", 333, "2fsvalue333"));
+
+ flush();
+
+ execute("UPDATE %s SET m = m + ? WHERE k = 0 AND c = 0",
+ map(1, "value1"));
+
+ execute("UPDATE %s SET sm = sm + ? WHERE k = 0",
+ map(1, "svalue1"));
+
+ flush();
+
+ assertRows(execute("SELECT m[22] FROM %s WHERE k = 0 AND c = 0"),
+ row("value22")
+ );
+ assertRows(execute("SELECT m[1], m[22], m[333] FROM %s WHERE k = 0 AND c = 0"),
+ row("value1", "value22", "value333")
+ );
+ assertRows(execute("SELECT m[20 .. 25] FROM %s WHERE k = 0 AND c = 0"),
+ row(map(22, "value22"))
+ );
+
+ execute("INSERT INTO %s(k, c, l, m, fm, o) VALUES (0, 1, 'foobar', ?, ?, 42)",
+ map(1, "value1_2", 333, "value333_2"),
+ map(1, "fvalue1_2", 333, "fvalue333_2"));
+
+ assertRows(execute("SELECT c, m[1], fm[1] FROM %s WHERE k = 0"),
+ row(0, "value1", "fvalue1"),
+ row(1, "value1_2", "fvalue1_2")
+ );
+ assertRows(execute("SELECT c, sm[1], fsm[1] FROM %s WHERE k = 0"),
+ row(0, "svalue1", "fsvalue1"),
+ row(1, "svalue1", "fsvalue1")
+ );
+
+ // with UDF as slice arg
+
+ String f = createFunction(KEYSPACE, "int",
+ "CREATE FUNCTION %s(arg int) " +
+ "CALLED ON NULL INPUT " +
+ "RETURNS int " +
+ "LANGUAGE java AS 'return arg;'");
+
+ assertRows(execute("SELECT k, c, l, m[" + f +"(1).." + f +"(22)], o FROM %s WHERE k = 0"),
+ row(0, 0, "foobar", map(1, "value1",
+ 22, "value22"), 42),
+ row(0, 1, "foobar", map(1, "value1_2"), 42)
+ );
+
+ assertRows(execute("SELECT k, c, l, m[" + f +"(?).." + f +"(?)], o FROM %s WHERE k = 0", 1, 22),
+ row(0, 0, "foobar", map(1, "value1",
+ 22, "value22"), 42),
+ row(0, 1, "foobar", map(1, "value1_2"), 42)
+ );
+
+ // with UDF taking a map
+
+ f = createFunction(KEYSPACE, "map<int,text>",
+ "CREATE FUNCTION %s(m text) " +
+ "CALLED ON NULL INPUT " +
+ "RETURNS TEXT " +
+ "LANGUAGE java AS $$return m;$$");
+
+ assertRows(execute("SELECT k, c, " + f + "(m[1]) FROM %s WHERE k = 0"),
+ row(0, 0, "value1"),
+ row(0, 1, "value1_2"));
+
+ // with UDF taking multiple cols
+
+ f = createFunction(KEYSPACE, "map<int,text>,map<int,text>,int,int",
+ "CREATE FUNCTION %s(m1 map<int,text>, m2 text, k int, c int) " +
+ "CALLED ON NULL INPUT " +
+ "RETURNS TEXT " +
+ "LANGUAGE java AS $$return m1.get(1) + ':' + m2 + ':' + k + ':' + c;$$");
+
+ assertRows(execute("SELECT " + f + "(m, m[1], k, c) FROM %s WHERE k = 0"),
+ row("value1:value1:0:0"),
+ row("value1_2:value1_2:0:1"));
+
+ // with nested UDF + aggregation and multiple cols
+
+ f = createFunction(KEYSPACE, "int,int",
+ "CREATE FUNCTION %s(k int, c int) " +
+ "CALLED ON NULL INPUT " +
+ "RETURNS int " +
+ "LANGUAGE java AS $$return k + c;$$");
+
+ assertColumnNames(execute("SELECT max(" + f + "(k, c)) as sel1, max(" + f + "(k, c)) FROM %s WHERE k = 0"),
+ "sel1", "system.max(" + f + "(k, c))");
+ assertRows(execute("SELECT max(" + f + "(k, c)) as sel1, max(" + f + "(k, c)) FROM %s WHERE k = 0"),
+ row(1, 1));
+
+ assertColumnNames(execute("SELECT max(" + f + "(k, c)) as sel1, max(" + f + "(k, c)) FROM %s"),
+ "sel1", "system.max(" + f + "(k, c))");
+ assertRows(execute("SELECT max(" + f + "(k, c)) as sel1, max(" + f + "(k, c)) FROM %s"),
+ row(2, 2));
+
+ // prepared parameters
+
+ assertRows(execute("SELECT c, m[?], fm[?] FROM %s WHERE k = 0", 1, 1),
+ row(0, "value1", "fvalue1"),
+ row(1, "value1_2", "fvalue1_2")
+ );
+ assertRows(execute("SELECT c, sm[?], fsm[?] FROM %s WHERE k = 0", 1, 1),
+ row(0, "svalue1", "fsvalue1"),
+ row(1, "svalue1", "fsvalue1")
+ );
+ assertRows(execute("SELECT k, l, m[?..?], o FROM %s WHERE k = 0", 1, 22),
+ row(0, "foobar", map(1, "value1",
+ 22, "value22"), 42),
+ row(0, "foobar", map(1, "value1_2"), 42)
+ );
+ }
+
+ @Test
+ public void testMapOperationOnPartKey() throws Throwable
+ {
+ createTable("CREATE TABLE %s (k frozen<map<text, text>> PRIMARY KEY, l text, o int)");
+
+ execute("INSERT INTO %s(k, l, o) VALUES (?, 'foobar', 42)", map("1", "value1", "22", "value22", "333", "value333"));
+
+ assertRows(execute("SELECT l, k['1'], o FROM %s WHERE k = ?", map("1", "value1", "22", "value22", "333", "value333")),
+ row("foobar", "value1", 42)
+ );
+
+ assertRows(execute("SELECT l, k['22'], o FROM %s WHERE k = ?", map("1", "value1", "22", "value22", "333", "value333")),
+ row("foobar", "value22", 42)
+ );
+
+ assertRows(execute("SELECT l, k['333'], o FROM %s WHERE k = ?", map("1", "value1", "22", "value22", "333", "value333")),
+ row("foobar", "value333", 42)
+ );
+
+ assertRows(execute("SELECT l, k['foobar'], o FROM %s WHERE k = ?", map("1", "value1", "22", "value22", "333", "value333")),
+ row("foobar", null, 42)
+ );
+
+ assertRows(execute("SELECT l, k['1'..'22'], o FROM %s WHERE k = ?", map("1", "value1", "22", "value22", "333", "value333")),
+ row("foobar", map("1", "value1",
+ "22", "value22"), 42)
+ );
+
+ assertRows(execute("SELECT l, k[''..'23'], o FROM %s WHERE k = ?", map("1", "value1", "22", "value22", "333", "value333")),
+ row("foobar", map("1", "value1",
+ "22", "value22"), 42)
+ );
+
+ assertRows(execute("SELECT l, k['2'..'3'], o FROM %s WHERE k = ?", map("1", "value1", "22", "value22", "333", "value333")),
+ row("foobar", map("22", "value22"), 42)
+ );
+
+ assertRows(execute("SELECT l, k['22'..], o FROM %s WHERE k = ?", map("1", "value1", "22", "value22", "333", "value333")),
+ row("foobar", map("22", "value22",
+ "333", "value333"), 42)
+ );
+
+ assertRows(execute("SELECT l, k[..'22'], o FROM %s WHERE k = ?", map("1", "value1", "22", "value22", "333", "value333")),
+ row("foobar", map("1", "value1",
+ "22", "value22"), 42)
+ );
+
+ assertRows(execute("SELECT l, k, o FROM %s WHERE k = ?", map("1", "value1", "22", "value22", "333", "value333")),
+ row("foobar", map("1", "value1",
+ "22", "value22",
+ "333", "value333"), 42)
+ );
+ }
+
+ @Test
+ public void testMapOperationOnClustKey() throws Throwable
+ {
+ createTable("CREATE TABLE %s (k int, c frozen<map<text, text>>, l text, o int, PRIMARY KEY (k, c))");
+
+ execute("INSERT INTO %s(k, c, l, o) VALUES (0, ?, 'foobar', 42)", map("1", "value1", "22", "value22", "333", "value333"));
+
+ assertRows(execute("SELECT k, l, c['1'], o FROM %s WHERE k = 0 AND c = ?", map("1", "value1", "22", "value22", "333", "value333")),
+ row(0, "foobar", "value1", 42)
+ );
+
+ assertRows(execute("SELECT k, l, c['22'], o FROM %s WHERE k = 0 AND c = ?", map("1", "value1", "22", "value22", "333", "value333")),
+ row(0, "foobar", "value22", 42)
+ );
+
+ assertRows(execute("SELECT k, l, c['333'], o FROM %s WHERE k = 0 AND c = ?", map("1", "value1", "22", "value22", "333", "value333")),
+ row(0, "foobar", "value333", 42)
+ );
+
+ assertRows(execute("SELECT k, l, c['foobar'], o FROM %s WHERE k = 0 AND c = ?", map("1", "value1", "22", "value22", "333", "value333")),
+ row(0, "foobar", null, 42)
+ );
+
+ assertRows(execute("SELECT k, l, c['1'..'22'], o FROM %s WHERE k = 0 AND c = ?", map("1", "value1", "22", "value22", "333", "value333")),
+ row(0, "foobar", map("1", "value1",
+ "22", "value22"), 42)
+ );
+
+ assertRows(execute("SELECT k, l, c[''..'23'], o FROM %s WHERE k = 0 AND c = ?", map("1", "value1", "22", "value22", "333", "value333")),
+ row(0, "foobar", map("1", "value1",
+ "22", "value22"), 42)
+ );
+
+ assertRows(execute("SELECT k, l, c['2'..'3'], o FROM %s WHERE k = 0 AND c = ?", map("1", "value1", "22", "value22", "333", "value333")),
+ row(0, "foobar", map("22", "value22"), 42)
+ );
+
+ assertRows(execute("SELECT k, l, c['22'..], o FROM %s WHERE k = 0 AND c = ?", map("1", "value1", "22", "value22", "333", "value333")),
+ row(0, "foobar", map("22", "value22",
+ "333", "value333"), 42)
+ );
+
+ assertRows(execute("SELECT k, l, c[..'22'], o FROM %s WHERE k = 0 AND c = ?", map("1", "value1", "22", "value22", "333", "value333")),
+ row(0, "foobar", map("1", "value1",
+ "22", "value22"), 42)
+ );
+
+ assertRows(execute("SELECT k, l, c, o FROM %s WHERE k = 0 AND c = ?", map("1", "value1", "22", "value22", "333", "value333")),
+ row(0, "foobar", map("1", "value1",
+ "22", "value22",
+ "333", "value333"), 42)
+ );
+ }
+
+ @Test
+ public void testSetOperation() throws Throwable
+ {
+ createTable("CREATE TABLE %s (k int, c int, l text, " +
+ "s set<text>, " +
+ "fs frozen<set<text>>, " +
+ "ss set<text> STATIC, " +
+ "fss frozen<set<text>> STATIC, " +
+ "o int, PRIMARY KEY (k, c))");
+
+ execute("INSERT INTO %s(k, c, l, s, fs, ss, fss, o) VALUES (0, 0, 'foobar', ?, ?, ?, ?, 42)",
+ set("1", "22", "333"),
+ set("f1", "f22", "f333"),
+ set("s1", "s22", "s333"),
+ set("fs1", "fs22", "fs333"));
+
+ flush();
+
+ execute("UPDATE %s SET s = s + ? WHERE k = 0 AND c = 0", set("22_2"));
+
+ execute("UPDATE %s SET ss = ss + ? WHERE k = 0", set("s22_2"));
+
+ flush();
+
+ execute("INSERT INTO %s(k, c, l, s, o) VALUES (0, 1, 'foobar', ?, 42)",
+ set("22", "333"));
+
+ assertRows(execute("SELECT c, s, fs, ss, fss FROM %s WHERE k = 0"),
+ row(0, set("1", "22", "22_2", "333"), set("f1", "f22", "f333"), set("s1", "s22", "s22_2", "s333"), set("fs1", "fs22", "fs333")),
+ row(1, set("22", "333"), null, set("s1", "s22", "s22_2", "s333"), set("fs1", "fs22", "fs333"))
+ );
+
+ assertRows(execute("SELECT c, s['1'], fs['f1'], ss['s1'], fss['fs1'] FROM %s WHERE k = 0"),
+ row(0, "1", "f1", "s1", "fs1"),
+ row(1, null, null, "s1", "fs1")
+ );
+
+ assertRows(execute("SELECT s['1'], fs['f1'], ss['s1'], fss['fs1'] FROM %s WHERE k = 0 AND c = 0"),
+ row("1", "f1", "s1", "fs1")
+ );
+
+ assertRows(execute("SELECT k, c, l, s['1'], fs['f1'], ss['s1'], fss['fs1'], o FROM %s WHERE k = 0"),
+ row(0, 0, "foobar", "1", "f1", "s1", "fs1", 42),
+ row(0, 1, "foobar", null, null, "s1", "fs1", 42)
+ );
+
+ assertColumnNames(execute("SELECT k, l, s['1'], o FROM %s WHERE k = 0"),
+ "k", "l", "s['1']", "o");
+
+ assertRows(execute("SELECT k, l, s['22'], o FROM %s WHERE k = 0 AND c = 0"),
+ row(0, "foobar", "22", 42)
+ );
+
+ assertRows(execute("SELECT k, l, s['333'], o FROM %s WHERE k = 0 AND c = 0"),
+ row(0, "foobar", "333", 42)
+ );
+
+ assertRows(execute("SELECT k, l, s['foobar'], o FROM %s WHERE k = 0 AND c = 0"),
+ row(0, "foobar", null, 42)
+ );
+
+ assertRows(execute("SELECT k, l, s['1'..'22'], o FROM %s WHERE k = 0 AND c = 0"),
+ row(0, "foobar", set("1", "22"), 42)
+ );
+ assertColumnNames(execute("SELECT k, l, s[''..'22'], o FROM %s WHERE k = 0"),
+ "k", "l", "s[''..'22']", "o");
+
+ assertRows(execute("SELECT k, l, s[''..'23'], o FROM %s WHERE k = 0 AND c = 0"),
+ row(0, "foobar", set("1", "22", "22_2"), 42)
+ );
+
+ assertRows(execute("SELECT k, l, s['2'..'3'], o FROM %s WHERE k = 0 AND c = 0"),
+ row(0, "foobar", set("22", "22_2"), 42)
+ );
+
+ assertRows(execute("SELECT k, l, s['22'..], o FROM %s WHERE k = 0"),
+ row(0, "foobar", set("22", "22_2", "333"), 42),
+ row(0, "foobar", set("22", "333"), 42)
+ );
+
+ assertRows(execute("SELECT k, l, s[..'22'], o FROM %s WHERE k = 0"),
+ row(0, "foobar", set("1", "22"), 42),
+ row(0, "foobar", set("22"), 42)
+ );
+
+ assertRows(execute("SELECT k, l, s, o FROM %s WHERE k = 0"),
+ row(0, "foobar", set("1", "22", "22_2", "333"), 42),
+ row(0, "foobar", set("22", "333"), 42)
+ );
+ }
+
+ @Test
+ public void testCollectionSliceOnMV() throws Throwable
+ {
+ createTable("CREATE TABLE %s (k int, c int, l text, m map<text, text>, o int, PRIMARY KEY (k, c))");
+ assertInvalidMessage("Can only select columns by name when defining a materialized view (got m['abc'])",
+ "CREATE MATERIALIZED VIEW " + KEYSPACE + ".view1 AS SELECT m['abc'] FROM %s WHERE k IS NOT NULL AND c IS NOT NULL AND m IS NOT NULL PRIMARY KEY (c, k)");
+ assertInvalidMessage("Can only select columns by name when defining a materialized view (got m['abc'..'def'])",
+ "CREATE MATERIALIZED VIEW " + KEYSPACE + ".view1 AS SELECT m['abc'..'def'] FROM %s WHERE k IS NOT NULL AND c IS NOT NULL AND m IS NOT NULL PRIMARY KEY (c, k)");
+ }
+
+ @Test
+ public void testElementAccessOnList() throws Throwable
+ {
+ createTable("CREATE TABLE %s (pk int PRIMARY KEY, l list<int>)");
+ execute("INSERT INTO %s (pk, l) VALUES (1, [1, 2, 3])");
+
+ assertInvalidMessage("Element selection is only allowed on sets and maps, but l is a list",
+ "SELECT pk, l[0] FROM %s");
+
+ assertInvalidMessage("Slice selection is only allowed on sets and maps, but l is a list",
+ "SELECT pk, l[1..3] FROM %s");
+ }
+
+ @Test
+ public void testCollectionOperationResultSetMetadata() throws Throwable
+ {
+ createTable("CREATE TABLE %s (k int PRIMARY KEY," +
+ "m map<text, text>," +
+ "fm frozen<map<text, text>>," +
+ "s set<text>," +
+ "fs frozen<set<text>>)");
+
+ execute("INSERT INTO %s (k, m, fm, s, fs) VALUES (?, ?, ?, ?, ?)",
+ 0,
+ map("1", "one", "2", "two"),
+ map("1", "one", "2", "two"),
+ set("1", "2", "3"),
+ set("1", "2", "3"));
+
+ String cql = "SELECT k, " +
+ "m, m['2'], m['2'..'3'], m[..'2'], m['3'..], " +
+ "fm, fm['2'], fm['2'..'3'], fm[..'2'], fm['3'..], " +
+ "s, s['2'], s['2'..'3'], s[..'2'], s['3'..], " +
+ "fs, fs['2'], fs['2'..'3'], fs[..'2'], fs['3'..] " +
+ "FROM " + KEYSPACE + '.' + currentTable() + " WHERE k = 0";
+ UntypedResultSet result = execute(cql);
+ Iterator<ColumnSpecification> meta = result.metadata().iterator();
+ meta.next();
+ for (int i = 0; i < 4; i++)
+ {
+ // take the "full" collection selection
+ ColumnSpecification ref = meta.next();
+ ColumnSpecification selSingle = meta.next();
+ assertEquals(ref.toString(), UTF8Type.instance, selSingle.type);
+ for (int selOrSlice = 0; selOrSlice < 3; selOrSlice++)
+ {
+ ColumnSpecification selSlice = meta.next();
+ assertEquals(ref.toString(), ref.type, selSlice.type);
+ }
+ }
+
+ assertRows(result,
+ row(0,
+ map("1", "one", "2", "two"), "two", map("2", "two"), map("1", "one", "2", "two"), null,
+ map("1", "one", "2", "two"), "two", map("2", "two"), map("1", "one", "2", "two"), map(),
+ set("1", "2", "3"), "2", set("2", "3"), set("1", "2"), set("3"),
+ set("1", "2", "3"), "2", set("2", "3"), set("1", "2"), set("3")));
+
+ Session session = sessionNet();
+ ResultSet rset = session.execute(cql);
+ ColumnDefinitions colDefs = rset.getColumnDefinitions();
+ Iterator<ColumnDefinitions.Definition> colDefIter = colDefs.asList().iterator();
+ colDefIter.next();
+ for (int i = 0; i < 4; i++)
+ {
+ // take the "full" collection selection
+ ColumnDefinitions.Definition ref = colDefIter.next();
+ ColumnDefinitions.Definition selSingle = colDefIter.next();
+ assertEquals(ref.getName(), DataType.NativeType.text(), selSingle.getType());
+ for (int selOrSlice = 0; selOrSlice < 3; selOrSlice++)
+ {
+ ColumnDefinitions.Definition selSlice = colDefIter.next();
+ assertEquals(ref.getName() + ' ' + ref.getType(), ref.getType(), selSlice.getType());
+ }
+ }
+ }
+
+ @Test
+ public void testFrozenCollectionNestedAccess() throws Throwable
+ {
+ createTable("CREATE TABLE %s (k int PRIMARY KEY, m map<text, frozen<map<text, set<int>>>>)");
+
+ execute("INSERT INTO %s(k, m) VALUES (0, ?)", map("1", map("a", set(1, 2, 4), "b", set(3)), "2", map("a", set(2, 4))));
+
+ assertRows(execute("SELECT m[?] FROM %s WHERE k = 0", "1"), row(map("a", set(1, 2, 4), "b", set(3))));
+ assertRows(execute("SELECT m[?][?] FROM %s WHERE k = 0", "1", "a"), row(set(1, 2, 4)));
+ assertRows(execute("SELECT m[?][?][?] FROM %s WHERE k = 0", "1", "a", 2), row(2));
+ assertRows(execute("SELECT m[?][?][?..?] FROM %s WHERE k = 0", "1", "a", 2, 3), row(set(2)));
+
+ // Checks it still work after flush
+ flush();
+
+ assertRows(execute("SELECT m[?] FROM %s WHERE k = 0", "1"), row(map("a", set(1, 2, 4), "b", set(3))));
+ assertRows(execute("SELECT m[?][?] FROM %s WHERE k = 0", "1", "a"), row(set(1, 2, 4)));
+ assertRows(execute("SELECT m[?][?][?] FROM %s WHERE k = 0", "1", "a", 2), row(2));
+ assertRows(execute("SELECT m[?][?][?..?] FROM %s WHERE k = 0", "1", "a", 2, 3), row(set(2)));
+ }
+
+ @Test
+ public void testUDTAndCollectionNestedAccess() throws Throwable
+ {
+ String type = createType("CREATE TYPE %s (s set<int>, m map<text, text>)");
+
+ assertInvalidMessage("Non-frozen UDTs are not allowed inside collections",
+ "CREATE TABLE " + KEYSPACE + ".t (k int PRIMARY KEY, v map<text, " + type + ">)");
+
+ String mapType = "map<text, frozen<" + type + ">>";
+ for (boolean frozen : new boolean[]{false, true})
+ {
+ mapType = frozen ? "frozen<" + mapType + ">" : mapType;
+
+ createTable("CREATE TABLE %s (k int PRIMARY KEY, v " + mapType + ")");
+
+ execute("INSERT INTO %s(k, v) VALUES (0, ?)", map("abc", userType("s", set(2, 4, 6), "m", map("a", "v1", "d", "v2"))));
+
+ beforeAndAfterFlush(() ->
+ {
+ assertRows(execute("SELECT v[?].s FROM %s WHERE k = 0", "abc"), row(set(2, 4, 6)));
+ assertRows(execute("SELECT v[?].m[..?] FROM %s WHERE k = 0", "abc", "b"), row(map("a", "v1")));
+ assertRows(execute("SELECT v[?].m[?] FROM %s WHERE k = 0", "abc", "d"), row("v2"));
+ });
+ }
+
+ assertInvalidMessage("Non-frozen UDTs with nested non-frozen collections are not supported",
+ "CREATE TABLE " + KEYSPACE + ".t (k int PRIMARY KEY, v " + type + ")");
+
+ type = createType("CREATE TYPE %s (s frozen<set<int>>, m frozen<map<text, text>>)");
+
+ for (boolean frozen : new boolean[]{false, true})
+ {
+ type = frozen ? "frozen<" + type + ">" : type;
+
+ createTable("CREATE TABLE %s (k int PRIMARY KEY, v " + type + ")");
+
+ execute("INSERT INTO %s(k, v) VALUES (0, ?)", userType("s", set(2, 4, 6), "m", map("a", "v1", "d", "v2")));
+
+ beforeAndAfterFlush(() ->
+ {
+ assertRows(execute("SELECT v.s[?] FROM %s WHERE k = 0", 2), row(2));
+ assertRows(execute("SELECT v.s[?..?] FROM %s WHERE k = 0", 2, 5), row(set(2, 4)));
+ assertRows(execute("SELECT v.s[..?] FROM %s WHERE k = 0", 3), row(set(2)));
+ assertRows(execute("SELECT v.m[..?] FROM %s WHERE k = 0", "b"), row(map("a", "v1")));
+ assertRows(execute("SELECT v.m[?] FROM %s WHERE k = 0", "d"), row("v2"));
+ });
+ }
+ }
+
+ @Test
+ public void testMapOverlappingSlices() throws Throwable
+ {
+ createTable("CREATE TABLE %s (k int PRIMARY KEY, m map<int, int>)");
+
+ execute("INSERT INTO %s(k, m) VALUES (?, ?)", 0, map(0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5));
+
+ flush();
+
+ assertRows(execute("SELECT m[7..8] FROM %s WHERE k=?", 0),
+ row((Map<Integer, Integer>) null));
+
+ assertRows(execute("SELECT m[0..3] FROM %s WHERE k=?", 0),
+ row(map(0, 0, 1, 1, 2, 2, 3, 3)));
+
+ assertRows(execute("SELECT m[0..3], m[2..4] FROM %s WHERE k=?", 0),
+ row(map(0, 0, 1, 1, 2, 2, 3, 3), map(2, 2, 3, 3, 4, 4)));
+
+ assertRows(execute("SELECT m, m[2..4] FROM %s WHERE k=?", 0),
+ row(map(0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5), map(2, 2, 3, 3, 4, 4)));
+
+ assertRows(execute("SELECT m[..3], m[3..4] FROM %s WHERE k=?", 0),
+ row(map(0, 0, 1, 1, 2, 2, 3, 3), map(3, 3, 4, 4)));
+
+ assertRows(execute("SELECT m[1..3], m[2] FROM %s WHERE k=?", 0),
+ row(map(1, 1, 2, 2, 3, 3), 2));
+ }
+
+ @Test
+ public void testMapOverlappingSlicesWithDoubles() throws Throwable
+ {
+ createTable("CREATE TABLE %s (k int PRIMARY KEY, m map<double, double>)");
+
+ execute("INSERT INTO %s(k, m) VALUES (?, ?)", 0, map(0.0, 0.0, 1.1, 1.1, 2.2, 2.2, 3.0, 3.0, 4.4, 4.4, 5.5, 5.5));
+
+ flush();
+
+ assertRows(execute("SELECT m[0.0..3.0] FROM %s WHERE k=?", 0),
+ row(map(0.0, 0.0, 1.1, 1.1, 2.2, 2.2, 3.0, 3.0)));
+
+ assertRows(execute("SELECT m[0...3.], m[2.2..4.4] FROM %s WHERE k=?", 0),
+ row(map(0.0, 0.0, 1.1, 1.1, 2.2, 2.2, 3.0, 3.0), map(2.2, 2.2, 3.0, 3.0, 4.4, 4.4)));
+
+ assertRows(execute("SELECT m, m[2.2..4.4] FROM %s WHERE k=?", 0),
+ row(map(0.0, 0.0, 1.1, 1.1, 2.2, 2.2, 3.0, 3.0, 4.4, 4.4, 5.5, 5.5), map(2.2, 2.2, 3.0, 3.0, 4.4, 4.4)));
+
+ assertRows(execute("SELECT m[..3.], m[3...4.4] FROM %s WHERE k=?", 0),
+ row(map(0.0, 0.0, 1.1, 1.1, 2.2, 2.2, 3.0, 3.0), map(3.0, 3.0, 4.4, 4.4)));
+
+ assertRows(execute("SELECT m[1.1..3.0], m[2.2] FROM %s WHERE k=?", 0),
+ row(map(1.1, 1.1, 2.2, 2.2, 3.0, 3.0), 2.2));
+ }
+
+ @Test
+ public void testNestedAccessWithNestedMap() throws Throwable
+ {
+ createTable("CREATE TABLE %s (id text PRIMARY KEY, m map<float, frozen<map<int, text>>>)");
+
+ execute("INSERT INTO %s (id,m) VALUES ('1', {1: {2: 'one-two'}})");
+
+ flush();
+
+ assertRows(execute("SELECT m[1][2] FROM %s WHERE id = '1'"),
+ row("one-two"));
+
+ assertRows(execute("SELECT m[1..][2] FROM %s WHERE id = '1'"),
+ row((Map) null));
+
+ assertRows(execute("SELECT m[1][..2] FROM %s WHERE id = '1'"),
+ row(map(2, "one-two")));
+
+ assertRows(execute("SELECT m[1..][..2] FROM %s WHERE id = '1'"),
+ row(map(1F, map(2, "one-two"))));
+ }
+
+ @Test
public void testInsertingCollectionsWithInvalidElements() throws Throwable
{
createTable("CREATE TABLE %s (k int PRIMARY KEY, s frozen<set<tuple<int, text, double>>>)");
@@ -1071,4 +1915,68 @@
assertInvalidMessage("Invalid map literal for m: value (1, '1', 1.0, 1) is not of type frozen<tuple<int, text, double>>",
"INSERT INTO %s (k, m) VALUES (0, {1 : (1, '1', 1.0, 1)})");
}
+
+ @Test
+ public void testSelectionOfEmptyCollections() throws Throwable
+ {
+ createTable("CREATE TABLE %s (k int PRIMARY KEY, m frozen<map<text, int>>, s frozen<set<int>>)");
+
+ execute("INSERT INTO %s(k) VALUES (0)");
+ execute("INSERT INTO %s(k, m, s) VALUES (1, {}, {})");
+ execute("INSERT INTO %s(k, m, s) VALUES (2, ?, ?)", map(), set());
+ execute("INSERT INTO %s(k, m, s) VALUES (3, {'2':2}, {2})");
+
+ beforeAndAfterFlush(() ->
+ {
+ assertRows(execute("SELECT m, s FROM %s WHERE k = 0"), row(null, null));
+ assertRows(execute("SELECT m['0'], s[0] FROM %s WHERE k = 0"), row(null, null));
+ assertRows(execute("SELECT m['0'..'1'], s[0..1] FROM %s WHERE k = 0"), row(null, null));
+ assertRows(execute("SELECT m['0'..'1']['3'..'5'], s[0..1][3..5] FROM %s WHERE k = 0"), row(null, null));
+
+ assertRows(execute("SELECT m, s FROM %s WHERE k = 1"), row(map(), set()));
+ assertRows(execute("SELECT m['0'], s[0] FROM %s WHERE k = 1"), row(null, null));
+ assertRows(execute("SELECT m['0'..'1'], s[0..1] FROM %s WHERE k = 1"), row(map(), set()));
+ assertRows(execute("SELECT m['0'..'1']['3'..'5'], s[0..1][3..5] FROM %s WHERE k = 1"), row(map(), set()));
+
+ assertRows(execute("SELECT m, s FROM %s WHERE k = 2"), row(map(), set()));
+ assertRows(execute("SELECT m['0'], s[0] FROM %s WHERE k = 2"), row(null, null));
+ assertRows(execute("SELECT m['0'..'1'], s[0..1] FROM %s WHERE k = 2"), row(map(), set()));
+ assertRows(execute("SELECT m['0'..'1']['3'..'5'], s[0..1][3..5] FROM %s WHERE k = 2"), row(map(), set()));
+
+ assertRows(execute("SELECT m, s FROM %s WHERE k = 3"), row(map("2", 2), set(2)));
+ assertRows(execute("SELECT m['0'], s[0] FROM %s WHERE k = 3"), row(null, null));
+ assertRows(execute("SELECT m['0'..'1'], s[0..1] FROM %s WHERE k = 3"), row(map(), set()));
+ assertRows(execute("SELECT m['0'..'1']['3'..'5'], s[0..1][3..5] FROM %s WHERE k = 3"), row(map(), set()));
+ });
+
+ createTable("CREATE TABLE %s (k int PRIMARY KEY, m map<text, int>, s set<int>)");
+
+ execute("INSERT INTO %s(k) VALUES (0)");
+ execute("INSERT INTO %s(k, m, s) VALUES (1, {}, {})");
+ execute("INSERT INTO %s(k, m, s) VALUES (2, ?, ?)", map(), set());
+ execute("INSERT INTO %s(k, m, s) VALUES (3, {'2':2}, {2})");
+
+ beforeAndAfterFlush(() ->
+ {
+ assertRows(execute("SELECT m, s FROM %s WHERE k = 0"), row(null, null));
+ assertRows(execute("SELECT m['0'], s[0] FROM %s WHERE k = 0"), row(null, null));
+ assertRows(execute("SELECT m['0'..'1'], s[0..1] FROM %s WHERE k = 0"), row(null, null));
+ assertRows(execute("SELECT m['0'..'1']['3'..'5'], s[0..1][3..5] FROM %s WHERE k = 0"), row(null, null));
+
+ assertRows(execute("SELECT m, s FROM %s WHERE k = 1"), row(null, null));
+ assertRows(execute("SELECT m['0'], s[0] FROM %s WHERE k = 1"), row(null, null));
+ assertRows(execute("SELECT m['0'..'1'], s[0..1] FROM %s WHERE k = 1"), row(null, null));
+ assertRows(execute("SELECT m['0'..'1']['3'..'5'], s[0..1][3..5] FROM %s WHERE k = 1"), row(null, null));
+
+ assertRows(execute("SELECT m, s FROM %s WHERE k = 2"), row(null, null));
+ assertRows(execute("SELECT m['0'], s[0] FROM %s WHERE k = 2"), row(null, null));
+ assertRows(execute("SELECT m['0'..'1'], s[0..1] FROM %s WHERE k = 2"), row(null, null));
+ assertRows(execute("SELECT m['0'..'1']['3'..'5'], s[0..1][3..5] FROM %s WHERE k = 2"), row(null, null));
+
+ assertRows(execute("SELECT m, s FROM %s WHERE k = 3"), row(map("2", 2), set(2)));
+ assertRows(execute("SELECT m['0'], s[0] FROM %s WHERE k = 3"), row(null, null));
+ assertRows(execute("SELECT m['0'..'1'], s[0..1] FROM %s WHERE k = 3"), row(null, null));
+ assertRows(execute("SELECT m['0'..'1']['3'..'5'], s[0..1][3..5] FROM %s WHERE k = 3"), row(null, null));
+ });
+ }
}
diff --git a/test/unit/org/apache/cassandra/cql3/validation/entities/CountersTest.java b/test/unit/org/apache/cassandra/cql3/validation/entities/CountersTest.java
index 94e1c52..adb824b 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/entities/CountersTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/entities/CountersTest.java
@@ -26,48 +26,6 @@
public class CountersTest extends CQLTester
{
/**
- * Check for a table with counters,
- * migrated from cql_tests.py:TestCQL.counters_test()
- */
- @Test
- public void testCounters() throws Throwable
- {
- createTable("CREATE TABLE %s (userid int, url text, total counter, PRIMARY KEY (userid, url)) WITH COMPACT STORAGE");
-
- execute("UPDATE %s SET total = total + 1 WHERE userid = 1 AND url = 'http://foo.com'");
- assertRows(execute("SELECT total FROM %s WHERE userid = 1 AND url = 'http://foo.com'"),
- row(1L));
-
- execute("UPDATE %s SET total = total - 4 WHERE userid = 1 AND url = 'http://foo.com'");
- assertRows(execute("SELECT total FROM %s WHERE userid = 1 AND url = 'http://foo.com'"),
- row(-3L));
-
- execute("UPDATE %s SET total = total+1 WHERE userid = 1 AND url = 'http://foo.com'");
- assertRows(execute("SELECT total FROM %s WHERE userid = 1 AND url = 'http://foo.com'"),
- row(-2L));
-
- execute("UPDATE %s SET total = total -2 WHERE userid = 1 AND url = 'http://foo.com'");
- assertRows(execute("SELECT total FROM %s WHERE userid = 1 AND url = 'http://foo.com'"),
- row(-4L));
-
- execute("UPDATE %s SET total += 6 WHERE userid = 1 AND url = 'http://foo.com'");
- assertRows(execute("SELECT total FROM %s WHERE userid = 1 AND url = 'http://foo.com'"),
- row(2L));
-
- execute("UPDATE %s SET total -= 1 WHERE userid = 1 AND url = 'http://foo.com'");
- assertRows(execute("SELECT total FROM %s WHERE userid = 1 AND url = 'http://foo.com'"),
- row(1L));
-
- execute("UPDATE %s SET total += -2 WHERE userid = 1 AND url = 'http://foo.com'");
- assertRows(execute("SELECT total FROM %s WHERE userid = 1 AND url = 'http://foo.com'"),
- row(-1L));
-
- execute("UPDATE %s SET total -= -2 WHERE userid = 1 AND url = 'http://foo.com'");
- assertRows(execute("SELECT total FROM %s WHERE userid = 1 AND url = 'http://foo.com'"),
- row(1L));
- }
-
- /**
* Test for the validation bug of #4706,
* migrated from cql_tests.py:TestCQL.validate_counter_regular_test()
*/
@@ -131,75 +89,69 @@
@Test
public void testCounterFiltering() throws Throwable
{
- for (String compactStorageClause : new String[]{ "", " WITH COMPACT STORAGE" })
- {
- createTable("CREATE TABLE %s (k int PRIMARY KEY, a counter)" + compactStorageClause);
+ createTable("CREATE TABLE %s (k int PRIMARY KEY, a counter)");
- for (int i = 0; i < 10; i++)
- execute("UPDATE %s SET a = a + ? WHERE k = ?", (long) i, i);
+ for (int i = 0; i < 10; i++)
+ execute("UPDATE %s SET a = a + ? WHERE k = ?", (long) i, i);
- execute("UPDATE %s SET a = a + ? WHERE k = ?", 6L, 10);
+ execute("UPDATE %s SET a = a + ? WHERE k = ?", 6L, 10);
- // GT
- assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE a > ? ALLOW FILTERING", 5L),
- row(6, 6L),
- row(7, 7L),
- row(8, 8L),
- row(9, 9L),
- row(10, 6L));
+ // GT
+ assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE a > ? ALLOW FILTERING", 5L),
+ row(6, 6L),
+ row(7, 7L),
+ row(8, 8L),
+ row(9, 9L),
+ row(10, 6L));
- // GTE
- assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE a >= ? ALLOW FILTERING", 6L),
- row(6, 6L),
- row(7, 7L),
- row(8, 8L),
- row(9, 9L),
- row(10, 6L));
+ // GTE
+ assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE a >= ? ALLOW FILTERING", 6L),
+ row(6, 6L),
+ row(7, 7L),
+ row(8, 8L),
+ row(9, 9L),
+ row(10, 6L));
- // LT
- assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE a < ? ALLOW FILTERING", 3L),
- row(0, 0L),
- row(1, 1L),
- row(2, 2L));
+ // LT
+ assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE a < ? ALLOW FILTERING", 3L),
+ row(0, 0L),
+ row(1, 1L),
+ row(2, 2L));
- // LTE
- assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE a <= ? ALLOW FILTERING", 3L),
- row(0, 0L),
- row(1, 1L),
- row(2, 2L),
- row(3, 3L));
+ // LTE
+ assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE a <= ? ALLOW FILTERING", 3L),
+ row(0, 0L),
+ row(1, 1L),
+ row(2, 2L),
+ row(3, 3L));
- // EQ
- assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE a = ? ALLOW FILTERING", 6L),
- row(6, 6L),
- row(10, 6L));
- }
+ // EQ
+ assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE a = ? ALLOW FILTERING", 6L),
+ row(6, 6L),
+ row(10, 6L));
}
@Test
public void testCounterFilteringWithNull() throws Throwable
{
- for (String compactStorageClause : new String[]{ "", " WITH COMPACT STORAGE" })
- {
- createTable("CREATE TABLE %s (k int PRIMARY KEY, a counter, b counter)" + compactStorageClause);
- execute("UPDATE %s SET a = a + ? WHERE k = ?", 1L, 1);
+ createTable("CREATE TABLE %s (k int PRIMARY KEY, a counter, b counter)");
+ execute("UPDATE %s SET a = a + ? WHERE k = ?", 1L, 1);
- assertRows(execute("SELECT * FROM %s WHERE a > ? ALLOW FILTERING", 0L),
- row(1, 1L, null));
- // GT
- assertEmpty(execute("SELECT * FROM %s WHERE b > ? ALLOW FILTERING", 1L));
- // GTE
- assertEmpty(execute("SELECT * FROM %s WHERE b >= ? ALLOW FILTERING", 1L));
- // LT
- assertEmpty(execute("SELECT * FROM %s WHERE b < ? ALLOW FILTERING", 1L));
- // LTE
- assertEmpty(execute("SELECT * FROM %s WHERE b <= ? ALLOW FILTERING", 1L));
- // EQ
- assertEmpty(execute("SELECT * FROM %s WHERE b = ? ALLOW FILTERING", 1L));
- // with null
- assertInvalidMessage("Invalid null value for counter increment/decrement",
- "SELECT * FROM %s WHERE b = null ALLOW FILTERING");
- }
+ assertRows(execute("SELECT * FROM %s WHERE a > ? ALLOW FILTERING", 0L),
+ row(1, 1L, null));
+ // GT
+ assertEmpty(execute("SELECT * FROM %s WHERE b > ? ALLOW FILTERING", 1L));
+ // GTE
+ assertEmpty(execute("SELECT * FROM %s WHERE b >= ? ALLOW FILTERING", 1L));
+ // LT
+ assertEmpty(execute("SELECT * FROM %s WHERE b < ? ALLOW FILTERING", 1L));
+ // LTE
+ assertEmpty(execute("SELECT * FROM %s WHERE b <= ? ALLOW FILTERING", 1L));
+ // EQ
+ assertEmpty(execute("SELECT * FROM %s WHERE b = ? ALLOW FILTERING", 1L));
+ // with null
+ assertInvalidMessage("Invalid null value for counter increment/decrement",
+ "SELECT * FROM %s WHERE b = null ALLOW FILTERING");
}
/**
@@ -208,34 +160,74 @@
@Test
public void testProhibitReversedCounterAsPartOfPrimaryKey() throws Throwable
{
- assertInvalidThrowMessage("counter type is not supported for PRIMARY KEY part a",
+ assertInvalidThrowMessage("counter type is not supported for PRIMARY KEY column 'a'",
InvalidRequestException.class, String.format("CREATE TABLE %s.%s (a counter, b int, PRIMARY KEY (b, a)) WITH CLUSTERING ORDER BY (a desc);", KEYSPACE, createTableName()));
}
/**
- * Test for the bug of #11726.
+ * Check that a counter batch works as intended
*/
@Test
- public void testCounterAndColumnSelection() throws Throwable
+ public void testCounterBatch() throws Throwable
{
- for (String compactStorageClause : new String[]{ "", " WITH COMPACT STORAGE" })
- {
- createTable("CREATE TABLE %s (k int PRIMARY KEY, c counter)" + compactStorageClause);
+ createTable("CREATE TABLE %s (userid int, url text, total counter, PRIMARY KEY (userid, url))");
- // Flush 2 updates in different sstable so that the following select does a merge, which is what triggers
- // the problem from #11726
+ // Ensure we handle updates to the same CQL row in the same partition properly
+ execute("BEGIN UNLOGGED BATCH " +
+ "UPDATE %1$s SET total = total + 1 WHERE userid = 1 AND url = 'http://foo.com'; " +
+ "UPDATE %1$s SET total = total + 1 WHERE userid = 1 AND url = 'http://foo.com'; " +
+ "UPDATE %1$s SET total = total + 1 WHERE userid = 1 AND url = 'http://foo.com'; " +
+ "APPLY BATCH; ");
+ assertRows(execute("SELECT total FROM %s WHERE userid = 1 AND url = 'http://foo.com'"),
+ row(3L));
- execute("UPDATE %s SET c = c + ? WHERE k = ?", 1L, 0);
+ // Ensure we handle different CQL rows in the same partition properly
+ execute("BEGIN UNLOGGED BATCH " +
+ "UPDATE %1$s SET total = total + 1 WHERE userid = 1 AND url = 'http://bar.com'; " +
+ "UPDATE %1$s SET total = total + 1 WHERE userid = 1 AND url = 'http://baz.com'; " +
+ "UPDATE %1$s SET total = total + 1 WHERE userid = 1 AND url = 'http://bad.com'; " +
+ "APPLY BATCH; ");
+ assertRows(execute("SELECT url, total FROM %s WHERE userid = 1"),
+ row("http://bad.com", 1L),
+ row("http://bar.com", 1L),
+ row("http://baz.com", 1L),
+ row("http://foo.com", 3L)); // from previous batch
- flush();
+ // Different counters in the same CQL Row
+ createTable("CREATE TABLE %s (userid int, url text, first counter, second counter, third counter, PRIMARY KEY (userid, url))");
+ execute("BEGIN UNLOGGED BATCH " +
+ "UPDATE %1$s SET first = first + 1 WHERE userid = 1 AND url = 'http://foo.com'; " +
+ "UPDATE %1$s SET first = first + 1 WHERE userid = 1 AND url = 'http://foo.com'; " +
+ "UPDATE %1$s SET second = second + 1 WHERE userid = 1 AND url = 'http://foo.com'; " +
+ "APPLY BATCH; ");
+ assertRows(execute("SELECT first, second, third FROM %s WHERE userid = 1 AND url = 'http://foo.com'"),
+ row(2L, 1L, null));
- execute("UPDATE %s SET c = c + ? WHERE k = ?", 1L, 0);
+ // Different counters in different CQL Rows
+ execute("BEGIN UNLOGGED BATCH " +
+ "UPDATE %1$s SET first = first + 1 WHERE userid = 1 AND url = 'http://bad.com'; " +
+ "UPDATE %1$s SET first = first + 1, second = second + 1 WHERE userid = 1 AND url = 'http://bar.com'; " +
+ "UPDATE %1$s SET first = first - 1, second = second - 1 WHERE userid = 1 AND url = 'http://bar.com'; " +
+ "UPDATE %1$s SET second = second + 1 WHERE userid = 1 AND url = 'http://baz.com'; " +
+ "APPLY BATCH; ");
+ assertRows(execute("SELECT url, first, second, third FROM %s WHERE userid = 1"),
+ row("http://bad.com", 1L, null, null),
+ row("http://bar.com", 0L, 0L, null),
+ row("http://baz.com", null, 1L, null),
+ row("http://foo.com", 2L, 1L, null)); // from previous batch
- flush();
- // Querying, but not including the counter. Pre-CASSANDRA-11726, this made us query the counter but include
- // it's value, which broke at merge (post-CASSANDRA-11726 are special cases to never skip values).
- assertRows(execute("SELECT k FROM %s"), row(0));
- }
+ // Different counters in different partitions
+ execute("BEGIN UNLOGGED BATCH " +
+ "UPDATE %1$s SET first = first + 1 WHERE userid = 2 AND url = 'http://bad.com'; " +
+ "UPDATE %1$s SET first = first + 1, second = second + 1 WHERE userid = 3 AND url = 'http://bar.com'; " +
+ "UPDATE %1$s SET first = first - 1, second = second - 1 WHERE userid = 4 AND url = 'http://bar.com'; " +
+ "UPDATE %1$s SET second = second + 1 WHERE userid = 5 AND url = 'http://baz.com'; " +
+ "APPLY BATCH; ");
+ assertRowsIgnoringOrder(execute("SELECT userid, url, first, second, third FROM %s WHERE userid IN (2, 3, 4, 5)"),
+ row(2, "http://bad.com", 1L, null, null),
+ row(3, "http://bar.com", 1L, 1L, null),
+ row(4, "http://bar.com", -1L, -1L, null),
+ row(5, "http://baz.com", null, 1L, null));
}
}
diff --git a/test/unit/org/apache/cassandra/cql3/validation/entities/FrozenCollectionsTest.java b/test/unit/org/apache/cassandra/cql3/validation/entities/FrozenCollectionsTest.java
index e50528b..23ac0ff 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/entities/FrozenCollectionsTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/entities/FrozenCollectionsTest.java
@@ -183,272 +183,263 @@
@Test
public void testClusteringKeyUsage() throws Throwable
{
- for (String option : Arrays.asList("", " WITH COMPACT STORAGE"))
- {
- createTable("CREATE TABLE %s (a int, b frozen<set<int>>, c int, PRIMARY KEY (a, b))" + option);
+ createTable("CREATE TABLE %s (a int, b frozen<set<int>>, c int, PRIMARY KEY (a, b))");
- execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", 0, set(), 1);
- execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", 0, set(1, 2, 3), 1);
- execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", 0, set(4, 5, 6), 0);
- execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", 0, set(7, 8, 9), 0);
+ execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", 0, set(), 1);
+ execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", 0, set(1, 2, 3), 1);
+ execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", 0, set(4, 5, 6), 0);
+ execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", 0, set(7, 8, 9), 0);
- // overwrite with an update
- execute("UPDATE %s SET c=? WHERE a=? AND b=?", 0, 0, set());
- execute("UPDATE %s SET c=? WHERE a=? AND b=?", 0, 0, set(1, 2, 3));
+ // overwrite with an update
+ execute("UPDATE %s SET c=? WHERE a=? AND b=?", 0, 0, set());
+ execute("UPDATE %s SET c=? WHERE a=? AND b=?", 0, 0, set(1, 2, 3));
- assertRows(execute("SELECT * FROM %s"),
- row(0, set(), 0),
- row(0, set(1, 2, 3), 0),
- row(0, set(4, 5, 6), 0),
- row(0, set(7, 8, 9), 0)
- );
+ assertRows(execute("SELECT * FROM %s"),
+ row(0, set(), 0),
+ row(0, set(1, 2, 3), 0),
+ row(0, set(4, 5, 6), 0),
+ row(0, set(7, 8, 9), 0)
+ );
- assertRows(execute("SELECT b FROM %s"),
- row(set()),
- row(set(1, 2, 3)),
- row(set(4, 5, 6)),
- row(set(7, 8, 9))
- );
+ assertRows(execute("SELECT b FROM %s"),
+ row(set()),
+ row(set(1, 2, 3)),
+ row(set(4, 5, 6)),
+ row(set(7, 8, 9))
+ );
- assertRows(execute("SELECT * FROM %s LIMIT 2"),
- row(0, set(), 0),
- row(0, set(1, 2, 3), 0)
- );
+ assertRows(execute("SELECT * FROM %s LIMIT 2"),
+ row(0, set(), 0),
+ row(0, set(1, 2, 3), 0)
+ );
- assertRows(execute("SELECT * FROM %s WHERE a=? AND b=?", 0, set(4, 5, 6)),
- row(0, set(4, 5, 6), 0)
- );
+ assertRows(execute("SELECT * FROM %s WHERE a=? AND b=?", 0, set(4, 5, 6)),
+ row(0, set(4, 5, 6), 0)
+ );
- assertRows(execute("SELECT * FROM %s WHERE a=? AND b=?", 0, set()),
- row(0, set(), 0)
- );
+ assertRows(execute("SELECT * FROM %s WHERE a=? AND b=?", 0, set()),
+ row(0, set(), 0)
+ );
- assertRows(execute("SELECT * FROM %s WHERE a=? AND b IN ?", 0, list(set(4, 5, 6), set())),
- row(0, set(), 0),
- row(0, set(4, 5, 6), 0)
- );
+ assertRows(execute("SELECT * FROM %s WHERE a=? AND b IN ?", 0, list(set(4, 5, 6), set())),
+ row(0, set(), 0),
+ row(0, set(4, 5, 6), 0)
+ );
- assertRows(execute("SELECT * FROM %s WHERE a=? AND b > ?", 0, set(4, 5, 6)),
- row(0, set(7, 8, 9), 0)
- );
+ assertRows(execute("SELECT * FROM %s WHERE a=? AND b > ?", 0, set(4, 5, 6)),
+ row(0, set(7, 8, 9), 0)
+ );
- assertRows(execute("SELECT * FROM %s WHERE a=? AND b >= ?", 0, set(4, 5, 6)),
- row(0, set(4, 5, 6), 0),
- row(0, set(7, 8, 9), 0)
- );
+ assertRows(execute("SELECT * FROM %s WHERE a=? AND b >= ?", 0, set(4, 5, 6)),
+ row(0, set(4, 5, 6), 0),
+ row(0, set(7, 8, 9), 0)
+ );
- assertRows(execute("SELECT * FROM %s WHERE a=? AND b < ?", 0, set(4, 5, 6)),
- row(0, set(), 0),
- row(0, set(1, 2, 3), 0)
- );
+ assertRows(execute("SELECT * FROM %s WHERE a=? AND b < ?", 0, set(4, 5, 6)),
+ row(0, set(), 0),
+ row(0, set(1, 2, 3), 0)
+ );
- assertRows(execute("SELECT * FROM %s WHERE a=? AND b <= ?", 0, set(4, 5, 6)),
- row(0, set(), 0),
- row(0, set(1, 2, 3), 0),
- row(0, set(4, 5, 6), 0)
- );
+ assertRows(execute("SELECT * FROM %s WHERE a=? AND b <= ?", 0, set(4, 5, 6)),
+ row(0, set(), 0),
+ row(0, set(1, 2, 3), 0),
+ row(0, set(4, 5, 6), 0)
+ );
- assertRows(execute("SELECT * FROM %s WHERE a=? AND b > ? AND b <= ?", 0, set(1, 2, 3), set(4, 5, 6)),
- row(0, set(4, 5, 6), 0)
- );
+ assertRows(execute("SELECT * FROM %s WHERE a=? AND b > ? AND b <= ?", 0, set(1, 2, 3), set(4, 5, 6)),
+ row(0, set(4, 5, 6), 0)
+ );
- execute("DELETE FROM %s WHERE a=? AND b=?", 0, set());
- execute("DELETE FROM %s WHERE a=? AND b=?", 0, set(4, 5, 6));
- assertRows(execute("SELECT * FROM %s"),
- row(0, set(1, 2, 3), 0),
- row(0, set(7, 8, 9), 0)
- );
- }
+ execute("DELETE FROM %s WHERE a=? AND b=?", 0, set());
+ execute("DELETE FROM %s WHERE a=? AND b=?", 0, set(4, 5, 6));
+ assertRows(execute("SELECT * FROM %s"),
+ row(0, set(1, 2, 3), 0),
+ row(0, set(7, 8, 9), 0)
+ );
}
@Test
public void testNestedClusteringKeyUsage() throws Throwable
{
- for (String option : Arrays.asList("", " WITH COMPACT STORAGE"))
- {
- createTable("CREATE TABLE %s (a int, b frozen<map<set<int>, list<int>>>, c frozen<set<int>>, d int, PRIMARY KEY (a, b, c))" + option);
+ createTable("CREATE TABLE %s (a int, b frozen<map<set<int>, list<int>>>, c frozen<set<int>>, d int, PRIMARY KEY (a, b, c))");
- execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 0, map(), set(), 0);
- execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 0, map(set(), list(1, 2, 3)), set(), 0);
- execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 0, map(set(1, 2, 3), list(1, 2, 3)), set(1, 2, 3), 0);
- execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 0, map(set(4, 5, 6), list(1, 2, 3)), set(1, 2, 3), 0);
- execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 0, map(set(7, 8, 9), list(1, 2, 3)), set(1, 2, 3), 0);
+ execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 0, map(), set(), 0);
+ execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 0, map(set(), list(1, 2, 3)), set(), 0);
+ execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 0, map(set(1, 2, 3), list(1, 2, 3)), set(1, 2, 3), 0);
+ execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 0, map(set(4, 5, 6), list(1, 2, 3)), set(1, 2, 3), 0);
+ execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 0, map(set(7, 8, 9), list(1, 2, 3)), set(1, 2, 3), 0);
- assertRows(execute("SELECT * FROM %s"),
- row(0, map(), set(), 0),
- row(0, map(set(), list(1, 2, 3)), set(), 0),
- row(0, map(set(1, 2, 3), list(1, 2, 3)), set(1, 2, 3), 0),
- row(0, map(set(4, 5, 6), list(1, 2, 3)), set(1, 2, 3), 0),
- row(0, map(set(7, 8, 9), list(1, 2, 3)), set(1, 2, 3), 0)
- );
+ assertRows(execute("SELECT * FROM %s"),
+ row(0, map(), set(), 0),
+ row(0, map(set(), list(1, 2, 3)), set(), 0),
+ row(0, map(set(1, 2, 3), list(1, 2, 3)), set(1, 2, 3), 0),
+ row(0, map(set(4, 5, 6), list(1, 2, 3)), set(1, 2, 3), 0),
+ row(0, map(set(7, 8, 9), list(1, 2, 3)), set(1, 2, 3), 0)
+ );
- assertRows(execute("SELECT b FROM %s"),
- row(map()),
- row(map(set(), list(1, 2, 3))),
- row(map(set(1, 2, 3), list(1, 2, 3))),
- row(map(set(4, 5, 6), list(1, 2, 3))),
- row(map(set(7, 8, 9), list(1, 2, 3)))
- );
+ assertRows(execute("SELECT b FROM %s"),
+ row(map()),
+ row(map(set(), list(1, 2, 3))),
+ row(map(set(1, 2, 3), list(1, 2, 3))),
+ row(map(set(4, 5, 6), list(1, 2, 3))),
+ row(map(set(7, 8, 9), list(1, 2, 3)))
+ );
- assertRows(execute("SELECT c FROM %s"),
- row(set()),
- row(set()),
- row(set(1, 2, 3)),
- row(set(1, 2, 3)),
- row(set(1, 2, 3))
- );
+ assertRows(execute("SELECT c FROM %s"),
+ row(set()),
+ row(set()),
+ row(set(1, 2, 3)),
+ row(set(1, 2, 3)),
+ row(set(1, 2, 3))
+ );
- assertRows(execute("SELECT * FROM %s LIMIT 3"),
- row(0, map(), set(), 0),
- row(0, map(set(), list(1, 2, 3)), set(), 0),
- row(0, map(set(1, 2, 3), list(1, 2, 3)), set(1, 2, 3), 0)
- );
+ assertRows(execute("SELECT * FROM %s LIMIT 3"),
+ row(0, map(), set(), 0),
+ row(0, map(set(), list(1, 2, 3)), set(), 0),
+ row(0, map(set(1, 2, 3), list(1, 2, 3)), set(1, 2, 3), 0)
+ );
- assertRows(execute("SELECT * FROM %s WHERE a=0 ORDER BY b DESC LIMIT 4"),
- row(0, map(set(7, 8, 9), list(1, 2, 3)), set(1, 2, 3), 0),
- row(0, map(set(4, 5, 6), list(1, 2, 3)), set(1, 2, 3), 0),
- row(0, map(set(1, 2, 3), list(1, 2, 3)), set(1, 2, 3), 0),
- row(0, map(set(), list(1, 2, 3)), set(), 0)
- );
+ assertRows(execute("SELECT * FROM %s WHERE a=0 ORDER BY b DESC LIMIT 4"),
+ row(0, map(set(7, 8, 9), list(1, 2, 3)), set(1, 2, 3), 0),
+ row(0, map(set(4, 5, 6), list(1, 2, 3)), set(1, 2, 3), 0),
+ row(0, map(set(1, 2, 3), list(1, 2, 3)), set(1, 2, 3), 0),
+ row(0, map(set(), list(1, 2, 3)), set(), 0)
+ );
- assertRows(execute("SELECT * FROM %s WHERE a=? AND b=?", 0, map()),
- row(0, map(), set(), 0)
- );
+ assertRows(execute("SELECT * FROM %s WHERE a=? AND b=?", 0, map()),
+ row(0, map(), set(), 0)
+ );
- assertRows(execute("SELECT * FROM %s WHERE a=? AND b=?", 0, map(set(), list(1, 2, 3))),
- row(0, map(set(), list(1, 2, 3)), set(), 0)
- );
+ assertRows(execute("SELECT * FROM %s WHERE a=? AND b=?", 0, map(set(), list(1, 2, 3))),
+ row(0, map(set(), list(1, 2, 3)), set(), 0)
+ );
- assertRows(execute("SELECT * FROM %s WHERE a=? AND b=?", 0, map(set(1, 2, 3), list(1, 2, 3))),
- row(0, map(set(1, 2, 3), list(1, 2, 3)), set(1, 2, 3), 0)
- );
+ assertRows(execute("SELECT * FROM %s WHERE a=? AND b=?", 0, map(set(1, 2, 3), list(1, 2, 3))),
+ row(0, map(set(1, 2, 3), list(1, 2, 3)), set(1, 2, 3), 0)
+ );
- assertRows(execute("SELECT * FROM %s WHERE a=? AND b=? AND c=?", 0, map(set(), list(1, 2, 3)), set()),
- row(0, map(set(), list(1, 2, 3)), set(), 0)
- );
+ assertRows(execute("SELECT * FROM %s WHERE a=? AND b=? AND c=?", 0, map(set(), list(1, 2, 3)), set()),
+ row(0, map(set(), list(1, 2, 3)), set(), 0)
+ );
- assertRows(execute("SELECT * FROM %s WHERE a=? AND (b, c) IN ?", 0, list(tuple(map(set(4, 5, 6), list(1, 2, 3)), set(1, 2, 3)),
- tuple(map(), set()))),
- row(0, map(), set(), 0),
- row(0, map(set(4, 5, 6), list(1, 2, 3)), set(1, 2, 3), 0)
- );
+ assertRows(execute("SELECT * FROM %s WHERE a=? AND (b, c) IN ?", 0, list(tuple(map(set(4, 5, 6), list(1, 2, 3)), set(1, 2, 3)),
+ tuple(map(), set()))),
+ row(0, map(), set(), 0),
+ row(0, map(set(4, 5, 6), list(1, 2, 3)), set(1, 2, 3), 0)
+ );
- assertRows(execute("SELECT * FROM %s WHERE a=? AND b > ?", 0, map(set(4, 5, 6), list(1, 2, 3))),
- row(0, map(set(7, 8, 9), list(1, 2, 3)), set(1, 2, 3), 0)
- );
+ assertRows(execute("SELECT * FROM %s WHERE a=? AND b > ?", 0, map(set(4, 5, 6), list(1, 2, 3))),
+ row(0, map(set(7, 8, 9), list(1, 2, 3)), set(1, 2, 3), 0)
+ );
- assertRows(execute("SELECT * FROM %s WHERE a=? AND b >= ?", 0, map(set(4, 5, 6), list(1, 2, 3))),
- row(0, map(set(4, 5, 6), list(1, 2, 3)), set(1, 2, 3), 0),
- row(0, map(set(7, 8, 9), list(1, 2, 3)), set(1, 2, 3), 0)
- );
+ assertRows(execute("SELECT * FROM %s WHERE a=? AND b >= ?", 0, map(set(4, 5, 6), list(1, 2, 3))),
+ row(0, map(set(4, 5, 6), list(1, 2, 3)), set(1, 2, 3), 0),
+ row(0, map(set(7, 8, 9), list(1, 2, 3)), set(1, 2, 3), 0)
+ );
- assertRows(execute("SELECT * FROM %s WHERE a=? AND b < ?", 0, map(set(4, 5, 6), list(1, 2, 3))),
- row(0, map(), set(), 0),
- row(0, map(set(), list(1, 2, 3)), set(), 0),
- row(0, map(set(1, 2, 3), list(1, 2, 3)), set(1, 2, 3), 0)
- );
+ assertRows(execute("SELECT * FROM %s WHERE a=? AND b < ?", 0, map(set(4, 5, 6), list(1, 2, 3))),
+ row(0, map(), set(), 0),
+ row(0, map(set(), list(1, 2, 3)), set(), 0),
+ row(0, map(set(1, 2, 3), list(1, 2, 3)), set(1, 2, 3), 0)
+ );
- assertRows(execute("SELECT * FROM %s WHERE a=? AND b <= ?", 0, map(set(4, 5, 6), list(1, 2, 3))),
- row(0, map(), set(), 0),
- row(0, map(set(), list(1, 2, 3)), set(), 0),
- row(0, map(set(1, 2, 3), list(1, 2, 3)), set(1, 2, 3), 0),
- row(0, map(set(4, 5, 6), list(1, 2, 3)), set(1, 2, 3), 0)
- );
+ assertRows(execute("SELECT * FROM %s WHERE a=? AND b <= ?", 0, map(set(4, 5, 6), list(1, 2, 3))),
+ row(0, map(), set(), 0),
+ row(0, map(set(), list(1, 2, 3)), set(), 0),
+ row(0, map(set(1, 2, 3), list(1, 2, 3)), set(1, 2, 3), 0),
+ row(0, map(set(4, 5, 6), list(1, 2, 3)), set(1, 2, 3), 0)
+ );
- assertRows(execute("SELECT * FROM %s WHERE a=? AND b > ? AND b <= ?", 0, map(set(1, 2, 3), list(1, 2, 3)), map(set(4, 5, 6), list(1, 2, 3))),
- row(0, map(set(4, 5, 6), list(1, 2, 3)), set(1, 2, 3), 0)
- );
+ assertRows(execute("SELECT * FROM %s WHERE a=? AND b > ? AND b <= ?", 0, map(set(1, 2, 3), list(1, 2, 3)), map(set(4, 5, 6), list(1, 2, 3))),
+ row(0, map(set(4, 5, 6), list(1, 2, 3)), set(1, 2, 3), 0)
+ );
- execute("DELETE FROM %s WHERE a=? AND b=? AND c=?", 0, map(), set());
- assertEmpty(execute("SELECT * FROM %s WHERE a=? AND b=? AND c=?", 0, map(), set()));
+ execute("DELETE FROM %s WHERE a=? AND b=? AND c=?", 0, map(), set());
+ assertEmpty(execute("SELECT * FROM %s WHERE a=? AND b=? AND c=?", 0, map(), set()));
- execute("DELETE FROM %s WHERE a=? AND b=? AND c=?", 0, map(set(), list(1, 2, 3)), set());
- assertEmpty(execute("SELECT * FROM %s WHERE a=? AND b=? AND c=?", 0, map(set(), list(1, 2, 3)), set()));
+ execute("DELETE FROM %s WHERE a=? AND b=? AND c=?", 0, map(set(), list(1, 2, 3)), set());
+ assertEmpty(execute("SELECT * FROM %s WHERE a=? AND b=? AND c=?", 0, map(set(), list(1, 2, 3)), set()));
- execute("DELETE FROM %s WHERE a=? AND b=? AND c=?", 0, map(set(4, 5, 6), list(1, 2, 3)), set(1, 2, 3));
- assertEmpty(execute("SELECT * FROM %s WHERE a=? AND b=? AND c=?", 0, map(set(4, 5, 6), list(1, 2, 3)), set(1, 2, 3)));
+ execute("DELETE FROM %s WHERE a=? AND b=? AND c=?", 0, map(set(4, 5, 6), list(1, 2, 3)), set(1, 2, 3));
+ assertEmpty(execute("SELECT * FROM %s WHERE a=? AND b=? AND c=?", 0, map(set(4, 5, 6), list(1, 2, 3)), set(1, 2, 3)));
- assertRows(execute("SELECT * FROM %s"),
- row(0, map(set(1, 2, 3), list(1, 2, 3)), set(1, 2, 3), 0),
- row(0, map(set(7, 8, 9), list(1, 2, 3)), set(1, 2, 3), 0)
- );
- }
+ assertRows(execute("SELECT * FROM %s"),
+ row(0, map(set(1, 2, 3), list(1, 2, 3)), set(1, 2, 3), 0),
+ row(0, map(set(7, 8, 9), list(1, 2, 3)), set(1, 2, 3), 0)
+ );
}
@Test
public void testNormalColumnUsage() throws Throwable
{
- for (String option : Arrays.asList("", " WITH COMPACT STORAGE"))
- {
- createTable("CREATE TABLE %s (a int PRIMARY KEY, b frozen<map<set<int>, list<int>>>, c frozen<set<int>>)" + option);
+ createTable("CREATE TABLE %s (a int PRIMARY KEY, b frozen<map<set<int>, list<int>>>, c frozen<set<int>>)");
- execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", 0, map(), set());
- execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", 1, map(set(), list(99999, 999999, 99999)), set());
- execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", 2, map(set(1, 2, 3), list(1, 2, 3)), set(1, 2, 3));
- execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", 3, map(set(4, 5, 6), list(1, 2, 3)), set(1, 2, 3));
- execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", 4, map(set(7, 8, 9), list(1, 2, 3)), set(1, 2, 3));
+ execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", 0, map(), set());
+ execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", 1, map(set(), list(99999, 999999, 99999)), set());
+ execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", 2, map(set(1, 2, 3), list(1, 2, 3)), set(1, 2, 3));
+ execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", 3, map(set(4, 5, 6), list(1, 2, 3)), set(1, 2, 3));
+ execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", 4, map(set(7, 8, 9), list(1, 2, 3)), set(1, 2, 3));
- // overwrite with update
- execute ("UPDATE %s SET b=? WHERE a=?", map(set(), list(1, 2, 3)), 1);
+ // overwrite with update
+ execute("UPDATE %s SET b=? WHERE a=?", map(set(), list(1, 2, 3)), 1);
- assertRows(execute("SELECT * FROM %s"),
- row(0, map(), set()),
- row(1, map(set(), list(1, 2, 3)), set()),
- row(2, map(set(1, 2, 3), list(1, 2, 3)), set(1, 2, 3)),
- row(3, map(set(4, 5, 6), list(1, 2, 3)), set(1, 2, 3)),
- row(4, map(set(7, 8, 9), list(1, 2, 3)), set(1, 2, 3))
- );
+ assertRows(execute("SELECT * FROM %s"),
+ row(0, map(), set()),
+ row(1, map(set(), list(1, 2, 3)), set()),
+ row(2, map(set(1, 2, 3), list(1, 2, 3)), set(1, 2, 3)),
+ row(3, map(set(4, 5, 6), list(1, 2, 3)), set(1, 2, 3)),
+ row(4, map(set(7, 8, 9), list(1, 2, 3)), set(1, 2, 3))
+ );
- assertRows(execute("SELECT b FROM %s"),
- row(map()),
- row(map(set(), list(1, 2, 3))),
- row(map(set(1, 2, 3), list(1, 2, 3))),
- row(map(set(4, 5, 6), list(1, 2, 3))),
- row(map(set(7, 8, 9), list(1, 2, 3)))
- );
+ assertRows(execute("SELECT b FROM %s"),
+ row(map()),
+ row(map(set(), list(1, 2, 3))),
+ row(map(set(1, 2, 3), list(1, 2, 3))),
+ row(map(set(4, 5, 6), list(1, 2, 3))),
+ row(map(set(7, 8, 9), list(1, 2, 3)))
+ );
- assertRows(execute("SELECT c FROM %s"),
- row(set()),
- row(set()),
- row(set(1, 2, 3)),
- row(set(1, 2, 3)),
- row(set(1, 2, 3))
- );
+ assertRows(execute("SELECT c FROM %s"),
+ row(set()),
+ row(set()),
+ row(set(1, 2, 3)),
+ row(set(1, 2, 3)),
+ row(set(1, 2, 3))
+ );
- assertRows(execute("SELECT * FROM %s WHERE a=?", 3),
- row(3, map(set(4, 5, 6), list(1, 2, 3)), set(1, 2, 3))
- );
+ assertRows(execute("SELECT * FROM %s WHERE a=?", 3),
+ row(3, map(set(4, 5, 6), list(1, 2, 3)), set(1, 2, 3))
+ );
- execute("UPDATE %s SET b=? WHERE a=?", null, 1);
- assertRows(execute("SELECT * FROM %s WHERE a=?", 1),
- row(1, null, set())
- );
+ execute("UPDATE %s SET b=? WHERE a=?", null, 1);
+ assertRows(execute("SELECT * FROM %s WHERE a=?", 1),
+ row(1, null, set())
+ );
- execute("UPDATE %s SET b=? WHERE a=?", map(), 1);
- assertRows(execute("SELECT * FROM %s WHERE a=?", 1),
- row(1, map(), set())
- );
+ execute("UPDATE %s SET b=? WHERE a=?", map(), 1);
+ assertRows(execute("SELECT * FROM %s WHERE a=?", 1),
+ row(1, map(), set())
+ );
- execute("UPDATE %s SET c=? WHERE a=?", null, 2);
- assertRows(execute("SELECT * FROM %s WHERE a=?", 2),
- row(2, map(set(1, 2, 3), list(1, 2, 3)), null)
- );
+ execute("UPDATE %s SET c=? WHERE a=?", null, 2);
+ assertRows(execute("SELECT * FROM %s WHERE a=?", 2),
+ row(2, map(set(1, 2, 3), list(1, 2, 3)), null)
+ );
- execute("UPDATE %s SET c=? WHERE a=?", set(), 2);
- assertRows(execute("SELECT * FROM %s WHERE a=?", 2),
- row(2, map(set(1, 2, 3), list(1, 2, 3)), set())
- );
+ execute("UPDATE %s SET c=? WHERE a=?", set(), 2);
+ assertRows(execute("SELECT * FROM %s WHERE a=?", 2),
+ row(2, map(set(1, 2, 3), list(1, 2, 3)), set())
+ );
- execute("DELETE b FROM %s WHERE a=?", 3);
- assertRows(execute("SELECT * FROM %s WHERE a=?", 3),
- row(3, null, set(1, 2, 3))
- );
+ execute("DELETE b FROM %s WHERE a=?", 3);
+ assertRows(execute("SELECT * FROM %s WHERE a=?", 3),
+ row(3, null, set(1, 2, 3))
+ );
- execute("DELETE c FROM %s WHERE a=?", 4);
- assertRows(execute("SELECT * FROM %s WHERE a=?", 4),
- row(4, map(set(7, 8, 9), list(1, 2, 3)), null)
- );
- }
+ execute("DELETE c FROM %s WHERE a=?", 4);
+ assertRows(execute("SELECT * FROM %s WHERE a=?", 4),
+ row(4, map(set(7, 8, 9), list(1, 2, 3)), null)
+ );
}
@Test
@@ -563,10 +554,10 @@
createTable("CREATE TABLE %s (a frozen<map<int, text>> PRIMARY KEY, b frozen<map<int, text>>)");
// for now, we don't support indexing values or keys of collections in the primary key
- assertInvalidIndexCreationWithMessage("CREATE INDEX ON %s (full(a))", "Cannot create secondary index on partition key column");
- assertInvalidIndexCreationWithMessage("CREATE INDEX ON %s (keys(a))", "Cannot create secondary index on partition key column");
+ assertInvalidIndexCreationWithMessage("CREATE INDEX ON %s (full(a))", "Cannot create secondary index on the only partition key column");
+ assertInvalidIndexCreationWithMessage("CREATE INDEX ON %s (keys(a))", "Cannot create secondary index on the only partition key column");
assertInvalidIndexCreationWithMessage("CREATE INDEX ON %s (keys(b))", "Cannot create keys() index on frozen column b. " +
- "Frozen collections only support full() indexes");
+ "Frozen collections are immutable and must be fully indexed");
createTable("CREATE TABLE %s (a int, b frozen<list<int>>, c frozen<set<int>>, d frozen<map<int, text>>, PRIMARY KEY (a, b))");
diff --git a/test/unit/org/apache/cassandra/cql3/validation/entities/JsonTest.java b/test/unit/org/apache/cassandra/cql3/validation/entities/JsonTest.java
index b048397..71c1d62 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/entities/JsonTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/entities/JsonTest.java
@@ -17,6 +17,7 @@
*/
package org.apache.cassandra.cql3.validation.entities;
+import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.Json;
import org.apache.cassandra.cql3.CQLTester;
import org.apache.cassandra.cql3.Duration;
@@ -44,10 +45,17 @@
public class JsonTest extends CQLTester
{
+ // This method will be ran instead of the CQLTester#setUpClass
@BeforeClass
- public static void setUp()
+ public static void setUpClass()
{
+ if (ROW_CACHE_SIZE_IN_MB > 0)
+ DatabaseDescriptor.setRowCacheSizeInMB(ROW_CACHE_SIZE_IN_MB);
+
StorageService.instance.setPartitionerUnsafe(ByteOrderedPartitioner.instance);
+
+ // Once per-JVM is enough
+ prepareServer();
}
@Test
@@ -1356,10 +1364,10 @@
future.get(30, TimeUnit.SECONDS);
executor.shutdown();
- Assert.assertTrue(executor.awaitTermination(30, TimeUnit.SECONDS));
+ Assert.assertTrue(executor.awaitTermination(1, TimeUnit.MINUTES));
}
- @Test
+ @Test
public void emptyStringJsonSerializationTest() throws Throwable
{
createTable("create table %s(id INT, name TEXT, PRIMARY KEY(id));");
@@ -1377,7 +1385,7 @@
@Test
public void testJsonOrdering() throws Throwable
{
- createTable("CREATE TABLE %s( PRIMARY KEY (a, b), a INT, b INT);");
+ createTable("CREATE TABLE %s(a INT, b INT, PRIMARY KEY (a, b))");
execute("INSERT INTO %s(a, b) VALUES (20, 30);");
execute("INSERT INTO %s(a, b) VALUES (100, 200);");
@@ -1393,7 +1401,7 @@
row("{\"a\": 100}"),
row("{\"a\": 20}"));
- // Check ordering with alias
+ // Check ordering with alias
assertRows(execute("SELECT JSON a, b as c FROM %s WHERE a IN (20, 100) ORDER BY b"),
row("{\"a\": 20, \"c\": 30}"),
row("{\"a\": 100, \"c\": 200}"));
@@ -1402,7 +1410,7 @@
row("{\"a\": 100, \"c\": 200}"),
row("{\"a\": 20, \"c\": 30}"));
- // Check ordering with CAST
+ // Check ordering with CAST
assertRows(execute("SELECT JSON a, CAST(b AS FLOAT) FROM %s WHERE a IN (20, 100) ORDER BY b"),
row("{\"a\": 20, \"cast(b as float)\": 30.0}"),
row("{\"a\": 100, \"cast(b as float)\": 200.0}"));
diff --git a/test/unit/org/apache/cassandra/cql3/validation/entities/SecondaryIndexTest.java b/test/unit/org/apache/cassandra/cql3/validation/entities/SecondaryIndexTest.java
index ce77081..947e8b5 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/entities/SecondaryIndexTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/entities/SecondaryIndexTest.java
@@ -22,24 +22,25 @@
import java.util.concurrent.Callable;
import java.util.concurrent.CountDownLatch;
+import com.google.common.collect.ImmutableSet;
+
import org.apache.commons.lang3.StringUtils;
import org.junit.Test;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.CQLTester;
import org.apache.cassandra.cql3.ColumnIdentifier;
-import org.apache.cassandra.cql3.Duration;
import org.apache.cassandra.cql3.QueryProcessor;
import org.apache.cassandra.cql3.restrictions.StatementRestrictions;
-import org.apache.cassandra.cql3.statements.IndexTarget;
+import org.apache.cassandra.cql3.statements.schema.IndexTarget;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.DeletionTime;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.db.rows.Cell;
import org.apache.cassandra.db.rows.Row;
-import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.exceptions.SyntaxException;
import org.apache.cassandra.index.IndexNotAvailableException;
import org.apache.cassandra.index.SecondaryIndexManager;
@@ -53,6 +54,8 @@
import org.apache.cassandra.utils.MD5Digest;
import org.apache.cassandra.utils.Pair;
+import static java.lang.String.format;
+
import static org.apache.cassandra.Util.throwAssert;
import static org.apache.cassandra.utils.ByteBufferUtil.EMPTY_BYTE_BUFFER;
import static org.apache.cassandra.utils.ByteBufferUtil.bytes;
@@ -97,27 +100,27 @@
*/
private void testCreateAndDropIndex(String indexName, boolean addKeyspaceOnDrop) throws Throwable
{
- execute("USE system");
- assertInvalidMessage(String.format("Index '%s' could not be found",
- removeQuotes(indexName.toLowerCase(Locale.US))),
- "DROP INDEX " + indexName + ";");
+ assertInvalidMessage(format("Index '%s.%s' doesn't exist",
+ KEYSPACE,
+ removeQuotes(indexName.toLowerCase(Locale.US))),
+ format("DROP INDEX %s.%s", KEYSPACE, indexName));
createTable("CREATE TABLE %s (a int primary key, b int);");
createIndex("CREATE INDEX " + indexName + " ON %s(b);");
createIndex("CREATE INDEX IF NOT EXISTS " + indexName + " ON %s(b);");
- assertInvalidMessage(String.format("Index %s already exists",
- removeQuotes(indexName.toLowerCase(Locale.US))),
+ assertInvalidMessage(format("Index '%s' already exists",
+ removeQuotes(indexName.toLowerCase(Locale.US))),
"CREATE INDEX " + indexName + " ON %s(b)");
// IF NOT EXISTS should apply in cases where the new index differs from an existing one in name only
String otherIndexName = "index_" + System.nanoTime();
- assertEquals(1, getCurrentColumnFamilyStore().metadata.getIndexes().size());
+ assertEquals(1, getCurrentColumnFamilyStore().metadata().indexes.size());
createIndex("CREATE INDEX IF NOT EXISTS " + otherIndexName + " ON %s(b)");
- assertEquals(1, getCurrentColumnFamilyStore().metadata.getIndexes().size());
- assertInvalidMessage(String.format("Index %s is a duplicate of existing index %s",
- removeQuotes(otherIndexName.toLowerCase(Locale.US)),
- removeQuotes(indexName.toLowerCase(Locale.US))),
+ assertEquals(1, getCurrentColumnFamilyStore().metadata().indexes.size());
+ assertInvalidMessage(format("Index %s is a duplicate of existing index %s",
+ removeQuotes(otherIndexName.toLowerCase(Locale.US)),
+ removeQuotes(indexName.toLowerCase(Locale.US))),
"CREATE INDEX " + otherIndexName + " ON %s(b)");
execute("INSERT INTO %s (a, b) values (?, ?);", 0, 0);
@@ -126,26 +129,24 @@
execute("INSERT INTO %s (a, b) values (?, ?);", 3, 1);
assertRows(execute("SELECT * FROM %s where b = ?", 1), row(1, 1), row(3, 1));
- assertInvalidMessage(String.format("Index '%s' could not be found in any of the tables of keyspace 'system'",
- removeQuotes(indexName.toLowerCase(Locale.US))),
- "DROP INDEX " + indexName);
if (addKeyspaceOnDrop)
{
- dropIndex("DROP INDEX " + KEYSPACE + "." + indexName);
+ dropIndex(format("DROP INDEX %s.%s", KEYSPACE, indexName));
}
else
{
execute("USE " + KEYSPACE);
- execute("DROP INDEX " + indexName);
+ execute(format("DROP INDEX %s", indexName));
}
assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
"SELECT * FROM %s where b = ?", 1);
- dropIndex("DROP INDEX IF EXISTS " + indexName);
- assertInvalidMessage(String.format("Index '%s' could not be found",
- removeQuotes(indexName.toLowerCase(Locale.US))),
- "DROP INDEX " + indexName);
+ dropIndex(format("DROP INDEX IF EXISTS %s.%s", KEYSPACE, indexName));
+ assertInvalidMessage(format("Index '%s.%s' doesn't exist",
+ KEYSPACE,
+ removeQuotes(indexName.toLowerCase(Locale.US))),
+ format("DROP INDEX %s.%s", KEYSPACE, indexName));
}
/**
@@ -240,20 +241,10 @@
public void testUnknownCompressionOptions() throws Throwable
{
String tableName = createTableName();
- assertInvalidThrow(SyntaxException.class, String.format("CREATE TABLE %s (key varchar PRIMARY KEY, password varchar, gender varchar) WITH compression_parameters:sstable_compressor = 'DeflateCompressor'", tableName));
+ assertInvalidThrow(SyntaxException.class, format("CREATE TABLE %s (key varchar PRIMARY KEY, password varchar, gender varchar) WITH compression_parameters:sstable_compressor = 'DeflateCompressor'", tableName));
- assertInvalidThrow(ConfigurationException.class, String.format("CREATE TABLE %s (key varchar PRIMARY KEY, password varchar, gender varchar) WITH compression = { 'sstable_compressor': 'DeflateCompressor' }",
- tableName));
- }
-
- /**
- * Check one can use arbitrary name for datacenter when creating keyspace (#4278),
- * migrated from cql_tests.py:TestCQL.keyspace_creation_options_test()
- */
- @Test
- public void testDataCenterName() throws Throwable
- {
- execute("CREATE KEYSPACE Foo WITH replication = { 'class' : 'NetworkTopologyStrategy', 'us-east' : 1, 'us-west' : 1 };");
+ assertInvalidThrow(ConfigurationException.class, format("CREATE TABLE %s (key varchar PRIMARY KEY, password varchar, gender varchar) WITH compression = { 'sstable_compressor': 'DeflateCompressor' }",
+ tableName));
}
/**
@@ -569,14 +560,14 @@
{
String indexName = columnName + "_idx";
SecondaryIndexManager indexManager = getCurrentColumnFamilyStore().indexManager;
- createIndex(String.format("CREATE INDEX %s on %%s(%s)", indexName, columnName));
+ createIndex(format("CREATE INDEX %s on %%s(%s)", indexName, columnName));
IndexMetadata indexDef = indexManager.getIndexByName(indexName).getIndexMetadata();
- assertEquals(String.format("values(%s)", columnName), indexDef.options.get(IndexTarget.TARGET_OPTION_NAME));
- dropIndex(String.format("DROP INDEX %s.%s", KEYSPACE, indexName));
+ assertEquals(format("values(%s)", columnName), indexDef.options.get(IndexTarget.TARGET_OPTION_NAME));
+ dropIndex(format("DROP INDEX %s.%s", KEYSPACE, indexName));
assertFalse(indexManager.hasIndexes());
- createIndex(String.format("CREATE INDEX %s on %%s(values(%s))", indexName, columnName));
+ createIndex(format("CREATE INDEX %s on %%s(values(%s))", indexName, columnName));
assertEquals(indexDef, indexManager.getIndexByName(indexName).getIndexMetadata());
- dropIndex(String.format("DROP INDEX %s.%s", KEYSPACE, indexName));
+ dropIndex(format("DROP INDEX %s.%s", KEYSPACE, indexName));
}
@Test
@@ -605,15 +596,15 @@
private void createAndDropIndexWithQuotedColumnIdentifier(String target) throws Throwable
{
String indexName = "test_mixed_case_idx";
- createIndex(String.format("CREATE INDEX %s ON %%s(%s)", indexName, target));
+ createIndex(format("CREATE INDEX %s ON %%s(%s)", indexName, target));
SecondaryIndexManager indexManager = getCurrentColumnFamilyStore().indexManager;
IndexMetadata indexDef = indexManager.getIndexByName(indexName).getIndexMetadata();
- dropIndex(String.format("DROP INDEX %s.%s", KEYSPACE, indexName));
+ dropIndex(format("DROP INDEX %s.%s", KEYSPACE, indexName));
// verify we can re-create the index using the target string
- createIndex(String.format("CREATE INDEX %s ON %%s(%s)",
- indexName, indexDef.options.get(IndexTarget.TARGET_OPTION_NAME)));
+ createIndex(format("CREATE INDEX %s ON %%s(%s)",
+ indexName, indexDef.options.get(IndexTarget.TARGET_OPTION_NAME)));
assertEquals(indexDef, indexManager.getIndexByName(indexName).getIndexMetadata());
- dropIndex(String.format("DROP INDEX %s.%s", KEYSPACE, indexName));
+ dropIndex(format("DROP INDEX %s.%s", KEYSPACE, indexName));
}
@@ -662,23 +653,6 @@
}
@Test
- public void testCompactTableWithValueOver64k() throws Throwable
- {
- createTable("CREATE TABLE %s(a int, b blob, PRIMARY KEY (a)) WITH COMPACT STORAGE");
- createIndex("CREATE INDEX ON %s(b)");
- failInsert("INSERT INTO %s (a, b) VALUES (0, ?)", ByteBuffer.allocate(TOO_BIG));
- failInsert("INSERT INTO %s (a, b) VALUES (0, ?) IF NOT EXISTS", ByteBuffer.allocate(TOO_BIG));
- failInsert("BEGIN BATCH\n" +
- "INSERT INTO %s (a, b) VALUES (0, ?);\n" +
- "APPLY BATCH",
- ByteBuffer.allocate(TOO_BIG));
- failInsert("BEGIN BATCH\n" +
- "INSERT INTO %s (a, b) VALUES (0, ?) IF NOT EXISTS;\n" +
- "APPLY BATCH",
- ByteBuffer.allocate(TOO_BIG));
- }
-
- @Test
public void testIndexOnPartitionKeyInsertValueOver64k() throws Throwable
{
createTable("CREATE TABLE %s(a int, b int, c blob, PRIMARY KEY ((a, b)))");
@@ -692,7 +666,7 @@
// the indexed value passes validation, but the batch size will
// exceed the default failure threshold, so temporarily raise it
// (the non-conditional batch doesn't hit this because
- // BatchStatement::executeInternal skips the size check but CAS
+ // BatchStatement::executeLocally skips the size check but CAS
// path does not)
long batchSizeThreshold = DatabaseDescriptor.getBatchSizeFailThreshold();
try
@@ -745,7 +719,7 @@
// the indexed value passes validation, but the batch size will
// exceed the default failure threshold, so temporarily raise it
// (the non-conditional batch doesn't hit this because
- // BatchStatement::executeInternal skips the size check but CAS
+ // BatchStatement::executeLocally skips the size check but CAS
// path does not)
long batchSizeThreshold = DatabaseDescriptor.getBatchSizeFailThreshold();
try
@@ -782,15 +756,15 @@
public void prepareStatementsWithLIKEClauses() throws Throwable
{
createTable("CREATE TABLE %s (a int, c1 text, c2 text, v1 text, v2 text, v3 int, PRIMARY KEY (a, c1, c2))");
- createIndex(String.format("CREATE CUSTOM INDEX c1_idx on %%s(c1) USING '%s' WITH OPTIONS = {'mode' : 'PREFIX'}",
- SASIIndex.class.getName()));
- createIndex(String.format("CREATE CUSTOM INDEX c2_idx on %%s(c2) USING '%s' WITH OPTIONS = {'mode' : 'CONTAINS'}",
- SASIIndex.class.getName()));
- createIndex(String.format("CREATE CUSTOM INDEX v1_idx on %%s(v1) USING '%s' WITH OPTIONS = {'mode' : 'PREFIX'}",
- SASIIndex.class.getName()));
- createIndex(String.format("CREATE CUSTOM INDEX v2_idx on %%s(v2) USING '%s' WITH OPTIONS = {'mode' : 'CONTAINS'}",
- SASIIndex.class.getName()));
- createIndex(String.format("CREATE CUSTOM INDEX v3_idx on %%s(v3) USING '%s'", SASIIndex.class.getName()));
+ createIndex(format("CREATE CUSTOM INDEX c1_idx on %%s(c1) USING '%s' WITH OPTIONS = {'mode' : 'PREFIX'}",
+ SASIIndex.class.getName()));
+ createIndex(format("CREATE CUSTOM INDEX c2_idx on %%s(c2) USING '%s' WITH OPTIONS = {'mode' : 'CONTAINS'}",
+ SASIIndex.class.getName()));
+ createIndex(format("CREATE CUSTOM INDEX v1_idx on %%s(v1) USING '%s' WITH OPTIONS = {'mode' : 'PREFIX'}",
+ SASIIndex.class.getName()));
+ createIndex(format("CREATE CUSTOM INDEX v2_idx on %%s(v2) USING '%s' WITH OPTIONS = {'mode' : 'CONTAINS'}",
+ SASIIndex.class.getName()));
+ createIndex(format("CREATE CUSTOM INDEX v3_idx on %%s(v3) USING '%s'", SASIIndex.class.getName()));
forcePreparedValues();
// prefix mode indexes support prefix/contains/matches
@@ -892,37 +866,21 @@
row("B"), row("E"));
}
- /**
- * Migrated from cql_tests.py:TestCQL.invalid_clustering_indexing_test()
- */
- @Test
- public void testIndexesOnClusteringInvalid() throws Throwable
- {
- createTable("CREATE TABLE %s (a int, b int, c int, d int, PRIMARY KEY ((a, b))) WITH COMPACT STORAGE");
- assertInvalid("CREATE INDEX ON %s (a)");
- assertInvalid("CREATE INDEX ON %s (b)");
-
- createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY (a, b)) WITH COMPACT STORAGE");
- assertInvalid("CREATE INDEX ON %s (a)");
- assertInvalid("CREATE INDEX ON %s (b)");
- assertInvalid("CREATE INDEX ON %s (c)");
- }
-
@Test
public void testMultipleIndexesOnOneColumn() throws Throwable
{
String indexClassName = StubIndex.class.getName();
createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY ((a), b))");
// uses different options otherwise the two indexes are considered duplicates
- createIndex(String.format("CREATE CUSTOM INDEX c_idx_1 ON %%s(c) USING '%s' WITH OPTIONS = {'foo':'a'}", indexClassName));
- createIndex(String.format("CREATE CUSTOM INDEX c_idx_2 ON %%s(c) USING '%s' WITH OPTIONS = {'foo':'b'}", indexClassName));
+ createIndex(format("CREATE CUSTOM INDEX c_idx_1 ON %%s(c) USING '%s' WITH OPTIONS = {'foo':'a'}", indexClassName));
+ createIndex(format("CREATE CUSTOM INDEX c_idx_2 ON %%s(c) USING '%s' WITH OPTIONS = {'foo':'b'}", indexClassName));
ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
- CFMetaData cfm = cfs.metadata;
- StubIndex index1 = (StubIndex)cfs.indexManager.getIndex(cfm.getIndexes()
+ TableMetadata cfm = cfs.metadata();
+ StubIndex index1 = (StubIndex)cfs.indexManager.getIndex(cfm.indexes
.get("c_idx_1")
.orElseThrow(throwAssert("index not found")));
- StubIndex index2 = (StubIndex)cfs.indexManager.getIndex(cfm.getIndexes()
+ StubIndex index2 = (StubIndex)cfs.indexManager.getIndex(cfm.indexes
.get("c_idx_2")
.orElseThrow(throwAssert("index not found")));
Object[] row1a = row(0, 0, 0);
@@ -957,11 +915,11 @@
String indexClassName = StubIndex.class.getName();
createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY ((a), b))");
- createIndex(String.format("CREATE CUSTOM INDEX c_idx ON %%s(c) USING '%s'", indexClassName));
+ createIndex(format("CREATE CUSTOM INDEX c_idx ON %%s(c) USING '%s'", indexClassName));
ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
- CFMetaData cfm = cfs.metadata;
- StubIndex index1 = (StubIndex) cfs.indexManager.getIndex(cfm.getIndexes()
+ TableMetadata cfm = cfs.metadata();
+ StubIndex index1 = (StubIndex) cfs.indexManager.getIndex(cfm.indexes
.get("c_idx")
.orElseThrow(throwAssert("index not found")));
@@ -1013,11 +971,11 @@
// Any columns which are unchanged by the update are not passed to the Indexer
// Note that for simplicity this test resets the index between each scenario
createTable("CREATE TABLE %s (k int, c int, v1 int, v2 int, PRIMARY KEY (k,c))");
- createIndex(String.format("CREATE CUSTOM INDEX test_index ON %%s() USING '%s'", StubIndex.class.getName()));
+ createIndex(format("CREATE CUSTOM INDEX test_index ON %%s() USING '%s'", StubIndex.class.getName()));
execute("INSERT INTO %s (k, c, v1, v2) VALUES (0, 0, 0, 0) USING TIMESTAMP 0");
- ColumnDefinition v1 = getCurrentColumnFamilyStore().metadata.getColumnDefinition(new ColumnIdentifier("v1", true));
- ColumnDefinition v2 = getCurrentColumnFamilyStore().metadata.getColumnDefinition(new ColumnIdentifier("v2", true));
+ ColumnMetadata v1 = getCurrentColumnFamilyStore().metadata().getColumn(new ColumnIdentifier("v1", true));
+ ColumnMetadata v2 = getCurrentColumnFamilyStore().metadata().getColumn(new ColumnIdentifier("v2", true));
StubIndex index = (StubIndex)getCurrentColumnFamilyStore().indexManager.getIndexByName("test_index");
assertEquals(1, index.rowsInserted.size());
@@ -1097,21 +1055,96 @@
}
}
+ @Test // A Bad init could leave an index only accepting reads
+ public void testReadOnlyIndex() throws Throwable
+ {
+ // On successful initialization both reads and writes go through
+ String tableName = createTable("CREATE TABLE %s (pk int, ck int, value int, PRIMARY KEY (pk, ck))");
+ String indexName = createIndex("CREATE CUSTOM INDEX ON %s (value) USING '" + ReadOnlyOnFailureIndex.class.getName() + "'");
+ assertTrue(waitForIndex(keyspace(), tableName, indexName));
+ execute("SELECT value FROM %s WHERE value = 1");
+ execute("INSERT INTO %s (pk, ck, value) VALUES (?, ?, ?)", 1, 1, 1);
+ ReadOnlyOnFailureIndex index = (ReadOnlyOnFailureIndex) getCurrentColumnFamilyStore().indexManager.getIndexByName(indexName);
+ assertEquals(1, index.rowsInserted.size());
+
+ // Upon rebuild, both reads and writes still go through
+ getCurrentColumnFamilyStore().indexManager.rebuildIndexesBlocking(ImmutableSet.of(indexName));
+ assertEquals(1, index.rowsInserted.size());
+ execute("SELECT value FROM %s WHERE value = 1");
+ execute("INSERT INTO %s (pk, ck, value) VALUES (?, ?, ?)", 2, 1, 1);
+ assertEquals(2, index.rowsInserted.size());
+ dropIndex(format("DROP INDEX %s.%s", KEYSPACE, indexName));
+
+ // On bad initial build writes are not forwarded to the index
+ ReadOnlyOnFailureIndex.failInit = true;
+ indexName = createIndex("CREATE CUSTOM INDEX ON %s (value) USING '" + ReadOnlyOnFailureIndex.class.getName() + "'");
+ index = (ReadOnlyOnFailureIndex) getCurrentColumnFamilyStore().indexManager.getIndexByName(indexName);
+ assertTrue(waitForIndexBuilds(keyspace(), indexName));
+ assertInvalidThrow(IndexNotAvailableException.class, "SELECT value FROM %s WHERE value = 1");
+ execute("INSERT INTO %s (pk, ck, value) VALUES (?, ?, ?)", 1, 1, 1);
+ assertEquals(0, index.rowsInserted.size());
+
+ // Upon recovery, we can index data again
+ index.reset();
+ getCurrentColumnFamilyStore().indexManager.rebuildIndexesBlocking(ImmutableSet.of(indexName));
+ assertEquals(2, index.rowsInserted.size());
+ execute("SELECT value FROM %s WHERE value = 1");
+ execute("INSERT INTO %s (pk, ck, value) VALUES (?, ?, ?)", 2, 1, 1);
+ assertEquals(3, index.rowsInserted.size());
+ dropIndex(format("DROP INDEX %s.%s", KEYSPACE, indexName));
+ }
+
+ @Test // A Bad init could leave an index only accepting writes
+ public void testWriteOnlyIndex() throws Throwable
+ {
+ // On successful initialization both reads and writes go through
+ String tableName = createTable("CREATE TABLE %s (pk int, ck int, value int, PRIMARY KEY (pk, ck))");
+ String indexName = createIndex("CREATE CUSTOM INDEX ON %s (value) USING '" + WriteOnlyOnFailureIndex.class.getName() + "'");
+ assertTrue(waitForIndex(keyspace(), tableName, indexName));
+ execute("SELECT value FROM %s WHERE value = 1");
+ execute("INSERT INTO %s (pk, ck, value) VALUES (?, ?, ?)", 1, 1, 1);
+ WriteOnlyOnFailureIndex index = (WriteOnlyOnFailureIndex) getCurrentColumnFamilyStore().indexManager.getIndexByName(indexName);
+ assertEquals(1, index.rowsInserted.size());
+
+ // Upon rebuild, both reads and writes still go through
+ getCurrentColumnFamilyStore().indexManager.rebuildIndexesBlocking(ImmutableSet.of(indexName));
+ assertEquals(1, index.rowsInserted.size());
+ execute("SELECT value FROM %s WHERE value = 1");
+ execute("INSERT INTO %s (pk, ck, value) VALUES (?, ?, ?)", 2, 1, 1);
+ assertEquals(2, index.rowsInserted.size());
+ dropIndex(format("DROP INDEX %s.%s", KEYSPACE, indexName));
+
+ // On bad initial build writes are forwarded to the index
+ WriteOnlyOnFailureIndex.failInit = true;
+ indexName = createIndex("CREATE CUSTOM INDEX ON %s (value) USING '" + WriteOnlyOnFailureIndex.class.getName() + "'");
+ index = (WriteOnlyOnFailureIndex) getCurrentColumnFamilyStore().indexManager.getIndexByName(indexName);
+ assertTrue(waitForIndexBuilds(keyspace(), indexName));
+ execute("INSERT INTO %s (pk, ck, value) VALUES (?, ?, ?)", 1, 1, 1);
+ assertEquals(1, index.rowsInserted.size());
+ assertInvalidThrow(IndexNotAvailableException.class, "SELECT value FROM %s WHERE value = 1");
+
+ // Upon recovery, we can query data again
+ index.reset();
+ getCurrentColumnFamilyStore().indexManager.rebuildIndexesBlocking(ImmutableSet.of(indexName));
+ assertEquals(2, index.rowsInserted.size());
+ execute("SELECT value FROM %s WHERE value = 1");
+ execute("INSERT INTO %s (pk, ck, value) VALUES (?, ?, ?)", 2, 1, 1);
+ assertEquals(3, index.rowsInserted.size());
+ dropIndex(format("DROP INDEX %s.%s", KEYSPACE, indexName));
+ }
+
@Test
public void droppingIndexInvalidatesPreparedStatements() throws Throwable
{
createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY ((a), b))");
- createIndex("CREATE INDEX c_idx ON %s(c)");
- MD5Digest cqlId = prepareStatement("SELECT * FROM %s.%s WHERE c=?", false).statementId;
- Integer thriftId = prepareStatement("SELECT * FROM %s.%s WHERE c=?", true).toThriftPreparedResult().getItemId();
+ String indexName = createIndex("CREATE INDEX ON %s(c)");
+ MD5Digest cqlId = prepareStatement("SELECT * FROM %s.%s WHERE c=?").statementId;
assertNotNull(QueryProcessor.instance.getPrepared(cqlId));
- assertNotNull(QueryProcessor.instance.getPreparedForThrift(thriftId));
- dropIndex("DROP INDEX %s.c_idx");
+ dropIndex("DROP INDEX %s." + indexName);
assertNull(QueryProcessor.instance.getPrepared(cqlId));
- assertNull(QueryProcessor.instance.getPreparedForThrift(thriftId));
}
// See CASSANDRA-11021
@@ -1247,27 +1280,6 @@
}
@Test
- public void testEmptyRestrictionValueWithSecondaryIndexAndCompactTables() throws Throwable
- {
- createTable("CREATE TABLE %s (pk blob, c blob, v blob, PRIMARY KEY ((pk), c)) WITH COMPACT STORAGE");
- assertInvalidMessage("Secondary indexes are not supported on COMPACT STORAGE tables that have clustering columns",
- "CREATE INDEX on %s(c)");
-
- createTable("CREATE TABLE %s (pk blob PRIMARY KEY, v blob) WITH COMPACT STORAGE");
- createIndex("CREATE INDEX on %s(v)");
-
- execute("INSERT INTO %s (pk, v) VALUES (?, ?)", bytes("foo123"), bytes("1"));
-
- // Test restrictions on non-primary key value
- assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND v = textAsBlob('');"));
-
- execute("INSERT INTO %s (pk, v) VALUES (?, ?)", bytes("foo124"), EMPTY_BYTE_BUFFER);
-
- assertRows(execute("SELECT * FROM %s WHERE v = textAsBlob('');"),
- row(bytes("foo124"), EMPTY_BYTE_BUFFER));
- }
-
- @Test
public void testPartitionKeyWithIndex() throws Throwable
{
createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY ((a, b)))");
@@ -1442,10 +1454,11 @@
Object udt2 = userType("a", 2);
execute("INSERT INTO %s (k, v) VALUES (?, ?)", 0, udt1);
- execute("CREATE INDEX idx ON %s (v)");
+ String indexName = createIndex("CREATE INDEX ON %s (v)");
+
execute("INSERT INTO %s (k, v) VALUES (?, ?)", 1, udt2);
execute("INSERT INTO %s (k, v) VALUES (?, ?)", 1, udt1);
- assertTrue(waitForIndex(keyspace(), tableName, "idx"));
+ assertTrue(waitForIndex(keyspace(), tableName, indexName));
assertRows(execute("SELECT * FROM %s WHERE v = ?", udt1), row(1, udt1), row(0, udt1));
assertEmpty(execute("SELECT * FROM %s WHERE v = ?", udt2));
@@ -1453,8 +1466,9 @@
execute("DELETE FROM %s WHERE k = 0");
assertRows(execute("SELECT * FROM %s WHERE v = ?", udt1), row(1, udt1));
- dropIndex("DROP INDEX %s.idx");
- assertInvalidMessage("Index 'idx' could not be found", "DROP INDEX " + KEYSPACE + ".idx");
+ dropIndex("DROP INDEX %s." + indexName);
+ assertInvalidMessage(format("Index '%s.%s' doesn't exist", KEYSPACE, indexName),
+ format("DROP INDEX %s.%s", KEYSPACE, indexName));
assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
"SELECT * FROM %s WHERE v = ?", udt1);
}
@@ -1469,12 +1483,12 @@
Object udt2 = userType("a", 2);
execute("INSERT INTO %s (k, v) VALUES (?, ?)", 1, set(udt1, udt2));
- assertInvalidMessage("Frozen collections only support full()", "CREATE INDEX idx ON %s (keys(v))");
- assertInvalidMessage("Frozen collections only support full()", "CREATE INDEX idx ON %s (values(v))");
- execute("CREATE INDEX idx ON %s (full(v))");
+ assertInvalidMessage("Frozen collections are immutable and must be fully indexed", "CREATE INDEX idx ON %s (keys(v))");
+ assertInvalidMessage("Frozen collections are immutable and must be fully indexed", "CREATE INDEX idx ON %s (values(v))");
+ String indexName = createIndex("CREATE INDEX ON %s (full(v))");
execute("INSERT INTO %s (k, v) VALUES (?, ?)", 2, set(udt2));
- assertTrue(waitForIndex(keyspace(), tableName, "idx"));
+ assertTrue(waitForIndex(keyspace(), tableName, indexName));
assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
"SELECT * FROM %s WHERE v CONTAINS ?", udt1);
@@ -1485,8 +1499,9 @@
execute("DELETE FROM %s WHERE k = 2");
assertEmpty(execute("SELECT * FROM %s WHERE v = ?", set(udt2)));
- dropIndex("DROP INDEX %s.idx");
- assertInvalidMessage("Index 'idx' could not be found", "DROP INDEX " + KEYSPACE + ".idx");
+ dropIndex("DROP INDEX %s." + indexName);
+ assertInvalidMessage(format("Index '%s.%s' doesn't exist", KEYSPACE, indexName),
+ format("DROP INDEX %s.%s", KEYSPACE, indexName));
assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
"SELECT * FROM %s WHERE v CONTAINS ?", udt1);
}
@@ -1502,14 +1517,14 @@
execute("INSERT INTO %s (k, v) VALUES (?, ?)", 1, set(udt1));
assertInvalidMessage("Cannot create index on keys of column v with non-map type",
- "CREATE INDEX idx ON %s (keys(v))");
+ "CREATE INDEX ON %s (keys(v))");
assertInvalidMessage("full() indexes can only be created on frozen collections",
- "CREATE INDEX idx ON %s (full(v))");
- execute("CREATE INDEX idx ON %s (values(v))");
+ "CREATE INDEX ON %s (full(v))");
+ String indexName = createIndex("CREATE INDEX ON %s (values(v))");
execute("INSERT INTO %s (k, v) VALUES (?, ?)", 2, set(udt2));
execute("UPDATE %s SET v = v + ? WHERE k = ?", set(udt2), 1);
- assertTrue(waitForIndex(keyspace(), tableName, "idx"));
+ assertTrue(waitForIndex(keyspace(), tableName, indexName));
assertRows(execute("SELECT * FROM %s WHERE v CONTAINS ?", udt1), row(1, set(udt1, udt2)));
assertRows(execute("SELECT * FROM %s WHERE v CONTAINS ?", udt2), row(1, set(udt1, udt2)), row(2, set(udt2)));
@@ -1518,8 +1533,9 @@
assertEmpty(execute("SELECT * FROM %s WHERE v CONTAINS ?", udt1));
assertRows(execute("SELECT * FROM %s WHERE v CONTAINS ?", udt2), row(2, set(udt2)));
- dropIndex("DROP INDEX %s.idx");
- assertInvalidMessage("Index 'idx' could not be found", "DROP INDEX " + KEYSPACE + ".idx");
+ dropIndex("DROP INDEX %s." + indexName);
+ assertInvalidMessage(format("Index '%s.%s' doesn't exist", KEYSPACE, indexName),
+ format("DROP INDEX %s.%s", KEYSPACE, indexName));
assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
"SELECT * FROM %s WHERE v CONTAINS ?", udt1);
}
@@ -1529,9 +1545,9 @@
{
String type = createType("CREATE TYPE %s (a int)");
createTable("CREATE TABLE %s (k int PRIMARY KEY, v " + type + ")");
- assertInvalidMessage("Secondary indexes are not supported on non-frozen UDTs", "CREATE INDEX ON %s (v)");
- assertInvalidMessage("Non-collection columns support only simple indexes", "CREATE INDEX ON %s (keys(v))");
- assertInvalidMessage("Non-collection columns support only simple indexes", "CREATE INDEX ON %s (values(v))");
+ assertInvalidMessage("Cannot create index on non-frozen UDT column v", "CREATE INDEX ON %s (v)");
+ assertInvalidMessage("Cannot create keys() index on v. Non-collection columns only support simple indexes", "CREATE INDEX ON %s (keys(v))");
+ assertInvalidMessage("Cannot create values() index on v. Non-collection columns only support simple indexes", "CREATE INDEX ON %s (values(v))");
assertInvalidMessage("full() indexes can only be created on frozen collections", "CREATE INDEX ON %s (full(v))");
}
@@ -1577,66 +1593,22 @@
assertEmpty(execute("SELECT * FROM %s WHERE a = 5"));
}
- @Test
- public void testIndicesOnCompactTable() throws Throwable
+ private ResultMessage.Prepared prepareStatement(String cql)
{
- assertInvalidMessage("COMPACT STORAGE with composite PRIMARY KEY allows no more than one column not part of the PRIMARY KEY (got: v1, v2)",
- "CREATE TABLE test (pk int, c int, v1 int, v2 int, PRIMARY KEY(pk, c)) WITH COMPACT STORAGE");
-
- createTable("CREATE TABLE %s (pk int, c int, v int, PRIMARY KEY(pk, c)) WITH COMPACT STORAGE");
- assertInvalidMessage("Secondary indexes are not supported on COMPACT STORAGE tables that have clustering columns",
- "CREATE INDEX ON %s(v)");
-
- createTable("CREATE TABLE %s (pk int PRIMARY KEY, v int) WITH COMPACT STORAGE");
- createIndex("CREATE INDEX ON %s(v)");
-
- execute("INSERT INTO %s (pk, v) VALUES (?, ?)", 1, 1);
- execute("INSERT INTO %s (pk, v) VALUES (?, ?)", 2, 1);
- execute("INSERT INTO %s (pk, v) VALUES (?, ?)", 3, 3);
-
- assertRows(execute("SELECT pk, v FROM %s WHERE v = 1"),
- row(1, 1),
- row(2, 1));
-
- assertRows(execute("SELECT pk, v FROM %s WHERE v = 3"),
- row(3, 3));
-
- assertEmpty(execute("SELECT pk, v FROM %s WHERE v = 5"));
-
- createTable("CREATE TABLE %s (pk int PRIMARY KEY, v1 int, v2 int) WITH COMPACT STORAGE");
- createIndex("CREATE INDEX ON %s(v1)");
-
- execute("INSERT INTO %s (pk, v1, v2) VALUES (?, ?, ?)", 1, 1, 1);
- execute("INSERT INTO %s (pk, v1, v2) VALUES (?, ?, ?)", 2, 1, 2);
- execute("INSERT INTO %s (pk, v1, v2) VALUES (?, ?, ?)", 3, 3, 3);
-
- assertRows(execute("SELECT pk, v2 FROM %s WHERE v1 = 1"),
- row(1, 1),
- row(2, 2));
-
- assertRows(execute("SELECT pk, v2 FROM %s WHERE v1 = 3"),
- row(3, 3));
-
- assertEmpty(execute("SELECT pk, v2 FROM %s WHERE v1 = 5"));
- }
-
- private ResultMessage.Prepared prepareStatement(String cql, boolean forThrift)
- {
- return QueryProcessor.prepare(String.format(cql, KEYSPACE, currentTable()),
- ClientState.forInternalCalls(),
- forThrift);
+ return QueryProcessor.prepare(format(cql, KEYSPACE, currentTable()),
+ ClientState.forInternalCalls());
}
- private void validateCell(Cell cell, ColumnDefinition def, ByteBuffer val, long timestamp)
+ private void validateCell(Cell cell, ColumnMetadata def, ByteBuffer val, long timestamp)
{
assertNotNull(cell);
assertEquals(0, def.type.compare(cell.value(), val));
assertEquals(timestamp, cell.timestamp());
}
- private static void assertColumnValue(int expected, String name, Row row, CFMetaData cfm)
+ private static void assertColumnValue(int expected, String name, Row row, TableMetadata cfm)
{
- ColumnDefinition col = cfm.getColumnDefinition(new ColumnIdentifier(name, true));
+ ColumnMetadata col = cfm.getColumn(new ColumnIdentifier(name, true));
AbstractType<?> type = col.type;
assertEquals(expected, type.compose(row.getCell(col).value()));
}
@@ -1669,4 +1641,62 @@
return super.getInvalidateTask();
}
}
+
+ /**
+ * {@code StubIndex} that only supports some load. Could be intentional or a result of a bad init.
+ */
+ public static class LoadTypeConstrainedIndex extends StubIndex
+ {
+ static volatile boolean failInit = false;
+ final LoadType supportedLoadOnFailure;
+
+ LoadTypeConstrainedIndex(ColumnFamilyStore baseCfs, IndexMetadata indexDef, LoadType supportedLoadOnFailure)
+ {
+ super(baseCfs, indexDef);
+ this.supportedLoadOnFailure = supportedLoadOnFailure;
+ }
+
+ @Override
+ public LoadType getSupportedLoadTypeOnFailure(boolean isInitialBuild)
+ {
+ return supportedLoadOnFailure;
+ }
+
+ @Override
+ public void reset()
+ {
+ super.reset();
+ failInit = false;
+ }
+
+ @Override
+ public Callable<?> getInitializationTask()
+ {
+ if (failInit)
+ return () -> {throw new IllegalStateException("Index is configured to fail.");};
+
+ return null;
+ }
+
+ public boolean shouldBuildBlocking()
+ {
+ return true;
+ }
+ }
+
+ public static class ReadOnlyOnFailureIndex extends LoadTypeConstrainedIndex
+ {
+ public ReadOnlyOnFailureIndex(ColumnFamilyStore baseCfs, IndexMetadata indexDef)
+ {
+ super(baseCfs, indexDef, LoadType.READ);
+ }
+ }
+
+ public static class WriteOnlyOnFailureIndex extends LoadTypeConstrainedIndex
+ {
+ public WriteOnlyOnFailureIndex(ColumnFamilyStore baseCfs, IndexMetadata indexDef)
+ {
+ super(baseCfs, indexDef, LoadType.WRITE);
+ }
+ }
}
diff --git a/test/unit/org/apache/cassandra/cql3/validation/entities/TimestampTest.java b/test/unit/org/apache/cassandra/cql3/validation/entities/TimestampTest.java
index b41163c..63cd2b7 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/entities/TimestampTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/entities/TimestampTest.java
@@ -19,7 +19,7 @@
import org.junit.Test;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.cassandra.cql3.CQLTester;
import static junit.framework.Assert.assertNull;
@@ -181,4 +181,42 @@
row(2, 2, null, null),
row(3, 3, 3, 1L));
}
+
+ @Test
+ public void testTimestampAndTTLPrepared() throws Throwable
+ {
+
+ createTable("CREATE TABLE %s (k int , c int, i int, PRIMARY KEY (k, c))");
+ execute("INSERT INTO %s (k, c, i) VALUES (1, 1, 1) USING TIMESTAMP ? AND TTL ?;", 1L,5);
+ execute("INSERT INTO %s (k, c) VALUES (1, 2) USING TIMESTAMP ? AND TTL ? ;", 1L, 5);
+ execute("INSERT INTO %s (k, c, i) VALUES (1, 3, 1) USING TIMESTAMP ? AND TTL ?;", 1L, 5);
+ execute("INSERT INTO %s (k, c) VALUES (2, 2) USING TIMESTAMP ? AND TTL ?;", 2L, 5);
+ execute("INSERT INTO %s (k, c, i) VALUES (3, 3, 3) USING TIMESTAMP ? AND TTL ?;", 1L, 5);
+ assertRows(execute("SELECT k, c, i, writetime(i) FROM %s "),
+ row(1, 1, 1, 1L),
+ row(1, 2, null, null),
+ row(1, 3, 1, 1L),
+ row(2, 2, null, null),
+ row(3, 3, 3, 1L));
+ Thread.sleep(6*1000);
+ assertEmpty(execute("SELECT k, c, i, writetime(i) FROM %s "));
+ }
+
+ @Test
+ public void testTimestampAndTTLUpdatePrepared() throws Throwable
+ {
+
+ createTable("CREATE TABLE %s (k int , c int, i int, PRIMARY KEY (k, c))");
+ execute("UPDATE %s USING TIMESTAMP ? AND TTL ? SET i=1 WHERE k=1 AND c = 1 ;", 1L, 5);
+ execute("UPDATE %s USING TIMESTAMP ? AND TTL ? SET i=1 WHERE k=1 AND c = 3 ;", 1L, 5);
+ execute("UPDATE %s USING TIMESTAMP ? AND TTL ? SET i=1 WHERE k=2 AND c = 2 ;", 2L, 5);
+ execute("UPDATE %s USING TIMESTAMP ? AND TTL ? SET i=3 WHERE k=3 AND c = 3 ;", 1L, 5);
+ assertRows(execute("SELECT k, c, i, writetime(i) FROM %s "),
+ row(1, 1, 1, 1L),
+ row(1, 3, 1, 1L),
+ row(2, 2, 1, 2L),
+ row(3, 3, 3, 1L));
+ Thread.sleep(6*1000);
+ assertEmpty(execute("SELECT k, c, i, writetime(i) FROM %s "));
+ }
}
diff --git a/test/unit/org/apache/cassandra/cql3/validation/entities/TupleTypeTest.java b/test/unit/org/apache/cassandra/cql3/validation/entities/TupleTypeTest.java
index bace751..28430cb 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/entities/TupleTypeTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/entities/TupleTypeTest.java
@@ -217,6 +217,14 @@
}
@Test
+ public void testTupleModification() throws Throwable
+ {
+ createTable("CREATE TABLE %s(pk int PRIMARY KEY, value tuple<int, int>)");
+ assertInvalidMessage("Invalid operation (value = value + (1, 1)) for tuple column value",
+ "UPDATE %s SET value += (1, 1) WHERE k=0;");
+ }
+
+ @Test
public void testReversedTypeTuple() throws Throwable
{
// CASSANDRA-13717
@@ -226,3 +234,4 @@
assertRows(execute("SELECT tdemo FROM %s"), row(tuple( df.parse("2017-02-03 03:05+0000"), "Europe")));
}
}
+
diff --git a/test/unit/org/apache/cassandra/cql3/validation/entities/TypeTest.java b/test/unit/org/apache/cassandra/cql3/validation/entities/TypeTest.java
index 60a0fdc..ceb96b6 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/entities/TypeTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/entities/TypeTest.java
@@ -22,15 +22,14 @@
import org.junit.Test;
import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
public class TypeTest extends CQLTester
{
@Test
public void testNonExistingOnes() throws Throwable
{
- assertInvalidMessage("No user type named", "DROP TYPE " + KEYSPACE + ".type_does_not_exist");
- assertInvalidMessage("Cannot drop type in unknown keyspace", "DROP TYPE keyspace_does_not_exist.type_does_not_exist");
+ assertInvalidMessage(String.format("Type '%s.type_does_not_exist' doesn't exist", KEYSPACE), "DROP TYPE " + KEYSPACE + ".type_does_not_exist");
+ assertInvalidMessage("Type 'keyspace_does_not_exist.type_does_not_exist' doesn't exist", "DROP TYPE keyspace_does_not_exist.type_does_not_exist");
execute("DROP TYPE IF EXISTS " + KEYSPACE + ".type_does_not_exist");
execute("DROP TYPE IF EXISTS keyspace_does_not_exist.type_does_not_exist");
diff --git a/test/unit/org/apache/cassandra/cql3/validation/entities/UFAuthTest.java b/test/unit/org/apache/cassandra/cql3/validation/entities/UFAuthTest.java
index 3affe9a..4a2d71f 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/entities/UFAuthTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/entities/UFAuthTest.java
@@ -28,7 +28,7 @@
import org.apache.cassandra.auth.*;
import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.config.Schema;
+import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.cql3.*;
import org.apache.cassandra.cql3.functions.Function;
import org.apache.cassandra.cql3.functions.FunctionName;
@@ -226,7 +226,7 @@
functions.add(functionName);
statements.add(stmt);
}
- BatchStatement batch = new BatchStatement(-1, BatchStatement.Type.LOGGED, statements, Attributes.none());
+ BatchStatement batch = new BatchStatement(BatchStatement.Type.LOGGED, VariableSpecifications.empty(), statements, Attributes.none());
assertUnauthorized(batch, functions);
grantExecuteOnFunction(functions.get(0));
@@ -236,7 +236,7 @@
assertUnauthorized(batch, functions.subList(2, functions.size()));
grantExecuteOnFunction(functions.get(2));
- batch.checkAccess(clientState);
+ batch.authorize(clientState);
}
@Test
@@ -313,7 +313,7 @@
// with terminal arguments, so evaluated at prepare time
String cql = String.format("UPDATE %s SET v2 = 0 WHERE k = blobasint(intasblob(0)) and v1 = 0",
KEYSPACE + "." + currentTable());
- getStatement(cql).checkAccess(clientState);
+ getStatement(cql).authorize(clientState);
// with non-terminal arguments, so evaluated at execution
String functionName = createSimpleFunction();
@@ -321,7 +321,7 @@
cql = String.format("UPDATE %s SET v2 = 0 WHERE k = blobasint(intasblob(%s)) and v1 = 0",
KEYSPACE + "." + currentTable(),
functionCall(functionName));
- getStatement(cql).checkAccess(clientState);
+ getStatement(cql).authorize(clientState);
}
@Test
@@ -343,7 +343,7 @@
assertUnauthorized(aggDef, fFunc, "int");
grantExecuteOnFunction(fFunc);
- getStatement(aggDef).checkAccess(clientState);
+ getStatement(aggDef).authorize(clientState);
}
@Test
@@ -361,24 +361,24 @@
String cql = String.format("SELECT %s(v1) FROM %s",
aggregate,
KEYSPACE + "." + currentTable());
- getStatement(cql).checkAccess(clientState);
+ getStatement(cql).authorize(clientState);
// check that revoking EXECUTE permission on any one of the
// component functions means we lose the ability to execute it
revokeExecuteOnFunction(aggregate);
assertUnauthorized(cql, aggregate, "int");
grantExecuteOnFunction(aggregate);
- getStatement(cql).checkAccess(clientState);
+ getStatement(cql).authorize(clientState);
revokeExecuteOnFunction(sFunc);
assertUnauthorized(cql, sFunc, "int, int");
grantExecuteOnFunction(sFunc);
- getStatement(cql).checkAccess(clientState);
+ getStatement(cql).authorize(clientState);
revokeExecuteOnFunction(fFunc);
assertUnauthorized(cql, fFunc, "int");
grantExecuteOnFunction(fFunc);
- getStatement(cql).checkAccess(clientState);
+ getStatement(cql).authorize(clientState);
}
@Test
@@ -410,7 +410,7 @@
assertUnauthorized(cql, aggregate, "int");
grantExecuteOnFunction(aggregate);
- getStatement(cql).checkAccess(clientState);
+ getStatement(cql).authorize(clientState);
}
@Test
@@ -442,7 +442,7 @@
assertUnauthorized(cql, innerFunc, "int");
grantExecuteOnFunction(innerFunc);
- getStatement(cql).checkAccess(clientState);
+ getStatement(cql).authorize(clientState);
}
@Test
@@ -484,7 +484,7 @@
grantExecuteOnFunction(innerFunction);
// now execution of both is permitted
- getStatement(cql).checkAccess(clientState);
+ getStatement(cql).authorize(clientState);
}
private void assertPermissionsOnFunction(String cql, String functionName) throws Throwable
@@ -496,14 +496,14 @@
{
assertUnauthorized(cql, functionName, argTypes);
grantExecuteOnFunction(functionName);
- getStatement(cql).checkAccess(clientState);
+ getStatement(cql).authorize(clientState);
}
private void assertUnauthorized(BatchStatement batch, Iterable<String> functionNames) throws Throwable
{
try
{
- batch.checkAccess(clientState);
+ batch.authorize(clientState);
fail("Expected an UnauthorizedException, but none was thrown");
}
catch (UnauthorizedException e)
@@ -520,7 +520,7 @@
{
try
{
- getStatement(cql).checkAccess(clientState);
+ getStatement(cql).authorize(clientState);
fail("Expected an UnauthorizedException, but none was thrown");
}
catch (UnauthorizedException e)
@@ -625,7 +625,7 @@
private CQLStatement getStatement(String cql)
{
- return QueryProcessor.getStatement(cql, clientState).statement;
+ return QueryProcessor.getStatement(cql, clientState);
}
private FunctionResource functionResource(String functionName)
diff --git a/test/unit/org/apache/cassandra/cql3/validation/entities/UFIdentificationTest.java b/test/unit/org/apache/cassandra/cql3/validation/entities/UFIdentificationTest.java
index b2288e4..bba5c92 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/entities/UFIdentificationTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/entities/UFIdentificationTest.java
@@ -28,6 +28,7 @@
import org.apache.cassandra.cql3.Attributes;
import org.apache.cassandra.cql3.CQLStatement;
import org.apache.cassandra.cql3.QueryProcessor;
+import org.apache.cassandra.cql3.VariableSpecifications;
import org.apache.cassandra.cql3.functions.Function;
import org.apache.cassandra.cql3.statements.BatchStatement;
import org.apache.cassandra.cql3.statements.ModificationStatement;
@@ -307,7 +308,7 @@
statements.add(modificationStatement(cql("INSERT INTO %s (key, i_cc, t_cc) VALUES (2, 2, %s)",
functionCall(tFunc, "'foo'"))));
- BatchStatement batch = new BatchStatement(-1, BatchStatement.Type.LOGGED, statements, Attributes.none());
+ BatchStatement batch = new BatchStatement(BatchStatement.Type.LOGGED, VariableSpecifications.empty(), statements, Attributes.none());
assertFunctions(batch, iFunc, iFunc2, tFunc);
}
@@ -320,18 +321,18 @@
statements.add(modificationStatement(cql("UPDATE %s SET i_val = %s WHERE key=0 AND i_cc=1 and t_cc='foo' IF s_val = %s",
functionCall(iFunc, "0"), functionCall(sFunc, "{1}"))));
- BatchStatement batch = new BatchStatement(-1, BatchStatement.Type.LOGGED, statements, Attributes.none());
+ BatchStatement batch = new BatchStatement(BatchStatement.Type.LOGGED, VariableSpecifications.empty(), statements, Attributes.none());
assertFunctions(batch, iFunc, lFunc, sFunc);
}
private ModificationStatement modificationStatement(String cql)
{
- return (ModificationStatement) QueryProcessor.getStatement(cql, ClientState.forInternalCalls()).statement;
+ return (ModificationStatement) QueryProcessor.getStatement(cql, ClientState.forInternalCalls());
}
private void assertFunctions(String cql, String... function)
{
- CQLStatement stmt = QueryProcessor.getStatement(cql, ClientState.forInternalCalls()).statement;
+ CQLStatement stmt = QueryProcessor.getStatement(cql, ClientState.forInternalCalls());
assertFunctions(stmt, function);
}
diff --git a/test/unit/org/apache/cassandra/cql3/validation/entities/UFJavaTest.java b/test/unit/org/apache/cassandra/cql3/validation/entities/UFJavaTest.java
index 4d46b8b..596e944 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/entities/UFJavaTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/entities/UFJavaTest.java
@@ -33,13 +33,13 @@
import com.datastax.driver.core.TupleType;
import com.datastax.driver.core.TupleValue;
import com.datastax.driver.core.UDTValue;
-import org.apache.cassandra.config.Schema;
import org.apache.cassandra.cql3.CQL3Type;
import org.apache.cassandra.cql3.CQLTester;
import org.apache.cassandra.cql3.UntypedResultSet;
import org.apache.cassandra.cql3.functions.FunctionName;
import org.apache.cassandra.exceptions.FunctionExecutionException;
import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.transport.ProtocolVersion;
public class UFJavaTest extends CQLTester
@@ -106,15 +106,15 @@
catch (InvalidRequestException e)
{
Assert.assertTrue(e.getMessage(), e.getMessage().contains("Java source compilation failed"));
- Assert.assertTrue(e.getMessage(), e.getMessage().contains("foobarbaz cannot be resolved to a type"));
+ Assert.assertTrue(e.getMessage(), e.getMessage().contains("foobarbaz cannot be resolved"));
}
}
@Test
public void testJavaFunctionInvalidReturn() throws Throwable
{
- assertInvalidMessage("system keyspace is not user-modifiable",
- "CREATE OR REPLACE FUNCTION jfir(val double) " +
+ assertInvalidMessage("cannot convert from long to Double",
+ "CREATE OR REPLACE FUNCTION " + KEYSPACE + ".jfir(val double) " +
"RETURNS NULL ON NULL INPUT " +
"RETURNS double " +
"LANGUAGE JAVA\n" +
@@ -476,7 +476,7 @@
{
List<Row> rowsNet = executeNet(version, "SELECT f_use1(udt) FROM %s WHERE key = 1").all();
Assert.assertEquals(1, rowsNet.size());
- UDTValue udtVal = rowsNet.get(0).getUDTValue(0);
+ com.datastax.driver.core.UDTValue udtVal = rowsNet.get(0).getUDTValue(0);
Assert.assertEquals("one", udtVal.getString("txt"));
Assert.assertEquals(1, udtVal.getInt("i"));
}
@@ -717,13 +717,15 @@
"(key int primary key, lst list<frozen<%s>>, st set<frozen<%s>>, mp map<int, frozen<%s>>)",
type, type, type));
+ // The mix of the package names org.apache.cassandra.cql3.functions.types and com.datastax.driver.core is
+ // intentional to test the replacement of com.datastax.driver.core with org.apache.cassandra.cql3.functions.types.
String fName1 = createFunction(KEYSPACE, "list<frozen<" + type + ">>",
"CREATE FUNCTION %s( lst list<frozen<" + type + ">> ) " +
"RETURNS NULL ON NULL INPUT " +
"RETURNS text " +
"LANGUAGE java\n" +
"AS $$" +
- " com.datastax.driver.core.UDTValue udtVal = (com.datastax.driver.core.UDTValue)lst.get(1);" +
+ " org.apache.cassandra.cql3.functions.types.UDTValue udtVal = (com.datastax.driver.core.UDTValue)lst.get(1);" +
" return udtVal.getString(\"txt\");$$;");
String fName2 = createFunction(KEYSPACE, "set<frozen<" + type + ">>",
"CREATE FUNCTION %s( st set<frozen<" + type + ">> ) " +
@@ -731,7 +733,7 @@
"RETURNS text " +
"LANGUAGE java\n" +
"AS $$" +
- " com.datastax.driver.core.UDTValue udtVal = (com.datastax.driver.core.UDTValue)st.iterator().next();" +
+ " com.datastax.driver.core.UDTValue udtVal = (org.apache.cassandra.cql3.functions.types.UDTValue)st.iterator().next();" +
" return udtVal.getString(\"txt\");$$;");
String fName3 = createFunction(KEYSPACE, "map<int, frozen<" + type + ">>",
"CREATE FUNCTION %s( mp map<int, frozen<" + type + ">> ) " +
@@ -739,7 +741,7 @@
"RETURNS text " +
"LANGUAGE java\n" +
"AS $$" +
- " com.datastax.driver.core.UDTValue udtVal = (com.datastax.driver.core.UDTValue)mp.get(Integer.valueOf(3));" +
+ " org.apache.cassandra.cql3.functions.types.UDTValue udtVal = (com.datastax.driver.core.UDTValue)mp.get(Integer.valueOf(3));" +
" return udtVal.getString(\"txt\");$$;");
execute("INSERT INTO %s (key, lst, st, mp) values (1, " +
diff --git a/test/unit/org/apache/cassandra/cql3/validation/entities/UFPureScriptTest.java b/test/unit/org/apache/cassandra/cql3/validation/entities/UFPureScriptTest.java
deleted file mode 100644
index d5f17e1..0000000
--- a/test/unit/org/apache/cassandra/cql3/validation/entities/UFPureScriptTest.java
+++ /dev/null
@@ -1,407 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.cql3.validation.entities;
-
-import java.math.BigDecimal;
-import java.math.BigInteger;
-import java.util.Arrays;
-import java.util.Date;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeMap;
-import java.util.TreeSet;
-import java.util.UUID;
-
-import org.junit.Assert;
-import org.junit.Test;
-
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.cql3.CQLTester;
-import org.apache.cassandra.cql3.UntypedResultSet;
-import org.apache.cassandra.cql3.functions.FunctionName;
-import org.apache.cassandra.exceptions.FunctionExecutionException;
-import org.apache.cassandra.transport.ProtocolVersion;
-import org.apache.cassandra.utils.UUIDGen;
-
-public class UFPureScriptTest extends CQLTester
-{
- // Just JavaScript UDFs to check how UDF - especially security/class-loading/sandboxing stuff -
- // behaves, if no Java UDF has been executed before.
-
- // Do not add any other test here - especially none using Java UDFs
-
- @Test
- public void testJavascriptSimpleCollections() throws Throwable
- {
- createTable("CREATE TABLE %s (key int primary key, lst list<double>, st set<text>, mp map<int, boolean>)");
-
- String fName1 = createFunction(KEYSPACE_PER_TEST, "list<double>",
- "CREATE FUNCTION %s( lst list<double> ) " +
- "RETURNS NULL ON NULL INPUT " +
- "RETURNS list<double> " +
- "LANGUAGE javascript\n" +
- "AS 'lst;';");
- String fName2 = createFunction(KEYSPACE_PER_TEST, "set<text>",
- "CREATE FUNCTION %s( st set<text> ) " +
- "RETURNS NULL ON NULL INPUT " +
- "RETURNS set<text> " +
- "LANGUAGE javascript\n" +
- "AS 'st;';");
- String fName3 = createFunction(KEYSPACE_PER_TEST, "map<int, boolean>",
- "CREATE FUNCTION %s( mp map<int, boolean> ) " +
- "RETURNS NULL ON NULL INPUT " +
- "RETURNS map<int, boolean> " +
- "LANGUAGE javascript\n" +
- "AS 'mp;';");
-
- List<Double> list = Arrays.asList(1d, 2d, 3d);
- Set<String> set = new TreeSet<>(Arrays.asList("one", "three", "two"));
- Map<Integer, Boolean> map = new TreeMap<>();
- map.put(1, true);
- map.put(2, false);
- map.put(3, true);
-
- execute("INSERT INTO %s (key, lst, st, mp) VALUES (1, ?, ?, ?)", list, set, map);
-
- assertRows(execute("SELECT lst, st, mp FROM %s WHERE key = 1"),
- row(list, set, map));
-
- assertRows(execute("SELECT " + fName1 + "(lst), " + fName2 + "(st), " + fName3 + "(mp) FROM %s WHERE key = 1"),
- row(list, set, map));
-
- for (ProtocolVersion version : PROTOCOL_VERSIONS)
- assertRowsNet(version,
- executeNet(version, "SELECT " + fName1 + "(lst), " + fName2 + "(st), " + fName3 + "(mp) FROM %s WHERE key = 1"),
- row(list, set, map));
- }
-
- @Test
- public void testJavascriptTupleType() throws Throwable
- {
- createTable("CREATE TABLE %s (key int primary key, tup frozen<tuple<double, text, int, boolean>>)");
-
- String fName = createFunction(KEYSPACE_PER_TEST, "tuple<double, text, int, boolean>",
- "CREATE FUNCTION %s( tup tuple<double, text, int, boolean> ) " +
- "RETURNS NULL ON NULL INPUT " +
- "RETURNS tuple<double, text, int, boolean> " +
- "LANGUAGE javascript\n" +
- "AS $$tup;$$;");
-
- Object t = tuple(1d, "foo", 2, true);
-
- execute("INSERT INTO %s (key, tup) VALUES (1, ?)", t);
-
- assertRows(execute("SELECT tup FROM %s WHERE key = 1"),
- row(t));
-
- assertRows(execute("SELECT " + fName + "(tup) FROM %s WHERE key = 1"),
- row(t));
- }
-
- @Test
- public void testJavascriptUserType() throws Throwable
- {
- String type = createType("CREATE TYPE %s (txt text, i int)");
-
- createTable("CREATE TABLE %s (key int primary key, udt frozen<" + type + ">)");
-
- String fUdt1 = createFunction(KEYSPACE, type,
- "CREATE FUNCTION %s( udt " + type + " ) " +
- "RETURNS NULL ON NULL INPUT " +
- "RETURNS " + type + ' ' +
- "LANGUAGE javascript\n" +
- "AS $$" +
- " udt;$$;");
- String fUdt2 = createFunction(KEYSPACE, type,
- "CREATE FUNCTION %s( udt " + type + " ) " +
- "RETURNS NULL ON NULL INPUT " +
- "RETURNS text " +
- "LANGUAGE javascript\n" +
- "AS $$" +
- " udt.getString(\"txt\");$$;");
- String fUdt3 = createFunction(KEYSPACE, type,
- "CREATE FUNCTION %s( udt " + type + " ) " +
- "RETURNS NULL ON NULL INPUT " +
- "RETURNS int " +
- "LANGUAGE javascript\n" +
- "AS $$" +
- " udt.getInt(\"i\");$$;");
-
- execute("INSERT INTO %s (key, udt) VALUES (1, {txt: 'one', i:1})");
-
- UntypedResultSet rows = execute("SELECT " + fUdt1 + "(udt) FROM %s WHERE key = 1");
- Assert.assertEquals(1, rows.size());
- assertRows(execute("SELECT " + fUdt2 + "(udt) FROM %s WHERE key = 1"),
- row("one"));
- assertRows(execute("SELECT " + fUdt3 + "(udt) FROM %s WHERE key = 1"),
- row(1));
- }
-
- @Test
- public void testJavascriptUTCollections() throws Throwable
- {
- String type = createType("CREATE TYPE %s (txt text, i int)");
-
- createTable(String.format("CREATE TABLE %%s " +
- "(key int primary key, lst list<frozen<%s>>, st set<frozen<%s>>, mp map<int, frozen<%s>>)",
- type, type, type));
-
- String fName = createFunction(KEYSPACE, "list<frozen<" + type + ">>",
- "CREATE FUNCTION %s( lst list<frozen<" + type + ">> ) " +
- "RETURNS NULL ON NULL INPUT " +
- "RETURNS text " +
- "LANGUAGE javascript\n" +
- "AS $$" +
- " lst.get(1).getString(\"txt\");$$;");
- createFunctionOverload(fName, "set<frozen<" + type + ">>",
- "CREATE FUNCTION %s( st set<frozen<" + type + ">> ) " +
- "RETURNS NULL ON NULL INPUT " +
- "RETURNS text " +
- "LANGUAGE javascript\n" +
- "AS $$" +
- " st.iterator().next().getString(\"txt\");$$;");
- createFunctionOverload(fName, "map<int, frozen<" + type + ">>",
- "CREATE FUNCTION %s( mp map<int, frozen<" + type + ">> ) " +
- "RETURNS NULL ON NULL INPUT " +
- "RETURNS text " +
- "LANGUAGE javascript\n" +
- "AS $$" +
- " mp.get(java.lang.Integer.valueOf(3)).getString(\"txt\");$$;");
-
- execute("INSERT INTO %s (key, lst, st, mp) values (1, " +
- // list<frozen<UDT>>
- "[ {txt: 'one', i:1}, {txt: 'three', i:1}, {txt: 'one', i:1} ] , " +
- // set<frozen<UDT>>
- "{ {txt: 'one', i:1}, {txt: 'three', i:3}, {txt: 'two', i:2} }, " +
- // map<int, frozen<UDT>>
- "{ 1: {txt: 'one', i:1}, 2: {txt: 'one', i:3}, 3: {txt: 'two', i:2} })");
-
- assertRows(execute("SELECT " + fName + "(lst) FROM %s WHERE key = 1"),
- row("three"));
- assertRows(execute("SELECT " + fName + "(st) FROM %s WHERE key = 1"),
- row("one"));
- assertRows(execute("SELECT " + fName + "(mp) FROM %s WHERE key = 1"),
- row("two"));
-
- String cqlSelect = "SELECT " + fName + "(lst), " + fName + "(st), " + fName + "(mp) FROM %s WHERE key = 1";
- assertRows(execute(cqlSelect),
- row("three", "one", "two"));
-
- // same test - but via native protocol
- for (ProtocolVersion version : PROTOCOL_VERSIONS)
- assertRowsNet(version,
- executeNet(version, cqlSelect),
- row("three", "one", "two"));
- }
-
- @Test
- public void testJavascriptFunction() throws Throwable
- {
- createTable("CREATE TABLE %s (key int primary key, val double)");
-
- String functionBody = '\n' +
- " Math.sin(val);\n";
-
- String fName = createFunction(KEYSPACE, "double",
- "CREATE OR REPLACE FUNCTION %s(val double) " +
- "RETURNS NULL ON NULL INPUT " +
- "RETURNS double " +
- "LANGUAGE javascript\n" +
- "AS '" + functionBody + "';");
-
- FunctionName fNameName = parseFunctionName(fName);
-
- assertRows(execute("SELECT language, body FROM system_schema.functions WHERE keyspace_name=? AND function_name=?",
- fNameName.keyspace, fNameName.name),
- row("javascript", functionBody));
-
- execute("INSERT INTO %s (key, val) VALUES (?, ?)", 1, 1d);
- execute("INSERT INTO %s (key, val) VALUES (?, ?)", 2, 2d);
- execute("INSERT INTO %s (key, val) VALUES (?, ?)", 3, 3d);
- assertRows(execute("SELECT key, val, " + fName + "(val) FROM %s"),
- row(1, 1d, Math.sin(1d)),
- row(2, 2d, Math.sin(2d)),
- row(3, 3d, Math.sin(3d))
- );
- }
-
- @Test
- public void testJavascriptBadReturnType() throws Throwable
- {
- createTable("CREATE TABLE %s (key int primary key, val double)");
-
- String fName = createFunction(KEYSPACE, "double",
- "CREATE OR REPLACE FUNCTION %s(val double) " +
- "RETURNS NULL ON NULL INPUT " +
- "RETURNS double " +
- "LANGUAGE javascript\n" +
- "AS '\"string\";';");
-
- execute("INSERT INTO %s (key, val) VALUES (?, ?)", 1, 1d);
- // throws IRE with ClassCastException
- assertInvalidMessage("Invalid value for CQL type double", "SELECT key, val, " + fName + "(val) FROM %s");
- }
-
- @Test
- public void testJavascriptThrow() throws Throwable
- {
- createTable("CREATE TABLE %s (key int primary key, val double)");
-
- String fName = createFunction(KEYSPACE, "double",
- "CREATE OR REPLACE FUNCTION %s(val double) " +
- "RETURNS NULL ON NULL INPUT " +
- "RETURNS double " +
- "LANGUAGE javascript\n" +
- "AS 'throw \"fool\";';");
-
- execute("INSERT INTO %s (key, val) VALUES (?, ?)", 1, 1d);
- // throws IRE with ScriptException
- assertInvalidThrowMessage("fool", FunctionExecutionException.class,
- "SELECT key, val, " + fName + "(val) FROM %s");
- }
-
- @Test
- public void testScriptReturnTypeCasting() throws Throwable
- {
- createTable("CREATE TABLE %s (key int primary key, val double)");
- execute("INSERT INTO %s (key, val) VALUES (?, ?)", 1, 1d);
-
- Object[][] variations = {
- new Object[] { "true", "boolean", true },
- new Object[] { "false", "boolean", false },
- new Object[] { "100", "tinyint", (byte)100 },
- new Object[] { "100.", "tinyint", (byte)100 },
- new Object[] { "100", "smallint", (short)100 },
- new Object[] { "100.", "smallint", (short)100 },
- new Object[] { "100", "int", 100 },
- new Object[] { "100.", "int", 100 },
- new Object[] { "100", "double", 100d },
- new Object[] { "100.", "double", 100d },
- new Object[] { "100", "bigint", 100L },
- new Object[] { "100.", "bigint", 100L },
- new Object[] { "100", "varint", BigInteger.valueOf(100L) },
- new Object[] { "100.", "varint", BigInteger.valueOf(100L) },
- new Object[] { "parseInt(\"100\");", "decimal", BigDecimal.valueOf(100d) },
- new Object[] { "100.", "decimal", BigDecimal.valueOf(100d) },
- };
-
- for (Object[] variation : variations)
- {
- Object functionBody = variation[0];
- Object returnType = variation[1];
- Object expectedResult = variation[2];
-
- String fName = createFunction(KEYSPACE, "double",
- "CREATE OR REPLACE FUNCTION %s(val double) " +
- "RETURNS NULL ON NULL INPUT " +
- "RETURNS " +returnType + ' ' +
- "LANGUAGE javascript " +
- "AS '" + functionBody + ";';");
- assertRows(execute("SELECT key, val, " + fName + "(val) FROM %s"),
- row(1, 1d, expectedResult));
- }
- }
-
- @Test
- public void testScriptParamReturnTypes() throws Throwable
- {
- UUID ruuid = UUID.randomUUID();
- UUID tuuid = UUIDGen.getTimeUUID();
-
- createTable("CREATE TABLE %s (key int primary key, " +
- "tival tinyint, sival smallint, ival int, lval bigint, fval float, dval double, vval varint, ddval decimal, " +
- "timval time, dtval date, tsval timestamp, uval uuid, tuval timeuuid)");
- execute("INSERT INTO %s (key, tival, sival, ival, lval, fval, dval, vval, ddval, timval, dtval, tsval, uval, tuval) VALUES " +
- "(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", 1,
- (byte)1, (short)1, 1, 1L, 1f, 1d, BigInteger.valueOf(1L), BigDecimal.valueOf(1d), 1L, Integer.MAX_VALUE, new Date(1), ruuid, tuuid);
-
- Object[][] variations = {
- new Object[] { "tinyint", "tival", (byte)1, (byte)2 },
- new Object[] { "smallint", "sival", (short)1, (short)2 },
- new Object[] { "int", "ival", 1, 2 },
- new Object[] { "bigint", "lval", 1L, 2L },
- new Object[] { "float", "fval", 1f, 2f },
- new Object[] { "double", "dval", 1d, 2d },
- new Object[] { "varint", "vval", BigInteger.valueOf(1L), BigInteger.valueOf(2L) },
- new Object[] { "decimal", "ddval", BigDecimal.valueOf(1d), BigDecimal.valueOf(2d) },
- new Object[] { "time", "timval", 1L, 2L },
- };
-
- for (Object[] variation : variations)
- {
- Object type = variation[0];
- Object col = variation[1];
- Object expected1 = variation[2];
- Object expected2 = variation[3];
- String fName = createFunction(KEYSPACE, type.toString(),
- "CREATE OR REPLACE FUNCTION %s(val " + type + ") " +
- "RETURNS NULL ON NULL INPUT " +
- "RETURNS " + type + ' ' +
- "LANGUAGE javascript " +
- "AS 'val+1;';");
- assertRows(execute("SELECT key, " + col + ", " + fName + '(' + col + ") FROM %s"),
- row(1, expected1, expected2));
- }
-
- variations = new Object[][] {
- new Object[] { "timestamp","tsval", new Date(1), new Date(1) },
- new Object[] { "uuid", "uval", ruuid, ruuid },
- new Object[] { "timeuuid", "tuval", tuuid, tuuid },
- new Object[] { "date", "dtval", Integer.MAX_VALUE, Integer.MAX_VALUE },
- };
-
- for (Object[] variation : variations)
- {
- Object type = variation[0];
- Object col = variation[1];
- Object expected1 = variation[2];
- Object expected2 = variation[3];
- String fName = createFunction(KEYSPACE, type.toString(),
- "CREATE OR REPLACE FUNCTION %s(val " + type + ") " +
- "RETURNS NULL ON NULL INPUT " +
- "RETURNS " + type + ' ' +
- "LANGUAGE javascript " +
- "AS 'val;';");
- assertRows(execute("SELECT key, " + col + ", " + fName + '(' + col + ") FROM %s"),
- row(1, expected1, expected2));
- }
- }
-
- @Test
- public void testJavascriptDisabled() throws Throwable
- {
- createTable("CREATE TABLE %s (key int primary key, val double)");
-
- DatabaseDescriptor.enableScriptedUserDefinedFunctions(false);
- try
- {
- assertInvalid("double",
- "CREATE OR REPLACE FUNCTION " + KEYSPACE + ".assertNotEnabled(val double) " +
- "RETURNS NULL ON NULL INPUT " +
- "RETURNS double " +
- "LANGUAGE javascript\n" +
- "AS 'Math.sin(val);';");
- }
- finally
- {
- DatabaseDescriptor.enableScriptedUserDefinedFunctions(true);
- }
- }
-}
diff --git a/test/unit/org/apache/cassandra/cql3/validation/entities/UFSecurityTest.java b/test/unit/org/apache/cassandra/cql3/validation/entities/UFSecurityTest.java
index 4e45a8a..7422db43 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/entities/UFSecurityTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/entities/UFSecurityTest.java
@@ -57,11 +57,22 @@
assertAccessControlException("System.getProperty(\"foo.bar.baz\"); return 0d;", e);
}
+ String[] cfnSources =
+ { "try { Class.forName(\"" + UDHelper.class.getName() + "\"); } catch (Exception e) { throw new RuntimeException(e); } return 0d;",
+ "try { Class.forName(\"sun.misc.Unsafe\"); } catch (Exception e) { throw new RuntimeException(e); } return 0d;" };
+ for (String source : cfnSources)
+ {
+ assertInvalidMessage("Java UDF validation failed: [call to java.lang.Class.forName()]",
+ "CREATE OR REPLACE FUNCTION " + KEYSPACE + ".invalid_class_access(val double) " +
+ "RETURNS NULL ON NULL INPUT " +
+ "RETURNS double " +
+ "LANGUAGE JAVA\n" +
+ "AS '" + source + "';");
+ }
+
String[][] typesAndSources =
{
- {"", "try { Class.forName(\"" + UDHelper.class.getName() + "\"); } catch (Exception e) { throw new RuntimeException(e); } return 0d;"},
{"sun.misc.Unsafe", "sun.misc.Unsafe.getUnsafe(); return 0d;"},
- {"", "try { Class.forName(\"sun.misc.Unsafe\"); } catch (Exception e) { throw new RuntimeException(e); } return 0d;"},
{"java.nio.file.FileSystems", "try {" +
" java.nio.file.FileSystems.getDefault(); return 0d;" +
"} catch (Exception t) {" +
diff --git a/test/unit/org/apache/cassandra/cql3/validation/entities/UFTest.java b/test/unit/org/apache/cassandra/cql3/validation/entities/UFTest.java
index 6f3616c..d21d159 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/entities/UFTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/entities/UFTest.java
@@ -17,6 +17,7 @@
*/
package org.apache.cassandra.cql3.validation.entities;
+import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
@@ -27,7 +28,6 @@
import com.datastax.driver.core.*;
import com.datastax.driver.core.exceptions.InvalidQueryException;
-import org.apache.cassandra.config.Schema;
import org.apache.cassandra.cql3.CQLTester;
import org.apache.cassandra.cql3.QueryProcessor;
import org.apache.cassandra.cql3.UntypedResultSet;
@@ -36,12 +36,13 @@
import org.apache.cassandra.cql3.functions.UDFunction;
import org.apache.cassandra.db.marshal.CollectionType;
import org.apache.cassandra.exceptions.InvalidRequestException;
-import org.apache.cassandra.exceptions.SyntaxException;
import org.apache.cassandra.schema.KeyspaceMetadata;
+import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.service.ClientState;
import org.apache.cassandra.transport.*;
import org.apache.cassandra.transport.ProtocolVersion;
import org.apache.cassandra.transport.messages.ResultMessage;
+import org.apache.cassandra.utils.ByteBufferUtil;
public class UFTest extends CQLTester
{
@@ -57,10 +58,21 @@
@Test
public void testNonExistingOnes() throws Throwable
{
- assertInvalidThrowMessage("Cannot drop non existing function", InvalidRequestException.class, "DROP FUNCTION " + KEYSPACE + ".func_does_not_exist");
- assertInvalidThrowMessage("Cannot drop non existing function", InvalidRequestException.class, "DROP FUNCTION " + KEYSPACE + ".func_does_not_exist(int,text)");
- assertInvalidThrowMessage("Cannot drop non existing function", InvalidRequestException.class, "DROP FUNCTION keyspace_does_not_exist.func_does_not_exist");
- assertInvalidThrowMessage("Cannot drop non existing function", InvalidRequestException.class, "DROP FUNCTION keyspace_does_not_exist.func_does_not_exist(int,text)");
+ assertInvalidThrowMessage(String.format("Function '%s.func_does_not_exist' doesn't exist", KEYSPACE),
+ InvalidRequestException.class,
+ "DROP FUNCTION " + KEYSPACE + ".func_does_not_exist");
+
+ assertInvalidThrowMessage(String.format("Function '%s.func_does_not_exist(int, text)' doesn't exist", KEYSPACE),
+ InvalidRequestException.class,
+ "DROP FUNCTION " + KEYSPACE + ".func_does_not_exist(int, text)");
+
+ assertInvalidThrowMessage("Function 'keyspace_does_not_exist.func_does_not_exist' doesn't exist",
+ InvalidRequestException.class,
+ "DROP FUNCTION keyspace_does_not_exist.func_does_not_exist");
+
+ assertInvalidThrowMessage("Function 'keyspace_does_not_exist.func_does_not_exist(int, text)' doesn't exist",
+ InvalidRequestException.class,
+ "DROP FUNCTION keyspace_does_not_exist.func_does_not_exist(int, text)");
execute("DROP FUNCTION IF EXISTS " + KEYSPACE + ".func_does_not_exist");
execute("DROP FUNCTION IF EXISTS " + KEYSPACE + ".func_does_not_exist(int,text)");
@@ -99,7 +111,7 @@
"RETURNS NULL ON NULL INPUT " +
"RETURNS int " +
"LANGUAGE javascript " +
- "AS '\"string\";';");
+ "AS '\"string1\";';");
assertLastSchemaChange(Event.SchemaChange.Change.UPDATED, Event.SchemaChange.Target.FUNCTION,
KEYSPACE, parseFunctionName(f).name,
@@ -158,16 +170,16 @@
ResultMessage.Prepared preparedSelect1 = QueryProcessor.prepare(
String.format("SELECT key, %s(d) FROM %s.%s", fSin, KEYSPACE, currentTable()),
- ClientState.forInternalCalls(), false);
+ ClientState.forInternalCalls());
ResultMessage.Prepared preparedSelect2 = QueryProcessor.prepare(
String.format("SELECT key FROM %s.%s", KEYSPACE, currentTable()),
- ClientState.forInternalCalls(), false);
+ ClientState.forInternalCalls());
ResultMessage.Prepared preparedInsert1 = QueryProcessor.prepare(
String.format("INSERT INTO %s.%s (key, d) VALUES (?, %s(?))", KEYSPACE, currentTable(), fSin),
- ClientState.forInternalCalls(), false);
+ ClientState.forInternalCalls());
ResultMessage.Prepared preparedInsert2 = QueryProcessor.prepare(
String.format("INSERT INTO %s.%s (key, d) VALUES (?, ?)", KEYSPACE, currentTable()),
- ClientState.forInternalCalls(), false);
+ ClientState.forInternalCalls());
Assert.assertNotNull(QueryProcessor.instance.getPrepared(preparedSelect1.statementId));
Assert.assertNotNull(QueryProcessor.instance.getPrepared(preparedSelect2.statementId));
@@ -192,10 +204,10 @@
preparedSelect1= QueryProcessor.prepare(
String.format("SELECT key, %s(d) FROM %s.%s", fSin, KEYSPACE, currentTable()),
- ClientState.forInternalCalls(), false);
+ ClientState.forInternalCalls());
preparedInsert1 = QueryProcessor.prepare(
String.format("INSERT INTO %s.%s (key, d) VALUES (?, %s(?))", KEYSPACE, currentTable(), fSin),
- ClientState.forInternalCalls(), false);
+ ClientState.forInternalCalls());
Assert.assertNotNull(QueryProcessor.instance.getPrepared(preparedSelect1.statementId));
Assert.assertNotNull(QueryProcessor.instance.getPrepared(preparedInsert1.statementId));
@@ -261,7 +273,7 @@
KEYSPACE,
currentTable(),
literalArgs),
- ClientState.forInternalCalls(), false);
+ ClientState.forInternalCalls());
Assert.assertNotNull(QueryProcessor.instance.getPrepared(prepared.statementId));
return prepared;
}
@@ -277,7 +289,7 @@
KEYSPACE,
currentTable(),
function),
- ClientState.forInternalCalls(), false);
+ ClientState.forInternalCalls());
Assert.assertNotNull(QueryProcessor.instance.getPrepared(prepared.statementId));
return prepared;
}
@@ -292,7 +304,7 @@
String.format("INSERT INTO %s.%s (key, val) VALUES (?, ?)",
KEYSPACE,
currentTable()),
- ClientState.forInternalCalls(), false);
+ ClientState.forInternalCalls());
Assert.assertNotNull(QueryProcessor.instance.getPrepared(control.statementId));
// a function that we'll drop and verify that statements which use it to
@@ -368,7 +380,7 @@
"CREATE OR REPLACE FUNCTION " + fSin + " ( input double ) " +
"CALLED ON NULL INPUT " +
"RETURNS text " +
- "LANGUAGE java AS 'return Double.valueOf(42d);'");
+ "LANGUAGE java AS 'return \"42d\";'");
// proper replacement
execute("CREATE OR REPLACE FUNCTION " + fSin + " ( input double ) " +
@@ -401,13 +413,13 @@
execute("DROP FUNCTION " + fSin2);
// Drop unexisting function
- assertInvalidMessage("Cannot drop non existing function", "DROP FUNCTION " + fSin);
+ assertInvalidMessage(String.format("Function '%s' doesn't exist", fSin), "DROP FUNCTION " + fSin);
// but don't complain with "IF EXISTS"
execute("DROP FUNCTION IF EXISTS " + fSin);
// can't drop native functions
- assertInvalidMessage("system keyspace is not user-modifiable", "DROP FUNCTION totimestamp");
- assertInvalidMessage("system keyspace is not user-modifiable", "DROP FUNCTION uuid");
+ assertInvalidMessage("System keyspace 'system' is not user-modifiable", "DROP FUNCTION totimestamp");
+ assertInvalidMessage("System keyspace 'system' is not user-modifiable", "DROP FUNCTION uuid");
// sin() no longer exists
assertInvalidMessage("Unknown function", "SELECT key, sin(d) FROM %s");
@@ -508,8 +520,8 @@
assertEmpty(execute("SELECT v FROM %s WHERE k = " + fOverload + "((varchar)?)", "foo"));
// no such functions exist...
- assertInvalidMessage("non existing function", "DROP FUNCTION " + fOverload + "(boolean)");
- assertInvalidMessage("non existing function", "DROP FUNCTION " + fOverload + "(bigint)");
+ assertInvalidMessage(String.format("Function '%s(boolean)' doesn't exist", fOverload), "DROP FUNCTION " + fOverload + "(boolean)");
+ assertInvalidMessage(String.format("Function '%s(bigint)' doesn't exist", fOverload), "DROP FUNCTION " + fOverload + "(bigint)");
// 'overloaded' has multiple overloads - so it has to fail (CASSANDRA-7812)
assertInvalidMessage("matches multiple function definitions", "DROP FUNCTION " + fOverload);
@@ -640,43 +652,43 @@
"AS 'return null;';");
- assertInvalidMessage("system keyspace is not user-modifiable",
+ assertInvalidMessage("System keyspace 'system' is not user-modifiable",
"CREATE OR REPLACE FUNCTION system.jnft(val double) " +
"RETURNS NULL ON NULL INPUT " +
"RETURNS double " +
"LANGUAGE JAVA\n" +
"AS 'return null;';");
- assertInvalidMessage("system keyspace is not user-modifiable",
+ assertInvalidMessage("System keyspace 'system' is not user-modifiable",
"CREATE OR REPLACE FUNCTION system.totimestamp(val timeuuid) " +
"RETURNS NULL ON NULL INPUT " +
"RETURNS timestamp " +
"LANGUAGE JAVA\n" +
"AS 'return null;';");
- assertInvalidMessage("system keyspace is not user-modifiable",
+ assertInvalidMessage("System keyspace 'system' is not user-modifiable",
"DROP FUNCTION system.now");
- // KS for executeInternal() is system
- assertInvalidMessage("system keyspace is not user-modifiable",
+ // KS for executeLocally() is system
+ assertInvalidMessage("System keyspace 'system' is not user-modifiable",
"CREATE OR REPLACE FUNCTION jnft(val double) " +
"RETURNS NULL ON NULL INPUT " +
"RETURNS double " +
"LANGUAGE JAVA\n" +
"AS 'return null;';");
- assertInvalidMessage("system keyspace is not user-modifiable",
+ assertInvalidMessage("System keyspace 'system' is not user-modifiable",
"CREATE OR REPLACE FUNCTION totimestamp(val timeuuid) " +
"RETURNS NULL ON NULL INPUT " +
"RETURNS timestamp " +
"LANGUAGE JAVA\n" +
"AS 'return null;';");
- assertInvalidMessage("system keyspace is not user-modifiable",
+ assertInvalidMessage("System keyspace 'system' is not user-modifiable",
"DROP FUNCTION now");
}
@Test
public void testFunctionNonExistingKeyspace() throws Throwable
{
- assertInvalidMessage("Keyspace this_ks_does_not_exist doesn't exist",
+ assertInvalidMessage("Keyspace 'this_ks_does_not_exist' doesn't exist",
"CREATE OR REPLACE FUNCTION this_ks_does_not_exist.jnft(val double) " +
"RETURNS NULL ON NULL INPUT " +
"RETURNS double " +
@@ -689,7 +701,7 @@
{
dropPerTestKeyspace();
- assertInvalidMessage("Keyspace " + KEYSPACE_PER_TEST + " doesn't exist",
+ assertInvalidMessage("Keyspace '" + KEYSPACE_PER_TEST + "' doesn't exist",
"CREATE OR REPLACE FUNCTION " + KEYSPACE_PER_TEST + ".jnft(val double) " +
"RETURNS NULL ON NULL INPUT " +
"RETURNS double " +
@@ -740,7 +752,7 @@
Assert.assertEquals(1, Schema.instance.getFunctions(fNameName).size());
ResultMessage.Prepared prepared = QueryProcessor.prepare(String.format("SELECT key, %s(udt) FROM %s.%s", fName, KEYSPACE, currentTable()),
- ClientState.forInternalCalls(), false);
+ ClientState.forInternalCalls());
Assert.assertNotNull(QueryProcessor.instance.getPrepared(prepared.statementId));
// UT still referenced by table
@@ -760,7 +772,7 @@
@Test
public void testDuplicateArgNames() throws Throwable
{
- assertInvalidMessage("duplicate argument names for given function",
+ assertInvalidMessage("Duplicate argument names for given function",
"CREATE OR REPLACE FUNCTION " + KEYSPACE + ".scrinv(val double, val text) " +
"RETURNS NULL ON NULL INPUT " +
"RETURNS text " +
@@ -822,7 +834,7 @@
"LANGUAGE JAVA\n" +
"AS 'throw new RuntimeException();';");
- KeyspaceMetadata ksm = Schema.instance.getKSMetaData(KEYSPACE_PER_TEST);
+ KeyspaceMetadata ksm = Schema.instance.getKeyspaceMetadata(KEYSPACE_PER_TEST);
UDFunction f = (UDFunction) ksm.functions.get(parseFunctionName(fName)).iterator().next();
UDFunction broken = UDFunction.createBrokenFunction(f.name(),
@@ -833,7 +845,7 @@
"java",
f.body(),
new InvalidRequestException("foo bar is broken"));
- Schema.instance.setKeyspaceMetadata(ksm.withSwapped(ksm.functions.without(f.name(), f.argTypes()).with(broken)));
+ Schema.instance.load(ksm.withSwapped(ksm.functions.without(f.name(), f.argTypes()).with(broken)));
assertInvalidThrowMessage("foo bar is broken", InvalidRequestException.class,
"SELECT key, " + fName + "(dval) FROM %s");
@@ -873,246 +885,110 @@
}
@Test
- public void testArgumentGenerics() throws Throwable
+ public void testEmptyString() throws Throwable
{
createTable("CREATE TABLE %s (key int primary key, sval text, aval ascii, bval blob, empty_int int)");
+ execute("INSERT INTO %s (key, sval, aval, bval, empty_int) VALUES (?, ?, ?, ?, blobAsInt(0x))", 1, "", "", ByteBuffer.allocate(0));
- String typeName = createType("CREATE TYPE %s (txt text, i int)");
+ String fNameSRC = createFunction(KEYSPACE_PER_TEST, "text",
+ "CREATE OR REPLACE FUNCTION %s(val text) " +
+ "CALLED ON NULL INPUT " +
+ "RETURNS text " +
+ "LANGUAGE JAVA\n" +
+ "AS 'return val;'");
- createFunction(KEYSPACE, "map<text,bigint>,list<text>",
- "CREATE FUNCTION IF NOT EXISTS %s(state map<text,bigint>, styles list<text>)\n" +
- " RETURNS NULL ON NULL INPUT\n" +
- " RETURNS map<text,bigint>\n" +
- " LANGUAGE java\n" +
- " AS $$\n" +
- " for (String style : styles) {\n" +
- " if (state.containsKey(style)) {\n" +
- " state.put(style, state.get(style) + 1L);\n" +
- " } else {\n" +
- " state.put(style, 1L);\n" +
- " }\n" +
- " }\n" +
- " return state;\n" +
- " $$");
+ String fNameSCC = createFunction(KEYSPACE_PER_TEST, "text",
+ "CREATE OR REPLACE FUNCTION %s(val text) " +
+ "CALLED ON NULL INPUT " +
+ "RETURNS text " +
+ "LANGUAGE JAVA\n" +
+ "AS 'return \"\";'");
- createFunction(KEYSPACE, "text",
- "CREATE OR REPLACE FUNCTION %s(" +
- " listText list<text>," +
- " setText set<text>," +
- " mapTextInt map<text, int>," +
- " mapListTextSetInt map<frozen<list<text>>, frozen<set<int>>>," +
- " mapTextTuple map<text, frozen<tuple<int, text>>>," +
- " mapTextType map<text, frozen<" + typeName + ">>" +
- ") " +
- "CALLED ON NULL INPUT " +
- "RETURNS map<frozen<list<text>>, frozen<set<int>>> " +
- "LANGUAGE JAVA\n" +
- "AS $$" +
- " for (String s : listtext) {};" +
- " for (String s : settext) {};" +
- " for (String s : maptextint.keySet()) {};" +
- " for (Integer s : maptextint.values()) {};" +
- " for (java.util.List<String> l : maplisttextsetint.keySet()) {};" +
- " for (java.util.Set<Integer> s : maplisttextsetint.values()) {};" +
- " for (com.datastax.driver.core.TupleValue t : maptexttuple.values()) {};" +
- " for (com.datastax.driver.core.UDTValue u : maptexttype.values()) {};" +
- " return maplisttextsetint;" +
- "$$");
- }
+ String fNameSRN = createFunction(KEYSPACE_PER_TEST, "text",
+ "CREATE OR REPLACE FUNCTION %s(val text) " +
+ "RETURNS NULL ON NULL INPUT " +
+ "RETURNS text " +
+ "LANGUAGE JAVA\n" +
+ "AS 'return val;'");
- @Test
- public void testArgAndReturnTypes() throws Throwable
- {
+ String fNameSCN = createFunction(KEYSPACE_PER_TEST, "text",
+ "CREATE OR REPLACE FUNCTION %s(val text) " +
+ "RETURNS NULL ON NULL INPUT " +
+ "RETURNS text " +
+ "LANGUAGE JAVA\n" +
+ "AS 'return \"\";'");
- String type = KEYSPACE + '.' + createType("CREATE TYPE %s (txt text, i int)");
+ String fNameBRC = createFunction(KEYSPACE_PER_TEST, "blob",
+ "CREATE OR REPLACE FUNCTION %s(val blob) " +
+ "CALLED ON NULL INPUT " +
+ "RETURNS blob " +
+ "LANGUAGE JAVA\n" +
+ "AS 'return val;'");
- createTable("CREATE TABLE %s (key int primary key, udt frozen<" + type + ">)");
- execute("INSERT INTO %s (key, udt) VALUES (1, {txt: 'foo', i: 42})");
+ String fNameBCC = createFunction(KEYSPACE_PER_TEST, "blob",
+ "CREATE OR REPLACE FUNCTION %s(val blob) " +
+ "CALLED ON NULL INPUT " +
+ "RETURNS blob " +
+ "LANGUAGE JAVA\n" +
+ "AS 'return ByteBuffer.allocate(0);'");
- // Java UDFs
+ String fNameBRN = createFunction(KEYSPACE_PER_TEST, "blob",
+ "CREATE OR REPLACE FUNCTION %s(val blob) " +
+ "RETURNS NULL ON NULL INPUT " +
+ "RETURNS blob " +
+ "LANGUAGE JAVA\n" +
+ "AS 'return val;'");
- String f = createFunction(KEYSPACE, "int",
- "CREATE OR REPLACE FUNCTION %s(val int) " +
- "RETURNS NULL ON NULL INPUT " +
- "RETURNS " + type + ' ' +
- "LANGUAGE JAVA\n" +
- "AS 'return udfContext.newReturnUDTValue();';");
+ String fNameBCN = createFunction(KEYSPACE_PER_TEST, "blob",
+ "CREATE OR REPLACE FUNCTION %s(val blob) " +
+ "RETURNS NULL ON NULL INPUT " +
+ "RETURNS blob " +
+ "LANGUAGE JAVA\n" +
+ "AS 'return ByteBuffer.allocate(0);'");
- assertRows(execute("SELECT " + f + "(key) FROM %s"),
- row(userType("txt", null, "i", null)));
+ String fNameIRC = createFunction(KEYSPACE_PER_TEST, "int",
+ "CREATE OR REPLACE FUNCTION %s(val int) " +
+ "CALLED ON NULL INPUT " +
+ "RETURNS int " +
+ "LANGUAGE JAVA\n" +
+ "AS 'return val;'");
- f = createFunction(KEYSPACE, "int",
- "CREATE OR REPLACE FUNCTION %s(val " + type + ") " +
- "RETURNS NULL ON NULL INPUT " +
- "RETURNS " + type + ' ' +
- "LANGUAGE JAVA\n" +
- "AS $$" +
- " com.datastax.driver.core.UDTValue udt = udfContext.newArgUDTValue(\"val\");" +
- " udt.setString(\"txt\", \"baz\");" +
- " udt.setInt(\"i\", 88);" +
- " return udt;" +
- "$$;");
+ String fNameICC = createFunction(KEYSPACE_PER_TEST, "int",
+ "CREATE OR REPLACE FUNCTION %s(val int) " +
+ "CALLED ON NULL INPUT " +
+ "RETURNS int " +
+ "LANGUAGE JAVA\n" +
+ "AS 'return 0;'");
- assertRows(execute("SELECT " + f + "(udt) FROM %s"),
- row(userType("txt", "baz", "i", 88)));
+ String fNameIRN = createFunction(KEYSPACE_PER_TEST, "int",
+ "CREATE OR REPLACE FUNCTION %s(val int) " +
+ "RETURNS NULL ON NULL INPUT " +
+ "RETURNS int " +
+ "LANGUAGE JAVA\n" +
+ "AS 'return val;'");
- f = createFunction(KEYSPACE, "int",
- "CREATE OR REPLACE FUNCTION %s(val " + type + ") " +
- "RETURNS NULL ON NULL INPUT " +
- "RETURNS tuple<text, int>" +
- "LANGUAGE JAVA\n" +
- "AS $$" +
- " com.datastax.driver.core.TupleValue tv = udfContext.newReturnTupleValue();" +
- " tv.setString(0, \"baz\");" +
- " tv.setInt(1, 88);" +
- " return tv;" +
- "$$;");
+ String fNameICN = createFunction(KEYSPACE_PER_TEST, "int",
+ "CREATE OR REPLACE FUNCTION %s(val int) " +
+ "RETURNS NULL ON NULL INPUT " +
+ "RETURNS int " +
+ "LANGUAGE JAVA\n" +
+ "AS 'return 0;'");
- assertRows(execute("SELECT " + f + "(udt) FROM %s"),
- row(tuple("baz", 88)));
-
- // JavaScript UDFs
-
- f = createFunction(KEYSPACE, "int",
- "CREATE OR REPLACE FUNCTION %s(val int) " +
- "RETURNS NULL ON NULL INPUT " +
- "RETURNS " + type + ' ' +
- "LANGUAGE JAVASCRIPT\n" +
- "AS $$" +
- " udt = udfContext.newReturnUDTValue();" +
- " udt;" +
- "$$;");
-
- assertRows(execute("SELECT " + f + "(key) FROM %s"),
- row(userType("txt", null, "i", null)));
-
- f = createFunction(KEYSPACE, "int",
- "CREATE OR REPLACE FUNCTION %s(val " + type + ") " +
- "RETURNS NULL ON NULL INPUT " +
- "RETURNS " + type + ' ' +
- "LANGUAGE JAVASCRIPT\n" +
- "AS $$" +
- " udt = udfContext.newArgUDTValue(0);" +
- " udt.setString(\"txt\", \"baz\");" +
- " udt.setInt(\"i\", 88);" +
- " udt;" +
- "$$;");
-
- assertRows(execute("SELECT " + f + "(udt) FROM %s"),
- row(userType("txt", "baz", "i", 88)));
-
- f = createFunction(KEYSPACE, "int",
- "CREATE OR REPLACE FUNCTION %s(val " + type + ") " +
- "RETURNS NULL ON NULL INPUT " +
- "RETURNS tuple<text, int>" +
- "LANGUAGE JAVASCRIPT\n" +
- "AS $$" +
- " tv = udfContext.newReturnTupleValue();" +
- " tv.setString(0, \"baz\");" +
- " tv.setInt(1, 88);" +
- " tv;" +
- "$$;");
-
- assertRows(execute("SELECT " + f + "(udt) FROM %s"),
- row(tuple("baz", 88)));
-
- createFunction(KEYSPACE, "map",
- "CREATE FUNCTION %s(my_map map<text, text>)\n" +
- " CALLED ON NULL INPUT\n" +
- " RETURNS text\n" +
- " LANGUAGE java\n" +
- " AS $$\n" +
- " String buffer = \"\";\n" +
- " for(java.util.Map.Entry<String, String> entry: my_map.entrySet()) {\n" +
- " buffer = buffer + entry.getKey() + \": \" + entry.getValue() + \", \";\n" +
- " }\n" +
- " return buffer;\n" +
- " $$;\n");
- }
-
- @Test
- public void testImportJavaUtil() throws Throwable
- {
- createFunction(KEYSPACE, "list<text>",
- "CREATE OR REPLACE FUNCTION %s(listText list<text>) " +
- "CALLED ON NULL INPUT " +
- "RETURNS set<text> " +
- "LANGUAGE JAVA\n" +
- "AS $$\n" +
- " Set<String> set = new HashSet<String>(); " +
- " for (String s : listtext) {" +
- " set.add(s);" +
- " }" +
- " return set;" +
- "$$");
-
- }
-
- @Test
- public void testAnyUserTupleType() throws Throwable
- {
- createTable("CREATE TABLE %s (key int primary key, sval text)");
- execute("INSERT INTO %s (key, sval) VALUES (1, 'foo')");
-
- String udt = createType("CREATE TYPE %s (a int, b text, c bigint)");
-
- String fUdt = createFunction(KEYSPACE, "text",
- "CREATE OR REPLACE FUNCTION %s(arg text) " +
- "CALLED ON NULL INPUT " +
- "RETURNS " + udt + " " +
- "LANGUAGE JAVA\n" +
- "AS $$\n" +
- " UDTValue udt = udfContext.newUDTValue(\"" + udt + "\");" +
- " udt.setInt(\"a\", 42);" +
- " udt.setString(\"b\", \"42\");" +
- " udt.setLong(\"c\", 4242);" +
- " return udt;" +
- "$$");
-
- assertRows(execute("SELECT " + fUdt + "(sval) FROM %s"),
- row(userType("a", 42, "b", "42", "c", 4242L)));
-
- String fTup = createFunction(KEYSPACE, "text",
- "CREATE OR REPLACE FUNCTION %s(arg text) " +
- "CALLED ON NULL INPUT " +
- "RETURNS tuple<int, " + udt + "> " +
- "LANGUAGE JAVA\n" +
- "AS $$\n" +
- " UDTValue udt = udfContext.newUDTValue(\"" + udt + "\");" +
- " udt.setInt(\"a\", 42);" +
- " udt.setString(\"b\", \"42\");" +
- " udt.setLong(\"c\", 4242);" +
- " TupleValue tup = udfContext.newTupleValue(\"tuple<int," + udt + ">\");" +
- " tup.setInt(0, 88);" +
- " tup.setUDTValue(1, udt);" +
- " return tup;" +
- "$$");
-
- assertRows(execute("SELECT " + fTup + "(sval) FROM %s"),
- row(tuple(88, userType("a", 42, "b", "42", "c", 4242L))));
- }
-
- @Test(expected = SyntaxException.class)
- public void testEmptyFunctionName() throws Throwable
- {
- execute("CREATE FUNCTION IF NOT EXISTS " + KEYSPACE + ".\"\" (arg int)\n" +
- " RETURNS NULL ON NULL INPUT\n" +
- " RETURNS int\n" +
- " LANGUAGE java\n" +
- " AS $$\n" +
- " return a;\n" +
- " $$");
- }
-
- @Test(expected = SyntaxException.class)
- public void testEmptyArgName() throws Throwable
- {
- execute("CREATE FUNCTION IF NOT EXISTS " + KEYSPACE + ".myfn (\"\" int)\n" +
- " RETURNS NULL ON NULL INPUT\n" +
- " RETURNS int\n" +
- " LANGUAGE java\n" +
- " AS $$\n" +
- " return a;\n" +
- " $$");
+ assertRows(execute("SELECT " + fNameSRC + "(sval) FROM %s"), row(""));
+ assertRows(execute("SELECT " + fNameSRN + "(sval) FROM %s"), row(""));
+ assertRows(execute("SELECT " + fNameSCC + "(sval) FROM %s"), row(""));
+ assertRows(execute("SELECT " + fNameSCN + "(sval) FROM %s"), row(""));
+ assertRows(execute("SELECT " + fNameSRC + "(aval) FROM %s"), row(""));
+ assertRows(execute("SELECT " + fNameSRN + "(aval) FROM %s"), row(""));
+ assertRows(execute("SELECT " + fNameSCC + "(aval) FROM %s"), row(""));
+ assertRows(execute("SELECT " + fNameSCN + "(aval) FROM %s"), row(""));
+ assertRows(execute("SELECT " + fNameBRC + "(bval) FROM %s"), row(ByteBufferUtil.EMPTY_BYTE_BUFFER));
+ assertRows(execute("SELECT " + fNameBRN + "(bval) FROM %s"), row(ByteBufferUtil.EMPTY_BYTE_BUFFER));
+ assertRows(execute("SELECT " + fNameBCC + "(bval) FROM %s"), row(ByteBufferUtil.EMPTY_BYTE_BUFFER));
+ assertRows(execute("SELECT " + fNameBCN + "(bval) FROM %s"), row(ByteBufferUtil.EMPTY_BYTE_BUFFER));
+ assertRows(execute("SELECT " + fNameIRC + "(empty_int) FROM %s"), row(new Object[]{ null }));
+ assertRows(execute("SELECT " + fNameIRN + "(empty_int) FROM %s"), row(new Object[]{ null }));
+ assertRows(execute("SELECT " + fNameICC + "(empty_int) FROM %s"), row(0));
+ assertRows(execute("SELECT " + fNameICN + "(empty_int) FROM %s"), row(new Object[]{ null }));
}
}
diff --git a/test/unit/org/apache/cassandra/cql3/validation/entities/UFTypesTest.java b/test/unit/org/apache/cassandra/cql3/validation/entities/UFTypesTest.java
index 3f1bcb1..9680bd5 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/entities/UFTypesTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/entities/UFTypesTest.java
@@ -293,14 +293,14 @@
execute("INSERT INTO %s (a, b) VALUES (?, ?)", 2, set(4, 5, 6));
execute("INSERT INTO %s (a, b) VALUES (?, ?)", 3, set(7, 8, 9));
- assertInvalidMessage("The function arguments should not be frozen",
+ assertInvalidMessage("Argument 'frozen<set<int>>' cannot be frozen; remove frozen<> modifier from 'frozen<set<int>>'",
"CREATE OR REPLACE FUNCTION " + KEYSPACE + ".frozenSetArg(values frozen<set<int>>) " +
"CALLED ON NULL INPUT " +
"RETURNS int " +
"LANGUAGE java\n" +
"AS 'int sum = 0; for (Object value : values) {sum += value;} return sum;';");
- assertInvalidMessage("The function return type should not be frozen",
+ assertInvalidMessage("Return type 'frozen<set<int>>' cannot be frozen; remove frozen<> modifier from 'frozen<set<int>>'",
"CREATE OR REPLACE FUNCTION " + KEYSPACE + ".frozenReturnType(values set<int>) " +
"CALLED ON NULL INPUT " +
"RETURNS frozen<set<int>> " +
@@ -331,7 +331,7 @@
assertRows(execute("SELECT a FROM %s WHERE b = " + functionName + "(?)", set(1, 2, 3)),
row(1));
- assertInvalidMessage("The function arguments should not be frozen",
+ assertInvalidMessage("Argument 'frozen<set<int>>' cannot be frozen; remove frozen<> modifier from 'frozen<set<int>>'",
"DROP FUNCTION " + functionName + "(frozen<set<int>>);");
}
@@ -346,14 +346,14 @@
execute("INSERT INTO %s (a, b) VALUES (?, ?)", 2, list(4, 5, 6));
execute("INSERT INTO %s (a, b) VALUES (?, ?)", 3, list(7, 8, 9));
- assertInvalidMessage("The function arguments should not be frozen",
+ assertInvalidMessage("Argument 'frozen<list<int>>' cannot be frozen; remove frozen<> modifier from 'frozen<list<int>>'",
"CREATE OR REPLACE FUNCTION " + KEYSPACE + ".withFrozenArg(values frozen<list<int>>) " +
"CALLED ON NULL INPUT " +
"RETURNS int " +
"LANGUAGE java\n" +
"AS 'int sum = 0; for (Object value : values) {sum += value;} return sum;';");
- assertInvalidMessage("The function return type should not be frozen",
+ assertInvalidMessage("Return type 'frozen<list<int>>' cannot be frozen; remove frozen<> modifier from 'frozen<list<int>>'",
"CREATE OR REPLACE FUNCTION " + KEYSPACE + ".frozenReturnType(values list<int>) " +
"CALLED ON NULL INPUT " +
"RETURNS frozen<list<int>> " +
@@ -384,7 +384,7 @@
assertRows(execute("SELECT a FROM %s WHERE b = " + functionName + "(?)", set(1, 2, 3)),
row(1));
- assertInvalidMessage("The function arguments should not be frozen",
+ assertInvalidMessage("frozen<list<int>>' cannot be frozen; remove frozen<> modifier from 'frozen<list<int>>'",
"DROP FUNCTION " + functionName + "(frozen<list<int>>);");
}
@@ -399,14 +399,14 @@
execute("INSERT INTO %s (a, b) VALUES (?, ?)", 2, map(4, 4, 5, 5, 6, 6));
execute("INSERT INTO %s (a, b) VALUES (?, ?)", 3, map(7, 7, 8, 8, 9, 9));
- assertInvalidMessage("The function arguments should not be frozen",
+ assertInvalidMessage("Argument 'frozen<map<int, int>>' cannot be frozen; remove frozen<> modifier from 'frozen<map<int, int>>'",
"CREATE OR REPLACE FUNCTION " + KEYSPACE + ".withFrozenArg(values frozen<map<int, int>>) " +
"CALLED ON NULL INPUT " +
"RETURNS int " +
"LANGUAGE java\n" +
"AS 'int sum = 0; for (Object value : values.values()) {sum += value;} return sum;';");
- assertInvalidMessage("The function return type should not be frozen",
+ assertInvalidMessage("Return type 'frozen<map<int, int>>' cannot be frozen; remove frozen<> modifier from 'frozen<map<int, int>>'",
"CREATE OR REPLACE FUNCTION " + KEYSPACE + ".frozenReturnType(values map<int, int>) " +
"CALLED ON NULL INPUT " +
"RETURNS frozen<map<int, int>> " +
@@ -437,7 +437,7 @@
assertRows(execute("SELECT a FROM %s WHERE b = " + functionName + "(?)", map(1, 1, 2, 2, 3, 3)),
row(1));
- assertInvalidMessage("The function arguments should not be frozen",
+ assertInvalidMessage("frozen<map<int, int>>' cannot be frozen; remove frozen<> modifier from 'frozen<map<int, int>>",
"DROP FUNCTION " + functionName + "(frozen<map<int, int>>);");
}
@@ -452,14 +452,14 @@
execute("INSERT INTO %s (a, b) VALUES (?, ?)", 2, tuple(4, 5));
execute("INSERT INTO %s (a, b) VALUES (?, ?)", 3, tuple(7, 8));
- assertInvalidMessage("The function arguments should not be frozen",
+ assertInvalidMessage("Argument 'tuple<int, int>' cannot be frozen; remove frozen<> modifier from 'tuple<int, int>'",
"CREATE OR REPLACE FUNCTION " + KEYSPACE + ".withFrozenArg(values frozen<tuple<int, int>>) " +
"CALLED ON NULL INPUT " +
"RETURNS text " +
"LANGUAGE java\n" +
"AS 'return values.toString();';");
- assertInvalidMessage("The function return type should not be frozen",
+ assertInvalidMessage("Return type 'tuple<int, int>' cannot be frozen; remove frozen<> modifier from 'tuple<int, int>'",
"CREATE OR REPLACE FUNCTION " + KEYSPACE + ".frozenReturnType(values tuple<int, int>) " +
"CALLED ON NULL INPUT " +
"RETURNS frozen<tuple<int, int>> " +
@@ -490,7 +490,7 @@
assertRows(execute("SELECT a FROM %s WHERE b = " + functionName + "(?)", tuple(1, 2)),
row(1));
- assertInvalidMessage("The function arguments should not be frozen",
+ assertInvalidMessage("Argument 'tuple<int, int>' cannot be frozen; remove frozen<> modifier from 'tuple<int, int>'",
"DROP FUNCTION " + functionName + "(frozen<tuple<int, int>>);");
}
@@ -506,14 +506,14 @@
execute("INSERT INTO %s (a, b) VALUES (?, {f : ?})", 2, 4);
execute("INSERT INTO %s (a, b) VALUES (?, {f : ?})", 3, 7);
- assertInvalidMessage("The function arguments should not be frozen",
+ assertInvalidMessage("cannot be frozen; remove frozen<> modifier",
"CREATE OR REPLACE FUNCTION " + KEYSPACE + ".withFrozenArg(values frozen<" + myType + ">) " +
"CALLED ON NULL INPUT " +
"RETURNS text " +
"LANGUAGE java\n" +
"AS 'return values.toString();';");
- assertInvalidMessage("The function return type should not be frozen",
+ assertInvalidMessage("cannot be frozen; remove frozen<> modifier",
"CREATE OR REPLACE FUNCTION " + KEYSPACE + ".frozenReturnType(values " + myType + ") " +
"CALLED ON NULL INPUT " +
"RETURNS frozen<" + myType + "> " +
@@ -544,7 +544,7 @@
assertRows(execute("SELECT a FROM %s WHERE b = " + functionName + "({f: ?})", 1),
row(1));
- assertInvalidMessage("The function arguments should not be frozen",
+ assertInvalidMessage(String.format("frozen<%s>' cannot be frozen; remove frozen<> modifier from 'frozen<%s>'", myType, myType),
"DROP FUNCTION " + functionName + "(frozen<" + myType + ">);");
}
}
diff --git a/test/unit/org/apache/cassandra/cql3/validation/entities/UFVerifierTest.java b/test/unit/org/apache/cassandra/cql3/validation/entities/UFVerifierTest.java
index 9a8e682..15a5070 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/entities/UFVerifierTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/entities/UFVerifierTest.java
@@ -31,26 +31,7 @@
import org.apache.cassandra.cql3.CQLTester;
import org.apache.cassandra.cql3.functions.UDFByteCodeVerifier;
-import org.apache.cassandra.cql3.validation.entities.udfverify.CallClone;
-import org.apache.cassandra.cql3.validation.entities.udfverify.CallComDatastax;
-import org.apache.cassandra.cql3.validation.entities.udfverify.CallFinalize;
-import org.apache.cassandra.cql3.validation.entities.udfverify.CallOrgApache;
-import org.apache.cassandra.cql3.validation.entities.udfverify.ClassWithField;
-import org.apache.cassandra.cql3.validation.entities.udfverify.ClassWithInitializer;
-import org.apache.cassandra.cql3.validation.entities.udfverify.ClassWithInitializer2;
-import org.apache.cassandra.cql3.validation.entities.udfverify.ClassWithInitializer3;
-import org.apache.cassandra.cql3.validation.entities.udfverify.ClassWithInnerClass;
-import org.apache.cassandra.cql3.validation.entities.udfverify.ClassWithInnerClass2;
-import org.apache.cassandra.cql3.validation.entities.udfverify.ClassWithStaticInitializer;
-import org.apache.cassandra.cql3.validation.entities.udfverify.ClassWithStaticInnerClass;
-import org.apache.cassandra.cql3.validation.entities.udfverify.GoodClass;
-import org.apache.cassandra.cql3.validation.entities.udfverify.UseOfSynchronized;
-import org.apache.cassandra.cql3.validation.entities.udfverify.UseOfSynchronizedWithNotify;
-import org.apache.cassandra.cql3.validation.entities.udfverify.UseOfSynchronizedWithNotifyAll;
-import org.apache.cassandra.cql3.validation.entities.udfverify.UseOfSynchronizedWithWait;
-import org.apache.cassandra.cql3.validation.entities.udfverify.UseOfSynchronizedWithWaitL;
-import org.apache.cassandra.cql3.validation.entities.udfverify.UseOfSynchronizedWithWaitLI;
-import org.apache.cassandra.cql3.validation.entities.udfverify.UsingMapEntry;
+import org.apache.cassandra.cql3.validation.entities.udfverify.*;
import static org.junit.Assert.assertEquals;
@@ -159,13 +140,6 @@
}
@Test
- public void testCallComDatastax()
- {
- assertEquals(new HashSet<>(Collections.singletonList("call to com.datastax.driver.core.DataType.cint()")),
- verify("com/", CallComDatastax.class));
- }
-
- @Test
public void testCallOrgApache()
{
assertEquals(new HashSet<>(Collections.singletonList("call to org.apache.cassandra.config.DatabaseDescriptor.getClusterName()")),
diff --git a/test/unit/org/apache/cassandra/cql3/validation/entities/UserTypesTest.java b/test/unit/org/apache/cassandra/cql3/validation/entities/UserTypesTest.java
index 646484c..e39dd35 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/entities/UserTypesTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/entities/UserTypesTest.java
@@ -24,7 +24,6 @@
import org.apache.cassandra.cql3.CQLTester;
import org.apache.cassandra.dht.ByteOrderedPartitioner;
-import org.apache.cassandra.exceptions.*;
import org.apache.cassandra.service.StorageService;
public class UserTypesTest extends CQLTester
@@ -110,9 +109,9 @@
String myType = KEYSPACE + '.' + typename;
// non-frozen UDTs in a table PK
- assertInvalidMessage("Invalid non-frozen user-defined type for PRIMARY KEY component k",
+ assertInvalidMessage("Invalid non-frozen user-defined type \"" + myType + "\" for PRIMARY KEY column 'k'",
"CREATE TABLE " + KEYSPACE + ".wrong (k " + myType + " PRIMARY KEY , v int)");
- assertInvalidMessage("Invalid non-frozen user-defined type for PRIMARY KEY component k2",
+ assertInvalidMessage("Invalid non-frozen user-defined type \"" + myType + "\" for PRIMARY KEY column 'k2'",
"CREATE TABLE " + KEYSPACE + ".wrong (k1 int, k2 " + myType + ", v int, PRIMARY KEY (k1, k2))");
// non-frozen UDTs in a collection
@@ -621,7 +620,7 @@
private void assertInvalidAlterDropStatements(String t) throws Throwable
{
assertInvalidMessage("Cannot alter user type " + typeWithKs(t), "ALTER TYPE " + typeWithKs(t) + " RENAME foo TO bar;");
- assertInvalidMessage("Cannot drop user type " + typeWithKs(t), "DROP TYPE " + typeWithKs(t) + ';');
+ assertInvalidMessage("Cannot drop user type '" + typeWithKs(t), "DROP TYPE " + typeWithKs(t) + ';');
}
@Test
@@ -879,25 +878,6 @@
row(1, 1,set(userType("a", 1), userType("a", 1, "b", 1), userType("a", 1, "b", 2), userType("a", 2), userType("a", 2, "b", 1)), 2));
}
- @Test(expected = SyntaxException.class)
- public void emptyTypeNameTest() throws Throwable
- {
- execute("CREATE TYPE \"\" (a int, b int)");
- }
-
- @Test(expected = SyntaxException.class)
- public void emptyFieldNameTest() throws Throwable
- {
- execute("CREATE TYPE mytype (\"\" int, b int)");
- }
-
- @Test(expected = SyntaxException.class)
- public void renameColumnToEmpty() throws Throwable
- {
- String typeName = createType("CREATE TYPE %s (a int, b int)");
- execute(String.format("ALTER TYPE %s.%s RENAME b TO \"\"", keyspace(), typeName));
- }
-
private String typeWithKs(String type1)
{
return keyspace() + '.' + type1;
diff --git a/test/unit/org/apache/cassandra/cql3/validation/entities/VirtualTableTest.java b/test/unit/org/apache/cassandra/cql3/validation/entities/VirtualTableTest.java
new file mode 100644
index 0000000..a503a60
--- /dev/null
+++ b/test/unit/org/apache/cassandra/cql3/validation/entities/VirtualTableTest.java
@@ -0,0 +1,372 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.validation.entities;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+
+import com.google.common.collect.ImmutableList;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.db.Mutation;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.marshal.LongType;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.db.partitions.Partition;
+import org.apache.cassandra.db.partitions.PartitionUpdate;
+import org.apache.cassandra.db.virtual.AbstractVirtualTable;
+import org.apache.cassandra.db.virtual.SimpleDataSet;
+import org.apache.cassandra.db.virtual.VirtualKeyspace;
+import org.apache.cassandra.db.virtual.VirtualKeyspaceRegistry;
+import org.apache.cassandra.db.virtual.VirtualTable;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.service.StorageServiceMBean;
+import org.apache.cassandra.triggers.ITrigger;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+public class VirtualTableTest extends CQLTester
+{
+ private static final String KS_NAME = "test_virtual_ks";
+ private static final String VT1_NAME = "vt1";
+ private static final String VT2_NAME = "vt2";
+
+ private static class WritableVirtualTable extends AbstractVirtualTable
+ {
+ private final ColumnMetadata valueColumn;
+ private final Map<String, Integer> backingMap = new HashMap<>();
+
+ WritableVirtualTable(String keyspaceName, String tableName)
+ {
+ super(TableMetadata.builder(keyspaceName, tableName)
+ .kind(TableMetadata.Kind.VIRTUAL)
+ .addPartitionKeyColumn("key", UTF8Type.instance)
+ .addRegularColumn("value", Int32Type.instance)
+ .build());
+ valueColumn = metadata().regularColumns().getSimple(0);
+ }
+
+ @Override
+ public DataSet data()
+ {
+ SimpleDataSet data = new SimpleDataSet(metadata());
+ backingMap.forEach((key, value) -> data.row(key).column("value", value));
+ return data;
+ }
+
+ @Override
+ public void apply(PartitionUpdate update)
+ {
+ String key = (String) metadata().partitionKeyType.compose(update.partitionKey().getKey());
+ update.forEach(row ->
+ {
+ Integer value = Int32Type.instance.compose(row.getCell(valueColumn).value());
+ backingMap.put(key, value);
+ });
+ }
+ }
+
+ @BeforeClass
+ public static void setUpClass()
+ {
+ TableMetadata vt1Metadata =
+ TableMetadata.builder(KS_NAME, VT1_NAME)
+ .kind(TableMetadata.Kind.VIRTUAL)
+ .addPartitionKeyColumn("pk", UTF8Type.instance)
+ .addClusteringColumn("c", UTF8Type.instance)
+ .addRegularColumn("v1", Int32Type.instance)
+ .addRegularColumn("v2", LongType.instance)
+ .build();
+
+ SimpleDataSet vt1data = new SimpleDataSet(vt1Metadata);
+
+ vt1data.row("pk1", "c1").column("v1", 11).column("v2", 11L)
+ .row("pk2", "c1").column("v1", 21).column("v2", 21L)
+ .row("pk1", "c2").column("v1", 12).column("v2", 12L)
+ .row("pk2", "c2").column("v1", 22).column("v2", 22L)
+ .row("pk1", "c3").column("v1", 13).column("v2", 13L)
+ .row("pk2", "c3").column("v1", 23).column("v2", 23L);
+
+ VirtualTable vt1 = new AbstractVirtualTable(vt1Metadata)
+ {
+ public DataSet data()
+ {
+ return vt1data;
+ }
+ };
+ VirtualTable vt2 = new WritableVirtualTable(KS_NAME, VT2_NAME);
+
+ VirtualKeyspaceRegistry.instance.register(new VirtualKeyspace(KS_NAME, ImmutableList.of(vt1, vt2)));
+
+ CQLTester.setUpClass();
+ }
+
+ @Test
+ public void testQueries() throws Throwable
+ {
+ assertRowsNet(executeNet("SELECT * FROM test_virtual_ks.vt1 WHERE pk = 'UNKNOWN'"));
+
+ assertRowsNet(executeNet("SELECT * FROM test_virtual_ks.vt1 WHERE pk = 'pk1' AND c = 'UNKNOWN'"));
+
+ // Test DISTINCT query
+ assertRowsNet(executeNet("SELECT DISTINCT pk FROM test_virtual_ks.vt1"),
+ row("pk1"),
+ row("pk2"));
+
+ assertRowsNet(executeNet("SELECT DISTINCT pk FROM test_virtual_ks.vt1 WHERE token(pk) > token('pk1')"),
+ row("pk2"));
+
+ // Test single partition queries
+ assertRowsNet(executeNet("SELECT v1, v2 FROM test_virtual_ks.vt1 WHERE pk = 'pk1' AND c = 'c1'"),
+ row(11, 11L));
+
+ assertRowsNet(executeNet("SELECT c, v1, v2 FROM test_virtual_ks.vt1 WHERE pk = 'pk1' AND c IN ('c1', 'c2')"),
+ row("c1", 11, 11L),
+ row("c2", 12, 12L));
+
+ assertRowsNet(executeNet("SELECT c, v1, v2 FROM test_virtual_ks.vt1 WHERE pk = 'pk1' AND c IN ('c2', 'c1') ORDER BY c DESC"),
+ row("c2", 12, 12L),
+ row("c1", 11, 11L));
+
+ // Test multi-partition queries
+ assertRows(execute("SELECT * FROM test_virtual_ks.vt1 WHERE pk IN ('pk2', 'pk1') AND c IN ('c2', 'c1')"),
+ row("pk1", "c1", 11, 11L),
+ row("pk1", "c2", 12, 12L),
+ row("pk2", "c1", 21, 21L),
+ row("pk2", "c2", 22, 22L));
+
+ assertRows(execute("SELECT pk, c, v1 FROM test_virtual_ks.vt1 WHERE pk IN ('pk2', 'pk1') AND c IN ('c2', 'c1') ORDER BY c DESC"),
+ row("pk1", "c2", 12),
+ row("pk2", "c2", 22),
+ row("pk1", "c1", 11),
+ row("pk2", "c1", 21));
+
+ assertRows(execute("SELECT pk, c, v1 FROM test_virtual_ks.vt1 WHERE pk IN ('pk2', 'pk1') AND c IN ('c2', 'c1') ORDER BY c DESC LIMIT 1"),
+ row("pk1", "c2", 12));
+
+ assertRows(execute("SELECT c, v1, v2 FROM test_virtual_ks.vt1 WHERE pk IN ('pk2', 'pk1') AND c IN ('c2', 'c1' , 'c3') ORDER BY c DESC PER PARTITION LIMIT 1"),
+ row("c3", 13, 13L),
+ row("c3", 23, 23L));
+
+ assertRows(execute("SELECT count(*) FROM test_virtual_ks.vt1 WHERE pk IN ('pk2', 'pk1') AND c IN ('c2', 'c1')"),
+ row(4L));
+
+ for (int pageSize = 1; pageSize < 5; pageSize++)
+ {
+ assertRowsNet(executeNetWithPaging("SELECT pk, c, v1, v2 FROM test_virtual_ks.vt1 WHERE pk IN ('pk2', 'pk1') AND c IN ('c2', 'c1')", pageSize),
+ row("pk1", "c1", 11, 11L),
+ row("pk1", "c2", 12, 12L),
+ row("pk2", "c1", 21, 21L),
+ row("pk2", "c2", 22, 22L));
+
+ assertRowsNet(executeNetWithPaging("SELECT * FROM test_virtual_ks.vt1 WHERE pk IN ('pk2', 'pk1') AND c IN ('c2', 'c1') LIMIT 2", pageSize),
+ row("pk1", "c1", 11, 11L),
+ row("pk1", "c2", 12, 12L));
+
+ assertRowsNet(executeNetWithPaging("SELECT count(*) FROM test_virtual_ks.vt1 WHERE pk IN ('pk2', 'pk1') AND c IN ('c2', 'c1')", pageSize),
+ row(4L));
+ }
+
+ // Test range queries
+ for (int pageSize = 1; pageSize < 4; pageSize++)
+ {
+ assertRowsNet(executeNetWithPaging("SELECT * FROM test_virtual_ks.vt1 WHERE token(pk) < token('pk2') AND c IN ('c2', 'c1') ALLOW FILTERING", pageSize),
+ row("pk1", "c1", 11, 11L),
+ row("pk1", "c2", 12, 12L));
+
+ assertRowsNet(executeNetWithPaging("SELECT * FROM test_virtual_ks.vt1 WHERE token(pk) < token('pk2') AND c IN ('c2', 'c1') LIMIT 1 ALLOW FILTERING", pageSize),
+ row("pk1", "c1", 11, 11L));
+
+ assertRowsNet(executeNetWithPaging("SELECT * FROM test_virtual_ks.vt1 WHERE token(pk) <= token('pk2') AND c > 'c1' PER PARTITION LIMIT 1 ALLOW FILTERING", pageSize),
+ row("pk1", "c2", 12, 12L),
+ row("pk2", "c2", 22, 22L));
+
+ assertRowsNet(executeNetWithPaging("SELECT count(*) FROM test_virtual_ks.vt1 WHERE token(pk) = token('pk2') AND c < 'c3' ALLOW FILTERING", pageSize),
+ row(2L));
+ }
+ }
+
+ @Test
+ public void testModifications() throws Throwable
+ {
+ // check for clean state
+ assertRows(execute("SELECT * FROM test_virtual_ks.vt2"));
+
+ // fill the table, test UNLOGGED batch
+ execute("BEGIN UNLOGGED BATCH " +
+ "UPDATE test_virtual_ks.vt2 SET value = 1 WHERE key ='pk1';" +
+ "UPDATE test_virtual_ks.vt2 SET value = 2 WHERE key ='pk2';" +
+ "UPDATE test_virtual_ks.vt2 SET value = 3 WHERE key ='pk3';" +
+ "APPLY BATCH");
+ assertRows(execute("SELECT * FROM test_virtual_ks.vt2"),
+ row("pk1", 1),
+ row("pk2", 2),
+ row("pk3", 3));
+
+ // test that LOGGED batches don't allow virtual table updates
+ assertInvalidMessage("Cannot include a virtual table statement in a logged batch",
+ "BEGIN BATCH " +
+ "UPDATE test_virtual_ks.vt2 SET value = 1 WHERE key ='pk1';" +
+ "UPDATE test_virtual_ks.vt2 SET value = 2 WHERE key ='pk2';" +
+ "UPDATE test_virtual_ks.vt2 SET value = 3 WHERE key ='pk3';" +
+ "APPLY BATCH");
+
+ // test that UNLOGGED batch doesn't allow mixing updates for regular and virtual tables
+ createTable("CREATE TABLE %s (key text PRIMARY KEY, value int)");
+ assertInvalidMessage("Mutations for virtual and regular tables cannot exist in the same batch",
+ "BEGIN UNLOGGED BATCH " +
+ "UPDATE test_virtual_ks.vt2 SET value = 1 WHERE key ='pk1'" +
+ "UPDATE %s SET value = 2 WHERE key ='pk2'" +
+ "UPDATE test_virtual_ks.vt2 SET value = 3 WHERE key ='pk3'" +
+ "APPLY BATCH");
+
+ // update a single value with UPDATE
+ execute("UPDATE test_virtual_ks.vt2 SET value = 11 WHERE key ='pk1'");
+ assertRows(execute("SELECT * FROM test_virtual_ks.vt2 WHERE key = 'pk1'"),
+ row("pk1", 11));
+
+ // update a single value with INSERT
+ executeNet("INSERT INTO test_virtual_ks.vt2 (key, value) VALUES ('pk2', 22)");
+ assertRows(execute("SELECT * FROM test_virtual_ks.vt2 WHERE key = 'pk2'"),
+ row("pk2", 22));
+
+ // test that deletions are (currently) rejected
+ assertInvalidMessage("Virtual tables don't support DELETE statements",
+ "DELETE FROM test_virtual_ks.vt2 WHERE key ='pk1'");
+
+ // test that TTL is (currently) rejected with INSERT and UPDATE
+ assertInvalidMessage("Expiring columns are not supported by virtual tables",
+ "INSERT INTO test_virtual_ks.vt2 (key, value) VALUES ('pk1', 11) USING TTL 86400");
+ assertInvalidMessage("Expiring columns are not supported by virtual tables",
+ "UPDATE test_virtual_ks.vt2 USING TTL 86400 SET value = 11 WHERE key ='pk1'");
+
+ // test that LWT is (currently) rejected with virtual tables in batches
+ assertInvalidMessage("Conditional BATCH statements cannot include mutations for virtual tables",
+ "BEGIN UNLOGGED BATCH " +
+ "UPDATE test_virtual_ks.vt2 SET value = 3 WHERE key ='pk3' IF value = 2;" +
+ "APPLY BATCH");
+
+ // test that LWT is (currently) rejected with virtual tables in UPDATEs
+ assertInvalidMessage("Conditional updates are not supported by virtual tables",
+ "UPDATE test_virtual_ks.vt2 SET value = 3 WHERE key ='pk3' IF value = 2");
+
+ // test that LWT is (currently) rejected with virtual tables in INSERTs
+ assertInvalidMessage("Conditional updates are not supported by virtual tables",
+ "INSERT INTO test_virtual_ks.vt2 (key, value) VALUES ('pk2', 22) IF NOT EXISTS");
+ }
+
+ @Test
+ public void testInvalidDDLOperations() throws Throwable
+ {
+ assertInvalidMessage("Virtual keyspace 'test_virtual_ks' is not user-modifiable",
+ "DROP KEYSPACE test_virtual_ks");
+
+ assertInvalidMessage("Virtual keyspace 'test_virtual_ks' is not user-modifiable",
+ "ALTER KEYSPACE test_virtual_ks WITH durable_writes = false");
+
+ assertInvalidMessage("Virtual keyspace 'test_virtual_ks' is not user-modifiable",
+ "CREATE TABLE test_virtual_ks.test (id int PRIMARY KEY)");
+
+ assertInvalidMessage("Virtual keyspace 'test_virtual_ks' is not user-modifiable",
+ "CREATE TYPE test_virtual_ks.type (id int)");
+
+ assertInvalidMessage("Virtual keyspace 'test_virtual_ks' is not user-modifiable",
+ "DROP TABLE test_virtual_ks.vt1");
+
+ assertInvalidMessage("Virtual keyspace 'test_virtual_ks' is not user-modifiable",
+ "ALTER TABLE test_virtual_ks.vt1 DROP v1");
+
+ assertInvalidMessage("Error during truncate: Cannot truncate virtual tables",
+ "TRUNCATE TABLE test_virtual_ks.vt1");
+
+ assertInvalidMessage("Virtual keyspace 'test_virtual_ks' is not user-modifiable",
+ "CREATE INDEX ON test_virtual_ks.vt1 (v1)");
+
+ assertInvalidMessage("Virtual keyspace 'test_virtual_ks' is not user-modifiable",
+ "CREATE MATERIALIZED VIEW test_virtual_ks.mvt1 AS SELECT c, v1 FROM test_virtual_ks.vt1 WHERE c IS NOT NULL PRIMARY KEY(c)");
+
+ assertInvalidMessage("Virtual keyspace 'test_virtual_ks' is not user-modifiable",
+ "CREATE TRIGGER test_trigger ON test_virtual_ks.vt1 USING '" + TestTrigger.class.getName() + '\'');
+ }
+
+ /**
+ * Noop trigger for audit log testing
+ */
+ public static class TestTrigger implements ITrigger
+ {
+ public Collection<Mutation> augment(Partition update)
+ {
+ return null;
+ }
+ }
+
+ @Test
+ public void testMBeansMethods() throws Throwable
+ {
+ StorageServiceMBean mbean = StorageService.instance;
+
+ assertJMXFails(() -> mbean.forceKeyspaceCompaction(false, KS_NAME));
+ assertJMXFails(() -> mbean.forceKeyspaceCompaction(false, KS_NAME, VT1_NAME));
+
+ assertJMXFails(() -> mbean.scrub(true, true, true, true, 1, KS_NAME));
+ assertJMXFails(() -> mbean.scrub(true, true, true, true, 1, KS_NAME, VT1_NAME));
+
+ assertJMXFails(() -> mbean.verify(true, KS_NAME));
+ assertJMXFails(() -> mbean.verify(true, KS_NAME, VT1_NAME));
+
+ assertJMXFails(() -> mbean.upgradeSSTables(KS_NAME, false, 1));
+ assertJMXFails(() -> mbean.upgradeSSTables(KS_NAME, false, 1, VT1_NAME));
+
+ assertJMXFails(() -> mbean.garbageCollect("ROW", 1, KS_NAME, VT1_NAME));
+
+ assertJMXFails(() -> mbean.forceKeyspaceFlush(KS_NAME));
+ assertJMXFails(() -> mbean.forceKeyspaceFlush(KS_NAME, VT1_NAME));
+
+ assertJMXFails(() -> mbean.truncate(KS_NAME, VT1_NAME));
+
+ assertJMXFails(() -> mbean.loadNewSSTables(KS_NAME, VT1_NAME));
+
+ assertJMXFails(() -> mbean.getAutoCompactionStatus(KS_NAME));
+ assertJMXFails(() -> mbean.getAutoCompactionStatus(KS_NAME, VT1_NAME));
+ }
+
+ @FunctionalInterface
+ private static interface ThrowingRunnable
+ {
+ public void run() throws Throwable;
+ }
+
+ private void assertJMXFails(ThrowingRunnable r) throws Throwable
+ {
+ try
+ {
+ r.run();
+ fail();
+ }
+ catch (IllegalArgumentException e)
+ {
+ assertEquals("Cannot perform any operations against virtual keyspace " + KS_NAME, e.getMessage());
+ }
+ }
+}
diff --git a/test/unit/org/apache/cassandra/cql3/validation/entities/udfverify/CallClone.java b/test/unit/org/apache/cassandra/cql3/validation/entities/udfverify/CallClone.java
index 4e0e1d3..1d13188 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/entities/udfverify/CallClone.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/entities/udfverify/CallClone.java
@@ -21,7 +21,7 @@
import java.nio.ByteBuffer;
import java.util.List;
-import com.datastax.driver.core.TypeCodec;
+import org.apache.cassandra.cql3.functions.types.TypeCodec;
import org.apache.cassandra.cql3.functions.JavaUDF;
import org.apache.cassandra.cql3.functions.UDFContext;
import org.apache.cassandra.transport.ProtocolVersion;
diff --git a/test/unit/org/apache/cassandra/cql3/validation/entities/udfverify/CallComDatastax.java b/test/unit/org/apache/cassandra/cql3/validation/entities/udfverify/CallComDatastax.java
deleted file mode 100644
index c4cef58..0000000
--- a/test/unit/org/apache/cassandra/cql3/validation/entities/udfverify/CallComDatastax.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.cql3.validation.entities.udfverify;
-
-import java.nio.ByteBuffer;
-import java.util.List;
-
-import com.datastax.driver.core.DataType;
-import com.datastax.driver.core.TypeCodec;
-import org.apache.cassandra.cql3.functions.JavaUDF;
-import org.apache.cassandra.cql3.functions.UDFContext;
-import org.apache.cassandra.transport.ProtocolVersion;
-
-/**
- * Used by {@link org.apache.cassandra.cql3.validation.entities.UFVerifierTest}.
- */
-public final class CallComDatastax extends JavaUDF
-{
- public CallComDatastax(TypeCodec<Object> returnDataType, TypeCodec<Object>[] argDataTypes, UDFContext udfContext)
- {
- super(returnDataType, argDataTypes, udfContext);
- }
-
- protected Object executeAggregateImpl(ProtocolVersion protocolVersion, Object firstParam, List<ByteBuffer> params)
- {
- throw new UnsupportedOperationException();
- }
-
- protected ByteBuffer executeImpl(ProtocolVersion protocolVersion, List<ByteBuffer> params)
- {
- DataType.cint();
- return null;
- }
-}
diff --git a/test/unit/org/apache/cassandra/cql3/validation/entities/udfverify/CallFinalize.java b/test/unit/org/apache/cassandra/cql3/validation/entities/udfverify/CallFinalize.java
index dfb523e..786129f 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/entities/udfverify/CallFinalize.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/entities/udfverify/CallFinalize.java
@@ -21,7 +21,7 @@
import java.nio.ByteBuffer;
import java.util.List;
-import com.datastax.driver.core.TypeCodec;
+import org.apache.cassandra.cql3.functions.types.TypeCodec;
import org.apache.cassandra.cql3.functions.JavaUDF;
import org.apache.cassandra.cql3.functions.UDFContext;
import org.apache.cassandra.transport.ProtocolVersion;
diff --git a/test/unit/org/apache/cassandra/cql3/validation/entities/udfverify/CallOrgApache.java b/test/unit/org/apache/cassandra/cql3/validation/entities/udfverify/CallOrgApache.java
index 34a82ed..77664ac 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/entities/udfverify/CallOrgApache.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/entities/udfverify/CallOrgApache.java
@@ -21,7 +21,7 @@
import java.nio.ByteBuffer;
import java.util.List;
-import com.datastax.driver.core.TypeCodec;
+import org.apache.cassandra.cql3.functions.types.TypeCodec;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.functions.JavaUDF;
import org.apache.cassandra.cql3.functions.UDFContext;
diff --git a/test/unit/org/apache/cassandra/cql3/validation/entities/udfverify/ClassWithField.java b/test/unit/org/apache/cassandra/cql3/validation/entities/udfverify/ClassWithField.java
index 33625fe..5fa12ed 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/entities/udfverify/ClassWithField.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/entities/udfverify/ClassWithField.java
@@ -21,7 +21,7 @@
import java.nio.ByteBuffer;
import java.util.List;
-import com.datastax.driver.core.TypeCodec;
+import org.apache.cassandra.cql3.functions.types.TypeCodec;
import org.apache.cassandra.cql3.functions.JavaUDF;
import org.apache.cassandra.cql3.functions.UDFContext;
import org.apache.cassandra.transport.ProtocolVersion;
diff --git a/test/unit/org/apache/cassandra/cql3/validation/entities/udfverify/ClassWithInitializer.java b/test/unit/org/apache/cassandra/cql3/validation/entities/udfverify/ClassWithInitializer.java
index 4f83bfe..e1c866a 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/entities/udfverify/ClassWithInitializer.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/entities/udfverify/ClassWithInitializer.java
@@ -21,7 +21,7 @@
import java.nio.ByteBuffer;
import java.util.List;
-import com.datastax.driver.core.TypeCodec;
+import org.apache.cassandra.cql3.functions.types.TypeCodec;
import org.apache.cassandra.cql3.functions.JavaUDF;
import org.apache.cassandra.cql3.functions.UDFContext;
import org.apache.cassandra.transport.ProtocolVersion;
diff --git a/test/unit/org/apache/cassandra/cql3/validation/entities/udfverify/ClassWithInitializer2.java b/test/unit/org/apache/cassandra/cql3/validation/entities/udfverify/ClassWithInitializer2.java
index df4c78a..15d60de 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/entities/udfverify/ClassWithInitializer2.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/entities/udfverify/ClassWithInitializer2.java
@@ -21,7 +21,7 @@
import java.nio.ByteBuffer;
import java.util.List;
-import com.datastax.driver.core.TypeCodec;
+import org.apache.cassandra.cql3.functions.types.TypeCodec;
import org.apache.cassandra.cql3.functions.JavaUDF;
import org.apache.cassandra.cql3.functions.UDFContext;
import org.apache.cassandra.transport.ProtocolVersion;
diff --git a/test/unit/org/apache/cassandra/cql3/validation/entities/udfverify/ClassWithInitializer3.java b/test/unit/org/apache/cassandra/cql3/validation/entities/udfverify/ClassWithInitializer3.java
index d30ada3..fc5d61c 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/entities/udfverify/ClassWithInitializer3.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/entities/udfverify/ClassWithInitializer3.java
@@ -21,7 +21,7 @@
import java.nio.ByteBuffer;
import java.util.List;
-import com.datastax.driver.core.TypeCodec;
+import org.apache.cassandra.cql3.functions.types.TypeCodec;
import org.apache.cassandra.cql3.functions.JavaUDF;
import org.apache.cassandra.cql3.functions.UDFContext;
import org.apache.cassandra.transport.ProtocolVersion;
diff --git a/test/unit/org/apache/cassandra/cql3/validation/entities/udfverify/ClassWithInnerClass.java b/test/unit/org/apache/cassandra/cql3/validation/entities/udfverify/ClassWithInnerClass.java
index 091597f..b387061 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/entities/udfverify/ClassWithInnerClass.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/entities/udfverify/ClassWithInnerClass.java
@@ -21,7 +21,7 @@
import java.nio.ByteBuffer;
import java.util.List;
-import com.datastax.driver.core.TypeCodec;
+import org.apache.cassandra.cql3.functions.types.TypeCodec;
import org.apache.cassandra.cql3.functions.JavaUDF;
import org.apache.cassandra.cql3.functions.UDFContext;
import org.apache.cassandra.transport.ProtocolVersion;
diff --git a/test/unit/org/apache/cassandra/cql3/validation/entities/udfverify/ClassWithInnerClass2.java b/test/unit/org/apache/cassandra/cql3/validation/entities/udfverify/ClassWithInnerClass2.java
index ac5b06f..0d2db32 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/entities/udfverify/ClassWithInnerClass2.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/entities/udfverify/ClassWithInnerClass2.java
@@ -21,7 +21,7 @@
import java.nio.ByteBuffer;
import java.util.List;
-import com.datastax.driver.core.TypeCodec;
+import org.apache.cassandra.cql3.functions.types.TypeCodec;
import org.apache.cassandra.cql3.functions.JavaUDF;
import org.apache.cassandra.cql3.functions.UDFContext;
import org.apache.cassandra.transport.ProtocolVersion;
diff --git a/test/unit/org/apache/cassandra/cql3/validation/entities/udfverify/ClassWithStaticInitializer.java b/test/unit/org/apache/cassandra/cql3/validation/entities/udfverify/ClassWithStaticInitializer.java
index c927667..bc7b253 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/entities/udfverify/ClassWithStaticInitializer.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/entities/udfverify/ClassWithStaticInitializer.java
@@ -21,7 +21,7 @@
import java.nio.ByteBuffer;
import java.util.List;
-import com.datastax.driver.core.TypeCodec;
+import org.apache.cassandra.cql3.functions.types.TypeCodec;
import org.apache.cassandra.cql3.functions.JavaUDF;
import org.apache.cassandra.cql3.functions.UDFContext;
import org.apache.cassandra.transport.ProtocolVersion;
diff --git a/test/unit/org/apache/cassandra/cql3/validation/entities/udfverify/ClassWithStaticInnerClass.java b/test/unit/org/apache/cassandra/cql3/validation/entities/udfverify/ClassWithStaticInnerClass.java
index 9ce5d71..2b65621 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/entities/udfverify/ClassWithStaticInnerClass.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/entities/udfverify/ClassWithStaticInnerClass.java
@@ -21,7 +21,7 @@
import java.nio.ByteBuffer;
import java.util.List;
-import com.datastax.driver.core.TypeCodec;
+import org.apache.cassandra.cql3.functions.types.TypeCodec;
import org.apache.cassandra.cql3.functions.JavaUDF;
import org.apache.cassandra.cql3.functions.UDFContext;
import org.apache.cassandra.transport.ProtocolVersion;
diff --git a/test/unit/org/apache/cassandra/cql3/validation/entities/udfverify/GoodClass.java b/test/unit/org/apache/cassandra/cql3/validation/entities/udfverify/GoodClass.java
index 7275ef5..4bab493 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/entities/udfverify/GoodClass.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/entities/udfverify/GoodClass.java
@@ -21,7 +21,7 @@
import java.nio.ByteBuffer;
import java.util.List;
-import com.datastax.driver.core.TypeCodec;
+import org.apache.cassandra.cql3.functions.types.TypeCodec;
import org.apache.cassandra.cql3.functions.JavaUDF;
import org.apache.cassandra.cql3.functions.UDFContext;
import org.apache.cassandra.transport.ProtocolVersion;
diff --git a/test/unit/org/apache/cassandra/cql3/validation/entities/udfverify/UseOfSynchronized.java b/test/unit/org/apache/cassandra/cql3/validation/entities/udfverify/UseOfSynchronized.java
index c036f63..2c39a83 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/entities/udfverify/UseOfSynchronized.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/entities/udfverify/UseOfSynchronized.java
@@ -21,7 +21,7 @@
import java.nio.ByteBuffer;
import java.util.List;
-import com.datastax.driver.core.TypeCodec;
+import org.apache.cassandra.cql3.functions.types.TypeCodec;
import org.apache.cassandra.cql3.functions.JavaUDF;
import org.apache.cassandra.cql3.functions.UDFContext;
import org.apache.cassandra.transport.ProtocolVersion;
diff --git a/test/unit/org/apache/cassandra/cql3/validation/entities/udfverify/UseOfSynchronizedWithNotify.java b/test/unit/org/apache/cassandra/cql3/validation/entities/udfverify/UseOfSynchronizedWithNotify.java
index 3eb673a..6240ad1 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/entities/udfverify/UseOfSynchronizedWithNotify.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/entities/udfverify/UseOfSynchronizedWithNotify.java
@@ -21,7 +21,7 @@
import java.nio.ByteBuffer;
import java.util.List;
-import com.datastax.driver.core.TypeCodec;
+import org.apache.cassandra.cql3.functions.types.TypeCodec;
import org.apache.cassandra.cql3.functions.JavaUDF;
import org.apache.cassandra.cql3.functions.UDFContext;
import org.apache.cassandra.transport.ProtocolVersion;
diff --git a/test/unit/org/apache/cassandra/cql3/validation/entities/udfverify/UseOfSynchronizedWithNotifyAll.java b/test/unit/org/apache/cassandra/cql3/validation/entities/udfverify/UseOfSynchronizedWithNotifyAll.java
index d9841f7..60e81aa 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/entities/udfverify/UseOfSynchronizedWithNotifyAll.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/entities/udfverify/UseOfSynchronizedWithNotifyAll.java
@@ -21,7 +21,7 @@
import java.nio.ByteBuffer;
import java.util.List;
-import com.datastax.driver.core.TypeCodec;
+import org.apache.cassandra.cql3.functions.types.TypeCodec;
import org.apache.cassandra.cql3.functions.JavaUDF;
import org.apache.cassandra.cql3.functions.UDFContext;
import org.apache.cassandra.transport.ProtocolVersion;
diff --git a/test/unit/org/apache/cassandra/cql3/validation/entities/udfverify/UseOfSynchronizedWithWait.java b/test/unit/org/apache/cassandra/cql3/validation/entities/udfverify/UseOfSynchronizedWithWait.java
index b4e4af3..71951bb 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/entities/udfverify/UseOfSynchronizedWithWait.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/entities/udfverify/UseOfSynchronizedWithWait.java
@@ -21,7 +21,7 @@
import java.nio.ByteBuffer;
import java.util.List;
-import com.datastax.driver.core.TypeCodec;
+import org.apache.cassandra.cql3.functions.types.TypeCodec;
import org.apache.cassandra.cql3.functions.JavaUDF;
import org.apache.cassandra.cql3.functions.UDFContext;
import org.apache.cassandra.transport.ProtocolVersion;
diff --git a/test/unit/org/apache/cassandra/cql3/validation/entities/udfverify/UseOfSynchronizedWithWaitL.java b/test/unit/org/apache/cassandra/cql3/validation/entities/udfverify/UseOfSynchronizedWithWaitL.java
index 24d4a21..13e1501 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/entities/udfverify/UseOfSynchronizedWithWaitL.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/entities/udfverify/UseOfSynchronizedWithWaitL.java
@@ -21,7 +21,7 @@
import java.nio.ByteBuffer;
import java.util.List;
-import com.datastax.driver.core.TypeCodec;
+import org.apache.cassandra.cql3.functions.types.TypeCodec;
import org.apache.cassandra.cql3.functions.JavaUDF;
import org.apache.cassandra.cql3.functions.UDFContext;
import org.apache.cassandra.transport.ProtocolVersion;
diff --git a/test/unit/org/apache/cassandra/cql3/validation/entities/udfverify/UseOfSynchronizedWithWaitLI.java b/test/unit/org/apache/cassandra/cql3/validation/entities/udfverify/UseOfSynchronizedWithWaitLI.java
index 5f61bf6..b0b344d 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/entities/udfverify/UseOfSynchronizedWithWaitLI.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/entities/udfverify/UseOfSynchronizedWithWaitLI.java
@@ -21,7 +21,7 @@
import java.nio.ByteBuffer;
import java.util.List;
-import com.datastax.driver.core.TypeCodec;
+import org.apache.cassandra.cql3.functions.types.TypeCodec;
import org.apache.cassandra.cql3.functions.JavaUDF;
import org.apache.cassandra.cql3.functions.UDFContext;
import org.apache.cassandra.transport.ProtocolVersion;
diff --git a/test/unit/org/apache/cassandra/cql3/validation/entities/udfverify/UsingMapEntry.java b/test/unit/org/apache/cassandra/cql3/validation/entities/udfverify/UsingMapEntry.java
index 0b95b90..cf6ee64 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/entities/udfverify/UsingMapEntry.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/entities/udfverify/UsingMapEntry.java
@@ -23,7 +23,7 @@
import java.util.List;
import java.util.Map;
-import com.datastax.driver.core.TypeCodec;
+import org.apache.cassandra.cql3.functions.types.TypeCodec;
import org.apache.cassandra.cql3.functions.JavaUDF;
import org.apache.cassandra.cql3.functions.UDFContext;
import org.apache.cassandra.transport.ProtocolVersion;
diff --git a/test/unit/org/apache/cassandra/cql3/validation/miscellaneous/CrcCheckChanceTest.java b/test/unit/org/apache/cassandra/cql3/validation/miscellaneous/CrcCheckChanceTest.java
index 2760ae5..246f512 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/miscellaneous/CrcCheckChanceTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/miscellaneous/CrcCheckChanceTest.java
@@ -23,7 +23,7 @@
import org.junit.Test;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.CQLTester;
import org.apache.cassandra.db.ColumnFamilyStore;
@@ -70,11 +70,11 @@
ColumnFamilyStore indexCfs = cfs.indexManager.getAllIndexColumnFamilyStores().iterator().next();
cfs.forceBlockingFlush();
- Assert.assertEquals(0.99, cfs.getCrcCheckChance());
- Assert.assertEquals(0.99, cfs.getLiveSSTables().iterator().next().getCrcCheckChance());
+ Assert.assertEquals(0.99, cfs.getCrcCheckChance(), 0.0);
+ Assert.assertEquals(0.99, cfs.getLiveSSTables().iterator().next().getCrcCheckChance(), 0.0);
- Assert.assertEquals(0.99, indexCfs.getCrcCheckChance());
- Assert.assertEquals(0.99, indexCfs.getLiveSSTables().iterator().next().getCrcCheckChance());
+ Assert.assertEquals(0.99, indexCfs.getCrcCheckChance(), 0.0);
+ Assert.assertEquals(0.99, indexCfs.getLiveSSTables().iterator().next().getCrcCheckChance(), 0.0);
//Test for stack overflow
if (newFormat)
@@ -114,10 +114,10 @@
//Now let's change via JMX
cfs.setCrcCheckChance(0.01);
- Assert.assertEquals(0.01, cfs.getCrcCheckChance());
- Assert.assertEquals(0.01, cfs.getLiveSSTables().iterator().next().getCrcCheckChance());
- Assert.assertEquals(0.01, indexCfs.getCrcCheckChance());
- Assert.assertEquals(0.01, indexCfs.getLiveSSTables().iterator().next().getCrcCheckChance());
+ Assert.assertEquals(0.01, cfs.getCrcCheckChance(), 0.0);
+ Assert.assertEquals(0.01, cfs.getLiveSSTables().iterator().next().getCrcCheckChance(), 0.0);
+ Assert.assertEquals(0.01, indexCfs.getCrcCheckChance(), 0.0);
+ Assert.assertEquals(0.01, indexCfs.getLiveSSTables().iterator().next().getCrcCheckChance(), 0.0);
assertRows(execute("SELECT * FROM %s WHERE p=?", "p1"),
row("p1", "k1", "sv1", "v1"),
@@ -135,19 +135,19 @@
alterTable("ALTER TABLE %s WITH compression = {'sstable_compression': 'LZ4Compressor', 'crc_check_chance': 0.5}");
//We should be able to get the new value by accessing directly the schema metadata
- Assert.assertEquals(0.5, cfs.metadata.params.crcCheckChance);
+ Assert.assertEquals(0.5, cfs.metadata().params.crcCheckChance, 0.0);
//but previous JMX-set value will persist until next restart
- Assert.assertEquals(0.01, cfs.getLiveSSTables().iterator().next().getCrcCheckChance());
- Assert.assertEquals(0.01, indexCfs.getCrcCheckChance());
- Assert.assertEquals(0.01, indexCfs.getLiveSSTables().iterator().next().getCrcCheckChance());
+ Assert.assertEquals(0.01, cfs.getLiveSSTables().iterator().next().getCrcCheckChance(), 0.0);
+ Assert.assertEquals(0.01, indexCfs.getCrcCheckChance(), 0.0);
+ Assert.assertEquals(0.01, indexCfs.getLiveSSTables().iterator().next().getCrcCheckChance(), 0.0);
//Verify the call used by JMX still works
cfs.setCrcCheckChance(0.03);
- Assert.assertEquals(0.03, cfs.getCrcCheckChance());
- Assert.assertEquals(0.03, cfs.getLiveSSTables().iterator().next().getCrcCheckChance());
- Assert.assertEquals(0.03, indexCfs.getCrcCheckChance());
- Assert.assertEquals(0.03, indexCfs.getLiveSSTables().iterator().next().getCrcCheckChance());
+ Assert.assertEquals(0.03, cfs.getCrcCheckChance(), 0.0);
+ Assert.assertEquals(0.03, cfs.getLiveSSTables().iterator().next().getCrcCheckChance(), 0.0);
+ Assert.assertEquals(0.03, indexCfs.getCrcCheckChance(), 0.0);
+ Assert.assertEquals(0.03, indexCfs.getLiveSSTables().iterator().next().getCrcCheckChance(), 0.0);
// Also check that any open readers also use the updated value
// note: only compressed files currently perform crc checks, so only the dfile reader is relevant here
@@ -156,12 +156,12 @@
try (RandomAccessReader baseDataReader = baseSSTable.openDataReader();
RandomAccessReader idxDataReader = idxSSTable.openDataReader())
{
- Assert.assertEquals(0.03, baseDataReader.getCrcCheckChance());
- Assert.assertEquals(0.03, idxDataReader.getCrcCheckChance());
+ Assert.assertEquals(0.03, baseDataReader.getCrcCheckChance(), 0.0);
+ Assert.assertEquals(0.03, idxDataReader.getCrcCheckChance(), 0.0);
cfs.setCrcCheckChance(0.31);
- Assert.assertEquals(0.31, baseDataReader.getCrcCheckChance());
- Assert.assertEquals(0.31, idxDataReader.getCrcCheckChance());
+ Assert.assertEquals(0.31, baseDataReader.getCrcCheckChance(), 0.0);
+ Assert.assertEquals(0.31, idxDataReader.getCrcCheckChance(), 0.0);
}
}
diff --git a/test/unit/org/apache/cassandra/cql3/validation/miscellaneous/OverflowTest.java b/test/unit/org/apache/cassandra/cql3/validation/miscellaneous/OverflowTest.java
index 6fdedc2..71d632d 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/miscellaneous/OverflowTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/miscellaneous/OverflowTest.java
@@ -107,8 +107,6 @@
{
createTable("CREATE TABLE %s ( k int PRIMARY KEY, c int ) WITH "
+ "comment = 'My comment' "
- + "AND read_repair_chance = 0.5 "
- + "AND dclocal_read_repair_chance = 0.5 "
+ "AND gc_grace_seconds = 4 "
+ "AND bloom_filter_fp_chance = 0.01 "
+ "AND compaction = { 'class' : 'LeveledCompactionStrategy', 'sstable_size_in_mb' : 10, 'fanout_size' : 5 } "
@@ -117,8 +115,6 @@
execute("ALTER TABLE %s WITH "
+ "comment = 'other comment' "
- + "AND read_repair_chance = 0.3 "
- + "AND dclocal_read_repair_chance = 0.3 "
+ "AND gc_grace_seconds = 100 "
+ "AND bloom_filter_fp_chance = 0.1 "
+ "AND compaction = { 'class': 'SizeTieredCompactionStrategy', 'min_sstable_size' : 42 } "
@@ -167,20 +163,6 @@
}
/**
- * Test regression from #5189,
- * migrated from cql_tests.py:TestCQL.compact_metadata_test()
- */
- @Test
- public void testCompactMetadata() throws Throwable
- {
- createTable("CREATE TABLE %s (id int primary key, i int ) WITH COMPACT STORAGE");
-
- execute("INSERT INTO %s (id, i) VALUES (1, 2)");
- assertRows(execute("SELECT * FROM %s"),
- row(1, 2));
- }
-
- /**
* Migrated from cql_tests.py:TestCQL.conversion_functions_test()
*/
@Test
@@ -237,22 +219,6 @@
// Test empty IN() in UPDATE
execute("UPDATE %s SET v = 3 WHERE k1 IN () AND k2 = 2");
assertArrayEquals(rows, getRows(execute("SELECT * FROM %s")));
-
- // Same test, but for compact
- createTable("CREATE TABLE %s (k1 int, k2 int, v int, PRIMARY KEY (k1, k2)) WITH COMPACT STORAGE");
-
- rows = fill();
-
- assertEmpty(execute("SELECT v FROM %s WHERE k1 IN ()"));
- assertEmpty(execute("SELECT v FROM %s WHERE k1 = 0 AND k2 IN ()"));
-
- // Test empty IN() in DELETE
- execute("DELETE FROM %s WHERE k1 IN ()");
- assertArrayEquals(rows, getRows(execute("SELECT * FROM %s")));
-
- // Test empty IN() in UPDATE
- execute("UPDATE %s SET v = 3 WHERE k1 IN () AND k2 = 2");
- assertArrayEquals(rows, getRows(execute("SELECT * FROM %s")));
}
/**
diff --git a/test/unit/org/apache/cassandra/cql3/validation/miscellaneous/TombstonesTest.java b/test/unit/org/apache/cassandra/cql3/validation/miscellaneous/TombstonesTest.java
index 72ed887..85048ae 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/miscellaneous/TombstonesTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/miscellaneous/TombstonesTest.java
@@ -24,10 +24,14 @@
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
+
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.db.filter.TombstoneOverwhelmingException;
+import static junit.framework.Assert.assertEquals;
import static junit.framework.Assert.assertTrue;
import static junit.framework.Assert.fail;
@@ -36,34 +40,45 @@
*/
public class TombstonesTest extends CQLTester
{
- static final int ORIGINAL_THRESHOLD = DatabaseDescriptor.getTombstoneFailureThreshold();
- static final int THRESHOLD = 100;
+ static final int ORIGINAL_FAILURE_THRESHOLD = DatabaseDescriptor.getTombstoneFailureThreshold();
+ static final int FAILURE_THRESHOLD = 100;
+
+ static final int ORIGINAL_WARN_THRESHOLD = DatabaseDescriptor.getTombstoneFailureThreshold();
+ static final int WARN_THRESHOLD = 50;
@BeforeClass
public static void setUp() throws Throwable
{
DatabaseDescriptor.daemonInitialization();
- DatabaseDescriptor.setTombstoneFailureThreshold(THRESHOLD);
+ DatabaseDescriptor.setTombstoneFailureThreshold(FAILURE_THRESHOLD);
+ DatabaseDescriptor.setTombstoneWarnThreshold(WARN_THRESHOLD);
}
@AfterClass
public static void tearDown()
{
- DatabaseDescriptor.setTombstoneFailureThreshold(ORIGINAL_THRESHOLD);
+ DatabaseDescriptor.setTombstoneFailureThreshold(ORIGINAL_FAILURE_THRESHOLD);
+ DatabaseDescriptor.setTombstoneWarnThreshold(ORIGINAL_WARN_THRESHOLD);
}
@Test
public void testBelowThresholdSelect() throws Throwable
{
- createTable("CREATE TABLE %s (a text, b text, c text, PRIMARY KEY (a, b));");
+
+ String tableName = createTable("CREATE TABLE %s (a text, b text, c text, PRIMARY KEY (a, b));");
+ ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(tableName);
+ long oldFailures = cfs.metric.tombstoneFailures.getCount();
+ long oldWarnings = cfs.metric.tombstoneWarnings.getCount();
// insert exactly the amount of tombstones that shouldn't trigger an exception
- for (int i = 0; i < THRESHOLD; i++)
+ for (int i = 0; i < FAILURE_THRESHOLD; i++)
execute("INSERT INTO %s (a, b, c) VALUES ('key', 'column" + i + "', null);");
try
{
execute("SELECT * FROM %s WHERE a = 'key';");
+ assertEquals(oldFailures, cfs.metric.tombstoneFailures.getCount());
+ assertEquals(oldWarnings, cfs.metric.tombstoneWarnings.getCount());
}
catch (Throwable e)
{
@@ -74,10 +89,13 @@
@Test
public void testBeyondThresholdSelect() throws Throwable
{
- createTable("CREATE TABLE %s (a text, b text, c text, PRIMARY KEY (a, b));");
+ String tableName = createTable("CREATE TABLE %s (a text, b text, c text, PRIMARY KEY (a, b));");
+ ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(tableName);
+ long oldFailures = cfs.metric.tombstoneFailures.getCount();
+ long oldWarnings = cfs.metric.tombstoneWarnings.getCount();
// insert exactly the amount of tombstones that *SHOULD* trigger an exception
- for (int i = 0; i < THRESHOLD + 1; i++)
+ for (int i = 0; i < FAILURE_THRESHOLD + 1; i++)
execute("INSERT INTO %s (a, b, c) VALUES ('key', 'column" + i + "', null);");
try
@@ -88,19 +106,24 @@
catch (Throwable e)
{
String error = "Expected exception instanceof TombstoneOverwhelmingException instead got "
- + System.lineSeparator()
- + Throwables.getStackTraceAsString(e);
+ + System.lineSeparator()
+ + Throwables.getStackTraceAsString(e);
assertTrue(error, e instanceof TombstoneOverwhelmingException);
+ assertEquals(oldWarnings, cfs.metric.tombstoneWarnings.getCount());
+ assertEquals(oldFailures + 1, cfs.metric.tombstoneFailures.getCount());
}
}
@Test
public void testAllShadowedSelect() throws Throwable
{
- createTable("CREATE TABLE %s (a text, b text, c text, PRIMARY KEY (a, b));");
+ String tableName = createTable("CREATE TABLE %s (a text, b text, c text, PRIMARY KEY (a, b));");
+ ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(tableName);
+ long oldFailures = cfs.metric.tombstoneFailures.getCount();
+ long oldWarnings = cfs.metric.tombstoneWarnings.getCount();
// insert exactly the amount of tombstones that *SHOULD* normally trigger an exception
- for (int i = 0; i < THRESHOLD + 1; i++)
+ for (int i = 0; i < FAILURE_THRESHOLD + 1; i++)
execute("INSERT INTO %s (a, b, c) VALUES ('key', 'column" + i + "', null);");
// delete all with a partition level tombstone
@@ -109,6 +132,8 @@
try
{
execute("SELECT * FROM %s WHERE a = 'key';");
+ assertEquals(oldFailures, cfs.metric.tombstoneFailures.getCount());
+ assertEquals(oldWarnings, cfs.metric.tombstoneWarnings.getCount());
}
catch (Throwable e)
{
@@ -119,9 +144,12 @@
@Test
public void testLiveShadowedCellsSelect() throws Throwable
{
- createTable("CREATE TABLE %s (a text, b text, c text, PRIMARY KEY (a, b));");
+ String tableName = createTable("CREATE TABLE %s (a text, b text, c text, PRIMARY KEY (a, b));");
+ ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(tableName);
+ long oldFailures = cfs.metric.tombstoneFailures.getCount();
+ long oldWarnings = cfs.metric.tombstoneWarnings.getCount();
- for (int i = 0; i < THRESHOLD + 1; i++)
+ for (int i = 0; i < FAILURE_THRESHOLD + 1; i++)
execute("INSERT INTO %s (a, b, c) VALUES ('key', 'column" + i + "', 'column');");
// delete all with a partition level tombstone
@@ -130,6 +158,8 @@
try
{
execute("SELECT * FROM %s WHERE a = 'key';");
+ assertEquals(oldFailures, cfs.metric.tombstoneFailures.getCount());
+ assertEquals(oldWarnings, cfs.metric.tombstoneWarnings.getCount());
}
catch (Throwable e)
{
@@ -140,9 +170,12 @@
@Test
public void testExpiredTombstones() throws Throwable
{
- createTable("CREATE TABLE %s (a text, b text, c text, PRIMARY KEY (a, b)) WITH gc_grace_seconds = 1;");
+ String tableName = createTable("CREATE TABLE %s (a text, b text, c text, PRIMARY KEY (a, b)) WITH gc_grace_seconds = 1;");
+ ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(tableName);
+ long oldFailures = cfs.metric.tombstoneFailures.getCount();
+ long oldWarnings = cfs.metric.tombstoneWarnings.getCount();
- for (int i = 0; i < THRESHOLD + 1; i++)
+ for (int i = 0; i < FAILURE_THRESHOLD + 1; i++)
execute("INSERT INTO %s (a, b, c) VALUES ('key', 'column" + i + "', null);");
// not yet past gc grace - must throw a TOE
@@ -154,6 +187,9 @@
catch (Throwable e)
{
assertTrue(e instanceof TombstoneOverwhelmingException);
+
+ assertEquals(++oldFailures, cfs.metric.tombstoneFailures.getCount());
+ assertEquals(oldWarnings, cfs.metric.tombstoneWarnings.getCount());
}
// sleep past gc grace
@@ -163,10 +199,37 @@
try
{
execute("SELECT * FROM %s WHERE a = 'key';");
+ assertEquals(oldFailures, cfs.metric.tombstoneFailures.getCount());
+ assertEquals(oldWarnings, cfs.metric.tombstoneWarnings.getCount());
}
catch (Throwable e)
{
fail("SELECT with expired tombstones beyond the threshold should not have failed, but has: " + e);
}
}
+
+ @Test
+ public void testBeyondWarnThresholdSelect() throws Throwable
+ {
+ String tableName = createTable("CREATE TABLE %s (a text, b text, c text, PRIMARY KEY (a,b));");
+ ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(tableName);
+ long oldFailures = cfs.metric.tombstoneFailures.getCount();
+ long oldWarnings = cfs.metric.tombstoneWarnings.getCount();
+
+ // insert the number of tombstones that *SHOULD* trigger an Warning
+ for (int i = 0; i < WARN_THRESHOLD + 1; i++)
+ execute("INSERT INTO %s (a, b, c ) VALUES ('key', 'cc" + i + "', null);");
+ try
+ {
+ execute("SELECT * FROM %s WHERE a = 'key';");
+ assertEquals(oldWarnings + 1, cfs.metric.tombstoneWarnings.getCount());
+ assertEquals(oldFailures, cfs.metric.tombstoneFailures.getCount());
+ }
+ catch (Throwable e)
+ {
+ fail("SELECT with tombstones below the failure threshold and above warning threashhold should not have failed, but has: " + e);
+ }
+ }
+
+
}
diff --git a/test/unit/org/apache/cassandra/cql3/validation/operations/AggregationTest.java b/test/unit/org/apache/cassandra/cql3/validation/operations/AggregationTest.java
index 9841482..703d0ad 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/operations/AggregationTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/operations/AggregationTest.java
@@ -38,26 +38,25 @@
import org.slf4j.LoggerFactory;
import ch.qos.logback.classic.LoggerContext;
+import ch.qos.logback.classic.joran.ReconfigureOnChangeTask;
import ch.qos.logback.classic.spi.TurboFilterList;
import ch.qos.logback.classic.turbo.ReconfigureOnChangeFilter;
import ch.qos.logback.classic.turbo.TurboFilter;
-import org.apache.cassandra.config.Schema;
-import org.apache.cassandra.config.SchemaConstants;
+import org.apache.cassandra.schema.SchemaConstants;
import org.apache.cassandra.cql3.CQLTester;
import org.apache.cassandra.cql3.QueryProcessor;
import org.apache.cassandra.cql3.UntypedResultSet;
import org.apache.cassandra.cql3.UntypedResultSet.Row;
-import org.apache.cassandra.cql3.functions.UDAggregate;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.db.marshal.TypeParser;
import org.apache.cassandra.exceptions.FunctionExecutionException;
import org.apache.cassandra.exceptions.InvalidRequestException;
-import org.apache.cassandra.schema.KeyspaceMetadata;
import org.apache.cassandra.service.ClientState;
import org.apache.cassandra.transport.Event;
import org.apache.cassandra.transport.ProtocolVersion;
import org.apache.cassandra.transport.messages.ResultMessage;
+import static ch.qos.logback.core.CoreConstants.RECONFIGURE_ON_CHANGE_TASK;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
@@ -68,10 +67,21 @@
@Test
public void testNonExistingOnes() throws Throwable
{
- assertInvalidThrowMessage("Cannot drop non existing aggregate", InvalidRequestException.class, "DROP AGGREGATE " + KEYSPACE + ".aggr_does_not_exist");
- assertInvalidThrowMessage("Cannot drop non existing aggregate", InvalidRequestException.class, "DROP AGGREGATE " + KEYSPACE + ".aggr_does_not_exist(int,text)");
- assertInvalidThrowMessage("Cannot drop non existing aggregate", InvalidRequestException.class, "DROP AGGREGATE keyspace_does_not_exist.aggr_does_not_exist");
- assertInvalidThrowMessage("Cannot drop non existing aggregate", InvalidRequestException.class, "DROP AGGREGATE keyspace_does_not_exist.aggr_does_not_exist(int,text)");
+ assertInvalidThrowMessage(String.format("Aggregate '%s.aggr_does_not_exist' doesn't exist", KEYSPACE),
+ InvalidRequestException.class,
+ "DROP AGGREGATE " + KEYSPACE + ".aggr_does_not_exist");
+
+ assertInvalidThrowMessage(String.format("Aggregate '%s.aggr_does_not_exist(int, text)' doesn't exist", KEYSPACE),
+ InvalidRequestException.class,
+ "DROP AGGREGATE " + KEYSPACE + ".aggr_does_not_exist(int,text)");
+
+ assertInvalidThrowMessage("Aggregate 'keyspace_does_not_exist.aggr_does_not_exist' doesn't exist",
+ InvalidRequestException.class,
+ "DROP AGGREGATE keyspace_does_not_exist.aggr_does_not_exist");
+
+ assertInvalidThrowMessage("Aggregate 'keyspace_does_not_exist.aggr_does_not_exist(int, text)' doesn't exist",
+ InvalidRequestException.class,
+ "DROP AGGREGATE keyspace_does_not_exist.aggr_does_not_exist(int,text)");
execute("DROP AGGREGATE IF EXISTS " + KEYSPACE + ".aggr_does_not_exist");
execute("DROP AGGREGATE IF EXISTS " + KEYSPACE + ".aggr_does_not_exist(int,text)");
@@ -321,26 +331,6 @@
}
@Test
- public void testFunctionsWithCompactStorage() throws Throwable
- {
- createTable("CREATE TABLE %s (a int , b int, c double, primary key(a, b) ) WITH COMPACT STORAGE");
-
- execute("INSERT INTO %s (a, b, c) VALUES (1, 1, 11.5)");
- execute("INSERT INTO %s (a, b, c) VALUES (1, 2, 9.5)");
- execute("INSERT INTO %s (a, b, c) VALUES (1, 3, 9.0)");
-
- assertRows(execute("SELECT max(b), min(b), sum(b), avg(b) , max(c), sum(c), avg(c) FROM %s"),
- row(3, 1, 6, 2, 11.5, 30.0, 10.0));
-
- assertRows(execute("SELECT COUNT(*) FROM %s"), row(3L));
- assertRows(execute("SELECT COUNT(1) FROM %s"), row(3L));
- assertRows(execute("SELECT COUNT(*) FROM %s WHERE a = 1 AND b > 1"), row(2L));
- assertRows(execute("SELECT COUNT(1) FROM %s WHERE a = 1 AND b > 1"), row(2L));
- assertRows(execute("SELECT max(b), min(b), sum(b), avg(b) , max(c), sum(c), avg(c) FROM %s WHERE a = 1 AND b > 1"),
- row(3, 2, 5, 2, 9.5, 18.5, 9.25));
- }
-
- @Test
public void testInvalidCalls() throws Throwable
{
createTable("CREATE TABLE %s (a int, b int, c int, primary key (a, b))");
@@ -437,7 +427,7 @@
schemaChange("CREATE OR REPLACE AGGREGATE " + a + "(double) " +
"SFUNC " + shortFunctionName(f) + " " +
"STYPE double " +
- "INITCOND 0");
+ "INITCOND 1");
assertLastSchemaChange(Event.SchemaChange.Change.UPDATED, Event.SchemaChange.Target.AGGREGATE,
KEYSPACE, parseFunctionName(a).name,
@@ -482,7 +472,7 @@
// DROP AGGREGATE must not succeed against a scalar
assertInvalidMessage("matches multiple function definitions", "DROP AGGREGATE " + f);
- assertInvalidMessage("non existing", "DROP AGGREGATE " + f + "(double, double)");
+ assertInvalidMessage("doesn't exist", "DROP AGGREGATE " + f + "(double, double)");
String a = createAggregate(KEYSPACE,
"double",
@@ -499,7 +489,7 @@
// DROP FUNCTION must not succeed against an aggregate
assertInvalidMessage("matches multiple function definitions", "DROP FUNCTION " + a);
- assertInvalidMessage("non existing function", "DROP FUNCTION " + a + "(double)");
+ assertInvalidMessage("doesn't exist", "DROP FUNCTION " + a + "(double)");
// ambigious
assertInvalidMessage("matches multiple function definitions", "DROP AGGREGATE " + a);
@@ -678,37 +668,37 @@
"LANGUAGE java " +
"AS 'return a.toString();'");
- assertInvalidMessage("does not exist or is not a scalar function",
+ assertInvalidMessage("doesn't exist",
"CREATE AGGREGATE " + KEYSPACE + ".aggrInvalid(double)" +
"SFUNC " + shortFunctionName(fState) + " " +
"STYPE double " +
"FINALFUNC " + shortFunctionName(fFinal));
- assertInvalidMessage("does not exist or is not a scalar function",
+ assertInvalidMessage("doesn't exist",
"CREATE AGGREGATE " + KEYSPACE + ".aggrInvalid(int)" +
"SFUNC " + shortFunctionName(fState) + " " +
"STYPE double " +
"FINALFUNC " + shortFunctionName(fFinal));
- assertInvalidMessage("does not exist or is not a scalar function",
+ assertInvalidMessage("doesn't exist",
"CREATE AGGREGATE " + KEYSPACE + ".aggrInvalid(double)" +
"SFUNC " + shortFunctionName(fState) + " " +
"STYPE int " +
"FINALFUNC " + shortFunctionName(fFinal));
- assertInvalidMessage("does not exist or is not a scalar function",
+ assertInvalidMessage("doesn't exist",
"CREATE AGGREGATE " + KEYSPACE + ".aggrInvalid(double)" +
"SFUNC " + shortFunctionName(fState) + " " +
"STYPE int");
- assertInvalidMessage("does not exist or is not a scalar function",
+ assertInvalidMessage("doesn't exist",
"CREATE AGGREGATE " + KEYSPACE + ".aggrInvalid(int)" +
"SFUNC " + shortFunctionName(fState) + " " +
"STYPE double");
- assertInvalidMessage("does not exist or is not a scalar function",
+ assertInvalidMessage("doesn't exist",
"CREATE AGGREGATE " + KEYSPACE + ".aggrInvalid(double)" +
"SFUNC " + shortFunctionName(fState2) + " " +
"STYPE double " +
"FINALFUNC " + shortFunctionName(fFinal));
- assertInvalidMessage("does not exist or is not a scalar function",
+ assertInvalidMessage("doesn't exist",
"CREATE AGGREGATE " + KEYSPACE + ".aggrInvalid(double)" +
"SFUNC " + shortFunctionName(fState) + " " +
"STYPE double " +
@@ -734,13 +724,13 @@
"LANGUAGE java " +
"AS 'return a.toString();'");
- assertInvalidMessage("does not exist or is not a scalar function",
+ assertInvalidMessage("doesn't exist",
"CREATE AGGREGATE " + KEYSPACE + ".aggrInvalid(int)" +
"SFUNC " + shortFunctionName(fState) + "_not_there " +
"STYPE int " +
"FINALFUNC " + shortFunctionName(fFinal));
- assertInvalidMessage("does not exist or is not a scalar function",
+ assertInvalidMessage("doesn't exist",
"CREATE AGGREGATE " + KEYSPACE + ".aggrInvalid(int)" +
"SFUNC " + shortFunctionName(fState) + " " +
"STYPE int " +
@@ -823,7 +813,7 @@
@Test
public void testJavaAggregateWithoutStateOrFinal() throws Throwable
{
- assertInvalidMessage("does not exist or is not a scalar function",
+ assertInvalidMessage("doesn't exist",
"CREATE AGGREGATE " + KEYSPACE + ".jSumFooNE1(int) " +
"SFUNC jSumFooNEstate " +
"STYPE int");
@@ -836,7 +826,7 @@
"LANGUAGE java " +
"AS 'return Integer.valueOf(a + b);'");
- assertInvalidMessage("does not exist or is not a scalar function",
+ assertInvalidMessage("doesn't exist",
"CREATE AGGREGATE " + KEYSPACE + ".jSumFooNE2(int) " +
"SFUNC " + shortFunctionName(f) + " " +
"STYPE int " +
@@ -1119,7 +1109,7 @@
"SFUNC " + shortFunctionName(fState) + " " +
"STYPE int");
- ResultMessage.Prepared prepared = QueryProcessor.prepare("SELECT " + a + "(b) FROM " + otherKS + ".jsdp", ClientState.forInternalCalls(), false);
+ ResultMessage.Prepared prepared = QueryProcessor.prepare("SELECT " + a + "(b) FROM " + otherKS + ".jsdp", ClientState.forInternalCalls());
assertNotNull(QueryProcessor.instance.getPrepared(prepared.statementId));
execute("DROP AGGREGATE " + a + "(int)");
@@ -1131,7 +1121,7 @@
"SFUNC " + shortFunctionName(fState) + " " +
"STYPE int");
- prepared = QueryProcessor.prepare("SELECT " + a + "(b) FROM " + otherKS + ".jsdp", ClientState.forInternalCalls(), false);
+ prepared = QueryProcessor.prepare("SELECT " + a + "(b) FROM " + otherKS + ".jsdp", ClientState.forInternalCalls());
assertNotNull(QueryProcessor.instance.getPrepared(prepared.statementId));
execute("DROP KEYSPACE " + otherKS + ";");
@@ -1162,12 +1152,12 @@
"SFUNC " + shortFunctionName(fState) + " " +
"STYPE int ");
- assertInvalidMessage("does not exist or is not a scalar function",
+ assertInvalidMessage("doesn't exist",
"CREATE AGGREGATE " + KEYSPACE + ".aggInv(int) " +
"SFUNC " + shortFunctionName(a) + " " +
"STYPE int ");
- assertInvalidMessage("does not exist or is not a scalar function",
+ assertInvalidMessage("isn't a scalar function",
"CREATE AGGREGATE " + KEYSPACE + ".aggInv(int) " +
"SFUNC " + shortFunctionName(fState) + " " +
"STYPE int " +
@@ -1309,41 +1299,6 @@
}
@Test
- public void testBrokenAggregate() throws Throwable
- {
- createTable("CREATE TABLE %s (key int primary key, val int)");
- execute("INSERT INTO %s (key, val) VALUES (?, ?)", 1, 1);
-
- String fState = createFunction(KEYSPACE,
- "int, int",
- "CREATE FUNCTION %s(a int, b int) " +
- "CALLED ON NULL INPUT " +
- "RETURNS int " +
- "LANGUAGE javascript " +
- "AS 'a + b;'");
-
- String a = createAggregate(KEYSPACE,
- "int",
- "CREATE AGGREGATE %s(int) " +
- "SFUNC " + shortFunctionName(fState) + " " +
- "STYPE int ");
-
- KeyspaceMetadata ksm = Schema.instance.getKSMetaData(keyspace());
- UDAggregate f = (UDAggregate) ksm.functions.get(parseFunctionName(a)).iterator().next();
-
- UDAggregate broken = UDAggregate.createBroken(f.name(),
- f.argTypes(),
- f.returnType(),
- null,
- new InvalidRequestException("foo bar is broken"));
-
- Schema.instance.setKeyspaceMetadata(ksm.withSwapped(ksm.functions.without(f.name(), f.argTypes()).with(broken)));
-
- assertInvalidThrowMessage("foo bar is broken", InvalidRequestException.class,
- "SELECT " + a + "(val) FROM %s");
- }
-
- @Test
public void testWrongStateType() throws Throwable
{
createTable("CREATE TABLE %s (key int primary key, val int)");
@@ -1468,7 +1423,7 @@
"LANGUAGE java " +
"AS 'return state;'");
- assertInvalidMessage("The function state type should not be frozen",
+ assertInvalidMessage("cannot be frozen",
"CREATE AGGREGATE %s(set<int>) " +
"SFUNC " + parseFunctionName(fState).name + ' ' +
"STYPE frozen<set<int>> " +
@@ -1489,7 +1444,7 @@
assertRows(execute("SELECT " + aggregation + "(b) FROM %s"),
row(set(7, 8, 9)));
- assertInvalidMessage("The function arguments should not be frozen",
+ assertInvalidMessage("Argument 'frozen<set<int>>' cannot be frozen; remove frozen<> modifier from 'frozen<set<int>>'",
"DROP AGGREGATE %s (frozen<set<int>>);");
}
@@ -1520,7 +1475,7 @@
"LANGUAGE java " +
"AS 'return state;'");
- assertInvalidMessage("The function state type should not be frozen",
+ assertInvalidMessage("cannot be frozen",
"CREATE AGGREGATE %s(list<int>) " +
"SFUNC " + parseFunctionName(fState).name + ' ' +
"STYPE frozen<list<int>> " +
@@ -1538,7 +1493,7 @@
assertRows(execute("SELECT " + aggregation + "(b) FROM %s"),
row(list(7, 8, 9)));
- assertInvalidMessage("The function arguments should not be frozen",
+ assertInvalidMessage("Argument 'frozen<list<int>>' cannot be frozen; remove frozen<> modifier from 'frozen<list<int>>'",
"DROP AGGREGATE %s (frozen<list<int>>);");
}
@@ -1569,7 +1524,7 @@
"LANGUAGE java " +
"AS 'return state;'");
- assertInvalidMessage("The function state type should not be frozen",
+ assertInvalidMessage("cannot be frozen",
"CREATE AGGREGATE %s(map<int, int>) " +
"SFUNC " + parseFunctionName(fState).name + ' ' +
"STYPE frozen<map<int, int>> " +
@@ -1587,7 +1542,7 @@
assertRows(execute("SELECT " + aggregation + "(b) FROM %s"),
row(map(7, 8, 9, 10)));
- assertInvalidMessage("The function arguments should not be frozen",
+ assertInvalidMessage("Argument 'frozen<map<int, int>>' cannot be frozen; remove frozen<> modifier from 'frozen<map<int, int>>'",
"DROP AGGREGATE %s (frozen<map<int, int>>);");
}
@@ -1618,7 +1573,7 @@
"LANGUAGE java " +
"AS 'return state;'");
- assertInvalidMessage("The function state type should not be frozen",
+ assertInvalidMessage("cannot be frozen",
"CREATE AGGREGATE %s(tuple<int, int>) " +
"SFUNC " + parseFunctionName(fState).name + ' ' +
"STYPE frozen<tuple<int, int>> " +
@@ -1636,7 +1591,7 @@
assertRows(execute("SELECT " + aggregation + "(b) FROM %s"),
row(tuple(7, 8)));
- assertInvalidMessage("The function arguments should not be frozen",
+ assertInvalidMessage("Argument 'tuple<int, int>' cannot be frozen; remove frozen<> modifier from 'tuple<int, int>'",
"DROP AGGREGATE %s (frozen<tuple<int, int>>);");
}
@@ -1668,7 +1623,7 @@
"LANGUAGE java " +
"AS 'return state;'");
- assertInvalidMessage("The function state type should not be frozen",
+ assertInvalidMessage("cannot be frozen",
"CREATE AGGREGATE %s(" + myType + ") " +
"SFUNC " + parseFunctionName(fState).name + ' ' +
"STYPE frozen<" + myType + "> " +
@@ -1686,7 +1641,7 @@
assertRows(execute("SELECT " + aggregation + "(b).f FROM %s"),
row(7));
- assertInvalidMessage("The function arguments should not be frozen",
+ assertInvalidMessage(String.format("Argument 'frozen<%s>' cannot be frozen; remove frozen<> modifier from 'frozen<%s>'", myType, myType),
"DROP AGGREGATE %s (frozen<" + myType + ">);");
}
@@ -1869,6 +1824,16 @@
break;
}
}
+
+ ReconfigureOnChangeTask roct = (ReconfigureOnChangeTask) ctx.getObject(RECONFIGURE_ON_CHANGE_TASK);
+ if (roct != null)
+ {
+ // New functionality in logback - they replaced ReconfigureOnChangeFilter (which runs in the logging code)
+ // with an async ReconfigureOnChangeTask - i.e. in a thread that does not become sandboxed.
+ // Let the test run anyway, just we cannot reconfigure it (and it is pointless to reconfigure).
+ return;
+ }
+
assertTrue("ReconfigureOnChangeFilter not in logback's turbo-filter list - do that by adding scan=\"true\" to logback-test.xml's configuration element", done);
}
diff --git a/test/unit/org/apache/cassandra/cql3/validation/operations/AlterTest.java b/test/unit/org/apache/cassandra/cql3/validation/operations/AlterTest.java
index 238dd07..fc3a61e 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/operations/AlterTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/operations/AlterTest.java
@@ -17,29 +17,53 @@
*/
package org.apache.cassandra.cql3.validation.operations;
-import org.junit.Assert;
+import java.util.UUID;
import org.junit.Test;
-import org.apache.cassandra.config.SchemaConstants;
+import com.datastax.driver.core.PreparedStatement;
+
+import org.apache.cassandra.dht.OrderPreservingPartitioner;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.TokenMetadata;
+import org.apache.cassandra.schema.SchemaConstants;
import org.apache.cassandra.cql3.CQLTester;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Keyspace;
-import org.apache.cassandra.db.marshal.IntegerType;
import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.exceptions.InvalidRequestException;
import org.apache.cassandra.exceptions.SyntaxException;
import org.apache.cassandra.schema.SchemaKeyspace;
+import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.utils.FBUtilities;
import static java.lang.String.format;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
public class AlterTest extends CQLTester
{
@Test
+ public void testDropColumnAsPreparedStatement() throws Throwable
+ {
+ String table = createTable("CREATE TABLE %s (key int PRIMARY KEY, value int);");
+
+ PreparedStatement prepared = sessionNet().prepare("ALTER TABLE " + KEYSPACE + "." + table + " DROP value;");
+
+ executeNet("INSERT INTO %s (key, value) VALUES (1, 1)");
+ assertRowsNet(executeNet("SELECT * FROM %s"), row(1, 1));
+
+ sessionNet().execute(prepared.bind());
+
+ executeNet("ALTER TABLE %s ADD value int");
+
+ assertRows(execute("SELECT * FROM %s"), row(1, null));
+ }
+
+ @Test
public void testAddList() throws Throwable
{
createTable("CREATE TABLE %s (id text PRIMARY KEY, content text);");
- execute("ALTER TABLE %s ADD myCollection list<text>;");
+ alterTable("ALTER TABLE %s ADD myCollection list<text>;");
execute("INSERT INTO %s (id, content , myCollection) VALUES ('test', 'first test', ['first element']);");
assertRows(execute("SELECT * FROM %s;"), row("test", "first test", list("first element")));
@@ -50,7 +74,7 @@
{
createTable("CREATE TABLE %s (id text PRIMARY KEY, content text, myCollection list<text>);");
execute("INSERT INTO %s (id, content , myCollection) VALUES ('test', 'first test', ['first element']);");
- execute("ALTER TABLE %s DROP myCollection;");
+ alterTable("ALTER TABLE %s DROP myCollection;");
assertRows(execute("SELECT * FROM %s;"), row("test", "first test"));
}
@@ -59,7 +83,7 @@
public void testAddMap() throws Throwable
{
createTable("CREATE TABLE %s (id text PRIMARY KEY, content text);");
- execute("ALTER TABLE %s ADD myCollection map<text, text>;");
+ alterTable("ALTER TABLE %s ADD myCollection map<text, text>;");
execute("INSERT INTO %s (id, content , myCollection) VALUES ('test', 'first test', { '1' : 'first element'});");
assertRows(execute("SELECT * FROM %s;"), row("test", "first test", map("1", "first element")));
@@ -70,7 +94,7 @@
{
createTable("CREATE TABLE %s (id text PRIMARY KEY, content text, myCollection map<text, text>);");
execute("INSERT INTO %s (id, content , myCollection) VALUES ('test', 'first test', { '1' : 'first element'});");
- execute("ALTER TABLE %s DROP myCollection;");
+ alterTable("ALTER TABLE %s DROP myCollection;");
assertRows(execute("SELECT * FROM %s;"), row("test", "first test"));
}
@@ -80,9 +104,8 @@
{
createTable("CREATE TABLE %s (id text PRIMARY KEY, content text, myCollection list<text>);");
execute("INSERT INTO %s (id, content , myCollection) VALUES ('test', 'first test', ['first element']);");
- execute("ALTER TABLE %s DROP myCollection;");
- execute("ALTER TABLE %s ADD myCollection list<text>;");
-
+ alterTable("ALTER TABLE %s DROP myCollection;");
+ alterTable("ALTER TABLE %s ADD myCollection list<text>;");
assertRows(execute("SELECT * FROM %s;"), row("test", "first test", null));
execute("UPDATE %s set myCollection = ['second element'] WHERE id = 'test';");
assertRows(execute("SELECT * FROM %s;"), row("test", "first test", list("second element")));
@@ -93,7 +116,7 @@
{
createTable("CREATE TABLE %s (id text PRIMARY KEY, content text, myCollection list<text>);");
execute("INSERT INTO %s (id, content , myCollection) VALUES ('test', 'first test', ['first element']);");
- execute("ALTER TABLE %s DROP myCollection;");
+ alterTable("ALTER TABLE %s DROP myCollection;");
assertInvalid("ALTER TABLE %s ADD myCollection map<int, int>;");
}
@@ -108,8 +131,8 @@
// flush is necessary since otherwise the values of `todrop` will get discarded during
// alter statement
flush(true);
- execute("ALTER TABLE %s DROP todrop USING TIMESTAMP 20000;");
- execute("ALTER TABLE %s ADD todrop int;");
+ alterTable("ALTER TABLE %s DROP todrop USING TIMESTAMP 20000;");
+ alterTable("ALTER TABLE %s ADD todrop int;");
execute("INSERT INTO %s (id, c1, v1, todrop) VALUES (?, ?, ?, ?) USING TIMESTAMP ?", 1, 100, 100, 100, 30000L);
assertRows(execute("SELECT id, c1, v1, todrop FROM %s"),
row(1, 0, 0, null),
@@ -121,6 +144,21 @@
}
@Test
+ public void testDropAddWithDifferentKind() throws Throwable
+ {
+ createTable("CREATE TABLE %s (a int, b int, c int, d int static, PRIMARY KEY (a, b));");
+
+ alterTable("ALTER TABLE %s DROP c;");
+ alterTable("ALTER TABLE %s DROP d;");
+
+ assertInvalidMessage("Cannot re-add previously dropped column 'c' of kind STATIC, incompatible with previous kind REGULAR",
+ "ALTER TABLE %s ADD c int static;");
+
+ assertInvalidMessage("Cannot re-add previously dropped column 'd' of kind REGULAR, incompatible with previous kind STATIC",
+ "ALTER TABLE %s ADD d int;");
+ }
+
+ @Test
public void testDropStaticWithTimestamp() throws Throwable
{
createTable("CREATE TABLE %s (id int, c1 int, v1 int, todrop int static, PRIMARY KEY (id, c1));");
@@ -130,8 +168,8 @@
// flush is necessary since otherwise the values of `todrop` will get discarded during
// alter statement
flush(true);
- execute("ALTER TABLE %s DROP todrop USING TIMESTAMP 20000;");
- execute("ALTER TABLE %s ADD todrop int static;");
+ alterTable("ALTER TABLE %s DROP todrop USING TIMESTAMP 20000;");
+ alterTable("ALTER TABLE %s ADD todrop int static;");
execute("INSERT INTO %s (id, c1, v1, todrop) VALUES (?, ?, ?, ?) USING TIMESTAMP ?", 1, 100, 100, 100, 30000L);
// static column value with largest timestmap will be available again
assertRows(execute("SELECT id, c1, v1, todrop FROM %s"),
@@ -153,9 +191,9 @@
// flush is necessary since otherwise the values of `todrop1` and `todrop2` will get discarded during
// alter statement
flush(true);
- execute("ALTER TABLE %s DROP (todrop1, todrop2) USING TIMESTAMP 20000;");
- execute("ALTER TABLE %s ADD todrop1 int;");
- execute("ALTER TABLE %s ADD todrop2 int;");
+ alterTable("ALTER TABLE %s DROP (todrop1, todrop2) USING TIMESTAMP 20000;");
+ alterTable("ALTER TABLE %s ADD todrop1 int;");
+ alterTable("ALTER TABLE %s ADD todrop2 int;");
execute("INSERT INTO %s (id, c1, v1, todrop1, todrop2) VALUES (?, ?, ?, ?, ?) USING TIMESTAMP ?", 1, 100, 100, 100, 100, 40000L);
assertRows(execute("SELECT id, c1, v1, todrop1, todrop2 FROM %s"),
@@ -185,12 +223,12 @@
ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(tableName);
alterTable("ALTER TABLE %s WITH min_index_interval=256 AND max_index_interval=512");
- assertEquals(256, cfs.metadata.params.minIndexInterval);
- assertEquals(512, cfs.metadata.params.maxIndexInterval);
+ assertEquals(256, cfs.metadata().params.minIndexInterval);
+ assertEquals(512, cfs.metadata().params.maxIndexInterval);
alterTable("ALTER TABLE %s WITH caching = {}");
- assertEquals(256, cfs.metadata.params.minIndexInterval);
- assertEquals(512, cfs.metadata.params.maxIndexInterval);
+ assertEquals(256, cfs.metadata().params.minIndexInterval);
+ assertEquals(512, cfs.metadata().params.maxIndexInterval);
}
/**
@@ -211,13 +249,13 @@
row(ks1, true),
row(ks2, false));
- schemaChange("ALTER KEYSPACE " + ks1 + " WITH replication = { 'class' : 'NetworkTopologyStrategy', 'dc1' : 1 } AND durable_writes=False");
+ schemaChange("ALTER KEYSPACE " + ks1 + " WITH replication = { 'class' : 'NetworkTopologyStrategy', '" + DATA_CENTER + "' : 1 } AND durable_writes=False");
schemaChange("ALTER KEYSPACE " + ks2 + " WITH durable_writes=true");
assertRowsIgnoringOrderAndExtra(execute("SELECT keyspace_name, durable_writes, replication FROM system_schema.keyspaces"),
row(KEYSPACE, true, map("class", "org.apache.cassandra.locator.SimpleStrategy", "replication_factor", "1")),
row(KEYSPACE_PER_TEST, true, map("class", "org.apache.cassandra.locator.SimpleStrategy", "replication_factor", "1")),
- row(ks1, false, map("class", "org.apache.cassandra.locator.NetworkTopologyStrategy", "dc1", "1")),
+ row(ks1, false, map("class", "org.apache.cassandra.locator.NetworkTopologyStrategy", DATA_CENTER, "1")),
row(ks2, true, map("class", "org.apache.cassandra.locator.SimpleStrategy", "replication_factor", "1")));
execute("USE " + ks1);
@@ -232,22 +270,148 @@
}
@Test
+ public void testCreateAlterNetworkTopologyWithDefaults() throws Throwable
+ {
+ TokenMetadata metadata = StorageService.instance.getTokenMetadata();
+ metadata.clearUnsafe();
+ InetAddressAndPort local = FBUtilities.getBroadcastAddressAndPort();
+ InetAddressAndPort remote = InetAddressAndPort.getByName("127.0.0.4");
+ metadata.updateHostId(UUID.randomUUID(), local);
+ metadata.updateNormalToken(new OrderPreservingPartitioner.StringToken("A"), local);
+ metadata.updateHostId(UUID.randomUUID(), remote);
+ metadata.updateNormalToken(new OrderPreservingPartitioner.StringToken("B"), remote);
+
+ // With two datacenters we should respect anything passed in as a manual override
+ String ks1 = createKeyspace("CREATE KEYSPACE %s WITH replication={ 'class' : 'NetworkTopologyStrategy', 'replication_factor' : 1, '" + DATA_CENTER_REMOTE + "': 3}");
+
+ assertRowsIgnoringOrderAndExtra(execute("SELECT keyspace_name, durable_writes, replication FROM system_schema.keyspaces"),
+ row(KEYSPACE, true, map("class", "org.apache.cassandra.locator.SimpleStrategy", "replication_factor", "1")),
+ row(KEYSPACE_PER_TEST, true, map("class", "org.apache.cassandra.locator.SimpleStrategy", "replication_factor", "1")),
+ row(ks1, true, map("class", "org.apache.cassandra.locator.NetworkTopologyStrategy", DATA_CENTER, "1", DATA_CENTER_REMOTE, "3")));
+
+ // Should be able to remove data centers
+ schemaChange("ALTER KEYSPACE " + ks1 + " WITH replication = { 'class' : 'NetworkTopologyStrategy', '" + DATA_CENTER + "' : 0, '" + DATA_CENTER_REMOTE + "': 3 }");
+
+ assertRowsIgnoringOrderAndExtra(execute("SELECT keyspace_name, durable_writes, replication FROM system_schema.keyspaces"),
+ row(KEYSPACE, true, map("class", "org.apache.cassandra.locator.SimpleStrategy", "replication_factor", "1")),
+ row(KEYSPACE_PER_TEST, true, map("class", "org.apache.cassandra.locator.SimpleStrategy", "replication_factor", "1")),
+ row(ks1, true, map("class", "org.apache.cassandra.locator.NetworkTopologyStrategy", DATA_CENTER_REMOTE, "3")));
+
+ // The auto-expansion should not change existing replication counts; do not let the user shoot themselves in the foot
+ schemaChange("ALTER KEYSPACE " + ks1 + " WITH replication = { 'class' : 'NetworkTopologyStrategy', 'replication_factor' : 1 } AND durable_writes=True");
+
+ assertRowsIgnoringOrderAndExtra(execute("SELECT keyspace_name, durable_writes, replication FROM system_schema.keyspaces"),
+ row(KEYSPACE, true, map("class", "org.apache.cassandra.locator.SimpleStrategy", "replication_factor", "1")),
+ row(KEYSPACE_PER_TEST, true, map("class", "org.apache.cassandra.locator.SimpleStrategy", "replication_factor", "1")),
+ row(ks1, true, map("class", "org.apache.cassandra.locator.NetworkTopologyStrategy", DATA_CENTER, "1", DATA_CENTER_REMOTE, "3")));
+
+ // The keyspace should be fully functional
+ execute("USE " + ks1);
+
+ assertInvalidThrow(ConfigurationException.class, "CREATE TABLE tbl1 (a int PRIMARY KEY, b int) WITH compaction = { 'min_threshold' : 4 }");
+
+ execute("CREATE TABLE tbl1 (a int PRIMARY KEY, b int) WITH compaction = { 'class' : 'SizeTieredCompactionStrategy', 'min_threshold' : 7 }");
+
+ assertRows(execute("SELECT table_name, compaction FROM system_schema.tables WHERE keyspace_name='" + ks1 + "'"),
+ row("tbl1", map("class", "org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy",
+ "min_threshold", "7",
+ "max_threshold", "32")));
+ metadata.clearUnsafe();
+ }
+
+ @Test
+ public void testCreateSimpleAlterNTSDefaults() throws Throwable
+ {
+ TokenMetadata metadata = StorageService.instance.getTokenMetadata();
+ metadata.clearUnsafe();
+ InetAddressAndPort local = FBUtilities.getBroadcastAddressAndPort();
+ InetAddressAndPort remote = InetAddressAndPort.getByName("127.0.0.4");
+ metadata.updateHostId(UUID.randomUUID(), local);
+ metadata.updateNormalToken(new OrderPreservingPartitioner.StringToken("A"), local);
+ metadata.updateHostId(UUID.randomUUID(), remote);
+ metadata.updateNormalToken(new OrderPreservingPartitioner.StringToken("B"), remote);
+
+ // Let's create a keyspace first with SimpleStrategy
+ String ks1 = createKeyspace("CREATE KEYSPACE %s WITH replication={ 'class' : 'SimpleStrategy', 'replication_factor' : 2}");
+
+ assertRowsIgnoringOrderAndExtra(execute("SELECT keyspace_name, durable_writes, replication FROM system_schema.keyspaces"),
+ row(KEYSPACE, true, map("class", "org.apache.cassandra.locator.SimpleStrategy", "replication_factor", "1")),
+ row(KEYSPACE_PER_TEST, true, map("class", "org.apache.cassandra.locator.SimpleStrategy", "replication_factor", "1")),
+ row(ks1, true, map("class", "org.apache.cassandra.locator.SimpleStrategy", "replication_factor", "2")));
+
+ // Now we should be able to ALTER to NetworkTopologyStrategy directly from SimpleStrategy without supplying replication_factor
+ schemaChange("ALTER KEYSPACE " + ks1 + " WITH replication = { 'class' : 'NetworkTopologyStrategy'}");
+
+ assertRowsIgnoringOrderAndExtra(execute("SELECT keyspace_name, durable_writes, replication FROM system_schema.keyspaces"),
+ row(KEYSPACE, true, map("class", "org.apache.cassandra.locator.SimpleStrategy", "replication_factor", "1")),
+ row(KEYSPACE_PER_TEST, true, map("class", "org.apache.cassandra.locator.SimpleStrategy", "replication_factor", "1")),
+ row(ks1, true, map("class", "org.apache.cassandra.locator.NetworkTopologyStrategy", DATA_CENTER, "2", DATA_CENTER_REMOTE, "2")));
+
+ schemaChange("ALTER KEYSPACE " + ks1 + " WITH replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 3}");
+ schemaChange("ALTER KEYSPACE " + ks1 + " WITH replication = { 'class' : 'NetworkTopologyStrategy', 'replication_factor': 2}");
+
+ assertRowsIgnoringOrderAndExtra(execute("SELECT keyspace_name, durable_writes, replication FROM system_schema.keyspaces"),
+ row(KEYSPACE, true, map("class", "org.apache.cassandra.locator.SimpleStrategy", "replication_factor", "1")),
+ row(KEYSPACE_PER_TEST, true, map("class", "org.apache.cassandra.locator.SimpleStrategy", "replication_factor", "1")),
+ row(ks1, true, map("class", "org.apache.cassandra.locator.NetworkTopologyStrategy", DATA_CENTER, "2", DATA_CENTER_REMOTE, "2")));
+ }
+
+ /**
+ * Test {@link ConfigurationException} thrown on alter keyspace to no DC option in replication configuration.
+ */
+ @Test
+ public void testAlterKeyspaceWithNoOptionThrowsConfigurationException() throws Throwable
+ {
+ // Create keyspaces
+ execute("CREATE KEYSPACE testABC WITH replication={ 'class' : 'NetworkTopologyStrategy', '" + DATA_CENTER + "' : 3 }");
+ execute("CREATE KEYSPACE testXYZ WITH replication={ 'class' : 'SimpleStrategy', 'replication_factor' : 3 }");
+
+ // Try to alter the created keyspace without any option
+ assertInvalidThrow(ConfigurationException.class, "ALTER KEYSPACE testABC WITH replication={ 'class' : 'NetworkTopologyStrategy' }");
+ assertInvalidThrow(ConfigurationException.class, "ALTER KEYSPACE testXYZ WITH replication={ 'class' : 'SimpleStrategy' }");
+
+ // Make sure that the alter works as expected
+ alterTable("ALTER KEYSPACE testABC WITH replication={ 'class' : 'NetworkTopologyStrategy', '" + DATA_CENTER + "' : 2 }");
+ alterTable("ALTER KEYSPACE testXYZ WITH replication={ 'class' : 'SimpleStrategy', 'replication_factor' : 2 }");
+
+ // clean up
+ execute("DROP KEYSPACE IF EXISTS testABC");
+ execute("DROP KEYSPACE IF EXISTS testXYZ");
+ }
+
+ /**
+ * Test {@link ConfigurationException} thrown when altering a keyspace to invalid DC option in replication configuration.
+ */
+ @Test
+ public void testAlterKeyspaceWithNTSOnlyAcceptsConfiguredDataCenterNames() throws Throwable
+ {
+ // Create a keyspace with expected DC name.
+ createKeyspace("CREATE KEYSPACE %s WITH replication = {'class' : 'NetworkTopologyStrategy', '" + DATA_CENTER + "' : 2 }");
+
+ // try modifying the keyspace
+ assertAlterKeyspaceThrowsException(ConfigurationException.class,
+ "Unrecognized strategy option {INVALID_DC} passed to NetworkTopologyStrategy for keyspace " + currentKeyspace(),
+ "ALTER KEYSPACE %s WITH replication = { 'class' : 'NetworkTopologyStrategy', 'INVALID_DC' : 2 }");
+
+ alterKeyspace("ALTER KEYSPACE %s WITH replication = {'class' : 'NetworkTopologyStrategy', '" + DATA_CENTER + "' : 3 }");
+
+ // Mix valid and invalid, should throw an exception
+ assertAlterKeyspaceThrowsException(ConfigurationException.class,
+ "Unrecognized strategy option {INVALID_DC} passed to NetworkTopologyStrategy for keyspace " + currentKeyspace(),
+ "ALTER KEYSPACE %s WITH replication={ 'class' : 'NetworkTopologyStrategy', '" + DATA_CENTER + "' : 2 , 'INVALID_DC': 1}");
+ }
+
+ @Test
public void testAlterKeyspaceWithMultipleInstancesOfSameDCThrowsSyntaxException() throws Throwable
{
- try
- {
- // Create a keyspace
- execute("CREATE KEYSPACE testABC WITH replication = {'class' : 'NetworkTopologyStrategy', 'dc1' : 2}");
+ // Create a keyspace
+ createKeyspace("CREATE KEYSPACE %s WITH replication = {'class' : 'NetworkTopologyStrategy', '" + DATA_CENTER + "' : 2}");
- // try modifying the keyspace
- assertInvalidThrow(SyntaxException.class, "ALTER KEYSPACE testABC WITH replication = {'class' : 'NetworkTopologyStrategy', 'dc1' : 2, 'dc1' : 3 }");
- execute("ALTER KEYSPACE testABC WITH replication = {'class' : 'NetworkTopologyStrategy', 'dc1' : 3}");
- }
- finally
- {
- // clean-up
- execute("DROP KEYSPACE IF EXISTS testABC");
- }
+ // try modifying the keyspace
+ assertAlterTableThrowsException(SyntaxException.class,
+ "",
+ "ALTER KEYSPACE %s WITH replication = {'class' : 'NetworkTopologyStrategy', '" + DATA_CENTER + "' : 2, '" + DATA_CENTER + "' : 3 }");
+ alterKeyspace("ALTER KEYSPACE %s WITH replication = {'class' : 'NetworkTopologyStrategy', '" + DATA_CENTER + "' : 3}");
}
/**
@@ -261,99 +425,27 @@
execute("UPDATE %s SET t = '111' WHERE id = 1");
- execute("ALTER TABLE %s ADD l list<text>");
+ alterTable("ALTER TABLE %s ADD l list<text>");
assertRows(execute("SELECT * FROM %s"),
row(1, null, "111"));
- execute("ALTER TABLE %s ADD m map<int, text>");
+ alterTable("ALTER TABLE %s ADD m map<int, text>");
assertRows(execute("SELECT * FROM %s"),
row(1, null, null, "111"));
}
- @Test(expected = InvalidRequestException.class)
- public void testDropComplexAddSimpleColumn() throws Throwable
+ /**
+ * Test for 7744,
+ * migrated from cql_tests.py:TestCQL.downgrade_to_compact_bug_test()
+ */
+ @Test
+ public void testDowngradeToCompact() throws Throwable
{
createTable("create table %s (k int primary key, v set<text>)");
- execute("alter table %s drop v");
- execute("alter table %s add v text");
- }
-
- @Test(expected = InvalidRequestException.class)
- public void testDropSimpleAddComplexColumn() throws Throwable
- {
- createTable("create table %s (k int primary key, v text)");
- execute("alter table %s drop v");
- execute("alter table %s add v set<text>");
- }
-
- @Test(expected = InvalidRequestException.class)
- public void testDropMultiCellAddFrozenColumn() throws Throwable
- {
- createTable("create table %s (k int primary key, v set<text>)");
- execute("alter table %s drop v");
- execute("alter table %s add v frozen<set<text>>");
- }
-
- @Test(expected = InvalidRequestException.class)
- public void testDropFrozenAddMultiCellColumn() throws Throwable
- {
- createTable("create table %s (k int primary key, v frozen<set<text>>)");
- execute("alter table %s drop v");
- execute("alter table %s add v set<text>");
- }
-
- @Test
- public void testDropTimeUUIDAddUUIDColumn() throws Throwable
- {
- createTable("create table %s (k int primary key, v timeuuid)");
- execute("alter table %s drop v");
- execute("alter table %s add v uuid");
- }
-
- @Test(expected = InvalidRequestException.class)
- public void testDropUUIDAddTimeUUIDColumn() throws Throwable
- {
- createTable("create table %s (k int primary key, v uuid)");
- execute("alter table %s drop v");
- execute("alter table %s add v timeuuid");
- }
-
- @Test
- public void testDropAddSameType() throws Throwable
- {
- createTable("create table %s (k int primary key, v1 timeuuid, v2 set<uuid>, v3 frozen<list<text>>)");
-
- execute("alter table %s drop v1");
- execute("alter table %s add v1 timeuuid");
-
- execute("alter table %s drop v2");
- execute("alter table %s add v2 set<uuid>");
-
- execute("alter table %s drop v3");
- execute("alter table %s add v3 frozen<list<text>>");
- }
-
- @Test(expected = InvalidRequestException.class)
- public void testDropRegularAddStatic() throws Throwable
- {
- createTable("create table %s (k int, c int, v uuid, PRIMARY KEY (k, c))");
- execute("alter table %s drop v");
- execute("alter table %s add v uuid static");
- }
-
- @Test(expected = InvalidRequestException.class)
- public void testDropStaticAddRegular() throws Throwable
- {
- createTable("create table %s (k int, c int, v uuid static, PRIMARY KEY (k, c))");
- execute("alter table %s drop v");
- execute("alter table %s add v uuid");
- }
-
- @Test(expected = SyntaxException.class)
- public void renameToEmptyTest() throws Throwable
- {
- createTable("CREATE TABLE %s (k int, c1 int, v int, PRIMARY KEY (k, c1))");
- execute("ALTER TABLE %s RENAME c1 TO \"\"");
+ execute("insert into %s (k, v) VALUES (0, {'f'})");
+ flush();
+ alterTable("alter table %s drop v");
+ alterTable("alter table %s add v1 int");
}
@Test
@@ -364,7 +456,7 @@
"ALTER KEYSPACE ks WITH WITH DURABLE_WRITES = true" };
for (String stmt : stmts) {
- assertInvalidSyntaxMessage("no viable alternative at input 'WITH'", stmt);
+ assertAlterTableThrowsException(SyntaxException.class, "no viable alternative at input 'WITH'", stmt);
}
}
@@ -378,9 +470,9 @@
SchemaKeyspace.TABLES),
KEYSPACE,
currentTable()),
- row(map("chunk_length_in_kb", "64", "class", "org.apache.cassandra.io.compress.LZ4Compressor")));
+ row(map("chunk_length_in_kb", "16", "class", "org.apache.cassandra.io.compress.LZ4Compressor")));
- execute("ALTER TABLE %s WITH compression = { 'class' : 'SnappyCompressor', 'chunk_length_in_kb' : 32 };");
+ alterTable("ALTER TABLE %s WITH compression = { 'class' : 'SnappyCompressor', 'chunk_length_in_kb' : 32 };");
assertRows(execute(format("SELECT compression FROM %s.%s WHERE keyspace_name = ? and table_name = ?;",
SchemaConstants.SCHEMA_KEYSPACE_NAME,
@@ -389,7 +481,7 @@
currentTable()),
row(map("chunk_length_in_kb", "32", "class", "org.apache.cassandra.io.compress.SnappyCompressor")));
- execute("ALTER TABLE %s WITH compression = { 'sstable_compression' : 'LZ4Compressor', 'chunk_length_kb' : 64 };");
+ alterTable("ALTER TABLE %s WITH compression = { 'class' : 'LZ4Compressor', 'chunk_length_in_kb' : 64 };");
assertRows(execute(format("SELECT compression FROM %s.%s WHERE keyspace_name = ? and table_name = ?;",
SchemaConstants.SCHEMA_KEYSPACE_NAME,
@@ -398,7 +490,35 @@
currentTable()),
row(map("chunk_length_in_kb", "64", "class", "org.apache.cassandra.io.compress.LZ4Compressor")));
- execute("ALTER TABLE %s WITH compression = { 'sstable_compression' : '', 'chunk_length_kb' : 32 };");
+ alterTable("ALTER TABLE %s WITH compression = { 'class' : 'LZ4Compressor', 'min_compress_ratio' : 2 };");
+
+ assertRows(execute(format("SELECT compression FROM %s.%s WHERE keyspace_name = ? and table_name = ?;",
+ SchemaConstants.SCHEMA_KEYSPACE_NAME,
+ SchemaKeyspace.TABLES),
+ KEYSPACE,
+ currentTable()),
+ row(map("chunk_length_in_kb", "16", "class", "org.apache.cassandra.io.compress.LZ4Compressor", "min_compress_ratio", "2.0")));
+
+ alterTable("ALTER TABLE %s WITH compression = { 'class' : 'LZ4Compressor', 'min_compress_ratio' : 1 };");
+
+ assertRows(execute(format("SELECT compression FROM %s.%s WHERE keyspace_name = ? and table_name = ?;",
+ SchemaConstants.SCHEMA_KEYSPACE_NAME,
+ SchemaKeyspace.TABLES),
+ KEYSPACE,
+ currentTable()),
+ row(map("chunk_length_in_kb", "16", "class", "org.apache.cassandra.io.compress.LZ4Compressor", "min_compress_ratio", "1.0")));
+
+ alterTable("ALTER TABLE %s WITH compression = { 'class' : 'LZ4Compressor', 'min_compress_ratio' : 0 };");
+
+ assertRows(execute(format("SELECT compression FROM %s.%s WHERE keyspace_name = ? and table_name = ?;",
+ SchemaConstants.SCHEMA_KEYSPACE_NAME,
+ SchemaKeyspace.TABLES),
+ KEYSPACE,
+ currentTable()),
+ row(map("chunk_length_in_kb", "16", "class", "org.apache.cassandra.io.compress.LZ4Compressor")));
+
+ alterTable("ALTER TABLE %s WITH compression = { 'class' : 'SnappyCompressor', 'chunk_length_in_kb' : 32 };");
+ alterTable("ALTER TABLE %s WITH compression = { 'enabled' : 'false'};");
assertRows(execute(format("SELECT compression FROM %s.%s WHERE keyspace_name = ? and table_name = ?;",
SchemaConstants.SCHEMA_KEYSPACE_NAME,
@@ -407,42 +527,57 @@
currentTable()),
row(map("enabled", "false")));
- execute("ALTER TABLE %s WITH compression = { 'class' : 'SnappyCompressor', 'chunk_length_in_kb' : 32 };");
- execute("ALTER TABLE %s WITH compression = { 'enabled' : 'false'};");
+ assertAlterTableThrowsException(ConfigurationException.class,
+ "Missing sub-option 'class' for the 'compression' option.",
+ "ALTER TABLE %s WITH compression = {'chunk_length_in_kb' : 32};");
- assertRows(execute(format("SELECT compression FROM %s.%s WHERE keyspace_name = ? and table_name = ?;",
- SchemaConstants.SCHEMA_KEYSPACE_NAME,
- SchemaKeyspace.TABLES),
- KEYSPACE,
- currentTable()),
- row(map("enabled", "false")));
+ assertAlterTableThrowsException(ConfigurationException.class,
+ "The 'class' option must not be empty. To disable compression use 'enabled' : false",
+ "ALTER TABLE %s WITH compression = { 'class' : ''};");
- assertThrowsConfigurationException("Missing sub-option 'class' for the 'compression' option.",
- "ALTER TABLE %s WITH compression = {'chunk_length_in_kb' : 32};");
+ assertAlterTableThrowsException(ConfigurationException.class,
+ "If the 'enabled' option is set to false no other options must be specified",
+ "ALTER TABLE %s WITH compression = { 'enabled' : 'false', 'class' : 'SnappyCompressor'};");
- assertThrowsConfigurationException("The 'class' option must not be empty. To disable compression use 'enabled' : false",
- "ALTER TABLE %s WITH compression = { 'class' : ''};");
+ assertAlterTableThrowsException(ConfigurationException.class,
+ "The 'sstable_compression' option must not be used if the compression algorithm is already specified by the 'class' option",
+ "ALTER TABLE %s WITH compression = { 'sstable_compression' : 'SnappyCompressor', 'class' : 'SnappyCompressor'};");
- assertThrowsConfigurationException("If the 'enabled' option is set to false no other options must be specified",
- "ALTER TABLE %s WITH compression = { 'enabled' : 'false', 'class' : 'SnappyCompressor'};");
+ assertAlterTableThrowsException(ConfigurationException.class,
+ "The 'chunk_length_kb' option must not be used if the chunk length is already specified by the 'chunk_length_in_kb' option",
+ "ALTER TABLE %s WITH compression = { 'class' : 'SnappyCompressor', 'chunk_length_kb' : 32 , 'chunk_length_in_kb' : 32 };");
- assertThrowsConfigurationException("The 'sstable_compression' option must not be used if the compression algorithm is already specified by the 'class' option",
- "ALTER TABLE %s WITH compression = { 'sstable_compression' : 'SnappyCompressor', 'class' : 'SnappyCompressor'};");
+ assertAlterTableThrowsException(ConfigurationException.class,
+ "Invalid negative min_compress_ratio",
+ "ALTER TABLE %s WITH compression = { 'class' : 'SnappyCompressor', 'min_compress_ratio' : -1 };");
- assertThrowsConfigurationException("The 'chunk_length_kb' option must not be used if the chunk length is already specified by the 'chunk_length_in_kb' option",
- "ALTER TABLE %s WITH compression = { 'class' : 'SnappyCompressor', 'chunk_length_kb' : 32 , 'chunk_length_in_kb' : 32 };");
+ assertAlterTableThrowsException(ConfigurationException.class,
+ "min_compress_ratio can either be 0 or greater than or equal to 1",
+ "ALTER TABLE %s WITH compression = { 'class' : 'SnappyCompressor', 'min_compress_ratio' : 0.5 };");
}
- private void assertThrowsConfigurationException(String errorMsg, String alterStmt) throws Throwable
+ private void assertAlterKeyspaceThrowsException(Class<? extends Throwable> clazz, String msg, String stmt)
+ {
+ assertThrowsException(clazz, msg, () -> {alterKeyspaceMayThrow(stmt);});
+ }
+
+ private void assertAlterTableThrowsException(Class<? extends Throwable> clazz, String msg, String stmt)
+ {
+ assertThrowsException(clazz, msg, () -> {alterTableMayThrow(stmt);});
+ }
+
+ private static void assertThrowsException(Class<? extends Throwable> clazz, String msg, CheckedFunction function)
{
try
{
- execute(alterStmt);
- Assert.fail("Query should be invalid but no error was thrown. Query is: " + alterStmt);
+ function.apply();
+ fail("An error should havee been thrown but was not.");
}
- catch (ConfigurationException e)
+ catch (Throwable e)
{
- assertEquals(errorMsg, e.getMessage());
+ assertTrue("Unexpected exception type (expected: " + clazz + ", value: " + e.getClass() + ")",
+ clazz.isAssignableFrom(e.getClass()));
+ assertTrue("Expecting the error message to contains: '" + msg + "' but was " + e.getMessage(), e.getMessage().contains(msg));
}
}
@@ -463,7 +598,7 @@
flush();
- execute("ALTER TABLE %s DROP x");
+ alterTable("ALTER TABLE %s DROP x");
compact();
@@ -492,59 +627,8 @@
if (flushAfterInsert)
flush();
- execute("ALTER TABLE %s DROP x");
+ alterTable("ALTER TABLE %s DROP x");
assertEmpty(execute("SELECT * FROM %s"));
}
-
- /**
- * Test for CASSANDRA-13917
- */
- @Test
- public void testAlterWithCompactStaticFormat() throws Throwable
- {
- createTable("CREATE TABLE %s (a int PRIMARY KEY, b int, c int) WITH COMPACT STORAGE");
-
- assertInvalidMessage("Cannot rename unknown column column1 in keyspace",
- "ALTER TABLE %s RENAME column1 TO column2");
-
- assertInvalidMessage("Cannot rename unknown column value in keyspace",
- "ALTER TABLE %s RENAME value TO value2");
- }
-
- /**
- * Test for CASSANDRA-13917
- */
- @Test
- public void testAlterWithCompactNonStaticFormat() throws Throwable
- {
- createTable("CREATE TABLE %s (a int, b int, PRIMARY KEY (a, b)) WITH COMPACT STORAGE");
- assertInvalidMessage("Cannot rename unknown column column1 in keyspace",
- "ALTER TABLE %s RENAME column1 TO column2");
-
- createTable("CREATE TABLE %s (a int, b int, v int, PRIMARY KEY (a, b)) WITH COMPACT STORAGE");
- assertInvalidMessage("Cannot rename unknown column column1 in keyspace",
- "ALTER TABLE %s RENAME column1 TO column2");
- }
-
- @Test
- public void testAlterTableAlterType() throws Throwable
- {
- createTable("CREATE TABLE %s (a int, b int, PRIMARY KEY (a,b)) WITH COMPACT STORAGE");
- assertInvalidMessage(String.format("Compact value type can only be changed to BytesType, but %s was given.",
- IntegerType.instance),
- "ALTER TABLE %s ALTER value TYPE 'org.apache.cassandra.db.marshal.IntegerType'");
-
- execute("ALTER TABLE %s ALTER value TYPE 'org.apache.cassandra.db.marshal.BytesType'");
-
- createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY (a,b)) WITH COMPACT STORAGE");
- assertInvalidMessage("Altering of types is not allowed",
- "ALTER TABLE %s ALTER c TYPE 'org.apache.cassandra.db.marshal.BytesType'");
-
- createTable("CREATE TABLE %s (a int, value int, PRIMARY KEY (a,value)) WITH COMPACT STORAGE");
- assertInvalidMessage("Altering of types is not allowed",
- "ALTER TABLE %s ALTER value TYPE 'org.apache.cassandra.db.marshal.IntegerType'");
- execute("ALTER TABLE %s ALTER value1 TYPE 'org.apache.cassandra.db.marshal.BytesType'");
- }
-
}
diff --git a/test/unit/org/apache/cassandra/cql3/validation/operations/BatchTest.java b/test/unit/org/apache/cassandra/cql3/validation/operations/BatchTest.java
index d0bdd15..bc220fd 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/operations/BatchTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/operations/BatchTest.java
@@ -104,39 +104,6 @@
}
@Test
- public void testBatchRangeDelete() throws Throwable
- {
- createTable("CREATE TABLE %s (partitionKey int," +
- "clustering int," +
- "value int," +
- " PRIMARY KEY (partitionKey, clustering)) WITH COMPACT STORAGE");
-
- int value = 0;
- for (int partitionKey = 0; partitionKey < 4; partitionKey++)
- for (int clustering1 = 0; clustering1 < 5; clustering1++)
- execute("INSERT INTO %s (partitionKey, clustering, value) VALUES (?, ?, ?)",
- partitionKey, clustering1, value++);
-
- execute("BEGIN BATCH " +
- "DELETE FROM %1$s WHERE partitionKey = 1;" +
- "DELETE FROM %1$s WHERE partitionKey = 0 AND clustering >= 4;" +
- "DELETE FROM %1$s WHERE partitionKey = 0 AND clustering <= 0;" +
- "DELETE FROM %1$s WHERE partitionKey = 2 AND clustering >= 0 AND clustering <= 3;" +
- "DELETE FROM %1$s WHERE partitionKey = 2 AND clustering <= 3 AND clustering >= 4;" +
- "DELETE FROM %1$s WHERE partitionKey = 3 AND (clustering) >= (3) AND (clustering) <= (6);" +
- "APPLY BATCH;");
-
- assertRows(execute("SELECT * FROM %s"),
- row(0, 1, 1),
- row(0, 2, 2),
- row(0, 3, 3),
- row(2, 4, 14),
- row(3, 0, 15),
- row(3, 1, 16),
- row(3, 2, 17));
- }
-
- @Test
public void testBatchUpdate() throws Throwable
{
createTable("CREATE TABLE %s (partitionKey int," +
@@ -201,6 +168,26 @@
}
@Test
+ public void testBatchMultipleTablePrepare() throws Throwable
+ {
+ String tbl1 = KEYSPACE + "." + createTableName();
+ String tbl2 = KEYSPACE + "." + createTableName();
+
+ schemaChange(String.format("CREATE TABLE %s (k1 int PRIMARY KEY, v1 int)", tbl1));
+ schemaChange(String.format("CREATE TABLE %s (k2 int PRIMARY KEY, v2 int)", tbl2));
+
+ String query = "BEGIN BATCH " +
+ String.format("UPDATE %s SET v1 = 1 WHERE k1 = ?;", tbl1) +
+ String.format("UPDATE %s SET v2 = 2 WHERE k2 = ?;", tbl2) +
+ "APPLY BATCH;";
+ prepare(query);
+ execute(query, 0, 1);
+
+ assertRows(execute(String.format("SELECT * FROM %s", tbl1)), row(0, 1));
+ assertRows(execute(String.format("SELECT * FROM %s", tbl2)), row(1, 2));
+ }
+
+ @Test
public void testBatchWithInRestriction() throws Throwable
{
createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY (a,b))");
@@ -241,7 +228,7 @@
}
@Test
- public void testBatchAndConditionalInteraction() throws Throwable
+ public void testBatchTTLConditionalInteraction() throws Throwable
{
createTable(String.format("CREATE TABLE %s.clustering (\n" +
@@ -256,10 +243,16 @@
execute("DELETE FROM " + KEYSPACE +".clustering WHERE id=1");
String clusteringInsert = "INSERT INTO " + KEYSPACE + ".clustering(id, clustering1, clustering2, clustering3, val) VALUES(%s, %s, %s, %s, %s); ";
+ String clusteringTTLInsert = "INSERT INTO " + KEYSPACE + ".clustering(id, clustering1, clustering2, clustering3, val) VALUES(%s, %s, %s, %s, %s) USING TTL %s; ";
+ String clusteringConditionalInsert = "INSERT INTO " + KEYSPACE + ".clustering(id, clustering1, clustering2, clustering3, val) VALUES(%s, %s, %s, %s, %s) IF NOT EXISTS; ";
+ String clusteringConditionalTTLInsert = "INSERT INTO " + KEYSPACE + ".clustering(id, clustering1, clustering2, clustering3, val) VALUES(%s, %s, %s, %s, %s) IF NOT EXISTS USING TTL %s; ";
String clusteringUpdate = "UPDATE " + KEYSPACE + ".clustering SET val=%s WHERE id=%s AND clustering1=%s AND clustering2=%s AND clustering3=%s ;";
+ String clusteringTTLUpdate = "UPDATE " + KEYSPACE + ".clustering USING TTL %s SET val=%s WHERE id=%s AND clustering1=%s AND clustering2=%s AND clustering3=%s ;";
String clusteringConditionalUpdate = "UPDATE " + KEYSPACE + ".clustering SET val=%s WHERE id=%s AND clustering1=%s AND clustering2=%s AND clustering3=%s IF val=%s ;";
+ String clusteringConditionalTTLUpdate = "UPDATE " + KEYSPACE + ".clustering USING TTL %s SET val=%s WHERE id=%s AND clustering1=%s AND clustering2=%s AND clustering3=%s IF val=%s ;";
String clusteringDelete = "DELETE FROM " + KEYSPACE + ".clustering WHERE id=%s AND clustering1=%s AND clustering2=%s AND clustering3=%s ;";
String clusteringRangeDelete = "DELETE FROM " + KEYSPACE + ".clustering WHERE id=%s AND clustering1=%s ;";
+ String clusteringConditionalDelete = "DELETE FROM " + KEYSPACE + ".clustering WHERE id=%s AND clustering1=%s AND clustering2=%s AND clustering3=%s IF val=%s ; ";
execute("BEGIN BATCH " + String.format(clusteringInsert, 1, 1, 1, 1, 1) + " APPLY BATCH");
@@ -300,7 +293,6 @@
cmd4.append("APPLY BATCH ");
execute(cmd4.toString());
- System.out.println(execute("SELECT * FROM " + KEYSPACE+".clustering WHERE id=1"));
assertRows(execute("SELECT * FROM " + KEYSPACE+".clustering WHERE id=1"),
row(1, 1, 1, 2, 234),
row(1, 1, 2, 3, 23),
@@ -314,7 +306,6 @@
cmd5.append("APPLY BATCH ");
execute(cmd5.toString());
- System.out.println(execute("SELECT * FROM " + KEYSPACE+".clustering WHERE id=1"));
assertRows(execute("SELECT * FROM " + KEYSPACE+".clustering WHERE id=1"),
row(1, 1, 1, 2, 1234),
row(1, 1, 2, 3, 23)
@@ -327,7 +318,6 @@
cmd6.append("APPLY BATCH ");
execute(cmd6.toString());
- System.out.println(execute("SELECT * FROM " + KEYSPACE+".clustering WHERE id=1"));
assertRows(execute("SELECT * FROM " + KEYSPACE+".clustering WHERE id=1"),
row(1, 1, 1, 2, 1),
row(1, 1, 2, 3, 23),
@@ -342,12 +332,319 @@
cmd7.append("APPLY BATCH ");
execute(cmd7.toString());
- System.out.println(execute("SELECT * FROM " + KEYSPACE+".clustering WHERE id=1"));
assertRows(execute("SELECT * FROM " + KEYSPACE+".clustering WHERE id=1"),
row(1, 1, 1, 2, 1),
row(1, 1, 2, 3, 23),
row(1, 3, 4, 5, 345)
);
+
+ StringBuilder cmd8 = new StringBuilder();
+ cmd8.append("BEGIN BATCH ");
+ cmd8.append(String.format(clusteringConditionalDelete, 1, 3, 4, 5, 345));
+ cmd8.append(String.format(clusteringRangeDelete, 1, 1));
+ cmd8.append(String.format(clusteringInsert, 1, 2, 3, 4, 5));
+ cmd8.append("APPLY BATCH ");
+ execute(cmd8.toString());
+
+ assertRows(execute("SELECT * FROM " + KEYSPACE+".clustering WHERE id=1"),
+ row(1, 2, 3, 4, 5)
+ );
+
+ StringBuilder cmd9 = new StringBuilder();
+ cmd9.append("BEGIN BATCH ");
+ cmd9.append(String.format(clusteringConditionalInsert, 1, 3, 4, 5, 345));
+ cmd9.append(String.format(clusteringDelete, 1, 2, 3, 4));
+ cmd9.append("APPLY BATCH ");
+ execute(cmd9.toString());
+
+ assertRows(execute("SELECT * FROM " + KEYSPACE+".clustering WHERE id=1"),
+ row(1, 3, 4, 5, 345)
+ );
+
+ StringBuilder cmd10 = new StringBuilder();
+ cmd10.append("BEGIN BATCH ");
+ cmd10.append(String.format(clusteringTTLInsert, 1, 2, 3, 4, 5, 5));
+ cmd10.append(String.format(clusteringConditionalTTLUpdate, 10, 5, 1, 3, 4, 5, 345));
+ cmd10.append("APPLY BATCH ");
+ execute(cmd10.toString());
+
+ assertRows(execute("SELECT * FROM " + KEYSPACE+".clustering WHERE id=1"),
+ row(1, 2, 3, 4, 5), // 5 second TTL
+ row(1, 3, 4, 5, 5) // 10 second TTL
+ );
+
+ Thread.sleep(6000);
+
+ assertRows(execute("SELECT * FROM " + KEYSPACE+".clustering WHERE id=1"),
+ row(1, 3, 4, 5, 5) // now 4 second TTL
+ );
+
+ StringBuilder cmd11 = new StringBuilder();
+ cmd11.append("BEGIN BATCH ");
+ cmd11.append(String.format(clusteringConditionalTTLInsert, 1, 2, 3, 4, 5, 5));
+ cmd11.append(String.format(clusteringInsert,1, 4, 5, 6, 7));
+ cmd11.append("APPLY BATCH ");
+ execute(cmd11.toString());
+
+ assertRows(execute("SELECT * FROM " + KEYSPACE+".clustering WHERE id=1"),
+ row(1, 2, 3, 4, 5), // This one has 5 seconds left
+ row(1, 3, 4, 5, 5), // This one should have 4 seconds left
+ row(1, 4, 5, 6, 7) // This one has no TTL
+ );
+
+ Thread.sleep(6000);
+
+ assertRows(execute("SELECT * FROM " + KEYSPACE+".clustering WHERE id=1"),
+ row(1, 3, 4, 5, null), // We had a row here before from cmd9, but we've ttl'd out the value in cmd11
+ row(1, 4, 5, 6, 7)
+ );
+
+ StringBuilder cmd12 = new StringBuilder();
+ cmd12.append("BEGIN BATCH ");
+ cmd12.append(String.format(clusteringConditionalTTLUpdate, 5, 5, 1, 3, 4, 5, null));
+ cmd12.append(String.format(clusteringTTLUpdate, 5, 8, 1, 4, 5, 6));
+ cmd12.append("APPLY BATCH ");
+ execute(cmd12.toString());
+
+ assertRows(execute("SELECT * FROM " + KEYSPACE+".clustering WHERE id=1"),
+ row(1, 3, 4, 5, 5),
+ row(1, 4, 5, 6, 8)
+ );
+
+ Thread.sleep(6000);
+ assertRows(execute("SELECT * FROM " + KEYSPACE+".clustering WHERE id=1"),
+ row(1, 3, 4, 5, null),
+ row(1, 4, 5, 6, null)
+ );
+ }
+
+
+ @Test
+ public void testBatchStaticTTLConditionalInteraction() throws Throwable
+ {
+
+ createTable(String.format("CREATE TABLE %s.clustering_static (\n" +
+ " id int,\n" +
+ " clustering1 int,\n" +
+ " clustering2 int,\n" +
+ " clustering3 int,\n" +
+ " sval int static, \n" +
+ " val int, \n" +
+ " PRIMARY KEY(id, clustering1, clustering2, clustering3)" +
+ ")", KEYSPACE));
+
+ execute("DELETE FROM " + KEYSPACE +".clustering_static WHERE id=1");
+
+ String clusteringInsert = "INSERT INTO " + KEYSPACE + ".clustering_static(id, clustering1, clustering2, clustering3, val) VALUES(%s, %s, %s, %s, %s); ";
+ String clusteringTTLInsert = "INSERT INTO " + KEYSPACE + ".clustering_static(id, clustering1, clustering2, clustering3, val) VALUES(%s, %s, %s, %s, %s) USING TTL %s; ";
+ String clusteringStaticInsert = "INSERT INTO " + KEYSPACE + ".clustering_static(id, clustering1, clustering2, clustering3, sval, val) VALUES(%s, %s, %s, %s, %s, %s); ";
+ String clusteringConditionalInsert = "INSERT INTO " + KEYSPACE + ".clustering_static(id, clustering1, clustering2, clustering3, val) VALUES(%s, %s, %s, %s, %s) IF NOT EXISTS; ";
+ String clusteringConditionalTTLInsert = "INSERT INTO " + KEYSPACE + ".clustering_static(id, clustering1, clustering2, clustering3, val) VALUES(%s, %s, %s, %s, %s) IF NOT EXISTS USING TTL %s; ";
+ String clusteringUpdate = "UPDATE " + KEYSPACE + ".clustering_static SET val=%s WHERE id=%s AND clustering1=%s AND clustering2=%s AND clustering3=%s ;";
+ String clusteringStaticUpdate = "UPDATE " + KEYSPACE + ".clustering_static SET sval=%s WHERE id=%s ;";
+ String clusteringTTLUpdate = "UPDATE " + KEYSPACE + ".clustering_static USING TTL %s SET val=%s WHERE id=%s AND clustering1=%s AND clustering2=%s AND clustering3=%s ;";
+ String clusteringStaticConditionalUpdate = "UPDATE " + KEYSPACE + ".clustering_static SET val=%s WHERE id=%s AND clustering1=%s AND clustering2=%s AND clustering3=%s IF sval=%s ;";
+ String clusteringConditionalTTLUpdate = "UPDATE " + KEYSPACE + ".clustering_static USING TTL %s SET val=%s WHERE id=%s AND clustering1=%s AND clustering2=%s AND clustering3=%s IF val=%s ;";
+ String clusteringStaticConditionalTTLUpdate = "UPDATE " + KEYSPACE + ".clustering_static USING TTL %s SET val=%s WHERE id=%s AND clustering1=%s AND clustering2=%s AND clustering3=%s IF sval=%s ;";
+ String clusteringStaticConditionalStaticUpdate = "UPDATE " + KEYSPACE +".clustering_static SET sval=%s WHERE id=%s IF sval=%s; ";
+ String clusteringDelete = "DELETE FROM " + KEYSPACE + ".clustering_static WHERE id=%s AND clustering1=%s AND clustering2=%s AND clustering3=%s ;";
+ String clusteringRangeDelete = "DELETE FROM " + KEYSPACE + ".clustering_static WHERE id=%s AND clustering1=%s ;";
+ String clusteringConditionalDelete = "DELETE FROM " + KEYSPACE + ".clustering_static WHERE id=%s AND clustering1=%s AND clustering2=%s AND clustering3=%s IF val=%s ; ";
+ String clusteringStaticConditionalDelete = "DELETE FROM " + KEYSPACE + ".clustering_static WHERE id=%s AND clustering1=%s AND clustering2=%s AND clustering3=%s IF sval=%s ; ";
+
+
+ execute("BEGIN BATCH " + String.format(clusteringStaticInsert, 1, 1, 1, 1, 1, 1) + " APPLY BATCH");
+
+ assertRows(execute("SELECT * FROM " + KEYSPACE+".clustering_static WHERE id=1"), row(1, 1, 1, 1, 1, 1));
+
+ StringBuilder cmd2 = new StringBuilder();
+ cmd2.append("BEGIN BATCH ");
+ cmd2.append(String.format(clusteringInsert, 1, 1, 1, 2, 2));
+ cmd2.append(String.format(clusteringStaticConditionalUpdate, 11, 1, 1, 1, 1, 1));
+ cmd2.append("APPLY BATCH ");
+ execute(cmd2.toString());
+
+
+ assertRows(execute("SELECT * FROM " + KEYSPACE+".clustering_static WHERE id=1"),
+ row(1, 1, 1, 1, 1, 11),
+ row(1, 1, 1, 2, 1, 2)
+ );
+
+
+ StringBuilder cmd3 = new StringBuilder();
+ cmd3.append("BEGIN BATCH ");
+ cmd3.append(String.format(clusteringInsert, 1, 1, 2, 3, 23));
+ cmd3.append(String.format(clusteringStaticUpdate, 22, 1));
+ cmd3.append(String.format(clusteringDelete, 1, 1, 1, 1));
+ cmd3.append("APPLY BATCH ");
+ execute(cmd3.toString());
+
+ assertRows(execute("SELECT * FROM " + KEYSPACE+".clustering_static WHERE id=1"),
+ row(1, 1, 1, 2, 22, 2),
+ row(1, 1, 2, 3, 22, 23)
+ );
+
+ StringBuilder cmd4 = new StringBuilder();
+ cmd4.append("BEGIN BATCH ");
+ cmd4.append(String.format(clusteringInsert, 1, 2, 3, 4, 1234));
+ cmd4.append(String.format(clusteringStaticConditionalTTLUpdate, 5, 234, 1, 1, 1, 2, 22));
+ cmd4.append("APPLY BATCH ");
+ execute(cmd4.toString());
+
+ assertRows(execute("SELECT * FROM " + KEYSPACE+".clustering_static WHERE id=1"),
+ row(1, 1, 1, 2, 22, 234),
+ row(1, 1, 2, 3, 22, 23),
+ row(1, 2, 3, 4, 22, 1234)
+ );
+
+ Thread.sleep(6000);
+
+ assertRows(execute("SELECT * FROM " + KEYSPACE+".clustering_static WHERE id=1"),
+ row(1, 1, 1, 2, 22, null),
+ row(1, 1, 2, 3, 22, 23),
+ row(1, 2, 3, 4, 22, 1234)
+ );
+
+ StringBuilder cmd5 = new StringBuilder();
+ cmd5.append("BEGIN BATCH ");
+ cmd5.append(String.format(clusteringRangeDelete, 1, 2));
+ cmd5.append(String.format(clusteringStaticConditionalUpdate, 1234, 1, 1, 1, 2, 22));
+ cmd5.append("APPLY BATCH ");
+ execute(cmd5.toString());
+
+ assertRows(execute("SELECT * FROM " + KEYSPACE+".clustering_static WHERE id=1"),
+ row(1, 1, 1, 2, 22, 1234),
+ row(1, 1, 2, 3, 22, 23)
+ );
+
+ StringBuilder cmd6 = new StringBuilder();
+ cmd6.append("BEGIN BATCH ");
+ cmd6.append(String.format(clusteringUpdate, 345, 1, 3, 4, 5));
+ cmd6.append(String.format(clusteringStaticConditionalUpdate, 1, 1, 1, 1, 2, 22));
+ cmd6.append("APPLY BATCH ");
+ execute(cmd6.toString());
+
+ assertRows(execute("SELECT * FROM " + KEYSPACE+".clustering_static WHERE id=1"),
+ row(1, 1, 1, 2, 22, 1),
+ row(1, 1, 2, 3, 22, 23),
+ row(1, 3, 4, 5, 22, 345)
+ );
+
+
+ StringBuilder cmd7 = new StringBuilder();
+ cmd7.append("BEGIN BATCH ");
+ cmd7.append(String.format(clusteringDelete, 1, 3, 4, 5));
+ cmd7.append(String.format(clusteringStaticConditionalUpdate, 2300, 1, 1, 2, 3, 1)); // SHOULD NOT MATCH
+ cmd7.append("APPLY BATCH ");
+ execute(cmd7.toString());
+
+ assertRows(execute("SELECT * FROM " + KEYSPACE+".clustering_static WHERE id=1"),
+ row(1, 1, 1, 2, 22, 1),
+ row(1, 1, 2, 3, 22, 23),
+ row(1, 3, 4, 5, 22, 345)
+ );
+
+ StringBuilder cmd8 = new StringBuilder();
+ cmd8.append("BEGIN BATCH ");
+ cmd8.append(String.format(clusteringConditionalDelete, 1, 3, 4, 5, 345));
+ cmd8.append(String.format(clusteringRangeDelete, 1, 1));
+ cmd8.append(String.format(clusteringInsert, 1, 2, 3, 4, 5));
+ cmd8.append("APPLY BATCH ");
+ execute(cmd8.toString());
+
+ assertRows(execute("SELECT * FROM " + KEYSPACE+".clustering_static WHERE id=1"),
+ row(1, 2, 3, 4, 22, 5)
+ );
+
+ StringBuilder cmd9 = new StringBuilder();
+ cmd9.append("BEGIN BATCH ");
+ cmd9.append(String.format(clusteringConditionalInsert, 1, 3, 4, 5, 345));
+ cmd9.append(String.format(clusteringDelete, 1, 2, 3, 4));
+ cmd9.append("APPLY BATCH ");
+ execute(cmd9.toString());
+
+ assertRows(execute("SELECT * FROM " + KEYSPACE+".clustering_static WHERE id=1"),
+ row(1, 3, 4, 5, 22, 345)
+ );
+
+ StringBuilder cmd10 = new StringBuilder();
+ cmd10.append("BEGIN BATCH ");
+ cmd10.append(String.format(clusteringTTLInsert, 1, 2, 3, 4, 5, 5));
+ cmd10.append(String.format(clusteringConditionalTTLUpdate, 10, 5, 1, 3, 4, 5, 345));
+ cmd10.append("APPLY BATCH ");
+ execute(cmd10.toString());
+
+ assertRows(execute("SELECT * FROM " + KEYSPACE+".clustering_static WHERE id=1"),
+ row(1, 2, 3, 4, 22, 5), // 5 second TTL
+ row(1, 3, 4, 5, 22, 5) // 10 second TTL
+ );
+
+ Thread.sleep(6000);
+
+ assertRows(execute("SELECT * FROM " + KEYSPACE+".clustering_static WHERE id=1"),
+ row(1, 3, 4, 5, 22, 5) // now 4 second TTL
+ );
+
+ StringBuilder cmd11 = new StringBuilder();
+ cmd11.append("BEGIN BATCH ");
+ cmd11.append(String.format(clusteringConditionalTTLInsert, 1, 2, 3, 4, 5, 5));
+ cmd11.append(String.format(clusteringInsert,1, 4, 5, 6, 7));
+ cmd11.append("APPLY BATCH ");
+ execute(cmd11.toString());
+
+ assertRows(execute("SELECT * FROM " + KEYSPACE+".clustering_static WHERE id=1"),
+ row(1, 2, 3, 4, 22, 5), // This one has 5 seconds left
+ row(1, 3, 4, 5, 22, 5), // This one should have 4 seconds left
+ row(1, 4, 5, 6, 22, 7) // This one has no TTL
+ );
+
+ Thread.sleep(6000);
+
+ assertRows(execute("SELECT * FROM " + KEYSPACE+".clustering_static WHERE id=1"),
+ row(1, 3, 4, 5, 22, null), // We had a row here before from cmd9, but we've ttl'd out the value in cmd11
+ row(1, 4, 5, 6, 22, 7)
+ );
+
+ StringBuilder cmd12 = new StringBuilder();
+ cmd12.append("BEGIN BATCH ");
+ cmd12.append(String.format(clusteringConditionalTTLUpdate, 5, 5, 1, 3, 4, 5, null));
+ cmd12.append(String.format(clusteringTTLUpdate, 5, 8, 1, 4, 5, 6));
+ cmd12.append("APPLY BATCH ");
+ execute(cmd12.toString());
+
+ assertRows(execute("SELECT * FROM " + KEYSPACE+".clustering_static WHERE id=1"),
+ row(1, 3, 4, 5, 22, 5),
+ row(1, 4, 5, 6, 22, 8)
+ );
+
+ Thread.sleep(6000);
+ assertRows(execute("SELECT * FROM " + KEYSPACE+".clustering_static WHERE id=1"),
+ row(1, 3, 4, 5, 22, null),
+ row(1, 4, 5, 6, 22, null)
+ );
+
+ StringBuilder cmd13 = new StringBuilder();
+ cmd13.append("BEGIN BATCH ");
+ cmd13.append(String.format(clusteringStaticConditionalDelete, 1, 3, 4, 5, 22));
+ cmd13.append(String.format(clusteringInsert, 1, 2, 3, 4, 5));
+ cmd13.append("APPLY BATCH ");
+ execute(cmd13.toString());
+
+ assertRows(execute("SELECT * FROM " + KEYSPACE+".clustering_static WHERE id=1"),
+ row(1, 2, 3, 4, 22, 5),
+ row(1, 4, 5, 6, 22, null)
+ );
+
+ StringBuilder cmd14 = new StringBuilder();
+ cmd14.append("BEGIN BATCH ");
+ cmd14.append(String.format(clusteringStaticConditionalStaticUpdate, 23, 1, 22));
+ cmd14.append(String.format(clusteringDelete, 1, 4, 5, 6));
+ cmd14.append("APPLY BATCH ");
+ execute(cmd14.toString());
+
+ assertRows(execute("SELECT * FROM " + KEYSPACE+".clustering_static WHERE id=1"),
+ row(1, 2, 3, 4, 23, 5)
+ );
}
}
diff --git a/test/unit/org/apache/cassandra/cql3/validation/operations/CreateTest.java b/test/unit/org/apache/cassandra/cql3/validation/operations/CreateTest.java
index edb6668..1c3b449 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/operations/CreateTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/operations/CreateTest.java
@@ -25,17 +25,22 @@
import org.junit.Test;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.Schema;
-import org.apache.cassandra.config.SchemaConstants;
+import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.CQLTester;
import org.apache.cassandra.cql3.Duration;
import org.apache.cassandra.db.Mutation;
import org.apache.cassandra.db.partitions.Partition;
import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.exceptions.InvalidRequestException;
import org.apache.cassandra.exceptions.SyntaxException;
import org.apache.cassandra.io.util.DataOutputBuffer;
+import org.apache.cassandra.locator.AbstractEndpointSnitch;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.IEndpointSnitch;
+import org.apache.cassandra.locator.Replica;
import org.apache.cassandra.schema.SchemaKeyspace;
+import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.schema.*;
import org.apache.cassandra.triggers.ITrigger;
import org.apache.cassandra.utils.ByteBufferUtil;
@@ -45,18 +50,10 @@
import static junit.framework.Assert.assertTrue;
import static junit.framework.Assert.fail;
import static org.apache.cassandra.cql3.Duration.*;
-import static org.junit.Assert.assertEquals;
public class CreateTest extends CQLTester
{
@Test
- public void testCQL3PartitionKeyOnlyTable()
- {
- createTable("CREATE TABLE %s (id text PRIMARY KEY);");
- assertFalse(currentTableMetadata().isThriftCompatible());
- }
-
- @Test
public void testCreateTableWithSmallintColumns() throws Throwable
{
createTable("CREATE TABLE %s (a text, b smallint, c smallint, primary key (a, b));");
@@ -94,14 +91,14 @@
@Test
public void testCreateTableWithDurationColumns() throws Throwable
{
- assertInvalidMessage("duration type is not supported for PRIMARY KEY part a",
- "CREATE TABLE test (a duration PRIMARY KEY, b int);");
+ assertInvalidMessage("duration type is not supported for PRIMARY KEY column 'a'",
+ "CREATE TABLE cql_test_keyspace.table0 (a duration PRIMARY KEY, b int);");
- assertInvalidMessage("duration type is not supported for PRIMARY KEY part b",
- "CREATE TABLE test (a text, b duration, c duration, primary key (a, b));");
+ assertInvalidMessage("duration type is not supported for PRIMARY KEY column 'b'",
+ "CREATE TABLE cql_test_keyspace.table0 (a text, b duration, c duration, primary key (a, b));");
- assertInvalidMessage("duration type is not supported for PRIMARY KEY part b",
- "CREATE TABLE test (a text, b duration, c duration, primary key (a, b)) with clustering order by (b DESC);");
+ assertInvalidMessage("duration type is not supported for PRIMARY KEY column 'b'",
+ "CREATE TABLE cql_test_keyspace.table0 (a text, b duration, c duration, primary key (a, b)) with clustering order by (b DESC);");
createTable("CREATE TABLE %s (a int, b int, c duration, primary key (a, b));");
execute("INSERT INTO %s (a, b, c) VALUES (1, 1, 1y2mo)");
@@ -181,25 +178,25 @@
// Test duration within Map
assertInvalidMessage("Durations are not allowed as map keys: map<duration, text>",
- "CREATE TABLE test(pk int PRIMARY KEY, m map<duration, text>)");
+ "CREATE TABLE cql_test_keyspace.table0(pk int PRIMARY KEY, m map<duration, text>)");
createTable("CREATE TABLE %s(pk int PRIMARY KEY, m map<text, duration>)");
execute("INSERT INTO %s (pk, m) VALUES (1, {'one month' : 1mo, '60 days' : 60d})");
assertRows(execute("SELECT * FROM %s"),
row(1, map("one month", Duration.from("1mo"), "60 days", Duration.from("60d"))));
- assertInvalidMessage("duration type is not supported for PRIMARY KEY part m",
- "CREATE TABLE %s(m frozen<map<text, duration>> PRIMARY KEY, v int)");
+ assertInvalidMessage("duration type is not supported for PRIMARY KEY column 'm'",
+ "CREATE TABLE cql_test_keyspace.table0(m frozen<map<text, duration>> PRIMARY KEY, v int)");
- assertInvalidMessage("duration type is not supported for PRIMARY KEY part m",
- "CREATE TABLE %s(pk int, m frozen<map<text, duration>>, v int, PRIMARY KEY (pk, m))");
+ assertInvalidMessage("duration type is not supported for PRIMARY KEY column 'm'",
+ "CREATE TABLE cql_test_keyspace.table0(pk int, m frozen<map<text, duration>>, v int, PRIMARY KEY (pk, m))");
// Test duration within Set
assertInvalidMessage("Durations are not allowed inside sets: set<duration>",
- "CREATE TABLE %s(pk int PRIMARY KEY, s set<duration>)");
+ "CREATE TABLE cql_test_keyspace.table0(pk int PRIMARY KEY, s set<duration>)");
assertInvalidMessage("Durations are not allowed inside sets: frozen<set<duration>>",
- "CREATE TABLE %s(s frozen<set<duration>> PRIMARY KEY, v int)");
+ "CREATE TABLE cql_test_keyspace.table0(s frozen<set<duration>> PRIMARY KEY, v int)");
// Test duration within List
createTable("CREATE TABLE %s(pk int PRIMARY KEY, l list<duration>)");
@@ -207,8 +204,8 @@
assertRows(execute("SELECT * FROM %s"),
row(1, list(Duration.from("1mo"), Duration.from("60d"))));
- assertInvalidMessage("duration type is not supported for PRIMARY KEY part l",
- "CREATE TABLE %s(l frozen<list<duration>> PRIMARY KEY, v int)");
+ assertInvalidMessage("duration type is not supported for PRIMARY KEY column 'l'",
+ "CREATE TABLE cql_test_keyspace.table0(l frozen<list<duration>> PRIMARY KEY, v int)");
// Test duration within Tuple
createTable("CREATE TABLE %s(pk int PRIMARY KEY, t tuple<int, duration>)");
@@ -216,8 +213,8 @@
assertRows(execute("SELECT * FROM %s"),
row(1, tuple(1, Duration.from("1mo"))));
- assertInvalidMessage("duration type is not supported for PRIMARY KEY part t",
- "CREATE TABLE %s(t frozen<tuple<int, duration>> PRIMARY KEY, v int)");
+ assertInvalidMessage("duration type is not supported for PRIMARY KEY column 't'",
+ "CREATE TABLE cql_test_keyspace.table0(t frozen<tuple<int, duration>> PRIMARY KEY, v int)");
// Test duration within UDT
String typename = createType("CREATE TYPE %s (a duration)");
@@ -227,12 +224,12 @@
assertRows(execute("SELECT * FROM %s"),
row(1, userType("a", Duration.from("1mo"))));
- assertInvalidMessage("duration type is not supported for PRIMARY KEY part u",
- "CREATE TABLE %s(pk int, u frozen<" + myType + ">, v int, PRIMARY KEY(pk, u))");
+ assertInvalidMessage("duration type is not supported for PRIMARY KEY column 'u'",
+ "CREATE TABLE cql_test_keyspace.table0(pk int, u frozen<" + myType + ">, v int, PRIMARY KEY(pk, u))");
// Test duration with several level of depth
- assertInvalidMessage("duration type is not supported for PRIMARY KEY part m",
- "CREATE TABLE %s(pk int, m frozen<map<text, list<tuple<int, duration>>>>, v int, PRIMARY KEY (pk, m))");
+ assertInvalidMessage("duration type is not supported for PRIMARY KEY column 'm'",
+ "CREATE TABLE cql_test_keyspace.table0(pk int, m frozen<map<text, list<tuple<int, duration>>>>, v int, PRIMARY KEY (pk, m))");
}
private ByteBuffer duration(long months, long days, long nanoseconds) throws IOException
@@ -286,142 +283,7 @@
row(id1, 36, null, null));
}
- /**
- * Creation and basic operations on a static table with compact storage,
- * migrated from cql_tests.py:TestCQL.noncomposite_static_cf_test()
- */
- @Test
- public void testDenseStaticTable() throws Throwable
- {
- createTable("CREATE TABLE %s (userid uuid PRIMARY KEY, firstname text, lastname text, age int) WITH COMPACT STORAGE");
-
- UUID id1 = UUID.fromString("550e8400-e29b-41d4-a716-446655440000");
- UUID id2 = UUID.fromString("f47ac10b-58cc-4372-a567-0e02b2c3d479");
-
- execute("INSERT INTO %s (userid, firstname, lastname, age) VALUES (?, ?, ?, ?)", id1, "Frodo", "Baggins", 32);
- execute("UPDATE %s SET firstname = ?, lastname = ?, age = ? WHERE userid = ?", "Samwise", "Gamgee", 33, id2);
-
- assertRows(execute("SELECT firstname, lastname FROM %s WHERE userid = ?", id1),
- row("Frodo", "Baggins"));
-
- assertRows(execute("SELECT * FROM %s WHERE userid = ?", id1),
- row(id1, 32, "Frodo", "Baggins"));
-
- assertRows(execute("SELECT * FROM %s"),
- row(id2, 33, "Samwise", "Gamgee"),
- row(id1, 32, "Frodo", "Baggins")
- );
-
- String batch = "BEGIN BATCH "
- + "INSERT INTO %1$s (userid, age) VALUES (?, ?) "
- + "UPDATE %1$s SET age = ? WHERE userid = ? "
- + "DELETE firstname, lastname FROM %1$s WHERE userid = ? "
- + "DELETE firstname, lastname FROM %1$s WHERE userid = ? "
- + "APPLY BATCH";
-
- execute(batch, id1, 36, 37, id2, id1, id2);
-
- assertRows(execute("SELECT * FROM %s"),
- row(id2, 37, null, null),
- row(id1, 36, null, null));
- }
-
- /**
- * Creation and basic operations on a non-composite table with compact storage,
- * migrated from cql_tests.py:TestCQL.dynamic_cf_test()
- */
- @Test
- public void testDenseNonCompositeTable() throws Throwable
- {
- createTable("CREATE TABLE %s (userid uuid, url text, time bigint, PRIMARY KEY (userid, url)) WITH COMPACT STORAGE");
-
- UUID id1 = UUID.fromString("550e8400-e29b-41d4-a716-446655440000");
- UUID id2 = UUID.fromString("f47ac10b-58cc-4372-a567-0e02b2c3d479");
- UUID id3 = UUID.fromString("810e8500-e29b-41d4-a716-446655440000");
-
- execute("INSERT INTO %s (userid, url, time) VALUES (?, ?, ?)", id1, "http://foo.bar", 42L);
- execute("INSERT INTO %s (userid, url, time) VALUES (?, ?, ?)", id1, "http://foo-2.bar", 24L);
- execute("INSERT INTO %s (userid, url, time) VALUES (?, ?, ?)", id1, "http://bar.bar", 128L);
- execute("UPDATE %s SET time = 24 WHERE userid = ? and url = 'http://bar.foo'", id2);
- execute("UPDATE %s SET time = 12 WHERE userid IN (?, ?) and url = 'http://foo-3'", id2, id1);
-
- assertRows(execute("SELECT url, time FROM %s WHERE userid = ?", id1),
- row("http://bar.bar", 128L),
- row("http://foo-2.bar", 24L),
- row("http://foo-3", 12L),
- row("http://foo.bar", 42L));
-
- assertRows(execute("SELECT * FROM %s WHERE userid = ?", id2),
- row(id2, "http://bar.foo", 24L),
- row(id2, "http://foo-3", 12L));
-
- assertRows(execute("SELECT time FROM %s"),
- row(24L), // id2
- row(12L),
- row(128L), // id1
- row(24L),
- row(12L),
- row(42L)
- );
-
- // Check we don't allow empty values for url since this is the full underlying cell name (#6152)
- assertInvalid("INSERT INTO %s (userid, url, time) VALUES (?, '', 42)", id3);
- }
-
- /**
- * Creation and basic operations on a composite table with compact storage,
- * migrated from cql_tests.py:TestCQL.dense_cf_test()
- */
- @Test
- public void testDenseCompositeTable() throws Throwable
- {
- createTable("CREATE TABLE %s (userid uuid, ip text, port int, time bigint, PRIMARY KEY (userid, ip, port)) WITH COMPACT STORAGE");
-
- UUID id1 = UUID.fromString("550e8400-e29b-41d4-a716-446655440000");
- UUID id2 = UUID.fromString("f47ac10b-58cc-4372-a567-0e02b2c3d479");
-
- execute("INSERT INTO %s (userid, ip, port, time) VALUES (?, '192.168.0.1', 80, 42)", id1);
- execute("INSERT INTO %s (userid, ip, port, time) VALUES (?, '192.168.0.2', 80, 24)", id1);
- execute("INSERT INTO %s (userid, ip, port, time) VALUES (?, '192.168.0.2', 90, 42)", id1);
- execute("UPDATE %s SET time = 24 WHERE userid = ? AND ip = '192.168.0.2' AND port = 80", id2);
-
- // we don't have to include all of the clustering columns (see CASSANDRA-7990)
- execute("INSERT INTO %s (userid, ip, time) VALUES (?, '192.168.0.3', 42)", id2);
- execute("UPDATE %s SET time = 42 WHERE userid = ? AND ip = '192.168.0.4'", id2);
-
- assertRows(execute("SELECT ip, port, time FROM %s WHERE userid = ?", id1),
- row("192.168.0.1", 80, 42L),
- row("192.168.0.2", 80, 24L),
- row("192.168.0.2", 90, 42L));
-
- assertRows(execute("SELECT ip, port, time FROM %s WHERE userid = ? and ip >= '192.168.0.2'", id1),
- row("192.168.0.2", 80, 24L),
- row("192.168.0.2", 90, 42L));
-
- assertRows(execute("SELECT ip, port, time FROM %s WHERE userid = ? and ip = '192.168.0.2'", id1),
- row("192.168.0.2", 80, 24L),
- row("192.168.0.2", 90, 42L));
-
- assertEmpty(execute("SELECT ip, port, time FROM %s WHERE userid = ? and ip > '192.168.0.2'", id1));
-
- assertRows(execute("SELECT ip, port, time FROM %s WHERE userid = ? AND ip = '192.168.0.3'", id2),
- row("192.168.0.3", null, 42L));
-
- assertRows(execute("SELECT ip, port, time FROM %s WHERE userid = ? AND ip = '192.168.0.4'", id2),
- row("192.168.0.4", null, 42L));
-
- execute("DELETE time FROM %s WHERE userid = ? AND ip = '192.168.0.2' AND port = 80", id1);
-
- assertRowCount(execute("SELECT * FROM %s WHERE userid = ?", id1), 2);
-
- execute("DELETE FROM %s WHERE userid = ?", id1);
- assertEmpty(execute("SELECT * FROM %s WHERE userid = ?", id1));
-
- execute("DELETE FROM %s WHERE userid = ? AND ip = '192.168.0.3'", id2);
- assertEmpty(execute("SELECT * FROM %s WHERE userid = ? AND ip = '192.168.0.3'", id2));
- }
-
- /**
+ /**
* Creation and basic operations on a composite table,
* migrated from cql_tests.py:TestCQL.sparse_cf_test()
*/
@@ -465,8 +327,6 @@
assertInvalid("CREATE TABLE test (key text PRIMARY KEY, key int)");
assertInvalid("CREATE TABLE test (key text PRIMARY KEY, c int, c text)");
-
- assertInvalid("CREATE TABLE test (key text, key2 text, c int, d text, PRIMARY KEY (key, key2)) WITH COMPACT STORAGE");
}
/**
@@ -476,7 +336,7 @@
@Test
public void testObsoleteTableProperties() throws Throwable
{
- assertInvalidThrow(SyntaxException.class, "CREATE TABLE test (foo text PRIMARY KEY, c int) WITH default_validation=timestamp");
+ assertInvalidThrow(SyntaxException.class, "CREATE TABLE cql_test_keyspace.table0 (foo text PRIMARY KEY, c int) WITH default_validation=timestamp");
createTable("CREATE TABLE %s (foo text PRIMARY KEY, c int)");
assertInvalidThrow(SyntaxException.class, "ALTER TABLE %s WITH default_validation=int");
@@ -497,7 +357,7 @@
"CREATE KEYSPACE My_much_much_too_long_identifier_that_should_not_work WITH replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 }");
execute("DROP KEYSPACE testXYZ");
- assertInvalidThrow(ConfigurationException.class, "DROP KEYSPACE non_existing");
+ assertInvalidThrow(InvalidRequestException.class, "DROP KEYSPACE non_existing");
execute("CREATE KEYSPACE testXYZ WITH replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 }");
@@ -505,12 +365,39 @@
execute("DROP KEYSPACE testXYZ");
}
+ /**
+ * Test {@link ConfigurationException} is thrown on create keyspace with invalid DC option in replication configuration .
+ */
+ @Test
+ public void testCreateKeyspaceWithNTSOnlyAcceptsConfiguredDataCenterNames() throws Throwable
+ {
+ assertInvalidThrow(ConfigurationException.class, "CREATE KEYSPACE testABC WITH replication = { 'class' : 'NetworkTopologyStrategy', 'INVALID_DC' : 2 }");
+ execute("CREATE KEYSPACE testABC WITH replication = {'class' : 'NetworkTopologyStrategy', '" + DATA_CENTER + "' : 2 }");
+
+ // Mix valid and invalid, should throw an exception
+ assertInvalidThrow(ConfigurationException.class, "CREATE KEYSPACE testXYZ WITH replication={ 'class' : 'NetworkTopologyStrategy', '" + DATA_CENTER + "' : 2 , 'INVALID_DC': 1}");
+
+ // clean-up
+ execute("DROP KEYSPACE IF EXISTS testABC");
+ execute("DROP KEYSPACE IF EXISTS testXYZ");
+ }
+
+ /**
+ * Test {@link ConfigurationException} is thrown on create keyspace without any options.
+ */
+ @Test
+ public void testConfigurationExceptionThrownWhenCreateKeyspaceWithNoOptions() throws Throwable
+ {
+ assertInvalidThrow(ConfigurationException.class, "CREATE KEYSPACE testXYZ with replication = { 'class': 'NetworkTopologyStrategy' }");
+ assertInvalidThrow(ConfigurationException.class, "CREATE KEYSPACE testXYZ WITH replication = { 'class' : 'SimpleStrategy' }");
+ }
+
@Test
public void testCreateKeyspaceWithMultipleInstancesOfSameDCThrowsException() throws Throwable
{
try
{
- assertInvalidThrow(SyntaxException.class, "CREATE KEYSPACE testABC WITH replication = {'class' : 'NetworkTopologyStrategy', 'dc1' : 2, 'dc1' : 3 }");
+ assertInvalidThrow(SyntaxException.class, "CREATE KEYSPACE testABC WITH replication = {'class' : 'NetworkTopologyStrategy', '" + DATA_CENTER + "' : 2, '" + DATA_CENTER + "' : 3 }");
}
finally
{
@@ -527,13 +414,12 @@
public void testTable() throws Throwable
{
String table1 = createTable(" CREATE TABLE %s (k int PRIMARY KEY, c int)");
- createTable(" CREATE TABLE %s (k int, name int, value int, PRIMARY KEY(k, name)) WITH COMPACT STORAGE ");
- createTable(" CREATE TABLE %s (k int, c int, PRIMARY KEY (k),)");
+ createTable("CREATE TABLE %s (k int, c int, PRIMARY KEY (k),)");
String table4 = createTableName();
// repeated column
- assertInvalidMessage("Multiple definition of identifier k", String.format("CREATE TABLE %s (k int PRIMARY KEY, c int, k text)", table4));
+ assertInvalidMessage("Duplicate column 'k' declaration for table", String.format("CREATE TABLE %s (k int PRIMARY KEY, c int, k text)", table4));
// compact storage limitations
assertInvalidThrow(SyntaxException.class,
@@ -545,17 +431,6 @@
}
/**
- * Test truncate statement,
- * migrated from cql_tests.py:TestCQL.table_test().
- */
- @Test
- public void testTruncate() throws Throwable
- {
- createTable(" CREATE TABLE %s (k int, name int, value int, PRIMARY KEY(k, name)) WITH COMPACT STORAGE ");
- execute("TRUNCATE %s");
- }
-
- /**
* Migrated from cql_tests.py:TestCQL.multiordering_validation_test()
*/
@Test
@@ -638,48 +513,34 @@
}
@Test
- public void testCreateIndexOnCompactTableWithClusteringColumns() throws Throwable
+ // tests CASSANDRA-4278
+ public void testHyphenDatacenters() throws Throwable
{
- createTable("CREATE TABLE %s (a int, b int , c int, PRIMARY KEY (a, b)) WITH COMPACT STORAGE;");
+ IEndpointSnitch snitch = DatabaseDescriptor.getEndpointSnitch();
- assertInvalidMessage("Secondary indexes are not supported on COMPACT STORAGE tables that have clustering columns",
- "CREATE INDEX ON %s (a);");
+ // Register an EndpointSnitch which returns fixed values for test.
+ DatabaseDescriptor.setEndpointSnitch(new AbstractEndpointSnitch()
+ {
+ @Override
+ public String getRack(InetAddressAndPort endpoint) { return RACK1; }
- assertInvalidMessage("Secondary indexes are not supported on COMPACT STORAGE tables that have clustering columns",
- "CREATE INDEX ON %s (b);");
+ @Override
+ public String getDatacenter(InetAddressAndPort endpoint) { return "us-east-1"; }
- assertInvalidMessage("Secondary indexes are not supported on COMPACT STORAGE tables that have clustering columns",
- "CREATE INDEX ON %s (c);");
- }
+ @Override
+ public int compareEndpoints(InetAddressAndPort target, Replica a1, Replica a2) { return 0; }
+ });
- @Test
- public void testCreateIndexOnCompactTableWithoutClusteringColumns() throws Throwable
- {
- createTable("CREATE TABLE %s (a int PRIMARY KEY, b int) WITH COMPACT STORAGE;");
+ // this forces the dc above to be added to the list of known datacenters (fixes static init problem
+ // with this group of tests), ok to remove at some point if doing so doesn't break the test
+ StorageService.instance.getTokenMetadata().updateHostId(UUID.randomUUID(), InetAddressAndPort.getByName("127.0.0.255"));
+ execute("CREATE KEYSPACE Foo WITH replication = { 'class' : 'NetworkTopologyStrategy', 'us-east-1' : 1 };");
- assertInvalidMessage("Secondary indexes are not supported on PRIMARY KEY columns in COMPACT STORAGE tables",
- "CREATE INDEX ON %s (a);");
+ // Restore the previous EndpointSnitch
+ DatabaseDescriptor.setEndpointSnitch(snitch);
- createIndex("CREATE INDEX ON %s (b);");
-
- execute("INSERT INTO %s (a, b) values (1, 1)");
- execute("INSERT INTO %s (a, b) values (2, 4)");
- execute("INSERT INTO %s (a, b) values (3, 6)");
-
- assertRows(execute("SELECT * FROM %s WHERE b = ?", 4), row(2, 4));
- }
-
- /**
- * Test for CASSANDRA-13917
- */
- @Test
- public void testCreateIndextWithCompactStaticFormat() throws Throwable
- {
- createTable("CREATE TABLE %s (a int PRIMARY KEY, b int, c int) WITH COMPACT STORAGE");
- assertInvalidMessage("Undefined column name column1",
- "CREATE INDEX column1_index on %s (column1)");
- assertInvalidMessage("Undefined column name value",
- "CREATE INDEX value_index on %s (value)");
+ // clean up
+ execute("DROP KEYSPACE IF EXISTS Foo");
}
@Test
@@ -703,7 +564,7 @@
SchemaKeyspace.TABLES),
KEYSPACE,
currentTable()),
- row(map("chunk_length_in_kb", "64", "class", "org.apache.cassandra.io.compress.LZ4Compressor")));
+ row(map("chunk_length_in_kb", "16", "class", "org.apache.cassandra.io.compress.LZ4Compressor")));
createTable("CREATE TABLE %s (a text, b int, c int, primary key (a, b))"
+ " WITH compression = { 'class' : 'SnappyCompressor', 'chunk_length_in_kb' : 32 };");
@@ -736,6 +597,36 @@
row(map("chunk_length_in_kb", "32", "class", "org.apache.cassandra.io.compress.SnappyCompressor")));
createTable("CREATE TABLE %s (a text, b int, c int, primary key (a, b))"
+ + " WITH compression = { 'sstable_compression' : 'SnappyCompressor', 'min_compress_ratio' : 2 };");
+
+ assertRows(execute(format("SELECT compression FROM %s.%s WHERE keyspace_name = ? and table_name = ?;",
+ SchemaConstants.SCHEMA_KEYSPACE_NAME,
+ SchemaKeyspace.TABLES),
+ KEYSPACE,
+ currentTable()),
+ row(map("chunk_length_in_kb", "16", "class", "org.apache.cassandra.io.compress.SnappyCompressor", "min_compress_ratio", "2.0")));
+
+ createTable("CREATE TABLE %s (a text, b int, c int, primary key (a, b))"
+ + " WITH compression = { 'sstable_compression' : 'SnappyCompressor', 'min_compress_ratio' : 1 };");
+
+ assertRows(execute(format("SELECT compression FROM %s.%s WHERE keyspace_name = ? and table_name = ?;",
+ SchemaConstants.SCHEMA_KEYSPACE_NAME,
+ SchemaKeyspace.TABLES),
+ KEYSPACE,
+ currentTable()),
+ row(map("chunk_length_in_kb", "16", "class", "org.apache.cassandra.io.compress.SnappyCompressor", "min_compress_ratio", "1.0")));
+
+ createTable("CREATE TABLE %s (a text, b int, c int, primary key (a, b))"
+ + " WITH compression = { 'sstable_compression' : 'SnappyCompressor', 'min_compress_ratio' : 0 };");
+
+ assertRows(execute(format("SELECT compression FROM %s.%s WHERE keyspace_name = ? and table_name = ?;",
+ SchemaConstants.SCHEMA_KEYSPACE_NAME,
+ SchemaKeyspace.TABLES),
+ KEYSPACE,
+ currentTable()),
+ row(map("chunk_length_in_kb", "16", "class", "org.apache.cassandra.io.compress.SnappyCompressor")));
+
+ createTable("CREATE TABLE %s (a text, b int, c int, primary key (a, b))"
+ " WITH compression = { 'sstable_compression' : '', 'chunk_length_kb' : 32 };");
assertRows(execute(format("SELECT compression FROM %s.%s WHERE keyspace_name = ? and table_name = ?;",
@@ -779,35 +670,55 @@
"CREATE TABLE %s (a text, b int, c int, primary key (a, b))"
+ " WITH compression = { 'class' : 'SnappyCompressor', 'chunk_length_kb' : 32 , 'chunk_length_in_kb' : 32 };");
+ assertThrowsConfigurationException("chunk_length_in_kb must be a power of 2",
+ "CREATE TABLE %s (a text, b int, c int, primary key (a, b))"
+ + " WITH compression = { 'class' : 'SnappyCompressor', 'chunk_length_in_kb' : 31 };");
+
+ assertThrowsConfigurationException("Invalid negative or null chunk_length_in_kb",
+ "CREATE TABLE %s (a text, b int, c int, primary key (a, b))"
+ + " WITH compression = { 'class' : 'SnappyCompressor', 'chunk_length_in_kb' : -1 };");
+
+ assertThrowsConfigurationException("Invalid negative min_compress_ratio",
+ "CREATE TABLE %s (a text, b int, c int, primary key (a, b))"
+ + " WITH compression = { 'class' : 'SnappyCompressor', 'min_compress_ratio' : -1 };");
+
assertThrowsConfigurationException("Unknown compression options unknownOption",
"CREATE TABLE %s (a text, b int, c int, primary key (a, b))"
+ " WITH compression = { 'class' : 'SnappyCompressor', 'unknownOption' : 32 };");
}
- private void assertThrowsConfigurationException(String errorMsg, String createStmt) {
- try
- {
- createTable(createStmt);
- fail("Query should be invalid but no error was thrown. Query is: " + createStmt);
- }
- catch (RuntimeException e)
- {
- Throwable cause = e.getCause();
- assertTrue("The exception should be a ConfigurationException", cause instanceof ConfigurationException);
- assertEquals(errorMsg, cause.getMessage());
- }
- }
+ @Test
+ public void compactTableTest() throws Throwable
+ {
+ assertInvalidMessage("COMPACT STORAGE tables are not allowed starting with version 4.0",
+ "CREATE TABLE compact_table_create (id text PRIMARY KEY, content text) WITH COMPACT STORAGE;");
+ }
+
+ private void assertThrowsConfigurationException(String errorMsg, String createStmt)
+ {
+ try
+ {
+ createTable(createStmt);
+ fail("Query should be invalid but no error was thrown. Query is: " + createStmt);
+ }
+ catch (RuntimeException e)
+ {
+ Throwable cause = e.getCause();
+ assertTrue("The exception should be a ConfigurationException", cause instanceof ConfigurationException);
+ assertEquals(errorMsg, cause.getMessage());
+ }
+ }
private void assertTriggerExists(String name)
{
- CFMetaData cfm = Schema.instance.getCFMetaData(keyspace(), currentTable()).copy();
- assertTrue("the trigger does not exist", cfm.getTriggers().get(name).isPresent());
+ TableMetadata metadata = Schema.instance.getTableMetadata(keyspace(), currentTable());
+ assertTrue("the trigger does not exist", metadata.triggers.get(name).isPresent());
}
private void assertTriggerDoesNotExists(String name)
{
- CFMetaData cfm = Schema.instance.getCFMetaData(keyspace(), currentTable()).copy();
- assertFalse("the trigger exists", cfm.getTriggers().get(name).isPresent());
+ TableMetadata metadata = Schema.instance.getTableMetadata(keyspace(), currentTable());
+ assertFalse("the trigger exists", metadata.triggers.get(name).isPresent());
}
public static class TestTrigger implements ITrigger
diff --git a/test/unit/org/apache/cassandra/cql3/validation/operations/DeleteTest.java b/test/unit/org/apache/cassandra/cql3/validation/operations/DeleteTest.java
index 66554da..0c88044 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/operations/DeleteTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/operations/DeleteTest.java
@@ -146,20 +146,6 @@
assertRows(execute("SELECT * FROM %s"),
row("abc", 4, "xyz", "some other value"));
-
- createTable("CREATE TABLE %s (username varchar, id int, name varchar, stuff varchar, PRIMARY KEY(username, id, name)) WITH COMPACT STORAGE");
-
- execute("INSERT INTO %s (username, id, name, stuff) VALUES (?, ?, ?, ?)", "abc", 2, "rst", "some value");
- execute("INSERT INTO %s (username, id, name, stuff) VALUES (?, ?, ?, ?)", "abc", 4, "xyz", "some other value");
-
- assertRows(execute("SELECT * FROM %s"),
- row("abc", 2, "rst", "some value"),
- row("abc", 4, "xyz", "some other value"));
-
- execute("DELETE FROM %s WHERE username='abc' AND id=2");
-
- assertRows(execute("SELECT * FROM %s"),
- row("abc", 4, "xyz", "some other value"));
}
/**
@@ -438,64 +424,54 @@
private void testDeleteWithNoClusteringColumns(boolean forceFlush) throws Throwable
{
- for (String compactOption : new String[] {"", " WITH COMPACT STORAGE" })
- {
- createTable("CREATE TABLE %s (partitionKey int PRIMARY KEY," +
- "value int)" + compactOption);
+ createTable("CREATE TABLE %s (partitionKey int PRIMARY KEY," +
+ "value int)");
- execute("INSERT INTO %s (partitionKey, value) VALUES (0, 0)");
- execute("INSERT INTO %s (partitionKey, value) VALUES (1, 1)");
- execute("INSERT INTO %s (partitionKey, value) VALUES (2, 2)");
- execute("INSERT INTO %s (partitionKey, value) VALUES (3, 3)");
- flush(forceFlush);
+ execute("INSERT INTO %s (partitionKey, value) VALUES (0, 0)");
+ execute("INSERT INTO %s (partitionKey, value) VALUES (1, 1)");
+ execute("INSERT INTO %s (partitionKey, value) VALUES (2, 2)");
+ execute("INSERT INTO %s (partitionKey, value) VALUES (3, 3)");
+ flush(forceFlush);
- execute("DELETE value FROM %s WHERE partitionKey = ?", 0);
- flush(forceFlush);
+ execute("DELETE value FROM %s WHERE partitionKey = ?", 0);
+ flush(forceFlush);
- if (isEmpty(compactOption))
- {
- assertRows(execute("SELECT * FROM %s WHERE partitionKey = ?", 0),
- row(0, null));
- }
- else
- {
- assertEmpty(execute("SELECT * FROM %s WHERE partitionKey = ?", 0));
- }
+ assertRows(execute("SELECT * FROM %s WHERE partitionKey = ?", 0),
+ row(0, null));
- execute("DELETE FROM %s WHERE partitionKey IN (?, ?)", 0, 1);
- flush(forceFlush);
- assertRows(execute("SELECT * FROM %s"),
- row(2, 2),
- row(3, 3));
+ execute("DELETE FROM %s WHERE partitionKey IN (?, ?)", 0, 1);
+ flush(forceFlush);
+ assertRows(execute("SELECT * FROM %s"),
+ row(2, 2),
+ row(3, 3));
- // test invalid queries
+ // test invalid queries
- // token function
- assertInvalidMessage("The token function cannot be used in WHERE clauses for DELETE statements",
- "DELETE FROM %s WHERE token(partitionKey) = token(?)", 0);
+ // token function
+ assertInvalidMessage("The token function cannot be used in WHERE clauses for DELETE statements",
+ "DELETE FROM %s WHERE token(partitionKey) = token(?)", 0);
- // multiple time same primary key element in WHERE clause
- assertInvalidMessage("partitionkey cannot be restricted by more than one relation if it includes an Equal",
- "DELETE FROM %s WHERE partitionKey = ? AND partitionKey = ?", 0, 1);
+ // multiple time same primary key element in WHERE clause
+ assertInvalidMessage("partitionkey cannot be restricted by more than one relation if it includes an Equal",
+ "DELETE FROM %s WHERE partitionKey = ? AND partitionKey = ?", 0, 1);
- // unknown identifiers
- assertInvalidMessage("Undefined column name unknown",
- "DELETE unknown FROM %s WHERE partitionKey = ?", 0);
+ // unknown identifiers
+ assertInvalidMessage("Undefined column name unknown",
+ "DELETE unknown FROM %s WHERE partitionKey = ?", 0);
- assertInvalidMessage("Undefined column name partitionkey1",
- "DELETE FROM %s WHERE partitionKey1 = ?", 0);
+ assertInvalidMessage("Undefined column name partitionkey1",
+ "DELETE FROM %s WHERE partitionKey1 = ?", 0);
- // Invalid operator in the where clause
- assertInvalidMessage("Only EQ and IN relation are supported on the partition key (unless you use the token() function)",
- "DELETE FROM %s WHERE partitionKey > ? ", 0);
+ // Invalid operator in the where clause
+ assertInvalidMessage("Only EQ and IN relation are supported on the partition key (unless you use the token() function)",
+ "DELETE FROM %s WHERE partitionKey > ? ", 0);
- assertInvalidMessage("Cannot use CONTAINS on non-collection column partitionkey",
- "DELETE FROM %s WHERE partitionKey CONTAINS ?", 0);
+ assertInvalidMessage("Cannot use CONTAINS on non-collection column partitionkey",
+ "DELETE FROM %s WHERE partitionKey CONTAINS ?", 0);
- // Non primary key in the where clause
- assertInvalidMessage("Non PRIMARY KEY columns found in where clause: value",
- "DELETE FROM %s WHERE partitionKey = ? AND value = ?", 0, 1);
- }
+ // Non primary key in the where clause
+ assertInvalidMessage("Non PRIMARY KEY columns found in where clause: value",
+ "DELETE FROM %s WHERE partitionKey = ? AND value = ?", 0, 1);
}
@Test
@@ -507,87 +483,77 @@
private void testDeleteWithOneClusteringColumns(boolean forceFlush) throws Throwable
{
- for (String compactOption : new String[] {"", " WITH COMPACT STORAGE" })
- {
- createTable("CREATE TABLE %s (partitionKey int," +
- "clustering int," +
- "value int," +
- " PRIMARY KEY (partitionKey, clustering))" + compactOption);
+ createTable("CREATE TABLE %s (partitionKey int," +
+ "clustering int," +
+ "value int," +
+ " PRIMARY KEY (partitionKey, clustering))");
- execute("INSERT INTO %s (partitionKey, clustering, value) VALUES (0, 0, 0)");
- execute("INSERT INTO %s (partitionKey, clustering, value) VALUES (0, 1, 1)");
- execute("INSERT INTO %s (partitionKey, clustering, value) VALUES (0, 2, 2)");
- execute("INSERT INTO %s (partitionKey, clustering, value) VALUES (0, 3, 3)");
- execute("INSERT INTO %s (partitionKey, clustering, value) VALUES (0, 4, 4)");
- execute("INSERT INTO %s (partitionKey, clustering, value) VALUES (0, 5, 5)");
- execute("INSERT INTO %s (partitionKey, clustering, value) VALUES (1, 0, 6)");
- flush(forceFlush);
+ execute("INSERT INTO %s (partitionKey, clustering, value) VALUES (0, 0, 0)");
+ execute("INSERT INTO %s (partitionKey, clustering, value) VALUES (0, 1, 1)");
+ execute("INSERT INTO %s (partitionKey, clustering, value) VALUES (0, 2, 2)");
+ execute("INSERT INTO %s (partitionKey, clustering, value) VALUES (0, 3, 3)");
+ execute("INSERT INTO %s (partitionKey, clustering, value) VALUES (0, 4, 4)");
+ execute("INSERT INTO %s (partitionKey, clustering, value) VALUES (0, 5, 5)");
+ execute("INSERT INTO %s (partitionKey, clustering, value) VALUES (1, 0, 6)");
+ flush(forceFlush);
- execute("DELETE value FROM %s WHERE partitionKey = ? AND clustering = ?", 0, 1);
- flush(forceFlush);
- if (isEmpty(compactOption))
- {
- assertRows(execute("SELECT * FROM %s WHERE partitionKey = ? AND clustering = ?", 0, 1),
- row(0, 1, null));
- }
- else
- {
- assertEmpty(execute("SELECT * FROM %s WHERE partitionKey = ? AND clustering = ?", 0, 1));
- }
+ execute("DELETE value FROM %s WHERE partitionKey = ? AND clustering = ?", 0, 1);
+ flush(forceFlush);
+ assertRows(execute("SELECT * FROM %s WHERE partitionKey = ? AND clustering = ?", 0, 1),
+ row(0, 1, null));
- execute("DELETE FROM %s WHERE partitionKey = ? AND clustering = ?", 0, 1);
- flush(forceFlush);
- assertEmpty(execute("SELECT value FROM %s WHERE partitionKey = ? AND clustering = ?", 0, 1));
+ execute("DELETE FROM %s WHERE partitionKey = ? AND clustering = ?", 0, 1);
+ flush(forceFlush);
+ assertEmpty(execute("SELECT value FROM %s WHERE partitionKey = ? AND clustering = ?", 0, 1));
- execute("DELETE FROM %s WHERE partitionKey IN (?, ?) AND clustering = ?", 0, 1, 0);
- flush(forceFlush);
- assertRows(execute("SELECT * FROM %s WHERE partitionKey IN (?, ?)", 0, 1),
- row(0, 2, 2),
- row(0, 3, 3),
- row(0, 4, 4),
- row(0, 5, 5));
+ execute("DELETE FROM %s WHERE partitionKey IN (?, ?) AND clustering = ?", 0, 1, 0);
+ flush(forceFlush);
+ assertRows(execute("SELECT * FROM %s WHERE partitionKey IN (?, ?)", 0, 1),
+ row(0, 2, 2),
+ row(0, 3, 3),
+ row(0, 4, 4),
+ row(0, 5, 5));
- execute("DELETE FROM %s WHERE partitionKey = ? AND (clustering) IN ((?), (?))", 0, 4, 5);
- flush(forceFlush);
- assertRows(execute("SELECT * FROM %s WHERE partitionKey IN (?, ?)", 0, 1),
- row(0, 2, 2),
- row(0, 3, 3));
+ execute("DELETE FROM %s WHERE partitionKey = ? AND (clustering) IN ((?), (?))", 0, 4, 5);
+ flush(forceFlush);
+ assertRows(execute("SELECT * FROM %s WHERE partitionKey IN (?, ?)", 0, 1),
+ row(0, 2, 2),
+ row(0, 3, 3));
- // test invalid queries
+ // test invalid queries
- // missing primary key element
- assertInvalidMessage("Some partition key parts are missing: partitionkey",
- "DELETE FROM %s WHERE clustering = ?", 1);
+ // missing primary key element
+ assertInvalidMessage("Some partition key parts are missing: partitionkey",
+ "DELETE FROM %s WHERE clustering = ?", 1);
- // token function
- assertInvalidMessage("The token function cannot be used in WHERE clauses for DELETE statements",
- "DELETE FROM %s WHERE token(partitionKey) = token(?) AND clustering = ? ", 0, 1);
+ // token function
+ assertInvalidMessage("The token function cannot be used in WHERE clauses for DELETE statements",
+ "DELETE FROM %s WHERE token(partitionKey) = token(?) AND clustering = ? ", 0, 1);
- // multiple time same primary key element in WHERE clause
- assertInvalidMessage("clustering cannot be restricted by more than one relation if it includes an Equal",
- "DELETE FROM %s WHERE partitionKey = ? AND clustering = ? AND clustering = ?", 0, 1, 1);
+ // multiple time same primary key element in WHERE clause
+ assertInvalidMessage("clustering cannot be restricted by more than one relation if it includes an Equal",
+ "DELETE FROM %s WHERE partitionKey = ? AND clustering = ? AND clustering = ?", 0, 1, 1);
- // unknown identifiers
- assertInvalidMessage("Undefined column name value1",
- "DELETE value1 FROM %s WHERE partitionKey = ? AND clustering = ?", 0, 1);
+ // unknown identifiers
+ assertInvalidMessage("Undefined column name value1",
+ "DELETE value1 FROM %s WHERE partitionKey = ? AND clustering = ?", 0, 1);
- assertInvalidMessage("Undefined column name partitionkey1",
- "DELETE FROM %s WHERE partitionKey1 = ? AND clustering = ?", 0, 1);
+ assertInvalidMessage("Undefined column name partitionkey1",
+ "DELETE FROM %s WHERE partitionKey1 = ? AND clustering = ?", 0, 1);
- assertInvalidMessage("Undefined column name clustering_3",
- "DELETE FROM %s WHERE partitionKey = ? AND clustering_3 = ?", 0, 1);
+ assertInvalidMessage("Undefined column name clustering_3",
+ "DELETE FROM %s WHERE partitionKey = ? AND clustering_3 = ?", 0, 1);
- // Invalid operator in the where clause
- assertInvalidMessage("Only EQ and IN relation are supported on the partition key (unless you use the token() function)",
- "DELETE FROM %s WHERE partitionKey > ? AND clustering = ?", 0, 1);
+ // Invalid operator in the where clause
+ assertInvalidMessage("Only EQ and IN relation are supported on the partition key (unless you use the token() function)",
+ "DELETE FROM %s WHERE partitionKey > ? AND clustering = ?", 0, 1);
- assertInvalidMessage("Cannot use CONTAINS on non-collection column partitionkey",
- "DELETE FROM %s WHERE partitionKey CONTAINS ? AND clustering = ?", 0, 1);
+ assertInvalidMessage("Cannot use CONTAINS on non-collection column partitionkey",
+ "DELETE FROM %s WHERE partitionKey CONTAINS ? AND clustering = ?", 0, 1);
- // Non primary key in the where clause
- assertInvalidMessage("Non PRIMARY KEY columns found in where clause: value",
- "DELETE FROM %s WHERE partitionKey = ? AND clustering = ? AND value = ?", 0, 1, 3);
- }
+ // Non primary key in the where clause
+ assertInvalidMessage("Non PRIMARY KEY columns found in where clause: value",
+ "DELETE FROM %s WHERE partitionKey = ? AND clustering = ? AND value = ?", 0, 1, 3);
}
@Test
@@ -599,125 +565,96 @@
private void testDeleteWithTwoClusteringColumns(boolean forceFlush) throws Throwable
{
- for (String compactOption : new String[] { "", " WITH COMPACT STORAGE" })
- {
- createTable("CREATE TABLE %s (partitionKey int," +
- "clustering_1 int," +
- "clustering_2 int," +
- "value int," +
- " PRIMARY KEY (partitionKey, clustering_1, clustering_2))" + compactOption);
+ createTable("CREATE TABLE %s (partitionKey int," +
+ "clustering_1 int," +
+ "clustering_2 int," +
+ "value int," +
+ " PRIMARY KEY (partitionKey, clustering_1, clustering_2))");
- execute("INSERT INTO %s (partitionKey, clustering_1, clustering_2, value) VALUES (0, 0, 0, 0)");
- execute("INSERT INTO %s (partitionKey, clustering_1, clustering_2, value) VALUES (0, 0, 1, 1)");
- execute("INSERT INTO %s (partitionKey, clustering_1, clustering_2, value) VALUES (0, 0, 2, 2)");
- execute("INSERT INTO %s (partitionKey, clustering_1, clustering_2, value) VALUES (0, 0, 3, 3)");
- execute("INSERT INTO %s (partitionKey, clustering_1, clustering_2, value) VALUES (0, 1, 1, 4)");
- execute("INSERT INTO %s (partitionKey, clustering_1, clustering_2, value) VALUES (0, 1, 2, 5)");
- execute("INSERT INTO %s (partitionKey, clustering_1, clustering_2, value) VALUES (1, 0, 0, 6)");
- flush(forceFlush);
+ execute("INSERT INTO %s (partitionKey, clustering_1, clustering_2, value) VALUES (0, 0, 0, 0)");
+ execute("INSERT INTO %s (partitionKey, clustering_1, clustering_2, value) VALUES (0, 0, 1, 1)");
+ execute("INSERT INTO %s (partitionKey, clustering_1, clustering_2, value) VALUES (0, 0, 2, 2)");
+ execute("INSERT INTO %s (partitionKey, clustering_1, clustering_2, value) VALUES (0, 0, 3, 3)");
+ execute("INSERT INTO %s (partitionKey, clustering_1, clustering_2, value) VALUES (0, 1, 1, 4)");
+ execute("INSERT INTO %s (partitionKey, clustering_1, clustering_2, value) VALUES (0, 1, 2, 5)");
+ execute("INSERT INTO %s (partitionKey, clustering_1, clustering_2, value) VALUES (1, 0, 0, 6)");
+ flush(forceFlush);
- execute("DELETE value FROM %s WHERE partitionKey = ? AND clustering_1 = ? AND clustering_2 = ?", 0, 1, 1);
- flush(forceFlush);
+ execute("DELETE value FROM %s WHERE partitionKey = ? AND clustering_1 = ? AND clustering_2 = ?", 0, 1, 1);
+ flush(forceFlush);
- if (isEmpty(compactOption))
- {
- assertRows(execute("SELECT * FROM %s WHERE partitionKey = ? AND clustering_1 = ? AND clustering_2 = ?",
- 0, 1, 1),
- row(0, 1, 1, null));
- }
- else
- {
- assertEmpty(execute("SELECT * FROM %s WHERE partitionKey = ? AND clustering_1 = ? AND clustering_2 = ?",
- 0, 1, 1));
- }
+ assertRows(execute("SELECT * FROM %s WHERE partitionKey = ? AND clustering_1 = ? AND clustering_2 = ?",
+ 0, 1, 1),
+ row(0, 1, 1, null));
- execute("DELETE FROM %s WHERE partitionKey = ? AND (clustering_1, clustering_2) = (?, ?)", 0, 1, 1);
- flush(forceFlush);
- assertEmpty(execute("SELECT value FROM %s WHERE partitionKey = ? AND clustering_1 = ? AND clustering_2 = ?",
- 0, 1, 1));
+ execute("DELETE FROM %s WHERE partitionKey = ? AND (clustering_1, clustering_2) = (?, ?)", 0, 1, 1);
+ flush(forceFlush);
+ assertEmpty(execute("SELECT value FROM %s WHERE partitionKey = ? AND clustering_1 = ? AND clustering_2 = ?",
+ 0, 1, 1));
- execute("DELETE FROM %s WHERE partitionKey IN (?, ?) AND clustering_1 = ? AND clustering_2 = ?", 0, 1, 0, 0);
- flush(forceFlush);
- assertRows(execute("SELECT * FROM %s WHERE partitionKey IN (?, ?)", 0, 1),
- row(0, 0, 1, 1),
- row(0, 0, 2, 2),
- row(0, 0, 3, 3),
- row(0, 1, 2, 5));
+ execute("DELETE FROM %s WHERE partitionKey IN (?, ?) AND clustering_1 = ? AND clustering_2 = ?", 0, 1, 0, 0);
+ flush(forceFlush);
+ assertRows(execute("SELECT * FROM %s WHERE partitionKey IN (?, ?)", 0, 1),
+ row(0, 0, 1, 1),
+ row(0, 0, 2, 2),
+ row(0, 0, 3, 3),
+ row(0, 1, 2, 5));
- Object[][] rows;
- if (isEmpty(compactOption))
- {
- rows = new Object[][]{row(0, 0, 1, 1),
- row(0, 0, 2, null),
- row(0, 0, 3, null),
- row(0, 1, 2, 5)};
- }
- else
- {
- rows = new Object[][]{row(0, 0, 1, 1), row(0, 1, 2, 5)};
- }
+ execute("DELETE value FROM %s WHERE partitionKey = ? AND clustering_1 = ? AND clustering_2 IN (?, ?)", 0, 0, 2, 3);
+ flush(forceFlush);
+ assertRows(execute("SELECT * FROM %s WHERE partitionKey IN (?, ?)", 0, 1), row(0, 0, 1, 1),
+ row(0, 0, 2, null),
+ row(0, 0, 3, null),
+ row(0, 1, 2, 5));
- execute("DELETE value FROM %s WHERE partitionKey = ? AND clustering_1 = ? AND clustering_2 IN (?, ?)", 0, 0, 2, 3);
- flush(forceFlush);
- assertRows(execute("SELECT * FROM %s WHERE partitionKey IN (?, ?)", 0, 1), rows);
+ execute("DELETE FROM %s WHERE partitionKey = ? AND (clustering_1, clustering_2) IN ((?, ?), (?, ?))", 0, 0, 2, 1, 2);
+ flush(forceFlush);
+ assertRows(execute("SELECT * FROM %s WHERE partitionKey IN (?, ?)", 0, 1),
+ row(0, 0, 1, 1),
+ row(0, 0, 3, null));
- if (isEmpty(compactOption))
- {
- rows = new Object[][]{row(0, 0, 1, 1),
- row(0, 0, 3, null)};
- }
- else
- {
- rows = new Object[][]{row(0, 0, 1, 1)};
- }
+ execute("DELETE FROM %s WHERE partitionKey = ? AND (clustering_1) IN ((?), (?)) AND clustering_2 = ?", 0, 0, 2, 3);
+ flush(forceFlush);
+ assertRows(execute("SELECT * FROM %s WHERE partitionKey IN (?, ?)", 0, 1),
+ row(0, 0, 1, 1));
- execute("DELETE FROM %s WHERE partitionKey = ? AND (clustering_1, clustering_2) IN ((?, ?), (?, ?))", 0, 0, 2, 1, 2);
- flush(forceFlush);
- assertRows(execute("SELECT * FROM %s WHERE partitionKey IN (?, ?)", 0, 1), rows);
+ // test invalid queries
- execute("DELETE FROM %s WHERE partitionKey = ? AND (clustering_1) IN ((?), (?)) AND clustering_2 = ?", 0, 0, 2, 3);
- flush(forceFlush);
- assertRows(execute("SELECT * FROM %s WHERE partitionKey IN (?, ?)", 0, 1),
- row(0, 0, 1, 1));
+ // missing primary key element
+ assertInvalidMessage("Some partition key parts are missing: partitionkey",
+ "DELETE FROM %s WHERE clustering_1 = ? AND clustering_2 = ?", 1, 1);
- // test invalid queries
+ assertInvalidMessage("PRIMARY KEY column \"clustering_2\" cannot be restricted as preceding column \"clustering_1\" is not restricted",
+ "DELETE FROM %s WHERE partitionKey = ? AND clustering_2 = ?", 0, 1);
- // missing primary key element
- assertInvalidMessage("Some partition key parts are missing: partitionkey",
- "DELETE FROM %s WHERE clustering_1 = ? AND clustering_2 = ?", 1, 1);
+ // token function
+ assertInvalidMessage("The token function cannot be used in WHERE clauses for DELETE statements",
+ "DELETE FROM %s WHERE token(partitionKey) = token(?) AND clustering_1 = ? AND clustering_2 = ?", 0, 1, 1);
- assertInvalidMessage("PRIMARY KEY column \"clustering_2\" cannot be restricted as preceding column \"clustering_1\" is not restricted",
- "DELETE FROM %s WHERE partitionKey = ? AND clustering_2 = ?", 0, 1);
+ // multiple time same primary key element in WHERE clause
+ assertInvalidMessage("clustering_1 cannot be restricted by more than one relation if it includes an Equal",
+ "DELETE FROM %s WHERE partitionKey = ? AND clustering_1 = ? AND clustering_2 = ? AND clustering_1 = ?", 0, 1, 1, 1);
- // token function
- assertInvalidMessage("The token function cannot be used in WHERE clauses for DELETE statements",
- "DELETE FROM %s WHERE token(partitionKey) = token(?) AND clustering_1 = ? AND clustering_2 = ?", 0, 1, 1);
+ // unknown identifiers
+ assertInvalidMessage("Undefined column name value1",
+ "DELETE value1 FROM %s WHERE partitionKey = ? AND clustering_1 = ? AND clustering_2 = ?", 0, 1, 1);
- // multiple time same primary key element in WHERE clause
- assertInvalidMessage("clustering_1 cannot be restricted by more than one relation if it includes an Equal",
- "DELETE FROM %s WHERE partitionKey = ? AND clustering_1 = ? AND clustering_2 = ? AND clustering_1 = ?", 0, 1, 1, 1);
+ assertInvalidMessage("Undefined column name partitionkey1",
+ "DELETE FROM %s WHERE partitionKey1 = ? AND clustering_1 = ? AND clustering_2 = ?", 0, 1, 1);
- // unknown identifiers
- assertInvalidMessage("Undefined column name value1",
- "DELETE value1 FROM %s WHERE partitionKey = ? AND clustering_1 = ? AND clustering_2 = ?", 0, 1, 1);
+ assertInvalidMessage("Undefined column name clustering_3",
+ "DELETE FROM %s WHERE partitionKey = ? AND clustering_1 = ? AND clustering_3 = ?", 0, 1, 1);
- assertInvalidMessage("Undefined column name partitionkey1",
- "DELETE FROM %s WHERE partitionKey1 = ? AND clustering_1 = ? AND clustering_2 = ?", 0, 1, 1);
+ // Invalid operator in the where clause
+ assertInvalidMessage("Only EQ and IN relation are supported on the partition key (unless you use the token() function)",
+ "DELETE FROM %s WHERE partitionKey > ? AND clustering_1 = ? AND clustering_2 = ?", 0, 1, 1);
- assertInvalidMessage("Undefined column name clustering_3",
- "DELETE FROM %s WHERE partitionKey = ? AND clustering_1 = ? AND clustering_3 = ?", 0, 1, 1);
+ assertInvalidMessage("Cannot use CONTAINS on non-collection column partitionkey",
+ "DELETE FROM %s WHERE partitionKey CONTAINS ? AND clustering_1 = ? AND clustering_2 = ?", 0, 1, 1);
- // Invalid operator in the where clause
- assertInvalidMessage("Only EQ and IN relation are supported on the partition key (unless you use the token() function)",
- "DELETE FROM %s WHERE partitionKey > ? AND clustering_1 = ? AND clustering_2 = ?", 0, 1, 1);
-
- assertInvalidMessage("Cannot use CONTAINS on non-collection column partitionkey",
- "DELETE FROM %s WHERE partitionKey CONTAINS ? AND clustering_1 = ? AND clustering_2 = ?", 0, 1, 1);
-
- // Non primary key in the where clause
- assertInvalidMessage("Non PRIMARY KEY columns found in where clause: value",
- "DELETE FROM %s WHERE partitionKey = ? AND clustering_1 = ? AND clustering_2 = ? AND value = ?", 0, 1, 1, 3);
- }
+ // Non primary key in the where clause
+ assertInvalidMessage("Non PRIMARY KEY columns found in where clause: value",
+ "DELETE FROM %s WHERE partitionKey = ? AND clustering_1 = ? AND clustering_2 = ? AND value = ?", 0, 1, 1, 3);
}
@Test
@@ -762,95 +699,92 @@
private void testDeleteWithRangeAndOneClusteringColumn(boolean forceFlush) throws Throwable
{
- for (String compactOption : new String[] { "", " WITH COMPACT STORAGE" })
- {
- createTable("CREATE TABLE %s (partitionKey int," +
- "clustering int," +
- "value int," +
- " PRIMARY KEY (partitionKey, clustering))" + compactOption);
+ createTable("CREATE TABLE %s (partitionKey int," +
+ "clustering int," +
+ "value int," +
+ " PRIMARY KEY (partitionKey, clustering))");
- int value = 0;
- for (int partitionKey = 0; partitionKey < 5; partitionKey++)
- for (int clustering1 = 0; clustering1 < 5; clustering1++)
- execute("INSERT INTO %s (partitionKey, clustering, value) VALUES (?, ?, ?)",
- partitionKey, clustering1, value++);
+ int value = 0;
+ for (int partitionKey = 0; partitionKey < 5; partitionKey++)
+ for (int clustering1 = 0; clustering1 < 5; clustering1++)
+ execute("INSERT INTO %s (partitionKey, clustering, value) VALUES (?, ?, ?)",
+ partitionKey, clustering1, value++);
- flush(forceFlush);
+ flush(forceFlush);
- // test delete partition
- execute("DELETE FROM %s WHERE partitionKey = ?", 1);
- flush(forceFlush);
- assertEmpty(execute("SELECT * FROM %s WHERE partitionKey = ?", 1));
+ // test delete partition
+ execute("DELETE FROM %s WHERE partitionKey = ?", 1);
+ flush(forceFlush);
+ assertEmpty(execute("SELECT * FROM %s WHERE partitionKey = ?", 1));
- // test slices on the first clustering column
+ // test slices on the first clustering column
- execute("DELETE FROM %s WHERE partitionKey = ? AND clustering >= ?", 0, 4);
- flush(forceFlush);
- assertRows(execute("SELECT * FROM %s WHERE partitionKey = ?", 0),
- row(0, 0, 0),
- row(0, 1, 1),
- row(0, 2, 2),
- row(0, 3, 3));
+ execute("DELETE FROM %s WHERE partitionKey = ? AND clustering >= ?", 0, 4);
+ flush(forceFlush);
+ assertRows(execute("SELECT * FROM %s WHERE partitionKey = ?", 0),
+ row(0, 0, 0),
+ row(0, 1, 1),
+ row(0, 2, 2),
+ row(0, 3, 3));
- execute("DELETE FROM %s WHERE partitionKey = ? AND clustering > ?", 0, 2);
- flush(forceFlush);
- assertRows(execute("SELECT * FROM %s WHERE partitionKey = ?", 0),
- row(0, 0, 0),
- row(0, 1, 1),
- row(0, 2, 2));
+ execute("DELETE FROM %s WHERE partitionKey = ? AND clustering > ?", 0, 2);
+ flush(forceFlush);
+ assertRows(execute("SELECT * FROM %s WHERE partitionKey = ?", 0),
+ row(0, 0, 0),
+ row(0, 1, 1),
+ row(0, 2, 2));
- execute("DELETE FROM %s WHERE partitionKey = ? AND clustering <= ?", 0, 0);
- flush(forceFlush);
- assertRows(execute("SELECT * FROM %s WHERE partitionKey = ?", 0),
- row(0, 1, 1),
- row(0, 2, 2));
+ execute("DELETE FROM %s WHERE partitionKey = ? AND clustering <= ?", 0, 0);
+ flush(forceFlush);
+ assertRows(execute("SELECT * FROM %s WHERE partitionKey = ?", 0),
+ row(0, 1, 1),
+ row(0, 2, 2));
- execute("DELETE FROM %s WHERE partitionKey = ? AND clustering < ?", 0, 2);
- flush(forceFlush);
- assertRows(execute("SELECT * FROM %s WHERE partitionKey = ?", 0),
- row(0, 2, 2));
+ execute("DELETE FROM %s WHERE partitionKey = ? AND clustering < ?", 0, 2);
+ flush(forceFlush);
+ assertRows(execute("SELECT * FROM %s WHERE partitionKey = ?", 0),
+ row(0, 2, 2));
- execute("DELETE FROM %s WHERE partitionKey = ? AND clustering >= ? AND clustering < ?", 2, 0, 3);
- flush(forceFlush);
- assertRows(execute("SELECT * FROM %s WHERE partitionKey = ?", 2),
- row(2, 3, 13),
- row(2, 4, 14));
+ execute("DELETE FROM %s WHERE partitionKey = ? AND clustering >= ? AND clustering < ?", 2, 0, 3);
+ flush(forceFlush);
+ assertRows(execute("SELECT * FROM %s WHERE partitionKey = ?", 2),
+ row(2, 3, 13),
+ row(2, 4, 14));
- execute("DELETE FROM %s WHERE partitionKey = ? AND clustering > ? AND clustering <= ?", 2, 3, 5);
- flush(forceFlush);
- assertRows(execute("SELECT * FROM %s WHERE partitionKey = ?", 2),
- row(2, 3, 13));
+ execute("DELETE FROM %s WHERE partitionKey = ? AND clustering > ? AND clustering <= ?", 2, 3, 5);
+ flush(forceFlush);
+ assertRows(execute("SELECT * FROM %s WHERE partitionKey = ?", 2),
+ row(2, 3, 13));
- execute("DELETE FROM %s WHERE partitionKey = ? AND clustering < ? AND clustering > ?", 2, 3, 5);
- flush(forceFlush);
- assertRows(execute("SELECT * FROM %s WHERE partitionKey = ?", 2),
- row(2, 3, 13));
+ execute("DELETE FROM %s WHERE partitionKey = ? AND clustering < ? AND clustering > ?", 2, 3, 5);
+ flush(forceFlush);
+ assertRows(execute("SELECT * FROM %s WHERE partitionKey = ?", 2),
+ row(2, 3, 13));
- // test multi-column slices
- execute("DELETE FROM %s WHERE partitionKey = ? AND (clustering) > (?)", 3, 2);
- flush(forceFlush);
- assertRows(execute("SELECT * FROM %s WHERE partitionKey = ?", 3),
- row(3, 0, 15),
- row(3, 1, 16),
- row(3, 2, 17));
+ // test multi-column slices
+ execute("DELETE FROM %s WHERE partitionKey = ? AND (clustering) > (?)", 3, 2);
+ flush(forceFlush);
+ assertRows(execute("SELECT * FROM %s WHERE partitionKey = ?", 3),
+ row(3, 0, 15),
+ row(3, 1, 16),
+ row(3, 2, 17));
- execute("DELETE FROM %s WHERE partitionKey = ? AND (clustering) < (?)", 3, 1);
- flush(forceFlush);
- assertRows(execute("SELECT * FROM %s WHERE partitionKey = ?", 3),
- row(3, 1, 16),
- row(3, 2, 17));
+ execute("DELETE FROM %s WHERE partitionKey = ? AND (clustering) < (?)", 3, 1);
+ flush(forceFlush);
+ assertRows(execute("SELECT * FROM %s WHERE partitionKey = ?", 3),
+ row(3, 1, 16),
+ row(3, 2, 17));
- execute("DELETE FROM %s WHERE partitionKey = ? AND (clustering) >= (?) AND (clustering) <= (?)", 3, 0, 1);
- flush(forceFlush);
- assertRows(execute("SELECT * FROM %s WHERE partitionKey = ?", 3),
- row(3, 2, 17));
+ execute("DELETE FROM %s WHERE partitionKey = ? AND (clustering) >= (?) AND (clustering) <= (?)", 3, 0, 1);
+ flush(forceFlush);
+ assertRows(execute("SELECT * FROM %s WHERE partitionKey = ?", 3),
+ row(3, 2, 17));
- // Test invalid queries
- assertInvalidMessage("Range deletions are not supported for specific columns",
- "DELETE value FROM %s WHERE partitionKey = ? AND clustering >= ?", 2, 1);
- assertInvalidMessage("Range deletions are not supported for specific columns",
- "DELETE value FROM %s WHERE partitionKey = ?", 2);
- }
+ // Test invalid queries
+ assertInvalidMessage("Range deletions are not supported for specific columns",
+ "DELETE value FROM %s WHERE partitionKey = ? AND clustering >= ?", 2, 1);
+ assertInvalidMessage("Range deletions are not supported for specific columns",
+ "DELETE value FROM %s WHERE partitionKey = ?", 2);
}
@Test
@@ -862,194 +796,193 @@
private void testDeleteWithRangeAndTwoClusteringColumns(boolean forceFlush) throws Throwable
{
- for (String compactOption : new String[] { "", " WITH COMPACT STORAGE" })
- {
- createTable("CREATE TABLE %s (partitionKey int," +
+ createTable("CREATE TABLE %s (partitionKey int," +
"clustering_1 int," +
"clustering_2 int," +
"value int," +
- " PRIMARY KEY (partitionKey, clustering_1, clustering_2))" + compactOption);
+ " PRIMARY KEY (partitionKey, clustering_1, clustering_2))");
- int value = 0;
- for (int partitionKey = 0; partitionKey < 5; partitionKey++)
- for (int clustering1 = 0; clustering1 < 5; clustering1++)
- for (int clustering2 = 0; clustering2 < 5; clustering2++) {
- execute("INSERT INTO %s (partitionKey, clustering_1, clustering_2, value) VALUES (?, ?, ?, ?)",
- partitionKey, clustering1, clustering2, value++);}
- flush(forceFlush);
+ int value = 0;
+ for (int partitionKey = 0; partitionKey < 5; partitionKey++)
+ for (int clustering1 = 0; clustering1 < 5; clustering1++)
+ for (int clustering2 = 0; clustering2 < 5; clustering2++)
+ {
+ execute("INSERT INTO %s (partitionKey, clustering_1, clustering_2, value) VALUES (?, ?, ?, ?)",
+ partitionKey, clustering1, clustering2, value++);
+ }
+ flush(forceFlush);
- // test unspecified second clustering column
- execute("DELETE FROM %s WHERE partitionKey = ? AND clustering_1 = ?", 0, 1);
- flush(forceFlush);
- assertRows(execute("SELECT * FROM %s WHERE partitionKey = ? AND clustering_1 < ?", 0, 2),
- row(0, 0, 0, 0),
- row(0, 0, 1, 1),
- row(0, 0, 2, 2),
- row(0, 0, 3, 3),
- row(0, 0, 4, 4));
+ // test unspecified second clustering column
+ execute("DELETE FROM %s WHERE partitionKey = ? AND clustering_1 = ?", 0, 1);
+ flush(forceFlush);
+ assertRows(execute("SELECT * FROM %s WHERE partitionKey = ? AND clustering_1 < ?", 0, 2),
+ row(0, 0, 0, 0),
+ row(0, 0, 1, 1),
+ row(0, 0, 2, 2),
+ row(0, 0, 3, 3),
+ row(0, 0, 4, 4));
- // test delete partition
- execute("DELETE FROM %s WHERE partitionKey = ?", 1);
- flush(forceFlush);
- assertEmpty(execute("SELECT * FROM %s WHERE partitionKey = ?", 1));
+ // test delete partition
+ execute("DELETE FROM %s WHERE partitionKey = ?", 1);
+ flush(forceFlush);
+ assertEmpty(execute("SELECT * FROM %s WHERE partitionKey = ?", 1));
- // test slices on the second clustering column
+ // test slices on the second clustering column
- execute("DELETE FROM %s WHERE partitionKey = ? AND clustering_1 = ? AND clustering_2 < ?", 0, 0, 2);
- flush(forceFlush);
- assertRows(execute("SELECT * FROM %s WHERE partitionKey = ? AND clustering_1 < ?", 0, 2),
- row(0, 0, 2, 2),
- row(0, 0, 3, 3),
- row(0, 0, 4, 4));
+ execute("DELETE FROM %s WHERE partitionKey = ? AND clustering_1 = ? AND clustering_2 < ?", 0, 0, 2);
+ flush(forceFlush);
+ assertRows(execute("SELECT * FROM %s WHERE partitionKey = ? AND clustering_1 < ?", 0, 2),
+ row(0, 0, 2, 2),
+ row(0, 0, 3, 3),
+ row(0, 0, 4, 4));
- execute("DELETE FROM %s WHERE partitionKey = ? AND clustering_1 = ? AND clustering_2 <= ?", 0, 0, 3);
- flush(forceFlush);
- assertRows(execute("SELECT * FROM %s WHERE partitionKey = ? AND clustering_1 < ?", 0, 2),
- row(0, 0, 4, 4));
+ execute("DELETE FROM %s WHERE partitionKey = ? AND clustering_1 = ? AND clustering_2 <= ?", 0, 0, 3);
+ flush(forceFlush);
+ assertRows(execute("SELECT * FROM %s WHERE partitionKey = ? AND clustering_1 < ?", 0, 2),
+ row(0, 0, 4, 4));
- execute("DELETE FROM %s WHERE partitionKey = ? AND clustering_1 = ? AND clustering_2 > ? ", 0, 2, 2);
- flush(forceFlush);
- assertRows(execute("SELECT * FROM %s WHERE partitionKey = ? AND clustering_1 = ?", 0, 2),
- row(0, 2, 0, 10),
- row(0, 2, 1, 11),
- row(0, 2, 2, 12));
+ execute("DELETE FROM %s WHERE partitionKey = ? AND clustering_1 = ? AND clustering_2 > ? ", 0, 2, 2);
+ flush(forceFlush);
+ assertRows(execute("SELECT * FROM %s WHERE partitionKey = ? AND clustering_1 = ?", 0, 2),
+ row(0, 2, 0, 10),
+ row(0, 2, 1, 11),
+ row(0, 2, 2, 12));
- execute("DELETE FROM %s WHERE partitionKey = ? AND clustering_1 = ? AND clustering_2 >= ? ", 0, 2, 1);
- flush(forceFlush);
- assertRows(execute("SELECT * FROM %s WHERE partitionKey = ? AND clustering_1 = ?", 0, 2),
- row(0, 2, 0, 10));
+ execute("DELETE FROM %s WHERE partitionKey = ? AND clustering_1 = ? AND clustering_2 >= ? ", 0, 2, 1);
+ flush(forceFlush);
+ assertRows(execute("SELECT * FROM %s WHERE partitionKey = ? AND clustering_1 = ?", 0, 2),
+ row(0, 2, 0, 10));
- execute("DELETE FROM %s WHERE partitionKey = ? AND clustering_1 = ? AND clustering_2 > ? AND clustering_2 < ? ",
- 0, 3, 1, 4);
- flush(forceFlush);
- assertRows(execute("SELECT * FROM %s WHERE partitionKey = ? AND clustering_1 = ?", 0, 3),
- row(0, 3, 0, 15),
- row(0, 3, 1, 16),
- row(0, 3, 4, 19));
+ execute("DELETE FROM %s WHERE partitionKey = ? AND clustering_1 = ? AND clustering_2 > ? AND clustering_2 < ? ",
+ 0, 3, 1, 4);
+ flush(forceFlush);
+ assertRows(execute("SELECT * FROM %s WHERE partitionKey = ? AND clustering_1 = ?", 0, 3),
+ row(0, 3, 0, 15),
+ row(0, 3, 1, 16),
+ row(0, 3, 4, 19));
- execute("DELETE FROM %s WHERE partitionKey = ? AND clustering_1 = ? AND clustering_2 > ? AND clustering_2 < ? ",
- 0, 3, 4, 1);
- flush(forceFlush);
- assertRows(execute("SELECT * FROM %s WHERE partitionKey = ? AND clustering_1 = ?", 0, 3),
- row(0, 3, 0, 15),
- row(0, 3, 1, 16),
- row(0, 3, 4, 19));
+ execute("DELETE FROM %s WHERE partitionKey = ? AND clustering_1 = ? AND clustering_2 > ? AND clustering_2 < ? ",
+ 0, 3, 4, 1);
+ flush(forceFlush);
+ assertRows(execute("SELECT * FROM %s WHERE partitionKey = ? AND clustering_1 = ?", 0, 3),
+ row(0, 3, 0, 15),
+ row(0, 3, 1, 16),
+ row(0, 3, 4, 19));
- execute("DELETE FROM %s WHERE partitionKey = ? AND clustering_1 = ? AND clustering_2 >= ? AND clustering_2 <= ? ",
- 0, 3, 1, 4);
- flush(forceFlush);
- assertRows(execute("SELECT * FROM %s WHERE partitionKey = ? AND clustering_1 = ?", 0, 3),
- row(0, 3, 0, 15));
+ execute("DELETE FROM %s WHERE partitionKey = ? AND clustering_1 = ? AND clustering_2 >= ? AND clustering_2 <= ? ",
+ 0, 3, 1, 4);
+ flush(forceFlush);
+ assertRows(execute("SELECT * FROM %s WHERE partitionKey = ? AND clustering_1 = ?", 0, 3),
+ row(0, 3, 0, 15));
- // test slices on the first clustering column
+ // test slices on the first clustering column
- execute("DELETE FROM %s WHERE partitionKey = ? AND clustering_1 >= ?", 0, 4);
- flush(forceFlush);
- assertRows(execute("SELECT * FROM %s WHERE partitionKey = ?", 0),
- row(0, 0, 4, 4),
- row(0, 2, 0, 10),
- row(0, 3, 0, 15));
+ execute("DELETE FROM %s WHERE partitionKey = ? AND clustering_1 >= ?", 0, 4);
+ flush(forceFlush);
+ assertRows(execute("SELECT * FROM %s WHERE partitionKey = ?", 0),
+ row(0, 0, 4, 4),
+ row(0, 2, 0, 10),
+ row(0, 3, 0, 15));
- execute("DELETE FROM %s WHERE partitionKey = ? AND clustering_1 > ?", 0, 3);
- flush(forceFlush);
- assertRows(execute("SELECT * FROM %s WHERE partitionKey = ?", 0),
- row(0, 0, 4, 4),
- row(0, 2, 0, 10),
- row(0, 3, 0, 15));
+ execute("DELETE FROM %s WHERE partitionKey = ? AND clustering_1 > ?", 0, 3);
+ flush(forceFlush);
+ assertRows(execute("SELECT * FROM %s WHERE partitionKey = ?", 0),
+ row(0, 0, 4, 4),
+ row(0, 2, 0, 10),
+ row(0, 3, 0, 15));
- execute("DELETE FROM %s WHERE partitionKey = ? AND clustering_1 < ?", 0, 3);
- flush(forceFlush);
- assertRows(execute("SELECT * FROM %s WHERE partitionKey = ?", 0),
- row(0, 3, 0, 15));
+ execute("DELETE FROM %s WHERE partitionKey = ? AND clustering_1 < ?", 0, 3);
+ flush(forceFlush);
+ assertRows(execute("SELECT * FROM %s WHERE partitionKey = ?", 0),
+ row(0, 3, 0, 15));
- execute("DELETE FROM %s WHERE partitionKey = ? AND clustering_1 >= ? AND clustering_1 < ?", 2, 0, 3);
- flush(forceFlush);
- assertRows(execute("SELECT * FROM %s WHERE partitionKey = ?", 2),
- row(2, 3, 0, 65),
- row(2, 3, 1, 66),
- row(2, 3, 2, 67),
- row(2, 3, 3, 68),
- row(2, 3, 4, 69),
- row(2, 4, 0, 70),
- row(2, 4, 1, 71),
- row(2, 4, 2, 72),
- row(2, 4, 3, 73),
- row(2, 4, 4, 74));
+ execute("DELETE FROM %s WHERE partitionKey = ? AND clustering_1 >= ? AND clustering_1 < ?", 2, 0, 3);
+ flush(forceFlush);
+ assertRows(execute("SELECT * FROM %s WHERE partitionKey = ?", 2),
+ row(2, 3, 0, 65),
+ row(2, 3, 1, 66),
+ row(2, 3, 2, 67),
+ row(2, 3, 3, 68),
+ row(2, 3, 4, 69),
+ row(2, 4, 0, 70),
+ row(2, 4, 1, 71),
+ row(2, 4, 2, 72),
+ row(2, 4, 3, 73),
+ row(2, 4, 4, 74));
- execute("DELETE FROM %s WHERE partitionKey = ? AND clustering_1 > ? AND clustering_1 <= ?", 2, 3, 5);
- flush(forceFlush);
- assertRows(execute("SELECT * FROM %s WHERE partitionKey = ?", 2),
- row(2, 3, 0, 65),
- row(2, 3, 1, 66),
- row(2, 3, 2, 67),
- row(2, 3, 3, 68),
- row(2, 3, 4, 69));
+ execute("DELETE FROM %s WHERE partitionKey = ? AND clustering_1 > ? AND clustering_1 <= ?", 2, 3, 5);
+ flush(forceFlush);
+ assertRows(execute("SELECT * FROM %s WHERE partitionKey = ?", 2),
+ row(2, 3, 0, 65),
+ row(2, 3, 1, 66),
+ row(2, 3, 2, 67),
+ row(2, 3, 3, 68),
+ row(2, 3, 4, 69));
- execute("DELETE FROM %s WHERE partitionKey = ? AND clustering_1 < ? AND clustering_1 > ?", 2, 3, 5);
- flush(forceFlush);
- assertRows(execute("SELECT * FROM %s WHERE partitionKey = ?", 2),
- row(2, 3, 0, 65),
- row(2, 3, 1, 66),
- row(2, 3, 2, 67),
- row(2, 3, 3, 68),
- row(2, 3, 4, 69));
+ execute("DELETE FROM %s WHERE partitionKey = ? AND clustering_1 < ? AND clustering_1 > ?", 2, 3, 5);
+ flush(forceFlush);
+ assertRows(execute("SELECT * FROM %s WHERE partitionKey = ?", 2),
+ row(2, 3, 0, 65),
+ row(2, 3, 1, 66),
+ row(2, 3, 2, 67),
+ row(2, 3, 3, 68),
+ row(2, 3, 4, 69));
- // test multi-column slices
- execute("DELETE FROM %s WHERE partitionKey = ? AND (clustering_1, clustering_2) > (?, ?)", 2, 3, 3);
- flush(forceFlush);
- assertRows(execute("SELECT * FROM %s WHERE partitionKey = ?", 2),
- row(2, 3, 0, 65),
- row(2, 3, 1, 66),
- row(2, 3, 2, 67),
- row(2, 3, 3, 68));
+ // test multi-column slices
+ execute("DELETE FROM %s WHERE partitionKey = ? AND (clustering_1, clustering_2) > (?, ?)", 2, 3, 3);
+ flush(forceFlush);
+ assertRows(execute("SELECT * FROM %s WHERE partitionKey = ?", 2),
+ row(2, 3, 0, 65),
+ row(2, 3, 1, 66),
+ row(2, 3, 2, 67),
+ row(2, 3, 3, 68));
- execute("DELETE FROM %s WHERE partitionKey = ? AND (clustering_1, clustering_2) < (?, ?)", 2, 3, 1);
- flush(forceFlush);
- assertRows(execute("SELECT * FROM %s WHERE partitionKey = ?", 2),
- row(2, 3, 1, 66),
- row(2, 3, 2, 67),
- row(2, 3, 3, 68));
+ execute("DELETE FROM %s WHERE partitionKey = ? AND (clustering_1, clustering_2) < (?, ?)", 2, 3, 1);
+ flush(forceFlush);
+ assertRows(execute("SELECT * FROM %s WHERE partitionKey = ?", 2),
+ row(2, 3, 1, 66),
+ row(2, 3, 2, 67),
+ row(2, 3, 3, 68));
- execute("DELETE FROM %s WHERE partitionKey = ? AND (clustering_1, clustering_2) >= (?, ?) AND (clustering_1) <= (?)", 2, 3, 2, 4);
- flush(forceFlush);
- assertRows(execute("SELECT * FROM %s WHERE partitionKey = ?", 2),
- row(2, 3, 1, 66));
+ execute("DELETE FROM %s WHERE partitionKey = ? AND (clustering_1, clustering_2) >= (?, ?) AND (clustering_1) <= (?)", 2, 3, 2, 4);
+ flush(forceFlush);
+ assertRows(execute("SELECT * FROM %s WHERE partitionKey = ?", 2),
+ row(2, 3, 1, 66));
- // Test with a mix of single column and multi-column restrictions
- execute("DELETE FROM %s WHERE partitionKey = ? AND clustering_1 = ? AND (clustering_2) < (?)", 3, 0, 3);
- flush(forceFlush);
- assertRows(execute("SELECT * FROM %s WHERE partitionKey = ? AND clustering_1 = ?", 3, 0),
- row(3, 0, 3, 78),
- row(3, 0, 4, 79));
+ // Test with a mix of single column and multi-column restrictions
+ execute("DELETE FROM %s WHERE partitionKey = ? AND clustering_1 = ? AND (clustering_2) < (?)", 3, 0, 3);
+ flush(forceFlush);
+ assertRows(execute("SELECT * FROM %s WHERE partitionKey = ? AND clustering_1 = ?", 3, 0),
+ row(3, 0, 3, 78),
+ row(3, 0, 4, 79));
- execute("DELETE FROM %s WHERE partitionKey = ? AND clustering_1 IN (?, ?) AND (clustering_2) >= (?)", 3, 0, 1, 3);
- flush(forceFlush);
- assertRows(execute("SELECT * FROM %s WHERE partitionKey = ? AND clustering_1 IN (?, ?)", 3, 0, 1),
- row(3, 1, 0, 80),
- row(3, 1, 1, 81),
- row(3, 1, 2, 82));
+ execute("DELETE FROM %s WHERE partitionKey = ? AND clustering_1 IN (?, ?) AND (clustering_2) >= (?)", 3, 0, 1, 3);
+ flush(forceFlush);
+ assertRows(execute("SELECT * FROM %s WHERE partitionKey = ? AND clustering_1 IN (?, ?)", 3, 0, 1),
+ row(3, 1, 0, 80),
+ row(3, 1, 1, 81),
+ row(3, 1, 2, 82));
- execute("DELETE FROM %s WHERE partitionKey = ? AND (clustering_1) IN ((?), (?)) AND clustering_2 < ?", 3, 0, 1, 1);
- flush(forceFlush);
- assertRows(execute("SELECT * FROM %s WHERE partitionKey = ? AND clustering_1 IN (?, ?)", 3, 0, 1),
- row(3, 1, 1, 81),
- row(3, 1, 2, 82));
+ execute("DELETE FROM %s WHERE partitionKey = ? AND (clustering_1) IN ((?), (?)) AND clustering_2 < ?", 3, 0, 1, 1);
+ flush(forceFlush);
+ assertRows(execute("SELECT * FROM %s WHERE partitionKey = ? AND clustering_1 IN (?, ?)", 3, 0, 1),
+ row(3, 1, 1, 81),
+ row(3, 1, 2, 82));
- execute("DELETE FROM %s WHERE partitionKey = ? AND (clustering_1) = (?) AND clustering_2 >= ?", 3, 1, 2);
- flush(forceFlush);
- assertRows(execute("SELECT * FROM %s WHERE partitionKey = ? AND clustering_1 IN (?, ?)", 3, 0, 1),
- row(3, 1, 1, 81));
+ execute("DELETE FROM %s WHERE partitionKey = ? AND (clustering_1) = (?) AND clustering_2 >= ?", 3, 1, 2);
+ flush(forceFlush);
+ assertRows(execute("SELECT * FROM %s WHERE partitionKey = ? AND clustering_1 IN (?, ?)", 3, 0, 1),
+ row(3, 1, 1, 81));
- // Test invalid queries
- assertInvalidMessage("Range deletions are not supported for specific columns",
- "DELETE value FROM %s WHERE partitionKey = ? AND (clustering_1, clustering_2) >= (?, ?)", 2, 3, 1);
- assertInvalidMessage("Range deletions are not supported for specific columns",
- "DELETE value FROM %s WHERE partitionKey = ? AND clustering_1 >= ?", 2, 3);
- assertInvalidMessage("Range deletions are not supported for specific columns",
- "DELETE value FROM %s WHERE partitionKey = ? AND clustering_1 = ?", 2, 3);
- assertInvalidMessage("Range deletions are not supported for specific columns",
- "DELETE value FROM %s WHERE partitionKey = ?", 2);
- }
+ // Test invalid queries
+ assertInvalidMessage("Range deletions are not supported for specific columns",
+ "DELETE value FROM %s WHERE partitionKey = ? AND (clustering_1, clustering_2) >= (?, ?)", 2, 3, 1);
+ assertInvalidMessage("Range deletions are not supported for specific columns",
+ "DELETE value FROM %s WHERE partitionKey = ? AND clustering_1 >= ?", 2, 3);
+ assertInvalidMessage("Range deletions are not supported for specific columns",
+ "DELETE value FROM %s WHERE partitionKey = ? AND clustering_1 = ?", 2, 3);
+ assertInvalidMessage("Range deletions are not supported for specific columns",
+ "DELETE value FROM %s WHERE partitionKey = ?", 2);
}
@Test
@@ -1217,109 +1150,93 @@
@Test
public void testDeleteWithEmptyRestrictionValue() throws Throwable
{
- for (String options : new String[] { "", " WITH COMPACT STORAGE" })
- {
- createTable("CREATE TABLE %s (pk blob, c blob, v blob, PRIMARY KEY (pk, c))" + options);
+ createTable("CREATE TABLE %s (pk blob, c blob, v blob, PRIMARY KEY (pk, c))");
- if (StringUtils.isEmpty(options))
- {
- execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?)", bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("1"));
- execute("DELETE FROM %s WHERE pk = textAsBlob('foo123') AND c = textAsBlob('');");
+ execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?)", bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("1"));
+ execute("DELETE FROM %s WHERE pk = textAsBlob('foo123') AND c = textAsBlob('');");
- assertEmpty(execute("SELECT * FROM %s"));
+ assertEmpty(execute("SELECT * FROM %s"));
- execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?)", bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("1"));
- execute("DELETE FROM %s WHERE pk = textAsBlob('foo123') AND c IN (textAsBlob(''), textAsBlob('1'));");
+ execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?)", bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("1"));
+ execute("DELETE FROM %s WHERE pk = textAsBlob('foo123') AND c IN (textAsBlob(''), textAsBlob('1'));");
- assertEmpty(execute("SELECT * FROM %s"));
+ assertEmpty(execute("SELECT * FROM %s"));
- execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?)", bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("1"));
- execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?)", bytes("foo123"), bytes("1"), bytes("1"));
- execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?)", bytes("foo123"), bytes("2"), bytes("2"));
+ execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?)", bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("1"));
+ execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?)", bytes("foo123"), bytes("1"), bytes("1"));
+ execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?)", bytes("foo123"), bytes("2"), bytes("2"));
- execute("DELETE FROM %s WHERE pk = textAsBlob('foo123') AND c > textAsBlob('')");
+ execute("DELETE FROM %s WHERE pk = textAsBlob('foo123') AND c > textAsBlob('')");
- assertRows(execute("SELECT * FROM %s"),
- row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("1")));
+ assertRows(execute("SELECT * FROM %s"),
+ row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("1")));
- execute("DELETE FROM %s WHERE pk = textAsBlob('foo123') AND c >= textAsBlob('')");
+ execute("DELETE FROM %s WHERE pk = textAsBlob('foo123') AND c >= textAsBlob('')");
- assertEmpty(execute("SELECT * FROM %s"));
- }
- else
- {
- assertInvalid("Invalid empty or null value for column c",
- "DELETE FROM %s WHERE pk = textAsBlob('foo123') AND c = textAsBlob('')");
- assertInvalid("Invalid empty or null value for column c",
- "DELETE FROM %s WHERE pk = textAsBlob('foo123') AND c IN (textAsBlob(''), textAsBlob('1'))");
- }
+ assertEmpty(execute("SELECT * FROM %s"));
- execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?)", bytes("foo123"), bytes("1"), bytes("1"));
- execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?)", bytes("foo123"), bytes("2"), bytes("2"));
+ execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?)", bytes("foo123"), bytes("1"), bytes("1"));
+ execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?)", bytes("foo123"), bytes("2"), bytes("2"));
- execute("DELETE FROM %s WHERE pk = textAsBlob('foo123') AND c > textAsBlob('')");
+ execute("DELETE FROM %s WHERE pk = textAsBlob('foo123') AND c > textAsBlob('')");
- assertEmpty(execute("SELECT * FROM %s"));
+ assertEmpty(execute("SELECT * FROM %s"));
- execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?)", bytes("foo123"), bytes("1"), bytes("1"));
- execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?)", bytes("foo123"), bytes("2"), bytes("2"));
+ execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?)", bytes("foo123"), bytes("1"), bytes("1"));
+ execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?)", bytes("foo123"), bytes("2"), bytes("2"));
- execute("DELETE FROM %s WHERE pk = textAsBlob('foo123') AND c <= textAsBlob('')");
- execute("DELETE FROM %s WHERE pk = textAsBlob('foo123') AND c < textAsBlob('')");
+ execute("DELETE FROM %s WHERE pk = textAsBlob('foo123') AND c <= textAsBlob('')");
+ execute("DELETE FROM %s WHERE pk = textAsBlob('foo123') AND c < textAsBlob('')");
- assertRows(execute("SELECT * FROM %s"),
- row(bytes("foo123"), bytes("1"), bytes("1")),
- row(bytes("foo123"), bytes("2"), bytes("2")));
- }
+ assertRows(execute("SELECT * FROM %s"),
+ row(bytes("foo123"), bytes("1"), bytes("1")),
+ row(bytes("foo123"), bytes("2"), bytes("2")));
}
@Test
public void testDeleteWithMultipleClusteringColumnsAndEmptyRestrictionValue() throws Throwable
{
- for (String options : new String[] { "", " WITH COMPACT STORAGE" })
- {
- createTable("CREATE TABLE %s (pk blob, c1 blob, c2 blob, v blob, PRIMARY KEY (pk, c1, c2))" + options);
+ createTable("CREATE TABLE %s (pk blob, c1 blob, c2 blob, v blob, PRIMARY KEY (pk, c1, c2))");
- execute("INSERT INTO %s (pk, c1, c2, v) VALUES (?, ?, ?, ?)", bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("1"), bytes("1"));
- execute("DELETE FROM %s WHERE pk = textAsBlob('foo123') AND c1 = textAsBlob('');");
+ execute("INSERT INTO %s (pk, c1, c2, v) VALUES (?, ?, ?, ?)", bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("1"), bytes("1"));
+ execute("DELETE FROM %s WHERE pk = textAsBlob('foo123') AND c1 = textAsBlob('');");
- assertEmpty(execute("SELECT * FROM %s"));
+ assertEmpty(execute("SELECT * FROM %s"));
- execute("INSERT INTO %s (pk, c1, c2, v) VALUES (?, ?, ?, ?)", bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("1"), bytes("1"));
- execute("DELETE FROM %s WHERE pk = textAsBlob('foo123') AND c1 IN (textAsBlob(''), textAsBlob('1')) AND c2 = textAsBlob('1');");
+ execute("INSERT INTO %s (pk, c1, c2, v) VALUES (?, ?, ?, ?)", bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("1"), bytes("1"));
+ execute("DELETE FROM %s WHERE pk = textAsBlob('foo123') AND c1 IN (textAsBlob(''), textAsBlob('1')) AND c2 = textAsBlob('1');");
- assertEmpty(execute("SELECT * FROM %s"));
+ assertEmpty(execute("SELECT * FROM %s"));
- execute("INSERT INTO %s (pk, c1, c2, v) VALUES (?, ?, ?, ?)", bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("1"), bytes("0"));
- execute("INSERT INTO %s (pk, c1, c2, v) VALUES (?, ?, ?, ?)", bytes("foo123"), bytes("1"), bytes("1"), bytes("1"));
- execute("INSERT INTO %s (pk, c1, c2, v) VALUES (?, ?, ?, ?)", bytes("foo123"), bytes("1"), bytes("2"), bytes("3"));
+ execute("INSERT INTO %s (pk, c1, c2, v) VALUES (?, ?, ?, ?)", bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("1"), bytes("0"));
+ execute("INSERT INTO %s (pk, c1, c2, v) VALUES (?, ?, ?, ?)", bytes("foo123"), bytes("1"), bytes("1"), bytes("1"));
+ execute("INSERT INTO %s (pk, c1, c2, v) VALUES (?, ?, ?, ?)", bytes("foo123"), bytes("1"), bytes("2"), bytes("3"));
- execute("DELETE FROM %s WHERE pk = textAsBlob('foo123') AND c1 > textAsBlob('')");
+ execute("DELETE FROM %s WHERE pk = textAsBlob('foo123') AND c1 > textAsBlob('')");
- assertRows(execute("SELECT * FROM %s"),
- row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("1"), bytes("0")));
+ assertRows(execute("SELECT * FROM %s"),
+ row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("1"), bytes("0")));
- execute("DELETE FROM %s WHERE pk = textAsBlob('foo123') AND c1 >= textAsBlob('')");
+ execute("DELETE FROM %s WHERE pk = textAsBlob('foo123') AND c1 >= textAsBlob('')");
- assertEmpty(execute("SELECT * FROM %s"));
+ assertEmpty(execute("SELECT * FROM %s"));
- execute("INSERT INTO %s (pk, c1, c2, v) VALUES (?, ?, ?, ?)", bytes("foo123"), bytes("1"), bytes("1"), bytes("1"));
- execute("INSERT INTO %s (pk, c1, c2, v) VALUES (?, ?, ?, ?)", bytes("foo123"), bytes("1"), bytes("2"), bytes("3"));
+ execute("INSERT INTO %s (pk, c1, c2, v) VALUES (?, ?, ?, ?)", bytes("foo123"), bytes("1"), bytes("1"), bytes("1"));
+ execute("INSERT INTO %s (pk, c1, c2, v) VALUES (?, ?, ?, ?)", bytes("foo123"), bytes("1"), bytes("2"), bytes("3"));
- execute("DELETE FROM %s WHERE pk = textAsBlob('foo123') AND c1 > textAsBlob('')");
+ execute("DELETE FROM %s WHERE pk = textAsBlob('foo123') AND c1 > textAsBlob('')");
- assertEmpty(execute("SELECT * FROM %s"));
+ assertEmpty(execute("SELECT * FROM %s"));
- execute("INSERT INTO %s (pk, c1, c2, v) VALUES (?, ?, ?, ?)", bytes("foo123"), bytes("1"), bytes("1"), bytes("1"));
- execute("INSERT INTO %s (pk, c1, c2, v) VALUES (?, ?, ?, ?)", bytes("foo123"), bytes("1"), bytes("2"), bytes("3"));
+ execute("INSERT INTO %s (pk, c1, c2, v) VALUES (?, ?, ?, ?)", bytes("foo123"), bytes("1"), bytes("1"), bytes("1"));
+ execute("INSERT INTO %s (pk, c1, c2, v) VALUES (?, ?, ?, ?)", bytes("foo123"), bytes("1"), bytes("2"), bytes("3"));
- execute("DELETE FROM %s WHERE pk = textAsBlob('foo123') AND c1 <= textAsBlob('')");
- execute("DELETE FROM %s WHERE pk = textAsBlob('foo123') AND c1 < textAsBlob('')");
+ execute("DELETE FROM %s WHERE pk = textAsBlob('foo123') AND c1 <= textAsBlob('')");
+ execute("DELETE FROM %s WHERE pk = textAsBlob('foo123') AND c1 < textAsBlob('')");
- assertRows(execute("SELECT * FROM %s"),
- row(bytes("foo123"), bytes("1"), bytes("1"), bytes("1")),
- row(bytes("foo123"), bytes("1"), bytes("2"), bytes("3")));
- }
+ assertRows(execute("SELECT * FROM %s"),
+ row(bytes("foo123"), bytes("1"), bytes("1"), bytes("1")),
+ row(bytes("foo123"), bytes("1"), bytes("2"), bytes("3")));
}
/**
@@ -1506,70 +1423,6 @@
execute("DELETE FROM %s WHERE k = ? AND a >= ? AND a < ?", "a", 0, 2);
}
- /**
- * Test for CASSANDRA-13917
- */
- @Test
- public void testWithCompactStaticFormat() throws Throwable
- {
- createTable("CREATE TABLE %s (a int PRIMARY KEY, b int, c int) WITH COMPACT STORAGE");
- testWithCompactFormat();
-
- // if column1 is present, hidden column is called column2
- createTable("CREATE TABLE %s (a int PRIMARY KEY, b int, c int, column1 int) WITH COMPACT STORAGE");
- assertInvalidMessage("Undefined column name column2",
- "DELETE FROM %s WHERE a = 1 AND column2= 1");
- assertInvalidMessage("Undefined column name column2",
- "DELETE FROM %s WHERE a = 1 AND column2 = 1 AND value1 = 1");
- assertInvalidMessage("Undefined column name column2",
- "DELETE column2 FROM %s WHERE a = 1");
-
- // if value is present, hidden column is called value1
- createTable("CREATE TABLE %s (a int PRIMARY KEY, b int, c int, value int) WITH COMPACT STORAGE");
- assertInvalidMessage("Undefined column name value1",
- "DELETE FROM %s WHERE a = 1 AND value1 = 1");
- assertInvalidMessage("Undefined column name value1",
- "DELETE FROM %s WHERE a = 1 AND value1 = 1 AND column1 = 1");
- assertInvalidMessage("Undefined column name value1",
- "DELETE value1 FROM %s WHERE a = 1");
- }
-
- /**
- * Test for CASSANDRA-13917
- */
- @Test
- public void testWithCompactNonStaticFormat() throws Throwable
- {
- createTable("CREATE TABLE %s (a int, b int, PRIMARY KEY (a, b)) WITH COMPACT STORAGE");
- execute("INSERT INTO %s (a, b) VALUES (1, 1)");
- execute("INSERT INTO %s (a, b) VALUES (2, 1)");
- assertRows(execute("SELECT a, b FROM %s"),
- row(1, 1),
- row(2, 1));
- testWithCompactFormat();
-
- createTable("CREATE TABLE %s (a int, b int, v int, PRIMARY KEY (a, b)) WITH COMPACT STORAGE");
- execute("INSERT INTO %s (a, b, v) VALUES (1, 1, 3)");
- execute("INSERT INTO %s (a, b, v) VALUES (2, 1, 4)");
- assertRows(execute("SELECT a, b, v FROM %s"),
- row(1, 1, 3),
- row(2, 1, 4));
- testWithCompactFormat();
- }
-
- private void testWithCompactFormat() throws Throwable
- {
- assertInvalidMessage("Undefined column name value",
- "DELETE FROM %s WHERE a = 1 AND value = 1");
- assertInvalidMessage("Undefined column name column1",
- "DELETE FROM %s WHERE a = 1 AND column1= 1");
- assertInvalidMessage("Undefined column name value",
- "DELETE FROM %s WHERE a = 1 AND value = 1 AND column1 = 1");
- assertInvalidMessage("Undefined column name value",
- "DELETE value FROM %s WHERE a = 1");
- assertInvalidMessage("Undefined column name column1",
- "DELETE column1 FROM %s WHERE a = 1");
- }
/**
* Checks if the memtable is empty or not
diff --git a/test/unit/org/apache/cassandra/cql3/validation/operations/DropCompactStorageThriftTest.java b/test/unit/org/apache/cassandra/cql3/validation/operations/DropCompactStorageThriftTest.java
deleted file mode 100644
index 973412a..0000000
--- a/test/unit/org/apache/cassandra/cql3/validation/operations/DropCompactStorageThriftTest.java
+++ /dev/null
@@ -1,641 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.cql3.validation.operations;
-
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import java.util.UUID;
-
-import org.junit.Test;
-
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
-import org.apache.cassandra.cql3.ColumnSpecification;
-import org.apache.cassandra.cql3.UntypedResultSet;
-import org.apache.cassandra.cql3.statements.IndexTarget;
-import org.apache.cassandra.db.Keyspace;
-import org.apache.cassandra.db.marshal.AbstractType;
-import org.apache.cassandra.db.marshal.AsciiType;
-import org.apache.cassandra.db.marshal.BytesType;
-import org.apache.cassandra.db.marshal.CompositeType;
-import org.apache.cassandra.db.marshal.EmptyType;
-import org.apache.cassandra.db.marshal.Int32Type;
-import org.apache.cassandra.db.marshal.MapType;
-import org.apache.cassandra.db.marshal.UTF8Type;
-import org.apache.cassandra.index.TargetParser;
-import org.apache.cassandra.locator.SimpleStrategy;
-import org.apache.cassandra.serializers.MarshalException;
-import org.apache.cassandra.thrift.Cassandra;
-import org.apache.cassandra.thrift.CfDef;
-import org.apache.cassandra.thrift.Column;
-import org.apache.cassandra.thrift.ColumnDef;
-import org.apache.cassandra.thrift.ColumnOrSuperColumn;
-import org.apache.cassandra.thrift.ColumnParent;
-import org.apache.cassandra.thrift.IndexType;
-import org.apache.cassandra.thrift.KsDef;
-import org.apache.cassandra.thrift.Mutation;
-import org.apache.cassandra.thrift.SuperColumn;
-import org.apache.cassandra.utils.ByteBufferUtil;
-import org.apache.cassandra.utils.Pair;
-import org.apache.cassandra.utils.UUIDGen;
-
-import static junit.framework.Assert.assertFalse;
-import static junit.framework.Assert.assertTrue;
-import static org.apache.cassandra.thrift.ConsistencyLevel.ONE;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-
-public class DropCompactStorageThriftTest extends ThriftCQLTester
-{
- @Test
- public void thriftCreatedTableTest() throws Throwable
- {
- final String KEYSPACE = "thrift_created_table_test_ks";
- final String TABLE = "test_table_1";
-
- CfDef cfDef = new CfDef().setDefault_validation_class(Int32Type.instance.toString())
- .setKey_validation_class(AsciiType.instance.toString())
- .setComparator_type(AsciiType.instance.toString())
- .setColumn_metadata(Arrays.asList(new ColumnDef(ByteBufferUtil.bytes("col1"),
- AsciiType.instance.toString())
- .setIndex_name("col1Index")
- .setIndex_type(IndexType.KEYS),
- new ColumnDef(ByteBufferUtil.bytes("col2"),
- AsciiType.instance.toString())
- .setIndex_name("col2Index")
- .setIndex_type(IndexType.KEYS)))
- .setKeyspace(KEYSPACE)
- .setName(TABLE);
-
- KsDef ksDef = new KsDef(KEYSPACE,
- SimpleStrategy.class.getName(),
- Arrays.asList(cfDef));
- ksDef.setStrategy_options(Collections.singletonMap("replication_factor", "1"));
-
- Cassandra.Client client = getClient();
- client.system_add_keyspace(ksDef);
- client.set_keyspace(KEYSPACE);
-
- client.insert(UTF8Type.instance.decompose("key1"),
- new ColumnParent(TABLE),
- getColumnForInsert(ByteBufferUtil.bytes("col1"), ByteBufferUtil.bytes("val1")),
- ONE);
-
- client.insert(UTF8Type.instance.decompose("key1"),
- new ColumnParent(TABLE),
- getColumnForInsert(ByteBufferUtil.bytes("col2"), ByteBufferUtil.bytes("val2")),
- ONE);
-
- client.insert(UTF8Type.instance.decompose("key1"),
- new ColumnParent(TABLE),
- getColumnForInsert(ByteBufferUtil.bytes("dynamicKey1"), ByteBufferUtil.bytes(100)),
- ONE);
-
- client.insert(UTF8Type.instance.decompose("key1"),
- new ColumnParent(TABLE),
- getColumnForInsert(ByteBufferUtil.bytes("dynamicKey2"), ByteBufferUtil.bytes(200)),
- ONE);
-
-
- execute(String.format("ALTER TABLE %s.%s DROP COMPACT STORAGE;", KEYSPACE, TABLE));
- UntypedResultSet resultSet = execute(String.format("select * from %s.%s",
- KEYSPACE, TABLE));
- assertColumnType(AsciiType.instance, resultSet, "key");
- assertColumnType(AsciiType.instance, resultSet, "column1");
- assertColumnType(Int32Type.instance, resultSet, "value");
- assertColumnType(AsciiType.instance, resultSet, "col1");
- assertColumnType(AsciiType.instance, resultSet, "col2");
-
- assertRows(resultSet,
- row("key1", "dynamicKey1", "val1", "val2", 100),
- row("key1", "dynamicKey2", "val1", "val2", 200));
- }
-
- @Test
- public void thriftStaticCompatTableTest() throws Throwable
- {
- String KEYSPACE = keyspace();
- String TABLE = createTable("CREATE TABLE %s (key ascii PRIMARY KEY, val ascii) WITH COMPACT STORAGE");
-
- Cassandra.Client client = getClient();
- client.set_keyspace(KEYSPACE);
-
- client.insert(UTF8Type.instance.decompose("key1"),
- new ColumnParent(TABLE),
- getColumnForInsert(ByteBufferUtil.bytes("val"), ByteBufferUtil.bytes("val1")),
- ONE);
-
- client.insert(UTF8Type.instance.decompose("key1"),
- new ColumnParent(TABLE),
- getColumnForInsert(ByteBufferUtil.bytes("dynamicKey1"), ByteBufferUtil.bytes("dynamicValue1")),
- ONE);
-
- client.insert(UTF8Type.instance.decompose("key1"),
- new ColumnParent(TABLE),
- getColumnForInsert(ByteBufferUtil.bytes("dynamicKey2"), ByteBufferUtil.bytes("dynamicValue2")),
- ONE);
-
- execute(String.format("ALTER TABLE %s.%s DROP COMPACT STORAGE;", KEYSPACE, TABLE));
- UntypedResultSet resultSet = execute(String.format("select * from %s.%s",
- KEYSPACE, TABLE));
- assertColumnType(AsciiType.instance, resultSet, "key");
- assertColumnType(UTF8Type.instance, resultSet, "column1");
- assertColumnType(AsciiType.instance, resultSet, "val");
- assertColumnType(BytesType.instance, resultSet, "value");
-
- // Values are interpreted as bytes by default:
- assertRows(resultSet,
- row("key1", "dynamicKey1", "val1", ByteBufferUtil.bytes("dynamicValue1")),
- row("key1", "dynamicKey2", "val1", ByteBufferUtil.bytes("dynamicValue2")));
- }
-
- @Test
- public void testSparseCompactTableIndex() throws Throwable
- {
- createTable("CREATE TABLE %s (key ascii PRIMARY KEY, val ascii) WITH COMPACT STORAGE");
-
- // Indexes are allowed only on the sparse compact tables
- createIndex("CREATE INDEX ON %s(val)");
- for (int i = 0; i < 10; i++)
- execute("INSERT INTO %s (key, val) VALUES (?, ?)", Integer.toString(i), Integer.toString(i * 10));
-
- alterTable("ALTER TABLE %s DROP COMPACT STORAGE");
-
- assertRows(execute("SELECT * FROM %s WHERE val = '50'"),
- row("5", null, "50", null));
- assertRows(execute("SELECT * FROM %s WHERE key = '5'"),
- row("5", null, "50", null));
- }
-
- @Test
- public void thriftCompatTableTest() throws Throwable
- {
- String KEYSPACE = keyspace();
- String TABLE = createTable("CREATE TABLE %s (pkey ascii, ckey ascii, PRIMARY KEY (pkey, ckey)) WITH COMPACT STORAGE");
-
- Cassandra.Client client = getClient();
- client.set_keyspace(KEYSPACE);
-
- client.insert(UTF8Type.instance.decompose("key1"),
- new ColumnParent(TABLE),
- getColumnForInsert(ByteBufferUtil.bytes("ckeyValue1"), ByteBufferUtil.EMPTY_BYTE_BUFFER),
- ONE);
-
- client.insert(UTF8Type.instance.decompose("key1"),
- new ColumnParent(TABLE),
- getColumnForInsert(ByteBufferUtil.bytes("ckeyValue2"), ByteBufferUtil.EMPTY_BYTE_BUFFER),
- ONE);
-
- execute(String.format("ALTER TABLE %s.%s DROP COMPACT STORAGE;", KEYSPACE, TABLE));
- UntypedResultSet resultSet = execute(String.format("select * from %s.%s",
- KEYSPACE, TABLE));
- assertColumnType(AsciiType.instance, resultSet, "pkey");
- assertColumnType(AsciiType.instance, resultSet, "ckey");
- assertColumnType(EmptyType.instance, resultSet, "value");
-
- // Value is always empty
- assertRows(resultSet,
- row("key1", "ckeyValue1", ByteBufferUtil.EMPTY_BYTE_BUFFER),
- row("key1", "ckeyValue2", ByteBufferUtil.EMPTY_BYTE_BUFFER));
- }
-
- @Test
- public void thriftDenseTableTest() throws Throwable
- {
- String KEYSPACE = keyspace();
- String TABLE = createTable("CREATE TABLE %s (pkey text, ckey text, v text, PRIMARY KEY (pkey, ckey)) WITH COMPACT STORAGE");
-
- Cassandra.Client client = getClient();
- client.set_keyspace(KEYSPACE);
-
- client.insert(UTF8Type.instance.decompose("key1"),
- new ColumnParent(TABLE),
- getColumnForInsert(ByteBufferUtil.bytes("ckey1"), ByteBufferUtil.bytes("cvalue1")),
- ONE);
-
- client.insert(UTF8Type.instance.decompose("key1"),
- new ColumnParent(TABLE),
- getColumnForInsert(ByteBufferUtil.bytes("ckey2"), ByteBufferUtil.bytes("cvalue2")),
- ONE);
-
- execute(String.format("ALTER TABLE %s.%s DROP COMPACT STORAGE;", KEYSPACE, TABLE));
- UntypedResultSet resultSet = execute(String.format("select * from %s.%s",
- KEYSPACE, TABLE));
- assertColumnType(UTF8Type.instance, resultSet, "pkey");
- assertColumnType(UTF8Type.instance, resultSet, "ckey");
- assertColumnType(UTF8Type.instance, resultSet, "v");
-
- assertRows(resultSet,
- row("key1", "ckey1", "cvalue1"),
- row("key1", "ckey2", "cvalue2"));
- }
-
- @Test
- public void thriftTableWithIntKey() throws Throwable
- {
- final String KEYSPACE = "thrift_table_with_int_key_ks";
- final String TABLE = "test_table_1";
-
- ByteBuffer columnName = ByteBufferUtil.bytes("columnname");
- CfDef cfDef = new CfDef().setDefault_validation_class(UTF8Type.instance.toString())
- .setKey_validation_class(BytesType.instance.toString())
- .setComparator_type(BytesType.instance.toString())
- .setColumn_metadata(Arrays.asList(new ColumnDef(columnName,
- Int32Type.instance.toString())
- .setIndex_name("col1Index")
- .setIndex_type(IndexType.KEYS)))
- .setKeyspace(KEYSPACE)
- .setName(TABLE);
-
- KsDef ksDef = new KsDef(KEYSPACE,
- SimpleStrategy.class.getName(),
- Arrays.asList(cfDef));
- ksDef.setStrategy_options(Collections.singletonMap("replication_factor", "1"));
-
- Cassandra.Client client = getClient();
- client.system_add_keyspace(ksDef);
- client.set_keyspace(KEYSPACE);
-
- client.insert(UTF8Type.instance.decompose("key1"),
- new ColumnParent(TABLE),
- getColumnForInsert(columnName, ByteBufferUtil.bytes(100)),
- ONE);
-
- execute(String.format("ALTER TABLE %s.%s DROP COMPACT STORAGE;", KEYSPACE, TABLE));
- UntypedResultSet resultSet = execute(String.format("select * from %s.%s",
- KEYSPACE, TABLE));
-
- assertEquals(resultSet.metadata()
- .stream()
- .filter((cs) -> cs.name.toString().equals(BytesType.instance.getString(columnName)))
- .findFirst()
- .get().type,
- Int32Type.instance);
-
- assertRows(resultSet,
- row(UTF8Type.instance.decompose("key1"), null, 100, null));
- }
-
- @Test
- public void thriftCompatTableWithSupercolumnsTest() throws Throwable
- {
- final String KEYSPACE = "thrift_compact_table_with_supercolumns_test";
- final String TABLE = "test_table_1";
-
- CfDef cfDef = new CfDef().setColumn_type("Super")
- .setSubcomparator_type(Int32Type.instance.toString())
- .setComparator_type(AsciiType.instance.toString())
- .setDefault_validation_class(AsciiType.instance.toString())
- .setKey_validation_class(AsciiType.instance.toString())
- .setKeyspace(KEYSPACE)
- .setName(TABLE);
-
- KsDef ksDef = new KsDef(KEYSPACE,
- SimpleStrategy.class.getName(),
- Arrays.asList(cfDef));
- ksDef.setStrategy_options(Collections.singletonMap("replication_factor", "1"));
-
- Cassandra.Client client = getClient();
- client.system_add_keyspace(ksDef);
-
- client.set_keyspace(KEYSPACE);
-
- Mutation mutation = new Mutation();
- ColumnOrSuperColumn csoc = new ColumnOrSuperColumn();
- csoc.setSuper_column(getSuperColumnForInsert(ByteBufferUtil.bytes("val1"),
- Arrays.asList(getColumnForInsert(ByteBufferUtil.bytes(1), ByteBufferUtil.bytes("value1")),
- getColumnForInsert(ByteBufferUtil.bytes(2), ByteBufferUtil.bytes("value2")),
- getColumnForInsert(ByteBufferUtil.bytes(3), ByteBufferUtil.bytes("value3")))));
- mutation.setColumn_or_supercolumn(csoc);
-
- Mutation mutation2 = new Mutation();
- ColumnOrSuperColumn csoc2 = new ColumnOrSuperColumn();
- csoc2.setSuper_column(getSuperColumnForInsert(ByteBufferUtil.bytes("val2"),
- Arrays.asList(getColumnForInsert(ByteBufferUtil.bytes(4), ByteBufferUtil.bytes("value7")),
- getColumnForInsert(ByteBufferUtil.bytes(5), ByteBufferUtil.bytes("value8")),
- getColumnForInsert(ByteBufferUtil.bytes(6), ByteBufferUtil.bytes("value9")))));
- mutation2.setColumn_or_supercolumn(csoc2);
-
- client.batch_mutate(Collections.singletonMap(ByteBufferUtil.bytes("key1"),
- Collections.singletonMap(TABLE, Arrays.asList(mutation, mutation2))),
- ONE);
-
- execute(String.format("ALTER TABLE %s.%s DROP COMPACT STORAGE;", KEYSPACE, TABLE));
- UntypedResultSet resultSet = execute(String.format("select * from %s.%s",
- KEYSPACE, TABLE));
- assertColumnType(AsciiType.instance, resultSet, "key");
- assertColumnType(AsciiType.instance, resultSet, "column1");
- assertColumnType(MapType.getInstance(Int32Type.instance, AsciiType.instance, true), resultSet, "");
-
- assertRows(resultSet,
- row("key1", "val1", map(1, "value1", 2, "value2", 3, "value3")),
- row("key1", "val2", map(4, "value7", 5, "value8", 6, "value9")));
-
- assertRows(execute(String.format("SELECT \"\" FROM %s.%s;", KEYSPACE, TABLE)),
- row(map(1, "value1", 2, "value2", 3, "value3")),
- row(map(4, "value7", 5, "value8", 6, "value9")));
-
- assertInvalidMessage("Range deletions are not supported for specific columns",
- String.format("DELETE \"\" FROM %s.%s WHERE key=?;", KEYSPACE, TABLE),
- "key1");
-
- execute(String.format("TRUNCATE %s.%s;", KEYSPACE, TABLE));
-
- execute(String.format("INSERT INTO %s.%s (key, column1, \"\") VALUES (?, ?, ?);", KEYSPACE, TABLE),
- "key3", "val1", map(7, "value7", 8, "value8"));
-
- assertRows(execute(String.format("SELECT \"\" FROM %s.%s;", KEYSPACE, TABLE)),
- row(map(7, "value7", 8, "value8")));
- }
-
- @Test
- public void thriftCreatedTableWithCompositeColumnsTest() throws Throwable
- {
- final String KEYSPACE = "thrift_created_table_with_composites_test_ks";
- final String TABLE = "test_table_1";
-
- CompositeType type = CompositeType.getInstance(AsciiType.instance, AsciiType.instance, AsciiType.instance);
- CfDef cfDef = new CfDef().setDefault_validation_class(AsciiType.instance.toString())
- .setComparator_type(type.toString())
- .setKey_validation_class(AsciiType.instance.toString())
- .setKeyspace(KEYSPACE)
- .setName(TABLE);
-
- KsDef ksDef = new KsDef(KEYSPACE,
- SimpleStrategy.class.getName(),
- Arrays.asList(cfDef));
- ksDef.setStrategy_options(Collections.singletonMap("replication_factor", "1"));
-
- Cassandra.Client client = getClient();
- client.system_add_keyspace(ksDef);
- client.set_keyspace(KEYSPACE);
-
- client.insert(UTF8Type.instance.decompose("key1"),
- new ColumnParent(TABLE),
- getColumnForInsert(type.decompose("a", "b", "c"), ByteBufferUtil.bytes("val1")),
- ONE);
-
- client.insert(UTF8Type.instance.decompose("key1"),
- new ColumnParent(TABLE),
- getColumnForInsert(type.decompose("d", "e", "f"), ByteBufferUtil.bytes("val2")),
- ONE);
-
- execute(String.format("ALTER TABLE %s.%s DROP COMPACT STORAGE;", KEYSPACE, TABLE));
- UntypedResultSet resultSet = execute(String.format("select * from %s.%s",
- KEYSPACE, TABLE));
-
- assertColumnType(AsciiType.instance, resultSet, "key");
- assertColumnType(AsciiType.instance, resultSet, "column1");
- assertColumnType(AsciiType.instance, resultSet, "column2");
- assertColumnType(AsciiType.instance, resultSet, "column3");
- assertColumnType(AsciiType.instance, resultSet, "value");
-
- assertRows(resultSet,
- row("key1", "a", "b", "c", "val1"),
- row("key1", "d", "e", "f", "val2"));
- }
-
- @Test
- public void compactTableWithoutClusteringKeyTest() throws Throwable
- {
- String KEYSPACE = keyspace();
- String TABLE = createTable("CREATE TABLE %s (pkey text PRIMARY KEY, s1 text, s2 text) WITH COMPACT STORAGE");
-
- Cassandra.Client client = getClient();
- client.set_keyspace(KEYSPACE);
-
- client.insert(UTF8Type.instance.decompose("key1"),
- new ColumnParent(TABLE),
- getColumnForInsert(ByteBufferUtil.bytes("ckey1"), ByteBufferUtil.bytes("val1")),
- ONE);
-
- client.insert(UTF8Type.instance.decompose("key1"),
- new ColumnParent(TABLE),
- getColumnForInsert(ByteBufferUtil.bytes("ckey2"), ByteBufferUtil.bytes("val2")),
- ONE);
-
- client.insert(UTF8Type.instance.decompose("key1"),
- new ColumnParent(TABLE),
- getColumnForInsert(ByteBufferUtil.bytes("s1"), ByteBufferUtil.bytes("s1Val")),
- ONE);
-
- client.insert(UTF8Type.instance.decompose("key1"),
- new ColumnParent(TABLE),
- getColumnForInsert(ByteBufferUtil.bytes("s2"), ByteBufferUtil.bytes("s2Val")),
- ONE);
-
- execute(String.format("ALTER TABLE %s.%s DROP COMPACT STORAGE;", KEYSPACE, TABLE));
- UntypedResultSet resultSet = execute(String.format("select * from %s.%s",
- KEYSPACE, TABLE));
-
- assertColumnType(UTF8Type.instance, resultSet, "pkey");
- assertColumnType(UTF8Type.instance, resultSet, "s1");
- assertColumnType(UTF8Type.instance, resultSet, "s2");
- assertColumnType(UTF8Type.instance, resultSet, "column1");
- assertColumnType(BytesType.instance, resultSet, "value");
-
- assertRows(resultSet,
- row("key1", "ckey1", "s1Val", "s2Val", ByteBufferUtil.bytes("val1")),
- row("key1", "ckey2", "s1Val", "s2Val", ByteBufferUtil.bytes("val2")));
- }
-
- @Test
- public void denseTableTestTest() throws Throwable
- {
- String KEYSPACE = keyspace();
- String TABLE = createTable("CREATE TABLE %s (pkey text PRIMARY KEY, s text) WITH COMPACT STORAGE");
-
- Cassandra.Client client = getClient();
- client.set_keyspace(KEYSPACE);
-
- client.insert(UTF8Type.instance.decompose("key1"),
- new ColumnParent(TABLE),
- getColumnForInsert(ByteBufferUtil.bytes("ckey1"), ByteBufferUtil.bytes("val1")),
- ONE);
-
- client.insert(UTF8Type.instance.decompose("key1"),
- new ColumnParent(TABLE),
- getColumnForInsert(ByteBufferUtil.bytes("ckey2"), ByteBufferUtil.bytes("val2")),
- ONE);
-
- client.insert(UTF8Type.instance.decompose("key1"),
- new ColumnParent(TABLE),
- getColumnForInsert(ByteBufferUtil.bytes("s"), ByteBufferUtil.bytes("sval1")),
- ONE);
-
- client.insert(UTF8Type.instance.decompose("key1"),
- new ColumnParent(TABLE),
- getColumnForInsert(ByteBufferUtil.bytes("s"), ByteBufferUtil.bytes("sval2")),
- ONE);
-
- // `s` becomes static, `column1` becomes a clustering key, `value` becomes visible
- execute(String.format("ALTER TABLE %s.%s DROP COMPACT STORAGE;", KEYSPACE, TABLE));
- UntypedResultSet resultSet = execute(String.format("select * from %s.%s",
- KEYSPACE, TABLE));
- assertColumnType(UTF8Type.instance, resultSet, "pkey");
- assertColumnType(UTF8Type.instance, resultSet, "s");
- assertColumnType(UTF8Type.instance, resultSet, "column1");
- assertColumnType(BytesType.instance, resultSet, "value");
-
- assertRows(resultSet,
- row("key1", "ckey1", "sval2", ByteBufferUtil.bytes("val1")),
- row("key1", "ckey2", "sval2", ByteBufferUtil.bytes("val2")));
- }
-
- @Test
- public void denseCompositeWithIndexesTest() throws Throwable
- {
- final String KEYSPACE = "thrift_dense_composite_table_test_ks";
- final String TABLE = "dense_composite_table";
-
- ByteBuffer aCol = createDynamicCompositeKey(ByteBufferUtil.bytes("a"));
- ByteBuffer bCol = createDynamicCompositeKey(ByteBufferUtil.bytes("b"));
- ByteBuffer cCol = createDynamicCompositeKey(ByteBufferUtil.bytes("c"));
-
- String compositeType = "DynamicCompositeType(a => BytesType, b => TimeUUIDType, c => UTF8Type)";
-
- CfDef cfDef = new CfDef();
- cfDef.setName(TABLE);
- cfDef.setComparator_type(compositeType);
- cfDef.setKeyspace(KEYSPACE);
-
- cfDef.setColumn_metadata(
- Arrays.asList(new ColumnDef(aCol, "BytesType").setIndex_type(IndexType.KEYS).setIndex_name(KEYSPACE + "_a"),
- new ColumnDef(bCol, "BytesType").setIndex_type(IndexType.KEYS).setIndex_name(KEYSPACE + "_b"),
- new ColumnDef(cCol, "BytesType").setIndex_type(IndexType.KEYS).setIndex_name(KEYSPACE + "_c")));
-
-
- KsDef ksDef = new KsDef(KEYSPACE,
- SimpleStrategy.class.getName(),
- Collections.singletonList(cfDef));
- ksDef.setStrategy_options(Collections.singletonMap("replication_factor", "1"));
-
- Cassandra.Client client = getClient();
- client.system_add_keyspace(ksDef);
- client.set_keyspace(KEYSPACE);
-
- CFMetaData cfm = Keyspace.open(KEYSPACE).getColumnFamilyStore(TABLE).metadata;
- assertFalse(cfm.isCQLTable());
-
- List<Pair<ColumnDefinition, IndexTarget.Type>> compactTableTargets = new ArrayList<>();
- compactTableTargets.add(TargetParser.parse(cfm, "a"));
- compactTableTargets.add(TargetParser.parse(cfm, "b"));
- compactTableTargets.add(TargetParser.parse(cfm, "c"));
-
- execute(String.format("ALTER TABLE %s.%s DROP COMPACT STORAGE", KEYSPACE, TABLE));
- cfm = Keyspace.open(KEYSPACE).getColumnFamilyStore(TABLE).metadata;
- assertTrue(cfm.isCQLTable());
-
- List<Pair<ColumnDefinition, IndexTarget.Type>> cqlTableTargets = new ArrayList<>();
- cqlTableTargets.add(TargetParser.parse(cfm, "a"));
- cqlTableTargets.add(TargetParser.parse(cfm, "b"));
- cqlTableTargets.add(TargetParser.parse(cfm, "c"));
-
- assertEquals(compactTableTargets, cqlTableTargets);
- }
-
- private static ByteBuffer createDynamicCompositeKey(Object... objects)
- {
- int length = 0;
-
- for (Object object : objects)
- {
- length += 2 * Short.BYTES + Byte.BYTES;
- if (object instanceof String)
- length += ((String) object).length();
- else if (object instanceof UUID)
- length += 2 * Long.BYTES;
- else if (object instanceof ByteBuffer)
- length += ((ByteBuffer) object).remaining();
- else
- throw new MarshalException(object.getClass().getName() + " is not recognized as a valid type for this composite");
- }
-
- ByteBuffer out = ByteBuffer.allocate(length);
-
- for (Object object : objects)
- {
- if (object instanceof String)
- {
- String cast = (String) object;
-
- out.putShort((short) (0x8000 | 's'));
- out.putShort((short) cast.length());
- out.put(cast.getBytes());
- out.put((byte) 0);
- }
- else if (object instanceof UUID)
- {
- out.putShort((short) (0x8000 | 't'));
- out.putShort((short) 16);
- out.put(UUIDGen.decompose((UUID) object));
- out.put((byte) 0);
- }
- else if (object instanceof ByteBuffer)
- {
- ByteBuffer bytes = ((ByteBuffer) object).duplicate();
- out.putShort((short) (0x8000 | 'b'));
- out.putShort((short) bytes.remaining());
- out.put(bytes);
- out.put((byte) 0);
- }
- else
- {
- throw new MarshalException(object.getClass().getName() + " is not recognized as a valid type for this composite");
- }
- }
-
- return out;
- }
-
- private Column getColumnForInsert(ByteBuffer columnName, ByteBuffer value)
- {
- Column column = new Column();
- column.setName(columnName);
- column.setValue(value);
- column.setTimestamp(System.currentTimeMillis());
- return column;
- }
-
- private SuperColumn getSuperColumnForInsert(ByteBuffer columnName, List<Column> columns)
- {
- SuperColumn column = new SuperColumn();
- column.setName(columnName);
- for (Column c : columns)
- column.addToColumns(c);
- return column;
- }
-
- private static void assertColumnType(AbstractType t, UntypedResultSet resultSet, String columnName)
- {
- for (ColumnSpecification columnSpecification : resultSet.metadata())
- {
- if (columnSpecification.name.toString().equals(columnName))
- {
- assertEquals(t, columnSpecification.type);
- return;
- }
- }
-
- fail(String.format("Could not find a column with name '%s'", columnName));
- }
-}
diff --git a/test/unit/org/apache/cassandra/cql3/validation/operations/DropRecreateAndRestoreTest.java b/test/unit/org/apache/cassandra/cql3/validation/operations/DropRecreateAndRestoreTest.java
index f491d24..19aba64 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/operations/DropRecreateAndRestoreTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/operations/DropRecreateAndRestoreTest.java
@@ -19,7 +19,6 @@
import java.io.File;
import java.util.List;
-import java.util.UUID;
import org.junit.Test;
@@ -30,6 +29,7 @@
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.exceptions.InvalidRequestException;
import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.schema.TableId;
public class DropRecreateAndRestoreTest extends CQLTester
{
@@ -43,7 +43,7 @@
long time = System.currentTimeMillis();
- UUID id = currentTableMetadata().cfId;
+ TableId id = currentTableMetadata().id;
assertRows(execute("SELECT * FROM %s"), row(0, 0, 0), row(0, 1, 1));
Thread.sleep(5);
@@ -84,7 +84,7 @@
public void testCreateWithIdDuplicate() throws Throwable
{
createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY(a, b))");
- UUID id = currentTableMetadata().cfId;
+ TableId id = currentTableMetadata().id;
execute(String.format("CREATE TABLE %%s (a int, b int, c int, PRIMARY KEY(a, b)) WITH ID = %s", id));
}
@@ -98,7 +98,7 @@
public void testAlterWithId() throws Throwable
{
createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY(a, b))");
- UUID id = currentTableMetadata().cfId;
+ TableId id = currentTableMetadata().id;
execute(String.format("ALTER TABLE %%s WITH ID = %s", id));
}
}
diff --git a/test/unit/org/apache/cassandra/cql3/validation/operations/DropTest.java b/test/unit/org/apache/cassandra/cql3/validation/operations/DropTest.java
index 692eb45..90130ad 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/operations/DropTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/operations/DropTest.java
@@ -27,8 +27,8 @@
@Test
public void testNonExistingOnes() throws Throwable
{
- assertInvalidMessage("Cannot drop non existing table", "DROP TABLE " + KEYSPACE + ".table_does_not_exist");
- assertInvalidMessage("Cannot drop table in unknown keyspace", "DROP TABLE keyspace_does_not_exist.table_does_not_exist");
+ assertInvalidMessage(String.format("Table '%s.table_does_not_exist' doesn't exist", KEYSPACE), "DROP TABLE " + KEYSPACE + ".table_does_not_exist");
+ assertInvalidMessage("Table 'keyspace_does_not_exist.table_does_not_exist' doesn't exist", "DROP TABLE keyspace_does_not_exist.table_does_not_exist");
execute("DROP TABLE IF EXISTS " + KEYSPACE + ".table_does_not_exist");
execute("DROP TABLE IF EXISTS keyspace_does_not_exist.table_does_not_exist");
diff --git a/test/unit/org/apache/cassandra/cql3/validation/operations/InsertTest.java b/test/unit/org/apache/cassandra/cql3/validation/operations/InsertTest.java
index 06cc1a3..0f01f3e 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/operations/InsertTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/operations/InsertTest.java
@@ -26,7 +26,6 @@
import org.apache.cassandra.cql3.UntypedResultSet;
import org.apache.cassandra.cql3.UntypedResultSet.Row;
import org.apache.cassandra.exceptions.InvalidRequestException;
-import org.apache.cassandra.utils.ByteBufferUtil;
public class InsertTest extends CQLTester
{
@@ -119,54 +118,6 @@
}
@Test
- public void testInsertWithCompactFormat() throws Throwable
- {
- testInsertWithCompactFormat(false);
- testInsertWithCompactFormat(true);
- }
-
- private void testInsertWithCompactFormat(boolean forceFlush) throws Throwable
- {
- createTable("CREATE TABLE %s (partitionKey int," +
- "clustering int," +
- "value int," +
- " PRIMARY KEY (partitionKey, clustering)) WITH COMPACT STORAGE");
-
- execute("INSERT INTO %s (partitionKey, clustering, value) VALUES (0, 0, 0)");
- execute("INSERT INTO %s (partitionKey, clustering, value) VALUES (0, 1, 1)");
- flush(forceFlush);
-
- assertRows(execute("SELECT * FROM %s"),
- row(0, 0, 0),
- row(0, 1, 1));
-
- // Invalid Null values for the clustering key or the regular column
- assertInvalidMessage("Some clustering keys are missing: clustering",
- "INSERT INTO %s (partitionKey, value) VALUES (0, 0)");
- assertInvalidMessage("Column value is mandatory for this COMPACT STORAGE table",
- "INSERT INTO %s (partitionKey, clustering) VALUES (0, 0)");
-
- // Missing primary key columns
- assertInvalidMessage("Some partition key parts are missing: partitionkey",
- "INSERT INTO %s (clustering, value) VALUES (0, 1)");
-
- // multiple time the same value
- assertInvalidMessage("The column names contains duplicates",
- "INSERT INTO %s (partitionKey, clustering, value, value) VALUES (0, 0, 2, 2)");
-
- // multiple time same primary key element in WHERE clause
- assertInvalidMessage("The column names contains duplicates",
- "INSERT INTO %s (partitionKey, clustering, clustering, value) VALUES (0, 0, 0, 2)");
-
- // unknown identifiers
- assertInvalidMessage("Undefined column name clusteringx",
- "INSERT INTO %s (partitionKey, clusteringx, value) VALUES (0, 0, 2)");
-
- assertInvalidMessage("Undefined column name valuex",
- "INSERT INTO %s (partitionKey, clustering, valuex) VALUES (0, 0, 2)");
- }
-
- @Test
public void testInsertWithTwoClusteringColumns() throws Throwable
{
testInsertWithTwoClusteringColumns(false);
@@ -211,126 +162,6 @@
"INSERT INTO %s (partitionKey, clustering_1, clustering_2, valuex) VALUES (0, 0, 0, 2)");
}
- /**
- * Test for CASSANDRA-13917
- */
- @Test
- public void testInsertWithCompactStaticFormat() throws Throwable
- {
- testWithCompactTable("CREATE TABLE %s (a int PRIMARY KEY, b int, c int) WITH COMPACT STORAGE");
-
- // if column1 is present, hidden column is called column2
- createTable("CREATE TABLE %s (a int PRIMARY KEY, b int, c int, column1 int) WITH COMPACT STORAGE");
- execute("INSERT INTO %s (a, b, c, column1) VALUES (1, 1, 1, 1)");
- assertInvalidMessage("Undefined column name column2",
- "INSERT INTO %s (a, b, c, column2) VALUES (1, 1, 1, 1)");
-
- // if value is present, hidden column is called value1
- createTable("CREATE TABLE %s (a int PRIMARY KEY, b int, c int, value int) WITH COMPACT STORAGE");
- execute("INSERT INTO %s (a, b, c, value) VALUES (1, 1, 1, 1)");
- assertInvalidMessage("Undefined column name value1",
- "INSERT INTO %s (a, b, c, value1) VALUES (1, 1, 1, 1)");
- }
-
- /**
- * Test for CASSANDRA-13917
- */
- @Test
- public void testInsertWithCompactNonStaticFormat() throws Throwable
- {
- testWithCompactTable("CREATE TABLE %s (a int, b int, PRIMARY KEY (a, b)) WITH COMPACT STORAGE");
- testWithCompactTable("CREATE TABLE %s (a int, b int, v int, PRIMARY KEY (a, b)) WITH COMPACT STORAGE");
- }
-
- private void testWithCompactTable(String tableQuery) throws Throwable
- {
- createTable(tableQuery);
-
- // pass correct types to the hidden columns
- assertInvalidMessage("Undefined column name column1",
- "INSERT INTO %s (a, b, column1) VALUES (?, ?, ?)",
- 1, 1, ByteBufferUtil.bytes('a'));
- assertInvalidMessage("Undefined column name value",
- "INSERT INTO %s (a, b, value) VALUES (?, ?, ?)",
- 1, 1, ByteBufferUtil.bytes('a'));
- assertInvalidMessage("Undefined column name column1",
- "INSERT INTO %s (a, b, column1, value) VALUES (?, ?, ?, ?)",
- 1, 1, ByteBufferUtil.bytes('a'), ByteBufferUtil.bytes('b'));
- assertInvalidMessage("Undefined column name value",
- "INSERT INTO %s (a, b, value, column1) VALUES (?, ?, ?, ?)",
- 1, 1, ByteBufferUtil.bytes('a'), ByteBufferUtil.bytes('b'));
-
- // pass incorrect types to the hidden columns
- assertInvalidMessage("Undefined column name value",
- "INSERT INTO %s (a, b, value) VALUES (?, ?, ?)",
- 1, 1, 1);
- assertInvalidMessage("Undefined column name column1",
- "INSERT INTO %s (a, b, column1) VALUES (?, ?, ?)",
- 1, 1, 1);
- assertEmpty(execute("SELECT * FROM %s"));
-
- // pass null to the hidden columns
- assertInvalidMessage("Undefined column name value",
- "INSERT INTO %s (a, b, value) VALUES (?, ?, ?)",
- 1, 1, null);
- assertInvalidMessage("Undefined column name column1",
- "INSERT INTO %s (a, b, column1) VALUES (?, ?, ?)",
- 1, 1, null);
- }
-
- @Test
- public void testInsertWithCompactStorageAndTwoClusteringColumns() throws Throwable
- {
- testInsertWithCompactStorageAndTwoClusteringColumns(false);
- testInsertWithCompactStorageAndTwoClusteringColumns(true);
- }
-
- private void testInsertWithCompactStorageAndTwoClusteringColumns(boolean forceFlush) throws Throwable
- {
- createTable("CREATE TABLE %s (partitionKey int," +
- "clustering_1 int," +
- "clustering_2 int," +
- "value int," +
- " PRIMARY KEY (partitionKey, clustering_1, clustering_2)) WITH COMPACT STORAGE");
-
- execute("INSERT INTO %s (partitionKey, clustering_1, value) VALUES (0, 0, 0)");
- execute("INSERT INTO %s (partitionKey, clustering_1, clustering_2, value) VALUES (0, 0, 0, 0)");
- execute("INSERT INTO %s (partitionKey, clustering_1, clustering_2, value) VALUES (0, 0, 1, 1)");
- flush(forceFlush);
-
- assertRows(execute("SELECT * FROM %s"),
- row(0, 0, null, 0),
- row(0, 0, 0, 0),
- row(0, 0, 1, 1));
-
- // Invalid Null values for the clustering key or the regular column
- assertInvalidMessage("PRIMARY KEY column \"clustering_2\" cannot be restricted as preceding column \"clustering_1\" is not restricted",
- "INSERT INTO %s (partitionKey, clustering_2, value) VALUES (0, 0, 0)");
- assertInvalidMessage("Column value is mandatory for this COMPACT STORAGE table",
- "INSERT INTO %s (partitionKey, clustering_1, clustering_2) VALUES (0, 0, 0)");
-
- // Missing primary key columns
- assertInvalidMessage("Some partition key parts are missing: partitionkey",
- "INSERT INTO %s (clustering_1, clustering_2, value) VALUES (0, 0, 1)");
- assertInvalidMessage("PRIMARY KEY column \"clustering_2\" cannot be restricted as preceding column \"clustering_1\" is not restricted",
- "INSERT INTO %s (partitionKey, clustering_2, value) VALUES (0, 0, 2)");
-
- // multiple time the same value
- assertInvalidMessage("The column names contains duplicates",
- "INSERT INTO %s (partitionKey, clustering_1, value, clustering_2, value) VALUES (0, 0, 2, 0, 2)");
-
- // multiple time same primary key element in WHERE clause
- assertInvalidMessage("The column names contains duplicates",
- "INSERT INTO %s (partitionKey, clustering_1, clustering_1, clustering_2, value) VALUES (0, 0, 0, 0, 2)");
-
- // unknown identifiers
- assertInvalidMessage("Undefined column name clustering_1x",
- "INSERT INTO %s (partitionKey, clustering_1x, clustering_2, value) VALUES (0, 0, 0, 2)");
-
- assertInvalidMessage("Undefined column name valuex",
- "INSERT INTO %s (partitionKey, clustering_1, clustering_2, valuex) VALUES (0, 0, 0, 2)");
- }
-
@Test
public void testInsertWithAStaticColumn() throws Throwable
{
diff --git a/test/unit/org/apache/cassandra/cql3/validation/operations/InsertUpdateIfConditionTest.java b/test/unit/org/apache/cassandra/cql3/validation/operations/InsertUpdateIfConditionTest.java
index e86071a..edfe57a 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/operations/InsertUpdateIfConditionTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/operations/InsertUpdateIfConditionTest.java
@@ -23,7 +23,7 @@
import org.junit.Test;
-import org.apache.cassandra.config.SchemaConstants;
+import org.apache.cassandra.schema.SchemaConstants;
import org.apache.cassandra.cql3.CQLTester;
import org.apache.cassandra.cql3.Duration;
import org.apache.cassandra.exceptions.InvalidRequestException;
@@ -65,7 +65,10 @@
"UPDATE %s SET v1 = 3, v2 = 'bar' WHERE k = 0 IF v1 = ?", unset());
// Shouldn't apply
+
assertRows(execute("UPDATE %s SET v1 = 3, v2 = 'bar' WHERE k = 0 IF v1 = ?", 4), row(false));
+ assertRows(execute("UPDATE %s SET v1 = 3, v2 = 'bar' WHERE k = 0 IF v1 IN (?, ?)", 1, 2), row(false));
+ assertRows(execute("UPDATE %s SET v1 = 3, v2 = 'bar' WHERE k = 0 IF v1 IN ?", list(1, 2)), row(false));
assertRows(execute("UPDATE %s SET v1 = 3, v2 = 'bar' WHERE k = 0 IF EXISTS"), row(false));
// Should apply
@@ -81,11 +84,15 @@
// Should apply (note: we want v2 before v1 in the statement order to exercise #5786)
assertRows(execute("UPDATE %s SET v2 = 'bar', v1 = 3 WHERE k = 0 IF v1 = ?", 2), row(true));
+ assertRows(execute("UPDATE %s SET v2 = 'bar', v1 = 2 WHERE k = 0 IF v1 IN (?, ?)", 2, 3), row(true));
+ assertRows(execute("UPDATE %s SET v2 = 'bar', v1 = 3 WHERE k = 0 IF v1 IN ?", list(2, 3)), row(true));
+ assertRows(execute("UPDATE %s SET v2 = 'bar', v1 = 3 WHERE k = 0 IF v1 IN ?", list(1, null, 3)), row(true));
assertRows(execute("UPDATE %s SET v2 = 'bar', v1 = 3 WHERE k = 0 IF EXISTS"), row(true));
assertRows(execute("SELECT * FROM %s"), row(0, 3, "bar", null));
// Shouldn't apply, only one condition is ok
assertRows(execute("UPDATE %s SET v1 = 5, v2 = 'foobar' WHERE k = 0 IF v1 = ? AND v2 = ?", 3, "foo"), row(false, 3, "bar"));
+ assertRows(execute("UPDATE %s SET v1 = 5, v2 = 'foobar' WHERE k = 0 IF v1 = 3 AND v2 = 'foo'"), row(false, 3, "bar"));
assertRows(execute("SELECT * FROM %s"), row(0, 3, "bar", null));
// Should apply
@@ -161,7 +168,6 @@
"UPDATE %s SET v1 = 3, v2 = 'bar' WHERE k = 0 IF v1 >= ?", unset());
assertInvalidMessage("Invalid 'unset' value in condition",
"UPDATE %s SET v1 = 3, v2 = 'bar' WHERE k = 0 IF v1 != ?", unset());
-
}
/**
@@ -539,24 +545,6 @@
row(true));
}
- /**
- * Test for CAS with compact storage table, and #6813 in particular,
- * migrated from cql_tests.py:TestCQL.cas_and_compact_test()
- */
- @Test
- public void testCompactStorage() throws Throwable
- {
- createTable("CREATE TABLE %s (partition text, key text, owner text, PRIMARY KEY (partition, key) ) WITH COMPACT STORAGE");
-
- execute("INSERT INTO %s (partition, key, owner) VALUES ('a', 'b', null)");
- assertRows(execute("UPDATE %s SET owner='z' WHERE partition='a' AND key='b' IF owner=null"), row(true));
-
- assertRows(execute("UPDATE %s SET owner='b' WHERE partition='a' AND key='b' IF owner='a'"), row(false, "z"));
- assertRows(execute("UPDATE %s SET owner='b' WHERE partition='a' AND key='b' IF owner='z'"), row(true));
-
- assertRows(execute("INSERT INTO %s (partition, key, owner) VALUES ('a', 'c', 'x') IF NOT EXISTS"), row(true));
- }
-
@Test
public void testWholeUDT() throws Throwable
{
@@ -1426,10 +1414,8 @@
{
String tableName = createTable("CREATE TABLE %s (id text PRIMARY KEY, value1 blob, value2 blob)with comment = 'foo'");
- execute("use " + KEYSPACE);
-
// try dropping when doesn't exist
- schemaChange("DROP INDEX IF EXISTS myindex");
+ schemaChange(format("DROP INDEX IF EXISTS %s.myindex", KEYSPACE));
// create and confirm
createIndex("CREATE INDEX IF NOT EXISTS myindex ON %s (value1)");
@@ -1440,7 +1426,7 @@
execute("CREATE INDEX IF NOT EXISTS myindex ON %s (value1)");
// drop and confirm
- execute("DROP INDEX IF EXISTS myindex");
+ execute(format("DROP INDEX IF EXISTS %s.myindex", KEYSPACE));
Object[][] rows = getRows(execute("select index_name from system.\"IndexInfo\" where table_name = ?", tableName));
assertEquals(0, rows.length);
@@ -1986,39 +1972,156 @@
String typename = createType("CREATE TYPE %s (a int, b text)");
String myType = KEYSPACE + '.' + typename;
- createTable("CREATE TABLE %s (k int PRIMARY KEY, v frozen<" + myType + "> )");
+ for (boolean frozen : new boolean[] {false, true})
+ {
+ createTable(String.format("CREATE TABLE %%s (k int PRIMARY KEY, v %s)",
+ frozen
+ ? "frozen<" + myType + ">"
+ : myType));
Object v = userType("a", 0, "b", "abc");
- execute("INSERT INTO %s (k, v) VALUES (?, ?)", 0, v);
+ execute("INSERT INTO %s (k, v) VALUES (0, ?)", v);
// Does not apply
assertRows(execute("UPDATE %s SET v = {a: 0, b: 'bc'} WHERE k = 0 IF v IN (?, ?)", userType("a", 1, "b", "abc"), userType("a", 0, "b", "ac")),
row(false, v));
assertRows(execute("UPDATE %s SET v = {a: 0, b: 'bc'} WHERE k = 0 IF v IN (?, ?)", userType("a", 1, "b", "abc"), null),
row(false, v));
- assertRows(execute("UPDATE %s SET v = {a: 0, b: 'bc'} WHERE k = 0 IF v IN (?, ?)", userType("a", 1, "b", "abc"), unset()),
- row(false, v));
assertRows(execute("UPDATE %s SET v = {a: 0, b: 'bc'} WHERE k = 0 IF v IN (?, ?)", null, null),
row(false, v));
+ assertRows(execute("UPDATE %s SET v = {a: 0, b: 'bc'} WHERE k = 0 IF v IN (?, ?)", userType("a", 1, "b", "abc"), unset()),
+ row(false, v));
assertRows(execute("UPDATE %s SET v = {a: 0, b: 'bc'} WHERE k = 0 IF v IN (?, ?)", unset(), unset()),
row(false, v));
assertRows(execute("UPDATE %s SET v = {a: 0, b: 'bc'} WHERE k = 0 IF v IN ?", list(userType("a", 1, "b", "abc"), userType("a", 0, "b", "ac"))),
row(false, v));
+ assertRows(execute("UPDATE %s SET v = {a: 0, b: 'bc'} WHERE k = 0 IF v.a IN (?, ?)", 1, 2),
+ row(false, v));
+ assertRows(execute("UPDATE %s SET v = {a: 0, b: 'bc'} WHERE k = 0 IF v.a IN (?, ?)", 1, null),
+ row(false, v));
+ assertRows(execute("UPDATE %s SET v = {a: 0, b: 'bc'} WHERE k = 0 IF v.a IN ?", list(1, 2)),
+ row(false, v));
+ assertRows(execute("UPDATE %s SET v = {a: 0, b: 'bc'} WHERE k = 0 IF v.a IN (?, ?)", 1, unset()),
+ row(false, v));
// Does apply
assertRows(execute("UPDATE %s SET v = {a: 0, b: 'bc'} WHERE k = 0 IF v IN (?, ?)", userType("a", 0, "b", "abc"), userType("a", 0, "b", "ac")),
row(true));
assertRows(execute("UPDATE %s SET v = {a: 1, b: 'bc'} WHERE k = 0 IF v IN (?, ?)", userType("a", 0, "b", "bc"), null),
row(true));
- assertRows(execute("UPDATE %s SET v = {a: 1, b: 'ac'} WHERE k = 0 IF v IN (?, ?, ?)", userType("a", 0, "b", "bc"), unset(), userType("a", 1, "b", "bc")),
+ assertRows(execute("UPDATE %s SET v = {a: 0, b: 'abc'} WHERE k = 0 IF v IN ?", list(userType("a", 1, "b", "bc"), userType("a", 0, "b", "ac"))),
row(true));
- assertRows(execute("UPDATE %s SET v = {a: 0, b: 'abc'} WHERE k = 0 IF v IN ?", list(userType("a", 1, "b", "ac"), userType("a", 0, "b", "ac"))),
+ assertRows(execute("UPDATE %s SET v = {a: 1, b: 'bc'} WHERE k = 0 IF v IN (?, ?, ?)", userType("a", 1, "b", "bc"), unset(), userType("a", 0, "b", "abc")),
+ row(true));
+ assertRows(execute("UPDATE %s SET v = {a: 0, b: 'bc'} WHERE k = 0 IF v.a IN (?, ?)", 1, 0),
+ row(true));
+ assertRows(execute("UPDATE %s SET v = {a: 0, b: 'abc'} WHERE k = 0 IF v.a IN (?, ?)", 0, null),
+ row(true));
+ assertRows(execute("UPDATE %s SET v = {a: 0, b: 'bc'} WHERE k = 0 IF v.a IN ?", list(0, 1)),
+ row(true));
+ assertRows(execute("UPDATE %s SET v = {a: 0, b: 'abc'} WHERE k = 0 IF v.a IN (?, ?, ?)", 1, unset(), 0),
row(true));
assertInvalidMessage("Invalid null list in IN condition",
"UPDATE %s SET v = {a: 0, b: 'bc'} WHERE k = 0 IF v IN ?", (List<ByteBuffer>) null);
assertInvalidMessage("Invalid 'unset' value in condition",
"UPDATE %s SET v = {a: 0, b: 'bc'} WHERE k = 0 IF v IN ?", unset());
+ assertInvalidMessage("Invalid 'unset' value in condition",
+ "UPDATE %s SET v = {a: 0, b: 'bc'} WHERE k = 0 IF v.a IN ?", unset());
+ }
+ }
+
+ @Test
+ public void testConditionalOnDurationColumns() throws Throwable
+ {
+ createTable(" CREATE TABLE %s (k int PRIMARY KEY, v int, d duration)");
+
+ assertInvalidMessage("Slice conditions ( > ) are not supported on durations",
+ "UPDATE %s SET v = 3 WHERE k = 0 IF d > 1s");
+ assertInvalidMessage("Slice conditions ( >= ) are not supported on durations",
+ "UPDATE %s SET v = 3 WHERE k = 0 IF d >= 1s");
+ assertInvalidMessage("Slice conditions ( <= ) are not supported on durations",
+ "UPDATE %s SET v = 3 WHERE k = 0 IF d <= 1s");
+ assertInvalidMessage("Slice conditions ( < ) are not supported on durations",
+ "UPDATE %s SET v = 3 WHERE k = 0 IF d < 1s");
+
+ execute("INSERT INTO %s (k, v, d) VALUES (1, 1, 2s)");
+
+ assertRows(execute("UPDATE %s SET v = 4 WHERE k = 1 IF d = 1s"), row(false, Duration.from("2s")));
+ assertRows(execute("UPDATE %s SET v = 3 WHERE k = 1 IF d = 2s"), row(true));
+
+ assertRows(execute("SELECT * FROM %s WHERE k = 1"), row(1, Duration.from("2s"), 3));
+
+ assertRows(execute("UPDATE %s SET d = 10s WHERE k = 1 IF d != 2s"), row(false, Duration.from("2s")));
+ assertRows(execute("UPDATE %s SET v = 6 WHERE k = 1 IF d != 1s"), row(true));
+
+ assertRows(execute("SELECT * FROM %s WHERE k = 1"), row(1, Duration.from("2s"), 6));
+
+ assertRows(execute("UPDATE %s SET v = 5 WHERE k = 1 IF d IN (1s, 5s)"), row(false, Duration.from("2s")));
+ assertRows(execute("UPDATE %s SET d = 10s WHERE k = 1 IF d IN (1s, 2s)"), row(true));
+
+ assertRows(execute("SELECT * FROM %s WHERE k = 1"), row(1, Duration.from("10s"), 6));
+ }
+
+ @Test
+ public void testConditionalOnDurationWithinLists() throws Throwable
+ {
+ for (Boolean frozen : new Boolean[]{Boolean.FALSE, Boolean.TRUE})
+ {
+ String listType = String.format(frozen ? "frozen<%s>" : "%s", "list<duration>");
+
+ createTable("CREATE TABLE %s (k int PRIMARY KEY, v int, l " + listType + " )");
+
+ assertInvalidMessage("Slice conditions are not supported on collections containing durations",
+ "UPDATE %s SET v = 3 WHERE k = 0 IF l > [1s, 2s]");
+ assertInvalidMessage("Slice conditions are not supported on collections containing durations",
+ "UPDATE %s SET v = 3 WHERE k = 0 IF l >= [1s, 2s]");
+ assertInvalidMessage("Slice conditions are not supported on collections containing durations",
+ "UPDATE %s SET v = 3 WHERE k = 0 IF l <= [1s, 2s]");
+ assertInvalidMessage("Slice conditions are not supported on collections containing durations",
+ "UPDATE %s SET v = 3 WHERE k = 0 IF l < [1s, 2s]");
+
+ execute("INSERT INTO %s (k, v, l) VALUES (1, 1, [1s, 2s])");
+
+ assertRows(execute("UPDATE %s SET v = 4 WHERE k = 1 IF l = [2s]"), row(false, list(Duration.from("1000ms"), Duration.from("2s"))));
+ assertRows(execute("UPDATE %s SET v = 3 WHERE k = 1 IF l = [1s, 2s]"), row(true));
+
+ assertRows(execute("SELECT * FROM %s WHERE k = 1"), row(1, list(Duration.from("1000ms"), Duration.from("2s")), 3));
+
+ assertRows(execute("UPDATE %s SET l = [10s] WHERE k = 1 IF l != [1s, 2s]"), row(false, list(Duration.from("1000ms"), Duration.from("2s"))));
+ assertRows(execute("UPDATE %s SET v = 6 WHERE k = 1 IF l != [1s]"), row(true));
+
+ assertRows(execute("SELECT * FROM %s WHERE k = 1"), row(1, list(Duration.from("1000ms"), Duration.from("2s")), 6));
+
+ assertRows(execute("UPDATE %s SET v = 5 WHERE k = 1 IF l IN ([1s], [1s, 5s])"), row(false, list(Duration.from("1000ms"), Duration.from("2s"))));
+ assertRows(execute("UPDATE %s SET l = [5s, 10s] WHERE k = 1 IF l IN ([1s], [1s, 2s])"), row(true));
+
+ assertRows(execute("SELECT * FROM %s WHERE k = 1"), row(1, list(Duration.from("5s"), Duration.from("10s")), 6));
+
+ assertInvalidMessage("Slice conditions ( > ) are not supported on durations",
+ "UPDATE %s SET v = 3 WHERE k = 0 IF l[0] > 1s");
+ assertInvalidMessage("Slice conditions ( >= ) are not supported on durations",
+ "UPDATE %s SET v = 3 WHERE k = 0 IF l[0] >= 1s");
+ assertInvalidMessage("Slice conditions ( <= ) are not supported on durations",
+ "UPDATE %s SET v = 3 WHERE k = 0 IF l[0] <= 1s");
+ assertInvalidMessage("Slice conditions ( < ) are not supported on durations",
+ "UPDATE %s SET v = 3 WHERE k = 0 IF l[0] < 1s");
+
+ assertRows(execute("UPDATE %s SET v = 4 WHERE k = 1 IF l[0] = 2s"), row(false, list(Duration.from("5s"), Duration.from("10s"))));
+ assertRows(execute("UPDATE %s SET v = 3 WHERE k = 1 IF l[0] = 5s"), row(true));
+
+ assertRows(execute("SELECT * FROM %s WHERE k = 1"), row(1, list(Duration.from("5s"), Duration.from("10s")), 3));
+
+ assertRows(execute("UPDATE %s SET l = [10s] WHERE k = 1 IF l[1] != 10s"), row(false, list(Duration.from("5s"), Duration.from("10s"))));
+ assertRows(execute("UPDATE %s SET v = 6 WHERE k = 1 IF l[1] != 1s"), row(true));
+
+ assertRows(execute("SELECT * FROM %s WHERE k = 1"), row(1, list(Duration.from("5s"), Duration.from("10s")), 6));
+
+ assertRows(execute("UPDATE %s SET v = 5 WHERE k = 1 IF l[0] IN (2s, 10s)"), row(false, list(Duration.from("5s"), Duration.from("10s"))));
+ assertRows(execute("UPDATE %s SET l = [6s, 12s] WHERE k = 1 IF l[0] IN (5s, 10s)"), row(true));
+
+ assertRows(execute("SELECT * FROM %s WHERE k = 1"), row(1, list(Duration.from("6s"), Duration.from("12s")), 6));
+ }
}
@Test
@@ -2079,95 +2182,63 @@
}
@Test
- public void testConditionalOnDurationColumns() throws Throwable
- {
- createTable(" CREATE TABLE %s (k int PRIMARY KEY, v int, d duration)");
-
- assertInvalidMessage("Slice conditions are not supported on durations",
- "UPDATE %s SET v = 3 WHERE k = 0 IF d > 1s");
- assertInvalidMessage("Slice conditions are not supported on durations",
- "UPDATE %s SET v = 3 WHERE k = 0 IF d >= 1s");
- assertInvalidMessage("Slice conditions are not supported on durations",
- "UPDATE %s SET v = 3 WHERE k = 0 IF d <= 1s");
- assertInvalidMessage("Slice conditions are not supported on durations",
- "UPDATE %s SET v = 3 WHERE k = 0 IF d < 1s");
-
- execute("INSERT INTO %s (k, v, d) VALUES (1, 1, 2s)");
-
- assertRows(execute("UPDATE %s SET v = 4 WHERE k = 1 IF d = 1s"), row(false, Duration.from("2s")));
- assertRows(execute("UPDATE %s SET v = 3 WHERE k = 1 IF d = 2s"), row(true));
-
- assertRows(execute("SELECT * FROM %s WHERE k = 1"), row(1, Duration.from("2s"), 3));
-
- assertRows(execute("UPDATE %s SET d = 10s WHERE k = 1 IF d != 2s"), row(false, Duration.from("2s")));
- assertRows(execute("UPDATE %s SET v = 6 WHERE k = 1 IF d != 1s"), row(true));
-
- assertRows(execute("SELECT * FROM %s WHERE k = 1"), row(1, Duration.from("2s"), 6));
-
- assertRows(execute("UPDATE %s SET v = 5 WHERE k = 1 IF d IN (1s, 5s)"), row(false, Duration.from("2s")));
- assertRows(execute("UPDATE %s SET d = 10s WHERE k = 1 IF d IN (1s, 2s)"), row(true));
-
- assertRows(execute("SELECT * FROM %s WHERE k = 1"), row(1, Duration.from("10s"), 6));
- }
-
- @Test
- public void testConditionalOnDurationWithinLists() throws Throwable
+ public void testConditionalOnDurationWithinMaps() throws Throwable
{
for (Boolean frozen : new Boolean[]{Boolean.FALSE, Boolean.TRUE})
{
- String listType = String.format(frozen ? "frozen<%s>" : "%s", "list<duration>");
+ String mapType = String.format(frozen ? "frozen<%s>" : "%s", "map<int, duration>");
- createTable("CREATE TABLE %s (k int PRIMARY KEY, v int, l " + listType + " )");
+ createTable("CREATE TABLE %s (k int PRIMARY KEY, v int, m " + mapType + " )");
assertInvalidMessage("Slice conditions are not supported on collections containing durations",
- "UPDATE %s SET v = 3 WHERE k = 0 IF l > [1s, 2s]");
+ "UPDATE %s SET v = 3 WHERE k = 0 IF m > {1: 1s, 2: 2s}");
assertInvalidMessage("Slice conditions are not supported on collections containing durations",
- "UPDATE %s SET v = 3 WHERE k = 0 IF l >= [1s, 2s]");
+ "UPDATE %s SET v = 3 WHERE k = 0 IF m >= {1: 1s, 2: 2s}");
assertInvalidMessage("Slice conditions are not supported on collections containing durations",
- "UPDATE %s SET v = 3 WHERE k = 0 IF l <= [1s, 2s]");
+ "UPDATE %s SET v = 3 WHERE k = 0 IF m <= {1: 1s, 2: 2s}");
assertInvalidMessage("Slice conditions are not supported on collections containing durations",
- "UPDATE %s SET v = 3 WHERE k = 0 IF l < [1s, 2s]");
+ "UPDATE %s SET v = 3 WHERE k = 0 IF m < {1: 1s, 2: 2s}");
- execute("INSERT INTO %s (k, v, l) VALUES (1, 1, [1s, 2s])");
+ execute("INSERT INTO %s (k, v, m) VALUES (1, 1, {1: 1s, 2: 2s})");
- assertRows(execute("UPDATE %s SET v = 4 WHERE k = 1 IF l = [2s]"), row(false, list(Duration.from("1000ms"), Duration.from("2s"))));
- assertRows(execute("UPDATE %s SET v = 3 WHERE k = 1 IF l = [1s, 2s]"), row(true));
+ assertRows(execute("UPDATE %s SET v = 4 WHERE k = 1 IF m = {2: 2s}"), row(false, map(1, Duration.from("1000ms"), 2, Duration.from("2s"))));
+ assertRows(execute("UPDATE %s SET v = 3 WHERE k = 1 IF m = {1: 1s, 2: 2s}"), row(true));
- assertRows(execute("SELECT * FROM %s WHERE k = 1"), row(1, list(Duration.from("1000ms"), Duration.from("2s")), 3));
+ assertRows(execute("SELECT * FROM %s WHERE k = 1"), row(1, map(1, Duration.from("1000ms"), 2, Duration.from("2s")), 3));
- assertRows(execute("UPDATE %s SET l = [10s] WHERE k = 1 IF l != [1s, 2s]"), row(false, list(Duration.from("1000ms"), Duration.from("2s"))));
- assertRows(execute("UPDATE %s SET v = 6 WHERE k = 1 IF l != [1s]"), row(true));
+ assertRows(execute("UPDATE %s SET m = {1 :10s} WHERE k = 1 IF m != {1: 1s, 2: 2s}"), row(false, map(1, Duration.from("1000ms"), 2, Duration.from("2s"))));
+ assertRows(execute("UPDATE %s SET v = 6 WHERE k = 1 IF m != {1: 1s}"), row(true));
- assertRows(execute("SELECT * FROM %s WHERE k = 1"), row(1, list(Duration.from("1000ms"), Duration.from("2s")), 6));
+ assertRows(execute("SELECT * FROM %s WHERE k = 1"), row(1, map(1, Duration.from("1000ms"), 2, Duration.from("2s")), 6));
- assertRows(execute("UPDATE %s SET v = 5 WHERE k = 1 IF l IN ([1s], [1s, 5s])"), row(false, list(Duration.from("1000ms"), Duration.from("2s"))));
- assertRows(execute("UPDATE %s SET l = [5s, 10s] WHERE k = 1 IF l IN ([1s], [1s, 2s])"), row(true));
+ assertRows(execute("UPDATE %s SET v = 5 WHERE k = 1 IF m IN ({1: 1s}, {1: 5s})"), row(false, map(1, Duration.from("1000ms"), 2, Duration.from("2s"))));
+ assertRows(execute("UPDATE %s SET m = {1: 5s, 2: 10s} WHERE k = 1 IF m IN ({1: 1s}, {1: 1s, 2: 2s})"), row(true));
- assertRows(execute("SELECT * FROM %s WHERE k = 1"), row(1, list(Duration.from("5s"), Duration.from("10s")), 6));
+ assertRows(execute("SELECT * FROM %s WHERE k = 1"), row(1, map(1, Duration.from("5s"), 2, Duration.from("10s")), 6));
- assertInvalidMessage("Slice conditions are not supported on durations",
- "UPDATE %s SET v = 3 WHERE k = 0 IF l[0] > 1s");
- assertInvalidMessage("Slice conditions are not supported on durations",
- "UPDATE %s SET v = 3 WHERE k = 0 IF l[0] >= 1s");
- assertInvalidMessage("Slice conditions are not supported on durations",
- "UPDATE %s SET v = 3 WHERE k = 0 IF l[0] <= 1s");
- assertInvalidMessage("Slice conditions are not supported on durations",
- "UPDATE %s SET v = 3 WHERE k = 0 IF l[0] < 1s");
+ assertInvalidMessage("Slice conditions ( > ) are not supported on durations",
+ "UPDATE %s SET v = 3 WHERE k = 0 IF m[1] > 1s");
+ assertInvalidMessage("Slice conditions ( >= ) are not supported on durations",
+ "UPDATE %s SET v = 3 WHERE k = 0 IF m[1] >= 1s");
+ assertInvalidMessage("Slice conditions ( <= ) are not supported on durations",
+ "UPDATE %s SET v = 3 WHERE k = 0 IF m[1] <= 1s");
+ assertInvalidMessage("Slice conditions ( < ) are not supported on durations",
+ "UPDATE %s SET v = 3 WHERE k = 0 IF m[1] < 1s");
- assertRows(execute("UPDATE %s SET v = 4 WHERE k = 1 IF l[0] = 2s"), row(false, list(Duration.from("5s"), Duration.from("10s"))));
- assertRows(execute("UPDATE %s SET v = 3 WHERE k = 1 IF l[0] = 5s"), row(true));
+ assertRows(execute("UPDATE %s SET v = 4 WHERE k = 1 IF m[1] = 2s"), row(false, map(1, Duration.from("5s"), 2, Duration.from("10s"))));
+ assertRows(execute("UPDATE %s SET v = 3 WHERE k = 1 IF m[1] = 5s"), row(true));
- assertRows(execute("SELECT * FROM %s WHERE k = 1"), row(1, list(Duration.from("5s"), Duration.from("10s")), 3));
+ assertRows(execute("SELECT * FROM %s WHERE k = 1"), row(1, map(1, Duration.from("5s"), 2, Duration.from("10s")), 3));
- assertRows(execute("UPDATE %s SET l = [10s] WHERE k = 1 IF l[1] != 10s"), row(false, list(Duration.from("5s"), Duration.from("10s"))));
- assertRows(execute("UPDATE %s SET v = 6 WHERE k = 1 IF l[1] != 1s"), row(true));
+ assertRows(execute("UPDATE %s SET m = {1: 10s} WHERE k = 1 IF m[2] != 10s"), row(false, map(1, Duration.from("5s"), 2, Duration.from("10s"))));
+ assertRows(execute("UPDATE %s SET v = 6 WHERE k = 1 IF m[2] != 1s"), row(true));
- assertRows(execute("SELECT * FROM %s WHERE k = 1"), row(1, list(Duration.from("5s"), Duration.from("10s")), 6));
+ assertRows(execute("SELECT * FROM %s WHERE k = 1"), row(1, map(1, Duration.from("5s"), 2, Duration.from("10s")), 6));
- assertRows(execute("UPDATE %s SET v = 5 WHERE k = 1 IF l[0] IN (2s, 10s)"), row(false, list(Duration.from("5s"), Duration.from("10s"))));
- assertRows(execute("UPDATE %s SET l = [6s, 12s] WHERE k = 1 IF l[0] IN (5s, 10s)"), row(true));
+ assertRows(execute("UPDATE %s SET v = 5 WHERE k = 1 IF m[1] IN (2s, 10s)"), row(false, map(1, Duration.from("5s"), 2, Duration.from("10s"))));
+ assertRows(execute("UPDATE %s SET m = {1: 6s, 2: 12s} WHERE k = 1 IF m[1] IN (5s, 10s)"), row(true));
- assertRows(execute("SELECT * FROM %s WHERE k = 1"), row(1, list(Duration.from("6s"), Duration.from("12s")), 6));
+ assertRows(execute("SELECT * FROM %s WHERE k = 1"), row(1, map(1, Duration.from("6s"), 2, Duration.from("12s")), 6));
}
}
@@ -2229,67 +2300,6 @@
}
@Test
- public void testConditionalOnDurationWithinMaps() throws Throwable
- {
- for (Boolean frozen : new Boolean[]{Boolean.FALSE, Boolean.TRUE})
- {
- String mapType = String.format(frozen ? "frozen<%s>" : "%s", "map<int, duration>");
-
- createTable("CREATE TABLE %s (k int PRIMARY KEY, v int, m " + mapType + " )");
-
- assertInvalidMessage("Slice conditions are not supported on collections containing durations",
- "UPDATE %s SET v = 3 WHERE k = 0 IF m > {1: 1s, 2: 2s}");
- assertInvalidMessage("Slice conditions are not supported on collections containing durations",
- "UPDATE %s SET v = 3 WHERE k = 0 IF m >= {1: 1s, 2: 2s}");
- assertInvalidMessage("Slice conditions are not supported on collections containing durations",
- "UPDATE %s SET v = 3 WHERE k = 0 IF m <= {1: 1s, 2: 2s}");
- assertInvalidMessage("Slice conditions are not supported on collections containing durations",
- "UPDATE %s SET v = 3 WHERE k = 0 IF m < {1: 1s, 2: 2s}");
-
- execute("INSERT INTO %s (k, v, m) VALUES (1, 1, {1: 1s, 2: 2s})");
-
- assertRows(execute("UPDATE %s SET v = 4 WHERE k = 1 IF m = {2: 2s}"), row(false, map(1, Duration.from("1000ms"), 2, Duration.from("2s"))));
- assertRows(execute("UPDATE %s SET v = 3 WHERE k = 1 IF m = {1: 1s, 2: 2s}"), row(true));
-
- assertRows(execute("SELECT * FROM %s WHERE k = 1"), row(1, map(1, Duration.from("1000ms"), 2, Duration.from("2s")), 3));
-
- assertRows(execute("UPDATE %s SET m = {1 :10s} WHERE k = 1 IF m != {1: 1s, 2: 2s}"), row(false, map(1, Duration.from("1000ms"), 2, Duration.from("2s"))));
- assertRows(execute("UPDATE %s SET v = 6 WHERE k = 1 IF m != {1: 1s}"), row(true));
-
- assertRows(execute("SELECT * FROM %s WHERE k = 1"), row(1, map(1, Duration.from("1000ms"), 2, Duration.from("2s")), 6));
-
- assertRows(execute("UPDATE %s SET v = 5 WHERE k = 1 IF m IN ({1: 1s}, {1: 5s})"), row(false, map(1, Duration.from("1000ms"), 2, Duration.from("2s"))));
- assertRows(execute("UPDATE %s SET m = {1: 5s, 2: 10s} WHERE k = 1 IF m IN ({1: 1s}, {1: 1s, 2: 2s})"), row(true));
-
- assertRows(execute("SELECT * FROM %s WHERE k = 1"), row(1, map(1, Duration.from("5s"), 2, Duration.from("10s")), 6));
-
- assertInvalidMessage("Slice conditions are not supported on durations",
- "UPDATE %s SET v = 3 WHERE k = 0 IF m[1] > 1s");
- assertInvalidMessage("Slice conditions are not supported on durations",
- "UPDATE %s SET v = 3 WHERE k = 0 IF m[1] >= 1s");
- assertInvalidMessage("Slice conditions are not supported on durations",
- "UPDATE %s SET v = 3 WHERE k = 0 IF m[1] <= 1s");
- assertInvalidMessage("Slice conditions are not supported on durations",
- "UPDATE %s SET v = 3 WHERE k = 0 IF m[1] < 1s");
-
- assertRows(execute("UPDATE %s SET v = 4 WHERE k = 1 IF m[1] = 2s"), row(false, map(1, Duration.from("5s"), 2, Duration.from("10s"))));
- assertRows(execute("UPDATE %s SET v = 3 WHERE k = 1 IF m[1] = 5s"), row(true));
-
- assertRows(execute("SELECT * FROM %s WHERE k = 1"), row(1, map(1, Duration.from("5s"), 2, Duration.from("10s")), 3));
-
- assertRows(execute("UPDATE %s SET m = {1: 10s} WHERE k = 1 IF m[2] != 10s"), row(false, map(1, Duration.from("5s"), 2, Duration.from("10s"))));
- assertRows(execute("UPDATE %s SET v = 6 WHERE k = 1 IF m[2] != 1s"), row(true));
-
- assertRows(execute("SELECT * FROM %s WHERE k = 1"), row(1, map(1, Duration.from("5s"), 2, Duration.from("10s")), 6));
-
- assertRows(execute("UPDATE %s SET v = 5 WHERE k = 1 IF m[1] IN (2s, 10s)"), row(false, map(1, Duration.from("5s"), 2, Duration.from("10s"))));
- assertRows(execute("UPDATE %s SET m = {1: 6s, 2: 12s} WHERE k = 1 IF m[1] IN (5s, 10s)"), row(true));
-
- assertRows(execute("SELECT * FROM %s WHERE k = 1"), row(1, map(1, Duration.from("6s"), 2, Duration.from("12s")), 6));
- }
- }
-
- @Test
public void testConditionalOnDurationWithinUdts() throws Throwable
{
String udt = createType("CREATE TYPE %s (i int, d duration)");
@@ -2326,13 +2336,13 @@
assertRows(execute("SELECT * FROM %s WHERE k = 1"), row(1, userType("i", 1, "d", Duration.from("10s")), 6));
- assertInvalidMessage("Slice conditions are not supported on durations",
+ assertInvalidMessage("Slice conditions ( > ) are not supported on durations",
"UPDATE %s SET v = 3 WHERE k = 0 IF u.d > 1s");
- assertInvalidMessage("Slice conditions are not supported on durations",
+ assertInvalidMessage("Slice conditions ( >= ) are not supported on durations",
"UPDATE %s SET v = 3 WHERE k = 0 IF u.d >= 1s");
- assertInvalidMessage("Slice conditions are not supported on durations",
+ assertInvalidMessage("Slice conditions ( <= ) are not supported on durations",
"UPDATE %s SET v = 3 WHERE k = 0 IF u.d <= 1s");
- assertInvalidMessage("Slice conditions are not supported on durations",
+ assertInvalidMessage("Slice conditions ( < ) are not supported on durations",
"UPDATE %s SET v = 3 WHERE k = 0 IF u.d < 1s");
assertRows(execute("UPDATE %s SET v = 4 WHERE k = 1 IF u.d = 2s"), row(false, userType("i", 1, "d", Duration.from("10s"))));
diff --git a/test/unit/org/apache/cassandra/cql3/validation/operations/SelectGroupByTest.java b/test/unit/org/apache/cassandra/cql3/validation/operations/SelectGroupByTest.java
index 5c51494..17d06ac 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/operations/SelectGroupByTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/operations/SelectGroupByTest.java
@@ -26,594 +26,582 @@
@Test
public void testGroupByWithoutPaging() throws Throwable
{
- for (String compactOption : new String[] { "", " WITH COMPACT STORAGE" })
- {
- createTable("CREATE TABLE %s (a int, b int, c int, d int, e int, primary key (a, b, c, d))"
- + compactOption);
+ createTable("CREATE TABLE %s (a int, b int, c int, d int, e int, primary key (a, b, c, d))");
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (1, 2, 1, 3, 6)");
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (1, 2, 2, 6, 12)");
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (1, 3, 2, 12, 24)");
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (1, 4, 2, 12, 24)");
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (1, 4, 2, 6, 12)");
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (2, 2, 3, 3, 6)");
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (2, 4, 3, 6, 12)");
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (3, 3, 2, 12, 24)");
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (4, 8, 2, 12, 24)");
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (1, 2, 1, 3, 6)");
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (1, 2, 2, 6, 12)");
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (1, 3, 2, 12, 24)");
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (1, 4, 2, 12, 24)");
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (1, 4, 2, 6, 12)");
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (2, 2, 3, 3, 6)");
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (2, 4, 3, 6, 12)");
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (3, 3, 2, 12, 24)");
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (4, 8, 2, 12, 24)");
- // Makes sure that we have some tombstones
- execute("DELETE FROM %s WHERE a = 1 AND b = 3 AND c = 2 AND d = 12");
- execute("DELETE FROM %s WHERE a = 3");
+ // Makes sure that we have some tombstones
+ execute("DELETE FROM %s WHERE a = 1 AND b = 3 AND c = 2 AND d = 12");
+ execute("DELETE FROM %s WHERE a = 3");
- // Range queries
- assertRows(execute("SELECT a, b, e, count(b), max(e) FROM %s GROUP BY a"),
- row(1, 2, 6, 4L, 24),
- row(2, 2, 6, 2L, 12),
- row(4, 8, 24, 1L, 24));
+ // Range queries
+ assertRows(execute("SELECT a, b, e, count(b), max(e) FROM %s GROUP BY a"),
+ row(1, 2, 6, 4L, 24),
+ row(2, 2, 6, 2L, 12),
+ row(4, 8, 24, 1L, 24));
- assertRows(execute("SELECT a, b, e, count(b), max(e) FROM %s GROUP BY a, b"),
- row(1, 2, 6, 2L, 12),
- row(1, 4, 12, 2L, 24),
- row(2, 2, 6, 1L, 6),
- row(2, 4, 12, 1L, 12),
- row(4, 8, 24, 1L, 24));
+ assertRows(execute("SELECT a, b, e, count(b), max(e) FROM %s GROUP BY a, b"),
+ row(1, 2, 6, 2L, 12),
+ row(1, 4, 12, 2L, 24),
+ row(2, 2, 6, 1L, 6),
+ row(2, 4, 12, 1L, 12),
+ row(4, 8, 24, 1L, 24));
- assertRows(execute("SELECT a, b, e, count(b), max(e) FROM %s WHERE b = 2 GROUP BY a, b ALLOW FILTERING"),
- row(1, 2, 6, 2L, 12),
- row(2, 2, 6, 1L, 6));
+ assertRows(execute("SELECT a, b, e, count(b), max(e) FROM %s WHERE b = 2 GROUP BY a, b ALLOW FILTERING"),
+ row(1, 2, 6, 2L, 12),
+ row(2, 2, 6, 1L, 6));
- assertEmpty(execute("SELECT a, b, e, count(b), max(e) FROM %s WHERE b IN () GROUP BY a, b ALLOW FILTERING"));
+ assertEmpty(execute("SELECT a, b, e, count(b), max(e) FROM %s WHERE b IN () GROUP BY a, b ALLOW FILTERING"));
- // Range queries without aggregates
- assertRows(execute("SELECT a, b, c, d FROM %s GROUP BY a, b, c"),
- row(1, 2, 1, 3),
- row(1, 2, 2, 6),
- row(1, 4, 2, 6),
- row(2, 2, 3, 3),
- row(2, 4, 3, 6),
- row(4, 8, 2, 12));
+ // Range queries without aggregates
+ assertRows(execute("SELECT a, b, c, d FROM %s GROUP BY a, b, c"),
+ row(1, 2, 1, 3),
+ row(1, 2, 2, 6),
+ row(1, 4, 2, 6),
+ row(2, 2, 3, 3),
+ row(2, 4, 3, 6),
+ row(4, 8, 2, 12));
- assertRows(execute("SELECT a, b, c, d FROM %s GROUP BY a, b"),
- row(1, 2, 1, 3),
- row(1, 4, 2, 6),
- row(2, 2, 3, 3),
- row(2, 4, 3, 6),
- row(4, 8, 2, 12));
+ assertRows(execute("SELECT a, b, c, d FROM %s GROUP BY a, b"),
+ row(1, 2, 1, 3),
+ row(1, 4, 2, 6),
+ row(2, 2, 3, 3),
+ row(2, 4, 3, 6),
+ row(4, 8, 2, 12));
- // Range queries with wildcard
- assertRows(execute("SELECT * FROM %s GROUP BY a, b, c"),
- row(1, 2, 1, 3, 6),
- row(1, 2, 2, 6, 12),
- row(1, 4, 2, 6, 12),
- row(2, 2, 3, 3, 6),
- row(2, 4, 3, 6, 12),
- row(4, 8, 2, 12, 24));
+ // Range queries with wildcard
+ assertRows(execute("SELECT * FROM %s GROUP BY a, b, c"),
+ row(1, 2, 1, 3, 6),
+ row(1, 2, 2, 6, 12),
+ row(1, 4, 2, 6, 12),
+ row(2, 2, 3, 3, 6),
+ row(2, 4, 3, 6, 12),
+ row(4, 8, 2, 12, 24));
- assertRows(execute("SELECT * FROM %s GROUP BY a, b"),
- row(1, 2, 1, 3, 6),
- row(1, 4, 2, 6, 12),
- row(2, 2, 3, 3, 6),
- row(2, 4, 3, 6, 12),
- row(4, 8, 2, 12, 24));
+ assertRows(execute("SELECT * FROM %s GROUP BY a, b"),
+ row(1, 2, 1, 3, 6),
+ row(1, 4, 2, 6, 12),
+ row(2, 2, 3, 3, 6),
+ row(2, 4, 3, 6, 12),
+ row(4, 8, 2, 12, 24));
- // Range query with LIMIT
- assertRows(execute("SELECT a, b, e, count(b), max(e) FROM %s GROUP BY a, b LIMIT 2"),
- row(1, 2, 6, 2L, 12),
- row(1, 4, 12, 2L, 24));
+ // Range query with LIMIT
+ assertRows(execute("SELECT a, b, e, count(b), max(e) FROM %s GROUP BY a, b LIMIT 2"),
+ row(1, 2, 6, 2L, 12),
+ row(1, 4, 12, 2L, 24));
- // Range queries with PER PARTITION LIMIT
- assertRows(execute("SELECT a, b, e, count(b), max(e) FROM %s GROUP BY a, b PER PARTITION LIMIT 1"),
- row(1, 2, 6, 2L, 12),
- row(2, 2, 6, 1L, 6),
- row(4, 8, 24, 1L, 24));
+ // Range queries with PER PARTITION LIMIT
+ assertRows(execute("SELECT a, b, e, count(b), max(e) FROM %s GROUP BY a, b PER PARTITION LIMIT 1"),
+ row(1, 2, 6, 2L, 12),
+ row(2, 2, 6, 1L, 6),
+ row(4, 8, 24, 1L, 24));
- assertRows(execute("SELECT a, b, e, count(b), max(e) FROM %s GROUP BY a PER PARTITION LIMIT 2"),
- row(1, 2, 6, 4L, 24),
- row(2, 2, 6, 2L, 12),
- row(4, 8, 24, 1L, 24));
+ assertRows(execute("SELECT a, b, e, count(b), max(e) FROM %s GROUP BY a PER PARTITION LIMIT 2"),
+ row(1, 2, 6, 4L, 24),
+ row(2, 2, 6, 2L, 12),
+ row(4, 8, 24, 1L, 24));
- // Range query with PER PARTITION LIMIT and LIMIT
- assertRows(execute("SELECT a, b, e, count(b), max(e) FROM %s GROUP BY a, b PER PARTITION LIMIT 1 LIMIT 2"),
- row(1, 2, 6, 2L, 12),
- row(2, 2, 6, 1L, 6));
+ // Range query with PER PARTITION LIMIT and LIMIT
+ assertRows(execute("SELECT a, b, e, count(b), max(e) FROM %s GROUP BY a, b PER PARTITION LIMIT 1 LIMIT 2"),
+ row(1, 2, 6, 2L, 12),
+ row(2, 2, 6, 1L, 6));
- assertRows(execute("SELECT a, b, e, count(b), max(e) FROM %s GROUP BY a PER PARTITION LIMIT 2"),
- row(1, 2, 6, 4L, 24),
- row(2, 2, 6, 2L, 12),
- row(4, 8, 24, 1L, 24));
+ assertRows(execute("SELECT a, b, e, count(b), max(e) FROM %s GROUP BY a PER PARTITION LIMIT 2"),
+ row(1, 2, 6, 4L, 24),
+ row(2, 2, 6, 2L, 12),
+ row(4, 8, 24, 1L, 24));
- // Range queries without aggregates and with LIMIT
- assertRows(execute("SELECT a, b, c, d FROM %s GROUP BY a, b, c LIMIT 3"),
- row(1, 2, 1, 3),
- row(1, 2, 2, 6),
- row(1, 4, 2, 6));
+ // Range queries without aggregates and with LIMIT
+ assertRows(execute("SELECT a, b, c, d FROM %s GROUP BY a, b, c LIMIT 3"),
+ row(1, 2, 1, 3),
+ row(1, 2, 2, 6),
+ row(1, 4, 2, 6));
- assertRows(execute("SELECT a, b, c, d FROM %s GROUP BY a, b LIMIT 3"),
- row(1, 2, 1, 3),
- row(1, 4, 2, 6),
- row(2, 2, 3, 3));
+ assertRows(execute("SELECT a, b, c, d FROM %s GROUP BY a, b LIMIT 3"),
+ row(1, 2, 1, 3),
+ row(1, 4, 2, 6),
+ row(2, 2, 3, 3));
- // Range queries with wildcard and with LIMIT
- assertRows(execute("SELECT * FROM %s GROUP BY a, b, c LIMIT 3"),
- row(1, 2, 1, 3, 6),
- row(1, 2, 2, 6, 12),
- row(1, 4, 2, 6, 12));
+ // Range queries with wildcard and with LIMIT
+ assertRows(execute("SELECT * FROM %s GROUP BY a, b, c LIMIT 3"),
+ row(1, 2, 1, 3, 6),
+ row(1, 2, 2, 6, 12),
+ row(1, 4, 2, 6, 12));
- assertRows(execute("SELECT * FROM %s GROUP BY a, b LIMIT 3"),
- row(1, 2, 1, 3, 6),
- row(1, 4, 2, 6, 12),
- row(2, 2, 3, 3, 6));
+ assertRows(execute("SELECT * FROM %s GROUP BY a, b LIMIT 3"),
+ row(1, 2, 1, 3, 6),
+ row(1, 4, 2, 6, 12),
+ row(2, 2, 3, 3, 6));
- // Range queries without aggregates and with PER PARTITION LIMIT
- assertRows(execute("SELECT a, b, c, d FROM %s GROUP BY a, b, c PER PARTITION LIMIT 2"),
- row(1, 2, 1, 3),
- row(1, 2, 2, 6),
- row(2, 2, 3, 3),
- row(2, 4, 3, 6),
- row(4, 8, 2, 12));
+ // Range queries without aggregates and with PER PARTITION LIMIT
+ assertRows(execute("SELECT a, b, c, d FROM %s GROUP BY a, b, c PER PARTITION LIMIT 2"),
+ row(1, 2, 1, 3),
+ row(1, 2, 2, 6),
+ row(2, 2, 3, 3),
+ row(2, 4, 3, 6),
+ row(4, 8, 2, 12));
- assertRows(execute("SELECT a, b, c, d FROM %s GROUP BY a, b PER PARTITION LIMIT 1"),
- row(1, 2, 1, 3),
- row(2, 2, 3, 3),
- row(4, 8, 2, 12));
+ assertRows(execute("SELECT a, b, c, d FROM %s GROUP BY a, b PER PARTITION LIMIT 1"),
+ row(1, 2, 1, 3),
+ row(2, 2, 3, 3),
+ row(4, 8, 2, 12));
- // Range queries with wildcard and with PER PARTITION LIMIT
- assertRows(execute("SELECT * FROM %s GROUP BY a, b, c PER PARTITION LIMIT 2"),
- row(1, 2, 1, 3, 6),
- row(1, 2, 2, 6, 12),
- row(2, 2, 3, 3, 6),
- row(2, 4, 3, 6, 12),
- row(4, 8, 2, 12, 24));
+ // Range queries with wildcard and with PER PARTITION LIMIT
+ assertRows(execute("SELECT * FROM %s GROUP BY a, b, c PER PARTITION LIMIT 2"),
+ row(1, 2, 1, 3, 6),
+ row(1, 2, 2, 6, 12),
+ row(2, 2, 3, 3, 6),
+ row(2, 4, 3, 6, 12),
+ row(4, 8, 2, 12, 24));
- assertRows(execute("SELECT * FROM %s GROUP BY a, b PER PARTITION LIMIT 1"),
- row(1, 2, 1, 3, 6),
- row(2, 2, 3, 3, 6),
- row(4, 8, 2, 12, 24));
+ assertRows(execute("SELECT * FROM %s GROUP BY a, b PER PARTITION LIMIT 1"),
+ row(1, 2, 1, 3, 6),
+ row(2, 2, 3, 3, 6),
+ row(4, 8, 2, 12, 24));
- // Range queries without aggregates, with PER PARTITION LIMIT and LIMIT
- assertRows(execute("SELECT a, b, c, d FROM %s GROUP BY a, b, c PER PARTITION LIMIT 2 LIMIT 3"),
- row(1, 2, 1, 3),
- row(1, 2, 2, 6),
- row(2, 2, 3, 3));
+ // Range queries without aggregates, with PER PARTITION LIMIT and LIMIT
+ assertRows(execute("SELECT a, b, c, d FROM %s GROUP BY a, b, c PER PARTITION LIMIT 2 LIMIT 3"),
+ row(1, 2, 1, 3),
+ row(1, 2, 2, 6),
+ row(2, 2, 3, 3));
- // Range queries with wildcard, with PER PARTITION LIMIT and LIMIT
- assertRows(execute("SELECT * FROM %s GROUP BY a, b, c PER PARTITION LIMIT 2 LIMIT 3"),
- row(1, 2, 1, 3, 6),
- row(1, 2, 2, 6, 12),
- row(2, 2, 3, 3, 6));
+ // Range queries with wildcard, with PER PARTITION LIMIT and LIMIT
+ assertRows(execute("SELECT * FROM %s GROUP BY a, b, c PER PARTITION LIMIT 2 LIMIT 3"),
+ row(1, 2, 1, 3, 6),
+ row(1, 2, 2, 6, 12),
+ row(2, 2, 3, 3, 6));
- // Range query with DISTINCT
- assertRows(execute("SELECT DISTINCT a, count(a)FROM %s GROUP BY a"),
- row(1, 1L),
- row(2, 1L),
- row(4, 1L));
+ // Range query with DISTINCT
+ assertRows(execute("SELECT DISTINCT a, count(a)FROM %s GROUP BY a"),
+ row(1, 1L),
+ row(2, 1L),
+ row(4, 1L));
- assertInvalidMessage("Grouping on clustering columns is not allowed for SELECT DISTINCT queries",
- "SELECT DISTINCT a, count(a)FROM %s GROUP BY a, b");
+ assertInvalidMessage("Grouping on clustering columns is not allowed for SELECT DISTINCT queries",
+ "SELECT DISTINCT a, count(a)FROM %s GROUP BY a, b");
- // Range query with DISTINCT and LIMIT
- assertRows(execute("SELECT DISTINCT a, count(a)FROM %s GROUP BY a LIMIT 2"),
- row(1, 1L),
- row(2, 1L));
+ // Range query with DISTINCT and LIMIT
+ assertRows(execute("SELECT DISTINCT a, count(a)FROM %s GROUP BY a LIMIT 2"),
+ row(1, 1L),
+ row(2, 1L));
- assertInvalidMessage("Grouping on clustering columns is not allowed for SELECT DISTINCT queries",
- "SELECT DISTINCT a, count(a)FROM %s GROUP BY a, b LIMIT 2");
+ assertInvalidMessage("Grouping on clustering columns is not allowed for SELECT DISTINCT queries",
+ "SELECT DISTINCT a, count(a)FROM %s GROUP BY a, b LIMIT 2");
- // Range query with ORDER BY
- assertInvalidMessage("ORDER BY is only supported when the partition key is restricted by an EQ or an IN",
- "SELECT a, b, c, count(b), max(e) FROM %s GROUP BY a, b ORDER BY b DESC, c DESC");
+ // Range query with ORDER BY
+ assertInvalidMessage("ORDER BY is only supported when the partition key is restricted by an EQ or an IN",
+ "SELECT a, b, c, count(b), max(e) FROM %s GROUP BY a, b ORDER BY b DESC, c DESC");
- // Single partition queries
- assertRows(execute("SELECT a, b, e, count(b), max(e) FROM %s WHERE a = 1 GROUP BY a, b, c"),
- row(1, 2, 6, 1L, 6),
- row(1, 2, 12, 1L, 12),
- row(1, 4, 12, 2L, 24));
+ // Single partition queries
+ assertRows(execute("SELECT a, b, e, count(b), max(e) FROM %s WHERE a = 1 GROUP BY a, b, c"),
+ row(1, 2, 6, 1L, 6),
+ row(1, 2, 12, 1L, 12),
+ row(1, 4, 12, 2L, 24));
- assertRows(execute("SELECT a, b, e, count(b), max(e) FROM %s WHERE a = 1 GROUP BY b, c"),
- row(1, 2, 6, 1L, 6),
- row(1, 2, 12, 1L, 12),
- row(1, 4, 12, 2L, 24));
+ assertRows(execute("SELECT a, b, e, count(b), max(e) FROM %s WHERE a = 1 GROUP BY b, c"),
+ row(1, 2, 6, 1L, 6),
+ row(1, 2, 12, 1L, 12),
+ row(1, 4, 12, 2L, 24));
- assertRows(execute("SELECT a, b, e, count(b), max(e) FROM %s WHERE a = 1 AND b = 2 GROUP BY a, b, c"),
- row(1, 2, 6, 1L, 6),
- row(1, 2, 12, 1L, 12));
+ assertRows(execute("SELECT a, b, e, count(b), max(e) FROM %s WHERE a = 1 AND b = 2 GROUP BY a, b, c"),
+ row(1, 2, 6, 1L, 6),
+ row(1, 2, 12, 1L, 12));
- assertRows(execute("SELECT a, b, e, count(b), max(e) FROM %s WHERE a = 1 AND b = 2 GROUP BY a, c"),
- row(1, 2, 6, 1L, 6),
- row(1, 2, 12, 1L, 12));
+ assertRows(execute("SELECT a, b, e, count(b), max(e) FROM %s WHERE a = 1 AND b = 2 GROUP BY a, c"),
+ row(1, 2, 6, 1L, 6),
+ row(1, 2, 12, 1L, 12));
- assertRows(execute("SELECT a, b, e, count(b), max(e) FROM %s WHERE a = 1 AND b = 2 GROUP BY c"),
- row(1, 2, 6, 1L, 6),
- row(1, 2, 12, 1L, 12));
+ assertRows(execute("SELECT a, b, e, count(b), max(e) FROM %s WHERE a = 1 AND b = 2 GROUP BY c"),
+ row(1, 2, 6, 1L, 6),
+ row(1, 2, 12, 1L, 12));
- // Single partition queries without aggregates
- assertRows(execute("SELECT a, b, c, d FROM %s WHERE a = 1 GROUP BY a, b"),
- row(1, 2, 1, 3),
- row(1, 4, 2, 6));
+ // Single partition queries without aggregates
+ assertRows(execute("SELECT a, b, c, d FROM %s WHERE a = 1 GROUP BY a, b"),
+ row(1, 2, 1, 3),
+ row(1, 4, 2, 6));
- assertRows(execute("SELECT a, b, c, d FROM %s WHERE a = 1 GROUP BY a, b, c"),
- row(1, 2, 1, 3),
- row(1, 2, 2, 6),
- row(1, 4, 2, 6));
+ assertRows(execute("SELECT a, b, c, d FROM %s WHERE a = 1 GROUP BY a, b, c"),
+ row(1, 2, 1, 3),
+ row(1, 2, 2, 6),
+ row(1, 4, 2, 6));
- assertRows(execute("SELECT a, b, c, d FROM %s WHERE a = 1 GROUP BY b, c"),
- row(1, 2, 1, 3),
- row(1, 2, 2, 6),
- row(1, 4, 2, 6));
+ assertRows(execute("SELECT a, b, c, d FROM %s WHERE a = 1 GROUP BY b, c"),
+ row(1, 2, 1, 3),
+ row(1, 2, 2, 6),
+ row(1, 4, 2, 6));
- assertRows(execute("SELECT a, b, c, d FROM %s WHERE a = 1 and token(a) = token(1) GROUP BY b, c"),
- row(1, 2, 1, 3),
- row(1, 2, 2, 6),
- row(1, 4, 2, 6));
+ assertRows(execute("SELECT a, b, c, d FROM %s WHERE a = 1 and token(a) = token(1) GROUP BY b, c"),
+ row(1, 2, 1, 3),
+ row(1, 2, 2, 6),
+ row(1, 4, 2, 6));
- // Single partition queries with wildcard
- assertRows(execute("SELECT * FROM %s WHERE a = 1 GROUP BY a, b, c"),
- row(1, 2, 1, 3, 6),
- row(1, 2, 2, 6, 12),
- row(1, 4, 2, 6, 12));
+ // Single partition queries with wildcard
+ assertRows(execute("SELECT * FROM %s WHERE a = 1 GROUP BY a, b, c"),
+ row(1, 2, 1, 3, 6),
+ row(1, 2, 2, 6, 12),
+ row(1, 4, 2, 6, 12));
- assertRows(execute("SELECT * FROM %s WHERE a = 1 GROUP BY a, b"),
- row(1, 2, 1, 3, 6),
- row(1, 4, 2, 6, 12));
+ assertRows(execute("SELECT * FROM %s WHERE a = 1 GROUP BY a, b"),
+ row(1, 2, 1, 3, 6),
+ row(1, 4, 2, 6, 12));
- // Single partition queries with DISTINCT
- assertRows(execute("SELECT DISTINCT a, count(a)FROM %s WHERE a = 1 GROUP BY a"),
- row(1, 1L));
+ // Single partition queries with DISTINCT
+ assertRows(execute("SELECT DISTINCT a, count(a)FROM %s WHERE a = 1 GROUP BY a"),
+ row(1, 1L));
- assertInvalidMessage("Grouping on clustering columns is not allowed for SELECT DISTINCT queries",
- "SELECT DISTINCT a, count(a)FROM %s WHERE a = 1 GROUP BY a, b");
+ assertInvalidMessage("Grouping on clustering columns is not allowed for SELECT DISTINCT queries",
+ "SELECT DISTINCT a, count(a)FROM %s WHERE a = 1 GROUP BY a, b");
- // Single partition queries with LIMIT
- assertRows(execute("SELECT a, b, e, count(b), max(e) FROM %s WHERE a = 1 GROUP BY a, b, c LIMIT 10"),
- row(1, 2, 6, 1L, 6),
- row(1, 2, 12, 1L, 12),
- row(1, 4, 12, 2L, 24));
+ // Single partition queries with LIMIT
+ assertRows(execute("SELECT a, b, e, count(b), max(e) FROM %s WHERE a = 1 GROUP BY a, b, c LIMIT 10"),
+ row(1, 2, 6, 1L, 6),
+ row(1, 2, 12, 1L, 12),
+ row(1, 4, 12, 2L, 24));
- assertRows(execute("SELECT a, b, e, count(b), max(e) FROM %s WHERE a = 1 GROUP BY a, b, c LIMIT 2"),
- row(1, 2, 6, 1L, 6),
- row(1, 2, 12, 1L, 12));
+ assertRows(execute("SELECT a, b, e, count(b), max(e) FROM %s WHERE a = 1 GROUP BY a, b, c LIMIT 2"),
+ row(1, 2, 6, 1L, 6),
+ row(1, 2, 12, 1L, 12));
- assertRows(execute("SELECT count(b), max(e) FROM %s WHERE a = 1 GROUP BY a, b, c LIMIT 1"),
- row(1L, 6));
+ assertRows(execute("SELECT count(b), max(e) FROM %s WHERE a = 1 GROUP BY a, b, c LIMIT 1"),
+ row(1L, 6));
- // Single partition queries with PER PARTITION LIMIT
- assertRows(execute("SELECT a, b, e, count(b), max(e) FROM %s WHERE a = 1 GROUP BY a, b, c PER PARTITION LIMIT 10"),
- row(1, 2, 6, 1L, 6),
- row(1, 2, 12, 1L, 12),
- row(1, 4, 12, 2L, 24));
+ // Single partition queries with PER PARTITION LIMIT
+ assertRows(execute("SELECT a, b, e, count(b), max(e) FROM %s WHERE a = 1 GROUP BY a, b, c PER PARTITION LIMIT 10"),
+ row(1, 2, 6, 1L, 6),
+ row(1, 2, 12, 1L, 12),
+ row(1, 4, 12, 2L, 24));
- assertRows(execute("SELECT a, b, e, count(b), max(e) FROM %s WHERE a = 1 GROUP BY a, b, c PER PARTITION LIMIT 2"),
- row(1, 2, 6, 1L, 6),
- row(1, 2, 12, 1L, 12));
+ assertRows(execute("SELECT a, b, e, count(b), max(e) FROM %s WHERE a = 1 GROUP BY a, b, c PER PARTITION LIMIT 2"),
+ row(1, 2, 6, 1L, 6),
+ row(1, 2, 12, 1L, 12));
- assertRows(execute("SELECT count(b), max(e) FROM %s WHERE a = 1 GROUP BY a, b, c PER PARTITION LIMIT 1"),
- row(1L, 6));
+ assertRows(execute("SELECT count(b), max(e) FROM %s WHERE a = 1 GROUP BY a, b, c PER PARTITION LIMIT 1"),
+ row(1L, 6));
- // Single partition queries without aggregates and with LIMIT
- assertRows(execute("SELECT a, b, c, d FROM %s WHERE a = 1 GROUP BY a, b LIMIT 2"),
- row(1, 2, 1, 3),
- row(1, 4, 2, 6));
+ // Single partition queries without aggregates and with LIMIT
+ assertRows(execute("SELECT a, b, c, d FROM %s WHERE a = 1 GROUP BY a, b LIMIT 2"),
+ row(1, 2, 1, 3),
+ row(1, 4, 2, 6));
- assertRows(execute("SELECT a, b, c, d FROM %s WHERE a = 1 GROUP BY a, b LIMIT 1"),
- row(1, 2, 1, 3));
+ assertRows(execute("SELECT a, b, c, d FROM %s WHERE a = 1 GROUP BY a, b LIMIT 1"),
+ row(1, 2, 1, 3));
- assertRows(execute("SELECT a, b, c, d FROM %s WHERE a = 1 GROUP BY a, b, c LIMIT 2"),
- row(1, 2, 1, 3),
- row(1, 2, 2, 6));
+ assertRows(execute("SELECT a, b, c, d FROM %s WHERE a = 1 GROUP BY a, b, c LIMIT 2"),
+ row(1, 2, 1, 3),
+ row(1, 2, 2, 6));
- // Single partition queries with wildcard and with LIMIT
- assertRows(execute("SELECT * FROM %s WHERE a = 1 GROUP BY a, b, c LIMIT 2"),
- row(1, 2, 1, 3, 6),
- row(1, 2, 2, 6, 12));
+ // Single partition queries with wildcard and with LIMIT
+ assertRows(execute("SELECT * FROM %s WHERE a = 1 GROUP BY a, b, c LIMIT 2"),
+ row(1, 2, 1, 3, 6),
+ row(1, 2, 2, 6, 12));
- assertRows(execute("SELECT * FROM %s WHERE a = 1 GROUP BY a, b LIMIT 1"),
- row(1, 2, 1, 3, 6));
+ assertRows(execute("SELECT * FROM %s WHERE a = 1 GROUP BY a, b LIMIT 1"),
+ row(1, 2, 1, 3, 6));
- // Single partition queries without aggregates and with PER PARTITION LIMIT
- assertRows(execute("SELECT a, b, c, d FROM %s WHERE a = 1 GROUP BY a, b PER PARTITION LIMIT 2"),
- row(1, 2, 1, 3),
- row(1, 4, 2, 6));
+ // Single partition queries without aggregates and with PER PARTITION LIMIT
+ assertRows(execute("SELECT a, b, c, d FROM %s WHERE a = 1 GROUP BY a, b PER PARTITION LIMIT 2"),
+ row(1, 2, 1, 3),
+ row(1, 4, 2, 6));
- assertRows(execute("SELECT a, b, c, d FROM %s WHERE a = 1 GROUP BY a, b PER PARTITION LIMIT 1"),
- row(1, 2, 1, 3));
+ assertRows(execute("SELECT a, b, c, d FROM %s WHERE a = 1 GROUP BY a, b PER PARTITION LIMIT 1"),
+ row(1, 2, 1, 3));
- assertRows(execute("SELECT a, b, c, d FROM %s WHERE a = 1 GROUP BY a, b, c PER PARTITION LIMIT 2"),
- row(1, 2, 1, 3),
- row(1, 2, 2, 6));
+ assertRows(execute("SELECT a, b, c, d FROM %s WHERE a = 1 GROUP BY a, b, c PER PARTITION LIMIT 2"),
+ row(1, 2, 1, 3),
+ row(1, 2, 2, 6));
- // Single partition queries with wildcard and with PER PARTITION LIMIT
- assertRows(execute("SELECT * FROM %s WHERE a = 1 GROUP BY a, b, c PER PARTITION LIMIT 2"),
- row(1, 2, 1, 3, 6),
- row(1, 2, 2, 6, 12));
+ // Single partition queries with wildcard and with PER PARTITION LIMIT
+ assertRows(execute("SELECT * FROM %s WHERE a = 1 GROUP BY a, b, c PER PARTITION LIMIT 2"),
+ row(1, 2, 1, 3, 6),
+ row(1, 2, 2, 6, 12));
- assertRows(execute("SELECT * FROM %s WHERE a = 1 GROUP BY a, b PER PARTITION LIMIT 1"),
- row(1, 2, 1, 3, 6));
+ assertRows(execute("SELECT * FROM %s WHERE a = 1 GROUP BY a, b PER PARTITION LIMIT 1"),
+ row(1, 2, 1, 3, 6));
- // Single partition queries with ORDER BY
- assertRows(execute("SELECT a, b, e, count(b), max(e) FROM %s WHERE a = 1 GROUP BY a, b, c ORDER BY b DESC, c DESC"),
- row(1, 4, 24, 2L, 24),
- row(1, 2, 12, 1L, 12),
- row(1, 2, 6, 1L, 6));
+ // Single partition queries with ORDER BY
+ assertRows(execute("SELECT a, b, e, count(b), max(e) FROM %s WHERE a = 1 GROUP BY a, b, c ORDER BY b DESC, c DESC"),
+ row(1, 4, 24, 2L, 24),
+ row(1, 2, 12, 1L, 12),
+ row(1, 2, 6, 1L, 6));
- // Single partition queries with ORDER BY and PER PARTITION LIMIT
- assertRows(execute("SELECT a, b, e, count(b), max(e) FROM %s WHERE a = 1 GROUP BY a, b, c ORDER BY b DESC, c DESC PER PARTITION LIMIT 1"),
- row(1, 4, 24, 2L, 24));
+ // Single partition queries with ORDER BY and PER PARTITION LIMIT
+ assertRows(execute("SELECT a, b, e, count(b), max(e) FROM %s WHERE a = 1 GROUP BY a, b, c ORDER BY b DESC, c DESC PER PARTITION LIMIT 1"),
+ row(1, 4, 24, 2L, 24));
- // Single partition queries with ORDER BY and LIMIT
- assertRows(execute("SELECT a, b, e, count(b), max(e) FROM %s WHERE a = 1 GROUP BY a, b, c ORDER BY b DESC, c DESC LIMIT 2"),
- row(1, 4, 24, 2L, 24),
- row(1, 2, 12, 1L, 12));
+ // Single partition queries with ORDER BY and LIMIT
+ assertRows(execute("SELECT a, b, e, count(b), max(e) FROM %s WHERE a = 1 GROUP BY a, b, c ORDER BY b DESC, c DESC LIMIT 2"),
+ row(1, 4, 24, 2L, 24),
+ row(1, 2, 12, 1L, 12));
- // Multi-partitions queries
- assertRows(execute("SELECT a, b, e, count(b), max(e) FROM %s WHERE a IN (1, 2, 4) GROUP BY a, b, c"),
- row(1, 2, 6, 1L, 6),
- row(1, 2, 12, 1L, 12),
- row(1, 4, 12, 2L, 24),
- row(2, 2, 6, 1L, 6),
- row(2, 4, 12, 1L, 12),
- row(4, 8, 24, 1L, 24));
+ // Multi-partitions queries
+ assertRows(execute("SELECT a, b, e, count(b), max(e) FROM %s WHERE a IN (1, 2, 4) GROUP BY a, b, c"),
+ row(1, 2, 6, 1L, 6),
+ row(1, 2, 12, 1L, 12),
+ row(1, 4, 12, 2L, 24),
+ row(2, 2, 6, 1L, 6),
+ row(2, 4, 12, 1L, 12),
+ row(4, 8, 24, 1L, 24));
- assertRows(execute("SELECT a, b, e, count(b), max(e) FROM %s WHERE a IN (1, 2, 4) AND b = 2 GROUP BY a, b, c"),
- row(1, 2, 6, 1L, 6),
- row(1, 2, 12, 1L, 12),
- row(2, 2, 6, 1L, 6));
+ assertRows(execute("SELECT a, b, e, count(b), max(e) FROM %s WHERE a IN (1, 2, 4) AND b = 2 GROUP BY a, b, c"),
+ row(1, 2, 6, 1L, 6),
+ row(1, 2, 12, 1L, 12),
+ row(2, 2, 6, 1L, 6));
- // Multi-partitions queries without aggregates
- assertRows(execute("SELECT a, b, c, d FROM %s WHERE a IN (1, 2, 4) GROUP BY a, b"),
- row(1, 2, 1, 3),
- row(1, 4, 2, 6),
- row(2, 2, 3, 3),
- row(2, 4, 3, 6),
- row(4, 8, 2, 12));
+ // Multi-partitions queries without aggregates
+ assertRows(execute("SELECT a, b, c, d FROM %s WHERE a IN (1, 2, 4) GROUP BY a, b"),
+ row(1, 2, 1, 3),
+ row(1, 4, 2, 6),
+ row(2, 2, 3, 3),
+ row(2, 4, 3, 6),
+ row(4, 8, 2, 12));
- assertRows(execute("SELECT a, b, c, d FROM %s WHERE a IN (1, 2, 4) GROUP BY a, b, c"),
- row(1, 2, 1, 3),
- row(1, 2, 2, 6),
- row(1, 4, 2, 6),
- row(2, 2, 3, 3),
- row(2, 4, 3, 6),
- row(4, 8, 2, 12));
+ assertRows(execute("SELECT a, b, c, d FROM %s WHERE a IN (1, 2, 4) GROUP BY a, b, c"),
+ row(1, 2, 1, 3),
+ row(1, 2, 2, 6),
+ row(1, 4, 2, 6),
+ row(2, 2, 3, 3),
+ row(2, 4, 3, 6),
+ row(4, 8, 2, 12));
- // Multi-partitions with wildcard
- assertRows(execute("SELECT * FROM %s WHERE a IN (1, 2, 4) GROUP BY a, b, c"),
- row(1, 2, 1, 3, 6),
- row(1, 2, 2, 6, 12),
- row(1, 4, 2, 6, 12),
- row(2, 2, 3, 3, 6),
- row(2, 4, 3, 6, 12),
- row(4, 8, 2, 12, 24));
+ // Multi-partitions with wildcard
+ assertRows(execute("SELECT * FROM %s WHERE a IN (1, 2, 4) GROUP BY a, b, c"),
+ row(1, 2, 1, 3, 6),
+ row(1, 2, 2, 6, 12),
+ row(1, 4, 2, 6, 12),
+ row(2, 2, 3, 3, 6),
+ row(2, 4, 3, 6, 12),
+ row(4, 8, 2, 12, 24));
- assertRows(execute("SELECT * FROM %s WHERE a IN (1, 2, 4) GROUP BY a, b"),
- row(1, 2, 1, 3, 6),
- row(1, 4, 2, 6, 12),
- row(2, 2, 3, 3, 6),
- row(2, 4, 3, 6, 12),
- row(4, 8, 2, 12, 24));
+ assertRows(execute("SELECT * FROM %s WHERE a IN (1, 2, 4) GROUP BY a, b"),
+ row(1, 2, 1, 3, 6),
+ row(1, 4, 2, 6, 12),
+ row(2, 2, 3, 3, 6),
+ row(2, 4, 3, 6, 12),
+ row(4, 8, 2, 12, 24));
- // Multi-partitions query with DISTINCT
- assertRows(execute("SELECT DISTINCT a, count(a)FROM %s WHERE a IN (1, 2, 4) GROUP BY a"),
- row(1, 1L),
- row(2, 1L),
- row(4, 1L));
+ // Multi-partitions query with DISTINCT
+ assertRows(execute("SELECT DISTINCT a, count(a)FROM %s WHERE a IN (1, 2, 4) GROUP BY a"),
+ row(1, 1L),
+ row(2, 1L),
+ row(4, 1L));
- assertInvalidMessage("Grouping on clustering columns is not allowed for SELECT DISTINCT queries",
- "SELECT DISTINCT a, count(a)FROM %s WHERE a IN (1, 2, 4) GROUP BY a, b");
+ assertInvalidMessage("Grouping on clustering columns is not allowed for SELECT DISTINCT queries",
+ "SELECT DISTINCT a, count(a)FROM %s WHERE a IN (1, 2, 4) GROUP BY a, b");
- // Multi-partitions query with DISTINCT and LIMIT
- assertRows(execute("SELECT DISTINCT a, count(a)FROM %s WHERE a IN (1, 2, 4) GROUP BY a LIMIT 2"),
- row(1, 1L),
- row(2, 1L));
+ // Multi-partitions query with DISTINCT and LIMIT
+ assertRows(execute("SELECT DISTINCT a, count(a)FROM %s WHERE a IN (1, 2, 4) GROUP BY a LIMIT 2"),
+ row(1, 1L),
+ row(2, 1L));
- // Multi-partitions queries with PER PARTITION LIMIT
- assertRows(execute("SELECT a, b, e, count(b), max(e) FROM %s WHERE a IN (1, 2, 4) GROUP BY a, b, c PER PARTITION LIMIT 1"),
- row(1, 2, 6, 1L, 6),
- row(2, 2, 6, 1L, 6),
- row(4, 8, 24, 1L, 24));
+ // Multi-partitions queries with PER PARTITION LIMIT
+ assertRows(execute("SELECT a, b, e, count(b), max(e) FROM %s WHERE a IN (1, 2, 4) GROUP BY a, b, c PER PARTITION LIMIT 1"),
+ row(1, 2, 6, 1L, 6),
+ row(2, 2, 6, 1L, 6),
+ row(4, 8, 24, 1L, 24));
- assertRows(execute("SELECT a, b, e, count(b), max(e) FROM %s WHERE a IN (1, 2, 4) GROUP BY a, b, c PER PARTITION LIMIT 2"),
- row(1, 2, 6, 1L, 6),
- row(1, 2, 12, 1L, 12),
- row(2, 2, 6, 1L, 6),
- row(2, 4, 12, 1L, 12),
- row(4, 8, 24, 1L, 24));
+ assertRows(execute("SELECT a, b, e, count(b), max(e) FROM %s WHERE a IN (1, 2, 4) GROUP BY a, b, c PER PARTITION LIMIT 2"),
+ row(1, 2, 6, 1L, 6),
+ row(1, 2, 12, 1L, 12),
+ row(2, 2, 6, 1L, 6),
+ row(2, 4, 12, 1L, 12),
+ row(4, 8, 24, 1L, 24));
- // Multi-partitions with wildcard and PER PARTITION LIMIT
- assertRows(execute("SELECT * FROM %s WHERE a IN (1, 2, 4) GROUP BY a, b, c PER PARTITION LIMIT 2"),
- row(1, 2, 1, 3, 6),
- row(1, 2, 2, 6, 12),
- row(2, 2, 3, 3, 6),
- row(2, 4, 3, 6, 12),
- row(4, 8, 2, 12, 24));
+ // Multi-partitions with wildcard and PER PARTITION LIMIT
+ assertRows(execute("SELECT * FROM %s WHERE a IN (1, 2, 4) GROUP BY a, b, c PER PARTITION LIMIT 2"),
+ row(1, 2, 1, 3, 6),
+ row(1, 2, 2, 6, 12),
+ row(2, 2, 3, 3, 6),
+ row(2, 4, 3, 6, 12),
+ row(4, 8, 2, 12, 24));
- assertRows(execute("SELECT * FROM %s WHERE a IN (1, 2, 4) GROUP BY a, b PER PARTITION LIMIT 1"),
- row(1, 2, 1, 3, 6),
- row(2, 2, 3, 3, 6),
- row(4, 8, 2, 12, 24));
+ assertRows(execute("SELECT * FROM %s WHERE a IN (1, 2, 4) GROUP BY a, b PER PARTITION LIMIT 1"),
+ row(1, 2, 1, 3, 6),
+ row(2, 2, 3, 3, 6),
+ row(4, 8, 2, 12, 24));
- // Multi-partitions queries with ORDER BY
- assertRows(execute("SELECT a, b, c, count(b), max(e) FROM %s WHERE a IN (1, 2, 4) GROUP BY a, b ORDER BY b DESC, c DESC"),
- row(4, 8, 2, 1L, 24),
- row(2, 4, 3, 1L, 12),
- row(1, 4, 2, 2L, 24),
- row(2, 2, 3, 1L, 6),
- row(1, 2, 2, 2L, 12));
+ // Multi-partitions queries with ORDER BY
+ assertRows(execute("SELECT a, b, c, count(b), max(e) FROM %s WHERE a IN (1, 2, 4) GROUP BY a, b ORDER BY b DESC, c DESC"),
+ row(4, 8, 2, 1L, 24),
+ row(2, 4, 3, 1L, 12),
+ row(1, 4, 2, 2L, 24),
+ row(2, 2, 3, 1L, 6),
+ row(1, 2, 2, 2L, 12));
- assertRows(execute("SELECT a, b, c, d FROM %s WHERE a IN (1, 2, 4) GROUP BY a, b, c ORDER BY b DESC, c DESC"),
- row(4, 8, 2, 12),
- row(2, 4, 3, 6),
- row(1, 4, 2, 12),
- row(2, 2, 3, 3),
- row(1, 2, 2, 6),
- row(1, 2, 1, 3));
+ assertRows(execute("SELECT a, b, c, d FROM %s WHERE a IN (1, 2, 4) GROUP BY a, b, c ORDER BY b DESC, c DESC"),
+ row(4, 8, 2, 12),
+ row(2, 4, 3, 6),
+ row(1, 4, 2, 12),
+ row(2, 2, 3, 3),
+ row(1, 2, 2, 6),
+ row(1, 2, 1, 3));
- // Multi-partitions queries with ORDER BY and LIMIT
- assertRows(execute("SELECT a, b, c, d FROM %s WHERE a IN (1, 2, 4) GROUP BY a, b ORDER BY b DESC, c DESC LIMIT 3"),
- row(4, 8, 2, 12),
- row(2, 4, 3, 6),
- row(1, 4, 2, 12));
+ // Multi-partitions queries with ORDER BY and LIMIT
+ assertRows(execute("SELECT a, b, c, d FROM %s WHERE a IN (1, 2, 4) GROUP BY a, b ORDER BY b DESC, c DESC LIMIT 3"),
+ row(4, 8, 2, 12),
+ row(2, 4, 3, 6),
+ row(1, 4, 2, 12));
- // Multi-partitions with wildcard, ORDER BY and LIMIT
- assertRows(execute("SELECT * FROM %s WHERE a IN (1, 2, 4) GROUP BY a, b, c ORDER BY b DESC, c DESC LIMIT 3"),
- row(4, 8, 2, 12, 24),
- row(2, 4, 3, 6, 12),
- row(1, 4, 2, 12, 24));
+ // Multi-partitions with wildcard, ORDER BY and LIMIT
+ assertRows(execute("SELECT * FROM %s WHERE a IN (1, 2, 4) GROUP BY a, b, c ORDER BY b DESC, c DESC LIMIT 3"),
+ row(4, 8, 2, 12, 24),
+ row(2, 4, 3, 6, 12),
+ row(1, 4, 2, 12, 24));
- // Invalid queries
- assertInvalidMessage("Group by is currently only supported on the columns of the PRIMARY KEY, got e",
- "SELECT a, b, d, count(b), max(c) FROM %s WHERE a = 1 GROUP BY a, e");
+ // Invalid queries
+ assertInvalidMessage("Group by is currently only supported on the columns of the PRIMARY KEY, got e",
+ "SELECT a, b, d, count(b), max(c) FROM %s WHERE a = 1 GROUP BY a, e");
- assertInvalidMessage("Group by currently only support groups of columns following their declared order in the PRIMARY KEY",
- "SELECT a, b, d, count(b), max(c) FROM %s WHERE a = 1 GROUP BY c");
+ assertInvalidMessage("Group by currently only support groups of columns following their declared order in the PRIMARY KEY",
+ "SELECT a, b, d, count(b), max(c) FROM %s WHERE a = 1 GROUP BY c");
- assertInvalidMessage("Group by currently only support groups of columns following their declared order in the PRIMARY KEY",
- "SELECT a, b, d, count(b), max(c) FROM %s WHERE a = 1 GROUP BY a, c, b");
+ assertInvalidMessage("Group by currently only support groups of columns following their declared order in the PRIMARY KEY",
+ "SELECT a, b, d, count(b), max(c) FROM %s WHERE a = 1 GROUP BY a, c, b");
- assertInvalidMessage("Group by currently only support groups of columns following their declared order in the PRIMARY KEY",
- "SELECT a, b, d, count(b), max(c) FROM %s WHERE a = 1 GROUP BY a, a");
+ assertInvalidMessage("Group by currently only support groups of columns following their declared order in the PRIMARY KEY",
+ "SELECT a, b, d, count(b), max(c) FROM %s WHERE a = 1 GROUP BY a, a");
- assertInvalidMessage("Group by currently only support groups of columns following their declared order in the PRIMARY KEY",
- "SELECT a, b, c, d FROM %s WHERE token(a) = token(1) GROUP BY b, c");
+ assertInvalidMessage("Group by currently only support groups of columns following their declared order in the PRIMARY KEY",
+ "SELECT a, b, c, d FROM %s WHERE token(a) = token(1) GROUP BY b, c");
- assertInvalidMessage("Undefined column name clustering1",
- "SELECT a, b as clustering1, max(c) FROM %s WHERE a = 1 GROUP BY a, clustering1");
+ assertInvalidMessage("Undefined column name clustering1",
+ "SELECT a, b as clustering1, max(c) FROM %s WHERE a = 1 GROUP BY a, clustering1");
- assertInvalidMessage("Undefined column name z",
- "SELECT a, b, max(c) FROM %s WHERE a = 1 GROUP BY a, b, z");
+ assertInvalidMessage("Undefined column name z",
+ "SELECT a, b, max(c) FROM %s WHERE a = 1 GROUP BY a, b, z");
- // Test with composite partition key
- createTable("CREATE TABLE %s (a int, b int, c int, d int, e int, primary key ((a, b), c, d))" + compactOption);
+ // Test with composite partition key
+ createTable("CREATE TABLE %s (a int, b int, c int, d int, e int, primary key ((a, b), c, d))");
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (1, 1, 1, 3, 6)");
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (1, 1, 2, 6, 12)");
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (1, 1, 3, 12, 24)");
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (1, 2, 1, 12, 24)");
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (1, 2, 2, 6, 12)");
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (1, 1, 1, 3, 6)");
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (1, 1, 2, 6, 12)");
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (1, 1, 3, 12, 24)");
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (1, 2, 1, 12, 24)");
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (1, 2, 2, 6, 12)");
- assertInvalidMessage("Group by is not supported on only a part of the partition key",
- "SELECT a, b, max(d) FROM %s GROUP BY a");
+ assertInvalidMessage("Group by is not supported on only a part of the partition key",
+ "SELECT a, b, max(d) FROM %s GROUP BY a");
- assertRows(execute("SELECT a, b, max(d) FROM %s GROUP BY a, b"),
- row(1, 2, 12),
- row(1, 1, 12));
+ assertRows(execute("SELECT a, b, max(d) FROM %s GROUP BY a, b"),
+ row(1, 2, 12),
+ row(1, 1, 12));
- assertRows(execute("SELECT a, b, max(d) FROM %s WHERE a = 1 AND b = 1 GROUP BY b"),
- row(1, 1, 12));
+ assertRows(execute("SELECT a, b, max(d) FROM %s WHERE a = 1 AND b = 1 GROUP BY b"),
+ row(1, 1, 12));
- // Test with table without clustering key
- createTable("CREATE TABLE %s (a int primary key, b int, c int)" + compactOption);
+ // Test with table without clustering key
+ createTable("CREATE TABLE %s (a int primary key, b int, c int)");
- execute("INSERT INTO %s (a, b, c) VALUES (1, 3, 6)");
- execute("INSERT INTO %s (a, b, c) VALUES (2, 6, 12)");
- execute("INSERT INTO %s (a, b, c) VALUES (3, 12, 24)");
+ execute("INSERT INTO %s (a, b, c) VALUES (1, 3, 6)");
+ execute("INSERT INTO %s (a, b, c) VALUES (2, 6, 12)");
+ execute("INSERT INTO %s (a, b, c) VALUES (3, 12, 24)");
- assertInvalidMessage("Group by currently only support groups of columns following their declared order in the PRIMARY KEY",
- "SELECT a, max(c) FROM %s WHERE a = 1 GROUP BY a, a");
- }
+ assertInvalidMessage("Group by currently only support groups of columns following their declared order in the PRIMARY KEY",
+ "SELECT a, max(c) FROM %s WHERE a = 1 GROUP BY a, a");
}
@Test
public void testGroupByWithoutPagingWithDeletions() throws Throwable
{
- for (String compactOption : new String[] { "", " WITH COMPACT STORAGE" })
- {
- createTable("CREATE TABLE %s (a int, b int, c int, d int, e int, primary key (a, b, c, d))"
- + compactOption);
+ createTable("CREATE TABLE %s (a int, b int, c int, d int, e int, primary key (a, b, c, d))");
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (1, 2, 1, 3, 6)");
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (1, 2, 1, 6, 12)");
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (1, 2, 1, 9, 18)");
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (1, 2, 1, 12, 24)");
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (1, 2, 2, 3, 6)");
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (1, 2, 2, 6, 12)");
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (1, 2, 2, 9, 18)");
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (1, 2, 2, 12, 24)");
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (1, 2, 3, 3, 6)");
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (1, 2, 3, 6, 12)");
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (1, 2, 3, 9, 18)");
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (1, 2, 3, 12, 24)");
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (1, 2, 1, 3, 6)");
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (1, 2, 1, 6, 12)");
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (1, 2, 1, 9, 18)");
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (1, 2, 1, 12, 24)");
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (1, 2, 2, 3, 6)");
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (1, 2, 2, 6, 12)");
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (1, 2, 2, 9, 18)");
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (1, 2, 2, 12, 24)");
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (1, 2, 3, 3, 6)");
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (1, 2, 3, 6, 12)");
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (1, 2, 3, 9, 18)");
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (1, 2, 3, 12, 24)");
- execute("DELETE FROM %s WHERE a = 1 AND b = 2 AND c = 1 AND d = 12");
- execute("DELETE FROM %s WHERE a = 1 AND b = 2 AND c = 2 AND d = 9");
+ execute("DELETE FROM %s WHERE a = 1 AND b = 2 AND c = 1 AND d = 12");
+ execute("DELETE FROM %s WHERE a = 1 AND b = 2 AND c = 2 AND d = 9");
- assertRows(execute("SELECT a, b, c, count(b), max(d) FROM %s GROUP BY a, b, c"),
- row(1, 2, 1, 3L, 9),
- row(1, 2, 2, 3L, 12),
- row(1, 2, 3, 4L, 12));
- }
+ assertRows(execute("SELECT a, b, c, count(b), max(d) FROM %s GROUP BY a, b, c"),
+ row(1, 2, 1, 3L, 9),
+ row(1, 2, 2, 3L, 12),
+ row(1, 2, 3, 4L, 12));
}
@Test
public void testGroupByWithRangeNamesQueryWithoutPaging() throws Throwable
{
- for (String compactOption : new String[] { "", " WITH COMPACT STORAGE" })
- {
- createTable("CREATE TABLE %s (a int, b int, c int, d int, primary key (a, b, c))"
- + compactOption);
+ createTable("CREATE TABLE %s (a int, b int, c int, d int, primary key (a, b, c))");
- for (int i = 1; i < 5; i++)
- for (int j = 1; j < 5; j++)
- for (int k = 1; k < 5; k++)
- execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", i, j, k, i + j);
+ for (int i = 1; i < 5; i++)
+ for (int j = 1; j < 5; j++)
+ for (int k = 1; k < 5; k++)
+ execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", i, j, k, i + j);
- // Makes sure that we have some tombstones
- execute("DELETE FROM %s WHERE a = 3");
+ // Makes sure that we have some tombstones
+ execute("DELETE FROM %s WHERE a = 3");
- // Range queries
- assertRows(execute("SELECT a, b, d, count(b), max(d) FROM %s WHERE b = 1 and c IN (1, 2) GROUP BY a ALLOW FILTERING"),
- row(1, 1, 2, 2L, 2),
- row(2, 1, 3, 2L, 3),
- row(4, 1, 5, 2L, 5));
+ // Range queries
+ assertRows(execute("SELECT a, b, d, count(b), max(d) FROM %s WHERE b = 1 and c IN (1, 2) GROUP BY a ALLOW FILTERING"),
+ row(1, 1, 2, 2L, 2),
+ row(2, 1, 3, 2L, 3),
+ row(4, 1, 5, 2L, 5));
- assertRows(execute("SELECT a, b, d, count(b), max(d) FROM %s WHERE b = 1 and c IN (1, 2) GROUP BY a, b ALLOW FILTERING"),
- row(1, 1, 2, 2L, 2),
- row(2, 1, 3, 2L, 3),
- row(4, 1, 5, 2L, 5));
+ assertRows(execute("SELECT a, b, d, count(b), max(d) FROM %s WHERE b = 1 and c IN (1, 2) GROUP BY a, b ALLOW FILTERING"),
+ row(1, 1, 2, 2L, 2),
+ row(2, 1, 3, 2L, 3),
+ row(4, 1, 5, 2L, 5));
- assertRows(execute("SELECT a, b, d, count(b), max(d) FROM %s WHERE b IN (1, 2) and c IN (1, 2) GROUP BY a, b ALLOW FILTERING"),
- row(1, 1, 2, 2L, 2),
- row(1, 2, 3, 2L, 3),
- row(2, 1, 3, 2L, 3),
- row(2, 2, 4, 2L, 4),
- row(4, 1, 5, 2L, 5),
- row(4, 2, 6, 2L, 6));
+ assertRows(execute("SELECT a, b, d, count(b), max(d) FROM %s WHERE b IN (1, 2) and c IN (1, 2) GROUP BY a, b ALLOW FILTERING"),
+ row(1, 1, 2, 2L, 2),
+ row(1, 2, 3, 2L, 3),
+ row(2, 1, 3, 2L, 3),
+ row(2, 2, 4, 2L, 4),
+ row(4, 1, 5, 2L, 5),
+ row(4, 2, 6, 2L, 6));
- // Range queries with LIMIT
- assertRows(execute("SELECT a, b, d, count(b), max(d) FROM %s WHERE b = 1 and c IN (1, 2) GROUP BY a LIMIT 5 ALLOW FILTERING"),
- row(1, 1, 2, 2L, 2),
- row(2, 1, 3, 2L, 3),
- row(4, 1, 5, 2L, 5));
+ // Range queries with LIMIT
+ assertRows(execute("SELECT a, b, d, count(b), max(d) FROM %s WHERE b = 1 and c IN (1, 2) GROUP BY a LIMIT 5 ALLOW FILTERING"),
+ row(1, 1, 2, 2L, 2),
+ row(2, 1, 3, 2L, 3),
+ row(4, 1, 5, 2L, 5));
- assertRows(execute("SELECT a, b, d, count(b), max(d) FROM %s WHERE b = 1 and c IN (1, 2) GROUP BY a, b LIMIT 3 ALLOW FILTERING"),
- row(1, 1, 2, 2L, 2),
- row(2, 1, 3, 2L, 3),
- row(4, 1, 5, 2L, 5));
+ assertRows(execute("SELECT a, b, d, count(b), max(d) FROM %s WHERE b = 1 and c IN (1, 2) GROUP BY a, b LIMIT 3 ALLOW FILTERING"),
+ row(1, 1, 2, 2L, 2),
+ row(2, 1, 3, 2L, 3),
+ row(4, 1, 5, 2L, 5));
- assertRows(execute("SELECT a, b, d, count(b), max(d) FROM %s WHERE b IN (1, 2) and c IN (1, 2) GROUP BY a, b LIMIT 3 ALLOW FILTERING"),
- row(1, 1, 2, 2L, 2),
- row(1, 2, 3, 2L, 3),
- row(2, 1, 3, 2L, 3));
+ assertRows(execute("SELECT a, b, d, count(b), max(d) FROM %s WHERE b IN (1, 2) and c IN (1, 2) GROUP BY a, b LIMIT 3 ALLOW FILTERING"),
+ row(1, 1, 2, 2L, 2),
+ row(1, 2, 3, 2L, 3),
+ row(2, 1, 3, 2L, 3));
- // Range queries with PER PARTITION LIMIT
- assertRows(execute("SELECT a, b, d, count(b), max(d) FROM %s WHERE b = 1 and c IN (1, 2) GROUP BY a, b PER PARTITION LIMIT 2 ALLOW FILTERING"),
- row(1, 1, 2, 2L, 2),
- row(2, 1, 3, 2L, 3),
- row(4, 1, 5, 2L, 5));
+ // Range queries with PER PARTITION LIMIT
+ assertRows(execute("SELECT a, b, d, count(b), max(d) FROM %s WHERE b = 1 and c IN (1, 2) GROUP BY a, b PER PARTITION LIMIT 2 ALLOW FILTERING"),
+ row(1, 1, 2, 2L, 2),
+ row(2, 1, 3, 2L, 3),
+ row(4, 1, 5, 2L, 5));
- assertRows(execute("SELECT a, b, d, count(b), max(d) FROM %s WHERE b IN (1, 2) and c IN (1, 2) GROUP BY a, b PER PARTITION LIMIT 1 ALLOW FILTERING"),
- row(1, 1, 2, 2L, 2),
- row(2, 1, 3, 2L, 3),
- row(4, 1, 5, 2L, 5));
+ assertRows(execute("SELECT a, b, d, count(b), max(d) FROM %s WHERE b IN (1, 2) and c IN (1, 2) GROUP BY a, b PER PARTITION LIMIT 1 ALLOW FILTERING"),
+ row(1, 1, 2, 2L, 2),
+ row(2, 1, 3, 2L, 3),
+ row(4, 1, 5, 2L, 5));
- // Range queries with PER PARTITION LIMIT and LIMIT
- assertRows(execute("SELECT a, b, d, count(b), max(d) FROM %s WHERE b = 1 and c IN (1, 2) GROUP BY a, b PER PARTITION LIMIT 2 LIMIT 5 ALLOW FILTERING"),
- row(1, 1, 2, 2L, 2),
- row(2, 1, 3, 2L, 3),
- row(4, 1, 5, 2L, 5));
+ // Range queries with PER PARTITION LIMIT and LIMIT
+ assertRows(execute("SELECT a, b, d, count(b), max(d) FROM %s WHERE b = 1 and c IN (1, 2) GROUP BY a, b PER PARTITION LIMIT 2 LIMIT 5 ALLOW FILTERING"),
+ row(1, 1, 2, 2L, 2),
+ row(2, 1, 3, 2L, 3),
+ row(4, 1, 5, 2L, 5));
- assertRows(execute("SELECT a, b, d, count(b), max(d) FROM %s WHERE b IN (1, 2) and c IN (1, 2) GROUP BY a, b PER PARTITION LIMIT 1 LIMIT 2 ALLOW FILTERING"),
- row(1, 1, 2, 2L, 2),
- row(2, 1, 3, 2L, 3));
- }
+ assertRows(execute("SELECT a, b, d, count(b), max(d) FROM %s WHERE b IN (1, 2) and c IN (1, 2) GROUP BY a, b PER PARTITION LIMIT 1 LIMIT 2 ALLOW FILTERING"),
+ row(1, 1, 2, 2L, 2),
+ row(2, 1, 3, 2L, 3));
}
@Test
@@ -1080,491 +1068,483 @@
@Test
public void testGroupByWithPaging() throws Throwable
{
- for (String compactOption : new String[] { "", " WITH COMPACT STORAGE" })
+ createTable("CREATE TABLE %s (a int, b int, c int, d int, e int, primary key (a, b, c, d))");
+
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (1, 2, 1, 3, 6)");
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (1, 2, 2, 6, 12)");
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (1, 3, 2, 12, 24)");
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (1, 4, 2, 12, 24)");
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (1, 4, 2, 6, 12)");
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (2, 2, 3, 3, 6)");
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (2, 4, 3, 6, 12)");
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (3, 3, 2, 12, 24)");
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (4, 8, 2, 12, 24)");
+
+ // Makes sure that we have some tombstones
+ execute("DELETE FROM %s WHERE a = 1 AND b = 3 AND c = 2 AND d = 12");
+ execute("DELETE FROM %s WHERE a = 3");
+
+ for (int pageSize = 1; pageSize < 10; pageSize++)
{
- createTable("CREATE TABLE %s (a int, b int, c int, d int, e int, primary key (a, b, c, d))"
- + compactOption);
+ // Range queries
+ assertRowsNet(executeNetWithPaging("SELECT a, b, e, count(b), max(e) FROM %s GROUP BY a", pageSize),
+ row(1, 2, 6, 4L, 24),
+ row(2, 2, 6, 2L, 12),
+ row(4, 8, 24, 1L, 24));
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (1, 2, 1, 3, 6)");
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (1, 2, 2, 6, 12)");
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (1, 3, 2, 12, 24)");
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (1, 4, 2, 12, 24)");
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (1, 4, 2, 6, 12)");
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (2, 2, 3, 3, 6)");
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (2, 4, 3, 6, 12)");
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (3, 3, 2, 12, 24)");
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (4, 8, 2, 12, 24)");
+ assertRowsNet(executeNetWithPaging("SELECT a, b, e, count(b), max(e) FROM %s GROUP BY a, b", pageSize),
+ row(1, 2, 6, 2L, 12),
+ row(1, 4, 12, 2L, 24),
+ row(2, 2, 6, 1L, 6),
+ row(2, 4, 12, 1L, 12),
+ row(4, 8, 24, 1L, 24));
- // Makes sure that we have some tombstones
- execute("DELETE FROM %s WHERE a = 1 AND b = 3 AND c = 2 AND d = 12");
- execute("DELETE FROM %s WHERE a = 3");
+ assertRowsNet(executeNetWithPaging("SELECT a, b, e, count(b), max(e) FROM %s", pageSize),
+ row(1, 2, 6, 7L, 24));
- for (int pageSize = 1; pageSize < 10; pageSize++)
- {
- // Range queries
- assertRowsNet(executeNetWithPaging("SELECT a, b, e, count(b), max(e) FROM %s GROUP BY a", pageSize),
- row(1, 2, 6, 4L, 24),
- row(2, 2, 6, 2L, 12),
- row(4, 8, 24, 1L, 24));
+ assertRowsNet(executeNetWithPaging("SELECT a, b, e, count(b), max(e) FROM %s WHERE b = 2 GROUP BY a, b ALLOW FILTERING",
+ pageSize),
+ row(1, 2, 6, 2L, 12),
+ row(2, 2, 6, 1L, 6));
- assertRowsNet(executeNetWithPaging("SELECT a, b, e, count(b), max(e) FROM %s GROUP BY a, b", pageSize),
- row(1, 2, 6, 2L, 12),
- row(1, 4, 12, 2L, 24),
- row(2, 2, 6, 1L, 6),
- row(2, 4, 12, 1L, 12),
- row(4, 8, 24, 1L, 24));
+ assertRowsNet(executeNetWithPaging("SELECT a, b, e, count(b), max(e) FROM %s WHERE b = 2 ALLOW FILTERING",
+ pageSize),
+ row(1, 2, 6, 3L, 12));
- assertRowsNet(executeNetWithPaging("SELECT a, b, e, count(b), max(e) FROM %s", pageSize),
- row(1, 2, 6, 7L, 24));
+ // Range queries without aggregates
+ assertRowsNet(executeNetWithPaging("SELECT a, b, c, d FROM %s GROUP BY a, b, c", pageSize),
+ row(1, 2, 1, 3),
+ row(1, 2, 2, 6),
+ row(1, 4, 2, 6),
+ row(2, 2, 3, 3),
+ row(2, 4, 3, 6),
+ row(4, 8, 2, 12));
- assertRowsNet(executeNetWithPaging("SELECT a, b, e, count(b), max(e) FROM %s WHERE b = 2 GROUP BY a, b ALLOW FILTERING",
- pageSize),
- row(1, 2, 6, 2L, 12),
- row(2, 2, 6, 1L, 6));
+ assertRowsNet(executeNetWithPaging("SELECT a, b, c, d FROM %s GROUP BY a, b", pageSize),
+ row(1, 2, 1, 3),
+ row(1, 4, 2, 6),
+ row(2, 2, 3, 3),
+ row(2, 4, 3, 6),
+ row(4, 8, 2, 12));
- assertRowsNet(executeNetWithPaging("SELECT a, b, e, count(b), max(e) FROM %s WHERE b = 2 ALLOW FILTERING",
- pageSize),
- row(1, 2, 6, 3L, 12));
+ // Range queries with wildcard
+ assertRowsNet(executeNetWithPaging("SELECT * FROM %s GROUP BY a, b, c", pageSize),
+ row(1, 2, 1, 3, 6),
+ row(1, 2, 2, 6, 12),
+ row(1, 4, 2, 6, 12),
+ row(2, 2, 3, 3, 6),
+ row(2, 4, 3, 6, 12),
+ row(4, 8, 2, 12, 24));
- // Range queries without aggregates
- assertRowsNet(executeNetWithPaging("SELECT a, b, c, d FROM %s GROUP BY a, b, c", pageSize),
- row(1, 2, 1, 3),
- row(1, 2, 2, 6),
- row(1, 4, 2, 6),
- row(2, 2, 3, 3),
- row(2, 4, 3, 6),
- row(4, 8, 2, 12));
+ assertRowsNet(executeNetWithPaging("SELECT * FROM %s GROUP BY a, b", pageSize),
+ row(1, 2, 1, 3, 6),
+ row(1, 4, 2, 6, 12),
+ row(2, 2, 3, 3, 6),
+ row(2, 4, 3, 6, 12),
+ row(4, 8, 2, 12, 24));
- assertRowsNet(executeNetWithPaging("SELECT a, b, c, d FROM %s GROUP BY a, b", pageSize),
- row(1, 2, 1, 3),
- row(1, 4, 2, 6),
- row(2, 2, 3, 3),
- row(2, 4, 3, 6),
- row(4, 8, 2, 12));
+ // Range query with LIMIT
+ assertRowsNet(executeNetWithPaging("SELECT a, b, e, count(b), max(e) FROM %s GROUP BY a, b LIMIT 2",
+ pageSize),
+ row(1, 2, 6, 2L, 12),
+ row(1, 4, 12, 2L, 24));
- // Range queries with wildcard
- assertRowsNet(executeNetWithPaging("SELECT * FROM %s GROUP BY a, b, c", pageSize),
- row(1, 2, 1, 3, 6),
- row(1, 2, 2, 6, 12),
- row(1, 4, 2, 6, 12),
- row(2, 2, 3, 3, 6),
- row(2, 4, 3, 6, 12),
- row(4, 8, 2, 12, 24));
+ assertRowsNet(executeNetWithPaging("SELECT a, b, e, count(b), max(e) FROM %s LIMIT 2",
+ pageSize),
+ row(1, 2, 6, 7L, 24));
- assertRowsNet(executeNetWithPaging("SELECT * FROM %s GROUP BY a, b", pageSize),
- row(1, 2, 1, 3, 6),
- row(1, 4, 2, 6, 12),
- row(2, 2, 3, 3, 6),
- row(2, 4, 3, 6, 12),
- row(4, 8, 2, 12, 24));
+ // Range queries with PER PARTITION LIMIT
+ assertRowsNet(executeNetWithPaging("SELECT a, b, e, count(b), max(e) FROM %s GROUP BY a, b PER PARTITION LIMIT 3", pageSize),
+ row(1, 2, 6, 2L, 12),
+ row(1, 4, 12, 2L, 24),
+ row(2, 2, 6, 1L, 6),
+ row(2, 4, 12, 1L, 12),
+ row(4, 8, 24, 1L, 24));
- // Range query with LIMIT
- assertRowsNet(executeNetWithPaging("SELECT a, b, e, count(b), max(e) FROM %s GROUP BY a, b LIMIT 2",
- pageSize),
- row(1, 2, 6, 2L, 12),
- row(1, 4, 12, 2L, 24));
+ assertRowsNet(executeNetWithPaging("SELECT a, b, e, count(b), max(e) FROM %s GROUP BY a, b PER PARTITION LIMIT 1", pageSize),
+ row(1, 2, 6, 2L, 12),
+ row(2, 2, 6, 1L, 6),
+ row(4, 8, 24, 1L, 24));
- assertRowsNet(executeNetWithPaging("SELECT a, b, e, count(b), max(e) FROM %s LIMIT 2",
- pageSize),
- row(1, 2, 6, 7L, 24));
+ // Range query with PER PARTITION LIMIT
+ assertRowsNet(executeNetWithPaging("SELECT a, b, e, count(b), max(e) FROM %s GROUP BY a, b PER PARTITION LIMIT 1 LIMIT 2", pageSize),
+ row(1, 2, 6, 2L, 12),
+ row(2, 2, 6, 1L, 6));
- // Range queries with PER PARTITION LIMIT
- assertRowsNet(executeNetWithPaging("SELECT a, b, e, count(b), max(e) FROM %s GROUP BY a, b PER PARTITION LIMIT 3", pageSize),
- row(1, 2, 6, 2L, 12),
- row(1, 4, 12, 2L, 24),
- row(2, 2, 6, 1L, 6),
- row(2, 4, 12, 1L, 12),
- row(4, 8, 24, 1L, 24));
+ // Range query without aggregates and with PER PARTITION LIMIT
+ assertRowsNet(executeNetWithPaging("SELECT a, b, c, d FROM %s GROUP BY a, b, c PER PARTITION LIMIT 2", pageSize),
+ row(1, 2, 1, 3),
+ row(1, 2, 2, 6),
+ row(2, 2, 3, 3),
+ row(2, 4, 3, 6),
+ row(4, 8, 2, 12));
- assertRowsNet(executeNetWithPaging("SELECT a, b, e, count(b), max(e) FROM %s GROUP BY a, b PER PARTITION LIMIT 1", pageSize),
- row(1, 2, 6, 2L, 12),
- row(2, 2, 6, 1L, 6),
- row(4, 8, 24, 1L, 24));
+ // Range queries without aggregates and with LIMIT
+ assertRowsNet(executeNetWithPaging("SELECT a, b, c, d FROM %s GROUP BY a, b, c LIMIT 3", pageSize),
+ row(1, 2, 1, 3),
+ row(1, 2, 2, 6),
+ row(1, 4, 2, 6));
- // Range query with PER PARTITION LIMIT
- assertRowsNet(executeNetWithPaging("SELECT a, b, e, count(b), max(e) FROM %s GROUP BY a, b PER PARTITION LIMIT 1 LIMIT 2", pageSize),
- row(1, 2, 6, 2L, 12),
- row(2, 2, 6, 1L, 6));
+ assertRowsNet(executeNetWithPaging("SELECT a, b, c, d FROM %s GROUP BY a, b LIMIT 3", pageSize),
+ row(1, 2, 1, 3),
+ row(1, 4, 2, 6),
+ row(2, 2, 3, 3));
- // Range query without aggregates and with PER PARTITION LIMIT
- assertRowsNet(executeNetWithPaging("SELECT a, b, c, d FROM %s GROUP BY a, b, c PER PARTITION LIMIT 2", pageSize),
- row(1, 2, 1, 3),
- row(1, 2, 2, 6),
- row(2, 2, 3, 3),
- row(2, 4, 3, 6),
- row(4, 8, 2, 12));
+ // Range query without aggregates, with PER PARTITION LIMIT and with LIMIT
+ assertRowsNet(executeNetWithPaging("SELECT a, b, c, d FROM %s GROUP BY a, b, c PER PARTITION LIMIT 2 LIMIT 3", pageSize),
+ row(1, 2, 1, 3),
+ row(1, 2, 2, 6),
+ row(2, 2, 3, 3));
- // Range queries with wildcard and with LIMIT
- assertRowsNet(executeNetWithPaging("SELECT * FROM %s GROUP BY a, b, c LIMIT 3", pageSize),
- row(1, 2, 1, 3, 6),
- row(1, 2, 2, 6, 12),
- row(1, 4, 2, 6, 12));
+ // Range queries with wildcard and with LIMIT
+ assertRowsNet(executeNetWithPaging("SELECT * FROM %s GROUP BY a, b, c LIMIT 3", pageSize),
+ row(1, 2, 1, 3, 6),
+ row(1, 2, 2, 6, 12),
+ row(1, 4, 2, 6, 12));
- assertRowsNet(executeNetWithPaging("SELECT * FROM %s GROUP BY a, b LIMIT 3", pageSize),
- row(1, 2, 1, 3, 6),
- row(1, 4, 2, 6, 12),
- row(2, 2, 3, 3, 6));
+ assertRowsNet(executeNetWithPaging("SELECT * FROM %s GROUP BY a, b LIMIT 3", pageSize),
+ row(1, 2, 1, 3, 6),
+ row(1, 4, 2, 6, 12),
+ row(2, 2, 3, 3, 6));
- // Range queries with wildcard and with PER PARTITION LIMIT
- assertRowsNet(executeNetWithPaging("SELECT * FROM %s GROUP BY a, b, c PER PARTITION LIMIT 2", pageSize),
- row(1, 2, 1, 3, 6),
- row(1, 2, 2, 6, 12),
- row(2, 2, 3, 3, 6),
- row(2, 4, 3, 6, 12),
- row(4, 8, 2, 12, 24));
+ // Range queries with wildcard and with PER PARTITION LIMIT
+ assertRowsNet(executeNetWithPaging("SELECT * FROM %s GROUP BY a, b, c PER PARTITION LIMIT 2", pageSize),
+ row(1, 2, 1, 3, 6),
+ row(1, 2, 2, 6, 12),
+ row(2, 2, 3, 3, 6),
+ row(2, 4, 3, 6, 12),
+ row(4, 8, 2, 12, 24));
- assertRowsNet(executeNetWithPaging("SELECT * FROM %s GROUP BY a, b PER PARTITION LIMIT 1", pageSize),
- row(1, 2, 1, 3, 6),
- row(2, 2, 3, 3, 6),
- row(4, 8, 2, 12, 24));
+ assertRowsNet(executeNetWithPaging("SELECT * FROM %s GROUP BY a, b PER PARTITION LIMIT 1", pageSize),
+ row(1, 2, 1, 3, 6),
+ row(2, 2, 3, 3, 6),
+ row(4, 8, 2, 12, 24));
- // Range queries without aggregates and with LIMIT
- assertRowsNet(executeNetWithPaging("SELECT a, b, c, d FROM %s GROUP BY a, b, c LIMIT 3", pageSize),
- row(1, 2, 1, 3),
- row(1, 2, 2, 6),
- row(1, 4, 2, 6));
+ // Range queries with wildcard, with PER PARTITION LIMIT and LIMIT
+ assertRowsNet(executeNetWithPaging("SELECT * FROM %s GROUP BY a, b, c PER PARTITION LIMIT 2 LIMIT 3", pageSize),
+ row(1, 2, 1, 3, 6),
+ row(1, 2, 2, 6, 12),
+ row(2, 2, 3, 3, 6));
- assertRowsNet(executeNetWithPaging("SELECT a, b, c, d FROM %s GROUP BY a, b LIMIT 3", pageSize),
- row(1, 2, 1, 3),
- row(1, 4, 2, 6),
- row(2, 2, 3, 3));
+ // Range query with DISTINCT
+ assertRowsNet(executeNetWithPaging("SELECT DISTINCT a, count(a)FROM %s GROUP BY a", pageSize),
+ row(1, 1L),
+ row(2, 1L),
+ row(4, 1L));
- // Range query without aggregates, with PER PARTITION LIMIT and with LIMIT
- assertRowsNet(executeNetWithPaging("SELECT a, b, c, d FROM %s GROUP BY a, b, c PER PARTITION LIMIT 2 LIMIT 3", pageSize),
- row(1, 2, 1, 3),
- row(1, 2, 2, 6),
- row(2, 2, 3, 3));
+ assertRowsNet(executeNetWithPaging("SELECT DISTINCT a, count(a)FROM %s", pageSize),
+ row(1, 3L));
- // Range queries with wildcard, with PER PARTITION LIMIT and LIMIT
- assertRowsNet(executeNetWithPaging("SELECT * FROM %s GROUP BY a, b, c PER PARTITION LIMIT 2 LIMIT 3", pageSize),
- row(1, 2, 1, 3, 6),
- row(1, 2, 2, 6, 12),
- row(2, 2, 3, 3, 6));
+ // Range query with DISTINCT and LIMIT
+ assertRowsNet(executeNetWithPaging("SELECT DISTINCT a, count(a)FROM %s GROUP BY a LIMIT 2", pageSize),
+ row(1, 1L),
+ row(2, 1L));
- // Range query with DISTINCT
- assertRowsNet(executeNetWithPaging("SELECT DISTINCT a, count(a)FROM %s GROUP BY a", pageSize),
- row(1, 1L),
- row(2, 1L),
- row(4, 1L));
+ assertRowsNet(executeNetWithPaging("SELECT DISTINCT a, count(a)FROM %s LIMIT 2", pageSize),
+ row(1, 3L));
- assertRowsNet(executeNetWithPaging("SELECT DISTINCT a, count(a)FROM %s", pageSize),
- row(1, 3L));
+ // Range query with ORDER BY
+ assertInvalidMessage("ORDER BY is only supported when the partition key is restricted by an EQ or an IN",
+ "SELECT a, b, c, count(b), max(e) FROM %s GROUP BY a, b ORDER BY b DESC, c DESC");
- // Range query with DISTINCT and LIMIT
- assertRowsNet(executeNetWithPaging("SELECT DISTINCT a, count(a)FROM %s GROUP BY a LIMIT 2", pageSize),
- row(1, 1L),
- row(2, 1L));
+ // Single partition queries
+ assertRowsNet(executeNetWithPaging("SELECT a, b, e, count(b), max(e) FROM %s WHERE a = 1 GROUP BY a, b, c",
+ pageSize),
+ row(1, 2, 6, 1L, 6),
+ row(1, 2, 12, 1L, 12),
+ row(1, 4, 12, 2L, 24));
- assertRowsNet(executeNetWithPaging("SELECT DISTINCT a, count(a)FROM %s LIMIT 2", pageSize),
- row(1, 3L));
+ assertRowsNet(executeNetWithPaging("SELECT a, b, e, count(b), max(e) FROM %s WHERE a = 1", pageSize),
+ row(1, 2, 6, 4L, 24));
- // Range query with ORDER BY
- assertInvalidMessage("ORDER BY is only supported when the partition key is restricted by an EQ or an IN",
- "SELECT a, b, c, count(b), max(e) FROM %s GROUP BY a, b ORDER BY b DESC, c DESC");
+ assertRowsNet(executeNetWithPaging("SELECT a, b, e, count(b), max(e) FROM %s WHERE a = 1 AND b = 2 GROUP BY a, b, c",
+ pageSize),
+ row(1, 2, 6, 1L, 6),
+ row(1, 2, 12, 1L, 12));
- // Single partition queries
- assertRowsNet(executeNetWithPaging("SELECT a, b, e, count(b), max(e) FROM %s WHERE a = 1 GROUP BY a, b, c",
- pageSize),
- row(1, 2, 6, 1L, 6),
- row(1, 2, 12, 1L, 12),
- row(1, 4, 12, 2L, 24));
+ assertRowsNet(executeNetWithPaging("SELECT a, b, e, count(b), max(e) FROM %s WHERE a = 1 AND b = 2",
+ pageSize),
+ row(1, 2, 6, 2L, 12));
- assertRowsNet(executeNetWithPaging("SELECT a, b, e, count(b), max(e) FROM %s WHERE a = 1", pageSize),
- row(1, 2, 6, 4L, 24));
+ // Single partition queries without aggregates
+ assertRowsNet(executeNetWithPaging("SELECT a, b, c, d FROM %s WHERE a = 1 GROUP BY a, b", pageSize),
+ row(1, 2, 1, 3),
+ row(1, 4, 2, 6));
- assertRowsNet(executeNetWithPaging("SELECT a, b, e, count(b), max(e) FROM %s WHERE a = 1 AND b = 2 GROUP BY a, b, c",
- pageSize),
- row(1, 2, 6, 1L, 6),
- row(1, 2, 12, 1L, 12));
+ assertRowsNet(executeNetWithPaging("SELECT a, b, c, d FROM %s WHERE a = 1 GROUP BY a, b, c", pageSize),
+ row(1, 2, 1, 3),
+ row(1, 2, 2, 6),
+ row(1, 4, 2, 6));
- assertRowsNet(executeNetWithPaging("SELECT a, b, e, count(b), max(e) FROM %s WHERE a = 1 AND b = 2",
- pageSize),
- row(1, 2, 6, 2L, 12));
+ // Single partition queries with wildcard
+ assertRowsNet(executeNetWithPaging("SELECT * FROM %s WHERE a = 1 GROUP BY a, b, c", pageSize),
+ row(1, 2, 1, 3, 6),
+ row(1, 2, 2, 6, 12),
+ row(1, 4, 2, 6, 12));
- // Single partition queries without aggregates
- assertRowsNet(executeNetWithPaging("SELECT a, b, c, d FROM %s WHERE a = 1 GROUP BY a, b", pageSize),
- row(1, 2, 1, 3),
- row(1, 4, 2, 6));
-
- assertRowsNet(executeNetWithPaging("SELECT a, b, c, d FROM %s WHERE a = 1 GROUP BY a, b, c", pageSize),
- row(1, 2, 1, 3),
- row(1, 2, 2, 6),
- row(1, 4, 2, 6));
-
- // Single partition queries with wildcard
- assertRowsNet(executeNetWithPaging("SELECT * FROM %s WHERE a = 1 GROUP BY a, b, c", pageSize),
- row(1, 2, 1, 3, 6),
- row(1, 2, 2, 6, 12),
- row(1, 4, 2, 6, 12));
-
- assertRowsNet(executeNetWithPaging("SELECT * FROM %s WHERE a = 1 GROUP BY a, b", pageSize),
- row(1, 2, 1, 3, 6),
- row(1, 4, 2, 6, 12));
+ assertRowsNet(executeNetWithPaging("SELECT * FROM %s WHERE a = 1 GROUP BY a, b", pageSize),
+ row(1, 2, 1, 3, 6),
+ row(1, 4, 2, 6, 12));
- // Single partition query with DISTINCT
- assertRowsNet(executeNetWithPaging("SELECT DISTINCT a, count(a)FROM %s WHERE a = 1 GROUP BY a",
- pageSize),
- row(1, 1L));
+ // Single partition query with DISTINCT
+ assertRowsNet(executeNetWithPaging("SELECT DISTINCT a, count(a)FROM %s WHERE a = 1 GROUP BY a",
+ pageSize),
+ row(1, 1L));
- assertRowsNet(executeNetWithPaging("SELECT DISTINCT a, count(a)FROM %s WHERE a = 1 GROUP BY a",
- pageSize),
- row(1, 1L));
+ assertRowsNet(executeNetWithPaging("SELECT DISTINCT a, count(a)FROM %s WHERE a = 1 GROUP BY a",
+ pageSize),
+ row(1, 1L));
- // Single partition queries with LIMIT
- assertRowsNet(executeNetWithPaging("SELECT a, b, e, count(b), max(e) FROM %s WHERE a = 1 GROUP BY a, b, c LIMIT 10",
- pageSize),
- row(1, 2, 6, 1L, 6),
- row(1, 2, 12, 1L, 12),
- row(1, 4, 12, 2L, 24));
+ // Single partition queries with LIMIT
+ assertRowsNet(executeNetWithPaging("SELECT a, b, e, count(b), max(e) FROM %s WHERE a = 1 GROUP BY a, b, c LIMIT 10",
+ pageSize),
+ row(1, 2, 6, 1L, 6),
+ row(1, 2, 12, 1L, 12),
+ row(1, 4, 12, 2L, 24));
- assertRowsNet(executeNetWithPaging("SELECT a, b, e, count(b), max(e) FROM %s WHERE a = 1 GROUP BY a, b, c LIMIT 2",
- pageSize),
- row(1, 2, 6, 1L, 6),
- row(1, 2, 12, 1L, 12));
+ assertRowsNet(executeNetWithPaging("SELECT a, b, e, count(b), max(e) FROM %s WHERE a = 1 GROUP BY a, b, c LIMIT 2",
+ pageSize),
+ row(1, 2, 6, 1L, 6),
+ row(1, 2, 12, 1L, 12));
- assertRowsNet(executeNetWithPaging("SELECT a, b, e, count(b), max(e) FROM %s WHERE a = 1 LIMIT 2",
- pageSize),
- row(1, 2, 6, 4L, 24));
+ assertRowsNet(executeNetWithPaging("SELECT a, b, e, count(b), max(e) FROM %s WHERE a = 1 LIMIT 2",
+ pageSize),
+ row(1, 2, 6, 4L, 24));
- assertRowsNet(executeNetWithPaging("SELECT count(b), max(e) FROM %s WHERE a = 1 GROUP BY a, b, c LIMIT 1",
- pageSize),
- row(1L, 6));
+ assertRowsNet(executeNetWithPaging("SELECT count(b), max(e) FROM %s WHERE a = 1 GROUP BY a, b, c LIMIT 1",
+ pageSize),
+ row(1L, 6));
- // Single partition queries with wildcard and with LIMIT
- assertRowsNet(executeNetWithPaging("SELECT * FROM %s WHERE a = 1 GROUP BY a, b, c LIMIT 2", pageSize),
- row(1, 2, 1, 3, 6),
- row(1, 2, 2, 6, 12));
+ // Single partition query with PER PARTITION LIMIT
+ assertRowsNet(executeNetWithPaging("SELECT a, b, e, count(b), max(e) FROM %s WHERE a = 1 GROUP BY a, b, c PER PARTITION LIMIT 2",
+ pageSize),
+ row(1, 2, 6, 1L, 6),
+ row(1, 2, 12, 1L, 12));
- assertRowsNet(executeNetWithPaging("SELECT * FROM %s WHERE a = 1 GROUP BY a, b LIMIT 1", pageSize),
- row(1, 2, 1, 3, 6));
+ // Single partition queries without aggregates and with LIMIT
+ assertRowsNet(executeNetWithPaging("SELECT a, b, c, d FROM %s WHERE a = 1 GROUP BY a, b LIMIT 2",
+ pageSize),
+ row(1, 2, 1, 3),
+ row(1, 4, 2, 6));
- // Single partition queries with wildcard and with PER PARTITION LIMIT
- assertRowsNet(executeNetWithPaging("SELECT * FROM %s WHERE a = 1 GROUP BY a, b, c PER PARTITION LIMIT 2", pageSize),
- row(1, 2, 1, 3, 6),
- row(1, 2, 2, 6, 12));
+ assertRowsNet(executeNetWithPaging("SELECT a, b, c, d FROM %s WHERE a = 1 GROUP BY a, b LIMIT 1",
+ pageSize),
+ row(1, 2, 1, 3));
- assertRowsNet(executeNetWithPaging("SELECT * FROM %s WHERE a = 1 GROUP BY a, b PER PARTITION LIMIT 1", pageSize),
- row(1, 2, 1, 3, 6));
+ assertRowsNet(executeNetWithPaging("SELECT a, b, c, d FROM %s WHERE a = 1 GROUP BY a, b, c LIMIT 2",
+ pageSize),
+ row(1, 2, 1, 3),
+ row(1, 2, 2, 6));
- // Single partition query with PER PARTITION LIMIT
- assertRowsNet(executeNetWithPaging("SELECT a, b, e, count(b), max(e) FROM %s WHERE a = 1 GROUP BY a, b, c PER PARTITION LIMIT 2",
- pageSize),
- row(1, 2, 6, 1L, 6),
- row(1, 2, 12, 1L, 12));
+ // Single partition queries with wildcard and with LIMIT
+ assertRowsNet(executeNetWithPaging("SELECT * FROM %s WHERE a = 1 GROUP BY a, b, c LIMIT 2", pageSize),
+ row(1, 2, 1, 3, 6),
+ row(1, 2, 2, 6, 12));
- // Single partition queries without aggregates and with LIMIT
- assertRowsNet(executeNetWithPaging("SELECT a, b, c, d FROM %s WHERE a = 1 GROUP BY a, b LIMIT 2",
- pageSize),
- row(1, 2, 1, 3),
- row(1, 4, 2, 6));
+ assertRowsNet(executeNetWithPaging("SELECT * FROM %s WHERE a = 1 GROUP BY a, b LIMIT 1", pageSize),
+ row(1, 2, 1, 3, 6));
- assertRowsNet(executeNetWithPaging("SELECT a, b, c, d FROM %s WHERE a = 1 GROUP BY a, b LIMIT 1",
- pageSize),
- row(1, 2, 1, 3));
+ // Single partition queries with wildcard and with PER PARTITION LIMIT
+ assertRowsNet(executeNetWithPaging("SELECT * FROM %s WHERE a = 1 GROUP BY a, b, c PER PARTITION LIMIT 2", pageSize),
+ row(1, 2, 1, 3, 6),
+ row(1, 2, 2, 6, 12));
- assertRowsNet(executeNetWithPaging("SELECT a, b, c, d FROM %s WHERE a = 1 GROUP BY a, b, c LIMIT 2",
- pageSize),
- row(1, 2, 1, 3),
- row(1, 2, 2, 6));
+ assertRowsNet(executeNetWithPaging("SELECT * FROM %s WHERE a = 1 GROUP BY a, b PER PARTITION LIMIT 1", pageSize),
+ row(1, 2, 1, 3, 6));
- // Single partition queries with ORDER BY
- assertRowsNet(executeNetWithPaging("SELECT a, b, e, count(b), max(e) FROM %s WHERE a = 1 GROUP BY a, b, c ORDER BY b DESC, c DESC",
- pageSize),
- row(1, 4, 24, 2L, 24),
- row(1, 2, 12, 1L, 12),
- row(1, 2, 6, 1L, 6));
+ // Single partition queries with ORDER BY
+ assertRowsNet(executeNetWithPaging("SELECT a, b, e, count(b), max(e) FROM %s WHERE a = 1 GROUP BY a, b, c ORDER BY b DESC, c DESC",
+ pageSize),
+ row(1, 4, 24, 2L, 24),
+ row(1, 2, 12, 1L, 12),
+ row(1, 2, 6, 1L, 6));
- assertRowsNet(executeNetWithPaging("SELECT a, b, e, count(b), max(e) FROM %s WHERE a = 1 ORDER BY b DESC, c DESC",
- pageSize),
- row(1, 4, 24, 4L, 24));
+ assertRowsNet(executeNetWithPaging("SELECT a, b, e, count(b), max(e) FROM %s WHERE a = 1 ORDER BY b DESC, c DESC",
+ pageSize),
+ row(1, 4, 24, 4L, 24));
- // Single partition queries with ORDER BY and LIMIT
- assertRowsNet(executeNetWithPaging("SELECT a, b, e, count(b), max(e) FROM %s WHERE a = 1 GROUP BY a, b, c ORDER BY b DESC, c DESC LIMIT 2",
- pageSize),
- row(1, 4, 24, 2L, 24),
- row(1, 2, 12, 1L, 12));
+ // Single partition queries with ORDER BY and LIMIT
+ assertRowsNet(executeNetWithPaging("SELECT a, b, e, count(b), max(e) FROM %s WHERE a = 1 GROUP BY a, b, c ORDER BY b DESC, c DESC LIMIT 2",
+ pageSize),
+ row(1, 4, 24, 2L, 24),
+ row(1, 2, 12, 1L, 12));
- assertRowsNet(executeNetWithPaging("SELECT a, b, e, count(b), max(e) FROM %s WHERE a = 1 ORDER BY b DESC, c DESC LIMIT 2",
- pageSize),
- row(1, 4, 24, 4L, 24));
+ assertRowsNet(executeNetWithPaging("SELECT a, b, e, count(b), max(e) FROM %s WHERE a = 1 ORDER BY b DESC, c DESC LIMIT 2",
+ pageSize),
+ row(1, 4, 24, 4L, 24));
- // Single partition queries with ORDER BY and PER PARTITION LIMIT
- assertRowsNet(executeNetWithPaging("SELECT a, b, e, count(b), max(e) FROM %s WHERE a = 1 GROUP BY a, b, c ORDER BY b DESC, c DESC PER PARTITION LIMIT 2",
- pageSize),
- row(1, 4, 24, 2L, 24),
- row(1, 2, 12, 1L, 12));
+ // Single partition queries with ORDER BY and PER PARTITION LIMIT
+ assertRowsNet(executeNetWithPaging("SELECT a, b, e, count(b), max(e) FROM %s WHERE a = 1 GROUP BY a, b, c ORDER BY b DESC, c DESC PER PARTITION LIMIT 2",
+ pageSize),
+ row(1, 4, 24, 2L, 24),
+ row(1, 2, 12, 1L, 12));
- // Multi-partitions queries
- assertRowsNet(executeNetWithPaging("SELECT a, b, e, count(b), max(e) FROM %s WHERE a IN (1, 2, 4) GROUP BY a, b, c",
- pageSize),
- row(1, 2, 6, 1L, 6),
- row(1, 2, 12, 1L, 12),
- row(1, 4, 12, 2L, 24),
- row(2, 2, 6, 1L, 6),
- row(2, 4, 12, 1L, 12),
- row(4, 8, 24, 1L, 24));
+ // Multi-partitions queries
+ assertRowsNet(executeNetWithPaging("SELECT a, b, e, count(b), max(e) FROM %s WHERE a IN (1, 2, 4) GROUP BY a, b, c",
+ pageSize),
+ row(1, 2, 6, 1L, 6),
+ row(1, 2, 12, 1L, 12),
+ row(1, 4, 12, 2L, 24),
+ row(2, 2, 6, 1L, 6),
+ row(2, 4, 12, 1L, 12),
+ row(4, 8, 24, 1L, 24));
- assertRowsNet(executeNetWithPaging("SELECT a, b, e, count(b), max(e) FROM %s WHERE a IN (1, 2, 4)",
- pageSize),
- row(1, 2, 6, 7L, 24));
+ assertRowsNet(executeNetWithPaging("SELECT a, b, e, count(b), max(e) FROM %s WHERE a IN (1, 2, 4)",
+ pageSize),
+ row(1, 2, 6, 7L, 24));
- assertRowsNet(executeNetWithPaging("SELECT a, b, e, count(b), max(e) FROM %s WHERE a IN (1, 2, 4) AND b = 2 GROUP BY a, b, c",
- pageSize),
- row(1, 2, 6, 1L, 6),
- row(1, 2, 12, 1L, 12),
- row(2, 2, 6, 1L, 6));
+ assertRowsNet(executeNetWithPaging("SELECT a, b, e, count(b), max(e) FROM %s WHERE a IN (1, 2, 4) AND b = 2 GROUP BY a, b, c",
+ pageSize),
+ row(1, 2, 6, 1L, 6),
+ row(1, 2, 12, 1L, 12),
+ row(2, 2, 6, 1L, 6));
- assertRowsNet(executeNetWithPaging("SELECT a, b, e, count(b), max(e) FROM %s WHERE a IN (1, 2, 4) AND b = 2",
- pageSize),
- row(1, 2, 6, 3L, 12));
+ assertRowsNet(executeNetWithPaging("SELECT a, b, e, count(b), max(e) FROM %s WHERE a IN (1, 2, 4) AND b = 2",
+ pageSize),
+ row(1, 2, 6, 3L, 12));
- // Multi-partitions queries with PER PARTITION LIMIT
- assertRowsNet(executeNetWithPaging("SELECT a, b, e, count(b), max(e) FROM %s WHERE a IN (1, 2, 4) GROUP BY a, b, c PER PARTITION LIMIT 2",
- pageSize),
- row(1, 2, 6, 1L, 6),
- row(1, 2, 12, 1L, 12),
- row(2, 2, 6, 1L, 6),
- row(2, 4, 12, 1L, 12),
- row(4, 8, 24, 1L, 24));
+ // Multi-partitions queries with PER PARTITION LIMIT
+ assertRowsNet(executeNetWithPaging("SELECT a, b, e, count(b), max(e) FROM %s WHERE a IN (1, 2, 4) GROUP BY a, b, c PER PARTITION LIMIT 2",
+ pageSize),
+ row(1, 2, 6, 1L, 6),
+ row(1, 2, 12, 1L, 12),
+ row(2, 2, 6, 1L, 6),
+ row(2, 4, 12, 1L, 12),
+ row(4, 8, 24, 1L, 24));
- assertRowsNet(executeNetWithPaging("SELECT a, b, e, count(b), max(e) FROM %s WHERE a IN (1, 2, 4) GROUP BY a, b, c PER PARTITION LIMIT 1",
- pageSize),
- row(1, 2, 6, 1L, 6),
- row(2, 2, 6, 1L, 6),
- row(4, 8, 24, 1L, 24));
+ assertRowsNet(executeNetWithPaging("SELECT a, b, e, count(b), max(e) FROM %s WHERE a IN (1, 2, 4) GROUP BY a, b, c PER PARTITION LIMIT 1",
+ pageSize),
+ row(1, 2, 6, 1L, 6),
+ row(2, 2, 6, 1L, 6),
+ row(4, 8, 24, 1L, 24));
- // Multi-partitions queries without aggregates
- assertRowsNet(executeNetWithPaging("SELECT a, b, c, d FROM %s WHERE a IN (1, 2, 4) GROUP BY a, b",
- pageSize),
- row(1, 2, 1, 3),
- row(1, 4, 2, 6),
- row(2, 2, 3, 3),
- row(2, 4, 3, 6),
- row(4, 8, 2, 12));
+ // Multi-partitions queries without aggregates
+ assertRowsNet(executeNetWithPaging("SELECT a, b, c, d FROM %s WHERE a IN (1, 2, 4) GROUP BY a, b",
+ pageSize),
+ row(1, 2, 1, 3),
+ row(1, 4, 2, 6),
+ row(2, 2, 3, 3),
+ row(2, 4, 3, 6),
+ row(4, 8, 2, 12));
- assertRowsNet(executeNetWithPaging("SELECT a, b, c, d FROM %s WHERE a IN (1, 2, 4) GROUP BY a, b, c",
- pageSize),
- row(1, 2, 1, 3),
- row(1, 2, 2, 6),
- row(1, 4, 2, 6),
- row(2, 2, 3, 3),
- row(2, 4, 3, 6),
- row(4, 8, 2, 12));
+ assertRowsNet(executeNetWithPaging("SELECT a, b, c, d FROM %s WHERE a IN (1, 2, 4) GROUP BY a, b, c",
+ pageSize),
+ row(1, 2, 1, 3),
+ row(1, 2, 2, 6),
+ row(1, 4, 2, 6),
+ row(2, 2, 3, 3),
+ row(2, 4, 3, 6),
+ row(4, 8, 2, 12));
- // Multi-partitions with wildcard
- assertRowsNet(executeNetWithPaging("SELECT * FROM %s WHERE a IN (1, 2, 4) GROUP BY a, b, c", pageSize),
- row(1, 2, 1, 3, 6),
- row(1, 2, 2, 6, 12),
- row(1, 4, 2, 6, 12),
- row(2, 2, 3, 3, 6),
- row(2, 4, 3, 6, 12),
- row(4, 8, 2, 12, 24));
+ // Multi-partitions with wildcard
+ assertRowsNet(executeNetWithPaging("SELECT * FROM %s WHERE a IN (1, 2, 4) GROUP BY a, b, c", pageSize),
+ row(1, 2, 1, 3, 6),
+ row(1, 2, 2, 6, 12),
+ row(1, 4, 2, 6, 12),
+ row(2, 2, 3, 3, 6),
+ row(2, 4, 3, 6, 12),
+ row(4, 8, 2, 12, 24));
- assertRowsNet(executeNetWithPaging("SELECT * FROM %s WHERE a IN (1, 2, 4) GROUP BY a, b", pageSize),
- row(1, 2, 1, 3, 6),
- row(1, 4, 2, 6, 12),
- row(2, 2, 3, 3, 6),
- row(2, 4, 3, 6, 12),
- row(4, 8, 2, 12, 24));
+ assertRowsNet(executeNetWithPaging("SELECT * FROM %s WHERE a IN (1, 2, 4) GROUP BY a, b", pageSize),
+ row(1, 2, 1, 3, 6),
+ row(1, 4, 2, 6, 12),
+ row(2, 2, 3, 3, 6),
+ row(2, 4, 3, 6, 12),
+ row(4, 8, 2, 12, 24));
- // Multi-partitions queries with DISTINCT
- assertRowsNet(executeNetWithPaging("SELECT DISTINCT a, count(a)FROM %s WHERE a IN (1, 2, 4) GROUP BY a",
- pageSize),
- row(1, 1L),
- row(2, 1L),
- row(4, 1L));
+ // Multi-partitions queries with DISTINCT
+ assertRowsNet(executeNetWithPaging("SELECT DISTINCT a, count(a)FROM %s WHERE a IN (1, 2, 4) GROUP BY a",
+ pageSize),
+ row(1, 1L),
+ row(2, 1L),
+ row(4, 1L));
- assertRowsNet(executeNetWithPaging("SELECT DISTINCT a, count(a)FROM %s WHERE a IN (1, 2, 4)",
- pageSize),
- row(1, 3L));
+ assertRowsNet(executeNetWithPaging("SELECT DISTINCT a, count(a)FROM %s WHERE a IN (1, 2, 4)",
+ pageSize),
+ row(1, 3L));
- // Multi-partitions query with DISTINCT and LIMIT
- assertRowsNet(executeNetWithPaging("SELECT DISTINCT a, count(a)FROM %s WHERE a IN (1, 2, 4) GROUP BY a LIMIT 2",
- pageSize),
- row(1, 1L),
- row(2, 1L));
+ // Multi-partitions query with DISTINCT and LIMIT
+ assertRowsNet(executeNetWithPaging("SELECT DISTINCT a, count(a)FROM %s WHERE a IN (1, 2, 4) GROUP BY a LIMIT 2",
+ pageSize),
+ row(1, 1L),
+ row(2, 1L));
- assertRowsNet(executeNetWithPaging("SELECT DISTINCT a, count(a)FROM %s WHERE a IN (1, 2, 4) LIMIT 2",
- pageSize),
- row(1, 3L));
- }
+ assertRowsNet(executeNetWithPaging("SELECT DISTINCT a, count(a)FROM %s WHERE a IN (1, 2, 4) LIMIT 2",
+ pageSize),
+ row(1, 3L));
}
}
@Test
public void testGroupByWithRangeNamesQueryWithPaging() throws Throwable
{
- for (String compactOption : new String[] { "", " WITH COMPACT STORAGE" })
+ createTable("CREATE TABLE %s (a int, b int, c int, d int, primary key (a, b, c))");
+
+ for (int i = 1; i < 5; i++)
+ for (int j = 1; j < 5; j++)
+ for (int k = 1; k < 5; k++)
+ execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", i, j, k, i + j);
+
+ // Makes sure that we have some tombstones
+ execute("DELETE FROM %s WHERE a = 3");
+
+ for (int pageSize = 1; pageSize < 2; pageSize++)
{
- createTable("CREATE TABLE %s (a int, b int, c int, d int, primary key (a, b, c))"
- + compactOption);
+ // Range queries
+ assertRowsNet(executeNetWithPaging("SELECT a, b, d, count(b), max(d) FROM %s WHERE b = 1 and c IN (1, 2) GROUP BY a ALLOW FILTERING", pageSize),
+ row(1, 1, 2, 2L, 2),
+ row(2, 1, 3, 2L, 3),
+ row(4, 1, 5, 2L, 5));
- for (int i = 1; i < 5; i++)
- for (int j = 1; j < 5; j++)
- for (int k = 1; k < 5; k++)
- execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", i, j, k, i + j);
+ assertRowsNet(executeNetWithPaging("SELECT a, b, d, count(b), max(d) FROM %s WHERE b = 1 and c IN (1, 2) GROUP BY a, b ALLOW FILTERING", pageSize),
+ row(1, 1, 2, 2L, 2),
+ row(2, 1, 3, 2L, 3),
+ row(4, 1, 5, 2L, 5));
- // Makes sure that we have some tombstones
- execute("DELETE FROM %s WHERE a = 3");
+ assertRowsNet(executeNetWithPaging("SELECT a, b, d, count(b), max(d) FROM %s WHERE b IN (1, 2) and c IN (1, 2) GROUP BY a, b ALLOW FILTERING", pageSize),
+ row(1, 1, 2, 2L, 2),
+ row(1, 2, 3, 2L, 3),
+ row(2, 1, 3, 2L, 3),
+ row(2, 2, 4, 2L, 4),
+ row(4, 1, 5, 2L, 5),
+ row(4, 2, 6, 2L, 6));
- for (int pageSize = 1; pageSize < 2; pageSize++)
- {
- // Range queries
- assertRowsNet(executeNetWithPaging("SELECT a, b, d, count(b), max(d) FROM %s WHERE b = 1 and c IN (1, 2) GROUP BY a ALLOW FILTERING", pageSize),
- row(1, 1, 2, 2L, 2),
- row(2, 1, 3, 2L, 3),
- row(4, 1, 5, 2L, 5));
+ // Range queries with LIMIT
+ assertRowsNet(executeNetWithPaging("SELECT a, b, d, count(b), max(d) FROM %s WHERE b = 1 and c IN (1, 2) GROUP BY a LIMIT 5 ALLOW FILTERING", pageSize),
+ row(1, 1, 2, 2L, 2),
+ row(2, 1, 3, 2L, 3),
+ row(4, 1, 5, 2L, 5));
- assertRowsNet(executeNetWithPaging("SELECT a, b, d, count(b), max(d) FROM %s WHERE b = 1 and c IN (1, 2) GROUP BY a, b ALLOW FILTERING", pageSize),
- row(1, 1, 2, 2L, 2),
- row(2, 1, 3, 2L, 3),
- row(4, 1, 5, 2L, 5));
+ assertRowsNet(executeNetWithPaging("SELECT a, b, d, count(b), max(d) FROM %s WHERE b = 1 and c IN (1, 2) GROUP BY a, b LIMIT 3 ALLOW FILTERING", pageSize),
+ row(1, 1, 2, 2L, 2),
+ row(2, 1, 3, 2L, 3),
+ row(4, 1, 5, 2L, 5));
- assertRowsNet(executeNetWithPaging("SELECT a, b, d, count(b), max(d) FROM %s WHERE b IN (1, 2) and c IN (1, 2) GROUP BY a, b ALLOW FILTERING", pageSize),
- row(1, 1, 2, 2L, 2),
- row(1, 2, 3, 2L, 3),
- row(2, 1, 3, 2L, 3),
- row(2, 2, 4, 2L, 4),
- row(4, 1, 5, 2L, 5),
- row(4, 2, 6, 2L, 6));
+ assertRowsNet(executeNetWithPaging("SELECT a, b, d, count(b), max(d) FROM %s WHERE b IN (1, 2) and c IN (1, 2) GROUP BY a, b LIMIT 3 ALLOW FILTERING", pageSize),
+ row(1, 1, 2, 2L, 2),
+ row(1, 2, 3, 2L, 3),
+ row(2, 1, 3, 2L, 3));
- // Range queries with LIMIT
- assertRowsNet(executeNetWithPaging("SELECT a, b, d, count(b), max(d) FROM %s WHERE b = 1 and c IN (1, 2) GROUP BY a LIMIT 5 ALLOW FILTERING", pageSize),
- row(1, 1, 2, 2L, 2),
- row(2, 1, 3, 2L, 3),
- row(4, 1, 5, 2L, 5));
+ // Range queries with PER PARTITION LIMIT
+ assertRowsNet(executeNetWithPaging("SELECT a, b, d, count(b), max(d) FROM %s WHERE b = 1 and c IN (1, 2) GROUP BY a, b PER PARTITION LIMIT 2 ALLOW FILTERING", pageSize),
+ row(1, 1, 2, 2L, 2),
+ row(2, 1, 3, 2L, 3),
+ row(4, 1, 5, 2L, 5));
- assertRowsNet(executeNetWithPaging("SELECT a, b, d, count(b), max(d) FROM %s WHERE b = 1 and c IN (1, 2) GROUP BY a, b LIMIT 3 ALLOW FILTERING", pageSize),
- row(1, 1, 2, 2L, 2),
- row(2, 1, 3, 2L, 3),
- row(4, 1, 5, 2L, 5));
+ assertRowsNet(executeNetWithPaging("SELECT a, b, d, count(b), max(d) FROM %s WHERE b IN (1, 2) and c IN (1, 2) GROUP BY a, b PER PARTITION LIMIT 1 ALLOW FILTERING", pageSize),
+ row(1, 1, 2, 2L, 2),
+ row(2, 1, 3, 2L, 3),
+ row(4, 1, 5, 2L, 5));
- assertRowsNet(executeNetWithPaging("SELECT a, b, d, count(b), max(d) FROM %s WHERE b IN (1, 2) and c IN (1, 2) GROUP BY a, b LIMIT 3 ALLOW FILTERING", pageSize),
- row(1, 1, 2, 2L, 2),
- row(1, 2, 3, 2L, 3),
- row(2, 1, 3, 2L, 3));
+ // Range queries with PER PARTITION LIMIT and LIMIT
+ assertRowsNet(executeNetWithPaging("SELECT a, b, d, count(b), max(d) FROM %s WHERE b = 1 and c IN (1, 2) GROUP BY a, b PER PARTITION LIMIT 2 LIMIT 5 ALLOW FILTERING", pageSize),
+ row(1, 1, 2, 2L, 2),
+ row(2, 1, 3, 2L, 3),
+ row(4, 1, 5, 2L, 5));
- // Range queries with PER PARTITION LIMIT
- assertRowsNet(executeNetWithPaging("SELECT a, b, d, count(b), max(d) FROM %s WHERE b = 1 and c IN (1, 2) GROUP BY a, b PER PARTITION LIMIT 2 ALLOW FILTERING", pageSize),
- row(1, 1, 2, 2L, 2),
- row(2, 1, 3, 2L, 3),
- row(4, 1, 5, 2L, 5));
-
- assertRowsNet(executeNetWithPaging("SELECT a, b, d, count(b), max(d) FROM %s WHERE b IN (1, 2) and c IN (1, 2) GROUP BY a, b PER PARTITION LIMIT 1 ALLOW FILTERING", pageSize),
- row(1, 1, 2, 2L, 2),
- row(2, 1, 3, 2L, 3),
- row(4, 1, 5, 2L, 5));
-
- // Range queries with PER PARTITION LIMIT and LIMIT
- assertRowsNet(executeNetWithPaging("SELECT a, b, d, count(b), max(d) FROM %s WHERE b = 1 and c IN (1, 2) GROUP BY a, b PER PARTITION LIMIT 2 LIMIT 5 ALLOW FILTERING", pageSize),
- row(1, 1, 2, 2L, 2),
- row(2, 1, 3, 2L, 3),
- row(4, 1, 5, 2L, 5));
-
- assertRowsNet(executeNetWithPaging("SELECT a, b, d, count(b), max(d) FROM %s WHERE b IN (1, 2) and c IN (1, 2) GROUP BY a, b PER PARTITION LIMIT 1 LIMIT 2 ALLOW FILTERING", pageSize),
- row(1, 1, 2, 2L, 2),
- row(2, 1, 3, 2L, 3));
- }
+ assertRowsNet(executeNetWithPaging("SELECT a, b, d, count(b), max(d) FROM %s WHERE b IN (1, 2) and c IN (1, 2) GROUP BY a, b PER PARTITION LIMIT 1 LIMIT 2 ALLOW FILTERING", pageSize),
+ row(1, 1, 2, 2L, 2),
+ row(2, 1, 3, 2L, 3));
}
}
diff --git a/test/unit/org/apache/cassandra/cql3/validation/operations/SelectLimitTest.java b/test/unit/org/apache/cassandra/cql3/validation/operations/SelectLimitTest.java
index 5c45451..2e419e1 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/operations/SelectLimitTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/operations/SelectLimitTest.java
@@ -30,51 +30,14 @@
public class SelectLimitTest extends CQLTester
{
+ // This method will be ran instead of the CQLTester#setUpClass
@BeforeClass
- public static void setUp()
+ public static void setUpClass()
{
StorageService.instance.setPartitionerUnsafe(ByteOrderedPartitioner.instance);
DatabaseDescriptor.setPartitionerUnsafe(ByteOrderedPartitioner.instance);
- }
- /**
- * Test limit across a partition range, requires byte ordered partitioner,
- * migrated from cql_tests.py:TestCQL.limit_range_test()
- */
- @Test
- public void testPartitionRange() throws Throwable
- {
- createTable("CREATE TABLE %s (userid int, url text, time bigint, PRIMARY KEY (userid, url)) WITH COMPACT STORAGE");
-
- for (int i = 0; i < 100; i++)
- for (String tld : new String[] { "com", "org", "net" })
- execute("INSERT INTO %s (userid, url, time) VALUES (?, ?, ?)", i, String.format("http://foo.%s", tld), 42L);
-
- assertRows(execute("SELECT * FROM %s WHERE token(userid) >= token(2) LIMIT 1"),
- row(2, "http://foo.com", 42L));
-
- assertRows(execute("SELECT * FROM %s WHERE token(userid) > token(2) LIMIT 1"),
- row(3, "http://foo.com", 42L));
- }
-
- /**
- * Test limit across a column range,
- * migrated from cql_tests.py:TestCQL.limit_multiget_test()
- */
- @Test
- public void testColumnRange() throws Throwable
- {
- createTable("CREATE TABLE %s (userid int, url text, time bigint, PRIMARY KEY (userid, url)) WITH COMPACT STORAGE");
-
- for (int i = 0; i < 100; i++)
- for (String tld : new String[] { "com", "org", "net" })
- execute("INSERT INTO %s (userid, url, time) VALUES (?, ?, ?)", i, String.format("http://foo.%s", tld), 42L);
-
- // Check that we do limit the output to 1 *and* that we respect query
- // order of keys (even though 48 is after 2)
- assertRows(execute("SELECT * FROM %s WHERE userid IN (48, 2) LIMIT 1"),
- row(2, "http://foo.com", 42L));
-
+ prepareServer();
}
/**
@@ -95,103 +58,11 @@
}
@Test
- public void testLimitInStaticTable() throws Throwable
- {
- createTable("CREATE TABLE %s (k int, v int, PRIMARY KEY (k) ) WITH COMPACT STORAGE ");
-
- for (int i = 0; i < 10; i++)
- execute("INSERT INTO %s(k, v) VALUES (?, ?)", i, i);
-
- assertRows(execute("SELECT * FROM %s LIMIT 5"),
- row(0, 0),
- row(1, 1),
- row(2, 2),
- row(3, 3),
- row(4, 4));
-
- assertRows(execute("SELECT v FROM %s LIMIT 5"),
- row(0),
- row(1),
- row(2),
- row(3),
- row(4));
-
- assertRows(execute("SELECT k FROM %s LIMIT 5"),
- row(0),
- row(1),
- row(2),
- row(3),
- row(4));
-
- assertRows(execute("SELECT DISTINCT k FROM %s LIMIT 5"),
- row(0),
- row(1),
- row(2),
- row(3),
- row(4));
- }
-
- /**
- * Check for #7052 bug,
- * migrated from cql_tests.py:TestCQL.limit_compact_table()
- */
- @Test
- public void testLimitInCompactTable() throws Throwable
- {
- createTable("CREATE TABLE %s (k int, v int, PRIMARY KEY (k, v) ) WITH COMPACT STORAGE ");
-
- for (int i = 0; i < 4; i++)
- for (int j = 0; j < 4; j++)
- execute("INSERT INTO %s(k, v) VALUES (?, ?)", i, j);
-
- assertRows(execute("SELECT v FROM %s WHERE k=0 AND v > 0 AND v <= 4 LIMIT 2"),
- row(1),
- row(2));
- assertRows(execute("SELECT v FROM %s WHERE k=0 AND v > -1 AND v <= 4 LIMIT 2"),
- row(0),
- row(1));
- assertRows(execute("SELECT * FROM %s WHERE k IN (0, 1, 2) AND v > 0 AND v <= 4 LIMIT 2"),
- row(0, 1),
- row(0, 2));
- assertRows(execute("SELECT * FROM %s WHERE k IN (0, 1, 2) AND v > -1 AND v <= 4 LIMIT 2"),
- row(0, 0),
- row(0, 1));
- assertRows(execute("SELECT * FROM %s WHERE k IN (0, 1, 2) AND v > 0 AND v <= 4 LIMIT 6"),
- row(0, 1),
- row(0, 2),
- row(0, 3),
- row(1, 1),
- row(1, 2),
- row(1, 3));
- assertRows(execute("SELECT * FROM %s WHERE v > 1 AND v <= 3 LIMIT 6 ALLOW FILTERING"),
- row(0, 2),
- row(0, 3),
- row(1, 2),
- row(1, 3),
- row(2, 2),
- row(2, 3));
- }
-
- @Test
public void testPerPartitionLimit() throws Throwable
{
- perPartitionLimitTest(false);
- }
-
- @Test
- public void testPerPartitionLimitWithCompactStorage() throws Throwable
- {
- perPartitionLimitTest(true);
- }
-
- private void perPartitionLimitTest(boolean withCompactStorage) throws Throwable
- {
String query = "CREATE TABLE %s (a int, b int, c int, PRIMARY KEY (a, b))";
- if (withCompactStorage)
- createTable(query + " WITH COMPACT STORAGE");
- else
- createTable(query);
+ createTable(query);
for (int i = 0; i < 5; i++)
{
diff --git a/test/unit/org/apache/cassandra/cql3/validation/operations/SelectMultiColumnRelationTest.java b/test/unit/org/apache/cassandra/cql3/validation/operations/SelectMultiColumnRelationTest.java
index 30c727b..5062448 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/operations/SelectMultiColumnRelationTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/operations/SelectMultiColumnRelationTest.java
@@ -32,781 +32,754 @@
@Test
public void testSingleClusteringInvalidQueries() throws Throwable
{
- for (String compactOption : new String[] { "", " WITH COMPACT STORAGE" })
- {
- createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY (a, b))" + compactOption);
+ createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY (a, b))");
- assertInvalidSyntax("SELECT * FROM %s WHERE () = (?, ?)", 1, 2);
- assertInvalidMessage("b cannot be restricted by more than one relation if it includes an Equal",
- "SELECT * FROM %s WHERE a = 0 AND (b) = (?) AND (b) > (?)", 0, 0);
- assertInvalidMessage("More than one restriction was found for the start bound on b",
- "SELECT * FROM %s WHERE a = 0 AND (b) > (?) AND (b) > (?)", 0, 1);
- assertInvalidMessage("More than one restriction was found for the start bound on b",
- "SELECT * FROM %s WHERE a = 0 AND (b) > (?) AND b > ?", 0, 1);
- assertInvalidMessage("Multi-column relations can only be applied to clustering columns but was applied to: a",
- "SELECT * FROM %s WHERE (a, b) = (?, ?)", 0, 0);
- }
+ assertInvalidSyntax("SELECT * FROM %s WHERE () = (?, ?)", 1, 2);
+ assertInvalidMessage("b cannot be restricted by more than one relation if it includes an Equal",
+ "SELECT * FROM %s WHERE a = 0 AND (b) = (?) AND (b) > (?)", 0, 0);
+ assertInvalidMessage("More than one restriction was found for the start bound on b",
+ "SELECT * FROM %s WHERE a = 0 AND (b) > (?) AND (b) > (?)", 0, 1);
+ assertInvalidMessage("More than one restriction was found for the start bound on b",
+ "SELECT * FROM %s WHERE a = 0 AND (b) > (?) AND b > ?", 0, 1);
+ assertInvalidMessage("Multi-column relations can only be applied to clustering columns but was applied to: a",
+ "SELECT * FROM %s WHERE (a, b) = (?, ?)", 0, 0);
}
@Test
public void testMultiClusteringInvalidQueries() throws Throwable
{
- for (String compactOption : new String[] { "", " WITH COMPACT STORAGE" })
- {
- createTable("CREATE TABLE %s (a int, b int, c int, d int, PRIMARY KEY (a, b, c, d))" + compactOption);
+ createTable("CREATE TABLE %s (a int, b int, c int, d int, PRIMARY KEY (a, b, c, d))");
- assertInvalidSyntax("SELECT * FROM %s WHERE a = 0 AND (b, c) > ()");
- assertInvalidMessage("Expected 2 elements in value tuple, but got 3: (?, ?, ?)",
- "SELECT * FROM %s WHERE a = 0 AND (b, c) > (?, ?, ?)", 1, 2, 3);
- assertInvalidMessage("Invalid null value in condition for column c",
- "SELECT * FROM %s WHERE a = 0 AND (b, c) > (?, ?)", 1, null);
+ assertInvalidSyntax("SELECT * FROM %s WHERE a = 0 AND (b, c) > ()");
+ assertInvalidMessage("Expected 2 elements in value tuple, but got 3: (?, ?, ?)",
+ "SELECT * FROM %s WHERE a = 0 AND (b, c) > (?, ?, ?)", 1, 2, 3);
+ assertInvalidMessage("Invalid null value in condition for column c",
+ "SELECT * FROM %s WHERE a = 0 AND (b, c) > (?, ?)", 1, null);
- // Wrong order of columns
- assertInvalidMessage("Clustering columns must appear in the PRIMARY KEY order in multi-column relations: (d, c, b) = (?, ?, ?)",
- "SELECT * FROM %s WHERE a = 0 AND (d, c, b) = (?, ?, ?)", 0, 0, 0);
- assertInvalidMessage("Clustering columns must appear in the PRIMARY KEY order in multi-column relations: (d, c, b) > (?, ?, ?)",
- "SELECT * FROM %s WHERE a = 0 AND (d, c, b) > (?, ?, ?)", 0, 0, 0);
+ // Wrong order of columns
+ assertInvalidMessage("Clustering columns must appear in the PRIMARY KEY order in multi-column relations: (d, c, b) = (?, ?, ?)",
+ "SELECT * FROM %s WHERE a = 0 AND (d, c, b) = (?, ?, ?)", 0, 0, 0);
+ assertInvalidMessage("Clustering columns must appear in the PRIMARY KEY order in multi-column relations: (d, c, b) > (?, ?, ?)",
+ "SELECT * FROM %s WHERE a = 0 AND (d, c, b) > (?, ?, ?)", 0, 0, 0);
- // Wrong number of values
- assertInvalidMessage("Expected 3 elements in value tuple, but got 2: (?, ?)",
- "SELECT * FROM %s WHERE a=0 AND (b, c, d) IN ((?, ?))", 0, 1);
- assertInvalidMessage("Expected 3 elements in value tuple, but got 5: (?, ?, ?, ?, ?)",
- "SELECT * FROM %s WHERE a=0 AND (b, c, d) IN ((?, ?, ?, ?, ?))", 0, 1, 2, 3, 4);
+ // Wrong number of values
+ assertInvalidMessage("Expected 3 elements in value tuple, but got 2: (?, ?)",
+ "SELECT * FROM %s WHERE a=0 AND (b, c, d) IN ((?, ?))", 0, 1);
+ assertInvalidMessage("Expected 3 elements in value tuple, but got 5: (?, ?, ?, ?, ?)",
+ "SELECT * FROM %s WHERE a=0 AND (b, c, d) IN ((?, ?, ?, ?, ?))", 0, 1, 2, 3, 4);
- // Missing first clustering column
- assertInvalidMessage("PRIMARY KEY column \"c\" cannot be restricted as preceding column \"b\" is not restricted",
- "SELECT * FROM %s WHERE a = 0 AND (c, d) = (?, ?)", 0, 0);
- assertInvalidMessage("PRIMARY KEY column \"c\" cannot be restricted as preceding column \"b\" is not restricted",
- "SELECT * FROM %s WHERE a = 0 AND (c, d) > (?, ?)", 0, 0);
+ // Missing first clustering column
+ assertInvalidMessage("PRIMARY KEY column \"c\" cannot be restricted as preceding column \"b\" is not restricted",
+ "SELECT * FROM %s WHERE a = 0 AND (c, d) = (?, ?)", 0, 0);
+ assertInvalidMessage("PRIMARY KEY column \"c\" cannot be restricted as preceding column \"b\" is not restricted",
+ "SELECT * FROM %s WHERE a = 0 AND (c, d) > (?, ?)", 0, 0);
- // Nulls
- assertInvalidMessage("Invalid null value for column d",
- "SELECT * FROM %s WHERE a = 0 AND (b, c, d) = (?, ?, ?)", 1, 2, null);
- assertInvalidMessage("Invalid null value for column d",
- "SELECT * FROM %s WHERE a = 0 AND (b, c, d) IN ((?, ?, ?))", 1, 2, null);
- assertInvalidMessage("Invalid null value in condition for columns: [b, c, d]",
- "SELECT * FROM %s WHERE a = 0 AND (b, c, d) IN ((?, ?, ?), (?, ?, ?))", 1, 2, null, 2, 1 ,4);
+ // Nulls
+ assertInvalidMessage("Invalid null value for column d",
+ "SELECT * FROM %s WHERE a = 0 AND (b, c, d) = (?, ?, ?)", 1, 2, null);
+ assertInvalidMessage("Invalid null value for column d",
+ "SELECT * FROM %s WHERE a = 0 AND (b, c, d) IN ((?, ?, ?))", 1, 2, null);
+ assertInvalidMessage("Invalid null value in condition for columns: [b, c, d]",
+ "SELECT * FROM %s WHERE a = 0 AND (b, c, d) IN ((?, ?, ?), (?, ?, ?))", 1, 2, null, 2, 1, 4);
- // Wrong type for 'd'
- assertInvalid("SELECT * FROM %s WHERE a = 0 AND (b, c, d) = (?, ?, ?)", 1, 2, "foobar");
- assertInvalid("SELECT * FROM %s WHERE a = 0 AND b = (?, ?, ?)", 1, 2, 3);
+ // Wrong type for 'd'
+ assertInvalid("SELECT * FROM %s WHERE a = 0 AND (b, c, d) = (?, ?, ?)", 1, 2, "foobar");
+ assertInvalid("SELECT * FROM %s WHERE a = 0 AND b = (?, ?, ?)", 1, 2, 3);
- // Mix single and tuple inequalities
- assertInvalidMessage("Column \"c\" cannot be restricted by two inequalities not starting with the same column",
- "SELECT * FROM %s WHERE a = 0 AND (b, c, d) > (?, ?, ?) AND c < ?", 0, 1, 0, 1);
- assertInvalidMessage("Column \"c\" cannot be restricted by two inequalities not starting with the same column",
- "SELECT * FROM %s WHERE a = 0 AND c > ? AND (b, c, d) < (?, ?, ?)", 1, 1, 1, 0);
+ // Mix single and tuple inequalities
+ assertInvalidMessage("Column \"c\" cannot be restricted by two inequalities not starting with the same column",
+ "SELECT * FROM %s WHERE a = 0 AND (b, c, d) > (?, ?, ?) AND c < ?", 0, 1, 0, 1);
+ assertInvalidMessage("Column \"c\" cannot be restricted by two inequalities not starting with the same column",
+ "SELECT * FROM %s WHERE a = 0 AND c > ? AND (b, c, d) < (?, ?, ?)", 1, 1, 1, 0);
- assertInvalidMessage("Multi-column relations can only be applied to clustering columns but was applied to: a",
- "SELECT * FROM %s WHERE (a, b, c, d) IN ((?, ?, ?, ?))", 0, 1, 2, 3);
- assertInvalidMessage("PRIMARY KEY column \"c\" cannot be restricted as preceding column \"b\" is not restricted",
- "SELECT * FROM %s WHERE (c, d) IN ((?, ?))", 0, 1);
+ assertInvalidMessage("Multi-column relations can only be applied to clustering columns but was applied to: a",
+ "SELECT * FROM %s WHERE (a, b, c, d) IN ((?, ?, ?, ?))", 0, 1, 2, 3);
+ assertInvalidMessage("PRIMARY KEY column \"c\" cannot be restricted as preceding column \"b\" is not restricted",
+ "SELECT * FROM %s WHERE (c, d) IN ((?, ?))", 0, 1);
- assertInvalidMessage("Clustering column \"c\" cannot be restricted (preceding column \"b\" is restricted by a non-EQ relation)",
- "SELECT * FROM %s WHERE a = ? AND b > ? AND (c, d) IN ((?, ?))", 0, 0, 0, 0);
+ assertInvalidMessage("Clustering column \"c\" cannot be restricted (preceding column \"b\" is restricted by a non-EQ relation)",
+ "SELECT * FROM %s WHERE a = ? AND b > ? AND (c, d) IN ((?, ?))", 0, 0, 0, 0);
- assertInvalidMessage("Clustering column \"c\" cannot be restricted (preceding column \"b\" is restricted by a non-EQ relation)",
- "SELECT * FROM %s WHERE a = ? AND b > ? AND (c, d) > (?, ?)", 0, 0, 0, 0);
- assertInvalidMessage("PRIMARY KEY column \"c\" cannot be restricted (preceding column \"b\" is restricted by a non-EQ relation)",
- "SELECT * FROM %s WHERE a = ? AND (c, d) > (?, ?) AND b > ? ", 0, 0, 0, 0);
+ assertInvalidMessage("Clustering column \"c\" cannot be restricted (preceding column \"b\" is restricted by a non-EQ relation)",
+ "SELECT * FROM %s WHERE a = ? AND b > ? AND (c, d) > (?, ?)", 0, 0, 0, 0);
+ assertInvalidMessage("PRIMARY KEY column \"c\" cannot be restricted (preceding column \"b\" is restricted by a non-EQ relation)",
+ "SELECT * FROM %s WHERE a = ? AND (c, d) > (?, ?) AND b > ? ", 0, 0, 0, 0);
- assertInvalidMessage("Column \"c\" cannot be restricted by two inequalities not starting with the same column",
- "SELECT * FROM %s WHERE a = ? AND (b, c) > (?, ?) AND (b) < (?) AND (c) < (?)", 0, 0, 0, 0, 0);
- assertInvalidMessage("Column \"c\" cannot be restricted by two inequalities not starting with the same column",
- "SELECT * FROM %s WHERE a = ? AND (c) < (?) AND (b, c) > (?, ?) AND (b) < (?)", 0, 0, 0, 0, 0);
- assertInvalidMessage("Clustering column \"c\" cannot be restricted (preceding column \"b\" is restricted by a non-EQ relation)",
- "SELECT * FROM %s WHERE a = ? AND (b) < (?) AND (c) < (?) AND (b, c) > (?, ?)", 0, 0, 0, 0, 0);
- assertInvalidMessage("Clustering column \"c\" cannot be restricted (preceding column \"b\" is restricted by a non-EQ relation)",
- "SELECT * FROM %s WHERE a = ? AND (b) < (?) AND c < ? AND (b, c) > (?, ?)", 0, 0, 0, 0, 0);
+ assertInvalidMessage("Column \"c\" cannot be restricted by two inequalities not starting with the same column",
+ "SELECT * FROM %s WHERE a = ? AND (b, c) > (?, ?) AND (b) < (?) AND (c) < (?)", 0, 0, 0, 0, 0);
+ assertInvalidMessage("Column \"c\" cannot be restricted by two inequalities not starting with the same column",
+ "SELECT * FROM %s WHERE a = ? AND (c) < (?) AND (b, c) > (?, ?) AND (b) < (?)", 0, 0, 0, 0, 0);
+ assertInvalidMessage("Clustering column \"c\" cannot be restricted (preceding column \"b\" is restricted by a non-EQ relation)",
+ "SELECT * FROM %s WHERE a = ? AND (b) < (?) AND (c) < (?) AND (b, c) > (?, ?)", 0, 0, 0, 0, 0);
+ assertInvalidMessage("Clustering column \"c\" cannot be restricted (preceding column \"b\" is restricted by a non-EQ relation)",
+ "SELECT * FROM %s WHERE a = ? AND (b) < (?) AND c < ? AND (b, c) > (?, ?)", 0, 0, 0, 0, 0);
- assertInvalidMessage("Column \"c\" cannot be restricted by two inequalities not starting with the same column",
- "SELECT * FROM %s WHERE a = ? AND (b, c) > (?, ?) AND (c) < (?)", 0, 0, 0, 0);
- }
+ assertInvalidMessage("Column \"c\" cannot be restricted by two inequalities not starting with the same column",
+ "SELECT * FROM %s WHERE a = ? AND (b, c) > (?, ?) AND (c) < (?)", 0, 0, 0, 0);
}
@Test
public void testMultiAndSingleColumnRelationMix() throws Throwable
{
- for (String compactOption : new String[]{"", " WITH COMPACT STORAGE"})
- {
- createTable("CREATE TABLE %s (a int, b int, c int, d int, PRIMARY KEY (a, b, c, d))" + compactOption);
+ createTable("CREATE TABLE %s (a int, b int, c int, d int, PRIMARY KEY (a, b, c, d))");
- execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 0, 0, 0, 0);
- execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 0, 0, 1, 0);
- execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 0, 0, 1, 1);
+ execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 0, 0, 0, 0);
+ execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 0, 0, 1, 0);
+ execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 0, 0, 1, 1);
- execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 0, 1, 0, 0);
- execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 0, 1, 1, 0);
- execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 0, 1, 1, 1);
+ execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 0, 1, 0, 0);
+ execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 0, 1, 1, 0);
+ execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 0, 1, 1, 1);
- assertRows(execute("SELECT * FROM %s WHERE a = ? and b = ? and (c, d) = (?, ?)", 0, 1, 0, 0),
- row(0, 1, 0, 0));
+ assertRows(execute("SELECT * FROM %s WHERE a = ? and b = ? and (c, d) = (?, ?)", 0, 1, 0, 0),
+ row(0, 1, 0, 0));
- assertRows(execute("SELECT * FROM %s WHERE a = ? and b IN (?, ?) and (c, d) = (?, ?)", 0, 0, 1, 0, 0),
- row(0, 0, 0, 0),
- row(0, 1, 0, 0));
+ assertRows(execute("SELECT * FROM %s WHERE a = ? and b IN (?, ?) and (c, d) = (?, ?)", 0, 0, 1, 0, 0),
+ row(0, 0, 0, 0),
+ row(0, 1, 0, 0));
- assertRows(execute("SELECT * FROM %s WHERE a = ? and b = ? and (c) IN ((?))", 0, 1, 0),
- row(0, 1, 0, 0));
+ assertRows(execute("SELECT * FROM %s WHERE a = ? and b = ? and (c) IN ((?))", 0, 1, 0),
+ row(0, 1, 0, 0));
- assertRows(execute("SELECT * FROM %s WHERE a = ? and b IN (?, ?) and (c) IN ((?))", 0, 0, 1, 0),
- row(0, 0, 0, 0),
- row(0, 1, 0, 0));
+ assertRows(execute("SELECT * FROM %s WHERE a = ? and b IN (?, ?) and (c) IN ((?))", 0, 0, 1, 0),
+ row(0, 0, 0, 0),
+ row(0, 1, 0, 0));
- assertRows(execute("SELECT * FROM %s WHERE a = ? and b = ? and (c) IN ((?), (?))", 0, 1, 0, 1),
- row(0, 1, 0, 0),
- row(0, 1, 1, 0),
- row(0, 1, 1, 1));
+ assertRows(execute("SELECT * FROM %s WHERE a = ? and b = ? and (c) IN ((?), (?))", 0, 1, 0, 1),
+ row(0, 1, 0, 0),
+ row(0, 1, 1, 0),
+ row(0, 1, 1, 1));
- assertRows(execute("SELECT * FROM %s WHERE a = ? and b = ? and (c, d) IN ((?, ?))", 0, 1, 0, 0),
- row(0, 1, 0, 0));
+ assertRows(execute("SELECT * FROM %s WHERE a = ? and b = ? and (c, d) IN ((?, ?))", 0, 1, 0, 0),
+ row(0, 1, 0, 0));
- assertRows(execute("SELECT * FROM %s WHERE a = ? and b = ? and (c, d) IN ((?, ?), (?, ?))", 0, 1, 0, 0, 1, 1),
- row(0, 1, 0, 0),
- row(0, 1, 1, 1));
+ assertRows(execute("SELECT * FROM %s WHERE a = ? and b = ? and (c, d) IN ((?, ?), (?, ?))", 0, 1, 0, 0, 1, 1),
+ row(0, 1, 0, 0),
+ row(0, 1, 1, 1));
- assertRows(execute("SELECT * FROM %s WHERE a = ? and b IN (?, ?) and (c, d) IN ((?, ?), (?, ?))", 0, 0, 1, 0, 0, 1, 1),
- row(0, 0, 0, 0),
- row(0, 0, 1, 1),
- row(0, 1, 0, 0),
- row(0, 1, 1, 1));
+ assertRows(execute("SELECT * FROM %s WHERE a = ? and b IN (?, ?) and (c, d) IN ((?, ?), (?, ?))", 0, 0, 1, 0, 0, 1, 1),
+ row(0, 0, 0, 0),
+ row(0, 0, 1, 1),
+ row(0, 1, 0, 0),
+ row(0, 1, 1, 1));
- assertRows(execute("SELECT * FROM %s WHERE a = ? and b = ? and (c, d) > (?, ?)", 0, 1, 0, 0),
- row(0, 1, 1, 0),
- row(0, 1, 1, 1));
+ assertRows(execute("SELECT * FROM %s WHERE a = ? and b = ? and (c, d) > (?, ?)", 0, 1, 0, 0),
+ row(0, 1, 1, 0),
+ row(0, 1, 1, 1));
- assertRows(execute("SELECT * FROM %s WHERE a = ? and b IN (?, ?) and (c, d) > (?, ?)", 0, 0, 1, 0, 0),
- row(0, 0, 1, 0),
- row(0, 0, 1, 1),
- row(0, 1, 1, 0),
- row(0, 1, 1, 1));
+ assertRows(execute("SELECT * FROM %s WHERE a = ? and b IN (?, ?) and (c, d) > (?, ?)", 0, 0, 1, 0, 0),
+ row(0, 0, 1, 0),
+ row(0, 0, 1, 1),
+ row(0, 1, 1, 0),
+ row(0, 1, 1, 1));
- assertRows(execute("SELECT * FROM %s WHERE a = ? and b = ? and (c, d) > (?, ?) and (c) <= (?) ", 0, 1, 0, 0, 1),
- row(0, 1, 1, 0),
- row(0, 1, 1, 1));
+ assertRows(execute("SELECT * FROM %s WHERE a = ? and b = ? and (c, d) > (?, ?) and (c) <= (?) ", 0, 1, 0, 0, 1),
+ row(0, 1, 1, 0),
+ row(0, 1, 1, 1));
- assertRows(execute("SELECT * FROM %s WHERE a = ? and b = ? and (c, d) > (?, ?) and c <= ? ", 0, 1, 0, 0, 1),
- row(0, 1, 1, 0),
- row(0, 1, 1, 1));
+ assertRows(execute("SELECT * FROM %s WHERE a = ? and b = ? and (c, d) > (?, ?) and c <= ? ", 0, 1, 0, 0, 1),
+ row(0, 1, 1, 0),
+ row(0, 1, 1, 1));
- assertRows(execute("SELECT * FROM %s WHERE a = ? and b = ? and (c, d) >= (?, ?) and (c, d) < (?, ?)", 0, 1, 0, 0, 1, 1),
- row(0, 1, 0, 0),
- row(0, 1, 1, 0));
+ assertRows(execute("SELECT * FROM %s WHERE a = ? and b = ? and (c, d) >= (?, ?) and (c, d) < (?, ?)", 0, 1, 0, 0, 1, 1),
+ row(0, 1, 0, 0),
+ row(0, 1, 1, 0));
- assertRows(execute("SELECT * FROM %s WHERE a = ? and (b, c) = (?, ?) and d = ?", 0, 0, 1, 0),
- row(0, 0, 1, 0));
+ assertRows(execute("SELECT * FROM %s WHERE a = ? and (b, c) = (?, ?) and d = ?", 0, 0, 1, 0),
+ row(0, 0, 1, 0));
- assertRows(execute("SELECT * FROM %s WHERE a = ? and (b, c) IN ((?, ?), (?, ?)) and d = ?", 0, 0, 1, 0, 0, 0),
- row(0, 0, 0, 0),
- row(0, 0, 1, 0));
+ assertRows(execute("SELECT * FROM %s WHERE a = ? and (b, c) IN ((?, ?), (?, ?)) and d = ?", 0, 0, 1, 0, 0, 0),
+ row(0, 0, 0, 0),
+ row(0, 0, 1, 0));
- assertRows(execute("SELECT * FROM %s WHERE a = ? and b = ? and (c) = (?) and d = ?", 0, 0, 1, 0),
- row(0, 0, 1, 0));
+ assertRows(execute("SELECT * FROM %s WHERE a = ? and b = ? and (c) = (?) and d = ?", 0, 0, 1, 0),
+ row(0, 0, 1, 0));
- assertRows(execute("SELECT * FROM %s WHERE a = ? and (b, c) = (?, ?) and d IN (?, ?)", 0, 0, 1, 0, 2),
- row(0, 0, 1, 0));
+ assertRows(execute("SELECT * FROM %s WHERE a = ? and (b, c) = (?, ?) and d IN (?, ?)", 0, 0, 1, 0, 2),
+ row(0, 0, 1, 0));
- assertRows(execute("SELECT * FROM %s WHERE a = ? and b = ? and (c) = (?) and d IN (?, ?)", 0, 0, 1, 0, 2),
- row(0, 0, 1, 0));
+ assertRows(execute("SELECT * FROM %s WHERE a = ? and b = ? and (c) = (?) and d IN (?, ?)", 0, 0, 1, 0, 2),
+ row(0, 0, 1, 0));
- assertRows(execute("SELECT * FROM %s WHERE a = ? and (b, c) = (?, ?) and d >= ?", 0, 0, 1, 0),
- row(0, 0, 1, 0),
- row(0, 0, 1, 1));
+ assertRows(execute("SELECT * FROM %s WHERE a = ? and (b, c) = (?, ?) and d >= ?", 0, 0, 1, 0),
+ row(0, 0, 1, 0),
+ row(0, 0, 1, 1));
- assertRows(execute("SELECT * FROM %s WHERE a = ? and d < 1 and (b, c) = (?, ?) and d >= ?", 0, 0, 1, 0),
- row(0, 0, 1, 0));
+ assertRows(execute("SELECT * FROM %s WHERE a = ? and d < 1 and (b, c) = (?, ?) and d >= ?", 0, 0, 1, 0),
+ row(0, 0, 1, 0));
- assertRows(execute("SELECT * FROM %s WHERE a = ? and d < 1 and (b, c) IN ((?, ?), (?, ?)) and d >= ?", 0, 0, 1, 0, 0, 0),
- row(0, 0, 0, 0),
- row(0, 0, 1, 0));
- }
+ assertRows(execute("SELECT * FROM %s WHERE a = ? and d < 1 and (b, c) IN ((?, ?), (?, ?)) and d >= ?", 0, 0, 1, 0, 0, 0),
+ row(0, 0, 0, 0),
+ row(0, 0, 1, 0));
}
@Test
public void testSeveralMultiColumnRelation() throws Throwable
{
- for (String compactOption : new String[]{"", " WITH COMPACT STORAGE"})
- {
- createTable("CREATE TABLE %s (a int, b int, c int, d int, PRIMARY KEY (a, b, c, d))" + compactOption);
+ createTable("CREATE TABLE %s (a int, b int, c int, d int, PRIMARY KEY (a, b, c, d))");
- execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 0, 0, 0, 0);
- execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 0, 0, 1, 0);
- execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 0, 0, 1, 1);
+ execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 0, 0, 0, 0);
+ execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 0, 0, 1, 0);
+ execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 0, 0, 1, 1);
- execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 0, 1, 0, 0);
- execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 0, 1, 1, 0);
- execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 0, 1, 1, 1);
+ execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 0, 1, 0, 0);
+ execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 0, 1, 1, 0);
+ execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 0, 1, 1, 1);
- assertRows(execute("SELECT * FROM %s WHERE a = ? and (b) = (?) and (c, d) = (?, ?)", 0, 1, 0, 0),
- row(0, 1, 0, 0));
+ assertRows(execute("SELECT * FROM %s WHERE a = ? and (b) = (?) and (c, d) = (?, ?)", 0, 1, 0, 0),
+ row(0, 1, 0, 0));
- assertRows(execute("SELECT * FROM %s WHERE a = ? and (b) IN ((?), (?)) and (c, d) = (?, ?)", 0, 0, 1, 0, 0),
- row(0, 0, 0, 0),
- row(0, 1, 0, 0));
+ assertRows(execute("SELECT * FROM %s WHERE a = ? and (b) IN ((?), (?)) and (c, d) = (?, ?)", 0, 0, 1, 0, 0),
+ row(0, 0, 0, 0),
+ row(0, 1, 0, 0));
- assertRows(execute("SELECT * FROM %s WHERE a = ? and (b) = (?) and (c) IN ((?))", 0, 1, 0),
- row(0, 1, 0, 0));
+ assertRows(execute("SELECT * FROM %s WHERE a = ? and (b) = (?) and (c) IN ((?))", 0, 1, 0),
+ row(0, 1, 0, 0));
- assertRows(execute("SELECT * FROM %s WHERE a = ? and (b) IN ((?),(?)) and (c) IN ((?))", 0, 0, 1, 0),
- row(0, 0, 0, 0),
- row(0, 1, 0, 0));
+ assertRows(execute("SELECT * FROM %s WHERE a = ? and (b) IN ((?),(?)) and (c) IN ((?))", 0, 0, 1, 0),
+ row(0, 0, 0, 0),
+ row(0, 1, 0, 0));
- assertRows(execute("SELECT * FROM %s WHERE a = ? and (b) = (?) and (c) IN ((?), (?))", 0, 1, 0, 1),
- row(0, 1, 0, 0),
- row(0, 1, 1, 0),
- row(0, 1, 1, 1));
+ assertRows(execute("SELECT * FROM %s WHERE a = ? and (b) = (?) and (c) IN ((?), (?))", 0, 1, 0, 1),
+ row(0, 1, 0, 0),
+ row(0, 1, 1, 0),
+ row(0, 1, 1, 1));
- assertRows(execute("SELECT * FROM %s WHERE a = ? and (b) = (?) and (c, d) IN ((?, ?))", 0, 1, 0, 0),
- row(0, 1, 0, 0));
+ assertRows(execute("SELECT * FROM %s WHERE a = ? and (b) = (?) and (c, d) IN ((?, ?))", 0, 1, 0, 0),
+ row(0, 1, 0, 0));
- assertRows(execute("SELECT * FROM %s WHERE a = ? and (b) = (?) and (c, d) IN ((?, ?), (?, ?))", 0, 1, 0, 0, 1, 1),
- row(0, 1, 0, 0),
- row(0, 1, 1, 1));
+ assertRows(execute("SELECT * FROM %s WHERE a = ? and (b) = (?) and (c, d) IN ((?, ?), (?, ?))", 0, 1, 0, 0, 1, 1),
+ row(0, 1, 0, 0),
+ row(0, 1, 1, 1));
- assertRows(execute("SELECT * FROM %s WHERE a = ? and (b) IN ((?), (?)) and (c, d) IN ((?, ?), (?, ?))", 0, 0, 1, 0, 0, 1, 1),
- row(0, 0, 0, 0),
- row(0, 0, 1, 1),
- row(0, 1, 0, 0),
- row(0, 1, 1, 1));
+ assertRows(execute("SELECT * FROM %s WHERE a = ? and (b) IN ((?), (?)) and (c, d) IN ((?, ?), (?, ?))", 0, 0, 1, 0, 0, 1, 1),
+ row(0, 0, 0, 0),
+ row(0, 0, 1, 1),
+ row(0, 1, 0, 0),
+ row(0, 1, 1, 1));
- assertRows(execute("SELECT * FROM %s WHERE a = ? and (b) = (?) and (c, d) > (?, ?)", 0, 1, 0, 0),
- row(0, 1, 1, 0),
- row(0, 1, 1, 1));
+ assertRows(execute("SELECT * FROM %s WHERE a = ? and (b) = (?) and (c, d) > (?, ?)", 0, 1, 0, 0),
+ row(0, 1, 1, 0),
+ row(0, 1, 1, 1));
- assertRows(execute("SELECT * FROM %s WHERE a = ? and (b) IN ((?),(?)) and (c, d) > (?, ?)", 0, 0, 1, 0, 0),
- row(0, 0, 1, 0),
- row(0, 0, 1, 1),
- row(0, 1, 1, 0),
- row(0, 1, 1, 1));
+ assertRows(execute("SELECT * FROM %s WHERE a = ? and (b) IN ((?),(?)) and (c, d) > (?, ?)", 0, 0, 1, 0, 0),
+ row(0, 0, 1, 0),
+ row(0, 0, 1, 1),
+ row(0, 1, 1, 0),
+ row(0, 1, 1, 1));
- assertRows(execute("SELECT * FROM %s WHERE a = ? and (b) = (?) and (c, d) > (?, ?) and (c) <= (?) ", 0, 1, 0, 0, 1),
- row(0, 1, 1, 0),
- row(0, 1, 1, 1));
+ assertRows(execute("SELECT * FROM %s WHERE a = ? and (b) = (?) and (c, d) > (?, ?) and (c) <= (?) ", 0, 1, 0, 0, 1),
+ row(0, 1, 1, 0),
+ row(0, 1, 1, 1));
- assertRows(execute("SELECT * FROM %s WHERE a = ? and (b) = (?) and (c, d) > (?, ?) and c <= ? ", 0, 1, 0, 0, 1),
- row(0, 1, 1, 0),
- row(0, 1, 1, 1));
+ assertRows(execute("SELECT * FROM %s WHERE a = ? and (b) = (?) and (c, d) > (?, ?) and c <= ? ", 0, 1, 0, 0, 1),
+ row(0, 1, 1, 0),
+ row(0, 1, 1, 1));
- assertRows(execute("SELECT * FROM %s WHERE a = ? and (b) = (?) and (c, d) >= (?, ?) and (c, d) < (?, ?)", 0, 1, 0, 0, 1, 1),
- row(0, 1, 0, 0),
- row(0, 1, 1, 0));
+ assertRows(execute("SELECT * FROM %s WHERE a = ? and (b) = (?) and (c, d) >= (?, ?) and (c, d) < (?, ?)", 0, 1, 0, 0, 1, 1),
+ row(0, 1, 0, 0),
+ row(0, 1, 1, 0));
- assertRows(execute("SELECT * FROM %s WHERE a = ? and (b, c) = (?, ?) and d = ?", 0, 0, 1, 0),
- row(0, 0, 1, 0));
+ assertRows(execute("SELECT * FROM %s WHERE a = ? and (b, c) = (?, ?) and d = ?", 0, 0, 1, 0),
+ row(0, 0, 1, 0));
- assertRows(execute("SELECT * FROM %s WHERE a = ? and (b, c) IN ((?, ?), (?, ?)) and d = ?", 0, 0, 1, 0, 0, 0),
- row(0, 0, 0, 0),
- row(0, 0, 1, 0));
+ assertRows(execute("SELECT * FROM %s WHERE a = ? and (b, c) IN ((?, ?), (?, ?)) and d = ?", 0, 0, 1, 0, 0, 0),
+ row(0, 0, 0, 0),
+ row(0, 0, 1, 0));
- assertRows(execute("SELECT * FROM %s WHERE a = ? and (d) < (1) and (b, c) = (?, ?) and (d) >= (?)", 0, 0, 1, 0),
- row(0, 0, 1, 0));
+ assertRows(execute("SELECT * FROM %s WHERE a = ? and (d) < (1) and (b, c) = (?, ?) and (d) >= (?)", 0, 0, 1, 0),
+ row(0, 0, 1, 0));
- assertRows(execute("SELECT * FROM %s WHERE a = ? and (d) < (1) and (b, c) IN ((?, ?), (?, ?)) and (d) >= (?)", 0, 0, 1, 0, 0, 0),
- row(0, 0, 0, 0),
- row(0, 0, 1, 0));
- }
+ assertRows(execute("SELECT * FROM %s WHERE a = ? and (d) < (1) and (b, c) IN ((?, ?), (?, ?)) and (d) >= (?)", 0, 0, 1, 0, 0, 0),
+ row(0, 0, 0, 0),
+ row(0, 0, 1, 0));
}
@Test
public void testSinglePartitionInvalidQueries() throws Throwable
{
- for (String compactOption : new String[]{"", " WITH COMPACT STORAGE"})
- {
- createTable("CREATE TABLE %s (a int PRIMARY KEY, b int)" + compactOption);
- assertInvalidMessage("Multi-column relations can only be applied to clustering columns but was applied to: a",
- "SELECT * FROM %s WHERE (a) > (?)", 0);
- assertInvalidMessage("Multi-column relations can only be applied to clustering columns but was applied to: a",
- "SELECT * FROM %s WHERE (a) = (?)", 0);
- assertInvalidMessage("Multi-column relations can only be applied to clustering columns but was applied to: b",
- "SELECT * FROM %s WHERE (b) = (?)", 0);
- }
+ createTable("CREATE TABLE %s (a int PRIMARY KEY, b int)");
+ assertInvalidMessage("Multi-column relations can only be applied to clustering columns but was applied to: a",
+ "SELECT * FROM %s WHERE (a) > (?)", 0);
+ assertInvalidMessage("Multi-column relations can only be applied to clustering columns but was applied to: a",
+ "SELECT * FROM %s WHERE (a) = (?)", 0);
+ assertInvalidMessage("Multi-column relations can only be applied to clustering columns but was applied to: b",
+ "SELECT * FROM %s WHERE (b) = (?)", 0);
}
@Test
public void testSingleClustering() throws Throwable
{
- for (String compactOption : new String[]{"", " WITH COMPACT STORAGE"})
- {
- createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY (a, b))" + compactOption);
+ createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY (a, b))");
- execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", 0, 0, 0);
- execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", 0, 1, 0);
- execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", 0, 2, 0);
+ execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", 0, 0, 0);
+ execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", 0, 1, 0);
+ execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", 0, 2, 0);
- // Equalities
+ // Equalities
- assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b) = (?)", 0, 1),
- row(0, 1, 0)
- );
+ assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b) = (?)", 0, 1),
+ row(0, 1, 0)
+ );
- // Same but check the whole tuple can be prepared
- assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b) = ?", 0, tuple(1)),
- row(0, 1, 0)
- );
+ // Same but check the whole tuple can be prepared
+ assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b) = ?", 0, tuple(1)),
+ row(0, 1, 0)
+ );
- assertEmpty(execute("SELECT * FROM %s WHERE a = ? AND (b) = (?)", 0, 3));
+ assertEmpty(execute("SELECT * FROM %s WHERE a = ? AND (b) = (?)", 0, 3));
- // Inequalities
+ // Inequalities
- assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b) > (?)", 0, 0),
- row(0, 1, 0),
- row(0, 2, 0)
- );
+ assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b) > (?)", 0, 0),
+ row(0, 1, 0),
+ row(0, 2, 0)
+ );
- assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b) >= (?)", 0, 1),
- row(0, 1, 0),
- row(0, 2, 0)
- );
+ assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b) >= (?)", 0, 1),
+ row(0, 1, 0),
+ row(0, 2, 0)
+ );
- assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b) < (?)", 0, 2),
- row(0, 0, 0),
- row(0, 1, 0)
- );
+ assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b) < (?)", 0, 2),
+ row(0, 0, 0),
+ row(0, 1, 0)
+ );
- assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b) <= (?)", 0, 1),
- row(0, 0, 0),
- row(0, 1, 0)
- );
+ assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b) <= (?)", 0, 1),
+ row(0, 0, 0),
+ row(0, 1, 0)
+ );
- assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b) > (?) AND (b) < (?)", 0, 0, 2),
- row(0, 1, 0)
- );
+ assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b) > (?) AND (b) < (?)", 0, 0, 2),
+ row(0, 1, 0)
+ );
- assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b) > (?) AND b < ?", 0, 0, 2),
- row(0, 1, 0)
- );
+ assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b) > (?) AND b < ?", 0, 0, 2),
+ row(0, 1, 0)
+ );
- assertRows(execute("SELECT * FROM %s WHERE a = ? AND b > ? AND (b) < (?)", 0, 0, 2),
- row(0, 1, 0)
- );
- }
+ assertRows(execute("SELECT * FROM %s WHERE a = ? AND b > ? AND (b) < (?)", 0, 0, 2),
+ row(0, 1, 0)
+ );
}
@Test
public void testNonEqualsRelation() throws Throwable
{
- for (String compactOption : new String[]{"", " WITH COMPACT STORAGE"})
- {
- createTable("CREATE TABLE %s (a int PRIMARY KEY, b int)" + compactOption);
- assertInvalidMessage("Unsupported \"!=\" relation: (b) != (0)",
- "SELECT * FROM %s WHERE a = 0 AND (b) != (0)");
- }
+ createTable("CREATE TABLE %s (a int PRIMARY KEY, b int)");
+ assertInvalidMessage("Unsupported \"!=\" relation: (b) != (0)",
+ "SELECT * FROM %s WHERE a = 0 AND (b) != (0)");
}
@Test
public void testMultipleClustering() throws Throwable
{
- for (String compactOption : new String[]{"", " WITH COMPACT STORAGE"})
- {
- createTable("CREATE TABLE %s (a int, b int, c int, d int, PRIMARY KEY (a, b, c, d))" + compactOption);
+ createTable("CREATE TABLE %s (a int, b int, c int, d int, PRIMARY KEY (a, b, c, d))");
- execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 0, 0, 0, 0);
- execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 0, 0, 1, 0);
- execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 0, 0, 1, 1);
+ execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 0, 0, 0, 0);
+ execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 0, 0, 1, 0);
+ execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 0, 0, 1, 1);
- execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 0, 1, 0, 0);
- execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 0, 1, 1, 0);
- execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 0, 1, 1, 1);
+ execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 0, 1, 0, 0);
+ execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 0, 1, 1, 0);
+ execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 0, 1, 1, 1);
- // Empty query
- assertEmpty(execute("SELECT * FROM %s WHERE a = 0 AND (b, c, d) IN ()"));
+ // Empty query
+ assertEmpty(execute("SELECT * FROM %s WHERE a = 0 AND (b, c, d) IN ()"));
- // Equalities
+ // Equalities
- assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b) = (?)", 0, 1),
- row(0, 1, 0, 0),
- row(0, 1, 1, 0),
- row(0, 1, 1, 1)
- );
+ assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b) = (?)", 0, 1),
+ row(0, 1, 0, 0),
+ row(0, 1, 1, 0),
+ row(0, 1, 1, 1)
+ );
- // Same with whole tuple prepared
- assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b) = ?", 0, tuple(1)),
- row(0, 1, 0, 0),
- row(0, 1, 1, 0),
- row(0, 1, 1, 1)
- );
+ // Same with whole tuple prepared
+ assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b) = ?", 0, tuple(1)),
+ row(0, 1, 0, 0),
+ row(0, 1, 1, 0),
+ row(0, 1, 1, 1)
+ );
- assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b, c) = (?, ?)", 0, 1, 1),
- row(0, 1, 1, 0),
- row(0, 1, 1, 1)
- );
+ assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b, c) = (?, ?)", 0, 1, 1),
+ row(0, 1, 1, 0),
+ row(0, 1, 1, 1)
+ );
- // Same with whole tuple prepared
- assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b, c) = ?", 0, tuple(1, 1)),
- row(0, 1, 1, 0),
- row(0, 1, 1, 1)
- );
+ // Same with whole tuple prepared
+ assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b, c) = ?", 0, tuple(1, 1)),
+ row(0, 1, 1, 0),
+ row(0, 1, 1, 1)
+ );
- assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b, c, d) = (?, ?, ?)", 0, 1, 1, 1),
- row(0, 1, 1, 1)
- );
+ assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b, c, d) = (?, ?, ?)", 0, 1, 1, 1),
+ row(0, 1, 1, 1)
+ );
- // Same with whole tuple prepared
- assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b, c, d) = ?", 0, tuple(1, 1, 1)),
- row(0, 1, 1, 1)
- );
+ // Same with whole tuple prepared
+ assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b, c, d) = ?", 0, tuple(1, 1, 1)),
+ row(0, 1, 1, 1)
+ );
- // Inequalities
+ // Inequalities
- assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b) > (?)", 0, 0),
- row(0, 1, 0, 0),
- row(0, 1, 1, 0),
- row(0, 1, 1, 1)
- );
+ assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b) > (?)", 0, 0),
+ row(0, 1, 0, 0),
+ row(0, 1, 1, 0),
+ row(0, 1, 1, 1)
+ );
- assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b) >= (?)", 0, 0),
- row(0, 0, 0, 0),
- row(0, 0, 1, 0),
- row(0, 0, 1, 1),
- row(0, 1, 0, 0),
- row(0, 1, 1, 0),
- row(0, 1, 1, 1)
- );
+ assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b) >= (?)", 0, 0),
+ row(0, 0, 0, 0),
+ row(0, 0, 1, 0),
+ row(0, 0, 1, 1),
+ row(0, 1, 0, 0),
+ row(0, 1, 1, 0),
+ row(0, 1, 1, 1)
+ );
- assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b, c) > (?, ?)", 0, 1, 0),
- row(0, 1, 1, 0),
- row(0, 1, 1, 1)
- );
+ assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b, c) > (?, ?)", 0, 1, 0),
+ row(0, 1, 1, 0),
+ row(0, 1, 1, 1)
+ );
- assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b, c) >= (?, ?)", 0, 1, 0),
- row(0, 1, 0, 0),
- row(0, 1, 1, 0),
- row(0, 1, 1, 1)
- );
+ assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b, c) >= (?, ?)", 0, 1, 0),
+ row(0, 1, 0, 0),
+ row(0, 1, 1, 0),
+ row(0, 1, 1, 1)
+ );
- assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b, c, d) > (?, ?, ?)", 0, 1, 1, 0),
- row(0, 1, 1, 1)
- );
+ assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b, c, d) > (?, ?, ?)", 0, 1, 1, 0),
+ row(0, 1, 1, 1)
+ );
- assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b, c, d) >= (?, ?, ?)", 0, 1, 1, 0),
- row(0, 1, 1, 0),
- row(0, 1, 1, 1)
- );
+ assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b, c, d) >= (?, ?, ?)", 0, 1, 1, 0),
+ row(0, 1, 1, 0),
+ row(0, 1, 1, 1)
+ );
- assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b) < (?)", 0, 1),
- row(0, 0, 0, 0),
- row(0, 0, 1, 0),
- row(0, 0, 1, 1)
- );
+ assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b) < (?)", 0, 1),
+ row(0, 0, 0, 0),
+ row(0, 0, 1, 0),
+ row(0, 0, 1, 1)
+ );
- assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b) <= (?)", 0, 1),
- row(0, 0, 0, 0),
- row(0, 0, 1, 0),
- row(0, 0, 1, 1),
- row(0, 1, 0, 0),
- row(0, 1, 1, 0),
- row(0, 1, 1, 1)
- );
+ assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b) <= (?)", 0, 1),
+ row(0, 0, 0, 0),
+ row(0, 0, 1, 0),
+ row(0, 0, 1, 1),
+ row(0, 1, 0, 0),
+ row(0, 1, 1, 0),
+ row(0, 1, 1, 1)
+ );
- assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b, c) < (?, ?)", 0, 0, 1),
- row(0, 0, 0, 0)
- );
+ assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b, c) < (?, ?)", 0, 0, 1),
+ row(0, 0, 0, 0)
+ );
- assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b, c) <= (?, ?)", 0, 0, 1),
- row(0, 0, 0, 0),
- row(0, 0, 1, 0),
- row(0, 0, 1, 1)
- );
+ assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b, c) <= (?, ?)", 0, 0, 1),
+ row(0, 0, 0, 0),
+ row(0, 0, 1, 0),
+ row(0, 0, 1, 1)
+ );
- assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b, c, d) < (?, ?, ?)", 0, 0, 1, 1),
- row(0, 0, 0, 0),
- row(0, 0, 1, 0)
- );
+ assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b, c, d) < (?, ?, ?)", 0, 0, 1, 1),
+ row(0, 0, 0, 0),
+ row(0, 0, 1, 0)
+ );
- assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b, c, d) <= (?, ?, ?)", 0, 0, 1, 1),
- row(0, 0, 0, 0),
- row(0, 0, 1, 0),
- row(0, 0, 1, 1)
- );
+ assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b, c, d) <= (?, ?, ?)", 0, 0, 1, 1),
+ row(0, 0, 0, 0),
+ row(0, 0, 1, 0),
+ row(0, 0, 1, 1)
+ );
- assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b, c, d) > (?, ?, ?) AND (b) < (?)", 0, 0, 1, 0, 1),
- row(0, 0, 1, 1)
- );
+ assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b, c, d) > (?, ?, ?) AND (b) < (?)", 0, 0, 1, 0, 1),
+ row(0, 0, 1, 1)
+ );
- assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b, c, d) > (?, ?, ?) AND b < ?", 0, 0, 1, 0, 1),
- row(0, 0, 1, 1)
- );
+ assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b, c, d) > (?, ?, ?) AND b < ?", 0, 0, 1, 0, 1),
+ row(0, 0, 1, 1)
+ );
- assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b, c, d) > (?, ?, ?) AND (b, c) < (?, ?)", 0, 0, 1, 1, 1, 1),
- row(0, 1, 0, 0)
- );
+ assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b, c, d) > (?, ?, ?) AND (b, c) < (?, ?)", 0, 0, 1, 1, 1, 1),
+ row(0, 1, 0, 0)
+ );
- assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b, c, d) > (?, ?, ?) AND (b, c, d) < (?, ?, ?)", 0, 0, 1, 1, 1, 1, 0),
- row(0, 1, 0, 0)
- );
+ assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b, c, d) > (?, ?, ?) AND (b, c, d) < (?, ?, ?)", 0, 0, 1, 1, 1, 1, 0),
+ row(0, 1, 0, 0)
+ );
- // Same with whole tuple prepared
- assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b, c, d) > ? AND (b, c, d) < ?", 0, tuple(0, 1, 1), tuple(1, 1, 0)),
- row(0, 1, 0, 0)
- );
+ // Same with whole tuple prepared
+ assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b, c, d) > ? AND (b, c, d) < ?", 0, tuple(0, 1, 1), tuple(1, 1, 0)),
+ row(0, 1, 0, 0)
+ );
- // reversed
- assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b) > (?) ORDER BY b DESC, c DESC, d DESC", 0, 0),
- row(0, 1, 1, 1),
- row(0, 1, 1, 0),
- row(0, 1, 0, 0)
- );
+ // reversed
+ assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b) > (?) ORDER BY b DESC, c DESC, d DESC", 0, 0),
+ row(0, 1, 1, 1),
+ row(0, 1, 1, 0),
+ row(0, 1, 0, 0)
+ );
- assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b) >= (?) ORDER BY b DESC, c DESC, d DESC", 0, 0),
- row(0, 1, 1, 1),
- row(0, 1, 1, 0),
- row(0, 1, 0, 0),
- row(0, 0, 1, 1),
- row(0, 0, 1, 0),
- row(0, 0, 0, 0)
- );
+ assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b) >= (?) ORDER BY b DESC, c DESC, d DESC", 0, 0),
+ row(0, 1, 1, 1),
+ row(0, 1, 1, 0),
+ row(0, 1, 0, 0),
+ row(0, 0, 1, 1),
+ row(0, 0, 1, 0),
+ row(0, 0, 0, 0)
+ );
- assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b, c) > (?, ?) ORDER BY b DESC, c DESC, d DESC", 0, 1, 0),
- row(0, 1, 1, 1),
- row(0, 1, 1, 0)
- );
+ assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b, c) > (?, ?) ORDER BY b DESC, c DESC, d DESC", 0, 1, 0),
+ row(0, 1, 1, 1),
+ row(0, 1, 1, 0)
+ );
- assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b, c) >= (?, ?) ORDER BY b DESC, c DESC, d DESC", 0, 1, 0),
- row(0, 1, 1, 1),
- row(0, 1, 1, 0),
- row(0, 1, 0, 0)
- );
+ assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b, c) >= (?, ?) ORDER BY b DESC, c DESC, d DESC", 0, 1, 0),
+ row(0, 1, 1, 1),
+ row(0, 1, 1, 0),
+ row(0, 1, 0, 0)
+ );
- assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b, c, d) > (?, ?, ?) ORDER BY b DESC, c DESC, d DESC", 0, 1, 1, 0),
- row(0, 1, 1, 1)
- );
+ assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b, c, d) > (?, ?, ?) ORDER BY b DESC, c DESC, d DESC", 0, 1, 1, 0),
+ row(0, 1, 1, 1)
+ );
- assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b, c, d) >= (?, ?, ?) ORDER BY b DESC, c DESC, d DESC", 0, 1, 1, 0),
- row(0, 1, 1, 1),
- row(0, 1, 1, 0)
- );
+ assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b, c, d) >= (?, ?, ?) ORDER BY b DESC, c DESC, d DESC", 0, 1, 1, 0),
+ row(0, 1, 1, 1),
+ row(0, 1, 1, 0)
+ );
- assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b) < (?) ORDER BY b DESC, c DESC, d DESC", 0, 1),
- row(0, 0, 1, 1),
- row(0, 0, 1, 0),
- row(0, 0, 0, 0)
- );
+ assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b) < (?) ORDER BY b DESC, c DESC, d DESC", 0, 1),
+ row(0, 0, 1, 1),
+ row(0, 0, 1, 0),
+ row(0, 0, 0, 0)
+ );
- assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b) <= (?) ORDER BY b DESC, c DESC, d DESC", 0, 1),
- row(0, 1, 1, 1),
- row(0, 1, 1, 0),
- row(0, 1, 0, 0),
- row(0, 0, 1, 1),
- row(0, 0, 1, 0),
- row(0, 0, 0, 0)
- );
+ assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b) <= (?) ORDER BY b DESC, c DESC, d DESC", 0, 1),
+ row(0, 1, 1, 1),
+ row(0, 1, 1, 0),
+ row(0, 1, 0, 0),
+ row(0, 0, 1, 1),
+ row(0, 0, 1, 0),
+ row(0, 0, 0, 0)
+ );
- assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b, c) < (?, ?) ORDER BY b DESC, c DESC, d DESC", 0, 0, 1),
- row(0, 0, 0, 0)
- );
+ assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b, c) < (?, ?) ORDER BY b DESC, c DESC, d DESC", 0, 0, 1),
+ row(0, 0, 0, 0)
+ );
- assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b, c) <= (?, ?) ORDER BY b DESC, c DESC, d DESC", 0, 0, 1),
- row(0, 0, 1, 1),
- row(0, 0, 1, 0),
- row(0, 0, 0, 0)
- );
+ assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b, c) <= (?, ?) ORDER BY b DESC, c DESC, d DESC", 0, 0, 1),
+ row(0, 0, 1, 1),
+ row(0, 0, 1, 0),
+ row(0, 0, 0, 0)
+ );
- assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b, c, d) < (?, ?, ?) ORDER BY b DESC, c DESC, d DESC", 0, 0, 1, 1),
- row(0, 0, 1, 0),
- row(0, 0, 0, 0)
- );
+ assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b, c, d) < (?, ?, ?) ORDER BY b DESC, c DESC, d DESC", 0, 0, 1, 1),
+ row(0, 0, 1, 0),
+ row(0, 0, 0, 0)
+ );
- assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b, c, d) <= (?, ?, ?) ORDER BY b DESC, c DESC, d DESC", 0, 0, 1, 1),
- row(0, 0, 1, 1),
- row(0, 0, 1, 0),
- row(0, 0, 0, 0)
- );
+ assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b, c, d) <= (?, ?, ?) ORDER BY b DESC, c DESC, d DESC", 0, 0, 1, 1),
+ row(0, 0, 1, 1),
+ row(0, 0, 1, 0),
+ row(0, 0, 0, 0)
+ );
- assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b, c, d) > (?, ?, ?) AND (b) < (?) ORDER BY b DESC, c DESC, d DESC", 0, 0, 1, 0, 1),
- row(0, 0, 1, 1)
- );
+ assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b, c, d) > (?, ?, ?) AND (b) < (?) ORDER BY b DESC, c DESC, d DESC", 0, 0, 1, 0, 1),
+ row(0, 0, 1, 1)
+ );
- assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b, c, d) > (?, ?, ?) AND b < ? ORDER BY b DESC, c DESC, d DESC", 0, 0, 1, 0, 1),
- row(0, 0, 1, 1)
- );
+ assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b, c, d) > (?, ?, ?) AND b < ? ORDER BY b DESC, c DESC, d DESC", 0, 0, 1, 0, 1),
+ row(0, 0, 1, 1)
+ );
- assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b, c, d) > (?, ?, ?) AND (b, c) < (?, ?) ORDER BY b DESC, c DESC, d DESC", 0, 0, 1, 1, 1, 1),
- row(0, 1, 0, 0)
- );
+ assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b, c, d) > (?, ?, ?) AND (b, c) < (?, ?) ORDER BY b DESC, c DESC, d DESC", 0, 0, 1, 1, 1, 1),
+ row(0, 1, 0, 0)
+ );
- assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b, c, d) > (?, ?, ?) AND (b, c, d) < (?, ?, ?) ORDER BY b DESC, c DESC, d DESC", 0, 0, 1, 1, 1, 1, 0),
- row(0, 1, 0, 0)
- );
+ assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b, c, d) > (?, ?, ?) AND (b, c, d) < (?, ?, ?) ORDER BY b DESC, c DESC, d DESC", 0, 0, 1, 1, 1, 1, 0),
+ row(0, 1, 0, 0)
+ );
- // IN
+ // IN
- assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b, c, d) IN ((?, ?, ?), (?, ?, ?))", 0, 0, 1, 0, 0, 1, 1),
- row(0, 0, 1, 0),
- row(0, 0, 1, 1)
- );
+ assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b, c, d) IN ((?, ?, ?), (?, ?, ?))", 0, 0, 1, 0, 0, 1, 1),
+ row(0, 0, 1, 0),
+ row(0, 0, 1, 1)
+ );
- // same query but with whole tuple prepared
- assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b, c, d) IN (?, ?)", 0, tuple(0, 1, 0), tuple(0, 1, 1)),
- row(0, 0, 1, 0),
- row(0, 0, 1, 1)
- );
+ // same query but with whole tuple prepared
+ assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b, c, d) IN (?, ?)", 0, tuple(0, 1, 0), tuple(0, 1, 1)),
+ row(0, 0, 1, 0),
+ row(0, 0, 1, 1)
+ );
- // same query but with whole IN list prepared
- assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b, c, d) IN ?", 0, list(tuple(0, 1, 0), tuple(0, 1, 1))),
- row(0, 0, 1, 0),
- row(0, 0, 1, 1)
- );
+ // same query but with whole IN list prepared
+ assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b, c, d) IN ?", 0, list(tuple(0, 1, 0), tuple(0, 1, 1))),
+ row(0, 0, 1, 0),
+ row(0, 0, 1, 1)
+ );
- // same query, but reversed order for the IN values
- assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b, c, d) IN (?, ?)", 0, tuple(0, 1, 1), tuple(0, 1, 0)),
- row(0, 0, 1, 0),
- row(0, 0, 1, 1)
- );
+ // same query, but reversed order for the IN values
+ assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b, c, d) IN (?, ?)", 0, tuple(0, 1, 1), tuple(0, 1, 0)),
+ row(0, 0, 1, 0),
+ row(0, 0, 1, 1)
+ );
- assertRows(execute("SELECT * FROM %s WHERE a = ? and (b, c) IN ((?, ?))", 0, 0, 1),
- row(0, 0, 1, 0),
- row(0, 0, 1, 1)
- );
+ assertRows(execute("SELECT * FROM %s WHERE a = ? and (b, c) IN ((?, ?))", 0, 0, 1),
+ row(0, 0, 1, 0),
+ row(0, 0, 1, 1)
+ );
- assertRows(execute("SELECT * FROM %s WHERE a = ? and (b) IN ((?))", 0, 0),
- row(0, 0, 0, 0),
- row(0, 0, 1, 0),
- row(0, 0, 1, 1)
- );
+ assertRows(execute("SELECT * FROM %s WHERE a = ? and (b) IN ((?))", 0, 0),
+ row(0, 0, 0, 0),
+ row(0, 0, 1, 0),
+ row(0, 0, 1, 1)
+ );
- assertEmpty(execute("SELECT * FROM %s WHERE a = ? and (b) IN ()", 0));
+ assertEmpty(execute("SELECT * FROM %s WHERE a = ? and (b) IN ()", 0));
- assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b, c) IN ((?, ?)) ORDER BY b DESC, c DESC, d DESC", 0, 0, 1),
- row(0, 0, 1, 1),
- row(0, 0, 1, 0)
- );
+ assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b, c) IN ((?, ?)) ORDER BY b DESC, c DESC, d DESC", 0, 0, 1),
+ row(0, 0, 1, 1),
+ row(0, 0, 1, 0)
+ );
- assertEmpty(execute("SELECT * FROM %s WHERE a = ? AND (b, c) IN () ORDER BY b DESC, c DESC, d DESC", 0));
+ assertEmpty(execute("SELECT * FROM %s WHERE a = ? AND (b, c) IN () ORDER BY b DESC, c DESC, d DESC", 0));
- // IN on both partition key and clustering key
- execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 1, 0, 0, 0);
- execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 1, 0, 1, 0);
- execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 1, 0, 1, 1);
+ // IN on both partition key and clustering key
+ execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 1, 0, 0, 0);
+ execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 1, 0, 1, 0);
+ execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 1, 0, 1, 1);
- assertRows(execute("SELECT * FROM %s WHERE a IN (?, ?) AND (b, c, d) IN (?, ?)", 0, 1, tuple(0, 1, 0), tuple(0, 1, 1)),
- row(0, 0, 1, 0),
- row(0, 0, 1, 1),
- row(1, 0, 1, 0),
- row(1, 0, 1, 1)
- );
+ assertRows(execute("SELECT * FROM %s WHERE a IN (?, ?) AND (b, c, d) IN (?, ?)", 0, 1, tuple(0, 1, 0), tuple(0, 1, 1)),
+ row(0, 0, 1, 0),
+ row(0, 0, 1, 1),
+ row(1, 0, 1, 0),
+ row(1, 0, 1, 1)
+ );
- // same but with whole IN lists prepared
- assertRows(execute("SELECT * FROM %s WHERE a IN ? AND (b, c, d) IN ?", list(0, 1), list(tuple(0, 1, 0), tuple(0, 1, 1))),
- row(0, 0, 1, 0),
- row(0, 0, 1, 1),
- row(1, 0, 1, 0),
- row(1, 0, 1, 1)
- );
+ // same but with whole IN lists prepared
+ assertRows(execute("SELECT * FROM %s WHERE a IN ? AND (b, c, d) IN ?", list(0, 1), list(tuple(0, 1, 0), tuple(0, 1, 1))),
+ row(0, 0, 1, 0),
+ row(0, 0, 1, 1),
+ row(1, 0, 1, 0),
+ row(1, 0, 1, 1)
+ );
- // same query, but reversed order for the IN values
- assertRows(execute("SELECT * FROM %s WHERE a IN (?, ?) AND (b, c, d) IN (?, ?)", 1, 0, tuple(0, 1, 1), tuple(0, 1, 0)),
- row(0, 0, 1, 0),
- row(0, 0, 1, 1),
- row(1, 0, 1, 0),
- row(1, 0, 1, 1)
- );
+ // same query, but reversed order for the IN values
+ assertRows(execute("SELECT * FROM %s WHERE a IN (?, ?) AND (b, c, d) IN (?, ?)", 1, 0, tuple(0, 1, 1), tuple(0, 1, 0)),
+ row(0, 0, 1, 0),
+ row(0, 0, 1, 1),
+ row(1, 0, 1, 0),
+ row(1, 0, 1, 1)
+ );
- assertRows(execute("SELECT * FROM %s WHERE a IN (?, ?) and (b, c) IN ((?, ?))", 0, 1, 0, 1),
- row(0, 0, 1, 0),
- row(0, 0, 1, 1),
- row(1, 0, 1, 0),
- row(1, 0, 1, 1)
- );
+ assertRows(execute("SELECT * FROM %s WHERE a IN (?, ?) and (b, c) IN ((?, ?))", 0, 1, 0, 1),
+ row(0, 0, 1, 0),
+ row(0, 0, 1, 1),
+ row(1, 0, 1, 0),
+ row(1, 0, 1, 1)
+ );
- assertRows(execute("SELECT * FROM %s WHERE a IN (?, ?) and (b) IN ((?))", 0, 1, 0),
- row(0, 0, 0, 0),
- row(0, 0, 1, 0),
- row(0, 0, 1, 1),
- row(1, 0, 0, 0),
- row(1, 0, 1, 0),
- row(1, 0, 1, 1)
- );
- }
+ assertRows(execute("SELECT * FROM %s WHERE a IN (?, ?) and (b) IN ((?))", 0, 1, 0),
+ row(0, 0, 0, 0),
+ row(0, 0, 1, 0),
+ row(0, 0, 1, 1),
+ row(1, 0, 0, 0),
+ row(1, 0, 1, 0),
+ row(1, 0, 1, 1)
+ );
}
@Test
public void testMultipleClusteringReversedComponents() throws Throwable
{
- for (String compactOption : new String[]{"", " COMPACT STORAGE AND"})
- {
- createTable("CREATE TABLE %s (a int, b int, c int, d int, PRIMARY KEY (a, b, c, d)) WITH" + compactOption + " CLUSTERING ORDER BY (b DESC, c ASC, d DESC)");
+ createTable("CREATE TABLE %s (a int, b int, c int, d int, PRIMARY KEY (a, b, c, d)) WITH CLUSTERING ORDER BY (b DESC, c ASC, d DESC)");
- // b and d are reversed in the clustering order
- execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 0, 1, 0, 0);
- execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 0, 1, 1, 1);
- execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 0, 1, 1, 0);
+ // b and d are reversed in the clustering order
+ execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 0, 1, 0, 0);
+ execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 0, 1, 1, 1);
+ execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 0, 1, 1, 0);
- execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 0, 0, 0, 0);
- execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 0, 0, 1, 1);
- execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 0, 0, 1, 0);
+ execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 0, 0, 0, 0);
+ execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 0, 0, 1, 1);
+ execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 0, 0, 1, 0);
- assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b) > (?)", 0, 0),
- row(0, 1, 0, 0),
- row(0, 1, 1, 1),
- row(0, 1, 1, 0)
- );
+ assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b) > (?)", 0, 0),
+ row(0, 1, 0, 0),
+ row(0, 1, 1, 1),
+ row(0, 1, 1, 0)
+ );
- assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b) >= (?)", 0, 0),
- row(0, 1, 0, 0),
- row(0, 1, 1, 1),
- row(0, 1, 1, 0),
- row(0, 0, 0, 0),
- row(0, 0, 1, 1),
- row(0, 0, 1, 0)
- );
+ assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b) >= (?)", 0, 0),
+ row(0, 1, 0, 0),
+ row(0, 1, 1, 1),
+ row(0, 1, 1, 0),
+ row(0, 0, 0, 0),
+ row(0, 0, 1, 1),
+ row(0, 0, 1, 0)
+ );
- assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b) < (?)", 0, 1),
- row(0, 0, 0, 0),
- row(0, 0, 1, 1),
- row(0, 0, 1, 0)
- );
+ assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b) < (?)", 0, 1),
+ row(0, 0, 0, 0),
+ row(0, 0, 1, 1),
+ row(0, 0, 1, 0)
+ );
- assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b) <= (?)", 0, 1),
- row(0, 1, 0, 0),
- row(0, 1, 1, 1),
- row(0, 1, 1, 0),
- row(0, 0, 0, 0),
- row(0, 0, 1, 1),
- row(0, 0, 1, 0)
- );
+ assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b) <= (?)", 0, 1),
+ row(0, 1, 0, 0),
+ row(0, 1, 1, 1),
+ row(0, 1, 1, 0),
+ row(0, 0, 0, 0),
+ row(0, 0, 1, 1),
+ row(0, 0, 1, 0)
+ );
- assertRows(execute("SELECT * FROM %s WHERE a=? AND (b, c, d) IN ((?, ?, ?), (?, ?, ?))", 0, 1, 1, 1, 0, 1, 1),
- row(0, 1, 1, 1),
- row(0, 0, 1, 1)
- );
+ assertRows(execute("SELECT * FROM %s WHERE a=? AND (b, c, d) IN ((?, ?, ?), (?, ?, ?))", 0, 1, 1, 1, 0, 1, 1),
+ row(0, 1, 1, 1),
+ row(0, 0, 1, 1)
+ );
- // same query, but reversed order for the IN values
- assertRows(execute("SELECT * FROM %s WHERE a=? AND (b, c, d) IN ((?, ?, ?), (?, ?, ?))", 0, 0, 1, 1, 1, 1, 1),
- row(0, 1, 1, 1),
- row(0, 0, 1, 1)
- );
+ // same query, but reversed order for the IN values
+ assertRows(execute("SELECT * FROM %s WHERE a=? AND (b, c, d) IN ((?, ?, ?), (?, ?, ?))", 0, 0, 1, 1, 1, 1, 1),
+ row(0, 1, 1, 1),
+ row(0, 0, 1, 1)
+ );
- assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b, c, d) IN (?, ?, ?, ?, ?, ?)",
- 0, tuple(1, 0, 0), tuple(1, 1, 1), tuple(1, 1, 0), tuple(0, 0, 0), tuple(0, 1, 1), tuple(0, 1, 0)),
- row(0, 1, 0, 0),
- row(0, 1, 1, 1),
- row(0, 1, 1, 0),
- row(0, 0, 0, 0),
- row(0, 0, 1, 1),
- row(0, 0, 1, 0)
- );
+ assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b, c, d) IN (?, ?, ?, ?, ?, ?)",
+ 0, tuple(1, 0, 0), tuple(1, 1, 1), tuple(1, 1, 0), tuple(0, 0, 0), tuple(0, 1, 1), tuple(0, 1, 0)),
+ row(0, 1, 0, 0),
+ row(0, 1, 1, 1),
+ row(0, 1, 1, 0),
+ row(0, 0, 0, 0),
+ row(0, 0, 1, 1),
+ row(0, 0, 1, 0)
+ );
- assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b, c) IN (?)", 0, tuple(0, 1)),
- row(0, 0, 1, 1),
- row(0, 0, 1, 0)
- );
+ assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b, c) IN (?)", 0, tuple(0, 1)),
+ row(0, 0, 1, 1),
+ row(0, 0, 1, 0)
+ );
- assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b, c) IN (?)", 0, tuple(0, 0)),
- row(0, 0, 0, 0)
- );
+ assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b, c) IN (?)", 0, tuple(0, 0)),
+ row(0, 0, 0, 0)
+ );
- assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b) IN ((?))", 0, 0),
- row(0, 0, 0, 0),
- row(0, 0, 1, 1),
- row(0, 0, 1, 0)
- );
+ assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b) IN ((?))", 0, 0),
+ row(0, 0, 0, 0),
+ row(0, 0, 1, 1),
+ row(0, 0, 1, 0)
+ );
- assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b, c) > (?, ?)", 0, 1, 0),
- row(0,1, 1, 1),
- row(0, 1, 1, 0)
- );
- }
+ assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b, c) > (?, ?)", 0, 1, 0),
+ row(0, 1, 1, 1),
+ row(0, 1, 1, 0)
+ );
}
@Test
@@ -1017,16 +990,13 @@
@Test
public void testINWithDuplicateValue() throws Throwable
{
- for (String compactOption : new String[] { "", " WITH COMPACT STORAGE" })
- {
- createTable("CREATE TABLE %s (k1 int, k2 int, v int, PRIMARY KEY (k1, k2))" + compactOption);
- execute("INSERT INTO %s (k1, k2, v) VALUES (?, ?, ?)", 1, 1, 1);
+ createTable("CREATE TABLE %s (k1 int, k2 int, v int, PRIMARY KEY (k1, k2))");
+ execute("INSERT INTO %s (k1, k2, v) VALUES (?, ?, ?)", 1, 1, 1);
- assertRows(execute("SELECT * FROM %s WHERE k1 IN (?, ?) AND (k2) IN ((?), (?))", 1, 1, 1, 2),
- row(1, 1, 1));
- assertRows(execute("SELECT * FROM %s WHERE k1 = ? AND (k2) IN ((?), (?))", 1, 1, 1),
- row(1, 1, 1));
- }
+ assertRows(execute("SELECT * FROM %s WHERE k1 IN (?, ?) AND (k2) IN ((?), (?))", 1, 1, 1, 2),
+ row(1, 1, 1));
+ assertRows(execute("SELECT * FROM %s WHERE k1 = ? AND (k2) IN ((?), (?))", 1, 1, 1),
+ row(1, 1, 1));
}
@Test
@@ -1054,868 +1024,852 @@
@Test
public void testMixedOrderColumns1() throws Throwable
{
- for (String compactOption : new String[]{"", " COMPACT STORAGE AND "})
- {
- createTable("CREATE TABLE %s (a int, b int, c int, d int, e int, PRIMARY KEY (a, b, c, d, e)) WITH " +
- compactOption +
- " CLUSTERING ORDER BY (b DESC, c ASC, d DESC, e ASC)");
+ createTable("CREATE TABLE %s (a int, b int, c int, d int, e int, PRIMARY KEY (a, b, c, d, e)) WITH " +
+ " CLUSTERING ORDER BY (b DESC, c ASC, d DESC, e ASC)");
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 2, 0, -1, 0);
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 2, 0, -1, 1);
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 2, 0, 1, 1);
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, -1, 0, 0);
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, -1, 1, 1);
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, -1, 1, 0);
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, 0, 1, -1);
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, 0, 1, 1);
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, 0, 0, -1);
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, 0, 0, 0);
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, 0, 0, 1);
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, 0, -1, -1);
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, 1, 0, -1);
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, 1, 0, 0);
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, 1, 0, -1);
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, 1, 0, 0);
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, 1, 0, 1);
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, 1, -1, 0);
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 0, 0, 0, 0);
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, -1, 0, -1, 0);
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, -1, 0, 0, 0);
- assertRows(execute(
- "SELECT * FROM %s" +
- " WHERE a = ? " +
- "AND (b,c,d,e)<=(?,?,?,?) " +
- "AND (b)>(?)", 0, 2, 0, 1, 1, -1),
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 2, 0, -1, 0);
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 2, 0, -1, 1);
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 2, 0, 1, 1);
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, -1, 0, 0);
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, -1, 1, 1);
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, -1, 1, 0);
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, 0, 1, -1);
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, 0, 1, 1);
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, 0, 0, -1);
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, 0, 0, 0);
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, 0, 0, 1);
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, 0, -1, -1);
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, 1, 0, -1);
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, 1, 0, 0);
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, 1, 0, -1);
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, 1, 0, 0);
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, 1, 0, 1);
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, 1, -1, 0);
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 0, 0, 0, 0);
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, -1, 0, -1, 0);
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, -1, 0, 0, 0);
+ assertRows(execute(
+ "SELECT * FROM %s" +
+ " WHERE a = ? " +
+ "AND (b,c,d,e)<=(?,?,?,?) " +
+ "AND (b)>(?)", 0, 2, 0, 1, 1, -1),
- row(0, 2, 0, 1, 1),
- row(0, 2, 0, -1, 0),
- row(0, 2, 0, -1, 1),
- row(0, 1, -1, 1, 0),
- row(0, 1, -1, 1, 1),
- row(0, 1, -1, 0, 0),
- row(0, 1, 0, 1, -1),
- row(0, 1, 0, 1, 1),
- row(0, 1, 0, 0, -1),
- row(0, 1, 0, 0, 0),
- row(0, 1, 0, 0, 1),
- row(0, 1, 0, -1, -1),
- row(0, 1, 1, 0, -1),
- row(0, 1, 1, 0, 0),
- row(0, 1, 1, 0, 1),
- row(0, 1, 1, -1, 0),
- row(0, 0, 0, 0, 0)
- );
+ row(0, 2, 0, 1, 1),
+ row(0, 2, 0, -1, 0),
+ row(0, 2, 0, -1, 1),
+ row(0, 1, -1, 1, 0),
+ row(0, 1, -1, 1, 1),
+ row(0, 1, -1, 0, 0),
+ row(0, 1, 0, 1, -1),
+ row(0, 1, 0, 1, 1),
+ row(0, 1, 0, 0, -1),
+ row(0, 1, 0, 0, 0),
+ row(0, 1, 0, 0, 1),
+ row(0, 1, 0, -1, -1),
+ row(0, 1, 1, 0, -1),
+ row(0, 1, 1, 0, 0),
+ row(0, 1, 1, 0, 1),
+ row(0, 1, 1, -1, 0),
+ row(0, 0, 0, 0, 0)
+ );
- assertRows(execute(
- "SELECT * FROM %s" +
- " WHERE a = ? " +
- "AND (b,c,d,e)<=(?,?,?,?) " +
- "AND (b)>=(?)", 0, 2, 0, 1, 1, -1),
+ assertRows(execute(
+ "SELECT * FROM %s" +
+ " WHERE a = ? " +
+ "AND (b,c,d,e)<=(?,?,?,?) " +
+ "AND (b)>=(?)", 0, 2, 0, 1, 1, -1),
- row(0, 2, 0, 1, 1),
- row(0, 2, 0, -1, 0),
- row(0, 2, 0, -1, 1),
- row(0, 1, -1, 1, 0),
- row(0, 1, -1, 1, 1),
- row(0, 1, -1, 0, 0),
- row(0, 1, 0, 1, -1),
- row(0, 1, 0, 1, 1),
- row(0, 1, 0, 0, -1),
- row(0, 1, 0, 0, 0),
- row(0, 1, 0, 0, 1),
- row(0, 1, 0, -1, -1),
- row(0, 1, 1, 0, -1),
- row(0, 1, 1, 0, 0),
- row(0, 1, 1, 0, 1),
- row(0, 1, 1, -1, 0),
- row(0, 0, 0, 0, 0),
- row(0, -1, 0, 0, 0),
- row(0, -1, 0, -1, 0)
- );
+ row(0, 2, 0, 1, 1),
+ row(0, 2, 0, -1, 0),
+ row(0, 2, 0, -1, 1),
+ row(0, 1, -1, 1, 0),
+ row(0, 1, -1, 1, 1),
+ row(0, 1, -1, 0, 0),
+ row(0, 1, 0, 1, -1),
+ row(0, 1, 0, 1, 1),
+ row(0, 1, 0, 0, -1),
+ row(0, 1, 0, 0, 0),
+ row(0, 1, 0, 0, 1),
+ row(0, 1, 0, -1, -1),
+ row(0, 1, 1, 0, -1),
+ row(0, 1, 1, 0, 0),
+ row(0, 1, 1, 0, 1),
+ row(0, 1, 1, -1, 0),
+ row(0, 0, 0, 0, 0),
+ row(0, -1, 0, 0, 0),
+ row(0, -1, 0, -1, 0)
+ );
- assertRows(execute(
- "SELECT * FROM %s" +
- " WHERE a = ? " +
- "AND (b,c,d)>=(?,?,?)" +
- "AND (b,c,d,e)<(?,?,?,?) ", 0, 1, 1, 0, 1, 1, 0, 1),
- row(0, 1, 1, 0, -1),
- row(0, 1, 1, 0, 0)
+ assertRows(execute(
+ "SELECT * FROM %s" +
+ " WHERE a = ? " +
+ "AND (b,c,d)>=(?,?,?)" +
+ "AND (b,c,d,e)<(?,?,?,?) ", 0, 1, 1, 0, 1, 1, 0, 1),
+ row(0, 1, 1, 0, -1),
+ row(0, 1, 1, 0, 0)
- );
+ );
- assertRows(execute(
- "SELECT * FROM %s" +
- " WHERE a = ? " +
- "AND (b,c,d,e)>(?,?,?,?)" +
- "AND (b,c,d)<=(?,?,?) ", 0, -1, 0, -1, -1, 2, 0, -1),
+ assertRows(execute(
+ "SELECT * FROM %s" +
+ " WHERE a = ? " +
+ "AND (b,c,d,e)>(?,?,?,?)" +
+ "AND (b,c,d)<=(?,?,?) ", 0, -1, 0, -1, -1, 2, 0, -1),
- row(0, 2, 0, -1, 0),
- row(0, 2, 0, -1, 1),
- row(0, 1, -1, 1, 0),
- row(0, 1, -1, 1, 1),
- row(0, 1, -1, 0, 0),
- row(0, 1, 0, 1, -1),
- row(0, 1, 0, 1, 1),
- row(0, 1, 0, 0, -1),
- row(0, 1, 0, 0, 0),
- row(0, 1, 0, 0, 1),
- row(0, 1, 0, -1, -1),
- row(0, 1, 1, 0, -1),
- row(0, 1, 1, 0, 0),
- row(0, 1, 1, 0, 1),
- row(0, 1, 1, -1, 0),
- row(0, 0, 0, 0, 0),
- row(0, -1, 0, 0, 0),
- row(0, -1, 0, -1, 0)
- );
+ row(0, 2, 0, -1, 0),
+ row(0, 2, 0, -1, 1),
+ row(0, 1, -1, 1, 0),
+ row(0, 1, -1, 1, 1),
+ row(0, 1, -1, 0, 0),
+ row(0, 1, 0, 1, -1),
+ row(0, 1, 0, 1, 1),
+ row(0, 1, 0, 0, -1),
+ row(0, 1, 0, 0, 0),
+ row(0, 1, 0, 0, 1),
+ row(0, 1, 0, -1, -1),
+ row(0, 1, 1, 0, -1),
+ row(0, 1, 1, 0, 0),
+ row(0, 1, 1, 0, 1),
+ row(0, 1, 1, -1, 0),
+ row(0, 0, 0, 0, 0),
+ row(0, -1, 0, 0, 0),
+ row(0, -1, 0, -1, 0)
+ );
- assertRows(execute(
- "SELECT * FROM %s" +
- " WHERE a = ? " +
- "AND (b,c,d,e) < (?,?,?,?) " +
- "AND (b,c,d,e)>(?,?,?,?)", 0, 1, 0, 0, 0, 1, 0, -1, -1),
- row(0, 1, 0, 0, -1)
- );
+ assertRows(execute(
+ "SELECT * FROM %s" +
+ " WHERE a = ? " +
+ "AND (b,c,d,e) < (?,?,?,?) " +
+ "AND (b,c,d,e)>(?,?,?,?)", 0, 1, 0, 0, 0, 1, 0, -1, -1),
+ row(0, 1, 0, 0, -1)
+ );
- assertRows(execute(
- "SELECT * FROM %s" +
- " WHERE a = ? " +
- "AND (b,c,d,e) <= (?,?,?,?) " +
- "AND (b,c,d,e)>(?,?,?,?)", 0, 1, 0, 0, 0, 1, 0, -1, -1),
- row(0, 1, 0, 0, -1),
- row(0, 1, 0, 0, 0)
- );
+ assertRows(execute(
+ "SELECT * FROM %s" +
+ " WHERE a = ? " +
+ "AND (b,c,d,e) <= (?,?,?,?) " +
+ "AND (b,c,d,e)>(?,?,?,?)", 0, 1, 0, 0, 0, 1, 0, -1, -1),
+ row(0, 1, 0, 0, -1),
+ row(0, 1, 0, 0, 0)
+ );
- assertRows(execute(
- "SELECT * FROM %s" +
- " WHERE a = ? " +
- "AND (b)<(?) " +
- "AND (b,c,d,e)>(?,?,?,?)", 0, 2, -1, 0, -1, -1),
+ assertRows(execute(
+ "SELECT * FROM %s" +
+ " WHERE a = ? " +
+ "AND (b)<(?) " +
+ "AND (b,c,d,e)>(?,?,?,?)", 0, 2, -1, 0, -1, -1),
- row(0, 1, -1, 1, 0),
- row(0, 1, -1, 1, 1),
- row(0, 1, -1, 0, 0),
- row(0, 1, 0, 1, -1),
- row(0, 1, 0, 1, 1),
- row(0, 1, 0, 0, -1),
- row(0, 1, 0, 0, 0),
- row(0, 1, 0, 0, 1),
- row(0, 1, 0, -1, -1),
- row(0, 1, 1, 0, -1),
- row(0, 1, 1, 0, 0),
- row(0, 1, 1, 0, 1),
- row(0, 1, 1, -1, 0),
- row(0, 0, 0, 0, 0),
- row(0, -1, 0, 0, 0),
- row(0, -1, 0, -1, 0)
+ row(0, 1, -1, 1, 0),
+ row(0, 1, -1, 1, 1),
+ row(0, 1, -1, 0, 0),
+ row(0, 1, 0, 1, -1),
+ row(0, 1, 0, 1, 1),
+ row(0, 1, 0, 0, -1),
+ row(0, 1, 0, 0, 0),
+ row(0, 1, 0, 0, 1),
+ row(0, 1, 0, -1, -1),
+ row(0, 1, 1, 0, -1),
+ row(0, 1, 1, 0, 0),
+ row(0, 1, 1, 0, 1),
+ row(0, 1, 1, -1, 0),
+ row(0, 0, 0, 0, 0),
+ row(0, -1, 0, 0, 0),
+ row(0, -1, 0, -1, 0)
- );
+ );
- assertRows(execute(
- "SELECT * FROM %s" +
- " WHERE a = ? " +
- "AND (b)<(?) " +
- "AND (b)>(?)", 0, 2, -1),
+ assertRows(execute(
+ "SELECT * FROM %s" +
+ " WHERE a = ? " +
+ "AND (b)<(?) " +
+ "AND (b)>(?)", 0, 2, -1),
- row(0, 1, -1, 1, 0),
- row(0, 1, -1, 1, 1),
- row(0, 1, -1, 0, 0),
- row(0, 1, 0, 1, -1),
- row(0, 1, 0, 1, 1),
- row(0, 1, 0, 0, -1),
- row(0, 1, 0, 0, 0),
- row(0, 1, 0, 0, 1),
- row(0, 1, 0, -1, -1),
- row(0, 1, 1, 0, -1),
- row(0, 1, 1, 0, 0),
- row(0, 1, 1, 0, 1),
- row(0, 1, 1, -1, 0),
- row(0, 0, 0, 0, 0)
+ row(0, 1, -1, 1, 0),
+ row(0, 1, -1, 1, 1),
+ row(0, 1, -1, 0, 0),
+ row(0, 1, 0, 1, -1),
+ row(0, 1, 0, 1, 1),
+ row(0, 1, 0, 0, -1),
+ row(0, 1, 0, 0, 0),
+ row(0, 1, 0, 0, 1),
+ row(0, 1, 0, -1, -1),
+ row(0, 1, 1, 0, -1),
+ row(0, 1, 1, 0, 0),
+ row(0, 1, 1, 0, 1),
+ row(0, 1, 1, -1, 0),
+ row(0, 0, 0, 0, 0)
- );
+ );
- assertRows(execute(
- "SELECT * FROM %s" +
- " WHERE a = ? " +
- "AND (b)<(?) " +
- "AND (b)>=(?)", 0, 2, -1),
+ assertRows(execute(
+ "SELECT * FROM %s" +
+ " WHERE a = ? " +
+ "AND (b)<(?) " +
+ "AND (b)>=(?)", 0, 2, -1),
- row(0, 1, -1, 1, 0),
- row(0, 1, -1, 1, 1),
- row(0, 1, -1, 0, 0),
- row(0, 1, 0, 1, -1),
- row(0, 1, 0, 1, 1),
- row(0, 1, 0, 0, -1),
- row(0, 1, 0, 0, 0),
- row(0, 1, 0, 0, 1),
- row(0, 1, 0, -1, -1),
- row(0, 1, 1, 0, -1),
- row(0, 1, 1, 0, 0),
- row(0, 1, 1, 0, 1),
- row(0, 1, 1, -1, 0),
- row(0, 0, 0, 0, 0),
- row(0, -1, 0, 0, 0),
- row(0, -1, 0, -1, 0)
- );
+ row(0, 1, -1, 1, 0),
+ row(0, 1, -1, 1, 1),
+ row(0, 1, -1, 0, 0),
+ row(0, 1, 0, 1, -1),
+ row(0, 1, 0, 1, 1),
+ row(0, 1, 0, 0, -1),
+ row(0, 1, 0, 0, 0),
+ row(0, 1, 0, 0, 1),
+ row(0, 1, 0, -1, -1),
+ row(0, 1, 1, 0, -1),
+ row(0, 1, 1, 0, 0),
+ row(0, 1, 1, 0, 1),
+ row(0, 1, 1, -1, 0),
+ row(0, 0, 0, 0, 0),
+ row(0, -1, 0, 0, 0),
+ row(0, -1, 0, -1, 0)
+ );
- assertRows(execute(
- "SELECT * FROM %s" +
- " WHERE a = ? " +
- "AND (b,c,d,e)<=(?,?,?,?) " +
- "AND (b,c,d,e)>(?,?,?,?)", 0, 2, 0, 1, 1, -1, 0, -1, -1),
+ assertRows(execute(
+ "SELECT * FROM %s" +
+ " WHERE a = ? " +
+ "AND (b,c,d,e)<=(?,?,?,?) " +
+ "AND (b,c,d,e)>(?,?,?,?)", 0, 2, 0, 1, 1, -1, 0, -1, -1),
- row(0, 2, 0, 1, 1),
- row(0, 2, 0, -1, 0),
- row(0, 2, 0, -1, 1),
- row(0, 1, -1, 1, 0),
- row(0, 1, -1, 1, 1),
- row(0, 1, -1, 0, 0),
- row(0, 1, 0, 1, -1),
- row(0, 1, 0, 1, 1),
- row(0, 1, 0, 0, -1),
- row(0, 1, 0, 0, 0),
- row(0, 1, 0, 0, 1),
- row(0, 1, 0, -1, -1),
- row(0, 1, 1, 0, -1),
- row(0, 1, 1, 0, 0),
- row(0, 1, 1, 0, 1),
- row(0, 1, 1, -1, 0),
- row(0, 0, 0, 0, 0),
- row(0, -1, 0, 0, 0),
- row(0, -1, 0, -1, 0)
- );
+ row(0, 2, 0, 1, 1),
+ row(0, 2, 0, -1, 0),
+ row(0, 2, 0, -1, 1),
+ row(0, 1, -1, 1, 0),
+ row(0, 1, -1, 1, 1),
+ row(0, 1, -1, 0, 0),
+ row(0, 1, 0, 1, -1),
+ row(0, 1, 0, 1, 1),
+ row(0, 1, 0, 0, -1),
+ row(0, 1, 0, 0, 0),
+ row(0, 1, 0, 0, 1),
+ row(0, 1, 0, -1, -1),
+ row(0, 1, 1, 0, -1),
+ row(0, 1, 1, 0, 0),
+ row(0, 1, 1, 0, 1),
+ row(0, 1, 1, -1, 0),
+ row(0, 0, 0, 0, 0),
+ row(0, -1, 0, 0, 0),
+ row(0, -1, 0, -1, 0)
+ );
- assertRows(execute(
- "SELECT * FROM %s" +
- " WHERE a = ? " +
- "AND (b,c)<=(?,?) " +
- "AND (b,c,d,e)>(?,?,?,?)", 0, 2, 0, -1, 0, -1, -1),
+ assertRows(execute(
+ "SELECT * FROM %s" +
+ " WHERE a = ? " +
+ "AND (b,c)<=(?,?) " +
+ "AND (b,c,d,e)>(?,?,?,?)", 0, 2, 0, -1, 0, -1, -1),
- row(0, 2, 0, 1, 1),
- row(0, 2, 0, -1, 0),
- row(0, 2, 0, -1, 1),
- row(0, 1, -1, 1, 0),
- row(0, 1, -1, 1, 1),
- row(0, 1, -1, 0, 0),
- row(0, 1, 0, 1, -1),
- row(0, 1, 0, 1, 1),
- row(0, 1, 0, 0, -1),
- row(0, 1, 0, 0, 0),
- row(0, 1, 0, 0, 1),
- row(0, 1, 0, -1, -1),
- row(0, 1, 1, 0, -1),
- row(0, 1, 1, 0, 0),
- row(0, 1, 1, 0, 1),
- row(0, 1, 1, -1, 0),
- row(0, 0, 0, 0, 0),
- row(0, -1, 0, 0, 0),
- row(0, -1, 0, -1, 0)
- );
+ row(0, 2, 0, 1, 1),
+ row(0, 2, 0, -1, 0),
+ row(0, 2, 0, -1, 1),
+ row(0, 1, -1, 1, 0),
+ row(0, 1, -1, 1, 1),
+ row(0, 1, -1, 0, 0),
+ row(0, 1, 0, 1, -1),
+ row(0, 1, 0, 1, 1),
+ row(0, 1, 0, 0, -1),
+ row(0, 1, 0, 0, 0),
+ row(0, 1, 0, 0, 1),
+ row(0, 1, 0, -1, -1),
+ row(0, 1, 1, 0, -1),
+ row(0, 1, 1, 0, 0),
+ row(0, 1, 1, 0, 1),
+ row(0, 1, 1, -1, 0),
+ row(0, 0, 0, 0, 0),
+ row(0, -1, 0, 0, 0),
+ row(0, -1, 0, -1, 0)
+ );
- assertRows(execute(
- "SELECT * FROM %s" +
- " WHERE a = ? " +
- "AND (b,c,d)<=(?,?,?) " +
- "AND (b,c,d,e)>(?,?,?,?)", 0, 2, 0, -1, -1, 0, -1, -1),
+ assertRows(execute(
+ "SELECT * FROM %s" +
+ " WHERE a = ? " +
+ "AND (b,c,d)<=(?,?,?) " +
+ "AND (b,c,d,e)>(?,?,?,?)", 0, 2, 0, -1, -1, 0, -1, -1),
- row(0, 2, 0, -1, 0),
- row(0, 2, 0, -1, 1),
- row(0, 1, -1, 1, 0),
- row(0, 1, -1, 1, 1),
- row(0, 1, -1, 0, 0),
- row(0, 1, 0, 1, -1),
- row(0, 1, 0, 1, 1),
- row(0, 1, 0, 0, -1),
- row(0, 1, 0, 0, 0),
- row(0, 1, 0, 0, 1),
- row(0, 1, 0, -1, -1),
- row(0, 1, 1, 0, -1),
- row(0, 1, 1, 0, 0),
- row(0, 1, 1, 0, 1),
- row(0, 1, 1, -1, 0),
- row(0, 0, 0, 0, 0),
- row(0, -1, 0, 0, 0),
- row(0, -1, 0, -1, 0)
- );
+ row(0, 2, 0, -1, 0),
+ row(0, 2, 0, -1, 1),
+ row(0, 1, -1, 1, 0),
+ row(0, 1, -1, 1, 1),
+ row(0, 1, -1, 0, 0),
+ row(0, 1, 0, 1, -1),
+ row(0, 1, 0, 1, 1),
+ row(0, 1, 0, 0, -1),
+ row(0, 1, 0, 0, 0),
+ row(0, 1, 0, 0, 1),
+ row(0, 1, 0, -1, -1),
+ row(0, 1, 1, 0, -1),
+ row(0, 1, 1, 0, 0),
+ row(0, 1, 1, 0, 1),
+ row(0, 1, 1, -1, 0),
+ row(0, 0, 0, 0, 0),
+ row(0, -1, 0, 0, 0),
+ row(0, -1, 0, -1, 0)
+ );
- assertRows(execute(
- "SELECT * FROM %s" +
- " WHERE a = ? " +
- "AND (b,c,d,e)>(?,?,?,?)" +
- "AND (b,c,d)<=(?,?,?) ", 0, -1, 0, -1, -1, 2, 0, -1),
+ assertRows(execute(
+ "SELECT * FROM %s" +
+ " WHERE a = ? " +
+ "AND (b,c,d,e)>(?,?,?,?)" +
+ "AND (b,c,d)<=(?,?,?) ", 0, -1, 0, -1, -1, 2, 0, -1),
- row(0, 2, 0, -1, 0),
- row(0, 2, 0, -1, 1),
- row(0, 1, -1, 1, 0),
- row(0, 1, -1, 1, 1),
- row(0, 1, -1, 0, 0),
- row(0, 1, 0, 1, -1),
- row(0, 1, 0, 1, 1),
- row(0, 1, 0, 0, -1),
- row(0, 1, 0, 0, 0),
- row(0, 1, 0, 0, 1),
- row(0, 1, 0, -1, -1),
- row(0, 1, 1, 0, -1),
- row(0, 1, 1, 0, 0),
- row(0, 1, 1, 0, 1),
- row(0, 1, 1, -1, 0),
- row(0, 0, 0, 0, 0),
- row(0, -1, 0, 0, 0),
- row(0, -1, 0, -1, 0)
- );
+ row(0, 2, 0, -1, 0),
+ row(0, 2, 0, -1, 1),
+ row(0, 1, -1, 1, 0),
+ row(0, 1, -1, 1, 1),
+ row(0, 1, -1, 0, 0),
+ row(0, 1, 0, 1, -1),
+ row(0, 1, 0, 1, 1),
+ row(0, 1, 0, 0, -1),
+ row(0, 1, 0, 0, 0),
+ row(0, 1, 0, 0, 1),
+ row(0, 1, 0, -1, -1),
+ row(0, 1, 1, 0, -1),
+ row(0, 1, 1, 0, 0),
+ row(0, 1, 1, 0, 1),
+ row(0, 1, 1, -1, 0),
+ row(0, 0, 0, 0, 0),
+ row(0, -1, 0, 0, 0),
+ row(0, -1, 0, -1, 0)
+ );
- assertRows(execute(
- "SELECT * FROM %s" +
- " WHERE a = ? " +
- "AND (b,c,d)>=(?,?,?)" +
- "AND (b,c,d,e)<(?,?,?,?) ", 0, 1, 1, 0, 1, 1, 0, 1),
- row(0, 1, 1, 0, -1),
- row(0, 1, 1, 0, 0)
- );
- assertRows(execute(
- "SELECT * FROM %s" +
- " WHERE a = ? " +
- "AND (b,c,d,e)<(?,?,?,?) " +
- "AND (b,c,d)>=(?,?,?)", 0, 1, 1, 0, 1, 1, 1, 0),
- row(0, 1, 1, 0, -1),
- row(0, 1, 1, 0, 0)
+ assertRows(execute(
+ "SELECT * FROM %s" +
+ " WHERE a = ? " +
+ "AND (b,c,d)>=(?,?,?)" +
+ "AND (b,c,d,e)<(?,?,?,?) ", 0, 1, 1, 0, 1, 1, 0, 1),
+ row(0, 1, 1, 0, -1),
+ row(0, 1, 1, 0, 0)
+ );
+ assertRows(execute(
+ "SELECT * FROM %s" +
+ " WHERE a = ? " +
+ "AND (b,c,d,e)<(?,?,?,?) " +
+ "AND (b,c,d)>=(?,?,?)", 0, 1, 1, 0, 1, 1, 1, 0),
+ row(0, 1, 1, 0, -1),
+ row(0, 1, 1, 0, 0)
- );
+ );
- assertRows(execute(
- "SELECT * FROM %s" +
- " WHERE a = ? " +
- "AND (b,c)<(?,?) " +
- "AND (b,c,d,e)>(?,?,?,?)", 0, 2, 0, -1, 0, -1, -1),
- row(0, 1, -1, 1, 0),
- row(0, 1, -1, 1, 1),
- row(0, 1, -1, 0, 0),
- row(0, 1, 0, 1, -1),
- row(0, 1, 0, 1, 1),
- row(0, 1, 0, 0, -1),
- row(0, 1, 0, 0, 0),
- row(0, 1, 0, 0, 1),
- row(0, 1, 0, -1, -1),
- row(0, 1, 1, 0, -1),
- row(0, 1, 1, 0, 0),
- row(0, 1, 1, 0, 1),
- row(0, 1, 1, -1, 0),
- row(0, 0, 0, 0, 0),
- row(0, -1, 0, 0, 0),
- row(0, -1, 0, -1, 0)
- );
+ assertRows(execute(
+ "SELECT * FROM %s" +
+ " WHERE a = ? " +
+ "AND (b,c)<(?,?) " +
+ "AND (b,c,d,e)>(?,?,?,?)", 0, 2, 0, -1, 0, -1, -1),
+ row(0, 1, -1, 1, 0),
+ row(0, 1, -1, 1, 1),
+ row(0, 1, -1, 0, 0),
+ row(0, 1, 0, 1, -1),
+ row(0, 1, 0, 1, 1),
+ row(0, 1, 0, 0, -1),
+ row(0, 1, 0, 0, 0),
+ row(0, 1, 0, 0, 1),
+ row(0, 1, 0, -1, -1),
+ row(0, 1, 1, 0, -1),
+ row(0, 1, 1, 0, 0),
+ row(0, 1, 1, 0, 1),
+ row(0, 1, 1, -1, 0),
+ row(0, 0, 0, 0, 0),
+ row(0, -1, 0, 0, 0),
+ row(0, -1, 0, -1, 0)
+ );
- assertRows(execute(
- "SELECT * FROM %s" +
- " WHERE a = ? " +
- "AND (b,c)<(?,?) " +
- "AND (b,c,d,e)>(?,?,?,?)", 0, 2, 0, -1, 0, -1, -1),
- row(0, 1, -1, 1, 0),
- row(0, 1, -1, 1, 1),
- row(0, 1, -1, 0, 0),
- row(0, 1, 0, 1, -1),
- row(0, 1, 0, 1, 1),
- row(0, 1, 0, 0, -1),
- row(0, 1, 0, 0, 0),
- row(0, 1, 0, 0, 1),
- row(0, 1, 0, -1, -1),
- row(0, 1, 1, 0, -1),
- row(0, 1, 1, 0, 0),
- row(0, 1, 1, 0, 1),
- row(0, 1, 1, -1, 0),
- row(0, 0, 0, 0, 0),
- row(0, -1, 0, 0, 0),
- row(0, -1, 0, -1, 0)
- );
+ assertRows(execute(
+ "SELECT * FROM %s" +
+ " WHERE a = ? " +
+ "AND (b,c)<(?,?) " +
+ "AND (b,c,d,e)>(?,?,?,?)", 0, 2, 0, -1, 0, -1, -1),
+ row(0, 1, -1, 1, 0),
+ row(0, 1, -1, 1, 1),
+ row(0, 1, -1, 0, 0),
+ row(0, 1, 0, 1, -1),
+ row(0, 1, 0, 1, 1),
+ row(0, 1, 0, 0, -1),
+ row(0, 1, 0, 0, 0),
+ row(0, 1, 0, 0, 1),
+ row(0, 1, 0, -1, -1),
+ row(0, 1, 1, 0, -1),
+ row(0, 1, 1, 0, 0),
+ row(0, 1, 1, 0, 1),
+ row(0, 1, 1, -1, 0),
+ row(0, 0, 0, 0, 0),
+ row(0, -1, 0, 0, 0),
+ row(0, -1, 0, -1, 0)
+ );
- assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b,c,d,e) <= (?,?,?,?)", 0, 1, 0, 0, 0),
- row(0, 1, -1, 1, 0),
- row(0, 1, -1, 1, 1),
- row(0, 1, -1, 0, 0),
- row(0, 1, 0, 0, -1),
- row(0, 1, 0, 0, 0),
- row(0, 1, 0, -1, -1),
- row(0, 0, 0, 0, 0),
- row(0, -1, 0, 0, 0),
- row(0, -1, 0, -1, 0)
- );
+ assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b,c,d,e) <= (?,?,?,?)", 0, 1, 0, 0, 0),
+ row(0, 1, -1, 1, 0),
+ row(0, 1, -1, 1, 1),
+ row(0, 1, -1, 0, 0),
+ row(0, 1, 0, 0, -1),
+ row(0, 1, 0, 0, 0),
+ row(0, 1, 0, -1, -1),
+ row(0, 0, 0, 0, 0),
+ row(0, -1, 0, 0, 0),
+ row(0, -1, 0, -1, 0)
+ );
- assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b,c,d,e) > (?,?,?,?)", 0, 1, 0, 0, 0),
- row(0, 2, 0, 1, 1),
- row(0, 2, 0, -1, 0),
- row(0, 2, 0, -1, 1),
- row(0, 1, 0, 1, -1),
- row(0, 1, 0, 1, 1),
- row(0, 1, 0, 0, 1),
- row(0, 1, 1, 0, -1),
- row(0, 1, 1, 0, 0),
- row(0, 1, 1, 0, 1),
- row(0, 1, 1, -1, 0)
- );
+ assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b,c,d,e) > (?,?,?,?)", 0, 1, 0, 0, 0),
+ row(0, 2, 0, 1, 1),
+ row(0, 2, 0, -1, 0),
+ row(0, 2, 0, -1, 1),
+ row(0, 1, 0, 1, -1),
+ row(0, 1, 0, 1, 1),
+ row(0, 1, 0, 0, 1),
+ row(0, 1, 1, 0, -1),
+ row(0, 1, 1, 0, 0),
+ row(0, 1, 1, 0, 1),
+ row(0, 1, 1, -1, 0)
+ );
- assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b,c,d,e) >= (?,?,?,?)", 0, 1, 0, 0, 0),
- row(0, 2, 0, 1, 1),
- row(0, 2, 0, -1, 0),
- row(0, 2, 0, -1, 1),
- row(0, 1, 0, 1, -1),
- row(0, 1, 0, 1, 1),
- row(0, 1, 0, 0, 0),
- row(0, 1, 0, 0, 1),
- row(0, 1, 1, 0, -1),
- row(0, 1, 1, 0, 0),
- row(0, 1, 1, 0, 1),
- row(0, 1, 1, -1, 0)
- );
+ assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b,c,d,e) >= (?,?,?,?)", 0, 1, 0, 0, 0),
+ row(0, 2, 0, 1, 1),
+ row(0, 2, 0, -1, 0),
+ row(0, 2, 0, -1, 1),
+ row(0, 1, 0, 1, -1),
+ row(0, 1, 0, 1, 1),
+ row(0, 1, 0, 0, 0),
+ row(0, 1, 0, 0, 1),
+ row(0, 1, 1, 0, -1),
+ row(0, 1, 1, 0, 0),
+ row(0, 1, 1, 0, 1),
+ row(0, 1, 1, -1, 0)
+ );
- assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b,c,d) >= (?,?,?)", 0, 1, 0, 0),
- row(0, 2, 0, 1, 1),
- row(0, 2, 0, -1, 0),
- row(0, 2, 0, -1, 1),
- row(0, 1, 0, 1, -1),
- row(0, 1, 0, 1, 1),
- row(0, 1, 0, 0, -1),
- row(0, 1, 0, 0, 0),
- row(0, 1, 0, 0, 1),
- row(0, 1, 1, 0, -1),
- row(0, 1, 1, 0, 0),
- row(0, 1, 1, 0, 1),
- row(0, 1, 1, -1, 0)
- );
+ assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b,c,d) >= (?,?,?)", 0, 1, 0, 0),
+ row(0, 2, 0, 1, 1),
+ row(0, 2, 0, -1, 0),
+ row(0, 2, 0, -1, 1),
+ row(0, 1, 0, 1, -1),
+ row(0, 1, 0, 1, 1),
+ row(0, 1, 0, 0, -1),
+ row(0, 1, 0, 0, 0),
+ row(0, 1, 0, 0, 1),
+ row(0, 1, 1, 0, -1),
+ row(0, 1, 1, 0, 0),
+ row(0, 1, 1, 0, 1),
+ row(0, 1, 1, -1, 0)
+ );
- assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b,c,d) > (?,?,?)", 0, 1, 0, 0),
- row(0, 2, 0, 1, 1),
- row(0, 2, 0, -1, 0),
- row(0, 2, 0, -1, 1),
- row(0, 1, 0, 1, -1),
- row(0, 1, 0, 1, 1),
- row(0, 1, 1, 0, -1),
- row(0, 1, 1, 0, 0),
- row(0, 1, 1, 0, 1),
- row(0, 1, 1, -1, 0)
- );
- }
+ assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b,c,d) > (?,?,?)", 0, 1, 0, 0),
+ row(0, 2, 0, 1, 1),
+ row(0, 2, 0, -1, 0),
+ row(0, 2, 0, -1, 1),
+ row(0, 1, 0, 1, -1),
+ row(0, 1, 0, 1, 1),
+ row(0, 1, 1, 0, -1),
+ row(0, 1, 1, 0, 0),
+ row(0, 1, 1, 0, 1),
+ row(0, 1, 1, -1, 0)
+ );
}
@Test
public void testMixedOrderColumns2() throws Throwable
{
- for (String compactOption : new String[]{"", " COMPACT STORAGE AND "})
- {
- createTable("CREATE TABLE %s (a int, b int, c int, d int, e int, PRIMARY KEY (a, b, c, d, e)) WITH " +
- compactOption +
- "CLUSTERING ORDER BY (b DESC, c ASC, d ASC, e ASC)");
+ createTable("CREATE TABLE %s (a int, b int, c int, d int, e int, PRIMARY KEY (a, b, c, d, e)) WITH " +
+ "CLUSTERING ORDER BY (b DESC, c ASC, d ASC, e ASC)");
- // b and d are reversed in the clustering order
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 2, 0, -1, 0);
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 2, 0, -1, 1);
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, -1, 0, 0);
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, -1, 1, 0);
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, -1, 1, 1);
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, 0, 1, -1);
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, 0, 1, 1);
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, 0, 0, -1);
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, 0, 0, 0);
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, 0, 0, 1);
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, 0, -1, -1);
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, 1, 0, -1);
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, 1, 0, 0);
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, 1, 0, -1);
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, 1, 0, 0);
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, 1, 0, 1);
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, 1, -1, 0);
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 0, 0, 0, 0);
+ // b and d are reversed in the clustering order
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 2, 0, -1, 0);
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 2, 0, -1, 1);
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, -1, 0, 0);
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, -1, 1, 0);
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, -1, 1, 1);
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, 0, 1, -1);
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, 0, 1, 1);
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, 0, 0, -1);
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, 0, 0, 0);
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, 0, 0, 1);
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, 0, -1, -1);
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, 1, 0, -1);
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, 1, 0, 0);
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, 1, 0, -1);
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, 1, 0, 0);
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, 1, 0, 1);
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, 1, -1, 0);
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 0, 0, 0, 0);
- assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b,c,d,e) <= (?,?,?,?)", 0, 1, 0, 0, 0),
- row(0, 1, -1, 0, 0),
- row(0, 1, -1, 1, 0),
- row(0, 1, -1, 1, 1),
- row(0, 1, 0, -1, -1),
- row(0, 1, 0, 0, -1),
- row(0, 1, 0, 0, 0),
- row(0, 0, 0, 0, 0)
- );
+ assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b,c,d,e) <= (?,?,?,?)", 0, 1, 0, 0, 0),
+ row(0, 1, -1, 0, 0),
+ row(0, 1, -1, 1, 0),
+ row(0, 1, -1, 1, 1),
+ row(0, 1, 0, -1, -1),
+ row(0, 1, 0, 0, -1),
+ row(0, 1, 0, 0, 0),
+ row(0, 0, 0, 0, 0)
+ );
- assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b,c,d,e) > (?,?,?,?)", 0, 1, 0, 0, 0),
- row(0, 2, 0, -1, 0),
- row(0, 2, 0, -1, 1),
- row(0, 1, 0, 0, 1),
- row(0, 1, 0, 1, -1),
- row(0, 1, 0, 1, 1),
- row(0, 1, 1, -1, 0),
- row(0, 1, 1, 0, -1),
- row(0, 1, 1, 0, 0),
- row(0, 1, 1, 0, 1)
- );
- assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b,c,d,e) >= (?,?,?,?)", 0, 1, 0, 0, 0),
- row(0, 2, 0, -1, 0),
- row(0, 2, 0, -1, 1),
- row(0, 1, 0, 0, 0),
- row(0, 1, 0, 0, 1),
- row(0, 1, 0, 1, -1),
- row(0, 1, 0, 1, 1),
- row(0, 1, 1, -1, 0),
- row(0, 1, 1, 0, -1),
- row(0, 1, 1, 0, 0),
- row(0, 1, 1, 0, 1)
- );
- }
+ assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b,c,d,e) > (?,?,?,?)", 0, 1, 0, 0, 0),
+ row(0, 2, 0, -1, 0),
+ row(0, 2, 0, -1, 1),
+ row(0, 1, 0, 0, 1),
+ row(0, 1, 0, 1, -1),
+ row(0, 1, 0, 1, 1),
+ row(0, 1, 1, -1, 0),
+ row(0, 1, 1, 0, -1),
+ row(0, 1, 1, 0, 0),
+ row(0, 1, 1, 0, 1)
+ );
+ assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b,c,d,e) >= (?,?,?,?)", 0, 1, 0, 0, 0),
+ row(0, 2, 0, -1, 0),
+ row(0, 2, 0, -1, 1),
+ row(0, 1, 0, 0, 0),
+ row(0, 1, 0, 0, 1),
+ row(0, 1, 0, 1, -1),
+ row(0, 1, 0, 1, 1),
+ row(0, 1, 1, -1, 0),
+ row(0, 1, 1, 0, -1),
+ row(0, 1, 1, 0, 0),
+ row(0, 1, 1, 0, 1)
+ );
}
@Test
public void testMixedOrderColumns3() throws Throwable
{
- for (String compactOption : new String[]{"", " COMPACT STORAGE AND "})
- {
- createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY (a, b, c)) WITH " +
- compactOption +
- "CLUSTERING ORDER BY (b DESC, c ASC)");
+ createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY (a, b, c)) WITH " +
+ "CLUSTERING ORDER BY (b DESC, c ASC)");
- execute("INSERT INTO %s (a, b, c) VALUES (?,?,?);", 0, 2, 3);
- execute("INSERT INTO %s (a, b, c) VALUES (?,?,?);", 0, 2, 4);
- execute("INSERT INTO %s (a, b, c) VALUES (?,?,?);", 0, 4, 4);
- execute("INSERT INTO %s (a, b, c) VALUES (?,?,?);", 0, 3, 4);
- execute("INSERT INTO %s (a, b, c) VALUES (?,?,?);", 0, 4, 5);
- execute("INSERT INTO %s (a, b, c) VALUES (?,?,?);", 0, 4, 6);
+ execute("INSERT INTO %s (a, b, c) VALUES (?,?,?);", 0, 2, 3);
+ execute("INSERT INTO %s (a, b, c) VALUES (?,?,?);", 0, 2, 4);
+ execute("INSERT INTO %s (a, b, c) VALUES (?,?,?);", 0, 4, 4);
+ execute("INSERT INTO %s (a, b, c) VALUES (?,?,?);", 0, 3, 4);
+ execute("INSERT INTO %s (a, b, c) VALUES (?,?,?);", 0, 4, 5);
+ execute("INSERT INTO %s (a, b, c) VALUES (?,?,?);", 0, 4, 6);
- assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b,c)>=(?,?) AND (b,c)<(?,?) ALLOW FILTERING", 0, 2, 3, 4, 5),
- row(0, 4, 4), row(0, 3, 4), row(0, 2, 3), row(0, 2, 4)
- );
- assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b,c)>=(?,?) AND (b,c)<=(?,?) ALLOW FILTERING", 0, 2, 3, 4, 5),
- row(0, 4, 4), row(0, 4, 5), row(0, 3, 4), row(0, 2, 3), row(0, 2, 4)
- );
- assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b,c)<(?,?) ALLOW FILTERING", 0, 4, 5),
- row(0, 4, 4), row(0, 3, 4), row(0, 2, 3), row(0, 2, 4)
- );
+ assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b,c)>=(?,?) AND (b,c)<(?,?) ALLOW FILTERING", 0, 2, 3, 4, 5),
+ row(0, 4, 4), row(0, 3, 4), row(0, 2, 3), row(0, 2, 4)
+ );
+ assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b,c)>=(?,?) AND (b,c)<=(?,?) ALLOW FILTERING", 0, 2, 3, 4, 5),
+ row(0, 4, 4), row(0, 4, 5), row(0, 3, 4), row(0, 2, 3), row(0, 2, 4)
+ );
+ assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b,c)<(?,?) ALLOW FILTERING", 0, 4, 5),
+ row(0, 4, 4), row(0, 3, 4), row(0, 2, 3), row(0, 2, 4)
+ );
- assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b,c)>(?,?) ALLOW FILTERING", 0, 4, 5),
- row(0, 4, 6)
- );
+ assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b,c)>(?,?) ALLOW FILTERING", 0, 4, 5),
+ row(0, 4, 6)
+ );
- assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b)<(?) and (b)>(?) ALLOW FILTERING", 0, 4, 2),
- row(0, 3, 4)
- );
- }
+ assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b)<(?) and (b)>(?) ALLOW FILTERING", 0, 4, 2),
+ row(0, 3, 4)
+ );
}
@Test
public void testMixedOrderColumns4() throws Throwable
{
- for (String compactOption : new String[]{"", " COMPACT STORAGE AND "})
- {
- createTable("CREATE TABLE %s (a int, b int, c int, d int, e int, PRIMARY KEY (a, b, c, d, e)) WITH " +
- compactOption +
- "CLUSTERING ORDER BY (b ASC, c DESC, d DESC, e ASC)");
+ createTable("CREATE TABLE %s (a int, b int, c int, d int, e int, PRIMARY KEY (a, b, c, d, e)) WITH " +
+ "CLUSTERING ORDER BY (b ASC, c DESC, d DESC, e ASC)");
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 2, 0, -1, 0);
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 2, 0, -1, 1);
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 2, 0, 1, 1);
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 2, -1, 1, 1);
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 2, -3, 1, 1);
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, -1, 0, 0);
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, -1, 1, 1);
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, -1, 1, 0);
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, 0, 1, -1);
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, 0, 1, 1);
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, 0, 0, -1);
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, 0, 0, 0);
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, 0, 0, 1);
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, 0, -1, -1);
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, 1, 0, -1);
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, 1, 0, 0);
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, 1, 0, -1);
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, 1, 0, 0);
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, 1, 0, 1);
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, 1, -1, 0);
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 0, 0, 0, 0);
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, -1, 0, -1, 0);
- execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, -1, 0, 0, 0);
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 2, 0, -1, 0);
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 2, 0, -1, 1);
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 2, 0, 1, 1);
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 2, -1, 1, 1);
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 2, -3, 1, 1);
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, -1, 0, 0);
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, -1, 1, 1);
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, -1, 1, 0);
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, 0, 1, -1);
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, 0, 1, 1);
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, 0, 0, -1);
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, 0, 0, 0);
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, 0, 0, 1);
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, 0, -1, -1);
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, 1, 0, -1);
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, 1, 0, 0);
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, 1, 0, -1);
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, 1, 0, 0);
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, 1, 0, 1);
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 1, 1, -1, 0);
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, 0, 0, 0, 0);
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, -1, 0, -1, 0);
+ execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, -1, 0, 0, 0);
- assertRows(execute(
- "SELECT * FROM %s" +
- " WHERE a = ? " +
- "AND (b,c,d,e)<(?,?,?,?) " +
- "AND (b,c,d,e)>(?,?,?,?)", 0, 2, 0, 1, 1, -1, 0, -1, -1),
+ assertRows(execute(
+ "SELECT * FROM %s" +
+ " WHERE a = ? " +
+ "AND (b,c,d,e)<(?,?,?,?) " +
+ "AND (b,c,d,e)>(?,?,?,?)", 0, 2, 0, 1, 1, -1, 0, -1, -1),
- row(0, -1, 0, 0, 0),
- row(0, -1, 0, -1, 0),
- row(0, 0, 0, 0, 0),
- row(0, 1, 1, 0, -1),
- row(0, 1, 1, 0, 0),
- row(0, 1, 1, 0, 1),
- row(0, 1, 1, -1, 0),
- row(0, 1, 0, 1, -1),
- row(0, 1, 0, 1, 1),
- row(0, 1, 0, 0, -1),
- row(0, 1, 0, 0, 0),
- row(0, 1, 0, 0, 1),
- row(0, 1, 0, -1, -1),
- row(0, 1, -1, 1, 0),
- row(0, 1, -1, 1, 1),
- row(0, 1, -1, 0, 0),
- row(0, 2, 0, -1, 0),
- row(0, 2, 0, -1, 1),
- row(0, 2, -1, 1, 1),
- row(0, 2, -3, 1, 1)
+ row(0, -1, 0, 0, 0),
+ row(0, -1, 0, -1, 0),
+ row(0, 0, 0, 0, 0),
+ row(0, 1, 1, 0, -1),
+ row(0, 1, 1, 0, 0),
+ row(0, 1, 1, 0, 1),
+ row(0, 1, 1, -1, 0),
+ row(0, 1, 0, 1, -1),
+ row(0, 1, 0, 1, 1),
+ row(0, 1, 0, 0, -1),
+ row(0, 1, 0, 0, 0),
+ row(0, 1, 0, 0, 1),
+ row(0, 1, 0, -1, -1),
+ row(0, 1, -1, 1, 0),
+ row(0, 1, -1, 1, 1),
+ row(0, 1, -1, 0, 0),
+ row(0, 2, 0, -1, 0),
+ row(0, 2, 0, -1, 1),
+ row(0, 2, -1, 1, 1),
+ row(0, 2, -3, 1, 1)
- );
+ );
- assertRows(execute(
- "SELECT * FROM %s" +
- " WHERE a = ? " +
- "AND (b,c,d,e) < (?,?,?,?) " +
- "AND (b,c,d,e)>(?,?,?,?)", 0, 1, 0, 0, 0, 1, 0, -1, -1),
- row(0, 1, 0, 0, -1)
- );
+ assertRows(execute(
+ "SELECT * FROM %s" +
+ " WHERE a = ? " +
+ "AND (b,c,d,e) < (?,?,?,?) " +
+ "AND (b,c,d,e)>(?,?,?,?)", 0, 1, 0, 0, 0, 1, 0, -1, -1),
+ row(0, 1, 0, 0, -1)
+ );
- assertRows(execute(
- "SELECT * FROM %s" +
- " WHERE a = ? " +
- "AND (b,c,d,e) <= (?,?,?,?) " +
- "AND (b,c,d,e)>(?,?,?,?)", 0, 1, 0, 0, 0, 1, 0, -1, -1),
- row(0, 1, 0, 0, -1),
- row(0, 1, 0, 0, 0)
- );
+ assertRows(execute(
+ "SELECT * FROM %s" +
+ " WHERE a = ? " +
+ "AND (b,c,d,e) <= (?,?,?,?) " +
+ "AND (b,c,d,e)>(?,?,?,?)", 0, 1, 0, 0, 0, 1, 0, -1, -1),
+ row(0, 1, 0, 0, -1),
+ row(0, 1, 0, 0, 0)
+ );
- assertRows(execute(
- "SELECT * FROM %s" +
- " WHERE a = ? " +
- "AND (b,c,d,e)<=(?,?,?,?) " +
- "AND (b,c,d,e)>(?,?,?,?)", 0, 2, 0, 1, 1, -1, 0, -1, -1),
+ assertRows(execute(
+ "SELECT * FROM %s" +
+ " WHERE a = ? " +
+ "AND (b,c,d,e)<=(?,?,?,?) " +
+ "AND (b,c,d,e)>(?,?,?,?)", 0, 2, 0, 1, 1, -1, 0, -1, -1),
- row(0, -1, 0, 0, 0),
- row(0, -1, 0, -1, 0),
- row(0, 0, 0, 0, 0),
- row(0, 1, 1, 0, -1),
- row(0, 1, 1, 0, 0),
- row(0, 1, 1, 0, 1),
- row(0, 1, 1, -1, 0),
- row(0, 1, 0, 1, -1),
- row(0, 1, 0, 1, 1),
- row(0, 1, 0, 0, -1),
- row(0, 1, 0, 0, 0),
- row(0, 1, 0, 0, 1),
- row(0, 1, 0, -1, -1),
- row(0, 1, -1, 1, 0),
- row(0, 1, -1, 1, 1),
- row(0, 1, -1, 0, 0),
- row(0, 2, 0, 1, 1),
- row(0, 2, 0, -1, 0),
- row(0, 2, 0, -1, 1),
- row(0, 2, -1, 1, 1),
- row(0, 2, -3, 1, 1)
- );
+ row(0, -1, 0, 0, 0),
+ row(0, -1, 0, -1, 0),
+ row(0, 0, 0, 0, 0),
+ row(0, 1, 1, 0, -1),
+ row(0, 1, 1, 0, 0),
+ row(0, 1, 1, 0, 1),
+ row(0, 1, 1, -1, 0),
+ row(0, 1, 0, 1, -1),
+ row(0, 1, 0, 1, 1),
+ row(0, 1, 0, 0, -1),
+ row(0, 1, 0, 0, 0),
+ row(0, 1, 0, 0, 1),
+ row(0, 1, 0, -1, -1),
+ row(0, 1, -1, 1, 0),
+ row(0, 1, -1, 1, 1),
+ row(0, 1, -1, 0, 0),
+ row(0, 2, 0, 1, 1),
+ row(0, 2, 0, -1, 0),
+ row(0, 2, 0, -1, 1),
+ row(0, 2, -1, 1, 1),
+ row(0, 2, -3, 1, 1)
+ );
- assertRows(execute(
- "SELECT * FROM %s" +
- " WHERE a = ? " +
- "AND (b,c)<=(?,?) " +
- "AND (b,c,d,e)>(?,?,?,?)", 0, 2, 0, -1, 0, -1, -1),
+ assertRows(execute(
+ "SELECT * FROM %s" +
+ " WHERE a = ? " +
+ "AND (b,c)<=(?,?) " +
+ "AND (b,c,d,e)>(?,?,?,?)", 0, 2, 0, -1, 0, -1, -1),
- row(0, -1, 0, 0, 0),
- row(0, -1, 0, -1, 0),
- row(0, 0, 0, 0, 0),
- row(0, 1, 1, 0, -1),
- row(0, 1, 1, 0, 0),
- row(0, 1, 1, 0, 1),
- row(0, 1, 1, -1, 0),
- row(0, 1, 0, 1, -1),
- row(0, 1, 0, 1, 1),
- row(0, 1, 0, 0, -1),
- row(0, 1, 0, 0, 0),
- row(0, 1, 0, 0, 1),
- row(0, 1, 0, -1, -1),
- row(0, 1, -1, 1, 0),
- row(0, 1, -1, 1, 1),
- row(0, 1, -1, 0, 0),
- row(0, 2, 0, 1, 1),
- row(0, 2, 0, -1, 0),
- row(0, 2, 0, -1, 1),
- row(0, 2, -1, 1, 1),
- row(0, 2, -3, 1, 1)
- );
+ row(0, -1, 0, 0, 0),
+ row(0, -1, 0, -1, 0),
+ row(0, 0, 0, 0, 0),
+ row(0, 1, 1, 0, -1),
+ row(0, 1, 1, 0, 0),
+ row(0, 1, 1, 0, 1),
+ row(0, 1, 1, -1, 0),
+ row(0, 1, 0, 1, -1),
+ row(0, 1, 0, 1, 1),
+ row(0, 1, 0, 0, -1),
+ row(0, 1, 0, 0, 0),
+ row(0, 1, 0, 0, 1),
+ row(0, 1, 0, -1, -1),
+ row(0, 1, -1, 1, 0),
+ row(0, 1, -1, 1, 1),
+ row(0, 1, -1, 0, 0),
+ row(0, 2, 0, 1, 1),
+ row(0, 2, 0, -1, 0),
+ row(0, 2, 0, -1, 1),
+ row(0, 2, -1, 1, 1),
+ row(0, 2, -3, 1, 1)
+ );
- assertRows(execute(
- "SELECT * FROM %s" +
- " WHERE a = ? " +
- "AND (b,c)<(?,?) " +
- "AND (b,c,d,e)>(?,?,?,?)", 0, 2, 0, -1, 0, -1, -1),
- row(0, -1, 0, 0, 0),
- row(0, -1, 0, -1, 0),
- row(0, 0, 0, 0, 0),
- row(0, 1, 1, 0, -1),
- row(0, 1, 1, 0, 0),
- row(0, 1, 1, 0, 1),
- row(0, 1, 1, -1, 0),
- row(0, 1, 0, 1, -1),
- row(0, 1, 0, 1, 1),
- row(0, 1, 0, 0, -1),
- row(0, 1, 0, 0, 0),
- row(0, 1, 0, 0, 1),
- row(0, 1, 0, -1, -1),
- row(0, 1, -1, 1, 0),
- row(0, 1, -1, 1, 1),
- row(0, 1, -1, 0, 0),
- row(0, 2, -1, 1, 1),
- row(0, 2, -3, 1, 1)
- );
+ assertRows(execute(
+ "SELECT * FROM %s" +
+ " WHERE a = ? " +
+ "AND (b,c)<(?,?) " +
+ "AND (b,c,d,e)>(?,?,?,?)", 0, 2, 0, -1, 0, -1, -1),
+ row(0, -1, 0, 0, 0),
+ row(0, -1, 0, -1, 0),
+ row(0, 0, 0, 0, 0),
+ row(0, 1, 1, 0, -1),
+ row(0, 1, 1, 0, 0),
+ row(0, 1, 1, 0, 1),
+ row(0, 1, 1, -1, 0),
+ row(0, 1, 0, 1, -1),
+ row(0, 1, 0, 1, 1),
+ row(0, 1, 0, 0, -1),
+ row(0, 1, 0, 0, 0),
+ row(0, 1, 0, 0, 1),
+ row(0, 1, 0, -1, -1),
+ row(0, 1, -1, 1, 0),
+ row(0, 1, -1, 1, 1),
+ row(0, 1, -1, 0, 0),
+ row(0, 2, -1, 1, 1),
+ row(0, 2, -3, 1, 1)
+ );
- assertRows(execute(
- "SELECT * FROM %s" +
- " WHERE a = ? " +
- "AND (b,c,d,e)<=(?,?,?,?) " +
- "AND (b)>=(?)", 0, 2, 0, 1, 1, -1),
+ assertRows(execute(
+ "SELECT * FROM %s" +
+ " WHERE a = ? " +
+ "AND (b,c,d,e)<=(?,?,?,?) " +
+ "AND (b)>=(?)", 0, 2, 0, 1, 1, -1),
- row(0, -1, 0, 0, 0),
- row(0, -1, 0, -1, 0),
- row(0, 0, 0, 0, 0),
- row(0, 1, 1, 0, -1),
- row(0, 1, 1, 0, 0),
- row(0, 1, 1, 0, 1),
- row(0, 1, 1, -1, 0),
- row(0, 1, 0, 1, -1),
- row(0, 1, 0, 1, 1),
- row(0, 1, 0, 0, -1),
- row(0, 1, 0, 0, 0),
- row(0, 1, 0, 0, 1),
- row(0, 1, 0, -1, -1),
- row(0, 1, -1, 1, 0),
- row(0, 1, -1, 1, 1),
- row(0, 1, -1, 0, 0),
- row(0, 2, 0, 1, 1),
- row(0, 2, 0, -1, 0),
- row(0, 2, 0, -1, 1),
- row(0, 2, -1, 1, 1),
- row(0, 2, -3, 1, 1)
- );
+ row(0, -1, 0, 0, 0),
+ row(0, -1, 0, -1, 0),
+ row(0, 0, 0, 0, 0),
+ row(0, 1, 1, 0, -1),
+ row(0, 1, 1, 0, 0),
+ row(0, 1, 1, 0, 1),
+ row(0, 1, 1, -1, 0),
+ row(0, 1, 0, 1, -1),
+ row(0, 1, 0, 1, 1),
+ row(0, 1, 0, 0, -1),
+ row(0, 1, 0, 0, 0),
+ row(0, 1, 0, 0, 1),
+ row(0, 1, 0, -1, -1),
+ row(0, 1, -1, 1, 0),
+ row(0, 1, -1, 1, 1),
+ row(0, 1, -1, 0, 0),
+ row(0, 2, 0, 1, 1),
+ row(0, 2, 0, -1, 0),
+ row(0, 2, 0, -1, 1),
+ row(0, 2, -1, 1, 1),
+ row(0, 2, -3, 1, 1)
+ );
- assertRows(execute(
- "SELECT * FROM %s" +
- " WHERE a = ? " +
- "AND (b,c,d,e)<=(?,?,?,?) " +
- "AND (b)>(?)", 0, 2, 0, 1, 1, -1),
+ assertRows(execute(
+ "SELECT * FROM %s" +
+ " WHERE a = ? " +
+ "AND (b,c,d,e)<=(?,?,?,?) " +
+ "AND (b)>(?)", 0, 2, 0, 1, 1, -1),
- row(0, 0, 0, 0, 0),
- row(0, 1, 1, 0, -1),
- row(0, 1, 1, 0, 0),
- row(0, 1, 1, 0, 1),
- row(0, 1, 1, -1, 0),
- row(0, 1, 0, 1, -1),
- row(0, 1, 0, 1, 1),
- row(0, 1, 0, 0, -1),
- row(0, 1, 0, 0, 0),
- row(0, 1, 0, 0, 1),
- row(0, 1, 0, -1, -1),
- row(0, 1, -1, 1, 0),
- row(0, 1, -1, 1, 1),
- row(0, 1, -1, 0, 0),
- row(0, 2, 0, 1, 1),
- row(0, 2, 0, -1, 0),
- row(0, 2, 0, -1, 1),
- row(0, 2, -1, 1, 1),
- row(0, 2, -3, 1, 1)
- );
+ row(0, 0, 0, 0, 0),
+ row(0, 1, 1, 0, -1),
+ row(0, 1, 1, 0, 0),
+ row(0, 1, 1, 0, 1),
+ row(0, 1, 1, -1, 0),
+ row(0, 1, 0, 1, -1),
+ row(0, 1, 0, 1, 1),
+ row(0, 1, 0, 0, -1),
+ row(0, 1, 0, 0, 0),
+ row(0, 1, 0, 0, 1),
+ row(0, 1, 0, -1, -1),
+ row(0, 1, -1, 1, 0),
+ row(0, 1, -1, 1, 1),
+ row(0, 1, -1, 0, 0),
+ row(0, 2, 0, 1, 1),
+ row(0, 2, 0, -1, 0),
+ row(0, 2, 0, -1, 1),
+ row(0, 2, -1, 1, 1),
+ row(0, 2, -3, 1, 1)
+ );
- assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b,c,d,e) <= (?,?,?,?)", 0, 1, 0, 0, 0),
- row(0, -1, 0, 0, 0),
- row(0, -1, 0, -1, 0),
- row(0, 0, 0, 0, 0),
- row(0, 1, 0, 0, -1),
- row(0, 1, 0, 0, 0),
- row(0, 1, 0, -1, -1),
- row(0, 1, -1, 1, 0),
- row(0, 1, -1, 1, 1),
- row(0, 1, -1, 0, 0)
- );
+ assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b,c,d,e) <= (?,?,?,?)", 0, 1, 0, 0, 0),
+ row(0, -1, 0, 0, 0),
+ row(0, -1, 0, -1, 0),
+ row(0, 0, 0, 0, 0),
+ row(0, 1, 0, 0, -1),
+ row(0, 1, 0, 0, 0),
+ row(0, 1, 0, -1, -1),
+ row(0, 1, -1, 1, 0),
+ row(0, 1, -1, 1, 1),
+ row(0, 1, -1, 0, 0)
+ );
- assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b,c,d,e) > (?,?,?,?)", 0, 1, 0, 0, 0),
- row(0, 1, 1, 0, -1),
- row(0, 1, 1, 0, 0),
- row(0, 1, 1, 0, 1),
- row(0, 1, 1, -1, 0),
- row(0, 1, 0, 1, -1),
- row(0, 1, 0, 1, 1),
- row(0, 1, 0, 0, 1),
- row(0, 2, 0, 1, 1),
- row(0, 2, 0, -1, 0),
- row(0, 2, 0, -1, 1),
- row(0, 2, -1, 1, 1),
- row(0, 2, -3, 1, 1)
+ assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b,c,d,e) > (?,?,?,?)", 0, 1, 0, 0, 0),
+ row(0, 1, 1, 0, -1),
+ row(0, 1, 1, 0, 0),
+ row(0, 1, 1, 0, 1),
+ row(0, 1, 1, -1, 0),
+ row(0, 1, 0, 1, -1),
+ row(0, 1, 0, 1, 1),
+ row(0, 1, 0, 0, 1),
+ row(0, 2, 0, 1, 1),
+ row(0, 2, 0, -1, 0),
+ row(0, 2, 0, -1, 1),
+ row(0, 2, -1, 1, 1),
+ row(0, 2, -3, 1, 1)
- );
+ );
- assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b,c,d,e) >= (?,?,?,?)", 0, 1, 0, 0, 0),
- row(0, 1, 1, 0, -1),
- row(0, 1, 1, 0, 0),
- row(0, 1, 1, 0, 1),
- row(0, 1, 1, -1, 0),
- row(0, 1, 0, 1, -1),
- row(0, 1, 0, 1, 1),
- row(0, 1, 0, 0, 0),
- row(0, 1, 0, 0, 1),
- row(0, 2, 0, 1, 1),
- row(0, 2, 0, -1, 0),
- row(0, 2, 0, -1, 1),
- row(0, 2, -1, 1, 1),
- row(0, 2, -3, 1, 1)
- );
+ assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b,c,d,e) >= (?,?,?,?)", 0, 1, 0, 0, 0),
+ row(0, 1, 1, 0, -1),
+ row(0, 1, 1, 0, 0),
+ row(0, 1, 1, 0, 1),
+ row(0, 1, 1, -1, 0),
+ row(0, 1, 0, 1, -1),
+ row(0, 1, 0, 1, 1),
+ row(0, 1, 0, 0, 0),
+ row(0, 1, 0, 0, 1),
+ row(0, 2, 0, 1, 1),
+ row(0, 2, 0, -1, 0),
+ row(0, 2, 0, -1, 1),
+ row(0, 2, -1, 1, 1),
+ row(0, 2, -3, 1, 1)
+ );
- assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b,c,d) >= (?,?,?)", 0, 1, 0, 0),
- row(0, 1, 1, 0, -1),
- row(0, 1, 1, 0, 0),
- row(0, 1, 1, 0, 1),
- row(0, 1, 1, -1, 0),
- row(0, 1, 0, 1, -1),
- row(0, 1, 0, 1, 1),
- row(0, 1, 0, 0, -1),
- row(0, 1, 0, 0, 0),
- row(0, 1, 0, 0, 1),
- row(0, 2, 0, 1, 1),
- row(0, 2, 0, -1, 0),
- row(0, 2, 0, -1, 1),
- row(0, 2, -1, 1, 1),
- row(0, 2, -3, 1, 1)
- );
+ assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b,c,d) >= (?,?,?)", 0, 1, 0, 0),
+ row(0, 1, 1, 0, -1),
+ row(0, 1, 1, 0, 0),
+ row(0, 1, 1, 0, 1),
+ row(0, 1, 1, -1, 0),
+ row(0, 1, 0, 1, -1),
+ row(0, 1, 0, 1, 1),
+ row(0, 1, 0, 0, -1),
+ row(0, 1, 0, 0, 0),
+ row(0, 1, 0, 0, 1),
+ row(0, 2, 0, 1, 1),
+ row(0, 2, 0, -1, 0),
+ row(0, 2, 0, -1, 1),
+ row(0, 2, -1, 1, 1),
+ row(0, 2, -3, 1, 1)
+ );
- assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b,c,d) > (?,?,?)", 0, 1, 0, 0),
- row(0, 1, 1, 0, -1),
- row(0, 1, 1, 0, 0),
- row(0, 1, 1, 0, 1),
- row(0, 1, 1, -1, 0),
- row(0, 1, 0, 1, -1),
- row(0, 1, 0, 1, 1),
- row(0, 2, 0, 1, 1),
- row(0, 2, 0, -1, 0),
- row(0, 2, 0, -1, 1),
- row(0, 2, -1, 1, 1),
- row(0, 2, -3, 1, 1)
- );
+ assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b,c,d) > (?,?,?)", 0, 1, 0, 0),
+ row(0, 1, 1, 0, -1),
+ row(0, 1, 1, 0, 0),
+ row(0, 1, 1, 0, 1),
+ row(0, 1, 1, -1, 0),
+ row(0, 1, 0, 1, -1),
+ row(0, 1, 0, 1, 1),
+ row(0, 2, 0, 1, 1),
+ row(0, 2, 0, -1, 0),
+ row(0, 2, 0, -1, 1),
+ row(0, 2, -1, 1, 1),
+ row(0, 2, -3, 1, 1)
+ );
- assertRows(execute(
- "SELECT * FROM %s" +
- " WHERE a = ? " +
- "AND (b) < (?) ", 0, 0),
- row(0, -1, 0, 0, 0), row(0, -1, 0, -1, 0)
- );
- assertRows(execute(
- "SELECT * FROM %s" +
- " WHERE a = ? " +
- "AND (b) <= (?) ", 0, -1),
- row(0, -1, 0, 0, 0), row(0, -1, 0, -1, 0)
- );
- assertRows(execute(
- "SELECT * FROM %s" +
- " WHERE a = ? " +
- "AND (b,c,d,e) < (?,?,?,?) and (b,c,d,e) > (?,?,?,?) ", 0, 2, 0, 0, 0, 2, -2, 0, 0),
- row(0, 2, 0, -1, 0),
- row(0, 2, 0, -1, 1),
- row(0, 2, -1, 1, 1)
- );
- }
+ assertRows(execute(
+ "SELECT * FROM %s" +
+ " WHERE a = ? " +
+ "AND (b) < (?) ", 0, 0),
+ row(0, -1, 0, 0, 0), row(0, -1, 0, -1, 0)
+ );
+ assertRows(execute(
+ "SELECT * FROM %s" +
+ " WHERE a = ? " +
+ "AND (b) <= (?) ", 0, -1),
+ row(0, -1, 0, 0, 0), row(0, -1, 0, -1, 0)
+ );
+ assertRows(execute(
+ "SELECT * FROM %s" +
+ " WHERE a = ? " +
+ "AND (b,c,d,e) < (?,?,?,?) and (b,c,d,e) > (?,?,?,?) ", 0, 2, 0, 0, 0, 2, -2, 0, 0),
+ row(0, 2, 0, -1, 0),
+ row(0, 2, 0, -1, 1),
+ row(0, 2, -1, 1, 1)
+ );
}
@Test
diff --git a/test/unit/org/apache/cassandra/cql3/validation/operations/SelectOrderByTest.java b/test/unit/org/apache/cassandra/cql3/validation/operations/SelectOrderByTest.java
index 06aa2fd..8a3ae03 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/operations/SelectOrderByTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/operations/SelectOrderByTest.java
@@ -29,72 +29,66 @@
@Test
public void testNormalSelectionOrderSingleClustering() throws Throwable
{
- for (String descOption : new String[]{"", " WITH CLUSTERING ORDER BY (b DESC)"})
- {
- createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY (a, b))" + descOption);
- execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", 0, 0, 0);
- execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", 0, 1, 1);
- execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", 0, 2, 2);
+ createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY (a, b))");
+ execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", 0, 0, 0);
+ execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", 0, 1, 1);
+ execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", 0, 2, 2);
- beforeAndAfterFlush(() -> {
- assertRows(execute("SELECT * FROM %s WHERE a=? ORDER BY b ASC", 0),
- row(0, 0, 0),
- row(0, 1, 1),
- row(0, 2, 2)
- );
+ beforeAndAfterFlush(() -> {
+ assertRows(execute("SELECT * FROM %s WHERE a=? ORDER BY b ASC", 0),
+ row(0, 0, 0),
+ row(0, 1, 1),
+ row(0, 2, 2)
+ );
- assertRows(execute("SELECT * FROM %s WHERE a=? ORDER BY b DESC", 0),
- row(0, 2, 2),
- row(0, 1, 1),
- row(0, 0, 0)
- );
+ assertRows(execute("SELECT * FROM %s WHERE a=? ORDER BY b DESC", 0),
+ row(0, 2, 2),
+ row(0, 1, 1),
+ row(0, 0, 0)
+ );
- // order by the only column in the selection
- assertRows(execute("SELECT b FROM %s WHERE a=? ORDER BY b ASC", 0),
- row(0), row(1), row(2));
+ // order by the only column in the selection
+ assertRows(execute("SELECT b FROM %s WHERE a=? ORDER BY b ASC", 0),
+ row(0), row(1), row(2));
- assertRows(execute("SELECT b FROM %s WHERE a=? ORDER BY b DESC", 0),
- row(2), row(1), row(0));
+ assertRows(execute("SELECT b FROM %s WHERE a=? ORDER BY b DESC", 0),
+ row(2), row(1), row(0));
- // order by a column not in the selection
- assertRows(execute("SELECT c FROM %s WHERE a=? ORDER BY b ASC", 0),
- row(0), row(1), row(2));
+ // order by a column not in the selection
+ assertRows(execute("SELECT c FROM %s WHERE a=? ORDER BY b ASC", 0),
+ row(0), row(1), row(2));
- assertRows(execute("SELECT c FROM %s WHERE a=? ORDER BY b DESC", 0),
- row(2), row(1), row(0));
- });
- }
+ assertRows(execute("SELECT c FROM %s WHERE a=? ORDER BY b DESC", 0),
+ row(2), row(1), row(0));
+ });
}
@Test
public void testFunctionSelectionOrderSingleClustering() throws Throwable
{
- for (String descOption : new String[]{"", " WITH CLUSTERING ORDER BY (b DESC)"})
- {
- createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY (a, b))" + descOption);
- execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", 0, 0, 0);
- execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", 0, 1, 1);
- execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", 0, 2, 2);
+ createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY (a, b))");
+ execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", 0, 0, 0);
+ execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", 0, 1, 1);
+ execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", 0, 2, 2);
- beforeAndAfterFlush(() -> {
- // order by the only column in the selection
- assertRows(execute("SELECT blobAsInt(intAsBlob(b)) FROM %s WHERE a=? ORDER BY b ASC", 0),
- row(0), row(1), row(2));
+ beforeAndAfterFlush(() -> {
+ // order by the only column in the selection
+ assertRows(execute("SELECT blobAsInt(intAsBlob(b)) FROM %s WHERE a=? ORDER BY b ASC", 0),
+ row(0), row(1), row(2));
- assertRows(execute("SELECT blobAsInt(intAsBlob(b)) FROM %s WHERE a=? ORDER BY b DESC", 0),
- row(2), row(1), row(0));
+ assertRows(execute("SELECT blobAsInt(intAsBlob(b)) FROM %s WHERE a=? ORDER BY b DESC", 0),
+ row(2), row(1), row(0));
- // order by a column not in the selection
- assertRows(execute("SELECT blobAsInt(intAsBlob(c)) FROM %s WHERE a=? ORDER BY b ASC", 0),
- row(0), row(1), row(2));
+ // order by a column not in the selection
+ assertRows(execute("SELECT blobAsInt(intAsBlob(c)) FROM %s WHERE a=? ORDER BY b ASC", 0),
+ row(0), row(1), row(2));
- assertRows(execute("SELECT blobAsInt(intAsBlob(c)) FROM %s WHERE a=? ORDER BY b DESC", 0),
- row(2), row(1), row(0));
+ assertRows(execute("SELECT blobAsInt(intAsBlob(c)) FROM %s WHERE a=? ORDER BY b DESC", 0),
+ row(2), row(1), row(0));
- assertInvalid("SELECT * FROM %s WHERE a=? ORDER BY c ASC", 0);
- assertInvalid("SELECT * FROM %s WHERE a=? ORDER BY c DESC", 0);
- });
- }
+ assertInvalid("SELECT * FROM %s WHERE a=? ORDER BY c ASC", 0);
+ assertInvalid("SELECT * FROM %s WHERE a=? ORDER BY c DESC", 0);
+ });
}
@Test
@@ -102,26 +96,23 @@
{
String type = createType("CREATE TYPE %s (a int)");
- for (String descOption : new String[]{"", " WITH CLUSTERING ORDER BY (b DESC)"})
- {
- createTable("CREATE TABLE %s (a int, b int, c frozen<" + type + " >, PRIMARY KEY (a, b))" + descOption);
- execute("INSERT INTO %s (a, b, c) VALUES (?, ?, {a: ?})", 0, 0, 0);
- execute("INSERT INTO %s (a, b, c) VALUES (?, ?, {a: ?})", 0, 1, 1);
- execute("INSERT INTO %s (a, b, c) VALUES (?, ?, {a: ?})", 0, 2, 2);
+ createTable("CREATE TABLE %s (a int, b int, c frozen<" + type + " >, PRIMARY KEY (a, b))");
+ execute("INSERT INTO %s (a, b, c) VALUES (?, ?, {a: ?})", 0, 0, 0);
+ execute("INSERT INTO %s (a, b, c) VALUES (?, ?, {a: ?})", 0, 1, 1);
+ execute("INSERT INTO %s (a, b, c) VALUES (?, ?, {a: ?})", 0, 2, 2);
- beforeAndAfterFlush(() -> {
- // order by a column not in the selection
- assertRows(execute("SELECT c.a FROM %s WHERE a=? ORDER BY b ASC", 0),
- row(0), row(1), row(2));
+ beforeAndAfterFlush(() -> {
+ // order by a column not in the selection
+ assertRows(execute("SELECT c.a FROM %s WHERE a=? ORDER BY b ASC", 0),
+ row(0), row(1), row(2));
- assertRows(execute("SELECT c.a FROM %s WHERE a=? ORDER BY b DESC", 0),
- row(2), row(1), row(0));
+ assertRows(execute("SELECT c.a FROM %s WHERE a=? ORDER BY b DESC", 0),
+ row(2), row(1), row(0));
- assertRows(execute("SELECT blobAsInt(intAsBlob(c.a)) FROM %s WHERE a=? ORDER BY b DESC", 0),
- row(2), row(1), row(0));
- });
- dropTable("DROP TABLE %s");
- }
+ assertRows(execute("SELECT blobAsInt(intAsBlob(c.a)) FROM %s WHERE a=? ORDER BY b DESC", 0),
+ row(2), row(1), row(0));
+ });
+ dropTable("DROP TABLE %s");
}
@Test
@@ -245,77 +236,6 @@
}
/**
- * Check ORDER BY support in SELECT statement
- * migrated from cql_tests.py:TestCQL.order_by_test()
- */
- @Test
- public void testSimpleOrderBy() throws Throwable
- {
- createTable("CREATE TABLE %s (k int, c int, v int, PRIMARY KEY (k, c)) WITH COMPACT STORAGE");
-
- for (int i = 0; i < 10; i++)
- execute("INSERT INTO %s (k, c, v) VALUES (0, ?, ?)", i, i);
-
- beforeAndAfterFlush(() -> {
- assertRows(execute("SELECT v FROM %s WHERE k = 0 ORDER BY c DESC"),
- row(9), row(8), row(7), row(6), row(5), row(4), row(3), row(2), row(1), row(0));
- });
-
- createTable("CREATE TABLE %s (k int, c1 int, c2 int, v int, PRIMARY KEY (k, c1, c2)) WITH COMPACT STORAGE");
-
- for (int i = 0; i < 4; i++)
- for (int j = 0; j < 2; j++)
- execute("INSERT INTO %s (k, c1, c2, v) VALUES (0, ?, ?, ?)", i, j, i * 2 + j);
-
- beforeAndAfterFlush(() -> {
- assertInvalid("SELECT v FROM %s WHERE k = 0 ORDER BY c DESC");
- assertInvalid("SELECT v FROM %s WHERE k = 0 ORDER BY c2 DESC");
- assertInvalid("SELECT v FROM %s WHERE k = 0 ORDER BY k DESC");
-
- assertRows(execute("SELECT v FROM %s WHERE k = 0 ORDER BY c1 DESC"),
- row(7), row(6), row(5), row(4), row(3), row(2), row(1), row(0));
-
- assertRows(execute("SELECT v FROM %s WHERE k = 0 ORDER BY c1"),
- row(0), row(1), row(2), row(3), row(4), row(5), row(6), row(7));
- });
- }
-
- /**
- * More ORDER BY checks (#4160)
- * migrated from cql_tests.py:TestCQL.more_order_by_test()
- */
- @Test
- public void testMoreOrderBy() throws Throwable
- {
- createTable("CREATE TABLE %s (row text, number int, string text, PRIMARY KEY(row, number)) WITH COMPACT STORAGE ");
-
- execute("INSERT INTO %s (row, number, string) VALUES ('row', 1, 'one')");
- execute("INSERT INTO %s (row, number, string) VALUES ('row', 2, 'two')");
- execute("INSERT INTO %s (row, number, string) VALUES ('row', 3, 'three')");
- execute("INSERT INTO %s (row, number, string) VALUES ('row', 4, 'four')");
-
- beforeAndAfterFlush(() -> {
- assertRows(execute("SELECT number FROM %s WHERE row='row' AND number < 3 ORDER BY number ASC"),
- row(1), row(2));
-
- assertRows(execute("SELECT number FROM %s WHERE row='row' AND number >= 3 ORDER BY number ASC"),
- row(3), row(4));
-
- assertRows(execute("SELECT number FROM %s WHERE row='row' AND number < 3 ORDER BY number DESC"),
- row(2), row(1));
-
- assertRows(execute("SELECT number FROM %s WHERE row='row' AND number >= 3 ORDER BY number DESC"),
- row(4), row(3));
-
- assertRows(execute("SELECT number FROM %s WHERE row='row' AND number > 3 ORDER BY number DESC"),
- row(4));
-
- assertRows(execute("SELECT number FROM %s WHERE row='row' AND number <= 3 ORDER BY number DESC"),
- row(3), row(2), row(1));
- });
- }
-
- /**
* Check we don't allow order by on row key (#4246)
* migrated from cql_tests.py:TestCQL.order_by_validation_test()
*/
@@ -455,6 +375,23 @@
}
@Test
+ public void testOrderByForInClauseWithCollectionElementSelection() throws Throwable
+ {
+ createTable("CREATE TABLE %s (pk int, c frozen<set<int>>, v int, PRIMARY KEY (pk, c))");
+
+ execute("INSERT INTO %s (pk, c, v) VALUES (0, {1, 2}, 0)");
+ execute("INSERT INTO %s (pk, c, v) VALUES (0, {1, 2, 3}, 1)");
+ execute("INSERT INTO %s (pk, c, v) VALUES (1, {2, 3}, 2)");
+
+ beforeAndAfterFlush(() -> {
+ assertRows(execute("SELECT c[2], v FROM %s WHERE pk = 0 ORDER BY c"),
+ row(2, 0), row(2, 1));
+ assertRows(execute("SELECT c[2], v FROM %s WHERE pk IN (0, 1) ORDER BY c"),
+ row(2, 0), row(2, 1), row(2, 2));
+ });
+ }
+
+ @Test
public void testOrderByForInClauseWithNullValue() throws Throwable
{
createTable("CREATE TABLE %s (a int, b int, c int, s int static, d int, PRIMARY KEY (a, b, c))");
@@ -702,6 +639,132 @@
}
}
+ /**
+ * Test that ORDER BY columns allow skipping equality-restricted clustering columns, see CASSANDRA-10271.
+ */
+ @Test
+ public void testAllowSkippingEqualityAndSingleValueInRestrictedClusteringColumns() throws Throwable
+ {
+ createTable("CREATE TABLE %s (a int, b int, c int, d int, PRIMARY KEY (a, b, c))");
+ execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 0, 0, 0, 0);
+ execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 0, 0, 1, 1);
+ execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 0, 0, 2, 2);
+ execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 0, 1, 0, 3);
+ execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 0, 1, 1, 4);
+ execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 0, 1, 2, 5);
+
+ assertInvalidMessage("Order by is currently only supported on the clustered columns of the PRIMARY KEY, got d",
+ "SELECT * FROM %s WHERE a=? ORDER BY d DESC", 0);
+
+ assertInvalidMessage("Order by is currently only supported on the clustered columns of the PRIMARY KEY, got d",
+ "SELECT * FROM %s WHERE a=? ORDER BY b ASC, c ASC, d ASC", 0);
+
+ String errorMsg = "Order by currently only supports the ordering of columns following their declared order in the PRIMARY KEY";
+
+ assertRows(execute("SELECT * FROM %s WHERE a=? AND b=? ORDER BY c", 0, 0),
+ row(0, 0, 0, 0),
+ row(0, 0, 1, 1),
+ row(0, 0, 2, 2)
+ );
+
+ assertRows(execute("SELECT * FROM %s WHERE a=? AND b=? ORDER BY c ASC", 0, 0),
+ row(0, 0, 0, 0),
+ row(0, 0, 1, 1),
+ row(0, 0, 2, 2)
+ );
+
+ assertRows(execute("SELECT * FROM %s WHERE a=? AND b=? ORDER BY c DESC", 0, 0),
+ row(0, 0, 2, 2),
+ row(0, 0, 1, 1),
+ row(0, 0, 0, 0)
+ );
+
+ assertRows(execute("SELECT * FROM %s WHERE a=? AND b=? AND c>=? ORDER BY c ASC", 0, 0, 1),
+ row(0, 0, 1, 1),
+ row(0, 0, 2, 2));
+
+ assertRows(execute("SELECT * FROM %s WHERE a=? AND b=? AND c>=? ORDER BY c DESC", 0, 0, 1),
+ row(0, 0, 2, 2),
+ row(0, 0, 1, 1));
+
+ assertRows(execute("SELECT * FROM %s WHERE a=? AND b=? AND c IN (?, ?) ORDER BY c ASC", 0, 0, 1, 2),
+ row(0, 0, 1, 1),
+ row(0, 0, 2, 2));
+
+ assertRows(execute("SELECT * FROM %s WHERE a=? AND b=? AND c IN (?, ?) ORDER BY c DESC", 0, 0, 1, 2),
+ row(0, 0, 2, 2),
+ row(0, 0, 1, 1));
+
+ assertInvalidMessage(errorMsg, "SELECT * FROM %s WHERE a=? AND b<? ORDER BY c DESC", 0, 1);
+
+ assertInvalidMessage(errorMsg, "SELECT * FROM %s WHERE a=? AND (b, c) > (?, ?) ORDER BY c", 0, 0, 0);
+ assertInvalidMessage(errorMsg, "SELECT * FROM %s WHERE a=? AND (b, c) >= (?, ?) ORDER BY c", 0, 0, 0);
+ assertInvalidMessage(errorMsg, "SELECT * FROM %s WHERE a=? AND (b, c) < (?, ?) ORDER BY c", 0, 0, 0);
+ assertInvalidMessage(errorMsg, "SELECT * FROM %s WHERE a=? AND (b, c) <= (?, ?) ORDER BY c", 0, 0, 0);
+ assertRows(execute("SELECT * FROM %s WHERE a=? AND (b, c) = (?, ?) ORDER BY c ASC", 0, 0, 0),
+ row(0, 0, 0, 0));
+ assertRows(execute("SELECT * FROM %s WHERE a=? AND (b, c) = (?, ?) ORDER BY c DESC", 0, 0, 0),
+ row(0, 0, 0, 0));
+
+ assertInvalidMessage(errorMsg, "SELECT * FROM %s WHERE a=? AND (b, c) > ? ORDER BY c", 0, tuple(0, 0));
+ assertInvalidMessage(errorMsg, "SELECT * FROM %s WHERE a=? AND (b, c) >= ? ORDER BY c", 0, tuple(0, 0));
+ assertInvalidMessage(errorMsg, "SELECT * FROM %s WHERE a=? AND (b, c) < ? ORDER BY c", 0, tuple(0, 0));
+ assertInvalidMessage(errorMsg, "SELECT * FROM %s WHERE a=? AND (b, c) <= ? ORDER BY c", 0, tuple(0, 0));
+ assertRows(execute("SELECT * FROM %s WHERE a=? AND (b, c) = ? ORDER BY c ASC", 0, tuple(0, 0)),
+ row(0, 0, 0, 0));
+ assertRows(execute("SELECT * FROM %s WHERE a=? AND (b, c) = ? ORDER BY c DESC", 0, tuple(0, 0)),
+ row(0, 0, 0, 0));
+
+ assertRows(execute("SELECT * FROM %s WHERE a IN (?, ?) AND b=? AND c>=? ORDER BY c ASC", 0, 1, 0, 0),
+ row(0, 0, 0, 0),
+ row(0, 0, 1, 1),
+ row(0, 0, 2, 2));
+
+ assertRows(execute("SELECT * FROM %s WHERE a IN (?, ?) AND b=? AND c>=? ORDER BY c DESC", 0, 1, 0, 0),
+ row(0, 0, 2, 2),
+ row(0, 0, 1, 1),
+ row(0, 0, 0, 0));
+
+ assertRows(execute("SELECT * FROM %s WHERE a IN (?, ?) AND b=? ORDER BY c ASC", 0, 1, 0),
+ row(0, 0, 0, 0),
+ row(0, 0, 1, 1),
+ row(0, 0, 2, 2));
+
+ assertRows(execute("SELECT * FROM %s WHERE a IN (?, ?) AND b=? ORDER BY c DESC", 0, 1, 0),
+ row(0, 0, 2, 2),
+ row(0, 0, 1, 1),
+ row(0, 0, 0, 0));
+
+ assertRows(execute("SELECT * FROM %s WHERE a=? AND b IN (?) ORDER BY c ASC", 0, 1),
+ row(0, 1, 0, 3),
+ row(0, 1, 1, 4),
+ row(0, 1, 2, 5));
+
+ assertRows(execute("SELECT * FROM %s WHERE a=? AND b IN (?) ORDER BY c DESC", 0, 1),
+ row(0, 1, 2, 5),
+ row(0, 1, 1, 4),
+ row(0, 1, 0, 3));
+
+ assertRows(execute("SELECT * FROM %s WHERE a=? AND (b, c) IN ((?, ?)) ORDER BY c ASC", 0, 1, 1),
+ row(0, 1, 1, 4));
+
+ assertRows(execute("SELECT * FROM %s WHERE a=? AND (b, c) IN ((?, ?)) ORDER BY c DESC", 0, 1, 1),
+ row(0, 1, 1, 4));
+
+ assertRows(execute("SELECT * FROM %s WHERE a=? AND b IN (?, ?) AND c=? ORDER BY b ASC", 0, 0, 1, 2),
+ row(0, 0, 2, 2),
+ row(0, 1, 2, 5));
+
+ assertRows(execute("SELECT * FROM %s WHERE a=? AND b IN (?, ?) AND c=? ORDER BY b DESC", 0, 0, 1, 2),
+ row(0, 1, 2, 5),
+ row(0, 0, 2, 2));
+
+ assertInvalidMessage(errorMsg, "SELECT * FROM %s WHERE a=? AND b IN ? ORDER BY c", 0, list(0));
+ assertInvalidMessage(errorMsg, "SELECT * FROM %s WHERE a=? AND b IN (?,?) ORDER BY c", 0, 1, 3);
+ assertInvalidMessage(errorMsg, "SELECT * FROM %s WHERE a=? AND (b,c) IN ? ORDER BY c", 0, list(tuple(0, 0)));
+ assertInvalidMessage(errorMsg, "SELECT * FROM %s WHERE a=? AND (b,c) IN ((?,?), (?,?)) ORDER BY c", 0, 0, 0, 0, 1);
+ }
+
@Test
public void testSelectWithReversedTypeInReverseOrderWithStaticColumnsWithoutStaticRow() throws Throwable
{
diff --git a/test/unit/org/apache/cassandra/cql3/validation/operations/SelectOrderedPartitionerTest.java b/test/unit/org/apache/cassandra/cql3/validation/operations/SelectOrderedPartitionerTest.java
index 003258a..a14a2a4 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/operations/SelectOrderedPartitionerTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/operations/SelectOrderedPartitionerTest.java
@@ -30,7 +30,6 @@
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.CQLTester;
-import org.apache.cassandra.cql3.UntypedResultSet;
import org.apache.cassandra.cql3.restrictions.StatementRestrictions;
import org.apache.cassandra.dht.ByteOrderedPartitioner;
@@ -395,19 +394,6 @@
row(0, 1),
row(1, 0),
row(1, 1));
-
- // Check for dense tables too
- createTable(" CREATE TABLE %s (k int, c int, PRIMARY KEY (k, c)) WITH COMPACT STORAGE");
-
- for (int k = 0; k < 2; k++)
- for (int c = 0; c < 2; c++)
- execute("INSERT INTO %s (k, c) VALUES (?, ?)", k, c);
-
- assertRows(execute("SELECT * FROM %s"),
- row(0, 0),
- row(0, 1),
- row(1, 0),
- row(1, 1));
}
/**
diff --git a/test/unit/org/apache/cassandra/cql3/validation/operations/SelectSingleColumnRelationTest.java b/test/unit/org/apache/cassandra/cql3/validation/operations/SelectSingleColumnRelationTest.java
index 7e5afda..3795ce5 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/operations/SelectSingleColumnRelationTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/operations/SelectSingleColumnRelationTest.java
@@ -77,23 +77,6 @@
execute("insert into %s (a, b, c, d) values (?, ?, ?, ?)", "first", 3, 7, 3);
execute("insert into %s (a, b, c, d) values (?, ?, ?, ?)", "second", 4, 8, 4);
- testSelectQueriesWithClusteringColumnRelations();
- }
-
- @Test
- public void testClusteringColumnRelationsWithCompactStorage() throws Throwable
- {
- createTable("CREATE TABLE %s (a text, b int, c int, d int, primary key(a, b, c)) WITH COMPACT STORAGE;");
- execute("insert into %s (a, b, c, d) values (?, ?, ?, ?)", "first", 1, 5, 1);
- execute("insert into %s (a, b, c, d) values (?, ?, ?, ?)", "first", 2, 6, 2);
- execute("insert into %s (a, b, c, d) values (?, ?, ?, ?)", "first", 3, 7, 3);
- execute("insert into %s (a, b, c, d) values (?, ?, ?, ?)", "second", 4, 8, 4);
-
- testSelectQueriesWithClusteringColumnRelations();
- }
-
- private void testSelectQueriesWithClusteringColumnRelations() throws Throwable
- {
assertRows(execute("select * from %s where a in (?, ?)", "first", "second"),
row("first", 1, 5, 1),
row("first", 2, 6, 2),
@@ -223,7 +206,7 @@
@Test
public void testClusteringColumnRelationsWithClusteringOrder() throws Throwable
{
- createTable("CREATE TABLE %s (a text, b int, c int, d int, primary key(a, b, c)) WITH CLUSTERING ORDER BY (b DESC);");
+ createTable("CREATE TABLE %s (a text, b int, c int, d int, primary key(a, b, c)) WITH CLUSTERING ORDER BY (b DESC, c ASC);");
execute("insert into %s (a, b, c, d) values (?, ?, ?, ?)", "first", 1, 5, 1);
execute("insert into %s (a, b, c, d) values (?, ?, ?, ?)", "first", 2, 6, 2);
execute("insert into %s (a, b, c, d) values (?, ?, ?, ?)", "first", 3, 7, 3);
@@ -424,36 +407,30 @@
@Test
public void testEmptyIN() throws Throwable
{
- for (String compactOption : new String[] { "", " WITH COMPACT STORAGE" })
- {
- createTable("CREATE TABLE %s (k1 int, k2 int, v int, PRIMARY KEY (k1, k2))" + compactOption);
+ createTable("CREATE TABLE %s (k1 int, k2 int, v int, PRIMARY KEY (k1, k2))");
- for (int i = 0; i <= 2; i++)
- for (int j = 0; j <= 2; j++)
- execute("INSERT INTO %s (k1, k2, v) VALUES (?, ?, ?)", i, j, i + j);
+ for (int i = 0; i <= 2; i++)
+ for (int j = 0; j <= 2; j++)
+ execute("INSERT INTO %s (k1, k2, v) VALUES (?, ?, ?)", i, j, i + j);
- assertEmpty(execute("SELECT v FROM %s WHERE k1 IN ()"));
- assertEmpty(execute("SELECT v FROM %s WHERE k1 = 0 AND k2 IN ()"));
- }
+ assertEmpty(execute("SELECT v FROM %s WHERE k1 IN ()"));
+ assertEmpty(execute("SELECT v FROM %s WHERE k1 = 0 AND k2 IN ()"));
}
@Test
public void testINWithDuplicateValue() throws Throwable
{
- for (String compactOption : new String[] { "", " WITH COMPACT STORAGE" })
- {
- createTable("CREATE TABLE %s (k1 int, k2 int, v int, PRIMARY KEY (k1, k2))" + compactOption);
- execute("INSERT INTO %s (k1, k2, v) VALUES (?, ?, ?)", 1, 1, 1);
+ createTable("CREATE TABLE %s (k1 int, k2 int, v int, PRIMARY KEY (k1, k2))");
+ execute("INSERT INTO %s (k1, k2, v) VALUES (?, ?, ?)", 1, 1, 1);
- assertRows(execute("SELECT * FROM %s WHERE k1 IN (?, ?)", 1, 1),
- row(1, 1, 1));
+ assertRows(execute("SELECT * FROM %s WHERE k1 IN (?, ?)", 1, 1),
+ row(1, 1, 1));
- assertRows(execute("SELECT * FROM %s WHERE k1 IN (?, ?) AND k2 IN (?, ?)", 1, 1, 1, 1),
- row(1, 1, 1));
+ assertRows(execute("SELECT * FROM %s WHERE k1 IN (?, ?) AND k2 IN (?, ?)", 1, 1, 1, 1),
+ row(1, 1, 1));
- assertRows(execute("SELECT * FROM %s WHERE k1 = ? AND k2 IN (?, ?)", 1, 1, 1),
- row(1, 1, 1));
- }
+ assertRows(execute("SELECT * FROM %s WHERE k1 = ? AND k2 IN (?, ?)", 1, 1, 1),
+ row(1, 1, 1));
}
@Test
diff --git a/test/unit/org/apache/cassandra/cql3/validation/operations/SelectTest.java b/test/unit/org/apache/cassandra/cql3/validation/operations/SelectTest.java
index 1d45448..d7c1e25 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/operations/SelectTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/operations/SelectTest.java
@@ -60,7 +60,6 @@
);
// Ascending order
-
assertRows(execute("SELECT * FROM %s WHERE p=? ORDER BY c ASC", "p1"),
row("p1", "k1", "sv1", "v1"),
row("p1", "k2", "sv1", "v2")
@@ -313,65 +312,6 @@
assertRowCount(execute("SELECT firstname, lastname FROM %s WHERE userid IN (?, ?)", id1, id2), 2);
}
- /**
- * Check query with KEY IN clause for wide row tables
- * migrated from cql_tests.py:TestCQL.in_clause_wide_rows_test()
- */
- @Test
- public void testSelectKeyInForWideRows() throws Throwable
- {
- createTable("CREATE TABLE %s (k int, c int, v int, PRIMARY KEY (k, c)) WITH COMPACT STORAGE");
-
- for (int i = 0; i < 10; i++)
- execute("INSERT INTO %s (k, c, v) VALUES (0, ?, ?)", i, i);
-
- assertRows(execute("SELECT v FROM %s WHERE k = 0 AND c IN (5, 2, 8)"),
- row(2), row(5), row(8));
-
- createTable("CREATE TABLE %s (k int, c1 int, c2 int, v int, PRIMARY KEY (k, c1, c2)) WITH COMPACT STORAGE");
-
- for (int i = 0; i < 10; i++)
- execute("INSERT INTO %s (k, c1, c2, v) VALUES (0, 0, ?, ?)", i, i);
-
- assertEmpty(execute("SELECT v FROM %s WHERE k = 0 AND c1 IN (5, 2, 8) AND c2 = 3"));
-
- assertRows(execute("SELECT v FROM %s WHERE k = 0 AND c1 = 0 AND c2 IN (5, 2, 8)"),
- row(2), row(5), row(8));
- }
-
- /**
- * Check SELECT respects inclusive and exclusive bounds
- * migrated from cql_tests.py:TestCQL.exclusive_slice_test()
- */
- @Test
- public void testSelectBounds() throws Throwable
- {
- createTable("CREATE TABLE %s (k int, c int, v int, PRIMARY KEY (k, c)) WITH COMPACT STORAGE");
-
- for (int i = 0; i < 10; i++)
- execute("INSERT INTO %s (k, c, v) VALUES (0, ?, ?)", i, i);
-
- assertRowCount(execute("SELECT v FROM %s WHERE k = 0"), 10);
-
- assertRows(execute("SELECT v FROM %s WHERE k = 0 AND c >= 2 AND c <= 6"),
- row(2), row(3), row(4), row(5), row(6));
-
- assertRows(execute("SELECT v FROM %s WHERE k = 0 AND c > 2 AND c <= 6"),
- row(3), row(4), row(5), row(6));
-
- assertRows(execute("SELECT v FROM %s WHERE k = 0 AND c >= 2 AND c < 6"),
- row(2), row(3), row(4), row(5));
-
- assertRows(execute("SELECT v FROM %s WHERE k = 0 AND c > 2 AND c < 6"),
- row(3), row(4), row(5));
-
- assertRows(execute("SELECT v FROM %s WHERE k = 0 AND c > 2 AND c <= 6 LIMIT 2"),
- row(3), row(4));
-
- assertRows(execute("SELECT v FROM %s WHERE k = 0 AND c >= 2 AND c < 6 ORDER BY c DESC LIMIT 2"),
- row(5), row(4));
- }
-
@Test
public void testSetContainsWithIndex() throws Throwable
{
@@ -800,128 +740,6 @@
}
/**
- * Test for #4716 bug and more generally for good behavior of ordering,
- * migrated from cql_tests.py:TestCQL.reversed_compact_test()
- */
- @Test
- public void testReverseCompact() throws Throwable
- {
- createTable("CREATE TABLE %s ( k text, c int, v int, PRIMARY KEY (k, c) ) WITH COMPACT STORAGE AND CLUSTERING ORDER BY (c DESC)");
-
- for (int i = 0; i < 10; i++)
- execute("INSERT INTO %s (k, c, v) VALUES ('foo', ?, ?)", i, i);
-
- assertRows(execute("SELECT c FROM %s WHERE c > 2 AND c < 6 AND k = 'foo'"),
- row(5), row(4), row(3));
-
- assertRows(execute("SELECT c FROM %s WHERE c >= 2 AND c <= 6 AND k = 'foo'"),
- row(6), row(5), row(4), row(3), row(2));
-
- assertRows(execute("SELECT c FROM %s WHERE c > 2 AND c < 6 AND k = 'foo' ORDER BY c ASC"),
- row(3), row(4), row(5));
-
- assertRows(execute("SELECT c FROM %s WHERE c >= 2 AND c <= 6 AND k = 'foo' ORDER BY c ASC"),
- row(2), row(3), row(4), row(5), row(6));
-
- assertRows(execute("SELECT c FROM %s WHERE c > 2 AND c < 6 AND k = 'foo' ORDER BY c DESC"),
- row(5), row(4), row(3));
-
- assertRows(execute("SELECT c FROM %s WHERE c >= 2 AND c <= 6 AND k = 'foo' ORDER BY c DESC"),
- row(6), row(5), row(4), row(3), row(2));
-
- createTable("CREATE TABLE %s ( k text, c int, v int, PRIMARY KEY (k, c) ) WITH COMPACT STORAGE");
-
- for (int i = 0; i < 10; i++)
- execute("INSERT INTO %s(k, c, v) VALUES ('foo', ?, ?)", i, i);
-
- assertRows(execute("SELECT c FROM %s WHERE c > 2 AND c < 6 AND k = 'foo'"),
- row(3), row(4), row(5));
-
- assertRows(execute("SELECT c FROM %s WHERE c >= 2 AND c <= 6 AND k = 'foo'"),
- row(2), row(3), row(4), row(5), row(6));
-
- assertRows(execute("SELECT c FROM %s WHERE c > 2 AND c < 6 AND k = 'foo' ORDER BY c ASC"),
- row(3), row(4), row(5));
-
- assertRows(execute("SELECT c FROM %s WHERE c >= 2 AND c <= 6 AND k = 'foo' ORDER BY c ASC"),
- row(2), row(3), row(4), row(5), row(6));
-
- assertRows(execute("SELECT c FROM %s WHERE c > 2 AND c < 6 AND k = 'foo' ORDER BY c DESC"),
- row(5), row(4), row(3));
-
- assertRows(execute("SELECT c FROM %s WHERE c >= 2 AND c <= 6 AND k = 'foo' ORDER BY c DESC"),
- row(6), row(5), row(4), row(3), row(2));
- }
-
- /**
- * Test for the bug from #4760 and #4759,
- * migrated from cql_tests.py:TestCQL.reversed_compact_multikey_test()
- */
- @Test
- public void testReversedCompactMultikey() throws Throwable
- {
- createTable("CREATE TABLE %s (key text, c1 int, c2 int, value text, PRIMARY KEY(key, c1, c2) ) WITH COMPACT STORAGE AND CLUSTERING ORDER BY(c1 DESC, c2 DESC)");
-
- for (int i = 0; i < 3; i++)
- for (int j = 0; j < 3; j++)
- execute("INSERT INTO %s (key, c1, c2, value) VALUES ('foo', ?, ?, 'bar')", i, j);
-
- // Equalities
- assertRows(execute("SELECT c1, c2 FROM %s WHERE key='foo' AND c1 = 1"),
- row(1, 2), row(1, 1), row(1, 0));
-
- assertRows(execute("SELECT c1, c2 FROM %s WHERE key='foo' AND c1 = 1 ORDER BY c1 ASC, c2 ASC"),
- row(1, 0), row(1, 1), row(1, 2));
-
- assertRows(execute("SELECT c1, c2 FROM %s WHERE key='foo' AND c1 = 1 ORDER BY c1 DESC, c2 DESC"),
- row(1, 2), row(1, 1), row(1, 0));
-
- // GT
- assertRows(execute("SELECT c1, c2 FROM %s WHERE key='foo' AND c1 > 1"),
- row(2, 2), row(2, 1), row(2, 0));
-
- assertRows(execute("SELECT c1, c2 FROM %s WHERE key='foo' AND c1 > 1 ORDER BY c1 ASC, c2 ASC"),
- row(2, 0), row(2, 1), row(2, 2));
-
- assertRows(execute("SELECT c1, c2 FROM %s WHERE key='foo' AND c1 > 1 ORDER BY c1 DESC, c2 DESC"),
- row(2, 2), row(2, 1), row(2, 0));
-
- assertRows(execute("SELECT c1, c2 FROM %s WHERE key='foo' AND c1 >= 1"),
- row(2, 2), row(2, 1), row(2, 0), row(1, 2), row(1, 1), row(1, 0));
-
- assertRows(execute("SELECT c1, c2 FROM %s WHERE key='foo' AND c1 >= 1 ORDER BY c1 ASC, c2 ASC"),
- row(1, 0), row(1, 1), row(1, 2), row(2, 0), row(2, 1), row(2, 2));
-
- assertRows(execute("SELECT c1, c2 FROM %s WHERE key='foo' AND c1 >= 1 ORDER BY c1 ASC"),
- row(1, 0), row(1, 1), row(1, 2), row(2, 0), row(2, 1), row(2, 2));
-
- assertRows(execute("SELECT c1, c2 FROM %s WHERE key='foo' AND c1 >= 1 ORDER BY c1 DESC, c2 DESC"),
- row(2, 2), row(2, 1), row(2, 0), row(1, 2), row(1, 1), row(1, 0));
-
- // LT
- assertRows(execute("SELECT c1, c2 FROM %s WHERE key='foo' AND c1 < 1"),
- row(0, 2), row(0, 1), row(0, 0));
-
- assertRows(execute("SELECT c1, c2 FROM %s WHERE key='foo' AND c1 < 1 ORDER BY c1 ASC, c2 ASC"),
- row(0, 0), row(0, 1), row(0, 2));
-
- assertRows(execute("SELECT c1, c2 FROM %s WHERE key='foo' AND c1 < 1 ORDER BY c1 DESC, c2 DESC"),
- row(0, 2), row(0, 1), row(0, 0));
-
- assertRows(execute("SELECT c1, c2 FROM %s WHERE key='foo' AND c1 <= 1"),
- row(1, 2), row(1, 1), row(1, 0), row(0, 2), row(0, 1), row(0, 0));
-
- assertRows(execute("SELECT c1, c2 FROM %s WHERE key='foo' AND c1 <= 1 ORDER BY c1 ASC, c2 ASC"),
- row(0, 0), row(0, 1), row(0, 2), row(1, 0), row(1, 1), row(1, 2));
-
- assertRows(execute("SELECT c1, c2 FROM %s WHERE key='foo' AND c1 <= 1 ORDER BY c1 ASC"),
- row(0, 0), row(0, 1), row(0, 2), row(1, 0), row(1, 1), row(1, 2));
-
- assertRows(execute("SELECT c1, c2 FROM %s WHERE key='foo' AND c1 <= 1 ORDER BY c1 DESC, c2 DESC"),
- row(1, 2), row(1, 1), row(1, 0), row(0, 2), row(0, 1), row(0, 0));
- }
-
- /**
* Migrated from cql_tests.py:TestCQL.bug_4882_test()
*/
@Test
@@ -1032,26 +850,7 @@
@Test
public void testMultiSelects() throws Throwable
{
- doTestVariousSelects(false);
- }
-
- /**
- * Migrated from cql_tests.py:TestCQL.multi_in_compact_test()
- */
- @Test
- public void testMultiSelectsCompactStorage() throws Throwable
- {
- doTestVariousSelects(true);
- }
-
-
- public void doTestVariousSelects(boolean compact) throws Throwable
- {
- createTable(
- "CREATE TABLE %s (group text, zipcode text, state text, fips_regions int, city text, PRIMARY KEY (group, zipcode, state, fips_regions))"
- + (compact
- ? " WITH COMPACT STORAGE"
- : ""));
+ createTable("CREATE TABLE %s (group text, zipcode text, state text, fips_regions int, city text, PRIMARY KEY (group, zipcode, state, fips_regions))");
String str = "INSERT INTO %s (group, zipcode, state, fips_regions, city) VALUES (?, ?, ?, ?, ?)";
execute(str, "test", "06029", "CT", 9, "Ellington");
@@ -1086,22 +885,6 @@
}
/**
- * Migrated from cql_tests.py:TestCQL.multi_in_compact_non_composite_test()
- */
- @Test
- public void testMultiSelectsNonCompositeCompactStorage() throws Throwable
- {
- createTable("CREATE TABLE %s (key int, c int, v int, PRIMARY KEY (key, c)) WITH COMPACT STORAGE");
-
- execute("INSERT INTO %s (key, c, v) VALUES (0, 0, 0)");
- execute("INSERT INTO %s (key, c, v) VALUES (0, 1, 1)");
- execute("INSERT INTO %s (key, c, v) VALUES (0, 2, 2)");
-
- assertRows(execute("SELECT * FROM %s WHERE key=0 AND c IN (0, 2)"),
- row(0, 0, 0), row(0, 2, 2));
- }
-
- /**
* Migrated from cql_tests.py:TestCQL.ticket_5230_test()
*/
@Test
@@ -1190,7 +973,135 @@
}
/**
->>>>>>> cassandra-3.0
+ * Migrated from cql_tests.py:TestCQL.select_distinct_test()
+ */
+ @Test
+ public void testSelectDistinct() throws Throwable
+ {
+ // Test a regular(CQL3) table.
+ createTable("CREATE TABLE %s (pk0 int, pk1 int, ck0 int, val int, PRIMARY KEY((pk0, pk1), ck0))");
+
+ for (int i = 0; i < 3; i++)
+ {
+ execute("INSERT INTO %s (pk0, pk1, ck0, val) VALUES (?, ?, 0, 0)", i, i);
+ execute("INSERT INTO %s (pk0, pk1, ck0, val) VALUES (?, ?, 1, 1)", i, i);
+ }
+
+ assertRows(execute("SELECT DISTINCT pk0, pk1 FROM %s LIMIT 1"),
+ row(0, 0));
+
+ assertRows(execute("SELECT DISTINCT pk0, pk1 FROM %s LIMIT 3"),
+ row(0, 0),
+ row(2, 2),
+ row(1, 1));
+
+ // Test selection validation.
+ assertInvalidMessage("queries must request all the partition key columns", "SELECT DISTINCT pk0 FROM %s");
+ assertInvalidMessage("queries must only request partition key columns", "SELECT DISTINCT pk0, pk1, ck0 FROM %s");
+ }
+
+ /**
+ * Migrated from cql_tests.py:TestCQL.select_distinct_with_deletions_test()
+ */
+ @Test
+ public void testSelectDistinctWithDeletions() throws Throwable
+ {
+ createTable("CREATE TABLE %s (k int PRIMARY KEY, c int, v int)");
+
+ for (int i = 0; i < 10; i++)
+ execute("INSERT INTO %s (k, c, v) VALUES (?, ?, ?)", i, i, i);
+
+ Object[][] rows = getRows(execute("SELECT DISTINCT k FROM %s"));
+ Assert.assertEquals(10, rows.length);
+ Object key_to_delete = rows[3][0];
+
+ execute("DELETE FROM %s WHERE k=?", key_to_delete);
+
+ rows = getRows(execute("SELECT DISTINCT k FROM %s"));
+ Assert.assertEquals(9, rows.length);
+
+ rows = getRows(execute("SELECT DISTINCT k FROM %s LIMIT 5"));
+ Assert.assertEquals(5, rows.length);
+
+ rows = getRows(execute("SELECT DISTINCT k FROM %s"));
+ Assert.assertEquals(9, rows.length);
+ }
+
+ @Test
+ public void testSelectDistinctWithWhereClause() throws Throwable {
+ createTable("CREATE TABLE %s (k int, a int, b int, PRIMARY KEY (k, a))");
+ createIndex("CREATE INDEX ON %s (b)");
+
+ for (int i = 0; i < 10; i++)
+ {
+ execute("INSERT INTO %s (k, a, b) VALUES (?, ?, ?)", i, i, i);
+ execute("INSERT INTO %s (k, a, b) VALUES (?, ?, ?)", i, i * 10, i * 10);
+ }
+
+ String distinctQueryErrorMsg = "SELECT DISTINCT with WHERE clause only supports restriction by partition key and/or static columns.";
+ assertInvalidMessage(distinctQueryErrorMsg,
+ "SELECT DISTINCT k FROM %s WHERE a >= 80 ALLOW FILTERING");
+
+ assertInvalidMessage(distinctQueryErrorMsg,
+ "SELECT DISTINCT k FROM %s WHERE k IN (1, 2, 3) AND a = 10");
+
+ assertInvalidMessage(distinctQueryErrorMsg,
+ "SELECT DISTINCT k FROM %s WHERE b = 5");
+
+ assertRows(execute("SELECT DISTINCT k FROM %s WHERE k = 1"),
+ row(1));
+ assertRows(execute("SELECT DISTINCT k FROM %s WHERE k IN (5, 6, 7)"),
+ row(5),
+ row(6),
+ row(7));
+
+ // With static columns
+ createTable("CREATE TABLE %s (k int, a int, s int static, b int, PRIMARY KEY (k, a))");
+ createIndex("CREATE INDEX ON %s (b)");
+ for (int i = 0; i < 10; i++)
+ {
+ execute("INSERT INTO %s (k, a, b, s) VALUES (?, ?, ?, ?)", i, i, i, i);
+ execute("INSERT INTO %s (k, a, b, s) VALUES (?, ?, ?, ?)", i, i * 10, i * 10, i * 10);
+ }
+
+ assertRows(execute("SELECT DISTINCT s FROM %s WHERE k = 5"),
+ row(50));
+ assertRows(execute("SELECT DISTINCT s FROM %s WHERE k IN (5, 6, 7)"),
+ row(50),
+ row(60),
+ row(70));
+ }
+
+ @Test
+ public void testSelectDistinctWithWhereClauseOnStaticColumn() throws Throwable
+ {
+ createTable("CREATE TABLE %s (k int, a int, s int static, s1 int static, b int, PRIMARY KEY (k, a))");
+
+ for (int i = 0; i < 10; i++)
+ {
+ execute("INSERT INTO %s (k, a, b, s, s1) VALUES (?, ?, ?, ?, ?)", i, i, i, i, i);
+ execute("INSERT INTO %s (k, a, b, s, s1) VALUES (?, ?, ?, ?, ?)", i, i * 10, i * 10, i * 10, i * 10);
+ }
+
+ execute("INSERT INTO %s (k, a, b, s, s1) VALUES (?, ?, ?, ?, ?)", 2, 10, 10, 10, 10);
+
+ beforeAndAfterFlush(() -> {
+ assertRows(execute("SELECT DISTINCT k, s, s1 FROM %s WHERE s = 90 AND s1 = 90 ALLOW FILTERING"),
+ row(9, 90, 90));
+
+ assertRows(execute("SELECT DISTINCT k, s, s1 FROM %s WHERE s = 90 AND s1 = 90 ALLOW FILTERING"),
+ row(9, 90, 90));
+
+ assertRows(execute("SELECT DISTINCT k, s, s1 FROM %s WHERE s = 10 AND s1 = 10 ALLOW FILTERING"),
+ row(1, 10, 10),
+ row(2, 10, 10));
+
+ assertRows(execute("SELECT DISTINCT k, s, s1 FROM %s WHERE k = 1 AND s = 10 AND s1 = 10 ALLOW FILTERING"),
+ row(1, 10, 10));
+ });
+ }
+
+ /**
* Migrated from cql_tests.py:TestCQL.bug_6327_test()
*/
@Test
@@ -1199,7 +1110,7 @@
createTable("CREATE TABLE %s ( k int, v int, PRIMARY KEY (k, v))");
execute("INSERT INTO %s (k, v) VALUES (0, 0)");
-
+
flush();
assertRows(execute("SELECT v FROM %s WHERE k=0 AND v IN (1, 0)"),
@@ -1431,154 +1342,6 @@
}
@Test
- public void testFilteringOnCompactTablesWithoutIndices() throws Throwable
- {
- //----------------------------------------------
- // Test COMPACT table with clustering columns
- //----------------------------------------------
- createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY (a, b)) WITH COMPACT STORAGE");
-
- execute("INSERT INTO %s (a, b, c) VALUES (1, 2, 4)");
- execute("INSERT INTO %s (a, b, c) VALUES (1, 3, 6)");
- execute("INSERT INTO %s (a, b, c) VALUES (1, 4, 4)");
- execute("INSERT INTO %s (a, b, c) VALUES (2, 3, 7)");
-
- // Adds tomstones
- execute("INSERT INTO %s (a, b, c) VALUES (1, 1, 4)");
- execute("INSERT INTO %s (a, b, c) VALUES (2, 2, 7)");
- execute("DELETE FROM %s WHERE a = 1 AND b = 1");
- execute("DELETE FROM %s WHERE a = 2 AND b = 2");
-
- beforeAndAfterFlush(() -> {
-
- // Checks filtering
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE a = 1 AND b = 4 AND c = 4");
-
- assertRows(execute("SELECT * FROM %s WHERE a = 1 AND b = 4 AND c = 4 ALLOW FILTERING"),
- row(1, 4, 4));
-
- assertInvalidMessage("IN predicates on non-primary-key columns (c) is not yet supported",
- "SELECT * FROM %s WHERE a IN (1, 2) AND c IN (6, 7)");
-
- assertInvalidMessage("IN predicates on non-primary-key columns (c) is not yet supported",
- "SELECT * FROM %s WHERE a IN (1, 2) AND c IN (6, 7) ALLOW FILTERING");
-
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE c > 4");
-
- assertRows(execute("SELECT * FROM %s WHERE c > 4 ALLOW FILTERING"),
- row(1, 3, 6),
- row(2, 3, 7));
-
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE b < 3 AND c <= 4");
-
- assertRows(execute("SELECT * FROM %s WHERE b < 3 AND c <= 4 ALLOW FILTERING"),
- row(1, 2, 4));
-
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE c >= 3 AND c <= 6");
-
- assertRows(execute("SELECT * FROM %s WHERE c >= 3 AND c <= 6 ALLOW FILTERING"),
- row(1, 2, 4),
- row(1, 3, 6),
- row(1, 4, 4));
-
- });
-
- // Checks filtering with null
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE c = null");
- assertInvalidMessage("Unsupported null value for column c",
- "SELECT * FROM %s WHERE c = null ALLOW FILTERING");
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE c > null");
- assertInvalidMessage("Unsupported null value for column c",
- "SELECT * FROM %s WHERE c > null ALLOW FILTERING");
-
- // Checks filtering with unset
- assertInvalidMessage("Unsupported unset value for column c",
- "SELECT * FROM %s WHERE c = ? ALLOW FILTERING",
- unset());
- assertInvalidMessage("Unsupported unset value for column c",
- "SELECT * FROM %s WHERE c > ? ALLOW FILTERING",
- unset());
-
- //----------------------------------------------
- // Test COMPACT table without clustering columns
- //----------------------------------------------
- createTable("CREATE TABLE %s (a int PRIMARY KEY, b int, c int) WITH COMPACT STORAGE");
-
- execute("INSERT INTO %s (a, b, c) VALUES (1, 2, 4)");
- execute("INSERT INTO %s (a, b, c) VALUES (2, 1, 6)");
- execute("INSERT INTO %s (a, b, c) VALUES (3, 2, 4)");
- execute("INSERT INTO %s (a, b, c) VALUES (4, 1, 7)");
-
- // Adds tomstones
- execute("INSERT INTO %s (a, b, c) VALUES (0, 1, 4)");
- execute("INSERT INTO %s (a, b, c) VALUES (5, 2, 7)");
- execute("DELETE FROM %s WHERE a = 0");
- execute("DELETE FROM %s WHERE a = 5");
-
- beforeAndAfterFlush(() -> {
-
- // Checks filtering
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE a = 1 AND b = 4 AND c = 4");
-
- assertRows(execute("SELECT * FROM %s WHERE a = 1 AND b = 2 AND c = 4 ALLOW FILTERING"),
- row(1, 2, 4));
-
- assertInvalidMessage("IN predicates on non-primary-key columns (c) is not yet supported",
- "SELECT * FROM %s WHERE a IN (1, 2) AND c IN (6, 7)");
-
- assertInvalidMessage("IN predicates on non-primary-key columns (c) is not yet supported",
- "SELECT * FROM %s WHERE a IN (1, 2) AND c IN (6, 7) ALLOW FILTERING");
-
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE c > 4");
-
- assertRows(execute("SELECT * FROM %s WHERE c > 4 ALLOW FILTERING"),
- row(2, 1, 6),
- row(4, 1, 7));
-
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE b < 3 AND c <= 4");
-
- assertRows(execute("SELECT * FROM %s WHERE b < 3 AND c <= 4 ALLOW FILTERING"),
- row(1, 2, 4),
- row(3, 2, 4));
-
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE c >= 3 AND c <= 6");
-
- assertRows(execute("SELECT * FROM %s WHERE c >= 3 AND c <= 6 ALLOW FILTERING"),
- row(1, 2, 4),
- row(2, 1, 6),
- row(3, 2, 4));
- });
-
- // Checks filtering with null
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE c = null");
- assertInvalidMessage("Unsupported null value for column c",
- "SELECT * FROM %s WHERE c = null ALLOW FILTERING");
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE c > null");
- assertInvalidMessage("Unsupported null value for column c",
- "SELECT * FROM %s WHERE c > null ALLOW FILTERING");
-
- // // Checks filtering with unset
- assertInvalidMessage("Unsupported unset value for column c",
- "SELECT * FROM %s WHERE c = ? ALLOW FILTERING",
- unset());
- assertInvalidMessage("Unsupported unset value for column c",
- "SELECT * FROM %s WHERE c > ? ALLOW FILTERING",
- unset());
- }
-
- @Test
public void testFilteringWithoutIndicesWithCollections() throws Throwable
{
createTable("CREATE TABLE %s (a int, b int, c list<int>, d set<int>, e map<int, int>, PRIMARY KEY (a, b))");
@@ -1825,331 +1588,12 @@
unset());
}
- @Test
- public void testFilteringOnCompactTablesWithoutIndicesAndWithLists() throws Throwable
- {
- //----------------------------------------------
- // Test COMPACT table with clustering columns
- //----------------------------------------------
- createTable("CREATE TABLE %s (a int, b int, c frozen<list<int>>, PRIMARY KEY (a, b)) WITH COMPACT STORAGE");
-
- execute("INSERT INTO %s (a, b, c) VALUES (1, 2, [4, 2])");
- execute("INSERT INTO %s (a, b, c) VALUES (1, 3, [6, 2])");
- execute("INSERT INTO %s (a, b, c) VALUES (1, 4, [4, 1])");
- execute("INSERT INTO %s (a, b, c) VALUES (2, 3, [7, 1])");
-
- beforeAndAfterFlush(() -> {
-
- // Checks filtering
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE a = 1 AND b = 4 AND c = [4, 1]");
-
- assertRows(execute("SELECT * FROM %s WHERE a = 1 AND b = 4 AND c = [4, 1] ALLOW FILTERING"),
- row(1, 4, list(4, 1)));
-
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE c > [4, 2]");
-
- assertRows(execute("SELECT * FROM %s WHERE c > [4, 2] ALLOW FILTERING"),
- row(1, 3, list(6, 2)),
- row(2, 3, list(7, 1)));
-
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE b <= 3 AND c < [6, 2]");
-
- assertRows(execute("SELECT * FROM %s WHERE b <= 3 AND c < [6, 2] ALLOW FILTERING"),
- row(1, 2, list(4, 2)));
-
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE c >= [4, 2] AND c <= [6, 4]");
-
- assertRows(execute("SELECT * FROM %s WHERE c >= [4, 2] AND c <= [6, 4] ALLOW FILTERING"),
- row(1, 2, list(4, 2)),
- row(1, 3, list(6, 2)));
-
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE c CONTAINS 2");
-
- assertRows(execute("SELECT * FROM %s WHERE c CONTAINS 2 ALLOW FILTERING"),
- row(1, 2, list(4, 2)),
- row(1, 3, list(6, 2)));
-
- assertInvalidMessage("Cannot use CONTAINS KEY on non-map column c",
- "SELECT * FROM %s WHERE c CONTAINS KEY 2 ALLOW FILTERING");
-
- assertRows(execute("SELECT * FROM %s WHERE c CONTAINS 2 AND c CONTAINS 6 ALLOW FILTERING"),
- row(1, 3, list(6, 2)));
- });
-
- // Checks filtering with null
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE c = null");
- assertInvalidMessage("Unsupported null value for column c",
- "SELECT * FROM %s WHERE c = null ALLOW FILTERING");
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE c > null");
- assertInvalidMessage("Unsupported null value for column c",
- "SELECT * FROM %s WHERE c > null ALLOW FILTERING");
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE c CONTAINS null");
- assertInvalidMessage("Unsupported null value for column c",
- "SELECT * FROM %s WHERE c CONTAINS null ALLOW FILTERING");
-
- // Checks filtering with unset
- assertInvalidMessage("Unsupported unset value for column c",
- "SELECT * FROM %s WHERE c = ? ALLOW FILTERING",
- unset());
- assertInvalidMessage("Unsupported unset value for column c",
- "SELECT * FROM %s WHERE c > ? ALLOW FILTERING",
- unset());
- assertInvalidMessage("Unsupported unset value for column c",
- "SELECT * FROM %s WHERE c CONTAINS ? ALLOW FILTERING",
- unset());
-
- //----------------------------------------------
- // Test COMPACT table without clustering columns
- //----------------------------------------------
- createTable("CREATE TABLE %s (a int PRIMARY KEY, b int, c frozen<list<int>>) WITH COMPACT STORAGE");
-
- execute("INSERT INTO %s (a, b, c) VALUES (1, 2, [4, 2])");
- execute("INSERT INTO %s (a, b, c) VALUES (2, 1, [6, 2])");
- execute("INSERT INTO %s (a, b, c) VALUES (3, 2, [4, 1])");
- execute("INSERT INTO %s (a, b, c) VALUES (4, 1, [7, 1])");
-
- beforeAndAfterFlush(() -> {
-
- // Checks filtering
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE a = 1 AND b = 2 AND c = [4, 2]");
-
- assertRows(execute("SELECT * FROM %s WHERE a = 1 AND b = 2 AND c = [4, 2] ALLOW FILTERING"),
- row(1, 2, list(4, 2)));
-
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE c > [4, 2]");
-
- assertRows(execute("SELECT * FROM %s WHERE c > [4, 2] ALLOW FILTERING"),
- row(2, 1, list(6, 2)),
- row(4, 1, list(7, 1)));
-
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE b < 3 AND c <= [4, 2]");
-
- assertRows(execute("SELECT * FROM %s WHERE b < 3 AND c <= [4, 2] ALLOW FILTERING"),
- row(1, 2, list(4, 2)),
- row(3, 2, list(4, 1)));
-
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE c >= [4, 3] AND c <= [7]");
-
- assertRows(execute("SELECT * FROM %s WHERE c >= [4, 3] AND c <= [7] ALLOW FILTERING"),
- row(2, 1, list(6, 2)));
-
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE c CONTAINS 2");
-
- assertRows(execute("SELECT * FROM %s WHERE c CONTAINS 2 ALLOW FILTERING"),
- row(1, 2, list(4, 2)),
- row(2, 1, list(6, 2)));
-
- assertInvalidMessage("Cannot use CONTAINS KEY on non-map column c",
- "SELECT * FROM %s WHERE c CONTAINS KEY 2 ALLOW FILTERING");
-
- assertRows(execute("SELECT * FROM %s WHERE c CONTAINS 2 AND c CONTAINS 6 ALLOW FILTERING"),
- row(2, 1, list(6, 2)));
- });
-
- // Checks filtering with null
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE c = null");
- assertInvalidMessage("Unsupported null value for column c",
- "SELECT * FROM %s WHERE c = null ALLOW FILTERING");
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE c > null");
- assertInvalidMessage("Unsupported null value for column c",
- "SELECT * FROM %s WHERE c > null ALLOW FILTERING");
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE c CONTAINS null");
- assertInvalidMessage("Unsupported null value for column c",
- "SELECT * FROM %s WHERE c CONTAINS null ALLOW FILTERING");
-
- // Checks filtering with unset
- assertInvalidMessage("Unsupported unset value for column c",
- "SELECT * FROM %s WHERE c = ? ALLOW FILTERING",
- unset());
- assertInvalidMessage("Unsupported unset value for column c",
- "SELECT * FROM %s WHERE c > ? ALLOW FILTERING",
- unset());
- assertInvalidMessage("Unsupported unset value for column c",
- "SELECT * FROM %s WHERE c CONTAINS ? ALLOW FILTERING",
- unset());
- }
-
- @Test
- public void testFilteringOnCompactTablesWithoutIndicesAndWithSets() throws Throwable
- {
- //----------------------------------------------
- // Test COMPACT table with clustering columns
- //----------------------------------------------
- createTable("CREATE TABLE %s (a int, b int, c frozen<set<int>>, PRIMARY KEY (a, b)) WITH COMPACT STORAGE");
-
- execute("INSERT INTO %s (a, b, c) VALUES (1, 2, {4, 2})");
- execute("INSERT INTO %s (a, b, c) VALUES (1, 3, {6, 2})");
- execute("INSERT INTO %s (a, b, c) VALUES (1, 4, {4, 1})");
- execute("INSERT INTO %s (a, b, c) VALUES (2, 3, {7, 1})");
-
- beforeAndAfterFlush(() -> {
-
- // Checks filtering
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE a = 1 AND b = 4 AND c = {4, 1}");
-
- assertRows(execute("SELECT * FROM %s WHERE a = 1 AND b = 4 AND c = {4, 1} ALLOW FILTERING"),
- row(1, 4, set(4, 1)));
-
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE c > {4, 2}");
-
- assertRows(execute("SELECT * FROM %s WHERE c > {4, 2} ALLOW FILTERING"),
- row(1, 3, set(6, 2)));
-
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE b <= 3 AND c < {6, 2}");
-
- assertRows(execute("SELECT * FROM %s WHERE b <= 3 AND c < {6, 2} ALLOW FILTERING"),
- row(1, 2, set(2, 4)),
- row(2, 3, set(1, 7)));
-
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE c >= {4, 2} AND c <= {6, 4}");
-
- assertRows(execute("SELECT * FROM %s WHERE c >= {4, 2} AND c <= {6, 4} ALLOW FILTERING"),
- row(1, 2, set(4, 2)),
- row(1, 3, set(6, 2)));
-
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE c CONTAINS 2");
-
- assertRows(execute("SELECT * FROM %s WHERE c CONTAINS 2 ALLOW FILTERING"),
- row(1, 2, set(4, 2)),
- row(1, 3, set(6, 2)));
-
- assertInvalidMessage("Cannot use CONTAINS KEY on non-map column c",
- "SELECT * FROM %s WHERE c CONTAINS KEY 2 ALLOW FILTERING");
-
- assertRows(execute("SELECT * FROM %s WHERE c CONTAINS 2 AND c CONTAINS 6 ALLOW FILTERING"),
- row(1, 3, set(6, 2)));
- });
- // Checks filtering with null
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE c = null");
- assertInvalidMessage("Unsupported null value for column c",
- "SELECT * FROM %s WHERE c = null ALLOW FILTERING");
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE c > null");
- assertInvalidMessage("Unsupported null value for column c",
- "SELECT * FROM %s WHERE c > null ALLOW FILTERING");
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE c CONTAINS null");
- assertInvalidMessage("Unsupported null value for column c",
- "SELECT * FROM %s WHERE c CONTAINS null ALLOW FILTERING");
-
- // Checks filtering with unset
- assertInvalidMessage("Unsupported unset value for column c",
- "SELECT * FROM %s WHERE c = ? ALLOW FILTERING",
- unset());
- assertInvalidMessage("Unsupported unset value for column c",
- "SELECT * FROM %s WHERE c > ? ALLOW FILTERING",
- unset());
- assertInvalidMessage("Unsupported unset value for column c",
- "SELECT * FROM %s WHERE c CONTAINS ? ALLOW FILTERING",
- unset());
-
- //----------------------------------------------
- // Test COMPACT table without clustering columns
- //----------------------------------------------
- createTable("CREATE TABLE %s (a int PRIMARY KEY, b int, c frozen<set<int>>) WITH COMPACT STORAGE");
-
- execute("INSERT INTO %s (a, b, c) VALUES (1, 2, {4, 2})");
- execute("INSERT INTO %s (a, b, c) VALUES (2, 1, {6, 2})");
- execute("INSERT INTO %s (a, b, c) VALUES (3, 2, {4, 1})");
- execute("INSERT INTO %s (a, b, c) VALUES (4, 1, {7, 1})");
-
- beforeAndAfterFlush(() -> {
-
- // Checks filtering
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE a = 1 AND b = 2 AND c = {4, 2}");
-
- assertRows(execute("SELECT * FROM %s WHERE a = 1 AND b = 2 AND c = {4, 2} ALLOW FILTERING"),
- row(1, 2, set(4, 2)));
-
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE c > {4, 2}");
-
- assertRows(execute("SELECT * FROM %s WHERE c > {4, 2} ALLOW FILTERING"),
- row(2, 1, set(6, 2)));
-
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE b < 3 AND c <= {4, 2}");
-
- assertRows(execute("SELECT * FROM %s WHERE b < 3 AND c <= {4, 2} ALLOW FILTERING"),
- row(1, 2, set(4, 2)),
- row(4, 1, set(1, 7)),
- row(3, 2, set(4, 1)));
-
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE c >= {4, 3} AND c <= {7}");
-
- assertRows(execute("SELECT * FROM %s WHERE c >= {5, 2} AND c <= {7} ALLOW FILTERING"),
- row(2, 1, set(6, 2)));
-
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE c CONTAINS 2");
-
- assertRows(execute("SELECT * FROM %s WHERE c CONTAINS 2 ALLOW FILTERING"),
- row(1, 2, set(4, 2)),
- row(2, 1, set(6, 2)));
-
- assertInvalidMessage("Cannot use CONTAINS KEY on non-map column c",
- "SELECT * FROM %s WHERE c CONTAINS KEY 2 ALLOW FILTERING");
-
- assertRows(execute("SELECT * FROM %s WHERE c CONTAINS 2 AND c CONTAINS 6 ALLOW FILTERING"),
- row(2, 1, set(6, 2)));
- });
-
- // Checks filtering with null
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE c = null");
- assertInvalidMessage("Unsupported null value for column c",
- "SELECT * FROM %s WHERE c = null ALLOW FILTERING");
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE c > null");
- assertInvalidMessage("Unsupported null value for column c",
- "SELECT * FROM %s WHERE c > null ALLOW FILTERING");
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE c CONTAINS null");
- assertInvalidMessage("Unsupported null value for column c",
- "SELECT * FROM %s WHERE c CONTAINS null ALLOW FILTERING");
-
- // Checks filtering with unset
- assertInvalidMessage("Unsupported unset value for column c",
- "SELECT * FROM %s WHERE c = ? ALLOW FILTERING",
- unset());
- assertInvalidMessage("Unsupported unset value for column c",
- "SELECT * FROM %s WHERE c > ? ALLOW FILTERING",
- unset());
- assertInvalidMessage("Unsupported unset value for column c",
- "SELECT * FROM %s WHERE c CONTAINS ? ALLOW FILTERING",
- unset());
- }
@Test
public void testIndexQueryWithValueOver64K() throws Throwable
{
- String tableName = createTable("CREATE TABLE %s (a int, b int, c blob, PRIMARY KEY (a, b))");
- String idx = tableName + "_c_idx";
- createIndex("CREATE INDEX " + idx + " ON %s (c)");
+ createTable("CREATE TABLE %s (a int, b int, c blob, PRIMARY KEY (a, b))");
+ String idx = createIndex("CREATE INDEX ON %s (c)");
execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", 0, 0, bytes(1));
execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", 0, 1, bytes(2));
@@ -2211,41 +1655,6 @@
assertInvalidMessage("queries must only request partition key columns",
"SELECT DISTINCT pk0, pk1, ck0 FROM %s ALLOW FILTERING");
});
-
- // Test a 'compact storage' table.
- createTable("CREATE TABLE %s (pk0 int, pk1 int, val int, PRIMARY KEY((pk0, pk1))) WITH COMPACT STORAGE");
-
- for (int i = 0; i < 3; i++)
- execute("INSERT INTO %s (pk0, pk1, val) VALUES (?, ?, ?)", i, i, i);
-
- beforeAndAfterFlush(() -> {
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT DISTINCT pk0, pk1 FROM %s WHERE pk1 = 1 LIMIT 3");
-
- assertRows(execute("SELECT DISTINCT pk0, pk1 FROM %s WHERE pk0 < 2 AND pk1 = 1 LIMIT 1 ALLOW FILTERING"),
- row(1, 1));
-
- assertRows(execute("SELECT DISTINCT pk0, pk1 FROM %s WHERE pk1 > 1 LIMIT 3 ALLOW FILTERING"),
- row(2, 2));
- });
-
- // Test a 'wide row' thrift table.
- createTable("CREATE TABLE %s (pk int, name text, val int, PRIMARY KEY(pk, name)) WITH COMPACT STORAGE");
-
- for (int i = 0; i < 3; i++)
- {
- execute("INSERT INTO %s (pk, name, val) VALUES (?, 'name0', 0)", i);
- execute("INSERT INTO %s (pk, name, val) VALUES (?, 'name1', 1)", i);
- }
-
- beforeAndAfterFlush(() -> {
- assertRows(execute("SELECT DISTINCT pk FROM %s WHERE pk > 1 LIMIT 1 ALLOW FILTERING"),
- row(2));
-
- assertRows(execute("SELECT DISTINCT pk FROM %s WHERE pk > 0 LIMIT 3 ALLOW FILTERING"),
- row(1),
- row(2));
- });
}
@Test
@@ -2450,7 +1859,7 @@
});
// test clutering order
- createTable("CREATE TABLE %s (a int, b int, c int, d int, e int, PRIMARY KEY ((a, b), c, d)) WITH CLUSTERING ORDER BY (c DESC)");
+ createTable("CREATE TABLE %s (a int, b int, c int, d int, e int, PRIMARY KEY ((a, b), c, d)) WITH CLUSTERING ORDER BY (c DESC, d ASC)");
execute("INSERT INTO %s (a,b,c,d,e) VALUES (11, 11, 13, 14, 15)");
execute("INSERT INTO %s (a,b,c,d,e) VALUES (11, 11, 14, 17, 18)");
@@ -2565,868 +1974,39 @@
@Test
public void testAllowFilteringOnPartitionKeyWithCounters() throws Throwable
{
- for (String compactStorageClause : new String[] { "", " WITH COMPACT STORAGE" })
- {
- createTable("CREATE TABLE %s (a int, b int, c int, cnt counter, PRIMARY KEY ((a, b), c))"
- + compactStorageClause);
+ createTable("CREATE TABLE %s (a int, b int, c int, cnt counter, PRIMARY KEY ((a, b), c))");
- execute("UPDATE %s SET cnt = cnt + ? WHERE a = ? AND b = ? AND c = ?", 14L, 11, 12, 13);
- execute("UPDATE %s SET cnt = cnt + ? WHERE a = ? AND b = ? AND c = ?", 24L, 21, 22, 23);
- execute("UPDATE %s SET cnt = cnt + ? WHERE a = ? AND b = ? AND c = ?", 27L, 21, 22, 26);
- execute("UPDATE %s SET cnt = cnt + ? WHERE a = ? AND b = ? AND c = ?", 34L, 31, 32, 33);
- execute("UPDATE %s SET cnt = cnt + ? WHERE a = ? AND b = ? AND c = ?", 24L, 41, 42, 43);
-
- beforeAndAfterFlush(() -> {
-
- assertRows(executeFilteringOnly("SELECT * FROM %s WHERE cnt = 24"),
- row(41, 42, 43, 24L),
- row(21, 22, 23, 24L));
- assertRows(executeFilteringOnly("SELECT * FROM %s WHERE b > 22 AND cnt = 24"),
- row(41, 42, 43, 24L));
- assertRows(executeFilteringOnly("SELECT * FROM %s WHERE b > 10 AND b < 25 AND cnt = 24"),
- row(21, 22, 23, 24L));
- assertRows(executeFilteringOnly("SELECT * FROM %s WHERE b > 10 AND c < 25 AND cnt = 24"),
- row(21, 22, 23, 24L));
-
- assertInvalidMessage(
- "ORDER BY is only supported when the partition key is restricted by an EQ or an IN.",
- "SELECT * FROM %s WHERE a = 21 AND b > 10 AND cnt > 23 ORDER BY c DESC ALLOW FILTERING");
-
- assertRows(executeFilteringOnly("SELECT * FROM %s WHERE a = 21 AND b = 22 AND cnt > 23 ORDER BY c DESC"),
- row(21, 22, 26, 27L),
- row(21, 22, 23, 24L));
-
- assertRows(executeFilteringOnly("SELECT * FROM %s WHERE cnt > 20 AND cnt < 30"),
- row(41, 42, 43, 24L),
- row(21, 22, 23, 24L),
- row(21, 22, 26, 27L));
- });
- }
- }
-
- @Test
- public void testAllowFilteringOnPartitionKeyOnCompactTablesWithoutIndicesAndWithLists() throws Throwable
- {
- // ----------------------------------------------
- // Test COMPACT table with clustering columns
- // ----------------------------------------------
- createTable("CREATE TABLE %s (a int, b int, c frozen<list<int>>, PRIMARY KEY (a, b)) WITH COMPACT STORAGE");
-
- execute("INSERT INTO %s (a, b, c) VALUES (1, 2, [4, 2])");
- execute("INSERT INTO %s (a, b, c) VALUES (1, 3, [6, 2])");
- execute("INSERT INTO %s (a, b, c) VALUES (1, 4, [4, 1])");
- execute("INSERT INTO %s (a, b, c) VALUES (2, 3, [7, 1])");
+ execute("UPDATE %s SET cnt = cnt + ? WHERE a = ? AND b = ? AND c = ?", 14L, 11, 12, 13);
+ execute("UPDATE %s SET cnt = cnt + ? WHERE a = ? AND b = ? AND c = ?", 24L, 21, 22, 23);
+ execute("UPDATE %s SET cnt = cnt + ? WHERE a = ? AND b = ? AND c = ?", 27L, 21, 22, 26);
+ execute("UPDATE %s SET cnt = cnt + ? WHERE a = ? AND b = ? AND c = ?", 34L, 31, 32, 33);
+ execute("UPDATE %s SET cnt = cnt + ? WHERE a = ? AND b = ? AND c = ?", 24L, 41, 42, 43);
beforeAndAfterFlush(() -> {
- // Checks filtering
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE a >= 1 AND b = 4 AND c = [4, 1]");
+ assertRows(executeFilteringOnly("SELECT * FROM %s WHERE cnt = 24"),
+ row(41, 42, 43, 24L),
+ row(21, 22, 23, 24L));
+ assertRows(executeFilteringOnly("SELECT * FROM %s WHERE b > 22 AND cnt = 24"),
+ row(41, 42, 43, 24L));
+ assertRows(executeFilteringOnly("SELECT * FROM %s WHERE b > 10 AND b < 25 AND cnt = 24"),
+ row(21, 22, 23, 24L));
+ assertRows(executeFilteringOnly("SELECT * FROM %s WHERE b > 10 AND c < 25 AND cnt = 24"),
+ row(21, 22, 23, 24L));
- assertRows(execute("SELECT * FROM %s WHERE a >= 1 AND b >= 4 AND c = [4, 1] ALLOW FILTERING"),
- row(1, 4, list(4, 1)));
+ assertInvalidMessage(
+ "ORDER BY is only supported when the partition key is restricted by an EQ or an IN.",
+ "SELECT * FROM %s WHERE a = 21 AND b > 10 AND cnt > 23 ORDER BY c DESC ALLOW FILTERING");
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE a > 0 AND c > [4, 2]");
+ assertRows(executeFilteringOnly("SELECT * FROM %s WHERE a = 21 AND b = 22 AND cnt > 23 ORDER BY c DESC"),
+ row(21, 22, 26, 27L),
+ row(21, 22, 23, 24L));
- assertRows(execute("SELECT * FROM %s WHERE a >= 1 AND c > [4, 2] ALLOW FILTERING"),
- row(1, 3, list(6, 2)),
- row(2, 3, list(7, 1)));
-
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE a > 1 AND b <= 3 AND c < [6, 2]");
-
- assertRows(execute("SELECT * FROM %s WHERE a <= 1 AND b <= 3 AND c < [6, 2] ALLOW FILTERING"),
- row(1, 2, list(4, 2)));
-
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE a <= 1 AND c >= [4, 2] AND c <= [6, 4]");
-
- assertRows(execute("SELECT * FROM %s WHERE a > 0 AND b <= 3 AND c >= [4, 2] AND c <= [6, 4] ALLOW FILTERING"),
- row(1, 2, list(4, 2)),
- row(1, 3, list(6, 2)));
-
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE a > 1 AND c CONTAINS 2");
-
- assertRows(execute("SELECT * FROM %s WHERE a > 0 AND c CONTAINS 2 ALLOW FILTERING"),
- row(1, 2, list(4, 2)),
- row(1, 3, list(6, 2)));
-
- assertInvalidMessage("Cannot use CONTAINS KEY on non-map column c",
- "SELECT * FROM %s WHERE a > 1 AND c CONTAINS KEY 2 ALLOW FILTERING");
-
- assertRows(execute("SELECT * FROM %s WHERE a < 2 AND c CONTAINS 2 AND c CONTAINS 6 ALLOW FILTERING"),
- row(1, 3, list(6, 2)));
+ assertRows(executeFilteringOnly("SELECT * FROM %s WHERE cnt > 20 AND cnt < 30"),
+ row(41, 42, 43, 24L),
+ row(21, 22, 23, 24L),
+ row(21, 22, 26, 27L));
});
- // Checks filtering with null
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE a > 1 AND c = null");
- assertInvalidMessage("Unsupported null value for column c",
- "SELECT * FROM %s WHERE a > 1 AND c = null ALLOW FILTERING");
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE a > 1 AND c > null");
- assertInvalidMessage("Unsupported null value for column c",
- "SELECT * FROM %s WHERE a > 1 AND c > null ALLOW FILTERING");
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE a > 1 AND c CONTAINS null");
- assertInvalidMessage("Unsupported null value for column c",
- "SELECT * FROM %s WHERE a > 1 AND c CONTAINS null ALLOW FILTERING");
-
- // Checks filtering with unset
- assertInvalidMessage("Unsupported unset value for column c",
- "SELECT * FROM %s WHERE a > 1 AND c = ? ALLOW FILTERING",
- unset());
- assertInvalidMessage("Unsupported unset value for column c",
- "SELECT * FROM %s WHERE a > 1 AND c > ? ALLOW FILTERING",
- unset());
- assertInvalidMessage("Unsupported unset value for column c",
- "SELECT * FROM %s WHERE a > 1 AND c CONTAINS ? ALLOW FILTERING",
- unset());
-
- // ----------------------------------------------
- // Test COMPACT table without clustering columns
- // ----------------------------------------------
- createTable("CREATE TABLE %s (a int PRIMARY KEY, b int, c frozen<list<int>>) WITH COMPACT STORAGE");
-
- execute("INSERT INTO %s (a, b, c) VALUES (1, 2, [4, 2])");
- execute("INSERT INTO %s (a, b, c) VALUES (2, 1, [6, 2])");
- execute("INSERT INTO %s (a, b, c) VALUES (3, 2, [4, 1])");
- execute("INSERT INTO %s (a, b, c) VALUES (4, 1, [7, 1])");
-
- beforeAndAfterFlush(() -> {
-
- // Checks filtering
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE a >= 1 AND b = 2 AND c = [4, 2]");
-
- assertRows(execute("SELECT * FROM %s WHERE a >= 1 AND b = 2 AND c = [4, 2] ALLOW FILTERING"),
- row(1, 2, list(4, 2)));
-
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE a > 1 AND c > [4, 2]");
-
- assertRows(execute("SELECT * FROM %s WHERE a > 3 AND c > [4, 2] ALLOW FILTERING"),
- row(4, 1, list(7, 1)));
-
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE a < 1 AND b < 3 AND c <= [4, 2]");
-
- assertRows(execute("SELECT * FROM %s WHERE a < 3 AND b < 3 AND c <= [4, 2] ALLOW FILTERING"),
- row(1, 2, list(4, 2)));
-
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE a > 1 AND c >= [4, 3] AND c <= [7]");
-
- assertRows(execute("SELECT * FROM %s WHERE a >= 2 AND c >= [4, 3] AND c <= [7] ALLOW FILTERING"),
- row(2, 1, list(6, 2)));
-
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE a > 3 AND c CONTAINS 2");
-
- assertRows(execute("SELECT * FROM %s WHERE a >= 1 AND c CONTAINS 2 ALLOW FILTERING"),
- row(1, 2, list(4, 2)),
- row(2, 1, list(6, 2)));
-
- assertInvalidMessage("Cannot use CONTAINS KEY on non-map column c",
- "SELECT * FROM %s WHERE a >=1 AND c CONTAINS KEY 2 ALLOW FILTERING");
-
- assertRows(execute("SELECT * FROM %s WHERE a < 3 AND c CONTAINS 2 AND c CONTAINS 6 ALLOW FILTERING"),
- row(2, 1, list(6, 2)));
- });
-
- // Checks filtering with null
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE a > 1 AND c = null");
- assertInvalidMessage("Unsupported null value for column c",
- "SELECT * FROM %s WHERE a > 1 AND c = null ALLOW FILTERING");
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE a > 1 AND c > null");
- assertInvalidMessage("Unsupported null value for column c",
- "SELECT * FROM %s WHERE a > 1 AND c > null ALLOW FILTERING");
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE a > 1 AND c CONTAINS null");
- assertInvalidMessage("Unsupported null value for column c",
- "SELECT * FROM %s WHERE a > 1 AND c CONTAINS null ALLOW FILTERING");
-
- // Checks filtering with unset
- assertInvalidMessage("Unsupported unset value for column c",
- "SELECT * FROM %s WHERE a > 1 AND c = ? ALLOW FILTERING",
- unset());
- assertInvalidMessage("Unsupported unset value for column c",
- "SELECT * FROM %s WHERE a > 1 AND c > ? ALLOW FILTERING",
- unset());
- assertInvalidMessage("Unsupported unset value for column c",
- "SELECT * FROM %s WHERE a > 1 AND c CONTAINS ? ALLOW FILTERING",
- unset());
- }
-
-
- @Test
- public void testAllowFilteringOnPartitionKeyOnCompactTablesWithoutIndicesAndWithMaps() throws Throwable
- {
- //----------------------------------------------
- // Test COMPACT table with clustering columns
- //----------------------------------------------
- createTable("CREATE TABLE %s (a int, b int, c frozen<map<int, int>>, PRIMARY KEY (a, b)) WITH COMPACT STORAGE");
-
- execute("INSERT INTO %s (a, b, c) VALUES (1, 2, {4 : 2})");
- execute("INSERT INTO %s (a, b, c) VALUES (1, 3, {6 : 2})");
- execute("INSERT INTO %s (a, b, c) VALUES (1, 4, {4 : 1})");
- execute("INSERT INTO %s (a, b, c) VALUES (2, 3, {7 : 1})");
-
- beforeAndAfterFlush(() -> {
-
- // Checks filtering
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE a >= 1 AND b = 4 AND c = {4 : 1}");
-
- assertRows(execute("SELECT * FROM %s WHERE a <= 1 AND b = 4 AND c = {4 : 1} ALLOW FILTERING"),
- row(1, 4, map(4, 1)));
-
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE a > 1 AND c > {4 : 2}");
-
- assertRows(execute("SELECT * FROM %s WHERE a > 1 AND c > {4 : 2} ALLOW FILTERING"),
- row(2, 3, map(7, 1)));
-
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE a > 1 AND b <= 3 AND c < {6 : 2}");
-
- assertRows(execute("SELECT * FROM %s WHERE a >= 1 AND b <= 3 AND c < {6 : 2} ALLOW FILTERING"),
- row(1, 2, map(4, 2)));
-
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE a > 1 AND c >= {4 : 2} AND c <= {6 : 4}");
-
- assertRows(execute("SELECT * FROM %s WHERE a > 0 AND c >= {4 : 2} AND c <= {6 : 4} ALLOW FILTERING"),
- row(1, 2, map(4, 2)),
- row(1, 3, map(6, 2)));
-
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE a > 10 AND c CONTAINS 2");
-
- assertRows(execute("SELECT * FROM %s WHERE a > 0 AND c CONTAINS 2 ALLOW FILTERING"),
- row(1, 2, map(4, 2)),
- row(1, 3, map(6, 2)));
-
- assertRows(execute("SELECT * FROM %s WHERE a < 2 AND c CONTAINS KEY 6 ALLOW FILTERING"),
- row(1, 3, map(6, 2)));
-
- assertRows(execute("SELECT * FROM %s WHERE a >= 1 AND c CONTAINS 2 AND c CONTAINS KEY 6 ALLOW FILTERING"),
- row(1, 3, map(6, 2)));
- });
-
- // Checks filtering with null
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE a >= 1 AND c = null");
- assertInvalidMessage("Unsupported null value for column c",
- "SELECT * FROM %s WHERE a >= 1 AND c = null ALLOW FILTERING");
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE a >= 1 AND c > null");
- assertInvalidMessage("Unsupported null value for column c",
- "SELECT * FROM %s WHERE a >= 1 AND c > null ALLOW FILTERING");
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE a >= 1 AND c CONTAINS null");
- assertInvalidMessage("Unsupported null value for column c",
- "SELECT * FROM %s WHERE a >= 1 AND c CONTAINS null ALLOW FILTERING");
- assertInvalidMessage("Unsupported null value for column c",
- "SELECT * FROM %s WHERE a >= 1 AND c CONTAINS KEY null ALLOW FILTERING");
-
- // Checks filtering with unset
- assertInvalidMessage("Unsupported unset value for column c",
- "SELECT * FROM %s WHERE a >= 1 AND c = ? ALLOW FILTERING",
- unset());
- assertInvalidMessage("Unsupported unset value for column c",
- "SELECT * FROM %s WHERE a >= 1 AND c > ? ALLOW FILTERING",
- unset());
- assertInvalidMessage("Unsupported unset value for column c",
- "SELECT * FROM %s WHERE a >= 1 AND c CONTAINS ? ALLOW FILTERING",
- unset());
- assertInvalidMessage("Unsupported unset value for column c",
- "SELECT * FROM %s WHERE a >= 1 AND c CONTAINS KEY ? ALLOW FILTERING",
- unset());
-
- //----------------------------------------------
- // Test COMPACT table without clustering columns
- //----------------------------------------------
- createTable("CREATE TABLE %s (a int PRIMARY KEY, b int, c frozen<map<int, int>>) WITH COMPACT STORAGE");
-
- execute("INSERT INTO %s (a, b, c) VALUES (1, 2, {4 : 2})");
- execute("INSERT INTO %s (a, b, c) VALUES (2, 1, {6 : 2})");
- execute("INSERT INTO %s (a, b, c) VALUES (3, 2, {4 : 1})");
- execute("INSERT INTO %s (a, b, c) VALUES (4, 1, {7 : 1})");
-
- beforeAndAfterFlush(() -> {
-
- // Checks filtering
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE a >= 1 AND b = 2 AND c = {4 : 2}");
-
- assertRows(execute("SELECT * FROM %s WHERE a >= 1 AND b = 2 AND c = {4 : 2} ALLOW FILTERING"),
- row(1, 2, map(4, 2)));
-
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE a >= 1 AND c > {4 : 2}");
-
- assertRows(execute("SELECT * FROM %s WHERE a >= 1 AND c > {4 : 2} ALLOW FILTERING"),
- row(2, 1, map(6, 2)),
- row(4, 1, map(7, 1)));
-
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE a >= 1 AND b < 3 AND c <= {4 : 2}");
-
- assertRows(execute("SELECT * FROM %s WHERE a >= 1 AND b < 3 AND c <= {4 : 2} ALLOW FILTERING"),
- row(1, 2, map(4, 2)),
- row(3, 2, map(4, 1)));
-
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE a >= 1 AND c >= {4 : 3} AND c <= {7 : 1}");
-
- assertRows(execute("SELECT * FROM %s WHERE a >= 2 AND c >= {5 : 2} AND c <= {7 : 0} ALLOW FILTERING"),
- row(2, 1, map(6, 2)));
-
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE a >= 1 AND c CONTAINS 2");
-
- assertRows(execute("SELECT * FROM %s WHERE a >= 1 AND c CONTAINS 2 ALLOW FILTERING"),
- row(1, 2, map(4, 2)),
- row(2, 1, map(6, 2)));
-
- assertRows(execute("SELECT * FROM %s WHERE a > 0 AND c CONTAINS KEY 4 ALLOW FILTERING"),
- row(1, 2, map(4, 2)),
- row(3, 2, map(4, 1)));
-
- assertRows(execute("SELECT * FROM %s WHERE a >= 2 AND c CONTAINS 2 AND c CONTAINS KEY 6 ALLOW FILTERING"),
- row(2, 1, map(6, 2)));
- });
-
- // Checks filtering with null
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE a >= 1 AND c = null");
- assertInvalidMessage("Unsupported null value for column c",
- "SELECT * FROM %s WHERE a >= 1 AND c = null ALLOW FILTERING");
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE a >= 1 AND c > null");
- assertInvalidMessage("Unsupported null value for column c",
- "SELECT * FROM %s WHERE a >= 1 AND c > null ALLOW FILTERING");
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE a >= 1 AND c CONTAINS null");
- assertInvalidMessage("Unsupported null value for column c",
- "SELECT * FROM %s WHERE a >= 1 AND c CONTAINS null ALLOW FILTERING");
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE a >= 1 AND c CONTAINS KEY null");
- assertInvalidMessage("Unsupported null value for column c",
- "SELECT * FROM %s WHERE a >= 1 AND c CONTAINS KEY null ALLOW FILTERING");
-
- // Checks filtering with unset
- assertInvalidMessage("Unsupported unset value for column c",
- "SELECT * FROM %s WHERE a >= 1 AND c = ? ALLOW FILTERING",
- unset());
- assertInvalidMessage("Unsupported unset value for column c",
- "SELECT * FROM %s WHERE a >= 1 AND c > ? ALLOW FILTERING",
- unset());
- assertInvalidMessage("Unsupported unset value for column c",
- "SELECT * FROM %s WHERE a >= 1 AND c CONTAINS ? ALLOW FILTERING",
- unset());
- assertInvalidMessage("Unsupported unset value for column c",
- "SELECT * FROM %s WHERE a >= 1 AND c CONTAINS KEY ? ALLOW FILTERING",
- unset());
- }
-
- @Test
- public void testAllowFilteringOnPartitionKeyOnCompactTablesWithoutIndicesAndWithSets() throws Throwable
- {
- //----------------------------------------------
- // Test COMPACT table with clustering columns
- //----------------------------------------------
- createTable("CREATE TABLE %s (a int, b int, c frozen<set<int>>, PRIMARY KEY (a, b)) WITH COMPACT STORAGE");
-
- execute("INSERT INTO %s (a, b, c) VALUES (1, 2, {4, 2})");
- execute("INSERT INTO %s (a, b, c) VALUES (1, 3, {6, 2})");
- execute("INSERT INTO %s (a, b, c) VALUES (1, 4, {4, 1})");
- execute("INSERT INTO %s (a, b, c) VALUES (2, 3, {7, 1})");
-
- beforeAndAfterFlush(() -> {
-
- // Checks filtering
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE a >= 1 AND b = 4 AND c = {4, 1}");
-
- assertRows(execute("SELECT * FROM %s WHERE a >= 1 AND b = 4 AND c = {4, 1} ALLOW FILTERING"),
- row(1, 4, set(4, 1)));
-
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE a >= 1 AND c > {4, 2}");
-
- assertRows(execute("SELECT * FROM %s WHERE a >= 1 AND c > {4, 2} ALLOW FILTERING"),
- row(1, 3, set(6, 2)));
-
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE a >= 1 AND b <= 3 AND c < {6, 2}");
-
- assertRows(execute("SELECT * FROM %s WHERE a > 0 AND b <= 3 AND c < {6, 2} ALLOW FILTERING"),
- row(1, 2, set(2, 4)),
- row(2, 3, set(1, 7)));
-
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE a >= 1 AND c >= {4, 2} AND c <= {6, 4}");
-
- assertRows(execute("SELECT * FROM %s WHERE a >= 0 AND c >= {4, 2} AND c <= {6, 4} ALLOW FILTERING"),
- row(1, 2, set(4, 2)),
- row(1, 3, set(6, 2)));
-
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE a >= 1 AND c CONTAINS 2");
-
- assertRows(execute("SELECT * FROM %s WHERE a < 2 AND c CONTAINS 2 ALLOW FILTERING"),
- row(1, 2, set(4, 2)),
- row(1, 3, set(6, 2)));
-
- assertInvalidMessage("Cannot use CONTAINS KEY on non-map column c",
- "SELECT * FROM %s WHERE a >= 1 AND c CONTAINS KEY 2 ALLOW FILTERING");
-
- assertRows(execute("SELECT * FROM %s WHERE a >= 1 AND c CONTAINS 2 AND c CONTAINS 6 ALLOW FILTERING"),
- row(1, 3, set(6, 2)));
- });
- // Checks filtering with null
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE a >= 1 AND c = null");
- assertInvalidMessage("Unsupported null value for column c",
- "SELECT * FROM %s WHERE a >= 1 AND c = null ALLOW FILTERING");
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE a >= 1 AND c > null");
- assertInvalidMessage("Unsupported null value for column c",
- "SELECT * FROM %s WHERE a >= 1 AND c > null ALLOW FILTERING");
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE a >= 1 AND c CONTAINS null");
- assertInvalidMessage("Unsupported null value for column c",
- "SELECT * FROM %s WHERE a >= 1 AND c CONTAINS null ALLOW FILTERING");
-
- // Checks filtering with unset
- assertInvalidMessage("Unsupported unset value for column c",
- "SELECT * FROM %s WHERE a >= 1 AND c = ? ALLOW FILTERING",
- unset());
- assertInvalidMessage("Unsupported unset value for column c",
- "SELECT * FROM %s WHERE a >= 1 AND c > ? ALLOW FILTERING",
- unset());
- assertInvalidMessage("Unsupported unset value for column c",
- "SELECT * FROM %s WHERE a >= 1 AND c CONTAINS ? ALLOW FILTERING",
- unset());
-
- //----------------------------------------------
- // Test COMPACT table without clustering columns
- //----------------------------------------------
- createTable("CREATE TABLE %s (a int PRIMARY KEY, b int, c frozen<set<int>>) WITH COMPACT STORAGE");
-
- execute("INSERT INTO %s (a, b, c) VALUES (1, 2, {4, 2})");
- execute("INSERT INTO %s (a, b, c) VALUES (2, 1, {6, 2})");
- execute("INSERT INTO %s (a, b, c) VALUES (3, 2, {4, 1})");
- execute("INSERT INTO %s (a, b, c) VALUES (4, 1, {7, 1})");
-
- beforeAndAfterFlush(() -> {
-
- // Checks filtering
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE a >= 1 AND b = 2 AND c = {4, 2}");
-
- assertRows(execute("SELECT * FROM %s WHERE a >= 1 AND b = 2 AND c = {4, 2} ALLOW FILTERING"),
- row(1, 2, set(4, 2)));
-
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE a >= 1 AND c > {4, 2}");
-
- assertRows(execute("SELECT * FROM %s WHERE a >= 1 AND c > {4, 2} ALLOW FILTERING"),
- row(2, 1, set(6, 2)));
-
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE a >= 1 AND b < 3 AND c <= {4, 2}");
-
- assertRows(execute("SELECT * FROM %s WHERE a <= 4 AND b < 3 AND c <= {4, 2} ALLOW FILTERING"),
- row(1, 2, set(4, 2)),
- row(4, 1, set(1, 7)),
- row(3, 2, set(4, 1)));
-
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE a >= 1 AND c >= {4, 3} AND c <= {7}");
-
- assertRows(execute("SELECT * FROM %s WHERE a < 3 AND c >= {5, 2} AND c <= {7} ALLOW FILTERING"),
- row(2, 1, set(6, 2)));
-
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE a >= 1 AND c CONTAINS 2");
-
- assertRows(execute("SELECT * FROM %s WHERE a >= 0 AND c CONTAINS 2 ALLOW FILTERING"),
- row(1, 2, set(4, 2)),
- row(2, 1, set(6, 2)));
-
- assertInvalidMessage("Cannot use CONTAINS KEY on non-map column c",
- "SELECT * FROM %s WHERE a >= 1 AND c CONTAINS KEY 2 ALLOW FILTERING");
-
- assertRows(execute("SELECT * FROM %s WHERE a >= 2 AND c CONTAINS 2 AND c CONTAINS 6 ALLOW FILTERING"),
- row(2, 1, set(6, 2)));
- });
-
- // Checks filtering with null
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE a >= 1 AND c = null");
- assertInvalidMessage("Unsupported null value for column c",
- "SELECT * FROM %s WHERE a >= 1 AND c = null ALLOW FILTERING");
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE a >= 1 AND c > null");
- assertInvalidMessage("Unsupported null value for column c",
- "SELECT * FROM %s WHERE a >= 1 AND c > null ALLOW FILTERING");
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE a >= 1 AND c CONTAINS null");
- assertInvalidMessage("Unsupported null value for column c",
- "SELECT * FROM %s WHERE a >= 1 AND c CONTAINS null ALLOW FILTERING");
-
- // Checks filtering with unset
- assertInvalidMessage("Unsupported unset value for column c",
- "SELECT * FROM %s WHERE a >= 1 AND c = ? ALLOW FILTERING",
- unset());
- assertInvalidMessage("Unsupported unset value for column c",
- "SELECT * FROM %s WHERE a >= 1 AND c > ? ALLOW FILTERING",
- unset());
- assertInvalidMessage("Unsupported unset value for column c",
- "SELECT * FROM %s WHERE a >= 1 AND c CONTAINS ? ALLOW FILTERING",
- unset());
- }
-
- @Test
- public void testAllowFilteringOnPartitionKeyOnCompactTablesWithoutIndices() throws Throwable
- {
- // ----------------------------------------------
- // Test COMPACT table with clustering columns
- // ----------------------------------------------
- createTable("CREATE TABLE %s (a int, b int, c int, d int, PRIMARY KEY ((a, b), c)) WITH COMPACT STORAGE");
-
- execute("INSERT INTO %s (a, b, c, d) VALUES (1, 2, 4, 5)");
- execute("INSERT INTO %s (a, b, c, d) VALUES (1, 3, 6, 7)");
- execute("INSERT INTO %s (a, b, c, d) VALUES (1, 4, 4, 5)");
- execute("INSERT INTO %s (a, b, c, d) VALUES (2, 3, 7, 8)");
-
- // Adds tomstones
- execute("INSERT INTO %s (a, b, c, d) VALUES (1, 1, 4, 5)");
- execute("INSERT INTO %s (a, b, c, d) VALUES (2, 2, 7, 8)");
- execute("DELETE FROM %s WHERE a = 1 AND b = 1 AND c = 4");
- execute("DELETE FROM %s WHERE a = 2 AND b = 2 AND c = 7");
-
- beforeAndAfterFlush(() -> {
- assertRows(execute("SELECT * FROM %s WHERE a = 1 AND b = 4 AND c = 4"),
- row(1, 4, 4, 5));
-
- // Checks filtering
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE a = 1 AND b = 4 AND c = 4 AND d = 5");
-
- assertRows(execute("SELECT * FROM %s WHERE a = 1 AND b = 4 AND c = 4 ALLOW FILTERING"),
- row(1, 4, 4, 5));
-
- assertInvalidMessage("IN predicates on non-primary-key columns (d) is not yet supported",
- "SELECT * FROM %s WHERE a IN (1, 2) AND b = 3 AND d IN (6, 7)");
-
- assertInvalidMessage("IN predicates on non-primary-key columns (d) is not yet supported",
- "SELECT * FROM %s WHERE a IN (1, 2) AND b = 3 AND d IN (6, 7) ALLOW FILTERING");
-
- assertRows(execute("SELECT * FROM %s WHERE a < 2 AND c > 4 AND c <= 6 ALLOW FILTERING"),
- row(1, 3, 6, 7));
-
- assertRows(execute("SELECT * FROM %s WHERE a <= 1 AND b >= 2 AND c >= 4 AND d <= 8 ALLOW FILTERING"),
- row(1, 3, 6, 7),
- row(1, 4, 4, 5),
- row(1, 2, 4, 5));
-
- assertRows(execute("SELECT * FROM %s WHERE a = 1 AND c >= 4 AND d <= 8 ALLOW FILTERING"),
- row(1, 3, 6, 7),
- row(1, 4, 4, 5),
- row(1, 2, 4, 5));
-
- assertRows(execute("SELECT * FROM %s WHERE a >= 2 AND c >= 4 AND d <= 8 ALLOW FILTERING"),
- row(2, 3, 7, 8));
- });
-
- // Checks filtering with null
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE d = null");
- assertInvalidMessage("Unsupported null value for column a",
- "SELECT * FROM %s WHERE a = null ALLOW FILTERING");
- assertInvalidMessage("Unsupported null value for column a",
- "SELECT * FROM %s WHERE a > null ALLOW FILTERING");
-
- // Checks filtering with unset
- assertInvalidMessage("Unsupported unset value for column a",
- "SELECT * FROM %s WHERE a = ? ALLOW FILTERING",
- unset());
- assertInvalidMessage("Unsupported unset value for column a",
- "SELECT * FROM %s WHERE a > ? ALLOW FILTERING",
- unset());
-
- //----------------------------------------------
- // Test COMPACT table without clustering columns
- //----------------------------------------------
- createTable("CREATE TABLE %s (a int primary key, b int, c int) WITH COMPACT STORAGE");
-
- execute("INSERT INTO %s (a, b, c) VALUES (1, 2, 4)");
- execute("INSERT INTO %s (a, b, c) VALUES (2, 1, 6)");
- execute("INSERT INTO %s (a, b, c) VALUES (3, 2, 4)");
- execute("INSERT INTO %s (a, b, c) VALUES (4, 1, 7)");
-
- // Adds tomstones
- execute("INSERT INTO %s (a, b, c) VALUES (0, 1, 4)");
- execute("INSERT INTO %s (a, b, c) VALUES (5, 2, 7)");
- execute("DELETE FROM %s WHERE a = 0");
- execute("DELETE FROM %s WHERE a = 5");
-
- beforeAndAfterFlush(() -> {
-
- // Checks filtering
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE a = 1 AND b = 4 AND c = 4");
-
- assertRows(execute("SELECT * FROM %s WHERE a = 1 AND b = 2 AND c = 4 ALLOW FILTERING"),
- row(1, 2, 4));
-
- assertRows(execute("SELECT * FROM %s WHERE a = 1 AND b = 2 ALLOW FILTERING"),
- row(1, 2, 4));
-
- assertRows(execute("SELECT * FROM %s WHERE b >= 2 AND c <= 4 ALLOW FILTERING"),
- row(1, 2, 4),
- row(3, 2, 4));
-
- assertRows(execute("SELECT * FROM %s WHERE a = 1 ALLOW FILTERING"),
- row(1, 2, 4));
-
- assertRows(execute("SELECT * FROM %s WHERE b >= 2 ALLOW FILTERING"),
- row(1, 2, 4),
- row(3, 2, 4));
-
- assertRows(execute("SELECT * FROM %s WHERE a >= 2 AND b <=1 ALLOW FILTERING"),
- row(2, 1, 6),
- row(4, 1, 7));
-
- assertRows(execute("SELECT * FROM %s WHERE a = 1 AND c >= 4 ALLOW FILTERING"),
- row(1, 2, 4));
-
- assertInvalidMessage("IN predicates on non-primary-key columns (b) is not yet supported",
- "SELECT * FROM %s WHERE a = 1 AND b IN (1, 2) AND c IN (6, 7)");
-
- assertInvalidMessage("IN predicates on non-primary-key columns (c) is not yet supported",
- "SELECT * FROM %s WHERE a IN (1, 2) AND c IN (6, 7) ALLOW FILTERING");
-
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE c > 4");
-
- assertRows(execute("SELECT * FROM %s WHERE c > 4 ALLOW FILTERING"),
- row(2, 1, 6),
- row(4, 1, 7));
-
- assertRows(execute("SELECT * FROM %s WHERE a >= 1 AND b >= 2 AND c <= 4 ALLOW FILTERING"),
- row(1, 2, 4),
- row(3, 2, 4));
-
- assertRows(execute("SELECT * FROM %s WHERE a < 3 AND c <= 4 ALLOW FILTERING"),
- row(1, 2, 4));
-
- assertRows(execute("SELECT * FROM %s WHERE a < 3 AND b >= 2 AND c <= 4 ALLOW FILTERING"),
- row(1, 2, 4));
-
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE c >= 3 AND c <= 6");
-
- assertRows(execute("SELECT * FROM %s WHERE c <=6 ALLOW FILTERING"),
- row(1, 2, 4),
- row(2, 1, 6),
- row(3, 2, 4));
-
- assertRows(execute("SELECT * FROM %s WHERE token(a) >= token(2)"),
- row(2, 1, 6),
- row(4, 1, 7),
- row(3, 2, 4));
-
- assertRows(execute("SELECT * FROM %s WHERE token(a) >= token(2) ALLOW FILTERING"),
- row(2, 1, 6),
- row(4, 1, 7),
- row(3, 2, 4));
-
- assertRows(execute("SELECT * FROM %s WHERE token(a) >= token(2) AND b = 1 ALLOW FILTERING"),
- row(2, 1, 6),
- row(4, 1, 7));
-
- });
- }
-
- @Test
- public void testFilteringOnCompactTablesWithoutIndicesAndWithMaps() throws Throwable
- {
- //----------------------------------------------
- // Test COMPACT table with clustering columns
- //----------------------------------------------
- createTable("CREATE TABLE %s (a int, b int, c frozen<map<int, int>>, PRIMARY KEY (a, b)) WITH COMPACT STORAGE");
-
- execute("INSERT INTO %s (a, b, c) VALUES (1, 2, {4 : 2})");
- execute("INSERT INTO %s (a, b, c) VALUES (1, 3, {6 : 2})");
- execute("INSERT INTO %s (a, b, c) VALUES (1, 4, {4 : 1})");
- execute("INSERT INTO %s (a, b, c) VALUES (2, 3, {7 : 1})");
-
- beforeAndAfterFlush(() -> {
-
- // Checks filtering
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE a = 1 AND b = 4 AND c = {4 : 1}");
-
- assertRows(execute("SELECT * FROM %s WHERE a = 1 AND b = 4 AND c = {4 : 1} ALLOW FILTERING"),
- row(1, 4, map(4, 1)));
-
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE c > {4 : 2}");
-
- assertRows(execute("SELECT * FROM %s WHERE c > {4 : 2} ALLOW FILTERING"),
- row(1, 3, map(6, 2)),
- row(2, 3, map(7, 1)));
-
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE b <= 3 AND c < {6 : 2}");
-
- assertRows(execute("SELECT * FROM %s WHERE b <= 3 AND c < {6 : 2} ALLOW FILTERING"),
- row(1, 2, map(4, 2)));
-
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE c >= {4 : 2} AND c <= {6 : 4}");
-
- assertRows(execute("SELECT * FROM %s WHERE c >= {4 : 2} AND c <= {6 : 4} ALLOW FILTERING"),
- row(1, 2, map(4, 2)),
- row(1, 3, map(6, 2)));
-
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE c CONTAINS 2");
-
- assertRows(execute("SELECT * FROM %s WHERE c CONTAINS 2 ALLOW FILTERING"),
- row(1, 2, map(4, 2)),
- row(1, 3, map(6, 2)));
-
- assertRows(execute("SELECT * FROM %s WHERE c CONTAINS KEY 6 ALLOW FILTERING"),
- row(1, 3, map(6, 2)));
-
- assertRows(execute("SELECT * FROM %s WHERE c CONTAINS 2 AND c CONTAINS KEY 6 ALLOW FILTERING"),
- row(1, 3, map(6, 2)));
- });
-
- // Checks filtering with null
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE c = null");
- assertInvalidMessage("Unsupported null value for column c",
- "SELECT * FROM %s WHERE c = null ALLOW FILTERING");
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE c > null");
- assertInvalidMessage("Unsupported null value for column c",
- "SELECT * FROM %s WHERE c > null ALLOW FILTERING");
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE c CONTAINS null");
- assertInvalidMessage("Unsupported null value for column c",
- "SELECT * FROM %s WHERE c CONTAINS null ALLOW FILTERING");
- assertInvalidMessage("Unsupported null value for column c",
- "SELECT * FROM %s WHERE c CONTAINS KEY null ALLOW FILTERING");
-
- // Checks filtering with unset
- assertInvalidMessage("Unsupported unset value for column c",
- "SELECT * FROM %s WHERE c = ? ALLOW FILTERING",
- unset());
- assertInvalidMessage("Unsupported unset value for column c",
- "SELECT * FROM %s WHERE c > ? ALLOW FILTERING",
- unset());
- assertInvalidMessage("Unsupported unset value for column c",
- "SELECT * FROM %s WHERE c CONTAINS ? ALLOW FILTERING",
- unset());
- assertInvalidMessage("Unsupported unset value for column c",
- "SELECT * FROM %s WHERE c CONTAINS KEY ? ALLOW FILTERING",
- unset());
-
- //----------------------------------------------
- // Test COMPACT table without clustering columns
- //----------------------------------------------
- createTable("CREATE TABLE %s (a int PRIMARY KEY, b int, c frozen<map<int, int>>) WITH COMPACT STORAGE");
-
- execute("INSERT INTO %s (a, b, c) VALUES (1, 2, {4 : 2})");
- execute("INSERT INTO %s (a, b, c) VALUES (2, 1, {6 : 2})");
- execute("INSERT INTO %s (a, b, c) VALUES (3, 2, {4 : 1})");
- execute("INSERT INTO %s (a, b, c) VALUES (4, 1, {7 : 1})");
-
- beforeAndAfterFlush(() -> {
-
- // Checks filtering
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE a = 1 AND b = 2 AND c = {4 : 2}");
-
- assertRows(execute("SELECT * FROM %s WHERE a = 1 AND b = 2 AND c = {4 : 2} ALLOW FILTERING"),
- row(1, 2, map(4, 2)));
-
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE c > {4 : 2}");
-
- assertRows(execute("SELECT * FROM %s WHERE c > {4 : 2} ALLOW FILTERING"),
- row(2, 1, map(6, 2)),
- row(4, 1, map(7, 1)));
-
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE b < 3 AND c <= {4 : 2}");
-
- assertRows(execute("SELECT * FROM %s WHERE b < 3 AND c <= {4 : 2} ALLOW FILTERING"),
- row(1, 2, map(4, 2)),
- row(3, 2, map(4, 1)));
-
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE c >= {4 : 3} AND c <= {7 : 1}");
-
- assertRows(execute("SELECT * FROM %s WHERE c >= {5 : 2} AND c <= {7 : 0} ALLOW FILTERING"),
- row(2, 1, map(6, 2)));
-
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE c CONTAINS 2");
-
- assertRows(execute("SELECT * FROM %s WHERE c CONTAINS 2 ALLOW FILTERING"),
- row(1, 2, map(4, 2)),
- row(2, 1, map(6, 2)));
-
- assertRows(execute("SELECT * FROM %s WHERE c CONTAINS KEY 4 ALLOW FILTERING"),
- row(1, 2, map(4, 2)),
- row(3, 2, map(4, 1)));
-
- assertRows(execute("SELECT * FROM %s WHERE c CONTAINS 2 AND c CONTAINS KEY 6 ALLOW FILTERING"),
- row(2, 1, map(6, 2)));
- });
-
- // Checks filtering with null
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE c = null");
- assertInvalidMessage("Unsupported null value for column c",
- "SELECT * FROM %s WHERE c = null ALLOW FILTERING");
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE c > null");
- assertInvalidMessage("Unsupported null value for column c",
- "SELECT * FROM %s WHERE c > null ALLOW FILTERING");
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE c CONTAINS null");
- assertInvalidMessage("Unsupported null value for column c",
- "SELECT * FROM %s WHERE c CONTAINS null ALLOW FILTERING");
- assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
- "SELECT * FROM %s WHERE c CONTAINS KEY null");
- assertInvalidMessage("Unsupported null value for column c",
- "SELECT * FROM %s WHERE c CONTAINS KEY null ALLOW FILTERING");
-
- // Checks filtering with unset
- assertInvalidMessage("Unsupported unset value for column c",
- "SELECT * FROM %s WHERE c = ? ALLOW FILTERING",
- unset());
- assertInvalidMessage("Unsupported unset value for column c",
- "SELECT * FROM %s WHERE c > ? ALLOW FILTERING",
- unset());
- assertInvalidMessage("Unsupported unset value for column c",
- "SELECT * FROM %s WHERE c CONTAINS ? ALLOW FILTERING",
- unset());
- assertInvalidMessage("Unsupported unset value for column c",
- "SELECT * FROM %s WHERE c CONTAINS KEY ? ALLOW FILTERING",
- unset());
}
@Test
@@ -3742,99 +2322,6 @@
}
@Test
- public void filteringOnCompactTable() throws Throwable
- {
- createTable("CREATE TABLE %s (a int, b int, c int, d int, PRIMARY KEY (a, b, c)) WITH COMPACT STORAGE");
-
- execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 11, 12, 13, 14);
- execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 21, 22, 23, 24);
- execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 21, 25, 26, 27);
- execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 31, 32, 33, 34);
-
- beforeAndAfterFlush(() -> {
-
- assertRows(executeFilteringOnly("SELECT * FROM %s WHERE c > 13"),
- row(21, 22, 23, 24),
- row(21, 25, 26, 27),
- row(31, 32, 33, 34));
-
- assertRows(executeFilteringOnly("SELECT * FROM %s WHERE c > 13 AND c < 33"),
- row(21, 22, 23, 24),
- row(21, 25, 26, 27));
-
- assertRows(executeFilteringOnly("SELECT * FROM %s WHERE c > 13 AND b < 32"),
- row(21, 22, 23, 24),
- row(21, 25, 26, 27));
-
- assertRows(executeFilteringOnly("SELECT * FROM %s WHERE a = 21 AND c > 13 AND b < 32 ORDER BY b DESC"),
- row(21, 25, 26, 27),
- row(21, 22, 23, 24));
-
- assertRows(executeFilteringOnly("SELECT * FROM %s WHERE a IN (21, 31) AND c > 13 ORDER BY b DESC"),
- row(31, 32, 33, 34),
- row(21, 25, 26, 27),
- row(21, 22, 23, 24));
-
- assertRows(executeFilteringOnly("SELECT * FROM %s WHERE c > 13 AND d < 34"),
- row(21, 22, 23, 24),
- row(21, 25, 26, 27));
-
- assertRows(executeFilteringOnly("SELECT * FROM %s WHERE c > 13"),
- row(21, 22, 23, 24),
- row(21, 25, 26, 27),
- row(31, 32, 33, 34));
- });
-
- // with frozen in clustering key
- createTable("CREATE TABLE %s (a int, b int, c frozen<list<int>>, d int, PRIMARY KEY (a, b, c)) WITH COMPACT STORAGE");
-
- execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 11, 12, list(1, 3), 14);
- execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 21, 22, list(2, 3), 24);
- execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 21, 25, list(2, 6), 27);
- execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 31, 32, list(3, 3), 34);
-
- beforeAndAfterFlush(() -> {
-
- assertRows(executeFilteringOnly("SELECT * FROM %s WHERE c CONTAINS 2"),
- row(21, 22, list(2, 3), 24),
- row(21, 25, list(2, 6), 27));
-
- assertRows(executeFilteringOnly("SELECT * FROM %s WHERE c CONTAINS 2 AND b < 25"),
- row(21, 22, list(2, 3), 24));
-
- assertRows(executeFilteringOnly("SELECT * FROM %s WHERE c CONTAINS 2 AND c CONTAINS 3"),
- row(21, 22, list(2, 3), 24));
-
- assertRows(executeFilteringOnly("SELECT * FROM %s WHERE b > 12 AND c CONTAINS 2 AND d < 27"),
- row(21, 22, list(2, 3), 24));
- });
-
- // with frozen in value
- createTable("CREATE TABLE %s (a int, b int, c int, d frozen<list<int>>, PRIMARY KEY (a, b, c)) WITH COMPACT STORAGE");
-
- execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 11, 12, 13, list(1, 4));
- execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 21, 22, 23, list(2, 4));
- execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 21, 25, 25, list(2, 6));
- execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 31, 32, 34, list(3, 4));
-
- beforeAndAfterFlush(() -> {
-
- assertRows(executeFilteringOnly("SELECT * FROM %s WHERE d CONTAINS 2"),
- row(21, 22, 23, list(2, 4)),
- row(21, 25, 25, list(2, 6)));
-
- assertRows(executeFilteringOnly("SELECT * FROM %s WHERE d CONTAINS 2 AND b < 25"),
- row(21, 22, 23, list(2, 4)));
-
- assertRows(executeFilteringOnly("SELECT * FROM %s WHERE d CONTAINS 2 AND d CONTAINS 4"),
- row(21, 22, 23, list(2, 4)));
-
- assertRows(executeFilteringOnly("SELECT * FROM %s WHERE b > 12 AND c < 25 AND d CONTAINS 2"),
- row(21, 22, 23, list(2, 4)));
- });
- }
-
- @Test
public void testCustomIndexWithFiltering() throws Throwable
{
// Test for CASSANDRA-11310 compatibility with 2i
@@ -3860,36 +2347,33 @@
@Test
public void testFilteringWithCounters() throws Throwable
{
- for (String compactStorageClause: new String[] {"", " WITH COMPACT STORAGE"})
- {
- createTable("CREATE TABLE %s (a int, b int, c int, cnt counter, PRIMARY KEY (a, b, c))" + compactStorageClause);
+ createTable("CREATE TABLE %s (a int, b int, c int, cnt counter, PRIMARY KEY (a, b, c))");
- execute("UPDATE %s SET cnt = cnt + ? WHERE a = ? AND b = ? AND c = ?", 14L, 11, 12, 13);
- execute("UPDATE %s SET cnt = cnt + ? WHERE a = ? AND b = ? AND c = ?", 24L, 21, 22, 23);
- execute("UPDATE %s SET cnt = cnt + ? WHERE a = ? AND b = ? AND c = ?", 27L, 21, 25, 26);
- execute("UPDATE %s SET cnt = cnt + ? WHERE a = ? AND b = ? AND c = ?", 34L, 31, 32, 33);
- execute("UPDATE %s SET cnt = cnt + ? WHERE a = ? AND b = ? AND c = ?", 24L, 41, 42, 43);
+ execute("UPDATE %s SET cnt = cnt + ? WHERE a = ? AND b = ? AND c = ?", 14L, 11, 12, 13);
+ execute("UPDATE %s SET cnt = cnt + ? WHERE a = ? AND b = ? AND c = ?", 24L, 21, 22, 23);
+ execute("UPDATE %s SET cnt = cnt + ? WHERE a = ? AND b = ? AND c = ?", 27L, 21, 25, 26);
+ execute("UPDATE %s SET cnt = cnt + ? WHERE a = ? AND b = ? AND c = ?", 34L, 31, 32, 33);
+ execute("UPDATE %s SET cnt = cnt + ? WHERE a = ? AND b = ? AND c = ?", 24L, 41, 42, 43);
- beforeAndAfterFlush(() -> {
+ beforeAndAfterFlush(() -> {
- assertRows(executeFilteringOnly("SELECT * FROM %s WHERE cnt = 24"),
- row(21, 22, 23, 24L),
- row(41, 42, 43, 24L));
- assertRows(executeFilteringOnly("SELECT * FROM %s WHERE b > 22 AND cnt = 24"),
- row(41, 42, 43, 24L));
- assertRows(executeFilteringOnly("SELECT * FROM %s WHERE b > 10 AND b < 25 AND cnt = 24"),
- row(21, 22, 23, 24L));
- assertRows(executeFilteringOnly("SELECT * FROM %s WHERE b > 10 AND c < 25 AND cnt = 24"),
- row(21, 22, 23, 24L));
- assertRows(executeFilteringOnly("SELECT * FROM %s WHERE a = 21 AND b > 10 AND cnt > 23 ORDER BY b DESC"),
- row(21, 25, 26, 27L),
- row(21, 22, 23, 24L));
- assertRows(executeFilteringOnly("SELECT * FROM %s WHERE cnt > 20 AND cnt < 30"),
- row(21, 22, 23, 24L),
- row(21, 25, 26, 27L),
- row(41, 42, 43, 24L));
- });
- }
+ assertRows(executeFilteringOnly("SELECT * FROM %s WHERE cnt = 24"),
+ row(21, 22, 23, 24L),
+ row(41, 42, 43, 24L));
+ assertRows(executeFilteringOnly("SELECT * FROM %s WHERE b > 22 AND cnt = 24"),
+ row(41, 42, 43, 24L));
+ assertRows(executeFilteringOnly("SELECT * FROM %s WHERE b > 10 AND b < 25 AND cnt = 24"),
+ row(21, 22, 23, 24L));
+ assertRows(executeFilteringOnly("SELECT * FROM %s WHERE b > 10 AND c < 25 AND cnt = 24"),
+ row(21, 22, 23, 24L));
+ assertRows(executeFilteringOnly("SELECT * FROM %s WHERE a = 21 AND b > 10 AND cnt > 23 ORDER BY b DESC"),
+ row(21, 25, 26, 27L),
+ row(21, 22, 23, 24L));
+ assertRows(executeFilteringOnly("SELECT * FROM %s WHERE cnt > 20 AND cnt < 30"),
+ row(21, 22, 23, 24L),
+ row(21, 25, 26, 27L),
+ row(41, 42, 43, 24L));
+ });
}
private UntypedResultSet executeFilteringOnly(String statement) throws Throwable
@@ -3899,76 +2383,62 @@
}
/**
- * Check select with and without compact storage, with different column
- * order. See CASSANDRA-10988
+ * Check select with ith different column order. See CASSANDRA-10988
*/
@Test
public void testClusteringOrderWithSlice() throws Throwable
{
- for (String compactOption : new String[] { "", " COMPACT STORAGE AND" })
- {
- // non-compound, ASC order
- createTable("CREATE TABLE %s (a text, b int, PRIMARY KEY (a, b)) WITH" +
- compactOption +
- " CLUSTERING ORDER BY (b ASC)");
+ // non-compound, ASC order
+ createTable("CREATE TABLE %s (a text, b int, PRIMARY KEY (a, b)) WITH CLUSTERING ORDER BY (b ASC)");
- execute("INSERT INTO %s (a, b) VALUES ('a', 2)");
- execute("INSERT INTO %s (a, b) VALUES ('a', 3)");
- assertRows(execute("SELECT * FROM %s WHERE a = 'a' AND b > 0"),
- row("a", 2),
- row("a", 3));
+ execute("INSERT INTO %s (a, b) VALUES ('a', 2)");
+ execute("INSERT INTO %s (a, b) VALUES ('a', 3)");
+ assertRows(execute("SELECT * FROM %s WHERE a = 'a' AND b > 0"),
+ row("a", 2),
+ row("a", 3));
- assertRows(execute("SELECT * FROM %s WHERE a = 'a' AND b > 0 ORDER BY b DESC"),
- row("a", 3),
- row("a", 2));
+ assertRows(execute("SELECT * FROM %s WHERE a = 'a' AND b > 0 ORDER BY b DESC"),
+ row("a", 3),
+ row("a", 2));
- // non-compound, DESC order
- createTable("CREATE TABLE %s (a text, b int, PRIMARY KEY (a, b)) WITH" +
- compactOption +
- " CLUSTERING ORDER BY (b DESC)");
+ // non-compound, DESC order
+ createTable("CREATE TABLE %s (a text, b int, PRIMARY KEY (a, b)) WITH CLUSTERING ORDER BY (b DESC)");
- execute("INSERT INTO %s (a, b) VALUES ('a', 2)");
- execute("INSERT INTO %s (a, b) VALUES ('a', 3)");
- assertRows(execute("SELECT * FROM %s WHERE a = 'a' AND b > 0"),
- row("a", 3),
- row("a", 2));
+ execute("INSERT INTO %s (a, b) VALUES ('a', 2)");
+ execute("INSERT INTO %s (a, b) VALUES ('a', 3)");
+ assertRows(execute("SELECT * FROM %s WHERE a = 'a' AND b > 0"),
+ row("a", 3),
+ row("a", 2));
- assertRows(execute("SELECT * FROM %s WHERE a = 'a' AND b > 0 ORDER BY b ASC"),
- row("a", 2),
- row("a", 3));
+ assertRows(execute("SELECT * FROM %s WHERE a = 'a' AND b > 0 ORDER BY b ASC"),
+ row("a", 2),
+ row("a", 3));
- // compound, first column DESC order
- createTable("CREATE TABLE %s (a text, b int, c int, PRIMARY KEY (a, b, c)) WITH" +
- compactOption +
- " CLUSTERING ORDER BY (b DESC)"
- );
+ // compound, first column DESC order
+ createTable("CREATE TABLE %s (a text, b int, c int, PRIMARY KEY (a, b, c)) WITH CLUSTERING ORDER BY (b DESC, c ASC)");
- execute("INSERT INTO %s (a, b, c) VALUES ('a', 2, 4)");
- execute("INSERT INTO %s (a, b, c) VALUES ('a', 3, 5)");
- assertRows(execute("SELECT * FROM %s WHERE a = 'a' AND b > 0"),
- row("a", 3, 5),
- row("a", 2, 4));
+ execute("INSERT INTO %s (a, b, c) VALUES ('a', 2, 4)");
+ execute("INSERT INTO %s (a, b, c) VALUES ('a', 3, 5)");
+ assertRows(execute("SELECT * FROM %s WHERE a = 'a' AND b > 0"),
+ row("a", 3, 5),
+ row("a", 2, 4));
- assertRows(execute("SELECT * FROM %s WHERE a = 'a' AND b > 0 ORDER BY b ASC"),
- row("a", 2, 4),
- row("a", 3, 5));
+ assertRows(execute("SELECT * FROM %s WHERE a = 'a' AND b > 0 ORDER BY b ASC"),
+ row("a", 2, 4),
+ row("a", 3, 5));
- // compound, mixed order
- createTable("CREATE TABLE %s (a text, b int, c int, PRIMARY KEY (a, b, c)) WITH" +
- compactOption +
- " CLUSTERING ORDER BY (b ASC, c DESC)"
- );
+ // compound, mixed order
+ createTable("CREATE TABLE %s (a text, b int, c int, PRIMARY KEY (a, b, c)) WITH CLUSTERING ORDER BY (b ASC, c DESC)");
- execute("INSERT INTO %s (a, b, c) VALUES ('a', 2, 4)");
- execute("INSERT INTO %s (a, b, c) VALUES ('a', 3, 5)");
- assertRows(execute("SELECT * FROM %s WHERE a = 'a' AND b > 0"),
- row("a", 2, 4),
- row("a", 3, 5));
+ execute("INSERT INTO %s (a, b, c) VALUES ('a', 2, 4)");
+ execute("INSERT INTO %s (a, b, c) VALUES ('a', 3, 5)");
+ assertRows(execute("SELECT * FROM %s WHERE a = 'a' AND b > 0"),
+ row("a", 2, 4),
+ row("a", 3, 5));
- assertRows(execute("SELECT * FROM %s WHERE a = 'a' AND b > 0 ORDER BY b ASC"),
- row("a", 2, 4),
- row("a", 3, 5));
- }
+ assertRows(execute("SELECT * FROM %s WHERE a = 'a' AND b > 0 ORDER BY b ASC"),
+ row("a", 2, 4),
+ row("a", 3, 5));
}
@Test
@@ -4032,226 +2502,210 @@
@Test
public void testEmptyRestrictionValue() throws Throwable
{
- for (String options : new String[] { "", " WITH COMPACT STORAGE" })
- {
- createTable("CREATE TABLE %s (pk blob, c blob, v blob, PRIMARY KEY ((pk), c))" + options);
- execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?)",
- bytes("foo123"), bytes("1"), bytes("1"));
- execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?)",
- bytes("foo123"), bytes("2"), bytes("2"));
+ createTable("CREATE TABLE %s (pk blob, c blob, v blob, PRIMARY KEY ((pk), c))");
+ execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?)",
+ bytes("foo123"), bytes("1"), bytes("1"));
+ execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?)",
+ bytes("foo123"), bytes("2"), bytes("2"));
- beforeAndAfterFlush(() -> {
+ beforeAndAfterFlush(() -> {
- assertInvalidMessage("Key may not be empty", "SELECT * FROM %s WHERE pk = textAsBlob('');");
- assertInvalidMessage("Key may not be empty", "SELECT * FROM %s WHERE pk IN (textAsBlob(''), textAsBlob('1'));");
+ assertInvalidMessage("Key may not be empty", "SELECT * FROM %s WHERE pk = textAsBlob('');");
+ assertInvalidMessage("Key may not be empty", "SELECT * FROM %s WHERE pk IN (textAsBlob(''), textAsBlob('1'));");
- assertInvalidMessage("Key may not be empty",
- "INSERT INTO %s (pk, c, v) VALUES (?, ?, ?)",
- EMPTY_BYTE_BUFFER, bytes("2"), bytes("2"));
+ assertInvalidMessage("Key may not be empty",
+ "INSERT INTO %s (pk, c, v) VALUES (?, ?, ?)",
+ EMPTY_BYTE_BUFFER, bytes("2"), bytes("2"));
- // Test clustering columns restrictions
- assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c = textAsBlob('');"));
+ // Test clustering columns restrictions
+ assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c = textAsBlob('');"));
- assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c) = (textAsBlob(''));"));
+ assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c) = (textAsBlob(''));"));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c IN (textAsBlob(''), textAsBlob('1'));"),
- row(bytes("foo123"), bytes("1"), bytes("1")));
+ assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c IN (textAsBlob(''), textAsBlob('1'));"),
+ row(bytes("foo123"), bytes("1"), bytes("1")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c) IN ((textAsBlob('')), (textAsBlob('1')));"),
- row(bytes("foo123"), bytes("1"), bytes("1")));
+ assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c) IN ((textAsBlob('')), (textAsBlob('1')));"),
+ row(bytes("foo123"), bytes("1"), bytes("1")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c > textAsBlob('');"),
- row(bytes("foo123"), bytes("1"), bytes("1")),
- row(bytes("foo123"), bytes("2"), bytes("2")));
+ assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c > textAsBlob('');"),
+ row(bytes("foo123"), bytes("1"), bytes("1")),
+ row(bytes("foo123"), bytes("2"), bytes("2")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c) > (textAsBlob(''));"),
- row(bytes("foo123"), bytes("1"), bytes("1")),
- row(bytes("foo123"), bytes("2"), bytes("2")));
+ assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c) > (textAsBlob(''));"),
+ row(bytes("foo123"), bytes("1"), bytes("1")),
+ row(bytes("foo123"), bytes("2"), bytes("2")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c >= textAsBlob('');"),
- row(bytes("foo123"), bytes("1"), bytes("1")),
- row(bytes("foo123"), bytes("2"), bytes("2")));
+ assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c >= textAsBlob('');"),
+ row(bytes("foo123"), bytes("1"), bytes("1")),
+ row(bytes("foo123"), bytes("2"), bytes("2")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c) >= (textAsBlob(''));"),
- row(bytes("foo123"), bytes("1"), bytes("1")),
- row(bytes("foo123"), bytes("2"), bytes("2")));
+ assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c) >= (textAsBlob(''));"),
+ row(bytes("foo123"), bytes("1"), bytes("1")),
+ row(bytes("foo123"), bytes("2"), bytes("2")));
- assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c <= textAsBlob('');"));
+ assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c <= textAsBlob('');"));
- assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c) <= (textAsBlob(''));"));
+ assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c) <= (textAsBlob(''));"));
- assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c < textAsBlob('');"));
+ assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c < textAsBlob('');"));
- assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c) < (textAsBlob(''));"));
+ assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c) < (textAsBlob(''));"));
- assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c > textAsBlob('') AND c < textAsBlob('');"));
- });
+ assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c > textAsBlob('') AND c < textAsBlob('');"));
+ });
- if (options.contains("COMPACT"))
- {
- assertInvalidMessage("Invalid empty or null value for column c",
- "INSERT INTO %s (pk, c, v) VALUES (?, ?, ?)",
- bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("4"));
- }
- else
- {
- execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?)",
- bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("4"));
- beforeAndAfterFlush(() -> {
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c = textAsBlob('');"),
- row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("4")));
+ execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?)",
+ bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("4"));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c) = (textAsBlob(''));"),
- row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("4")));
+ beforeAndAfterFlush(() -> {
+ assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c = textAsBlob('');"),
+ row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("4")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c IN (textAsBlob(''), textAsBlob('1'));"),
- row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("4")),
- row(bytes("foo123"), bytes("1"), bytes("1")));
+ assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c) = (textAsBlob(''));"),
+ row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("4")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c) IN ((textAsBlob('')), (textAsBlob('1')));"),
- row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("4")),
- row(bytes("foo123"), bytes("1"), bytes("1")));
+ assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c IN (textAsBlob(''), textAsBlob('1'));"),
+ row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("4")),
+ row(bytes("foo123"), bytes("1"), bytes("1")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c > textAsBlob('');"),
- row(bytes("foo123"), bytes("1"), bytes("1")),
- row(bytes("foo123"), bytes("2"), bytes("2")));
+ assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c) IN ((textAsBlob('')), (textAsBlob('1')));"),
+ row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("4")),
+ row(bytes("foo123"), bytes("1"), bytes("1")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c) > (textAsBlob(''));"),
- row(bytes("foo123"), bytes("1"), bytes("1")),
- row(bytes("foo123"), bytes("2"), bytes("2")));
+ assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c > textAsBlob('');"),
+ row(bytes("foo123"), bytes("1"), bytes("1")),
+ row(bytes("foo123"), bytes("2"), bytes("2")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c >= textAsBlob('');"),
- row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("4")),
- row(bytes("foo123"), bytes("1"), bytes("1")),
- row(bytes("foo123"), bytes("2"), bytes("2")));
+ assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c) > (textAsBlob(''));"),
+ row(bytes("foo123"), bytes("1"), bytes("1")),
+ row(bytes("foo123"), bytes("2"), bytes("2")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c) >= (textAsBlob(''));"),
- row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("4")),
- row(bytes("foo123"), bytes("1"), bytes("1")),
- row(bytes("foo123"), bytes("2"), bytes("2")));
+ assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c >= textAsBlob('');"),
+ row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("4")),
+ row(bytes("foo123"), bytes("1"), bytes("1")),
+ row(bytes("foo123"), bytes("2"), bytes("2")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c <= textAsBlob('');"),
- row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("4")));
+ assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c) >= (textAsBlob(''));"),
+ row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("4")),
+ row(bytes("foo123"), bytes("1"), bytes("1")),
+ row(bytes("foo123"), bytes("2"), bytes("2")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c) <= (textAsBlob(''));"),
- row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("4")));
+ assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c <= textAsBlob('');"),
+ row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("4")));
- assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c < textAsBlob('');"));
+ assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c) <= (textAsBlob(''));"),
+ row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("4")));
- assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c) < (textAsBlob(''));"));
+ assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c < textAsBlob('');"));
- assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c >= textAsBlob('') AND c < textAsBlob('');"));
- });
- }
+ assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c) < (textAsBlob(''));"));
- // Test restrictions on non-primary key value
- assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND v = textAsBlob('') ALLOW FILTERING;"));
+ assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c >= textAsBlob('') AND c < textAsBlob('');"));
+ });
- execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?)",
- bytes("foo123"), bytes("3"), EMPTY_BYTE_BUFFER);
+ // Test restrictions on non-primary key value
+ assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND v = textAsBlob('') ALLOW FILTERING;"));
- beforeAndAfterFlush(() -> {
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND v = textAsBlob('') ALLOW FILTERING;"),
- row(bytes("foo123"), bytes("3"), EMPTY_BYTE_BUFFER));
- });
- }
+ execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?)",
+ bytes("foo123"), bytes("3"), EMPTY_BYTE_BUFFER);
+
+ beforeAndAfterFlush(() -> {
+ assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND v = textAsBlob('') ALLOW FILTERING;"),
+ row(bytes("foo123"), bytes("3"), EMPTY_BYTE_BUFFER));
+ });
}
@Test
public void testEmptyRestrictionValueWithMultipleClusteringColumns() throws Throwable
{
- for (String options : new String[] { "", " WITH COMPACT STORAGE" })
- {
- createTable("CREATE TABLE %s (pk blob, c1 blob, c2 blob, v blob, PRIMARY KEY (pk, c1, c2))" + options);
- execute("INSERT INTO %s (pk, c1, c2, v) VALUES (?, ?, ?, ?)", bytes("foo123"), bytes("1"), bytes("1"), bytes("1"));
- execute("INSERT INTO %s (pk, c1, c2, v) VALUES (?, ?, ?, ?)", bytes("foo123"), bytes("1"), bytes("2"), bytes("2"));
+ createTable("CREATE TABLE %s (pk blob, c1 blob, c2 blob, v blob, PRIMARY KEY (pk, c1, c2))");
+ execute("INSERT INTO %s (pk, c1, c2, v) VALUES (?, ?, ?, ?)", bytes("foo123"), bytes("1"), bytes("1"), bytes("1"));
+ execute("INSERT INTO %s (pk, c1, c2, v) VALUES (?, ?, ?, ?)", bytes("foo123"), bytes("1"), bytes("2"), bytes("2"));
- beforeAndAfterFlush(() -> {
+ beforeAndAfterFlush(() -> {
- assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c1 = textAsBlob('');"));
+ assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c1 = textAsBlob('');"));
- assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c1 = textAsBlob('1') AND c2 = textAsBlob('');"));
+ assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c1 = textAsBlob('1') AND c2 = textAsBlob('');"));
- assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c1, c2) = (textAsBlob('1'), textAsBlob(''));"));
+ assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c1, c2) = (textAsBlob('1'), textAsBlob(''));"));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c1 IN (textAsBlob(''), textAsBlob('1')) AND c2 = textAsBlob('1');"),
- row(bytes("foo123"), bytes("1"), bytes("1"), bytes("1")));
+ assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c1 IN (textAsBlob(''), textAsBlob('1')) AND c2 = textAsBlob('1');"),
+ row(bytes("foo123"), bytes("1"), bytes("1"), bytes("1")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c1 = textAsBlob('1') AND c2 IN (textAsBlob(''), textAsBlob('1'));"),
- row(bytes("foo123"), bytes("1"), bytes("1"), bytes("1")));
+ assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c1 = textAsBlob('1') AND c2 IN (textAsBlob(''), textAsBlob('1'));"),
+ row(bytes("foo123"), bytes("1"), bytes("1"), bytes("1")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c1, c2) IN ((textAsBlob(''), textAsBlob('1')), (textAsBlob('1'), textAsBlob('1')));"),
- row(bytes("foo123"), bytes("1"), bytes("1"), bytes("1")));
+ assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c1, c2) IN ((textAsBlob(''), textAsBlob('1')), (textAsBlob('1'), textAsBlob('1')));"),
+ row(bytes("foo123"), bytes("1"), bytes("1"), bytes("1")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c1 > textAsBlob('');"),
- row(bytes("foo123"), bytes("1"), bytes("1"), bytes("1")),
- row(bytes("foo123"), bytes("1"), bytes("2"), bytes("2")));
+ assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c1 > textAsBlob('');"),
+ row(bytes("foo123"), bytes("1"), bytes("1"), bytes("1")),
+ row(bytes("foo123"), bytes("1"), bytes("2"), bytes("2")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c1 = textAsBlob('1') AND c2 > textAsBlob('');"),
- row(bytes("foo123"), bytes("1"), bytes("1"), bytes("1")),
- row(bytes("foo123"), bytes("1"), bytes("2"), bytes("2")));
+ assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c1 = textAsBlob('1') AND c2 > textAsBlob('');"),
+ row(bytes("foo123"), bytes("1"), bytes("1"), bytes("1")),
+ row(bytes("foo123"), bytes("1"), bytes("2"), bytes("2")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c1, c2) > (textAsBlob(''), textAsBlob('1'));"),
- row(bytes("foo123"), bytes("1"), bytes("1"), bytes("1")),
- row(bytes("foo123"), bytes("1"), bytes("2"), bytes("2")));
+ assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c1, c2) > (textAsBlob(''), textAsBlob('1'));"),
+ row(bytes("foo123"), bytes("1"), bytes("1"), bytes("1")),
+ row(bytes("foo123"), bytes("1"), bytes("2"), bytes("2")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c1 = textAsBlob('1') AND c2 >= textAsBlob('');"),
- row(bytes("foo123"), bytes("1"), bytes("1"), bytes("1")),
- row(bytes("foo123"), bytes("1"), bytes("2"), bytes("2")));
+ assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c1 = textAsBlob('1') AND c2 >= textAsBlob('');"),
+ row(bytes("foo123"), bytes("1"), bytes("1"), bytes("1")),
+ row(bytes("foo123"), bytes("1"), bytes("2"), bytes("2")));
- assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c1 = textAsBlob('1') AND c2 <= textAsBlob('');"));
+ assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c1 = textAsBlob('1') AND c2 <= textAsBlob('');"));
- assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c1, c2) <= (textAsBlob('1'), textAsBlob(''));"));
- });
+ assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c1, c2) <= (textAsBlob('1'), textAsBlob(''));"));
+ });
- execute("INSERT INTO %s (pk, c1, c2, v) VALUES (?, ?, ?, ?)",
- bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("1"), bytes("4"));
+ execute("INSERT INTO %s (pk, c1, c2, v) VALUES (?, ?, ?, ?)",
+ bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("1"), bytes("4"));
- beforeAndAfterFlush(() -> {
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c1 = textAsBlob('');"),
- row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("1"), bytes("4")));
+ beforeAndAfterFlush(() -> {
+ assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c1 = textAsBlob('');"),
+ row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("1"), bytes("4")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c1 = textAsBlob('') AND c2 = textAsBlob('1');"),
- row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("1"), bytes("4")));
+ assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c1 = textAsBlob('') AND c2 = textAsBlob('1');"),
+ row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("1"), bytes("4")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c1, c2) = (textAsBlob(''), textAsBlob('1'));"),
- row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("1"), bytes("4")));
+ assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c1, c2) = (textAsBlob(''), textAsBlob('1'));"),
+ row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("1"), bytes("4")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c1 IN (textAsBlob(''), textAsBlob('1')) AND c2 = textAsBlob('1');"),
- row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("1"), bytes("4")),
- row(bytes("foo123"), bytes("1"), bytes("1"), bytes("1")));
+ assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c1 IN (textAsBlob(''), textAsBlob('1')) AND c2 = textAsBlob('1');"),
+ row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("1"), bytes("4")),
+ row(bytes("foo123"), bytes("1"), bytes("1"), bytes("1")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c1, c2) IN ((textAsBlob(''), textAsBlob('1')), (textAsBlob('1'), textAsBlob('1')));"),
- row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("1"), bytes("4")),
- row(bytes("foo123"), bytes("1"), bytes("1"), bytes("1")));
+ assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c1, c2) IN ((textAsBlob(''), textAsBlob('1')), (textAsBlob('1'), textAsBlob('1')));"),
+ row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("1"), bytes("4")),
+ row(bytes("foo123"), bytes("1"), bytes("1"), bytes("1")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c1, c2) > (textAsBlob(''), textAsBlob('1'));"),
- row(bytes("foo123"), bytes("1"), bytes("1"), bytes("1")),
- row(bytes("foo123"), bytes("1"), bytes("2"), bytes("2")));
+ assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c1, c2) > (textAsBlob(''), textAsBlob('1'));"),
+ row(bytes("foo123"), bytes("1"), bytes("1"), bytes("1")),
+ row(bytes("foo123"), bytes("1"), bytes("2"), bytes("2")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c1, c2) >= (textAsBlob(''), textAsBlob('1'));"),
- row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("1"), bytes("4")),
- row(bytes("foo123"), bytes("1"), bytes("1"), bytes("1")),
- row(bytes("foo123"), bytes("1"), bytes("2"), bytes("2")));
+ assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c1, c2) >= (textAsBlob(''), textAsBlob('1'));"),
+ row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("1"), bytes("4")),
+ row(bytes("foo123"), bytes("1"), bytes("1"), bytes("1")),
+ row(bytes("foo123"), bytes("1"), bytes("2"), bytes("2")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c1, c2) <= (textAsBlob(''), textAsBlob('1'));"),
- row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("1"), bytes("4")));
+ assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c1, c2) <= (textAsBlob(''), textAsBlob('1'));"),
+ row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("1"), bytes("4")));
- assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c1, c2) < (textAsBlob(''), textAsBlob('1'));"));
- });
- }
+ assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c1, c2) < (textAsBlob(''), textAsBlob('1'));"));
+ });
}
@Test
public void testEmptyRestrictionValueWithOrderBy() throws Throwable
{
- for (String options : new String[] { "",
- " WITH COMPACT STORAGE",
- " WITH CLUSTERING ORDER BY (c DESC)",
- " WITH COMPACT STORAGE AND CLUSTERING ORDER BY (c DESC)"})
+ for (String options : new String[]{ "",
+ " WITH CLUSTERING ORDER BY (c DESC)" })
{
- String orderingClause = options.contains("ORDER") ? "" : "ORDER BY c DESC" ;
+ String orderingClause = options.contains("ORDER") ? "" : "ORDER BY c DESC";
createTable("CREATE TABLE %s (pk blob, c blob, v blob, PRIMARY KEY ((pk), c))" + options);
execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?)",
@@ -4276,55 +2730,41 @@
assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c < textAsBlob('')" + orderingClause));
assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c <= textAsBlob('')" + orderingClause));
-
});
- if (options.contains("COMPACT"))
- {
- assertInvalidMessage("Invalid empty or null value for column c",
- "INSERT INTO %s (pk, c, v) VALUES (?, ?, ?)",
- bytes("foo123"),
- EMPTY_BYTE_BUFFER,
- bytes("4"));
- }
- else
- {
- execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?)",
- bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("4"));
+ execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?)",
+ bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("4"));
- beforeAndAfterFlush(() -> {
+ beforeAndAfterFlush(() -> {
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c IN (textAsBlob(''), textAsBlob('1'))" + orderingClause),
- row(bytes("foo123"), bytes("1"), bytes("1")),
- row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("4")));
+ assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c IN (textAsBlob(''), textAsBlob('1'))" + orderingClause),
+ row(bytes("foo123"), bytes("1"), bytes("1")),
+ row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("4")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c > textAsBlob('')" + orderingClause),
- row(bytes("foo123"), bytes("2"), bytes("2")),
- row(bytes("foo123"), bytes("1"), bytes("1")));
+ assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c > textAsBlob('')" + orderingClause),
+ row(bytes("foo123"), bytes("2"), bytes("2")),
+ row(bytes("foo123"), bytes("1"), bytes("1")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c >= textAsBlob('')" + orderingClause),
- row(bytes("foo123"), bytes("2"), bytes("2")),
- row(bytes("foo123"), bytes("1"), bytes("1")),
- row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("4")));
+ assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c >= textAsBlob('')" + orderingClause),
+ row(bytes("foo123"), bytes("2"), bytes("2")),
+ row(bytes("foo123"), bytes("1"), bytes("1")),
+ row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("4")));
- assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c < textAsBlob('')" + orderingClause));
+ assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c < textAsBlob('')" + orderingClause));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c <= textAsBlob('')" + orderingClause),
- row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("4")));
- });
- }
+ assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c <= textAsBlob('')" + orderingClause),
+ row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("4")));
+ });
}
}
@Test
public void testEmptyRestrictionValueWithMultipleClusteringColumnsAndOrderBy() throws Throwable
{
- for (String options : new String[] { "",
- " WITH COMPACT STORAGE",
- " WITH CLUSTERING ORDER BY (c1 DESC, c2 DESC)",
- " WITH COMPACT STORAGE AND CLUSTERING ORDER BY (c1 DESC, c2 DESC)"})
+ for (String options : new String[]{ "",
+ " WITH CLUSTERING ORDER BY (c1 DESC, c2 DESC)" })
{
- String orderingClause = options.contains("ORDER") ? "" : "ORDER BY c1 DESC, c2 DESC" ;
+ String orderingClause = options.contains("ORDER") ? "" : "ORDER BY c1 DESC, c2 DESC";
createTable("CREATE TABLE %s (pk blob, c1 blob, c2 blob, v blob, PRIMARY KEY (pk, c1, c2))" + options);
execute("INSERT INTO %s (pk, c1, c2, v) VALUES (?, ?, ?, ?)", bytes("foo123"), bytes("1"), bytes("1"), bytes("1"));
@@ -4375,6 +2815,16 @@
}
@Test
+ public void testWithDistinctAndJsonAsColumnName() throws Throwable
+ {
+ createTable("CREATE TABLE %s (distinct int, json int, value int, PRIMARY KEY(distinct, json))");
+ execute("INSERT INTO %s (distinct, json, value) VALUES (0, 0, 0)");
+
+ assertRows(execute("SELECT distinct, json FROM %s"), row(0, 0));
+ assertRows(execute("SELECT distinct distinct FROM %s"), row(0));
+ }
+
+ @Test
public void testFilteringOnDurationColumn() throws Throwable
{
createTable("CREATE TABLE %s (k int PRIMARY KEY, d duration)");
@@ -4604,7 +3054,6 @@
i++;
}
-
}
@Test
@@ -4637,83 +3086,77 @@
}
}
- /**
- * Test for CASSANDRA-13917
- */
- @Test
- public void testWithCompactStaticFormat() throws Throwable
+ @Test // CASSANDRA-14989
+ public void testTokenFctAcceptsValidArguments() throws Throwable
{
- createTable("CREATE TABLE %s (a int PRIMARY KEY, b int, c int) WITH COMPACT STORAGE");
- execute("INSERT INTO %s (a, b, c) VALUES (1, 1, 1)");
- execute("INSERT INTO %s (a, b, c) VALUES (2, 1, 1)");
- assertRows(execute("SELECT a, b, c FROM %s"),
- row(1, 1, 1),
- row(2, 1, 1));
- testWithCompactFormat();
-
- // if column column1 is present, hidden column is called column2
- createTable("CREATE TABLE %s (a int PRIMARY KEY, b int, c int, column1 int) WITH COMPACT STORAGE");
- execute("INSERT INTO %s (a, b, c, column1) VALUES (1, 1, 1, 1)");
- execute("INSERT INTO %s (a, b, c, column1) VALUES (2, 1, 1, 2)");
- assertRows(execute("SELECT a, b, c, column1 FROM %s"),
- row(1, 1, 1, 1),
- row(2, 1, 1, 2));
- assertInvalidMessage("Undefined column name column2",
- "SELECT a, column2, value FROM %s");
-
- // if column value is present, hidden column is called value1
- createTable("CREATE TABLE %s (a int PRIMARY KEY, b int, c int, value int) WITH COMPACT STORAGE");
- execute("INSERT INTO %s (a, b, c, value) VALUES (1, 1, 1, 1)");
- execute("INSERT INTO %s (a, b, c, value) VALUES (2, 1, 1, 2)");
- assertRows(execute("SELECT a, b, c, value FROM %s"),
- row(1, 1, 1, 1),
- row(2, 1, 1, 2));
- assertInvalidMessage("Undefined column name value1",
- "SELECT a, value1, value FROM %s");
+ createTable("CREATE TABLE %s (k1 uuid, k2 text, PRIMARY KEY ((k1, k2)))");
+ execute("INSERT INTO %s (k1, k2) VALUES (uuid(), 'k2')");
+ assertRowCount(execute("SELECT token(k1, k2) FROM %s"), 1);
}
- /**
- * Test for CASSANDRA-13917
- */
@Test
- public void testWithCompactNonStaticFormat() throws Throwable
+ public void testTokenFctRejectsInvalidColumnName() throws Throwable
{
- createTable("CREATE TABLE %s (a int, b int, PRIMARY KEY (a, b)) WITH COMPACT STORAGE");
- execute("INSERT INTO %s (a, b) VALUES (1, 1)");
- execute("INSERT INTO %s (a, b) VALUES (2, 1)");
- assertRows(execute("SELECT a, b FROM %s"),
- row(1, 1),
- row(2, 1));
- testWithCompactFormat();
-
- createTable("CREATE TABLE %s (a int, b int, v int, PRIMARY KEY (a, b)) WITH COMPACT STORAGE");
- execute("INSERT INTO %s (a, b, v) VALUES (1, 1, 3)");
- execute("INSERT INTO %s (a, b, v) VALUES (2, 1, 4)");
- assertRows(execute("SELECT a, b, v FROM %s"),
- row(1, 1, 3),
- row(2, 1, 4));
- testWithCompactFormat();
+ createTable("CREATE TABLE %s (k1 uuid, k2 text, PRIMARY KEY ((k1, k2)))");
+ execute("INSERT INTO %s (k1, k2) VALUES (uuid(), 'k2')");
+ assertInvalidMessage("Undefined column name ", "SELECT token(s1, k1) FROM %s");
}
- private void testWithCompactFormat() throws Throwable
+ @Test
+ public void testTokenFctRejectsInvalidColumnType() throws Throwable
{
- assertInvalidMessage("Undefined column name value",
- "SELECT * FROM %s WHERE a IN (1,2,3) ORDER BY value ASC");
- assertInvalidMessage("Undefined column name column1",
- "SELECT * FROM %s WHERE a IN (1,2,3) ORDER BY column1 ASC");
- assertInvalidMessage("Undefined column name column1",
- "SELECT column1 FROM %s");
- assertInvalidMessage("Undefined column name value",
- "SELECT value FROM %s");
- assertInvalidMessage("Undefined column name value",
- "SELECT value, column1 FROM %s");
- assertInvalid("Undefined column name column1",
- "SELECT * FROM %s WHERE column1 = null ALLOW FILTERING");
- assertInvalid("Undefined column name value",
- "SELECT * FROM %s WHERE value = null ALLOW FILTERING");
- assertInvalidMessage("Undefined column name column1",
- "SELECT WRITETIME(column1) FROM %s");
- assertInvalidMessage("Undefined column name value",
- "SELECT WRITETIME(value) FROM %s");
+ createTable("CREATE TABLE %s (k1 uuid, k2 text, PRIMARY KEY ((k1, k2)))");
+ execute("INSERT INTO %s (k1, k2) VALUES (uuid(), 'k2')");
+ assertInvalidMessage("Type error: k2 cannot be passed as argument 0 of function system.token of type uuid",
+ "SELECT token(k2, k1) FROM %s");
+ }
+
+ @Test
+ public void testTokenFctRejectsInvalidColumnCount() throws Throwable
+ {
+ createTable("CREATE TABLE %s (k1 uuid, k2 text, PRIMARY KEY ((k1, k2)))");
+ execute("INSERT INTO %s (k1, k2) VALUES (uuid(), 'k2')");
+ assertInvalidMessage("Invalid number of arguments in call to function system.token: 2 required but 1 provided",
+ "SELECT token(k1) FROM %s");
+ }
+
+ @Test
+ public void testCreatingUDFWithSameNameAsBuiltin_PrefersCompatibleArgs_SameKeyspace() throws Throwable
+ {
+ createTable("CREATE TABLE %s (k1 uuid, k2 text, PRIMARY KEY ((k1, k2)))");
+ createFunctionOverload(KEYSPACE + ".token", "double",
+ "CREATE FUNCTION %s (val double) RETURNS null ON null INPUT RETURNS double LANGUAGE java AS 'return 10.0d;'");
+ execute("INSERT INTO %s (k1, k2) VALUES (uuid(), 'k2')");
+ assertRows(execute("SELECT token(10) FROM %s"), row(10.0d));
+ }
+
+ @Test
+ public void testCreatingUDFWithSameNameAsBuiltin_FullyQualifiedFunctionNameWorks() throws Throwable
+ {
+ createTable("CREATE TABLE %s (k1 uuid, k2 text, PRIMARY KEY ((k1, k2)))");
+ createFunctionOverload(KEYSPACE + ".token", "double",
+ "CREATE FUNCTION %s (val double) RETURNS null ON null INPUT RETURNS double LANGUAGE java AS 'return 10.0d;'");
+ execute("INSERT INTO %s (k1, k2) VALUES (uuid(), 'k2')");
+ assertRows(execute("SELECT " + KEYSPACE + ".token(10) FROM %s"), row(10.0d));
+ }
+
+ @Test
+ public void testCreatingUDFWithSameNameAsBuiltin_PrefersCompatibleArgs() throws Throwable
+ {
+ createTable("CREATE TABLE %s (k1 uuid, k2 text, PRIMARY KEY ((k1, k2)))");
+ createFunctionOverload(KEYSPACE + ".token", "double",
+ "CREATE FUNCTION %s (val double) RETURNS null ON null INPUT RETURNS double LANGUAGE java AS 'return 10.0d;'");
+ execute("INSERT INTO %s (k1, k2) VALUES (uuid(), 'k2')");
+ assertRowCount(execute("SELECT token(k1, k2) FROM %s"), 1);
+ }
+
+ @Test
+ public void testCreatingUDFWithSameNameAsBuiltin_FullyQualifiedFunctionNameWorks_SystemKeyspace() throws Throwable
+ {
+ createTable("CREATE TABLE %s (k1 uuid, k2 text, PRIMARY KEY ((k1, k2)))");
+ createFunctionOverload(KEYSPACE + ".token", "double",
+ "CREATE FUNCTION %s (val double) RETURNS null ON null INPUT RETURNS double LANGUAGE java AS 'return 10.0d;'");
+ execute("INSERT INTO %s (k1, k2) VALUES (uuid(), 'k2')");
+ assertRowCount(execute("SELECT system.token(k1, k2) FROM %s"), 1);
}
}
diff --git a/test/unit/org/apache/cassandra/cql3/validation/operations/TTLTest.java b/test/unit/org/apache/cassandra/cql3/validation/operations/TTLTest.java
index fc70974..99ca7dc 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/operations/TTLTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/operations/TTLTest.java
@@ -9,6 +9,8 @@
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
+import org.apache.cassandra.config.Config;
+import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.Attributes;
import org.apache.cassandra.cql3.CQLTester;
import org.apache.cassandra.cql3.UntypedResultSet;
@@ -19,6 +21,8 @@
import org.apache.cassandra.exceptions.InvalidRequestException;
import org.apache.cassandra.utils.FBUtilities;
+import org.junit.After;
+import org.junit.Before;
import org.junit.Test;
public class TTLTest extends CQLTester
@@ -31,6 +35,18 @@
public static final String SIMPLE_CLUSTERING = "table2";
public static final String COMPLEX_NOCLUSTERING = "table3";
public static final String COMPLEX_CLUSTERING = "table4";
+ private Config.CorruptedTombstoneStrategy corruptTombstoneStrategy;
+ @Before
+ public void before()
+ {
+ corruptTombstoneStrategy = DatabaseDescriptor.getCorruptedTombstoneStrategy();
+ }
+
+ @After
+ public void after()
+ {
+ DatabaseDescriptor.setCorruptedTombstoneStrategy(corruptTombstoneStrategy);
+ }
@Test
public void testTTLPerRequestLimit() throws Throwable
@@ -167,9 +183,12 @@
@Test
public void testRecoverOverflowedExpirationWithScrub() throws Throwable
{
+ // this tests writes corrupt tombstones on purpose, disable the strategy:
+ DatabaseDescriptor.setCorruptedTombstoneStrategy(Config.CorruptedTombstoneStrategy.disabled);
baseTestRecoverOverflowedExpiration(false, false);
baseTestRecoverOverflowedExpiration(true, false);
baseTestRecoverOverflowedExpiration(true, true);
+ // we reset the corrupted ts strategy after each test in @After above
}
public void testCapExpirationDateOverflowPolicy(ExpirationDateOverflowHandling.ExpirationDateOverflowPolicy policy) throws Throwable
diff --git a/test/unit/org/apache/cassandra/cql3/validation/operations/ThriftCQLTester.java b/test/unit/org/apache/cassandra/cql3/validation/operations/ThriftCQLTester.java
deleted file mode 100644
index 5d4d1a0..0000000
--- a/test/unit/org/apache/cassandra/cql3/validation/operations/ThriftCQLTester.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.cql3.validation.operations;
-
-import java.io.IOException;
-import java.net.ServerSocket;
-
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.cql3.*;
-import org.apache.cassandra.dht.ByteOrderedPartitioner;
-import org.apache.cassandra.service.*;
-import org.apache.cassandra.thrift.*;
-import org.apache.cassandra.utils.FBUtilities;
-import org.apache.thrift.protocol.TBinaryProtocol;
-
-public class ThriftCQLTester extends CQLTester
-{
- private Cassandra.Client client;
-
- private static ThriftServer thriftServer;
- private static int thriftPort;
-
- static {
- try (ServerSocket serverSocket = new ServerSocket(0))
- {
- thriftPort = serverSocket.getLocalPort();
- }
- catch (IOException e)
- {
- // ignore
- }
- }
-
- @BeforeClass
- public static void setup() throws Exception
- {
- StorageService.instance.initServer(0);
-
- if (thriftServer == null || ! thriftServer.isRunning())
- {
- thriftServer = new ThriftServer(FBUtilities.getLocalAddress(), thriftPort, 50);
- thriftServer.start();
- }
- }
-
- @AfterClass
- public static void teardown()
- {
- if (thriftServer != null && thriftServer.isRunning())
- {
- thriftServer.stop();
- }
- }
-
- public Cassandra.Client getClient() throws Throwable
- {
- return getClient(FBUtilities.getLocalAddress().getHostName(), thriftPort);
- }
-
- public Cassandra.Client getClient(String hostname, int thriftPort) throws Throwable
- {
- if (client == null)
- client = new Cassandra.Client(new TBinaryProtocol(new TFramedTransportFactory().openTransport(hostname, thriftPort)));
-
- return client;
- }
-}
diff --git a/test/unit/org/apache/cassandra/cql3/validation/operations/UpdateTest.java b/test/unit/org/apache/cassandra/cql3/validation/operations/UpdateTest.java
index 1a8b49b..973531b 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/operations/UpdateTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/operations/UpdateTest.java
@@ -29,7 +29,6 @@
import org.apache.cassandra.cql3.UntypedResultSet.Row;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Keyspace;
-import org.apache.cassandra.utils.ByteBufferUtil;
import static org.apache.commons.lang3.StringUtils.isEmpty;
import static org.junit.Assert.assertTrue;
@@ -72,125 +71,113 @@
private void testUpdate(boolean forceFlush) throws Throwable
{
- for (String compactOption : new String[] {"", " WITH COMPACT STORAGE" })
- {
- createTable("CREATE TABLE %s (partitionKey int," +
+ createTable("CREATE TABLE %s (partitionKey int," +
"clustering_1 int," +
"value int," +
- " PRIMARY KEY (partitionKey, clustering_1))" + compactOption);
+ " PRIMARY KEY (partitionKey, clustering_1))");
- execute("INSERT INTO %s (partitionKey, clustering_1, value) VALUES (0, 0, 0)");
- execute("INSERT INTO %s (partitionKey, clustering_1, value) VALUES (0, 1, 1)");
- execute("INSERT INTO %s (partitionKey, clustering_1, value) VALUES (0, 2, 2)");
- execute("INSERT INTO %s (partitionKey, clustering_1, value) VALUES (0, 3, 3)");
- execute("INSERT INTO %s (partitionKey, clustering_1, value) VALUES (1, 0, 4)");
+ execute("INSERT INTO %s (partitionKey, clustering_1, value) VALUES (0, 0, 0)");
+ execute("INSERT INTO %s (partitionKey, clustering_1, value) VALUES (0, 1, 1)");
+ execute("INSERT INTO %s (partitionKey, clustering_1, value) VALUES (0, 2, 2)");
+ execute("INSERT INTO %s (partitionKey, clustering_1, value) VALUES (0, 3, 3)");
+ execute("INSERT INTO %s (partitionKey, clustering_1, value) VALUES (1, 0, 4)");
- flush(forceFlush);
+ flush(forceFlush);
- execute("UPDATE %s SET value = ? WHERE partitionKey = ? AND clustering_1 = ?", 7, 0, 1);
- flush(forceFlush);
- assertRows(execute("SELECT value FROM %s WHERE partitionKey = ? AND clustering_1 = ?",
- 0, 1),
- row(7));
+ execute("UPDATE %s SET value = ? WHERE partitionKey = ? AND clustering_1 = ?", 7, 0, 1);
+ flush(forceFlush);
+ assertRows(execute("SELECT value FROM %s WHERE partitionKey = ? AND clustering_1 = ?",
+ 0, 1),
+ row(7));
- execute("UPDATE %s SET value = ? WHERE partitionKey = ? AND (clustering_1) = (?)", 8, 0, 2);
- flush(forceFlush);
- assertRows(execute("SELECT value FROM %s WHERE partitionKey = ? AND clustering_1 = ?",
- 0, 2),
- row(8));
+ execute("UPDATE %s SET value = ? WHERE partitionKey = ? AND (clustering_1) = (?)", 8, 0, 2);
+ flush(forceFlush);
+ assertRows(execute("SELECT value FROM %s WHERE partitionKey = ? AND clustering_1 = ?",
+ 0, 2),
+ row(8));
- execute("UPDATE %s SET value = ? WHERE partitionKey IN (?, ?) AND clustering_1 = ?", 9, 0, 1, 0);
- flush(forceFlush);
- assertRows(execute("SELECT * FROM %s WHERE partitionKey IN (?, ?) AND clustering_1 = ?",
- 0, 1, 0),
- row(0, 0, 9),
- row(1, 0, 9));
+ execute("UPDATE %s SET value = ? WHERE partitionKey IN (?, ?) AND clustering_1 = ?", 9, 0, 1, 0);
+ flush(forceFlush);
+ assertRows(execute("SELECT * FROM %s WHERE partitionKey IN (?, ?) AND clustering_1 = ?",
+ 0, 1, 0),
+ row(0, 0, 9),
+ row(1, 0, 9));
- execute("UPDATE %s SET value = ? WHERE partitionKey IN ? AND clustering_1 = ?", 19, Arrays.asList(0, 1), 0);
- flush(forceFlush);
- assertRows(execute("SELECT * FROM %s WHERE partitionKey IN ? AND clustering_1 = ?",
- Arrays.asList(0, 1), 0),
- row(0, 0, 19),
- row(1, 0, 19));
+ execute("UPDATE %s SET value = ? WHERE partitionKey IN ? AND clustering_1 = ?", 19, Arrays.asList(0, 1), 0);
+ flush(forceFlush);
+ assertRows(execute("SELECT * FROM %s WHERE partitionKey IN ? AND clustering_1 = ?",
+ Arrays.asList(0, 1), 0),
+ row(0, 0, 19),
+ row(1, 0, 19));
- execute("UPDATE %s SET value = ? WHERE partitionKey = ? AND clustering_1 IN (?, ?)", 10, 0, 1, 0);
- flush(forceFlush);
- assertRows(execute("SELECT * FROM %s WHERE partitionKey = ? AND clustering_1 IN (?, ?)",
- 0, 1, 0),
- row(0, 0, 10),
- row(0, 1, 10));
+ execute("UPDATE %s SET value = ? WHERE partitionKey = ? AND clustering_1 IN (?, ?)", 10, 0, 1, 0);
+ flush(forceFlush);
+ assertRows(execute("SELECT * FROM %s WHERE partitionKey = ? AND clustering_1 IN (?, ?)",
+ 0, 1, 0),
+ row(0, 0, 10),
+ row(0, 1, 10));
- execute("UPDATE %s SET value = ? WHERE partitionKey = ? AND (clustering_1) IN ((?), (?))", 20, 0, 0, 1);
- flush(forceFlush);
- assertRows(execute("SELECT * FROM %s WHERE partitionKey = ? AND (clustering_1) IN ((?), (?))",
- 0, 0, 1),
- row(0, 0, 20),
- row(0, 1, 20));
+ execute("UPDATE %s SET value = ? WHERE partitionKey = ? AND (clustering_1) IN ((?), (?))", 20, 0, 0, 1);
+ flush(forceFlush);
+ assertRows(execute("SELECT * FROM %s WHERE partitionKey = ? AND (clustering_1) IN ((?), (?))",
+ 0, 0, 1),
+ row(0, 0, 20),
+ row(0, 1, 20));
- execute("UPDATE %s SET value = ? WHERE partitionKey = ? AND clustering_1 = ?", null, 0, 0);
- flush(forceFlush);
+ execute("UPDATE %s SET value = ? WHERE partitionKey = ? AND clustering_1 = ?", null, 0, 0);
+ flush(forceFlush);
- if (isEmpty(compactOption))
- {
- assertRows(execute("SELECT * FROM %s WHERE partitionKey = ? AND (clustering_1) IN ((?), (?))",
- 0, 0, 1),
- row(0, 0, null),
- row(0, 1, 20));
- }
- else
- {
- assertRows(execute("SELECT * FROM %s WHERE partitionKey = ? AND (clustering_1) IN ((?), (?))",
- 0, 0, 1),
- row(0, 1, 20));
- }
+ assertRows(execute("SELECT * FROM %s WHERE partitionKey = ? AND (clustering_1) IN ((?), (?))",
+ 0, 0, 1),
+ row(0, 0, null),
+ row(0, 1, 20));
- // test invalid queries
+ // test invalid queries
- // missing primary key element
- assertInvalidMessage("Some partition key parts are missing: partitionkey",
- "UPDATE %s SET value = ? WHERE clustering_1 = ? ", 7, 1);
+ // missing primary key element
+ assertInvalidMessage("Some partition key parts are missing: partitionkey",
+ "UPDATE %s SET value = ? WHERE clustering_1 = ? ", 7, 1);
- assertInvalidMessage("Some clustering keys are missing: clustering_1",
- "UPDATE %s SET value = ? WHERE partitionKey = ?", 7, 0);
+ assertInvalidMessage("Some clustering keys are missing: clustering_1",
+ "UPDATE %s SET value = ? WHERE partitionKey = ?", 7, 0);
- assertInvalidMessage("Some clustering keys are missing: clustering_1",
- "UPDATE %s SET value = ? WHERE partitionKey = ?", 7, 0);
+ assertInvalidMessage("Some clustering keys are missing: clustering_1",
+ "UPDATE %s SET value = ? WHERE partitionKey = ?", 7, 0);
- // token function
- assertInvalidMessage("The token function cannot be used in WHERE clauses for UPDATE statements",
- "UPDATE %s SET value = ? WHERE token(partitionKey) = token(?) AND clustering_1 = ?",
- 7, 0, 1);
+ // token function
+ assertInvalidMessage("The token function cannot be used in WHERE clauses for UPDATE statements",
+ "UPDATE %s SET value = ? WHERE token(partitionKey) = token(?) AND clustering_1 = ?",
+ 7, 0, 1);
- // multiple time the same value
- assertInvalidSyntax("UPDATE %s SET value = ?, value = ? WHERE partitionKey = ? AND clustering_1 = ?", 7, 0, 1);
+ // multiple time the same value
+ assertInvalidSyntax("UPDATE %s SET value = ?, value = ? WHERE partitionKey = ? AND clustering_1 = ?", 7, 0, 1);
- // multiple time same primary key element in WHERE clause
- assertInvalidMessage("clustering_1 cannot be restricted by more than one relation if it includes an Equal",
- "UPDATE %s SET value = ? WHERE partitionKey = ? AND clustering_1 = ? AND clustering_1 = ?", 7, 0, 1, 1);
+ // multiple time same primary key element in WHERE clause
+ assertInvalidMessage("clustering_1 cannot be restricted by more than one relation if it includes an Equal",
+ "UPDATE %s SET value = ? WHERE partitionKey = ? AND clustering_1 = ? AND clustering_1 = ?", 7, 0, 1, 1);
- // unknown identifiers
- assertInvalidMessage("Undefined column name value1",
- "UPDATE %s SET value1 = ? WHERE partitionKey = ? AND clustering_1 = ?", 7, 0, 1);
+ // unknown identifiers
+ assertInvalidMessage("Undefined column name value1",
+ "UPDATE %s SET value1 = ? WHERE partitionKey = ? AND clustering_1 = ?", 7, 0, 1);
- assertInvalidMessage("Undefined column name partitionkey1",
- "UPDATE %s SET value = ? WHERE partitionKey1 = ? AND clustering_1 = ?", 7, 0, 1);
+ assertInvalidMessage("Undefined column name partitionkey1",
+ "UPDATE %s SET value = ? WHERE partitionKey1 = ? AND clustering_1 = ?", 7, 0, 1);
- assertInvalidMessage("Undefined column name clustering_3",
- "UPDATE %s SET value = ? WHERE partitionKey = ? AND clustering_3 = ?", 7, 0, 1);
+ assertInvalidMessage("Undefined column name clustering_3",
+ "UPDATE %s SET value = ? WHERE partitionKey = ? AND clustering_3 = ?", 7, 0, 1);
- // Invalid operator in the where clause
- assertInvalidMessage("Only EQ and IN relation are supported on the partition key (unless you use the token() function)",
- "UPDATE %s SET value = ? WHERE partitionKey > ? AND clustering_1 = ?", 7, 0, 1);
+ // Invalid operator in the where clause
+ assertInvalidMessage("Only EQ and IN relation are supported on the partition key (unless you use the token() function)",
+ "UPDATE %s SET value = ? WHERE partitionKey > ? AND clustering_1 = ?", 7, 0, 1);
- assertInvalidMessage("Cannot use CONTAINS on non-collection column partitionkey",
- "UPDATE %s SET value = ? WHERE partitionKey CONTAINS ? AND clustering_1 = ?", 7, 0, 1);
+ assertInvalidMessage("Cannot use CONTAINS on non-collection column partitionkey",
+ "UPDATE %s SET value = ? WHERE partitionKey CONTAINS ? AND clustering_1 = ?", 7, 0, 1);
- assertInvalidMessage("Non PRIMARY KEY columns found in where clause: value",
- "UPDATE %s SET value = ? WHERE partitionKey = ? AND clustering_1 = ? AND value = ?", 7, 0, 1, 3);
+ assertInvalidMessage("Non PRIMARY KEY columns found in where clause: value",
+ "UPDATE %s SET value = ? WHERE partitionKey = ? AND clustering_1 = ? AND value = ?", 7, 0, 1, 3);
- assertInvalidMessage("Slice restrictions are not supported on the clustering columns in UPDATE statements",
- "UPDATE %s SET value = ? WHERE partitionKey = ? AND clustering_1 > ?", 7, 0, 1);
- }
+ assertInvalidMessage("Slice restrictions are not supported on the clustering columns in UPDATE statements",
+ "UPDATE %s SET value = ? WHERE partitionKey = ? AND clustering_1 > ?", 7, 0, 1);
}
@Test
@@ -245,142 +232,127 @@
private void testUpdateWithTwoClusteringColumns(boolean forceFlush) throws Throwable
{
- for (String compactOption : new String[] { "", " WITH COMPACT STORAGE" })
- {
- createTable("CREATE TABLE %s (partitionKey int," +
+ createTable("CREATE TABLE %s (partitionKey int," +
"clustering_1 int," +
"clustering_2 int," +
"value int," +
- " PRIMARY KEY (partitionKey, clustering_1, clustering_2))" + compactOption);
+ " PRIMARY KEY (partitionKey, clustering_1, clustering_2))");
- execute("INSERT INTO %s (partitionKey, clustering_1, clustering_2, value) VALUES (0, 0, 0, 0)");
- execute("INSERT INTO %s (partitionKey, clustering_1, clustering_2, value) VALUES (0, 0, 1, 1)");
- execute("INSERT INTO %s (partitionKey, clustering_1, clustering_2, value) VALUES (0, 0, 2, 2)");
- execute("INSERT INTO %s (partitionKey, clustering_1, clustering_2, value) VALUES (0, 0, 3, 3)");
- execute("INSERT INTO %s (partitionKey, clustering_1, clustering_2, value) VALUES (0, 1, 1, 4)");
- execute("INSERT INTO %s (partitionKey, clustering_1, clustering_2, value) VALUES (0, 1, 2, 5)");
- execute("INSERT INTO %s (partitionKey, clustering_1, clustering_2, value) VALUES (1, 0, 0, 6)");
- flush(forceFlush);
+ execute("INSERT INTO %s (partitionKey, clustering_1, clustering_2, value) VALUES (0, 0, 0, 0)");
+ execute("INSERT INTO %s (partitionKey, clustering_1, clustering_2, value) VALUES (0, 0, 1, 1)");
+ execute("INSERT INTO %s (partitionKey, clustering_1, clustering_2, value) VALUES (0, 0, 2, 2)");
+ execute("INSERT INTO %s (partitionKey, clustering_1, clustering_2, value) VALUES (0, 0, 3, 3)");
+ execute("INSERT INTO %s (partitionKey, clustering_1, clustering_2, value) VALUES (0, 1, 1, 4)");
+ execute("INSERT INTO %s (partitionKey, clustering_1, clustering_2, value) VALUES (0, 1, 2, 5)");
+ execute("INSERT INTO %s (partitionKey, clustering_1, clustering_2, value) VALUES (1, 0, 0, 6)");
+ flush(forceFlush);
- execute("UPDATE %s SET value = ? WHERE partitionKey = ? AND clustering_1 = ? AND clustering_2 = ?", 7, 0, 1, 1);
- flush(forceFlush);
- assertRows(execute("SELECT value FROM %s WHERE partitionKey = ? AND clustering_1 = ? AND clustering_2 = ?",
- 0, 1, 1),
- row(7));
+ execute("UPDATE %s SET value = ? WHERE partitionKey = ? AND clustering_1 = ? AND clustering_2 = ?", 7, 0, 1, 1);
+ flush(forceFlush);
+ assertRows(execute("SELECT value FROM %s WHERE partitionKey = ? AND clustering_1 = ? AND clustering_2 = ?",
+ 0, 1, 1),
+ row(7));
- execute("UPDATE %s SET value = ? WHERE partitionKey = ? AND (clustering_1, clustering_2) = (?, ?)", 8, 0, 1, 2);
- flush(forceFlush);
- assertRows(execute("SELECT value FROM %s WHERE partitionKey = ? AND clustering_1 = ? AND clustering_2 = ?",
- 0, 1, 2),
- row(8));
+ execute("UPDATE %s SET value = ? WHERE partitionKey = ? AND (clustering_1, clustering_2) = (?, ?)", 8, 0, 1, 2);
+ flush(forceFlush);
+ assertRows(execute("SELECT value FROM %s WHERE partitionKey = ? AND clustering_1 = ? AND clustering_2 = ?",
+ 0, 1, 2),
+ row(8));
- execute("UPDATE %s SET value = ? WHERE partitionKey IN (?, ?) AND clustering_1 = ? AND clustering_2 = ?", 9, 0, 1, 0, 0);
- flush(forceFlush);
- assertRows(execute("SELECT * FROM %s WHERE partitionKey IN (?, ?) AND clustering_1 = ? AND clustering_2 = ?",
- 0, 1, 0, 0),
- row(0, 0, 0, 9),
- row(1, 0, 0, 9));
+ execute("UPDATE %s SET value = ? WHERE partitionKey IN (?, ?) AND clustering_1 = ? AND clustering_2 = ?", 9, 0, 1, 0, 0);
+ flush(forceFlush);
+ assertRows(execute("SELECT * FROM %s WHERE partitionKey IN (?, ?) AND clustering_1 = ? AND clustering_2 = ?",
+ 0, 1, 0, 0),
+ row(0, 0, 0, 9),
+ row(1, 0, 0, 9));
- execute("UPDATE %s SET value = ? WHERE partitionKey IN ? AND clustering_1 = ? AND clustering_2 = ?", 9, Arrays.asList(0, 1), 0, 0);
- flush(forceFlush);
- assertRows(execute("SELECT * FROM %s WHERE partitionKey IN ? AND clustering_1 = ? AND clustering_2 = ?",
- Arrays.asList(0, 1), 0, 0),
- row(0, 0, 0, 9),
- row(1, 0, 0, 9));
+ execute("UPDATE %s SET value = ? WHERE partitionKey IN ? AND clustering_1 = ? AND clustering_2 = ?", 9, Arrays.asList(0, 1), 0, 0);
+ flush(forceFlush);
+ assertRows(execute("SELECT * FROM %s WHERE partitionKey IN ? AND clustering_1 = ? AND clustering_2 = ?",
+ Arrays.asList(0, 1), 0, 0),
+ row(0, 0, 0, 9),
+ row(1, 0, 0, 9));
- execute("UPDATE %s SET value = ? WHERE partitionKey = ? AND clustering_1 = ? AND clustering_2 IN (?, ?)", 12, 0, 1, 1, 2);
- flush(forceFlush);
- assertRows(execute("SELECT * FROM %s WHERE partitionKey = ? AND clustering_1 = ? AND clustering_2 IN (?, ?)",
- 0, 1, 1, 2),
- row(0, 1, 1, 12),
- row(0, 1, 2, 12));
+ execute("UPDATE %s SET value = ? WHERE partitionKey = ? AND clustering_1 = ? AND clustering_2 IN (?, ?)", 12, 0, 1, 1, 2);
+ flush(forceFlush);
+ assertRows(execute("SELECT * FROM %s WHERE partitionKey = ? AND clustering_1 = ? AND clustering_2 IN (?, ?)",
+ 0, 1, 1, 2),
+ row(0, 1, 1, 12),
+ row(0, 1, 2, 12));
- execute("UPDATE %s SET value = ? WHERE partitionKey = ? AND clustering_1 IN (?, ?) AND clustering_2 IN (?, ?)", 10, 0, 1, 0, 1, 2);
- flush(forceFlush);
- assertRows(execute("SELECT * FROM %s WHERE partitionKey = ? AND clustering_1 IN (?, ?) AND clustering_2 IN (?, ?)",
- 0, 1, 0, 1, 2),
- row(0, 0, 1, 10),
- row(0, 0, 2, 10),
- row(0, 1, 1, 10),
- row(0, 1, 2, 10));
+ execute("UPDATE %s SET value = ? WHERE partitionKey = ? AND clustering_1 IN (?, ?) AND clustering_2 IN (?, ?)", 10, 0, 1, 0, 1, 2);
+ flush(forceFlush);
+ assertRows(execute("SELECT * FROM %s WHERE partitionKey = ? AND clustering_1 IN (?, ?) AND clustering_2 IN (?, ?)",
+ 0, 1, 0, 1, 2),
+ row(0, 0, 1, 10),
+ row(0, 0, 2, 10),
+ row(0, 1, 1, 10),
+ row(0, 1, 2, 10));
- execute("UPDATE %s SET value = ? WHERE partitionKey = ? AND (clustering_1, clustering_2) IN ((?, ?), (?, ?))", 20, 0, 0, 2, 1, 2);
- flush(forceFlush);
- assertRows(execute("SELECT * FROM %s WHERE partitionKey = ? AND (clustering_1, clustering_2) IN ((?, ?), (?, ?))",
- 0, 0, 2, 1, 2),
- row(0, 0, 2, 20),
- row(0, 1, 2, 20));
+ execute("UPDATE %s SET value = ? WHERE partitionKey = ? AND (clustering_1, clustering_2) IN ((?, ?), (?, ?))", 20, 0, 0, 2, 1, 2);
+ flush(forceFlush);
+ assertRows(execute("SELECT * FROM %s WHERE partitionKey = ? AND (clustering_1, clustering_2) IN ((?, ?), (?, ?))",
+ 0, 0, 2, 1, 2),
+ row(0, 0, 2, 20),
+ row(0, 1, 2, 20));
- execute("UPDATE %s SET value = ? WHERE partitionKey = ? AND clustering_1 = ? AND clustering_2 = ?", null, 0, 0, 2);
- flush(forceFlush);
+ execute("UPDATE %s SET value = ? WHERE partitionKey = ? AND clustering_1 = ? AND clustering_2 = ?", null, 0, 0, 2);
+ flush(forceFlush);
- if (isEmpty(compactOption))
- {
- assertRows(execute("SELECT * FROM %s WHERE partitionKey = ? AND (clustering_1, clustering_2) IN ((?, ?), (?, ?))",
- 0, 0, 2, 1, 2),
- row(0, 0, 2, null),
- row(0, 1, 2, 20));
- }
- else
- {
- assertRows(execute("SELECT * FROM %s WHERE partitionKey = ? AND (clustering_1, clustering_2) IN ((?, ?), (?, ?))",
- 0, 0, 2, 1, 2),
- row(0, 1, 2, 20));
- }
+ assertRows(execute("SELECT * FROM %s WHERE partitionKey = ? AND (clustering_1, clustering_2) IN ((?, ?), (?, ?))",
+ 0, 0, 2, 1, 2),
+ row(0, 0, 2, null),
+ row(0, 1, 2, 20));
- // test invalid queries
+ // test invalid queries
- // missing primary key element
- assertInvalidMessage("Some partition key parts are missing: partitionkey",
- "UPDATE %s SET value = ? WHERE clustering_1 = ? AND clustering_2 = ?", 7, 1, 1);
+ // missing primary key element
+ assertInvalidMessage("Some partition key parts are missing: partitionkey",
+ "UPDATE %s SET value = ? WHERE clustering_1 = ? AND clustering_2 = ?", 7, 1, 1);
- String errorMsg = isEmpty(compactOption) ? "Some clustering keys are missing: clustering_1"
- : "PRIMARY KEY column \"clustering_2\" cannot be restricted as preceding column \"clustering_1\" is not restricted";
+ assertInvalidMessage("Some clustering keys are missing: clustering_1",
+ "UPDATE %s SET value = ? WHERE partitionKey = ? AND clustering_2 = ?", 7, 0, 1);
- assertInvalidMessage(errorMsg,
- "UPDATE %s SET value = ? WHERE partitionKey = ? AND clustering_2 = ?", 7, 0, 1);
+ assertInvalidMessage("Some clustering keys are missing: clustering_1, clustering_2",
+ "UPDATE %s SET value = ? WHERE partitionKey = ?", 7, 0);
- assertInvalidMessage("Some clustering keys are missing: clustering_1, clustering_2",
- "UPDATE %s SET value = ? WHERE partitionKey = ?", 7, 0);
+ // token function
+ assertInvalidMessage("The token function cannot be used in WHERE clauses for UPDATE statements",
+ "UPDATE %s SET value = ? WHERE token(partitionKey) = token(?) AND clustering_1 = ? AND clustering_2 = ?",
+ 7, 0, 1, 1);
- // token function
- assertInvalidMessage("The token function cannot be used in WHERE clauses for UPDATE statements",
- "UPDATE %s SET value = ? WHERE token(partitionKey) = token(?) AND clustering_1 = ? AND clustering_2 = ?",
- 7, 0, 1, 1);
+ // multiple time the same value
+ assertInvalidSyntax("UPDATE %s SET value = ?, value = ? WHERE partitionKey = ? AND clustering_1 = ? AND clustering_2 = ?", 7, 0, 1, 1);
- // multiple time the same value
- assertInvalidSyntax("UPDATE %s SET value = ?, value = ? WHERE partitionKey = ? AND clustering_1 = ? AND clustering_2 = ?", 7, 0, 1, 1);
+ // multiple time same primary key element in WHERE clause
+ assertInvalidMessage("clustering_1 cannot be restricted by more than one relation if it includes an Equal",
+ "UPDATE %s SET value = ? WHERE partitionKey = ? AND clustering_1 = ? AND clustering_2 = ? AND clustering_1 = ?", 7, 0, 1, 1, 1);
- // multiple time same primary key element in WHERE clause
- assertInvalidMessage("clustering_1 cannot be restricted by more than one relation if it includes an Equal",
- "UPDATE %s SET value = ? WHERE partitionKey = ? AND clustering_1 = ? AND clustering_2 = ? AND clustering_1 = ?", 7, 0, 1, 1, 1);
+ // unknown identifiers
+ assertInvalidMessage("Undefined column name value1",
+ "UPDATE %s SET value1 = ? WHERE partitionKey = ? AND clustering_1 = ? AND clustering_2 = ?", 7, 0, 1, 1);
- // unknown identifiers
- assertInvalidMessage("Undefined column name value1",
- "UPDATE %s SET value1 = ? WHERE partitionKey = ? AND clustering_1 = ? AND clustering_2 = ?", 7, 0, 1, 1);
+ assertInvalidMessage("Undefined column name partitionkey1",
+ "UPDATE %s SET value = ? WHERE partitionKey1 = ? AND clustering_1 = ? AND clustering_2 = ?", 7, 0, 1, 1);
- assertInvalidMessage("Undefined column name partitionkey1",
- "UPDATE %s SET value = ? WHERE partitionKey1 = ? AND clustering_1 = ? AND clustering_2 = ?", 7, 0, 1, 1);
+ assertInvalidMessage("Undefined column name clustering_3",
+ "UPDATE %s SET value = ? WHERE partitionKey = ? AND clustering_1 = ? AND clustering_3 = ?", 7, 0, 1, 1);
- assertInvalidMessage("Undefined column name clustering_3",
- "UPDATE %s SET value = ? WHERE partitionKey = ? AND clustering_1 = ? AND clustering_3 = ?", 7, 0, 1, 1);
+ // Invalid operator in the where clause
+ assertInvalidMessage("Only EQ and IN relation are supported on the partition key (unless you use the token() function)",
+ "UPDATE %s SET value = ? WHERE partitionKey > ? AND clustering_1 = ? AND clustering_2 = ?", 7, 0, 1, 1);
- // Invalid operator in the where clause
- assertInvalidMessage("Only EQ and IN relation are supported on the partition key (unless you use the token() function)",
- "UPDATE %s SET value = ? WHERE partitionKey > ? AND clustering_1 = ? AND clustering_2 = ?", 7, 0, 1, 1);
+ assertInvalidMessage("Cannot use CONTAINS on non-collection column partitionkey",
+ "UPDATE %s SET value = ? WHERE partitionKey CONTAINS ? AND clustering_1 = ? AND clustering_2 = ?", 7, 0, 1, 1);
- assertInvalidMessage("Cannot use CONTAINS on non-collection column partitionkey",
- "UPDATE %s SET value = ? WHERE partitionKey CONTAINS ? AND clustering_1 = ? AND clustering_2 = ?", 7, 0, 1, 1);
+ assertInvalidMessage("Non PRIMARY KEY columns found in where clause: value",
+ "UPDATE %s SET value = ? WHERE partitionKey = ? AND clustering_1 = ? AND clustering_2 = ? AND value = ?", 7, 0, 1, 1, 3);
- assertInvalidMessage("Non PRIMARY KEY columns found in where clause: value",
- "UPDATE %s SET value = ? WHERE partitionKey = ? AND clustering_1 = ? AND clustering_2 = ? AND value = ?", 7, 0, 1, 1, 3);
+ assertInvalidMessage("Slice restrictions are not supported on the clustering columns in UPDATE statements",
+ "UPDATE %s SET value = ? WHERE partitionKey = ? AND clustering_1 > ?", 7, 0, 1);
- assertInvalidMessage("Slice restrictions are not supported on the clustering columns in UPDATE statements",
- "UPDATE %s SET value = ? WHERE partitionKey = ? AND clustering_1 > ?", 7, 0, 1);
-
- assertInvalidMessage("Slice restrictions are not supported on the clustering columns in UPDATE statements",
- "UPDATE %s SET value = ? WHERE partitionKey = ? AND (clustering_1, clustering_2) > (?, ?)", 7, 0, 1, 1);
- }
+ assertInvalidMessage("Slice restrictions are not supported on the clustering columns in UPDATE statements",
+ "UPDATE %s SET value = ? WHERE partitionKey = ? AND (clustering_1, clustering_2) > (?, ?)", 7, 0, 1, 1);
}
@Test
@@ -392,49 +364,46 @@
public void testUpdateWithMultiplePartitionKeyComponents(boolean forceFlush) throws Throwable
{
- for (String compactOption : new String[] { "", " WITH COMPACT STORAGE" })
- {
- createTable("CREATE TABLE %s (partitionKey_1 int," +
+ createTable("CREATE TABLE %s (partitionKey_1 int," +
"partitionKey_2 int," +
"clustering_1 int," +
"clustering_2 int," +
"value int," +
- " PRIMARY KEY ((partitionKey_1, partitionKey_2), clustering_1, clustering_2))" + compactOption);
+ " PRIMARY KEY ((partitionKey_1, partitionKey_2), clustering_1, clustering_2))");
- execute("INSERT INTO %s (partitionKey_1, partitionKey_2, clustering_1, clustering_2, value) VALUES (0, 0, 0, 0, 0)");
- execute("INSERT INTO %s (partitionKey_1, partitionKey_2, clustering_1, clustering_2, value) VALUES (0, 1, 0, 1, 1)");
- execute("INSERT INTO %s (partitionKey_1, partitionKey_2, clustering_1, clustering_2, value) VALUES (0, 1, 1, 1, 2)");
- execute("INSERT INTO %s (partitionKey_1, partitionKey_2, clustering_1, clustering_2, value) VALUES (1, 0, 0, 1, 3)");
- execute("INSERT INTO %s (partitionKey_1, partitionKey_2, clustering_1, clustering_2, value) VALUES (1, 1, 0, 1, 3)");
- flush(forceFlush);
+ execute("INSERT INTO %s (partitionKey_1, partitionKey_2, clustering_1, clustering_2, value) VALUES (0, 0, 0, 0, 0)");
+ execute("INSERT INTO %s (partitionKey_1, partitionKey_2, clustering_1, clustering_2, value) VALUES (0, 1, 0, 1, 1)");
+ execute("INSERT INTO %s (partitionKey_1, partitionKey_2, clustering_1, clustering_2, value) VALUES (0, 1, 1, 1, 2)");
+ execute("INSERT INTO %s (partitionKey_1, partitionKey_2, clustering_1, clustering_2, value) VALUES (1, 0, 0, 1, 3)");
+ execute("INSERT INTO %s (partitionKey_1, partitionKey_2, clustering_1, clustering_2, value) VALUES (1, 1, 0, 1, 3)");
+ flush(forceFlush);
- execute("UPDATE %s SET value = ? WHERE partitionKey_1 = ? AND partitionKey_2 = ? AND clustering_1 = ? AND clustering_2 = ?", 7, 0, 0, 0, 0);
- flush(forceFlush);
- assertRows(execute("SELECT value FROM %s WHERE partitionKey_1 = ? AND partitionKey_2 = ? AND clustering_1 = ? AND clustering_2 = ?",
- 0, 0, 0, 0),
- row(7));
+ execute("UPDATE %s SET value = ? WHERE partitionKey_1 = ? AND partitionKey_2 = ? AND clustering_1 = ? AND clustering_2 = ?", 7, 0, 0, 0, 0);
+ flush(forceFlush);
+ assertRows(execute("SELECT value FROM %s WHERE partitionKey_1 = ? AND partitionKey_2 = ? AND clustering_1 = ? AND clustering_2 = ?",
+ 0, 0, 0, 0),
+ row(7));
- execute("UPDATE %s SET value = ? WHERE partitionKey_1 IN (?, ?) AND partitionKey_2 = ? AND clustering_1 = ? AND clustering_2 = ?", 9, 0, 1, 1, 0, 1);
- flush(forceFlush);
- assertRows(execute("SELECT * FROM %s WHERE partitionKey_1 IN (?, ?) AND partitionKey_2 = ? AND clustering_1 = ? AND clustering_2 = ?",
- 0, 1, 1, 0, 1),
- row(0, 1, 0, 1, 9),
- row(1, 1, 0, 1, 9));
+ execute("UPDATE %s SET value = ? WHERE partitionKey_1 IN (?, ?) AND partitionKey_2 = ? AND clustering_1 = ? AND clustering_2 = ?", 9, 0, 1, 1, 0, 1);
+ flush(forceFlush);
+ assertRows(execute("SELECT * FROM %s WHERE partitionKey_1 IN (?, ?) AND partitionKey_2 = ? AND clustering_1 = ? AND clustering_2 = ?",
+ 0, 1, 1, 0, 1),
+ row(0, 1, 0, 1, 9),
+ row(1, 1, 0, 1, 9));
- execute("UPDATE %s SET value = ? WHERE partitionKey_1 IN (?, ?) AND partitionKey_2 IN (?, ?) AND clustering_1 = ? AND clustering_2 = ?", 10, 0, 1, 0, 1, 0, 1);
- flush(forceFlush);
- assertRows(execute("SELECT * FROM %s"),
- row(0, 0, 0, 0, 7),
- row(0, 0, 0, 1, 10),
- row(0, 1, 0, 1, 10),
- row(0, 1, 1, 1, 2),
- row(1, 0, 0, 1, 10),
- row(1, 1, 0, 1, 10));
+ execute("UPDATE %s SET value = ? WHERE partitionKey_1 IN (?, ?) AND partitionKey_2 IN (?, ?) AND clustering_1 = ? AND clustering_2 = ?", 10, 0, 1, 0, 1, 0, 1);
+ flush(forceFlush);
+ assertRows(execute("SELECT * FROM %s"),
+ row(0, 0, 0, 0, 7),
+ row(0, 0, 0, 1, 10),
+ row(0, 1, 0, 1, 10),
+ row(0, 1, 1, 1, 2),
+ row(1, 0, 0, 1, 10),
+ row(1, 1, 0, 1, 10));
- // missing primary key element
- assertInvalidMessage("Some partition key parts are missing: partitionkey_2",
- "UPDATE %s SET value = ? WHERE partitionKey_1 = ? AND clustering_1 = ? AND clustering_2 = ?", 7, 1, 1);
- }
+ // missing primary key element
+ assertInvalidMessage("Some partition key parts are missing: partitionkey_2",
+ "UPDATE %s SET value = ? WHERE partitionKey_1 = ? AND clustering_1 = ? AND clustering_2 = ?", 7, 1, 1);
}
@Test
@@ -668,60 +637,4 @@
ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(currentTable());
return cfs.metric.allMemtablesLiveDataSize.getValue() == 0;
}
-
- /**
- * Test for CASSANDRA-13917
- */
- @Test
- public void testUpdateWithCompactStaticFormat() throws Throwable
- {
- testWithCompactFormat("CREATE TABLE %s (a int PRIMARY KEY, b int, c int) WITH COMPACT STORAGE");
-
- assertInvalidMessage("Undefined column name column1",
- "UPDATE %s SET b = 1 WHERE column1 = ?",
- ByteBufferUtil.bytes('a'));
- assertInvalidMessage("Undefined column name value",
- "UPDATE %s SET b = 1 WHERE value = ?",
- ByteBufferUtil.bytes('a'));
-
- // if column1 is present, hidden column is called column2
- createTable("CREATE TABLE %s (a int PRIMARY KEY, b int, c int, column1 int) WITH COMPACT STORAGE");
- execute("INSERT INTO %s (a, b, c, column1) VALUES (1, 1, 1, 1)");
- execute("UPDATE %s SET column1 = 6 WHERE a = 1");
- assertInvalidMessage("Undefined column name column2", "UPDATE %s SET column2 = 6 WHERE a = 0");
- assertInvalidMessage("Undefined column name value", "UPDATE %s SET value = 6 WHERE a = 0");
-
- // if value is present, hidden column is called value1
- createTable("CREATE TABLE %s (a int PRIMARY KEY, b int, c int, value int) WITH COMPACT STORAGE");
- execute("INSERT INTO %s (a, b, c, value) VALUES (1, 1, 1, 1)");
- execute("UPDATE %s SET value = 6 WHERE a = 1");
- assertInvalidMessage("Undefined column name column1", "UPDATE %s SET column1 = 6 WHERE a = 1");
- assertInvalidMessage("Undefined column name value1", "UPDATE %s SET value1 = 6 WHERE a = 1");
- }
-
- /**
- * Test for CASSANDRA-13917
- */
- @Test
- public void testUpdateWithCompactNonStaticFormat() throws Throwable
- {
- testWithCompactFormat("CREATE TABLE %s (a int, b int, PRIMARY KEY (a, b)) WITH COMPACT STORAGE");
- testWithCompactFormat("CREATE TABLE %s (a int, b int, v int, PRIMARY KEY (a, b)) WITH COMPACT STORAGE");
- }
-
- private void testWithCompactFormat(String tableQuery) throws Throwable
- {
- createTable(tableQuery);
- // pass correct types to hidden columns
- assertInvalidMessage("Undefined column name column1",
- "UPDATE %s SET column1 = ? WHERE a = 0",
- ByteBufferUtil.bytes('a'));
- assertInvalidMessage("Undefined column name value",
- "UPDATE %s SET value = ? WHERE a = 0",
- ByteBufferUtil.bytes('a'));
-
- // pass incorrect types to hidden columns
- assertInvalidMessage("Undefined column name column1", "UPDATE %s SET column1 = 6 WHERE a = 0");
- assertInvalidMessage("Undefined column name value", "UPDATE %s SET value = 6 WHERE a = 0");
- }
}
diff --git a/test/unit/org/apache/cassandra/db/CellTest.java b/test/unit/org/apache/cassandra/db/CellTest.java
index c68b4ec..14d05c6 100644
--- a/test/unit/org/apache/cassandra/db/CellTest.java
+++ b/test/unit/org/apache/cassandra/db/CellTest.java
@@ -23,20 +23,20 @@
import com.google.common.collect.Lists;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.ColumnIdentifier;
import org.apache.cassandra.cql3.FieldIdentifier;
import org.apache.cassandra.db.marshal.*;
import org.apache.cassandra.db.rows.*;
import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.serializers.MarshalException;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FBUtilities;
@@ -54,13 +54,14 @@
private static final String CF_STANDARD1 = "Standard1";
private static final String CF_COLLECTION = "Collection1";
- private static final CFMetaData cfm = SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD1);
- private static final CFMetaData cfm2 = CFMetaData.Builder.create(KEYSPACE1, CF_COLLECTION)
- .addPartitionKey("k", IntegerType.instance)
- .addClusteringColumn("c", IntegerType.instance)
- .addRegularColumn("v", IntegerType.instance)
- .addRegularColumn("m", MapType.getInstance(IntegerType.instance, IntegerType.instance, true))
- .build();
+ private static final TableMetadata cfm = SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD1).build();
+ private static final TableMetadata cfm2 =
+ TableMetadata.builder(KEYSPACE1, CF_COLLECTION)
+ .addPartitionKeyColumn("k", IntegerType.instance)
+ .addClusteringColumn("c", IntegerType.instance)
+ .addRegularColumn("v", IntegerType.instance)
+ .addRegularColumn("m", MapType.getInstance(IntegerType.instance, IntegerType.instance, true))
+ .build();
@BeforeClass
public static void defineSchema() throws ConfigurationException
@@ -69,14 +70,14 @@
SchemaLoader.createKeyspace(KEYSPACE1, KeyspaceParams.simple(1), cfm, cfm2);
}
- private static ColumnDefinition fakeColumn(String name, AbstractType<?> type)
+ private static ColumnMetadata fakeColumn(String name, AbstractType<?> type)
{
- return new ColumnDefinition("fakeKs",
- "fakeTable",
- ColumnIdentifier.getInterned(name, false),
- type,
- ColumnDefinition.NO_POSITION,
- ColumnDefinition.Kind.REGULAR);
+ return new ColumnMetadata("fakeKs",
+ "fakeTable",
+ ColumnIdentifier.getInterned(name, false),
+ type,
+ ColumnMetadata.NO_POSITION,
+ ColumnMetadata.Kind.REGULAR);
}
@Test
@@ -130,7 +131,7 @@
@Test
public void testValidate()
{
- ColumnDefinition c;
+ ColumnMetadata c;
// Valid cells
c = fakeColumn("c", Int32Type.instance);
@@ -173,7 +174,7 @@
asList(f1, f2),
asList(Int32Type.instance, UTF8Type.instance),
true);
- ColumnDefinition c;
+ ColumnMetadata c;
// Valid cells
c = fakeColumn("c", udt);
@@ -207,7 +208,7 @@
asList(Int32Type.instance, UTF8Type.instance),
false);
- ColumnDefinition c = fakeColumn("c", udt);
+ ColumnMetadata c = fakeColumn("c", udt);
ByteBuffer val = udt(bb(1), bb("foo"));
// Valid cells
@@ -243,11 +244,13 @@
Assert.assertEquals(-1, testExpiring("val", "a", 2, 1, null, "val", 1L, 2));
Assert.assertEquals(-1, testExpiring("val", "a", 1, 2, null, null, null, 1));
- Assert.assertEquals(1, testExpiring("val", "a", 1, 2, null, "val", null, 1));
+ Assert.assertEquals(-1, testExpiring("val", "a", 1, 2, null, "val", null, 1));
+ Assert.assertEquals(1, testExpiring("val", "a", 1, 1, null, "val", null, 2));
+ Assert.assertEquals(1, testExpiring("val", "a", 1, 1, null, "val", null, 1));
// newer value
Assert.assertEquals(-1, testExpiring("val", "b", 2, 1, null, "a", null, null));
- Assert.assertEquals(-1, testExpiring("val", "b", 2, 1, null, "a", null, 2));
+ Assert.assertEquals(-1, testExpiring("val", "b", 2, 1, null, "a", null, 1));
}
class SimplePurger implements DeletionPurger
@@ -349,7 +352,7 @@
@Test
public void testComplexCellReconcile()
{
- ColumnDefinition m = cfm2.getColumnDefinition(new ColumnIdentifier("m", false));
+ ColumnMetadata m = cfm2.getColumn(new ColumnIdentifier("m", false));
int now1 = FBUtilities.nowInSeconds();
long ts1 = now1*1000000L;
@@ -366,7 +369,7 @@
List<Cell> cells2 = Lists.newArrayList(r2m2, r2m3, r2m4);
RowBuilder builder = new RowBuilder();
- Cells.reconcileComplex(m, cells1.iterator(), cells2.iterator(), DeletionTime.LIVE, builder, now2 + 1);
+ Cells.reconcileComplex(m, cells1.iterator(), cells2.iterator(), DeletionTime.LIVE, builder);
Assert.assertEquals(Lists.newArrayList(r1m1, r2m2, r2m3, r2m4), builder.cells);
}
@@ -383,32 +386,31 @@
Cell c1 = expiring(cfm, n1, v1, t1, et1);
Cell c2 = expiring(cfm, n2, v2, t2, et2);
- int now = FBUtilities.nowInSeconds();
- if (Cells.reconcile(c1, c2, now) == c1)
- return Cells.reconcile(c2, c1, now) == c1 ? -1 : 0;
- return Cells.reconcile(c2, c1, now) == c2 ? 1 : 0;
+ if (Cells.reconcile(c1, c2) == c1)
+ return Cells.reconcile(c2, c1) == c1 ? -1 : 0;
+ return Cells.reconcile(c2, c1) == c2 ? 1 : 0;
}
- private Cell regular(CFMetaData cfm, String columnName, String value, long timestamp)
+ private Cell regular(TableMetadata cfm, String columnName, String value, long timestamp)
{
- ColumnDefinition cdef = cfm.getColumnDefinition(ByteBufferUtil.bytes(columnName));
+ ColumnMetadata cdef = cfm.getColumn(ByteBufferUtil.bytes(columnName));
return BufferCell.live(cdef, timestamp, ByteBufferUtil.bytes(value));
}
- private Cell expiring(CFMetaData cfm, String columnName, String value, long timestamp, int localExpirationTime)
+ private Cell expiring(TableMetadata cfm, String columnName, String value, long timestamp, int localExpirationTime)
{
return expiring(cfm, columnName, value, timestamp, 1, localExpirationTime);
}
- private Cell expiring(CFMetaData cfm, String columnName, String value, long timestamp, int ttl, int localExpirationTime)
+ private Cell expiring(TableMetadata cfm, String columnName, String value, long timestamp, int ttl, int localExpirationTime)
{
- ColumnDefinition cdef = cfm.getColumnDefinition(ByteBufferUtil.bytes(columnName));
+ ColumnMetadata cdef = cfm.getColumn(ByteBufferUtil.bytes(columnName));
return new BufferCell(cdef, timestamp, ttl, localExpirationTime, ByteBufferUtil.bytes(value), null);
}
- private Cell deleted(CFMetaData cfm, String columnName, int localDeletionTime, long timestamp)
+ private Cell deleted(TableMetadata cfm, String columnName, int localDeletionTime, long timestamp)
{
- ColumnDefinition cdef = cfm.getColumnDefinition(ByteBufferUtil.bytes(columnName));
+ ColumnMetadata cdef = cfm.getColumn(ByteBufferUtil.bytes(columnName));
return BufferCell.tombstone(cdef, timestamp, localDeletionTime);
}
}
diff --git a/test/unit/org/apache/cassandra/db/CleanupTest.java b/test/unit/org/apache/cassandra/db/CleanupTest.java
index 552e6d1..9965361 100644
--- a/test/unit/org/apache/cassandra/db/CleanupTest.java
+++ b/test/unit/org/apache/cassandra/db/CleanupTest.java
@@ -19,7 +19,6 @@
package org.apache.cassandra.db;
import java.io.IOException;
-import java.net.InetAddress;
import java.net.UnknownHostException;
import java.nio.ByteBuffer;
import java.util.AbstractMap;
@@ -39,8 +38,9 @@
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.Util;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.schema.KeyspaceMetadata;
import org.apache.cassandra.cql3.Operator;
import org.apache.cassandra.db.compaction.CompactionManager;
@@ -94,13 +94,13 @@
DatabaseDescriptor.setEndpointSnitch(new AbstractNetworkTopologySnitch()
{
@Override
- public String getRack(InetAddress endpoint)
+ public String getRack(InetAddressAndPort endpoint)
{
return "RC1";
}
@Override
- public String getDatacenter(InetAddress endpoint)
+ public String getDatacenter(InetAddressAndPort endpoint)
{
return "DC1";
}
@@ -123,6 +123,7 @@
Keyspace keyspace = Keyspace.open(KEYSPACE1);
ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF_STANDARD1);
+
// insert data and verify we get it back w/ range query
fillCF(cfs, "val", LOOPS);
@@ -130,6 +131,7 @@
List<Long> expectedMaxTimestamps = getMaxTimestampList(cfs);
assertEquals(LOOPS, Util.getAll(Util.cmd(cfs).build()).size());
+
// with one token in the ring, owned by the local node, cleanup should be a no-op
CompactionManager.instance.performCleanup(cfs, 2);
@@ -151,7 +153,7 @@
fillCF(cfs, "birthdate", LOOPS);
assertEquals(LOOPS, Util.getAll(Util.cmd(cfs).build()).size());
- ColumnDefinition cdef = cfs.metadata.getColumnDefinition(COLUMN);
+ ColumnMetadata cdef = cfs.metadata().getColumn(COLUMN);
String indexName = "birthdate_key_index";
long start = System.nanoTime();
while (!cfs.getBuiltIndexes().contains(indexName) && System.nanoTime() - start < TimeUnit.SECONDS.toNanos(10))
@@ -167,8 +169,8 @@
byte[] tk1 = new byte[1], tk2 = new byte[1];
tk1[0] = 2;
tk2[0] = 1;
- tmd.updateNormalToken(new BytesToken(tk1), InetAddress.getByName("127.0.0.1"));
- tmd.updateNormalToken(new BytesToken(tk2), InetAddress.getByName("127.0.0.2"));
+ tmd.updateNormalToken(new BytesToken(tk1), InetAddressAndPort.getByName("127.0.0.1"));
+ tmd.updateNormalToken(new BytesToken(tk2), InetAddressAndPort.getByName("127.0.0.2"));
CompactionManager.instance.performCleanup(cfs, 2);
@@ -199,12 +201,13 @@
byte[] tk1 = new byte[1], tk2 = new byte[1];
tk1[0] = 2;
tk2[0] = 1;
- tmd.updateNormalToken(new BytesToken(tk1), InetAddress.getByName("127.0.0.1"));
- tmd.updateNormalToken(new BytesToken(tk2), InetAddress.getByName("127.0.0.2"));
+ tmd.updateNormalToken(new BytesToken(tk1), InetAddressAndPort.getByName("127.0.0.1"));
+ tmd.updateNormalToken(new BytesToken(tk2), InetAddressAndPort.getByName("127.0.0.2"));
CompactionManager.instance.performCleanup(cfs, 2);
assertEquals(0, Util.getAll(Util.cmd(cfs).build()).size());
}
+
@Test
public void testCleanupWithNoTokenRange() throws Exception
{
@@ -222,9 +225,9 @@
TokenMetadata tmd = StorageService.instance.getTokenMetadata();
tmd.clearUnsafe();
- tmd.updateHostId(UUID.randomUUID(), InetAddress.getByName("127.0.0.1"));
+ tmd.updateHostId(UUID.randomUUID(), InetAddressAndPort.getByName("127.0.0.1"));
byte[] tk1 = {2};
- tmd.updateNormalToken(new BytesToken(tk1), InetAddress.getByName("127.0.0.1"));
+ tmd.updateNormalToken(new BytesToken(tk1), InetAddressAndPort.getByName("127.0.0.1"));
Keyspace keyspace = Keyspace.open(KEYSPACE2);
@@ -258,32 +261,35 @@
Keyspace keyspace = Keyspace.open(KEYSPACE3);
ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF_STANDARD3);
cfs.disableAutoCompaction();
- for (byte i = 0; i < 100; i++)
- {
- new RowUpdateBuilder(cfs.metadata, System.currentTimeMillis(), ByteBuffer.wrap(new byte[] {i}))
- .clustering(COLUMN)
- .add("val", VALUE)
- .build()
- .applyUnsafe();
- cfs.forceBlockingFlush();
- }
TokenMetadata tmd = StorageService.instance.getTokenMetadata();
tmd.clearUnsafe();
- tmd.updateHostId(UUID.randomUUID(), InetAddress.getByName("127.0.0.1"));
- tmd.updateNormalToken(token(new byte[] {50}), InetAddress.getByName("127.0.0.1"));
+ tmd.updateNormalToken(token(new byte[]{ 50 }), InetAddressAndPort.getByName("127.0.0.1"));
+
+ for (byte i = 0; i < 100; i++)
+ {
+ new RowUpdateBuilder(cfs.metadata(), System.currentTimeMillis(), ByteBuffer.wrap(new byte[]{ i }))
+ .clustering(COLUMN)
+ .add("val", VALUE)
+ .build()
+ .applyUnsafe();
+ cfs.forceBlockingFlush();
+ }
+
Set<SSTableReader> beforeFirstCleanup = Sets.newHashSet(cfs.getLiveSSTables());
// single token - 127.0.0.1 owns everything, cleanup should be noop
cfs.forceCleanup(2);
assertEquals(beforeFirstCleanup, cfs.getLiveSSTables());
- tmd.updateNormalToken(token(new byte[] {120}), InetAddress.getByName("127.0.0.2"));
+ tmd.updateNormalToken(token(new byte[]{ 120 }), InetAddressAndPort.getByName("127.0.0.2"));
+
cfs.forceCleanup(2);
for (SSTableReader sstable : cfs.getLiveSSTables())
{
assertEquals(sstable.first, sstable.last); // single-token sstables
- assertTrue(sstable.first.getToken().compareTo(token(new byte[]{50})) <= 0);
+ assertTrue(sstable.first.getToken().compareTo(token(new byte[]{ 50 })) <= 0);
// with single-token sstables they should all either be skipped or dropped:
assertTrue(beforeFirstCleanup.contains(sstable));
}
+
}
@@ -304,8 +310,8 @@
byte[] tk1 = new byte[1], tk2 = new byte[1];
tk1[0] = 2;
tk2[0] = 1;
- tmd.updateNormalToken(new BytesToken(tk1), InetAddress.getByName("127.0.0.1"));
- tmd.updateNormalToken(new BytesToken(tk2), InetAddress.getByName("127.0.0.2"));
+ tmd.updateNormalToken(new BytesToken(tk1), InetAddressAndPort.getByName("127.0.0.1"));
+ tmd.updateNormalToken(new BytesToken(tk2), InetAddressAndPort.getByName("127.0.0.2"));
for(SSTableReader r: cfs.getLiveSSTables())
CompactionManager.instance.forceUserDefinedCleanup(r.getFilename());
@@ -394,7 +400,7 @@
{
String key = String.valueOf(i);
// create a row and update the birthdate value, test that the index query fetches the new version
- new RowUpdateBuilder(cfs.metadata, System.currentTimeMillis(), ByteBufferUtil.bytes(key))
+ new RowUpdateBuilder(cfs.metadata(), System.currentTimeMillis(), ByteBufferUtil.bytes(key))
.clustering(COLUMN)
.add(colName, VALUE)
.build()
diff --git a/test/unit/org/apache/cassandra/db/CleanupTransientTest.java b/test/unit/org/apache/cassandra/db/CleanupTransientTest.java
new file mode 100644
index 0000000..9789183
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/CleanupTransientTest.java
@@ -0,0 +1,195 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.UUID;
+
+import org.apache.cassandra.locator.RangesAtEndpoint;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.Util;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.db.partitions.FilteredPartition;
+import org.apache.cassandra.dht.IPartitioner;
+import org.apache.cassandra.dht.RandomPartitioner;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.locator.AbstractNetworkTopologySnitch;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.Replica;
+import org.apache.cassandra.locator.TokenMetadata;
+import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.service.PendingRangeCalculatorService;
+import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.utils.ByteBufferUtil;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class CleanupTransientTest
+{
+ private static final IPartitioner partitioner = RandomPartitioner.instance;
+ private static IPartitioner oldPartitioner;
+
+ public static final int LOOPS = 200;
+ public static final String KEYSPACE1 = "CleanupTest1";
+ public static final String CF_INDEXED1 = "Indexed1";
+ public static final String CF_STANDARD1 = "Standard1";
+
+ public static final String KEYSPACE2 = "CleanupTestMultiDc";
+ public static final String CF_INDEXED2 = "Indexed2";
+ public static final String CF_STANDARD2 = "Standard2";
+
+ public static final ByteBuffer COLUMN = ByteBufferUtil.bytes("birthdate");
+ public static final ByteBuffer VALUE = ByteBuffer.allocate(8);
+ static
+ {
+ VALUE.putLong(20101229);
+ VALUE.flip();
+ }
+
+ @BeforeClass
+ public static void setup() throws Exception
+ {
+ DatabaseDescriptor.daemonInitialization();
+ DatabaseDescriptor.setTransientReplicationEnabledUnsafe(true);
+ oldPartitioner = StorageService.instance.setPartitionerUnsafe(partitioner);
+ SchemaLoader.prepareServer();
+ SchemaLoader.createKeyspace(KEYSPACE1,
+ KeyspaceParams.simple("2/1"),
+ SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD1),
+ SchemaLoader.compositeIndexCFMD(KEYSPACE1, CF_INDEXED1, true));
+
+ StorageService ss = StorageService.instance;
+ final int RING_SIZE = 2;
+
+ TokenMetadata tmd = ss.getTokenMetadata();
+ tmd.clearUnsafe();
+ ArrayList<Token> endpointTokens = new ArrayList<>();
+ ArrayList<Token> keyTokens = new ArrayList<>();
+ List<InetAddressAndPort> hosts = new ArrayList<>();
+ List<UUID> hostIds = new ArrayList<>();
+
+ endpointTokens.add(RandomPartitioner.MINIMUM);
+ endpointTokens.add(RandomPartitioner.instance.midpoint(RandomPartitioner.MINIMUM, new RandomPartitioner.BigIntegerToken(RandomPartitioner.MAXIMUM)));
+
+ Util.createInitialRing(ss, partitioner, endpointTokens, keyTokens, hosts, hostIds, RING_SIZE);
+ PendingRangeCalculatorService.instance.blockUntilFinished();
+
+
+ DatabaseDescriptor.setEndpointSnitch(new AbstractNetworkTopologySnitch()
+ {
+ @Override
+ public String getRack(InetAddressAndPort endpoint)
+ {
+ return "RC1";
+ }
+
+ @Override
+ public String getDatacenter(InetAddressAndPort endpoint)
+ {
+ return "DC1";
+ }
+ });
+ }
+
+ @Test
+ public void testCleanup() throws Exception
+ {
+ Keyspace keyspace = Keyspace.open(KEYSPACE1);
+ ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF_STANDARD1);
+
+
+ // insert data and verify we get it back w/ range query
+ fillCF(cfs, "val", LOOPS);
+
+ // record max timestamps of the sstables pre-cleanup
+ List<Long> expectedMaxTimestamps = getMaxTimestampList(cfs);
+
+ assertEquals(LOOPS, Util.getAll(Util.cmd(cfs).build()).size());
+
+ // with two tokens RF=2/1 and the sstable not repaired this should do nothing
+ CompactionManager.instance.performCleanup(cfs, 2);
+
+ // ensure max timestamp of the sstables are retained post-cleanup
+ assert expectedMaxTimestamps.equals(getMaxTimestampList(cfs));
+
+ // check data is still there
+ assertEquals(LOOPS, Util.getAll(Util.cmd(cfs).build()).size());
+
+ //Get an exact count of how many partitions are in the fully replicated range and should
+ //be retained
+ int fullCount = 0;
+ RangesAtEndpoint localRanges = StorageService.instance.getLocalReplicas(keyspace.getName()).filter(Replica::isFull);
+ for (FilteredPartition partition : Util.getAll(Util.cmd(cfs).build()))
+ {
+ Token token = partition.partitionKey().getToken();
+ for (Replica r : localRanges)
+ {
+ if (r.range().contains(token))
+ fullCount++;
+ }
+ }
+
+ SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
+ sstable.descriptor.getMetadataSerializer().mutateRepairMetadata(sstable.descriptor, 1, null, false);
+ sstable.reloadSSTableMetadata();
+
+ // This should remove approximately 50% of the data, specifically whatever was transiently replicated
+ CompactionManager.instance.performCleanup(cfs, 2);
+
+ // ensure max timestamp of the sstables are retained post-cleanup
+ assert expectedMaxTimestamps.equals(getMaxTimestampList(cfs));
+
+ // check less data is there, all transient data should be gone since the table was repaired
+ assertEquals(fullCount, Util.getAll(Util.cmd(cfs).build()).size());
+ }
+
+ protected void fillCF(ColumnFamilyStore cfs, String colName, int rowsPerSSTable)
+ {
+ CompactionManager.instance.disableAutoCompaction();
+
+ for (int i = 0; i < rowsPerSSTable; i++)
+ {
+ String key = String.valueOf(i);
+ // create a row and update the birthdate value, test that the index query fetches the new version
+ new RowUpdateBuilder(cfs.metadata(), System.currentTimeMillis(), ByteBufferUtil.bytes(key))
+ .clustering(COLUMN)
+ .add(colName, VALUE)
+ .build()
+ .applyUnsafe();
+ }
+
+ cfs.forceBlockingFlush();
+ }
+
+ protected List<Long> getMaxTimestampList(ColumnFamilyStore cfs)
+ {
+ List<Long> list = new LinkedList<Long>();
+ for (SSTableReader sstable : cfs.getLiveSSTables())
+ list.add(sstable.getMaxTimestamp());
+ return list;
+ }
+}
diff --git a/test/unit/org/apache/cassandra/db/ColumnFamilyMetricTest.java b/test/unit/org/apache/cassandra/db/ColumnFamilyMetricTest.java
index 2f7aaa5..c016f9b 100644
--- a/test/unit/org/apache/cassandra/db/ColumnFamilyMetricTest.java
+++ b/test/unit/org/apache/cassandra/db/ColumnFamilyMetricTest.java
@@ -55,7 +55,7 @@
for (int j = 0; j < 10; j++)
{
- new RowUpdateBuilder(cfs.metadata, FBUtilities.timestampMicros(), String.valueOf(j))
+ new RowUpdateBuilder(cfs.metadata(), FBUtilities.timestampMicros(), String.valueOf(j))
.clustering("0")
.add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER)
.build()
@@ -91,7 +91,7 @@
// This confirms another test/set up did not overflow the histogram
store.metric.colUpdateTimeDeltaHistogram.cf.getSnapshot().get999thPercentile();
- new RowUpdateBuilder(store.metadata, 0, "4242")
+ new RowUpdateBuilder(store.metadata(), 0, "4242")
.clustering("0")
.add("val", ByteBufferUtil.bytes("0"))
.build()
@@ -101,7 +101,7 @@
store.metric.colUpdateTimeDeltaHistogram.cf.getSnapshot().get999thPercentile();
// smallest time delta that would overflow the histogram if unfiltered
- new RowUpdateBuilder(store.metadata, 18165375903307L, "4242")
+ new RowUpdateBuilder(store.metadata(), 18165375903307L, "4242")
.clustering("0")
.add("val", ByteBufferUtil.bytes("0"))
.build()
diff --git a/test/unit/org/apache/cassandra/db/ColumnFamilyStoreCQLHelperTest.java b/test/unit/org/apache/cassandra/db/ColumnFamilyStoreCQLHelperTest.java
deleted file mode 100644
index 090d7e9..0000000
--- a/test/unit/org/apache/cassandra/db/ColumnFamilyStoreCQLHelperTest.java
+++ /dev/null
@@ -1,691 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.db;
-
-import java.io.FileReader;
-import java.nio.ByteBuffer;
-import java.nio.charset.Charset;
-import java.util.*;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.io.Files;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-import org.apache.cassandra.*;
-import org.apache.cassandra.config.*;
-import org.apache.cassandra.cql3.*;
-import org.apache.cassandra.cql3.statements.*;
-import org.apache.cassandra.db.marshal.*;
-import org.apache.cassandra.exceptions.*;
-import org.apache.cassandra.index.sasi.*;
-import org.apache.cassandra.schema.*;
-import org.apache.cassandra.utils.*;
-import org.json.simple.JSONArray;
-import org.json.simple.JSONObject;
-import org.json.simple.parser.JSONParser;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-public class ColumnFamilyStoreCQLHelperTest extends CQLTester
-{
- @Before
- public void defineSchema() throws ConfigurationException
- {
- SchemaLoader.prepareServer();
- }
-
- @Test
- public void testUserTypesCQL()
- {
- String keyspace = "cql_test_keyspace_user_types";
- String table = "test_table_user_types";
-
- UserType typeA = new UserType(keyspace, ByteBufferUtil.bytes("a"),
- Arrays.asList(FieldIdentifier.forUnquoted("a1"),
- FieldIdentifier.forUnquoted("a2"),
- FieldIdentifier.forUnquoted("a3")),
- Arrays.asList(IntegerType.instance,
- IntegerType.instance,
- IntegerType.instance),
- true);
-
- UserType typeB = new UserType(keyspace, ByteBufferUtil.bytes("b"),
- Arrays.asList(FieldIdentifier.forUnquoted("b1"),
- FieldIdentifier.forUnquoted("b2"),
- FieldIdentifier.forUnquoted("b3")),
- Arrays.asList(typeA,
- typeA,
- typeA),
- true);
-
- UserType typeC = new UserType(keyspace, ByteBufferUtil.bytes("c"),
- Arrays.asList(FieldIdentifier.forUnquoted("c1"),
- FieldIdentifier.forUnquoted("c2"),
- FieldIdentifier.forUnquoted("c3")),
- Arrays.asList(typeB,
- typeB,
- typeB),
- true);
-
- CFMetaData cfm = CFMetaData.Builder.create(keyspace, table)
- .addPartitionKey("pk1", IntegerType.instance)
- .addClusteringColumn("ck1", IntegerType.instance)
- .addRegularColumn("reg1", typeC)
- .addRegularColumn("reg2", ListType.getInstance(IntegerType.instance, false))
- .addRegularColumn("reg3", MapType.getInstance(AsciiType.instance, IntegerType.instance, true))
- .build();
-
- SchemaLoader.createKeyspace(keyspace,
- KeyspaceParams.simple(1),
- Tables.of(cfm),
- Types.of(typeA, typeB, typeC));
-
- ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
-
- assertEquals(ImmutableList.of("CREATE TYPE cql_test_keyspace_user_types.a(a1 varint, a2 varint, a3 varint);",
- "CREATE TYPE cql_test_keyspace_user_types.b(b1 a, b2 a, b3 a);",
- "CREATE TYPE cql_test_keyspace_user_types.c(c1 b, c2 b, c3 b);"),
- ColumnFamilyStoreCQLHelper.getUserTypesAsCQL(cfs.metadata));
- }
-
- @Test
- public void testDroppedColumnsCQL()
- {
- String keyspace = "cql_test_keyspace_dropped_columns";
- String table = "test_table_dropped_columns";
-
- CFMetaData cfm = CFMetaData.Builder.create(keyspace, table)
- .addPartitionKey("pk1", IntegerType.instance)
- .addClusteringColumn("ck1", IntegerType.instance)
- .addRegularColumn("reg1", IntegerType.instance)
- .addRegularColumn("reg2", IntegerType.instance)
- .addRegularColumn("reg3", IntegerType.instance)
- .build();
-
-
- ColumnDefinition reg1 = cfm.getColumnDefinition(ByteBufferUtil.bytes("reg1"));
- ColumnDefinition reg2 = cfm.getColumnDefinition(ByteBufferUtil.bytes("reg2"));
- ColumnDefinition reg3 = cfm.getColumnDefinition(ByteBufferUtil.bytes("reg3"));
-
- cfm.removeColumnDefinition(reg1);
- cfm.removeColumnDefinition(reg2);
- cfm.removeColumnDefinition(reg3);
-
- cfm.recordColumnDrop(reg1, 10000);
- cfm.recordColumnDrop(reg2, 20000);
- cfm.recordColumnDrop(reg3, 30000);
-
- SchemaLoader.createKeyspace(keyspace,
- KeyspaceParams.simple(1),
- cfm);
-
- ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
-
- assertEquals(ImmutableList.of("ALTER TABLE cql_test_keyspace_dropped_columns.test_table_dropped_columns DROP reg1 USING TIMESTAMP 10000;",
- "ALTER TABLE cql_test_keyspace_dropped_columns.test_table_dropped_columns DROP reg3 USING TIMESTAMP 30000;",
- "ALTER TABLE cql_test_keyspace_dropped_columns.test_table_dropped_columns DROP reg2 USING TIMESTAMP 20000;"),
- ColumnFamilyStoreCQLHelper.getDroppedColumnsAsCQL(cfs.metadata));
-
- assertTrue(ColumnFamilyStoreCQLHelper.getCFMetadataAsCQL(cfs.metadata, true).startsWith(
- "CREATE TABLE IF NOT EXISTS cql_test_keyspace_dropped_columns.test_table_dropped_columns (\n" +
- "\tpk1 varint,\n" +
- "\tck1 varint,\n" +
- "\treg1 varint,\n" +
- "\treg3 varint,\n" +
- "\treg2 varint,\n" +
- "\tPRIMARY KEY (pk1, ck1))"));
- }
-
- @Test
- public void testReaddedColumns()
- {
- String keyspace = "cql_test_keyspace_readded_columns";
- String table = "test_table_readded_columns";
-
- CFMetaData cfm = CFMetaData.Builder.create(keyspace, table)
- .addPartitionKey("pk1", IntegerType.instance)
- .addClusteringColumn("ck1", IntegerType.instance)
- .addRegularColumn("reg1", IntegerType.instance)
- .addStaticColumn("reg2", IntegerType.instance)
- .addRegularColumn("reg3", IntegerType.instance)
- .build();
-
- ColumnDefinition reg1 = cfm.getColumnDefinition(ByteBufferUtil.bytes("reg1"));
- ColumnDefinition reg2 = cfm.getColumnDefinition(ByteBufferUtil.bytes("reg2"));
-
- cfm.removeColumnDefinition(reg1);
- cfm.removeColumnDefinition(reg2);
-
- cfm.recordColumnDrop(reg1, 10000);
- cfm.recordColumnDrop(reg2, 20000);
-
- cfm.addColumnDefinition(reg1);
- cfm.addColumnDefinition(reg2);
-
- SchemaLoader.createKeyspace(keyspace,
- KeyspaceParams.simple(1),
- cfm);
-
- ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
-
- // when re-adding, column is present in CREATE, then in DROP and then in ADD again, to record DROP with a proper timestamp
- assertTrue(ColumnFamilyStoreCQLHelper.getCFMetadataAsCQL(cfs.metadata, true).startsWith(
- "CREATE TABLE IF NOT EXISTS cql_test_keyspace_readded_columns.test_table_readded_columns (\n" +
- "\tpk1 varint,\n" +
- "\tck1 varint,\n" +
- "\treg2 varint static,\n" +
- "\treg1 varint,\n" +
- "\treg3 varint,\n" +
- "\tPRIMARY KEY (pk1, ck1))"));
-
- assertEquals(ImmutableList.of("ALTER TABLE cql_test_keyspace_readded_columns.test_table_readded_columns DROP reg1 USING TIMESTAMP 10000;",
- "ALTER TABLE cql_test_keyspace_readded_columns.test_table_readded_columns ADD reg1 varint;",
- "ALTER TABLE cql_test_keyspace_readded_columns.test_table_readded_columns DROP reg2 USING TIMESTAMP 20000;",
- "ALTER TABLE cql_test_keyspace_readded_columns.test_table_readded_columns ADD reg2 varint static;"),
- ColumnFamilyStoreCQLHelper.getDroppedColumnsAsCQL(cfs.metadata));
- }
-
- @Test
- public void testCfmColumnsCQL()
- {
- String keyspace = "cql_test_keyspace_create_table";
- String table = "test_table_create_table";
-
- CFMetaData cfm = CFMetaData.Builder.create(keyspace, table)
- .addPartitionKey("pk1", IntegerType.instance)
- .addPartitionKey("pk2", AsciiType.instance)
- .addClusteringColumn("ck1", ReversedType.getInstance(IntegerType.instance))
- .addClusteringColumn("ck2", IntegerType.instance)
- .addStaticColumn("st1", AsciiType.instance)
- .addRegularColumn("reg1", AsciiType.instance)
- .addRegularColumn("reg2", ListType.getInstance(IntegerType.instance, false))
- .addRegularColumn("reg3", MapType.getInstance(AsciiType.instance, IntegerType.instance, true))
- .build();
-
- SchemaLoader.createKeyspace(keyspace,
- KeyspaceParams.simple(1),
- cfm);
-
- ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
-
- assertTrue(ColumnFamilyStoreCQLHelper.getCFMetadataAsCQL(cfs.metadata, true).startsWith(
- "CREATE TABLE IF NOT EXISTS cql_test_keyspace_create_table.test_table_create_table (\n" +
- "\tpk1 varint,\n" +
- "\tpk2 ascii,\n" +
- "\tck1 varint,\n" +
- "\tck2 varint,\n" +
- "\tst1 ascii static,\n" +
- "\treg1 ascii,\n" +
- "\treg2 frozen<list<varint>>,\n" +
- "\treg3 map<ascii, varint>,\n" +
- "\tPRIMARY KEY ((pk1, pk2), ck1, ck2))\n" +
- "\tWITH ID = " + cfs.metadata.cfId + "\n" +
- "\tAND CLUSTERING ORDER BY (ck1 DESC, ck2 ASC)"));
- }
-
- @Test
- public void testCfmCompactStorageCQL()
- {
- String keyspace = "cql_test_keyspace_compact";
- String table = "test_table_compact";
-
- CFMetaData cfm = CFMetaData.Builder.createDense(keyspace, table, true, false)
- .addPartitionKey("pk1", IntegerType.instance)
- .addPartitionKey("pk2", AsciiType.instance)
- .addClusteringColumn("ck1", ReversedType.getInstance(IntegerType.instance))
- .addClusteringColumn("ck2", IntegerType.instance)
- .addRegularColumn("reg", IntegerType.instance)
- .build();
-
- SchemaLoader.createKeyspace(keyspace,
- KeyspaceParams.simple(1),
- cfm);
-
- ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
-
- assertTrue(ColumnFamilyStoreCQLHelper.getCFMetadataAsCQL(cfs.metadata, true).startsWith(
- "CREATE TABLE IF NOT EXISTS cql_test_keyspace_compact.test_table_compact (\n" +
- "\tpk1 varint,\n" +
- "\tpk2 ascii,\n" +
- "\tck1 varint,\n" +
- "\tck2 varint,\n" +
- "\treg varint,\n" +
- "\tPRIMARY KEY ((pk1, pk2), ck1, ck2))\n" +
- "\tWITH ID = " + cfm.cfId + "\n" +
- "\tAND COMPACT STORAGE\n" +
- "\tAND CLUSTERING ORDER BY (ck1 DESC, ck2 ASC)"));
- }
-
- @Test
- public void testCfmCounterCQL()
- {
- String keyspace = "cql_test_keyspace_counter";
- String table = "test_table_counter";
-
- CFMetaData cfm = CFMetaData.Builder.createDense(keyspace, table, true, true)
- .addPartitionKey("pk1", IntegerType.instance)
- .addPartitionKey("pk2", AsciiType.instance)
- .addClusteringColumn("ck1", ReversedType.getInstance(IntegerType.instance))
- .addClusteringColumn("ck2", IntegerType.instance)
- .addRegularColumn("cnt", CounterColumnType.instance)
- .build();
-
- SchemaLoader.createKeyspace(keyspace,
- KeyspaceParams.simple(1),
- cfm);
-
- ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
-
- assertTrue(ColumnFamilyStoreCQLHelper.getCFMetadataAsCQL(cfs.metadata, true).startsWith(
- "CREATE TABLE IF NOT EXISTS cql_test_keyspace_counter.test_table_counter (\n" +
- "\tpk1 varint,\n" +
- "\tpk2 ascii,\n" +
- "\tck1 varint,\n" +
- "\tck2 varint,\n" +
- "\tcnt counter,\n" +
- "\tPRIMARY KEY ((pk1, pk2), ck1, ck2))\n" +
- "\tWITH ID = " + cfm.cfId + "\n" +
- "\tAND COMPACT STORAGE\n" +
- "\tAND CLUSTERING ORDER BY (ck1 DESC, ck2 ASC)"));
- }
-
- @Test
- public void testCfmOptionsCQL()
- {
- String keyspace = "cql_test_keyspace_options";
- String table = "test_table_options";
-
- CFMetaData cfm = CFMetaData.Builder.create(keyspace, table)
- .addPartitionKey("pk1", IntegerType.instance)
- .addClusteringColumn("cl1", IntegerType.instance)
- .addRegularColumn("reg1", AsciiType.instance)
- .build();
-
- cfm.recordColumnDrop(cfm.getColumnDefinition(ByteBuffer.wrap("reg1".getBytes())), FBUtilities.timestampMicros());
- cfm.bloomFilterFpChance(1.0);
- cfm.comment("comment");
- cfm.compaction(CompactionParams.lcs(Collections.singletonMap("sstable_size_in_mb", "1")));
- cfm.compression(CompressionParams.lz4(1 << 16));
- cfm.dcLocalReadRepairChance(0.2);
- cfm.crcCheckChance(0.3);
- cfm.defaultTimeToLive(4);
- cfm.gcGraceSeconds(5);
- cfm.minIndexInterval(6);
- cfm.maxIndexInterval(7);
- cfm.memtableFlushPeriod(8);
- cfm.readRepairChance(0.9);
- cfm.speculativeRetry(SpeculativeRetryParam.always());
- cfm.extensions(ImmutableMap.of("ext1",
- ByteBuffer.wrap("val1".getBytes())));
-
- SchemaLoader.createKeyspace(keyspace,
- KeyspaceParams.simple(1),
- cfm);
- ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
-
- assertTrue(ColumnFamilyStoreCQLHelper.getCFMetadataAsCQL(cfs.metadata, true).endsWith(
- "AND bloom_filter_fp_chance = 1.0\n" +
- "\tAND dclocal_read_repair_chance = 0.2\n" +
- "\tAND crc_check_chance = 0.3\n" +
- "\tAND default_time_to_live = 4\n" +
- "\tAND gc_grace_seconds = 5\n" +
- "\tAND min_index_interval = 6\n" +
- "\tAND max_index_interval = 7\n" +
- "\tAND memtable_flush_period_in_ms = 8\n" +
- "\tAND read_repair_chance = 0.9\n" +
- "\tAND speculative_retry = 'ALWAYS'\n" +
- "\tAND comment = 'comment'\n" +
- "\tAND caching = { 'keys': 'ALL', 'rows_per_partition': 'NONE' }\n" +
- "\tAND compaction = { 'class': 'org.apache.cassandra.db.compaction.LeveledCompactionStrategy', 'sstable_size_in_mb': '1' }\n" +
- "\tAND compression = { 'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor' }\n" +
- "\tAND cdc = false\n" +
- "\tAND extensions = { 'ext1': 0x76616c31 };"
- ));
- }
-
- @Test
- public void testCfmIndexJson()
- {
- String keyspace = "cql_test_keyspace_3";
- String table = "test_table_3";
-
- CFMetaData cfm = CFMetaData.Builder.create(keyspace, table)
- .addPartitionKey("pk1", IntegerType.instance)
- .addClusteringColumn("cl1", IntegerType.instance)
- .addRegularColumn("reg1", AsciiType.instance)
- .build();
-
- cfm.indexes(cfm.getIndexes()
- .with(IndexMetadata.fromIndexTargets(cfm,
- Collections.singletonList(new IndexTarget(cfm.getColumnDefinition(ByteBufferUtil.bytes("reg1")).name,
- IndexTarget.Type.VALUES)),
- "indexName",
- IndexMetadata.Kind.COMPOSITES,
- Collections.emptyMap()))
- .with(IndexMetadata.fromIndexTargets(cfm,
- Collections.singletonList(new IndexTarget(cfm.getColumnDefinition(ByteBufferUtil.bytes("reg1")).name,
- IndexTarget.Type.KEYS)),
- "indexName2",
- IndexMetadata.Kind.COMPOSITES,
- Collections.emptyMap()))
- .with(IndexMetadata.fromIndexTargets(cfm,
- Collections.singletonList(new IndexTarget(cfm.getColumnDefinition(ByteBufferUtil.bytes("reg1")).name,
- IndexTarget.Type.KEYS_AND_VALUES)),
- "indexName3",
- IndexMetadata.Kind.COMPOSITES,
- Collections.emptyMap()))
- .with(IndexMetadata.fromIndexTargets(cfm,
- Collections.singletonList(new IndexTarget(cfm.getColumnDefinition(ByteBufferUtil.bytes("reg1")).name,
- IndexTarget.Type.KEYS_AND_VALUES)),
- "indexName4",
- IndexMetadata.Kind.CUSTOM,
- Collections.singletonMap(IndexTarget.CUSTOM_INDEX_OPTION_NAME,
- SASIIndex.class.getName()))
- ));
-
-
- SchemaLoader.createKeyspace(keyspace,
- KeyspaceParams.simple(1),
- cfm);
-
- ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
-
- assertEquals(ImmutableList.of("CREATE INDEX \"indexName\" ON cql_test_keyspace_3.test_table_3 (reg1);",
- "CREATE INDEX \"indexName2\" ON cql_test_keyspace_3.test_table_3 (reg1);",
- "CREATE INDEX \"indexName3\" ON cql_test_keyspace_3.test_table_3 (reg1);",
- "CREATE CUSTOM INDEX \"indexName4\" ON cql_test_keyspace_3.test_table_3 (reg1) USING 'org.apache.cassandra.index.sasi.SASIIndex';"),
- ColumnFamilyStoreCQLHelper.getIndexesAsCQL(cfs.metadata));
- }
-
- private final static String SNAPSHOT = "testsnapshot";
-
- @Test
- public void testSnapshot() throws Throwable
- {
- String typeA = createType("CREATE TYPE %s (a1 varint, a2 varint, a3 varint);");
- String typeB = createType("CREATE TYPE %s (b1 frozen<" + typeA + ">, b2 frozen<" + typeA + ">, b3 frozen<" + typeA + ">);");
- String typeC = createType("CREATE TYPE %s (c1 frozen<" + typeB + ">, c2 frozen<" + typeB + ">, c3 frozen<" + typeB + ">);");
-
- String tableName = createTable("CREATE TABLE IF NOT EXISTS %s (" +
- "pk1 varint," +
- "pk2 ascii," +
- "ck1 varint," +
- "ck2 varint," +
- "reg1 " + typeC + "," +
- "reg2 int," +
- "reg3 int," +
- "PRIMARY KEY ((pk1, pk2), ck1, ck2)) WITH " +
- "CLUSTERING ORDER BY (ck1 ASC, ck2 DESC);");
-
- alterTable("ALTER TABLE %s DROP reg3 USING TIMESTAMP 10000;");
- alterTable("ALTER TABLE %s ADD reg3 int;");
-
- for (int i = 0; i < 10; i++)
- execute("INSERT INTO %s (pk1, pk2, ck1, ck2, reg1, reg2) VALUES (?, ?, ?, ?, ?, ?)", i, i + 1, i + 2, i + 3, null, i + 5);
-
- ColumnFamilyStore cfs = Keyspace.open(keyspace()).getColumnFamilyStore(tableName);
- cfs.snapshot(SNAPSHOT);
-
- String schema = Files.toString(cfs.getDirectories().getSnapshotSchemaFile(SNAPSHOT), Charset.defaultCharset());
- assertTrue(schema.contains(String.format("CREATE TYPE %s.%s(a1 varint, a2 varint, a3 varint);", keyspace(), typeA)));
- assertTrue(schema.contains(String.format("CREATE TYPE %s.%s(a1 varint, a2 varint, a3 varint);", keyspace(), typeA)));
- assertTrue(schema.contains(String.format("CREATE TYPE %s.%s(b1 frozen<%s>, b2 frozen<%s>, b3 frozen<%s>);", keyspace(), typeB, typeA, typeA, typeA)));
- assertTrue(schema.contains(String.format("CREATE TYPE %s.%s(c1 frozen<%s>, c2 frozen<%s>, c3 frozen<%s>);", keyspace(), typeC, typeB, typeB, typeB)));
-
- schema = schema.substring(schema.indexOf("CREATE TABLE")); // trim to ensure order
-
- assertTrue(schema.startsWith("CREATE TABLE IF NOT EXISTS " + keyspace() + "." + tableName + " (\n" +
- "\tpk1 varint,\n" +
- "\tpk2 ascii,\n" +
- "\tck1 varint,\n" +
- "\tck2 varint,\n" +
- "\treg2 int,\n" +
- "\treg3 int,\n" +
- "\treg1 " + typeC + ",\n" +
- "\tPRIMARY KEY ((pk1, pk2), ck1, ck2))\n" +
- "\tWITH ID = " + cfs.metadata.cfId + "\n" +
- "\tAND CLUSTERING ORDER BY (ck1 ASC, ck2 DESC)"));
-
- schema = schema.substring(schema.indexOf("ALTER"));
- assertTrue(schema.startsWith(String.format("ALTER TABLE %s.%s DROP reg3 USING TIMESTAMP 10000;", keyspace(), tableName)));
- assertTrue(schema.contains(String.format("ALTER TABLE %s.%s ADD reg3 int;", keyspace(), tableName)));
-
- JSONObject manifest = (JSONObject) new JSONParser().parse(new FileReader(cfs.getDirectories().getSnapshotManifestFile(SNAPSHOT)));
- JSONArray files = (JSONArray) manifest.get("files");
- Assert.assertEquals(1, files.size());
- }
-
- @Test
- public void testSystemKsSnapshot() throws Throwable
- {
- ColumnFamilyStore cfs = Keyspace.open("system").getColumnFamilyStore("peers");
- cfs.snapshot(SNAPSHOT);
-
- Assert.assertTrue(cfs.getDirectories().getSnapshotManifestFile(SNAPSHOT).exists());
- Assert.assertFalse(cfs.getDirectories().getSnapshotSchemaFile(SNAPSHOT).exists());
- }
-
- @Test
- public void testDroppedType() throws Throwable
- {
- String typeA = createType("CREATE TYPE %s (a1 varint, a2 varint, a3 varint);");
- String typeB = createType("CREATE TYPE %s (b1 frozen<" + typeA + ">, b2 frozen<" + typeA + ">, b3 frozen<" + typeA + ">);");
-
- String tableName = createTable("CREATE TABLE IF NOT EXISTS %s (" +
- "pk1 varint," +
- "ck1 varint," +
- "reg1 " + typeB + "," +
- "reg2 varint," +
- "PRIMARY KEY (pk1, ck1));");
-
- alterTable("ALTER TABLE %s DROP reg1 USING TIMESTAMP 10000;");
-
- Runnable validate = () -> {
- try
- {
- ColumnFamilyStore cfs = Keyspace.open(keyspace()).getColumnFamilyStore(tableName);
- cfs.snapshot(SNAPSHOT);
- String schema = Files.toString(cfs.getDirectories().getSnapshotSchemaFile(SNAPSHOT), Charset.defaultCharset());
-
- // When both column and it's type are dropped, the type in column definition gets substituted with a tuple
- assertTrue(schema.startsWith("CREATE TABLE IF NOT EXISTS " + keyspace() + "." + tableName + " (\n" +
- "\tpk1 varint,\n" +
- "\tck1 varint,\n" +
- "\treg2 varint,\n" +
- "\treg1 frozen<tuple<frozen<tuple<varint, varint, varint>>, frozen<tuple<varint, varint, varint>>, frozen<tuple<varint, varint, varint>>>>,\n" +
- "\tPRIMARY KEY (pk1, ck1))"));
- assertTrue(schema.contains("ALTER TABLE " + keyspace() + "." + tableName + " DROP reg1 USING TIMESTAMP 10000;"));
- }
- catch (Exception e)
- {
- throw new RuntimeException(e);
- }
- };
-
- // Validate before and after the type drop
- validate.run();
- schemaChange("DROP TYPE " + keyspace() + "." + typeB);
- schemaChange("DROP TYPE " + keyspace() + "." + typeA);
- validate.run();
- }
-
- @Test
- public void testDenseTable() throws Throwable
- {
- String tableName = createTable("CREATE TABLE IF NOT EXISTS %s (" +
- "pk1 varint PRIMARY KEY," +
- "reg1 int)" +
- " WITH COMPACT STORAGE");
-
- ColumnFamilyStore cfs = Keyspace.open(keyspace()).getColumnFamilyStore(tableName);
-
- assertTrue(ColumnFamilyStoreCQLHelper.getCFMetadataAsCQL(cfs.metadata, true).startsWith(
- "CREATE TABLE IF NOT EXISTS " + keyspace() + "." + tableName + " (\n" +
- "\tpk1 varint PRIMARY KEY,\n" +
- "\treg1 int)\n" +
- "\tWITH ID = " + cfs.metadata.cfId + "\n" +
- "\tAND COMPACT STORAGE"));
- }
-
- @Test
- public void testStaticCompactTable() throws Throwable
- {
- String tableName = createTable("CREATE TABLE IF NOT EXISTS %s (" +
- "pk1 varint PRIMARY KEY," +
- "reg1 int," +
- "reg2 int)" +
- " WITH COMPACT STORAGE");
-
- ColumnFamilyStore cfs = Keyspace.open(keyspace()).getColumnFamilyStore(tableName);
-
- assertTrue(ColumnFamilyStoreCQLHelper.getCFMetadataAsCQL(cfs.metadata, true).startsWith(
- "CREATE TABLE IF NOT EXISTS " + keyspace() + "." + tableName + " (\n" +
- "\tpk1 varint PRIMARY KEY,\n" +
- "\treg1 int,\n" +
- "\treg2 int)\n" +
- "\tWITH ID = " + cfs.metadata.cfId + "\n" +
- "\tAND COMPACT STORAGE"));
- }
-
- @Test
- public void testStaticCompactWithCounters() throws Throwable
- {
- String tableName = createTable("CREATE TABLE IF NOT EXISTS %s (" +
- "pk1 varint PRIMARY KEY," +
- "reg1 counter," +
- "reg2 counter)" +
- " WITH COMPACT STORAGE");
-
-
- ColumnFamilyStore cfs = Keyspace.open(keyspace()).getColumnFamilyStore(tableName);
-
- assertTrue(ColumnFamilyStoreCQLHelper.getCFMetadataAsCQL(cfs.metadata, true).startsWith(
- "CREATE TABLE IF NOT EXISTS " + keyspace() + "." + tableName + " (\n" +
- "\tpk1 varint PRIMARY KEY,\n" +
- "\treg1 counter,\n" +
- "\treg2 counter)\n" +
- "\tWITH ID = " + cfs.metadata.cfId + "\n" +
- "\tAND COMPACT STORAGE"));
- }
-
- @Test
- public void testDenseCompactTableWithoutRegulars() throws Throwable
- {
- String tableName = createTable("CREATE TABLE IF NOT EXISTS %s (" +
- "pk1 varint," +
- "ck1 int," +
- "PRIMARY KEY (pk1, ck1))" +
- " WITH COMPACT STORAGE");
-
- ColumnFamilyStore cfs = Keyspace.open(keyspace()).getColumnFamilyStore(tableName);
-
- assertTrue(ColumnFamilyStoreCQLHelper.getCFMetadataAsCQL(cfs.metadata, true).startsWith(
- "CREATE TABLE IF NOT EXISTS " + keyspace() + "." + tableName + " (\n" +
- "\tpk1 varint,\n" +
- "\tck1 int,\n" +
- "\tPRIMARY KEY (pk1, ck1))\n" +
- "\tWITH ID = " + cfs.metadata.cfId + "\n" +
- "\tAND COMPACT STORAGE"));
- }
-
- @Test
- public void testCompactDynamic() throws Throwable
- {
- String tableName = createTable("CREATE TABLE IF NOT EXISTS %s (" +
- "pk1 varint," +
- "ck1 int," +
- "reg int," +
- "PRIMARY KEY (pk1, ck1))" +
- " WITH COMPACT STORAGE");
-
- ColumnFamilyStore cfs = Keyspace.open(keyspace()).getColumnFamilyStore(tableName);
-
- assertTrue(ColumnFamilyStoreCQLHelper.getCFMetadataAsCQL(cfs.metadata, true).startsWith(
- "CREATE TABLE IF NOT EXISTS " + keyspace() + "." + tableName + " (\n" +
- "\tpk1 varint,\n" +
- "\tck1 int,\n" +
- "\treg int,\n" +
- "\tPRIMARY KEY (pk1, ck1))\n" +
- "\tWITH ID = " + cfs.metadata.cfId + "\n" +
- "\tAND COMPACT STORAGE"));
- }
-
- @Test
- public void testDynamicComposite() throws Throwable
- {
- Map<Byte, AbstractType<?>> aliases = new HashMap<>();
- aliases.put((byte)'a', BytesType.instance);
- aliases.put((byte)'b', BytesType.instance);
- aliases.put((byte)'c', BytesType.instance);
-
- String DYNAMIC_COMPOSITE = "dynamic_composite";
- AbstractType<?> dynamicComposite = DynamicCompositeType.getInstance(aliases);
-
- SchemaLoader.createKeyspace(DYNAMIC_COMPOSITE,
- KeyspaceParams.simple(1),
- SchemaLoader.denseCFMD(DYNAMIC_COMPOSITE, DYNAMIC_COMPOSITE, dynamicComposite));
-
- ColumnFamilyStore cfs = Keyspace.open(DYNAMIC_COMPOSITE).getColumnFamilyStore(DYNAMIC_COMPOSITE);
-
- assertTrue(ColumnFamilyStoreCQLHelper.getCFMetadataAsCQL(cfs.metadata, true).startsWith(
- "CREATE TABLE IF NOT EXISTS " + DYNAMIC_COMPOSITE + "." + DYNAMIC_COMPOSITE + " (\n" +
- "\tkey ascii,\n" +
- "\tcols 'org.apache.cassandra.db.marshal.DynamicCompositeType(a=>org.apache.cassandra.db.marshal.BytesType,b=>org.apache.cassandra.db.marshal.BytesType,c=>org.apache.cassandra.db.marshal.BytesType)',\n" +
- "\tval ascii,\n" +
- "\tPRIMARY KEY (key, cols))\n" +
- "\tWITH ID = " + cfs.metadata.cfId + "\n" +
- "\tAND COMPACT STORAGE"));
- }
-
- @Test
- public void superColumnFamilyTest() throws Throwable
- {
- final String KEYSPACE = "thrift_compact_table_with_supercolumns_test";
- final String TABLE = "test_table_1";
-
- CFMetaData cfm = CFMetaData.Builder.createSuper(KEYSPACE, TABLE, false)
- .addPartitionKey("key", BytesType.instance)
- .addClusteringColumn("column1", AsciiType.instance)
- .addRegularColumn("", MapType.getInstance(Int32Type.instance, AsciiType.instance, true))
- .build();
-
- SchemaLoader.createKeyspace(KEYSPACE,
- KeyspaceParams.simple(1),
- cfm);
-
- ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(TABLE);
-
- assertTrue(ColumnFamilyStoreCQLHelper.getCFMetadataAsCQL(cfs.metadata, true).startsWith(
- "/*\n" +
- "Warning: Table " + KEYSPACE + "." + TABLE + " omitted because it has constructs not compatible with CQL (was created via legacy API).\n\n" +
- "Approximate structure, for reference:\n" +
- "(this should not be used to reproduce this schema)\n\n" +
- "CREATE TABLE IF NOT EXISTS " + KEYSPACE + "." + TABLE + " (\n" +
- "\tkey blob,\n" +
- "\tcolumn1 ascii,\n" +
- "\t\"\" map<int, ascii>,\n" +
- "\tPRIMARY KEY (key, column1))\n" +
- "\tWITH ID = " + cfs.metadata.cfId + "\n" +
- "\tAND COMPACT STORAGE"));
- }
-}
diff --git a/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java b/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java
index a3564bb..c3ea0a0 100644
--- a/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java
+++ b/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java
@@ -39,22 +39,20 @@
import com.google.common.collect.Iterators;
import org.apache.cassandra.*;
-import org.apache.cassandra.config.*;
import org.apache.cassandra.cql3.Operator;
import org.apache.cassandra.db.lifecycle.SSTableSet;
import org.apache.cassandra.db.rows.*;
import org.apache.cassandra.db.partitions.*;
-import org.apache.cassandra.db.marshal.*;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.io.sstable.Component;
import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.io.sstable.format.SSTableFormat;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.metrics.ClearableHistogram;
+import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.schema.KeyspaceParams;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FBUtilities;
-import org.apache.cassandra.utils.Pair;
import org.apache.cassandra.utils.WrappedRunnable;
import static junit.framework.Assert.assertNotNull;
@@ -103,14 +101,14 @@
Keyspace keyspace = Keyspace.open(KEYSPACE1);
ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF_STANDARD1);
- new RowUpdateBuilder(cfs.metadata, 0, "key1")
+ new RowUpdateBuilder(cfs.metadata(), 0, "key1")
.clustering("Column1")
.add("val", "asdf")
.build()
.applyUnsafe();
cfs.forceBlockingFlush();
- new RowUpdateBuilder(cfs.metadata, 1, "key1")
+ new RowUpdateBuilder(cfs.metadata(), 1, "key1")
.clustering("Column1")
.add("val", "asdf")
.build()
@@ -129,7 +127,7 @@
ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF_STANDARD1);
List<Mutation> rms = new LinkedList<>();
- rms.add(new RowUpdateBuilder(cfs.metadata, 0, "key1")
+ rms.add(new RowUpdateBuilder(cfs.metadata(), 0, "key1")
.clustering("Column1")
.add("val", "asdf")
.build());
@@ -148,7 +146,7 @@
Keyspace keyspace = Keyspace.open(KEYSPACE1);
final ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF_STANDARD2);
- RowUpdateBuilder.deleteRow(cfs.metadata, FBUtilities.timestampMicros(), "key1", "Column1").applyUnsafe();
+ RowUpdateBuilder.deleteRow(cfs.metadata(), FBUtilities.timestampMicros(), "key1", "Column1").applyUnsafe();
Runnable r = new WrappedRunnable()
{
@@ -163,93 +161,6 @@
reTest(cfs, r);
}
- // TODO: Implement this once we have hooks to super columns available in CQL context
-// @Test
-// public void testDeleteSuperRowSticksAfterFlush() throws Throwable
-// {
-// String keyspaceName = KEYSPACE1;
-// String cfName= CF_SUPER1;
-//
-// Keyspace keyspace = Keyspace.open(keyspaceName);
-// ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfName);
-//
-// ByteBuffer scfName = ByteBufferUtil.bytes("SuperDuper");
-// DecoratedKey key = Util.dk("flush-resurrection");
-//
-// // create an isolated sstable.
-// putColSuper(cfs, key, 0, ByteBufferUtil.bytes("val"), ByteBufferUtil.bytes(1L), ByteBufferUtil.bytes(1L), ByteBufferUtil.bytes("val1"));
-
-// putColsSuper(cfs, key, scfName,
-// new BufferCell(cellname(1L), ByteBufferUtil.bytes("val1"), 1),
-// new BufferCell(cellname(2L), ByteBufferUtil.bytes("val2"), 1),
-// new BufferCell(cellname(3L), ByteBufferUtil.bytes("val3"), 1));
-// cfs.forceBlockingFlush();
-//
-// // insert, don't flush.
-// putColsSuper(cfs, key, scfName,
-// new BufferCell(cellname(4L), ByteBufferUtil.bytes("val4"), 1),
-// new BufferCell(cellname(5L), ByteBufferUtil.bytes("val5"), 1),
-// new BufferCell(cellname(6L), ByteBufferUtil.bytes("val6"), 1));
-//
-// // verify insert.
-// final SlicePredicate sp = new SlicePredicate();
-// sp.setSlice_range(new SliceRange());
-// sp.getSlice_range().setCount(100);
-// sp.getSlice_range().setStart(ArrayUtils.EMPTY_BYTE_ARRAY);
-// sp.getSlice_range().setFinish(ArrayUtils.EMPTY_BYTE_ARRAY);
-//
-// assertRowAndColCount(1, 6, false, cfs.getRangeSlice(Util.range("f", "g"), null, ThriftValidation.asIFilter(sp, cfs.metadata, scfName), 100));
-//
-// // delete
-// Mutation rm = new Mutation(keyspace.getName(), key.getKey());
-// rm.deleteRange(cfName, SuperColumns.startOf(scfName), SuperColumns.endOf(scfName), 2);
-// rm.applyUnsafe();
-//
-// // verify delete.
-// assertRowAndColCount(1, 0, false, cfs.getRangeSlice(Util.range("f", "g"), null, ThriftValidation.asIFilter(sp, cfs.metadata, scfName), 100));
-//
-// // flush
-// cfs.forceBlockingFlush();
-//
-// // re-verify delete.
-// assertRowAndColCount(1, 0, false, cfs.getRangeSlice(Util.range("f", "g"), null, ThriftValidation.asIFilter(sp, cfs.metadata, scfName), 100));
-//
-// // late insert.
-// putColsSuper(cfs, key, scfName,
-// new BufferCell(cellname(4L), ByteBufferUtil.bytes("val4"), 1L),
-// new BufferCell(cellname(7L), ByteBufferUtil.bytes("val7"), 1L));
-//
-// // re-verify delete.
-// assertRowAndColCount(1, 0, false, cfs.getRangeSlice(Util.range("f", "g"), null, ThriftValidation.asIFilter(sp, cfs.metadata, scfName), 100));
-//
-// // make sure new writes are recognized.
-// putColsSuper(cfs, key, scfName,
-// new BufferCell(cellname(3L), ByteBufferUtil.bytes("val3"), 3),
-// new BufferCell(cellname(8L), ByteBufferUtil.bytes("val8"), 3),
-// new BufferCell(cellname(9L), ByteBufferUtil.bytes("val9"), 3));
-// assertRowAndColCount(1, 3, false, cfs.getRangeSlice(Util.range("f", "g"), null, ThriftValidation.asIFilter(sp, cfs.metadata, scfName), 100));
-// }
-
-// private static void assertRowAndColCount(int rowCount, int colCount, boolean isDeleted, Collection<Row> rows) throws CharacterCodingException
-// {
-// assert rows.size() == rowCount : "rowcount " + rows.size();
-// for (Row row : rows)
-// {
-// assert row.cf != null : "cf was null";
-// assert row.cf.getColumnCount() == colCount : "colcount " + row.cf.getColumnCount() + "|" + str(row.cf);
-// if (isDeleted)
-// assert row.cf.isMarkedForDelete() : "cf not marked for delete";
-// }
-// }
-//
-// private static String str(ColumnFamily cf) throws CharacterCodingException
-// {
-// StringBuilder sb = new StringBuilder();
-// for (Cell col : cf.getSortedColumns())
-// sb.append(String.format("(%s,%s,%d),", ByteBufferUtil.string(col.name().toByteBuffer()), ByteBufferUtil.string(col.value()), col.timestamp()));
-// return sb.toString();
-// }
-
@Test
public void testDeleteStandardRowSticksAfterFlush() throws Throwable
{
@@ -263,22 +174,24 @@
ByteBuffer val = ByteBufferUtil.bytes("val1");
// insert
- ColumnDefinition newCol = ColumnDefinition.regularDef(cfs.metadata, ByteBufferUtil.bytes("val2"), AsciiType.instance);
- new RowUpdateBuilder(cfs.metadata, 0, "key1").clustering("Column1").add("val", "val1").build().applyUnsafe();
- new RowUpdateBuilder(cfs.metadata, 0, "key2").clustering("Column1").add("val", "val1").build().applyUnsafe();
+ Mutation.SimpleBuilder builder = Mutation.simpleBuilder(keyspaceName, cfs.metadata().partitioner.decorateKey(ByteBufferUtil.bytes("val2")));
+ builder.update(cfName).row("Column1").add("val", "val1").build();
+
+ new RowUpdateBuilder(cfs.metadata(), 0, "key1").clustering("Column1").add("val", "val1").build().applyUnsafe();
+ new RowUpdateBuilder(cfs.metadata(), 0, "key2").clustering("Column1").add("val", "val1").build().applyUnsafe();
assertRangeCount(cfs, col, val, 2);
// flush.
cfs.forceBlockingFlush();
// insert, don't flush
- new RowUpdateBuilder(cfs.metadata, 1, "key3").clustering("Column1").add("val", "val1").build().applyUnsafe();
- new RowUpdateBuilder(cfs.metadata, 1, "key4").clustering("Column1").add("val", "val1").build().applyUnsafe();
+ new RowUpdateBuilder(cfs.metadata(), 1, "key3").clustering("Column1").add("val", "val1").build().applyUnsafe();
+ new RowUpdateBuilder(cfs.metadata(), 1, "key4").clustering("Column1").add("val", "val1").build().applyUnsafe();
assertRangeCount(cfs, col, val, 4);
// delete (from sstable and memtable)
- RowUpdateBuilder.deleteRow(cfs.metadata, 5, "key1", "Column1").applyUnsafe();
- RowUpdateBuilder.deleteRow(cfs.metadata, 5, "key3", "Column1").applyUnsafe();
+ RowUpdateBuilder.deleteRow(cfs.metadata(), 5, "key1", "Column1").applyUnsafe();
+ RowUpdateBuilder.deleteRow(cfs.metadata(), 5, "key3", "Column1").applyUnsafe();
// verify delete
assertRangeCount(cfs, col, val, 2);
@@ -290,15 +203,15 @@
assertRangeCount(cfs, col, val, 2);
// simulate a 'late' insertion that gets put in after the deletion. should get inserted, but fail on read.
- new RowUpdateBuilder(cfs.metadata, 2, "key1").clustering("Column1").add("val", "val1").build().applyUnsafe();
- new RowUpdateBuilder(cfs.metadata, 2, "key3").clustering("Column1").add("val", "val1").build().applyUnsafe();
+ new RowUpdateBuilder(cfs.metadata(), 2, "key1").clustering("Column1").add("val", "val1").build().applyUnsafe();
+ new RowUpdateBuilder(cfs.metadata(), 2, "key3").clustering("Column1").add("val", "val1").build().applyUnsafe();
// should still be nothing there because we deleted this row. 2nd breakage, but was undetected because of 1837.
assertRangeCount(cfs, col, val, 2);
// make sure that new writes are recognized.
- new RowUpdateBuilder(cfs.metadata, 10, "key5").clustering("Column1").add("val", "val1").build().applyUnsafe();
- new RowUpdateBuilder(cfs.metadata, 10, "key6").clustering("Column1").add("val", "val1").build().applyUnsafe();
+ new RowUpdateBuilder(cfs.metadata(), 10, "key5").clustering("Column1").add("val", "val1").build().applyUnsafe();
+ new RowUpdateBuilder(cfs.metadata(), 10, "key6").clustering("Column1").add("val", "val1").build().applyUnsafe();
assertRangeCount(cfs, col, val, 4);
// and it remains so after flush. (this wasn't failing before, but it's good to check.)
@@ -331,7 +244,7 @@
cfs.snapshot("nonEphemeralSnapshot", null, false, false);
cfs.snapshot("ephemeralSnapshot", null, true, false);
- Map<String, Pair<Long, Long>> snapshotDetails = cfs.getSnapshotDetails();
+ Map<String, Directories.SnapshotSizeDetails> snapshotDetails = cfs.getSnapshotDetails();
assertEquals(2, snapshotDetails.size());
assertTrue(snapshotDetails.containsKey("ephemeralSnapshot"));
assertTrue(snapshotDetails.containsKey("nonEphemeralSnapshot"));
@@ -350,9 +263,9 @@
public void testBackupAfterFlush() throws Throwable
{
ColumnFamilyStore cfs = Keyspace.open(KEYSPACE2).getColumnFamilyStore(CF_STANDARD1);
- new RowUpdateBuilder(cfs.metadata, 0, ByteBufferUtil.bytes("key1")).clustering("Column1").add("val", "asdf").build().applyUnsafe();
+ new RowUpdateBuilder(cfs.metadata(), 0, ByteBufferUtil.bytes("key1")).clustering("Column1").add("val", "asdf").build().applyUnsafe();
cfs.forceBlockingFlush();
- new RowUpdateBuilder(cfs.metadata, 0, ByteBufferUtil.bytes("key2")).clustering("Column1").add("val", "asdf").build().applyUnsafe();
+ new RowUpdateBuilder(cfs.metadata(), 0, ByteBufferUtil.bytes("key2")).clustering("Column1").add("val", "asdf").build().applyUnsafe();
cfs.forceBlockingFlush();
for (int version = 1; version <= 2; ++version)
@@ -412,7 +325,7 @@
// ColumnFamilyStore cfs = Keyspace.open(ks).getColumnFamilyStore(cf);
// SSTableDeletingTask.waitForDeletions();
//
-// final CFMetaData cfmeta = Schema.instance.getCFMetaData(ks, cf);
+// final CFMetaData cfmeta = Schema.instance.getTableMetadataRef(ks, cf);
// Directories dir = new Directories(cfs.metadata);
//
// // clear old SSTables (probably left by CFS.clearUnsafe() calls in other tests)
@@ -498,10 +411,10 @@
private void assertRangeCount(ColumnFamilyStore cfs, ByteBuffer col, ByteBuffer val, int count)
{
- assertRangeCount(cfs, cfs.metadata.getColumnDefinition(col), val, count);
+ assertRangeCount(cfs, cfs.metadata().getColumn(col), val, count);
}
- private void assertRangeCount(ColumnFamilyStore cfs, ColumnDefinition col, ByteBuffer val, int count)
+ private void assertRangeCount(ColumnFamilyStore cfs, ColumnMetadata col, ByteBuffer val, int count)
{
int found = 0;
@@ -526,7 +439,7 @@
ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF_INDEX1);
cfs.truncateBlocking();
- UpdateBuilder builder = UpdateBuilder.create(cfs.metadata, "key")
+ UpdateBuilder builder = UpdateBuilder.create(cfs.metadata.get(), "key")
.newRow()
.add("birthdate", 1L)
.add("notbirthdate", 2L);
@@ -557,9 +470,9 @@
{
ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD1);
- ColumnFamilyStore.scrubDataDirectories(cfs.metadata);
+ ColumnFamilyStore.scrubDataDirectories(cfs.metadata());
- new RowUpdateBuilder(cfs.metadata, 2, "key").clustering("name").add("val", "2").build().applyUnsafe();
+ new RowUpdateBuilder(cfs.metadata(), 2, "key").clustering("name").add("val", "2").build().applyUnsafe();
cfs.forceBlockingFlush();
// Nuke the metadata and reload that sstable
@@ -573,9 +486,9 @@
ssTable.selfRef().release();
- ColumnFamilyStore.scrubDataDirectories(cfs.metadata);
+ ColumnFamilyStore.scrubDataDirectories(cfs.metadata());
- List<File> ssTableFiles = new Directories(cfs.metadata).sstableLister(Directories.OnTxnErr.THROW).listFiles();
+ List<File> ssTableFiles = new Directories(cfs.metadata()).sstableLister(Directories.OnTxnErr.THROW).listFiles();
assertNotNull(ssTableFiles);
assertEquals(0, ssTableFiles.size());
}
diff --git a/test/unit/org/apache/cassandra/db/ColumnsTest.java b/test/unit/org/apache/cassandra/db/ColumnsTest.java
index 1f34c88..6dc1832 100644
--- a/test/unit/org/apache/cassandra/db/ColumnsTest.java
+++ b/test/unit/org/apache/cassandra/db/ColumnsTest.java
@@ -28,20 +28,21 @@
import com.google.common.collect.Lists;
import org.apache.cassandra.cql3.ColumnIdentifier;
+import org.apache.cassandra.db.commitlog.CommitLog;
import org.apache.cassandra.db.marshal.BytesType;
import org.junit.AfterClass;
import org.junit.Test;
-import junit.framework.Assert;
-import org.apache.cassandra.MockSchema;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.junit.Assert;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.db.marshal.SetType;
import org.apache.cassandra.db.marshal.UTF8Type;
import org.apache.cassandra.io.util.DataInputBuffer;
import org.apache.cassandra.io.util.DataOutputBuffer;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.MockSchema;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.utils.btree.BTreeSet;
import static org.apache.cassandra.utils.ByteBufferUtil.bytes;
@@ -51,16 +52,17 @@
static
{
DatabaseDescriptor.daemonInitialization();
+ CommitLog.instance.start();
}
- private static final CFMetaData cfMetaData = MockSchema.newCFS().metadata;
+ private static final TableMetadata TABLE_METADATA = MockSchema.newCFS().metadata();
@Test
public void testDeserializeCorruption() throws IOException
{
ColumnsCheck check = randomSmall(1, 0, 3, 0);
Columns superset = check.columns;
- List<ColumnDefinition> minus1 = new ArrayList<>(check.definitions);
+ List<ColumnMetadata> minus1 = new ArrayList<>(check.definitions);
minus1.remove(3);
Columns minus2 = check.columns
.without(check.columns.getSimple(3))
@@ -93,8 +95,8 @@
{
// pick some arbitrary groupings of columns to remove at-once (to avoid factorial complexity)
// whatever is left after each removal, we perform this logic on again, recursively
- List<List<ColumnDefinition>> removeGroups = shuffleAndGroup(Lists.newArrayList(input.definitions));
- for (List<ColumnDefinition> defs : removeGroups)
+ List<List<ColumnMetadata>> removeGroups = shuffleAndGroup(Lists.newArrayList(input.definitions));
+ for (List<ColumnMetadata> defs : removeGroups)
{
ColumnsCheck subset = input.remove(defs);
@@ -106,7 +108,7 @@
// test .mergeTo
Columns otherSubset = input.columns;
- for (ColumnDefinition def : subset.definitions)
+ for (ColumnMetadata def : subset.definitions)
{
otherSubset = otherSubset.without(def);
assertContents(otherSubset.mergeTo(subset.columns), input.definitions);
@@ -131,7 +133,7 @@
testSerialize(randomColumns.columns, randomColumns.definitions);
}
- private void testSerialize(Columns columns, List<ColumnDefinition> definitions) throws IOException
+ private void testSerialize(Columns columns, List<ColumnMetadata> definitions) throws IOException
{
try (DataOutputBuffer out = new DataOutputBuffer())
{
@@ -165,7 +167,7 @@
for (int i = 0; i < 50; i++)
names.add("regular_" + i);
- List<ColumnDefinition> defs = new ArrayList<>();
+ List<ColumnMetadata> defs = new ArrayList<>();
addRegular(names, defs);
Columns columns = Columns.from(new HashSet<>(defs));
@@ -181,18 +183,18 @@
@Test
public void testStaticColumns()
{
- testColumns(ColumnDefinition.Kind.STATIC);
+ testColumns(ColumnMetadata.Kind.STATIC);
}
@Test
public void testRegularColumns()
{
- testColumns(ColumnDefinition.Kind.REGULAR);
+ testColumns(ColumnMetadata.Kind.REGULAR);
}
- private void testColumns(ColumnDefinition.Kind kind)
+ private void testColumns(ColumnMetadata.Kind kind)
{
- List<ColumnDefinition> definitions = ImmutableList.of(
+ List<ColumnMetadata> definitions = ImmutableList.of(
def("a", UTF8Type.instance, kind),
def("b", SetType.getInstance(UTF8Type.instance, true), kind),
def("c", UTF8Type.instance, kind),
@@ -208,9 +210,9 @@
Assert.assertEquals(4, columns.simpleColumnCount());
// test simpleColumns()
- List<ColumnDefinition> simpleColumnsExpected =
+ List<ColumnMetadata> simpleColumnsExpected =
ImmutableList.of(definitions.get(0), definitions.get(2), definitions.get(4), definitions.get(6));
- List<ColumnDefinition> simpleColumnsActual = new ArrayList<>();
+ List<ColumnMetadata> simpleColumnsActual = new ArrayList<>();
Iterators.addAll(simpleColumnsActual, columns.simpleColumns());
Assert.assertEquals(simpleColumnsExpected, simpleColumnsActual);
@@ -218,9 +220,9 @@
Assert.assertEquals(4, columns.complexColumnCount());
// test complexColumns()
- List<ColumnDefinition> complexColumnsExpected =
+ List<ColumnMetadata> complexColumnsExpected =
ImmutableList.of(definitions.get(1), definitions.get(3), definitions.get(5), definitions.get(7));
- List<ColumnDefinition> complexColumnsActual = new ArrayList<>();
+ List<ColumnMetadata> complexColumnsActual = new ArrayList<>();
Iterators.addAll(complexColumnsActual, columns.complexColumns());
Assert.assertEquals(complexColumnsExpected, complexColumnsActual);
@@ -228,8 +230,8 @@
Assert.assertEquals(8, columns.size());
// test selectOrderIterator()
- List<ColumnDefinition> columnsExpected = definitions;
- List<ColumnDefinition> columnsActual = new ArrayList<>();
+ List<ColumnMetadata> columnsExpected = definitions;
+ List<ColumnMetadata> columnsActual = new ArrayList<>();
Iterators.addAll(columnsActual, columns.selectOrderIterator());
Assert.assertEquals(columnsExpected, columnsActual);
}
@@ -238,8 +240,8 @@
{
testSerializeSubset(input.columns, input.columns, input.definitions);
testSerializeSubset(input.columns, Columns.NONE, Collections.emptyList());
- List<List<ColumnDefinition>> removeGroups = shuffleAndGroup(Lists.newArrayList(input.definitions));
- for (List<ColumnDefinition> defs : removeGroups)
+ List<List<ColumnMetadata>> removeGroups = shuffleAndGroup(Lists.newArrayList(input.definitions));
+ for (List<ColumnMetadata> defs : removeGroups)
{
Collections.sort(defs);
ColumnsCheck subset = input.remove(defs);
@@ -247,7 +249,7 @@
}
}
- private void testSerializeSubset(Columns superset, Columns subset, List<ColumnDefinition> subsetDefinitions) throws IOException
+ private void testSerializeSubset(Columns superset, Columns subset, List<ColumnMetadata> subsetDefinitions) throws IOException
{
try (DataOutputBuffer out = new DataOutputBuffer())
{
@@ -260,17 +262,17 @@
}
}
- private static void assertContents(Columns columns, List<ColumnDefinition> defs)
+ private static void assertContents(Columns columns, List<ColumnMetadata> defs)
{
Assert.assertEquals(defs, Lists.newArrayList(columns));
boolean hasSimple = false, hasComplex = false;
int firstComplexIdx = 0;
int i = 0;
- Iterator<ColumnDefinition> simple = columns.simpleColumns();
- Iterator<ColumnDefinition> complex = columns.complexColumns();
- Iterator<ColumnDefinition> all = columns.iterator();
- Predicate<ColumnDefinition> predicate = columns.inOrderInclusionTester();
- for (ColumnDefinition def : defs)
+ Iterator<ColumnMetadata> simple = columns.simpleColumns();
+ Iterator<ColumnMetadata> complex = columns.complexColumns();
+ Iterator<ColumnMetadata> all = columns.iterator();
+ Predicate<ColumnMetadata> predicate = columns.inOrderInclusionTester();
+ for (ColumnMetadata def : defs)
{
Assert.assertEquals(def, all.next());
Assert.assertTrue(columns.contains(def));
@@ -303,9 +305,9 @@
// check select order
if (!columns.hasSimple() || !columns.getSimple(0).kind.isPrimaryKeyKind())
{
- List<ColumnDefinition> selectOrderDefs = new ArrayList<>(defs);
+ List<ColumnMetadata> selectOrderDefs = new ArrayList<>(defs);
Collections.sort(selectOrderDefs, (a, b) -> a.name.bytes.compareTo(b.name.bytes));
- List<ColumnDefinition> selectOrderColumns = new ArrayList<>();
+ List<ColumnMetadata> selectOrderColumns = new ArrayList<>();
Iterators.addAll(selectOrderColumns, columns.selectOrderIterator());
Assert.assertEquals(selectOrderDefs, selectOrderColumns);
}
@@ -347,27 +349,27 @@
private static class ColumnsCheck
{
final Columns columns;
- final List<ColumnDefinition> definitions;
+ final List<ColumnMetadata> definitions;
- private ColumnsCheck(Columns columns, List<ColumnDefinition> definitions)
+ private ColumnsCheck(Columns columns, List<ColumnMetadata> definitions)
{
this.columns = columns;
this.definitions = definitions;
}
- private ColumnsCheck(List<ColumnDefinition> definitions)
+ private ColumnsCheck(List<ColumnMetadata> definitions)
{
this.columns = Columns.from(BTreeSet.of(definitions));
this.definitions = definitions;
}
- ColumnsCheck remove(List<ColumnDefinition> remove)
+ ColumnsCheck remove(List<ColumnMetadata> remove)
{
Columns subset = columns;
- for (ColumnDefinition def : remove)
+ for (ColumnMetadata def : remove)
subset = subset.without(def);
Assert.assertEquals(columns.size() - remove.size(), subset.size());
- List<ColumnDefinition> remainingDefs = Lists.newArrayList(columns);
+ List<ColumnMetadata> remainingDefs = Lists.newArrayList(columns);
remainingDefs.removeAll(remove);
return new ColumnsCheck(subset, remainingDefs);
}
@@ -417,7 +419,7 @@
for (char c = 'a' ; c <= 'z' ; c++)
names .add(Character.toString(c));
- List<ColumnDefinition> result = new ArrayList<>();
+ List<ColumnMetadata> result = new ArrayList<>();
addPartition(select(names, pkCount), result);
addClustering(select(names, clCount), result);
addRegular(select(names, regularCount), result);
@@ -441,7 +443,7 @@
private static ColumnsCheck randomHuge(int pkCount, int clCount, int regularCount, int complexCount)
{
- List<ColumnDefinition> result = new ArrayList<>();
+ List<ColumnMetadata> result = new ArrayList<>();
Set<String> usedNames = new HashSet<>();
addPartition(names(pkCount, usedNames), result);
addClustering(names(clCount, usedNames), result);
@@ -468,48 +470,49 @@
return names;
}
- private static void addPartition(List<String> names, List<ColumnDefinition> results)
+ private static void addPartition(List<String> names, List<ColumnMetadata> results)
{
for (String name : names)
- results.add(ColumnDefinition.partitionKeyDef(cfMetaData, bytes(name), UTF8Type.instance, 0));
+ results.add(ColumnMetadata.partitionKeyColumn(TABLE_METADATA, bytes(name), UTF8Type.instance, 0));
}
- private static void addClustering(List<String> names, List<ColumnDefinition> results)
+ private static void addClustering(List<String> names, List<ColumnMetadata> results)
{
int i = 0;
for (String name : names)
- results.add(ColumnDefinition.clusteringDef(cfMetaData, bytes(name), UTF8Type.instance, i++));
+ results.add(ColumnMetadata.clusteringColumn(TABLE_METADATA, bytes(name), UTF8Type.instance, i++));
}
- private static void addRegular(List<String> names, List<ColumnDefinition> results)
+ private static void addRegular(List<String> names, List<ColumnMetadata> results)
{
for (String name : names)
- results.add(ColumnDefinition.regularDef(cfMetaData, bytes(name), UTF8Type.instance));
+ results.add(ColumnMetadata.regularColumn(TABLE_METADATA, bytes(name), UTF8Type.instance));
}
- private static void addComplex(List<String> names, List<ColumnDefinition> results)
+ private static void addComplex(List<String> names, List<ColumnMetadata> results)
{
for (String name : names)
- results.add(ColumnDefinition.regularDef(cfMetaData, bytes(name), SetType.getInstance(UTF8Type.instance, true)));
+ results.add(ColumnMetadata.regularColumn(TABLE_METADATA, bytes(name), SetType.getInstance(UTF8Type.instance, true)));
}
- private static ColumnDefinition def(String name, AbstractType<?> type, ColumnDefinition.Kind kind)
+ private static ColumnMetadata def(String name, AbstractType<?> type, ColumnMetadata.Kind kind)
{
- return new ColumnDefinition(cfMetaData, bytes(name), type, ColumnDefinition.NO_POSITION, kind);
+ return new ColumnMetadata(TABLE_METADATA, bytes(name), type, ColumnMetadata.NO_POSITION, kind);
}
- private static CFMetaData mock(Columns columns)
+ private static TableMetadata mock(Columns columns)
{
if (columns.isEmpty())
- return cfMetaData;
- CFMetaData.Builder builder = CFMetaData.Builder.create(cfMetaData.ksName, cfMetaData.cfName);
+ return TABLE_METADATA;
+
+ TableMetadata.Builder builder = TableMetadata.builder(TABLE_METADATA.keyspace, TABLE_METADATA.name);
boolean hasPartitionKey = false;
- for (ColumnDefinition def : columns)
+ for (ColumnMetadata def : columns)
{
switch (def.kind)
{
case PARTITION_KEY:
- builder.addPartitionKey(def.name, def.type);
+ builder.addPartitionKeyColumn(def.name, def.type);
hasPartitionKey = true;
break;
case CLUSTERING:
@@ -521,7 +524,7 @@
}
}
if (!hasPartitionKey)
- builder.addPartitionKey("219894021498309239rufejsfjdksfjheiwfhjes", UTF8Type.instance);
+ builder.addPartitionKeyColumn("219894021498309239rufejsfjdksfjheiwfhjes", UTF8Type.instance);
return builder.build();
}
}
diff --git a/test/unit/org/apache/cassandra/db/CounterCacheTest.java b/test/unit/org/apache/cassandra/db/CounterCacheTest.java
index 4cfd848..a913133 100644
--- a/test/unit/org/apache/cassandra/db/CounterCacheTest.java
+++ b/test/unit/org/apache/cassandra/db/CounterCacheTest.java
@@ -20,8 +20,8 @@
import java.util.Collections;
import java.util.concurrent.ExecutionException;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.dht.Bounds;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.schema.KeyspaceParams;
@@ -33,7 +33,7 @@
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.db.marshal.*;
import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.config.Schema;
+import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.exceptions.WriteTimeoutException;
import org.apache.cassandra.service.CacheService;
@@ -52,15 +52,14 @@
{
SchemaLoader.prepareServer();
- CFMetaData counterTable = CFMetaData.Builder.create(KEYSPACE1, COUNTER1, false, true, true)
- .addPartitionKey("key", Int32Type.instance)
- .addClusteringColumn("name", Int32Type.instance)
- .addRegularColumn("c", CounterColumnType.instance)
- .build();
+ TableMetadata.Builder counterTable =
+ TableMetadata.builder(KEYSPACE1, COUNTER1)
+ .isCounter(true)
+ .addPartitionKeyColumn("key", Int32Type.instance)
+ .addClusteringColumn("name", Int32Type.instance)
+ .addRegularColumn("c", CounterColumnType.instance);
- SchemaLoader.createKeyspace(KEYSPACE1,
- KeyspaceParams.simple(1),
- counterTable);
+ SchemaLoader.createKeyspace(KEYSPACE1, KeyspaceParams.simple(1), counterTable);
}
@AfterClass
@@ -76,9 +75,9 @@
cfs.truncateBlocking();
CacheService.instance.invalidateCounterCache();
- Clustering c1 = CBuilder.create(cfs.metadata.comparator).add(ByteBufferUtil.bytes(1)).build();
- Clustering c2 = CBuilder.create(cfs.metadata.comparator).add(ByteBufferUtil.bytes(2)).build();
- ColumnDefinition cd = cfs.metadata.getColumnDefinition(ByteBufferUtil.bytes("c"));
+ Clustering c1 = CBuilder.create(cfs.metadata().comparator).add(ByteBufferUtil.bytes(1)).build();
+ Clustering c2 = CBuilder.create(cfs.metadata().comparator).add(ByteBufferUtil.bytes(2)).build();
+ ColumnMetadata cd = cfs.metadata().getColumn(ByteBufferUtil.bytes("c"));
assertEquals(0, CacheService.instance.counterCache.size());
assertNull(cfs.getCachedCounter(bytes(1), c1, cd, null));
@@ -104,9 +103,9 @@
cfs.truncateBlocking();
CacheService.instance.invalidateCounterCache();
- Clustering c1 = CBuilder.create(cfs.metadata.comparator).add(ByteBufferUtil.bytes(1)).build();
- Clustering c2 = CBuilder.create(cfs.metadata.comparator).add(ByteBufferUtil.bytes(2)).build();
- ColumnDefinition cd = cfs.metadata.getColumnDefinition(ByteBufferUtil.bytes("c"));
+ Clustering c1 = CBuilder.create(cfs.metadata().comparator).add(ByteBufferUtil.bytes(1)).build();
+ Clustering c2 = CBuilder.create(cfs.metadata().comparator).add(ByteBufferUtil.bytes(2)).build();
+ ColumnMetadata cd = cfs.metadata().getColumn(ByteBufferUtil.bytes("c"));
assertEquals(0, CacheService.instance.counterCache.size());
assertNull(cfs.getCachedCounter(bytes(1), c1, cd, null));
@@ -149,10 +148,10 @@
cfs.truncateBlocking();
CacheService.instance.invalidateCounterCache();
- new CounterMutation(new RowUpdateBuilder(cfs.metadata, 0, bytes(1)).clustering(1).add("c", 1L).build(), ConsistencyLevel.ONE).apply();
- new CounterMutation(new RowUpdateBuilder(cfs.metadata, 0, bytes(1)).clustering(2).add("c", 2L).build(), ConsistencyLevel.ONE).apply();
- new CounterMutation(new RowUpdateBuilder(cfs.metadata, 0, bytes(2)).clustering(1).add("c", 1L).build(), ConsistencyLevel.ONE).apply();
- new CounterMutation(new RowUpdateBuilder(cfs.metadata, 0, bytes(2)).clustering(2).add("c", 2L).build(), ConsistencyLevel.ONE).apply();
+ new CounterMutation(new RowUpdateBuilder(cfs.metadata(), 0, bytes(1)).clustering(1).add("c", 1L).build(), ConsistencyLevel.ONE).apply();
+ new CounterMutation(new RowUpdateBuilder(cfs.metadata(), 0, bytes(1)).clustering(2).add("c", 2L).build(), ConsistencyLevel.ONE).apply();
+ new CounterMutation(new RowUpdateBuilder(cfs.metadata(), 0, bytes(2)).clustering(1).add("c", 1L).build(), ConsistencyLevel.ONE).apply();
+ new CounterMutation(new RowUpdateBuilder(cfs.metadata(), 0, bytes(2)).clustering(2).add("c", 2L).build(), ConsistencyLevel.ONE).apply();
assertEquals(4, CacheService.instance.counterCache.size());
@@ -165,9 +164,9 @@
CacheService.instance.counterCache.loadSaved();
assertEquals(4, CacheService.instance.counterCache.size());
- Clustering c1 = CBuilder.create(cfs.metadata.comparator).add(ByteBufferUtil.bytes(1)).build();
- Clustering c2 = CBuilder.create(cfs.metadata.comparator).add(ByteBufferUtil.bytes(2)).build();
- ColumnDefinition cd = cfs.metadata.getColumnDefinition(ByteBufferUtil.bytes("c"));
+ Clustering c1 = CBuilder.create(cfs.metadata().comparator).add(ByteBufferUtil.bytes(1)).build();
+ Clustering c2 = CBuilder.create(cfs.metadata().comparator).add(ByteBufferUtil.bytes(2)).build();
+ ColumnMetadata cd = cfs.metadata().getColumn(ByteBufferUtil.bytes("c"));
assertEquals(1L, cfs.getCachedCounter(bytes(1), c1, cd, null).count);
assertEquals(2L, cfs.getCachedCounter(bytes(1), c2, cd, null).count);
@@ -182,10 +181,10 @@
cfs.truncateBlocking();
CacheService.instance.invalidateCounterCache();
- new CounterMutation(new RowUpdateBuilder(cfs.metadata, 0, bytes(1)).clustering(1).add("c", 1L).build(), ConsistencyLevel.ONE).apply();
- new CounterMutation(new RowUpdateBuilder(cfs.metadata, 0, bytes(1)).clustering(2).add("c", 2L).build(), ConsistencyLevel.ONE).apply();
- new CounterMutation(new RowUpdateBuilder(cfs.metadata, 0, bytes(2)).clustering(1).add("c", 1L).build(), ConsistencyLevel.ONE).apply();
- new CounterMutation(new RowUpdateBuilder(cfs.metadata, 0, bytes(2)).clustering(2).add("c", 2L).build(), ConsistencyLevel.ONE).apply();
+ new CounterMutation(new RowUpdateBuilder(cfs.metadata(), 0, bytes(1)).clustering(1).add("c", 1L).build(), ConsistencyLevel.ONE).apply();
+ new CounterMutation(new RowUpdateBuilder(cfs.metadata(), 0, bytes(1)).clustering(2).add("c", 2L).build(), ConsistencyLevel.ONE).apply();
+ new CounterMutation(new RowUpdateBuilder(cfs.metadata(), 0, bytes(2)).clustering(1).add("c", 1L).build(), ConsistencyLevel.ONE).apply();
+ new CounterMutation(new RowUpdateBuilder(cfs.metadata(), 0, bytes(2)).clustering(2).add("c", 2L).build(), ConsistencyLevel.ONE).apply();
// flush the counter cache and invalidate
CacheService.instance.counterCache.submitWrite(Integer.MAX_VALUE).get();
@@ -213,10 +212,10 @@
cfs.truncateBlocking();
CacheService.instance.invalidateCounterCache();
- new CounterMutation(new RowUpdateBuilder(cfs.metadata, 0, bytes(1)).clustering(1).add("c", 1L).build(), ConsistencyLevel.ONE).apply();
- new CounterMutation(new RowUpdateBuilder(cfs.metadata, 0, bytes(1)).clustering(2).add("c", 2L).build(), ConsistencyLevel.ONE).apply();
- new CounterMutation(new RowUpdateBuilder(cfs.metadata, 0, bytes(2)).clustering(1).add("c", 1L).build(), ConsistencyLevel.ONE).apply();
- new CounterMutation(new RowUpdateBuilder(cfs.metadata, 0, bytes(2)).clustering(2).add("c", 2L).build(), ConsistencyLevel.ONE).apply();
+ new CounterMutation(new RowUpdateBuilder(cfs.metadata(), 0, bytes(1)).clustering(1).add("c", 1L).build(), ConsistencyLevel.ONE).apply();
+ new CounterMutation(new RowUpdateBuilder(cfs.metadata(), 0, bytes(1)).clustering(2).add("c", 2L).build(), ConsistencyLevel.ONE).apply();
+ new CounterMutation(new RowUpdateBuilder(cfs.metadata(), 0, bytes(2)).clustering(1).add("c", 1L).build(), ConsistencyLevel.ONE).apply();
+ new CounterMutation(new RowUpdateBuilder(cfs.metadata(), 0, bytes(2)).clustering(2).add("c", 2L).build(), ConsistencyLevel.ONE).apply();
// flush the counter cache and invalidate
CacheService.instance.counterCache.submitWrite(Integer.MAX_VALUE).get();
diff --git a/test/unit/org/apache/cassandra/db/CounterCellTest.java b/test/unit/org/apache/cassandra/db/CounterCellTest.java
index 5208cb2..4ce9802 100644
--- a/test/unit/org/apache/cassandra/db/CounterCellTest.java
+++ b/test/unit/org/apache/cassandra/db/CounterCellTest.java
@@ -19,8 +19,6 @@
package org.apache.cassandra.db;
import java.nio.ByteBuffer;
-import java.security.MessageDigest;
-import java.util.Arrays;
import org.junit.AfterClass;
import org.junit.Assert;
@@ -28,7 +26,7 @@
import org.junit.Test;
import org.apache.cassandra.SchemaLoader;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.db.rows.BTreeRow;
import org.apache.cassandra.db.rows.BufferCell;
import org.apache.cassandra.db.rows.Cell;
@@ -101,27 +99,27 @@
private Cell createLegacyCounterCell(ColumnFamilyStore cfs, ByteBuffer colName, long count, long ts)
{
- ColumnDefinition cDef = cfs.metadata.getColumnDefinition(colName);
+ ColumnMetadata cDef = cfs.metadata().getColumn(colName);
ByteBuffer val = CounterContext.instance().createLocal(count);
return BufferCell.live(cDef, ts, val);
}
private Cell createCounterCell(ColumnFamilyStore cfs, ByteBuffer colName, CounterId id, long count, long ts)
{
- ColumnDefinition cDef = cfs.metadata.getColumnDefinition(colName);
+ ColumnMetadata cDef = cfs.metadata().getColumn(colName);
ByteBuffer val = CounterContext.instance().createGlobal(id, ts, count);
return BufferCell.live(cDef, ts, val);
}
private Cell createCounterCellFromContext(ColumnFamilyStore cfs, ByteBuffer colName, ContextState context, long ts)
{
- ColumnDefinition cDef = cfs.metadata.getColumnDefinition(colName);
+ ColumnMetadata cDef = cfs.metadata().getColumn(colName);
return BufferCell.live(cDef, ts, context.context);
}
private Cell createDeleted(ColumnFamilyStore cfs, ByteBuffer colName, long ts, int localDeletionTime)
{
- ColumnDefinition cDef = cfs.metadata.getColumnDefinition(colName);
+ ColumnMetadata cDef = cfs.metadata().getColumn(colName);
return BufferCell.tombstone(cDef, ts, localDeletionTime);
}
@@ -137,43 +135,43 @@
// both deleted, diff deletion time, same ts
left = createDeleted(cfs, col, 2, 5);
right = createDeleted(cfs, col, 2, 10);
- assert Cells.reconcile(left, right, 10) == right;
+ assert Cells.reconcile(left, right) == right;
// diff ts
right = createLegacyCounterCell(cfs, col, 1, 10);
- assert Cells.reconcile(left, right, 10) == left;
+ assert Cells.reconcile(left, right) == left;
// < tombstone
left = createDeleted(cfs, col, 6, 6);
right = createLegacyCounterCell(cfs, col, 1, 5);
- assert Cells.reconcile(left, right, 10) == left;
+ assert Cells.reconcile(left, right) == left;
// > tombstone
left = createDeleted(cfs, col, 1, 1);
right = createLegacyCounterCell(cfs, col, 1, 5);
- assert Cells.reconcile(left, right, 10) == left;
+ assert Cells.reconcile(left, right) == left;
// == tombstone
left = createDeleted(cfs, col, 8, 8);
right = createLegacyCounterCell(cfs, col, 1, 8);
- assert Cells.reconcile(left, right, 10) == left;
+ assert Cells.reconcile(left, right) == left;
// live + live
left = createLegacyCounterCell(cfs, col, 1, 2);
right = createLegacyCounterCell(cfs, col, 3, 5);
- Cell reconciled = Cells.reconcile(left, right, 10);
+ Cell reconciled = Cells.reconcile(left, right);
assertEquals(CounterContext.instance().total(reconciled.value()), 4);
assertEquals(reconciled.timestamp(), 5L);
// Add, don't change TS
Cell addTen = createLegacyCounterCell(cfs, col, 10, 4);
- reconciled = Cells.reconcile(reconciled, addTen, 10);
+ reconciled = Cells.reconcile(reconciled, addTen);
assertEquals(CounterContext.instance().total(reconciled.value()), 14);
assertEquals(reconciled.timestamp(), 5L);
// Add w/new TS
Cell addThree = createLegacyCounterCell(cfs, col, 3, 7);
- reconciled = Cells.reconcile(reconciled, addThree, 10);
+ reconciled = Cells.reconcile(reconciled, addThree);
assertEquals(CounterContext.instance().total(reconciled.value()), 17);
assertEquals(reconciled.timestamp(), 7L);
@@ -181,7 +179,7 @@
assert reconciled.localDeletionTime() == Integer.MAX_VALUE;
Cell deleted = createDeleted(cfs, col, 8, 8);
- reconciled = Cells.reconcile(reconciled, deleted, 10);
+ reconciled = Cells.reconcile(reconciled, deleted);
assert reconciled.localDeletionTime() == 8;
}
@@ -264,8 +262,8 @@
ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(COUNTER1);
ByteBuffer col = ByteBufferUtil.bytes("val");
- MessageDigest digest1 = MessageDigest.getInstance("md5");
- MessageDigest digest2 = MessageDigest.getInstance("md5");
+ Digest digest1 = Digest.forReadResponse();
+ Digest digest2 = Digest.forReadResponse();
CounterContext.ContextState state = CounterContext.ContextState.allocate(0, 2, 2);
state.writeRemote(CounterId.fromInt(1), 4L, 4L);
@@ -275,13 +273,13 @@
Cell original = createCounterCellFromContext(cfs, col, state, 5);
- ColumnDefinition cDef = cfs.metadata.getColumnDefinition(col);
+ ColumnMetadata cDef = cfs.metadata().getColumn(col);
Cell cleared = BufferCell.live(cDef, 5, CounterContext.instance().clearAllLocal(state.context));
original.digest(digest1);
cleared.digest(digest2);
- assert Arrays.equals(digest1.digest(), digest2.digest());
+ assertArrayEquals(digest1.digest(), digest2.digest());
}
@Test
@@ -290,15 +288,15 @@
// For DB-1881
ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(COUNTER1);
- ColumnDefinition emptyColDef = cfs.metadata.getColumnDefinition(ByteBufferUtil.bytes("val2"));
+ ColumnMetadata emptyColDef = cfs.metadata().getColumn(ByteBufferUtil.bytes("val2"));
BufferCell emptyCell = BufferCell.live(emptyColDef, 0, ByteBuffer.allocate(0));
- Row.Builder builder = BTreeRow.unsortedBuilder(0);
+ Row.Builder builder = BTreeRow.unsortedBuilder();
builder.newRow(Clustering.make(AsciiSerializer.instance.serialize("test")));
builder.addCell(emptyCell);
Row row = builder.build();
- MessageDigest digest = MessageDigest.getInstance("md5");
+ Digest digest = Digest.forReadResponse();
row.digest(digest);
assertNotNull(digest.digest());
}
diff --git a/test/unit/org/apache/cassandra/db/CounterMutationTest.java b/test/unit/org/apache/cassandra/db/CounterMutationTest.java
index c8d4703..9be0960 100644
--- a/test/unit/org/apache/cassandra/db/CounterMutationTest.java
+++ b/test/unit/org/apache/cassandra/db/CounterMutationTest.java
@@ -21,7 +21,7 @@
import org.junit.Test;
import org.apache.cassandra.Util;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.db.rows.Row;
import org.apache.cassandra.db.context.CounterContext;
@@ -53,7 +53,7 @@
{
ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF1);
cfs.truncateBlocking();
- ColumnDefinition cDef = cfs.metadata.getColumnDefinition(ByteBufferUtil.bytes("val"));
+ ColumnMetadata cDef = cfs.metadata().getColumn(ByteBufferUtil.bytes("val"));
// Do the initial update (+1)
addAndCheck(cfs, 1, 1);
@@ -67,8 +67,8 @@
private void addAndCheck(ColumnFamilyStore cfs, long toAdd, long expected)
{
- ColumnDefinition cDef = cfs.metadata.getColumnDefinition(ByteBufferUtil.bytes("val"));
- Mutation m = new RowUpdateBuilder(cfs.metadata, 5, "key1").clustering("cc").add("val", toAdd).build();
+ ColumnMetadata cDef = cfs.metadata().getColumn(ByteBufferUtil.bytes("val"));
+ Mutation m = new RowUpdateBuilder(cfs.metadata(), 5, "key1").clustering("cc").add("val", toAdd).build();
new CounterMutation(m, ConsistencyLevel.ONE).apply();
Row row = Util.getOnlyRow(Util.cmd(cfs).includeRow("cc").columns("val").build());
@@ -93,10 +93,10 @@
private void addTwoAndCheck(ColumnFamilyStore cfs, long addOne, long expectedOne, long addTwo, long expectedTwo)
{
- ColumnDefinition cDefOne = cfs.metadata.getColumnDefinition(ByteBufferUtil.bytes("val"));
- ColumnDefinition cDefTwo = cfs.metadata.getColumnDefinition(ByteBufferUtil.bytes("val2"));
+ ColumnMetadata cDefOne = cfs.metadata().getColumn(ByteBufferUtil.bytes("val"));
+ ColumnMetadata cDefTwo = cfs.metadata().getColumn(ByteBufferUtil.bytes("val2"));
- Mutation m = new RowUpdateBuilder(cfs.metadata, 5, "key1")
+ Mutation m = new RowUpdateBuilder(cfs.metadata(), 5, "key1")
.clustering("cc")
.add("val", addOne)
.add("val2", addTwo)
@@ -118,36 +118,36 @@
cfsTwo.truncateBlocking();
// Do the update (+1, -1), (+2, -2)
- Mutation batch = new Mutation(KEYSPACE1, Util.dk("key1"));
- batch.add(new RowUpdateBuilder(cfsOne.metadata, 5, "key1")
+ Mutation.PartitionUpdateCollector batch = new Mutation.PartitionUpdateCollector(KEYSPACE1, Util.dk("key1"));
+ batch.add(new RowUpdateBuilder(cfsOne.metadata(), 5, "key1")
.clustering("cc")
.add("val", 1L)
.add("val2", -1L)
- .build().get(cfsOne.metadata));
+ .build().getPartitionUpdate(cfsOne.metadata()));
- batch.add(new RowUpdateBuilder(cfsTwo.metadata, 5, "key1")
+ batch.add(new RowUpdateBuilder(cfsTwo.metadata(), 5, "key1")
.clustering("cc")
.add("val", 2L)
.add("val2", -2L)
- .build().get(cfsTwo.metadata));
+ .build().getPartitionUpdate(cfsTwo.metadata()));
- new CounterMutation(batch, ConsistencyLevel.ONE).apply();
+ new CounterMutation(batch.build(), ConsistencyLevel.ONE).apply();
- ColumnDefinition c1cfs1 = cfsOne.metadata.getColumnDefinition(ByteBufferUtil.bytes("val"));
- ColumnDefinition c2cfs1 = cfsOne.metadata.getColumnDefinition(ByteBufferUtil.bytes("val2"));
+ ColumnMetadata c1cfs1 = cfsOne.metadata().getColumn(ByteBufferUtil.bytes("val"));
+ ColumnMetadata c2cfs1 = cfsOne.metadata().getColumn(ByteBufferUtil.bytes("val2"));
Row row = Util.getOnlyRow(Util.cmd(cfsOne).includeRow("cc").columns("val", "val2").build());
assertEquals(1L, CounterContext.instance().total(row.getCell(c1cfs1).value()));
assertEquals(-1L, CounterContext.instance().total(row.getCell(c2cfs1).value()));
- ColumnDefinition c1cfs2 = cfsTwo.metadata.getColumnDefinition(ByteBufferUtil.bytes("val"));
- ColumnDefinition c2cfs2 = cfsTwo.metadata.getColumnDefinition(ByteBufferUtil.bytes("val2"));
+ ColumnMetadata c1cfs2 = cfsTwo.metadata().getColumn(ByteBufferUtil.bytes("val"));
+ ColumnMetadata c2cfs2 = cfsTwo.metadata().getColumn(ByteBufferUtil.bytes("val2"));
row = Util.getOnlyRow(Util.cmd(cfsTwo).includeRow("cc").columns("val", "val2").build());
assertEquals(2L, CounterContext.instance().total(row.getCell(c1cfs2).value()));
assertEquals(-2L, CounterContext.instance().total(row.getCell(c2cfs2).value()));
// Check the caches, separately
- CBuilder cb = CBuilder.create(cfsOne.metadata.comparator);
+ CBuilder cb = CBuilder.create(cfsOne.metadata().comparator);
cb.add("cc");
assertEquals(1L, cfsOne.getCachedCounter(Util.dk("key1").getKey(), cb.build(), c1cfs1, null).count);
@@ -162,12 +162,12 @@
{
ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF1);
cfs.truncateBlocking();
- ColumnDefinition cOne = cfs.metadata.getColumnDefinition(ByteBufferUtil.bytes("val"));
- ColumnDefinition cTwo = cfs.metadata.getColumnDefinition(ByteBufferUtil.bytes("val2"));
+ ColumnMetadata cOne = cfs.metadata().getColumn(ByteBufferUtil.bytes("val"));
+ ColumnMetadata cTwo = cfs.metadata().getColumn(ByteBufferUtil.bytes("val2"));
// Do the initial update (+1, -1)
new CounterMutation(
- new RowUpdateBuilder(cfs.metadata, 5, "key1")
+ new RowUpdateBuilder(cfs.metadata(), 5, "key1")
.clustering("cc")
.add("val", 1L)
.add("val2", -1L)
@@ -180,7 +180,7 @@
// Remove the first counter, increment the second counter
new CounterMutation(
- new RowUpdateBuilder(cfs.metadata, 5, "key1")
+ new RowUpdateBuilder(cfs.metadata(), 5, "key1")
.clustering("cc")
.delete(cOne)
.add("val2", -5L)
@@ -193,7 +193,7 @@
// Increment the first counter, make sure it's still shadowed by the tombstone
new CounterMutation(
- new RowUpdateBuilder(cfs.metadata, 5, "key1")
+ new RowUpdateBuilder(cfs.metadata(), 5, "key1")
.clustering("cc")
.add("val", 1L)
.build(),
@@ -202,12 +202,12 @@
assertEquals(null, row.getCell(cOne));
// Get rid of the complete partition
- RowUpdateBuilder.deleteRow(cfs.metadata, 6, "key1", "cc").applyUnsafe();
+ RowUpdateBuilder.deleteRow(cfs.metadata(), 6, "key1", "cc").applyUnsafe();
Util.assertEmpty(Util.cmd(cfs).includeRow("cc").columns("val", "val2").build());
// Increment both counters, ensure that both stay dead
new CounterMutation(
- new RowUpdateBuilder(cfs.metadata, 6, "key1")
+ new RowUpdateBuilder(cfs.metadata(), 6, "key1")
.clustering("cc")
.add("val", 1L)
.add("val2", 1L)
diff --git a/test/unit/org/apache/cassandra/db/DeletePartitionTest.java b/test/unit/org/apache/cassandra/db/DeletePartitionTest.java
index a65befd..6ed43f7 100644
--- a/test/unit/org/apache/cassandra/db/DeletePartitionTest.java
+++ b/test/unit/org/apache/cassandra/db/DeletePartitionTest.java
@@ -23,7 +23,7 @@
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.Util;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.db.rows.Row;
import org.apache.cassandra.db.partitions.*;
import org.apache.cassandra.schema.KeyspaceParams;
@@ -59,10 +59,10 @@
public void testDeletePartition(DecoratedKey key, boolean flushBeforeRemove, boolean flushAfterRemove)
{
ColumnFamilyStore store = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD1);
- ColumnDefinition column = store.metadata.getColumnDefinition(ByteBufferUtil.bytes("val"));
+ ColumnMetadata column = store.metadata().getColumn(ByteBufferUtil.bytes("val"));
// write
- new RowUpdateBuilder(store.metadata, 0, key.getKey())
+ new RowUpdateBuilder(store.metadata(), 0, key.getKey())
.clustering("Column1")
.add("val", "asdf")
.build()
@@ -78,8 +78,9 @@
store.forceBlockingFlush();
// delete the partition
- new Mutation(KEYSPACE1, key)
- .add(PartitionUpdate.fullPartitionDelete(store.metadata, key, 0, FBUtilities.nowInSeconds()))
+ new Mutation.PartitionUpdateCollector(KEYSPACE1, key)
+ .add(PartitionUpdate.fullPartitionDelete(store.metadata(), key, 0, FBUtilities.nowInSeconds()))
+ .build()
.applyUnsafe();
if (flushAfterRemove)
diff --git a/test/unit/org/apache/cassandra/db/DigestTest.java b/test/unit/org/apache/cassandra/db/DigestTest.java
new file mode 100644
index 0000000..4fd12d0
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/DigestTest.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db;
+
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+
+import com.google.common.hash.Hashing;
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.Hex;
+import org.apache.cassandra.utils.UUIDGen;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+
+public class DigestTest
+{
+ private static final Logger logger = LoggerFactory.getLogger(DigestTest.class);
+
+ @Test
+ public void hashEmptyBytes() throws Exception {
+ Assert.assertArrayEquals(Hex.hexToBytes("d41d8cd98f00b204e9800998ecf8427e"),
+ Digest.forReadResponse().update(ByteBufferUtil.EMPTY_BYTE_BUFFER).digest());
+ }
+
+ @Test
+ public void hashBytesFromTinyDirectByteBuffer() throws Exception {
+ ByteBuffer directBuf = ByteBuffer.allocateDirect(8);
+ directBuf.putLong(5L).position(0);
+ directBuf.position(0);
+ assertArrayEquals(Hex.hexToBytes("aaa07454fa93ed2d37b4c5da9f2f87fd"),
+ Digest.forReadResponse().update(directBuf).digest());
+ }
+
+ @Test
+ public void hashBytesFromLargerDirectByteBuffer() throws Exception {
+ ByteBuffer directBuf = ByteBuffer.allocateDirect(1024);
+ for (int i = 0; i < 100; i++) {
+ directBuf.putInt(i);
+ }
+ directBuf.position(0);
+ assertArrayEquals(Hex.hexToBytes("daf10ea8894783b1b2618309494cde21"),
+ Digest.forReadResponse().update(directBuf).digest());
+ }
+
+ @Test
+ public void hashBytesFromTinyOnHeapByteBuffer() throws Exception {
+ ByteBuffer onHeapBuf = ByteBuffer.allocate(8);
+ onHeapBuf.putLong(5L);
+ onHeapBuf.position(0);
+ assertArrayEquals(Hex.hexToBytes("aaa07454fa93ed2d37b4c5da9f2f87fd"),
+ Digest.forReadResponse().update(onHeapBuf).digest());
+ }
+
+ @Test
+ public void hashBytesFromLargerOnHeapByteBuffer() throws Exception {
+ ByteBuffer onHeapBuf = ByteBuffer.allocate(1024);
+ for (int i = 0; i < 100; i++) {
+ onHeapBuf.putInt(i);
+ }
+ onHeapBuf.position(0);
+ assertArrayEquals(Hex.hexToBytes("daf10ea8894783b1b2618309494cde21"),
+ Digest.forReadResponse().update(onHeapBuf).digest());
+ }
+
+ @Test
+ public void testValidatorDigest()
+ {
+ Digest[] digests = new Digest[]
+ {
+ Digest.forValidator(),
+ new Digest(Hashing.murmur3_128(1000).newHasher()),
+ new Digest(Hashing.murmur3_128(2000).newHasher())
+ };
+ byte [] random = UUIDGen.getTimeUUIDBytes();
+
+ for (Digest digest : digests)
+ {
+ digest.updateWithByte((byte) 33)
+ .update(random, 0, random.length)
+ .update(ByteBuffer.wrap(random))
+ .update(random, 0, 3)
+ .updateWithBoolean(false)
+ .updateWithInt(77)
+ .updateWithLong(101);
+ }
+
+ long len = Byte.BYTES
+ + random.length * 2 // both the byte[] and the ByteBuffer
+ + 3 // 3 bytes from the random byte[]
+ + Byte.BYTES
+ + Integer.BYTES
+ + Long.BYTES;
+
+ assertEquals(len, digests[0].inputBytes());
+ byte[] h = digests[0].digest();
+ assertArrayEquals(digests[1].digest(), Arrays.copyOfRange(h, 0, 16));
+ assertArrayEquals(digests[2].digest(), Arrays.copyOfRange(h, 16, 32));
+ }
+
+}
diff --git a/test/unit/org/apache/cassandra/db/DirectoriesTest.java b/test/unit/org/apache/cassandra/db/DirectoriesTest.java
index e217f03..dd4e51f 100644
--- a/test/unit/org/apache/cassandra/db/DirectoriesTest.java
+++ b/test/unit/org/apache/cassandra/db/DirectoriesTest.java
@@ -34,11 +34,12 @@
import org.junit.BeforeClass;
import org.junit.Test;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.cql3.ColumnIdentifier;
+import org.apache.cassandra.schema.Indexes;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.config.Config.DiskFailurePolicy;
import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.cql3.statements.IndexTarget;
+import org.apache.cassandra.cql3.statements.schema.IndexTarget;
import org.apache.cassandra.db.Directories.DataDirectory;
import org.apache.cassandra.db.marshal.UTF8Type;
import org.apache.cassandra.index.internal.CassandraIndex;
@@ -50,7 +51,6 @@
import org.apache.cassandra.schema.IndexMetadata;
import org.apache.cassandra.service.DefaultFSErrorHandler;
import org.apache.cassandra.utils.ByteBufferUtil;
-import org.apache.cassandra.utils.Pair;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
@@ -64,7 +64,7 @@
private static final String KS = "ks";
private static final String[] TABLES = new String[] { "cf1", "ks" };
- private static final Set<CFMetaData> CFM = new HashSet<>(TABLES.length);
+ private static final Set<TableMetadata> CFM = new HashSet<>(TABLES.length);
private static final Map<String, List<File>> files = new HashMap<>();
@@ -77,15 +77,13 @@
for (String table : TABLES)
{
- UUID tableID = CFMetaData.generateLegacyCfId(KS, table);
- CFM.add(CFMetaData.Builder.create(KS, table)
- .withId(tableID)
- .addPartitionKey("thekey", UTF8Type.instance)
- .addClusteringColumn("thecolumn", UTF8Type.instance)
- .build());
+ CFM.add(TableMetadata.builder(KS, table)
+ .addPartitionKeyColumn("thekey", UTF8Type.instance)
+ .addClusteringColumn("thecolumn", UTF8Type.instance)
+ .build());
}
- tempDataDir = File.createTempFile("cassandra", "unittest");
+ tempDataDir = FileUtils.createTempFile("cassandra", "unittest");
tempDataDir.delete(); // hack to create a temp dir
tempDataDir.mkdir();
@@ -103,23 +101,23 @@
private static void createTestFiles() throws IOException
{
- for (CFMetaData cfm : CFM)
+ for (TableMetadata cfm : CFM)
{
List<File> fs = new ArrayList<>();
- files.put(cfm.cfName, fs);
+ files.put(cfm.name, fs);
File dir = cfDir(cfm);
dir.mkdirs();
- createFakeSSTable(dir, cfm.cfName, 1, fs);
- createFakeSSTable(dir, cfm.cfName, 2, fs);
+ createFakeSSTable(dir, cfm.name, 1, fs);
+ createFakeSSTable(dir, cfm.name, 2, fs);
File backupDir = new File(dir, Directories.BACKUPS_SUBDIR);
backupDir.mkdir();
- createFakeSSTable(backupDir, cfm.cfName, 1, fs);
+ createFakeSSTable(backupDir, cfm.name, 1, fs);
File snapshotDir = new File(dir, Directories.SNAPSHOT_SUBDIR + File.separator + "42");
snapshotDir.mkdirs();
- createFakeSSTable(snapshotDir, cfm.cfName, 1, fs);
+ createFakeSSTable(snapshotDir, cfm.name, 1, fs);
}
}
@@ -134,33 +132,33 @@
}
}
- private static File cfDir(CFMetaData metadata)
+ private static File cfDir(TableMetadata metadata)
{
- String cfId = ByteBufferUtil.bytesToHex(ByteBufferUtil.bytes(metadata.cfId));
- int idx = metadata.cfName.indexOf(Directories.SECONDARY_INDEX_NAME_SEPARATOR);
+ String tableId = metadata.id.toHexString();
+ int idx = metadata.name.indexOf(Directories.SECONDARY_INDEX_NAME_SEPARATOR);
if (idx >= 0)
{
// secondary index
return new File(tempDataDir,
- metadata.ksName + File.separator +
- metadata.cfName.substring(0, idx) + '-' + cfId + File.separator +
- metadata.cfName.substring(idx));
+ metadata.keyspace + File.separator +
+ metadata.name.substring(0, idx) + '-' + tableId + File.separator +
+ metadata.name.substring(idx));
}
else
{
- return new File(tempDataDir, metadata.ksName + File.separator + metadata.cfName + '-' + cfId);
+ return new File(tempDataDir, metadata.keyspace + File.separator + metadata.name + '-' + tableId);
}
}
@Test
public void testStandardDirs() throws IOException
{
- for (CFMetaData cfm : CFM)
+ for (TableMetadata cfm : CFM)
{
Directories directories = new Directories(cfm);
assertEquals(cfDir(cfm), directories.getDirectoryForNewSSTables());
- Descriptor desc = new Descriptor(cfDir(cfm), KS, cfm.cfName, 1, SSTableFormat.Type.BIG);
+ Descriptor desc = new Descriptor(cfDir(cfm), KS, cfm.name, 1, SSTableFormat.Type.BIG);
File snapshotDir = new File(cfDir(cfm), File.separator + Directories.SNAPSHOT_SUBDIR + File.separator + "42");
assertEquals(snapshotDir.getCanonicalFile(), Directories.getSnapshotDirectory(desc, "42"));
@@ -172,21 +170,22 @@
@Test
public void testSecondaryIndexDirectories()
{
- UUID tableID = CFMetaData.generateLegacyCfId(KS, "cf");
- CFMetaData PARENT_CFM = CFMetaData.Builder.create(KS, "cf")
- .withId(tableID)
- .addPartitionKey("thekey", UTF8Type.instance)
- .addClusteringColumn("col", UTF8Type.instance)
- .build();
- ColumnDefinition col = PARENT_CFM.getColumnDefinition(ByteBufferUtil.bytes("col"));
+ TableMetadata.Builder builder =
+ TableMetadata.builder(KS, "cf")
+ .addPartitionKeyColumn("thekey", UTF8Type.instance)
+ .addClusteringColumn("col", UTF8Type.instance);
+
+ ColumnIdentifier col = ColumnIdentifier.getInterned("col", true);
IndexMetadata indexDef =
- IndexMetadata.fromIndexTargets(PARENT_CFM,
- Collections.singletonList(new IndexTarget(col.name, IndexTarget.Type.VALUES)),
+ IndexMetadata.fromIndexTargets(
+ Collections.singletonList(new IndexTarget(col, IndexTarget.Type.VALUES)),
"idx",
IndexMetadata.Kind.KEYS,
Collections.emptyMap());
- PARENT_CFM.indexes(PARENT_CFM.getIndexes().with(indexDef));
- CFMetaData INDEX_CFM = CassandraIndex.indexCfsMetadata(PARENT_CFM, indexDef);
+ builder.indexes(Indexes.of(indexDef));
+
+ TableMetadata PARENT_CFM = builder.build();
+ TableMetadata INDEX_CFM = CassandraIndex.indexCfsMetadata(PARENT_CFM, indexDef);
Directories parentDirectories = new Directories(PARENT_CFM);
Directories indexDirectories = new Directories(INDEX_CFM);
// secondary index has its own directory
@@ -194,8 +193,8 @@
{
assertEquals(cfDir(INDEX_CFM), dir);
}
- Descriptor parentDesc = new Descriptor(parentDirectories.getDirectoryForNewSSTables(), KS, PARENT_CFM.cfName, 0, SSTableFormat.Type.BIG);
- Descriptor indexDesc = new Descriptor(indexDirectories.getDirectoryForNewSSTables(), KS, INDEX_CFM.cfName, 0, SSTableFormat.Type.BIG);
+ Descriptor parentDesc = new Descriptor(parentDirectories.getDirectoryForNewSSTables(), KS, PARENT_CFM.name, 0, SSTableFormat.Type.BIG);
+ Descriptor indexDesc = new Descriptor(indexDirectories.getDirectoryForNewSSTables(), KS, INDEX_CFM.name, 0, SSTableFormat.Type.BIG);
// snapshot dir should be created under its parent's
File parentSnapshotDirectory = Directories.getSnapshotDirectory(parentDesc, "test");
@@ -212,22 +211,22 @@
indexDirectories.snapshotCreationTime("test"));
// check true snapshot size
- Descriptor parentSnapshot = new Descriptor(parentSnapshotDirectory, KS, PARENT_CFM.cfName, 0, SSTableFormat.Type.BIG);
+ Descriptor parentSnapshot = new Descriptor(parentSnapshotDirectory, KS, PARENT_CFM.name, 0, SSTableFormat.Type.BIG);
createFile(parentSnapshot.filenameFor(Component.DATA), 30);
- Descriptor indexSnapshot = new Descriptor(indexSnapshotDirectory, KS, INDEX_CFM.cfName, 0, SSTableFormat.Type.BIG);
+ Descriptor indexSnapshot = new Descriptor(indexSnapshotDirectory, KS, INDEX_CFM.name, 0, SSTableFormat.Type.BIG);
createFile(indexSnapshot.filenameFor(Component.DATA), 40);
assertEquals(30, parentDirectories.trueSnapshotsSize());
assertEquals(40, indexDirectories.trueSnapshotsSize());
// check snapshot details
- Map<String, Pair<Long, Long>> parentSnapshotDetail = parentDirectories.getSnapshotDetails();
+ Map<String, Directories.SnapshotSizeDetails> parentSnapshotDetail = parentDirectories.getSnapshotDetails();
assertTrue(parentSnapshotDetail.containsKey("test"));
- assertEquals(30L, parentSnapshotDetail.get("test").right.longValue());
+ assertEquals(30L, parentSnapshotDetail.get("test").dataSizeBytes);
- Map<String, Pair<Long, Long>> indexSnapshotDetail = indexDirectories.getSnapshotDetails();
+ Map<String, Directories.SnapshotSizeDetails> indexSnapshotDetail = indexDirectories.getSnapshotDetails();
assertTrue(indexSnapshotDetail.containsKey("test"));
- assertEquals(40L, indexSnapshotDetail.get("test").right.longValue());
+ assertEquals(40L, indexSnapshotDetail.get("test").dataSizeBytes);
// check backup directory
File parentBackupDirectory = Directories.getBackupsDirectory(parentDesc);
@@ -250,20 +249,20 @@
@Test
public void testSSTableLister()
{
- for (CFMetaData cfm : CFM)
+ for (TableMetadata cfm : CFM)
{
Directories directories = new Directories(cfm);
checkFiles(cfm, directories);
}
}
- private void checkFiles(CFMetaData cfm, Directories directories)
+ private void checkFiles(TableMetadata cfm, Directories directories)
{
Directories.SSTableLister lister;
Set<File> listed;// List all but no snapshot, backup
lister = directories.sstableLister(Directories.OnTxnErr.THROW);
listed = new HashSet<>(lister.listFiles());
- for (File f : files.get(cfm.cfName))
+ for (File f : files.get(cfm.name))
{
if (f.getPath().contains(Directories.SNAPSHOT_SUBDIR) || f.getPath().contains(Directories.BACKUPS_SUBDIR))
assertFalse(f + " should not be listed", listed.contains(f));
@@ -274,7 +273,7 @@
// List all but including backup (but no snapshot)
lister = directories.sstableLister(Directories.OnTxnErr.THROW).includeBackups(true);
listed = new HashSet<>(lister.listFiles());
- for (File f : files.get(cfm.cfName))
+ for (File f : files.get(cfm.name))
{
if (f.getPath().contains(Directories.SNAPSHOT_SUBDIR))
assertFalse(f + " should not be listed", listed.contains(f));
@@ -285,7 +284,7 @@
// Skip temporary and compacted
lister = directories.sstableLister(Directories.OnTxnErr.THROW).skipTemporary(true);
listed = new HashSet<>(lister.listFiles());
- for (File f : files.get(cfm.cfName))
+ for (File f : files.get(cfm.name))
{
if (f.getPath().contains(Directories.SNAPSHOT_SUBDIR) || f.getPath().contains(Directories.BACKUPS_SUBDIR))
assertFalse(f + " should not be listed", listed.contains(f));
@@ -299,7 +298,7 @@
@Test
public void testTemporaryFile() throws IOException
{
- for (CFMetaData cfm : CFM)
+ for (TableMetadata cfm : CFM)
{
Directories directories = new Directories(cfm);
@@ -355,14 +354,14 @@
@Test
public void testMTSnapshots() throws Exception
{
- for (final CFMetaData cfm : CFM)
+ for (final TableMetadata cfm : CFM)
{
final Directories directories = new Directories(cfm);
assertEquals(cfDir(cfm), directories.getDirectoryForNewSSTables());
final String n = Long.toString(System.nanoTime());
Callable<File> directoryGetter = new Callable<File>() {
public File call() throws Exception {
- Descriptor desc = new Descriptor(cfDir(cfm), KS, cfm.cfName, 1, SSTableFormat.Type.BIG);
+ Descriptor desc = new Descriptor(cfDir(cfm), KS, cfm.name, 1, SSTableFormat.Type.BIG);
return Directories.getSnapshotDirectory(desc, n);
}
};
@@ -483,7 +482,7 @@
paths.add(new DataDirectory(new File("/tmp/aa")));
paths.add(new DataDirectory(new File("/tmp/a")));
- for (CFMetaData cfm : CFM)
+ for (TableMetadata cfm : CFM)
{
Directories dirs = new Directories(cfm, paths);
for (DataDirectory dir : paths)
@@ -523,7 +522,7 @@
paths.add(new DataDirectory(new File("/tmp/aa")));
paths.add(new DataDirectory(new File("/tmp/aaa")));
- for (CFMetaData cfm : CFM)
+ for (TableMetadata cfm : CFM)
{
Directories dirs = new Directories(cfm, paths);
for (DataDirectory dir : paths)
@@ -554,12 +553,12 @@
DataDirectory dd1 = new DataDirectory(ddir1.toFile());
DataDirectory dd2 = new DataDirectory(ddir2.toFile());
- for (CFMetaData tm : CFM)
+ for (TableMetadata tm : CFM)
{
Directories dirs = new Directories(tm, Sets.newHashSet(dd1, dd2));
- Descriptor desc = Descriptor.fromFilename(ddir1.resolve(getNewFilename(tm, false)).toString());
+ Descriptor desc = Descriptor.fromFilename(ddir1.resolve(getNewFilename(tm, false)).toFile());
assertEquals(ddir1.toFile(), dirs.getDataDirectoryForFile(desc).location);
- desc = Descriptor.fromFilename(ddir2.resolve(getNewFilename(tm, false)).toString());
+ desc = Descriptor.fromFilename(ddir2.resolve(getNewFilename(tm, false)).toFile());
assertEquals(ddir2.toFile(), dirs.getDataDirectoryForFile(desc).location);
}
}
@@ -595,28 +594,28 @@
Path ddir1 = Files.createDirectories(p.resolve("datadir1"));
Path ddir2 = Files.createDirectories(p.resolve("datadir11"));
- for (CFMetaData metadata : CFM)
+ for (TableMetadata tm : CFM)
{
- Path keyspacedir = Files.createDirectories(ddir2.resolve(metadata.ksName));
- String tabledir = metadata.cfName + (oldStyle ? "" : Component.separator + ByteBufferUtil.bytesToHex(ByteBufferUtil.bytes(metadata.cfId)));
+ Path keyspacedir = Files.createDirectories(ddir2.resolve(tm.keyspace));
+ String tabledir = tm.name + (oldStyle ? "" : Component.separator + tm.id.toHexString());
Files.createSymbolicLink(keyspacedir.resolve(tabledir), symlinktarget);
}
DataDirectory dd1 = new DataDirectory(ddir1.toFile());
DataDirectory dd2 = new DataDirectory(ddir2.toFile());
- for (CFMetaData tm : CFM)
+ for (TableMetadata tm : CFM)
{
Directories dirs = new Directories(tm, Sets.newHashSet(dd1, dd2));
- Descriptor desc = Descriptor.fromFilename(ddir1.resolve(getNewFilename(tm, oldStyle)).toFile().toString());
+ Descriptor desc = Descriptor.fromFilename(ddir1.resolve(getNewFilename(tm, oldStyle)).toFile());
assertEquals(ddir1.toFile(), dirs.getDataDirectoryForFile(desc).location);
- desc = Descriptor.fromFilename(ddir2.resolve(getNewFilename(tm, oldStyle)).toFile().toString());
+ desc = Descriptor.fromFilename(ddir2.resolve(getNewFilename(tm, oldStyle)).toFile());
assertEquals(ddir2.toFile(), dirs.getDataDirectoryForFile(desc).location);
}
}
- private String getNewFilename(CFMetaData metadata, boolean oldStyle)
+ private String getNewFilename(TableMetadata tm, boolean oldStyle)
{
- return metadata.ksName + File.separator + metadata.cfName + (oldStyle ? "" : Component.separator + ByteBufferUtil.bytesToHex(ByteBufferUtil.bytes(metadata.cfId))) + "/na-1-big-Data.db";
+ return tm.keyspace + File.separator + tm.name + (oldStyle ? "" : Component.separator + tm.id.toHexString()) + "/na-1-big-Data.db";
}
private List<Directories.DataDirectoryCandidate> getWriteableDirectories(DataDirectory[] dataDirectories, long writeSize)
diff --git a/test/unit/org/apache/cassandra/db/DiskBoundaryManagerTest.java b/test/unit/org/apache/cassandra/db/DiskBoundaryManagerTest.java
index febcfeb..3cd501e 100644
--- a/test/unit/org/apache/cassandra/db/DiskBoundaryManagerTest.java
+++ b/test/unit/org/apache/cassandra/db/DiskBoundaryManagerTest.java
@@ -19,7 +19,6 @@
package org.apache.cassandra.db;
import java.io.File;
-import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.List;
@@ -30,6 +29,7 @@
import org.apache.cassandra.cql3.CQLTester;
import org.apache.cassandra.dht.BootStrapper;
+import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.locator.TokenMetadata;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.FBUtilities;
@@ -51,9 +51,9 @@
{
DisallowedDirectories.clearUnwritableUnsafe();
TokenMetadata metadata = StorageService.instance.getTokenMetadata();
- metadata.updateNormalTokens(BootStrapper.getRandomTokens(metadata, 10), FBUtilities.getBroadcastAddress());
+ metadata.updateNormalTokens(BootStrapper.getRandomTokens(metadata, 10), FBUtilities.getBroadcastAddressAndPort());
createTable("create table %s (id int primary key, x text)");
- dirs = new Directories(getCurrentColumnFamilyStore().metadata, Lists.newArrayList(new Directories.DataDirectory(new File("/tmp/1")),
+ dirs = new Directories(getCurrentColumnFamilyStore().metadata(), Lists.newArrayList(new Directories.DataDirectory(new File("/tmp/1")),
new Directories.DataDirectory(new File("/tmp/2")),
new Directories.DataDirectory(new File("/tmp/3"))));
mock = new MockCFS(getCurrentColumnFamilyStore(), dirs);
@@ -86,7 +86,7 @@
public void updateTokensTest() throws UnknownHostException
{
DiskBoundaries dbv1 = dbm.getDiskBoundaries(mock);
- StorageService.instance.getTokenMetadata().updateNormalTokens(BootStrapper.getRandomTokens(StorageService.instance.getTokenMetadata(), 10), InetAddress.getByName("127.0.0.10"));
+ StorageService.instance.getTokenMetadata().updateNormalTokens(BootStrapper.getRandomTokens(StorageService.instance.getTokenMetadata(), 10), InetAddressAndPort.getByName("127.0.0.10"));
DiskBoundaries dbv2 = dbm.getDiskBoundaries(mock);
assertFalse(dbv1.equals(dbv2));
}
diff --git a/test/unit/org/apache/cassandra/db/ImportTest.java b/test/unit/org/apache/cassandra/db/ImportTest.java
new file mode 100644
index 0000000..4094aa4
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/ImportTest.java
@@ -0,0 +1,630 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Random;
+import java.util.Set;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.junit.Test;
+
+import org.apache.cassandra.cache.RowCacheKey;
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.cql3.UntypedResultSet;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.dht.BootStrapper;
+import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.io.sstable.Component;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.TokenMetadata;
+import org.apache.cassandra.service.CacheService;
+import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.utils.FBUtilities;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class ImportTest extends CQLTester
+{
+ @Test
+ public void basicImportTest() throws Throwable
+ {
+ createTable("create table %s (id int primary key, d int)");
+ for (int i = 0; i < 10; i++)
+ execute("insert into %s (id, d) values (?, ?)", i, i);
+ getCurrentColumnFamilyStore().forceBlockingFlush();
+ Set<SSTableReader> sstables = getCurrentColumnFamilyStore().getLiveSSTables();
+ getCurrentColumnFamilyStore().clearUnsafe();
+
+ File backupdir = moveToBackupDir(sstables);
+
+ assertEquals(0, execute("select * from %s").size());
+
+ SSTableImporter.Options options = SSTableImporter.Options.options(backupdir.toString()).build();
+ SSTableImporter importer = new SSTableImporter(getCurrentColumnFamilyStore());
+ importer.importNewSSTables(options);
+
+ assertEquals(10, execute("select * from %s").size());
+ }
+
+ @Test
+ public void basicImportMultiDirTest() throws Throwable
+ {
+ createTable("create table %s (id int primary key, d int)");
+ for (int i = 0; i < 10; i++)
+ execute("insert into %s (id, d) values (?, ?)", i, i);
+ getCurrentColumnFamilyStore().forceBlockingFlush();
+ Set<SSTableReader> sstables = getCurrentColumnFamilyStore().getLiveSSTables();
+ getCurrentColumnFamilyStore().clearUnsafe();
+
+ File backupdir = moveToBackupDir(sstables);
+ for (int i = 10; i < 20; i++)
+ execute("insert into %s (id, d) values (?, ?)", i, i);
+ getCurrentColumnFamilyStore().forceBlockingFlush();
+ sstables = getCurrentColumnFamilyStore().getLiveSSTables();
+ getCurrentColumnFamilyStore().clearUnsafe();
+
+ File backupdir2 = moveToBackupDir(sstables);
+
+ assertEquals(0, execute("select * from %s").size());
+
+ SSTableImporter.Options options = SSTableImporter.Options.options(Sets.newHashSet(backupdir.toString(), backupdir2.toString())).build();
+ SSTableImporter importer = new SSTableImporter(getCurrentColumnFamilyStore());
+ importer.importNewSSTables(options);
+
+ assertEquals(20, execute("select * from %s").size());
+
+ }
+
+
+ @Test
+ @Deprecated
+ public void refreshTest() throws Throwable
+ {
+ createTable("create table %s (id int primary key, d int)");
+ for (int i = 0; i < 10; i++)
+ execute("insert into %s (id, d) values (?, ?)", i, i);
+ getCurrentColumnFamilyStore().forceBlockingFlush();
+ Set<SSTableReader> sstables = getCurrentColumnFamilyStore().getLiveSSTables();
+ getCurrentColumnFamilyStore().clearUnsafe();
+ sstables.forEach(s -> s.selfRef().release());
+ assertEquals(0, execute("select * from %s").size());
+ getCurrentColumnFamilyStore().loadNewSSTables();
+ assertEquals(10, execute("select * from %s").size());
+ }
+
+ @Test
+ public void importResetLevelTest() throws Throwable
+ {
+ createTable("create table %s (id int primary key, d int)");
+ for (int i = 0; i < 10; i++)
+ execute("insert into %s (id, d) values (?, ?)", i, i);
+ getCurrentColumnFamilyStore().forceBlockingFlush();
+ Set<SSTableReader> sstables = getCurrentColumnFamilyStore().getLiveSSTables();
+ getCurrentColumnFamilyStore().clearUnsafe();
+ for (SSTableReader sstable : sstables)
+ sstable.descriptor.getMetadataSerializer().mutateLevel(sstable.descriptor, 8);
+ File backupdir = moveToBackupDir(sstables);
+ assertEquals(0, execute("select * from %s").size());
+
+ SSTableImporter.Options options = SSTableImporter.Options.options(backupdir.toString()).build();
+ SSTableImporter importer = new SSTableImporter(getCurrentColumnFamilyStore());
+ importer.importNewSSTables(options);
+
+ assertEquals(10, execute("select * from %s").size());
+ sstables = getCurrentColumnFamilyStore().getLiveSSTables();
+ assertEquals(1, sstables.size());
+ for (SSTableReader sstable : sstables)
+ assertEquals(8, sstable.getSSTableLevel());
+
+ getCurrentColumnFamilyStore().clearUnsafe();
+ backupdir = moveToBackupDir(sstables);
+
+ options = SSTableImporter.Options.options(backupdir.toString()).resetLevel(true).build();
+ importer.importNewSSTables(options);
+
+ sstables = getCurrentColumnFamilyStore().getLiveSSTables();
+ assertEquals(1, sstables.size());
+ for (SSTableReader sstable : getCurrentColumnFamilyStore().getLiveSSTables())
+ assertEquals(0, sstable.getSSTableLevel());
+ }
+
+
+ @Test
+ public void importClearRepairedTest() throws Throwable
+ {
+ createTable("create table %s (id int primary key, d int)");
+ for (int i = 0; i < 10; i++)
+ execute("insert into %s (id, d) values (?, ?)", i, i);
+ getCurrentColumnFamilyStore().forceBlockingFlush();
+ Set<SSTableReader> sstables = getCurrentColumnFamilyStore().getLiveSSTables();
+ getCurrentColumnFamilyStore().clearUnsafe();
+ for (SSTableReader sstable : sstables)
+ sstable.descriptor.getMetadataSerializer().mutateRepairMetadata(sstable.descriptor, 111, null, false);
+
+ File backupdir = moveToBackupDir(sstables);
+ assertEquals(0, execute("select * from %s").size());
+
+ SSTableImporter.Options options = SSTableImporter.Options.options(backupdir.toString()).build();
+ SSTableImporter importer = new SSTableImporter(getCurrentColumnFamilyStore());
+ importer.importNewSSTables(options);
+
+ assertEquals(10, execute("select * from %s").size());
+ sstables = getCurrentColumnFamilyStore().getLiveSSTables();
+ assertEquals(1, sstables.size());
+ for (SSTableReader sstable : sstables)
+ assertTrue(sstable.isRepaired());
+
+ getCurrentColumnFamilyStore().clearUnsafe();
+ backupdir = moveToBackupDir(sstables);
+
+ options = SSTableImporter.Options.options(backupdir.toString()).clearRepaired(true).build();
+ importer.importNewSSTables(options);
+ sstables = getCurrentColumnFamilyStore().getLiveSSTables();
+ assertEquals(1, sstables.size());
+ for (SSTableReader sstable : getCurrentColumnFamilyStore().getLiveSSTables())
+ assertFalse(sstable.isRepaired());
+ }
+
+ private File moveToBackupDir(Set<SSTableReader> sstables) throws IOException
+ {
+ Path temp = Files.createTempDirectory("importtest");
+ SSTableReader sst = sstables.iterator().next();
+ String tabledir = sst.descriptor.directory.getName();
+ String ksdir = sst.descriptor.directory.getParentFile().getName();
+ Path backupdir = createDirectories(temp.toString(), ksdir, tabledir);
+ for (SSTableReader sstable : sstables)
+ {
+ sstable.selfRef().release();
+ for (File f : sstable.descriptor.directory.listFiles())
+ {
+ if (f.toString().contains(sstable.descriptor.baseFilename()))
+ {
+ System.out.println("move " + f.toPath() + " to " + backupdir);
+ File moveFileTo = new File(backupdir.toFile(), f.getName());
+ moveFileTo.deleteOnExit();
+ Files.move(f.toPath(), moveFileTo.toPath());
+ }
+ }
+ }
+ return backupdir.toFile();
+ }
+
+ private Path createDirectories(String base, String ... subdirs)
+ {
+ File b = new File(base);
+ b.mkdir();
+ System.out.println("mkdir "+b);
+ b.deleteOnExit();
+ for (String subdir : subdirs)
+ {
+ b = new File(b, subdir);
+ b.mkdir();
+ System.out.println("mkdir "+b);
+ b.deleteOnExit();
+ }
+ return b.toPath();
+ }
+
+ @Test
+ public void testGetCorrectDirectory() throws Throwable
+ {
+ TokenMetadata metadata = StorageService.instance.getTokenMetadata();
+ metadata.updateNormalTokens(BootStrapper.getRandomTokens(metadata, 10), FBUtilities.getBroadcastAddressAndPort());
+ createTable("create table %s (id int primary key, d int)");
+ getCurrentColumnFamilyStore().disableAutoCompaction();
+
+ // generate sstables with different first tokens
+ for (int i = 0; i < 10; i++)
+ {
+ execute("insert into %s (id, d) values (?, ?)", i, i);
+ getCurrentColumnFamilyStore().forceBlockingFlush();
+ }
+
+ Set<SSTableReader> toMove = getCurrentColumnFamilyStore().getLiveSSTables();
+ getCurrentColumnFamilyStore().clearUnsafe();
+ File dir = moveToBackupDir(toMove);
+
+ Directories dirs = new Directories(getCurrentColumnFamilyStore().metadata(), Lists.newArrayList(new Directories.DataDirectory(new File("/tmp/1")),
+ new Directories.DataDirectory(new File("/tmp/2")),
+ new Directories.DataDirectory(new File("/tmp/3"))));
+ MockCFS mock = new MockCFS(getCurrentColumnFamilyStore(), dirs);
+ SSTableImporter importer = new SSTableImporter(mock);
+
+ importer.importNewSSTables(SSTableImporter.Options.options(dir.toString()).build());
+ for (SSTableReader sstable : mock.getLiveSSTables())
+ {
+ File movedDir = sstable.descriptor.directory.getCanonicalFile();
+ File correctDir = mock.getDiskBoundaries().getCorrectDiskForSSTable(sstable).location.getCanonicalFile();
+ assertTrue(movedDir.toString().startsWith(correctDir.toString()));
+ }
+ for (SSTableReader sstable : mock.getLiveSSTables())
+ sstable.selfRef().release();
+ }
+
+ private void testCorruptHelper(boolean verify) throws Throwable
+ {
+ createTable("create table %s (id int primary key, d int)");
+ for (int i = 0; i < 10; i++)
+ execute("insert into %s (id, d) values (?, ?)", i, i);
+ getCurrentColumnFamilyStore().forceBlockingFlush();
+ SSTableReader sstableToCorrupt = getCurrentColumnFamilyStore().getLiveSSTables().iterator().next();
+ for (int i = 0; i < 10; i++)
+ execute("insert into %s (id, d) values (?, ?)", i + 10, i);
+ getCurrentColumnFamilyStore().forceBlockingFlush();
+ Set<SSTableReader> sstables = getCurrentColumnFamilyStore().getLiveSSTables();
+
+ getCurrentColumnFamilyStore().clearUnsafe();
+
+ String filenameToCorrupt = sstableToCorrupt.descriptor.filenameFor(Component.STATS);
+ try (RandomAccessFile file = new RandomAccessFile(filenameToCorrupt, "rw"))
+ {
+ file.seek(0);
+ file.writeBytes(StringUtils.repeat('z', 2));
+ }
+
+ File backupdir = moveToBackupDir(sstables);
+
+ // now move a correct sstable to another directory to make sure that directory gets properly imported
+ for (int i = 100; i < 130; i++)
+ execute("insert into %s (id, d) values (?, ?)", i, i);
+ getCurrentColumnFamilyStore().forceBlockingFlush();
+ Set<SSTableReader> correctSSTables = getCurrentColumnFamilyStore().getLiveSSTables();
+
+ getCurrentColumnFamilyStore().clearUnsafe();
+ File backupdirCorrect = moveToBackupDir(correctSSTables);
+
+ Set<File> beforeImport = Sets.newHashSet(backupdir.listFiles());
+ // first we moved out 2 sstables, one correct and one corrupt in to a single directory (backupdir)
+ // then we moved out 1 sstable, a correct one (in backupdirCorrect).
+ // now import should fail import on backupdir, but import the one in backupdirCorrect.
+ SSTableImporter.Options options = SSTableImporter.Options.options(Sets.newHashSet(backupdir.toString(), backupdirCorrect.toString())).verifySSTables(verify).build();
+ SSTableImporter importer = new SSTableImporter(getCurrentColumnFamilyStore());
+ List<String> failedDirectories = importer.importNewSSTables(options);
+ assertEquals(Collections.singletonList(backupdir.toString()), failedDirectories);
+ UntypedResultSet res = execute("SELECT * FROM %s");
+ for (UntypedResultSet.Row r : res)
+ {
+ int pk = r.getInt("id");
+ assertTrue("pk = "+pk, pk >= 100 && pk < 130);
+ }
+ assertEquals("Data dir should contain one file", 1, countFiles(getCurrentColumnFamilyStore().getDirectories().getDirectoryForNewSSTables()));
+ assertEquals("backupdir contained 2 files before import, should still contain 2 after failing to import it", beforeImport, Sets.newHashSet(backupdir.listFiles()));
+ assertEquals("backupdirCorrect contained 1 file before import, should be empty after import", 0, countFiles(backupdirCorrect));
+ }
+
+ private int countFiles(File dir)
+ {
+ int fileCount = 0;
+
+ for (File f : dir.listFiles())
+ {
+ if (f.isFile() && f.toString().contains("-Data.db"))
+ {
+ fileCount++;
+ }
+ }
+ return fileCount;
+ }
+
+ @Test
+ public void testImportCorrupt() throws Throwable
+ {
+ testCorruptHelper(true);
+ }
+
+ @Test
+ public void testImportCorruptWithoutValidation() throws Throwable
+ {
+ testCorruptHelper(false);
+ }
+
+ @Test
+ public void testImportOutOfRange() throws Throwable
+ {
+ createTable("create table %s (id int primary key, d int)");
+ for (int i = 0; i < 1000; i++)
+ execute("insert into %s (id, d) values (?, ?)", i, i);
+ getCurrentColumnFamilyStore().forceBlockingFlush();
+ Set<SSTableReader> sstables = getCurrentColumnFamilyStore().getLiveSSTables();
+
+ getCurrentColumnFamilyStore().clearUnsafe();
+
+ TokenMetadata tmd = StorageService.instance.getTokenMetadata();
+
+ tmd.updateNormalTokens(BootStrapper.getRandomTokens(tmd, 5), InetAddressAndPort.getByName("127.0.0.1"));
+ tmd.updateNormalTokens(BootStrapper.getRandomTokens(tmd, 5), InetAddressAndPort.getByName("127.0.0.2"));
+ tmd.updateNormalTokens(BootStrapper.getRandomTokens(tmd, 5), InetAddressAndPort.getByName("127.0.0.3"));
+
+
+ File backupdir = moveToBackupDir(sstables);
+ try
+ {
+ SSTableImporter.Options options = SSTableImporter.Options.options(backupdir.toString()).verifySSTables(true).verifyTokens(true).build();
+ SSTableImporter importer = new SSTableImporter(getCurrentColumnFamilyStore());
+ List<String> failed = importer.importNewSSTables(options);
+ assertEquals(Collections.singletonList(backupdir.toString()), failed);
+
+ // verify that we check the tokens if verifySSTables == false but verifyTokens == true:
+ options = SSTableImporter.Options.options(backupdir.toString()).verifySSTables(false).verifyTokens(true).build();
+ importer = new SSTableImporter(getCurrentColumnFamilyStore());
+ failed = importer.importNewSSTables(options);
+ assertEquals(Collections.singletonList(backupdir.toString()), failed);
+
+ // and that we can import with it disabled:
+ options = SSTableImporter.Options.options(backupdir.toString()).verifySSTables(true).verifyTokens(false).build();
+ importer = new SSTableImporter(getCurrentColumnFamilyStore());
+ failed = importer.importNewSSTables(options);
+ assertTrue(failed.isEmpty());
+
+ }
+ finally
+ {
+ tmd.clearUnsafe();
+ }
+ }
+
+ @Test
+ public void testImportOutOfRangeExtendedVerify() throws Throwable
+ {
+ createTable("create table %s (id int primary key, d int)");
+ for (int i = 0; i < 1000; i++)
+ execute("insert into %s (id, d) values (?, ?)", i, i);
+ getCurrentColumnFamilyStore().forceBlockingFlush();
+ Set<SSTableReader> sstables = getCurrentColumnFamilyStore().getLiveSSTables();
+
+ getCurrentColumnFamilyStore().clearUnsafe();
+
+ TokenMetadata tmd = StorageService.instance.getTokenMetadata();
+
+ tmd.updateNormalTokens(BootStrapper.getRandomTokens(tmd, 5), InetAddressAndPort.getByName("127.0.0.1"));
+ tmd.updateNormalTokens(BootStrapper.getRandomTokens(tmd, 5), InetAddressAndPort.getByName("127.0.0.2"));
+ tmd.updateNormalTokens(BootStrapper.getRandomTokens(tmd, 5), InetAddressAndPort.getByName("127.0.0.3"));
+
+
+ File backupdir = moveToBackupDir(sstables);
+ try
+ {
+ SSTableImporter.Options options = SSTableImporter.Options.options(backupdir.toString())
+ .verifySSTables(true)
+ .verifyTokens(true)
+ .extendedVerify(true).build();
+ SSTableImporter importer = new SSTableImporter(getCurrentColumnFamilyStore());
+ List<String> failedDirectories = importer.importNewSSTables(options);
+ assertEquals(Collections.singletonList(backupdir.toString()), failedDirectories);
+ }
+ finally
+ {
+ tmd.clearUnsafe();
+ }
+ }
+
+
+ @Test
+ public void testImportInvalidateCache() throws Throwable
+ {
+ createTable("create table %s (id int primary key, d int) WITH caching = { 'keys': 'NONE', 'rows_per_partition': 'ALL' }");
+ for (int i = 0; i < 10; i++)
+ execute("insert into %s (id, d) values (?, ?)", i, i);
+ getCurrentColumnFamilyStore().forceBlockingFlush();
+ CacheService.instance.setRowCacheCapacityInMB(1);
+
+ Set<RowCacheKey> keysToInvalidate = new HashSet<>();
+
+ // populate the row cache with keys from the sstable we are about to remove
+ for (int i = 0; i < 10; i++)
+ {
+ execute("SELECT * FROM %s WHERE id = ?", i);
+ }
+ Iterator<RowCacheKey> it = CacheService.instance.rowCache.keyIterator();
+ while (it.hasNext())
+ {
+ keysToInvalidate.add(it.next());
+ }
+ SSTableReader sstableToImport = getCurrentColumnFamilyStore().getLiveSSTables().iterator().next();
+ getCurrentColumnFamilyStore().clearUnsafe();
+
+
+ for (int i = 10; i < 20; i++)
+ execute("insert into %s (id, d) values (?, ?)", i, i);
+ getCurrentColumnFamilyStore().forceBlockingFlush();
+
+ Set<RowCacheKey> allCachedKeys = new HashSet<>();
+
+ // populate row cache with sstable we are keeping
+ for (int i = 10; i < 20; i++)
+ {
+ execute("SELECT * FROM %s WHERE id = ?", i);
+ }
+ it = CacheService.instance.rowCache.keyIterator();
+ while (it.hasNext())
+ {
+ allCachedKeys.add(it.next());
+ }
+ assertEquals(20, CacheService.instance.rowCache.size());
+ File backupdir = moveToBackupDir(Collections.singleton(sstableToImport));
+ // make sure we don't wipe caches with invalidateCaches = false:
+ Set<SSTableReader> beforeFirstImport = getCurrentColumnFamilyStore().getLiveSSTables();
+
+ SSTableImporter.Options options = SSTableImporter.Options.options(backupdir.toString()).verifySSTables(true).verifyTokens(true).build();
+ SSTableImporter importer = new SSTableImporter(getCurrentColumnFamilyStore());
+ importer.importNewSSTables(options);
+ assertEquals(20, CacheService.instance.rowCache.size());
+ Set<SSTableReader> toMove = Sets.difference(getCurrentColumnFamilyStore().getLiveSSTables(), beforeFirstImport);
+ getCurrentColumnFamilyStore().clearUnsafe();
+ // move away the sstable we just imported again:
+ backupdir = moveToBackupDir(toMove);
+ beforeFirstImport.forEach(s -> s.selfRef().release());
+ options = SSTableImporter.Options.options(backupdir.toString()).verifySSTables(true).verifyTokens(true).invalidateCaches(true).build();
+ importer.importNewSSTables(options);
+ assertEquals(10, CacheService.instance.rowCache.size());
+ it = CacheService.instance.rowCache.keyIterator();
+ while (it.hasNext())
+ {
+ // make sure the keys from the sstable we are importing are invalidated and that the other one is still there
+ RowCacheKey rck = it.next();
+ assertTrue(allCachedKeys.contains(rck));
+ assertFalse(keysToInvalidate.contains(rck));
+ }
+ }
+
+ @Test
+ public void testImportCacheEnabledWithoutSrcDir() throws Throwable
+ {
+ createTable("create table %s (id int primary key, d int) WITH caching = { 'keys': 'NONE', 'rows_per_partition': 'ALL' }");
+ for (int i = 0; i < 10; i++)
+ execute("insert into %s (id, d) values (?, ?)", i, i);
+ getCurrentColumnFamilyStore().forceBlockingFlush();
+ Set<SSTableReader> sstables = getCurrentColumnFamilyStore().getLiveSSTables();
+ CacheService.instance.setRowCacheCapacityInMB(1);
+ getCurrentColumnFamilyStore().clearUnsafe();
+ sstables.forEach(s -> s.selfRef().release());
+ SSTableImporter.Options options = SSTableImporter.Options.options().invalidateCaches(true).build();
+ SSTableImporter importer = new SSTableImporter(getCurrentColumnFamilyStore());
+ importer.importNewSSTables(options);
+ assertEquals(1, getCurrentColumnFamilyStore().getLiveSSTables().size());
+ }
+
+ @Test
+ public void testRefreshCorrupt() throws Throwable
+ {
+ createTable("create table %s (id int primary key, d int) WITH caching = { 'keys': 'NONE', 'rows_per_partition': 'ALL' }");
+ for (int i = 0; i < 10; i++)
+ execute("insert into %s (id, d) values (?, ?)", i, i);
+ getCurrentColumnFamilyStore().forceBlockingFlush();
+ Set<SSTableReader> sstables = getCurrentColumnFamilyStore().getLiveSSTables();
+ getCurrentColumnFamilyStore().clearUnsafe();
+ sstables.forEach(s -> s.selfRef().release());
+ // corrupt the sstable which is still in the data directory
+ SSTableReader sstableToCorrupt = sstables.iterator().next();
+ String filenameToCorrupt = sstableToCorrupt.descriptor.filenameFor(Component.STATS);
+ try (RandomAccessFile file = new RandomAccessFile(filenameToCorrupt, "rw"))
+ {
+ file.seek(0);
+ file.writeBytes(StringUtils.repeat('z', 2));
+ }
+
+ for (int i = 10; i < 20; i++)
+ execute("insert into %s (id, d) values (?, ?)", i, i);
+ getCurrentColumnFamilyStore().forceBlockingFlush();
+ for (int i = 20; i < 30; i++)
+ execute("insert into %s (id, d) values (?, ?)", i, i);
+ getCurrentColumnFamilyStore().forceBlockingFlush();
+
+ Set<SSTableReader> expectedFiles = new HashSet<>(getCurrentColumnFamilyStore().getLiveSSTables());
+
+ SSTableImporter.Options options = SSTableImporter.Options.options().build();
+ SSTableImporter importer = new SSTableImporter(getCurrentColumnFamilyStore());
+ boolean gotException = false;
+ try
+ {
+ importer.importNewSSTables(options);
+ }
+ catch (Throwable t)
+ {
+ gotException = true;
+ }
+ assertTrue(gotException);
+ assertEquals(2, getCurrentColumnFamilyStore().getLiveSSTables().size());
+ // for nodetool refresh we leave corrupt sstables in the data directory
+ assertEquals(3, countFiles(sstableToCorrupt.descriptor.directory));
+ int rowCount = 0;
+ for (UntypedResultSet.Row r : execute("SELECT * FROM %s"))
+ {
+ rowCount++;
+ int pk = r.getInt("id");
+ assertTrue("pk = "+pk, pk >= 10 && pk < 30);
+ }
+ assertEquals(20, rowCount);
+ assertEquals(expectedFiles, getCurrentColumnFamilyStore().getLiveSSTables());
+ for (SSTableReader sstable : expectedFiles)
+ assertTrue(new File(sstable.descriptor.filenameFor(Component.DATA)).exists());
+ getCurrentColumnFamilyStore().truncateBlocking();
+ LifecycleTransaction.waitForDeletions();
+ for (File f : sstableToCorrupt.descriptor.directory.listFiles()) // clean up the corrupt files which truncate does not handle
+ f.delete();
+
+ }
+
+ /**
+ * If a user gives a bad directory we don't import any directories - we should let the user correct the directories
+ */
+ @Test
+ public void importBadDirectoryTest() throws Throwable
+ {
+ createTable("create table %s (id int primary key, d int)");
+ for (int i = 0; i < 10; i++)
+ execute("insert into %s (id, d) values (?, ?)", i, i);
+ getCurrentColumnFamilyStore().forceBlockingFlush();
+ Set<SSTableReader> sstables = getCurrentColumnFamilyStore().getLiveSSTables();
+ getCurrentColumnFamilyStore().clearUnsafe();
+
+ File backupdir = moveToBackupDir(sstables);
+ for (int i = 10; i < 20; i++)
+ execute("insert into %s (id, d) values (?, ?)", i, i);
+ getCurrentColumnFamilyStore().forceBlockingFlush();
+ sstables = getCurrentColumnFamilyStore().getLiveSSTables();
+ getCurrentColumnFamilyStore().clearUnsafe();
+
+ File backupdir2 = moveToBackupDir(sstables);
+
+ assertEquals(0, execute("select * from %s").size());
+
+ SSTableImporter.Options options = SSTableImporter.Options.options(Sets.newHashSet(backupdir.toString(), backupdir2.toString(), "/tmp/DOESNTEXIST")).build();
+ SSTableImporter importer = new SSTableImporter(getCurrentColumnFamilyStore());
+ boolean gotException = false;
+ try
+ {
+ importer.importNewSSTables(options);
+ }
+ catch (Throwable t)
+ {
+ gotException = true;
+ }
+ assertTrue(gotException);
+ assertEquals(0, execute("select * from %s").size());
+ assertEquals(0, getCurrentColumnFamilyStore().getLiveSSTables().size());
+ }
+
+ private static class MockCFS extends ColumnFamilyStore
+ {
+ public MockCFS(ColumnFamilyStore cfs, Directories dirs)
+ {
+ super(cfs.keyspace, cfs.getTableName(), 0, cfs.metadata, dirs, false, false, true);
+ }
+ }
+}
diff --git a/test/unit/org/apache/cassandra/db/KeyCacheTest.java b/test/unit/org/apache/cassandra/db/KeyCacheTest.java
index ada6b5b..1819b18 100644
--- a/test/unit/org/apache/cassandra/db/KeyCacheTest.java
+++ b/test/unit/org/apache/cassandra/db/KeyCacheTest.java
@@ -269,8 +269,8 @@
Mutation rm;
// inserts
- new RowUpdateBuilder(cfs.metadata, 0, "key1").clustering("1").build().applyUnsafe();
- new RowUpdateBuilder(cfs.metadata, 0, "key2").clustering("2").build().applyUnsafe();
+ new RowUpdateBuilder(cfs.metadata(), 0, "key1").clustering("1").build().applyUnsafe();
+ new RowUpdateBuilder(cfs.metadata(), 0, "key2").clustering("2").build().applyUnsafe();
// to make sure we have SSTable
cfs.forceBlockingFlush();
@@ -287,7 +287,7 @@
throw new IllegalStateException();
Util.compactAll(cfs, Integer.MAX_VALUE).get();
- boolean noEarlyOpen = DatabaseDescriptor.getSSTablePreempiveOpenIntervalInMB() < 0;
+ boolean noEarlyOpen = DatabaseDescriptor.getSSTablePreemptiveOpenIntervalInMB() < 0;
// after compaction cache should have entries for new SSTables,
// but since we have kept a reference to the old sstables,
diff --git a/test/unit/org/apache/cassandra/db/KeyspaceTest.java b/test/unit/org/apache/cassandra/db/KeyspaceTest.java
index 3c3b04b..3e088fb 100644
--- a/test/unit/org/apache/cassandra/db/KeyspaceTest.java
+++ b/test/unit/org/apache/cassandra/db/KeyspaceTest.java
@@ -101,20 +101,20 @@
{
// slice with limit 1
Row row = Util.getOnlyRow(Util.cmd(cfs, "0").columns("c").withLimit(1).build());
- assertEquals(ByteBufferUtil.bytes(0), row.getCell(cfs.metadata.getColumnDefinition(new ColumnIdentifier("c", false))).value());
+ assertEquals(ByteBufferUtil.bytes(0), row.getCell(cfs.metadata().getColumn(new ColumnIdentifier("c", false))).value());
// fetch each row by name
for (int i = 0; i < 2; i++)
{
row = Util.getOnlyRow(Util.cmd(cfs, "0").columns("c").includeRow(i).build());
- assertEquals(ByteBufferUtil.bytes(i), row.getCell(cfs.metadata.getColumnDefinition(new ColumnIdentifier("c", false))).value());
+ assertEquals(ByteBufferUtil.bytes(i), row.getCell(cfs.metadata().getColumn(new ColumnIdentifier("c", false))).value());
}
// fetch each row by slice
for (int i = 0; i < 2; i++)
{
row = Util.getOnlyRow(Util.cmd(cfs, "0").columns("c").fromIncl(i).toIncl(i).build());
- assertEquals(ByteBufferUtil.bytes(i), row.getCell(cfs.metadata.getColumnDefinition(new ColumnIdentifier("c", false))).value());
+ assertEquals(ByteBufferUtil.bytes(i), row.getCell(cfs.metadata().getColumn(new ColumnIdentifier("c", false))).value());
}
if (round == 0)
@@ -166,7 +166,7 @@
for (int i = sliceEnd; i >= sliceStart; i--)
{
Row row = rowIterator.next();
- Cell cell = row.getCell(cfs.metadata.getColumnDefinition(new ColumnIdentifier("c", false)));
+ Cell cell = row.getCell(cfs.metadata().getColumn(new ColumnIdentifier("c", false)));
assertEquals(ByteBufferUtil.bytes(columnValuePrefix + i), cell.value());
}
}
@@ -175,7 +175,7 @@
for (int i = sliceStart; i <= sliceEnd; i++)
{
Row row = rowIterator.next();
- Cell cell = row.getCell(cfs.metadata.getColumnDefinition(new ColumnIdentifier("c", false)));
+ Cell cell = row.getCell(cfs.metadata().getColumn(new ColumnIdentifier("c", false)));
assertEquals(ByteBufferUtil.bytes(columnValuePrefix + i), cell.value());
}
}
@@ -226,7 +226,7 @@
{
execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", "0", i, i);
- PartitionColumns.of(cfs.metadata.getColumnDefinition(new ColumnIdentifier("c", false)));
+ RegularAndStaticColumns columns = RegularAndStaticColumns.of(cfs.metadata().getColumn(new ColumnIdentifier("c", false)));
ClusteringIndexSliceFilter filter = new ClusteringIndexSliceFilter(Slices.ALL, false);
SinglePartitionReadCommand command = singlePartitionSlice(cfs, "0", filter, null);
try (ReadExecutionController executionController = command.executionController();
@@ -235,7 +235,7 @@
try (RowIterator rowIterator = iterator.next())
{
Row row = rowIterator.next();
- Cell cell = row.getCell(cfs.metadata.getColumnDefinition(new ColumnIdentifier("c", false)));
+ Cell cell = row.getCell(cfs.metadata().getColumn(new ColumnIdentifier("c", false)));
assertEquals(ByteBufferUtil.bytes(i), cell.value());
}
}
@@ -250,7 +250,7 @@
if (columnValues.length == 0)
{
if (iterator.hasNext())
- fail("Didn't expect any results, but got rows starting with: " + iterator.next().next().toString(cfs.metadata));
+ fail("Didn't expect any results, but got rows starting with: " + iterator.next().next().toString(cfs.metadata()));
return;
}
@@ -259,7 +259,7 @@
for (int expected : columnValues)
{
Row row = rowIterator.next();
- Cell cell = row.getCell(cfs.metadata.getColumnDefinition(new ColumnIdentifier("c", false)));
+ Cell cell = row.getCell(cfs.metadata().getColumn(new ColumnIdentifier("c", false)));
assertEquals(
String.format("Expected %s, but got %s", ByteBufferUtil.bytesToHex(ByteBufferUtil.bytes(expected)), ByteBufferUtil.bytesToHex(cell.value())),
ByteBufferUtil.bytes(expected), cell.value());
@@ -287,7 +287,7 @@
? DataLimits.NONE
: DataLimits.cqlLimits(rowLimit);
return SinglePartitionReadCommand.create(
- cfs.metadata, FBUtilities.nowInSeconds(), ColumnFilter.all(cfs.metadata), RowFilter.NONE, limit, Util.dk(key), filter);
+ cfs.metadata(), FBUtilities.nowInSeconds(), ColumnFilter.all(cfs.metadata()), RowFilter.NONE, limit, Util.dk(key), filter);
}
@Test
@@ -456,17 +456,17 @@
{
ClusteringIndexSliceFilter filter = slices(cfs, 1000, null, false);
SinglePartitionReadCommand command = SinglePartitionReadCommand.create(
- cfs.metadata, FBUtilities.nowInSeconds(), ColumnFilter.all(cfs.metadata), RowFilter.NONE, DataLimits.cqlLimits(3), Util.dk("0"), filter);
+ cfs.metadata(), FBUtilities.nowInSeconds(), ColumnFilter.all(cfs.metadata()), RowFilter.NONE, DataLimits.cqlLimits(3), Util.dk("0"), filter);
assertRowsInResult(cfs, command, 1000, 1001, 1002);
filter = slices(cfs, 1195, null, false);
command = SinglePartitionReadCommand.create(
- cfs.metadata, FBUtilities.nowInSeconds(), ColumnFilter.all(cfs.metadata), RowFilter.NONE, DataLimits.cqlLimits(3), Util.dk("0"), filter);
+ cfs.metadata(), FBUtilities.nowInSeconds(), ColumnFilter.all(cfs.metadata()), RowFilter.NONE, DataLimits.cqlLimits(3), Util.dk("0"), filter);
assertRowsInResult(cfs, command, 1195, 1196, 1197);
filter = slices(cfs, null, 1996, true);
command = SinglePartitionReadCommand.create(
- cfs.metadata, FBUtilities.nowInSeconds(), ColumnFilter.all(cfs.metadata), RowFilter.NONE, DataLimits.cqlLimits(1000), Util.dk("0"), filter);
+ cfs.metadata(), FBUtilities.nowInSeconds(), ColumnFilter.all(cfs.metadata()), RowFilter.NONE, DataLimits.cqlLimits(1000), Util.dk("0"), filter);
int[] expectedValues = new int[997];
for (int i = 0, v = 1996; v >= 1000; i++, v--)
expectedValues[i] = v;
@@ -474,22 +474,22 @@
filter = slices(cfs, 1990, null, false);
command = SinglePartitionReadCommand.create(
- cfs.metadata, FBUtilities.nowInSeconds(), ColumnFilter.all(cfs.metadata), RowFilter.NONE, DataLimits.cqlLimits(3), Util.dk("0"), filter);
+ cfs.metadata(), FBUtilities.nowInSeconds(), ColumnFilter.all(cfs.metadata()), RowFilter.NONE, DataLimits.cqlLimits(3), Util.dk("0"), filter);
assertRowsInResult(cfs, command, 1990, 1991, 1992);
filter = slices(cfs, null, null, true);
command = SinglePartitionReadCommand.create(
- cfs.metadata, FBUtilities.nowInSeconds(), ColumnFilter.all(cfs.metadata), RowFilter.NONE, DataLimits.cqlLimits(3), Util.dk("0"), filter);
+ cfs.metadata(), FBUtilities.nowInSeconds(), ColumnFilter.all(cfs.metadata()), RowFilter.NONE, DataLimits.cqlLimits(3), Util.dk("0"), filter);
assertRowsInResult(cfs, command, 1999, 1998, 1997);
filter = slices(cfs, null, 9000, true);
command = SinglePartitionReadCommand.create(
- cfs.metadata, FBUtilities.nowInSeconds(), ColumnFilter.all(cfs.metadata), RowFilter.NONE, DataLimits.cqlLimits(3), Util.dk("0"), filter);
+ cfs.metadata(), FBUtilities.nowInSeconds(), ColumnFilter.all(cfs.metadata()), RowFilter.NONE, DataLimits.cqlLimits(3), Util.dk("0"), filter);
assertRowsInResult(cfs, command, 1999, 1998, 1997);
filter = slices(cfs, 9000, null, false);
command = SinglePartitionReadCommand.create(
- cfs.metadata, FBUtilities.nowInSeconds(), ColumnFilter.all(cfs.metadata), RowFilter.NONE, DataLimits.cqlLimits(3), Util.dk("0"), filter);
+ cfs.metadata(), FBUtilities.nowInSeconds(), ColumnFilter.all(cfs.metadata()), RowFilter.NONE, DataLimits.cqlLimits(3), Util.dk("0"), filter);
assertRowsInResult(cfs, command);
}
}
diff --git a/test/unit/org/apache/cassandra/db/LegacyCellNameTest.java b/test/unit/org/apache/cassandra/db/LegacyCellNameTest.java
deleted file mode 100644
index 902c47e..0000000
--- a/test/unit/org/apache/cassandra/db/LegacyCellNameTest.java
+++ /dev/null
@@ -1,112 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.db;
-
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.DatabaseDescriptor;
-
-import static junit.framework.Assert.assertTrue;
-
-public class LegacyCellNameTest
-{
- @BeforeClass
- public static void setupDD()
- {
- DatabaseDescriptor.daemonInitialization();
- }
-
- @Test
- public void testColumnSameNameAsPartitionKeyCompactStorage() throws Exception
- {
- CFMetaData cfm = CFMetaData.compile("CREATE TABLE cs (" +
- "k int PRIMARY KEY, v int)" +
- " WITH COMPACT STORAGE", "ks");
-
- LegacyLayout.LegacyCellName cellName
- = LegacyLayout.decodeCellName(cfm,
- LegacyLayout.makeLegacyComparator(cfm)
- .fromString("k"));
-
- assertTrue(cellName.column.isRegular());
- }
-
- @Test
- public void testColumnSameNameAsClusteringKeyCompactStorage() throws Exception
- {
- CFMetaData cfm = CFMetaData.compile("CREATE TABLE cs (" +
- "k int PRIMARY KEY, v int)" +
- " WITH COMPACT STORAGE", "ks");
-
- LegacyLayout.LegacyCellName cellName
- = LegacyLayout.decodeCellName(cfm,
- LegacyLayout.makeLegacyComparator(cfm)
- .fromString("column1"));
-
- assertTrue(cellName.column.isRegular());
- }
-
- @Test
- public void testColumnSameNameAsPartitionKeyCql3() throws Exception
- {
- CFMetaData cfm = CFMetaData.compile("CREATE TABLE cs (" +
- "k int PRIMARY KEY, v int)", "ks");
-
- LegacyLayout.LegacyCellName cellName
- = LegacyLayout.decodeCellName(cfm,
- LegacyLayout.makeLegacyComparator(cfm)
- .fromString("k"));
-
- // When being grouped into Rows by LegacyLayout.CellGrouper,
- // primary key columns are filtered out
- assertTrue(cellName.column.isPrimaryKeyColumn());
- }
-
- @Test
- public void testCompositeWithColumnNameSameAsClusteringKeyCql3() throws Exception
- {
- CFMetaData cfm = CFMetaData.compile("CREATE TABLE cs (" +
- "k int, c text, v int, PRIMARY KEY(k, c))", "ks");
-
- LegacyLayout.LegacyCellName cellName
- = LegacyLayout.decodeCellName(cfm,
- LegacyLayout.makeLegacyComparator(cfm)
- .fromString("c_value:c"));
-
- // When being grouped into Rows by LegacyLayout.CellGrouper,
- // primary key columns are filtered out
- assertTrue(cellName.column.isPrimaryKeyColumn());
- }
-
- // This throws IllegalArgumentException not because the cellname's value matches
- // the clustering key name, but because when converted to a Composite, the buffer
- // contains only a single component and so has no column name component
- @Test(expected=IllegalArgumentException.class)
- public void testColumnSameNameAsClusteringKeyCql3() throws Exception
- {
- CFMetaData cfm = CFMetaData.compile("CREATE TABLE cs (" +
- "k int, c text, v int, PRIMARY KEY(k, c))", "ks");
-
- LegacyLayout.LegacyCellName cellName
- = LegacyLayout.decodeCellName(cfm,
- LegacyLayout.makeLegacyComparator(cfm)
- .fromString("c"));
- }
-}
diff --git a/test/unit/org/apache/cassandra/db/LegacyLayoutTest.java b/test/unit/org/apache/cassandra/db/LegacyLayoutTest.java
deleted file mode 100644
index 65565e1..0000000
--- a/test/unit/org/apache/cassandra/db/LegacyLayoutTest.java
+++ /dev/null
@@ -1,486 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.db;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-
-import org.junit.AfterClass;
-import org.apache.cassandra.db.LegacyLayout.CellGrouper;
-import org.apache.cassandra.db.LegacyLayout.LegacyBound;
-import org.apache.cassandra.db.LegacyLayout.LegacyCell;
-import org.apache.cassandra.db.LegacyLayout.LegacyRangeTombstone;
-import org.apache.cassandra.db.filter.ColumnFilter;
-import org.apache.cassandra.db.marshal.MapType;
-import org.apache.cassandra.db.marshal.UTF8Type;
-import org.apache.cassandra.db.rows.BufferCell;
-import org.apache.cassandra.db.rows.Cell;
-import org.apache.cassandra.db.rows.RowIterator;
-import org.apache.cassandra.db.rows.SerializationHelper;
-import org.apache.cassandra.db.rows.UnfilteredRowIterator;
-import org.apache.cassandra.db.rows.UnfilteredRowIteratorSerializer;
-import org.apache.cassandra.db.transform.FilteredRows;
-import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.io.util.DataInputBuffer;
-import org.apache.cassandra.io.util.DataOutputBuffer;
-import org.apache.cassandra.net.MessagingService;
-import org.apache.cassandra.serializers.Int32Serializer;
-import org.apache.cassandra.serializers.UTF8Serializer;
-import org.apache.cassandra.service.MigrationManager;
-import org.apache.cassandra.utils.FBUtilities;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import org.apache.cassandra.SchemaLoader;
-import org.apache.cassandra.Util;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.cql3.ColumnIdentifier;
-import org.apache.cassandra.cql3.QueryProcessor;
-import org.apache.cassandra.cql3.UntypedResultSet;
-import org.apache.cassandra.db.marshal.Int32Type;
-import org.apache.cassandra.db.marshal.SetType;
-import org.apache.cassandra.db.partitions.PartitionUpdate;
-import org.apache.cassandra.db.rows.BTreeRow;
-import org.apache.cassandra.db.rows.Row;
-import org.apache.cassandra.dht.Murmur3Partitioner;
-import org.apache.cassandra.schema.KeyspaceParams;
-import org.apache.cassandra.utils.Hex;
-
-import static org.apache.cassandra.net.MessagingService.VERSION_21;
-import static org.apache.cassandra.utils.ByteBufferUtil.bytes;
-import static org.junit.Assert.*;
-
-public class LegacyLayoutTest
-{
- static Util.PartitionerSwitcher sw;
- static String KEYSPACE = "Keyspace1";
-
- @BeforeClass
- public static void defineSchema() throws ConfigurationException
- {
- DatabaseDescriptor.daemonInitialization();
- sw = Util.switchPartitioner(Murmur3Partitioner.instance);
- SchemaLoader.loadSchema();
- SchemaLoader.createKeyspace(KEYSPACE, KeyspaceParams.simple(1));
- }
-
- @AfterClass
- public static void resetPartitioner()
- {
- sw.close();
- }
-
- @Test
- public void testFromUnfilteredRowIterator() throws Throwable
- {
- CFMetaData table = CFMetaData.Builder.create("ks", "table")
- .withPartitioner(Murmur3Partitioner.instance)
- .addPartitionKey("k", Int32Type.instance)
- .addRegularColumn("a", SetType.getInstance(Int32Type.instance, true))
- .addRegularColumn("b", SetType.getInstance(Int32Type.instance, true))
- .build();
-
- ColumnDefinition a = table.getColumnDefinition(new ColumnIdentifier("a", false));
- ColumnDefinition b = table.getColumnDefinition(new ColumnIdentifier("b", false));
-
- Row.Builder builder = BTreeRow.unsortedBuilder(0);
- builder.newRow(Clustering.EMPTY);
- builder.addComplexDeletion(a, new DeletionTime(1L, 1));
- builder.addComplexDeletion(b, new DeletionTime(1L, 1));
- Row row = builder.build();
-
- ByteBuffer key = bytes(1);
- PartitionUpdate upd = PartitionUpdate.singleRowUpdate(table, key, row);
-
- LegacyLayout.LegacyUnfilteredPartition p = LegacyLayout.fromUnfilteredRowIterator(null, upd.unfilteredIterator());
- assertEquals(DeletionTime.LIVE, p.partitionDeletion);
- assertEquals(0, p.cells.size());
-
- LegacyLayout.LegacyRangeTombstoneList l = p.rangeTombstones;
- assertEquals("a", l.starts[0].collectionName.name.toString());
- assertEquals("a", l.ends[0].collectionName.name.toString());
-
- assertEquals("b", l.starts[1].collectionName.name.toString());
- assertEquals("b", l.ends[1].collectionName.name.toString());
- }
-
- /**
- * Tests with valid sstables containing duplicate RT entries at index boundaries
- * in 2.1 format, where DATA below is a > 1000 byte long string of letters,
- * and the column index is set to 1kb
-
- [
- {"key": "1",
- "cells": [["1:_","1:!",1513015245,"t",1513015263],
- ["1:1:","",1513015467727335],
- ["1:1:val1","DATA",1513015467727335],
- ["1:_","1:!",1513015245,"t",1513015263],
- ["1:1:val2","DATA",1513015467727335],
- ["1:_","1:!",1513015245,"t",1513015263],
- ["1:1:val3","DATA",1513015467727335],
- ["1:_","1:!",1513015245,"t",1513015263],
- ["1:2:","",1513015458470156],
- ["1:2:val1","DATA",1513015458470156],
- ["1:_","1:!",1513015245,"t",1513015263],
- ["1:2:val2","DATA",1513015458470156],
- ["1:_","1:!",1513015245,"t",1513015263],
- ["1:2:val3","DATA",1513015458470156],
- ["1:_","1:!",1513015245,"t",1513015263],
- ["1:3:","",1513015450253602],
- ["1:3:val1","DATA",1513015450253602],
- ["1:_","1:!",1513015245,"t",1513015263],
- ["1:3:val2","DATA",1513015450253602],
- ["1:_","1:!",1513015245,"t",1513015263],
- ["1:3:val3","DATA",1513015450253602]]}
- ]
- *
- * See CASSANDRA-14008 for details.
- */
- @Test
- public void testRTBetweenColumns() throws Throwable
- {
- QueryProcessor.executeInternal(String.format("CREATE TABLE \"%s\".legacy_ka_repeated_rt (k1 int, c1 int, c2 int, val1 text, val2 text, val3 text, primary key (k1, c1, c2))", KEYSPACE));
-
- Keyspace keyspace = Keyspace.open(KEYSPACE);
- ColumnFamilyStore cfs = keyspace.getColumnFamilyStore("legacy_ka_repeated_rt");
-
- Path legacySSTableRoot = Paths.get("test/data/legacy-sstables/ka/legacy_tables/legacy_ka_repeated_rt/");
-
- for (String filename : new String[]{ "Keyspace1-legacy_ka_repeated_rt-ka-1-CompressionInfo.db",
- "Keyspace1-legacy_ka_repeated_rt-ka-1-Data.db",
- "Keyspace1-legacy_ka_repeated_rt-ka-1-Digest.sha1",
- "Keyspace1-legacy_ka_repeated_rt-ka-1-Filter.db",
- "Keyspace1-legacy_ka_repeated_rt-ka-1-Index.db",
- "Keyspace1-legacy_ka_repeated_rt-ka-1-Statistics.db",
- "Keyspace1-legacy_ka_repeated_rt-ka-1-Summary.db",
- "Keyspace1-legacy_ka_repeated_rt-ka-1-TOC.txt" })
- {
- Files.copy(Paths.get(legacySSTableRoot.toString(), filename), cfs.getDirectories().getDirectoryForNewSSTables().toPath().resolve(filename));
- }
-
- cfs.loadNewSSTables();
-
- UntypedResultSet rs = QueryProcessor.executeInternal(String.format("SELECT * FROM \"%s\".legacy_ka_repeated_rt WHERE k1=1", KEYSPACE));
- assertEquals(3, rs.size());
-
- UntypedResultSet rs2 = QueryProcessor.executeInternal(String.format("SELECT * FROM \"%s\".legacy_ka_repeated_rt WHERE k1=1 AND c1=1", KEYSPACE));
- assertEquals(3, rs2.size());
-
- for (int i = 1; i <= 3; i++)
- {
- UntypedResultSet rs3 = QueryProcessor.executeInternal(String.format("SELECT * FROM \"%s\".legacy_ka_repeated_rt WHERE k1=1 AND c1=1 AND c2=%s", KEYSPACE, i));
- assertEquals(1, rs3.size());
- }
-
- }
-
-
- private static UnfilteredRowIterator roundTripVia21(UnfilteredRowIterator partition) throws IOException
- {
- try (DataOutputBuffer out = new DataOutputBuffer())
- {
- LegacyLayout.serializeAsLegacyPartition(null, partition, out, VERSION_21);
- try (DataInputBuffer in = new DataInputBuffer(out.buffer(), false))
- {
- return LegacyLayout.deserializeLegacyPartition(in, VERSION_21, SerializationHelper.Flag.LOCAL, partition.partitionKey().getKey());
- }
- }
- }
-
- @Test
- public void testStaticRangeTombstoneRoundTripUnexpectedDeletion() throws Throwable
- {
- // this variant of the bug deletes a row with the same clustering key value as the name of the static collection
- QueryProcessor.executeInternal(String.format("CREATE TABLE \"%s\".legacy_static_rt_rt_1 (pk int, ck1 text, ck2 text, v int, s set<text> static, primary key (pk, ck1, ck2))", KEYSPACE));
- Keyspace keyspace = Keyspace.open(KEYSPACE);
- CFMetaData table = keyspace.getColumnFamilyStore("legacy_static_rt_rt_1").metadata;
- ColumnDefinition v = table.getColumnDefinition(new ColumnIdentifier("v", false));
- ColumnDefinition bug = table.getColumnDefinition(new ColumnIdentifier("s", false));
-
- Row.Builder builder;
- builder = BTreeRow.unsortedBuilder(0);
- builder.newRow(Clustering.STATIC_CLUSTERING);
- builder.addComplexDeletion(bug, new DeletionTime(1L, 1));
- Row staticRow = builder.build();
-
- builder = BTreeRow.unsortedBuilder(0);
- builder.newRow(new BufferClustering(UTF8Serializer.instance.serialize("s"), UTF8Serializer.instance.serialize("anything")));
- builder.addCell(new BufferCell(v, 1L, Cell.NO_TTL, Cell.NO_DELETION_TIME, Int32Serializer.instance.serialize(1), null));
- Row row = builder.build();
-
- DecoratedKey pk = table.decorateKey(bytes(1));
- PartitionUpdate upd = PartitionUpdate.singleRowUpdate(table, pk, row, staticRow);
-
- try (RowIterator before = FilteredRows.filter(upd.unfilteredIterator(), FBUtilities.nowInSeconds());
- RowIterator after = FilteredRows.filter(roundTripVia21(upd.unfilteredIterator()), FBUtilities.nowInSeconds()))
- {
- while (before.hasNext() || after.hasNext())
- assertEquals(before.hasNext() ? before.next() : null, after.hasNext() ? after.next() : null);
- }
- }
-
- @Test
- public void testStaticRangeTombstoneRoundTripCorruptRead() throws Throwable
- {
- // this variant of the bug corrupts the byte stream of the partition, so that a sequential read starting before
- // this partition will fail with a CorruptSSTableException, and possible yield junk results
- QueryProcessor.executeInternal(String.format("CREATE TABLE \"%s\".legacy_static_rt_rt_2 (pk int, ck int, nameWithLengthGreaterThan4 set<int> static, primary key (pk, ck))", KEYSPACE));
- Keyspace keyspace = Keyspace.open(KEYSPACE);
- CFMetaData table = keyspace.getColumnFamilyStore("legacy_static_rt_rt_2").metadata;
-
- ColumnDefinition bug = table.getColumnDefinition(new ColumnIdentifier("nameWithLengthGreaterThan4", false));
-
- Row.Builder builder = BTreeRow.unsortedBuilder(0);
- builder.newRow(Clustering.STATIC_CLUSTERING);
- builder.addComplexDeletion(bug, new DeletionTime(1L, 1));
- Row row = builder.build();
-
- DecoratedKey pk = table.decorateKey(bytes(1));
- PartitionUpdate upd = PartitionUpdate.singleRowUpdate(table, pk, row);
-
- UnfilteredRowIterator afterRoundTripVia32 = roundTripVia21(upd.unfilteredIterator());
- try (DataOutputBuffer out = new DataOutputBuffer())
- {
- // we only encounter a corruption/serialization error after writing this to a 3.0 format and reading it back
- UnfilteredRowIteratorSerializer.serializer.serialize(afterRoundTripVia32, ColumnFilter.all(table), out, MessagingService.current_version);
- try (DataInputBuffer in = new DataInputBuffer(out.buffer(), false);
- UnfilteredRowIterator afterSerialization = UnfilteredRowIteratorSerializer.serializer.deserialize(in, MessagingService.current_version, table, ColumnFilter.all(table), SerializationHelper.Flag.LOCAL))
- {
- while (afterSerialization.hasNext())
- afterSerialization.next();
- }
- }
- }
-
- @Test
- public void testCollectionDeletionRoundTripForDroppedColumn() throws Throwable
- {
- // this variant of the bug deletes a row with the same clustering key value as the name of the static collection
- QueryProcessor.executeInternal(String.format("CREATE TABLE \"%s\".legacy_rt_rt_dc (pk int, ck1 text, v int, s set<text>, primary key (pk, ck1))", KEYSPACE));
- Keyspace keyspace = Keyspace.open(KEYSPACE);
- CFMetaData table = keyspace.getColumnFamilyStore("legacy_rt_rt_dc").metadata;
- ColumnDefinition v = table.getColumnDefinition(new ColumnIdentifier("v", false));
- ColumnDefinition bug = table.getColumnDefinition(new ColumnIdentifier("s", false));
-
- Row.Builder builder;
- builder = BTreeRow.unsortedBuilder(0);
- builder.newRow(new BufferClustering(UTF8Serializer.instance.serialize("a")));
- builder.addCell(BufferCell.live(v, 0L, Int32Serializer.instance.serialize(1), null));
- builder.addComplexDeletion(bug, new DeletionTime(1L, 1));
- Row row = builder.build();
-
- DecoratedKey pk = table.decorateKey(bytes(1));
- PartitionUpdate upd = PartitionUpdate.singleRowUpdate(table, pk, row);
-
- // we need to perform the round trip in two parts here, with a column drop inbetween
- try (RowIterator before = FilteredRows.filter(upd.unfilteredIterator(), FBUtilities.nowInSeconds());
- DataOutputBuffer serialized21 = new DataOutputBuffer())
- {
- LegacyLayout.serializeAsLegacyPartition(null, upd.unfilteredIterator(), serialized21, VERSION_21);
- QueryProcessor.executeInternal(String.format("ALTER TABLE \"%s\".legacy_rt_rt_dc DROP s", KEYSPACE));
- try (DataInputBuffer in = new DataInputBuffer(serialized21.buffer(), false))
- {
- try (UnfilteredRowIterator deser21 = LegacyLayout.deserializeLegacyPartition(in, VERSION_21, SerializationHelper.Flag.LOCAL, upd.partitionKey().getKey());
- RowIterator after = FilteredRows.filter(deser21, FBUtilities.nowInSeconds());)
- {
- while (before.hasNext() || after.hasNext())
- assertEquals(before.hasNext() ? before.next() : null, after.hasNext() ? after.next() : null);
- }
- }
-
- }
- }
-
- @Test
- public void testDecodeLegacyPagedRangeCommandSerializer() throws IOException
- {
- /*
- Run on 2.1
- public static void main(String[] args) throws IOException, ConfigurationException
- {
- Gossiper.instance.start((int) (System.currentTimeMillis() / 1000));
- Keyspace.setInitialized();
- CFMetaData cfMetaData = CFMetaData.sparseCFMetaData("ks", "cf", UTF8Type.instance)
- .addColumnDefinition(new ColumnDefinition("ks", "cf", new ColumnIdentifier("v", true), SetType.getInstance(Int32Type.instance, false), null, null, null, null, ColumnDefinition.Kind.REGULAR));
- KSMetaData ksMetaData = KSMetaData.testMetadata("ks", SimpleStrategy.class, KSMetaData.optsWithRF(3), cfMetaData);
- MigrationManager.announceNewKeyspace(ksMetaData);
- RowPosition position = RowPosition.ForKey.get(ByteBufferUtil.EMPTY_BYTE_BUFFER, new Murmur3Partitioner());
- SliceQueryFilter filter = new IdentityQueryFilter();
- Composite cellName = CellNames.compositeSparseWithCollection(new ByteBuffer[0], Int32Type.instance.decompose(1), new ColumnIdentifier("v", true), false);
- try (DataOutputBuffer buffer = new DataOutputBuffer(1024))
- {
- PagedRangeCommand command = new PagedRangeCommand("ks", "cf", 1, AbstractBounds.bounds(position, true, position, true), filter, cellName, filter.finish(), Collections.emptyList(), 1, true);
- PagedRangeCommand.serializer.serialize(command, buffer, MessagingService.current_version);
- System.out.println(Hex.bytesToHex(buffer.toByteArray()));
- }
- }
- */
-
- DatabaseDescriptor.daemonInitialization();
- Keyspace.setInitialized();
- CFMetaData table = CFMetaData.Builder.create("ks", "cf")
- .addPartitionKey("k", Int32Type.instance)
- .addRegularColumn("v", SetType.getInstance(Int32Type.instance, true))
- .build();
- SchemaLoader.createKeyspace("ks", KeyspaceParams.simple(1));
- MigrationManager.announceNewColumnFamily(table);
-
- byte[] bytes = Hex.hexToBytes("00026b73000263660000000000000001fffffffe01000000088000000000000000010000000880000000000000000000000100000000007fffffffffffffff000b00017600000400000001000000000000000000000101");
- ReadCommand.legacyPagedRangeCommandSerializer.deserialize(new DataInputBuffer(bytes), VERSION_21);
- }
-
- @Test
- public void testDecodeCollectionPageBoundary()
- {
- CFMetaData table = CFMetaData.Builder.create("ks", "cf")
- .addPartitionKey("k", Int32Type.instance)
- .addRegularColumn("v", SetType.getInstance(Int32Type.instance, true))
- .build();
-
- ColumnDefinition v = table.getColumnDefinition(new ColumnIdentifier("v", false));
- ByteBuffer bound = LegacyLayout.encodeCellName(table, Clustering.EMPTY, v.name.bytes, Int32Type.instance.decompose(1));
-
- LegacyLayout.decodeSliceBound(table, bound, true);
- }
-
- @Test
- public void testAsymmetricRTBoundSerializedSize()
- {
- CFMetaData table = CFMetaData.Builder.create("ks", "cf")
- .addPartitionKey("k", Int32Type.instance)
- .addClusteringColumn("c1", Int32Type.instance)
- .addClusteringColumn("c2", Int32Type.instance)
- .addRegularColumn("v", Int32Type.instance)
- .build();
-
- ByteBuffer one = Int32Type.instance.decompose(1);
- ByteBuffer two = Int32Type.instance.decompose(2);
- PartitionUpdate p = new PartitionUpdate(table, table.decorateKey(one), table.partitionColumns(), 0);
- p.add(new RangeTombstone(Slice.make(new ClusteringBound(ClusteringPrefix.Kind.EXCL_START_BOUND, new ByteBuffer[] { one, one }),
- new ClusteringBound(ClusteringPrefix.Kind.INCL_END_BOUND, new ByteBuffer[] { two })),
- new DeletionTime(1, 1)
- ));
-
- LegacyLayout.fromUnfilteredRowIterator(null, p.unfilteredIterator());
- LegacyLayout.serializedSizeAsLegacyPartition(null, p.unfilteredIterator(), VERSION_21);
- }
-
- @Test
- public void testCellGrouper()
- {
- // CREATE TABLE %s (pk int, ck int, v map<text, text>, PRIMARY KEY (pk, ck))
- CFMetaData cfm = CFMetaData.Builder.create("ks", "table")
- .addPartitionKey("pk", Int32Type.instance)
- .addClusteringColumn("ck", Int32Type.instance)
- .addRegularColumn("v", MapType.getInstance(UTF8Type.instance, UTF8Type.instance, true))
- .build();
- SerializationHelper helper = new SerializationHelper(cfm, MessagingService.VERSION_22, SerializationHelper.Flag.LOCAL, ColumnFilter.all(cfm));
- LegacyLayout.CellGrouper cg = new LegacyLayout.CellGrouper(cfm, helper);
-
- ClusteringBound startBound = ClusteringBound.create(ClusteringPrefix.Kind.INCL_START_BOUND, new ByteBuffer[] {bytes(2)});
- ClusteringBound endBound = ClusteringBound.create(ClusteringPrefix.Kind.EXCL_END_BOUND, new ByteBuffer[] {bytes(2)});
- LegacyLayout.LegacyBound start = new LegacyLayout.LegacyBound(startBound, false, cfm.getColumnDefinition(bytes("v")));
- LegacyLayout.LegacyBound end = new LegacyLayout.LegacyBound(endBound, false, cfm.getColumnDefinition(bytes("v")));
- LegacyLayout.LegacyRangeTombstone lrt = new LegacyLayout.LegacyRangeTombstone(start, end, new DeletionTime(2, 1588598040));
- assertTrue(cg.addAtom(lrt));
-
- // add a real cell
- LegacyLayout.LegacyCell cell = new LegacyLayout.LegacyCell(LegacyLayout.LegacyCell.Kind.REGULAR,
- new LegacyLayout.LegacyCellName(Clustering.make(bytes(2)),
- cfm.getColumnDefinition(bytes("v")),
- bytes("g")),
- bytes("v"), 3, Integer.MAX_VALUE, 0);
- assertTrue(cg.addAtom(cell));
-
- // add legacy range tombstone where collection name is null for the end bound (this gets translated to a row tombstone)
- startBound = ClusteringBound.create(ClusteringPrefix.Kind.EXCL_START_BOUND, new ByteBuffer[] {bytes(2)});
- endBound = ClusteringBound.create(ClusteringPrefix.Kind.EXCL_END_BOUND, new ByteBuffer[] {bytes(2)});
- start = new LegacyLayout.LegacyBound(startBound, false, cfm.getColumnDefinition(bytes("v")));
- end = new LegacyLayout.LegacyBound(endBound, false, null);
- assertTrue(cg.addAtom(new LegacyLayout.LegacyRangeTombstone(start, end, new DeletionTime(1, 1588598040))));
- }
-
- private static LegacyCell cell(Clustering clustering, ColumnDefinition column, ByteBuffer value, long timestamp)
- {
- return new LegacyCell(LegacyCell.Kind.REGULAR,
- new LegacyLayout.LegacyCellName(clustering, column, null),
- value,
- timestamp,
- Cell.NO_DELETION_TIME,
- Cell.NO_TTL);
- }
-
- /**
- * This tests that when {@link CellGrouper} gets a collection tombstone for
- * a non-fetched collection, then that tombstone does not incorrectly stop the grouping of the current row, as
- * was done before CASSANDRA-15805.
- *
- * <p>Please note that this rely on a query only _fetching_ some of the table columns, which in practice only
- * happens for thrift queries, and thrift queries shouldn't mess up with CQL tables and collection tombstones,
- * so this test is not of the utmost importance. Nonetheless, the pre-CASSANDRA-15805 behavior was incorrect and
- * this ensure it is fixed.
- */
- @Test
- public void testCellGrouperOnNonFecthedCollectionTombstone()
- {
- // CREATE TABLE %s (pk int, ck int, a text, b set<text>, c text, PRIMARY KEY (pk, ck))
- CFMetaData cfm = CFMetaData.Builder.create("ks", "table")
- .addPartitionKey("pk", Int32Type.instance)
- .addClusteringColumn("ck", Int32Type.instance)
- .addRegularColumn("a", UTF8Type.instance)
- .addRegularColumn("b", SetType.getInstance(UTF8Type.instance, true))
- .addRegularColumn("c", UTF8Type.instance)
- .build();
-
- // Creates a filter that _only_ fetches a and c, but not b.
- ColumnFilter filter = ColumnFilter.selectionBuilder()
- .add(cfm.getColumnDefinition(bytes("a")))
- .add(cfm.getColumnDefinition(bytes("c")))
- .build();
- SerializationHelper helper = new SerializationHelper(cfm,
- MessagingService.VERSION_22,
- SerializationHelper.Flag.LOCAL,
- filter);
- CellGrouper grouper = new CellGrouper(cfm, helper);
- Clustering clustering = new BufferClustering(bytes(1));
-
- // We add a cell for a, then a collection tombstone for b, and then a cell for c (for the same clustering).
- // All those additions should return 'true' as all belong to the same row.
- LegacyCell ca = cell(clustering, cfm.getColumnDefinition(bytes("a")), bytes("v1"), 1);
- assertTrue(grouper.addAtom(ca));
-
- ClusteringBound startBound = ClusteringBound.inclusiveStartOf(bytes(1));
- ClusteringBound endBound = ClusteringBound.inclusiveEndOf(bytes(1));
- ColumnDefinition bDef = cfm.getColumnDefinition(bytes("b"));
- assert bDef != null;
- LegacyBound start = new LegacyBound(startBound, false, bDef);
- LegacyBound end = new LegacyBound(endBound, false, bDef);
- LegacyRangeTombstone rtb = new LegacyRangeTombstone(start, end, new DeletionTime(1, 1588598040));
- assertTrue(rtb.isCollectionTombstone()); // Ensure we're testing what we think
- assertTrue(grouper.addAtom(rtb));
-
- LegacyCell cc = cell(clustering, cfm.getColumnDefinition(bytes("c")), bytes("v2"), 1);
- assertTrue(grouper.addAtom(cc));
- }
-}
diff --git a/test/unit/org/apache/cassandra/db/LegacyLayoutValidationTest.java b/test/unit/org/apache/cassandra/db/LegacyLayoutValidationTest.java
deleted file mode 100644
index 4d565ca..0000000
--- a/test/unit/org/apache/cassandra/db/LegacyLayoutValidationTest.java
+++ /dev/null
@@ -1,226 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.db;
-
-import java.nio.ByteBuffer;
-import java.util.Iterator;
-
-import com.google.common.collect.Iterators;
-import org.junit.Test;
-
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.cql3.ColumnIdentifier;
-import org.apache.cassandra.db.LegacyLayout.LegacyCell;
-import org.apache.cassandra.db.LegacyLayout.LegacyCellName;
-import org.apache.cassandra.db.marshal.Int32Type;
-import org.apache.cassandra.db.marshal.UTF8Type;
-import org.apache.cassandra.db.rows.UnfilteredRowIterator;
-import org.apache.cassandra.serializers.MarshalException;
-
-import static org.apache.cassandra.utils.ByteBufferUtil.hexToBytes;
-
-public class LegacyLayoutValidationTest
-{
- static final String KEYSPACE = "ks";
-
- static
- {
- DatabaseDescriptor.daemonInitialization();
- }
-
- private static final CFMetaData FIXED = CFMetaData.Builder.create("ks", "cf")
- .addPartitionKey("k", Int32Type.instance)
- .addClusteringColumn("c1", Int32Type.instance)
- .addClusteringColumn("c2", Int32Type.instance)
- .addRegularColumn("v1", Int32Type.instance)
- .addRegularColumn("v2", Int32Type.instance)
- .build();
-
- private static final CFMetaData COMPACT_FIXED = CFMetaData.Builder.create("ks", "cf", true, false, false)
- .addPartitionKey("k", Int32Type.instance)
- .addClusteringColumn("c", Int32Type.instance)
- .addRegularColumn("v", Int32Type.instance)
- .build();
-
- private static final CFMetaData VARIABLE = CFMetaData.Builder.create("ks", "cf")
- .addPartitionKey("k", Int32Type.instance)
- .addClusteringColumn("c1", UTF8Type.instance)
- .addClusteringColumn("c2", UTF8Type.instance)
- .addRegularColumn("v1", UTF8Type.instance)
- .addRegularColumn("v2", UTF8Type.instance)
- .build();
-
- private static final CFMetaData COMPACT_VARIABLE = CFMetaData.Builder.create("ks", "cf", true, false, false)
- .addPartitionKey("k", Int32Type.instance)
- .addClusteringColumn("c", UTF8Type.instance)
- .addRegularColumn("v", UTF8Type.instance)
- .build();
-
- @Test
- public void fixedClusteringSuccess()
- {
- Clustering clustering = Clustering.make(Int32Type.instance.decompose(1), Int32Type.instance.decompose(2));
- ByteBuffer serialized = LegacyLayout.encodeClustering(FIXED, clustering);
- LegacyLayout.decodeClustering(FIXED, serialized);
- }
-
- @Test (expected = MarshalException.class)
- public void fixedClusteringFailure()
- {
- Clustering clustering = Clustering.make(Int32Type.instance.decompose(1), hexToBytes("07000000000001"));
- ByteBuffer serialized = LegacyLayout.encodeClustering(FIXED, clustering);
- LegacyLayout.decodeClustering(FIXED, serialized);
- }
-
- @Test
- public void variableClusteringSuccess()
- {
- Clustering clustering = Clustering.make(UTF8Type.instance.decompose("one"), UTF8Type.instance.decompose("two,three"));
- ByteBuffer serialized = LegacyLayout.encodeClustering(VARIABLE, clustering);
- LegacyLayout.decodeClustering(VARIABLE, serialized);
- }
-
- @Test
- public void fixedCompactClusteringSuccess()
- {
- Clustering clustering = Clustering.make(Int32Type.instance.decompose(2));
- ByteBuffer serialized = LegacyLayout.encodeClustering(COMPACT_FIXED, clustering);
- LegacyLayout.decodeClustering(COMPACT_FIXED, serialized);
- }
-
- @Test (expected = MarshalException.class)
- public void fixedCompactClusteringFailure()
- {
- Clustering clustering = Clustering.make(hexToBytes("07000000000001"));
- ByteBuffer serialized = LegacyLayout.encodeClustering(COMPACT_FIXED, clustering);
- LegacyLayout.decodeClustering(COMPACT_FIXED, serialized);
- }
-
- @Test
- public void variableCompactClusteringSuccess()
- {
- Clustering clustering = Clustering.make(UTF8Type.instance.decompose("two,three"));
- ByteBuffer serialized = LegacyLayout.encodeClustering(COMPACT_VARIABLE, clustering);
- LegacyLayout.decodeClustering(COMPACT_VARIABLE, serialized);
- }
-
- @Test
- public void fixedBoundSuccess()
- {
- Clustering clustering = Clustering.make(Int32Type.instance.decompose(1), Int32Type.instance.decompose(2));
- ByteBuffer serialized = LegacyLayout.encodeClustering(FIXED, clustering);
- LegacyLayout.decodeSliceBound(FIXED, serialized, true);
- }
-
- @Test (expected = MarshalException.class)
- public void fixedBoundFailure()
- {
- Clustering clustering = Clustering.make(Int32Type.instance.decompose(1), hexToBytes("07000000000001"));
- ByteBuffer serialized = LegacyLayout.encodeClustering(FIXED, clustering);
- LegacyLayout.decodeSliceBound(FIXED, serialized, true);
- }
-
- @Test
- public void variableBoundSuccess()
- {
- Clustering clustering = Clustering.make(UTF8Type.instance.decompose("one"), UTF8Type.instance.decompose("two,three"));
- ByteBuffer serialized = LegacyLayout.encodeClustering(VARIABLE, clustering);
- LegacyLayout.decodeSliceBound(VARIABLE, serialized, true);
- }
-
- @Test
- public void fixedCompactBoundSuccess()
- {
- Clustering clustering = Clustering.make(Int32Type.instance.decompose(1));
- ByteBuffer serialized = LegacyLayout.encodeClustering(COMPACT_FIXED, clustering);
- LegacyLayout.decodeSliceBound(COMPACT_FIXED, serialized, true);
- }
-
- @Test (expected = MarshalException.class)
- public void fixedCompactBoundFailure()
- {
- Clustering clustering = Clustering.make(hexToBytes("07000000000001"));
- ByteBuffer serialized = LegacyLayout.encodeClustering(COMPACT_FIXED, clustering);
- LegacyLayout.decodeSliceBound(COMPACT_FIXED, serialized, true);
- }
-
- @Test
- public void variableCompactBoundSuccess()
- {
- Clustering clustering = Clustering.make(UTF8Type.instance.decompose("one"));
- ByteBuffer serialized = LegacyLayout.encodeClustering(COMPACT_VARIABLE, clustering);
- LegacyLayout.decodeSliceBound(COMPACT_VARIABLE, serialized, true);
- }
-
- private static LegacyCell cell(CFMetaData cfm, Clustering clustering, String name, ByteBuffer value) throws UnknownColumnException
- {
- ColumnDefinition definition = cfm.getColumnDefinition(new ColumnIdentifier(name, false));
-
- ByteBuffer cellName = LegacyCellName.create(clustering, definition).encode(cfm);
- return LegacyCell.regular(cfm, null, cellName, value, 0);
-
- }
-
- @Test
- public void fixedValueSuccess() throws Throwable
- {
- DecoratedKey dk = DatabaseDescriptor.getPartitioner().decorateKey(Int32Type.instance.decompose(1000000));
- LegacyLayout.LegacyDeletionInfo deletionInfo = LegacyLayout.LegacyDeletionInfo.live();
- Clustering clustering = Clustering.make(Int32Type.instance.decompose(1), Int32Type.instance.decompose(2));
- Iterator<LegacyCell> cells = Iterators.forArray(cell(FIXED, clustering, "v1", Int32Type.instance.decompose(3)),
- cell(FIXED, clustering, "v2", Int32Type.instance.decompose(4)));
- try (UnfilteredRowIterator iter = LegacyLayout.toUnfilteredRowIterator(FIXED, dk, deletionInfo, cells))
- {
- while (iter.hasNext())
- iter.next();
- }
- }
-
- @Test (expected = MarshalException.class)
- public void fixedValueFailure() throws Throwable
- {
- DecoratedKey dk = DatabaseDescriptor.getPartitioner().decorateKey(Int32Type.instance.decompose(1000000));
- LegacyLayout.LegacyDeletionInfo deletionInfo = LegacyLayout.LegacyDeletionInfo.live();
- Clustering clustering = Clustering.make(Int32Type.instance.decompose(1), Int32Type.instance.decompose(2));
- Iterator<LegacyCell> cells = Iterators.forArray(cell(FIXED, clustering, "v1", Int32Type.instance.decompose(3)),
- cell(FIXED, clustering, "v2", hexToBytes("0000")));
- try (UnfilteredRowIterator iter = LegacyLayout.toUnfilteredRowIterator(FIXED, dk, deletionInfo, cells))
- {
- while (iter.hasNext())
- iter.next();
- }
- }
-
- @Test
- public void variableValueSuccess() throws Throwable
- {
- DecoratedKey dk = DatabaseDescriptor.getPartitioner().decorateKey(Int32Type.instance.decompose(1000000));
- LegacyLayout.LegacyDeletionInfo deletionInfo = LegacyLayout.LegacyDeletionInfo.live();
- Clustering clustering = Clustering.make(Int32Type.instance.decompose(1), Int32Type.instance.decompose(2));
- Iterator<LegacyCell> cells = Iterators.forArray(cell(VARIABLE, clustering, "v1", UTF8Type.instance.decompose("3")),
- cell(VARIABLE, clustering, "v2", hexToBytes("0000")));
- try (UnfilteredRowIterator iter = LegacyLayout.toUnfilteredRowIterator(VARIABLE, dk, deletionInfo, cells))
- {
- while (iter.hasNext())
- iter.next();
- }
- }
-}
diff --git a/test/unit/org/apache/cassandra/db/MmapFileTest.java b/test/unit/org/apache/cassandra/db/MmapFileTest.java
index 0c67ff7..71a218e 100644
--- a/test/unit/org/apache/cassandra/db/MmapFileTest.java
+++ b/test/unit/org/apache/cassandra/db/MmapFileTest.java
@@ -29,7 +29,7 @@
import org.junit.Assert;
import org.junit.Test;
-import sun.nio.ch.DirectBuffer;
+import org.apache.cassandra.io.util.FileUtils;
public class MmapFileTest
{
@@ -48,9 +48,9 @@
Assert.assertEquals("# of mapped buffers should be 0", Long.valueOf(0L), mmapCount);
Assert.assertEquals("amount of mapped memory should be 0", Long.valueOf(0L), mmapMemoryUsed);
- File f1 = File.createTempFile("MmapFileTest1", ".bin");
- File f2 = File.createTempFile("MmapFileTest2", ".bin");
- File f3 = File.createTempFile("MmapFileTest2", ".bin");
+ File f1 = FileUtils.createTempFile("MmapFileTest1", ".bin");
+ File f2 = FileUtils.createTempFile("MmapFileTest2", ".bin");
+ File f3 = FileUtils.createTempFile("MmapFileTest2", ".bin");
try
{
@@ -87,8 +87,8 @@
buffer.putInt(42);
buffer.putInt(42);
buffer.putInt(42);
-
- ((DirectBuffer) buffer).cleaner().clean();
+
+ FileUtils.clean(buffer);
}
mmapCount = (Long) mbs.getAttribute(bpmName, "Count");
@@ -115,7 +115,7 @@
buffer.putInt(42);
buffer.putInt(42);
- ((DirectBuffer) buffer).cleaner().clean();
+ FileUtils.clean(buffer);
}
mmapCount = (Long) mbs.getAttribute(bpmName, "Count");
@@ -140,7 +140,7 @@
buffer.putInt(42);
buffer.putInt(42);
- ((DirectBuffer) buffer).cleaner().clean();
+ FileUtils.clean(buffer);
}
mmapCount = (Long) mbs.getAttribute(bpmName, "Count");
diff --git a/test/unit/org/apache/cassandra/db/MutationExceededMaxSizeExceptionTest.java b/test/unit/org/apache/cassandra/db/MutationExceededMaxSizeExceptionTest.java
new file mode 100644
index 0000000..81d9735
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/MutationExceededMaxSizeExceptionTest.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import org.junit.Test;
+
+import static org.apache.cassandra.db.MutationExceededMaxSizeException.makeTopKeysString;
+import static org.junit.Assert.*;
+
+public class MutationExceededMaxSizeExceptionTest
+{
+
+ @Test
+ public void testMakePKString()
+ {
+ List<String> keys = Arrays.asList("aaa", "bbb", "ccc");
+
+ assertEquals(0, makeTopKeysString(new ArrayList<>(), 1024).length());
+ assertEquals("aaa and 2 more.", makeTopKeysString(new ArrayList<>(keys), 0));
+ assertEquals("aaa and 2 more.", makeTopKeysString(new ArrayList<>(keys), 5));
+ assertEquals("aaa, bbb, ccc", makeTopKeysString(new ArrayList<>(keys), 13));
+ assertEquals("aaa, bbb, ccc", makeTopKeysString(new ArrayList<>(keys), 1024));
+ assertEquals("aaa, bbb and 1 more.", makeTopKeysString(new ArrayList<>(keys), 8));
+ assertEquals("aaa, bbb and 1 more.", makeTopKeysString(new ArrayList<>(keys), 10));
+ }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/db/NameSortTest.java b/test/unit/org/apache/cassandra/db/NameSortTest.java
index 1da6ea6..0b00f40 100644
--- a/test/unit/org/apache/cassandra/db/NameSortTest.java
+++ b/test/unit/org/apache/cassandra/db/NameSortTest.java
@@ -23,7 +23,7 @@
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.Util;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.db.rows.Row;
import org.apache.cassandra.db.partitions.*;
import org.apache.cassandra.db.marshal.AsciiType;
@@ -77,7 +77,7 @@
for (int i = 0; i < N; i++)
{
ByteBuffer key = ByteBufferUtil.bytes(Integer.toString(i));
- RowUpdateBuilder rub = new RowUpdateBuilder(cfs.metadata, 0, key);
+ RowUpdateBuilder rub = new RowUpdateBuilder(cfs.metadata(), 0, key);
rub.clustering("cc");
for (int j = 0; j < 8; j++)
rub.add("val" + j, j % 2 == 0 ? "a" : "b");
@@ -94,7 +94,7 @@
{
for (Row r : partition)
{
- for (ColumnDefinition cd : r.columns())
+ for (ColumnMetadata cd : r.columns())
{
if (r.getCell(cd) == null)
continue;
diff --git a/test/unit/org/apache/cassandra/db/NativeCellTest.java b/test/unit/org/apache/cassandra/db/NativeCellTest.java
index 69e615b..a63fb32 100644
--- a/test/unit/org/apache/cassandra/db/NativeCellTest.java
+++ b/test/unit/org/apache/cassandra/db/NativeCellTest.java
@@ -29,7 +29,7 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.cql3.ColumnIdentifier;
import org.apache.cassandra.db.marshal.BytesType;
import org.apache.cassandra.db.marshal.SetType;
@@ -61,7 +61,7 @@
{
for (int run = 0 ; run < 1000 ; run++)
{
- Row.Builder builder = BTreeRow.unsortedBuilder(1);
+ Row.Builder builder = BTreeRow.unsortedBuilder();
builder.newRow(rndclustering());
int count = 1 + rand.nextInt(10);
for (int i = 0 ; i < count ; i++)
@@ -92,7 +92,7 @@
private static void rndcd(Row.Builder builder)
{
- ColumnDefinition col = rndcol();
+ ColumnMetadata col = rndcol();
if (!col.isComplex())
{
builder.addCell(rndcell(col));
@@ -105,19 +105,19 @@
}
}
- private static ColumnDefinition rndcol()
+ private static ColumnMetadata rndcol()
{
UUID uuid = new UUID(rand.nextLong(), rand.nextLong());
boolean isComplex = rand.nextBoolean();
- return new ColumnDefinition("",
- "",
- ColumnIdentifier.getInterned(uuid.toString(), false),
+ return new ColumnMetadata("",
+ "",
+ ColumnIdentifier.getInterned(uuid.toString(), false),
isComplex ? new SetType<>(BytesType.instance, true) : BytesType.instance,
- -1,
- ColumnDefinition.Kind.REGULAR);
+ -1,
+ ColumnMetadata.Kind.REGULAR);
}
- private static Cell rndcell(ColumnDefinition col)
+ private static Cell rndcell(ColumnMetadata col)
{
long timestamp = rand.nextLong();
int ttl = rand.nextInt();
diff --git a/test/unit/org/apache/cassandra/db/OldFormatDeserializerTest.java b/test/unit/org/apache/cassandra/db/OldFormatDeserializerTest.java
deleted file mode 100644
index cb37229..0000000
--- a/test/unit/org/apache/cassandra/db/OldFormatDeserializerTest.java
+++ /dev/null
@@ -1,172 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.db;
-
-import java.util.function.Supplier;
-
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.db.UnfilteredDeserializer.OldFormatDeserializer.UnfilteredIterator;
-import org.apache.cassandra.db.marshal.Int32Type;
-import org.apache.cassandra.db.rows.RangeTombstoneMarker;
-import org.apache.cassandra.db.rows.SerializationHelper;
-import org.apache.cassandra.db.rows.Unfiltered;
-import org.apache.cassandra.dht.Murmur3Partitioner;
-import org.apache.cassandra.net.MessagingService;
-import org.apache.cassandra.utils.ByteBufferUtil;
-import org.apache.cassandra.utils.FBUtilities;
-
-import static org.junit.Assert.*;
-
-public class OldFormatDeserializerTest
-{
- @BeforeClass
- public static void beforeClass()
- {
- DatabaseDescriptor.daemonInitialization();
- }
-
- @Test
- public void testRangeTombstones() throws Exception
- {
- CFMetaData metadata = CFMetaData.Builder.create("ks", "table")
- .withPartitioner(Murmur3Partitioner.instance)
- .addPartitionKey("k", Int32Type.instance)
- .addClusteringColumn("v", Int32Type.instance)
- .build();
-
- Supplier<LegacyLayout.LegacyAtom> atomSupplier = supplier(rt(0, 10, 42),
- rt(5, 15, 42));
-
- UnfilteredIterator iterator = new UnfilteredIterator(metadata,
- DeletionTime.LIVE,
- new SerializationHelper(metadata, MessagingService.current_version, SerializationHelper.Flag.LOCAL),
- atomSupplier);
-
- // As the deletion time are the same, we want this to produce a single range tombstone covering from 0 to 15.
-
- assertTrue(iterator.hasNext());
-
- Unfiltered first = iterator.next();
- assertTrue(first.isRangeTombstoneMarker());
- RangeTombstoneMarker start = (RangeTombstoneMarker)first;
- assertTrue(start.isOpen(false));
- assertFalse(start.isClose(false));
- assertEquals(0, toInt(start.openBound(false)));
- assertEquals(42, start.openDeletionTime(false).markedForDeleteAt());
-
- Unfiltered second = iterator.next();
- assertTrue(second.isRangeTombstoneMarker());
- RangeTombstoneMarker end = (RangeTombstoneMarker)second;
- assertTrue(end.isClose(false));
- assertFalse(end.isOpen(false));
- assertEquals(15, toInt(end.closeBound(false)));
- assertEquals(42, end.closeDeletionTime(false).markedForDeleteAt());
-
- assertFalse(iterator.hasNext());
- }
-
- @Test
- public void testRangeTombstonesSameStart() throws Exception
- {
- CFMetaData metadata = CFMetaData.Builder.create("ks", "table")
- .withPartitioner(Murmur3Partitioner.instance)
- .addPartitionKey("k", Int32Type.instance)
- .addClusteringColumn("v", Int32Type.instance)
- .build();
-
- // Multiple RT that have the same start (we _can_ get this in the legacy format!)
- Supplier<LegacyLayout.LegacyAtom> atomSupplier = supplier(rt(1, 2, 3),
- rt(1, 2, 5),
- rt(1, 5, 4));
-
- UnfilteredIterator iterator = new UnfilteredIterator(metadata,
- DeletionTime.LIVE,
- new SerializationHelper(metadata, MessagingService.current_version, SerializationHelper.Flag.LOCAL),
- atomSupplier);
-
- // We should be entirely ignoring the first tombston (shadowed by 2nd one) so we should generate
- // [1, 2]@5 (2, 5]@4 (but where both range actually form a boundary)
-
- assertTrue(iterator.hasNext());
-
- Unfiltered first = iterator.next();
- System.out.println(">> " + first.toString(metadata));
- assertTrue(first.isRangeTombstoneMarker());
- RangeTombstoneMarker start = (RangeTombstoneMarker)first;
- assertTrue(start.isOpen(false));
- assertFalse(start.isClose(false));
- assertEquals(1, toInt(start.openBound(false)));
- assertEquals(5, start.openDeletionTime(false).markedForDeleteAt());
-
- Unfiltered second = iterator.next();
- assertTrue(second.isRangeTombstoneMarker());
- RangeTombstoneMarker middle = (RangeTombstoneMarker)second;
- assertTrue(middle.isClose(false));
- assertTrue(middle.isOpen(false));
- assertEquals(2, toInt(middle.closeBound(false)));
- assertEquals(2, toInt(middle.openBound(false)));
- assertEquals(5, middle.closeDeletionTime(false).markedForDeleteAt());
- assertEquals(4, middle.openDeletionTime(false).markedForDeleteAt());
-
- Unfiltered third = iterator.next();
- assertTrue(third.isRangeTombstoneMarker());
- RangeTombstoneMarker end = (RangeTombstoneMarker)third;
- assertTrue(end.isClose(false));
- assertFalse(end.isOpen(false));
- assertEquals(5, toInt(end.closeBound(false)));
- assertEquals(4, end.closeDeletionTime(false).markedForDeleteAt());
-
- assertFalse(iterator.hasNext());
- }
-
- private static int toInt(ClusteringPrefix prefix)
- {
- assertTrue(prefix.size() == 1);
- return ByteBufferUtil.toInt(prefix.get(0));
- }
-
- private static Supplier<LegacyLayout.LegacyAtom> supplier(LegacyLayout.LegacyAtom... atoms)
- {
- return new Supplier<LegacyLayout.LegacyAtom>()
- {
- int i = 0;
-
- public LegacyLayout.LegacyAtom get()
- {
- return i >= atoms.length ? null : atoms[i++];
- }
- };
- }
-
- private static LegacyLayout.LegacyAtom rt(int start, int end, int deletion)
- {
- return new LegacyLayout.LegacyRangeTombstone(bound(start, true), bound(end, false), new DeletionTime(deletion, FBUtilities.nowInSeconds()));
- }
-
- private static LegacyLayout.LegacyBound bound(int b, boolean isStart)
- {
- return new LegacyLayout.LegacyBound(isStart ? ClusteringBound.inclusiveStartOf(ByteBufferUtil.bytes(b)) : ClusteringBound.inclusiveEndOf(ByteBufferUtil.bytes(b)),
- false,
- null);
- }
-}
diff --git a/test/unit/org/apache/cassandra/db/PartitionRangeReadTest.java b/test/unit/org/apache/cassandra/db/PartitionRangeReadTest.java
index c418afc..8a666fc 100644
--- a/test/unit/org/apache/cassandra/db/PartitionRangeReadTest.java
+++ b/test/unit/org/apache/cassandra/db/PartitionRangeReadTest.java
@@ -19,12 +19,10 @@
package org.apache.cassandra.db;
import java.math.BigInteger;
-import java.net.InetAddress;
import java.nio.ByteBuffer;
import java.nio.charset.CharacterCodingException;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Collections;
import java.util.Iterator;
import java.util.List;
@@ -32,13 +30,10 @@
import org.junit.BeforeClass;
import org.junit.Test;
-import static org.apache.cassandra.db.ConsistencyLevel.ONE;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import org.apache.cassandra.*;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.marshal.AsciiType;
import org.apache.cassandra.db.rows.Row;
@@ -48,8 +43,12 @@
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.locator.ReplicaPlan;
+import org.apache.cassandra.locator.ReplicaPlans;
import org.apache.cassandra.locator.TokenMetadata;
+import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.service.StorageProxy;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.ByteBufferUtil;
@@ -63,8 +62,6 @@
public static final String CF_STANDARDINT = "StandardInteger1";
public static final String CF_COMPACT1 = "Compact1";
- private static final List<InetAddress> LOCAL = Collections.singletonList(FBUtilities.getBroadcastAddress());
-
@BeforeClass
public static void defineSchema() throws ConfigurationException
{
@@ -73,12 +70,12 @@
KeyspaceParams.simple(1),
SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD1),
SchemaLoader.denseCFMD(KEYSPACE1, CF_STANDARDINT, IntegerType.instance),
- CFMetaData.Builder.create(KEYSPACE1, CF_COMPACT1, false, false, false)
- .addPartitionKey("key", AsciiType.instance)
- .addClusteringColumn("column1", AsciiType.instance)
- .addRegularColumn("value", AsciiType.instance)
- .addStaticColumn("val", AsciiType.instance)
- .build());
+ TableMetadata.builder(KEYSPACE1, CF_COMPACT1)
+ .isCompound(false)
+ .addPartitionKeyColumn("key", AsciiType.instance)
+ .addClusteringColumn("column1", AsciiType.instance)
+ .addRegularColumn("value", AsciiType.instance)
+ .addStaticColumn("val", AsciiType.instance));
SchemaLoader.createKeyspace(KEYSPACE2,
KeyspaceParams.simple(1),
SchemaLoader.standardCFMD(KEYSPACE2, CF_STANDARD1));
@@ -88,10 +85,10 @@
public void testInclusiveBounds()
{
ColumnFamilyStore cfs = Keyspace.open(KEYSPACE2).getColumnFamilyStore(CF_STANDARD1);
- new RowUpdateBuilder(cfs.metadata, 0, ByteBufferUtil.bytes("key1"))
+ new RowUpdateBuilder(cfs.metadata(), 0, ByteBufferUtil.bytes("key1"))
.clustering("cc1")
.add("val", "asdf").build().applyUnsafe();
- new RowUpdateBuilder(cfs.metadata, 0, ByteBufferUtil.bytes("key2"))
+ new RowUpdateBuilder(cfs.metadata(), 0, ByteBufferUtil.bytes("key2"))
.clustering("cc2")
.add("val", "asdf").build().applyUnsafe();
@@ -107,18 +104,18 @@
cfs.truncateBlocking();
ByteBuffer col = ByteBufferUtil.bytes("val");
- ColumnDefinition cDef = cfs.metadata.getColumnDefinition(col);
+ ColumnMetadata cDef = cfs.metadata().getColumn(col);
// insert two columns that represent the same integer but have different binary forms (the
// second one is padded with extra zeros)
- new RowUpdateBuilder(cfs.metadata, 0, "k1")
+ new RowUpdateBuilder(cfs.metadata(), 0, "k1")
.clustering(new BigInteger(new byte[]{1}))
.add("val", "val1")
.build()
.applyUnsafe();
cfs.forceBlockingFlush();
- new RowUpdateBuilder(cfs.metadata, 1, "k1")
+ new RowUpdateBuilder(cfs.metadata(), 1, "k1")
.clustering(new BigInteger(new byte[]{0, 0, 1}))
.add("val", "val2")
.build()
@@ -140,10 +137,16 @@
ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_COMPACT1);
for (int i = 0; i < 10; i++)
{
- new RowUpdateBuilder(cfs.metadata, 0, Integer.toString(i))
+ new RowUpdateBuilder(cfs.metadata(), 0, Integer.toString(i))
.add("val", "abcd")
.build()
.applyUnsafe();
+
+ new RowUpdateBuilder(cfs.metadata(), 0, Integer.toString(i))
+ .clustering("column1")
+ .add("value", "")
+ .build()
+ .applyUnsafe();
}
assertEquals(10, Util.getAll(Util.cmd(cfs).build()).size());
@@ -163,7 +166,7 @@
for (int i = 0; i < 10; ++i)
{
- RowUpdateBuilder builder = new RowUpdateBuilder(cfs.metadata, 10, String.valueOf(i));
+ RowUpdateBuilder builder = new RowUpdateBuilder(cfs.metadata(), 10, String.valueOf(i));
builder.clustering("c");
builder.add("val", String.valueOf(i));
builder.build().applyUnsafe();
@@ -171,7 +174,7 @@
cfs.forceBlockingFlush();
- ColumnDefinition cDef = cfs.metadata.getColumnDefinition(ByteBufferUtil.bytes("val"));
+ ColumnMetadata cDef = cfs.metadata().getColumn(ByteBufferUtil.bytes("val"));
List<FilteredPartition> partitions;
@@ -200,212 +203,6 @@
assertTrue(partitions.get(partitions.size() - 1).iterator().next().getCell(cDef).value().equals(ByteBufferUtil.bytes("6")));
}
- // TODO: Port or remove, depending on what DataLimits.thriftLimits (per cell) looks like
-// @Test
-// public void testRangeSliceColumnsLimit() throws Throwable
-// {
-// String keyspaceName = KEYSPACE1;
-// String cfName = CF_STANDARD1;
-// Keyspace keyspace = Keyspace.open(keyspaceName);
-// ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfName);
-// cfs.clearUnsafe();
-//
-// Cell[] cols = new Cell[5];
-// for (int i = 0; i < 5; i++)
-// cols[i] = column("c" + i, "value", 1);
-//
-// putColsStandard(cfs, Util.dk("a"), cols[0], cols[1], cols[2], cols[3], cols[4]);
-// putColsStandard(cfs, Util.dk("b"), cols[0], cols[1]);
-// putColsStandard(cfs, Util.dk("c"), cols[0], cols[1], cols[2], cols[3]);
-// cfs.forceBlockingFlush();
-//
-// SlicePredicate sp = new SlicePredicate();
-// sp.setSlice_range(new SliceRange());
-// sp.getSlice_range().setCount(1);
-// sp.getSlice_range().setStart(ArrayUtils.EMPTY_BYTE_ARRAY);
-// sp.getSlice_range().setFinish(ArrayUtils.EMPTY_BYTE_ARRAY);
-//
-// assertTotalColCount(cfs.getRangeSlice(Util.range("", ""),
-// null,
-// ThriftValidation.asIFilter(sp, cfs.metadata, null),
-// 3,
-// System.currentTimeMillis(),
-// true,
-// false),
-// 3);
-// assertTotalColCount(cfs.getRangeSlice(Util.range("", ""),
-// null,
-// ThriftValidation.asIFilter(sp, cfs.metadata, null),
-// 5,
-// System.currentTimeMillis(),
-// true,
-// false),
-// 5);
-// assertTotalColCount(cfs.getRangeSlice(Util.range("", ""),
-// null,
-// ThriftValidation.asIFilter(sp, cfs.metadata, null),
-// 8,
-// System.currentTimeMillis(),
-// true,
-// false),
-// 8);
-// assertTotalColCount(cfs.getRangeSlice(Util.range("", ""),
-// null,
-// ThriftValidation.asIFilter(sp, cfs.metadata, null),
-// 10,
-// System.currentTimeMillis(),
-// true,
-// false),
-// 10);
-// assertTotalColCount(cfs.getRangeSlice(Util.range("", ""),
-// null,
-// ThriftValidation.asIFilter(sp, cfs.metadata, null),
-// 100,
-// System.currentTimeMillis(),
-// true,
-// false),
-// 11);
-//
-// // Check that when querying by name, we always include all names for a
-// // gien row even if it means returning more columns than requested (this is necesseray for CQL)
-// sp = new SlicePredicate();
-// sp.setColumn_names(Arrays.asList(
-// ByteBufferUtil.bytes("c0"),
-// ByteBufferUtil.bytes("c1"),
-// ByteBufferUtil.bytes("c2")
-// ));
-//
-// assertTotalColCount(cfs.getRangeSlice(Util.range("", ""),
-// null,
-// ThriftValidation.asIFilter(sp, cfs.metadata, null),
-// 1,
-// System.currentTimeMillis(),
-// true,
-// false),
-// 3);
-// assertTotalColCount(cfs.getRangeSlice(Util.range("", ""),
-// null,
-// ThriftValidation.asIFilter(sp, cfs.metadata, null),
-// 4,
-// System.currentTimeMillis(),
-// true,
-// false),
-// 5);
-// assertTotalColCount(cfs.getRangeSlice(Util.range("", ""),
-// null,
-// ThriftValidation.asIFilter(sp, cfs.metadata, null),
-// 5,
-// System.currentTimeMillis(),
-// true,
-// false),
-// 5);
-// assertTotalColCount(cfs.getRangeSlice(Util.range("", ""),
-// null,
-// ThriftValidation.asIFilter(sp, cfs.metadata, null),
-// 6,
-// System.currentTimeMillis(),
-// true,
-// false),
-// 8);
-// assertTotalColCount(cfs.getRangeSlice(Util.range("", ""),
-// null,
-// ThriftValidation.asIFilter(sp, cfs.metadata, null),
-// 100,
-// System.currentTimeMillis(),
-// true,
-// false),
-// 8);
-// }
-
- // TODO: Port or remove, depending on what DataLimits.thriftLimits (per cell) looks like
-// @Test
-// public void testRangeSlicePaging() throws Throwable
-// {
-// String keyspaceName = KEYSPACE1;
-// String cfName = CF_STANDARD1;
-// Keyspace keyspace = Keyspace.open(keyspaceName);
-// ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfName);
-// cfs.clearUnsafe();
-//
-// Cell[] cols = new Cell[4];
-// for (int i = 0; i < 4; i++)
-// cols[i] = column("c" + i, "value", 1);
-//
-// DecoratedKey ka = Util.dk("a");
-// DecoratedKey kb = Util.dk("b");
-// DecoratedKey kc = Util.dk("c");
-//
-// PartitionPosition min = Util.rp("");
-//
-// putColsStandard(cfs, ka, cols[0], cols[1], cols[2], cols[3]);
-// putColsStandard(cfs, kb, cols[0], cols[1], cols[2]);
-// putColsStandard(cfs, kc, cols[0], cols[1], cols[2], cols[3]);
-// cfs.forceBlockingFlush();
-//
-// SlicePredicate sp = new SlicePredicate();
-// sp.setSlice_range(new SliceRange());
-// sp.getSlice_range().setCount(1);
-// sp.getSlice_range().setStart(ArrayUtils.EMPTY_BYTE_ARRAY);
-// sp.getSlice_range().setFinish(ArrayUtils.EMPTY_BYTE_ARRAY);
-//
-// Collection<Row> rows;
-// Row row, row1, row2;
-// IDiskAtomFilter filter = ThriftValidation.asIFilter(sp, cfs.metadata, null);
-//
-// rows = cfs.getRangeSlice(cfs.makeExtendedFilter(Util.range("", ""), filter, null, 3, true, true, System.currentTimeMillis()));
-// assert rows.size() == 1 : "Expected 1 row, got " + toString(rows);
-// row = rows.iterator().next();
-// assertColumnNames(row, "c0", "c1", "c2");
-//
-// sp.getSlice_range().setStart(ByteBufferUtil.getArray(ByteBufferUtil.bytes("c2")));
-// filter = ThriftValidation.asIFilter(sp, cfs.metadata, null);
-// rows = cfs.getRangeSlice(cfs.makeExtendedFilter(new Bounds<PartitionPosition>(ka, min), filter, null, 3, true, true, System.currentTimeMillis()));
-// assert rows.size() == 2 : "Expected 2 rows, got " + toString(rows);
-// Iterator<Row> iter = rows.iterator();
-// row1 = iter.next();
-// row2 = iter.next();
-// assertColumnNames(row1, "c2", "c3");
-// assertColumnNames(row2, "c0");
-//
-// sp.getSlice_range().setStart(ByteBufferUtil.getArray(ByteBufferUtil.bytes("c0")));
-// filter = ThriftValidation.asIFilter(sp, cfs.metadata, null);
-// rows = cfs.getRangeSlice(cfs.makeExtendedFilter(new Bounds<PartitionPosition>(row2.key, min), filter, null, 3, true, true, System.currentTimeMillis()));
-// assert rows.size() == 1 : "Expected 1 row, got " + toString(rows);
-// row = rows.iterator().next();
-// assertColumnNames(row, "c0", "c1", "c2");
-//
-// sp.getSlice_range().setStart(ByteBufferUtil.getArray(ByteBufferUtil.bytes("c2")));
-// filter = ThriftValidation.asIFilter(sp, cfs.metadata, null);
-// rows = cfs.getRangeSlice(cfs.makeExtendedFilter(new Bounds<PartitionPosition>(row.key, min), filter, null, 3, true, true, System.currentTimeMillis()));
-// assert rows.size() == 2 : "Expected 2 rows, got " + toString(rows);
-// iter = rows.iterator();
-// row1 = iter.next();
-// row2 = iter.next();
-// assertColumnNames(row1, "c2");
-// assertColumnNames(row2, "c0", "c1");
-//
-// // Paging within bounds
-// SliceQueryFilter sf = new SliceQueryFilter(cellname("c1"),
-// cellname("c2"),
-// false,
-// 0);
-// rows = cfs.getRangeSlice(cfs.makeExtendedFilter(new Bounds<PartitionPosition>(ka, kc), sf, cellname("c2"), cellname("c1"), null, 2, true, System.currentTimeMillis()));
-// assert rows.size() == 2 : "Expected 2 rows, got " + toString(rows);
-// iter = rows.iterator();
-// row1 = iter.next();
-// row2 = iter.next();
-// assertColumnNames(row1, "c2");
-// assertColumnNames(row2, "c1");
-//
-// rows = cfs.getRangeSlice(cfs.makeExtendedFilter(new Bounds<PartitionPosition>(kb, kc), sf, cellname("c1"), cellname("c1"), null, 10, true, System.currentTimeMillis()));
-// assert rows.size() == 2 : "Expected 2 rows, got " + toString(rows);
-// iter = rows.iterator();
-// row1 = iter.next();
-// row2 = iter.next();
-// assertColumnNames(row1, "c1", "c2");
-// assertColumnNames(row2, "c1");
-// }
-
@Test
public void testComputeConcurrencyFactor()
{
@@ -439,16 +236,16 @@
int vnodeCount = 0;
Keyspace keyspace = Keyspace.open(KEYSPACE1);
- List<StorageProxy.RangeForQuery> ranges = new ArrayList<>();
+ List<ReplicaPlan.ForRangeRead> ranges = new ArrayList<>();
for (int i = 0; i + 1 < tokens.size(); i++)
{
Range<PartitionPosition> range = Range.makeRowRange(tokens.get(i), tokens.get(i + 1));
- ranges.add(new StorageProxy.RangeForQuery(range, LOCAL, LOCAL, 1));
+ ranges.add(ReplicaPlans.forRangeRead(keyspace, ConsistencyLevel.ONE, range, 1));
vnodeCount++;
}
- StorageProxy.RangeMerger merge = new StorageProxy.RangeMerger(ranges.iterator(), keyspace, ONE);
- StorageProxy.RangeForQuery mergedRange = Iterators.getOnlyElement(merge);
+ StorageProxy.RangeMerger merge = new StorageProxy.RangeMerger(ranges.iterator(), keyspace, ConsistencyLevel.ONE);
+ ReplicaPlan.ForRangeRead mergedRange = Iterators.getOnlyElement(merge);
// all ranges are merged as test has only one node.
assertEquals(vnodeCount, mergedRange.vnodeCount());
}
@@ -466,7 +263,7 @@
int rows = 100;
for (int i = 0; i < rows; ++i)
{
- RowUpdateBuilder builder = new RowUpdateBuilder(cfs.metadata, 10, String.valueOf(i));
+ RowUpdateBuilder builder = new RowUpdateBuilder(cfs.metadata(), 10, String.valueOf(i));
builder.clustering("c");
builder.add("val", String.valueOf(i));
builder.build().applyUnsafe();
@@ -476,36 +273,36 @@
PartitionRangeReadCommand command = (PartitionRangeReadCommand) Util.cmd(cfs).build();
// without range merger, there will be 2 batches requested: 1st batch with 1 range and 2nd batch with remaining ranges
- Iterator<StorageProxy.RangeForQuery> ranges = rangeIterator(command, keyspace, false);
- StorageProxy.RangeCommandIterator data = new StorageProxy.RangeCommandIterator(ranges, command, 1, 1000, vnodeCount, keyspace, ONE, System.nanoTime());
+ Iterator<ReplicaPlan.ForRangeRead> ranges = rangeIterator(command, keyspace, false);
+ StorageProxy.RangeCommandIterator data = new StorageProxy.RangeCommandIterator(ranges, command, 1, 1000, vnodeCount, System.nanoTime());
verifyRangeCommandIterator(data, rows, 2, vnodeCount);
// without range merger and initial cf=5, there will be 1 batches requested: 5 vnode ranges for 1st batch
ranges = rangeIterator(command, keyspace, false);
- data = new StorageProxy.RangeCommandIterator(ranges, command, vnodeCount, 1000, vnodeCount, keyspace, ONE, System.nanoTime());
+ data = new StorageProxy.RangeCommandIterator(ranges, command, vnodeCount, 1000, vnodeCount, System.nanoTime());
verifyRangeCommandIterator(data, rows, 1, vnodeCount);
// without range merger and max cf=1, there will be 5 batches requested: 1 vnode range per batch
ranges = rangeIterator(command, keyspace, false);
- data = new StorageProxy.RangeCommandIterator(ranges, command, 1, 1, vnodeCount, keyspace, ONE, System.nanoTime());
+ data = new StorageProxy.RangeCommandIterator(ranges, command, 1, 1, vnodeCount, System.nanoTime());
verifyRangeCommandIterator(data, rows, vnodeCount, vnodeCount);
// with range merger, there will be only 1 batch requested, as all ranges share the same replica - localhost
ranges = rangeIterator(command, keyspace, true);
- data = new StorageProxy.RangeCommandIterator(ranges, command, 1, 1000, vnodeCount, keyspace, ONE, System.nanoTime());
+ data = new StorageProxy.RangeCommandIterator(ranges, command, 1, 1000, vnodeCount, System.nanoTime());
verifyRangeCommandIterator(data, rows, 1, vnodeCount);
// with range merger and max cf=1, there will be only 1 batch requested, as all ranges share the same replica - localhost
ranges = rangeIterator(command, keyspace, true);
- data = new StorageProxy.RangeCommandIterator(ranges, command, 1, 1, vnodeCount, keyspace, ONE, System.nanoTime());
+ data = new StorageProxy.RangeCommandIterator(ranges, command, 1, 1, vnodeCount, System.nanoTime());
verifyRangeCommandIterator(data, rows, 1, vnodeCount);
}
- private Iterator<StorageProxy.RangeForQuery> rangeIterator(PartitionRangeReadCommand command, Keyspace keyspace, boolean withRangeMerger)
+ private Iterator<ReplicaPlan.ForRangeRead> rangeIterator(PartitionRangeReadCommand command, Keyspace keyspace, boolean withRangeMerger)
{
- Iterator<StorageProxy.RangeForQuery> ranges = new StorageProxy.RangeIterator(command, keyspace, ONE);
+ Iterator<ReplicaPlan.ForRangeRead> ranges = new StorageProxy.RangeIterator(command, keyspace, ConsistencyLevel.ONE);
if (withRangeMerger)
- ranges = new StorageProxy.RangeMerger(ranges, keyspace, ONE);
+ ranges = new StorageProxy.RangeMerger(ranges, keyspace, ConsistencyLevel.ONE);
return ranges;
}
@@ -527,7 +324,7 @@
TokenMetadata tmd = StorageService.instance.getTokenMetadata();
tmd.clearUnsafe();
- tmd.updateNormalTokens(tokens, FBUtilities.getBroadcastAddress());
+ tmd.updateNormalTokens(tokens, FBUtilities.getBroadcastAddressAndPort());
return tokens;
}
diff --git a/test/unit/org/apache/cassandra/db/PartitionTest.java b/test/unit/org/apache/cassandra/db/PartitionTest.java
index 7216ab7..be3a9e4 100644
--- a/test/unit/org/apache/cassandra/db/PartitionTest.java
+++ b/test/unit/org/apache/cassandra/db/PartitionTest.java
@@ -19,13 +19,15 @@
package org.apache.cassandra.db;
import java.io.IOException;
-import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.Arrays;
+import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
-import org.apache.cassandra.config.ColumnDefinition;
+
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.db.rows.EncodingStats;
import org.apache.cassandra.db.rows.UnfilteredRowIterators;
import org.apache.cassandra.db.marshal.AsciiType;
@@ -42,6 +44,7 @@
import org.apache.cassandra.utils.FBUtilities;
import static junit.framework.Assert.assertTrue;
+import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
@@ -67,7 +70,7 @@
public void testSingleColumn() throws IOException
{
ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD1);
- PartitionUpdate update = new RowUpdateBuilder(cfs.metadata, 5, "key1")
+ PartitionUpdate update = new RowUpdateBuilder(cfs.metadata(), 5, "key1")
.clustering("c")
.add("val", "val1")
.buildUpdate();
@@ -80,7 +83,7 @@
CachedPartition deserialized = CachedPartition.cacheSerializer.deserialize(new DataInputBuffer(bufOut.getData()));
assert deserialized != null;
- assert deserialized.metadata().cfName.equals(CF_STANDARD1);
+ assert deserialized.metadata().name.equals(CF_STANDARD1);
assert deserialized.partitionKey().equals(partition.partitionKey());
}
@@ -88,7 +91,7 @@
public void testManyColumns() throws IOException
{
ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_TENCOL);
- RowUpdateBuilder builder = new RowUpdateBuilder(cfs.metadata, 5, "key1")
+ RowUpdateBuilder builder = new RowUpdateBuilder(cfs.metadata(), 5, "key1")
.clustering("c")
.add("val", "val1");
@@ -108,7 +111,7 @@
assertTrue(deserialized.columns().regulars.getSimple(1).equals(partition.columns().regulars.getSimple(1)));
assertTrue(deserialized.columns().regulars.getSimple(5).equals(partition.columns().regulars.getSimple(5)));
- ColumnDefinition cDef = cfs.metadata.getColumnDefinition(ByteBufferUtil.bytes("val8"));
+ ColumnMetadata cDef = cfs.metadata().getColumn(ByteBufferUtil.bytes("val8"));
assertTrue(partition.lastRow().getCell(cDef).value().equals(deserialized.lastRow().getCell(cDef).value()));
assert deserialized.partitionKey().equals(partition.partitionKey());
}
@@ -119,52 +122,40 @@
testDigest(MessagingService.current_version);
}
- @Test
- public void testLegacyDigest() throws NoSuchAlgorithmException
- {
- testDigest(MessagingService.VERSION_22);
- }
-
public void testDigest(int version) throws NoSuchAlgorithmException
{
ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_TENCOL);
try
{
- RowUpdateBuilder builder = new RowUpdateBuilder(cfs.metadata, 5, "key1").clustering("c").add("val", "val1");
+ RowUpdateBuilder builder = new RowUpdateBuilder(cfs.metadata(), 5, "key1").clustering("c").add("val", "val1");
for (int i = 0; i < 10; i++)
builder.add("val" + i, "val" + i);
builder.build().applyUnsafe();
- new RowUpdateBuilder(cfs.metadata, 5, "key2").clustering("c").add("val", "val2").build().applyUnsafe();
+ new RowUpdateBuilder(cfs.metadata(), 5, "key2").clustering("c").add("val", "val2").build().applyUnsafe();
ReadCommand cmd1 = Util.cmd(cfs, "key1").build();
ReadCommand cmd2 = Util.cmd(cfs, "key2").build();
ImmutableBTreePartition p1 = Util.getOnlyPartitionUnfiltered(cmd1);
ImmutableBTreePartition p2 = Util.getOnlyPartitionUnfiltered(cmd2);
- MessageDigest digest1 = MessageDigest.getInstance("MD5");
- MessageDigest digest2 = MessageDigest.getInstance("MD5");
- UnfilteredRowIterators.digest(cmd1, p1.unfilteredIterator(), digest1, version);
- UnfilteredRowIterators.digest(cmd2, p2.unfilteredIterator(), digest2, version);
- assertFalse(Arrays.equals(digest1.digest(), digest2.digest()));
+ byte[] digest1 = getDigest(p1.unfilteredIterator(), version);
+ byte[] digest2 = getDigest(p2.unfilteredIterator(), version);
+ assertFalse(Arrays.equals(digest1, digest2));
p1 = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, "key2").build());
p2 = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, "key2").build());
- digest1 = MessageDigest.getInstance("MD5");
- digest2 = MessageDigest.getInstance("MD5");
- UnfilteredRowIterators.digest(cmd1, p1.unfilteredIterator(), digest1, version);
- UnfilteredRowIterators.digest(cmd2, p2.unfilteredIterator(), digest2, version);
- assertTrue(Arrays.equals(digest1.digest(), digest2.digest()));
+ digest1 = getDigest(p1.unfilteredIterator(), version);
+ digest2 = getDigest(p2.unfilteredIterator(), version);
+ assertArrayEquals(digest1, digest2);
p1 = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, "key2").build());
- RowUpdateBuilder.deleteRow(cfs.metadata, 6, "key2", "c").applyUnsafe();
+ RowUpdateBuilder.deleteRow(cfs.metadata(), 6, "key2", "c").applyUnsafe();
p2 = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, "key2").build());
- digest1 = MessageDigest.getInstance("MD5");
- digest2 = MessageDigest.getInstance("MD5");
- UnfilteredRowIterators.digest(cmd1, p1.unfilteredIterator(), digest1, version);
- UnfilteredRowIterators.digest(cmd2, p2.unfilteredIterator(), digest2, version);
- assertFalse(Arrays.equals(digest1.digest(), digest2.digest()));
+ digest1 = getDigest(p1.unfilteredIterator(), version);
+ digest2 = getDigest(p2.unfilteredIterator(), version);
+ assertFalse(Arrays.equals(digest1, digest2));
}
finally
{
@@ -172,6 +163,13 @@
}
}
+ private byte[] getDigest(UnfilteredRowIterator partition, int version)
+ {
+ Digest digest = Digest.forReadResponse();
+ UnfilteredRowIterators.digest(partition, digest, version);
+ return digest.digest();
+ }
+
@Test
public void testColumnStatsRecordsRowDeletesCorrectly()
{
@@ -179,12 +177,12 @@
int localDeletionTime = (int) (timestamp / 1000);
ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_TENCOL);
- RowUpdateBuilder builder = new RowUpdateBuilder(cfs.metadata, 5, "key1").clustering("c").add("val", "val1");
+ RowUpdateBuilder builder = new RowUpdateBuilder(cfs.metadata(), 5, "key1").clustering("c").add("val", "val1");
for (int i = 0; i < 10; i++)
builder.add("val" + i, "val" + i);
builder.build().applyUnsafe();
- RowUpdateBuilder.deleteRowAt(cfs.metadata, 10L, localDeletionTime, "key1", "c").applyUnsafe();
+ RowUpdateBuilder.deleteRowAt(cfs.metadata(), 10L, localDeletionTime, "key1", "c").applyUnsafe();
ImmutableBTreePartition partition = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, "key1").build());
EncodingStats stats = partition.stats();
assertEquals(localDeletionTime, stats.minLocalDeletionTime);
diff --git a/test/unit/org/apache/cassandra/db/RangeTombstoneListTest.java b/test/unit/org/apache/cassandra/db/RangeTombstoneListTest.java
index d3dc835..d4f7e59 100644
--- a/test/unit/org/apache/cassandra/db/RangeTombstoneListTest.java
+++ b/test/unit/org/apache/cassandra/db/RangeTombstoneListTest.java
@@ -19,22 +19,39 @@
package org.apache.cassandra.db;
import java.nio.ByteBuffer;
-import java.util.*;
-import java.util.regex.Pattern;
+import java.util.Iterator;
+import java.util.Random;
+import java.util.function.Consumer;
import java.util.regex.Matcher;
+import java.util.regex.Pattern;
import com.google.common.base.Joiner;
-
+import org.junit.BeforeClass;
import org.junit.Test;
-import static org.junit.Assert.*;
+import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.distributed.impl.IsolatedExecutor;
+import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.ByteBufferUtil;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
public class RangeTombstoneListTest
{
private static final ClusteringComparator cmp = new ClusteringComparator(Int32Type.instance);
+ @BeforeClass
+ public static void beforeClass()
+ {
+ // Needed to initialize initial_range_tombstone_allocation_size and range_tombstone_resize_factor
+ DatabaseDescriptor.daemonInitialization();
+ }
+
@Test
public void sortedAdditionTest()
{
@@ -539,6 +556,72 @@
assertEquals(6, l.searchDeletionTime(clustering(1000)).markedForDeleteAt());
}
+ @Test
+ public void testSetResizeFactor()
+ {
+ double original = DatabaseDescriptor.getRangeTombstoneListGrowthFactor();
+ final StorageService storageService = StorageService.instance;
+ final Consumer<Throwable> expectIllegalStateExceptio = exception -> {
+ assertSame(IllegalStateException.class, exception.getClass());
+ assertEquals("Not updating range_tombstone_resize_factor as growth factor must be in the range [1.2, 5.0] inclusive" , exception.getMessage());
+ };
+ try
+ {
+ // prevent bad ones
+ assertHasException(() -> storageService.setRangeTombstoneListResizeGrowthFactor(-1), expectIllegalStateExceptio);
+ assertHasException(() -> storageService.setRangeTombstoneListResizeGrowthFactor(0), expectIllegalStateExceptio);
+ assertHasException(() -> storageService.setRangeTombstoneListResizeGrowthFactor(1.1), expectIllegalStateExceptio);
+ assertHasException(() -> storageService.setRangeTombstoneListResizeGrowthFactor(5.1), expectIllegalStateExceptio);
+
+ // accept good ones
+ storageService.setRangeTombstoneListResizeGrowthFactor(1.2);
+ storageService.setRangeTombstoneListResizeGrowthFactor(2.0);
+ storageService.setRangeTombstoneListResizeGrowthFactor(5.0);
+ }
+ finally
+ {
+ storageService.setRangeTombstoneListResizeGrowthFactor(original);
+ }
+ }
+
+ @Test
+ public void testSetInitialAllocationSize()
+ {
+ int original = DatabaseDescriptor.getInitialRangeTombstoneListAllocationSize();
+ final StorageService storageService = StorageService.instance;
+ final Consumer<Throwable> expectIllegalStateExceptio = exception -> {
+ assertSame(String.format("The actual exception message:<%s>", exception.getMessage()), IllegalStateException.class, exception.getClass());
+ assertEquals("Not updating initial_range_tombstone_allocation_size as it must be in the range [0, 1024] inclusive" , exception.getMessage());
+ };
+ try
+ {
+ // prevent bad ones
+ assertHasException(() -> storageService.setInitialRangeTombstoneListAllocationSize(-1), expectIllegalStateExceptio);
+ assertHasException(() -> storageService.setInitialRangeTombstoneListAllocationSize(1025), expectIllegalStateExceptio);
+
+ // accept good ones
+ storageService.setInitialRangeTombstoneListAllocationSize(1);
+ storageService.setInitialRangeTombstoneListAllocationSize(1024);
+ }
+ finally
+ {
+ storageService.setInitialRangeTombstoneListAllocationSize(original);
+ }
+ }
+
+ private void assertHasException(IsolatedExecutor.ThrowingRunnable block, Consumer<Throwable> verifier)
+ {
+ try
+ {
+ block.run();
+ fail("Expect the code block to throw but not");
+ }
+ catch (Throwable throwable)
+ {
+ verifier.accept(throwable);
+ }
+ }
+
private static void assertRT(RangeTombstone expected, RangeTombstone actual)
{
assertTrue(String.format("%s != %s", toString(expected), toString(actual)), cmp.compare(expected.deletedSlice().start(), actual.deletedSlice().start()) == 0);
diff --git a/test/unit/org/apache/cassandra/db/RangeTombstoneTest.java b/test/unit/org/apache/cassandra/db/RangeTombstoneTest.java
index 363ef72..3d1d003 100644
--- a/test/unit/org/apache/cassandra/db/RangeTombstoneTest.java
+++ b/test/unit/org/apache/cassandra/db/RangeTombstoneTest.java
@@ -31,8 +31,8 @@
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.UpdateBuilder;
import org.apache.cassandra.Util;
-import org.apache.cassandra.config.ColumnDefinition;
-import org.apache.cassandra.cql3.statements.IndexTarget;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.cql3.statements.schema.IndexTarget;
import org.apache.cassandra.db.compaction.CompactionManager;
import org.apache.cassandra.db.filter.ColumnFilter;
import org.apache.cassandra.db.marshal.Int32Type;
@@ -45,9 +45,12 @@
import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
import org.apache.cassandra.schema.IndexMetadata;
import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.schema.MigrationManager;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FBUtilities;
+import static org.apache.cassandra.SchemaLoader.standardCFMD;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
@@ -63,12 +66,7 @@
SchemaLoader.prepareServer();
SchemaLoader.createKeyspace(KSNAME,
KeyspaceParams.simple(1),
- SchemaLoader.standardCFMD(KSNAME,
- CFNAME,
- 1,
- UTF8Type.instance,
- Int32Type.instance,
- Int32Type.instance));
+ standardCFMD(KSNAME, CFNAME, 1, UTF8Type.instance, Int32Type.instance, Int32Type.instance));
}
@Test
@@ -76,27 +74,27 @@
{
Keyspace keyspace = Keyspace.open(KSNAME);
ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CFNAME);
- boolean enforceStrictLiveness = cfs.metadata.enforceStrictLiveness();
+ boolean enforceStrictLiveness = cfs.metadata().enforceStrictLiveness();
// Inserting data
String key = "k1";
UpdateBuilder builder;
- builder = UpdateBuilder.create(cfs.metadata, key).withTimestamp(0);
+ builder = UpdateBuilder.create(cfs.metadata(), key).withTimestamp(0);
for (int i = 0; i < 40; i += 2)
builder.newRow(i).add("val", i);
builder.applyUnsafe();
cfs.forceBlockingFlush();
- new RowUpdateBuilder(cfs.metadata, 1, key).addRangeTombstone(10, 22).build().applyUnsafe();
+ new RowUpdateBuilder(cfs.metadata(), 1, key).addRangeTombstone(10, 22).build().applyUnsafe();
- builder = UpdateBuilder.create(cfs.metadata, key).withTimestamp(2);
+ builder = UpdateBuilder.create(cfs.metadata(), key).withTimestamp(2);
for (int i = 1; i < 40; i += 2)
builder.newRow(i).add("val", i);
builder.applyUnsafe();
- new RowUpdateBuilder(cfs.metadata, 3, key).addRangeTombstone(19, 27).build().applyUnsafe();
+ new RowUpdateBuilder(cfs.metadata(), 3, key).addRangeTombstone(19, 27).build().applyUnsafe();
// We don't flush to test with both a range tomsbtone in memtable and in sstable
// Queries by name
@@ -140,14 +138,14 @@
// Inserting data
String key = "k111";
- UpdateBuilder builder = UpdateBuilder.create(cfs.metadata, key).withTimestamp(0);
+ UpdateBuilder builder = UpdateBuilder.create(cfs.metadata(), key).withTimestamp(0);
for (int i = 0; i < 40; i += 2)
builder.newRow(i).add("val", i);
builder.applyUnsafe();
- new RowUpdateBuilder(cfs.metadata, 1, key).addRangeTombstone(5, 10).build().applyUnsafe();
+ new RowUpdateBuilder(cfs.metadata(), 1, key).addRangeTombstone(5, 10).build().applyUnsafe();
- new RowUpdateBuilder(cfs.metadata, 2, key).addRangeTombstone(15, 20).build().applyUnsafe();
+ new RowUpdateBuilder(cfs.metadata(), 2, key).addRangeTombstone(15, 20).build().applyUnsafe();
ImmutableBTreePartition partition;
@@ -215,14 +213,14 @@
sb.add(ClusteringBound.create(cfs.getComparator(), true, true, 1), ClusteringBound.create(cfs.getComparator(), false, true, 10));
sb.add(ClusteringBound.create(cfs.getComparator(), true, true, 16), ClusteringBound.create(cfs.getComparator(), false, true, 20));
- partition = Util.getOnlyPartitionUnfiltered(SinglePartitionReadCommand.create(cfs.metadata, FBUtilities.nowInSeconds(), Util.dk(key), sb.build()));
+ partition = Util.getOnlyPartitionUnfiltered(SinglePartitionReadCommand.create(cfs.metadata(), FBUtilities.nowInSeconds(), Util.dk(key), sb.build()));
rt = rangeTombstones(partition);
assertEquals(2, rt.size());
}
private Collection<RangeTombstone> rangeTombstones(ImmutableBTreePartition partition)
{
- List<RangeTombstone> tombstones = new ArrayList<RangeTombstone>();
+ List<RangeTombstone> tombstones = new ArrayList<>();
Iterators.addAll(tombstones, partition.deletionInfo().rangeIterator(false));
return tombstones;
}
@@ -236,7 +234,7 @@
String key = "rt_times";
int nowInSec = FBUtilities.nowInSeconds();
- new Mutation(PartitionUpdate.fullPartitionDelete(cfs.metadata, Util.dk(key), 1000, nowInSec)).apply();
+ new Mutation(PartitionUpdate.fullPartitionDelete(cfs.metadata(), Util.dk(key), 1000, nowInSec)).apply();
cfs.forceBlockingFlush();
SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
@@ -254,11 +252,11 @@
cfs.truncateBlocking();
String key = "rt_times";
- UpdateBuilder.create(cfs.metadata, key).withTimestamp(999).newRow(5).add("val", 5).apply();
+ UpdateBuilder.create(cfs.metadata(), key).withTimestamp(999).newRow(5).add("val", 5).apply();
key = "rt_times2";
int nowInSec = FBUtilities.nowInSeconds();
- new Mutation(PartitionUpdate.fullPartitionDelete(cfs.metadata, Util.dk(key), 1000, nowInSec)).apply();
+ new Mutation(PartitionUpdate.fullPartitionDelete(cfs.metadata(), Util.dk(key), 1000, nowInSec)).apply();
cfs.forceBlockingFlush();
SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
@@ -277,7 +275,7 @@
String key = "rt_times";
int nowInSec = FBUtilities.nowInSeconds();
- new RowUpdateBuilder(cfs.metadata, nowInSec, 1000L, key).addRangeTombstone(1, 2).build().apply();
+ new RowUpdateBuilder(cfs.metadata(), nowInSec, 1000L, key).addRangeTombstone(1, 2).build().apply();
cfs.forceBlockingFlush();
SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
@@ -295,11 +293,11 @@
cfs.truncateBlocking();
String key = "rt_times";
- UpdateBuilder.create(cfs.metadata, key).withTimestamp(999).newRow(5).add("val", 5).apply();
+ UpdateBuilder.create(cfs.metadata(), key).withTimestamp(999).newRow(5).add("val", 5).apply();
key = "rt_times2";
int nowInSec = FBUtilities.nowInSeconds();
- new Mutation(PartitionUpdate.fullPartitionDelete(cfs.metadata, Util.dk(key), 1000, nowInSec)).apply();
+ new Mutation(PartitionUpdate.fullPartitionDelete(cfs.metadata(), Util.dk(key), 1000, nowInSec)).apply();
cfs.forceBlockingFlush();
cfs.forceBlockingFlush();
@@ -322,17 +320,17 @@
{
Keyspace ks = Keyspace.open(KSNAME);
ColumnFamilyStore cfs = ks.getColumnFamilyStore(CFNAME);
- cfs.metadata.gcGraceSeconds(2);
+ MigrationManager.announceTableUpdate(cfs.metadata().unbuild().gcGraceSeconds(2).build(), true);
String key = "7810";
- UpdateBuilder builder = UpdateBuilder.create(cfs.metadata, key).withTimestamp(0);
+ UpdateBuilder builder = UpdateBuilder.create(cfs.metadata(), key).withTimestamp(0);
for (int i = 10; i < 20; i ++)
builder.newRow(i).add("val", i);
builder.apply();
cfs.forceBlockingFlush();
- new RowUpdateBuilder(cfs.metadata, 1, key).addRangeTombstone(10, 11).build().apply();
+ new RowUpdateBuilder(cfs.metadata(), 1, key).addRangeTombstone(10, 11).build().apply();
cfs.forceBlockingFlush();
Thread.sleep(5);
@@ -345,16 +343,16 @@
{
Keyspace ks = Keyspace.open(KSNAME);
ColumnFamilyStore cfs = ks.getColumnFamilyStore(CFNAME);
- cfs.metadata.gcGraceSeconds(2);
+ MigrationManager.announceTableUpdate(cfs.metadata().unbuild().gcGraceSeconds(2).build(), true);
String key = "7808_1";
- UpdateBuilder builder = UpdateBuilder.create(cfs.metadata, key).withTimestamp(0);
+ UpdateBuilder builder = UpdateBuilder.create(cfs.metadata(), key).withTimestamp(0);
for (int i = 0; i < 40; i += 2)
builder.newRow(i).add("val", i);
builder.apply();
cfs.forceBlockingFlush();
- new Mutation(PartitionUpdate.fullPartitionDelete(cfs.metadata, Util.dk(key), 1, 1)).apply();
+ new Mutation(PartitionUpdate.fullPartitionDelete(cfs.metadata(), Util.dk(key), 1, 1)).apply();
cfs.forceBlockingFlush();
Thread.sleep(5);
cfs.forceMajorCompaction();
@@ -365,18 +363,18 @@
{
Keyspace ks = Keyspace.open(KSNAME);
ColumnFamilyStore cfs = ks.getColumnFamilyStore(CFNAME);
- cfs.metadata.gcGraceSeconds(2);
+ MigrationManager.announceTableUpdate(cfs.metadata().unbuild().gcGraceSeconds(2).build(), true);
String key = "7808_2";
- UpdateBuilder builder = UpdateBuilder.create(cfs.metadata, key).withTimestamp(0);
+ UpdateBuilder builder = UpdateBuilder.create(cfs.metadata(), key).withTimestamp(0);
for (int i = 10; i < 20; i ++)
builder.newRow(i).add("val", i);
builder.apply();
cfs.forceBlockingFlush();
- new Mutation(PartitionUpdate.fullPartitionDelete(cfs.metadata, Util.dk(key), 0, 0)).apply();
+ new Mutation(PartitionUpdate.fullPartitionDelete(cfs.metadata(), Util.dk(key), 0, 0)).apply();
- UpdateBuilder.create(cfs.metadata, key).withTimestamp(1).newRow(5).add("val", 5).apply();
+ UpdateBuilder.create(cfs.metadata(), key).withTimestamp(1).newRow(5).add("val", 5).apply();
cfs.forceBlockingFlush();
Thread.sleep(5);
@@ -390,23 +388,23 @@
CompactionManager.instance.disableAutoCompaction();
Keyspace keyspace = Keyspace.open(KSNAME);
ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CFNAME);
- boolean enforceStrictLiveness = cfs.metadata.enforceStrictLiveness();
+ boolean enforceStrictLiveness = cfs.metadata().enforceStrictLiveness();
// Inserting data
String key = "k2";
- UpdateBuilder builder = UpdateBuilder.create(cfs.metadata, key).withTimestamp(0);
+ UpdateBuilder builder = UpdateBuilder.create(cfs.metadata(), key).withTimestamp(0);
for (int i = 0; i < 20; i++)
builder.newRow(i).add("val", i);
builder.applyUnsafe();
cfs.forceBlockingFlush();
- new RowUpdateBuilder(cfs.metadata, 1, key).addRangeTombstone(5, 15).build().applyUnsafe();
+ new RowUpdateBuilder(cfs.metadata(), 1, key).addRangeTombstone(5, 15).build().applyUnsafe();
cfs.forceBlockingFlush();
- new RowUpdateBuilder(cfs.metadata, 1, key).addRangeTombstone(5, 10).build().applyUnsafe();
+ new RowUpdateBuilder(cfs.metadata(), 1, key).addRangeTombstone(5, 10).build().applyUnsafe();
cfs.forceBlockingFlush();
- new RowUpdateBuilder(cfs.metadata, 2, key).addRangeTombstone(5, 8).build().applyUnsafe();
+ new RowUpdateBuilder(cfs.metadata(), 2, key).addRangeTombstone(5, 8).build().applyUnsafe();
cfs.forceBlockingFlush();
Partition partition = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, key).build());
@@ -448,18 +446,18 @@
// Inserting data
String key = "k3";
- UpdateBuilder.create(cfs.metadata, key).withTimestamp(0).newRow(2).add("val", 2).applyUnsafe();
+ UpdateBuilder.create(cfs.metadata(), key).withTimestamp(0).newRow(2).add("val", 2).applyUnsafe();
cfs.forceBlockingFlush();
- new RowUpdateBuilder(cfs.metadata, 1, key).addRangeTombstone(0, 10).build().applyUnsafe();
- UpdateBuilder.create(cfs.metadata, key).withTimestamp(2).newRow(1).add("val", 1).applyUnsafe();
+ new RowUpdateBuilder(cfs.metadata(), 1, key).addRangeTombstone(0, 10).build().applyUnsafe();
+ UpdateBuilder.create(cfs.metadata(), key).withTimestamp(2).newRow(1).add("val", 1).applyUnsafe();
cfs.forceBlockingFlush();
// Get the last value of the row
FilteredPartition partition = Util.getOnlyPartition(Util.cmd(cfs, key).build());
assertTrue(partition.rowCount() > 0);
- int last = i(partition.unfilteredIterator(ColumnFilter.all(cfs.metadata), Slices.ALL, true).next().clustering().get(0));
+ int last = i(partition.unfilteredIterator(ColumnFilter.all(cfs.metadata()), Slices.ALL, true).next().clustering().get(0));
assertEquals("Last column should be column 1 since column 2 has been deleted", 1, last);
}
@@ -474,19 +472,27 @@
cfs.truncateBlocking();
cfs.disableAutoCompaction();
- ColumnDefinition cd = cfs.metadata.getColumnDefinition(indexedColumnName).copy();
+ ColumnMetadata cd = cfs.metadata().getColumn(indexedColumnName).copy();
IndexMetadata indexDef =
- IndexMetadata.fromIndexTargets(cfs.metadata,
- Collections.singletonList(new IndexTarget(cd.name, IndexTarget.Type.VALUES)),
+ IndexMetadata.fromIndexTargets(
+ Collections.singletonList(new IndexTarget(cd.name, IndexTarget.Type.VALUES)),
"test_index",
IndexMetadata.Kind.CUSTOM,
ImmutableMap.of(IndexTarget.CUSTOM_INDEX_OPTION_NAME,
StubIndex.class.getName()));
- if (!cfs.metadata.getIndexes().get("test_index").isPresent())
- cfs.metadata.indexes(cfs.metadata.getIndexes().with(indexDef));
+ TableMetadata current = cfs.metadata();
- Future<?> rebuild = cfs.indexManager.addIndex(indexDef);
+ if (!current.indexes.get("test_index").isPresent())
+ {
+ TableMetadata updated =
+ current.unbuild()
+ .indexes(current.indexes.with(indexDef))
+ .build();
+ MigrationManager.announceTableUpdate(updated, true);
+ }
+
+ Future<?> rebuild = cfs.indexManager.addIndex(indexDef, false);
// If rebuild there is, wait for the rebuild to finish so it doesn't race with the following insertions
if (rebuild != null)
rebuild.get();
@@ -498,13 +504,13 @@
.orElseThrow(() -> new RuntimeException(new AssertionError("Index not found")));
index.reset();
- UpdateBuilder builder = UpdateBuilder.create(cfs.metadata, key).withTimestamp(0);
+ UpdateBuilder builder = UpdateBuilder.create(cfs.metadata(), key).withTimestamp(0);
for (int i = 0; i < 10; i++)
builder.newRow(i).add("val", i);
builder.applyUnsafe();
cfs.forceBlockingFlush();
- new RowUpdateBuilder(cfs.metadata, 0, key).addRangeTombstone(0, 7).build().applyUnsafe();
+ new RowUpdateBuilder(cfs.metadata(), 0, key).addRangeTombstone(0, 7).build().applyUnsafe();
cfs.forceBlockingFlush();
assertEquals(10, index.rowsInserted.size());
@@ -528,13 +534,13 @@
cfs.truncateBlocking();
cfs.disableAutoCompaction();
- UpdateBuilder builder = UpdateBuilder.create(cfs.metadata, key).withTimestamp(0);
+ UpdateBuilder builder = UpdateBuilder.create(cfs.metadata(), key).withTimestamp(0);
for (int i = 0; i < 10; i += 2)
builder.newRow(i).add("val", i);
builder.applyUnsafe();
cfs.forceBlockingFlush();
- new RowUpdateBuilder(cfs.metadata, 0, key).addRangeTombstone(0, 7).build().applyUnsafe();
+ new RowUpdateBuilder(cfs.metadata(), 0, key).addRangeTombstone(0, 7).build().applyUnsafe();
cfs.forceBlockingFlush();
// there should be 2 sstables
@@ -552,11 +558,11 @@
{
// after compaction, we should have a single RT with a single row (the row 8)
Unfiltered u1 = iter.next();
- assertTrue("Expecting open marker, got " + u1.toString(cfs.metadata), u1 instanceof RangeTombstoneMarker);
+ assertTrue("Expecting open marker, got " + u1.toString(cfs.metadata()), u1 instanceof RangeTombstoneMarker);
Unfiltered u2 = iter.next();
- assertTrue("Expecting close marker, got " + u2.toString(cfs.metadata), u2 instanceof RangeTombstoneMarker);
+ assertTrue("Expecting close marker, got " + u2.toString(cfs.metadata()), u2 instanceof RangeTombstoneMarker);
Unfiltered u3 = iter.next();
- assertTrue("Expecting row, got " + u3.toString(cfs.metadata), u3 instanceof Row);
+ assertTrue("Expecting row, got " + u3.toString(cfs.metadata()), u3 instanceof Row);
}
}
}
@@ -572,19 +578,27 @@
cfs.truncateBlocking();
cfs.disableAutoCompaction();
- ColumnDefinition cd = cfs.metadata.getColumnDefinition(indexedColumnName).copy();
+ ColumnMetadata cd = cfs.metadata().getColumn(indexedColumnName).copy();
IndexMetadata indexDef =
- IndexMetadata.fromIndexTargets(cfs.metadata,
- Collections.singletonList(new IndexTarget(cd.name, IndexTarget.Type.VALUES)),
+ IndexMetadata.fromIndexTargets(
+ Collections.singletonList(new IndexTarget(cd.name, IndexTarget.Type.VALUES)),
"test_index",
IndexMetadata.Kind.CUSTOM,
ImmutableMap.of(IndexTarget.CUSTOM_INDEX_OPTION_NAME,
StubIndex.class.getName()));
- if (!cfs.metadata.getIndexes().get("test_index").isPresent())
- cfs.metadata.indexes(cfs.metadata.getIndexes().with(indexDef));
+ TableMetadata current = cfs.metadata();
- Future<?> rebuild = cfs.indexManager.addIndex(indexDef);
+ if (!current.indexes.get("test_index").isPresent())
+ {
+ TableMetadata updated =
+ current.unbuild()
+ .indexes(current.indexes.with(indexDef))
+ .build();
+ MigrationManager.announceTableUpdate(updated, true);
+ }
+
+ Future<?> rebuild = cfs.indexManager.addIndex(indexDef, false);
// If rebuild there is, wait for the rebuild to finish so it doesn't race with the following insertions
if (rebuild != null)
rebuild.get();
@@ -592,13 +606,13 @@
StubIndex index = (StubIndex)cfs.indexManager.getIndexByName("test_index");
index.reset();
- UpdateBuilder.create(cfs.metadata, key).withTimestamp(0).newRow(1).add("val", 1).applyUnsafe();
+ UpdateBuilder.create(cfs.metadata(), key).withTimestamp(0).newRow(1).add("val", 1).applyUnsafe();
// add a RT which hides the column we just inserted
- new RowUpdateBuilder(cfs.metadata, 1, key).addRangeTombstone(0, 1).build().applyUnsafe();
+ new RowUpdateBuilder(cfs.metadata(), 1, key).addRangeTombstone(0, 1).build().applyUnsafe();
// now re-insert that column
- UpdateBuilder.create(cfs.metadata, key).withTimestamp(2).newRow(1).add("val", 1).applyUnsafe();
+ UpdateBuilder.create(cfs.metadata(), key).withTimestamp(2).newRow(1).add("val", 1).applyUnsafe();
cfs.forceBlockingFlush();
diff --git a/test/unit/org/apache/cassandra/db/ReadCommandTest.java b/test/unit/org/apache/cassandra/db/ReadCommandTest.java
index 774645e..c3687f1 100644
--- a/test/unit/org/apache/cassandra/db/ReadCommandTest.java
+++ b/test/unit/org/apache/cassandra/db/ReadCommandTest.java
@@ -19,16 +19,22 @@
package org.apache.cassandra.db;
import java.io.IOException;
+import java.io.OutputStream;
+import java.net.UnknownHostException;
import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.List;
+import java.util.*;
+import java.util.function.Supplier;
+import java.util.stream.IntStream;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.Util;
-import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.filter.ClusteringIndexSliceFilter;
import org.apache.cassandra.db.filter.ColumnFilter;
@@ -36,24 +42,51 @@
import org.apache.cassandra.db.filter.RowFilter;
import org.apache.cassandra.db.marshal.AsciiType;
import org.apache.cassandra.db.marshal.BytesType;
-import org.apache.cassandra.db.partitions.FilteredPartition;
-import org.apache.cassandra.db.partitions.PartitionIterator;
-import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
-import org.apache.cassandra.db.partitions.UnfilteredPartitionIterators;
+import org.apache.cassandra.db.marshal.CounterColumnType;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.marshal.ReversedType;
+import org.apache.cassandra.db.marshal.SetType;
+import org.apache.cassandra.db.partitions.*;
import org.apache.cassandra.db.rows.Row;
import org.apache.cassandra.db.rows.RowIterator;
import org.apache.cassandra.db.rows.SerializationHelper;
+import org.apache.cassandra.db.rows.Unfiltered;
+import org.apache.cassandra.db.rows.DeserializationHelper;
import org.apache.cassandra.db.rows.UnfilteredRowIterator;
import org.apache.cassandra.db.rows.UnfilteredRowIterators;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.util.DataInputBuffer;
import org.apache.cassandra.io.util.DataOutputBuffer;
+import org.apache.cassandra.io.util.WrappedDataOutputStreamPlus;
+import org.apache.cassandra.locator.EndpointsForToken;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.ReplicaUtils;
+import org.apache.cassandra.net.Message;
+import org.apache.cassandra.metrics.ClearableHistogram;
import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.net.Verb;
+import org.apache.cassandra.repair.consistent.LocalSessionAccessor;
+import org.apache.cassandra.schema.CachingParams;
+import org.apache.cassandra.schema.KeyspaceMetadata;
import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.schema.TableParams;
+import org.apache.cassandra.service.ActiveRepairService;
+import org.apache.cassandra.streaming.PreviewKind;
+import org.apache.cassandra.tracing.Tracing;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.UUIDGen;
+import static org.apache.cassandra.utils.ByteBufferUtil.EMPTY_BYTE_BUFFER;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
public class ReadCommandTest
{
@@ -63,50 +96,100 @@
private static final String CF3 = "Standard3";
private static final String CF4 = "Standard4";
private static final String CF5 = "Standard5";
+ private static final String CF6 = "Standard6";
+ private static final String CF7 = "Counter7";
+ private static final String CF8 = "Standard8";
+ private static final String CF9 = "Standard9";
+
+ private static final InetAddressAndPort REPAIR_COORDINATOR;
+ static {
+ try
+ {
+ REPAIR_COORDINATOR = InetAddressAndPort.getByName("10.0.0.1");
+ }
+ catch (UnknownHostException e)
+ {
+
+ throw new AssertionError(e);
+ }
+ }
@BeforeClass
public static void defineSchema() throws ConfigurationException
{
DatabaseDescriptor.daemonInitialization();
- CFMetaData metadata1 = SchemaLoader.standardCFMD(KEYSPACE, CF1);
+ TableMetadata.Builder metadata1 = SchemaLoader.standardCFMD(KEYSPACE, CF1);
- CFMetaData metadata2 = CFMetaData.Builder.create(KEYSPACE, CF2)
- .addPartitionKey("key", BytesType.instance)
- .addClusteringColumn("col", AsciiType.instance)
- .addRegularColumn("a", AsciiType.instance)
- .addRegularColumn("b", AsciiType.instance).build();
+ TableMetadata.Builder metadata2 =
+ TableMetadata.builder(KEYSPACE, CF2)
+ .addPartitionKeyColumn("key", BytesType.instance)
+ .addClusteringColumn("col", AsciiType.instance)
+ .addRegularColumn("a", AsciiType.instance)
+ .addRegularColumn("b", AsciiType.instance);
- CFMetaData metadata3 = CFMetaData.Builder.create(KEYSPACE, CF3)
- .addPartitionKey("key", BytesType.instance)
- .addClusteringColumn("col", AsciiType.instance)
- .addRegularColumn("a", AsciiType.instance)
- .addRegularColumn("b", AsciiType.instance)
- .addRegularColumn("c", AsciiType.instance)
- .addRegularColumn("d", AsciiType.instance)
- .addRegularColumn("e", AsciiType.instance)
- .addRegularColumn("f", AsciiType.instance).build();
+ TableMetadata.Builder metadata3 =
+ TableMetadata.builder(KEYSPACE, CF3)
+ .addPartitionKeyColumn("key", BytesType.instance)
+ .addClusteringColumn("col", AsciiType.instance)
+ .addRegularColumn("a", AsciiType.instance)
+ .addRegularColumn("b", AsciiType.instance)
+ .addRegularColumn("c", AsciiType.instance)
+ .addRegularColumn("d", AsciiType.instance)
+ .addRegularColumn("e", AsciiType.instance)
+ .addRegularColumn("f", AsciiType.instance);
- CFMetaData metadata4 = CFMetaData.Builder.create(KEYSPACE, CF4)
- .addPartitionKey("key", BytesType.instance)
- .addClusteringColumn("col", AsciiType.instance)
- .addRegularColumn("a", AsciiType.instance)
- .addRegularColumn("b", AsciiType.instance)
- .addRegularColumn("c", AsciiType.instance)
- .addRegularColumn("d", AsciiType.instance)
- .addRegularColumn("e", AsciiType.instance)
- .addRegularColumn("f", AsciiType.instance).build();
+ TableMetadata.Builder metadata4 =
+ TableMetadata.builder(KEYSPACE, CF4)
+ .addPartitionKeyColumn("key", BytesType.instance)
+ .addClusteringColumn("col", AsciiType.instance)
+ .addRegularColumn("a", AsciiType.instance)
+ .addRegularColumn("b", AsciiType.instance)
+ .addRegularColumn("c", AsciiType.instance)
+ .addRegularColumn("d", AsciiType.instance)
+ .addRegularColumn("e", AsciiType.instance)
+ .addRegularColumn("f", AsciiType.instance);
- CFMetaData metadata5 = CFMetaData.Builder.create(KEYSPACE, CF5)
- .addPartitionKey("key", BytesType.instance)
- .addClusteringColumn("col", AsciiType.instance)
- .addRegularColumn("a", AsciiType.instance)
- .addRegularColumn("b", AsciiType.instance)
- .addRegularColumn("c", AsciiType.instance)
- .addRegularColumn("d", AsciiType.instance)
- .addRegularColumn("e", AsciiType.instance)
- .addRegularColumn("f", AsciiType.instance).build();
+ TableMetadata.Builder metadata5 =
+ TableMetadata.builder(KEYSPACE, CF5)
+ .addPartitionKeyColumn("key", BytesType.instance)
+ .addClusteringColumn("col", AsciiType.instance)
+ .addRegularColumn("a", AsciiType.instance)
+ .addRegularColumn("b", AsciiType.instance)
+ .addRegularColumn("c", AsciiType.instance)
+ .addRegularColumn("d", AsciiType.instance)
+ .addRegularColumn("e", AsciiType.instance)
+ .addRegularColumn("f", AsciiType.instance);
+ TableMetadata.Builder metadata6 =
+ TableMetadata.builder(KEYSPACE, CF6)
+ .addPartitionKeyColumn("key", BytesType.instance)
+ .addStaticColumn("s", AsciiType.instance)
+ .addClusteringColumn("col", AsciiType.instance)
+ .addRegularColumn("a", AsciiType.instance)
+ .addRegularColumn("b", AsciiType.instance)
+ .caching(CachingParams.CACHE_EVERYTHING);
+
+ TableMetadata.Builder metadata7 =
+ TableMetadata.builder(KEYSPACE, CF7)
+ .flags(EnumSet.of(TableMetadata.Flag.COUNTER))
+ .addPartitionKeyColumn("key", BytesType.instance)
+ .addClusteringColumn("col", AsciiType.instance)
+ .addRegularColumn("c", CounterColumnType.instance);
+
+ TableMetadata.Builder metadata8 =
+ TableMetadata.builder(KEYSPACE, CF8)
+ .addPartitionKeyColumn("key", BytesType.instance)
+ .addClusteringColumn("col", AsciiType.instance)
+ .addRegularColumn("a", AsciiType.instance)
+ .addRegularColumn("b", AsciiType.instance)
+ .addRegularColumn("c", SetType.getInstance(AsciiType.instance, true));
+
+ TableMetadata.Builder metadata9 =
+ TableMetadata.builder(KEYSPACE, CF9)
+ .addPartitionKeyColumn("key", Int32Type.instance)
+ .addClusteringColumn("col", ReversedType.getInstance(Int32Type.instance))
+ .addRegularColumn("a", AsciiType.instance);
SchemaLoader.prepareServer();
SchemaLoader.createKeyspace(KEYSPACE,
@@ -115,7 +198,13 @@
metadata2,
metadata3,
metadata4,
- metadata5);
+ metadata5,
+ metadata6,
+ metadata7,
+ metadata8,
+ metadata9);
+
+ LocalSessionAccessor.startup();
}
@Test
@@ -123,7 +212,7 @@
{
ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(CF1);
- new RowUpdateBuilder(cfs.metadata, 0, ByteBufferUtil.bytes("key1"))
+ new RowUpdateBuilder(cfs.metadata(), 0, ByteBufferUtil.bytes("key1"))
.clustering("Column1")
.add("val", ByteBufferUtil.bytes("abcd"))
.build()
@@ -131,7 +220,7 @@
cfs.forceBlockingFlush();
- new RowUpdateBuilder(cfs.metadata, 0, ByteBufferUtil.bytes("key2"))
+ new RowUpdateBuilder(cfs.metadata(), 0, ByteBufferUtil.bytes("key2"))
.clustering("Column1")
.add("val", ByteBufferUtil.bytes("abcd"))
.build()
@@ -151,7 +240,7 @@
cfs.truncateBlocking();
- new RowUpdateBuilder(cfs.metadata, 0, ByteBufferUtil.bytes("key"))
+ new RowUpdateBuilder(cfs.metadata(), 0, ByteBufferUtil.bytes("key"))
.clustering("cc")
.add("a", ByteBufferUtil.bytes("abcd"))
.build()
@@ -159,7 +248,7 @@
cfs.forceBlockingFlush();
- new RowUpdateBuilder(cfs.metadata, 0, ByteBufferUtil.bytes("key"))
+ new RowUpdateBuilder(cfs.metadata(), 0, ByteBufferUtil.bytes("key"))
.clustering("dd")
.add("a", ByteBufferUtil.bytes("abcd"))
.build()
@@ -182,7 +271,7 @@
cfs.truncateBlocking();
- new RowUpdateBuilder(cfs.metadata, 0, ByteBufferUtil.bytes("key"))
+ new RowUpdateBuilder(cfs.metadata(), 0, ByteBufferUtil.bytes("key"))
.clustering("cc")
.add("a", ByteBufferUtil.bytes("abcd"))
.build()
@@ -190,7 +279,7 @@
cfs.forceBlockingFlush();
- new RowUpdateBuilder(cfs.metadata, 0, ByteBufferUtil.bytes("key"))
+ new RowUpdateBuilder(cfs.metadata(), 0, ByteBufferUtil.bytes("key"))
.clustering("dd")
.add("a", ByteBufferUtil.bytes("abcd"))
.build()
@@ -236,9 +325,68 @@
// Given the data above, when the keys are sorted and the deletions removed, we should
// get these clustering rows in this order
String[] expectedRows = new String[] { "aa", "ff", "ee", "cc", "dd", "cc", "bb"};
- int nowInSeconds = FBUtilities.nowInSeconds();
- List<UnfilteredPartitionIterator> iterators = writeAndThenReadPartitions(cfs, groups, nowInSeconds);
+ List<ByteBuffer> buffers = new ArrayList<>(groups.length);
+ int nowInSeconds = FBUtilities.nowInSeconds();
+ ColumnFilter columnFilter = ColumnFilter.allRegularColumnsBuilder(cfs.metadata()).build();
+ RowFilter rowFilter = RowFilter.create();
+ Slice slice = Slice.make(ClusteringBound.BOTTOM, ClusteringBound.TOP);
+ ClusteringIndexSliceFilter sliceFilter = new ClusteringIndexSliceFilter(Slices.with(cfs.metadata().comparator, slice), false);
+
+ for (String[][] group : groups)
+ {
+ cfs.truncateBlocking();
+
+ List<SinglePartitionReadCommand> commands = new ArrayList<>(group.length);
+
+ for (String[] data : group)
+ {
+ if (data[0].equals("1"))
+ {
+ new RowUpdateBuilder(cfs.metadata(), 0, ByteBufferUtil.bytes(data[1]))
+ .clustering(data[2])
+ .add(data[3], ByteBufferUtil.bytes("blah"))
+ .build()
+ .apply();
+ }
+ else
+ {
+ RowUpdateBuilder.deleteRow(cfs.metadata(), FBUtilities.timestampMicros(), ByteBufferUtil.bytes(data[1]), data[2]).apply();
+ }
+ commands.add(SinglePartitionReadCommand.create(cfs.metadata(), nowInSeconds, columnFilter, rowFilter, DataLimits.NONE, Util.dk(data[1]), sliceFilter));
+ }
+
+ cfs.forceBlockingFlush();
+
+ ReadQuery query = new SinglePartitionReadCommand.Group(commands, DataLimits.NONE);
+
+ try (ReadExecutionController executionController = query.executionController();
+ UnfilteredPartitionIterator iter = query.executeLocally(executionController);
+ DataOutputBuffer buffer = new DataOutputBuffer())
+ {
+ UnfilteredPartitionIterators.serializerForIntraNode().serialize(iter,
+ columnFilter,
+ buffer,
+ MessagingService.current_version);
+ buffers.add(buffer.buffer());
+ }
+ }
+
+ // deserialize, merge and check the results are all there
+ List<UnfilteredPartitionIterator> iterators = new ArrayList<>();
+
+ for (ByteBuffer buffer : buffers)
+ {
+ try (DataInputBuffer in = new DataInputBuffer(buffer, true))
+ {
+ iterators.add(UnfilteredPartitionIterators.serializerForIntraNode().deserialize(in,
+ MessagingService.current_version,
+ cfs.metadata(),
+ columnFilter,
+ DeserializationHelper.Flag.LOCAL));
+ }
+ }
+
UnfilteredPartitionIterators.MergeListener listener =
new UnfilteredPartitionIterators.MergeListener()
{
@@ -253,7 +401,7 @@
}
};
- try (PartitionIterator partitionIterator = UnfilteredPartitionIterators.filter(UnfilteredPartitionIterators.merge(iterators, nowInSeconds, listener), nowInSeconds))
+ try (PartitionIterator partitionIterator = UnfilteredPartitionIterators.filter(UnfilteredPartitionIterators.merge(iterators, listener), nowInSeconds))
{
int i = 0;
@@ -266,7 +414,7 @@
while (rowIterator.hasNext())
{
Row row = rowIterator.next();
- assertEquals("col=" + expectedRows[i++], row.clustering().toString(cfs.metadata));
+ assertEquals("col=" + expectedRows[i++], row.clustering().toString(cfs.metadata()));
//System.out.print(row.toString(cfs.metadata, true));
}
}
@@ -277,11 +425,37 @@
}
}
- /**
- * This test will create several partitions with several rows each. Then, it will perform up to 5 row deletions on
- * some partitions. We check that when reading the partitions, the maximum number of tombstones reported in the
- * metrics is indeed equal to 5.
- */
+ @Test
+ public void testSerializer() throws IOException
+ {
+ ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(CF2);
+
+ new RowUpdateBuilder(cfs.metadata.get(), 0, ByteBufferUtil.bytes("key"))
+ .clustering("dd")
+ .add("a", ByteBufferUtil.bytes("abcd"))
+ .build()
+ .apply();
+
+ ReadCommand readCommand = Util.cmd(cfs, Util.dk("key")).includeRow("dd").build();
+ int messagingVersion = MessagingService.current_version;
+ FakeOutputStream out = new FakeOutputStream();
+ Tracing.instance.newSession(Tracing.TraceType.QUERY);
+ Message<ReadCommand> messageOut = Message.out(Verb.READ_REQ, readCommand);
+ long size = messageOut.serializedSize(messagingVersion);
+ Message.serializer.serialize(messageOut, new WrappedDataOutputStreamPlus(out), messagingVersion);
+ Assert.assertEquals(size, out.count);
+ }
+
+ static class FakeOutputStream extends OutputStream
+ {
+ long count;
+
+ public void write(int b) throws IOException
+ {
+ count++;
+ }
+ }
+
@Test
public void testCountDeletedRows() throws Exception
{
@@ -318,18 +492,59 @@
new String[] { "-1", "key2", "dd", "d" }
}
};
- int nowInSeconds = FBUtilities.nowInSeconds();
- writeAndThenReadPartitions(cfs, groups, nowInSeconds);
+ List<ByteBuffer> buffers = new ArrayList<>(groups.length);
+ int nowInSeconds = FBUtilities.nowInSeconds();
+ ColumnFilter columnFilter = ColumnFilter.allRegularColumnsBuilder(cfs.metadata()).build();
+ RowFilter rowFilter = RowFilter.create();
+ Slice slice = Slice.make(ClusteringBound.BOTTOM, ClusteringBound.TOP);
+ ClusteringIndexSliceFilter sliceFilter = new ClusteringIndexSliceFilter(
+ Slices.with(cfs.metadata().comparator, slice), false);
+
+ for (String[][] group : groups)
+ {
+ cfs.truncateBlocking();
+
+ List<SinglePartitionReadCommand> commands = new ArrayList<>(group.length);
+
+ for (String[] data : group)
+ {
+ if (data[0].equals("1"))
+ {
+ new RowUpdateBuilder(cfs.metadata(), 0, ByteBufferUtil.bytes(data[1]))
+ .clustering(data[2])
+ .add(data[3], ByteBufferUtil.bytes("blah"))
+ .build()
+ .apply();
+ }
+ else
+ {
+ RowUpdateBuilder.deleteRow(cfs.metadata(), FBUtilities.timestampMicros(),
+ ByteBufferUtil.bytes(data[1]), data[2]).apply();
+ }
+ commands.add(SinglePartitionReadCommand.create(cfs.metadata(), nowInSeconds, columnFilter, rowFilter,
+ DataLimits.NONE, Util.dk(data[1]), sliceFilter));
+ }
+
+ cfs.forceBlockingFlush();
+
+ ReadQuery query = new SinglePartitionReadCommand.Group(commands, DataLimits.NONE);
+
+ try (ReadExecutionController executionController = query.executionController();
+ UnfilteredPartitionIterator iter = query.executeLocally(executionController);
+ DataOutputBuffer buffer = new DataOutputBuffer())
+ {
+ UnfilteredPartitionIterators.serializerForIntraNode().serialize(iter,
+ columnFilter,
+ buffer,
+ MessagingService.current_version);
+ buffers.add(buffer.buffer());
+ }
+ }
assertEquals(5, cfs.metric.tombstoneScannedHistogram.cf.getSnapshot().getMax());
}
- /**
- * This test will create several partitions with several rows each and no deletions. We check that when reading the
- * partitions, the maximum number of tombstones reported in the metrics is equal to 1, which is apparently the
- * default max value for histograms in the metrics lib (equivalent to having no element reported).
- */
@Test
public void testCountWithNoDeletedRow() throws Exception
{
@@ -354,26 +569,13 @@
}
};
- int nowInSeconds = FBUtilities.nowInSeconds();
-
- writeAndThenReadPartitions(cfs, groups, nowInSeconds);
-
- assertEquals(1, cfs.metric.tombstoneScannedHistogram.cf.getSnapshot().getMax());
- }
-
- /**
- * Writes rows to the column family store using the groups as input and then reads them. Returns the iterators from
- * the read.
- */
- private List<UnfilteredPartitionIterator> writeAndThenReadPartitions(ColumnFamilyStore cfs, String[][][] groups,
- int nowInSeconds) throws IOException
- {
List<ByteBuffer> buffers = new ArrayList<>(groups.length);
- ColumnFilter columnFilter = ColumnFilter.allColumnsBuilder(cfs.metadata).build();
+ int nowInSeconds = FBUtilities.nowInSeconds();
+ ColumnFilter columnFilter = ColumnFilter.allRegularColumnsBuilder(cfs.metadata()).build();
RowFilter rowFilter = RowFilter.create();
Slice slice = Slice.make(ClusteringBound.BOTTOM, ClusteringBound.TOP);
ClusteringIndexSliceFilter sliceFilter = new ClusteringIndexSliceFilter(
- Slices.with(cfs.metadata.comparator, slice), false);
+ Slices.with(cfs.metadata().comparator, slice), false);
for (String[][] group : groups)
{
@@ -385,7 +587,7 @@
{
if (data[0].equals("1"))
{
- new RowUpdateBuilder(cfs.metadata, 0, ByteBufferUtil.bytes(data[1]))
+ new RowUpdateBuilder(cfs.metadata(), 0, ByteBufferUtil.bytes(data[1]))
.clustering(data[2])
.add(data[3], ByteBufferUtil.bytes("blah"))
.build()
@@ -393,10 +595,10 @@
}
else
{
- RowUpdateBuilder.deleteRow(cfs.metadata, FBUtilities.timestampMicros(),
+ RowUpdateBuilder.deleteRow(cfs.metadata(), FBUtilities.timestampMicros(),
ByteBufferUtil.bytes(data[1]), data[2]).apply();
}
- commands.add(SinglePartitionReadCommand.create(cfs.metadata, nowInSeconds, columnFilter, rowFilter,
+ commands.add(SinglePartitionReadCommand.create(cfs.metadata(), nowInSeconds, columnFilter, rowFilter,
DataLimits.NONE, Util.dk(data[1]), sliceFilter));
}
@@ -416,22 +618,702 @@
}
}
- // deserialize, merge and check the results are all there
- List<UnfilteredPartitionIterator> iterators = new ArrayList<>();
+ assertEquals(1, cfs.metric.tombstoneScannedHistogram.cf.getSnapshot().getMax());
+ }
- for (ByteBuffer buffer : buffers)
+ @Test
+ public void testSinglePartitionSliceRepairedDataTracking() throws Exception
+ {
+ ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(CF2);
+ ReadCommand readCommand = Util.cmd(cfs, Util.dk("key")).build();
+ testRepairedDataTracking(cfs, readCommand);
+ }
+
+ @Test
+ public void testPartitionRangeRepairedDataTracking() throws Exception
+ {
+ ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(CF2);
+ ReadCommand readCommand = Util.cmd(cfs).build();
+ testRepairedDataTracking(cfs, readCommand);
+ }
+
+ @Test
+ public void testSinglePartitionNamesRepairedDataTracking() throws Exception
+ {
+ ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(CF2);
+ ReadCommand readCommand = Util.cmd(cfs, Util.dk("key")).includeRow("cc").includeRow("dd").build();
+ testRepairedDataTracking(cfs, readCommand);
+ }
+
+ @Test
+ public void testSinglePartitionNamesSkipsOptimisationsIfTrackingRepairedData()
+ {
+ // when tracking, the optimizations of querying sstables in timestamp order and
+ // returning once all requested columns are not available so just assert that
+ // all sstables are read when performing such queries
+ ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(CF2);
+ cfs.truncateBlocking();
+ cfs.disableAutoCompaction();
+
+ new RowUpdateBuilder(cfs.metadata(), 0, ByteBufferUtil.bytes("key"))
+ .clustering("dd")
+ .add("a", ByteBufferUtil.bytes("abcd"))
+ .build()
+ .apply();
+
+ cfs.forceBlockingFlush();
+
+ new RowUpdateBuilder(cfs.metadata(), 1, ByteBufferUtil.bytes("key"))
+ .clustering("dd")
+ .add("a", ByteBufferUtil.bytes("wxyz"))
+ .build()
+ .apply();
+
+ cfs.forceBlockingFlush();
+ List<SSTableReader> sstables = new ArrayList<>(cfs.getLiveSSTables());
+ assertEquals(2, sstables.size());
+ Collections.sort(sstables, SSTableReader.maxTimestampDescending);
+
+ ReadCommand readCommand = Util.cmd(cfs, Util.dk("key")).includeRow("dd").columns("a").build();
+
+ assertEquals(0, readCount(sstables.get(0)));
+ assertEquals(0, readCount(sstables.get(1)));
+ ReadCommand withTracking = readCommand.copy();
+ withTracking.trackRepairedStatus();
+ Util.getAll(withTracking);
+ assertEquals(1, readCount(sstables.get(0)));
+ assertEquals(1, readCount(sstables.get(1)));
+
+ // same command without tracking touches only the table with the higher timestamp
+ Util.getAll(readCommand.copy());
+ assertEquals(2, readCount(sstables.get(0)));
+ assertEquals(1, readCount(sstables.get(1)));
+ }
+
+ @Test
+ public void dontIncludeLegacyCounterContextInDigest() throws IOException
+ {
+ // Serializations of a CounterContext containing legacy (pre-2.1) shards
+ // can legitimately differ across replicas. For this reason, the context
+ // bytes are omitted from the repaired digest if they contain legacy shards.
+ // This clearly has a tradeoff with the efficacy of the digest, without doing
+ // so false positive digest mismatches will be reported for scenarios where
+ // there is nothing that can be done to "fix" the replicas
+ ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(CF7);
+ cfs.truncateBlocking();
+ cfs.disableAutoCompaction();
+
+ // insert a row with the counter column having value 0, in a legacy shard.
+ new RowUpdateBuilder(cfs.metadata(), 0, ByteBufferUtil.bytes("key"))
+ .clustering("aa")
+ .addLegacyCounterCell("c", 0L)
+ .build()
+ .apply();
+ cfs.forceBlockingFlush();
+ cfs.getLiveSSTables().forEach(sstable -> mutateRepaired(cfs, sstable, 111, null));
+
+ // execute a read and capture the digest
+ ReadCommand readCommand = Util.cmd(cfs, Util.dk("key")).build();
+ ByteBuffer digestWithLegacyCounter0 = performReadAndVerifyRepairedInfo(readCommand, 1, 1, true);
+ assertFalse(EMPTY_BYTE_BUFFER.equals(digestWithLegacyCounter0));
+
+ // truncate, then re-insert the same partition, but this time with a legacy
+ // shard having the value 1. The repaired digest should match the previous, as
+ // the values (context) are not included, only the cell metadata (ttl, timestamp, etc)
+ cfs.truncateBlocking();
+ new RowUpdateBuilder(cfs.metadata(), 0, ByteBufferUtil.bytes("key"))
+ .clustering("aa")
+ .addLegacyCounterCell("c", 1L)
+ .build()
+ .apply();
+ cfs.forceBlockingFlush();
+ cfs.getLiveSSTables().forEach(sstable -> mutateRepaired(cfs, sstable, 111, null));
+
+ ByteBuffer digestWithLegacyCounter1 = performReadAndVerifyRepairedInfo(readCommand, 1, 1, true);
+ assertEquals(digestWithLegacyCounter0, digestWithLegacyCounter1);
+
+ // truncate, then re-insert the same partition, but this time with a non-legacy
+ // counter cell present. The repaired digest should not match the previous ones
+ // as this time the value (context) is included.
+ cfs.truncateBlocking();
+ new RowUpdateBuilder(cfs.metadata(), 0, ByteBufferUtil.bytes("key"))
+ .clustering("aa")
+ .add("c", 1L)
+ .build()
+ .apply();
+ cfs.forceBlockingFlush();
+ cfs.getLiveSSTables().forEach(sstable -> mutateRepaired(cfs, sstable, 111, null));
+
+ ByteBuffer digestWithCounterCell = performReadAndVerifyRepairedInfo(readCommand, 1, 1, true);
+ assertFalse(EMPTY_BYTE_BUFFER.equals(digestWithCounterCell));
+ assertFalse(digestWithLegacyCounter0.equals(digestWithCounterCell));
+ assertFalse(digestWithLegacyCounter1.equals(digestWithCounterCell));
+ }
+
+ /**
+ * Writes a single partition containing a single row and reads using a partition read. The single
+ * row includes 1 live simple column, 1 simple tombstone and 1 complex column with a complex
+ * deletion and a live cell. The repaired data digests generated by executing the same query
+ * before and after the tombstones become eligible for purging should not match each other.
+ * Also, neither digest should be empty as the partition is not made empty by the purging.
+ */
+ @Test
+ public void purgeGCableTombstonesBeforeCalculatingDigest() throws Exception
+ {
+ ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(CF8);
+ cfs.truncateBlocking();
+ cfs.disableAutoCompaction();
+ setGCGrace(cfs, 600);
+
+ DecoratedKey[] keys = new DecoratedKey[] { Util.dk("key0"), Util.dk("key1"), Util.dk("key2"), Util.dk("key3") };
+ int nowInSec = FBUtilities.nowInSeconds();
+ TableMetadata cfm = cfs.metadata();
+
+ // A simple tombstone
+ new RowUpdateBuilder(cfs.metadata(), 0, keys[0]).clustering("cc").delete("a").build().apply();
+
+ // Collection with an associated complex deletion
+ PartitionUpdate.SimpleBuilder builder = PartitionUpdate.simpleBuilder(cfs.metadata(), keys[1]).timestamp(0);
+ builder.row("cc").add("c", ImmutableSet.of("element1", "element2"));
+ builder.buildAsMutation().apply();
+
+ // RangeTombstone and a row (not covered by the RT). The row contains a regular tombstone which will not be
+ // purged. This is to prevent the partition from being fully purged and removed from the final results
+ new RowUpdateBuilder(cfs.metadata(), nowInSec, 0L, keys[2]).addRangeTombstone("aa", "bb").build().apply();
+ new RowUpdateBuilder(cfs.metadata(), nowInSec+ 1000, 1000L, keys[2]).clustering("cc").delete("a").build().apply();
+
+ // Partition with 2 rows, one fully deleted
+ new RowUpdateBuilder(cfs.metadata.get(), 0, keys[3]).clustering("bb").add("a", ByteBufferUtil.bytes("a")).delete("b").build().apply();
+ RowUpdateBuilder.deleteRow(cfs.metadata(), 0, keys[3], "cc").apply();
+ cfs.forceBlockingFlush();
+ cfs.getLiveSSTables().forEach(sstable -> mutateRepaired(cfs, sstable, 111, null));
+
+ Map<DecoratedKey, ByteBuffer> digestsWithTombstones = new HashMap<>();
+ //Tombstones are not yet purgable
+ for (DecoratedKey key : keys)
{
- try (DataInputBuffer in = new DataInputBuffer(buffer, true))
- {
- iterators.add(UnfilteredPartitionIterators.serializerForIntraNode().deserialize(in,
- MessagingService.current_version,
- cfs.metadata,
- columnFilter,
- SerializationHelper.Flag.LOCAL));
- }
+ ReadCommand cmd = Util.cmd(cfs, key).withNowInSeconds(nowInSec).build();
+ cmd.trackRepairedStatus();
+ Partition partition = Util.getOnlyPartitionUnfiltered(cmd);
+ assertFalse(partition.isEmpty());
+ partition.unfilteredIterator().forEachRemaining(u -> {
+ // must be either a RT, or a row containing some kind of deletion
+ assertTrue(u.isRangeTombstoneMarker() || ((Row)u).hasDeletion(cmd.nowInSec()));
+ });
+ ByteBuffer digestWithTombstones = cmd.getRepairedDataDigest();
+ // None should generate an empty digest
+ assertDigestsDiffer(EMPTY_BYTE_BUFFER, digestWithTombstones);
+ digestsWithTombstones.put(key, digestWithTombstones);
}
- return iterators;
+ // Make tombstones eligible for purging and re-run cmd with an incremented nowInSec
+ setGCGrace(cfs, 0);
+
+ //Tombstones are now purgable, so won't be in the read results and produce different digests
+ for (DecoratedKey key : keys)
+ {
+ ReadCommand cmd = Util.cmd(cfs, key).withNowInSeconds(nowInSec + 60).build();
+ cmd.trackRepairedStatus();
+ Partition partition = Util.getOnlyPartitionUnfiltered(cmd);
+ assertFalse(partition.isEmpty());
+ partition.unfilteredIterator().forEachRemaining(u -> {
+ // After purging, only rows without any deletions should remain.
+ // The one exception is "key2:cc" which has a regular column tombstone which is not
+ // eligible for purging. This is to prevent the partition from being fully purged
+ // when its RT is removed.
+ assertTrue(u.isRow());
+ Row r = (Row)u;
+ assertTrue(!r.hasDeletion(cmd.nowInSec())
+ || (key.equals(keys[2]) && r.clustering()
+ .get(0)
+ .equals(AsciiType.instance.fromString("cc"))));
+
+ });
+ ByteBuffer digestWithoutTombstones = cmd.getRepairedDataDigest();
+ // not an empty digest
+ assertDigestsDiffer(EMPTY_BYTE_BUFFER, digestWithoutTombstones);
+ // should not match the pre-purge digest
+ assertDigestsDiffer(digestsWithTombstones.get(key), digestWithoutTombstones);
+ }
+ }
+
+ @Test
+ public void testRepairedDataOverreadMetrics()
+ {
+ ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(CF9);
+ cfs.truncateBlocking();
+ cfs.disableAutoCompaction();
+ cfs.metadata().withSwapped(cfs.metadata().params.unbuild()
+ .caching(CachingParams.CACHE_NOTHING)
+ .build());
+ // Insert and repair
+ insert(cfs, IntStream.range(0, 10), () -> IntStream.range(0, 10));
+ cfs.forceBlockingFlush();
+ cfs.getLiveSSTables().forEach(sstable -> mutateRepaired(cfs, sstable, 111, null));
+ // Insert and leave unrepaired
+ insert(cfs, IntStream.range(0, 10), () -> IntStream.range(10, 20));
+
+ // Single partition reads
+ int limit = 5;
+ ReadCommand cmd = Util.cmd(cfs, ByteBufferUtil.bytes(0)).withLimit(limit).build();
+ assertEquals(0, getAndResetOverreadCount(cfs));
+
+ // No overreads if not tracking
+ readAndCheckRowCount(Collections.singletonList(Util.getOnlyPartition(cmd)), limit);
+ assertEquals(0, getAndResetOverreadCount(cfs));
+
+ // Overread up to (limit - 1) if tracking is enabled
+ cmd = cmd.copy();
+ cmd.trackRepairedStatus();
+ readAndCheckRowCount(Collections.singletonList(Util.getOnlyPartition(cmd)), limit);
+ // overread count is always < limit as the first read is counted during merging (and so is expected)
+ assertEquals(limit - 1, getAndResetOverreadCount(cfs));
+
+ // if limit already requires reading all repaired data, no overreads should be recorded
+ limit = 20;
+ cmd = Util.cmd(cfs, ByteBufferUtil.bytes(0)).withLimit(limit).build();
+ readAndCheckRowCount(Collections.singletonList(Util.getOnlyPartition(cmd)), limit);
+ assertEquals(0, getAndResetOverreadCount(cfs));
+
+ // Range reads
+ limit = 5;
+ cmd = Util.cmd(cfs).withLimit(limit).build();
+ assertEquals(0, getAndResetOverreadCount(cfs));
+ // No overreads if not tracking
+ readAndCheckRowCount(Util.getAll(cmd), limit);
+ assertEquals(0, getAndResetOverreadCount(cfs));
+
+ // Overread up to (limit - 1) if tracking is enabled
+ cmd = cmd.copy();
+ cmd.trackRepairedStatus();
+ readAndCheckRowCount(Util.getAll(cmd), limit);
+ assertEquals(limit - 1, getAndResetOverreadCount(cfs));
+
+ // if limit already requires reading all repaired data, no overreads should be recorded
+ limit = 100;
+ cmd = Util.cmd(cfs).withLimit(limit).build();
+ readAndCheckRowCount(Util.getAll(cmd), limit);
+ assertEquals(0, getAndResetOverreadCount(cfs));
+ }
+
+ private void setGCGrace(ColumnFamilyStore cfs, int gcGrace)
+ {
+ TableParams newParams = cfs.metadata().params.unbuild().gcGraceSeconds(gcGrace).build();
+ KeyspaceMetadata keyspaceMetadata = Schema.instance.getKeyspaceMetadata(cfs.metadata().keyspace);
+ Schema.instance.load(
+ keyspaceMetadata.withSwapped(
+ keyspaceMetadata.tables.withSwapped(
+ cfs.metadata().withSwapped(newParams))));
+ }
+
+ private long getAndResetOverreadCount(ColumnFamilyStore cfs)
+ {
+ // always clear the histogram after reading to make comparisons & asserts easier
+ long rows = cfs.metric.repairedDataTrackingOverreadRows.cf.getSnapshot().getMax();
+ ((ClearableHistogram)cfs.metric.repairedDataTrackingOverreadRows.cf).clear();
+ return rows;
+ }
+
+ private void readAndCheckRowCount(Iterable<FilteredPartition> partitions, int expected)
+ {
+ int count = 0;
+ for (Partition partition : partitions)
+ {
+ assertFalse(partition.isEmpty());
+ try (UnfilteredRowIterator iter = partition.unfilteredIterator())
+ {
+ while (iter.hasNext())
+ {
+ iter.next();
+ count++;
+ }
+ }
+ }
+ assertEquals(expected, count);
+ }
+
+ private void insert(ColumnFamilyStore cfs, IntStream partitionIds, Supplier<IntStream> rowIds)
+ {
+ partitionIds.mapToObj(ByteBufferUtil::bytes)
+ .forEach( pk ->
+ rowIds.get().forEach( c ->
+ new RowUpdateBuilder(cfs.metadata(), 0, pk)
+ .clustering(c)
+ .add("a", ByteBufferUtil.bytes("abcd"))
+ .build()
+ .apply()
+
+ ));
+ }
+
+ private void assertDigestsDiffer(ByteBuffer b0, ByteBuffer b1)
+ {
+ assertTrue(ByteBufferUtil.compareUnsigned(b0, b1) != 0);
+ }
+
+ @Test
+ public void partitionReadFullyPurged() throws Exception
+ {
+ ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(CF6);
+ ReadCommand partitionRead = Util.cmd(cfs, Util.dk("key")).build();
+ fullyPurgedPartitionCreatesEmptyDigest(cfs, partitionRead);
+ }
+
+ @Test
+ public void rangeReadFullyPurged() throws Exception
+ {
+ ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(CF6);
+ ReadCommand rangeRead = Util.cmd(cfs).build();
+ fullyPurgedPartitionCreatesEmptyDigest(cfs, rangeRead);
+ }
+
+ /**
+ * Writes a single partition containing only a single row deletion and reads with either a range or
+ * partition query. Before the row deletion is eligible for purging, it should appear in the query
+ * results and cause a non-empty repaired data digest to be generated. Repeating the query after
+ * the row deletion is eligible for purging, both the result set and the repaired data digest should
+ * be empty.
+ */
+ private void fullyPurgedPartitionCreatesEmptyDigest(ColumnFamilyStore cfs, ReadCommand command) throws Exception
+ {
+ cfs.truncateBlocking();
+ cfs.disableAutoCompaction();
+ setGCGrace(cfs, 600);
+
+ // Partition with a fully deleted static row and a single, fully deleted regular row
+ RowUpdateBuilder.deleteRow(cfs.metadata(), 0, ByteBufferUtil.bytes("key")).apply();
+ RowUpdateBuilder.deleteRow(cfs.metadata(), 0, ByteBufferUtil.bytes("key"), "cc").apply();
+ cfs.forceBlockingFlush();
+ cfs.getLiveSSTables().forEach(sstable -> mutateRepaired(cfs, sstable, 111, null));
+
+ command.trackRepairedStatus();
+ List<ImmutableBTreePartition> partitions = Util.getAllUnfiltered(command);
+ assertEquals(1, partitions.size());
+ ByteBuffer digestWithTombstones = command.getRepairedDataDigest();
+ assertTrue(ByteBufferUtil.compareUnsigned(EMPTY_BYTE_BUFFER, digestWithTombstones) != 0);
+
+ // Make tombstones eligible for purging and re-run cmd with an incremented nowInSec
+ setGCGrace(cfs, 0);
+
+ AbstractReadCommandBuilder builder = command instanceof PartitionRangeReadCommand
+ ? Util.cmd(cfs)
+ : Util.cmd(cfs, Util.dk("key"));
+ builder.withNowInSeconds(command.nowInSec() + 60);
+ command = builder.build();
+ command.trackRepairedStatus();
+
+ partitions = Util.getAllUnfiltered(command);
+ assertTrue(partitions.isEmpty());
+ ByteBuffer digestWithoutTombstones = command.getRepairedDataDigest();
+ assertEquals(0, ByteBufferUtil.compareUnsigned(EMPTY_BYTE_BUFFER, digestWithoutTombstones));
+ }
+
+ /**
+ * Verifies that during range reads which include multiple partitions, fully purged partitions
+ * have no material effect on the calculated digest. This test writes two sstables, each containing
+ * a single partition; the first is live and the second fully deleted and eligible for purging.
+ * Initially, only the sstable containing the live partition is marked repaired, while a range read
+ * which covers both partitions is performed to generate a digest. Then the sstable containing the
+ * purged partition is also marked repaired and the query reexecuted. The digests produced by both
+ * queries should match as the digest calculation should exclude the fully purged partition.
+ */
+ @Test
+ public void mixedPurgedAndNonPurgedPartitions()
+ {
+ ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(CF6);
+ cfs.truncateBlocking();
+ cfs.disableAutoCompaction();
+ setGCGrace(cfs, 0);
+
+ ReadCommand command = Util.cmd(cfs).withNowInSeconds(FBUtilities.nowInSeconds() + 60).build();
+
+ // Live partition in a repaired sstable, so included in the digest calculation
+ new RowUpdateBuilder(cfs.metadata.get(), 0, ByteBufferUtil.bytes("key-0")).clustering("cc").add("a", ByteBufferUtil.bytes("a")).build().apply();
+ cfs.forceBlockingFlush();
+ cfs.getLiveSSTables().forEach(sstable -> mutateRepaired(cfs, sstable, 111, null));
+ // Fully deleted partition (static and regular rows) in an unrepaired sstable, so not included in the intial digest
+ RowUpdateBuilder.deleteRow(cfs.metadata(), 0, ByteBufferUtil.bytes("key-1")).apply();
+ RowUpdateBuilder.deleteRow(cfs.metadata(), 0, ByteBufferUtil.bytes("key-1"), "cc").apply();
+ cfs.forceBlockingFlush();
+
+ command.trackRepairedStatus();
+ List<ImmutableBTreePartition> partitions = Util.getAllUnfiltered(command);
+ assertEquals(1, partitions.size());
+ ByteBuffer digestWithoutPurgedPartition = command.getRepairedDataDigest();
+ assertTrue(ByteBufferUtil.compareUnsigned(EMPTY_BYTE_BUFFER, digestWithoutPurgedPartition) != 0);
+
+ // mark the sstable containing the purged partition as repaired, so both partitions are now
+ // read during in the digest calculation. Because the purged partition is entirely
+ // discarded, the resultant digest should match the earlier one.
+ cfs.getLiveSSTables().forEach(sstable -> mutateRepaired(cfs, sstable, 111, null));
+ command = Util.cmd(cfs).withNowInSeconds(command.nowInSec()).build();
+ command.trackRepairedStatus();
+
+ partitions = Util.getAllUnfiltered(command);
+ assertEquals(1, partitions.size());
+ ByteBuffer digestWithPurgedPartition = command.getRepairedDataDigest();
+ assertEquals(0, ByteBufferUtil.compareUnsigned(digestWithPurgedPartition, digestWithoutPurgedPartition));
+ }
+
+ @Test
+ public void purgingConsidersRepairedDataOnly() throws Exception
+ {
+ // 2 sstables, first is repaired and contains data that is all purgeable
+ // the second is unrepaired and contains non-purgable data. Even though
+ // the partition itself is not fully purged, the repaired data digest
+ // should be empty as there was no non-purgeable, repaired data read.
+ ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(CF6);
+ cfs.truncateBlocking();
+ cfs.disableAutoCompaction();
+ setGCGrace(cfs, 0);
+
+ // Partition with a fully deleted static row and a single, fully deleted row which will be fully purged
+ DecoratedKey key = Util.dk("key");
+ RowUpdateBuilder.deleteRow(cfs.metadata(), 0, key).apply();
+ RowUpdateBuilder.deleteRow(cfs.metadata(), 0, key, "cc").apply();
+ cfs.forceBlockingFlush();
+ cfs.getLiveSSTables().forEach(sstable -> mutateRepaired(cfs, sstable, 111, null));
+
+ new RowUpdateBuilder(cfs.metadata(), 1, key).clustering("cc").add("a", ByteBufferUtil.bytes("a")).build().apply();
+ cfs.forceBlockingFlush();
+
+ int nowInSec = FBUtilities.nowInSeconds() + 10;
+ ReadCommand cmd = Util.cmd(cfs, key).withNowInSeconds(nowInSec).build();
+ cmd.trackRepairedStatus();
+ Partition partition = Util.getOnlyPartitionUnfiltered(cmd);
+ assertFalse(partition.isEmpty());
+ // check that
+ try (UnfilteredRowIterator rows = partition.unfilteredIterator())
+ {
+ assertFalse(rows.isEmpty());
+ Unfiltered unfiltered = rows.next();
+ assertFalse(rows.hasNext());
+ assertTrue(unfiltered.isRow());
+ assertFalse(((Row) unfiltered).hasDeletion(nowInSec));
+ }
+ assertEquals(EMPTY_BYTE_BUFFER, cmd.getRepairedDataDigest());
+ }
+
+ private long readCount(SSTableReader sstable)
+ {
+ return sstable.getReadMeter().count();
+ }
+
+ @Test
+ public void skipRowCacheIfTrackingRepairedData()
+ {
+ ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(CF6);
+
+ cfs.truncateBlocking();
+ cfs.disableAutoCompaction();
+
+ new RowUpdateBuilder(cfs.metadata(), 0, ByteBufferUtil.bytes("key"))
+ .clustering("cc")
+ .add("a", ByteBufferUtil.bytes("abcd"))
+ .build()
+ .apply();
+
+ cfs.forceBlockingFlush();
+
+ ReadCommand readCommand = Util.cmd(cfs, Util.dk("key")).build();
+ assertTrue(cfs.isRowCacheEnabled());
+ // warm the cache
+ assertFalse(Util.getAll(readCommand).isEmpty());
+ long cacheHits = cfs.metric.rowCacheHit.getCount();
+
+ Util.getAll(readCommand);
+ assertTrue(cfs.metric.rowCacheHit.getCount() > cacheHits);
+ cacheHits = cfs.metric.rowCacheHit.getCount();
+
+ ReadCommand withRepairedInfo = readCommand.copy();
+ withRepairedInfo.trackRepairedStatus();
+ Util.getAll(withRepairedInfo);
+ assertEquals(cacheHits, cfs.metric.rowCacheHit.getCount());
+ }
+
+ @Test (expected = IllegalArgumentException.class)
+ public void copyFullAsTransientTest()
+ {
+ ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(CF6);
+ ReadCommand readCommand = Util.cmd(cfs, Util.dk("key")).build();
+ readCommand.copyAsTransientQuery(ReplicaUtils.full(FBUtilities.getBroadcastAddressAndPort()));
+ }
+
+ @Test (expected = IllegalArgumentException.class)
+ public void copyTransientAsDigestQuery()
+ {
+ ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(CF6);
+ ReadCommand readCommand = Util.cmd(cfs, Util.dk("key")).build();
+ readCommand.copyAsDigestQuery(ReplicaUtils.trans(FBUtilities.getBroadcastAddressAndPort()));
+ }
+
+ @Test (expected = IllegalArgumentException.class)
+ public void copyMultipleFullAsTransientTest()
+ {
+ ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(CF6);
+ DecoratedKey key = Util.dk("key");
+ Token token = key.getToken();
+ // Address is unimportant for this test
+ InetAddressAndPort addr = FBUtilities.getBroadcastAddressAndPort();
+ ReadCommand readCommand = Util.cmd(cfs, key).build();
+ readCommand.copyAsTransientQuery(EndpointsForToken.of(token,
+ ReplicaUtils.trans(addr, token),
+ ReplicaUtils.full(addr, token)));
+ }
+
+ @Test (expected = IllegalArgumentException.class)
+ public void copyMultipleTransientAsDigestQuery()
+ {
+ ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(CF6);
+ DecoratedKey key = Util.dk("key");
+ Token token = key.getToken();
+ // Address is unimportant for this test
+ InetAddressAndPort addr = FBUtilities.getBroadcastAddressAndPort();
+ ReadCommand readCommand = Util.cmd(cfs, key).build();
+ readCommand.copyAsDigestQuery(EndpointsForToken.of(token,
+ ReplicaUtils.trans(addr, token),
+ ReplicaUtils.full(addr, token)));
+ }
+
+ private void testRepairedDataTracking(ColumnFamilyStore cfs, ReadCommand readCommand) throws IOException
+ {
+ cfs.truncateBlocking();
+ cfs.disableAutoCompaction();
+
+ new RowUpdateBuilder(cfs.metadata(), 0, ByteBufferUtil.bytes("key"))
+ .clustering("cc")
+ .add("a", ByteBufferUtil.bytes("abcd"))
+ .build()
+ .apply();
+
+ cfs.forceBlockingFlush();
+
+ new RowUpdateBuilder(cfs.metadata(), 1, ByteBufferUtil.bytes("key"))
+ .clustering("dd")
+ .add("a", ByteBufferUtil.bytes("abcd"))
+ .build()
+ .apply();
+
+ cfs.forceBlockingFlush();
+ List<SSTableReader> sstables = new ArrayList<>(cfs.getLiveSSTables());
+ assertEquals(2, sstables.size());
+ sstables.forEach(sstable -> assertFalse(sstable.isRepaired() || sstable.isPendingRepair()));
+ SSTableReader sstable1 = sstables.get(0);
+ SSTableReader sstable2 = sstables.get(1);
+
+ int numPartitions = 1;
+ int rowsPerPartition = 2;
+
+ // Capture all the digest versions as we mutate the table's repaired status. Each time
+ // we make a change, we expect a different digest.
+ Set<ByteBuffer> digests = new HashSet<>();
+ // first time round, nothing has been marked repaired so we expect digest to be an empty buffer and to be marked conclusive
+ ByteBuffer digest = performReadAndVerifyRepairedInfo(readCommand, numPartitions, rowsPerPartition, true);
+ assertEquals(EMPTY_BYTE_BUFFER, digest);
+ digests.add(digest);
+
+ // add a pending repair session to table1, digest should remain the same but now we expect it to be marked inconclusive
+ UUID session1 = UUIDGen.getTimeUUID();
+ mutateRepaired(cfs, sstable1, ActiveRepairService.UNREPAIRED_SSTABLE, session1);
+ digests.add(performReadAndVerifyRepairedInfo(readCommand, numPartitions, rowsPerPartition, false));
+ assertEquals(1, digests.size());
+
+ // add a different pending session to table2, digest should remain the same and still consider it inconclusive
+ UUID session2 = UUIDGen.getTimeUUID();
+ mutateRepaired(cfs, sstable2, ActiveRepairService.UNREPAIRED_SSTABLE, session2);
+ digests.add(performReadAndVerifyRepairedInfo(readCommand, numPartitions, rowsPerPartition, false));
+ assertEquals(1, digests.size());
+
+ // mark one table repaired
+ mutateRepaired(cfs, sstable1, 111, null);
+ // this time, digest should not be empty, session2 still means that the result is inconclusive
+ digests.add(performReadAndVerifyRepairedInfo(readCommand, numPartitions, rowsPerPartition, false));
+ assertEquals(2, digests.size());
+
+ // mark the second table repaired
+ mutateRepaired(cfs, sstable2, 222, null);
+ // digest should be updated again and as there are no longer any pending sessions, it should be considered conclusive
+ digests.add(performReadAndVerifyRepairedInfo(readCommand, numPartitions, rowsPerPartition, true));
+ assertEquals(3, digests.size());
+
+ // insert a partition tombstone into the memtable, then re-check the repaired info.
+ // This is to ensure that when the optimisations which skip reading from sstables
+ // when a newer partition tombstone has already been cause the digest to be marked
+ // as inconclusive.
+ // the exception to this case is for partition range reads, where we always read
+ // and generate digests for all sstables, so we only test this path for single partition reads
+ if (readCommand.isLimitedToOnePartition())
+ {
+ new Mutation(PartitionUpdate.simpleBuilder(cfs.metadata(), ByteBufferUtil.bytes("key"))
+ .delete()
+ .build()).apply();
+ digest = performReadAndVerifyRepairedInfo(readCommand, 0, rowsPerPartition, false);
+ assertEquals(EMPTY_BYTE_BUFFER, digest);
+
+ // now flush so we have an unrepaired table with the deletion and repeat the check
+ cfs.forceBlockingFlush();
+ digest = performReadAndVerifyRepairedInfo(readCommand, 0, rowsPerPartition, false);
+ assertEquals(EMPTY_BYTE_BUFFER, digest);
+ }
+ }
+
+ private void mutateRepaired(ColumnFamilyStore cfs, SSTableReader sstable, long repairedAt, UUID pendingSession)
+ {
+ try
+ {
+ sstable.descriptor.getMetadataSerializer().mutateRepairMetadata(sstable.descriptor, repairedAt, pendingSession, false);
+ sstable.reloadSSTableMetadata();
+ }
+ catch (IOException e)
+ {
+ e.printStackTrace();
+ fail("Caught IOException when mutating sstable metadata");
+ }
+
+ if (pendingSession != null)
+ {
+ // setup a minimal repair session. This is necessary because we
+ // check for sessions which have exceeded timeout and been purged
+ Range<Token> range = new Range<>(cfs.metadata().partitioner.getMinimumToken(),
+ cfs.metadata().partitioner.getRandomToken());
+ ActiveRepairService.instance.registerParentRepairSession(pendingSession,
+ REPAIR_COORDINATOR,
+ Lists.newArrayList(cfs),
+ Sets.newHashSet(range),
+ true,
+ repairedAt,
+ true,
+ PreviewKind.NONE);
+
+ LocalSessionAccessor.prepareUnsafe(pendingSession, null, Sets.newHashSet(REPAIR_COORDINATOR));
+ }
+ }
+
+ private ByteBuffer performReadAndVerifyRepairedInfo(ReadCommand command,
+ int expectedPartitions,
+ int expectedRowsPerPartition,
+ boolean expectConclusive)
+ {
+ // perform equivalent read command multiple times and assert that
+ // the repaired data info is always consistent. Return the digest
+ // so we can verify that it changes when the repaired status of
+ // the queried tables does.
+ Set<ByteBuffer> digests = new HashSet<>();
+ for (int i = 0; i < 10; i++)
+ {
+ ReadCommand withRepairedInfo = command.copy();
+ withRepairedInfo.trackRepairedStatus();
+
+ List<FilteredPartition> partitions = Util.getAll(withRepairedInfo);
+ assertEquals(expectedPartitions, partitions.size());
+ partitions.forEach(p -> assertEquals(expectedRowsPerPartition, p.rowCount()));
+
+ ByteBuffer digest = withRepairedInfo.getRepairedDataDigest();
+ digests.add(digest);
+ assertEquals(1, digests.size());
+ assertEquals(expectConclusive, withRepairedInfo.isRepairedDataDigestConclusive());
+ }
+ return digests.iterator().next();
}
}
diff --git a/test/unit/org/apache/cassandra/db/ReadCommandVerbHandlerTest.java b/test/unit/org/apache/cassandra/db/ReadCommandVerbHandlerTest.java
new file mode 100644
index 0000000..8682273
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/ReadCommandVerbHandlerTest.java
@@ -0,0 +1,176 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db;
+
+import java.net.UnknownHostException;
+import java.util.Random;
+import java.util.UUID;
+
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.db.filter.ClusteringIndexSliceFilter;
+import org.apache.cassandra.db.filter.ColumnFilter;
+import org.apache.cassandra.db.filter.DataLimits;
+import org.apache.cassandra.db.filter.RowFilter;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.TokenMetadata;
+import org.apache.cassandra.net.Message;
+import org.apache.cassandra.net.MessageFlag;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.net.ParamType;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.FBUtilities;
+
+import static org.apache.cassandra.net.Verb.*;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class ReadCommandVerbHandlerTest
+{
+ private final static Random random = new Random();
+
+ private static ReadCommandVerbHandler handler;
+ private static TableMetadata metadata;
+ private static TableMetadata metadata_with_transient;
+ private static DecoratedKey KEY;
+
+ private static final String TEST_NAME = "read_command_vh_test_";
+ private static final String KEYSPACE = TEST_NAME + "cql_keyspace_replicated";
+ private static final String KEYSPACE_WITH_TRANSIENT = TEST_NAME + "ks_with_transient";
+ private static final String TABLE = "table1";
+
+ @BeforeClass
+ public static void init() throws Throwable
+ {
+ SchemaLoader.loadSchema();
+ SchemaLoader.schemaDefinition(TEST_NAME);
+ metadata = Schema.instance.getTableMetadata(KEYSPACE, TABLE);
+ metadata_with_transient = Schema.instance.getTableMetadata(KEYSPACE_WITH_TRANSIENT, TABLE);
+ KEY = key(metadata, 1);
+
+ TokenMetadata tmd = StorageService.instance.getTokenMetadata();
+ tmd.updateNormalToken(KEY.getToken(), InetAddressAndPort.getByName("127.0.0.2"));
+ tmd.updateNormalToken(key(metadata, 2).getToken(), InetAddressAndPort.getByName("127.0.0.3"));
+ tmd.updateNormalToken(key(metadata, 3).getToken(), FBUtilities.getBroadcastAddressAndPort());
+ }
+
+ @Before
+ public void setup()
+ {
+ MessagingService.instance().inboundSink.clear();
+ MessagingService.instance().outboundSink.clear();
+ MessagingService.instance().outboundSink.add((message, to) -> false);
+ MessagingService.instance().inboundSink.add((message) -> false);
+
+ handler = new ReadCommandVerbHandler();
+ }
+
+ @Test
+ public void setRepairedDataTrackingFlagIfHeaderPresent()
+ {
+ ReadCommand command = command(metadata);
+ assertFalse(command.isTrackingRepairedStatus());
+
+ handler.doVerb(Message.builder(READ_REQ, command)
+ .from(peer())
+ .withFlag(MessageFlag.TRACK_REPAIRED_DATA)
+ .withId(messageId())
+ .build());
+ assertTrue(command.isTrackingRepairedStatus());
+ }
+
+ @Test
+ public void dontSetRepairedDataTrackingFlagUnlessHeaderPresent()
+ {
+ ReadCommand command = command(metadata);
+ assertFalse(command.isTrackingRepairedStatus());
+ handler.doVerb(Message.builder(READ_REQ, command)
+ .from(peer())
+ .withId(messageId())
+ .withParam(ParamType.TRACE_SESSION, UUID.randomUUID())
+ .build());
+ assertFalse(command.isTrackingRepairedStatus());
+ }
+
+ @Test
+ public void dontSetRepairedDataTrackingFlagIfHeadersEmpty()
+ {
+ ReadCommand command = command(metadata);
+ assertFalse(command.isTrackingRepairedStatus());
+ handler.doVerb(Message.builder(READ_REQ, command)
+ .withId(messageId())
+ .from(peer())
+ .build());
+ assertFalse(command.isTrackingRepairedStatus());
+ }
+
+ @Test (expected = InvalidRequestException.class)
+ public void rejectsRequestWithNonMatchingTransientness()
+ {
+ ReadCommand command = command(metadata_with_transient);
+ handler.doVerb(Message.builder(READ_REQ, command)
+ .from(peer())
+ .withId(messageId())
+ .build());
+ }
+
+ private static int messageId()
+ {
+ return random.nextInt();
+ }
+
+ private static InetAddressAndPort peer()
+ {
+ try
+ {
+ return InetAddressAndPort.getByAddress(new byte[]{ 127, 0, 0, 9});
+ }
+ catch (UnknownHostException e)
+ {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private static SinglePartitionReadCommand command(TableMetadata metadata)
+ {
+ return new SinglePartitionReadCommand(false,
+ 0,
+ false,
+ metadata,
+ FBUtilities.nowInSeconds(),
+ ColumnFilter.all(metadata),
+ RowFilter.NONE,
+ DataLimits.NONE,
+ KEY,
+ new ClusteringIndexSliceFilter(Slices.ALL, false),
+ null);
+ }
+
+ private static DecoratedKey key(TableMetadata metadata, int key)
+ {
+ return metadata.partitioner.decorateKey(ByteBufferUtil.bytes(key));
+ }
+}
diff --git a/test/unit/org/apache/cassandra/db/ReadMessageTest.java b/test/unit/org/apache/cassandra/db/ReadMessageTest.java
index f76bf93..5b05253 100644
--- a/test/unit/org/apache/cassandra/db/ReadMessageTest.java
+++ b/test/unit/org/apache/cassandra/db/ReadMessageTest.java
@@ -28,8 +28,8 @@
import org.junit.Test;
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.Util;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.commitlog.CommitLogTestReplayer;
import org.apache.cassandra.db.partitions.PartitionUpdate;
@@ -59,29 +59,34 @@
{
DatabaseDescriptor.daemonInitialization();
- CFMetaData cfForReadMetadata = CFMetaData.Builder.create(KEYSPACE1, CF_FOR_READ_TEST)
- .addPartitionKey("key", BytesType.instance)
- .addClusteringColumn("col1", AsciiType.instance)
- .addClusteringColumn("col2", AsciiType.instance)
- .addRegularColumn("a", AsciiType.instance)
- .addRegularColumn("b", AsciiType.instance).build();
+ TableMetadata.Builder cfForReadMetadata =
+ TableMetadata.builder(KEYSPACE1, CF_FOR_READ_TEST)
+ .addPartitionKeyColumn("key", BytesType.instance)
+ .addClusteringColumn("col1", AsciiType.instance)
+ .addClusteringColumn("col2", AsciiType.instance)
+ .addRegularColumn("a", AsciiType.instance)
+ .addRegularColumn("b", AsciiType.instance);
- CFMetaData cfForCommitMetadata1 = CFMetaData.Builder.create(KEYSPACE1, CF_FOR_COMMIT_TEST)
- .addPartitionKey("key", BytesType.instance)
- .addClusteringColumn("name", AsciiType.instance)
- .addRegularColumn("commit1", AsciiType.instance).build();
+ TableMetadata.Builder cfForCommitMetadata1 =
+ TableMetadata.builder(KEYSPACE1, CF_FOR_COMMIT_TEST)
+ .addPartitionKeyColumn("key", BytesType.instance)
+ .addClusteringColumn("name", AsciiType.instance)
+ .addRegularColumn("commit1", AsciiType.instance);
- CFMetaData cfForCommitMetadata2 = CFMetaData.Builder.create(KEYSPACENOCOMMIT, CF_FOR_COMMIT_TEST)
- .addPartitionKey("key", BytesType.instance)
- .addClusteringColumn("name", AsciiType.instance)
- .addRegularColumn("commit2", AsciiType.instance).build();
+ TableMetadata.Builder cfForCommitMetadata2 =
+ TableMetadata.builder(KEYSPACENOCOMMIT, CF_FOR_COMMIT_TEST)
+ .addPartitionKeyColumn("key", BytesType.instance)
+ .addClusteringColumn("name", AsciiType.instance)
+ .addRegularColumn("commit2", AsciiType.instance);
SchemaLoader.prepareServer();
+
SchemaLoader.createKeyspace(KEYSPACE1,
KeyspaceParams.simple(1),
SchemaLoader.standardCFMD(KEYSPACE1, CF),
cfForReadMetadata,
cfForCommitMetadata1);
+
SchemaLoader.createKeyspace(KEYSPACENOCOMMIT,
KeyspaceParams.simpleTransient(1),
SchemaLoader.standardCFMD(KEYSPACENOCOMMIT, CF),
@@ -158,13 +163,13 @@
{
ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF);
- new RowUpdateBuilder(cfs.metadata, 0, ByteBufferUtil.bytes("key1"))
+ new RowUpdateBuilder(cfs.metadata(), 0, ByteBufferUtil.bytes("key1"))
.clustering("Column1")
.add("val", ByteBufferUtil.bytes("abcd"))
.build()
.apply();
- ColumnDefinition col = cfs.metadata.getColumnDefinition(ByteBufferUtil.bytes("val"));
+ ColumnMetadata col = cfs.metadata().getColumn(ByteBufferUtil.bytes("val"));
int found = 0;
for (FilteredPartition partition : Util.getAll(Util.cmd(cfs).build()))
{
@@ -184,20 +189,20 @@
ColumnFamilyStore cfsnocommit = Keyspace.open(KEYSPACENOCOMMIT).getColumnFamilyStore(CF_FOR_COMMIT_TEST);
- new RowUpdateBuilder(cfs.metadata, 0, ByteBufferUtil.bytes("row"))
+ new RowUpdateBuilder(cfs.metadata(), 0, ByteBufferUtil.bytes("row"))
.clustering("c")
.add("commit1", ByteBufferUtil.bytes("abcd"))
.build()
.apply();
- new RowUpdateBuilder(cfsnocommit.metadata, 0, ByteBufferUtil.bytes("row"))
+ new RowUpdateBuilder(cfsnocommit.metadata(), 0, ByteBufferUtil.bytes("row"))
.clustering("c")
.add("commit2", ByteBufferUtil.bytes("abcd"))
.build()
.apply();
- Checker checker = new Checker(cfs.metadata.getColumnDefinition(ByteBufferUtil.bytes("commit1")),
- cfsnocommit.metadata.getColumnDefinition(ByteBufferUtil.bytes("commit2")));
+ Checker checker = new Checker(cfs.metadata().getColumn(ByteBufferUtil.bytes("commit1")),
+ cfsnocommit.metadata().getColumn(ByteBufferUtil.bytes("commit2")));
CommitLogTestReplayer replayer = new CommitLogTestReplayer(checker);
replayer.examineCommitLog();
@@ -208,13 +213,13 @@
static class Checker implements Predicate<Mutation>
{
- private final ColumnDefinition withCommit;
- private final ColumnDefinition withoutCommit;
+ private final ColumnMetadata withCommit;
+ private final ColumnMetadata withoutCommit;
boolean commitLogMessageFound = false;
boolean noCommitLogMessageFound = false;
- public Checker(ColumnDefinition withCommit, ColumnDefinition withoutCommit)
+ public Checker(ColumnMetadata withCommit, ColumnMetadata withoutCommit)
{
this.withCommit = withCommit;
this.withoutCommit = withoutCommit;
diff --git a/test/unit/org/apache/cassandra/db/ReadResponseTest.java b/test/unit/org/apache/cassandra/db/ReadResponseTest.java
index 52ab8bb..6e1a804 100644
--- a/test/unit/org/apache/cassandra/db/ReadResponseTest.java
+++ b/test/unit/org/apache/cassandra/db/ReadResponseTest.java
@@ -15,85 +15,247 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.cassandra.db;
-import java.util.*;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Random;
-import org.junit.After;
import org.junit.Before;
import org.junit.Test;
-import org.apache.cassandra.Util;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.cql3.CQLTester;
-import org.apache.cassandra.db.rows.Rows;
-import org.apache.cassandra.db.rows.UnfilteredRowIterators;
-import org.apache.cassandra.db.partitions.ImmutableBTreePartition;
+import org.apache.cassandra.db.filter.ColumnFilter;
+import org.apache.cassandra.db.filter.DataLimits;
+import org.apache.cassandra.db.filter.RowFilter;
+import org.apache.cassandra.db.marshal.Int32Type;
import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
-import org.apache.cassandra.db.marshal.AsciiType;
-import org.apache.cassandra.dht.ByteOrderedPartitioner;
-import org.apache.cassandra.dht.IPartitioner;
+import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.io.util.DataInputBuffer;
+import org.apache.cassandra.io.util.DataOutputBuffer;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.FBUtilities;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
-public class ReadResponseTest extends CQLTester
+public class ReadResponseTest
{
- private IPartitioner partitionerToRestore;
+
+ private final Random random = new Random();
+ private TableMetadata metadata;
@Before
- public void setupPartitioner()
+ public void setup()
{
- // Using an ordered partitioner to be able to predict keys order in the following tests.
- partitionerToRestore = DatabaseDescriptor.setPartitionerUnsafe(ByteOrderedPartitioner.instance);
- }
-
- @After
- public void resetPartitioner()
- {
- DatabaseDescriptor.setPartitionerUnsafe(partitionerToRestore);
+ metadata = TableMetadata.builder("ks", "t1")
+ .addPartitionKeyColumn("p", Int32Type.instance)
+ .addRegularColumn("v", Int32Type.instance)
+ .partitioner(Murmur3Partitioner.instance)
+ .build();
}
@Test
- public void testLegacyResponseSkipWrongBounds()
+ public void fromCommandWithConclusiveRepairedDigest()
{
- createTable("CREATE TABLE %s (k text PRIMARY KEY)");
-
- ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
-
- // Test that if a legacy response contains keys at the boundary of the requested key range that shouldn't be present, those
- // are properly skipped. See CASSANDRA-9857 for context.
-
- List<ImmutableBTreePartition> responses = Arrays.asList(makePartition(cfs.metadata, "k1"),
- makePartition(cfs.metadata, "k2"),
- makePartition(cfs.metadata, "k3"));
- ReadResponse.LegacyRemoteDataResponse response = new ReadResponse.LegacyRemoteDataResponse(responses);
-
- assertPartitions(response.makeIterator(Util.cmd(cfs).fromKeyExcl("k1").toKeyExcl("k3").build()), "k2");
- assertPartitions(response.makeIterator(Util.cmd(cfs).fromKeyExcl("k0").toKeyExcl("k3").build()), "k1", "k2");
- assertPartitions(response.makeIterator(Util.cmd(cfs).fromKeyExcl("k1").toKeyExcl("k4").build()), "k2", "k3");
-
- assertPartitions(response.makeIterator(Util.cmd(cfs).fromKeyIncl("k1").toKeyExcl("k3").build()), "k1", "k2");
- assertPartitions(response.makeIterator(Util.cmd(cfs).fromKeyIncl("k1").toKeyExcl("k4").build()), "k1", "k2", "k3");
+ ByteBuffer digest = digest();
+ ReadCommand command = command(key(), metadata, digest, true);
+ ReadResponse response = command.createResponse(EmptyIterators.unfilteredPartition(metadata));
+ assertTrue(response.isRepairedDigestConclusive());
+ assertEquals(digest, response.repairedDataDigest());
+ verifySerDe(response);
}
- private void assertPartitions(UnfilteredPartitionIterator actual, String... expectedKeys)
+ @Test
+ public void fromCommandWithInconclusiveRepairedDigest()
{
- int i = 0;
- while (i < expectedKeys.length && actual.hasNext())
+ ByteBuffer digest = digest();
+ ReadCommand command = command(key(), metadata, digest, false);
+ ReadResponse response = command.createResponse(EmptyIterators.unfilteredPartition(metadata));
+ assertFalse(response.isRepairedDigestConclusive());
+ assertEquals(digest, response.repairedDataDigest());
+ verifySerDe(response);
+ }
+
+ @Test
+ public void fromCommandWithConclusiveEmptyRepairedDigest()
+ {
+ ReadCommand command = command(key(), metadata, ByteBufferUtil.EMPTY_BYTE_BUFFER, true);
+ ReadResponse response = command.createResponse(EmptyIterators.unfilteredPartition(metadata));
+ assertTrue(response.isRepairedDigestConclusive());
+ assertEquals(ByteBufferUtil.EMPTY_BYTE_BUFFER, response.repairedDataDigest());
+ verifySerDe(response);
+ }
+
+ @Test
+ public void fromCommandWithInconclusiveEmptyRepairedDigest()
+ {
+ ReadCommand command = command(key(), metadata, ByteBufferUtil.EMPTY_BYTE_BUFFER, false);
+ ReadResponse response = command.createResponse(EmptyIterators.unfilteredPartition(metadata));
+ assertFalse(response.isRepairedDigestConclusive());
+ assertEquals(ByteBufferUtil.EMPTY_BYTE_BUFFER, response.repairedDataDigest());
+ verifySerDe(response);
+ }
+
+ /*
+ * Digest responses should never include repaired data tracking as we only request
+ * it in read repair or for range queries
+ */
+ @Test (expected = UnsupportedOperationException.class)
+ public void digestResponseErrorsIfRepairedDataDigestRequested()
+ {
+ ReadCommand command = digestCommand(key(), metadata);
+ ReadResponse response = command.createResponse(EmptyIterators.unfilteredPartition(metadata));
+ assertTrue(response.isDigestResponse());
+ assertFalse(response.mayIncludeRepairedDigest());
+ response.repairedDataDigest();
+ }
+
+ @Test (expected = UnsupportedOperationException.class)
+ public void digestResponseErrorsIfIsConclusiveRequested()
+ {
+ ReadCommand command = digestCommand(key(), metadata);
+ ReadResponse response = command.createResponse(EmptyIterators.unfilteredPartition(metadata));
+ assertTrue(response.isDigestResponse());
+ assertFalse(response.mayIncludeRepairedDigest());
+ response.isRepairedDigestConclusive();
+ }
+
+ @Test (expected = UnsupportedOperationException.class)
+ public void digestResponseErrorsIfIteratorRequested()
+ {
+ ReadCommand command = digestCommand(key(), metadata);
+ ReadResponse response = command.createResponse(EmptyIterators.unfilteredPartition(metadata));
+ assertTrue(response.isDigestResponse());
+ assertFalse(response.mayIncludeRepairedDigest());
+ response.makeIterator(command);
+ }
+
+ @Test
+ public void makeDigestDoesntConsiderRepairedDataInfo()
+ {
+ // It shouldn't be possible to get false positive DigestMismatchExceptions based
+ // on differing repaired data tracking info because it isn't requested on initial
+ // requests, only following a digest mismatch. Having a test doesn't hurt though
+ int key = key();
+ ByteBuffer digest1 = digest();
+ ReadCommand command1 = command(key, metadata, digest1, true);
+ ReadResponse response1 = command1.createResponse(EmptyIterators.unfilteredPartition(metadata));
+
+ ByteBuffer digest2 = digest();
+ ReadCommand command2 = command(key, metadata, digest2, false);
+ ReadResponse response2 = command1.createResponse(EmptyIterators.unfilteredPartition(metadata));
+
+ assertEquals(response1.digest(command1), response2.digest(command2));
+ }
+
+ private void verifySerDe(ReadResponse response) {
+ // check that roundtripping through ReadResponse.serializer behaves as expected.
+ // ReadResponses from pre-4.0 nodes will never contain repaired data digest
+ // or pending session info, but we run all messages through both pre/post 4.0
+ // serde to check that the defaults are correctly applied
+ roundTripSerialization(response, MessagingService.current_version);
+ roundTripSerialization(response, MessagingService.VERSION_30);
+
+ }
+
+ private void roundTripSerialization(ReadResponse response, int version)
+ {
+ try
{
- String actualKey = AsciiType.instance.getString(actual.next().partitionKey().getKey());
- assertEquals(expectedKeys[i++], actualKey);
+ DataOutputBuffer out = new DataOutputBuffer();
+ ReadResponse.serializer.serialize(response, out, version);
+
+ DataInputBuffer in = new DataInputBuffer(out.buffer(), false);
+ ReadResponse deser = ReadResponse.serializer.deserialize(in, version);
+ if (version < MessagingService.VERSION_40)
+ {
+ assertFalse(deser.mayIncludeRepairedDigest());
+ // even though that means they should never be used, verify that the default values are present
+ assertEquals(ByteBufferUtil.EMPTY_BYTE_BUFFER, deser.repairedDataDigest());
+ assertTrue(deser.isRepairedDigestConclusive());
+ }
+ else
+ {
+ assertTrue(deser.mayIncludeRepairedDigest());
+ assertEquals(response.repairedDataDigest(), deser.repairedDataDigest());
+ assertEquals(response.isRepairedDigestConclusive(), deser.isRepairedDigestConclusive());
+ }
+ }
+ catch (IOException e)
+ {
+ fail("Caught unexpected IOException during SerDe: " + e.getMessage());
+ }
+ }
+
+
+ private int key()
+ {
+ return random.nextInt();
+ }
+
+ private ByteBuffer digest()
+ {
+ byte[] bytes = new byte[4];
+ random.nextBytes(bytes);
+ return ByteBuffer.wrap(bytes);
+ }
+
+ private ReadCommand digestCommand(int key, TableMetadata metadata)
+ {
+ return new StubReadCommand(key, metadata, true, ByteBufferUtil.EMPTY_BYTE_BUFFER, true);
+ }
+
+ private ReadCommand command(int key, TableMetadata metadata, ByteBuffer repairedDigest, boolean conclusive)
+ {
+ return new StubReadCommand(key, metadata, false, repairedDigest, conclusive);
+ }
+
+ private static class StubReadCommand extends SinglePartitionReadCommand
+ {
+
+ private final ByteBuffer repairedDigest;
+ private final boolean conclusive;
+
+ StubReadCommand(int key, TableMetadata metadata,
+ boolean isDigest,
+ final ByteBuffer repairedDigest,
+ final boolean conclusive)
+ {
+ super(isDigest,
+ 0,
+ false,
+ metadata,
+ FBUtilities.nowInSeconds(),
+ ColumnFilter.all(metadata),
+ RowFilter.NONE,
+ DataLimits.NONE,
+ metadata.partitioner.decorateKey(ByteBufferUtil.bytes(key)),
+ null,
+ null);
+ this.repairedDigest = repairedDigest;
+ this.conclusive = conclusive;
}
- if (i < expectedKeys.length)
- throw new AssertionError("Got less results than expected: " + expectedKeys[i] + " is not in the result");
- if (actual.hasNext())
- throw new AssertionError("Got more results than expected: first unexpected key is " + AsciiType.instance.getString(actual.next().partitionKey().getKey()));
- }
+ @Override
+ public ByteBuffer getRepairedDataDigest()
+ {
+ return repairedDigest;
+ }
- private static ImmutableBTreePartition makePartition(CFMetaData metadata, String key)
- {
- return ImmutableBTreePartition.create(UnfilteredRowIterators.noRowsIterator(metadata, Util.dk(key), Rows.EMPTY_STATIC_ROW, new DeletionTime(0, 0), false));
+ @Override
+ public boolean isRepairedDataDigestConclusive()
+ {
+ return conclusive;
+ }
+
+ public UnfilteredPartitionIterator executeLocally(ReadExecutionController controller)
+ {
+ return EmptyIterators.unfilteredPartition(this.metadata());
+ }
}
}
diff --git a/test/unit/org/apache/cassandra/db/RecoveryManagerFlushedTest.java b/test/unit/org/apache/cassandra/db/RecoveryManagerFlushedTest.java
index 0b20343..fc34942 100644
--- a/test/unit/org/apache/cassandra/db/RecoveryManagerFlushedTest.java
+++ b/test/unit/org/apache/cassandra/db/RecoveryManagerFlushedTest.java
@@ -36,7 +36,8 @@
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.config.ParameterizedClass;
-import org.apache.cassandra.config.SchemaConstants;
+import org.apache.cassandra.io.compress.ZstdCompressor;
+import org.apache.cassandra.schema.SchemaConstants;
import org.apache.cassandra.db.compaction.CompactionManager;
import org.apache.cassandra.db.commitlog.CommitLog;
import org.apache.cassandra.exceptions.ConfigurationException;
@@ -71,7 +72,8 @@
{null, EncryptionContextGenerator.createContext(true)}, // Encryption
{new ParameterizedClass(LZ4Compressor.class.getName(), Collections.emptyMap()), EncryptionContextGenerator.createDisabledContext()},
{new ParameterizedClass(SnappyCompressor.class.getName(), Collections.emptyMap()), EncryptionContextGenerator.createDisabledContext()},
- {new ParameterizedClass(DeflateCompressor.class.getName(), Collections.emptyMap()), EncryptionContextGenerator.createDisabledContext()}});
+ {new ParameterizedClass(DeflateCompressor.class.getName(), Collections.emptyMap()), EncryptionContextGenerator.createDisabledContext()},
+ {new ParameterizedClass(ZstdCompressor.class.getName(), Collections.emptyMap()), EncryptionContextGenerator.createDisabledContext()}});
}
@Before
@@ -126,7 +128,7 @@
{
Keyspace keyspace = Keyspace.open(KEYSPACE1);
ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfname);
- new RowUpdateBuilder(cfs.metadata, 0, key)
+ new RowUpdateBuilder(cfs.metadata(), 0, key)
.clustering("c")
.add("val", "val1")
.build()
diff --git a/test/unit/org/apache/cassandra/db/RecoveryManagerMissingHeaderTest.java b/test/unit/org/apache/cassandra/db/RecoveryManagerMissingHeaderTest.java
index 8897700..4044fff 100644
--- a/test/unit/org/apache/cassandra/db/RecoveryManagerMissingHeaderTest.java
+++ b/test/unit/org/apache/cassandra/db/RecoveryManagerMissingHeaderTest.java
@@ -42,6 +42,7 @@
import org.apache.cassandra.io.compress.DeflateCompressor;
import org.apache.cassandra.io.compress.LZ4Compressor;
import org.apache.cassandra.io.compress.SnappyCompressor;
+import org.apache.cassandra.io.compress.ZstdCompressor;
import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.schema.KeyspaceParams;
import org.apache.cassandra.security.EncryptionContext;
@@ -70,7 +71,8 @@
{null, EncryptionContextGenerator.createContext(true)}, // Encryption
{new ParameterizedClass(LZ4Compressor.class.getName(), Collections.emptyMap()), EncryptionContextGenerator.createDisabledContext()},
{new ParameterizedClass(SnappyCompressor.class.getName(), Collections.emptyMap()), EncryptionContextGenerator.createDisabledContext()},
- {new ParameterizedClass(DeflateCompressor.class.getName(), Collections.emptyMap()), EncryptionContextGenerator.createDisabledContext()}});
+ {new ParameterizedClass(DeflateCompressor.class.getName(), Collections.emptyMap()), EncryptionContextGenerator.createDisabledContext()},
+ {new ParameterizedClass(ZstdCompressor.class.getName(), Collections.emptyMap()), EncryptionContextGenerator.createDisabledContext()}});
}
@Before
@@ -98,11 +100,11 @@
Keyspace keyspace2 = Keyspace.open(KEYSPACE2);
DecoratedKey dk = Util.dk("keymulti");
- UnfilteredRowIterator upd1 = Util.apply(new RowUpdateBuilder(keyspace1.getColumnFamilyStore(CF_STANDARD1).metadata, 1L, 0, "keymulti")
+ UnfilteredRowIterator upd1 = Util.apply(new RowUpdateBuilder(keyspace1.getColumnFamilyStore(CF_STANDARD1).metadata(), 1L, 0, "keymulti")
.clustering("col1").add("val", "1")
.build());
- UnfilteredRowIterator upd2 = Util.apply(new RowUpdateBuilder(keyspace2.getColumnFamilyStore(CF_STANDARD3).metadata, 1L, 0, "keymulti")
+ UnfilteredRowIterator upd2 = Util.apply(new RowUpdateBuilder(keyspace2.getColumnFamilyStore(CF_STANDARD3).metadata(), 1L, 0, "keymulti")
.clustering("col1").add("val", "1")
.build());
diff --git a/test/unit/org/apache/cassandra/db/RecoveryManagerTest.java b/test/unit/org/apache/cassandra/db/RecoveryManagerTest.java
index cc9c667..527c6a7 100644
--- a/test/unit/org/apache/cassandra/db/RecoveryManagerTest.java
+++ b/test/unit/org/apache/cassandra/db/RecoveryManagerTest.java
@@ -30,21 +30,6 @@
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicReference;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.cassandra.Util;
-import org.apache.cassandra.concurrent.NamedThreadFactory;
-import org.apache.cassandra.config.ColumnDefinition;
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.config.ParameterizedClass;
-import org.apache.cassandra.db.rows.*;
-import org.apache.cassandra.db.context.CounterContext;
-import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.io.compress.DeflateCompressor;
-import org.apache.cassandra.io.compress.LZ4Compressor;
-import org.apache.cassandra.io.compress.SnappyCompressor;
-
import org.junit.Assert;
import org.junit.Before;
import org.junit.BeforeClass;
@@ -53,16 +38,31 @@
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameters;
-import static org.junit.Assert.assertEquals;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.Util;
+import org.apache.cassandra.concurrent.NamedThreadFactory;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.config.ParameterizedClass;
import org.apache.cassandra.db.commitlog.CommitLog;
import org.apache.cassandra.db.commitlog.CommitLogArchiver;
+import org.apache.cassandra.db.commitlog.CommitLogReplayer;
+import org.apache.cassandra.db.context.CounterContext;
+import org.apache.cassandra.db.rows.*;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.io.compress.DeflateCompressor;
+import org.apache.cassandra.io.compress.LZ4Compressor;
+import org.apache.cassandra.io.compress.SnappyCompressor;
+import org.apache.cassandra.io.compress.ZstdCompressor;
+import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.schema.KeyspaceParams;
import org.apache.cassandra.security.EncryptionContext;
import org.apache.cassandra.security.EncryptionContextGenerator;
import org.apache.cassandra.utils.ByteBufferUtil;
-import org.apache.cassandra.db.commitlog.CommitLogReplayer;
+
+import static org.junit.Assert.assertEquals;
@RunWith(Parameterized.class)
public class RecoveryManagerTest
@@ -91,7 +91,8 @@
{null, EncryptionContextGenerator.createContext(true)}, // Encryption
{new ParameterizedClass(LZ4Compressor.class.getName(), Collections.emptyMap()), EncryptionContextGenerator.createDisabledContext()},
{new ParameterizedClass(SnappyCompressor.class.getName(), Collections.emptyMap()), EncryptionContextGenerator.createDisabledContext()},
- {new ParameterizedClass(DeflateCompressor.class.getName(), Collections.emptyMap()), EncryptionContextGenerator.createDisabledContext()}});
+ {new ParameterizedClass(DeflateCompressor.class.getName(), Collections.emptyMap()), EncryptionContextGenerator.createDisabledContext()},
+ {new ParameterizedClass(ZstdCompressor.class.getName(), Collections.emptyMap()), EncryptionContextGenerator.createDisabledContext()}});
}
@Before
@@ -143,11 +144,11 @@
Keyspace keyspace1 = Keyspace.open(KEYSPACE1);
Keyspace keyspace2 = Keyspace.open(KEYSPACE2);
- UnfilteredRowIterator upd1 = Util.apply(new RowUpdateBuilder(keyspace1.getColumnFamilyStore(CF_STANDARD1).metadata, 1L, 0, "keymulti")
+ UnfilteredRowIterator upd1 = Util.apply(new RowUpdateBuilder(keyspace1.getColumnFamilyStore(CF_STANDARD1).metadata(), 1L, 0, "keymulti")
.clustering("col1").add("val", "1")
.build());
- UnfilteredRowIterator upd2 = Util.apply(new RowUpdateBuilder(keyspace2.getColumnFamilyStore(CF_STANDARD3).metadata, 1L, 0, "keymulti")
+ UnfilteredRowIterator upd2 = Util.apply(new RowUpdateBuilder(keyspace2.getColumnFamilyStore(CF_STANDARD3).metadata(), 1L, 0, "keymulti")
.clustering("col2").add("val", "1")
.build());
@@ -206,11 +207,11 @@
Keyspace keyspace1 = Keyspace.open(KEYSPACE1);
Keyspace keyspace2 = Keyspace.open(KEYSPACE2);
- UnfilteredRowIterator upd1 = Util.apply(new RowUpdateBuilder(keyspace1.getColumnFamilyStore(CF_STANDARD1).metadata, 1L, 0, "keymulti")
+ UnfilteredRowIterator upd1 = Util.apply(new RowUpdateBuilder(keyspace1.getColumnFamilyStore(CF_STANDARD1).metadata(), 1L, 0, "keymulti")
.clustering("col1").add("val", "1")
.build());
- UnfilteredRowIterator upd2 = Util.apply(new RowUpdateBuilder(keyspace2.getColumnFamilyStore(CF_STANDARD3).metadata, 1L, 0, "keymulti")
+ UnfilteredRowIterator upd2 = Util.apply(new RowUpdateBuilder(keyspace2.getColumnFamilyStore(CF_STANDARD3).metadata(), 1L, 0, "keymulti")
.clustering("col2").add("val", "1")
.build());
@@ -233,7 +234,7 @@
for (int i = 0; i < 10; ++i)
{
- new CounterMutation(new RowUpdateBuilder(cfs.metadata, 1L, 0, "key")
+ new CounterMutation(new RowUpdateBuilder(cfs.metadata(), 1L, 0, "key")
.clustering("cc").add("val", CounterContext.instance().createLocal(1L))
.build(), ConsistencyLevel.ALL).apply();
}
@@ -242,7 +243,7 @@
int replayed = CommitLog.instance.resetUnsafe(false);
- ColumnDefinition counterCol = cfs.metadata.getColumnDefinition(ByteBufferUtil.bytes("val"));
+ ColumnMetadata counterCol = cfs.metadata().getColumn(ByteBufferUtil.bytes("val"));
Row row = Util.getOnlyRow(Util.cmd(cfs).includeRow("cc").columns("val").build());
assertEquals(10L, CounterContext.instance().total(row.getCell(counterCol).value()));
}
@@ -259,7 +260,7 @@
for (int i = 0; i < 10; ++i)
{
long ts = TimeUnit.MILLISECONDS.toMicros(timeMS + (i * 1000));
- new RowUpdateBuilder(cfs.metadata, ts, "name-" + i)
+ new RowUpdateBuilder(cfs.metadata(), ts, "name-" + i)
.clustering("cc")
.add("val", Integer.toString(i))
.build()
@@ -288,7 +289,7 @@
for (int i = 0; i < 10; ++i)
{
long ts = TimeUnit.MILLISECONDS.toMicros(timeMS + (i * 1000));
- new RowUpdateBuilder(cfs.metadata, ts, "name-" + i)
+ new RowUpdateBuilder(cfs.metadata(), ts, "name-" + i)
.add("val", Integer.toString(i))
.build()
.apply();
@@ -322,7 +323,7 @@
else
ts = TimeUnit.MILLISECONDS.toMicros(timeMS + (i * 1000));
- new RowUpdateBuilder(cfs.metadata, ts, "name-" + i)
+ new RowUpdateBuilder(cfs.metadata(), ts, "name-" + i)
.clustering("cc")
.add("val", Integer.toString(i))
.build()
diff --git a/test/unit/org/apache/cassandra/db/RecoveryManagerTruncateTest.java b/test/unit/org/apache/cassandra/db/RecoveryManagerTruncateTest.java
index 738888f..a51cd21 100644
--- a/test/unit/org/apache/cassandra/db/RecoveryManagerTruncateTest.java
+++ b/test/unit/org/apache/cassandra/db/RecoveryManagerTruncateTest.java
@@ -32,6 +32,7 @@
import org.apache.cassandra.io.compress.DeflateCompressor;
import org.apache.cassandra.io.compress.LZ4Compressor;
import org.apache.cassandra.io.compress.SnappyCompressor;
+import org.apache.cassandra.io.compress.ZstdCompressor;
import org.apache.cassandra.schema.KeyspaceParams;
import org.apache.cassandra.security.EncryptionContext;
import org.apache.cassandra.security.EncryptionContextGenerator;
@@ -68,7 +69,8 @@
{null, EncryptionContextGenerator.createContext(true)}, // Encryption
{new ParameterizedClass(LZ4Compressor.class.getName(), Collections.emptyMap()), EncryptionContextGenerator.createDisabledContext()},
{new ParameterizedClass(SnappyCompressor.class.getName(), Collections.emptyMap()), EncryptionContextGenerator.createDisabledContext()},
- {new ParameterizedClass(DeflateCompressor.class.getName(), Collections.emptyMap()), EncryptionContextGenerator.createDisabledContext()}});
+ {new ParameterizedClass(DeflateCompressor.class.getName(), Collections.emptyMap()), EncryptionContextGenerator.createDisabledContext()},
+ {new ParameterizedClass(ZstdCompressor.class.getName(), Collections.emptyMap()), EncryptionContextGenerator.createDisabledContext()}});
}
@Before
@@ -93,7 +95,7 @@
ColumnFamilyStore cfs = keyspace.getColumnFamilyStore("Standard1");
// add a single cell
- new RowUpdateBuilder(cfs.metadata, 0, "key1")
+ new RowUpdateBuilder(cfs.metadata(), 0, "key1")
.clustering("cc")
.add("val", "val1")
.build()
diff --git a/test/unit/org/apache/cassandra/db/RepairedDataInfoTest.java b/test/unit/org/apache/cassandra/db/RepairedDataInfoTest.java
new file mode 100644
index 0000000..c43be98
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/RepairedDataInfoTest.java
@@ -0,0 +1,344 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db;
+
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.stream.IntStream;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.commitlog.CommitLog;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.schema.MockSchema;
+import org.apache.cassandra.db.filter.DataLimits;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.db.partitions.AbstractUnfilteredPartitionIterator;
+import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
+import org.apache.cassandra.db.rows.AbstractUnfilteredRowIterator;
+import org.apache.cassandra.db.rows.BTreeRow;
+import org.apache.cassandra.db.rows.BufferCell;
+import org.apache.cassandra.db.rows.Cell;
+import org.apache.cassandra.db.rows.EncodingStats;
+import org.apache.cassandra.db.rows.RangeTombstoneBoundMarker;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.db.rows.Rows;
+import org.apache.cassandra.db.rows.Unfiltered;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.utils.FBUtilities;
+
+import static org.apache.cassandra.Util.clustering;
+import static org.apache.cassandra.Util.dk;
+import static org.apache.cassandra.utils.ByteBufferUtil.*;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+
+public class RepairedDataInfoTest
+{
+ private static ColumnFamilyStore cfs;
+ private static TableMetadata metadata;
+ private static ColumnMetadata valueMetadata;
+ private static ColumnMetadata staticMetadata;
+
+ private final int nowInSec = FBUtilities.nowInSeconds();
+
+ @BeforeClass
+ public static void setUp()
+ {
+ DatabaseDescriptor.daemonInitialization();
+ CommitLog.instance.start();
+ MockSchema.cleanup();
+ String ks = "repaired_data_info_test";
+ cfs = MockSchema.newCFS(ks, metadata -> metadata.addStaticColumn("s", UTF8Type.instance));
+ metadata = cfs.metadata();
+ valueMetadata = metadata.regularColumns().getSimple(0);
+ staticMetadata = metadata.staticColumns().getSimple(0);
+ }
+
+ @Test
+ public void withTrackingAppliesRepairedDataCounter()
+ {
+ DataLimits.Counter counter = DataLimits.cqlLimits(15).newCounter(nowInSec, false, false, false).onlyCount();
+ RepairedDataInfo info = new RepairedDataInfo(counter);
+ info.prepare(cfs, nowInSec, Integer.MAX_VALUE);
+ UnfilteredRowIterator[] partitions = new UnfilteredRowIterator[3];
+ for (int i=0; i<3; i++)
+ partitions[i] = partition(bytes(i), rows(0, 5, nowInSec));
+
+ UnfilteredPartitionIterator iter = partitions(partitions);
+ iter = info.withRepairedDataInfo(iter);
+ consume(iter);
+
+ assertEquals(15, counter.counted());
+ assertEquals(5, counter.countedInCurrentPartition());
+ }
+
+ @Test
+ public void digestOfSinglePartitionWithSingleRowAndEmptyStaticRow()
+ {
+ Digest manualDigest = Digest.forRepairedDataTracking();
+ Row[] rows = rows(0, 1, nowInSec);
+ UnfilteredRowIterator partition = partition(bytes(0), rows);
+ addToDigest(manualDigest,
+ partition.partitionKey().getKey(),
+ partition.partitionLevelDeletion(),
+ Rows.EMPTY_STATIC_ROW,
+ rows);
+ byte[] fromRepairedInfo = consume(partition);
+ assertArrayEquals(manualDigest.digest(), fromRepairedInfo);
+ }
+
+ @Test
+ public void digestOfSinglePartitionWithMultipleRowsAndEmptyStaticRow()
+ {
+ Digest manualDigest = Digest.forRepairedDataTracking();
+ Row[] rows = rows(0, 5, nowInSec);
+ UnfilteredRowIterator partition = partition(bytes(0), rows);
+ addToDigest(manualDigest,
+ partition.partitionKey().getKey(),
+ partition.partitionLevelDeletion(),
+ Rows.EMPTY_STATIC_ROW,
+ rows);
+ byte[] fromRepairedInfo = consume(partition);
+ assertArrayEquals(manualDigest.digest(), fromRepairedInfo);
+ }
+
+ @Test
+ public void digestOfSinglePartitionWithMultipleRowsAndTombstones()
+ {
+ Digest manualDigest = Digest.forRepairedDataTracking();
+ Unfiltered[] unfiltereds = new Unfiltered[]
+ {
+ open(0), close(0),
+ row(1, 1, nowInSec),
+ open(2), close(4),
+ row(5, 7, nowInSec)
+ };
+ UnfilteredRowIterator partition = partition(bytes(0), unfiltereds);
+ addToDigest(manualDigest,
+ partition.partitionKey().getKey(),
+ partition.partitionLevelDeletion(),
+ Rows.EMPTY_STATIC_ROW,
+ unfiltereds);
+ byte[] fromRepairedInfo = consume(partition);
+ assertArrayEquals(manualDigest.digest(), fromRepairedInfo);
+ }
+
+ @Test
+ public void digestOfMultiplePartitionsWithMultipleRowsAndNonEmptyStaticRows()
+ {
+ Digest manualDigest = Digest.forRepairedDataTracking();
+ Row staticRow = staticRow(nowInSec);
+ Row[] rows = rows(0, 5, nowInSec);
+ UnfilteredRowIterator[] partitionsArray = new UnfilteredRowIterator[5];
+ for (int i=0; i<5; i++)
+ {
+ UnfilteredRowIterator partition = partitionWithStaticRow(bytes(i), staticRow, rows);
+ partitionsArray[i] = partition;
+ addToDigest(manualDigest,
+ partition.partitionKey().getKey(),
+ partition.partitionLevelDeletion(),
+ staticRow,
+ rows);
+ }
+
+ UnfilteredPartitionIterator partitions = partitions(partitionsArray);
+ byte[] fromRepairedInfo = consume(partitions);
+ assertArrayEquals(manualDigest.digest(), fromRepairedInfo);
+ }
+
+ @Test
+ public void digestOfFullyPurgedPartition()
+ {
+ int deletionTime = nowInSec - cfs.metadata().params.gcGraceSeconds - 1;
+ DeletionTime deletion = new DeletionTime(((long)deletionTime * 1000), deletionTime);
+ Row staticRow = staticRow(nowInSec, deletion);
+ Row row = row(1, nowInSec, deletion);
+ UnfilteredRowIterator partition = partitionWithStaticRow(bytes(0), staticRow, row);
+
+ // The partition is fully purged, so nothing should be added to the digest
+ byte[] fromRepairedInfo = consume(partition);
+ assertEquals(0, fromRepairedInfo.length);
+ }
+
+ @Test
+ public void digestOfEmptyPartition()
+ {
+ // Static row is read greedily during transformation and if the underlying
+ // SSTableIterator doesn't contain the partition, an empty but non-null
+ // static row is read and digested.
+ UnfilteredRowIterator partition = partition(bytes(0));
+ // The partition is completely empty, so nothing should be added to the digest
+ byte[] fromRepairedInfo = consume(partition);
+ assertEquals(0, fromRepairedInfo.length);
+ }
+
+ private RepairedDataInfo info()
+ {
+ return new RepairedDataInfo(DataLimits.NONE.newCounter(nowInSec, false, false, false));
+ }
+
+ private Digest addToDigest(Digest aggregate,
+ ByteBuffer partitionKey,
+ DeletionTime deletion,
+ Row staticRow,
+ Unfiltered...unfiltereds)
+ {
+ Digest perPartitionDigest = Digest.forRepairedDataTracking();
+ if (staticRow != null && !staticRow.isEmpty())
+ staticRow.digest(perPartitionDigest);
+ perPartitionDigest.update(partitionKey);
+ deletion.digest(perPartitionDigest);
+ for (Unfiltered unfiltered : unfiltereds)
+ unfiltered.digest(perPartitionDigest);
+ byte[] rowDigestBytes = perPartitionDigest.digest();
+ aggregate.update(rowDigestBytes, 0, rowDigestBytes.length);
+ return aggregate;
+ }
+
+ private byte[] consume(UnfilteredPartitionIterator partitions)
+ {
+ RepairedDataInfo info = info();
+ info.prepare(cfs, nowInSec, Integer.MAX_VALUE);
+ partitions.forEachRemaining(partition ->
+ {
+ try (UnfilteredRowIterator iter = info.withRepairedDataInfo(partition))
+ {
+ iter.forEachRemaining(u -> {});
+ }
+ });
+ return getArray(info.getDigest());
+ }
+
+ private byte[] consume(UnfilteredRowIterator partition)
+ {
+ RepairedDataInfo info = info();
+ info.prepare(cfs, nowInSec, Integer.MAX_VALUE);
+ try (UnfilteredRowIterator iter = info.withRepairedDataInfo(partition))
+ {
+ iter.forEachRemaining(u -> {});
+ }
+ return getArray(info.getDigest());
+ }
+
+ public static Cell cell(ColumnMetadata def, Object value)
+ {
+ ByteBuffer bb = value instanceof ByteBuffer ? (ByteBuffer)value : ((AbstractType)def.type).decompose(value);
+ return new BufferCell(def, 1L, BufferCell.NO_TTL, BufferCell.NO_DELETION_TIME, bb, null);
+ }
+
+ private Row staticRow(int nowInSec)
+ {
+ Row.Builder builder = BTreeRow.unsortedBuilder();
+ builder.newRow(Clustering.STATIC_CLUSTERING);
+ builder.addCell(cell(staticMetadata, "static value"));
+ return builder.build();
+ }
+
+ private Row staticRow(int nowInSec, DeletionTime deletion)
+ {
+ Row.Builder builder = BTreeRow.unsortedBuilder();
+ builder.newRow(Clustering.STATIC_CLUSTERING);
+ builder.addRowDeletion(new Row.Deletion(deletion, false));
+ return builder.build();
+ }
+
+ private Row row(int clustering, int value, int nowInSec)
+ {
+ Row.Builder builder = BTreeRow.unsortedBuilder();
+ builder.newRow(clustering(metadata.comparator, Integer.toString(clustering)));
+ builder.addCell(cell(valueMetadata, Integer.toString(value)));
+ return builder.build();
+ }
+
+ private Row row(int clustering, int nowInSec, DeletionTime deletion)
+ {
+ Row.Builder builder = BTreeRow.unsortedBuilder();
+ builder.newRow(clustering(metadata.comparator, Integer.toString(clustering)));
+ builder.addRowDeletion(new Row.Deletion(deletion, false));
+ return builder.build();
+ }
+
+ private Row[] rows(int clusteringStart, int clusteringEnd, int nowInSec)
+ {
+ return IntStream.range(clusteringStart, clusteringEnd)
+ .mapToObj(v -> row(v, v, nowInSec))
+ .toArray(Row[]::new);
+ }
+
+ private RangeTombstoneBoundMarker open(int start)
+ {
+ return new RangeTombstoneBoundMarker(
+ ClusteringBound.create(ClusteringBound.boundKind(true, true),
+ new ByteBuffer[] { Clustering.make(Int32Type.instance.decompose(start)).get(0)}),
+ new DeletionTime(FBUtilities.timestampMicros(), FBUtilities.nowInSeconds()));
+ }
+
+ private RangeTombstoneBoundMarker close(int close)
+ {
+ return new RangeTombstoneBoundMarker(
+ ClusteringBound.create(ClusteringBound.boundKind(false, true),
+ new ByteBuffer[] { Clustering.make(Int32Type.instance.decompose(close)).get(0)}),
+ new DeletionTime(FBUtilities.timestampMicros(), FBUtilities.nowInSeconds()));
+ }
+
+ private UnfilteredRowIterator partition(ByteBuffer pk, Unfiltered... unfiltereds)
+ {
+ return partitionWithStaticRow(pk, Rows.EMPTY_STATIC_ROW, unfiltereds);
+ }
+
+ private UnfilteredRowIterator partitionWithStaticRow(ByteBuffer pk, Row staticRow, Unfiltered... unfiltereds)
+ {
+ Iterator<Unfiltered> unfilteredIterator = Arrays.asList(unfiltereds).iterator();
+ return new AbstractUnfilteredRowIterator(metadata, dk(pk), DeletionTime.LIVE, metadata.regularAndStaticColumns(), staticRow, false, EncodingStats.NO_STATS) {
+ protected Unfiltered computeNext()
+ {
+ return unfilteredIterator.hasNext() ? unfilteredIterator.next() : endOfData();
+ }
+ };
+ }
+
+ private static UnfilteredPartitionIterator partitions(UnfilteredRowIterator...partitions)
+ {
+ Iterator<UnfilteredRowIterator> partitionsIter = Arrays.asList(partitions).iterator();
+ return new AbstractUnfilteredPartitionIterator()
+ {
+ public TableMetadata metadata()
+ {
+ return metadata;
+ }
+
+ public boolean hasNext()
+ {
+ return partitionsIter.hasNext();
+ }
+
+ public UnfilteredRowIterator next()
+ {
+ return partitionsIter.next();
+ }
+ };
+ }
+}
diff --git a/test/unit/org/apache/cassandra/db/RepairedDataTombstonesTest.java b/test/unit/org/apache/cassandra/db/RepairedDataTombstonesTest.java
index b814ea6..a864786 100644
--- a/test/unit/org/apache/cassandra/db/RepairedDataTombstonesTest.java
+++ b/test/unit/org/apache/cassandra/db/RepairedDataTombstonesTest.java
@@ -308,7 +308,7 @@
public static void repair(ColumnFamilyStore cfs, SSTableReader sstable) throws IOException
{
- sstable.descriptor.getMetadataSerializer().mutateRepairedAt(sstable.descriptor, 1);
+ sstable.descriptor.getMetadataSerializer().mutateRepairMetadata(sstable.descriptor, 1, null, false);
sstable.reloadSSTableMetadata();
cfs.getTracker().notifySSTableRepairedStatusChanged(Collections.singleton(sstable));
}
diff --git a/test/unit/org/apache/cassandra/db/RowCacheCQLTest.java b/test/unit/org/apache/cassandra/db/RowCacheCQLTest.java
index a8f7e3d..27cfebd 100644
--- a/test/unit/org/apache/cassandra/db/RowCacheCQLTest.java
+++ b/test/unit/org/apache/cassandra/db/RowCacheCQLTest.java
@@ -15,6 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.cassandra.db;
import org.junit.Test;
diff --git a/test/unit/org/apache/cassandra/db/RowCacheTest.java b/test/unit/org/apache/cassandra/db/RowCacheTest.java
index 558c187..5ca1eef 100644
--- a/test/unit/org/apache/cassandra/db/RowCacheTest.java
+++ b/test/unit/org/apache/cassandra/db/RowCacheTest.java
@@ -18,7 +18,6 @@
package org.apache.cassandra.db;
-import java.net.InetAddress;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Arrays;
@@ -32,8 +31,8 @@
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.Util;
import org.apache.cassandra.cache.RowCacheKey;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.Schema;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.db.marshal.AsciiType;
import org.apache.cassandra.db.rows.*;
import org.apache.cassandra.db.compaction.CompactionManager;
@@ -99,9 +98,9 @@
ByteBuffer key = ByteBufferUtil.bytes("rowcachekey");
DecoratedKey dk = cachedStore.decorateKey(key);
- RowCacheKey rck = new RowCacheKey(cachedStore.metadata.ksAndCFName, dk);
+ RowCacheKey rck = new RowCacheKey(cachedStore.metadata(), dk);
- RowUpdateBuilder rub = new RowUpdateBuilder(cachedStore.metadata, System.currentTimeMillis(), key);
+ RowUpdateBuilder rub = new RowUpdateBuilder(cachedStore.metadata(), System.currentTimeMillis(), key);
rub.clustering(String.valueOf(0));
rub.add("val", ByteBufferUtil.bytes("val" + 0));
rub.build().applyUnsafe();
@@ -301,8 +300,8 @@
byte[] tk1, tk2;
tk1 = "key1000".getBytes();
tk2 = "key1050".getBytes();
- tmd.updateNormalToken(new BytesToken(tk1), InetAddress.getByName("127.0.0.1"));
- tmd.updateNormalToken(new BytesToken(tk2), InetAddress.getByName("127.0.0.2"));
+ tmd.updateNormalToken(new BytesToken(tk1), InetAddressAndPort.getByName("127.0.0.1"));
+ tmd.updateNormalToken(new BytesToken(tk2), InetAddressAndPort.getByName("127.0.0.2"));
store.cleanupCache();
assertEquals(50, CacheService.instance.rowCache.size());
CacheService.instance.setRowCacheCapacityInMB(0);
@@ -414,11 +413,11 @@
ByteBuffer key = ByteBufferUtil.bytes("rowcachekey");
DecoratedKey dk = cachedStore.decorateKey(key);
- RowCacheKey rck = new RowCacheKey(cachedStore.metadata.ksAndCFName, dk);
+ RowCacheKey rck = new RowCacheKey(cachedStore.metadata(), dk);
String values[] = new String[200];
for (int i = 0; i < 200; i++)
{
- RowUpdateBuilder rub = new RowUpdateBuilder(cachedStore.metadata, System.currentTimeMillis(), key);
+ RowUpdateBuilder rub = new RowUpdateBuilder(cachedStore.metadata(), System.currentTimeMillis(), key);
rub.clustering(String.valueOf(i));
values[i] = "val" + i;
rub.add("val", ByteBufferUtil.bytes(values[i]));
@@ -548,12 +547,10 @@
private static void readData(String keyspace, String columnFamily, int offset, int numberOfRows)
{
ColumnFamilyStore store = Keyspace.open(keyspace).getColumnFamilyStore(columnFamily);
- CFMetaData cfm = Schema.instance.getCFMetaData(keyspace, columnFamily);
for (int i = offset; i < offset + numberOfRows; i++)
{
DecoratedKey key = Util.dk("key" + i);
- Clustering cl = Clustering.make(ByteBufferUtil.bytes("col" + i));
Util.getAll(Util.cmd(store, key).build());
}
}
diff --git a/test/unit/org/apache/cassandra/db/RowIndexEntryTest.java b/test/unit/org/apache/cassandra/db/RowIndexEntryTest.java
index 6c8eed5..392a1a0 100644
--- a/test/unit/org/apache/cassandra/db/RowIndexEntryTest.java
+++ b/test/unit/org/apache/cassandra/db/RowIndexEntryTest.java
@@ -33,7 +33,9 @@
import org.apache.cassandra.Util;
import org.apache.cassandra.cache.IMeasurableMemory;
-import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.cql3.statements.schema.CreateTableStatement;
+import org.apache.cassandra.db.rows.SerializationHelper;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.CQLTester;
import org.apache.cassandra.db.columniterator.AbstractSSTableIterator;
@@ -131,18 +133,21 @@
private static class DoubleSerializer implements AutoCloseable
{
- CFMetaData cfMeta = CFMetaData.compile("CREATE TABLE pipe.dev_null (pk bigint, ck bigint, val text, PRIMARY KEY(pk, ck))", "foo");
+ TableMetadata metadata =
+ CreateTableStatement.parse("CREATE TABLE pipe.dev_null (pk bigint, ck bigint, val text, PRIMARY KEY(pk, ck))", "foo")
+ .build();
+
Version version = BigFormat.latestVersion;
DeletionTime deletionInfo = new DeletionTime(FBUtilities.timestampMicros(), FBUtilities.nowInSeconds());
LivenessInfo primaryKeyLivenessInfo = LivenessInfo.EMPTY;
Row.Deletion deletion = Row.Deletion.LIVE;
- SerializationHeader header = new SerializationHeader(true, cfMeta, cfMeta.partitionColumns(), EncodingStats.NO_STATS);
+ SerializationHeader header = new SerializationHeader(true, metadata, metadata.regularAndStaticColumns(), EncodingStats.NO_STATS);
// create C-11206 + old serializer instances
- RowIndexEntry.IndexSerializer rieSerializer = new RowIndexEntry.Serializer(cfMeta, version, header);
- Pre_C_11206_RowIndexEntry.Serializer oldSerializer = new Pre_C_11206_RowIndexEntry.Serializer(cfMeta, version, header);
+ RowIndexEntry.IndexSerializer rieSerializer = new RowIndexEntry.Serializer(version, header);
+ Pre_C_11206_RowIndexEntry.Serializer oldSerializer = new Pre_C_11206_RowIndexEntry.Serializer(metadata, version, header);
@SuppressWarnings({ "resource", "IOResourceOpenedButNotSafelyClosed" })
final DataOutputBuffer rieOutput = new DataOutputBuffer(1024);
@@ -161,12 +166,12 @@
DoubleSerializer() throws IOException
{
SequentialWriterOption option = SequentialWriterOption.newBuilder().bufferSize(1024).build();
- File f = File.createTempFile("RowIndexEntryTest-", "db");
+ File f = FileUtils.createTempFile("RowIndexEntryTest-", "db");
dataWriterNew = new SequentialWriter(f, option);
columnIndex = new org.apache.cassandra.db.ColumnIndex(header, dataWriterNew, version, Collections.emptyList(),
rieSerializer.indexInfoSerializer());
- f = File.createTempFile("RowIndexEntryTest-", "db");
+ f = FileUtils.createTempFile("RowIndexEntryTest-", "db");
dataWriterOld = new SequentialWriter(f, option);
}
@@ -201,7 +206,7 @@
private AbstractUnfilteredRowIterator makeRowIter(Row staticRow, DecoratedKey partitionKey,
Iterator<Clustering> clusteringIter, SequentialWriter dataWriter)
{
- return new AbstractUnfilteredRowIterator(cfMeta, partitionKey, deletionInfo, cfMeta.partitionColumns(),
+ return new AbstractUnfilteredRowIterator(metadata, partitionKey, deletionInfo, metadata.regularAndStaticColumns(),
staticRow, false, new EncodingStats(0, 0, 0))
{
protected Unfiltered computeNext()
@@ -225,7 +230,7 @@
private Unfiltered buildRow(Clustering clustering)
{
BTree.Builder<ColumnData> builder = BTree.builder(ColumnData.comparator);
- builder.add(BufferCell.live(cfMeta.partitionColumns().iterator().next(),
+ builder.add(BufferCell.live(metadata.regularAndStaticColumns().iterator().next(),
1L,
ByteBuffer.allocate(0)));
return BTreeRow.create(clustering, primaryKeyLivenessInfo, deletion, builder.build());
@@ -256,7 +261,7 @@
Collection<SSTableFlushObserver> observers,
Version version) throws IOException
{
- assert !iterator.isEmpty() && version.storeRows();
+ assert !iterator.isEmpty();
Builder builder = new Builder(iterator, output, header, observers, version.correspondingMessagingVersion());
return builder.build();
@@ -275,6 +280,7 @@
{
private final UnfilteredRowIterator iterator;
private final SequentialWriter writer;
+ private final SerializationHelper helper;
private final SerializationHeader header;
private final int version;
@@ -302,6 +308,7 @@
{
this.iterator = iterator;
this.writer = writer;
+ this.helper = new SerializationHelper(header);
this.header = header;
this.version = version;
this.observers = observers == null ? Collections.emptyList() : observers;
@@ -313,7 +320,7 @@
ByteBufferUtil.writeWithShortLength(iterator.partitionKey().getKey(), writer);
DeletionTime.serializer.serialize(iterator.partitionLevelDeletion(), writer);
if (header.hasStatic())
- UnfilteredSerializer.serializer.serializeStaticRow(iterator.staticRow(), header, writer, version);
+ UnfilteredSerializer.serializer.serializeStaticRow(iterator.staticRow(), helper, writer, version);
}
public ColumnIndex build() throws IOException
@@ -354,7 +361,7 @@
startPosition = pos;
}
- UnfilteredSerializer.serializer.serialize(unfiltered, header, writer, pos - previousRowStart, version);
+ UnfilteredSerializer.serializer.serialize(unfiltered, helper, writer, pos - previousRowStart, version);
// notify observers about each new row
if (!observers.isEmpty())
@@ -404,8 +411,8 @@
Pre_C_11206_RowIndexEntry simple = new Pre_C_11206_RowIndexEntry(123);
DataOutputBuffer buffer = new DataOutputBuffer();
- SerializationHeader header = new SerializationHeader(true, cfs.metadata, cfs.metadata.partitionColumns(), EncodingStats.NO_STATS);
- Pre_C_11206_RowIndexEntry.Serializer serializer = new Pre_C_11206_RowIndexEntry.Serializer(cfs.metadata, BigFormat.latestVersion, header);
+ SerializationHeader header = new SerializationHeader(true, cfs.metadata(), cfs.metadata().regularAndStaticColumns(), EncodingStats.NO_STATS);
+ Pre_C_11206_RowIndexEntry.Serializer serializer = new Pre_C_11206_RowIndexEntry.Serializer(cfs.metadata(), BigFormat.latestVersion, header);
serializer.serialize(simple, buffer);
@@ -417,12 +424,12 @@
ImmutableBTreePartition partition = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs).build());
- File tempFile = File.createTempFile("row_index_entry_test", null);
+ File tempFile = FileUtils.createTempFile("row_index_entry_test", null);
tempFile.deleteOnExit();
SequentialWriter writer = new SequentialWriter(tempFile);
ColumnIndex columnIndex = RowIndexEntryTest.ColumnIndex.writeAndBuildIndex(partition.unfilteredIterator(), writer, header, Collections.emptySet(), BigFormat.latestVersion);
Pre_C_11206_RowIndexEntry withIndex = Pre_C_11206_RowIndexEntry.create(0xdeadbeef, DeletionTime.LIVE, columnIndex);
- IndexInfo.Serializer indexSerializer = cfs.metadata.serializers().indexInfoSerializer(BigFormat.latestVersion, header);
+ IndexInfo.Serializer indexSerializer = IndexInfo.serializer(BigFormat.latestVersion, header);
// sanity check
assertTrue(columnIndex.columnsIndex.size() >= 3);
@@ -565,16 +572,14 @@
private final IndexInfo.Serializer idxSerializer;
private final Version version;
- Serializer(CFMetaData metadata, Version version, SerializationHeader header)
+ Serializer(TableMetadata metadata, Version version, SerializationHeader header)
{
- this.idxSerializer = metadata.serializers().indexInfoSerializer(version, header);
+ this.idxSerializer = IndexInfo.serializer(version, header);
this.version = version;
}
public void serialize(Pre_C_11206_RowIndexEntry rie, DataOutputPlus out) throws IOException
{
- assert version.storeRows() : "We read old index files but we should never write them";
-
out.writeUnsignedVInt(rie.position);
out.writeUnsignedVInt(rie.promotedSize(idxSerializer));
@@ -622,35 +627,6 @@
public Pre_C_11206_RowIndexEntry deserialize(DataInputPlus in) throws IOException
{
- if (!version.storeRows())
- {
- long position = in.readLong();
-
- int size = in.readInt();
- if (size > 0)
- {
- DeletionTime deletionTime = DeletionTime.serializer.deserialize(in);
-
- int entries = in.readInt();
- List<IndexInfo> columnsIndex = new ArrayList<>(entries);
-
- long headerLength = 0L;
- for (int i = 0; i < entries; i++)
- {
- IndexInfo info = idxSerializer.deserialize(in);
- columnsIndex.add(info);
- if (i == 0)
- headerLength = info.offset;
- }
-
- return new Pre_C_11206_RowIndexEntry.IndexedEntry(position, deletionTime, headerLength, columnsIndex);
- }
- else
- {
- return new Pre_C_11206_RowIndexEntry(position);
- }
- }
-
long position = in.readUnsignedVInt();
int size = (int)in.readUnsignedVInt();
@@ -678,7 +654,7 @@
// should be used instead.
static long readPosition(DataInputPlus in, Version version) throws IOException
{
- return version.storeRows() ? in.readUnsignedVInt() : in.readLong();
+ return in.readUnsignedVInt();
}
public static void skip(DataInputPlus in, Version version) throws IOException
@@ -689,7 +665,7 @@
private static void skipPromotedIndex(DataInputPlus in, Version version) throws IOException
{
- int size = version.storeRows() ? (int)in.readUnsignedVInt() : in.readInt();
+ int size = (int)in.readUnsignedVInt();
if (size <= 0)
return;
@@ -698,8 +674,6 @@
public int serializedSize(Pre_C_11206_RowIndexEntry rie)
{
- assert version.storeRows() : "We read old index files but we should never write them";
-
int indexedSize = 0;
if (rie.isIndexed())
{
diff --git a/test/unit/org/apache/cassandra/db/RowTest.java b/test/unit/org/apache/cassandra/db/RowTest.java
index 5fdb98a..fe54299 100644
--- a/test/unit/org/apache/cassandra/db/RowTest.java
+++ b/test/unit/org/apache/cassandra/db/RowTest.java
@@ -29,8 +29,8 @@
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.Util;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.ColumnIdentifier;
import org.apache.cassandra.db.rows.*;
@@ -54,22 +54,23 @@
private int nowInSeconds;
private DecoratedKey dk;
private ColumnFamilyStore cfs;
- private CFMetaData cfm;
+ private TableMetadata metadata;
@BeforeClass
public static void defineSchema() throws ConfigurationException
{
DatabaseDescriptor.daemonInitialization();
- CFMetaData cfMetadata = CFMetaData.Builder.create(KEYSPACE1, CF_STANDARD1)
- .addPartitionKey("key", BytesType.instance)
- .addClusteringColumn("col1", AsciiType.instance)
- .addRegularColumn("a", AsciiType.instance)
- .addRegularColumn("b", AsciiType.instance)
- .build();
+
+ TableMetadata.Builder metadata =
+ TableMetadata.builder(KEYSPACE1, CF_STANDARD1)
+ .addPartitionKeyColumn("key", BytesType.instance)
+ .addClusteringColumn("col1", AsciiType.instance)
+ .addRegularColumn("a", AsciiType.instance)
+ .addRegularColumn("b", AsciiType.instance);
+
SchemaLoader.prepareServer();
- SchemaLoader.createKeyspace(KEYSPACE1,
- KeyspaceParams.simple(1),
- cfMetadata);
+
+ SchemaLoader.createKeyspace(KEYSPACE1, KeyspaceParams.simple(1), metadata);
}
@Before
@@ -78,19 +79,19 @@
nowInSeconds = FBUtilities.nowInSeconds();
dk = Util.dk("key0");
cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD1);
- cfm = cfs.metadata;
+ metadata = cfs.metadata();
}
@Test
- public void testMergeRangeTombstones() throws InterruptedException
+ public void testMergeRangeTombstones()
{
- PartitionUpdate update1 = new PartitionUpdate(cfm, dk, cfm.partitionColumns(), 1);
+ PartitionUpdate.Builder update1 = new PartitionUpdate.Builder(metadata, dk, metadata.regularAndStaticColumns(), 1);
writeRangeTombstone(update1, "1", "11", 123, 123);
writeRangeTombstone(update1, "2", "22", 123, 123);
writeRangeTombstone(update1, "3", "31", 123, 123);
writeRangeTombstone(update1, "4", "41", 123, 123);
- PartitionUpdate update2 = new PartitionUpdate(cfm, dk, cfm.partitionColumns(), 1);
+ PartitionUpdate.Builder update2 = new PartitionUpdate.Builder(metadata, dk, metadata.regularAndStaticColumns(), 1);
writeRangeTombstone(update2, "1", "11", 123, 123);
writeRangeTombstone(update2, "111", "112", 1230, 123);
writeRangeTombstone(update2, "2", "24", 123, 123);
@@ -98,7 +99,7 @@
writeRangeTombstone(update2, "4", "41", 123, 1230);
writeRangeTombstone(update2, "5", "51", 123, 1230);
- try (UnfilteredRowIterator merged = UnfilteredRowIterators.merge(ImmutableList.of(update1.unfilteredIterator(), update2.unfilteredIterator()), nowInSeconds))
+ try (UnfilteredRowIterator merged = UnfilteredRowIterators.merge(ImmutableList.of(update1.build().unfilteredIterator(), update2.build().unfilteredIterator())))
{
Object[][] expected = new Object[][]{ { "1", "11", 123l, 123 },
{ "111", "112", 1230l, 123 },
@@ -128,17 +129,17 @@
@Test
public void testResolve()
{
- ColumnDefinition defA = cfm.getColumnDefinition(new ColumnIdentifier("a", true));
- ColumnDefinition defB = cfm.getColumnDefinition(new ColumnIdentifier("b", true));
+ ColumnMetadata defA = metadata.getColumn(new ColumnIdentifier("a", true));
+ ColumnMetadata defB = metadata.getColumn(new ColumnIdentifier("b", true));
- Row.Builder builder = BTreeRow.unsortedBuilder(nowInSeconds);
- builder.newRow(cfm.comparator.make("c1"));
- writeSimpleCellValue(builder, cfm, defA, "a1", 0);
- writeSimpleCellValue(builder, cfm, defA, "a2", 1);
- writeSimpleCellValue(builder, cfm, defB, "b1", 1);
+ Row.Builder builder = BTreeRow.unsortedBuilder();
+ builder.newRow(metadata.comparator.make("c1"));
+ writeSimpleCellValue(builder, defA, "a1", 0);
+ writeSimpleCellValue(builder, defA, "a2", 1);
+ writeSimpleCellValue(builder, defB, "b1", 1);
Row row = builder.build();
- PartitionUpdate update = PartitionUpdate.singleRowUpdate(cfm, dk, row);
+ PartitionUpdate update = PartitionUpdate.singleRowUpdate(metadata, dk, row);
Unfiltered unfiltered = update.unfilteredIterator().next();
assertTrue(unfiltered.kind() == Unfiltered.Kind.ROW);
@@ -152,11 +153,11 @@
public void testExpiringColumnExpiration() throws IOException
{
int ttl = 1;
- ColumnDefinition def = cfm.getColumnDefinition(new ColumnIdentifier("a", true));
+ ColumnMetadata def = metadata.getColumn(new ColumnIdentifier("a", true));
Cell cell = BufferCell.expiring(def, 0, ttl, nowInSeconds, ((AbstractType) def.cellValueType()).decompose("a1"));
- PartitionUpdate update = PartitionUpdate.singleRowUpdate(cfm, dk, BTreeRow.singleCellRow(cfm.comparator.make("c1"), cell));
+ PartitionUpdate update = PartitionUpdate.singleRowUpdate(metadata, dk, BTreeRow.singleCellRow(metadata.comparator.make("c1"), cell));
new Mutation(update).applyUnsafe();
// when we read with a nowInSeconds before the cell has expired,
@@ -172,14 +173,14 @@
@Test
public void testHashCode()
{
- ColumnDefinition defA = cfm.getColumnDefinition(new ColumnIdentifier("a", true));
- ColumnDefinition defB = cfm.getColumnDefinition(new ColumnIdentifier("b", true));
+ ColumnMetadata defA = metadata.getColumn(new ColumnIdentifier("a", true));
+ ColumnMetadata defB = metadata.getColumn(new ColumnIdentifier("b", true));
- Row.Builder builder = BTreeRow.unsortedBuilder(nowInSeconds);
- builder.newRow(cfm.comparator.make("c1"));
- writeSimpleCellValue(builder, cfm, defA, "a1", 0);
- writeSimpleCellValue(builder, cfm, defA, "a2", 1);
- writeSimpleCellValue(builder, cfm, defB, "b1", 1);
+ Row.Builder builder = BTreeRow.unsortedBuilder();
+ builder.newRow(metadata.comparator.make("c1"));
+ writeSimpleCellValue(builder, defA, "a1", 0);
+ writeSimpleCellValue(builder, defA, "a2", 1);
+ writeSimpleCellValue(builder, defB, "b1", 1);
Row row = builder.build();
Map<Row, Integer> map = new HashMap<>();
@@ -189,7 +190,7 @@
private void assertRangeTombstoneMarkers(ClusteringBound start, ClusteringBound end, DeletionTime deletionTime, Object[] expected)
{
- AbstractType clusteringType = (AbstractType)cfm.comparator.subtype(0);
+ AbstractType clusteringType = (AbstractType) metadata.comparator.subtype(0);
assertEquals(1, start.size());
assertEquals(start.kind(), ClusteringPrefix.Kind.INCL_START_BOUND);
@@ -203,18 +204,17 @@
assertEquals(expected[3], deletionTime.localDeletionTime());
}
- public void writeRangeTombstone(PartitionUpdate update, Object start, Object end, long markedForDeleteAt, int localDeletionTime)
+ public void writeRangeTombstone(PartitionUpdate.Builder update, Object start, Object end, long markedForDeleteAt, int localDeletionTime)
{
ClusteringComparator comparator = cfs.getComparator();
update.add(new RangeTombstone(Slice.make(comparator.make(start), comparator.make(end)), new DeletionTime(markedForDeleteAt, localDeletionTime)));
}
private void writeSimpleCellValue(Row.Builder builder,
- CFMetaData cfm,
- ColumnDefinition columnDefinition,
+ ColumnMetadata columnMetadata,
String value,
long timestamp)
{
- builder.addCell(BufferCell.live(columnDefinition, timestamp, ((AbstractType) columnDefinition.cellValueType()).decompose(value)));
+ builder.addCell(BufferCell.live(columnMetadata, timestamp, ((AbstractType) columnMetadata.cellValueType()).decompose(value)));
}
}
diff --git a/test/unit/org/apache/cassandra/db/RowUpdateBuilder.java b/test/unit/org/apache/cassandra/db/RowUpdateBuilder.java
index 8535616..3a07a00 100644
--- a/test/unit/org/apache/cassandra/db/RowUpdateBuilder.java
+++ b/test/unit/org/apache/cassandra/db/RowUpdateBuilder.java
@@ -18,15 +18,15 @@
package org.apache.cassandra.db;
import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.List;
+import java.util.*;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
-import org.apache.cassandra.db.partitions.PartitionUpdate;
-import org.apache.cassandra.db.rows.BTreeRow;
-import org.apache.cassandra.db.rows.Row;
-import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.cql3.ColumnIdentifier;
+import org.apache.cassandra.db.context.CounterContext;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.db.rows.*;
+import org.apache.cassandra.db.partitions.*;
+import org.apache.cassandra.utils.*;
/**
* Convenience object to create single row updates for tests.
@@ -48,22 +48,22 @@
this.updateBuilder = updateBuilder;
}
- public RowUpdateBuilder(CFMetaData metadata, long timestamp, Object partitionKey)
+ public RowUpdateBuilder(TableMetadata metadata, long timestamp, Object partitionKey)
{
this(metadata, FBUtilities.nowInSeconds(), timestamp, partitionKey);
}
- public RowUpdateBuilder(CFMetaData metadata, int localDeletionTime, long timestamp, Object partitionKey)
+ public RowUpdateBuilder(TableMetadata metadata, int localDeletionTime, long timestamp, Object partitionKey)
{
this(metadata, localDeletionTime, timestamp, metadata.params.defaultTimeToLive, partitionKey);
}
- public RowUpdateBuilder(CFMetaData metadata, long timestamp, int ttl, Object partitionKey)
+ public RowUpdateBuilder(TableMetadata metadata, long timestamp, int ttl, Object partitionKey)
{
this(metadata, FBUtilities.nowInSeconds(), timestamp, ttl, partitionKey);
}
- public RowUpdateBuilder(CFMetaData metadata, int localDeletionTime, long timestamp, int ttl, Object partitionKey)
+ public RowUpdateBuilder(TableMetadata metadata, int localDeletionTime, long timestamp, int ttl, Object partitionKey)
{
this(PartitionUpdate.simpleBuilder(metadata, partitionKey));
@@ -72,6 +72,12 @@
this.updateBuilder.nowInSec(localDeletionTime);
}
+ public RowUpdateBuilder timestamp(long ts)
+ {
+ updateBuilder.timestamp(ts);
+ return this;
+ }
+
private Row.SimpleBuilder rowBuilder()
{
// Normally, rowBuilder is created by the call to clustering(), but we allow skipping that call for an empty
@@ -111,49 +117,37 @@
public PartitionUpdate buildUpdate()
{
- PartitionUpdate update = updateBuilder.build();
for (RangeTombstone rt : rts)
- update.add(rt);
- return update;
+ updateBuilder.addRangeTombstone(rt);
+ return updateBuilder.build();
}
- private static void deleteRow(PartitionUpdate update, long timestamp, int localDeletionTime, Object... clusteringValues)
+ private static void deleteRow(PartitionUpdate.Builder updateBuilder, long timestamp, int localDeletionTime, Object... clusteringValues)
{
- assert clusteringValues.length == update.metadata().comparator.size() || (clusteringValues.length == 0 && !update.columns().statics.isEmpty());
-
- boolean isStatic = clusteringValues.length != update.metadata().comparator.size();
- Row.Builder builder = BTreeRow.sortedBuilder();
-
- if (isStatic)
- builder.newRow(Clustering.STATIC_CLUSTERING);
- else
- builder.newRow(clusteringValues.length == 0 ? Clustering.EMPTY : update.metadata().comparator.make(clusteringValues));
- builder.addRowDeletion(Row.Deletion.regular(new DeletionTime(timestamp, localDeletionTime)));
-
- update.add(builder.build());
+ SimpleBuilders.RowBuilder b = new SimpleBuilders.RowBuilder(updateBuilder.metadata(), clusteringValues);
+ b.nowInSec(localDeletionTime).timestamp(timestamp).delete();
+ updateBuilder.add(b.build());
}
- public static Mutation deleteRow(CFMetaData metadata, long timestamp, Object key, Object... clusteringValues)
+ public static Mutation deleteRow(TableMetadata metadata, long timestamp, Object key, Object... clusteringValues)
{
return deleteRowAt(metadata, timestamp, FBUtilities.nowInSeconds(), key, clusteringValues);
}
- public static Mutation deleteRowAt(CFMetaData metadata, long timestamp, int localDeletionTime, Object key, Object... clusteringValues)
+ public static Mutation deleteRowAt(TableMetadata metadata, long timestamp, int localDeletionTime, Object key, Object... clusteringValues)
{
- PartitionUpdate update = new PartitionUpdate(metadata, makeKey(metadata, key), metadata.partitionColumns(), 0);
+ PartitionUpdate.Builder update = new PartitionUpdate.Builder(metadata, makeKey(metadata, key), metadata.regularAndStaticColumns(), 0);
deleteRow(update, timestamp, localDeletionTime, clusteringValues);
- // note that the created mutation may get further update later on, so we don't use the ctor that create a singletonMap
- // underneath (this class if for convenience, not performance)
- return new Mutation(update.metadata().ksName, update.partitionKey()).add(update);
+ return new Mutation.PartitionUpdateCollector(update.metadata().keyspace, update.partitionKey()).add(update.build()).build();
}
- private static DecoratedKey makeKey(CFMetaData metadata, Object... partitionKey)
+ private static DecoratedKey makeKey(TableMetadata metadata, Object... partitionKey)
{
if (partitionKey.length == 1 && partitionKey[0] instanceof DecoratedKey)
return (DecoratedKey)partitionKey[0];
- ByteBuffer key = CFMetaData.serializePartitionKey(metadata.getKeyValidatorAsClusteringComparator().make(partitionKey));
- return metadata.decorateKey(key);
+ ByteBuffer key = metadata.partitionKeyAsClusteringComparator().make(partitionKey).serializeAsPartitionKey();
+ return metadata.partitioner.decorateKey(key);
}
public RowUpdateBuilder addRangeTombstone(RangeTombstone rt)
@@ -174,9 +168,9 @@
return this;
}
- public RowUpdateBuilder add(ColumnDefinition columnDefinition, Object value)
+ public RowUpdateBuilder add(ColumnMetadata columnMetadata, Object value)
{
- return add(columnDefinition.name.toString(), value);
+ return add(columnMetadata.name.toString(), value);
}
public RowUpdateBuilder delete(String columnName)
@@ -185,8 +179,16 @@
return this;
}
- public RowUpdateBuilder delete(ColumnDefinition columnDefinition)
+ public RowUpdateBuilder delete(ColumnMetadata columnMetadata)
{
- return delete(columnDefinition.name.toString());
+ return delete(columnMetadata.name.toString());
+ }
+
+ public RowUpdateBuilder addLegacyCounterCell(String columnName, long value)
+ {
+ assert updateBuilder.metadata().getColumn(new ColumnIdentifier(columnName, true)).isCounterColumn();
+ ByteBuffer val = CounterContext.instance().createLocal(value);
+ rowBuilder().add(columnName, val);
+ return this;
}
}
diff --git a/test/unit/org/apache/cassandra/db/SchemaCQLHelperTest.java b/test/unit/org/apache/cassandra/db/SchemaCQLHelperTest.java
new file mode 100644
index 0000000..6857dd3
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/SchemaCQLHelperTest.java
@@ -0,0 +1,468 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.io.Files;
+
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import org.apache.cassandra.*;
+import org.apache.cassandra.cql3.*;
+import org.apache.cassandra.cql3.statements.schema.IndexTarget;
+import org.apache.cassandra.db.marshal.*;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.index.sasi.SASIIndex;
+import org.apache.cassandra.schema.*;
+import org.apache.cassandra.service.reads.SpeculativeRetryPolicy;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.FBUtilities;
+import org.json.simple.JSONArray;
+import org.json.simple.JSONObject;
+import org.json.simple.parser.JSONParser;
+
+import java.io.FileReader;
+import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.stream.Collectors;
+
+import static org.hamcrest.CoreMatchers.allOf;
+import static org.hamcrest.CoreMatchers.containsString;
+import static org.hamcrest.CoreMatchers.startsWith;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThat;
+
+public class SchemaCQLHelperTest extends CQLTester
+{
+ @Before
+ public void defineSchema() throws ConfigurationException
+ {
+ SchemaLoader.prepareServer();
+ }
+
+ @Test
+ public void testUserTypesCQL()
+ {
+ String keyspace = "cql_test_keyspace_user_types";
+ String table = "test_table_user_types";
+
+ UserType typeA = new UserType(keyspace, ByteBufferUtil.bytes("a"),
+ Arrays.asList(FieldIdentifier.forUnquoted("a1"),
+ FieldIdentifier.forUnquoted("a2"),
+ FieldIdentifier.forUnquoted("a3")),
+ Arrays.asList(IntegerType.instance,
+ IntegerType.instance,
+ IntegerType.instance),
+ true);
+
+ UserType typeB = new UserType(keyspace, ByteBufferUtil.bytes("b"),
+ Arrays.asList(FieldIdentifier.forUnquoted("b1"),
+ FieldIdentifier.forUnquoted("b2"),
+ FieldIdentifier.forUnquoted("b3")),
+ Arrays.asList(typeA,
+ typeA,
+ typeA),
+ true);
+
+ UserType typeC = new UserType(keyspace, ByteBufferUtil.bytes("c"),
+ Arrays.asList(FieldIdentifier.forUnquoted("c1"),
+ FieldIdentifier.forUnquoted("c2"),
+ FieldIdentifier.forUnquoted("c3")),
+ Arrays.asList(typeB,
+ typeB,
+ typeB),
+ true);
+
+ TableMetadata cfm =
+ TableMetadata.builder(keyspace, table)
+ .addPartitionKeyColumn("pk1", IntegerType.instance)
+ .addClusteringColumn("ck1", IntegerType.instance)
+ .addRegularColumn("reg1", typeC.freeze())
+ .addRegularColumn("reg2", ListType.getInstance(IntegerType.instance, false))
+ .addRegularColumn("reg3", MapType.getInstance(AsciiType.instance, IntegerType.instance, true))
+ .build();
+
+ SchemaLoader.createKeyspace(keyspace, KeyspaceParams.simple(1), Tables.of(cfm), Types.of(typeA, typeB, typeC));
+
+ ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
+
+ assertEquals(ImmutableList.of("CREATE TYPE cql_test_keyspace_user_types.a (\n" +
+ " a1 varint,\n" +
+ " a2 varint,\n" +
+ " a3 varint\n" +
+ ");",
+ "CREATE TYPE cql_test_keyspace_user_types.b (\n" +
+ " b1 a,\n" +
+ " b2 a,\n" +
+ " b3 a\n" +
+ ");",
+ "CREATE TYPE cql_test_keyspace_user_types.c (\n" +
+ " c1 b,\n" +
+ " c2 b,\n" +
+ " c3 b\n" +
+ ");"),
+ SchemaCQLHelper.getUserTypesAsCQL(cfs.metadata(), cfs.keyspace.getMetadata().types).collect(Collectors.toList()));
+ }
+
+ @Test
+ public void testDroppedColumnsCQL()
+ {
+ String keyspace = "cql_test_keyspace_dropped_columns";
+ String table = "test_table_dropped_columns";
+
+ TableMetadata.Builder builder =
+ TableMetadata.builder(keyspace, table)
+ .addPartitionKeyColumn("pk1", IntegerType.instance)
+ .addClusteringColumn("ck1", IntegerType.instance)
+ .addStaticColumn("st1", IntegerType.instance)
+ .addRegularColumn("reg1", IntegerType.instance)
+ .addRegularColumn("reg2", IntegerType.instance)
+ .addRegularColumn("reg3", IntegerType.instance);
+
+ ColumnMetadata st1 = builder.getColumn(ByteBufferUtil.bytes("st1"));
+ ColumnMetadata reg1 = builder.getColumn(ByteBufferUtil.bytes("reg1"));
+ ColumnMetadata reg2 = builder.getColumn(ByteBufferUtil.bytes("reg2"));
+ ColumnMetadata reg3 = builder.getColumn(ByteBufferUtil.bytes("reg3"));
+
+ builder.removeRegularOrStaticColumn(st1.name)
+ .removeRegularOrStaticColumn(reg1.name)
+ .removeRegularOrStaticColumn(reg2.name)
+ .removeRegularOrStaticColumn(reg3.name);
+
+ builder.recordColumnDrop(st1, 5000)
+ .recordColumnDrop(reg1, 10000)
+ .recordColumnDrop(reg2, 20000)
+ .recordColumnDrop(reg3, 30000);
+
+ SchemaLoader.createKeyspace(keyspace, KeyspaceParams.simple(1), builder);
+
+ ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
+
+ String expected = "CREATE TABLE IF NOT EXISTS cql_test_keyspace_dropped_columns.test_table_dropped_columns (\n" +
+ " pk1 varint,\n" +
+ " ck1 varint,\n" +
+ " reg1 varint,\n" +
+ " reg3 varint,\n" +
+ " reg2 varint,\n" +
+ " st1 varint static,\n" +
+ " PRIMARY KEY (pk1, ck1)\n) WITH ID =";
+ String actual = SchemaCQLHelper.getTableMetadataAsCQL(cfs.metadata(), true, true, true);
+
+ assertThat(actual,
+ allOf(startsWith(expected),
+ containsString("ALTER TABLE cql_test_keyspace_dropped_columns.test_table_dropped_columns DROP reg1 USING TIMESTAMP 10000;"),
+ containsString("ALTER TABLE cql_test_keyspace_dropped_columns.test_table_dropped_columns DROP reg3 USING TIMESTAMP 30000;"),
+ containsString("ALTER TABLE cql_test_keyspace_dropped_columns.test_table_dropped_columns DROP reg2 USING TIMESTAMP 20000;"),
+ containsString("ALTER TABLE cql_test_keyspace_dropped_columns.test_table_dropped_columns DROP st1 USING TIMESTAMP 5000;")));
+ }
+
+ @Test
+ public void testReaddedColumns()
+ {
+ String keyspace = "cql_test_keyspace_readded_columns";
+ String table = "test_table_readded_columns";
+
+ TableMetadata.Builder builder =
+ TableMetadata.builder(keyspace, table)
+ .addPartitionKeyColumn("pk1", IntegerType.instance)
+ .addClusteringColumn("ck1", IntegerType.instance)
+ .addRegularColumn("reg1", IntegerType.instance)
+ .addStaticColumn("st1", IntegerType.instance)
+ .addRegularColumn("reg2", IntegerType.instance);
+
+ ColumnMetadata reg1 = builder.getColumn(ByteBufferUtil.bytes("reg1"));
+ ColumnMetadata st1 = builder.getColumn(ByteBufferUtil.bytes("st1"));
+
+ builder.removeRegularOrStaticColumn(reg1.name);
+ builder.removeRegularOrStaticColumn(st1.name);
+
+ builder.recordColumnDrop(reg1, 10000);
+ builder.recordColumnDrop(st1, 20000);
+
+ builder.addColumn(reg1);
+ builder.addColumn(st1);
+
+ SchemaLoader.createKeyspace(keyspace, KeyspaceParams.simple(1), builder);
+
+ ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
+
+ // when re-adding, column is present as both column and as dropped column record.
+ String actual = SchemaCQLHelper.getTableMetadataAsCQL(cfs.metadata(), true, true, true);
+ String expected = "CREATE TABLE IF NOT EXISTS cql_test_keyspace_readded_columns.test_table_readded_columns (\n" +
+ " pk1 varint,\n" +
+ " ck1 varint,\n" +
+ " reg2 varint,\n" +
+ " reg1 varint,\n" +
+ " st1 varint static,\n" +
+ " PRIMARY KEY (pk1, ck1)\n" +
+ ") WITH ID";
+
+ assertThat(actual,
+ allOf(startsWith(expected),
+ containsString("ALTER TABLE cql_test_keyspace_readded_columns.test_table_readded_columns DROP reg1 USING TIMESTAMP 10000;"),
+ containsString("ALTER TABLE cql_test_keyspace_readded_columns.test_table_readded_columns ADD reg1 varint;"),
+ containsString("ALTER TABLE cql_test_keyspace_readded_columns.test_table_readded_columns DROP st1 USING TIMESTAMP 20000;"),
+ containsString("ALTER TABLE cql_test_keyspace_readded_columns.test_table_readded_columns ADD st1 varint static;")));
+ }
+
+ @Test
+ public void testCfmColumnsCQL()
+ {
+ String keyspace = "cql_test_keyspace_create_table";
+ String table = "test_table_create_table";
+
+ TableMetadata.Builder metadata =
+ TableMetadata.builder(keyspace, table)
+ .addPartitionKeyColumn("pk1", IntegerType.instance)
+ .addPartitionKeyColumn("pk2", AsciiType.instance)
+ .addClusteringColumn("ck1", ReversedType.getInstance(IntegerType.instance))
+ .addClusteringColumn("ck2", IntegerType.instance)
+ .addStaticColumn("st1", AsciiType.instance)
+ .addRegularColumn("reg1", AsciiType.instance)
+ .addRegularColumn("reg2", ListType.getInstance(IntegerType.instance, false))
+ .addRegularColumn("reg3", MapType.getInstance(AsciiType.instance, IntegerType.instance, true));
+
+ SchemaLoader.createKeyspace(keyspace, KeyspaceParams.simple(1), metadata);
+
+ ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
+
+ assertThat(SchemaCQLHelper.getTableMetadataAsCQL(cfs.metadata(), true, true, true),
+ startsWith(
+ "CREATE TABLE IF NOT EXISTS cql_test_keyspace_create_table.test_table_create_table (\n" +
+ " pk1 varint,\n" +
+ " pk2 ascii,\n" +
+ " ck1 varint,\n" +
+ " ck2 varint,\n" +
+ " st1 ascii static,\n" +
+ " reg1 ascii,\n" +
+ " reg2 frozen<list<varint>>,\n" +
+ " reg3 map<ascii, varint>,\n" +
+ " PRIMARY KEY ((pk1, pk2), ck1, ck2)\n" +
+ ") WITH ID = " + cfs.metadata.id + "\n" +
+ " AND CLUSTERING ORDER BY (ck1 DESC, ck2 ASC)"));
+ }
+
+ @Test
+ public void testCfmOptionsCQL()
+ {
+ String keyspace = "cql_test_keyspace_options";
+ String table = "test_table_options";
+
+ TableMetadata.Builder builder = TableMetadata.builder(keyspace, table);
+ long droppedTimestamp = FBUtilities.timestampMicros();
+ builder.addPartitionKeyColumn("pk1", IntegerType.instance)
+ .addClusteringColumn("cl1", IntegerType.instance)
+ .addRegularColumn("reg1", AsciiType.instance)
+ .bloomFilterFpChance(1.0)
+ .comment("comment")
+ .compaction(CompactionParams.lcs(Collections.singletonMap("sstable_size_in_mb", "1")))
+ .compression(CompressionParams.lz4(1 << 16, 1 << 15))
+ .crcCheckChance(0.3)
+ .defaultTimeToLive(4)
+ .gcGraceSeconds(5)
+ .minIndexInterval(6)
+ .maxIndexInterval(7)
+ .memtableFlushPeriod(8)
+ .speculativeRetry(SpeculativeRetryPolicy.fromString("always"))
+ .additionalWritePolicy(SpeculativeRetryPolicy.fromString("always"))
+ .extensions(ImmutableMap.of("ext1", ByteBuffer.wrap("val1".getBytes())))
+ .recordColumnDrop(ColumnMetadata.regularColumn(keyspace, table, "reg1", AsciiType.instance),
+ droppedTimestamp);
+
+ SchemaLoader.createKeyspace(keyspace, KeyspaceParams.simple(1), builder);
+
+ ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
+
+ assertThat(SchemaCQLHelper.getTableMetadataAsCQL(cfs.metadata(), true, true, true),
+ containsString("CLUSTERING ORDER BY (cl1 ASC)\n" +
+ " AND additional_write_policy = 'ALWAYS'\n" +
+ " AND bloom_filter_fp_chance = 1.0\n" +
+ " AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}\n" +
+ " AND cdc = false\n" +
+ " AND comment = 'comment'\n" +
+ " AND compaction = {'class': 'org.apache.cassandra.db.compaction.LeveledCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4', 'sstable_size_in_mb': '1'}\n" +
+ " AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor', 'min_compress_ratio': '2.0'}\n" +
+ " AND crc_check_chance = 0.3\n" +
+ " AND default_time_to_live = 4\n" +
+ " AND extensions = {'ext1': 0x76616c31}\n" +
+ " AND gc_grace_seconds = 5\n" +
+ " AND max_index_interval = 7\n" +
+ " AND memtable_flush_period_in_ms = 8\n" +
+ " AND min_index_interval = 6\n" +
+ " AND read_repair = 'BLOCKING'\n" +
+ " AND speculative_retry = 'ALWAYS';"
+ ));
+ }
+
+ @Test
+ public void testCfmIndexJson()
+ {
+ String keyspace = "cql_test_keyspace_3";
+ String table = "test_table_3";
+
+ TableMetadata.Builder builder =
+ TableMetadata.builder(keyspace, table)
+ .addPartitionKeyColumn("pk1", IntegerType.instance)
+ .addClusteringColumn("cl1", IntegerType.instance)
+ .addRegularColumn("reg1", AsciiType.instance);
+
+ ColumnIdentifier reg1 = ColumnIdentifier.getInterned("reg1", true);
+
+ builder.indexes(
+ Indexes.of(IndexMetadata.fromIndexTargets(
+ Collections.singletonList(new IndexTarget(reg1, IndexTarget.Type.VALUES)),
+ "indexName",
+ IndexMetadata.Kind.COMPOSITES,
+ Collections.emptyMap()),
+ IndexMetadata.fromIndexTargets(
+ Collections.singletonList(new IndexTarget(reg1, IndexTarget.Type.KEYS)),
+ "indexName2",
+ IndexMetadata.Kind.COMPOSITES,
+ Collections.emptyMap()),
+ IndexMetadata.fromIndexTargets(
+ Collections.singletonList(new IndexTarget(reg1, IndexTarget.Type.KEYS_AND_VALUES)),
+ "indexName3",
+ IndexMetadata.Kind.COMPOSITES,
+ Collections.emptyMap()),
+ IndexMetadata.fromIndexTargets(
+ Collections.singletonList(new IndexTarget(reg1, IndexTarget.Type.KEYS_AND_VALUES)),
+ "indexName4",
+ IndexMetadata.Kind.CUSTOM,
+ Collections.singletonMap(IndexTarget.CUSTOM_INDEX_OPTION_NAME, SASIIndex.class.getName()))));
+
+
+ SchemaLoader.createKeyspace(keyspace, KeyspaceParams.simple(1), builder);
+
+ ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
+
+ assertEquals(ImmutableList.of("CREATE INDEX \"indexName\" ON cql_test_keyspace_3.test_table_3 (values(reg1));",
+ "CREATE INDEX \"indexName2\" ON cql_test_keyspace_3.test_table_3 (keys(reg1));",
+ "CREATE INDEX \"indexName3\" ON cql_test_keyspace_3.test_table_3 (entries(reg1));",
+ "CREATE CUSTOM INDEX \"indexName4\" ON cql_test_keyspace_3.test_table_3 (entries(reg1)) USING 'org.apache.cassandra.index.sasi.SASIIndex';"),
+ SchemaCQLHelper.getIndexesAsCQL(cfs.metadata()).collect(Collectors.toList()));
+ }
+
+ private final static String SNAPSHOT = "testsnapshot";
+
+ @Test
+ public void testSnapshot() throws Throwable
+ {
+ String typeA = createType("CREATE TYPE %s (a1 varint, a2 varint, a3 varint);");
+ String typeB = createType("CREATE TYPE %s (b1 frozen<" + typeA + ">, b2 frozen<" + typeA + ">, b3 frozen<" + typeA + ">);");
+ String typeC = createType("CREATE TYPE %s (c1 frozen<" + typeB + ">, c2 frozen<" + typeB + ">, c3 frozen<" + typeB + ">);");
+
+ String tableName = createTable("CREATE TABLE IF NOT EXISTS %s (" +
+ "pk1 varint," +
+ "pk2 ascii," +
+ "ck1 varint," +
+ "ck2 varint," +
+ "reg1 " + typeC + "," +
+ "reg2 int," +
+ "reg3 int," +
+ "PRIMARY KEY ((pk1, pk2), ck1, ck2)) WITH " +
+ "CLUSTERING ORDER BY (ck1 ASC, ck2 DESC);");
+
+ alterTable("ALTER TABLE %s DROP reg3 USING TIMESTAMP 10000;");
+ alterTable("ALTER TABLE %s ADD reg3 int;");
+
+ for (int i = 0; i < 10; i++)
+ execute("INSERT INTO %s (pk1, pk2, ck1, ck2, reg1, reg2) VALUES (?, ?, ?, ?, ?, ?)", i, i + 1, i + 2, i + 3, null, i + 5);
+
+ ColumnFamilyStore cfs = Keyspace.open(keyspace()).getColumnFamilyStore(tableName);
+ cfs.snapshot(SNAPSHOT);
+
+ String schema = Files.toString(cfs.getDirectories().getSnapshotSchemaFile(SNAPSHOT), Charset.defaultCharset());
+ assertThat(schema,
+ allOf(containsString(String.format("CREATE TYPE %s.%s (\n" +
+ " a1 varint,\n" +
+ " a2 varint,\n" +
+ " a3 varint\n" +
+ ");", keyspace(), typeA)),
+ containsString(String.format("CREATE TYPE %s.%s (\n" +
+ " a1 varint,\n" +
+ " a2 varint,\n" +
+ " a3 varint\n" +
+ ");", keyspace(), typeA)),
+ containsString(String.format("CREATE TYPE %s.%s (\n" +
+ " b1 frozen<%s>,\n" +
+ " b2 frozen<%s>,\n" +
+ " b3 frozen<%s>\n" +
+ ");", keyspace(), typeB, typeA, typeA, typeA)),
+ containsString(String.format("CREATE TYPE %s.%s (\n" +
+ " c1 frozen<%s>,\n" +
+ " c2 frozen<%s>,\n" +
+ " c3 frozen<%s>\n" +
+ ");", keyspace(), typeC, typeB, typeB, typeB))));
+
+ schema = schema.substring(schema.indexOf("CREATE TABLE")); // trim to ensure order
+ String expected = "CREATE TABLE IF NOT EXISTS " + keyspace() + "." + tableName + " (\n" +
+ " pk1 varint,\n" +
+ " pk2 ascii,\n" +
+ " ck1 varint,\n" +
+ " ck2 varint,\n" +
+ " reg2 int,\n" +
+ " reg1 " + typeC+ ",\n" +
+ " reg3 int,\n" +
+ " PRIMARY KEY ((pk1, pk2), ck1, ck2)\n" +
+ ") WITH ID = " + cfs.metadata.id + "\n" +
+ " AND CLUSTERING ORDER BY (ck1 ASC, ck2 DESC)";
+
+ assertThat(schema,
+ allOf(startsWith(expected),
+ containsString("ALTER TABLE " + keyspace() + "." + tableName + " DROP reg3 USING TIMESTAMP 10000;"),
+ containsString("ALTER TABLE " + keyspace() + "." + tableName + " ADD reg3 int;")));
+
+ JSONObject manifest = (JSONObject) new JSONParser().parse(new FileReader(cfs.getDirectories().getSnapshotManifestFile(SNAPSHOT)));
+ JSONArray files = (JSONArray) manifest.get("files");
+ Assert.assertEquals(1, files.size());
+ }
+
+ @Test
+ public void testSystemKsSnapshot()
+ {
+ ColumnFamilyStore cfs = Keyspace.open("system").getColumnFamilyStore("peers");
+ cfs.snapshot(SNAPSHOT);
+
+ Assert.assertTrue(cfs.getDirectories().getSnapshotManifestFile(SNAPSHOT).exists());
+ Assert.assertFalse(cfs.getDirectories().getSnapshotSchemaFile(SNAPSHOT).exists());
+ }
+
+ @Test
+ public void testBooleanCompositeKey() throws Throwable
+ {
+ createTable("CREATE TABLE %s (t_id boolean, id boolean, ck boolean, nk boolean, PRIMARY KEY ((t_id, id), ck))");
+
+ execute("insert into %s (t_id, id, ck, nk) VALUES (true, false, false, true)");
+ assertRows(execute("select * from %s"), row(true, false, false, true));
+
+ // CASSANDRA-14752 -
+ // a problem with composite boolean types meant that calling this would
+ // prevent any boolean values to be inserted afterwards
+ ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
+ cfs.getSSTablesForKey("false:true");
+
+ execute("insert into %s (t_id, id, ck, nk) VALUES (true, true, false, true)");
+ assertRows(execute("select t_id, id, ck, nk from %s"), row(true, false, false, true), row(true, true, false, true));
+ }
+}
diff --git a/test/unit/org/apache/cassandra/db/ScrubTest.java b/test/unit/org/apache/cassandra/db/ScrubTest.java
index f53a6be..28962db 100644
--- a/test/unit/org/apache/cassandra/db/ScrubTest.java
+++ b/test/unit/org/apache/cassandra/db/ScrubTest.java
@@ -34,7 +34,6 @@
import org.apache.cassandra.*;
import org.apache.cassandra.cache.ChunkCache;
-import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.Operator;
import org.apache.cassandra.cql3.QueryProcessor;
@@ -43,7 +42,6 @@
import org.apache.cassandra.db.compaction.OperationType;
import org.apache.cassandra.db.compaction.Scrubber;
import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
-import org.apache.cassandra.db.marshal.Int32Type;
import org.apache.cassandra.db.marshal.LongType;
import org.apache.cassandra.db.marshal.UUIDType;
import org.apache.cassandra.db.partitions.Partition;
@@ -61,8 +59,14 @@
import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.TableMetadataRef;
import org.apache.cassandra.utils.ByteBufferUtil;
+import static org.apache.cassandra.SchemaLoader.counterCFMD;
+import static org.apache.cassandra.SchemaLoader.createKeyspace;
+import static org.apache.cassandra.SchemaLoader.getCompressionParameters;
+import static org.apache.cassandra.SchemaLoader.loadSchema;
+import static org.apache.cassandra.SchemaLoader.standardCFMD;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
@@ -93,19 +97,18 @@
@BeforeClass
public static void defineSchema() throws ConfigurationException
{
- SchemaLoader.loadSchema();
- SchemaLoader.createKeyspace(KEYSPACE,
- KeyspaceParams.simple(1),
- SchemaLoader.standardCFMD(KEYSPACE, CF),
- SchemaLoader.standardCFMD(KEYSPACE, CF2),
- SchemaLoader.standardCFMD(KEYSPACE, CF3),
- SchemaLoader.counterCFMD(KEYSPACE, COUNTER_CF)
- .compression(SchemaLoader.getCompressionParameters(COMPRESSION_CHUNK_LENGTH)),
- SchemaLoader.standardCFMD(KEYSPACE, CF_UUID, 0, UUIDType.instance),
- SchemaLoader.keysIndexCFMD(KEYSPACE, CF_INDEX1, true),
- SchemaLoader.compositeIndexCFMD(KEYSPACE, CF_INDEX2, true),
- SchemaLoader.keysIndexCFMD(KEYSPACE, CF_INDEX1_BYTEORDERED, true).copy(ByteOrderedPartitioner.instance),
- SchemaLoader.compositeIndexCFMD(KEYSPACE, CF_INDEX2_BYTEORDERED, true).copy(ByteOrderedPartitioner.instance));
+ loadSchema();
+ createKeyspace(KEYSPACE,
+ KeyspaceParams.simple(1),
+ standardCFMD(KEYSPACE, CF),
+ standardCFMD(KEYSPACE, CF2),
+ standardCFMD(KEYSPACE, CF3),
+ counterCFMD(KEYSPACE, COUNTER_CF).compression(getCompressionParameters(COMPRESSION_CHUNK_LENGTH)),
+ standardCFMD(KEYSPACE, CF_UUID, 0, UUIDType.instance),
+ SchemaLoader.keysIndexCFMD(KEYSPACE, CF_INDEX1, true),
+ SchemaLoader.compositeIndexCFMD(KEYSPACE, CF_INDEX2, true),
+ SchemaLoader.keysIndexCFMD(KEYSPACE, CF_INDEX1_BYTEORDERED, true).partitioner(ByteOrderedPartitioner.instance),
+ SchemaLoader.compositeIndexCFMD(KEYSPACE, CF_INDEX2_BYTEORDERED, true).partitioner(ByteOrderedPartitioner.instance));
}
@Test
@@ -316,7 +319,7 @@
DatabaseDescriptor.setPartitionerUnsafe(new ByteOrderedPartitioner());
// Create out-of-order SSTable
- File tempDir = File.createTempFile("ScrubTest.testScrubOutOfOrder", "").getParentFile();
+ File tempDir = FileUtils.createTempFile("ScrubTest.testScrubOutOfOrder", "").getParentFile();
// create ks/cf directory
File tempDataDir = new File(tempDir, String.join(File.separator, KEYSPACE, CF3));
tempDataDir.mkdirs();
@@ -329,8 +332,7 @@
cfs.clearUnsafe();
List<String> keys = Arrays.asList("t", "a", "b", "z", "c", "y", "d");
- String filename = cfs.getSSTablePath(tempDataDir);
- Descriptor desc = Descriptor.fromFilename(filename);
+ Descriptor desc = cfs.newSSTableDescriptor(tempDataDir);
LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.WRITE);
try (SSTableTxnWriter writer = new SSTableTxnWriter(txn, createTestWriter(desc, (long) keys.size(), cfs.metadata, txn)))
@@ -338,7 +340,7 @@
for (String k : keys)
{
- PartitionUpdate update = UpdateBuilder.create(cfs.metadata, Util.dk(k))
+ PartitionUpdate update = UpdateBuilder.create(cfs.metadata(), Util.dk(k))
.newRow("someName").add("val", "someValue")
.build();
@@ -450,7 +452,7 @@
{
for (int i = 0; i < partitionsPerSSTable; i++)
{
- PartitionUpdate update = UpdateBuilder.create(cfs.metadata, String.valueOf(i))
+ PartitionUpdate update = UpdateBuilder.create(cfs.metadata(), String.valueOf(i))
.newRow("r1").add("val", "1")
.newRow("r1").add("val", "1")
.build();
@@ -466,7 +468,7 @@
assertTrue(values.length % 2 == 0);
for (int i = 0; i < values.length; i +=2)
{
- UpdateBuilder builder = UpdateBuilder.create(cfs.metadata, String.valueOf(i));
+ UpdateBuilder builder = UpdateBuilder.create(cfs.metadata(), String.valueOf(i));
if (composite)
{
builder.newRow("c" + i)
@@ -489,7 +491,7 @@
{
for (int i = 0; i < partitionsPerSSTable; i++)
{
- PartitionUpdate update = UpdateBuilder.create(cfs.metadata, String.valueOf(i))
+ PartitionUpdate update = UpdateBuilder.create(cfs.metadata(), String.valueOf(i))
.newRow("r1").add("val", 100L)
.build();
new CounterMutation(new Mutation(update), ConsistencyLevel.ONE).apply();
@@ -513,39 +515,12 @@
QueryProcessor.process("CREATE TABLE \"Keyspace1\".test_scrub_validation (a text primary key, b int)", ConsistencyLevel.ONE);
ColumnFamilyStore cfs2 = keyspace.getColumnFamilyStore("test_scrub_validation");
- new Mutation(UpdateBuilder.create(cfs2.metadata, "key").newRow().add("b", Int32Type.instance.decompose(1)).build()).apply();
+ new Mutation(UpdateBuilder.create(cfs2.metadata(), "key").newRow().add("b", LongType.instance.decompose(1L)).build()).apply();
cfs2.forceBlockingFlush();
CompactionManager.instance.performScrub(cfs2, false, false, 2);
}
- /**
- * For CASSANDRA-6892 too, check that for a compact table with one cluster column, we can insert whatever
- * we want as value for the clustering column, including something that would conflict with a CQL column definition.
- */
- @Test
- public void testValidationCompactStorage() throws Exception
- {
- QueryProcessor.process(String.format("CREATE TABLE \"%s\".test_compact_dynamic_columns (a int, b text, c text, PRIMARY KEY (a, b)) WITH COMPACT STORAGE", KEYSPACE), ConsistencyLevel.ONE);
-
- Keyspace keyspace = Keyspace.open(KEYSPACE);
- ColumnFamilyStore cfs = keyspace.getColumnFamilyStore("test_compact_dynamic_columns");
-
- QueryProcessor.executeInternal(String.format("INSERT INTO \"%s\".test_compact_dynamic_columns (a, b, c) VALUES (0, 'a', 'foo')", KEYSPACE));
- QueryProcessor.executeInternal(String.format("INSERT INTO \"%s\".test_compact_dynamic_columns (a, b, c) VALUES (0, 'b', 'bar')", KEYSPACE));
- QueryProcessor.executeInternal(String.format("INSERT INTO \"%s\".test_compact_dynamic_columns (a, b, c) VALUES (0, 'c', 'boo')", KEYSPACE));
- cfs.forceBlockingFlush();
- CompactionManager.instance.performScrub(cfs, true, true, 2);
-
- // Scrub is silent, but it will remove broken records. So reading everything back to make sure nothing to "scrubbed away"
- UntypedResultSet rs = QueryProcessor.executeInternal(String.format("SELECT * FROM \"%s\".test_compact_dynamic_columns", KEYSPACE));
- assertEquals(3, rs.size());
-
- Iterator<UntypedResultSet.Row> iter = rs.iterator();
- assertEquals("foo", iter.next().getString("c"));
- assertEquals("bar", iter.next().getString("c"));
- assertEquals("boo", iter.next().getString("c"));
- }
@Test /* CASSANDRA-5174 */
public void testScrubKeysIndex_preserveOrder() throws IOException, ExecutionException, InterruptedException
@@ -637,11 +612,11 @@
assertOrdered(Util.cmd(cfs).filterOn(colName, Operator.EQ, 1L).build(), numRows / 2);
}
- private static SSTableMultiWriter createTestWriter(Descriptor descriptor, long keyCount, CFMetaData metadata, LifecycleTransaction txn)
+ private static SSTableMultiWriter createTestWriter(Descriptor descriptor, long keyCount, TableMetadataRef metadata, LifecycleTransaction txn)
{
- SerializationHeader header = new SerializationHeader(true, metadata, metadata.partitionColumns(), EncodingStats.NO_STATS);
- MetadataCollector collector = new MetadataCollector(metadata.comparator).sstableLevel(0);
- return new TestMultiWriter(new TestWriter(descriptor, keyCount, 0, metadata, collector, header, txn), txn);
+ SerializationHeader header = new SerializationHeader(true, metadata.get(), metadata.get().regularAndStaticColumns(), EncodingStats.NO_STATS);
+ MetadataCollector collector = new MetadataCollector(metadata.get().comparator).sstableLevel(0);
+ return new TestMultiWriter(new TestWriter(descriptor, keyCount, 0, null, false, metadata, collector, header, txn), txn);
}
private static class TestMultiWriter extends SimpleSSTableMultiWriter
@@ -657,10 +632,10 @@
*/
private static class TestWriter extends BigTableWriter
{
- TestWriter(Descriptor descriptor, long keyCount, long repairedAt, CFMetaData metadata,
+ TestWriter(Descriptor descriptor, long keyCount, long repairedAt, UUID pendingRepair, boolean isTransient, TableMetadataRef metadata,
MetadataCollector collector, SerializationHeader header, LifecycleTransaction txn)
{
- super(descriptor, keyCount, repairedAt, metadata, collector, header, Collections.emptySet(), txn);
+ super(descriptor, keyCount, repairedAt, pendingRepair, isTransient, metadata, collector, header, Collections.emptySet(), txn);
}
@Override
@@ -711,41 +686,4 @@
rs = QueryProcessor.executeInternal(String.format("SELECT * FROM \"%s\".cf_with_duplicates_3_0", KEYSPACE));
assertEquals(0, rs.size());
}
-
- @Test
- public void testUpgradeSstablesWithDuplicates() throws Exception
- {
- DatabaseDescriptor.setPartitionerUnsafe(Murmur3Partitioner.instance);
- String cf = "cf_with_duplicates_2_0";
- QueryProcessor.process(String.format("CREATE TABLE \"%s\".%s (a int, b int, c int, PRIMARY KEY (a, b))", KEYSPACE, cf), ConsistencyLevel.ONE);
-
- Keyspace keyspace = Keyspace.open(KEYSPACE);
- ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cf);
-
- Path legacySSTableRoot = Paths.get(System.getProperty(INVALID_LEGACY_SSTABLE_ROOT_PROP),
- "Keyspace1",
- cf);
-
- for (String filename : new String[]{ "lb-1-big-CompressionInfo.db",
- "lb-1-big-Data.db",
- "lb-1-big-Digest.adler32",
- "lb-1-big-Filter.db",
- "lb-1-big-Index.db",
- "lb-1-big-Statistics.db",
- "lb-1-big-Summary.db",
- "lb-1-big-TOC.txt" })
- {
- Files.copy(Paths.get(legacySSTableRoot.toString(), filename), cfs.getDirectories().getDirectoryForNewSSTables().toPath().resolve(filename));
- }
-
- cfs.loadNewSSTables();
-
- cfs.sstablesRewrite(true, 1);
-
- UntypedResultSet rs = QueryProcessor.executeInternal(String.format("SELECT * FROM \"%s\".%s", KEYSPACE, cf));
- assertEquals(1, rs.size());
- QueryProcessor.executeInternal(String.format("DELETE FROM \"%s\".%s WHERE a=1 AND b =2", KEYSPACE, cf));
- rs = QueryProcessor.executeInternal(String.format("SELECT * FROM \"%s\".%s", KEYSPACE, cf));
- assertEquals(0, rs.size());
- }
}
diff --git a/test/unit/org/apache/cassandra/db/SecondaryIndexTest.java b/test/unit/org/apache/cassandra/db/SecondaryIndexTest.java
index e9a0db6..bfe1c6d 100644
--- a/test/unit/org/apache/cassandra/db/SecondaryIndexTest.java
+++ b/test/unit/org/apache/cassandra/db/SecondaryIndexTest.java
@@ -30,9 +30,9 @@
import org.junit.Test;
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.Util;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.cql3.Operator;
-import org.apache.cassandra.cql3.statements.IndexTarget;
+import org.apache.cassandra.cql3.statements.schema.IndexTarget;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.db.partitions.*;
import org.apache.cassandra.db.rows.Row;
@@ -40,6 +40,8 @@
import org.apache.cassandra.index.Index;
import org.apache.cassandra.schema.IndexMetadata;
import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.schema.MigrationManager;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FBUtilities;
@@ -84,10 +86,10 @@
{
ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(WITH_COMPOSITE_INDEX);
- new RowUpdateBuilder(cfs.metadata, 0, "k1").clustering("c").add("birthdate", 1L).add("notbirthdate", 1L).build().applyUnsafe();
- new RowUpdateBuilder(cfs.metadata, 0, "k2").clustering("c").add("birthdate", 2L).add("notbirthdate", 2L).build().applyUnsafe();
- new RowUpdateBuilder(cfs.metadata, 0, "k3").clustering("c").add("birthdate", 1L).add("notbirthdate", 2L).build().applyUnsafe();
- new RowUpdateBuilder(cfs.metadata, 0, "k4").clustering("c").add("birthdate", 3L).add("notbirthdate", 2L).build().applyUnsafe();
+ new RowUpdateBuilder(cfs.metadata(), 0, "k1").clustering("c").add("birthdate", 1L).add("notbirthdate", 1L).build().applyUnsafe();
+ new RowUpdateBuilder(cfs.metadata(), 0, "k2").clustering("c").add("birthdate", 2L).add("notbirthdate", 2L).build().applyUnsafe();
+ new RowUpdateBuilder(cfs.metadata(), 0, "k3").clustering("c").add("birthdate", 1L).add("notbirthdate", 2L).build().applyUnsafe();
+ new RowUpdateBuilder(cfs.metadata(), 0, "k4").clustering("c").add("birthdate", 3L).add("notbirthdate", 2L).build().applyUnsafe();
// basic single-expression query
List<FilteredPartition> partitions = Util.getAll(Util.cmd(cfs).fromKeyExcl("k1").toKeyIncl("k3").columns("birthdate").build());
@@ -160,7 +162,7 @@
for (int i = 0; i < 100; i++)
{
- new RowUpdateBuilder(cfs.metadata, FBUtilities.timestampMicros(), "key" + i)
+ new RowUpdateBuilder(cfs.metadata(), FBUtilities.timestampMicros(), "key" + i)
.clustering("c")
.add("birthdate", 34L)
.add("notbirthdate", ByteBufferUtil.bytes((long) (i % 2)))
@@ -192,15 +194,15 @@
{
ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(WITH_COMPOSITE_INDEX);
ByteBuffer bBB = ByteBufferUtil.bytes("birthdate");
- ColumnDefinition bDef = cfs.metadata.getColumnDefinition(bBB);
+ ColumnMetadata bDef = cfs.metadata().getColumn(bBB);
ByteBuffer col = ByteBufferUtil.bytes("birthdate");
// Confirm addition works
- new RowUpdateBuilder(cfs.metadata, 0, "k1").clustering("c").add("birthdate", 1L).build().applyUnsafe();
+ new RowUpdateBuilder(cfs.metadata(), 0, "k1").clustering("c").add("birthdate", 1L).build().applyUnsafe();
assertIndexedOne(cfs, col, 1L);
// delete the column directly
- RowUpdateBuilder.deleteRow(cfs.metadata, 1, "k1", "c").applyUnsafe();
+ RowUpdateBuilder.deleteRow(cfs.metadata(), 1, "k1", "c").applyUnsafe();
assertIndexedNone(cfs, col, 1L);
// verify that it's not being indexed under any other value either
@@ -208,26 +210,26 @@
assertNull(rc.getIndex(cfs));
// resurrect w/ a newer timestamp
- new RowUpdateBuilder(cfs.metadata, 2, "k1").clustering("c").add("birthdate", 1L).build().apply();;
+ new RowUpdateBuilder(cfs.metadata(), 2, "k1").clustering("c").add("birthdate", 1L).build().apply();;
assertIndexedOne(cfs, col, 1L);
// verify that row and delete w/ older timestamp does nothing
- RowUpdateBuilder.deleteRow(cfs.metadata, 1, "k1", "c").applyUnsafe();
+ RowUpdateBuilder.deleteRow(cfs.metadata(), 1, "k1", "c").applyUnsafe();
assertIndexedOne(cfs, col, 1L);
// similarly, column delete w/ older timestamp should do nothing
- new RowUpdateBuilder(cfs.metadata, 1, "k1").clustering("c").delete(bDef).build().applyUnsafe();
+ new RowUpdateBuilder(cfs.metadata(), 1, "k1").clustering("c").delete(bDef).build().applyUnsafe();
assertIndexedOne(cfs, col, 1L);
// delete the entire row (w/ newer timestamp this time)
// todo - checking the # of index searchers for the command is probably not the best thing to test here
- RowUpdateBuilder.deleteRow(cfs.metadata, 3, "k1", "c").applyUnsafe();
+ RowUpdateBuilder.deleteRow(cfs.metadata(), 3, "k1", "c").applyUnsafe();
rc = Util.cmd(cfs).build();
assertNull(rc.getIndex(cfs));
// make sure obsolete mutations don't generate an index entry
// todo - checking the # of index searchers for the command is probably not the best thing to test here
- new RowUpdateBuilder(cfs.metadata, 3, "k1").clustering("c").add("birthdate", 1L).build().apply();;
+ new RowUpdateBuilder(cfs.metadata(), 3, "k1").clustering("c").add("birthdate", 1L).build().apply();;
rc = Util.cmd(cfs).build();
assertNull(rc.getIndex(cfs));
}
@@ -240,8 +242,8 @@
ByteBuffer col = ByteBufferUtil.bytes("birthdate");
// create a row and update the birthdate value, test that the index query fetches the new version
- new RowUpdateBuilder(cfs.metadata, 1, "testIndexUpdate").clustering("c").add("birthdate", 100L).build().applyUnsafe();
- new RowUpdateBuilder(cfs.metadata, 2, "testIndexUpdate").clustering("c").add("birthdate", 200L).build().applyUnsafe();
+ new RowUpdateBuilder(cfs.metadata(), 1, "testIndexUpdate").clustering("c").add("birthdate", 100L).build().applyUnsafe();
+ new RowUpdateBuilder(cfs.metadata(), 2, "testIndexUpdate").clustering("c").add("birthdate", 200L).build().applyUnsafe();
// Confirm old version fetch fails
assertIndexedNone(cfs, col, 100L);
@@ -262,23 +264,23 @@
ByteBuffer col = ByteBufferUtil.bytes("birthdate");
// create a row and update the birthdate value with an expiring column
- new RowUpdateBuilder(cfs.metadata, 1L, 500, "K100").clustering("c").add("birthdate", 100L).build().applyUnsafe();
+ new RowUpdateBuilder(cfs.metadata(), 1L, 500, "K100").clustering("c").add("birthdate", 100L).build().applyUnsafe();
assertIndexedOne(cfs, col, 100L);
// requires a 1s sleep because we calculate local expiry time as (now() / 1000) + ttl
TimeUnit.SECONDS.sleep(1);
// now overwrite with the same name/value/ttl, but the local expiry time will be different
- new RowUpdateBuilder(cfs.metadata, 1L, 500, "K100").clustering("c").add("birthdate", 100L).build().applyUnsafe();
+ new RowUpdateBuilder(cfs.metadata(), 1L, 500, "K100").clustering("c").add("birthdate", 100L).build().applyUnsafe();
assertIndexedOne(cfs, col, 100L);
// check that modifying the indexed value using the same timestamp behaves as expected
- new RowUpdateBuilder(cfs.metadata, 1L, 500, "K101").clustering("c").add("birthdate", 101L).build().applyUnsafe();
+ new RowUpdateBuilder(cfs.metadata(), 1L, 500, "K101").clustering("c").add("birthdate", 101L).build().applyUnsafe();
assertIndexedOne(cfs, col, 101L);
TimeUnit.SECONDS.sleep(1);
- new RowUpdateBuilder(cfs.metadata, 1L, 500, "K101").clustering("c").add("birthdate", 102L).build().applyUnsafe();
+ new RowUpdateBuilder(cfs.metadata(), 1L, 500, "K101").clustering("c").add("birthdate", 102L).build().applyUnsafe();
// Confirm 101 is gone
assertIndexedNone(cfs, col, 101L);
@@ -295,13 +297,13 @@
ByteBuffer col = ByteBufferUtil.bytes("birthdate");
// create a row and update the "birthdate" value
- new RowUpdateBuilder(cfs.metadata, 1, "k1").noRowMarker().add("birthdate", 1L).build().applyUnsafe();
+ new RowUpdateBuilder(cfs.metadata(), 1, "k1").noRowMarker().add("birthdate", 1L).build().applyUnsafe();
// force a flush, so our index isn't being read from a memtable
keyspace.getColumnFamilyStore(WITH_KEYS_INDEX).forceBlockingFlush();
// now apply another update, but force the index update to be skipped
- keyspace.apply(new RowUpdateBuilder(cfs.metadata, 2, "k1").noRowMarker().add("birthdate", 2L).build(),
+ keyspace.apply(new RowUpdateBuilder(cfs.metadata(), 2, "k1").noRowMarker().add("birthdate", 2L).build(),
true,
false);
@@ -314,7 +316,7 @@
// now, reset back to the original value, still skipping the index update, to
// make sure the value was expunged from the index when it was discovered to be inconsistent
- keyspace.apply(new RowUpdateBuilder(cfs.metadata, 3, "k1").noRowMarker().add("birthdate", 1L).build(),
+ keyspace.apply(new RowUpdateBuilder(cfs.metadata(), 3, "k1").noRowMarker().add("birthdate", 1L).build(),
true,
false);
assertIndexedNone(cfs, col, 1L);
@@ -345,7 +347,7 @@
ByteBuffer col = ByteBufferUtil.bytes(colName);
// create a row and update the author value
- RowUpdateBuilder builder = new RowUpdateBuilder(cfs.metadata, 0, "k1");
+ RowUpdateBuilder builder = new RowUpdateBuilder(cfs.metadata(), 0, "k1");
if (!isStatic)
builder = builder.clustering("c");
builder.add(colName, 10l).build().applyUnsafe();
@@ -358,7 +360,7 @@
assertIndexedOne(cfs, col, 10l);
// now apply another update, but force the index update to be skipped
- builder = new RowUpdateBuilder(cfs.metadata, 0, "k1");
+ builder = new RowUpdateBuilder(cfs.metadata(), 0, "k1");
if (!isStatic)
builder = builder.clustering("c");
builder.add(colName, 20l);
@@ -374,7 +376,7 @@
// now, reset back to the original value, still skipping the index update, to
// make sure the value was expunged from the index when it was discovered to be inconsistent
// TODO: Figure out why this is re-inserting
- builder = new RowUpdateBuilder(cfs.metadata, 2, "k1");
+ builder = new RowUpdateBuilder(cfs.metadata(), 2, "k1");
if (!isStatic)
builder = builder.clustering("c");
builder.add(colName, 10L);
@@ -394,10 +396,10 @@
ByteBuffer colName = ByteBufferUtil.bytes("birthdate");
// Insert indexed value.
- new RowUpdateBuilder(cfs.metadata, 1, "k1").clustering("c").add("birthdate", 10l).build().applyUnsafe();
+ new RowUpdateBuilder(cfs.metadata(), 1, "k1").clustering("c").add("birthdate", 10l).build().applyUnsafe();
// Now delete the value
- RowUpdateBuilder.deleteRow(cfs.metadata, 2, "k1", "c").applyUnsafe();
+ RowUpdateBuilder.deleteRow(cfs.metadata(), 2, "k1", "c").applyUnsafe();
// We want the data to be gcable, but even if gcGrace == 0, we still need to wait 1 second
// since we won't gc on a tie.
@@ -417,10 +419,10 @@
ByteBuffer colName = ByteBufferUtil.bytes("birthdate");
// Insert indexed value.
- new RowUpdateBuilder(cfs.metadata, 1, "k1").add("birthdate", 10l).build().applyUnsafe();
+ new RowUpdateBuilder(cfs.metadata(), 1, "k1").add("birthdate", 10l).build().applyUnsafe();
// Now delete the value
- RowUpdateBuilder.deleteRow(cfs.metadata, 2, "k1").applyUnsafe();
+ RowUpdateBuilder.deleteRow(cfs.metadata(), 2, "k1").applyUnsafe();
// We want the data to be gcable, but even if gcGrace == 0, we still need to wait 1 second
// since we won't gc on a tie.
@@ -439,14 +441,14 @@
ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(WITH_COMPOSITE_INDEX);
Mutation rm;
- new RowUpdateBuilder(cfs.metadata, 0, "kk1").clustering("c").add("birthdate", 1L).build().applyUnsafe();
- new RowUpdateBuilder(cfs.metadata, 0, "kk1").clustering("c").add("notbirthdate", 1L).build().applyUnsafe();
- new RowUpdateBuilder(cfs.metadata, 0, "kk2").clustering("c").add("birthdate", 1L).build().applyUnsafe();
- new RowUpdateBuilder(cfs.metadata, 0, "kk2").clustering("c").add("notbirthdate", 2L).build().applyUnsafe();
- new RowUpdateBuilder(cfs.metadata, 0, "kk3").clustering("c").add("birthdate", 1L).build().applyUnsafe();
- new RowUpdateBuilder(cfs.metadata, 0, "kk3").clustering("c").add("notbirthdate", 2L).build().applyUnsafe();
- new RowUpdateBuilder(cfs.metadata, 0, "kk4").clustering("c").add("birthdate", 1L).build().applyUnsafe();
- new RowUpdateBuilder(cfs.metadata, 0, "kk4").clustering("c").add("notbirthdate", 2L).build().applyUnsafe();
+ new RowUpdateBuilder(cfs.metadata(), 0, "kk1").clustering("c").add("birthdate", 1L).build().applyUnsafe();
+ new RowUpdateBuilder(cfs.metadata(), 0, "kk1").clustering("c").add("notbirthdate", 1L).build().applyUnsafe();
+ new RowUpdateBuilder(cfs.metadata(), 0, "kk2").clustering("c").add("birthdate", 1L).build().applyUnsafe();
+ new RowUpdateBuilder(cfs.metadata(), 0, "kk2").clustering("c").add("notbirthdate", 2L).build().applyUnsafe();
+ new RowUpdateBuilder(cfs.metadata(), 0, "kk3").clustering("c").add("birthdate", 1L).build().applyUnsafe();
+ new RowUpdateBuilder(cfs.metadata(), 0, "kk3").clustering("c").add("notbirthdate", 2L).build().applyUnsafe();
+ new RowUpdateBuilder(cfs.metadata(), 0, "kk4").clustering("c").add("birthdate", 1L).build().applyUnsafe();
+ new RowUpdateBuilder(cfs.metadata(), 0, "kk4").clustering("c").add("notbirthdate", 2L).build().applyUnsafe();
// basic single-expression query, limit 1
Util.getOnlyRow(Util.cmd(cfs)
@@ -463,28 +465,39 @@
ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(COMPOSITE_INDEX_TO_BE_ADDED);
// create a row and update the birthdate value, test that the index query fetches the new version
- new RowUpdateBuilder(cfs.metadata, 0, "k1").clustering("c").add("birthdate", 1L).build().applyUnsafe();
+ new RowUpdateBuilder(cfs.metadata(), 0, "k1").clustering("c").add("birthdate", 1L).build().applyUnsafe();
String indexName = "birthdate_index";
- ColumnDefinition old = cfs.metadata.getColumnDefinition(ByteBufferUtil.bytes("birthdate"));
+ ColumnMetadata old = cfs.metadata().getColumn(ByteBufferUtil.bytes("birthdate"));
IndexMetadata indexDef =
- IndexMetadata.fromIndexTargets(cfs.metadata,
- Collections.singletonList(new IndexTarget(old.name, IndexTarget.Type.VALUES)),
+ IndexMetadata.fromIndexTargets(
+ Collections.singletonList(new IndexTarget(old.name, IndexTarget.Type.VALUES)),
indexName,
IndexMetadata.Kind.COMPOSITES,
Collections.EMPTY_MAP);
- cfs.metadata.indexes(cfs.metadata.getIndexes().with(indexDef));
- Future<?> future = cfs.indexManager.addIndex(indexDef);
- future.get();
+
+ TableMetadata current = cfs.metadata();
+
+ TableMetadata updated =
+ current.unbuild()
+ .indexes(current.indexes.with(indexDef))
+ .build();
+ MigrationManager.announceTableUpdate(updated, true);
+
+ // wait for the index to be built
+ Index index = cfs.indexManager.getIndex(indexDef);
+ do
+ {
+ TimeUnit.MILLISECONDS.sleep(100);
+ }
+ while (!cfs.indexManager.isIndexQueryable(index));
// we had a bug (CASSANDRA-2244) where index would get created but not flushed -- check for that
// the way we find the index cfs is a bit convoluted at the moment
- boolean flushed = false;
ColumnFamilyStore indexCfs = cfs.indexManager.getIndex(indexDef)
.getBackingTable()
.orElseThrow(throwAssert("Index not found"));
- flushed = !indexCfs.getLiveSSTables().isEmpty();
- assertTrue(flushed);
+ assertFalse(indexCfs.getLiveSSTables().isEmpty());
assertIndexedOne(cfs, ByteBufferUtil.bytes("birthdate"), 1L);
// validate that drop clears it out & rebuild works (CASSANDRA-2320)
@@ -493,7 +506,7 @@
assertFalse(cfs.getBuiltIndexes().contains(indexName));
// rebuild & re-query
- future = cfs.indexManager.addIndex(indexDef);
+ Future future = cfs.indexManager.addIndex(indexDef, false);
future.get();
assertIndexedOne(cfs, ByteBufferUtil.bytes("birthdate"), 1L);
}
@@ -506,7 +519,7 @@
ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(WITH_KEYS_INDEX);
for (int i = 0; i < 10; i++)
- new RowUpdateBuilder(cfs.metadata, 0, "k" + i).noRowMarker().add("birthdate", 1l).build().applyUnsafe();
+ new RowUpdateBuilder(cfs.metadata(), 0, "k" + i).noRowMarker().add("birthdate", 1l).build().applyUnsafe();
assertIndexedCount(cfs, ByteBufferUtil.bytes("birthdate"), 1l, 10);
cfs.forceBlockingFlush();
@@ -519,10 +532,10 @@
ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(WITH_MULTIPLE_COMPOSITE_INDEX);
// creates rows such that birthday_index has 1 partition (key = 1L) with 4 rows -- mean row count = 4, and notbirthdate_index has 2 partitions with 2 rows each -- mean row count = 2
- new RowUpdateBuilder(cfs.metadata, 0, "k1").clustering("c").add("birthdate", 1L).add("notbirthdate", 2L).build().applyUnsafe();
- new RowUpdateBuilder(cfs.metadata, 0, "k2").clustering("c").add("birthdate", 1L).add("notbirthdate", 2L).build().applyUnsafe();
- new RowUpdateBuilder(cfs.metadata, 0, "k3").clustering("c").add("birthdate", 1L).add("notbirthdate", 3L).build().applyUnsafe();
- new RowUpdateBuilder(cfs.metadata, 0, "k4").clustering("c").add("birthdate", 1L).add("notbirthdate", 3L).build().applyUnsafe();
+ new RowUpdateBuilder(cfs.metadata(), 0, "k1").clustering("c").add("birthdate", 1L).add("notbirthdate", 2L).build().applyUnsafe();
+ new RowUpdateBuilder(cfs.metadata(), 0, "k2").clustering("c").add("birthdate", 1L).add("notbirthdate", 2L).build().applyUnsafe();
+ new RowUpdateBuilder(cfs.metadata(), 0, "k3").clustering("c").add("birthdate", 1L).add("notbirthdate", 3L).build().applyUnsafe();
+ new RowUpdateBuilder(cfs.metadata(), 0, "k4").clustering("c").add("birthdate", 1L).add("notbirthdate", 3L).build().applyUnsafe();
cfs.forceBlockingFlush();
ReadCommand rc = Util.cmd(cfs)
@@ -546,7 +559,7 @@
}
private void assertIndexedCount(ColumnFamilyStore cfs, ByteBuffer col, Object val, int count)
{
- ColumnDefinition cdef = cfs.metadata.getColumnDefinition(col);
+ ColumnMetadata cdef = cfs.metadata().getColumn(col);
ReadCommand rc = Util.cmd(cfs).filterOn(cdef.name.toString(), Operator.EQ, ((AbstractType) cdef.cellValueType()).decompose(val)).build();
Index.Searcher searcher = rc.getIndex(cfs).searcherFor(rc);
diff --git a/test/unit/org/apache/cassandra/db/SerializationHeaderTest.java b/test/unit/org/apache/cassandra/db/SerializationHeaderTest.java
index 84fb51c..1092a90 100644
--- a/test/unit/org/apache/cassandra/db/SerializationHeaderTest.java
+++ b/test/unit/org/apache/cassandra/db/SerializationHeaderTest.java
@@ -19,8 +19,6 @@
package org.apache.cassandra.db;
import com.google.common.io.Files;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.ColumnIdentifier;
import org.apache.cassandra.db.compaction.OperationType;
@@ -41,6 +39,9 @@
import org.apache.cassandra.io.sstable.format.big.BigFormat;
import org.apache.cassandra.io.sstable.format.big.BigTableWriter;
import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.schema.TableMetadataRef;
import org.junit.Assert;
import org.junit.Test;
@@ -65,28 +66,36 @@
public void testWrittenAsDifferentKind() throws Exception
{
final String tableName = "testWrittenAsDifferentKind";
- final String schemaCqlWithStatic = String.format("CREATE TABLE %s (k int, c int, v int static, PRIMARY KEY(k, c))", tableName);
- final String schemaCqlWithRegular = String.format("CREATE TABLE %s (k int, c int, v int, PRIMARY KEY(k, c))", tableName);
+// final String schemaCqlWithStatic = String.format("CREATE TABLE %s (k int, c int, v int static, PRIMARY KEY(k, c))", tableName);
+// final String schemaCqlWithRegular = String.format("CREATE TABLE %s (k int, c int, v int, PRIMARY KEY(k, c))", tableName);
ColumnIdentifier v = ColumnIdentifier.getInterned("v", false);
- CFMetaData schemaWithStatic = CFMetaData.compile(schemaCqlWithStatic, KEYSPACE);
- CFMetaData schemaWithRegular = CFMetaData.compile(schemaCqlWithRegular, KEYSPACE);
- ColumnDefinition columnStatic = schemaWithStatic.getColumnDefinition(v);
- ColumnDefinition columnRegular = schemaWithRegular.getColumnDefinition(v);
- schemaWithStatic.recordColumnDrop(columnRegular, 0L);
- schemaWithRegular.recordColumnDrop(columnStatic, 0L);
+ TableMetadata schemaWithStatic = TableMetadata.builder(KEYSPACE, tableName)
+ .addPartitionKeyColumn("k", Int32Type.instance)
+ .addClusteringColumn("c", Int32Type.instance)
+ .addStaticColumn("v", Int32Type.instance)
+ .build();
+ TableMetadata schemaWithRegular = TableMetadata.builder(KEYSPACE, tableName)
+ .addPartitionKeyColumn("k", Int32Type.instance)
+ .addClusteringColumn("c", Int32Type.instance)
+ .addRegularColumn("v", Int32Type.instance)
+ .build();
+ ColumnMetadata columnStatic = schemaWithStatic.getColumn(v);
+ ColumnMetadata columnRegular = schemaWithRegular.getColumn(v);
+ schemaWithStatic = schemaWithStatic.unbuild().recordColumnDrop(columnRegular, 0L).build();
+ schemaWithRegular = schemaWithRegular.unbuild().recordColumnDrop(columnStatic, 0L).build();
final AtomicInteger generation = new AtomicInteger();
File dir = Files.createTempDir();
try
{
- BiFunction<CFMetaData, Function<ByteBuffer, Clustering>, Callable<Descriptor>> writer = (schema, clusteringFunction) -> () -> {
- Descriptor descriptor = new Descriptor(BigFormat.latestVersion, dir, schema.ksName, schema.cfName, generation.incrementAndGet(), SSTableFormat.Type.BIG, Component.DIGEST_CRC32);
+ BiFunction<TableMetadata, Function<ByteBuffer, Clustering>, Callable<Descriptor>> writer = (schema, clusteringFunction) -> () -> {
+ Descriptor descriptor = new Descriptor(BigFormat.latestVersion, dir, schema.keyspace, schema.name, generation.incrementAndGet(), SSTableFormat.Type.BIG);
SerializationHeader header = SerializationHeader.makeWithoutStats(schema);
try (LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.WRITE);
- SSTableWriter sstableWriter = BigTableWriter.create(schema, descriptor, 1, 0L, 0, header, Collections.emptyList(), txn))
+ SSTableWriter sstableWriter = BigTableWriter.create(TableMetadataRef.forOfflineTools(schema), descriptor, 1, 0L, null, false, 0, header, Collections.emptyList(), txn))
{
- ColumnDefinition cd = schema.getColumnDefinition(v);
+ ColumnMetadata cd = schema.getColumn(v);
for (int i = 0 ; i < 5 ; ++i) {
final ByteBuffer value = Int32Type.instance.decompose(i);
Cell cell = BufferCell.live(cd, 1L, value);
@@ -102,8 +111,8 @@
Descriptor sstableWithRegular = writer.apply(schemaWithRegular, BufferClustering::new).call();
Descriptor sstableWithStatic = writer.apply(schemaWithStatic, value -> Clustering.STATIC_CLUSTERING).call();
- SSTableReader readerWithStatic = SSTableReader.openNoValidation(sstableWithStatic, schemaWithRegular);
- SSTableReader readerWithRegular = SSTableReader.openNoValidation(sstableWithRegular, schemaWithStatic);
+ SSTableReader readerWithStatic = SSTableReader.openNoValidation(sstableWithStatic, TableMetadataRef.forOfflineTools(schemaWithRegular));
+ SSTableReader readerWithRegular = SSTableReader.openNoValidation(sstableWithRegular, TableMetadataRef.forOfflineTools(schemaWithStatic));
try (ISSTableScanner partitions = readerWithStatic.getScanner()) {
for (int i = 0 ; i < 5 ; ++i)
diff --git a/test/unit/org/apache/cassandra/db/SinglePartitionSliceCommandTest.java b/test/unit/org/apache/cassandra/db/SinglePartitionSliceCommandTest.java
index f5a8cc8..67fd314 100644
--- a/test/unit/org/apache/cassandra/db/SinglePartitionSliceCommandTest.java
+++ b/test/unit/org/apache/cassandra/db/SinglePartitionSliceCommandTest.java
@@ -40,12 +40,12 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.SchemaLoader;
-import org.apache.cassandra.Util;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.config.Schema;
+import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.cql3.ColumnIdentifier;
+import org.apache.cassandra.Util;
import org.apache.cassandra.cql3.QueryOptions;
import org.apache.cassandra.cql3.QueryProcessor;
import org.apache.cassandra.cql3.UntypedResultSet;
@@ -84,40 +84,37 @@
private static final String KEYSPACE = "ks";
private static final String TABLE = "tbl";
- private static CFMetaData cfm;
- private static ColumnDefinition v;
- private static ColumnDefinition s;
+ private static TableMetadata metadata;
+ private static ColumnMetadata v;
+ private static ColumnMetadata s;
private static final String TABLE_SCLICES = "tbl_slices";
- private static CFMetaData CFM_SLICES;
+ private static TableMetadata CFM_SLICES;
@BeforeClass
public static void defineSchema() throws ConfigurationException
{
DatabaseDescriptor.daemonInitialization();
- cfm = CFMetaData.Builder.create(KEYSPACE, TABLE)
- .addPartitionKey("k", UTF8Type.instance)
- .addStaticColumn("s", UTF8Type.instance)
- .addClusteringColumn("i", IntegerType.instance)
- .addRegularColumn("v", UTF8Type.instance)
- .build();
+ metadata =
+ TableMetadata.builder(KEYSPACE, TABLE)
+ .addPartitionKeyColumn("k", UTF8Type.instance)
+ .addStaticColumn("s", UTF8Type.instance)
+ .addClusteringColumn("i", IntegerType.instance)
+ .addRegularColumn("v", UTF8Type.instance)
+ .build();
- CFM_SLICES = CFMetaData.Builder.create(KEYSPACE, TABLE_SCLICES)
- .addPartitionKey("k", UTF8Type.instance)
- .addClusteringColumn("c1", Int32Type.instance)
- .addClusteringColumn("c2", Int32Type.instance)
- .addRegularColumn("v", IntegerType.instance)
- .build();
+ CFM_SLICES = TableMetadata.builder(KEYSPACE, TABLE_SCLICES)
+ .addPartitionKeyColumn("k", UTF8Type.instance)
+ .addClusteringColumn("c1", Int32Type.instance)
+ .addClusteringColumn("c2", Int32Type.instance)
+ .addRegularColumn("v", IntegerType.instance)
+ .build();
SchemaLoader.prepareServer();
- SchemaLoader.createKeyspace(KEYSPACE, KeyspaceParams.simple(1), cfm, CFM_SLICES);
-
- cfm = Schema.instance.getCFMetaData(KEYSPACE, TABLE);
- v = cfm.getColumnDefinition(new ColumnIdentifier("v", true));
- s = cfm.getColumnDefinition(new ColumnIdentifier("s", true));
-
- CFM_SLICES = Schema.instance.getCFMetaData(KEYSPACE, TABLE_SCLICES);
+ SchemaLoader.createKeyspace(KEYSPACE, KeyspaceParams.simple(1), metadata, CFM_SLICES);
+ v = metadata.getColumn(new ColumnIdentifier("v", true));
+ s = metadata.getColumn(new ColumnIdentifier("s", true));
}
@Before
@@ -128,62 +125,6 @@
}
@Test
- public void staticColumnsAreFiltered() throws IOException
- {
- DecoratedKey key = cfm.decorateKey(ByteBufferUtil.bytes("k"));
-
- UntypedResultSet rows;
-
- QueryProcessor.executeInternal("INSERT INTO ks.tbl (k, s, i, v) VALUES ('k', 's', 0, 'v')");
- QueryProcessor.executeInternal("DELETE v FROM ks.tbl WHERE k='k' AND i=0");
- QueryProcessor.executeInternal("DELETE FROM ks.tbl WHERE k='k' AND i=0");
- rows = QueryProcessor.executeInternal("SELECT * FROM ks.tbl WHERE k='k' AND i=0");
-
- for (UntypedResultSet.Row row: rows)
- {
- logger.debug("Current: k={}, s={}, v={}", (row.has("k") ? row.getString("k") : null), (row.has("s") ? row.getString("s") : null), (row.has("v") ? row.getString("v") : null));
- }
-
- assert rows.isEmpty();
-
- ColumnFilter columnFilter = ColumnFilter.selection(PartitionColumns.of(v));
- ByteBuffer zero = ByteBufferUtil.bytes(0);
- Slices slices = Slices.with(cfm.comparator, Slice.make(ClusteringBound.inclusiveStartOf(zero), ClusteringBound.inclusiveEndOf(zero)));
- ClusteringIndexSliceFilter sliceFilter = new ClusteringIndexSliceFilter(slices, false);
- ReadCommand cmd = SinglePartitionReadCommand.create(true,
- cfm,
- FBUtilities.nowInSeconds(),
- columnFilter,
- RowFilter.NONE,
- DataLimits.NONE,
- key,
- sliceFilter);
-
- DataOutputBuffer out = new DataOutputBuffer((int) ReadCommand.legacyReadCommandSerializer.serializedSize(cmd, MessagingService.VERSION_21));
- ReadCommand.legacyReadCommandSerializer.serialize(cmd, out, MessagingService.VERSION_21);
- DataInputPlus in = new DataInputBuffer(out.buffer(), true);
- cmd = ReadCommand.legacyReadCommandSerializer.deserialize(in, MessagingService.VERSION_21);
-
- logger.debug("ReadCommand: {}", cmd);
- try (ReadExecutionController controller = cmd.executionController();
- UnfilteredPartitionIterator partitionIterator = cmd.executeLocally(controller))
- {
- ReadResponse response = ReadResponse.createDataResponse(partitionIterator, cmd);
-
- logger.debug("creating response: {}", response);
- try (UnfilteredPartitionIterator pIter = response.makeIterator(cmd))
- {
- assert pIter.hasNext();
- try (UnfilteredRowIterator partition = pIter.next())
- {
- LegacyLayout.LegacyUnfilteredPartition rowIter = LegacyLayout.fromUnfilteredRowIterator(cmd, partition);
- Assert.assertEquals(Collections.emptyList(), rowIter.cells);
- }
- }
- }
- }
-
- @Test
public void testMultiNamesCommandWithFlush()
{
testMultiNamesOrSlicesCommand(true, false);
@@ -235,7 +176,7 @@
int uniqueCk1 = 2;
int uniqueCk2 = 3;
- DecoratedKey key = CFM_SLICES.decorateKey(ByteBufferUtil.bytes("k"));
+ DecoratedKey key = Util.dk(ByteBufferUtil.bytes("k"));
QueryProcessor.executeInternal(String.format("DELETE FROM ks.tbl_slices USING TIMESTAMP %d WHERE k='k' AND c1=%d",
deletionTime,
ck1));
@@ -308,7 +249,7 @@
Assert.assertTrue(ri.columns().contains(s));
Row staticRow = ri.staticRow();
Iterator<Cell> cellIterator = staticRow.cells().iterator();
- Assert.assertTrue(staticRow.toString(cfm, true), cellIterator.hasNext());
+ Assert.assertTrue(staticRow.toString(metadata, true), cellIterator.hasNext());
Cell cell = cellIterator.next();
Assert.assertEquals(s, cell.column());
Assert.assertEquals(ByteBufferUtil.bytesToHex(cell.value()), ByteBufferUtil.bytes("s"), cell.value());
@@ -318,15 +259,14 @@
@Test
public void staticColumnsAreReturned() throws IOException
{
- DecoratedKey key = cfm.decorateKey(ByteBufferUtil.bytes("k1"));
+ DecoratedKey key = metadata.partitioner.decorateKey(ByteBufferUtil.bytes("k1"));
QueryProcessor.executeInternal("INSERT INTO ks.tbl (k, s) VALUES ('k1', 's')");
Assert.assertFalse(QueryProcessor.executeInternal("SELECT s FROM ks.tbl WHERE k='k1'").isEmpty());
- ColumnFilter columnFilter = ColumnFilter.selection(PartitionColumns.of(s));
+ ColumnFilter columnFilter = ColumnFilter.selection(RegularAndStaticColumns.of(s));
ClusteringIndexSliceFilter sliceFilter = new ClusteringIndexSliceFilter(Slices.NONE, false);
- ReadCommand cmd = SinglePartitionReadCommand.create(true,
- cfm,
+ ReadCommand cmd = SinglePartitionReadCommand.create(metadata,
FBUtilities.nowInSeconds(),
columnFilter,
RowFilter.NONE,
@@ -361,7 +301,7 @@
}
// check (de)serialized iterator for sstable static cell
- Schema.instance.getColumnFamilyStoreInstance(cfm.cfId).forceBlockingFlush();
+ Schema.instance.getColumnFamilyStoreInstance(metadata.id).forceBlockingFlush();
try (ReadExecutionController executionController = cmd.executionController(); UnfilteredPartitionIterator pi = cmd.executeLocally(executionController))
{
response = ReadResponse.createDataResponse(pi, cmd);
@@ -379,20 +319,18 @@
@Test
public void toCQLStringIsSafeToCall() throws IOException
{
- DecoratedKey key = cfm.decorateKey(ByteBufferUtil.bytes("k1"));
+ DecoratedKey key = metadata.partitioner.decorateKey(ByteBufferUtil.bytes("k1"));
- ColumnFilter columnFilter = ColumnFilter.selection(PartitionColumns.of(s));
+ ColumnFilter columnFilter = ColumnFilter.selection(RegularAndStaticColumns.of(s));
Slice slice = Slice.make(ClusteringBound.BOTTOM, ClusteringBound.inclusiveEndOf(ByteBufferUtil.bytes("i1")));
- ClusteringIndexSliceFilter sliceFilter = new ClusteringIndexSliceFilter(Slices.with(cfm.comparator, slice), false);
- ReadCommand cmd = SinglePartitionReadCommand.create(true,
- cfm,
+ ClusteringIndexSliceFilter sliceFilter = new ClusteringIndexSliceFilter(Slices.with(metadata.comparator, slice), false);
+ ReadCommand cmd = SinglePartitionReadCommand.create(metadata,
FBUtilities.nowInSeconds(),
columnFilter,
RowFilter.NONE,
DataLimits.NONE,
key,
sliceFilter);
-
String ret = cmd.toCQLString();
Assert.assertNotNull(ret);
Assert.assertFalse(ret.isEmpty());
@@ -401,12 +339,12 @@
public static List<Unfiltered> getUnfilteredsFromSinglePartition(String q)
{
- SelectStatement stmt = (SelectStatement) QueryProcessor.parseStatement(q).prepare(ClientState.forInternalCalls()).statement;
+ SelectStatement stmt = (SelectStatement) QueryProcessor.parseStatement(q).prepare(ClientState.forInternalCalls());
List<Unfiltered> unfiltereds = new ArrayList<>();
- SinglePartitionReadCommand.Group query = (SinglePartitionReadCommand.Group) stmt.getQuery(QueryOptions.DEFAULT, 0);
- Assert.assertEquals(1, query.commands.size());
- SinglePartitionReadCommand command = Iterables.getOnlyElement(query.commands);
+ SinglePartitionReadQuery.Group<SinglePartitionReadCommand> query = (SinglePartitionReadQuery.Group<SinglePartitionReadCommand>) stmt.getQuery(QueryOptions.DEFAULT, 0);
+ Assert.assertEquals(1, query.queries.size());
+ SinglePartitionReadCommand command = Iterables.getOnlyElement(query.queries);
try (ReadExecutionController controller = ReadExecutionController.forCommand(command);
UnfilteredPartitionIterator partitions = command.executeLocally(controller))
{
@@ -447,15 +385,14 @@
public void sstableFiltering()
{
QueryProcessor.executeOnceInternal("CREATE TABLE ks.legacy_mc_inaccurate_min_max (k int, c1 int, c2 int, c3 int, v int, primary key (k, c1, c2, c3))");
- CFMetaData metadata = Schema.instance.getCFMetaData("ks", "legacy_mc_inaccurate_min_max");
- ColumnFamilyStore cfs = Schema.instance.getColumnFamilyStoreInstance(metadata.cfId);
+ TableMetadata metadata = Schema.instance.getTableMetadata("ks", "legacy_mc_inaccurate_min_max");
+ ColumnFamilyStore cfs = Schema.instance.getColumnFamilyStoreInstance(metadata.id);
QueryProcessor.executeOnceInternal("INSERT INTO ks.legacy_mc_inaccurate_min_max (k, c1, c2, c3, v) VALUES (100, 2, 2, 2, 2)");
QueryProcessor.executeOnceInternal("DELETE FROM ks.legacy_mc_inaccurate_min_max WHERE k=100 AND c1=1");
assertQueryReturnsSingleRT("SELECT * FROM ks.legacy_mc_inaccurate_min_max WHERE k=100 AND c1=1 AND c2=1");
cfs.forceBlockingFlush();
assertQueryReturnsSingleRT("SELECT * FROM ks.legacy_mc_inaccurate_min_max WHERE k=100 AND c1=1 AND c2=1");
-
assertQueryReturnsSingleRT("SELECT * FROM ks.legacy_mc_inaccurate_min_max WHERE k=100 AND c1=1 AND c2=1 AND c3=1"); // clustering names
cfs.truncateBlocking();
@@ -464,9 +401,10 @@
Slice slice = Slice.make(Clustering.make(bb(2), bb(3)), Clustering.make(bb(10), bb(10)));
RangeTombstone rt = new RangeTombstone(slice, new DeletionTime(TimeUnit.MILLISECONDS.toMicros(nowMillis),
Ints.checkedCast(TimeUnit.MILLISECONDS.toSeconds(nowMillis))));
- PartitionUpdate update = new PartitionUpdate(cfs.metadata, bb(100), cfs.metadata.partitionColumns(), 1);
- update.add(rt);
- new Mutation(update).apply();
+
+ PartitionUpdate.Builder builder = new PartitionUpdate.Builder(metadata, bb(100), metadata.regularAndStaticColumns(), 1);
+ builder.add(rt);
+ new Mutation(builder.build()).apply();
assertQueryReturnsSingleRT("SELECT * FROM ks.legacy_mc_inaccurate_min_max WHERE k=100 AND c1=3 AND c2=2");
cfs.forceBlockingFlush();
diff --git a/test/unit/org/apache/cassandra/db/SystemKeyspaceMigrator40Test.java b/test/unit/org/apache/cassandra/db/SystemKeyspaceMigrator40Test.java
new file mode 100644
index 0000000..a14db00
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/SystemKeyspaceMigrator40Test.java
@@ -0,0 +1,218 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db;
+
+import java.net.InetAddress;
+import java.nio.ByteBuffer;
+import java.util.UUID;
+
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import org.junit.Test;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.cql3.UntypedResultSet;
+import org.apache.cassandra.db.marshal.BytesType;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.db.marshal.UUIDType;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.utils.UUIDGen;
+
+import static org.junit.Assert.assertEquals;
+
+public class SystemKeyspaceMigrator40Test extends CQLTester
+{
+ @Test
+ public void testMigratePeers() throws Throwable
+ {
+ String insert = String.format("INSERT INTO %s ("
+ + "peer, "
+ + "data_center, "
+ + "host_id, "
+ + "preferred_ip, "
+ + "rack, "
+ + "release_version, "
+ + "rpc_address, "
+ + "schema_version, "
+ + "tokens) "
+ + " values ( ?, ?, ? , ? , ?, ?, ?, ?, ?)",
+ SystemKeyspaceMigrator40.legacyPeersName);
+ UUID hostId = UUIDGen.getTimeUUID();
+ UUID schemaVersion = UUIDGen.getTimeUUID();
+ execute(insert,
+ InetAddress.getByName("127.0.0.1"),
+ "dcFoo",
+ hostId,
+ InetAddress.getByName("127.0.0.2"),
+ "rackFoo", "4.0",
+ InetAddress.getByName("127.0.0.3"),
+ schemaVersion,
+ ImmutableSet.of("foobar"));
+ SystemKeyspaceMigrator40.migrate();
+
+ int rowCount = 0;
+ for (UntypedResultSet.Row row : execute(String.format("SELECT * FROM %s", SystemKeyspaceMigrator40.peersName)))
+ {
+ rowCount++;
+ assertEquals(InetAddress.getByName("127.0.0.1"), row.getInetAddress("peer"));
+ assertEquals(DatabaseDescriptor.getStoragePort(), row.getInt("peer_port"));
+ assertEquals("dcFoo", row.getString("data_center"));
+ assertEquals(hostId, row.getUUID("host_id"));
+ assertEquals(InetAddress.getByName("127.0.0.2"), row.getInetAddress("preferred_ip"));
+ assertEquals(DatabaseDescriptor.getStoragePort(), row.getInt("preferred_port"));
+ assertEquals("rackFoo", row.getString("rack"));
+ assertEquals("4.0", row.getString("release_version"));
+ assertEquals(InetAddress.getByName("127.0.0.3"), row.getInetAddress("native_address"));
+ assertEquals(DatabaseDescriptor.getNativeTransportPort(), row.getInt("native_port"));
+ assertEquals(schemaVersion, row.getUUID("schema_version"));
+ assertEquals(ImmutableSet.of("foobar"), row.getSet("tokens", UTF8Type.instance));
+ }
+ assertEquals(1, rowCount);
+
+ //Test nulls/missing don't prevent the row from propagating
+ execute(String.format("TRUNCATE %s", SystemKeyspaceMigrator40.legacyPeersName));
+ execute(String.format("TRUNCATE %s", SystemKeyspaceMigrator40.peersName));
+
+ execute(String.format("INSERT INTO %s (peer) VALUES (?)", SystemKeyspaceMigrator40.legacyPeersName),
+ InetAddress.getByName("127.0.0.1"));
+ SystemKeyspaceMigrator40.migrate();
+
+ rowCount = 0;
+ for (UntypedResultSet.Row row : execute(String.format("SELECT * FROM %s", SystemKeyspaceMigrator40.peersName)))
+ {
+ rowCount++;
+ }
+ assertEquals(1, rowCount);
+ }
+
+ @Test
+ public void testMigratePeerEvents() throws Throwable
+ {
+ String insert = String.format("INSERT INTO %s ("
+ + "peer, "
+ + "hints_dropped) "
+ + " values ( ?, ? )",
+ SystemKeyspaceMigrator40.legacyPeerEventsName);
+ UUID uuid = UUIDGen.getTimeUUID();
+ execute(insert,
+ InetAddress.getByName("127.0.0.1"),
+ ImmutableMap.of(uuid, 42));
+ SystemKeyspaceMigrator40.migrate();
+
+ int rowCount = 0;
+ for (UntypedResultSet.Row row : execute(String.format("SELECT * FROM %s", SystemKeyspaceMigrator40.peerEventsName)))
+ {
+ rowCount++;
+ assertEquals(InetAddress.getByName("127.0.0.1"), row.getInetAddress("peer"));
+ assertEquals(DatabaseDescriptor.getStoragePort(), row.getInt("peer_port"));
+ assertEquals(ImmutableMap.of(uuid, 42), row.getMap("hints_dropped", UUIDType.instance, Int32Type.instance));
+ }
+ assertEquals(1, rowCount);
+
+ //Test nulls/missing don't prevent the row from propagating
+ execute(String.format("TRUNCATE %s", SystemKeyspaceMigrator40.legacyPeerEventsName));
+ execute(String.format("TRUNCATE %s", SystemKeyspaceMigrator40.peerEventsName));
+
+ execute(String.format("INSERT INTO %s (peer) VALUES (?)", SystemKeyspaceMigrator40.legacyPeerEventsName),
+ InetAddress.getByName("127.0.0.1"));
+ SystemKeyspaceMigrator40.migrate();
+
+ rowCount = 0;
+ for (UntypedResultSet.Row row : execute(String.format("SELECT * FROM %s", SystemKeyspaceMigrator40.peerEventsName)))
+ {
+ rowCount++;
+ }
+ assertEquals(1, rowCount);
+ }
+
+ @Test
+ public void testMigrateTransferredRanges() throws Throwable
+ {
+ String insert = String.format("INSERT INTO %s ("
+ + "operation, "
+ + "peer, "
+ + "keyspace_name, "
+ + "ranges) "
+ + " values ( ?, ?, ?, ? )",
+ SystemKeyspaceMigrator40.legacyTransferredRangesName);
+ execute(insert,
+ "foo",
+ InetAddress.getByName("127.0.0.1"),
+ "bar",
+ ImmutableSet.of(ByteBuffer.wrap(new byte[] { 42 })));
+ SystemKeyspaceMigrator40.migrate();
+
+ int rowCount = 0;
+ for (UntypedResultSet.Row row : execute(String.format("SELECT * FROM %s", SystemKeyspaceMigrator40.transferredRangesName)))
+ {
+ rowCount++;
+ assertEquals("foo", row.getString("operation"));
+ assertEquals(InetAddress.getByName("127.0.0.1"), row.getInetAddress("peer"));
+ assertEquals(DatabaseDescriptor.getStoragePort(), row.getInt("peer_port"));
+ assertEquals("bar", row.getString("keyspace_name"));
+ assertEquals(ImmutableSet.of(ByteBuffer.wrap(new byte[] { 42 })), row.getSet("ranges", BytesType.instance));
+ }
+ assertEquals(1, rowCount);
+
+ //Test nulls/missing don't prevent the row from propagating
+ execute(String.format("TRUNCATE %s", SystemKeyspaceMigrator40.legacyTransferredRangesName));
+ execute(String.format("TRUNCATE %s", SystemKeyspaceMigrator40.transferredRangesName));
+
+ execute(String.format("INSERT INTO %s (operation, peer, keyspace_name) VALUES (?, ?, ?)", SystemKeyspaceMigrator40.legacyTransferredRangesName),
+ "foo",
+ InetAddress.getByName("127.0.0.1"),
+ "bar");
+ SystemKeyspaceMigrator40.migrate();
+
+ rowCount = 0;
+ for (UntypedResultSet.Row row : execute(String.format("SELECT * FROM %s", SystemKeyspaceMigrator40.transferredRangesName)))
+ {
+ rowCount++;
+ }
+ assertEquals(1, rowCount);
+ }
+
+ @Test
+ public void testMigrateAvailableRanges() throws Throwable
+ {
+ Range<Token> testRange = new Range<>(DatabaseDescriptor.getPartitioner().getRandomToken(), DatabaseDescriptor.getPartitioner().getRandomToken());
+ String insert = String.format("INSERT INTO %s ("
+ + "keyspace_name, "
+ + "ranges) "
+ + " values ( ?, ? )",
+ SystemKeyspaceMigrator40.legacyAvailableRangesName);
+ execute(insert,
+ "foo",
+ ImmutableSet.of(SystemKeyspace.rangeToBytes(testRange)));
+ SystemKeyspaceMigrator40.migrate();
+
+ int rowCount = 0;
+ for (UntypedResultSet.Row row : execute(String.format("SELECT * FROM %s", SystemKeyspaceMigrator40.availableRangesName)))
+ {
+ rowCount++;
+ assertEquals("foo", row.getString("keyspace_name"));
+ assertEquals(ImmutableSet.of(testRange), SystemKeyspace.rawRangesToRangeSet(row.getSet("full_ranges", BytesType.instance), DatabaseDescriptor.getPartitioner()));
+ }
+ assertEquals(1, rowCount);
+ }
+}
diff --git a/test/unit/org/apache/cassandra/db/SystemKeyspaceTest.java b/test/unit/org/apache/cassandra/db/SystemKeyspaceTest.java
index 92f2a56..0a6d551 100644
--- a/test/unit/org/apache/cassandra/db/SystemKeyspaceTest.java
+++ b/test/unit/org/apache/cassandra/db/SystemKeyspaceTest.java
@@ -17,46 +17,36 @@
*/
package org.apache.cassandra.db;
-import java.io.File;
import java.io.IOException;
-import java.net.InetAddress;
import java.net.UnknownHostException;
-import java.nio.file.Path;
-import java.nio.file.Paths;
import java.util.*;
-import java.util.concurrent.Future;
-import org.apache.commons.io.FileUtils;
import org.junit.BeforeClass;
import org.junit.Test;
-import org.apache.cassandra.concurrent.Stage;
-import org.apache.cassandra.concurrent.StageManager;
import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.config.SchemaConstants;
+import org.apache.cassandra.db.commitlog.CommitLog;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.schema.SchemaConstants;
import org.apache.cassandra.cql3.QueryProcessor;
import org.apache.cassandra.cql3.UntypedResultSet;
import org.apache.cassandra.dht.ByteOrderedPartitioner.BytesToken;
import org.apache.cassandra.dht.Token;
-import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.schema.SchemaKeyspace;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.CassandraVersion;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
public class SystemKeyspaceTest
{
- private static final String MIGRATION_SSTABLES_ROOT = "migration-sstable-root";
-
- // any file name will do but unrelated files in our folders tend to be log files or very old data files
- private static final String UNRELATED_FILE_NAME = "system.log";
- private static final String UNRELATED_FOLDER_NAME = "snapshot-abc";
-
@BeforeClass
public static void prepSnapshotTracker()
{
DatabaseDescriptor.daemonInitialization();
+ CommitLog.instance.start();
if (FBUtilities.isWindows)
WindowsFailedSnapshotTracker.deleteOldSnapshots();
@@ -66,7 +56,7 @@
public void testLocalTokens()
{
// Remove all existing tokens
- Collection<Token> current = SystemKeyspace.loadTokens().asMap().get(FBUtilities.getLocalAddress());
+ Collection<Token> current = SystemKeyspace.loadTokens().asMap().get(FBUtilities.getLocalAddressAndPort());
if (current != null && !current.isEmpty())
SystemKeyspace.updateTokens(current);
@@ -87,9 +77,8 @@
public void testNonLocalToken() throws UnknownHostException
{
BytesToken token = new BytesToken(ByteBufferUtil.bytes("token3"));
- InetAddress address = InetAddress.getByName("127.0.0.2");
- Future<?> future = SystemKeyspace.updateTokens(address, Collections.singletonList(token), StageManager.getStage(Stage.MUTATION));
- FBUtilities.waitOnFuture(future);
+ InetAddressAndPort address = InetAddressAndPort.getByName("127.0.0.2");
+ SystemKeyspace.updateTokens(address, Collections.<Token>singletonList(token));
assert SystemKeyspace.loadTokens().get(address).contains(token);
SystemKeyspace.removeEndpoint(address);
assert !SystemKeyspace.loadTokens().containsValue(token);
@@ -108,7 +97,7 @@
if (FBUtilities.isWindows)
assertEquals(expectedCount, getDeferredDeletionCount());
else
- assertTrue(getSystemSnapshotFiles().isEmpty());
+ assertTrue(getSystemSnapshotFiles(SchemaConstants.SYSTEM_KEYSPACE_NAME).isEmpty());
}
private int getDeferredDeletionCount()
@@ -145,7 +134,11 @@
// Compare versions again & verify that snapshots were created for all tables in the system ks
SystemKeyspace.snapshotOnVersionChange();
- assertEquals(SystemKeyspace.metadata().tables.size(), getSystemSnapshotFiles().size());
+
+ Set<String> snapshottedSystemTables = getSystemSnapshotFiles(SchemaConstants.SYSTEM_KEYSPACE_NAME);
+ SystemKeyspace.metadata().tables.forEach(t -> assertTrue(snapshottedSystemTables.contains(t.name)));
+ Set<String> snapshottedSchemaTables = getSystemSnapshotFiles(SchemaConstants.SCHEMA_KEYSPACE_NAME);
+ SchemaKeyspace.metadata().tables.forEach(t -> assertTrue(snapshottedSchemaTables.contains(t.name)));
// clear out the snapshots & set the previous recorded version equal to the latest, we shouldn't
// see any new snapshots created this time.
@@ -162,176 +155,6 @@
Keyspace.clearSnapshot(null, SchemaConstants.SYSTEM_KEYSPACE_NAME);
}
- @Test
- public void testMigrateEmptyDataDirs() throws IOException
- {
- File dataDir = Paths.get(DatabaseDescriptor.getAllDataFileLocations()[0]).toFile();
- if (new File(dataDir, "Emptykeyspace1").exists())
- FileUtils.deleteDirectory(new File(dataDir, "Emptykeyspace1"));
- assertTrue(new File(dataDir, "Emptykeyspace1").mkdirs());
- assertEquals(0, numLegacyFiles());
- SystemKeyspace.migrateDataDirs();
- assertEquals(0, numLegacyFiles());
-
- assertTrue(new File(dataDir, "Emptykeyspace1/table1").mkdirs());
- assertEquals(0, numLegacyFiles());
- SystemKeyspace.migrateDataDirs();
- assertEquals(0, numLegacyFiles());
-
- assertTrue(new File(dataDir, "Emptykeyspace1/wrong_file").createNewFile());
- assertEquals(0, numLegacyFiles());
- SystemKeyspace.migrateDataDirs();
- assertEquals(0, numLegacyFiles());
-
- }
-
- @Test
- public void testMigrateDataDirs_2_1() throws IOException
- {
- testMigrateDataDirs("2.1", 5); // see test data for num legacy files
- }
-
- @Test
- public void testMigrateDataDirs_2_2() throws IOException
- {
- testMigrateDataDirs("2.2", 7); // see test data for num legacy files
- }
-
- private void testMigrateDataDirs(String version, int numLegacyFiles) throws IOException
- {
- Path migrationSSTableRoot = Paths.get(System.getProperty(MIGRATION_SSTABLES_ROOT), version);
- Path dataDir = Paths.get(DatabaseDescriptor.getAllDataFileLocations()[0]);
-
- FileUtils.copyDirectory(migrationSSTableRoot.toFile(), dataDir.toFile());
-
- assertEquals(numLegacyFiles, numLegacyFiles());
-
- SystemKeyspace.migrateDataDirs();
-
- assertEquals(0, numLegacyFiles());
- }
-
- private static int numLegacyFiles()
- {
- int ret = 0;
- Iterable<String> dirs = Arrays.asList(DatabaseDescriptor.getAllDataFileLocations());
- for (String dataDir : dirs)
- {
- File dir = new File(dataDir);
- for (File ksdir : dir.listFiles((d, n) -> new File(d, n).isDirectory()))
- {
- for (File cfdir : ksdir.listFiles((d, n) -> new File(d, n).isDirectory()))
- {
- if (Descriptor.isLegacyFile(cfdir))
- {
- ret++;
- }
- else
- {
- File[] legacyFiles = cfdir.listFiles((d, n) -> Descriptor.isLegacyFile(new File(d, n)));
- if (legacyFiles != null)
- ret += legacyFiles.length;
- }
- }
- }
- }
- return ret;
- }
-
- @Test
- public void testMigrateDataDirs_UnrelatedFiles_2_1() throws IOException
- {
- testMigrateDataDirsWithUnrelatedFiles("2.1");
- }
-
- @Test
- public void testMigrateDataDirs_UnrelatedFiles_2_2() throws IOException
- {
- testMigrateDataDirsWithUnrelatedFiles("2.2");
- }
-
- private void testMigrateDataDirsWithUnrelatedFiles(String version) throws IOException
- {
- Path migrationSSTableRoot = Paths.get(System.getProperty(MIGRATION_SSTABLES_ROOT), version);
- Path dataDir = Paths.get(DatabaseDescriptor.getAllDataFileLocations()[0]);
-
- FileUtils.copyDirectory(migrationSSTableRoot.toFile(), dataDir.toFile());
-
- addUnRelatedFiles(dataDir);
-
- SystemKeyspace.migrateDataDirs();
-
- checkUnrelatedFiles(dataDir);
- }
-
- /**
- * Add some extra and totally unrelated files to the data dir and its sub-folders
- */
- private void addUnRelatedFiles(Path dataDir) throws IOException
- {
- File dir = new File(dataDir.toString());
- createAndCheck(dir, UNRELATED_FILE_NAME, false);
- createAndCheck(dir, UNRELATED_FOLDER_NAME, true);
-
- for (File ksdir : dir.listFiles((d, n) -> new File(d, n).isDirectory()))
- {
- createAndCheck(ksdir, UNRELATED_FILE_NAME, false);
- createAndCheck(ksdir, UNRELATED_FOLDER_NAME, true);
-
- for (File cfdir : ksdir.listFiles((d, n) -> new File(d, n).isDirectory()))
- {
- createAndCheck(cfdir, UNRELATED_FILE_NAME, false);
- createAndCheck(cfdir, UNRELATED_FOLDER_NAME, true);
- }
- }
- }
-
- /**
- * Make sure the extra files are still in the data dir and its sub-folders, then
- * remove them.
- */
- private void checkUnrelatedFiles(Path dataDir) throws IOException
- {
- File dir = new File(dataDir.toString());
- checkAndDelete(dir, UNRELATED_FILE_NAME, false);
- checkAndDelete(dir, UNRELATED_FOLDER_NAME, true);
-
- for (File ksdir : dir.listFiles((d, n) -> new File(d, n).isDirectory()))
- {
- checkAndDelete(ksdir, UNRELATED_FILE_NAME, false);
- checkAndDelete(ksdir, UNRELATED_FOLDER_NAME, true);
-
- for (File cfdir : ksdir.listFiles((d, n) -> new File(d, n).isDirectory()))
- {
- checkAndDelete(cfdir, UNRELATED_FILE_NAME, false);
- checkAndDelete(cfdir, UNRELATED_FOLDER_NAME, true);
- }
- }
- }
-
- private void createAndCheck(File dir, String fileName, boolean isDir) throws IOException
- {
- File f = new File(dir, fileName);
-
- if (isDir)
- f.mkdir();
- else
- f.createNewFile();
-
- assertTrue(f.exists());
- }
-
- private void checkAndDelete(File dir, String fileName, boolean isDir) throws IOException
- {
- File f = new File(dir, fileName);
- assertTrue(f.exists());
-
- if (isDir)
- FileUtils.deleteDirectory(f);
- else
- f.delete();
- }
-
private String getOlderVersionString()
{
String version = FBUtilities.getReleaseVersionString();
@@ -340,13 +163,13 @@
return (String.format("%s.%s.%s", semver.major - 1, semver.minor, semver.patch));
}
- private Set<String> getSystemSnapshotFiles()
+ private Set<String> getSystemSnapshotFiles(String keyspace)
{
Set<String> snapshottedTableNames = new HashSet<>();
- for (ColumnFamilyStore cfs : Keyspace.open(SchemaConstants.SYSTEM_KEYSPACE_NAME).getColumnFamilyStores())
+ for (ColumnFamilyStore cfs : Keyspace.open(keyspace).getColumnFamilyStores())
{
if (!cfs.getSnapshotDetails().isEmpty())
- snapshottedTableNames.add(cfs.getColumnFamilyName());
+ snapshottedTableNames.add(cfs.getTableName());
}
return snapshottedTableNames;
}
diff --git a/test/unit/org/apache/cassandra/db/TransformerTest.java b/test/unit/org/apache/cassandra/db/TransformerTest.java
index fe87af8..9c2ed91 100644
--- a/test/unit/org/apache/cassandra/db/TransformerTest.java
+++ b/test/unit/org/apache/cassandra/db/TransformerTest.java
@@ -23,9 +23,9 @@
import org.junit.Test;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.cassandra.Util;
-import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.marshal.BytesType;
import org.apache.cassandra.db.marshal.Int32Type;
@@ -44,18 +44,18 @@
DatabaseDescriptor.daemonInitialization();
}
- static final CFMetaData metadata = metadata();
+ static final TableMetadata metadata = metadata();
static final DecoratedKey partitionKey = new BufferDecoratedKey(new Murmur3Partitioner.LongToken(0L), ByteBufferUtil.EMPTY_BYTE_BUFFER);
- static final Row staticRow = BTreeRow.singleCellRow(Clustering.STATIC_CLUSTERING, new BufferCell(metadata.partitionColumns().columns(true).getSimple(0), 0L, 0, 0, ByteBufferUtil.bytes(-1), null));
+ static final Row staticRow = BTreeRow.singleCellRow(Clustering.STATIC_CLUSTERING, new BufferCell(metadata.regularAndStaticColumns().columns(true).getSimple(0), 0L, 0, 0, ByteBufferUtil.bytes(-1), null));
- static CFMetaData metadata()
+ static TableMetadata metadata()
{
- CFMetaData.Builder builder = CFMetaData.Builder.create("", "");
- builder.addPartitionKey("pk", BytesType.instance);
- builder.addClusteringColumn("c", Int32Type.instance);
- builder.addStaticColumn("s", Int32Type.instance);
- builder.addRegularColumn("v", Int32Type.instance);
- return builder.build();
+ return TableMetadata.builder("", "")
+ .addPartitionKeyColumn("pk", BytesType.instance)
+ .addClusteringColumn("c", Int32Type.instance)
+ .addStaticColumn("s", Int32Type.instance)
+ .addRegularColumn("v", Int32Type.instance)
+ .build();
}
// Mock Data
@@ -78,7 +78,7 @@
return (U) row(i);
}
- public CFMetaData metadata()
+ public TableMetadata metadata()
{
return metadata;
}
@@ -88,9 +88,9 @@
return false;
}
- public PartitionColumns columns()
+ public RegularAndStaticColumns columns()
{
- return metadata.partitionColumns();
+ return metadata.regularAndStaticColumns();
}
public DecoratedKey partitionKey()
@@ -150,7 +150,7 @@
private static Row row(int i)
{
return BTreeRow.singleCellRow(Util.clustering(metadata.comparator, i),
- new BufferCell(metadata.partitionColumns().columns(false).getSimple(0), 1L, BufferCell.NO_TTL, BufferCell.NO_DELETION_TIME, ByteBufferUtil.bytes(i), null));
+ new BufferCell(metadata.regularAndStaticColumns().columns(false).getSimple(0), 1L, BufferCell.NO_TTL, BufferCell.NO_DELETION_TIME, ByteBufferUtil.bytes(i), null));
}
// Transformations that check mock data ranges
diff --git a/test/unit/org/apache/cassandra/db/VerifyTest.java b/test/unit/org/apache/cassandra/db/VerifyTest.java
index a332f74..df2acb4 100644
--- a/test/unit/org/apache/cassandra/db/VerifyTest.java
+++ b/test/unit/org/apache/cassandra/db/VerifyTest.java
@@ -21,7 +21,6 @@
import com.google.common.base.Charsets;
import org.apache.cassandra.OrderedJUnit4ClassRunner;
-import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.Util;
import org.apache.cassandra.cache.ChunkCache;
import org.apache.cassandra.UpdateBuilder;
@@ -29,15 +28,23 @@
import org.apache.cassandra.db.compaction.CompactionManager;
import org.apache.cassandra.db.compaction.Verifier;
import org.apache.cassandra.db.marshal.UUIDType;
+import org.apache.cassandra.dht.ByteOrderedPartitioner;
+import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.exceptions.WriteTimeoutException;
import org.apache.cassandra.io.FSWriteError;
+import org.apache.cassandra.io.compress.CorruptBlockException;
import org.apache.cassandra.io.sstable.Component;
import org.apache.cassandra.io.sstable.CorruptSSTableException;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.TokenMetadata;
import org.apache.cassandra.schema.CompressionParams;
import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.commons.lang3.StringUtils;
import org.junit.BeforeClass;
@@ -45,13 +52,19 @@
import org.junit.runner.RunWith;
import java.io.*;
+import java.net.UnknownHostException;
import java.nio.file.Files;
+import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.zip.CRC32;
import java.util.zip.CheckedInputStream;
+import static org.apache.cassandra.SchemaLoader.counterCFMD;
+import static org.apache.cassandra.SchemaLoader.createKeyspace;
+import static org.apache.cassandra.SchemaLoader.loadSchema;
+import static org.apache.cassandra.SchemaLoader.standardCFMD;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -80,27 +93,27 @@
{
CompressionParams compressionParameters = CompressionParams.snappy(32768);
- SchemaLoader.loadSchema();
- SchemaLoader.createKeyspace(KEYSPACE,
- KeyspaceParams.simple(1),
- SchemaLoader.standardCFMD(KEYSPACE, CF).compression(compressionParameters),
- SchemaLoader.standardCFMD(KEYSPACE, CF2).compression(compressionParameters),
- SchemaLoader.standardCFMD(KEYSPACE, CF3),
- SchemaLoader.standardCFMD(KEYSPACE, CF4),
- SchemaLoader.standardCFMD(KEYSPACE, CORRUPT_CF),
- SchemaLoader.standardCFMD(KEYSPACE, CORRUPT_CF2),
- SchemaLoader.counterCFMD(KEYSPACE, COUNTER_CF).compression(compressionParameters),
- SchemaLoader.counterCFMD(KEYSPACE, COUNTER_CF2).compression(compressionParameters),
- SchemaLoader.counterCFMD(KEYSPACE, COUNTER_CF3),
- SchemaLoader.counterCFMD(KEYSPACE, COUNTER_CF4),
- SchemaLoader.counterCFMD(KEYSPACE, CORRUPTCOUNTER_CF),
- SchemaLoader.counterCFMD(KEYSPACE, CORRUPTCOUNTER_CF2),
- SchemaLoader.standardCFMD(KEYSPACE, CF_UUID, 0, UUIDType.instance));
+ loadSchema();
+ createKeyspace(KEYSPACE,
+ KeyspaceParams.simple(1),
+ standardCFMD(KEYSPACE, CF).compression(compressionParameters),
+ standardCFMD(KEYSPACE, CF2).compression(compressionParameters),
+ standardCFMD(KEYSPACE, CF3),
+ standardCFMD(KEYSPACE, CF4),
+ standardCFMD(KEYSPACE, CORRUPT_CF),
+ standardCFMD(KEYSPACE, CORRUPT_CF2),
+ counterCFMD(KEYSPACE, COUNTER_CF).compression(compressionParameters),
+ counterCFMD(KEYSPACE, COUNTER_CF2).compression(compressionParameters),
+ counterCFMD(KEYSPACE, COUNTER_CF3),
+ counterCFMD(KEYSPACE, COUNTER_CF4),
+ counterCFMD(KEYSPACE, CORRUPTCOUNTER_CF),
+ counterCFMD(KEYSPACE, CORRUPTCOUNTER_CF2),
+ standardCFMD(KEYSPACE, CF_UUID, 0, UUIDType.instance));
}
@Test
- public void testVerifyCorrect() throws IOException
+ public void testVerifyCorrect()
{
CompactionManager.instance.disableAutoCompaction();
Keyspace keyspace = Keyspace.open(KEYSPACE);
@@ -110,9 +123,9 @@
SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
- try (Verifier verifier = new Verifier(cfs, sstable, false))
+ try (Verifier verifier = new Verifier(cfs, sstable, false, Verifier.options().invokeDiskFailurePolicy(true).build()))
{
- verifier.verify(false);
+ verifier.verify();
}
catch (CorruptSSTableException err)
{
@@ -121,7 +134,7 @@
}
@Test
- public void testVerifyCounterCorrect() throws IOException
+ public void testVerifyCounterCorrect()
{
CompactionManager.instance.disableAutoCompaction();
Keyspace keyspace = Keyspace.open(KEYSPACE);
@@ -130,10 +143,9 @@
fillCounterCF(cfs, 2);
SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
-
- try (Verifier verifier = new Verifier(cfs, sstable, false))
+ try (Verifier verifier = new Verifier(cfs, sstable, false, Verifier.options().invokeDiskFailurePolicy(true).build()))
{
- verifier.verify(false);
+ verifier.verify();
}
catch (CorruptSSTableException err)
{
@@ -142,7 +154,7 @@
}
@Test
- public void testExtendedVerifyCorrect() throws IOException
+ public void testExtendedVerifyCorrect()
{
CompactionManager.instance.disableAutoCompaction();
Keyspace keyspace = Keyspace.open(KEYSPACE);
@@ -151,10 +163,9 @@
fillCF(cfs, 2);
SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
-
- try (Verifier verifier = new Verifier(cfs, sstable, false))
+ try (Verifier verifier = new Verifier(cfs, sstable, false, Verifier.options().invokeDiskFailurePolicy(true).build()))
{
- verifier.verify(true);
+ verifier.verify();
}
catch (CorruptSSTableException err)
{
@@ -163,7 +174,7 @@
}
@Test
- public void testExtendedVerifyCounterCorrect() throws IOException
+ public void testExtendedVerifyCounterCorrect()
{
CompactionManager.instance.disableAutoCompaction();
Keyspace keyspace = Keyspace.open(KEYSPACE);
@@ -173,9 +184,9 @@
SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
- try (Verifier verifier = new Verifier(cfs, sstable, false))
+ try (Verifier verifier = new Verifier(cfs, sstable, false, Verifier.options().invokeDiskFailurePolicy(true).extendedVerification(true).build()))
{
- verifier.verify(true);
+ verifier.verify();
}
catch (CorruptSSTableException err)
{
@@ -184,7 +195,7 @@
}
@Test
- public void testVerifyCorrectUncompressed() throws IOException
+ public void testVerifyCorrectUncompressed()
{
CompactionManager.instance.disableAutoCompaction();
Keyspace keyspace = Keyspace.open(KEYSPACE);
@@ -194,9 +205,9 @@
SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
- try (Verifier verifier = new Verifier(cfs, sstable, false))
+ try (Verifier verifier = new Verifier(cfs, sstable, false, Verifier.options().invokeDiskFailurePolicy(true).build()))
{
- verifier.verify(false);
+ verifier.verify();
}
catch (CorruptSSTableException err)
{
@@ -205,7 +216,7 @@
}
@Test
- public void testVerifyCounterCorrectUncompressed() throws IOException
+ public void testVerifyCounterCorrectUncompressed()
{
CompactionManager.instance.disableAutoCompaction();
Keyspace keyspace = Keyspace.open(KEYSPACE);
@@ -215,9 +226,9 @@
SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
- try (Verifier verifier = new Verifier(cfs, sstable, false))
+ try (Verifier verifier = new Verifier(cfs, sstable, false, Verifier.options().invokeDiskFailurePolicy(true).build()))
{
- verifier.verify(false);
+ verifier.verify();
}
catch (CorruptSSTableException err)
{
@@ -226,7 +237,7 @@
}
@Test
- public void testExtendedVerifyCorrectUncompressed() throws IOException
+ public void testExtendedVerifyCorrectUncompressed()
{
CompactionManager.instance.disableAutoCompaction();
Keyspace keyspace = Keyspace.open(KEYSPACE);
@@ -236,9 +247,9 @@
SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
- try (Verifier verifier = new Verifier(cfs, sstable, false))
+ try (Verifier verifier = new Verifier(cfs, sstable, false, Verifier.options().extendedVerification(true).invokeDiskFailurePolicy(true).build()))
{
- verifier.verify(true);
+ verifier.verify();
}
catch (CorruptSSTableException err)
{
@@ -247,7 +258,7 @@
}
@Test
- public void testExtendedVerifyCounterCorrectUncompressed() throws IOException
+ public void testExtendedVerifyCounterCorrectUncompressed()
{
CompactionManager.instance.disableAutoCompaction();
Keyspace keyspace = Keyspace.open(KEYSPACE);
@@ -257,9 +268,9 @@
SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
- try (Verifier verifier = new Verifier(cfs, sstable, false))
+ try (Verifier verifier = new Verifier(cfs, sstable, false, Verifier.options().extendedVerification(true).invokeDiskFailurePolicy(true).build()))
{
- verifier.verify(true);
+ verifier.verify();
}
catch (CorruptSSTableException err)
{
@@ -282,19 +293,26 @@
SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
- try (RandomAccessFile file = new RandomAccessFile(sstable.descriptor.filenameFor(sstable.descriptor.digestComponent), "rw"))
+ try (RandomAccessFile file = new RandomAccessFile(sstable.descriptor.filenameFor(Component.DIGEST), "rw"))
{
Long correctChecksum = Long.valueOf(file.readLine());
-
- writeChecksum(++correctChecksum, sstable.descriptor.filenameFor(sstable.descriptor.digestComponent));
+
+ writeChecksum(++correctChecksum, sstable.descriptor.filenameFor(Component.DIGEST));
}
- try (Verifier verifier = new Verifier(cfs, sstable, false))
+ try (Verifier verifier = new Verifier(cfs, sstable, false, Verifier.options().invokeDiskFailurePolicy(true).build()))
{
- verifier.verify(false);
+ verifier.verify();
fail("Expected a CorruptSSTableException to be thrown");
}
catch (CorruptSSTableException err) {}
+
+ try (Verifier verifier = new Verifier(cfs, sstable, false, Verifier.options().invokeDiskFailurePolicy(false).build()))
+ {
+ verifier.verify();
+ fail("Expected a RuntimeException to be thrown");
+ }
+ catch (RuntimeException err) {}
}
@@ -325,24 +343,26 @@
ChunkCache.instance.invalidateFile(sstable.getFilename());
// Update the Digest to have the right Checksum
- writeChecksum(simpleFullChecksum(sstable.getFilename()), sstable.descriptor.filenameFor(sstable.descriptor.digestComponent));
+ writeChecksum(simpleFullChecksum(sstable.getFilename()), sstable.descriptor.filenameFor(Component.DIGEST));
- try (Verifier verifier = new Verifier(cfs, sstable, false))
+ try (Verifier verifier = new Verifier(cfs, sstable, false, Verifier.options().invokeDiskFailurePolicy(true).build()))
{
// First a simple verify checking digest, which should succeed
try
{
- verifier.verify(false);
+ verifier.verify();
}
catch (CorruptSSTableException err)
{
fail("Simple verify should have succeeded as digest matched");
}
-
+ }
+ try (Verifier verifier = new Verifier(cfs, sstable, false, Verifier.options().invokeDiskFailurePolicy(true).extendedVerification(true).build()))
+ {
// Now try extended verify
try
{
- verifier.verify(true);
+ verifier.verify();
}
catch (CorruptSSTableException err)
@@ -353,13 +373,13 @@
}
}
- @Test(expected = CorruptSSTableException.class)
+ @Test
public void testVerifyBrokenSSTableMetadata() throws IOException, WriteTimeoutException
{
CompactionManager.instance.disableAutoCompaction();
Keyspace keyspace = Keyspace.open(KEYSPACE);
ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CORRUPT_CF2);
-
+ cfs.truncateBlocking();
fillCF(cfs, 2);
Util.getAll(Util.cmd(cfs).build());
@@ -371,11 +391,90 @@
file.seek(0);
file.writeBytes(StringUtils.repeat('z', 2));
file.close();
-
- try (Verifier verifier = new Verifier(cfs, sstable, false))
+ try (Verifier verifier = new Verifier(cfs, sstable, false, Verifier.options().invokeDiskFailurePolicy(true).build()))
{
- verifier.verify(false);
+ verifier.verify();
+ fail("Expected a CorruptSSTableException to be thrown");
}
+ catch (CorruptSSTableException err)
+ {}
+ try (Verifier verifier = new Verifier(cfs, sstable, false, Verifier.options().invokeDiskFailurePolicy(false).build()))
+ {
+ verifier.verify();
+ fail("Expected a RuntimeException to be thrown");
+ }
+ catch (CorruptSSTableException err) { fail("wrong exception thrown"); }
+ catch (RuntimeException err)
+ {}
+ }
+
+ @Test
+ public void testVerifyMutateRepairStatus() throws IOException, WriteTimeoutException
+ {
+ CompactionManager.instance.disableAutoCompaction();
+ Keyspace keyspace = Keyspace.open(KEYSPACE);
+ ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CORRUPT_CF2);
+ cfs.truncateBlocking();
+ fillCF(cfs, 2);
+
+ Util.getAll(Util.cmd(cfs).build());
+
+ // make the sstable repaired:
+ SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
+ sstable.descriptor.getMetadataSerializer().mutateRepairMetadata(sstable.descriptor, System.currentTimeMillis(), sstable.getPendingRepair(), sstable.isTransient());
+ sstable.reloadSSTableMetadata();
+
+ // break the sstable:
+ Long correctChecksum;
+ try (RandomAccessFile file = new RandomAccessFile(sstable.descriptor.filenameFor(Component.DIGEST), "rw"))
+ {
+ correctChecksum = Long.parseLong(file.readLine());
+ }
+ writeChecksum(++correctChecksum, sstable.descriptor.filenameFor(Component.DIGEST));
+ try (Verifier verifier = new Verifier(cfs, sstable, false, Verifier.options().mutateRepairStatus(false).invokeDiskFailurePolicy(true).build()))
+ {
+ verifier.verify();
+ fail("Expected a CorruptSSTableException to be thrown");
+ }
+ catch (CorruptSSTableException err)
+ {}
+
+ assertTrue(sstable.isRepaired());
+
+ // now the repair status should be changed:
+ try (Verifier verifier = new Verifier(cfs, sstable, false, Verifier.options().mutateRepairStatus(true).invokeDiskFailurePolicy(true).build()))
+ {
+ verifier.verify();
+ fail("Expected a CorruptSSTableException to be thrown");
+ }
+ catch (CorruptSSTableException err)
+ {}
+ assertFalse(sstable.isRepaired());
+ }
+
+ @Test(expected = RuntimeException.class)
+ public void testOutOfRangeTokens() throws IOException
+ {
+ Keyspace keyspace = Keyspace.open(KEYSPACE);
+ ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF);
+ fillCF(cfs, 100);
+ TokenMetadata tmd = StorageService.instance.getTokenMetadata();
+ byte[] tk1 = new byte[1], tk2 = new byte[1];
+ tk1[0] = 2;
+ tk2[0] = 1;
+ tmd.updateNormalToken(new ByteOrderedPartitioner.BytesToken(tk1), InetAddressAndPort.getByName("127.0.0.1"));
+ tmd.updateNormalToken(new ByteOrderedPartitioner.BytesToken(tk2), InetAddressAndPort.getByName("127.0.0.2"));
+
+ SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
+ try (Verifier verifier = new Verifier(cfs, sstable, false, Verifier.options().checkOwnsTokens(true).extendedVerification(true).build()))
+ {
+ verifier.verify();
+ }
+ finally
+ {
+ StorageService.instance.getTokenMetadata().clearUnsafe();
+ }
+
}
@Test
@@ -388,7 +487,7 @@
fillCF(cfs, 2);
SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
- sstable.descriptor.getMetadataSerializer().mutateRepairedAt(sstable.descriptor, 1);
+ sstable.descriptor.getMetadataSerializer().mutateRepairMetadata(sstable.descriptor, 1, sstable.getPendingRepair(), sstable.isTransient());
sstable.reloadSSTableMetadata();
cfs.getTracker().notifySSTableRepairedStatusChanged(Collections.singleton(sstable));
assertTrue(sstable.isRepaired());
@@ -396,14 +495,14 @@
sstable = cfs.getLiveSSTables().iterator().next();
Long correctChecksum;
- try (RandomAccessFile file = new RandomAccessFile(sstable.descriptor.filenameFor(sstable.descriptor.digestComponent), "rw"))
+ try (RandomAccessFile file = new RandomAccessFile(sstable.descriptor.filenameFor(Component.DIGEST), "rw"))
{
correctChecksum = Long.parseLong(file.readLine());
}
- writeChecksum(++correctChecksum, sstable.descriptor.filenameFor(sstable.descriptor.digestComponent));
- try (Verifier verifier = new Verifier(cfs, sstable, false))
+ writeChecksum(++correctChecksum, sstable.descriptor.filenameFor(Component.DIGEST));
+ try (Verifier verifier = new Verifier(cfs, sstable, false, Verifier.options().invokeDiskFailurePolicy(true).mutateRepairStatus(true).build()))
{
- verifier.verify(false);
+ verifier.verify();
fail("should be corrupt");
}
catch (CorruptSSTableException e)
@@ -411,12 +510,193 @@
assertFalse(sstable.isRepaired());
}
+ @Test
+ public void testVerifyIndex() throws IOException
+ {
+ testBrokenComponentHelper(Component.PRIMARY_INDEX);
+ }
+ @Test
+ public void testVerifyBf() throws IOException
+ {
+ testBrokenComponentHelper(Component.FILTER);
+ }
+
+ @Test
+ public void testVerifyIndexSummary() throws IOException
+ {
+ testBrokenComponentHelper(Component.SUMMARY);
+ }
+
+ private void testBrokenComponentHelper(Component componentToBreak) throws IOException
+ {
+ CompactionManager.instance.disableAutoCompaction();
+ Keyspace keyspace = Keyspace.open(KEYSPACE);
+ ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CORRUPT_CF2);
+
+ fillCF(cfs, 2);
+
+ SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
+ try (Verifier verifier = new Verifier(cfs, sstable, false, Verifier.options().build()))
+ {
+ verifier.verify(); //still not corrupt, should pass
+ }
+ String filenameToCorrupt = sstable.descriptor.filenameFor(componentToBreak);
+ try (RandomAccessFile file = new RandomAccessFile(filenameToCorrupt, "rw"))
+ {
+ file.setLength(3);
+ }
+
+ try (Verifier verifier = new Verifier(cfs, sstable, false, Verifier.options().invokeDiskFailurePolicy(true).build()))
+ {
+ verifier.verify();
+ fail("should throw exception");
+ }
+ catch(CorruptSSTableException e)
+ {
+ //expected
+ }
+ }
+
+ @Test
+ public void testQuick() throws IOException
+ {
+ CompactionManager.instance.disableAutoCompaction();
+ Keyspace keyspace = Keyspace.open(KEYSPACE);
+ ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CORRUPT_CF);
+
+ fillCF(cfs, 2);
+
+ Util.getAll(Util.cmd(cfs).build());
+
+ SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
+
+
+ try (RandomAccessFile file = new RandomAccessFile(sstable.descriptor.filenameFor(Component.DIGEST), "rw"))
+ {
+ Long correctChecksum = Long.valueOf(file.readLine());
+
+ writeChecksum(++correctChecksum, sstable.descriptor.filenameFor(Component.DIGEST));
+ }
+
+ try (Verifier verifier = new Verifier(cfs, sstable, false, Verifier.options().invokeDiskFailurePolicy(true).build()))
+ {
+ verifier.verify();
+ fail("Expected a CorruptSSTableException to be thrown");
+ }
+ catch (CorruptSSTableException err) {}
+
+ try (Verifier verifier = new Verifier(cfs, sstable, false, Verifier.options().invokeDiskFailurePolicy(true).quick(true).build())) // with quick = true we don't verify the digest
+ {
+ verifier.verify();
+ }
+
+ try (Verifier verifier = new Verifier(cfs, sstable, false, Verifier.options().invokeDiskFailurePolicy(true).build()))
+ {
+ verifier.verify();
+ fail("Expected a RuntimeException to be thrown");
+ }
+ catch (CorruptSSTableException err) {}
+ }
+
+ @Test
+ public void testRangeOwnHelper()
+ {
+ List<Range<Token>> normalized = new ArrayList<>();
+ normalized.add(r(Long.MIN_VALUE, Long.MIN_VALUE + 1));
+ normalized.add(r(Long.MIN_VALUE + 5, Long.MIN_VALUE + 6));
+ normalized.add(r(Long.MIN_VALUE + 10, Long.MIN_VALUE + 11));
+ normalized.add(r(0,10));
+ normalized.add(r(10,11));
+ normalized.add(r(20,25));
+ normalized.add(r(26,200));
+
+ Verifier.RangeOwnHelper roh = new Verifier.RangeOwnHelper(normalized);
+
+ roh.validate(dk(1));
+ roh.validate(dk(10));
+ roh.validate(dk(11));
+ roh.validate(dk(21));
+ roh.validate(dk(25));
+ boolean gotException = false;
+ try
+ {
+ roh.validate(dk(26));
+ }
+ catch (Throwable t)
+ {
+ gotException = true;
+ }
+ assertTrue(gotException);
+ }
+
+ @Test(expected = AssertionError.class)
+ public void testRangeOwnHelperBadToken()
+ {
+ List<Range<Token>> normalized = new ArrayList<>();
+ normalized.add(r(0,10));
+ Verifier.RangeOwnHelper roh = new Verifier.RangeOwnHelper(normalized);
+ roh.validate(dk(1));
+ // call with smaller token to get exception
+ roh.validate(dk(0));
+ }
+
+
+ @Test
+ public void testRangeOwnHelperNormalize()
+ {
+ List<Range<Token>> normalized = Range.normalize(Collections.singletonList(r(0,0)));
+ Verifier.RangeOwnHelper roh = new Verifier.RangeOwnHelper(normalized);
+ roh.validate(dk(Long.MIN_VALUE));
+ roh.validate(dk(0));
+ roh.validate(dk(Long.MAX_VALUE));
+ }
+
+ @Test
+ public void testRangeOwnHelperNormalizeWrap()
+ {
+ List<Range<Token>> normalized = Range.normalize(Collections.singletonList(r(Long.MAX_VALUE - 1000,Long.MIN_VALUE + 1000)));
+ Verifier.RangeOwnHelper roh = new Verifier.RangeOwnHelper(normalized);
+ roh.validate(dk(Long.MIN_VALUE));
+ roh.validate(dk(Long.MAX_VALUE));
+ boolean gotException = false;
+ try
+ {
+ roh.validate(dk(26));
+ }
+ catch (Throwable t)
+ {
+ gotException = true;
+ }
+ assertTrue(gotException);
+ }
+
+ @Test
+ public void testEmptyRanges()
+ {
+ new Verifier.RangeOwnHelper(Collections.emptyList()).validate(dk(1));
+ }
+
+ private DecoratedKey dk(long l)
+ {
+ return new BufferDecoratedKey(t(l), ByteBufferUtil.EMPTY_BYTE_BUFFER);
+ }
+
+ private Range<Token> r(long s, long e)
+ {
+ return new Range<>(t(s), t(e));
+ }
+
+ private Token t(long t)
+ {
+ return new Murmur3Partitioner.LongToken(t);
+ }
+
protected void fillCF(ColumnFamilyStore cfs, int partitionsPerSSTable)
{
for (int i = 0; i < partitionsPerSSTable; i++)
{
- UpdateBuilder.create(cfs.metadata, String.valueOf(i))
+ UpdateBuilder.create(cfs.metadata(), String.valueOf(i))
.newRow("c1").add("val", "1")
.newRow("c2").add("val", "2")
.apply();
@@ -429,7 +709,7 @@
{
for (int i = 0; i < partitionsPerSSTable; i++)
{
- UpdateBuilder.create(cfs.metadata, String.valueOf(i))
+ UpdateBuilder.create(cfs.metadata(), String.valueOf(i))
.newRow("c1").add("val", 100L)
.apply();
}
@@ -450,7 +730,7 @@
}
}
- protected void writeChecksum(long checksum, String filePath)
+ public static void writeChecksum(long checksum, String filePath)
{
File outFile = new File(filePath);
BufferedWriter out = null;
diff --git a/test/unit/org/apache/cassandra/db/commitlog/AbstractCommitLogServiceTest.java b/test/unit/org/apache/cassandra/db/commitlog/AbstractCommitLogServiceTest.java
index bc5cb29..741b145 100644
--- a/test/unit/org/apache/cassandra/db/commitlog/AbstractCommitLogServiceTest.java
+++ b/test/unit/org/apache/cassandra/db/commitlog/AbstractCommitLogServiceTest.java
@@ -28,7 +28,6 @@
import org.apache.cassandra.config.Config;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.commitlog.AbstractCommitLogService.SyncRunnable;
-import org.apache.cassandra.utils.Clock;
import org.apache.cassandra.utils.FreeRunningClock;
import static org.apache.cassandra.db.commitlog.AbstractCommitLogService.DEFAULT_MARKER_INTERVAL_MILLIS;
diff --git a/test/unit/org/apache/cassandra/db/commitlog/BatchCommitLogTest.java b/test/unit/org/apache/cassandra/db/commitlog/BatchCommitLogTest.java
index 1f8dbdf..fb7dda1 100644
--- a/test/unit/org/apache/cassandra/db/commitlog/BatchCommitLogTest.java
+++ b/test/unit/org/apache/cassandra/db/commitlog/BatchCommitLogTest.java
@@ -18,51 +18,44 @@
package org.apache.cassandra.db.commitlog;
-import static org.junit.Assert.*;
-
import java.nio.ByteBuffer;
import java.util.concurrent.TimeUnit;
-import org.apache.cassandra.SchemaLoader;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
import org.apache.cassandra.config.Config;
import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.config.ParameterizedClass;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.db.Mutation;
import org.apache.cassandra.db.RowUpdateBuilder;
-import org.apache.cassandra.db.compaction.CompactionManager;
-import org.apache.cassandra.db.marshal.AsciiType;
-import org.apache.cassandra.db.marshal.BytesType;
-import org.apache.cassandra.schema.KeyspaceParams;
-import org.junit.BeforeClass;
-import org.junit.Test;
+import org.apache.cassandra.security.EncryptionContext;
-public class BatchCommitLogTest
+public class BatchCommitLogTest extends CommitLogTest
{
private static final long CL_BATCH_SYNC_WINDOW = 1000; // 1 second
- private static final String KEYSPACE1 = "CommitLogTest";
- private static final String STANDARD1 = "Standard1";
+
+ public BatchCommitLogTest(ParameterizedClass commitLogCompression, EncryptionContext encryptionContext)
+ {
+ super(commitLogCompression, encryptionContext);
+ }
@BeforeClass
- public static void before()
+ public static void setCommitLogModeDetails()
{
DatabaseDescriptor.daemonInitialization();
DatabaseDescriptor.setCommitLogSync(Config.CommitLogSync.batch);
- DatabaseDescriptor.setCommitLogSyncBatchWindow(CL_BATCH_SYNC_WINDOW);
-
- KeyspaceParams.DEFAULT_LOCAL_DURABLE_WRITES = false;
- SchemaLoader.prepareServer();
- SchemaLoader.createKeyspace(KEYSPACE1,
- KeyspaceParams.simple(1),
- SchemaLoader.standardCFMD(KEYSPACE1, STANDARD1, 0, AsciiType.instance, BytesType.instance));
- CompactionManager.instance.disableAutoCompaction();
+ beforeClass();
}
@Test
public void testBatchCLSyncImmediately()
{
ColumnFamilyStore cfs1 = Keyspace.open(KEYSPACE1).getColumnFamilyStore(STANDARD1);
- Mutation m = new RowUpdateBuilder(cfs1.metadata, 0, "key")
+ Mutation m = new RowUpdateBuilder(cfs1.metadata.get(), 0, "key")
.clustering("bytes")
.add("val", ByteBuffer.allocate(10 * 1024))
.build();
@@ -70,7 +63,7 @@
long startNano = System.nanoTime();
CommitLog.instance.add(m);
long delta = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNano);
- assertTrue("Expect batch commitlog sync immediately, but took " + delta, delta < CL_BATCH_SYNC_WINDOW);
+ Assert.assertTrue("Expect batch commitlog sync immediately, but took " + delta, delta < CL_BATCH_SYNC_WINDOW);
}
@Test
@@ -79,7 +72,7 @@
long startNano = System.nanoTime();
CommitLog.instance.shutdownBlocking();
long delta = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNano);
- assertTrue("Expect batch commitlog shutdown immediately, but took " + delta, delta < CL_BATCH_SYNC_WINDOW);
+ Assert.assertTrue("Expect batch commitlog shutdown immediately, but took " + delta, delta < CL_BATCH_SYNC_WINDOW);
CommitLog.instance.start();
}
}
diff --git a/test/unit/org/apache/cassandra/db/commitlog/CDCTestReplayer.java b/test/unit/org/apache/cassandra/db/commitlog/CDCTestReplayer.java
new file mode 100644
index 0000000..fa3295a
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/commitlog/CDCTestReplayer.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db.commitlog;
+
+import java.io.File;
+import java.io.IOException;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.Mutation;
+import org.apache.cassandra.db.rows.DeserializationHelper;
+import org.apache.cassandra.io.util.DataInputBuffer;
+import org.apache.cassandra.io.util.RebufferingInputStream;
+
+/**
+ * Utility class that flags the replayer as having seen a CDC mutation and calculates offset but doesn't apply mutations
+ */
+public class CDCTestReplayer extends CommitLogReplayer
+{
+ private static final Logger logger = LoggerFactory.getLogger(CDCTestReplayer.class);
+
+ public CDCTestReplayer() throws IOException
+ {
+ super(CommitLog.instance, CommitLogPosition.NONE, null, ReplayFilter.create());
+ CommitLog.instance.sync(true);
+ commitLogReader = new CommitLogTestReader();
+ }
+
+ public void examineCommitLog() throws IOException
+ {
+ replayFiles(new File(DatabaseDescriptor.getCommitLogLocation()).listFiles());
+ }
+
+ private class CommitLogTestReader extends CommitLogReader
+ {
+ @Override
+ protected void readMutation(CommitLogReadHandler handler,
+ byte[] inputBuffer,
+ int size,
+ CommitLogPosition minPosition,
+ final int entryLocation,
+ final CommitLogDescriptor desc) throws IOException
+ {
+ RebufferingInputStream bufIn = new DataInputBuffer(inputBuffer, 0, size);
+ Mutation mutation;
+ try
+ {
+ mutation = Mutation.serializer.deserialize(bufIn, desc.getMessagingVersion(), DeserializationHelper.Flag.LOCAL);
+ if (mutation.trackedByCDC())
+ sawCDCMutation = true;
+ }
+ catch (IOException e)
+ {
+ // Test fails.
+ throw new AssertionError(e);
+ }
+ }
+ }
+}
diff --git a/test/unit/org/apache/cassandra/db/commitlog/CommitLogCQLTest.java b/test/unit/org/apache/cassandra/db/commitlog/CommitLogCQLTest.java
index 7235600..4725bcf 100644
--- a/test/unit/org/apache/cassandra/db/commitlog/CommitLogCQLTest.java
+++ b/test/unit/org/apache/cassandra/db/commitlog/CommitLogCQLTest.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.cassandra.db.commitlog;
import java.util.ArrayList;
diff --git a/test/unit/org/apache/cassandra/db/commitlog/CommitLogChainedMarkersTest.java b/test/unit/org/apache/cassandra/db/commitlog/CommitLogChainedMarkersTest.java
index 959029a..fb90d59 100644
--- a/test/unit/org/apache/cassandra/db/commitlog/CommitLogChainedMarkersTest.java
+++ b/test/unit/org/apache/cassandra/db/commitlog/CommitLogChainedMarkersTest.java
@@ -75,7 +75,7 @@
byte[] entropy = new byte[1024];
new Random().nextBytes(entropy);
- final Mutation m = new RowUpdateBuilder(cfs1.metadata, 0, "k")
+ final Mutation m = new RowUpdateBuilder(cfs1.metadata.get(), 0, "k")
.clustering("bytes")
.add("val", ByteBuffer.wrap(entropy))
.build();
@@ -88,7 +88,7 @@
ArrayList<File> toCheck = CommitLogReaderTest.getCommitLogs();
CommitLogReader reader = new CommitLogReader();
- CommitLogReaderTest.TestCLRHandler testHandler = new CommitLogReaderTest.TestCLRHandler(cfs1.metadata);
+ CommitLogReaderTest.TestCLRHandler testHandler = new CommitLogReaderTest.TestCLRHandler(cfs1.metadata.get());
for (File f : toCheck)
reader.readCommitLogSegment(testHandler, f, CommitLogReader.ALL_MUTATIONS, false);
diff --git a/test/unit/org/apache/cassandra/db/commitlog/CommitLogDescriptorTest.java b/test/unit/org/apache/cassandra/db/commitlog/CommitLogDescriptorTest.java
index a6bbdb6..53c6769 100644
--- a/test/unit/org/apache/cassandra/db/commitlog/CommitLogDescriptorTest.java
+++ b/test/unit/org/apache/cassandra/db/commitlog/CommitLogDescriptorTest.java
@@ -29,7 +29,6 @@
import org.junit.Before;
import org.junit.Test;
-import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.config.ParameterizedClass;
import org.apache.cassandra.config.TransparentDataEncryptionOptions;
import org.apache.cassandra.exceptions.ConfigurationException;
@@ -69,8 +68,6 @@
neverEnabledEncryption = EncryptionContextGenerator.createDisabledContext();
TransparentDataEncryptionOptions disaabledTdeOptions = new TransparentDataEncryptionOptions(false, enabledTdeOptions.cipher, enabledTdeOptions.key_alias, enabledTdeOptions.key_provider);
previouslyEnabledEncryption = new EncryptionContext(disaabledTdeOptions);
-
- DatabaseDescriptor.daemonInitialization();
}
@Test
@@ -109,10 +106,10 @@
public void testDescriptorPersistence() throws IOException
{
testDescriptorPersistence(new CommitLogDescriptor(11, null, neverEnabledEncryption));
- testDescriptorPersistence(new CommitLogDescriptor(CommitLogDescriptor.VERSION_21, 13, null, neverEnabledEncryption));
- testDescriptorPersistence(new CommitLogDescriptor(CommitLogDescriptor.VERSION_22, 15, null, neverEnabledEncryption));
- testDescriptorPersistence(new CommitLogDescriptor(CommitLogDescriptor.VERSION_22, 17, new ParameterizedClass("LZ4Compressor", null), neverEnabledEncryption));
- testDescriptorPersistence(new CommitLogDescriptor(CommitLogDescriptor.VERSION_22, 19,
+ testDescriptorPersistence(new CommitLogDescriptor(CommitLogDescriptor.current_version, 13, null, neverEnabledEncryption));
+ testDescriptorPersistence(new CommitLogDescriptor(CommitLogDescriptor.current_version, 15, null, neverEnabledEncryption));
+ testDescriptorPersistence(new CommitLogDescriptor(CommitLogDescriptor.current_version, 17, new ParameterizedClass("LZ4Compressor", null), neverEnabledEncryption));
+ testDescriptorPersistence(new CommitLogDescriptor(CommitLogDescriptor.current_version, 19,
new ParameterizedClass("StubbyCompressor", ImmutableMap.of("parameter1", "value1", "flag2", "55", "argument3", "null")
), neverEnabledEncryption));
}
@@ -125,7 +122,7 @@
for (int i=0; i<65535; ++i)
params.put("key"+i, Integer.toString(i, 16));
try {
- CommitLogDescriptor desc = new CommitLogDescriptor(CommitLogDescriptor.VERSION_22,
+ CommitLogDescriptor desc = new CommitLogDescriptor(CommitLogDescriptor.current_version,
21,
new ParameterizedClass("LZ4Compressor", params),
neverEnabledEncryption);
diff --git a/test/unit/org/apache/cassandra/db/commitlog/CommitLogInitWithExceptionTest.java b/test/unit/org/apache/cassandra/db/commitlog/CommitLogInitWithExceptionTest.java
new file mode 100644
index 0000000..690d9ba
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/commitlog/CommitLogInitWithExceptionTest.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.commitlog;
+
+
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.apache.cassandra.CassandraIsolatedJunit4ClassRunner;
+import org.apache.cassandra.config.Config;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.utils.JVMStabilityInspector;
+
+@RunWith(CassandraIsolatedJunit4ClassRunner.class)
+public class CommitLogInitWithExceptionTest
+{
+ private static Thread initThread;
+
+ @BeforeClass
+ public static void setUp()
+ {
+ DatabaseDescriptor.daemonInitialization();
+
+ if (DatabaseDescriptor.getDiskFailurePolicy() == Config.DiskFailurePolicy.die ||
+ DatabaseDescriptor.getDiskFailurePolicy() == Config.DiskFailurePolicy.ignore)
+ {
+ DatabaseDescriptor.setDiskFailurePolicy(Config.DiskFailurePolicy.stop);
+ }
+
+ DatabaseDescriptor.setCommitLogSegmentMgrProvider(c -> new MockCommitLogSegmentMgr(c, DatabaseDescriptor.getCommitLogLocation()));
+
+ JVMStabilityInspector.killerHook = (t) -> {
+ Assert.assertEquals("MOCK EXCEPTION: createSegment", t.getMessage());
+
+ try
+ {
+ // Avoid JVM exit. The JVM still needs to run other junit tests.
+ return false;
+ }
+ finally
+ {
+ Assert.assertNotNull(initThread);
+ // We have to manually stop init thread because the JVM does not exit actually.
+ initThread.stop();
+ }
+ };
+ }
+
+ @Test(timeout = 30000)
+ public void testCommitLogInitWithException() {
+ // This line will trigger initialization process because it's the first time to access CommitLog class.
+ initThread = new Thread(CommitLog.instance::start);
+
+ initThread.setName("initThread");
+ initThread.start();
+
+ try
+ {
+ initThread.join(); // Should not block here
+ }
+ catch (InterruptedException expected)
+ {
+ }
+
+ Assert.assertFalse(initThread.isAlive());
+
+ try
+ {
+ Thread.sleep(1000); // Wait for COMMIT-LOG-ALLOCATOR exit
+ }
+ catch (InterruptedException e)
+ {
+ Assert.fail();
+ }
+
+ Assert.assertEquals(Thread.State.TERMINATED, CommitLog.instance.segmentManager.managerThread.getState()); // exit successfully
+ }
+
+ private static class MockCommitLogSegmentMgr extends CommitLogSegmentManagerStandard {
+
+ public MockCommitLogSegmentMgr(CommitLog commitLog, String storageDirectory)
+ {
+ super(commitLog, storageDirectory);
+ }
+
+ @Override
+ public CommitLogSegment createSegment()
+ {
+ throw new RuntimeException("MOCK EXCEPTION: createSegment");
+ }
+ }
+
+}
diff --git a/test/unit/org/apache/cassandra/db/commitlog/CommitLogReaderTest.java b/test/unit/org/apache/cassandra/db/commitlog/CommitLogReaderTest.java
index edff3b7..ca76e45 100644
--- a/test/unit/org/apache/cassandra/db/commitlog/CommitLogReaderTest.java
+++ b/test/unit/org/apache/cassandra/db/commitlog/CommitLogReaderTest.java
@@ -27,8 +27,8 @@
import org.junit.BeforeClass;
import org.junit.Test;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.config.Config;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.CQLTester;
@@ -158,19 +158,19 @@
}
/**
- * Since we have both cfm and non mixed into the CL, we ignore updates that aren't for the cfm the test handler
+ * Since we have both table and non mixed into the CL, we ignore updates that aren't for the table the test handler
* is configured to check.
* @param handler
* @param offset integer offset of count we expect to see in record
*/
private void confirmReadOrder(TestCLRHandler handler, int offset)
{
- ColumnDefinition cd = currentTableMetadata().getColumnDefinition(new ColumnIdentifier("data", false));
+ ColumnMetadata cd = currentTableMetadata().getColumn(new ColumnIdentifier("data", false));
int i = 0;
int j = 0;
while (i + j < handler.seenMutationCount())
{
- PartitionUpdate pu = handler.seenMutations.get(i + j).get(currentTableMetadata());
+ PartitionUpdate pu = handler.seenMutations.get(i + j).getPartitionUpdate(currentTableMetadata());
if (pu == null)
{
j++;
@@ -208,17 +208,17 @@
public List<Mutation> seenMutations = new ArrayList<Mutation>();
public boolean sawStopOnErrorCheck = false;
- private final CFMetaData cfm;
+ private final TableMetadata metadata;
// Accept all
public TestCLRHandler()
{
- this.cfm = null;
+ this.metadata = null;
}
- public TestCLRHandler(CFMetaData cfm)
+ public TestCLRHandler(TableMetadata metadata)
{
- this.cfm = cfm;
+ this.metadata = metadata;
}
public boolean shouldSkipSegmentOnError(CommitLogReadException exception) throws IOException
@@ -234,7 +234,7 @@
public void handleMutation(Mutation m, int size, int entryLocation, CommitLogDescriptor desc)
{
- if ((cfm == null) || (cfm != null && m.get(cfm) != null)) {
+ if ((metadata == null) || (metadata != null && m.getPartitionUpdate(metadata) != null)) {
seenMutations.add(m);
}
}
diff --git a/test/unit/org/apache/cassandra/db/commitlog/CommitLogSegmentBackpressureTest.java b/test/unit/org/apache/cassandra/db/commitlog/CommitLogSegmentBackpressureTest.java
index 0076eb6..6b167b2 100644
--- a/test/unit/org/apache/cassandra/db/commitlog/CommitLogSegmentBackpressureTest.java
+++ b/test/unit/org/apache/cassandra/db/commitlog/CommitLogSegmentBackpressureTest.java
@@ -95,7 +95,7 @@
ColumnFamilyStore cfs1 = Keyspace.open(KEYSPACE1).getColumnFamilyStore(STANDARD1);
- final Mutation m = new RowUpdateBuilder(cfs1.metadata, 0, "k").clustering("bytes")
+ final Mutation m = new RowUpdateBuilder(cfs1.metadata(), 0, "k").clustering("bytes")
.add("val", ByteBuffer.wrap(entropy))
.build();
diff --git a/test/unit/org/apache/cassandra/db/commitlog/CommitLogSegmentManagerCDCTest.java b/test/unit/org/apache/cassandra/db/commitlog/CommitLogSegmentManagerCDCTest.java
index 92a4bf8..4128b71 100644
--- a/test/unit/org/apache/cassandra/db/commitlog/CommitLogSegmentManagerCDCTest.java
+++ b/test/unit/org/apache/cassandra/db/commitlog/CommitLogSegmentManagerCDCTest.java
@@ -18,57 +18,61 @@
package org.apache.cassandra.db.commitlog;
-import java.io.File;
-import java.io.IOException;
+import java.io.*;
import java.nio.ByteBuffer;
-import java.util.Random;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.*;
import org.junit.Assert;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
-import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.CQLTester;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.db.RowUpdateBuilder;
import org.apache.cassandra.db.commitlog.CommitLogSegment.CDCState;
-import org.apache.cassandra.exceptions.WriteTimeoutException;
+import org.apache.cassandra.exceptions.CDCWriteException;
import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.schema.TableMetadata;
public class CommitLogSegmentManagerCDCTest extends CQLTester
{
- private static Random random = new Random();
+ private static final Random random = new Random();
@BeforeClass
public static void setUpClass()
{
DatabaseDescriptor.setCDCEnabled(true);
+ DatabaseDescriptor.setCDCSpaceInMB(1024);
CQLTester.setUpClass();
}
@Before
- public void before() throws IOException
+ public void beforeTest() throws Throwable
{
- CommitLog.instance.resetUnsafe(true);
- for (File f : new File(DatabaseDescriptor.getCDCLogLocation()).listFiles())
- FileUtils.deleteWithConfirm(f);
+ super.beforeTest();
+ // Need to clean out any files from previous test runs. Prevents flaky test failures.
+ CommitLog.instance.stopUnsafe(true);
+ CommitLog.instance.start();
+ ((CommitLogSegmentManagerCDC)CommitLog.instance.segmentManager).updateCDCTotalSize();
}
@Test
- public void testCDCWriteTimeout() throws Throwable
+ public void testCDCWriteFailure() throws Throwable
{
createTable("CREATE TABLE %s (idx int, data text, primary key(idx)) WITH cdc=true;");
CommitLogSegmentManagerCDC cdcMgr = (CommitLogSegmentManagerCDC)CommitLog.instance.segmentManager;
- CFMetaData cfm = currentTableMetadata();
+ TableMetadata cfm = currentTableMetadata();
// Confirm that logic to check for whether or not we can allocate new CDC segments works
Integer originalCDCSize = DatabaseDescriptor.getCDCSpaceInMB();
try
{
DatabaseDescriptor.setCDCSpaceInMB(32);
- // Spin until we hit CDC capacity and make sure we get a WriteTimeout
+ // Spin until we hit CDC capacity and make sure we get a CDCWriteException
try
{
// Should trigger on anything < 20:1 compression ratio during compressed test
@@ -78,9 +82,9 @@
.add("data", randomizeBuffer(DatabaseDescriptor.getCommitLogSegmentSize() / 3))
.build().apply();
}
- Assert.fail("Expected WriteTimeoutException from full CDC but did not receive it.");
+ Assert.fail("Expected CDCWriteException from full CDC but did not receive it.");
}
- catch (WriteTimeoutException e)
+ catch (CDCWriteException e)
{
// expected, do nothing
}
@@ -111,45 +115,6 @@
}
@Test
- public void testCLSMCDCDiscardLogic() throws Throwable
- {
- CommitLogSegmentManagerCDC cdcMgr = (CommitLogSegmentManagerCDC)CommitLog.instance.segmentManager;
-
- createTable("CREATE TABLE %s (idx int, data text, primary key(idx)) WITH cdc=false;");
- for (int i = 0; i < 8; i++)
- {
- new RowUpdateBuilder(currentTableMetadata(), 0, i)
- .add("data", randomizeBuffer(DatabaseDescriptor.getCommitLogSegmentSize() / 4)) // fit 3 in a segment
- .build().apply();
- }
-
- // Should have 4 segments CDC since we haven't flushed yet, 3 PERMITTED, one of which is active, and 1 PERMITTED, in waiting
- Assert.assertEquals(4 * DatabaseDescriptor.getCommitLogSegmentSize(), cdcMgr.updateCDCTotalSize());
- expectCurrentCDCState(CDCState.PERMITTED);
- CommitLog.instance.forceRecycleAllSegments();
-
- // on flush, these PERMITTED should be deleted
- Assert.assertEquals(0, new File(DatabaseDescriptor.getCDCLogLocation()).listFiles().length);
-
- createTable("CREATE TABLE %s (idx int, data text, primary key(idx)) WITH cdc=true;");
- for (int i = 0; i < 8; i++)
- {
- new RowUpdateBuilder(currentTableMetadata(), 0, i)
- .add("data", randomizeBuffer(DatabaseDescriptor.getCommitLogSegmentSize() / 4))
- .build().apply();
- }
- // 4 total again, 3 CONTAINS, 1 in waiting PERMITTED
- Assert.assertEquals(4 * DatabaseDescriptor.getCommitLogSegmentSize(), cdcMgr.updateCDCTotalSize());
- CommitLog.instance.forceRecycleAllSegments();
- expectCurrentCDCState(CDCState.PERMITTED);
-
- // On flush, PERMITTED is deleted, CONTAINS is preserved.
- cdcMgr.awaitManagementTasksCompletion();
- int seen = getCDCRawCount();
- Assert.assertTrue("Expected >3 files in cdc_raw, saw: " + seen, seen >= 3);
- }
-
- @Test
public void testSegmentFlaggingOnCreation() throws Throwable
{
CommitLogSegmentManagerCDC cdcMgr = (CommitLogSegmentManagerCDC)CommitLog.instance.segmentManager;
@@ -159,8 +124,8 @@
try
{
DatabaseDescriptor.setCDCSpaceInMB(16);
- CFMetaData ccfm = Keyspace.open(keyspace()).getColumnFamilyStore(ct).metadata;
- // Spin until we hit CDC capacity and make sure we get a WriteTimeout
+ TableMetadata ccfm = Keyspace.open(keyspace()).getColumnFamilyStore(ct).metadata();
+ // Spin until we hit CDC capacity and make sure we get a CDCWriteException
try
{
for (int i = 0; i < 1000; i++)
@@ -169,15 +134,17 @@
.add("data", randomizeBuffer(DatabaseDescriptor.getCommitLogSegmentSize() / 3))
.build().apply();
}
- Assert.fail("Expected WriteTimeoutException from full CDC but did not receive it.");
+ Assert.fail("Expected CDCWriteException from full CDC but did not receive it.");
}
- catch (WriteTimeoutException e) { }
+ catch (CDCWriteException e) { }
expectCurrentCDCState(CDCState.FORBIDDEN);
CommitLog.instance.forceRecycleAllSegments();
cdcMgr.awaitManagementTasksCompletion();
- new File(DatabaseDescriptor.getCDCLogLocation()).listFiles()[0].delete();
+ // Delete all files in cdc_raw
+ for (File f : new File(DatabaseDescriptor.getCDCLogLocation()).listFiles())
+ f.delete();
cdcMgr.updateCDCTotalSize();
// Confirm cdc update process changes flag on active segment
expectCurrentCDCState(CDCState.PERMITTED);
@@ -186,12 +153,6 @@
for (File f : new File(DatabaseDescriptor.getCDCLogLocation()).listFiles()) {
FileUtils.deleteWithConfirm(f);
}
-
- // Set space to 0, confirm newly allocated segments are FORBIDDEN
- DatabaseDescriptor.setCDCSpaceInMB(0);
- CommitLog.instance.forceRecycleAllSegments();
- CommitLog.instance.segmentManager.awaitManagementTasksCompletion();
- expectCurrentCDCState(CDCState.FORBIDDEN);
}
finally
{
@@ -199,6 +160,270 @@
}
}
+ @Test
+ public void testCDCIndexFileWriteOnSync() throws IOException
+ {
+ createTable("CREATE TABLE %s (idx int, data text, primary key(idx)) WITH cdc=true;");
+ new RowUpdateBuilder(currentTableMetadata(), 0, 1)
+ .add("data", randomizeBuffer(DatabaseDescriptor.getCommitLogSegmentSize() / 3))
+ .build().apply();
+
+ CommitLog.instance.sync(true);
+ CommitLogSegment currentSegment = CommitLog.instance.segmentManager.allocatingFrom();
+ int syncOffset = currentSegment.lastSyncedOffset;
+
+ // Confirm index file is written
+ File cdcIndexFile = currentSegment.getCDCIndexFile();
+ Assert.assertTrue("Index file not written: " + cdcIndexFile, cdcIndexFile.exists());
+
+ // Read index value and confirm it's == end from last sync
+ BufferedReader in = new BufferedReader(new FileReader(cdcIndexFile));
+ String input = in.readLine();
+ Integer offset = Integer.parseInt(input);
+ Assert.assertEquals(syncOffset, (long)offset);
+ in.close();
+ }
+
+ @Test
+ public void testCompletedFlag() throws IOException
+ {
+ createTable("CREATE TABLE %s (idx int, data text, primary key(idx)) WITH cdc=true;");
+ CommitLogSegment initialSegment = CommitLog.instance.segmentManager.allocatingFrom();
+ Integer originalCDCSize = DatabaseDescriptor.getCDCSpaceInMB();
+
+ DatabaseDescriptor.setCDCSpaceInMB(8);
+ try
+ {
+ for (int i = 0; i < 1000; i++)
+ {
+ new RowUpdateBuilder(currentTableMetadata(), 0, 1)
+ .add("data", randomizeBuffer(DatabaseDescriptor.getCommitLogSegmentSize() / 3))
+ .build().apply();
+ }
+ }
+ catch (CDCWriteException ce)
+ {
+ // pass. Expected since we'll have a file or two linked on restart of CommitLog due to replay
+ }
+ finally
+ {
+ DatabaseDescriptor.setCDCSpaceInMB(originalCDCSize);
+ }
+
+ CommitLog.instance.forceRecycleAllSegments();
+
+ // Confirm index file is written
+ File cdcIndexFile = initialSegment.getCDCIndexFile();
+ Assert.assertTrue("Index file not written: " + cdcIndexFile, cdcIndexFile.exists());
+
+ // Read index file and confirm second line is COMPLETED
+ BufferedReader in = new BufferedReader(new FileReader(cdcIndexFile));
+ String input = in.readLine();
+ input = in.readLine();
+ Assert.assertTrue("Expected COMPLETED in index file, got: " + input, input.equals("COMPLETED"));
+ in.close();
+ }
+
+ @Test
+ public void testDeleteLinkOnDiscardNoCDC() throws Throwable
+ {
+ createTable("CREATE TABLE %s (idx int, data text, primary key(idx)) WITH cdc=false;");
+ new RowUpdateBuilder(currentTableMetadata(), 0, 1)
+ .add("data", randomizeBuffer(DatabaseDescriptor.getCommitLogSegmentSize() / 3))
+ .build().apply();
+ CommitLogSegment currentSegment = CommitLog.instance.segmentManager.allocatingFrom();
+
+ // Confirm that, with no CDC data present, we've hard-linked but have no index file
+ Path linked = new File(DatabaseDescriptor.getCDCLogLocation(), currentSegment.logFile.getName()).toPath();
+ File cdcIndexFile = currentSegment.getCDCIndexFile();
+ Assert.assertTrue("File does not exist: " + linked, Files.exists(linked));
+ Assert.assertFalse("Expected index file to not be created but found: " + cdcIndexFile, cdcIndexFile.exists());
+
+ // Sync and confirm no index written as index is written on flush
+ CommitLog.instance.sync(true);
+ Assert.assertTrue("File does not exist: " + linked, Files.exists(linked));
+ Assert.assertFalse("Expected index file to not be created but found: " + cdcIndexFile, cdcIndexFile.exists());
+
+ // Force a full recycle and confirm hard-link is deleted
+ CommitLog.instance.forceRecycleAllSegments();
+ CommitLog.instance.segmentManager.awaitManagementTasksCompletion();
+ Assert.assertFalse("Expected hard link to CLS to be deleted on non-cdc segment: " + linked, Files.exists(linked));
+ }
+
+ @Test
+ public void testRetainLinkOnDiscardCDC() throws Throwable
+ {
+ createTable("CREATE TABLE %s (idx int, data text, primary key(idx)) WITH cdc=true;");
+ CommitLogSegment currentSegment = CommitLog.instance.segmentManager.allocatingFrom();
+ File cdcIndexFile = currentSegment.getCDCIndexFile();
+ Assert.assertFalse("Expected no index file before flush but found: " + cdcIndexFile, cdcIndexFile.exists());
+
+ new RowUpdateBuilder(currentTableMetadata(), 0, 1)
+ .add("data", randomizeBuffer(DatabaseDescriptor.getCommitLogSegmentSize() / 3))
+ .build().apply();
+
+ Path linked = new File(DatabaseDescriptor.getCDCLogLocation(), currentSegment.logFile.getName()).toPath();
+ // Confirm that, with CDC data present but not yet flushed, we've hard-linked but have no index file
+ Assert.assertTrue("File does not exist: " + linked, Files.exists(linked));
+
+ // Sync and confirm index written as index is written on flush
+ CommitLog.instance.sync(true);
+ Assert.assertTrue("File does not exist: " + linked, Files.exists(linked));
+ Assert.assertTrue("Expected cdc index file after flush but found none: " + cdcIndexFile, cdcIndexFile.exists());
+
+ // Force a full recycle and confirm all files remain
+ CommitLog.instance.forceRecycleAllSegments();
+ Assert.assertTrue("File does not exist: " + linked, Files.exists(linked));
+ Assert.assertTrue("Expected cdc index file after recycle but found none: " + cdcIndexFile, cdcIndexFile.exists());
+ }
+
+ @Test
+ public void testReplayLogic() throws IOException
+ {
+ // Assert.assertEquals(0, new File(DatabaseDescriptor.getCDCLogLocation()).listFiles().length);
+ String table_name = createTable("CREATE TABLE %s (idx int, data text, primary key(idx)) WITH cdc=true;");
+ Integer originalCDCSize = DatabaseDescriptor.getCDCSpaceInMB();
+
+ DatabaseDescriptor.setCDCSpaceInMB(8);
+ TableMetadata ccfm = Keyspace.open(keyspace()).getColumnFamilyStore(table_name).metadata();
+ try
+ {
+ for (int i = 0; i < 1000; i++)
+ {
+ new RowUpdateBuilder(ccfm, 0, i)
+ .add("data", randomizeBuffer(DatabaseDescriptor.getCommitLogSegmentSize() / 3))
+ .build().apply();
+ }
+ Assert.fail("Expected CDCWriteException from full CDC but did not receive it.");
+ }
+ catch (CDCWriteException e)
+ {
+ // pass
+ }
+ finally
+ {
+ DatabaseDescriptor.setCDCSpaceInMB(originalCDCSize);
+ }
+
+ CommitLog.instance.sync(true);
+ CommitLog.instance.stopUnsafe(false);
+
+ // Build up a list of expected index files after replay and then clear out cdc_raw
+ List<CDCIndexData> oldData = parseCDCIndexData();
+ for (File f : new File(DatabaseDescriptor.getCDCLogLocation()).listFiles())
+ FileUtils.deleteWithConfirm(f.getAbsolutePath());
+
+ try
+ {
+ Assert.assertEquals("Expected 0 files in CDC folder after deletion. ",
+ 0, new File(DatabaseDescriptor.getCDCLogLocation()).listFiles().length);
+ }
+ finally
+ {
+ // If we don't have a started commitlog, assertions will cause the test to hang. I assume it's some assumption
+ // hang in the shutdown on CQLTester trying to clean up / drop keyspaces / tables and hanging applying
+ // mutations.
+ CommitLog.instance.start();
+ CommitLog.instance.segmentManager.awaitManagementTasksCompletion();
+ }
+ CDCTestReplayer replayer = new CDCTestReplayer();
+ replayer.examineCommitLog();
+
+ // Rough sanity check -> should be files there now.
+ Assert.assertTrue("Expected non-zero number of files in CDC folder after restart.",
+ new File(DatabaseDescriptor.getCDCLogLocation()).listFiles().length > 0);
+
+ // Confirm all the old indexes in old are present and >= the original offset, as we flag the entire segment
+ // as cdc written on a replay.
+ List<CDCIndexData> newData = parseCDCIndexData();
+ for (CDCIndexData cid : oldData)
+ {
+ boolean found = false;
+ for (CDCIndexData ncid : newData)
+ {
+ if (cid.fileName.equals(ncid.fileName))
+ {
+ Assert.assertTrue("New CDC index file expected to have >= offset in old.", ncid.offset >= cid.offset);
+ found = true;
+ }
+ }
+ if (!found)
+ {
+ StringBuilder errorMessage = new StringBuilder();
+ errorMessage.append(String.format("Missing old CDCIndexData in new set after replay: %s\n", cid));
+ errorMessage.append("List of CDCIndexData in new set of indexes after replay:\n");
+ for (CDCIndexData ncid : newData)
+ errorMessage.append(String.format(" %s\n", ncid));
+ Assert.fail(errorMessage.toString());
+ }
+ }
+
+ // And make sure we don't have new CDC Indexes we don't expect
+ for (CDCIndexData ncid : newData)
+ {
+ boolean found = false;
+ for (CDCIndexData cid : oldData)
+ {
+ if (cid.fileName.equals(ncid.fileName))
+ found = true;
+ }
+ if (!found)
+ Assert.fail(String.format("Unexpected new CDCIndexData found after replay: %s\n", ncid));
+ }
+ }
+
+ private List<CDCIndexData> parseCDCIndexData()
+ {
+ List<CDCIndexData> results = new ArrayList<>();
+ try
+ {
+ for (File f : new File(DatabaseDescriptor.getCDCLogLocation()).listFiles())
+ {
+ if (f.getName().contains("_cdc.idx"))
+ results.add(new CDCIndexData(f));
+ }
+ }
+ catch (IOException e)
+ {
+ Assert.fail(String.format("Failed to parse CDCIndexData: %s", e.getMessage()));
+ }
+ return results;
+ }
+
+ private static class CDCIndexData
+ {
+ private final String fileName;
+ private final int offset;
+
+ CDCIndexData(File f) throws IOException
+ {
+ String line = "";
+ try (BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(f))))
+ {
+ line = br.readLine();
+ }
+ catch (Exception e)
+ {
+ throw e;
+ }
+ fileName = f.getName();
+ offset = Integer.parseInt(line);
+ }
+
+ @Override
+ public String toString()
+ {
+ return String.format("%s,%d", fileName, offset);
+ }
+
+ @Override
+ public boolean equals(Object other)
+ {
+ CDCIndexData cid = (CDCIndexData)other;
+ return fileName.equals(cid.fileName) && offset == cid.offset;
+ }
+ }
+
private ByteBuffer randomizeBuffer(int size)
{
byte[] toWrap = new byte[size];
@@ -211,9 +436,15 @@
return new File(DatabaseDescriptor.getCDCLogLocation()).listFiles().length;
}
- private void expectCurrentCDCState(CDCState state)
+ private void expectCurrentCDCState(CDCState expectedState)
{
- Assert.assertEquals("Received unexpected CDCState on current allocatingFrom segment.",
- state, CommitLog.instance.segmentManager.allocatingFrom().getCDCState());
+ CDCState currentState = CommitLog.instance.segmentManager.allocatingFrom().getCDCState();
+ if (currentState != expectedState)
+ {
+ logger.error("expectCurrentCDCState violation! Expected state: {}. Found state: {}. Current CDC allocation: {}",
+ expectedState, currentState, ((CommitLogSegmentManagerCDC)CommitLog.instance.segmentManager).updateCDCTotalSize());
+ Assert.fail(String.format("Received unexpected CDCState on current allocatingFrom segment. Expected: %s. Received: %s",
+ expectedState, currentState));
+ }
}
}
diff --git a/test/unit/org/apache/cassandra/db/commitlog/CommitLogTest.java b/test/unit/org/apache/cassandra/db/commitlog/CommitLogTest.java
index 895cfd0..0e7f30d 100644
--- a/test/unit/org/apache/cassandra/db/commitlog/CommitLogTest.java
+++ b/test/unit/org/apache/cassandra/db/commitlog/CommitLogTest.java
@@ -19,6 +19,7 @@
package org.apache.cassandra.db.commitlog;
import java.io.*;
+import java.math.BigInteger;
import java.nio.ByteBuffer;
import java.util.*;
import java.util.concurrent.Callable;
@@ -29,16 +30,19 @@
import java.util.zip.Checksum;
import com.google.common.collect.Iterables;
+import com.google.common.io.Files;
import org.junit.*;
-import com.google.common.io.Files;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameters;
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.Util;
-import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.io.compress.ZstdCompressor;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.config.ParameterizedClass;
import org.apache.cassandra.config.Config.DiskFailurePolicy;
@@ -66,16 +70,26 @@
import org.apache.cassandra.utils.vint.VIntCoding;
import org.junit.After;
+
+import static org.apache.cassandra.db.commitlog.CommitLogSegment.ENTRY_OVERHEAD_SIZE;
import static org.apache.cassandra.utils.ByteBufferUtil.bytes;
+
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
+import org.apache.cassandra.db.marshal.IntegerType;
+import org.apache.cassandra.db.marshal.MapType;
+import org.apache.cassandra.db.marshal.SetType;
+import org.apache.cassandra.db.marshal.UTF8Type;
+
+@Ignore
@RunWith(Parameterized.class)
-public class CommitLogTest
+public abstract class CommitLogTest
{
- private static final String KEYSPACE1 = "CommitLogTest";
+ protected static final String KEYSPACE1 = "CommitLogTest";
private static final String KEYSPACE2 = "CommitLogTestNonDurable";
- private static final String STANDARD1 = "Standard1";
+ protected static final String STANDARD1 = "Standard1";
private static final String STANDARD2 = "Standard2";
private static final String CUSTOM1 = "Custom1";
@@ -96,10 +110,10 @@
{null, EncryptionContextGenerator.createContext(true)}, // Encryption
{new ParameterizedClass(LZ4Compressor.class.getName(), Collections.emptyMap()), EncryptionContextGenerator.createDisabledContext()},
{new ParameterizedClass(SnappyCompressor.class.getName(), Collections.emptyMap()), EncryptionContextGenerator.createDisabledContext()},
- {new ParameterizedClass(DeflateCompressor.class.getName(), Collections.emptyMap()), EncryptionContextGenerator.createDisabledContext()}});
+ {new ParameterizedClass(DeflateCompressor.class.getName(), Collections.emptyMap()), EncryptionContextGenerator.createDisabledContext()},
+ {new ParameterizedClass(ZstdCompressor.class.getName(), Collections.emptyMap()), EncryptionContextGenerator.createDisabledContext()}});
}
- @BeforeClass
public static void beforeClass() throws ConfigurationException
{
// Disable durable writes for system keyspaces to prevent system mutations, e.g. sstable_activity,
@@ -109,13 +123,12 @@
SchemaLoader.prepareServer();
- CFMetaData custom = CFMetaData.compile(String.format("CREATE TABLE \"%s\" (" +
- "k int," +
- "c1 frozen<map<text, text>>," +
- "c2 frozen<set<text>>," +
- "s int static," +
- "PRIMARY KEY (k, c1, c2)" +
- ");", CUSTOM1), KEYSPACE1);
+ TableMetadata.Builder custom =
+ TableMetadata.builder(KEYSPACE1, CUSTOM1)
+ .addPartitionKeyColumn("k", IntegerType.instance)
+ .addClusteringColumn("c1", MapType.getInstance(UTF8Type.instance, UTF8Type.instance, false))
+ .addClusteringColumn("c2", SetType.getInstance(UTF8Type.instance, false))
+ .addStaticColumn("s", IntegerType.instance);
SchemaLoader.createKeyspace(KEYSPACE1,
KeyspaceParams.simple(1),
@@ -167,10 +180,9 @@
}
@Test
- public void testRecoveryWithFinalEmptyLog() throws Exception
+ public void testRecoveryWithEmptyFinalLog() throws Exception
{
- // Even though it's empty, it's the last commitlog segment, so allowTruncation=true should allow it to pass
- CommitLog.instance.recoverFiles(new File[]{tmpFile(CommitLogDescriptor.current_version)});
+ CommitLog.instance.recoverFiles(tmpFile(CommitLogDescriptor.current_version));
}
/**
@@ -226,15 +238,9 @@
}
@Test
- public void testRecoveryWithEmptyLog20() throws Exception
- {
- CommitLog.instance.recoverFiles(tmpFile(CommitLogDescriptor.VERSION_20));
- }
-
- @Test
public void testRecoveryWithZeroLog() throws Exception
{
- testRecovery(new byte[10], null);
+ testRecovery(new byte[10], CommitLogReplayException.class);
}
@Test
@@ -245,23 +251,10 @@
}
@Test
- public void testRecoveryWithShortPadding() throws Exception
- {
- // If we have 0-3 bytes remaining, commitlog replayer
- // should pass, because there's insufficient room
- // left in the segment for the legacy size marker.
- testRecovery(new byte[1], null);
- testRecovery(new byte[2], null);
- testRecovery(new byte[3], null);
- }
-
- @Test
public void testRecoveryWithShortSize() throws Exception
{
- byte[] data = new byte[5];
- data[3] = 1; // Not a legacy marker, give it a fake (short) size
runExpecting(() -> {
- testRecovery(data, CommitLogDescriptor.VERSION_20);
+ testRecovery(new byte[2], CommitLogDescriptor.current_version);
return null;
}, CommitLogReplayException.class);
}
@@ -322,7 +315,7 @@
ColumnFamilyStore cfs2 = ks.getColumnFamilyStore(STANDARD2);
// Roughly 32 MB mutation
- Mutation m = new RowUpdateBuilder(cfs1.metadata, 0, "k")
+ Mutation m = new RowUpdateBuilder(cfs1.metadata(), 0, "k")
.clustering("bytes")
.add("val", ByteBuffer.allocate(DatabaseDescriptor.getCommitLogSegmentSize() / 4))
.build();
@@ -335,7 +328,7 @@
CommitLog.instance.add(m);
// Adding new mutation on another CF
- Mutation m2 = new RowUpdateBuilder(cfs2.metadata, 0, "k")
+ Mutation m2 = new RowUpdateBuilder(cfs2.metadata(), 0, "k")
.clustering("bytes")
.add("val", ByteBuffer.allocate(4))
.build();
@@ -343,8 +336,8 @@
assertEquals(2, CommitLog.instance.segmentManager.getActiveSegments().size());
- UUID cfid2 = m2.getColumnFamilyIds().iterator().next();
- CommitLog.instance.discardCompletedSegments(cfid2, CommitLogPosition.NONE, CommitLog.instance.getCurrentPosition());
+ TableId id2 = m2.getTableIds().iterator().next();
+ CommitLog.instance.discardCompletedSegments(id2, CommitLogPosition.NONE, CommitLog.instance.getCurrentPosition());
// Assert we still have both our segments
assertEquals(2, CommitLog.instance.segmentManager.getActiveSegments().size());
@@ -358,7 +351,7 @@
ColumnFamilyStore cfs2 = ks.getColumnFamilyStore(STANDARD2);
// Roughly 32 MB mutation
- Mutation rm = new RowUpdateBuilder(cfs1.metadata, 0, "k")
+ Mutation rm = new RowUpdateBuilder(cfs1.metadata(), 0, "k")
.clustering("bytes")
.add("val", ByteBuffer.allocate((DatabaseDescriptor.getCommitLogSegmentSize()/4) - 1))
.build();
@@ -370,14 +363,14 @@
assertEquals(1, CommitLog.instance.segmentManager.getActiveSegments().size());
// "Flush": this won't delete anything
- UUID cfid1 = rm.getColumnFamilyIds().iterator().next();
+ TableId id1 = rm.getTableIds().iterator().next();
CommitLog.instance.sync(true);
- CommitLog.instance.discardCompletedSegments(cfid1, CommitLogPosition.NONE, CommitLog.instance.getCurrentPosition());
+ CommitLog.instance.discardCompletedSegments(id1, CommitLogPosition.NONE, CommitLog.instance.getCurrentPosition());
assertEquals(1, CommitLog.instance.segmentManager.getActiveSegments().size());
// Adding new mutation on another CF, large enough (including CL entry overhead) that a new segment is created
- Mutation rm2 = new RowUpdateBuilder(cfs2.metadata, 0, "k")
+ Mutation rm2 = new RowUpdateBuilder(cfs2.metadata(), 0, "k")
.clustering("bytes")
.add("val", ByteBuffer.allocate(DatabaseDescriptor.getMaxMutationSize() - 200))
.build();
@@ -395,8 +388,8 @@
// "Flush" second cf: The first segment should be deleted since we
// didn't write anything on cf1 since last flush (and we flush cf2)
- UUID cfid2 = rm2.getColumnFamilyIds().iterator().next();
- CommitLog.instance.discardCompletedSegments(cfid2, CommitLogPosition.NONE, CommitLog.instance.getCurrentPosition());
+ TableId id2 = rm2.getTableIds().iterator().next();
+ CommitLog.instance.discardCompletedSegments(id2, CommitLogPosition.NONE, CommitLog.instance.getCurrentPosition());
segments = CommitLog.instance.segmentManager.getActiveSegments();
@@ -408,9 +401,9 @@
private String getDirtyCFIds(Collection<CommitLogSegment> segments)
{
- return "Dirty cfIds: <"
+ return "Dirty tableIds: <"
+ String.join(", ", segments.stream()
- .map(CommitLogSegment::getDirtyCFIDs)
+ .map(CommitLogSegment::getDirtyTableIds)
.flatMap(uuids -> uuids.stream())
.distinct()
.map(uuid -> uuid.toString()).collect(Collectors.toList()))
@@ -423,7 +416,7 @@
// We don't want to allocate a size of 0 as this is optimized under the hood and our computation would
// break testEqualRecordLimit
int allocSize = 1;
- Mutation rm = new RowUpdateBuilder(cfs.metadata, 0, key)
+ Mutation rm = new RowUpdateBuilder(cfs.metadata(), 0, key)
.clustering(colName)
.add("val", ByteBuffer.allocate(allocSize)).build();
@@ -431,7 +424,7 @@
max -= CommitLogSegment.ENTRY_OVERHEAD_SIZE; // log entry overhead
// Note that the size of the value if vint encoded. So we first compute the ovehead of the mutation without the value and it's size
- int mutationOverhead = (int)Mutation.serializer.serializedSize(rm, MessagingService.current_version) - (VIntCoding.computeVIntSize(allocSize) + allocSize);
+ int mutationOverhead = rm.serializedSize(MessagingService.current_version) - (VIntCoding.computeVIntSize(allocSize) + allocSize);
max -= mutationOverhead;
// Now, max is the max for both the value and it's size. But we want to know how much we can allocate, i.e. the size of the value.
@@ -451,25 +444,66 @@
public void testEqualRecordLimit() throws Exception
{
ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(STANDARD1);
- Mutation rm = new RowUpdateBuilder(cfs.metadata, 0, "k")
+ Mutation rm = new RowUpdateBuilder(cfs.metadata(), 0, "k")
.clustering("bytes")
.add("val", ByteBuffer.allocate(getMaxRecordDataSize()))
.build();
CommitLog.instance.add(rm);
}
- @Test(expected = IllegalArgumentException.class)
+ @Test(expected = MutationExceededMaxSizeException.class)
public void testExceedRecordLimit() throws Exception
{
Keyspace ks = Keyspace.open(KEYSPACE1);
ColumnFamilyStore cfs = ks.getColumnFamilyStore(STANDARD1);
- Mutation rm = new RowUpdateBuilder(cfs.metadata, 0, "k")
+ Mutation rm = new RowUpdateBuilder(cfs.metadata(), 0, "k")
.clustering("bytes")
.add("val", ByteBuffer.allocate(1 + getMaxRecordDataSize()))
.build();
CommitLog.instance.add(rm);
throw new AssertionError("mutation larger than limit was accepted");
}
+ @Test
+ public void testExceedRecordLimitWithMultiplePartitions() throws Exception
+ {
+ CommitLog.instance.resetUnsafe(true);
+ List<Mutation> mutations = new ArrayList<>();
+ Keyspace ks = Keyspace.open(KEYSPACE1);
+ char[] keyChars = new char[MutationExceededMaxSizeException.PARTITION_MESSAGE_LIMIT];
+ Arrays.fill(keyChars, 'k');
+ String key = new String(keyChars);
+
+ // large mutation
+ mutations.add(new RowUpdateBuilder(ks.getColumnFamilyStore(STANDARD1).metadata(), 0, key)
+ .clustering("bytes")
+ .add("val", ByteBuffer.allocate(1 + getMaxRecordDataSize()))
+ .build());
+
+ // smaller mutation
+ mutations.add(new RowUpdateBuilder(ks.getColumnFamilyStore(STANDARD2).metadata(), 0, key)
+ .clustering("bytes")
+ .add("val", ByteBuffer.allocate(1 + getMaxRecordDataSize() - 1024))
+ .build());
+
+ Mutation mutation = Mutation.merge(mutations);
+ try
+ {
+ CommitLog.instance.add(Mutation.merge(mutations));
+ throw new AssertionError("mutation larger than limit was accepted");
+ }
+ catch (MutationExceededMaxSizeException exception)
+ {
+ String message = exception.getMessage();
+
+ long mutationSize = mutation.serializedSize(MessagingService.current_version) + ENTRY_OVERHEAD_SIZE;
+ final String expectedMessagePrefix = String.format("Encountered an oversized mutation (%d/%d) for keyspace: %s.",
+ mutationSize,
+ DatabaseDescriptor.getMaxMutationSize(),
+ KEYSPACE1);
+ assertTrue(message.startsWith(expectedMessagePrefix));
+ assertTrue(message.contains(String.format("%s.%s and 1 more.", STANDARD1, key)));
+ }
+ }
protected void testRecoveryWithBadSizeArgument(int size, int dataSize) throws Exception
{
@@ -530,9 +564,9 @@
return Collections.singletonMap(EncryptionContext.ENCRYPTION_IV, Hex.bytesToHex(buf));
}
- protected File tmpFile(int version) throws IOException
+ protected File tmpFile(int version)
{
- File logFile = File.createTempFile("CommitLog-" + version + "-", ".log");
+ File logFile = FileUtils.createTempFile("CommitLog-" + version + "-", ".log");
assert logFile.length() == 0;
return logFile;
}
@@ -618,8 +652,7 @@
{
ParameterizedClass commitLogCompression = DatabaseDescriptor.getCommitLogCompression();
EncryptionContext encryptionContext = DatabaseDescriptor.getEncryptionContext();
- runExpecting(() -> testRecovery(logData, CommitLogDescriptor.VERSION_20), expected);
- runExpecting(() -> testRecovery(new CommitLogDescriptor(4, commitLogCompression, encryptionContext), logData), expected);
+ runExpecting(() -> testRecovery(logData, CommitLogDescriptor.current_version), expected);
}
@Test
@@ -634,10 +667,10 @@
ColumnFamilyStore cfs1 = ks.getColumnFamilyStore(STANDARD1);
ColumnFamilyStore cfs2 = ks.getColumnFamilyStore(STANDARD2);
- new RowUpdateBuilder(cfs1.metadata, 0, "k").clustering("bytes").add("val", ByteBuffer.allocate(100)).build().applyUnsafe();
+ new RowUpdateBuilder(cfs1.metadata(), 0, "k").clustering("bytes").add("val", ByteBuffer.allocate(100)).build().applyUnsafe();
cfs1.truncateBlocking();
DatabaseDescriptor.setAutoSnapshot(prev);
- Mutation m2 = new RowUpdateBuilder(cfs2.metadata, 0, "k")
+ Mutation m2 = new RowUpdateBuilder(cfs2.metadata(), 0, "k")
.clustering("bytes")
.add("val", ByteBuffer.allocate(DatabaseDescriptor.getCommitLogSegmentSize() / 4))
.build();
@@ -649,8 +682,8 @@
CommitLogPosition position = CommitLog.instance.getCurrentPosition();
for (Keyspace keyspace : Keyspace.system())
for (ColumnFamilyStore syscfs : keyspace.getColumnFamilyStores())
- CommitLog.instance.discardCompletedSegments(syscfs.metadata.cfId, CommitLogPosition.NONE, position);
- CommitLog.instance.discardCompletedSegments(cfs2.metadata.cfId, CommitLogPosition.NONE, position);
+ CommitLog.instance.discardCompletedSegments(syscfs.metadata().id, CommitLogPosition.NONE, position);
+ CommitLog.instance.discardCompletedSegments(cfs2.metadata().id, CommitLogPosition.NONE, position);
assertEquals(1, CommitLog.instance.segmentManager.getActiveSegments().size());
}
finally
@@ -667,10 +700,10 @@
{
DatabaseDescriptor.setAutoSnapshot(false);
Keyspace notDurableKs = Keyspace.open(KEYSPACE2);
- Assert.assertFalse(notDurableKs.getMetadata().params.durableWrites);
+ assertFalse(notDurableKs.getMetadata().params.durableWrites);
ColumnFamilyStore cfs = notDurableKs.getColumnFamilyStore("Standard1");
- new RowUpdateBuilder(cfs.metadata, 0, "key1")
+ new RowUpdateBuilder(cfs.metadata(), 0, "key1")
.clustering("bytes").add("val", bytes("abcd"))
.build()
.applyUnsafe();
@@ -693,14 +726,14 @@
{
int cellCount = 0;
ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(STANDARD1);
- final Mutation rm1 = new RowUpdateBuilder(cfs.metadata, 0, "k1")
+ final Mutation rm1 = new RowUpdateBuilder(cfs.metadata(), 0, "k1")
.clustering("bytes")
.add("val", bytes("this is a string"))
.build();
cellCount += 1;
CommitLog.instance.add(rm1);
- final Mutation rm2 = new RowUpdateBuilder(cfs.metadata, 0, "k2")
+ final Mutation rm2 = new RowUpdateBuilder(cfs.metadata(), 0, "k2")
.clustering("bytes")
.add("val", bytes("this is a string"))
.build();
@@ -709,9 +742,9 @@
CommitLog.instance.sync(true);
- SimpleCountingReplayer replayer = new SimpleCountingReplayer(CommitLog.instance, CommitLogPosition.NONE, cfs.metadata);
+ SimpleCountingReplayer replayer = new SimpleCountingReplayer(CommitLog.instance, CommitLogPosition.NONE, cfs.metadata());
List<String> activeSegments = CommitLog.instance.getActiveSegmentNames();
- Assert.assertFalse(activeSegments.isEmpty());
+ assertFalse(activeSegments.isEmpty());
File[] files = new File(CommitLog.instance.segmentManager.storageDirectory).listFiles((file, name) -> activeSegments.contains(name));
replayer.replayFiles(files);
@@ -730,7 +763,7 @@
for (int i = 0; i < max; i++)
{
- final Mutation rm1 = new RowUpdateBuilder(cfs.metadata, 0, "k" + 1)
+ final Mutation rm1 = new RowUpdateBuilder(cfs.metadata(), 0, "k" + 1)
.clustering("bytes")
.add("val", bytes("this is a string"))
.build();
@@ -746,9 +779,9 @@
CommitLog.instance.sync(true);
- SimpleCountingReplayer replayer = new SimpleCountingReplayer(CommitLog.instance, commitLogPosition, cfs.metadata);
+ SimpleCountingReplayer replayer = new SimpleCountingReplayer(CommitLog.instance, commitLogPosition, cfs.metadata());
List<String> activeSegments = CommitLog.instance.getActiveSegmentNames();
- Assert.assertFalse(activeSegments.isEmpty());
+ assertFalse(activeSegments.isEmpty());
File[] files = new File(CommitLog.instance.segmentManager.storageDirectory).listFiles((file, name) -> activeSegments.contains(name));
replayer.replayFiles(files);
@@ -759,15 +792,15 @@
class SimpleCountingReplayer extends CommitLogReplayer
{
private final CommitLogPosition filterPosition;
- private final CFMetaData metadata;
+ private final TableMetadata metadata;
int cells;
int skipped;
- SimpleCountingReplayer(CommitLog commitLog, CommitLogPosition filterPosition, CFMetaData cfm)
+ SimpleCountingReplayer(CommitLog commitLog, CommitLogPosition filterPosition, TableMetadata metadata)
{
super(commitLog, filterPosition, Collections.emptyMap(), ReplayFilter.create());
this.filterPosition = filterPosition;
- this.metadata = cfm;
+ this.metadata = metadata;
}
@SuppressWarnings("resource")
@@ -788,7 +821,7 @@
{
// Only process mutations for the CF's we're testing against, since we can't deterministically predict
// whether or not system keyspaces will be mutated during a test.
- if (partitionUpdate.metadata().cfName.equals(metadata.cfName))
+ if (partitionUpdate.metadata().name.equals(metadata.name))
{
for (Row row : partitionUpdate)
cells += Iterables.size(row.cells());
@@ -810,7 +843,7 @@
for (int i = 0 ; i < 5 ; i++)
{
- new RowUpdateBuilder(cfs.metadata, 0, "k")
+ new RowUpdateBuilder(cfs.metadata(), 0, "k")
.clustering("c" + i).add("val", ByteBuffer.allocate(100))
.build()
.apply();
@@ -854,7 +887,7 @@
for (int i = 0 ; i < 5 ; i++)
{
- new RowUpdateBuilder(cfs.metadata, 0, "k")
+ new RowUpdateBuilder(cfs.metadata(), 0, "k")
.clustering("c" + i).add("val", ByteBuffer.allocate(100))
.build()
.apply();
@@ -936,9 +969,9 @@
{
ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CUSTOM1);
- RowUpdateBuilder rb = new RowUpdateBuilder(cfs.metadata, 0, 1);
+ RowUpdateBuilder rb = new RowUpdateBuilder(cfs.metadata(), 0, BigInteger.ONE);
- rb.add("s", 2);
+ rb.add("s", BigInteger.valueOf(2));
Mutation rm = rb.build();
CommitLog.instance.add(rm);
@@ -956,7 +989,6 @@
}
Assert.assertEquals(replayed, 1);
-
}
}
diff --git a/test/unit/org/apache/cassandra/db/commitlog/CommitLogTestReplayer.java b/test/unit/org/apache/cassandra/db/commitlog/CommitLogTestReplayer.java
index 9a22b04..5b87d68 100644
--- a/test/unit/org/apache/cassandra/db/commitlog/CommitLogTestReplayer.java
+++ b/test/unit/org/apache/cassandra/db/commitlog/CommitLogTestReplayer.java
@@ -26,7 +26,7 @@
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.Mutation;
-import org.apache.cassandra.db.rows.SerializationHelper;
+import org.apache.cassandra.db.rows.DeserializationHelper;
import org.apache.cassandra.io.util.DataInputBuffer;
import org.apache.cassandra.io.util.RebufferingInputStream;
@@ -65,7 +65,7 @@
Mutation mutation;
try
{
- mutation = Mutation.serializer.deserialize(bufIn, desc.getMessagingVersion(), SerializationHelper.Flag.LOCAL);
+ mutation = Mutation.serializer.deserialize(bufIn, desc.getMessagingVersion(), DeserializationHelper.Flag.LOCAL);
Assert.assertTrue(processor.apply(mutation));
}
catch (IOException e)
diff --git a/test/unit/org/apache/cassandra/db/commitlog/CommitLogUpgradeTest.java b/test/unit/org/apache/cassandra/db/commitlog/CommitLogUpgradeTest.java
index d55b59f..7d3689e 100644
--- a/test/unit/org/apache/cassandra/db/commitlog/CommitLogUpgradeTest.java
+++ b/test/unit/org/apache/cassandra/db/commitlog/CommitLogUpgradeTest.java
@@ -24,9 +24,8 @@
import java.io.*;
import java.nio.ByteBuffer;
import java.util.Properties;
-import java.util.UUID;
-import junit.framework.Assert;
+import org.junit.Assert;
import com.google.common.base.Predicate;
@@ -36,9 +35,11 @@
import org.junit.Test;
import org.apache.cassandra.SchemaLoader;
-import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.schema.KeyspaceMetadata;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.config.Schema;
+import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.db.Mutation;
import org.apache.cassandra.db.rows.Cell;
import org.apache.cassandra.db.rows.Row;
@@ -46,10 +47,10 @@
import org.apache.cassandra.db.marshal.BytesType;
import org.apache.cassandra.db.partitions.PartitionUpdate;
import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.Tables;
import org.apache.cassandra.security.EncryptionContextGenerator;
import org.apache.cassandra.utils.JVMStabilityInspector;
import org.apache.cassandra.utils.KillerForTests;
-import org.apache.cassandra.db.commitlog.CommitLogReplayer.CommitLogReplayException;
/**
* Note: if you are looking to create new test cases for this test, check out
@@ -76,12 +77,15 @@
private KillerForTests killerForTests;
private boolean shouldBeKilled = false;
- static CFMetaData metadata = CFMetaData.Builder.createDense(KEYSPACE, TABLE, false, false)
- .addPartitionKey("key", AsciiType.instance)
- .addClusteringColumn("col", AsciiType.instance)
- .addRegularColumn("val", BytesType.instance)
- .build()
- .compression(SchemaLoader.getCompressionParameters());
+ static TableMetadata metadata =
+ TableMetadata.builder(KEYSPACE, TABLE)
+ .isCompound(false)
+ .isDense(true)
+ .addPartitionKeyColumn("key", AsciiType.instance)
+ .addClusteringColumn("col", AsciiType.instance)
+ .addRegularColumn("val", BytesType.instance)
+ .compression(SchemaLoader.getCompressionParameters())
+ .build();
@Before
public void prepareToBeKilled()
@@ -98,83 +102,6 @@
}
@Test
- public void test20() throws Exception
- {
- testRestore(DATA_DIR + "2.0");
- }
-
- @Test
- public void test21() throws Exception
- {
- testRestore(DATA_DIR + "2.1");
- }
-
- @Test
- public void test22() throws Exception
- {
- testRestore(DATA_DIR + "2.2");
- }
-
- @Test
- public void test22_LZ4() throws Exception
- {
- testRestore(DATA_DIR + "2.2-lz4");
- }
-
- @Test
- public void test22_Snappy() throws Exception
- {
- testRestore(DATA_DIR + "2.2-snappy");
- }
-
- public void test22_truncated() throws Exception
- {
- testRestore(DATA_DIR + "2.2-lz4-truncated");
- }
-
- @Test(expected = CommitLogReplayException.class)
- public void test22_bitrot() throws Exception
- {
- shouldBeKilled = true;
- testRestore(DATA_DIR + "2.2-lz4-bitrot");
- }
-
- @Test
- public void test22_bitrot_ignored() throws Exception
- {
- try
- {
- System.setProperty(CommitLogReplayer.IGNORE_REPLAY_ERRORS_PROPERTY, "true");
- testRestore(DATA_DIR + "2.2-lz4-bitrot");
- }
- finally
- {
- System.clearProperty(CommitLogReplayer.IGNORE_REPLAY_ERRORS_PROPERTY);
- }
- }
-
- @Test(expected = CommitLogReplayException.class)
- public void test22_bitrot2() throws Exception
- {
- shouldBeKilled = true;
- testRestore(DATA_DIR + "2.2-lz4-bitrot2");
- }
-
- @Test
- public void test22_bitrot2_ignored() throws Exception
- {
- try
- {
- System.setProperty(CommitLogReplayer.IGNORE_REPLAY_ERRORS_PROPERTY, "true");
- testRestore(DATA_DIR + "2.2-lz4-bitrot2");
- }
- finally
- {
- System.clearProperty(CommitLogReplayer.IGNORE_REPLAY_ERRORS_PROPERTY);
- }
- }
-
- @Test
public void test34_encrypted() throws Exception
{
testRestore(DATA_DIR + "3.4-encrypted");
@@ -184,9 +111,7 @@
public static void initialize()
{
SchemaLoader.loadSchema();
- SchemaLoader.createKeyspace(KEYSPACE,
- KeyspaceParams.simple(1),
- metadata);
+ SchemaLoader.createKeyspace(KEYSPACE, KeyspaceParams.simple(1), metadata);
DatabaseDescriptor.setEncryptionContext(EncryptionContextGenerator.createContext(true));
}
@@ -200,13 +125,9 @@
String cfidString = prop.getProperty(CFID_PROPERTY);
if (cfidString != null)
{
- UUID cfid = UUID.fromString(cfidString);
- if (Schema.instance.getCF(cfid) == null)
- {
- CFMetaData cfm = Schema.instance.getCFMetaData(KEYSPACE, TABLE);
- Schema.instance.unload(cfm);
- Schema.instance.load(cfm.copy(cfid));
- }
+ TableId tableId = TableId.fromString(cfidString);
+ if (Schema.instance.getTableMetadata(tableId) == null)
+ Schema.instance.load(KeyspaceMetadata.create(KEYSPACE, KeyspaceParams.simple(1), Tables.of(metadata.unbuild().id(tableId).build())));
}
Hasher hasher = new Hasher();
diff --git a/test/unit/org/apache/cassandra/db/commitlog/CommitLogUpgradeTestMaker.java b/test/unit/org/apache/cassandra/db/commitlog/CommitLogUpgradeTestMaker.java
index 5a03f9f..680a0e7 100644
--- a/test/unit/org/apache/cassandra/db/commitlog/CommitLogUpgradeTestMaker.java
+++ b/test/unit/org/apache/cassandra/db/commitlog/CommitLogUpgradeTestMaker.java
@@ -33,12 +33,13 @@
import java.util.concurrent.atomic.AtomicLong;
import com.google.common.util.concurrent.RateLimiter;
+import org.junit.Assert;
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.Util;
import org.apache.cassandra.UpdateBuilder;
import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.config.Schema;
+import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.db.Mutation;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.io.util.FileUtils;
@@ -92,9 +93,7 @@
}
SchemaLoader.loadSchema();
- SchemaLoader.createKeyspace(KEYSPACE,
- KeyspaceParams.simple(1),
- metadata);
+ SchemaLoader.createKeyspace(KEYSPACE, KeyspaceParams.simple(1), metadata);
}
public void makeLog() throws IOException, InterruptedException
@@ -112,7 +111,7 @@
Thread.sleep(runTimeMs);
stop = true;
scheduled.shutdown();
- scheduled.awaitTermination(2, TimeUnit.SECONDS);
+ Assert.assertTrue(scheduled.awaitTermination(1, TimeUnit.MINUTES));
int hash = 0;
int cells = 0;
@@ -134,7 +133,7 @@
FileUtils.createHardLink(f, new File(dataDir, f.getName()));
Properties prop = new Properties();
- prop.setProperty(CFID_PROPERTY, Schema.instance.getId(KEYSPACE, TABLE).toString());
+ prop.setProperty(CFID_PROPERTY, Schema.instance.getTableMetadata(KEYSPACE, TABLE).id.toString());
prop.setProperty(CELLS_PROPERTY, Integer.toString(cells));
prop.setProperty(HASH_PROPERTY, Integer.toString(hash));
prop.store(new FileOutputStream(new File(dataDir, PROPERTIES_FILE)),
@@ -236,7 +235,7 @@
rl.acquire();
ByteBuffer key = randomBytes(16, tlr);
- UpdateBuilder builder = UpdateBuilder.create(Schema.instance.getCFMetaData(KEYSPACE, TABLE), Util.dk(key));
+ UpdateBuilder builder = UpdateBuilder.create(Schema.instance.getTableMetadata(KEYSPACE, TABLE), Util.dk(key));
for (int ii = 0; ii < numCells; ii++)
{
diff --git a/test/unit/org/apache/cassandra/db/commitlog/CommitlogShutdownTest.java b/test/unit/org/apache/cassandra/db/commitlog/CommitlogShutdownTest.java
index 91a3f02..711cf65 100644
--- a/test/unit/org/apache/cassandra/db/commitlog/CommitlogShutdownTest.java
+++ b/test/unit/org/apache/cassandra/db/commitlog/CommitlogShutdownTest.java
@@ -40,6 +40,7 @@
import org.apache.cassandra.db.marshal.AsciiType;
import org.apache.cassandra.db.marshal.BytesType;
import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.TableId;
import org.jboss.byteman.contrib.bmunit.BMRule;
import org.jboss.byteman.contrib.bmunit.BMUnitRunner;
@@ -74,10 +75,9 @@
CompactionManager.instance.disableAutoCompaction();
- CommitLog.instance.resetUnsafe(true);
ColumnFamilyStore cfs1 = Keyspace.open(KEYSPACE1).getColumnFamilyStore(STANDARD1);
- final Mutation m = new RowUpdateBuilder(cfs1.metadata, 0, "k")
+ final Mutation m = new RowUpdateBuilder(cfs1.metadata.get(), 0, "k")
.clustering("bytes")
.add("val", ByteBuffer.wrap(entropy))
.build();
@@ -89,8 +89,8 @@
}
// schedule discarding completed segments and immediately issue a shutdown
- UUID cfid = m.getColumnFamilyIds().iterator().next();
- CommitLog.instance.discardCompletedSegments(cfid, CommitLogPosition.NONE, CommitLog.instance.getCurrentPosition());
+ TableId tableId = m.getTableIds().iterator().next();
+ CommitLog.instance.discardCompletedSegments(tableId, CommitLogPosition.NONE, CommitLog.instance.getCurrentPosition());
CommitLog.instance.shutdownBlocking();
// the shutdown should block until all logs except the currently active one and perhaps a new, empty one are gone
diff --git a/test/unit/org/apache/cassandra/db/commitlog/GroupCommitLogTest.java b/test/unit/org/apache/cassandra/db/commitlog/GroupCommitLogTest.java
new file mode 100644
index 0000000..8b0a506
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/commitlog/GroupCommitLogTest.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.commitlog;
+
+import org.junit.BeforeClass;
+
+import org.apache.cassandra.config.Config;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.config.ParameterizedClass;
+import org.apache.cassandra.security.EncryptionContext;
+
+public class GroupCommitLogTest extends CommitLogTest
+{
+ public GroupCommitLogTest(ParameterizedClass commitLogCompression, EncryptionContext encryptionContext)
+ {
+ super(commitLogCompression, encryptionContext);
+ }
+
+ @BeforeClass
+ public static void setCommitLogModeDetails()
+ {
+ DatabaseDescriptor.daemonInitialization();
+ DatabaseDescriptor.setCommitLogSync(Config.CommitLogSync.group);
+ DatabaseDescriptor.setCommitLogSyncGroupWindow(1);
+ beforeClass();
+ }
+}
diff --git a/test/unit/org/apache/cassandra/db/commitlog/SegmentReaderTest.java b/test/unit/org/apache/cassandra/db/commitlog/SegmentReaderTest.java
index d9d5e4c..ce20935 100644
--- a/test/unit/org/apache/cassandra/db/commitlog/SegmentReaderTest.java
+++ b/test/unit/org/apache/cassandra/db/commitlog/SegmentReaderTest.java
@@ -41,7 +41,9 @@
import org.apache.cassandra.io.compress.ICompressor;
import org.apache.cassandra.io.compress.LZ4Compressor;
import org.apache.cassandra.io.compress.SnappyCompressor;
+import org.apache.cassandra.io.compress.ZstdCompressor;
import org.apache.cassandra.io.util.FileDataInput;
+import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.io.util.RandomAccessReader;
import org.apache.cassandra.security.CipherFactory;
import org.apache.cassandra.security.EncryptionUtils;
@@ -77,6 +79,12 @@
compressedSegmenter(DeflateCompressor.create(null));
}
+ @Test
+ public void compressedSegmenter_Zstd() throws IOException
+ {
+ compressedSegmenter(ZstdCompressor.create(Collections.emptyMap()));
+ }
+
private void compressedSegmenter(ICompressor compressor) throws IOException
{
int rawSize = (1 << 15) - 137;
@@ -93,7 +101,7 @@
compressor.compress(plainTextBuffer, compBuffer);
compBuffer.flip();
- File compressedFile = File.createTempFile("compressed-segment-", ".log");
+ File compressedFile = FileUtils.createTempFile("compressed-segment-", ".log");
compressedFile.deleteOnExit();
FileOutputStream fos = new FileOutputStream(compressedFile);
fos.getChannel().write(compBuffer);
@@ -180,7 +188,7 @@
ByteBuffer compressedBuffer = EncryptionUtils.compress(plainTextBuffer, null, true, context.getCompressor());
Cipher cipher = cipherFactory.getEncryptor(context.getTransparentDataEncryptionOptions().cipher, context.getTransparentDataEncryptionOptions().key_alias);
- File encryptedFile = File.createTempFile("encrypted-segment-", ".log");
+ File encryptedFile = FileUtils.createTempFile("encrypted-segment-", ".log");
encryptedFile.deleteOnExit();
FileChannel channel = new RandomAccessFile(encryptedFile, "rw").getChannel();
channel.write(ByteBufferUtil.bytes(plainTextLength));
diff --git a/test/unit/org/apache/cassandra/db/commitlog/SnapshotDeletingTest.java b/test/unit/org/apache/cassandra/db/commitlog/SnapshotDeletingTest.java
index 413e716..b3dc070 100644
--- a/test/unit/org/apache/cassandra/db/commitlog/SnapshotDeletingTest.java
+++ b/test/unit/org/apache/cassandra/db/commitlog/SnapshotDeletingTest.java
@@ -25,7 +25,7 @@
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.Util;
-import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.DecoratedKey;
@@ -90,7 +90,7 @@
private void populate(int rowCount) {
long timestamp = System.currentTimeMillis();
- CFMetaData cfm = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD1).metadata;
+ TableMetadata cfm = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD1).metadata();
for (int i = 0; i <= rowCount; i++)
{
DecoratedKey key = Util.dk(Integer.toString(i));
diff --git a/test/unit/org/apache/cassandra/db/compaction/AbstractCompactionStrategyTest.java b/test/unit/org/apache/cassandra/db/compaction/AbstractCompactionStrategyTest.java
index 481b394..4092f54 100644
--- a/test/unit/org/apache/cassandra/db/compaction/AbstractCompactionStrategyTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/AbstractCompactionStrategyTest.java
@@ -26,7 +26,7 @@
import org.junit.BeforeClass;
import org.junit.Test;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.Util;
import org.apache.cassandra.db.ColumnFamilyStore;
@@ -59,7 +59,7 @@
SchemaLoader.standardCFMD(KEYSPACE1, LCS_TABLE)
.compaction(CompactionParams.lcs(Collections.emptyMap())),
SchemaLoader.standardCFMD(KEYSPACE1, STCS_TABLE)
- .compaction(CompactionParams.scts(Collections.emptyMap())),
+ .compaction(CompactionParams.stcs(Collections.emptyMap())),
SchemaLoader.standardCFMD(KEYSPACE1, DTCS_TABLE)
.compaction(CompactionParams.create(DateTieredCompactionStrategy.class, Collections.emptyMap())),
SchemaLoader.standardCFMD(KEYSPACE1, TWCS_TABLE)
@@ -134,7 +134,7 @@
long timestamp = System.currentTimeMillis();
DecoratedKey dk = Util.dk(String.format("%03d", key));
ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(table);
- new RowUpdateBuilder(cfs.metadata, timestamp, dk.getKey())
+ new RowUpdateBuilder(cfs.metadata(), timestamp, dk.getKey())
.clustering(String.valueOf(key))
.add("val", "val")
.build()
diff --git a/test/unit/org/apache/cassandra/db/compaction/AbstractPendingRepairTest.java b/test/unit/org/apache/cassandra/db/compaction/AbstractPendingRepairTest.java
new file mode 100644
index 0000000..de7ddfc
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/compaction/AbstractPendingRepairTest.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.UUID;
+
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Ignore;
+
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.cql3.QueryProcessor;
+import org.apache.cassandra.cql3.statements.schema.CreateTableStatement;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.repair.AbstractRepairTest;
+import org.apache.cassandra.repair.consistent.LocalSessionAccessor;
+import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.service.ActiveRepairService;
+
+@Ignore
+public class AbstractPendingRepairTest extends AbstractRepairTest
+{
+ protected String ks;
+ protected final String tbl = "tbl";
+ protected TableMetadata cfm;
+ protected ColumnFamilyStore cfs;
+ protected CompactionStrategyManager csm;
+ protected static ActiveRepairService ARS;
+
+ private int nextSSTableKey = 0;
+
+ @BeforeClass
+ public static void setupClass()
+ {
+ SchemaLoader.prepareServer();
+ ARS = ActiveRepairService.instance;
+ LocalSessionAccessor.startup();
+
+ // cutoff messaging service
+ MessagingService.instance().outboundSink.add((message, to) -> false);
+ MessagingService.instance().inboundSink.add((message) -> false);
+ }
+
+ @Before
+ public void setup()
+ {
+ ks = "ks_" + System.currentTimeMillis();
+ cfm = CreateTableStatement.parse(String.format("CREATE TABLE %s.%s (k INT PRIMARY KEY, v INT)", ks, tbl), ks).build();
+ SchemaLoader.createKeyspace(ks, KeyspaceParams.simple(1), cfm);
+ cfs = Schema.instance.getColumnFamilyStoreInstance(cfm.id);
+ csm = cfs.getCompactionStrategyManager();
+ nextSSTableKey = 0;
+ cfs.disableAutoCompaction();
+ }
+
+ /**
+ * creates and returns an sstable
+ *
+ * @param orphan if true, the sstable will be removed from the unrepaired strategy
+ */
+ SSTableReader makeSSTable(boolean orphan)
+ {
+ int pk = nextSSTableKey++;
+ Set<SSTableReader> pre = cfs.getLiveSSTables();
+ QueryProcessor.executeInternal(String.format("INSERT INTO %s.%s (k, v) VALUES(?, ?)", ks, tbl), pk, pk);
+ cfs.forceBlockingFlush();
+ Set<SSTableReader> post = cfs.getLiveSSTables();
+ Set<SSTableReader> diff = new HashSet<>(post);
+ diff.removeAll(pre);
+ assert diff.size() == 1;
+ SSTableReader sstable = diff.iterator().next();
+ if (orphan)
+ {
+ csm.getUnrepairedUnsafe().allStrategies().forEach(acs -> acs.removeSSTable(sstable));
+ }
+ return sstable;
+ }
+
+ public static void mutateRepaired(SSTableReader sstable, long repairedAt, UUID pendingRepair, boolean isTransient)
+ {
+ try
+ {
+ sstable.descriptor.getMetadataSerializer().mutateRepairMetadata(sstable.descriptor, repairedAt, pendingRepair, isTransient);
+ sstable.reloadSSTableMetadata();
+ }
+ catch (IOException e)
+ {
+ throw new AssertionError(e);
+ }
+ }
+
+ public static void mutateRepaired(SSTableReader sstable, long repairedAt)
+ {
+ mutateRepaired(sstable, repairedAt, ActiveRepairService.NO_PENDING_REPAIR, false);
+ }
+
+ public static void mutateRepaired(SSTableReader sstable, UUID pendingRepair, boolean isTransient)
+ {
+ mutateRepaired(sstable, ActiveRepairService.UNREPAIRED_SSTABLE, pendingRepair, isTransient);
+ }
+}
diff --git a/test/unit/org/apache/cassandra/db/compaction/ActiveCompactionsTest.java b/test/unit/org/apache/cassandra/db/compaction/ActiveCompactionsTest.java
new file mode 100644
index 0000000..be5e7df
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/compaction/ActiveCompactionsTest.java
@@ -0,0 +1,196 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ExecutionException;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Sets;
+import org.junit.Test;
+
+import org.apache.cassandra.cache.AutoSavingCache;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.db.view.View;
+import org.apache.cassandra.db.view.ViewBuilderTask;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.index.Index;
+import org.apache.cassandra.index.SecondaryIndexBuilder;
+import org.apache.cassandra.io.sstable.IndexSummaryRedistribution;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.service.CacheService;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+public class ActiveCompactionsTest extends CQLTester
+{
+ @Test
+ public void testSecondaryIndexTracking() throws Throwable
+ {
+ createTable("CREATE TABLE %s (pk int, ck int, a int, b int, PRIMARY KEY (pk, ck))");
+ String idxName = createIndex("CREATE INDEX on %s(a)");
+ getCurrentColumnFamilyStore().disableAutoCompaction();
+ for (int i = 0; i < 5; i++)
+ {
+ execute("INSERT INTO %s (pk, ck, a, b) VALUES ("+i+", 2, 3, 4)");
+ getCurrentColumnFamilyStore().forceBlockingFlush();
+ }
+
+ Index idx = getCurrentColumnFamilyStore().indexManager.getIndexByName(idxName);
+ Set<SSTableReader> sstables = getCurrentColumnFamilyStore().getLiveSSTables();
+ SecondaryIndexBuilder builder = idx.getBuildTaskSupport().getIndexBuildTask(getCurrentColumnFamilyStore(), Collections.singleton(idx), sstables);
+
+ MockActiveCompactions mockActiveCompactions = new MockActiveCompactions();
+ CompactionManager.instance.submitIndexBuild(builder, mockActiveCompactions).get();
+
+ assertTrue(mockActiveCompactions.finished);
+ assertNotNull(mockActiveCompactions.holder);
+ assertEquals(sstables, mockActiveCompactions.holder.getCompactionInfo().getSSTables());
+ }
+
+ @Test
+ public void testIndexSummaryRedistributionTracking() throws Throwable
+ {
+ createTable("CREATE TABLE %s (pk int, ck int, a int, b int, PRIMARY KEY (pk, ck))");
+ getCurrentColumnFamilyStore().disableAutoCompaction();
+ for (int i = 0; i < 5; i++)
+ {
+ execute("INSERT INTO %s (pk, ck, a, b) VALUES ("+i+", 2, 3, 4)");
+ getCurrentColumnFamilyStore().forceBlockingFlush();
+ }
+ Set<SSTableReader> sstables = getCurrentColumnFamilyStore().getLiveSSTables();
+ try (LifecycleTransaction txn = getCurrentColumnFamilyStore().getTracker().tryModify(sstables, OperationType.INDEX_SUMMARY))
+ {
+ Map<TableId, LifecycleTransaction> transactions = ImmutableMap.<TableId, LifecycleTransaction>builder().put(getCurrentColumnFamilyStore().metadata().id, txn).build();
+ IndexSummaryRedistribution isr = new IndexSummaryRedistribution(transactions, 0, 1000);
+ MockActiveCompactions mockActiveCompactions = new MockActiveCompactions();
+ CompactionManager.instance.runIndexSummaryRedistribution(isr, mockActiveCompactions);
+ assertTrue(mockActiveCompactions.finished);
+ assertNotNull(mockActiveCompactions.holder);
+ // index redistribution operates over all keyspaces/tables, we always cancel them
+ assertTrue(mockActiveCompactions.holder.getCompactionInfo().getSSTables().isEmpty());
+ assertTrue(mockActiveCompactions.holder.getCompactionInfo().shouldStop((sstable) -> false));
+ }
+ }
+
+ @Test
+ public void testViewBuildTracking() throws Throwable
+ {
+ createTable("CREATE TABLE %s (k1 int, c1 int , val int, PRIMARY KEY (k1, c1))");
+ getCurrentColumnFamilyStore().disableAutoCompaction();
+ for (int i = 0; i < 5; i++)
+ {
+ execute("INSERT INTO %s (k1, c1, val) VALUES ("+i+", 2, 3)");
+ getCurrentColumnFamilyStore().forceBlockingFlush();
+ }
+ execute(String.format("CREATE MATERIALIZED VIEW %s.view1 AS SELECT k1, c1, val FROM %s.%s WHERE k1 IS NOT NULL AND c1 IS NOT NULL AND val IS NOT NULL PRIMARY KEY (val, k1, c1)", keyspace(), keyspace(), currentTable()));
+ View view = Iterables.getOnlyElement(getCurrentColumnFamilyStore().viewManager);
+
+ Token token = DatabaseDescriptor.getPartitioner().getMinimumToken();
+ ViewBuilderTask vbt = new ViewBuilderTask(getCurrentColumnFamilyStore(), view, new Range<>(token, token), token, 0);
+
+ MockActiveCompactions mockActiveCompactions = new MockActiveCompactions();
+ CompactionManager.instance.submitViewBuilder(vbt, mockActiveCompactions).get();
+ assertTrue(mockActiveCompactions.finished);
+ assertTrue(mockActiveCompactions.holder.getCompactionInfo().getSSTables().isEmpty());
+ // this should stop for all compactions, even if it doesn't pick any sstables;
+ assertTrue(mockActiveCompactions.holder.getCompactionInfo().shouldStop((sstable) -> false));
+ }
+
+ @Test
+ public void testScrubOne() throws Throwable
+ {
+ createTable("CREATE TABLE %s (pk int, ck int, a int, b int, PRIMARY KEY (pk, ck))");
+ getCurrentColumnFamilyStore().disableAutoCompaction();
+ for (int i = 0; i < 5; i++)
+ {
+ execute("INSERT INTO %s (pk, ck, a, b) VALUES (" + i + ", 2, 3, 4)");
+ getCurrentColumnFamilyStore().forceBlockingFlush();
+ }
+
+ SSTableReader sstable = Iterables.getFirst(getCurrentColumnFamilyStore().getLiveSSTables(), null);
+ try (LifecycleTransaction txn = getCurrentColumnFamilyStore().getTracker().tryModify(sstable, OperationType.SCRUB))
+ {
+ MockActiveCompactions mockActiveCompactions = new MockActiveCompactions();
+ CompactionManager.instance.scrubOne(getCurrentColumnFamilyStore(), txn, true, false, false, mockActiveCompactions);
+
+ assertTrue(mockActiveCompactions.finished);
+ assertEquals(mockActiveCompactions.holder.getCompactionInfo().getSSTables(), Sets.newHashSet(sstable));
+ assertFalse(mockActiveCompactions.holder.getCompactionInfo().shouldStop((s) -> false));
+ assertTrue(mockActiveCompactions.holder.getCompactionInfo().shouldStop((s) -> true));
+ }
+
+ }
+
+ @Test
+ public void testVerifyOne() throws Throwable
+ {
+ createTable("CREATE TABLE %s (pk int, ck int, a int, b int, PRIMARY KEY (pk, ck))");
+ getCurrentColumnFamilyStore().disableAutoCompaction();
+ for (int i = 0; i < 5; i++)
+ {
+ execute("INSERT INTO %s (pk, ck, a, b) VALUES (" + i + ", 2, 3, 4)");
+ getCurrentColumnFamilyStore().forceBlockingFlush();
+ }
+
+ SSTableReader sstable = Iterables.getFirst(getCurrentColumnFamilyStore().getLiveSSTables(), null);
+ MockActiveCompactions mockActiveCompactions = new MockActiveCompactions();
+ CompactionManager.instance.verifyOne(getCurrentColumnFamilyStore(), sstable, new Verifier.Options.Builder().build(), mockActiveCompactions);
+ assertTrue(mockActiveCompactions.finished);
+ assertEquals(mockActiveCompactions.holder.getCompactionInfo().getSSTables(), Sets.newHashSet(sstable));
+ assertFalse(mockActiveCompactions.holder.getCompactionInfo().shouldStop((s) -> false));
+ assertTrue(mockActiveCompactions.holder.getCompactionInfo().shouldStop((s) -> true));
+ }
+
+ @Test
+ public void testSubmitCacheWrite() throws ExecutionException, InterruptedException
+ {
+ AutoSavingCache.Writer writer = CacheService.instance.keyCache.getWriter(100);
+ MockActiveCompactions mockActiveCompactions = new MockActiveCompactions();
+ CompactionManager.instance.submitCacheWrite(writer, mockActiveCompactions).get();
+ assertTrue(mockActiveCompactions.finished);
+ assertTrue(mockActiveCompactions.holder.getCompactionInfo().getSSTables().isEmpty());
+ }
+
+ private static class MockActiveCompactions implements ActiveCompactionsTracker
+ {
+ public CompactionInfo.Holder holder;
+ public boolean finished = false;
+ public void beginCompaction(CompactionInfo.Holder ci)
+ {
+ holder = ci;
+ }
+
+ public void finishCompaction(CompactionInfo.Holder ci)
+ {
+ finished = true;
+ }
+ }
+}
diff --git a/test/unit/org/apache/cassandra/db/compaction/AntiCompactionBytemanTest.java b/test/unit/org/apache/cassandra/db/compaction/AntiCompactionBytemanTest.java
index ba6f3a1..38d2607 100644
--- a/test/unit/org/apache/cassandra/db/compaction/AntiCompactionBytemanTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/AntiCompactionBytemanTest.java
@@ -19,11 +19,11 @@
package org.apache.cassandra.db.compaction;
import java.util.ArrayList;
-import java.util.Collections;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
+import java.util.UUID;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -32,13 +32,18 @@
import org.junit.Test;
import org.junit.runner.RunWith;
+import org.apache.cassandra.Util;
import org.apache.cassandra.cql3.CQLTester;
import org.apache.cassandra.cql3.UntypedResultSet;
import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
import org.apache.cassandra.dht.Murmur3Partitioner;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.RangesAtEndpoint;
+import org.apache.cassandra.locator.Replica;
import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.utils.FBUtilities;
import org.jboss.byteman.contrib.bmunit.BMRule;
import org.jboss.byteman.contrib.bmunit.BMRules;
import org.jboss.byteman.contrib.bmunit.BMUnitRunner;
@@ -61,6 +66,7 @@
createTable("create table %s (id int primary key, i int)");
execute("insert into %s (id, i) values (1, 1)");
execute("insert into %s (id, i) values (2, 1)");
+ execute("insert into %s (id, i) values (3, 1)");
getCurrentColumnFamilyStore().forceBlockingFlush();
UntypedResultSet res = execute("select token(id) as tok from %s");
Iterator<UntypedResultSet.Row> it = res.iterator();
@@ -75,6 +81,12 @@
long first = tokens.get(0) - 10;
long last = tokens.get(0) + 10;
Range<Token> toRepair = new Range<>(new Murmur3Partitioner.LongToken(first), new Murmur3Partitioner.LongToken(last));
+ first = tokens.get(1) - 10;
+ last = tokens.get(1) + 10;
+ Range<Token> pending = new Range<>(new Murmur3Partitioner.LongToken(first), new Murmur3Partitioner.LongToken(last));
+
+ RangesAtEndpoint ranges = new RangesAtEndpoint.Builder(FBUtilities.getBroadcastAddressAndPort()).add(Replica.fullReplica(FBUtilities.getBroadcastAddressAndPort(), toRepair))
+ .add(Replica.transientReplica(InetAddressAndPort.getByName("127.0.0.1"), pending)).build();
AtomicBoolean failed = new AtomicBoolean(false);
AtomicBoolean finished = new AtomicBoolean(false);
@@ -100,7 +112,7 @@
UntypedResultSet.Row r = rowIter.next();
ids.add(r.getInt("id"));
}
- if (!Sets.newHashSet(1,2).equals(ids))
+ if (!Sets.newHashSet(1,2,3).equals(ids))
{
failed.set(true);
return;
@@ -114,12 +126,12 @@
try (LifecycleTransaction txn = getCurrentColumnFamilyStore().getTracker().tryModify(getCurrentColumnFamilyStore().getLiveSSTables(), OperationType.ANTICOMPACTION))
{
- CompactionManager.instance.antiCompactGroup(getCurrentColumnFamilyStore(), Collections.singleton(toRepair), txn, 123);
+ CompactionManager.instance.antiCompactGroup(getCurrentColumnFamilyStore(), ranges, txn, UUID.randomUUID(), () -> false);
}
finished.set(true);
t.join();
assertFalse(failed.get());
assertFalse(getCurrentColumnFamilyStore().getLiveSSTables().contains(sstableBefore));
- AntiCompactionTest.assertOnDiskState(getCurrentColumnFamilyStore(), 2);
+ Util.assertOnDiskState(getCurrentColumnFamilyStore(), 3);
}
}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/db/compaction/AntiCompactionTest.java b/test/unit/org/apache/cassandra/db/compaction/AntiCompactionTest.java
index a85be24..b2618e5 100644
--- a/test/unit/org/apache/cassandra/db/compaction/AntiCompactionTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/AntiCompactionTest.java
@@ -19,25 +19,33 @@
import java.io.File;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
+import java.util.Collections;
+import java.util.Iterator;
import java.util.List;
import java.util.Set;
-import java.util.SortedSet;
-import java.util.TreeSet;
import java.util.UUID;
-import java.util.concurrent.ExecutionException;
+import java.util.function.Predicate;
import java.util.stream.Collectors;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
-import com.google.common.util.concurrent.ListenableFuture;
-import com.google.common.util.concurrent.RateLimiter;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.After;
import org.junit.Test;
-import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.RangesAtEndpoint;
+import org.apache.cassandra.locator.Replica;
+import org.apache.cassandra.schema.MockSchema;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
import org.apache.cassandra.db.rows.EncodingStats;
@@ -46,16 +54,22 @@
import org.apache.cassandra.dht.ByteOrderedPartitioner.BytesToken;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
-import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.io.sstable.*;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.schema.KeyspaceParams;
import org.apache.cassandra.service.ActiveRepairService;
import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.streaming.PreviewKind;
import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.UUIDGen;
import org.apache.cassandra.utils.concurrent.Refs;
import org.apache.cassandra.UpdateBuilder;
+import org.apache.cassandra.utils.concurrent.Transactional;
+import static org.apache.cassandra.service.ActiveRepairService.NO_PENDING_REPAIR;
+import static org.apache.cassandra.service.ActiveRepairService.UNREPAIRED_SSTABLE;
+import static org.apache.cassandra.Util.assertOnDiskState;
import static org.hamcrest.CoreMatchers.is;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
@@ -68,16 +82,21 @@
{
private static final String KEYSPACE1 = "AntiCompactionTest";
private static final String CF = "AntiCompactionTest";
- private static CFMetaData cfm;
+ private static final Collection<Range<Token>> NO_RANGES = Collections.emptyList();
+
+ private static TableMetadata metadata;
+ private static ColumnFamilyStore cfs;
+ private static InetAddressAndPort local;
+
@BeforeClass
- public static void defineSchema() throws ConfigurationException
+ public static void defineSchema() throws Throwable
{
SchemaLoader.prepareServer();
- cfm = SchemaLoader.standardCFMD(KEYSPACE1, CF);
- SchemaLoader.createKeyspace(KEYSPACE1,
- KeyspaceParams.simple(1),
- cfm);
+ metadata = SchemaLoader.standardCFMD(KEYSPACE1, CF).build();
+ SchemaLoader.createKeyspace(KEYSPACE1, KeyspaceParams.simple(1), metadata);
+ cfs = Schema.instance.getColumnFamilyStoreInstance(metadata.id);
+ local = InetAddressAndPort.getByName("127.0.0.1");
}
@After
@@ -88,44 +107,86 @@
store.truncateBlocking();
}
- @Test
- public void antiCompactOne() throws Exception
+ private void registerParentRepairSession(UUID sessionID, Iterable<Range<Token>> ranges, long repairedAt, UUID pendingRepair) throws IOException
{
- ColumnFamilyStore store = prepareColumnFamilyStore();
- Collection<SSTableReader> sstables = getUnrepairedSSTables(store);
- assertEquals(store.getLiveSSTables().size(), sstables.size());
- Range<Token> range = new Range<Token>(new BytesToken("0".getBytes()), new BytesToken("4".getBytes()));
- List<Range<Token>> ranges = Arrays.asList(range);
+ ActiveRepairService.instance.registerParentRepairSession(sessionID,
+ InetAddressAndPort.getByName("10.0.0.1"),
+ Lists.newArrayList(cfs), ImmutableSet.copyOf(ranges),
+ pendingRepair != null || repairedAt != UNREPAIRED_SSTABLE,
+ repairedAt, true, PreviewKind.NONE);
+ }
- int repairedKeys = 0;
- int nonRepairedKeys = 0;
+ private static RangesAtEndpoint atEndpoint(Collection<Range<Token>> full, Collection<Range<Token>> trans)
+ {
+ RangesAtEndpoint.Builder builder = RangesAtEndpoint.builder(local);
+ for (Range<Token> range : full)
+ builder.add(new Replica(local, range, true));
+
+ for (Range<Token> range : trans)
+ builder.add(new Replica(local, range, false));
+
+ return builder.build();
+ }
+
+ private static Collection<Range<Token>> range(int l, int r)
+ {
+ return Collections.singleton(new Range<>(new BytesToken(Integer.toString(l).getBytes()), new BytesToken(Integer.toString(r).getBytes())));
+ }
+
+ private static class SSTableStats
+ {
+ int numLiveSSTables = 0;
+ int pendingKeys = 0;
+ int transKeys = 0;
+ int unrepairedKeys = 0;
+ }
+
+ private SSTableStats antiCompactRanges(ColumnFamilyStore store, RangesAtEndpoint ranges) throws IOException
+ {
+ UUID sessionID = UUID.randomUUID();
+ Collection<SSTableReader> sstables = getUnrepairedSSTables(store);
try (LifecycleTransaction txn = store.getTracker().tryModify(sstables, OperationType.ANTICOMPACTION);
Refs<SSTableReader> refs = Refs.ref(sstables))
{
if (txn == null)
throw new IllegalStateException();
- long repairedAt = 1000;
- UUID parentRepairSession = UUID.randomUUID();
- CompactionManager.instance.performAnticompaction(store, ranges, refs, txn, repairedAt, parentRepairSession);
+ registerParentRepairSession(sessionID, ranges.ranges(), FBUtilities.nowInSeconds(), sessionID);
+ CompactionManager.instance.performAnticompaction(store, ranges, refs, txn, sessionID, () -> false);
}
- assertEquals(2, store.getLiveSSTables().size());
+ SSTableStats stats = new SSTableStats();
+ stats.numLiveSSTables = store.getLiveSSTables().size();
+
+ Predicate<Token> fullContains = t -> Iterables.any(ranges.onlyFull().ranges(), r -> r.contains(t));
+ Predicate<Token> transContains = t -> Iterables.any(ranges.onlyTransient().ranges(), r -> r.contains(t));
for (SSTableReader sstable : store.getLiveSSTables())
{
- try (ISSTableScanner scanner = sstable.getScanner((RateLimiter) null))
+ assertFalse(sstable.isRepaired());
+ assertEquals(sstable.isPendingRepair() ? sessionID : NO_PENDING_REPAIR, sstable.getPendingRepair());
+ try (ISSTableScanner scanner = sstable.getScanner())
{
while (scanner.hasNext())
{
UnfilteredRowIterator row = scanner.next();
- if (sstable.isRepaired())
+ Token token = row.partitionKey().getToken();
+ if (sstable.isPendingRepair() && !sstable.isTransient())
{
- assertTrue(range.contains(row.partitionKey().getToken()));
- repairedKeys++;
+ assertTrue(fullContains.test(token));
+ assertFalse(transContains.test(token));
+ stats.pendingKeys++;
+ }
+ else if (sstable.isPendingRepair() && sstable.isTransient())
+ {
+
+ assertTrue(transContains.test(token));
+ assertFalse(fullContains.test(token));
+ stats.transKeys++;
}
else
{
- assertFalse(range.contains(row.partitionKey().getToken()));
- nonRepairedKeys++;
+ assertFalse(fullContains.test(token));
+ assertFalse(transContains.test(token));
+ stats.unrepairedKeys++;
}
}
}
@@ -136,8 +197,42 @@
assertEquals(1, sstable.selfRef().globalCount());
}
assertEquals(0, store.getTracker().getCompacting().size());
- assertEquals(repairedKeys, 4);
- assertEquals(nonRepairedKeys, 6);
+ return stats;
+ }
+
+ @Test
+ public void antiCompactOneFull() throws Exception
+ {
+ ColumnFamilyStore store = prepareColumnFamilyStore();
+ SSTableStats stats = antiCompactRanges(store, atEndpoint(range(0, 4), NO_RANGES));
+ assertEquals(2, stats.numLiveSSTables);
+ assertEquals(stats.pendingKeys, 4);
+ assertEquals(stats.transKeys, 0);
+ assertEquals(stats.unrepairedKeys, 6);
+ assertOnDiskState(store, 2);
+ }
+
+ @Test
+ public void antiCompactOneMixed() throws Exception
+ {
+ ColumnFamilyStore store = prepareColumnFamilyStore();
+ SSTableStats stats = antiCompactRanges(store, atEndpoint(range(0, 4), range(4, 8)));
+ assertEquals(3, stats.numLiveSSTables);
+ assertEquals(stats.pendingKeys, 4);
+ assertEquals(stats.transKeys, 4);
+ assertEquals(stats.unrepairedKeys, 2);
+ assertOnDiskState(store, 3);
+ }
+
+ @Test
+ public void antiCompactOneTransOnly() throws Exception
+ {
+ ColumnFamilyStore store = prepareColumnFamilyStore();
+ SSTableStats stats = antiCompactRanges(store, atEndpoint(NO_RANGES, range(0, 4)));
+ assertEquals(2, stats.numLiveSSTables);
+ assertEquals(stats.pendingKeys, 0);
+ assertEquals(stats.transKeys, 4);
+ assertEquals(stats.unrepairedKeys, 6);
assertOnDiskState(store, 2);
}
@@ -150,12 +245,14 @@
SSTableReader s = writeFile(cfs, 1000);
cfs.addSSTable(s);
Range<Token> range = new Range<Token>(new BytesToken(ByteBufferUtil.bytes(0)), new BytesToken(ByteBufferUtil.bytes(500)));
+ List<Range<Token>> ranges = Arrays.asList(range);
Collection<SSTableReader> sstables = cfs.getLiveSSTables();
UUID parentRepairSession = UUID.randomUUID();
+ registerParentRepairSession(parentRepairSession, ranges, UNREPAIRED_SSTABLE, UUIDGen.getTimeUUID());
try (LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.ANTICOMPACTION);
Refs<SSTableReader> refs = Refs.ref(sstables))
{
- CompactionManager.instance.performAnticompaction(cfs, Arrays.asList(range), refs, txn, 12345, parentRepairSession);
+ CompactionManager.instance.performAnticompaction(cfs, atEndpoint(ranges, NO_RANGES), refs, txn, parentRepairSession, () -> false);
}
long sum = 0;
long rows = 0;
@@ -172,13 +269,13 @@
private SSTableReader writeFile(ColumnFamilyStore cfs, int count)
{
File dir = cfs.getDirectories().getDirectoryForNewSSTables();
- String filename = cfs.getSSTablePath(dir);
+ Descriptor desc = cfs.newSSTableDescriptor(dir);
- try (SSTableTxnWriter writer = SSTableTxnWriter.create(cfs, filename, 0, 0, new SerializationHeader(true, cfm, cfm.partitionColumns(), EncodingStats.NO_STATS)))
+ try (SSTableTxnWriter writer = SSTableTxnWriter.create(cfs, desc, 0, 0, NO_PENDING_REPAIR, false, new SerializationHeader(true, cfs.metadata(), cfs.metadata().regularAndStaticColumns(), EncodingStats.NO_STATS)))
{
for (int i = 0; i < count; i++)
{
- UpdateBuilder builder = UpdateBuilder.create(cfm, ByteBufferUtil.bytes(i));
+ UpdateBuilder builder = UpdateBuilder.create(metadata, ByteBufferUtil.bytes(i));
for (int j = 0; j < count * 5; j++)
builder.newRow("c" + j).add("val", "value1");
writer.append(builder.build().unfilteredIterator());
@@ -196,7 +293,7 @@
for (int i = 0; i < 10; i++)
{
String localSuffix = Integer.toString(i);
- new RowUpdateBuilder(cfm, System.currentTimeMillis(), localSuffix + "-" + Suffix)
+ new RowUpdateBuilder(metadata, System.currentTimeMillis(), localSuffix + "-" + Suffix)
.clustering("c")
.add("val", "val" + localSuffix)
.build()
@@ -206,7 +303,7 @@
}
@Test
- public void antiCompactTen() throws InterruptedException, IOException
+ public void antiCompactTenFull() throws InterruptedException, IOException
{
Keyspace keyspace = Keyspace.open(KEYSPACE1);
ColumnFamilyStore store = keyspace.getColumnFamilyStore(CF);
@@ -216,180 +313,86 @@
{
generateSStable(store,Integer.toString(table));
}
- Collection<SSTableReader> sstables = getUnrepairedSSTables(store);
- assertEquals(store.getLiveSSTables().size(), sstables.size());
-
- Range<Token> range = new Range<Token>(new BytesToken("0".getBytes()), new BytesToken("4".getBytes()));
- List<Range<Token>> ranges = Arrays.asList(range);
-
- long repairedAt = 1000;
- UUID parentRepairSession = UUID.randomUUID();
- try (LifecycleTransaction txn = store.getTracker().tryModify(sstables, OperationType.ANTICOMPACTION);
- Refs<SSTableReader> refs = Refs.ref(sstables))
- {
- CompactionManager.instance.performAnticompaction(store, ranges, refs, txn, repairedAt, parentRepairSession);
- }
+ SSTableStats stats = antiCompactRanges(store, atEndpoint(range(0, 4), NO_RANGES));
/*
Anticompaction will be anti-compacting 10 SSTables but will be doing this two at a time
so there will be no net change in the number of sstables
*/
- assertEquals(10, store.getLiveSSTables().size());
- int repairedKeys = 0;
- int nonRepairedKeys = 0;
- for (SSTableReader sstable : store.getLiveSSTables())
- {
- try (ISSTableScanner scanner = sstable.getScanner((RateLimiter) null))
- {
- while (scanner.hasNext())
- {
- try (UnfilteredRowIterator row = scanner.next())
- {
- if (sstable.isRepaired())
- {
- assertTrue(range.contains(row.partitionKey().getToken()));
- assertEquals(repairedAt, sstable.getSSTableMetadata().repairedAt);
- repairedKeys++;
- }
- else
- {
- assertFalse(range.contains(row.partitionKey().getToken()));
- assertEquals(ActiveRepairService.UNREPAIRED_SSTABLE, sstable.getSSTableMetadata().repairedAt);
- nonRepairedKeys++;
- }
- }
- }
- }
- }
- assertEquals(repairedKeys, 40);
- assertEquals(nonRepairedKeys, 60);
+ assertEquals(10, stats.numLiveSSTables);
+ assertEquals(stats.pendingKeys, 40);
+ assertEquals(stats.transKeys, 0);
+ assertEquals(stats.unrepairedKeys, 60);
assertOnDiskState(store, 10);
}
@Test
- public void shouldMutateRepairedAt() throws InterruptedException, IOException
+ public void antiCompactTenTrans() throws InterruptedException, IOException
{
- ColumnFamilyStore store = prepareColumnFamilyStore();
- Collection<SSTableReader> sstables = getUnrepairedSSTables(store);
- assertEquals(store.getLiveSSTables().size(), sstables.size());
- Range<Token> range = new Range<Token>(new BytesToken("/".getBytes()), new BytesToken("9999".getBytes()));
- List<Range<Token>> ranges = Arrays.asList(range);
- UUID parentRepairSession = UUID.randomUUID();
+ Keyspace keyspace = Keyspace.open(KEYSPACE1);
+ ColumnFamilyStore store = keyspace.getColumnFamilyStore(CF);
+ store.disableAutoCompaction();
- try (LifecycleTransaction txn = store.getTracker().tryModify(sstables, OperationType.ANTICOMPACTION);
- Refs<SSTableReader> refs = Refs.ref(sstables))
+ for (int table = 0; table < 10; table++)
{
- CompactionManager.instance.performAnticompaction(store, ranges, refs, txn, 1, parentRepairSession);
+ generateSStable(store,Integer.toString(table));
}
-
- SSTableReader sstable = Iterables.get(store.getLiveSSTables(), 0);
- assertThat(store.getLiveSSTables().size(), is(1));
- assertThat(sstable.isRepaired(), is(true));
- assertThat(sstable.selfRef().globalCount(), is(1));
- assertThat(store.getTracker().getCompacting().size(), is(0));
- assertOnDiskState(store, 1);
+ SSTableStats stats = antiCompactRanges(store, atEndpoint(NO_RANGES, range(0, 4)));
+ /*
+ Anticompaction will be anti-compacting 10 SSTables but will be doing this two at a time
+ so there will be no net change in the number of sstables
+ */
+ assertEquals(10, stats.numLiveSSTables);
+ assertEquals(stats.pendingKeys, 0);
+ assertEquals(stats.transKeys, 40);
+ assertEquals(stats.unrepairedKeys, 60);
+ assertOnDiskState(store, 10);
}
@Test
- public void shouldAntiCompactSSTable() throws IOException, InterruptedException, ExecutionException
+ public void antiCompactTenMixed() throws InterruptedException, IOException
+ {
+ Keyspace keyspace = Keyspace.open(KEYSPACE1);
+ ColumnFamilyStore store = keyspace.getColumnFamilyStore(CF);
+ store.disableAutoCompaction();
+
+ for (int table = 0; table < 10; table++)
+ {
+ generateSStable(store,Integer.toString(table));
+ }
+ SSTableStats stats = antiCompactRanges(store, atEndpoint(range(0, 4), range(4, 8)));
+ assertEquals(15, stats.numLiveSSTables);
+ assertEquals(stats.pendingKeys, 40);
+ assertEquals(stats.transKeys, 40);
+ assertEquals(stats.unrepairedKeys, 20);
+ assertOnDiskState(store, 15);
+ }
+
+ @Test
+ public void shouldMutatePendingRepair() throws InterruptedException, IOException
{
ColumnFamilyStore store = prepareColumnFamilyStore();
Collection<SSTableReader> sstables = getUnrepairedSSTables(store);
assertEquals(store.getLiveSSTables().size(), sstables.size());
- // SSTable range is 0 - 10, repair just a subset of the ranges (0 - 4) of the SSTable. Should result in
- // one repaired and one unrepaired SSTable
- Range<Token> range = new Range<Token>(new BytesToken("/".getBytes()), new BytesToken("4".getBytes()));
+ // the sstables start at "0".getBytes() = 48, we need to include that first token, with "/".getBytes() = 47
+ Range<Token> range = new Range<Token>(new BytesToken("/".getBytes()), new BytesToken("9999".getBytes()));
List<Range<Token>> ranges = Arrays.asList(range);
- UUID parentRepairSession = UUID.randomUUID();
+ UUID pendingRepair = UUID.randomUUID();
+ registerParentRepairSession(pendingRepair, ranges, UNREPAIRED_SSTABLE, pendingRepair);
try (LifecycleTransaction txn = store.getTracker().tryModify(sstables, OperationType.ANTICOMPACTION);
Refs<SSTableReader> refs = Refs.ref(sstables))
{
- CompactionManager.instance.performAnticompaction(store, ranges, refs, txn, 1, parentRepairSession);
+ CompactionManager.instance.performAnticompaction(store, atEndpoint(ranges, NO_RANGES), refs, txn, pendingRepair, () -> false);
}
- SortedSet<SSTableReader> sstablesSorted = new TreeSet<>(SSTableReader.generationReverseComparator.reversed());
- sstablesSorted.addAll(store.getLiveSSTables());
-
- SSTableReader sstable = sstablesSorted.first();
- assertThat(store.getLiveSSTables().size(), is(2));
- assertThat(sstable.isRepaired(), is(true));
- assertThat(sstable.selfRef().globalCount(), is(1));
+ assertThat(store.getLiveSSTables().size(), is(1));
+ assertThat(Iterables.get(store.getLiveSSTables(), 0).isRepaired(), is(false));
+ assertThat(Iterables.get(store.getLiveSSTables(), 0).isPendingRepair(), is(true));
+ assertThat(Iterables.get(store.getLiveSSTables(), 0).selfRef().globalCount(), is(1));
assertThat(store.getTracker().getCompacting().size(), is(0));
-
- // Test we don't anti-compact already repaired SSTables. repairedAt shouldn't change for the already repaired SSTable (first)
- sstables = store.getLiveSSTables();
- // Range that's a subset of the repaired SSTable's ranges, so would cause an anti-compaction (if it wasn't repaired)
- range = new Range<Token>(new BytesToken("/".getBytes()), new BytesToken("2".getBytes()));
- ranges = Arrays.asList(range);
- try (Refs<SSTableReader> refs = Refs.ref(sstables))
- {
- // use different repairedAt to ensure it doesn't change
- ListenableFuture fut = CompactionManager.instance.submitAntiCompaction(store, ranges, refs, 200, parentRepairSession);
- fut.get();
- }
-
- sstablesSorted.clear();
- sstablesSorted.addAll(store.getLiveSSTables());
- assertThat(sstablesSorted.size(), is(2));
- assertThat(sstablesSorted.first().isRepaired(), is(true));
- assertThat(sstablesSorted.last().isRepaired(), is(false));
- assertThat(sstablesSorted.first().getSSTableMetadata().repairedAt, is(1L));
- assertThat(sstablesSorted.last().getSSTableMetadata().repairedAt, is(0L));
- assertThat(sstablesSorted.first().selfRef().globalCount(), is(1));
- assertThat(sstablesSorted.last().selfRef().globalCount(), is(1));
- assertThat(store.getTracker().getCompacting().size(), is(0));
-
- // Test repairing all the ranges of the repaired SSTable. Should mutate repairedAt without anticompacting,
- // but leave the unrepaired SSTable as is.
- range = new Range<Token>(new BytesToken("/".getBytes()), new BytesToken("4".getBytes()));
- ranges = Arrays.asList(range);
-
- try (Refs<SSTableReader> refs = Refs.ref(sstables))
- {
- // Same repaired at, but should be changed on the repaired SSTable now
- ListenableFuture fut = CompactionManager.instance.submitAntiCompaction(store, ranges, refs, 200, parentRepairSession);
- fut.get();
- }
-
- sstablesSorted.clear();
- sstablesSorted.addAll(store.getLiveSSTables());
-
- assertThat(sstablesSorted.size(), is(2));
- assertThat(sstablesSorted.first().isRepaired(), is(true));
- assertThat(sstablesSorted.last().isRepaired(), is(false));
- assertThat(sstablesSorted.first().getSSTableMetadata().repairedAt, is(200L));
- assertThat(sstablesSorted.last().getSSTableMetadata().repairedAt, is(0L));
- assertThat(sstablesSorted.first().selfRef().globalCount(), is(1));
- assertThat(sstablesSorted.last().selfRef().globalCount(), is(1));
- assertThat(store.getTracker().getCompacting().size(), is(0));
-
- // Repair whole range. Should mutate repairedAt on repaired SSTable (again) and
- // mark unrepaired SSTable as repaired
- range = new Range<Token>(new BytesToken("/".getBytes()), new BytesToken("999".getBytes()));
- ranges = Arrays.asList(range);
-
- try (Refs<SSTableReader> refs = Refs.ref(sstables))
- {
- // Both SSTables should have repairedAt of 400
- ListenableFuture fut = CompactionManager.instance.submitAntiCompaction(store, ranges, refs, 400, parentRepairSession);
- fut.get();
- }
-
- sstablesSorted.clear();
- sstablesSorted.addAll(store.getLiveSSTables());
-
- assertThat(sstablesSorted.size(), is(2));
- assertThat(sstablesSorted.first().isRepaired(), is(true));
- assertThat(sstablesSorted.last().isRepaired(), is(true));
- assertThat(sstablesSorted.first().getSSTableMetadata().repairedAt, is(400L));
- assertThat(sstablesSorted.last().getSSTableMetadata().repairedAt, is(400L));
- assertThat(sstablesSorted.first().selfRef().globalCount(), is(1));
- assertThat(sstablesSorted.last().selfRef().globalCount(), is(1));
- assertThat(store.getTracker().getCompacting().size(), is(0));
- assertOnDiskState(store, 2);
+ assertOnDiskState(store, 1);
}
-
@Test
public void shouldSkipAntiCompactionForNonIntersectingRange() throws InterruptedException, IOException
{
@@ -401,22 +404,29 @@
{
generateSStable(store,Integer.toString(table));
}
+ int refCountBefore = Iterables.get(store.getLiveSSTables(), 0).selfRef().globalCount();
Collection<SSTableReader> sstables = getUnrepairedSSTables(store);
assertEquals(store.getLiveSSTables().size(), sstables.size());
Range<Token> range = new Range<Token>(new BytesToken("-1".getBytes()), new BytesToken("-10".getBytes()));
List<Range<Token>> ranges = Arrays.asList(range);
UUID parentRepairSession = UUID.randomUUID();
-
+ registerParentRepairSession(parentRepairSession, ranges, UNREPAIRED_SSTABLE, null);
+ boolean gotException = false;
try (LifecycleTransaction txn = store.getTracker().tryModify(sstables, OperationType.ANTICOMPACTION);
Refs<SSTableReader> refs = Refs.ref(sstables))
{
- CompactionManager.instance.performAnticompaction(store, ranges, refs, txn, 1, parentRepairSession);
+ CompactionManager.instance.performAnticompaction(store, atEndpoint(ranges, NO_RANGES), refs, txn, parentRepairSession, () -> false);
+ }
+ catch (IllegalStateException e)
+ {
+ gotException = true;
}
- assertThat(store.getLiveSSTables().size(), is(10));
+ assertTrue(gotException);
assertThat(Iterables.get(store.getLiveSSTables(), 0).isRepaired(), is(false));
- assertOnDiskState(store, 10);
+ assertEquals(refCountBefore, Iterables.get(store.getLiveSSTables(), 0).selfRef().globalCount());
+ assertOnDiskState(cfs, 10);
}
private ColumnFamilyStore prepareColumnFamilyStore()
@@ -426,7 +436,7 @@
store.disableAutoCompaction();
for (int i = 0; i < 10; i++)
{
- new RowUpdateBuilder(cfm, System.currentTimeMillis(), Integer.toString(i))
+ new RowUpdateBuilder(metadata, System.currentTimeMillis(), Integer.toString(i))
.clustering("c")
.add("val", "val")
.build()
@@ -449,25 +459,127 @@
return ImmutableSet.copyOf(cfs.getTracker().getView().sstables(SSTableSet.LIVE, (s) -> !s.isRepaired()));
}
- public static void assertOnDiskState(ColumnFamilyStore cfs, int expectedSSTableCount)
+ /**
+ * If the parent repair session is missing, we should still clean up
+ */
+ @Test
+ public void missingParentRepairSession() throws Exception
{
- LifecycleTransaction.waitForDeletions();
- assertEquals(expectedSSTableCount, cfs.getLiveSSTables().size());
- Set<Integer> liveGenerations = cfs.getLiveSSTables().stream().map(sstable -> sstable.descriptor.generation).collect(Collectors.toSet());
- int fileCount = 0;
- for (File f : cfs.getDirectories().getCFDirectories())
+ Keyspace keyspace = Keyspace.open(KEYSPACE1);
+ ColumnFamilyStore store = keyspace.getColumnFamilyStore(CF);
+ store.disableAutoCompaction();
+
+ for (int table = 0; table < 10; table++)
{
- for (File sst : f.listFiles())
- {
- if (sst.getName().contains("Data"))
- {
- Descriptor d = Descriptor.fromFilename(sst.getAbsolutePath());
- assertTrue(liveGenerations.contains(d.generation));
- fileCount++;
- }
- }
+ generateSStable(store,Integer.toString(table));
}
- assertEquals(expectedSSTableCount, fileCount);
+ Collection<SSTableReader> sstables = getUnrepairedSSTables(store);
+ assertEquals(10, sstables.size());
+
+ Range<Token> range = new Range<Token>(new BytesToken("-1".getBytes()), new BytesToken("-10".getBytes()));
+ List<Range<Token>> ranges = Arrays.asList(range);
+
+ UUID missingRepairSession = UUIDGen.getTimeUUID();
+ try (LifecycleTransaction txn = store.getTracker().tryModify(sstables, OperationType.ANTICOMPACTION);
+ Refs<SSTableReader> refs = Refs.ref(sstables))
+ {
+ Assert.assertFalse(refs.isEmpty());
+ try
+ {
+ CompactionManager.instance.performAnticompaction(store, atEndpoint(ranges, NO_RANGES), refs, txn, missingRepairSession, () -> false);
+ Assert.fail("expected RuntimeException");
+ }
+ catch (RuntimeException e)
+ {
+ // expected
+ }
+ Assert.assertEquals(Transactional.AbstractTransactional.State.ABORTED, txn.state());
+ Assert.assertTrue(refs.isEmpty());
+ }
}
+ @Test
+ public void testSSTablesToInclude()
+ {
+ ColumnFamilyStore cfs = MockSchema.newCFS("anticomp");
+ List<SSTableReader> sstables = new ArrayList<>();
+ sstables.add(MockSchema.sstable(1, 10, 100, cfs));
+ sstables.add(MockSchema.sstable(2, 100, 200, cfs));
+
+ Range<Token> r = new Range<>(t(10), t(100)); // should include sstable 1 and 2 above, but none is fully contained (Range is (x, y])
+
+ Iterator<SSTableReader> sstableIterator = sstables.iterator();
+ Set<SSTableReader> fullyContainedSSTables = CompactionManager.findSSTablesToAnticompact(sstableIterator, Collections.singletonList(r), UUID.randomUUID());
+ assertTrue(fullyContainedSSTables.isEmpty());
+ assertEquals(2, sstables.size());
+ }
+
+ @Test
+ public void testSSTablesToInclude2()
+ {
+ ColumnFamilyStore cfs = MockSchema.newCFS("anticomp");
+ List<SSTableReader> sstables = new ArrayList<>();
+ SSTableReader sstable1 = MockSchema.sstable(1, 10, 100, cfs);
+ SSTableReader sstable2 = MockSchema.sstable(2, 100, 200, cfs);
+ sstables.add(sstable1);
+ sstables.add(sstable2);
+
+ Range<Token> r = new Range<>(t(9), t(100)); // sstable 1 is fully contained
+
+ Iterator<SSTableReader> sstableIterator = sstables.iterator();
+ Set<SSTableReader> fullyContainedSSTables = CompactionManager.findSSTablesToAnticompact(sstableIterator, Collections.singletonList(r), UUID.randomUUID());
+ assertEquals(Collections.singleton(sstable1), fullyContainedSSTables);
+ assertEquals(Collections.singletonList(sstable2), sstables);
+ }
+
+ @Test(expected = IllegalStateException.class)
+ public void testSSTablesToNotInclude()
+ {
+ ColumnFamilyStore cfs = MockSchema.newCFS("anticomp");
+ List<SSTableReader> sstables = new ArrayList<>();
+ SSTableReader sstable1 = MockSchema.sstable(1, 0, 5, cfs);
+ sstables.add(sstable1);
+
+ Range<Token> r = new Range<>(t(9), t(100)); // sstable is not intersecting and should not be included
+
+ CompactionManager.validateSSTableBoundsForAnticompaction(UUID.randomUUID(), sstables, atEndpoint(Collections.singletonList(r), NO_RANGES));
+ }
+
+ @Test(expected = IllegalStateException.class)
+ public void testSSTablesToNotInclude2()
+ {
+ ColumnFamilyStore cfs = MockSchema.newCFS("anticomp");
+ List<SSTableReader> sstables = new ArrayList<>();
+ SSTableReader sstable1 = MockSchema.sstable(1, 10, 10, cfs);
+ SSTableReader sstable2 = MockSchema.sstable(2, 100, 200, cfs);
+ sstables.add(sstable1);
+ sstables.add(sstable2);
+
+ Range<Token> r = new Range<>(t(10), t(11)); // no sstable included, throw
+
+ CompactionManager.validateSSTableBoundsForAnticompaction(UUID.randomUUID(), sstables, atEndpoint(Collections.singletonList(r), NO_RANGES));
+ }
+
+ @Test
+ public void testSSTablesToInclude4()
+ {
+ ColumnFamilyStore cfs = MockSchema.newCFS("anticomp");
+ List<SSTableReader> sstables = new ArrayList<>();
+ SSTableReader sstable1 = MockSchema.sstable(1, 10, 100, cfs);
+ SSTableReader sstable2 = MockSchema.sstable(2, 100, 200, cfs);
+ sstables.add(sstable1);
+ sstables.add(sstable2);
+
+ Range<Token> r = new Range<>(t(9), t(200)); // sstable 2 is fully contained - last token is equal
+
+ Iterator<SSTableReader> sstableIterator = sstables.iterator();
+ Set<SSTableReader> fullyContainedSSTables = CompactionManager.findSSTablesToAnticompact(sstableIterator, Collections.singletonList(r), UUID.randomUUID());
+ assertEquals(Sets.newHashSet(sstable1, sstable2), fullyContainedSSTables);
+ assertTrue(sstables.isEmpty());
+ }
+
+ private Token t(long t)
+ {
+ return new Murmur3Partitioner.LongToken(t);
+ }
}
diff --git a/test/unit/org/apache/cassandra/db/compaction/CancelCompactionsTest.java b/test/unit/org/apache/cassandra/db/compaction/CancelCompactionsTest.java
index bcbe92d..beed019 100644
--- a/test/unit/org/apache/cassandra/db/compaction/CancelCompactionsTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/CancelCompactionsTest.java
@@ -18,22 +18,441 @@
package org.apache.cassandra.db.compaction;
+import java.util.ArrayList;
import java.util.Collections;
+import java.util.HashSet;
import java.util.List;
+import java.util.Set;
+import java.util.UUID;
import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+import com.google.common.collect.ImmutableSet;
import com.google.common.util.concurrent.Uninterruptibles;
import org.junit.Test;
-import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.CQLTester;
-import org.apache.cassandra.metrics.CompactionMetrics;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.db.repair.PendingAntiCompaction;
+import org.apache.cassandra.dht.IPartitioner;
+import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.index.Index;
+import org.apache.cassandra.index.StubIndex;
+import org.apache.cassandra.index.internal.CollatedViewIndexBuilder;
+import org.apache.cassandra.io.sstable.ISSTableScanner;
+import org.apache.cassandra.io.sstable.ReducingKeyIterator;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.RangesAtEndpoint;
+import org.apache.cassandra.locator.Replica;
+import org.apache.cassandra.schema.MockSchema;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.service.ActiveRepairService;
+import org.apache.cassandra.streaming.PreviewKind;
+import org.apache.cassandra.utils.FBUtilities;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
public class CancelCompactionsTest extends CQLTester
{
+ /**
+ * makes sure we only cancel compactions if the precidate says we have overlapping sstables
+ */
+ @Test
+ public void cancelTest() throws InterruptedException
+ {
+ ColumnFamilyStore cfs = MockSchema.newCFS();
+ List<SSTableReader> sstables = createSSTables(cfs, 10, 0);
+ Set<SSTableReader> toMarkCompacting = new HashSet<>(sstables.subList(0, 3));
+
+ TestCompactionTask tct = new TestCompactionTask(cfs, toMarkCompacting);
+ try
+ {
+ tct.start();
+
+ List<CompactionInfo.Holder> activeCompactions = getActiveCompactionsForTable(cfs);
+ assertEquals(1, activeCompactions.size());
+ assertEquals(activeCompactions.get(0).getCompactionInfo().getSSTables(), toMarkCompacting);
+ // predicate requires the non-compacting sstables, should not cancel the one currently compacting:
+ cfs.runWithCompactionsDisabled(() -> null, (sstable) -> !toMarkCompacting.contains(sstable), false, false, true);
+ assertEquals(1, activeCompactions.size());
+ assertFalse(activeCompactions.get(0).isStopRequested());
+
+ // predicate requires the compacting ones - make sure stop is requested and that when we abort that
+ // compaction we actually run the callable (countdown the latch)
+ CountDownLatch cdl = new CountDownLatch(1);
+ Thread t = new Thread(() -> cfs.runWithCompactionsDisabled(() -> { cdl.countDown(); return null; }, toMarkCompacting::contains, false, false, true));
+ t.start();
+ while (!activeCompactions.get(0).isStopRequested())
+ Thread.sleep(100);
+
+ // cdl.countDown will not get executed until we have aborted all compactions for the sstables in toMarkCompacting
+ assertFalse(cdl.await(2, TimeUnit.SECONDS));
+ tct.abort();
+ // now the compactions are aborted and we can successfully wait for the latch
+ t.join();
+ assertTrue(cdl.await(2, TimeUnit.SECONDS));
+ }
+ finally
+ {
+ tct.abort();
+ }
+ }
+
+ /**
+ * make sure we only cancel relevant compactions when there are multiple ongoing compactions
+ */
+ @Test
+ public void multipleCompactionsCancelTest() throws InterruptedException
+ {
+ ColumnFamilyStore cfs = MockSchema.newCFS();
+ List<SSTableReader> sstables = createSSTables(cfs, 10, 0);
+
+ List<TestCompactionTask> tcts = new ArrayList<>();
+ tcts.add(new TestCompactionTask(cfs, new HashSet<>(sstables.subList(0, 3))));
+ tcts.add(new TestCompactionTask(cfs, new HashSet<>(sstables.subList(6, 9))));
+
+ try
+ {
+ tcts.forEach(TestCompactionTask::start);
+
+ List<CompactionInfo.Holder> activeCompactions = getActiveCompactionsForTable(cfs);
+ assertEquals(2, activeCompactions.size());
+
+ Set<Set<SSTableReader>> compactingSSTables = new HashSet<>();
+ compactingSSTables.add(activeCompactions.get(0).getCompactionInfo().getSSTables());
+ compactingSSTables.add(activeCompactions.get(1).getCompactionInfo().getSSTables());
+ Set<Set<SSTableReader>> expectedSSTables = new HashSet<>();
+ expectedSSTables.add(new HashSet<>(sstables.subList(0, 3)));
+ expectedSSTables.add(new HashSet<>(sstables.subList(6, 9)));
+ assertEquals(compactingSSTables, expectedSSTables);
+
+ cfs.runWithCompactionsDisabled(() -> null, (sstable) -> false, false, false, true);
+ assertEquals(2, activeCompactions.size());
+ assertTrue(activeCompactions.stream().noneMatch(CompactionInfo.Holder::isStopRequested));
+
+ CountDownLatch cdl = new CountDownLatch(1);
+ // start a compaction which only needs the sstables where first token is > 50 - these are the sstables compacted by tcts.get(1)
+ Thread t = new Thread(() -> cfs.runWithCompactionsDisabled(() -> { cdl.countDown(); return null; }, (sstable) -> first(sstable) > 50, false, false, true));
+ t.start();
+ activeCompactions = getActiveCompactionsForTable(cfs);
+ assertEquals(2, activeCompactions.size());
+ Thread.sleep(500);
+ for (CompactionInfo.Holder holder : activeCompactions)
+ {
+ if (holder.getCompactionInfo().getSSTables().containsAll(sstables.subList(6, 9)))
+ assertTrue(holder.isStopRequested());
+ else
+ assertFalse(holder.isStopRequested());
+ }
+ tcts.get(1).abort();
+ assertEquals(1, CompactionManager.instance.active.getCompactions().size());
+ cdl.await();
+ t.join();
+ }
+ finally
+ {
+ tcts.forEach(TestCompactionTask::abort);
+ }
+ }
+
+ /**
+ * Makes sure sub range compaction now only cancels the relevant compactions, not all of them
+ */
+ @Test
+ public void testSubrangeCompaction() throws InterruptedException
+ {
+ ColumnFamilyStore cfs = MockSchema.newCFS();
+ List<SSTableReader> sstables = createSSTables(cfs, 10, 0);
+
+ List<TestCompactionTask> tcts = new ArrayList<>();
+ tcts.add(new TestCompactionTask(cfs, new HashSet<>(sstables.subList(0, 2))));
+ tcts.add(new TestCompactionTask(cfs, new HashSet<>(sstables.subList(3, 4))));
+ tcts.add(new TestCompactionTask(cfs, new HashSet<>(sstables.subList(5, 7))));
+ tcts.add(new TestCompactionTask(cfs, new HashSet<>(sstables.subList(8, 9))));
+ try
+ {
+ tcts.forEach(TestCompactionTask::start);
+
+ List<CompactionInfo.Holder> activeCompactions = getActiveCompactionsForTable(cfs);
+ assertEquals(4, activeCompactions.size());
+ Range<Token> range = new Range<>(token(0), token(49));
+ Thread t = new Thread(() -> {
+ try
+ {
+ cfs.forceCompactionForTokenRange(Collections.singleton(range));
+ }
+ catch (Throwable e)
+ {
+ throw new RuntimeException(e);
+ }
+ });
+
+ t.start();
+
+ Thread.sleep(500);
+ assertEquals(4, getActiveCompactionsForTable(cfs).size());
+ List<TestCompactionTask> toAbort = new ArrayList<>();
+ for (CompactionInfo.Holder holder : getActiveCompactionsForTable(cfs))
+ {
+ if (holder.getCompactionInfo().getSSTables().stream().anyMatch(sstable -> sstable.intersects(Collections.singleton(range))))
+ {
+ assertTrue(holder.isStopRequested());
+ for (TestCompactionTask tct : tcts)
+ if (tct.sstables.equals(holder.getCompactionInfo().getSSTables()))
+ toAbort.add(tct);
+ }
+ else
+ assertFalse(holder.isStopRequested());
+ }
+ assertEquals(2, toAbort.size());
+ toAbort.forEach(TestCompactionTask::abort);
+ t.join();
+
+ }
+ finally
+ {
+ tcts.forEach(TestCompactionTask::abort);
+ }
+ }
+
+ @Test
+ public void testAnticompaction() throws InterruptedException, ExecutionException
+ {
+ ColumnFamilyStore cfs = MockSchema.newCFS();
+ List<SSTableReader> sstables = createSSTables(cfs, 10, 0);
+ List<SSTableReader> alreadyRepairedSSTables = createSSTables(cfs, 10, 10);
+ for (SSTableReader sstable : alreadyRepairedSSTables)
+ AbstractPendingRepairTest.mutateRepaired(sstable, System.currentTimeMillis());
+ assertEquals(20, cfs.getLiveSSTables().size());
+ List<TestCompactionTask> tcts = new ArrayList<>();
+ tcts.add(new TestCompactionTask(cfs, new HashSet<>(sstables.subList(0, 2))));
+ tcts.add(new TestCompactionTask(cfs, new HashSet<>(sstables.subList(3, 4))));
+ tcts.add(new TestCompactionTask(cfs, new HashSet<>(sstables.subList(5, 7))));
+ tcts.add(new TestCompactionTask(cfs, new HashSet<>(sstables.subList(8, 9))));
+
+ List<TestCompactionTask> nonAffectedTcts = new ArrayList<>();
+ nonAffectedTcts.add(new TestCompactionTask(cfs, new HashSet<>(alreadyRepairedSSTables)));
+
+ try
+ {
+ tcts.forEach(TestCompactionTask::start);
+ nonAffectedTcts.forEach(TestCompactionTask::start);
+ List<CompactionInfo.Holder> activeCompactions = getActiveCompactionsForTable(cfs);
+ assertEquals(5, activeCompactions.size());
+ // make sure that sstables are fully contained so that the metadata gets mutated
+ Range<Token> range = new Range<>(token(-1), token(49));
+
+ UUID prsid = UUID.randomUUID();
+ ActiveRepairService.instance.registerParentRepairSession(prsid, InetAddressAndPort.getLocalHost(), Collections.singletonList(cfs), Collections.singleton(range), true, 1, true, PreviewKind.NONE);
+
+ InetAddressAndPort local = FBUtilities.getBroadcastAddressAndPort();
+ RangesAtEndpoint rae = RangesAtEndpoint.builder(local).add(new Replica(local, range, true)).build();
+
+ PendingAntiCompaction pac = new PendingAntiCompaction(prsid, Collections.singleton(cfs), rae, Executors.newSingleThreadExecutor(), () -> false);
+ Future<?> fut = pac.run();
+ Thread.sleep(600);
+ List<TestCompactionTask> toAbort = new ArrayList<>();
+ for (CompactionInfo.Holder holder : getActiveCompactionsForTable(cfs))
+ {
+ if (holder.getCompactionInfo().getSSTables().stream().anyMatch(sstable -> sstable.intersects(Collections.singleton(range)) && !sstable.isRepaired() && !sstable.isPendingRepair()))
+ {
+ assertTrue(holder.isStopRequested());
+ for (TestCompactionTask tct : tcts)
+ if (tct.sstables.equals(holder.getCompactionInfo().getSSTables()))
+ toAbort.add(tct);
+ }
+ else
+ assertFalse(holder.isStopRequested());
+ }
+ assertEquals(2, toAbort.size());
+ toAbort.forEach(TestCompactionTask::abort);
+ fut.get();
+ for (SSTableReader sstable : sstables)
+ assertTrue(!sstable.intersects(Collections.singleton(range)) || sstable.isPendingRepair());
+ }
+ finally
+ {
+ tcts.forEach(TestCompactionTask::abort);
+ nonAffectedTcts.forEach(TestCompactionTask::abort);
+ }
+ }
+
+ /**
+ * Make sure index rebuilds get cancelled
+ */
+ @Test
+ public void testIndexRebuild() throws ExecutionException, InterruptedException
+ {
+ ColumnFamilyStore cfs = MockSchema.newCFS();
+ List<SSTableReader> sstables = createSSTables(cfs, 5, 0);
+ Index idx = new StubIndex(cfs, null);
+ CountDownLatch indexBuildStarted = new CountDownLatch(1);
+ CountDownLatch indexBuildRunning = new CountDownLatch(1);
+ CountDownLatch compactionsStopped = new CountDownLatch(1);
+ ReducingKeyIterator reducingKeyIterator = new ReducingKeyIterator(sstables)
+ {
+ @Override
+ public boolean hasNext()
+ {
+ indexBuildStarted.countDown();
+ try
+ {
+ indexBuildRunning.await();
+ }
+ catch (InterruptedException e)
+ {
+ throw new RuntimeException();
+ }
+ return false;
+ }
+ };
+ Future<?> f = CompactionManager.instance.submitIndexBuild(new CollatedViewIndexBuilder(cfs, Collections.singleton(idx), reducingKeyIterator, ImmutableSet.copyOf(sstables)));
+ // wait for hasNext to get called
+ indexBuildStarted.await();
+ assertEquals(1, getActiveCompactionsForTable(cfs).size());
+ boolean foundCompaction = false;
+ for (CompactionInfo.Holder holder : getActiveCompactionsForTable(cfs))
+ {
+ if (holder.getCompactionInfo().getSSTables().equals(new HashSet<>(sstables)))
+ {
+ assertFalse(holder.isStopRequested());
+ foundCompaction = true;
+ }
+ }
+ assertTrue(foundCompaction);
+ cfs.runWithCompactionsDisabled(() -> {compactionsStopped.countDown(); return null;}, (sstable) -> true, false, false, true);
+ // wait for the runWithCompactionsDisabled callable
+ compactionsStopped.await();
+ assertEquals(1, getActiveCompactionsForTable(cfs).size());
+ foundCompaction = false;
+ for (CompactionInfo.Holder holder : getActiveCompactionsForTable(cfs))
+ {
+ if (holder.getCompactionInfo().getSSTables().equals(new HashSet<>(sstables)))
+ {
+ assertTrue(holder.isStopRequested());
+ foundCompaction = true;
+ }
+ }
+ assertTrue(foundCompaction);
+ // signal that the index build should be finished
+ indexBuildRunning.countDown();
+ f.get();
+ assertTrue(getActiveCompactionsForTable(cfs).isEmpty());
+ }
+
+ long first(SSTableReader sstable)
+ {
+ return (long)sstable.first.getToken().getTokenValue();
+ }
+
+ Token token(long t)
+ {
+ return new Murmur3Partitioner.LongToken(t);
+ }
+
+ private List<SSTableReader> createSSTables(ColumnFamilyStore cfs, int count, int startGeneration)
+ {
+ List<SSTableReader> sstables = new ArrayList<>();
+ for (int i = 0; i < count; i++)
+ {
+ long first = i * 10;
+ long last = (i + 1) * 10 - 1;
+ sstables.add(MockSchema.sstable(startGeneration + i, 0, true, first, last, cfs));
+ }
+ cfs.disableAutoCompaction();
+ cfs.addSSTables(sstables);
+ return sstables;
+ }
+
+ private static class TestCompactionTask
+ {
+ private ColumnFamilyStore cfs;
+ private final Set<SSTableReader> sstables;
+ private LifecycleTransaction txn;
+ private CompactionController controller;
+ private CompactionIterator ci;
+ private List<ISSTableScanner> scanners;
+
+ public TestCompactionTask(ColumnFamilyStore cfs, Set<SSTableReader> sstables)
+ {
+ this.cfs = cfs;
+ this.sstables = sstables;
+ }
+
+ public void start()
+ {
+ scanners = sstables.stream().map(SSTableReader::getScanner).collect(Collectors.toList());
+ txn = cfs.getTracker().tryModify(sstables, OperationType.COMPACTION);
+ assertNotNull(txn);
+ controller = new CompactionController(cfs, sstables, Integer.MIN_VALUE);
+ ci = new CompactionIterator(txn.opType(), scanners, controller, FBUtilities.nowInSeconds(), UUID.randomUUID());
+ CompactionManager.instance.active.beginCompaction(ci);
+ }
+
+ public void abort()
+ {
+ if (controller != null)
+ controller.close();
+ if (ci != null)
+ ci.close();
+ if (txn != null)
+ txn.abort();
+ if (scanners != null)
+ scanners.forEach(ISSTableScanner::close);
+ CompactionManager.instance.active.finishCompaction(ci);
+
+ }
+ }
+
+ @Test
+ public void test2iCancellation() throws Throwable
+ {
+ createTable("create table %s (id int primary key, something int)");
+ createIndex("create index on %s(something)");
+ getCurrentColumnFamilyStore().disableAutoCompaction();
+ for (int i = 0; i < 10; i++)
+ execute("insert into %s (id, something) values (?, ?)", i, i);
+ flush();
+ ColumnFamilyStore idx = getCurrentColumnFamilyStore().indexManager.getAllIndexColumnFamilyStores().iterator().next();
+ Set<SSTableReader> sstables = new HashSet<>();
+ try (LifecycleTransaction txn = idx.getTracker().tryModify(idx.getLiveSSTables(), OperationType.COMPACTION))
+ {
+ getCurrentColumnFamilyStore().runWithCompactionsDisabled(() -> true, (sstable) -> { sstables.add(sstable); return true;}, false, false, false);
+ }
+ // the predicate only gets compacting sstables, and we are only compacting the 2i sstables - with interruptIndexes = false we should see no sstables here
+ assertTrue(sstables.isEmpty());
+ }
+
+ @Test
+ public void testSubrangeCompactionWith2i() throws Throwable
+ {
+ createTable("create table %s (id int primary key, something int)");
+ createIndex("create index on %s(something)");
+ getCurrentColumnFamilyStore().disableAutoCompaction();
+ for (int i = 0; i < 10; i++)
+ execute("insert into %s (id, something) values (?, ?)", i, i);
+ flush();
+ ColumnFamilyStore idx = getCurrentColumnFamilyStore().indexManager.getAllIndexColumnFamilyStores().iterator().next();
+ try (LifecycleTransaction txn = idx.getTracker().tryModify(idx.getLiveSSTables(), OperationType.COMPACTION))
+ {
+ IPartitioner partitioner = getCurrentColumnFamilyStore().getPartitioner();
+ getCurrentColumnFamilyStore().forceCompactionForTokenRange(Collections.singleton(new Range<>(partitioner.getMinimumToken(), partitioner.getMaximumToken())));
+ }
+ }
+
@Test
public void testStandardCompactionTaskCancellation() throws Throwable
{
@@ -55,14 +474,13 @@
if (ct != null)
break;
}
- if (ct != null)
- break;
+ if (ct != null) break;
}
assertNotNull(ct);
CountDownLatch waitForBeginCompaction = new CountDownLatch(1);
CountDownLatch waitForStart = new CountDownLatch(1);
- Iterable<CFMetaData> metadatas = Collections.singleton(getCurrentColumnFamilyStore().metadata);
+ Iterable<TableMetadata> metadatas = Collections.singleton(getCurrentColumnFamilyStore().metadata());
/*
Here we ask strategies to pause & interrupt compactions right before calling beginCompaction in CompactionTask
The code running in the separate thread below mimics CFS#runWithCompactionsDisabled but we only allow
@@ -71,16 +489,16 @@
Thread t = new Thread(() -> {
Uninterruptibles.awaitUninterruptibly(waitForBeginCompaction);
getCurrentColumnFamilyStore().getCompactionStrategyManager().pause();
- CompactionManager.instance.interruptCompactionFor(metadatas, false);
+ CompactionManager.instance.interruptCompactionFor(metadatas, (s) -> true, false);
waitForStart.countDown();
- CompactionManager.instance.waitForCessation(Collections.singleton(getCurrentColumnFamilyStore()));
+ CompactionManager.instance.waitForCessation(Collections.singleton(getCurrentColumnFamilyStore()), (s) -> true);
getCurrentColumnFamilyStore().getCompactionStrategyManager().resume();
});
t.start();
try
{
- ct.execute(new CompactionMetrics()
+ ct.execute(new ActiveCompactions()
{
@Override
public void beginCompaction(CompactionInfo.Holder ci)
@@ -102,4 +520,12 @@
t.join();
}
}
+
+ private List<CompactionInfo.Holder> getActiveCompactionsForTable(ColumnFamilyStore cfs)
+ {
+ return CompactionManager.instance.active.getCompactions()
+ .stream()
+ .filter(holder -> holder.getCompactionInfo().getTable().orElse("unknown").equalsIgnoreCase(cfs.name))
+ .collect(Collectors.toList());
+ }
}
diff --git a/test/unit/org/apache/cassandra/db/compaction/CompactionControllerTest.java b/test/unit/org/apache/cassandra/db/compaction/CompactionControllerTest.java
index 052206e..0ab714a 100644
--- a/test/unit/org/apache/cassandra/db/compaction/CompactionControllerTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/CompactionControllerTest.java
@@ -20,6 +20,7 @@
import java.nio.ByteBuffer;
import java.util.Set;
+import java.util.function.LongPredicate;
import java.util.function.Predicate;
import com.google.common.collect.Sets;
@@ -28,7 +29,7 @@
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.Util;
-import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.Keyspace;
@@ -59,16 +60,18 @@
SchemaLoader.prepareServer();
SchemaLoader.createKeyspace(KEYSPACE,
KeyspaceParams.simple(1),
- CFMetaData.Builder.create(KEYSPACE, CF1, true, false, false)
- .addPartitionKey("pk", AsciiType.instance)
- .addClusteringColumn("ck", AsciiType.instance)
- .addRegularColumn("val", AsciiType.instance)
- .build(),
- CFMetaData.Builder.create(KEYSPACE, CF2, true, false, false)
- .addPartitionKey("pk", AsciiType.instance)
- .addClusteringColumn("ck", AsciiType.instance)
- .addRegularColumn("val", AsciiType.instance)
- .build());
+ TableMetadata.builder(KEYSPACE, CF1)
+ .isCompound(false)
+ .isDense(true)
+ .addPartitionKeyColumn("pk", AsciiType.instance)
+ .addClusteringColumn("ck", AsciiType.instance)
+ .addRegularColumn("val", AsciiType.instance),
+ TableMetadata.builder(KEYSPACE, CF2)
+ .isCompound(false)
+ .isDense(true)
+ .addPartitionKeyColumn("pk", AsciiType.instance)
+ .addClusteringColumn("ck", AsciiType.instance)
+ .addRegularColumn("val", AsciiType.instance));
}
@Test
@@ -85,7 +88,7 @@
long timestamp3 = timestamp2 - 5; // oldest timestamp
// add to first memtable
- applyMutation(cfs.metadata, key, timestamp1);
+ applyMutation(cfs.metadata(), key, timestamp1);
// check max purgeable timestamp without any sstables
try(CompactionController controller = new CompactionController(cfs, null, 0))
@@ -99,7 +102,7 @@
Set<SSTableReader> compacting = Sets.newHashSet(cfs.getLiveSSTables()); // first sstable is compacting
// create another sstable
- applyMutation(cfs.metadata, key, timestamp2);
+ applyMutation(cfs.metadata(), key, timestamp2);
cfs.forceBlockingFlush();
// check max purgeable timestamp when compacting the first sstable with and without a memtable
@@ -107,7 +110,7 @@
{
assertPurgeBoundary(controller.getPurgeEvaluator(key), timestamp2);
- applyMutation(cfs.metadata, key, timestamp3);
+ applyMutation(cfs.metadata(), key, timestamp3);
assertPurgeBoundary(controller.getPurgeEvaluator(key), timestamp3); //second sstable and second memtable
}
@@ -118,9 +121,9 @@
//newest to oldest
try (CompactionController controller = new CompactionController(cfs, null, 0))
{
- applyMutation(cfs.metadata, key, timestamp1);
- applyMutation(cfs.metadata, key, timestamp2);
- applyMutation(cfs.metadata, key, timestamp3);
+ applyMutation(cfs.metadata(), key, timestamp1);
+ applyMutation(cfs.metadata(), key, timestamp2);
+ applyMutation(cfs.metadata(), key, timestamp3);
assertPurgeBoundary(controller.getPurgeEvaluator(key), timestamp3); //memtable only
}
@@ -130,9 +133,9 @@
//oldest to newest
try (CompactionController controller = new CompactionController(cfs, null, 0))
{
- applyMutation(cfs.metadata, key, timestamp3);
- applyMutation(cfs.metadata, key, timestamp2);
- applyMutation(cfs.metadata, key, timestamp1);
+ applyMutation(cfs.metadata(), key, timestamp3);
+ applyMutation(cfs.metadata(), key, timestamp2);
+ applyMutation(cfs.metadata(), key, timestamp1);
assertPurgeBoundary(controller.getPurgeEvaluator(key), timestamp3);
}
@@ -152,14 +155,14 @@
long timestamp3 = timestamp2 - 5; // oldest timestamp
// create sstable with tombstone that should be expired in no older timestamps
- applyDeleteMutation(cfs.metadata, key, timestamp2);
+ applyDeleteMutation(cfs.metadata(), key, timestamp2);
cfs.forceBlockingFlush();
// first sstable with tombstone is compacting
Set<SSTableReader> compacting = Sets.newHashSet(cfs.getLiveSSTables());
// create another sstable with more recent timestamp
- applyMutation(cfs.metadata, key, timestamp1);
+ applyMutation(cfs.metadata(), key, timestamp1);
cfs.forceBlockingFlush();
// second sstable is overlapping
@@ -173,7 +176,7 @@
assertEquals(compacting.iterator().next(), expired.iterator().next());
// however if we add an older mutation to the memtable then the sstable should not be expired
- applyMutation(cfs.metadata, key, timestamp3);
+ applyMutation(cfs.metadata(), key, timestamp3);
expired = CompactionController.getFullyExpiredSSTables(cfs, compacting, overlapping, gcBefore);
assertNotNull(expired);
assertEquals(0, expired.size());
@@ -184,7 +187,7 @@
assertEquals(1, expired.size());
}
- private void applyMutation(CFMetaData cfm, DecoratedKey key, long timestamp)
+ private void applyMutation(TableMetadata cfm, DecoratedKey key, long timestamp)
{
ByteBuffer val = ByteBufferUtil.bytes(1L);
@@ -195,13 +198,13 @@
.applyUnsafe();
}
- private void applyDeleteMutation(CFMetaData cfm, DecoratedKey key, long timestamp)
+ private void applyDeleteMutation(TableMetadata cfm, DecoratedKey key, long timestamp)
{
new Mutation(PartitionUpdate.fullPartitionDelete(cfm, key, timestamp, FBUtilities.nowInSeconds()))
.applyUnsafe();
}
- private void assertPurgeBoundary(Predicate<Long> evaluator, long boundary)
+ private void assertPurgeBoundary(LongPredicate evaluator, long boundary)
{
assertFalse(evaluator.test(boundary));
assertTrue(evaluator.test(boundary - 1));
diff --git a/test/unit/org/apache/cassandra/db/compaction/CompactionExecutorTest.java b/test/unit/org/apache/cassandra/db/compaction/CompactionExecutorTest.java
index 2f8b5b2..ab3d9e5 100644
--- a/test/unit/org/apache/cassandra/db/compaction/CompactionExecutorTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/CompactionExecutorTest.java
@@ -22,6 +22,7 @@
import java.util.concurrent.TimeUnit;
import org.junit.After;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.apache.cassandra.concurrent.DebuggableThreadPoolExecutor;
@@ -62,7 +63,7 @@
public void destroy() throws Exception
{
executor.shutdown();
- executor.awaitTermination(1, TimeUnit.MINUTES);
+ Assert.assertTrue(executor.awaitTermination(1, TimeUnit.MINUTES));
}
@Test
diff --git a/test/unit/org/apache/cassandra/db/compaction/CompactionIteratorTest.java b/test/unit/org/apache/cassandra/db/compaction/CompactionIteratorTest.java
index 58c5a00..719cd7d 100644
--- a/test/unit/org/apache/cassandra/db/compaction/CompactionIteratorTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/CompactionIteratorTest.java
@@ -22,7 +22,6 @@
import static org.apache.cassandra.db.transform.DuplicateRowCheckerTest.rows;
import static org.junit.Assert.*;
-import java.net.InetAddress;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -33,7 +32,6 @@
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.Util;
-import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.CQLTester;
import org.apache.cassandra.db.ColumnFamilyStore;
@@ -45,11 +43,12 @@
import org.apache.cassandra.db.partitions.AbstractUnfilteredPartitionIterator;
import org.apache.cassandra.db.rows.*;
import org.apache.cassandra.io.sstable.ISSTableScanner;
-import org.apache.cassandra.net.IMessageSink;
-import org.apache.cassandra.net.MessageIn;
-import org.apache.cassandra.net.MessageOut;
-import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.net.Message;
import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FBUtilities;
@@ -62,7 +61,7 @@
private static final String CFNAME = "Integer1";
static final DecoratedKey kk;
- static final CFMetaData metadata;
+ static final TableMetadata metadata;
private static final int RANGE = 1000;
private static final int COUNT = 100;
@@ -81,7 +80,7 @@
1,
UTF8Type.instance,
Int32Type.instance,
- Int32Type.instance));
+ Int32Type.instance).build());
}
// See org.apache.cassandra.db.rows.UnfilteredRowsGenerator.parse for the syntax used in these tests.
@@ -321,6 +320,67 @@
}
}
+ @Test
+ public void transformTest()
+ {
+ UnfilteredRowsGenerator generator = new UnfilteredRowsGenerator(metadata.comparator, false);
+ List<List<Unfiltered>> inputLists = parse(new String[] {"10[100] 11[100] 12[100]"}, generator);
+ List<List<Unfiltered>> tombstoneLists = parse(new String[] {}, generator);
+ List<Iterable<UnfilteredRowIterator>> content = ImmutableList.copyOf(Iterables.transform(inputLists, list -> ImmutableList.of(listToIterator(list, kk))));
+ Map<DecoratedKey, Iterable<UnfilteredRowIterator>> transformedSources = new TreeMap<>();
+ transformedSources.put(kk, Iterables.transform(tombstoneLists, list -> listToIterator(list, kk)));
+ try (CompactionController controller = new Controller(Keyspace.openAndGetStore(metadata), transformedSources, GC_BEFORE);
+ CompactionIterator iter = new CompactionIterator(OperationType.COMPACTION,
+ Lists.transform(content, x -> new Scanner(x)),
+ controller, NOW, null))
+ {
+ assertTrue(iter.hasNext());
+ UnfilteredRowIterator rows = iter.next();
+ assertTrue(rows.hasNext());
+ assertNotNull(rows.next());
+
+ iter.stop();
+ try
+ {
+ // Will call Transformation#applyToRow
+ rows.hasNext();
+ fail("Should have thrown CompactionInterruptedException");
+ }
+ catch (CompactionInterruptedException e)
+ {
+ // ignore
+ }
+ }
+ }
+
+ @Test
+ public void transformPartitionTest()
+ {
+ UnfilteredRowsGenerator generator = new UnfilteredRowsGenerator(metadata.comparator, false);
+ List<List<Unfiltered>> inputLists = parse(new String[] {"10[100] 11[100] 12[100]"}, generator);
+ List<List<Unfiltered>> tombstoneLists = parse(new String[] {}, generator);
+ List<Iterable<UnfilteredRowIterator>> content = ImmutableList.copyOf(Iterables.transform(inputLists, list -> ImmutableList.of(listToIterator(list, kk))));
+ Map<DecoratedKey, Iterable<UnfilteredRowIterator>> transformedSources = new TreeMap<>();
+ transformedSources.put(kk, Iterables.transform(tombstoneLists, list -> listToIterator(list, kk)));
+ try (CompactionController controller = new Controller(Keyspace.openAndGetStore(metadata), transformedSources, GC_BEFORE);
+ CompactionIterator iter = new CompactionIterator(OperationType.COMPACTION,
+ Lists.transform(content, x -> new Scanner(x)),
+ controller, NOW, null))
+ {
+ iter.stop();
+ try
+ {
+ // Will call Transformation#applyToPartition
+ iter.hasNext();
+ fail("Should have thrown CompactionInterruptedException");
+ }
+ catch (CompactionInterruptedException e)
+ {
+ // ignore
+ }
+ }
+ }
+
class Controller extends CompactionController
{
private final Map<DecoratedKey, Iterable<UnfilteredRowIterator>> tombstoneSources;
@@ -349,13 +409,7 @@
}
@Override
- public boolean isForThrift()
- {
- return false;
- }
-
- @Override
- public CFMetaData metadata()
+ public TableMetadata metadata()
{
return metadata;
}
@@ -397,9 +451,9 @@
}
@Override
- public String getBackingFiles()
+ public Set<SSTableReader> getBackingSSTables()
{
- return null;
+ return ImmutableSet.of();
}
}
@@ -412,26 +466,13 @@
createTable("CREATE TABLE %s (pk text, ck1 int, ck2 int, v int, PRIMARY KEY (pk, ck1, ck2))");
for (int i = 0; i < 10; i++)
execute("insert into %s (pk, ck1, ck2, v) values (?, ?, ?, ?)", "key", i, i, i);
- flush();
+ getCurrentColumnFamilyStore().forceBlockingFlush();
DatabaseDescriptor.setSnapshotOnDuplicateRowDetection(true);
- CFMetaData metadata = getCurrentColumnFamilyStore().metadata;
+ TableMetadata metadata = getCurrentColumnFamilyStore().metadata();
- final HashMap<InetAddress, MessageOut> sentMessages = new HashMap<>();
- IMessageSink sink = new IMessageSink()
- {
- public boolean allowOutgoingMessage(MessageOut message, int id, InetAddress to)
- {
- sentMessages.put(to, message);
- return false;
- }
-
- public boolean allowIncomingMessage(MessageIn message, int id)
- {
- return false;
- }
- };
- MessagingService.instance().addMessageSink(sink);
+ final HashMap<InetAddressAndPort, Message<?>> sentMessages = new HashMap<>();
+ MessagingService.instance().outboundSink.add((message, to) -> { sentMessages.put(to, message); return false;});
// no duplicates
sentMessages.clear();
@@ -451,7 +492,7 @@
private void iterate(Unfiltered...unfiltereds)
{
ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
- DecoratedKey key = cfs.metadata.partitioner.decorateKey(ByteBufferUtil.bytes("key"));
+ DecoratedKey key = cfs.getPartitioner().decorateKey(ByteBufferUtil.bytes("key"));
try (CompactionController controller = new CompactionController(cfs, Integer.MAX_VALUE);
UnfilteredRowIterator rows = rows(metadata, key, false, unfiltereds);
ISSTableScanner scanner = new Scanner(Collections.singletonList(rows));
diff --git a/test/unit/org/apache/cassandra/db/compaction/CompactionStrategyManagerPendingRepairTest.java b/test/unit/org/apache/cassandra/db/compaction/CompactionStrategyManagerPendingRepairTest.java
new file mode 100644
index 0000000..9f2bc2e
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/compaction/CompactionStrategyManagerPendingRepairTest.java
@@ -0,0 +1,374 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.UUID;
+
+import com.google.common.collect.Iterables;
+import org.junit.Assert;
+import org.junit.Test;
+
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.notifications.SSTableAddedNotification;
+import org.apache.cassandra.notifications.SSTableDeletingNotification;
+import org.apache.cassandra.notifications.SSTableListChangedNotification;
+import org.apache.cassandra.notifications.SSTableRepairStatusChanged;
+import org.apache.cassandra.repair.consistent.LocalSessionAccessor;
+import org.apache.cassandra.service.ActiveRepairService;
+import org.apache.cassandra.utils.FBUtilities;
+
+/**
+ * Tests CompactionStrategyManager's handling of pending repair sstables
+ */
+public class CompactionStrategyManagerPendingRepairTest extends AbstractPendingRepairTest
+{
+
+ private boolean transientContains(SSTableReader sstable)
+ {
+ return csm.getTransientRepairsUnsafe().containsSSTable(sstable);
+ }
+
+ private boolean pendingContains(SSTableReader sstable)
+ {
+ return csm.getPendingRepairsUnsafe().containsSSTable(sstable);
+ }
+
+ private boolean repairedContains(SSTableReader sstable)
+ {
+ return csm.getRepairedUnsafe().containsSSTable(sstable);
+ }
+
+ private boolean unrepairedContains(SSTableReader sstable)
+ {
+ return csm.getUnrepairedUnsafe().containsSSTable(sstable);
+ }
+
+ private boolean hasPendingStrategiesFor(UUID sessionID)
+ {
+ return !Iterables.isEmpty(csm.getPendingRepairsUnsafe().getStrategiesFor(sessionID));
+ }
+
+ private boolean hasTransientStrategiesFor(UUID sessionID)
+ {
+ return !Iterables.isEmpty(csm.getTransientRepairsUnsafe().getStrategiesFor(sessionID));
+ }
+
+ /**
+ * Pending repair strategy should be created when we encounter a new pending id
+ */
+ @Test
+ public void sstableAdded()
+ {
+ UUID repairID = registerSession(cfs, true, true);
+ LocalSessionAccessor.prepareUnsafe(repairID, COORDINATOR, PARTICIPANTS);
+ Assert.assertTrue(Iterables.isEmpty(csm.getPendingRepairsUnsafe().allStrategies()));
+
+ SSTableReader sstable = makeSSTable(true);
+ Assert.assertFalse(sstable.isRepaired());
+ Assert.assertFalse(sstable.isPendingRepair());
+
+ mutateRepaired(sstable, repairID, false);
+ Assert.assertFalse(sstable.isRepaired());
+ Assert.assertTrue(sstable.isPendingRepair());
+ Assert.assertFalse(hasPendingStrategiesFor(repairID));
+ Assert.assertFalse(hasTransientStrategiesFor(repairID));
+
+ // add the sstable
+ csm.handleNotification(new SSTableAddedNotification(Collections.singleton(sstable), null), cfs.getTracker());
+ Assert.assertFalse(repairedContains(sstable));
+ Assert.assertFalse(unrepairedContains(sstable));
+ Assert.assertTrue(pendingContains(sstable));
+ Assert.assertTrue(hasPendingStrategiesFor(repairID));
+ Assert.assertFalse(hasTransientStrategiesFor(repairID));
+ }
+
+ @Test
+ public void sstableListChangedAddAndRemove()
+ {
+ UUID repairID = registerSession(cfs, true, true);
+ LocalSessionAccessor.prepareUnsafe(repairID, COORDINATOR, PARTICIPANTS);
+
+ SSTableReader sstable1 = makeSSTable(true);
+ mutateRepaired(sstable1, repairID, false);
+
+ SSTableReader sstable2 = makeSSTable(true);
+ mutateRepaired(sstable2, repairID, false);
+
+ Assert.assertFalse(repairedContains(sstable1));
+ Assert.assertFalse(unrepairedContains(sstable1));
+ Assert.assertFalse(repairedContains(sstable2));
+ Assert.assertFalse(unrepairedContains(sstable2));
+ Assert.assertFalse(hasPendingStrategiesFor(repairID));
+ Assert.assertFalse(hasTransientStrategiesFor(repairID));
+
+ // add only
+ SSTableListChangedNotification notification;
+ notification = new SSTableListChangedNotification(Collections.singleton(sstable1),
+ Collections.emptyList(),
+ OperationType.COMPACTION);
+ csm.handleNotification(notification, cfs.getTracker());
+
+ Assert.assertFalse(repairedContains(sstable1));
+ Assert.assertFalse(unrepairedContains(sstable1));
+ Assert.assertTrue(pendingContains(sstable1));
+ Assert.assertFalse(repairedContains(sstable2));
+ Assert.assertFalse(unrepairedContains(sstable2));
+ Assert.assertFalse(pendingContains(sstable2));
+ Assert.assertTrue(hasPendingStrategiesFor(repairID));
+ Assert.assertFalse(hasTransientStrategiesFor(repairID));
+
+ // remove and add
+ notification = new SSTableListChangedNotification(Collections.singleton(sstable2),
+ Collections.singleton(sstable1),
+ OperationType.COMPACTION);
+ csm.handleNotification(notification, cfs.getTracker());
+
+ Assert.assertFalse(repairedContains(sstable1));
+ Assert.assertFalse(unrepairedContains(sstable1));
+ Assert.assertFalse(pendingContains(sstable1));
+ Assert.assertFalse(repairedContains(sstable2));
+ Assert.assertFalse(unrepairedContains(sstable2));
+ Assert.assertTrue(pendingContains(sstable2));
+ }
+
+ @Test
+ public void sstableRepairStatusChanged()
+ {
+ UUID repairID = registerSession(cfs, true, true);
+ LocalSessionAccessor.prepareUnsafe(repairID, COORDINATOR, PARTICIPANTS);
+
+ // add as unrepaired
+ SSTableReader sstable = makeSSTable(false);
+ Assert.assertTrue(unrepairedContains(sstable));
+ Assert.assertFalse(repairedContains(sstable));
+ Assert.assertFalse(hasPendingStrategiesFor(repairID));
+ Assert.assertFalse(hasTransientStrategiesFor(repairID));
+
+ SSTableRepairStatusChanged notification;
+
+ // change to pending repaired
+ mutateRepaired(sstable, repairID, false);
+ notification = new SSTableRepairStatusChanged(Collections.singleton(sstable));
+ csm.handleNotification(notification, cfs.getTracker());
+ Assert.assertFalse(unrepairedContains(sstable));
+ Assert.assertFalse(repairedContains(sstable));
+ Assert.assertTrue(hasPendingStrategiesFor(repairID));
+ Assert.assertFalse(hasTransientStrategiesFor(repairID));
+ Assert.assertTrue(pendingContains(sstable));
+
+ // change to repaired
+ mutateRepaired(sstable, System.currentTimeMillis());
+ notification = new SSTableRepairStatusChanged(Collections.singleton(sstable));
+ csm.handleNotification(notification, cfs.getTracker());
+ Assert.assertFalse(unrepairedContains(sstable));
+ Assert.assertTrue(repairedContains(sstable));
+ Assert.assertFalse(pendingContains(sstable));
+ }
+
+ @Test
+ public void sstableDeleted()
+ {
+ UUID repairID = registerSession(cfs, true, true);
+ LocalSessionAccessor.prepareUnsafe(repairID, COORDINATOR, PARTICIPANTS);
+
+ SSTableReader sstable = makeSSTable(true);
+ mutateRepaired(sstable, repairID, false);
+ csm.handleNotification(new SSTableAddedNotification(Collections.singleton(sstable), null), cfs.getTracker());
+ Assert.assertTrue(pendingContains(sstable));
+
+ // delete sstable
+ SSTableDeletingNotification notification = new SSTableDeletingNotification(sstable);
+ csm.handleNotification(notification, cfs.getTracker());
+ Assert.assertFalse(pendingContains(sstable));
+ Assert.assertFalse(unrepairedContains(sstable));
+ Assert.assertFalse(repairedContains(sstable));
+ }
+
+ /**
+ * CompactionStrategyManager.getStrategies should include
+ * pending repair strategies when appropriate
+ */
+ @Test
+ public void getStrategies()
+ {
+ UUID repairID = registerSession(cfs, true, true);
+ LocalSessionAccessor.prepareUnsafe(repairID, COORDINATOR, PARTICIPANTS);
+
+ List<List<AbstractCompactionStrategy>> strategies;
+
+ strategies = csm.getStrategies();
+ Assert.assertEquals(3, strategies.size());
+ Assert.assertTrue(strategies.get(2).isEmpty());
+
+ SSTableReader sstable = makeSSTable(true);
+ mutateRepaired(sstable, repairID, false);
+ csm.handleNotification(new SSTableAddedNotification(Collections.singleton(sstable), null), cfs.getTracker());
+
+ strategies = csm.getStrategies();
+ Assert.assertEquals(3, strategies.size());
+ Assert.assertFalse(strategies.get(2).isEmpty());
+ }
+
+ /**
+ * Tests that finalized repairs result in cleanup compaction tasks
+ * which reclassify the sstables as repaired
+ */
+ @Test
+ public void cleanupCompactionFinalized()
+ {
+ UUID repairID = registerSession(cfs, true, true);
+ LocalSessionAccessor.prepareUnsafe(repairID, COORDINATOR, PARTICIPANTS);
+ SSTableReader sstable = makeSSTable(true);
+ mutateRepaired(sstable, repairID, false);
+ csm.handleNotification(new SSTableAddedNotification(Collections.singleton(sstable), null), cfs.getTracker());
+ LocalSessionAccessor.finalizeUnsafe(repairID);
+ Assert.assertTrue(hasPendingStrategiesFor(repairID));
+ Assert.assertFalse(hasTransientStrategiesFor(repairID));
+ Assert.assertTrue(pendingContains(sstable));
+ Assert.assertTrue(sstable.isPendingRepair());
+ Assert.assertFalse(sstable.isRepaired());
+
+ cfs.getCompactionStrategyManager().enable(); // enable compaction to fetch next background task
+ AbstractCompactionTask compactionTask = csm.getNextBackgroundTask(FBUtilities.nowInSeconds());
+ Assert.assertNotNull(compactionTask);
+ Assert.assertSame(PendingRepairManager.RepairFinishedCompactionTask.class, compactionTask.getClass());
+
+ // run the compaction
+ compactionTask.execute(ActiveCompactionsTracker.NOOP);
+
+ Assert.assertTrue(repairedContains(sstable));
+ Assert.assertFalse(unrepairedContains(sstable));
+ Assert.assertFalse(pendingContains(sstable));
+ Assert.assertFalse(hasPendingStrategiesFor(repairID));
+ Assert.assertFalse(hasTransientStrategiesFor(repairID));
+
+ // sstable should have pendingRepair cleared, and repairedAt set correctly
+ long expectedRepairedAt = ActiveRepairService.instance.getParentRepairSession(repairID).repairedAt;
+ Assert.assertFalse(sstable.isPendingRepair());
+ Assert.assertTrue(sstable.isRepaired());
+ Assert.assertEquals(expectedRepairedAt, sstable.getSSTableMetadata().repairedAt);
+ }
+
+ /**
+ * Tests that failed repairs result in cleanup compaction tasks
+ * which reclassify the sstables as unrepaired
+ */
+ @Test
+ public void cleanupCompactionFailed()
+ {
+ UUID repairID = registerSession(cfs, true, true);
+ LocalSessionAccessor.prepareUnsafe(repairID, COORDINATOR, PARTICIPANTS);
+ SSTableReader sstable = makeSSTable(true);
+ mutateRepaired(sstable, repairID, false);
+ csm.handleNotification(new SSTableAddedNotification(Collections.singleton(sstable), null), cfs.getTracker());
+ LocalSessionAccessor.failUnsafe(repairID);
+
+ Assert.assertTrue(hasPendingStrategiesFor(repairID));
+ Assert.assertFalse(hasTransientStrategiesFor(repairID));
+ Assert.assertTrue(pendingContains(sstable));
+ Assert.assertTrue(sstable.isPendingRepair());
+ Assert.assertFalse(sstable.isRepaired());
+
+ cfs.getCompactionStrategyManager().enable(); // enable compaction to fetch next background task
+ AbstractCompactionTask compactionTask = csm.getNextBackgroundTask(FBUtilities.nowInSeconds());
+ Assert.assertNotNull(compactionTask);
+ Assert.assertSame(PendingRepairManager.RepairFinishedCompactionTask.class, compactionTask.getClass());
+
+ // run the compaction
+ compactionTask.execute(ActiveCompactionsTracker.NOOP);
+
+ Assert.assertFalse(repairedContains(sstable));
+ Assert.assertTrue(unrepairedContains(sstable));
+ Assert.assertFalse(hasPendingStrategiesFor(repairID));
+ Assert.assertFalse(hasTransientStrategiesFor(repairID));
+
+ // sstable should have pendingRepair cleared, and repairedAt set correctly
+ Assert.assertFalse(sstable.isPendingRepair());
+ Assert.assertFalse(sstable.isRepaired());
+ Assert.assertEquals(ActiveRepairService.UNREPAIRED_SSTABLE, sstable.getSSTableMetadata().repairedAt);
+ }
+
+ @Test
+ public void finalizedSessionTransientCleanup()
+ {
+ Assert.assertTrue(cfs.getLiveSSTables().isEmpty());
+ UUID repairID = registerSession(cfs, true, true);
+ LocalSessionAccessor.prepareUnsafe(repairID, COORDINATOR, PARTICIPANTS);
+ SSTableReader sstable = makeSSTable(true);
+ mutateRepaired(sstable, repairID, true);
+ csm.handleNotification(new SSTableAddedNotification(Collections.singleton(sstable), null), cfs.getTracker());
+ LocalSessionAccessor.finalizeUnsafe(repairID);
+
+ Assert.assertFalse(hasPendingStrategiesFor(repairID));
+ Assert.assertTrue(hasTransientStrategiesFor(repairID));
+ Assert.assertTrue(transientContains(sstable));
+ Assert.assertFalse(pendingContains(sstable));
+ Assert.assertFalse(repairedContains(sstable));
+ Assert.assertFalse(unrepairedContains(sstable));
+
+ cfs.getCompactionStrategyManager().enable(); // enable compaction to fetch next background task
+ AbstractCompactionTask compactionTask = csm.getNextBackgroundTask(FBUtilities.nowInSeconds());
+ Assert.assertNotNull(compactionTask);
+ Assert.assertSame(PendingRepairManager.RepairFinishedCompactionTask.class, compactionTask.getClass());
+
+ // run the compaction
+ compactionTask.execute(ActiveCompactionsTracker.NOOP);
+
+ Assert.assertTrue(cfs.getLiveSSTables().isEmpty());
+ Assert.assertFalse(hasPendingStrategiesFor(repairID));
+ Assert.assertFalse(hasTransientStrategiesFor(repairID));
+ }
+
+ @Test
+ public void failedSessionTransientCleanup()
+ {
+ Assert.assertTrue(cfs.getLiveSSTables().isEmpty());
+ UUID repairID = registerSession(cfs, true, true);
+ LocalSessionAccessor.prepareUnsafe(repairID, COORDINATOR, PARTICIPANTS);
+ SSTableReader sstable = makeSSTable(true);
+ mutateRepaired(sstable, repairID, true);
+ csm.handleNotification(new SSTableAddedNotification(Collections.singleton(sstable), null), cfs.getTracker());
+ LocalSessionAccessor.failUnsafe(repairID);
+
+ Assert.assertFalse(hasPendingStrategiesFor(repairID));
+ Assert.assertTrue(hasTransientStrategiesFor(repairID));
+ Assert.assertTrue(transientContains(sstable));
+ Assert.assertFalse(pendingContains(sstable));
+ Assert.assertFalse(repairedContains(sstable));
+ Assert.assertFalse(unrepairedContains(sstable));
+
+ cfs.getCompactionStrategyManager().enable(); // enable compaction to fetch next background task
+ AbstractCompactionTask compactionTask = csm.getNextBackgroundTask(FBUtilities.nowInSeconds());
+ Assert.assertNotNull(compactionTask);
+ Assert.assertSame(PendingRepairManager.RepairFinishedCompactionTask.class, compactionTask.getClass());
+
+ // run the compaction
+ compactionTask.execute(ActiveCompactionsTracker.NOOP);
+
+ Assert.assertFalse(cfs.getLiveSSTables().isEmpty());
+ Assert.assertFalse(hasPendingStrategiesFor(repairID));
+ Assert.assertFalse(hasTransientStrategiesFor(repairID));
+ Assert.assertFalse(transientContains(sstable));
+ Assert.assertFalse(pendingContains(sstable));
+ Assert.assertFalse(repairedContains(sstable));
+ Assert.assertTrue(unrepairedContains(sstable));
+ }
+}
diff --git a/test/unit/org/apache/cassandra/db/compaction/CompactionStrategyManagerTest.java b/test/unit/org/apache/cassandra/db/compaction/CompactionStrategyManagerTest.java
index 2120757..d29ab52 100644
--- a/test/unit/org/apache/cassandra/db/compaction/CompactionStrategyManagerTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/CompactionStrategyManagerTest.java
@@ -19,16 +19,29 @@
package org.apache.cassandra.db.compaction;
import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
import com.google.common.io.Files;
import org.junit.AfterClass;
+import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.Util;
import org.apache.cassandra.config.DatabaseDescriptor;
@@ -36,10 +49,10 @@
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.Directories;
import org.apache.cassandra.db.DiskBoundaries;
-import org.apache.cassandra.db.DiskBoundaryManager;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.db.PartitionPosition;
import org.apache.cassandra.db.RowUpdateBuilder;
+import org.apache.cassandra.db.compaction.AbstractStrategyHolder.GroupedSSTableContainer;
import org.apache.cassandra.dht.ByteOrderedPartitioner;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.io.sstable.format.SSTableReader;
@@ -48,13 +61,20 @@
import org.apache.cassandra.schema.CompactionParams;
import org.apache.cassandra.schema.KeyspaceParams;
import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.UUIDGen;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertSame;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
public class CompactionStrategyManagerTest
{
+ private static final Logger logger = LoggerFactory.getLogger(CompactionStrategyManagerTest.class);
+
private static final String KS_PREFIX = "Keyspace1";
private static final String TABLE_PREFIX = "CF_STANDARD";
@@ -72,6 +92,17 @@
* disk assignment based on its generation - See {@link this#getSSTableIndex(Integer[], SSTableReader)}
*/
originalPartitioner = StorageService.instance.setPartitionerUnsafe(ByteOrderedPartitioner.instance);
+ SchemaLoader.createKeyspace(KS_PREFIX,
+ KeyspaceParams.simple(1),
+ SchemaLoader.standardCFMD(KS_PREFIX, TABLE_PREFIX)
+ .compaction(CompactionParams.stcs(Collections.emptyMap())));
+ }
+
+ @Before
+ public void setUp() throws Exception
+ {
+ ColumnFamilyStore cfs = Keyspace.open(KS_PREFIX).getColumnFamilyStore(TABLE_PREFIX);
+ cfs.truncateBlocking();
}
@AfterClass
@@ -82,19 +113,30 @@
}
@Test
- public void testSSTablesAssignedToCorrectCompactionStrategy()
+ public void testSSTablesAssignedToCorrectCompactionStrategy() throws IOException
{
// Creates 100 SSTables with keys 0-99
int numSSTables = 100;
- SchemaLoader.createKeyspace(KS_PREFIX,
- KeyspaceParams.simple(1),
- SchemaLoader.standardCFMD(KS_PREFIX, TABLE_PREFIX)
- .compaction(CompactionParams.scts(Collections.emptyMap())));
ColumnFamilyStore cfs = Keyspace.open(KS_PREFIX).getColumnFamilyStore(TABLE_PREFIX);
cfs.disableAutoCompaction();
+ Set<SSTableReader> previousSSTables = cfs.getLiveSSTables();
for (int i = 0; i < numSSTables; i++)
{
createSSTableWithKey(KS_PREFIX, TABLE_PREFIX, i);
+ Set<SSTableReader> currentSSTables = cfs.getLiveSSTables();
+ Set<SSTableReader> newSSTables = Sets.difference(currentSSTables, previousSSTables);
+ assertEquals(1, newSSTables.size());
+ if (i % 3 == 0)
+ {
+ //make 1 third of sstables repaired
+ cfs.getCompactionStrategyManager().mutateRepaired(newSSTables, System.currentTimeMillis(), null, false);
+ }
+ else if (i % 3 == 1)
+ {
+ //make 1 third of sstables pending repair
+ cfs.getCompactionStrategyManager().mutateRepaired(newSSTables, 0, UUIDGen.getTimeUUID(), false);
+ }
+ previousSSTables = currentSSTables;
}
// Creates a CompactionStrategymanager with different numbers of disks and check
@@ -116,7 +158,7 @@
final Integer[] boundaries = computeBoundaries(numSSTables, numDisks);
MockBoundaryManager mockBoundaryManager = new MockBoundaryManager(cfs, boundaries);
- System.out.println("Boundaries for " + numDisks + " disks is " + Arrays.toString(boundaries));
+ logger.debug("Boundaries for {} disks is {}", numDisks, Arrays.toString(boundaries));
CompactionStrategyManager csm = new CompactionStrategyManager(cfs, mockBoundaryManager::getBoundaries,
true);
@@ -133,7 +175,7 @@
updateBoundaries(mockBoundaryManager, boundaries, delta);
// Check that SSTables are still assigned to the previous boundary layout
- System.out.println("Old boundaries: " + Arrays.toString(previousBoundaries) + " New boundaries: " + Arrays.toString(boundaries));
+ logger.debug("Old boundaries: {} New boundaries: {}", Arrays.toString(previousBoundaries), Arrays.toString(boundaries));
for (SSTableReader reader : cfs.getLiveSSTables())
{
verifySSTableIsAssignedToCorrectStrategy(previousBoundaries, csm, reader);
@@ -152,12 +194,209 @@
assertFalse(((SizeTieredCompactionStrategy)csm.compactionStrategyFor(reader)).sstables.contains(reader));
// Add SSTable again and check that is correctly assigned
- csm.handleNotification(new SSTableAddedNotification(Collections.singleton(reader)), this);
+ csm.handleNotification(new SSTableAddedNotification(Collections.singleton(reader), null), this);
verifySSTableIsAssignedToCorrectStrategy(boundaries, csm, reader);
}
}
}
+ @Test
+ public void testAutomaticUpgradeConcurrency() throws Exception
+ {
+ ColumnFamilyStore cfs = Keyspace.open(KS_PREFIX).getColumnFamilyStore(TABLE_PREFIX);
+ DatabaseDescriptor.setAutomaticSSTableUpgradeEnabled(true);
+ DatabaseDescriptor.setMaxConcurrentAutoUpgradeTasks(1);
+
+ // latch to block CompactionManager.BackgroundCompactionCandidate#maybeRunUpgradeTask
+ // inside the currentlyBackgroundUpgrading check - with max_concurrent_auto_upgrade_tasks = 1 this will make
+ // sure that BackgroundCompactionCandidate#maybeRunUpgradeTask returns false until the latch has been counted down
+ CountDownLatch latch = new CountDownLatch(1);
+ AtomicInteger upgradeTaskCount = new AtomicInteger(0);
+ MockCFSForCSM mock = new MockCFSForCSM(cfs, latch, upgradeTaskCount);
+
+ CompactionManager.BackgroundCompactionCandidate r = CompactionManager.instance.getBackgroundCompactionCandidate(mock);
+ CompactionStrategyManager mgr = mock.getCompactionStrategyManager();
+ // basic idea is that we start a thread which will be able to get in to the currentlyBackgroundUpgrading-guarded
+ // code in CompactionManager, then we try to run a bunch more of the upgrade tasks which should return false
+ // due to the currentlyBackgroundUpgrading count being >= max_concurrent_auto_upgrade_tasks
+ Thread t = new Thread(() -> r.maybeRunUpgradeTask(mgr));
+ t.start();
+ Thread.sleep(100); // let the thread start and grab the task
+ assertEquals(1, CompactionManager.instance.currentlyBackgroundUpgrading.get());
+ assertFalse(r.maybeRunUpgradeTask(mgr));
+ assertFalse(r.maybeRunUpgradeTask(mgr));
+ latch.countDown();
+ t.join();
+ assertEquals(1, upgradeTaskCount.get()); // we should only call findUpgradeSSTableTask once when concurrency = 1
+ assertEquals(0, CompactionManager.instance.currentlyBackgroundUpgrading.get());
+
+ DatabaseDescriptor.setAutomaticSSTableUpgradeEnabled(false);
+ }
+
+ @Test
+ public void testAutomaticUpgradeConcurrency2() throws Exception
+ {
+ ColumnFamilyStore cfs = Keyspace.open(KS_PREFIX).getColumnFamilyStore(TABLE_PREFIX);
+ DatabaseDescriptor.setAutomaticSSTableUpgradeEnabled(true);
+ DatabaseDescriptor.setMaxConcurrentAutoUpgradeTasks(2);
+ // latch to block CompactionManager.BackgroundCompactionCandidate#maybeRunUpgradeTask
+ // inside the currentlyBackgroundUpgrading check - with max_concurrent_auto_upgrade_tasks = 1 this will make
+ // sure that BackgroundCompactionCandidate#maybeRunUpgradeTask returns false until the latch has been counted down
+ CountDownLatch latch = new CountDownLatch(1);
+ AtomicInteger upgradeTaskCount = new AtomicInteger();
+ MockCFSForCSM mock = new MockCFSForCSM(cfs, latch, upgradeTaskCount);
+
+ CompactionManager.BackgroundCompactionCandidate r = CompactionManager.instance.getBackgroundCompactionCandidate(mock);
+ CompactionStrategyManager mgr = mock.getCompactionStrategyManager();
+
+ // basic idea is that we start 2 threads who will be able to get in to the currentlyBackgroundUpgrading-guarded
+ // code in CompactionManager, then we try to run a bunch more of the upgrade task which should return false
+ // due to the currentlyBackgroundUpgrading count being >= max_concurrent_auto_upgrade_tasks
+ Thread t = new Thread(() -> r.maybeRunUpgradeTask(mgr));
+ t.start();
+ Thread t2 = new Thread(() -> r.maybeRunUpgradeTask(mgr));
+ t2.start();
+ Thread.sleep(100); // let the threads start and grab the task
+ assertEquals(2, CompactionManager.instance.currentlyBackgroundUpgrading.get());
+ assertFalse(r.maybeRunUpgradeTask(mgr));
+ assertFalse(r.maybeRunUpgradeTask(mgr));
+ assertFalse(r.maybeRunUpgradeTask(mgr));
+ assertEquals(2, CompactionManager.instance.currentlyBackgroundUpgrading.get());
+ latch.countDown();
+ t.join();
+ t2.join();
+ assertEquals(2, upgradeTaskCount.get());
+ assertEquals(0, CompactionManager.instance.currentlyBackgroundUpgrading.get());
+
+ DatabaseDescriptor.setMaxConcurrentAutoUpgradeTasks(1);
+ DatabaseDescriptor.setAutomaticSSTableUpgradeEnabled(false);
+ }
+
+ private static void assertHolderExclusivity(boolean isRepaired, boolean isPendingRepair, boolean isTransient, Class<? extends AbstractStrategyHolder> expectedType)
+ {
+ ColumnFamilyStore cfs = Keyspace.open(KS_PREFIX).getColumnFamilyStore(TABLE_PREFIX);
+ CompactionStrategyManager csm = cfs.getCompactionStrategyManager();
+
+ AbstractStrategyHolder holder = csm.getHolder(isRepaired, isPendingRepair, isTransient);
+ assertNotNull(holder);
+ assertSame(expectedType, holder.getClass());
+
+ int matches = 0;
+ for (AbstractStrategyHolder other : csm.getHolders())
+ {
+ if (other.managesRepairedGroup(isRepaired, isPendingRepair, isTransient))
+ {
+ assertSame("holder assignment should be mutually exclusive", holder, other);
+ matches++;
+ }
+ }
+ assertEquals(1, matches);
+ }
+
+ private static void assertInvalieHolderConfig(boolean isRepaired, boolean isPendingRepair, boolean isTransient)
+ {
+ ColumnFamilyStore cfs = Keyspace.open(KS_PREFIX).getColumnFamilyStore(TABLE_PREFIX);
+ CompactionStrategyManager csm = cfs.getCompactionStrategyManager();
+ try
+ {
+ csm.getHolder(isRepaired, isPendingRepair, isTransient);
+ fail("Expected IllegalArgumentException");
+ }
+ catch (IllegalArgumentException e)
+ {
+ // expected
+ }
+ }
+
+ /**
+ * If an sstable can be be assigned to a strategy holder, it shouldn't be possibly to
+ * assign it to any of the other holders.
+ */
+ @Test
+ public void testMutualExclusiveHolderClassification() throws Exception
+ {
+ assertHolderExclusivity(false, false, false, CompactionStrategyHolder.class);
+ assertHolderExclusivity(true, false, false, CompactionStrategyHolder.class);
+ assertHolderExclusivity(false, true, false, PendingRepairHolder.class);
+ assertHolderExclusivity(false, true, true, PendingRepairHolder.class);
+ assertInvalieHolderConfig(true, true, false);
+ assertInvalieHolderConfig(true, true, true);
+ assertInvalieHolderConfig(false, false, true);
+ assertInvalieHolderConfig(true, false, true);
+ }
+
+ PartitionPosition forKey(int key)
+ {
+ DecoratedKey dk = Util.dk(String.format("%04d", key));
+ return dk.getToken().minKeyBound();
+ }
+
+ /**
+ * Test that csm.groupSSTables correctly groups sstables by repaired status and directory
+ */
+ @Test
+ public void groupSSTables() throws Exception
+ {
+ final int numDir = 4;
+ ColumnFamilyStore cfs = createJBODMockCFS(numDir);
+ Keyspace.open(cfs.keyspace.getName()).getColumnFamilyStore(cfs.name).disableAutoCompaction();
+ assertTrue(cfs.getLiveSSTables().isEmpty());
+ List<SSTableReader> transientRepairs = new ArrayList<>();
+ List<SSTableReader> pendingRepair = new ArrayList<>();
+ List<SSTableReader> unrepaired = new ArrayList<>();
+ List<SSTableReader> repaired = new ArrayList<>();
+
+ for (int i = 0; i < numDir; i++)
+ {
+ int key = 100 * i;
+ transientRepairs.add(createSSTableWithKey(cfs.keyspace.getName(), cfs.name, key++));
+ pendingRepair.add(createSSTableWithKey(cfs.keyspace.getName(), cfs.name, key++));
+ unrepaired.add(createSSTableWithKey(cfs.keyspace.getName(), cfs.name, key++));
+ repaired.add(createSSTableWithKey(cfs.keyspace.getName(), cfs.name, key++));
+ }
+
+ cfs.getCompactionStrategyManager().mutateRepaired(transientRepairs, 0, UUID.randomUUID(), true);
+ cfs.getCompactionStrategyManager().mutateRepaired(pendingRepair, 0, UUID.randomUUID(), false);
+ cfs.getCompactionStrategyManager().mutateRepaired(repaired, 1000, null, false);
+
+ DiskBoundaries boundaries = new DiskBoundaries(cfs, cfs.getDirectories().getWriteableLocations(),
+ Lists.newArrayList(forKey(100), forKey(200), forKey(300)),
+ 10, 10);
+
+ CompactionStrategyManager csm = new CompactionStrategyManager(cfs, () -> boundaries, true);
+
+ List<GroupedSSTableContainer> grouped = csm.groupSSTables(Iterables.concat( transientRepairs, pendingRepair, repaired, unrepaired));
+
+ for (int x=0; x<grouped.size(); x++)
+ {
+ GroupedSSTableContainer group = grouped.get(x);
+ AbstractStrategyHolder holder = csm.getHolders().get(x);
+ for (int y=0; y<numDir; y++)
+ {
+ SSTableReader sstable = Iterables.getOnlyElement(group.getGroup(y));
+ assertTrue(holder.managesSSTable(sstable));
+ SSTableReader expected;
+ if (sstable.isRepaired())
+ expected = repaired.get(y);
+ else if (sstable.isPendingRepair())
+ {
+ if (sstable.isTransient())
+ {
+ expected = transientRepairs.get(y);
+ }
+ else
+ {
+ expected = pendingRepair.get(y);
+ }
+ }
+ else
+ expected = unrepaired.get(y);
+
+ assertSame(expected, sstable);
+ }
+ }
+ }
+
private MockCFS createJBODMockCFS(int disks)
{
// Create #disks data directories to simulate JBOD
@@ -170,7 +409,7 @@
}
ColumnFamilyStore cfs = Keyspace.open(KS_PREFIX).getColumnFamilyStore(TABLE_PREFIX);
- MockCFS mockCFS = new MockCFS(cfs, new Directories(cfs.metadata, directories));
+ MockCFS mockCFS = new MockCFS(cfs, new Directories(cfs.metadata(), directories));
mockCFS.disableAutoCompaction();
mockCFS.addSSTables(cfs.getLiveSSTables());
return mockCFS;
@@ -217,18 +456,13 @@
return result;
}
- /**
- * Since each SSTable contains keys from 0-99, and each sstable
- * generation is numbered from 1-100, since we are using ByteOrderedPartitioner
- * we can compute the sstable position in the disk boundaries by finding
- * the generation position relative to the boundaries
- */
private int getSSTableIndex(Integer[] boundaries, SSTableReader reader)
{
int index = 0;
- while (boundaries[index] < reader.descriptor.generation)
+ int firstKey = Integer.parseInt(new String(ByteBufferUtil.getArray(reader.first.getKey())));
+ while (boundaries[index] <= firstKey)
index++;
- System.out.println("Index for SSTable " + reader.descriptor.generation + " on boundary " + Arrays.toString(boundaries) + " is " + index);
+ logger.debug("Index for SSTable {} on boundary {} is {}", reader.descriptor.generation, Arrays.toString(boundaries), index);
return index;
}
@@ -266,17 +500,20 @@
}
}
- private static void createSSTableWithKey(String keyspace, String table, int key)
+ private static SSTableReader createSSTableWithKey(String keyspace, String table, int key)
{
long timestamp = System.currentTimeMillis();
DecoratedKey dk = Util.dk(String.format("%04d", key));
ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
- new RowUpdateBuilder(cfs.metadata, timestamp, dk.getKey())
+ new RowUpdateBuilder(cfs.metadata(), timestamp, dk.getKey())
.clustering(Integer.toString(key))
.add("val", "val")
.build()
.applyUnsafe();
+ Set<SSTableReader> before = cfs.getLiveSSTables();
cfs.forceBlockingFlush();
+ Set<SSTableReader> after = cfs.getLiveSSTables();
+ return Iterables.getOnlyElement(Sets.difference(after, before));
}
// just to be able to override the data directories
@@ -287,4 +524,50 @@
super(cfs.keyspace, cfs.getTableName(), 0, cfs.metadata, dirs, false, false, true);
}
}
+
+ private static class MockCFSForCSM extends ColumnFamilyStore
+ {
+ private final CountDownLatch latch;
+ private final AtomicInteger upgradeTaskCount;
+
+ private MockCFSForCSM(ColumnFamilyStore cfs, CountDownLatch latch, AtomicInteger upgradeTaskCount)
+ {
+ super(cfs.keyspace, cfs.name, 10, cfs.metadata, cfs.getDirectories(), true, false, false);
+ this.latch = latch;
+ this.upgradeTaskCount = upgradeTaskCount;
+ }
+ @Override
+ public CompactionStrategyManager getCompactionStrategyManager()
+ {
+ return new MockCSM(this, latch, upgradeTaskCount);
+ }
+ }
+
+ private static class MockCSM extends CompactionStrategyManager
+ {
+ private final CountDownLatch latch;
+ private final AtomicInteger upgradeTaskCount;
+
+ private MockCSM(ColumnFamilyStore cfs, CountDownLatch latch, AtomicInteger upgradeTaskCount)
+ {
+ super(cfs);
+ this.latch = latch;
+ this.upgradeTaskCount = upgradeTaskCount;
+ }
+
+ @Override
+ public AbstractCompactionTask findUpgradeSSTableTask()
+ {
+ try
+ {
+ latch.await();
+ upgradeTaskCount.incrementAndGet();
+ }
+ catch (InterruptedException e)
+ {
+ throw new RuntimeException(e);
+ }
+ return null;
+ }
+ }
}
diff --git a/test/unit/org/apache/cassandra/db/compaction/CompactionTaskTest.java b/test/unit/org/apache/cassandra/db/compaction/CompactionTaskTest.java
new file mode 100644
index 0000000..af74603
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/compaction/CompactionTaskTest.java
@@ -0,0 +1,157 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+import java.util.UUID;
+
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.cql3.QueryProcessor;
+import org.apache.cassandra.cql3.statements.schema.CreateTableStatement;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.service.ActiveRepairService;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.UUIDGen;
+import org.apache.cassandra.utils.concurrent.Transactional;
+
+public class CompactionTaskTest
+{
+ private static TableMetadata cfm;
+ private static ColumnFamilyStore cfs;
+
+ @BeforeClass
+ public static void setUpClass() throws Exception
+ {
+ SchemaLoader.prepareServer();
+ cfm = CreateTableStatement.parse("CREATE TABLE tbl (k INT PRIMARY KEY, v INT)", "coordinatorsessiontest").build();
+ SchemaLoader.createKeyspace("ks", KeyspaceParams.simple(1), cfm);
+ cfs = Schema.instance.getColumnFamilyStoreInstance(cfm.id);
+ }
+
+ @Before
+ public void setUp() throws Exception
+ {
+ cfs.getCompactionStrategyManager().enable();
+ cfs.truncateBlocking();
+ }
+
+ @Test
+ public void compactionInterruption() throws Exception
+ {
+ cfs.getCompactionStrategyManager().disable();
+ QueryProcessor.executeInternal("INSERT INTO ks.tbl (k, v) VALUES (1, 1);");
+ QueryProcessor.executeInternal("INSERT INTO ks.tbl (k, v) VALUES (2, 2);");
+ cfs.forceBlockingFlush();
+ QueryProcessor.executeInternal("INSERT INTO ks.tbl (k, v) VALUES (3, 3);");
+ QueryProcessor.executeInternal("INSERT INTO ks.tbl (k, v) VALUES (4, 4);");
+ cfs.forceBlockingFlush();
+ Set<SSTableReader> sstables = cfs.getLiveSSTables();
+
+ Assert.assertEquals(2, sstables.size());
+
+ LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.COMPACTION);
+ Assert.assertNotNull(txn);
+ CompactionTask task = new CompactionTask(cfs, txn, 0);
+ Assert.assertNotNull(task);
+ cfs.getCompactionStrategyManager().pause();
+ try
+ {
+ task.execute(CompactionManager.instance.active);
+ Assert.fail("Expected CompactionInterruptedException");
+ }
+ catch (CompactionInterruptedException e)
+ {
+ // expected
+ }
+ Assert.assertEquals(Transactional.AbstractTransactional.State.ABORTED, txn.state());
+ }
+
+ private static void mutateRepaired(SSTableReader sstable, long repairedAt, UUID pendingRepair, boolean isTransient) throws IOException
+ {
+ sstable.descriptor.getMetadataSerializer().mutateRepairMetadata(sstable.descriptor, repairedAt, pendingRepair, isTransient);
+ sstable.reloadSSTableMetadata();
+ }
+
+ /**
+ * If we try to create a compaction task that will mix
+ * repaired/unrepaired/pending repair sstables, it should fail
+ */
+ @Test
+ public void mixedSSTableFailure() throws Exception
+ {
+ cfs.getCompactionStrategyManager().disable();
+ QueryProcessor.executeInternal("INSERT INTO ks.tbl (k, v) VALUES (1, 1);");
+ cfs.forceBlockingFlush();
+ QueryProcessor.executeInternal("INSERT INTO ks.tbl (k, v) VALUES (2, 2);");
+ cfs.forceBlockingFlush();
+ QueryProcessor.executeInternal("INSERT INTO ks.tbl (k, v) VALUES (3, 3);");
+ cfs.forceBlockingFlush();
+ QueryProcessor.executeInternal("INSERT INTO ks.tbl (k, v) VALUES (4, 4);");
+ cfs.forceBlockingFlush();
+
+ List<SSTableReader> sstables = new ArrayList<>(cfs.getLiveSSTables());
+ Assert.assertEquals(4, sstables.size());
+
+ SSTableReader unrepaired = sstables.get(0);
+ SSTableReader repaired = sstables.get(1);
+ SSTableReader pending1 = sstables.get(2);
+ SSTableReader pending2 = sstables.get(3);
+
+ mutateRepaired(repaired, FBUtilities.nowInSeconds(), ActiveRepairService.NO_PENDING_REPAIR, false);
+ mutateRepaired(pending1, ActiveRepairService.UNREPAIRED_SSTABLE, UUIDGen.getTimeUUID(), false);
+ mutateRepaired(pending2, ActiveRepairService.UNREPAIRED_SSTABLE, UUIDGen.getTimeUUID(), false);
+
+ LifecycleTransaction txn = null;
+ List<SSTableReader> toCompact = new ArrayList<>(sstables);
+ for (int i=0; i<sstables.size(); i++)
+ {
+ try
+ {
+ txn = cfs.getTracker().tryModify(sstables, OperationType.COMPACTION);
+ Assert.assertNotNull(txn);
+ CompactionTask task = new CompactionTask(cfs, txn, 0);
+ Assert.fail("Expected IllegalArgumentException");
+ }
+ catch (IllegalArgumentException e)
+ {
+ // expected
+ }
+ finally
+ {
+ if (txn != null)
+ txn.abort();
+ }
+ Collections.rotate(toCompact, 1);
+ }
+ }
+}
diff --git a/test/unit/org/apache/cassandra/db/compaction/CompactionsBytemanTest.java b/test/unit/org/apache/cassandra/db/compaction/CompactionsBytemanTest.java
index d5f2800..95069f1 100644
--- a/test/unit/org/apache/cassandra/db/compaction/CompactionsBytemanTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/CompactionsBytemanTest.java
@@ -18,11 +18,9 @@
package org.apache.cassandra.db.compaction;
-import java.io.IOException;
+import java.util.concurrent.TimeUnit;
import java.util.Collection;
import java.util.Collections;
-import java.util.HashSet;
-import java.util.Set;
import java.util.function.Consumer;
import java.util.stream.Collectors;
@@ -35,10 +33,9 @@
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.io.sstable.Descriptor;
-import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.metrics.CompactionMetrics;
import org.apache.cassandra.utils.FBUtilities;
import org.jboss.byteman.contrib.bmunit.BMRule;
+import org.jboss.byteman.contrib.bmunit.BMRules;
import org.jboss.byteman.contrib.bmunit.BMUnitRunner;
import static org.junit.Assert.assertEquals;
@@ -48,6 +45,74 @@
@RunWith(BMUnitRunner.class)
public class CompactionsBytemanTest extends CQLTester
{
+ /*
+ Return false for the first time hasAvailableDiskSpace is called. i.e first SSTable is too big
+ Create 5 SSTables. After compaction, there should be 2 left - 1 as the 9 SStables which were merged,
+ and the other the SSTable that was 'too large' and failed the hasAvailableDiskSpace check
+ */
+ @Test
+ @BMRules(rules = { @BMRule(name = "One SSTable too big for remaining disk space test",
+ targetClass = "Directories",
+ targetMethod = "hasAvailableDiskSpace",
+ condition = "not flagged(\"done\")",
+ action = "flag(\"done\"); return false;") } )
+ public void testSSTableNotEnoughDiskSpaceForCompactionGetsDropped() throws Throwable
+ {
+ createLowGCGraceTable();
+ final ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
+ for (int i = 0; i < 5; i++)
+ {
+ createPossiblyExpiredSSTable(cfs, false);
+ }
+ assertEquals(5, getCurrentColumnFamilyStore().getLiveSSTables().size());
+ cfs.forceMajorCompaction(false);
+ assertEquals(2, getCurrentColumnFamilyStore().getLiveSSTables().size());
+ dropTable("DROP TABLE %s");
+ }
+
+ /*
+ Always return false for hasAvailableDiskSpace. i.e node has no more space
+ Create 2 expired SSTables and 1 long lived one. After compaction, there should only be 1 left,
+ as the 2 expired ones would have been compacted away.
+ */
+ @Test
+ @BMRules(rules = { @BMRule(name = "No disk space with expired SSTables test",
+ targetClass = "Directories",
+ targetMethod = "hasAvailableDiskSpace",
+ action = "return false;") } )
+ public void testExpiredSSTablesStillGetDroppedWithNoDiskSpace() throws Throwable
+ {
+ createLowGCGraceTable();
+ final ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
+ createPossiblyExpiredSSTable(cfs, true);
+ createPossiblyExpiredSSTable(cfs, true);
+ createPossiblyExpiredSSTable(cfs, false);
+ assertEquals(3, cfs.getLiveSSTables().size());
+ Thread.sleep(TimeUnit.SECONDS.toMillis((long)1.5)); // give some time to expire.
+ cfs.forceMajorCompaction(false);
+ assertEquals(1, cfs.getLiveSSTables().size());
+ dropTable("DROP TABLE %s");
+ }
+
+ /*
+ Always return false for hasAvailableDiskSpace. i.e node has no more space
+ Create 2 SSTables. Compaction will not succeed and will throw Runtime Exception
+ */
+ @Test(expected = RuntimeException.class)
+ @BMRules(rules = { @BMRule(name = "No disk space with expired SSTables test",
+ targetClass = "Directories",
+ targetMethod = "hasAvailableDiskSpace",
+ action = "return false;") } )
+ public void testRuntimeExceptionWhenNoDiskSpaceForCompaction() throws Throwable
+ {
+ createLowGCGraceTable();
+ final ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
+ createPossiblyExpiredSSTable(cfs, false);
+ createPossiblyExpiredSSTable(cfs, false);
+ cfs.forceMajorCompaction(false);
+ dropTable("DROP TABLE %s");
+ }
+
@Test
@BMRule(name = "Delay background compaction task future check",
targetClass = "CompactionManager",
@@ -57,8 +122,8 @@
action = "Thread.sleep(1000)")
public void testCompactingCFCounting() throws Throwable
{
- String table = createTable("CREATE TABLE %s (k INT, c INT, v INT, PRIMARY KEY (k, c))");
- ColumnFamilyStore cfs = Keyspace.open(CQLTester.KEYSPACE).getColumnFamilyStore(table);
+ createTable("CREATE TABLE %s (k INT, c INT, v INT, PRIMARY KEY (k, c))");
+ ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
cfs.enableAutoCompaction();
execute("INSERT INTO %s (k, c, v) VALUES (?, ?, ?)", 0, 1, 1);
@@ -69,6 +134,24 @@
assertEquals(0, CompactionManager.instance.compactingCF.count(cfs));
}
+ private void createPossiblyExpiredSSTable(final ColumnFamilyStore cfs, final boolean expired) throws Throwable
+ {
+ if (expired)
+ {
+ execute("INSERT INTO %s (id, val) values (1, 'expired') USING TTL 1");
+ Thread.sleep(TimeUnit.SECONDS.toMillis((long)1.5));
+ }
+ else
+ {
+ execute("INSERT INTO %s (id, val) values (2, 'immortal')");
+ }
+ cfs.forceBlockingFlush();
+ }
+
+ private void createLowGCGraceTable(){
+ createTable("CREATE TABLE %s (id int PRIMARY KEY, val text) with compaction = {'class':'SizeTieredCompactionStrategy', 'enabled': 'false'} AND gc_grace_seconds=0");
+ }
+
@Test
@BMRule(name = "Stop all compactions",
targetClass = "CompactionTask",
@@ -111,7 +194,7 @@
}
cfs.forceBlockingFlush();
}
- setRepaired(cfs, cfs.getLiveSSTables());
+ cfs.getCompactionStrategyManager().mutateRepaired(cfs.getLiveSSTables(), System.currentTimeMillis(), null, false);
for (int i = 0; i < 5; i++)
{
for (int j = 0; j < 10; j++)
@@ -122,7 +205,7 @@
}
assertTrue(cfs.getTracker().getCompacting().isEmpty());
- assertTrue(CompactionMetrics.getCompactions().stream().noneMatch(h -> h.getCompactionInfo().getCFMetaData().equals(cfs.metadata)));
+ assertTrue(CompactionManager.instance.active.getCompactions().stream().noneMatch(h -> h.getCompactionInfo().getTableMetadata().equals(cfs.metadata)));
try
{
@@ -137,19 +220,7 @@
}
assertTrue(cfs.getTracker().getCompacting().isEmpty());
- assertTrue(CompactionMetrics.getCompactions().stream().noneMatch(h -> h.getCompactionInfo().getCFMetaData().equals(cfs.metadata)));
+ assertTrue(CompactionManager.instance.active.getCompactions().stream().noneMatch(h -> h.getCompactionInfo().getTableMetadata().equals(cfs.metadata)));
}
-
- private void setRepaired(ColumnFamilyStore cfs, Iterable<SSTableReader> sstables) throws IOException
- {
- Set<SSTableReader> changed = new HashSet<>();
- for (SSTableReader sstable: sstables)
- {
- sstable.descriptor.getMetadataSerializer().mutateRepairedAt(sstable.descriptor, System.currentTimeMillis());
- sstable.reloadSSTableMetadata();
- changed.add(sstable);
- }
- cfs.getTracker().notifySSTableRepairedStatusChanged(changed);
- }
}
diff --git a/test/unit/org/apache/cassandra/db/compaction/CompactionsCQLTest.java b/test/unit/org/apache/cassandra/db/compaction/CompactionsCQLTest.java
index 7873ac9..14b3d6d 100644
--- a/test/unit/org/apache/cassandra/db/compaction/CompactionsCQLTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/CompactionsCQLTest.java
@@ -17,16 +17,42 @@
*/
package org.apache.cassandra.db.compaction;
+import java.nio.ByteBuffer;
+import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Random;
+import org.apache.commons.lang.StringUtils;
+import org.junit.After;
+import org.junit.Before;
import org.junit.Test;
import org.apache.cassandra.cql3.CQLTester;
import org.apache.cassandra.cql3.UntypedResultSet;
+import org.apache.cassandra.config.Config;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DeletionTime;
+import org.apache.cassandra.db.Mutation;
+import org.apache.cassandra.db.RangeTombstone;
+import org.apache.cassandra.db.RowUpdateBuilder;
+import org.apache.cassandra.db.Slice;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.db.partitions.PartitionUpdate;
+import org.apache.cassandra.db.rows.Cell;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.db.rows.Unfiltered;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.io.sstable.CorruptSSTableException;
+import org.apache.cassandra.io.sstable.ISSTableScanner;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.serializers.MarshalException;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -35,6 +61,21 @@
public static final int SLEEP_TIME = 5000;
+ private Config.CorruptedTombstoneStrategy strategy;
+
+ @Before
+ public void before()
+ {
+ strategy = DatabaseDescriptor.getCorruptedTombstoneStrategy();
+ }
+
+ @After
+ public void after()
+ {
+ DatabaseDescriptor.setCorruptedTombstoneStrategy(DatabaseDescriptor.getCorruptedTombstoneStrategy());
+ }
+
+
@Test
public void testTriggerMinorCompactionSTCS() throws Throwable
{
@@ -226,6 +267,338 @@
getCurrentColumnFamilyStore().setCompactionParameters(localOptions);
}
+ @Test
+ public void testPerCFSNeverPurgeTombstonesCell() throws Throwable
+ {
+ testPerCFSNeverPurgeTombstonesHelper(true);
+ }
+
+ @Test
+ public void testPerCFSNeverPurgeTombstones() throws Throwable
+ {
+ testPerCFSNeverPurgeTombstonesHelper(false);
+ }
+
+ @Test
+ public void testCompactionInvalidRTs() throws Throwable
+ {
+ // set the corruptedTombstoneStrategy to exception since these tests require it - if someone changed the default
+ // in test/conf/cassandra.yaml they would start failing
+ DatabaseDescriptor.setCorruptedTombstoneStrategy(Config.CorruptedTombstoneStrategy.exception);
+ prepare();
+ // write a range tombstone with negative local deletion time (LDTs are not set by user and should not be negative):
+ RangeTombstone rt = new RangeTombstone(Slice.ALL, new DeletionTime(System.currentTimeMillis(), -1));
+ RowUpdateBuilder rub = new RowUpdateBuilder(getCurrentColumnFamilyStore().metadata(), System.currentTimeMillis() * 1000, 22).clustering(33).addRangeTombstone(rt);
+ rub.build().apply();
+ getCurrentColumnFamilyStore().forceBlockingFlush();
+ compactAndValidate();
+ readAndValidate(true);
+ readAndValidate(false);
+ }
+
+ @Test
+ public void testCompactionInvalidTombstone() throws Throwable
+ {
+ DatabaseDescriptor.setCorruptedTombstoneStrategy(Config.CorruptedTombstoneStrategy.exception);
+ prepare();
+ // write a standard tombstone with negative local deletion time (LDTs are not set by user and should not be negative):
+ RowUpdateBuilder rub = new RowUpdateBuilder(getCurrentColumnFamilyStore().metadata(), -1, System.currentTimeMillis() * 1000, 22).clustering(33).delete("b");
+ rub.build().apply();
+ getCurrentColumnFamilyStore().forceBlockingFlush();
+ compactAndValidate();
+ readAndValidate(true);
+ readAndValidate(false);
+ }
+
+ @Test
+ public void testCompactionInvalidPartitionDeletion() throws Throwable
+ {
+ DatabaseDescriptor.setCorruptedTombstoneStrategy(Config.CorruptedTombstoneStrategy.exception);
+ prepare();
+ // write a partition deletion with negative local deletion time (LDTs are not set by user and should not be negative)::
+ PartitionUpdate pu = PartitionUpdate.simpleBuilder(getCurrentColumnFamilyStore().metadata(), 22).nowInSec(-1).delete().build();
+ new Mutation(pu).apply();
+ getCurrentColumnFamilyStore().forceBlockingFlush();
+ compactAndValidate();
+ readAndValidate(true);
+ readAndValidate(false);
+ }
+
+ @Test
+ public void testCompactionInvalidRowDeletion() throws Throwable
+ {
+ DatabaseDescriptor.setCorruptedTombstoneStrategy(Config.CorruptedTombstoneStrategy.exception);
+ prepare();
+ // write a row deletion with negative local deletion time (LDTs are not set by user and should not be negative):
+ RowUpdateBuilder.deleteRowAt(getCurrentColumnFamilyStore().metadata(), System.currentTimeMillis() * 1000, -1, 22, 33).apply();
+ getCurrentColumnFamilyStore().forceBlockingFlush();
+ compactAndValidate();
+ readAndValidate(true);
+ readAndValidate(false);
+ }
+
+ private void prepare() throws Throwable
+ {
+ createTable("CREATE TABLE %s (id int, id2 int, b text, primary key (id, id2))");
+ for (int i = 0; i < 2; i++)
+ execute("INSERT INTO %s (id, id2, b) VALUES (?, ?, ?)", i, i, String.valueOf(i));
+ }
+
+ @Test
+ public void testIndexedReaderRowDeletion() throws Throwable
+ {
+ // write enough data to make sure we use an IndexedReader when doing a read, and make sure it fails when reading a corrupt row deletion
+ DatabaseDescriptor.setCorruptedTombstoneStrategy(Config.CorruptedTombstoneStrategy.exception);
+ int maxSizePre = DatabaseDescriptor.getColumnIndexSizeInKB();
+ DatabaseDescriptor.setColumnIndexSize(1024);
+ prepareWide();
+ RowUpdateBuilder.deleteRowAt(getCurrentColumnFamilyStore().metadata(), System.currentTimeMillis() * 1000, -1, 22, 33).apply();
+ getCurrentColumnFamilyStore().forceBlockingFlush();
+ readAndValidate(true);
+ readAndValidate(false);
+ DatabaseDescriptor.setColumnIndexSize(maxSizePre);
+ }
+
+ @Test
+ public void testIndexedReaderTombstone() throws Throwable
+ {
+ // write enough data to make sure we use an IndexedReader when doing a read, and make sure it fails when reading a corrupt standard tombstone
+ DatabaseDescriptor.setCorruptedTombstoneStrategy(Config.CorruptedTombstoneStrategy.exception);
+ int maxSizePre = DatabaseDescriptor.getColumnIndexSizeInKB();
+ DatabaseDescriptor.setColumnIndexSize(1024);
+ prepareWide();
+ RowUpdateBuilder rub = new RowUpdateBuilder(getCurrentColumnFamilyStore().metadata(), -1, System.currentTimeMillis() * 1000, 22).clustering(33).delete("b");
+ rub.build().apply();
+ getCurrentColumnFamilyStore().forceBlockingFlush();
+ readAndValidate(true);
+ readAndValidate(false);
+ DatabaseDescriptor.setColumnIndexSize(maxSizePre);
+ }
+
+ @Test
+ public void testIndexedReaderRT() throws Throwable
+ {
+ // write enough data to make sure we use an IndexedReader when doing a read, and make sure it fails when reading a corrupt range tombstone
+ DatabaseDescriptor.setCorruptedTombstoneStrategy(Config.CorruptedTombstoneStrategy.exception);
+ final int maxSizePreKB = DatabaseDescriptor.getColumnIndexSizeInKB();
+ DatabaseDescriptor.setColumnIndexSize(1024);
+ prepareWide();
+ RangeTombstone rt = new RangeTombstone(Slice.ALL, new DeletionTime(System.currentTimeMillis(), -1));
+ RowUpdateBuilder rub = new RowUpdateBuilder(getCurrentColumnFamilyStore().metadata(), System.currentTimeMillis() * 1000, 22).clustering(33).addRangeTombstone(rt);
+ rub.build().apply();
+ getCurrentColumnFamilyStore().forceBlockingFlush();
+ readAndValidate(true);
+ readAndValidate(false);
+ DatabaseDescriptor.setColumnIndexSize(maxSizePreKB);
+ }
+
+
+ @Test
+ public void testLCSThresholdParams() throws Throwable
+ {
+ createTable("create table %s (id int, id2 int, t blob, primary key (id, id2)) with compaction = {'class':'LeveledCompactionStrategy', 'sstable_size_in_mb':'1', 'max_threshold':'60'}");
+ ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
+ cfs.disableAutoCompaction();
+ byte [] b = new byte[100 * 1024];
+ new Random().nextBytes(b);
+ ByteBuffer value = ByteBuffer.wrap(b);
+ for (int i = 0; i < 50; i++)
+ {
+ for (int j = 0; j < 10; j++)
+ {
+ execute("insert into %s (id, id2, t) values (?, ?, ?)", i, j, value);
+ }
+ cfs.forceBlockingFlush();
+ }
+ assertEquals(50, cfs.getLiveSSTables().size());
+ LeveledCompactionStrategy lcs = (LeveledCompactionStrategy) cfs.getCompactionStrategyManager().getUnrepairedUnsafe().first();
+ AbstractCompactionTask act = lcs.getNextBackgroundTask(0);
+ // we should be compacting all 50 sstables:
+ assertEquals(50, act.transaction.originals().size());
+ act.execute(ActiveCompactionsTracker.NOOP);
+ }
+
+ @Test
+ public void testSTCSinL0() throws Throwable
+ {
+ createTable("create table %s (id int, id2 int, t blob, primary key (id, id2)) with compaction = {'class':'LeveledCompactionStrategy', 'sstable_size_in_mb':'1', 'max_threshold':'60'}");
+ ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
+ cfs.disableAutoCompaction();
+ execute("insert into %s (id, id2, t) values (?, ?, ?)", 1,1,"L1");
+ cfs.forceBlockingFlush();
+ cfs.forceMajorCompaction();
+ SSTableReader l1sstable = cfs.getLiveSSTables().iterator().next();
+ assertEquals(1, l1sstable.getSSTableLevel());
+ // now we have a single L1 sstable, create many L0 ones:
+ byte [] b = new byte[100 * 1024];
+ new Random().nextBytes(b);
+ ByteBuffer value = ByteBuffer.wrap(b);
+ for (int i = 0; i < 50; i++)
+ {
+ for (int j = 0; j < 10; j++)
+ {
+ execute("insert into %s (id, id2, t) values (?, ?, ?)", i, j, value);
+ }
+ cfs.forceBlockingFlush();
+ }
+ assertEquals(51, cfs.getLiveSSTables().size());
+
+ // mark the L1 sstable as compacting to make sure we trigger STCS in L0:
+ LifecycleTransaction txn = cfs.getTracker().tryModify(l1sstable, OperationType.COMPACTION);
+ LeveledCompactionStrategy lcs = (LeveledCompactionStrategy) cfs.getCompactionStrategyManager().getUnrepairedUnsafe().first();
+ AbstractCompactionTask act = lcs.getNextBackgroundTask(0);
+ // note that max_threshold is 60 (more than the amount of L0 sstables), but MAX_COMPACTING_L0 is 32, which means we will trigger STCS with at most max_threshold sstables
+ assertEquals(50, act.transaction.originals().size());
+ assertEquals(0, ((LeveledCompactionTask)act).getLevel());
+ assertTrue(act.transaction.originals().stream().allMatch(s -> s.getSSTableLevel() == 0));
+ txn.abort(); // unmark the l1 sstable compacting
+ act.execute(ActiveCompactionsTracker.NOOP);
+ }
+
+ private void prepareWide() throws Throwable
+ {
+ createTable("CREATE TABLE %s (id int, id2 int, b text, primary key (id, id2))");
+ for (int i = 0; i < 100; i++)
+ execute("INSERT INTO %s (id, id2, b) VALUES (?, ?, ?)", 22, i, StringUtils.repeat("ABCDEFG", 10));
+ }
+
+ private void compactAndValidate()
+ {
+ boolean gotException = false;
+ try
+ {
+ getCurrentColumnFamilyStore().forceMajorCompaction();
+ }
+ catch(Throwable t)
+ {
+ gotException = true;
+ Throwable cause = t;
+ while (cause != null && !(cause instanceof MarshalException))
+ cause = cause.getCause();
+ assertNotNull(cause);
+ MarshalException me = (MarshalException) cause;
+ assertTrue(me.getMessage().contains(getCurrentColumnFamilyStore().metadata.keyspace+"."+getCurrentColumnFamilyStore().metadata.name));
+ assertTrue(me.getMessage().contains("Key 22"));
+ }
+ assertTrue(gotException);
+ assertSuspectAndReset(getCurrentColumnFamilyStore().getLiveSSTables());
+ }
+
+ private void readAndValidate(boolean asc) throws Throwable
+ {
+ execute("select * from %s where id = 0 order by id2 "+(asc ? "ASC" : "DESC"));
+
+ boolean gotException = false;
+ try
+ {
+ for (UntypedResultSet.Row r : execute("select * from %s")) {}
+ }
+ catch (Throwable t)
+ {
+ assertTrue(t instanceof CorruptSSTableException);
+ gotException = true;
+ Throwable cause = t;
+ while (cause != null && !(cause instanceof MarshalException))
+ cause = cause.getCause();
+ assertNotNull(cause);
+ MarshalException me = (MarshalException) cause;
+ assertTrue(me.getMessage().contains("Key 22"));
+ }
+ assertSuspectAndReset(getCurrentColumnFamilyStore().getLiveSSTables());
+ assertTrue(gotException);
+ gotException = false;
+ try
+ {
+ execute("select * from %s where id = 22 order by id2 "+(asc ? "ASC" : "DESC"));
+ }
+ catch (Throwable t)
+ {
+ assertTrue(t instanceof CorruptSSTableException);
+ gotException = true;
+ Throwable cause = t;
+ while (cause != null && !(cause instanceof MarshalException))
+ cause = cause.getCause();
+ assertNotNull(cause);
+ MarshalException me = (MarshalException) cause;
+ assertTrue(me.getMessage().contains("Key 22"));
+ }
+ assertTrue(gotException);
+ assertSuspectAndReset(getCurrentColumnFamilyStore().getLiveSSTables());
+ }
+
+ public void testPerCFSNeverPurgeTombstonesHelper(boolean deletedCell) throws Throwable
+ {
+ createTable("CREATE TABLE %s (id int primary key, b text) with gc_grace_seconds = 0");
+ for (int i = 0; i < 100; i++)
+ {
+ execute("INSERT INTO %s (id, b) VALUES (?, ?)", i, String.valueOf(i));
+ }
+ getCurrentColumnFamilyStore().forceBlockingFlush();
+
+ assertTombstones(getCurrentColumnFamilyStore().getLiveSSTables().iterator().next(), false);
+ if (deletedCell)
+ execute("UPDATE %s SET b=null WHERE id = ?", 50);
+ else
+ execute("DELETE FROM %s WHERE id = ?", 50);
+ getCurrentColumnFamilyStore().setNeverPurgeTombstones(false);
+ getCurrentColumnFamilyStore().forceBlockingFlush();
+ Thread.sleep(2000); // wait for gcgs to pass
+ getCurrentColumnFamilyStore().forceMajorCompaction();
+ assertTombstones(getCurrentColumnFamilyStore().getLiveSSTables().iterator().next(), false);
+ if (deletedCell)
+ execute("UPDATE %s SET b=null WHERE id = ?", 44);
+ else
+ execute("DELETE FROM %s WHERE id = ?", 44);
+ getCurrentColumnFamilyStore().setNeverPurgeTombstones(true);
+ getCurrentColumnFamilyStore().forceBlockingFlush();
+ Thread.sleep(1100);
+ getCurrentColumnFamilyStore().forceMajorCompaction();
+ assertTombstones(getCurrentColumnFamilyStore().getLiveSSTables().iterator().next(), true);
+ // disable it again and make sure the tombstone is gone:
+ getCurrentColumnFamilyStore().setNeverPurgeTombstones(false);
+ getCurrentColumnFamilyStore().forceMajorCompaction();
+ assertTombstones(getCurrentColumnFamilyStore().getLiveSSTables().iterator().next(), false);
+ getCurrentColumnFamilyStore().truncateBlocking();
+ }
+
+ private void assertSuspectAndReset(Collection<SSTableReader> sstables)
+ {
+ assertFalse(sstables.isEmpty());
+ for (SSTableReader sstable : sstables)
+ {
+ assertTrue(sstable.isMarkedSuspect());
+ sstable.unmarkSuspect();
+ }
+ }
+
+ private void assertTombstones(SSTableReader sstable, boolean expectTS)
+ {
+ boolean foundTombstone = false;
+ try(ISSTableScanner scanner = sstable.getScanner())
+ {
+ while (scanner.hasNext())
+ {
+ try (UnfilteredRowIterator iter = scanner.next())
+ {
+ if (!iter.partitionLevelDeletion().isLive())
+ foundTombstone = true;
+ while (iter.hasNext())
+ {
+ Unfiltered unfiltered = iter.next();
+ assertTrue(unfiltered instanceof Row);
+ for (Cell c : ((Row)unfiltered).cells())
+ {
+ if (c.isTombstone())
+ foundTombstone = true;
+ }
+
+ }
+ }
+ }
+ }
+ assertEquals(expectTS, foundTombstone);
+ }
+
public boolean verifyStrategies(CompactionStrategyManager manager, Class<? extends AbstractCompactionStrategy> expected)
{
boolean found = false;
diff --git a/test/unit/org/apache/cassandra/db/compaction/CompactionsPurgeTest.java b/test/unit/org/apache/cassandra/db/compaction/CompactionsPurgeTest.java
index f5b1641..a0d52aa 100644
--- a/test/unit/org/apache/cassandra/db/compaction/CompactionsPurgeTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/CompactionsPurgeTest.java
@@ -19,17 +19,17 @@
package org.apache.cassandra.db.compaction;
import java.util.Collection;
-import java.util.List;
import java.util.concurrent.ExecutionException;
+import com.google.common.collect.Iterables;
import org.junit.BeforeClass;
import org.junit.Test;
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.Util;
-import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.cql3.QueryProcessor;
import org.apache.cassandra.cql3.UntypedResultSet;
+import org.apache.cassandra.cql3.statements.schema.CreateTableStatement;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.partitions.ImmutableBTreePartition;
import org.apache.cassandra.db.rows.Row;
@@ -59,23 +59,27 @@
public static void defineSchema() throws ConfigurationException
{
SchemaLoader.prepareServer();
+
SchemaLoader.createKeyspace(KEYSPACE1,
KeyspaceParams.simple(1),
SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD1),
SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD2));
+
SchemaLoader.createKeyspace(KEYSPACE2,
KeyspaceParams.simple(1),
SchemaLoader.standardCFMD(KEYSPACE2, CF_STANDARD1));
+
SchemaLoader.createKeyspace(KEYSPACE_CACHED,
KeyspaceParams.simple(1),
SchemaLoader.standardCFMD(KEYSPACE_CACHED, CF_CACHED).caching(CachingParams.CACHE_EVERYTHING));
+
SchemaLoader.createKeyspace(KEYSPACE_CQL,
KeyspaceParams.simple(1),
- CFMetaData.compile("CREATE TABLE " + CF_CQL + " ("
- + "k int PRIMARY KEY,"
- + "v1 text,"
- + "v2 int"
- + ")", KEYSPACE_CQL));
+ CreateTableStatement.parse("CREATE TABLE " + CF_CQL + " ("
+ + "k int PRIMARY KEY,"
+ + "v1 text,"
+ + "v2 int"
+ + ")", KEYSPACE_CQL));
}
@Test
@@ -92,7 +96,7 @@
// inserts
for (int i = 0; i < 10; i++)
{
- RowUpdateBuilder builder = new RowUpdateBuilder(cfs.metadata, 0, key);
+ RowUpdateBuilder builder = new RowUpdateBuilder(cfs.metadata(), 0, key);
builder.clustering(String.valueOf(i))
.add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER)
.build().applyUnsafe();
@@ -103,12 +107,12 @@
// deletes
for (int i = 0; i < 10; i++)
{
- RowUpdateBuilder.deleteRow(cfs.metadata, 1, key, String.valueOf(i)).applyUnsafe();
+ RowUpdateBuilder.deleteRow(cfs.metadata(), 1, key, String.valueOf(i)).applyUnsafe();
}
cfs.forceBlockingFlush();
// resurrect one column
- RowUpdateBuilder builder = new RowUpdateBuilder(cfs.metadata, 2, key);
+ RowUpdateBuilder builder = new RowUpdateBuilder(cfs.metadata(), 2, key);
builder.clustering(String.valueOf(5))
.add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER)
.build().applyUnsafe();
@@ -137,7 +141,7 @@
// inserts
for (int i = 0; i < 10; i++)
{
- RowUpdateBuilder builder = new RowUpdateBuilder(cfs.metadata, 0, key);
+ RowUpdateBuilder builder = new RowUpdateBuilder(cfs.metadata(), 0, key);
builder.clustering(String.valueOf(i))
.add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER)
.build().applyUnsafe();
@@ -147,7 +151,7 @@
// deletes
for (int i = 0; i < 10; i++)
{
- RowUpdateBuilder.deleteRow(cfs.metadata, Long.MAX_VALUE, key, String.valueOf(i)).applyUnsafe();
+ RowUpdateBuilder.deleteRow(cfs.metadata(), Long.MAX_VALUE, key, String.valueOf(i)).applyUnsafe();
}
cfs.forceBlockingFlush();
@@ -155,7 +159,7 @@
FBUtilities.waitOnFutures(CompactionManager.instance.submitMaximal(cfs, Integer.MAX_VALUE, false));
// resurrect one column
- RowUpdateBuilder builder = new RowUpdateBuilder(cfs.metadata, 2, key);
+ RowUpdateBuilder builder = new RowUpdateBuilder(cfs.metadata(), 2, key);
builder.clustering(String.valueOf(5))
.add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER)
.build().applyUnsafe();
@@ -182,15 +186,16 @@
// inserts
for (int i = 0; i < 10; i++)
{
- RowUpdateBuilder builder = new RowUpdateBuilder(cfs.metadata, 0, key);
+ RowUpdateBuilder builder = new RowUpdateBuilder(cfs.metadata(), 0, key);
builder.clustering(String.valueOf(i))
.add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER)
.build().applyUnsafe();
}
cfs.forceBlockingFlush();
- new Mutation(KEYSPACE1, dk(key))
- .add(PartitionUpdate.fullPartitionDelete(cfs.metadata, dk(key), Long.MAX_VALUE, FBUtilities.nowInSeconds()))
+ new Mutation.PartitionUpdateCollector(KEYSPACE1, dk(key))
+ .add(PartitionUpdate.fullPartitionDelete(cfs.metadata(), dk(key), Long.MAX_VALUE, FBUtilities.nowInSeconds()))
+ .build()
.applyUnsafe();
cfs.forceBlockingFlush();
@@ -198,7 +203,7 @@
FBUtilities.waitOnFutures(CompactionManager.instance.submitMaximal(cfs, Integer.MAX_VALUE, false));
// resurrect one column
- RowUpdateBuilder builder = new RowUpdateBuilder(cfs.metadata, 2, key);
+ RowUpdateBuilder builder = new RowUpdateBuilder(cfs.metadata(), 2, key);
builder.clustering(String.valueOf(5))
.add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER)
.build().applyUnsafe();
@@ -225,14 +230,14 @@
// inserts
for (int i = 0; i < 10; i++)
{
- RowUpdateBuilder builder = new RowUpdateBuilder(cfs.metadata, 0, key);
+ RowUpdateBuilder builder = new RowUpdateBuilder(cfs.metadata(), 0, key);
builder.clustering(String.valueOf(i))
.add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER)
.build().applyUnsafe();
}
cfs.forceBlockingFlush();
- new RowUpdateBuilder(cfs.metadata, Long.MAX_VALUE, dk(key))
+ new RowUpdateBuilder(cfs.metadata(), Long.MAX_VALUE, dk(key))
.addRangeTombstone(String.valueOf(0), String.valueOf(9)).build().applyUnsafe();
cfs.forceBlockingFlush();
@@ -240,7 +245,7 @@
FBUtilities.waitOnFutures(CompactionManager.instance.submitMaximal(cfs, Integer.MAX_VALUE, false));
// resurrect one column
- RowUpdateBuilder builder = new RowUpdateBuilder(cfs.metadata, 2, key);
+ RowUpdateBuilder builder = new RowUpdateBuilder(cfs.metadata(), 2, key);
builder.clustering(String.valueOf(5))
.add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER)
.build().applyUnsafe();
@@ -268,7 +273,7 @@
// inserts
for (int i = 0; i < 10; i++)
{
- RowUpdateBuilder builder = new RowUpdateBuilder(cfs.metadata, 0, key);
+ RowUpdateBuilder builder = new RowUpdateBuilder(cfs.metadata(), 0, key);
builder.clustering(String.valueOf(i))
.add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER)
.build().applyUnsafe();
@@ -278,7 +283,7 @@
// deletes
for (int i = 0; i < 10; i++)
{
- RowUpdateBuilder.deleteRow(cfs.metadata, 1, key, String.valueOf(i)).applyUnsafe();
+ RowUpdateBuilder.deleteRow(cfs.metadata(), 1, key, String.valueOf(i)).applyUnsafe();
}
cfs.forceBlockingFlush();
@@ -292,15 +297,16 @@
cfs.forceBlockingFlush();
Collection<SSTableReader> sstablesIncomplete = cfs.getLiveSSTables();
- RowUpdateBuilder builder = new RowUpdateBuilder(cfs.metadata, 2, "key1");
+ RowUpdateBuilder builder = new RowUpdateBuilder(cfs.metadata(), 2, "key1");
builder.clustering(String.valueOf(5))
.add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER)
.build().applyUnsafe();
cfs.forceBlockingFlush();
- List<AbstractCompactionTask> tasks = cfs.getCompactionStrategyManager().getUserDefinedTasks(sstablesIncomplete, Integer.MAX_VALUE);
- assertEquals(1, tasks.size());
- tasks.get(0).execute(null);
+ try (CompactionTasks tasks = cfs.getCompactionStrategyManager().getUserDefinedTasks(sstablesIncomplete, Integer.MAX_VALUE))
+ {
+ Iterables.getOnlyElement(tasks).execute(ActiveCompactionsTracker.NOOP);
+ }
// verify that minor compaction does GC when key is provably not
// present in a non-compacted sstable
@@ -323,35 +329,36 @@
Keyspace keyspace = Keyspace.open(KEYSPACE2);
String cfName = "Standard1";
ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfName);
- final boolean enforceStrictLiveness = cfs.metadata.enforceStrictLiveness();
+ final boolean enforceStrictLiveness = cfs.metadata().enforceStrictLiveness();
String key3 = "key3";
// inserts
- new RowUpdateBuilder(cfs.metadata, 8, key3)
+ new RowUpdateBuilder(cfs.metadata(), 8, key3)
.clustering("c1")
.add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER)
.build().applyUnsafe();
- new RowUpdateBuilder(cfs.metadata, 8, key3)
+ new RowUpdateBuilder(cfs.metadata(), 8, key3)
.clustering("c2")
.add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER)
.build().applyUnsafe();
cfs.forceBlockingFlush();
// delete c1
- RowUpdateBuilder.deleteRow(cfs.metadata, 10, key3, "c1").applyUnsafe();
+ RowUpdateBuilder.deleteRow(cfs.metadata(), 10, key3, "c1").applyUnsafe();
cfs.forceBlockingFlush();
Collection<SSTableReader> sstablesIncomplete = cfs.getLiveSSTables();
// delete c2 so we have new delete in a diffrent SSTable
- RowUpdateBuilder.deleteRow(cfs.metadata, 9, key3, "c2").applyUnsafe();
+ RowUpdateBuilder.deleteRow(cfs.metadata(), 9, key3, "c2").applyUnsafe();
cfs.forceBlockingFlush();
// compact the sstables with the c1/c2 data and the c1 tombstone
- List<AbstractCompactionTask> tasks = cfs.getCompactionStrategyManager().getUserDefinedTasks(sstablesIncomplete, Integer.MAX_VALUE);
- assertEquals(1, tasks.size());
- tasks.get(0).execute(null);
+ try (CompactionTasks tasks = cfs.getCompactionStrategyManager().getUserDefinedTasks(sstablesIncomplete, Integer.MAX_VALUE))
+ {
+ Iterables.getOnlyElement(tasks).execute(ActiveCompactionsTracker.NOOP);
+ }
// We should have both the c1 and c2 tombstones still. Since the min timestamp in the c2 tombstone
// sstable is older than the c1 tombstone, it is invalid to throw out the c1 tombstone.
@@ -375,7 +382,7 @@
// inserts
for (int i = 0; i < 5; i++)
{
- RowUpdateBuilder builder = new RowUpdateBuilder(cfs.metadata, 0, key);
+ RowUpdateBuilder builder = new RowUpdateBuilder(cfs.metadata(), 0, key);
builder.clustering(String.valueOf(i))
.add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER)
.build().applyUnsafe();
@@ -384,7 +391,7 @@
// deletes
for (int i = 0; i < 5; i++)
{
- RowUpdateBuilder.deleteRow(cfs.metadata, 1, key, String.valueOf(i)).applyUnsafe();
+ RowUpdateBuilder.deleteRow(cfs.metadata(), 1, key, String.valueOf(i)).applyUnsafe();
}
cfs.forceBlockingFlush();
assertEquals(String.valueOf(cfs.getLiveSSTables()), 1, cfs.getLiveSSTables().size()); // inserts & deletes were in the same memtable -> only deletes in sstable
@@ -412,20 +419,20 @@
// inserts
for (int i = 0; i < 10; i++)
{
- RowUpdateBuilder builder = new RowUpdateBuilder(cfs.metadata, 0, key);
+ RowUpdateBuilder builder = new RowUpdateBuilder(cfs.metadata(), 0, key);
builder.clustering(String.valueOf(i))
.add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER)
.build().applyUnsafe();
}
// deletes partition
- Mutation rm = new Mutation(KEYSPACE_CACHED, dk(key));
- rm.add(PartitionUpdate.fullPartitionDelete(cfs.metadata, dk(key), 1, FBUtilities.nowInSeconds()));
- rm.applyUnsafe();
+ Mutation.PartitionUpdateCollector rm = new Mutation.PartitionUpdateCollector(KEYSPACE_CACHED, dk(key));
+ rm.add(PartitionUpdate.fullPartitionDelete(cfs.metadata(), dk(key), 1, FBUtilities.nowInSeconds()));
+ rm.build().applyUnsafe();
// Adds another unrelated partition so that the sstable is not considered fully expired. We do not
// invalidate the row cache in that latter case.
- new RowUpdateBuilder(cfs.metadata, 0, "key4").clustering("c").add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER).build().applyUnsafe();
+ new RowUpdateBuilder(cfs.metadata(), 0, "key4").clustering("c").add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER).build().applyUnsafe();
// move the key up in row cache (it should not be empty since we have the partition deletion info)
assertFalse(Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, key).build()).isEmpty());
@@ -452,16 +459,16 @@
// inserts
for (int i = 0; i < 10; i++)
{
- RowUpdateBuilder builder = new RowUpdateBuilder(cfs.metadata, i, key);
+ RowUpdateBuilder builder = new RowUpdateBuilder(cfs.metadata(), i, key);
builder.clustering(String.valueOf(i))
.add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER)
.build().applyUnsafe();
}
// deletes partition with timestamp such that not all columns are deleted
- Mutation rm = new Mutation(KEYSPACE1, dk(key));
- rm.add(PartitionUpdate.fullPartitionDelete(cfs.metadata, dk(key), 4, FBUtilities.nowInSeconds()));
- rm.applyUnsafe();
+ Mutation.PartitionUpdateCollector rm = new Mutation.PartitionUpdateCollector(KEYSPACE1, dk(key));
+ rm.add(PartitionUpdate.fullPartitionDelete(cfs.metadata(), dk(key), 4, FBUtilities.nowInSeconds()));
+ rm.build().applyUnsafe();
ImmutableBTreePartition partition = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, key).build());
assertFalse(partition.partitionLevelDeletion().isLive());
@@ -474,7 +481,7 @@
// re-inserts with timestamp lower than delete
for (int i = 0; i < 5; i++)
{
- RowUpdateBuilder builder = new RowUpdateBuilder(cfs.metadata, i, key);
+ RowUpdateBuilder builder = new RowUpdateBuilder(cfs.metadata(), i, key);
builder.clustering(String.valueOf(i))
.add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER)
.build().applyUnsafe();
diff --git a/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java b/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java
index c1bddd1..941ef13 100644
--- a/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java
@@ -20,7 +20,11 @@
import java.io.File;
import java.nio.ByteBuffer;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
import java.util.concurrent.TimeUnit;
import org.junit.BeforeClass;
@@ -31,37 +35,54 @@
import org.apache.cassandra.OrderedJUnit4ClassRunner;
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.Util;
-import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.cql3.ColumnIdentifier;
-import org.apache.cassandra.db.*;
-import org.apache.cassandra.db.columniterator.SSTableIterator;
+import org.apache.cassandra.db.Clustering;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.DeletionTime;
+import org.apache.cassandra.db.Directories;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.db.RangeTombstone;
+import org.apache.cassandra.db.ReadExecutionController;
+import org.apache.cassandra.db.RowUpdateBuilder;
+import org.apache.cassandra.db.SinglePartitionReadCommand;
+import org.apache.cassandra.db.Slice;
+import org.apache.cassandra.db.Slices;
import org.apache.cassandra.db.filter.ClusteringIndexSliceFilter;
import org.apache.cassandra.db.filter.ColumnFilter;
import org.apache.cassandra.db.filter.DataLimits;
import org.apache.cassandra.db.filter.RowFilter;
+import org.apache.cassandra.db.marshal.AsciiType;
import org.apache.cassandra.db.partitions.FilteredPartition;
import org.apache.cassandra.db.partitions.ImmutableBTreePartition;
import org.apache.cassandra.db.partitions.PartitionIterator;
import org.apache.cassandra.db.rows.Cell;
-import org.apache.cassandra.db.rows.ColumnData;
import org.apache.cassandra.db.rows.Row;
import org.apache.cassandra.db.rows.RowIterator;
import org.apache.cassandra.db.rows.Unfiltered;
import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.dht.Bounds;
+import org.apache.cassandra.dht.ByteOrderedPartitioner;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.dht.*;
import org.apache.cassandra.io.sstable.Component;
import org.apache.cassandra.io.sstable.ISSTableScanner;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
import org.apache.cassandra.schema.CompactionParams;
import org.apache.cassandra.schema.KeyspaceParams;
-import org.apache.cassandra.schema.TableParams;
-import org.apache.cassandra.service.MigrationManager;
+import org.apache.cassandra.schema.MigrationManager;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FBUtilities;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertTrue;
@RunWith(OrderedJUnit4ClassRunner.class)
public class CompactionsTest
@@ -90,18 +111,37 @@
SchemaLoader.createKeyspace(KEYSPACE1,
KeyspaceParams.simple(1),
SchemaLoader.denseCFMD(KEYSPACE1, CF_DENSE1)
- .compaction(CompactionParams.scts(compactionOptions)),
+ .compaction(CompactionParams.stcs(compactionOptions)),
SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD1)
- .compaction(CompactionParams.scts(compactionOptions)),
+ .compaction(CompactionParams.stcs(compactionOptions)),
SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD2),
SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD3),
SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD4),
- SchemaLoader.standardCFMD(KEYSPACE1, CF_SUPER1),
- SchemaLoader.standardCFMD(KEYSPACE1, CF_SUPER5),
- SchemaLoader.standardCFMD(KEYSPACE1, CF_SUPERGC)
+ SchemaLoader.superCFMD(KEYSPACE1, CF_SUPER1, AsciiType.instance),
+ SchemaLoader.superCFMD(KEYSPACE1, CF_SUPER5, AsciiType.instance),
+ SchemaLoader.superCFMD(KEYSPACE1, CF_SUPERGC, AsciiType.instance)
.gcGraceSeconds(0));
}
+ public static long populate(String ks, String cf, int startRowKey, int endRowKey, int ttl)
+ {
+ long timestamp = System.currentTimeMillis();
+ TableMetadata cfm = Keyspace.open(ks).getColumnFamilyStore(cf).metadata();
+ for (int i = startRowKey; i <= endRowKey; i++)
+ {
+ DecoratedKey key = Util.dk(Integer.toString(i));
+ for (int j = 0; j < 10; j++)
+ {
+ new RowUpdateBuilder(cfm, timestamp, j > 0 ? ttl : 0, key.getKey())
+ .clustering(Integer.toString(j))
+ .add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER)
+ .build()
+ .applyUnsafe();
+ }
+ }
+ return timestamp;
+ }
+
// Test to see if sstable has enough expired columns, it is compacted itself.
@Test
public void testSingleSSTableCompaction() throws Exception
@@ -109,7 +149,7 @@
Keyspace keyspace = Keyspace.open(KEYSPACE1);
ColumnFamilyStore store = keyspace.getColumnFamilyStore(CF_DENSE1);
store.clearUnsafe();
- store.metadata.gcGraceSeconds(1);
+ MigrationManager.announceTableUpdate(store.metadata().unbuild().gcGraceSeconds(1).build(), true);
// disable compaction while flushing
store.disableAutoCompaction();
@@ -140,31 +180,12 @@
assertMaxTimestamp(store, timestamp);
}
- public static long populate(String ks, String cf, int startRowKey, int endRowKey, int ttl)
- {
- long timestamp = System.currentTimeMillis();
- CFMetaData cfm = Keyspace.open(ks).getColumnFamilyStore(cf).metadata;
- for (int i = startRowKey; i <= endRowKey; i++)
- {
- DecoratedKey key = Util.dk(Integer.toString(i));
- for (int j = 0; j < 10; j++)
- {
- new RowUpdateBuilder(cfm, timestamp, j > 0 ? ttl : 0, key.getKey())
- .clustering(Integer.toString(j))
- .add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER)
- .build()
- .applyUnsafe();
- }
- }
- return timestamp;
- }
-
@Test
public void testSuperColumnTombstones()
{
Keyspace keyspace = Keyspace.open(KEYSPACE1);
ColumnFamilyStore cfs = keyspace.getColumnFamilyStore("Super1");
- CFMetaData table = cfs.metadata;
+ TableMetadata table = cfs.metadata();
cfs.disableAutoCompaction();
DecoratedKey key = Util.dk("tskey");
@@ -172,9 +193,9 @@
// a subcolumn
new RowUpdateBuilder(table, FBUtilities.timestampMicros(), key.getKey())
- .clustering(ByteBufferUtil.bytes("cols"))
- .add("val", "val1")
- .build().applyUnsafe();
+ .clustering(ByteBufferUtil.bytes("cols"))
+ .add("val", "val1")
+ .build().applyUnsafe();
cfs.forceBlockingFlush();
// shadow the subcolumn with a supercolumn tombstone
@@ -208,8 +229,7 @@
ColumnFamilyStore store = keyspace.getColumnFamilyStore(CF_STANDARD1);
store.clearUnsafe();
- MigrationManager.announceColumnFamilyUpdate(store.metadata.params(TableParams.builder(store.metadata.params).gcGraceSeconds(1).compaction(CompactionParams.scts(compactionOptions)).build()), true);
- store.reload();
+ MigrationManager.announceTableUpdate(store.metadata().unbuild().gcGraceSeconds(1).compaction(CompactionParams.stcs(compactionOptions)).build(), true);
// disable compaction while flushing
store.disableAutoCompaction();
@@ -246,14 +266,13 @@
long newSize1 = it.next().uncompressedLength();
long newSize2 = it.next().uncompressedLength();
assertEquals("candidate sstable should not be tombstone-compacted because its key range overlap with other sstable",
- originalSize1, newSize1);
+ originalSize1, newSize1);
assertEquals("candidate sstable should not be tombstone-compacted because its key range overlap with other sstable",
- originalSize2, newSize2);
+ originalSize2, newSize2);
// now let's enable the magic property
compactionOptions.put("unchecked_tombstone_compaction", "true");
- MigrationManager.announceColumnFamilyUpdate(store.metadata.params(TableParams.builder(store.metadata.params).gcGraceSeconds(1).compaction(CompactionParams.scts(compactionOptions)).build()), true);
- store.reload();
+ MigrationManager.announceTableUpdate(store.metadata().unbuild().gcGraceSeconds(1).compaction(CompactionParams.stcs(compactionOptions)).build(), true);
//submit background task again and wait for it to complete
FBUtilities.waitOnFutures(CompactionManager.instance.submitBackground(store));
@@ -282,13 +301,14 @@
assertEquals(maxTimestampExpected, maxTimestampObserved);
}
+
@Test
public void testDontPurgeAccidentally() throws InterruptedException
{
- testDontPurgeAccidentally("test1", CF_SUPER5);
+ testDontPurgeAccidentally("test1", "Super5");
// Use CF with gc_grace=0, see last bug of CASSANDRA-2786
- testDontPurgeAccidentally("test1", CF_SUPERGC);
+ testDontPurgeAccidentally("test1", "SuperDirectGC");
}
@Test
@@ -297,7 +317,7 @@
Keyspace keyspace = Keyspace.open(KEYSPACE1);
final String cfname = "Standard3"; // use clean(no sstable) CF
ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfname);
- CFMetaData table = cfs.metadata;
+ TableMetadata table = cfs.metadata();
// disable compaction while flushing
cfs.disableAutoCompaction();
@@ -331,7 +351,7 @@
assertEquals( prevGeneration + 1, sstables.iterator().next().descriptor.generation);
}
- public static void writeSSTableWithRangeTombstoneMaskingOneColumn(ColumnFamilyStore cfs, CFMetaData table, int[] dks) {
+ public static void writeSSTableWithRangeTombstoneMaskingOneColumn(ColumnFamilyStore cfs, TableMetadata table, int[] dks) {
for (int dk : dks)
{
RowUpdateBuilder deletedRowUpdateBuilder = new RowUpdateBuilder(table, 1, Util.dk(Integer.toString(dk)));
@@ -358,7 +378,7 @@
// disable compaction while flushing
cfs.disableAutoCompaction();
- final CFMetaData table = cfs.metadata;
+ final TableMetadata table = cfs.metadata();
Directories dir = cfs.getDirectories();
ArrayList<DecoratedKey> keys = new ArrayList<DecoratedKey>();
@@ -388,17 +408,17 @@
for (FilteredPartition p : Util.getAll(Util.cmd(cfs).build()))
{
k.add(p.partitionKey());
- final SinglePartitionReadCommand command = SinglePartitionReadCommand.create(cfs.metadata, FBUtilities.nowInSeconds(), ColumnFilter.all(cfs.metadata), RowFilter.NONE, DataLimits.NONE, p.partitionKey(), new ClusteringIndexSliceFilter(Slices.ALL, false));
+ final SinglePartitionReadCommand command = SinglePartitionReadCommand.create(cfs.metadata(), FBUtilities.nowInSeconds(), ColumnFilter.all(cfs.metadata()), RowFilter.NONE, DataLimits.NONE, p.partitionKey(), new ClusteringIndexSliceFilter(Slices.ALL, false));
try (ReadExecutionController executionController = command.executionController();
PartitionIterator iterator = command.executeInternal(executionController))
{
try (RowIterator rowIterator = iterator.next())
{
Row row = rowIterator.next();
- Cell cell = row.getCell(cfs.metadata.getColumnDefinition(new ColumnIdentifier("val", false)));
+ Cell cell = row.getCell(cfs.metadata().getColumn(new ColumnIdentifier("val", false)));
assertEquals(ByteBufferUtil.bytes("a"), cell.value());
assertEquals(3, cell.timestamp());
- assertNotSame(ByteBufferUtil.bytes("01"), row.clustering().getRawValues()[0]);
+ assertNotEquals(ByteBufferUtil.bytes("01"), row.clustering().getRawValues()[0]);
assertEquals(ByteBufferUtil.bytes("02"), row.clustering().getRawValues()[0]);
}
}
@@ -418,7 +438,7 @@
// This test catches the regression of CASSANDRA-2786
Keyspace keyspace = Keyspace.open(KEYSPACE1);
ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfname);
- CFMetaData table = cfs.metadata;
+ TableMetadata table = cfs.metadata();
// disable compaction while flushing
cfs.clearUnsafe();
@@ -499,7 +519,7 @@
{
long timestamp = System.currentTimeMillis();
DecoratedKey dk = Util.dk(String.format("%03d", key));
- new RowUpdateBuilder(Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD1).metadata, timestamp, dk.getKey())
+ new RowUpdateBuilder(Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD1).metadata(), timestamp, dk.getKey())
.add("val", "val")
.build()
.applyUnsafe();
diff --git a/test/unit/org/apache/cassandra/db/compaction/CorruptedSSTablesCompactionsTest.java b/test/unit/org/apache/cassandra/db/compaction/CorruptedSSTablesCompactionsTest.java
index 231c2b5..95542a1 100644
--- a/test/unit/org/apache/cassandra/db/compaction/CorruptedSSTablesCompactionsTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/CorruptedSSTablesCompactionsTest.java
@@ -94,7 +94,7 @@
/**
* Return a table metadata, we use types with fixed size to increase the chance of detecting corrupt data
*/
- private static CFMetaData makeTable(String tableName)
+ private static TableMetadata.Builder makeTable(String tableName)
{
return SchemaLoader.standardCFMD(KEYSPACE1, tableName, 1, LongType.instance, LongType.instance, LongType.instance);
}
@@ -135,7 +135,7 @@
final ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(tableName);
final int ROWS_PER_SSTABLE = 10;
- final int SSTABLES = cfs.metadata.params.minIndexInterval * 2 / ROWS_PER_SSTABLE;
+ final int SSTABLES = cfs.metadata().params.minIndexInterval * 2 / ROWS_PER_SSTABLE;
final int SSTABLES_TO_CORRUPT = 8;
assertTrue(String.format("Not enough sstables (%d), expected at least %d sstables to corrupt", SSTABLES, SSTABLES_TO_CORRUPT),
@@ -154,7 +154,7 @@
{
DecoratedKey key = Util.dk(String.valueOf(i), LongType.instance);
long timestamp = j * ROWS_PER_SSTABLE + i;
- new RowUpdateBuilder(cfs.metadata, timestamp, key.getKey())
+ new RowUpdateBuilder(cfs.metadata(), timestamp, key.getKey())
.clustering(Long.valueOf(i))
.add("val", Long.valueOf(i))
.build()
diff --git a/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java b/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java
index aa886b4..f75842d 100644
--- a/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java
@@ -227,7 +227,7 @@
for (int r = 0; r < numSSTables; r++)
{
DecoratedKey key = Util.dk(String.valueOf(r));
- new RowUpdateBuilder(cfs.metadata, r, key.getKey())
+ new RowUpdateBuilder(cfs.metadata(), r, key.getKey())
.clustering("column")
.add("val", value).build().applyUnsafe();
@@ -263,7 +263,7 @@
for (int r = 0; r < numSSTables; r++)
{
DecoratedKey key = Util.dk(String.valueOf(r));
- new RowUpdateBuilder(cfs.metadata, r, key.getKey())
+ new RowUpdateBuilder(cfs.metadata(), r, key.getKey())
.clustering("column")
.add("val", value).build().applyUnsafe();
@@ -300,7 +300,7 @@
// create 2 sstables
DecoratedKey key = Util.dk(String.valueOf("expired"));
- new RowUpdateBuilder(cfs.metadata, System.currentTimeMillis(), 1, key.getKey())
+ new RowUpdateBuilder(cfs.metadata(), System.currentTimeMillis(), 1, key.getKey())
.clustering("column")
.add("val", value).build().applyUnsafe();
@@ -309,7 +309,7 @@
Thread.sleep(10);
key = Util.dk(String.valueOf("nonexpired"));
- new RowUpdateBuilder(cfs.metadata, System.currentTimeMillis(), key.getKey())
+ new RowUpdateBuilder(cfs.metadata(), System.currentTimeMillis(), key.getKey())
.clustering("column")
.add("val", value).build().applyUnsafe();
@@ -353,7 +353,7 @@
for (int i = 0; i < 10; i++)
{
DecoratedKey key = Util.dk(String.valueOf(r));
- new RowUpdateBuilder(cfs.metadata, timestamp, key.getKey())
+ new RowUpdateBuilder(cfs.metadata(), timestamp, key.getKey())
.clustering("column")
.add("val", bigValue).build().applyUnsafe();
}
@@ -363,7 +363,7 @@
for (int r = 0; r < numSSTables / 2; r++)
{
DecoratedKey key = Util.dk(String.valueOf(r));
- new RowUpdateBuilder(cfs.metadata, timestamp, key.getKey())
+ new RowUpdateBuilder(cfs.metadata(), timestamp, key.getKey())
.clustering("column")
.add("val", value).build().applyUnsafe();
cfs.forceBlockingFlush();
diff --git a/test/unit/org/apache/cassandra/db/compaction/LeveledCompactionStrategyTest.java b/test/unit/org/apache/cassandra/db/compaction/LeveledCompactionStrategyTest.java
index b1d467e..8a8ed13 100644
--- a/test/unit/org/apache/cassandra/db/compaction/LeveledCompactionStrategyTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/LeveledCompactionStrategyTest.java
@@ -17,17 +17,20 @@
*/
package org.apache.cassandra.db.compaction;
+import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
+import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
+import java.util.Map;
import java.util.Random;
import java.util.UUID;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.junit.After;
import org.junit.Before;
import org.junit.BeforeClass;
@@ -51,6 +54,9 @@
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.notifications.SSTableAddedNotification;
import org.apache.cassandra.notifications.SSTableRepairStatusChanged;
+import org.apache.cassandra.repair.ValidationManager;
+import org.apache.cassandra.schema.MockSchema;
+import org.apache.cassandra.streaming.PreviewKind;
import org.apache.cassandra.repair.RepairJobDesc;
import org.apache.cassandra.repair.Validator;
import org.apache.cassandra.schema.CompactionParams;
@@ -121,7 +127,7 @@
// Adds enough data to trigger multiple sstable per level
for (int r = 0; r < rows; r++)
{
- UpdateBuilder update = UpdateBuilder.create(cfs.metadata, String.valueOf(r));
+ UpdateBuilder update = UpdateBuilder.create(cfs.metadata(), String.valueOf(r));
for (int c = 0; c < columns; c++)
update.newRow("column" + c).add("val", value);
update.applyUnsafe();
@@ -177,7 +183,7 @@
// Adds enough data to trigger multiple sstable per level
for (int r = 0; r < rows; r++)
{
- UpdateBuilder update = UpdateBuilder.create(cfs.metadata, String.valueOf(r));
+ UpdateBuilder update = UpdateBuilder.create(cfs.metadata(), String.valueOf(r));
for (int c = 0; c < columns; c++)
update.newRow("column" + c).add("val", value);
update.applyUnsafe();
@@ -193,10 +199,18 @@
Range<Token> range = new Range<>(Util.token(""), Util.token(""));
int gcBefore = keyspace.getColumnFamilyStore(CF_STANDARDDLEVELED).gcBefore(FBUtilities.nowInSeconds());
UUID parentRepSession = UUID.randomUUID();
- ActiveRepairService.instance.registerParentRepairSession(parentRepSession, FBUtilities.getBroadcastAddress(), Arrays.asList(cfs), Arrays.asList(range), false, System.currentTimeMillis(), true);
+ ActiveRepairService.instance.registerParentRepairSession(parentRepSession,
+ FBUtilities.getBroadcastAddressAndPort(),
+ Arrays.asList(cfs),
+ Arrays.asList(range),
+ false,
+ ActiveRepairService.UNREPAIRED_SSTABLE,
+ true,
+ PreviewKind.NONE);
RepairJobDesc desc = new RepairJobDesc(parentRepSession, UUID.randomUUID(), KEYSPACE1, CF_STANDARDDLEVELED, Arrays.asList(range));
- Validator validator = new Validator(desc, FBUtilities.getBroadcastAddress(), gcBefore);
- CompactionManager.instance.submitValidation(cfs, validator).get();
+ Validator validator = new Validator(desc, FBUtilities.getBroadcastAddressAndPort(), gcBefore, PreviewKind.NONE);
+
+ ValidationManager.instance.submitValidation(cfs, validator).get();
}
/**
@@ -243,7 +257,7 @@
int columns = 10;
for (int r = 0; r < rows; r++)
{
- UpdateBuilder update = UpdateBuilder.create(cfs.metadata, String.valueOf(r));
+ UpdateBuilder update = UpdateBuilder.create(cfs.metadata(), String.valueOf(r));
for (int c = 0; c < columns; c++)
update.newRow("column" + c).add("val", value);
update.applyUnsafe();
@@ -280,7 +294,7 @@
// Adds enough data to trigger multiple sstable per level
for (int r = 0; r < rows; r++)
{
- UpdateBuilder update = UpdateBuilder.create(cfs.metadata, String.valueOf(r));
+ UpdateBuilder update = UpdateBuilder.create(cfs.metadata(), String.valueOf(r));
for (int c = 0; c < columns; c++)
update.newRow("column" + c).add("val", value);
update.applyUnsafe();
@@ -321,7 +335,7 @@
// Adds enough data to trigger multiple sstable per level
for (int r = 0; r < rows; r++)
{
- UpdateBuilder update = UpdateBuilder.create(cfs.metadata, String.valueOf(r));
+ UpdateBuilder update = UpdateBuilder.create(cfs.metadata(), String.valueOf(r));
for (int c = 0; c < columns; c++)
update.newRow("column" + c).add("val", value);
update.applyUnsafe();
@@ -330,7 +344,7 @@
waitForLeveling(cfs);
cfs.disableAutoCompaction();
- while(CompactionManager.instance.isCompacting(Arrays.asList(cfs)))
+ while(CompactionManager.instance.isCompacting(Arrays.asList(cfs), (sstable) -> true))
Thread.sleep(100);
CompactionStrategyManager manager = cfs.getCompactionStrategyManager();
@@ -354,7 +368,7 @@
SSTableReader sstable1 = unrepaired.manifest.generations[2].get(0);
SSTableReader sstable2 = unrepaired.manifest.generations[1].get(0);
- sstable1.descriptor.getMetadataSerializer().mutateRepairedAt(sstable1.descriptor, System.currentTimeMillis());
+ sstable1.descriptor.getMetadataSerializer().mutateRepairMetadata(sstable1.descriptor, System.currentTimeMillis(), null, false);
sstable1.reloadSSTableMetadata();
assertTrue(sstable1.isRepaired());
@@ -370,7 +384,7 @@
assertFalse(unrepaired.manifest.generations[2].contains(sstable1));
unrepaired.removeSSTable(sstable2);
- manager.handleNotification(new SSTableAddedNotification(singleton(sstable2)), this);
+ manager.handleNotification(new SSTableAddedNotification(singleton(sstable2), null), this);
assertTrue(unrepaired.manifest.getLevel(1).contains(sstable2));
assertFalse(repaired.manifest.getLevel(1).contains(sstable2));
}
@@ -399,7 +413,7 @@
// create 10 sstables that contain data for both key1 and key2
for (int i = 0; i < numIterations; i++) {
for (DecoratedKey key : keys) {
- UpdateBuilder update = UpdateBuilder.create(cfs.metadata, key);
+ UpdateBuilder update = UpdateBuilder.create(cfs.metadata(), key);
for (int c = 0; c < columns; c++)
update.newRow("column" + c).add("val", value);
update.applyUnsafe();
@@ -410,7 +424,7 @@
// create 20 more sstables with 10 containing data for key1 and other 10 containing data for key2
for (int i = 0; i < numIterations; i++) {
for (DecoratedKey key : keys) {
- UpdateBuilder update = UpdateBuilder.create(cfs.metadata, key);
+ UpdateBuilder update = UpdateBuilder.create(cfs.metadata(), key);
for (int c = 0; c < columns; c++)
update.newRow("column" + c).add("val", value);
update.applyUnsafe();
@@ -427,7 +441,7 @@
Collection<Range<Token>> tokenRanges = new ArrayList<>(Arrays.asList(tokenRange));
cfs.forceCompactionForTokenRange(tokenRanges);
- while(CompactionManager.instance.isCompacting(Arrays.asList(cfs))) {
+ while(CompactionManager.instance.isCompacting(Arrays.asList(cfs), (sstable) -> true)) {
Thread.sleep(100);
}
@@ -440,12 +454,64 @@
cfs.forceCompactionForTokenRange(tokenRanges2);
- while(CompactionManager.instance.isCompacting(Arrays.asList(cfs))) {
+ while(CompactionManager.instance.isCompacting(Arrays.asList(cfs), (sstable) -> true)) {
Thread.sleep(100);
}
// the 11 tables containing key1 should all compact to 1 table
assertEquals(1, cfs.getLiveSSTables().size());
+ // Set it up again
+ cfs.truncateBlocking();
+
+ // create 10 sstables that contain data for both key1 and key2
+ for (int i = 0; i < numIterations; i++)
+ {
+ for (DecoratedKey key : keys)
+ {
+ UpdateBuilder update = UpdateBuilder.create(cfs.metadata(), key);
+ for (int c = 0; c < columns; c++)
+ update.newRow("column" + c).add("val", value);
+ update.applyUnsafe();
+ }
+ cfs.forceBlockingFlush();
+ }
+
+ // create 20 more sstables with 10 containing data for key1 and other 10 containing data for key2
+ for (int i = 0; i < numIterations; i++)
+ {
+ for (DecoratedKey key : keys)
+ {
+ UpdateBuilder update = UpdateBuilder.create(cfs.metadata(), key);
+ for (int c = 0; c < columns; c++)
+ update.newRow("column" + c).add("val", value);
+ update.applyUnsafe();
+ cfs.forceBlockingFlush();
+ }
+ }
+
+ // We should have a total of 30 sstables again
+ assertEquals(30, cfs.getLiveSSTables().size());
+
+ // This time, we're going to make sure the token range wraps around, to cover the full range
+ Range<Token> wrappingRange;
+ if (key1.getToken().compareTo(key2.getToken()) < 0)
+ {
+ wrappingRange = new Range<>(key2.getToken(), key1.getToken());
+ }
+ else
+ {
+ wrappingRange = new Range<>(key1.getToken(), key2.getToken());
+ }
+ Collection<Range<Token>> wrappingRanges = new ArrayList<>(Arrays.asList(wrappingRange));
+ cfs.forceCompactionForTokenRange(wrappingRanges);
+
+ while(CompactionManager.instance.isCompacting(Arrays.asList(cfs), (sstable) -> true))
+ {
+ Thread.sleep(100);
+ }
+
+ // should all compact to 1 table
+ assertEquals(1, cfs.getLiveSSTables().size());
}
@Test
@@ -461,7 +527,7 @@
cfs.disableAutoCompaction();
for (int r = 0; r < rows; r++)
{
- UpdateBuilder update = UpdateBuilder.create(cfs.metadata, String.valueOf(r));
+ UpdateBuilder update = UpdateBuilder.create(cfs.metadata(), String.valueOf(r));
for (int c = 0; c < columns; c++)
update.newRow("column" + c).add("val", value);
update.applyUnsafe();
@@ -480,4 +546,70 @@
lastMaxTimeStamp = sstable.getMaxTimestamp();
}
}
+
+ @Test
+ public void testDisableSTCSInL0() throws IOException
+ {
+ /*
+ First creates a bunch of sstables in L1, then overloads L0 with 50 sstables. Now with STCS in L0 enabled
+ we should get a compaction task where the target level is 0, then we disable STCS-in-L0 and make sure that
+ the compaction task we get targets L1 or higher.
+ */
+ ColumnFamilyStore cfs = MockSchema.newCFS();
+ Map<String, String> localOptions = new HashMap<>();
+ localOptions.put("class", "LeveledCompactionStrategy");
+ localOptions.put("sstable_size_in_mb", "1");
+ cfs.setCompactionParameters(localOptions);
+ List<SSTableReader> sstables = new ArrayList<>();
+ for (int i = 0; i < 11; i++)
+ {
+ SSTableReader l1sstable = MockSchema.sstable(i, 1 * 1024 * 1024, cfs);
+ l1sstable.descriptor.getMetadataSerializer().mutateLevel(l1sstable.descriptor, 1);
+ l1sstable.reloadSSTableMetadata();
+ sstables.add(l1sstable);
+ }
+
+ for (int i = 100; i < 150; i++)
+ sstables.add(MockSchema.sstable(i, 1 * 1024 * 1024, cfs));
+
+ cfs.disableAutoCompaction();
+ cfs.addSSTables(sstables);
+ assertEquals(0, getTaskLevel(cfs));
+
+ try
+ {
+ CompactionManager.instance.setDisableSTCSInL0(true);
+ assertTrue(getTaskLevel(cfs) > 0);
+ }
+ finally
+ {
+ CompactionManager.instance.setDisableSTCSInL0(false);
+ }
+ }
+
+ private int getTaskLevel(ColumnFamilyStore cfs)
+ {
+ int level = -1;
+ for (List<AbstractCompactionStrategy> strategies : cfs.getCompactionStrategyManager().getStrategies())
+ {
+ for (AbstractCompactionStrategy strategy : strategies)
+ {
+ AbstractCompactionTask task = strategy.getNextBackgroundTask(0);
+ if (task != null)
+ {
+ try
+ {
+ assertTrue(task instanceof LeveledCompactionTask);
+ LeveledCompactionTask lcsTask = (LeveledCompactionTask) task;
+ level = Math.max(level, lcsTask.getLevel());
+ }
+ finally
+ {
+ task.transaction.abort();
+ }
+ }
+ }
+ }
+ return level;
+ }
}
diff --git a/test/unit/org/apache/cassandra/db/compaction/OneCompactionTest.java b/test/unit/org/apache/cassandra/db/compaction/OneCompactionTest.java
index f55bf52..0c469dc 100644
--- a/test/unit/org/apache/cassandra/db/compaction/OneCompactionTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/OneCompactionTest.java
@@ -64,7 +64,7 @@
Set<String> inserted = new HashSet<>();
for (int j = 0; j < insertsPerTable; j++) {
String key = String.valueOf(j);
- new RowUpdateBuilder(store.metadata, j, key)
+ new RowUpdateBuilder(store.metadata(), j, key)
.clustering("0")
.add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER)
.build()
diff --git a/test/unit/org/apache/cassandra/db/compaction/PendingRepairManagerTest.java b/test/unit/org/apache/cassandra/db/compaction/PendingRepairManagerTest.java
new file mode 100644
index 0000000..9f4cf8d
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/compaction/PendingRepairManagerTest.java
@@ -0,0 +1,308 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.UUID;
+
+import com.google.common.collect.Lists;
+import org.junit.Assert;
+import org.junit.Test;
+
+import org.apache.cassandra.db.lifecycle.SSTableSet;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.repair.consistent.LocalSessionAccessor;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.UUIDGen;
+
+public class PendingRepairManagerTest extends AbstractPendingRepairTest
+{
+ /**
+ * If a local session is ongoing, it should not be cleaned up
+ */
+ @Test
+ public void needsCleanupInProgress()
+ {
+ PendingRepairManager prm = csm.getPendingRepairManagers().get(0);
+
+ UUID repairID = registerSession(cfs, true, true);
+ LocalSessionAccessor.prepareUnsafe(repairID, COORDINATOR, PARTICIPANTS);
+ SSTableReader sstable = makeSSTable(true);
+ mutateRepaired(sstable, repairID, false);
+ prm.addSSTable(sstable);
+ Assert.assertNotNull(prm.get(repairID));
+
+ Assert.assertFalse(prm.canCleanup(repairID));
+ }
+
+ /**
+ * If a local session is finalized, it should be cleaned up
+ */
+ @Test
+ public void needsCleanupFinalized()
+ {
+ PendingRepairManager prm = csm.getPendingRepairManagers().get(0);
+
+ UUID repairID = registerSession(cfs, true, true);
+ LocalSessionAccessor.prepareUnsafe(repairID, COORDINATOR, PARTICIPANTS);
+ SSTableReader sstable = makeSSTable(true);
+ mutateRepaired(sstable, repairID, false);
+ prm.addSSTable(sstable);
+ Assert.assertNotNull(prm.get(repairID));
+ LocalSessionAccessor.finalizeUnsafe(repairID);
+
+ Assert.assertTrue(prm.canCleanup(repairID));
+ }
+
+ /**
+ * If a local session has failed, it should be cleaned up
+ */
+ @Test
+ public void needsCleanupFailed()
+ {
+ PendingRepairManager prm = csm.getPendingRepairManagers().get(0);
+
+ UUID repairID = registerSession(cfs, true, true);
+ LocalSessionAccessor.prepareUnsafe(repairID, COORDINATOR, PARTICIPANTS);
+ SSTableReader sstable = makeSSTable(true);
+ mutateRepaired(sstable, repairID, false);
+ prm.addSSTable(sstable);
+ Assert.assertNotNull(prm.get(repairID));
+ LocalSessionAccessor.failUnsafe(repairID);
+
+ Assert.assertTrue(prm.canCleanup(repairID));
+ }
+
+ @Test
+ public void needsCleanupNoSession()
+ {
+ UUID fakeID = UUIDGen.getTimeUUID();
+ PendingRepairManager prm = new PendingRepairManager(cfs, null, false);
+ Assert.assertTrue(prm.canCleanup(fakeID));
+ }
+
+ @Test
+ public void estimateRemainingTasksInProgress()
+ {
+ PendingRepairManager prm = csm.getPendingRepairManagers().get(0);
+
+ UUID repairID = registerSession(cfs, true, true);
+ LocalSessionAccessor.prepareUnsafe(repairID, COORDINATOR, PARTICIPANTS);
+ SSTableReader sstable = makeSSTable(true);
+ mutateRepaired(sstable, repairID, false);
+ prm.addSSTable(sstable);
+ Assert.assertNotNull(prm.get(repairID));
+
+ Assert.assertEquals(0, prm.getEstimatedRemainingTasks());
+ Assert.assertEquals(0, prm.getNumPendingRepairFinishedTasks());
+ }
+
+ @Test
+ public void estimateRemainingFinishedRepairTasks()
+ {
+ PendingRepairManager prm = csm.getPendingRepairManagers().get(0);
+
+ UUID repairID = registerSession(cfs, true, true);
+ LocalSessionAccessor.prepareUnsafe(repairID, COORDINATOR, PARTICIPANTS);
+ SSTableReader sstable = makeSSTable(true);
+ mutateRepaired(sstable, repairID, false);
+ prm.addSSTable(sstable);
+ Assert.assertNotNull(prm.get(repairID));
+ Assert.assertNotNull(prm.get(repairID));
+ LocalSessionAccessor.finalizeUnsafe(repairID);
+
+ Assert.assertEquals(0, prm.getEstimatedRemainingTasks());
+ Assert.assertEquals(1, prm.getNumPendingRepairFinishedTasks());
+ }
+
+ @Test
+ public void getNextBackgroundTask()
+ {
+ PendingRepairManager prm = csm.getPendingRepairManagers().get(0);
+
+ UUID repairID = registerSession(cfs, true, true);
+ LocalSessionAccessor.prepareUnsafe(repairID, COORDINATOR, PARTICIPANTS);
+ SSTableReader sstable = makeSSTable(true);
+ mutateRepaired(sstable, repairID, false);
+ prm.addSSTable(sstable);
+
+ repairID = registerSession(cfs, true, true);
+ LocalSessionAccessor.prepareUnsafe(repairID, COORDINATOR, PARTICIPANTS);
+ sstable = makeSSTable(true);
+ mutateRepaired(sstable, repairID, false);
+ prm.addSSTable(sstable);
+ LocalSessionAccessor.finalizeUnsafe(repairID);
+
+ Assert.assertEquals(2, prm.getSessions().size());
+ Assert.assertNull(prm.getNextBackgroundTask(FBUtilities.nowInSeconds()));
+ AbstractCompactionTask compactionTask = prm.getNextRepairFinishedTask();
+ try
+ {
+ Assert.assertNotNull(compactionTask);
+ Assert.assertSame(PendingRepairManager.RepairFinishedCompactionTask.class, compactionTask.getClass());
+ PendingRepairManager.RepairFinishedCompactionTask cleanupTask = (PendingRepairManager.RepairFinishedCompactionTask) compactionTask;
+ Assert.assertEquals(repairID, cleanupTask.getSessionID());
+ }
+ finally
+ {
+ compactionTask.transaction.abort();
+ }
+ }
+
+ @Test
+ public void getNextBackgroundTaskNoSessions()
+ {
+ PendingRepairManager prm = csm.getPendingRepairManagers().get(0);
+ Assert.assertNull(prm.getNextBackgroundTask(FBUtilities.nowInSeconds()));
+ }
+
+ /**
+ * If all sessions should be cleaned up, getNextBackgroundTask should return null
+ */
+ @Test
+ public void getNextBackgroundTaskAllCleanup() throws Exception
+ {
+ PendingRepairManager prm = csm.getPendingRepairManagers().get(0);
+ UUID repairID = registerSession(cfs, true, true);
+ LocalSessionAccessor.prepareUnsafe(repairID, COORDINATOR, PARTICIPANTS);
+
+ SSTableReader sstable = makeSSTable(true);
+ mutateRepaired(sstable, repairID, false);
+ prm.addSSTable(sstable);
+ Assert.assertNotNull(prm.get(repairID));
+ Assert.assertNotNull(prm.get(repairID));
+ LocalSessionAccessor.finalizeUnsafe(repairID);
+
+ Assert.assertNull(prm.getNextBackgroundTask(FBUtilities.nowInSeconds()));
+
+ }
+
+ @Test
+ public void maximalTaskNeedsCleanup()
+ {
+ PendingRepairManager prm = csm.getPendingRepairManagers().get(0);
+
+ UUID repairID = registerSession(cfs, true, true);
+ LocalSessionAccessor.prepareUnsafe(repairID, COORDINATOR, PARTICIPANTS);
+ SSTableReader sstable = makeSSTable(true);
+ mutateRepaired(sstable, repairID, false);
+ prm.addSSTable(sstable);
+ Assert.assertNotNull(prm.get(repairID));
+ Assert.assertNotNull(prm.get(repairID));
+ LocalSessionAccessor.finalizeUnsafe(repairID);
+
+ Collection<AbstractCompactionTask> tasks = prm.getMaximalTasks(FBUtilities.nowInSeconds(), false);
+ try
+ {
+ Assert.assertEquals(1, tasks.size());
+ }
+ finally
+ {
+ tasks.stream().forEach(t -> t.transaction.abort());
+ }
+ }
+
+ @Test
+ public void userDefinedTaskTest()
+ {
+ PendingRepairManager prm = csm.getPendingRepairManagers().get(0);
+ UUID repairId = registerSession(cfs, true, true);
+ SSTableReader sstable = makeSSTable(true);
+ mutateRepaired(sstable, repairId, false);
+ prm.addSSTable(sstable);
+
+ try (CompactionTasks tasks = csm.getUserDefinedTasks(Collections.singleton(sstable), 100))
+ {
+ Assert.assertEquals(1, tasks.size());
+ }
+ }
+
+ @Test
+ public void mixedPendingSessionsTest()
+ {
+ PendingRepairManager prm = csm.getPendingRepairManagers().get(0);
+ UUID repairId = registerSession(cfs, true, true);
+ UUID repairId2 = registerSession(cfs, true, true);
+ SSTableReader sstable = makeSSTable(true);
+ SSTableReader sstable2 = makeSSTable(true);
+
+ mutateRepaired(sstable, repairId, false);
+ mutateRepaired(sstable2, repairId2, false);
+ prm.addSSTable(sstable);
+ prm.addSSTable(sstable2);
+ try (CompactionTasks tasks = csm.getUserDefinedTasks(Lists.newArrayList(sstable, sstable2), 100))
+ {
+ Assert.assertEquals(2, tasks.size());
+ }
+ }
+
+ /**
+ * Tests that a IllegalSSTableArgumentException is thrown if we try to get
+ * scanners for an sstable that isn't pending repair
+ */
+ @Test(expected = PendingRepairManager.IllegalSSTableArgumentException.class)
+ public void getScannersInvalidSSTable() throws Exception
+ {
+ PendingRepairManager prm = csm.getPendingRepairManagers().get(0);
+ SSTableReader sstable = makeSSTable(true);
+ prm.getScanners(Collections.singleton(sstable), Collections.singleton(RANGE1));
+ }
+
+ /**
+ * Tests that a IllegalSSTableArgumentException is thrown if we try to get
+ * scanners for an sstable that isn't pending repair
+ */
+ @Test(expected = PendingRepairManager.IllegalSSTableArgumentException.class)
+ public void getOrCreateInvalidSSTable() throws Exception
+ {
+ PendingRepairManager prm = csm.getPendingRepairManagers().get(0);
+ SSTableReader sstable = makeSSTable(true);
+ prm.getOrCreate(sstable);
+ }
+
+ @Test
+ public void sessionHasData()
+ {
+ PendingRepairManager prm = csm.getPendingRepairManagers().get(0);
+
+ UUID repairID = registerSession(cfs, true, true);
+ LocalSessionAccessor.prepareUnsafe(repairID, COORDINATOR, PARTICIPANTS);
+
+ Assert.assertFalse(prm.hasDataForSession(repairID));
+ SSTableReader sstable = makeSSTable(true);
+ mutateRepaired(sstable, repairID, false);
+ prm.addSSTable(sstable);
+ Assert.assertTrue(prm.hasDataForSession(repairID));
+ }
+
+ @Test
+ public void noEmptyCompactionTask()
+ {
+ PendingRepairManager prm = csm.getPendingRepairManagers().get(0);
+ SSTableReader sstable = makeSSTable(false);
+ UUID id = UUID.randomUUID();
+ mutateRepaired(sstable, id, false);
+ prm.getOrCreate(sstable);
+ cfs.truncateBlocking();
+ Assert.assertFalse(cfs.getSSTables(SSTableSet.LIVE).iterator().hasNext());
+ Assert.assertNull(cfs.getCompactionStrategyManager().getNextBackgroundTask(0));
+
+ }
+}
diff --git a/test/unit/org/apache/cassandra/db/compaction/SingleSSTableLCSTaskTest.java b/test/unit/org/apache/cassandra/db/compaction/SingleSSTableLCSTaskTest.java
new file mode 100644
index 0000000..61cf302
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/compaction/SingleSSTableLCSTaskTest.java
@@ -0,0 +1,155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+import java.io.RandomAccessFile;
+import java.nio.ByteBuffer;
+import java.util.Random;
+
+import org.apache.commons.lang3.StringUtils;
+import org.junit.Test;
+
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.io.sstable.Component;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class SingleSSTableLCSTaskTest extends CQLTester
+{
+ @Test
+ public void basicTest() throws Throwable
+ {
+ createTable("create table %s (id int primary key, t text) with compaction = {'class':'LeveledCompactionStrategy','single_sstable_uplevel':true}");
+ ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
+ execute("insert into %s (id, t) values (1, 'meep')");
+ cfs.forceBlockingFlush();
+ SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
+
+ try (LifecycleTransaction txn = cfs.getTracker().tryModify(sstable, OperationType.COMPACTION))
+ {
+ if (txn != null)
+ {
+ SingleSSTableLCSTask task = new SingleSSTableLCSTask(cfs, txn, 2);
+ task.executeInternal(null);
+ }
+ }
+ assertEquals(1, cfs.getLiveSSTables().size());
+ cfs.getLiveSSTables().forEach(s -> assertEquals(2, s.getSSTableLevel()));
+ // make sure compaction strategy is notified:
+ LeveledCompactionStrategy lcs = (LeveledCompactionStrategy) cfs.getCompactionStrategyManager().getUnrepairedUnsafe().first();
+ for (int i = 0; i < lcs.manifest.getLevelCount(); i++)
+ {
+ if (i == 2)
+ assertEquals(1, lcs.getLevelSize(i));
+ else
+ assertEquals(0, lcs.getLevelSize(i));
+ }
+ assertTrue(cfs.getTracker().getCompacting().isEmpty());
+ }
+
+ @Test
+ public void compactionTest() throws Throwable
+ {
+ compactionTestHelper(true);
+ }
+
+ @Test
+ public void uplevelDisabledTest() throws Throwable
+ {
+ compactionTestHelper(false);
+ }
+
+ private void compactionTestHelper(boolean singleSSTUplevel) throws Throwable
+ {
+ createTable("create table %s (id int, id2 int, t blob, primary key (id, id2))" +
+ "with compaction = {'class':'LeveledCompactionStrategy', 'single_sstable_uplevel':" + singleSSTUplevel + ", 'sstable_size_in_mb':'1', 'max_threshold':'1000'}");
+ ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
+ cfs.disableAutoCompaction();
+ byte[] b = new byte[10 * 1024];
+ new Random().nextBytes(b);
+ ByteBuffer value = ByteBuffer.wrap(b);
+ for (int i = 0; i < 5000; i++)
+ {
+ for (int j = 0; j < 10; j++)
+ {
+ execute("insert into %s (id, id2, t) values (?, ?, ?)", i, j, value);
+ }
+ if (i % 100 == 0)
+ cfs.forceBlockingFlush();
+ }
+ // now we have a bunch of data in L0, first compaction will be a normal one, containing all sstables:
+ LeveledCompactionStrategy lcs = (LeveledCompactionStrategy) cfs.getCompactionStrategyManager().getUnrepairedUnsafe().first();
+ AbstractCompactionTask act = lcs.getNextBackgroundTask(0);
+ act.execute(ActiveCompactionsTracker.NOOP);
+
+ // now all sstables are laid out non-overlapping in L1, this means that the rest of the compactions
+ // will be single sstable ones, make sure that we use SingleSSTableLCSTask if singleSSTUplevel is true:
+ while (lcs.getEstimatedRemainingTasks() > 0)
+ {
+ act = lcs.getNextBackgroundTask(0);
+ assertEquals(singleSSTUplevel, act instanceof SingleSSTableLCSTask);
+ act.execute(ActiveCompactionsTracker.NOOP);
+ }
+ assertEquals(0, lcs.getLevelSize(0));
+ int l1size = lcs.getLevelSize(1);
+ // this should be 10, but it might vary a bit depending on partition sizes etc
+ assertTrue(l1size >= 8 && l1size <= 12);
+ assertTrue(lcs.getLevelSize(2) > 0);
+ }
+
+ @Test
+ public void corruptMetadataTest() throws Throwable
+ {
+ createTable("create table %s (id int primary key, t text) with compaction = {'class':'LeveledCompactionStrategy','single_sstable_uplevel':true}");
+ ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
+ execute("insert into %s (id, t) values (1, 'meep')");
+ cfs.forceBlockingFlush();
+ SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
+
+ String filenameToCorrupt = sstable.descriptor.filenameFor(Component.STATS);
+ RandomAccessFile file = new RandomAccessFile(filenameToCorrupt, "rw");
+ file.seek(0);
+ file.writeBytes(StringUtils.repeat('z', 2));
+ file.close();
+ boolean gotException = false;
+ try (LifecycleTransaction txn = cfs.getTracker().tryModify(sstable, OperationType.COMPACTION))
+ {
+ if (txn != null)
+ {
+ SingleSSTableLCSTask task = new SingleSSTableLCSTask(cfs, txn, 2);
+ task.executeInternal(null);
+ }
+ }
+ catch (Throwable t)
+ {
+ gotException = true;
+ }
+ assertTrue(gotException);
+ assertEquals(1, cfs.getLiveSSTables().size());
+ for (SSTableReader sst : cfs.getLiveSSTables())
+ assertEquals(0, sst.getSSTableMetadata().sstableLevel);
+ LeveledCompactionStrategy lcs = (LeveledCompactionStrategy) cfs.getCompactionStrategyManager().getUnrepairedUnsafe().first();
+ assertEquals(1, lcs.getLevelSize(0));
+ assertTrue(cfs.getTracker().getCompacting().isEmpty());
+ }
+}
diff --git a/test/unit/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategyTest.java b/test/unit/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategyTest.java
index ff0f444..00c4a86 100644
--- a/test/unit/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategyTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategyTest.java
@@ -162,7 +162,7 @@
for (int r = 0; r < numSSTables; r++)
{
String key = String.valueOf(r);
- new RowUpdateBuilder(cfs.metadata, 0, key)
+ new RowUpdateBuilder(cfs.metadata(), 0, key)
.clustering("column").add("val", value)
.build().applyUnsafe();
cfs.forceBlockingFlush();
diff --git a/test/unit/org/apache/cassandra/db/compaction/TTLExpiryTest.java b/test/unit/org/apache/cassandra/db/compaction/TTLExpiryTest.java
index 9fafc74..a2352fc 100644
--- a/test/unit/org/apache/cassandra/db/compaction/TTLExpiryTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/TTLExpiryTest.java
@@ -31,7 +31,7 @@
import org.apache.cassandra.OrderedJUnit4ClassRunner;
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.Util;
-import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.filter.ColumnFilter;
import org.apache.cassandra.db.lifecycle.SSTableSet;
@@ -42,6 +42,7 @@
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.io.sstable.ISSTableScanner;
import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.MigrationManager;
import org.apache.cassandra.tools.SSTableExpiredBlockers;
import org.apache.cassandra.utils.ByteBufferUtil;
@@ -64,17 +65,17 @@
SchemaLoader.createKeyspace(KEYSPACE1,
KeyspaceParams.simple(1),
- CFMetaData.Builder.create(KEYSPACE1, CF_STANDARD1)
- .addPartitionKey("pKey", AsciiType.instance)
- .addRegularColumn("col1", AsciiType.instance)
- .addRegularColumn("col", AsciiType.instance)
- .addRegularColumn("col311", AsciiType.instance)
- .addRegularColumn("col2", AsciiType.instance)
- .addRegularColumn("col3", AsciiType.instance)
- .addRegularColumn("col7", AsciiType.instance)
- .addRegularColumn("col8", MapType.getInstance(AsciiType.instance, AsciiType.instance, true))
- .addRegularColumn("shadow", AsciiType.instance)
- .build().gcGraceSeconds(0));
+ TableMetadata.builder(KEYSPACE1, CF_STANDARD1)
+ .addPartitionKeyColumn("pKey", AsciiType.instance)
+ .addRegularColumn("col1", AsciiType.instance)
+ .addRegularColumn("col", AsciiType.instance)
+ .addRegularColumn("col311", AsciiType.instance)
+ .addRegularColumn("col2", AsciiType.instance)
+ .addRegularColumn("col3", AsciiType.instance)
+ .addRegularColumn("col7", AsciiType.instance)
+ .addRegularColumn("col8", MapType.getInstance(AsciiType.instance, AsciiType.instance, true))
+ .addRegularColumn("shadow", AsciiType.instance)
+ .gcGraceSeconds(0));
}
@Test
@@ -82,36 +83,36 @@
{
ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore("Standard1");
cfs.disableAutoCompaction();
- cfs.metadata.gcGraceSeconds(0);
+ MigrationManager.announceTableUpdate(cfs.metadata().unbuild().gcGraceSeconds(0).build(), true);
String key = "ttl";
- new RowUpdateBuilder(cfs.metadata, 1L, 1, key)
+ new RowUpdateBuilder(cfs.metadata(), 1L, 1, key)
.add("col1", ByteBufferUtil.EMPTY_BYTE_BUFFER)
.build()
.applyUnsafe();
- new RowUpdateBuilder(cfs.metadata, 3L, 1, key)
+ new RowUpdateBuilder(cfs.metadata(), 3L, 1, key)
.add("col2", ByteBufferUtil.EMPTY_BYTE_BUFFER)
.build()
.applyUnsafe();
cfs.forceBlockingFlush();
- new RowUpdateBuilder(cfs.metadata, 2L, 1, key)
+ new RowUpdateBuilder(cfs.metadata(), 2L, 1, key)
.add("col1", ByteBufferUtil.EMPTY_BYTE_BUFFER)
.build()
.applyUnsafe();
- new RowUpdateBuilder(cfs.metadata, 5L, 1, key)
+ new RowUpdateBuilder(cfs.metadata(), 5L, 1, key)
.add("col2", ByteBufferUtil.EMPTY_BYTE_BUFFER)
.build()
.applyUnsafe();
cfs.forceBlockingFlush();
- new RowUpdateBuilder(cfs.metadata, 4L, 1, key)
+ new RowUpdateBuilder(cfs.metadata(), 4L, 1, key)
.add("col1", ByteBufferUtil.EMPTY_BYTE_BUFFER)
.build()
.applyUnsafe();
- new RowUpdateBuilder(cfs.metadata, 7L, 1, key)
+ new RowUpdateBuilder(cfs.metadata(), 7L, 1, key)
.add("shadow", ByteBufferUtil.EMPTY_BYTE_BUFFER)
.build()
.applyUnsafe();
@@ -119,12 +120,12 @@
cfs.forceBlockingFlush();
- new RowUpdateBuilder(cfs.metadata, 6L, 3, key)
+ new RowUpdateBuilder(cfs.metadata(), 6L, 3, key)
.add("shadow", ByteBufferUtil.EMPTY_BYTE_BUFFER)
.build()
.applyUnsafe();
- new RowUpdateBuilder(cfs.metadata, 8L, 1, key)
+ new RowUpdateBuilder(cfs.metadata(), 8L, 1, key)
.add("col2", ByteBufferUtil.EMPTY_BYTE_BUFFER)
.build()
.applyUnsafe();
@@ -163,10 +164,10 @@
cfs.truncateBlocking();
cfs.disableAutoCompaction();
// To reproduce #10944, we need our gcBefore to be equal to the locaDeletionTime. A gcGrace of 1 will (almost always) give us that.
- cfs.metadata.gcGraceSeconds(force10944Bug ? 1 : 0);
+ MigrationManager.announceTableUpdate(cfs.metadata().unbuild().gcGraceSeconds(force10944Bug ? 1 : 0).build(), true);
long timestamp = System.currentTimeMillis();
String key = "ttl";
- new RowUpdateBuilder(cfs.metadata, timestamp, 1, key)
+ new RowUpdateBuilder(cfs.metadata(), timestamp, 1, key)
.add("col", ByteBufferUtil.EMPTY_BYTE_BUFFER)
.add("col7", ByteBufferUtil.EMPTY_BYTE_BUFFER)
.build()
@@ -174,7 +175,7 @@
cfs.forceBlockingFlush();
- new RowUpdateBuilder(cfs.metadata, timestamp, 1, key)
+ new RowUpdateBuilder(cfs.metadata(), timestamp, 1, key)
.add("col2", ByteBufferUtil.EMPTY_BYTE_BUFFER)
.add("col8", Collections.singletonMap("bar", "foo"))
.delete("col1")
@@ -185,14 +186,14 @@
cfs.forceBlockingFlush();
// To reproduce #10944, we need to avoid the optimization that get rid of full sstable because everything
// is known to be gcAble, so keep some data non-expiring in that case.
- new RowUpdateBuilder(cfs.metadata, timestamp, force10944Bug ? 0 : 1, key)
+ new RowUpdateBuilder(cfs.metadata(), timestamp, force10944Bug ? 0 : 1, key)
.add("col3", ByteBufferUtil.EMPTY_BYTE_BUFFER)
.build()
.applyUnsafe();
cfs.forceBlockingFlush();
- new RowUpdateBuilder(cfs.metadata, timestamp, 1, key)
+ new RowUpdateBuilder(cfs.metadata(), timestamp, 1, key)
.add("col311", ByteBufferUtil.EMPTY_BYTE_BUFFER)
.build()
.applyUnsafe();
@@ -211,28 +212,28 @@
ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore("Standard1");
cfs.truncateBlocking();
cfs.disableAutoCompaction();
- cfs.metadata.gcGraceSeconds(0);
+ MigrationManager.announceTableUpdate(cfs.metadata().unbuild().gcGraceSeconds(0).build(), true);
long timestamp = System.currentTimeMillis();
String key = "ttl";
- new RowUpdateBuilder(cfs.metadata, timestamp, 1, key)
+ new RowUpdateBuilder(cfs.metadata(), timestamp, 1, key)
.add("col", ByteBufferUtil.EMPTY_BYTE_BUFFER)
.add("col7", ByteBufferUtil.EMPTY_BYTE_BUFFER)
.build()
.applyUnsafe();
cfs.forceBlockingFlush();
- new RowUpdateBuilder(cfs.metadata, timestamp, 1, key)
+ new RowUpdateBuilder(cfs.metadata(), timestamp, 1, key)
.add("col2", ByteBufferUtil.EMPTY_BYTE_BUFFER)
.build()
.applyUnsafe();
cfs.forceBlockingFlush();
- new RowUpdateBuilder(cfs.metadata, timestamp, 1, key)
+ new RowUpdateBuilder(cfs.metadata(), timestamp, 1, key)
.add("col3", ByteBufferUtil.EMPTY_BYTE_BUFFER)
.build()
.applyUnsafe();
cfs.forceBlockingFlush();
String noTTLKey = "nottl";
- new RowUpdateBuilder(cfs.metadata, timestamp, noTTLKey)
+ new RowUpdateBuilder(cfs.metadata(), timestamp, noTTLKey)
.add("col311", ByteBufferUtil.EMPTY_BYTE_BUFFER)
.build()
.applyUnsafe();
@@ -243,9 +244,8 @@
cfs.enableAutoCompaction(true);
assertEquals(1, cfs.getLiveSSTables().size());
SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
- ISSTableScanner scanner = sstable.getScanner(ColumnFilter.all(sstable.metadata),
+ ISSTableScanner scanner = sstable.getScanner(ColumnFilter.all(cfs.metadata()),
DataRange.allData(cfs.getPartitioner()),
- false,
SSTableReadsListener.NOOP_LISTENER);
assertTrue(scanner.hasNext());
while(scanner.hasNext())
@@ -262,9 +262,9 @@
ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore("Standard1");
cfs.truncateBlocking();
cfs.disableAutoCompaction();
- cfs.metadata.gcGraceSeconds(0);
+ MigrationManager.announceTableUpdate(cfs.metadata().unbuild().gcGraceSeconds(0).build(), true);
- new RowUpdateBuilder(cfs.metadata, System.currentTimeMillis(), "test")
+ new RowUpdateBuilder(cfs.metadata(), System.currentTimeMillis(), "test")
.noRowMarker()
.add("col1", ByteBufferUtil.EMPTY_BYTE_BUFFER)
.build()
@@ -274,7 +274,7 @@
SSTableReader blockingSSTable = cfs.getSSTables(SSTableSet.LIVE).iterator().next();
for (int i = 0; i < 10; i++)
{
- new RowUpdateBuilder(cfs.metadata, System.currentTimeMillis(), "test")
+ new RowUpdateBuilder(cfs.metadata(), System.currentTimeMillis(), "test")
.noRowMarker()
.delete("col1")
.build()
diff --git a/test/unit/org/apache/cassandra/db/compaction/TimeWindowCompactionStrategyTest.java b/test/unit/org/apache/cassandra/db/compaction/TimeWindowCompactionStrategyTest.java
index b67bf16..75e4998 100644
--- a/test/unit/org/apache/cassandra/db/compaction/TimeWindowCompactionStrategyTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/TimeWindowCompactionStrategyTest.java
@@ -164,7 +164,7 @@
for (int r = 0; r < 3; r++)
{
DecoratedKey key = Util.dk(String.valueOf(r));
- new RowUpdateBuilder(cfs.metadata, r, key.getKey())
+ new RowUpdateBuilder(cfs.metadata(), r, key.getKey())
.clustering("column")
.add("val", value).build().applyUnsafe();
@@ -175,7 +175,7 @@
{
// And add progressively more cells into each sstable
DecoratedKey key = Util.dk(String.valueOf(r));
- new RowUpdateBuilder(cfs.metadata, r, key.getKey())
+ new RowUpdateBuilder(cfs.metadata(), r, key.getKey())
.clustering("column")
.add("val", value).build().applyUnsafe();
cfs.forceBlockingFlush();
@@ -220,7 +220,7 @@
DecoratedKey key = Util.dk(String.valueOf(r));
for(int i = 0 ; i < r ; i++)
{
- new RowUpdateBuilder(cfs.metadata, tstamp + r, key.getKey())
+ new RowUpdateBuilder(cfs.metadata(), tstamp + r, key.getKey())
.clustering("column")
.add("val", value).build().applyUnsafe();
}
@@ -255,7 +255,7 @@
// create 2 sstables
DecoratedKey key = Util.dk(String.valueOf("expired"));
- new RowUpdateBuilder(cfs.metadata, System.currentTimeMillis(), 1, key.getKey())
+ new RowUpdateBuilder(cfs.metadata(), System.currentTimeMillis(), 1, key.getKey())
.clustering("column")
.add("val", value).build().applyUnsafe();
@@ -264,7 +264,7 @@
Thread.sleep(10);
key = Util.dk(String.valueOf("nonexpired"));
- new RowUpdateBuilder(cfs.metadata, System.currentTimeMillis(), key.getKey())
+ new RowUpdateBuilder(cfs.metadata(), System.currentTimeMillis(), key.getKey())
.clustering("column")
.add("val", value).build().applyUnsafe();
@@ -303,7 +303,7 @@
// create 2 sstables
DecoratedKey key = Util.dk(String.valueOf("expired"));
- new RowUpdateBuilder(cfs.metadata, System.currentTimeMillis(), 1, key.getKey())
+ new RowUpdateBuilder(cfs.metadata(), System.currentTimeMillis(), 1, key.getKey())
.clustering("column")
.add("val", value).build().applyUnsafe();
@@ -311,11 +311,11 @@
SSTableReader expiredSSTable = cfs.getLiveSSTables().iterator().next();
Thread.sleep(10);
- new RowUpdateBuilder(cfs.metadata, System.currentTimeMillis() - 1000, key.getKey())
+ new RowUpdateBuilder(cfs.metadata(), System.currentTimeMillis() - 1000, key.getKey())
.clustering("column")
.add("val", value).build().applyUnsafe();
key = Util.dk(String.valueOf("nonexpired"));
- new RowUpdateBuilder(cfs.metadata, System.currentTimeMillis(), key.getKey())
+ new RowUpdateBuilder(cfs.metadata(), System.currentTimeMillis(), key.getKey())
.clustering("column")
.add("val", value).build().applyUnsafe();
diff --git a/test/unit/org/apache/cassandra/db/compaction/ValidationExecutorTest.java b/test/unit/org/apache/cassandra/db/compaction/ValidationExecutorTest.java
new file mode 100644
index 0000000..a175bdd
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/compaction/ValidationExecutorTest.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.locks.Condition;
+
+import com.google.common.util.concurrent.Uninterruptibles;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.utils.concurrent.SimpleCondition;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.Assert.assertEquals;
+
+public class ValidationExecutorTest
+{
+
+ CompactionManager.ValidationExecutor validationExecutor;
+
+ @Before
+ public void setup()
+ {
+ DatabaseDescriptor.clientInitialization();
+ // required for static initialization of CompactionManager
+ DatabaseDescriptor.setConcurrentCompactors(2);
+ DatabaseDescriptor.setConcurrentValidations(2);
+
+ // shutdown the singleton CompactionManager to ensure MBeans are unregistered
+ CompactionManager.instance.forceShutdown();
+ }
+
+ @After
+ public void tearDown()
+ {
+ if (null != validationExecutor)
+ validationExecutor.shutdownNow();
+ }
+
+ @Test
+ public void testQueueOnValidationSubmission() throws InterruptedException
+ {
+ Condition taskBlocked = new SimpleCondition();
+ AtomicInteger threadsAvailable = new AtomicInteger(DatabaseDescriptor.getConcurrentValidations());
+ CountDownLatch taskComplete = new CountDownLatch(5);
+ validationExecutor = new CompactionManager.ValidationExecutor();
+
+ ExecutorService testExecutor = Executors.newSingleThreadExecutor();
+ for (int i=0; i< 5; i++)
+ testExecutor.submit(() -> {
+ threadsAvailable.decrementAndGet();
+ validationExecutor.submit(new Task(taskBlocked, taskComplete));
+ });
+
+ // wait for all tasks to be submitted & check that the excess ones were queued
+ while (threadsAvailable.get() > 0)
+ TimeUnit.MILLISECONDS.sleep(10);
+
+ assertEquals(2, validationExecutor.getActiveTaskCount());
+ assertEquals(3, validationExecutor.getPendingTaskCount());
+
+ taskBlocked.signalAll();
+ taskComplete.await(10, TimeUnit.SECONDS);
+ validationExecutor.shutdownNow();
+ }
+
+ @Test
+ public void testAdjustPoolSize()
+ {
+ // adjusting the pool size should dynamically set core and max pool
+ // size to DatabaseDescriptor::getConcurrentValidations
+
+ validationExecutor = new CompactionManager.ValidationExecutor();
+
+ int corePoolSize = validationExecutor.getCorePoolSize();
+ int maxPoolSize = validationExecutor.getMaximumPoolSize();
+
+ DatabaseDescriptor.setConcurrentValidations(corePoolSize * 2);
+ validationExecutor.adjustPoolSize();
+ assertThat(validationExecutor.getCorePoolSize()).isEqualTo(corePoolSize * 2);
+ assertThat(validationExecutor.getMaximumPoolSize()).isEqualTo(maxPoolSize * 2);
+ validationExecutor.shutdownNow();
+ }
+
+ private static class Task implements Runnable
+ {
+ private final Condition blocked;
+ private final CountDownLatch complete;
+
+ Task(Condition blocked, CountDownLatch complete)
+ {
+ this.blocked = blocked;
+ this.complete = complete;
+ }
+
+ public void run()
+ {
+ Uninterruptibles.awaitUninterruptibly(blocked, 10, TimeUnit.SECONDS);
+ complete.countDown();
+ }
+ }
+}
diff --git a/test/unit/org/apache/cassandra/db/composites/CTypeTest.java b/test/unit/org/apache/cassandra/db/composites/CTypeTest.java
index 9b261e6..999417e 100644
--- a/test/unit/org/apache/cassandra/db/composites/CTypeTest.java
+++ b/test/unit/org/apache/cassandra/db/composites/CTypeTest.java
@@ -30,42 +30,42 @@
{
CompositeType baseType = CompositeType.getInstance(AsciiType.instance, UUIDType.instance, LongType.instance);
- ByteBuffer a1 = baseType.builder()
- .add(ByteBufferUtil.bytes("a"))
- .add(UUIDType.instance.fromString("00000000-0000-0000-0000-000000000000"))
- .add(ByteBufferUtil.bytes(1)).build();
- ByteBuffer a2 = baseType.builder()
- .add(ByteBufferUtil.bytes("a"))
- .add(UUIDType.instance.fromString("00000000-0000-0000-0000-000000000000"))
- .add(ByteBufferUtil.bytes(100)).build();
- ByteBuffer b1 = baseType.builder()
- .add(ByteBufferUtil.bytes("a"))
- .add(UUIDType.instance.fromString("ffffffff-ffff-ffff-ffff-ffffffffffff"))
- .add(ByteBufferUtil.bytes(1)).build();
- ByteBuffer b2 = baseType.builder()
- .add(ByteBufferUtil.bytes("a"))
- .add(UUIDType.instance.fromString("ffffffff-ffff-ffff-ffff-ffffffffffff"))
- .add(ByteBufferUtil.bytes(100)).build();
- ByteBuffer c1 = baseType.builder()
- .add(ByteBufferUtil.bytes("z"))
- .add(UUIDType.instance.fromString("00000000-0000-0000-0000-000000000000"))
- .add(ByteBufferUtil.bytes(1)).build();
- ByteBuffer c2 = baseType.builder()
- .add(ByteBufferUtil.bytes("z"))
- .add(UUIDType.instance.fromString("00000000-0000-0000-0000-000000000000"))
- .add(ByteBufferUtil.bytes(100)).build();
- ByteBuffer d1 = baseType.builder()
- .add(ByteBufferUtil.bytes("z"))
- .add(UUIDType.instance.fromString("ffffffff-ffff-ffff-ffff-ffffffffffff"))
- .add(ByteBufferUtil.bytes(1)).build();
- ByteBuffer d2 = baseType.builder()
- .add(ByteBufferUtil.bytes("z"))
- .add(UUIDType.instance.fromString("ffffffff-ffff-ffff-ffff-ffffffffffff"))
- .add(ByteBufferUtil.bytes(100)).build();
- ByteBuffer z1 = baseType.builder()
- .add(ByteBufferUtil.EMPTY_BYTE_BUFFER)
- .add(UUIDType.instance.fromString("ffffffff-ffff-ffff-ffff-ffffffffffff"))
- .add(ByteBufferUtil.bytes(100)).build();
+ ByteBuffer a1 = CompositeType.build(
+ ByteBufferUtil.bytes("a"),
+ UUIDType.instance.fromString("00000000-0000-0000-0000-000000000000"),
+ ByteBufferUtil.bytes(1));
+ ByteBuffer a2 = CompositeType.build(
+ ByteBufferUtil.bytes("a"),
+ UUIDType.instance.fromString("00000000-0000-0000-0000-000000000000"),
+ ByteBufferUtil.bytes(100));
+ ByteBuffer b1 = CompositeType.build(
+ ByteBufferUtil.bytes("a"),
+ UUIDType.instance.fromString("ffffffff-ffff-ffff-ffff-ffffffffffff"),
+ ByteBufferUtil.bytes(1));
+ ByteBuffer b2 = CompositeType.build(
+ ByteBufferUtil.bytes("a"),
+ UUIDType.instance.fromString("ffffffff-ffff-ffff-ffff-ffffffffffff"),
+ ByteBufferUtil.bytes(100));
+ ByteBuffer c1 = CompositeType.build(
+ ByteBufferUtil.bytes("z"),
+ UUIDType.instance.fromString("00000000-0000-0000-0000-000000000000"),
+ ByteBufferUtil.bytes(1));
+ ByteBuffer c2 = CompositeType.build(
+ ByteBufferUtil.bytes("z"),
+ UUIDType.instance.fromString("00000000-0000-0000-0000-000000000000"),
+ ByteBufferUtil.bytes(100));
+ ByteBuffer d1 = CompositeType.build(
+ ByteBufferUtil.bytes("z"),
+ UUIDType.instance.fromString("ffffffff-ffff-ffff-ffff-ffffffffffff"),
+ ByteBufferUtil.bytes(1));
+ ByteBuffer d2 = CompositeType.build(
+ ByteBufferUtil.bytes("z"),
+ UUIDType.instance.fromString("ffffffff-ffff-ffff-ffff-ffffffffffff"),
+ ByteBufferUtil.bytes(100));
+ ByteBuffer z1 = CompositeType.build(
+ ByteBufferUtil.EMPTY_BYTE_BUFFER,
+ UUIDType.instance.fromString("ffffffff-ffff-ffff-ffff-ffffffffffff"),
+ ByteBufferUtil.bytes(100));
assert baseType.compare(a1,a2) < 0;
assert baseType.compare(a2,b1) < 0;
@@ -105,8 +105,8 @@
public void testSimpleType2()
{
CompositeType baseType = CompositeType.getInstance(UUIDType.instance);
- ByteBuffer a = baseType.builder().add(UUIDType.instance.fromString("00000000-0000-0000-0000-000000000000")).build();
- ByteBuffer z = baseType.builder().add(UUIDType.instance.fromString("ffffffff-ffff-ffff-ffff-ffffffffffff")).build();
+ ByteBuffer a = CompositeType.build(UUIDType.instance.fromString("00000000-0000-0000-0000-000000000000"));
+ ByteBuffer z = CompositeType.build(UUIDType.instance.fromString("ffffffff-ffff-ffff-ffff-ffffffffffff"));
assert baseType.compare(a,z) < 0;
assert baseType.compare(z,a) > 0;
@@ -118,8 +118,8 @@
public void testSimpleType1()
{
CompositeType baseType = CompositeType.getInstance(BytesType.instance);
- ByteBuffer a = baseType.builder().add(ByteBufferUtil.bytes("a")).build();
- ByteBuffer z = baseType.builder().add(ByteBufferUtil.bytes("z")).build();
+ ByteBuffer a = CompositeType.build(ByteBufferUtil.bytes("a"));
+ ByteBuffer z = CompositeType.build(ByteBufferUtil.bytes("z"));
assert baseType.compare(a,z) < 0;
assert baseType.compare(z,a) > 0;
diff --git a/test/unit/org/apache/cassandra/db/context/CounterContextTest.java b/test/unit/org/apache/cassandra/db/context/CounterContextTest.java
index 6994046..4437365 100644
--- a/test/unit/org/apache/cassandra/db/context/CounterContextTest.java
+++ b/test/unit/org/apache/cassandra/db/context/CounterContextTest.java
@@ -28,7 +28,7 @@
import org.apache.cassandra.Util;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.ClockAndCount;
-import org.apache.cassandra.db.LegacyLayout.LegacyCell;
+import org.apache.cassandra.db.commitlog.CommitLog;
import org.apache.cassandra.db.context.CounterContext.Relationship;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.CounterId;
@@ -56,6 +56,7 @@
public static void setupDD()
{
DatabaseDescriptor.daemonInitialization();
+ CommitLog.instance.start();
}
@Test
@@ -556,8 +557,6 @@
assertEquals(ClockAndCount.create(1L, 10L), cc.getClockAndCountOf(updateContext, CounterContext.UPDATE_CLOCK_ID));
assertTrue(cc.isUpdate(updateContext));
- LegacyCell updateCell = LegacyCell.counter(null, updateContext);
- assertTrue(updateCell.isCounterUpdate());
/*
@@ -571,7 +570,5 @@
ByteBuffer notUpdateContext = notUpdateContextState.context;
assertFalse(cc.isUpdate(notUpdateContext));
- LegacyCell notUpdateCell = LegacyCell.counter(null, notUpdateContext);
- assertFalse(notUpdateCell.isCounterUpdate());
}
}
diff --git a/test/unit/org/apache/cassandra/db/filter/ColumnFilterTest.java b/test/unit/org/apache/cassandra/db/filter/ColumnFilterTest.java
index db06d20..42f6957 100644
--- a/test/unit/org/apache/cassandra/db/filter/ColumnFilterTest.java
+++ b/test/unit/org/apache/cassandra/db/filter/ColumnFilterTest.java
@@ -18,53 +18,193 @@
package org.apache.cassandra.db.filter;
-import org.junit.Test;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
-import junit.framework.Assert;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.db.RegularAndStaticColumns;
import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.marshal.SetType;
+import org.apache.cassandra.db.rows.CellPath;
import org.apache.cassandra.dht.Murmur3Partitioner;
import org.apache.cassandra.io.util.DataInputBuffer;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputBuffer;
import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.utils.ByteBufferUtil;
+import org.junit.Test;
+
+import org.junit.Assert;
public class ColumnFilterTest
{
final static ColumnFilter.Serializer serializer = new ColumnFilter.Serializer();
@Test
- public void columnFilterSerialisationRoundTrip() throws Exception
+ public void testColumnFilterSerialisationRoundTrip() throws Exception
{
- CFMetaData metadata = CFMetaData.Builder.create("ks", "table")
- .withPartitioner(Murmur3Partitioner.instance)
- .addPartitionKey("pk", Int32Type.instance)
- .addClusteringColumn("ck", Int32Type.instance)
- .addRegularColumn("v1", Int32Type.instance)
- .addRegularColumn("v2", Int32Type.instance)
- .addRegularColumn("v3", Int32Type.instance)
- .build();
+ TableMetadata metadata = TableMetadata.builder("ks", "table")
+ .partitioner(Murmur3Partitioner.instance)
+ .addPartitionKeyColumn("pk", Int32Type.instance)
+ .addClusteringColumn("ck", Int32Type.instance)
+ .addRegularColumn("v1", Int32Type.instance)
+ .addRegularColumn("v2", Int32Type.instance)
+ .addRegularColumn("v3", Int32Type.instance)
+ .build();
- ColumnDefinition v1 = metadata.getColumnDefinition(ByteBufferUtil.bytes("v1"));
+ ColumnMetadata v1 = metadata.getColumn(ByteBufferUtil.bytes("v1"));
- testRoundTrip(ColumnFilter.all(metadata), metadata, MessagingService.VERSION_30);
- testRoundTrip(ColumnFilter.all(metadata), metadata, MessagingService.VERSION_3014);
+ ColumnFilter columnFilter;
- testRoundTrip(ColumnFilter.selection(metadata.partitionColumns().without(v1)), metadata, MessagingService.VERSION_30);
- testRoundTrip(ColumnFilter.selection(metadata.partitionColumns().without(v1)), metadata, MessagingService.VERSION_3014);
+ columnFilter = ColumnFilter.all(metadata);
+ testRoundTrip(columnFilter, ColumnFilter.Serializer.maybeUpdateForBackwardCompatility(columnFilter, MessagingService.VERSION_30), metadata, MessagingService.VERSION_30);
+ testRoundTrip(columnFilter, ColumnFilter.Serializer.maybeUpdateForBackwardCompatility(columnFilter, MessagingService.VERSION_3014), metadata, MessagingService.VERSION_3014);
+ testRoundTrip(columnFilter, metadata, MessagingService.VERSION_40);
- testRoundTrip(ColumnFilter.selection(metadata, metadata.partitionColumns().without(v1)), metadata, MessagingService.VERSION_30);
- testRoundTrip(ColumnFilter.selection(metadata, metadata.partitionColumns().without(v1)), metadata, MessagingService.VERSION_3014);
+ testRoundTrip(ColumnFilter.selection(metadata.regularAndStaticColumns().without(v1)), metadata, MessagingService.VERSION_30);
+ testRoundTrip(ColumnFilter.selection(metadata.regularAndStaticColumns().without(v1)), metadata, MessagingService.VERSION_3014);
+ testRoundTrip(ColumnFilter.selection(metadata.regularAndStaticColumns().without(v1)), metadata, MessagingService.VERSION_40);
+
+ columnFilter = ColumnFilter.selection(metadata, metadata.regularAndStaticColumns().without(v1));
+ testRoundTrip(columnFilter, ColumnFilter.Serializer.maybeUpdateForBackwardCompatility(columnFilter, MessagingService.VERSION_30), metadata, MessagingService.VERSION_30);
+ testRoundTrip(columnFilter, ColumnFilter.Serializer.maybeUpdateForBackwardCompatility(columnFilter, MessagingService.VERSION_3014), metadata, MessagingService.VERSION_3014);
+ testRoundTrip(columnFilter, metadata, MessagingService.VERSION_40);
+
+ // Table with static column
+ metadata = TableMetadata.builder("ks", "table")
+ .partitioner(Murmur3Partitioner.instance)
+ .addPartitionKeyColumn("pk", Int32Type.instance)
+ .addClusteringColumn("ck", Int32Type.instance)
+ .addStaticColumn("s1", Int32Type.instance)
+ .addRegularColumn("v1", Int32Type.instance)
+ .addRegularColumn("v2", Int32Type.instance)
+ .addRegularColumn("v3", Int32Type.instance)
+ .build();
+
+ v1 = metadata.getColumn(ByteBufferUtil.bytes("v1"));
+ ColumnMetadata s1 = metadata.getColumn(ByteBufferUtil.bytes("s1"));
+
+ columnFilter = ColumnFilter.all(metadata);
+ testRoundTrip(columnFilter, ColumnFilter.Serializer.maybeUpdateForBackwardCompatility(columnFilter, MessagingService.VERSION_30), metadata, MessagingService.VERSION_30);
+ testRoundTrip(columnFilter, ColumnFilter.Serializer.maybeUpdateForBackwardCompatility(columnFilter, MessagingService.VERSION_3014), metadata, MessagingService.VERSION_3014);
+ testRoundTrip(columnFilter, metadata, MessagingService.VERSION_40);
+
+ testRoundTrip(ColumnFilter.selection(metadata.regularAndStaticColumns().without(v1)), metadata, MessagingService.VERSION_30);
+ testRoundTrip(ColumnFilter.selection(metadata.regularAndStaticColumns().without(v1)), metadata, MessagingService.VERSION_3014);
+ testRoundTrip(ColumnFilter.selection(metadata.regularAndStaticColumns().without(v1)), metadata, MessagingService.VERSION_40);
+
+ testRoundTrip(ColumnFilter.selection(metadata.regularAndStaticColumns().without(v1).without(s1)), metadata, MessagingService.VERSION_30);
+ testRoundTrip(ColumnFilter.selection(metadata.regularAndStaticColumns().without(v1).without(s1)), metadata, MessagingService.VERSION_3014);
+ testRoundTrip(ColumnFilter.selection(metadata.regularAndStaticColumns().without(v1).without(s1)), metadata, MessagingService.VERSION_40);
+
+ columnFilter = ColumnFilter.selection(metadata, metadata.regularAndStaticColumns().without(v1));
+ testRoundTrip(columnFilter, ColumnFilter.Serializer.maybeUpdateForBackwardCompatility(columnFilter, MessagingService.VERSION_30), metadata, MessagingService.VERSION_30);
+ testRoundTrip(columnFilter, ColumnFilter.Serializer.maybeUpdateForBackwardCompatility(columnFilter, MessagingService.VERSION_3014), metadata, MessagingService.VERSION_3014);
+ testRoundTrip(columnFilter, metadata, MessagingService.VERSION_40);
+
+ columnFilter = ColumnFilter.selection(metadata, metadata.regularAndStaticColumns().without(v1).without(s1));
+ testRoundTrip(columnFilter, ColumnFilter.Serializer.maybeUpdateForBackwardCompatility(columnFilter, MessagingService.VERSION_30), metadata, MessagingService.VERSION_30);
+ testRoundTrip(columnFilter, ColumnFilter.Serializer.maybeUpdateForBackwardCompatility(columnFilter, MessagingService.VERSION_3014), metadata, MessagingService.VERSION_3014);
+ testRoundTrip(columnFilter, metadata, MessagingService.VERSION_40);
}
- static void testRoundTrip(ColumnFilter columnFilter, CFMetaData metadata, int version) throws Exception
+ @Test
+ public void testColumnFilterConstruction()
+ {
+ // all regular column
+ TableMetadata metadata = TableMetadata.builder("ks", "table")
+ .partitioner(Murmur3Partitioner.instance)
+ .addPartitionKeyColumn("pk", Int32Type.instance)
+ .addClusteringColumn("ck", Int32Type.instance)
+ .addRegularColumn("v1", Int32Type.instance)
+ .addRegularColumn("v2", Int32Type.instance)
+ .addRegularColumn("v3", Int32Type.instance)
+ .build();
+ ColumnFilter columnFilter = ColumnFilter.all(metadata);
+ assertTrue(columnFilter.fetchAllRegulars);
+ assertEquals(metadata.regularAndStaticColumns(), columnFilter.fetched);
+ assertNull(columnFilter.queried);
+ assertEquals("*", columnFilter.toString());
+
+ RegularAndStaticColumns queried = RegularAndStaticColumns.builder()
+ .add(metadata.getColumn(ByteBufferUtil.bytes("v1"))).build();
+ columnFilter = ColumnFilter.selection(queried);
+ assertFalse(columnFilter.fetchAllRegulars);
+ assertEquals(queried, columnFilter.fetched);
+ assertEquals(queried, columnFilter.queried);
+ assertEquals("v1", columnFilter.toString());
+
+ // with static column
+ metadata = TableMetadata.builder("ks", "table")
+ .partitioner(Murmur3Partitioner.instance)
+ .addPartitionKeyColumn("pk", Int32Type.instance)
+ .addClusteringColumn("ck", Int32Type.instance)
+ .addStaticColumn("sc1", Int32Type.instance)
+ .addStaticColumn("sc2", Int32Type.instance)
+ .addRegularColumn("v1", Int32Type.instance)
+ .build();
+
+ columnFilter = ColumnFilter.all(metadata);
+ assertTrue(columnFilter.fetchAllRegulars);
+ assertEquals(metadata.regularAndStaticColumns(), columnFilter.fetched);
+ assertNull(columnFilter.queried);
+ assertEquals("*", columnFilter.toString());
+
+ queried = RegularAndStaticColumns.builder()
+ .add(metadata.getColumn(ByteBufferUtil.bytes("v1"))).build();
+ columnFilter = ColumnFilter.selection(metadata, queried);
+ assertEquals("v1", columnFilter.toString());
+
+ // only static
+ metadata = TableMetadata.builder("ks", "table")
+ .partitioner(Murmur3Partitioner.instance)
+ .addPartitionKeyColumn("pk", Int32Type.instance)
+ .addClusteringColumn("ck", Int32Type.instance)
+ .addStaticColumn("sc", Int32Type.instance)
+ .build();
+
+ columnFilter = ColumnFilter.all(metadata);
+ assertTrue(columnFilter.fetchAllRegulars);
+ assertEquals(metadata.regularAndStaticColumns(), columnFilter.fetched);
+ assertNull(columnFilter.queried);
+ assertEquals("*", columnFilter.toString());
+
+ // with collection type
+ metadata = TableMetadata.builder("ks", "table")
+ .partitioner(Murmur3Partitioner.instance)
+ .addPartitionKeyColumn("pk", Int32Type.instance)
+ .addClusteringColumn("ck", Int32Type.instance)
+ .addRegularColumn("v1", Int32Type.instance)
+ .addRegularColumn("set", SetType.getInstance(Int32Type.instance, true))
+ .build();
+
+ columnFilter = ColumnFilter.all(metadata);
+ assertTrue(columnFilter.fetchAllRegulars);
+ assertEquals(metadata.regularAndStaticColumns(), columnFilter.fetched);
+ assertNull(columnFilter.queried);
+ assertEquals("*", columnFilter.toString());
+
+ columnFilter = ColumnFilter.selectionBuilder().add(metadata.getColumn(ByteBufferUtil.bytes("v1")))
+ .select(metadata.getColumn(ByteBufferUtil.bytes("set")), CellPath.create(ByteBufferUtil.bytes(1)))
+ .build();
+ assertEquals("set[1], v1", columnFilter.toString());
+ }
+
+ static void testRoundTrip(ColumnFilter columnFilter, TableMetadata metadata, int version) throws Exception
+ {
+ testRoundTrip(columnFilter, columnFilter, metadata, version);
+ }
+
+ static void testRoundTrip(ColumnFilter columnFilter, ColumnFilter expected, TableMetadata metadata, int version) throws Exception
{
DataOutputBuffer output = new DataOutputBuffer();
serializer.serialize(columnFilter, output, version);
Assert.assertEquals(serializer.serializedSize(columnFilter, version), output.position());
DataInputPlus input = new DataInputBuffer(output.buffer(), false);
- Assert.assertEquals(serializer.deserialize(input, version, metadata), columnFilter);
+ ColumnFilter deserialized = serializer.deserialize(input, version, metadata);
+ Assert.assertEquals(deserialized, expected);
}
}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/db/filter/RowFilterTest.java b/test/unit/org/apache/cassandra/db/filter/RowFilterTest.java
index 9313c3a..333d3f8 100644
--- a/test/unit/org/apache/cassandra/db/filter/RowFilterTest.java
+++ b/test/unit/org/apache/cassandra/db/filter/RowFilterTest.java
@@ -25,16 +25,13 @@
import org.junit.Assert;
import org.junit.Test;
-import org.apache.cassandra.SchemaLoader;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
import org.apache.cassandra.cql3.ColumnIdentifier;
import org.apache.cassandra.cql3.Operator;
import org.apache.cassandra.db.Clustering;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.DeletionTime;
import org.apache.cassandra.db.LivenessInfo;
-import org.apache.cassandra.db.PartitionColumns;
+import org.apache.cassandra.db.RegularAndStaticColumns;
import org.apache.cassandra.db.marshal.Int32Type;
import org.apache.cassandra.db.partitions.SingletonUnfilteredPartitionIterator;
import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
@@ -46,6 +43,8 @@
import org.apache.cassandra.db.rows.Rows;
import org.apache.cassandra.db.rows.Unfiltered;
import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.utils.btree.BTree;
public class RowFilterTest
@@ -55,14 +54,13 @@
public void testCQLFilterClose()
{
// CASSANDRA-15126
- SchemaLoader.prepareServer();
- CFMetaData metadata = CFMetaData.Builder.create("testks", "testcf")
- .addPartitionKey("pk", Int32Type.instance)
- .addStaticColumn("s", Int32Type.instance)
- .addRegularColumn("r", Int32Type.instance)
- .build();
- ColumnDefinition s = metadata.getColumnDefinition(new ColumnIdentifier("s", true));
- ColumnDefinition r = metadata.getColumnDefinition(new ColumnIdentifier("r", true));
+ TableMetadata metadata = TableMetadata.builder("testks", "testcf")
+ .addPartitionKeyColumn("pk", Int32Type.instance)
+ .addStaticColumn("s", Int32Type.instance)
+ .addRegularColumn("r", Int32Type.instance)
+ .build();
+ ColumnMetadata s = metadata.getColumn(new ColumnIdentifier("s", true));
+ ColumnMetadata r = metadata.getColumn(new ColumnIdentifier("r", true));
ByteBuffer one = Int32Type.instance.decompose(1);
RowFilter filter = RowFilter.NONE.withNewExpressions(new ArrayList<>());
@@ -72,9 +70,9 @@
{
public DeletionTime partitionLevelDeletion() { return null; }
public EncodingStats stats() { return null; }
- public CFMetaData metadata() { return metadata; }
+ public TableMetadata metadata() { return metadata; }
public boolean isReverseOrder() { return false; }
- public PartitionColumns columns() { return null; }
+ public RegularAndStaticColumns columns() { return null; }
public DecoratedKey partitionKey() { return null; }
public boolean hasNext() { return false; }
public Unfiltered next() { return null; }
@@ -89,7 +87,7 @@
{
closed.set(true);
}
- }, false), 1);
+ }), 1);
Assert.assertFalse(iter.hasNext());
Assert.assertTrue(closed.get());
@@ -101,9 +99,9 @@
boolean hasNext = true;
public DeletionTime partitionLevelDeletion() { return null; }
public EncodingStats stats() { return null; }
- public CFMetaData metadata() { return metadata; }
+ public TableMetadata metadata() { return metadata; }
public boolean isReverseOrder() { return false; }
- public PartitionColumns columns() { return null; }
+ public RegularAndStaticColumns columns() { return null; }
public DecoratedKey partitionKey() { return null; }
public Row staticRow() { return Rows.EMPTY_STATIC_ROW; }
public boolean hasNext()
@@ -123,7 +121,7 @@
{
closed.set(true);
}
- }, false), 1);
+ }), 1);
Assert.assertFalse(iter.hasNext());
Assert.assertTrue(closed.get());
}
diff --git a/test/unit/org/apache/cassandra/db/filter/SliceTest.java b/test/unit/org/apache/cassandra/db/filter/SliceTest.java
index 9188c94..6c04500 100644
--- a/test/unit/org/apache/cassandra/db/filter/SliceTest.java
+++ b/test/unit/org/apache/cassandra/db/filter/SliceTest.java
@@ -30,6 +30,7 @@
import org.apache.cassandra.db.marshal.Int32Type;
import org.apache.cassandra.utils.ByteBufferUtil;
+import static org.apache.cassandra.db.ClusteringPrefix.Kind.*;
import static org.junit.Assert.*;
public class SliceTest
@@ -43,8 +44,8 @@
types.add(Int32Type.instance);
ClusteringComparator cc = new ClusteringComparator(types);
- ClusteringPrefix.Kind sk = ClusteringPrefix.Kind.INCL_START_BOUND;
- ClusteringPrefix.Kind ek = ClusteringPrefix.Kind.INCL_END_BOUND;
+ ClusteringPrefix.Kind sk = INCL_START_BOUND;
+ ClusteringPrefix.Kind ek = INCL_END_BOUND;
// filter falls entirely before sstable
Slice slice = Slice.make(makeBound(sk, 0, 0, 0), makeBound(ek, 1, 0, 0));
@@ -274,8 +275,8 @@
types.add(Int32Type.instance);
ClusteringComparator cc = new ClusteringComparator(types);
- ClusteringPrefix.Kind sk = ClusteringPrefix.Kind.INCL_START_BOUND;
- ClusteringPrefix.Kind ek = ClusteringPrefix.Kind.INCL_END_BOUND;
+ ClusteringPrefix.Kind sk = INCL_START_BOUND;
+ ClusteringPrefix.Kind ek = INCL_END_BOUND;
// slice does intersect
Slice slice = Slice.make(makeBound(sk), makeBound(ek));
@@ -323,6 +324,26 @@
assertSlicesNormalization(cc, slices(s(-1, 2), s(-1, 3), s(5, 9)), slices(s(-1, 3), s(5, 9)));
}
+ @Test
+ public void testIsEmpty()
+ {
+ List<AbstractType<?>> types = new ArrayList<>();
+ types.add(Int32Type.instance);
+ types.add(Int32Type.instance);
+ ClusteringComparator cc = new ClusteringComparator(types);
+
+ assertFalse(Slice.isEmpty(cc, makeBound(INCL_START_BOUND, 5, 0), makeBound(INCL_END_BOUND, 5, 0)));
+ assertFalse(Slice.isEmpty(cc, makeBound(INCL_START_BOUND, 5, 0), makeBound(EXCL_END_BOUND, 5, 1)));
+ assertFalse(Slice.isEmpty(cc, makeBound(INCL_START_BOUND, 5), makeBound(EXCL_END_BOUND, 5, 1)));
+
+ assertTrue(Slice.isEmpty(cc, makeBound(EXCL_START_BOUND, 5), makeBound(EXCL_END_BOUND, 5)));
+ assertTrue(Slice.isEmpty(cc, makeBound(EXCL_START_BOUND, 5), makeBound(EXCL_END_BOUND, 5, 1)));
+ assertTrue(Slice.isEmpty(cc, makeBound(EXCL_START_BOUND, 5, 1), makeBound(EXCL_END_BOUND, 5, 1)));
+ assertTrue(Slice.isEmpty(cc, makeBound(INCL_START_BOUND, 5, 0), makeBound(INCL_END_BOUND, 4, 0)));
+ assertTrue(Slice.isEmpty(cc, makeBound(INCL_START_BOUND, 5, 0), makeBound(EXCL_END_BOUND, 5)));
+ assertTrue(Slice.isEmpty(cc, makeBound(INCL_START_BOUND, 5, 0), makeBound(EXCL_END_BOUND, 3, 0)));
+ }
+
private static ClusteringBound makeBound(ClusteringPrefix.Kind kind, Integer... components)
{
ByteBuffer[] values = new ByteBuffer[components.length];
@@ -343,8 +364,8 @@
private static Slice s(int start, int finish)
{
- return Slice.make(makeBound(ClusteringPrefix.Kind.INCL_START_BOUND, start),
- makeBound(ClusteringPrefix.Kind.INCL_END_BOUND, finish));
+ return Slice.make(makeBound(INCL_START_BOUND, start),
+ makeBound(INCL_END_BOUND, finish));
}
private Slice[] slices(Slice... slices)
diff --git a/test/unit/org/apache/cassandra/db/lifecycle/HelpersTest.java b/test/unit/org/apache/cassandra/db/lifecycle/HelpersTest.java
index 6aa7bc4..7acd3e6 100644
--- a/test/unit/org/apache/cassandra/db/lifecycle/HelpersTest.java
+++ b/test/unit/org/apache/cassandra/db/lifecycle/HelpersTest.java
@@ -30,12 +30,13 @@
import org.junit.BeforeClass;
import org.junit.Test;
-import junit.framework.Assert;
-import org.apache.cassandra.MockSchema;
+import org.junit.Assert;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.commitlog.CommitLog;
import org.apache.cassandra.db.compaction.OperationType;
import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.schema.MockSchema;
import static junit.framework.Assert.assertEquals;
import static junit.framework.Assert.assertNotNull;
@@ -48,6 +49,7 @@
public static void setUp()
{
DatabaseDescriptor.daemonInitialization();
+ CommitLog.instance.start();
MockSchema.cleanup();
}
diff --git a/test/unit/org/apache/cassandra/db/lifecycle/LifecycleTransactionTest.java b/test/unit/org/apache/cassandra/db/lifecycle/LifecycleTransactionTest.java
index 4514b72..1e0d157 100644
--- a/test/unit/org/apache/cassandra/db/lifecycle/LifecycleTransactionTest.java
+++ b/test/unit/org/apache/cassandra/db/lifecycle/LifecycleTransactionTest.java
@@ -27,16 +27,16 @@
import org.junit.BeforeClass;
import org.junit.Test;
-import junit.framework.Assert;
-import org.apache.cassandra.MockSchema;
+import org.junit.Assert;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Memtable;
import org.apache.cassandra.db.commitlog.CommitLogPosition;
import org.apache.cassandra.db.compaction.OperationType;
-import org.apache.cassandra.db.lifecycle.LifecycleTransaction.ReaderState;
import org.apache.cassandra.db.lifecycle.LifecycleTransaction.ReaderState.Action;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction.ReaderState;
import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.schema.MockSchema;
import org.apache.cassandra.utils.Pair;
import org.apache.cassandra.utils.concurrent.AbstractTransactionalTest;
import org.apache.cassandra.utils.concurrent.Transactional.AbstractTransactional.State;
diff --git a/test/unit/org/apache/cassandra/db/lifecycle/LogTransactionTest.java b/test/unit/org/apache/cassandra/db/lifecycle/LogTransactionTest.java
index 35f4894..e5ff138 100644
--- a/test/unit/org/apache/cassandra/db/lifecycle/LogTransactionTest.java
+++ b/test/unit/org/apache/cassandra/db/lifecycle/LogTransactionTest.java
@@ -32,15 +32,8 @@
import com.google.common.collect.Sets;
import org.junit.BeforeClass;
import org.junit.Test;
+import org.junit.Assert;
-import static junit.framework.Assert.assertNotNull;
-import static junit.framework.Assert.assertNull;
-import static junit.framework.Assert.fail;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-import junit.framework.Assert;
-import org.apache.cassandra.MockSchema;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Directories;
import org.apache.cassandra.db.SerializationHeader;
@@ -51,12 +44,20 @@
import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
import org.apache.cassandra.io.sstable.metadata.MetadataType;
import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
-import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.io.util.FileHandle;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.schema.MockSchema;
import org.apache.cassandra.utils.AlwaysPresentFilter;
import org.apache.cassandra.utils.concurrent.AbstractTransactionalTest;
import org.apache.cassandra.utils.concurrent.Transactional;
+import static junit.framework.Assert.assertNotNull;
+import static junit.framework.Assert.assertNull;
+import static junit.framework.Assert.fail;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
public class LogTransactionTest extends AbstractTransactionalTest
{
private static final String KEYSPACE = "TransactionLogsTest";
@@ -89,7 +90,7 @@
{
this.cfs = cfs;
this.txnLogs = txnLogs;
- this.dataFolder = new Directories(cfs.metadata).getDirectoryForNewSSTables();
+ this.dataFolder = new Directories(cfs.metadata()).getDirectoryForNewSSTables();
this.sstableOld = sstable(dataFolder, cfs, 0, 128);
this.sstableNew = sstable(dataFolder, cfs, 1, 128);
@@ -201,7 +202,7 @@
public void testUntrack() throws Throwable
{
ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
- File dataFolder = new Directories(cfs.metadata).getDirectoryForNewSSTables();
+ File dataFolder = new Directories(cfs.metadata()).getDirectoryForNewSSTables();
SSTableReader sstableNew = sstable(dataFolder, cfs, 1, 128);
// complete a transaction without keep the new files since they were untracked
@@ -224,7 +225,7 @@
public void testCommitSameDesc() throws Throwable
{
ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
- File dataFolder = new Directories(cfs.metadata).getDirectoryForNewSSTables();
+ File dataFolder = new Directories(cfs.metadata()).getDirectoryForNewSSTables();
SSTableReader sstableOld1 = sstable(dataFolder, cfs, 0, 128);
SSTableReader sstableOld2 = sstable(dataFolder, cfs, 0, 256);
SSTableReader sstableNew = sstable(dataFolder, cfs, 1, 128);
@@ -255,7 +256,7 @@
public void testCommitOnlyNew() throws Throwable
{
ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
- File dataFolder = new Directories(cfs.metadata).getDirectoryForNewSSTables();
+ File dataFolder = new Directories(cfs.metadata()).getDirectoryForNewSSTables();
SSTableReader sstable = sstable(dataFolder, cfs, 0, 128);
LogTransaction log = new LogTransaction(OperationType.COMPACTION);
@@ -273,7 +274,7 @@
public void testCommitOnlyOld() throws Throwable
{
ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
- File dataFolder = new Directories(cfs.metadata).getDirectoryForNewSSTables();
+ File dataFolder = new Directories(cfs.metadata()).getDirectoryForNewSSTables();
SSTableReader sstable = sstable(dataFolder, cfs, 0, 128);
LogTransaction log = new LogTransaction(OperationType.COMPACTION);
@@ -294,7 +295,7 @@
{
ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
- File origiFolder = new Directories(cfs.metadata).getDirectoryForNewSSTables();
+ File origiFolder = new Directories(cfs.metadata()).getDirectoryForNewSSTables();
File dataFolder1 = new File(origiFolder, "1");
File dataFolder2 = new File(origiFolder, "2");
Files.createDirectories(dataFolder1.toPath());
@@ -330,7 +331,7 @@
public void testAbortOnlyNew() throws Throwable
{
ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
- File dataFolder = new Directories(cfs.metadata).getDirectoryForNewSSTables();
+ File dataFolder = new Directories(cfs.metadata()).getDirectoryForNewSSTables();
SSTableReader sstable = sstable(dataFolder, cfs, 0, 128);
LogTransaction log = new LogTransaction(OperationType.COMPACTION);
@@ -348,7 +349,7 @@
public void testAbortOnlyOld() throws Throwable
{
ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
- File dataFolder = new Directories(cfs.metadata).getDirectoryForNewSSTables();
+ File dataFolder = new Directories(cfs.metadata()).getDirectoryForNewSSTables();
SSTableReader sstable = sstable(dataFolder, cfs, 0, 128);
LogTransaction log = new LogTransaction(OperationType.COMPACTION);
@@ -370,7 +371,7 @@
{
ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
- File origiFolder = new Directories(cfs.metadata).getDirectoryForNewSSTables();
+ File origiFolder = new Directories(cfs.metadata()).getDirectoryForNewSSTables();
File dataFolder1 = new File(origiFolder, "1");
File dataFolder2 = new File(origiFolder, "2");
Files.createDirectories(dataFolder1.toPath());
@@ -405,7 +406,7 @@
public void testRemoveUnfinishedLeftovers_abort() throws Throwable
{
ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
- File dataFolder = new Directories(cfs.metadata).getDirectoryForNewSSTables();
+ File dataFolder = new Directories(cfs.metadata()).getDirectoryForNewSSTables();
SSTableReader sstableOld = sstable(dataFolder, cfs, 0, 128);
SSTableReader sstableNew = sstable(dataFolder, cfs, 1, 128);
@@ -424,10 +425,10 @@
Assert.assertEquals(tmpFiles, getTemporaryFiles(sstableNew.descriptor.directory));
// normally called at startup
- LogTransaction.removeUnfinishedLeftovers(cfs.metadata);
+ LogTransaction.removeUnfinishedLeftovers(cfs.metadata());
// sstableOld should be only table left
- Directories directories = new Directories(cfs.metadata);
+ Directories directories = new Directories(cfs.metadata());
Map<Descriptor, Set<Component>> sstables = directories.sstableLister(Directories.OnTxnErr.THROW).list();
assertEquals(1, sstables.size());
@@ -442,7 +443,7 @@
public void testRemoveUnfinishedLeftovers_commit() throws Throwable
{
ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
- File dataFolder = new Directories(cfs.metadata).getDirectoryForNewSSTables();
+ File dataFolder = new Directories(cfs.metadata()).getDirectoryForNewSSTables();
SSTableReader sstableOld = sstable(dataFolder, cfs, 0, 128);
SSTableReader sstableNew = sstable(dataFolder, cfs, 1, 128);
@@ -464,10 +465,10 @@
Assert.assertEquals(tmpFiles, getTemporaryFiles(sstableOld.descriptor.directory));
// normally called at startup
- LogTransaction.removeUnfinishedLeftovers(cfs.metadata);
+ LogTransaction.removeUnfinishedLeftovers(cfs.metadata());
// sstableNew should be only table left
- Directories directories = new Directories(cfs.metadata);
+ Directories directories = new Directories(cfs.metadata());
Map<Descriptor, Set<Component>> sstables = directories.sstableLister(Directories.OnTxnErr.THROW).list();
assertEquals(1, sstables.size());
@@ -483,7 +484,7 @@
{
ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
- File origiFolder = new Directories(cfs.metadata).getDirectoryForNewSSTables();
+ File origiFolder = new Directories(cfs.metadata()).getDirectoryForNewSSTables();
File dataFolder1 = new File(origiFolder, "1");
File dataFolder2 = new File(origiFolder, "2");
Files.createDirectories(dataFolder1.toPath());
@@ -534,7 +535,7 @@
{
ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
- File origiFolder = new Directories(cfs.metadata).getDirectoryForNewSSTables();
+ File origiFolder = new Directories(cfs.metadata()).getDirectoryForNewSSTables();
File dataFolder1 = new File(origiFolder, "1");
File dataFolder2 = new File(origiFolder, "2");
Files.createDirectories(dataFolder1.toPath());
@@ -634,7 +635,7 @@
Assert.assertEquals(2, logFiles.size());
// insert a partial sstable record and a full commit record
- String sstableRecord = LogRecord.make(LogRecord.Type.ADD, Collections.emptyList(), 0, "abc-").raw;
+ String sstableRecord = LogRecord.make(LogRecord.Type.ADD, Collections.emptyList(), 0, "abc").raw;
int toChop = sstableRecord.length() / 2;
FileUtils.append(logFiles.get(0), sstableRecord.substring(0, sstableRecord.length() - toChop));
FileUtils.append(logFiles.get(1), sstableRecord);
@@ -653,7 +654,7 @@
Assert.assertEquals(2, logFiles.size());
// insert a partial sstable record and a full commit record
- String sstableRecord = LogRecord.make(LogRecord.Type.ADD, Collections.emptyList(), 0, "abc-").raw;
+ String sstableRecord = LogRecord.make(LogRecord.Type.ADD, Collections.emptyList(), 0, "abc").raw;
int toChop = sstableRecord.length() / 2;
FileUtils.append(logFiles.get(0), sstableRecord);
FileUtils.append(logFiles.get(1), sstableRecord.substring(0, sstableRecord.length() - toChop));
@@ -709,7 +710,7 @@
{
ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
- File origiFolder = new Directories(cfs.metadata).getDirectoryForNewSSTables();
+ File origiFolder = new Directories(cfs.metadata()).getDirectoryForNewSSTables();
File dataFolder1 = new File(origiFolder, "1");
File dataFolder2 = new File(origiFolder, "2");
Files.createDirectories(dataFolder1.toPath());
@@ -766,7 +767,7 @@
public void testGetTemporaryFiles() throws IOException
{
ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
- File dataFolder = new Directories(cfs.metadata).getDirectoryForNewSSTables();
+ File dataFolder = new Directories(cfs.metadata()).getDirectoryForNewSSTables();
SSTableReader sstable1 = sstable(dataFolder, cfs, 0, 128);
Set<File> tmpFiles = getTemporaryFiles(dataFolder);
@@ -775,7 +776,7 @@
try(LogTransaction log = new LogTransaction(OperationType.WRITE))
{
- Directories directories = new Directories(cfs.metadata);
+ Directories directories = new Directories(cfs.metadata());
File[] beforeSecondSSTable = dataFolder.listFiles(pathname -> !pathname.isDirectory());
@@ -834,7 +835,7 @@
{
ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
- File origiFolder = new Directories(cfs.metadata).getDirectoryForNewSSTables();
+ File origiFolder = new Directories(cfs.metadata()).getDirectoryForNewSSTables();
File dataFolder1 = new File(origiFolder, "1");
File dataFolder2 = new File(origiFolder, "2");
Files.createDirectories(dataFolder1.toPath());
@@ -993,7 +994,7 @@
private static void testCorruptRecord(BiConsumer<LogTransaction, SSTableReader> modifier, boolean isRecoverable) throws IOException
{
ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
- File dataFolder = new Directories(cfs.metadata).getDirectoryForNewSSTables();
+ File dataFolder = new Directories(cfs.metadata()).getDirectoryForNewSSTables();
SSTableReader sstableOld = sstable(dataFolder, cfs, 0, 128);
SSTableReader sstableNew = sstable(dataFolder, cfs, 1, 128);
@@ -1028,7 +1029,7 @@
{ // the corruption is recoverable but the commit record is unreadable so the transaction is still in progress
//This should remove new files
- LogTransaction.removeUnfinishedLeftovers(cfs.metadata);
+ LogTransaction.removeUnfinishedLeftovers(cfs.metadata());
// make sure to exclude the old files that were deleted by the modifier
assertFiles(dataFolder.getPath(), oldFiles);
@@ -1037,7 +1038,7 @@
{ // if an intermediate line was also modified, it should ignore the tx log file
//This should not remove any files
- LogTransaction.removeUnfinishedLeftovers(cfs.metadata);
+ LogTransaction.removeUnfinishedLeftovers(cfs.metadata());
assertFiles(dataFolder.getPath(), Sets.newHashSet(Iterables.concat(newFiles,
oldFiles,
@@ -1065,7 +1066,7 @@
private static void testObsoletedFilesChanged(Consumer<SSTableReader> modifier) throws IOException
{
ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
- File dataFolder = new Directories(cfs.metadata).getDirectoryForNewSSTables();
+ File dataFolder = new Directories(cfs.metadata()).getDirectoryForNewSSTables();
SSTableReader sstableOld = sstable(dataFolder, cfs, 0, 128);
SSTableReader sstableNew = sstable(dataFolder, cfs, 1, 128);
@@ -1083,7 +1084,7 @@
log.txnFile().commit();
//This should not remove the old files
- LogTransaction.removeUnfinishedLeftovers(cfs.metadata);
+ LogTransaction.removeUnfinishedLeftovers(cfs.metadata());
assertFiles(dataFolder.getPath(), Sets.newHashSet(Iterables.concat(
sstableNew.getAllFilePaths(),
@@ -1108,7 +1109,7 @@
public void testGetTemporaryFilesSafeAfterObsoletion() throws Throwable
{
ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
- File dataFolder = new Directories(cfs.metadata).getDirectoryForNewSSTables();
+ File dataFolder = new Directories(cfs.metadata()).getDirectoryForNewSSTables();
SSTableReader sstable = sstable(dataFolder, cfs, 0, 128);
LogTransaction logs = new LogTransaction(OperationType.COMPACTION);
@@ -1132,7 +1133,7 @@
public void testGetTemporaryFilesThrowsIfCompletingAfterObsoletion() throws Throwable
{
ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
- File dataFolder = new Directories(cfs.metadata).getDirectoryForNewSSTables();
+ File dataFolder = new Directories(cfs.metadata()).getDirectoryForNewSSTables();
SSTableReader sstable = sstable(dataFolder, cfs, 0, 128);
LogTransaction logs = new LogTransaction(OperationType.COMPACTION);
@@ -1180,9 +1181,9 @@
FileHandle dFile = new FileHandle.Builder(descriptor.filenameFor(Component.DATA)).complete();
FileHandle iFile = new FileHandle.Builder(descriptor.filenameFor(Component.PRIMARY_INDEX)).complete();
- SerializationHeader header = SerializationHeader.make(cfs.metadata, Collections.emptyList());
- StatsMetadata metadata = (StatsMetadata) new MetadataCollector(cfs.metadata.comparator)
- .finalizeMetadata(cfs.metadata.partitioner.getClass().getCanonicalName(), 0.01f, -1, header)
+ SerializationHeader header = SerializationHeader.make(cfs.metadata(), Collections.emptyList());
+ StatsMetadata metadata = (StatsMetadata) new MetadataCollector(cfs.metadata().comparator)
+ .finalizeMetadata(cfs.metadata().partitioner.getClass().getCanonicalName(), 0.01f, -1, null, false, header)
.get(MetadataType.STATS);
SSTableReader reader = SSTableReader.internalOpen(descriptor,
components,
diff --git a/test/unit/org/apache/cassandra/db/lifecycle/RealTransactionsTest.java b/test/unit/org/apache/cassandra/db/lifecycle/RealTransactionsTest.java
index 595610e..b7b7d4a 100644
--- a/test/unit/org/apache/cassandra/db/lifecycle/RealTransactionsTest.java
+++ b/test/unit/org/apache/cassandra/db/lifecycle/RealTransactionsTest.java
@@ -28,11 +28,10 @@
import org.junit.BeforeClass;
import org.junit.Test;
-import junit.framework.Assert;
-import org.apache.cassandra.MockSchema;
+import org.junit.Assert;
import org.apache.cassandra.SchemaLoader;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.Schema;
+import org.apache.cassandra.schema.TableMetadataRef;
+import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.db.SerializationHeader;
@@ -65,8 +64,6 @@
@BeforeClass
public static void setUp()
{
- MockSchema.cleanup();
-
SchemaLoader.prepareServer();
SchemaLoader.createKeyspace(KEYSPACE,
KeyspaceParams.simple(1),
@@ -160,14 +157,16 @@
{
long lastCheckObsoletion = System.nanoTime();
File directory = txn.originals().iterator().next().descriptor.directory;
- Descriptor desc = Descriptor.fromFilename(cfs.getSSTablePath(directory));
- CFMetaData metadata = Schema.instance.getCFMetaData(desc);
+ Descriptor desc = cfs.newSSTableDescriptor(directory);
+ TableMetadataRef metadata = Schema.instance.getTableMetadataRef(desc);
rewriter.switchWriter(SSTableWriter.create(metadata,
desc,
0,
0,
+ null,
+ false,
0,
- SerializationHeader.make(cfs.metadata, txn.originals()),
+ SerializationHeader.make(cfs.metadata(), txn.originals()),
cfs.indexManager.listIndexes(),
txn));
while (ci.hasNext())
diff --git a/test/unit/org/apache/cassandra/db/lifecycle/TrackerTest.java b/test/unit/org/apache/cassandra/db/lifecycle/TrackerTest.java
index 65c7d0e..910445f 100644
--- a/test/unit/org/apache/cassandra/db/lifecycle/TrackerTest.java
+++ b/test/unit/org/apache/cassandra/db/lifecycle/TrackerTest.java
@@ -21,6 +21,7 @@
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
+import java.util.Optional;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
@@ -33,8 +34,7 @@
import org.junit.BeforeClass;
import org.junit.Test;
-import junit.framework.Assert;
-import org.apache.cassandra.MockSchema;
+import org.junit.Assert;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Memtable;
@@ -43,6 +43,8 @@
import org.apache.cassandra.db.compaction.OperationType;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.notifications.*;
+import org.apache.cassandra.schema.CachingParams;
+import org.apache.cassandra.schema.MockSchema;
import org.apache.cassandra.utils.concurrent.OpOrder;
import static com.google.common.collect.ImmutableSet.copyOf;
@@ -75,6 +77,7 @@
public static void setUp()
{
DatabaseDescriptor.daemonInitialization();
+ CommitLog.instance.start();
MockSchema.cleanup();
}
@@ -145,7 +148,7 @@
@Test
public void testAddInitialSSTables()
{
- ColumnFamilyStore cfs = MockSchema.newCFS();
+ ColumnFamilyStore cfs = MockSchema.newCFS(metadata -> metadata.caching(CachingParams.CACHE_KEYS));
Tracker tracker = cfs.getTracker();
List<SSTableReader> readers = ImmutableList.of(MockSchema.sstable(0, 17, cfs),
MockSchema.sstable(1, 121, cfs),
@@ -155,7 +158,7 @@
Assert.assertEquals(3, tracker.view.get().sstables.size());
for (SSTableReader reader : readers)
- Assert.assertTrue(reader.isKeyCacheSetup());
+ Assert.assertTrue(reader.isKeyCacheEnabled());
Assert.assertEquals(17 + 121 + 9, cfs.metric.liveDiskSpaceUsed.getCount());
}
@@ -165,7 +168,7 @@
{
boolean backups = DatabaseDescriptor.isIncrementalBackupsEnabled();
DatabaseDescriptor.setIncrementalBackupsEnabled(false);
- ColumnFamilyStore cfs = MockSchema.newCFS();
+ ColumnFamilyStore cfs = MockSchema.newCFS(metadata -> metadata.caching(CachingParams.CACHE_KEYS));
Tracker tracker = cfs.getTracker();
MockListener listener = new MockListener(false);
tracker.subscribe(listener);
@@ -177,7 +180,7 @@
Assert.assertEquals(3, tracker.view.get().sstables.size());
for (SSTableReader reader : readers)
- Assert.assertTrue(reader.isKeyCacheSetup());
+ Assert.assertTrue(reader.isKeyCacheEnabled());
Assert.assertEquals(17 + 121 + 9, cfs.metric.liveDiskSpaceUsed.getCount());
Assert.assertEquals(1, listener.senders.size());
@@ -262,7 +265,7 @@
{
boolean backups = DatabaseDescriptor.isIncrementalBackupsEnabled();
DatabaseDescriptor.setIncrementalBackupsEnabled(false);
- ColumnFamilyStore cfs = MockSchema.newCFS();
+ ColumnFamilyStore cfs = MockSchema.newCFS(metadata -> metadata.caching(CachingParams.CACHE_KEYS));
MockListener listener = new MockListener(false);
Tracker tracker = cfs.getTracker();
tracker.subscribe(listener);
@@ -305,8 +308,9 @@
Assert.assertEquals(2, listener.received.size());
Assert.assertEquals(prev2, ((MemtableDiscardedNotification) listener.received.get(0)).memtable);
Assert.assertEquals(singleton(reader), ((SSTableAddedNotification) listener.received.get(1)).added);
+ Assert.assertEquals(Optional.of(prev2), ((SSTableAddedNotification) listener.received.get(1)).memtable());
listener.received.clear();
- Assert.assertTrue(reader.isKeyCacheSetup());
+ Assert.assertTrue(reader.isKeyCacheEnabled());
Assert.assertEquals(10, cfs.metric.liveDiskSpaceUsed.getCount());
// test invalidated CFS
@@ -326,6 +330,7 @@
Assert.assertEquals(prev1, ((MemtableSwitchedNotification) listener.received.get(0)).memtable);
Assert.assertEquals(prev1, ((MemtableDiscardedNotification) listener.received.get(1)).memtable);
Assert.assertEquals(singleton(reader), ((SSTableAddedNotification) listener.received.get(2)).added);
+ Assert.assertEquals(Optional.of(prev1), ((SSTableAddedNotification) listener.received.get(2)).memtable());
Assert.assertTrue(listener.received.get(3) instanceof SSTableDeletingNotification);
Assert.assertEquals(1, ((SSTableListChangedNotification) listener.received.get(4)).removed.size());
DatabaseDescriptor.setIncrementalBackupsEnabled(backups);
@@ -356,12 +361,17 @@
tracker.notifyRenewed(memtable);
Assert.assertEquals(memtable, ((MemtableRenewedNotification) listener.received.get(0)).renewed);
listener.received.clear();
+ tracker.notifySSTableMetadataChanged(r1, r1.getSSTableMetadata());
+ Assert.assertEquals(((SSTableMetadataChanged)listener.received.get(0)).sstable, r1);
+ Assert.assertEquals(r1.getSSTableMetadata(), ((SSTableMetadataChanged)listener.received.get(0)).oldMetadata);
+ listener.received.clear();
tracker.unsubscribe(listener);
MockListener failListener = new MockListener(true);
tracker.subscribe(failListener);
tracker.subscribe(listener);
- Assert.assertNotNull(tracker.notifyAdded(singleton(r1), null));
+ Assert.assertNotNull(tracker.notifyAdded(singleton(r1), null, null));
Assert.assertEquals(singleton(r1), ((SSTableAddedNotification) listener.received.get(0)).added);
+ Assert.assertFalse(((SSTableAddedNotification) listener.received.get(0)).memtable().isPresent());
listener.received.clear();
Assert.assertNotNull(tracker.notifySSTablesChanged(singleton(r1), singleton(r2), OperationType.COMPACTION, null));
Assert.assertEquals(singleton(r1), ((SSTableListChangedNotification) listener.received.get(0)).removed);
diff --git a/test/unit/org/apache/cassandra/db/lifecycle/ViewTest.java b/test/unit/org/apache/cassandra/db/lifecycle/ViewTest.java
index a0e6e5f..fd32087 100644
--- a/test/unit/org/apache/cassandra/db/lifecycle/ViewTest.java
+++ b/test/unit/org/apache/cassandra/db/lifecycle/ViewTest.java
@@ -31,14 +31,15 @@
import org.junit.BeforeClass;
import org.junit.Test;
-import junit.framework.Assert;
-import org.apache.cassandra.MockSchema;
+import org.junit.Assert;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Memtable;
import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.db.commitlog.CommitLog;
import org.apache.cassandra.dht.AbstractBounds;
import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.schema.MockSchema;
import static com.google.common.collect.ImmutableSet.copyOf;
import static com.google.common.collect.ImmutableSet.of;
@@ -52,6 +53,7 @@
public static void setUp()
{
DatabaseDescriptor.daemonInitialization();
+ CommitLog.instance.start();
MockSchema.cleanup();
}
diff --git a/test/unit/org/apache/cassandra/db/marshal/CompositeTypeTest.java b/test/unit/org/apache/cassandra/db/marshal/CompositeTypeTest.java
index cc66e71..9813fd9 100644
--- a/test/unit/org/apache/cassandra/db/marshal/CompositeTypeTest.java
+++ b/test/unit/org/apache/cassandra/db/marshal/CompositeTypeTest.java
@@ -29,7 +29,7 @@
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.Util;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.rows.Cell;
import org.apache.cassandra.db.rows.Row;
@@ -189,13 +189,13 @@
ByteBuffer key = ByteBufferUtil.bytes("k");
long ts = FBUtilities.timestampMicros();
- new RowUpdateBuilder(cfs.metadata, ts, key).clustering(cname5).add("val", "cname5").build().applyUnsafe();
- new RowUpdateBuilder(cfs.metadata, ts, key).clustering(cname1).add("val", "cname1").build().applyUnsafe();
- new RowUpdateBuilder(cfs.metadata, ts, key).clustering(cname4).add("val", "cname4").build().applyUnsafe();
- new RowUpdateBuilder(cfs.metadata, ts, key).clustering(cname2).add("val", "cname2").build().applyUnsafe();
- new RowUpdateBuilder(cfs.metadata, ts, key).clustering(cname3).add("val", "cname3").build().applyUnsafe();
+ new RowUpdateBuilder(cfs.metadata(), ts, key).clustering(cname5).add("val", "cname5").build().applyUnsafe();
+ new RowUpdateBuilder(cfs.metadata(), ts, key).clustering(cname1).add("val", "cname1").build().applyUnsafe();
+ new RowUpdateBuilder(cfs.metadata(), ts, key).clustering(cname4).add("val", "cname4").build().applyUnsafe();
+ new RowUpdateBuilder(cfs.metadata(), ts, key).clustering(cname2).add("val", "cname2").build().applyUnsafe();
+ new RowUpdateBuilder(cfs.metadata(), ts, key).clustering(cname3).add("val", "cname3").build().applyUnsafe();
- ColumnDefinition cdef = cfs.metadata.getColumnDefinition(ByteBufferUtil.bytes("val"));
+ ColumnMetadata cdef = cfs.metadata().getColumn(ByteBufferUtil.bytes("val"));
ImmutableBTreePartition readPartition = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, key).build());
Iterator<Row> iter = readPartition.iterator();
@@ -261,13 +261,14 @@
new String[]{ "foo!", "b:ar" },
};
+
for (String[] input : inputs)
{
- CompositeType.Builder builder = new CompositeType.Builder(comp);
- for (String part : input)
- builder.add(UTF8Type.instance.fromString(part));
+ ByteBuffer[] bbs = new ByteBuffer[input.length];
+ for (int i = 0; i < input.length; i++)
+ bbs[i] = UTF8Type.instance.fromString(input[i]);
- ByteBuffer value = comp.fromString(comp.getString(builder.build()));
+ ByteBuffer value = comp.fromString(comp.getString(CompositeType.build(bbs)));
ByteBuffer[] splitted = comp.split(value);
for (int i = 0; i < splitted.length; i++)
assertEquals(input[i], UTF8Type.instance.getString(splitted[i]));
diff --git a/test/unit/org/apache/cassandra/db/marshal/DynamicCompositeTypeTest.java b/test/unit/org/apache/cassandra/db/marshal/DynamicCompositeTypeTest.java
index 0a3c39c..068daf6 100644
--- a/test/unit/org/apache/cassandra/db/marshal/DynamicCompositeTypeTest.java
+++ b/test/unit/org/apache/cassandra/db/marshal/DynamicCompositeTypeTest.java
@@ -31,7 +31,7 @@
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.Util;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.rows.Cell;
import org.apache.cassandra.db.rows.Row;
@@ -193,13 +193,13 @@
ByteBuffer key = ByteBufferUtil.bytes("k");
long ts = FBUtilities.timestampMicros();
- new RowUpdateBuilder(cfs.metadata, ts, key).clustering(cname5).add("val", "cname5").build().applyUnsafe();
- new RowUpdateBuilder(cfs.metadata, ts, key).clustering(cname1).add("val", "cname1").build().applyUnsafe();
- new RowUpdateBuilder(cfs.metadata, ts, key).clustering(cname4).add("val", "cname4").build().applyUnsafe();
- new RowUpdateBuilder(cfs.metadata, ts, key).clustering(cname2).add("val", "cname2").build().applyUnsafe();
- new RowUpdateBuilder(cfs.metadata, ts, key).clustering(cname3).add("val", "cname3").build().applyUnsafe();
+ new RowUpdateBuilder(cfs.metadata(), ts, key).clustering(cname5).add("val", "cname5").build().applyUnsafe();
+ new RowUpdateBuilder(cfs.metadata(), ts, key).clustering(cname1).add("val", "cname1").build().applyUnsafe();
+ new RowUpdateBuilder(cfs.metadata(), ts, key).clustering(cname4).add("val", "cname4").build().applyUnsafe();
+ new RowUpdateBuilder(cfs.metadata(), ts, key).clustering(cname2).add("val", "cname2").build().applyUnsafe();
+ new RowUpdateBuilder(cfs.metadata(), ts, key).clustering(cname3).add("val", "cname3").build().applyUnsafe();
- ColumnDefinition cdef = cfs.metadata.getColumnDefinition(ByteBufferUtil.bytes("val"));
+ ColumnMetadata cdef = cfs.metadata().getColumn(ByteBufferUtil.bytes("val"));
ImmutableBTreePartition readPartition = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, key).build());
Iterator<Row> iter = readPartition.iterator();
@@ -230,13 +230,13 @@
ByteBuffer key = ByteBufferUtil.bytes("kr");
long ts = FBUtilities.timestampMicros();
- new RowUpdateBuilder(cfs.metadata, ts, key).clustering(cname5).add("val", "cname5").build().applyUnsafe();
- new RowUpdateBuilder(cfs.metadata, ts, key).clustering(cname1).add("val", "cname1").build().applyUnsafe();
- new RowUpdateBuilder(cfs.metadata, ts, key).clustering(cname4).add("val", "cname4").build().applyUnsafe();
- new RowUpdateBuilder(cfs.metadata, ts, key).clustering(cname2).add("val", "cname2").build().applyUnsafe();
- new RowUpdateBuilder(cfs.metadata, ts, key).clustering(cname3).add("val", "cname3").build().applyUnsafe();
+ new RowUpdateBuilder(cfs.metadata(), ts, key).clustering(cname5).add("val", "cname5").build().applyUnsafe();
+ new RowUpdateBuilder(cfs.metadata(), ts, key).clustering(cname1).add("val", "cname1").build().applyUnsafe();
+ new RowUpdateBuilder(cfs.metadata(), ts, key).clustering(cname4).add("val", "cname4").build().applyUnsafe();
+ new RowUpdateBuilder(cfs.metadata(), ts, key).clustering(cname2).add("val", "cname2").build().applyUnsafe();
+ new RowUpdateBuilder(cfs.metadata(), ts, key).clustering(cname3).add("val", "cname3").build().applyUnsafe();
- ColumnDefinition cdef = cfs.metadata.getColumnDefinition(ByteBufferUtil.bytes("val"));
+ ColumnMetadata cdef = cfs.metadata().getColumn(ByteBufferUtil.bytes("val"));
ImmutableBTreePartition readPartition = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, key).build());
Iterator<Row> iter = readPartition.iterator();
diff --git a/test/unit/org/apache/cassandra/db/marshal/EmptyTypeTest.java b/test/unit/org/apache/cassandra/db/marshal/EmptyTypeTest.java
index 423e304..b362666 100644
--- a/test/unit/org/apache/cassandra/db/marshal/EmptyTypeTest.java
+++ b/test/unit/org/apache/cassandra/db/marshal/EmptyTypeTest.java
@@ -2,7 +2,6 @@
import java.nio.ByteBuffer;
-import org.junit.Assert;
import org.junit.Test;
import org.apache.cassandra.io.util.DataInputPlus;
@@ -11,13 +10,15 @@
import org.apache.cassandra.utils.ByteBufferUtil;
import org.mockito.Mockito;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
public class EmptyTypeTest
{
@Test
public void isFixed()
{
- Assert.assertEquals(0, EmptyType.instance.valueLengthIfFixed());
+ assertThat(EmptyType.instance.valueLengthIfFixed()).isEqualTo(0);
}
@Test
@@ -35,17 +36,8 @@
DataOutputPlus output = Mockito.mock(DataOutputPlus.class);
ByteBuffer rejected = ByteBuffer.wrap("this better fail".getBytes());
- boolean thrown = false;
- try
- {
- EmptyType.instance.writeValue(rejected, output);
- }
- catch (AssertionError e)
- {
- thrown = true;
- }
- Assert.assertTrue("writeValue did not reject non-empty input", thrown);
-
+ assertThatThrownBy(() -> EmptyType.instance.writeValue(rejected, output))
+ .isInstanceOf(AssertionError.class);
Mockito.verifyNoInteractions(output);
}
@@ -55,12 +47,14 @@
DataInputPlus input = Mockito.mock(DataInputPlus.class);
ByteBuffer buffer = EmptyType.instance.readValue(input);
- Assert.assertNotNull(buffer);
- Assert.assertFalse("empty type returned back non-empty data", buffer.hasRemaining());
+ assertThat(buffer)
+ .isNotNull()
+ .matches(b -> !b.hasRemaining());
buffer = EmptyType.instance.readValue(input, 42);
- Assert.assertNotNull(buffer);
- Assert.assertFalse("empty type returned back non-empty data", buffer.hasRemaining());
+ assertThat(buffer)
+ .isNotNull()
+ .matches(b -> !b.hasRemaining());
Mockito.verifyNoInteractions(input);
}
@@ -69,26 +63,21 @@
public void decompose()
{
ByteBuffer buffer = EmptyType.instance.decompose(null);
- Assert.assertEquals(0, buffer.remaining());
+ assertThat(buffer.remaining()).isEqualTo(0);
}
@Test
public void composeEmptyInput()
{
Void result = EmptyType.instance.compose(ByteBufferUtil.EMPTY_BYTE_BUFFER);
- Assert.assertNull(result);
+ assertThat(result).isNull();
}
@Test
public void composeNonEmptyInput()
{
- try
- {
- EmptyType.instance.compose(ByteBufferUtil.bytes("should fail"));
- Assert.fail("compose is expected to reject non-empty values, but did not");
- }
- catch (MarshalException e) {
- Assert.assertTrue(e.getMessage().startsWith("EmptyType only accept empty values"));
- }
+ assertThatThrownBy(() -> EmptyType.instance.compose(ByteBufferUtil.bytes("should fail")))
+ .isInstanceOf(MarshalException.class)
+ .hasMessage("EmptyType only accept empty values");
}
}
diff --git a/test/unit/org/apache/cassandra/db/marshal/JsonConversionTest.java b/test/unit/org/apache/cassandra/db/marshal/JsonConversionTest.java
index beb37e2..847ebef 100644
--- a/test/unit/org/apache/cassandra/db/marshal/JsonConversionTest.java
+++ b/test/unit/org/apache/cassandra/db/marshal/JsonConversionTest.java
@@ -21,10 +21,11 @@
import static org.junit.Assert.assertEquals;
import java.nio.ByteBuffer;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.cassandra.cql3.QueryOptions;
import org.apache.cassandra.transport.ProtocolVersion;
import org.apache.cassandra.utils.UUIDGen;
-import com.fasterxml.jackson.databind.ObjectMapper;
import org.junit.Test;
public class JsonConversionTest
diff --git a/test/unit/org/apache/cassandra/db/marshal/SimpleDateTypeTest.java b/test/unit/org/apache/cassandra/db/marshal/SimpleDateTypeTest.java
index 5c9ed4e..f45c55f 100644
--- a/test/unit/org/apache/cassandra/db/marshal/SimpleDateTypeTest.java
+++ b/test/unit/org/apache/cassandra/db/marshal/SimpleDateTypeTest.java
@@ -95,29 +95,29 @@
SimpleDateSerializer.instance.deserialize(d1),
SimpleDateSerializer.instance.deserialize(d2));
- d1 = SimpleDateType.instance.fromString("-10000-10-10");
- d2 = SimpleDateType.instance.fromString("10000-10-10");
+ d1 = SimpleDateType.instance.fromString("-2200-10-10");
+ d2 = SimpleDateType.instance.fromString("2200-10-10");
assert SimpleDateType.instance.compare(d1, d2) < 0 :
String.format("Failed neg/pos string comparison with %s and %s",
SimpleDateSerializer.instance.deserialize(d1),
SimpleDateSerializer.instance.deserialize(d2));
d1 = SimpleDateType.instance.fromString("1969-12-31");
- d2 = SimpleDateType.instance.fromString("1970-1-1");
+ d2 = SimpleDateType.instance.fromString("1970-01-01");
assert SimpleDateType.instance.compare(d1, d2) < 0 :
String.format("Failed pre/post epoch comparison with %s and %s",
SimpleDateSerializer.instance.deserialize(d1),
SimpleDateSerializer.instance.deserialize(d2));
- d1 = SimpleDateType.instance.fromString("1970-1-1");
- d2 = SimpleDateType.instance.fromString("1970-1-1");
+ d1 = SimpleDateType.instance.fromString("1970-01-01");
+ d2 = SimpleDateType.instance.fromString("1970-01-01");
assert SimpleDateType.instance.compare(d1, d2) == 0 :
String.format("Failed == date from string comparison with %s and %s",
SimpleDateSerializer.instance.deserialize(d1),
SimpleDateSerializer.instance.deserialize(d2));
- d1 = SimpleDateType.instance.fromString("1970-1-1");
- d2 = SimpleDateType.instance.fromString("1970-1-2");
+ d1 = SimpleDateType.instance.fromString("1970-01-01");
+ d2 = SimpleDateType.instance.fromString("1970-01-02");
assert SimpleDateType.instance.compare(d1, d2) < 0 :
String.format("Failed post epoch string comparison with %s and %s",
SimpleDateSerializer.instance.deserialize(d1),
diff --git a/test/unit/org/apache/cassandra/db/marshal/TimeUUIDTypeTest.java b/test/unit/org/apache/cassandra/db/marshal/TimeUUIDTypeTest.java
index b6160bf..9a71e06 100644
--- a/test/unit/org/apache/cassandra/db/marshal/TimeUUIDTypeTest.java
+++ b/test/unit/org/apache/cassandra/db/marshal/TimeUUIDTypeTest.java
@@ -21,7 +21,7 @@
import java.nio.ByteBuffer;
import java.util.UUID;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.cassandra.serializers.MarshalException;
import org.junit.Test;
import static org.junit.Assert.assertEquals;
diff --git a/test/unit/org/apache/cassandra/db/marshal/UUIDTypeTest.java b/test/unit/org/apache/cassandra/db/marshal/UUIDTypeTest.java
index 335860c..12a8871 100644
--- a/test/unit/org/apache/cassandra/db/marshal/UUIDTypeTest.java
+++ b/test/unit/org/apache/cassandra/db/marshal/UUIDTypeTest.java
@@ -30,7 +30,7 @@
import org.junit.Test;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.UUIDGen;
import org.slf4j.Logger;
diff --git a/test/unit/org/apache/cassandra/db/monitoring/MonitoringTaskTest.java b/test/unit/org/apache/cassandra/db/monitoring/MonitoringTaskTest.java
index acc988f..dc8c317 100644
--- a/test/unit/org/apache/cassandra/db/monitoring/MonitoringTaskTest.java
+++ b/test/unit/org/apache/cassandra/db/monitoring/MonitoringTaskTest.java
@@ -32,6 +32,11 @@
import org.junit.BeforeClass;
import org.junit.Test;
+import org.apache.cassandra.utils.ApproximateTime;
+
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+import static java.util.concurrent.TimeUnit.NANOSECONDS;
+import static org.apache.cassandra.utils.MonotonicClock.approxTime;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
@@ -39,8 +44,8 @@
public class MonitoringTaskTest
{
- private static final long timeout = 100;
- private static final long slowTimeout = 10;
+ private static final long timeout = MILLISECONDS.toNanos(100);
+ private static final long slowTimeout = MILLISECONDS.toNanos(10);
private static final long MAX_SPIN_TIME_NANOS = TimeUnit.SECONDS.toNanos(5);
@@ -90,8 +95,8 @@
private static void waitForOperationsToComplete(List<Monitorable> operations) throws InterruptedException
{
- long timeout = operations.stream().map(Monitorable::timeout).reduce(0L, Long::max);
- Thread.sleep(timeout * 2 + ApproximateTime.precision());
+ long timeout = operations.stream().map(Monitorable::timeoutNanos).reduce(0L, Long::max);
+ Thread.sleep(NANOSECONDS.toMillis(timeout * 2 + approxTime.error()));
long start = System.nanoTime();
while(System.nanoTime() - start <= MAX_SPIN_TIME_NANOS)
@@ -109,8 +114,8 @@
private static void waitForOperationsToBeReportedAsSlow(List<Monitorable> operations) throws InterruptedException
{
- long timeout = operations.stream().map(Monitorable::slowTimeout).reduce(0L, Long::max);
- Thread.sleep(timeout * 2 + ApproximateTime.precision());
+ long timeout = operations.stream().map(Monitorable::slowTimeoutNanos).reduce(0L, Long::max);
+ Thread.sleep(NANOSECONDS.toMillis(timeout * 2 + approxTime.error()));
long start = System.nanoTime();
while(System.nanoTime() - start <= MAX_SPIN_TIME_NANOS)
@@ -124,7 +129,7 @@
@Test
public void testAbort() throws InterruptedException
{
- Monitorable operation = new TestMonitor("Test abort", System.currentTimeMillis(), false, timeout, slowTimeout);
+ Monitorable operation = new TestMonitor("Test abort", System.nanoTime(), false, timeout, slowTimeout);
waitForOperationsToComplete(operation);
assertTrue(operation.isAborted());
@@ -135,7 +140,7 @@
@Test
public void testAbortIdemPotent() throws InterruptedException
{
- Monitorable operation = new TestMonitor("Test abort", System.currentTimeMillis(), false, timeout, slowTimeout);
+ Monitorable operation = new TestMonitor("Test abort", System.nanoTime(), false, timeout, slowTimeout);
waitForOperationsToComplete(operation);
assertTrue(operation.abort());
@@ -148,7 +153,7 @@
@Test
public void testAbortCrossNode() throws InterruptedException
{
- Monitorable operation = new TestMonitor("Test for cross node", System.currentTimeMillis(), true, timeout, slowTimeout);
+ Monitorable operation = new TestMonitor("Test for cross node", System.nanoTime(), true, timeout, slowTimeout);
waitForOperationsToComplete(operation);
assertTrue(operation.isAborted());
@@ -159,7 +164,7 @@
@Test
public void testComplete() throws InterruptedException
{
- Monitorable operation = new TestMonitor("Test complete", System.currentTimeMillis(), false, timeout, slowTimeout);
+ Monitorable operation = new TestMonitor("Test complete", System.nanoTime(), false, timeout, slowTimeout);
operation.complete();
waitForOperationsToComplete(operation);
@@ -171,7 +176,7 @@
@Test
public void testCompleteIdemPotent() throws InterruptedException
{
- Monitorable operation = new TestMonitor("Test complete", System.currentTimeMillis(), false, timeout, slowTimeout);
+ Monitorable operation = new TestMonitor("Test complete", System.nanoTime(), false, timeout, slowTimeout);
operation.complete();
waitForOperationsToComplete(operation);
@@ -185,7 +190,7 @@
@Test
public void testReportSlow() throws InterruptedException
{
- Monitorable operation = new TestMonitor("Test report slow", System.currentTimeMillis(), false, timeout, slowTimeout);
+ Monitorable operation = new TestMonitor("Test report slow", System.nanoTime(), false, timeout, slowTimeout);
waitForOperationsToBeReportedAsSlow(operation);
assertTrue(operation.isSlow());
@@ -199,7 +204,7 @@
public void testNoReportSlowIfZeroSlowTimeout() throws InterruptedException
{
// when the slow timeout is set to zero then operation won't be reported as slow
- Monitorable operation = new TestMonitor("Test report slow disabled", System.currentTimeMillis(), false, timeout, 0);
+ Monitorable operation = new TestMonitor("Test report slow disabled", System.nanoTime(), false, timeout, 0);
waitForOperationsToBeReportedAsSlow(operation);
assertTrue(operation.isSlow());
@@ -212,7 +217,7 @@
@Test
public void testReport() throws InterruptedException
{
- Monitorable operation = new TestMonitor("Test report", System.currentTimeMillis(), false, timeout, slowTimeout);
+ Monitorable operation = new TestMonitor("Test report", System.nanoTime(), false, timeout, slowTimeout);
waitForOperationsToComplete(operation);
assertTrue(operation.isSlow());
@@ -220,10 +225,10 @@
assertFalse(operation.isCompleted());
// aborted operations are not logged as slow
- assertFalse(MonitoringTask.instance.logSlowOperations(ApproximateTime.currentTimeMillis()));
+ assertFalse(MonitoringTask.instance.logSlowOperations(approxTime.now()));
assertEquals(0, MonitoringTask.instance.getSlowOperations().size());
- assertTrue(MonitoringTask.instance.logFailedOperations(ApproximateTime.currentTimeMillis()));
+ assertTrue(MonitoringTask.instance.logFailedOperations(approxTime.now()));
assertEquals(0, MonitoringTask.instance.getFailedOperations().size());
}
@@ -233,20 +238,20 @@
MonitoringTask.instance = MonitoringTask.make(10, -1);
try
{
- Monitorable operation1 = new TestMonitor("Test report 1", System.currentTimeMillis(), false, timeout, slowTimeout);
+ Monitorable operation1 = new TestMonitor("Test report 1", System.nanoTime(), false, timeout, slowTimeout);
waitForOperationsToComplete(operation1);
assertTrue(operation1.isAborted());
assertFalse(operation1.isCompleted());
- Monitorable operation2 = new TestMonitor("Test report 2", System.currentTimeMillis(), false, timeout, slowTimeout);
+ Monitorable operation2 = new TestMonitor("Test report 2", System.nanoTime(), false, timeout, slowTimeout);
waitForOperationsToBeReportedAsSlow(operation2);
operation2.complete();
assertFalse(operation2.isAborted());
assertTrue(operation2.isCompleted());
- Thread.sleep(ApproximateTime.precision() + 500);
+ Thread.sleep(2 * NANOSECONDS.toMillis(approxTime.error()) + 500);
assertEquals(0, MonitoringTask.instance.getFailedOperations().size());
assertEquals(0, MonitoringTask.instance.getSlowOperations().size());
}
@@ -266,12 +271,12 @@
for (int i = 0; i < opCount; i++)
{
executorService.submit(() ->
- operations.add(new TestMonitor(UUID.randomUUID().toString(), System.currentTimeMillis(), false, timeout, slowTimeout))
+ operations.add(new TestMonitor(UUID.randomUUID().toString(), System.nanoTime(), false, timeout, slowTimeout))
);
}
executorService.shutdown();
- assertTrue(executorService.awaitTermination(30, TimeUnit.SECONDS));
+ assertTrue(executorService.awaitTermination(1, TimeUnit.MINUTES));
assertEquals(opCount, operations.size());
waitForOperationsToComplete(operations);
@@ -311,14 +316,14 @@
for (int j = 0; j < numTimes; j++)
{
Monitorable operation1 = new TestMonitor(operationName,
- System.currentTimeMillis(),
+ System.nanoTime(),
false,
timeout,
slowTimeout);
waitForOperationsToComplete(operation1);
Monitorable operation2 = new TestMonitor(operationName,
- System.currentTimeMillis(),
+ System.nanoTime(),
false,
timeout,
slowTimeout);
@@ -366,7 +371,7 @@
try
{
Monitorable operation = new TestMonitor("Test testMultipleThreadsSameName failed",
- System.currentTimeMillis(),
+ System.nanoTime(),
false,
timeout,
slowTimeout);
@@ -400,7 +405,7 @@
try
{
Monitorable operation = new TestMonitor("Test testMultipleThreadsSameName slow",
- System.currentTimeMillis(),
+ System.nanoTime(),
false,
timeout,
slowTimeout);
@@ -436,7 +441,7 @@
try
{
Monitorable operation = new TestMonitor("Test thread " + Thread.currentThread().getName(),
- System.currentTimeMillis(),
+ System.nanoTime(),
false,
timeout,
slowTimeout);
diff --git a/test/unit/org/apache/cassandra/db/partition/PartitionImplementationTest.java b/test/unit/org/apache/cassandra/db/partition/PartitionImplementationTest.java
index 781ec9c..5162548 100644
--- a/test/unit/org/apache/cassandra/db/partition/PartitionImplementationTest.java
+++ b/test/unit/org/apache/cassandra/db/partition/PartitionImplementationTest.java
@@ -35,8 +35,8 @@
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.Util;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.cql3.ColumnIdentifier;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.filter.ColumnFilter;
@@ -62,7 +62,7 @@
private static final int TIMESTAMP = KEY_RANGE + 1;
- private static CFMetaData cfm;
+ private static TableMetadata metadata;
private Random rand = new Random(2);
@BeforeClass
@@ -70,15 +70,15 @@
{
SchemaLoader.prepareServer();
- cfm = CFMetaData.Builder.create(KEYSPACE, CF)
- .addPartitionKey("pk", AsciiType.instance)
- .addClusteringColumn("ck", AsciiType.instance)
- .addRegularColumn("col", AsciiType.instance)
- .addStaticColumn("static_col", AsciiType.instance)
- .build();
- SchemaLoader.createKeyspace(KEYSPACE,
- KeyspaceParams.simple(1),
- cfm);
+ metadata =
+ TableMetadata.builder(KEYSPACE, CF)
+ .addPartitionKeyColumn("pk", AsciiType.instance)
+ .addClusteringColumn("ck", AsciiType.instance)
+ .addRegularColumn("col", AsciiType.instance)
+ .addStaticColumn("static_col", AsciiType.instance)
+ .build();
+
+ SchemaLoader.createKeyspace(KEYSPACE, KeyspaceParams.simple(1), metadata);
}
private List<Row> generateRows()
@@ -100,8 +100,8 @@
Row makeRow(Clustering clustering, String colValue)
{
- ColumnDefinition defCol = cfm.getColumnDefinition(new ColumnIdentifier("col", true));
- Row.Builder row = BTreeRow.unsortedBuilder(TIMESTAMP);
+ ColumnMetadata defCol = metadata.getColumn(new ColumnIdentifier("col", true));
+ Row.Builder row = BTreeRow.unsortedBuilder();
row.newRow(clustering);
row.addCell(BufferCell.live(defCol, TIMESTAMP, ByteBufferUtil.bytes(colValue)));
return row.build();
@@ -109,8 +109,8 @@
Row makeStaticRow()
{
- ColumnDefinition defCol = cfm.getColumnDefinition(new ColumnIdentifier("static_col", true));
- Row.Builder row = BTreeRow.unsortedBuilder(TIMESTAMP);
+ ColumnMetadata defCol = metadata.getColumn(new ColumnIdentifier("static_col", true));
+ Row.Builder row = BTreeRow.unsortedBuilder();
row.newRow(Clustering.STATIC_CLUSTERING);
row.addCell(BufferCell.live(defCol, TIMESTAMP, ByteBufferUtil.bytes("static value")));
return row.build();
@@ -148,7 +148,7 @@
markers.add(open);
markers.add(close);
}
- markers.sort(cfm.comparator);
+ markers.sort(metadata.comparator);
RangeTombstoneMarker toAdd = null;
Set<DeletionTime> open = new HashSet<>();
@@ -163,7 +163,7 @@
{
if (toAdd != null)
{
- if (cfm.comparator.compare(toAdd, marker) != 0)
+ if (metadata.comparator.compare(toAdd, marker) != 0)
content.add(toAdd);
else
{
@@ -187,7 +187,7 @@
{
if (toAdd != null)
{
- if (cfm.comparator.compare(toAdd, marker) != 0)
+ if (metadata.comparator.compare(toAdd, marker) != 0)
content.add(toAdd);
else
{
@@ -212,7 +212,7 @@
private Clustering clustering(int i)
{
- return cfm.comparator.make(String.format("Row%06d", i));
+ return metadata.comparator.make(String.format("Row%06d", i));
}
private void test(Supplier<Collection<? extends Unfiltered>> content, Row staticRow)
@@ -233,18 +233,18 @@
private void testIter(Supplier<Collection<? extends Unfiltered>> contentSupplier, Row staticRow)
{
- NavigableSet<Clusterable> sortedContent = new TreeSet<Clusterable>(cfm.comparator);
+ NavigableSet<Clusterable> sortedContent = new TreeSet<Clusterable>(metadata.comparator);
sortedContent.addAll(contentSupplier.get());
AbstractBTreePartition partition;
- try (UnfilteredRowIterator iter = new Util.UnfilteredSource(cfm, Util.dk("pk"), staticRow, sortedContent.stream().map(x -> (Unfiltered) x).iterator()))
+ try (UnfilteredRowIterator iter = new Util.UnfilteredSource(metadata, Util.dk("pk"), staticRow, sortedContent.stream().map(x -> (Unfiltered) x).iterator()))
{
partition = ImmutableBTreePartition.create(iter);
}
- ColumnDefinition defCol = cfm.getColumnDefinition(new ColumnIdentifier("col", true));
+ ColumnMetadata defCol = metadata.getColumn(new ColumnIdentifier("col", true));
ColumnFilter cf = ColumnFilter.selectionBuilder().add(defCol).build();
- Function<? super Clusterable, ? extends Clusterable> colFilter = x -> x instanceof Row ? ((Row) x).filter(cf, cfm) : x;
- Slices slices = Slices.with(cfm.comparator, Slice.make(clustering(KEY_RANGE / 4), clustering(KEY_RANGE * 3 / 4)));
+ Function<? super Clusterable, ? extends Clusterable> colFilter = x -> x instanceof Row ? ((Row) x).filter(cf, metadata) : x;
+ Slices slices = Slices.with(metadata.comparator, Slice.make(clustering(KEY_RANGE / 4), clustering(KEY_RANGE * 3 / 4)));
Slices multiSlices = makeSlices();
// lastRow
@@ -278,44 +278,44 @@
// unfiltered iterator
assertIteratorsEqual(sortedContent.iterator(),
- partition.unfilteredIterator(ColumnFilter.all(cfm), Slices.ALL, false));
+ partition.unfilteredIterator(ColumnFilter.all(metadata), Slices.ALL, false));
// column-filtered
assertIteratorsEqual(sortedContent.stream().map(colFilter).iterator(),
partition.unfilteredIterator(cf, Slices.ALL, false));
// sliced
assertIteratorsEqual(slice(sortedContent, slices.get(0)),
- partition.unfilteredIterator(ColumnFilter.all(cfm), slices, false));
+ partition.unfilteredIterator(ColumnFilter.all(metadata), slices, false));
assertIteratorsEqual(streamOf(slice(sortedContent, slices.get(0))).map(colFilter).iterator(),
partition.unfilteredIterator(cf, slices, false));
// randomly multi-sliced
assertIteratorsEqual(slice(sortedContent, multiSlices),
- partition.unfilteredIterator(ColumnFilter.all(cfm), multiSlices, false));
+ partition.unfilteredIterator(ColumnFilter.all(metadata), multiSlices, false));
assertIteratorsEqual(streamOf(slice(sortedContent, multiSlices)).map(colFilter).iterator(),
partition.unfilteredIterator(cf, multiSlices, false));
// reversed
assertIteratorsEqual(sortedContent.descendingIterator(),
- partition.unfilteredIterator(ColumnFilter.all(cfm), Slices.ALL, true));
+ partition.unfilteredIterator(ColumnFilter.all(metadata), Slices.ALL, true));
assertIteratorsEqual(sortedContent.descendingSet().stream().map(colFilter).iterator(),
partition.unfilteredIterator(cf, Slices.ALL, true));
assertIteratorsEqual(invert(slice(sortedContent, slices.get(0))),
- partition.unfilteredIterator(ColumnFilter.all(cfm), slices, true));
+ partition.unfilteredIterator(ColumnFilter.all(metadata), slices, true));
assertIteratorsEqual(streamOf(invert(slice(sortedContent, slices.get(0)))).map(colFilter).iterator(),
partition.unfilteredIterator(cf, slices, true));
assertIteratorsEqual(invert(slice(sortedContent, multiSlices)),
- partition.unfilteredIterator(ColumnFilter.all(cfm), multiSlices, true));
+ partition.unfilteredIterator(ColumnFilter.all(metadata), multiSlices, true));
assertIteratorsEqual(streamOf(invert(slice(sortedContent, multiSlices))).map(colFilter).iterator(),
partition.unfilteredIterator(cf, multiSlices, true));
// search iterator
- testSearchIterator(sortedContent, partition, ColumnFilter.all(cfm), false);
+ testSearchIterator(sortedContent, partition, ColumnFilter.all(metadata), false);
testSearchIterator(sortedContent, partition, cf, false);
- testSearchIterator(sortedContent, partition, ColumnFilter.all(cfm), true);
+ testSearchIterator(sortedContent, partition, ColumnFilter.all(metadata), true);
testSearchIterator(sortedContent, partition, cf, true);
// sliceable iter
- testSlicingOfIterators(sortedContent, partition, ColumnFilter.all(cfm), false);
+ testSlicingOfIterators(sortedContent, partition, ColumnFilter.all(metadata), false);
testSlicingOfIterators(sortedContent, partition, cf, false);
- testSlicingOfIterators(sortedContent, partition, ColumnFilter.all(cfm), true);
+ testSlicingOfIterators(sortedContent, partition, ColumnFilter.all(metadata), true);
testSlicingOfIterators(sortedContent, partition, cf, true);
}
@@ -338,14 +338,14 @@
assertEquals(expected == null, row == null);
if (row == null)
continue;
- assertRowsEqual(expected.filter(cf, cfm), row);
+ assertRowsEqual(expected.filter(cf, metadata), row);
}
}
Slices makeSlices()
{
int pos = 0;
- Slices.Builder builder = new Slices.Builder(cfm.comparator);
+ Slices.Builder builder = new Slices.Builder(metadata.comparator);
while (pos <= KEY_RANGE)
{
int skip = rand.nextInt(KEY_RANGE / 10) * (rand.nextInt(3) + 2 / 3); // increased chance of getting 0
@@ -362,13 +362,13 @@
void testSlicingOfIterators(NavigableSet<Clusterable> sortedContent, AbstractBTreePartition partition, ColumnFilter cf, boolean reversed)
{
- Function<? super Clusterable, ? extends Clusterable> colFilter = x -> x instanceof Row ? ((Row) x).filter(cf, cfm) : x;
+ Function<? super Clusterable, ? extends Clusterable> colFilter = x -> x instanceof Row ? ((Row) x).filter(cf, metadata) : x;
Slices slices = makeSlices();
// fetch each slice in turn
for (Slice slice : (Iterable<Slice>) () -> directed(slices, reversed))
{
- try (UnfilteredRowIterator slicedIter = partition.unfilteredIterator(cf, Slices.with(cfm.comparator, slice), reversed))
+ try (UnfilteredRowIterator slicedIter = partition.unfilteredIterator(cf, Slices.with(metadata.comparator, slice), reversed))
{
assertIteratorsEqual(streamOf(directed(slice(sortedContent, slice), reversed)).map(colFilter).iterator(),
slicedIter);
@@ -441,8 +441,8 @@
Clusterable[] a2 = (Clusterable[]) Iterators.toArray(it2, Clusterable.class);
if (Arrays.equals(a1, a2))
return;
- String a1s = Stream.of(a1).map(x -> "\n" + (x instanceof Unfiltered ? ((Unfiltered) x).toString(cfm) : x.toString())).collect(Collectors.toList()).toString();
- String a2s = Stream.of(a2).map(x -> "\n" + (x instanceof Unfiltered ? ((Unfiltered) x).toString(cfm) : x.toString())).collect(Collectors.toList()).toString();
+ String a1s = Stream.of(a1).map(x -> "\n" + (x instanceof Unfiltered ? ((Unfiltered) x).toString(metadata) : x.toString())).collect(Collectors.toList()).toString();
+ String a2s = Stream.of(a2).map(x -> "\n" + (x instanceof Unfiltered ? ((Unfiltered) x).toString(metadata) : x.toString())).collect(Collectors.toList()).toString();
assertArrayEquals("Arrays differ. Expected " + a1s + " was " + a2s, a1, a2);
}
@@ -451,7 +451,7 @@
NavigableSet<Clusterable> nexts = sortedContent.tailSet(cl, true);
if (nexts.isEmpty())
return null;
- Row row = nexts.first() instanceof Row && cfm.comparator.compare(cl, nexts.first()) == 0 ? (Row) nexts.first() : null;
+ Row row = nexts.first() instanceof Row && metadata.comparator.compare(cl, nexts.first()) == 0 ? (Row) nexts.first() : null;
for (Clusterable next : nexts)
if (next instanceof RangeTombstoneMarker)
{
@@ -459,7 +459,7 @@
if (!rt.isClose(false))
return row;
DeletionTime delTime = rt.closeDeletionTime(false);
- return row == null ? BTreeRow.emptyDeletedRow(cl, Deletion.regular(delTime)) : row.filter(ColumnFilter.all(cfm), delTime, true, cfm);
+ return row == null ? BTreeRow.emptyDeletedRow(cl, Deletion.regular(delTime)) : row.filter(ColumnFilter.all(metadata), delTime, true, metadata);
}
return row;
}
diff --git a/test/unit/org/apache/cassandra/db/partition/PartitionUpdateTest.java b/test/unit/org/apache/cassandra/db/partition/PartitionUpdateTest.java
index df23e4f..a4555c8 100644
--- a/test/unit/org/apache/cassandra/db/partition/PartitionUpdateTest.java
+++ b/test/unit/org/apache/cassandra/db/partition/PartitionUpdateTest.java
@@ -18,41 +18,14 @@
package org.apache.cassandra.db.partition;
import org.apache.cassandra.UpdateBuilder;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-
-import com.google.common.collect.Lists;
-
-import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.cql3.CQLTester;
-import org.apache.cassandra.db.Clustering;
-import org.apache.cassandra.db.DecoratedKey;
-import org.apache.cassandra.db.DeletionTime;
-import org.apache.cassandra.db.Mutation;
import org.apache.cassandra.db.RowUpdateBuilder;
-import org.apache.cassandra.db.filter.ColumnFilter;
import org.apache.cassandra.db.partitions.PartitionUpdate;
-import org.apache.cassandra.db.rows.BTreeRow;
-import org.apache.cassandra.db.rows.BufferCell;
-import org.apache.cassandra.db.rows.Cell;
-import org.apache.cassandra.db.rows.CellPath;
-import org.apache.cassandra.db.rows.EncodingStats;
-import org.apache.cassandra.db.rows.Row;
-import org.apache.cassandra.db.rows.RowAndDeletionMergeIterator;
-import org.apache.cassandra.db.rows.Rows;
-import org.apache.cassandra.db.rows.UnfilteredRowIterator;
-import org.apache.cassandra.dht.Murmur3Partitioner;
-import org.apache.cassandra.io.sstable.ISSTableScanner;
-import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FBUtilities;
import org.junit.Test;
-import junit.framework.Assert;
-
-import static org.junit.Assert.assertEquals;
-
+import org.junit.Assert;
public class PartitionUpdateTest extends CQLTester
{
@@ -60,7 +33,7 @@
public void testOperationCount()
{
createTable("CREATE TABLE %s (key text, clustering int, a int, s int static, PRIMARY KEY(key, clustering))");
- CFMetaData cfm = currentTableMetadata();
+ TableMetadata cfm = currentTableMetadata();
UpdateBuilder builder = UpdateBuilder.create(cfm, "key0");
Assert.assertEquals(0, builder.build().operationCount());
@@ -79,7 +52,7 @@
public void testMutationSize()
{
createTable("CREATE TABLE %s (key text, clustering int, a int, s int static, PRIMARY KEY(key, clustering))");
- CFMetaData cfm = currentTableMetadata();
+ TableMetadata cfm = currentTableMetadata();
UpdateBuilder builder = UpdateBuilder.create(cfm, "key0");
builder.newRow().add("s", 1);
@@ -99,133 +72,17 @@
}
@Test
- public void testOperationCountWithCompactTable()
+ public void testUpdateAllTimestamp()
{
- createTable("CREATE TABLE %s (key text PRIMARY KEY, a int) WITH COMPACT STORAGE");
- CFMetaData cfm = currentTableMetadata();
+ createTable("CREATE TABLE %s (key text, clustering int, a int, b int, c int, s int static, PRIMARY KEY(key, clustering))");
+ TableMetadata cfm = currentTableMetadata();
- PartitionUpdate update = new RowUpdateBuilder(cfm, FBUtilities.timestampMicros(), "key0").add("a", 1)
- .buildUpdate();
- Assert.assertEquals(1, update.operationCount());
+ long timestamp = FBUtilities.timestampMicros();
+ RowUpdateBuilder rub = new RowUpdateBuilder(cfm, timestamp, "key0").clustering(1).add("a", 1);
+ PartitionUpdate pu = rub.buildUpdate();
+ PartitionUpdate pu2 = new PartitionUpdate.Builder(pu, 0).updateAllTimestamp(0).build();
- update = new RowUpdateBuilder(cfm, FBUtilities.timestampMicros(), "key0").buildUpdate();
- Assert.assertEquals(0, update.operationCount());
- }
-
- /**
- * Makes sure we merge duplicate rows, see CASSANDRA-15789
- */
- @Test
- public void testDuplicate()
- {
- createTable("CREATE TABLE %s (pk int, ck int, v map<text, text>, PRIMARY KEY (pk, ck))");
- CFMetaData cfm = currentTableMetadata();
-
- DecoratedKey dk = Murmur3Partitioner.instance.decorateKey(ByteBufferUtil.bytes(1));
-
- List<Row> rows = new ArrayList<>();
- Row.Builder builder = BTreeRow.unsortedBuilder(FBUtilities.nowInSeconds());
- builder.newRow(Clustering.make(ByteBufferUtil.bytes(2)));
- builder.addComplexDeletion(cfm.getColumnDefinition(ByteBufferUtil.bytes("v")), new DeletionTime(2, 1588586647));
-
- Cell c = BufferCell.live(cfm.getColumnDefinition(ByteBufferUtil.bytes("v")), 3, ByteBufferUtil.bytes("h"), CellPath.create(ByteBufferUtil.bytes("g")));
- builder.addCell(c);
-
- Row r = builder.build();
- rows.add(r);
-
- builder.newRow(Clustering.make(ByteBufferUtil.bytes(2)));
- builder.addRowDeletion(new Row.Deletion(new DeletionTime(1588586647, 1), false));
- r = builder.build();
- rows.add(r);
-
- RowAndDeletionMergeIterator rmi = new RowAndDeletionMergeIterator(cfm,
- dk,
- DeletionTime.LIVE,
- ColumnFilter.all(cfm),
- Rows.EMPTY_STATIC_ROW,
- false,
- EncodingStats.NO_STATS,
- rows.iterator(),
- Collections.emptyIterator(),
- true);
-
- PartitionUpdate pu = PartitionUpdate.fromPre30Iterator(rmi, ColumnFilter.all(cfm));
- pu.iterator();
-
- Mutation m = new Mutation(getCurrentColumnFamilyStore().keyspace.getName(), dk);
- m.add(pu);
- m.apply();
- getCurrentColumnFamilyStore().forceBlockingFlush();
-
- SSTableReader sst = getCurrentColumnFamilyStore().getLiveSSTables().iterator().next();
- int count = 0;
- try (ISSTableScanner scanner = sst.getScanner())
- {
- while (scanner.hasNext())
- {
- try (UnfilteredRowIterator iter = scanner.next())
- {
- while (iter.hasNext())
- {
- iter.next();
- count++;
- }
- }
- }
- }
- assertEquals(1, count);
- }
-
- /**
- * Makes sure we don't create duplicates when merging 2 partition updates
- */
- @Test
- public void testMerge()
- {
- createTable("CREATE TABLE %s (pk int, ck int, v map<text, text>, PRIMARY KEY (pk, ck))");
- CFMetaData cfm = currentTableMetadata();
-
- DecoratedKey dk = Murmur3Partitioner.instance.decorateKey(ByteBufferUtil.bytes(1));
-
- Row.Builder builder = BTreeRow.unsortedBuilder(FBUtilities.nowInSeconds());
- builder.newRow(Clustering.make(ByteBufferUtil.bytes(2)));
- builder.addComplexDeletion(cfm.getColumnDefinition(ByteBufferUtil.bytes("v")), new DeletionTime(2, 1588586647));
- Cell c = BufferCell.live(cfm.getColumnDefinition(ByteBufferUtil.bytes("v")), 3, ByteBufferUtil.bytes("h"), CellPath.create(ByteBufferUtil.bytes("g")));
- builder.addCell(c);
- Row r = builder.build();
-
- PartitionUpdate p1 = new PartitionUpdate(cfm, dk, cfm.partitionColumns(), 2);
- p1.add(r);
-
- builder.newRow(Clustering.make(ByteBufferUtil.bytes(2)));
- builder.addRowDeletion(new Row.Deletion(new DeletionTime(1588586647, 1), false));
- r = builder.build();
- PartitionUpdate p2 = new PartitionUpdate(cfm, dk, cfm.partitionColumns(), 2);
- p2.add(r);
-
- Mutation m = new Mutation(getCurrentColumnFamilyStore().keyspace.getName(), dk);
- m.add(PartitionUpdate.merge(Lists.newArrayList(p1, p2)));
- m.apply();
-
- getCurrentColumnFamilyStore().forceBlockingFlush();
-
- SSTableReader sst = getCurrentColumnFamilyStore().getLiveSSTables().iterator().next();
- int count = 0;
- try (ISSTableScanner scanner = sst.getScanner())
- {
- while (scanner.hasNext())
- {
- try (UnfilteredRowIterator iter = scanner.next())
- {
- while (iter.hasNext())
- {
- iter.next();
- count++;
- }
- }
- }
- }
- assertEquals(1, count);
+ Assert.assertTrue(pu.maxTimestamp() > 0);
+ Assert.assertTrue(pu2.maxTimestamp() == 0);
}
}
diff --git a/test/unit/org/apache/cassandra/db/partitions/PurgeFunctionTest.java b/test/unit/org/apache/cassandra/db/partitions/PurgeFunctionTest.java
index 7f85aea..ce569b5 100644
--- a/test/unit/org/apache/cassandra/db/partitions/PurgeFunctionTest.java
+++ b/test/unit/org/apache/cassandra/db/partitions/PurgeFunctionTest.java
@@ -19,13 +19,12 @@
import java.nio.ByteBuffer;
import java.util.Iterator;
-import java.util.function.Predicate;
+import java.util.function.LongPredicate;
import com.google.common.collect.Iterators;
import org.junit.Before;
import org.junit.Test;
-import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.ClusteringPrefix.Kind;
import org.apache.cassandra.db.*;
@@ -34,6 +33,7 @@
import org.apache.cassandra.db.rows.*;
import org.apache.cassandra.db.transform.Transformation;
import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.utils.FBUtilities;
import static org.junit.Assert.assertEquals;
@@ -46,7 +46,7 @@
private static final String KEYSPACE = "PurgeFunctionTest";
private static final String TABLE = "table";
- private CFMetaData metadata;
+ private TableMetadata metadata;
private DecoratedKey key;
private static UnfilteredPartitionIterator withoutPurgeableTombstones(UnfilteredPartitionIterator iterator, int gcBefore)
@@ -55,10 +55,10 @@
{
private WithoutPurgeableTombstones()
{
- super(iterator.isForThrift(), FBUtilities.nowInSeconds(), gcBefore, Integer.MAX_VALUE, false, false);
+ super(FBUtilities.nowInSeconds(), gcBefore, Integer.MAX_VALUE, false, false);
}
- protected Predicate<Long> getPurgeEvaluator()
+ protected LongPredicate getPurgeEvaluator()
{
return time -> true;
}
@@ -73,11 +73,10 @@
DatabaseDescriptor.setPartitionerUnsafe(Murmur3Partitioner.instance);
metadata =
- CFMetaData.Builder
- .create(KEYSPACE, TABLE)
- .addPartitionKey("pk", UTF8Type.instance)
- .addClusteringColumn("ck", UTF8Type.instance)
- .build();
+ TableMetadata.builder(KEYSPACE, TABLE)
+ .addPartitionKeyColumn("pk", UTF8Type.instance)
+ .addClusteringColumn("ck", UTF8Type.instance)
+ .build();
key = Murmur3Partitioner.instance.decorateKey(bytes("key"));
}
@@ -219,7 +218,7 @@
new AbstractUnfilteredRowIterator(metadata,
key,
DeletionTime.LIVE,
- metadata.partitionColumns(),
+ metadata.regularAndStaticColumns(),
Rows.EMPTY_STATIC_ROW,
isReversedOrder,
EncodingStats.NO_STATS)
@@ -230,7 +229,7 @@
}
};
- return new SingletonUnfilteredPartitionIterator(rowIter, false);
+ return new SingletonUnfilteredPartitionIterator(rowIter);
}
private RangeTombstoneBoundMarker bound(ClusteringPrefix.Kind kind,
diff --git a/test/unit/org/apache/cassandra/db/repair/AbstractPendingAntiCompactionTest.java b/test/unit/org/apache/cassandra/db/repair/AbstractPendingAntiCompactionTest.java
new file mode 100644
index 0000000..62b7db1
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/repair/AbstractPendingAntiCompactionTest.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.repair;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.UUID;
+
+import com.google.common.collect.Sets;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Ignore;
+
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.ColumnIdentifier;
+import org.apache.cassandra.cql3.QueryProcessor;
+import org.apache.cassandra.cql3.statements.schema.CreateTableStatement;
+import org.apache.cassandra.cql3.statements.schema.IndexTarget;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.repair.AbstractRepairTest;
+import org.apache.cassandra.repair.consistent.LocalSessionAccessor;
+import org.apache.cassandra.schema.IndexMetadata;
+import org.apache.cassandra.schema.Indexes;
+import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.service.ActiveRepairService;
+
+@Ignore
+public abstract class AbstractPendingAntiCompactionTest
+{
+
+ static final Collection<Range<Token>> FULL_RANGE;
+ static final Collection<Range<Token>> NO_RANGES = Collections.emptyList();
+ static InetAddressAndPort local;
+
+ static
+ {
+ DatabaseDescriptor.daemonInitialization();
+ Token minToken = DatabaseDescriptor.getPartitioner().getMinimumToken();
+ FULL_RANGE = Collections.singleton(new Range<>(minToken, minToken));
+ }
+
+ String ks;
+ final String tbl = "tbl";
+ final String tbl2 = "tbl2";
+
+ TableMetadata cfm;
+ ColumnFamilyStore cfs;
+ ColumnFamilyStore cfs2;
+
+ @BeforeClass
+ public static void setupClass() throws Throwable
+ {
+ SchemaLoader.prepareServer();
+ local = InetAddressAndPort.getByName("127.0.0.1");
+ ActiveRepairService.instance.consistent.local.start();
+ }
+
+ @Before
+ public void setup()
+ {
+ ks = "ks_" + System.currentTimeMillis();
+ cfm = CreateTableStatement.parse(String.format("CREATE TABLE %s.%s (k INT PRIMARY KEY, v INT)", ks, tbl), ks).build();
+
+ Indexes.Builder indexes = Indexes.builder();
+ indexes.add(IndexMetadata.fromIndexTargets(Collections.singletonList(new IndexTarget(new ColumnIdentifier("v", true),
+ IndexTarget.Type.VALUES)),
+ tbl2 + "_idx",
+ IndexMetadata.Kind.COMPOSITES, Collections.emptyMap()));
+
+ TableMetadata cfm2 = CreateTableStatement.parse(String.format("CREATE TABLE %s.%s (k INT PRIMARY KEY, v INT)", ks, tbl2), ks).indexes(indexes.build()).build();
+
+ SchemaLoader.createKeyspace(ks, KeyspaceParams.simple(1), cfm, cfm2);
+ cfs = Schema.instance.getColumnFamilyStoreInstance(cfm.id);
+ cfs2 = Schema.instance.getColumnFamilyStoreInstance(cfm2.id);
+ }
+
+ void makeSSTables(int num)
+ {
+ makeSSTables(num, cfs, 2);
+ }
+
+ void makeSSTables(int num, ColumnFamilyStore cfs, int rowsPerSSTable)
+ {
+ for (int i = 0; i < num; i++)
+ {
+ int val = i * rowsPerSSTable; // multiplied to prevent ranges from overlapping
+ for (int j = 0; j < rowsPerSSTable; j++)
+ QueryProcessor.executeInternal(String.format("INSERT INTO %s.%s (k, v) VALUES (?, ?)", ks, cfs.getTableName()), val + j, val + j);
+ cfs.forceBlockingFlush();
+ }
+ Assert.assertEquals(num, cfs.getLiveSSTables().size());
+ }
+
+ UUID prepareSession()
+ {
+ UUID sessionID = AbstractRepairTest.registerSession(cfs, true, true);
+ LocalSessionAccessor.prepareUnsafe(sessionID, AbstractRepairTest.COORDINATOR, Sets.newHashSet(AbstractRepairTest.COORDINATOR));
+ return sessionID;
+ }
+
+}
diff --git a/test/unit/org/apache/cassandra/db/repair/CompactionManagerGetSSTablesForValidationTest.java b/test/unit/org/apache/cassandra/db/repair/CompactionManagerGetSSTablesForValidationTest.java
new file mode 100644
index 0000000..3b29cc5
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/repair/CompactionManagerGetSSTablesForValidationTest.java
@@ -0,0 +1,181 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.repair;
+
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.Set;
+import java.util.UUID;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.cql3.statements.schema.CreateTableStatement;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.cql3.QueryProcessor;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.streaming.PreviewKind;
+import org.apache.cassandra.repair.RepairJobDesc;
+import org.apache.cassandra.repair.Validator;
+import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.service.ActiveRepairService;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.UUIDGen;
+
+import static org.apache.cassandra.db.repair.CassandraValidationIterator.getSSTablesToValidate;
+
+/**
+ * Tests correct sstables are returned from CompactionManager.getSSTablesForValidation
+ * for consistent, legacy incremental, and full repairs
+ */
+public class CompactionManagerGetSSTablesForValidationTest
+{
+ private String ks;
+ private static final String tbl = "tbl";
+ private ColumnFamilyStore cfs;
+ private static InetAddressAndPort coordinator;
+
+ private static Token MT;
+
+ private SSTableReader repaired;
+ private SSTableReader unrepaired;
+ private SSTableReader pendingRepair;
+
+ private UUID sessionID;
+ private RepairJobDesc desc;
+
+ @BeforeClass
+ public static void setupClass() throws Exception
+ {
+ SchemaLoader.prepareServer();
+ coordinator = InetAddressAndPort.getByName("10.0.0.1");
+ MT = DatabaseDescriptor.getPartitioner().getMinimumToken();
+ }
+
+ @Before
+ public void setup() throws Exception
+ {
+ ks = "ks_" + System.currentTimeMillis();
+ TableMetadata cfm = CreateTableStatement.parse(String.format("CREATE TABLE %s.%s (k INT PRIMARY KEY, v INT)", ks, tbl), ks).build();
+ SchemaLoader.createKeyspace(ks, KeyspaceParams.simple(1), cfm);
+ cfs = Schema.instance.getColumnFamilyStoreInstance(cfm.id);
+ }
+
+ private void makeSSTables()
+ {
+ for (int i=0; i<3; i++)
+ {
+ QueryProcessor.executeInternal(String.format("INSERT INTO %s.%s (k, v) VALUES(?, ?)", ks, tbl), i, i);
+ cfs.forceBlockingFlush();
+ }
+ Assert.assertEquals(3, cfs.getLiveSSTables().size());
+
+ }
+
+ private void registerRepair(boolean incremental) throws Exception
+ {
+ sessionID = UUIDGen.getTimeUUID();
+ Range<Token> range = new Range<>(MT, MT);
+ ActiveRepairService.instance.registerParentRepairSession(sessionID,
+ coordinator,
+ Lists.newArrayList(cfs),
+ Sets.newHashSet(range),
+ incremental,
+ incremental ? System.currentTimeMillis() : ActiveRepairService.UNREPAIRED_SSTABLE,
+ true,
+ PreviewKind.NONE);
+ desc = new RepairJobDesc(sessionID, UUIDGen.getTimeUUID(), ks, tbl, Collections.singleton(range));
+ }
+
+ private void modifySSTables() throws Exception
+ {
+ Iterator<SSTableReader> iter = cfs.getLiveSSTables().iterator();
+
+ repaired = iter.next();
+ repaired.descriptor.getMetadataSerializer().mutateRepairMetadata(repaired.descriptor, System.currentTimeMillis(), null, false);
+ repaired.reloadSSTableMetadata();
+
+ pendingRepair = iter.next();
+ pendingRepair.descriptor.getMetadataSerializer().mutateRepairMetadata(pendingRepair.descriptor, ActiveRepairService.UNREPAIRED_SSTABLE, sessionID, false);
+ pendingRepair.reloadSSTableMetadata();
+
+ unrepaired = iter.next();
+
+ Assert.assertFalse(iter.hasNext());
+ }
+
+ @Test
+ public void consistentRepair() throws Exception
+ {
+ makeSSTables();
+ registerRepair(true);
+ modifySSTables();
+
+ // get sstables for repair
+ Validator validator = new Validator(desc, coordinator, FBUtilities.nowInSeconds(), true, PreviewKind.NONE);
+ Set<SSTableReader> sstables = Sets.newHashSet(getSSTablesToValidate(cfs, validator.desc.ranges, validator.desc.parentSessionId, validator.isIncremental));
+ Assert.assertNotNull(sstables);
+ Assert.assertEquals(1, sstables.size());
+ Assert.assertTrue(sstables.contains(pendingRepair));
+ }
+
+ @Test
+ public void legacyIncrementalRepair() throws Exception
+ {
+ makeSSTables();
+ registerRepair(true);
+ modifySSTables();
+
+ // get sstables for repair
+ Validator validator = new Validator(desc, coordinator, FBUtilities.nowInSeconds(), false, PreviewKind.NONE);
+ Set<SSTableReader> sstables = Sets.newHashSet(getSSTablesToValidate(cfs, validator.desc.ranges, validator.desc.parentSessionId, validator.isIncremental));
+ Assert.assertNotNull(sstables);
+ Assert.assertEquals(2, sstables.size());
+ Assert.assertTrue(sstables.contains(pendingRepair));
+ Assert.assertTrue(sstables.contains(unrepaired));
+ }
+
+ @Test
+ public void fullRepair() throws Exception
+ {
+ makeSSTables();
+ registerRepair(false);
+ modifySSTables();
+
+ // get sstables for repair
+ Validator validator = new Validator(desc, coordinator, FBUtilities.nowInSeconds(), false, PreviewKind.NONE);
+ Set<SSTableReader> sstables = Sets.newHashSet(getSSTablesToValidate(cfs, validator.desc.ranges, validator.desc.parentSessionId, validator.isIncremental));
+ Assert.assertNotNull(sstables);
+ Assert.assertEquals(3, sstables.size());
+ Assert.assertTrue(sstables.contains(pendingRepair));
+ Assert.assertTrue(sstables.contains(unrepaired));
+ Assert.assertTrue(sstables.contains(repaired));
+ }
+}
diff --git a/test/unit/org/apache/cassandra/db/repair/PendingAntiCompactionBytemanTest.java b/test/unit/org/apache/cassandra/db/repair/PendingAntiCompactionBytemanTest.java
new file mode 100644
index 0000000..127a0a4
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/repair/PendingAntiCompactionBytemanTest.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.repair;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.UUID;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+import com.google.common.collect.Lists;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.apache.cassandra.db.compaction.CompactionInterruptedException;
+import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.locator.RangesAtEndpoint;
+import org.apache.cassandra.locator.Replica;
+import org.jboss.byteman.contrib.bmunit.BMRule;
+import org.jboss.byteman.contrib.bmunit.BMRules;
+import org.jboss.byteman.contrib.bmunit.BMUnitRunner;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+@RunWith(BMUnitRunner.class)
+public class PendingAntiCompactionBytemanTest extends AbstractPendingAntiCompactionTest
+{
+ @BMRules(rules = { @BMRule(name = "Throw exception anticompaction",
+ targetClass = "Range$OrderedRangeContainmentChecker",
+ targetMethod = "test",
+ action = "throw new org.apache.cassandra.db.compaction.CompactionInterruptedException(null);")} )
+ @Test
+ public void testExceptionAnticompaction() throws InterruptedException
+ {
+ cfs.disableAutoCompaction();
+ cfs2.disableAutoCompaction();
+ ExecutorService es = Executors.newFixedThreadPool(1);
+ makeSSTables(4, cfs, 5);
+ makeSSTables(4, cfs2, 5);
+ List<Range<Token>> ranges = new ArrayList<>();
+
+ for (SSTableReader sstable : cfs.getLiveSSTables())
+ {
+ ranges.add(new Range<>(sstable.first.getToken(), sstable.last.getToken()));
+ }
+ UUID prsid = prepareSession();
+ try
+ {
+ PendingAntiCompaction pac = new PendingAntiCompaction(prsid, Lists.newArrayList(cfs, cfs2), atEndpoint(ranges, NO_RANGES), es, () -> false);
+ pac.run().get();
+ fail("PAC should throw exception when anticompaction throws exception!");
+ }
+ catch (ExecutionException e)
+ {
+ assertTrue(e.getCause() instanceof CompactionInterruptedException);
+ }
+ // Note that since we fail the PAC immediately when any of the anticompactions fail we need to wait for the other
+ // AC to finish as well before asserting that we have nothing compacting.
+ CompactionManager.instance.waitForCessation(Lists.newArrayList(cfs, cfs2), (sstable) -> true);
+ // and make sure nothing is marked compacting
+ assertTrue(cfs.getTracker().getCompacting().isEmpty());
+ assertTrue(cfs2.getTracker().getCompacting().isEmpty());
+ assertEquals(4, cfs.getLiveSSTables().size());
+ assertEquals(4, cfs2.getLiveSSTables().size());
+ }
+
+ private static RangesAtEndpoint atEndpoint(Collection<Range<Token>> full, Collection<Range<Token>> trans)
+ {
+ RangesAtEndpoint.Builder builder = RangesAtEndpoint.builder(local);
+ for (Range<Token> range : full)
+ builder.add(new Replica(local, range, true));
+
+ for (Range<Token> range : trans)
+ builder.add(new Replica(local, range, false));
+
+ return builder.build();
+ }
+}
diff --git a/test/unit/org/apache/cassandra/db/repair/PendingAntiCompactionTest.java b/test/unit/org/apache/cassandra/db/repair/PendingAntiCompactionTest.java
new file mode 100644
index 0000000..1c5c245
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/repair/PendingAntiCompactionTest.java
@@ -0,0 +1,762 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.repair;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.stream.Collectors;
+
+import javax.annotation.Nullable;
+
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.common.util.concurrent.FutureCallback;
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.ListenableFutureTask;
+import com.google.common.util.concurrent.ListeningExecutorService;
+import com.google.common.util.concurrent.MoreExecutors;
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.QueryProcessor;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.compaction.AbstractPendingRepairTest;
+import org.apache.cassandra.db.compaction.CompactionController;
+import org.apache.cassandra.db.compaction.CompactionInfo;
+import org.apache.cassandra.db.compaction.CompactionInterruptedException;
+import org.apache.cassandra.db.compaction.CompactionIterator;
+import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.db.compaction.OperationType;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.dht.ByteOrderedPartitioner;
+import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.io.sstable.ISSTableScanner;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.RangesAtEndpoint;
+import org.apache.cassandra.locator.Replica;
+import org.apache.cassandra.repair.consistent.LocalSessionAccessor;
+import org.apache.cassandra.schema.MockSchema;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.service.ActiveRepairService;
+import org.apache.cassandra.streaming.PreviewKind;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.UUIDGen;
+import org.apache.cassandra.utils.WrappedRunnable;
+import org.apache.cassandra.utils.concurrent.Transactional;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+public class PendingAntiCompactionTest extends AbstractPendingAntiCompactionTest
+{
+ static final Logger logger = LoggerFactory.getLogger(PendingAntiCompactionTest.class);
+
+ private static class InstrumentedAcquisitionCallback extends PendingAntiCompaction.AcquisitionCallback
+ {
+ public InstrumentedAcquisitionCallback(UUID parentRepairSession, RangesAtEndpoint ranges)
+ {
+ super(parentRepairSession, ranges, () -> false);
+ }
+
+ Set<TableId> submittedCompactions = new HashSet<>();
+
+ ListenableFuture<?> submitPendingAntiCompaction(PendingAntiCompaction.AcquireResult result)
+ {
+ submittedCompactions.add(result.cfs.metadata.id);
+ result.abort(); // prevent ref leak complaints
+ return ListenableFutureTask.create(() -> {}, null);
+ }
+ }
+
+ /**
+ * verify the pending anti compaction happy path
+ */
+ @Test
+ public void successCase() throws Exception
+ {
+ Assert.assertSame(ByteOrderedPartitioner.class, DatabaseDescriptor.getPartitioner().getClass());
+ cfs.disableAutoCompaction();
+
+ // create 2 sstables, one that will be split, and another that will be moved
+ for (int i = 0; i < 8; i++)
+ {
+ QueryProcessor.executeInternal(String.format("INSERT INTO %s.%s (k, v) VALUES (?, ?)", ks, tbl), i, i);
+ }
+ cfs.forceBlockingFlush();
+ for (int i = 8; i < 12; i++)
+ {
+ QueryProcessor.executeInternal(String.format("INSERT INTO %s.%s (k, v) VALUES (?, ?)", ks, tbl), i, i);
+ }
+ cfs.forceBlockingFlush();
+ assertEquals(2, cfs.getLiveSSTables().size());
+
+ Token left = ByteOrderedPartitioner.instance.getToken(ByteBufferUtil.bytes((int) 6));
+ Token right = ByteOrderedPartitioner.instance.getToken(ByteBufferUtil.bytes((int) 16));
+ List<ColumnFamilyStore> tables = Lists.newArrayList(cfs);
+ Collection<Range<Token>> ranges = Collections.singleton(new Range<>(left, right));
+
+ // create a session so the anti compaction can fine it
+ UUID sessionID = UUIDGen.getTimeUUID();
+ ActiveRepairService.instance.registerParentRepairSession(sessionID, InetAddressAndPort.getLocalHost(), tables, ranges, true, 1, true, PreviewKind.NONE);
+
+ PendingAntiCompaction pac;
+ ExecutorService executor = Executors.newSingleThreadExecutor();
+ try
+ {
+ pac = new PendingAntiCompaction(sessionID, tables, atEndpoint(ranges, NO_RANGES), executor, () -> false);
+ pac.run().get();
+ }
+ finally
+ {
+ executor.shutdown();
+ }
+
+ assertEquals(3, cfs.getLiveSSTables().size());
+ int pendingRepair = 0;
+ for (SSTableReader sstable : cfs.getLiveSSTables())
+ {
+ if (sstable.isPendingRepair())
+ pendingRepair++;
+ }
+ assertEquals(2, pendingRepair);
+ }
+
+ @Test
+ public void acquisitionSuccess() throws Exception
+ {
+ cfs.disableAutoCompaction();
+ makeSSTables(6);
+ List<SSTableReader> sstables = new ArrayList<>(cfs.getLiveSSTables());
+ List<SSTableReader> expected = sstables.subList(0, 3);
+ Collection<Range<Token>> ranges = new HashSet<>();
+ for (SSTableReader sstable : expected)
+ {
+ ranges.add(new Range<>(sstable.first.getToken(), sstable.last.getToken()));
+ }
+
+ PendingAntiCompaction.AcquisitionCallable acquisitionCallable = new PendingAntiCompaction.AcquisitionCallable(cfs, ranges, UUIDGen.getTimeUUID(), 0, 0);
+
+ logger.info("SSTables: {}", sstables);
+ logger.info("Expected: {}", expected);
+ PendingAntiCompaction.AcquireResult result = acquisitionCallable.call();
+ assertNotNull(result);
+ logger.info("Originals: {}", result.txn.originals());
+ assertEquals(3, result.txn.originals().size());
+ for (SSTableReader sstable : expected)
+ {
+ logger.info("Checking {}", sstable);
+ assertTrue(result.txn.originals().contains(sstable));
+ }
+
+ assertEquals(Transactional.AbstractTransactional.State.IN_PROGRESS, result.txn.state());
+ result.abort();
+ }
+
+ @Test
+ public void repairedSSTablesAreNotAcquired() throws Exception
+ {
+ cfs.disableAutoCompaction();
+ makeSSTables(2);
+
+ List<SSTableReader> sstables = new ArrayList<>(cfs.getLiveSSTables());
+ assertEquals(2, sstables.size());
+ SSTableReader repaired = sstables.get(0);
+ SSTableReader unrepaired = sstables.get(1);
+ assertTrue(repaired.intersects(FULL_RANGE));
+ assertTrue(unrepaired.intersects(FULL_RANGE));
+
+ repaired.descriptor.getMetadataSerializer().mutateRepairMetadata(repaired.descriptor, 1, null, false);
+ repaired.reloadSSTableMetadata();
+
+ PendingAntiCompaction.AcquisitionCallable acquisitionCallable = new PendingAntiCompaction.AcquisitionCallable(cfs, FULL_RANGE, UUIDGen.getTimeUUID(), 0, 0);
+ PendingAntiCompaction.AcquireResult result = acquisitionCallable.call();
+ assertNotNull(result);
+
+ logger.info("Originals: {}", result.txn.originals());
+ assertEquals(1, result.txn.originals().size());
+ assertTrue(result.txn.originals().contains(unrepaired));
+ result.abort(); // release sstable refs
+ }
+
+ @Test
+ public void finalizedPendingRepairSSTablesAreNotAcquired() throws Exception
+ {
+ cfs.disableAutoCompaction();
+ makeSSTables(2);
+
+ List<SSTableReader> sstables = new ArrayList<>(cfs.getLiveSSTables());
+ assertEquals(2, sstables.size());
+ SSTableReader repaired = sstables.get(0);
+ SSTableReader unrepaired = sstables.get(1);
+ assertTrue(repaired.intersects(FULL_RANGE));
+ assertTrue(unrepaired.intersects(FULL_RANGE));
+
+ UUID sessionId = prepareSession();
+ LocalSessionAccessor.finalizeUnsafe(sessionId);
+ repaired.descriptor.getMetadataSerializer().mutateRepairMetadata(repaired.descriptor, 0, sessionId, false);
+ repaired.reloadSSTableMetadata();
+ assertTrue(repaired.isPendingRepair());
+
+ PendingAntiCompaction.AcquisitionCallable acquisitionCallable = new PendingAntiCompaction.AcquisitionCallable(cfs, FULL_RANGE, UUIDGen.getTimeUUID(), 0, 0);
+ PendingAntiCompaction.AcquireResult result = acquisitionCallable.call();
+ assertNotNull(result);
+
+ logger.info("Originals: {}", result.txn.originals());
+ assertEquals(1, result.txn.originals().size());
+ assertTrue(result.txn.originals().contains(unrepaired));
+ result.abort(); // releases sstable refs
+ }
+
+ @Test
+ public void conflictingSessionAcquisitionFailure() throws Exception
+ {
+ cfs.disableAutoCompaction();
+ makeSSTables(2);
+
+ List<SSTableReader> sstables = new ArrayList<>(cfs.getLiveSSTables());
+ assertEquals(2, sstables.size());
+ SSTableReader repaired = sstables.get(0);
+ SSTableReader unrepaired = sstables.get(1);
+ assertTrue(repaired.intersects(FULL_RANGE));
+ assertTrue(unrepaired.intersects(FULL_RANGE));
+
+ UUID sessionId = prepareSession();
+ repaired.descriptor.getMetadataSerializer().mutateRepairMetadata(repaired.descriptor, 0, sessionId, false);
+ repaired.reloadSSTableMetadata();
+ assertTrue(repaired.isPendingRepair());
+
+ PendingAntiCompaction.AcquisitionCallable acquisitionCallable = new PendingAntiCompaction.AcquisitionCallable(cfs, FULL_RANGE, UUIDGen.getTimeUUID(), 0, 0);
+ PendingAntiCompaction.AcquireResult result = acquisitionCallable.call();
+ assertNull(result);
+ }
+
+ @Test
+ public void pendingRepairNoSSTablesExist() throws Exception
+ {
+ cfs.disableAutoCompaction();
+
+ assertEquals(0, cfs.getLiveSSTables().size());
+
+ PendingAntiCompaction.AcquisitionCallable acquisitionCallable = new PendingAntiCompaction.AcquisitionCallable(cfs, FULL_RANGE, UUIDGen.getTimeUUID(), 0, 0);
+ PendingAntiCompaction.AcquireResult result = acquisitionCallable.call();
+ assertNotNull(result);
+
+ result.abort(); // There's nothing to release, but we should exit cleanly
+ }
+
+ /**
+ * anti compaction task should be submitted if everything is ok
+ */
+ @Test
+ public void callbackSuccess() throws Exception
+ {
+ cfs.disableAutoCompaction();
+ makeSSTables(2);
+
+ PendingAntiCompaction.AcquisitionCallable acquisitionCallable = new PendingAntiCompaction.AcquisitionCallable(cfs, FULL_RANGE, UUIDGen.getTimeUUID(), 0, 0);
+ PendingAntiCompaction.AcquireResult result = acquisitionCallable.call();
+ assertNotNull(result);
+
+ InstrumentedAcquisitionCallback cb = new InstrumentedAcquisitionCallback(UUIDGen.getTimeUUID(), atEndpoint(FULL_RANGE, NO_RANGES));
+ assertTrue(cb.submittedCompactions.isEmpty());
+ cb.apply(Lists.newArrayList(result));
+
+ assertEquals(1, cb.submittedCompactions.size());
+ assertTrue(cb.submittedCompactions.contains(cfm.id));
+ }
+
+ /**
+ * If one of the supplied AcquireResults is null, either an Exception was thrown, or
+ * we couldn't get a transaction for the sstables. In either case we need to cancel the repair, and release
+ * any sstables acquired for other tables
+ */
+ @Test
+ public void callbackNullResult() throws Exception
+ {
+ cfs.disableAutoCompaction();
+ makeSSTables(2);
+
+ PendingAntiCompaction.AcquisitionCallable acquisitionCallable = new PendingAntiCompaction.AcquisitionCallable(cfs, FULL_RANGE, UUIDGen.getTimeUUID(), 0, 0);
+ PendingAntiCompaction.AcquireResult result = acquisitionCallable.call();
+ assertNotNull(result);
+ assertEquals(Transactional.AbstractTransactional.State.IN_PROGRESS, result.txn.state());
+
+ InstrumentedAcquisitionCallback cb = new InstrumentedAcquisitionCallback(UUIDGen.getTimeUUID(), atEndpoint(FULL_RANGE, Collections.emptyList()));
+ assertTrue(cb.submittedCompactions.isEmpty());
+ cb.apply(Lists.newArrayList(result, null));
+
+ assertTrue(cb.submittedCompactions.isEmpty());
+ assertEquals(Transactional.AbstractTransactional.State.ABORTED, result.txn.state());
+ }
+
+ /**
+ * If an AcquireResult has a null txn, there were no sstables to acquire references
+ * for, so no anti compaction should have been submitted.
+ */
+ @Test
+ public void callbackNullTxn() throws Exception
+ {
+ cfs.disableAutoCompaction();
+ makeSSTables(2);
+
+ PendingAntiCompaction.AcquisitionCallable acquisitionCallable = new PendingAntiCompaction.AcquisitionCallable(cfs, FULL_RANGE, UUIDGen.getTimeUUID(), 0, 0);
+ PendingAntiCompaction.AcquireResult result = acquisitionCallable.call();
+ assertNotNull(result);
+
+ ColumnFamilyStore cfs2 = Schema.instance.getColumnFamilyStoreInstance(Schema.instance.getTableMetadata("system", "peers").id);
+ PendingAntiCompaction.AcquireResult fakeResult = new PendingAntiCompaction.AcquireResult(cfs2, null, null);
+
+ InstrumentedAcquisitionCallback cb = new InstrumentedAcquisitionCallback(UUIDGen.getTimeUUID(), atEndpoint(FULL_RANGE, NO_RANGES));
+ assertTrue(cb.submittedCompactions.isEmpty());
+ cb.apply(Lists.newArrayList(result, fakeResult));
+
+ assertEquals(1, cb.submittedCompactions.size());
+ assertTrue(cb.submittedCompactions.contains(cfm.id));
+ assertFalse(cb.submittedCompactions.contains(cfs2.metadata.id));
+ }
+
+
+ @Test
+ public void singleAnticompaction() throws Exception
+ {
+ cfs.disableAutoCompaction();
+ makeSSTables(2);
+
+ PendingAntiCompaction.AcquisitionCallable acquisitionCallable = new PendingAntiCompaction.AcquisitionCallable(cfs, FULL_RANGE, UUIDGen.getTimeUUID(), 0, 0);
+ PendingAntiCompaction.AcquireResult result = acquisitionCallable.call();
+ UUID sessionID = UUIDGen.getTimeUUID();
+ ActiveRepairService.instance.registerParentRepairSession(sessionID,
+ InetAddressAndPort.getByName("127.0.0.1"),
+ Lists.newArrayList(cfs),
+ FULL_RANGE,
+ true,0,
+ true,
+ PreviewKind.NONE);
+ CompactionManager.instance.performAnticompaction(result.cfs, atEndpoint(FULL_RANGE, NO_RANGES), result.refs, result.txn, sessionID, () -> false);
+
+ }
+
+ @Test (expected = CompactionInterruptedException.class)
+ public void cancelledAntiCompaction() throws Exception
+ {
+ cfs.disableAutoCompaction();
+ makeSSTables(1);
+
+ PendingAntiCompaction.AcquisitionCallable acquisitionCallable = new PendingAntiCompaction.AcquisitionCallable(cfs, FULL_RANGE, UUIDGen.getTimeUUID(), 0, 0);
+ PendingAntiCompaction.AcquireResult result = acquisitionCallable.call();
+ UUID sessionID = UUIDGen.getTimeUUID();
+ ActiveRepairService.instance.registerParentRepairSession(sessionID,
+ InetAddressAndPort.getByName("127.0.0.1"),
+ Lists.newArrayList(cfs),
+ FULL_RANGE,
+ true,0,
+ true,
+ PreviewKind.NONE);
+
+ // attempt to anti-compact the sstable in half
+ SSTableReader sstable = Iterables.getOnlyElement(cfs.getLiveSSTables());
+ Token left = cfs.getPartitioner().midpoint(sstable.first.getToken(), sstable.last.getToken());
+ Token right = sstable.last.getToken();
+ CompactionManager.instance.performAnticompaction(result.cfs,
+ atEndpoint(Collections.singleton(new Range<>(left, right)), NO_RANGES),
+ result.refs, result.txn, sessionID, () -> true);
+ }
+
+ /**
+ * Makes sure that PendingAntiCompaction fails when anticompaction throws exception
+ */
+ @Test
+ public void antiCompactionException()
+ {
+ cfs.disableAutoCompaction();
+ makeSSTables(2);
+ UUID prsid = UUID.randomUUID();
+ ListeningExecutorService es = MoreExecutors.listeningDecorator(MoreExecutors.newDirectExecutorService());
+ PendingAntiCompaction pac = new PendingAntiCompaction(prsid, Collections.singleton(cfs), atEndpoint(FULL_RANGE, NO_RANGES), es, () -> false) {
+ @Override
+ protected AcquisitionCallback getAcquisitionCallback(UUID prsId, RangesAtEndpoint tokenRanges)
+ {
+ return new AcquisitionCallback(prsid, tokenRanges, () -> false)
+ {
+ @Override
+ ListenableFuture<?> submitPendingAntiCompaction(AcquireResult result)
+ {
+ Runnable r = new WrappedRunnable()
+ {
+ protected void runMayThrow()
+ {
+ throw new CompactionInterruptedException(null);
+ }
+ };
+ return es.submit(r);
+ }
+ };
+ }
+ };
+ ListenableFuture<?> fut = pac.run();
+ try
+ {
+ fut.get();
+ fail("Should throw exception");
+ }
+ catch(Throwable t)
+ {
+ }
+ }
+
+ @Test
+ public void testBlockedAcquisition() throws ExecutionException, InterruptedException, TimeoutException
+ {
+ cfs.disableAutoCompaction();
+ ExecutorService es = Executors.newFixedThreadPool(1);
+
+ makeSSTables(2);
+ UUID prsid = UUID.randomUUID();
+ Set<SSTableReader> sstables = cfs.getLiveSSTables();
+ List<ISSTableScanner> scanners = sstables.stream().map(SSTableReader::getScanner).collect(Collectors.toList());
+ try
+ {
+ try (LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.ANTICOMPACTION);
+ CompactionController controller = new CompactionController(cfs, sstables, 0);
+ CompactionIterator ci = CompactionManager.getAntiCompactionIterator(scanners, controller, 0, UUID.randomUUID(), CompactionManager.instance.active, () -> false))
+ {
+ // `ci` is our imaginary ongoing anticompaction which makes no progress until after 30s
+ // now we try to start a new AC, which will try to cancel all ongoing compactions
+
+ CompactionManager.instance.active.beginCompaction(ci);
+ PendingAntiCompaction pac = new PendingAntiCompaction(prsid, Collections.singleton(cfs), atEndpoint(FULL_RANGE, NO_RANGES), 0, 0, es, () -> false);
+ ListenableFuture fut = pac.run();
+ try
+ {
+ fut.get(30, TimeUnit.SECONDS);
+ fail("the future should throw exception since we try to start a new anticompaction when one is already running");
+ }
+ catch (ExecutionException e)
+ {
+ assertTrue(e.getCause() instanceof PendingAntiCompaction.SSTableAcquisitionException);
+ }
+
+ assertEquals(1, getCompactionsFor(cfs).size());
+ for (CompactionInfo.Holder holder : getCompactionsFor(cfs))
+ assertFalse(holder.isStopRequested());
+ }
+ }
+ finally
+ {
+ es.shutdown();
+ ISSTableScanner.closeAllAndPropagate(scanners, null);
+ }
+ }
+
+ private List<CompactionInfo.Holder> getCompactionsFor(ColumnFamilyStore cfs)
+ {
+ List<CompactionInfo.Holder> compactions = new ArrayList<>();
+ for (CompactionInfo.Holder holder : CompactionManager.instance.active.getCompactions())
+ {
+ if (holder.getCompactionInfo().getTableMetadata().equals(cfs.metadata()))
+ compactions.add(holder);
+ }
+ return compactions;
+ }
+
+ @Test
+ public void testUnblockedAcquisition() throws ExecutionException, InterruptedException
+ {
+ cfs.disableAutoCompaction();
+ ExecutorService es = Executors.newFixedThreadPool(1);
+ makeSSTables(2);
+ UUID prsid = prepareSession();
+ Set<SSTableReader> sstables = cfs.getLiveSSTables();
+ List<ISSTableScanner> scanners = sstables.stream().map(SSTableReader::getScanner).collect(Collectors.toList());
+ try
+ {
+ try (LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.ANTICOMPACTION);
+ CompactionController controller = new CompactionController(cfs, sstables, 0);
+ CompactionIterator ci = new CompactionIterator(OperationType.COMPACTION, scanners, controller, 0, UUID.randomUUID()))
+ {
+ // `ci` is our imaginary ongoing anticompaction which makes no progress until after 5s
+ // now we try to start a new AC, which will try to cancel all ongoing compactions
+
+ CompactionManager.instance.active.beginCompaction(ci);
+ PendingAntiCompaction pac = new PendingAntiCompaction(prsid, Collections.singleton(cfs), atEndpoint(FULL_RANGE, NO_RANGES), es, () -> false);
+ ListenableFuture fut = pac.run();
+ try
+ {
+ fut.get(5, TimeUnit.SECONDS);
+ }
+ catch (TimeoutException e)
+ {
+ // expected, we wait 1 minute for compactions to get cancelled in runWithCompactionsDisabled, but we are not iterating
+ // CompactionIterator so the compaction is not actually cancelled
+ }
+ try
+ {
+ assertTrue(ci.hasNext());
+ ci.next();
+ fail("CompactionIterator should be abortable");
+ }
+ catch (CompactionInterruptedException e)
+ {
+ CompactionManager.instance.active.finishCompaction(ci);
+ txn.abort();
+ // expected
+ }
+ CountDownLatch cdl = new CountDownLatch(1);
+ Futures.addCallback(fut, new FutureCallback<Object>()
+ {
+ public void onSuccess(@Nullable Object o)
+ {
+ cdl.countDown();
+ }
+
+ public void onFailure(Throwable throwable)
+ {
+ }
+ }, MoreExecutors.directExecutor());
+ assertTrue(cdl.await(1, TimeUnit.MINUTES));
+ }
+ }
+ finally
+ {
+ es.shutdown();
+ }
+ }
+
+ @Test
+ public void testSSTablePredicateOngoingAntiCompaction()
+ {
+ ColumnFamilyStore cfs = MockSchema.newCFS();
+ cfs.disableAutoCompaction();
+ List<SSTableReader> sstables = new ArrayList<>();
+ List<SSTableReader> repairedSSTables = new ArrayList<>();
+ List<SSTableReader> pendingSSTables = new ArrayList<>();
+ for (int i = 1; i <= 10; i++)
+ {
+ SSTableReader sstable = MockSchema.sstable(i, i * 10, i * 10 + 9, cfs);
+ sstables.add(sstable);
+ }
+ for (int i = 1; i <= 10; i++)
+ {
+ SSTableReader sstable = MockSchema.sstable(i + 10, i * 10, i * 10 + 9, cfs);
+ AbstractPendingRepairTest.mutateRepaired(sstable, System.currentTimeMillis());
+ repairedSSTables.add(sstable);
+ }
+ for (int i = 1; i <= 10; i++)
+ {
+ SSTableReader sstable = MockSchema.sstable(i + 20, i * 10, i * 10 + 9, cfs);
+ AbstractPendingRepairTest.mutateRepaired(sstable, UUID.randomUUID(), false);
+ pendingSSTables.add(sstable);
+ }
+
+ cfs.addSSTables(sstables);
+ cfs.addSSTables(repairedSSTables);
+
+ // if we are compacting the non-repaired non-pending sstables, we should get an error
+ tryPredicate(cfs, sstables, null, true);
+ // make sure we don't try to grab pending or repaired sstables;
+ tryPredicate(cfs, repairedSSTables, sstables, false);
+ tryPredicate(cfs, pendingSSTables, sstables, false);
+ }
+
+ private void tryPredicate(ColumnFamilyStore cfs, List<SSTableReader> compacting, List<SSTableReader> expectedLive, boolean shouldFail)
+ {
+ CompactionInfo.Holder holder = new CompactionInfo.Holder()
+ {
+ public CompactionInfo getCompactionInfo()
+ {
+ return new CompactionInfo(cfs.metadata(), OperationType.ANTICOMPACTION, 0, 1000, UUID.randomUUID(), compacting);
+ }
+
+ public boolean isGlobal()
+ {
+ return false;
+ }
+ };
+ CompactionManager.instance.active.beginCompaction(holder);
+ try
+ {
+ PendingAntiCompaction.AntiCompactionPredicate predicate =
+ new PendingAntiCompaction.AntiCompactionPredicate(Collections.singleton(new Range<>(new Murmur3Partitioner.LongToken(0), new Murmur3Partitioner.LongToken(100))),
+ UUID.randomUUID());
+ Set<SSTableReader> live = cfs.getLiveSSTables().stream().filter(predicate).collect(Collectors.toSet());
+ if (shouldFail)
+ fail("should fail - we try to grab already anticompacting sstables for anticompaction");
+ assertEquals(live, new HashSet<>(expectedLive));
+ }
+ catch (PendingAntiCompaction.SSTableAcquisitionException e)
+ {
+ if (!shouldFail)
+ fail("We should not fail filtering sstables");
+ }
+ finally
+ {
+ CompactionManager.instance.active.finishCompaction(holder);
+ }
+ }
+
+ @Test
+ public void testRetries() throws InterruptedException, ExecutionException
+ {
+ ColumnFamilyStore cfs = MockSchema.newCFS();
+ cfs.addSSTable(MockSchema.sstable(1, true, cfs));
+ CountDownLatch cdl = new CountDownLatch(5);
+ ExecutorService es = Executors.newFixedThreadPool(1);
+ CompactionInfo.Holder holder = new CompactionInfo.Holder()
+ {
+ public CompactionInfo getCompactionInfo()
+ {
+ return new CompactionInfo(cfs.metadata(), OperationType.ANTICOMPACTION, 0, 0, UUID.randomUUID(), cfs.getLiveSSTables());
+ }
+
+ public boolean isGlobal()
+ {
+ return false;
+ }
+ };
+ try
+ {
+ PendingAntiCompaction.AntiCompactionPredicate acp = new PendingAntiCompaction.AntiCompactionPredicate(FULL_RANGE, UUID.randomUUID())
+ {
+ @Override
+ public boolean apply(SSTableReader sstable)
+ {
+ cdl.countDown();
+ if (cdl.getCount() > 0)
+ throw new PendingAntiCompaction.SSTableAcquisitionException("blah");
+ return true;
+ }
+ };
+ CompactionManager.instance.active.beginCompaction(holder);
+ PendingAntiCompaction.AcquisitionCallable acquisitionCallable = new PendingAntiCompaction.AcquisitionCallable(cfs, UUID.randomUUID(), 10, 1, acp);
+ Future f = es.submit(acquisitionCallable);
+ cdl.await();
+ assertNotNull(f.get());
+ }
+ finally
+ {
+ es.shutdown();
+ CompactionManager.instance.active.finishCompaction(holder);
+ }
+ }
+
+ @Test
+ public void testRetriesTimeout() throws InterruptedException, ExecutionException
+ {
+ ColumnFamilyStore cfs = MockSchema.newCFS();
+ cfs.addSSTable(MockSchema.sstable(1, true, cfs));
+ ExecutorService es = Executors.newFixedThreadPool(1);
+ CompactionInfo.Holder holder = new CompactionInfo.Holder()
+ {
+ public CompactionInfo getCompactionInfo()
+ {
+ return new CompactionInfo(cfs.metadata(), OperationType.ANTICOMPACTION, 0, 0, UUID.randomUUID(), cfs.getLiveSSTables());
+ }
+
+ public boolean isGlobal()
+ {
+ return false;
+ }
+ };
+ try
+ {
+ PendingAntiCompaction.AntiCompactionPredicate acp = new PendingAntiCompaction.AntiCompactionPredicate(FULL_RANGE, UUID.randomUUID())
+ {
+ @Override
+ public boolean apply(SSTableReader sstable)
+ {
+ throw new PendingAntiCompaction.SSTableAcquisitionException("blah");
+ }
+ };
+ CompactionManager.instance.active.beginCompaction(holder);
+ PendingAntiCompaction.AcquisitionCallable acquisitionCallable = new PendingAntiCompaction.AcquisitionCallable(cfs, UUID.randomUUID(), 2, 1000, acp);
+ Future fut = es.submit(acquisitionCallable);
+ assertNull(fut.get());
+ }
+ finally
+ {
+ es.shutdown();
+ CompactionManager.instance.active.finishCompaction(holder);
+ }
+ }
+
+ @Test
+ public void testWith2i() throws ExecutionException, InterruptedException
+ {
+ cfs2.disableAutoCompaction();
+ makeSSTables(2, cfs2, 100);
+ ColumnFamilyStore idx = cfs2.indexManager.getAllIndexColumnFamilyStores().iterator().next();
+ ExecutorService es = Executors.newFixedThreadPool(1);
+ try
+ {
+ UUID prsid = prepareSession();
+ for (SSTableReader sstable : cfs2.getLiveSSTables())
+ assertFalse(sstable.isPendingRepair());
+
+ // mark the sstables pending, with a 2i compaction going, which should be untouched;
+ try (LifecycleTransaction txn = idx.getTracker().tryModify(idx.getLiveSSTables(), OperationType.COMPACTION))
+ {
+ PendingAntiCompaction pac = new PendingAntiCompaction(prsid, Collections.singleton(cfs2), atEndpoint(FULL_RANGE, NO_RANGES), es, () -> false);
+ pac.run().get();
+ }
+ // and make sure it succeeded;
+ for (SSTableReader sstable : cfs2.getLiveSSTables())
+ assertTrue(sstable.isPendingRepair());
+ }
+ finally
+ {
+ es.shutdown();
+ }
+ }
+
+ private static RangesAtEndpoint atEndpoint(Collection<Range<Token>> full, Collection<Range<Token>> trans)
+ {
+ RangesAtEndpoint.Builder builder = RangesAtEndpoint.builder(local);
+ for (Range<Token> range : full)
+ builder.add(new Replica(local, range, true));
+
+ for (Range<Token> range : trans)
+ builder.add(new Replica(local, range, false));
+
+ return builder.build();
+ }
+}
diff --git a/test/unit/org/apache/cassandra/db/rows/ColumnDefinitionVersionComparatorTest.java b/test/unit/org/apache/cassandra/db/rows/ColumnDefinitionVersionComparatorTest.java
deleted file mode 100644
index 97db48f..0000000
--- a/test/unit/org/apache/cassandra/db/rows/ColumnDefinitionVersionComparatorTest.java
+++ /dev/null
@@ -1,201 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.db.rows;
-
-import java.nio.ByteBuffer;
-import java.util.Set;
-
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-import org.apache.cassandra.config.ColumnDefinition;
-import org.apache.cassandra.db.marshal.*;
-
-import static java.util.Arrays.asList;
-import static org.apache.cassandra.cql3.FieldIdentifier.forUnquoted;
-import static org.apache.cassandra.utils.ByteBufferUtil.bytes;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-
-public class ColumnDefinitionVersionComparatorTest
-{
- private UserType udtWith2Fields;
- private UserType udtWith3Fields;
-
- @Before
- public void setUp()
- {
- udtWith2Fields = new UserType("ks",
- bytes("myType"),
- asList(forUnquoted("a"), forUnquoted("b")),
- asList(Int32Type.instance, Int32Type.instance),
- false);
- udtWith3Fields = new UserType("ks",
- bytes("myType"),
- asList(forUnquoted("a"), forUnquoted("b"), forUnquoted("c")),
- asList(Int32Type.instance, Int32Type.instance, Int32Type.instance),
- false);
- }
-
- @After
- public void tearDown()
- {
- udtWith2Fields = null;
- udtWith3Fields = null;
- }
-
- @Test
- public void testWithSimpleTypes()
- {
- checkComparisonResults(Int32Type.instance, BytesType.instance);
- checkComparisonResults(EmptyType.instance, BytesType.instance);
- }
-
- @Test
- public void testWithTuples()
- {
- checkComparisonResults(new TupleType(asList(Int32Type.instance, Int32Type.instance)),
- new TupleType(asList(Int32Type.instance, Int32Type.instance, Int32Type.instance)));
- }
-
- @Test
- public void testWithUDTs()
- {
- checkComparisonResults(udtWith2Fields, udtWith3Fields);
- }
-
- @Test
- public void testWithUDTsNestedWithinSet()
- {
- for (boolean isMultiCell : new boolean[]{false, true})
- {
- SetType<ByteBuffer> set1 = SetType.getInstance(udtWith2Fields, isMultiCell);
- SetType<ByteBuffer> set2 = SetType.getInstance(udtWith3Fields, isMultiCell);
- checkComparisonResults(set1, set2);
- }
- }
-
- @Test
- public void testWithUDTsNestedWithinList()
- {
- for (boolean isMultiCell : new boolean[]{false, true})
- {
- ListType<ByteBuffer> list1 = ListType.getInstance(udtWith2Fields, isMultiCell);
- ListType<ByteBuffer> list2 = ListType.getInstance(udtWith3Fields, isMultiCell);
- checkComparisonResults(list1, list2);
- }
- }
-
- @Test
- public void testWithUDTsNestedWithinMap()
- {
- for (boolean isMultiCell : new boolean[]{false, true})
- {
- MapType<ByteBuffer, Integer> map1 = MapType.getInstance(udtWith2Fields, Int32Type.instance, isMultiCell);
- MapType<ByteBuffer, Integer> map2 = MapType.getInstance(udtWith3Fields, Int32Type.instance, isMultiCell);
- checkComparisonResults(map1, map2);
- }
-
- for (boolean isMultiCell : new boolean[]{false, true})
- {
- MapType<Integer, ByteBuffer> map1 = MapType.getInstance(Int32Type.instance, udtWith2Fields, isMultiCell);
- MapType<Integer, ByteBuffer> map2 = MapType.getInstance(Int32Type.instance, udtWith3Fields, isMultiCell);
- checkComparisonResults(map1, map2);
- }
- }
-
- @Test
- public void testWithUDTsNestedWithinTuple()
- {
- TupleType tuple1 = new TupleType(asList(udtWith2Fields, Int32Type.instance));
- TupleType tuple2 = new TupleType(asList(udtWith3Fields, Int32Type.instance));
- checkComparisonResults(tuple1, tuple2);
- }
-
- @Test
- public void testWithUDTsNestedWithinComposite()
- {
- CompositeType composite1 = CompositeType.getInstance(asList(udtWith2Fields, Int32Type.instance));
- CompositeType composite2 = CompositeType.getInstance(asList(udtWith3Fields, Int32Type.instance));
- checkComparisonResults(composite1, composite2);
- }
-
- @Test
- public void testWithDeeplyNestedUDT()
- {
- for (boolean isMultiCell : new boolean[]{false, true})
- {
- ListType<Set<ByteBuffer>> list1 = ListType.getInstance(SetType.getInstance(new TupleType(asList(udtWith2Fields, Int32Type.instance)), isMultiCell), isMultiCell);
- ListType<Set<ByteBuffer>> list2 = ListType.getInstance(SetType.getInstance(new TupleType(asList(udtWith3Fields, Int32Type.instance)), isMultiCell), isMultiCell);
- checkComparisonResults(list1, list2);
- }
- }
-
- @Test
- public void testInvalidComparison()
- {
- assertInvalidComparison("Found 2 incompatible versions of column c in ks.t: one of type org.apache.cassandra.db.marshal.Int32Type and one of type org.apache.cassandra.db.marshal.UTF8Type (but both types are incompatible)",
- Int32Type.instance,
- UTF8Type.instance);
- assertInvalidComparison("Found 2 incompatible versions of column c in ks.t: one of type org.apache.cassandra.db.marshal.FrozenType(org.apache.cassandra.db.marshal.UserType(ks,6d7954797065,61:org.apache.cassandra.db.marshal.Int32Type,62:org.apache.cassandra.db.marshal.Int32Type)) and one of type org.apache.cassandra.db.marshal.Int32Type (but both types are incompatible)",
- udtWith2Fields,
- Int32Type.instance);
- assertInvalidComparison("Found 2 incompatible versions of column c in ks.t: one of type org.apache.cassandra.db.marshal.SetType(org.apache.cassandra.db.marshal.UTF8Type) and one of type org.apache.cassandra.db.marshal.SetType(org.apache.cassandra.db.marshal.InetAddressType) (but both types are incompatible)",
- SetType.getInstance(UTF8Type.instance, true),
- SetType.getInstance(InetAddressType.instance, true));
- assertInvalidComparison("Found 2 incompatible versions of column c in ks.t: one of type org.apache.cassandra.db.marshal.ListType(org.apache.cassandra.db.marshal.UTF8Type) and one of type org.apache.cassandra.db.marshal.ListType(org.apache.cassandra.db.marshal.InetAddressType) (but both types are incompatible)",
- ListType.getInstance(UTF8Type.instance, true),
- ListType.getInstance(InetAddressType.instance, true));
- assertInvalidComparison("Found 2 incompatible versions of column c in ks.t: one of type org.apache.cassandra.db.marshal.MapType(org.apache.cassandra.db.marshal.UTF8Type,org.apache.cassandra.db.marshal.IntegerType) and one of type org.apache.cassandra.db.marshal.MapType(org.apache.cassandra.db.marshal.InetAddressType,org.apache.cassandra.db.marshal.IntegerType) (but both types are incompatible)",
- MapType.getInstance(UTF8Type.instance, IntegerType.instance, true),
- MapType.getInstance(InetAddressType.instance, IntegerType.instance, true));
- assertInvalidComparison("Found 2 incompatible versions of column c in ks.t: one of type org.apache.cassandra.db.marshal.MapType(org.apache.cassandra.db.marshal.IntegerType,org.apache.cassandra.db.marshal.UTF8Type) and one of type org.apache.cassandra.db.marshal.MapType(org.apache.cassandra.db.marshal.IntegerType,org.apache.cassandra.db.marshal.InetAddressType) (but both types are incompatible)",
- MapType.getInstance(IntegerType.instance, UTF8Type.instance, true),
- MapType.getInstance(IntegerType.instance, InetAddressType.instance, true));
- }
-
- private void assertInvalidComparison(String expectedMessage, AbstractType<?> oldVersion, AbstractType<?> newVersion)
- {
- try
- {
- checkComparisonResults(oldVersion, newVersion);
- fail("comparison doesn't throw expected IllegalArgumentException: " + expectedMessage);
- }
- catch (IllegalArgumentException e)
- {
- System.out.println(e.getMessage());
- assertEquals(expectedMessage, e.getMessage());
- }
- }
-
- private void checkComparisonResults(AbstractType<?> oldVersion, AbstractType<?> newVersion)
- {
- assertEquals(0, compare(oldVersion, oldVersion));
- assertEquals(0, compare(newVersion, newVersion));
- assertEquals(-1, compare(oldVersion, newVersion));
- assertEquals(1, compare(newVersion, oldVersion));
- }
-
- private static int compare(AbstractType<?> left, AbstractType<?> right)
- {
- ColumnDefinition v1 = ColumnDefinition.regularDef("ks", "t", "c", left);
- ColumnDefinition v2 = ColumnDefinition.regularDef("ks", "t", "c", right);
- return ColumnDefinitionVersionComparator.INSTANCE.compare(v1, v2);
- }
-}
diff --git a/test/unit/org/apache/cassandra/db/rows/ColumnMetadataVersionComparatorTest.java b/test/unit/org/apache/cassandra/db/rows/ColumnMetadataVersionComparatorTest.java
new file mode 100644
index 0000000..854421a
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/rows/ColumnMetadataVersionComparatorTest.java
@@ -0,0 +1,201 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db.rows;
+
+import java.nio.ByteBuffer;
+import java.util.Set;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import org.apache.cassandra.db.marshal.*;
+import org.apache.cassandra.schema.ColumnMetadata;
+
+import static java.util.Arrays.asList;
+import static org.apache.cassandra.cql3.FieldIdentifier.forUnquoted;
+import static org.apache.cassandra.utils.ByteBufferUtil.bytes;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+public class ColumnMetadataVersionComparatorTest
+{
+ private UserType udtWith2Fields;
+ private UserType udtWith3Fields;
+
+ @Before
+ public void setUp()
+ {
+ udtWith2Fields = new UserType("ks",
+ bytes("myType"),
+ asList(forUnquoted("a"), forUnquoted("b")),
+ asList(Int32Type.instance, Int32Type.instance),
+ false);
+ udtWith3Fields = new UserType("ks",
+ bytes("myType"),
+ asList(forUnquoted("a"), forUnquoted("b"), forUnquoted("c")),
+ asList(Int32Type.instance, Int32Type.instance, Int32Type.instance),
+ false);
+ }
+
+ @After
+ public void tearDown()
+ {
+ udtWith2Fields = null;
+ udtWith3Fields = null;
+ }
+
+ @Test
+ public void testWithSimpleTypes()
+ {
+ checkComparisonResults(Int32Type.instance, BytesType.instance);
+ checkComparisonResults(EmptyType.instance, BytesType.instance);
+ }
+
+ @Test
+ public void testWithTuples()
+ {
+ checkComparisonResults(new TupleType(asList(Int32Type.instance, Int32Type.instance)),
+ new TupleType(asList(Int32Type.instance, Int32Type.instance, Int32Type.instance)));
+ }
+
+ @Test
+ public void testWithUDTs()
+ {
+ checkComparisonResults(udtWith2Fields, udtWith3Fields);
+ }
+
+ @Test
+ public void testWithUDTsNestedWithinSet()
+ {
+ for (boolean isMultiCell : new boolean[]{false, true})
+ {
+ SetType<ByteBuffer> set1 = SetType.getInstance(udtWith2Fields, isMultiCell);
+ SetType<ByteBuffer> set2 = SetType.getInstance(udtWith3Fields, isMultiCell);
+ checkComparisonResults(set1, set2);
+ }
+ }
+
+ @Test
+ public void testWithUDTsNestedWithinList()
+ {
+ for (boolean isMultiCell : new boolean[]{false, true})
+ {
+ ListType<ByteBuffer> list1 = ListType.getInstance(udtWith2Fields, isMultiCell);
+ ListType<ByteBuffer> list2 = ListType.getInstance(udtWith3Fields, isMultiCell);
+ checkComparisonResults(list1, list2);
+ }
+ }
+
+ @Test
+ public void testWithUDTsNestedWithinMap()
+ {
+ for (boolean isMultiCell : new boolean[]{false, true})
+ {
+ MapType<ByteBuffer, Integer> map1 = MapType.getInstance(udtWith2Fields, Int32Type.instance, isMultiCell);
+ MapType<ByteBuffer, Integer> map2 = MapType.getInstance(udtWith3Fields, Int32Type.instance, isMultiCell);
+ checkComparisonResults(map1, map2);
+ }
+
+ for (boolean isMultiCell : new boolean[]{false, true})
+ {
+ MapType<Integer, ByteBuffer> map1 = MapType.getInstance(Int32Type.instance, udtWith2Fields, isMultiCell);
+ MapType<Integer, ByteBuffer> map2 = MapType.getInstance(Int32Type.instance, udtWith3Fields, isMultiCell);
+ checkComparisonResults(map1, map2);
+ }
+ }
+
+ @Test
+ public void testWithUDTsNestedWithinTuple()
+ {
+ TupleType tuple1 = new TupleType(asList(udtWith2Fields, Int32Type.instance));
+ TupleType tuple2 = new TupleType(asList(udtWith3Fields, Int32Type.instance));
+ checkComparisonResults(tuple1, tuple2);
+ }
+
+ @Test
+ public void testWithUDTsNestedWithinComposite()
+ {
+ CompositeType composite1 = CompositeType.getInstance(asList(udtWith2Fields, Int32Type.instance));
+ CompositeType composite2 = CompositeType.getInstance(asList(udtWith3Fields, Int32Type.instance));
+ checkComparisonResults(composite1, composite2);
+ }
+
+ @Test
+ public void testWithDeeplyNestedUDT()
+ {
+ for (boolean isMultiCell : new boolean[]{false, true})
+ {
+ ListType<Set<ByteBuffer>> list1 = ListType.getInstance(SetType.getInstance(new TupleType(asList(udtWith2Fields, Int32Type.instance)), isMultiCell), isMultiCell);
+ ListType<Set<ByteBuffer>> list2 = ListType.getInstance(SetType.getInstance(new TupleType(asList(udtWith3Fields, Int32Type.instance)), isMultiCell), isMultiCell);
+ checkComparisonResults(list1, list2);
+ }
+ }
+
+ @Test
+ public void testInvalidComparison()
+ {
+ assertInvalidComparison("Found 2 incompatible versions of column c in ks.t: one of type org.apache.cassandra.db.marshal.Int32Type and one of type org.apache.cassandra.db.marshal.UTF8Type (but both types are incompatible)",
+ Int32Type.instance,
+ UTF8Type.instance);
+ assertInvalidComparison("Found 2 incompatible versions of column c in ks.t: one of type org.apache.cassandra.db.marshal.FrozenType(org.apache.cassandra.db.marshal.UserType(ks,6d7954797065,61:org.apache.cassandra.db.marshal.Int32Type,62:org.apache.cassandra.db.marshal.Int32Type)) and one of type org.apache.cassandra.db.marshal.Int32Type (but both types are incompatible)",
+ udtWith2Fields,
+ Int32Type.instance);
+ assertInvalidComparison("Found 2 incompatible versions of column c in ks.t: one of type org.apache.cassandra.db.marshal.SetType(org.apache.cassandra.db.marshal.UTF8Type) and one of type org.apache.cassandra.db.marshal.SetType(org.apache.cassandra.db.marshal.InetAddressType) (but both types are incompatible)",
+ SetType.getInstance(UTF8Type.instance, true),
+ SetType.getInstance(InetAddressType.instance, true));
+ assertInvalidComparison("Found 2 incompatible versions of column c in ks.t: one of type org.apache.cassandra.db.marshal.ListType(org.apache.cassandra.db.marshal.UTF8Type) and one of type org.apache.cassandra.db.marshal.ListType(org.apache.cassandra.db.marshal.InetAddressType) (but both types are incompatible)",
+ ListType.getInstance(UTF8Type.instance, true),
+ ListType.getInstance(InetAddressType.instance, true));
+ assertInvalidComparison("Found 2 incompatible versions of column c in ks.t: one of type org.apache.cassandra.db.marshal.MapType(org.apache.cassandra.db.marshal.UTF8Type,org.apache.cassandra.db.marshal.IntegerType) and one of type org.apache.cassandra.db.marshal.MapType(org.apache.cassandra.db.marshal.InetAddressType,org.apache.cassandra.db.marshal.IntegerType) (but both types are incompatible)",
+ MapType.getInstance(UTF8Type.instance, IntegerType.instance, true),
+ MapType.getInstance(InetAddressType.instance, IntegerType.instance, true));
+ assertInvalidComparison("Found 2 incompatible versions of column c in ks.t: one of type org.apache.cassandra.db.marshal.MapType(org.apache.cassandra.db.marshal.IntegerType,org.apache.cassandra.db.marshal.UTF8Type) and one of type org.apache.cassandra.db.marshal.MapType(org.apache.cassandra.db.marshal.IntegerType,org.apache.cassandra.db.marshal.InetAddressType) (but both types are incompatible)",
+ MapType.getInstance(IntegerType.instance, UTF8Type.instance, true),
+ MapType.getInstance(IntegerType.instance, InetAddressType.instance, true));
+ }
+
+ private void assertInvalidComparison(String expectedMessage, AbstractType<?> oldVersion, AbstractType<?> newVersion)
+ {
+ try
+ {
+ checkComparisonResults(oldVersion, newVersion);
+ fail("comparison doesn't throw expected IllegalArgumentException: " + expectedMessage);
+ }
+ catch (IllegalArgumentException e)
+ {
+ System.out.println(e.getMessage());
+ assertEquals(expectedMessage, e.getMessage());
+ }
+ }
+
+ private void checkComparisonResults(AbstractType<?> oldVersion, AbstractType<?> newVersion)
+ {
+ assertEquals(0, compare(oldVersion, oldVersion));
+ assertEquals(0, compare(newVersion, newVersion));
+ assertEquals(-1, compare(oldVersion, newVersion));
+ assertEquals(1, compare(newVersion, oldVersion));
+ }
+
+ private static int compare(AbstractType<?> left, AbstractType<?> right)
+ {
+ ColumnMetadata v1 = ColumnMetadata.regularColumn("ks", "t", "c", left);
+ ColumnMetadata v2 = ColumnMetadata.regularColumn("ks", "t", "c", right);
+ return ColumnMetadataVersionComparator.INSTANCE.compare(v1, v2);
+ }
+}
diff --git a/test/unit/org/apache/cassandra/db/rows/DigestBackwardCompatibilityTest.java b/test/unit/org/apache/cassandra/db/rows/DigestBackwardCompatibilityTest.java
deleted file mode 100644
index a72d397..0000000
--- a/test/unit/org/apache/cassandra/db/rows/DigestBackwardCompatibilityTest.java
+++ /dev/null
@@ -1,179 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.db.rows;
-
-import java.nio.ByteBuffer;
-import java.security.MessageDigest;
-
-import org.junit.Test;
-
-import org.apache.cassandra.Util;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
-import org.apache.cassandra.cql3.CQLTester;
-import org.apache.cassandra.db.*;
-import org.apache.cassandra.db.partitions.*;
-import org.apache.cassandra.db.context.CounterContext;
-import org.apache.cassandra.net.MessagingService;
-import org.apache.cassandra.utils.ByteBufferUtil;
-import org.apache.cassandra.utils.CounterId;
-import org.apache.cassandra.utils.FBUtilities;
-
-import static org.junit.Assert.assertEquals;
-
-/**
- * Test that digest for pre-3.0 versions are properly computed (they match the value computed on pre-3.0 nodes).
- *
- * The concreted 'hard-coded' digests this file tests against have been generated on a 2.2 node using basically
- * the same test file but with 2 modifications:
- * 1. readAndDigest is modified to work on 2.2 (the actual modification is in the method as a comment)
- * 2. the assertions are replace by simple println() of the generated digest.
- *
- * Note that we only compare against 2.2 since digests should be fixed between version before 3.0 (this would be a bug
- * of previous version otherwise).
- */
-public class DigestBackwardCompatibilityTest extends CQLTester
-{
- private ByteBuffer readAndDigest(String partitionKey)
- {
- /*
- * In 2.2, this must be replaced by:
- * ColumnFamily partition = getCurrentColumnFamilyStore().getColumnFamily(QueryFilter.getIdentityFilter(Util.dk(partitionKey), currentTable(), System.currentTimeMillis()));
- * return ColumnFamily.digest(partition);
- */
-
- ReadCommand cmd = Util.cmd(getCurrentColumnFamilyStore(), partitionKey).build();
- ImmutableBTreePartition partition = Util.getOnlyPartitionUnfiltered(cmd);
- MessageDigest digest = FBUtilities.threadLocalMD5Digest();
- UnfilteredRowIterators.digest(cmd, partition.unfilteredIterator(), digest, MessagingService.VERSION_22);
- return ByteBuffer.wrap(digest.digest());
- }
-
- private void assertDigest(String expected, ByteBuffer actual)
- {
- String toTest = ByteBufferUtil.bytesToHex(actual);
- assertEquals(String.format("[digest from 2.2] %s != %s [digest from 3.0]", expected, toTest), expected, toTest);
- }
-
- @Test
- public void testCQLTable() throws Throwable
- {
- createTable("CREATE TABLE %s (k text, t int, v1 text, v2 int, PRIMARY KEY (k, t))");
-
- String key = "someKey";
-
- for (int i = 0; i < 10; i++)
- execute("INSERT INTO %s(k, t, v1, v2) VALUES (?, ?, ?, ?) USING TIMESTAMP ? AND TTL ?", key, i, "v" + i, i, 1L, 200);
-
- // ColumnFamily(table_0 [0::false:0@1!200,0:v1:false:2@1!200,0:v2:false:4@1!200,1::false:0@1!200,1:v1:false:2@1!200,1:v2:false:4@1!200,2::false:0@1!200,2:v1:false:2@1!200,2:v2:false:4@1!200,3::false:0@1!200,3:v1:false:2@1!200,3:v2:false:4@1!200,4::false:0@1!200,4:v1:false:2@1!200,4:v2:false:4@1!200,5::false:0@1!200,5:v1:false:2@1!200,5:v2:false:4@1!200,6::false:0@1!200,6:v1:false:2@1!200,6:v2:false:4@1!200,7::false:0@1!200,7:v1:false:2@1!200,7:v2:false:4@1!200,8::false:0@1!200,8:v1:false:2@1!200,8:v2:false:4@1!200,9::false:0@1!200,9:v1:false:2@1!200,9:v2:false:4@1!200,])
- assertDigest("aa608035cf6574a97061b5c166b64939", readAndDigest(key));
-
- // This is a cell deletion
- execute("DELETE v1 FROM %s USING TIMESTAMP ? WHERE k = ? AND t = ?", 2L, key, 2);
-
- // This is a range tombstone
- execute("DELETE FROM %s USING TIMESTAMP ? WHERE k = ? AND t = ?", 3L, key, 4);
-
- // This is a partition level deletion (but we use an older tombstone so it doesn't get rid of everything and keeps the test interesting)
- execute("DELETE FROM %s USING TIMESTAMP ? WHERE k = ?", 0L, key);
-
- // ColumnFamily(table_0 -{deletedAt=0, localDeletion=1441012270, ranges=[4:_-4:!, deletedAt=3, localDeletion=1441012270]}- [0::false:0@1!200,0:v1:false:2@1!200,0:v2:false:4@1!200,1::false:0@1!200,1:v1:false:2@1!200,1:v2:false:4@1!200,2::false:0@1!200,2:v1:true:4@2,2:v2:false:4@1!200,3::false:0@1!200,3:v1:false:2@1!200,3:v2:false:4@1!200,5::false:0@1!200,5:v1:false:2@1!200,5:v2:false:4@1!200,6::false:0@1!200,6:v1:false:2@1!200,6:v2:false:4@1!200,7::false:0@1!200,7:v1:false:2@1!200,7:v2:false:4@1!200,8::false:0@1!200,8:v1:false:2@1!200,8:v2:false:4@1!200,9::false:0@1!200,9:v1:false:2@1!200,9:v2:false:4@1!200,])
- assertDigest("b5f38d2dc7b917d221f98ab1641f82bf", readAndDigest(key));
- }
-
- @Test
- public void testCompactTable() throws Throwable
- {
- createTable("CREATE TABLE %s (k text, t int, v text, PRIMARY KEY (k, t)) WITH COMPACT STORAGE");
-
- String key = "someKey";
-
- for (int i = 0; i < 10; i++)
- execute("INSERT INTO %s(k, t, v) VALUES (?, ?, ?) USING TIMESTAMP ? AND TTL ?", key, i, "v" + i, 1L, 200);
-
- assertDigest("44785ddd7c62c73287b448b6063645e5", readAndDigest(key));
-
- // This is a cell deletion
- execute("DELETE FROM %s USING TIMESTAMP ? WHERE k = ? AND t = ?", 2L, key, 2);
-
- // This is a partition level deletion (but we use an older tombstone so it doesn't get rid of everything and keeps the test interesting)
- execute("DELETE FROM %s USING TIMESTAMP ? WHERE k = ?", 0L, key);
-
- assertDigest("55d9bd6335276395d83b18eb17f9abe7", readAndDigest(key));
- }
-
- @Test
- public void testStaticCompactTable() throws Throwable
- {
- createTable("CREATE TABLE %s (k text PRIMARY KEY, v1 text, v2 int) WITH COMPACT STORAGE");
-
- String key = "someKey";
- execute("INSERT INTO %s(k, v1, v2) VALUES (?, ?, ?) USING TIMESTAMP ?", key, "v", 0, 1L);
-
- assertDigest("d2080f9f57d6edf92da1fdaaa76573d3", readAndDigest(key));
- }
-
- @Test
- public void testTableWithCollection() throws Throwable
- {
- createTable("CREATE TABLE %s (k text PRIMARY KEY, m map<text, text>)");
-
- String key = "someKey";
-
- execute("INSERT INTO %s(k, m) VALUES (?, { 'foo' : 'value1', 'bar' : 'value2' }) USING TIMESTAMP ?", key, 1L);
-
- // ColumnFamily(table_2 -{deletedAt=-9223372036854775808, localDeletion=2147483647, ranges=[m:_-m:!, deletedAt=0, localDeletion=1441012271]}- [:false:0@1,m:626172:false:6@1,m:666f6f:false:6@1,])
- assertDigest("708f3fc8bc8149cc3513eef300bf0182", readAndDigest(key));
-
- // This is a collection range tombstone
- execute("DELETE m FROM %s USING TIMESTAMP ? WHERE k = ?", 2L, key);
-
- // ColumnFamily(table_2 -{deletedAt=-9223372036854775808, localDeletion=2147483647, ranges=[m:_-m:!, deletedAt=2, localDeletion=1441012271]}- [:false:0@1,])
- assertDigest("f39937fc3ed96956ef507e81717fa5cd", readAndDigest(key));
- }
-
- @Test
- public void testCounterTable() throws Throwable
- {
- /*
- * We can't use CQL to insert counters as both the timestamp and counter ID are automatically assigned and unpredictable.
- * So we need to built it ourselves in a way that is totally equivalent between 2.2 and 3.0 which makes the test a little
- * bit less readable. In any case, the code to generate the equivalent mutation on 2.2 is:
- * ColumnFamily cf = ArrayBackedSortedColumns.factory.create(getCurrentColumnFamilyStore().metadata);
- * ByteBuffer value = CounterContext.instance().createGlobal(CounterId.fromInt(1), 1L, 42L);
- * cf.addColumn(new BufferCounterCell(CellNames.simpleSparse(new ColumnIdentifier("c", true)) , value, 0L, Long.MIN_VALUE));
- * new Mutation(KEYSPACE, ByteBufferUtil.bytes(key), cf).applyUnsafe();
- *
- * Also note that we use COMPACT STORAGE only because it has no bearing on the test and was slightly easier in 2.2 to create
- * the mutation.
- */
-
- createTable("CREATE TABLE %s (k text PRIMARY KEY, c counter) WITH COMPACT STORAGE");
-
- String key = "someKey";
-
- CFMetaData metadata = getCurrentColumnFamilyStore().metadata;
- ColumnDefinition column = metadata.getColumnDefinition(ByteBufferUtil.bytes("c"));
- ByteBuffer value = CounterContext.instance().createGlobal(CounterId.fromInt(1), 1L, 42L);
- Row row = BTreeRow.singleCellRow(Clustering.STATIC_CLUSTERING, BufferCell.live(column, 0L, value));
-
- new Mutation(PartitionUpdate.singleRowUpdate(metadata, Util.dk(key), row)).applyUnsafe();
-
- assertDigest("3a5f7b48c320538b4cd2f829e05c6db3", readAndDigest(key));
- }
-}
diff --git a/test/unit/org/apache/cassandra/db/rows/EncodingStatsTest.java b/test/unit/org/apache/cassandra/db/rows/EncodingStatsTest.java
new file mode 100644
index 0000000..1ac092a
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/rows/EncodingStatsTest.java
@@ -0,0 +1,170 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.rows;
+
+import java.util.function.Function;
+
+import com.google.common.collect.ImmutableList;
+import org.junit.Assert;
+import org.junit.Test;
+
+import org.apache.cassandra.db.LivenessInfo;
+
+import static org.quicktheories.QuickTheory.qt;
+import static org.quicktheories.generators.SourceDSL.integers;
+import static org.quicktheories.generators.SourceDSL.longs;
+
+public class EncodingStatsTest
+{
+ @Test
+ public void testCollectWithNoStats()
+ {
+ EncodingStats none = EncodingStats.merge(ImmutableList.of(
+ EncodingStats.NO_STATS,
+ EncodingStats.NO_STATS,
+ EncodingStats.NO_STATS
+ ), Function.identity());
+ Assert.assertEquals(none, EncodingStats.NO_STATS);
+ }
+
+ @Test
+ public void testCollectWithNoStatsWithEmpty()
+ {
+ EncodingStats none = EncodingStats.merge(ImmutableList.of(
+ EncodingStats.NO_STATS,
+ EncodingStats.NO_STATS,
+ new EncodingStats(LivenessInfo.NO_TIMESTAMP, LivenessInfo.NO_EXPIRATION_TIME, 0)
+ ), Function.identity());
+ Assert.assertEquals(none, EncodingStats.NO_STATS);
+ }
+
+ @Test
+ public void testCollectWithNoStatsWithTimestamp()
+ {
+ EncodingStats single = new EncodingStats(1, LivenessInfo.NO_EXPIRATION_TIME, 0);
+ EncodingStats result = EncodingStats.merge(ImmutableList.of(
+ EncodingStats.NO_STATS,
+ EncodingStats.NO_STATS,
+ single,
+ EncodingStats.NO_STATS
+ ), Function.identity());
+ Assert.assertEquals(single, result);
+ }
+
+ @Test
+ public void testCollectWithNoStatsWithExpires()
+ {
+ EncodingStats single = new EncodingStats(LivenessInfo.NO_TIMESTAMP, 1, 0);
+ EncodingStats result = EncodingStats.merge(ImmutableList.of(
+ EncodingStats.NO_STATS,
+ single,
+ EncodingStats.NO_STATS
+ ), Function.identity());
+ Assert.assertEquals(single, result);
+ }
+
+ @Test
+ public void testCollectWithNoStatsWithTTL()
+ {
+ EncodingStats single = new EncodingStats(LivenessInfo.NO_TIMESTAMP, LivenessInfo.NO_EXPIRATION_TIME, 1);
+ EncodingStats result = EncodingStats.merge(ImmutableList.of(
+ EncodingStats.NO_STATS,
+ single,
+ EncodingStats.NO_STATS
+ ), Function.identity());
+ Assert.assertEquals(single, result);
+ }
+
+ @Test
+ public void testCollectOneEach()
+ {
+ EncodingStats tsp = new EncodingStats(1, LivenessInfo.NO_EXPIRATION_TIME, 0);
+ EncodingStats exp = new EncodingStats(LivenessInfo.NO_TIMESTAMP, 1, 0);
+ EncodingStats ttl = new EncodingStats(LivenessInfo.NO_TIMESTAMP, LivenessInfo.NO_EXPIRATION_TIME, 1);
+ EncodingStats result = EncodingStats.merge(ImmutableList.of(
+ tsp,
+ exp,
+ ttl
+ ), Function.identity());
+ Assert.assertEquals(new EncodingStats(1, 1, 1), result);
+ }
+
+ @Test
+ public void testTimestamp()
+ {
+ EncodingStats one = new EncodingStats(1, LivenessInfo.NO_EXPIRATION_TIME, 0);
+ EncodingStats two = new EncodingStats(2, LivenessInfo.NO_EXPIRATION_TIME, 0);
+ EncodingStats thr = new EncodingStats(3, LivenessInfo.NO_EXPIRATION_TIME, 0);
+ EncodingStats result = EncodingStats.merge(ImmutableList.of(
+ one,
+ two,
+ thr
+ ), Function.identity());
+ Assert.assertEquals(one, result);
+ }
+
+ @Test
+ public void testExpires()
+ {
+ EncodingStats one = new EncodingStats(LivenessInfo.NO_TIMESTAMP,1, 0);
+ EncodingStats two = new EncodingStats(LivenessInfo.NO_TIMESTAMP,2, 0);
+ EncodingStats thr = new EncodingStats(LivenessInfo.NO_TIMESTAMP,3, 0);
+ EncodingStats result = EncodingStats.merge(ImmutableList.of(
+ one,
+ two,
+ thr
+ ), Function.identity());
+ Assert.assertEquals(one, result);
+ }
+
+ @Test
+ public void testTTL()
+ {
+ EncodingStats one = new EncodingStats(LivenessInfo.NO_TIMESTAMP, LivenessInfo.NO_EXPIRATION_TIME,1);
+ EncodingStats two = new EncodingStats(LivenessInfo.NO_TIMESTAMP, LivenessInfo.NO_EXPIRATION_TIME,2);
+ EncodingStats thr = new EncodingStats(LivenessInfo.NO_TIMESTAMP, LivenessInfo.NO_EXPIRATION_TIME,3);
+ EncodingStats result = EncodingStats.merge(ImmutableList.of(
+ thr,
+ one,
+ two
+ ), Function.identity());
+ Assert.assertEquals(one, result);
+ }
+
+ @Test
+ public void testEncodingStatsCollectWithNone()
+ {
+ qt().forAll(longs().between(Long.MIN_VALUE+1, Long.MAX_VALUE),
+ integers().between(0, Integer.MAX_VALUE-1),
+ integers().allPositive())
+ .asWithPrecursor(EncodingStats::new)
+ .check((timestamp, expires, ttl, stats) ->
+ {
+ EncodingStats result = EncodingStats.merge(ImmutableList.of(
+ EncodingStats.NO_STATS,
+ stats,
+ EncodingStats.NO_STATS
+ ), Function.identity());
+ return result.minTTL == ttl
+ && result.minLocalDeletionTime == expires
+ && result.minTimestamp == timestamp;
+ });
+ }
+
+}
diff --git a/test/unit/org/apache/cassandra/db/rows/RowAndDeletionMergeIteratorTest.java b/test/unit/org/apache/cassandra/db/rows/RowAndDeletionMergeIteratorTest.java
index 2f48000..f590d36 100644
--- a/test/unit/org/apache/cassandra/db/rows/RowAndDeletionMergeIteratorTest.java
+++ b/test/unit/org/apache/cassandra/db/rows/RowAndDeletionMergeIteratorTest.java
@@ -30,18 +30,18 @@
import org.junit.Test;
import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.db.ClusteringPrefix;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.filter.ColumnFilter;
import org.apache.cassandra.db.marshal.AbstractType;
-import org.apache.cassandra.config.ColumnDefinition;
import org.apache.cassandra.cql3.ColumnIdentifier;
import org.apache.cassandra.db.partitions.PartitionUpdate;
import org.apache.cassandra.db.marshal.Int32Type;
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.Util;
-import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.db.marshal.AsciiType;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.schema.KeyspaceParams;
@@ -56,23 +56,22 @@
private int nowInSeconds;
private DecoratedKey dk;
private ColumnFamilyStore cfs;
- private CFMetaData cfm;
- private ColumnDefinition defA;
+ private TableMetadata cfm;
+ private ColumnMetadata defA;
@BeforeClass
public static void defineSchema() throws ConfigurationException
{
DatabaseDescriptor.daemonInitialization();
- CFMetaData cfMetadata = CFMetaData.Builder.create(KEYSPACE1, CF_STANDARD1)
- .addPartitionKey("key", AsciiType.instance)
- .addClusteringColumn("col1", Int32Type.instance)
- .addRegularColumn("a", Int32Type.instance)
- .build();
- SchemaLoader.prepareServer();
- SchemaLoader.createKeyspace(KEYSPACE1,
- KeyspaceParams.simple(1),
- cfMetadata);
+ TableMetadata.Builder builder =
+ TableMetadata.builder(KEYSPACE1, CF_STANDARD1)
+ .addPartitionKeyColumn("key", AsciiType.instance)
+ .addClusteringColumn("col1", Int32Type.instance)
+ .addRegularColumn("a", Int32Type.instance);
+
+ SchemaLoader.prepareServer();
+ SchemaLoader.createKeyspace(KEYSPACE1, KeyspaceParams.simple(1), builder);
}
@Before
@@ -81,8 +80,8 @@
nowInSeconds = FBUtilities.nowInSeconds();
dk = Util.dk("key0");
cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD1);
- cfm = cfs.metadata;
- defA = cfm.getColumnDefinition(new ColumnIdentifier("a", true));
+ cfm = cfs.metadata();
+ defA = cfm.getColumn(new ColumnIdentifier("a", true));
}
@Test
@@ -353,7 +352,7 @@
@Test
public void testWithNoopBoundaryMarkers()
{
- PartitionUpdate update = new PartitionUpdate(cfm, dk, cfm.partitionColumns(), 1);
+ PartitionUpdate update = PartitionUpdate.emptyUpdate(cfm, dk);
RangeTombstoneList rtl = new RangeTombstoneList(cfm.comparator, 10);
rtl.add(rt(1, 2, 5, 5));
rtl.add(rt(3, 4, 5, 5));
@@ -394,11 +393,11 @@
private Iterator<Row> createRowIterator()
{
- PartitionUpdate update = new PartitionUpdate(cfm, dk, cfm.partitionColumns(), 1);
+ PartitionUpdate.Builder update = new PartitionUpdate.Builder(cfm, dk, cfm.regularAndStaticColumns(), 1);
for (int i = 0; i < 5; i++)
addRow(update, i, i);
- return update.iterator();
+ return update.build().iterator();
}
private UnfilteredRowIterator createMergeIterator(Iterator<Row> rows, Iterator<RangeTombstone> tombstones, boolean reversed)
@@ -423,14 +422,14 @@
true);
}
- private void addRow(PartitionUpdate update, int col1, int a)
+ private void addRow(PartitionUpdate.Builder update, int col1, int a)
{
update.add(BTreeRow.singleCellRow(update.metadata().comparator.make(col1), makeCell(defA, a, 0)));
}
- private Cell makeCell(ColumnDefinition columnDefinition, int value, long timestamp)
+ private Cell makeCell(ColumnMetadata columnMetadata, int value, long timestamp)
{
- return BufferCell.live(columnDefinition, timestamp, ((AbstractType)columnDefinition.cellValueType()).decompose(value));
+ return BufferCell.live(columnMetadata, timestamp, ((AbstractType) columnMetadata.cellValueType()).decompose(value));
}
private static RangeTombstone atLeast(int start, long tstamp, int delTime)
diff --git a/test/unit/org/apache/cassandra/db/rows/RowBuilder.java b/test/unit/org/apache/cassandra/db/rows/RowBuilder.java
index ede2ccd..21522a5 100644
--- a/test/unit/org/apache/cassandra/db/rows/RowBuilder.java
+++ b/test/unit/org/apache/cassandra/db/rows/RowBuilder.java
@@ -21,7 +21,7 @@
import java.util.LinkedList;
import java.util.List;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.db.Clustering;
import org.apache.cassandra.db.DeletionTime;
import org.apache.cassandra.db.LivenessInfo;
@@ -38,7 +38,7 @@
public Clustering clustering = null;
public LivenessInfo livenessInfo = null;
public Row.Deletion deletionTime = null;
- public List<Pair<ColumnDefinition, DeletionTime>> complexDeletions = new LinkedList<>();
+ public List<Pair<ColumnMetadata, DeletionTime>> complexDeletions = new LinkedList<>();
@Override
public Builder copy()
@@ -79,7 +79,7 @@
deletionTime = deletion;
}
- public void addComplexDeletion(ColumnDefinition column, DeletionTime complexDeletion)
+ public void addComplexDeletion(ColumnMetadata column, DeletionTime complexDeletion)
{
complexDeletions.add(Pair.create(column, complexDeletion));
}
diff --git a/test/unit/org/apache/cassandra/db/rows/RowsTest.java b/test/unit/org/apache/cassandra/db/rows/RowsTest.java
index accb4c9..00a9af4 100644
--- a/test/unit/org/apache/cassandra/db/rows/RowsTest.java
+++ b/test/unit/org/apache/cassandra/db/rows/RowsTest.java
@@ -33,8 +33,8 @@
import org.junit.Assert;
import org.junit.Test;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.ColumnIdentifier;
import org.apache.cassandra.db.Clustering;
@@ -50,23 +50,24 @@
{
private static final String KEYSPACE = "rows_test";
private static final String KCVM_TABLE = "kcvm";
- private static final CFMetaData kcvm;
- private static final ColumnDefinition v;
- private static final ColumnDefinition m;
+ private static final TableMetadata kcvm;
+ private static final ColumnMetadata v;
+ private static final ColumnMetadata m;
private static final Clustering c1;
static
{
DatabaseDescriptor.daemonInitialization();
- kcvm = CFMetaData.Builder.create(KEYSPACE, KCVM_TABLE)
- .addPartitionKey("k", IntegerType.instance)
- .addClusteringColumn("c", IntegerType.instance)
- .addRegularColumn("v", IntegerType.instance)
- .addRegularColumn("m", MapType.getInstance(IntegerType.instance, IntegerType.instance, true))
- .build();
+ kcvm =
+ TableMetadata.builder(KEYSPACE, KCVM_TABLE)
+ .addPartitionKeyColumn("k", IntegerType.instance)
+ .addClusteringColumn("c", IntegerType.instance)
+ .addRegularColumn("v", IntegerType.instance)
+ .addRegularColumn("m", MapType.getInstance(IntegerType.instance, IntegerType.instance, true))
+ .build();
- v = kcvm.getColumnDefinition(new ColumnIdentifier("v", false));
- m = kcvm.getColumnDefinition(new ColumnIdentifier("m", false));
+ v = kcvm.getColumn(new ColumnIdentifier("v", false));
+ m = kcvm.getColumn(new ColumnIdentifier("m", false));
c1 = kcvm.comparator.make(BigInteger.valueOf(1));
}
@@ -158,8 +159,8 @@
updates++;
}
- Map<ColumnDefinition, List<MergedPair<DeletionTime>>> complexDeletions = new HashMap<>();
- public void onComplexDeletion(int i, Clustering clustering, ColumnDefinition column, DeletionTime merged, DeletionTime original)
+ Map<ColumnMetadata, List<MergedPair<DeletionTime>>> complexDeletions = new HashMap<>();
+ public void onComplexDeletion(int i, Clustering clustering, ColumnMetadata column, DeletionTime merged, DeletionTime original)
{
updateClustering(clustering);
if (!complexDeletions.containsKey(column)) complexDeletions.put(column, new LinkedList<>());
@@ -210,7 +211,7 @@
private static Row.Builder createBuilder(Clustering c, int now, ByteBuffer vVal, ByteBuffer mKey, ByteBuffer mVal)
{
long ts = secondToTs(now);
- Row.Builder builder = BTreeRow.unsortedBuilder(now);
+ Row.Builder builder = BTreeRow.unsortedBuilder();
builder.newRow(c);
builder.addPrimaryKeyLivenessInfo(LivenessInfo.create(ts, now));
if (vVal != null)
@@ -231,7 +232,7 @@
{
int now = FBUtilities.nowInSeconds();
long ts = secondToTs(now);
- Row.Builder originalBuilder = BTreeRow.unsortedBuilder(now);
+ Row.Builder originalBuilder = BTreeRow.unsortedBuilder();
originalBuilder.newRow(c1);
LivenessInfo liveness = LivenessInfo.create(ts, now);
originalBuilder.addPrimaryKeyLivenessInfo(liveness);
@@ -260,7 +261,7 @@
{
int now = FBUtilities.nowInSeconds();
long ts = secondToTs(now);
- Row.Builder builder = BTreeRow.unsortedBuilder(now);
+ Row.Builder builder = BTreeRow.unsortedBuilder();
builder.newRow(c1);
LivenessInfo liveness = LivenessInfo.create(ts, now);
builder.addPrimaryKeyLivenessInfo(liveness);
@@ -298,7 +299,7 @@
{
int now1 = FBUtilities.nowInSeconds();
long ts1 = secondToTs(now1);
- Row.Builder r1Builder = BTreeRow.unsortedBuilder(now1);
+ Row.Builder r1Builder = BTreeRow.unsortedBuilder();
r1Builder.newRow(c1);
LivenessInfo r1Liveness = LivenessInfo.create(ts1, now1);
r1Builder.addPrimaryKeyLivenessInfo(r1Liveness);
@@ -314,7 +315,7 @@
int now2 = now1 + 1;
long ts2 = secondToTs(now2);
- Row.Builder r2Builder = BTreeRow.unsortedBuilder(now2);
+ Row.Builder r2Builder = BTreeRow.unsortedBuilder();
r2Builder.newRow(c1);
LivenessInfo r2Liveness = LivenessInfo.create(ts2, now2);
r2Builder.addPrimaryKeyLivenessInfo(r2Liveness);
@@ -330,7 +331,7 @@
Row r1 = r1Builder.build();
Row r2 = r2Builder.build();
- Row merged = Rows.merge(r1, r2, now2 + 1);
+ Row merged = Rows.merge(r1, r2);
Assert.assertEquals(r1ComplexDeletion, merged.getComplexColumnData(m).complexDeletion());
@@ -374,7 +375,7 @@
{
int now1 = FBUtilities.nowInSeconds();
long ts1 = secondToTs(now1);
- Row.Builder r1Builder = BTreeRow.unsortedBuilder(now1);
+ Row.Builder r1Builder = BTreeRow.unsortedBuilder();
r1Builder.newRow(c1);
LivenessInfo r1Liveness = LivenessInfo.create(ts1, now1);
r1Builder.addPrimaryKeyLivenessInfo(r1Liveness);
@@ -382,7 +383,7 @@
// mergedData == null
int now2 = now1 + 1;
long ts2 = secondToTs(now2);
- Row.Builder r2Builder = BTreeRow.unsortedBuilder(now2);
+ Row.Builder r2Builder = BTreeRow.unsortedBuilder();
r2Builder.newRow(c1);
LivenessInfo r2Liveness = LivenessInfo.create(ts2, now2);
r2Builder.addPrimaryKeyLivenessInfo(r2Liveness);
@@ -428,7 +429,7 @@
{
int now1 = FBUtilities.nowInSeconds();
long ts1 = secondToTs(now1);
- Row.Builder r1Builder = BTreeRow.unsortedBuilder(now1);
+ Row.Builder r1Builder = BTreeRow.unsortedBuilder();
r1Builder.newRow(c1);
LivenessInfo r1Liveness = LivenessInfo.create(ts1, now1);
r1Builder.addPrimaryKeyLivenessInfo(r1Liveness);
@@ -436,7 +437,7 @@
// mergedData == null
int now2 = now1 + 1;
long ts2 = secondToTs(now2);
- Row.Builder r2Builder = BTreeRow.unsortedBuilder(now2);
+ Row.Builder r2Builder = BTreeRow.unsortedBuilder();
r2Builder.newRow(c1);
LivenessInfo r2Liveness = LivenessInfo.create(ts2, now2);
r2Builder.addPrimaryKeyLivenessInfo(r2Liveness);
@@ -493,7 +494,7 @@
updateBuilder.addCell(expectedMCell);
RowBuilder builder = new RowBuilder();
- long td = Rows.merge(existingBuilder.build(), updateBuilder.build(), builder, now2 + 1);
+ long td = Rows.merge(existingBuilder.build(), updateBuilder.build(), builder);
Assert.assertEquals(c1, builder.clustering);
Assert.assertEquals(LivenessInfo.create(ts2, now2), builder.livenessInfo);
@@ -517,7 +518,7 @@
updateBuilder.addRowDeletion(expectedDeletion);
RowBuilder builder = new RowBuilder();
- Rows.merge(existingBuilder.build(), updateBuilder.build(), builder, now3 + 1);
+ Rows.merge(existingBuilder.build(), updateBuilder.build(), builder);
Assert.assertEquals(expectedDeletion, builder.deletionTime);
Assert.assertEquals(Collections.emptyList(), builder.complexDeletions);
@@ -541,7 +542,7 @@
updateBuilder.addRowDeletion(expectedDeletion);
RowBuilder builder = new RowBuilder();
- Rows.merge(existingBuilder.build(), updateBuilder.build(), builder, now3 + 1);
+ Rows.merge(existingBuilder.build(), updateBuilder.build(), builder);
Assert.assertEquals(expectedDeletion, builder.deletionTime);
Assert.assertEquals(LivenessInfo.EMPTY, builder.livenessInfo);
@@ -550,14 +551,14 @@
}
// Creates a dummy cell for a (regular) column for the provided name and without a cellPath.
- private static Cell liveCell(ColumnDefinition name)
+ private static Cell liveCell(ColumnMetadata name)
{
return liveCell(name, -1);
}
// Creates a dummy cell for a (regular) column for the provided name.
// If path >= 0, the cell will have a CellPath containing path as an Int32Type.
- private static Cell liveCell(ColumnDefinition name, int path)
+ private static Cell liveCell(ColumnMetadata name, int path)
{
CellPath cp = path < 0 ? null : CellPath.create(ByteBufferUtil.bytes(path));
return new BufferCell(name, 0L, Cell.NO_TTL, Cell.NO_DELETION_TIME, ByteBuffer.allocate(1), cp);
@@ -593,20 +594,21 @@
// Creates a table with
// - 3 Simple columns: a, c and e
// - 2 Complex columns: b and d
- CFMetaData metadata = CFMetaData.Builder.create("dummy_ks", "dummy_tbl")
- .addPartitionKey("k", BytesType.instance)
- .addRegularColumn("a", BytesType.instance)
- .addRegularColumn("b", MapType.getInstance(Int32Type.instance, BytesType.instance, true))
- .addRegularColumn("c", BytesType.instance)
- .addRegularColumn("d", MapType.getInstance(Int32Type.instance, BytesType.instance, true))
- .addRegularColumn("e", BytesType.instance)
- .build();
+ TableMetadata metadata =
+ TableMetadata.builder("dummy_ks", "dummy_tbl")
+ .addPartitionKeyColumn("k", BytesType.instance)
+ .addRegularColumn("a", BytesType.instance)
+ .addRegularColumn("b", MapType.getInstance(Int32Type.instance, BytesType.instance, true))
+ .addRegularColumn("c", BytesType.instance)
+ .addRegularColumn("d", MapType.getInstance(Int32Type.instance, BytesType.instance, true))
+ .addRegularColumn("e", BytesType.instance)
+ .build();
- ColumnDefinition a = metadata.getColumnDefinition(new ColumnIdentifier("a", false));
- ColumnDefinition b = metadata.getColumnDefinition(new ColumnIdentifier("b", false));
- ColumnDefinition c = metadata.getColumnDefinition(new ColumnIdentifier("c", false));
- ColumnDefinition d = metadata.getColumnDefinition(new ColumnIdentifier("d", false));
- ColumnDefinition e = metadata.getColumnDefinition(new ColumnIdentifier("e", false));
+ ColumnMetadata a = metadata.getColumn(new ColumnIdentifier("a", false));
+ ColumnMetadata b = metadata.getColumn(new ColumnIdentifier("b", false));
+ ColumnMetadata c = metadata.getColumn(new ColumnIdentifier("c", false));
+ ColumnMetadata d = metadata.getColumn(new ColumnIdentifier("d", false));
+ ColumnMetadata e = metadata.getColumn(new ColumnIdentifier("e", false));
Row row;
diff --git a/test/unit/org/apache/cassandra/db/rows/ThrottledUnfilteredIteratorTest.java b/test/unit/org/apache/cassandra/db/rows/ThrottledUnfilteredIteratorTest.java
new file mode 100644
index 0000000..cc886f1
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/rows/ThrottledUnfilteredIteratorTest.java
@@ -0,0 +1,695 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.rows;
+
+import static org.apache.cassandra.SchemaLoader.standardCFMD;
+import static org.junit.Assert.*;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.SortedMap;
+import java.util.TreeMap;
+
+import com.google.common.collect.Iterators;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.UpdateBuilder;
+import org.apache.cassandra.Util;
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.db.AbstractReadCommandBuilder;
+import org.apache.cassandra.db.BufferDecoratedKey;
+import org.apache.cassandra.db.Clustering;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.DeletionTime;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.ReadCommand;
+import org.apache.cassandra.db.ReadExecutionController;
+import org.apache.cassandra.db.RegularAndStaticColumns;
+import org.apache.cassandra.db.RowUpdateBuilder;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.db.partitions.AbstractUnfilteredPartitionIterator;
+import org.apache.cassandra.db.partitions.ImmutableBTreePartition;
+import org.apache.cassandra.db.partitions.Partition;
+import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
+import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.io.sstable.ISSTableScanner;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.CloseableIterator;
+import org.apache.cassandra.utils.FBUtilities;
+
+public class ThrottledUnfilteredIteratorTest extends CQLTester
+{
+ private static final String KSNAME = "ThrottledUnfilteredIteratorTest";
+ private static final String CFNAME = "StandardInteger1";
+
+ @BeforeClass
+ public static void defineSchema() throws ConfigurationException
+ {
+ SchemaLoader.prepareServer();
+ SchemaLoader.createKeyspace(KSNAME,
+ KeyspaceParams.simple(1),
+ standardCFMD(KSNAME, CFNAME, 1, UTF8Type.instance, Int32Type.instance, Int32Type.instance));
+ }
+
+ static final TableMetadata metadata;
+ static final ColumnMetadata v1Metadata;
+ static final ColumnMetadata v2Metadata;
+ static final ColumnMetadata staticMetadata;
+
+ static
+ {
+ metadata = TableMetadata.builder("", "")
+ .addPartitionKeyColumn("pk", Int32Type.instance)
+ .addClusteringColumn("ck1", Int32Type.instance)
+ .addClusteringColumn("ck2", Int32Type.instance)
+ .addRegularColumn("v1", Int32Type.instance)
+ .addRegularColumn("v2", Int32Type.instance)
+ .addStaticColumn("s1", Int32Type.instance)
+ .build();
+ v1Metadata = metadata.regularAndStaticColumns().columns(false).getSimple(0);
+ v2Metadata = metadata.regularAndStaticColumns().columns(false).getSimple(1);
+ staticMetadata = metadata.regularAndStaticColumns().columns(true).getSimple(0);
+ }
+
+ @Test
+ public void emptyPartitionDeletionTest() throws Throwable
+ {
+ // create cell tombstone, range tombstone, partition deletion
+ createTable("CREATE TABLE %s (pk int, ck1 int, ck2 int, v1 int, v2 int, PRIMARY KEY (pk, ck1, ck2))");
+ // partition deletion
+ execute("DELETE FROM %s USING TIMESTAMP 160 WHERE pk=1");
+
+ // flush and generate 1 sstable
+ ColumnFamilyStore cfs = Keyspace.open(keyspace()).getColumnFamilyStore(currentTable());
+ cfs.forceBlockingFlush();
+ cfs.disableAutoCompaction();
+ cfs.forceMajorCompaction();
+
+ assertEquals(1, cfs.getLiveSSTables().size());
+ SSTableReader reader = cfs.getLiveSSTables().iterator().next();
+
+ try (ISSTableScanner scanner = reader.getScanner();
+ CloseableIterator<UnfilteredRowIterator> throttled = ThrottledUnfilteredIterator.throttle(scanner, 100))
+ {
+ assertTrue(throttled.hasNext());
+ UnfilteredRowIterator iterator = throttled.next();
+ assertFalse(throttled.hasNext());
+ assertFalse(iterator.hasNext());
+ assertEquals(iterator.partitionLevelDeletion().markedForDeleteAt(), 160);
+ }
+
+ // test opt out
+ try (ISSTableScanner scanner = reader.getScanner();
+ CloseableIterator<UnfilteredRowIterator> throttled = ThrottledUnfilteredIterator.throttle(scanner, 0))
+ {
+ assertEquals(scanner, throttled);
+ }
+ }
+
+ @Test
+ public void emptyStaticTest() throws Throwable
+ {
+ // create cell tombstone, range tombstone, partition deletion
+ createTable("CREATE TABLE %s (pk int, ck1 int, ck2 int, v1 int, v2 int static, PRIMARY KEY (pk, ck1, ck2))");
+ // partition deletion
+ execute("UPDATE %s SET v2 = 160 WHERE pk = 1");
+
+ // flush and generate 1 sstable
+ ColumnFamilyStore cfs = Keyspace.open(keyspace()).getColumnFamilyStore(currentTable());
+ cfs.forceBlockingFlush();
+ cfs.disableAutoCompaction();
+ cfs.forceMajorCompaction();
+
+ assertEquals(1, cfs.getLiveSSTables().size());
+ SSTableReader reader = cfs.getLiveSSTables().iterator().next();
+
+ try (ISSTableScanner scanner = reader.getScanner();
+ CloseableIterator<UnfilteredRowIterator> throttled = ThrottledUnfilteredIterator.throttle(scanner, 100))
+ {
+ assertTrue(throttled.hasNext());
+ UnfilteredRowIterator iterator = throttled.next();
+ assertFalse(throttled.hasNext());
+ assertFalse(iterator.hasNext());
+ assertEquals(Int32Type.instance.getSerializer().deserialize(iterator.staticRow().cells().iterator().next().value()), new Integer(160));
+ }
+
+ // test opt out
+ try (ISSTableScanner scanner = reader.getScanner();
+ CloseableIterator<UnfilteredRowIterator> throttled = ThrottledUnfilteredIterator.throttle(scanner, 0))
+ {
+ assertEquals(scanner, throttled);
+ }
+ }
+
+ @Test
+ public void complexThrottleWithTombstoneTest() throws Throwable
+ {
+ // create cell tombstone, range tombstone, partition deletion
+ createTable("CREATE TABLE %s (pk int, ck1 int, ck2 int, v1 int, v2 int, PRIMARY KEY (pk, ck1, ck2))");
+
+ for (int ck1 = 1; ck1 <= 150; ck1++)
+ for (int ck2 = 1; ck2 <= 150; ck2++)
+ {
+ int timestamp = ck1, v1 = ck1, v2 = ck2;
+ execute("INSERT INTO %s(pk,ck1,ck2,v1,v2) VALUES(1,?,?,?,?) using timestamp "
+ + timestamp, ck1, ck2, v1, v2);
+ }
+
+ for (int ck1 = 1; ck1 <= 100; ck1++)
+ for (int ck2 = 1; ck2 <= 100; ck2++)
+ {
+ if (ck1 % 2 == 0 || ck1 % 3 == 0) // range tombstone
+ execute("DELETE FROM %s USING TIMESTAMP 170 WHERE pk=1 AND ck1=?", ck1);
+ else if (ck1 == ck2) // row tombstone
+ execute("DELETE FROM %s USING TIMESTAMP 180 WHERE pk=1 AND ck1=? AND ck2=?", ck1, ck2);
+ else if (ck1 == ck2 - 1) // cell tombstone
+ execute("DELETE v2 FROM %s USING TIMESTAMP 190 WHERE pk=1 AND ck1=? AND ck2=?", ck1, ck2);
+ }
+
+ // range deletion
+ execute("DELETE FROM %s USING TIMESTAMP 150 WHERE pk=1 AND ck1 > 100 AND ck1 < 120");
+ execute("DELETE FROM %s USING TIMESTAMP 150 WHERE pk=1 AND ck1 = 50 AND ck2 < 120");
+ // partition deletion
+ execute("DELETE FROM %s USING TIMESTAMP 160 WHERE pk=1");
+
+ // flush and generate 1 sstable
+ ColumnFamilyStore cfs = Keyspace.open(keyspace()).getColumnFamilyStore(currentTable());
+ cfs.forceBlockingFlush();
+ cfs.disableAutoCompaction();
+ cfs.forceMajorCompaction();
+
+ assertEquals(1, cfs.getLiveSSTables().size());
+ SSTableReader reader = cfs.getLiveSSTables().iterator().next();
+
+ try (ISSTableScanner scanner = reader.getScanner())
+ {
+ try (UnfilteredRowIterator rowIterator = scanner.next())
+ {
+ // only 1 partition data
+ assertFalse(scanner.hasNext());
+ List<Unfiltered> expectedUnfiltereds = new ArrayList<>();
+ rowIterator.forEachRemaining(expectedUnfiltereds::add);
+
+ // test different throttle
+ for (Integer throttle : Arrays.asList(2, 3, 4, 5, 11, 41, 99, 1000, 10001))
+ {
+ try (ISSTableScanner scannerForThrottle = reader.getScanner())
+ {
+ assertTrue(scannerForThrottle.hasNext());
+ try (UnfilteredRowIterator rowIteratorForThrottle = scannerForThrottle.next())
+ {
+ assertFalse(scannerForThrottle.hasNext());
+ verifyThrottleIterator(expectedUnfiltereds,
+ rowIteratorForThrottle,
+ new ThrottledUnfilteredIterator(rowIteratorForThrottle, throttle),
+ throttle);
+ }
+ }
+ }
+ }
+ }
+ }
+
+ private void verifyThrottleIterator(List<Unfiltered> expectedUnfiltereds,
+ UnfilteredRowIterator rowIteratorForThrottle,
+ ThrottledUnfilteredIterator throttledIterator,
+ int throttle)
+ {
+ List<Unfiltered> output = new ArrayList<>();
+
+ boolean isRevered = rowIteratorForThrottle.isReverseOrder();
+ boolean isFirst = true;
+
+ while (throttledIterator.hasNext())
+ {
+ UnfilteredRowIterator splittedIterator = throttledIterator.next();
+ assertMetadata(rowIteratorForThrottle, splittedIterator, isFirst);
+
+ List<Unfiltered> splittedUnfiltereds = new ArrayList<>();
+
+ splittedIterator.forEachRemaining(splittedUnfiltereds::add);
+
+ int remain = expectedUnfiltereds.size() - output.size();
+ int expectedSize = remain >= throttle ? throttle : remain;
+ if (splittedUnfiltereds.size() != expectedSize)
+ {
+ assertEquals(expectedSize + 1, splittedUnfiltereds.size());
+ // the extra unfilter must be close bound marker
+ Unfiltered last = splittedUnfiltereds.get(expectedSize);
+ assertTrue(last.isRangeTombstoneMarker());
+ RangeTombstoneMarker marker = (RangeTombstoneMarker) last;
+ assertFalse(marker.isBoundary());
+ assertTrue(marker.isClose(isRevered));
+ }
+ output.addAll(splittedUnfiltereds);
+ if (isFirst)
+ isFirst = false;
+ }
+ int index = 0;
+ RangeTombstoneMarker openMarker = null;
+ for (int i = 0; i < expectedUnfiltereds.size(); i++)
+ {
+ Unfiltered expected = expectedUnfiltereds.get(i);
+ Unfiltered data = output.get(i);
+
+ // verify that all tombstone are paired
+ if (data.isRangeTombstoneMarker())
+ {
+ RangeTombstoneMarker marker = (RangeTombstoneMarker) data;
+ if (marker.isClose(isRevered))
+ {
+ assertNotNull(openMarker);
+ openMarker = null;
+ }
+ if (marker.isOpen(isRevered))
+ {
+ assertNull(openMarker);
+ openMarker = marker;
+ }
+ }
+ if (expected.equals(data))
+ {
+ index++;
+ }
+ else // because of created closeMarker and openMarker
+ {
+ assertNotNull(openMarker);
+ DeletionTime openDeletionTime = openMarker.openDeletionTime(isRevered);
+ // only boundary or row will create extra closeMarker and openMarker
+ if (expected.isRangeTombstoneMarker())
+ {
+ RangeTombstoneMarker marker = (RangeTombstoneMarker) expected;
+ assertTrue(marker.isBoundary());
+ RangeTombstoneBoundaryMarker boundary = (RangeTombstoneBoundaryMarker) marker;
+ assertEquals(boundary.createCorrespondingCloseMarker(isRevered), data);
+ assertEquals(boundary.createCorrespondingOpenMarker(isRevered), output.get(index + 1));
+ assertEquals(openDeletionTime, boundary.endDeletionTime());
+
+ openMarker = boundary.createCorrespondingOpenMarker(isRevered);
+ }
+ else
+ {
+ ByteBuffer[] byteBuffers = expected.clustering().getRawValues();
+ RangeTombstoneBoundMarker closeMarker = RangeTombstoneBoundMarker.exclusiveClose(isRevered,
+ byteBuffers,
+ openDeletionTime);
+
+ RangeTombstoneBoundMarker nextOpenMarker = RangeTombstoneBoundMarker.inclusiveOpen(isRevered,
+ byteBuffers,
+ openDeletionTime);
+ assertEquals(closeMarker, data);
+ assertEquals(nextOpenMarker, output.get(index + 1));
+
+ openMarker = nextOpenMarker;
+ }
+ index += 2;
+ }
+ }
+ assertNull(openMarker);
+ assertEquals(output.size(), index);
+ }
+
+ @Test
+ public void simpleThrottleTest()
+ {
+ simpleThrottleTest(false);
+ }
+
+ @Test
+ public void skipTest()
+ {
+ simpleThrottleTest(true);
+ }
+
+ public void simpleThrottleTest(boolean skipOdd)
+ {
+ // all live rows with partition deletion
+ ThrottledUnfilteredIterator throttledIterator;
+ UnfilteredRowIterator origin;
+
+ List<Row> rows = new ArrayList<>();
+ int rowCount = 1111;
+
+ for (int i = 0; i < rowCount; i++)
+ rows.add(createRow(i, createCell(v1Metadata, i), createCell(v2Metadata, i)));
+
+ // testing different throttle limit
+ for (int throttle = 2; throttle < 1200; throttle += 21)
+ {
+ origin = rows(metadata.regularAndStaticColumns(),
+ 1,
+ new DeletionTime(0, 100),
+ createStaticRow(createCell(staticMetadata, 160)),
+ rows.toArray(new Row[0]));
+ throttledIterator = new ThrottledUnfilteredIterator(origin, throttle);
+
+ int splittedCount = (int) Math.ceil(rowCount*1.0/throttle);
+ for (int i = 1; i <= splittedCount; i++)
+ {
+ UnfilteredRowIterator splitted = throttledIterator.next();
+ assertMetadata(origin, splitted, i == 1);
+ // no op
+ splitted.close();
+
+ int start = (i - 1) * throttle;
+ int end = i == splittedCount ? rowCount : i * throttle;
+ if (skipOdd && (i % 2) == 0)
+ {
+ assertRows(splitted, rows.subList(start, end).toArray(new Row[0]));
+ }
+ }
+ assertTrue(!throttledIterator.hasNext());
+ }
+ }
+
+ @Test
+ public void throttledPartitionIteratorTest()
+ {
+ // all live rows with partition deletion
+ CloseableIterator<UnfilteredRowIterator> throttledIterator;
+ UnfilteredPartitionIterator origin;
+
+ SortedMap<Integer, List<Row>> partitions = new TreeMap<>();
+ int partitionCount = 13;
+ int baseRowsPerPartition = 1111;
+
+ for (int i = 1; i <= partitionCount; i++)
+ {
+ ArrayList<Row> rows = new ArrayList<>();
+ for (int j = 0; j < (baseRowsPerPartition + i); j++)
+ rows.add(createRow(i, createCell(v1Metadata, j), createCell(v2Metadata, j)));
+ partitions.put(i, rows);
+ }
+
+ // testing different throttle limit
+ for (int throttle = 2; throttle < 1200; throttle += 21)
+ {
+ origin = partitions(metadata.regularAndStaticColumns(),
+ new DeletionTime(0, 100),
+ createStaticRow(createCell(staticMetadata, 160)),
+ partitions);
+ throttledIterator = ThrottledUnfilteredIterator.throttle(origin, throttle);
+
+ int currentPartition = 0;
+ int rowsInPartition = 0;
+ int expectedSplitCount = 0;
+ int currentSplit = 1;
+ while (throttledIterator.hasNext())
+ {
+ UnfilteredRowIterator splitted = throttledIterator.next();
+ if (currentSplit > expectedSplitCount)
+ {
+ currentPartition++;
+ rowsInPartition = partitions.get(currentPartition).size();
+ expectedSplitCount = (int) Math.ceil(rowsInPartition * 1.0 / throttle);
+ currentSplit = 1;
+ }
+ UnfilteredRowIterator current = rows(metadata.regularAndStaticColumns(),
+ currentPartition,
+ new DeletionTime(0, 100),
+ createStaticRow(createCell(staticMetadata, 160)),
+ partitions.get(currentPartition).toArray(new Row[0]));
+ assertMetadata(current, splitted, currentSplit == 1);
+ // no op
+ splitted.close();
+
+ int start = (currentSplit - 1) * throttle;
+ int end = currentSplit == expectedSplitCount ? rowsInPartition : currentSplit * throttle;
+ assertRows(splitted, partitions.get(currentPartition).subList(start, end).toArray(new Row[0]));
+ currentSplit++;
+ }
+ }
+
+
+ origin = partitions(metadata.regularAndStaticColumns(),
+ new DeletionTime(0, 100),
+ Rows.EMPTY_STATIC_ROW,
+ partitions);
+ try
+ {
+ try (CloseableIterator<UnfilteredRowIterator> throttled = ThrottledUnfilteredIterator.throttle(origin, 10))
+ {
+ int i = 0;
+ while (throttled.hasNext())
+ {
+ assertEquals(dk(1), throttled.next().partitionKey());
+ if (i++ == 10)
+ {
+ throw new RuntimeException("Dummy exception");
+ }
+ }
+ fail("Should not reach here");
+ }
+ }
+ catch (RuntimeException rte)
+ {
+ int iteratedPartitions = 2;
+ while (iteratedPartitions <= partitionCount)
+ {
+ // check that original iterator was not closed
+ assertTrue(origin.hasNext());
+ // check it's possible to fetch second partition from original iterator
+ assertEquals(dk(iteratedPartitions++), origin.next().partitionKey());
+ }
+ }
+
+ }
+
+ private void assertMetadata(UnfilteredRowIterator origin, UnfilteredRowIterator splitted, boolean isFirst)
+ {
+ assertEquals(splitted.columns(), origin.columns());
+ assertEquals(splitted.partitionKey(), origin.partitionKey());
+ assertEquals(splitted.isReverseOrder(), origin.isReverseOrder());
+ assertEquals(splitted.metadata(), origin.metadata());
+ assertEquals(splitted.stats(), origin.stats());
+
+ if (isFirst)
+ {
+ assertEquals(origin.partitionLevelDeletion(), splitted.partitionLevelDeletion());
+ assertEquals(origin.staticRow(), splitted.staticRow());
+ }
+ else
+ {
+ assertEquals(DeletionTime.LIVE, splitted.partitionLevelDeletion());
+ assertEquals(Rows.EMPTY_STATIC_ROW, splitted.staticRow());
+ }
+ }
+
+ public static void assertRows(UnfilteredRowIterator iterator, Row... rows)
+ {
+ Iterator<Row> rowsIterator = Arrays.asList(rows).iterator();
+
+ while (iterator.hasNext() && rowsIterator.hasNext())
+ assertEquals(iterator.next(), rowsIterator.next());
+
+ assertTrue(iterator.hasNext() == rowsIterator.hasNext());
+ }
+
+ private static DecoratedKey dk(int pk)
+ {
+ return new BufferDecoratedKey(new Murmur3Partitioner.LongToken(pk), ByteBufferUtil.bytes(pk));
+ }
+
+ private static UnfilteredRowIterator rows(RegularAndStaticColumns columns,
+ int pk,
+ DeletionTime partitionDeletion,
+ Row staticRow,
+ Unfiltered... rows)
+ {
+ Iterator<Unfiltered> rowsIterator = Arrays.asList(rows).iterator();
+ return new AbstractUnfilteredRowIterator(metadata, dk(pk), partitionDeletion, columns, staticRow, false, EncodingStats.NO_STATS) {
+ protected Unfiltered computeNext()
+ {
+ return rowsIterator.hasNext() ? rowsIterator.next() : endOfData();
+ }
+ };
+ }
+
+ private static UnfilteredPartitionIterator partitions(RegularAndStaticColumns columns,
+ DeletionTime partitionDeletion,
+ Row staticRow,
+ SortedMap<Integer, List<Row>> partitions)
+ {
+ Iterator<Map.Entry<Integer, List<Row>>> partitionIt = partitions.entrySet().iterator();
+ return new AbstractUnfilteredPartitionIterator() {
+ public boolean hasNext()
+ {
+ return partitionIt.hasNext();
+ }
+
+ public UnfilteredRowIterator next()
+ {
+ Map.Entry<Integer, List<Row>> next = partitionIt.next();
+ Iterator<Row> rowsIterator = next.getValue().iterator();
+ return new AbstractUnfilteredRowIterator(metadata, dk(next.getKey()), partitionDeletion, columns, staticRow, false, EncodingStats.NO_STATS) {
+ protected Unfiltered computeNext()
+ {
+ return rowsIterator.hasNext() ? rowsIterator.next() : endOfData();
+ }
+ };
+ }
+
+ public TableMetadata metadata()
+ {
+ return metadata;
+ }
+ };
+ }
+
+
+ private static Row createRow(int ck, Cell... columns)
+ {
+ return createRow(ck, ck, columns);
+ }
+
+ private static Row createRow(int ck1, int ck2, Cell... columns)
+ {
+ BTreeRow.Builder builder = new BTreeRow.Builder(true);
+ builder.newRow(Util.clustering(metadata.comparator, ck1, ck2));
+ for (Cell cell : columns)
+ builder.addCell(cell);
+ return builder.build();
+ }
+
+ private static Row createStaticRow(Cell... columns)
+ {
+ Row.Builder builder = new BTreeRow.Builder(true);
+ builder.newRow(Clustering.STATIC_CLUSTERING);
+ for (Cell cell : columns)
+ builder.addCell(cell);
+ return builder.build();
+ }
+
+ private static Cell createCell(ColumnMetadata metadata, int v)
+ {
+ return createCell(metadata, v, 100L, BufferCell.NO_DELETION_TIME);
+ }
+
+ private static Cell createCell(ColumnMetadata metadata, int v, long timestamp, int localDeletionTime)
+ {
+ return new BufferCell(metadata,
+ timestamp,
+ BufferCell.NO_TTL,
+ localDeletionTime,
+ ByteBufferUtil.bytes(v),
+ null);
+ }
+
+ @Test
+ public void testThrottledIteratorWithRangeDeletions() throws Exception
+ {
+ Keyspace keyspace = Keyspace.open(KSNAME);
+ ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CFNAME);
+
+ // Inserting data
+ String key = "k1";
+
+ UpdateBuilder builder;
+
+ builder = UpdateBuilder.create(cfs.metadata(), key).withTimestamp(0);
+ for (int i = 0; i < 40; i += 2)
+ builder.newRow(i).add("val", i);
+ builder.applyUnsafe();
+
+ new RowUpdateBuilder(cfs.metadata(), 1, key).addRangeTombstone(10, 22).build().applyUnsafe();
+
+ cfs.forceBlockingFlush();
+
+ builder = UpdateBuilder.create(cfs.metadata(), key).withTimestamp(2);
+ for (int i = 1; i < 40; i += 2)
+ builder.newRow(i).add("val", i);
+ builder.applyUnsafe();
+
+ new RowUpdateBuilder(cfs.metadata(), 3, key).addRangeTombstone(19, 27).build().applyUnsafe();
+ // We don't flush to test with both a range tomsbtone in memtable and in sstable
+
+ // Queries by name
+ int[] live = new int[]{ 4, 9, 11, 17, 28 };
+ int[] dead = new int[]{ 12, 19, 21, 24, 27 };
+
+ AbstractReadCommandBuilder.PartitionRangeBuilder cmdBuilder = Util.cmd(cfs);
+
+ ReadCommand cmd = cmdBuilder.build();
+
+ for (int batchSize = 2; batchSize <= 40; batchSize++)
+ {
+ List<UnfilteredRowIterator> unfilteredRowIterators = new LinkedList<>();
+
+ try (ReadExecutionController executionController = cmd.executionController();
+ UnfilteredPartitionIterator iterator = cmd.executeLocally(executionController))
+ {
+ assertTrue(iterator.hasNext());
+ Iterator<UnfilteredRowIterator> throttled = ThrottledUnfilteredIterator.throttle(iterator, batchSize);
+ while (throttled.hasNext())
+ {
+ UnfilteredRowIterator next = throttled.next();
+ ImmutableBTreePartition materializedPartition = ImmutableBTreePartition.create(next);
+ int unfilteredCount = Iterators.size(materializedPartition.unfilteredIterator());
+
+ System.out.println("batchsize " + batchSize + " unfilteredCount " + unfilteredCount + " materializedPartition " + materializedPartition);
+
+ if (throttled.hasNext())
+ {
+ if (unfilteredCount != batchSize)
+ {
+ //when there is extra unfiltered, it must be close bound marker
+ assertEquals(batchSize + 1, unfilteredCount);
+ Unfiltered last = Iterators.getLast(materializedPartition.unfilteredIterator());
+ assertTrue(last.isRangeTombstoneMarker());
+ RangeTombstoneMarker marker = (RangeTombstoneMarker) last;
+ assertFalse(marker.isBoundary());
+ assertTrue(marker.isClose(false));
+ }
+ }
+ else
+ {
+ //only last batch can be smaller than batchSize
+ assertTrue(unfilteredCount <= batchSize + 1);
+ }
+ unfilteredRowIterators.add(materializedPartition.unfilteredIterator());
+ }
+ assertFalse(iterator.hasNext());
+ }
+
+ // Verify throttled data after merge
+ Partition partition = ImmutableBTreePartition.create(UnfilteredRowIterators.merge(unfilteredRowIterators));
+
+ int nowInSec = FBUtilities.nowInSeconds();
+
+ for (int i : live)
+ assertTrue("Row " + i + " should be live", partition.getRow(Clustering.make(ByteBufferUtil.bytes((i)))).hasLiveData(nowInSec, cfs.metadata().enforceStrictLiveness()));
+ for (int i : dead)
+ assertFalse("Row " + i + " shouldn't be live", partition.getRow(Clustering.make(ByteBufferUtil.bytes((i)))).hasLiveData(nowInSec, cfs.metadata().enforceStrictLiveness()));
+ }
+ }
+}
diff --git a/test/unit/org/apache/cassandra/db/rows/UnfilteredRowIteratorsMergeTest.java b/test/unit/org/apache/cassandra/db/rows/UnfilteredRowIteratorsMergeTest.java
index 4578ad1..7f1b735 100644
--- a/test/unit/org/apache/cassandra/db/rows/UnfilteredRowIteratorsMergeTest.java
+++ b/test/unit/org/apache/cassandra/db/rows/UnfilteredRowIteratorsMergeTest.java
@@ -19,7 +19,7 @@
import java.nio.ByteBuffer;
import java.util.*;
-import java.util.function.Function;
+import java.util.function.IntUnaryOperator;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
@@ -31,7 +31,7 @@
import org.junit.Test;
import org.apache.cassandra.Util;
-import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.marshal.AsciiType;
@@ -47,11 +47,13 @@
}
static DecoratedKey partitionKey = Util.dk("key");
static DeletionTime partitionLevelDeletion = DeletionTime.LIVE;
- static CFMetaData metadata = CFMetaData.Builder.create("UnfilteredRowIteratorsMergeTest", "Test").
- addPartitionKey("key", AsciiType.instance).
- addClusteringColumn("clustering", Int32Type.instance).
- addRegularColumn("data", Int32Type.instance).
- build();
+ static TableMetadata metadata =
+ TableMetadata.builder("UnfilteredRowIteratorsMergeTest", "Test")
+ .addPartitionKeyColumn("key", AsciiType.instance)
+ .addClusteringColumn("clustering", Int32Type.instance)
+ .addRegularColumn("data", Int32Type.instance)
+ .build();
+
static Comparator<Clusterable> comparator = new ClusteringComparator(Int32Type.instance);
static int nowInSec = FBUtilities.nowInSeconds();
@@ -110,7 +112,7 @@
System.out.println("\nSeed " + seed);
Random r = new Random(seed);
- List<Function<Integer, Integer>> timeGenerators = ImmutableList.of(
+ List<IntUnaryOperator> timeGenerators = ImmutableList.of(
x -> -1,
x -> DEL_RANGE,
x -> r.nextInt(DEL_RANGE)
@@ -149,25 +151,24 @@
public UnfilteredRowIterator mergeIterators(List<UnfilteredRowIterator> us, boolean iterations)
{
- int now = FBUtilities.nowInSeconds();
if (iterations)
{
UnfilteredRowIterator mi = us.get(0);
int i;
for (i = 1; i + 2 <= ITERATORS; i += 2)
- mi = UnfilteredRowIterators.merge(ImmutableList.of(mi, us.get(i), us.get(i+1)), now);
+ mi = UnfilteredRowIterators.merge(ImmutableList.of(mi, us.get(i), us.get(i+1)));
if (i + 1 <= ITERATORS)
- mi = UnfilteredRowIterators.merge(ImmutableList.of(mi, us.get(i)), now);
+ mi = UnfilteredRowIterators.merge(ImmutableList.of(mi, us.get(i)));
return mi;
}
else
{
- return UnfilteredRowIterators.merge(us, now);
+ return UnfilteredRowIterators.merge(us);
}
}
@SuppressWarnings("unused")
- private List<Unfiltered> generateSource(Random r, Function<Integer, Integer> timeGenerator)
+ private List<Unfiltered> generateSource(Random r, IntUnaryOperator timeGenerator)
{
int[] positions = new int[ITEMS + 1];
for (int i=0; i<ITEMS; ++i)
@@ -385,10 +386,10 @@
return Clustering.make(Int32Type.instance.decompose(i));
}
- static Row emptyRowAt(int pos, Function<Integer, Integer> timeGenerator)
+ static Row emptyRowAt(int pos, IntUnaryOperator timeGenerator)
{
final Clustering clustering = clusteringFor(pos);
- final LivenessInfo live = LivenessInfo.create(timeGenerator.apply(pos), nowInSec);
+ final LivenessInfo live = LivenessInfo.create(timeGenerator.applyAsInt(pos), nowInSec);
return BTreeRow.noCellLiveRow(clustering, live);
}
@@ -424,7 +425,7 @@
super(UnfilteredRowIteratorsMergeTest.metadata,
UnfilteredRowIteratorsMergeTest.partitionKey,
UnfilteredRowIteratorsMergeTest.partitionLevelDeletion,
- UnfilteredRowIteratorsMergeTest.metadata.partitionColumns(),
+ UnfilteredRowIteratorsMergeTest.metadata.regularAndStaticColumns(),
null,
reversed,
EncodingStats.NO_STATS);
diff --git a/test/unit/org/apache/cassandra/db/rows/UnfilteredRowIteratorsTest.java b/test/unit/org/apache/cassandra/db/rows/UnfilteredRowIteratorsTest.java
index cce8599..a936504 100644
--- a/test/unit/org/apache/cassandra/db/rows/UnfilteredRowIteratorsTest.java
+++ b/test/unit/org/apache/cassandra/db/rows/UnfilteredRowIteratorsTest.java
@@ -23,41 +23,37 @@
import org.junit.Test;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.cassandra.Util;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
-import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.BufferDecoratedKey;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.DeletionTime;
import org.apache.cassandra.db.EmptyIterators;
-import org.apache.cassandra.db.PartitionColumns;
+import org.apache.cassandra.db.RegularAndStaticColumns;
import org.apache.cassandra.db.filter.DataLimits;
import org.apache.cassandra.db.marshal.Int32Type;
import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FBUtilities;
public class UnfilteredRowIteratorsTest
{
- static final CFMetaData metadata;
- static final ColumnDefinition v1Metadata;
- static final ColumnDefinition v2Metadata;
+ static final TableMetadata metadata;
+ static final ColumnMetadata v1Metadata;
+ static final ColumnMetadata v2Metadata;
static
{
- // Required because of metadata creation, assertion is thrown otherwise
- DatabaseDescriptor.daemonInitialization();
-
- metadata = CFMetaData.Builder.create("", "")
- .addPartitionKey("pk", Int32Type.instance)
- .addClusteringColumn("ck", Int32Type.instance)
- .addRegularColumn("v1", Int32Type.instance)
- .addRegularColumn("v2", Int32Type.instance)
- .build();
- v1Metadata = metadata.partitionColumns().columns(false).getSimple(0);
- v2Metadata = metadata.partitionColumns().columns(false).getSimple(1);
+ metadata = TableMetadata.builder("", "")
+ .addPartitionKeyColumn("pk", Int32Type.instance)
+ .addClusteringColumn("ck", Int32Type.instance)
+ .addRegularColumn("v1", Int32Type.instance)
+ .addRegularColumn("v2", Int32Type.instance)
+ .build();
+ v1Metadata = metadata.regularAndStaticColumns().columns(false).getSimple(0);
+ v2Metadata = metadata.regularAndStaticColumns().columns(false).getSimple(1);
}
@@ -66,14 +62,14 @@
{
UnfilteredRowIterator iter1, iter2, iter3, concat;
// simple concatenation
- iter1 = rows(metadata.partitionColumns(), 1,
+ iter1 = rows(metadata.regularAndStaticColumns(), 1,
row(1, cell(v1Metadata, 1), cell(v2Metadata, 1)),
row(2, cell(v1Metadata, 2), cell(v2Metadata, 2)));
- iter2 = rows(metadata.partitionColumns(), 1,
+ iter2 = rows(metadata.regularAndStaticColumns(), 1,
row(3, cell(v1Metadata, 3), cell(v2Metadata, 3)),
row(4, cell(v1Metadata, 4), cell(v2Metadata, 4)));
concat = UnfilteredRowIterators.concat(iter1, iter2);
- Assert.assertEquals(concat.columns(), metadata.partitionColumns());
+ Assert.assertEquals(concat.columns(), metadata.regularAndStaticColumns());
assertRows(concat,
row(1, cell(v1Metadata, 1), cell(v2Metadata, 1)),
row(2, cell(v1Metadata, 2), cell(v2Metadata, 2)),
@@ -81,65 +77,65 @@
row(4, cell(v1Metadata, 4), cell(v2Metadata, 4)));
// concat with RHS empty iterator
- iter1 = rows(metadata.partitionColumns(), 1,
+ iter1 = rows(metadata.regularAndStaticColumns(), 1,
row(1, cell(v1Metadata, 1), cell(v2Metadata, 1)),
row(2, cell(v1Metadata, 2), cell(v2Metadata, 2)));
- Assert.assertEquals(concat.columns(), metadata.partitionColumns());
+ Assert.assertEquals(concat.columns(), metadata.regularAndStaticColumns());
assertRows(UnfilteredRowIterators.concat(iter1, EmptyIterators.unfilteredRow(metadata, dk(1), false, Rows.EMPTY_STATIC_ROW, DeletionTime.LIVE)),
row(1, cell(v1Metadata, 1), cell(v2Metadata, 1)),
row(2, cell(v1Metadata, 2), cell(v2Metadata, 2)));
// concat with LHS empty iterator
- iter1 = rows(metadata.partitionColumns(), 1,
+ iter1 = rows(metadata.regularAndStaticColumns(), 1,
row(1, cell(v1Metadata, 1), cell(v2Metadata, 1)),
row(2, cell(v1Metadata, 2), cell(v2Metadata, 2)));
- Assert.assertEquals(concat.columns(), metadata.partitionColumns());
+ Assert.assertEquals(concat.columns(), metadata.regularAndStaticColumns());
assertRows(UnfilteredRowIterators.concat(EmptyIterators.unfilteredRow(metadata, dk(1), false, Rows.EMPTY_STATIC_ROW, DeletionTime.LIVE), iter1),
row(1, cell(v1Metadata, 1), cell(v2Metadata, 1)),
row(2, cell(v1Metadata, 2), cell(v2Metadata, 2)));
// concat with different columns
- iter1 = rows(metadata.partitionColumns().without(v1Metadata), 1,
+ iter1 = rows(metadata.regularAndStaticColumns().without(v1Metadata), 1,
row(1, cell(v2Metadata, 1)), row(2, cell(v2Metadata, 2)));
- iter2 = rows(metadata.partitionColumns().without(v2Metadata), 1,
+ iter2 = rows(metadata.regularAndStaticColumns().without(v2Metadata), 1,
row(3, cell(v1Metadata, 3)), row(4, cell(v1Metadata, 4)));
concat = UnfilteredRowIterators.concat(iter1, iter2);
- Assert.assertEquals(concat.columns(), PartitionColumns.of(v1Metadata).mergeTo(PartitionColumns.of(v2Metadata)));
+ Assert.assertEquals(concat.columns(), RegularAndStaticColumns.of(v1Metadata).mergeTo(RegularAndStaticColumns.of(v2Metadata)));
assertRows(concat,
row(1, cell(v2Metadata, 1)), row(2, cell(v2Metadata, 2)),
row(3, cell(v1Metadata, 3)), row(4, cell(v1Metadata, 4)));
// concat with CQL limits
- iter1 = rows(metadata.partitionColumns(), 1,
+ iter1 = rows(metadata.regularAndStaticColumns(), 1,
row(1, cell(v1Metadata, 1), cell(v2Metadata, 1)),
row(2, cell(v1Metadata, 2), cell(v2Metadata, 2)));
- iter2 = rows(metadata.partitionColumns(), 1,
+ iter2 = rows(metadata.regularAndStaticColumns(), 1,
row(3, cell(v1Metadata, 3), cell(v2Metadata, 3)),
row(4, cell(v1Metadata, 4), cell(v2Metadata, 4)));
concat = UnfilteredRowIterators.concat(DataLimits.cqlLimits(1).filter(iter1, FBUtilities.nowInSeconds(), true),
DataLimits.cqlLimits(1).filter(iter2, FBUtilities.nowInSeconds(), true));
- Assert.assertEquals(concat.columns(), metadata.partitionColumns());
+ Assert.assertEquals(concat.columns(), metadata.regularAndStaticColumns());
assertRows(concat,
row(1, cell(v1Metadata, 1), cell(v2Metadata, 1)),
row(3, cell(v1Metadata, 3), cell(v2Metadata, 3)));
// concat concatenated iterators
- iter1 = rows(metadata.partitionColumns(), 1,
+ iter1 = rows(metadata.regularAndStaticColumns(), 1,
row(1, cell(v1Metadata, 1), cell(v2Metadata, 1)),
row(2, cell(v1Metadata, 2), cell(v2Metadata, 2)));
- iter2 = rows(metadata.partitionColumns(), 1,
+ iter2 = rows(metadata.regularAndStaticColumns(), 1,
row(3, cell(v1Metadata, 3), cell(v2Metadata, 3)),
row(4, cell(v1Metadata, 4), cell(v2Metadata, 4)));
concat = UnfilteredRowIterators.concat(DataLimits.cqlLimits(1).filter(iter1, FBUtilities.nowInSeconds(), true),
DataLimits.cqlLimits(1).filter(iter2, FBUtilities.nowInSeconds(), true));
- iter3 = rows(metadata.partitionColumns(), 1,
+ iter3 = rows(metadata.regularAndStaticColumns(), 1,
row(4, cell(v1Metadata, 4), cell(v2Metadata, 4)),
row(5, cell(v1Metadata, 5), cell(v2Metadata, 5)));
concat = UnfilteredRowIterators.concat(concat, DataLimits.cqlLimits(1).filter(iter3, FBUtilities.nowInSeconds(), true));
- Assert.assertEquals(concat.columns(), metadata.partitionColumns());
+ Assert.assertEquals(concat.columns(), metadata.regularAndStaticColumns());
assertRows(concat,
row(1, cell(v1Metadata, 1), cell(v2Metadata, 1)),
row(3, cell(v1Metadata, 3), cell(v2Metadata, 3)),
@@ -161,7 +157,7 @@
return new BufferDecoratedKey(new Murmur3Partitioner.LongToken(pk), ByteBufferUtil.bytes(pk));
}
- public static UnfilteredRowIterator rows(PartitionColumns columns, int pk, Row... rows)
+ public static UnfilteredRowIterator rows(RegularAndStaticColumns columns, int pk, Row... rows)
{
Iterator<Row> rowsIterator = Arrays.asList(rows).iterator();
return new AbstractUnfilteredRowIterator(metadata, dk(pk), DeletionTime.LIVE, columns, Rows.EMPTY_STATIC_ROW, false, EncodingStats.NO_STATS) {
@@ -181,7 +177,7 @@
return builder.build();
}
- public Cell cell(ColumnDefinition metadata, int v)
+ public Cell cell(ColumnMetadata metadata, int v)
{
return new BufferCell(metadata,
1L, BufferCell.NO_TTL, BufferCell.NO_DELETION_TIME, ByteBufferUtil.bytes(v), null);
diff --git a/test/unit/org/apache/cassandra/db/rows/UnfilteredRowsGenerator.java b/test/unit/org/apache/cassandra/db/rows/UnfilteredRowsGenerator.java
index 7cdccdb..71b28e8 100644
--- a/test/unit/org/apache/cassandra/db/rows/UnfilteredRowsGenerator.java
+++ b/test/unit/org/apache/cassandra/db/rows/UnfilteredRowsGenerator.java
@@ -19,16 +19,16 @@
import java.nio.ByteBuffer;
import java.util.*;
-import java.util.function.Function;
+import java.util.function.IntUnaryOperator;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.junit.Assert;
-import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.marshal.Int32Type;
import org.apache.cassandra.db.rows.Unfiltered.Kind;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.utils.btree.BTree;
public class UnfilteredRowsGenerator
@@ -111,7 +111,7 @@
}
}
- public List<Unfiltered> generateSource(Random r, int items, int range, int del_range, Function<Integer, Integer> timeGenerator)
+ public List<Unfiltered> generateSource(Random r, int items, int range, int del_range, IntUnaryOperator timeGenerator)
{
int[] positions = new int[items + 1];
for (int i=0; i<items; ++i)
@@ -219,10 +219,10 @@
return out;
}
- static Row emptyRowAt(int pos, Function<Integer, Integer> timeGenerator)
+ static Row emptyRowAt(int pos, IntUnaryOperator timeGenerator)
{
final Clustering clustering = clusteringFor(pos);
- final LivenessInfo live = LivenessInfo.create(timeGenerator.apply(pos), UnfilteredRowIteratorsMergeTest.nowInSec);
+ final LivenessInfo live = LivenessInfo.create(timeGenerator.applyAsInt(pos), UnfilteredRowIteratorsMergeTest.nowInSec);
return BTreeRow.noCellLiveRow(clustering, live);
}
@@ -289,12 +289,12 @@
new DeletionTime(delTime, delTime));
}
- public static UnfilteredRowIterator source(Iterable<Unfiltered> content, CFMetaData metadata, DecoratedKey partitionKey)
+ public static UnfilteredRowIterator source(Iterable<Unfiltered> content, TableMetadata metadata, DecoratedKey partitionKey)
{
return source(content, metadata, partitionKey, DeletionTime.LIVE);
}
- public static UnfilteredRowIterator source(Iterable<Unfiltered> content, CFMetaData metadata, DecoratedKey partitionKey, DeletionTime delTime)
+ public static UnfilteredRowIterator source(Iterable<Unfiltered> content, TableMetadata metadata, DecoratedKey partitionKey, DeletionTime delTime)
{
return new Source(content.iterator(), metadata, partitionKey, delTime, false);
}
@@ -303,12 +303,12 @@
{
Iterator<Unfiltered> content;
- protected Source(Iterator<Unfiltered> content, CFMetaData metadata, DecoratedKey partitionKey, DeletionTime partitionLevelDeletion, boolean reversed)
+ protected Source(Iterator<Unfiltered> content, TableMetadata metadata, DecoratedKey partitionKey, DeletionTime partitionLevelDeletion, boolean reversed)
{
super(metadata,
partitionKey,
partitionLevelDeletion,
- metadata.partitionColumns(),
+ metadata.regularAndStaticColumns(),
Rows.EMPTY_STATIC_ROW,
reversed,
EncodingStats.NO_STATS);
diff --git a/test/unit/org/apache/cassandra/db/streaming/CassandraEntireSSTableStreamWriterTest.java b/test/unit/org/apache/cassandra/db/streaming/CassandraEntireSSTableStreamWriterTest.java
new file mode 100644
index 0000000..00a48d1
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/streaming/CassandraEntireSSTableStreamWriterTest.java
@@ -0,0 +1,210 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db.streaming;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.WritableByteChannel;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Queue;
+import java.util.UUID;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.Unpooled;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.channel.ChannelOutboundHandlerAdapter;
+import io.netty.channel.ChannelPromise;
+import io.netty.channel.DefaultFileRegion;
+import io.netty.channel.embedded.EmbeddedChannel;
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.RowUpdateBuilder;
+import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.io.sstable.SSTableMultiWriter;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.util.DataInputBuffer;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.net.SharedDefaultFileRegion;
+import org.apache.cassandra.net.AsyncStreamingOutputPlus;
+import org.apache.cassandra.schema.CachingParams;
+import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.streaming.DefaultConnectionFactory;
+import org.apache.cassandra.streaming.PreviewKind;
+import org.apache.cassandra.streaming.SessionInfo;
+import org.apache.cassandra.streaming.StreamCoordinator;
+import org.apache.cassandra.streaming.StreamEventHandler;
+import org.apache.cassandra.streaming.StreamOperation;
+import org.apache.cassandra.streaming.StreamResultFuture;
+import org.apache.cassandra.streaming.StreamSession;
+import org.apache.cassandra.streaming.StreamSummary;
+import org.apache.cassandra.streaming.messages.StreamMessageHeader;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.FBUtilities;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+public class CassandraEntireSSTableStreamWriterTest
+{
+ public static final String KEYSPACE = "CassandraEntireSSTableStreamWriterTest";
+ public static final String CF_STANDARD = "Standard1";
+ public static final String CF_INDEXED = "Indexed1";
+ public static final String CF_STANDARDLOWINDEXINTERVAL = "StandardLowIndexInterval";
+
+ private static SSTableReader sstable;
+ private static ColumnFamilyStore store;
+
+ @BeforeClass
+ public static void defineSchemaAndPrepareSSTable()
+ {
+ SchemaLoader.prepareServer();
+ SchemaLoader.createKeyspace(KEYSPACE,
+ KeyspaceParams.simple(1),
+ SchemaLoader.standardCFMD(KEYSPACE, CF_STANDARD),
+ SchemaLoader.compositeIndexCFMD(KEYSPACE, CF_INDEXED, true),
+ SchemaLoader.standardCFMD(KEYSPACE, CF_STANDARDLOWINDEXINTERVAL)
+ .minIndexInterval(8)
+ .maxIndexInterval(256)
+ .caching(CachingParams.CACHE_NOTHING));
+
+ Keyspace keyspace = Keyspace.open(KEYSPACE);
+ store = keyspace.getColumnFamilyStore("Standard1");
+
+ // insert data and compact to a single sstable
+ CompactionManager.instance.disableAutoCompaction();
+ for (int j = 0; j < 10; j++)
+ {
+ new RowUpdateBuilder(store.metadata(), j, String.valueOf(j))
+ .clustering("0")
+ .add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER)
+ .build()
+ .applyUnsafe();
+ }
+ store.forceBlockingFlush();
+ CompactionManager.instance.performMaximal(store, false);
+
+ sstable = store.getLiveSSTables().iterator().next();
+ }
+
+ @Test
+ public void testBlockWriterOverWire() throws IOException
+ {
+ StreamSession session = setupStreamingSessionForTest();
+
+ CassandraEntireSSTableStreamWriter writer = new CassandraEntireSSTableStreamWriter(sstable, session, CassandraOutgoingFile.getComponentManifest(sstable));
+
+ EmbeddedChannel channel = new EmbeddedChannel();
+ AsyncStreamingOutputPlus out = new AsyncStreamingOutputPlus(channel);
+ writer.write(out);
+
+ Queue msgs = channel.outboundMessages();
+
+ assertTrue(msgs.peek() instanceof DefaultFileRegion);
+ }
+
+ @Test
+ public void testBlockReadingAndWritingOverWire() throws Exception
+ {
+ StreamSession session = setupStreamingSessionForTest();
+ InetAddressAndPort peer = FBUtilities.getBroadcastAddressAndPort();
+
+ CassandraEntireSSTableStreamWriter writer = new CassandraEntireSSTableStreamWriter(sstable, session, CassandraOutgoingFile.getComponentManifest(sstable));
+
+ // This is needed as Netty releases the ByteBuffers as soon as the channel is flushed
+ ByteBuf serializedFile = Unpooled.buffer(8192);
+ EmbeddedChannel channel = createMockNettyChannel(serializedFile);
+ AsyncStreamingOutputPlus out = new AsyncStreamingOutputPlus(channel);
+
+ writer.write(out);
+
+ session.prepareReceiving(new StreamSummary(sstable.metadata().id, 1, 5104));
+
+ CassandraStreamHeader header =
+ CassandraStreamHeader.builder()
+ .withSSTableFormat(sstable.descriptor.formatType)
+ .withSSTableVersion(sstable.descriptor.version)
+ .withSSTableLevel(0)
+ .withEstimatedKeys(sstable.estimatedKeys())
+ .withSections(Collections.emptyList())
+ .withSerializationHeader(sstable.header.toComponent())
+ .withComponentManifest(CassandraOutgoingFile.getComponentManifest(sstable))
+ .isEntireSSTable(true)
+ .withFirstKey(sstable.first)
+ .withTableId(sstable.metadata().id)
+ .build();
+
+ CassandraEntireSSTableStreamReader reader = new CassandraEntireSSTableStreamReader(new StreamMessageHeader(sstable.metadata().id, peer, session.planId(), false, 0, 0, 0, null), header, session);
+
+ SSTableMultiWriter sstableWriter = reader.read(new DataInputBuffer(serializedFile.nioBuffer(), false));
+ Collection<SSTableReader> newSstables = sstableWriter.finished();
+
+ assertEquals(1, newSstables.size());
+ }
+
+ private EmbeddedChannel createMockNettyChannel(ByteBuf serializedFile) throws Exception
+ {
+ WritableByteChannel wbc = new WritableByteChannel()
+ {
+ private boolean isOpen = true;
+ public int write(ByteBuffer src) throws IOException
+ {
+ int size = src.limit();
+ serializedFile.writeBytes(src);
+ return size;
+ }
+
+ public boolean isOpen()
+ {
+ return isOpen;
+ }
+
+ public void close() throws IOException
+ {
+ isOpen = false;
+ }
+ };
+
+ return new EmbeddedChannel(new ChannelOutboundHandlerAdapter() {
+ @Override
+ public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise promise) throws Exception
+ {
+ ((SharedDefaultFileRegion) msg).transferTo(wbc, 0);
+ super.write(ctx, msg, promise);
+ }
+ });
+ }
+
+ private StreamSession setupStreamingSessionForTest()
+ {
+ StreamCoordinator streamCoordinator = new StreamCoordinator(StreamOperation.BOOTSTRAP, 1, new DefaultConnectionFactory(), false, false, null, PreviewKind.NONE);
+ StreamResultFuture future = StreamResultFuture.createInitiator(UUID.randomUUID(), StreamOperation.BOOTSTRAP, Collections.<StreamEventHandler>emptyList(), streamCoordinator);
+
+ InetAddressAndPort peer = FBUtilities.getBroadcastAddressAndPort();
+ streamCoordinator.addSessionInfo(new SessionInfo(peer, 0, peer, Collections.emptyList(), Collections.emptyList(), StreamSession.State.INITIALIZED));
+
+ StreamSession session = streamCoordinator.getOrCreateNextSession(peer);
+ session.init(future);
+ return session;
+ }
+}
diff --git a/test/unit/org/apache/cassandra/db/streaming/CassandraOutgoingFileTest.java b/test/unit/org/apache/cassandra/db/streaming/CassandraOutgoingFileTest.java
new file mode 100644
index 0000000..9d663b5
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/streaming/CassandraOutgoingFileTest.java
@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.streaming;
+
+import java.util.Arrays;
+import java.util.List;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.RowUpdateBuilder;
+import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.io.sstable.KeyIterator;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.schema.CachingParams;
+import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.streaming.StreamOperation;
+import org.apache.cassandra.utils.ByteBufferUtil;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class CassandraOutgoingFileTest
+{
+ public static final String KEYSPACE = "CassandraOutgoingFileTest";
+ public static final String CF_STANDARD = "Standard1";
+ public static final String CF_INDEXED = "Indexed1";
+ public static final String CF_STANDARDLOWINDEXINTERVAL = "StandardLowIndexInterval";
+
+ private static SSTableReader sstable;
+ private static ColumnFamilyStore store;
+
+ @BeforeClass
+ public static void defineSchemaAndPrepareSSTable()
+ {
+ SchemaLoader.prepareServer();
+ SchemaLoader.createKeyspace(KEYSPACE,
+ KeyspaceParams.simple(1),
+ SchemaLoader.standardCFMD(KEYSPACE, CF_STANDARD),
+ SchemaLoader.compositeIndexCFMD(KEYSPACE, CF_INDEXED, true),
+ SchemaLoader.standardCFMD(KEYSPACE, CF_STANDARDLOWINDEXINTERVAL)
+ .minIndexInterval(8)
+ .maxIndexInterval(256)
+ .caching(CachingParams.CACHE_NOTHING));
+
+ Keyspace keyspace = Keyspace.open(KEYSPACE);
+ store = keyspace.getColumnFamilyStore(CF_STANDARD);
+
+ // insert data and compact to a single sstable
+ CompactionManager.instance.disableAutoCompaction();
+ for (int j = 0; j < 10; j++)
+ {
+ new RowUpdateBuilder(store.metadata(), j, String.valueOf(j))
+ .clustering("0")
+ .add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER)
+ .build()
+ .applyUnsafe();
+ }
+ store.forceBlockingFlush();
+ CompactionManager.instance.performMaximal(store, false);
+
+ sstable = store.getLiveSSTables().iterator().next();
+ }
+
+ @Test
+ public void validateFullyContainedIn_SingleContiguousRange_Succeeds()
+ {
+ List<Range<Token>> requestedRanges = Arrays.asList(new Range<>(store.getPartitioner().getMinimumToken(), sstable.last.getToken()));
+
+ List<SSTableReader.PartitionPositionBounds> sections = sstable.getPositionsForRanges(requestedRanges);
+ CassandraOutgoingFile cof = new CassandraOutgoingFile(StreamOperation.BOOTSTRAP, sstable.ref(),
+ sections,
+ requestedRanges, sstable.estimatedKeys());
+
+ assertTrue(cof.contained(sections, sstable));
+ }
+
+ @Test
+ public void validateFullyContainedIn_PartialOverlap_Fails()
+ {
+ List<Range<Token>> requestedRanges = Arrays.asList(new Range<>(store.getPartitioner().getMinimumToken(), getTokenAtIndex(2)));
+
+ List<SSTableReader.PartitionPositionBounds> sections = sstable.getPositionsForRanges(requestedRanges);
+ CassandraOutgoingFile cof = new CassandraOutgoingFile(StreamOperation.BOOTSTRAP, sstable.ref(),
+ sections,
+ requestedRanges, sstable.estimatedKeys());
+
+ assertFalse(cof.contained(sections, sstable));
+ }
+
+ @Test
+ public void validateFullyContainedIn_SplitRange_Succeeds()
+ {
+ List<Range<Token>> requestedRanges = Arrays.asList(new Range<>(store.getPartitioner().getMinimumToken(), getTokenAtIndex(4)),
+ new Range<>(getTokenAtIndex(2), getTokenAtIndex(6)),
+ new Range<>(getTokenAtIndex(5), sstable.last.getToken()));
+ requestedRanges = Range.normalize(requestedRanges);
+
+ List<SSTableReader.PartitionPositionBounds> sections = sstable.getPositionsForRanges(requestedRanges);
+ CassandraOutgoingFile cof = new CassandraOutgoingFile(StreamOperation.BOOTSTRAP, sstable.ref(),
+ sections,
+ requestedRanges, sstable.estimatedKeys());
+
+ assertTrue(cof.contained(sections, sstable));
+ }
+
+ private DecoratedKey getKeyAtIndex(int i)
+ {
+ int count = 0;
+ DecoratedKey key;
+
+ try (KeyIterator iter = new KeyIterator(sstable.descriptor, sstable.metadata()))
+ {
+ do
+ {
+ key = iter.next();
+ count++;
+ } while (iter.hasNext() && count < i);
+ }
+ return key;
+ }
+
+ private Token getTokenAtIndex(int i)
+ {
+ return getKeyAtIndex(i).getToken();
+ }
+}
diff --git a/test/unit/org/apache/cassandra/db/streaming/CassandraStreamHeaderTest.java b/test/unit/org/apache/cassandra/db/streaming/CassandraStreamHeaderTest.java
new file mode 100644
index 0000000..e48abf6
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/streaming/CassandraStreamHeaderTest.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db.streaming;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+
+import org.junit.Test;
+
+import org.apache.cassandra.cql3.statements.schema.CreateTableStatement;
+import org.apache.cassandra.db.SerializationHeader;
+import org.apache.cassandra.db.streaming.CassandraStreamHeader.CassandraStreamHeaderSerializer;
+import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.io.sstable.Component;
+import org.apache.cassandra.io.sstable.format.SSTableFormat;
+import org.apache.cassandra.io.sstable.format.big.BigFormat;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.serializers.SerializationUtils;
+import org.apache.cassandra.utils.ByteBufferUtil;
+
+public class CassandraStreamHeaderTest
+{
+ @Test
+ public void serializerTest()
+ {
+ String ddl = "CREATE TABLE tbl (k INT PRIMARY KEY, v INT)";
+ TableMetadata metadata = CreateTableStatement.parse(ddl, "ks").build();
+ CassandraStreamHeader header =
+ CassandraStreamHeader.builder()
+ .withSSTableFormat(SSTableFormat.Type.BIG)
+ .withSSTableVersion(BigFormat.latestVersion)
+ .withSSTableLevel(0)
+ .withEstimatedKeys(0)
+ .withSections(Collections.emptyList())
+ .withSerializationHeader(SerializationHeader.makeWithoutStats(metadata).toComponent())
+ .withTableId(metadata.id)
+ .build();
+
+ SerializationUtils.assertSerializationCycle(header, CassandraStreamHeader.serializer);
+ }
+
+ @Test
+ public void serializerTest_EntireSSTableTransfer()
+ {
+ String ddl = "CREATE TABLE tbl (k INT PRIMARY KEY, v INT)";
+ TableMetadata metadata = CreateTableStatement.parse(ddl, "ks").build();
+
+ ComponentManifest manifest = new ComponentManifest(new LinkedHashMap<Component, Long>() {{ put(Component.DATA, 100L); }});
+
+ CassandraStreamHeader header =
+ CassandraStreamHeader.builder()
+ .withSSTableFormat(SSTableFormat.Type.BIG)
+ .withSSTableVersion(BigFormat.latestVersion)
+ .withSSTableLevel(0)
+ .withEstimatedKeys(0)
+ .withSections(Collections.emptyList())
+ .withSerializationHeader(SerializationHeader.makeWithoutStats(metadata).toComponent())
+ .withComponentManifest(manifest)
+ .isEntireSSTable(true)
+ .withFirstKey(Murmur3Partitioner.instance.decorateKey(ByteBufferUtil.EMPTY_BYTE_BUFFER))
+ .withTableId(metadata.id)
+ .build();
+
+ SerializationUtils.assertSerializationCycle(header, new TestableCassandraStreamHeaderSerializer());
+ }
+
+ private static class TestableCassandraStreamHeaderSerializer extends CassandraStreamHeaderSerializer
+ {
+ @Override
+ public CassandraStreamHeader deserialize(DataInputPlus in, int version) throws IOException
+ {
+ return deserialize(in, version, tableId -> Murmur3Partitioner.instance);
+ }
+ }
+}
diff --git a/test/unit/org/apache/cassandra/db/streaming/CassandraStreamManagerTest.java b/test/unit/org/apache/cassandra/db/streaming/CassandraStreamManagerTest.java
new file mode 100644
index 0000000..0b37d66
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/streaming/CassandraStreamManagerTest.java
@@ -0,0 +1,242 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.streaming;
+
+import java.io.IOException;
+import java.net.UnknownHostException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+import com.google.common.util.concurrent.Uninterruptibles;
+
+import org.apache.cassandra.locator.RangesAtEndpoint;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.concurrent.NamedThreadFactory;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.QueryProcessor;
+import org.apache.cassandra.cql3.statements.schema.CreateTableStatement;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.dht.IPartitioner;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.service.ActiveRepairService;
+import org.apache.cassandra.streaming.DefaultConnectionFactory;
+import org.apache.cassandra.streaming.OutgoingStream;
+import org.apache.cassandra.streaming.PreviewKind;
+import org.apache.cassandra.streaming.StreamConnectionFactory;
+import org.apache.cassandra.streaming.StreamOperation;
+import org.apache.cassandra.streaming.StreamSession;
+import org.apache.cassandra.utils.UUIDGen;
+import org.apache.cassandra.utils.concurrent.Ref;
+
+import static org.apache.cassandra.service.ActiveRepairService.NO_PENDING_REPAIR;
+
+public class CassandraStreamManagerTest
+{
+ private static final String KEYSPACE = null;
+ private String keyspace = null;
+ private static final String table = "tbl";
+ private static final StreamConnectionFactory connectionFactory = new DefaultConnectionFactory();
+
+ private TableMetadata tbm;
+ private ColumnFamilyStore cfs;
+
+ @BeforeClass
+ public static void setupClass() throws Exception
+ {
+ SchemaLoader.prepareServer();
+ }
+
+ @Before
+ public void createKeyspace() throws Exception
+ {
+ keyspace = String.format("ks_%s", System.currentTimeMillis());
+ tbm = CreateTableStatement.parse(String.format("CREATE TABLE %s (k INT PRIMARY KEY, v INT)", table), keyspace).build();
+ SchemaLoader.createKeyspace(keyspace, KeyspaceParams.simple(1), tbm);
+ cfs = Schema.instance.getColumnFamilyStoreInstance(tbm.id);
+ }
+
+ private static StreamSession session(UUID pendingRepair)
+ {
+ try
+ {
+ return new StreamSession(StreamOperation.REPAIR,
+ InetAddressAndPort.getByName("127.0.0.1"),
+ connectionFactory,
+ false,
+ 0,
+ pendingRepair,
+ PreviewKind.NONE);
+ }
+ catch (UnknownHostException e)
+ {
+ throw new AssertionError(e);
+ }
+ }
+
+ private SSTableReader createSSTable(Runnable queryable)
+ {
+ Set<SSTableReader> before = cfs.getLiveSSTables();
+ queryable.run();
+ cfs.forceBlockingFlush();
+ Set<SSTableReader> after = cfs.getLiveSSTables();
+
+ Set<SSTableReader> diff = Sets.difference(after, before);
+ return Iterables.getOnlyElement(diff);
+ }
+
+ private static void mutateRepaired(SSTableReader sstable, long repairedAt, UUID pendingRepair, boolean isTransient) throws IOException
+ {
+ Descriptor descriptor = sstable.descriptor;
+ descriptor.getMetadataSerializer().mutateRepairMetadata(descriptor, repairedAt, pendingRepair, isTransient);
+ sstable.reloadSSTableMetadata();
+
+ }
+
+ private static Set<SSTableReader> sstablesFromStreams(Collection<OutgoingStream> streams)
+ {
+ Set<SSTableReader> sstables = new HashSet<>();
+ for (OutgoingStream stream: streams)
+ {
+ Ref<SSTableReader> ref = CassandraOutgoingFile.fromStream(stream).getRef();
+ sstables.add(ref.get());
+ ref.release();
+ }
+ return sstables;
+ }
+
+ private Set<SSTableReader> getReadersForRange(Range<Token> range)
+ {
+ Collection<OutgoingStream> streams = cfs.getStreamManager().createOutgoingStreams(session(NO_PENDING_REPAIR),
+ RangesAtEndpoint.toDummyList(Collections.singleton(range)),
+ NO_PENDING_REPAIR,
+ PreviewKind.NONE);
+ return sstablesFromStreams(streams);
+ }
+
+ private Set<SSTableReader> selectReaders(UUID pendingRepair)
+ {
+ IPartitioner partitioner = DatabaseDescriptor.getPartitioner();
+ Collection<Range<Token>> ranges = Lists.newArrayList(new Range<Token>(partitioner.getMinimumToken(), partitioner.getMinimumToken()));
+ Collection<OutgoingStream> streams = cfs.getStreamManager().createOutgoingStreams(session(pendingRepair), RangesAtEndpoint.toDummyList(ranges), pendingRepair, PreviewKind.NONE);
+ return sstablesFromStreams(streams);
+ }
+
+ @Test
+ public void incrementalSSTableSelection() throws Exception
+ {
+ // CASSANDRA-15825 Make sure a compaction won't be triggered under our feet removing the sstables mid-flight
+ cfs.disableAutoCompaction();
+
+ // make 3 tables, 1 unrepaired, 2 pending repair with different repair ids, and 1 repaired
+ SSTableReader sstable1 = createSSTable(() -> QueryProcessor.executeInternal(String.format("INSERT INTO %s.%s (k, v) VALUES (1, 1)", keyspace, table)));
+ SSTableReader sstable2 = createSSTable(() -> QueryProcessor.executeInternal(String.format("INSERT INTO %s.%s (k, v) VALUES (2, 2)", keyspace, table)));
+ SSTableReader sstable3 = createSSTable(() -> QueryProcessor.executeInternal(String.format("INSERT INTO %s.%s (k, v) VALUES (3, 3)", keyspace, table)));
+ SSTableReader sstable4 = createSSTable(() -> QueryProcessor.executeInternal(String.format("INSERT INTO %s.%s (k, v) VALUES (4, 4)", keyspace, table)));
+
+
+ UUID pendingRepair = UUIDGen.getTimeUUID();
+ long repairedAt = System.currentTimeMillis();
+ mutateRepaired(sstable2, ActiveRepairService.UNREPAIRED_SSTABLE, pendingRepair, false);
+ mutateRepaired(sstable3, ActiveRepairService.UNREPAIRED_SSTABLE, UUIDGen.getTimeUUID(), false);
+ mutateRepaired(sstable4, repairedAt, NO_PENDING_REPAIR, false);
+
+
+
+ // no pending repair should return all sstables
+ Assert.assertEquals(Sets.newHashSet(sstable1, sstable2, sstable3, sstable4), selectReaders(NO_PENDING_REPAIR));
+
+ // a pending repair arg should only return sstables with the same pending repair id
+ Assert.assertEquals(Sets.newHashSet(sstable2), selectReaders(pendingRepair));
+ }
+
+ @Test
+ public void testSSTableSectionsForRanges() throws Exception
+ {
+ cfs.truncateBlocking();
+
+ createSSTable(() -> {
+ QueryProcessor.executeInternal(String.format("INSERT INTO %s.%s (k, v) VALUES (1, 1)", keyspace, table));
+ QueryProcessor.executeInternal(String.format("INSERT INTO %s.%s (k, v) VALUES (2, 2)", keyspace, table));
+ });
+
+ Collection<SSTableReader> allSSTables = cfs.getLiveSSTables();
+ Assert.assertEquals(1, allSSTables.size());
+ final Token firstToken = allSSTables.iterator().next().first.getToken();
+ DatabaseDescriptor.setSSTablePreemptiveOpenIntervalInMB(1);
+
+ Set<SSTableReader> sstablesBeforeRewrite = getReadersForRange(new Range<>(firstToken, firstToken));
+ Assert.assertEquals(1, sstablesBeforeRewrite.size());
+ final AtomicInteger checkCount = new AtomicInteger();
+ // needed since we get notified when compaction is done as well - we can't get sections for ranges for obsoleted sstables
+ final AtomicBoolean done = new AtomicBoolean(false);
+ final AtomicBoolean failed = new AtomicBoolean(false);
+ Runnable r = new Runnable()
+ {
+ public void run()
+ {
+ while (!done.get())
+ {
+ Range<Token> range = new Range<Token>(firstToken, firstToken);
+ Set<SSTableReader> sstables = getReadersForRange(range);
+ if (sstables.size() != 1)
+ failed.set(true);
+ checkCount.incrementAndGet();
+ Uninterruptibles.sleepUninterruptibly(5, TimeUnit.MILLISECONDS);
+ }
+ }
+ };
+ Thread t = NamedThreadFactory.createThread(r);
+ try
+ {
+ t.start();
+ cfs.forceMajorCompaction();
+ // reset
+ }
+ finally
+ {
+ DatabaseDescriptor.setSSTablePreemptiveOpenIntervalInMB(50);
+ done.set(true);
+ t.join(20);
+ }
+ Assert.assertFalse(failed.get());
+ Assert.assertTrue(checkCount.get() >= 2);
+ cfs.truncateBlocking();
+ }
+}
diff --git a/test/unit/org/apache/cassandra/db/streaming/ComponentManifestTest.java b/test/unit/org/apache/cassandra/db/streaming/ComponentManifestTest.java
new file mode 100644
index 0000000..4909263
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/streaming/ComponentManifestTest.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.streaming;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.LinkedHashMap;
+
+import org.junit.Test;
+
+import org.apache.cassandra.io.sstable.Component;
+import org.apache.cassandra.io.util.DataInputBuffer;
+import org.apache.cassandra.io.util.DataOutputBufferFixed;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.serializers.SerializationUtils;
+
+import static org.junit.Assert.assertNotEquals;
+
+public class ComponentManifestTest
+{
+ @Test
+ public void testSerialization()
+ {
+ ComponentManifest expected = new ComponentManifest(new LinkedHashMap<Component, Long>() {{ put(Component.DATA, 100L); }});
+ SerializationUtils.assertSerializationCycle(expected, ComponentManifest.serializer);
+ }
+
+ @Test(expected = EOFException.class)
+ public void testSerialization_FailsOnBadBytes() throws IOException
+ {
+ ByteBuffer buf = ByteBuffer.allocate(512);
+ ComponentManifest expected = new ComponentManifest(new LinkedHashMap<Component, Long>() {{ put(Component.DATA, 100L); }});
+
+ DataOutputBufferFixed out = new DataOutputBufferFixed(buf);
+
+ ComponentManifest.serializer.serialize(expected, out, MessagingService.VERSION_40);
+
+ buf.putInt(0, -100);
+
+ DataInputBuffer in = new DataInputBuffer(out.buffer(), false);
+ ComponentManifest actual = ComponentManifest.serializer.deserialize(in, MessagingService.VERSION_40);
+ assertNotEquals(expected, actual);
+ }
+}
diff --git a/test/unit/org/apache/cassandra/db/streaming/StreamRequestTest.java b/test/unit/org/apache/cassandra/db/streaming/StreamRequestTest.java
new file mode 100644
index 0000000..9f5e656
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/streaming/StreamRequestTest.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.streaming;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.Util;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.dht.ByteOrderedPartitioner;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.io.util.DataInputBuffer;
+import org.apache.cassandra.io.util.DataOutputBuffer;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.RangesAtEndpoint;
+import org.apache.cassandra.locator.Replica;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.streaming.StreamRequest;
+
+public class StreamRequestTest
+{
+ private static InetAddressAndPort local;
+ private final String ks = "keyspace";
+ private final int version = MessagingService.current_version;
+
+ @BeforeClass
+ public static void setUp() throws Throwable
+ {
+ DatabaseDescriptor.daemonInitialization();
+ local = InetAddressAndPort.getByName("127.0.0.1");
+ }
+
+ @Test
+ public void serializationRoundTrip() throws Throwable
+ {
+ StreamRequest orig = new StreamRequest(ks,
+ atEndpoint(Arrays.asList(range(1, 2), range(3, 4), range(5, 6)),
+ Collections.emptyList()),
+ atEndpoint(Collections.emptyList(),
+ Arrays.asList(range(5, 6), range(7, 8))),
+ Arrays.asList("a", "b", "c"));
+
+ int expectedSize = (int) StreamRequest.serializer.serializedSize(orig, version);
+ try (DataOutputBuffer out = new DataOutputBuffer(expectedSize))
+ {
+ StreamRequest.serializer.serialize(orig, out, version);
+ Assert.assertEquals(expectedSize, out.buffer().limit());
+ try (DataInputBuffer in = new DataInputBuffer(out.buffer(), false))
+ {
+ StreamRequest decoded = StreamRequest.serializer.deserialize(in, version);
+
+ Assert.assertEquals(orig.keyspace, decoded.keyspace);
+ Util.assertRCEquals(orig.full, decoded.full);
+ Util.assertRCEquals(orig.transientReplicas, decoded.transientReplicas);
+ Assert.assertEquals(orig.columnFamilies, decoded.columnFamilies);
+ }
+ }
+ }
+
+ private static RangesAtEndpoint atEndpoint(Collection<Range<Token>> full, Collection<Range<Token>> trans)
+ {
+ RangesAtEndpoint.Builder builder = RangesAtEndpoint.builder(local);
+ for (Range<Token> range : full)
+ builder.add(new Replica(local, range, true));
+
+ for (Range<Token> range : trans)
+ builder.add(new Replica(local, range, false));
+
+ return builder.build();
+ }
+
+ private static Range<Token> range(int l, int r)
+ {
+ return new Range<>(new ByteOrderedPartitioner.BytesToken(Integer.toString(l).getBytes()),
+ new ByteOrderedPartitioner.BytesToken(Integer.toString(r).getBytes()));
+ }
+}
diff --git a/test/unit/org/apache/cassandra/db/transform/DuplicateRowCheckerTest.java b/test/unit/org/apache/cassandra/db/transform/DuplicateRowCheckerTest.java
index 432bce3..6c3a5c0 100644
--- a/test/unit/org/apache/cassandra/db/transform/DuplicateRowCheckerTest.java
+++ b/test/unit/org/apache/cassandra/db/transform/DuplicateRowCheckerTest.java
@@ -27,14 +27,15 @@
import com.google.common.collect.Iterators;
import org.junit.*;
-import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.CQLTester;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.db.partitions.*;
import org.apache.cassandra.db.rows.*;
+import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.net.*;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.utils.DiagnosticSnapshotService;
import org.apache.cassandra.utils.FBUtilities;
@@ -45,27 +46,14 @@
public class DuplicateRowCheckerTest extends CQLTester
{
ColumnFamilyStore cfs;
- CFMetaData metadata;
- static HashMap<InetAddress, MessageOut> sentMessages;
+ TableMetadata metadata;
+ static HashMap<InetAddressAndPort, Message<?>> sentMessages;
@BeforeClass
public static void setupMessaging()
{
sentMessages = new HashMap<>();
- IMessageSink sink = new IMessageSink()
- {
- public boolean allowOutgoingMessage(MessageOut message, int id, InetAddress to)
- {
- sentMessages.put(to, message);
- return false;
- }
-
- public boolean allowIncomingMessage(MessageIn message, int id)
- {
- return false;
- }
- };
- MessagingService.instance().addMessageSink(sink);
+ MessagingService.instance().outboundSink.add((message, to) -> { sentMessages.put(to, message); return false;});
}
@Before
@@ -80,7 +68,7 @@
execute("insert into %s (pk, ck1, ck2, v) values (?, ?, ?, ?)", "key", i, i, i);
getCurrentColumnFamilyStore().forceBlockingFlush();
- metadata = getCurrentColumnFamilyStore().metadata;
+ metadata = getCurrentColumnFamilyStore().metadata();
cfs = getCurrentColumnFamilyStore();
sentMessages.clear();
}
@@ -168,14 +156,14 @@
assertCommandIssued(sentMessages, true);
}
- public static void assertCommandIssued(HashMap<InetAddress, MessageOut> sent, boolean isExpected)
+ public static void assertCommandIssued(HashMap<InetAddressAndPort, Message<?>> sent, boolean isExpected)
{
assertEquals(isExpected, !sent.isEmpty());
if (isExpected)
{
assertEquals(1, sent.size());
- assertTrue(sent.containsKey(FBUtilities.getBroadcastAddress()));
- SnapshotCommand command = (SnapshotCommand) sent.get(FBUtilities.getBroadcastAddress()).payload;
+ assertTrue(sent.containsKey(FBUtilities.getBroadcastAddressAndPort()));
+ SnapshotCommand command = (SnapshotCommand) sent.get(FBUtilities.getBroadcastAddressAndPort()).payload;
assertTrue(command.snapshot_name.startsWith(DiagnosticSnapshotService.DUPLICATE_ROWS_DETECTED_SNAPSHOT_PREFIX));
}
}
@@ -200,7 +188,7 @@
return ((AbstractType<T>) type).decompose(value);
}
- public static Row makeRow(CFMetaData metadata, Object... clusteringValues)
+ public static Row makeRow(TableMetadata metadata, Object... clusteringValues)
{
ByteBuffer[] clusteringByteBuffers = new ByteBuffer[clusteringValues.length];
for (int i = 0; i < clusteringValues.length; i++)
@@ -209,7 +197,7 @@
return BTreeRow.noCellLiveRow(Clustering.make(clusteringByteBuffers), LivenessInfo.create(0, 0));
}
- public static UnfilteredRowIterator rows(CFMetaData metadata,
+ public static UnfilteredRowIterator rows(TableMetadata metadata,
DecoratedKey key,
boolean isReversedOrder,
Unfiltered... unfiltereds)
@@ -218,7 +206,7 @@
return new AbstractUnfilteredRowIterator(metadata,
key,
DeletionTime.LIVE,
- metadata.partitionColumns(),
+ metadata.regularAndStaticColumns(),
Rows.EMPTY_STATIC_ROW,
isReversedOrder,
EncodingStats.NO_STATS)
@@ -234,13 +222,13 @@
{
int nowInSecs = 0;
return DuplicateRowChecker.duringRead(FilteredPartitions.filter(unfiltered, nowInSecs),
- Collections.singletonList(FBUtilities.getBroadcastAddress()));
+ Collections.singletonList(FBUtilities.getBroadcastAddressAndPort()));
}
- public static UnfilteredPartitionIterator iter(CFMetaData metadata, boolean isReversedOrder, Unfiltered... unfiltereds)
+ public static UnfilteredPartitionIterator iter(TableMetadata metadata, boolean isReversedOrder, Unfiltered... unfiltereds)
{
DecoratedKey key = metadata.partitioner.decorateKey(bytes("key"));
UnfilteredRowIterator rowIter = rows(metadata, key, isReversedOrder, unfiltereds);
- return new SingletonUnfilteredPartitionIterator(rowIter, false);
+ return new SingletonUnfilteredPartitionIterator(rowIter);
}
}
diff --git a/test/unit/org/apache/cassandra/db/transform/RTTransformationsTest.java b/test/unit/org/apache/cassandra/db/transform/RTTransformationsTest.java
index b2ec2b2..e400f3a 100644
--- a/test/unit/org/apache/cassandra/db/transform/RTTransformationsTest.java
+++ b/test/unit/org/apache/cassandra/db/transform/RTTransformationsTest.java
@@ -25,7 +25,6 @@
import com.google.common.collect.Iterators;
-import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.ClusteringPrefix.Kind;
import org.apache.cassandra.db.marshal.AbstractType;
@@ -35,6 +34,7 @@
import org.apache.cassandra.db.rows.*;
import org.apache.cassandra.db.transform.RTBoundValidator.Stage;
import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.utils.FBUtilities;
import static org.apache.cassandra.db.transform.RTBoundCloser.close;
@@ -52,21 +52,20 @@
private final int nowInSec = FBUtilities.nowInSeconds();
- private CFMetaData metadata;
+ private TableMetadata metadata;
private DecoratedKey key;
@Before
public void setUp()
{
metadata =
- CFMetaData.Builder
- .create(KEYSPACE, TABLE)
- .addPartitionKey("pk", UTF8Type.instance)
- .addClusteringColumn("ck0", UTF8Type.instance)
- .addClusteringColumn("ck1", UTF8Type.instance)
- .addClusteringColumn("ck2", UTF8Type.instance)
- .withPartitioner(Murmur3Partitioner.instance)
- .build();
+ TableMetadata.builder(KEYSPACE, TABLE)
+ .addPartitionKeyColumn("pk", UTF8Type.instance)
+ .addClusteringColumn("ck0", UTF8Type.instance)
+ .addClusteringColumn("ck1", UTF8Type.instance)
+ .addClusteringColumn("ck2", UTF8Type.instance)
+ .partitioner(Murmur3Partitioner.instance)
+ .build();
key = Murmur3Partitioner.instance.decorateKey(bytes("key"));
}
@@ -413,7 +412,7 @@
new AbstractUnfilteredRowIterator(metadata,
key,
DeletionTime.LIVE,
- metadata.partitionColumns(),
+ metadata.regularAndStaticColumns(),
Rows.EMPTY_STATIC_ROW,
isReversedOrder,
EncodingStats.NO_STATS)
@@ -424,7 +423,7 @@
}
};
- return new SingletonUnfilteredPartitionIterator(rowIter, false);
+ return new SingletonUnfilteredPartitionIterator(rowIter);
}
private void assertIteratorsEqual(UnfilteredPartitionIterator iter1, UnfilteredPartitionIterator iter2)
diff --git a/test/unit/org/apache/cassandra/db/view/ViewBuilderTaskTest.java b/test/unit/org/apache/cassandra/db/view/ViewBuilderTaskTest.java
new file mode 100644
index 0000000..2341c73
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/view/ViewBuilderTaskTest.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.view;
+
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import org.junit.Test;
+
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.SystemKeyspace;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.dht.IPartitioner;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.schema.SchemaConstants;
+import org.apache.cassandra.transport.ProtocolVersion;
+
+import static org.junit.Assert.assertEquals;
+
+public class ViewBuilderTaskTest extends CQLTester
+{
+ private static final ProtocolVersion protocolVersion = ProtocolVersion.CURRENT;
+
+ @Test
+ public void testBuildRange() throws Throwable
+ {
+ requireNetwork();
+ execute("USE " + keyspace());
+ executeNet(protocolVersion, "USE " + keyspace());
+
+ String tableName = createTable("CREATE TABLE %s (" +
+ "k int, " +
+ "c int, " +
+ "v text, " +
+ "PRIMARY KEY(k, c))");
+
+ String viewName = tableName + "_view";
+ executeNet(protocolVersion, String.format("CREATE MATERIALIZED VIEW %s AS SELECT * FROM %%s " +
+ "WHERE v IS NOT NULL AND k IS NOT NULL AND c IS NOT NULL " +
+ "PRIMARY KEY (v, k, c)", viewName));
+
+ ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
+ View view = cfs.keyspace.viewManager.forTable(cfs.metadata().id).iterator().next();
+
+ // Insert the dataset
+ for (int k = 0; k < 100; k++)
+ for (int c = 0; c < 10; c++)
+ execute("INSERT INTO %s (k, c, v) VALUES (?, ?, ?)", k, c, String.valueOf(k));
+
+ // Retrieve the sorted tokens of the inserted rows
+ IPartitioner partitioner = cfs.metadata().partitioner;
+ List<Token> tokens = IntStream.range(0, 100)
+ .mapToObj(Int32Type.instance::decompose)
+ .map(partitioner::getToken)
+ .sorted()
+ .collect(Collectors.toList());
+
+ class Tester
+ {
+ private void test(int indexOfStartToken,
+ int indexOfEndToken,
+ Integer indexOfLastToken,
+ long keysBuilt,
+ long expectedKeysBuilt,
+ int expectedRowsInView) throws Throwable
+ {
+ // Truncate the materialized view (not the base table)
+ cfs.viewManager.forceBlockingFlush();
+ cfs.viewManager.truncateBlocking(cfs.forceBlockingFlush(), System.currentTimeMillis());
+ assertRowCount(execute("SELECT * FROM " + viewName), 0);
+
+ // Get the tokens from the referenced inserted rows
+ Token startToken = tokens.get(indexOfStartToken);
+ Token endToken = tokens.get(indexOfEndToken);
+ Token lastToken = indexOfLastToken == null ? null : tokens.get(indexOfLastToken);
+ Range<Token> range = new Range<>(startToken, endToken);
+
+ // Run the view build task, verifying the returned number of bult keys
+ long actualKeysBuilt = new ViewBuilderTask(cfs, view, range, lastToken, keysBuilt).call();
+ assertEquals(expectedKeysBuilt, actualKeysBuilt);
+
+ // Verify that the rows have been written to the MV
+ assertRowCount(execute("SELECT * FROM " + viewName), expectedRowsInView);
+
+ // Verify that the last position and number of bult keys have been stored
+ assertRows(execute(String.format("SELECT last_token, keys_built " +
+ "FROM %s.%s WHERE keyspace_name='%s' AND view_name='%s' " +
+ "AND start_token=? AND end_token=?",
+ SchemaConstants.SYSTEM_KEYSPACE_NAME,
+ SystemKeyspace.VIEW_BUILDS_IN_PROGRESS,
+ keyspace(),
+ viewName),
+ startToken.toString(), endToken.toString()),
+ row(endToken.toString(), expectedKeysBuilt));
+ }
+ }
+ Tester tester = new Tester();
+
+ // Build range from rows 0 to 100 without any recorded start position
+ tester.test(0, 10, null, 0, 10, 100);
+
+ // Build range from rows 100 to 200 starting at row 150
+ tester.test(10, 20, 15, 0, 5, 50);
+
+ // Build range from rows 300 to 400 starting at row 350 with 10 built keys
+ tester.test(30, 40, 35, 10, 15, 50);
+
+ // Build range from rows 400 to 500 starting at row 100 (out of range) with 10 built keys
+ tester.test(40, 50, 10, 10, 20, 100);
+
+ // Build range from rows 900 to 100 (wrap around) without any recorded start position
+ tester.test(90, 10, null, 0, 20, 200);
+
+ executeNet(protocolVersion, "DROP MATERIALIZED VIEW " + view.name);
+ }
+}
diff --git a/test/unit/org/apache/cassandra/db/view/ViewUtilsTest.java b/test/unit/org/apache/cassandra/db/view/ViewUtilsTest.java
index 89bb44a..7eebef7 100644
--- a/test/unit/org/apache/cassandra/db/view/ViewUtilsTest.java
+++ b/test/unit/org/apache/cassandra/db/view/ViewUtilsTest.java
@@ -18,7 +18,6 @@
package org.apache.cassandra.db.view;
-import java.net.InetAddress;
import java.util.HashMap;
import java.util.Map;
import java.util.Optional;
@@ -26,9 +25,11 @@
import org.junit.BeforeClass;
import org.junit.Test;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.config.Schema;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.Replica;
+import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.dht.OrderPreservingPartitioner.StringToken;
import org.apache.cassandra.exceptions.ConfigurationException;
@@ -59,12 +60,12 @@
metadata.clearUnsafe();
// DC1
- metadata.updateNormalToken(new StringToken("A"), InetAddress.getByName("127.0.0.1"));
- metadata.updateNormalToken(new StringToken("C"), InetAddress.getByName("127.0.0.2"));
+ metadata.updateNormalToken(new StringToken("A"), InetAddressAndPort.getByName("127.0.0.1"));
+ metadata.updateNormalToken(new StringToken("C"), InetAddressAndPort.getByName("127.0.0.2"));
// DC2
- metadata.updateNormalToken(new StringToken("B"), InetAddress.getByName("127.0.0.4"));
- metadata.updateNormalToken(new StringToken("D"), InetAddress.getByName("127.0.0.5"));
+ metadata.updateNormalToken(new StringToken("B"), InetAddressAndPort.getByName("127.0.0.4"));
+ metadata.updateNormalToken(new StringToken("D"), InetAddressAndPort.getByName("127.0.0.5"));
Map<String, String> replicationMap = new HashMap<>();
replicationMap.put(ReplicationParams.CLASS, NetworkTopologyStrategy.class.getName());
@@ -74,14 +75,14 @@
Keyspace.clear("Keyspace1");
KeyspaceMetadata meta = KeyspaceMetadata.create("Keyspace1", KeyspaceParams.create(false, replicationMap));
- Schema.instance.setKeyspaceMetadata(meta);
+ Schema.instance.load(meta);
- Optional<InetAddress> naturalEndpoint = ViewUtils.getViewNaturalEndpoint("Keyspace1",
- new StringToken("CA"),
- new StringToken("BB"));
+ Optional<Replica> naturalEndpoint = ViewUtils.getViewNaturalEndpoint("Keyspace1",
+ new StringToken("CA"),
+ new StringToken("BB"));
Assert.assertTrue(naturalEndpoint.isPresent());
- Assert.assertEquals(InetAddress.getByName("127.0.0.2"), naturalEndpoint.get());
+ Assert.assertEquals(InetAddressAndPort.getByName("127.0.0.2"), naturalEndpoint.get().endpoint());
}
@@ -92,12 +93,12 @@
metadata.clearUnsafe();
// DC1
- metadata.updateNormalToken(new StringToken("A"), InetAddress.getByName("127.0.0.1"));
- metadata.updateNormalToken(new StringToken("C"), InetAddress.getByName("127.0.0.2"));
+ metadata.updateNormalToken(new StringToken("A"), InetAddressAndPort.getByName("127.0.0.1"));
+ metadata.updateNormalToken(new StringToken("C"), InetAddressAndPort.getByName("127.0.0.2"));
// DC2
- metadata.updateNormalToken(new StringToken("B"), InetAddress.getByName("127.0.0.4"));
- metadata.updateNormalToken(new StringToken("D"), InetAddress.getByName("127.0.0.5"));
+ metadata.updateNormalToken(new StringToken("B"), InetAddressAndPort.getByName("127.0.0.4"));
+ metadata.updateNormalToken(new StringToken("D"), InetAddressAndPort.getByName("127.0.0.5"));
Map<String, String> replicationMap = new HashMap<>();
replicationMap.put(ReplicationParams.CLASS, NetworkTopologyStrategy.class.getName());
@@ -107,14 +108,14 @@
Keyspace.clear("Keyspace1");
KeyspaceMetadata meta = KeyspaceMetadata.create("Keyspace1", KeyspaceParams.create(false, replicationMap));
- Schema.instance.setKeyspaceMetadata(meta);
+ Schema.instance.load(meta);
- Optional<InetAddress> naturalEndpoint = ViewUtils.getViewNaturalEndpoint("Keyspace1",
- new StringToken("CA"),
- new StringToken("BB"));
+ Optional<Replica> naturalEndpoint = ViewUtils.getViewNaturalEndpoint("Keyspace1",
+ new StringToken("CA"),
+ new StringToken("BB"));
Assert.assertTrue(naturalEndpoint.isPresent());
- Assert.assertEquals(InetAddress.getByName("127.0.0.1"), naturalEndpoint.get());
+ Assert.assertEquals(InetAddressAndPort.getByName("127.0.0.1"), naturalEndpoint.get().endpoint());
}
@Test
@@ -124,12 +125,12 @@
metadata.clearUnsafe();
// DC1
- metadata.updateNormalToken(new StringToken("A"), InetAddress.getByName("127.0.0.1"));
- metadata.updateNormalToken(new StringToken("C"), InetAddress.getByName("127.0.0.2"));
+ metadata.updateNormalToken(new StringToken("A"), InetAddressAndPort.getByName("127.0.0.1"));
+ metadata.updateNormalToken(new StringToken("C"), InetAddressAndPort.getByName("127.0.0.2"));
// DC2
- metadata.updateNormalToken(new StringToken("B"), InetAddress.getByName("127.0.0.4"));
- metadata.updateNormalToken(new StringToken("D"), InetAddress.getByName("127.0.0.5"));
+ metadata.updateNormalToken(new StringToken("B"), InetAddressAndPort.getByName("127.0.0.4"));
+ metadata.updateNormalToken(new StringToken("D"), InetAddressAndPort.getByName("127.0.0.5"));
Map<String, String> replicationMap = new HashMap<>();
replicationMap.put(ReplicationParams.CLASS, NetworkTopologyStrategy.class.getName());
@@ -139,11 +140,11 @@
Keyspace.clear("Keyspace1");
KeyspaceMetadata meta = KeyspaceMetadata.create("Keyspace1", KeyspaceParams.create(false, replicationMap));
- Schema.instance.setKeyspaceMetadata(meta);
+ Schema.instance.load(meta);
- Optional<InetAddress> naturalEndpoint = ViewUtils.getViewNaturalEndpoint("Keyspace1",
- new StringToken("AB"),
- new StringToken("BB"));
+ Optional<Replica> naturalEndpoint = ViewUtils.getViewNaturalEndpoint("Keyspace1",
+ new StringToken("AB"),
+ new StringToken("BB"));
Assert.assertFalse(naturalEndpoint.isPresent());
}
diff --git a/test/unit/org/apache/cassandra/db/virtual/SettingsTableTest.java b/test/unit/org/apache/cassandra/db/virtual/SettingsTableTest.java
new file mode 100644
index 0000000..7a27282
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/virtual/SettingsTableTest.java
@@ -0,0 +1,245 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.virtual;
+
+import java.lang.reflect.Field;
+import java.lang.reflect.Modifier;
+import java.util.Arrays;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import com.google.common.collect.ImmutableList;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import com.datastax.driver.core.ResultSet;
+import com.datastax.driver.core.Row;
+import org.apache.cassandra.config.Config;
+import org.apache.cassandra.config.EncryptionOptions.ServerEncryptionOptions.InternodeEncryption;
+import org.apache.cassandra.config.ParameterizedClass;
+import org.apache.cassandra.cql3.CQLTester;
+
+public class SettingsTableTest extends CQLTester
+{
+ private static final String KS_NAME = "vts";
+
+ private Config config;
+ private SettingsTable table;
+
+ @BeforeClass
+ public static void setUpClass()
+ {
+ CQLTester.setUpClass();
+ }
+
+ @Before
+ public void config()
+ {
+ config = new Config();
+ table = new SettingsTable(KS_NAME, config);
+ VirtualKeyspaceRegistry.instance.register(new VirtualKeyspace(KS_NAME, ImmutableList.of(table)));
+ }
+
+ private String getValue(Field f)
+ {
+ Object untypedValue = table.getValue(f);
+ String value = null;
+ if (untypedValue != null)
+ {
+ if (untypedValue.getClass().isArray())
+ {
+ value = Arrays.toString((Object[]) untypedValue);
+ }
+ else
+ value = untypedValue.toString();
+ }
+ return value;
+ }
+
+ @Test
+ public void testSelectAll() throws Throwable
+ {
+ int paging = (int) (Math.random() * 100 + 1);
+ ResultSet result = executeNetWithPaging("SELECT * FROM vts.settings", paging);
+ int i = 0;
+ for (Row r : result)
+ {
+ i++;
+ String name = r.getString("name");
+ Field f = SettingsTable.FIELDS.get(name);
+ if (f != null) // skip overrides
+ Assert.assertEquals(getValue(f), r.getString("value"));
+ }
+ Assert.assertTrue(SettingsTable.FIELDS.size() <= i);
+ }
+
+ @Test
+ public void testSelectPartition() throws Throwable
+ {
+ List<Field> fields = Arrays.stream(Config.class.getFields())
+ .filter(f -> !Modifier.isStatic(f.getModifiers()))
+ .collect(Collectors.toList());
+ for (Field f : fields)
+ {
+ if (table.overrides.containsKey(f.getName()))
+ continue;
+
+ String q = "SELECT * FROM vts.settings WHERE name = '"+f.getName()+'\'';
+ assertRowsNet(executeNet(q), new Object[] {f.getName(), getValue(f)});
+ }
+ }
+
+ @Test
+ public void testSelectEmpty() throws Throwable
+ {
+ String q = "SELECT * FROM vts.settings WHERE name = 'EMPTY'";
+ assertRowsNet(executeNet(q));
+ }
+
+ @Test
+ public void testSelectOverride() throws Throwable
+ {
+ String q = "SELECT * FROM vts.settings WHERE name = 'server_encryption_options_enabled'";
+ assertRowsNet(executeNet(q), new Object[] {"server_encryption_options_enabled", "false"});
+ q = "SELECT * FROM vts.settings WHERE name = 'server_encryption_options_XYZ'";
+ assertRowsNet(executeNet(q));
+ }
+
+ private void check(String setting, String expected) throws Throwable
+ {
+ String q = "SELECT * FROM vts.settings WHERE name = '"+setting+'\'';
+ assertRowsNet(executeNet(q), new Object[] {setting, expected});
+ }
+
+ @Test
+ public void testEncryptionOverride() throws Throwable
+ {
+ String pre = "server_encryption_options_";
+ check(pre + "enabled", "false");
+ String all = "SELECT * FROM vts.settings WHERE " +
+ "name > 'server_encryption' AND name < 'server_encryptionz' ALLOW FILTERING";
+
+ Assert.assertEquals(9, executeNet(all).all().size());
+
+ check(pre + "algorithm", null);
+ config.server_encryption_options = config.server_encryption_options.withAlgorithm("SUPERSSL");
+ check(pre + "algorithm", "SUPERSSL");
+
+ check(pre + "cipher_suites", "[]");
+ config.server_encryption_options = config.server_encryption_options.withCipherSuites("c1", "c2");
+ check(pre + "cipher_suites", "[c1, c2]");
+
+ check(pre + "protocol", config.server_encryption_options.protocol);
+ config.server_encryption_options = config.server_encryption_options.withProtocol("TLSv5");
+ check(pre + "protocol", "TLSv5");
+
+ check(pre + "optional", "true");
+ config.server_encryption_options = config.server_encryption_options.withOptional(false);
+ check(pre + "optional", "false");
+
+ check(pre + "client_auth", "false");
+ config.server_encryption_options = config.server_encryption_options.withRequireClientAuth(true);
+ check(pre + "client_auth", "true");
+
+ check(pre + "endpoint_verification", "false");
+ config.server_encryption_options = config.server_encryption_options.withRequireEndpointVerification(true);
+ check(pre + "endpoint_verification", "true");
+
+ check(pre + "internode_encryption", "none");
+ config.server_encryption_options = config.server_encryption_options.withInternodeEncryption(InternodeEncryption.all);
+ check(pre + "internode_encryption", "all");
+ check(pre + "enabled", "true");
+
+ check(pre + "legacy_ssl_storage_port", "false");
+ config.server_encryption_options = config.server_encryption_options.withLegacySslStoragePort(true);
+ check(pre + "legacy_ssl_storage_port", "true");
+ }
+
+ @Test
+ public void testAuditOverride() throws Throwable
+ {
+ String pre = "audit_logging_options_";
+ check(pre + "enabled", "false");
+ String all = "SELECT * FROM vts.settings WHERE " +
+ "name > 'audit_logging' AND name < 'audit_loggingz' ALLOW FILTERING";
+
+ config.audit_logging_options.enabled = true;
+ Assert.assertEquals(9, executeNet(all).all().size());
+ check(pre + "enabled", "true");
+
+ check(pre + "logger", "BinAuditLogger");
+ config.audit_logging_options.logger = new ParameterizedClass("logger", null);
+ check(pre + "logger", "logger");
+
+ config.audit_logging_options.audit_logs_dir = "dir";
+ check(pre + "audit_logs_dir", "dir");
+
+ check(pre + "included_keyspaces", "");
+ config.audit_logging_options.included_keyspaces = "included_keyspaces";
+ check(pre + "included_keyspaces", "included_keyspaces");
+
+ check(pre + "excluded_keyspaces", "system,system_schema,system_virtual_schema");
+ config.audit_logging_options.excluded_keyspaces = "excluded_keyspaces";
+ check(pre + "excluded_keyspaces", "excluded_keyspaces");
+
+ check(pre + "included_categories", "");
+ config.audit_logging_options.included_categories = "included_categories";
+ check(pre + "included_categories", "included_categories");
+
+ check(pre + "excluded_categories", "");
+ config.audit_logging_options.excluded_categories = "excluded_categories";
+ check(pre + "excluded_categories", "excluded_categories");
+
+ check(pre + "included_users", "");
+ config.audit_logging_options.included_users = "included_users";
+ check(pre + "included_users", "included_users");
+
+ check(pre + "excluded_users", "");
+ config.audit_logging_options.excluded_users = "excluded_users";
+ check(pre + "excluded_users", "excluded_users");
+ }
+
+ @Test
+ public void testTransparentEncryptionOptionsOverride() throws Throwable
+ {
+ String pre = "transparent_data_encryption_options_";
+ check(pre + "enabled", "false");
+ String all = "SELECT * FROM vts.settings WHERE " +
+ "name > 'transparent_data_encryption_options' AND " +
+ "name < 'transparent_data_encryption_optionsz' ALLOW FILTERING";
+
+ config.transparent_data_encryption_options.enabled = true;
+ Assert.assertEquals(4, executeNet(all).all().size());
+ check(pre + "enabled", "true");
+
+ check(pre + "cipher", "AES/CBC/PKCS5Padding");
+ config.transparent_data_encryption_options.cipher = "cipher";
+ check(pre + "cipher", "cipher");
+
+ check(pre + "chunk_length_kb", "64");
+ config.transparent_data_encryption_options.chunk_length_kb = 5;
+ check(pre + "chunk_length_kb", "5");
+
+ check(pre + "iv_length", "16");
+ config.transparent_data_encryption_options.iv_length = 7;
+ check(pre + "iv_length", "7");
+ }
+}
diff --git a/test/unit/org/apache/cassandra/db/virtual/SystemPropertiesTableTest.java b/test/unit/org/apache/cassandra/db/virtual/SystemPropertiesTableTest.java
new file mode 100644
index 0000000..2ec0683
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/virtual/SystemPropertiesTableTest.java
@@ -0,0 +1,188 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.virtual;
+
+import java.lang.reflect.Field;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import com.datastax.driver.core.ResultSet;
+import com.datastax.driver.core.Row;
+import org.apache.cassandra.cql3.CQLTester;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Maps;
+
+public class SystemPropertiesTableTest extends CQLTester
+{
+ private static final String KS_NAME = "vts";
+ private static final Map<String, String> ORIGINAL_ENV_MAP = System.getenv();
+ private static final String TEST_PROP = "org.apache.cassandra.db.virtual.SystemPropertiesTableTest";
+
+ private SystemPropertiesTable table;
+
+ @BeforeClass
+ public static void setUpClass()
+ {
+ CQLTester.setUpClass();
+ }
+
+ @Before
+ public void config()
+ {
+ table = new SystemPropertiesTable(KS_NAME);
+ VirtualKeyspaceRegistry.instance.register(new VirtualKeyspace(KS_NAME, ImmutableList.of(table)));
+ }
+
+ @Test
+ public void testSelectAll() throws Throwable
+ {
+ ResultSet result = executeNet("SELECT * FROM vts.system_properties");
+
+ for (Row r : result)
+ Assert.assertEquals(System.getProperty(r.getString("name"), System.getenv(r.getString("name"))), r.getString("value"));
+ }
+
+ @Test
+ public void testSelectPartition() throws Throwable
+ {
+ List<String> properties = Stream.concat(System.getProperties().stringPropertyNames().stream(),
+ System.getenv().keySet().stream())
+ .filter(name -> SystemPropertiesTable.isCassandraRelevant(name))
+ .collect(Collectors.toList());
+
+ for (String property : properties)
+ {
+ String q = "SELECT * FROM vts.system_properties WHERE name = '" + property + '\'';
+ assertRowsNet(executeNet(q), new Object[] {property, System.getProperty(property, System.getenv(property))});
+ }
+ }
+
+ @Test
+ public void testSelectEmpty() throws Throwable
+ {
+ String q = "SELECT * FROM vts.system_properties WHERE name = 'EMPTY'";
+ assertRowsNet(executeNet(q));
+ }
+
+ @Test
+ public void testSelectProperty() throws Throwable
+ {
+ try
+ {
+ String value = "test_value";
+ System.setProperty(TEST_PROP, value);
+ String q = String.format("SELECT * FROM vts.system_properties WHERE name = '%s'", TEST_PROP);
+ assertRowsNet(executeNet(q), new Object[] {TEST_PROP, value});
+ }
+ finally
+ {
+ System.clearProperty(TEST_PROP);
+ }
+ }
+
+ @Test
+ public void testSelectEnv() throws Throwable
+ {
+ try
+ {
+ String value = "test_value";
+ addEnv(TEST_PROP, value);
+ String q = String.format("SELECT * FROM vts.system_properties WHERE name = '%s'", TEST_PROP);
+ assertRowsNet(executeNet(q), new Object[] {TEST_PROP, value});
+ }
+ finally
+ {
+ resetEnv();
+ }
+ }
+
+ @Test
+ public void testSelectPropertyOverEnv() throws Throwable
+ {
+ try
+ {
+ String value = "test_value";
+ System.setProperty(TEST_PROP, value);
+ addEnv(TEST_PROP, "wrong_value");
+ String q = String.format("SELECT * FROM vts.system_properties WHERE name = '%s'", TEST_PROP);
+ assertRowsNet(executeNet(q), new Object[] {TEST_PROP, value});
+ }
+ finally
+ {
+ System.clearProperty(TEST_PROP);
+ resetEnv();
+ }
+ }
+
+ private static void addEnv(String env, String value) throws ReflectiveOperationException
+ {
+ Map<String, String> envMap = Maps.newConcurrentMap();
+ envMap.putAll(System.getenv());
+ envMap.put(env, value);
+ setEnv(envMap);
+ }
+
+ private static void resetEnv() throws ReflectiveOperationException
+ {
+ setEnv(ORIGINAL_ENV_MAP);
+ }
+
+ private static void setEnv(Map<String, String> newenv) throws ReflectiveOperationException
+ {
+ try
+ {
+ Class<?> cls = Class.forName("java.lang.ProcessEnvironment");
+ Field field = cls.getDeclaredField("theEnvironment");
+ field.setAccessible(true);
+ Map<String, String> envMap = (Map<String, String>) field.get(null);
+ envMap.clear();
+ envMap.putAll(newenv);
+ field = cls.getDeclaredField("theCaseInsensitiveEnvironment");
+ field.setAccessible(true);
+ envMap = (Map<String, String>) field.get(null);
+ envMap.clear();
+ envMap.putAll(newenv);
+ }
+ catch (NoSuchFieldException ignore)
+ {
+ Class[] classes = Collections.class.getDeclaredClasses();
+ Map<String, String> envMap = System.getenv();
+ for(Class cl : classes) {
+ if("java.util.Collections$UnmodifiableMap".equals(cl.getName()))
+ {
+ Field field = cl.getDeclaredField("m");
+ field.setAccessible(true);
+ Object obj = field.get(envMap);
+ envMap = (Map<String, String>) obj;
+ envMap.clear();
+ envMap.putAll(newenv);
+ }
+ }
+ }
+ }
+}
diff --git a/test/unit/org/apache/cassandra/dht/BootStrapperTest.java b/test/unit/org/apache/cassandra/dht/BootStrapperTest.java
index ed15a70..c5cce58 100644
--- a/test/unit/org/apache/cassandra/dht/BootStrapperTest.java
+++ b/test/unit/org/apache/cassandra/dht/BootStrapperTest.java
@@ -18,18 +18,24 @@
package org.apache.cassandra.dht;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.fail;
+import org.junit.Assert;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.Collection;
-import java.util.HashSet;
import java.util.List;
import java.util.Map;
-import java.util.Set;
+import java.util.Random;
+import java.util.UUID;
+import com.google.common.base.Predicate;
+import com.google.common.base.Predicates;
import com.google.common.collect.Lists;
+import com.google.common.collect.Multimap;
+import org.apache.cassandra.dht.RangeStreamer.FetchReplica;
import org.apache.commons.math3.stat.descriptive.SummaryStatistics;
import org.junit.AfterClass;
@@ -40,7 +46,9 @@
import org.apache.cassandra.OrderedJUnit4ClassRunner;
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.config.Schema;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.Replica;
+import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.dht.tokenallocator.TokenAllocation;
import org.apache.cassandra.exceptions.ConfigurationException;
@@ -51,6 +59,7 @@
import org.apache.cassandra.locator.TokenMetadata;
import org.apache.cassandra.schema.KeyspaceParams;
import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.streaming.StreamOperation;
import org.apache.cassandra.utils.FBUtilities;
@RunWith(OrderedJUnit4ClassRunner.class)
@@ -58,6 +67,7 @@
{
static IPartitioner oldPartitioner;
+ static Predicate<Replica> originalAlivePredicate = RangeStreamer.ALIVE_PREDICATE;
@BeforeClass
public static void setup() throws ConfigurationException
{
@@ -66,12 +76,14 @@
SchemaLoader.startGossiper();
SchemaLoader.prepareServer();
SchemaLoader.schemaDefinition("BootStrapperTest");
+ RangeStreamer.ALIVE_PREDICATE = Predicates.alwaysTrue();
}
@AfterClass
public static void tearDown()
{
DatabaseDescriptor.setPartitionerUnsafe(oldPartitioner);
+ RangeStreamer.ALIVE_PREDICATE = originalAlivePredicate;
}
@Test
@@ -80,7 +92,7 @@
final int[] clusterSizes = new int[] { 1, 3, 5, 10, 100};
for (String keyspaceName : Schema.instance.getNonLocalStrategyKeyspaces())
{
- int replicationFactor = Keyspace.open(keyspaceName).getReplicationStrategy().getReplicationFactor();
+ int replicationFactor = Keyspace.open(keyspaceName).getReplicationStrategy().getReplicationFactor().allReplicas;
for (int clusterSize : clusterSizes)
if (clusterSize >= replicationFactor)
testSourceTargetComputation(keyspaceName, clusterSize, replicationFactor);
@@ -94,40 +106,37 @@
generateFakeEndpoints(numOldNodes);
Token myToken = tmd.partitioner.getRandomToken();
- InetAddress myEndpoint = InetAddress.getByName("127.0.0.1");
+ InetAddressAndPort myEndpoint = InetAddressAndPort.getByName("127.0.0.1");
assertEquals(numOldNodes, tmd.sortedTokens().size());
- RangeStreamer s = new RangeStreamer(tmd, null, myEndpoint, "Bootstrap", true, DatabaseDescriptor.getEndpointSnitch(), new StreamStateStore(), false);
IFailureDetector mockFailureDetector = new IFailureDetector()
{
- public boolean isAlive(InetAddress ep)
+ public boolean isAlive(InetAddressAndPort ep)
{
return true;
}
- public void interpret(InetAddress ep) { throw new UnsupportedOperationException(); }
- public void report(InetAddress ep) { throw new UnsupportedOperationException(); }
+ public void interpret(InetAddressAndPort ep) { throw new UnsupportedOperationException(); }
+ public void report(InetAddressAndPort ep) { throw new UnsupportedOperationException(); }
public void registerFailureDetectionEventListener(IFailureDetectionEventListener listener) { throw new UnsupportedOperationException(); }
public void unregisterFailureDetectionEventListener(IFailureDetectionEventListener listener) { throw new UnsupportedOperationException(); }
- public void remove(InetAddress ep) { throw new UnsupportedOperationException(); }
- public void forceConviction(InetAddress ep) { throw new UnsupportedOperationException(); }
+ public void remove(InetAddressAndPort ep) { throw new UnsupportedOperationException(); }
+ public void forceConviction(InetAddressAndPort ep) { throw new UnsupportedOperationException(); }
};
- s.addSourceFilter(new RangeStreamer.FailureDetectorSourceFilter(mockFailureDetector));
+ RangeStreamer s = new RangeStreamer(tmd, null, myEndpoint, StreamOperation.BOOTSTRAP, true, DatabaseDescriptor.getEndpointSnitch(), new StreamStateStore(), mockFailureDetector, false, 1);
+ assertNotNull(Keyspace.open(keyspaceName));
s.addRanges(keyspaceName, Keyspace.open(keyspaceName).getReplicationStrategy().getPendingAddressRanges(tmd, myToken, myEndpoint));
- Collection<Map.Entry<InetAddress, Collection<Range<Token>>>> toFetch = s.toFetch().get(keyspaceName);
+
+ Multimap<InetAddressAndPort, FetchReplica> toFetch = s.toFetch().get(keyspaceName);
// Check we get get RF new ranges in total
- Set<Range<Token>> ranges = new HashSet<>();
- for (Map.Entry<InetAddress, Collection<Range<Token>>> e : toFetch)
- ranges.addAll(e.getValue());
-
- assertEquals(replicationFactor, ranges.size());
+ assertEquals(replicationFactor, toFetch.size());
// there isn't any point in testing the size of these collections for any specific size. When a random partitioner
// is used, they will vary.
- assert toFetch.iterator().next().getValue().size() > 0;
- assert !toFetch.iterator().next().getKey().equals(myEndpoint);
+ assert toFetch.values().size() > 0;
+ assert toFetch.keys().stream().noneMatch(myEndpoint::equals);
return s;
}
@@ -142,6 +151,8 @@
generateFakeEndpoints(tmd, numOldNodes, numVNodes, "0", "0");
}
+ Random rand = new Random(1);
+
private void generateFakeEndpoints(TokenMetadata tmd, int numOldNodes, int numVNodes, String dc, String rack) throws UnknownHostException
{
IPartitioner p = tmd.partitioner;
@@ -149,10 +160,10 @@
for (int i = 1; i <= numOldNodes; i++)
{
// leave .1 for myEndpoint
- InetAddress addr = InetAddress.getByName("127." + dc + "." + rack + "." + (i + 1));
+ InetAddressAndPort addr = InetAddressAndPort.getByName("127." + dc + "." + rack + "." + (i + 1));
List<Token> tokens = Lists.newArrayListWithCapacity(numVNodes);
for (int j = 0; j < numVNodes; ++j)
- tokens.add(p.getRandomToken());
+ tokens.add(p.getRandomToken(rand));
tmd.updateNormalTokens(tokens, addr);
}
@@ -165,10 +176,21 @@
String ks = "BootStrapperTestKeyspace3";
TokenMetadata tm = new TokenMetadata();
generateFakeEndpoints(tm, 10, vn);
- InetAddress addr = FBUtilities.getBroadcastAddress();
+ InetAddressAndPort addr = FBUtilities.getBroadcastAddressAndPort();
allocateTokensForNode(vn, ks, tm, addr);
}
+ @Test
+ public void testAllocateTokensLocalRf() throws UnknownHostException
+ {
+ int vn = 16;
+ int allocateTokensForLocalRf = 3;
+ TokenMetadata tm = new TokenMetadata();
+ generateFakeEndpoints(tm, 10, vn);
+ InetAddressAndPort addr = FBUtilities.getBroadcastAddressAndPort();
+ allocateTokensForNode(vn, allocateTokensForLocalRf, tm, addr);
+ }
+
public void testAllocateTokensNetworkStrategy(int rackCount, int replicas) throws UnknownHostException
{
IEndpointSnitch oldSnitch = DatabaseDescriptor.getEndpointSnitch();
@@ -178,12 +200,19 @@
int vn = 16;
String ks = "BootStrapperTestNTSKeyspace" + rackCount + replicas;
String dc = "1";
+
+ // Register peers with expected DC for NetworkTopologyStrategy.
+ TokenMetadata metadata = StorageService.instance.getTokenMetadata();
+ metadata.clearUnsafe();
+ metadata.updateHostId(UUID.randomUUID(), InetAddressAndPort.getByName("127.1.0.99"));
+ metadata.updateHostId(UUID.randomUUID(), InetAddressAndPort.getByName("127.15.0.99"));
+
SchemaLoader.createKeyspace(ks, KeyspaceParams.nts(dc, replicas, "15", 15), SchemaLoader.standardCFMD(ks, "Standard1"));
TokenMetadata tm = StorageService.instance.getTokenMetadata();
tm.clearUnsafe();
for (int i = 0; i < rackCount; ++i)
generateFakeEndpoints(tm, 10, vn, dc, Integer.toString(i));
- InetAddress addr = InetAddress.getByName("127." + dc + ".0.99");
+ InetAddressAndPort addr = InetAddressAndPort.getByName("127." + dc + ".0.99");
allocateTokensForNode(vn, ks, tm, addr);
// Note: Not matching replication factor in second datacentre, but this should not affect us.
} finally {
@@ -221,7 +250,7 @@
testAllocateTokensNetworkStrategy(1, 1);
}
- private void allocateTokensForNode(int vn, String ks, TokenMetadata tm, InetAddress addr)
+ private void allocateTokensForNode(int vn, String ks, TokenMetadata tm, InetAddressAndPort addr)
{
SummaryStatistics os = TokenAllocation.replicatedOwnershipStats(tm.cloneOnlyTokenMap(), Keyspace.open(ks).getReplicationStrategy(), addr);
Collection<Token> tokens = BootStrapper.allocateTokens(tm, addr, ks, vn, 0);
@@ -231,6 +260,14 @@
verifyImprovement(os, ns);
}
+ private void allocateTokensForNode(int vn, int rf, TokenMetadata tm, InetAddressAndPort addr)
+ {
+ Collection<Token> tokens = BootStrapper.allocateTokens(tm, addr, rf, vn, 0);
+ assertEquals(vn, tokens.size());
+ tm.updateNormalTokens(tokens, addr);
+ // SummaryStatistics is not implemented for `allocate_tokens_for_local_replication_factor` so can't be verified
+ }
+
private void verifyImprovement(SummaryStatistics os, SummaryStatistics ns)
{
if (ns.getStandardDeviation() > os.getStandardDeviation())
@@ -239,7 +276,54 @@
}
}
-
+ @Test
+ public void testAllocateTokensRfEqRacks() throws UnknownHostException
+ {
+ IEndpointSnitch oldSnitch = DatabaseDescriptor.getEndpointSnitch();
+ try
+ {
+ DatabaseDescriptor.setEndpointSnitch(new RackInferringSnitch());
+ int vn = 8;
+ int replicas = 3;
+ int rackCount = replicas;
+ String ks = "BootStrapperTestNTSKeyspaceRfEqRacks";
+ String dc = "1";
+
+ TokenMetadata metadata = StorageService.instance.getTokenMetadata();
+ metadata.clearUnsafe();
+ metadata.updateHostId(UUID.randomUUID(), InetAddressAndPort.getByName("127.1.0.99"));
+ metadata.updateHostId(UUID.randomUUID(), InetAddressAndPort.getByName("127.15.0.99"));
+
+ SchemaLoader.createKeyspace(ks, KeyspaceParams.nts(dc, replicas, "15", 15), SchemaLoader.standardCFMD(ks, "Standard1"));
+ int base = 5;
+ for (int i = 0; i < rackCount; ++i)
+ generateFakeEndpoints(metadata, base << i, vn, dc, Integer.toString(i)); // unbalanced racks
+
+ int cnt = 5;
+ for (int i = 0; i < cnt; ++i)
+ allocateTokensForNode(vn, ks, metadata, InetAddressAndPort.getByName("127." + dc + ".0." + (99 + i)));
+
+ double target = 1.0 / (base + cnt);
+ double permittedOver = 1.0 / (2 * vn + 1) + 0.01;
+
+ Map<InetAddress, Float> ownership = StorageService.instance.effectiveOwnership(ks);
+ boolean failed = false;
+ for (Map.Entry<InetAddress, Float> o : ownership.entrySet())
+ {
+ int rack = o.getKey().getAddress()[2];
+ if (rack != 0)
+ continue;
+
+ System.out.format("Node %s owns %f ratio to optimal %.2f\n", o.getKey(), o.getValue(), o.getValue() / target);
+ if (o.getValue()/target > 1 + permittedOver)
+ failed = true;
+ }
+ Assert.assertFalse(String.format("One of the nodes in the rack has over %.2f%% overutilization.", permittedOver * 100), failed);
+ } finally {
+ DatabaseDescriptor.setEndpointSnitch(oldSnitch);
+ }
+ }
+
@Test
public void testAllocateTokensMultipleKeyspaces() throws UnknownHostException
{
@@ -251,14 +335,14 @@
TokenMetadata tm = new TokenMetadata();
generateFakeEndpoints(tm, 10, vn);
- InetAddress dcaddr = FBUtilities.getBroadcastAddress();
+ InetAddressAndPort dcaddr = FBUtilities.getBroadcastAddressAndPort();
SummaryStatistics os3 = TokenAllocation.replicatedOwnershipStats(tm, Keyspace.open(ks3).getReplicationStrategy(), dcaddr);
SummaryStatistics os2 = TokenAllocation.replicatedOwnershipStats(tm, Keyspace.open(ks2).getReplicationStrategy(), dcaddr);
String cks = ks3;
String nks = ks2;
for (int i=11; i<=20; ++i)
{
- allocateTokensForNode(vn, cks, tm, InetAddress.getByName("127.0.0." + (i + 1)));
+ allocateTokensForNode(vn, cks, tm, InetAddressAndPort.getByName("127.0.0." + (i + 1)));
String t = cks; cks = nks; nks = t;
}
diff --git a/test/unit/org/apache/cassandra/dht/KeyCollisionTest.java b/test/unit/org/apache/cassandra/dht/KeyCollisionTest.java
index c0ff9ca..5b5365d 100644
--- a/test/unit/org/apache/cassandra/dht/KeyCollisionTest.java
+++ b/test/unit/org/apache/cassandra/dht/KeyCollisionTest.java
@@ -27,7 +27,7 @@
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.Util;
import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.config.Schema;
+import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.db.RowUpdateBuilder;
@@ -95,7 +95,7 @@
private void insert(String key)
{
- RowUpdateBuilder builder = new RowUpdateBuilder(Schema.instance.getCFMetaData(KEYSPACE1, CF), FBUtilities.timestampMicros(), key);
+ RowUpdateBuilder builder = new RowUpdateBuilder(Schema.instance.getTableMetadata(KEYSPACE1, CF), FBUtilities.timestampMicros(), key);
builder.clustering("c").add("val", "asdf").build().applyUnsafe();
}
diff --git a/test/unit/org/apache/cassandra/dht/LengthPartitioner.java b/test/unit/org/apache/cassandra/dht/LengthPartitioner.java
index 97f2dcc..bd6f3d4 100644
--- a/test/unit/org/apache/cassandra/dht/LengthPartitioner.java
+++ b/test/unit/org/apache/cassandra/dht/LengthPartitioner.java
@@ -22,8 +22,8 @@
import java.util.*;
import java.util.concurrent.ThreadLocalRandom;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.Schema;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.db.BufferDecoratedKey;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.marshal.AbstractType;
@@ -143,12 +143,12 @@
for (String ks : Schema.instance.getKeyspaces())
{
- for (CFMetaData cfmd : Schema.instance.getTablesAndViews(ks))
+ for (TableMetadata cfmd : Schema.instance.getTablesAndViews(ks))
{
for (Range<Token> r : sortedRanges)
{
// Looping over every KS:CF:Range, get the splits size and add it to the count
- allTokens.put(r.right, allTokens.get(r.right) + StorageService.instance.getSplits(ks, cfmd.cfName, r, 1).size());
+ allTokens.put(r.right, allTokens.get(r.right) + StorageService.instance.getSplits(ks, cfmd.name, r, 1).size());
}
}
}
diff --git a/test/unit/org/apache/cassandra/dht/PartitionerTestCase.java b/test/unit/org/apache/cassandra/dht/PartitionerTestCase.java
index 33e9d60..ec535b0 100644
--- a/test/unit/org/apache/cassandra/dht/PartitionerTestCase.java
+++ b/test/unit/org/apache/cassandra/dht/PartitionerTestCase.java
@@ -29,6 +29,7 @@
import org.junit.Test;
import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.commitlog.CommitLog;
import org.apache.cassandra.service.StorageService;
import static org.junit.Assert.assertEquals;
@@ -47,6 +48,7 @@
public static void initDD()
{
DatabaseDescriptor.daemonInitialization();
+ CommitLog.instance.start();
}
@Before
diff --git a/test/unit/org/apache/cassandra/dht/RangeFetchMapCalculatorTest.java b/test/unit/org/apache/cassandra/dht/RangeFetchMapCalculatorTest.java
new file mode 100644
index 0000000..e2b09bc
--- /dev/null
+++ b/test/unit/org/apache/cassandra/dht/RangeFetchMapCalculatorTest.java
@@ -0,0 +1,426 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.dht;
+
+import java.net.UnknownHostException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Map;
+
+import com.google.common.base.Predicate;
+import com.google.common.collect.Multimap;
+import org.apache.cassandra.locator.EndpointsByRange;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.locator.AbstractNetworkTopologySnitch;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.Replica;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class RangeFetchMapCalculatorTest
+{
+ @BeforeClass
+ public static void setupUpSnitch()
+ {
+ DatabaseDescriptor.daemonInitialization();
+ DatabaseDescriptor.setPartitionerUnsafe(RandomPartitioner.instance);
+ DatabaseDescriptor.setEndpointSnitch(new AbstractNetworkTopologySnitch()
+ {
+ //Odd IPs are in DC1 and Even are in DC2. Endpoints upto .14 will have unique racks and
+ // then will be same for a set of three.
+ @Override
+ public String getRack(InetAddressAndPort endpoint)
+ {
+ return "RAC1";
+ }
+
+ @Override
+ public String getDatacenter(InetAddressAndPort endpoint)
+ {
+ if (getIPLastPart(endpoint) <= 50)
+ return DatabaseDescriptor.getLocalDataCenter();
+ else if (getIPLastPart(endpoint) % 2 == 0)
+ return DatabaseDescriptor.getLocalDataCenter();
+ else
+ return DatabaseDescriptor.getLocalDataCenter() + "Remote";
+ }
+
+ private int getIPLastPart(InetAddressAndPort endpoint)
+ {
+ String str = endpoint.address.toString();
+ int index = str.lastIndexOf(".");
+ return Integer.parseInt(str.substring(index + 1).trim());
+ }
+ });
+ }
+
+ @Test
+ public void testWithSingleSource() throws Exception
+ {
+ EndpointsByRange.Builder rangesWithSources = new EndpointsByRange.Builder();
+ addNonTrivialRangeAndSources(rangesWithSources, 1, 10, "127.0.0.1");
+ addNonTrivialRangeAndSources(rangesWithSources, 11, 20, "127.0.0.2");
+ addNonTrivialRangeAndSources(rangesWithSources, 21, 30, "127.0.0.3");
+ addNonTrivialRangeAndSources(rangesWithSources, 31, 40, "127.0.0.4");
+ addNonTrivialRangeAndSources(rangesWithSources, 41, 50, "127.0.0.5");
+
+ RangeFetchMapCalculator calculator = new RangeFetchMapCalculator(rangesWithSources.build(), Collections.emptyList(), "Test");
+ Multimap<InetAddressAndPort, Range<Token>> map = calculator.getRangeFetchMap();
+ validateRange(rangesWithSources, map);
+
+ Assert.assertEquals(4, map.asMap().keySet().size());
+ }
+
+ @Test
+ public void testWithNonOverlappingSource() throws Exception
+ {
+ EndpointsByRange.Builder rangesWithSources = new EndpointsByRange.Builder();
+ addNonTrivialRangeAndSources(rangesWithSources, 1, 10, "127.0.0.1", "127.0.0.2");
+ addNonTrivialRangeAndSources(rangesWithSources, 11, 20, "127.0.0.3", "127.0.0.4");
+ addNonTrivialRangeAndSources(rangesWithSources, 21, 30, "127.0.0.5", "127.0.0.6");
+ addNonTrivialRangeAndSources(rangesWithSources, 31, 40, "127.0.0.7", "127.0.0.8");
+ addNonTrivialRangeAndSources(rangesWithSources, 41, 50, "127.0.0.9", "127.0.0.10");
+
+ RangeFetchMapCalculator calculator = new RangeFetchMapCalculator(rangesWithSources.build(), Collections.emptyList(), "Test");
+ Multimap<InetAddressAndPort, Range<Token>> map = calculator.getRangeFetchMap();
+ validateRange(rangesWithSources, map);
+
+ Assert.assertEquals(5, map.asMap().keySet().size());
+ }
+
+ @Test
+ public void testWithRFThreeReplacement() throws Exception
+ {
+ EndpointsByRange.Builder rangesWithSources = new EndpointsByRange.Builder();
+ addNonTrivialRangeAndSources(rangesWithSources, 1, 10, "127.0.0.1", "127.0.0.2");
+ addNonTrivialRangeAndSources(rangesWithSources, 11, 20, "127.0.0.2", "127.0.0.3");
+ addNonTrivialRangeAndSources(rangesWithSources, 21, 30, "127.0.0.3", "127.0.0.4");
+
+ RangeFetchMapCalculator calculator = new RangeFetchMapCalculator(rangesWithSources.build(), Collections.emptyList(), "Test");
+ Multimap<InetAddressAndPort, Range<Token>> map = calculator.getRangeFetchMap();
+ validateRange(rangesWithSources, map);
+
+ //We should validate that it streamed from 3 unique sources
+ Assert.assertEquals(3, map.asMap().keySet().size());
+ }
+
+ @Test
+ public void testForMultipleRoundsComputation() throws Exception
+ {
+ EndpointsByRange.Builder rangesWithSources = new EndpointsByRange.Builder();
+ addNonTrivialRangeAndSources(rangesWithSources, 1, 10, "127.0.0.3");
+ addNonTrivialRangeAndSources(rangesWithSources, 11, 20, "127.0.0.3");
+ addNonTrivialRangeAndSources(rangesWithSources, 21, 30, "127.0.0.3");
+ addNonTrivialRangeAndSources(rangesWithSources, 31, 40, "127.0.0.3");
+ addNonTrivialRangeAndSources(rangesWithSources, 41, 50, "127.0.0.3", "127.0.0.2");
+
+ RangeFetchMapCalculator calculator = new RangeFetchMapCalculator(rangesWithSources.build(), Collections.emptyList(), "Test");
+ Multimap<InetAddressAndPort, Range<Token>> map = calculator.getRangeFetchMap();
+ validateRange(rangesWithSources, map);
+
+ //We should validate that it streamed from 2 unique sources
+ Assert.assertEquals(2, map.asMap().keySet().size());
+
+ assertArrays(Arrays.asList(generateNonTrivialRange(1, 10), generateNonTrivialRange(11, 20), generateNonTrivialRange(21, 30), generateNonTrivialRange(31, 40)),
+ map.asMap().get(InetAddressAndPort.getByName("127.0.0.3")));
+ assertArrays(Arrays.asList(generateNonTrivialRange(41, 50)), map.asMap().get(InetAddressAndPort.getByName("127.0.0.2")));
+ }
+
+ @Test
+ public void testForMultipleRoundsComputationWithLocalHost() throws Exception
+ {
+ EndpointsByRange.Builder rangesWithSources = new EndpointsByRange.Builder();
+ addNonTrivialRangeAndSources(rangesWithSources, 1, 10, "127.0.0.1");
+ addNonTrivialRangeAndSources(rangesWithSources, 11, 20, "127.0.0.1");
+ addNonTrivialRangeAndSources(rangesWithSources, 21, 30, "127.0.0.1");
+ addNonTrivialRangeAndSources(rangesWithSources, 31, 40, "127.0.0.1");
+ addNonTrivialRangeAndSources(rangesWithSources, 41, 50, "127.0.0.1", "127.0.0.2");
+
+ RangeFetchMapCalculator calculator = new RangeFetchMapCalculator(rangesWithSources.build(), Collections.emptyList(), "Test");
+ Multimap<InetAddressAndPort, Range<Token>> map = calculator.getRangeFetchMap();
+ validateRange(rangesWithSources, map);
+
+ //We should validate that it streamed from only non local host and only one range
+ Assert.assertEquals(1, map.asMap().keySet().size());
+
+ assertArrays(Arrays.asList(generateNonTrivialRange(41, 50)), map.asMap().get(InetAddressAndPort.getByName("127.0.0.2")));
+ }
+
+ @Test
+ public void testForEmptyGraph() throws Exception
+ {
+ EndpointsByRange.Builder rangesWithSources = new EndpointsByRange.Builder();
+ addNonTrivialRangeAndSources(rangesWithSources, 1, 10, "127.0.0.1");
+ addNonTrivialRangeAndSources(rangesWithSources, 11, 20, "127.0.0.1");
+ addNonTrivialRangeAndSources(rangesWithSources, 21, 30, "127.0.0.1");
+ addNonTrivialRangeAndSources(rangesWithSources, 31, 40, "127.0.0.1");
+ addNonTrivialRangeAndSources(rangesWithSources, 41, 50, "127.0.0.1");
+
+ RangeFetchMapCalculator calculator = new RangeFetchMapCalculator(rangesWithSources.build(), Collections.emptyList(), "Test");
+ Multimap<InetAddressAndPort, Range<Token>> map = calculator.getRangeFetchMap();
+ //All ranges map to local host so we will not stream anything.
+ assertTrue(map.isEmpty());
+ }
+
+ @Test
+ public void testWithNoSourceWithLocal() throws Exception
+ {
+ EndpointsByRange.Builder rangesWithSources = new EndpointsByRange.Builder();
+ addNonTrivialRangeAndSources(rangesWithSources, 1, 10, "127.0.0.1", "127.0.0.5");
+ addNonTrivialRangeAndSources(rangesWithSources, 11, 20, "127.0.0.2");
+ addNonTrivialRangeAndSources(rangesWithSources, 21, 30, "127.0.0.3");
+
+ //Return false for all except 127.0.0.5
+ final RangeStreamer.SourceFilter filter = new RangeStreamer.SourceFilter()
+ {
+ public boolean apply(Replica replica)
+ {
+ try
+ {
+ if (replica.endpoint().equals(InetAddressAndPort.getByName("127.0.0.5")))
+ return false;
+ else
+ return true;
+ }
+ catch (UnknownHostException e)
+ {
+ return true;
+ }
+ }
+
+ public String message(Replica replica)
+ {
+ return "Doesn't match 127.0.0.5";
+ }
+ };
+
+ RangeFetchMapCalculator calculator = new RangeFetchMapCalculator(rangesWithSources.build(), Arrays.asList(filter), "Test");
+ Multimap<InetAddressAndPort, Range<Token>> map = calculator.getRangeFetchMap();
+
+ validateRange(rangesWithSources, map);
+
+ //We should validate that it streamed from only non local host and only one range
+ Assert.assertEquals(2, map.asMap().keySet().size());
+
+ assertArrays(Arrays.asList(generateNonTrivialRange(11, 20)), map.asMap().get(InetAddressAndPort.getByName("127.0.0.2")));
+ assertArrays(Arrays.asList(generateNonTrivialRange(21, 30)), map.asMap().get(InetAddressAndPort.getByName("127.0.0.3")));
+ }
+
+ @Test (expected = IllegalStateException.class)
+ public void testWithNoLiveSource() throws Exception
+ {
+ EndpointsByRange.Builder rangesWithSources = new EndpointsByRange.Builder();
+ addNonTrivialRangeAndSources(rangesWithSources, 1, 10, "127.0.0.5");
+ addNonTrivialRangeAndSources(rangesWithSources, 11, 20, "127.0.0.2");
+ addNonTrivialRangeAndSources(rangesWithSources, 21, 30, "127.0.0.3");
+
+ final RangeStreamer.SourceFilter allDeadFilter = new RangeStreamer.SourceFilter()
+ {
+ public boolean apply(Replica replica)
+ {
+ return false;
+ }
+
+ public String message(Replica replica)
+ {
+ return "All dead";
+ }
+ };
+
+ RangeFetchMapCalculator calculator = new RangeFetchMapCalculator(rangesWithSources.build(), Arrays.asList(allDeadFilter), "Test");
+ calculator.getRangeFetchMap();
+ }
+
+ @Test
+ public void testForLocalDC() throws Exception
+ {
+ EndpointsByRange.Builder rangesWithSources = new EndpointsByRange.Builder();
+ addNonTrivialRangeAndSources(rangesWithSources, 1, 10, "127.0.0.1", "127.0.0.3", "127.0.0.53");
+ addNonTrivialRangeAndSources(rangesWithSources, 11, 20, "127.0.0.1", "127.0.0.3", "127.0.0.57");
+ addNonTrivialRangeAndSources(rangesWithSources, 21, 30, "127.0.0.2", "127.0.0.59", "127.0.0.61");
+
+ RangeFetchMapCalculator calculator = new RangeFetchMapCalculator(rangesWithSources.build(), new ArrayList<>(), "Test");
+ Multimap<InetAddressAndPort, Range<Token>> map = calculator.getRangeFetchMap();
+ validateRange(rangesWithSources, map);
+ Assert.assertEquals(2, map.asMap().size());
+
+ //Should have streamed from local DC endpoints
+ assertArrays(Arrays.asList(generateNonTrivialRange(21, 30)), map.asMap().get(InetAddressAndPort.getByName("127.0.0.2")));
+ assertArrays(Arrays.asList(generateNonTrivialRange(1, 10), generateNonTrivialRange(11, 20)), map.asMap().get(InetAddressAndPort.getByName("127.0.0.3")));
+ }
+
+ @Test
+ public void testForRemoteDC() throws Exception
+ {
+ EndpointsByRange.Builder rangesWithSources = new EndpointsByRange.Builder();
+ addNonTrivialRangeAndSources(rangesWithSources, 1, 10, "127.0.0.3", "127.0.0.51");
+ addNonTrivialRangeAndSources(rangesWithSources, 11, 20, "127.0.0.3", "127.0.0.55");
+ addNonTrivialRangeAndSources(rangesWithSources, 21, 30, "127.0.0.2", "127.0.0.59");
+
+ //Reject only 127.0.0.3 and accept everyone else
+ final RangeStreamer.SourceFilter localHostFilter = new RangeStreamer.SourceFilter()
+ {
+ public boolean apply(Replica replica)
+ {
+ try
+ {
+ if (replica.endpoint().equals(InetAddressAndPort.getByName("127.0.0.3")))
+ return false;
+ else
+ return true;
+ }
+ catch (UnknownHostException e)
+ {
+ return true;
+ }
+ }
+
+ public String message(Replica replica)
+ {
+ return "Not 127.0.0.3";
+ }
+ };
+
+ RangeFetchMapCalculator calculator = new RangeFetchMapCalculator(rangesWithSources.build(), Arrays.asList(localHostFilter), "Test");
+ Multimap<InetAddressAndPort, Range<Token>> map = calculator.getRangeFetchMap();
+ validateRange(rangesWithSources, map);
+ Assert.assertEquals(3, map.asMap().size());
+
+ //Should have streamed from remote DC endpoint
+ assertArrays(Arrays.asList(generateNonTrivialRange(1, 10)), map.asMap().get(InetAddressAndPort.getByName("127.0.0.51")));
+ assertArrays(Arrays.asList(generateNonTrivialRange(11, 20)), map.asMap().get(InetAddressAndPort.getByName("127.0.0.55")));
+ assertArrays(Arrays.asList(generateNonTrivialRange(21, 30)), map.asMap().get(InetAddressAndPort.getByName("127.0.0.2")));
+ }
+
+ @Test
+ public void testTrivialRanges() throws UnknownHostException
+ {
+ EndpointsByRange.Builder rangesWithSources = new EndpointsByRange.Builder();
+ // add non-trivial ranges
+ addNonTrivialRangeAndSources(rangesWithSources, 1, 10, "127.0.0.3", "127.0.0.51");
+ addNonTrivialRangeAndSources(rangesWithSources, 11, 20, "127.0.0.3", "127.0.0.55");
+ addNonTrivialRangeAndSources(rangesWithSources, 21, 30, "127.0.0.2", "127.0.0.59");
+ // and a trivial one:
+ addTrivialRangeAndSources(rangesWithSources, 1, 10, "127.0.0.3", "127.0.0.51");
+ RangeFetchMapCalculator calculator = new RangeFetchMapCalculator(rangesWithSources.build(), Collections.emptyList(), "Test");
+ Multimap<InetAddressAndPort, Range<Token>> optMap = calculator.getRangeFetchMapForNonTrivialRanges();
+ Multimap<InetAddressAndPort, Range<Token>> trivialMap = calculator.getRangeFetchMapForTrivialRanges(optMap);
+ assertTrue(trivialMap.get(InetAddressAndPort.getByName("127.0.0.3")).contains(generateTrivialRange(1,10)) ^
+ trivialMap.get(InetAddressAndPort.getByName("127.0.0.51")).contains(generateTrivialRange(1,10)));
+ assertFalse(optMap.containsKey(generateTrivialRange(1, 10)));
+ }
+
+ @Test(expected = IllegalStateException.class)
+ public void testNotEnoughEndpointsForTrivialRange() throws UnknownHostException
+ {
+ EndpointsByRange.Builder rangesWithSources = new EndpointsByRange.Builder();
+ // add non-trivial ranges
+ addNonTrivialRangeAndSources(rangesWithSources, 1, 10, "127.0.0.3", "127.0.0.51");
+ addNonTrivialRangeAndSources(rangesWithSources, 11, 20, "127.0.0.3", "127.0.0.55");
+ addNonTrivialRangeAndSources(rangesWithSources, 21, 30, "127.0.0.2", "127.0.0.59");
+ // and a trivial one:
+ addTrivialRangeAndSources(rangesWithSources, 1, 10, "127.0.0.3");
+
+ RangeStreamer.SourceFilter filter = new RangeStreamer.SourceFilter()
+ {
+ public boolean apply(Replica replica)
+ {
+ try
+ {
+ if (replica.endpoint().equals(InetAddressAndPort.getByName("127.0.0.3")))
+ return false;
+ }
+ catch (UnknownHostException e)
+ {
+ throw new RuntimeException(e);
+ }
+ return true;
+ }
+
+ public String message(Replica replica)
+ {
+ return "Not 127.0.0.3";
+ }
+ };
+ RangeFetchMapCalculator calculator = new RangeFetchMapCalculator(rangesWithSources.build(), Collections.singleton(filter), "Test");
+ Multimap<InetAddressAndPort, Range<Token>> optMap = calculator.getRangeFetchMapForNonTrivialRanges();
+ Multimap<InetAddressAndPort, Range<Token>> trivialMap = calculator.getRangeFetchMapForTrivialRanges(optMap);
+
+ }
+
+ private void assertArrays(Collection<Range<Token>> expected, Collection<Range<Token>> result)
+ {
+ Assert.assertEquals(expected.size(), result.size());
+ assertTrue(result.containsAll(expected));
+ }
+
+ private void validateRange(EndpointsByRange.Builder rangesWithSources, Multimap<InetAddressAndPort, Range<Token>> result)
+ {
+ for (Map.Entry<InetAddressAndPort, Range<Token>> entry : result.entries())
+ {
+ assertTrue(rangesWithSources.get(entry.getValue()).endpoints().contains(entry.getKey()));
+ }
+ }
+
+ private void addNonTrivialRangeAndSources(EndpointsByRange.Builder rangesWithSources, int left, int right, String... hosts) throws UnknownHostException
+ {
+ for (InetAddressAndPort endpoint : makeAddrs(hosts))
+ {
+ Range<Token> range = generateNonTrivialRange(left, right);
+ rangesWithSources.put(range, Replica.fullReplica(endpoint, range));
+ }
+ }
+
+ private void addTrivialRangeAndSources(EndpointsByRange.Builder rangesWithSources, int left, int right, String... hosts) throws UnknownHostException
+ {
+ for (InetAddressAndPort endpoint : makeAddrs(hosts))
+ {
+ Range<Token> range = generateTrivialRange(left, right);
+ rangesWithSources.put(range, Replica.fullReplica(endpoint, range));
+ }
+ }
+
+ private Collection<InetAddressAndPort> makeAddrs(String... hosts) throws UnknownHostException
+ {
+ ArrayList<InetAddressAndPort> addrs = new ArrayList<>(hosts.length);
+ for (String host : hosts)
+ addrs.add(InetAddressAndPort.getByName(host));
+ return addrs;
+ }
+
+ private Range<Token> generateNonTrivialRange(int left, int right)
+ {
+ // * 1000 to make sure we dont filter away any trivial ranges:
+ return new Range<>(new RandomPartitioner.BigIntegerToken(String.valueOf(left * 10000)), new RandomPartitioner.BigIntegerToken(String.valueOf(right * 10000)));
+ }
+
+ private Range<Token> generateTrivialRange(int left, int right)
+ {
+ Range<Token> r = new Range<>(new RandomPartitioner.BigIntegerToken(String.valueOf(left)), new RandomPartitioner.BigIntegerToken(String.valueOf(right)));
+ assertTrue(RangeFetchMapCalculator.isTrivial(r));
+ return r;
+ }
+}
diff --git a/test/unit/org/apache/cassandra/dht/RangeTest.java b/test/unit/org/apache/cassandra/dht/RangeTest.java
index 3657ced..29b120b 100644
--- a/test/unit/org/apache/cassandra/dht/RangeTest.java
+++ b/test/unit/org/apache/cassandra/dht/RangeTest.java
@@ -682,7 +682,7 @@
Range.OrderedRangeContainmentChecker checker = new Range.OrderedRangeContainmentChecker(ranges);
for (Token t : tokensToTest)
{
- if (checker.contains(t) != Range.isInRanges(t, ranges)) // avoid running Joiner.on(..) every iteration
+ if (checker.test(t) != Range.isInRanges(t, ranges)) // avoid running Joiner.on(..) every iteration
fail(String.format("This should never flap! If it does, it is a bug (ranges = %s, token = %s)", Joiner.on(",").join(ranges), t));
}
}
@@ -693,11 +693,11 @@
{
List<Range<Token>> ranges = asList(r(Long.MIN_VALUE, Long.MIN_VALUE + 1), r(Long.MAX_VALUE - 1, Long.MAX_VALUE));
Range.OrderedRangeContainmentChecker checker = new Range.OrderedRangeContainmentChecker(ranges);
- assertFalse(checker.contains(t(Long.MIN_VALUE)));
- assertTrue(checker.contains(t(Long.MIN_VALUE + 1)));
- assertFalse(checker.contains(t(0)));
- assertFalse(checker.contains(t(Long.MAX_VALUE - 1)));
- assertTrue(checker.contains(t(Long.MAX_VALUE)));
+ assertFalse(checker.test(t(Long.MIN_VALUE)));
+ assertTrue(checker.test(t(Long.MIN_VALUE + 1)));
+ assertFalse(checker.test(t(0)));
+ assertFalse(checker.test(t(Long.MAX_VALUE - 1)));
+ assertTrue(checker.test(t(Long.MAX_VALUE)));
}
private static Range<Token> r(long left, long right)
diff --git a/test/unit/org/apache/cassandra/dht/SplitterTest.java b/test/unit/org/apache/cassandra/dht/SplitterTest.java
index b39c09f..c591499 100644
--- a/test/unit/org/apache/cassandra/dht/SplitterTest.java
+++ b/test/unit/org/apache/cassandra/dht/SplitterTest.java
@@ -24,9 +24,14 @@
import java.util.List;
import java.util.Random;
import java.util.Set;
+import java.util.stream.Collectors;
import org.junit.Test;
+import org.apache.cassandra.utils.Pair;
+
+import static com.google.common.collect.Sets.newHashSet;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -50,42 +55,84 @@
{
randomSplitTestVNodes(new RandomPartitioner());
}
+
@Test
public void randomSplitTestVNodesMurmur3Partitioner()
{
randomSplitTestVNodes(new Murmur3Partitioner());
}
- public void randomSplitTestNoVNodes(IPartitioner partitioner)
+ @Test
+ public void testWithWeight()
{
+ List<Splitter.WeightedRange> ranges = new ArrayList<>();
+ ranges.add(new Splitter.WeightedRange(1.0, t(0, 10)));
+ ranges.add(new Splitter.WeightedRange(1.0, t(20, 30)));
+ ranges.add(new Splitter.WeightedRange(0.5, t(40, 60)));
+
+ List<Splitter.WeightedRange> ranges2 = new ArrayList<>();
+ ranges2.add(new Splitter.WeightedRange(1.0, t(0, 10)));
+ ranges2.add(new Splitter.WeightedRange(1.0, t(20, 30)));
+ ranges2.add(new Splitter.WeightedRange(1.0, t(40, 50)));
+ IPartitioner partitioner = Murmur3Partitioner.instance;
Splitter splitter = partitioner.splitter().get();
+
+ assertEquals(splitter.splitOwnedRanges(2, ranges, false), splitter.splitOwnedRanges(2, ranges2, false));
+ }
+
+ @Test
+ public void testWithWeight2()
+ {
+ List<Splitter.WeightedRange> ranges = new ArrayList<>();
+ ranges.add(new Splitter.WeightedRange(0.2, t(0, 10)));
+ ranges.add(new Splitter.WeightedRange(1.0, t(20, 30)));
+ ranges.add(new Splitter.WeightedRange(1.0, t(40, 50)));
+
+ List<Splitter.WeightedRange> ranges2 = new ArrayList<>();
+ ranges2.add(new Splitter.WeightedRange(1.0, t(0, 2)));
+ ranges2.add(new Splitter.WeightedRange(1.0, t(20, 30)));
+ ranges2.add(new Splitter.WeightedRange(1.0, t(40, 50)));
+ IPartitioner partitioner = Murmur3Partitioner.instance;
+ Splitter splitter = partitioner.splitter().get();
+
+ assertEquals(splitter.splitOwnedRanges(2, ranges, false), splitter.splitOwnedRanges(2, ranges2, false));
+ }
+
+ private Range<Token> t(long left, long right)
+ {
+ return new Range<>(new Murmur3Partitioner.LongToken(left), new Murmur3Partitioner.LongToken(right));
+ }
+
+ private static void randomSplitTestNoVNodes(IPartitioner partitioner)
+ {
+ Splitter splitter = getSplitter(partitioner);
Random r = new Random();
for (int i = 0; i < 10000; i++)
{
- List<Range<Token>> localRanges = generateLocalRanges(1, r.nextInt(4)+1, splitter, r, partitioner instanceof RandomPartitioner);
+ List<Splitter.WeightedRange> localRanges = generateLocalRanges(1, r.nextInt(4) + 1, splitter, r, partitioner instanceof RandomPartitioner);
List<Token> boundaries = splitter.splitOwnedRanges(r.nextInt(9) + 1, localRanges, false);
- assertTrue("boundaries = "+boundaries+" ranges = "+localRanges, assertRangeSizeEqual(localRanges, boundaries, partitioner, splitter, true));
+ assertTrue("boundaries = " + boundaries + " ranges = " + localRanges, assertRangeSizeEqual(localRanges, boundaries, partitioner, splitter, true));
}
}
- public void randomSplitTestVNodes(IPartitioner partitioner)
+ private static void randomSplitTestVNodes(IPartitioner partitioner)
{
- Splitter splitter = partitioner.splitter().get();
+ Splitter splitter = getSplitter(partitioner);
Random r = new Random();
for (int i = 0; i < 10000; i++)
{
// we need many tokens to be able to split evenly over the disks
int numTokens = 172 + r.nextInt(128);
int rf = r.nextInt(4) + 2;
- int parts = r.nextInt(5)+1;
- List<Range<Token>> localRanges = generateLocalRanges(numTokens, rf, splitter, r, partitioner instanceof RandomPartitioner);
+ int parts = r.nextInt(5) + 1;
+ List<Splitter.WeightedRange> localRanges = generateLocalRanges(numTokens, rf, splitter, r, partitioner instanceof RandomPartitioner);
List<Token> boundaries = splitter.splitOwnedRanges(parts, localRanges, true);
if (!assertRangeSizeEqual(localRanges, boundaries, partitioner, splitter, false))
fail(String.format("Could not split %d tokens with rf=%d into %d parts (localRanges=%s, boundaries=%s)", numTokens, rf, parts, localRanges, boundaries));
}
}
- private boolean assertRangeSizeEqual(List<Range<Token>> localRanges, List<Token> tokens, IPartitioner partitioner, Splitter splitter, boolean splitIndividualRanges)
+ private static boolean assertRangeSizeEqual(List<Splitter.WeightedRange> localRanges, List<Token> tokens, IPartitioner partitioner, Splitter splitter, boolean splitIndividualRanges)
{
Token start = partitioner.getMinimumToken();
List<BigInteger> splits = new ArrayList<>();
@@ -113,27 +160,27 @@
return allBalanced;
}
- private BigInteger sumOwnedBetween(List<Range<Token>> localRanges, Token start, Token end, Splitter splitter, boolean splitIndividualRanges)
+ private static BigInteger sumOwnedBetween(List<Splitter.WeightedRange> localRanges, Token start, Token end, Splitter splitter, boolean splitIndividualRanges)
{
BigInteger sum = BigInteger.ZERO;
- for (Range<Token> range : localRanges)
+ for (Splitter.WeightedRange range : localRanges)
{
if (splitIndividualRanges)
{
- Set<Range<Token>> intersections = new Range<>(start, end).intersectionWith(range);
+ Set<Range<Token>> intersections = new Range<>(start, end).intersectionWith(range.range());
for (Range<Token> intersection : intersections)
sum = sum.add(splitter.valueForToken(intersection.right).subtract(splitter.valueForToken(intersection.left)));
}
else
{
- if (new Range<>(start, end).contains(range.left))
- sum = sum.add(splitter.valueForToken(range.right).subtract(splitter.valueForToken(range.left)));
+ if (new Range<>(start, end).contains(range.left()))
+ sum = sum.add(splitter.valueForToken(range.right()).subtract(splitter.valueForToken(range.left())));
}
}
return sum;
}
- private List<Range<Token>> generateLocalRanges(int numTokens, int rf, Splitter splitter, Random r, boolean randomPartitioner)
+ private static List<Splitter.WeightedRange> generateLocalRanges(int numTokens, int rf, Splitter splitter, Random r, boolean randomPartitioner)
{
int localTokens = numTokens * rf;
List<Token> randomTokens = new ArrayList<>();
@@ -146,13 +193,330 @@
Collections.sort(randomTokens);
- List<Range<Token>> localRanges = new ArrayList<>(localTokens);
+ List<Splitter.WeightedRange> localRanges = new ArrayList<>(localTokens);
for (int i = 0; i < randomTokens.size() - 1; i++)
{
- assert randomTokens.get(i).compareTo(randomTokens.get(i+1)) < 0;
- localRanges.add(new Range<>(randomTokens.get(i), randomTokens.get(i+1)));
+ assert randomTokens.get(i).compareTo(randomTokens.get(i + 1)) < 0;
+ localRanges.add(new Splitter.WeightedRange(1.0, new Range<>(randomTokens.get(i), randomTokens.get(i + 1))));
i++;
}
return localRanges;
}
+
+ @Test
+ public void testSplitMurmur3Partitioner()
+ {
+ testSplit(new Murmur3Partitioner());
+ }
+
+ @Test
+ public void testSplitRandomPartitioner()
+ {
+ testSplit(new RandomPartitioner());
+ }
+
+ @SuppressWarnings("unchecked")
+ private static void testSplit(IPartitioner partitioner)
+ {
+ boolean isRandom = partitioner instanceof RandomPartitioner;
+ Splitter splitter = getSplitter(partitioner);
+ BigInteger min = splitter.valueForToken(partitioner.getMinimumToken());
+ BigInteger max = splitter.valueForToken(partitioner.getMaximumToken());
+ BigInteger first = isRandom ? RandomPartitioner.ZERO : min;
+ BigInteger last = isRandom ? max.subtract(BigInteger.valueOf(1)) : max;
+ BigInteger midpoint = last.add(first).divide(BigInteger.valueOf(2));
+
+ // regular single range
+ testSplit(partitioner, 1, newHashSet(Pair.create(1, 100)), newHashSet(Pair.create(1, 100)));
+ testSplit(partitioner, 2,
+ newHashSet(Pair.create(1, 100)),
+ newHashSet(Pair.create(1, 50), Pair.create(50, 100)));
+ testSplit(partitioner, 4,
+ newHashSet(Pair.create(1, 100)),
+ newHashSet(Pair.create(1, 25), Pair.create(25, 50), Pair.create(50, 75), Pair.create(75, 100)));
+ testSplit(partitioner, 5,
+ newHashSet(Pair.create(3, 79)),
+ newHashSet(Pair.create(3, 18), Pair.create(18, 33), Pair.create(33, 48), Pair.create(48, 63),
+ Pair.create(63, 79)));
+ testSplit(partitioner, 3,
+ newHashSet(Pair.create(3, 20)),
+ newHashSet(Pair.create(3, 8), Pair.create(8, 14), Pair.create(14, 20)));
+ testSplit(partitioner, 4,
+ newHashSet(Pair.create(3, 20)),
+ newHashSet(Pair.create(3, 7), Pair.create(7, 11), Pair.create(11, 15), Pair.create(15, 20)));
+
+ // single range too small to be partitioned
+ testSplit(partitioner, 1, newHashSet(Pair.create(1, 2)), newHashSet(Pair.create(1, 2)));
+ testSplit(partitioner, 2, newHashSet(Pair.create(1, 2)), newHashSet(Pair.create(1, 2)));
+ testSplit(partitioner, 4, newHashSet(Pair.create(1, 4)), newHashSet(Pair.create(1, 4)));
+ testSplit(partitioner, 8, newHashSet(Pair.create(1, 2)), newHashSet(Pair.create(1, 2)));
+
+ // single wrapping range
+ BigInteger cutpoint = isRandom ? midpoint.add(BigInteger.valueOf(7)) : min.add(BigInteger.valueOf(6));
+ testSplit(partitioner, 2,
+ newHashSet(Pair.create(8, 4)),
+ newHashSet(Pair.create(8, cutpoint), Pair.create(cutpoint, 4)));
+
+ // single range around partitioner min/max values
+ testSplit(partitioner, 2,
+ newHashSet(Pair.create(max.subtract(BigInteger.valueOf(8)), min)),
+ newHashSet(Pair.create(max.subtract(BigInteger.valueOf(8)), max.subtract(BigInteger.valueOf(4))),
+ Pair.create(max.subtract(BigInteger.valueOf(4)), isRandom ? first : max)));
+ testSplit(partitioner, 2,
+ newHashSet(Pair.create(max.subtract(BigInteger.valueOf(8)), max)),
+ newHashSet(Pair.create(max.subtract(BigInteger.valueOf(8)), max.subtract(BigInteger.valueOf(4))),
+ Pair.create(max.subtract(BigInteger.valueOf(4)), max)));
+ testSplit(partitioner, 2,
+ newHashSet(Pair.create(min, min.add(BigInteger.valueOf(8)))),
+ newHashSet(Pair.create(min, min.add(BigInteger.valueOf(4))),
+ Pair.create(min.add(BigInteger.valueOf(4)), min.add(BigInteger.valueOf(8)))));
+ testSplit(partitioner, 2,
+ newHashSet(Pair.create(max, min.add(BigInteger.valueOf(8)))),
+ newHashSet(Pair.create(max, min.add(BigInteger.valueOf(4))),
+ Pair.create(min.add(BigInteger.valueOf(4)), min.add(BigInteger.valueOf(8)))));
+ testSplit(partitioner, 2,
+ newHashSet(Pair.create(max.subtract(BigInteger.valueOf(4)), min.add(BigInteger.valueOf(4)))),
+ newHashSet(Pair.create(max.subtract(BigInteger.valueOf(4)), last),
+ Pair.create(last, min.add(BigInteger.valueOf(4)))));
+ testSplit(partitioner, 2,
+ newHashSet(Pair.create(max.subtract(BigInteger.valueOf(4)), min.add(BigInteger.valueOf(8)))),
+ newHashSet(Pair.create(max.subtract(BigInteger.valueOf(4)), min.add(BigInteger.valueOf(2))),
+ Pair.create(min.add(BigInteger.valueOf(2)), min.add(BigInteger.valueOf(8)))));
+
+ // multiple ranges
+ testSplit(partitioner, 1,
+ newHashSet(Pair.create(1, 100), Pair.create(200, 300)),
+ newHashSet(Pair.create(1, 100), Pair.create(200, 300)));
+ testSplit(partitioner, 2,
+ newHashSet(Pair.create(1, 100), Pair.create(200, 300)),
+ newHashSet(Pair.create(1, 100), Pair.create(200, 300)));
+ testSplit(partitioner, 4,
+ newHashSet(Pair.create(1, 100), Pair.create(200, 300)),
+ newHashSet(Pair.create(1, 50), Pair.create(50, 100), Pair.create(200, 250), Pair.create(250, 300)));
+ testSplit(partitioner, 4,
+ newHashSet(Pair.create(1, 100),
+ Pair.create(200, 300),
+ Pair.create(max.subtract(BigInteger.valueOf(4)), min.add(BigInteger.valueOf(4)))),
+ newHashSet(Pair.create(1, 50),
+ Pair.create(50, 100),
+ Pair.create(200, 250),
+ Pair.create(250, 300),
+ Pair.create(last, min.add(BigInteger.valueOf(4))),
+ Pair.create(max.subtract(BigInteger.valueOf(4)), last)));
+ }
+
+ private static void testSplit(IPartitioner partitioner, int parts, Set<Pair<Object, Object>> ranges, Set<Pair<Object, Object>> expected)
+ {
+ Splitter splitter = getSplitter(partitioner);
+ Set<Range<Token>> splittedRanges = splitter.split(ranges(partitioner, ranges), parts);
+ assertEquals(ranges(partitioner, expected), splittedRanges);
+ }
+
+ private static Set<Range<Token>> ranges(IPartitioner partitioner, Set<Pair<Object, Object>> pairs)
+ {
+ return pairs.stream().map(pair -> range(partitioner, pair)).collect(Collectors.toSet());
+ }
+
+ private static Range<Token> range(IPartitioner partitioner, Pair<?, ?> pair)
+ {
+ return new Range<>(token(partitioner, pair.left), token(partitioner, pair.right));
+ }
+
+ private static Token token(IPartitioner partitioner, Object n)
+ {
+ return partitioner.getTokenFactory().fromString(n.toString());
+ }
+
+ @Test
+ public void testTokensInRangeRandomPartitioner()
+ {
+ testTokensInRange(new RandomPartitioner());
+ }
+
+ @Test
+ public void testTokensInRangeMurmur3Partitioner()
+ {
+ testTokensInRange(new Murmur3Partitioner());
+ }
+
+ private static void testTokensInRange(IPartitioner partitioner)
+ {
+ Splitter splitter = getSplitter(partitioner);
+
+ // test full range
+ Range<Token> fullRange = new Range<>(partitioner.getMinimumToken(), partitioner.getMaximumToken());
+ BigInteger fullRangeSize = splitter.valueForToken(partitioner.getMaximumToken()).subtract(splitter.valueForToken(partitioner.getMinimumToken()));
+ assertEquals(fullRangeSize, splitter.tokensInRange(fullRange));
+ fullRange = new Range<>(splitter.tokenForValue(BigInteger.valueOf(-10)), splitter.tokenForValue(BigInteger.valueOf(-10)));
+ assertEquals(fullRangeSize, splitter.tokensInRange(fullRange));
+
+ // test small range
+ Range<Token> smallRange = new Range<>(splitter.tokenForValue(BigInteger.valueOf(-5)), splitter.tokenForValue(BigInteger.valueOf(5)));
+ assertEquals(BigInteger.valueOf(10), splitter.tokensInRange(smallRange));
+
+ // test wrap-around range
+ Range<Token> wrapAround = new Range<>(splitter.tokenForValue(BigInteger.valueOf(5)), splitter.tokenForValue(BigInteger.valueOf(-5)));
+ assertEquals(fullRangeSize.subtract(BigInteger.TEN), splitter.tokensInRange(wrapAround));
+ }
+
+ @Test
+ public void testElapsedTokensRandomPartitioner()
+ {
+ testElapsedMultiRange(new RandomPartitioner());
+ }
+
+ @Test
+ public void testElapsedTokensMurmur3Partitioner()
+ {
+ testElapsedMultiRange(new Murmur3Partitioner());
+ }
+
+ private static void testElapsedMultiRange(IPartitioner partitioner)
+ {
+ Splitter splitter = getSplitter(partitioner);
+ // small range
+ Range<Token> smallRange = new Range<>(splitter.tokenForValue(BigInteger.valueOf(-1)), splitter.tokenForValue(BigInteger.valueOf(1)));
+ testElapsedTokens(partitioner, smallRange, true);
+
+ // medium range
+ Range<Token> mediumRange = new Range<>(splitter.tokenForValue(BigInteger.valueOf(0)), splitter.tokenForValue(BigInteger.valueOf(123456789)));
+ testElapsedTokens(partitioner, mediumRange, true);
+
+ // wrapped range
+ BigInteger min = splitter.valueForToken(partitioner.getMinimumToken());
+ BigInteger max = splitter.valueForToken(partitioner.getMaximumToken());
+ Range<Token> wrappedRange = new Range<>(splitter.tokenForValue(max.subtract(BigInteger.valueOf(1350))),
+ splitter.tokenForValue(min.add(BigInteger.valueOf(20394))));
+ testElapsedTokens(partitioner, wrappedRange, true);
+
+ // full range
+ Range<Token> fullRange = new Range<>(partitioner.getMinimumToken(), partitioner.getMaximumToken());
+ testElapsedTokens(partitioner, fullRange, false);
+ }
+
+ private static void testElapsedTokens(IPartitioner partitioner, Range<Token> range, boolean partialRange)
+ {
+ Splitter splitter = getSplitter(partitioner);
+
+ BigInteger left = splitter.valueForToken(range.left);
+ BigInteger right = splitter.valueForToken(range.right);
+ BigInteger tokensInRange = splitter.tokensInRange(range);
+
+ // elapsedTokens(left, (left, right]) = 0
+ assertEquals(BigInteger.ZERO, splitter.elapsedTokens(splitter.tokenForValue(left), range));
+
+ // elapsedTokens(right, (left, right]) = tokensInRange((left, right])
+ assertEquals(tokensInRange, splitter.elapsedTokens(splitter.tokenForValue(right), range));
+
+ // elapsedTokens(left+1, (left, right]) = 1
+ assertEquals(BigInteger.ONE, splitter.elapsedTokens(splitter.tokenForValue(left.add(BigInteger.ONE)), range));
+
+ // elapsedTokens(right-1, (left, right]) = tokensInRange((left, right]) - 1
+ assertEquals(tokensInRange.subtract(BigInteger.ONE), splitter.elapsedTokens(splitter.tokenForValue(right.subtract(BigInteger.ONE)), range));
+
+ // elapsedTokens(midpoint, (left, right]) + tokensInRange((midpoint, right]) = tokensInRange
+ Token midpoint = partitioner.midpoint(range.left, range.right);
+ assertEquals(tokensInRange, splitter.elapsedTokens(midpoint, range).add(splitter.tokensInRange(new Range<>(midpoint, range.right))));
+
+ if (partialRange)
+ {
+ // elapsedTokens(right + 1, (left, right]) = 0
+ assertEquals(BigInteger.ZERO, splitter.elapsedTokens(splitter.tokenForValue(right.add(BigInteger.ONE)), range));
+ }
+ }
+
+ @Test
+ public void testPositionInRangeRandomPartitioner()
+ {
+ testPositionInRangeMultiRange(new RandomPartitioner());
+ }
+
+ @Test
+ public void testPositionInRangeMurmur3Partitioner()
+ {
+ testPositionInRangeMultiRange(new Murmur3Partitioner());
+ }
+
+ private static void testPositionInRangeMultiRange(IPartitioner partitioner)
+ {
+ Splitter splitter = getSplitter(partitioner);
+
+ // Test tiny range
+ Token start = splitter.tokenForValue(BigInteger.ZERO);
+ Token end = splitter.tokenForValue(BigInteger.valueOf(3));
+ Range<Token> range = new Range<>(start, end);
+ assertEquals(0.0, splitter.positionInRange(start, range), 0.01);
+ assertEquals(0.33, splitter.positionInRange(splitter.tokenForValue(BigInteger.valueOf(1)), range), 0.01);
+ assertEquals(0.66, splitter.positionInRange(splitter.tokenForValue(BigInteger.valueOf(2)), range), 0.01);
+ assertEquals(1.0, splitter.positionInRange(end, range), 0.01);
+ // Token not in range should return -1.0 for position
+ Token notInRange = splitter.tokenForValue(BigInteger.valueOf(10));
+ assertEquals(-1.0, splitter.positionInRange(notInRange, range), 0.0);
+
+
+ // Test medium range
+ start = splitter.tokenForValue(BigInteger.ZERO);
+ end = splitter.tokenForValue(BigInteger.valueOf(1000));
+ range = new Range<>(start, end);
+ testPositionInRange(partitioner, splitter, range);
+
+ // Test wrap-around range
+ start = splitter.tokenForValue(splitter.valueForToken(partitioner.getMaximumToken()).subtract(BigInteger.valueOf(123456789)));
+ end = splitter.tokenForValue(splitter.valueForToken(partitioner.getMinimumToken()).add(BigInteger.valueOf(123456789)));
+ range = new Range<>(start, end);
+ testPositionInRange(partitioner, splitter, range);
+
+ // Test full range
+ testPositionInRange(partitioner, splitter, new Range<>(partitioner.getMinimumToken(), partitioner.getMaximumToken()));
+ testPositionInRange(partitioner, splitter, new Range<>(partitioner.getMinimumToken(), partitioner.getMinimumToken()));
+ testPositionInRange(partitioner, splitter, new Range<>(partitioner.getMaximumToken(), partitioner.getMaximumToken()));
+ testPositionInRange(partitioner, splitter, new Range<>(splitter.tokenForValue(BigInteger.ONE), splitter.tokenForValue(BigInteger.ONE)));
+ }
+
+ private static void testPositionInRange(IPartitioner partitioner, Splitter splitter, Range<Token> range)
+ {
+ Range<Token> actualRange = range;
+ //full range case
+ if (range.left.equals(range.right))
+ {
+ actualRange = new Range<>(partitioner.getMinimumToken(), partitioner.getMaximumToken());
+ }
+ assertEquals(0.0, splitter.positionInRange(actualRange.left, range), 0.01);
+ assertEquals(0.25, splitter.positionInRange(getTokenInPosition(partitioner, actualRange, 0.25), range), 0.01);
+ assertEquals(0.37, splitter.positionInRange(getTokenInPosition(partitioner, actualRange, 0.373), range), 0.01);
+ assertEquals(0.5, splitter.positionInRange(getTokenInPosition(partitioner, actualRange, 0.5), range), 0.01);
+ assertEquals(0.75, splitter.positionInRange(getTokenInPosition(partitioner, actualRange, 0.75), range), 0.01);
+ assertEquals(0.99, splitter.positionInRange(getTokenInPosition(partitioner, actualRange, 0.999), range), 0.01);
+ assertEquals(1.0, splitter.positionInRange(actualRange.right, range), 0.01);
+ }
+
+ private static Token getTokenInPosition(IPartitioner partitioner, Range<Token> range, double position)
+ {
+ if (range.left.equals(range.right))
+ {
+ range = new Range<>(partitioner.getMinimumToken(), partitioner.getMaximumToken());
+ }
+ Splitter splitter = getSplitter(partitioner);
+ BigInteger totalTokens = splitter.tokensInRange(range);
+ BigInteger elapsedTokens = BigDecimal.valueOf(position).multiply(new BigDecimal(totalTokens)).toBigInteger();
+ BigInteger tokenInPosition = splitter.valueForToken(range.left).add(elapsedTokens);
+ return getWrappedToken(partitioner, tokenInPosition);
+ }
+
+ private static Token getWrappedToken(IPartitioner partitioner, BigInteger position)
+ {
+ Splitter splitter = getSplitter(partitioner);
+ BigInteger maxTokenValue = splitter.valueForToken(partitioner.getMaximumToken());
+ BigInteger minTokenValue = splitter.valueForToken(partitioner.getMinimumToken());
+ if (position.compareTo(maxTokenValue) > 0)
+ {
+ position = minTokenValue.add(position.subtract(maxTokenValue));
+ }
+ return splitter.tokenForValue(position);
+ }
+
+ private static Splitter getSplitter(IPartitioner partitioner)
+ {
+ return partitioner.splitter().orElseThrow(() -> new AssertionError(partitioner.getClass() + " must have a splitter"));
+ }
}
diff --git a/test/unit/org/apache/cassandra/dht/StreamStateStoreTest.java b/test/unit/org/apache/cassandra/dht/StreamStateStoreTest.java
index fd9b03e..b18d249 100644
--- a/test/unit/org/apache/cassandra/dht/StreamStateStoreTest.java
+++ b/test/unit/org/apache/cassandra/dht/StreamStateStoreTest.java
@@ -17,15 +17,19 @@
*/
package org.apache.cassandra.dht;
-import java.net.InetAddress;
import java.util.Collections;
+import org.apache.cassandra.db.commitlog.CommitLog;
+import org.apache.cassandra.locator.RangesAtEndpoint;
import org.junit.BeforeClass;
import org.junit.Test;
import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.streaming.DefaultConnectionFactory;
+import org.apache.cassandra.streaming.PreviewKind;
import org.apache.cassandra.streaming.StreamEvent;
+import org.apache.cassandra.streaming.StreamOperation;
import org.apache.cassandra.streaming.StreamSession;
import org.apache.cassandra.utils.FBUtilities;
@@ -39,6 +43,7 @@
public static void initDD()
{
DatabaseDescriptor.daemonInitialization();
+ CommitLog.instance.start();
}
@Test
@@ -49,9 +54,9 @@
Token.TokenFactory factory = p.getTokenFactory();
Range<Token> range = new Range<>(factory.fromString("0"), factory.fromString("100"));
- InetAddress local = FBUtilities.getBroadcastAddress();
- StreamSession session = new StreamSession(local, local, new DefaultConnectionFactory(), 0, true, false);
- session.addStreamRequest("keyspace1", Collections.singleton(range), Collections.singleton("cf"), 0);
+ InetAddressAndPort local = FBUtilities.getBroadcastAddressAndPort();
+ StreamSession session = new StreamSession(StreamOperation.BOOTSTRAP, local, new DefaultConnectionFactory(), false, 0, null, PreviewKind.NONE);
+ session.addStreamRequest("keyspace1", RangesAtEndpoint.toDummyList(Collections.singleton(range)), RangesAtEndpoint.toDummyList(Collections.emptyList()), Collections.singleton("cf"));
StreamStateStore store = new StreamStateStore();
// session complete event that is not completed makes data not available for keyspace/ranges
@@ -71,8 +76,8 @@
// add different range within the same keyspace
Range<Token> range2 = new Range<>(factory.fromString("100"), factory.fromString("200"));
- session = new StreamSession(local, local, new DefaultConnectionFactory(), 0, true, false);
- session.addStreamRequest("keyspace1", Collections.singleton(range2), Collections.singleton("cf"), 0);
+ session = new StreamSession(StreamOperation.BOOTSTRAP, local, new DefaultConnectionFactory(), false, 0, null, PreviewKind.NONE);
+ session.addStreamRequest("keyspace1", RangesAtEndpoint.toDummyList(Collections.singleton(range2)), RangesAtEndpoint.toDummyList(Collections.emptyList()), Collections.singleton("cf"));
session.state(StreamSession.State.COMPLETE);
store.handleStreamEvent(new StreamEvent.SessionCompleteEvent(session));
diff --git a/test/unit/org/apache/cassandra/diag/DiagnosticEventServiceTest.java b/test/unit/org/apache/cassandra/diag/DiagnosticEventServiceTest.java
new file mode 100644
index 0000000..a645c03
--- /dev/null
+++ b/test/unit/org/apache/cassandra/diag/DiagnosticEventServiceTest.java
@@ -0,0 +1,244 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.diag;
+
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.function.Consumer;
+
+import com.google.common.collect.ImmutableList;
+import org.junit.After;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+import org.apache.cassandra.OrderedJUnit4ClassRunner;
+import org.apache.cassandra.config.DatabaseDescriptor;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+@RunWith(OrderedJUnit4ClassRunner.class)
+public class DiagnosticEventServiceTest
+{
+
+ @BeforeClass
+ public static void setup()
+ {
+ DatabaseDescriptor.daemonInitialization();
+ }
+
+ @After
+ public void cleanup()
+ {
+ DiagnosticEventService.instance().cleanup();
+ }
+
+ @Test
+ public void testSubscribe()
+ {
+ DiagnosticEventService instance = DiagnosticEventService.instance();
+ assertFalse(instance.hasSubscribers(TestEvent1.class));
+ assertFalse(instance.hasSubscribers(TestEvent2.class));
+ Consumer<TestEvent1> consumer1 = (event) ->
+ {
+ };
+ Consumer<TestEvent1> consumer2 = (event) ->
+ {
+ };
+ Consumer<TestEvent1> consumer3 = (event) ->
+ {
+ };
+ instance.subscribe(TestEvent1.class, consumer1);
+ instance.subscribe(TestEvent1.class, consumer2);
+ instance.subscribe(TestEvent1.class, consumer3);
+ assertTrue(instance.hasSubscribers(TestEvent1.class));
+ assertFalse(instance.hasSubscribers(TestEvent2.class));
+ instance.unsubscribe(consumer1);
+ assertTrue(instance.hasSubscribers(TestEvent1.class));
+ assertFalse(instance.hasSubscribers(TestEvent2.class));
+ instance.unsubscribe(consumer2);
+ instance.unsubscribe(consumer3);
+ assertFalse(instance.hasSubscribers(TestEvent1.class));
+ assertFalse(instance.hasSubscribers(TestEvent2.class));
+ }
+
+ @Test
+ public void testSubscribeByType()
+ {
+ DiagnosticEventService instance = DiagnosticEventService.instance();
+ assertFalse(instance.hasSubscribers(TestEvent1.class));
+ assertFalse(instance.hasSubscribers(TestEvent2.class));
+ Consumer<TestEvent1> consumer1 = (event) ->
+ {
+ };
+ Consumer<TestEvent1> consumer2 = (event) ->
+ {
+ };
+ Consumer<TestEvent1> consumer3 = (event) ->
+ {
+ };
+
+ assertFalse(instance.hasSubscribers(TestEvent1.class, TestEventType.TEST1));
+ instance.subscribe(TestEvent1.class, TestEventType.TEST1, consumer1);
+ assertTrue(instance.hasSubscribers(TestEvent1.class, TestEventType.TEST1));
+ assertFalse(instance.hasSubscribers(TestEvent1.class, TestEventType.TEST2));
+
+ instance.subscribe(TestEvent1.class, TestEventType.TEST2, consumer2);
+ instance.subscribe(TestEvent1.class, TestEventType.TEST2, consumer2);
+ instance.subscribe(TestEvent1.class, TestEventType.TEST2, consumer2);
+ assertTrue(instance.hasSubscribers(TestEvent1.class, TestEventType.TEST2));
+
+ assertFalse(instance.hasSubscribers(TestEvent2.class));
+
+ instance.subscribe(TestEvent1.class, consumer3);
+ assertTrue(instance.hasSubscribers(TestEvent1.class));
+ assertTrue(instance.hasSubscribers(TestEvent1.class, TestEventType.TEST1));
+ assertTrue(instance.hasSubscribers(TestEvent1.class, TestEventType.TEST2));
+ assertTrue(instance.hasSubscribers(TestEvent1.class, TestEventType.TEST3));
+
+ instance.unsubscribe(consumer1);
+ assertTrue(instance.hasSubscribers(TestEvent1.class));
+ assertFalse(instance.hasSubscribers(TestEvent2.class));
+ instance.unsubscribe(consumer2);
+ instance.unsubscribe(consumer3);
+ assertFalse(instance.hasSubscribers(TestEvent1.class));
+ assertFalse(instance.hasSubscribers(TestEvent2.class));
+ }
+
+ @Test
+ public void testSubscribeAll()
+ {
+ DiagnosticEventService instance = DiagnosticEventService.instance();
+ assertFalse(instance.hasSubscribers(TestEvent1.class));
+ assertFalse(instance.hasSubscribers(TestEvent2.class));
+ Consumer<DiagnosticEvent> consumerAll1 = (event) ->
+ {
+ };
+ Consumer<DiagnosticEvent> consumerAll2 = (event) ->
+ {
+ };
+ Consumer<DiagnosticEvent> consumerAll3 = (event) ->
+ {
+ };
+ instance.subscribeAll(consumerAll1);
+ instance.subscribeAll(consumerAll2);
+ instance.subscribeAll(consumerAll3);
+ assertTrue(instance.hasSubscribers(TestEvent1.class));
+ assertTrue(instance.hasSubscribers(TestEvent2.class));
+ instance.unsubscribe(consumerAll1);
+ assertTrue(instance.hasSubscribers(TestEvent1.class));
+ assertTrue(instance.hasSubscribers(TestEvent2.class));
+ instance.unsubscribe(consumerAll2);
+ instance.unsubscribe(consumerAll3);
+ assertFalse(instance.hasSubscribers(TestEvent1.class));
+ assertFalse(instance.hasSubscribers(TestEvent2.class));
+ }
+
+ @Test
+ public void testCleanup()
+ {
+ DiagnosticEventService instance = DiagnosticEventService.instance();
+ assertFalse(instance.hasSubscribers(TestEvent1.class));
+ assertFalse(instance.hasSubscribers(TestEvent2.class));
+ Consumer<TestEvent1> consumer = (event) ->
+ {
+ };
+ instance.subscribe(TestEvent1.class, consumer);
+ Consumer<DiagnosticEvent> consumerAll = (event) ->
+ {
+ };
+ instance.subscribeAll(consumerAll);
+ assertTrue(instance.hasSubscribers(TestEvent1.class));
+ assertTrue(instance.hasSubscribers(TestEvent2.class));
+ instance.cleanup();
+ assertFalse(instance.hasSubscribers(TestEvent1.class));
+ assertFalse(instance.hasSubscribers(TestEvent2.class));
+ }
+
+ @Test
+ public void testPublish()
+ {
+ DiagnosticEventService instance = DiagnosticEventService.instance();
+ TestEvent1 a = new TestEvent1();
+ TestEvent1 b = new TestEvent1();
+ TestEvent1 c = new TestEvent1();
+ List<TestEvent1> events = ImmutableList.of(a, b, c, c, c);
+
+ List<DiagnosticEvent> consumed = new LinkedList<>();
+ Consumer<TestEvent1> consumer = consumed::add;
+ Consumer<DiagnosticEvent> consumerAll = consumed::add;
+
+ DatabaseDescriptor.setDiagnosticEventsEnabled(true);
+ instance.publish(c);
+ instance.subscribe(TestEvent1.class, consumer);
+ instance.publish(a);
+ instance.unsubscribe(consumer);
+ instance.publish(c);
+ instance.subscribeAll(consumerAll);
+ instance.publish(b);
+ instance.subscribe(TestEvent1.class, TestEventType.TEST3, consumer);
+ instance.publish(c);
+ instance.subscribe(TestEvent1.class, TestEventType.TEST1, consumer);
+ instance.publish(c);
+
+ assertEquals(events, consumed);
+ }
+
+ @Test
+ public void testEnabled()
+ {
+ DatabaseDescriptor.setDiagnosticEventsEnabled(false);
+ DiagnosticEventService.instance().subscribe(TestEvent1.class, (event) -> fail());
+ DiagnosticEventService.instance().publish(new TestEvent1());
+ DatabaseDescriptor.setDiagnosticEventsEnabled(true);
+ }
+
+ public static class TestEvent1 extends DiagnosticEvent
+ {
+ public TestEventType getType()
+ {
+ return TestEventType.TEST1;
+ }
+
+ public HashMap<String, Serializable> toMap()
+ {
+ return null;
+ }
+ }
+
+ public static class TestEvent2 extends DiagnosticEvent
+ {
+ public TestEventType getType()
+ {
+ return TestEventType.TEST2;
+ }
+
+ public HashMap<String, Serializable> toMap()
+ {
+ return null;
+ }
+ }
+
+ public enum TestEventType { TEST1, TEST2, TEST3 };
+}
diff --git a/test/unit/org/apache/cassandra/diag/store/DiagnosticEventMemoryStoreTest.java b/test/unit/org/apache/cassandra/diag/store/DiagnosticEventMemoryStoreTest.java
new file mode 100644
index 0000000..5e897b6
--- /dev/null
+++ b/test/unit/org/apache/cassandra/diag/store/DiagnosticEventMemoryStoreTest.java
@@ -0,0 +1,170 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.diag.store;
+
+import java.util.Map;
+import java.util.NavigableMap;
+
+import org.junit.Test;
+
+import org.apache.cassandra.diag.DiagnosticEvent;
+import org.apache.cassandra.diag.DiagnosticEventServiceTest.TestEvent1;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertSame;
+
+public class DiagnosticEventMemoryStoreTest
+{
+ @Test
+ public void testEmpty()
+ {
+ DiagnosticEventMemoryStore store = new DiagnosticEventMemoryStore();
+ assertEquals(0, store.size());
+ assertEquals(0, store.scan(0L, 10).size());
+ }
+
+ @Test
+ public void testSingle()
+ {
+ DiagnosticEventMemoryStore store = new DiagnosticEventMemoryStore();
+ store.store(new TestEvent1());
+ assertEquals(1, store.size());
+ assertEquals(1, store.scan(0L, 10).size());
+ }
+
+ @Test
+ public void testIdentity()
+ {
+ DiagnosticEventMemoryStore store = new DiagnosticEventMemoryStore();
+ TestEvent1 e1 = new TestEvent1();
+ TestEvent1 e2 = new TestEvent1();
+ TestEvent1 e3 = new TestEvent1();
+
+ store.store(e1);
+ store.store(e2);
+ store.store(e3);
+
+ assertEquals(3, store.size());
+
+ NavigableMap<Long, DiagnosticEvent> res = store.scan(0L, 10);
+ assertEquals(3, res.size());
+
+ Map.Entry<Long, DiagnosticEvent> entry = res.pollFirstEntry();
+ assertEquals(new Long(1), entry.getKey());
+ assertSame(e1, entry.getValue());
+
+ entry = res.pollFirstEntry();
+ assertEquals(new Long(2), entry.getKey());
+ assertSame(e2, entry.getValue());
+
+ entry = res.pollFirstEntry();
+ assertEquals(new Long(3), entry.getKey());
+ assertSame(e3, entry.getValue());
+ }
+
+ @Test
+ public void testLimit()
+ {
+ DiagnosticEventMemoryStore store = new DiagnosticEventMemoryStore();
+
+ TestEvent1 e1 = new TestEvent1();
+ TestEvent1 e2 = new TestEvent1();
+ TestEvent1 e3 = new TestEvent1();
+
+ store.store(e1);
+ store.store(e2);
+ store.store(e3);
+
+ NavigableMap<Long, DiagnosticEvent> res = store.scan(0L, 2);
+ assertEquals(2, res.size());
+
+ Map.Entry<Long, DiagnosticEvent> entry = res.pollFirstEntry();
+ assertEquals(new Long(1), entry.getKey());
+ assertSame(e1, entry.getValue());
+
+ entry = res.pollLastEntry();
+ assertEquals(new Long(2), entry.getKey());
+ assertSame(e2, entry.getValue());
+ }
+
+ @Test
+ public void testSeek()
+ {
+ DiagnosticEventMemoryStore store = new DiagnosticEventMemoryStore();
+
+ TestEvent1 e2 = new TestEvent1();
+ TestEvent1 e3 = new TestEvent1();
+
+ store.store(new TestEvent1());
+ store.store(e2);
+ store.store(e3);
+ store.store(new TestEvent1());
+ store.store(new TestEvent1());
+ store.store(new TestEvent1());
+
+ NavigableMap<Long, DiagnosticEvent> res = store.scan(2L, 2);
+ assertEquals(2, res.size());
+
+ Map.Entry<Long, DiagnosticEvent> entry = res.pollFirstEntry();
+ assertEquals(new Long(2), entry.getKey());
+ assertSame(e2, entry.getValue());
+
+ entry = res.pollLastEntry();
+ assertEquals(new Long(3), entry.getKey());
+ assertSame(e3, entry.getValue());
+ }
+
+ @Test
+ public void testMaxElements()
+ {
+ DiagnosticEventMemoryStore store = new DiagnosticEventMemoryStore();
+ store.setMaxSize(3);
+
+ store.store(new TestEvent1());
+ store.store(new TestEvent1());
+ store.store(new TestEvent1());
+ TestEvent1 e2 = new TestEvent1(); // 4
+ TestEvent1 e3 = new TestEvent1();
+ store.store(e2);
+ store.store(e3);
+ store.store(new TestEvent1()); // 6
+
+ assertEquals(3, store.size());
+
+ NavigableMap<Long, DiagnosticEvent> res = store.scan(4L, 2);
+ assertEquals(2, res.size());
+
+ Map.Entry<Long, DiagnosticEvent> entry = res.pollFirstEntry();
+ assertEquals(new Long(4), entry.getKey());
+ assertSame(e2, entry.getValue());
+
+ entry = res.pollFirstEntry();
+ assertEquals(new Long(5), entry.getKey());
+ assertSame(e3, entry.getValue());
+
+ store.store(new TestEvent1()); // 7
+ store.store(new TestEvent1());
+ store.store(new TestEvent1());
+
+ res = store.scan(4L, 2);
+ assertEquals(2, res.size());
+ assertEquals(new Long(7), res.pollFirstEntry().getKey());
+ assertEquals(new Long(8), res.pollLastEntry().getKey());
+ }
+}
diff --git a/test/unit/org/apache/cassandra/fql/FullQueryLoggerTest.java b/test/unit/org/apache/cassandra/fql/FullQueryLoggerTest.java
new file mode 100644
index 0000000..04baa09
--- /dev/null
+++ b/test/unit/org/apache/cassandra/fql/FullQueryLoggerTest.java
@@ -0,0 +1,735 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.fql;
+
+import java.io.File;
+import java.nio.ByteBuffer;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import javax.annotation.Nullable;
+
+import org.apache.commons.lang3.StringUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import io.netty.buffer.Unpooled;
+import net.openhft.chronicle.queue.ChronicleQueue;
+import net.openhft.chronicle.queue.ChronicleQueueBuilder;
+import net.openhft.chronicle.queue.ExcerptTailer;
+import net.openhft.chronicle.queue.RollCycles;
+import net.openhft.chronicle.wire.ValueIn;
+import net.openhft.chronicle.wire.WireOut;
+import org.apache.cassandra.Util;
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.cql3.QueryOptions;
+import org.apache.cassandra.cql3.statements.BatchStatement;
+import org.apache.cassandra.fql.FullQueryLogger.Query;
+import org.apache.cassandra.fql.FullQueryLogger.Batch;
+import org.apache.cassandra.cql3.statements.BatchStatement.Type;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.service.QueryState;
+import org.apache.cassandra.transport.ProtocolVersion;
+import org.apache.cassandra.utils.ObjectSizes;
+import org.apache.cassandra.utils.binlog.BinLogTest;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import static org.apache.cassandra.fql.FullQueryLogger.BATCH;
+import static org.apache.cassandra.fql.FullQueryLogger.BATCH_TYPE;
+import static org.apache.cassandra.fql.FullQueryLogger.GENERATED_NOW_IN_SECONDS;
+import static org.apache.cassandra.fql.FullQueryLogger.GENERATED_TIMESTAMP;
+import static org.apache.cassandra.fql.FullQueryLogger.PROTOCOL_VERSION;
+import static org.apache.cassandra.fql.FullQueryLogger.QUERIES;
+import static org.apache.cassandra.fql.FullQueryLogger.QUERY;
+import static org.apache.cassandra.fql.FullQueryLogger.QUERY_OPTIONS;
+import static org.apache.cassandra.fql.FullQueryLogger.QUERY_START_TIME;
+import static org.apache.cassandra.fql.FullQueryLogger.SINGLE_QUERY;
+import static org.apache.cassandra.fql.FullQueryLogger.TYPE;
+import static org.apache.cassandra.fql.FullQueryLogger.VALUES;
+import static org.apache.cassandra.fql.FullQueryLogger.VERSION;
+
+public class FullQueryLoggerTest extends CQLTester
+{
+ private static Path tempDir;
+
+ @BeforeClass
+ public static void beforeClass() throws Exception
+ {
+ tempDir = BinLogTest.tempDir();
+ }
+
+ @After
+ public void tearDown()
+ {
+ FullQueryLogger.instance.stop();
+ FullQueryLogger.instance.reset(tempDir.toString());
+ }
+
+ @Test(expected = NullPointerException.class)
+ public void testConfigureNullPath() throws Exception
+ {
+ FullQueryLogger.instance.enable(null, "", true, 1, 1, StringUtils.EMPTY, 10);
+ }
+
+ @Test(expected = NullPointerException.class)
+ public void testConfigureNullRollCycle() throws Exception
+ {
+ FullQueryLogger.instance.enable(BinLogTest.tempDir(), null, true, 1, 1, StringUtils.EMPTY, 10);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testConfigureInvalidRollCycle() throws Exception
+ {
+ FullQueryLogger.instance.enable(BinLogTest.tempDir(), "foobar", true, 1, 1, StringUtils.EMPTY, 10);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testConfigureInvalidMaxQueueWeight() throws Exception
+ {
+ FullQueryLogger.instance.enable(BinLogTest.tempDir(), "DAILY", true, 0, 1, StringUtils.EMPTY, 10);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testConfigureInvalidMaxQueueLogSize() throws Exception
+ {
+ FullQueryLogger.instance.enable(BinLogTest.tempDir(), "DAILY", true, 1, 0, StringUtils.EMPTY, 10);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testConfigureOverExistingFile()
+ {
+ File f = FileUtils.createTempFile("foo", "bar");
+ f.deleteOnExit();
+ FullQueryLogger.instance.enable(f.toPath(), "TEST_SECONDLY", true, 1, 1, StringUtils.EMPTY, 10);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testCanRead() throws Exception
+ {
+ tempDir.toFile().setReadable(false);
+ try
+ {
+ configureFQL();
+ }
+ finally
+ {
+ tempDir.toFile().setReadable(true);
+ }
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testCanWrite() throws Exception
+ {
+ tempDir.toFile().setWritable(false);
+ try
+ {
+ configureFQL();
+ }
+ finally
+ {
+ tempDir.toFile().setWritable(true);
+ }
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testCanExecute() throws Exception
+ {
+ tempDir.toFile().setExecutable(false);
+ try
+ {
+ configureFQL();
+ }
+ finally
+ {
+ tempDir.toFile().setExecutable(true);
+ }
+ }
+
+ @Test
+ public void testResetWithoutConfigure() throws Exception
+ {
+ FullQueryLogger.instance.reset(tempDir.toString());
+ FullQueryLogger.instance.reset(tempDir.toString());
+ }
+
+ @Test
+ public void stopWithoutConfigure() throws Exception
+ {
+ FullQueryLogger.instance.stop();
+ FullQueryLogger.instance.stop();
+ }
+
+ /**
+ * Both the last used and supplied directory should get cleaned
+ */
+ @Test
+ public void testResetCleansPaths() throws Exception
+ {
+ configureFQL();
+ File tempA = File.createTempFile("foo", "bar", tempDir.toFile());
+ assertTrue(tempA.exists());
+ File tempB = File.createTempFile("foo", "bar", BinLogTest.tempDir().toFile());
+ FullQueryLogger.instance.reset(tempB.getParent());
+ assertFalse(tempA.exists());
+ assertFalse(tempB.exists());
+ }
+
+ /**
+ * The last used and configured directory are the same and it shouldn't be an issue
+ */
+ @Test
+ public void testResetSamePath() throws Exception
+ {
+ configureFQL();
+ File tempA = File.createTempFile("foo", "bar", tempDir.toFile());
+ assertTrue(tempA.exists());
+ FullQueryLogger.instance.reset(tempA.getParent());
+ assertFalse(tempA.exists());
+ }
+
+ @Test(expected = IllegalStateException.class)
+ public void testDoubleConfigure() throws Exception
+ {
+ configureFQL();
+ configureFQL();
+ }
+
+ @Test
+ public void testCleansDirectory() throws Exception
+ {
+ assertTrue(new File(tempDir.toFile(), "foobar").createNewFile());
+ configureFQL();
+ assertEquals(tempDir.toFile().listFiles().length, 1);
+ assertEquals("directory-listing.cq4t", tempDir.toFile().listFiles()[0].getName());
+ }
+
+ @Test
+ public void testEnabledReset() throws Exception
+ {
+ assertFalse(FullQueryLogger.instance.isEnabled());
+ configureFQL();
+ assertTrue(FullQueryLogger.instance.isEnabled());
+ FullQueryLogger.instance.reset(tempDir.toString());
+ assertFalse(FullQueryLogger.instance.isEnabled());
+ }
+
+ @Test
+ public void testEnabledStop() throws Exception
+ {
+ assertFalse(FullQueryLogger.instance.isEnabled());
+ configureFQL();
+ assertTrue(FullQueryLogger.instance.isEnabled());
+ FullQueryLogger.instance.stop();
+ assertFalse(FullQueryLogger.instance.isEnabled());
+ }
+
+ /**
+ * Test that a thread will block if the FQL is over weight, and unblock once the backup is cleared
+ */
+ @Test
+ public void testBlocking() throws Exception
+ {
+ configureFQL();
+ //Prevent the bin log thread from making progress, causing the task queue to block
+ Semaphore blockBinLog = new Semaphore(0);
+ try
+ {
+ //Find out when the bin log thread has been blocked, necessary to not run into batch task drain behavior
+ Semaphore binLogBlocked = new Semaphore(0);
+ FullQueryLogger.instance.binLog.put(new Query("foo1", QueryOptions.DEFAULT, queryState(), 1)
+ {
+
+ public void writeMarshallablePayload(WireOut wire)
+ {
+ //Notify that the bin log is blocking now
+ binLogBlocked.release();
+ try
+ {
+ //Block the bin log thread so the task queue can be filled
+ blockBinLog.acquire();
+ }
+ catch (InterruptedException e)
+ {
+ e.printStackTrace();
+ }
+ super.writeMarshallablePayload(wire);
+ }
+
+ public void release()
+ {
+ super.release();
+ }
+ });
+
+ //Wait for the bin log thread to block so it can't batch drain tasks
+ Util.spinAssertEquals(true, binLogBlocked::tryAcquire, 60);
+
+ //Now fill the task queue
+ logQuery("foo2");
+
+ //Start a thread to block waiting on the bin log queue
+ Thread t = new Thread(() ->
+ {
+ logQuery("foo3");
+ //Should be able to log another query without an issue
+ logQuery("foo4");
+ });
+ t.start();
+ Thread.sleep(500);
+ //If thread state is terminated then the thread started, finished, and didn't block on the full task queue
+ assertTrue(t.getState() != Thread.State.TERMINATED);
+ }
+ finally
+ {
+ //Unblock the binlog thread
+ blockBinLog.release();
+ }
+ Util.spinAssertEquals(true, () -> checkForQueries(Arrays.asList("foo1", "foo2", "foo3", "foo4")), 60);
+ }
+
+ private boolean checkForQueries(List<String> queries)
+ {
+ try (ChronicleQueue queue = ChronicleQueueBuilder.single(tempDir.toFile()).rollCycle(RollCycles.TEST_SECONDLY).build())
+ {
+ ExcerptTailer tailer = queue.createTailer();
+ List<String> expectedQueries = new LinkedList<>(queries);
+ while (!expectedQueries.isEmpty())
+ {
+ if (!tailer.readDocument(wire -> {
+ assertEquals(expectedQueries.get(0), wire.read("query").text());
+ expectedQueries.remove(0);
+ }))
+ {
+ return false;
+ }
+ }
+ assertFalse(tailer.readDocument(wire -> {}));
+ return true;
+ }
+ }
+
+ @Test
+ public void testNonBlocking() throws Exception
+ {
+ FullQueryLogger.instance.enable(tempDir, "TEST_SECONDLY", false, 1, 1024 * 1024 * 256, StringUtils.EMPTY, 10);
+ //Prevent the bin log thread from making progress, causing the task queue to refuse tasks
+ Semaphore blockBinLog = new Semaphore(0);
+ try
+ {
+ //Find out when the bin log thread has been blocked, necessary to not run into batch task drain behavior
+ Semaphore binLogBlocked = new Semaphore(0);
+ FullQueryLogger.instance.binLog.put(new Query("foo1", QueryOptions.DEFAULT, queryState(), 1)
+ {
+
+ public void writeMarshallablePayload(WireOut wire)
+ {
+ //Notify that the bin log is blocking now
+ binLogBlocked.release();
+ try
+ {
+ //Block the bin log thread so the task queue can be filled
+ blockBinLog.acquire();
+ }
+ catch (InterruptedException e)
+ {
+ e.printStackTrace();
+ }
+ super.writeMarshallablePayload(wire);
+ }
+
+ public void release()
+ {
+ super.release();
+ }
+ });
+
+ //Wait for the bin log thread to block so it can't batch drain tasks
+ Util.spinAssertEquals(true, binLogBlocked::tryAcquire, 60);
+
+ //Now fill the task queue
+ logQuery("foo2");
+
+ //This sample should get dropped AKA released without being written
+ AtomicInteger releasedCount = new AtomicInteger(0);
+ AtomicInteger writtenCount = new AtomicInteger(0);
+ FullQueryLogger.instance.binLog.logRecord(new Query("foo3", QueryOptions.DEFAULT, queryState(), 1) {
+ public void writeMarshallablePayload(WireOut wire)
+ {
+ writtenCount.incrementAndGet();
+ super.writeMarshallablePayload(wire);
+ }
+
+ public void release()
+ {
+ releasedCount.incrementAndGet();
+ super.release();
+ }
+ });
+
+ Util.spinAssertEquals(1, releasedCount::get, 60);
+ assertEquals(0, writtenCount.get());
+ }
+ finally
+ {
+ blockBinLog.release();
+ }
+ //Wait for tasks to drain so there should be space in the queue
+ Util.spinAssertEquals(true, () -> checkForQueries(Arrays.asList("foo1", "foo2")), 60);
+ //Should be able to log again
+ logQuery("foo4");
+ Util.spinAssertEquals(true, () -> checkForQueries(Arrays.asList("foo1", "foo2", "foo4")), 60);
+ }
+
+ @Test
+ public void testRoundTripQuery() throws Exception
+ {
+ configureFQL();
+ logQuery("foo");
+ Util.spinAssertEquals(true, () -> checkForQueries(Arrays.asList("foo")), 60);
+ assertRoundTripQuery(null);
+ }
+
+ @Test
+ public void testRoundTripQueryWithKeyspace() throws Exception
+ {
+ configureFQL();
+ logQuery("foo", "abcdefg");
+ Util.spinAssertEquals(true, () -> checkForQueries(Arrays.asList("foo")), 60);
+ assertRoundTripQuery("abcdefg");
+ }
+
+ private void assertRoundTripQuery(@Nullable String keyspace)
+ {
+ try (ChronicleQueue queue = ChronicleQueueBuilder.single(tempDir.toFile()).rollCycle(RollCycles.TEST_SECONDLY).build())
+ {
+ ExcerptTailer tailer = queue.createTailer();
+ assertTrue(tailer.readDocument(wire ->
+ {
+ assertEquals(0, wire.read(VERSION).int16());
+ assertEquals(SINGLE_QUERY, wire.read(TYPE).text());
+
+ assertEquals(1L, wire.read(QUERY_START_TIME).int64());
+
+ ProtocolVersion protocolVersion = ProtocolVersion.decode(wire.read(PROTOCOL_VERSION).int32(), true);
+ assertEquals(ProtocolVersion.CURRENT, protocolVersion);
+
+ QueryOptions queryOptions = QueryOptions.codec.decode(Unpooled.wrappedBuffer(wire.read(QUERY_OPTIONS).bytes()), protocolVersion);
+ compareQueryOptions(QueryOptions.DEFAULT, queryOptions);
+
+ String wireKeyspace = wire.read(FullQueryLogger.KEYSPACE).text();
+ assertEquals(keyspace, wireKeyspace);
+
+ assertEquals("foo", wire.read(QUERY).text());
+ }));
+ }
+ }
+
+ @Test
+ public void testRoundTripBatchWithKeyspace() throws Exception
+ {
+ configureFQL();
+ logBatch(Type.UNLOGGED,
+ Arrays.asList("foo1", "foo2"),
+ Arrays.asList(Arrays.asList(ByteBuffer.allocate(1),
+ ByteBuffer.allocateDirect(2)),
+ Collections.emptyList()),
+ QueryOptions.DEFAULT,
+ queryState("abcdefgh"),
+ 1);
+
+ Util.spinAssertEquals(true, () ->
+ {
+ try (ChronicleQueue queue = ChronicleQueueBuilder.single(tempDir.toFile()).rollCycle(RollCycles.TEST_SECONDLY).build())
+ {
+ return queue.createTailer().readingDocument().isPresent();
+ }
+ }, 60);
+
+ assertRoundTripBatch("abcdefgh");
+ }
+
+ @Test
+ public void testRoundTripBatchWithKeyspaceNull() throws Exception
+ {
+ configureFQL();
+ logBatch(Type.UNLOGGED,
+ Arrays.asList("foo1", "foo2"),
+ Arrays.asList(Arrays.asList(ByteBuffer.allocate(1),
+ ByteBuffer.allocateDirect(2)),
+ Collections.emptyList()),
+ QueryOptions.DEFAULT,
+ queryState(),
+ 1);
+
+ Util.spinAssertEquals(true, () ->
+ {
+ try (ChronicleQueue queue = ChronicleQueueBuilder.single(tempDir.toFile()).rollCycle(RollCycles.TEST_SECONDLY).build())
+ {
+ return queue.createTailer().readingDocument().isPresent();
+ }
+ }, 60);
+
+ assertRoundTripBatch(null);
+ }
+
+
+ private void assertRoundTripBatch(@Nullable String keyspace)
+ {
+ try (ChronicleQueue queue = ChronicleQueueBuilder.single(tempDir.toFile()).rollCycle(RollCycles.TEST_SECONDLY).build())
+ {
+ ExcerptTailer tailer = queue.createTailer();
+ assertTrue(tailer.readDocument(wire -> {
+ assertEquals(0, wire.read(VERSION).int16());
+ assertEquals(BATCH, wire.read(TYPE).text());
+
+ assertEquals(1L, wire.read(QUERY_START_TIME).int64());
+
+ ProtocolVersion protocolVersion = ProtocolVersion.decode(wire.read(PROTOCOL_VERSION).int32(), true);
+ assertEquals(ProtocolVersion.CURRENT, protocolVersion);
+
+ QueryOptions queryOptions = QueryOptions.codec.decode(Unpooled.wrappedBuffer(wire.read(QUERY_OPTIONS).bytes()), protocolVersion);
+ compareQueryOptions(QueryOptions.DEFAULT, queryOptions);
+
+ assertEquals(Long.MIN_VALUE, wire.read(GENERATED_TIMESTAMP).int64());
+ assertEquals(Integer.MIN_VALUE, wire.read(GENERATED_NOW_IN_SECONDS).int32());
+ assertEquals(keyspace, wire.read(FullQueryLogger.KEYSPACE).text());
+ assertEquals("UNLOGGED", wire.read(BATCH_TYPE).text());
+ ValueIn in = wire.read(QUERIES);
+ assertEquals(2, in.int32());
+ assertEquals("foo1", in.text());
+ assertEquals("foo2", in.text());
+ in = wire.read(VALUES);
+ assertEquals(2, in.int32());
+ assertEquals(2, in.int32());
+ assertTrue(Arrays.equals(new byte[1], in.bytes()));
+ assertTrue(Arrays.equals(new byte[2], in.bytes()));
+ assertEquals(0, in.int32());
+ }));
+ }
+ }
+
+
+ @Test
+ public void testQueryWeight()
+ {
+ //Empty query should have some weight
+ Query query = new Query("", QueryOptions.DEFAULT, queryState(), 1);
+ assertTrue(query.weight() >= 95);
+
+ StringBuilder sb = new StringBuilder();
+ for (int ii = 0; ii < 1024 * 1024; ii++)
+ {
+ sb.append('a');
+ }
+ query = new Query(sb.toString(), QueryOptions.DEFAULT, queryState(), 1);
+
+ //A large query should be reflected in the size, * 2 since characters are still two bytes
+ assertTrue(query.weight() > ObjectSizes.measureDeep(sb.toString()));
+
+ //Large query options should be reflected
+ QueryOptions largeOptions = QueryOptions.forInternalCalls(Arrays.asList(ByteBuffer.allocate(1024 * 1024)));
+ query = new Query("", largeOptions, queryState(), 1);
+ assertTrue(query.weight() > 1024 * 1024);
+ System.out.printf("weight %d%n", query.weight());
+ }
+
+ @Test
+ public void testBatchWeight()
+ {
+ //An empty batch should have weight
+ Batch batch = new Batch(Type.UNLOGGED, new ArrayList<>(), new ArrayList<>(), QueryOptions.DEFAULT, queryState(), 1);
+ assertTrue(batch.weight() > 0);
+
+ // make sure that a batch with keyspace set has a higher weight
+ Batch batch2 = new Batch(Type.UNLOGGED, new ArrayList<>(), new ArrayList<>(), QueryOptions.DEFAULT, queryState("ABABA"), 1);
+ assertTrue(batch.weight() < batch2.weight());
+
+ StringBuilder sb = new StringBuilder();
+ for (int ii = 0; ii < 1024 * 1024; ii++)
+ {
+ sb.append('a');
+ }
+
+ //The weight of the list containing queries should be reflected
+ List<String> bigList = new ArrayList(100000);
+ for (int ii = 0; ii < 100000; ii++)
+ {
+ bigList.add("");
+ }
+ batch = new Batch(Type.UNLOGGED, bigList, new ArrayList<>(), QueryOptions.DEFAULT, queryState(), 1);
+ assertTrue(batch.weight() > ObjectSizes.measureDeep(bigList));
+
+ //The size of the query should be reflected
+ bigList = new ArrayList(1);
+ bigList.add(sb.toString());
+ batch = new Batch(Type.UNLOGGED, bigList, new ArrayList<>(), QueryOptions.DEFAULT, queryState(), 1);
+ assertTrue(batch.weight() > ObjectSizes.measureDeep(bigList));
+
+ bigList = null;
+ //The size of the list of values should be reflected
+ List<List<ByteBuffer>> bigValues = new ArrayList<>(100000);
+ for (int ii = 0; ii < 100000; ii++)
+ {
+ bigValues.add(new ArrayList<>(0));
+ }
+ bigValues.get(0).add(ByteBuffer.allocate(1024 * 1024 * 5));
+ batch = new Batch(Type.UNLOGGED, new ArrayList<>(), bigValues, QueryOptions.DEFAULT, queryState(), 1);
+ assertTrue(batch.weight() > ObjectSizes.measureDeep(bigValues));
+
+ //As should the size of the values
+ QueryOptions largeOptions = QueryOptions.forInternalCalls(Arrays.asList(ByteBuffer.allocate(1024 * 1024)));
+ batch = new Batch(Type.UNLOGGED, new ArrayList<>(), new ArrayList<>(), largeOptions, queryState(), 1);
+ assertTrue(batch.weight() > 1024 * 1024);
+ }
+
+ @Test(expected = NullPointerException.class)
+ public void testLogBatchNullType() throws Exception
+ {
+ logBatch(null, new ArrayList<>(), new ArrayList<>(), QueryOptions.DEFAULT, queryState(), 1);
+ }
+
+ @Test(expected = NullPointerException.class)
+ public void testLogBatchNullQueries() throws Exception
+ {
+ logBatch(Type.UNLOGGED, null, new ArrayList<>(), QueryOptions.DEFAULT, queryState(), 1);
+ }
+
+ @Test(expected = NullPointerException.class)
+ public void testLogBatchNullQueriesQuery() throws Exception
+ {
+ configureFQL();
+ logBatch(Type.UNLOGGED, Arrays.asList((String)null), new ArrayList<>(), QueryOptions.DEFAULT, queryState(), 1);
+ }
+
+ @Test(expected = NullPointerException.class)
+ public void testLogBatchNullValues() throws Exception
+ {
+ logBatch(Type.UNLOGGED, new ArrayList<>(), null, QueryOptions.DEFAULT, queryState(), 1);
+ }
+
+ @Test(expected = NullPointerException.class)
+ public void testLogBatchNullValuesValue() throws Exception
+ {
+ logBatch(Type.UNLOGGED, new ArrayList<>(), Arrays.asList((List<ByteBuffer>)null), null, queryState(), 1);
+ }
+
+ @Test(expected = NullPointerException.class)
+ public void testLogBatchNullQueryOptions() throws Exception
+ {
+ logBatch(Type.UNLOGGED, new ArrayList<>(), new ArrayList<>(), null, queryState(), 1);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testLogBatchNegativeTime() throws Exception
+ {
+ logBatch(Type.UNLOGGED, new ArrayList<>(), new ArrayList<>(), QueryOptions.DEFAULT, queryState(), -1);
+ }
+
+ @Test(expected = NullPointerException.class)
+ public void testLogQueryNullQuery() throws Exception
+ {
+ logQuery(null, QueryOptions.DEFAULT, queryState(), 1);
+ }
+
+ @Test(expected = NullPointerException.class)
+ public void testLogQueryNullQueryOptions() throws Exception
+ {
+ logQuery("", null, queryState(), 1);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testLogQueryNegativeTime() throws Exception
+ {
+ logQuery("", QueryOptions.DEFAULT, queryState(), -1);
+ }
+
+ private static void compareQueryOptions(QueryOptions a, QueryOptions b)
+ {
+ assertEquals(a.getClass(), b.getClass());
+ assertEquals(a.getProtocolVersion(), b.getProtocolVersion());
+ assertEquals(a.getPageSize(), b.getPageSize());
+ assertEquals(a.getConsistency(), b.getConsistency());
+ assertEquals(a.getPagingState(), b.getPagingState());
+ assertEquals(a.getValues(), b.getValues());
+ assertEquals(a.getSerialConsistency(), b.getSerialConsistency());
+ }
+
+ private void configureFQL() throws Exception
+ {
+ FullQueryLogger.instance.enable(tempDir, "TEST_SECONDLY", true, 1, 1024 * 1024 * 256, StringUtils.EMPTY, 10);
+ }
+
+ private void logQuery(String query)
+ {
+ FullQueryLogger.instance.querySuccess(null, query, QueryOptions.DEFAULT, queryState(), 1, null);
+ }
+
+ private void logQuery(String query, String keyspace)
+ {
+ logQuery(query, keyspace, 1);
+ }
+ private void logQuery(String query, String keyspace, long time)
+ {
+ FullQueryLogger.instance.querySuccess(null, query, QueryOptions.DEFAULT, queryState(keyspace), time, null);
+ }
+ private void logQuery(String query, QueryOptions options, QueryState state, long time)
+ {
+ FullQueryLogger.instance.querySuccess(null, query, options, state, time, null);
+ }
+
+ private void logBatch(BatchStatement.Type type,
+ List<String> queries,
+ List<List<ByteBuffer>> values,
+ QueryOptions options,
+ QueryState queryState,
+ long time)
+ {
+ FullQueryLogger.instance.batchSuccess(type,
+ Collections.emptyList(),
+ queries,
+ values,
+ options,
+ queryState,
+ time,
+ null);
+ }
+
+ private QueryState queryState(String keyspace)
+ {
+ ClientState clientState = ClientState.forInternalCalls(keyspace);
+ return new QueryState(clientState);
+ }
+
+ private QueryState queryState()
+ {
+ return new QueryState(ClientState.forInternalCalls());
+ }
+}
+
diff --git a/test/unit/org/apache/cassandra/gms/ArrivalWindowTest.java b/test/unit/org/apache/cassandra/gms/ArrivalWindowTest.java
index a539ca8..ea59300 100644
--- a/test/unit/org/apache/cassandra/gms/ArrivalWindowTest.java
+++ b/test/unit/org/apache/cassandra/gms/ArrivalWindowTest.java
@@ -25,26 +25,17 @@
import org.junit.Test;
-import java.net.InetAddress;
-
-import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.utils.FBUtilities;
-import org.junit.BeforeClass;
public class ArrivalWindowTest
{
- @BeforeClass
- public static void beforeClass()
- {
- DatabaseDescriptor.daemonInitialization();
- }
-
@Test
public void testWithNanoTime()
{
final ArrivalWindow windowWithNano = new ArrivalWindow(4);
final long toNano = 1000000L;
- InetAddress ep = FBUtilities.getLocalAddress();
+ InetAddressAndPort ep = FBUtilities.getLocalAddressAndPort();
windowWithNano.add(111 * toNano, ep);
windowWithNano.add(222 * toNano, ep);
windowWithNano.add(333 * toNano, ep);
diff --git a/test/unit/org/apache/cassandra/gms/EndpointStateTest.java b/test/unit/org/apache/cassandra/gms/EndpointStateTest.java
index 2453fe8..6e0cc75 100644
--- a/test/unit/org/apache/cassandra/gms/EndpointStateTest.java
+++ b/test/unit/org/apache/cassandra/gms/EndpointStateTest.java
@@ -72,7 +72,7 @@
public void run()
{
state.addApplicationState(ApplicationState.TOKENS, valueFactory.tokens(tokens));
- state.addApplicationState(ApplicationState.STATUS, valueFactory.normal(tokens));
+ state.addApplicationState(ApplicationState.STATUS_WITH_PORT, valueFactory.normal(tokens));
}
});
@@ -86,7 +86,7 @@
for (Map.Entry<ApplicationState, VersionedValue> entry : state.states())
values.put(entry.getKey(), entry.getValue());
- if (values.containsKey(ApplicationState.STATUS) && !values.containsKey(ApplicationState.TOKENS))
+ if (values.containsKey(ApplicationState.STATUS_WITH_PORT) && !values.containsKey(ApplicationState.TOKENS))
{
numFailures.incrementAndGet();
System.out.println(String.format("Failed: %s", values));
@@ -129,7 +129,7 @@
{
Map<ApplicationState, VersionedValue> states = new EnumMap<>(ApplicationState.class);
states.put(ApplicationState.TOKENS, valueFactory.tokens(tokens));
- states.put(ApplicationState.STATUS, valueFactory.normal(tokens));
+ states.put(ApplicationState.STATUS_WITH_PORT, valueFactory.normal(tokens));
state.addApplicationStates(states);
}
});
@@ -158,7 +158,7 @@
for (Map.Entry<ApplicationState, VersionedValue> entry : states)
values.put(entry.getKey(), entry.getValue());
- assertTrue(values.containsKey(ApplicationState.STATUS));
+ assertTrue(values.containsKey(ApplicationState.STATUS_WITH_PORT));
assertTrue(values.containsKey(ApplicationState.TOKENS));
assertTrue(values.containsKey(ApplicationState.INTERNAL_IP));
assertTrue(values.containsKey(ApplicationState.HOST_ID));
diff --git a/test/unit/org/apache/cassandra/gms/ExpireEndpointTest.java b/test/unit/org/apache/cassandra/gms/ExpireEndpointTest.java
index f6d9fa3..298549e 100644
--- a/test/unit/org/apache/cassandra/gms/ExpireEndpointTest.java
+++ b/test/unit/org/apache/cassandra/gms/ExpireEndpointTest.java
@@ -18,7 +18,6 @@
package org.apache.cassandra.gms;
-import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.UUID;
@@ -26,6 +25,7 @@
import org.junit.Test;
import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.service.StorageService;
import static org.junit.Assert.assertFalse;
@@ -43,7 +43,7 @@
@Test
public void testExpireEndpoint() throws UnknownHostException
{
- InetAddress hostAddress = InetAddress.getByName("127.0.0.2");
+ InetAddressAndPort hostAddress = InetAddressAndPort.getByName("127.0.0.2");
UUID hostId = UUID.randomUUID();
long expireTime = System.currentTimeMillis() - 1;
@@ -51,7 +51,7 @@
EndpointState endpointState = Gossiper.instance.getEndpointStateForEndpoint(hostAddress);
Gossiper.runInGossipStageBlocking(() -> Gossiper.instance.markDead(hostAddress, endpointState));
- endpointState.addApplicationState(ApplicationState.STATUS, StorageService.instance.valueFactory.removedNonlocal(hostId, expireTime));
+ endpointState.addApplicationState(ApplicationState.STATUS_WITH_PORT, StorageService.instance.valueFactory.removedNonlocal(hostId, expireTime));
Gossiper.instance.addExpireTimeForEndpoint(hostAddress, expireTime);
assertTrue("Expiring endpoint not unreachable before status check", Gossiper.instance.getUnreachableMembers().contains(hostAddress));
diff --git a/test/unit/org/apache/cassandra/gms/FailureDetectorTest.java b/test/unit/org/apache/cassandra/gms/FailureDetectorTest.java
index 0dff95a..77fabef 100644
--- a/test/unit/org/apache/cassandra/gms/FailureDetectorTest.java
+++ b/test/unit/org/apache/cassandra/gms/FailureDetectorTest.java
@@ -18,7 +18,6 @@
package org.apache.cassandra.gms;
-import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.Collections;
@@ -30,9 +29,11 @@
import org.apache.cassandra.Util;
import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.commitlog.CommitLog;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.RandomPartitioner;
import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.locator.TokenMetadata;
import org.apache.cassandra.service.StorageService;
@@ -45,8 +46,9 @@
{
// slow unit tests can cause problems with FailureDetector's GC pause handling
System.setProperty("cassandra.max_local_pause_in_ms", "20000");
+
DatabaseDescriptor.daemonInitialization();
- DatabaseDescriptor.createAllDirectories();
+ CommitLog.instance.start();
}
@Test
@@ -60,7 +62,7 @@
ArrayList<Token> endpointTokens = new ArrayList<>();
ArrayList<Token> keyTokens = new ArrayList<>();
- List<InetAddress> hosts = new ArrayList<>();
+ List<InetAddressAndPort> hosts = new ArrayList<>();
List<UUID> hostIds = new ArrayList<>();
// we want to convict if there is any heartbeat data present in the FD
@@ -69,12 +71,12 @@
// create a ring of 2 nodes
Util.createInitialRing(ss, partitioner, endpointTokens, keyTokens, hosts, hostIds, 3);
- InetAddress leftHost = hosts.get(1);
+ InetAddressAndPort leftHost = hosts.get(1);
FailureDetector.instance.report(leftHost);
// trigger handleStateLeft in StorageService
- ss.onChange(leftHost, ApplicationState.STATUS,
+ ss.onChange(leftHost, ApplicationState.STATUS_WITH_PORT,
valueFactory.left(Collections.singleton(endpointTokens.get(1)), Gossiper.computeExpireTime()));
// confirm that handleStateLeft was called and leftEndpoint was removed from TokenMetadata
diff --git a/test/unit/org/apache/cassandra/gms/GossipDigestTest.java b/test/unit/org/apache/cassandra/gms/GossipDigestTest.java
index 36e3b27..cb67a54 100644
--- a/test/unit/org/apache/cassandra/gms/GossipDigestTest.java
+++ b/test/unit/org/apache/cassandra/gms/GossipDigestTest.java
@@ -26,26 +26,16 @@
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputBuffer;
-import java.net.InetAddress;
-
-import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.net.MessagingService;
-
-import org.junit.BeforeClass;
import org.junit.Test;
public class GossipDigestTest
{
- @BeforeClass
- public static void beforeClass()
- {
- DatabaseDescriptor.daemonInitialization();
- }
-
@Test
public void test() throws IOException
{
- InetAddress endpoint = InetAddress.getByName("127.0.0.1");
+ InetAddressAndPort endpoint = InetAddressAndPort.getByName("127.0.0.1");
int generation = 0;
int maxVersion = 123;
GossipDigest expected = new GossipDigest(endpoint, generation, maxVersion);
diff --git a/test/unit/org/apache/cassandra/gms/GossiperTest.java b/test/unit/org/apache/cassandra/gms/GossiperTest.java
index b6b3ffb..fd760cb 100644
--- a/test/unit/org/apache/cassandra/gms/GossiperTest.java
+++ b/test/unit/org/apache/cassandra/gms/GossiperTest.java
@@ -26,20 +26,21 @@
import java.util.UUID;
import com.google.common.collect.ImmutableMap;
+import com.google.common.net.InetAddresses;
+import org.junit.After;
+import org.junit.Assert;
import org.junit.Before;
-import org.junit.BeforeClass;
import org.junit.Test;
-import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.Util;
import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.config.Schema;
+import org.apache.cassandra.db.commitlog.CommitLog;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.RandomPartitioner;
import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.SeedProvider;
import org.apache.cassandra.locator.TokenMetadata;
-import org.apache.cassandra.net.MessagingService;
-import org.apache.cassandra.schema.KeyspaceParams;
import org.apache.cassandra.service.StorageService;
import static org.junit.Assert.assertEquals;
@@ -48,15 +49,11 @@
public class GossiperTest
{
- @BeforeClass
- public static void before()
+ static
{
System.setProperty(Gossiper.Props.DISABLE_THREAD_VALIDATION, "true");
DatabaseDescriptor.daemonInitialization();
- SchemaLoader.prepareServer();
- SchemaLoader.createKeyspace("schema_test_ks",
- KeyspaceParams.simple(1),
- SchemaLoader.standardCFMD("schema_test_ks", "schema_test_cf"));
+ CommitLog.instance.start();
}
static final IPartitioner partitioner = new RandomPartitioner();
@@ -64,22 +61,68 @@
TokenMetadata tmd = StorageService.instance.getTokenMetadata();
ArrayList<Token> endpointTokens = new ArrayList<>();
ArrayList<Token> keyTokens = new ArrayList<>();
- List<InetAddress> hosts = new ArrayList<>();
+ List<InetAddressAndPort> hosts = new ArrayList<>();
List<UUID> hostIds = new ArrayList<>();
+ private SeedProvider originalSeedProvider;
+
@Before
public void setup()
{
tmd.clearUnsafe();
+ originalSeedProvider = DatabaseDescriptor.getSeedProvider();
+ }
+
+ @After
+ public void tearDown()
+ {
+ DatabaseDescriptor.setSeedProvider(originalSeedProvider);
}
@Test
- public void testLargeGenerationJump() throws UnknownHostException
+ public void testHaveVersion3Nodes() throws Exception
+ {
+ VersionedValue.VersionedValueFactory factory = new VersionedValue.VersionedValueFactory(null);
+ EndpointState es = new EndpointState(null);
+ es.addApplicationState(ApplicationState.RELEASE_VERSION, factory.releaseVersion("4.0-SNAPSHOT"));
+ Gossiper.instance.endpointStateMap.put(InetAddressAndPort.getByName("127.0.0.1"), es);
+ Gossiper.instance.liveEndpoints.add(InetAddressAndPort.getByName("127.0.0.1"));
+
+
+ es = new EndpointState(null);
+ es.addApplicationState(ApplicationState.RELEASE_VERSION, factory.releaseVersion("3.11.3"));
+ Gossiper.instance.endpointStateMap.put(InetAddressAndPort.getByName("127.0.0.2"), es);
+ Gossiper.instance.liveEndpoints.add(InetAddressAndPort.getByName("127.0.0.2"));
+
+
+ es = new EndpointState(null);
+ es.addApplicationState(ApplicationState.RELEASE_VERSION, factory.releaseVersion("3.0.0"));
+ Gossiper.instance.endpointStateMap.put(InetAddressAndPort.getByName("127.0.0.3"), es);
+ Gossiper.instance.liveEndpoints.add(InetAddressAndPort.getByName("127.0.0.3"));
+
+
+ assertTrue(Gossiper.instance.haveMajorVersion3NodesSupplier.get());
+
+ Gossiper.instance.endpointStateMap.remove(InetAddressAndPort.getByName("127.0.0.2"));
+ Gossiper.instance.liveEndpoints.remove(InetAddressAndPort.getByName("127.0.0.2"));
+
+
+ assertTrue(Gossiper.instance.haveMajorVersion3NodesSupplier.get());
+
+ Gossiper.instance.endpointStateMap.remove(InetAddressAndPort.getByName("127.0.0.3"));
+ Gossiper.instance.liveEndpoints.add(InetAddressAndPort.getByName("127.0.0.3"));
+
+ assertFalse(Gossiper.instance.haveMajorVersion3NodesSupplier.get());
+
+ }
+
+ @Test
+ public void testLargeGenerationJump() throws UnknownHostException, InterruptedException
{
Util.createInitialRing(ss, partitioner, endpointTokens, keyTokens, hosts, hostIds, 2);
try
{
- InetAddress remoteHostAddress = hosts.get(1);
+ InetAddressAndPort remoteHostAddress = hosts.get(1);
EndpointState initialRemoteState = Gossiper.instance.getEndpointStateForEndpoint(remoteHostAddress);
HeartBeatState initialRemoteHeartBeat = initialRemoteState.getHeartBeatState();
@@ -125,7 +168,7 @@
Util.createInitialRing(ss, partitioner, endpointTokens, keyTokens, hosts, hostIds, 2);
try
{
- InetAddress remoteHostAddress = hosts.get(1);
+ InetAddressAndPort remoteHostAddress = hosts.get(1);
EndpointState initialRemoteState = Gossiper.instance.getEndpointStateForEndpoint(remoteHostAddress);
HeartBeatState initialRemoteHeartBeat = initialRemoteState.getHeartBeatState();
@@ -143,23 +186,23 @@
Gossiper.instance.register(
new IEndpointStateChangeSubscriber()
{
- public void onJoin(InetAddress endpoint, EndpointState epState) { }
+ public void onJoin(InetAddressAndPort endpoint, EndpointState epState) { }
- public void beforeChange(InetAddress endpoint, EndpointState currentState, ApplicationState newStateKey, VersionedValue newValue) { }
+ public void beforeChange(InetAddressAndPort endpoint, EndpointState currentState, ApplicationState newStateKey, VersionedValue newValue) { }
- public void onChange(InetAddress endpoint, ApplicationState state, VersionedValue value)
+ public void onChange(InetAddressAndPort endpoint, ApplicationState state, VersionedValue value)
{
assertEquals(ApplicationState.TOKENS, state);
stateChangedNum++;
}
- public void onAlive(InetAddress endpoint, EndpointState state) { }
+ public void onAlive(InetAddressAndPort endpoint, EndpointState state) { }
- public void onDead(InetAddress endpoint, EndpointState state) { }
+ public void onDead(InetAddressAndPort endpoint, EndpointState state) { }
- public void onRemove(InetAddress endpoint) { }
+ public void onRemove(InetAddressAndPort endpoint) { }
- public void onRestart(InetAddress endpoint, EndpointState state) { }
+ public void onRestart(InetAddressAndPort endpoint, EndpointState state) { }
}
);
@@ -192,53 +235,126 @@
}
}
+ // Note: This test might fail if for some reason the node broadcast address is in 127.99.0.0/16
@Test
- public void testSchemaVersionUpdate() throws UnknownHostException, InterruptedException
+ public void testReloadSeeds() throws UnknownHostException
{
- Util.createInitialRing(ss, partitioner, endpointTokens, keyTokens, hosts, hostIds, 2);
- MessagingService.instance().listen();
- Gossiper.instance.start(1);
- InetAddress remoteHostAddress = hosts.get(1);
+ Gossiper gossiper = new Gossiper(false);
+ List<String> loadedList;
- EndpointState initialRemoteState = Gossiper.instance.getEndpointStateForEndpoint(remoteHostAddress);
- // Set to any 3.0 version
- Gossiper.instance.injectApplicationState(remoteHostAddress, ApplicationState.RELEASE_VERSION, StorageService.instance.valueFactory.releaseVersion("3.0.14"));
-
- Gossiper.instance.applyStateLocally(ImmutableMap.of(remoteHostAddress, initialRemoteState));
-
- // wait until the schema is set
- VersionedValue schema = null;
- for (int i = 0; i < 10; i++)
+ // Initialize the seed list directly to a known set to start with
+ gossiper.seeds.clear();
+ InetAddressAndPort addr = InetAddressAndPort.getByAddress(InetAddress.getByName("127.99.1.1"));
+ int nextSize = 4;
+ List<InetAddressAndPort> nextSeeds = new ArrayList<>(nextSize);
+ for (int i = 0; i < nextSize; i ++)
{
- EndpointState localState = Gossiper.instance.getEndpointStateForEndpoint(hosts.get(0));
- schema = localState.getApplicationState(ApplicationState.SCHEMA);
- if (schema != null)
- break;
- Thread.sleep(1000);
+ gossiper.seeds.add(addr);
+ nextSeeds.add(addr);
+ addr = InetAddressAndPort.getByAddress(InetAddresses.increment(addr.address));
+ }
+ Assert.assertEquals(nextSize, gossiper.seeds.size());
+
+ // Add another unique address to the list
+ addr = InetAddressAndPort.getByAddress(InetAddresses.increment(addr.address));
+ nextSeeds.add(addr);
+ nextSize++;
+ DatabaseDescriptor.setSeedProvider(new TestSeedProvider(nextSeeds));
+ loadedList = gossiper.reloadSeeds();
+
+ // Check that the new entry was added
+ Assert.assertEquals(nextSize, loadedList.size());
+ for (InetAddressAndPort a : nextSeeds)
+ assertTrue(loadedList.contains(a.toString()));
+
+ // Check that the return value of the reloadSeeds matches the content of the getSeeds call
+ // and that they both match the internal contents of the Gossiper seeds list
+ Assert.assertEquals(loadedList.size(), gossiper.getSeeds().size());
+ for (InetAddressAndPort a : gossiper.seeds)
+ {
+ assertTrue(loadedList.contains(a.toString()));
+ assertTrue(gossiper.getSeeds().contains(a.toString()));
}
- // schema is set and equals to "alternative" version
- assertTrue(schema != null);
- assertEquals(schema.value, Schema.instance.getAltVersion().toString());
+ // Add a duplicate of the last address to the seed provider list
+ int uniqueSize = nextSize;
+ nextSeeds.add(addr);
+ nextSize++;
+ DatabaseDescriptor.setSeedProvider(new TestSeedProvider(nextSeeds));
+ loadedList = gossiper.reloadSeeds();
- // Upgrade remote host version to the latest one (3.11)
- Gossiper.instance.injectApplicationState(remoteHostAddress, ApplicationState.RELEASE_VERSION, StorageService.instance.valueFactory.releaseVersion());
+ // Check that the number of seed nodes reported hasn't increased
+ Assert.assertEquals(uniqueSize, loadedList.size());
+ for (InetAddressAndPort a : nextSeeds)
+ assertTrue(loadedList.contains(a.toString()));
- Gossiper.instance.applyStateLocally(ImmutableMap.of(remoteHostAddress, initialRemoteState));
-
- // wait until the schema change
- VersionedValue newSchema = null;
- for (int i = 0; i < 10; i++)
+ // Create a new list that has no overlaps with the previous list
+ addr = InetAddressAndPort.getByAddress(InetAddress.getByName("127.99.2.1"));
+ int disjointSize = 3;
+ List<InetAddressAndPort> disjointSeeds = new ArrayList<>(disjointSize);
+ for (int i = 0; i < disjointSize; i ++)
{
- EndpointState localState = Gossiper.instance.getEndpointStateForEndpoint(hosts.get(0));
- newSchema = localState.getApplicationState(ApplicationState.SCHEMA);
- if (!schema.value.equals(newSchema.value))
- break;
- Thread.sleep(1000);
+ disjointSeeds.add(addr);
+ addr = InetAddressAndPort.getByAddress(InetAddresses.increment(addr.address));
+ }
+ DatabaseDescriptor.setSeedProvider(new TestSeedProvider(disjointSeeds));
+ loadedList = gossiper.reloadSeeds();
+
+ // Check that the list now contains exactly the new other list.
+ Assert.assertEquals(disjointSize, gossiper.getSeeds().size());
+ Assert.assertEquals(disjointSize, loadedList.size());
+ for (InetAddressAndPort a : disjointSeeds)
+ {
+ assertTrue(gossiper.getSeeds().contains(a.toString()));
+ assertTrue(loadedList.contains(a.toString()));
}
- // schema is changed and equals to real version
- assertFalse(schema.value.equals(newSchema.value));
- assertEquals(newSchema.value, Schema.instance.getRealVersion().toString());
+ // Set the seed node provider to return an empty list
+ DatabaseDescriptor.setSeedProvider(new TestSeedProvider(new ArrayList<InetAddressAndPort>()));
+ loadedList = gossiper.reloadSeeds();
+
+ // Check that the in memory seed node list was not modified
+ Assert.assertEquals(disjointSize, loadedList.size());
+ for (InetAddressAndPort a : disjointSeeds)
+ assertTrue(loadedList.contains(a.toString()));
+
+ // Change the seed provider to one that throws an unchecked exception
+ DatabaseDescriptor.setSeedProvider(new ErrorSeedProvider());
+ loadedList = gossiper.reloadSeeds();
+
+ // Check for the expected null response from a reload error
+ Assert.assertNull(loadedList);
+
+ // Check that the in memory seed node list was not modified and the exception was caught
+ Assert.assertEquals(disjointSize, gossiper.getSeeds().size());
+ for (InetAddressAndPort a : disjointSeeds)
+ assertTrue(gossiper.getSeeds().contains(a.toString()));
+ }
+
+ static class TestSeedProvider implements SeedProvider
+ {
+ private List<InetAddressAndPort> seeds;
+
+ TestSeedProvider(List<InetAddressAndPort> seeds)
+ {
+ this.seeds = seeds;
+ }
+
+ @Override
+ public List<InetAddressAndPort> getSeeds()
+ {
+ return seeds;
+ }
+ }
+
+ // A seed provider for testing which throws assertion errors when queried
+ static class ErrorSeedProvider implements SeedProvider
+ {
+ @Override
+ public List<InetAddressAndPort> getSeeds()
+ {
+ assert(false);
+ return new ArrayList<>();
+ }
}
}
diff --git a/test/unit/org/apache/cassandra/gms/PendingRangeCalculatorServiceTest.java b/test/unit/org/apache/cassandra/gms/PendingRangeCalculatorServiceTest.java
index 7892de4..0d6d199 100644
--- a/test/unit/org/apache/cassandra/gms/PendingRangeCalculatorServiceTest.java
+++ b/test/unit/org/apache/cassandra/gms/PendingRangeCalculatorServiceTest.java
@@ -18,7 +18,6 @@
package org.apache.cassandra.gms;
-import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.Collection;
@@ -32,10 +31,10 @@
import org.junit.runner.RunWith;
import org.apache.cassandra.SchemaLoader;
-import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.dht.ByteOrderedPartitioner;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.service.StorageService;
import org.jboss.byteman.contrib.bmunit.BMRule;
import org.jboss.byteman.contrib.bmunit.BMUnitRunner;
@@ -64,11 +63,11 @@
@BMRule(name = "Block pending range calculation",
targetClass = "TokenMetadata",
targetMethod = "calculatePendingRanges",
- targetLocation = "AT INVOKE org.apache.cassandra.locator.AbstractReplicationStrategy.getAddressRanges",
+ targetLocation = "AT INVOKE org.apache.cassandra.locator.AbstractReplicationStrategy.getAddressReplicas",
action = "org.apache.cassandra.gms.PendingRangeCalculatorServiceTest.calculationLock.lock()")
public void testDelayedResponse() throws UnknownHostException, InterruptedException
{
- InetAddress otherNodeAddr = InetAddress.getByName("127.0.0.2");
+ InetAddressAndPort otherNodeAddr = InetAddressAndPort.getByName("127.0.0.2");
UUID otherHostId = UUID.randomUUID();
// introduce node for first major state change
@@ -114,7 +113,7 @@
}
}
- private Map<InetAddress, EndpointState> getStates(InetAddress otherNodeAddr, UUID hostId, int ver, boolean bootstrapping)
+ private Map<InetAddressAndPort, EndpointState> getStates(InetAddressAndPort otherNodeAddr, UUID hostId, int ver, boolean bootstrapping)
{
HeartBeatState hb = new HeartBeatState(1, ver);
EndpointState state = new EndpointState(hb);
@@ -127,7 +126,7 @@
state.addApplicationState(ApplicationState.HOST_ID, StorageService.instance.valueFactory.hostId(hostId));
state.addApplicationState(ApplicationState.NET_VERSION, StorageService.instance.valueFactory.networkVersion());
- Map<InetAddress, EndpointState> states = new HashMap<>();
+ Map<InetAddressAndPort, EndpointState> states = new HashMap<>();
states.put(otherNodeAddr, state);
return states;
}
diff --git a/test/unit/org/apache/cassandra/gms/SerializationsTest.java b/test/unit/org/apache/cassandra/gms/SerializationsTest.java
index 0df266f..90ce10b 100644
--- a/test/unit/org/apache/cassandra/gms/SerializationsTest.java
+++ b/test/unit/org/apache/cassandra/gms/SerializationsTest.java
@@ -24,6 +24,7 @@
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.io.util.DataInputPlus.DataInputStreamPlus;
import org.apache.cassandra.io.util.DataOutputStreamPlus;
+import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.FBUtilities;
@@ -31,7 +32,6 @@
import org.junit.Test;
import java.io.IOException;
-import java.net.InetAddress;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
@@ -78,9 +78,9 @@
private void testGossipDigestWrite() throws IOException
{
- Map<InetAddress, EndpointState> states = new HashMap<InetAddress, EndpointState>();
- states.put(InetAddress.getByName("127.0.0.1"), Statics.EndpointSt);
- states.put(InetAddress.getByName("127.0.0.2"), Statics.EndpointSt);
+ Map<InetAddressAndPort, EndpointState> states = new HashMap<>();
+ states.put(InetAddressAndPort.getByName("127.0.0.1"), Statics.EndpointSt);
+ states.put(InetAddressAndPort.getByName("127.0.0.2"), Statics.EndpointSt);
GossipDigestAck ack = new GossipDigestAck(Statics.Digests, states);
GossipDigestAck2 ack2 = new GossipDigestAck2(states);
GossipDigestSyn syn = new GossipDigestSyn("Not a real cluster name",
@@ -132,9 +132,9 @@
{
HeartbeatSt.updateHeartBeat();
EndpointSt.addApplicationState(ApplicationState.LOAD, vv0);
- EndpointSt.addApplicationState(ApplicationState.STATUS, vv1);
+ EndpointSt.addApplicationState(ApplicationState.STATUS_WITH_PORT, vv1);
for (int i = 0; i < 100; i++)
- Digests.add(new GossipDigest(FBUtilities.getBroadcastAddress(), 100 + i, 1000 + 2 * i));
+ Digests.add(new GossipDigest(FBUtilities.getBroadcastAddressAndPort(), 100 + i, 1000 + 2 * i));
}
}
}
diff --git a/test/unit/org/apache/cassandra/gms/ShadowRoundTest.java b/test/unit/org/apache/cassandra/gms/ShadowRoundTest.java
index f8cc49c..2bcbc50 100644
--- a/test/unit/org/apache/cassandra/gms/ShadowRoundTest.java
+++ b/test/unit/org/apache/cassandra/gms/ShadowRoundTest.java
@@ -30,13 +30,14 @@
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.commitlog.CommitLog;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.locator.IEndpointSnitch;
import org.apache.cassandra.locator.PropertyFileSnitch;
-import org.apache.cassandra.net.MessageIn;
-import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.net.Message;
import org.apache.cassandra.net.MockMessagingService;
import org.apache.cassandra.net.MockMessagingSpy;
+import org.apache.cassandra.net.Verb;
import org.apache.cassandra.service.StorageService;
import static org.apache.cassandra.net.MockMessagingService.verb;
@@ -53,6 +54,7 @@
System.setProperty("cassandra.config", "cassandra-seeds.yaml");
DatabaseDescriptor.daemonInitialization();
+ CommitLog.instance.start();
IEndpointSnitch snitch = new PropertyFileSnitch();
DatabaseDescriptor.setEndpointSnitch(snitch);
Keyspace.setInitialized();
@@ -71,11 +73,11 @@
int noOfSeeds = Gossiper.instance.seeds.size();
final AtomicBoolean ackSend = new AtomicBoolean(false);
- MockMessagingSpy spySyn = MockMessagingService.when(verb(MessagingService.Verb.GOSSIP_DIGEST_SYN))
+ MockMessagingSpy spySyn = MockMessagingService.when(verb(Verb.GOSSIP_DIGEST_SYN))
.respondN((msgOut, to) ->
{
// ACK once to finish shadow round, then busy-spin until gossiper has been enabled
- // and then reply with remaining ACKs from other seeds
+ // and then respond with remaining ACKs from other seeds
if (!ackSend.compareAndSet(false, true))
{
while (!Gossiper.instance.isEnabled()) ;
@@ -87,15 +89,17 @@
Collections.singletonList(new GossipDigest(to, hb.getGeneration(), hb.getHeartBeatVersion())),
Collections.singletonMap(to, state));
- logger.debug("Simulating digest ACK reply");
- return MessageIn.create(to, payload, Collections.emptyMap(), MessagingService.Verb.GOSSIP_DIGEST_ACK, MessagingService.current_version);
+ logger.debug("Simulating digest ACK response");
+ return Message.builder(Verb.GOSSIP_DIGEST_ACK, payload)
+ .from(to)
+ .build();
}, noOfSeeds);
// GossipDigestAckVerbHandler will send ack2 for each ack received (after the shadow round)
- MockMessagingSpy spyAck2 = MockMessagingService.when(verb(MessagingService.Verb.GOSSIP_DIGEST_ACK2)).dontReply();
+ MockMessagingSpy spyAck2 = MockMessagingService.when(verb(Verb.GOSSIP_DIGEST_ACK2)).dontReply();
// Migration request messages should not be emitted during shadow round
- MockMessagingSpy spyMigrationReq = MockMessagingService.when(verb(MessagingService.Verb.MIGRATION_REQUEST)).dontReply();
+ MockMessagingSpy spyMigrationReq = MockMessagingService.when(verb(Verb.SCHEMA_PULL_REQ)).dontReply();
try
{
@@ -109,7 +113,7 @@
// we expect one SYN for each seed during shadow round + additional SYNs after gossiper has been enabled
assertTrue(spySyn.messagesIntercepted > noOfSeeds);
- // we don't expect to emit any GOSSIP_DIGEST_ACK2 or MIGRATION_REQUEST messages
+ // we don't expect to emit any GOSSIP_DIGEST_ACK2 or SCHEMA_PULL messages
assertEquals(0, spyAck2.messagesIntercepted);
assertEquals(0, spyMigrationReq.messagesIntercepted);
}
diff --git a/test/unit/org/apache/cassandra/hadoop/ColumnFamilyInputFormatTest.java b/test/unit/org/apache/cassandra/hadoop/ColumnFamilyInputFormatTest.java
deleted file mode 100644
index d4261bf..0000000
--- a/test/unit/org/apache/cassandra/hadoop/ColumnFamilyInputFormatTest.java
+++ /dev/null
@@ -1,52 +0,0 @@
-package org.apache.cassandra.hadoop;
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
-
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.cassandra.thrift.SlicePredicate;
-import org.apache.cassandra.utils.ByteBufferUtil;
-import org.apache.hadoop.conf.Configuration;
-import org.junit.Test;
-
-public class ColumnFamilyInputFormatTest
-{
- @Test
- public void testSlicePredicate()
- {
- long columnValue = 1271253600000l;
- ByteBuffer columnBytes = ByteBufferUtil.bytes(columnValue);
-
- List<ByteBuffer> columnNames = new ArrayList<ByteBuffer>();
- columnNames.add(columnBytes);
- SlicePredicate originalPredicate = new SlicePredicate().setColumn_names(columnNames);
-
- Configuration conf = new Configuration();
- ConfigHelper.setInputSlicePredicate(conf, originalPredicate);
-
- SlicePredicate rtPredicate = ConfigHelper.getInputSlicePredicate(conf);
- assert rtPredicate.column_names.size() == 1;
- assert originalPredicate.column_names.get(0).equals(rtPredicate.column_names.get(0));
- }
-}
diff --git a/test/unit/org/apache/cassandra/hints/AlteredHints.java b/test/unit/org/apache/cassandra/hints/AlteredHints.java
index 7efe08f..9b8e32f 100644
--- a/test/unit/org/apache/cassandra/hints/AlteredHints.java
+++ b/test/unit/org/apache/cassandra/hints/AlteredHints.java
@@ -32,8 +32,8 @@
import org.junit.BeforeClass;
import org.apache.cassandra.SchemaLoader;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.Schema;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.db.Mutation;
import org.apache.cassandra.db.RowUpdateBuilder;
import org.apache.cassandra.schema.KeyspaceParams;
@@ -51,7 +51,7 @@
private static Mutation createMutation(int index, long timestamp)
{
- CFMetaData table = Schema.instance.getCFMetaData(KEYSPACE, TABLE);
+ TableMetadata table = Schema.instance.getTableMetadata(KEYSPACE, TABLE);
return new RowUpdateBuilder(table, timestamp, bytes(index))
.clustering(bytes(index))
.add("val", bytes(index))
diff --git a/test/unit/org/apache/cassandra/hints/ChecksummedDataInputTest.java b/test/unit/org/apache/cassandra/hints/ChecksummedDataInputTest.java
index 7325e74..9f4cdfb 100644
--- a/test/unit/org/apache/cassandra/hints/ChecksummedDataInputTest.java
+++ b/test/unit/org/apache/cassandra/hints/ChecksummedDataInputTest.java
@@ -29,6 +29,7 @@
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.io.util.DataOutputBuffer;
+import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.io.util.RandomAccessReader;
import org.apache.cassandra.io.util.SequentialWriter;
import org.apache.cassandra.utils.ByteBufferUtil;
@@ -83,7 +84,7 @@
FBUtilities.updateChecksum(crc, buffer);
// save the buffer to file to create a RAR
- File file = File.createTempFile("testReadMethods", "1");
+ File file = FileUtils.createTempFile("testReadMethods", "1");
file.deleteOnExit();
try (SequentialWriter writer = new SequentialWriter(file))
{
@@ -158,7 +159,7 @@
}
// save the buffer to file to create a RAR
- File file = File.createTempFile("testResetCrc", "1");
+ File file = FileUtils.createTempFile("testResetCrc", "1");
file.deleteOnExit();
try (SequentialWriter writer = new SequentialWriter(file))
{
@@ -214,7 +215,7 @@
}
// save the buffer to file to create a RAR
- File file = File.createTempFile("testFailedCrc", "1");
+ File file = FileUtils.createTempFile("testFailedCrc", "1");
file.deleteOnExit();
try (SequentialWriter writer = new SequentialWriter(file))
{
diff --git a/test/unit/org/apache/cassandra/hints/HintMessageTest.java b/test/unit/org/apache/cassandra/hints/HintMessageTest.java
index 7ffaa54..bb015a8 100644
--- a/test/unit/org/apache/cassandra/hints/HintMessageTest.java
+++ b/test/unit/org/apache/cassandra/hints/HintMessageTest.java
@@ -23,8 +23,8 @@
import org.junit.Test;
import org.apache.cassandra.SchemaLoader;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.Schema;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.db.Mutation;
import org.apache.cassandra.db.RowUpdateBuilder;
import org.apache.cassandra.io.util.DataInputBuffer;
@@ -53,7 +53,7 @@
UUID hostId = UUID.randomUUID();
long now = FBUtilities.timestampMicros();
- CFMetaData table = Schema.instance.getCFMetaData(KEYSPACE, TABLE);
+ TableMetadata table = Schema.instance.getTableMetadata(KEYSPACE, TABLE);
Mutation mutation =
new RowUpdateBuilder(table, now, bytes("key"))
.clustering("column")
diff --git a/test/unit/org/apache/cassandra/hints/HintTest.java b/test/unit/org/apache/cassandra/hints/HintTest.java
index e4a33fd..e3e26d0 100644
--- a/test/unit/org/apache/cassandra/hints/HintTest.java
+++ b/test/unit/org/apache/cassandra/hints/HintTest.java
@@ -18,7 +18,6 @@
package org.apache.cassandra.hints;
import java.io.IOException;
-import java.net.InetAddress;
import java.util.Collections;
import java.util.UUID;
@@ -39,12 +38,16 @@
import org.apache.cassandra.io.util.DataInputBuffer;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputBuffer;
+import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.locator.TokenMetadata;
import org.apache.cassandra.metrics.StorageMetrics;
-import org.apache.cassandra.net.MessageIn;
+import org.apache.cassandra.net.Message;
import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.schema.KeyspaceParams;
-import org.apache.cassandra.schema.TableParams;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.schema.MigrationManager;
import org.apache.cassandra.service.StorageProxy;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.FBUtilities;
@@ -54,7 +57,7 @@
import static org.apache.cassandra.Util.dk;
import static org.apache.cassandra.hints.HintsTestUtil.assertHintsEqual;
import static org.apache.cassandra.hints.HintsTestUtil.assertPartitionsEqual;
-import static org.apache.cassandra.utils.ByteBufferUtil.bytes;
+import static org.apache.cassandra.net.Verb.HINT_REQ;
public class HintTest
{
@@ -78,13 +81,13 @@
public void resetGcGraceSeconds()
{
TokenMetadata tokenMeta = StorageService.instance.getTokenMetadata();
- InetAddress local = FBUtilities.getBroadcastAddress();
+ InetAddressAndPort local = FBUtilities.getBroadcastAddressAndPort();
tokenMeta.clearUnsafe();
tokenMeta.updateHostId(UUID.randomUUID(), local);
tokenMeta.updateNormalTokens(BootStrapper.getRandomTokens(tokenMeta, 1), local);
- for (CFMetaData table : Schema.instance.getTablesAndViews(KEYSPACE))
- table.gcGraceSeconds(TableParams.DEFAULT_GC_GRACE_SECONDS);
+ for (TableMetadata table : Schema.instance.getTablesAndViews(KEYSPACE))
+ MigrationManager.announceTableUpdate(table.unbuild().gcGraceSeconds(864000).build(), true);
}
@Test
@@ -125,7 +128,7 @@
// assert that we can read the inserted partitions
for (PartitionUpdate partition : mutation.getPartitionUpdates())
- assertPartitionsEqual(partition, readPartition(key, partition.metadata().cfName, partition.columns()));
+ assertPartitionsEqual(partition, readPartition(key, partition.metadata().name, partition.columns()));
}
@Test
@@ -150,9 +153,9 @@
assertNoPartitions(key, TABLE1);
// TABLE0 and TABLE2 updates should have been applied successfully
- PartitionUpdate upd0 = mutation.getPartitionUpdate(Schema.instance.getId(KEYSPACE, TABLE0));
+ PartitionUpdate upd0 = mutation.getPartitionUpdate(Schema.instance.getTableMetadata(KEYSPACE, TABLE0));
assertPartitionsEqual(upd0, readPartition(key, TABLE0, upd0.columns()));
- PartitionUpdate upd2 = mutation.getPartitionUpdate(Schema.instance.getId(KEYSPACE, TABLE2));
+ PartitionUpdate upd2 = mutation.getPartitionUpdate(Schema.instance.getTableMetadata(KEYSPACE, TABLE2));
assertPartitionsEqual(upd2, readPartition(key, TABLE2, upd2.columns()));
}
@@ -161,7 +164,6 @@
{
long now = FBUtilities.timestampMicros();
String key = "testApplyWithRegularExpiration";
- Mutation mutation = createMutation(key, now);
// sanity check that there is no data inside yet
assertNoPartitions(key, TABLE0);
@@ -169,8 +171,15 @@
assertNoPartitions(key, TABLE2);
// lower the GC GS on TABLE0 to 0 BEFORE the hint is created
- Schema.instance.getCFMetaData(KEYSPACE, TABLE0).gcGraceSeconds(0);
+ TableMetadata updated =
+ Schema.instance
+ .getTableMetadata(KEYSPACE, TABLE0)
+ .unbuild()
+ .gcGraceSeconds(0)
+ .build();
+ MigrationManager.announceTableUpdate(updated, true);
+ Mutation mutation = createMutation(key, now);
Hint.create(mutation, now / 1000).apply();
// all updates should have been skipped and not applied, as expired
@@ -184,8 +193,6 @@
{
long now = FBUtilities.timestampMicros();
String key = "testApplyWithGCGSReducedLater";
- Mutation mutation = createMutation(key, now);
- Hint hint = Hint.create(mutation, now / 1000);
// sanity check that there is no data inside yet
assertNoPartitions(key, TABLE0);
@@ -193,8 +200,16 @@
assertNoPartitions(key, TABLE2);
// lower the GC GS on TABLE0 AFTER the hint is already created
- Schema.instance.getCFMetaData(KEYSPACE, TABLE0).gcGraceSeconds(0);
+ TableMetadata updated =
+ Schema.instance
+ .getTableMetadata(KEYSPACE, TABLE0)
+ .unbuild()
+ .gcGraceSeconds(0)
+ .build();
+ MigrationManager.announceTableUpdate(updated, true);
+ Mutation mutation = createMutation(key, now);
+ Hint hint = Hint.create(mutation, now / 1000);
hint.apply();
// all updates should have been skipped and not applied, as expired
@@ -215,8 +230,8 @@
// Prepare metadata with injected stale endpoint serving the mutation key.
TokenMetadata tokenMeta = StorageService.instance.getTokenMetadata();
- InetAddress local = FBUtilities.getBroadcastAddress();
- InetAddress endpoint = InetAddress.getByName("1.1.1.1");
+ InetAddressAndPort local = FBUtilities.getBroadcastAddressAndPort();
+ InetAddressAndPort endpoint = InetAddressAndPort.getByName("1.1.1.1");
UUID localId = StorageService.instance.getLocalHostUUID();
UUID targetId = UUID.randomUUID();
tokenMeta.updateHostId(targetId, endpoint);
@@ -231,9 +246,7 @@
long totalHintCount = StorageProxy.instance.getTotalHints();
// Process hint message.
HintMessage message = new HintMessage(localId, hint);
- MessagingService.instance().getVerbHandler(MessagingService.Verb.HINT).doVerb(
- MessageIn.create(local, message, Collections.emptyMap(), MessagingService.Verb.HINT, MessagingService.current_version),
- -1);
+ HINT_REQ.handler().doVerb(Message.out(HINT_REQ, message));
// hint should not be applied as we no longer are a replica
assertNoPartitions(key, TABLE0);
@@ -256,8 +269,8 @@
// Prepare metadata with injected stale endpoint.
TokenMetadata tokenMeta = StorageService.instance.getTokenMetadata();
- InetAddress local = FBUtilities.getBroadcastAddress();
- InetAddress endpoint = InetAddress.getByName("1.1.1.1");
+ InetAddressAndPort local = FBUtilities.getBroadcastAddressAndPort();
+ InetAddressAndPort endpoint = InetAddressAndPort.getByName("1.1.1.1");
UUID localId = StorageService.instance.getLocalHostUUID();
UUID targetId = UUID.randomUUID();
tokenMeta.updateHostId(targetId, endpoint);
@@ -276,9 +289,8 @@
long totalHintCount = StorageMetrics.totalHints.getCount();
// Process hint message.
HintMessage message = new HintMessage(localId, hint);
- MessagingService.instance().getVerbHandler(MessagingService.Verb.HINT).doVerb(
- MessageIn.create(local, message, Collections.emptyMap(), MessagingService.Verb.HINT, MessagingService.current_version),
- -1);
+ HINT_REQ.<HintMessage>handler().doVerb(
+ Message.builder(HINT_REQ, message).from(local).build());
// hint should not be applied as we no longer are a replica
assertNoPartitions(key, TABLE0);
@@ -298,17 +310,17 @@
{
Mutation.SimpleBuilder builder = Mutation.simpleBuilder(KEYSPACE, dk(key));
- builder.update(Schema.instance.getCFMetaData(KEYSPACE, TABLE0))
+ builder.update(Schema.instance.getTableMetadata(KEYSPACE, TABLE0))
.timestamp(now)
.row("column0")
.add("val", "value0");
- builder.update(Schema.instance.getCFMetaData(KEYSPACE, TABLE1))
+ builder.update(Schema.instance.getTableMetadata(KEYSPACE, TABLE1))
.timestamp(now + 1)
.row("column1")
.add("val", "value1");
- builder.update(Schema.instance.getCFMetaData(KEYSPACE, TABLE2))
+ builder.update(Schema.instance.getTableMetadata(KEYSPACE, TABLE2))
.timestamp(now + 2)
.row("column2")
.add("val", "value2");
@@ -318,14 +330,14 @@
private static ColumnFamilyStore cfs(String table)
{
- return Schema.instance.getColumnFamilyStoreInstance(Schema.instance.getCFMetaData(KEYSPACE, table).cfId);
+ return Schema.instance.getColumnFamilyStoreInstance(Schema.instance.getTableMetadata(KEYSPACE, table).id);
}
- private static FilteredPartition readPartition(String key, String table, PartitionColumns columns)
+ private static FilteredPartition readPartition(String key, String table, RegularAndStaticColumns columns)
{
String[] columnNames = new String[columns.size()];
int i = 0;
- for (ColumnDefinition column : columns)
+ for (ColumnMetadata column : columns)
columnNames[i++] = column.name.toString();
return Util.getOnlyPartition(Util.cmd(cfs(table), key).columns(columnNames).build());
diff --git a/test/unit/org/apache/cassandra/hints/HintWriteTTLTest.java b/test/unit/org/apache/cassandra/hints/HintWriteTTLTest.java
new file mode 100644
index 0000000..21dbd7e
--- /dev/null
+++ b/test/unit/org/apache/cassandra/hints/HintWriteTTLTest.java
@@ -0,0 +1,169 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.hints;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.cql3.statements.schema.CreateTableStatement;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.Mutation;
+import org.apache.cassandra.db.partitions.PartitionUpdate;
+import org.apache.cassandra.io.util.DataInputBuffer;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.UUIDGen;
+
+public class HintWriteTTLTest
+{
+ private static int TTL = 500;
+ private static int GC_GRACE = 84600;
+
+ private static Hint makeHint(TableMetadata tbm, int key, int creationTime, int gcgs)
+ {
+ PartitionUpdate update = PartitionUpdate.fullPartitionDelete(tbm,
+ ByteBufferUtil.bytes(key),
+ s2m(creationTime),
+ creationTime);
+ Mutation mutation = new Mutation(update);
+ return Hint.create(mutation, s2m(creationTime), gcgs);
+ }
+
+ private static DecoratedKey hintKey(Hint hint)
+ {
+ return hint.mutation.key();
+ }
+
+ private static Hint deserialize(ByteBuffer bb) throws IOException
+ {
+ DataInputBuffer input = new DataInputBuffer(bb, true);
+ try
+ {
+ return Hint.serializer.deserialize(input, MessagingService.current_version);
+ }
+ finally
+ {
+ input.close();
+ }
+ }
+
+ private static Hint ttldHint = null;
+ private static Hint liveHint = null;
+ private static File hintFile = null;
+
+ @BeforeClass
+ public static void setupClass() throws Exception
+ {
+ System.setProperty("cassandra.maxHintTTL", Integer.toString(TTL));
+ SchemaLoader.prepareServer();
+ TableMetadata tbm = CreateTableStatement.parse("CREATE TABLE tbl (k INT PRIMARY KEY, v INT)", "ks").gcGraceSeconds(GC_GRACE).build();
+ SchemaLoader.createKeyspace("ks", KeyspaceParams.simple(1), tbm);
+
+ int nowInSeconds = FBUtilities.nowInSeconds();
+ liveHint = makeHint(tbm, 1, nowInSeconds, GC_GRACE);
+ ttldHint = makeHint(tbm, 2, nowInSeconds - (TTL + 1), GC_GRACE);
+
+
+ File directory = Files.createTempDirectory(null).toFile();
+ HintsDescriptor descriptor = new HintsDescriptor(UUIDGen.getTimeUUID(), s2m(nowInSeconds));
+
+ try (HintsWriter writer = HintsWriter.create(directory, descriptor);
+ HintsWriter.Session session = writer.newSession(ByteBuffer.allocate(1024)))
+ {
+ session.append(liveHint);
+ session.append(ttldHint);
+ hintFile = writer.getFile();
+ }
+ }
+
+ private static long s2m(int seconds)
+ {
+ return TimeUnit.SECONDS.toMillis(seconds);
+ }
+
+ @Test
+ public void isLive() throws Exception
+ {
+ // max ttl is set to 500
+ Assert.assertTrue(Hint.isLive(s2m(0), s2m(499), 500)); // still live
+ Assert.assertFalse(Hint.isLive(s2m(0), s2m(499), 499)); // expired due to hint's own ttl
+ Assert.assertFalse(Hint.isLive(s2m(0), s2m(500), 501)); // expired due to max ttl
+ }
+
+ @Test
+ public void hintIsLive() throws Exception
+ {
+ Assert.assertTrue(liveHint.isLive());
+ Assert.assertFalse(ttldHint.isLive());
+ }
+
+ @Test
+ public void hintIterator() throws Exception
+ {
+ List<Hint> hints = new ArrayList<>();
+ try (HintsReader reader = HintsReader.open(hintFile))
+ {
+ for (HintsReader.Page page: reader)
+ {
+ Iterator<Hint> iter = page.hintsIterator();
+ while (iter.hasNext())
+ {
+ hints.add(iter.next());
+ }
+ }
+ }
+
+ Assert.assertEquals(1, hints.size());
+ Assert.assertEquals(hintKey(liveHint), hintKey(hints.get(0)));
+ }
+
+ @Test
+ public void bufferIterator() throws Exception
+ {
+ List<Hint> hints = new ArrayList<>();
+ try (HintsReader reader = HintsReader.open(hintFile))
+ {
+ for (HintsReader.Page page: reader)
+ {
+ Iterator<ByteBuffer> iter = page.buffersIterator();
+ while (iter.hasNext())
+ {
+ hints.add(deserialize(iter.next()));
+ }
+ }
+ }
+
+ Assert.assertEquals(1, hints.size());
+ Assert.assertEquals(hintKey(liveHint), hintKey(hints.get(0)));
+ }
+}
diff --git a/test/unit/org/apache/cassandra/hints/HintsBufferPoolTest.java b/test/unit/org/apache/cassandra/hints/HintsBufferPoolTest.java
index 7c8d0be..1374d80 100644
--- a/test/unit/org/apache/cassandra/hints/HintsBufferPoolTest.java
+++ b/test/unit/org/apache/cassandra/hints/HintsBufferPoolTest.java
@@ -28,7 +28,6 @@
import static junit.framework.Assert.*;
-import java.lang.Thread.State;
import java.util.Queue;
import java.util.UUID;
import java.util.concurrent.ConcurrentLinkedQueue;
diff --git a/test/unit/org/apache/cassandra/hints/HintsBufferTest.java b/test/unit/org/apache/cassandra/hints/HintsBufferTest.java
index 08f7ec0..a4cb651 100644
--- a/test/unit/org/apache/cassandra/hints/HintsBufferTest.java
+++ b/test/unit/org/apache/cassandra/hints/HintsBufferTest.java
@@ -29,8 +29,6 @@
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.concurrent.NamedThreadFactory;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.Schema;
import org.apache.cassandra.db.Mutation;
import org.apache.cassandra.db.RowUpdateBuilder;
import org.apache.cassandra.db.rows.Cell;
@@ -39,6 +37,8 @@
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.TableMetadata;
import static junit.framework.Assert.*;
@@ -197,7 +197,7 @@
private static Mutation createMutation(int index, long timestamp)
{
- CFMetaData table = Schema.instance.getCFMetaData(KEYSPACE, TABLE);
+ TableMetadata table = Schema.instance.getTableMetadata(KEYSPACE, TABLE);
return new RowUpdateBuilder(table, timestamp, bytes(index))
.clustering(bytes(index))
.add("val", bytes(index))
diff --git a/test/unit/org/apache/cassandra/hints/HintsCatalogTest.java b/test/unit/org/apache/cassandra/hints/HintsCatalogTest.java
index 68acd0c..92cfc71 100644
--- a/test/unit/org/apache/cassandra/hints/HintsCatalogTest.java
+++ b/test/unit/org/apache/cassandra/hints/HintsCatalogTest.java
@@ -25,10 +25,9 @@
import com.google.common.collect.ImmutableMap;
import org.apache.cassandra.SchemaLoader;
-import org.apache.cassandra.config.Schema;
import org.apache.cassandra.db.Mutation;
-import org.apache.cassandra.db.RowUpdateBuilder;
import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.utils.FBUtilities;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -195,17 +194,17 @@
{
Mutation.SimpleBuilder builder = Mutation.simpleBuilder(KEYSPACE, dk(key));
- builder.update(Schema.instance.getCFMetaData(KEYSPACE, TABLE0))
+ builder.update(Schema.instance.getTableMetadata(KEYSPACE, TABLE0))
.timestamp(now)
.row("column0")
.add("val", "value0");
- builder.update(Schema.instance.getCFMetaData(KEYSPACE, TABLE1))
+ builder.update(Schema.instance.getTableMetadata(KEYSPACE, TABLE1))
.timestamp(now + 1)
.row("column1")
.add("val", "value1");
- builder.update(Schema.instance.getCFMetaData(KEYSPACE, TABLE2))
+ builder.update(Schema.instance.getTableMetadata(KEYSPACE, TABLE2))
.timestamp(now + 2)
.row("column2")
.add("val", "value2");
diff --git a/test/unit/org/apache/cassandra/hints/HintsCompressionTest.java b/test/unit/org/apache/cassandra/hints/HintsCompressionTest.java
index f82db49..faa8d27 100644
--- a/test/unit/org/apache/cassandra/hints/HintsCompressionTest.java
+++ b/test/unit/org/apache/cassandra/hints/HintsCompressionTest.java
@@ -26,6 +26,7 @@
import org.apache.cassandra.io.compress.ICompressor;
import org.apache.cassandra.io.compress.LZ4Compressor;
import org.apache.cassandra.io.compress.SnappyCompressor;
+import org.apache.cassandra.io.compress.ZstdCompressor;
public class HintsCompressionTest extends AlteredHints
{
@@ -77,4 +78,12 @@
compressorClass = DeflateCompressor.class;
multiFlushAndDeserializeTest();
}
+
+ @Test
+ public void zstdCompressor() throws Exception
+ {
+ compressorClass = ZstdCompressor.class;
+ multiFlushAndDeserializeTest();
+ }
+
}
diff --git a/test/unit/org/apache/cassandra/hints/HintsDescriptorTest.java b/test/unit/org/apache/cassandra/hints/HintsDescriptorTest.java
index 08487d1..2d9f972 100644
--- a/test/unit/org/apache/cassandra/hints/HintsDescriptorTest.java
+++ b/test/unit/org/apache/cassandra/hints/HintsDescriptorTest.java
@@ -29,9 +29,11 @@
import org.apache.cassandra.io.compress.LZ4Compressor;
import org.apache.cassandra.io.util.DataOutputBuffer;
+import org.apache.cassandra.net.MessagingService;
import static junit.framework.Assert.assertEquals;
import static junit.framework.Assert.assertNotSame;
+import static junit.framework.Assert.assertTrue;
import static junit.framework.Assert.fail;
public class HintsDescriptorTest
@@ -115,6 +117,17 @@
}
}
+ @Test
+ public void testMessagingVersion()
+ {
+ String errorMsg = "Please update the current Hints messaging version to match the current messaging version";
+ int messageVersion = HintsDescriptor.messagingVersion(HintsDescriptor.CURRENT_VERSION);
+ assertEquals(errorMsg, messageVersion, MessagingService.current_version);
+
+ HintsDescriptor descriptor = new HintsDescriptor(UUID.randomUUID(), HintsDescriptor.CURRENT_VERSION, System.currentTimeMillis(), ImmutableMap.of());
+ assertEquals(errorMsg, descriptor.messagingVersion(), MessagingService.current_version);
+ }
+
private static void testSerializeDeserializeLoop(HintsDescriptor descriptor) throws IOException
{
// serialize to a byte array
diff --git a/test/unit/org/apache/cassandra/hints/HintsReaderTest.java b/test/unit/org/apache/cassandra/hints/HintsReaderTest.java
index 70cf6e7..d95bd56 100644
--- a/test/unit/org/apache/cassandra/hints/HintsReaderTest.java
+++ b/test/unit/org/apache/cassandra/hints/HintsReaderTest.java
@@ -31,14 +31,15 @@
import org.junit.Test;
import org.apache.cassandra.SchemaLoader;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.Schema;
import org.apache.cassandra.db.Mutation;
import org.apache.cassandra.db.RowUpdateBuilder;
import org.apache.cassandra.db.rows.Cell;
import org.apache.cassandra.db.rows.Row;
import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.MigrationManager;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.TableMetadata;
import static junit.framework.Assert.assertEquals;
import static junit.framework.Assert.assertNotNull;
@@ -64,7 +65,7 @@
private static Mutation createMutation(int index, long timestamp, String ks, String tb)
{
- CFMetaData table = Schema.instance.getCFMetaData(ks, tb);
+ TableMetadata table = Schema.instance.getTableMetadata(ks, tb);
return new RowUpdateBuilder(table, timestamp, bytes(index))
.clustering(bytes(index))
.add("val", bytes(index))
@@ -157,11 +158,12 @@
KeyspaceParams.simple(1),
SchemaLoader.standardCFMD(ks, CF_STANDARD1),
SchemaLoader.standardCFMD(ks, CF_STANDARD2));
+
directory = Files.createTempDirectory(null).toFile();
try
{
generateHints(3, ks);
- Schema.instance.dropTable(ks, CF_STANDARD1);
+ MigrationManager.announceTableDrop(ks, CF_STANDARD1, true);
readHints(3, 1);
}
finally
diff --git a/test/unit/org/apache/cassandra/hints/HintsServiceTest.java b/test/unit/org/apache/cassandra/hints/HintsServiceTest.java
index 077a9d1..dddf336 100644
--- a/test/unit/org/apache/cassandra/hints/HintsServiceTest.java
+++ b/test/unit/org/apache/cassandra/hints/HintsServiceTest.java
@@ -17,8 +17,6 @@
*/
package org.apache.cassandra.hints;
-import java.net.InetAddress;
-import java.util.Collections;
import java.util.UUID;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
@@ -27,6 +25,7 @@
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.MoreExecutors;
import org.junit.After;
import org.junit.Before;
import org.junit.BeforeClass;
@@ -34,22 +33,25 @@
import com.datastax.driver.core.utils.MoreFutures;
import org.apache.cassandra.SchemaLoader;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.Schema;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.net.NoPayload;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.partitions.PartitionUpdate;
import org.apache.cassandra.gms.IFailureDetectionEventListener;
import org.apache.cassandra.gms.IFailureDetector;
import org.apache.cassandra.metrics.StorageMetrics;
-import org.apache.cassandra.net.MessageIn;
+import org.apache.cassandra.net.Message;
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.net.MockMessagingService;
import org.apache.cassandra.net.MockMessagingSpy;
import org.apache.cassandra.schema.KeyspaceParams;
import org.apache.cassandra.service.StorageService;
-import org.apache.cassandra.utils.FBUtilities;
import static org.apache.cassandra.Util.dk;
+import static org.apache.cassandra.net.Verb.HINT_REQ;
+import static org.apache.cassandra.net.Verb.HINT_RSP;
import static org.apache.cassandra.net.MockMessagingService.verb;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@@ -78,9 +80,10 @@
}
@Before
- public void reinstanciateService() throws ExecutionException, InterruptedException
+ public void reinstanciateService() throws Throwable
{
- MessagingService.instance().clearMessageSinks();
+ MessagingService.instance().inboundSink.clear();
+ MessagingService.instance().outboundSink.clear();
if (!HintsService.instance.isShutDown())
{
@@ -89,7 +92,9 @@
}
failureDetector.isAlive = true;
+
HintsService.instance = new HintsService(failureDetector);
+
HintsService.instance.startDispatch();
}
@@ -125,7 +130,7 @@
{
HintsService.instance.resumeDispatch();
}
- });
+ }, MoreExecutors.directExecutor());
Futures.allAsList(
noMessagesWhilePaused,
@@ -180,20 +185,16 @@
private MockMessagingSpy sendHintsAndResponses(int noOfHints, int noOfResponses)
{
// create spy for hint messages, but only create responses for noOfResponses hints
- MessageIn<HintResponse> messageIn = MessageIn.create(FBUtilities.getBroadcastAddress(),
- HintResponse.instance,
- Collections.emptyMap(),
- MessagingService.Verb.REQUEST_RESPONSE,
- MessagingService.current_version);
+ Message<NoPayload> message = Message.internalResponse(HINT_RSP, NoPayload.noPayload);
MockMessagingSpy spy;
if (noOfResponses != -1)
{
- spy = MockMessagingService.when(verb(MessagingService.Verb.HINT)).respondN(messageIn, noOfResponses);
+ spy = MockMessagingService.when(verb(HINT_REQ)).respondN(message, noOfResponses);
}
else
{
- spy = MockMessagingService.when(verb(MessagingService.Verb.HINT)).respond(messageIn);
+ spy = MockMessagingService.when(verb(HINT_REQ)).respond(message);
}
// create and write noOfHints using service
@@ -202,8 +203,8 @@
{
long now = System.currentTimeMillis();
DecoratedKey dkey = dk(String.valueOf(i));
- CFMetaData cfMetaData = Schema.instance.getCFMetaData(KEYSPACE, TABLE);
- PartitionUpdate.SimpleBuilder builder = PartitionUpdate.simpleBuilder(cfMetaData, dkey).timestamp(now);
+ TableMetadata metadata = Schema.instance.getTableMetadata(KEYSPACE, TABLE);
+ PartitionUpdate.SimpleBuilder builder = PartitionUpdate.simpleBuilder(metadata, dkey).timestamp(now);
builder.row("column0").add("val", "value0");
Hint hint = Hint.create(builder.buildAsMutation(), now);
HintsService.instance.write(hostId, hint);
@@ -215,17 +216,17 @@
{
private boolean isAlive = true;
- public boolean isAlive(InetAddress ep)
+ public boolean isAlive(InetAddressAndPort ep)
{
return isAlive;
}
- public void interpret(InetAddress ep)
+ public void interpret(InetAddressAndPort ep)
{
throw new UnsupportedOperationException();
}
- public void report(InetAddress ep)
+ public void report(InetAddressAndPort ep)
{
throw new UnsupportedOperationException();
}
@@ -240,12 +241,12 @@
throw new UnsupportedOperationException();
}
- public void remove(InetAddress ep)
+ public void remove(InetAddressAndPort ep)
{
throw new UnsupportedOperationException();
}
- public void forceConviction(InetAddress ep)
+ public void forceConviction(InetAddressAndPort ep)
{
throw new UnsupportedOperationException();
}
diff --git a/test/unit/org/apache/cassandra/hints/HintsTestUtil.java b/test/unit/org/apache/cassandra/hints/HintsTestUtil.java
index 89b532f..c1c6192 100644
--- a/test/unit/org/apache/cassandra/hints/HintsTestUtil.java
+++ b/test/unit/org/apache/cassandra/hints/HintsTestUtil.java
@@ -17,11 +17,8 @@
*/
package org.apache.cassandra.hints;
-import java.util.UUID;
-
import com.google.common.collect.Iterators;
-import org.apache.cassandra.db.Mutation;
import org.apache.cassandra.db.partitions.AbstractBTreePartition;
import org.apache.cassandra.db.partitions.PartitionUpdate;
@@ -30,15 +27,6 @@
final class HintsTestUtil
{
- static void assertMutationsEqual(Mutation expected, Mutation actual)
- {
- assertEquals(expected.key(), actual.key());
- assertEquals(expected.getPartitionUpdates().size(), actual.getPartitionUpdates().size());
-
- for (UUID id : expected.getColumnFamilyIds())
- assertPartitionsEqual(expected.getPartitionUpdate(id), actual.getPartitionUpdate(id));
- }
-
static void assertPartitionsEqual(AbstractBTreePartition expected, AbstractBTreePartition actual)
{
assertEquals(expected.partitionKey(), actual.partitionKey());
@@ -51,9 +39,9 @@
{
assertEquals(expected.mutation.getKeyspaceName(), actual.mutation.getKeyspaceName());
assertEquals(expected.mutation.key(), actual.mutation.key());
- assertEquals(expected.mutation.getColumnFamilyIds(), actual.mutation.getColumnFamilyIds());
+ assertEquals(expected.mutation.getTableIds(), actual.mutation.getTableIds());
for (PartitionUpdate partitionUpdate : expected.mutation.getPartitionUpdates())
- assertPartitionsEqual(partitionUpdate, actual.mutation.getPartitionUpdate(partitionUpdate.metadata().cfId));
+ assertPartitionsEqual(partitionUpdate, actual.mutation.getPartitionUpdate(partitionUpdate.metadata()));
assertEquals(expected.creationTime, actual.creationTime);
assertEquals(expected.gcgs, actual.gcgs);
}
diff --git a/test/unit/org/apache/cassandra/hints/LegacyHintsMigratorTest.java b/test/unit/org/apache/cassandra/hints/LegacyHintsMigratorTest.java
deleted file mode 100644
index 78849e3..0000000
--- a/test/unit/org/apache/cassandra/hints/LegacyHintsMigratorTest.java
+++ /dev/null
@@ -1,197 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.hints;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.nio.file.Files;
-import java.util.*;
-
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import org.apache.cassandra.SchemaLoader;
-import org.apache.cassandra.Util;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.Schema;
-import org.apache.cassandra.config.SchemaConstants;
-import org.apache.cassandra.db.*;
-import org.apache.cassandra.db.marshal.UUIDType;
-import org.apache.cassandra.db.partitions.PartitionUpdate;
-import org.apache.cassandra.db.rows.BTreeRow;
-import org.apache.cassandra.db.rows.BufferCell;
-import org.apache.cassandra.db.rows.Cell;
-import org.apache.cassandra.net.MessagingService;
-import org.apache.cassandra.schema.KeyspaceParams;
-import org.apache.cassandra.utils.FBUtilities;
-import org.apache.cassandra.utils.UUIDGen;
-
-import static junit.framework.Assert.assertEquals;
-import static junit.framework.Assert.assertNotNull;
-import static junit.framework.Assert.assertTrue;
-
-import static org.apache.cassandra.hints.HintsTestUtil.assertMutationsEqual;
-import static org.apache.cassandra.utils.ByteBufferUtil.bytes;
-
-// TODO: test split into several files
-@SuppressWarnings("deprecation")
-public class LegacyHintsMigratorTest
-{
- private static final String KEYSPACE = "legacy_hints_migrator_test";
- private static final String TABLE = "table";
-
- @BeforeClass
- public static void defineSchema()
- {
- SchemaLoader.prepareServer();
- SchemaLoader.createKeyspace(KEYSPACE, KeyspaceParams.simple(1), SchemaLoader.standardCFMD(KEYSPACE, TABLE));
- }
-
- @Test
- public void testNothingToMigrate() throws IOException
- {
- File directory = Files.createTempDirectory(null).toFile();
- try
- {
- testNothingToMigrate(directory);
- }
- finally
- {
- directory.deleteOnExit();
- }
- }
-
- private static void testNothingToMigrate(File directory)
- {
- // truncate system.hints to enseure nothing inside
- Keyspace.open(SchemaConstants.SYSTEM_KEYSPACE_NAME).getColumnFamilyStore(SystemKeyspace.LEGACY_HINTS).truncateBlocking();
- new LegacyHintsMigrator(directory, 128 * 1024 * 1024).migrate();
- HintsCatalog catalog = HintsCatalog.load(directory, HintsService.EMPTY_PARAMS);
- assertEquals(0, catalog.stores().count());
- }
-
- @Test
- public void testMigrationIsComplete() throws IOException
- {
- File directory = Files.createTempDirectory(null).toFile();
- try
- {
- testMigrationIsComplete(directory);
- }
- finally
- {
- directory.deleteOnExit();
- }
- }
-
- private static void testMigrationIsComplete(File directory)
- {
- long timestamp = System.currentTimeMillis();
-
- // write 100 mutations for each of the 10 generated endpoints
- Map<UUID, Queue<Mutation>> mutations = new HashMap<>();
- for (int i = 0; i < 10; i++)
- {
- UUID hostId = UUID.randomUUID();
- Queue<Mutation> queue = new LinkedList<>();
- mutations.put(hostId, queue);
-
- for (int j = 0; j < 100; j++)
- {
- Mutation mutation = createMutation(j, timestamp + j);
- queue.offer(mutation);
- Mutation legacyHint = createLegacyHint(mutation, timestamp, hostId);
- legacyHint.applyUnsafe();
- }
- }
-
- // run the migration
- new LegacyHintsMigrator(directory, 128 * 1024 * 1024).migrate();
-
- // validate that the hints table is truncated now
- assertTrue(Keyspace.open(SchemaConstants.SYSTEM_KEYSPACE_NAME).getColumnFamilyStore(SystemKeyspace.LEGACY_HINTS).isEmpty());
-
- HintsCatalog catalog = HintsCatalog.load(directory, HintsService.EMPTY_PARAMS);
-
- // assert that we've correctly loaded 10 hints stores
- assertEquals(10, catalog.stores().count());
-
- // for each of the 10 stores, make sure the mutations have been migrated correctly
- for (Map.Entry<UUID, Queue<Mutation>> entry : mutations.entrySet())
- {
- HintsStore store = catalog.get(entry.getKey());
- assertNotNull(store);
-
- HintsDescriptor descriptor = store.poll();
- assertNotNull(descriptor);
-
- // read all the hints
- Queue<Hint> actualHints = new LinkedList<>();
- try (HintsReader reader = HintsReader.open(new File(directory, descriptor.fileName())))
- {
- for (HintsReader.Page page : reader)
- page.hintsIterator().forEachRemaining(actualHints::offer);
- }
-
- // assert the size matches
- assertEquals(100, actualHints.size());
-
- // compare expected hints to actual hints
- for (int i = 0; i < 100; i++)
- {
- Hint hint = actualHints.poll();
- Mutation mutation = entry.getValue().poll();
- int ttl = mutation.smallestGCGS();
-
- assertEquals(timestamp, hint.creationTime);
- assertEquals(ttl, hint.gcgs);
- assertTrue(mutation + " != " + hint.mutation, Util.sameContent(mutation, hint.mutation));
- }
- }
- }
-
- // legacy hint mutation creation code, copied more or less verbatim from the previous implementation
- private static Mutation createLegacyHint(Mutation mutation, long now, UUID targetId)
- {
- int version = MessagingService.VERSION_21;
- int ttl = mutation.smallestGCGS();
- UUID hintId = UUIDGen.getTimeUUID();
-
- ByteBuffer key = UUIDType.instance.decompose(targetId);
- Clustering clustering = SystemKeyspace.LegacyHints.comparator.make(hintId, version);
- ByteBuffer value = ByteBuffer.wrap(FBUtilities.serialize(mutation, Mutation.serializer, version));
- Cell cell = BufferCell.expiring(SystemKeyspace.LegacyHints.compactValueColumn(),
- now,
- ttl,
- FBUtilities.nowInSeconds(),
- value);
- return new Mutation(PartitionUpdate.singleRowUpdate(SystemKeyspace.LegacyHints,
- key,
- BTreeRow.singleCellRow(clustering, cell)));
- }
-
- private static Mutation createMutation(int index, long timestamp)
- {
- CFMetaData table = Schema.instance.getCFMetaData(KEYSPACE, TABLE);
- return new RowUpdateBuilder(table, timestamp, bytes(index))
- .clustering(bytes(index))
- .add("val", bytes(index))
- .build();
- }
-}
diff --git a/test/unit/org/apache/cassandra/index/CustomIndexTest.java b/test/unit/org/apache/cassandra/index/CustomIndexTest.java
index 9fe3f65..2b2bb87 100644
--- a/test/unit/org/apache/cassandra/index/CustomIndexTest.java
+++ b/test/unit/org/apache/cassandra/index/CustomIndexTest.java
@@ -27,18 +27,17 @@
import java.util.stream.Collectors;
import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Sets;
import org.junit.Test;
import com.datastax.driver.core.exceptions.QueryValidationException;
import org.apache.cassandra.Util;
-import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.cql3.CQLTester;
import org.apache.cassandra.cql3.ColumnIdentifier;
import org.apache.cassandra.cql3.restrictions.IndexRestrictions;
import org.apache.cassandra.cql3.restrictions.StatementRestrictions;
-import org.apache.cassandra.cql3.statements.IndexTarget;
+import org.apache.cassandra.cql3.statements.schema.IndexTarget;
import org.apache.cassandra.cql3.statements.ModificationStatement;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.marshal.AbstractType;
@@ -55,8 +54,7 @@
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.concurrent.OpOrder;
-import static org.apache.cassandra.Util.throwAssert;
-import static org.apache.cassandra.cql3.statements.IndexTarget.CUSTOM_INDEX_OPTION_NAME;
+import static org.apache.cassandra.cql3.statements.schema.IndexTarget.CUSTOM_INDEX_OPTION_NAME;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
@@ -71,7 +69,7 @@
// test to ensure that we don't deadlock when flushing CFS backed custom indexers
// see CASSANDRA-10181
createTable("CREATE TABLE %s (a int, b int, c int, d int, PRIMARY KEY (a, b))");
- createIndex("CREATE CUSTOM INDEX myindex ON %s(c) USING 'org.apache.cassandra.index.internal.CustomCassandraIndex'");
+ createIndex("CREATE CUSTOM INDEX ON %s(c) USING 'org.apache.cassandra.index.internal.CustomCassandraIndex'");
execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 0, 0, 0, 2);
execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 0, 1, 0, 1);
@@ -84,7 +82,7 @@
// deadlocks and times out the test in the face of the synchronisation
// issues described in the comments on CASSANDRA-9669
createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY (a))");
- createIndex("CREATE CUSTOM INDEX b_index ON %s(b) USING 'org.apache.cassandra.index.StubIndex'");
+ createIndex("CREATE CUSTOM INDEX ON %s(b) USING 'org.apache.cassandra.index.StubIndex'");
execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", 0, 1, 2);
getCurrentColumnFamilyStore().truncateBlocking();
}
@@ -114,7 +112,7 @@
excluded.reset();
assertTrue(excluded.rowsInserted.isEmpty());
- indexManager.buildAllIndexesBlocking(getCurrentColumnFamilyStore().getLiveSSTables());
+ indexManager.rebuildIndexesBlocking(Sets.newHashSet(toInclude, toExclude));
assertEquals(3, included.rowsInserted.size());
assertTrue(excluded.rowsInserted.isEmpty());
@@ -165,11 +163,11 @@
{
createTable("CREATE TABLE %s(k int, c int, v1 int, v2 int, PRIMARY KEY (k,c))");
- assertInvalidMessage("Duplicate column v1 in index target list",
+ assertInvalidMessage("Duplicate column 'v1' in index target list",
String.format("CREATE CUSTOM INDEX ON %%s(v1, v1) USING '%s'",
StubIndex.class.getName()));
- assertInvalidMessage("Duplicate column v1 in index target list",
+ assertInvalidMessage("Duplicate column 'v1' in index target list",
String.format("CREATE CUSTOM INDEX ON %%s(v1, v1, c, c) USING '%s'",
StubIndex.class.getName()));
}
@@ -188,39 +186,39 @@
" PRIMARY KEY(k,c))");
assertInvalidMessage("Cannot create keys() index on frozen column fmap. " +
- "Frozen collections only support full() indexes",
+ "Frozen collections are immutable and must be fully indexed",
String.format("CREATE CUSTOM INDEX ON %%s(c, keys(fmap)) USING'%s'",
StubIndex.class.getName()));
assertInvalidMessage("Cannot create entries() index on frozen column fmap. " +
- "Frozen collections only support full() indexes",
+ "Frozen collections are immutable and must be fully indexed",
String.format("CREATE CUSTOM INDEX ON %%s(c, entries(fmap)) USING'%s'",
StubIndex.class.getName()));
assertInvalidMessage("Cannot create values() index on frozen column fmap. " +
- "Frozen collections only support full() indexes",
+ "Frozen collections are immutable and must be fully indexed",
String.format("CREATE CUSTOM INDEX ON %%s(c, fmap) USING'%s'", StubIndex.class.getName()));
assertInvalidMessage("Cannot create keys() index on frozen column flist. " +
- "Frozen collections only support full() indexes",
+ "Frozen collections are immutable and must be fully indexed",
String.format("CREATE CUSTOM INDEX ON %%s(c, keys(flist)) USING'%s'",
StubIndex.class.getName()));
assertInvalidMessage("Cannot create entries() index on frozen column flist. " +
- "Frozen collections only support full() indexes",
+ "Frozen collections are immutable and must be fully indexed",
String.format("CREATE CUSTOM INDEX ON %%s(c, entries(flist)) USING'%s'",
StubIndex.class.getName()));
assertInvalidMessage("Cannot create values() index on frozen column flist. " +
- "Frozen collections only support full() indexes",
+ "Frozen collections are immutable and must be fully indexed",
String.format("CREATE CUSTOM INDEX ON %%s(c, flist) USING'%s'", StubIndex.class.getName()));
assertInvalidMessage("Cannot create keys() index on frozen column fset. " +
- "Frozen collections only support full() indexes",
+ "Frozen collections are immutable and must be fully indexed",
String.format("CREATE CUSTOM INDEX ON %%s(c, keys(fset)) USING'%s'",
StubIndex.class.getName()));
assertInvalidMessage("Cannot create entries() index on frozen column fset. " +
- "Frozen collections only support full() indexes",
+ "Frozen collections are immutable and must be fully indexed",
String.format("CREATE CUSTOM INDEX ON %%s(c, entries(fset)) USING'%s'",
StubIndex.class.getName()));
assertInvalidMessage("Cannot create values() index on frozen column fset. " +
- "Frozen collections only support full() indexes",
+ "Frozen collections are immutable and must be fully indexed",
String.format("CREATE CUSTOM INDEX ON %%s(c, fset) USING'%s'", StubIndex.class.getName()));
createIndex(String.format("CREATE CUSTOM INDEX ON %%s(c, full(fmap)) USING'%s'", StubIndex.class.getName()));
@@ -234,21 +232,21 @@
createTable("CREATE TABLE %s(k int, c int, v1 int, v2 int, PRIMARY KEY(k,c))");
createIndex(String.format("CREATE CUSTOM INDEX ON %%s(v1, v2) USING '%s'", StubIndex.class.getName()));
- assertEquals(1, getCurrentColumnFamilyStore().metadata.getIndexes().size());
+ assertEquals(1, getCurrentColumnFamilyStore().metadata().indexes.size());
assertIndexCreated(currentTable() + "_idx", "v1", "v2");
createIndex(String.format("CREATE CUSTOM INDEX ON %%s(c, v1, v2) USING '%s'", StubIndex.class.getName()));
- assertEquals(2, getCurrentColumnFamilyStore().metadata.getIndexes().size());
+ assertEquals(2, getCurrentColumnFamilyStore().metadata().indexes.size());
assertIndexCreated(currentTable() + "_idx_1", "c", "v1", "v2");
createIndex(String.format("CREATE CUSTOM INDEX ON %%s(c, v2) USING '%s'", StubIndex.class.getName()));
- assertEquals(3, getCurrentColumnFamilyStore().metadata.getIndexes().size());
+ assertEquals(3, getCurrentColumnFamilyStore().metadata().indexes.size());
assertIndexCreated(currentTable() + "_idx_2", "c", "v2");
// duplicate the previous index with some additional options and check the name is generated as expected
createIndex(String.format("CREATE CUSTOM INDEX ON %%s(c, v2) USING '%s' WITH OPTIONS = {'foo':'bar'}",
StubIndex.class.getName()));
- assertEquals(4, getCurrentColumnFamilyStore().metadata.getIndexes().size());
+ assertEquals(4, getCurrentColumnFamilyStore().metadata().indexes.size());
Map<String, String> options = new HashMap<>();
options.put("foo", "bar");
assertIndexCreated(currentTable() + "_idx_3", options, "c", "v2");
@@ -280,7 +278,19 @@
testCreateIndex("idx_5", "c2", "v1");
testCreateIndex("idx_6", "v1", "v2");
testCreateIndex("idx_7", "pk2", "c2", "v2");
- testCreateIndex("idx_8", "pk1", "c1", "v1", "mval", "sval", "lval");
+
+ createIndex(String.format("CREATE CUSTOM INDEX idx_8 ON %%s(" +
+ " pk1, c1, v1, values(mval), values(sval), values(lval)" +
+ ") USING '%s'",
+ StubIndex.class.getName()));
+ assertIndexCreated("idx_8",
+ new HashMap<>(),
+ ImmutableList.of(indexTarget("pk1", IndexTarget.Type.SIMPLE),
+ indexTarget("c1", IndexTarget.Type.SIMPLE),
+ indexTarget("v1", IndexTarget.Type.SIMPLE),
+ indexTarget("mval", IndexTarget.Type.VALUES),
+ indexTarget("sval", IndexTarget.Type.VALUES),
+ indexTarget("lval", IndexTarget.Type.VALUES)));
createIndex(String.format("CREATE CUSTOM INDEX inc_frozen ON %%s(" +
" pk2, c2, v2, full(fmap), full(fset), full(flist)" +
@@ -288,9 +298,9 @@
StubIndex.class.getName()));
assertIndexCreated("inc_frozen",
new HashMap<>(),
- ImmutableList.of(indexTarget("pk2", IndexTarget.Type.VALUES),
- indexTarget("c2", IndexTarget.Type.VALUES),
- indexTarget("v2", IndexTarget.Type.VALUES),
+ ImmutableList.of(indexTarget("pk2", IndexTarget.Type.SIMPLE),
+ indexTarget("c2", IndexTarget.Type.SIMPLE),
+ indexTarget("v2", IndexTarget.Type.SIMPLE),
indexTarget("fmap", IndexTarget.Type.FULL),
indexTarget("fset", IndexTarget.Type.FULL),
indexTarget("flist", IndexTarget.Type.FULL)));
@@ -301,12 +311,12 @@
StubIndex.class.getName()));
assertIndexCreated("all_teh_things",
new HashMap<>(),
- ImmutableList.of(indexTarget("pk1", IndexTarget.Type.VALUES),
- indexTarget("pk2", IndexTarget.Type.VALUES),
- indexTarget("c1", IndexTarget.Type.VALUES),
- indexTarget("c2", IndexTarget.Type.VALUES),
- indexTarget("v1", IndexTarget.Type.VALUES),
- indexTarget("v2", IndexTarget.Type.VALUES),
+ ImmutableList.of(indexTarget("pk1", IndexTarget.Type.SIMPLE),
+ indexTarget("pk2", IndexTarget.Type.SIMPLE),
+ indexTarget("c1", IndexTarget.Type.SIMPLE),
+ indexTarget("c2", IndexTarget.Type.SIMPLE),
+ indexTarget("v1", IndexTarget.Type.SIMPLE),
+ indexTarget("v2", IndexTarget.Type.SIMPLE),
indexTarget("mval", IndexTarget.Type.KEYS),
indexTarget("lval", IndexTarget.Type.VALUES),
indexTarget("sval", IndexTarget.Type.VALUES),
@@ -321,16 +331,6 @@
String myType = KEYSPACE + '.' + createType("CREATE TYPE %s (a int, b int)");
createTable("CREATE TABLE %s (k int PRIMARY KEY, v1 int, v2 frozen<" + myType + ">)");
testCreateIndex("udt_idx", "v1", "v2");
- Indexes indexes = getCurrentColumnFamilyStore().metadata.getIndexes();
- IndexMetadata expected = IndexMetadata.fromIndexTargets(getCurrentColumnFamilyStore().metadata,
- ImmutableList.of(indexTarget("v1", IndexTarget.Type.VALUES),
- indexTarget("v2", IndexTarget.Type.VALUES)),
- "udt_idx",
- IndexMetadata.Kind.CUSTOM,
- ImmutableMap.of(CUSTOM_INDEX_OPTION_NAME,
- StubIndex.class.getName()));
- IndexMetadata actual = indexes.get("udt_idx").orElseThrow(throwAssert("Index udt_idx not found"));
- assertEquals(expected, actual);
}
@Test
@@ -356,13 +356,13 @@
execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", row);
- assertInvalidMessage(String.format(IndexRestrictions.INDEX_NOT_FOUND, indexName, keyspace(), currentTable()),
+ assertInvalidMessage(String.format(IndexRestrictions.INDEX_NOT_FOUND, indexName, currentTableMetadata().toString()),
String.format("SELECT * FROM %%s WHERE expr(%s, 'foo bar baz')", indexName));
createIndex(String.format("CREATE CUSTOM INDEX %s ON %%s(c) USING '%s'", indexName, StubIndex.class.getName()));
assertInvalidThrowMessage(Optional.of(ProtocolVersion.CURRENT),
- String.format(IndexRestrictions.INDEX_NOT_FOUND, "no_such_index", keyspace(), currentTable()),
+ String.format(IndexRestrictions.INDEX_NOT_FOUND, "no_such_index", currentTableMetadata().toString()),
QueryValidationException.class,
"SELECT * FROM %s WHERE expr(no_such_index, 'foo bar baz ')");
@@ -537,7 +537,7 @@
@Test
public void reloadIndexMetadataOnBaseCfsReload() throws Throwable
{
- // verify that whenever the base table CFMetadata is reloaded, a reload of the index
+ // verify that whenever the base table TableMetadata is reloaded, a reload of the index
// metadata is performed
createTable("CREATE TABLE %s (k int, v1 int, PRIMARY KEY(k))");
createIndex(String.format("CREATE CUSTOM INDEX reload_counter ON %%s() USING '%s'",
@@ -617,13 +617,13 @@
}
@Test
- public void validateOptionsWithCFMetaData() throws Throwable
+ public void validateOptionsWithTableMetadata() throws Throwable
{
createTable("CREATE TABLE %s(k int, c int, v1 int, v2 int, PRIMARY KEY(k,c))");
createIndex(String.format("CREATE CUSTOM INDEX ON %%s(c, v2) USING '%s' WITH OPTIONS = {'foo':'bar'}",
IndexWithOverloadedValidateOptions.class.getName()));
- CFMetaData cfm = getCurrentColumnFamilyStore().metadata;
- assertEquals(cfm, IndexWithOverloadedValidateOptions.cfm);
+ TableMetadata table = getCurrentColumnFamilyStore().metadata();
+ assertEquals(table, IndexWithOverloadedValidateOptions.table);
assertNotNull(IndexWithOverloadedValidateOptions.options);
assertEquals("bar", IndexWithOverloadedValidateOptions.options.get("foo"));
}
@@ -859,8 +859,7 @@
private void assertIndexCreated(String name, Map<String, String> options, String... targetColumnNames)
{
List<IndexTarget> targets = Arrays.stream(targetColumnNames)
- .map(s -> new IndexTarget(ColumnIdentifier.getInterned(s, true),
- IndexTarget.Type.VALUES))
+ .map(s -> new IndexTarget(ColumnIdentifier.getInterned(s, true), IndexTarget.Type.SIMPLE))
.collect(Collectors.toList());
assertIndexCreated(name, options, targets);
}
@@ -870,14 +869,13 @@
// all tests here use StubIndex as the custom index class,
// so add that to the map of options
options.put(CUSTOM_INDEX_OPTION_NAME, StubIndex.class.getName());
- CFMetaData cfm = getCurrentColumnFamilyStore().metadata;
- IndexMetadata expected = IndexMetadata.fromIndexTargets(cfm, targets, name, IndexMetadata.Kind.CUSTOM, options);
- Indexes indexes = getCurrentColumnFamilyStore().metadata.getIndexes();
+ IndexMetadata expected = IndexMetadata.fromIndexTargets(targets, name, IndexMetadata.Kind.CUSTOM, options);
+ Indexes indexes = getCurrentColumnFamilyStore().metadata().indexes;
for (IndexMetadata actual : indexes)
if (actual.equals(expected))
return;
- fail(String.format("Index %s not found in CFMetaData", expected));
+ fail(String.format("Index %s not found", expected));
}
private static IndexTarget indexTarget(String name, IndexTarget.Type type)
@@ -1034,7 +1032,7 @@
public static final class IndexWithOverloadedValidateOptions extends StubIndex
{
- public static CFMetaData cfm;
+ public static TableMetadata table;
public static Map<String, String> options;
public IndexWithOverloadedValidateOptions(ColumnFamilyStore baseCfs, IndexMetadata metadata)
@@ -1042,10 +1040,10 @@
super(baseCfs, metadata);
}
- public static Map<String, String> validateOptions(Map<String, String> options, CFMetaData cfm)
+ public static Map<String, String> validateOptions(Map<String, String> options, TableMetadata table)
{
IndexWithOverloadedValidateOptions.options = options;
- IndexWithOverloadedValidateOptions.cfm = cfm;
+ IndexWithOverloadedValidateOptions.table = table;
return new HashMap<>();
}
}
@@ -1084,15 +1082,16 @@
// various OpOrder.Groups, which it can obtain from this index.
public Indexer indexerFor(final DecoratedKey key,
- PartitionColumns columns,
+ RegularAndStaticColumns columns,
int nowInSec,
- OpOrder.Group opGroup,
+ WriteContext ctx,
IndexTransaction.Type transactionType)
{
+ CassandraWriteContext cassandraWriteContext = (CassandraWriteContext) ctx;
if (readOrderingAtStart == null)
readOrderingAtStart = baseCfs.readOrdering.getCurrent();
- writeGroups.add(opGroup);
+ writeGroups.add(cassandraWriteContext.getGroup());
return new Indexer()
{
diff --git a/test/unit/org/apache/cassandra/index/SecondaryIndexManagerTest.java b/test/unit/org/apache/cassandra/index/SecondaryIndexManagerTest.java
new file mode 100644
index 0000000..d8fb99f
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/SecondaryIndexManagerTest.java
@@ -0,0 +1,848 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index;
+
+import java.io.FileNotFoundException;
+import java.net.SocketException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import com.google.common.collect.Sets;
+import org.junit.After;
+import org.junit.Test;
+
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.SystemKeyspace;
+import org.apache.cassandra.db.compaction.CompactionInfo;
+import org.apache.cassandra.db.lifecycle.SSTableSet;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.notifications.SSTableAddedNotification;
+import org.apache.cassandra.schema.IndexMetadata;
+import org.apache.cassandra.utils.JVMStabilityInspector;
+import org.apache.cassandra.utils.KillerForTests;
+import org.apache.cassandra.utils.concurrent.Refs;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+public class SecondaryIndexManagerTest extends CQLTester
+{
+
+ @After
+ public void after()
+ {
+ TestingIndex.clear();
+ }
+
+ @Test
+ public void creatingIndexMarksTheIndexAsBuilt() throws Throwable
+ {
+ String tableName = createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY (a, b))");
+ String indexName = createIndex("CREATE INDEX ON %s(c)");
+
+ waitForIndex(KEYSPACE, tableName, indexName);
+ assertMarkedAsBuilt(indexName);
+ }
+
+ @Test
+ public void rebuilOrRecoveringIndexMarksTheIndexAsBuilt() throws Throwable
+ {
+ String tableName = createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY (a, b))");
+ String indexName = createIndex("CREATE INDEX ON %s(c)");
+
+ waitForIndex(KEYSPACE, tableName, indexName);
+ assertMarkedAsBuilt(indexName);
+
+ assertTrue(tryRebuild(indexName, false));
+ assertMarkedAsBuilt(indexName);
+ }
+
+ @Test
+ public void recreatingIndexMarksTheIndexAsBuilt() throws Throwable
+ {
+ String tableName = createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY (a, b))");
+ String indexName = createIndex("CREATE INDEX ON %s(c)");
+
+ waitForIndex(KEYSPACE, tableName, indexName);
+ assertMarkedAsBuilt(indexName);
+
+ // drop the index and verify that it has been removed from the built indexes table
+ dropIndex("DROP INDEX %s." + indexName);
+ assertNotMarkedAsBuilt(indexName);
+
+ // create the index again and verify that it's added to the built indexes table
+ createIndex(String.format("CREATE INDEX %s ON %%s(c)", indexName));
+ waitForIndex(KEYSPACE, tableName, indexName);
+ assertMarkedAsBuilt(indexName);
+ }
+
+ @Test
+ public void addingSSTablesMarksTheIndexAsBuilt() throws Throwable
+ {
+ String tableName = createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY (a, b))");
+ String indexName = createIndex("CREATE INDEX ON %s(c)");
+
+ waitForIndex(KEYSPACE, tableName, indexName);
+ assertMarkedAsBuilt(indexName);
+
+ ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
+ cfs.indexManager.markAllIndexesRemoved();
+ assertNotMarkedAsBuilt(indexName);
+
+ try (Refs<SSTableReader> sstables = Refs.ref(cfs.getSSTables(SSTableSet.CANONICAL)))
+ {
+ cfs.indexManager.handleNotification(new SSTableAddedNotification(sstables, null), cfs.getTracker());
+ assertMarkedAsBuilt(indexName);
+ }
+ }
+
+ @Test
+ public void cannotRebuildRecoverWhileInitializationIsInProgress() throws Throwable
+ {
+ // create an index which blocks on creation
+ TestingIndex.blockCreate();
+ String tableName = createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY (a, b))");
+ String defaultIndexName = createIndex(String.format("CREATE CUSTOM INDEX ON %%s(c) USING '%s'", TestingIndex.class.getName()));
+ String readOnlyIndexName = createIndex(String.format("CREATE CUSTOM INDEX ON %%s(b) USING '%s'", ReadOnlyOnFailureIndex.class.getName()));
+ String writeOnlyIndexName = createIndex(String.format("CREATE CUSTOM INDEX ON %%s(b) USING '%s'", WriteOnlyOnFailureIndex.class.getName()));
+
+ // try to rebuild/recover the index before the index creation task has finished
+ assertFalse(tryRebuild(defaultIndexName, false));
+ assertFalse(tryRebuild(readOnlyIndexName, false));
+ assertFalse(tryRebuild(writeOnlyIndexName, false));
+ assertNotMarkedAsBuilt(defaultIndexName);
+ assertNotMarkedAsBuilt(readOnlyIndexName);
+ assertNotMarkedAsBuilt(writeOnlyIndexName);
+
+ // check that the index is marked as built when the creation finishes
+ TestingIndex.unblockCreate();
+ waitForIndex(KEYSPACE, tableName, defaultIndexName);
+ waitForIndex(KEYSPACE, tableName, readOnlyIndexName);
+ waitForIndex(KEYSPACE, tableName, writeOnlyIndexName);
+ assertMarkedAsBuilt(defaultIndexName);
+ assertMarkedAsBuilt(readOnlyIndexName);
+ assertMarkedAsBuilt(writeOnlyIndexName);
+
+ // now verify you can rebuild/recover
+ assertTrue(tryRebuild(defaultIndexName, false));
+ assertTrue(tryRebuild(readOnlyIndexName, false));
+ assertTrue(tryRebuild(readOnlyIndexName, false));
+ assertMarkedAsBuilt(defaultIndexName);
+ assertMarkedAsBuilt(readOnlyIndexName);
+ assertMarkedAsBuilt(writeOnlyIndexName);
+ }
+
+ @Test
+ public void cannotRebuildOrRecoverWhileAnotherRebuildIsInProgress() throws Throwable
+ {
+ String tableName = createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY (a, b))");
+ String defaultIndexName = createIndex(String.format("CREATE CUSTOM INDEX ON %%s(c) USING '%s'", TestingIndex.class.getName()));
+ String readOnlyIndexName = createIndex(String.format("CREATE CUSTOM INDEX ON %%s(b) USING '%s'", ReadOnlyOnFailureIndex.class.getName()));
+ String writeOnlyIndexName = createIndex(String.format("CREATE CUSTOM INDEX ON %%s(b) USING '%s'", WriteOnlyOnFailureIndex.class.getName()));
+ final AtomicBoolean error = new AtomicBoolean();
+
+ // wait for index initialization and verify it's built:
+ waitForIndex(KEYSPACE, tableName, defaultIndexName);
+ waitForIndex(KEYSPACE, tableName, readOnlyIndexName);
+ waitForIndex(KEYSPACE, tableName, writeOnlyIndexName);
+ assertMarkedAsBuilt(defaultIndexName);
+ assertMarkedAsBuilt(readOnlyIndexName);
+ assertMarkedAsBuilt(writeOnlyIndexName);
+
+ // rebuild the index in another thread, but make it block:
+ TestingIndex.blockBuild();
+ Thread asyncBuild = new Thread(() -> {
+ try
+ {
+ tryRebuild(defaultIndexName, false);
+ }
+ catch (Throwable ex)
+ {
+ error.set(true);
+ }
+ });
+ asyncBuild.start();
+
+ // wait for the rebuild to block, so that we can proceed unblocking all further operations:
+ TestingIndex.waitBlockedOnBuild();
+
+ // do not block further builds:
+ TestingIndex.shouldBlockBuild = false;
+
+ // verify rebuilding the index before the previous index build task has finished fails
+ assertFalse(tryRebuild(defaultIndexName, false));
+ assertNotMarkedAsBuilt(defaultIndexName);
+
+ // check that the index is marked as built when the build finishes
+ TestingIndex.unblockBuild();
+ asyncBuild.join();
+ assertMarkedAsBuilt(defaultIndexName);
+ assertMarkedAsBuilt(readOnlyIndexName);
+ assertMarkedAsBuilt(writeOnlyIndexName);
+
+ // now verify you can rebuild
+ assertTrue(tryRebuild(defaultIndexName, false));
+ assertTrue(tryRebuild(readOnlyIndexName, false));
+ assertTrue(tryRebuild(writeOnlyIndexName, false));
+ assertMarkedAsBuilt(defaultIndexName);
+ assertMarkedAsBuilt(readOnlyIndexName);
+ assertMarkedAsBuilt(writeOnlyIndexName);
+ }
+
+ @Test
+ public void cannotRebuildWhileAnSSTableBuildIsInProgress() throws Throwable
+ {
+ final String tableName = createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY (a, b))");
+ final String indexName = createIndex(String.format("CREATE CUSTOM INDEX ON %%s(c) USING '%s'", TestingIndex.class.getName()));
+ final AtomicBoolean error = new AtomicBoolean();
+
+ // wait for index initialization and verify it's built:
+ waitForIndex(KEYSPACE, tableName, indexName);
+ assertMarkedAsBuilt(indexName);
+
+ // add sstables in another thread, but make it block:
+ TestingIndex.blockBuild();
+ Thread asyncBuild = new Thread(() -> {
+ ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
+ try (Refs<SSTableReader> sstables = Refs.ref(cfs.getSSTables(SSTableSet.CANONICAL)))
+ {
+ cfs.indexManager.handleNotification(new SSTableAddedNotification(sstables, null), cfs.getTracker());
+ }
+ catch (Throwable ex)
+ {
+ error.set(true);
+ }
+ });
+ asyncBuild.start();
+
+ // wait for the build to block, so that we can proceed unblocking all further operations:
+ TestingIndex.waitBlockedOnBuild();
+
+ // do not block further builds:
+ TestingIndex.shouldBlockBuild = false;
+
+ // verify rebuilding the index before the previous index build task has finished fails
+ assertFalse(tryRebuild(indexName, false));
+ assertNotMarkedAsBuilt(indexName);
+
+ // check that the index is marked as built when the build finishes
+ TestingIndex.unblockBuild();
+ asyncBuild.join();
+ assertMarkedAsBuilt(indexName);
+
+ // now verify you can rebuild
+ assertTrue(tryRebuild(indexName, false));
+ assertMarkedAsBuilt(indexName);
+ }
+
+ @Test
+ public void addingSSTableWhileRebuildIsInProgress() throws Throwable
+ {
+ final String tableName = createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY (a, b))");
+ final String indexName = createIndex(String.format("CREATE CUSTOM INDEX ON %%s(c) USING '%s'", TestingIndex.class.getName()));
+ final AtomicBoolean error = new AtomicBoolean();
+
+ // wait for index initialization and verify it's built:
+ waitForIndex(KEYSPACE, tableName, indexName);
+ assertMarkedAsBuilt(indexName);
+
+ // rebuild the index in another thread, but make it block:
+ TestingIndex.blockBuild();
+ Thread asyncBuild = new Thread(() -> {
+ try
+ {
+ tryRebuild(indexName, false);
+ }
+ catch (Throwable ex)
+ {
+ error.set(true);
+ }
+ });
+ asyncBuild.start();
+
+ // wait for the rebuild to block, so that we can proceed unblocking all further operations:
+ TestingIndex.waitBlockedOnBuild();
+
+ // do not block further builds:
+ TestingIndex.shouldBlockBuild = false;
+
+ // try adding sstables and verify they are built but the index is not marked as built because of the pending build:
+ ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
+ try (Refs<SSTableReader> sstables = Refs.ref(cfs.getSSTables(SSTableSet.CANONICAL)))
+ {
+ cfs.indexManager.handleNotification(new SSTableAddedNotification(sstables, null), cfs.getTracker());
+ assertNotMarkedAsBuilt(indexName);
+ }
+
+ // unblock the pending build:
+ TestingIndex.unblockBuild();
+ asyncBuild.join();
+
+ // verify the index is now built:
+ assertMarkedAsBuilt(indexName);
+ assertFalse(error.get());
+ }
+
+ @Test
+ public void addingSSTableWithBuildFailureWhileRebuildIsInProgress() throws Throwable
+ {
+ final String tableName = createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY (a, b))");
+ final String indexName = createIndex(String.format("CREATE CUSTOM INDEX ON %%s(c) USING '%s'", TestingIndex.class.getName()));
+ final AtomicBoolean error = new AtomicBoolean();
+
+ // wait for index initialization and verify it's built:
+ waitForIndex(KEYSPACE, tableName, indexName);
+ assertMarkedAsBuilt(indexName);
+
+ // rebuild the index in another thread, but make it block:
+ TestingIndex.blockBuild();
+ Thread asyncBuild = new Thread(() -> {
+ try
+ {
+ tryRebuild(indexName, false);
+ }
+ catch (Throwable ex)
+ {
+ error.set(true);
+ }
+ });
+ asyncBuild.start();
+
+ // wait for the rebuild to block, so that we can proceed unblocking all further operations:
+ TestingIndex.waitBlockedOnBuild();
+
+ // do not block further builds:
+ TestingIndex.shouldBlockBuild = false;
+
+ // try adding sstables but make the build fail:
+ TestingIndex.shouldFailBuild = true;
+ ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
+ try (Refs<SSTableReader> sstables = Refs.ref(cfs.getSSTables(SSTableSet.CANONICAL)))
+ {
+ cfs.indexManager.handleNotification(new SSTableAddedNotification(sstables, null), cfs.getTracker());
+ fail("Should have failed!");
+ }
+ catch (Throwable ex)
+ {
+ assertTrue(ex.getMessage().contains("configured to fail"));
+ }
+
+ // disable failures:
+ TestingIndex.shouldFailBuild = false;
+
+ // unblock the pending build:
+ TestingIndex.unblockBuild();
+ asyncBuild.join();
+
+ // verify the index is *not* built due to the failing sstable build:
+ assertNotMarkedAsBuilt(indexName);
+ assertFalse(error.get());
+ }
+
+ @Test
+ public void rebuildWithFailure() throws Throwable
+ {
+ final String tableName = createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY (a, b))");
+ final String indexName = createIndex(String.format("CREATE CUSTOM INDEX ON %%s(c) USING '%s'", TestingIndex.class.getName()));
+ waitForIndex(KEYSPACE, tableName, indexName);
+
+ // Rebuild the index with failure and verify it is not marked as built
+ TestingIndex.shouldFailBuild = true;
+ try
+ {
+ ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
+ cfs.indexManager.rebuildIndexesBlocking(Collections.singleton(indexName));
+ fail("Should have failed!");
+ }
+ catch (Throwable ex)
+ {
+ assertTrue(ex.getMessage().contains("configured to fail"));
+ }
+ assertNotMarkedAsBuilt(indexName);
+ }
+
+ @Test
+ public void initializingIndexNotQueryableButMaybeWritable() throws Throwable
+ {
+ TestingIndex.blockCreate();
+ String tableName = createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY (a, b))");
+ String defaultIndexName = createIndex(String.format("CREATE CUSTOM INDEX ON %%s(c) USING '%s'", TestingIndex.class.getName()));
+ String readOnlyIndexName = createIndex(String.format("CREATE CUSTOM INDEX ON %%s(c) USING '%s'", ReadOnlyOnFailureIndex.class.getName()));
+ String writeOnlyIndexName = createIndex(String.format("CREATE CUSTOM INDEX ON %%s(c) USING '%s'", WriteOnlyOnFailureIndex.class.getName()));
+
+ // the index shouldn't be queryable while the initialization hasn't finished
+ assertFalse(isQueryable(defaultIndexName));
+ assertFalse(isQueryable(readOnlyIndexName));
+ assertFalse(isQueryable(writeOnlyIndexName));
+ assertTrue(isWritable(defaultIndexName));
+ assertTrue(isWritable(readOnlyIndexName));
+ assertTrue(isWritable(writeOnlyIndexName));
+
+ // the index should be queryable once the initialization has finished
+ TestingIndex.unblockCreate();
+ waitForIndex(KEYSPACE, tableName, defaultIndexName);
+ waitForIndex(KEYSPACE, tableName, readOnlyIndexName);
+ waitForIndex(KEYSPACE, tableName, writeOnlyIndexName);
+ assertTrue(isQueryable(defaultIndexName));
+ assertTrue(isQueryable(readOnlyIndexName));
+ assertTrue(isQueryable(writeOnlyIndexName));
+ assertTrue(isWritable(defaultIndexName));
+ assertTrue(isWritable(readOnlyIndexName));
+ assertTrue(isWritable(writeOnlyIndexName));
+ }
+
+ @Test
+ public void initializingIndexNotQueryableButMaybeNotWritableAfterPartialRebuild() throws Throwable
+ {
+ TestingIndex.blockCreate();
+ String tableName = createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY (a, b))");
+ String defaultIndexName = createIndex(String.format("CREATE CUSTOM INDEX ON %%s(c) USING '%s'", TestingIndex.class.getName()));
+ String readOnlyIndexName = createIndex(String.format("CREATE CUSTOM INDEX ON %%s(c) USING '%s'", ReadOnlyOnFailureIndex.class.getName()));
+ String writeOnlyIndexName = createIndex(String.format("CREATE CUSTOM INDEX ON %%s(c) USING '%s'", WriteOnlyOnFailureIndex.class.getName()));
+
+ // the index should never be queryable while the initialization hasn't finished
+ assertFalse(isQueryable(defaultIndexName));
+ assertFalse(isQueryable(readOnlyIndexName));
+ assertFalse(isQueryable(writeOnlyIndexName));
+
+ // the index should always we writable while the initialization hasn't finished
+ assertTrue(isWritable(defaultIndexName));
+ assertTrue(isWritable(readOnlyIndexName));
+ assertTrue(isWritable(writeOnlyIndexName));
+
+ // a failing partial build doesn't set the index as queryable, but might set it as not writable
+ TestingIndex.shouldFailBuild = true;
+ ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
+ try
+ {
+ cfs.indexManager.handleNotification(new SSTableAddedNotification(cfs.getLiveSSTables(), null), this);
+ fail("Should have failed!");
+ }
+ catch (Throwable ex)
+ {
+ assertTrue(ex.getMessage().contains("configured to fail"));
+ }
+ assertFalse(isQueryable(defaultIndexName));
+ assertFalse(isQueryable(readOnlyIndexName));
+ assertFalse(isQueryable(writeOnlyIndexName));
+ assertTrue(isWritable(defaultIndexName));
+ assertFalse(isWritable(readOnlyIndexName));
+ assertTrue(isWritable(writeOnlyIndexName));
+
+ // a successful partial build doesn't set the index as queryable nor writable
+ TestingIndex.shouldFailBuild = false;
+ cfs.indexManager.handleNotification(new SSTableAddedNotification(cfs.getLiveSSTables(), null), this);
+ assertFalse(isQueryable(defaultIndexName));
+ assertFalse(isQueryable(readOnlyIndexName));
+ assertFalse(isQueryable(writeOnlyIndexName));
+ assertTrue(isWritable(defaultIndexName));
+ assertFalse(isWritable(readOnlyIndexName));
+ assertTrue(isWritable(writeOnlyIndexName));
+
+ // the index should be queryable once the initialization has finished
+ TestingIndex.unblockCreate();
+ waitForIndex(KEYSPACE, tableName, defaultIndexName);
+ assertTrue(isQueryable(defaultIndexName));
+ assertTrue(isQueryable(readOnlyIndexName));
+ assertTrue(isQueryable(writeOnlyIndexName));
+ assertTrue(isWritable(defaultIndexName));
+ assertTrue(isWritable(readOnlyIndexName));
+ assertTrue(isWritable(writeOnlyIndexName));
+ }
+
+ @Test
+ public void indexWithFailedInitializationIsQueryableAndWritableAfterFullRebuild() throws Throwable
+ {
+ TestingIndex.shouldFailCreate = true;
+ createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY (a, b))");
+ String defaultIndexName = createIndex(String.format("CREATE CUSTOM INDEX ON %%s(c) USING '%s'", TestingIndex.class.getName()));
+ String readOnlyIndexName = createIndex(String.format("CREATE CUSTOM INDEX ON %%s(c) USING '%s'", ReadOnlyOnFailureIndex.class.getName()));
+ String writeOnlyIndexName = createIndex(String.format("CREATE CUSTOM INDEX ON %%s(c) USING '%s'", WriteOnlyOnFailureIndex.class.getName()));
+ assertTrue(waitForIndexBuilds(KEYSPACE, defaultIndexName));
+ assertTrue(waitForIndexBuilds(KEYSPACE, readOnlyIndexName));
+ assertTrue(waitForIndexBuilds(KEYSPACE, writeOnlyIndexName));
+
+ tryRebuild(defaultIndexName, true);
+ tryRebuild(readOnlyIndexName, true);
+ tryRebuild(writeOnlyIndexName, true);
+ TestingIndex.shouldFailCreate = false;
+
+ // a successfull full rebuild should set the index as queryable/writable
+ ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
+ cfs.indexManager.rebuildIndexesBlocking(Sets.newHashSet(defaultIndexName, readOnlyIndexName, writeOnlyIndexName));
+ assertTrue(isQueryable(defaultIndexName));
+ assertTrue(isQueryable(readOnlyIndexName));
+ assertTrue(isQueryable(writeOnlyIndexName));
+ assertTrue(isWritable(defaultIndexName));
+ assertTrue(isWritable(readOnlyIndexName));
+ assertTrue(isWritable(writeOnlyIndexName));
+ }
+
+ @Test
+ public void indexWithFailedInitializationDoesNotChangeQueryabilityNorWritabilityAfterPartialRebuild() throws Throwable
+ {
+ TestingIndex.shouldFailCreate = true;
+ createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY (a, b))");
+ String defaultIndexName = createIndex(String.format("CREATE CUSTOM INDEX ON %%s(c) USING '%s'", TestingIndex.class.getName()));
+ String readOnlyIndexName = createIndex(String.format("CREATE CUSTOM INDEX ON %%s(c) USING '%s'", ReadOnlyOnFailureIndex.class.getName()));
+ String writeOnlyIndexName = createIndex(String.format("CREATE CUSTOM INDEX ON %%s(c) USING '%s'", WriteOnlyOnFailureIndex.class.getName()));
+ assertTrue(waitForIndexBuilds(KEYSPACE, defaultIndexName));
+ assertTrue(waitForIndexBuilds(KEYSPACE, readOnlyIndexName));
+ assertTrue(waitForIndexBuilds(KEYSPACE, writeOnlyIndexName));
+ TestingIndex.shouldFailCreate = false;
+
+ // the index should never be queryable, but it could be writable after the failed initialization
+ assertFalse(isQueryable(defaultIndexName));
+ assertFalse(isQueryable(readOnlyIndexName));
+ assertFalse(isQueryable(writeOnlyIndexName));
+ assertTrue(isWritable(defaultIndexName));
+ assertFalse(isWritable(readOnlyIndexName));
+ assertTrue(isWritable(writeOnlyIndexName));
+
+ // a successful partial build doesn't set the index as queryable nor writable
+ ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
+ cfs.indexManager.handleNotification(new SSTableAddedNotification(cfs.getLiveSSTables(), null), this);
+ assertTrue(waitForIndexBuilds(KEYSPACE, defaultIndexName));
+ assertTrue(waitForIndexBuilds(KEYSPACE, readOnlyIndexName));
+ assertTrue(waitForIndexBuilds(KEYSPACE, writeOnlyIndexName));
+ assertFalse(isQueryable(defaultIndexName));
+ assertFalse(isQueryable(readOnlyIndexName));
+ assertFalse(isQueryable(writeOnlyIndexName));
+ assertTrue(isWritable(defaultIndexName));
+ assertFalse(isWritable(readOnlyIndexName));
+ assertTrue(isWritable(writeOnlyIndexName));
+ }
+
+ @Test
+ public void handleJVMStablityOnFailedCreate()
+ {
+ handleJVMStablityOnFailedCreate(new SocketException("Should not fail"), false);
+ handleJVMStablityOnFailedCreate(new FileNotFoundException("Should not fail"), false);
+ handleJVMStablityOnFailedCreate(new SocketException("Too many open files"), true);
+ handleJVMStablityOnFailedCreate(new FileNotFoundException("Too many open files"), true);
+ handleJVMStablityOnFailedCreate(new RuntimeException("Should not fail"), false);
+ }
+
+ private void handleJVMStablityOnFailedCreate(Throwable throwable, boolean shouldKillJVM)
+ {
+ KillerForTests killerForTests = new KillerForTests();
+ JVMStabilityInspector.Killer originalKiller = JVMStabilityInspector.replaceKiller(killerForTests);
+
+ try
+ {
+ TestingIndex.shouldFailCreate = true;
+ TestingIndex.failedCreateThrowable = throwable;
+
+ createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY (a, b))");
+ String indexName = createIndex(String.format("CREATE CUSTOM INDEX ON %%s(c) USING '%s'", TestingIndex.class.getName()));
+ tryRebuild(indexName, true);
+ fail("Should have failed!");
+ }
+ catch (Throwable t)
+ {
+ assertEquals(shouldKillJVM, killerForTests.wasKilled());
+ }
+ finally
+ {
+ JVMStabilityInspector.replaceKiller(originalKiller);
+ TestingIndex.shouldFailCreate = false;
+ TestingIndex.failedCreateThrowable = null;
+ }
+ }
+
+ @Test
+ public void handleJVMStablityOnFailedRebuild() throws Throwable
+ {
+ handleJVMStablityOnFailedRebuild(new SocketException("Should not fail"), false);
+ handleJVMStablityOnFailedRebuild(new FileNotFoundException("Should not fail"), false);
+ handleJVMStablityOnFailedRebuild(new SocketException("Too many open files"), true);
+ handleJVMStablityOnFailedRebuild(new FileNotFoundException("Too many open files"), true);
+ handleJVMStablityOnFailedRebuild(new RuntimeException("Should not fail"), false);
+ }
+
+ private void handleJVMStablityOnFailedRebuild(Throwable throwable, boolean shouldKillJVM) throws Throwable
+ {
+ String tableName = createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY (a, b))");
+ String indexName = createIndex(String.format("CREATE CUSTOM INDEX ON %%s(c) USING '%s'", TestingIndex.class.getName()));
+ waitForIndex(KEYSPACE, tableName, indexName);
+
+ KillerForTests killerForTests = new KillerForTests();
+ JVMStabilityInspector.Killer originalKiller = JVMStabilityInspector.replaceKiller(killerForTests);
+
+ try
+ {
+ TestingIndex.shouldFailBuild = true;
+ TestingIndex.failedBuildTrowable = throwable;
+
+ getCurrentColumnFamilyStore().indexManager.rebuildIndexesBlocking(Collections.singleton(indexName));
+ fail("Should have failed!");
+ }
+ catch (Throwable t)
+ {
+ assertEquals(shouldKillJVM, killerForTests.wasKilled());
+ }
+ finally
+ {
+ JVMStabilityInspector.replaceKiller(originalKiller);
+ TestingIndex.shouldFailBuild = false;
+ TestingIndex.failedBuildTrowable = null;
+ }
+ }
+
+ private static void assertMarkedAsBuilt(String indexName)
+ {
+ List<String> indexes = SystemKeyspace.getBuiltIndexes(KEYSPACE, Collections.singleton(indexName));
+ assertEquals(1, indexes.size());
+ assertEquals(indexName, indexes.get(0));
+ }
+
+ private static void assertNotMarkedAsBuilt(String indexName)
+ {
+ List<String> indexes = SystemKeyspace.getBuiltIndexes(KEYSPACE, Collections.singleton(indexName));
+ assertTrue(indexes.isEmpty());
+ }
+
+ private boolean tryRebuild(String indexName, boolean wait) throws Throwable
+ {
+ ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
+ boolean done = false;
+ do
+ {
+ try
+ {
+ cfs.indexManager.rebuildIndexesBlocking(Collections.singleton(indexName));
+ done = true;
+ }
+ catch (IllegalStateException e)
+ {
+ assertTrue(e.getMessage().contains("currently in progress"));
+ }
+ Thread.sleep(500);
+ }
+ while (!done && wait);
+
+ return done;
+ }
+
+ private boolean isQueryable(String indexName)
+ {
+ SecondaryIndexManager manager = getCurrentColumnFamilyStore().indexManager;
+ Index index = manager.getIndexByName(indexName);
+ return manager.isIndexQueryable(index);
+ }
+
+ private boolean isWritable(String indexName)
+ {
+ SecondaryIndexManager manager = getCurrentColumnFamilyStore().indexManager;
+ Index index = manager.getIndexByName(indexName);
+ return manager.isIndexWritable(index);
+ }
+
+ public static class TestingIndex extends StubIndex
+ {
+ private static volatile CountDownLatch createLatch;
+ private static volatile CountDownLatch buildLatch;
+ private static volatile CountDownLatch createWaitLatch;
+ private static volatile CountDownLatch buildWaitLatch;
+ static volatile boolean shouldBlockCreate = false;
+ static volatile boolean shouldBlockBuild = false;
+ static volatile boolean shouldFailCreate = false;
+ static volatile boolean shouldFailBuild = false;
+ static volatile Throwable failedCreateThrowable;
+ static volatile Throwable failedBuildTrowable;
+
+ @SuppressWarnings("WeakerAccess")
+ public TestingIndex(ColumnFamilyStore baseCfs, IndexMetadata metadata)
+ {
+ super(baseCfs, metadata);
+ }
+
+ static void blockCreate()
+ {
+ shouldBlockCreate = true;
+ createLatch = new CountDownLatch(1);
+ createWaitLatch = new CountDownLatch(1);
+ }
+
+ static void blockBuild()
+ {
+ shouldBlockBuild = true;
+ buildLatch = new CountDownLatch(1);
+ buildWaitLatch = new CountDownLatch(1);
+ }
+
+ static void unblockCreate()
+ {
+ createLatch.countDown();
+ }
+
+ static void unblockBuild()
+ {
+ buildLatch.countDown();
+ }
+
+ static void waitBlockedOnCreate() throws InterruptedException
+ {
+ createWaitLatch.await();
+ }
+
+ static void waitBlockedOnBuild() throws InterruptedException
+ {
+ buildWaitLatch.await();
+ }
+
+ static void clear()
+ {
+ reset(createLatch);
+ reset(createWaitLatch);
+ reset(buildLatch);
+ reset(buildWaitLatch);
+ createLatch = null;
+ createWaitLatch = null;
+ buildLatch = null;
+ buildWaitLatch = null;
+ shouldBlockCreate = false;
+ shouldBlockBuild = false;
+ shouldFailCreate = false;
+ shouldFailBuild = false;
+ failedCreateThrowable = null;
+ failedBuildTrowable = null;
+ }
+
+ private static void reset(CountDownLatch latch)
+ {
+ if (latch == null)
+ return;
+
+ while (0L < latch.getCount())
+ latch.countDown();
+ }
+
+ public Callable<?> getInitializationTask()
+ {
+ return () ->
+ {
+ if (shouldBlockCreate && createLatch != null)
+ {
+ createWaitLatch.countDown();
+ createLatch.await();
+ }
+
+ if (shouldFailCreate)
+ {
+ throw failedCreateThrowable == null
+ ? new IllegalStateException("Index is configured to fail.")
+ : new RuntimeException(failedCreateThrowable);
+ }
+
+ return null;
+ };
+ }
+
+ public IndexBuildingSupport getBuildTaskSupport()
+ {
+ return new CollatedViewIndexBuildingSupport()
+ {
+ public SecondaryIndexBuilder getIndexBuildTask(ColumnFamilyStore cfs, Set<Index> indexes, Collection<SSTableReader> sstables)
+ {
+ try
+ {
+ if (shouldBlockBuild && buildLatch != null)
+ {
+ buildWaitLatch.countDown();
+ buildLatch.await();
+ }
+ final SecondaryIndexBuilder builder = super.getIndexBuildTask(cfs, indexes, sstables);
+ return new SecondaryIndexBuilder()
+ {
+
+ @Override
+ public void build()
+ {
+ if (shouldFailBuild)
+ {
+ throw failedBuildTrowable == null
+ ? new IllegalStateException("Index is configured to fail.")
+ : new RuntimeException(failedBuildTrowable);
+ }
+ builder.build();
+ }
+
+ @Override
+ public CompactionInfo getCompactionInfo()
+ {
+ return builder.getCompactionInfo();
+ }
+ };
+ }
+ catch (InterruptedException ex)
+ {
+ throw new RuntimeException(ex);
+ }
+ }
+ };
+ }
+
+ public boolean shouldBuildBlocking()
+ {
+ return true;
+ }
+ }
+
+ /**
+ * <code>TestingIndex</code> that only supports reads when initial build or full rebuild has failed.
+ */
+ public static class ReadOnlyOnFailureIndex extends TestingIndex
+ {
+ public ReadOnlyOnFailureIndex(ColumnFamilyStore baseCfs, IndexMetadata indexDef)
+ {
+ super(baseCfs, indexDef);
+ }
+
+ @Override
+ public LoadType getSupportedLoadTypeOnFailure(boolean isInitialBuild)
+ {
+ return LoadType.READ;
+ }
+ }
+
+ /**
+ * <code>TestingIndex</code> that only supports writes when initial build or full rebuild has failed.
+ */
+ public static class WriteOnlyOnFailureIndex extends TestingIndex
+ {
+ public WriteOnlyOnFailureIndex(ColumnFamilyStore baseCfs, IndexMetadata indexDef)
+ {
+ super(baseCfs, indexDef);
+ }
+
+ @Override
+ public LoadType getSupportedLoadTypeOnFailure(boolean isInitialBuild)
+ {
+ return LoadType.WRITE;
+ }
+ }
+}
diff --git a/test/unit/org/apache/cassandra/index/StubIndex.java b/test/unit/org/apache/cassandra/index/StubIndex.java
index c80f0d9..02ccbff 100644
--- a/test/unit/org/apache/cassandra/index/StubIndex.java
+++ b/test/unit/org/apache/cassandra/index/StubIndex.java
@@ -23,7 +23,7 @@
import java.util.function.BiFunction;
import org.apache.cassandra.Util;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.cql3.Operator;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.filter.RowFilter;
@@ -36,7 +36,6 @@
import org.apache.cassandra.index.transactions.IndexTransaction;
import org.apache.cassandra.schema.IndexMetadata;
import org.apache.cassandra.utils.Pair;
-import org.apache.cassandra.utils.concurrent.OpOrder;
/**
* Basic custom index implementation for testing.
@@ -77,12 +76,12 @@
return false;
}
- public boolean dependsOn(ColumnDefinition column)
+ public boolean dependsOn(ColumnMetadata column)
{
return false;
}
- public boolean supportsExpression(ColumnDefinition column, Operator operator)
+ public boolean supportsExpression(ColumnMetadata column, Operator operator)
{
return operator == Operator.EQ;
}
@@ -98,9 +97,9 @@
}
public Indexer indexerFor(final DecoratedKey key,
- PartitionColumns columns,
+ RegularAndStaticColumns columns,
int nowInSec,
- OpOrder.Group opGroup,
+ WriteContext ctx,
IndexTransaction.Type transactionType)
{
return new Indexer()
@@ -161,7 +160,7 @@
return Optional.empty();
}
- public Collection<ColumnDefinition> getIndexedColumns()
+ public Collection<ColumnMetadata> getIndexedColumns()
{
return Collections.emptySet();
}
diff --git a/test/unit/org/apache/cassandra/index/internal/CassandraIndexTest.java b/test/unit/org/apache/cassandra/index/internal/CassandraIndexTest.java
index a5c1f60..cffbaaf 100644
--- a/test/unit/org/apache/cassandra/index/internal/CassandraIndexTest.java
+++ b/test/unit/org/apache/cassandra/index/internal/CassandraIndexTest.java
@@ -27,9 +27,6 @@
import com.google.common.collect.*;
import org.junit.Test;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
-import org.apache.cassandra.config.SchemaConstants;
import org.apache.cassandra.cql3.CQLTester;
import org.apache.cassandra.cql3.UntypedResultSet;
import org.apache.cassandra.cql3.restrictions.StatementRestrictions;
@@ -41,6 +38,9 @@
import org.apache.cassandra.db.rows.Unfiltered;
import org.apache.cassandra.db.rows.UnfilteredRowIterator;
import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.SchemaConstants;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FBUtilities;
@@ -353,21 +353,6 @@
}
@Test
- public void indexOnRegularColumnWithCompactStorage() throws Throwable
- {
- new TestScript().tableDefinition("CREATE TABLE %s (k int, v int, PRIMARY KEY (k)) WITH COMPACT STORAGE;")
- .target("v")
- .withFirstRow(row(0, 0))
- .withSecondRow(row(1,1))
- .missingIndexMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE)
- .firstQueryExpression("v=0")
- .secondQueryExpression("v=1")
- .updateExpression("SET v=2")
- .postUpdateQueryExpression("v=2")
- .run();
- }
-
- @Test
public void indexOnStaticColumn() throws Throwable
{
Object[] row1 = row("k0", "c0", "s0");
@@ -419,14 +404,6 @@
}
@Test
- public void testIndexOnCompactTable() throws Throwable
- {
- createTable("CREATE TABLE %s (k int, v int, PRIMARY KEY (k)) WITH COMPACT STORAGE;");
- assertInvalidMessage("Undefined column name value",
- "CREATE INDEX idx_value ON %s(value)");
- }
-
- @Test
public void indexOnClusteringColumnWithoutRegularColumns() throws Throwable
{
Object[] row1 = row("k0", "c0");
@@ -580,7 +557,7 @@
// check that there are no other rows in the built indexes table
rs = execute(selectBuiltIndexesQuery);
int sizeAfterBuild = rs.size();
- assertRowsIgnoringOrderAndExtra(rs, row(KEYSPACE, indexName));
+ assertRowsIgnoringOrderAndExtra(rs, row(KEYSPACE, indexName, null));
// rebuild the index and verify the built status table
getCurrentColumnFamilyStore().rebuildSecondaryIndex(indexName);
@@ -589,7 +566,7 @@
// check that there are no other rows in the built indexes table
rs = execute(selectBuiltIndexesQuery);
assertEquals(sizeAfterBuild, rs.size());
- assertRowsIgnoringOrderAndExtra(rs, row(KEYSPACE, indexName));
+ assertRowsIgnoringOrderAndExtra(rs, row(KEYSPACE, indexName, null));
// check that dropping the index removes it from the built indexes table
dropIndex("DROP INDEX %s." + indexName);
@@ -599,21 +576,22 @@
&& row.getString("index_name").equals(indexName)));
}
+
// this is slightly annoying, but we cannot read rows from the methods in Util as
- // ReadCommand#executeInternal uses metadata retrieved via the cfId, which the index
+ // ReadCommand#executeLocally uses metadata retrieved via the tableId, which the index
// CFS inherits from the base CFS. This has the 'wrong' partitioner (the index table
// uses LocalPartition, the base table a real one, so we cannot read from the index
- // table with executeInternal
+ // table with executeLocally
private void assertIndexRowTtl(ColumnFamilyStore indexCfs, int indexedValue, int ttl) throws Throwable
{
DecoratedKey indexKey = indexCfs.decorateKey(ByteBufferUtil.bytes(indexedValue));
- ClusteringIndexFilter filter = new ClusteringIndexSliceFilter(Slices.with(indexCfs.metadata.comparator,
+ ClusteringIndexFilter filter = new ClusteringIndexSliceFilter(Slices.with(indexCfs.metadata().comparator,
Slice.ALL),
false);
- SinglePartitionReadCommand command = SinglePartitionReadCommand.create(indexCfs.metadata,
+ SinglePartitionReadCommand command = SinglePartitionReadCommand.create(indexCfs.metadata(),
FBUtilities.nowInSeconds(),
indexKey,
- ColumnFilter.all(indexCfs.metadata),
+ ColumnFilter.all(indexCfs.metadata()),
filter);
try (ReadExecutionController executionController = command.executionController();
UnfilteredRowIterator iter = command.queryMemtableAndDisk(indexCfs, executionController))
@@ -711,7 +689,7 @@
if (updateExpression != null)
assertNotNull(postUpdateQueryExpression);
- // first, create the table as we need the CFMetaData to build the other cql statements
+ // first, create the table as we need the Tablemetadata to build the other cql statements
String tableName = createTable(tableDefinition);
indexName = String.format("index_%s_%d", tableName, indexCounter++);
@@ -814,7 +792,7 @@
private void assertPrimaryKeyColumnsOnly(UntypedResultSet resultSet, Object[] row)
{
assertFalse(resultSet.isEmpty());
- CFMetaData cfm = getCurrentColumnFamilyStore().metadata;
+ TableMetadata cfm = getCurrentColumnFamilyStore().metadata();
int columnCount = cfm.partitionKeyColumns().size();
if (cfm.isCompound())
columnCount += cfm.clusteringColumns().size();
@@ -824,14 +802,12 @@
private String getInsertCql()
{
- CFMetaData cfm = getCurrentColumnFamilyStore().metadata;
+ TableMetadata metadata = getCurrentColumnFamilyStore().metadata();
String columns = Joiner.on(", ")
- .join(Iterators.transform(cfm.allColumnsInSelectOrder(),
+ .join(Iterators.transform(metadata.allColumnsInSelectOrder(),
(column) -> column.name.toString()));
- String markers = Joiner.on(", ").join(Iterators.transform(cfm.allColumnsInSelectOrder(),
- (column) -> {
- return "?";
- }));
+ String markers = Joiner.on(", ").join(Iterators.transform(metadata.allColumnsInSelectOrder(),
+ (column) -> "?"));
return String.format("INSERT INTO %%s (%s) VALUES (%s)", columns, markers);
}
@@ -850,15 +826,15 @@
private String getDeletePartitionCql()
{
- CFMetaData cfm = getCurrentColumnFamilyStore().metadata;
+ TableMetadata cfm = getCurrentColumnFamilyStore().metadata();
return StreamSupport.stream(cfm.partitionKeyColumns().spliterator(), false)
.map(column -> column.name.toString() + "=?")
.collect(Collectors.joining(" AND ", "DELETE FROM %s WHERE ", ""));
}
- private Stream<ColumnDefinition> getPrimaryKeyColumns()
+ private Stream<ColumnMetadata> getPrimaryKeyColumns()
{
- CFMetaData cfm = getCurrentColumnFamilyStore().metadata;
+ TableMetadata cfm = getCurrentColumnFamilyStore().metadata();
if (cfm.isCompactTable())
return cfm.partitionKeyColumns().stream();
else
@@ -867,7 +843,7 @@
private Object[] getPrimaryKeyValues(Object[] row)
{
- CFMetaData cfm = getCurrentColumnFamilyStore().metadata;
+ TableMetadata cfm = getCurrentColumnFamilyStore().metadata();
if (cfm.isCompactTable())
return getPartitionKeyValues(row);
@@ -876,7 +852,7 @@
private Object[] getPartitionKeyValues(Object[] row)
{
- CFMetaData cfm = getCurrentColumnFamilyStore().metadata;
+ TableMetadata cfm = getCurrentColumnFamilyStore().metadata();
return copyValuesFromRow(row, cfm.partitionKeyColumns().size());
}
diff --git a/test/unit/org/apache/cassandra/index/internal/CustomCassandraIndex.java b/test/unit/org/apache/cassandra/index/internal/CustomCassandraIndex.java
index 1173801..04db7f6 100644
--- a/test/unit/org/apache/cassandra/index/internal/CustomCassandraIndex.java
+++ b/test/unit/org/apache/cassandra/index/internal/CustomCassandraIndex.java
@@ -28,14 +28,17 @@
import java.util.stream.Collectors;
import java.util.stream.StreamSupport;
+import com.google.common.collect.ImmutableSet;
+
import org.apache.cassandra.index.TargetParser;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.schema.TableMetadataRef;
import org.apache.cassandra.cql3.Operator;
-import org.apache.cassandra.cql3.statements.IndexTarget;
+import org.apache.cassandra.cql3.statements.schema.IndexTarget;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.compaction.CompactionManager;
import org.apache.cassandra.db.filter.RowFilter;
@@ -56,7 +59,6 @@
import org.apache.cassandra.schema.IndexMetadata;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.Pair;
-import org.apache.cassandra.utils.concurrent.OpOrder;
import org.apache.cassandra.utils.concurrent.Refs;
import static org.apache.cassandra.index.internal.CassandraIndex.getFunctions;
@@ -73,7 +75,7 @@
public final ColumnFamilyStore baseCfs;
protected IndexMetadata metadata;
protected ColumnFamilyStore indexCfs;
- protected ColumnDefinition indexedColumn;
+ protected ColumnMetadata indexedColumn;
protected CassandraIndexFunctions functions;
public CustomCassandraIndex(ColumnFamilyStore baseCfs, IndexMetadata indexDef)
@@ -88,19 +90,19 @@
* @param operator
* @return
*/
- protected boolean supportsOperator(ColumnDefinition indexedColumn, Operator operator)
+ protected boolean supportsOperator(ColumnMetadata indexedColumn, Operator operator)
{
return operator.equals(Operator.EQ);
}
- public ColumnDefinition getIndexedColumn()
+ public ColumnMetadata getIndexedColumn()
{
return indexedColumn;
}
public ClusteringComparator getIndexComparator()
{
- return indexCfs.metadata.comparator;
+ return indexCfs.metadata().comparator;
}
public ColumnFamilyStore getIndexCfs()
@@ -150,7 +152,6 @@
{
setMetadata(indexDef);
return () -> {
- indexCfs.metadata.reloadIndexMetadataProperties(baseCfs.metadata);
indexCfs.reload();
return null;
};
@@ -159,12 +160,12 @@
private void setMetadata(IndexMetadata indexDef)
{
metadata = indexDef;
- Pair<ColumnDefinition, IndexTarget.Type> target = TargetParser.parse(baseCfs.metadata, indexDef);
+ Pair<ColumnMetadata, IndexTarget.Type> target = TargetParser.parse(baseCfs.metadata(), indexDef);
functions = getFunctions(indexDef, target);
- CFMetaData cfm = indexCfsMetadata(baseCfs.metadata, indexDef);
+ TableMetadata cfm = indexCfsMetadata(baseCfs.metadata(), indexDef);
indexCfs = ColumnFamilyStore.createColumnFamilyStore(baseCfs.keyspace,
- cfm.cfName,
- cfm,
+ cfm.name,
+ TableMetadataRef.forOfflineTools(cfm),
baseCfs.getTracker().loadsstables);
indexedColumn = target.left;
}
@@ -182,12 +183,12 @@
return true;
}
- public boolean dependsOn(ColumnDefinition column)
+ public boolean dependsOn(ColumnMetadata column)
{
return column.equals(indexedColumn);
}
- public boolean supportsExpression(ColumnDefinition column, Operator operator)
+ public boolean supportsExpression(ColumnMetadata column, Operator operator)
{
return indexedColumn.name.equals(column.name)
&& supportsOperator(indexedColumn, operator);
@@ -205,7 +206,7 @@
public long getEstimatedResultRows()
{
- return indexCfs.getMeanColumns();
+ return indexCfs.getMeanEstimatedCellPerPartitionCount();
}
/**
@@ -285,9 +286,9 @@
}
public Indexer indexerFor(final DecoratedKey key,
- final PartitionColumns columns,
+ final RegularAndStaticColumns columns,
final int nowInSec,
- final OpOrder.Group opGroup,
+ final WriteContext ctx,
final IndexTransaction.Type transactionType)
{
if (!isPrimaryKeyIndex() && !columns.contains(indexedColumn))
@@ -377,7 +378,7 @@
clustering,
cell,
LivenessInfo.withExpirationTime(cell.timestamp(), cell.ttl(), cell.localDeletionTime()),
- opGroup);
+ ctx);
}
private void removeCells(Clustering clustering, Iterable<Cell> cells)
@@ -394,7 +395,7 @@
if (cell == null || !cell.isLive(nowInSec))
return;
- delete(key.getKey(), clustering, cell, opGroup, nowInSec);
+ delete(key.getKey(), clustering, cell, ctx, nowInSec);
}
private void indexPrimaryKey(final Clustering clustering,
@@ -402,10 +403,10 @@
final Row.Deletion deletion)
{
if (liveness.timestamp() != LivenessInfo.NO_TIMESTAMP)
- insert(key.getKey(), clustering, null, liveness, opGroup);
+ insert(key.getKey(), clustering, null, liveness, ctx);
if (!deletion.isLive())
- delete(key.getKey(), clustering, deletion.time(), opGroup);
+ delete(key.getKey(), clustering, deletion.time(), ctx);
}
private LivenessInfo getPrimaryKeyIndexLiveness(Row row)
@@ -435,14 +436,14 @@
* @param indexKey the partition key in the index table
* @param indexClustering the clustering in the index table
* @param deletion deletion timestamp etc
- * @param opGroup the operation under which to perform the deletion
+ * @param ctx the context under which to perform the deletion
*/
public void deleteStaleEntry(DecoratedKey indexKey,
Clustering indexClustering,
DeletionTime deletion,
- OpOrder.Group opGroup)
+ WriteContext ctx)
{
- doDelete(indexKey, indexClustering, deletion, opGroup);
+ doDelete(indexKey, indexClustering, deletion, ctx);
logger.debug("Removed index entry for stale value {}", indexKey);
}
@@ -453,14 +454,14 @@
Clustering clustering,
Cell cell,
LivenessInfo info,
- OpOrder.Group opGroup)
+ WriteContext ctx)
{
DecoratedKey valueKey = getIndexKeyFor(getIndexedValue(rowKey,
clustering,
cell));
Row row = BTreeRow.noCellLiveRow(buildIndexClustering(rowKey, clustering, cell), info);
PartitionUpdate upd = partitionUpdate(valueKey, row);
- indexCfs.apply(upd, UpdateTransaction.NO_OP, opGroup, null);
+ indexCfs.getWriteHandler().write(upd, ctx, UpdateTransaction.NO_OP);
logger.debug("Inserted entry into index for value {}", valueKey);
}
@@ -470,7 +471,7 @@
private void delete(ByteBuffer rowKey,
Clustering clustering,
Cell cell,
- OpOrder.Group opGroup,
+ WriteContext ctx,
int nowInSec)
{
DecoratedKey valueKey = getIndexKeyFor(getIndexedValue(rowKey,
@@ -479,7 +480,7 @@
doDelete(valueKey,
buildIndexClustering(rowKey, clustering, cell),
new DeletionTime(cell.timestamp(), nowInSec),
- opGroup);
+ ctx);
}
/**
@@ -488,7 +489,7 @@
private void delete(ByteBuffer rowKey,
Clustering clustering,
DeletionTime deletion,
- OpOrder.Group opGroup)
+ WriteContext ctx)
{
DecoratedKey valueKey = getIndexKeyFor(getIndexedValue(rowKey,
clustering,
@@ -496,17 +497,17 @@
doDelete(valueKey,
buildIndexClustering(rowKey, clustering, null),
deletion,
- opGroup);
+ ctx);
}
private void doDelete(DecoratedKey indexKey,
Clustering indexClustering,
DeletionTime deletion,
- OpOrder.Group opGroup)
+ WriteContext ctx)
{
Row row = BTreeRow.emptyDeletedRow(indexClustering, Row.Deletion.regular(deletion));
PartitionUpdate upd = partitionUpdate(indexKey, row);
- indexCfs.apply(upd, UpdateTransaction.NO_OP, opGroup, null);
+ indexCfs.getWriteHandler().write(upd, ctx, UpdateTransaction.NO_OP);
logger.debug("Removed index entry for value {}", indexKey);
}
@@ -553,8 +554,8 @@
"Cannot index value of size %d for index %s on %s.%s(%s) (maximum allowed size=%d)",
value.remaining(),
metadata.name,
- baseCfs.metadata.ksName,
- baseCfs.metadata.cfName,
+ baseCfs.metadata.keyspace,
+ baseCfs.metadata.name,
indexedColumn.name.toString(),
FBUtilities.MAX_UNSIGNED_SHORT));
}
@@ -586,15 +587,15 @@
private PartitionUpdate partitionUpdate(DecoratedKey valueKey, Row row)
{
- return PartitionUpdate.singleRowUpdate(indexCfs.metadata, valueKey, row);
+ return PartitionUpdate.singleRowUpdate(indexCfs.metadata(), valueKey, row);
}
private void invalidate()
{
// interrupt in-progress compactions
Collection<ColumnFamilyStore> cfss = Collections.singleton(indexCfs);
- CompactionManager.instance.interruptCompactionForCFs(cfss, true);
- CompactionManager.instance.waitForCessation(cfss);
+ CompactionManager.instance.interruptCompactionForCFs(cfss, (sstable) -> true, true);
+ CompactionManager.instance.waitForCessation(cfss, (sstable) -> true);
indexCfs.keyspace.writeOrder.awaitNewBarrier();
indexCfs.forceBlockingFlush();
indexCfs.readOrdering.awaitNewBarrier();
@@ -629,10 +630,9 @@
if (sstables.isEmpty())
{
logger.info("No SSTable data for {}.{} to build index {} from, marking empty index as built",
- baseCfs.metadata.ksName,
- baseCfs.metadata.cfName,
+ baseCfs.metadata.keyspace,
+ baseCfs.metadata.name,
metadata.name);
- baseCfs.indexManager.markIndexBuilt(metadata.name);
return;
}
@@ -642,11 +642,11 @@
SecondaryIndexBuilder builder = new CollatedViewIndexBuilder(baseCfs,
Collections.singleton(this),
- new ReducingKeyIterator(sstables));
+ new ReducingKeyIterator(sstables),
+ ImmutableSet.copyOf(sstables));
Future<?> future = CompactionManager.instance.submitIndexBuild(builder);
FBUtilities.waitOnFuture(future);
indexCfs.forceBlockingFlush();
- baseCfs.indexManager.markIndexBuilt(metadata.name);
}
logger.info("Index build of {} complete", metadata.name);
}
diff --git a/test/unit/org/apache/cassandra/index/sasi/SASIIndexTest.java b/test/unit/org/apache/cassandra/index/sasi/SASIIndexTest.java
index 7def47c..3508d76 100644
--- a/test/unit/org/apache/cassandra/index/sasi/SASIIndexTest.java
+++ b/test/unit/org/apache/cassandra/index/sasi/SASIIndexTest.java
@@ -26,7 +26,6 @@
import java.nio.file.Path;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.*;
-import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ThreadLocalRandom;
@@ -35,15 +34,17 @@
import java.util.stream.Collectors;
import org.apache.cassandra.SchemaLoader;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
-import org.apache.cassandra.config.Schema;
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.cql3.Operator;
+import org.apache.cassandra.cql3.QueryProcessor;
+import org.apache.cassandra.cql3.UntypedResultSet;
import org.apache.cassandra.index.Index;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.cql3.*;
import org.apache.cassandra.cql3.Term;
-import org.apache.cassandra.cql3.statements.IndexTarget;
-import org.apache.cassandra.cql3.statements.SelectStatement;
+import org.apache.cassandra.cql3.statements.schema.IndexTarget;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.filter.ColumnFilter;
import org.apache.cassandra.db.filter.DataLimits;
@@ -72,27 +73,23 @@
import org.apache.cassandra.io.sstable.IndexSummaryManager;
import org.apache.cassandra.io.sstable.SSTable;
import org.apache.cassandra.schema.IndexMetadata;
-import org.apache.cassandra.schema.KeyspaceMetadata;
import org.apache.cassandra.schema.KeyspaceParams;
-import org.apache.cassandra.schema.Tables;
import org.apache.cassandra.serializers.MarshalException;
import org.apache.cassandra.serializers.TypeSerializer;
-import org.apache.cassandra.service.ClientState;
-import org.apache.cassandra.service.MigrationManager;
-import org.apache.cassandra.service.QueryState;
-import org.apache.cassandra.thrift.CqlRow;
-import org.apache.cassandra.transport.messages.ResultMessage;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.Pair;
import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
import com.google.common.util.concurrent.Uninterruptibles;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.junit.*;
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+
public class SASIIndexTest
{
private static final IPartitioner PARTITIONER;
@@ -113,13 +110,13 @@
public static void loadSchema() throws ConfigurationException
{
SchemaLoader.loadSchema();
- MigrationManager.announceNewKeyspace(KeyspaceMetadata.create(KS_NAME,
- KeyspaceParams.simpleTransient(1),
- Tables.of(SchemaLoader.sasiCFMD(KS_NAME, CF_NAME),
- SchemaLoader.clusteringSASICFMD(KS_NAME, CLUSTERING_CF_NAME_1),
- SchemaLoader.clusteringSASICFMD(KS_NAME, CLUSTERING_CF_NAME_2, "location"),
- SchemaLoader.staticSASICFMD(KS_NAME, STATIC_CF_NAME),
- SchemaLoader.fullTextSearchSASICFMD(KS_NAME, FTS_CF_NAME))));
+ SchemaLoader.createKeyspace(KS_NAME,
+ KeyspaceParams.simpleTransient(1),
+ SchemaLoader.sasiCFMD(KS_NAME, CF_NAME),
+ SchemaLoader.clusteringSASICFMD(KS_NAME, CLUSTERING_CF_NAME_1),
+ SchemaLoader.clusteringSASICFMD(KS_NAME, CLUSTERING_CF_NAME_2, "location"),
+ SchemaLoader.staticSASICFMD(KS_NAME, STATIC_CF_NAME),
+ SchemaLoader.fullTextSearchSASICFMD(KS_NAME, FTS_CF_NAME));
}
@Before
@@ -781,33 +778,33 @@
{
ColumnFamilyStore store = Keyspace.open(KS_NAME).getColumnFamilyStore(CF_NAME);
- Mutation rm1 = new Mutation(KS_NAME, decoratedKey(AsciiType.instance.decompose("key1")));
- rm1.add(PartitionUpdate.singleRowUpdate(store.metadata,
+ Mutation.PartitionUpdateCollector rm1 = new Mutation.PartitionUpdateCollector(KS_NAME, decoratedKey(AsciiType.instance.decompose("key1")));
+ rm1.add(PartitionUpdate.singleRowUpdate(store.metadata(),
rm1.key(),
- buildRow(buildCell(store.metadata,
+ buildRow(buildCell(store.metadata(),
UTF8Type.instance.decompose("/data/output/id"),
AsciiType.instance.decompose("jason"),
System.currentTimeMillis()))));
- Mutation rm2 = new Mutation(KS_NAME, decoratedKey(AsciiType.instance.decompose("key2")));
- rm2.add(PartitionUpdate.singleRowUpdate(store.metadata,
+ Mutation.PartitionUpdateCollector rm2 = new Mutation.PartitionUpdateCollector(KS_NAME, decoratedKey(AsciiType.instance.decompose("key2")));
+ rm2.add(PartitionUpdate.singleRowUpdate(store.metadata(),
rm2.key(),
- buildRow(buildCell(store.metadata,
+ buildRow(buildCell(store.metadata(),
UTF8Type.instance.decompose("/data/output/id"),
AsciiType.instance.decompose("pavel"),
System.currentTimeMillis()))));
- Mutation rm3 = new Mutation(KS_NAME, decoratedKey(AsciiType.instance.decompose("key3")));
- rm3.add(PartitionUpdate.singleRowUpdate(store.metadata,
+ Mutation.PartitionUpdateCollector rm3 = new Mutation.PartitionUpdateCollector(KS_NAME, decoratedKey(AsciiType.instance.decompose("key3")));
+ rm3.add(PartitionUpdate.singleRowUpdate(store.metadata(),
rm3.key(),
- buildRow(buildCell(store.metadata,
+ buildRow(buildCell(store.metadata(),
UTF8Type.instance.decompose("/data/output/id"),
AsciiType.instance.decompose("Aleksey"),
System.currentTimeMillis()))));
- rm1.apply();
- rm2.apply();
- rm3.apply();
+ rm1.build().apply();
+ rm2.build().apply();
+ rm3.build().apply();
if (forceFlush)
store.forceBlockingFlush();
@@ -833,14 +830,14 @@
Assert.assertTrue(rows.toString(), rows.isEmpty());
// now let's trigger index rebuild and check if we got the data back
- store.indexManager.buildIndexBlocking(store.indexManager.getIndexByName("data_output_id"));
+ store.indexManager.rebuildIndexesBlocking(Sets.newHashSet(store.name + "_data_output_id"));
rows = getIndexed(store, 10, buildExpression(dataOutputId, Operator.LIKE_CONTAINS, UTF8Type.instance.decompose("a")));
Assert.assertTrue(rows.toString(), Arrays.equals(new String[] { "key1", "key2" }, rows.toArray(new String[rows.size()])));
// also let's try to build an index for column which has no data to make sure that doesn't fail
- store.indexManager.buildIndexBlocking(store.indexManager.getIndexByName("first_name"));
- store.indexManager.buildIndexBlocking(store.indexManager.getIndexByName("data_output_id"));
+ store.indexManager.rebuildIndexesBlocking(Sets.newHashSet(store.name + "_first_name"));
+ store.indexManager.rebuildIndexesBlocking(Sets.newHashSet(store.name + "_data_output_id"));
rows = getIndexed(store, 10, buildExpression(dataOutputId, Operator.LIKE_CONTAINS, UTF8Type.instance.decompose("a")));
Assert.assertTrue(rows.toString(), Arrays.equals(new String[] { "key1", "key2" }, rows.toArray(new String[rows.size()])));
@@ -955,7 +952,7 @@
loadData(part5, true);
- int minIndexInterval = store.metadata.params.minIndexInterval;
+ int minIndexInterval = store.metadata().params.minIndexInterval;
try
{
redistributeSummaries(10, store, firstName, minIndexInterval * 2);
@@ -964,13 +961,13 @@
redistributeSummaries(10, store, firstName, minIndexInterval * 16);
} finally
{
- store.metadata.minIndexInterval(minIndexInterval);
+ setMinIndexInterval(minIndexInterval);
}
}
private void redistributeSummaries(int expected, ColumnFamilyStore store, ByteBuffer firstName, int minIndexInterval) throws IOException
{
- store.metadata.minIndexInterval(minIndexInterval);
+ setMinIndexInterval(minIndexInterval);
IndexSummaryManager.instance.redistributeSummaries();
store.forceBlockingFlush();
@@ -978,6 +975,10 @@
Assert.assertEquals(rows.toString(), expected, rows.size());
}
+ private void setMinIndexInterval(int minIndexInterval) {
+ QueryProcessor.executeOnceInternal(String.format("ALTER TABLE %s.%s WITH min_index_interval = %d;", KS_NAME, CF_NAME, minIndexInterval));
+ }
+
@Test
public void testTruncate()
{
@@ -1098,25 +1099,30 @@
int previousCount = 0;
- do
+ try
{
- // this loop figures out if number of search results monotonically increasing
- // to make sure that concurrent updates don't interfere with reads, uses first_name and age
- // indexes to test correctness of both Trie and SkipList ColumnIndex implementations.
+ do
+ {
+ // this loop figures out if number of search results monotonically increasing
+ // to make sure that concurrent updates don't interfere with reads, uses first_name and age
+ // indexes to test correctness of both Trie and SkipList ColumnIndex implementations.
+ Set<DecoratedKey> rows = getPaged(store, 100, buildExpression(firstName, Operator.LIKE_CONTAINS, UTF8Type.instance.decompose("a")),
+ buildExpression(age, Operator.EQ, Int32Type.instance.decompose(26)));
+
+ Assert.assertTrue(previousCount <= rows.size());
+ previousCount = rows.size();
+ }
+ while (updates.get() < writeCount);
+
+ // to make sure that after all of the writes are done we can read all "count" worth of rows
Set<DecoratedKey> rows = getPaged(store, 100, buildExpression(firstName, Operator.LIKE_CONTAINS, UTF8Type.instance.decompose("a")),
- buildExpression(age, Operator.EQ, Int32Type.instance.decompose(26)));
-
- Assert.assertTrue(previousCount <= rows.size());
- previousCount = rows.size();
+ buildExpression(age, Operator.EQ, Int32Type.instance.decompose(26)));
+ Assert.assertEquals(writeCount, rows.size());
+ } finally {
+ scheduler.shutdownNow();
+ Assert.assertTrue(scheduler.awaitTermination(1, TimeUnit.MINUTES));
}
- while (updates.get() < writeCount);
-
- // to make sure that after all of the right are done we can read all "count" worth of rows
- Set<DecoratedKey> rows = getPaged(store, 100, buildExpression(firstName, Operator.LIKE_CONTAINS, UTF8Type.instance.decompose("a")),
- buildExpression(age, Operator.EQ, Int32Type.instance.decompose(26)));
-
- Assert.assertEquals(writeCount, rows.size());
}
@Test
@@ -1180,20 +1186,21 @@
final ByteBuffer firstName = UTF8Type.instance.decompose("first_name");
final ByteBuffer age = UTF8Type.instance.decompose("age");
- Mutation rm = new Mutation(KS_NAME, decoratedKey(AsciiType.instance.decompose("key1")));
+ Mutation.PartitionUpdateCollector rm = new Mutation.PartitionUpdateCollector(KS_NAME, decoratedKey(AsciiType.instance.decompose("key1")));
update(rm, new ArrayList<Cell>()
{{
add(buildCell(age, LongType.instance.decompose(26L), System.currentTimeMillis()));
add(buildCell(firstName, AsciiType.instance.decompose("pavel"), System.currentTimeMillis()));
}});
- rm.apply();
+ rm.build().apply();
- try {
- store.forceBlockingFlush();
- Assert.fail("It was possible to insert data of wrong type into a column!");
- } catch (final Throwable ex) {
- Assert.assertTrue(ex.getMessage().endsWith("Expected exactly 4 bytes, but was 8"));
- }
+ store.forceBlockingFlush();
+
+ Set<String> rows = getIndexed(store, 10, buildExpression(firstName, Operator.EQ, UTF8Type.instance.decompose("a")),
+ buildExpression(age, Operator.GTE, Int32Type.instance.decompose(26)));
+
+ // index is expected to have 0 results because age value was of wrong type
+ Assert.assertEquals(0, rows.size());
}
@@ -1211,25 +1218,25 @@
final ByteBuffer comment = UTF8Type.instance.decompose("comment");
- Mutation rm = new Mutation(KS_NAME, decoratedKey("key1"));
+ Mutation.PartitionUpdateCollector rm = new Mutation.PartitionUpdateCollector(KS_NAME, decoratedKey("key1"));
update(rm, comment, UTF8Type.instance.decompose("ⓈⓅⒺⒸⒾⒶⓁ ⒞⒣⒜⒭⒮ and normal ones"), System.currentTimeMillis());
- rm.apply();
+ rm.build().apply();
- rm = new Mutation(KS_NAME, decoratedKey("key2"));
+ rm = new Mutation.PartitionUpdateCollector(KS_NAME, decoratedKey("key2"));
update(rm, comment, UTF8Type.instance.decompose("龍馭鬱"), System.currentTimeMillis());
- rm.apply();
+ rm.build().apply();
- rm = new Mutation(KS_NAME, decoratedKey("key3"));
+ rm = new Mutation.PartitionUpdateCollector(KS_NAME, decoratedKey("key3"));
update(rm, comment, UTF8Type.instance.decompose("インディアナ"), System.currentTimeMillis());
- rm.apply();
+ rm.build().apply();
- rm = new Mutation(KS_NAME, decoratedKey("key4"));
+ rm = new Mutation.PartitionUpdateCollector(KS_NAME, decoratedKey("key4"));
update(rm, comment, UTF8Type.instance.decompose("レストラン"), System.currentTimeMillis());
- rm.apply();
+ rm.build().apply();
- rm = new Mutation(KS_NAME, decoratedKey("key5"));
+ rm = new Mutation.PartitionUpdateCollector(KS_NAME, decoratedKey("key5"));
update(rm, comment, UTF8Type.instance.decompose("ベンジャミン ウエスト"), System.currentTimeMillis());
- rm.apply();
+ rm.build().apply();
if (forceFlush)
store.forceBlockingFlush();
@@ -1287,21 +1294,21 @@
final ByteBuffer comment = UTF8Type.instance.decompose("comment_suffix_split");
- Mutation rm = new Mutation(KS_NAME, decoratedKey("key1"));
+ Mutation.PartitionUpdateCollector rm = new Mutation.PartitionUpdateCollector(KS_NAME, decoratedKey("key1"));
update(rm, comment, UTF8Type.instance.decompose("龍馭鬱"), System.currentTimeMillis());
- rm.apply();
+ rm.build().apply();
- rm = new Mutation(KS_NAME, decoratedKey("key2"));
+ rm = new Mutation.PartitionUpdateCollector(KS_NAME, decoratedKey("key2"));
update(rm, comment, UTF8Type.instance.decompose("インディアナ"), System.currentTimeMillis());
- rm.apply();
+ rm.build().apply();
- rm = new Mutation(KS_NAME, decoratedKey("key3"));
+ rm = new Mutation.PartitionUpdateCollector(KS_NAME, decoratedKey("key3"));
update(rm, comment, UTF8Type.instance.decompose("レストラン"), System.currentTimeMillis());
- rm.apply();
+ rm.build().apply();
- rm = new Mutation(KS_NAME, decoratedKey("key4"));
+ rm = new Mutation.PartitionUpdateCollector(KS_NAME, decoratedKey("key4"));
update(rm, comment, UTF8Type.instance.decompose("ベンジャミン ウエスト"), System.currentTimeMillis());
- rm.apply();
+ rm.build().apply();
if (forceFlush)
store.forceBlockingFlush();
@@ -1356,9 +1363,9 @@
final ByteBuffer bigValue = UTF8Type.instance.decompose(new String(randomBytes));
- Mutation rm = new Mutation(KS_NAME, decoratedKey("key1"));
+ Mutation.PartitionUpdateCollector rm = new Mutation.PartitionUpdateCollector(KS_NAME, decoratedKey("key1"));
update(rm, comment, bigValue, System.currentTimeMillis());
- rm.apply();
+ rm.build().apply();
Set<String> rows;
@@ -1388,16 +1395,15 @@
ColumnFamilyStore store = loadData(data1, true);
RowFilter filter = RowFilter.create();
- filter.add(store.metadata.getColumnDefinition(firstName), Operator.LIKE_CONTAINS, AsciiType.instance.fromString("a"));
+ filter.add(store.metadata().getColumn(firstName), Operator.LIKE_CONTAINS, AsciiType.instance.fromString("a"));
ReadCommand command =
- PartitionRangeReadCommand.create(false,
- store.metadata,
+ PartitionRangeReadCommand.create(store.metadata(),
FBUtilities.nowInSeconds(),
- ColumnFilter.all(store.metadata),
+ ColumnFilter.all(store.metadata()),
filter,
DataLimits.NONE,
- DataRange.allData(store.metadata.partitioner));
+ DataRange.allData(store.metadata().partitioner));
try
{
new QueryPlan(store, command, 0).execute(ReadExecutionController.empty());
@@ -1417,7 +1423,7 @@
try (ReadExecutionController controller = command.executionController())
{
- Set<String> rows = getKeys(new QueryPlan(store, command, DatabaseDescriptor.getRangeRpcTimeout()).execute(controller));
+ Set<String> rows = getKeys(new QueryPlan(store, command, DatabaseDescriptor.getRangeRpcTimeout(MILLISECONDS)).execute(controller));
Assert.assertTrue(rows.toString(), Arrays.equals(new String[] { "key1", "key2", "key3", "key4" }, rows.toArray(new String[rows.size()])));
}
}
@@ -1437,37 +1443,37 @@
final ByteBuffer fullName = UTF8Type.instance.decompose("/output/full-name/");
- Mutation rm = new Mutation(KS_NAME, decoratedKey("key1"));
+ Mutation.PartitionUpdateCollector rm = new Mutation.PartitionUpdateCollector(KS_NAME, decoratedKey("key1"));
update(rm, fullName, UTF8Type.instance.decompose("美加 八田"), System.currentTimeMillis());
- rm.apply();
+ rm.build().apply();
- rm = new Mutation(KS_NAME, decoratedKey("key2"));
+ rm = new Mutation.PartitionUpdateCollector(KS_NAME, decoratedKey("key2"));
update(rm, fullName, UTF8Type.instance.decompose("仁美 瀧澤"), System.currentTimeMillis());
- rm.apply();
+ rm.build().apply();
- rm = new Mutation(KS_NAME, decoratedKey("key3"));
+ rm = new Mutation.PartitionUpdateCollector(KS_NAME, decoratedKey("key3"));
update(rm, fullName, UTF8Type.instance.decompose("晃宏 高須"), System.currentTimeMillis());
- rm.apply();
+ rm.build().apply();
- rm = new Mutation(KS_NAME, decoratedKey("key4"));
+ rm = new Mutation.PartitionUpdateCollector(KS_NAME, decoratedKey("key4"));
update(rm, fullName, UTF8Type.instance.decompose("弘孝 大竹"), System.currentTimeMillis());
- rm.apply();
+ rm.build().apply();
- rm = new Mutation(KS_NAME, decoratedKey("key5"));
+ rm = new Mutation.PartitionUpdateCollector(KS_NAME, decoratedKey("key5"));
update(rm, fullName, UTF8Type.instance.decompose("満枝 榎本"), System.currentTimeMillis());
- rm.apply();
+ rm.build().apply();
- rm = new Mutation(KS_NAME, decoratedKey("key6"));
+ rm = new Mutation.PartitionUpdateCollector(KS_NAME, decoratedKey("key6"));
update(rm, fullName, UTF8Type.instance.decompose("飛鳥 上原"), System.currentTimeMillis());
- rm.apply();
+ rm.build().apply();
- rm = new Mutation(KS_NAME, decoratedKey("key7"));
+ rm = new Mutation.PartitionUpdateCollector(KS_NAME, decoratedKey("key7"));
update(rm, fullName, UTF8Type.instance.decompose("大輝 鎌田"), System.currentTimeMillis());
- rm.apply();
+ rm.build().apply();
- rm = new Mutation(KS_NAME, decoratedKey("key8"));
+ rm = new Mutation.PartitionUpdateCollector(KS_NAME, decoratedKey("key8"));
update(rm, fullName, UTF8Type.instance.decompose("利久 寺地"), System.currentTimeMillis());
- rm.apply();
+ rm.build().apply();
store.forceBlockingFlush();
@@ -1493,17 +1499,17 @@
final ByteBuffer comment = UTF8Type.instance.decompose("address");
- Mutation rm = new Mutation(KS_NAME, decoratedKey("key1"));
+ Mutation.PartitionUpdateCollector rm = new Mutation.PartitionUpdateCollector(KS_NAME, decoratedKey("key1"));
update(rm, comment, UTF8Type.instance.decompose("577 Rogahn Valleys Apt. 178"), System.currentTimeMillis());
- rm.apply();
+ rm.build().apply();
- rm = new Mutation(KS_NAME, decoratedKey("key2"));
+ rm = new Mutation.PartitionUpdateCollector(KS_NAME, decoratedKey("key2"));
update(rm, comment, UTF8Type.instance.decompose("89809 Beverly Course Suite 089"), System.currentTimeMillis());
- rm.apply();
+ rm.build().apply();
- rm = new Mutation(KS_NAME, decoratedKey("key3"));
+ rm = new Mutation.PartitionUpdateCollector(KS_NAME, decoratedKey("key3"));
update(rm, comment, UTF8Type.instance.decompose("165 clydie oval apt. 399"), System.currentTimeMillis());
- rm.apply();
+ rm.build().apply();
if (forceFlush)
store.forceBlockingFlush();
@@ -1568,42 +1574,42 @@
final ByteBuffer name = UTF8Type.instance.decompose("first_name_prefix");
- Mutation rm;
+ Mutation.PartitionUpdateCollector rm;
- rm = new Mutation(KS_NAME, decoratedKey("key1"));
+ rm = new Mutation.PartitionUpdateCollector(KS_NAME, decoratedKey("key1"));
update(rm, name, UTF8Type.instance.decompose("Pavel"), System.currentTimeMillis());
- rm.apply();
+ rm.build().apply();
- rm = new Mutation(KS_NAME, decoratedKey("key2"));
+ rm = new Mutation.PartitionUpdateCollector(KS_NAME, decoratedKey("key2"));
update(rm, name, UTF8Type.instance.decompose("Jordan"), System.currentTimeMillis());
- rm.apply();
+ rm.build().apply();
- rm = new Mutation(KS_NAME, decoratedKey("key3"));
+ rm = new Mutation.PartitionUpdateCollector(KS_NAME, decoratedKey("key3"));
update(rm, name, UTF8Type.instance.decompose("Mikhail"), System.currentTimeMillis());
- rm.apply();
+ rm.build().apply();
- rm = new Mutation(KS_NAME, decoratedKey("key4"));
+ rm = new Mutation.PartitionUpdateCollector(KS_NAME, decoratedKey("key4"));
update(rm, name, UTF8Type.instance.decompose("Michael"), System.currentTimeMillis());
- rm.apply();
+ rm.build().apply();
- rm = new Mutation(KS_NAME, decoratedKey("key5"));
+ rm = new Mutation.PartitionUpdateCollector(KS_NAME, decoratedKey("key5"));
update(rm, name, UTF8Type.instance.decompose("Johnny"), System.currentTimeMillis());
- rm.apply();
+ rm.build().apply();
// first flush would make interval for name - 'johnny' -> 'pavel'
store.forceBlockingFlush();
- rm = new Mutation(KS_NAME, decoratedKey("key6"));
+ rm = new Mutation.PartitionUpdateCollector(KS_NAME, decoratedKey("key6"));
update(rm, name, UTF8Type.instance.decompose("Jason"), System.currentTimeMillis());
- rm.apply();
+ rm.build().apply();
- rm = new Mutation(KS_NAME, decoratedKey("key7"));
+ rm = new Mutation.PartitionUpdateCollector(KS_NAME, decoratedKey("key7"));
update(rm, name, UTF8Type.instance.decompose("Vijay"), System.currentTimeMillis());
- rm.apply();
+ rm.build().apply();
- rm = new Mutation(KS_NAME, decoratedKey("key8")); // this name is going to be tokenized
+ rm = new Mutation.PartitionUpdateCollector(KS_NAME, decoratedKey("key8")); // this name is going to be tokenized
update(rm, name, UTF8Type.instance.decompose("Jean-Claude"), System.currentTimeMillis());
- rm.apply();
+ rm.build().apply();
// this flush is going to produce range - 'jason' -> 'vijay'
store.forceBlockingFlush();
@@ -1683,7 +1689,7 @@
};
// first let's check that we get 'false' for 'isLiteral' if we don't set the option with special comparator
- ColumnDefinition columnA = ColumnDefinition.regularDef(KS_NAME, CF_NAME, "special-A", stringType);
+ ColumnMetadata columnA = ColumnMetadata.regularColumn(KS_NAME, CF_NAME, "special-A", stringType);
ColumnIndex indexA = new ColumnIndex(UTF8Type.instance, columnA, IndexMetadata.fromSchemaMetadata("special-index-A", IndexMetadata.Kind.CUSTOM, new HashMap<String, String>()
{{
@@ -1694,7 +1700,7 @@
Assert.assertEquals(false, indexA.isLiteral());
// now let's double-check that we do get 'true' when we set it
- ColumnDefinition columnB = ColumnDefinition.regularDef(KS_NAME, CF_NAME, "special-B", stringType);
+ ColumnMetadata columnB = ColumnMetadata.regularColumn(KS_NAME, CF_NAME, "special-B", stringType);
ColumnIndex indexB = new ColumnIndex(UTF8Type.instance, columnB, IndexMetadata.fromSchemaMetadata("special-index-B", IndexMetadata.Kind.CUSTOM, new HashMap<String, String>()
{{
@@ -1706,7 +1712,7 @@
Assert.assertEquals(true, indexB.isLiteral());
// and finally we should also get a 'true' if it's built-in UTF-8/ASCII comparator
- ColumnDefinition columnC = ColumnDefinition.regularDef(KS_NAME, CF_NAME, "special-C", UTF8Type.instance);
+ ColumnMetadata columnC = ColumnMetadata.regularColumn(KS_NAME, CF_NAME, "special-C", UTF8Type.instance);
ColumnIndex indexC = new ColumnIndex(UTF8Type.instance, columnC, IndexMetadata.fromSchemaMetadata("special-index-C", IndexMetadata.Kind.CUSTOM, new HashMap<String, String>()
{{
@@ -1716,7 +1722,7 @@
Assert.assertEquals(true, indexC.isIndexed());
Assert.assertEquals(true, indexC.isLiteral());
- ColumnDefinition columnD = ColumnDefinition.regularDef(KS_NAME, CF_NAME, "special-D", AsciiType.instance);
+ ColumnMetadata columnD = ColumnMetadata.regularColumn(KS_NAME, CF_NAME, "special-D", AsciiType.instance);
ColumnIndex indexD = new ColumnIndex(UTF8Type.instance, columnD, IndexMetadata.fromSchemaMetadata("special-index-D", IndexMetadata.Kind.CUSTOM, new HashMap<String, String>()
{{
@@ -1727,7 +1733,7 @@
Assert.assertEquals(true, indexD.isLiteral());
// and option should supersedes the comparator type
- ColumnDefinition columnE = ColumnDefinition.regularDef(KS_NAME, CF_NAME, "special-E", UTF8Type.instance);
+ ColumnMetadata columnE = ColumnMetadata.regularColumn(KS_NAME, CF_NAME, "special-E", UTF8Type.instance);
ColumnIndex indexE = new ColumnIndex(UTF8Type.instance, columnE, IndexMetadata.fromSchemaMetadata("special-index-E", IndexMetadata.Kind.CUSTOM, new HashMap<String, String>()
{{
@@ -1739,10 +1745,10 @@
Assert.assertEquals(false, indexE.isLiteral());
// test frozen-collection
- ColumnDefinition columnF = ColumnDefinition.regularDef(KS_NAME,
- CF_NAME,
- "special-F",
- ListType.getInstance(UTF8Type.instance, false));
+ ColumnMetadata columnF = ColumnMetadata.regularColumn(KS_NAME,
+ CF_NAME,
+ "special-F",
+ ListType.getInstance(UTF8Type.instance, false));
ColumnIndex indexF = new ColumnIndex(UTF8Type.instance, columnF, IndexMetadata.fromSchemaMetadata("special-index-F", IndexMetadata.Kind.CUSTOM, new HashMap<String, String>()
{{
@@ -1904,18 +1910,18 @@
UntypedResultSet.Row row1 = iterator.next();
Assert.assertEquals(20160401L, row1.getLong("date"));
- Assert.assertEquals(24.46, row1.getDouble("value"));
+ Assert.assertEquals(24.46, row1.getDouble("value"), 0.0);
Assert.assertEquals(2, row1.getInt("variance"));
UntypedResultSet.Row row2 = iterator.next();
Assert.assertEquals(20160402L, row2.getLong("date"));
- Assert.assertEquals(25.62, row2.getDouble("value"));
+ Assert.assertEquals(25.62, row2.getDouble("value"), 0.0);
Assert.assertEquals(5, row2.getInt("variance"));
UntypedResultSet.Row row3 = iterator.next();
Assert.assertEquals(20160403L, row3.getLong("date"));
- Assert.assertEquals(24.96, row3.getDouble("value"));
+ Assert.assertEquals(24.96, row3.getDouble("value"), 0.0);
Assert.assertEquals(4, row3.getInt("variance"));
@@ -1927,7 +1933,7 @@
row1 = results.one();
Assert.assertEquals(20160402L, row1.getLong("date"));
- Assert.assertEquals(1.04, row1.getDouble("value"));
+ Assert.assertEquals(1.04, row1.getDouble("value"), 0.0);
Assert.assertEquals(7, row1.getInt("variance"));
// Only non statc columns filtering
@@ -1940,26 +1946,26 @@
row1 = iterator.next();
Assert.assertEquals("TEMPERATURE", row1.getString("sensor_type"));
Assert.assertEquals(20160401L, row1.getLong("date"));
- Assert.assertEquals(24.46, row1.getDouble("value"));
+ Assert.assertEquals(24.46, row1.getDouble("value"), 0.0);
Assert.assertEquals(2, row1.getInt("variance"));
row2 = iterator.next();
Assert.assertEquals("TEMPERATURE", row2.getString("sensor_type"));
Assert.assertEquals(20160402L, row2.getLong("date"));
- Assert.assertEquals(25.62, row2.getDouble("value"));
+ Assert.assertEquals(25.62, row2.getDouble("value"), 0.0);
Assert.assertEquals(5, row2.getInt("variance"));
row3 = iterator.next();
Assert.assertEquals("TEMPERATURE", row3.getString("sensor_type"));
Assert.assertEquals(20160403L, row3.getLong("date"));
- Assert.assertEquals(24.96, row3.getDouble("value"));
+ Assert.assertEquals(24.96, row3.getDouble("value"), 0.0);
Assert.assertEquals(4, row3.getInt("variance"));
UntypedResultSet.Row row4 = iterator.next();
Assert.assertEquals("PRESSURE", row4.getString("sensor_type"));
Assert.assertEquals(20160402L, row4.getLong("date"));
- Assert.assertEquals(1.04, row4.getDouble("value"));
+ Assert.assertEquals(1.04, row4.getDouble("value"), 0.0);
Assert.assertEquals(7, row4.getInt("variance"));
}
@@ -1975,7 +1981,7 @@
store.forceBlockingFlush();
SSTable ssTable = store.getSSTables(SSTableSet.LIVE).iterator().next();
- Path path = FileSystems.getDefault().getPath(ssTable.getFilename().replace("-Data", "-SI_age"));
+ Path path = FileSystems.getDefault().getPath(ssTable.getFilename().replace("-Data", "-SI_" + CLUSTERING_CF_NAME_1 + "_age"));
// Overwrite index file with garbage
Writer writer = new FileWriter(path.toFile(), false);
@@ -1987,7 +1993,7 @@
Assert.assertTrue(executeCQL(CLUSTERING_CF_NAME_1, "SELECT * FROM %s.%s WHERE age = 27 AND name = 'Pavel'").isEmpty());
// Rebuld index
- store.rebuildSecondaryIndex("age");
+ store.rebuildSecondaryIndex(CLUSTERING_CF_NAME_1 + "_age");
long size2 = Files.readAttributes(path, BasicFileAttributes.class).size();
// Make sure that garbage was overwriten
@@ -2026,7 +2032,7 @@
try
{
// unsupported partition key column
- SASIIndex.validateOptions(Collections.singletonMap("target", "id"), store.metadata);
+ SASIIndex.validateOptions(Collections.singletonMap("target", "id"), store.metadata());
Assert.fail();
}
catch (ConfigurationException e)
@@ -2039,7 +2045,7 @@
// invalid index mode
SASIIndex.validateOptions(new HashMap<String, String>()
{{ put("target", "address"); put("mode", "NORMAL"); }},
- store.metadata);
+ store.metadata());
Assert.fail();
}
catch (ConfigurationException e)
@@ -2052,7 +2058,7 @@
// invalid SPARSE on the literal index
SASIIndex.validateOptions(new HashMap<String, String>()
{{ put("target", "address"); put("mode", "SPARSE"); }},
- store.metadata);
+ store.metadata());
Assert.fail();
}
catch (ConfigurationException e)
@@ -2065,7 +2071,7 @@
// invalid SPARSE on the explicitly literal index
SASIIndex.validateOptions(new HashMap<String, String>()
{{ put("target", "height"); put("mode", "SPARSE"); put("is_literal", "true"); }},
- store.metadata);
+ store.metadata());
Assert.fail();
}
catch (ConfigurationException e)
@@ -2078,7 +2084,7 @@
// SPARSE with analyzer
SASIIndex.validateOptions(new HashMap<String, String>()
{{ put("target", "height"); put("mode", "SPARSE"); put("analyzed", "true"); }},
- store.metadata);
+ store.metadata());
Assert.fail();
}
catch (ConfigurationException e)
@@ -2351,14 +2357,13 @@
put("key1", Pair.create("Pavel", 14));
}}, false);
- ColumnIndex index = ((SASIIndex) store.indexManager.getIndexByName("first_name")).getIndex();
+ ColumnIndex index = ((SASIIndex) store.indexManager.getIndexByName(store.name + "_first_name")).getIndex();
IndexMemtable beforeFlushMemtable = index.getCurrentMemtable();
PartitionRangeReadCommand command =
- PartitionRangeReadCommand.create(false,
- store.metadata,
+ PartitionRangeReadCommand.create(store.metadata(),
FBUtilities.nowInSeconds(),
- ColumnFilter.all(store.metadata),
+ ColumnFilter.all(store.metadata()),
RowFilter.NONE,
DataLimits.NONE,
DataRange.allData(store.getPartitioner()));
@@ -2447,8 +2452,8 @@
KS_NAME,
TABLE_NAME));
- Columns regulars = Schema.instance.getCFMetaData(KS_NAME, TABLE_NAME).partitionColumns().regulars;
- List<String> allColumns = regulars.stream().map(ColumnDefinition::toString).collect(Collectors.toList());
+ Columns regulars = Schema.instance.getTableMetadata(KS_NAME, TABLE_NAME).regularColumns();
+ List<String> allColumns = regulars.stream().map(ColumnMetadata::toString).collect(Collectors.toList());
List<String> textColumns = Arrays.asList("text_v", "ascii_v", "varchar_v");
new HashMap<Class<? extends AbstractAnalyzer>, List<String>>()
@@ -2515,7 +2520,7 @@
private static Set<String> getIndexed(ColumnFamilyStore store, int maxResults, Expression... expressions)
{
- return getIndexed(store, ColumnFilter.all(store.metadata), maxResults, expressions);
+ return getIndexed(store, ColumnFilter.all(store.metadata()), maxResults, expressions);
}
private static Set<String> getIndexed(ColumnFamilyStore store, ColumnFilter columnFilter, int maxResults, Expression... expressions)
@@ -2534,7 +2539,7 @@
do
{
count = 0;
- currentPage = getIndexed(store, ColumnFilter.all(store.metadata), lastKey, pageSize, expressions);
+ currentPage = getIndexed(store, ColumnFilter.all(store.metadata()), lastKey, pageSize, expressions);
if (currentPage == null)
break;
@@ -2563,15 +2568,14 @@
RowFilter filter = RowFilter.create();
for (Expression e : expressions)
- filter.add(store.metadata.getColumnDefinition(e.name), e.op, e.value);
+ filter.add(store.metadata().getColumn(e.name), e.op, e.value);
ReadCommand command =
- PartitionRangeReadCommand.create(false,
- store.metadata,
+ PartitionRangeReadCommand.create(store.metadata(),
FBUtilities.nowInSeconds(),
columnFilter,
filter,
- DataLimits.thriftLimits(maxResults, DataLimits.NO_LIMIT),
+ DataLimits.cqlLimits(maxResults),
range);
return command.executeLocally(command.executionController());
@@ -2579,7 +2583,7 @@
private static Mutation newMutation(String key, String firstName, String lastName, int age, long timestamp)
{
- Mutation rm = new Mutation(KS_NAME, decoratedKey(AsciiType.instance.decompose(key)));
+ Mutation.PartitionUpdateCollector rm = new Mutation.PartitionUpdateCollector(KS_NAME, decoratedKey(AsciiType.instance.decompose(key)));
List<Cell> cells = new ArrayList<>(3);
if (age >= 0)
@@ -2590,7 +2594,7 @@
cells.add(buildCell(ByteBufferUtil.bytes("last_name"), UTF8Type.instance.decompose(lastName), timestamp));
update(rm, cells);
- return rm;
+ return rm.build();
}
private static Set<String> getKeys(final UnfilteredPartitionIterator rows)
@@ -2631,18 +2635,11 @@
private Set<String> executeCQLWithKeys(String rawStatement) throws Exception
{
- SelectStatement statement = (SelectStatement) QueryProcessor.parseStatement(rawStatement).prepare(ClientState.forInternalCalls()).statement;
- ResultMessage.Rows cqlRows = statement.executeInternal(QueryState.forInternalCalls(), QueryOptions.DEFAULT);
-
Set<String> results = new TreeSet<>();
- for (CqlRow row : cqlRows.toThriftResult().getRows())
+ for (UntypedResultSet.Row row : QueryProcessor.executeOnceInternal(rawStatement))
{
- for (org.apache.cassandra.thrift.Column col : row.columns)
- {
- String columnName = UTF8Type.instance.getString(col.bufferForName());
- if (columnName.equals("id"))
- results.add(AsciiType.instance.getString(col.bufferForValue()));
- }
+ if (row.has("id"))
+ results.add(row.getString("id"));
}
return results;
@@ -2674,13 +2671,13 @@
private static Cell buildCell(ByteBuffer name, ByteBuffer value, long timestamp)
{
- CFMetaData cfm = Keyspace.open(KS_NAME).getColumnFamilyStore(CF_NAME).metadata;
- return BufferCell.live(cfm.getColumnDefinition(name), timestamp, value);
+ TableMetadata cfm = Keyspace.open(KS_NAME).getColumnFamilyStore(CF_NAME).metadata();
+ return BufferCell.live(cfm.getColumn(name), timestamp, value);
}
- private static Cell buildCell(CFMetaData cfm, ByteBuffer name, ByteBuffer value, long timestamp)
+ private static Cell buildCell(TableMetadata cfm, ByteBuffer name, ByteBuffer value, long timestamp)
{
- ColumnDefinition column = cfm.getColumnDefinition(name);
+ ColumnMetadata column = cfm.getColumn(name);
assert column != null;
return BufferCell.live(column, timestamp, value);
}
@@ -2690,16 +2687,16 @@
return new Expression(name, op, value);
}
- private static void update(Mutation rm, ByteBuffer name, ByteBuffer value, long timestamp)
+ private static void update(Mutation.PartitionUpdateCollector rm, ByteBuffer name, ByteBuffer value, long timestamp)
{
- CFMetaData metadata = Keyspace.open(KS_NAME).getColumnFamilyStore(CF_NAME).metadata;
+ TableMetadata metadata = Keyspace.open(KS_NAME).getColumnFamilyStore(CF_NAME).metadata();
rm.add(PartitionUpdate.singleRowUpdate(metadata, rm.key(), buildRow(buildCell(metadata, name, value, timestamp))));
}
- private static void update(Mutation rm, List<Cell> cells)
+ private static void update(Mutation.PartitionUpdateCollector rm, List<Cell> cells)
{
- CFMetaData metadata = Keyspace.open(KS_NAME).getColumnFamilyStore(CF_NAME).metadata;
+ TableMetadata metadata = Keyspace.open(KS_NAME).getColumnFamilyStore(CF_NAME).metadata();
rm.add(PartitionUpdate.singleRowUpdate(metadata, rm.key(), buildRow(cells)));
}
diff --git a/test/unit/org/apache/cassandra/index/sasi/disk/OnDiskIndexTest.java b/test/unit/org/apache/cassandra/index/sasi/disk/OnDiskIndexTest.java
index 10dc7a8..1afb7b4 100644
--- a/test/unit/org/apache/cassandra/index/sasi/disk/OnDiskIndexTest.java
+++ b/test/unit/org/apache/cassandra/index/sasi/disk/OnDiskIndexTest.java
@@ -38,6 +38,7 @@
import org.apache.cassandra.db.marshal.LongType;
import org.apache.cassandra.db.marshal.UTF8Type;
import org.apache.cassandra.io.util.DataOutputBuffer;
+import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.utils.MurmurHash;
import org.apache.cassandra.utils.Pair;
@@ -48,7 +49,7 @@
import com.google.common.collect.Iterators;
import com.google.common.collect.Sets;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -82,7 +83,7 @@
for (Map.Entry<ByteBuffer, TokenTreeBuilder> e : data.entrySet())
addAll(builder, e.getKey(), e.getValue());
- File index = File.createTempFile("on-disk-sa-string", "db");
+ File index = FileUtils.createTempFile("on-disk-sa-string", "db");
index.deleteOnExit();
builder.finish(index);
@@ -138,7 +139,7 @@
for (Map.Entry<ByteBuffer, TokenTreeBuilder> e : data.entrySet())
addAll(builder, e.getKey(), e.getValue());
- File index = File.createTempFile("on-disk-sa-int", "db");
+ File index = FileUtils.createTempFile("on-disk-sa-int", "db");
index.deleteOnExit();
builder.finish(index);
@@ -226,7 +227,7 @@
for (int i = 0; i < iterCheckNums.size(); i++)
iterTest.add(iterCheckNums.get(i), keyAt((long) i), i);
- File iterIndex = File.createTempFile("sa-iter", ".db");
+ File iterIndex = FileUtils.createTempFile("sa-iter", ".db");
iterIndex.deleteOnExit();
iterTest.finish(iterIndex);
@@ -278,7 +279,7 @@
addAll(this, UTF8Type.instance.decompose("Pavel"), keyBuilder(9L, 10L));
}};
- File index = File.createTempFile("on-disk-sa-multi-suffix-match", ".db");
+ File index = FileUtils.createTempFile("on-disk-sa-multi-suffix-match", ".db");
index.deleteOnExit();
builder.finish(index);
@@ -317,7 +318,7 @@
for (long i = 0; i < numIterations; i++)
builder.add(LongType.instance.decompose(start + i), keyAt(i), i);
- File index = File.createTempFile("on-disk-sa-sparse", "db");
+ File index = FileUtils.createTempFile("on-disk-sa-sparse", "db");
index.deleteOnExit();
builder.finish(index);
@@ -375,7 +376,7 @@
for (Map.Entry<ByteBuffer, TokenTreeBuilder> e : data.entrySet())
addAll(builder, e.getKey(), e.getValue());
- File index = File.createTempFile("on-disk-sa-except-test", "db");
+ File index = FileUtils.createTempFile("on-disk-sa-except-test", "db");
index.deleteOnExit();
builder.finish(index);
@@ -419,7 +420,7 @@
for (Map.Entry<ByteBuffer, TokenTreeBuilder> e : data.entrySet())
addAll(builder, e.getKey(), e.getValue());
- File index = File.createTempFile("on-disk-sa-except-int-test", "db");
+ File index = FileUtils.createTempFile("on-disk-sa-except-int-test", "db");
index.deleteOnExit();
builder.finish(index);
@@ -443,7 +444,7 @@
for (long i = lower; i <= upper; i++)
builder.add(LongType.instance.decompose(i), keyAt(i), i);
- File index = File.createTempFile("on-disk-sa-except-long-ranges", "db");
+ File index = FileUtils.createTempFile("on-disk-sa-except-long-ranges", "db");
index.deleteOnExit();
builder.finish(index);
@@ -535,8 +536,8 @@
builder2.add(e.getKey(), key, position);
}
- File index1 = File.createTempFile("on-disk-sa-int", "db");
- File index2 = File.createTempFile("on-disk-sa-int2", "db");
+ File index1 = FileUtils.createTempFile("on-disk-sa-int", "db");
+ File index2 = FileUtils.createTempFile("on-disk-sa-int2", "db");
index1.deleteOnExit();
index2.deleteOnExit();
@@ -569,7 +570,7 @@
for (Map.Entry<ByteBuffer, TokenTreeBuilder> entry : terms.entrySet())
addAll(builder, entry.getKey(), entry.getValue());
- File index = File.createTempFile("on-disk-sa-try-superblocks", ".db");
+ File index = FileUtils.createTempFile("on-disk-sa-try-superblocks", ".db");
index.deleteOnExit();
builder.finish(index);
@@ -597,7 +598,7 @@
for (long i = 0; i < 100000; i++)
builder.add(LongType.instance.decompose(i), keyAt(i), i);
- File index = File.createTempFile("on-disk-sa-multi-superblock-match", ".db");
+ File index = FileUtils.createTempFile("on-disk-sa-multi-superblock-match", ".db");
index.deleteOnExit();
builder.finish(index);
@@ -652,10 +653,10 @@
putAll(offsets, keyBuilder(100L + i));
}
- File indexA = File.createTempFile("on-disk-sa-partition-a", ".db");
+ File indexA = FileUtils.createTempFile("on-disk-sa-partition-a", ".db");
indexA.deleteOnExit();
- File indexB = File.createTempFile("on-disk-sa-partition-b", ".db");
+ File indexB = FileUtils.createTempFile("on-disk-sa-partition-b", ".db");
indexB.deleteOnExit();
builderA.finish(indexA);
@@ -682,7 +683,7 @@
Assert.assertEquals(actual, expected);
- File indexC = File.createTempFile("on-disk-sa-partition-final", ".db");
+ File indexC = FileUtils.createTempFile("on-disk-sa-partition-final", ".db");
indexC.deleteOnExit();
OnDiskIndexBuilder combined = new OnDiskIndexBuilder(UTF8Type.instance, LongType.instance, OnDiskIndexBuilder.Mode.PREFIX);
@@ -733,7 +734,7 @@
for (Map.Entry<ByteBuffer, TokenTreeBuilder> e : data.entrySet())
addAll(builder, e.getKey(), e.getValue());
- File index = File.createTempFile("on-disk-sa-prefix-contains-search", "db");
+ File index = FileUtils.createTempFile("on-disk-sa-prefix-contains-search", "db");
index.deleteOnExit();
builder.finish(index);
diff --git a/test/unit/org/apache/cassandra/index/sasi/disk/PerSSTableIndexWriterTest.java b/test/unit/org/apache/cassandra/index/sasi/disk/PerSSTableIndexWriterTest.java
index f19d962..97b3433 100644
--- a/test/unit/org/apache/cassandra/index/sasi/disk/PerSSTableIndexWriterTest.java
+++ b/test/unit/org/apache/cassandra/index/sasi/disk/PerSSTableIndexWriterTest.java
@@ -24,8 +24,6 @@
import java.util.concurrent.ThreadLocalRandom;
import org.apache.cassandra.SchemaLoader;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
import org.apache.cassandra.db.Clustering;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.DecoratedKey;
@@ -43,10 +41,12 @@
import org.apache.cassandra.io.FSError;
import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.schema.KeyspaceMetadata;
import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.schema.Tables;
-import org.apache.cassandra.service.MigrationManager;
+import org.apache.cassandra.schema.MigrationManager;
import org.apache.cassandra.utils.ByteBufferUtil;
import com.google.common.util.concurrent.Futures;
@@ -67,7 +67,7 @@
SchemaLoader.loadSchema();
MigrationManager.announceNewKeyspace(KeyspaceMetadata.create(KS_NAME,
KeyspaceParams.simpleTransient(1),
- Tables.of(SchemaLoader.sasiCFMD(KS_NAME, CF_NAME))));
+ Tables.of(SchemaLoader.sasiCFMD(KS_NAME, CF_NAME).build())));
}
@Test
@@ -78,12 +78,12 @@
final long timestamp = System.currentTimeMillis();
ColumnFamilyStore cfs = Keyspace.open(KS_NAME).getColumnFamilyStore(CF_NAME);
- ColumnDefinition column = cfs.metadata.getColumnDefinition(UTF8Type.instance.decompose("age"));
+ ColumnMetadata column = cfs.metadata().getColumn(UTF8Type.instance.decompose("age"));
- SASIIndex sasi = (SASIIndex) cfs.indexManager.getIndexByName("age");
+ SASIIndex sasi = (SASIIndex) cfs.indexManager.getIndexByName(cfs.name + "_age");
File directory = cfs.getDirectories().getDirectoryForNewSSTables();
- Descriptor descriptor = Descriptor.fromFilename(cfs.getSSTablePath(directory));
+ Descriptor descriptor = cfs.newSSTableDescriptor(directory);
PerSSTableIndexWriter indexWriter = (PerSSTableIndexWriter) sasi.getFlushObserver(descriptor, OperationType.FLUSH);
SortedMap<DecoratedKey, Row> expectedKeys = new TreeMap<>(DecoratedKey.comparator);
@@ -91,7 +91,7 @@
for (int i = 0; i < maxKeys; i++)
{
ByteBuffer key = ByteBufferUtil.bytes(String.format(keyFormat, i));
- expectedKeys.put(cfs.metadata.partitioner.decorateKey(key),
+ expectedKeys.put(cfs.metadata().partitioner.decorateKey(key),
BTreeRow.singleCellRow(Clustering.EMPTY,
BufferCell.live(column, timestamp, Int32Type.instance.decompose(i))));
}
@@ -136,7 +136,7 @@
OnDiskIndex index = new OnDiskIndex(new File(indexFile), Int32Type.instance, keyPosition -> {
ByteBuffer key = ByteBufferUtil.bytes(String.format(keyFormat, keyPosition));
- return cfs.metadata.partitioner.decorateKey(key);
+ return cfs.metadata().partitioner.decorateKey(key);
});
Assert.assertEquals(0, UTF8Type.instance.compare(index.minKey(), ByteBufferUtil.bytes(String.format(keyFormat, 0))));
@@ -170,12 +170,12 @@
final String columnName = "timestamp";
ColumnFamilyStore cfs = Keyspace.open(KS_NAME).getColumnFamilyStore(CF_NAME);
- ColumnDefinition column = cfs.metadata.getColumnDefinition(UTF8Type.instance.decompose(columnName));
+ ColumnMetadata column = cfs.metadata().getColumn(UTF8Type.instance.decompose(columnName));
- SASIIndex sasi = (SASIIndex) cfs.indexManager.getIndexByName(columnName);
+ SASIIndex sasi = (SASIIndex) cfs.indexManager.getIndexByName(cfs.name + "_" + columnName);
File directory = cfs.getDirectories().getDirectoryForNewSSTables();
- Descriptor descriptor = Descriptor.fromFilename(cfs.getSSTablePath(directory));
+ Descriptor descriptor = cfs.newSSTableDescriptor(directory);
PerSSTableIndexWriter indexWriter = (PerSSTableIndexWriter) sasi.getFlushObserver(descriptor, OperationType.FLUSH);
final long now = System.currentTimeMillis();
@@ -183,7 +183,7 @@
indexWriter.begin();
indexWriter.indexes.put(column, indexWriter.newIndex(sasi.getIndex()));
- populateSegment(cfs.metadata, indexWriter.getIndex(column), new HashMap<Long, Set<Integer>>()
+ populateSegment(cfs.metadata(), indexWriter.getIndex(column), new HashMap<Long, Set<Integer>>()
{{
put(now, new HashSet<>(Arrays.asList(0, 1)));
put(now + 1, new HashSet<>(Arrays.asList(2, 3)));
@@ -201,7 +201,7 @@
// now let's test multiple correct segments with yield incorrect final segment
for (int i = 0; i < 3; i++)
{
- populateSegment(cfs.metadata, index, new HashMap<Long, Set<Integer>>()
+ populateSegment(cfs.metadata(), index, new HashMap<Long, Set<Integer>>()
{{
put(now, new HashSet<>(Arrays.asList(random.nextInt(), random.nextInt(), random.nextInt())));
put(now + 1, new HashSet<>(Arrays.asList(random.nextInt(), random.nextInt(), random.nextInt())));
@@ -236,7 +236,7 @@
Assert.assertFalse(new File(index.outputFile).exists());
}
- private static void populateSegment(CFMetaData metadata, PerSSTableIndexWriter.Index index, Map<Long, Set<Integer>> data)
+ private static void populateSegment(TableMetadata metadata, PerSSTableIndexWriter.Index index, Map<Long, Set<Integer>> data)
{
for (Map.Entry<Long, Set<Integer>> value : data.entrySet())
{
diff --git a/test/unit/org/apache/cassandra/index/sasi/disk/TokenTreeTest.java b/test/unit/org/apache/cassandra/index/sasi/disk/TokenTreeTest.java
index 3d0850a..4339a62 100644
--- a/test/unit/org/apache/cassandra/index/sasi/disk/TokenTreeTest.java
+++ b/test/unit/org/apache/cassandra/index/sasi/disk/TokenTreeTest.java
@@ -42,12 +42,12 @@
import org.apache.cassandra.io.util.RandomAccessReader;
import org.apache.cassandra.io.util.SequentialWriter;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
import org.apache.commons.lang3.builder.HashCodeBuilder;
-import com.carrotsearch.hppc.LongOpenHashSet;
+import com.carrotsearch.hppc.LongHashSet;
import com.carrotsearch.hppc.LongSet;
import com.carrotsearch.hppc.cursors.LongCursor;
import com.google.common.base.Function;
@@ -56,18 +56,26 @@
{
private static final Function<Long, DecoratedKey> KEY_CONVERTER = new KeyConverter();
+ static LongSet singleOffset = new LongHashSet();
+ static LongSet bigSingleOffset = new LongHashSet();
+ static LongSet shortPackableCollision = new LongHashSet();
+ static LongSet intPackableCollision = new LongHashSet();
+ static LongSet multiCollision = new LongHashSet();
+ static LongSet unpackableCollision = new LongHashSet();
+
@BeforeClass
public static void setupDD()
{
DatabaseDescriptor.daemonInitialization();
+ singleOffset.add(1);
+ bigSingleOffset.add(2147521562L);
+ shortPackableCollision.add(2L); shortPackableCollision.add(3L);
+ intPackableCollision.add(6L); intPackableCollision.add(((long) Short.MAX_VALUE) + 1);
+ multiCollision.add(3L); multiCollision.add(4L); multiCollision.add(5L);
+ unpackableCollision.add(((long) Short.MAX_VALUE) + 1); unpackableCollision.add(((long) Short.MAX_VALUE) + 2);
}
- static LongSet singleOffset = new LongOpenHashSet() {{ add(1); }};
- static LongSet bigSingleOffset = new LongOpenHashSet() {{ add(2147521562L); }};
- static LongSet shortPackableCollision = new LongOpenHashSet() {{ add(2L); add(3L); }}; // can pack two shorts
- static LongSet intPackableCollision = new LongOpenHashSet() {{ add(6L); add(((long) Short.MAX_VALUE) + 1); }}; // can pack int & short
- static LongSet multiCollision = new LongOpenHashSet() {{ add(3L); add(4L); add(5L); }}; // can't pack
- static LongSet unpackableCollision = new LongOpenHashSet() {{ add(((long) Short.MAX_VALUE) + 1); add(((long) Short.MAX_VALUE) + 2); }}; // can't pack
+
final static SortedMap<Long, LongSet> simpleTokenMap = new TreeMap<Long, LongSet>()
{{
@@ -116,7 +124,7 @@
public void testSerializedSize(final TokenTreeBuilder builder) throws Exception
{
builder.finish();
- final File treeFile = File.createTempFile("token-tree-size-test", "tt");
+ final File treeFile = FileUtils.createTempFile("token-tree-size-test", "tt");
treeFile.deleteOnExit();
try (SequentialWriter writer = new SequentialWriter(treeFile, DEFAULT_OPT))
@@ -148,7 +156,7 @@
{
builder.finish();
- final File treeFile = File.createTempFile("token-tree-iterate-test1", "tt");
+ final File treeFile = FileUtils.createTempFile("token-tree-iterate-test1", "tt");
treeFile.deleteOnExit();
try (SequentialWriter writer = new SequentialWriter(treeFile, DEFAULT_OPT))
@@ -227,7 +235,7 @@
public void buildSerializeIterateAndSkip(TokenTreeBuilder builder, SortedMap<Long, LongSet> tokens) throws Exception
{
builder.finish();
- final File treeFile = File.createTempFile("token-tree-iterate-test2", "tt");
+ final File treeFile = FileUtils.createTempFile("token-tree-iterate-test2", "tt");
treeFile.deleteOnExit();
try (SequentialWriter writer = new SequentialWriter(treeFile, DEFAULT_OPT))
@@ -286,7 +294,7 @@
public void skipPastEnd(TokenTreeBuilder builder, SortedMap<Long, LongSet> tokens) throws Exception
{
builder.finish();
- final File treeFile = File.createTempFile("token-tree-skip-past-test", "tt");
+ final File treeFile = FileUtils.createTempFile("token-tree-skip-past-test", "tt");
treeFile.deleteOnExit();
try (SequentialWriter writer = new SequentialWriter(treeFile, DEFAULT_OPT))
@@ -422,7 +430,6 @@
LongSet found = result.getOffsets();
Assert.assertEquals(entry.getValue(), found);
-
}
}
@@ -430,7 +437,7 @@
private static TokenTree buildTree(TokenTreeBuilder builder) throws Exception
{
builder.finish();
- final File treeFile = File.createTempFile("token-tree-", "db");
+ final File treeFile = FileUtils.createTempFile("token-tree-", "db");
treeFile.deleteOnExit();
try (SequentialWriter writer = new SequentialWriter(treeFile, DEFAULT_OPT))
@@ -609,7 +616,7 @@
private static LongSet convert(long... values)
{
- LongSet result = new LongOpenHashSet(values.length);
+ LongSet result = new LongHashSet(values.length);
for (long v : values)
result.add(v);
@@ -640,7 +647,7 @@
{{
for (long i = minToken; i <= maxToken; i++)
{
- LongSet offsetSet = new LongOpenHashSet();
+ LongSet offsetSet = new LongHashSet();
offsetSet.add(i);
put(i, offsetSet);
}
@@ -648,7 +655,7 @@
final TokenTreeBuilder builder = isStatic ? new StaticTokenTreeBuilder(new FakeCombinedTerm(toks)) : new DynamicTokenTreeBuilder(toks);
builder.finish();
- final File treeFile = File.createTempFile("token-tree-get-test", "tt");
+ final File treeFile = FileUtils.createTempFile("token-tree-get-test", "tt");
treeFile.deleteOnExit();
try (SequentialWriter writer = new SequentialWriter(treeFile, DEFAULT_OPT))
diff --git a/test/unit/org/apache/cassandra/index/sasi/plan/ExpressionTest.java b/test/unit/org/apache/cassandra/index/sasi/plan/ExpressionTest.java
new file mode 100644
index 0000000..7457a85
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sasi/plan/ExpressionTest.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sasi.plan;
+
+import java.nio.ByteBuffer;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import org.junit.Test;
+
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.index.sasi.plan.Expression.Bound;
+
+public class ExpressionTest
+{
+
+ @Test
+ public void testBoundHashCode()
+ {
+ ByteBuffer buf1 = UTF8Type.instance.decompose("blah");
+ Expression.Bound b1 = new Expression.Bound(buf1, true);
+ ByteBuffer buf2 = UTF8Type.instance.decompose("blah");
+ Expression.Bound b2 = new Expression.Bound(buf2, true);
+ assertTrue(b1.equals(b2));
+ assertTrue(b1.hashCode() == b2.hashCode());
+ }
+
+ @Test
+ public void testNotMatchingBoundHashCode()
+ {
+ ByteBuffer buf1 = UTF8Type.instance.decompose("blah");
+ Expression.Bound b1 = new Expression.Bound(buf1, true);
+ ByteBuffer buf2 = UTF8Type.instance.decompose("blah2");
+ Expression.Bound b2 = new Expression.Bound(buf2, true);
+ assertFalse(b1.equals(b2));
+ assertFalse(b1.hashCode() == b2.hashCode());
+ }
+}
diff --git a/test/unit/org/apache/cassandra/index/sasi/plan/OperationTest.java b/test/unit/org/apache/cassandra/index/sasi/plan/OperationTest.java
index e388cd4..8273dec 100644
--- a/test/unit/org/apache/cassandra/index/sasi/plan/OperationTest.java
+++ b/test/unit/org/apache/cassandra/index/sasi/plan/OperationTest.java
@@ -25,8 +25,8 @@
import com.google.common.collect.Multimap;
import com.google.common.collect.Sets;
import org.apache.cassandra.SchemaLoader;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.cql3.Operator;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.filter.RowFilter;
@@ -37,10 +37,7 @@
import org.apache.cassandra.db.marshal.LongType;
import org.apache.cassandra.db.marshal.UTF8Type;
import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.schema.KeyspaceMetadata;
import org.apache.cassandra.schema.KeyspaceParams;
-import org.apache.cassandra.schema.Tables;
-import org.apache.cassandra.service.MigrationManager;
import org.apache.cassandra.utils.FBUtilities;
import org.junit.*;
@@ -61,11 +58,11 @@
{
System.setProperty("cassandra.config", "cassandra-murmur.yaml");
SchemaLoader.loadSchema();
- MigrationManager.announceNewKeyspace(KeyspaceMetadata.create(KS_NAME,
- KeyspaceParams.simpleTransient(1),
- Tables.of(SchemaLoader.sasiCFMD(KS_NAME, CF_NAME),
- SchemaLoader.clusteringSASICFMD(KS_NAME, CLUSTERING_CF_NAME),
- SchemaLoader.staticSASICFMD(KS_NAME, STATIC_CF_NAME))));
+ SchemaLoader.createKeyspace(KS_NAME,
+ KeyspaceParams.simpleTransient(1),
+ SchemaLoader.sasiCFMD(KS_NAME, CF_NAME),
+ SchemaLoader.clusteringSASICFMD(KS_NAME, CLUSTERING_CF_NAME),
+ SchemaLoader.staticSASICFMD(KS_NAME, STATIC_CF_NAME));
BACKEND = Keyspace.open(KS_NAME).getColumnFamilyStore(CF_NAME);
CLUSTERING_BACKEND = Keyspace.open(KS_NAME).getColumnFamilyStore(CLUSTERING_CF_NAME);
@@ -78,7 +75,7 @@
public void beforeTest()
{
controller = new QueryController(BACKEND,
- PartitionRangeReadCommand.allDataRead(BACKEND.metadata, FBUtilities.nowInSeconds()),
+ PartitionRangeReadCommand.allDataRead(BACKEND.metadata(), FBUtilities.nowInSeconds()),
TimeUnit.SECONDS.toMillis(10));
}
@@ -91,9 +88,9 @@
@Test
public void testAnalyze() throws Exception
{
- final ColumnDefinition firstName = getColumn(UTF8Type.instance.decompose("first_name"));
- final ColumnDefinition age = getColumn(UTF8Type.instance.decompose("age"));
- final ColumnDefinition comment = getColumn(UTF8Type.instance.decompose("comment"));
+ final ColumnMetadata firstName = getColumn(UTF8Type.instance.decompose("first_name"));
+ final ColumnMetadata age = getColumn(UTF8Type.instance.decompose("age"));
+ final ColumnMetadata comment = getColumn(UTF8Type.instance.decompose("comment"));
// age != 5 AND age > 1 AND age != 6 AND age <= 10
Map<Expression.Op, Expression> expressions = convert(Operation.analyzeGroup(controller, OperationType.AND,
@@ -184,8 +181,8 @@
}}, expressions.get(Expression.Op.EQ));
// comment = 'soft eng' and comment != 'likes do'
- ListMultimap<ColumnDefinition, Expression> e = Operation.analyzeGroup(controller, OperationType.OR,
- Arrays.asList(new SimpleExpression(comment, Operator.LIKE_MATCHES, UTF8Type.instance.decompose("soft eng")),
+ ListMultimap<ColumnMetadata, Expression> e = Operation.analyzeGroup(controller, OperationType.OR,
+ Arrays.asList(new SimpleExpression(comment, Operator.LIKE_MATCHES, UTF8Type.instance.decompose("soft eng")),
new SimpleExpression(comment, Operator.NEQ, UTF8Type.instance.decompose("likes do"))));
List<Expression> expectedExpressions = new ArrayList<Expression>(2)
@@ -274,8 +271,8 @@
@Test
public void testSatisfiedBy() throws Exception
{
- final ColumnDefinition timestamp = getColumn(UTF8Type.instance.decompose("timestamp"));
- final ColumnDefinition age = getColumn(UTF8Type.instance.decompose("age"));
+ final ColumnMetadata timestamp = getColumn(UTF8Type.instance.decompose("timestamp"));
+ final ColumnMetadata age = getColumn(UTF8Type.instance.decompose("age"));
Operation.Builder builder = new Operation.Builder(OperationType.AND, controller, new SimpleExpression(age, Operator.NEQ, Int32Type.instance.decompose(5)));
Operation op = builder.complete();
@@ -438,8 +435,8 @@
@Test
public void testAnalyzeNotIndexedButDefinedColumn() throws Exception
{
- final ColumnDefinition firstName = getColumn(UTF8Type.instance.decompose("first_name"));
- final ColumnDefinition height = getColumn(UTF8Type.instance.decompose("height"));
+ final ColumnMetadata firstName = getColumn(UTF8Type.instance.decompose("first_name"));
+ final ColumnMetadata height = getColumn(UTF8Type.instance.decompose("height"));
// first_name = 'a' AND height != 10
Map<Expression.Op, Expression> expressions;
@@ -490,7 +487,7 @@
@Test
public void testSatisfiedByWithMultipleTerms()
{
- final ColumnDefinition comment = getColumn(UTF8Type.instance.decompose("comment"));
+ final ColumnMetadata comment = getColumn(UTF8Type.instance.decompose("comment"));
Unfiltered row = buildRow(buildCell(comment,UTF8Type.instance.decompose("software engineer is working on a project"),System.currentTimeMillis()));
Row staticRow = buildRow(Clustering.STATIC_CLUSTERING);
@@ -511,10 +508,10 @@
@Test
public void testSatisfiedByWithClustering()
{
- ColumnDefinition location = getColumn(CLUSTERING_BACKEND, UTF8Type.instance.decompose("location"));
- ColumnDefinition age = getColumn(CLUSTERING_BACKEND, UTF8Type.instance.decompose("age"));
- ColumnDefinition height = getColumn(CLUSTERING_BACKEND, UTF8Type.instance.decompose("height"));
- ColumnDefinition score = getColumn(CLUSTERING_BACKEND, UTF8Type.instance.decompose("score"));
+ ColumnMetadata location = getColumn(CLUSTERING_BACKEND, UTF8Type.instance.decompose("location"));
+ ColumnMetadata age = getColumn(CLUSTERING_BACKEND, UTF8Type.instance.decompose("age"));
+ ColumnMetadata height = getColumn(CLUSTERING_BACKEND, UTF8Type.instance.decompose("height"));
+ ColumnMetadata score = getColumn(CLUSTERING_BACKEND, UTF8Type.instance.decompose("score"));
Unfiltered row = buildRow(Clustering.make(UTF8Type.instance.fromString("US"), Int32Type.instance.decompose(27)),
buildCell(height, Int32Type.instance.decompose(182), System.currentTimeMillis()),
@@ -567,7 +564,7 @@
Assert.assertTrue(builder.complete().satisfiedBy(row, staticRow, false));
}
- private Map<Expression.Op, Expression> convert(Multimap<ColumnDefinition, Expression> expressions)
+ private Map<Expression.Op, Expression> convert(Multimap<ColumnMetadata, Expression> expressions)
{
Map<Expression.Op, Expression> converted = new HashMap<>();
for (Expression expression : expressions.values())
@@ -583,8 +580,8 @@
@Test
public void testSatisfiedByWithStatic()
{
- final ColumnDefinition sensorType = getColumn(STATIC_BACKEND, UTF8Type.instance.decompose("sensor_type"));
- final ColumnDefinition value = getColumn(STATIC_BACKEND, UTF8Type.instance.decompose("value"));
+ final ColumnMetadata sensorType = getColumn(STATIC_BACKEND, UTF8Type.instance.decompose("sensor_type"));
+ final ColumnMetadata value = getColumn(STATIC_BACKEND, UTF8Type.instance.decompose("value"));
Unfiltered row = buildRow(Clustering.make(UTF8Type.instance.fromString("date"), LongType.instance.decompose(20160401L)),
buildCell(value, DoubleType.instance.decompose(24.56), System.currentTimeMillis()));
@@ -638,7 +635,7 @@
private static class SimpleExpression extends RowFilter.Expression
{
- SimpleExpression(ColumnDefinition column, Operator operator, ByteBuffer value)
+ SimpleExpression(ColumnMetadata column, Operator operator, ByteBuffer value)
{
super(column, operator, value);
}
@@ -650,7 +647,7 @@
}
@Override
- public boolean isSatisfiedBy(CFMetaData metadata, DecoratedKey partitionKey, Row row)
+ public boolean isSatisfiedBy(TableMetadata metadata, DecoratedKey partitionKey, Row row)
{
throw new UnsupportedOperationException();
}
@@ -684,23 +681,23 @@
return rowBuilder.build();
}
- private static Cell buildCell(ColumnDefinition column, ByteBuffer value, long timestamp)
+ private static Cell buildCell(ColumnMetadata column, ByteBuffer value, long timestamp)
{
return BufferCell.live(column, timestamp, value);
}
- private static Cell deletedCell(ColumnDefinition column, long timestamp, int nowInSeconds)
+ private static Cell deletedCell(ColumnMetadata column, long timestamp, int nowInSeconds)
{
return BufferCell.tombstone(column, timestamp, nowInSeconds);
}
- private static ColumnDefinition getColumn(ByteBuffer name)
+ private static ColumnMetadata getColumn(ByteBuffer name)
{
return getColumn(BACKEND, name);
}
- private static ColumnDefinition getColumn(ColumnFamilyStore cfs, ByteBuffer name)
+ private static ColumnMetadata getColumn(ColumnFamilyStore cfs, ByteBuffer name)
{
- return cfs.metadata.getColumnDefinition(name);
+ return cfs.metadata().getColumn(name);
}
}
diff --git a/test/unit/org/apache/cassandra/index/sasi/utils/LongIterator.java b/test/unit/org/apache/cassandra/index/sasi/utils/LongIterator.java
index 205d28f..e7ff5b8 100644
--- a/test/unit/org/apache/cassandra/index/sasi/utils/LongIterator.java
+++ b/test/unit/org/apache/cassandra/index/sasi/utils/LongIterator.java
@@ -23,7 +23,7 @@
import java.util.Iterator;
import java.util.List;
-import com.carrotsearch.hppc.LongOpenHashSet;
+import com.carrotsearch.hppc.LongHashSet;
import com.carrotsearch.hppc.LongSet;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.index.sasi.disk.Token;
@@ -84,7 +84,7 @@
@Override
public LongSet getOffsets()
{
- return new LongOpenHashSet(4);
+ return new LongHashSet(4);
}
@Override
diff --git a/test/unit/org/apache/cassandra/index/sasi/utils/MappedBufferTest.java b/test/unit/org/apache/cassandra/index/sasi/utils/MappedBufferTest.java
index 7ffebf1..e55f6ba 100644
--- a/test/unit/org/apache/cassandra/index/sasi/utils/MappedBufferTest.java
+++ b/test/unit/org/apache/cassandra/index/sasi/utils/MappedBufferTest.java
@@ -449,7 +449,7 @@
@Test
public void testOpenWithoutPageBits() throws IOException
{
- File tmp = File.createTempFile("mapped-buffer", "tmp");
+ File tmp = FileUtils.createTempFile("mapped-buffer", "tmp");
tmp.deleteOnExit();
RandomAccessFile file = new RandomAccessFile(tmp, "rw");
@@ -490,7 +490,7 @@
private MappedBuffer createTestFile(long numCount, int typeSize, int numPageBits, int padding) throws IOException
{
- final File testFile = File.createTempFile("mapped-buffer-test", "db");
+ final File testFile = FileUtils.createTempFile("mapped-buffer-test", "db");
testFile.deleteOnExit();
RandomAccessFile file = new RandomAccessFile(testFile, "rw");
diff --git a/test/unit/org/apache/cassandra/index/sasi/utils/RangeIntersectionIteratorTest.java b/test/unit/org/apache/cassandra/index/sasi/utils/RangeIntersectionIteratorTest.java
index 4dc9e3f..e796240 100644
--- a/test/unit/org/apache/cassandra/index/sasi/utils/RangeIntersectionIteratorTest.java
+++ b/test/unit/org/apache/cassandra/index/sasi/utils/RangeIntersectionIteratorTest.java
@@ -27,7 +27,7 @@
import org.apache.cassandra.index.sasi.utils.RangeIntersectionIterator.BounceIntersectionIterator;
import org.apache.cassandra.io.util.FileUtils;
-import com.carrotsearch.hppc.LongOpenHashSet;
+import com.carrotsearch.hppc.LongHashSet;
import com.carrotsearch.hppc.LongSet;
import org.junit.Assert;
@@ -387,7 +387,7 @@
for (int i = 0; i < ranges.length; i++)
{
int rangeSize = random.nextInt(16, 512);
- LongSet range = new LongOpenHashSet(rangeSize);
+ LongSet range = new LongHashSet(rangeSize);
for (int j = 0; j < rangeSize; j++)
range.add(random.nextLong(0, 100));
diff --git a/test/unit/org/apache/cassandra/io/DiskSpaceMetricsTest.java b/test/unit/org/apache/cassandra/io/DiskSpaceMetricsTest.java
index ddacc6b..73d5e22 100644
--- a/test/unit/org/apache/cassandra/io/DiskSpaceMetricsTest.java
+++ b/test/unit/org/apache/cassandra/io/DiskSpaceMetricsTest.java
@@ -1,10 +1,8 @@
package org.apache.cassandra.io;
import java.io.IOException;
-import java.util.Collections;
import java.util.List;
import java.util.Map;
-import java.util.UUID;
import java.util.concurrent.atomic.AtomicInteger;
import com.google.common.collect.ImmutableMap;
@@ -21,6 +19,7 @@
import org.apache.cassandra.io.sstable.IndexSummaryManager;
import org.apache.cassandra.io.sstable.IndexSummaryRedistribution;
import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.utils.FBUtilities;
public class DiskSpaceMetricsTest extends CQLTester
@@ -102,10 +101,10 @@
{
List<SSTableReader> sstables = Lists.newArrayList(cfs.getSSTables(SSTableSet.CANONICAL));
LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.UNKNOWN);
- Map<UUID, LifecycleTransaction> txns = ImmutableMap.of(cfs.metadata.cfId, txn);
+ Map<TableId, LifecycleTransaction> txns = ImmutableMap.of(cfs.metadata.id, txn);
// fail on the last file (* 3 because we call isStopRequested 3 times for each sstable, and we should fail on the last)
AtomicInteger countdown = new AtomicInteger(3 * sstables.size() - 1);
- IndexSummaryRedistribution redistribution = new IndexSummaryRedistribution(Collections.emptyList(), txns, 0) {
+ IndexSummaryRedistribution redistribution = new IndexSummaryRedistribution(txns, 0, 0) {
public boolean isStopRequested()
{
return countdown.decrementAndGet() == 0;
diff --git a/test/unit/org/apache/cassandra/io/compress/CQLCompressionTest.java b/test/unit/org/apache/cassandra/io/compress/CQLCompressionTest.java
index a2aff2f..108c70f 100644
--- a/test/unit/org/apache/cassandra/io/compress/CQLCompressionTest.java
+++ b/test/unit/org/apache/cassandra/io/compress/CQLCompressionTest.java
@@ -18,27 +18,51 @@
package org.apache.cassandra.io.compress;
+import java.util.Set;
+
+import org.junit.Before;
+import org.junit.BeforeClass;
import org.junit.Test;
+import org.apache.cassandra.config.Config;
+import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
public class CQLCompressionTest extends CQLTester
{
+ private static Config.FlushCompression defaultFlush;
+
+ @BeforeClass
+ public static void setUpClass()
+ {
+ CQLTester.setUpClass();
+
+ defaultFlush = DatabaseDescriptor.getFlushCompression();
+ }
+
+ @Before
+ public void resetDefaultFlush()
+ {
+ DatabaseDescriptor.setFlushCompression(defaultFlush);
+ }
+
@Test
public void lz4ParamsTest()
{
createTable("create table %s (id int primary key, uh text) with compression = {'class':'LZ4Compressor', 'lz4_high_compressor_level':3}");
- assertTrue(((LZ4Compressor)getCurrentColumnFamilyStore().metadata.params.compression.getSstableCompressor()).compressorType.equals(LZ4Compressor.LZ4_FAST_COMPRESSOR));
+ assertEquals(((LZ4Compressor) getCurrentColumnFamilyStore().metadata().params.compression.getSstableCompressor()).compressorType, LZ4Compressor.LZ4_FAST_COMPRESSOR);
createTable("create table %s (id int primary key, uh text) with compression = {'class':'LZ4Compressor', 'lz4_compressor_type':'high', 'lz4_high_compressor_level':13}");
- assertEquals(((LZ4Compressor)getCurrentColumnFamilyStore().metadata.params.compression.getSstableCompressor()).compressorType, LZ4Compressor.LZ4_HIGH_COMPRESSOR);
- assertEquals(((LZ4Compressor)getCurrentColumnFamilyStore().metadata.params.compression.getSstableCompressor()).compressionLevel, (Integer)13);
+ assertEquals(((LZ4Compressor)getCurrentColumnFamilyStore().metadata().params.compression.getSstableCompressor()).compressorType, LZ4Compressor.LZ4_HIGH_COMPRESSOR);
+ assertEquals(((LZ4Compressor)getCurrentColumnFamilyStore().metadata().params.compression.getSstableCompressor()).compressionLevel, (Integer)13);
createTable("create table %s (id int primary key, uh text) with compression = {'class':'LZ4Compressor'}");
- assertEquals(((LZ4Compressor)getCurrentColumnFamilyStore().metadata.params.compression.getSstableCompressor()).compressorType, LZ4Compressor.LZ4_FAST_COMPRESSOR);
- assertEquals(((LZ4Compressor)getCurrentColumnFamilyStore().metadata.params.compression.getSstableCompressor()).compressionLevel, (Integer)9);
+ assertEquals(((LZ4Compressor)getCurrentColumnFamilyStore().metadata().params.compression.getSstableCompressor()).compressorType, LZ4Compressor.LZ4_FAST_COMPRESSOR);
+ assertEquals(((LZ4Compressor)getCurrentColumnFamilyStore().metadata().params.compression.getSstableCompressor()).compressionLevel, (Integer)9);
}
@Test(expected = ConfigurationException.class)
@@ -53,4 +77,197 @@
throw e.getCause();
}
}
+
+ @Test
+ public void zstdParamsTest()
+ {
+ createTable("create table %s (id int primary key, uh text) with compression = {'class':'ZstdCompressor', 'compression_level':-22}");
+ assertTrue(((ZstdCompressor)getCurrentColumnFamilyStore().metadata().params.compression.getSstableCompressor()).getClass().equals(ZstdCompressor.class));
+ assertEquals(((ZstdCompressor)getCurrentColumnFamilyStore().metadata().params.compression.getSstableCompressor()).getCompressionLevel(), -22);
+
+ createTable("create table %s (id int primary key, uh text) with compression = {'class':'ZstdCompressor'}");
+ assertTrue(((ZstdCompressor)getCurrentColumnFamilyStore().metadata().params.compression.getSstableCompressor()).getClass().equals(ZstdCompressor.class));
+ assertEquals(((ZstdCompressor)getCurrentColumnFamilyStore().metadata().params.compression.getSstableCompressor()).getCompressionLevel(), ZstdCompressor.DEFAULT_COMPRESSION_LEVEL);
+ }
+
+ @Test(expected = ConfigurationException.class)
+ public void zstdBadParamsTest() throws Throwable
+ {
+ try
+ {
+ createTable("create table %s (id int primary key, uh text) with compression = {'class':'ZstdCompressor', 'compression_level':'100'}");
+ }
+ catch (RuntimeException e)
+ {
+ throw e.getCause();
+ }
+ }
+
+ @Test
+ public void lz4FlushTest() throws Throwable
+ {
+ createTable("CREATE TABLE %s (k text PRIMARY KEY, v text) WITH compression = {'sstable_compression': 'LZ4Compressor'};");
+ ColumnFamilyStore store = flushTwice();
+
+ // Should flush as LZ4 "fast"
+ Set<SSTableReader> sstables = store.getLiveSSTables();
+ sstables.forEach(sstable -> {
+ LZ4Compressor compressor = (LZ4Compressor) sstable.getCompressionMetadata().parameters.getSstableCompressor();
+ assertEquals(LZ4Compressor.LZ4_FAST_COMPRESSOR, compressor.compressorType);
+ });
+
+ // Should compact to LZ4 "fast"
+ compact();
+
+ sstables = store.getLiveSSTables();
+ assertEquals(sstables.size(), 1);
+ store.getLiveSSTables().forEach(sstable -> {
+ LZ4Compressor compressor = (LZ4Compressor) sstable.getCompressionMetadata().parameters.getSstableCompressor();
+ assertEquals(LZ4Compressor.LZ4_FAST_COMPRESSOR, compressor.compressorType);
+ });
+ }
+
+ @Test
+ public void lz4hcFlushTest() throws Throwable
+ {
+ createTable("CREATE TABLE %s (k text PRIMARY KEY, v text) WITH compression = " +
+ "{'sstable_compression': 'LZ4Compressor', 'lz4_compressor_type': 'high'};");
+ ColumnFamilyStore store = flushTwice();
+
+ // Should flush as LZ4 "fast" mode
+ Set<SSTableReader> sstables = store.getLiveSSTables();
+ sstables.forEach(sstable -> {
+ LZ4Compressor compressor = (LZ4Compressor) sstable.getCompressionMetadata().parameters.getSstableCompressor();
+ assertEquals(LZ4Compressor.LZ4_FAST_COMPRESSOR, compressor.compressorType);
+ });
+
+ // Should compact to LZ4 "high" mode
+ compact();
+
+ sstables = store.getLiveSSTables();
+ assertEquals(sstables.size(), 1);
+ store.getLiveSSTables().forEach(sstable -> {
+ LZ4Compressor compressor = (LZ4Compressor) sstable.getCompressionMetadata().parameters.getSstableCompressor();
+ assertEquals(LZ4Compressor.LZ4_HIGH_COMPRESSOR, compressor.compressorType);
+ });
+ }
+
+ @Test
+ public void zstdFlushTest() throws Throwable
+ {
+ createTable("CREATE TABLE %s (k text PRIMARY KEY, v text) WITH compression = {'sstable_compression': 'ZstdCompressor'};");
+ ColumnFamilyStore store = flushTwice();
+
+ // Should flush as LZ4
+ Set<SSTableReader> sstables = store.getLiveSSTables();
+ sstables.forEach(sstable -> {
+ assertTrue(sstable.getCompressionMetadata().parameters.getSstableCompressor() instanceof LZ4Compressor);
+ });
+
+ // Should compact to Zstd
+ compact();
+
+ sstables = store.getLiveSSTables();
+ assertEquals(sstables.size(), 1);
+ store.getLiveSSTables().forEach(sstable -> {
+ assertTrue(sstable.getCompressionMetadata().parameters.getSstableCompressor() instanceof ZstdCompressor);
+ });
+ }
+
+ @Test
+ public void deflateFlushTest() throws Throwable
+ {
+ createTable("CREATE TABLE %s (k text PRIMARY KEY, v text) WITH compression = {'sstable_compression': 'DeflateCompressor'};");
+ ColumnFamilyStore store = flushTwice();
+
+ // Should flush as LZ4
+ Set<SSTableReader> sstables = store.getLiveSSTables();
+ sstables.forEach(sstable -> {
+ assertTrue(sstable.getCompressionMetadata().parameters.getSstableCompressor() instanceof LZ4Compressor);
+ });
+
+ // Should compact to Deflate
+ compact();
+
+ sstables = store.getLiveSSTables();
+ assertEquals(sstables.size(), 1);
+ store.getLiveSSTables().forEach(sstable -> {
+ assertTrue(sstable.getCompressionMetadata().parameters.getSstableCompressor() instanceof DeflateCompressor);
+ });
+ }
+
+ @Test
+ public void useNoCompressorOnFlushTest() throws Throwable
+ {
+ DatabaseDescriptor.setFlushCompression(Config.FlushCompression.none);
+ createTable("CREATE TABLE %s (k text PRIMARY KEY, v text) WITH compression = {'sstable_compression': 'LZ4Compressor'};");
+ ColumnFamilyStore store = flushTwice();
+
+ // Should flush as Noop compressor
+ Set<SSTableReader> sstables = store.getLiveSSTables();
+ sstables.forEach(sstable -> {
+ assertTrue(sstable.getCompressionMetadata().parameters.getSstableCompressor() instanceof NoopCompressor);
+ });
+
+ // Should compact to LZ4
+ compact();
+
+ sstables = store.getLiveSSTables();
+ assertEquals(sstables.size(), 1);
+ store.getLiveSSTables().forEach(sstable -> {
+ assertTrue(sstable.getCompressionMetadata().parameters.getSstableCompressor() instanceof LZ4Compressor);
+ });
+ }
+
+ @Test
+ public void useTableCompressorOnFlushTest() throws Throwable
+ {
+ DatabaseDescriptor.setFlushCompression(Config.FlushCompression.table);
+
+ createTable("CREATE TABLE %s (k text PRIMARY KEY, v text) WITH compression = {'sstable_compression': 'ZstdCompressor'};");
+ ColumnFamilyStore store = flushTwice();
+
+ // Should flush as Zstd
+ Set<SSTableReader> sstables = store.getLiveSSTables();
+ sstables.forEach(sstable -> {
+ assertTrue(sstable.getCompressionMetadata().parameters.getSstableCompressor() instanceof ZstdCompressor);
+ });
+ }
+
+ @Test
+ public void zstdTableFlushTest() throws Throwable
+ {
+ createTable("CREATE TABLE %s (k text PRIMARY KEY, v text) WITH compression = {'sstable_compression': 'ZstdCompressor'};");
+ ColumnFamilyStore store = flushTwice();
+
+ // Should flush as LZ4
+ Set<SSTableReader> sstables = store.getLiveSSTables();
+ sstables.forEach(sstable -> {
+ assertTrue(sstable.getCompressionMetadata().parameters.getSstableCompressor() instanceof LZ4Compressor);
+ });
+
+ // Should compact to Zstd
+ compact();
+
+ sstables = store.getLiveSSTables();
+ assertEquals(sstables.size(), 1);
+ store.getLiveSSTables().forEach(sstable -> {
+ assertTrue(sstable.getCompressionMetadata().parameters.getSstableCompressor() instanceof ZstdCompressor);
+ });
+ }
+
+ private ColumnFamilyStore flushTwice() throws Throwable
+ {
+ ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
+
+ execute("INSERT INTO %s (k, v) values (?, ?)", "k1", "v1");
+ flush();
+ assertEquals(1, cfs.getLiveSSTables().size());
+
+ execute("INSERT INTO %s (k, v) values (?, ?)", "k2", "v2");
+ flush();
+ assertEquals(2, cfs.getLiveSSTables().size());
+
+ return cfs;
+ }
}
diff --git a/test/unit/org/apache/cassandra/io/compress/CompressedRandomAccessReaderTest.java b/test/unit/org/apache/cassandra/io/compress/CompressedRandomAccessReaderTest.java
index c718147..d3d81f0 100644
--- a/test/unit/org/apache/cassandra/io/compress/CompressedRandomAccessReaderTest.java
+++ b/test/unit/org/apache/cassandra/io/compress/CompressedRandomAccessReaderTest.java
@@ -25,6 +25,7 @@
import java.util.Arrays;
import java.util.Random;
+import org.assertj.core.api.Assertions;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -36,14 +37,13 @@
import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
import org.apache.cassandra.io.util.*;
import org.apache.cassandra.schema.CompressionParams;
-import org.apache.cassandra.utils.ChecksumType;
import org.apache.cassandra.utils.SyncUtil;
+import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertSame;
import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
public class CompressedRandomAccessReaderTest
{
@@ -57,30 +57,46 @@
public void testResetAndTruncate() throws IOException
{
// test reset in current buffer or previous one
- testResetAndTruncate(File.createTempFile("normal", "1"), false, false, 10);
- testResetAndTruncate(File.createTempFile("normal", "2"), false, false, CompressionParams.DEFAULT_CHUNK_LENGTH);
+ testResetAndTruncate(FileUtils.createTempFile("normal", "1"), false, false, 10, 0);
+ testResetAndTruncate(FileUtils.createTempFile("normal", "2"), false, false, CompressionParams.DEFAULT_CHUNK_LENGTH, 0);
}
@Test
public void testResetAndTruncateCompressed() throws IOException
{
// test reset in current buffer or previous one
- testResetAndTruncate(File.createTempFile("compressed", "1"), true, false, 10);
- testResetAndTruncate(File.createTempFile("compressed", "2"), true, false, CompressionParams.DEFAULT_CHUNK_LENGTH);
+ testResetAndTruncate(FileUtils.createTempFile("compressed", "1"), true, false, 10, 0);
+ testResetAndTruncate(FileUtils.createTempFile("compressed", "2"), true, false, CompressionParams.DEFAULT_CHUNK_LENGTH, 0);
}
@Test
public void testResetAndTruncateCompressedMmap() throws IOException
{
// test reset in current buffer or previous one
- testResetAndTruncate(File.createTempFile("compressed_mmap", "1"), true, true, 10);
- testResetAndTruncate(File.createTempFile("compressed_mmap", "2"), true, true, CompressionParams.DEFAULT_CHUNK_LENGTH);
+ testResetAndTruncate(FileUtils.createTempFile("compressed_mmap", "1"), true, true, 10, 0);
+ testResetAndTruncate(FileUtils.createTempFile("compressed_mmap", "2"), true, true, CompressionParams.DEFAULT_CHUNK_LENGTH, 0);
+ }
+
+ @Test
+ public void testResetAndTruncateCompressedUncompressedChunks() throws IOException
+ {
+ // test reset in current buffer or previous one
+ testResetAndTruncate(FileUtils.createTempFile("compressed_uchunks", "1"), true, false, 10, 3);
+ testResetAndTruncate(FileUtils.createTempFile("compressed_uchunks", "2"), true, false, CompressionParams.DEFAULT_CHUNK_LENGTH, 3);
+ }
+
+ @Test
+ public void testResetAndTruncateCompressedUncompressedChunksMmap() throws IOException
+ {
+ // test reset in current buffer or previous one
+ testResetAndTruncate(FileUtils.createTempFile("compressed_uchunks_mmap", "1"), true, true, 10, 3);
+ testResetAndTruncate(FileUtils.createTempFile("compressed_uchunks_mmap", "2"), true, true, CompressionParams.DEFAULT_CHUNK_LENGTH, 3);
}
@Test
public void test6791() throws IOException, ConfigurationException
{
- File f = File.createTempFile("compressed6791_", "3");
+ File f = FileUtils.createTempFile("compressed6791_", "3");
String filename = f.getAbsolutePath();
MetadataCollector sstableMetadataCollector = new MetadataCollector(new ClusteringComparator(BytesType.instance));
try(CompressedSequentialWriter writer = new CompressedSequentialWriter(f, filename + ".metadata",
@@ -105,9 +121,7 @@
}
try (FileHandle.Builder builder = new FileHandle.Builder(filename)
- .withCompressionMetadata(new CompressionMetadata(filename + ".metadata",
- f.length(),
- ChecksumType.CRC32));
+ .withCompressionMetadata(new CompressionMetadata(filename + ".metadata", f.length(), true));
FileHandle fh = builder.complete();
RandomAccessReader reader = fh.createReader())
{
@@ -138,22 +152,16 @@
try
{
writeSSTable(file, CompressionParams.snappy(chunkLength), 10);
- CompressionMetadata metadata = new CompressionMetadata(filename + ".metadata", file.length(), ChecksumType.CRC32);
+ CompressionMetadata metadata = new CompressionMetadata(filename + ".metadata", file.length(), true);
long chunks = 2761628520L;
long midPosition = (chunks / 2L) * chunkLength;
int idx = 8 * (int) (midPosition / chunkLength); // before patch
assertTrue("Expect integer overflow", idx < 0);
- try
- {
- metadata.chunkFor(midPosition);
- fail("Expected to throw EOF exception with chunk idx larger than total number of chunks in the sstable");
- }
- catch (CorruptSSTableException e)
- {
- assertTrue("Expect EOF, but got " + e.getCause(), e.getCause() instanceof EOFException);
- }
+ Throwable thrown = Assertions.catchThrowable(() -> metadata.chunkFor(midPosition));
+ assertThat(thrown).isInstanceOf(CorruptSSTableException.class)
+ .hasCauseInstanceOf(EOFException.class);
}
finally
{
@@ -165,12 +173,12 @@
}
}
- private static void testResetAndTruncate(File f, boolean compressed, boolean usemmap, int junkSize) throws IOException
+ private static void testResetAndTruncate(File f, boolean compressed, boolean usemmap, int junkSize, double minCompressRatio) throws IOException
{
final String filename = f.getAbsolutePath();
writeSSTable(f, compressed ? CompressionParams.snappy() : null, junkSize);
- CompressionMetadata compressionMetadata = compressed ? new CompressionMetadata(filename + ".metadata", f.length(), ChecksumType.CRC32) : null;
+ CompressionMetadata compressionMetadata = compressed ? new CompressionMetadata(filename + ".metadata", f.length(), true) : null;
try (FileHandle.Builder builder = new FileHandle.Builder(filename).mmapped(usemmap).withCompressionMetadata(compressionMetadata);
FileHandle fh = builder.complete();
RandomAccessReader reader = fh.createReader())
@@ -245,7 +253,7 @@
}
// open compression metadata and get chunk information
- CompressionMetadata meta = new CompressionMetadata(metadata.getPath(), file.length(), ChecksumType.CRC32);
+ CompressionMetadata meta = new CompressionMetadata(metadata.getPath(), file.length(), true);
CompressionMetadata.Chunk chunk = meta.chunkFor(0);
try (FileHandle.Builder builder = new FileHandle.Builder(file.getPath()).withCompressionMetadata(meta);
diff --git a/test/unit/org/apache/cassandra/io/compress/CompressedSequentialWriterReopenTest.java b/test/unit/org/apache/cassandra/io/compress/CompressedSequentialWriterReopenTest.java
index 1bc3454..461c13c 100644
--- a/test/unit/org/apache/cassandra/io/compress/CompressedSequentialWriterReopenTest.java
+++ b/test/unit/org/apache/cassandra/io/compress/CompressedSequentialWriterReopenTest.java
@@ -28,7 +28,6 @@
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.CQLTester;
-import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.utils.ByteBufferUtil;
import static org.junit.Assert.assertEquals;
@@ -85,7 +84,7 @@
execute("insert into %s (id, t) values (?, ?)", i, ByteBuffer.wrap(blob));
}
getCurrentColumnFamilyStore().forceBlockingFlush();
- DatabaseDescriptor.setSSTablePreempiveOpenIntervalInMB(1);
+ DatabaseDescriptor.setSSTablePreemptiveOpenIntervalInMB(1);
getCurrentColumnFamilyStore().forceMajorCompaction();
}
diff --git a/test/unit/org/apache/cassandra/io/compress/CompressedSequentialWriterTest.java b/test/unit/org/apache/cassandra/io/compress/CompressedSequentialWriterTest.java
index 52b18a9..57802cb 100644
--- a/test/unit/org/apache/cassandra/io/compress/CompressedSequentialWriterTest.java
+++ b/test/unit/org/apache/cassandra/io/compress/CompressedSequentialWriterTest.java
@@ -24,6 +24,7 @@
import java.nio.ByteBuffer;
import java.util.*;
+import static org.apache.cassandra.schema.CompressionParams.DEFAULT_CHUNK_LENGTH;
import static org.apache.commons.io.FileUtils.readFileToByteArray;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@@ -33,7 +34,7 @@
import org.junit.BeforeClass;
import org.junit.Test;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.ClusteringComparator;
@@ -42,7 +43,7 @@
import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
import org.apache.cassandra.io.util.*;
import org.apache.cassandra.schema.CompressionParams;
-import org.apache.cassandra.utils.ChecksumType;
+import org.apache.cassandra.utils.ByteBufferUtil;
public class CompressedSequentialWriterTest extends SequentialWriterTest
{
@@ -57,13 +58,22 @@
private void runTests(String testName) throws IOException
{
// Test small < 1 chunk data set
- testWrite(File.createTempFile(testName + "_small", "1"), 25);
+ testWrite(FileUtils.createTempFile(testName + "_small", "1"), 25, false);
// Test to confirm pipeline w/chunk-aligned data writes works
- testWrite(File.createTempFile(testName + "_chunkAligned", "1"), CompressionParams.DEFAULT_CHUNK_LENGTH);
+ testWrite(FileUtils.createTempFile(testName + "_chunkAligned", "1"), DEFAULT_CHUNK_LENGTH, false);
// Test to confirm pipeline on non-chunk boundaries works
- testWrite(File.createTempFile(testName + "_large", "1"), CompressionParams.DEFAULT_CHUNK_LENGTH * 3 + 100);
+ testWrite(FileUtils.createTempFile(testName + "_large", "1"), DEFAULT_CHUNK_LENGTH * 3 + 100, false);
+
+ // Test small < 1 chunk data set
+ testWrite(FileUtils.createTempFile(testName + "_small", "2"), 25, true);
+
+ // Test to confirm pipeline w/chunk-aligned data writes works
+ testWrite(FileUtils.createTempFile(testName + "_chunkAligned", "2"), DEFAULT_CHUNK_LENGTH, true);
+
+ // Test to confirm pipeline on non-chunk boundaries works
+ testWrite(FileUtils.createTempFile(testName + "_large", "2"), DEFAULT_CHUNK_LENGTH * 3 + 100, true);
}
@Test
@@ -87,7 +97,21 @@
runTests("Snappy");
}
- private void testWrite(File f, int bytesToTest) throws IOException
+ @Test
+ public void testZSTDWriter() throws IOException
+ {
+ compressionParameters = CompressionParams.zstd();
+ runTests("ZSTD");
+ }
+
+ @Test
+ public void testNoopWriter() throws IOException
+ {
+ compressionParameters = CompressionParams.noop();
+ runTests("Noop");
+ }
+
+ private void testWrite(File f, int bytesToTest, boolean useMemmap) throws IOException
{
final String filename = f.getAbsolutePath();
MetadataCollector sstableMetadataCollector = new MetadataCollector(new ClusteringComparator(Collections.singletonList(BytesType.instance)));
@@ -112,14 +136,14 @@
DataPosition mark = writer.mark();
// Write enough garbage to transition chunk
- for (int i = 0; i < CompressionParams.DEFAULT_CHUNK_LENGTH; i++)
+ for (int i = 0; i < DEFAULT_CHUNK_LENGTH; i++)
{
writer.write((byte)i);
}
- if (bytesToTest <= CompressionParams.DEFAULT_CHUNK_LENGTH)
- assertEquals(writer.getLastFlushOffset(), CompressionParams.DEFAULT_CHUNK_LENGTH);
+ if (bytesToTest <= DEFAULT_CHUNK_LENGTH)
+ assertEquals(writer.getLastFlushOffset(), DEFAULT_CHUNK_LENGTH);
else
- assertTrue(writer.getLastFlushOffset() % CompressionParams.DEFAULT_CHUNK_LENGTH == 0);
+ assertTrue(writer.getLastFlushOffset() % DEFAULT_CHUNK_LENGTH == 0);
writer.resetAndTruncate(mark);
writer.write(dataPost);
@@ -127,7 +151,7 @@
}
assert f.exists();
- try (FileHandle.Builder builder = new FileHandle.Builder(filename).withCompressionMetadata(new CompressionMetadata(filename + ".metadata", f.length(), ChecksumType.CRC32));
+ try (FileHandle.Builder builder = new FileHandle.Builder(filename).withCompressionMetadata(new CompressionMetadata(filename + ".metadata", f.length(), true));
FileHandle fh = builder.complete();
RandomAccessReader reader = fh.createReader())
{
@@ -154,6 +178,81 @@
}
}
+ @Test
+ public void testShortUncompressedChunk() throws IOException
+ {
+ // Test uncompressed chunk below threshold (CASSANDRA-14892)
+ compressionParameters = CompressionParams.lz4(DEFAULT_CHUNK_LENGTH, DEFAULT_CHUNK_LENGTH);
+ testWrite(FileUtils.createTempFile("14892", "1"), compressionParameters.maxCompressedLength() - 1, false);
+ }
+
+ @Test
+ public void testUncompressedChunks() throws IOException
+ {
+ for (double ratio = 1.25; ratio >= 1; ratio -= 1.0/16)
+ testUncompressedChunks(ratio);
+ }
+
+ private void testUncompressedChunks(double ratio) throws IOException
+ {
+ for (int compressedSizeExtra : new int[] {-3, 0, 1, 3, 15, 1051})
+ testUncompressedChunks(ratio, compressedSizeExtra);
+ }
+
+ private void testUncompressedChunks(double ratio, int compressedSizeExtra) throws IOException
+ {
+ for (int size = (int) (DEFAULT_CHUNK_LENGTH / ratio - 5); size <= DEFAULT_CHUNK_LENGTH / ratio + 5; ++size)
+ testUncompressedChunks(size, ratio, compressedSizeExtra);
+ }
+
+ private void testUncompressedChunks(int size, double ratio, int extra) throws IOException
+ {
+ // System.out.format("size %d ratio %f extra %d\n", size, ratio, extra);
+ ByteBuffer b = ByteBuffer.allocate(size);
+ ByteBufferUtil.writeZeroes(b, size);
+ b.flip();
+
+ File f = FileUtils.createTempFile("testUncompressedChunks", "1");
+ String filename = f.getPath();
+ MetadataCollector sstableMetadataCollector = new MetadataCollector(new ClusteringComparator(Collections.singletonList(BytesType.instance)));
+ compressionParameters = new CompressionParams(MockCompressor.class.getTypeName(),
+ MockCompressor.paramsFor(ratio, extra),
+ DEFAULT_CHUNK_LENGTH, ratio);
+ try (CompressedSequentialWriter writer = new CompressedSequentialWriter(f, f.getPath() + ".metadata",
+ null, SequentialWriterOption.DEFAULT,
+ compressionParameters,
+ sstableMetadataCollector))
+ {
+ writer.write(b);
+ writer.finish();
+ b.flip();
+ }
+
+ assert f.exists();
+ try (FileHandle.Builder builder = new FileHandle.Builder(filename).withCompressionMetadata(new CompressionMetadata(filename + ".metadata", f.length(), true));
+ FileHandle fh = builder.complete();
+ RandomAccessReader reader = fh.createReader())
+ {
+ assertEquals(size, reader.length());
+ byte[] result = new byte[(int)reader.length()];
+
+ reader.readFully(result);
+ assert(reader.isEOF());
+
+ assert Arrays.equals(b.array(), result);
+ }
+ finally
+ {
+ if (f.exists())
+ f.delete();
+ File metadata = new File(f + ".metadata");
+ if (metadata.exists())
+ metadata.delete();
+ }
+
+ }
+
+
private ByteBuffer makeBB(int size)
{
return compressionParameters.getSstableCompressor().preferredBufferType().allocate(size);
@@ -174,7 +273,7 @@
public void resetAndTruncateTest()
{
File tempFile = new File(Files.createTempDir(), "reset.txt");
- File offsetsFile = FileUtils.createTempFile("compressedsequentialwriter.offset", "test");
+ File offsetsFile = FileUtils.createDeletableTempFile("compressedsequentialwriter.offset", "test");
final int bufferSize = 48;
final int writeSize = 64;
byte[] toWrite = new byte[writeSize];
@@ -222,6 +321,7 @@
private static class TestableCSW extends TestableSW
{
final File offsetsFile;
+ static final int MAX_COMPRESSED = BUFFER_SIZE * 10; // Always compress for this test.
private TestableCSW() throws IOException
{
@@ -233,7 +333,7 @@
{
this(file, offsetsFile, new CompressedSequentialWriter(file, offsetsFile.getPath(),
null, SequentialWriterOption.DEFAULT,
- CompressionParams.lz4(BUFFER_SIZE),
+ CompressionParams.lz4(BUFFER_SIZE, MAX_COMPRESSED),
new MetadataCollector(new ClusteringComparator(UTF8Type.instance))));
}
@@ -262,6 +362,7 @@
Assert.assertTrue(offsets.readUTF().endsWith("LZ4Compressor"));
Assert.assertEquals(0, offsets.readInt());
Assert.assertEquals(BUFFER_SIZE, offsets.readInt());
+ Assert.assertEquals(MAX_COMPRESSED, offsets.readInt());
Assert.assertEquals(fullContents.length, offsets.readLong());
Assert.assertEquals(2, offsets.readInt());
Assert.assertEquals(0, offsets.readLong());
diff --git a/test/unit/org/apache/cassandra/io/compress/CompressorTest.java b/test/unit/org/apache/cassandra/io/compress/CompressorTest.java
index 617e04e..29e8453 100644
--- a/test/unit/org/apache/cassandra/io/compress/CompressorTest.java
+++ b/test/unit/org/apache/cassandra/io/compress/CompressorTest.java
@@ -31,6 +31,7 @@
import org.junit.Assert;
import org.junit.Test;
+import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.io.util.RandomAccessReader;
import org.apache.cassandra.utils.ByteBufferUtil;
@@ -41,7 +42,9 @@
ICompressor[] compressors = new ICompressor[] {
LZ4Compressor.create(Collections.<String, String>emptyMap()),
DeflateCompressor.create(Collections.<String, String>emptyMap()),
- SnappyCompressor.create(Collections.<String, String>emptyMap())
+ SnappyCompressor.create(Collections.<String, String>emptyMap()),
+ ZstdCompressor.create(Collections.emptyMap()),
+ NoopCompressor.create(Collections.emptyMap())
};
@Test
@@ -80,7 +83,7 @@
// need byte[] representation which direct buffers don't have
byte[] compressedBytes = new byte[compressed.capacity()];
- ByteBufferUtil.arrayCopy(compressed, outOffset, compressedBytes, outOffset, compressed.limit() - outOffset);
+ ByteBufferUtil.copyBytes(compressed, outOffset, compressedBytes, outOffset, compressed.limit() - outOffset);
final int decompressedLength = compressor.uncompress(compressedBytes, outOffset, compressed.remaining(), restored, restoreOffset);
@@ -121,7 +124,7 @@
src.flip();
// create a temp file
- File temp = File.createTempFile("tempfile", ".tmp");
+ File temp = FileUtils.createTempFile("tempfile", ".tmp");
temp.deleteOnExit();
// Prepend some random bytes to the output and compress
@@ -176,6 +179,20 @@
testByteBuffers();
}
+ @Test
+ public void testZstdByteBuffers() throws IOException
+ {
+ compressor = ZstdCompressor.create(Collections.<String, String>emptyMap());
+ testByteBuffers();
+ }
+
+ @Test
+ public void testNoopByteBuffers() throws IOException
+ {
+ compressor = NoopCompressor.create(Collections.emptyMap());
+ testByteBuffers();
+ }
+
private void testByteBuffers() throws IOException
{
assert compressor.supports(BufferType.OFF_HEAP);
diff --git a/test/unit/org/apache/cassandra/io/compress/MockCompressor.java b/test/unit/org/apache/cassandra/io/compress/MockCompressor.java
new file mode 100644
index 0000000..d57f4b7
--- /dev/null
+++ b/test/unit/org/apache/cassandra/io/compress/MockCompressor.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.compress;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Map;
+import java.util.Set;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+
+import org.apache.cassandra.utils.ByteBufferUtil;
+
+/**
+ * Mock compressor used to test the effect of different output sizes in compression.
+ * Writes only the size of the given buffer, and on decompression expands to sequence of that many 0s.
+ */
+public class MockCompressor implements ICompressor
+{
+ final int extra;
+ final double ratio;
+
+ public static Map<String, String> paramsFor(double ratio, int extra)
+ {
+ return ImmutableMap.of("extra", "" + extra, "ratio", "" + ratio);
+ }
+
+ public static MockCompressor create(Map<String, String> opts)
+ {
+ return new MockCompressor(Integer.parseInt(opts.get("extra")),
+ Double.parseDouble(opts.get("ratio")));
+ }
+
+ private MockCompressor(int extra, double ratio)
+ {
+ this.extra = extra;
+ this.ratio = ratio;
+ }
+
+ public int initialCompressedBufferLength(int chunkLength)
+ {
+ return (int) Math.ceil(chunkLength / ratio + extra);
+ }
+
+ public int uncompress(byte[] input, int inputOffset, int inputLength, byte[] output, int outputOffset)
+ throws IOException
+ {
+ final ByteBuffer outputBuffer = ByteBuffer.wrap(output, outputOffset, output.length - outputOffset);
+ uncompress(ByteBuffer.wrap(input, inputOffset, inputLength),
+ outputBuffer);
+ return outputBuffer.position();
+ }
+
+ public void compress(ByteBuffer input, ByteBuffer output) throws IOException
+ {
+ int inputLength = input.remaining();
+ int outputLength = initialCompressedBufferLength(inputLength);
+ // assume the input is all zeros, write its length and pad until the required size
+ output.putInt(inputLength);
+ for (int i = 4; i < outputLength; ++i)
+ output.put((byte) i);
+ input.position(input.limit());
+ }
+
+ public void uncompress(ByteBuffer input, ByteBuffer output) throws IOException
+ {
+ int outputLength = input.getInt();
+ ByteBufferUtil.writeZeroes(output, outputLength);
+ }
+
+ public BufferType preferredBufferType()
+ {
+ return BufferType.OFF_HEAP;
+ }
+
+ public boolean supports(BufferType bufferType)
+ {
+ return true;
+ }
+
+ public Set<String> supportedOptions()
+ {
+ return ImmutableSet.of("extra", "ratio");
+ }
+
+}
diff --git a/test/unit/org/apache/cassandra/io/compress/ZstdCompressorTest.java b/test/unit/org/apache/cassandra/io/compress/ZstdCompressorTest.java
new file mode 100644
index 0000000..70e32ad
--- /dev/null
+++ b/test/unit/org/apache/cassandra/io/compress/ZstdCompressorTest.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.compress;
+
+import java.util.Collections;
+
+import com.google.common.collect.ImmutableMap;
+import org.junit.Test;
+
+import com.github.luben.zstd.Zstd;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Zstd Compressor specific tests. General compressor tests are in {@link CompressorTest}
+ */
+public class ZstdCompressorTest
+{
+ @Test
+ public void emptyConfigurationUsesDefaultCompressionLevel()
+ {
+ ZstdCompressor compressor = ZstdCompressor.create(Collections.emptyMap());
+ assertEquals(ZstdCompressor.DEFAULT_COMPRESSION_LEVEL, compressor.getCompressionLevel());
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void badCompressionLevelParamThrowsExceptionMin()
+ {
+ ZstdCompressor.create(ImmutableMap.of(ZstdCompressor.COMPRESSION_LEVEL_OPTION_NAME, Integer.toString(Zstd.minCompressionLevel() - 1)));
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void badCompressionLevelParamThrowsExceptionMax()
+ {
+ ZstdCompressor.create(ImmutableMap.of(ZstdCompressor.COMPRESSION_LEVEL_OPTION_NAME, Integer.toString(Zstd.maxCompressionLevel() + 1)));
+ }
+}
diff --git a/test/unit/org/apache/cassandra/io/sstable/BigTableWriterTest.java b/test/unit/org/apache/cassandra/io/sstable/BigTableWriterTest.java
index 78964f4..9e3594b 100644
--- a/test/unit/org/apache/cassandra/io/sstable/BigTableWriterTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/BigTableWriterTest.java
@@ -23,7 +23,7 @@
import org.junit.BeforeClass;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.UpdateBuilder;
import org.apache.cassandra.db.ColumnFamilyStore;
@@ -64,24 +64,27 @@
private TestableBTW()
{
- this(cfs.getSSTablePath(cfs.getDirectories().getDirectoryForNewSSTables()));
+ this(cfs.newSSTableDescriptor(cfs.getDirectories().getDirectoryForNewSSTables()));
}
- private TestableBTW(String file)
+ private TestableBTW(Descriptor desc)
{
- this(file, SSTableTxnWriter.create(cfs, file, 0, 0, new SerializationHeader(true, cfs.metadata, cfs.metadata.partitionColumns(), EncodingStats.NO_STATS)));
+ this(desc, SSTableTxnWriter.create(cfs, desc, 0, 0, null, false,
+ new SerializationHeader(true, cfs.metadata(),
+ cfs.metadata().regularAndStaticColumns(),
+ EncodingStats.NO_STATS)));
}
- private TestableBTW(String file, SSTableTxnWriter sw)
+ private TestableBTW(Descriptor desc, SSTableTxnWriter sw)
{
super(sw);
- this.file = new File(file);
- this.descriptor = Descriptor.fromFilename(file);
+ this.file = new File(desc.filenameFor(Component.DATA));
+ this.descriptor = desc;
this.writer = sw;
for (int i = 0; i < 100; i++)
{
- UpdateBuilder update = UpdateBuilder.create(cfs.metadata, i);
+ UpdateBuilder update = UpdateBuilder.create(cfs.metadata(), i);
for (int j = 0; j < 10; j++)
update.newRow(j).add("val", SSTableRewriterTest.random(0, 1000));
writer.append(update.build().unfilteredIterator());
diff --git a/test/unit/org/apache/cassandra/io/sstable/CQLSSTableWriterClientTest.java b/test/unit/org/apache/cassandra/io/sstable/CQLSSTableWriterClientTest.java
index 273c400..61ac017 100644
--- a/test/unit/org/apache/cassandra/io/sstable/CQLSSTableWriterClientTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/CQLSSTableWriterClientTest.java
@@ -23,7 +23,6 @@
import com.google.common.io.Files;
import org.junit.After;
-import org.junit.AfterClass;
import org.junit.Before;
import org.junit.Test;
diff --git a/test/unit/org/apache/cassandra/io/sstable/CQLSSTableWriterTest.java b/test/unit/org/apache/cassandra/io/sstable/CQLSSTableWriterTest.java
index a400612..f035658 100644
--- a/test/unit/org/apache/cassandra/io/sstable/CQLSSTableWriterTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/CQLSSTableWriterTest.java
@@ -35,16 +35,16 @@
import org.apache.cassandra.config.*;
import org.apache.cassandra.cql3.*;
import org.apache.cassandra.cql3.functions.UDHelper;
+import org.apache.cassandra.cql3.functions.types.*;
import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.commitlog.CommitLog;
import org.apache.cassandra.dht.*;
import org.apache.cassandra.exceptions.*;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.TableMetadataRef;
import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.transport.ProtocolVersion;
import org.apache.cassandra.utils.*;
-import com.datastax.driver.core.DataType;
-import com.datastax.driver.core.ProtocolVersion;
-import com.datastax.driver.core.TypeCodec;
-import com.datastax.driver.core.UDTValue;
-import com.datastax.driver.core.UserType;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
@@ -60,6 +60,7 @@
@BeforeClass
public static void setup() throws Exception
{
+ CommitLog.instance.start();
SchemaLoader.cleanupAndLeaveDirs();
Keyspace.setInitialized();
StorageService.instance.initServer();
@@ -349,7 +350,7 @@
loadSSTables(dataDir, KS);
UntypedResultSet resultSet = QueryProcessor.executeInternal("SELECT * FROM " + KS + "." + TABLE);
- TypeCodec collectionCodec = UDHelper.codecFor(DataType.CollectionType.frozenList(tuple2Type));
+ TypeCodec collectionCodec = UDHelper.codecFor(DataType.CollectionType.list(tuple2Type));
TypeCodec tuple3Codec = UDHelper.codecFor(tuple3Type);
assertEquals(resultSet.size(), 100);
@@ -358,13 +359,13 @@
assertEquals(cnt,
row.getInt("k"));
List<UDTValue> values = (List<UDTValue>) collectionCodec.deserialize(row.getBytes("v1"),
- ProtocolVersion.NEWEST_SUPPORTED);
+ ProtocolVersion.CURRENT);
assertEquals(values.get(0).getInt("a"), cnt * 10);
assertEquals(values.get(0).getInt("b"), cnt * 20);
assertEquals(values.get(1).getInt("a"), cnt * 30);
assertEquals(values.get(1).getInt("b"), cnt * 40);
- UDTValue v2 = (UDTValue) tuple3Codec.deserialize(row.getBytes("v2"), ProtocolVersion.NEWEST_SUPPORTED);
+ UDTValue v2 = (UDTValue) tuple3Codec.deserialize(row.getBytes("v2"), ProtocolVersion.CURRENT);
assertEquals(v2.getInt("a"), cnt * 100);
assertEquals(v2.getInt("b"), cnt * 200);
@@ -427,7 +428,7 @@
assertEquals(cnt,
row.getInt("k"));
UDTValue nestedTpl = (UDTValue) nestedTupleCodec.deserialize(row.getBytes("v1"),
- ProtocolVersion.NEWEST_SUPPORTED);
+ ProtocolVersion.CURRENT);
assertEquals(nestedTpl.getInt("c"), cnt * 100);
UDTValue tpl = nestedTpl.getUDTValue("tpl");
assertEquals(tpl.getInt("a"), cnt * 200);
@@ -647,13 +648,13 @@
public void init(String keyspace)
{
this.keyspace = keyspace;
- for (Range<Token> range : StorageService.instance.getLocalRanges(ks))
- addRangeForEndpoint(range, FBUtilities.getBroadcastAddress());
+ for (Range<Token> range : StorageService.instance.getLocalReplicas(ks).ranges())
+ addRangeForEndpoint(range, FBUtilities.getBroadcastAddressAndPort());
}
- public CFMetaData getTableMetadata(String cfName)
+ public TableMetadataRef getTableMetadata(String cfName)
{
- return Schema.instance.getCFMetaData(keyspace, cfName);
+ return Schema.instance.getTableMetadataRef(keyspace, cfName);
}
}, new OutputHandler.SystemOutput(false, false));
diff --git a/test/unit/org/apache/cassandra/io/sstable/DescriptorTest.java b/test/unit/org/apache/cassandra/io/sstable/DescriptorTest.java
index 9c1dc84..5f79f57 100644
--- a/test/unit/org/apache/cassandra/io/sstable/DescriptorTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/DescriptorTest.java
@@ -29,6 +29,7 @@
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.Directories;
import org.apache.cassandra.io.sstable.format.SSTableFormat;
+import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.Pair;
@@ -44,7 +45,7 @@
public DescriptorTest() throws IOException
{
// create CF directories, one without CFID and one with it
- tempDataDir = File.createTempFile("DescriptorTest", null).getParentFile();
+ tempDataDir = FileUtils.createTempFile("DescriptorTest", null).getParentFile();
}
@BeforeClass
@@ -83,27 +84,19 @@
private void testFromFilenameFor(File dir)
{
- // normal
- checkFromFilename(new Descriptor(dir, ksname, cfname, 1, SSTableFormat.Type.BIG), false);
- // skip component (for streaming lock file)
- checkFromFilename(new Descriptor(dir, ksname, cfname, 2, SSTableFormat.Type.BIG), true);
+ checkFromFilename(new Descriptor(dir, ksname, cfname, 1, SSTableFormat.Type.BIG));
// secondary index
String idxName = "myidx";
File idxDir = new File(dir.getAbsolutePath() + File.separator + Directories.SECONDARY_INDEX_NAME_SEPARATOR + idxName);
- checkFromFilename(new Descriptor(idxDir, ksname, cfname + Directories.SECONDARY_INDEX_NAME_SEPARATOR + idxName, 4, SSTableFormat.Type.BIG), false);
-
- // legacy version
- checkFromFilename(new Descriptor("ja", dir, ksname, cfname, 1, SSTableFormat.Type.LEGACY), false);
- // legacy secondary index
- checkFromFilename(new Descriptor("ja", dir, ksname, cfname + Directories.SECONDARY_INDEX_NAME_SEPARATOR + idxName, 3, SSTableFormat.Type.LEGACY), false);
+ checkFromFilename(new Descriptor(idxDir, ksname, cfname + Directories.SECONDARY_INDEX_NAME_SEPARATOR + idxName, 4, SSTableFormat.Type.BIG));
}
- private void checkFromFilename(Descriptor original, boolean skipComponent)
+ private void checkFromFilename(Descriptor original)
{
- File file = new File(skipComponent ? original.baseFilename() : original.filenameFor(Component.DATA));
+ File file = new File(original.filenameFor(Component.DATA));
- Pair<Descriptor, String> pair = Descriptor.fromFilename(file.getParentFile(), file.getName(), skipComponent);
+ Pair<Descriptor, Component> pair = Descriptor.fromFilenameWithComponent(file);
Descriptor desc = pair.left;
assertEquals(original.directory, desc.directory);
@@ -111,15 +104,7 @@
assertEquals(original.cfname, desc.cfname);
assertEquals(original.version, desc.version);
assertEquals(original.generation, desc.generation);
-
- if (skipComponent)
- {
- assertNull(pair.right);
- }
- else
- {
- assertEquals(Component.DATA.name(), pair.right);
- }
+ assertEquals(Component.DATA, pair.right);
}
@Test
@@ -136,20 +121,10 @@
@Test
public void validateNames()
{
- // TODO tmp file name probably is not handled correctly after CASSANDRA-7066
String[] names = {
- // old formats
- "system-schema_keyspaces-jb-1-Data.db",
- //"system-schema_keyspaces-tmp-jb-1-Data.db",
- "system-schema_keyspaces-ka-1-big-Data.db",
- //"system-schema_keyspaces-tmp-ka-1-big-Data.db",
+ "ma-1-big-Data.db",
// 2ndary index
- "keyspace1-standard1.idx1-ka-1-big-Data.db",
- // new formats
- "la-1-big-Data.db",
- //"tmp-la-1-big-Data.db",
- // 2ndary index
- ".idx1" + File.separator + "la-1-big-Data.db",
+ ".idx1" + File.separator + "ma-1-big-Data.db",
};
for (String name : names)
diff --git a/test/unit/org/apache/cassandra/io/sstable/IndexSummaryManagerTest.java b/test/unit/org/apache/cassandra/io/sstable/IndexSummaryManagerTest.java
index b33ead2..d383e88 100644
--- a/test/unit/org/apache/cassandra/io/sstable/IndexSummaryManagerTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/IndexSummaryManagerTest.java
@@ -23,6 +23,7 @@
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Consumer;
+import java.util.stream.Collectors;
import com.google.common.base.Joiner;
import com.google.common.collect.Sets;
@@ -41,6 +42,7 @@
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.db.RowUpdateBuilder;
+import org.apache.cassandra.db.compaction.AntiCompactionTest;
import org.apache.cassandra.db.compaction.CompactionInfo;
import org.apache.cassandra.db.compaction.CompactionInterruptedException;
import org.apache.cassandra.db.compaction.CompactionManager;
@@ -53,11 +55,13 @@
import org.apache.cassandra.metrics.RestorableMeter;
import org.apache.cassandra.schema.CachingParams;
import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.MigrationManager;
+import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.utils.ByteBufferUtil;
import static com.google.common.collect.ImmutableMap.of;
import static java.util.Arrays.asList;
-import static org.apache.cassandra.db.compaction.AntiCompactionTest.assertOnDiskState;
+import static org.apache.cassandra.Util.assertOnDiskState;
import static org.apache.cassandra.io.sstable.Downsampling.BASE_SAMPLING_LEVEL;
import static org.apache.cassandra.io.sstable.IndexSummaryRedistribution.DOWNSAMPLE_THESHOLD;
import static org.apache.cassandra.io.sstable.IndexSummaryRedistribution.UPSAMPLE_THRESHOLD;
@@ -67,7 +71,6 @@
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
-
@RunWith(OrderedJUnit4ClassRunner.class)
public class IndexSummaryManagerTest
{
@@ -105,15 +108,15 @@
String cfname = CF_STANDARDLOWiINTERVAL; // index interval of 8, no key caching
Keyspace keyspace = Keyspace.open(ksname);
ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfname);
- originalMinIndexInterval = cfs.metadata.params.minIndexInterval;
- originalMaxIndexInterval = cfs.metadata.params.maxIndexInterval;
+ originalMinIndexInterval = cfs.metadata().params.minIndexInterval;
+ originalMaxIndexInterval = cfs.metadata().params.maxIndexInterval;
originalCapacity = IndexSummaryManager.instance.getMemoryPoolCapacityInMB();
}
@After
public void afterTest()
{
- for (CompactionInfo.Holder holder : CompactionMetrics.getCompactions())
+ for (CompactionInfo.Holder holder : CompactionManager.instance.active.getCompactions())
{
holder.stop();
}
@@ -122,8 +125,10 @@
String cfname = CF_STANDARDLOWiINTERVAL; // index interval of 8, no key caching
Keyspace keyspace = Keyspace.open(ksname);
ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfname);
- cfs.metadata.minIndexInterval(originalMinIndexInterval);
- cfs.metadata.maxIndexInterval(originalMaxIndexInterval);
+
+ MigrationManager.announceTableUpdate(cfs.metadata().unbuild().minIndexInterval(originalMinIndexInterval).build(), true);
+ MigrationManager.announceTableUpdate(cfs.metadata().unbuild().maxIndexInterval(originalMaxIndexInterval).build(), true);
+
IndexSummaryManager.instance.setMemoryPoolCapacityInMB(originalCapacity);
}
@@ -142,7 +147,7 @@
try (LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.UNKNOWN))
{
- sstables = redistributeSummaries(Collections.EMPTY_LIST, of(cfs.metadata.cfId, txn), originalOffHeapSize * sstables.size());
+ sstables = redistributeSummaries(Collections.EMPTY_LIST, of(cfs.metadata.id, txn), originalOffHeapSize * sstables.size());
}
for (SSTableReader sstable : sstables)
assertEquals(BASE_SAMPLING_LEVEL, sstable.getIndexSummarySamplingLevel());
@@ -155,7 +160,7 @@
for (int i = 0; i < numPartition; i++)
{
Row row = Util.getOnlyRowUnfiltered(Util.cmd(cfs, String.format("%3d", i)).build());
- Cell cell = row.getCell(cfs.metadata.getColumnDefinition(ByteBufferUtil.bytes("val")));
+ Cell cell = row.getCell(cfs.metadata().getColumn(ByteBufferUtil.bytes("val")));
assertNotNull(cell);
assertEquals(100, cell.value().array().length);
@@ -185,7 +190,7 @@
{
String key = String.format("%3d", p);
- new RowUpdateBuilder(cfs.metadata, 0, key)
+ new RowUpdateBuilder(cfs.metadata(), 0, key)
.clustering("column")
.add("val", value)
.build()
@@ -224,34 +229,34 @@
sstable.overrideReadMeter(new RestorableMeter(100.0, 100.0));
for (SSTableReader sstable : sstables)
- assertEquals(cfs.metadata.params.minIndexInterval, sstable.getEffectiveIndexInterval(), 0.001);
+ assertEquals(cfs.metadata().params.minIndexInterval, sstable.getEffectiveIndexInterval(), 0.001);
// double the min_index_interval
- cfs.metadata.minIndexInterval(originalMinIndexInterval * 2);
+ MigrationManager.announceTableUpdate(cfs.metadata().unbuild().minIndexInterval(originalMinIndexInterval * 2).build(), true);
IndexSummaryManager.instance.redistributeSummaries();
for (SSTableReader sstable : cfs.getLiveSSTables())
{
- assertEquals(cfs.metadata.params.minIndexInterval, sstable.getEffectiveIndexInterval(), 0.001);
- assertEquals(numRows / cfs.metadata.params.minIndexInterval, sstable.getIndexSummarySize());
+ assertEquals(cfs.metadata().params.minIndexInterval, sstable.getEffectiveIndexInterval(), 0.001);
+ assertEquals(numRows / cfs.metadata().params.minIndexInterval, sstable.getIndexSummarySize());
}
// return min_index_interval to its original value
- cfs.metadata.minIndexInterval(originalMinIndexInterval);
+ MigrationManager.announceTableUpdate(cfs.metadata().unbuild().minIndexInterval(originalMinIndexInterval).build(), true);
IndexSummaryManager.instance.redistributeSummaries();
for (SSTableReader sstable : cfs.getLiveSSTables())
{
- assertEquals(cfs.metadata.params.minIndexInterval, sstable.getEffectiveIndexInterval(), 0.001);
- assertEquals(numRows / cfs.metadata.params.minIndexInterval, sstable.getIndexSummarySize());
+ assertEquals(cfs.metadata().params.minIndexInterval, sstable.getEffectiveIndexInterval(), 0.001);
+ assertEquals(numRows / cfs.metadata().params.minIndexInterval, sstable.getIndexSummarySize());
}
// halve the min_index_interval, but constrain the available space to exactly what we have now; as a result,
// the summary shouldn't change
- cfs.metadata.minIndexInterval(originalMinIndexInterval / 2);
+ MigrationManager.announceTableUpdate(cfs.metadata().unbuild().minIndexInterval(originalMinIndexInterval / 2).build(), true);
SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
long summarySpace = sstable.getIndexSummaryOffHeapSize();
try (LifecycleTransaction txn = cfs.getTracker().tryModify(asList(sstable), OperationType.UNKNOWN))
{
- redistributeSummaries(Collections.EMPTY_LIST, of(cfs.metadata.cfId, txn), summarySpace);
+ redistributeSummaries(Collections.EMPTY_LIST, of(cfs.metadata.id, txn), summarySpace);
}
sstable = cfs.getLiveSSTables().iterator().next();
@@ -263,7 +268,7 @@
int previousSize = sstable.getIndexSummarySize();
try (LifecycleTransaction txn = cfs.getTracker().tryModify(asList(sstable), OperationType.UNKNOWN))
{
- redistributeSummaries(Collections.EMPTY_LIST, of(cfs.metadata.cfId, txn), (long) Math.ceil(summarySpace * 1.5));
+ redistributeSummaries(Collections.EMPTY_LIST, of(cfs.metadata.id, txn), (long) Math.ceil(summarySpace * 1.5));
}
sstable = cfs.getLiveSSTables().iterator().next();
assertEquals(previousSize * 1.5, (double) sstable.getIndexSummarySize(), 1);
@@ -271,10 +276,10 @@
// return min_index_interval to it's original value (double it), but only give the summary enough space
// to have an effective index interval of twice the new min
- cfs.metadata.minIndexInterval(originalMinIndexInterval);
+ MigrationManager.announceTableUpdate(cfs.metadata().unbuild().minIndexInterval(originalMinIndexInterval).build(), true);
try (LifecycleTransaction txn = cfs.getTracker().tryModify(asList(sstable), OperationType.UNKNOWN))
{
- redistributeSummaries(Collections.EMPTY_LIST, of(cfs.metadata.cfId, txn), (long) Math.ceil(summarySpace / 2.0));
+ redistributeSummaries(Collections.EMPTY_LIST, of(cfs.metadata.id, txn), (long) Math.ceil(summarySpace / 2.0));
}
sstable = cfs.getLiveSSTables().iterator().next();
assertEquals(originalMinIndexInterval * 2, sstable.getEffectiveIndexInterval(), 0.001);
@@ -283,14 +288,14 @@
// raise the min_index_interval above our current effective interval, but set the max_index_interval lower
// than what we actually have space for (meaning the index summary would ideally be smaller, but this would
// result in an effective interval above the new max)
- cfs.metadata.minIndexInterval(originalMinIndexInterval * 4);
- cfs.metadata.maxIndexInterval(originalMinIndexInterval * 4);
+ MigrationManager.announceTableUpdate(cfs.metadata().unbuild().minIndexInterval(originalMinIndexInterval * 4).build(), true);
+ MigrationManager.announceTableUpdate(cfs.metadata().unbuild().maxIndexInterval(originalMinIndexInterval * 4).build(), true);
try (LifecycleTransaction txn = cfs.getTracker().tryModify(asList(sstable), OperationType.UNKNOWN))
{
- redistributeSummaries(Collections.EMPTY_LIST, of(cfs.metadata.cfId, txn), 10);
+ redistributeSummaries(Collections.EMPTY_LIST, of(cfs.metadata.id, txn), 10);
}
sstable = cfs.getLiveSSTables().iterator().next();
- assertEquals(cfs.metadata.params.minIndexInterval, sstable.getEffectiveIndexInterval(), 0.001);
+ assertEquals(cfs.metadata().params.minIndexInterval, sstable.getEffectiveIndexInterval(), 0.001);
}
@Test
@@ -310,35 +315,35 @@
try (LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.UNKNOWN))
{
- redistributeSummaries(Collections.EMPTY_LIST, of(cfs.metadata.cfId, txn), 10);
+ redistributeSummaries(Collections.EMPTY_LIST, of(cfs.metadata.id, txn), 10);
}
sstables = new ArrayList<>(cfs.getLiveSSTables());
for (SSTableReader sstable : sstables)
- assertEquals(cfs.metadata.params.maxIndexInterval, sstable.getEffectiveIndexInterval(), 0.01);
+ assertEquals(cfs.metadata().params.maxIndexInterval, sstable.getEffectiveIndexInterval(), 0.01);
// halve the max_index_interval
- cfs.metadata.maxIndexInterval(cfs.metadata.params.maxIndexInterval / 2);
+ MigrationManager.announceTableUpdate(cfs.metadata().unbuild().maxIndexInterval(cfs.metadata().params.maxIndexInterval / 2).build(), true);
try (LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.UNKNOWN))
{
- redistributeSummaries(Collections.EMPTY_LIST, of(cfs.metadata.cfId, txn), 1);
+ redistributeSummaries(Collections.EMPTY_LIST, of(cfs.metadata.id, txn), 1);
}
sstables = new ArrayList<>(cfs.getLiveSSTables());
for (SSTableReader sstable : sstables)
{
- assertEquals(cfs.metadata.params.maxIndexInterval, sstable.getEffectiveIndexInterval(), 0.01);
- assertEquals(numRows / cfs.metadata.params.maxIndexInterval, sstable.getIndexSummarySize());
+ assertEquals(cfs.metadata().params.maxIndexInterval, sstable.getEffectiveIndexInterval(), 0.01);
+ assertEquals(numRows / cfs.metadata().params.maxIndexInterval, sstable.getIndexSummarySize());
}
// return max_index_interval to its original value
- cfs.metadata.maxIndexInterval(cfs.metadata.params.maxIndexInterval * 2);
+ MigrationManager.announceTableUpdate(cfs.metadata().unbuild().maxIndexInterval(cfs.metadata().params.maxIndexInterval * 2).build(), true);
try (LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.UNKNOWN))
{
- redistributeSummaries(Collections.EMPTY_LIST, of(cfs.metadata.cfId, txn), 1);
+ redistributeSummaries(Collections.EMPTY_LIST, of(cfs.metadata.id, txn), 1);
}
for (SSTableReader sstable : cfs.getLiveSSTables())
{
- assertEquals(cfs.metadata.params.maxIndexInterval, sstable.getEffectiveIndexInterval(), 0.01);
- assertEquals(numRows / cfs.metadata.params.maxIndexInterval, sstable.getIndexSummarySize());
+ assertEquals(cfs.metadata().params.maxIndexInterval, sstable.getEffectiveIndexInterval(), 0.01);
+ assertEquals(numRows / cfs.metadata().params.maxIndexInterval, sstable.getIndexSummarySize());
}
}
@@ -353,7 +358,7 @@
int numRows = 256;
createSSTables(ksname, cfname, numSSTables, numRows);
- int minSamplingLevel = (BASE_SAMPLING_LEVEL * cfs.metadata.params.minIndexInterval) / cfs.metadata.params.maxIndexInterval;
+ int minSamplingLevel = (BASE_SAMPLING_LEVEL * cfs.metadata().params.minIndexInterval) / cfs.metadata().params.maxIndexInterval;
List<SSTableReader> sstables = new ArrayList<>(cfs.getLiveSSTables());
for (SSTableReader sstable : sstables)
@@ -364,7 +369,7 @@
// there should be enough space to not downsample anything
try (LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.UNKNOWN))
{
- sstables = redistributeSummaries(Collections.EMPTY_LIST, of(cfs.metadata.cfId, txn), (singleSummaryOffHeapSpace * numSSTables));
+ sstables = redistributeSummaries(Collections.EMPTY_LIST, of(cfs.metadata.id, txn), (singleSummaryOffHeapSpace * numSSTables));
}
for (SSTableReader sstable : sstables)
assertEquals(BASE_SAMPLING_LEVEL, sstable.getIndexSummarySamplingLevel());
@@ -375,7 +380,7 @@
assert sstables.size() == 4;
try (LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.UNKNOWN))
{
- sstables = redistributeSummaries(Collections.EMPTY_LIST, of(cfs.metadata.cfId, txn), (singleSummaryOffHeapSpace * (numSSTables / 2)));
+ sstables = redistributeSummaries(Collections.EMPTY_LIST, of(cfs.metadata.id, txn), (singleSummaryOffHeapSpace * (numSSTables / 2)));
}
for (SSTableReader sstable : sstables)
assertEquals(BASE_SAMPLING_LEVEL / 2, sstable.getIndexSummarySamplingLevel());
@@ -384,7 +389,7 @@
// everything should get cut to a quarter
try (LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.UNKNOWN))
{
- sstables = redistributeSummaries(Collections.EMPTY_LIST, of(cfs.metadata.cfId, txn), (singleSummaryOffHeapSpace * (numSSTables / 4)));
+ sstables = redistributeSummaries(Collections.EMPTY_LIST, of(cfs.metadata.id, txn), (singleSummaryOffHeapSpace * (numSSTables / 4)));
}
for (SSTableReader sstable : sstables)
assertEquals(BASE_SAMPLING_LEVEL / 4, sstable.getIndexSummarySamplingLevel());
@@ -393,7 +398,7 @@
// upsample back up to half
try (LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.UNKNOWN))
{
- sstables = redistributeSummaries(Collections.EMPTY_LIST, of(cfs.metadata.cfId, txn), (singleSummaryOffHeapSpace * (numSSTables / 2) + 4));
+ sstables = redistributeSummaries(Collections.EMPTY_LIST, of(cfs.metadata.id, txn), (singleSummaryOffHeapSpace * (numSSTables / 2) + 4));
}
assert sstables.size() == 4;
for (SSTableReader sstable : sstables)
@@ -403,7 +408,7 @@
// upsample back up to the original index summary
try (LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.UNKNOWN))
{
- sstables = redistributeSummaries(Collections.EMPTY_LIST, of(cfs.metadata.cfId, txn), (singleSummaryOffHeapSpace * numSSTables));
+ sstables = redistributeSummaries(Collections.EMPTY_LIST, of(cfs.metadata.id, txn), (singleSummaryOffHeapSpace * numSSTables));
}
for (SSTableReader sstable : sstables)
assertEquals(BASE_SAMPLING_LEVEL, sstable.getIndexSummarySamplingLevel());
@@ -415,7 +420,7 @@
sstables.get(1).overrideReadMeter(new RestorableMeter(50.0, 50.0));
try (LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.UNKNOWN))
{
- sstables = redistributeSummaries(Collections.EMPTY_LIST, of(cfs.metadata.cfId, txn), (singleSummaryOffHeapSpace * 3));
+ sstables = redistributeSummaries(Collections.EMPTY_LIST, of(cfs.metadata.id, txn), (singleSummaryOffHeapSpace * 3));
}
Collections.sort(sstables, hotnessComparator);
assertEquals(BASE_SAMPLING_LEVEL / 2, sstables.get(0).getIndexSummarySamplingLevel());
@@ -431,7 +436,7 @@
sstables.get(1).overrideReadMeter(new RestorableMeter(higherRate, higherRate));
try (LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.UNKNOWN))
{
- sstables = redistributeSummaries(Collections.EMPTY_LIST, of(cfs.metadata.cfId, txn), (singleSummaryOffHeapSpace * 3));
+ sstables = redistributeSummaries(Collections.EMPTY_LIST, of(cfs.metadata.id, txn), (singleSummaryOffHeapSpace * 3));
}
Collections.sort(sstables, hotnessComparator);
assertEquals(BASE_SAMPLING_LEVEL / 2, sstables.get(0).getIndexSummarySamplingLevel());
@@ -449,7 +454,7 @@
try (LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.UNKNOWN))
{
- sstables = redistributeSummaries(Collections.EMPTY_LIST, of(cfs.metadata.cfId, txn), (singleSummaryOffHeapSpace * 3) + 50);
+ sstables = redistributeSummaries(Collections.EMPTY_LIST, of(cfs.metadata.id, txn), (singleSummaryOffHeapSpace * 3) + 50);
}
Collections.sort(sstables, hotnessComparator);
@@ -473,7 +478,7 @@
sstables.get(3).overrideReadMeter(new RestorableMeter(128.0, 128.0));
try (LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.UNKNOWN))
{
- sstables = redistributeSummaries(Collections.EMPTY_LIST, of(cfs.metadata.cfId, txn), (long) (singleSummaryOffHeapSpace + (singleSummaryOffHeapSpace * (92.0 / BASE_SAMPLING_LEVEL))));
+ sstables = redistributeSummaries(Collections.EMPTY_LIST, of(cfs.metadata.id, txn), (long) (singleSummaryOffHeapSpace + (singleSummaryOffHeapSpace * (92.0 / BASE_SAMPLING_LEVEL))));
}
Collections.sort(sstables, hotnessComparator);
assertEquals(1, sstables.get(0).getIndexSummarySize()); // at the min sampling level
@@ -486,7 +491,7 @@
// Don't leave enough space for even the minimal index summaries
try (LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.UNKNOWN))
{
- sstables = redistributeSummaries(Collections.EMPTY_LIST, of(cfs.metadata.cfId, txn), 10);
+ sstables = redistributeSummaries(Collections.EMPTY_LIST, of(cfs.metadata.id, txn), 10);
}
for (SSTableReader sstable : sstables)
assertEquals(1, sstable.getIndexSummarySize()); // at the min sampling level
@@ -509,7 +514,7 @@
for (int row = 0; row < numRows; row++)
{
String key = String.format("%3d", row);
- new RowUpdateBuilder(cfs.metadata, 0, key)
+ new RowUpdateBuilder(cfs.metadata(), 0, key)
.clustering("column")
.add("val", value)
.build()
@@ -529,7 +534,7 @@
{
sstable = sstable.cloneWithNewSummarySamplingLevel(cfs, samplingLevel);
assertEquals(samplingLevel, sstable.getIndexSummarySamplingLevel());
- int expectedSize = (numRows * samplingLevel) / (sstable.metadata.params.minIndexInterval * BASE_SAMPLING_LEVEL);
+ int expectedSize = (numRows * samplingLevel) / (cfs.metadata().params.minIndexInterval * BASE_SAMPLING_LEVEL);
assertEquals(expectedSize, sstable.getIndexSummarySize(), 1);
txn.update(sstable, true);
txn.checkpoint();
@@ -575,7 +580,7 @@
for (int row = 0; row < numRows; row++)
{
String key = String.format("%3d", row);
- new RowUpdateBuilder(cfs.metadata, 0, key)
+ new RowUpdateBuilder(cfs.metadata(), 0, key)
.clustering("column")
.add("val", value)
.build()
@@ -584,20 +589,20 @@
cfs.forceBlockingFlush();
}
- assertTrue(manager.getAverageIndexInterval() >= cfs.metadata.params.minIndexInterval);
+ assertTrue(manager.getAverageIndexInterval() >= cfs.metadata().params.minIndexInterval);
Map<String, Integer> intervals = manager.getIndexIntervals();
for (Map.Entry<String, Integer> entry : intervals.entrySet())
if (entry.getKey().contains(CF_STANDARDLOWiINTERVAL))
- assertEquals(cfs.metadata.params.minIndexInterval, entry.getValue(), 0.001);
+ assertEquals(cfs.metadata().params.minIndexInterval, entry.getValue(), 0.001);
manager.setMemoryPoolCapacityInMB(0);
manager.redistributeSummaries();
- assertTrue(manager.getAverageIndexInterval() > cfs.metadata.params.minIndexInterval);
+ assertTrue(manager.getAverageIndexInterval() > cfs.metadata().params.minIndexInterval);
intervals = manager.getIndexIntervals();
for (Map.Entry<String, Integer> entry : intervals.entrySet())
{
if (entry.getKey().contains(CF_STANDARDLOWiINTERVAL))
- assertTrue(entry.getValue() >= cfs.metadata.params.minIndexInterval);
+ assertTrue(entry.getValue() >= cfs.metadata().params.minIndexInterval);
}
}
@@ -610,7 +615,7 @@
@Test
public void testCancelIndexInterrupt() throws Exception
{
- testCancelIndexHelper((cfs) -> CompactionManager.instance.interruptCompactionFor(Collections.singleton(cfs.metadata), false));
+ testCancelIndexHelper((cfs) -> CompactionManager.instance.interruptCompactionFor(Collections.singleton(cfs.metadata()), (sstable) -> true, false));
}
public void testCancelIndexHelper(Consumer<ColumnFamilyStore> cancelFunction) throws Exception
@@ -619,11 +624,15 @@
String cfname = CF_STANDARDLOWiINTERVAL; // index interval of 8, no key caching
Keyspace keyspace = Keyspace.open(ksname);
final ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfname);
- final int numSSTables = 4;
+ cfs.disableAutoCompaction();
+ final int numSSTables = 8;
int numRows = 256;
createSSTables(ksname, cfname, numSSTables, numRows);
- final List<SSTableReader> sstables = new ArrayList<>(cfs.getLiveSSTables());
+ List<SSTableReader> allSSTables = new ArrayList<>(cfs.getLiveSSTables());
+ List<SSTableReader> sstables = allSSTables.subList(0, 4);
+ List<SSTableReader> compacting = allSSTables.subList(4, 8);
+
for (SSTableReader sstable : sstables)
sstable.overrideReadMeter(new RestorableMeter(100.0, 100.0));
@@ -633,53 +642,73 @@
final AtomicReference<CompactionInterruptedException> exception = new AtomicReference<>();
// barrier to control when redistribution runs
final CountDownLatch barrier = new CountDownLatch(1);
-
- Thread t = NamedThreadFactory.createThread(new Runnable()
+ CompactionInfo.Holder ongoingCompaction = new CompactionInfo.Holder()
{
- public void run()
+ public CompactionInfo getCompactionInfo()
{
- try
+ return new CompactionInfo(cfs.metadata(), OperationType.UNKNOWN, 0, 0, UUID.randomUUID(), compacting);
+ }
+
+ public boolean isGlobal()
+ {
+ return false;
+ }
+ };
+ try (LifecycleTransaction ignored = cfs.getTracker().tryModify(compacting, OperationType.UNKNOWN))
+ {
+ CompactionManager.instance.active.beginCompaction(ongoingCompaction);
+
+ Thread t = NamedThreadFactory.createThread(new Runnable()
+ {
+ public void run()
{
- // Don't leave enough space for even the minimal index summaries
- try (LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.UNKNOWN))
+ try
{
- IndexSummaryManager.redistributeSummaries(new ObservableRedistribution(Collections.EMPTY_LIST,
- of(cfs.metadata.cfId, txn),
- singleSummaryOffHeapSpace,
- barrier));
+ // Don't leave enough space for even the minimal index summaries
+ try (LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.UNKNOWN))
+ {
+ IndexSummaryManager.redistributeSummaries(new ObservableRedistribution(of(cfs.metadata.id, txn),
+ 0,
+ singleSummaryOffHeapSpace,
+ barrier));
+ }
+ }
+ catch (CompactionInterruptedException ex)
+ {
+ exception.set(ex);
+ }
+ catch (IOException ignored)
+ {
}
}
- catch (CompactionInterruptedException ex)
- {
- exception.set(ex);
- }
- catch (IOException ignored)
- {
- }
- }
- });
- t.start();
- while (CompactionManager.instance.getActiveCompactions() == 0 && t.isAlive())
- Thread.sleep(1);
- // to ensure that the stop condition check in IndexSummaryRedistribution::redistributeSummaries
- // is made *after* the halt request is made to the CompactionManager, don't allow the redistribution
- // to proceed until stopCompaction has been called.
- cancelFunction.accept(cfs);
- // allows the redistribution to proceed
- barrier.countDown();
- t.join();
+ });
+
+ t.start();
+ while (CompactionManager.instance.getActiveCompactions() < 2 && t.isAlive())
+ Thread.sleep(1);
+ // to ensure that the stop condition check in IndexSummaryRedistribution::redistributeSummaries
+ // is made *after* the halt request is made to the CompactionManager, don't allow the redistribution
+ // to proceed until stopCompaction has been called.
+ cancelFunction.accept(cfs);
+ // allows the redistribution to proceed
+ barrier.countDown();
+ t.join();
+ }
+ finally
+ {
+ CompactionManager.instance.active.finishCompaction(ongoingCompaction);
+ }
assertNotNull("Expected compaction interrupted exception", exception.get());
- assertTrue("Expected no active compactions", CompactionMetrics.getCompactions().isEmpty());
+ assertTrue("Expected no active compactions", CompactionManager.instance.active.getCompactions().isEmpty());
- Set<SSTableReader> beforeRedistributionSSTables = new HashSet<>(sstables);
+ Set<SSTableReader> beforeRedistributionSSTables = new HashSet<>(allSSTables);
Set<SSTableReader> afterCancelSSTables = new HashSet<>(cfs.getLiveSSTables());
Set<SSTableReader> disjoint = Sets.symmetricDifference(beforeRedistributionSSTables, afterCancelSSTables);
assertTrue(String.format("Mismatched files before and after cancelling redistribution: %s",
Joiner.on(",").join(disjoint)),
disjoint.isEmpty());
-
- assertOnDiskState(cfs, numSSTables);
+ assertOnDiskState(cfs, 8);
validateData(cfs, numRows);
}
@@ -706,7 +735,7 @@
{
try (AutoCloseable toresume = CompactionManager.instance.pauseGlobalCompaction())
{
- sstables = redistributeSummaries(Collections.emptyList(), of(cfs.metadata.cfId, txn), (singleSummaryOffHeapSpace * (numSSTables / 2)));
+ sstables = redistributeSummaries(Collections.emptyList(), of(cfs.metadata().id, txn), (singleSummaryOffHeapSpace * (numSSTables / 2)));
fail("The redistribution should fail - we got paused before adding to active compactions, but after marking compacting");
}
}
@@ -721,12 +750,13 @@
}
private static List<SSTableReader> redistributeSummaries(List<SSTableReader> compacting,
- Map<UUID, LifecycleTransaction> transactions,
+ Map<TableId, LifecycleTransaction> transactions,
long memoryPoolBytes)
throws IOException
{
- return IndexSummaryManager.redistributeSummaries(new IndexSummaryRedistribution(compacting,
- transactions,
+ long nonRedistributingOffHeapSize = compacting.stream().mapToLong(SSTableReader::getIndexSummaryOffHeapSize).sum();
+ return IndexSummaryManager.redistributeSummaries(new IndexSummaryRedistribution(transactions,
+ nonRedistributingOffHeapSize,
memoryPoolBytes));
}
@@ -734,12 +764,12 @@
{
CountDownLatch barrier;
- ObservableRedistribution(List<SSTableReader> compacting,
- Map<UUID, LifecycleTransaction> transactions,
+ ObservableRedistribution(Map<TableId, LifecycleTransaction> transactions,
+ long nonRedistributingOffHeapSize,
long memoryPoolBytes,
CountDownLatch barrier)
{
- super(compacting, transactions, memoryPoolBytes);
+ super(transactions, nonRedistributingOffHeapSize, memoryPoolBytes);
this.barrier = barrier;
}
diff --git a/test/unit/org/apache/cassandra/io/sstable/IndexSummaryRedistributionTest.java b/test/unit/org/apache/cassandra/io/sstable/IndexSummaryRedistributionTest.java
index 31a57e1..07a2212 100644
--- a/test/unit/org/apache/cassandra/io/sstable/IndexSummaryRedistributionTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/IndexSummaryRedistributionTest.java
@@ -38,6 +38,7 @@
import org.apache.cassandra.metrics.StorageMetrics;
import org.apache.cassandra.schema.CachingParams;
import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.MigrationManager;
import static org.junit.Assert.assertEquals;
@@ -78,23 +79,23 @@
long oldSize = 0;
for (SSTableReader sstable : sstables)
{
- assertEquals(cfs.metadata.params.minIndexInterval, sstable.getEffectiveIndexInterval(), 0.001);
+ assertEquals(cfs.metadata().params.minIndexInterval, sstable.getEffectiveIndexInterval(), 0.001);
oldSize += sstable.bytesOnDisk();
}
load = StorageMetrics.load.getCount();
long others = load - oldSize; // Other SSTables size, e.g. schema and other system SSTables
- int originalMinIndexInterval = cfs.metadata.params.minIndexInterval;
+ int originalMinIndexInterval = cfs.metadata().params.minIndexInterval;
// double the min_index_interval
- cfs.metadata.minIndexInterval(originalMinIndexInterval * 2);
+ MigrationManager.announceTableUpdate(cfs.metadata().unbuild().minIndexInterval(originalMinIndexInterval * 2).build(), true);
IndexSummaryManager.instance.redistributeSummaries();
long newSize = 0;
for (SSTableReader sstable : cfs.getLiveSSTables())
{
- assertEquals(cfs.metadata.params.minIndexInterval, sstable.getEffectiveIndexInterval(), 0.001);
- assertEquals(numRows / cfs.metadata.params.minIndexInterval, sstable.getIndexSummarySize());
+ assertEquals(cfs.metadata().params.minIndexInterval, sstable.getEffectiveIndexInterval(), 0.001);
+ assertEquals(numRows / cfs.metadata().params.minIndexInterval, sstable.getIndexSummarySize());
newSize += sstable.bytesOnDisk();
}
newSize += others;
@@ -118,7 +119,7 @@
for (int row = 0; row < numRows; row++)
{
String key = String.format("%3d", row);
- new RowUpdateBuilder(cfs.metadata, 0, key)
+ new RowUpdateBuilder(cfs.metadata(), 0, key)
.clustering("column")
.add("val", value)
.build()
diff --git a/test/unit/org/apache/cassandra/io/sstable/IndexSummaryTest.java b/test/unit/org/apache/cassandra/io/sstable/IndexSummaryTest.java
index d83b96c..01cd0dd 100644
--- a/test/unit/org/apache/cassandra/io/sstable/IndexSummaryTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/IndexSummaryTest.java
@@ -218,13 +218,13 @@
{
Pair<List<DecoratedKey>, IndexSummary> random = generateRandomIndex(100, 1);
DataOutputBuffer dos = new DataOutputBuffer();
- IndexSummary.serializer.serialize(random.right, dos, false);
+ IndexSummary.serializer.serialize(random.right, dos);
// write junk
dos.writeUTF("JUNK");
dos.writeUTF("JUNK");
FileUtils.closeQuietly(dos);
DataInputStream dis = new DataInputStream(new ByteArrayInputStream(dos.toByteArray()));
- IndexSummary is = IndexSummary.serializer.deserialize(dis, partitioner, false, 1, 1);
+ IndexSummary is = IndexSummary.serializer.deserialize(dis, partitioner, 1, 1);
for (int i = 0; i < 100; i++)
assertEquals(i, is.binarySearch(random.left.get(i)));
// read the junk
@@ -248,9 +248,9 @@
assertArrayEquals(new byte[0], summary.getKey(0));
DataOutputBuffer dos = new DataOutputBuffer();
- IndexSummary.serializer.serialize(summary, dos, false);
+ IndexSummary.serializer.serialize(summary, dos);
DataInputStream dis = new DataInputStream(new ByteArrayInputStream(dos.toByteArray()));
- IndexSummary loaded = IndexSummary.serializer.deserialize(dis, p, false, 1, 1);
+ IndexSummary loaded = IndexSummary.serializer.deserialize(dis, p, 1, 1);
assertEquals(1, loaded.size());
assertEquals(summary.getPosition(0), loaded.getPosition(0));
diff --git a/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java b/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java
index d06c2c8..7a18133 100644
--- a/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java
@@ -22,9 +22,13 @@
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
import java.util.List;
import java.util.Random;
+import java.util.UUID;
+import com.google.common.collect.Lists;
import com.google.common.collect.Iterables;
import org.junit.After;
import org.junit.Assert;
@@ -37,21 +41,23 @@
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.cql3.QueryOptions;
import org.apache.cassandra.cql3.QueryProcessor;
import org.apache.cassandra.cql3.UntypedResultSet;
+import org.apache.cassandra.cql3.statements.SelectStatement;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Keyspace;
-import org.apache.cassandra.db.LivenessInfo;
+import org.apache.cassandra.db.compaction.AbstractCompactionTask;
+import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.db.compaction.Verifier;
+import org.apache.cassandra.db.repair.PendingAntiCompaction;
+import org.apache.cassandra.db.streaming.CassandraOutgoingFile;
+import org.apache.cassandra.db.ReadExecutionController;
+import org.apache.cassandra.db.SinglePartitionReadCommand;
import org.apache.cassandra.db.SinglePartitionSliceCommandTest;
import org.apache.cassandra.db.compaction.Verifier;
-import org.apache.cassandra.db.lifecycle.SSTableSet;
-import org.apache.cassandra.db.marshal.BytesType;
-import org.apache.cassandra.db.marshal.SetType;
-import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
import org.apache.cassandra.db.rows.RangeTombstoneMarker;
-import org.apache.cassandra.db.rows.Row;
import org.apache.cassandra.db.rows.Unfiltered;
import org.apache.cassandra.db.rows.UnfilteredRowIterator;
import org.apache.cassandra.dht.IPartitioner;
@@ -62,17 +68,23 @@
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.sstable.format.Version;
import org.apache.cassandra.io.sstable.format.big.BigFormat;
+import org.apache.cassandra.service.ActiveRepairService;
import org.apache.cassandra.service.CacheService;
+import org.apache.cassandra.service.ClientState;
import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.streaming.OutgoingStream;
import org.apache.cassandra.streaming.StreamPlan;
import org.apache.cassandra.streaming.StreamSession;
+import org.apache.cassandra.streaming.StreamOperation;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.UUIDGen;
-import static org.apache.cassandra.cql3.CQLTester.assertRows;
-import static org.apache.cassandra.cql3.CQLTester.row;
+import static org.apache.cassandra.service.ActiveRepairService.NO_PENDING_REPAIR;
+import static org.apache.cassandra.service.ActiveRepairService.UNREPAIRED_SSTABLE;
import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
/**
* Tests backwards compatibility for SSTables
@@ -90,7 +102,7 @@
* See {@link #testGenerateSstables()} to generate sstables.
* Take care on commit as you need to add the sstable files using {@code git add -f}
*/
- public static final String[] legacyVersions = {"mc", "mb", "ma", "la", "ka", "jb"};
+ public static final String[] legacyVersions = {"na", "mc", "mb", "ma"};
// 1200 chars
static final String longString = "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789" +
@@ -111,7 +123,7 @@
{
String scp = System.getProperty(LEGACY_SSTABLE_PROP);
Assert.assertNotNull("System property " + LEGACY_SSTABLE_PROP + " not set", scp);
-
+
LEGACY_SSTABLE_ROOT = new File(scp).getAbsoluteFile();
Assert.assertTrue("System property " + LEGACY_SSTABLE_ROOT + " does not specify a directory", LEGACY_SSTABLE_ROOT.isDirectory());
@@ -140,9 +152,12 @@
*/
protected Descriptor getDescriptor(String legacyVersion, String table)
{
- return new Descriptor(legacyVersion, getTableDir(legacyVersion, table), "legacy_tables", table, 1,
- BigFormat.instance.getVersion(legacyVersion).hasNewFileName()?
- SSTableFormat.Type.BIG :SSTableFormat.Type.LEGACY);
+ return new Descriptor(SSTableFormat.Type.BIG.info.getVersion(legacyVersion),
+ getTableDir(legacyVersion, table),
+ "legacy_tables",
+ table,
+ 1,
+ SSTableFormat.Type.BIG);
}
@Test
@@ -161,6 +176,118 @@
doTestLegacyCqlTables();
}
+ @Test
+ public void testMutateMetadata() throws Exception
+ {
+ // we need to make sure we write old version metadata in the format for that version
+ for (String legacyVersion : legacyVersions)
+ {
+ logger.info("Loading legacy version: {}", legacyVersion);
+ truncateLegacyTables(legacyVersion);
+ loadLegacyTables(legacyVersion);
+ CacheService.instance.invalidateKeyCache();
+
+ for (ColumnFamilyStore cfs : Keyspace.open("legacy_tables").getColumnFamilyStores())
+ {
+ for (SSTableReader sstable : cfs.getLiveSSTables())
+ {
+ sstable.descriptor.getMetadataSerializer().mutateRepairMetadata(sstable.descriptor, 1234, NO_PENDING_REPAIR, false);
+ sstable.reloadSSTableMetadata();
+ assertEquals(1234, sstable.getRepairedAt());
+ if (sstable.descriptor.version.hasPendingRepair())
+ assertEquals(NO_PENDING_REPAIR, sstable.getPendingRepair());
+ }
+
+ boolean isTransient = false;
+ for (SSTableReader sstable : cfs.getLiveSSTables())
+ {
+ UUID random = UUID.randomUUID();
+ sstable.descriptor.getMetadataSerializer().mutateRepairMetadata(sstable.descriptor, UNREPAIRED_SSTABLE, random, isTransient);
+ sstable.reloadSSTableMetadata();
+ assertEquals(UNREPAIRED_SSTABLE, sstable.getRepairedAt());
+ if (sstable.descriptor.version.hasPendingRepair())
+ assertEquals(random, sstable.getPendingRepair());
+ if (sstable.descriptor.version.hasIsTransient())
+ assertEquals(isTransient, sstable.isTransient());
+
+ isTransient = !isTransient;
+ }
+ }
+ }
+ }
+
+ @Test
+ public void testMutateMetadataCSM() throws Exception
+ {
+ // we need to make sure we write old version metadata in the format for that version
+ for (String legacyVersion : legacyVersions)
+ {
+ // Skip 2.0.1 sstables as it doesn't have repaired information
+ if (legacyVersion.equals("jb"))
+ continue;
+ truncateTables(legacyVersion);
+ loadLegacyTables(legacyVersion);
+
+ for (ColumnFamilyStore cfs : Keyspace.open("legacy_tables").getColumnFamilyStores())
+ {
+ // set pending
+ for (SSTableReader sstable : cfs.getLiveSSTables())
+ {
+ UUID random = UUID.randomUUID();
+ try
+ {
+ cfs.getCompactionStrategyManager().mutateRepaired(Collections.singleton(sstable), UNREPAIRED_SSTABLE, random, false);
+ if (!sstable.descriptor.version.hasPendingRepair())
+ fail("We should fail setting pending repair on unsupported sstables "+sstable);
+ }
+ catch (IllegalStateException e)
+ {
+ if (sstable.descriptor.version.hasPendingRepair())
+ fail("We should succeed setting pending repair on "+legacyVersion + " sstables, failed on "+sstable);
+ }
+ }
+ // set transient
+ for (SSTableReader sstable : cfs.getLiveSSTables())
+ {
+ try
+ {
+ cfs.getCompactionStrategyManager().mutateRepaired(Collections.singleton(sstable), UNREPAIRED_SSTABLE, UUID.randomUUID(), true);
+ if (!sstable.descriptor.version.hasIsTransient())
+ fail("We should fail setting pending repair on unsupported sstables "+sstable);
+ }
+ catch (IllegalStateException e)
+ {
+ if (sstable.descriptor.version.hasIsTransient())
+ fail("We should succeed setting pending repair on "+legacyVersion + " sstables, failed on "+sstable);
+ }
+ }
+ }
+ }
+ }
+
+ @Test
+ public void testMutateLevel() throws Exception
+ {
+ // we need to make sure we write old version metadata in the format for that version
+ for (String legacyVersion : legacyVersions)
+ {
+ logger.info("Loading legacy version: {}", legacyVersion);
+ truncateLegacyTables(legacyVersion);
+ loadLegacyTables(legacyVersion);
+ CacheService.instance.invalidateKeyCache();
+
+ for (ColumnFamilyStore cfs : Keyspace.open("legacy_tables").getColumnFamilyStores())
+ {
+ for (SSTableReader sstable : cfs.getLiveSSTables())
+ {
+ sstable.descriptor.getMetadataSerializer().mutateLevel(sstable.descriptor, 1234);
+ sstable.reloadSSTableMetadata();
+ assertEquals(1234, sstable.getSSTableLevel());
+ }
+ }
+ }
+ }
+
private void doTestLegacyCqlTables() throws Exception
{
for (String legacyVersion : legacyVersions)
@@ -185,160 +312,12 @@
verifyReads(legacyVersion);
}
}
- @Test
- public void testReverseIterationOfLegacyIndexedSSTable() throws Exception
- {
- // During upgrades from 2.1 to 3.0, reverse queries can drop rows before upgradesstables is completed
- QueryProcessor.executeInternal("CREATE TABLE legacy_tables.legacy_ka_indexed (" +
- " p int," +
- " c int," +
- " v1 int," +
- " v2 int," +
- " PRIMARY KEY(p, c)" +
- ")");
- loadLegacyTable("legacy_%s_indexed%s", "ka", "");
- UntypedResultSet rs = QueryProcessor.executeInternal("SELECT * " +
- "FROM legacy_tables.legacy_ka_indexed " +
- "WHERE p=1 " +
- "ORDER BY c DESC");
- assertEquals(5000, rs.size());
- }
-
- @Test
- public void testReadingLegacyIndexedSSTableWithStaticColumns() throws Exception
- {
- // During upgrades from 2.1 to 3.0, reading from tables with static columns errors before upgradesstables
- // is completed
- QueryProcessor.executeInternal("CREATE TABLE legacy_tables.legacy_ka_indexed_static (" +
- " p int," +
- " c int," +
- " v1 int," +
- " v2 int," +
- " s1 int static," +
- " s2 int static," +
- " PRIMARY KEY(p, c)" +
- ")");
- loadLegacyTable("legacy_%s_indexed_static%s", "ka", "");
- UntypedResultSet rs = QueryProcessor.executeInternal("SELECT * " +
- "FROM legacy_tables.legacy_ka_indexed_static " +
- "WHERE p=1 ");
- assertEquals(5000, rs.size());
- }
-
- @Test
- public void test14766() throws Exception
- {
- /*
- * During upgrades from 2.1 to 3.0, reading from old sstables in reverse order could omit the very last row if the
- * last indexed block had only two Unfiltered-s. See CASSANDRA-14766 for details.
- *
- * The sstable used here has two indexed blocks, with 2 cells/rows of ~500 bytes each, with column index interval of 1kb.
- * Without the fix SELECT * returns 4 rows in ASC order, but only 3 rows in DESC order, omitting the last one.
- */
-
- QueryProcessor.executeInternal("CREATE TABLE legacy_tables.legacy_ka_14766 (pk int, ck int, value text, PRIMARY KEY (pk, ck));");
- loadLegacyTable("legacy_%s_14766%s", "ka", "");
-
- UntypedResultSet rs;
-
- // read all rows in ASC order, expect all 4 to be returned
- rs = QueryProcessor.executeInternal("SELECT * FROM legacy_tables.legacy_ka_14766 WHERE pk = 0 ORDER BY ck ASC;");
- assertEquals(4, rs.size());
-
- // read all rows in DESC order, expect all 4 to be returned
- rs = QueryProcessor.executeInternal("SELECT * FROM legacy_tables.legacy_ka_14766 WHERE pk = 0 ORDER BY ck DESC;");
- assertEquals(4, rs.size());
- }
-
- @Test
- public void test14803() throws Exception
- {
- /*
- * During upgrades from 2.1 to 3.0, reading from old sstables in reverse order could return early if the sstable
- * reverse iterator encounters an indexed block that only covers a single row, and that row starts in the next
- * indexed block.
- */
-
- QueryProcessor.executeInternal("CREATE TABLE legacy_tables.legacy_ka_14803 (k int, c int, v1 blob, v2 blob, PRIMARY KEY (k, c));");
- loadLegacyTable("legacy_%s_14803%s", "ka", "");
-
- UntypedResultSet forward = QueryProcessor.executeOnceInternal(String.format("SELECT * FROM legacy_tables.legacy_ka_14803 WHERE k=100"));
- UntypedResultSet reverse = QueryProcessor.executeOnceInternal(String.format("SELECT * FROM legacy_tables.legacy_ka_14803 WHERE k=100 ORDER BY c DESC"));
-
- logger.info("{} - {}", forward.size(), reverse.size());
- Assert.assertFalse(forward.isEmpty());
- assertEquals(forward.size(), reverse.size());
- }
-
- @Test
- public void test14873() throws Exception
- {
- /*
- * When reading 2.1 sstables in 3.0 in reverse order it's possible to wrongly return an empty result set if the
- * partition being read has a static row, and the read is performed backwards.
- */
-
- /*
- * Contents of the SSTable (column_index_size_in_kb: 1) below:
- *
- * insert into legacy_tables.legacy_ka_14873 (pkc, sc) values (0, 0);
- * insert into legacy_tables.legacy_ka_14873 (pkc, cc, rc) values (0, 5, '5555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555');
- * insert into legacy_tables.legacy_ka_14873 (pkc, cc, rc) values (0, 4, '4444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444');
- * insert into legacy_tables.legacy_ka_14873 (pkc, cc, rc) values (0, 3, '3333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333');
- * insert into legacy_tables.legacy_ka_14873 (pkc, cc, rc) values (0, 2, '2222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222');
- * insert into legacy_tables.legacy_ka_14873 (pkc, cc, rc) values (0, 1, '1111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111');
- */
-
- String ddl =
- "CREATE TABLE legacy_tables.legacy_ka_14873 ("
- + "pkc int, cc int, sc int static, rc text, PRIMARY KEY (pkc, cc)"
- + ") WITH CLUSTERING ORDER BY (cc DESC) AND compaction = {'enabled' : 'false', 'class' : 'LeveledCompactionStrategy'};";
- QueryProcessor.executeInternal(ddl);
- loadLegacyTable("legacy_%s_14873%s", "ka", "");
-
- UntypedResultSet forward =
- QueryProcessor.executeOnceInternal(
- String.format("SELECT * FROM legacy_tables.legacy_ka_14873 WHERE pkc = 0 AND cc > 0 ORDER BY cc DESC;"));
-
- UntypedResultSet reverse =
- QueryProcessor.executeOnceInternal(
- String.format("SELECT * FROM legacy_tables.legacy_ka_14873 WHERE pkc = 0 AND cc > 0 ORDER BY cc ASC;"));
-
- assertEquals(5, forward.size());
- assertEquals(5, reverse.size());
- }
-
- @Test
- public void testMultiBlockRangeTombstones() throws Exception
- {
- /**
- * During upgrades from 2.1 to 3.0, reading old sstables in reverse order would generate invalid sequences of
- * range tombstone bounds if their range tombstones spanned multiple column index blocks. The read would fail
- * in different ways depending on whether the 2.1 tables were produced by a flush or a compaction.
- */
-
- String version = "ka";
- for (String tableFmt : new String[]{"legacy_%s_compacted_multi_block_rt%s", "legacy_%s_flushed_multi_block_rt%s"})
- {
- String table = String.format(tableFmt, version, "");
- QueryProcessor.executeOnceInternal(String.format("CREATE TABLE legacy_tables.%s " +
- "(k int, c1 int, c2 int, v1 blob, v2 blob, " +
- "PRIMARY KEY (k, c1, c2))", table));
- loadLegacyTable(tableFmt, version, "");
-
- UntypedResultSet forward = QueryProcessor.executeOnceInternal(String.format("SELECT * FROM legacy_tables.%s WHERE k=100", table));
- UntypedResultSet reverse = QueryProcessor.executeOnceInternal(String.format("SELECT * FROM legacy_tables.%s WHERE k=100 ORDER BY c1 DESC, c2 DESC", table));
-
- Assert.assertFalse(forward.isEmpty());
- assertEquals(table, forward.size(), reverse.size());
- }
- }
@Test
public void testInaccurateSSTableMinMax() throws Exception
{
QueryProcessor.executeInternal("CREATE TABLE legacy_tables.legacy_mc_inaccurate_min_max (k int, c1 int, c2 int, c3 int, v int, primary key (k, c1, c2, c3))");
- loadLegacyTable("legacy_%s_inaccurate_min_max%s", "mc", "");
+ loadLegacyTable("legacy_%s_inaccurate_min_max", "mc");
/*
sstable has the following mutations:
@@ -348,7 +327,7 @@
String query = "SELECT * FROM legacy_tables.legacy_mc_inaccurate_min_max WHERE k=100 AND c1=1 AND c2=1";
List<Unfiltered> unfiltereds = SinglePartitionSliceCommandTest.getUnfilteredsFromSinglePartition(query);
- assertEquals(2, unfiltereds.size());
+ Assert.assertEquals(2, unfiltereds.size());
Assert.assertTrue(unfiltereds.get(0).isRangeTombstoneMarker());
Assert.assertTrue(((RangeTombstoneMarker) unfiltereds.get(0)).isOpen(false));
Assert.assertTrue(unfiltereds.get(1).isRangeTombstoneMarker());
@@ -356,342 +335,141 @@
}
@Test
- public void testVerifyOldSSTables() throws Exception
+ public void testVerifyOldSSTables() throws IOException
{
for (String legacyVersion : legacyVersions)
{
- loadLegacyTables(legacyVersion);
ColumnFamilyStore cfs = Keyspace.open("legacy_tables").getColumnFamilyStore(String.format("legacy_%s_simple", legacyVersion));
+ loadLegacyTable("legacy_%s_simple", legacyVersion);
+
for (SSTableReader sstable : cfs.getLiveSSTables())
{
- try (Verifier verifier = new Verifier(cfs, sstable, false))
+ try (Verifier verifier = new Verifier(cfs, sstable, false, Verifier.options().checkVersion(true).build()))
{
- verifier.verify(true);
+ verifier.verify();
+ if (!sstable.descriptor.version.isLatestVersion())
+ fail("Verify should throw RuntimeException for old sstables "+sstable);
}
+ catch (RuntimeException e)
+ {}
}
- }
- }
-
- @Test
- public void test14912() throws Exception
- {
- /*
- * When reading 2.1 sstables in 3.0, collection tombstones need to be checked against
- * the dropped columns stored in table metadata. Failure to do so can result in unreadable
- * rows if a column with the same name but incompatible type has subsequently been added.
- *
- * The original (i.e. pre-any ALTER statements) table definition for this test is:
- * CREATE TABLE legacy_tables.legacy_ka_14912 (k int PRIMARY KEY, v1 set<text>, v2 text);
- *
- * The SSTable loaded emulates data being written before the table is ALTERed and contains:
- *
- * insert into legacy_tables.legacy_ka_14912 (k, v1, v2) values (0, {}, 'abc') USING TIMESTAMP 1543244999672280;
- * insert into legacy_tables.legacy_ka_14912 (k, v1, v2) values (1, {'abc'}, 'abc') USING TIMESTAMP 1543244999672280;
- *
- * The timestamps of the (generated) collection tombstones are 1543244999672279, e.g. the <TIMESTAMP of the mutation> - 1
- */
-
- QueryProcessor.executeInternal("CREATE TABLE legacy_tables.legacy_ka_14912 (k int PRIMARY KEY, v1 text, v2 text)");
- loadLegacyTable("legacy_%s_14912%s", "ka", "");
- CFMetaData cfm = Keyspace.open("legacy_tables").getColumnFamilyStore("legacy_ka_14912").metadata;
- ColumnDefinition columnToDrop;
-
- /*
- * This first variant simulates the original v1 set<text> column being dropped
- * then re-added with the text type:
- * CREATE TABLE legacy_tables.legacy_ka_14912 (k int PRIMARY KEY, v1 set<text>, v2 text);
- * INSERT INTO legacy_tables.legacy)ka_14912 (k, v1, v2)...
- * ALTER TABLE legacy_tables.legacy_ka_14912 DROP v1;
- * ALTER TABLE legacy_tables.legacy_ka_14912 ADD v1 text;
- */
- columnToDrop = ColumnDefinition.regularDef(cfm,
- UTF8Type.instance.fromString("v1"),
- SetType.getInstance(UTF8Type.instance, true));
- cfm.recordColumnDrop(columnToDrop, 1543244999700000L);
- assertExpectedRowsWithDroppedCollection(true);
- // repeat the query, but simulate clock drift by shifting the recorded
- // drop time forward so that it occurs before the collection timestamp
- cfm.recordColumnDrop(columnToDrop, 1543244999600000L);
- assertExpectedRowsWithDroppedCollection(false);
-
- /*
- * This second test simulates the original v1 set<text> column being dropped
- * then re-added with some other, non-collection type (overwriting the dropped
- * columns record), then dropping and re-adding again as text type:
- * CREATE TABLE legacy_tables.legacy_ka_14912 (k int PRIMARY KEY, v1 set<text>, v2 text);
- * INSERT INTO legacy_tables.legacy_ka_14912 (k, v1, v2)...
- * ALTER TABLE legacy_tables.legacy_ka_14912 DROP v1;
- * ALTER TABLE legacy_tables.legacy_ka_14912 ADD v1 blob;
- * ALTER TABLE legacy_tables.legacy_ka_14912 DROP v1;
- * ALTER TABLE legacy_tables.legacy_ka_14912 ADD v1 text;
- */
- columnToDrop = ColumnDefinition.regularDef(cfm,
- UTF8Type.instance.fromString("v1"),
- BytesType.instance);
- cfm.recordColumnDrop(columnToDrop, 1543244999700000L);
- assertExpectedRowsWithDroppedCollection(true);
- // repeat the query, but simulate clock drift by shifting the recorded
- // drop time forward so that it occurs before the collection timestamp
- cfm.recordColumnDrop(columnToDrop, 1543244999600000L);
- assertExpectedRowsWithDroppedCollection(false);
- }
-
- @Test
- public void test15081() throws Exception
- {
- QueryProcessor.executeInternal("CREATE TABLE legacy_tables.legacy_ka_15081 (id int primary key, payload text)");
- loadLegacyTable("legacy_%s_15081%s", "ka", "");
- UntypedResultSet results =
- QueryProcessor.executeOnceInternal(
- String.format("SELECT * FROM legacy_tables.legacy_ka_15081"));
- assertRows(results, row(1, "hello world"));
- }
-
- @Test
- public void testReadingLegacyTablesWithIllegalCellNames() throws Exception {
- /**
- * The sstable can be generated externally with SSTableSimpleUnsortedWriter:
- *
- * [
- * {"key": "1",
- * "cells": [["a:aa:c1","61",1555000750634000],
- * ["a:aa:c2","6161",1555000750634000],
- * ["a:aa:pk","00000001",1555000750634000],
- * ["a:aa:v1","aaa",1555000750634000]]},
- * {"key": "2",
- * "cells": [["b:bb:c1","62",1555000750634000],
- * ["b:bb:c2","6262",1555000750634000],
- * ["b:bb:pk","00000002",1555000750634000],
- * ["b:bb:v1","bbb",1555000750634000]]}
- * ]
- * and an extra sstable with only the invalid cell name
- * [
- * {"key": "3",
- * "cells": [["a:aa:pk","68656c6c6f30",1570466358949]]}
- * ]
- *
- */
- String table = "legacy_ka_with_illegal_cell_names";
- QueryProcessor.executeInternal("CREATE TABLE legacy_tables." + table + " (" +
- " pk int," +
- " c1 text," +
- " c2 text," +
- " v1 text," +
- " PRIMARY KEY(pk, c1, c2))");
- loadLegacyTable("legacy_%s_with_illegal_cell_names%s", "ka", "");
- UntypedResultSet results =
- QueryProcessor.executeOnceInternal("SELECT * FROM legacy_tables."+table);
-
- assertRows(results, row(1, "a", "aa", "aaa"), row(2, "b", "bb", "bbb"), row (3, "a", "aa", null));
- Keyspace.open("legacy_tables").getColumnFamilyStore(table).forceMajorCompaction();
- }
-
- @Test
- public void testReadingLegacyTablesWithIllegalCellNamesPKLI() throws Exception {
- /**
- *
- * Makes sure we grab the correct PKLI when we have illegal columns
- *
- * sstable looks like this:
- * [
- * {"key": "3",
- * "cells": [["a:aa:","",100],
- * ["a:aa:pk","6d656570",200]]}
- * ]
- */
- /*
- this generates the stable on 2.1:
- CFMetaData metadata = CFMetaData.compile("create table legacy_tables.legacy_ka_with_illegal_cell_names_2 (pk int, c1 text, c2 text, v1 text, primary key (pk, c1, c2))", "legacy_tables");
- try (SSTableSimpleUnsortedWriter writer = new SSTableSimpleUnsortedWriter(new File("/tmp/sstable21"),
- metadata,
- new ByteOrderedPartitioner(),
- 10))
- {
- writer.newRow(bytes(3));
- writer.addColumn(new BufferCell(Util.cellname("a", "aa", ""), bytes(""), 100));
- writer.addColumn(new BufferCell(Util.cellname("a", "aa", "pk"), bytes("meep"), 200));
- }
- */
- String table = "legacy_ka_with_illegal_cell_names_2";
- QueryProcessor.executeInternal("CREATE TABLE legacy_tables." + table + " (" +
- " pk int," +
- " c1 text," +
- " c2 text," +
- " v1 text," +
- " PRIMARY KEY(pk, c1, c2))");
- loadLegacyTable("legacy_%s_with_illegal_cell_names_2%s", "ka", "");
- ColumnFamilyStore cfs = Keyspace.open("legacy_tables").getColumnFamilyStore(table);
- assertEquals(1, Iterables.size(cfs.getSSTables(SSTableSet.CANONICAL)));
- cfs.forceMajorCompaction();
- assertEquals(1, Iterables.size(cfs.getSSTables(SSTableSet.CANONICAL)));
- SSTableReader sstable = Iterables.getFirst(cfs.getSSTables(SSTableSet.CANONICAL), null);
- LivenessInfo livenessInfo = null;
- try (ISSTableScanner scanner = sstable.getScanner())
- {
- while (scanner.hasNext())
+ // make sure we don't throw any exception if not checking version:
+ for (SSTableReader sstable : cfs.getLiveSSTables())
{
- try (UnfilteredRowIterator iter = scanner.next())
+ try (Verifier verifier = new Verifier(cfs, sstable, false, Verifier.options().checkVersion(false).build()))
{
- while (iter.hasNext())
- {
- Unfiltered uf = iter.next();
- livenessInfo = ((Row)uf).primaryKeyLivenessInfo();
- }
+ verifier.verify();
+ }
+ catch (Throwable e)
+ {
+ fail("Verify should throw RuntimeException for old sstables "+sstable);
}
}
}
- assertNotNull(livenessInfo);
- assertEquals(100, livenessInfo.timestamp());
}
@Test
- public void testReadingIndexedLegacyTablesWithIllegalCellNames() throws Exception {
- /**
- * The sstable can be generated externally with SSTableSimpleUnsortedWriter:
- * column_index_size_in_kb: 1
- * [
- * {"key": "key",
- * "cells": [
- * ["00000:000000:a","00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",0],
- * ["00000:000000:b","00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",0]
- * ["00000:000000:c","00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",0]
- * ["00000:000000:z","00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",0]
- * ["00001:000001:a","00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",0],
- * ["00001:000001:b","00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",0]
- * ["00001:000001:c","00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",0]
- * ["00001:000001:z","00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",0]
- * .
- * .
- * .
- * ["00010:000010:a","00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",0],
- * ["00010:000010:b","00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",0]
- * ["00010:000010:c","00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",0]
- * ["00010:000010:z","00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",0]
- * ]
- * }
- * ]
- * Each row in the partition contains only 1 valid cell. The ones with the column name components 'a', 'b' & 'z' are illegal as they refer to PRIMARY KEY
- * columns, but SSTables such as this can be generated with offline tools and loaded via SSTableLoader or nodetool refresh (see CASSANDRA-15086) (see
- * CASSANDRA-15086) Only 'c' is a valid REGULAR column in the table schema.
- * In the initial fix for CASSANDRA-15086, the bytes read by OldFormatDeserializer for these invalid cells are not correctly accounted for, causing
- * ReverseIndexedReader to assert that the end of a block has been reached earlier than it actually has, which in turn causes rows to be incorrectly
- * ommitted from the results.
- *
- * This sstable has been crafted to hit a further potential error condition. Rows 00001:00001 and 00008:00008 interact with the index block boundaries
- * in a very specific way; for both of these rows, the (illegal) cells 'a' & 'b', along with the valid 'c' cell are at the end of an index block, but
- * the 'z' cell is over the boundary, in the following block. We need to ensure that the bytes consumed for the 'z' cell are properly accounted for and
- * not counted toward those for the next row on disk.
- */
- QueryProcessor.executeInternal("CREATE TABLE legacy_tables.legacy_ka_with_illegal_cell_names_indexed (" +
- " a text," +
- " b text," +
- " z text," +
- " c text," +
- " PRIMARY KEY(a, b, z))");
- loadLegacyTable("legacy_%s_with_illegal_cell_names_indexed%s", "ka", "");
- String queryForward = "SELECT * FROM legacy_tables.legacy_ka_with_illegal_cell_names_indexed WHERE a = 'key'";
- String queryReverse = queryForward + " ORDER BY b DESC, z DESC";
+ public void testPendingAntiCompactionOldSSTables() throws Exception
+ {
+ for (String legacyVersion : legacyVersions)
+ {
+ ColumnFamilyStore cfs = Keyspace.open("legacy_tables").getColumnFamilyStore(String.format("legacy_%s_simple", legacyVersion));
+ loadLegacyTable("legacy_%s_simple", legacyVersion);
- List<String> forward = new ArrayList<>();
- QueryProcessor.executeOnceInternal(queryForward).forEach(r -> forward.add(r.getString("b") + ":" + r.getString("z")));
-
- List<String> reverse = new ArrayList<>();
- QueryProcessor.executeOnceInternal(queryReverse).forEach(r -> reverse.add(r.getString("b") + ":" + r.getString("z")));
-
- assertEquals(11, reverse.size());
- assertEquals(11, forward.size());
- for (int i=0; i < 11; i++)
- assertEquals(forward.get(i), reverse.get(10 - i));
+ boolean shouldFail = !cfs.getLiveSSTables().stream().allMatch(sstable -> sstable.descriptor.version.hasPendingRepair());
+ IPartitioner p = Iterables.getFirst(cfs.getLiveSSTables(), null).getPartitioner();
+ Range<Token> r = new Range<>(p.getMinimumToken(), p.getMinimumToken());
+ PendingAntiCompaction.AcquisitionCallable acquisitionCallable = new PendingAntiCompaction.AcquisitionCallable(cfs, Collections.singleton(r), UUIDGen.getTimeUUID(), 0, 0);
+ PendingAntiCompaction.AcquireResult res = acquisitionCallable.call();
+ assertEquals(shouldFail, res == null);
+ if (res != null)
+ res.abort();
+ }
}
- private void assertExpectedRowsWithDroppedCollection(boolean droppedCheckSuccessful)
+ @Test
+ public void testAutomaticUpgrade() throws Exception
{
- for (int i=0; i<=1; i++)
+ for (String legacyVersion : legacyVersions)
{
- UntypedResultSet rows =
- QueryProcessor.executeOnceInternal(
- String.format("SELECT * FROM legacy_tables.legacy_ka_14912 WHERE k = %s;", i));
- assertEquals(1, rows.size());
- UntypedResultSet.Row row = rows.one();
-
- // If the best-effort attempt to filter dropped columns was successful, then the row
- // should not contain the v1 column at all. Likewise, if no column data was written,
- // only a tombstone, then no v1 column should be present.
- // However, if collection data was written (i.e. where k=1), then if the dropped column
- // check didn't filter the legacy cells, we should expect an empty column value as the
- // legacy collection tombstone won't cover it and the dropped column check doesn't filter
- // it.
- if (droppedCheckSuccessful || i == 0)
- Assert.assertFalse(row.has("v1"));
- else
- assertEquals("", row.getString("v1"));
-
- assertEquals("abc", row.getString("v2"));
+ logger.info("Loading legacy version: {}", legacyVersion);
+ truncateLegacyTables(legacyVersion);
+ loadLegacyTables(legacyVersion);
+ ColumnFamilyStore cfs = Keyspace.open("legacy_tables").getColumnFamilyStore(String.format("legacy_%s_simple", legacyVersion));
+ AbstractCompactionTask act = cfs.getCompactionStrategyManager().getNextBackgroundTask(0);
+ // there should be no compactions to run with auto upgrades disabled:
+ assertEquals(null, act);
}
+
+ DatabaseDescriptor.setAutomaticSSTableUpgradeEnabled(true);
+ for (String legacyVersion : legacyVersions)
+ {
+ logger.info("Loading legacy version: {}", legacyVersion);
+ truncateLegacyTables(legacyVersion);
+ loadLegacyTables(legacyVersion);
+ ColumnFamilyStore cfs = Keyspace.open("legacy_tables").getColumnFamilyStore(String.format("legacy_%s_simple", legacyVersion));
+ if (cfs.getLiveSSTables().stream().anyMatch(s -> !s.descriptor.version.isLatestVersion()))
+ assertTrue(cfs.metric.oldVersionSSTableCount.getValue() > 0);
+ while (cfs.getLiveSSTables().stream().anyMatch(s -> !s.descriptor.version.isLatestVersion()))
+ {
+ CompactionManager.instance.submitBackground(cfs);
+ Thread.sleep(100);
+ }
+ assertTrue(cfs.metric.oldVersionSSTableCount.getValue() == 0);
+ }
+ DatabaseDescriptor.setAutomaticSSTableUpgradeEnabled(false);
}
private void streamLegacyTables(String legacyVersion) throws Exception
{
- for (int compact = 0; compact <= 1; compact++)
- {
- logger.info("Streaming legacy version {}{}", legacyVersion, getCompactNameSuffix(compact));
- streamLegacyTable("legacy_%s_simple%s", legacyVersion, getCompactNameSuffix(compact));
- streamLegacyTable("legacy_%s_simple_counter%s", legacyVersion, getCompactNameSuffix(compact));
- streamLegacyTable("legacy_%s_clust%s", legacyVersion, getCompactNameSuffix(compact));
- streamLegacyTable("legacy_%s_clust_counter%s", legacyVersion, getCompactNameSuffix(compact));
- }
+ logger.info("Streaming legacy version {}", legacyVersion);
+ streamLegacyTable("legacy_%s_simple", legacyVersion);
+ streamLegacyTable("legacy_%s_simple_counter", legacyVersion);
+ streamLegacyTable("legacy_%s_clust", legacyVersion);
+ streamLegacyTable("legacy_%s_clust_counter", legacyVersion);
}
- private void streamLegacyTable(String tablePattern, String legacyVersion, String compactNameSuffix) throws Exception
+ private void streamLegacyTable(String tablePattern, String legacyVersion) throws Exception
{
- String table = String.format(tablePattern, legacyVersion, compactNameSuffix);
+ String table = String.format(tablePattern, legacyVersion);
SSTableReader sstable = SSTableReader.open(getDescriptor(legacyVersion, table));
IPartitioner p = sstable.getPartitioner();
List<Range<Token>> ranges = new ArrayList<>();
ranges.add(new Range<>(p.getMinimumToken(), p.getToken(ByteBufferUtil.bytes("100"))));
ranges.add(new Range<>(p.getToken(ByteBufferUtil.bytes("100")), p.getMinimumToken()));
- ArrayList<StreamSession.SSTableStreamingSections> details = new ArrayList<>();
- details.add(new StreamSession.SSTableStreamingSections(sstable.ref(),
- sstable.getPositionsForRanges(ranges),
- sstable.estimatedKeysForRanges(ranges), sstable.getSSTableMetadata().repairedAt));
- new StreamPlan("LegacyStreamingTest").transferFiles(FBUtilities.getBroadcastAddress(), details)
- .execute().get();
+ List<OutgoingStream> streams = Lists.newArrayList(new CassandraOutgoingFile(StreamOperation.OTHER,
+ sstable.ref(),
+ sstable.getPositionsForRanges(ranges),
+ ranges,
+ sstable.estimatedKeysForRanges(ranges)));
+ new StreamPlan(StreamOperation.OTHER).transferStreams(FBUtilities.getBroadcastAddressAndPort(), streams).execute().get();
}
private static void truncateLegacyTables(String legacyVersion) throws Exception
{
- for (int compact = 0; compact <= 1; compact++)
- {
- logger.info("Truncating legacy version {}{}", legacyVersion, getCompactNameSuffix(compact));
- Keyspace.open("legacy_tables").getColumnFamilyStore(String.format("legacy_%s_simple%s", legacyVersion, getCompactNameSuffix(compact))).truncateBlocking();
- Keyspace.open("legacy_tables").getColumnFamilyStore(String.format("legacy_%s_simple_counter%s", legacyVersion, getCompactNameSuffix(compact))).truncateBlocking();
- Keyspace.open("legacy_tables").getColumnFamilyStore(String.format("legacy_%s_clust%s", legacyVersion, getCompactNameSuffix(compact))).truncateBlocking();
- Keyspace.open("legacy_tables").getColumnFamilyStore(String.format("legacy_%s_clust_counter%s", legacyVersion, getCompactNameSuffix(compact))).truncateBlocking();
- }
+ logger.info("Truncating legacy version {}", legacyVersion);
+ Keyspace.open("legacy_tables").getColumnFamilyStore(String.format("legacy_%s_simple", legacyVersion)).truncateBlocking();
+ Keyspace.open("legacy_tables").getColumnFamilyStore(String.format("legacy_%s_simple_counter", legacyVersion)).truncateBlocking();
+ Keyspace.open("legacy_tables").getColumnFamilyStore(String.format("legacy_%s_clust", legacyVersion)).truncateBlocking();
+ Keyspace.open("legacy_tables").getColumnFamilyStore(String.format("legacy_%s_clust_counter", legacyVersion)).truncateBlocking();
}
private static void compactLegacyTables(String legacyVersion) throws Exception
{
- for (int compact = 0; compact <= 1; compact++)
- {
- logger.info("Compacting legacy version {}{}", legacyVersion, getCompactNameSuffix(compact));
- Keyspace.open("legacy_tables").getColumnFamilyStore(String.format("legacy_%s_simple%s", legacyVersion, getCompactNameSuffix(compact))).forceMajorCompaction();
- Keyspace.open("legacy_tables").getColumnFamilyStore(String.format("legacy_%s_simple_counter%s", legacyVersion, getCompactNameSuffix(compact))).forceMajorCompaction();
- Keyspace.open("legacy_tables").getColumnFamilyStore(String.format("legacy_%s_clust%s", legacyVersion, getCompactNameSuffix(compact))).forceMajorCompaction();
- Keyspace.open("legacy_tables").getColumnFamilyStore(String.format("legacy_%s_clust_counter%s", legacyVersion, getCompactNameSuffix(compact))).forceMajorCompaction();
- }
+ logger.info("Compacting legacy version {}", legacyVersion);
+ Keyspace.open("legacy_tables").getColumnFamilyStore(String.format("legacy_%s_simple", legacyVersion)).forceMajorCompaction();
+ Keyspace.open("legacy_tables").getColumnFamilyStore(String.format("legacy_%s_simple_counter", legacyVersion)).forceMajorCompaction();
+ Keyspace.open("legacy_tables").getColumnFamilyStore(String.format("legacy_%s_clust", legacyVersion)).forceMajorCompaction();
+ Keyspace.open("legacy_tables").getColumnFamilyStore(String.format("legacy_%s_clust_counter", legacyVersion)).forceMajorCompaction();
}
private static void loadLegacyTables(String legacyVersion) throws Exception
{
- for (int compact = 0; compact <= 1; compact++)
- {
- logger.info("Preparing legacy version {}{}", legacyVersion, getCompactNameSuffix(compact));
- loadLegacyTable("legacy_%s_simple%s", legacyVersion, getCompactNameSuffix(compact));
- loadLegacyTable("legacy_%s_simple_counter%s", legacyVersion, getCompactNameSuffix(compact));
- loadLegacyTable("legacy_%s_clust%s", legacyVersion, getCompactNameSuffix(compact));
- loadLegacyTable("legacy_%s_clust_counter%s", legacyVersion, getCompactNameSuffix(compact));
- }
+ logger.info("Preparing legacy version {}", legacyVersion);
+ loadLegacyTable("legacy_%s_simple", legacyVersion);
+ loadLegacyTable("legacy_%s_simple_counter", legacyVersion);
+ loadLegacyTable("legacy_%s_clust", legacyVersion);
+ loadLegacyTable("legacy_%s_clust_counter", legacyVersion);
}
private static void verifyCache(String legacyVersion, long startCount) throws InterruptedException, java.util.concurrent.ExecutionException
@@ -703,81 +481,74 @@
Assert.assertTrue(endCount > startCount);
CacheService.instance.keyCache.submitWrite(Integer.MAX_VALUE).get();
CacheService.instance.invalidateKeyCache();
- assertEquals(startCount, CacheService.instance.keyCache.size());
+ Assert.assertEquals(startCount, CacheService.instance.keyCache.size());
CacheService.instance.keyCache.loadSaved();
- if (BigFormat.instance.getVersion(legacyVersion).storeRows())
- assertEquals(endCount, CacheService.instance.keyCache.size());
- else
- assertEquals(startCount, CacheService.instance.keyCache.size());
+ Assert.assertEquals(endCount, CacheService.instance.keyCache.size());
}
private static void verifyReads(String legacyVersion)
{
- for (int compact = 0; compact <= 1; compact++)
+ for (int ck = 0; ck < 50; ck++)
{
- for (int ck = 0; ck < 50; ck++)
+ String ckValue = Integer.toString(ck) + longString;
+ for (int pk = 0; pk < 5; pk++)
{
- String ckValue = Integer.toString(ck) + longString;
- for (int pk = 0; pk < 5; pk++)
+ logger.debug("for pk={} ck={}", pk, ck);
+
+ String pkValue = Integer.toString(pk);
+ if (ck == 0)
{
- logger.debug("for pk={} ck={}", pk, ck);
-
- String pkValue = Integer.toString(pk);
- UntypedResultSet rs;
- if (ck == 0)
- {
- readSimpleTable(legacyVersion, getCompactNameSuffix(compact), pkValue);
- readSimpleCounterTable(legacyVersion, getCompactNameSuffix(compact), pkValue);
- }
-
- readClusteringTable(legacyVersion, getCompactNameSuffix(compact), ck, ckValue, pkValue);
- readClusteringCounterTable(legacyVersion, getCompactNameSuffix(compact), ckValue, pkValue);
+ readSimpleTable(legacyVersion, pkValue);
+ readSimpleCounterTable(legacyVersion, pkValue);
}
+
+ readClusteringTable(legacyVersion, ck, ckValue, pkValue);
+ readClusteringCounterTable(legacyVersion, ckValue, pkValue);
}
}
}
- private static void readClusteringCounterTable(String legacyVersion, String compactSuffix, String ckValue, String pkValue)
+ private static void readClusteringCounterTable(String legacyVersion, String ckValue, String pkValue)
{
- logger.debug("Read legacy_{}_clust_counter{}", legacyVersion, compactSuffix);
+ logger.debug("Read legacy_{}_clust_counter", legacyVersion);
UntypedResultSet rs;
- rs = QueryProcessor.executeInternal(String.format("SELECT val FROM legacy_tables.legacy_%s_clust_counter%s WHERE pk=? AND ck=?", legacyVersion, compactSuffix), pkValue, ckValue);
+ rs = QueryProcessor.executeInternal(String.format("SELECT val FROM legacy_tables.legacy_%s_clust_counter WHERE pk=? AND ck=?", legacyVersion), pkValue, ckValue);
Assert.assertNotNull(rs);
- assertEquals(1, rs.size());
- assertEquals(1L, rs.one().getLong("val"));
+ Assert.assertEquals(1, rs.size());
+ Assert.assertEquals(1L, rs.one().getLong("val"));
}
- private static void readClusteringTable(String legacyVersion, String compactSuffix, int ck, String ckValue, String pkValue)
+ private static void readClusteringTable(String legacyVersion, int ck, String ckValue, String pkValue)
{
- logger.debug("Read legacy_{}_clust{}", legacyVersion, compactSuffix);
+ logger.debug("Read legacy_{}_clust", legacyVersion);
UntypedResultSet rs;
- rs = QueryProcessor.executeInternal(String.format("SELECT val FROM legacy_tables.legacy_%s_clust%s WHERE pk=? AND ck=?", legacyVersion, compactSuffix), pkValue, ckValue);
+ rs = QueryProcessor.executeInternal(String.format("SELECT val FROM legacy_tables.legacy_%s_clust WHERE pk=? AND ck=?", legacyVersion), pkValue, ckValue);
assertLegacyClustRows(1, rs);
String ckValue2 = Integer.toString(ck < 10 ? 40 : ck - 1) + longString;
String ckValue3 = Integer.toString(ck > 39 ? 10 : ck + 1) + longString;
- rs = QueryProcessor.executeInternal(String.format("SELECT val FROM legacy_tables.legacy_%s_clust%s WHERE pk=? AND ck IN (?, ?, ?)", legacyVersion, compactSuffix), pkValue, ckValue, ckValue2, ckValue3);
+ rs = QueryProcessor.executeInternal(String.format("SELECT val FROM legacy_tables.legacy_%s_clust WHERE pk=? AND ck IN (?, ?, ?)", legacyVersion), pkValue, ckValue, ckValue2, ckValue3);
assertLegacyClustRows(3, rs);
}
- private static void readSimpleCounterTable(String legacyVersion, String compactSuffix, String pkValue)
+ private static void readSimpleCounterTable(String legacyVersion, String pkValue)
{
- logger.debug("Read legacy_{}_simple_counter{}", legacyVersion, compactSuffix);
+ logger.debug("Read legacy_{}_simple_counter", legacyVersion);
UntypedResultSet rs;
- rs = QueryProcessor.executeInternal(String.format("SELECT val FROM legacy_tables.legacy_%s_simple_counter%s WHERE pk=?", legacyVersion, compactSuffix), pkValue);
+ rs = QueryProcessor.executeInternal(String.format("SELECT val FROM legacy_tables.legacy_%s_simple_counter WHERE pk=?", legacyVersion), pkValue);
Assert.assertNotNull(rs);
- assertEquals(1, rs.size());
- assertEquals(1L, rs.one().getLong("val"));
+ Assert.assertEquals(1, rs.size());
+ Assert.assertEquals(1L, rs.one().getLong("val"));
}
- private static void readSimpleTable(String legacyVersion, String compactSuffix, String pkValue)
+ private static void readSimpleTable(String legacyVersion, String pkValue)
{
- logger.debug("Read simple: legacy_{}_simple{}", legacyVersion, compactSuffix);
+ logger.debug("Read simple: legacy_{}_simple", legacyVersion);
UntypedResultSet rs;
- rs = QueryProcessor.executeInternal(String.format("SELECT val FROM legacy_tables.legacy_%s_simple%s WHERE pk=?", legacyVersion, compactSuffix), pkValue);
+ rs = QueryProcessor.executeInternal(String.format("SELECT val FROM legacy_tables.legacy_%s_simple WHERE pk=?", legacyVersion), pkValue);
Assert.assertNotNull(rs);
- assertEquals(1, rs.size());
- assertEquals("foo bar baz", rs.one().getString("val"));
+ Assert.assertEquals(1, rs.size());
+ Assert.assertEquals("foo bar baz", rs.one().getString("val"));
}
private static void createKeyspace()
@@ -787,31 +558,18 @@
private static void createTables(String legacyVersion)
{
- for (int i=0; i<=1; i++)
- {
- String compactSuffix = getCompactNameSuffix(i);
- String tableSuffix = i == 0? "" : " WITH COMPACT STORAGE";
- QueryProcessor.executeInternal(String.format("CREATE TABLE legacy_tables.legacy_%s_simple%s (pk text PRIMARY KEY, val text)%s", legacyVersion, compactSuffix, tableSuffix));
- QueryProcessor.executeInternal(String.format("CREATE TABLE legacy_tables.legacy_%s_simple_counter%s (pk text PRIMARY KEY, val counter)%s", legacyVersion, compactSuffix, tableSuffix));
- QueryProcessor.executeInternal(String.format("CREATE TABLE legacy_tables.legacy_%s_clust%s (pk text, ck text, val text, PRIMARY KEY (pk, ck))%s", legacyVersion, compactSuffix, tableSuffix));
- QueryProcessor.executeInternal(String.format("CREATE TABLE legacy_tables.legacy_%s_clust_counter%s (pk text, ck text, val counter, PRIMARY KEY (pk, ck))%s", legacyVersion, compactSuffix, tableSuffix));
- }
- }
-
- private static String getCompactNameSuffix(int i)
- {
- return i == 0? "" : "_compact";
+ QueryProcessor.executeInternal(String.format("CREATE TABLE legacy_tables.legacy_%s_simple (pk text PRIMARY KEY, val text)", legacyVersion));
+ QueryProcessor.executeInternal(String.format("CREATE TABLE legacy_tables.legacy_%s_simple_counter (pk text PRIMARY KEY, val counter)", legacyVersion));
+ QueryProcessor.executeInternal(String.format("CREATE TABLE legacy_tables.legacy_%s_clust (pk text, ck text, val text, PRIMARY KEY (pk, ck))", legacyVersion));
+ QueryProcessor.executeInternal(String.format("CREATE TABLE legacy_tables.legacy_%s_clust_counter (pk text, ck text, val counter, PRIMARY KEY (pk, ck))", legacyVersion));
}
private static void truncateTables(String legacyVersion)
{
- for (int compact = 0; compact <= 1; compact++)
- {
- QueryProcessor.executeInternal(String.format("TRUNCATE legacy_tables.legacy_%s_simple%s", legacyVersion, getCompactNameSuffix(compact)));
- QueryProcessor.executeInternal(String.format("TRUNCATE legacy_tables.legacy_%s_simple_counter%s", legacyVersion, getCompactNameSuffix(compact)));
- QueryProcessor.executeInternal(String.format("TRUNCATE legacy_tables.legacy_%s_clust%s", legacyVersion, getCompactNameSuffix(compact)));
- QueryProcessor.executeInternal(String.format("TRUNCATE legacy_tables.legacy_%s_clust_counter%s", legacyVersion, getCompactNameSuffix(compact)));
- }
+ QueryProcessor.executeInternal(String.format("TRUNCATE legacy_tables.legacy_%s_simple", legacyVersion));
+ QueryProcessor.executeInternal(String.format("TRUNCATE legacy_tables.legacy_%s_simple_counter", legacyVersion));
+ QueryProcessor.executeInternal(String.format("TRUNCATE legacy_tables.legacy_%s_clust", legacyVersion));
+ QueryProcessor.executeInternal(String.format("TRUNCATE legacy_tables.legacy_%s_clust_counter", legacyVersion));
CacheService.instance.invalidateCounterCache();
CacheService.instance.invalidateKeyCache();
}
@@ -819,19 +577,19 @@
private static void assertLegacyClustRows(int count, UntypedResultSet rs)
{
Assert.assertNotNull(rs);
- assertEquals(count, rs.size());
+ Assert.assertEquals(count, rs.size());
for (int i = 0; i < count; i++)
{
for (UntypedResultSet.Row r : rs)
{
- assertEquals(128, r.getString("val").length());
+ Assert.assertEquals(128, r.getString("val").length());
}
}
}
- private static void loadLegacyTable(String tablePattern, String legacyVersion, String compactSuffix) throws IOException
+ private static void loadLegacyTable(String tablePattern, String legacyVersion) throws IOException
{
- String table = String.format(tablePattern, legacyVersion, compactSuffix);
+ String table = String.format(tablePattern, legacyVersion);
logger.info("Loading legacy table {}", table);
@@ -845,43 +603,6 @@
cfs.loadNewSSTables();
}
-
- /**
- * Test for CASSANDRA-15778
- */
- @Test
- public void testReadLegacyCqlCreatedTableWithBytes() throws Exception {
- String table = "legacy_ka_cql_created_dense_table_with_bytes";
- QueryProcessor.executeInternal("CREATE TABLE legacy_tables." + table + " (" +
- " k int," +
- " v text," +
- " PRIMARY KEY(k, v)) WITH COMPACT STORAGE");
- loadLegacyTable("legacy_%s_cql_created_dense_table_with_bytes%s", "ka", "");
- QueryProcessor.executeInternal("ALTER TABLE legacy_tables." + table + " ALTER value TYPE 'org.apache.cassandra.db.marshal.BytesType';");
- UntypedResultSet rs = QueryProcessor.executeInternal("SELECT * FROM legacy_tables." + table);
- Assert.assertNotNull(rs);
- assertEquals(1, rs.size());
- assertEquals(ByteBufferUtil.bytes("byte string"), rs.one().getBytes("value"));
- }
-
- /**
- * Test for CASSANDRA-15778
- */
- @Test
- public void testReadLegacyCqlCreatedTableWithInt() throws Exception {
- String table = "legacy_ka_cql_created_dense_table_with_int";
- QueryProcessor.executeInternal("CREATE TABLE legacy_tables." + table + " (" +
- " k int," +
- " v text," +
- " PRIMARY KEY(k, v)) WITH COMPACT STORAGE");
- loadLegacyTable("legacy_%s_cql_created_dense_table_with_int%s", "ka", "");
- QueryProcessor.executeInternal("ALTER TABLE legacy_tables." + table + " ALTER value TYPE 'org.apache.cassandra.db.marshal.BytesType';");
- UntypedResultSet rs = QueryProcessor.executeInternal("SELECT * FROM legacy_tables." + table);
- Assert.assertNotNull(rs);
- assertEquals(1, rs.size());
- assertEquals(ByteBufferUtil.bytes(0xaabbcc), rs.one().getBytes("value"));
- }
-
/**
* Generates sstables for 8 CQL tables (see {@link #createTables(String)}) in <i>current</i>
* sstable format (version) into {@code test/data/legacy-sstables/VERSION}, where
@@ -903,28 +624,24 @@
}
String randomString = sb.toString();
- for (int compact = 0; compact <= 1; compact++)
+ for (int pk = 0; pk < 5; pk++)
{
- for (int pk = 0; pk < 5; pk++)
+ String valPk = Integer.toString(pk);
+ QueryProcessor.executeInternal(String.format("INSERT INTO legacy_tables.legacy_%s_simple (pk, val) VALUES ('%s', '%s')",
+ BigFormat.latestVersion, valPk, "foo bar baz"));
+
+ QueryProcessor.executeInternal(String.format("UPDATE legacy_tables.legacy_%s_simple_counter SET val = val + 1 WHERE pk = '%s'",
+ BigFormat.latestVersion, valPk));
+
+ for (int ck = 0; ck < 50; ck++)
{
- String valPk = Integer.toString(pk);
- QueryProcessor.executeInternal(String.format("INSERT INTO legacy_tables.legacy_%s_simple%s (pk, val) VALUES ('%s', '%s')",
- BigFormat.latestVersion, getCompactNameSuffix(compact), valPk, "foo bar baz"));
+ String valCk = Integer.toString(ck);
- QueryProcessor.executeInternal(String.format("UPDATE legacy_tables.legacy_%s_simple_counter%s SET val = val + 1 WHERE pk = '%s'",
- BigFormat.latestVersion, getCompactNameSuffix(compact), valPk));
+ QueryProcessor.executeInternal(String.format("INSERT INTO legacy_tables.legacy_%s_clust (pk, ck, val) VALUES ('%s', '%s', '%s')",
+ BigFormat.latestVersion, valPk, valCk + longString, randomString));
- for (int ck = 0; ck < 50; ck++)
- {
- String valCk = Integer.toString(ck);
-
- QueryProcessor.executeInternal(String.format("INSERT INTO legacy_tables.legacy_%s_clust%s (pk, ck, val) VALUES ('%s', '%s', '%s')",
- BigFormat.latestVersion, getCompactNameSuffix(compact), valPk, valCk + longString, randomString));
-
- QueryProcessor.executeInternal(String.format("UPDATE legacy_tables.legacy_%s_clust_counter%s SET val = val + 1 WHERE pk = '%s' AND ck='%s'",
- BigFormat.latestVersion, getCompactNameSuffix(compact), valPk, valCk + longString));
-
- }
+ QueryProcessor.executeInternal(String.format("UPDATE legacy_tables.legacy_%s_clust_counter SET val = val + 1 WHERE pk = '%s' AND ck='%s'",
+ BigFormat.latestVersion, valPk, valCk + longString));
}
}
@@ -932,13 +649,10 @@
File ksDir = new File(LEGACY_SSTABLE_ROOT, String.format("%s/legacy_tables", BigFormat.latestVersion));
ksDir.mkdirs();
- for (int compact = 0; compact <= 1; compact++)
- {
- copySstablesFromTestData(String.format("legacy_%s_simple%s", BigFormat.latestVersion, getCompactNameSuffix(compact)), ksDir);
- copySstablesFromTestData(String.format("legacy_%s_simple_counter%s", BigFormat.latestVersion, getCompactNameSuffix(compact)), ksDir);
- copySstablesFromTestData(String.format("legacy_%s_clust%s", BigFormat.latestVersion, getCompactNameSuffix(compact)), ksDir);
- copySstablesFromTestData(String.format("legacy_%s_clust_counter%s", BigFormat.latestVersion, getCompactNameSuffix(compact)), ksDir);
- }
+ copySstablesFromTestData(String.format("legacy_%s_simple", BigFormat.latestVersion), ksDir);
+ copySstablesFromTestData(String.format("legacy_%s_simple_counter", BigFormat.latestVersion), ksDir);
+ copySstablesFromTestData(String.format("legacy_%s_clust", BigFormat.latestVersion), ksDir);
+ copySstablesFromTestData(String.format("legacy_%s_clust_counter", BigFormat.latestVersion), ksDir);
}
public static void copySstablesFromTestData(String table, File ksDir) throws IOException
diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableCorruptionDetectionTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableCorruptionDetectionTest.java
index f7ced23..2510c5e 100644
--- a/test/unit/org/apache/cassandra/io/sstable/SSTableCorruptionDetectionTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/SSTableCorruptionDetectionTest.java
@@ -70,22 +70,24 @@
private static LifecycleTransaction txn;
private static ColumnFamilyStore cfs;
private static SSTableReader ssTableReader;
+ private static Config.CorruptedTombstoneStrategy original;
@BeforeClass
public static void setUp()
{
- CFMetaData cfm = CFMetaData.Builder.create(keyspace, table)
- .addPartitionKey("pk", AsciiType.instance)
- .addClusteringColumn("ck1", AsciiType.instance)
- .addClusteringColumn("ck2", AsciiType.instance)
- .addRegularColumn("reg1", BytesType.instance)
- .addRegularColumn("reg2", BytesType.instance)
- .build();
+ // this test writes corrupted data on purpose, disable corrupted tombstone detection
+ original = DatabaseDescriptor.getCorruptedTombstoneStrategy();
+ DatabaseDescriptor.setCorruptedTombstoneStrategy(Config.CorruptedTombstoneStrategy.disabled);
+ TableMetadata.Builder cfm =
+ TableMetadata.builder(keyspace, table)
+ .addPartitionKeyColumn("pk", AsciiType.instance)
+ .addClusteringColumn("ck1", AsciiType.instance)
+ .addClusteringColumn("ck2", AsciiType.instance)
+ .addRegularColumn("reg1", BytesType.instance)
+ .addRegularColumn("reg2", BytesType.instance)
+ .compression(CompressionParams.noCompression());
- cfm.compression(CompressionParams.noCompression());
- SchemaLoader.createKeyspace(keyspace,
- KeyspaceParams.simple(1),
- cfm);
+ SchemaLoader.createKeyspace(keyspace, KeyspaceParams.simple(1), cfm);
cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
cfs.disableAutoCompaction();
@@ -105,7 +107,7 @@
writer = getWriter(cfs, dir, txn);
for (int i = 0; i < numberOfPks; i++)
{
- UpdateBuilder builder = UpdateBuilder.create(cfs.metadata, String.format("pkvalue_%07d", i)).withTimestamp(1);
+ UpdateBuilder builder = UpdateBuilder.create(cfs.metadata(), String.format("pkvalue_%07d", i)).withTimestamp(1);
byte[] reg1 = new byte[valueSize];
random.nextBytes(reg1);
byte[] reg2 = new byte[valueSize];
@@ -129,6 +131,7 @@
txn.abort();
writer.close();
+ DatabaseDescriptor.setCorruptedTombstoneStrategy(original);
}
@Test
@@ -213,8 +216,7 @@
DecoratedKey dk = Util.dk(String.format("pkvalue_%07d", i));
try (UnfilteredRowIterator rowIter = sstable.iterator(dk,
Slices.ALL,
- ColumnFilter.all(cfs.metadata),
- false,
+ ColumnFilter.all(cfs.metadata()),
false,
SSTableReadsListener.NOOP_LISTENER))
{
diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableHeaderFixTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableHeaderFixTest.java
index c2eadc4..d07187b 100644
--- a/test/unit/org/apache/cassandra/io/sstable/SSTableHeaderFixTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/SSTableHeaderFixTest.java
@@ -19,31 +19,27 @@
import java.io.File;
import java.nio.ByteBuffer;
-import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
-import java.util.UUID;
import java.util.function.Function;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
-import java.util.stream.Stream;
import com.google.common.collect.Sets;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.ColumnIdentifier;
import org.apache.cassandra.cql3.FieldIdentifier;
-import org.apache.cassandra.cql3.statements.IndexTarget;
+import org.apache.cassandra.cql3.statements.schema.IndexTarget;
import org.apache.cassandra.db.SerializationHeader;
import org.apache.cassandra.db.marshal.AbstractCompositeType;
import org.apache.cassandra.db.marshal.AbstractType;
@@ -59,15 +55,15 @@
import org.apache.cassandra.db.marshal.UTF8Type;
import org.apache.cassandra.db.marshal.UserType;
import org.apache.cassandra.db.rows.EncodingStats;
-import org.apache.cassandra.dht.Murmur3Partitioner;
import org.apache.cassandra.io.sstable.format.SSTableFormat;
import org.apache.cassandra.io.sstable.format.Version;
import org.apache.cassandra.io.sstable.format.big.BigFormat;
import org.apache.cassandra.io.sstable.metadata.MetadataType;
import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.io.util.SequentialWriter;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.schema.IndexMetadata;
-import org.apache.cassandra.schema.SchemaKeyspace;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FBUtilities;
@@ -85,7 +81,6 @@
static
{
DatabaseDescriptor.toolInitialization();
- DatabaseDescriptor.applyAddressConfig();
}
private File temporaryFolder;
@@ -167,12 +162,12 @@
private static final Version version = BigFormat.instance.getVersion("mc");
- private CFMetaData tableMetadata;
+ private TableMetadata tableMetadata;
private final Set<String> updatedColumns = new HashSet<>();
- private ColumnDefinition getColDef(String n)
+ private ColumnMetadata getColDef(String n)
{
- return tableMetadata.getColumnDefinition(ByteBufferUtil.bytes(n));
+ return tableMetadata.getColumn(ByteBufferUtil.bytes(n));
}
/**
@@ -187,9 +182,11 @@
SerializationHeader.Component header = readHeader(sstable);
assertFrozenUdt(header, false, true);
- ColumnDefinition cd = getColDef("regular_c");
- tableMetadata.removeColumnDefinition(cd);
- tableMetadata.addColumnDefinition(ColumnDefinition.regularDef("ks", "cf", "regular_c", FloatType.instance));
+ ColumnMetadata cd = getColDef("regular_c");
+ tableMetadata = tableMetadata.unbuild()
+ .removeRegularOrStaticColumn(cd.name)
+ .addRegularColumn("regular_c", FloatType.instance)
+ .build();
SSTableHeaderFix headerFix = builder().withPath(sstable.toPath())
.build();
@@ -208,9 +205,9 @@
{
File dir = temporaryFolder;
- List<ColumnDefinition> cols = new ArrayList<>();
- cols.add(ColumnDefinition.partitionKeyDef("ks", "cf", "pk", udtPK, 0));
- cols.add(ColumnDefinition.clusteringDef("ks", "cf", "ck", udtCK, 0));
+ TableMetadata.Builder cols = TableMetadata.builder("ks", "cf")
+ .addPartitionKeyColumn("pk", udtPK)
+ .addClusteringColumn("ck", udtCK);
commonColumns(cols);
File sstable = buildFakeSSTable(dir, 1, cols, false);
@@ -237,18 +234,20 @@
public void verifyWithUnknownColumnTest() throws Exception
{
File dir = temporaryFolder;
- List<ColumnDefinition> cols = new ArrayList<>();
- cols.add(ColumnDefinition.partitionKeyDef("ks", "cf", "pk", udtPK, 0));
- cols.add(ColumnDefinition.clusteringDef("ks", "cf", "ck", udtCK, 0));
+ TableMetadata.Builder cols = TableMetadata.builder("ks", "cf")
+ .addPartitionKeyColumn("pk", udtPK)
+ .addClusteringColumn("ck", udtCK);
commonColumns(cols);
- cols.add(ColumnDefinition.regularDef("ks", "cf", "solr_query", UTF8Type.instance));
+ cols.addRegularColumn("solr_query", UTF8Type.instance);
File sstable = buildFakeSSTable(dir, 1, cols, true);
SerializationHeader.Component header = readHeader(sstable);
assertFrozenUdt(header, false, true);
- ColumnDefinition cd = getColDef("solr_query");
- tableMetadata.removeColumnDefinition(cd);
+ ColumnMetadata cd = getColDef("solr_query");
+ tableMetadata = tableMetadata.unbuild()
+ .removeRegularOrStaticColumn(cd.name)
+ .build();
SSTableHeaderFix headerFix = builder().withPath(sstable.toPath())
.build();
@@ -270,19 +269,21 @@
public void verifyWithIndexedUnknownColumnTest() throws Exception
{
File dir = temporaryFolder;
- List<ColumnDefinition> cols = new ArrayList<>();
- cols.add(ColumnDefinition.partitionKeyDef("ks", "cf", "pk", udtPK, 0));
- cols.add(ColumnDefinition.clusteringDef("ks", "cf", "ck", udtCK, 0));
+ TableMetadata.Builder cols = TableMetadata.builder("ks", "cf")
+ .addPartitionKeyColumn("pk", udtPK)
+ .addClusteringColumn("ck", udtCK);
commonColumns(cols);
- cols.add(ColumnDefinition.regularDef("ks", "cf", "solr_query", UTF8Type.instance));
+ cols.addRegularColumn("solr_query", UTF8Type.instance);
File sstable = buildFakeSSTable(dir, 1, cols, true);
SerializationHeader.Component header = readHeader(sstable);
assertFrozenUdt(header, false, true);
- ColumnDefinition cd = getColDef("solr_query");
- tableMetadata.indexes(tableMetadata.getIndexes().with(IndexMetadata.fromSchemaMetadata("some search index", IndexMetadata.Kind.CUSTOM, Collections.singletonMap(IndexTarget.TARGET_OPTION_NAME, "solr_query"))));
- tableMetadata.removeColumnDefinition(cd);
+ ColumnMetadata cd = getColDef("solr_query");
+ tableMetadata = tableMetadata.unbuild()
+ .indexes(tableMetadata.indexes.with(IndexMetadata.fromSchemaMetadata("some search index", IndexMetadata.Kind.CUSTOM, Collections.singletonMap(IndexTarget.TARGET_OPTION_NAME, "solr_query"))))
+ .removeRegularOrStaticColumn(cd.name)
+ .build();
SSTableHeaderFix headerFix = builder().withPath(sstable.toPath())
.build();
@@ -301,21 +302,21 @@
{
File dir = temporaryFolder;
- List<ColumnDefinition> cols = new ArrayList<>();
- cols.add(ColumnDefinition.partitionKeyDef("ks", "cf", "pk", udtPK, 0));
- cols.add(ColumnDefinition.clusteringDef("ks", "cf", "ck", udtCK, 0));
+ TableMetadata.Builder cols = TableMetadata.builder("ks", "cf")
+ .addPartitionKeyColumn("pk", udtPK)
+ .addClusteringColumn("ck", udtCK);
commonColumns(cols);
- cols.add(ColumnDefinition.regularDef("ks", "cf", "tuple_in_tuple", tupleInTuple));
- cols.add(ColumnDefinition.regularDef("ks", "cf", "udt_nested", udtNested));
- cols.add(ColumnDefinition.regularDef("ks", "cf", "udt_in_tuple", udtInTuple));
- cols.add(ColumnDefinition.regularDef("ks", "cf", "tuple_in_composite", tupleInComposite));
- cols.add(ColumnDefinition.regularDef("ks", "cf", "udt_in_composite", udtInComposite));
- cols.add(ColumnDefinition.regularDef("ks", "cf", "udt_in_list", udtInList));
- cols.add(ColumnDefinition.regularDef("ks", "cf", "udt_in_set", udtInSet));
- cols.add(ColumnDefinition.regularDef("ks", "cf", "udt_in_map", udtInMap));
- cols.add(ColumnDefinition.regularDef("ks", "cf", "udt_in_frozen_list", udtInFrozenList));
- cols.add(ColumnDefinition.regularDef("ks", "cf", "udt_in_frozen_set", udtInFrozenSet));
- cols.add(ColumnDefinition.regularDef("ks", "cf", "udt_in_frozen_map", udtInFrozenMap));
+ cols.addRegularColumn("tuple_in_tuple", tupleInTuple)
+ .addRegularColumn("udt_nested", udtNested)
+ .addRegularColumn("udt_in_tuple", udtInTuple)
+ .addRegularColumn("tuple_in_composite", tupleInComposite)
+ .addRegularColumn("udt_in_composite", udtInComposite)
+ .addRegularColumn("udt_in_list", udtInList)
+ .addRegularColumn("udt_in_set", udtInSet)
+ .addRegularColumn("udt_in_map", udtInMap)
+ .addRegularColumn("udt_in_frozen_list", udtInFrozenList)
+ .addRegularColumn("udt_in_frozen_set", udtInFrozenSet)
+ .addRegularColumn("udt_in_frozen_map", udtInFrozenMap);
File sstable = buildFakeSSTable(dir, 1, cols, true);
SerializationHeader.Component header = readHeader(sstable);
@@ -340,33 +341,36 @@
{
File dir = temporaryFolder;
- List<ColumnDefinition> cols = new ArrayList<>();
- cols.add(ColumnDefinition.partitionKeyDef("ks", "cf", "pk", udtPK, 0));
- cols.add(ColumnDefinition.clusteringDef("ks", "cf", "ck", udtCK, 0));
+ TableMetadata.Builder cols = TableMetadata.builder("ks", "cf")
+ .addPartitionKeyColumn("pk", udtPK)
+ .addClusteringColumn("ck", udtCK);
commonColumns(cols);
- cols.add(ColumnDefinition.regularDef("ks", "cf", "tuple_in_tuple", tupleInTuple));
- cols.add(ColumnDefinition.regularDef("ks", "cf", "udt_nested", udtNested));
- cols.add(ColumnDefinition.regularDef("ks", "cf", "udt_in_tuple", udtInTuple));
- cols.add(ColumnDefinition.regularDef("ks", "cf", "tuple_in_composite", tupleInComposite));
- cols.add(ColumnDefinition.regularDef("ks", "cf", "udt_in_composite", udtInComposite));
- cols.add(ColumnDefinition.regularDef("ks", "cf", "udt_in_list", udtInList));
- cols.add(ColumnDefinition.regularDef("ks", "cf", "udt_in_set", udtInSet));
- cols.add(ColumnDefinition.regularDef("ks", "cf", "udt_in_map", udtInMap));
- cols.add(ColumnDefinition.regularDef("ks", "cf", "udt_in_frozen_list", udtInFrozenList));
- cols.add(ColumnDefinition.regularDef("ks", "cf", "udt_in_frozen_set", udtInFrozenSet));
- cols.add(ColumnDefinition.regularDef("ks", "cf", "udt_in_frozen_map", udtInFrozenMap));
+ cols.addRegularColumn("tuple_in_tuple", tupleInTuple)
+ .addRegularColumn("udt_nested", udtNested)
+ .addRegularColumn("udt_in_tuple", udtInTuple)
+ .addRegularColumn("tuple_in_composite", tupleInComposite)
+ .addRegularColumn("udt_in_composite", udtInComposite)
+ .addRegularColumn("udt_in_list", udtInList)
+ .addRegularColumn("udt_in_set", udtInSet)
+ .addRegularColumn("udt_in_map", udtInMap)
+ .addRegularColumn("udt_in_frozen_list", udtInFrozenList)
+ .addRegularColumn("udt_in_frozen_set", udtInFrozenSet)
+ .addRegularColumn("udt_in_frozen_map", udtInFrozenMap);
File sstable = buildFakeSSTable(dir, 1, cols, true);
+ cols = tableMetadata.unbuild();
for (String col : new String[]{"tuple_in_tuple", "udt_nested", "udt_in_tuple",
"tuple_in_composite", "udt_in_composite",
"udt_in_list", "udt_in_set", "udt_in_map",
"udt_in_frozen_list", "udt_in_frozen_set", "udt_in_frozen_map"})
{
- ColumnDefinition cd = getColDef(col);
- tableMetadata.removeColumnDefinition(cd);
- AbstractType<?> dropType = SchemaKeyspace.expandUserTypes(cd.type);
- tableMetadata.recordColumnDrop(new ColumnDefinition(cd.ksName, cd.cfName, cd.name, dropType, cd.position(), cd.kind), FBUtilities.timestampMicros());
+ ColumnIdentifier ci = new ColumnIdentifier(col, true);
+ ColumnMetadata cd = getColDef(col);
+ AbstractType<?> dropType = cd.type.expandUserTypes();
+ cols.removeRegularOrStaticColumn(ci)
+ .recordColumnDrop(new ColumnMetadata(cd.ksName, cd.cfName, cd.name, dropType, cd.position(), cd.kind), FBUtilities.timestampMicros());
}
+ tableMetadata = cols.build();
SerializationHeader.Component header = readHeader(sstable);
assertFrozenUdt(header, false, true);
@@ -390,9 +394,9 @@
{
File dir = temporaryFolder;
- List<ColumnDefinition> cols = new ArrayList<>();
- cols.add(ColumnDefinition.partitionKeyDef("ks", "cf", "pk", udtPK, 0));
- cols.add(ColumnDefinition.clusteringDef("ks", "cf", "ck", udtCK, 0));
+ TableMetadata.Builder cols = TableMetadata.builder("ks", "cf")
+ .addPartitionKeyColumn("pk", udtPK)
+ .addClusteringColumn("ck", udtCK);
ColSpec[] colSpecs = new ColSpec[]
{
@@ -410,14 +414,14 @@
true),
// 'frozen<udt>' / dropped
new ColSpec("frozen_udt_as_frozen_udt_dropped",
- SchemaKeyspace.expandUserTypes(makeUDT2("frozen_udt_as_frozen_udt_dropped", true).freezeNestedMulticellTypes().freeze()),
+ makeUDT2("frozen_udt_as_frozen_udt_dropped", true).freezeNestedMulticellTypes().freeze().expandUserTypes(),
makeUDT2("frozen_udt_as_frozen_udt_dropped", false),
makeUDT2("frozen_udt_as_frozen_udt_dropped", false),
true,
false),
// 'frozen<udt>' / dropped / as 'udt'
new ColSpec("frozen_udt_as_unfrozen_udt_dropped",
- SchemaKeyspace.expandUserTypes(makeUDT2("frozen_udt_as_unfrozen_udt_dropped", true).freezeNestedMulticellTypes().freeze()),
+ makeUDT2("frozen_udt_as_unfrozen_udt_dropped", true).freezeNestedMulticellTypes().freeze().expandUserTypes(),
makeUDT2("frozen_udt_as_unfrozen_udt_dropped", true),
makeUDT2("frozen_udt_as_unfrozen_udt_dropped", false),
true,
@@ -431,7 +435,7 @@
// 'udt' / dropped
// TODO unable to test dropping a non-frozen UDT, as that requires an unfrozen tuple as well
// new ColSpec("unfrozen_udt_as_unfrozen_udt_dropped",
-// SchemaKeyspace.expandUserTypes(makeUDT2("unfrozen_udt_as_unfrozen_udt_dropped", true).freezeNestedMulticellTypes()),
+// makeUDT2("unfrozen_udt_as_unfrozen_udt_dropped", true).freezeNestedMulticellTypes().expandUserTypes(),
// makeUDT2("unfrozen_udt_as_unfrozen_udt_dropped", true),
// makeUDT2("unfrozen_udt_as_unfrozen_udt_dropped", true),
// true,
@@ -450,25 +454,27 @@
false)
};
- Arrays.stream(colSpecs).forEach(c -> cols.add(ColumnDefinition.regularDef("ks", "cf", c.name,
- // use the initial column type for the serialization header header.
- c.preFix)));
+ Arrays.stream(colSpecs).forEach(c -> cols.addRegularColumn(c.name,
+ // use the initial column type for the serialization header header.
+ c.preFix));
Map<String, ColSpec> colSpecMap = Arrays.stream(colSpecs).collect(Collectors.toMap(c -> c.name, c -> c));
- File sstable = buildFakeSSTable(dir, 1, cols, (s) -> s.map(c -> {
+ File sstable = buildFakeSSTable(dir, 1, cols, c -> {
ColSpec cs = colSpecMap.get(c.name.toString());
if (cs == null)
return c;
// update the column type in the schema to the "correct" one.
- return new ColumnDefinition(c.ksName, c.cfName, c.name, cs.schema, c.position(), c.kind);
- }));
+ return c.withNewType(cs.schema);
+ });
Arrays.stream(colSpecs)
.filter(c -> c.dropped)
.forEach(c -> {
- ColumnDefinition cd = getColDef(c.name);
- tableMetadata.removeColumnDefinition(cd);
- tableMetadata.recordColumnDrop(cd, FBUtilities.timestampMicros());
+ ColumnMetadata cd = getColDef(c.name);
+ tableMetadata = tableMetadata.unbuild()
+ .removeRegularOrStaticColumn(cd.name)
+ .recordColumnDrop(cd, FBUtilities.timestampMicros())
+ .build();
});
SerializationHeader.Component header = readHeader(sstable);
@@ -531,10 +537,10 @@
{
File dir = temporaryFolder;
- List<ColumnDefinition> cols = new ArrayList<>();
- cols.add(ColumnDefinition.partitionKeyDef("ks", "cf", "pk1", UTF8Type.instance, 0));
- cols.add(ColumnDefinition.partitionKeyDef("ks", "cf", "pk2", udtPK, 1));
- cols.add(ColumnDefinition.clusteringDef("ks", "cf", "ck", udtCK, 0));
+ TableMetadata.Builder cols = TableMetadata.builder("ks", "cf")
+ .addPartitionKeyColumn("pk1", UTF8Type.instance)
+ .addPartitionKeyColumn("pk2", udtPK)
+ .addClusteringColumn("ck", udtCK);
commonColumns(cols);
File sstable = buildFakeSSTable(dir, 1, cols, false);
@@ -557,10 +563,10 @@
@Test
public void compositePartitionKey() throws Exception
{
- List<ColumnDefinition> cols = new ArrayList<>();
- cols.add(ColumnDefinition.partitionKeyDef("ks", "cf", "pk1", UTF8Type.instance, 0));
- cols.add(ColumnDefinition.partitionKeyDef("ks", "cf", "pk2", udtPK, 1));
- cols.add(ColumnDefinition.clusteringDef("ks", "cf", "ck", udtCK, 0));
+ TableMetadata.Builder cols = TableMetadata.builder("ks", "cf")
+ .addPartitionKeyColumn("pk1", UTF8Type.instance)
+ .addPartitionKeyColumn("pk2", udtPK)
+ .addClusteringColumn("ck", udtCK);
commonColumns(cols);
File dir = temporaryFolder;
@@ -588,10 +594,10 @@
@Test
public void compositeClusteringKey() throws Exception
{
- List<ColumnDefinition> cols = new ArrayList<>();
- cols.add(ColumnDefinition.partitionKeyDef("ks", "cf", "pk", udtPK, 0));
- cols.add(ColumnDefinition.clusteringDef("ks", "cf", "ck1", Int32Type.instance, 0));
- cols.add(ColumnDefinition.clusteringDef("ks", "cf", "ck2", udtCK, 1));
+ TableMetadata.Builder cols = TableMetadata.builder("ks", "cf")
+ .addPartitionKeyColumn("pk", udtPK)
+ .addClusteringColumn("ck1", Int32Type.instance)
+ .addClusteringColumn("ck2", udtCK);
commonColumns(cols);
File dir = temporaryFolder;
@@ -748,65 +754,60 @@
private File generateFakeSSTable(File dir, int generation)
{
- List<ColumnDefinition> cols = new ArrayList<>();
- cols.add(ColumnDefinition.partitionKeyDef("ks", "cf", "pk", udtPK, 0));
- cols.add(ColumnDefinition.clusteringDef("ks", "cf", "ck", udtCK, 0));
+ TableMetadata.Builder cols = TableMetadata.builder("ks", "cf")
+ .addPartitionKeyColumn("pk", udtPK)
+ .addClusteringColumn("ck", udtCK);
commonColumns(cols);
return buildFakeSSTable(dir, generation, cols, true);
}
- private void commonColumns(List<ColumnDefinition> cols)
+ private void commonColumns(TableMetadata.Builder cols)
{
- cols.add(ColumnDefinition.regularDef("ks", "cf", "regular_a", UTF8Type.instance));
- cols.add(ColumnDefinition.regularDef("ks", "cf", "regular_b", udtRegular));
- cols.add(ColumnDefinition.regularDef("ks", "cf", "regular_c", Int32Type.instance));
- cols.add(ColumnDefinition.staticDef("ks", "cf", "static_a", UTF8Type.instance));
- cols.add(ColumnDefinition.staticDef("ks", "cf", "static_b", udtStatic));
- cols.add(ColumnDefinition.staticDef("ks", "cf", "static_c", Int32Type.instance));
+ cols.addRegularColumn("regular_a", UTF8Type.instance)
+ .addRegularColumn("regular_b", udtRegular)
+ .addRegularColumn("regular_c", Int32Type.instance)
+ .addStaticColumn("static_a", UTF8Type.instance)
+ .addStaticColumn("static_b", udtStatic)
+ .addStaticColumn("static_c", Int32Type.instance);
}
- private File buildFakeSSTable(File dir, int generation, List<ColumnDefinition> cols, boolean freezeInSchema)
+ private File buildFakeSSTable(File dir, int generation, TableMetadata.Builder cols, boolean freezeInSchema)
{
return buildFakeSSTable(dir, generation, cols, freezeInSchema
- ? s -> s.map(c -> new ColumnDefinition(c.ksName, c.cfName, c.name, freezeUdt(c.type), c.position(), c.kind))
- : s -> s);
+ ? c -> c.withNewType(freezeUdt(c.type))
+ : c -> c);
}
- private File buildFakeSSTable(File dir, int generation, List<ColumnDefinition> cols, Function<Stream<ColumnDefinition>, Stream<ColumnDefinition>> freezer)
+ private File buildFakeSSTable(File dir, int generation, TableMetadata.Builder cols, Function<ColumnMetadata, ColumnMetadata> freezer)
{
- CFMetaData headerMetadata = CFMetaData.create("ks", "cf",
- UUID.randomUUID(),
- false, true, false, false, false,
- cols,
- new Murmur3Partitioner());
+ TableMetadata headerMetadata = cols.build();
- List<ColumnDefinition> schemaCols = freezer.apply(cols.stream()).collect(Collectors.toList());
- tableMetadata = CFMetaData.create("ks", "cf",
- UUID.randomUUID(),
- false, true, false, false, false,
- schemaCols,
- new Murmur3Partitioner());
+ TableMetadata.Builder schemaCols = TableMetadata.builder("ks", "cf");
+ for (ColumnMetadata cm : cols.columns())
+ schemaCols.addColumn(freezer.apply(cm));
+ tableMetadata = schemaCols.build();
try
{
- Descriptor desc = new Descriptor(version, dir, "ks", "cf", generation, SSTableFormat.Type.BIG, Component.DATA);
+
+ Descriptor desc = new Descriptor(version, dir, "ks", "cf", generation, SSTableFormat.Type.BIG);
// Just create the component files - we don't really need those.
for (Component component : requiredComponents)
assertTrue(new File(desc.filenameFor(component)).createNewFile());
- AbstractType<?> partitionKey = headerMetadata.getKeyValidator();
+ AbstractType<?> partitionKey = headerMetadata.partitionKeyType;
List<AbstractType<?>> clusteringKey = headerMetadata.clusteringColumns()
.stream()
.map(cd -> cd.type)
.collect(Collectors.toList());
- Map<ByteBuffer, AbstractType<?>> staticColumns = headerMetadata.allColumns()
+ Map<ByteBuffer, AbstractType<?>> staticColumns = headerMetadata.columns()
.stream()
- .filter(cd -> cd.kind == ColumnDefinition.Kind.STATIC)
+ .filter(cd -> cd.kind == ColumnMetadata.Kind.STATIC)
.collect(Collectors.toMap(cd -> cd.name.bytes, cd -> cd.type, (a, b) -> a));
- Map<ByteBuffer, AbstractType<?>> regularColumns = headerMetadata.allColumns()
+ Map<ByteBuffer, AbstractType<?>> regularColumns = headerMetadata.columns()
.stream()
- .filter(cd -> cd.kind == ColumnDefinition.Kind.REGULAR)
+ .filter(cd -> cd.kind == ColumnMetadata.Kind.REGULAR)
.collect(Collectors.toMap(cd -> cd.name.bytes, cd -> cd.type, (a, b) -> a));
File statsFile = new File(desc.filenameFor(Component.STATS));
@@ -955,7 +956,7 @@
private SerializationHeader.Component readHeader(File sstable) throws Exception
{
- Descriptor desc = Descriptor.fromFilename(sstable.getParentFile(), sstable.getName()).left;
+ Descriptor desc = Descriptor.fromFilename(sstable);
return (SerializationHeader.Component) desc.getMetadataSerializer().deserialize(desc, MetadataType.HEADER);
}
diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableLoaderTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableLoaderTest.java
index 72c7467..5d40f8c 100644
--- a/test/unit/org/apache/cassandra/io/sstable/SSTableLoaderTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/SSTableLoaderTest.java
@@ -31,13 +31,13 @@
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.Util;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.Schema;
+import org.apache.cassandra.locator.Replica;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.schema.TableMetadataRef;
+import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.partitions.*;
import org.apache.cassandra.db.marshal.AsciiType;
-import org.apache.cassandra.dht.Range;
-import org.apache.cassandra.dht.Token;
import org.apache.cassandra.io.FSWriteError;
import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.schema.KeyspaceParams;
@@ -55,6 +55,7 @@
public class SSTableLoaderTest
{
public static final String KEYSPACE1 = "SSTableLoaderTest";
+ public static final String KEYSPACE2 = "SSTableLoaderTest1";
public static final String CF_STANDARD1 = "Standard1";
public static final String CF_STANDARD2 = "Standard2";
@@ -69,6 +70,11 @@
SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD1),
SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD2));
+ SchemaLoader.createKeyspace(KEYSPACE2,
+ KeyspaceParams.simple(1),
+ SchemaLoader.standardCFMD(KEYSPACE2, CF_STANDARD1),
+ SchemaLoader.standardCFMD(KEYSPACE2, CF_STANDARD2));
+
StorageService.instance.initServer();
}
@@ -102,13 +108,13 @@
public void init(String keyspace)
{
this.keyspace = keyspace;
- for (Range<Token> range : StorageService.instance.getLocalRanges(KEYSPACE1))
- addRangeForEndpoint(range, FBUtilities.getBroadcastAddress());
+ for (Replica replica : StorageService.instance.getLocalReplicas(KEYSPACE1))
+ addRangeForEndpoint(replica.range(), FBUtilities.getBroadcastAddressAndPort());
}
- public CFMetaData getTableMetadata(String tableName)
+ public TableMetadataRef getTableMetadata(String tableName)
{
- return Schema.instance.getCFMetaData(keyspace, tableName);
+ return Schema.instance.getTableMetadataRef(keyspace, tableName);
}
}
@@ -117,7 +123,7 @@
{
File dataDir = new File(tmpdir.getAbsolutePath() + File.separator + KEYSPACE1 + File.separator + CF_STANDARD1);
assert dataDir.mkdirs();
- CFMetaData cfmeta = Schema.instance.getCFMetaData(KEYSPACE1, CF_STANDARD1);
+ TableMetadata metadata = Schema.instance.getTableMetadata(KEYSPACE1, CF_STANDARD1);
String schema = "CREATE TABLE %s.%s (key ascii, name ascii, val ascii, val1 ascii, PRIMARY KEY (key, name))";
String query = "INSERT INTO %s.%s (key, name, val) VALUES (?, ?, ?)";
@@ -143,7 +149,7 @@
assertEquals(1, partitions.size());
assertEquals("key1", AsciiType.instance.getString(partitions.get(0).partitionKey().getKey()));
assertEquals(ByteBufferUtil.bytes("100"), partitions.get(0).getRow(Clustering.make(ByteBufferUtil.bytes("col1")))
- .getCell(cfmeta.getColumnDefinition(ByteBufferUtil.bytes("val")))
+ .getCell(metadata.getColumn(ByteBufferUtil.bytes("val")))
.value());
// The stream future is signalled when the work is complete but before releasing references. Wait for release
@@ -207,6 +213,48 @@
latch.await();
}
+ @Test
+ public void testLoadingSSTableToDifferentKeyspace() throws Exception
+ {
+ File dataDir = new File(tmpdir.getAbsolutePath() + File.separator + KEYSPACE1 + File.separator + CF_STANDARD1);
+ assert dataDir.mkdirs();
+ TableMetadata metadata = Schema.instance.getTableMetadata(KEYSPACE1, CF_STANDARD1);
+
+ String schema = "CREATE TABLE %s.%s (key ascii, name ascii, val ascii, val1 ascii, PRIMARY KEY (key, name))";
+ String query = "INSERT INTO %s.%s (key, name, val) VALUES (?, ?, ?)";
+
+ try (CQLSSTableWriter writer = CQLSSTableWriter.builder()
+ .inDirectory(dataDir)
+ .forTable(String.format(schema, KEYSPACE1, CF_STANDARD1))
+ .using(String.format(query, KEYSPACE1, CF_STANDARD1))
+ .build())
+ {
+ writer.addRow("key1", "col1", "100");
+ }
+
+ ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD1);
+ cfs.forceBlockingFlush(); // wait for sstables to be on disk else we won't be able to stream them
+
+ final CountDownLatch latch = new CountDownLatch(1);
+ SSTableLoader loader = new SSTableLoader(dataDir, new TestClient(), new OutputHandler.SystemOutput(false, false), 1, KEYSPACE2);
+ loader.stream(Collections.emptySet(), completionStreamListener(latch)).get();
+
+ cfs = Keyspace.open(KEYSPACE2).getColumnFamilyStore(CF_STANDARD1);
+ cfs.forceBlockingFlush();
+
+ List<FilteredPartition> partitions = Util.getAll(Util.cmd(cfs).build());
+
+ assertEquals(1, partitions.size());
+ assertEquals("key1", AsciiType.instance.getString(partitions.get(0).partitionKey().getKey()));
+ assertEquals(ByteBufferUtil.bytes("100"), partitions.get(0).getRow(Clustering.make(ByteBufferUtil.bytes("col1")))
+ .getCell(metadata.getColumn(ByteBufferUtil.bytes("val")))
+ .value());
+
+ // The stream future is signalled when the work is complete but before releasing references. Wait for release
+ // before cleanup (CASSANDRA-10118).
+ latch.await();
+ }
+
StreamEventHandler completionStreamListener(final CountDownLatch latch)
{
return new StreamEventHandler() {
diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableMetadataTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableMetadataTest.java
index f39cf3b..ae63fc3 100644
--- a/test/unit/org/apache/cassandra/io/sstable/SSTableMetadataTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/SSTableMetadataTest.java
@@ -27,7 +27,7 @@
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.Util;
-import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.Keyspace;
@@ -60,11 +60,11 @@
SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD),
SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD2),
SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD3),
- CFMetaData.Builder.create(KEYSPACE1, CF_STANDARDCOMPOSITE2)
- .addPartitionKey("key", AsciiType.instance)
- .addClusteringColumn("name", AsciiType.instance)
- .addClusteringColumn("int", IntegerType.instance)
- .addRegularColumn("val", AsciiType.instance).build(),
+ TableMetadata.builder(KEYSPACE1, CF_STANDARDCOMPOSITE2)
+ .addPartitionKeyColumn("key", AsciiType.instance)
+ .addClusteringColumn("name", AsciiType.instance)
+ .addClusteringColumn("int", IntegerType.instance)
+ .addRegularColumn("val", AsciiType.instance),
SchemaLoader.counterCFMD(KEYSPACE1, CF_COUNTER1));
}
@@ -78,7 +78,7 @@
{
DecoratedKey key = Util.dk(Integer.toString(i));
for (int j = 0; j < 10; j++)
- new RowUpdateBuilder(store.metadata, timestamp, 10 + j, Integer.toString(i))
+ new RowUpdateBuilder(store.metadata(), timestamp, 10 + j, Integer.toString(i))
.clustering(Integer.toString(j))
.add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER)
.build()
@@ -86,7 +86,7 @@
}
- new RowUpdateBuilder(store.metadata, timestamp, 10000, "longttl")
+ new RowUpdateBuilder(store.metadata(), timestamp, 10000, "longttl")
.clustering("col")
.add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER)
.build()
@@ -104,7 +104,7 @@
}
- new RowUpdateBuilder(store.metadata, timestamp, 20000, "longttl2")
+ new RowUpdateBuilder(store.metadata(), timestamp, 20000, "longttl2")
.clustering("col")
.add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER)
.build()
@@ -153,14 +153,14 @@
long timestamp = System.currentTimeMillis();
DecoratedKey key = Util.dk("deletetest");
for (int i = 0; i<5; i++)
- new RowUpdateBuilder(store.metadata, timestamp, 100, "deletetest")
+ new RowUpdateBuilder(store.metadata(), timestamp, 100, "deletetest")
.clustering("deletecolumn" + i)
.add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER)
.build()
.applyUnsafe();
- new RowUpdateBuilder(store.metadata, timestamp, 1000, "deletetest")
+ new RowUpdateBuilder(store.metadata(), timestamp, 1000, "deletetest")
.clustering("todelete")
.add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER)
.build()
@@ -176,7 +176,7 @@
assertEquals(ttltimestamp + 1000, firstMaxDelTime, 10);
}
- RowUpdateBuilder.deleteRow(store.metadata, timestamp + 1, "deletetest", "todelete").applyUnsafe();
+ RowUpdateBuilder.deleteRow(store.metadata(), timestamp + 1, "deletetest", "todelete").applyUnsafe();
store.forceBlockingFlush();
assertEquals(2,store.getLiveSSTables().size());
@@ -208,7 +208,7 @@
String key = "row" + j;
for (int i = 100; i<150; i++)
{
- new RowUpdateBuilder(store.metadata, System.currentTimeMillis(), key)
+ new RowUpdateBuilder(store.metadata(), System.currentTimeMillis(), key)
.clustering(j + "col" + i)
.add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER)
.build()
@@ -229,7 +229,7 @@
for (int i = 101; i<299; i++)
{
- new RowUpdateBuilder(store.metadata, System.currentTimeMillis(), key)
+ new RowUpdateBuilder(store.metadata(), System.currentTimeMillis(), key)
.clustering(9 + "col" + i)
.add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER)
.build()
diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java
index cd7b4a7..580b099 100644
--- a/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java
@@ -22,11 +22,11 @@
import java.nio.ByteBuffer;
import java.nio.file.Files;
import java.nio.file.Path;
+import java.nio.file.attribute.FileAttribute;
import java.util.*;
import java.util.concurrent.*;
import com.google.common.collect.Sets;
-import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -39,6 +39,8 @@
import org.apache.cassandra.db.compaction.CompactionManager;
import org.apache.cassandra.db.compaction.OperationType;
import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.db.lifecycle.SSTableSet;
+import org.apache.cassandra.db.lifecycle.View;
import org.apache.cassandra.db.partitions.UnfilteredPartitionIterators;
import org.apache.cassandra.db.rows.Row;
import org.apache.cassandra.dht.IPartitioner;
@@ -54,10 +56,10 @@
import org.apache.cassandra.service.CacheService;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FilterFactory;
-import org.apache.cassandra.utils.Pair;
-import static org.apache.cassandra.cql3.QueryProcessor.executeInternal;
+import static org.apache.cassandra.cql3.QueryProcessor.executeInternal;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
@@ -103,7 +105,7 @@
CompactionManager.instance.disableAutoCompaction();
for (int j = 0; j < 10; j++)
{
- new RowUpdateBuilder(store.metadata, j, String.valueOf(j))
+ new RowUpdateBuilder(store.metadata(), j, String.valueOf(j))
.clustering("0")
.add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER)
.build()
@@ -125,11 +127,11 @@
// confirm that positions increase continuously
SSTableReader sstable = store.getLiveSSTables().iterator().next();
long previous = -1;
- for (Pair<Long,Long> section : sstable.getPositionsForRanges(ranges))
+ for (SSTableReader.PartitionPositionBounds section : sstable.getPositionsForRanges(ranges))
{
- assert previous <= section.left : previous + " ! < " + section.left;
- assert section.left < section.right : section.left + " ! < " + section.right;
- previous = section.right;
+ assert previous <= section.lowerPosition : previous + " ! < " + section.lowerPosition;
+ assert section.lowerPosition < section.upperPosition : section.lowerPosition + " ! < " + section.upperPosition;
+ previous = section.upperPosition;
}
}
@@ -149,7 +151,7 @@
CompactionManager.instance.disableAutoCompaction();
for (int j = 0; j < 100; j += 2)
{
- new RowUpdateBuilder(store.metadata, j, String.valueOf(j))
+ new RowUpdateBuilder(store.metadata(), j, String.valueOf(j))
.clustering("0")
.add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER)
.build()
@@ -191,7 +193,7 @@
for (int j = 0; j < 100; j += 2)
{
- new RowUpdateBuilder(store.metadata, j, String.valueOf(j))
+ new RowUpdateBuilder(store.metadata(), j, String.valueOf(j))
.clustering("0")
.add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER)
.build()
@@ -219,7 +221,7 @@
for (int j = 0; j < 10; j++)
{
- new RowUpdateBuilder(store.metadata, j, String.valueOf(j))
+ new RowUpdateBuilder(store.metadata(), j, String.valueOf(j))
.clustering("0")
.add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER)
.build()
@@ -252,7 +254,7 @@
for (int j = 0; j < 10; j++)
{
- new RowUpdateBuilder(store.metadata, j, String.valueOf(j))
+ new RowUpdateBuilder(store.metadata(), j, String.valueOf(j))
.clustering("0")
.add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER)
.build()
@@ -268,13 +270,13 @@
long p6 = sstable.getPosition(k(6), SSTableReader.Operator.EQ).position;
long p7 = sstable.getPosition(k(7), SSTableReader.Operator.EQ).position;
- Pair<Long, Long> p = sstable.getPositionsForRanges(makeRanges(t(2), t(6))).get(0);
+ SSTableReader.PartitionPositionBounds p = sstable.getPositionsForRanges(makeRanges(t(2), t(6))).get(0);
// range are start exclusive so we should start at 3
- assert p.left == p3;
+ assert p.lowerPosition == p3;
// to capture 6 we have to stop at the start of 7
- assert p.right == p7;
+ assert p.upperPosition == p7;
}
@Test
@@ -285,7 +287,7 @@
ColumnFamilyStore store = keyspace.getColumnFamilyStore(CF_INDEXED);
partitioner = store.getPartitioner();
- new RowUpdateBuilder(store.metadata, System.currentTimeMillis(), "k1")
+ new RowUpdateBuilder(store.metadata(), System.currentTimeMillis(), "k1")
.clustering("0")
.add("birthdate", 1L)
.build()
@@ -307,7 +309,7 @@
CompactionManager.instance.disableAutoCompaction();
for (int j = 0; j < 10; j++)
{
- new RowUpdateBuilder(store.metadata, j, String.valueOf(j))
+ new RowUpdateBuilder(store.metadata(), j, String.valueOf(j))
.clustering("0")
.add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER)
.build()
@@ -344,23 +346,22 @@
DecoratedKey firstKey = null, lastKey = null;
long timestamp = System.currentTimeMillis();
- for (int i = 0; i < store.metadata.params.minIndexInterval; i++)
+ for (int i = 0; i < store.metadata().params.minIndexInterval; i++)
{
DecoratedKey key = Util.dk(String.valueOf(i));
if (firstKey == null)
firstKey = key;
if (lastKey == null)
lastKey = key;
- if (store.metadata.getKeyValidator().compare(lastKey.getKey(), key.getKey()) < 0)
+ if (store.metadata().partitionKeyType.compare(lastKey.getKey(), key.getKey()) < 0)
lastKey = key;
- new RowUpdateBuilder(store.metadata, timestamp, key.getKey())
- .clustering("col")
- .add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER)
- .build()
- .applyUnsafe();
-
+ new RowUpdateBuilder(store.metadata(), timestamp, key.getKey())
+ .clustering("col")
+ .add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER)
+ .build()
+ .applyUnsafe();
}
store.forceBlockingFlush();
@@ -369,8 +370,6 @@
// test to see if sstable can be opened as expected
SSTableReader target = SSTableReader.open(desc);
- Assert.assertEquals(target.getIndexSummarySize(), 1);
- Assert.assertArrayEquals(ByteBufferUtil.getArray(firstKey.getKey()), target.getIndexSummaryKey(0));
assert target.first.equals(firstKey);
assert target.last.equals(lastKey);
@@ -460,7 +459,7 @@
Keyspace keyspace = Keyspace.open(KEYSPACE1);
ColumnFamilyStore store = keyspace.getColumnFamilyStore("Indexed1");
- new RowUpdateBuilder(store.metadata, System.currentTimeMillis(), "k1")
+ new RowUpdateBuilder(store.metadata(), System.currentTimeMillis(), "k1")
.clustering("0")
.add("birthdate", 1L)
.build()
@@ -489,7 +488,7 @@
ColumnFamilyStore store = keyspace.getColumnFamilyStore("Standard1");
partitioner = store.getPartitioner();
- new RowUpdateBuilder(store.metadata, 0, "k1")
+ new RowUpdateBuilder(store.metadata(), 0, "k1")
.clustering("xyz")
.add("val", "abc")
.build()
@@ -499,7 +498,7 @@
boolean foundScanner = false;
for (SSTableReader s : store.getLiveSSTables())
{
- try (ISSTableScanner scanner = s.getScanner(new Range<Token>(t(0), t(1)), null))
+ try (ISSTableScanner scanner = s.getScanner(new Range<Token>(t(0), t(1))))
{
scanner.next(); // throws exception pre 5407
foundScanner = true;
@@ -522,7 +521,7 @@
for (int j = 0; j < 130; j++)
{
- new RowUpdateBuilder(store.metadata, j, String.valueOf(j))
+ new RowUpdateBuilder(store.metadata(), j, String.valueOf(j))
.clustering("0")
.add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER)
.build()
@@ -538,7 +537,7 @@
ranges.add(new Range<Token>(t(98), t(99)));
SSTableReader sstable = store.getLiveSSTables().iterator().next();
- List<Pair<Long,Long>> sections = sstable.getPositionsForRanges(ranges);
+ List<SSTableReader.PartitionPositionBounds> sections = sstable.getPositionsForRanges(ranges);
assert sections.size() == 1 : "Expected to find range in sstable" ;
// re-open the same sstable as it would be during bulk loading
@@ -561,7 +560,7 @@
final int NUM_PARTITIONS = 512;
for (int j = 0; j < NUM_PARTITIONS; j++)
{
- new RowUpdateBuilder(store.metadata, j, String.format("%3d", j))
+ new RowUpdateBuilder(store.metadata(), j, String.format("%3d", j))
.clustering("0")
.add("val", String.format("%3d", j))
.build()
@@ -640,7 +639,7 @@
final int NUM_PARTITIONS = 512;
for (int j = 0; j < NUM_PARTITIONS; j++)
{
- new RowUpdateBuilder(store.metadata, j, String.format("%3d", j))
+ new RowUpdateBuilder(store.metadata(), j, String.format("%3d", j))
.clustering("0")
.add("val", String.format("%3d", j))
.build()
@@ -695,4 +694,90 @@
{
return new BufferDecoratedKey(t(i), ByteBufferUtil.bytes(String.valueOf(i)));
}
+
+ @Test(expected = RuntimeException.class)
+ public void testMoveAndOpenLiveSSTable()
+ {
+ Keyspace keyspace = Keyspace.open(KEYSPACE1);
+ ColumnFamilyStore cfs = keyspace.getColumnFamilyStore("Standard1");
+ SSTableReader sstable = getNewSSTable(cfs);
+ Descriptor notLiveDesc = new Descriptor(new File("/tmp"), "", "", 0);
+ SSTableReader.moveAndOpenSSTable(cfs, sstable.descriptor, notLiveDesc, sstable.components);
+ }
+
+ @Test(expected = RuntimeException.class)
+ public void testMoveAndOpenLiveSSTable2()
+ {
+ Keyspace keyspace = Keyspace.open(KEYSPACE1);
+ ColumnFamilyStore cfs = keyspace.getColumnFamilyStore("Standard1");
+ SSTableReader sstable = getNewSSTable(cfs);
+ Descriptor notLiveDesc = new Descriptor(new File("/tmp"), "", "", 0);
+ SSTableReader.moveAndOpenSSTable(cfs, notLiveDesc, sstable.descriptor, sstable.components);
+ }
+
+ @Test
+ public void testMoveAndOpenSSTable() throws IOException
+ {
+ Keyspace keyspace = Keyspace.open(KEYSPACE1);
+ ColumnFamilyStore cfs = keyspace.getColumnFamilyStore("Standard1");
+ SSTableReader sstable = getNewSSTable(cfs);
+ cfs.clearUnsafe();
+ sstable.selfRef().release();
+ File tmpdir = Files.createTempDirectory("testMoveAndOpen").toFile();
+ tmpdir.deleteOnExit();
+ Descriptor notLiveDesc = new Descriptor(tmpdir, sstable.descriptor.ksname, sstable.descriptor.cfname, 100);
+ // make sure the new directory is empty and that the old files exist:
+ for (Component c : sstable.components)
+ {
+ File f = new File(notLiveDesc.filenameFor(c));
+ assertFalse(f.exists());
+ assertTrue(new File(sstable.descriptor.filenameFor(c)).exists());
+ }
+ SSTableReader.moveAndOpenSSTable(cfs, sstable.descriptor, notLiveDesc, sstable.components);
+ // make sure the files were moved:
+ for (Component c : sstable.components)
+ {
+ File f = new File(notLiveDesc.filenameFor(c));
+ assertTrue(f.exists());
+ assertTrue(f.toString().contains("-100-"));
+ f.deleteOnExit();
+ assertFalse(new File(sstable.descriptor.filenameFor(c)).exists());
+ }
+ }
+
+
+
+ private SSTableReader getNewSSTable(ColumnFamilyStore cfs)
+ {
+
+ Set<SSTableReader> before = cfs.getLiveSSTables();
+ for (int j = 0; j < 100; j += 2)
+ {
+ new RowUpdateBuilder(cfs.metadata(), j, String.valueOf(j))
+ .clustering("0")
+ .add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER)
+ .build()
+ .applyUnsafe();
+ }
+ cfs.forceBlockingFlush();
+ return Sets.difference(cfs.getLiveSSTables(), before).iterator().next();
+ }
+
+ @Test
+ public void testGetApproximateKeyCount() throws InterruptedException
+ {
+ Keyspace keyspace = Keyspace.open(KEYSPACE1);
+ ColumnFamilyStore cfs = keyspace.getColumnFamilyStore("Standard1");
+ cfs.discardSSTables(System.currentTimeMillis()); //Cleaning all existing SSTables.
+ getNewSSTable(cfs);
+
+ try (ColumnFamilyStore.RefViewFragment viewFragment1 = cfs.selectAndReference(View.selectFunction(SSTableSet.CANONICAL)))
+ {
+ cfs.discardSSTables(System.currentTimeMillis());
+
+ TimeUnit.MILLISECONDS.sleep(1000); //Giving enough time to clear files.
+ List<SSTableReader> sstables = new ArrayList<>(viewFragment1.sstables);
+ assertEquals(50, SSTableReader.getApproximateKeyCount(sstables));
+ }
+ }
}
diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java
index d1b4092..7c47c8b 100644
--- a/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java
@@ -56,10 +56,10 @@
import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
import org.apache.cassandra.metrics.StorageMetrics;
+import org.apache.cassandra.streaming.PreviewKind;
import org.apache.cassandra.streaming.StreamSession;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FBUtilities;
-import org.apache.cassandra.utils.Pair;
import org.apache.cassandra.utils.UUIDGen;
import static org.junit.Assert.*;
@@ -75,7 +75,7 @@
for (int j = 0; j < 100; j ++)
{
- new RowUpdateBuilder(cfs.metadata, j, String.valueOf(j))
+ new RowUpdateBuilder(cfs.metadata(), j, String.valueOf(j))
.clustering("0")
.add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER)
.build()
@@ -171,14 +171,14 @@
{
SSTableReader c = txn.current(sstables.iterator().next());
Collection<Range<Token>> r = Arrays.asList(new Range<>(cfs.getPartitioner().getMinimumToken(), cfs.getPartitioner().getMinimumToken()));
- List<Pair<Long, Long>> tmplinkPositions = sstable.getPositionsForRanges(r);
- List<Pair<Long, Long>> compactingPositions = c.getPositionsForRanges(r);
+ List<SSTableReader.PartitionPositionBounds> tmplinkPositions = sstable.getPositionsForRanges(r);
+ List<SSTableReader.PartitionPositionBounds> compactingPositions = c.getPositionsForRanges(r);
assertEquals(1, tmplinkPositions.size());
assertEquals(1, compactingPositions.size());
- assertEquals(0, tmplinkPositions.get(0).left.longValue());
+ assertEquals(0, tmplinkPositions.get(0).lowerPosition);
// make sure we have no overlap between the early opened file and the compacting one:
- assertEquals(tmplinkPositions.get(0).right.longValue(), compactingPositions.get(0).left.longValue());
- assertEquals(c.uncompressedLength(), compactingPositions.get(0).right.longValue());
+ assertEquals(tmplinkPositions.get(0).upperPosition, compactingPositions.get(0).lowerPosition);
+ assertEquals(c.uncompressedLength(), compactingPositions.get(0).upperPosition);
}
}
}
@@ -692,7 +692,7 @@
String key = Integer.toString(i);
for (int j = 0; j < 10; j++)
- new RowUpdateBuilder(cfs.metadata, 100, key)
+ new RowUpdateBuilder(cfs.metadata(), 100, key)
.clustering(Integer.toString(j))
.add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER)
.build()
@@ -773,65 +773,6 @@
validateCFS(cfs);
}
- @Test
- public void testSSTableSectionsForRanges() throws IOException, InterruptedException, ExecutionException
- {
- Keyspace keyspace = Keyspace.open(KEYSPACE);
- final ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF);
- truncate(cfs);
-
- cfs.addSSTable(writeFile(cfs, 1000));
-
- Collection<SSTableReader> allSSTables = cfs.getLiveSSTables();
- assertEquals(1, allSSTables.size());
- final Token firstToken = allSSTables.iterator().next().first.getToken();
- DatabaseDescriptor.setSSTablePreempiveOpenIntervalInMB(1);
-
- List<StreamSession.SSTableStreamingSections> sectionsBeforeRewrite = StreamSession.getSSTableSectionsForRanges(
- Collections.singleton(new Range<Token>(firstToken, firstToken)),
- Collections.singleton(cfs), 0L, false);
- assertEquals(1, sectionsBeforeRewrite.size());
- for (StreamSession.SSTableStreamingSections section : sectionsBeforeRewrite)
- section.ref.release();
- final AtomicInteger checkCount = new AtomicInteger();
- // needed since we get notified when compaction is done as well - we can't get sections for ranges for obsoleted sstables
- final AtomicBoolean done = new AtomicBoolean(false);
- final AtomicBoolean failed = new AtomicBoolean(false);
- Runnable r = new Runnable()
- {
- public void run()
- {
- while (!done.get())
- {
- Set<Range<Token>> range = Collections.singleton(new Range<Token>(firstToken, firstToken));
- List<StreamSession.SSTableStreamingSections> sections = StreamSession.getSSTableSectionsForRanges(range, Collections.singleton(cfs), 0L, false);
- if (sections.size() != 1)
- failed.set(true);
- for (StreamSession.SSTableStreamingSections section : sections)
- section.ref.release();
- checkCount.incrementAndGet();
- Uninterruptibles.sleepUninterruptibly(5, TimeUnit.MILLISECONDS);
- }
- }
- };
- Thread t = NamedThreadFactory.createThread(r);
- try
- {
- t.start();
- cfs.forceMajorCompaction();
- // reset
- }
- finally
- {
- DatabaseDescriptor.setSSTablePreempiveOpenIntervalInMB(50);
- done.set(true);
- t.join(20);
- }
- assertFalse(failed.get());
- assertTrue(checkCount.get() >= 2);
- truncate(cfs);
- }
-
/**
* emulates anticompaction - writing from one source sstable to two new sstables
*
@@ -934,14 +875,14 @@
for (int f = 0 ; f < fileCount ; f++)
{
File dir = cfs.getDirectories().getDirectoryForNewSSTables();
- String filename = cfs.getSSTablePath(dir);
+ Descriptor desc = cfs.newSSTableDescriptor(dir);
- try (SSTableTxnWriter writer = SSTableTxnWriter.create(cfs, filename, 0, 0, new SerializationHeader(true, cfs.metadata, cfs.metadata.partitionColumns(), EncodingStats.NO_STATS)))
+ try (SSTableTxnWriter writer = SSTableTxnWriter.create(cfs, desc, 0, 0, null, false, new SerializationHeader(true, cfs.metadata(), cfs.metadata().regularAndStaticColumns(), EncodingStats.NO_STATS)))
{
int end = f == fileCount - 1 ? partitionCount : ((f + 1) * partitionCount) / fileCount;
for ( ; i < end ; i++)
{
- UpdateBuilder builder = UpdateBuilder.create(cfs.metadata, ByteBufferUtil.bytes(i));
+ UpdateBuilder builder = UpdateBuilder.create(cfs.metadata(), ByteBufferUtil.bytes(i));
for (int j = 0; j < cellCount ; j++)
builder.newRow(Integer.toString(i)).add("val", random(0, 1000));
diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableScannerTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableScannerTest.java
index d1db09a..eff95fc 100644
--- a/test/unit/org/apache/cassandra/io/sstable/SSTableScannerTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/SSTableScannerTest.java
@@ -24,14 +24,13 @@
import java.util.List;
import com.google.common.collect.Iterables;
-import com.google.common.util.concurrent.RateLimiter;
import org.junit.BeforeClass;
import org.junit.Test;
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.Util;
-import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.DataRange;
import org.apache.cassandra.db.DecoratedKey;
@@ -75,7 +74,7 @@
}
// we produce all DataRange variations that produce an inclusive start and exclusive end range
- private static Iterable<DataRange> dataRanges(CFMetaData metadata, int start, int end)
+ private static Iterable<DataRange> dataRanges(TableMetadata metadata, int start, int end)
{
if (end < start)
return dataRanges(metadata, start, end, false, true);
@@ -86,7 +85,7 @@
);
}
- private static Iterable<DataRange> dataRanges(CFMetaData metadata, int start, int end, boolean inclusiveStart, boolean inclusiveEnd)
+ private static Iterable<DataRange> dataRanges(TableMetadata metadata, int start, int end, boolean inclusiveStart, boolean inclusiveEnd)
{
List<DataRange> ranges = new ArrayList<>();
if (start == end + 1)
@@ -144,7 +143,7 @@
return token(key).maxKeyBound();
}
- private static DataRange dataRange(CFMetaData metadata, PartitionPosition start, boolean startInclusive, PartitionPosition end, boolean endInclusive)
+ private static DataRange dataRange(TableMetadata metadata, PartitionPosition start, boolean startInclusive, PartitionPosition end, boolean endInclusive)
{
Slices.Builder sb = new Slices.Builder(metadata.comparator);
ClusteringIndexSliceFilter filter = new ClusteringIndexSliceFilter(sb.build(), false);
@@ -166,7 +165,7 @@
return ranges;
}
- private static void insertRowWithKey(CFMetaData metadata, int key)
+ private static void insertRowWithKey(TableMetadata metadata, int key)
{
long timestamp = System.currentTimeMillis();
@@ -181,9 +180,11 @@
private static void assertScanMatches(SSTableReader sstable, int scanStart, int scanEnd, int ... boundaries)
{
assert boundaries.length % 2 == 0;
- for (DataRange range : dataRanges(sstable.metadata, scanStart, scanEnd))
+ for (DataRange range : dataRanges(sstable.metadata(), scanStart, scanEnd))
{
- try(ISSTableScanner scanner = sstable.getScanner(ColumnFilter.all(sstable.metadata), range, false, SSTableReadsListener.NOOP_LISTENER))
+ try(ISSTableScanner scanner = sstable.getScanner(ColumnFilter.all(sstable.metadata()),
+ range,
+ SSTableReadsListener.NOOP_LISTENER))
{
for (int b = 0; b < boundaries.length; b += 2)
for (int i = boundaries[b]; i <= boundaries[b + 1]; i++)
@@ -213,14 +214,14 @@
store.disableAutoCompaction();
for (int i = 2; i < 10; i++)
- insertRowWithKey(store.metadata, i);
+ insertRowWithKey(store.metadata(), i);
store.forceBlockingFlush();
assertEquals(1, store.getLiveSSTables().size());
SSTableReader sstable = store.getLiveSSTables().iterator().next();
// full range scan
- ISSTableScanner scanner = sstable.getScanner(RateLimiter.create(Double.MAX_VALUE));
+ ISSTableScanner scanner = sstable.getScanner();
for (int i = 2; i < 10; i++)
assertEquals(toKey(i), new String(scanner.next().partitionKey().getKey().array()));
@@ -319,14 +320,14 @@
for (int i = 0; i < 3; i++)
for (int j = 2; j < 10; j++)
- insertRowWithKey(store.metadata, i * 100 + j);
+ insertRowWithKey(store.metadata(), i * 100 + j);
store.forceBlockingFlush();
assertEquals(1, store.getLiveSSTables().size());
SSTableReader sstable = store.getLiveSSTables().iterator().next();
// full range scan
- ISSTableScanner fullScanner = sstable.getScanner(RateLimiter.create(Double.MAX_VALUE));
+ ISSTableScanner fullScanner = sstable.getScanner();
assertScanContainsRanges(fullScanner,
2, 9,
102, 109,
@@ -336,8 +337,7 @@
// scan all three ranges separately
ISSTableScanner scanner = sstable.getScanner(makeRanges(1, 9,
101, 109,
- 201, 209),
- null);
+ 201, 209));
assertScanContainsRanges(scanner,
2, 9,
102, 109,
@@ -345,16 +345,14 @@
// skip the first range
scanner = sstable.getScanner(makeRanges(101, 109,
- 201, 209),
- null);
+ 201, 209));
assertScanContainsRanges(scanner,
102, 109,
202, 209);
// skip the second range
scanner = sstable.getScanner(makeRanges(1, 9,
- 201, 209),
- null);
+ 201, 209));
assertScanContainsRanges(scanner,
2, 9,
202, 209);
@@ -362,8 +360,7 @@
// skip the last range
scanner = sstable.getScanner(makeRanges(1, 9,
- 101, 109),
- null);
+ 101, 109));
assertScanContainsRanges(scanner,
2, 9,
102, 109);
@@ -371,8 +368,7 @@
// the first scanned range stops short of the actual data in the first range
scanner = sstable.getScanner(makeRanges(1, 5,
101, 109,
- 201, 209),
- null);
+ 201, 209));
assertScanContainsRanges(scanner,
2, 5,
102, 109,
@@ -381,8 +377,7 @@
// the first scanned range requests data beyond actual data in the first range
scanner = sstable.getScanner(makeRanges(1, 20,
101, 109,
- 201, 209),
- null);
+ 201, 209));
assertScanContainsRanges(scanner,
2, 9,
102, 109,
@@ -392,8 +387,7 @@
// the middle scan range splits the outside two data ranges
scanner = sstable.getScanner(makeRanges(1, 5,
6, 205,
- 206, 209),
- null);
+ 206, 209));
assertScanContainsRanges(scanner,
2, 5,
7, 9,
@@ -407,8 +401,7 @@
101, 109,
150, 159,
201, 209,
- 1000, 1001),
- null);
+ 1000, 1001));
assertScanContainsRanges(scanner,
3, 9,
102, 109,
@@ -420,8 +413,7 @@
201, 209,
101, 109,
1000, 1001,
- 150, 159),
- null);
+ 150, 159));
assertScanContainsRanges(scanner,
2, 9,
102, 109,
@@ -430,12 +422,11 @@
// only empty ranges
scanner = sstable.getScanner(makeRanges(0, 1,
150, 159,
- 250, 259),
- null);
+ 250, 259));
assertFalse(scanner.hasNext());
// no ranges is equivalent to a full scan
- scanner = sstable.getScanner(new ArrayList<Range<Token>>(), null);
+ scanner = sstable.getScanner(new ArrayList<Range<Token>>());
assertFalse(scanner.hasNext());
}
@@ -449,19 +440,19 @@
// disable compaction while flushing
store.disableAutoCompaction();
- insertRowWithKey(store.metadata, 205);
+ insertRowWithKey(store.metadata(), 205);
store.forceBlockingFlush();
assertEquals(1, store.getLiveSSTables().size());
SSTableReader sstable = store.getLiveSSTables().iterator().next();
// full range scan
- ISSTableScanner fullScanner = sstable.getScanner(RateLimiter.create(Double.MAX_VALUE));
+ ISSTableScanner fullScanner = sstable.getScanner();
assertScanContainsRanges(fullScanner, 205, 205);
// scan three ranges separately
ISSTableScanner scanner = sstable.getScanner(makeRanges(101, 109,
- 201, 209), null);
+ 201, 209));
// this will currently fail
assertScanContainsRanges(scanner, 205, 205);
diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableUtils.java b/test/unit/org/apache/cassandra/io/sstable/SSTableUtils.java
index df9d1aa..731cee2 100644
--- a/test/unit/org/apache/cassandra/io/sstable/SSTableUtils.java
+++ b/test/unit/org/apache/cassandra/io/sstable/SSTableUtils.java
@@ -23,16 +23,18 @@
import java.io.IOException;
import java.util.*;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.Schema;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.rows.*;
import org.apache.cassandra.db.partitions.*;
import org.apache.cassandra.io.sstable.format.SSTableFormat;
import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.service.ActiveRepairService;
import org.apache.cassandra.Util;
+
+import static org.apache.cassandra.service.ActiveRepairService.*;
import static org.junit.Assert.assertEquals;
public class SSTableUtils
@@ -70,7 +72,7 @@
public static File tempSSTableFile(String keyspaceName, String cfname, int generation) throws IOException
{
- File tempdir = File.createTempFile(keyspaceName, cfname);
+ File tempdir = FileUtils.createTempFile(keyspaceName, cfname);
if(!tempdir.delete() || !tempdir.mkdir())
throw new IOException("Temporary directory creation failed.");
tempdir.deleteOnExit();
@@ -171,7 +173,7 @@
Map<String, PartitionUpdate> map = new HashMap<>();
for (String key : keys)
{
- RowUpdateBuilder builder = new RowUpdateBuilder(Schema.instance.getCFMetaData(ksname, cfname), 0, key);
+ RowUpdateBuilder builder = new RowUpdateBuilder(Schema.instance.getTableMetadata(ksname, cfname), 0, key);
builder.clustering(key).add("val", key);
map.put(key, builder.buildUpdate());
}
@@ -180,7 +182,7 @@
public Collection<SSTableReader> write(SortedMap<DecoratedKey, PartitionUpdate> sorted) throws IOException
{
- PartitionColumns.Builder builder = PartitionColumns.builder();
+ RegularAndStaticColumns.Builder builder = RegularAndStaticColumns.builder();
for (PartitionUpdate update : sorted.values())
builder.addAll(update.columns());
final Iterator<Map.Entry<DecoratedKey, PartitionUpdate>> iter = sorted.entrySet().iterator();
@@ -188,7 +190,7 @@
{
public SerializationHeader header()
{
- return new SerializationHeader(true, Schema.instance.getCFMetaData(ksname, cfname), builder.build(), EncodingStats.NO_STATS);
+ return new SerializationHeader(true, Schema.instance.getTableMetadata(ksname, cfname), builder.build(), EncodingStats.NO_STATS);
}
@Override
@@ -214,10 +216,10 @@
public Collection<SSTableReader> write(int expectedSize, Appender appender) throws IOException
{
File datafile = (dest == null) ? tempSSTableFile(ksname, cfname, generation) : new File(dest.filenameFor(Component.DATA));
- CFMetaData cfm = Schema.instance.getCFMetaData(ksname, cfname);
- ColumnFamilyStore cfs = Schema.instance.getColumnFamilyStoreInstance(cfm.cfId);
+ TableMetadata metadata = Schema.instance.getTableMetadata(ksname, cfname);
+ ColumnFamilyStore cfs = Schema.instance.getColumnFamilyStoreInstance(metadata.id);
SerializationHeader header = appender.header();
- SSTableTxnWriter writer = SSTableTxnWriter.create(cfs, datafile.getAbsolutePath(), expectedSize, ActiveRepairService.UNREPAIRED_SSTABLE, 0, header);
+ SSTableTxnWriter writer = SSTableTxnWriter.create(cfs, Descriptor.fromFilename(datafile.getAbsolutePath()), expectedSize, UNREPAIRED_SSTABLE, NO_PENDING_REPAIR, false, 0, header);
while (appender.append(writer)) { /* pass */ }
Collection<SSTableReader> readers = writer.finish(true);
diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTest.java
index 391927c..31d0b89 100644
--- a/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTest.java
@@ -20,6 +20,7 @@
import java.io.File;
import java.nio.ByteBuffer;
+import java.util.UUID;
import org.junit.Test;
@@ -32,9 +33,12 @@
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.sstable.format.SSTableReadsListener;
import org.apache.cassandra.io.sstable.format.SSTableWriter;
+import org.apache.cassandra.service.ActiveRepairService;
import org.apache.cassandra.utils.FBUtilities;
import static junit.framework.Assert.fail;
+import static org.apache.cassandra.service.ActiveRepairService.NO_PENDING_REPAIR;
+import static org.apache.cassandra.service.ActiveRepairService.UNREPAIRED_SSTABLE;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@@ -53,7 +57,7 @@
{
for (int i = 0; i < 10000; i++)
{
- UpdateBuilder builder = UpdateBuilder.create(cfs.metadata, random(i, 10)).withTimestamp(1);
+ UpdateBuilder builder = UpdateBuilder.create(cfs.metadata(), random(i, 10)).withTimestamp(1);
for (int j = 0; j < 100; j++)
builder.newRow("" + j).add("val", ByteBuffer.allocate(1000));
writer.append(builder.build().unfilteredIterator());
@@ -64,7 +68,7 @@
assertFileCounts(dir.list());
for (int i = 10000; i < 20000; i++)
{
- UpdateBuilder builder = UpdateBuilder.create(cfs.metadata, random(i, 10)).withTimestamp(1);
+ UpdateBuilder builder = UpdateBuilder.create(cfs.metadata(), random(i, 10)).withTimestamp(1);
for (int j = 0; j < 100; j++)
builder.newRow("" + j).add("val", ByteBuffer.allocate(1000));
writer.append(builder.build().unfilteredIterator());
@@ -108,7 +112,7 @@
{
for (int i = 0; i < 10000; i++)
{
- UpdateBuilder builder = UpdateBuilder.create(cfs.metadata, random(i, 10)).withTimestamp(1);
+ UpdateBuilder builder = UpdateBuilder.create(cfs.metadata(), random(i, 10)).withTimestamp(1);
for (int j = 0; j < 100; j++)
builder.newRow("" + j).add("val", ByteBuffer.allocate(1000));
writer.append(builder.build().unfilteredIterator());
@@ -117,7 +121,7 @@
assertFileCounts(dir.list());
for (int i = 10000; i < 20000; i++)
{
- UpdateBuilder builder = UpdateBuilder.create(cfs.metadata, random(i, 10)).withTimestamp(1);
+ UpdateBuilder builder = UpdateBuilder.create(cfs.metadata(), random(i, 10)).withTimestamp(1);
for (int j = 0; j < 100; j++)
builder.newRow("" + j).add("val", ByteBuffer.allocate(1000));
writer.append(builder.build().unfilteredIterator());
@@ -159,7 +163,7 @@
{
for (int i = 0; i < 10000; i++)
{
- UpdateBuilder builder = UpdateBuilder.create(cfs.metadata, random(i, 10)).withTimestamp(1);
+ UpdateBuilder builder = UpdateBuilder.create(cfs.metadata(), random(i, 10)).withTimestamp(1);
for (int j = 0; j < 100; j++)
builder.newRow("" + j).add("val", ByteBuffer.allocate(1000));
writer1.append(builder.build().unfilteredIterator());
@@ -168,7 +172,7 @@
assertFileCounts(dir.list());
for (int i = 10000; i < 20000; i++)
{
- UpdateBuilder builder = UpdateBuilder.create(cfs.metadata, random(i, 10)).withTimestamp(1);
+ UpdateBuilder builder = UpdateBuilder.create(cfs.metadata(), random(i, 10)).withTimestamp(1);
for (int j = 0; j < 100; j++)
builder.newRow("" + j).add("val", ByteBuffer.allocate(1000));
writer2.append(builder.build().unfilteredIterator());
@@ -213,7 +217,7 @@
try (SSTableWriter writer1 = getWriter(cfs, dir, txn))
{
- UpdateBuilder largeValue = UpdateBuilder.create(cfs.metadata, "large_value").withTimestamp(1);
+ UpdateBuilder largeValue = UpdateBuilder.create(cfs.metadata(), "large_value").withTimestamp(1);
largeValue.newRow("clustering").add("val", ByteBuffer.allocate(2 * 1024 * 1024));
writer1.append(largeValue.build().unfilteredIterator());
@@ -226,8 +230,7 @@
DecoratedKey dk = Util.dk("large_value");
UnfilteredRowIterator rowIter = sstable.iterator(dk,
Slices.ALL,
- ColumnFilter.all(cfs.metadata),
- false,
+ ColumnFilter.all(cfs.metadata()),
false,
SSTableReadsListener.NOOP_LISTENER);
while (rowIter.hasNext())
@@ -246,4 +249,60 @@
}
}
+ private static void assertValidRepairMetadata(long repairedAt, UUID pendingRepair, boolean isTransient)
+ {
+ Keyspace keyspace = Keyspace.open(KEYSPACE);
+ ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF_SMALL_MAX_VALUE);
+ File dir = cfs.getDirectories().getDirectoryForNewSSTables();
+ LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.STREAM);
+
+ try (SSTableWriter writer = getWriter(cfs, dir, txn, repairedAt, pendingRepair, isTransient))
+ {
+ // expected
+ }
+ catch (IllegalArgumentException e)
+ {
+ throw new AssertionError("Unexpected IllegalArgumentException", e);
+ }
+
+ txn.abort();
+ LifecycleTransaction.waitForDeletions();
+ }
+
+ private static void assertInvalidRepairMetadata(long repairedAt, UUID pendingRepair, boolean isTransient)
+ {
+ Keyspace keyspace = Keyspace.open(KEYSPACE);
+ ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF_SMALL_MAX_VALUE);
+ File dir = cfs.getDirectories().getDirectoryForNewSSTables();
+ LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.STREAM);
+
+ try (SSTableWriter writer = getWriter(cfs, dir, txn, repairedAt, pendingRepair, isTransient))
+ {
+ fail("Expected IllegalArgumentException");
+ }
+ catch (IllegalArgumentException e)
+ {
+ // expected
+ }
+
+ txn.abort();
+ LifecycleTransaction.waitForDeletions();
+ }
+
+ /**
+ * It should only be possible to create sstables marked transient that also have a pending repair
+ */
+ @Test
+ public void testRepairMetadataValidation()
+ {
+ assertValidRepairMetadata(UNREPAIRED_SSTABLE, NO_PENDING_REPAIR, false);
+ assertValidRepairMetadata(1, NO_PENDING_REPAIR, false);
+ assertValidRepairMetadata(UNREPAIRED_SSTABLE, UUID.randomUUID(), false);
+ assertValidRepairMetadata(UNREPAIRED_SSTABLE, UUID.randomUUID(), true);
+
+ assertInvalidRepairMetadata(UNREPAIRED_SSTABLE, NO_PENDING_REPAIR, true);
+ assertInvalidRepairMetadata(1, UUID.randomUUID(), false);
+ assertInvalidRepairMetadata(1, NO_PENDING_REPAIR, true);
+
+ }
}
diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTestBase.java b/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTestBase.java
index a123a22..962e1a1 100644
--- a/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTestBase.java
+++ b/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTestBase.java
@@ -22,6 +22,7 @@
import java.nio.ByteBuffer;
import java.util.HashSet;
import java.util.Set;
+import java.util.UUID;
import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.TimeUnit;
@@ -48,6 +49,7 @@
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
public class SSTableWriterTestBase extends SchemaLoader
{
@@ -161,10 +163,15 @@
assertFalse(CompactionManager.instance.submitMaximal(cfs, cfs.gcBefore((int) (System.currentTimeMillis() / 1000)), false).isEmpty());
}
+ public static SSTableWriter getWriter(ColumnFamilyStore cfs, File directory, LifecycleTransaction txn, long repairedAt, UUID pendingRepair, boolean isTransient)
+ {
+ Descriptor desc = cfs.newSSTableDescriptor(directory);
+ return SSTableWriter.create(desc, 0, repairedAt, pendingRepair, isTransient, new SerializationHeader(true, cfs.metadata(), cfs.metadata().regularAndStaticColumns(), EncodingStats.NO_STATS), cfs.indexManager.listIndexes(), txn);
+ }
+
public static SSTableWriter getWriter(ColumnFamilyStore cfs, File directory, LifecycleTransaction txn)
{
- String filename = cfs.getSSTablePath(directory);
- return SSTableWriter.create(filename, 0, 0, new SerializationHeader(true, cfs.metadata, cfs.metadata.partitionColumns(), EncodingStats.NO_STATS), cfs.indexManager.listIndexes(), txn);
+ return getWriter(cfs, directory, txn, 0, null, false);
}
public static ByteBuffer random(int i, int size)
diff --git a/test/unit/org/apache/cassandra/io/sstable/format/ClientModeSSTableTest.java b/test/unit/org/apache/cassandra/io/sstable/format/ClientModeSSTableTest.java
deleted file mode 100644
index 90522d6..0000000
--- a/test/unit/org/apache/cassandra/io/sstable/format/ClientModeSSTableTest.java
+++ /dev/null
@@ -1,139 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.io.sstable.format;
-
-import static org.apache.cassandra.utils.ByteBufferUtil.bytes;
-
-import java.io.File;
-import java.nio.ByteBuffer;
-
-import com.google.common.util.concurrent.Runnables;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import org.apache.cassandra.concurrent.ScheduledExecutors;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.db.Slices;
-import org.apache.cassandra.db.filter.ColumnFilter;
-import org.apache.cassandra.db.marshal.BytesType;
-import org.apache.cassandra.db.rows.UnfilteredRowIterator;
-import org.apache.cassandra.dht.ByteOrderedPartitioner;
-import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.io.sstable.Descriptor;
-
-/**
- * Tests backwards compatibility for SSTables
- */
-public class ClientModeSSTableTest
-{
- public static final String LEGACY_SSTABLE_PROP = "legacy-sstable-root";
- public static final String KSNAME = "Keyspace1";
- public static final String CFNAME = "Standard1";
-
- public static File LEGACY_SSTABLE_ROOT;
-
- static CFMetaData metadata;
-
- @BeforeClass
- public static void defineSchema() throws ConfigurationException
- {
- DatabaseDescriptor.clientInitialization();
- DatabaseDescriptor.applyAddressConfig();
-
- metadata = CFMetaData.Builder.createDense(KSNAME, CFNAME, false, false)
- .addPartitionKey("key", BytesType.instance)
- .addClusteringColumn("column", BytesType.instance)
- .addRegularColumn("value", BytesType.instance)
- .withPartitioner(ByteOrderedPartitioner.instance)
- .build();
-
- String scp = System.getProperty(LEGACY_SSTABLE_PROP);
- assert scp != null;
- LEGACY_SSTABLE_ROOT = new File(scp).getAbsoluteFile();
- assert LEGACY_SSTABLE_ROOT.isDirectory();
- }
-
- /**
- * Get a descriptor for the legacy sstable at the given version.
- */
- protected Descriptor getDescriptor(String ver)
- {
- File directory = new File(LEGACY_SSTABLE_ROOT + File.separator + ver + File.separator + KSNAME);
- return new Descriptor(ver, directory, KSNAME, CFNAME, 0, SSTableFormat.Type.LEGACY);
- }
-
- @Test
- public void testVersions() throws Throwable
- {
- boolean notSkipped = false;
-
- for (File version : LEGACY_SSTABLE_ROOT.listFiles())
- {
- if (!new File(LEGACY_SSTABLE_ROOT + File.separator + version.getName() + File.separator + KSNAME).isDirectory())
- continue;
- if (Version.validate(version.getName()) && SSTableFormat.Type.LEGACY.info.getVersion(version.getName()).isCompatible())
- {
- notSkipped = true;
- testVersion(version.getName());
- }
- }
-
- assert notSkipped;
- }
-
- public void testVersion(String version) throws Throwable
- {
- SSTableReader reader = null;
- try
- {
- reader = SSTableReader.openNoValidation(getDescriptor(version), metadata);
-
- ByteBuffer key = bytes(Integer.toString(100));
-
- try (UnfilteredRowIterator iter = reader.iterator(metadata.decorateKey(key),
- Slices.ALL,
- ColumnFilter.selection(metadata.partitionColumns()),
- false,
- false,
- SSTableReadsListener.NOOP_LISTENER))
- {
- assert iter.next().clustering().get(0).equals(key);
- }
- }
- catch (Throwable e)
- {
- System.err.println("Failed to read " + version);
- throw e;
- }
- finally
- {
- if (reader != null)
- {
- int globalTidyCount = SSTableReader.GlobalTidy.lookup.size();
- reader.selfRef().release();
- assert reader.selfRef().globalCount() == 0;
-
- // await clean-up to complete if started.
- ScheduledExecutors.nonPeriodicTasks.submit(Runnables.doNothing()).get();
- // Ensure clean-up completed.
- assert SSTableReader.GlobalTidy.lookup.size() < globalTidyCount;
- }
- }
- }
-}
diff --git a/test/unit/org/apache/cassandra/io/sstable/format/SSTableFlushObserverTest.java b/test/unit/org/apache/cassandra/io/sstable/format/SSTableFlushObserverTest.java
index f4c2f46..2b787ca 100644
--- a/test/unit/org/apache/cassandra/io/sstable/format/SSTableFlushObserverTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/format/SSTableFlushObserverTest.java
@@ -25,8 +25,9 @@
import java.util.Collections;
import java.util.Iterator;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.db.commitlog.CommitLog;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.Clustering;
import org.apache.cassandra.db.DecoratedKey;
@@ -45,13 +46,14 @@
import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
import org.apache.cassandra.io.util.FileDataInput;
import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.schema.TableMetadataRef;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.Pair;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.Multimap;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -62,6 +64,7 @@
public static void initDD()
{
DatabaseDescriptor.daemonInitialization();
+ CommitLog.instance.start();
}
private static final String KS_NAME = "test";
@@ -70,12 +73,13 @@
@Test
public void testFlushObserver()
{
- CFMetaData cfm = CFMetaData.Builder.create(KS_NAME, CF_NAME)
- .addPartitionKey("id", UTF8Type.instance)
- .addRegularColumn("first_name", UTF8Type.instance)
- .addRegularColumn("age", Int32Type.instance)
- .addRegularColumn("height", LongType.instance)
- .build();
+ TableMetadata cfm =
+ TableMetadata.builder(KS_NAME, CF_NAME)
+ .addPartitionKeyColumn("id", UTF8Type.instance)
+ .addRegularColumn("first_name", UTF8Type.instance)
+ .addRegularColumn("age", Int32Type.instance)
+ .addRegularColumn("height", LongType.instance)
+ .build();
LifecycleTransaction transaction = LifecycleTransaction.offline(OperationType.COMPACTION);
FlushObserver observer = new FlushObserver();
@@ -89,14 +93,14 @@
SSTableFormat.Type sstableFormat = SSTableFormat.Type.current();
- BigTableWriter writer = new BigTableWriter(new Descriptor(sstableFormat.info.getLatestVersion().version,
+ BigTableWriter writer = new BigTableWriter(new Descriptor(sstableFormat.info.getLatestVersion(),
directory,
KS_NAME, CF_NAME,
0,
sstableFormat),
- 10L, 0L, cfm,
+ 10L, 0L, null, false, TableMetadataRef.forOfflineTools(cfm),
new MetadataCollector(cfm.comparator).sstableLevel(0),
- new SerializationHeader(true, cfm, cfm.partitionColumns(), EncodingStats.NO_STATS),
+ new SerializationHeader(true, cfm, cfm.regularAndStaticColumns(), EncodingStats.NO_STATS),
Collections.singletonList(observer),
transaction);
@@ -161,12 +165,12 @@
{
private final Iterator<Unfiltered> rows;
- public RowIterator(CFMetaData cfm, ByteBuffer key, Collection<Unfiltered> content)
+ public RowIterator(TableMetadata cfm, ByteBuffer key, Collection<Unfiltered> content)
{
super(cfm,
DatabaseDescriptor.getPartitioner().decorateKey(key),
DeletionTime.LIVE,
- cfm.partitionColumns(),
+ cfm.regularAndStaticColumns(),
BTreeRow.emptyRow(Clustering.STATIC_CLUSTERING),
false,
EncodingStats.NO_STATS);
@@ -219,8 +223,8 @@
return rowBuilder.build();
}
- private static ColumnDefinition getColumn(CFMetaData cfm, String name)
+ private static ColumnMetadata getColumn(TableMetadata cfm, String name)
{
- return cfm.getColumnDefinition(UTF8Type.instance.fromString(name));
+ return cfm.getColumn(UTF8Type.instance.fromString(name));
}
}
diff --git a/test/unit/org/apache/cassandra/io/sstable/format/big/BigTableZeroCopyWriterTest.java b/test/unit/org/apache/cassandra/io/sstable/format/big/BigTableZeroCopyWriterTest.java
new file mode 100644
index 0000000..3cf96f2
--- /dev/null
+++ b/test/unit/org/apache/cassandra/io/sstable/format/big/BigTableZeroCopyWriterTest.java
@@ -0,0 +1,212 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.sstable.format.big;
+
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.nio.ByteBuffer;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.Collection;
+import java.util.Set;
+import java.util.function.Function;
+
+import com.google.common.collect.ImmutableSet;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import io.netty.buffer.Unpooled;
+import io.netty.channel.embedded.EmbeddedChannel;
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.Util;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.RowUpdateBuilder;
+import org.apache.cassandra.db.Slices;
+import org.apache.cassandra.db.compaction.OperationType;
+import org.apache.cassandra.db.filter.ColumnFilter;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.io.sstable.Component;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.format.SSTableReadsListener;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.FileHandle;
+import org.apache.cassandra.net.AsyncStreamingInputPlus;
+import org.apache.cassandra.schema.CachingParams;
+import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.TableMetadataRef;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.Pair;
+
+import static org.apache.cassandra.io.util.DataInputPlus.DataInputStreamPlus;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
+
+public class BigTableZeroCopyWriterTest
+{
+ public static final String KEYSPACE1 = "BigTableBlockWriterTest";
+ public static final String CF_STANDARD = "Standard1";
+ public static final String CF_STANDARD2 = "Standard2";
+ public static final String CF_INDEXED = "Indexed1";
+ public static final String CF_STANDARDLOWINDEXINTERVAL = "StandardLowIndexInterval";
+
+ public static SSTableReader sstable;
+ public static ColumnFamilyStore store;
+ private static int expectedRowCount;
+
+ @BeforeClass
+ public static void defineSchema() throws Exception
+ {
+ SchemaLoader.prepareServer();
+ SchemaLoader.createKeyspace(KEYSPACE1,
+ KeyspaceParams.simple(1),
+ SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD),
+ SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD2),
+ SchemaLoader.compositeIndexCFMD(KEYSPACE1, CF_INDEXED, true),
+ SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARDLOWINDEXINTERVAL)
+ .minIndexInterval(8)
+ .maxIndexInterval(256)
+ .caching(CachingParams.CACHE_NOTHING));
+
+ String ks = KEYSPACE1;
+ String cf = "Standard1";
+
+ // clear and create just one sstable for this test
+ Keyspace keyspace = Keyspace.open(ks);
+ store = keyspace.getColumnFamilyStore(cf);
+ store.clearUnsafe();
+ store.disableAutoCompaction();
+
+ DecoratedKey firstKey = null, lastKey = null;
+ long timestamp = System.currentTimeMillis();
+ for (int i = 0; i < store.metadata().params.minIndexInterval; i++)
+ {
+ DecoratedKey key = Util.dk(String.valueOf(i));
+ if (firstKey == null)
+ firstKey = key;
+ if (lastKey == null)
+ lastKey = key;
+ if (store.metadata().partitionKeyType.compare(lastKey.getKey(), key.getKey()) < 0)
+ lastKey = key;
+
+ new RowUpdateBuilder(store.metadata(), timestamp, key.getKey())
+ .clustering("col")
+ .add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER)
+ .build()
+ .applyUnsafe();
+ expectedRowCount++;
+ }
+ store.forceBlockingFlush();
+
+ sstable = store.getLiveSSTables().iterator().next();
+ }
+
+ @Test
+ public void writeDataFile_DataInputPlus()
+ {
+ writeDataTestCycle(buffer -> new DataInputStreamPlus(new ByteArrayInputStream(buffer.array())));
+ }
+
+ @Test
+ public void writeDataFile_RebufferingByteBufDataInputPlus()
+ {
+ try (AsyncStreamingInputPlus input = new AsyncStreamingInputPlus(new EmbeddedChannel()))
+ {
+ writeDataTestCycle(buffer ->
+ {
+ input.append(Unpooled.wrappedBuffer(buffer));
+ return input;
+ });
+
+ input.requestClosure();
+ }
+ }
+
+
+ private void writeDataTestCycle(Function<ByteBuffer, DataInputPlus> bufferMapper)
+ {
+ File dir = store.getDirectories().getDirectoryForNewSSTables();
+ Descriptor desc = store.newSSTableDescriptor(dir);
+ TableMetadataRef metadata = Schema.instance.getTableMetadataRef(desc);
+
+ LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.STREAM);
+ Set<Component> componentsToWrite = ImmutableSet.of(Component.DATA, Component.PRIMARY_INDEX,
+ Component.STATS);
+
+ BigTableZeroCopyWriter btzcw = new BigTableZeroCopyWriter(desc, metadata, txn, componentsToWrite);
+
+ for (Component component : componentsToWrite)
+ {
+ if (Files.exists(Paths.get(desc.filenameFor(component))))
+ {
+ Pair<DataInputPlus, Long> pair = getSSTableComponentData(sstable, component, bufferMapper);
+
+ btzcw.writeComponent(component.type, pair.left, pair.right);
+ }
+ }
+
+ Collection<SSTableReader> readers = btzcw.finish(true);
+
+ SSTableReader reader = readers.toArray(new SSTableReader[0])[0];
+
+ assertNotEquals(sstable.getFilename(), reader.getFilename());
+ assertEquals(sstable.estimatedKeys(), reader.estimatedKeys());
+ assertEquals(sstable.isPendingRepair(), reader.isPendingRepair());
+
+ assertRowCount(expectedRowCount);
+ }
+
+ private void assertRowCount(int expected)
+ {
+ int count = 0;
+ for (int i = 0; i < store.metadata().params.minIndexInterval; i++)
+ {
+ DecoratedKey dk = Util.dk(String.valueOf(i));
+ UnfilteredRowIterator rowIter = sstable.iterator(dk,
+ Slices.ALL,
+ ColumnFilter.all(store.metadata()),
+ false,
+ SSTableReadsListener.NOOP_LISTENER);
+ while (rowIter.hasNext())
+ {
+ rowIter.next();
+ count++;
+ }
+ }
+ assertEquals(expected, count);
+ }
+
+ private Pair<DataInputPlus, Long> getSSTableComponentData(SSTableReader sstable, Component component,
+ Function<ByteBuffer, DataInputPlus> bufferMapper)
+ {
+ FileHandle componentFile = new FileHandle.Builder(sstable.descriptor.filenameFor(component))
+ .bufferSize(1024).complete();
+ ByteBuffer buffer = ByteBuffer.allocate((int) componentFile.channel.size());
+ componentFile.channel.read(buffer, 0);
+ buffer.flip();
+
+ DataInputPlus inputPlus = bufferMapper.apply(buffer);
+
+ return Pair.create(inputPlus, componentFile.channel.size());
+ }
+}
diff --git a/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java b/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java
index 9df3e11..f109d8f 100644
--- a/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java
@@ -15,7 +15,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.cassandra.io.sstable.metadata;
import java.io.File;
@@ -29,7 +28,8 @@
import org.junit.Test;
import org.apache.cassandra.SchemaLoader;
-import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.SerializationHeader;
import org.apache.cassandra.db.commitlog.CommitLogPosition;
@@ -45,6 +45,8 @@
import org.apache.cassandra.io.util.RandomAccessReader;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
public class MetadataSerializerTest
{
@@ -78,7 +80,7 @@
throws IOException
{
// Serialize to tmp file
- File statsFile = File.createTempFile(Component.STATS.name, null);
+ File statsFile = FileUtils.createTempFile(Component.STATS.name, null);
try (DataOutputStreamPlus out = new BufferedDataOutputStreamPlus(new FileOutputStream(statsFile)))
{
serializer.serialize(metadata, out, version);
@@ -91,20 +93,19 @@
CommitLogPosition club = new CommitLogPosition(11L, 12);
CommitLogPosition cllb = new CommitLogPosition(9L, 12);
- CFMetaData cfm = SchemaLoader.standardCFMD("ks1", "cf1");
+ TableMetadata cfm = SchemaLoader.standardCFMD("ks1", "cf1").build();
MetadataCollector collector = new MetadataCollector(cfm.comparator)
.commitLogIntervals(new IntervalSet<>(cllb, club));
String partitioner = RandomPartitioner.class.getCanonicalName();
double bfFpChance = 0.1;
- Map<MetadataType, MetadataComponent> originalMetadata = collector.finalizeMetadata(partitioner, bfFpChance, 0, SerializationHeader.make(cfm, Collections.emptyList()));
- return originalMetadata;
+ return collector.finalizeMetadata(partitioner, bfFpChance, 0, null, false, SerializationHeader.make(cfm, Collections.emptyList()));
}
@Test
- public void testLaReadLb() throws IOException
+ public void testMaReadMa() throws IOException
{
- testOldReadsNew("la", "lb");
+ testOldReadsNew("ma", "ma");
}
@Test
@@ -120,11 +121,29 @@
}
@Test
+ public void testMbReadMb() throws IOException
+ {
+ testOldReadsNew("mb", "mb");
+ }
+
+ @Test
public void testMbReadMc() throws IOException
{
testOldReadsNew("mb", "mc");
}
+ @Test
+ public void testMcReadMc() throws IOException
+ {
+ testOldReadsNew("mc", "mc");
+ }
+
+ @Test
+ public void testNaReadNa() throws IOException
+ {
+ testOldReadsNew("na", "na");
+ }
+
public void testOldReadsNew(String oldV, String newV) throws IOException
{
Map<MetadataType, MetadataComponent> originalMetadata = constructMetadata();
@@ -134,7 +153,8 @@
File statsFileLb = serialize(originalMetadata, serializer, BigFormat.instance.getVersion(newV));
File statsFileLa = serialize(originalMetadata, serializer, BigFormat.instance.getVersion(oldV));
// Reading both as earlier version should yield identical results.
- Descriptor desc = new Descriptor(oldV, statsFileLb.getParentFile(), "", "", 0, SSTableFormat.Type.current());
+ SSTableFormat.Type stype = SSTableFormat.Type.current();
+ Descriptor desc = new Descriptor(stype.info.getVersion(oldV), statsFileLb.getParentFile(), "", "", 0, stype);
try (RandomAccessReader inLb = RandomAccessReader.open(statsFileLb);
RandomAccessReader inLa = RandomAccessReader.open(statsFileLa))
{
@@ -144,12 +164,19 @@
for (MetadataType type : MetadataType.values())
{
assertEquals(deserializedLa.get(type), deserializedLb.get(type));
- if (!originalMetadata.get(type).equals(deserializedLb.get(type)))
- {
- // Currently only STATS can be different. Change if no longer the case
- assertEquals(MetadataType.STATS, type);
- }
+
+ if (MetadataType.STATS != type)
+ assertEquals(originalMetadata.get(type), deserializedLb.get(type));
}
}
}
+
+ @Test
+ public void pendingRepairCompatibility()
+ {
+ Version mc = BigFormat.instance.getVersion("mc");
+ assertFalse(mc.hasPendingRepair());
+ Version na = BigFormat.instance.getVersion("na");
+ assertTrue(na.hasPendingRepair());
+ }
}
diff --git a/test/unit/org/apache/cassandra/io/util/BufferedDataOutputStreamTest.java b/test/unit/org/apache/cassandra/io/util/BufferedDataOutputStreamTest.java
index 7ca2273..c5c3b60 100644
--- a/test/unit/org/apache/cassandra/io/util/BufferedDataOutputStreamTest.java
+++ b/test/unit/org/apache/cassandra/io/util/BufferedDataOutputStreamTest.java
@@ -44,10 +44,15 @@
import com.google.common.primitives.UnsignedInteger;
import com.google.common.primitives.UnsignedLong;
+import static org.apache.cassandra.utils.FBUtilities.preventIllegalAccessWarnings;
import static org.junit.Assert.*;
public class BufferedDataOutputStreamTest
{
+ static
+ {
+ preventIllegalAccessWarnings();
+ }
@Test(expected = BufferOverflowException.class)
public void testDataOutputBufferFixedByes() throws Exception
@@ -610,48 +615,4 @@
}
}
- @Test
- public void testWriteExcessSlow() throws Exception
- {
- try (DataOutputBuffer dob = new DataOutputBuffer(4))
- {
- dob.strictFlushing = true;
- ByteBuffer buf = ByteBuffer.allocateDirect(8);
- buf.putLong(0, 42);
- dob.write(buf);
- assertEquals(42, ByteBuffer.wrap(dob.toByteArray()).getLong());
- }
- }
-
- @Test
- public void testApplyToChannel() throws Exception
- {
- setUp();
- Object obj = new Object();
- Object retval = ndosp.applyToChannel( channel -> {
- ByteBuffer buf = ByteBuffer.allocate(8);
- buf.putLong(0, 42);
- try
- {
- channel.write(buf);
- }
- catch (Exception e)
- {
- throw new RuntimeException(e);
- }
- return obj;
- });
- assertEquals(obj, retval);
- assertEquals(42, ByteBuffer.wrap(generated.toByteArray()).getLong());
- }
-
- @Test(expected = UnsupportedOperationException.class)
- public void testApplyToChannelThrowsForMisaligned() throws Exception
- {
- setUp();
- ndosp.strictFlushing = true;
- ndosp.applyToChannel( channel -> {
- return null;
- });
- }
}
diff --git a/test/unit/org/apache/cassandra/io/util/BufferedRandomAccessFileTest.java b/test/unit/org/apache/cassandra/io/util/BufferedRandomAccessFileTest.java
index 2386160..764190c 100644
--- a/test/unit/org/apache/cassandra/io/util/BufferedRandomAccessFileTest.java
+++ b/test/unit/org/apache/cassandra/io/util/BufferedRandomAccessFileTest.java
@@ -143,7 +143,7 @@
@Test
public void testReadAndWriteOnCapacity() throws IOException
{
- File tmpFile = File.createTempFile("readtest", "bin");
+ File tmpFile = FileUtils.createTempFile("readtest", "bin");
try (SequentialWriter w = new SequentialWriter(tmpFile))
{
// Fully write the file and sync..
@@ -170,7 +170,7 @@
@Test
public void testLength() throws IOException
{
- File tmpFile = File.createTempFile("lengthtest", "bin");
+ File tmpFile = FileUtils.createTempFile("lengthtest", "bin");
try (SequentialWriter w = new SequentialWriter(tmpFile))
{
assertEquals(0, w.length());
@@ -423,7 +423,7 @@
@Test
public void testBytesPastMark() throws IOException
{
- File tmpFile = File.createTempFile("overflowtest", "bin");
+ File tmpFile = FileUtils.createTempFile("overflowtest", "bin");
tmpFile.deleteOnExit();
// Create the BRAF by filename instead of by file.
@@ -588,7 +588,7 @@
@Test (expected=IllegalArgumentException.class)
public void testSetNegativeLength() throws IOException, IllegalArgumentException
{
- File tmpFile = File.createTempFile("set_negative_length", "bin");
+ File tmpFile = FileUtils.createTempFile("set_negative_length", "bin");
try (SequentialWriter file = new SequentialWriter(tmpFile))
{
file.truncate(-8L);
@@ -597,7 +597,7 @@
private SequentialWriter createTempFile(String name) throws IOException
{
- File tempFile = File.createTempFile(name, null);
+ File tempFile = FileUtils.createTempFile(name, null);
tempFile.deleteOnExit();
return new SequentialWriter(tempFile);
@@ -605,7 +605,7 @@
private File writeTemporaryFile(byte[] data) throws IOException
{
- File f = File.createTempFile("BRAFTestFile", null);
+ File f = FileUtils.createTempFile("BRAFTestFile", null);
f.deleteOnExit();
FileOutputStream fout = new FileOutputStream(f);
fout.write(data);
diff --git a/test/unit/org/apache/cassandra/io/util/ChecksummedRandomAccessReaderTest.java b/test/unit/org/apache/cassandra/io/util/ChecksummedRandomAccessReaderTest.java
index 545c3e3..4963712 100644
--- a/test/unit/org/apache/cassandra/io/util/ChecksummedRandomAccessReaderTest.java
+++ b/test/unit/org/apache/cassandra/io/util/ChecksummedRandomAccessReaderTest.java
@@ -42,8 +42,8 @@
@Test
public void readFully() throws IOException
{
- final File data = File.createTempFile("testReadFully", "data");
- final File crc = File.createTempFile("testReadFully", "crc");
+ final File data = FileUtils.createTempFile("testReadFully", "data");
+ final File crc = FileUtils.createTempFile("testReadFully", "crc");
final byte[] expected = new byte[70 * 1024]; // bit more than crc chunk size, so we can test rebuffering.
ThreadLocalRandom.current().nextBytes(expected);
@@ -70,8 +70,8 @@
@Test
public void seek() throws IOException
{
- final File data = File.createTempFile("testSeek", "data");
- final File crc = File.createTempFile("testSeek", "crc");
+ final File data = FileUtils.createTempFile("testSeek", "data");
+ final File crc = FileUtils.createTempFile("testSeek", "crc");
final byte[] dataBytes = new byte[70 * 1024]; // bit more than crc chunk size
ThreadLocalRandom.current().nextBytes(dataBytes);
@@ -104,8 +104,8 @@
@Test(expected = CorruptFileException.class)
public void corruptionDetection() throws IOException
{
- final File data = File.createTempFile("corruptionDetection", "data");
- final File crc = File.createTempFile("corruptionDetection", "crc");
+ final File data = FileUtils.createTempFile("corruptionDetection", "data");
+ final File crc = FileUtils.createTempFile("corruptionDetection", "crc");
final byte[] expected = new byte[5 * 1024];
Arrays.fill(expected, (byte) 0);
diff --git a/test/unit/org/apache/cassandra/io/util/ChecksummedSequentialWriterTest.java b/test/unit/org/apache/cassandra/io/util/ChecksummedSequentialWriterTest.java
index 29d4eea..6837d1d 100644
--- a/test/unit/org/apache/cassandra/io/util/ChecksummedSequentialWriterTest.java
+++ b/test/unit/org/apache/cassandra/io/util/ChecksummedSequentialWriterTest.java
@@ -26,7 +26,7 @@
import org.junit.After;
import org.junit.BeforeClass;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.cassandra.config.DatabaseDescriptor;
public class ChecksummedSequentialWriterTest extends SequentialWriterTest
diff --git a/test/unit/org/apache/cassandra/io/util/FileUtilsTest.java b/test/unit/org/apache/cassandra/io/util/FileUtilsTest.java
index 2f9ccd4..373232d 100644
--- a/test/unit/org/apache/cassandra/io/util/FileUtilsTest.java
+++ b/test/unit/org/apache/cassandra/io/util/FileUtilsTest.java
@@ -21,7 +21,7 @@
import java.io.File;
import java.io.IOException;
import java.io.RandomAccessFile;
-import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
@@ -46,20 +46,49 @@
}
@Test
+ public void testParseFileSize() throws Exception
+ {
+ // test straightforward conversions for each unit
+ assertEquals("FileUtils.parseFileSize() failed to parse a whole number of bytes",
+ 256L, FileUtils.parseFileSize("256 bytes"));
+ assertEquals("FileUtils.parseFileSize() failed to parse a whole number of kilobytes",
+ 2048L, FileUtils.parseFileSize("2 KiB"));
+ assertEquals("FileUtils.parseFileSize() failed to parse a whole number of megabytes",
+ 4194304L, FileUtils.parseFileSize("4 MiB"));
+ assertEquals("FileUtils.parseFileSize() failed to parse a whole number of gigabytes",
+ 3221225472L, FileUtils.parseFileSize("3 GiB"));
+ assertEquals("FileUtils.parseFileSize() failed to parse a whole number of terabytes",
+ 5497558138880L, FileUtils.parseFileSize("5 TiB"));
+ // test conversions of fractional units
+ assertEquals("FileUtils.parseFileSize() failed to parse a rational number of kilobytes",
+ 1536L, FileUtils.parseFileSize("1.5 KiB"));
+ assertEquals("FileUtils.parseFileSize() failed to parse a rational number of kilobytes",
+ 4434L, FileUtils.parseFileSize("4.33 KiB"));
+ assertEquals("FileUtils.parseFileSize() failed to parse a rational number of megabytes",
+ 2359296L, FileUtils.parseFileSize("2.25 MiB"));
+ assertEquals("FileUtils.parseFileSize() failed to parse a rational number of megabytes",
+ 3292529L, FileUtils.parseFileSize("3.14 MiB"));
+ assertEquals("FileUtils.parseFileSize() failed to parse a rational number of gigabytes",
+ 1299227607L, FileUtils.parseFileSize("1.21 GiB"));
+ assertEquals("FileUtils.parseFileSize() failed to parse a rational number of terabytes",
+ 6621259022467L, FileUtils.parseFileSize("6.022 TiB"));
+ }
+
+ @Test
public void testTruncate() throws IOException
{
- File file = FileUtils.createTempFile("testTruncate", "1");
+ File file = FileUtils.createDeletableTempFile("testTruncate", "1");
final String expected = "The quick brown fox jumps over the lazy dog";
Files.write(file.toPath(), expected.getBytes());
assertTrue(file.exists());
byte[] b = Files.readAllBytes(file.toPath());
- assertEquals(expected, new String(b, Charset.forName("UTF-8")));
+ assertEquals(expected, new String(b, StandardCharsets.UTF_8));
FileUtils.truncate(file.getAbsolutePath(), 10);
b = Files.readAllBytes(file.toPath());
- assertEquals("The quick ", new String(b, Charset.forName("UTF-8")));
+ assertEquals("The quick ", new String(b, StandardCharsets.UTF_8));
FileUtils.truncate(file.getAbsolutePath(), 0);
b = Files.readAllBytes(file.toPath());
diff --git a/test/unit/org/apache/cassandra/io/util/MemoryTest.java b/test/unit/org/apache/cassandra/io/util/MemoryTest.java
index 81dee7e..677992f 100644
--- a/test/unit/org/apache/cassandra/io/util/MemoryTest.java
+++ b/test/unit/org/apache/cassandra/io/util/MemoryTest.java
@@ -27,7 +27,7 @@
import org.junit.Test;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.cassandra.utils.memory.MemoryUtil;
import static org.junit.Assert.assertEquals;
diff --git a/test/unit/org/apache/cassandra/io/util/MmappedRegionsTest.java b/test/unit/org/apache/cassandra/io/util/MmappedRegionsTest.java
index 39c9689..2814bab 100644
--- a/test/unit/org/apache/cassandra/io/util/MmappedRegionsTest.java
+++ b/test/unit/org/apache/cassandra/io/util/MmappedRegionsTest.java
@@ -37,7 +37,6 @@
import org.apache.cassandra.io.compress.CompressionMetadata;
import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
import org.apache.cassandra.schema.CompressionParams;
-import org.apache.cassandra.utils.ChecksumType;
import static junit.framework.Assert.assertNull;
import static org.junit.Assert.assertEquals;
@@ -68,7 +67,7 @@
private static File writeFile(String fileName, ByteBuffer buffer) throws IOException
{
- File ret = File.createTempFile(fileName, "1");
+ File ret = FileUtils.createTempFile(fileName, "1");
ret.deleteOnExit();
try (SequentialWriter writer = new SequentialWriter(ret))
@@ -299,10 +298,10 @@
MmappedRegions.MAX_SEGMENT_SIZE = 1024;
ByteBuffer buffer = allocateBuffer(128 * 1024);
- File f = File.createTempFile("testMapForCompressionMetadata", "1");
+ File f = FileUtils.createTempFile("testMapForCompressionMetadata", "1");
f.deleteOnExit();
- File cf = File.createTempFile(f.getName() + ".metadata", "1");
+ File cf = FileUtils.createTempFile(f.getName() + ".metadata", "1");
cf.deleteOnExit();
MetadataCollector sstableMetadataCollector = new MetadataCollector(new ClusteringComparator(BytesType.instance));
@@ -314,7 +313,7 @@
writer.finish();
}
- CompressionMetadata metadata = new CompressionMetadata(cf.getAbsolutePath(), f.length(), ChecksumType.CRC32);
+ CompressionMetadata metadata = new CompressionMetadata(cf.getAbsolutePath(), f.length(), true);
try(ChannelProxy channel = new ChannelProxy(f);
MmappedRegions regions = MmappedRegions.map(channel, metadata))
{
diff --git a/test/unit/org/apache/cassandra/io/util/NIODataInputStreamTest.java b/test/unit/org/apache/cassandra/io/util/NIODataInputStreamTest.java
index 7b91ccb..7feeb43 100644
--- a/test/unit/org/apache/cassandra/io/util/NIODataInputStreamTest.java
+++ b/test/unit/org/apache/cassandra/io/util/NIODataInputStreamTest.java
@@ -222,7 +222,7 @@
is.read(new byte[10]);
assertEquals(8190 - 10 - 4096, is.available());
- File f = File.createTempFile("foo", "bar");
+ File f = FileUtils.createTempFile("foo", "bar");
RandomAccessFile fos = new RandomAccessFile(f, "rw");
fos.write(new byte[10]);
fos.seek(0);
diff --git a/test/unit/org/apache/cassandra/io/util/RandomAccessReaderTest.java b/test/unit/org/apache/cassandra/io/util/RandomAccessReaderTest.java
index 8941d2a..74a4e8d 100644
--- a/test/unit/org/apache/cassandra/io/util/RandomAccessReaderTest.java
+++ b/test/unit/org/apache/cassandra/io/util/RandomAccessReaderTest.java
@@ -29,13 +29,14 @@
import java.nio.channels.FileLock;
import java.nio.channels.ReadableByteChannel;
import java.nio.channels.WritableByteChannel;
-import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.Random;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
+import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -266,7 +267,7 @@
private static File writeFile(Parameters params) throws IOException
{
- final File f = File.createTempFile("testReadFully", "1");
+ final File f = FileUtils.createTempFile("testReadFully", "1");
f.deleteOnExit();
try(SequentialWriter writer = new SequentialWriter(f))
@@ -319,7 +320,7 @@
@Test
public void testReadBytes() throws IOException
{
- File f = File.createTempFile("testReadBytes", "1");
+ File f = FileUtils.createTempFile("testReadBytes", "1");
final String expected = "The quick brown fox jumps over the lazy dog";
try(SequentialWriter writer = new SequentialWriter(f))
@@ -338,7 +339,7 @@
assertEquals(expected.length(), reader.length());
ByteBuffer b = ByteBufferUtil.read(reader, expected.length());
- assertEquals(expected, new String(b.array(), Charset.forName("UTF-8")));
+ assertEquals(expected, new String(b.array(), StandardCharsets.UTF_8));
assertTrue(reader.isEOF());
assertEquals(0, reader.bytesRemaining());
@@ -348,7 +349,7 @@
@Test
public void testReset() throws IOException
{
- File f = File.createTempFile("testMark", "1");
+ File f = FileUtils.createTempFile("testMark", "1");
final String expected = "The quick brown fox jumps over the lazy dog";
final int numIterations = 10;
@@ -368,7 +369,7 @@
assertEquals(expected.length() * numIterations, reader.length());
ByteBuffer b = ByteBufferUtil.read(reader, expected.length());
- assertEquals(expected, new String(b.array(), Charset.forName("UTF-8")));
+ assertEquals(expected, new String(b.array(), StandardCharsets.UTF_8));
assertFalse(reader.isEOF());
assertEquals((numIterations - 1) * expected.length(), reader.bytesRemaining());
@@ -380,7 +381,7 @@
for (int i = 0; i < (numIterations - 1); i++)
{
b = ByteBufferUtil.read(reader, expected.length());
- assertEquals(expected, new String(b.array(), Charset.forName("UTF-8")));
+ assertEquals(expected, new String(b.array(), StandardCharsets.UTF_8));
}
assertTrue(reader.isEOF());
assertEquals(expected.length() * (numIterations - 1), reader.bytesPastMark());
@@ -393,7 +394,7 @@
for (int i = 0; i < (numIterations - 1); i++)
{
b = ByteBufferUtil.read(reader, expected.length());
- assertEquals(expected, new String(b.array(), Charset.forName("UTF-8")));
+ assertEquals(expected, new String(b.array(), StandardCharsets.UTF_8));
}
reader.reset();
@@ -403,7 +404,7 @@
for (int i = 0; i < (numIterations - 1); i++)
{
b = ByteBufferUtil.read(reader, expected.length());
- assertEquals(expected, new String(b.array(), Charset.forName("UTF-8")));
+ assertEquals(expected, new String(b.array(), StandardCharsets.UTF_8));
}
assertTrue(reader.isEOF());
@@ -424,7 +425,7 @@
private static void testSeek(int numThreads) throws IOException, InterruptedException
{
- final File f = File.createTempFile("testMark", "1");
+ final File f = FileUtils.createTempFile("testMark", "1");
final byte[] expected = new byte[1 << 16];
long seed = System.nanoTime();
@@ -494,7 +495,7 @@
executor.submit(worker);
executor.shutdown();
- executor.awaitTermination(1, TimeUnit.MINUTES);
+ Assert.assertTrue(executor.awaitTermination(1, TimeUnit.MINUTES));
}
}
}
diff --git a/test/unit/org/apache/cassandra/io/util/RewindableDataInputStreamPlusTest.java b/test/unit/org/apache/cassandra/io/util/RewindableDataInputStreamPlusTest.java
index 175ab53..08c9ddf 100644
--- a/test/unit/org/apache/cassandra/io/util/RewindableDataInputStreamPlusTest.java
+++ b/test/unit/org/apache/cassandra/io/util/RewindableDataInputStreamPlusTest.java
@@ -42,7 +42,7 @@
@Before
public void setup() throws Exception
{
- this.file = new File(System.getProperty("java.io.tmpdir"), "subdir/test.buffer");
+ this.file = new File(FileUtils.getTempDir(), "subdir/test.buffer");
}
@Test
@@ -378,7 +378,7 @@
//finish reading again previous sequence
reader.mark();
- //read 3 bytes - OK
+ //read 3 bytes - START
assertEquals('a', reader.readChar());
//read 1 more bytes - CAPACITY will exhaust when trying to reset :(
assertEquals(1, reader.readShort());
diff --git a/test/unit/org/apache/cassandra/io/util/SafeMemoryWriterTest.java b/test/unit/org/apache/cassandra/io/util/SafeMemoryWriterTest.java
index 12c8c98..8b37c2d 100644
--- a/test/unit/org/apache/cassandra/io/util/SafeMemoryWriterTest.java
+++ b/test/unit/org/apache/cassandra/io/util/SafeMemoryWriterTest.java
@@ -24,8 +24,6 @@
import org.junit.Assert;
import org.junit.Test;
-import sun.misc.VM;
-
import static org.junit.Assert.assertEquals;
public class SafeMemoryWriterTest
@@ -33,6 +31,28 @@
Random rand = new Random();
static final int CHUNK = 54321;
+ static final long maxDirectMemory;
+ static
+ {
+ try
+ {
+ Class<?> cVM;
+ try
+ {
+ cVM = Class.forName("jdk.internal.misc.VM");
+ }
+ catch (ClassNotFoundException e)
+ {
+ cVM = Class.forName("sun.misc.VM");
+ }
+ maxDirectMemory = (Long) cVM.getDeclaredMethod("maxDirectMemory").invoke(null);
+ }
+ catch (Exception e)
+ {
+ throw new RuntimeException(e);
+ }
+ }
+
@Test
public void testTrim() throws IOException
{
@@ -51,9 +71,9 @@
while (initialSize * 2 / 3 > 1024L * 1024L * DataOutputBuffer.DOUBLING_THRESHOLD)
initialSize = initialSize * 2 / 3;
- if (VM.maxDirectMemory() * 2 / 3 < testSize)
+ if (maxDirectMemory * 2 / 3 < testSize)
{
- testSize = VM.maxDirectMemory() * 2 / 3;
+ testSize = maxDirectMemory * 2 / 3;
System.err.format("Insufficient direct memory for full test, reducing to: %,d %x\n", testSize, testSize);
}
diff --git a/test/unit/org/apache/cassandra/io/util/SequentialWriterTest.java b/test/unit/org/apache/cassandra/io/util/SequentialWriterTest.java
index 002ed23..c1ffda2 100644
--- a/test/unit/org/apache/cassandra/io/util/SequentialWriterTest.java
+++ b/test/unit/org/apache/cassandra/io/util/SequentialWriterTest.java
@@ -31,7 +31,7 @@
import org.junit.BeforeClass;
import org.junit.Test;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.io.compress.BufferType;
diff --git a/test/unit/org/apache/cassandra/locator/AlibabaCloudSnitchTest.java b/test/unit/org/apache/cassandra/locator/AlibabaCloudSnitchTest.java
new file mode 100644
index 0000000..043e332
--- /dev/null
+++ b/test/unit/org/apache/cassandra/locator/AlibabaCloudSnitchTest.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.locator;
+
+import static org.junit.Assert.assertEquals;
+import java.io.IOException;
+import java.net.InetAddress;
+import java.util.EnumMap;
+import java.util.Map;
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.commitlog.CommitLog;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.gms.ApplicationState;
+import org.apache.cassandra.gms.Gossiper;
+import org.apache.cassandra.gms.VersionedValue;
+import org.apache.cassandra.service.StorageService;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class AlibabaCloudSnitchTest
+{
+ private static String az;
+
+ @BeforeClass
+ public static void setup() throws Exception
+ {
+ System.setProperty(Gossiper.Props.DISABLE_THREAD_VALIDATION, "true");
+ DatabaseDescriptor.daemonInitialization();
+ CommitLog.instance.start();
+ SchemaLoader.mkdirs();
+ SchemaLoader.cleanup();
+ Keyspace.setInitialized();
+ StorageService.instance.initServer(0);
+ }
+
+ private class TestAlibabaCloudSnitch extends AlibabaCloudSnitch
+ {
+ public TestAlibabaCloudSnitch() throws IOException, ConfigurationException
+ {
+ super();
+ }
+
+ @Override
+ String alibabaApiCall(String url) throws IOException, ConfigurationException
+ {
+ return az;
+ }
+ }
+
+ @Test
+ public void testRac() throws IOException, ConfigurationException
+ {
+ az = "cn-hangzhou-f";
+ AlibabaCloudSnitch snitch = new TestAlibabaCloudSnitch();
+ InetAddressAndPort local = InetAddressAndPort.getByName("127.0.0.1");
+ InetAddressAndPort nonlocal = InetAddressAndPort.getByName("127.0.0.7");
+
+ Gossiper.instance.addSavedEndpoint(nonlocal);
+ Map<ApplicationState, VersionedValue> stateMap = new EnumMap<>(ApplicationState.class);
+ stateMap.put(ApplicationState.DC, StorageService.instance.valueFactory.datacenter("cn-shanghai"));
+ stateMap.put(ApplicationState.RACK, StorageService.instance.valueFactory.datacenter("a"));
+ Gossiper.instance.getEndpointStateForEndpoint(nonlocal).addApplicationStates(stateMap);
+
+ assertEquals("cn-shanghai", snitch.getDatacenter(nonlocal));
+ assertEquals("a", snitch.getRack(nonlocal));
+
+ assertEquals("cn-hangzhou", snitch.getDatacenter(local));
+ assertEquals("f", snitch.getRack(local));
+ }
+
+ @Test
+ public void testNewRegions() throws IOException, ConfigurationException
+ {
+ az = "us-east-1a";
+ AlibabaCloudSnitch snitch = new TestAlibabaCloudSnitch();
+ InetAddressAndPort local = InetAddressAndPort.getByName("127.0.0.1");
+ assertEquals("us-east", snitch.getDatacenter(local));
+ assertEquals("1a", snitch.getRack(local));
+ }
+
+ @AfterClass
+ public static void tearDown()
+ {
+ StorageService.instance.stopClient();
+ }
+
+}
diff --git a/test/unit/org/apache/cassandra/locator/CloudstackSnitchTest.java b/test/unit/org/apache/cassandra/locator/CloudstackSnitchTest.java
index bc3e837..7d623a2 100644
--- a/test/unit/org/apache/cassandra/locator/CloudstackSnitchTest.java
+++ b/test/unit/org/apache/cassandra/locator/CloudstackSnitchTest.java
@@ -19,7 +19,6 @@
package org.apache.cassandra.locator;
import java.io.IOException;
-import java.net.InetAddress;
import java.util.EnumMap;
import java.util.Map;
@@ -30,6 +29,7 @@
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.commitlog.CommitLog;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.gms.ApplicationState;
import org.apache.cassandra.gms.Gossiper;
@@ -47,6 +47,7 @@
{
System.setProperty(Gossiper.Props.DISABLE_THREAD_VALIDATION, "true");
DatabaseDescriptor.daemonInitialization();
+ CommitLog.instance.start();
SchemaLoader.mkdirs();
SchemaLoader.cleanup();
Keyspace.setInitialized();
@@ -78,8 +79,8 @@
{
az = "ch-gva-1";
CloudstackSnitch snitch = new TestCloudstackSnitch();
- InetAddress local = InetAddress.getByName("127.0.0.1");
- InetAddress nonlocal = InetAddress.getByName("127.0.0.7");
+ InetAddressAndPort local = InetAddressAndPort.getByName("127.0.0.1");
+ InetAddressAndPort nonlocal = InetAddressAndPort.getByName("127.0.0.7");
Gossiper.instance.addSavedEndpoint(nonlocal);
Map<ApplicationState, VersionedValue> stateMap = new EnumMap<>(ApplicationState.class);
@@ -100,7 +101,7 @@
{
az = "ch-gva-1";
CloudstackSnitch snitch = new TestCloudstackSnitch();
- InetAddress local = InetAddress.getByName("127.0.0.1");
+ InetAddressAndPort local = InetAddressAndPort.getByName("127.0.0.1");
assertEquals("ch-gva", snitch.getDatacenter(local));
assertEquals("1", snitch.getRack(local));
diff --git a/test/unit/org/apache/cassandra/locator/DynamicEndpointSnitchTest.java b/test/unit/org/apache/cassandra/locator/DynamicEndpointSnitchTest.java
index 8a59a4a..069c222 100644
--- a/test/unit/org/apache/cassandra/locator/DynamicEndpointSnitchTest.java
+++ b/test/unit/org/apache/cassandra/locator/DynamicEndpointSnitchTest.java
@@ -19,18 +19,19 @@
package org.apache.cassandra.locator;
import java.io.IOException;
-import java.net.InetAddress;
import java.util.*;
+import java.util.concurrent.TimeUnit;
import org.junit.BeforeClass;
import org.junit.Test;
+import org.apache.cassandra.Util;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.FBUtilities;
-import static org.junit.Assert.assertEquals;
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
public class DynamicEndpointSnitchTest
{
@@ -41,16 +42,26 @@
DatabaseDescriptor.daemonInitialization();
}
- private static void setScores(DynamicEndpointSnitch dsnitch, int rounds, List<InetAddress> hosts, Integer... scores) throws InterruptedException
+ private static void setScores(DynamicEndpointSnitch dsnitch, int rounds, List<InetAddressAndPort> hosts, Integer... scores) throws InterruptedException
{
for (int round = 0; round < rounds; round++)
{
for (int i = 0; i < hosts.size(); i++)
- dsnitch.receiveTiming(hosts.get(i), scores[i]);
+ dsnitch.receiveTiming(hosts.get(i), scores[i], MILLISECONDS);
}
Thread.sleep(150);
}
+ private static EndpointsForRange full(InetAddressAndPort... endpoints)
+ {
+ EndpointsForRange.Builder rlist = EndpointsForRange.builder(ReplicaUtils.FULL_RANGE, endpoints.length);
+ for (InetAddressAndPort endpoint: endpoints)
+ {
+ rlist.add(ReplicaUtils.full(endpoint));
+ }
+ return rlist.build();
+ }
+
@Test
public void testSnitch() throws InterruptedException, IOException, ConfigurationException
{
@@ -58,50 +69,51 @@
StorageService.instance.unsafeInitialize();
SimpleSnitch ss = new SimpleSnitch();
DynamicEndpointSnitch dsnitch = new DynamicEndpointSnitch(ss, String.valueOf(ss.hashCode()));
- InetAddress self = FBUtilities.getBroadcastAddress();
- InetAddress host1 = InetAddress.getByName("127.0.0.2");
- InetAddress host2 = InetAddress.getByName("127.0.0.3");
- InetAddress host3 = InetAddress.getByName("127.0.0.4");
- InetAddress host4 = InetAddress.getByName("127.0.0.5");
- List<InetAddress> hosts = Arrays.asList(host1, host2, host3);
+ InetAddressAndPort self = FBUtilities.getBroadcastAddressAndPort();
+ InetAddressAndPort host1 = InetAddressAndPort.getByName("127.0.0.2");
+ InetAddressAndPort host2 = InetAddressAndPort.getByName("127.0.0.3");
+ InetAddressAndPort host3 = InetAddressAndPort.getByName("127.0.0.4");
+ InetAddressAndPort host4 = InetAddressAndPort.getByName("127.0.0.5");
+ List<InetAddressAndPort> hosts = Arrays.asList(host1, host2, host3);
// first, make all hosts equal
setScores(dsnitch, 1, hosts, 10, 10, 10);
- List<InetAddress> order = Arrays.asList(host1, host2, host3);
- assertEquals(order, dsnitch.getSortedListByProximity(self, Arrays.asList(host1, host2, host3)));
+ EndpointsForRange order = full(host1, host2, host3);
+ Util.assertRCEquals(order, dsnitch.sortedByProximity(self, full(host1, host2, host3)));
// make host1 a little worse
setScores(dsnitch, 1, hosts, 20, 10, 10);
- order = Arrays.asList(host2, host3, host1);
- assertEquals(order, dsnitch.getSortedListByProximity(self, Arrays.asList(host1, host2, host3)));
+ order = full(host2, host3, host1);
+ Util.assertRCEquals(order, dsnitch.sortedByProximity(self, full(host1, host2, host3)));
// make host2 as bad as host1
setScores(dsnitch, 2, hosts, 15, 20, 10);
- order = Arrays.asList(host3, host1, host2);
- assertEquals(order, dsnitch.getSortedListByProximity(self, Arrays.asList(host1, host2, host3)));
+ order = full(host3, host1, host2);
+ Util.assertRCEquals(order, dsnitch.sortedByProximity(self, full(host1, host2, host3)));
// make host3 the worst
setScores(dsnitch, 3, hosts, 10, 10, 30);
- order = Arrays.asList(host1, host2, host3);
- assertEquals(order, dsnitch.getSortedListByProximity(self, Arrays.asList(host1, host2, host3)));
+ order = full(host1, host2, host3);
+ Util.assertRCEquals(order, dsnitch.sortedByProximity(self, full(host1, host2, host3)));
// make host3 equal to the others
setScores(dsnitch, 5, hosts, 10, 10, 10);
- order = Arrays.asList(host1, host2, host3);
- assertEquals(order, dsnitch.getSortedListByProximity(self, Arrays.asList(host1, host2, host3)));
+ order = full(host1, host2, host3);
+ Util.assertRCEquals(order, dsnitch.sortedByProximity(self, full(host1, host2, host3)));
/// Tests CASSANDRA-6683 improvements
// make the scores differ enough from the ideal order that we sort by score; under the old
// dynamic snitch behavior (where we only compared neighbors), these wouldn't get sorted
setScores(dsnitch, 20, hosts, 10, 70, 20);
- order = Arrays.asList(host1, host3, host2);
- assertEquals(order, dsnitch.getSortedListByProximity(self, Arrays.asList(host1, host2, host3)));
+ order = full(host1, host3, host2);
+ Util.assertRCEquals(order, dsnitch.sortedByProximity(self, full(host1, host2, host3)));
- order = Arrays.asList(host4, host1, host3, host2);
- assertEquals(order, dsnitch.getSortedListByProximity(self, Arrays.asList(host1, host2, host3, host4)));
+ order = full(host4, host1, host3, host2);
+ Util.assertRCEquals(order, dsnitch.sortedByProximity(self, full(host1, host2, host3, host4)));
+
setScores(dsnitch, 20, hosts, 10, 10, 10);
- order = Arrays.asList(host1, host2, host3, host4);
- assertEquals(order, dsnitch.getSortedListByProximity(self, Arrays.asList(host1, host2, host3, host4)));
+ order = full(host4, host1, host2, host3);
+ Util.assertRCEquals(order, dsnitch.sortedByProximity(self, full(host1, host2, host3, host4)));
}
}
diff --git a/test/unit/org/apache/cassandra/locator/EC2SnitchTest.java b/test/unit/org/apache/cassandra/locator/EC2SnitchTest.java
index 0c71c92..13d5149 100644
--- a/test/unit/org/apache/cassandra/locator/EC2SnitchTest.java
+++ b/test/unit/org/apache/cassandra/locator/EC2SnitchTest.java
@@ -20,10 +20,11 @@
import java.io.IOException;
-import java.net.InetAddress;
-import java.net.UnknownHostException;
+import java.util.Collections;
import java.util.EnumMap;
+import java.util.HashSet;
import java.util.Map;
+import java.util.Set;
import org.junit.AfterClass;
import org.junit.Assert;
@@ -33,25 +34,34 @@
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.commitlog.CommitLog;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.gms.ApplicationState;
import org.apache.cassandra.gms.Gossiper;
import org.apache.cassandra.gms.VersionedValue;
-import org.apache.cassandra.net.MessagingService;
-import org.apache.cassandra.net.OutboundTcpConnectionPool;
import org.apache.cassandra.service.StorageService;
+import static org.apache.cassandra.locator.Ec2Snitch.EC2_NAMING_LEGACY;
import static org.junit.Assert.assertEquals;
public class EC2SnitchTest
{
private static String az;
+ private final SnitchProperties legacySnitchProps = new SnitchProperties()
+ {
+ public String get(String propertyName, String defaultValue)
+ {
+ return propertyName.equals("ec2_naming_scheme") ? EC2_NAMING_LEGACY : super.get(propertyName, defaultValue);
+ }
+ };
+
@BeforeClass
public static void setup() throws Exception
{
System.setProperty(Gossiper.Props.DISABLE_THREAD_VALIDATION, "true");
DatabaseDescriptor.daemonInitialization();
+ CommitLog.instance.start();
SchemaLoader.mkdirs();
SchemaLoader.cleanup();
Keyspace.setInitialized();
@@ -65,6 +75,11 @@
super();
}
+ public TestEC2Snitch(SnitchProperties props) throws IOException, ConfigurationException
+ {
+ super(props);
+ }
+
@Override
String awsApiCall(String url) throws IOException, ConfigurationException
{
@@ -73,12 +88,12 @@
}
@Test
- public void testRac() throws IOException, ConfigurationException
+ public void testLegacyRac() throws IOException, ConfigurationException
{
az = "us-east-1d";
- Ec2Snitch snitch = new TestEC2Snitch();
- InetAddress local = InetAddress.getByName("127.0.0.1");
- InetAddress nonlocal = InetAddress.getByName("127.0.0.7");
+ Ec2Snitch snitch = new TestEC2Snitch(legacySnitchProps);
+ InetAddressAndPort local = InetAddressAndPort.getByName("127.0.0.1");
+ InetAddressAndPort nonlocal = InetAddressAndPort.getByName("127.0.0.7");
Gossiper.instance.addSavedEndpoint(nonlocal);
Map<ApplicationState, VersionedValue> stateMap = new EnumMap<>(ApplicationState.class);
@@ -92,31 +107,131 @@
assertEquals("us-east", snitch.getDatacenter(local));
assertEquals("1d", snitch.getRack(local));
}
-
+
@Test
- public void testNewRegions() throws IOException, ConfigurationException
+ public void testLegacyNewRegions() throws IOException, ConfigurationException
{
az = "us-east-2d";
- Ec2Snitch snitch = new TestEC2Snitch();
- InetAddress local = InetAddress.getByName("127.0.0.1");
+ Ec2Snitch snitch = new TestEC2Snitch(legacySnitchProps);
+ InetAddressAndPort local = InetAddressAndPort.getByName("127.0.0.1");
assertEquals("us-east-2", snitch.getDatacenter(local));
assertEquals("2d", snitch.getRack(local));
}
@Test
- public void testEc2MRSnitch() throws UnknownHostException
+ public void testFullNamingScheme() throws IOException, ConfigurationException
{
- InetAddress me = InetAddress.getByName("127.0.0.2");
- InetAddress com_ip = InetAddress.getByName("127.0.0.3");
+ InetAddressAndPort local = InetAddressAndPort.getByName("127.0.0.1");
+ az = "us-east-2d";
+ Ec2Snitch snitch = new TestEC2Snitch();
- OutboundTcpConnectionPool pool = MessagingService.instance().getConnectionPool(me);
- Assert.assertEquals(me, pool.endPoint());
- pool.reset(com_ip);
- Assert.assertEquals(com_ip, pool.endPoint());
+ assertEquals("us-east-2", snitch.getDatacenter(local));
+ assertEquals("us-east-2d", snitch.getRack(local));
- MessagingService.instance().destroyConnectionPool(me);
- pool = MessagingService.instance().getConnectionPool(me);
- Assert.assertEquals(com_ip, pool.endPoint());
+ az = "us-west-1a";
+ snitch = new TestEC2Snitch();
+
+ assertEquals("us-west-1", snitch.getDatacenter(local));
+ assertEquals("us-west-1a", snitch.getRack(local));
+ }
+
+ @Test
+ public void validateDatacenter_RequiresLegacy_CorrectAmazonName()
+ {
+ Set<String> datacenters = new HashSet<>();
+ datacenters.add("us-east-1");
+ Assert.assertTrue(Ec2Snitch.validate(datacenters, Collections.emptySet(), true));
+ }
+
+ @Test
+ public void validateDatacenter_RequiresLegacy_LegacyName()
+ {
+ Set<String> datacenters = new HashSet<>();
+ datacenters.add("us-east");
+ Assert.assertTrue(Ec2Snitch.validate(datacenters, Collections.emptySet(), true));
+ }
+
+ @Test
+ public void validate_RequiresLegacy_HappyPath()
+ {
+ Set<String> datacenters = new HashSet<>();
+ datacenters.add("us-east");
+ Set<String> racks = new HashSet<>();
+ racks.add("1a");
+ Assert.assertTrue(Ec2Snitch.validate(datacenters, racks, true));
+ }
+
+ @Test
+ public void validate_RequiresLegacy_HappyPathWithDCSuffix()
+ {
+ Set<String> datacenters = new HashSet<>();
+ datacenters.add("us-east_CUSTOM_SUFFIX");
+ Set<String> racks = new HashSet<>();
+ racks.add("1a");
+ Assert.assertTrue(Ec2Snitch.validate(datacenters, racks, true));
+ }
+
+ @Test
+ public void validateRack_RequiresAmazonName_CorrectAmazonName()
+ {
+ Set<String> racks = new HashSet<>();
+ racks.add("us-east-1a");
+ Assert.assertTrue(Ec2Snitch.validate(Collections.emptySet(), racks, false));
+ }
+
+ @Test
+ public void validateRack_RequiresAmazonName_LegacyName()
+ {
+ Set<String> racks = new HashSet<>();
+ racks.add("1a");
+ Assert.assertFalse(Ec2Snitch.validate(Collections.emptySet(), racks, false));
+ }
+
+ @Test
+ public void validate_RequiresAmazonName_HappyPath()
+ {
+ Set<String> datacenters = new HashSet<>();
+ datacenters.add("us-east-1");
+ Set<String> racks = new HashSet<>();
+ racks.add("us-east-1a");
+ Assert.assertTrue(Ec2Snitch.validate(datacenters, racks, false));
+ }
+
+ @Test
+ public void validate_RequiresAmazonName_HappyPathWithDCSuffix()
+ {
+ Set<String> datacenters = new HashSet<>();
+ datacenters.add("us-east-1_CUSTOM_SUFFIX");
+ Set<String> racks = new HashSet<>();
+ racks.add("us-east-1a");
+ Assert.assertTrue(Ec2Snitch.validate(datacenters, racks, false));
+ }
+
+ /**
+ * Validate upgrades in legacy mode for regions that didn't change name between the standard and legacy modes.
+ */
+ @Test
+ public void validate_RequiresLegacy_DCValidStandardAndLegacy()
+ {
+ Set<String> datacenters = new HashSet<>();
+ datacenters.add("us-west-2");
+ Set<String> racks = new HashSet<>();
+ racks.add("2a");
+ racks.add("2b");
+ Assert.assertTrue(Ec2Snitch.validate(datacenters, racks, true));
+ }
+
+ /**
+ * Check that racks names are enough to detect a mismatch in naming conventions.
+ */
+ @Test
+ public void validate_RequiresLegacy_RackInvalidForLegacy()
+ {
+ Set<String> datacenters = new HashSet<>();
+ datacenters.add("us-west-2");
+ Set<String> racks = new HashSet<>();
+ racks.add("us-west-2a");
+ Assert.assertFalse(Ec2Snitch.validate(datacenters, racks, true));
}
@AfterClass
diff --git a/test/unit/org/apache/cassandra/locator/GoogleCloudSnitchTest.java b/test/unit/org/apache/cassandra/locator/GoogleCloudSnitchTest.java
index 2491ba9..ac1abe1 100644
--- a/test/unit/org/apache/cassandra/locator/GoogleCloudSnitchTest.java
+++ b/test/unit/org/apache/cassandra/locator/GoogleCloudSnitchTest.java
@@ -20,7 +20,6 @@
import java.io.IOException;
-import java.net.InetAddress;
import java.util.EnumMap;
import java.util.Map;
@@ -31,6 +30,7 @@
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.commitlog.CommitLog;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.gms.ApplicationState;
import org.apache.cassandra.gms.Gossiper;
@@ -48,6 +48,7 @@
{
System.setProperty(Gossiper.Props.DISABLE_THREAD_VALIDATION, "true");
DatabaseDescriptor.daemonInitialization();
+ CommitLog.instance.start();
SchemaLoader.mkdirs();
SchemaLoader.cleanup();
Keyspace.setInitialized();
@@ -73,8 +74,8 @@
{
az = "us-central1-a";
GoogleCloudSnitch snitch = new TestGoogleCloudSnitch();
- InetAddress local = InetAddress.getByName("127.0.0.1");
- InetAddress nonlocal = InetAddress.getByName("127.0.0.7");
+ InetAddressAndPort local = InetAddressAndPort.getByName("127.0.0.1");
+ InetAddressAndPort nonlocal = InetAddressAndPort.getByName("127.0.0.7");
Gossiper.instance.addSavedEndpoint(nonlocal);
Map<ApplicationState, VersionedValue> stateMap = new EnumMap<>(ApplicationState.class);
@@ -94,7 +95,7 @@
{
az = "asia-east1-a";
GoogleCloudSnitch snitch = new TestGoogleCloudSnitch();
- InetAddress local = InetAddress.getByName("127.0.0.1");
+ InetAddressAndPort local = InetAddressAndPort.getByName("127.0.0.1");
assertEquals("asia-east1", snitch.getDatacenter(local));
assertEquals("a", snitch.getRack(local));
}
diff --git a/test/unit/org/apache/cassandra/locator/GossipingPropertyFileSnitchTest.java b/test/unit/org/apache/cassandra/locator/GossipingPropertyFileSnitchTest.java
index 77734f7..da26003 100644
--- a/test/unit/org/apache/cassandra/locator/GossipingPropertyFileSnitchTest.java
+++ b/test/unit/org/apache/cassandra/locator/GossipingPropertyFileSnitchTest.java
@@ -18,9 +18,8 @@
package org.apache.cassandra.locator;
-import java.net.InetAddress;
+import java.net.UnknownHostException;
-import com.google.common.net.InetAddresses;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -45,7 +44,15 @@
final String endpointString, final String expectedDatacenter,
final String expectedRack)
{
- final InetAddress endpoint = InetAddresses.forString(endpointString);
+ final InetAddressAndPort endpoint;
+ try
+ {
+ endpoint = InetAddressAndPort.getByName(endpointString);
+ }
+ catch (UnknownHostException e)
+ {
+ throw new RuntimeException(e);
+ }
assertEquals(expectedDatacenter, snitch.getDatacenter(endpoint));
assertEquals(expectedRack, snitch.getRack(endpoint));
}
@@ -54,6 +61,6 @@
public void testLoadConfig() throws Exception
{
final GossipingPropertyFileSnitch snitch = new GossipingPropertyFileSnitch();
- checkEndpoint(snitch, FBUtilities.getBroadcastAddress().getHostAddress(), "DC1", "RAC1");
+ checkEndpoint(snitch, FBUtilities.getBroadcastAddressAndPort().toString(), "DC1", "RAC1");
}
}
diff --git a/test/unit/org/apache/cassandra/locator/InetAddressAndPortSerializerTest.java b/test/unit/org/apache/cassandra/locator/InetAddressAndPortSerializerTest.java
new file mode 100644
index 0000000..c6ea3d7
--- /dev/null
+++ b/test/unit/org/apache/cassandra/locator/InetAddressAndPortSerializerTest.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.locator;
+
+import java.nio.ByteBuffer;
+
+import org.junit.Test;
+
+import org.apache.cassandra.io.util.DataInputBuffer;
+import org.apache.cassandra.io.util.DataOutputBuffer;
+import org.apache.cassandra.net.MessagingService;
+
+import static org.apache.cassandra.locator.InetAddressAndPort.Serializer.inetAddressAndPortSerializer;
+import static org.junit.Assert.assertEquals;
+
+public class InetAddressAndPortSerializerTest
+{
+ @Test
+ public void testRoundtrip() throws Exception
+ {
+ InetAddressAndPort ipv4 = InetAddressAndPort.getByName("127.0.0.1:42");
+ InetAddressAndPort ipv6 = InetAddressAndPort.getByName("[2001:db8:0:0:0:ff00:42:8329]:42");
+
+ testAddress(ipv4, MessagingService.VERSION_30);
+ testAddress(ipv6, MessagingService.VERSION_30);
+ testAddress(ipv4, MessagingService.current_version);
+ testAddress(ipv6, MessagingService.current_version);
+ }
+
+ private void testAddress(InetAddressAndPort address, int version) throws Exception
+ {
+ ByteBuffer out;
+ try (DataOutputBuffer dob = new DataOutputBuffer())
+ {
+ inetAddressAndPortSerializer.serialize(address, dob, version);
+ out = dob.buffer();
+ }
+ assertEquals(out.remaining(), inetAddressAndPortSerializer.serializedSize(address, version));
+
+ InetAddressAndPort roundtripped;
+ try (DataInputBuffer dib = new DataInputBuffer(out, false))
+ {
+ roundtripped = inetAddressAndPortSerializer.deserialize(dib, version);
+ }
+
+ if (version >= MessagingService.VERSION_40)
+ {
+ assertEquals(address, roundtripped);
+ }
+ else
+ {
+ assertEquals(address.address, roundtripped.address);
+ assertEquals(InetAddressAndPort.getDefaultPort(), roundtripped.port);
+ }
+ }
+}
diff --git a/test/unit/org/apache/cassandra/locator/InetAddressAndPortTest.java b/test/unit/org/apache/cassandra/locator/InetAddressAndPortTest.java
new file mode 100644
index 0000000..c32b9a9
--- /dev/null
+++ b/test/unit/org/apache/cassandra/locator/InetAddressAndPortTest.java
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.locator;
+
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+public class InetAddressAndPortTest
+{
+ private static interface ThrowingRunnable
+ {
+ public void run() throws Throwable;
+ }
+
+ @Test
+ public void getByNameIPv4Test() throws Exception
+ {
+ //Negative port
+ shouldThrow(() -> InetAddressAndPort.getByName("127.0.0.1:-1"), IllegalArgumentException.class);
+ //Too large port
+ shouldThrow(() -> InetAddressAndPort.getByName("127.0.0.1:65536"), IllegalArgumentException.class);
+
+ //bad address, caught by InetAddress
+ shouldThrow(() -> InetAddressAndPort.getByName("127.0.0.1.0"), UnknownHostException.class);
+
+ //Test default port
+ InetAddressAndPort address = InetAddressAndPort.getByName("127.0.0.1");
+ assertEquals(InetAddress.getByName("127.0.0.1"), address.address);
+ assertEquals(InetAddressAndPort.defaultPort, address.port);
+
+ //Test overriding default port
+ address = InetAddressAndPort.getByName("127.0.0.1:42");
+ assertEquals(InetAddress.getByName("127.0.0.1"), address.address);
+ assertEquals(42, address.port);
+ }
+
+ @Test
+ public void getByNameIPv6Test() throws Exception
+ {
+ //Negative port
+ shouldThrow(() -> InetAddressAndPort.getByName("[2001:0db8:0000:0000:0000:ff00:0042:8329]:-1"), IllegalArgumentException.class);
+ //Too large port
+ shouldThrow(() -> InetAddressAndPort.getByName("[2001:0db8:0000:0000:0000:ff00:0042:8329]:65536"), IllegalArgumentException.class);
+
+ //bad address, caught by InetAddress
+ shouldThrow(() -> InetAddressAndPort.getByName("2001:0db8:0000:0000:0000:ff00:0042:8329:8329"), UnknownHostException.class);
+
+ //Test default port
+ InetAddressAndPort address = InetAddressAndPort.getByName("2001:0db8:0000:0000:0000:ff00:0042:8329");
+ assertEquals(InetAddress.getByName("2001:0db8:0000:0000:0000:ff00:0042:8329"), address.address);
+ assertEquals(InetAddressAndPort.defaultPort, address.port);
+
+ //Test overriding default port
+ address = InetAddressAndPort.getByName("[2001:0db8:0000:0000:0000:ff00:0042:8329]:42");
+ assertEquals(InetAddress.getByName("2001:0db8:0000:0000:0000:ff00:0042:8329"), address.address);
+ assertEquals(42, address.port);
+ }
+
+ @Test
+ public void compareAndEqualsAndHashCodeTest() throws Exception
+ {
+ InetAddressAndPort address1 = InetAddressAndPort.getByName("127.0.0.1:42");
+ InetAddressAndPort address4 = InetAddressAndPort.getByName("127.0.0.1:43");
+ InetAddressAndPort address5 = InetAddressAndPort.getByName("127.0.0.1:41");
+ InetAddressAndPort address6 = InetAddressAndPort.getByName("127.0.0.2:42");
+ InetAddressAndPort address7 = InetAddressAndPort.getByName("127.0.0.0:42");
+
+ assertEquals(0, address1.compareTo(address1));
+ assertEquals(-1, address1.compareTo(address4));
+ assertEquals(1, address1.compareTo(address5));
+ assertEquals(-1, address1.compareTo(address6));
+ assertEquals(1, address1.compareTo(address7));
+
+ assertEquals(address1, address1);
+ assertEquals(address1.hashCode(), address1.hashCode());
+ assertEquals(address1, InetAddressAndPort.getByName("127.0.0.1:42"));
+ assertEquals(address1.hashCode(), InetAddressAndPort.getByName("127.0.0.1:42").hashCode());
+ assertEquals(address1, InetAddressAndPort.getByNameOverrideDefaults("127.0.0.1", 42));
+ assertEquals(address1.hashCode(), InetAddressAndPort.getByNameOverrideDefaults("127.0.0.1", 42).hashCode());
+ int originalPort = InetAddressAndPort.defaultPort;
+ InetAddressAndPort.initializeDefaultPort(42);
+ try
+ {
+ assertEquals(address1, InetAddressAndPort.getByName("127.0.0.1"));
+ assertEquals(address1.hashCode(), InetAddressAndPort.getByName("127.0.0.1").hashCode());
+ }
+ finally
+ {
+ InetAddressAndPort.initializeDefaultPort(originalPort);
+ }
+ assertTrue(!address1.equals(address4));
+ assertTrue(!address1.equals(address5));
+ assertTrue(!address1.equals(address6));
+ assertTrue(!address1.equals(address7));
+ }
+
+ @Test
+ public void toStringTest() throws Exception
+ {
+ String ipv4 = "127.0.0.1:42";
+ String ipv6 = "[2001:db8:0:0:0:ff00:42:8329]:42";
+ assertEquals(ipv4, InetAddressAndPort.getByName(ipv4).toString());
+ assertEquals(ipv6, InetAddressAndPort.getByName(ipv6).toString());
+ }
+
+
+ private void shouldThrow(ThrowingRunnable t, Class expectedClass)
+ {
+ try
+ {
+ t.run();
+ }
+ catch (Throwable thrown)
+ {
+ assertEquals(thrown.getClass(), expectedClass);
+ return;
+ }
+ fail("Runnable didn't throw");
+ }
+
+}
diff --git a/test/unit/org/apache/cassandra/locator/NetworkTopologyStrategyTest.java b/test/unit/org/apache/cassandra/locator/NetworkTopologyStrategyTest.java
index 48dd573..3960bd0 100644
--- a/test/unit/org/apache/cassandra/locator/NetworkTopologyStrategyTest.java
+++ b/test/unit/org/apache/cassandra/locator/NetworkTopologyStrategyTest.java
@@ -19,13 +19,13 @@
package org.apache.cassandra.locator;
import java.io.IOException;
-import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.*;
import java.util.stream.Collectors;
import com.google.common.collect.HashMultimap;
import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Lists;
import com.google.common.collect.ImmutableMultimap;
import com.google.common.collect.Multimap;
@@ -36,14 +36,20 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.cassandra.Util;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.dht.Murmur3Partitioner.LongToken;
import org.apache.cassandra.dht.OrderPreservingPartitioner.StringToken;
+import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.locator.TokenMetadata.Topology;
import org.apache.cassandra.service.StorageService;
+import static org.apache.cassandra.locator.Replica.fullReplica;
+import static org.apache.cassandra.locator.Replica.transientReplica;
+
public class NetworkTopologyStrategyTest
{
private String keyspaceName = "Keyspace1";
@@ -53,6 +59,7 @@
public static void setupDD()
{
DatabaseDescriptor.daemonInitialization();
+ DatabaseDescriptor.setTransientReplicationEnabledUnsafe(true);
}
@Test
@@ -70,13 +77,14 @@
// Set the localhost to the tokenmetadata. Embedded cassandra way?
NetworkTopologyStrategy strategy = new NetworkTopologyStrategy(keyspaceName, metadata, snitch, configOptions);
- assert strategy.getReplicationFactor("DC1") == 3;
- assert strategy.getReplicationFactor("DC2") == 2;
- assert strategy.getReplicationFactor("DC3") == 1;
+ assert strategy.getReplicationFactor("DC1").allReplicas == 3;
+ assert strategy.getReplicationFactor("DC2").allReplicas == 2;
+ assert strategy.getReplicationFactor("DC3").allReplicas == 1;
// Query for the natural hosts
- ArrayList<InetAddress> endpoints = strategy.getNaturalEndpoints(new StringToken("123"));
- assert 6 == endpoints.size();
- assert 6 == new HashSet<InetAddress>(endpoints).size(); // ensure uniqueness
+ EndpointsForToken replicas = strategy.getNaturalReplicasForToken(new StringToken("123"));
+ assert 6 == replicas.size();
+ assert 6 == replicas.endpoints().size(); // ensure uniqueness
+ assert 6 == new HashSet<>(replicas.byEndpoint().values()).size(); // ensure uniqueness
}
@Test
@@ -94,13 +102,14 @@
// Set the localhost to the tokenmetadata. Embedded cassandra way?
NetworkTopologyStrategy strategy = new NetworkTopologyStrategy(keyspaceName, metadata, snitch, configOptions);
- assert strategy.getReplicationFactor("DC1") == 3;
- assert strategy.getReplicationFactor("DC2") == 3;
- assert strategy.getReplicationFactor("DC3") == 0;
+ assert strategy.getReplicationFactor("DC1").allReplicas == 3;
+ assert strategy.getReplicationFactor("DC2").allReplicas == 3;
+ assert strategy.getReplicationFactor("DC3").allReplicas == 0;
// Query for the natural hosts
- ArrayList<InetAddress> endpoints = strategy.getNaturalEndpoints(new StringToken("123"));
- assert 6 == endpoints.size();
- assert 6 == new HashSet<InetAddress>(endpoints).size(); // ensure uniqueness
+ EndpointsForToken replicas = strategy.getNaturalReplicasForToken(new StringToken("123"));
+ assert 6 == replicas.size();
+ assert 6 == replicas.endpoints().size(); // ensure uniqueness
+ assert 6 == new HashSet<>(replicas.byEndpoint().values()).size(); // ensure uniqueness
}
@Test
@@ -114,7 +123,7 @@
DatabaseDescriptor.setEndpointSnitch(snitch);
TokenMetadata metadata = new TokenMetadata();
Map<String, String> configOptions = new HashMap<String, String>();
- Multimap<InetAddress, Token> tokens = HashMultimap.create();
+ Multimap<InetAddressAndPort, Token> tokens = HashMultimap.create();
int totalRF = 0;
for (int dc = 0; dc < dcRacks.length; ++dc)
@@ -126,7 +135,7 @@
for (int ep = 1; ep <= dcEndpoints[dc]/dcRacks[dc]; ++ep)
{
byte[] ipBytes = new byte[]{10, (byte)dc, (byte)rack, (byte)ep};
- InetAddress address = InetAddress.getByAddress(ipBytes);
+ InetAddressAndPort address = InetAddressAndPort.getByAddress(ipBytes);
StringToken token = new StringToken(String.format("%02x%02x%02x", ep, rack, dc));
logger.debug("adding node {} at {}", address, token);
tokens.put(address, token);
@@ -139,12 +148,13 @@
for (String testToken : new String[]{"123456", "200000", "000402", "ffffff", "400200"})
{
- List<InetAddress> endpoints = strategy.calculateNaturalEndpoints(new StringToken(testToken), metadata);
- Set<InetAddress> epSet = new HashSet<InetAddress>(endpoints);
+ EndpointsForRange replicas = strategy.calculateNaturalReplicas(new StringToken(testToken), metadata);
+ Set<InetAddressAndPort> endpointSet = replicas.endpoints();
- Assert.assertEquals(totalRF, endpoints.size());
- Assert.assertEquals(totalRF, epSet.size());
- logger.debug("{}: {}", testToken, endpoints);
+ Assert.assertEquals(totalRF, replicas.size());
+ Assert.assertEquals(totalRF, new HashSet<>(replicas.byEndpoint().values()).size());
+ Assert.assertEquals(totalRF, endpointSet.size());
+ logger.debug("{}: {}", testToken, replicas);
}
}
@@ -173,7 +183,7 @@
public void tokenFactory(TokenMetadata metadata, String token, byte[] bytes) throws UnknownHostException
{
Token token1 = new StringToken(token);
- InetAddress add1 = InetAddress.getByAddress(bytes);
+ InetAddressAndPort add1 = InetAddressAndPort.getByAddress(bytes);
metadata.updateNormalToken(token1, add1);
}
@@ -185,9 +195,9 @@
final int RUNS = 10;
StorageService.instance.setPartitionerUnsafe(Murmur3Partitioner.instance);
Map<String, Integer> datacenters = ImmutableMap.of("rf1", 1, "rf3", 3, "rf5_1", 5, "rf5_2", 5, "rf5_3", 5);
- List<InetAddress> nodes = new ArrayList<>(NODES);
+ List<InetAddressAndPort> nodes = new ArrayList<>(NODES);
for (byte i=0; i<NODES; ++i)
- nodes.add(InetAddress.getByAddress(new byte[]{127, 0, 0, i}));
+ nodes.add(InetAddressAndPort.getByAddress(new byte[]{ 127, 0, 0, i}));
for (int run=0; run<RUNS; ++run)
{
Random rand = new Random();
@@ -210,8 +220,8 @@
for (int i=0; i<1000; ++i)
{
Token token = Murmur3Partitioner.instance.getRandomToken(rand);
- List<InetAddress> expected = calculateNaturalEndpoints(token, tokenMetadata, datacenters, snitch);
- List<InetAddress> actual = nts.calculateNaturalEndpoints(token, tokenMetadata);
+ List<InetAddressAndPort> expected = calculateNaturalEndpoints(token, tokenMetadata, datacenters, snitch);
+ List<InetAddressAndPort> actual = new ArrayList<>(nts.calculateNaturalReplicas(token, tokenMetadata).endpoints());
if (endpointsDiffer(expected, actual))
{
System.err.println("Endpoints mismatch for token " + token);
@@ -222,7 +232,7 @@
}
}
- private boolean endpointsDiffer(List<InetAddress> ep1, List<InetAddress> ep2)
+ private boolean endpointsDiffer(List<InetAddressAndPort> ep1, List<InetAddressAndPort> ep2)
{
// Because the old algorithm does not put the nodes in the correct order in the case where more replicas
// are required than there are racks in a dc, we accept different order as long as the primary
@@ -231,15 +241,15 @@
return false;
if (!ep1.get(0).equals(ep2.get(0)))
return true;
- Set<InetAddress> s1 = new HashSet<>(ep1);
- Set<InetAddress> s2 = new HashSet<>(ep2);
+ Set<InetAddressAndPort> s1 = new HashSet<>(ep1);
+ Set<InetAddressAndPort> s2 = new HashSet<>(ep2);
return !s1.equals(s2);
}
- IEndpointSnitch generateSnitch(Map<String, Integer> datacenters, Collection<InetAddress> nodes, Random rand)
+ IEndpointSnitch generateSnitch(Map<String, Integer> datacenters, Collection<InetAddressAndPort> nodes, Random rand)
{
- final Map<InetAddress, String> nodeToRack = new HashMap<>();
- final Map<InetAddress, String> nodeToDC = new HashMap<>();
+ final Map<InetAddressAndPort, String> nodeToRack = new HashMap<>();
+ final Map<InetAddressAndPort, String> nodeToDC = new HashMap<>();
Map<String, List<String>> racksPerDC = new HashMap<>();
datacenters.forEach((dc, rf) -> racksPerDC.put(dc, randomRacks(rf, rand)));
int rf = datacenters.values().stream().mapToInt(x -> x).sum();
@@ -251,7 +261,7 @@
dcs[pos++] = dce.getKey();
}
- for (InetAddress node : nodes)
+ for (InetAddressAndPort node : nodes)
{
String dc = dcs[rand.nextInt(rf)];
List<String> racks = racksPerDC.get(dc);
@@ -262,12 +272,12 @@
return new AbstractNetworkTopologySnitch()
{
- public String getRack(InetAddress endpoint)
+ public String getRack(InetAddressAndPort endpoint)
{
return nodeToRack.get(endpoint);
}
- public String getDatacenter(InetAddress endpoint)
+ public String getDatacenter(InetAddressAndPort endpoint)
{
return nodeToDC.get(endpoint);
}
@@ -284,20 +294,20 @@
}
// Copy of older endpoints calculation algorithm for comparison
- public static List<InetAddress> calculateNaturalEndpoints(Token searchToken, TokenMetadata tokenMetadata, Map<String, Integer> datacenters, IEndpointSnitch snitch)
+ public static List<InetAddressAndPort> calculateNaturalEndpoints(Token searchToken, TokenMetadata tokenMetadata, Map<String, Integer> datacenters, IEndpointSnitch snitch)
{
// we want to preserve insertion order so that the first added endpoint becomes primary
- Set<InetAddress> replicas = new LinkedHashSet<>();
+ Set<InetAddressAndPort> replicas = new LinkedHashSet<>();
// replicas we have found in each DC
- Map<String, Set<InetAddress>> dcReplicas = new HashMap<>(datacenters.size());
+ Map<String, Set<InetAddressAndPort>> dcReplicas = new HashMap<>(datacenters.size());
for (Map.Entry<String, Integer> dc : datacenters.entrySet())
- dcReplicas.put(dc.getKey(), new HashSet<InetAddress>(dc.getValue()));
+ dcReplicas.put(dc.getKey(), new HashSet<InetAddressAndPort>(dc.getValue()));
Topology topology = tokenMetadata.getTopology();
// all endpoints in each DC, so we can check when we have exhausted all the members of a DC
- Multimap<String, InetAddress> allEndpoints = topology.getDatacenterEndpoints();
+ Multimap<String, InetAddressAndPort> allEndpoints = topology.getDatacenterEndpoints();
// all racks in a DC so we can check when we have exhausted all racks in a DC
- Map<String, ImmutableMultimap<String, InetAddress>> racks = topology.getDatacenterRacks();
+ Map<String, ImmutableMultimap<String, InetAddressAndPort>> racks = topology.getDatacenterRacks();
assert !allEndpoints.isEmpty() && !racks.isEmpty() : "not aware of any cluster members";
// tracks the racks we have already placed replicas in
@@ -307,15 +317,15 @@
// tracks the endpoints that we skipped over while looking for unique racks
// when we relax the rack uniqueness we can append this to the current result so we don't have to wind back the iterator
- Map<String, Set<InetAddress>> skippedDcEndpoints = new HashMap<>(datacenters.size());
+ Map<String, Set<InetAddressAndPort>> skippedDcEndpoints = new HashMap<>(datacenters.size());
for (Map.Entry<String, Integer> dc : datacenters.entrySet())
- skippedDcEndpoints.put(dc.getKey(), new LinkedHashSet<InetAddress>());
+ skippedDcEndpoints.put(dc.getKey(), new LinkedHashSet<InetAddressAndPort>());
Iterator<Token> tokenIter = TokenMetadata.ringIterator(tokenMetadata.sortedTokens(), searchToken, false);
while (tokenIter.hasNext() && !hasSufficientReplicas(dcReplicas, allEndpoints, datacenters))
{
Token next = tokenIter.next();
- InetAddress ep = tokenMetadata.getEndpoint(next);
+ InetAddressAndPort ep = tokenMetadata.getEndpoint(next);
String dc = snitch.getDatacenter(ep);
// have we already found all replicas for this dc?
if (!datacenters.containsKey(dc) || hasSufficientReplicas(dc, dcReplicas, allEndpoints, datacenters))
@@ -342,10 +352,10 @@
// if we've run out of distinct racks, add the hosts we skipped past already (up to RF)
if (seenRacks.get(dc).size() == racks.get(dc).keySet().size())
{
- Iterator<InetAddress> skippedIt = skippedDcEndpoints.get(dc).iterator();
+ Iterator<InetAddressAndPort> skippedIt = skippedDcEndpoints.get(dc).iterator();
while (skippedIt.hasNext() && !hasSufficientReplicas(dc, dcReplicas, allEndpoints, datacenters))
{
- InetAddress nextSkipped = skippedIt.next();
+ InetAddressAndPort nextSkipped = skippedIt.next();
dcReplicas.get(dc).add(nextSkipped);
replicas.add(nextSkipped);
}
@@ -354,15 +364,15 @@
}
}
- return new ArrayList<InetAddress>(replicas);
+ return new ArrayList<InetAddressAndPort>(replicas);
}
- private static boolean hasSufficientReplicas(String dc, Map<String, Set<InetAddress>> dcReplicas, Multimap<String, InetAddress> allEndpoints, Map<String, Integer> datacenters)
+ private static boolean hasSufficientReplicas(String dc, Map<String, Set<InetAddressAndPort>> dcReplicas, Multimap<String, InetAddressAndPort> allEndpoints, Map<String, Integer> datacenters)
{
return dcReplicas.get(dc).size() >= Math.min(allEndpoints.get(dc).size(), getReplicationFactor(dc, datacenters));
}
- private static boolean hasSufficientReplicas(Map<String, Set<InetAddress>> dcReplicas, Multimap<String, InetAddress> allEndpoints, Map<String, Integer> datacenters)
+ private static boolean hasSufficientReplicas(Map<String, Set<InetAddressAndPort>> dcReplicas, Multimap<String, InetAddressAndPort> allEndpoints, Map<String, Integer> datacenters)
{
for (String dc : datacenters.keySet())
if (!hasSufficientReplicas(dc, dcReplicas, allEndpoints, datacenters))
@@ -375,4 +385,50 @@
Integer replicas = datacenters.get(dc);
return replicas == null ? 0 : replicas;
}
+
+ private static Token tk(long t)
+ {
+ return new LongToken(t);
+ }
+
+ private static Range<Token> range(long l, long r)
+ {
+ return new Range<>(tk(l), tk(r));
+ }
+
+ @Test
+ public void testTransientReplica() throws Exception
+ {
+ IEndpointSnitch snitch = new SimpleSnitch();
+ DatabaseDescriptor.setEndpointSnitch(snitch);
+
+ List<InetAddressAndPort> endpoints = Lists.newArrayList(InetAddressAndPort.getByName("127.0.0.1"),
+ InetAddressAndPort.getByName("127.0.0.2"),
+ InetAddressAndPort.getByName("127.0.0.3"),
+ InetAddressAndPort.getByName("127.0.0.4"));
+
+ Multimap<InetAddressAndPort, Token> tokens = HashMultimap.create();
+ tokens.put(endpoints.get(0), tk(100));
+ tokens.put(endpoints.get(1), tk(200));
+ tokens.put(endpoints.get(2), tk(300));
+ tokens.put(endpoints.get(3), tk(400));
+ TokenMetadata metadata = new TokenMetadata();
+ metadata.updateNormalTokens(tokens);
+
+ Map<String, String> configOptions = new HashMap<String, String>();
+ configOptions.put(snitch.getDatacenter((InetAddressAndPort) null), "3/1");
+
+ NetworkTopologyStrategy strategy = new NetworkTopologyStrategy(keyspaceName, metadata, snitch, configOptions);
+
+ Util.assertRCEquals(EndpointsForRange.of(fullReplica(endpoints.get(0), range(400, 100)),
+ fullReplica(endpoints.get(1), range(400, 100)),
+ transientReplica(endpoints.get(2), range(400, 100))),
+ strategy.getNaturalReplicasForToken(tk(99)));
+
+
+ Util.assertRCEquals(EndpointsForRange.of(fullReplica(endpoints.get(1), range(100, 200)),
+ fullReplica(endpoints.get(2), range(100, 200)),
+ transientReplica(endpoints.get(3), range(100, 200))),
+ strategy.getNaturalReplicasForToken(tk(101)));
+ }
}
diff --git a/test/unit/org/apache/cassandra/locator/OldNetworkTopologyStrategyTest.java b/test/unit/org/apache/cassandra/locator/OldNetworkTopologyStrategyTest.java
deleted file mode 100644
index e6e17cd..0000000
--- a/test/unit/org/apache/cassandra/locator/OldNetworkTopologyStrategyTest.java
+++ /dev/null
@@ -1,382 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.locator;
-
-import java.net.InetAddress;
-import java.net.UnknownHostException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.dht.RandomPartitioner.BigIntegerToken;
-import org.apache.cassandra.dht.Range;
-import org.apache.cassandra.dht.Token;
-import org.apache.cassandra.service.StorageService;
-import org.apache.cassandra.utils.Pair;
-
-import static org.junit.Assert.assertEquals;
-
-public class OldNetworkTopologyStrategyTest
-{
- private List<Token> keyTokens;
- private TokenMetadata tmd;
- private Map<String, ArrayList<InetAddress>> expectedResults;
-
- @BeforeClass
- public static void setupDD()
- {
- DatabaseDescriptor.daemonInitialization();
- }
-
- @Before
- public void init()
- {
- keyTokens = new ArrayList<Token>();
- tmd = new TokenMetadata();
- expectedResults = new HashMap<String, ArrayList<InetAddress>>();
- }
-
- /**
- * 4 same rack endpoints
- *
- * @throws java.net.UnknownHostException
- */
- @Test
- public void testBigIntegerEndpointsA() throws UnknownHostException
- {
- RackInferringSnitch endpointSnitch = new RackInferringSnitch();
-
- AbstractReplicationStrategy strategy = new OldNetworkTopologyStrategy("Keyspace1", tmd, endpointSnitch, optsWithRF(1));
- addEndpoint("0", "5", "254.0.0.1");
- addEndpoint("10", "15", "254.0.0.2");
- addEndpoint("20", "25", "254.0.0.3");
- addEndpoint("30", "35", "254.0.0.4");
-
- expectedResults.put("5", buildResult("254.0.0.2", "254.0.0.3", "254.0.0.4"));
- expectedResults.put("15", buildResult("254.0.0.3", "254.0.0.4", "254.0.0.1"));
- expectedResults.put("25", buildResult("254.0.0.4", "254.0.0.1", "254.0.0.2"));
- expectedResults.put("35", buildResult("254.0.0.1", "254.0.0.2", "254.0.0.3"));
-
- testGetEndpoints(strategy, keyTokens.toArray(new Token[0]));
- }
-
- /**
- * 3 same rack endpoints
- * 1 external datacenter
- *
- * @throws java.net.UnknownHostException
- */
- @Test
- public void testBigIntegerEndpointsB() throws UnknownHostException
- {
- RackInferringSnitch endpointSnitch = new RackInferringSnitch();
-
- AbstractReplicationStrategy strategy = new OldNetworkTopologyStrategy("Keyspace1", tmd, endpointSnitch, optsWithRF(1));
- addEndpoint("0", "5", "254.0.0.1");
- addEndpoint("10", "15", "254.0.0.2");
- addEndpoint("20", "25", "254.1.0.3");
- addEndpoint("30", "35", "254.0.0.4");
-
- expectedResults.put("5", buildResult("254.0.0.2", "254.1.0.3", "254.0.0.4"));
- expectedResults.put("15", buildResult("254.1.0.3", "254.0.0.4", "254.0.0.1"));
- expectedResults.put("25", buildResult("254.0.0.4", "254.1.0.3", "254.0.0.1"));
- expectedResults.put("35", buildResult("254.0.0.1", "254.1.0.3", "254.0.0.2"));
-
- testGetEndpoints(strategy, keyTokens.toArray(new Token[0]));
- }
-
- /**
- * 2 same rack endpoints
- * 1 same datacenter, different rack endpoints
- * 1 external datacenter
- *
- * @throws java.net.UnknownHostException
- */
- @Test
- public void testBigIntegerEndpointsC() throws UnknownHostException
- {
- RackInferringSnitch endpointSnitch = new RackInferringSnitch();
-
- AbstractReplicationStrategy strategy = new OldNetworkTopologyStrategy("Keyspace1", tmd, endpointSnitch, optsWithRF(1));
- addEndpoint("0", "5", "254.0.0.1");
- addEndpoint("10", "15", "254.0.0.2");
- addEndpoint("20", "25", "254.0.1.3");
- addEndpoint("30", "35", "254.1.0.4");
-
- expectedResults.put("5", buildResult("254.0.0.2", "254.0.1.3", "254.1.0.4"));
- expectedResults.put("15", buildResult("254.0.1.3", "254.1.0.4", "254.0.0.1"));
- expectedResults.put("25", buildResult("254.1.0.4", "254.0.0.1", "254.0.0.2"));
- expectedResults.put("35", buildResult("254.0.0.1", "254.0.1.3", "254.1.0.4"));
-
- testGetEndpoints(strategy, keyTokens.toArray(new Token[0]));
- }
-
- private ArrayList<InetAddress> buildResult(String... addresses) throws UnknownHostException
- {
- ArrayList<InetAddress> result = new ArrayList<InetAddress>();
- for (String address : addresses)
- {
- result.add(InetAddress.getByName(address));
- }
- return result;
- }
-
- private void addEndpoint(String endpointTokenID, String keyTokenID, String endpointAddress) throws UnknownHostException
- {
- BigIntegerToken endpointToken = new BigIntegerToken(endpointTokenID);
-
- BigIntegerToken keyToken = new BigIntegerToken(keyTokenID);
- keyTokens.add(keyToken);
-
- InetAddress ep = InetAddress.getByName(endpointAddress);
- tmd.updateNormalToken(endpointToken, ep);
- }
-
- private void testGetEndpoints(AbstractReplicationStrategy strategy, Token[] keyTokens)
- {
- for (Token keyToken : keyTokens)
- {
- List<InetAddress> endpoints = strategy.getNaturalEndpoints(keyToken);
- for (int j = 0; j < endpoints.size(); j++)
- {
- ArrayList<InetAddress> hostsExpected = expectedResults.get(keyToken.toString());
- assertEquals(endpoints.get(j), hostsExpected.get(j));
- }
- }
- }
-
- /**
- * test basic methods to move a node. For sure, it's not the best place, but it's easy to test
- *
- * @throws java.net.UnknownHostException
- */
- @Test
- public void testMoveLeft() throws UnknownHostException
- {
- // Moves to the left : nothing to fetch, last part to stream
-
- int movingNodeIdx = 1;
- BigIntegerToken newToken = new BigIntegerToken("21267647932558653966460912964485513216");
- BigIntegerToken[] tokens = initTokens();
- BigIntegerToken[] tokensAfterMove = initTokensAfterMove(tokens, movingNodeIdx, newToken);
- Pair<Set<Range<Token>>, Set<Range<Token>>> ranges = calculateStreamAndFetchRanges(tokens, tokensAfterMove, movingNodeIdx);
-
- assertEquals(ranges.left.iterator().next().left, tokensAfterMove[movingNodeIdx]);
- assertEquals(ranges.left.iterator().next().right, tokens[movingNodeIdx]);
- assertEquals("No data should be fetched", ranges.right.size(), 0);
-
- }
-
- @Test
- public void testMoveRight() throws UnknownHostException
- {
- // Moves to the right : last part to fetch, nothing to stream
-
- int movingNodeIdx = 1;
- BigIntegerToken newToken = new BigIntegerToken("35267647932558653966460912964485513216");
- BigIntegerToken[] tokens = initTokens();
- BigIntegerToken[] tokensAfterMove = initTokensAfterMove(tokens, movingNodeIdx, newToken);
- Pair<Set<Range<Token>>, Set<Range<Token>>> ranges = calculateStreamAndFetchRanges(tokens, tokensAfterMove, movingNodeIdx);
-
- assertEquals("No data should be streamed", ranges.left.size(), 0);
- assertEquals(ranges.right.iterator().next().left, tokens[movingNodeIdx]);
- assertEquals(ranges.right.iterator().next().right, tokensAfterMove[movingNodeIdx]);
-
- }
-
- @SuppressWarnings("unchecked")
- @Test
- public void testMoveMiddleOfRing() throws UnknownHostException
- {
- // moves to another position in the middle of the ring : should stream all its data, and fetch all its new data
-
- int movingNodeIdx = 1;
- int movingNodeIdxAfterMove = 4;
- BigIntegerToken newToken = new BigIntegerToken("90070591730234615865843651857942052864");
- BigIntegerToken[] tokens = initTokens();
- BigIntegerToken[] tokensAfterMove = initTokensAfterMove(tokens, movingNodeIdx, newToken);
- Pair<Set<Range<Token>>, Set<Range<Token>>> ranges = calculateStreamAndFetchRanges(tokens, tokensAfterMove, movingNodeIdx);
-
- // sort the results, so they can be compared
- Range<Token>[] toStream = ranges.left.toArray(new Range[0]);
- Range<Token>[] toFetch = ranges.right.toArray(new Range[0]);
- Arrays.sort(toStream);
- Arrays.sort(toFetch);
-
- // build expected ranges
- Range<Token>[] toStreamExpected = new Range[2];
- toStreamExpected[0] = new Range<Token>(getToken(movingNodeIdx - 2, tokens), getToken(movingNodeIdx - 1, tokens));
- toStreamExpected[1] = new Range<Token>(getToken(movingNodeIdx - 1, tokens), getToken(movingNodeIdx, tokens));
- Arrays.sort(toStreamExpected);
- Range<Token>[] toFetchExpected = new Range[2];
- toFetchExpected[0] = new Range<Token>(getToken(movingNodeIdxAfterMove - 1, tokens), getToken(movingNodeIdxAfterMove, tokens));
- toFetchExpected[1] = new Range<Token>(getToken(movingNodeIdxAfterMove, tokensAfterMove), getToken(movingNodeIdx, tokensAfterMove));
- Arrays.sort(toFetchExpected);
-
- assertEquals(Arrays.equals(toStream, toStreamExpected), true);
- assertEquals(Arrays.equals(toFetch, toFetchExpected), true);
- }
-
- @SuppressWarnings("unchecked")
- @Test
- public void testMoveAfterNextNeighbors() throws UnknownHostException
- {
- // moves after its next neighbor in the ring
-
- int movingNodeIdx = 1;
- int movingNodeIdxAfterMove = 2;
- BigIntegerToken newToken = new BigIntegerToken("52535295865117307932921825928971026432");
- BigIntegerToken[] tokens = initTokens();
- BigIntegerToken[] tokensAfterMove = initTokensAfterMove(tokens, movingNodeIdx, newToken);
- Pair<Set<Range<Token>>, Set<Range<Token>>> ranges = calculateStreamAndFetchRanges(tokens, tokensAfterMove, movingNodeIdx);
-
-
- // sort the results, so they can be compared
- Range<Token>[] toStream = ranges.left.toArray(new Range[0]);
- Range<Token>[] toFetch = ranges.right.toArray(new Range[0]);
- Arrays.sort(toStream);
- Arrays.sort(toFetch);
-
- // build expected ranges
- Range<Token>[] toStreamExpected = new Range[1];
- toStreamExpected[0] = new Range<Token>(getToken(movingNodeIdx - 2, tokens), getToken(movingNodeIdx - 1, tokens));
- Arrays.sort(toStreamExpected);
- Range<Token>[] toFetchExpected = new Range[2];
- toFetchExpected[0] = new Range<Token>(getToken(movingNodeIdxAfterMove - 1, tokens), getToken(movingNodeIdxAfterMove, tokens));
- toFetchExpected[1] = new Range<Token>(getToken(movingNodeIdxAfterMove, tokensAfterMove), getToken(movingNodeIdx, tokensAfterMove));
- Arrays.sort(toFetchExpected);
-
- assertEquals(Arrays.equals(toStream, toStreamExpected), true);
- assertEquals(Arrays.equals(toFetch, toFetchExpected), true);
- }
-
- @SuppressWarnings("unchecked")
- @Test
- public void testMoveBeforePreviousNeighbor() throws UnknownHostException
- {
- // moves before its previous neighbor in the ring
-
- int movingNodeIdx = 1;
- int movingNodeIdxAfterMove = 7;
- BigIntegerToken newToken = new BigIntegerToken("158873535527910577765226390751398592512");
- BigIntegerToken[] tokens = initTokens();
- BigIntegerToken[] tokensAfterMove = initTokensAfterMove(tokens, movingNodeIdx, newToken);
- Pair<Set<Range<Token>>, Set<Range<Token>>> ranges = calculateStreamAndFetchRanges(tokens, tokensAfterMove, movingNodeIdx);
-
- Range<Token>[] toStream = ranges.left.toArray(new Range[0]);
- Range<Token>[] toFetch = ranges.right.toArray(new Range[0]);
- Arrays.sort(toStream);
- Arrays.sort(toFetch);
-
- Range<Token>[] toStreamExpected = new Range[2];
- toStreamExpected[0] = new Range<Token>(getToken(movingNodeIdx, tokensAfterMove), getToken(movingNodeIdx - 1, tokensAfterMove));
- toStreamExpected[1] = new Range<Token>(getToken(movingNodeIdx - 1, tokens), getToken(movingNodeIdx, tokens));
- Arrays.sort(toStreamExpected);
- Range<Token>[] toFetchExpected = new Range[1];
- toFetchExpected[0] = new Range<Token>(getToken(movingNodeIdxAfterMove - 1, tokens), getToken(movingNodeIdxAfterMove, tokens));
- Arrays.sort(toFetchExpected);
-
- System.out.println("toStream : " + Arrays.toString(toStream));
- System.out.println("toFetch : " + Arrays.toString(toFetch));
- System.out.println("toStreamExpected : " + Arrays.toString(toStreamExpected));
- System.out.println("toFetchExpected : " + Arrays.toString(toFetchExpected));
-
- assertEquals(Arrays.equals(toStream, toStreamExpected), true);
- assertEquals(Arrays.equals(toFetch, toFetchExpected), true);
- }
-
- private BigIntegerToken[] initTokensAfterMove(BigIntegerToken[] tokens,
- int movingNodeIdx, BigIntegerToken newToken)
- {
- BigIntegerToken[] tokensAfterMove = tokens.clone();
- tokensAfterMove[movingNodeIdx] = newToken;
- return tokensAfterMove;
- }
-
- private BigIntegerToken[] initTokens()
- {
- BigIntegerToken[] tokens = new BigIntegerToken[] {
- new BigIntegerToken("0"), // just to be able to test
- new BigIntegerToken("34028236692093846346337460743176821145"),
- new BigIntegerToken("42535295865117307932921825928971026432"),
- new BigIntegerToken("63802943797675961899382738893456539648"),
- new BigIntegerToken("85070591730234615865843651857942052864"),
- new BigIntegerToken("106338239662793269832304564822427566080"),
- new BigIntegerToken("127605887595351923798765477786913079296"),
- new BigIntegerToken("148873535527910577765226390751398592512")
- };
- return tokens;
- }
-
- private TokenMetadata initTokenMetadata(BigIntegerToken[] tokens)
- throws UnknownHostException
- {
- TokenMetadata tokenMetadataCurrent = new TokenMetadata();
-
- int lastIPPart = 1;
- for (BigIntegerToken token : tokens)
- tokenMetadataCurrent.updateNormalToken(token, InetAddress.getByName("254.0.0." + Integer.toString(lastIPPart++)));
-
- return tokenMetadataCurrent;
- }
-
- private BigIntegerToken getToken(int idx, BigIntegerToken[] tokens)
- {
- if (idx >= tokens.length)
- idx = idx % tokens.length;
- while (idx < 0)
- idx += tokens.length;
-
- return tokens[idx];
-
- }
-
- private Pair<Set<Range<Token>>, Set<Range<Token>>> calculateStreamAndFetchRanges(BigIntegerToken[] tokens, BigIntegerToken[] tokensAfterMove, int movingNodeIdx) throws UnknownHostException
- {
- RackInferringSnitch endpointSnitch = new RackInferringSnitch();
-
- InetAddress movingNode = InetAddress.getByName("254.0.0." + Integer.toString(movingNodeIdx + 1));
-
-
- TokenMetadata tokenMetadataCurrent = initTokenMetadata(tokens);
- TokenMetadata tokenMetadataAfterMove = initTokenMetadata(tokensAfterMove);
- AbstractReplicationStrategy strategy = new OldNetworkTopologyStrategy("Keyspace1", tokenMetadataCurrent, endpointSnitch, optsWithRF(2));
-
- Collection<Range<Token>> currentRanges = strategy.getAddressRanges().get(movingNode);
- Collection<Range<Token>> updatedRanges = strategy.getPendingAddressRanges(tokenMetadataAfterMove, tokensAfterMove[movingNodeIdx], movingNode);
-
- Pair<Set<Range<Token>>, Set<Range<Token>>> ranges = StorageService.instance.calculateStreamAndFetchRanges(currentRanges, updatedRanges);
-
- return ranges;
- }
-
- private static Map<String, String> optsWithRF(int rf)
- {
- return Collections.singletonMap("replication_factor", Integer.toString(rf));
- }
-}
diff --git a/test/unit/org/apache/cassandra/locator/PendingRangeMapsTest.java b/test/unit/org/apache/cassandra/locator/PendingRangeMapsTest.java
index 7121550..8e0bc00 100644
--- a/test/unit/org/apache/cassandra/locator/PendingRangeMapsTest.java
+++ b/test/unit/org/apache/cassandra/locator/PendingRangeMapsTest.java
@@ -25,9 +25,7 @@
import org.apache.cassandra.dht.Token;
import org.junit.Test;
-import java.net.InetAddress;
import java.net.UnknownHostException;
-import java.util.Collection;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@@ -39,17 +37,29 @@
return new Range<Token>(new BigIntegerToken(left), new BigIntegerToken(right));
}
+ private static void addPendingRange(PendingRangeMaps pendingRangeMaps, Range<Token> range, String endpoint)
+ {
+ try
+ {
+ pendingRangeMaps.addPendingRange(range, Replica.fullReplica(InetAddressAndPort.getByName(endpoint), range));
+ }
+ catch (UnknownHostException e)
+ {
+ throw new AssertionError(e);
+ }
+ }
+
@Test
public void testPendingEndpoints() throws UnknownHostException
{
PendingRangeMaps pendingRangeMaps = new PendingRangeMaps();
- pendingRangeMaps.addPendingRange(genRange("5", "15"), InetAddress.getByName("127.0.0.1"));
- pendingRangeMaps.addPendingRange(genRange("15", "25"), InetAddress.getByName("127.0.0.2"));
- pendingRangeMaps.addPendingRange(genRange("25", "35"), InetAddress.getByName("127.0.0.3"));
- pendingRangeMaps.addPendingRange(genRange("35", "45"), InetAddress.getByName("127.0.0.4"));
- pendingRangeMaps.addPendingRange(genRange("45", "55"), InetAddress.getByName("127.0.0.5"));
- pendingRangeMaps.addPendingRange(genRange("45", "65"), InetAddress.getByName("127.0.0.6"));
+ addPendingRange(pendingRangeMaps, genRange("5", "15"), "127.0.0.1");
+ addPendingRange(pendingRangeMaps, genRange("15", "25"), "127.0.0.2");
+ addPendingRange(pendingRangeMaps, genRange("25", "35"), "127.0.0.3");
+ addPendingRange(pendingRangeMaps, genRange("35", "45"), "127.0.0.4");
+ addPendingRange(pendingRangeMaps, genRange("45", "55"), "127.0.0.5");
+ addPendingRange(pendingRangeMaps, genRange("45", "65"), "127.0.0.6");
assertEquals(0, pendingRangeMaps.pendingEndpointsFor(new BigIntegerToken("0")).size());
assertEquals(0, pendingRangeMaps.pendingEndpointsFor(new BigIntegerToken("5")).size());
@@ -62,8 +72,8 @@
assertEquals(2, pendingRangeMaps.pendingEndpointsFor(new BigIntegerToken("55")).size());
assertEquals(1, pendingRangeMaps.pendingEndpointsFor(new BigIntegerToken("65")).size());
- Collection<InetAddress> endpoints = pendingRangeMaps.pendingEndpointsFor(new BigIntegerToken("15"));
- assertTrue(endpoints.contains(InetAddress.getByName("127.0.0.1")));
+ EndpointsForToken replicas = pendingRangeMaps.pendingEndpointsFor(new BigIntegerToken("15"));
+ assertTrue(replicas.endpoints().contains(InetAddressAndPort.getByName("127.0.0.1")));
}
@Test
@@ -71,13 +81,13 @@
{
PendingRangeMaps pendingRangeMaps = new PendingRangeMaps();
- pendingRangeMaps.addPendingRange(genRange("5", "15"), InetAddress.getByName("127.0.0.1"));
- pendingRangeMaps.addPendingRange(genRange("15", "25"), InetAddress.getByName("127.0.0.2"));
- pendingRangeMaps.addPendingRange(genRange("25", "35"), InetAddress.getByName("127.0.0.3"));
- pendingRangeMaps.addPendingRange(genRange("35", "45"), InetAddress.getByName("127.0.0.4"));
- pendingRangeMaps.addPendingRange(genRange("45", "55"), InetAddress.getByName("127.0.0.5"));
- pendingRangeMaps.addPendingRange(genRange("45", "65"), InetAddress.getByName("127.0.0.6"));
- pendingRangeMaps.addPendingRange(genRange("65", "7"), InetAddress.getByName("127.0.0.7"));
+ addPendingRange(pendingRangeMaps, genRange("5", "15"), "127.0.0.1");
+ addPendingRange(pendingRangeMaps, genRange("15", "25"), "127.0.0.2");
+ addPendingRange(pendingRangeMaps, genRange("25", "35"), "127.0.0.3");
+ addPendingRange(pendingRangeMaps, genRange("35", "45"), "127.0.0.4");
+ addPendingRange(pendingRangeMaps, genRange("45", "55"), "127.0.0.5");
+ addPendingRange(pendingRangeMaps, genRange("45", "65"), "127.0.0.6");
+ addPendingRange(pendingRangeMaps, genRange("65", "7"), "127.0.0.7");
assertEquals(1, pendingRangeMaps.pendingEndpointsFor(new BigIntegerToken("0")).size());
assertEquals(1, pendingRangeMaps.pendingEndpointsFor(new BigIntegerToken("5")).size());
@@ -91,8 +101,8 @@
assertEquals(2, pendingRangeMaps.pendingEndpointsFor(new BigIntegerToken("55")).size());
assertEquals(1, pendingRangeMaps.pendingEndpointsFor(new BigIntegerToken("65")).size());
- Collection<InetAddress> endpoints = pendingRangeMaps.pendingEndpointsFor(new BigIntegerToken("6"));
- assertTrue(endpoints.contains(InetAddress.getByName("127.0.0.1")));
- assertTrue(endpoints.contains(InetAddress.getByName("127.0.0.7")));
+ EndpointsForToken replicas = pendingRangeMaps.pendingEndpointsFor(new BigIntegerToken("6"));
+ assertTrue(replicas.endpoints().contains(InetAddressAndPort.getByName("127.0.0.1")));
+ assertTrue(replicas.endpoints().contains(InetAddressAndPort.getByName("127.0.0.7")));
}
}
diff --git a/test/unit/org/apache/cassandra/locator/PendingRangesTest.java b/test/unit/org/apache/cassandra/locator/PendingRangesTest.java
new file mode 100644
index 0000000..48bf546
--- /dev/null
+++ b/test/unit/org/apache/cassandra/locator/PendingRangesTest.java
@@ -0,0 +1,260 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.locator;
+
+import java.net.UnknownHostException;
+import java.util.Collections;
+
+import com.google.common.collect.*;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.dht.Token;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class PendingRangesTest
+{
+ private static final String RACK1 = "RACK1";
+ private static final String DC1 = "DC1";
+ private static final String KEYSPACE = "ks";
+ private static final InetAddressAndPort PEER1 = peer(1);
+ private static final InetAddressAndPort PEER2 = peer(2);
+ private static final InetAddressAndPort PEER3 = peer(3);
+ private static final InetAddressAndPort PEER4 = peer(4);
+ private static final InetAddressAndPort PEER5 = peer(5);
+ private static final InetAddressAndPort PEER6 = peer(6);
+
+ private static final InetAddressAndPort PEER1A = peer(11);
+ private static final InetAddressAndPort PEER4A = peer(14);
+
+ private static final Token TOKEN1 = token(0);
+ private static final Token TOKEN2 = token(10);
+ private static final Token TOKEN3 = token(20);
+ private static final Token TOKEN4 = token(30);
+ private static final Token TOKEN5 = token(40);
+ private static final Token TOKEN6 = token(50);
+
+ @BeforeClass
+ public static void beforeClass() throws Throwable
+ {
+ DatabaseDescriptor.daemonInitialization();
+ IEndpointSnitch snitch = snitch();
+ DatabaseDescriptor.setEndpointSnitch(snitch);
+ }
+
+ @Test
+ public void calculatePendingRangesForConcurrentReplacements()
+ {
+ /*
+ * As described in CASSANDRA-14802, concurrent range movements can generate pending ranges
+ * which are far larger than strictly required, which in turn can impact availability.
+ *
+ * In the narrow case of straight replacement, the pending ranges should mirror the owned ranges
+ * of the nodes being replaced.
+ *
+ * Note: the following example is purely illustrative as the iteration order for processing
+ * bootstrapping endpoints is not guaranteed. Because of this, precisely which endpoints' pending
+ * ranges are correct/incorrect depends on the specifics of the ring. Concretely, the bootstrap tokens
+ * are ultimately backed by a HashMap, so iteration of bootstrapping nodes is based on the hashcodes
+ * of the endpoints.
+ *
+ * E.g. a 6 node cluster with tokens:
+ *
+ * nodeA : 0
+ * nodeB : 10
+ * nodeC : 20
+ * nodeD : 30
+ * nodeE : 40
+ * nodeF : 50
+ *
+ * with an RF of 3, this gives an initial ring of :
+ *
+ * nodeA : (50, 0], (40, 50], (30, 40]
+ * nodeB : (0, 10], (50, 0], (40, 50]
+ * nodeC : (10, 20], (0, 10], (50, 0]
+ * nodeD : (20, 30], (10, 20], (0, 10]
+ * nodeE : (30, 40], (20, 30], (10, 20]
+ * nodeF : (40, 50], (30, 40], (20, 30]
+ *
+ * If nodeA is replaced by node1A, then the pending ranges map should be:
+ * {
+ * (50, 0] : [node1A],
+ * (40, 50] : [node1A],
+ * (30, 40] : [node1A]
+ * }
+ *
+ * Starting a second concurrent replacement of a node with non-overlapping ranges
+ * (i.e. node4 for node4A) should result in a pending range map of:
+ * {
+ * (50, 0] : [node1A],
+ * (40, 50] : [node1A],
+ * (30, 40] : [node1A],
+ * (20, 30] : [node4A],
+ * (10, 20] : [node4A],
+ * (0, 10] : [node4A]
+ * }
+ *
+ * But, the bug in CASSANDRA-14802 causes it to be:
+ * {
+ * (50, 0] : [node1A],
+ * (40, 50] : [node1A],
+ * (30, 40] : [node1A],
+ * (20, 30] : [node4A],
+ * (10, 20] : [node4A],
+ * (50, 10] : [node4A]
+ * }
+ *
+ * so node4A incorrectly becomes a pending endpoint for an additional sub-range: (50, 0).
+ *
+ */
+ TokenMetadata tm = new TokenMetadata();
+ AbstractReplicationStrategy replicationStrategy = simpleStrategy(tm, 3);
+
+ // setup initial ring
+ addNode(tm, PEER1, TOKEN1);
+ addNode(tm, PEER2, TOKEN2);
+ addNode(tm, PEER3, TOKEN3);
+ addNode(tm, PEER4, TOKEN4);
+ addNode(tm, PEER5, TOKEN5);
+ addNode(tm, PEER6, TOKEN6);
+
+ // no pending ranges before any replacements
+ tm.calculatePendingRanges(replicationStrategy, KEYSPACE);
+ assertEquals(0, Iterators.size(tm.getPendingRanges(KEYSPACE).iterator()));
+
+ // Ranges initially owned by PEER1 and PEER4
+ RangesAtEndpoint peer1Ranges = replicationStrategy.getAddressReplicas(tm).get(PEER1);
+ RangesAtEndpoint peer4Ranges = replicationStrategy.getAddressReplicas(tm).get(PEER4);
+ // Replace PEER1 with PEER1A
+ replace(PEER1, PEER1A, TOKEN1, tm, replicationStrategy);
+ // The only pending ranges should be the ones previously belonging to PEER1
+ // and these should have a single pending endpoint, PEER1A
+ RangesByEndpoint.Builder b1 = new RangesByEndpoint.Builder();
+ peer1Ranges.iterator().forEachRemaining(replica -> b1.put(PEER1A, new Replica(PEER1A, replica.range(), replica.isFull())));
+ RangesByEndpoint expected = b1.build();
+ assertPendingRanges(tm.getPendingRanges(KEYSPACE), expected);
+ // Also verify the Multimap variant of getPendingRanges
+ assertPendingRanges(tm.getPendingRangesMM(KEYSPACE), expected);
+
+ // Replace PEER4 with PEER4A
+ replace(PEER4, PEER4A, TOKEN4, tm, replicationStrategy);
+ // Pending ranges should now include the ranges originally belonging
+ // to PEER1 (now pending for PEER1A) and the ranges originally belonging to PEER4
+ // (now pending for PEER4A).
+ RangesByEndpoint.Builder b2 = new RangesByEndpoint.Builder();
+ peer1Ranges.iterator().forEachRemaining(replica -> b2.put(PEER1A, new Replica(PEER1A, replica.range(), replica.isFull())));
+ peer4Ranges.iterator().forEachRemaining(replica -> b2.put(PEER4A, new Replica(PEER4A, replica.range(), replica.isFull())));
+ expected = b2.build();
+ assertPendingRanges(tm.getPendingRanges(KEYSPACE), expected);
+ assertPendingRanges(tm.getPendingRangesMM(KEYSPACE), expected);
+ }
+
+
+ private void assertPendingRanges(PendingRangeMaps pending, RangesByEndpoint expected)
+ {
+ RangesByEndpoint.Builder actual = new RangesByEndpoint.Builder();
+ pending.iterator().forEachRemaining(pendingRange -> {
+ Replica replica = Iterators.getOnlyElement(pendingRange.getValue().iterator());
+ actual.put(replica.endpoint(), replica);
+ });
+ assertRangesByEndpoint(expected, actual.build());
+ }
+
+ private void assertPendingRanges(EndpointsByRange pending, RangesByEndpoint expected)
+ {
+ RangesByEndpoint.Builder actual = new RangesByEndpoint.Builder();
+ pending.flattenEntries().forEach(entry -> actual.put(entry.getValue().endpoint(), entry.getValue()));
+ assertRangesByEndpoint(expected, actual.build());
+ }
+
+
+ private void assertRangesByEndpoint(RangesByEndpoint expected, RangesByEndpoint actual)
+ {
+ assertEquals(expected.keySet(), actual.keySet());
+ for (InetAddressAndPort endpoint : expected.keySet())
+ {
+ RangesAtEndpoint expectedReplicas = expected.get(endpoint);
+ RangesAtEndpoint actualReplicas = actual.get(endpoint);
+ assertEquals(expectedReplicas.size(), actualReplicas.size());
+ assertTrue(Iterables.all(expectedReplicas, actualReplicas::contains));
+ }
+ }
+
+ private void addNode(TokenMetadata tm, InetAddressAndPort replica, Token token)
+ {
+ tm.updateNormalTokens(Collections.singleton(token), replica);
+ }
+
+ private void replace(InetAddressAndPort toReplace,
+ InetAddressAndPort replacement,
+ Token token,
+ TokenMetadata tm,
+ AbstractReplicationStrategy replicationStrategy)
+ {
+ assertEquals(toReplace, tm.getEndpoint(token));
+ tm.addReplaceTokens(Collections.singleton(token), replacement, toReplace);
+ tm.calculatePendingRanges(replicationStrategy, KEYSPACE);
+ }
+
+ private static Token token(long token)
+ {
+ return Murmur3Partitioner.instance.getTokenFactory().fromString(Long.toString(token));
+ }
+
+ private static InetAddressAndPort peer(int addressSuffix)
+ {
+ try
+ {
+ return InetAddressAndPort.getByAddress(new byte[]{ 127, 0, 0, (byte) addressSuffix});
+ }
+ catch (UnknownHostException e)
+ {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private static IEndpointSnitch snitch()
+ {
+ return new AbstractNetworkTopologySnitch()
+ {
+ public String getRack(InetAddressAndPort endpoint)
+ {
+ return RACK1;
+ }
+
+ public String getDatacenter(InetAddressAndPort endpoint)
+ {
+ return DC1;
+ }
+ };
+ }
+
+ private static AbstractReplicationStrategy simpleStrategy(TokenMetadata tokenMetadata, int replicationFactor)
+ {
+ return new SimpleStrategy(KEYSPACE,
+ tokenMetadata,
+ DatabaseDescriptor.getEndpointSnitch(),
+ Collections.singletonMap("replication_factor", Integer.toString(replicationFactor)));
+ }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/locator/PropertyFileSnitchTest.java b/test/unit/org/apache/cassandra/locator/PropertyFileSnitchTest.java
index e756902..bd8f886 100644
--- a/test/unit/org/apache/cassandra/locator/PropertyFileSnitchTest.java
+++ b/test/unit/org/apache/cassandra/locator/PropertyFileSnitchTest.java
@@ -18,7 +18,7 @@
package org.apache.cassandra.locator;
import java.io.IOException;
-import java.net.InetAddress;
+import java.net.UnknownHostException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
@@ -32,8 +32,7 @@
import java.util.Map;
import java.util.Set;
import java.util.UUID;
-
-import com.google.common.net.InetAddresses;
+import java.util.regex.Matcher;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.dht.IPartitioner;
@@ -61,7 +60,7 @@
private Path backupFile;
private VersionedValue.VersionedValueFactory valueFactory;
- private Map<InetAddress, Set<Token>> tokenMap;
+ private Map<InetAddressAndPort, Set<Token>> tokenMap;
@BeforeClass
public static void setupDD()
@@ -79,17 +78,17 @@
restoreOrigConfigFile();
- InetAddress[] hosts = {
- InetAddress.getByName("127.0.0.1"), // this exists in the config file
- InetAddress.getByName("127.0.0.2"), // this exists in the config file
- InetAddress.getByName("127.0.0.9"), // this does not exist in the config file
+ InetAddressAndPort[] hosts = {
+ InetAddressAndPort.getByName("127.0.0.1"), // this exists in the config file
+ InetAddressAndPort.getByName("127.0.0.2"), // this exists in the config file
+ InetAddressAndPort.getByName("127.0.0.9"), // this does not exist in the config file
};
IPartitioner partitioner = new RandomPartitioner();
valueFactory = new VersionedValue.VersionedValueFactory(partitioner);
tokenMap = new HashMap<>();
- for (InetAddress host : hosts)
+ for (InetAddressAndPort host : hosts)
{
Set<Token> tokens = Collections.singleton(partitioner.getRandomToken());
Gossiper.instance.initializeNodeUnsafe(host, UUID.randomUUID(), 1);
@@ -118,13 +117,21 @@
for (String line : lines)
{
String[] info = line.split("=");
- if (info.length == 2 && replacements.containsKey(info[0]))
+ if (info.length == 2 && !line.startsWith("#") && !line.startsWith("default="))
{
- String replacement = replacements.get(info[0]);
- if (!replacement.isEmpty()) // empty means remove this line
- newLines.add(info[0] + '=' + replacement);
+ InetAddressAndPort address = InetAddressAndPort.getByName(info[0].replaceAll(Matcher.quoteReplacement("\\:"), ":"));
+ String replacement = replacements.get(address.toString());
+ if (replacement != null)
+ {
+ if (!replacement.isEmpty()) // empty means remove this line
+ newLines.add(info[0] + '=' + replacement);
- replaced.add(info[0]);
+ replaced.add(address.toString());
+ }
+ else
+ {
+ newLines.add(line);
+ }
}
else
{
@@ -139,21 +146,26 @@
continue;
if (!replacement.getValue().isEmpty()) // empty means remove this line so do nothing here
- newLines.add(replacement.getKey() + '=' + replacement.getValue());
+ {
+ String escaped = replacement.getKey().replaceAll(Matcher.quoteReplacement(":"), "\\\\:");
+ newLines.add(escaped + '=' + replacement.getValue());
+ }
}
Files.write(effectiveFile, newLines, StandardCharsets.UTF_8, StandardOpenOption.TRUNCATE_EXISTING);
}
- private void setNodeShutdown(InetAddress host)
+ private void setNodeShutdown(InetAddressAndPort host)
{
StorageService.instance.getTokenMetadata().removeEndpoint(host);
+ Gossiper.instance.injectApplicationState(host, ApplicationState.STATUS_WITH_PORT, valueFactory.shutdown(true));
Gossiper.instance.injectApplicationState(host, ApplicationState.STATUS, valueFactory.shutdown(true));
Gossiper.instance.markDead(host, Gossiper.instance.getEndpointStateForEndpoint(host));
}
- private void setNodeLive(InetAddress host)
+ private void setNodeLive(InetAddressAndPort host)
{
+ Gossiper.instance.injectApplicationState(host, ApplicationState.STATUS_WITH_PORT, valueFactory.normal(tokenMap.get(host)));
Gossiper.instance.injectApplicationState(host, ApplicationState.STATUS, valueFactory.normal(tokenMap.get(host)));
Gossiper.instance.realMarkAlive(host, Gossiper.instance.getEndpointStateForEndpoint(host));
StorageService.instance.getTokenMetadata().updateNormalTokens(tokenMap.get(host), host);
@@ -161,9 +173,9 @@
private static void checkEndpoint(final AbstractNetworkTopologySnitch snitch,
final String endpointString, final String expectedDatacenter,
- final String expectedRack)
+ final String expectedRack) throws UnknownHostException
{
- final InetAddress endpoint = InetAddresses.forString(endpointString);
+ final InetAddressAndPort endpoint = InetAddressAndPort.getByName(endpointString);
assertEquals(expectedDatacenter, snitch.getDatacenter(endpoint));
assertEquals(expectedRack, snitch.getRack(endpoint));
}
@@ -175,25 +187,25 @@
@Test
public void testChangeHostRack() throws Exception
{
- final InetAddress host = InetAddress.getByName("127.0.0.1");
+ final InetAddressAndPort host = InetAddressAndPort.getByName("127.0.0.1");
final PropertyFileSnitch snitch = new PropertyFileSnitch(/*refreshPeriodInSeconds*/1);
- checkEndpoint(snitch, host.getHostAddress(), "DC1", "RAC1");
+ checkEndpoint(snitch, host.toString(), "DC1", "RAC1");
try
{
setNodeLive(host);
Files.copy(effectiveFile, backupFile);
- replaceConfigFile(Collections.singletonMap(host.getHostAddress(), "DC1:RAC2"));
+ replaceConfigFile(Collections.singletonMap(host.toString(), "DC1:RAC2"));
Thread.sleep(1500);
- checkEndpoint(snitch, host.getHostAddress(), "DC1", "RAC1");
+ checkEndpoint(snitch, host.toString(), "DC1", "RAC1");
setNodeShutdown(host);
- replaceConfigFile(Collections.singletonMap(host.getHostAddress(), "DC1:RAC2"));
+ replaceConfigFile(Collections.singletonMap(host.toString(), "DC1:RAC2"));
Thread.sleep(1500);
- checkEndpoint(snitch, host.getHostAddress(), "DC1", "RAC2");
+ checkEndpoint(snitch, host.toString(), "DC1", "RAC2");
}
finally
{
@@ -209,25 +221,25 @@
@Test
public void testChangeHostDc() throws Exception
{
- final InetAddress host = InetAddress.getByName("127.0.0.1");
+ final InetAddressAndPort host = InetAddressAndPort.getByName("127.0.0.1");
final PropertyFileSnitch snitch = new PropertyFileSnitch(/*refreshPeriodInSeconds*/1);
- checkEndpoint(snitch, host.getHostAddress(), "DC1", "RAC1");
+ checkEndpoint(snitch, host.toString(), "DC1", "RAC1");
try
{
setNodeLive(host);
Files.copy(effectiveFile, backupFile);
- replaceConfigFile(Collections.singletonMap(host.getHostAddress(), "DC2:RAC1"));
+ replaceConfigFile(Collections.singletonMap(host.toString(), "DC2:RAC1"));
Thread.sleep(1500);
- checkEndpoint(snitch, host.getHostAddress(), "DC1", "RAC1");
+ checkEndpoint(snitch, host.toString(), "DC1", "RAC1");
setNodeShutdown(host);
- replaceConfigFile(Collections.singletonMap(host.getHostAddress(), "DC2:RAC1"));
+ replaceConfigFile(Collections.singletonMap(host.toString(), "DC2:RAC1"));
Thread.sleep(1500);
- checkEndpoint(snitch, host.getHostAddress(), "DC2", "RAC1");
+ checkEndpoint(snitch, host.toString(), "DC2", "RAC1");
}
finally
{
@@ -244,25 +256,25 @@
@Test
public void testAddHost() throws Exception
{
- final InetAddress host = InetAddress.getByName("127.0.0.9");
+ final InetAddressAndPort host = InetAddressAndPort.getByName("127.0.0.9");
final PropertyFileSnitch snitch = new PropertyFileSnitch(/*refreshPeriodInSeconds*/1);
- checkEndpoint(snitch, host.getHostAddress(), "DC1", "r1"); // default
+ checkEndpoint(snitch, host.toString(), "DC1", "r1"); // default
try
{
setNodeLive(host);
Files.copy(effectiveFile, backupFile);
- replaceConfigFile(Collections.singletonMap(host.getHostAddress(), "DC2:RAC2")); // add this line if not yet there
+ replaceConfigFile(Collections.singletonMap(host.toString(), "DC2:RAC2")); // add this line if not yet there
Thread.sleep(1500);
- checkEndpoint(snitch, host.getHostAddress(), "DC1", "r1"); // unchanged
+ checkEndpoint(snitch, host.toString(), "DC1", "r1"); // unchanged
setNodeShutdown(host);
- replaceConfigFile(Collections.singletonMap(host.getHostAddress(), "DC2:RAC2")); // add this line if not yet there
+ replaceConfigFile(Collections.singletonMap(host.toString(), "DC2:RAC2")); // add this line if not yet there
Thread.sleep(1500);
- checkEndpoint(snitch, host.getHostAddress(), "DC2", "RAC2"); // changed
+ checkEndpoint(snitch, host.toString(), "DC2", "RAC2"); // changed
}
finally
{
@@ -279,25 +291,25 @@
@Test
public void testRemoveHost() throws Exception
{
- final InetAddress host = InetAddress.getByName("127.0.0.2");
+ final InetAddressAndPort host = InetAddressAndPort.getByName("127.0.0.2");
final PropertyFileSnitch snitch = new PropertyFileSnitch(/*refreshPeriodInSeconds*/1);
- checkEndpoint(snitch, host.getHostAddress(), "DC1", "RAC2");
+ checkEndpoint(snitch, host.toString(), "DC1", "RAC2");
try
{
setNodeLive(host);
Files.copy(effectiveFile, backupFile);
- replaceConfigFile(Collections.singletonMap(host.getHostAddress(), "")); // removes line if found
+ replaceConfigFile(Collections.singletonMap(host.toString(), "")); // removes line if found
Thread.sleep(1500);
- checkEndpoint(snitch, host.getHostAddress(), "DC1", "RAC2"); // unchanged
+ checkEndpoint(snitch, host.toString(), "DC1", "RAC2"); // unchanged
setNodeShutdown(host);
- replaceConfigFile(Collections.singletonMap(host.getHostAddress(), "")); // removes line if found
+ replaceConfigFile(Collections.singletonMap(host.toString(), "")); // removes line if found
Thread.sleep(1500);
- checkEndpoint(snitch, host.getHostAddress(), "DC1", "r1"); // default
+ checkEndpoint(snitch, host.toString(), "DC1", "r1"); // default
}
finally
{
@@ -314,9 +326,9 @@
@Test
public void testChangeDefault() throws Exception
{
- final InetAddress host = InetAddress.getByName("127.0.0.9");
+ final InetAddressAndPort host = InetAddressAndPort.getByName("127.0.0.9");
final PropertyFileSnitch snitch = new PropertyFileSnitch(/*refreshPeriodInSeconds*/1);
- checkEndpoint(snitch, host.getHostAddress(), "DC1", "r1"); // default
+ checkEndpoint(snitch, host.toString(), "DC1", "r1"); // default
try
{
@@ -326,13 +338,13 @@
replaceConfigFile(Collections.singletonMap("default", "DC2:r2")); // change default
Thread.sleep(1500);
- checkEndpoint(snitch, host.getHostAddress(), "DC1", "r1"); // unchanged
+ checkEndpoint(snitch, host.toString(), "DC1", "r1"); // unchanged
setNodeShutdown(host);
replaceConfigFile(Collections.singletonMap("default", "DC2:r2")); // change default again (refresh file update)
Thread.sleep(1500);
- checkEndpoint(snitch, host.getHostAddress(), "DC2", "r2"); // default updated
+ checkEndpoint(snitch, host.toString(), "DC2", "r2"); // default updated
}
finally
{
diff --git a/test/unit/org/apache/cassandra/locator/ReconnectableSnitchHelperTest.java b/test/unit/org/apache/cassandra/locator/ReconnectableSnitchHelperTest.java
new file mode 100644
index 0000000..b1c3775
--- /dev/null
+++ b/test/unit/org/apache/cassandra/locator/ReconnectableSnitchHelperTest.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.locator;
+
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.util.Collections;
+
+import org.junit.After;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.auth.IInternodeAuthenticator;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.net.MessagingServiceTest;
+
+public class ReconnectableSnitchHelperTest
+{
+ static final IInternodeAuthenticator originalAuthenticator = DatabaseDescriptor.getInternodeAuthenticator();
+
+ @BeforeClass
+ public static void beforeClass() throws UnknownHostException
+ {
+ DatabaseDescriptor.daemonInitialization();
+ DatabaseDescriptor.setBackPressureStrategy(new MessagingServiceTest.MockBackPressureStrategy(Collections.emptyMap()));
+ DatabaseDescriptor.setBroadcastAddress(InetAddress.getByName("127.0.0.1"));
+ }
+
+ /**
+ * Make sure that if a node fails internode authentication and MessagingService returns a null
+ * pool that ReconnectableSnitchHelper fails gracefully.
+ */
+ @Test
+ public void failedAuthentication() throws Exception
+ {
+ DatabaseDescriptor.setInternodeAuthenticator(MessagingServiceTest.ALLOW_NOTHING_AUTHENTICATOR);
+ InetAddressAndPort address = InetAddressAndPort.getByName("127.0.0.250");
+ //Should tolerate null returns by MS for the connection
+ ReconnectableSnitchHelper.reconnect(address, address, null, null);
+ }
+
+ @After
+ public void replaceAuthenticator()
+ {
+ DatabaseDescriptor.setInternodeAuthenticator(originalAuthenticator);
+ }
+}
diff --git a/test/unit/org/apache/cassandra/locator/ReplicaCollectionTest.java b/test/unit/org/apache/cassandra/locator/ReplicaCollectionTest.java
new file mode 100644
index 0000000..e2d4797
--- /dev/null
+++ b/test/unit/org/apache/cassandra/locator/ReplicaCollectionTest.java
@@ -0,0 +1,572 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.locator;
+
+import com.google.common.base.Predicates;
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Multimap;
+import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.locator.ReplicaCollection.Builder.Conflict;
+import org.apache.cassandra.utils.FBUtilities;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.net.UnknownHostException;
+import java.util.ArrayList;
+import java.util.AbstractMap;
+import java.util.Comparator;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
+
+import static com.google.common.collect.Iterables.*;
+import static com.google.common.collect.Iterables.filter;
+import static org.apache.cassandra.locator.Replica.fullReplica;
+import static org.apache.cassandra.locator.Replica.transientReplica;
+import static org.apache.cassandra.locator.ReplicaUtils.*;
+
+public class ReplicaCollectionTest
+{
+
+ static class TestCase<C extends AbstractReplicaCollection<C>>
+ {
+ final boolean isBuilder;
+ final C test;
+ final List<Replica> canonicalList;
+ final Multimap<InetAddressAndPort, Replica> canonicalByEndpoint;
+ final Multimap<Range<Token>, Replica> canonicalByRange;
+
+ TestCase(boolean isBuilder, C test, List<Replica> canonicalList)
+ {
+ this.isBuilder = isBuilder;
+ this.test = test;
+ this.canonicalList = canonicalList;
+ this.canonicalByEndpoint = HashMultimap.create();
+ this.canonicalByRange = HashMultimap.create();
+ for (Replica replica : canonicalList)
+ canonicalByEndpoint.put(replica.endpoint(), replica);
+ for (Replica replica : canonicalList)
+ canonicalByRange.put(replica.range(), replica);
+ if (isBuilder)
+ Assert.assertTrue(test instanceof ReplicaCollection.Builder<?>);
+ }
+
+ void testSize()
+ {
+ Assert.assertEquals(canonicalList.size(), test.size());
+ }
+
+ void testEquals()
+ {
+ Assert.assertTrue(elementsEqual(canonicalList, test));
+ }
+
+ void testEndpoints()
+ {
+ // TODO: we should do more exhaustive tests of the collection
+ Assert.assertEquals(ImmutableSet.copyOf(canonicalByEndpoint.keySet()), ImmutableSet.copyOf(test.endpoints()));
+ try
+ {
+ test.endpoints().add(EP5);
+ Assert.fail();
+ } catch (UnsupportedOperationException e) {}
+ try
+ {
+ test.endpoints().remove(EP5);
+ Assert.fail();
+ } catch (UnsupportedOperationException e) {}
+
+ Assert.assertTrue(test.endpoints().containsAll(canonicalByEndpoint.keySet()));
+ for (InetAddressAndPort ep : canonicalByEndpoint.keySet())
+ Assert.assertTrue(test.endpoints().contains(ep));
+ for (InetAddressAndPort ep : ALL_EP)
+ if (!canonicalByEndpoint.containsKey(ep))
+ Assert.assertFalse(test.endpoints().contains(ep));
+ }
+
+ public void testOrderOfIteration()
+ {
+ Assert.assertEquals(canonicalList, ImmutableList.copyOf(test));
+ Assert.assertEquals(canonicalList, test.stream().collect(Collectors.toList()));
+ Assert.assertTrue(Iterables.elementsEqual(new LinkedHashSet<>(Lists.transform(canonicalList, Replica::endpoint)), test.endpoints()));
+ }
+
+ private void assertSubList(C subCollection, int from, int to)
+ {
+ if (from == to)
+ {
+ Assert.assertTrue(subCollection.isEmpty());
+ }
+ else
+ {
+ AbstractReplicaCollection.ReplicaList subList = this.test.list.subList(from, to);
+ if (!isBuilder)
+ Assert.assertSame(subList.contents, subCollection.list.contents);
+ Assert.assertEquals(subList, subCollection.list);
+ }
+ }
+
+ private void assertSubSequence(Iterable<Replica> subSequence, int from, int to)
+ {
+ AbstractReplicaCollection.ReplicaList subList = this.test.list.subList(from, to);
+ if (!elementsEqual(subList, subSequence))
+ {
+ elementsEqual(subList, subSequence);
+ }
+ Assert.assertTrue(elementsEqual(subList, subSequence));
+ }
+
+ void testSubList(int subListDepth, int filterDepth, int sortDepth)
+ {
+ if (!isBuilder)
+ Assert.assertSame(test, test.subList(0, test.size()));
+
+ if (test.isEmpty())
+ return;
+
+ Assert.assertSame(test.list.contents, test.subList(0, 1).list.contents);
+ TestCase<C> skipFront = new TestCase<>(false, test.subList(1, test.size()), canonicalList.subList(1, canonicalList.size()));
+ assertSubList(skipFront.test, 1, canonicalList.size());
+ skipFront.testAll(subListDepth - 1, filterDepth, sortDepth);
+ TestCase<C> skipBack = new TestCase<>(false, test.subList(0, test.size() - 1), canonicalList.subList(0, canonicalList.size() - 1));
+ assertSubList(skipBack.test, 0, canonicalList.size() - 1);
+ skipBack.testAll(subListDepth - 1, filterDepth, sortDepth);
+ }
+
+ void testFilter(int subListDepth, int filterDepth, int sortDepth)
+ {
+ if (!isBuilder)
+ Assert.assertSame(test, test.filter(Predicates.alwaysTrue()));
+
+ if (test.isEmpty())
+ return;
+
+ // remove start
+ // we recurse on the same subset in testSubList, so just corroborate we have the correct list here
+ {
+ Predicate<Replica> removeFirst = r -> !r.equals(canonicalList.get(0));
+ assertSubList(test.filter(removeFirst), 1, canonicalList.size());
+ assertSubList(test.filter(removeFirst, 1), 1, Math.min(canonicalList.size(), 2));
+ assertSubSequence(test.filterLazily(removeFirst), 1, canonicalList.size());
+ assertSubSequence(test.filterLazily(removeFirst, 1), 1, Math.min(canonicalList.size(), 2));
+ }
+
+ if (test.size() <= 1)
+ return;
+
+ // remove end
+ // we recurse on the same subset in testSubList, so just corroborate we have the correct list here
+ {
+ int last = canonicalList.size() - 1;
+ Predicate<Replica> removeLast = r -> !r.equals(canonicalList.get(last));
+ assertSubList(test.filter(removeLast), 0, last);
+ assertSubSequence(test.filterLazily(removeLast), 0, last);
+ }
+
+ if (test.size() <= 2)
+ return;
+
+ Predicate<Replica> removeMiddle = r -> !r.equals(canonicalList.get(canonicalList.size() / 2));
+ TestCase<C> filtered = new TestCase<>(false, test.filter(removeMiddle), ImmutableList.copyOf(filter(canonicalList, removeMiddle::test)));
+ filtered.testAll(subListDepth, filterDepth - 1, sortDepth);
+ Assert.assertTrue(elementsEqual(filtered.canonicalList, test.filterLazily(removeMiddle, Integer.MAX_VALUE)));
+ Assert.assertTrue(elementsEqual(limit(filter(canonicalList, removeMiddle::test), canonicalList.size() - 2), test.filterLazily(removeMiddle, canonicalList.size() - 2)));
+ }
+
+ void testCount()
+ {
+ Assert.assertEquals(0, test.count(Predicates.alwaysFalse()));
+
+ if (test.isEmpty())
+ {
+ Assert.assertEquals(0, test.count(Predicates.alwaysTrue()));
+ return;
+ }
+
+ for (int i = 0 ; i < canonicalList.size() ; ++i)
+ {
+ Replica discount = canonicalList.get(i);
+ Assert.assertEquals(canonicalList.size() - 1, test.count(r -> !r.equals(discount)));
+ }
+ }
+
+ void testContains()
+ {
+ for (Replica replica : canonicalList)
+ Assert.assertTrue(test.contains(replica));
+ Assert.assertFalse(test.contains(fullReplica(NULL_EP, NULL_RANGE)));
+ }
+
+ void testGet()
+ {
+ for (int i = 0 ; i < canonicalList.size() ; ++i)
+ Assert.assertEquals(canonicalList.get(i), test.get(i));
+ }
+
+ void testSort(int subListDepth, int filterDepth, int sortDepth)
+ {
+ final Comparator<Replica> comparator = (o1, o2) ->
+ {
+ boolean f1 = o1.equals(canonicalList.get(0));
+ boolean f2 = o2.equals(canonicalList.get(0));
+ return f1 == f2 ? 0 : f1 ? 1 : -1;
+ };
+ TestCase<C> sorted = new TestCase<>(false, test.sorted(comparator), ImmutableList.sortedCopyOf(comparator, canonicalList));
+ sorted.testAll(subListDepth, filterDepth, sortDepth - 1);
+ }
+
+ void testAll(int subListDepth, int filterDepth, int sortDepth)
+ {
+ testEndpoints();
+ testOrderOfIteration();
+ testContains();
+ testGet();
+ testEquals();
+ testSize();
+ testCount();
+ if (subListDepth > 0)
+ testSubList(subListDepth, filterDepth, sortDepth);
+ if (filterDepth > 0)
+ testFilter(subListDepth, filterDepth, sortDepth);
+ if (sortDepth > 0)
+ testSort(subListDepth, filterDepth, sortDepth);
+ }
+
+ public void testAll()
+ {
+ testAll(2, 2, 2);
+ }
+ }
+
+ static class RangesAtEndpointTestCase extends TestCase<RangesAtEndpoint>
+ {
+ RangesAtEndpointTestCase(boolean isBuilder, RangesAtEndpoint test, List<Replica> canonicalList)
+ {
+ super(isBuilder, test, canonicalList);
+ }
+
+ void testRanges()
+ {
+ Assert.assertEquals(ImmutableSet.copyOf(canonicalByRange.keySet()), ImmutableSet.copyOf(test.ranges()));
+ try
+ {
+ test.ranges().add(R5);
+ Assert.fail();
+ } catch (UnsupportedOperationException e) {}
+ try
+ {
+ test.ranges().remove(R5);
+ Assert.fail();
+ } catch (UnsupportedOperationException e) {}
+
+ Assert.assertTrue(test.ranges().containsAll(canonicalByRange.keySet()));
+ for (Range<Token> range : canonicalByRange.keySet())
+ Assert.assertTrue(test.ranges().contains(range));
+ for (Range<Token> range : ALL_R)
+ if (!canonicalByRange.containsKey(range))
+ Assert.assertFalse(test.ranges().contains(range));
+ }
+
+ void testByRange()
+ {
+ // check byEndppint() and byRange().entrySet()
+ Assert.assertFalse(test.byRange().containsKey(EP1));
+ Assert.assertFalse(test.byRange().entrySet().contains(EP1));
+ try
+ {
+ test.byRange().entrySet().contains(null);
+ Assert.fail();
+ } catch (NullPointerException | IllegalArgumentException e) {}
+ try
+ {
+ test.byRange().containsKey(null);
+ Assert.fail();
+ } catch (NullPointerException | IllegalArgumentException e) {}
+
+ for (Range<Token> r : ALL_R)
+ {
+ if (canonicalByRange.containsKey(r))
+ {
+ Assert.assertTrue(test.byRange().containsKey(r));
+ Assert.assertEquals(canonicalByRange.get(r), ImmutableSet.of(test.byRange().get(r)));
+ for (Replica replica : canonicalByRange.get(r))
+ Assert.assertTrue(test.byRange().entrySet().contains(new AbstractMap.SimpleImmutableEntry<>(r, replica)));
+ }
+ else
+ {
+ Assert.assertFalse(test.byRange().containsKey(r));
+ Assert.assertFalse(test.byRange().entrySet().contains(new AbstractMap.SimpleImmutableEntry<>(r, Replica.fullReplica(EP1, r))));
+ }
+ }
+ }
+
+ @Override
+ public void testOrderOfIteration()
+ {
+ super.testOrderOfIteration();
+ Assert.assertTrue(Iterables.elementsEqual(Lists.transform(canonicalList, Replica::range), test.ranges()));
+ Assert.assertTrue(Iterables.elementsEqual(canonicalList, test.byRange().values()));
+ Assert.assertTrue(Iterables.elementsEqual(
+ Lists.transform(canonicalList, r -> new AbstractMap.SimpleImmutableEntry<>(r.range(), r)),
+ test.byRange().entrySet()));
+ }
+
+ public void testUnwrap(int subListDepth, int filterDepth, int sortDepth)
+ {
+ List<Replica> canonUnwrap = new ArrayList<>();
+ for (Replica replica : canonicalList)
+ for (Range<Token> range : replica.range().unwrap())
+ canonUnwrap.add(replica.decorateSubrange(range));
+ RangesAtEndpoint testUnwrap = test.unwrap();
+ if (testUnwrap == test)
+ {
+ Assert.assertEquals(canonicalList, canonUnwrap);
+ }
+ else
+ {
+ new RangesAtEndpointTestCase(false, testUnwrap, canonUnwrap)
+ .testAllExceptUnwrap(subListDepth, filterDepth, sortDepth);
+ }
+ }
+
+ void testAllExceptUnwrap(int subListDepth, int filterDepth, int sortDepth)
+ {
+ super.testAll(subListDepth, filterDepth, sortDepth);
+ testRanges();
+ testByRange();
+ }
+
+ @Override
+ void testAll(int subListDepth, int filterDepth, int sortDepth)
+ {
+ testAllExceptUnwrap(subListDepth, filterDepth, sortDepth);
+ testUnwrap(subListDepth, filterDepth, sortDepth);
+ }
+ }
+
+ static class EndpointsTestCase<E extends Endpoints<E>> extends TestCase<E>
+ {
+ EndpointsTestCase(boolean isBuilder, E test, List<Replica> canonicalList)
+ {
+ super(isBuilder, test, canonicalList);
+ }
+
+ void testByEndpoint()
+ {
+ // check byEndppint() and byEndpoint().entrySet()
+ Assert.assertFalse(test.byEndpoint().containsKey(R1));
+ Assert.assertFalse(test.byEndpoint().entrySet().contains(EP1));
+ try
+ {
+ test.byEndpoint().entrySet().contains(null);
+ Assert.fail();
+ } catch (NullPointerException | IllegalArgumentException e) {}
+ try
+ {
+ test.byEndpoint().containsKey(null);
+ Assert.fail();
+ } catch (NullPointerException | IllegalArgumentException e) {}
+
+ for (InetAddressAndPort ep : ALL_EP)
+ {
+ if (canonicalByEndpoint.containsKey(ep))
+ {
+ Assert.assertTrue(test.byEndpoint().containsKey(ep));
+ Assert.assertEquals(canonicalByEndpoint.get(ep), ImmutableSet.of(test.byEndpoint().get(ep)));
+ for (Replica replica : canonicalByEndpoint.get(ep))
+ Assert.assertTrue(test.byEndpoint().entrySet().contains(new AbstractMap.SimpleImmutableEntry<>(ep, replica)));
+ }
+ else
+ {
+ Assert.assertFalse(test.byEndpoint().containsKey(ep));
+ Assert.assertFalse(test.byEndpoint().entrySet().contains(new AbstractMap.SimpleImmutableEntry<>(ep, Replica.fullReplica(ep, R1))));
+ }
+ }
+ }
+
+ @Override
+ public void testOrderOfIteration()
+ {
+ super.testOrderOfIteration();
+ Assert.assertTrue(Iterables.elementsEqual(canonicalList, test.byEndpoint().values()));
+ Assert.assertTrue(Iterables.elementsEqual(
+ Lists.transform(canonicalList, r -> new AbstractMap.SimpleImmutableEntry<>(r.endpoint(), r)),
+ test.byEndpoint().entrySet()));
+ }
+
+ @Override
+ void testAll(int subListDepth, int filterDepth, int sortDepth)
+ {
+ super.testAll(subListDepth, filterDepth, sortDepth);
+ testByEndpoint();
+ }
+ }
+
+ private static final ImmutableList<Replica> RANGES_AT_ENDPOINT = ImmutableList.of(
+ fullReplica(EP1, R1),
+ fullReplica(EP1, R2),
+ transientReplica(EP1, R3),
+ fullReplica(EP1, R4),
+ transientReplica(EP1, R5)
+ );
+
+ @Test
+ public void testRangesAtEndpoint()
+ {
+ ImmutableList<Replica> canonical = RANGES_AT_ENDPOINT;
+ new RangesAtEndpointTestCase(
+ false, RangesAtEndpoint.copyOf(canonical), canonical
+ ).testAll();
+ }
+
+ @Test
+ public void testMutableRangesAtEndpoint()
+ {
+ ImmutableList<Replica> canonical1 = RANGES_AT_ENDPOINT.subList(0, RANGES_AT_ENDPOINT.size());
+ RangesAtEndpoint.Builder test = new RangesAtEndpoint.Builder(RANGES_AT_ENDPOINT.get(0).endpoint(), canonical1.size());
+ test.addAll(canonical1, Conflict.NONE);
+ try
+ { // incorrect range
+ test.addAll(canonical1, Conflict.NONE);
+ Assert.fail();
+ } catch (IllegalArgumentException e) { }
+ test.addAll(canonical1, Conflict.DUPLICATE); // we ignore exact duplicates
+ try
+ { // invalid endpoint; always error
+ test.add(fullReplica(EP2, BROADCAST_RANGE), Conflict.ALL);
+ Assert.fail();
+ } catch (IllegalArgumentException e) { }
+ try
+ { // conflict on isFull/isTransient
+ test.add(fullReplica(EP1, R3), Conflict.DUPLICATE);
+ Assert.fail();
+ } catch (IllegalArgumentException e) { }
+ test.add(fullReplica(EP1, R3), Conflict.ALL);
+
+ new RangesAtEndpointTestCase(true, test, canonical1).testAll();
+
+ RangesAtEndpoint snapshot = test.subList(0, test.size());
+
+ ImmutableList<Replica> canonical2 = RANGES_AT_ENDPOINT;
+ test.addAll(canonical2.reverse(), Conflict.DUPLICATE);
+ new TestCase<>(false, snapshot, canonical1).testAll();
+ new TestCase<>(true, test, canonical2).testAll();
+ }
+
+ private static final ImmutableList<Replica> ENDPOINTS_FOR_X = ImmutableList.of(
+ fullReplica(EP1, R1),
+ fullReplica(EP2, R1),
+ transientReplica(EP3, R1),
+ fullReplica(EP4, R1),
+ transientReplica(EP5, R1)
+ );
+
+ @Test
+ public void testEndpointsForRange()
+ {
+ ImmutableList<Replica> canonical = ENDPOINTS_FOR_X;
+ new EndpointsTestCase<>(
+ false, EndpointsForRange.copyOf(canonical), canonical
+ ).testAll();
+ }
+
+ @Test
+ public void testMutableEndpointsForRange()
+ {
+ ImmutableList<Replica> canonical1 = ENDPOINTS_FOR_X.subList(0, ENDPOINTS_FOR_X.size() - 1);
+ EndpointsForRange.Builder test = new EndpointsForRange.Builder(R1, canonical1.size());
+ test.addAll(canonical1, Conflict.NONE);
+ try
+ { // incorrect range
+ test.addAll(canonical1, Conflict.NONE);
+ Assert.fail();
+ } catch (IllegalArgumentException e) { }
+ test.addAll(canonical1, Conflict.DUPLICATE); // we ignore exact duplicates
+ try
+ { // incorrect range
+ test.add(fullReplica(BROADCAST_EP, R2), Conflict.ALL);
+ Assert.fail();
+ } catch (IllegalArgumentException e) { }
+ try
+ { // conflict on isFull/isTransient
+ test.add(transientReplica(EP1, R1), Conflict.DUPLICATE);
+ Assert.fail();
+ } catch (IllegalArgumentException e) { }
+ test.add(transientReplica(EP1, R1), Conflict.ALL);
+
+ new EndpointsTestCase<>(true, test, canonical1).testAll();
+
+ EndpointsForRange snapshot = test.subList(0, test.size());
+
+ ImmutableList<Replica> canonical2 = ENDPOINTS_FOR_X;
+ test.addAll(canonical2.reverse(), Conflict.DUPLICATE);
+ new EndpointsTestCase<>(false, snapshot, canonical1).testAll();
+ new EndpointsTestCase<>(true, test, canonical2).testAll();
+ }
+
+ @Test
+ public void testEndpointsForToken()
+ {
+ ImmutableList<Replica> canonical = ENDPOINTS_FOR_X;
+ new EndpointsTestCase<>(
+ false, EndpointsForToken.copyOf(tk(1), canonical), canonical
+ ).testAll();
+ }
+
+ @Test
+ public void testMutableEndpointsForToken()
+ {
+ ImmutableList<Replica> canonical1 = ENDPOINTS_FOR_X.subList(0, ENDPOINTS_FOR_X.size() - 1);
+ EndpointsForToken.Builder test = new EndpointsForToken.Builder(tk(1), canonical1.size());
+ test.addAll(canonical1, Conflict.NONE);
+ try
+ { // incorrect range
+ test.addAll(canonical1, Conflict.NONE);
+ Assert.fail();
+ } catch (IllegalArgumentException e) { }
+ test.addAll(canonical1, Conflict.DUPLICATE); // we ignore exact duplicates
+ try
+ { // incorrect range
+ test.add(fullReplica(BROADCAST_EP, R2), Conflict.ALL);
+ Assert.fail();
+ } catch (IllegalArgumentException e) { }
+ try
+ { // conflict on isFull/isTransient
+ test.add(transientReplica(EP1, R1), Conflict.DUPLICATE);
+ Assert.fail();
+ } catch (IllegalArgumentException e) { }
+ test.add(transientReplica(EP1, R1), Conflict.ALL);
+
+ new EndpointsTestCase<>(true, test, canonical1).testAll();
+
+ EndpointsForToken snapshot = test.subList(0, test.size());
+
+ ImmutableList<Replica> canonical2 = ENDPOINTS_FOR_X;
+ test.addAll(canonical2.reverse(), Conflict.DUPLICATE);
+ new EndpointsTestCase<>(false, snapshot, canonical1).testAll();
+ new EndpointsTestCase<>(true, test, canonical2).testAll();
+ }
+}
diff --git a/test/unit/org/apache/cassandra/locator/ReplicaLayoutTest.java b/test/unit/org/apache/cassandra/locator/ReplicaLayoutTest.java
new file mode 100644
index 0000000..b5b60e3
--- /dev/null
+++ b/test/unit/org/apache/cassandra/locator/ReplicaLayoutTest.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.locator;
+
+import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.dht.Token;
+import org.junit.Assert;
+import org.junit.Test;
+
+import static org.apache.cassandra.locator.ReplicaUtils.*;
+
+public class ReplicaLayoutTest
+{
+ @Test
+ public void testConflictResolution()
+ {
+ final Token token = new Murmur3Partitioner.LongToken(1L);
+ final Replica f1 = Replica.fullReplica(EP1, R1);
+ final Replica f2 = Replica.fullReplica(EP2, R1);
+ final Replica t2 = Replica.transientReplica(EP2, R1);
+ final Replica f3 = Replica.fullReplica(EP3, R1);
+ final Replica t4 = Replica.transientReplica(EP4, R1);
+
+ {
+ // test no conflict
+ EndpointsForToken natural = EndpointsForToken.of(token, f1, f3);
+ EndpointsForToken pending = EndpointsForToken.of(token, t2, t4);
+ Assert.assertFalse(ReplicaLayout.haveWriteConflicts(natural, pending));
+ }
+ {
+ // test full in natural, transient in pending
+ EndpointsForToken natural = EndpointsForToken.of(token, f1, f2, f3);
+ EndpointsForToken pending = EndpointsForToken.of(token, t2, t4);
+ EndpointsForToken expectNatural = natural;
+ EndpointsForToken expectPending = EndpointsForToken.of(token, t4);
+ Assert.assertTrue(ReplicaLayout.haveWriteConflicts(natural, pending));
+ assertEquals(expectNatural, ReplicaLayout.resolveWriteConflictsInNatural(natural, pending));
+ assertEquals(expectPending, ReplicaLayout.resolveWriteConflictsInPending(natural, pending));
+ }
+ {
+ // test transient in natural, full in pending
+ EndpointsForToken natural = EndpointsForToken.of(token, f1, t2, f3);
+ EndpointsForToken pending = EndpointsForToken.of(token, f2, t4);
+ EndpointsForToken expectNatural = EndpointsForToken.of(token, f1, f2, f3);
+ EndpointsForToken expectPending = EndpointsForToken.of(token, t4);
+ Assert.assertTrue(ReplicaLayout.haveWriteConflicts(natural, pending));
+ assertEquals(expectNatural, ReplicaLayout.resolveWriteConflictsInNatural(natural, pending));
+ assertEquals(expectPending, ReplicaLayout.resolveWriteConflictsInPending(natural, pending));
+ }
+ }
+
+ private static void assertEquals(AbstractReplicaCollection<?> a, AbstractReplicaCollection<?> b)
+ {
+ Assert.assertEquals(a.list, b.list);
+ }
+
+}
diff --git a/test/unit/org/apache/cassandra/locator/ReplicaPlansTest.java b/test/unit/org/apache/cassandra/locator/ReplicaPlansTest.java
new file mode 100644
index 0000000..4d0dd47
--- /dev/null
+++ b/test/unit/org/apache/cassandra/locator/ReplicaPlansTest.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.locator;
+
+import com.google.common.base.Predicates;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.ConsistencyLevel;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.schema.KeyspaceMetadata;
+import org.apache.cassandra.schema.KeyspaceParams;
+import org.junit.Test;
+
+import java.util.Map;
+import java.util.Set;
+
+import static org.apache.cassandra.locator.Replica.fullReplica;
+import static org.apache.cassandra.locator.ReplicaUtils.*;
+
+public class ReplicaPlansTest
+{
+
+ static
+ {
+ DatabaseDescriptor.daemonInitialization();
+ }
+
+ static class Snitch extends AbstractNetworkTopologySnitch
+ {
+ final Set<InetAddressAndPort> dc1;
+ Snitch(Set<InetAddressAndPort> dc1)
+ {
+ this.dc1 = dc1;
+ }
+ @Override
+ public String getRack(InetAddressAndPort endpoint)
+ {
+ return dc1.contains(endpoint) ? "R1" : "R2";
+ }
+
+ @Override
+ public String getDatacenter(InetAddressAndPort endpoint)
+ {
+ return dc1.contains(endpoint) ? "DC1" : "DC2";
+ }
+ }
+
+ private static Keyspace ks(Set<InetAddressAndPort> dc1, Map<String, String> replication)
+ {
+ replication = ImmutableMap.<String, String>builder().putAll(replication).put("class", "NetworkTopologyStrategy").build();
+ Keyspace keyspace = Keyspace.mockKS(KeyspaceMetadata.create("blah", KeyspaceParams.create(false, replication)));
+ Snitch snitch = new Snitch(dc1);
+ DatabaseDescriptor.setEndpointSnitch(snitch);
+ keyspace.getReplicationStrategy().snitch = snitch;
+ return keyspace;
+ }
+
+ private static Replica full(InetAddressAndPort ep) { return fullReplica(ep, R1); }
+
+
+
+ @Test
+ public void testWriteEachQuorum()
+ {
+ IEndpointSnitch stash = DatabaseDescriptor.getEndpointSnitch();
+ final Token token = tk(1L);
+ try
+ {
+ {
+ // all full natural
+ Keyspace ks = ks(ImmutableSet.of(EP1, EP2, EP3), ImmutableMap.of("DC1", "3", "DC2", "3"));
+ EndpointsForToken natural = EndpointsForToken.of(token, full(EP1), full(EP2), full(EP3), full(EP4), full(EP5), full(EP6));
+ EndpointsForToken pending = EndpointsForToken.empty(token);
+ ReplicaPlan.ForTokenWrite plan = ReplicaPlans.forWrite(ks, ConsistencyLevel.EACH_QUORUM, natural, pending, Predicates.alwaysTrue(), ReplicaPlans.writeNormal);
+ assertEquals(natural, plan.liveAndDown);
+ assertEquals(natural, plan.live);
+ assertEquals(natural, plan.contacts());
+ }
+ {
+ // all natural and up, one transient in each DC
+ Keyspace ks = ks(ImmutableSet.of(EP1, EP2, EP3), ImmutableMap.of("DC1", "3", "DC2", "3"));
+ EndpointsForToken natural = EndpointsForToken.of(token, full(EP1), full(EP2), trans(EP3), full(EP4), full(EP5), trans(EP6));
+ EndpointsForToken pending = EndpointsForToken.empty(token);
+ ReplicaPlan.ForTokenWrite plan = ReplicaPlans.forWrite(ks, ConsistencyLevel.EACH_QUORUM, natural, pending, Predicates.alwaysTrue(), ReplicaPlans.writeNormal);
+ assertEquals(natural, plan.liveAndDown);
+ assertEquals(natural, plan.live);
+ EndpointsForToken expectContacts = EndpointsForToken.of(token, full(EP1), full(EP2), full(EP4), full(EP5));
+ assertEquals(expectContacts, plan.contacts());
+ }
+ }
+ finally
+ {
+ DatabaseDescriptor.setEndpointSnitch(stash);
+ }
+
+ {
+ // test simple
+
+ }
+ }
+
+}
diff --git a/test/unit/org/apache/cassandra/locator/ReplicaUtils.java b/test/unit/org/apache/cassandra/locator/ReplicaUtils.java
new file mode 100644
index 0000000..72c0a06
--- /dev/null
+++ b/test/unit/org/apache/cassandra/locator/ReplicaUtils.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.locator;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.utils.FBUtilities;
+import org.junit.Assert;
+
+import java.net.UnknownHostException;
+import java.util.List;
+
+import static org.apache.cassandra.locator.Replica.fullReplica;
+import static org.apache.cassandra.locator.Replica.transientReplica;
+
+public class ReplicaUtils
+{
+ public static final Range<Token> FULL_RANGE = new Range<>(Murmur3Partitioner.MINIMUM, Murmur3Partitioner.MINIMUM);
+ public static final AbstractBounds<PartitionPosition> FULL_BOUNDS = new Range<>(Murmur3Partitioner.MINIMUM.minKeyBound(), Murmur3Partitioner.MINIMUM.maxKeyBound());
+
+ public static Replica full(InetAddressAndPort endpoint)
+ {
+ return fullReplica(endpoint, FULL_RANGE);
+ }
+
+ public static Replica trans(InetAddressAndPort endpoint)
+ {
+ return transientReplica(endpoint, FULL_RANGE);
+ }
+
+ public static Replica full(InetAddressAndPort endpoint, Token token)
+ {
+ return fullReplica(endpoint, new Range<>(token, token));
+ }
+
+ public static Replica trans(InetAddressAndPort endpoint, Token token)
+ {
+ return transientReplica(endpoint, new Range<>(token, token));
+ }
+
+ static final InetAddressAndPort EP1, EP2, EP3, EP4, EP5, EP6, EP7, EP8, EP9, BROADCAST_EP, NULL_EP;
+ static final Range<Token> R1, R2, R3, R4, R5, R6, R7, R8, R9, BROADCAST_RANGE, NULL_RANGE, WRAP_RANGE;
+ static final List<InetAddressAndPort> ALL_EP;
+ static final List<Range<Token>> ALL_R;
+
+ static
+ {
+ try
+ {
+ EP1 = InetAddressAndPort.getByName("127.0.0.1");
+ EP2 = InetAddressAndPort.getByName("127.0.0.2");
+ EP3 = InetAddressAndPort.getByName("127.0.0.3");
+ EP4 = InetAddressAndPort.getByName("127.0.0.4");
+ EP5 = InetAddressAndPort.getByName("127.0.0.5");
+ EP6 = InetAddressAndPort.getByName("127.0.0.6");
+ EP7 = InetAddressAndPort.getByName("127.0.0.7");
+ EP8 = InetAddressAndPort.getByName("127.0.0.8");
+ EP9 = InetAddressAndPort.getByName("127.0.0.9");
+ BROADCAST_EP = FBUtilities.getBroadcastAddressAndPort();
+ NULL_EP = InetAddressAndPort.getByName("127.255.255.255");
+ R1 = range(0, 1);
+ R2 = range(1, 2);
+ R3 = range(2, 3);
+ R4 = range(3, 4);
+ R5 = range(4, 5);
+ R6 = range(5, 6);
+ R7 = range(6, 7);
+ R8 = range(7, 8);
+ R9 = range(8, 9);
+ BROADCAST_RANGE = range(10, 11);
+ NULL_RANGE = range(10000, 10001);
+ WRAP_RANGE = range(100000, 0);
+ ALL_EP = ImmutableList.of(EP1, EP2, EP3, EP4, EP5, BROADCAST_EP);
+ ALL_R = ImmutableList.of(R1, R2, R3, R4, R5, BROADCAST_RANGE, WRAP_RANGE);
+ }
+ catch (UnknownHostException e)
+ {
+ throw new RuntimeException(e);
+ }
+ }
+
+ static Token tk(long t)
+ {
+ return new Murmur3Partitioner.LongToken(t);
+ }
+
+ static Range<Token> range(long left, long right)
+ {
+ return new Range<>(tk(left), tk(right));
+ }
+
+ static void assertEquals(AbstractReplicaCollection<?> a, AbstractReplicaCollection<?> b)
+ {
+ Assert.assertEquals(a.list, b.list);
+ }
+
+}
diff --git a/test/unit/org/apache/cassandra/locator/ReplicationFactorTest.java b/test/unit/org/apache/cassandra/locator/ReplicationFactorTest.java
new file mode 100644
index 0000000..7d85b44
--- /dev/null
+++ b/test/unit/org/apache/cassandra/locator/ReplicationFactorTest.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.locator;
+
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.gms.Gossiper;
+
+public class ReplicationFactorTest
+{
+
+ @BeforeClass
+ public static void setupClass()
+ {
+ DatabaseDescriptor.daemonInitialization();
+ DatabaseDescriptor.setTransientReplicationEnabledUnsafe(true);
+ Gossiper.instance.start(1);
+ }
+
+ private static void assertRfParseFailure(String s)
+ {
+ try
+ {
+ ReplicationFactor.fromString(s);
+ Assert.fail("Expected IllegalArgumentException");
+ }
+ catch (IllegalArgumentException e)
+ {
+ // expected
+ }
+ }
+
+ private static void assertRfParse(String s, int expectedReplicas, int expectedTrans)
+ {
+ ReplicationFactor rf = ReplicationFactor.fromString(s);
+ Assert.assertEquals(expectedReplicas, rf.allReplicas);
+ Assert.assertEquals(expectedTrans, rf.transientReplicas());
+ Assert.assertEquals(expectedReplicas - expectedTrans, rf.fullReplicas);
+ }
+
+ @Test
+ public void parseTest()
+ {
+ assertRfParse("3", 3, 0);
+ assertRfParse("3/1", 3, 1);
+
+ assertRfParse("5", 5, 0);
+ assertRfParse("5/2", 5, 2);
+
+ assertRfParseFailure("-1");
+ assertRfParseFailure("3/3");
+ assertRfParseFailure("3/4");
+ }
+
+ @Test
+ public void roundTripParseTest()
+ {
+ String input = "3";
+ Assert.assertEquals(input, ReplicationFactor.fromString(input).toParseableString());
+
+ String transientInput = "3/1";
+ Assert.assertEquals(transientInput, ReplicationFactor.fromString(transientInput).toParseableString());
+ }
+}
diff --git a/test/unit/org/apache/cassandra/locator/ReplicationStrategyEndpointCacheTest.java b/test/unit/org/apache/cassandra/locator/ReplicationStrategyEndpointCacheTest.java
index c811811..2e9e32d 100644
--- a/test/unit/org/apache/cassandra/locator/ReplicationStrategyEndpointCacheTest.java
+++ b/test/unit/org/apache/cassandra/locator/ReplicationStrategyEndpointCacheTest.java
@@ -17,10 +17,7 @@
*/
package org.apache.cassandra.locator;
-import java.net.InetAddress;
-import java.util.ArrayList;
import java.util.HashMap;
-import java.util.List;
import java.util.Map;
import org.apache.commons.lang3.StringUtils;
@@ -28,6 +25,7 @@
import org.junit.Test;
import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.Util;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.dht.RandomPartitioner.BigIntegerToken;
import org.apache.cassandra.dht.Token;
@@ -45,8 +43,7 @@
public static void defineSchema() throws Exception
{
SchemaLoader.prepareServer();
- SchemaLoader.createKeyspace(KEYSPACE,
- KeyspaceParams.simple(5));
+ SchemaLoader.createKeyspace(KEYSPACE, KeyspaceParams.simple(5));
}
public void setup(Class stratClass, Map<String, String> strategyOptions) throws Exception
@@ -56,69 +53,67 @@
strategy = getStrategyWithNewTokenMetadata(Keyspace.open(KEYSPACE).getReplicationStrategy(), tmd);
- tmd.updateNormalToken(new BigIntegerToken(String.valueOf(10)), InetAddress.getByName("127.0.0.1"));
- tmd.updateNormalToken(new BigIntegerToken(String.valueOf(20)), InetAddress.getByName("127.0.0.2"));
- tmd.updateNormalToken(new BigIntegerToken(String.valueOf(30)), InetAddress.getByName("127.0.0.3"));
- tmd.updateNormalToken(new BigIntegerToken(String.valueOf(40)), InetAddress.getByName("127.0.0.4"));
- //tmd.updateNormalToken(new BigIntegerToken(String.valueOf(50)), InetAddress.getByName("127.0.0.5"));
- tmd.updateNormalToken(new BigIntegerToken(String.valueOf(60)), InetAddress.getByName("127.0.0.6"));
- tmd.updateNormalToken(new BigIntegerToken(String.valueOf(70)), InetAddress.getByName("127.0.0.7"));
- tmd.updateNormalToken(new BigIntegerToken(String.valueOf(80)), InetAddress.getByName("127.0.0.8"));
+ tmd.updateNormalToken(new BigIntegerToken(String.valueOf(10)), InetAddressAndPort.getByName("127.0.0.1"));
+ tmd.updateNormalToken(new BigIntegerToken(String.valueOf(20)), InetAddressAndPort.getByName("127.0.0.2"));
+ tmd.updateNormalToken(new BigIntegerToken(String.valueOf(30)), InetAddressAndPort.getByName("127.0.0.3"));
+ tmd.updateNormalToken(new BigIntegerToken(String.valueOf(40)), InetAddressAndPort.getByName("127.0.0.4"));
+ //tmd.updateNormalToken(new BigIntegerToken(String.valueOf(50)), InetAddressAndPort.getByName("127.0.0.5", null, null));
+ tmd.updateNormalToken(new BigIntegerToken(String.valueOf(60)), InetAddressAndPort.getByName("127.0.0.6"));
+ tmd.updateNormalToken(new BigIntegerToken(String.valueOf(70)), InetAddressAndPort.getByName("127.0.0.7"));
+ tmd.updateNormalToken(new BigIntegerToken(String.valueOf(80)), InetAddressAndPort.getByName("127.0.0.8"));
}
@Test
public void testEndpointsWereCached() throws Exception
{
runEndpointsWereCachedTest(FakeSimpleStrategy.class, null);
- runEndpointsWereCachedTest(FakeOldNetworkTopologyStrategy.class, null);
runEndpointsWereCachedTest(FakeNetworkTopologyStrategy.class, new HashMap<String, String>());
}
public void runEndpointsWereCachedTest(Class stratClass, Map<String, String> configOptions) throws Exception
{
setup(stratClass, configOptions);
- assert strategy.getNaturalEndpoints(searchToken).equals(strategy.getNaturalEndpoints(searchToken));
+ Util.assertRCEquals(strategy.getNaturalReplicasForToken(searchToken), strategy.getNaturalReplicasForToken(searchToken));
}
@Test
public void testCacheRespectsTokenChanges() throws Exception
{
runCacheRespectsTokenChangesTest(SimpleStrategy.class, null);
- runCacheRespectsTokenChangesTest(OldNetworkTopologyStrategy.class, null);
runCacheRespectsTokenChangesTest(NetworkTopologyStrategy.class, new HashMap<String, String>());
}
public void runCacheRespectsTokenChangesTest(Class stratClass, Map<String, String> configOptions) throws Exception
{
setup(stratClass, configOptions);
- ArrayList<InetAddress> initial;
- ArrayList<InetAddress> endpoints;
+ EndpointsForToken initial;
+ EndpointsForToken replicas;
- endpoints = strategy.getNaturalEndpoints(searchToken);
- assert endpoints.size() == 5 : StringUtils.join(endpoints, ",");
+ replicas = strategy.getNaturalReplicasForToken(searchToken);
+ assert replicas.size() == 5 : StringUtils.join(replicas, ",");
// test token addition, in DC2 before existing token
- initial = strategy.getNaturalEndpoints(searchToken);
- tmd.updateNormalToken(new BigIntegerToken(String.valueOf(35)), InetAddress.getByName("127.0.0.5"));
- endpoints = strategy.getNaturalEndpoints(searchToken);
- assert endpoints.size() == 5 : StringUtils.join(endpoints, ",");
- assert !endpoints.equals(initial);
+ initial = strategy.getNaturalReplicasForToken(searchToken);
+ tmd.updateNormalToken(new BigIntegerToken(String.valueOf(35)), InetAddressAndPort.getByName("127.0.0.5"));
+ replicas = strategy.getNaturalReplicasForToken(searchToken);
+ assert replicas.size() == 5 : StringUtils.join(replicas, ",");
+ Util.assertNotRCEquals(replicas, initial);
// test token removal, newly created token
- initial = strategy.getNaturalEndpoints(searchToken);
- tmd.removeEndpoint(InetAddress.getByName("127.0.0.5"));
- endpoints = strategy.getNaturalEndpoints(searchToken);
- assert endpoints.size() == 5 : StringUtils.join(endpoints, ",");
- assert !endpoints.contains(InetAddress.getByName("127.0.0.5"));
- assert !endpoints.equals(initial);
+ initial = strategy.getNaturalReplicasForToken(searchToken);
+ tmd.removeEndpoint(InetAddressAndPort.getByName("127.0.0.5"));
+ replicas = strategy.getNaturalReplicasForToken(searchToken);
+ assert replicas.size() == 5 : StringUtils.join(replicas, ",");
+ assert !replicas.endpoints().contains(InetAddressAndPort.getByName("127.0.0.5"));
+ Util.assertNotRCEquals(replicas, initial);
// test token change
- initial = strategy.getNaturalEndpoints(searchToken);
+ initial = strategy.getNaturalReplicasForToken(searchToken);
//move .8 after search token but before other DC3
- tmd.updateNormalToken(new BigIntegerToken(String.valueOf(25)), InetAddress.getByName("127.0.0.8"));
- endpoints = strategy.getNaturalEndpoints(searchToken);
- assert endpoints.size() == 5 : StringUtils.join(endpoints, ",");
- assert !endpoints.equals(initial);
+ tmd.updateNormalToken(new BigIntegerToken(String.valueOf(25)), InetAddressAndPort.getByName("127.0.0.8"));
+ replicas = strategy.getNaturalReplicasForToken(searchToken);
+ assert replicas.size() == 5 : StringUtils.join(replicas, ",");
+ Util.assertNotRCEquals(replicas, initial);
}
protected static class FakeSimpleStrategy extends SimpleStrategy
@@ -130,28 +125,11 @@
super(keyspaceName, tokenMetadata, snitch, configOptions);
}
- public List<InetAddress> calculateNaturalEndpoints(Token token, TokenMetadata metadata)
+ public EndpointsForRange calculateNaturalReplicas(Token token, TokenMetadata metadata)
{
- assert !called : "calculateNaturalEndpoints was already called, result should have been cached";
+ assert !called : "calculateNaturalReplicas was already called, result should have been cached";
called = true;
- return super.calculateNaturalEndpoints(token, metadata);
- }
- }
-
- protected static class FakeOldNetworkTopologyStrategy extends OldNetworkTopologyStrategy
- {
- private boolean called = false;
-
- public FakeOldNetworkTopologyStrategy(String keyspaceName, TokenMetadata tokenMetadata, IEndpointSnitch snitch, Map<String, String> configOptions)
- {
- super(keyspaceName, tokenMetadata, snitch, configOptions);
- }
-
- public List<InetAddress> calculateNaturalEndpoints(Token token, TokenMetadata metadata)
- {
- assert !called : "calculateNaturalEndpoints was already called, result should have been cached";
- called = true;
- return super.calculateNaturalEndpoints(token, metadata);
+ return super.calculateNaturalReplicas(token, metadata);
}
}
@@ -164,11 +142,11 @@
super(keyspaceName, tokenMetadata, snitch, configOptions);
}
- public List<InetAddress> calculateNaturalEndpoints(Token token, TokenMetadata metadata)
+ public EndpointsForRange calculateNaturalReplicas(Token token, TokenMetadata metadata)
{
- assert !called : "calculateNaturalEndpoints was already called, result should have been cached";
+ assert !called : "calculateNaturalReplicas was already called, result should have been cached";
called = true;
- return super.calculateNaturalEndpoints(token, metadata);
+ return super.calculateNaturalReplicas(token, metadata);
}
}
diff --git a/test/unit/org/apache/cassandra/locator/SimpleStrategyTest.java b/test/unit/org/apache/cassandra/locator/SimpleStrategyTest.java
index 0955985..9e24de7 100644
--- a/test/unit/org/apache/cassandra/locator/SimpleStrategyTest.java
+++ b/test/unit/org/apache/cassandra/locator/SimpleStrategyTest.java
@@ -17,18 +17,29 @@
*/
package org.apache.cassandra.locator;
-import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.ArrayList;
-import java.util.Collection;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
+import java.util.Map;
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Multimap;
+import org.junit.Before;
import org.junit.BeforeClass;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.ExpectedException;
import org.apache.cassandra.SchemaLoader;
-import org.apache.cassandra.config.Schema;
+import org.apache.cassandra.Util;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.OrderPreservingPartitioner;
@@ -48,12 +59,21 @@
public class SimpleStrategyTest
{
public static final String KEYSPACE1 = "SimpleStrategyTest";
+ public static final String MULTIDC = "MultiDCSimpleStrategyTest";
@BeforeClass
- public static void defineSchema() throws Exception
+ public static void defineSchema()
{
SchemaLoader.prepareServer();
SchemaLoader.createKeyspace(KEYSPACE1, KeyspaceParams.simple(1));
+ SchemaLoader.createKeyspace(MULTIDC, KeyspaceParams.simple(3));
+ DatabaseDescriptor.setTransientReplicationEnabledUnsafe(true);
+ }
+
+ @Before
+ public void resetSnitch()
+ {
+ DatabaseDescriptor.setEndpointSnitch(new SimpleSnitch());
}
@Test
@@ -88,6 +108,47 @@
verifyGetNaturalEndpoints(endpointTokens.toArray(new Token[0]), keyTokens.toArray(new Token[0]));
}
+ @Test
+ public void testMultiDCSimpleStrategyEndpoints() throws UnknownHostException
+ {
+ IEndpointSnitch snitch = new PropertyFileSnitch();
+ DatabaseDescriptor.setEndpointSnitch(snitch);
+
+ TokenMetadata metadata = new TokenMetadata();
+
+ AbstractReplicationStrategy strategy = getStrategy(MULTIDC, metadata, snitch);
+
+ // Topology taken directly from the topology_test.test_size_estimates_multidc dtest that regressed
+ Multimap<InetAddressAndPort, Token> dc1 = HashMultimap.create();
+ dc1.put(InetAddressAndPort.getByName("127.0.0.1"), new Murmur3Partitioner.LongToken(-6639341390736545756L));
+ dc1.put(InetAddressAndPort.getByName("127.0.0.1"), new Murmur3Partitioner.LongToken(-2688160409776496397L));
+ dc1.put(InetAddressAndPort.getByName("127.0.0.2"), new Murmur3Partitioner.LongToken(-2506475074448728501L));
+ dc1.put(InetAddressAndPort.getByName("127.0.0.2"), new Murmur3Partitioner.LongToken(8473270337963525440L));
+ metadata.updateNormalTokens(dc1);
+
+ Multimap<InetAddressAndPort, Token> dc2 = HashMultimap.create();
+ dc2.put(InetAddressAndPort.getByName("127.0.0.4"), new Murmur3Partitioner.LongToken(-3736333188524231709L));
+ dc2.put(InetAddressAndPort.getByName("127.0.0.4"), new Murmur3Partitioner.LongToken(8673615181726552074L));
+ metadata.updateNormalTokens(dc2);
+
+ Map<InetAddressAndPort, Integer> primaryCount = new HashMap<>();
+ Map<InetAddressAndPort, Integer> replicaCount = new HashMap<>();
+ for (Token t : metadata.sortedTokens())
+ {
+ EndpointsForToken replicas = strategy.getNaturalReplicasForToken(t);
+ primaryCount.compute(replicas.get(0).endpoint(), (k, v) -> (v == null) ? 1 : v + 1);
+ for (Replica replica : replicas)
+ replicaCount.compute(replica.endpoint(), (k, v) -> (v == null) ? 1 : v + 1);
+ }
+
+ // All three hosts should have 2 "primary" replica ranges and 6 total ranges with RF=3, 3 nodes and 2 DCs.
+ for (InetAddressAndPort addr : primaryCount.keySet())
+ {
+ assertEquals(2, (int) primaryCount.get(addr));
+ assertEquals(6, (int) replicaCount.get(addr));
+ }
+ }
+
// given a list of endpoint tokens, and a set of key tokens falling between the endpoint tokens,
// make sure that the Strategy picks the right endpoints for the keys.
private void verifyGetNaturalEndpoints(Token[] endpointTokens, Token[] keyTokens) throws UnknownHostException
@@ -97,23 +158,23 @@
for (String keyspaceName : Schema.instance.getNonLocalStrategyKeyspaces())
{
tmd = new TokenMetadata();
- strategy = getStrategy(keyspaceName, tmd);
- List<InetAddress> hosts = new ArrayList<InetAddress>();
+ strategy = getStrategy(keyspaceName, tmd, new SimpleSnitch());
+ List<InetAddressAndPort> hosts = new ArrayList<>();
for (int i = 0; i < endpointTokens.length; i++)
{
- InetAddress ep = InetAddress.getByName("127.0.0." + String.valueOf(i + 1));
+ InetAddressAndPort ep = InetAddressAndPort.getByName("127.0.0." + String.valueOf(i + 1));
tmd.updateNormalToken(endpointTokens[i], ep);
hosts.add(ep);
}
for (int i = 0; i < keyTokens.length; i++)
{
- List<InetAddress> endpoints = strategy.getNaturalEndpoints(keyTokens[i]);
- assertEquals(strategy.getReplicationFactor(), endpoints.size());
- List<InetAddress> correctEndpoints = new ArrayList<InetAddress>();
- for (int j = 0; j < endpoints.size(); j++)
+ EndpointsForToken replicas = strategy.getNaturalReplicasForToken(keyTokens[i]);
+ assertEquals(strategy.getReplicationFactor().allReplicas, replicas.size());
+ List<InetAddressAndPort> correctEndpoints = new ArrayList<>();
+ for (int j = 0; j < replicas.size(); j++)
correctEndpoints.add(hosts.get((i + j + 1) % hosts.size()));
- assertEquals(new HashSet<InetAddress>(correctEndpoints), new HashSet<InetAddress>(endpoints));
+ assertEquals(new HashSet<>(correctEndpoints), replicas.endpoints());
}
}
}
@@ -135,58 +196,137 @@
keyTokens[i] = new BigIntegerToken(String.valueOf(RING_SIZE * 2 * i + RING_SIZE));
}
- List<InetAddress> hosts = new ArrayList<InetAddress>();
+ List<InetAddressAndPort> hosts = new ArrayList<>();
for (int i = 0; i < endpointTokens.length; i++)
{
- InetAddress ep = InetAddress.getByName("127.0.0." + String.valueOf(i + 1));
+ InetAddressAndPort ep = InetAddressAndPort.getByName("127.0.0." + String.valueOf(i + 1));
tmd.updateNormalToken(endpointTokens[i], ep);
hosts.add(ep);
}
// bootstrap at the end of the ring
Token bsToken = new BigIntegerToken(String.valueOf(210));
- InetAddress bootstrapEndpoint = InetAddress.getByName("127.0.0.11");
+ InetAddressAndPort bootstrapEndpoint = InetAddressAndPort.getByName("127.0.0.11");
tmd.addBootstrapToken(bsToken, bootstrapEndpoint);
AbstractReplicationStrategy strategy = null;
for (String keyspaceName : Schema.instance.getNonLocalStrategyKeyspaces())
{
- strategy = getStrategy(keyspaceName, tmd);
+ strategy = getStrategy(keyspaceName, tmd, new SimpleSnitch());
PendingRangeCalculatorService.calculatePendingRanges(strategy, keyspaceName);
- int replicationFactor = strategy.getReplicationFactor();
+ int replicationFactor = strategy.getReplicationFactor().allReplicas;
for (int i = 0; i < keyTokens.length; i++)
{
- Collection<InetAddress> endpoints = tmd.getWriteEndpoints(keyTokens[i], keyspaceName, strategy.getNaturalEndpoints(keyTokens[i]));
- assertTrue(endpoints.size() >= replicationFactor);
+ EndpointsForToken replicas = tmd.getWriteEndpoints(keyTokens[i], keyspaceName, strategy.getNaturalReplicasForToken(keyTokens[i]));
+ assertTrue(replicas.size() >= replicationFactor);
for (int j = 0; j < replicationFactor; j++)
{
//Check that the old nodes are definitely included
- assertTrue(endpoints.contains(hosts.get((i + j + 1) % hosts.size())));
+ assertTrue(replicas.endpoints().contains(hosts.get((i + j + 1) % hosts.size())));
}
// bootstrapEndpoint should be in the endpoints for i in MAX-RF to MAX, but not in any earlier ep.
if (i < RING_SIZE - replicationFactor)
- assertFalse(endpoints.contains(bootstrapEndpoint));
+ assertFalse(replicas.endpoints().contains(bootstrapEndpoint));
else
- assertTrue(endpoints.contains(bootstrapEndpoint));
+ assertTrue(replicas.endpoints().contains(bootstrapEndpoint));
}
}
StorageServiceAccessor.setTokenMetadata(oldTmd);
}
- private AbstractReplicationStrategy getStrategy(String keyspaceName, TokenMetadata tmd)
+ private static Token tk(long t)
{
- KeyspaceMetadata ksmd = Schema.instance.getKSMetaData(keyspaceName);
+ return new Murmur3Partitioner.LongToken(t);
+ }
+
+ private static Range<Token> range(long l, long r)
+ {
+ return new Range<>(tk(l), tk(r));
+ }
+
+ @Test
+ public void transientReplica() throws Exception
+ {
+ IEndpointSnitch snitch = new SimpleSnitch();
+ DatabaseDescriptor.setEndpointSnitch(snitch);
+
+ List<InetAddressAndPort> endpoints = Lists.newArrayList(InetAddressAndPort.getByName("127.0.0.1"),
+ InetAddressAndPort.getByName("127.0.0.2"),
+ InetAddressAndPort.getByName("127.0.0.3"),
+ InetAddressAndPort.getByName("127.0.0.4"));
+
+ Multimap<InetAddressAndPort, Token> tokens = HashMultimap.create();
+ tokens.put(endpoints.get(0), tk(100));
+ tokens.put(endpoints.get(1), tk(200));
+ tokens.put(endpoints.get(2), tk(300));
+ tokens.put(endpoints.get(3), tk(400));
+ TokenMetadata metadata = new TokenMetadata();
+ metadata.updateNormalTokens(tokens);
+
+ Map<String, String> configOptions = new HashMap<String, String>();
+ configOptions.put("replication_factor", "3/1");
+
+ SimpleStrategy strategy = new SimpleStrategy("ks", metadata, snitch, configOptions);
+
+ Range<Token> range1 = range(400, 100);
+ Util.assertRCEquals(EndpointsForToken.of(range1.right,
+ Replica.fullReplica(endpoints.get(0), range1),
+ Replica.fullReplica(endpoints.get(1), range1),
+ Replica.transientReplica(endpoints.get(2), range1)),
+ strategy.getNaturalReplicasForToken(tk(99)));
+
+
+ Range<Token> range2 = range(100, 200);
+ Util.assertRCEquals(EndpointsForToken.of(range2.right,
+ Replica.fullReplica(endpoints.get(1), range2),
+ Replica.fullReplica(endpoints.get(2), range2),
+ Replica.transientReplica(endpoints.get(3), range2)),
+ strategy.getNaturalReplicasForToken(tk(101)));
+ }
+
+ @Rule
+ public ExpectedException expectedEx = ExpectedException.none();
+
+ @Test
+ public void testSimpleStrategyThrowsConfigurationException() throws ConfigurationException, UnknownHostException
+ {
+ expectedEx.expect(ConfigurationException.class);
+ expectedEx.expectMessage("SimpleStrategy requires a replication_factor strategy option.");
+
+ IEndpointSnitch snitch = new SimpleSnitch();
+ DatabaseDescriptor.setEndpointSnitch(snitch);
+
+ List<InetAddressAndPort> endpoints = Lists.newArrayList(InetAddressAndPort.getByName("127.0.0.1"),
+ InetAddressAndPort.getByName("127.0.0.2"),
+ InetAddressAndPort.getByName("127.0.0.3"));
+
+ Multimap<InetAddressAndPort, Token> tokens = HashMultimap.create();
+ tokens.put(endpoints.get(0), tk(100));
+ tokens.put(endpoints.get(1), tk(200));
+ tokens.put(endpoints.get(2), tk(300));
+
+ TokenMetadata metadata = new TokenMetadata();
+ metadata.updateNormalTokens(tokens);
+
+ Map<String, String> configOptions = new HashMap<>();
+
+ SimpleStrategy strategy = new SimpleStrategy("ks", metadata, snitch, configOptions);
+ }
+
+ private AbstractReplicationStrategy getStrategy(String keyspaceName, TokenMetadata tmd, IEndpointSnitch snitch)
+ {
+ KeyspaceMetadata ksmd = Schema.instance.getKeyspaceMetadata(keyspaceName);
return AbstractReplicationStrategy.createReplicationStrategy(
keyspaceName,
ksmd.params.replication.klass,
tmd,
- new SimpleSnitch(),
+ snitch,
ksmd.params.replication.options);
}
}
diff --git a/test/unit/org/apache/cassandra/locator/TokenMetadataTest.java b/test/unit/org/apache/cassandra/locator/TokenMetadataTest.java
index b462723..fb3f2bd 100644
--- a/test/unit/org/apache/cassandra/locator/TokenMetadataTest.java
+++ b/test/unit/org/apache/cassandra/locator/TokenMetadataTest.java
@@ -17,10 +17,10 @@
*/
package org.apache.cassandra.locator;
-import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.Map;
+import java.util.UUID;
import com.google.common.collect.ImmutableMultimap;
import com.google.common.collect.Iterators;
@@ -56,8 +56,8 @@
{
DatabaseDescriptor.daemonInitialization();
tmd = StorageService.instance.getTokenMetadata();
- tmd.updateNormalToken(token(ONE), InetAddress.getByName("127.0.0.1"));
- tmd.updateNormalToken(token(SIX), InetAddress.getByName("127.0.0.6"));
+ tmd.updateNormalToken(token(ONE), InetAddressAndPort.getByName("127.0.0.1"));
+ tmd.updateNormalToken(token(SIX), InetAddressAndPort.getByName("127.0.0.6"));
}
private static void testRingIterator(ArrayList<Token> ring, String start, boolean includeMin, String... expected)
@@ -98,8 +98,8 @@
@Test
public void testTopologyUpdate_RackConsolidation() throws UnknownHostException
{
- final InetAddress first = InetAddress.getByName("127.0.0.1");
- final InetAddress second = InetAddress.getByName("127.0.0.6");
+ final InetAddressAndPort first = InetAddressAndPort.getByName("127.0.0.1");
+ final InetAddressAndPort second = InetAddressAndPort.getByName("127.0.0.6");
final String DATA_CENTER = "datacenter1";
final String RACK1 = "rack1";
final String RACK2 = "rack2";
@@ -107,19 +107,19 @@
DatabaseDescriptor.setEndpointSnitch(new AbstractEndpointSnitch()
{
@Override
- public String getRack(InetAddress endpoint)
+ public String getRack(InetAddressAndPort endpoint)
{
return endpoint.equals(first) ? RACK1 : RACK2;
}
@Override
- public String getDatacenter(InetAddress endpoint)
+ public String getDatacenter(InetAddressAndPort endpoint)
{
return DATA_CENTER;
}
@Override
- public int compareEndpoints(InetAddress target, InetAddress a1, InetAddress a2)
+ public int compareEndpoints(InetAddressAndPort target, Replica a1, Replica a2)
{
return 0;
}
@@ -134,14 +134,14 @@
TokenMetadata.Topology topology = tokenMetadata.getTopology();
assertNotNull(topology);
- Multimap<String, InetAddress> allEndpoints = topology.getDatacenterEndpoints();
+ Multimap<String, InetAddressAndPort> allEndpoints = topology.getDatacenterEndpoints();
assertNotNull(allEndpoints);
assertTrue(allEndpoints.size() == 2);
assertTrue(allEndpoints.containsKey(DATA_CENTER));
assertTrue(allEndpoints.get(DATA_CENTER).contains(first));
assertTrue(allEndpoints.get(DATA_CENTER).contains(second));
- Map<String, ImmutableMultimap<String, InetAddress>> racks = topology.getDatacenterRacks();
+ Map<String, ImmutableMultimap<String, InetAddressAndPort>> racks = topology.getDatacenterRacks();
assertNotNull(racks);
assertTrue(racks.size() == 1);
assertTrue(racks.containsKey(DATA_CENTER));
@@ -154,19 +154,19 @@
DatabaseDescriptor.setEndpointSnitch(new AbstractEndpointSnitch()
{
@Override
- public String getRack(InetAddress endpoint)
+ public String getRack(InetAddressAndPort endpoint)
{
return RACK1;
}
@Override
- public String getDatacenter(InetAddress endpoint)
+ public String getDatacenter(InetAddressAndPort endpoint)
{
return DATA_CENTER;
}
@Override
- public int compareEndpoints(InetAddress target, InetAddress a1, InetAddress a2)
+ public int compareEndpoints(InetAddressAndPort target, Replica a1, Replica a2)
{
return 0;
}
@@ -196,8 +196,8 @@
@Test
public void testTopologyUpdate_RackExpansion() throws UnknownHostException
{
- final InetAddress first = InetAddress.getByName("127.0.0.1");
- final InetAddress second = InetAddress.getByName("127.0.0.6");
+ final InetAddressAndPort first = InetAddressAndPort.getByName("127.0.0.1");
+ final InetAddressAndPort second = InetAddressAndPort.getByName("127.0.0.6");
final String DATA_CENTER = "datacenter1";
final String RACK1 = "rack1";
final String RACK2 = "rack2";
@@ -205,19 +205,19 @@
DatabaseDescriptor.setEndpointSnitch(new AbstractEndpointSnitch()
{
@Override
- public String getRack(InetAddress endpoint)
+ public String getRack(InetAddressAndPort endpoint)
{
return RACK1;
}
@Override
- public String getDatacenter(InetAddress endpoint)
+ public String getDatacenter(InetAddressAndPort endpoint)
{
return DATA_CENTER;
}
@Override
- public int compareEndpoints(InetAddress target, InetAddress a1, InetAddress a2)
+ public int compareEndpoints(InetAddressAndPort target, Replica a1, Replica a2)
{
return 0;
}
@@ -232,14 +232,14 @@
TokenMetadata.Topology topology = tokenMetadata.getTopology();
assertNotNull(topology);
- Multimap<String, InetAddress> allEndpoints = topology.getDatacenterEndpoints();
+ Multimap<String, InetAddressAndPort> allEndpoints = topology.getDatacenterEndpoints();
assertNotNull(allEndpoints);
assertTrue(allEndpoints.size() == 2);
assertTrue(allEndpoints.containsKey(DATA_CENTER));
assertTrue(allEndpoints.get(DATA_CENTER).contains(first));
assertTrue(allEndpoints.get(DATA_CENTER).contains(second));
- Map<String, ImmutableMultimap<String, InetAddress>> racks = topology.getDatacenterRacks();
+ Map<String, ImmutableMultimap<String, InetAddressAndPort>> racks = topology.getDatacenterRacks();
assertNotNull(racks);
assertTrue(racks.size() == 1);
assertTrue(racks.containsKey(DATA_CENTER));
@@ -252,19 +252,19 @@
DatabaseDescriptor.setEndpointSnitch(new AbstractEndpointSnitch()
{
@Override
- public String getRack(InetAddress endpoint)
+ public String getRack(InetAddressAndPort endpoint)
{
return endpoint.equals(first) ? RACK1 : RACK2;
}
@Override
- public String getDatacenter(InetAddress endpoint)
+ public String getDatacenter(InetAddressAndPort endpoint)
{
return DATA_CENTER;
}
@Override
- public int compareEndpoints(InetAddress target, InetAddress a1, InetAddress a2)
+ public int compareEndpoints(InetAddressAndPort target, Replica a1, Replica a2)
{
return 0;
}
@@ -289,4 +289,42 @@
assertTrue(racks.get(DATA_CENTER).get(RACK1).contains(first));
assertTrue(racks.get(DATA_CENTER).get(RACK2).contains(second));
}
+
+ @Test
+ public void testEndpointSizes() throws UnknownHostException
+ {
+ final InetAddressAndPort first = InetAddressAndPort.getByName("127.0.0.1");
+ final InetAddressAndPort second = InetAddressAndPort.getByName("127.0.0.6");
+
+ tmd.updateNormalToken(token(ONE), first);
+ tmd.updateNormalToken(token(SIX), second);
+
+ TokenMetadata tokenMetadata = tmd.cloneOnlyTokenMap();
+ assertNotNull(tokenMetadata);
+
+ tokenMetadata.updateHostId(UUID.randomUUID(), first);
+ tokenMetadata.updateHostId(UUID.randomUUID(), second);
+
+ assertEquals(2, tokenMetadata.getSizeOfAllEndpoints());
+ assertEquals(0, tokenMetadata.getSizeOfLeavingEndpoints());
+ assertEquals(0, tokenMetadata.getSizeOfMovingEndpoints());
+
+ tokenMetadata.addLeavingEndpoint(first);
+ assertEquals(1, tokenMetadata.getSizeOfLeavingEndpoints());
+
+ tokenMetadata.removeEndpoint(first);
+ assertEquals(0, tokenMetadata.getSizeOfLeavingEndpoints());
+ assertEquals(1, tokenMetadata.getSizeOfAllEndpoints());
+
+ tokenMetadata.addMovingEndpoint(token(SIX), second);
+ assertEquals(1, tokenMetadata.getSizeOfMovingEndpoints());
+
+ tokenMetadata.removeFromMoving(second);
+ assertEquals(0, tokenMetadata.getSizeOfMovingEndpoints());
+
+ tokenMetadata.removeEndpoint(second);
+ assertEquals(0, tokenMetadata.getSizeOfAllEndpoints());
+ assertEquals(0, tokenMetadata.getSizeOfLeavingEndpoints());
+ assertEquals(0, tokenMetadata.getSizeOfMovingEndpoints());
+ }
}
diff --git a/test/unit/org/apache/cassandra/metrics/BatchMetricsTest.java b/test/unit/org/apache/cassandra/metrics/BatchMetricsTest.java
new file mode 100644
index 0000000..c3bf794
--- /dev/null
+++ b/test/unit/org/apache/cassandra/metrics/BatchMetricsTest.java
@@ -0,0 +1,198 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.metrics;
+
+import java.io.IOException;
+import java.util.concurrent.TimeUnit;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+import com.datastax.driver.core.BatchStatement;
+import com.datastax.driver.core.Cluster;
+import com.datastax.driver.core.PreparedStatement;
+import com.datastax.driver.core.Session;
+import org.apache.cassandra.OrderedJUnit4ClassRunner;
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.service.EmbeddedCassandraService;
+import org.apache.cassandra.metrics.DecayingEstimatedHistogramReservoir.EstimatedHistogramReservoirSnapshot;
+
+import static org.apache.cassandra.cql3.statements.BatchStatement.metrics;
+import static org.apache.cassandra.metrics.DecayingEstimatedHistogramReservoir.*;
+import static org.junit.Assert.assertEquals;
+import static org.quicktheories.QuickTheory.qt;
+import static org.quicktheories.generators.Generate.intArrays;
+import static org.quicktheories.generators.SourceDSL.integers;
+
+@RunWith(OrderedJUnit4ClassRunner.class)
+public class BatchMetricsTest extends SchemaLoader
+{
+ private static final int MAX_ROUNDS_TO_PERFORM = 3;
+ private static final int MAX_DISTINCT_PARTITIONS = 128;
+ private static final int MAX_STATEMENTS_PER_ROUND = 32;
+
+ private static EmbeddedCassandraService cassandra;
+
+ private static Cluster cluster;
+ private static Session session;
+
+ private static String KEYSPACE = "junit";
+ private static final String LOGGER_TABLE = "loggerbatchmetricstest";
+ private static final String COUNTER_TABLE = "counterbatchmetricstest";
+
+ private static PreparedStatement psLogger;
+ private static PreparedStatement psCounter;
+
+ @BeforeClass()
+ public static void setup() throws ConfigurationException, IOException
+ {
+ Schema.instance.clear();
+
+ cassandra = new EmbeddedCassandraService();
+ cassandra.start();
+
+ DatabaseDescriptor.setWriteRpcTimeout(TimeUnit.SECONDS.toMillis(10));
+
+ cluster = Cluster.builder().addContactPoint("127.0.0.1").withPort(DatabaseDescriptor.getNativeTransportPort()).build();
+ session = cluster.connect();
+
+ session.execute("CREATE KEYSPACE IF NOT EXISTS " + KEYSPACE + " WITH replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 };");
+ session.execute("USE " + KEYSPACE);
+ session.execute("CREATE TABLE IF NOT EXISTS " + LOGGER_TABLE + " (id int PRIMARY KEY, val text);");
+ session.execute("CREATE TABLE IF NOT EXISTS " + COUNTER_TABLE + " (id int PRIMARY KEY, val counter);");
+
+ psLogger = session.prepare("INSERT INTO " + KEYSPACE + '.' + LOGGER_TABLE + " (id, val) VALUES (?, ?);");
+ psCounter = session.prepare("UPDATE " + KEYSPACE + '.' + COUNTER_TABLE + " SET val = val + 1 WHERE id = ?;");
+ }
+
+ private void executeLoggerBatch(BatchStatement.Type batchStatementType, int distinctPartitions, int statementsPerPartition)
+ {
+ BatchStatement batch = new BatchStatement(batchStatementType);
+
+ for (int i = 0; i < distinctPartitions; i++)
+ {
+ for (int j = 0; j < statementsPerPartition; j++)
+ {
+ if (batchStatementType == BatchStatement.Type.UNLOGGED || batchStatementType == BatchStatement.Type.LOGGED)
+ batch.add(psLogger.bind(i, "aaaaaaaa"));
+ else if (batchStatementType == BatchStatement.Type.COUNTER)
+ batch.add(psCounter.bind(i));
+ else
+ throw new IllegalStateException("There is no a case for BatchStatement.Type." + batchStatementType.name());
+ }
+ }
+
+ session.execute(batch);
+ }
+
+ @Test
+ public void testLoggedPartitionsPerBatch()
+ {
+ qt().withExamples(25)
+ .forAll(intArrays(integers().between(1, MAX_ROUNDS_TO_PERFORM),
+ integers().between(1, MAX_STATEMENTS_PER_ROUND)),
+ integers().between(1, MAX_DISTINCT_PARTITIONS))
+ .checkAssert((rounds, distinctPartitions) ->
+ assertMetrics(BatchStatement.Type.LOGGED, rounds, distinctPartitions));
+ }
+
+ @Test
+ public void testUnloggedPartitionsPerBatch()
+ {
+ qt().withExamples(25)
+ .forAll(intArrays(integers().between(1, MAX_ROUNDS_TO_PERFORM),
+ integers().between(1, MAX_STATEMENTS_PER_ROUND)),
+ integers().between(1, MAX_DISTINCT_PARTITIONS))
+ .checkAssert((rounds, distinctPartitions) ->
+ assertMetrics(BatchStatement.Type.UNLOGGED, rounds, distinctPartitions));
+ }
+
+ @Test
+ public void testCounterPartitionsPerBatch()
+ {
+ qt().withExamples(10)
+ .forAll(intArrays(integers().between(1, MAX_ROUNDS_TO_PERFORM),
+ integers().between(1, MAX_STATEMENTS_PER_ROUND)),
+ integers().between(1, MAX_DISTINCT_PARTITIONS))
+ .checkAssert((rounds, distinctPartitions) ->
+ assertMetrics(BatchStatement.Type.COUNTER, rounds, distinctPartitions));
+ }
+
+ private void assertMetrics(BatchStatement.Type batchTypeTested, int[] rounds, int distinctPartitions)
+ {
+ // reset the histogram between runs
+ clearHistogram();
+
+ // roundsOfStatementsPerPartition - array length is the number of rounds to executeLoggerBatch() and each
+ // value in the array represents the number of statements to execute per partition on that round
+ for (int ix = 0; ix < rounds.length; ix++)
+ {
+ long partitionsPerLoggedBatchCountPre = metrics.partitionsPerLoggedBatch.getCount();
+ long expectedPartitionsPerLoggedBatchCount = partitionsPerLoggedBatchCountPre + (batchTypeTested == BatchStatement.Type.LOGGED ? 1 : 0);
+ long partitionsPerUnloggedBatchCountPre = metrics.partitionsPerUnloggedBatch.getCount();
+ long expectedPartitionsPerUnloggedBatchCount = partitionsPerUnloggedBatchCountPre + (batchTypeTested == BatchStatement.Type.UNLOGGED ? 1 : 0);
+ long partitionsPerCounterBatchCountPre = metrics.partitionsPerCounterBatch.getCount();
+ long expectedPartitionsPerCounterBatchCount = partitionsPerCounterBatchCountPre + (batchTypeTested == BatchStatement.Type.COUNTER ? 1 : 0);
+
+ executeLoggerBatch(batchTypeTested, distinctPartitions, rounds[ix]);
+
+ assertEquals(expectedPartitionsPerUnloggedBatchCount, metrics.partitionsPerUnloggedBatch.getCount());
+ assertEquals(expectedPartitionsPerLoggedBatchCount, metrics.partitionsPerLoggedBatch.getCount());
+ assertEquals(expectedPartitionsPerCounterBatchCount, metrics.partitionsPerCounterBatch.getCount());
+
+ EstimatedHistogramReservoirSnapshot partitionsPerLoggedBatchSnapshot = (EstimatedHistogramReservoirSnapshot) metrics.partitionsPerLoggedBatch.getSnapshot();
+ EstimatedHistogramReservoirSnapshot partitionsPerUnloggedBatchSnapshot = (EstimatedHistogramReservoirSnapshot) metrics.partitionsPerUnloggedBatch.getSnapshot();
+ EstimatedHistogramReservoirSnapshot partitionsPerCounterBatchSnapshot = (EstimatedHistogramReservoirSnapshot) metrics.partitionsPerCounterBatch.getSnapshot();
+
+ // BatchMetrics uses DecayingEstimatedHistogramReservoir which notes that the return of getMax()
+ // may be more than the actual max value recorded in the reservoir with similar but reverse properties
+ // for getMin(). uses getBucketingForValue() on the snapshot to identify the exact max. since the
+ // distinctPartitions doesn't change per test round these values shouldn't change.
+ Range expectedPartitionsPerLoggedBatchMinMax = batchTypeTested == BatchStatement.Type.LOGGED ?
+ determineExpectedMinMax(partitionsPerLoggedBatchSnapshot, distinctPartitions) :
+ new Range(0L, 0L);
+ Range expectedPartitionsPerUnloggedBatchMinMax = batchTypeTested == BatchStatement.Type.UNLOGGED ?
+ determineExpectedMinMax(partitionsPerUnloggedBatchSnapshot, distinctPartitions) :
+ new Range(0L, 0L);
+ Range expectedPartitionsPerCounterBatchMinMax = batchTypeTested == BatchStatement.Type.COUNTER ?
+ determineExpectedMinMax(partitionsPerCounterBatchSnapshot, distinctPartitions) :
+ new Range(0L, 0L);
+
+ assertEquals(expectedPartitionsPerLoggedBatchMinMax, new Range(partitionsPerLoggedBatchSnapshot.getMin(), partitionsPerLoggedBatchSnapshot.getMax()));
+ assertEquals(expectedPartitionsPerUnloggedBatchMinMax, new Range(partitionsPerUnloggedBatchSnapshot.getMin(), partitionsPerUnloggedBatchSnapshot.getMax()));
+ assertEquals(expectedPartitionsPerCounterBatchMinMax, new Range(partitionsPerCounterBatchSnapshot.getMin(), partitionsPerCounterBatchSnapshot.getMax()));
+ }
+ }
+
+ private void clearHistogram()
+ {
+ ((ClearableHistogram) metrics.partitionsPerLoggedBatch).clear();
+ ((ClearableHistogram) metrics.partitionsPerUnloggedBatch).clear();
+ ((ClearableHistogram) metrics.partitionsPerCounterBatch).clear();
+ }
+
+ private Range determineExpectedMinMax(EstimatedHistogramReservoirSnapshot snapshot, long value)
+ {
+ return snapshot.getBucketingRangeForValue(value);
+ }
+}
diff --git a/test/unit/org/apache/cassandra/metrics/BufferPoolMetricsTest.java b/test/unit/org/apache/cassandra/metrics/BufferPoolMetricsTest.java
new file mode 100644
index 0000000..fade96c
--- /dev/null
+++ b/test/unit/org/apache/cassandra/metrics/BufferPoolMetricsTest.java
@@ -0,0 +1,192 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.metrics;
+
+import java.util.Random;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+import org.apache.cassandra.OrderedJUnit4ClassRunner;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.io.compress.BufferType;
+import org.apache.cassandra.utils.memory.BufferPool;
+import org.apache.cassandra.utils.memory.BufferPoolTest;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatExceptionOfType;
+import static org.junit.Assert.assertEquals;
+
+@RunWith(OrderedJUnit4ClassRunner.class)
+public class BufferPoolMetricsTest
+{
+ private static final BufferPoolMetrics metrics = new BufferPoolMetrics();
+
+ @BeforeClass()
+ public static void setup() throws ConfigurationException
+ {
+ DatabaseDescriptor.daemonInitialization();
+ }
+
+ @Before
+ public void setUp()
+ {
+ BufferPool.MEMORY_USAGE_THRESHOLD = 16 * 1024L * 1024L;
+ }
+
+ @After
+ public void cleanUp()
+ {
+ BufferPoolTest.resetBufferPool();
+ metrics.misses.mark(metrics.misses.getCount() * -1);
+ }
+
+ @Test
+ public void testMetricsSize()
+ {
+ // basically want to test changes in the metric being reported as the buffer pool grows - starts at zero
+ assertThat(metrics.size.getValue()).isEqualTo(BufferPool.sizeInBytes())
+ .isEqualTo(0);
+
+ // the idea is to test changes in the sizeOfBufferPool metric which starts at zero. it will bump up
+ // after the first request for a ByteBuffer and the idea from there will be to keep requesting them
+ // until it bumps a second time at which point there should be some confidence that thie metric is
+ // behaving as expected. these assertions should occur well within the value of the MEMORY_USAGE_THRESHOLD
+ // given the maxBufferSize (just covering the case of the weirdest random seed in the multiverse i guess - a
+ // while loop might have sufficed as well but a definitive termination seemed nicer)
+ final long seed = System.currentTimeMillis();
+ final Random rand = new Random(seed);
+ final String assertionMessage = String.format("Failed with seed of %s", seed);
+ final long maxIterations = BufferPool.MEMORY_USAGE_THRESHOLD;
+ final int maxBufferSize = BufferPool.NORMAL_CHUNK_SIZE - 1;
+ int nextSizeToRequest;
+ long totalBytesRequestedFromPool = 0;
+ long initialSizeInBytesAfterZero = 0;
+ boolean exitedBeforeMax = false;
+ for (int ix = 0; ix < maxIterations; ix++)
+ {
+ nextSizeToRequest = rand.nextInt(maxBufferSize) + 1;
+ totalBytesRequestedFromPool = totalBytesRequestedFromPool + nextSizeToRequest;
+ BufferPool.get(nextSizeToRequest, BufferType.OFF_HEAP);
+
+ assertThat(metrics.size.getValue()).as(assertionMessage)
+ .isEqualTo(BufferPool.sizeInBytes())
+ .isGreaterThanOrEqualTo(totalBytesRequestedFromPool);
+
+ if (initialSizeInBytesAfterZero == 0)
+ {
+ initialSizeInBytesAfterZero = BufferPool.sizeInBytes();
+ }
+ else
+ {
+ // when the total bytes requested from the pool exceeds the initial size we should have
+ // asserted a bump in the sizeInBytes which means that we've asserted the metric increasing
+ // as a result of that bump - can stop trying to grow the pool further
+ if (totalBytesRequestedFromPool > initialSizeInBytesAfterZero)
+ {
+ exitedBeforeMax = true;
+ break;
+ }
+ }
+ }
+
+ assertThat(exitedBeforeMax).as(assertionMessage).isTrue();
+ assertEquals(0, metrics.misses.getCount());
+ }
+
+ @Test
+ public void testMetricsMisses()
+ {
+ assertEquals(0, metrics.misses.getCount());
+
+ final int tinyBufferSizeThatHits = BufferPool.NORMAL_CHUNK_SIZE - 1;
+ final int bigBufferSizeThatMisses = BufferPool.NORMAL_CHUNK_SIZE + 1;
+
+ int iterations = 16;
+ for (int ix = 0; ix < iterations; ix++)
+ {
+ BufferPool.get(tinyBufferSizeThatHits, BufferType.OFF_HEAP);
+ assertEquals(0, metrics.misses.getCount());
+ }
+
+ for (int ix = 0; ix < iterations; ix++)
+ {
+ BufferPool.get(bigBufferSizeThatMisses + ix, BufferType.OFF_HEAP);
+ assertEquals(ix + 1, metrics.misses.getCount());
+ }
+ }
+
+ @Test
+ public void testZeroSizeRequestsDontChangeMetrics()
+ {
+ assertEquals(0, metrics.misses.getCount());
+ assertThat(metrics.size.getValue()).isEqualTo(BufferPool.sizeInBytes())
+ .isEqualTo(0);
+
+ BufferPool.get(0, BufferType.OFF_HEAP);
+
+ assertEquals(0, metrics.misses.getCount());
+ assertThat(metrics.size.getValue()).isEqualTo(BufferPool.sizeInBytes())
+ .isEqualTo(0);
+
+ BufferPool.get(65536, BufferType.OFF_HEAP);
+ BufferPool.get(0, BufferType.OFF_HEAP);
+ BufferPool.get(0, BufferType.OFF_HEAP);
+ BufferPool.get(0, BufferType.OFF_HEAP);
+ BufferPool.get(0, BufferType.OFF_HEAP);
+
+ assertEquals(0, metrics.misses.getCount());
+ assertThat(metrics.size.getValue()).isEqualTo(BufferPool.sizeInBytes())
+ .isGreaterThanOrEqualTo(65536);
+ }
+
+ @Test
+ public void testFailedRequestsDontChangeMetrics()
+ {
+ assertEquals(0, metrics.misses.getCount());
+ assertThat(metrics.size.getValue()).isEqualTo(BufferPool.sizeInBytes())
+ .isEqualTo(0);
+
+ tryRequestNegativeBufferSize();
+
+ assertEquals(0, metrics.misses.getCount());
+ assertThat(metrics.size.getValue()).isEqualTo(BufferPool.sizeInBytes())
+ .isEqualTo(0);
+
+ BufferPool.get(65536, BufferType.OFF_HEAP);
+ tryRequestNegativeBufferSize();
+ tryRequestNegativeBufferSize();
+ tryRequestNegativeBufferSize();
+ tryRequestNegativeBufferSize();
+
+ assertEquals(0, metrics.misses.getCount());
+ assertThat(metrics.size.getValue()).isEqualTo(BufferPool.sizeInBytes())
+ .isGreaterThanOrEqualTo(65536);
+ }
+
+ private void tryRequestNegativeBufferSize()
+ {
+ assertThatExceptionOfType(IllegalArgumentException.class).isThrownBy(
+ () -> BufferPool.get(-1, BufferType.OFF_HEAP));
+ }
+}
diff --git a/test/unit/org/apache/cassandra/metrics/CQLMetricsTest.java b/test/unit/org/apache/cassandra/metrics/CQLMetricsTest.java
index 099a530..e186998 100644
--- a/test/unit/org/apache/cassandra/metrics/CQLMetricsTest.java
+++ b/test/unit/org/apache/cassandra/metrics/CQLMetricsTest.java
@@ -30,7 +30,7 @@
import org.apache.cassandra.OrderedJUnit4ClassRunner;
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.config.Schema;
+import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.cql3.QueryProcessor;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.service.EmbeddedCassandraService;
diff --git a/test/unit/org/apache/cassandra/metrics/CassandraMetricsRegistryTest.java b/test/unit/org/apache/cassandra/metrics/CassandraMetricsRegistryTest.java
index 4a9b874..cd9866c 100644
--- a/test/unit/org/apache/cassandra/metrics/CassandraMetricsRegistryTest.java
+++ b/test/unit/org/apache/cassandra/metrics/CassandraMetricsRegistryTest.java
@@ -20,17 +20,17 @@
*/
package org.apache.cassandra.metrics;
+import static org.junit.Assert.*;
+
import java.lang.management.ManagementFactory;
import java.util.Collection;
+import org.apache.cassandra.metrics.CassandraMetricsRegistry.MetricName;
import org.junit.Test;
import com.codahale.metrics.jvm.BufferPoolMetricSet;
import com.codahale.metrics.jvm.GarbageCollectorMetricSet;
import com.codahale.metrics.jvm.MemoryUsageGaugeSet;
-import org.apache.cassandra.metrics.CassandraMetricsRegistry.MetricName;
-
-import static org.junit.Assert.*;
public class CassandraMetricsRegistryTest
@@ -86,4 +86,25 @@
}
}
-}
\ No newline at end of file
+ @Test
+ public void testDeltaBaseCase()
+ {
+ long[] last = new long[10];
+ long[] now = new long[]{0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+ // difference between all zeros and a value should be the value
+ assertArrayEquals(now, CassandraMetricsRegistry.delta(now, last));
+ // the difference between itself should be all 0s
+ assertArrayEquals(last, CassandraMetricsRegistry.delta(now, now));
+ // verifying each value is calculated
+ assertArrayEquals(new long[] {1, 1, 1, 1, 1, 1, 1, 1, 1, 1},
+ CassandraMetricsRegistry.delta(new long[] {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, now));
+ }
+
+ @Test
+ public void testDeltaHistogramSizeChange()
+ {
+ long[] count = new long[]{0, 1, 2, 3, 4, 5};
+ assertArrayEquals(count, CassandraMetricsRegistry.delta(count, new long[3]));
+ assertArrayEquals(new long[6], CassandraMetricsRegistry.delta(count, new long[]{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}));
+ }
+}
diff --git a/test/unit/org/apache/cassandra/metrics/DecayingEstimatedHistogramReservoirTest.java b/test/unit/org/apache/cassandra/metrics/DecayingEstimatedHistogramReservoirTest.java
index ef1fed3..b62078c 100644
--- a/test/unit/org/apache/cassandra/metrics/DecayingEstimatedHistogramReservoirTest.java
+++ b/test/unit/org/apache/cassandra/metrics/DecayingEstimatedHistogramReservoirTest.java
@@ -18,20 +18,167 @@
package org.apache.cassandra.metrics;
+import java.util.Arrays;
+import java.util.BitSet;
+import java.util.Random;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.LockSupport;
+
+import org.assertj.core.api.Assertions;
+import org.junit.Assert;
+import org.junit.Ignore;
import org.junit.Test;
import com.codahale.metrics.Clock;
import com.codahale.metrics.Snapshot;
+import org.apache.cassandra.utils.EstimatedHistogram;
+import org.apache.cassandra.utils.Pair;
+import org.quicktheories.core.Gen;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
-
+import static org.quicktheories.QuickTheory.qt;
+import static org.quicktheories.generators.SourceDSL.*;
public class DecayingEstimatedHistogramReservoirTest
{
private static final double DOUBLE_ASSERT_DELTA = 0;
+ public static final int numExamples = 1000000;
+ public static final Gen<long[]> offsets = integers().from(DecayingEstimatedHistogramReservoir.DEFAULT_BUCKET_COUNT)
+ .upToAndIncluding(DecayingEstimatedHistogramReservoir.MAX_BUCKET_COUNT - 10)
+ .zip(booleans().all(), EstimatedHistogram::newOffsets);
+
+
+ @Test
+ public void testFindIndex()
+ {
+ qt().withExamples(numExamples)
+ .forAll(booleans().all()
+ .flatMap(b -> offsets.flatMap(offs -> this.offsetsAndValue(offs, b, 0))))
+ .check(this::checkFindIndex);
+ }
+
+ private boolean checkFindIndex(Pair<long[], Long> offsetsAndValue)
+ {
+ long[] offsets = offsetsAndValue.left;
+ long value = offsetsAndValue.right;
+
+ int model = findIndexModel(offsets, value);
+ int actual = DecayingEstimatedHistogramReservoir.findIndex(offsets, value);
+
+ return model == actual;
+ }
+
+ private int findIndexModel(long[] offsets, long value)
+ {
+ int modelIndex = Arrays.binarySearch(offsets, value);
+ if (modelIndex < 0)
+ modelIndex = -modelIndex - 1;
+
+ return modelIndex;
+ };
+
+ @Test
+ public void showEstimationWorks()
+ {
+ qt().withExamples(numExamples)
+ .forAll(offsets.flatMap(offs -> this.offsetsAndValue(offs, false, 9)))
+ .check(this::checkEstimation);
+ }
+
+ public boolean checkEstimation(Pair<long[], Long> offsetsAndValue)
+ {
+ long[] offsets = offsetsAndValue.left;
+ long value = offsetsAndValue.right;
+ boolean considerZeros = offsets[0] == 0;
+
+ int modelIndex = Arrays.binarySearch(offsets, value);
+ if (modelIndex < 0)
+ modelIndex = -modelIndex - 1;
+
+ int estimate = (int) DecayingEstimatedHistogramReservoir.fastLog12(value);
+
+ if (considerZeros)
+ return estimate - 3 == modelIndex || estimate - 2 == modelIndex;
+ else
+ return estimate - 4 == modelIndex || estimate - 3 == modelIndex;
+ }
+
+
+ private Gen<Pair<long[], Long>> offsetsAndValue(long[] offsets, boolean useMaxLong, long minValue)
+ {
+ return longs().between(minValue, useMaxLong ? Long.MAX_VALUE : offsets[offsets.length - 1] + 100)
+ .mix(longs().between(minValue, minValue + 10),50)
+ .map(value -> Pair.create(offsets, value));
+ }
+
+ //shows that the max before overflow is 238 buckets regardless of consider zeros
+ @Test
+ @Ignore
+ public void showHistorgramOffsetOverflow()
+ {
+ qt().forAll(integers().from(DecayingEstimatedHistogramReservoir.DEFAULT_BUCKET_COUNT).upToAndIncluding(1000))
+ .check(count -> {
+ long[] offsets = EstimatedHistogram.newOffsets(count, false);
+ for (long offset : offsets)
+ if (offset < 0)
+ return false;
+
+ return true;
+ });
+ }
+
+ @Test
+ public void testStriping() throws InterruptedException
+ {
+ TestClock clock = new TestClock();
+ int nStripes = 4;
+ DecayingEstimatedHistogramReservoir model = new DecayingEstimatedHistogramReservoir(clock);
+ DecayingEstimatedHistogramReservoir test = new DecayingEstimatedHistogramReservoir(DecayingEstimatedHistogramReservoir.DEFAULT_ZERO_CONSIDERATION,
+ DecayingEstimatedHistogramReservoir.DEFAULT_BUCKET_COUNT,
+ nStripes,
+ clock);
+
+ long seed = System.nanoTime();
+ System.out.println("DecayingEstimatedHistogramReservoirTest#testStriping.seed = " + seed);
+ Random valGen = new Random(seed);
+ ExecutorService executors = Executors.newFixedThreadPool(nStripes * 2);
+ for (int i = 0; i < 1_000_000; i++)
+ {
+ long value = Math.abs(valGen.nextInt());
+ executors.submit(() -> {
+ model.update(value);
+ LockSupport.parkNanos(2);
+ test.update(value);
+ });
+ }
+
+ executors.shutdown();
+ Assert.assertTrue(executors.awaitTermination(1, TimeUnit.MINUTES));
+
+ Snapshot modelSnapshot = model.getSnapshot();
+ Snapshot testSnapshot = test.getSnapshot();
+
+ assertEquals(modelSnapshot.getMean(), testSnapshot.getMean(), DOUBLE_ASSERT_DELTA);
+ assertEquals(modelSnapshot.getMin(), testSnapshot.getMin(), DOUBLE_ASSERT_DELTA);
+ assertEquals(modelSnapshot.getMax(), testSnapshot.getMax(), DOUBLE_ASSERT_DELTA);
+ assertEquals(modelSnapshot.getMedian(), testSnapshot.getMedian(), DOUBLE_ASSERT_DELTA);
+ for (double i = 0.0; i < 1.0; i += 0.1)
+ assertEquals(modelSnapshot.getValue(i), testSnapshot.getValue(i), DOUBLE_ASSERT_DELTA);
+
+
+ int stripedValues = 0;
+ for (int i = model.size(); i < model.size() * model.stripeCount(); i++)
+ {
+ stripedValues += model.stripedBucketValue(i, true);
+ }
+ assertTrue("no striping found", stripedValues > 0);
+ }
+
@Test
public void testSimple()
{
@@ -45,7 +192,7 @@
}
{
// 0 and 1 map to different buckets
- DecayingEstimatedHistogramReservoir histogram = new DecayingEstimatedHistogramReservoir(true, DecayingEstimatedHistogramReservoir.DEFAULT_BUCKET_COUNT);
+ DecayingEstimatedHistogramReservoir histogram = new DecayingEstimatedHistogramReservoir(true);
histogram.update(0);
assertEquals(1, histogram.getSnapshot().getValues()[0]);
histogram.update(1);
@@ -58,7 +205,7 @@
@Test
public void testOverflow()
{
- DecayingEstimatedHistogramReservoir histogram = new DecayingEstimatedHistogramReservoir(DecayingEstimatedHistogramReservoir.DEFAULT_ZERO_CONSIDERATION, 1);
+ DecayingEstimatedHistogramReservoir histogram = new DecayingEstimatedHistogramReservoir(DecayingEstimatedHistogramReservoir.DEFAULT_ZERO_CONSIDERATION, 1, 1);
histogram.update(100);
assert histogram.isOverflowed();
assertEquals(Long.MAX_VALUE, histogram.getSnapshot().getMax());
@@ -80,7 +227,7 @@
{
TestClock clock = new TestClock();
- DecayingEstimatedHistogramReservoir histogram = new DecayingEstimatedHistogramReservoir(DecayingEstimatedHistogramReservoir.DEFAULT_ZERO_CONSIDERATION, DecayingEstimatedHistogramReservoir.DEFAULT_BUCKET_COUNT, clock);
+ DecayingEstimatedHistogramReservoir histogram = new DecayingEstimatedHistogramReservoir(clock);
for (int i = 0; i < 40; i++)
histogram.update(0);
for (int i = 0; i < 20; i++)
@@ -92,7 +239,10 @@
{
TestClock clock = new TestClock();
- DecayingEstimatedHistogramReservoir histogram = new DecayingEstimatedHistogramReservoir(true, DecayingEstimatedHistogramReservoir.DEFAULT_BUCKET_COUNT, clock);
+ DecayingEstimatedHistogramReservoir histogram = new DecayingEstimatedHistogramReservoir(true,
+ DecayingEstimatedHistogramReservoir.DEFAULT_BUCKET_COUNT,
+ DecayingEstimatedHistogramReservoir.DEFAULT_STRIPE_COUNT,
+ clock);
for (int i = 0; i < 40; i++)
histogram.update(0);
for (int i = 0; i < 20; i++)
@@ -109,7 +259,7 @@
{
TestClock clock = new TestClock();
- DecayingEstimatedHistogramReservoir histogram = new DecayingEstimatedHistogramReservoir(DecayingEstimatedHistogramReservoir.DEFAULT_ZERO_CONSIDERATION, DecayingEstimatedHistogramReservoir.DEFAULT_BUCKET_COUNT, clock);
+ DecayingEstimatedHistogramReservoir histogram = new DecayingEstimatedHistogramReservoir(clock);
for (int i = 0; i < 20; i++)
histogram.update(10);
for (int i = 0; i < 40; i++)
@@ -128,7 +278,7 @@
{
TestClock clock = new TestClock();
- DecayingEstimatedHistogramReservoir histogram = new DecayingEstimatedHistogramReservoir(DecayingEstimatedHistogramReservoir.DEFAULT_ZERO_CONSIDERATION, 90, clock);
+ DecayingEstimatedHistogramReservoir histogram = new DecayingEstimatedHistogramReservoir(DecayingEstimatedHistogramReservoir.DEFAULT_ZERO_CONSIDERATION, 90, 1, clock);
histogram.update(23282687);
assertFalse(histogram.isOverflowed());
assertEquals(1, histogram.getSnapshot().getValues()[89]);
@@ -149,7 +299,7 @@
{
TestClock clock = new TestClock();
- DecayingEstimatedHistogramReservoir histogram = new DecayingEstimatedHistogramReservoir(DecayingEstimatedHistogramReservoir.DEFAULT_ZERO_CONSIDERATION, DecayingEstimatedHistogramReservoir.DEFAULT_BUCKET_COUNT, clock);
+ DecayingEstimatedHistogramReservoir histogram = new DecayingEstimatedHistogramReservoir(clock);
// percentile of empty histogram is 0
assertEquals(0D, histogram.getSnapshot().getValue(0.99), DOUBLE_ASSERT_DELTA);
@@ -164,7 +314,7 @@
{
TestClock clock = new TestClock();
- DecayingEstimatedHistogramReservoir histogram = new DecayingEstimatedHistogramReservoir(DecayingEstimatedHistogramReservoir.DEFAULT_ZERO_CONSIDERATION, DecayingEstimatedHistogramReservoir.DEFAULT_BUCKET_COUNT, clock);
+ DecayingEstimatedHistogramReservoir histogram = new DecayingEstimatedHistogramReservoir(clock);
histogram.update(1);
histogram.update(2);
@@ -182,7 +332,7 @@
{
TestClock clock = new TestClock();
- DecayingEstimatedHistogramReservoir histogram = new DecayingEstimatedHistogramReservoir(DecayingEstimatedHistogramReservoir.DEFAULT_ZERO_CONSIDERATION, DecayingEstimatedHistogramReservoir.DEFAULT_BUCKET_COUNT, clock);
+ DecayingEstimatedHistogramReservoir histogram = new DecayingEstimatedHistogramReservoir(clock);
for (int i = 11; i <= 20; i++)
histogram.update(i);
@@ -201,7 +351,10 @@
{
TestClock clock = new TestClock();
- DecayingEstimatedHistogramReservoir histogram = new DecayingEstimatedHistogramReservoir(true, DecayingEstimatedHistogramReservoir.DEFAULT_BUCKET_COUNT, clock);
+ DecayingEstimatedHistogramReservoir histogram = new DecayingEstimatedHistogramReservoir(true,
+ DecayingEstimatedHistogramReservoir.DEFAULT_BUCKET_COUNT,
+ DecayingEstimatedHistogramReservoir.DEFAULT_STRIPE_COUNT,
+ clock);
histogram.update(0);
histogram.update(0);
histogram.update(1);
@@ -219,7 +372,7 @@
{
TestClock clock = new TestClock();
- DecayingEstimatedHistogramReservoir histogram = new DecayingEstimatedHistogramReservoir(DecayingEstimatedHistogramReservoir.DEFAULT_ZERO_CONSIDERATION, DecayingEstimatedHistogramReservoir.DEFAULT_BUCKET_COUNT, clock);
+ DecayingEstimatedHistogramReservoir histogram = new DecayingEstimatedHistogramReservoir(clock);
// percentile of empty histogram is 0
assertEquals(0, histogram.getSnapshot().getValue(1.0), DOUBLE_ASSERT_DELTA);
@@ -312,7 +465,7 @@
{
TestClock clock = new TestClock();
- DecayingEstimatedHistogramReservoir histogram = new DecayingEstimatedHistogramReservoir(DecayingEstimatedHistogramReservoir.DEFAULT_ZERO_CONSIDERATION, DecayingEstimatedHistogramReservoir.DEFAULT_BUCKET_COUNT, clock);
+ DecayingEstimatedHistogramReservoir histogram = new DecayingEstimatedHistogramReservoir(clock);
// percentile of empty histogram is 0
assertEquals(0, histogram.getSnapshot().getValue(0.99), DOUBLE_ASSERT_DELTA);
@@ -334,7 +487,7 @@
{
TestClock clock = new TestClock();
- DecayingEstimatedHistogramReservoir histogram = new DecayingEstimatedHistogramReservoir(DecayingEstimatedHistogramReservoir.DEFAULT_ZERO_CONSIDERATION, DecayingEstimatedHistogramReservoir.DEFAULT_BUCKET_COUNT, clock);
+ DecayingEstimatedHistogramReservoir histogram = new DecayingEstimatedHistogramReservoir(clock);
histogram.update(20);
histogram.update(21);
@@ -360,7 +513,7 @@
{
TestClock clock = new TestClock();
- DecayingEstimatedHistogramReservoir histogram = new DecayingEstimatedHistogramReservoir(DecayingEstimatedHistogramReservoir.DEFAULT_ZERO_CONSIDERATION, DecayingEstimatedHistogramReservoir.DEFAULT_BUCKET_COUNT, clock);
+ DecayingEstimatedHistogramReservoir histogram = new DecayingEstimatedHistogramReservoir(clock);
clock.addMillis(DecayingEstimatedHistogramReservoir.LANDMARK_RESET_INTERVAL_IN_MS - 1_000L);
@@ -380,6 +533,52 @@
}
}
+ @Test
+ public void testAggregation()
+ {
+ TestClock clock = new TestClock();
+
+ DecayingEstimatedHistogramReservoir histogram = new DecayingEstimatedHistogramReservoir(clock);
+ DecayingEstimatedHistogramReservoir another = new DecayingEstimatedHistogramReservoir(clock);
+
+ clock.addMillis(DecayingEstimatedHistogramReservoir.LANDMARK_RESET_INTERVAL_IN_MS - 1_000L);
+
+ histogram.update(1000);
+ clock.addMillis(100);
+ another.update(2000);
+ clock.addMillis(100);
+ histogram.update(2000);
+ clock.addMillis(100);
+ another.update(3000);
+ clock.addMillis(100);
+ histogram.update(3000);
+ clock.addMillis(100);
+ another.update(4000);
+
+ DecayingEstimatedHistogramReservoir.EstimatedHistogramReservoirSnapshot snapshot = (DecayingEstimatedHistogramReservoir.EstimatedHistogramReservoirSnapshot) histogram.getSnapshot();
+ DecayingEstimatedHistogramReservoir.EstimatedHistogramReservoirSnapshot anotherSnapshot = (DecayingEstimatedHistogramReservoir.EstimatedHistogramReservoirSnapshot) another.getSnapshot();
+
+ assertEquals(2000, snapshot.getMean(), 500D);
+ assertEquals(3000, anotherSnapshot.getMean(), 500D);
+
+ snapshot.add(anotherSnapshot);
+
+ // Another had newer decayLandmark, the aggregated snapshot should use it
+ assertEquals(anotherSnapshot.getSnapshotLandmark(), snapshot.getSnapshotLandmark());
+ assertEquals(2500, snapshot.getMean(), 500D);
+ }
+
+ @Test
+ public void testSize()
+ {
+ TestClock clock = new TestClock();
+
+ DecayingEstimatedHistogramReservoir histogram = new DecayingEstimatedHistogramReservoir(clock);
+ histogram.update(42);
+ histogram.update(42);
+ assertEquals(2, histogram.getSnapshot().size());
+ }
+
private void assertEstimatedQuantile(long expectedValue, double actualValue)
{
assertTrue("Expected at least [" + expectedValue + "] but actual is [" + actualValue + "]", actualValue >= expectedValue);
diff --git a/test/unit/org/apache/cassandra/metrics/HintedHandOffMetricsTest.java b/test/unit/org/apache/cassandra/metrics/HintedHandOffMetricsTest.java
index 2394e0c..15feca4 100644
--- a/test/unit/org/apache/cassandra/metrics/HintedHandOffMetricsTest.java
+++ b/test/unit/org/apache/cassandra/metrics/HintedHandOffMetricsTest.java
@@ -20,7 +20,6 @@
*/
package org.apache.cassandra.metrics;
-import java.net.InetAddress;
import java.util.Map;
import java.util.UUID;
@@ -32,9 +31,11 @@
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.UntypedResultSet;
import org.apache.cassandra.db.SystemKeyspace;
+import org.apache.cassandra.db.commitlog.CommitLog;
import org.apache.cassandra.db.marshal.Int32Type;
import org.apache.cassandra.db.marshal.UUIDType;
import org.apache.cassandra.hints.HintsService;
+import org.apache.cassandra.locator.InetAddressAndPort;
import static org.junit.Assert.assertEquals;
import static org.apache.cassandra.cql3.QueryProcessor.executeInternal;
@@ -45,6 +46,7 @@
public static void initDD()
{
DatabaseDescriptor.daemonInitialization();
+ CommitLog.instance.start();
}
@Test
@@ -53,11 +55,15 @@
DatabaseDescriptor.getHintsDirectory().mkdirs();
for (int i = 0; i < 99; i++)
- HintsService.instance.metrics.incrPastWindow(InetAddress.getByName("127.0.0.1"));
+ HintsService.instance.metrics.incrPastWindow(InetAddressAndPort.getLocalHost());
HintsService.instance.metrics.log();
- UntypedResultSet rows = executeInternal("SELECT hints_dropped FROM system." + SystemKeyspace.PEER_EVENTS);
+ UntypedResultSet rows = executeInternal("SELECT hints_dropped FROM system." + SystemKeyspace.PEER_EVENTS_V2);
Map<UUID, Integer> returned = rows.one().getMap("hints_dropped", UUIDType.instance, Int32Type.instance);
assertEquals(Iterators.getLast(returned.values().iterator()).intValue(), 99);
+
+ rows = executeInternal("SELECT hints_dropped FROM system." + SystemKeyspace.LEGACY_PEER_EVENTS);
+ returned = rows.one().getMap("hints_dropped", UUIDType.instance, Int32Type.instance);
+ assertEquals(Iterators.getLast(returned.values().iterator()).intValue(), 99);
}
}
diff --git a/test/unit/org/apache/cassandra/metrics/KeyspaceMetricsTest.java b/test/unit/org/apache/cassandra/metrics/KeyspaceMetricsTest.java
new file mode 100644
index 0000000..ae92146
--- /dev/null
+++ b/test/unit/org/apache/cassandra/metrics/KeyspaceMetricsTest.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.metrics;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.service.EmbeddedCassandraService;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import com.datastax.driver.core.Cluster;
+import com.datastax.driver.core.Session;
+
+public class KeyspaceMetricsTest extends SchemaLoader
+{
+ private static Session session;
+
+ @BeforeClass()
+ public static void setup() throws ConfigurationException, IOException
+ {
+ Schema.instance.clear();
+
+ EmbeddedCassandraService cassandra = new EmbeddedCassandraService();
+ cassandra.start();
+
+ Cluster cluster = Cluster.builder().addContactPoint("127.0.0.1").withPort(DatabaseDescriptor.getNativeTransportPort()).build();
+ session = cluster.connect();
+ }
+
+ @Test
+ public void testMetricsCleanupOnDrop()
+ {
+ String keyspace = "keyspacemetricstest_metrics_cleanup";
+ CassandraMetricsRegistry registry = CassandraMetricsRegistry.Metrics;
+ Supplier<Stream<String>> metrics = () -> registry.getNames().stream().filter(m -> m.contains(keyspace));
+
+ // no metrics before creating
+ assertEquals(0, metrics.get().count());
+
+ session.execute(String.format("CREATE KEYSPACE %s WITH replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 };", keyspace));
+ // some metrics
+ assertTrue(metrics.get().count() > 0);
+
+ session.execute(String.format("DROP KEYSPACE %s;", keyspace));
+ // no metrics after drop
+ assertEquals(metrics.get().collect(Collectors.joining(",")), 0, metrics.get().count());
+ }
+}
diff --git a/test/unit/org/apache/cassandra/metrics/LatencyMetricsTest.java b/test/unit/org/apache/cassandra/metrics/LatencyMetricsTest.java
index 62cb88e..d61c550 100644
--- a/test/unit/org/apache/cassandra/metrics/LatencyMetricsTest.java
+++ b/test/unit/org/apache/cassandra/metrics/LatencyMetricsTest.java
@@ -18,12 +18,27 @@
package org.apache.cassandra.metrics;
+import java.util.concurrent.TimeUnit;
+
import org.junit.Test;
+import static junit.framework.Assert.assertEquals;
import static junit.framework.Assert.assertFalse;
public class LatencyMetricsTest
{
+ private final MetricNameFactory factory = new TestMetricsNameFactory();
+
+ private class TestMetricsNameFactory implements MetricNameFactory
+ {
+
+ @Override
+ public CassandraMetricsRegistry.MetricName createMetricName(String metricName)
+ {
+ return new CassandraMetricsRegistry.MetricName(TestMetricsNameFactory.class, metricName);
+ }
+ }
+
/**
* Test bitsets in a "real-world" environment, i.e., bloom filters
*/
@@ -31,14 +46,10 @@
public void testGetRecentLatency()
{
final LatencyMetrics l = new LatencyMetrics("test", "test");
- Runnable r = new Runnable()
- {
- public void run()
+ Runnable r = () -> {
+ for (int i = 0; i < 10000; i++)
{
- for (int i = 0; i < 10000; i++)
- {
- l.addNano(1000);
- }
+ l.addNano(1000);
}
};
new Thread(r).start();
@@ -49,4 +60,49 @@
assertFalse(recent.equals(Double.POSITIVE_INFINITY));
}
}
+
+ /**
+ * Test that parent LatencyMetrics are receiving updates from child metrics when reading
+ */
+ @Test
+ public void testReadMerging()
+ {
+ final LatencyMetrics parent = new LatencyMetrics("testMerge", "testMerge");
+ final LatencyMetrics child = new LatencyMetrics(factory, "testChild", parent);
+
+ for (int i = 0; i < 100; i++)
+ {
+ child.addNano(TimeUnit.NANOSECONDS.convert(i, TimeUnit.MILLISECONDS));
+ }
+
+ assertEquals(4950000, child.totalLatency.getCount());
+ assertEquals(child.totalLatency.getCount(), parent.totalLatency.getCount());
+ assertEquals(child.latency.getSnapshot().getMean(), parent.latency.getSnapshot().getMean(), 50D);
+
+ child.release();
+ parent.release();
+ }
+
+ @Test
+ public void testRelease()
+ {
+ final LatencyMetrics parent = new LatencyMetrics("testRelease", "testRelease");
+ final LatencyMetrics child = new LatencyMetrics(factory, "testChildRelease", parent);
+
+ for (int i = 0; i < 100; i++)
+ {
+ child.addNano(TimeUnit.NANOSECONDS.convert(i, TimeUnit.MILLISECONDS));
+ }
+
+ double mean = parent.latency.getSnapshot().getMean();
+ long count = parent.totalLatency.getCount();
+
+ child.release();
+
+ // Check that no value was lost with the release
+ assertEquals(count, parent.totalLatency.getCount());
+ assertEquals(mean, parent.latency.getSnapshot().getMean(), 50D);
+
+ parent.release();
+ }
}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/metrics/MaxSamplerTest.java b/test/unit/org/apache/cassandra/metrics/MaxSamplerTest.java
new file mode 100644
index 0000000..69278b8
--- /dev/null
+++ b/test/unit/org/apache/cassandra/metrics/MaxSamplerTest.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.metrics;
+
+import java.util.List;
+import java.util.concurrent.TimeoutException;
+
+import org.apache.cassandra.metrics.Sampler.Sample;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class MaxSamplerTest extends SamplerTest
+{
+ @Before
+ public void setSampler()
+ {
+ this.sampler = new MaxSampler<String>()
+ {
+ public String toString(String value)
+ {
+ return value;
+ }
+ };
+ }
+
+ @Test
+ public void testReturnsMax() throws TimeoutException
+ {
+ sampler.beginSampling(5, 100000);
+ add();
+ List<Sample<String>> result = sampler.finishSampling(10);
+ for (int i = 9995 ; i < 10000 ; i ++)
+ {
+ final String key = "test" + i;
+ Assert.assertTrue(result.stream().anyMatch(s -> s.value.equals(key)));
+ }
+ }
+
+ @Test
+ public void testSizeEqualsCapacity() throws TimeoutException
+ {
+ sampler.beginSampling(10, 100000);
+ add();
+ List<Sample<String>> result = sampler.finishSampling(10);
+ for (int i = 9990 ; i < 10000 ; i ++)
+ {
+ final String key = "test" + i;
+ Assert.assertTrue(result.stream().anyMatch(s -> s.value.equals(key)));
+ }
+ }
+
+ @Test
+ public void testCapacityLarger() throws TimeoutException
+ {
+
+ sampler.beginSampling(100, 100000);
+ add();
+ List<Sample<String>> result = sampler.finishSampling(10);
+ for (int i = 9990 ; i < 10000 ; i ++)
+ {
+ final String key = "test" + i;
+ Assert.assertTrue(result.stream().anyMatch(s -> s.value.equals(key)));
+ }
+ }
+
+ private void add() throws TimeoutException
+ {
+ for (int i = 0 ; i < 10000 ; i ++)
+ {
+ // dont load shed test data
+ if (i % 999 == 0)
+ waitForEmpty(1000);
+ sampler.addSample("test"+i, i);
+ }
+ waitForEmpty(1000);
+ }
+}
diff --git a/test/unit/org/apache/cassandra/metrics/SamplerTest.java b/test/unit/org/apache/cassandra/metrics/SamplerTest.java
new file mode 100644
index 0000000..3d24c1b
--- /dev/null
+++ b/test/unit/org/apache/cassandra/metrics/SamplerTest.java
@@ -0,0 +1,247 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.metrics;
+
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.cassandra.concurrent.NamedThreadFactory;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.metrics.Sampler.Sample;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.utils.FreeRunningClock;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import com.google.common.collect.Maps;
+import com.google.common.util.concurrent.Uninterruptibles;
+
+
+public class SamplerTest
+{
+ Sampler<String> sampler;
+
+
+ @BeforeClass
+ public static void initMessagingService() throws ConfigurationException
+ {
+ // required so the rejection policy doesnt fail on initializing
+ // static MessagingService resources
+ DatabaseDescriptor.daemonInitialization();
+ }
+
+ @Test
+ public void sampleLoadshedding() throws Exception
+ {
+ // dont need to run this in children tests
+ if (sampler != null) return;
+ AtomicInteger called = new AtomicInteger();
+ CountDownLatch latch = new CountDownLatch(1);
+ Sampler<String> waitSampler = new Sampler<String>()
+ {
+ protected void insert(String item, long value)
+ {
+ called.incrementAndGet();
+ try
+ {
+ latch.await(1, TimeUnit.MINUTES);
+ }
+ catch (InterruptedException e)
+ {
+ e.printStackTrace();
+ }
+ }
+
+ public boolean isEnabled()
+ {
+ return true;
+ }
+
+ public void beginSampling(int capacity, int durationMillis)
+ {
+ }
+
+ public List<Sample<String>> finishSampling(int count)
+ {
+ return null;
+ }
+
+ public String toString(String value)
+ {
+ return "";
+ }
+ };
+ // 1000 queued, 1 in progress, 1 to drop
+ for (int i = 0; i < 1002; i++)
+ {
+ waitSampler.addSample("TEST", 1);
+ }
+ latch.countDown();
+ waitForEmpty(1000);
+ Assert.assertEquals(1001, called.get());
+ Assert.assertEquals(1, MessagingService.instance().getDroppedMessages().get("_SAMPLE").intValue());
+ }
+
+ @Test
+ public void testSamplerOutOfOrder() throws TimeoutException
+ {
+ if(sampler == null) return;
+ sampler.beginSampling(10, 1000000);
+ insert(sampler);
+ waitForEmpty(1000);
+ List<Sample<String>> single = sampler.finishSampling(10);
+ single = sampler.finishSampling(10);
+ Assert.assertEquals(0, single.size());
+ }
+
+ @Test(expected=RuntimeException.class)
+ public void testWhileRunning()
+ {
+ if(sampler == null) throw new RuntimeException();
+ sampler.clock = new FreeRunningClock();
+ try
+ {
+ sampler.beginSampling(10, 1000000);
+ } catch (RuntimeException e)
+ {
+ Assert.fail(); // shouldnt fail on first call
+ }
+ // should throw Exception
+ sampler.beginSampling(10, 1000000);
+ }
+
+ @Test
+ public void testRepeatStartAfterTimeout()
+ {
+ if(sampler == null) return;
+ FreeRunningClock clock = new FreeRunningClock();
+ sampler.clock = clock;
+ try
+ {
+ sampler.beginSampling(10, 10);
+ } catch (RuntimeException e)
+ {
+ Assert.fail(); // shouldnt fail on first call
+ }
+ clock.advance(11, TimeUnit.MILLISECONDS);
+ sampler.beginSampling(10, 1000000);
+ }
+
+ /**
+ * checking for exceptions if not thread safe (MinMaxPQ and SS/HL are not)
+ */
+ @Test
+ public void testMultithreadedAccess() throws Exception
+ {
+ if(sampler == null) return;
+ final AtomicBoolean running = new AtomicBoolean(true);
+ final CountDownLatch latch = new CountDownLatch(1);
+
+ NamedThreadFactory.createThread(new Runnable()
+ {
+ public void run()
+ {
+ try
+ {
+ while (running.get())
+ {
+ insert(sampler);
+ }
+ } finally
+ {
+ latch.countDown();
+ }
+ }
+
+ }
+ , "inserter").start();
+ try
+ {
+ // start/stop in fast iterations
+ for(int i = 0; i<100; i++)
+ {
+ sampler.beginSampling(i, 100000);
+ sampler.finishSampling(i);
+ }
+ // start/stop with pause to let it build up past capacity
+ for(int i = 0; i<3; i++)
+ {
+ sampler.beginSampling(i, 100000);
+ Thread.sleep(250);
+ sampler.finishSampling(i);
+ }
+
+ // with empty results
+ running.set(false);
+ latch.await(1, TimeUnit.SECONDS);
+ waitForEmpty(1000);
+ for(int i = 0; i<10; i++)
+ {
+ sampler.beginSampling(i, 100000);
+ Thread.sleep(i);
+ sampler.finishSampling(i);
+ }
+ } finally
+ {
+ running.set(false);
+ }
+ }
+
+ public void insert(Sampler<String> sampler)
+ {
+ for(int i = 1; i <= 10; i++)
+ {
+ for(int j = 0; j < i; j++)
+ {
+ String key = "item" + i;
+ sampler.addSample(key, 1);
+ }
+ }
+ }
+
+ public void waitForEmpty(int timeoutMs) throws TimeoutException
+ {
+ int timeout = 0;
+ while (!Sampler.samplerExecutor.getQueue().isEmpty())
+ {
+ timeout++;
+ Uninterruptibles.sleepUninterruptibly(100, TimeUnit.MILLISECONDS);
+ if (timeout * 100 > timeoutMs)
+ {
+ throw new TimeoutException("sampler executor not cleared within timeout");
+ }
+ }
+ }
+
+ public <T> Map<T, Long> countMap(List<Sample<T>> target)
+ {
+ Map<T, Long> counts = Maps.newHashMap();
+ for(Sample<T> counter : target)
+ {
+ counts.put(counter.value, counter.count);
+ }
+ return counts;
+ }
+}
diff --git a/test/unit/org/apache/cassandra/metrics/TableMetricsTest.java b/test/unit/org/apache/cassandra/metrics/TableMetricsTest.java
new file mode 100644
index 0000000..56ad401
--- /dev/null
+++ b/test/unit/org/apache/cassandra/metrics/TableMetricsTest.java
@@ -0,0 +1,281 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.metrics;
+
+import java.io.IOException;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+import com.datastax.driver.core.BatchStatement;
+import com.datastax.driver.core.Cluster;
+import com.datastax.driver.core.PreparedStatement;
+import com.datastax.driver.core.Session;
+import org.apache.cassandra.OrderedJUnit4ClassRunner;
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.service.EmbeddedCassandraService;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+@RunWith(OrderedJUnit4ClassRunner.class)
+public class TableMetricsTest extends SchemaLoader
+{
+
+ private static Session session;
+
+ private static final String KEYSPACE = "junit";
+ private static final String TABLE = "tablemetricstest";
+ private static final String COUNTER_TABLE = "tablemetricscountertest";
+
+ @BeforeClass()
+ public static void setup() throws ConfigurationException, IOException
+ {
+ Schema.instance.clear();
+
+ EmbeddedCassandraService cassandra = new EmbeddedCassandraService();
+ cassandra.start();
+
+ Cluster cluster = Cluster.builder().addContactPoint("127.0.0.1").withPort(DatabaseDescriptor.getNativeTransportPort()).build();
+ session = cluster.connect();
+
+ session.execute(String.format("CREATE KEYSPACE IF NOT EXISTS %s WITH replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 };", KEYSPACE));
+ session.execute(String.format("CREATE TABLE IF NOT EXISTS %s.%s (id_c counter, id int, val text, PRIMARY KEY(id, val));", KEYSPACE, COUNTER_TABLE));
+ }
+
+ private ColumnFamilyStore recreateTable()
+ {
+ return recreateTable(TABLE);
+ }
+
+ private ColumnFamilyStore recreateTable(String table)
+ {
+ session.execute(String.format("DROP TABLE IF EXISTS %s.%s", KEYSPACE, table));
+ session.execute(String.format("CREATE TABLE IF NOT EXISTS %s.%s (id int, val1 text, val2 text, PRIMARY KEY(id, val1));", KEYSPACE, table));
+ return ColumnFamilyStore.getIfExists(KEYSPACE, table);
+ }
+
+ private void executeBatch(boolean isLogged, int distinctPartitions, int statementsPerPartition, String... tables)
+ {
+ if (tables == null || tables.length == 0)
+ {
+ tables = new String[] { TABLE };
+ }
+ BatchStatement.Type batchType;
+
+ if (isLogged)
+ {
+ batchType = BatchStatement.Type.LOGGED;
+ }
+ else
+ {
+ batchType = BatchStatement.Type.UNLOGGED;
+ }
+
+ BatchStatement batch = new BatchStatement(batchType);
+
+ for (String table : tables)
+ populateBatch(batch, table, distinctPartitions, statementsPerPartition);
+
+ session.execute(batch);
+ }
+
+ private static void populateBatch(BatchStatement batch, String table, int distinctPartitions, int statementsPerPartition)
+ {
+ PreparedStatement ps = session.prepare(String.format("INSERT INTO %s.%s (id, val1, val2) VALUES (?, ?, ?);", KEYSPACE, table));
+
+ for (int i=0; i<distinctPartitions; i++)
+ {
+ for (int j=0; j<statementsPerPartition; j++)
+ {
+ batch.add(ps.bind(i, j + "a", "b"));
+ }
+ }
+ }
+
+ @Test
+ public void testRegularStatementsExecuted()
+ {
+ ColumnFamilyStore cfs = recreateTable();
+ assertEquals(0, cfs.metric.coordinatorWriteLatency.getCount());
+ assertEquals(0.0, cfs.metric.coordinatorWriteLatency.getMeanRate(), 0.0);
+
+ for (int i = 0; i < 10; i++)
+ {
+ session.execute(String.format("INSERT INTO %s.%s (id, val1, val2) VALUES (%d, '%s', '%s')", KEYSPACE, TABLE, i, "val" + i, "val" + i));
+ }
+
+ assertEquals(10, cfs.metric.coordinatorWriteLatency.getCount());
+ assertGreaterThan(cfs.metric.coordinatorWriteLatency.getMeanRate(), 0);
+ }
+
+ @Test
+ public void testPreparedStatementsExecuted()
+ {
+ ColumnFamilyStore cfs = recreateTable();
+ PreparedStatement metricsStatement = session.prepare(String.format("INSERT INTO %s.%s (id, val1, val2) VALUES (?, ?, ?)", KEYSPACE, TABLE));
+
+ assertEquals(0, cfs.metric.coordinatorWriteLatency.getCount());
+ assertEquals(0.0, cfs.metric.coordinatorWriteLatency.getMeanRate(), 0.0);
+
+ for (int i = 0; i < 10; i++)
+ {
+ session.execute(metricsStatement.bind(i, "val" + i, "val" + i));
+ }
+
+ assertEquals(10, cfs.metric.coordinatorWriteLatency.getCount());
+ assertGreaterThan(cfs.metric.coordinatorWriteLatency.getMeanRate(), 0);
+ }
+
+ @Test
+ public void testLoggedPartitionsPerBatch()
+ {
+ ColumnFamilyStore cfs = recreateTable();
+ assertEquals(0, cfs.metric.coordinatorWriteLatency.getCount());
+ assertEquals(0.0, cfs.metric.coordinatorWriteLatency.getMeanRate(), 0.0);
+
+ executeBatch(true, 10, 2);
+ assertEquals(1, cfs.metric.coordinatorWriteLatency.getCount());
+
+ executeBatch(true, 20, 2);
+ assertEquals(2, cfs.metric.coordinatorWriteLatency.getCount()); // 2 for previous batch and this batch
+ assertGreaterThan(cfs.metric.coordinatorWriteLatency.getMeanRate(), 0);
+ }
+
+ @Test
+ public void testLoggedPartitionsPerBatchMultiTable()
+ {
+ ColumnFamilyStore first = recreateTable();
+ assertEquals(0, first.metric.coordinatorWriteLatency.getCount());
+ assertEquals(0.0, first.metric.coordinatorWriteLatency.getMeanRate(), 0.0);
+
+ ColumnFamilyStore second = recreateTable(TABLE + "_second");
+ assertEquals(0, second.metric.coordinatorWriteLatency.getCount());
+ assertEquals(0.0, second.metric.coordinatorWriteLatency.getMeanRate(), 0.0);
+
+ executeBatch(true, 10, 2, TABLE, TABLE + "_second");
+ assertEquals(1, first.metric.coordinatorWriteLatency.getCount());
+ assertEquals(1, second.metric.coordinatorWriteLatency.getCount());
+
+ executeBatch(true, 20, 2, TABLE, TABLE + "_second");
+ assertEquals(2, first.metric.coordinatorWriteLatency.getCount()); // 2 for previous batch and this batch
+ assertEquals(2, second.metric.coordinatorWriteLatency.getCount()); // 2 for previous batch and this batch
+ assertGreaterThan(first.metric.coordinatorWriteLatency.getMeanRate(), 0);
+ assertGreaterThan(second.metric.coordinatorWriteLatency.getMeanRate(), 0);
+ }
+
+ @Test
+ public void testUnloggedPartitionsPerBatch()
+ {
+ ColumnFamilyStore cfs = recreateTable();
+ assertEquals(0, cfs.metric.coordinatorWriteLatency.getCount());
+ assertEquals(0.0, cfs.metric.coordinatorWriteLatency.getMeanRate(), 0.0);
+
+ executeBatch(false, 5, 3);
+ assertEquals(1, cfs.metric.coordinatorWriteLatency.getCount());
+
+ executeBatch(false, 25, 2);
+ assertEquals(2, cfs.metric.coordinatorWriteLatency.getCount()); // 2 for previous batch and this batch
+ assertGreaterThan(cfs.metric.coordinatorWriteLatency.getMeanRate(), 0);
+ }
+
+ @Test
+ public void testUnloggedPartitionsPerBatchMultiTable()
+ {
+ ColumnFamilyStore first = recreateTable();
+ assertEquals(0, first.metric.coordinatorWriteLatency.getCount());
+ assertEquals(0.0, first.metric.coordinatorWriteLatency.getMeanRate(), 0.0);
+
+ ColumnFamilyStore second = recreateTable(TABLE + "_second");
+ assertEquals(0, second.metric.coordinatorWriteLatency.getCount());
+ assertEquals(0.0, second.metric.coordinatorWriteLatency.getMeanRate(), 0.0);
+
+ executeBatch(false, 5, 3, TABLE, TABLE + "_second");
+ assertEquals(1, first.metric.coordinatorWriteLatency.getCount());
+
+ executeBatch(false, 25, 2, TABLE, TABLE + "_second");
+ assertEquals(2, first.metric.coordinatorWriteLatency.getCount()); // 2 for previous batch and this batch
+ assertEquals(2, second.metric.coordinatorWriteLatency.getCount()); // 2 for previous batch and this batch
+ assertGreaterThan(first.metric.coordinatorWriteLatency.getMeanRate(), 0);
+ assertGreaterThan(second.metric.coordinatorWriteLatency.getMeanRate(), 0);
+ }
+
+ @Test
+ public void testCounterStatement()
+ {
+ ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(KEYSPACE, COUNTER_TABLE);
+ assertEquals(0, cfs.metric.coordinatorWriteLatency.getCount());
+ assertEquals(0.0, cfs.metric.coordinatorWriteLatency.getMeanRate(), 0.0);
+ session.execute(String.format("UPDATE %s.%s SET id_c = id_c + 1 WHERE id = 1 AND val = 'val1'", KEYSPACE, COUNTER_TABLE));
+ assertEquals(1, cfs.metric.coordinatorWriteLatency.getCount());
+ assertGreaterThan(cfs.metric.coordinatorWriteLatency.getMeanRate(), 0);
+ }
+
+ private static void assertGreaterThan(double actual, double expectedLessThan) {
+ assertTrue("Expected " + actual + " > " + expectedLessThan, actual > expectedLessThan);
+ }
+
+ @Test
+ public void testMetricsCleanupOnDrop()
+ {
+ String tableName = TABLE + "_metrics_cleanup";
+ CassandraMetricsRegistry registry = CassandraMetricsRegistry.Metrics;
+ Supplier<Stream<String>> metrics = () -> registry.getNames().stream().filter(m -> m.contains(tableName));
+
+ // no metrics before creating
+ assertEquals(0, metrics.get().count());
+
+ recreateTable(tableName);
+ // some metrics
+ assertTrue(metrics.get().count() > 0);
+
+ session.execute(String.format("DROP TABLE IF EXISTS %s.%s", KEYSPACE, tableName));
+ // no metrics after drop
+ assertEquals(metrics.get().collect(Collectors.joining(",")), 0, metrics.get().count());
+ }
+
+ @Test
+ public void testViewMetricsCleanupOnDrop()
+ {
+ String tableName = TABLE + "_metrics_cleanup";
+ String viewName = TABLE + "_materialized_view_cleanup";
+ CassandraMetricsRegistry registry = CassandraMetricsRegistry.Metrics;
+ Supplier<Stream<String>> metrics = () -> registry.getNames().stream().filter(m -> m.contains(viewName));
+
+ // no metrics before creating
+ assertEquals(0, metrics.get().count());
+
+ recreateTable(tableName);
+ session.execute(String.format("CREATE MATERIALIZED VIEW %s.%s AS SELECT id,val1 FROM %s.%s WHERE id IS NOT NULL AND val1 IS NOT NULL PRIMARY KEY (id,val1);", KEYSPACE, viewName, KEYSPACE, tableName));
+ // some metrics
+ assertTrue(metrics.get().count() > 0);
+
+ session.execute(String.format("DROP MATERIALIZED VIEW IF EXISTS %s.%s;", KEYSPACE, viewName));
+ // no metrics after drop
+ assertEquals(metrics.get().collect(Collectors.joining(",")), 0, metrics.get().count());
+ }
+}
diff --git a/test/unit/org/apache/cassandra/metrics/TopFrequencySamplerTest.java b/test/unit/org/apache/cassandra/metrics/TopFrequencySamplerTest.java
new file mode 100644
index 0000000..ae6e3f9
--- /dev/null
+++ b/test/unit/org/apache/cassandra/metrics/TopFrequencySamplerTest.java
@@ -0,0 +1,71 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+package org.apache.cassandra.metrics;
+
+import java.util.List;
+import java.util.concurrent.TimeoutException;
+
+import org.apache.cassandra.metrics.Sampler.Sample;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TopFrequencySamplerTest extends SamplerTest
+{
+ @Before
+ public void setSampler()
+ {
+ this.sampler = new FrequencySampler<String>()
+ {
+ public String toString(String value)
+ {
+ return value;
+ }
+ };
+ }
+
+ @Test
+ public void testSamplerSingleInsertionsEqualMulti() throws TimeoutException
+ {
+ sampler.beginSampling(10, 100000);
+ insert(sampler);
+ waitForEmpty(1000);
+ List<Sample<String>> single = sampler.finishSampling(10);
+
+ FrequencySampler<String> sampler2 = new FrequencySampler<String>()
+ {
+ public String toString(String value)
+ {
+ return value;
+ }
+ };
+ sampler2.beginSampling(10, 100000);
+ for(int i = 1; i <= 10; i++)
+ {
+ String key = "item" + i;
+ sampler2.addSample(key, i);
+ }
+ waitForEmpty(1000);
+ Assert.assertEquals(countMap(single), countMap(sampler2.finishSampling(10)));
+ }
+
+
+}
diff --git a/test/unit/org/apache/cassandra/net/AsyncChannelPromiseTest.java b/test/unit/org/apache/cassandra/net/AsyncChannelPromiseTest.java
new file mode 100644
index 0000000..c4e6295
--- /dev/null
+++ b/test/unit/org/apache/cassandra/net/AsyncChannelPromiseTest.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.net;
+
+import org.junit.After;
+import org.junit.Test;
+
+import io.netty.channel.ChannelPromise;
+import io.netty.channel.embedded.EmbeddedChannel;
+
+public class AsyncChannelPromiseTest extends TestAbstractAsyncPromise
+{
+ @After
+ public void shutdown()
+ {
+ exec.shutdownNow();
+ }
+
+ private ChannelPromise newPromise()
+ {
+ return new AsyncChannelPromise(new EmbeddedChannel());
+ }
+
+ @Test
+ public void testSuccess()
+ {
+ for (boolean setUncancellable : new boolean[] { false, true })
+ for (boolean tryOrSet : new boolean[]{ false, true })
+ testOneSuccess(newPromise(), setUncancellable, tryOrSet, null, null);
+ }
+
+ @Test
+ public void testFailure()
+ {
+ for (boolean setUncancellable : new boolean[] { false, true })
+ for (boolean tryOrSet : new boolean[] { false, true })
+ for (Throwable v : new Throwable[] { null, new NullPointerException() })
+ testOneFailure(newPromise(), setUncancellable, tryOrSet, v, null);
+ }
+
+
+ @Test
+ public void testCancellation()
+ {
+ for (boolean interruptIfRunning : new boolean[] { true, false })
+ testOneCancellation(newPromise(), interruptIfRunning, null);
+ }
+
+
+ @Test
+ public void testTimeout()
+ {
+ for (boolean setUncancellable : new boolean[] { true, false })
+ testOneTimeout(newPromise(), setUncancellable);
+ }
+
+}
diff --git a/test/unit/org/apache/cassandra/net/AsyncMessageOutputPlusTest.java b/test/unit/org/apache/cassandra/net/AsyncMessageOutputPlusTest.java
new file mode 100644
index 0000000..633207c
--- /dev/null
+++ b/test/unit/org/apache/cassandra/net/AsyncMessageOutputPlusTest.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.net;
+
+import java.io.IOException;
+
+import org.junit.Test;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.channel.embedded.EmbeddedChannel;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.net.FrameEncoder.PayloadAllocator;
+
+import static org.junit.Assert.assertEquals;
+
+public class AsyncMessageOutputPlusTest
+{
+
+ static
+ {
+ DatabaseDescriptor.daemonInitialization();
+ }
+
+ @Test
+ public void testSuccess() throws IOException
+ {
+ EmbeddedChannel channel = new TestChannel(4);
+ ByteBuf read;
+ try (AsyncMessageOutputPlus out = new AsyncMessageOutputPlus(channel, 32, Integer.MAX_VALUE, PayloadAllocator.simple))
+ {
+ out.writeInt(1);
+ assertEquals(0, out.flushed());
+ assertEquals(0, out.flushedToNetwork());
+ assertEquals(4, out.position());
+
+ out.doFlush(0);
+ assertEquals(4, out.flushed());
+ assertEquals(4, out.flushedToNetwork());
+
+ out.writeInt(2);
+ assertEquals(8, out.position());
+ assertEquals(4, out.flushed());
+ assertEquals(4, out.flushedToNetwork());
+
+ out.doFlush(0);
+ assertEquals(8, out.position());
+ assertEquals(8, out.flushed());
+ assertEquals(4, out.flushedToNetwork());
+
+ read = channel.readOutbound();
+ assertEquals(4, read.readableBytes());
+ assertEquals(1, read.getInt(0));
+ assertEquals(8, out.flushed());
+ assertEquals(8, out.flushedToNetwork());
+
+ read = channel.readOutbound();
+ assertEquals(4, read.readableBytes());
+ assertEquals(2, read.getInt(0));
+
+ out.write(new byte[64]);
+ assertEquals(72, out.position());
+ assertEquals(40, out.flushed());
+ assertEquals(40, out.flushedToNetwork());
+
+ out.doFlush(0);
+ assertEquals(72, out.position());
+ assertEquals(72, out.flushed());
+ assertEquals(40, out.flushedToNetwork());
+
+ read = channel.readOutbound();
+ assertEquals(32, read.readableBytes());
+ assertEquals(0, read.getLong(0));
+ assertEquals(72, out.position());
+ assertEquals(72, out.flushed());
+ assertEquals(72, out.flushedToNetwork());
+
+ read = channel.readOutbound();
+ assertEquals(32, read.readableBytes());
+ assertEquals(0, read.getLong(0));
+ }
+
+ }
+
+}
diff --git a/test/unit/org/apache/cassandra/net/AsyncOneResponseTest.java b/test/unit/org/apache/cassandra/net/AsyncOneResponseTest.java
new file mode 100644
index 0000000..3d0508c
--- /dev/null
+++ b/test/unit/org/apache/cassandra/net/AsyncOneResponseTest.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.net;
+
+import java.util.concurrent.TimeUnit;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import static org.junit.Assert.assertTrue;
+
+public class AsyncOneResponseTest
+{
+ @Test
+ public void getThrowsExceptionAfterTimeout() throws InterruptedException
+ {
+ AsyncOneResponse<Object> response = new AsyncOneResponse<>();
+ Thread.sleep(2000);
+ Assert.assertFalse(response.await(1, TimeUnit.SECONDS));
+ }
+
+ @Test
+ public void getThrowsExceptionAfterCorrectTimeout() throws InterruptedException
+ {
+ AsyncOneResponse<Object> response = new AsyncOneResponse<>();
+
+ final long expectedTimeoutMillis = 1000; // Should time out after roughly this time
+ final long schedulingError = 10; // Scheduling is imperfect
+
+ long startTime = System.nanoTime();
+ boolean timeout = !response.await(expectedTimeoutMillis, TimeUnit.MILLISECONDS);
+ long endTime = System.nanoTime();
+
+ assertTrue(timeout);
+ assertTrue(TimeUnit.NANOSECONDS.toMillis(endTime - startTime) > (expectedTimeoutMillis - schedulingError));
+ }
+}
diff --git a/test/unit/org/apache/cassandra/net/AsyncPromiseTest.java b/test/unit/org/apache/cassandra/net/AsyncPromiseTest.java
new file mode 100644
index 0000000..0d2a2e9
--- /dev/null
+++ b/test/unit/org/apache/cassandra/net/AsyncPromiseTest.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.net;
+
+import org.junit.After;
+import org.junit.Test;
+
+import io.netty.util.concurrent.ImmediateEventExecutor;
+import io.netty.util.concurrent.Promise;
+
+public class AsyncPromiseTest extends TestAbstractAsyncPromise
+{
+ @After
+ public void shutdown()
+ {
+ exec.shutdownNow();
+ }
+
+ private <V> Promise<V> newPromise()
+ {
+ return new AsyncPromise<>(ImmediateEventExecutor.INSTANCE);
+ }
+
+ @Test
+ public void testSuccess()
+ {
+ for (boolean setUncancellable : new boolean[] { false, true })
+ for (boolean tryOrSet : new boolean[]{ false, true })
+ for (Integer v : new Integer[]{ null, 1 })
+ testOneSuccess(newPromise(), setUncancellable, tryOrSet, v, 2);
+ }
+
+ @Test
+ public void testFailure()
+ {
+ for (boolean setUncancellable : new boolean[] { false, true })
+ for (boolean tryOrSet : new boolean[] { false, true })
+ for (Throwable v : new Throwable[] { null, new NullPointerException() })
+ testOneFailure(newPromise(), setUncancellable, tryOrSet, v, 2);
+ }
+
+
+ @Test
+ public void testCancellation()
+ {
+ for (boolean interruptIfRunning : new boolean[] { true, false })
+ testOneCancellation(newPromise(), interruptIfRunning, 2);
+ }
+
+
+ @Test
+ public void testTimeout()
+ {
+ for (boolean setUncancellable : new boolean[] { true, false })
+ testOneTimeout(newPromise(), setUncancellable);
+ }
+
+}
diff --git a/test/unit/org/apache/cassandra/net/AsyncStreamingInputPlusTest.java b/test/unit/org/apache/cassandra/net/AsyncStreamingInputPlusTest.java
new file mode 100644
index 0000000..b575747
--- /dev/null
+++ b/test/unit/org/apache/cassandra/net/AsyncStreamingInputPlusTest.java
@@ -0,0 +1,280 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.net;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.WritableByteChannel;
+import java.util.concurrent.TimeUnit;
+
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.Unpooled;
+import io.netty.channel.embedded.EmbeddedChannel;
+import org.apache.cassandra.io.util.BufferedDataOutputStreamPlus;
+import org.apache.cassandra.net.AsyncStreamingInputPlus;
+import org.apache.cassandra.net.AsyncStreamingInputPlus.InputTimeoutException;
+
+import static org.junit.Assert.assertFalse;
+
+public class AsyncStreamingInputPlusTest
+{
+ private EmbeddedChannel channel;
+ private AsyncStreamingInputPlus inputPlus;
+ private ByteBuf buf;
+
+ @Before
+ public void setUp()
+ {
+ channel = new EmbeddedChannel();
+ }
+
+ @After
+ public void tearDown()
+ {
+ channel.close();
+
+ if (buf != null && buf.refCnt() > 0)
+ buf.release(buf.refCnt());
+ }
+
+// @Test
+// public void isOpen()
+// {
+// Assert.assertTrue(inputPlus.isOpen());
+// inputPlus.requestClosure();
+// Assert.assertFalse(inputPlus.isOpen());
+// }
+
+ @Test
+ public void append_closed()
+ {
+ inputPlus = new AsyncStreamingInputPlus(channel);
+ inputPlus.requestClosure();
+ inputPlus.close();
+ buf = channel.alloc().buffer(4);
+ assertFalse(inputPlus.append(buf));
+ }
+
+ @Test
+ public void append_normal()
+ {
+ inputPlus = new AsyncStreamingInputPlus(channel);
+ int size = 4;
+ buf = channel.alloc().buffer(size);
+ buf.writerIndex(size);
+ inputPlus.append(buf);
+ Assert.assertEquals(buf.readableBytes(), inputPlus.unsafeAvailable());
+ }
+
+ @Test
+ public void read() throws IOException
+ {
+ inputPlus = new AsyncStreamingInputPlus(channel);
+ // put two buffers of 8 bytes each into the queue.
+ // then read an int, then a long. the latter tests offset into the inputPlus, as well as spanning across queued buffers.
+ // the values of those int/long will both be '42', but spread across both queue buffers.
+ ByteBuf buf = channel.alloc().buffer(8);
+ buf.writeInt(42);
+ buf.writerIndex(8);
+ inputPlus.append(buf);
+ buf = channel.alloc().buffer(8);
+ buf.writeInt(42);
+ buf.writerIndex(8);
+ inputPlus.append(buf);
+ Assert.assertEquals(16, inputPlus.unsafeAvailable());
+
+// ByteBuffer out = ByteBuffer.allocate(4);
+// int readCount = inputPlus.read(out);
+// Assert.assertEquals(4, readCount);
+// out.flip();
+// Assert.assertEquals(42, out.getInt());
+// Assert.assertEquals(12, inputPlus.unsafeAvailable());
+
+// out = ByteBuffer.allocate(8);
+// readCount = inputPlus.read(out);
+// Assert.assertEquals(8, readCount);
+// out.flip();
+// Assert.assertEquals(42, out.getLong());
+// Assert.assertEquals(4, inputPlus.unsafeAvailable());
+ }
+
+// @Test (expected = EOFException.class)
+// public void read_closed() throws IOException
+// {
+// inputPlus.requestClosure();
+// ByteBuffer buf = ByteBuffer.allocate(1);
+// inputPlus.read(buf);
+// }
+
+ @Test
+ public void available_closed()
+ {
+ inputPlus = new AsyncStreamingInputPlus(channel);
+ inputPlus.requestClosure();
+ inputPlus.unsafeAvailable();
+ }
+
+ @Test
+ public void available_HappyPath()
+ {
+ inputPlus = new AsyncStreamingInputPlus(channel);
+ int size = 4;
+ buf = channel.alloc().heapBuffer(size);
+ buf.writerIndex(size);
+ inputPlus.append(buf);
+ Assert.assertEquals(size, inputPlus.unsafeAvailable());
+ }
+
+ @Test
+ public void available_ClosedButWithBytes()
+ {
+ inputPlus = new AsyncStreamingInputPlus(channel);
+ int size = 4;
+ buf = channel.alloc().heapBuffer(size);
+ buf.writerIndex(size);
+ inputPlus.append(buf);
+ inputPlus.requestClosure();
+ Assert.assertEquals(size, inputPlus.unsafeAvailable());
+ }
+
+ @Test
+ public void consumeUntil_SingleBuffer_Partial_HappyPath() throws IOException
+ {
+ consumeUntilTestCycle(1, 8, 0, 4);
+ }
+
+ @Test
+ public void consumeUntil_SingleBuffer_AllBytes_HappyPath() throws IOException
+ {
+ consumeUntilTestCycle(1, 8, 0, 8);
+ }
+
+ @Test
+ public void consumeUntil_MultipleBufferr_Partial_HappyPath() throws IOException
+ {
+ consumeUntilTestCycle(2, 8, 0, 13);
+ }
+
+ @Test
+ public void consumeUntil_MultipleBuffer_AllBytes_HappyPath() throws IOException
+ {
+ consumeUntilTestCycle(2, 8, 0, 16);
+ }
+
+ @Test(expected = EOFException.class)
+ public void consumeUntil_SingleBuffer_Fails() throws IOException
+ {
+ consumeUntilTestCycle(1, 8, 0, 9);
+ }
+
+ @Test(expected = EOFException.class)
+ public void consumeUntil_MultipleBuffer_Fails() throws IOException
+ {
+ consumeUntilTestCycle(2, 8, 0, 17);
+ }
+
+ private void consumeUntilTestCycle(int nBuffs, int buffSize, int startOffset, int len) throws IOException
+ {
+ inputPlus = new AsyncStreamingInputPlus(channel);
+
+ byte[] expectedBytes = new byte[len];
+ int count = 0;
+ for (int j=0; j < nBuffs; j++)
+ {
+ ByteBuf buf = channel.alloc().buffer(buffSize);
+ for (int i = 0; i < buf.capacity(); i++)
+ {
+ buf.writeByte(j);
+ if (count >= startOffset && (count - startOffset) < len)
+ expectedBytes[count - startOffset] = (byte)j;
+ count++;
+ }
+
+ inputPlus.append(buf);
+ }
+ inputPlus.requestClosure();
+
+ TestableWritableByteChannel wbc = new TestableWritableByteChannel(len);
+
+ inputPlus.skipBytesFully(startOffset);
+ BufferedDataOutputStreamPlus writer = new BufferedDataOutputStreamPlus(wbc);
+ inputPlus.consume(buffer -> { writer.write(buffer); return buffer.remaining(); }, len);
+ writer.close();
+
+ Assert.assertEquals(String.format("Test with %d buffers starting at %d consuming %d bytes", nBuffs, startOffset, len),
+ len, wbc.writtenBytes.readableBytes());
+
+ Assert.assertArrayEquals(expectedBytes, wbc.writtenBytes.array());
+ }
+
+ private static class TestableWritableByteChannel implements WritableByteChannel
+ {
+ private boolean isOpen = true;
+ public ByteBuf writtenBytes;
+
+ public TestableWritableByteChannel(int initialCapacity)
+ {
+ writtenBytes = Unpooled.buffer(initialCapacity);
+ }
+
+ public int write(ByteBuffer src)
+ {
+ int size = src.remaining();
+ writtenBytes.writeBytes(src);
+ return size;
+ }
+
+ public boolean isOpen()
+ {
+ return isOpen;
+ }
+
+ public void close()
+ {
+ isOpen = false;
+ }
+ }
+
+ @Test
+ public void rebufferTimeout() throws IOException
+ {
+ long timeoutMillis = 1000;
+ inputPlus = new AsyncStreamingInputPlus(channel, timeoutMillis, TimeUnit.MILLISECONDS);
+
+ long startNanos = System.nanoTime();
+ try
+ {
+ inputPlus.readInt();
+ Assert.fail("should not have been able to read from the queue");
+ }
+ catch (InputTimeoutException e)
+ {
+ // this is the success case, and is expected. any other exception is a failure.
+ }
+
+ long durationNanos = System.nanoTime() - startNanos;
+ Assert.assertTrue(TimeUnit.MILLISECONDS.toNanos(timeoutMillis) <= durationNanos);
+ }
+}
diff --git a/test/unit/org/apache/cassandra/net/AsyncStreamingOutputPlusTest.java b/test/unit/org/apache/cassandra/net/AsyncStreamingOutputPlusTest.java
new file mode 100644
index 0000000..305dc55
--- /dev/null
+++ b/test/unit/org/apache/cassandra/net/AsyncStreamingOutputPlusTest.java
@@ -0,0 +1,172 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.net;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
+import java.nio.file.Files;
+import java.util.Random;
+
+import org.junit.Test;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.channel.embedded.EmbeddedChannel;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.streaming.StreamManager;
+import org.apache.cassandra.utils.FBUtilities;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class AsyncStreamingOutputPlusTest
+{
+
+ static
+ {
+ DatabaseDescriptor.daemonInitialization();
+ }
+
+ @Test
+ public void testSuccess() throws IOException
+ {
+ EmbeddedChannel channel = new TestChannel(4);
+ ByteBuf read;
+ try (AsyncStreamingOutputPlus out = new AsyncStreamingOutputPlus(channel))
+ {
+ out.writeInt(1);
+ assertEquals(0, out.flushed());
+ assertEquals(0, out.flushedToNetwork());
+ assertEquals(4, out.position());
+
+ out.doFlush(0);
+ assertEquals(4, out.flushed());
+ assertEquals(4, out.flushedToNetwork());
+
+ out.writeInt(2);
+ assertEquals(8, out.position());
+ assertEquals(4, out.flushed());
+ assertEquals(4, out.flushedToNetwork());
+
+ out.doFlush(0);
+ assertEquals(8, out.position());
+ assertEquals(8, out.flushed());
+ assertEquals(4, out.flushedToNetwork());
+
+ read = channel.readOutbound();
+ assertEquals(4, read.readableBytes());
+ assertEquals(1, read.getInt(0));
+ assertEquals(8, out.flushed());
+ assertEquals(8, out.flushedToNetwork());
+
+ read = channel.readOutbound();
+ assertEquals(4, read.readableBytes());
+ assertEquals(2, read.getInt(0));
+
+ out.write(new byte[16]);
+ assertEquals(24, out.position());
+ assertEquals(8, out.flushed());
+ assertEquals(8, out.flushedToNetwork());
+
+ out.doFlush(0);
+ assertEquals(24, out.position());
+ assertEquals(24, out.flushed());
+ assertEquals(24, out.flushedToNetwork());
+
+ read = channel.readOutbound();
+ assertEquals(16, read.readableBytes());
+ assertEquals(0, read.getLong(0));
+ assertEquals(0, read.getLong(8));
+ assertEquals(24, out.position());
+ assertEquals(24, out.flushed());
+ assertEquals(24, out.flushedToNetwork());
+
+ out.writeToChannel(alloc -> {
+ ByteBuffer buffer = alloc.get(16);
+ buffer.putLong(1);
+ buffer.putLong(2);
+ buffer.flip();
+ }, new StreamManager.StreamRateLimiter(FBUtilities.getBroadcastAddressAndPort()));
+
+ assertEquals(40, out.position());
+ assertEquals(40, out.flushed());
+ assertEquals(40, out.flushedToNetwork());
+
+ read = channel.readOutbound();
+ assertEquals(16, read.readableBytes());
+ assertEquals(1, read.getLong(0));
+ assertEquals(2, read.getLong(8));
+ }
+ }
+
+ @Test
+ public void testWriteFileToChannelZeroCopy() throws IOException
+ {
+ testWriteFileToChannel(true);
+ }
+
+ @Test
+ public void testWriteFileToChannelSSL() throws IOException
+ {
+ testWriteFileToChannel(false);
+ }
+
+ private void testWriteFileToChannel(boolean zeroCopy) throws IOException
+ {
+ File file = populateTempData("zero_copy_" + zeroCopy);
+ int length = (int) file.length();
+
+ EmbeddedChannel channel = new TestChannel(4);
+ StreamManager.StreamRateLimiter limiter = new StreamManager.StreamRateLimiter(FBUtilities.getBroadcastAddressAndPort());
+
+ try (RandomAccessFile raf = new RandomAccessFile(file.getPath(), "r");
+ FileChannel fileChannel = raf.getChannel();
+ AsyncStreamingOutputPlus out = new AsyncStreamingOutputPlus(channel))
+ {
+ assertTrue(fileChannel.isOpen());
+
+ if (zeroCopy)
+ out.writeFileToChannelZeroCopy(fileChannel, limiter, length, length, length * 2);
+ else
+ out.writeFileToChannel(fileChannel, limiter, length);
+
+ assertEquals(length, out.flushed());
+ assertEquals(length, out.flushedToNetwork());
+ assertEquals(length, out.position());
+
+ assertFalse(fileChannel.isOpen());
+ }
+ }
+
+ private File populateTempData(String name) throws IOException
+ {
+ File file = Files.createTempFile(name, ".txt").toFile();
+ file.deleteOnExit();
+
+ Random r = new Random();
+ byte [] content = new byte[16];
+ r.nextBytes(content);
+ Files.write(file.toPath(), content);
+
+ return file;
+ }
+}
diff --git a/test/unit/org/apache/cassandra/net/ChunkedInputPlusTest.java b/test/unit/org/apache/cassandra/net/ChunkedInputPlusTest.java
new file mode 100644
index 0000000..f90fcd1
--- /dev/null
+++ b/test/unit/org/apache/cassandra/net/ChunkedInputPlusTest.java
@@ -0,0 +1,159 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.net;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+import com.google.common.collect.Lists;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.net.ChunkedInputPlus;
+import org.apache.cassandra.net.ShareableBytes;
+
+import static org.junit.Assert.*;
+
+public class ChunkedInputPlusTest
+{
+ @BeforeClass
+ public static void setUp()
+ {
+ DatabaseDescriptor.clientInitialization();
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testEmptyIterable()
+ {
+ ChunkedInputPlus.of(Collections.emptyList());
+ }
+
+ @Test
+ public void testUnderRead() throws IOException
+ {
+ List<ShareableBytes> chunks = Lists.newArrayList(
+ chunk(1, 1), chunk(2, 2), chunk(3, 3)
+ );
+
+ try (ChunkedInputPlus input = ChunkedInputPlus.of(chunks))
+ {
+ byte[] readBytes = new byte[5];
+ input.readFully(readBytes);
+ assertArrayEquals(new byte[] { 1, 2, 2, 3, 3 }, readBytes);
+
+ assertFalse(chunks.get(0).hasRemaining());
+ assertFalse(chunks.get(1).hasRemaining());
+ assertTrue (chunks.get(2).hasRemaining());
+
+ assertTrue (chunks.get(0).isReleased());
+ assertTrue (chunks.get(1).isReleased());
+ assertFalse(chunks.get(2).isReleased());
+ }
+
+ // close should release the last chunk
+ assertTrue(chunks.get(2).isReleased());
+ }
+
+ @Test
+ public void testExactRead() throws IOException
+ {
+ List<ShareableBytes> chunks = Lists.newArrayList(
+ chunk(1, 1), chunk(2, 2), chunk(3, 3)
+ );
+
+ try (ChunkedInputPlus input = ChunkedInputPlus.of(chunks))
+ {
+ byte[] readBytes = new byte[6];
+ input.readFully(readBytes);
+ assertArrayEquals(new byte[] { 1, 2, 2, 3, 3, 3 }, readBytes);
+
+ assertFalse(chunks.get(0).hasRemaining());
+ assertFalse(chunks.get(1).hasRemaining());
+ assertFalse(chunks.get(2).hasRemaining());
+
+ assertTrue (chunks.get(0).isReleased());
+ assertTrue (chunks.get(1).isReleased());
+ assertFalse(chunks.get(2).isReleased());
+ }
+
+ // close should release the last chunk
+ assertTrue(chunks.get(2).isReleased());
+ }
+
+ @Test
+ public void testOverRead() throws IOException
+ {
+ List<ShareableBytes> chunks = Lists.newArrayList(
+ chunk(1, 1), chunk(2, 2), chunk(3, 3)
+ );
+
+ boolean eofCaught = false;
+ try (ChunkedInputPlus input = ChunkedInputPlus.of(chunks))
+ {
+ byte[] readBytes = new byte[7];
+ input.readFully(readBytes);
+ assertArrayEquals(new byte[] { 1, 2, 2, 3, 3, 3, 4 }, readBytes);
+ }
+ catch (EOFException e)
+ {
+ eofCaught = true;
+
+ assertFalse(chunks.get(0).hasRemaining());
+ assertFalse(chunks.get(1).hasRemaining());
+ assertFalse(chunks.get(2).hasRemaining());
+
+ assertTrue (chunks.get(2).isReleased());
+ assertTrue (chunks.get(1).isReleased());
+ assertTrue (chunks.get(2).isReleased());
+ }
+ assertTrue(eofCaught);
+ }
+
+ @Test
+ public void testRemainder() throws IOException
+ {
+ List<ShareableBytes> chunks = Lists.newArrayList(
+ chunk(1, 1), chunk(2, 2), chunk(3, 3)
+ );
+
+ try (ChunkedInputPlus input = ChunkedInputPlus.of(chunks))
+ {
+ byte[] readBytes = new byte[5];
+ input.readFully(readBytes);
+ assertArrayEquals(new byte[] { 1, 2, 2, 3, 3 }, readBytes);
+
+ assertEquals(1, input.remainder());
+
+ assertTrue(chunks.get(0).isReleased());
+ assertTrue(chunks.get(1).isReleased());
+ assertTrue(chunks.get(2).isReleased()); // should be released by remainder()
+ }
+ }
+
+ private ShareableBytes chunk(int size, int fill)
+ {
+ ByteBuffer buffer = ByteBuffer.allocate(size);
+ Arrays.fill(buffer.array(), (byte) fill);
+ return ShareableBytes.wrap(buffer);
+ }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/net/ConnectionTest.java b/test/unit/org/apache/cassandra/net/ConnectionTest.java
new file mode 100644
index 0000000..eb8d867
--- /dev/null
+++ b/test/unit/org/apache/cassandra/net/ConnectionTest.java
@@ -0,0 +1,954 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.net;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.Function;
+import java.util.function.Supplier;
+import java.util.function.ToLongFunction;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Sets;
+import com.google.common.util.concurrent.Uninterruptibles;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.channel.ChannelOutboundHandlerAdapter;
+import io.netty.channel.ChannelPromise;
+import org.apache.cassandra.config.Config;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.config.EncryptionOptions;
+import org.apache.cassandra.db.commitlog.CommitLog;
+import org.apache.cassandra.exceptions.RequestFailureReason;
+import org.apache.cassandra.exceptions.UnknownColumnException;
+import org.apache.cassandra.io.IVersionedAsymmetricSerializer;
+import org.apache.cassandra.io.IVersionedSerializer;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.utils.FBUtilities;
+
+import static java.util.concurrent.TimeUnit.MINUTES;
+import static java.util.concurrent.TimeUnit.SECONDS;
+import static org.apache.cassandra.net.MessagingService.VERSION_30;
+import static org.apache.cassandra.net.MessagingService.VERSION_3014;
+import static org.apache.cassandra.net.MessagingService.VERSION_40;
+import static org.apache.cassandra.net.NoPayload.noPayload;
+import static org.apache.cassandra.net.MessagingService.current_version;
+import static org.apache.cassandra.net.ConnectionUtils.*;
+import static org.apache.cassandra.net.ConnectionType.LARGE_MESSAGES;
+import static org.apache.cassandra.net.ConnectionType.SMALL_MESSAGES;
+import static org.apache.cassandra.net.OutboundConnectionSettings.Framing.LZ4;
+import static org.apache.cassandra.net.OutboundConnections.LARGE_MESSAGE_THRESHOLD;
+import static org.apache.cassandra.utils.MonotonicClock.approxTime;
+
+public class ConnectionTest
+{
+ private static final Logger logger = LoggerFactory.getLogger(ConnectionTest.class);
+ private static final SocketFactory factory = new SocketFactory();
+
+ private final Map<Verb, Supplier<? extends IVersionedAsymmetricSerializer<?, ?>>> serializers = new HashMap<>();
+ private final Map<Verb, Supplier<? extends IVerbHandler<?>>> handlers = new HashMap<>();
+ private final Map<Verb, ToLongFunction<TimeUnit>> timeouts = new HashMap<>();
+
+ private void unsafeSetSerializer(Verb verb, Supplier<? extends IVersionedAsymmetricSerializer<?, ?>> supplier) throws Throwable
+ {
+ serializers.putIfAbsent(verb, verb.unsafeSetSerializer(supplier));
+ }
+
+ private void unsafeSetHandler(Verb verb, Supplier<? extends IVerbHandler<?>> supplier) throws Throwable
+ {
+ handlers.putIfAbsent(verb, verb.unsafeSetHandler(supplier));
+ }
+
+ private void unsafeSetExpiration(Verb verb, ToLongFunction<TimeUnit> expiration) throws Throwable
+ {
+ timeouts.putIfAbsent(verb, verb.unsafeSetExpiration(expiration));
+ }
+
+ @After
+ public void resetVerbs() throws Throwable
+ {
+ for (Map.Entry<Verb, Supplier<? extends IVersionedAsymmetricSerializer<?, ?>>> e : serializers.entrySet())
+ e.getKey().unsafeSetSerializer(e.getValue());
+ serializers.clear();
+ for (Map.Entry<Verb, Supplier<? extends IVerbHandler<?>>> e : handlers.entrySet())
+ e.getKey().unsafeSetHandler(e.getValue());
+ handlers.clear();
+ for (Map.Entry<Verb, ToLongFunction<TimeUnit>> e : timeouts.entrySet())
+ e.getKey().unsafeSetExpiration(e.getValue());
+ timeouts.clear();
+ }
+
+ @BeforeClass
+ public static void startup()
+ {
+ DatabaseDescriptor.daemonInitialization();
+ CommitLog.instance.start();
+ }
+
+ @AfterClass
+ public static void cleanup() throws InterruptedException
+ {
+ factory.shutdownNow();
+ }
+
+ interface SendTest
+ {
+ void accept(InboundMessageHandlers inbound, OutboundConnection outbound, InetAddressAndPort endpoint) throws Throwable;
+ }
+
+ interface ManualSendTest
+ {
+ void accept(Settings settings, InboundSockets inbound, OutboundConnection outbound, InetAddressAndPort endpoint) throws Throwable;
+ }
+
+ static class Settings
+ {
+ static final Settings SMALL = new Settings(SMALL_MESSAGES);
+ static final Settings LARGE = new Settings(LARGE_MESSAGES);
+ final ConnectionType type;
+ final Function<OutboundConnectionSettings, OutboundConnectionSettings> outbound;
+ final Function<InboundConnectionSettings, InboundConnectionSettings> inbound;
+ Settings(ConnectionType type)
+ {
+ this(type, Function.identity(), Function.identity());
+ }
+ Settings(ConnectionType type, Function<OutboundConnectionSettings, OutboundConnectionSettings> outbound,
+ Function<InboundConnectionSettings, InboundConnectionSettings> inbound)
+ {
+ this.type = type;
+ this.outbound = outbound;
+ this.inbound = inbound;
+ }
+ Settings outbound(Function<OutboundConnectionSettings, OutboundConnectionSettings> outbound)
+ {
+ return new Settings(type, this.outbound.andThen(outbound), inbound);
+ }
+ Settings inbound(Function<InboundConnectionSettings, InboundConnectionSettings> inbound)
+ {
+ return new Settings(type, outbound, this.inbound.andThen(inbound));
+ }
+ Settings override(Settings settings)
+ {
+ return new Settings(settings.type != null ? settings.type : type,
+ outbound.andThen(settings.outbound),
+ inbound.andThen(settings.inbound));
+ }
+ }
+
+ static final EncryptionOptions.ServerEncryptionOptions encryptionOptions =
+ new EncryptionOptions.ServerEncryptionOptions()
+ .withLegacySslStoragePort(true)
+ .withOptional(true)
+ .withInternodeEncryption(EncryptionOptions.ServerEncryptionOptions.InternodeEncryption.all)
+ .withKeyStore("test/conf/cassandra_ssl_test.keystore")
+ .withKeyStorePassword("cassandra")
+ .withTrustStore("test/conf/cassandra_ssl_test.truststore")
+ .withTrustStorePassword("cassandra")
+ .withRequireClientAuth(false)
+ .withCipherSuites("TLS_RSA_WITH_AES_128_CBC_SHA");
+
+ static final AcceptVersions legacy = new AcceptVersions(VERSION_30, VERSION_30);
+
+ static final List<Function<Settings, Settings>> MODIFIERS = ImmutableList.of(
+ settings -> settings.outbound(outbound -> outbound.withAcceptVersions(legacy))
+ .inbound(inbound -> inbound.withAcceptMessaging(legacy)),
+ settings -> settings.outbound(outbound -> outbound.withEncryption(encryptionOptions))
+ .inbound(inbound -> inbound.withEncryption(encryptionOptions)),
+ settings -> settings.outbound(outbound -> outbound.withFraming(LZ4))
+ );
+
+ static final List<Settings> SETTINGS = applyPowerSet(
+ ImmutableList.of(Settings.SMALL, Settings.LARGE),
+ MODIFIERS
+ );
+
+ private static <T> List<T> applyPowerSet(List<T> settings, List<Function<T, T>> modifiers)
+ {
+ List<T> result = new ArrayList<>();
+ for (Set<Function<T, T>> set : Sets.powerSet(new HashSet<>(modifiers)))
+ {
+ for (T s : settings)
+ {
+ for (Function<T, T> f : set)
+ s = f.apply(s);
+ result.add(s);
+ }
+ }
+ return result;
+ }
+
+ private void test(Settings extraSettings, SendTest test) throws Throwable
+ {
+ for (Settings s : SETTINGS)
+ doTest(s.override(extraSettings), test);
+ }
+ private void test(SendTest test) throws Throwable
+ {
+ for (Settings s : SETTINGS)
+ doTest(s, test);
+ }
+
+ private void testManual(ManualSendTest test) throws Throwable
+ {
+ for (Settings s : SETTINGS)
+ doTestManual(s, test);
+ }
+
+ private void doTest(Settings settings, SendTest test) throws Throwable
+ {
+ doTestManual(settings, (ignore, inbound, outbound, endpoint) -> {
+ inbound.open().sync();
+ test.accept(MessagingService.instance().getInbound(endpoint), outbound, endpoint);
+ });
+ }
+
+ private void doTestManual(Settings settings, ManualSendTest test) throws Throwable
+ {
+ InetAddressAndPort endpoint = FBUtilities.getBroadcastAddressAndPort();
+ InboundConnectionSettings inboundSettings = settings.inbound.apply(new InboundConnectionSettings())
+ .withBindAddress(endpoint)
+ .withSocketFactory(factory);
+ InboundSockets inbound = new InboundSockets(Collections.singletonList(inboundSettings));
+ OutboundConnectionSettings outboundTemplate = settings.outbound.apply(new OutboundConnectionSettings(endpoint))
+ .withDefaultReserveLimits()
+ .withSocketFactory(factory)
+ .withDefaults(ConnectionCategory.MESSAGING);
+ ResourceLimits.EndpointAndGlobal reserveCapacityInBytes = new ResourceLimits.EndpointAndGlobal(new ResourceLimits.Concurrent(outboundTemplate.applicationSendQueueReserveEndpointCapacityInBytes), outboundTemplate.applicationSendQueueReserveGlobalCapacityInBytes);
+ OutboundConnection outbound = new OutboundConnection(settings.type, outboundTemplate, reserveCapacityInBytes);
+ try
+ {
+ logger.info("Running {} {} -> {}", outbound.messagingVersion(), outbound.settings(), inboundSettings);
+ test.accept(settings, inbound, outbound, endpoint);
+ }
+ finally
+ {
+ outbound.close(false);
+ inbound.close().get(30L, SECONDS);
+ outbound.close(false).get(30L, SECONDS);
+ resetVerbs();
+ MessagingService.instance().messageHandlers.clear();
+ }
+ }
+
+ @Test
+ public void testSendSmall() throws Throwable
+ {
+ test((inbound, outbound, endpoint) -> {
+ int version = outbound.settings().acceptVersions.max;
+ int count = 10;
+
+ CountDownLatch deliveryDone = new CountDownLatch(1);
+ CountDownLatch receiveDone = new CountDownLatch(count);
+
+ unsafeSetHandler(Verb._TEST_1, () -> msg -> receiveDone.countDown());
+ Message<?> message = Message.out(Verb._TEST_1, noPayload);
+ for (int i = 0 ; i < count ; ++i)
+ outbound.enqueue(message);
+
+ Assert.assertTrue(receiveDone.await(10, SECONDS));
+ outbound.unsafeRunOnDelivery(deliveryDone::countDown);
+ Assert.assertTrue(deliveryDone.await(10, SECONDS));
+
+ check(outbound).submitted(10)
+ .sent (10, 10 * message.serializedSize(version))
+ .pending ( 0, 0)
+ .overload ( 0, 0)
+ .expired ( 0, 0)
+ .error ( 0, 0)
+ .check();
+ check(inbound) .received (10, 10 * message.serializedSize(version))
+ .processed(10, 10 * message.serializedSize(version))
+ .pending ( 0, 0)
+ .expired ( 0, 0)
+ .error ( 0, 0)
+ .check();
+ });
+ }
+
+ @Test
+ public void testSendLarge() throws Throwable
+ {
+ test((inbound, outbound, endpoint) -> {
+ int version = outbound.settings().acceptVersions.max;
+ int count = 10;
+
+ CountDownLatch deliveryDone = new CountDownLatch(1);
+ CountDownLatch receiveDone = new CountDownLatch(count);
+
+ unsafeSetSerializer(Verb._TEST_1, () -> new IVersionedSerializer<Object>()
+ {
+ public void serialize(Object noPayload, DataOutputPlus out, int version) throws IOException
+ {
+ for (int i = 0 ; i < LARGE_MESSAGE_THRESHOLD + 1 ; ++i)
+ out.writeByte(i);
+ }
+ public Object deserialize(DataInputPlus in, int version) throws IOException
+ {
+ in.skipBytesFully(LARGE_MESSAGE_THRESHOLD + 1);
+ return noPayload;
+ }
+ public long serializedSize(Object noPayload, int version)
+ {
+ return LARGE_MESSAGE_THRESHOLD + 1;
+ }
+ });
+ unsafeSetHandler(Verb._TEST_1, () -> msg -> receiveDone.countDown());
+ Message<?> message = Message.builder(Verb._TEST_1, new Object())
+ .withExpiresAt(System.nanoTime() + SECONDS.toNanos(30L))
+ .build();
+ for (int i = 0 ; i < count ; ++i)
+ outbound.enqueue(message);
+ Assert.assertTrue(receiveDone.await(10, SECONDS));
+
+ outbound.unsafeRunOnDelivery(deliveryDone::countDown);
+ Assert.assertTrue(deliveryDone.await(10, SECONDS));
+
+ check(outbound).submitted(10)
+ .sent (10, 10 * message.serializedSize(version))
+ .pending ( 0, 0)
+ .overload ( 0, 0)
+ .expired ( 0, 0)
+ .error ( 0, 0)
+ .check();
+ check(inbound) .received (10, 10 * message.serializedSize(version))
+ .processed(10, 10 * message.serializedSize(version))
+ .pending ( 0, 0)
+ .expired ( 0, 0)
+ .error ( 0, 0)
+ .check();
+ });
+ }
+
+ @Test
+ public void testInsufficientSpace() throws Throwable
+ {
+ test(new Settings(null).outbound(settings -> settings
+ .withApplicationReserveSendQueueCapacityInBytes(1 << 15, new ResourceLimits.Concurrent(1 << 16))
+ .withApplicationSendQueueCapacityInBytes(1 << 16)),
+ (inbound, outbound, endpoint) -> {
+
+ CountDownLatch done = new CountDownLatch(1);
+ Message<?> message = Message.out(Verb._TEST_1, new Object());
+ MessagingService.instance().callbacks.addWithExpiration(new RequestCallback()
+ {
+ @Override
+ public void onFailure(InetAddressAndPort from, RequestFailureReason failureReason)
+ {
+ done.countDown();
+ }
+
+ @Override
+ public boolean invokeOnFailure()
+ {
+ return true;
+ }
+
+ @Override
+ public void onResponse(Message msg)
+ {
+ throw new IllegalStateException();
+ }
+
+ }, message, endpoint);
+ AtomicInteger delivered = new AtomicInteger();
+ unsafeSetSerializer(Verb._TEST_1, () -> new IVersionedSerializer<Object>()
+ {
+ public void serialize(Object o, DataOutputPlus out, int version) throws IOException
+ {
+ for (int i = 0 ; i <= 4 << 16 ; i += 8L)
+ out.writeLong(1L);
+ }
+
+ public Object deserialize(DataInputPlus in, int version) throws IOException
+ {
+ in.skipBytesFully(4 << 16);
+ return null;
+ }
+
+ public long serializedSize(Object o, int version)
+ {
+ return 4 << 16;
+ }
+ });
+ unsafeSetHandler(Verb._TEST_1, () -> msg -> delivered.incrementAndGet());
+ outbound.enqueue(message);
+ Assert.assertTrue(done.await(10, SECONDS));
+ Assert.assertEquals(0, delivered.get());
+ check(outbound).submitted( 1)
+ .sent ( 0, 0)
+ .pending ( 0, 0)
+ .overload ( 1, message.serializedSize(current_version))
+ .expired ( 0, 0)
+ .error ( 0, 0)
+ .check();
+ check(inbound) .received ( 0, 0)
+ .processed( 0, 0)
+ .pending ( 0, 0)
+ .expired ( 0, 0)
+ .error ( 0, 0)
+ .check();
+ });
+ }
+
+ @Test
+ public void testSerializeError() throws Throwable
+ {
+ test((inbound, outbound, endpoint) -> {
+ int version = outbound.settings().acceptVersions.max;
+ int count = 100;
+
+ CountDownLatch deliveryDone = new CountDownLatch(1);
+ CountDownLatch receiveDone = new CountDownLatch(90);
+
+ AtomicInteger serialized = new AtomicInteger();
+ Message<?> message = Message.builder(Verb._TEST_1, new Object())
+ .withExpiresAt(System.nanoTime() + SECONDS.toNanos(30L))
+ .build();
+ unsafeSetSerializer(Verb._TEST_1, () -> new IVersionedSerializer<Object>()
+ {
+ public void serialize(Object o, DataOutputPlus out, int version) throws IOException
+ {
+ int i = serialized.incrementAndGet();
+ if (0 == (i & 15))
+ {
+ if (0 == (i & 16))
+ out.writeByte(i);
+ throw new IOException();
+ }
+
+ if (1 != (i & 31))
+ out.writeByte(i);
+ }
+
+ public Object deserialize(DataInputPlus in, int version) throws IOException
+ {
+ in.readByte();
+ return null;
+ }
+
+ public long serializedSize(Object o, int version)
+ {
+ return 1;
+ }
+ });
+
+ unsafeSetHandler(Verb._TEST_1, () -> msg -> receiveDone.countDown());
+ for (int i = 0 ; i < count ; ++i)
+ outbound.enqueue(message);
+
+ Assert.assertTrue(receiveDone.await(1, MINUTES));
+ outbound.unsafeRunOnDelivery(deliveryDone::countDown);
+ Assert.assertTrue(deliveryDone.await(10, SECONDS));
+
+ check(outbound).submitted(100)
+ .sent ( 90, 90 * message.serializedSize(version))
+ .pending ( 0, 0)
+ .overload ( 0, 0)
+ .expired ( 0, 0)
+ .error ( 10, 10 * message.serializedSize(version))
+ .check();
+ check(inbound) .received ( 90, 90 * message.serializedSize(version))
+ .processed( 90, 90 * message.serializedSize(version))
+ .pending ( 0, 0)
+ .expired ( 0, 0)
+ .error ( 0, 0)
+ .check();
+ });
+ }
+
+ @Test
+ public void testTimeout() throws Throwable
+ {
+ test((inbound, outbound, endpoint) -> {
+ int version = outbound.settings().acceptVersions.max;
+ int count = 10;
+ CountDownLatch enqueueDone = new CountDownLatch(1);
+ CountDownLatch deliveryDone = new CountDownLatch(1);
+ AtomicInteger delivered = new AtomicInteger();
+ Verb._TEST_1.unsafeSetHandler(() -> msg -> delivered.incrementAndGet());
+ Message<?> message = Message.builder(Verb._TEST_1, noPayload)
+ .withExpiresAt(approxTime.now() + TimeUnit.DAYS.toNanos(1L))
+ .build();
+ long sentSize = message.serializedSize(version);
+ outbound.enqueue(message);
+ long timeoutMillis = 10L;
+ while (delivered.get() < 1);
+ outbound.unsafeRunOnDelivery(() -> Uninterruptibles.awaitUninterruptibly(enqueueDone, 1L, TimeUnit.DAYS));
+ message = Message.builder(Verb._TEST_1, noPayload)
+ .withExpiresAt(approxTime.now() + TimeUnit.MILLISECONDS.toNanos(timeoutMillis))
+ .build();
+ for (int i = 0 ; i < count ; ++i)
+ outbound.enqueue(message);
+ Uninterruptibles.sleepUninterruptibly(timeoutMillis * 2, TimeUnit.MILLISECONDS);
+ enqueueDone.countDown();
+ outbound.unsafeRunOnDelivery(deliveryDone::countDown);
+ Assert.assertTrue(deliveryDone.await(1, MINUTES));
+ Assert.assertEquals(1, delivered.get());
+ check(outbound).submitted( 11)
+ .sent ( 1, sentSize)
+ .pending ( 0, 0)
+ .overload ( 0, 0)
+ .expired ( 10, 10 * message.serializedSize(current_version))
+ .error ( 0, 0)
+ .check();
+ check(inbound) .received ( 1, sentSize)
+ .processed( 1, sentSize)
+ .pending ( 0, 0)
+ .expired ( 0, 0)
+ .error ( 0, 0)
+ .check();
+ });
+ }
+
+ @Test
+ public void testPre40() throws Throwable
+ {
+ MessagingService.instance().versions.set(FBUtilities.getBroadcastAddressAndPort(),
+ MessagingService.VERSION_30);
+
+ try
+ {
+ test((inbound, outbound, endpoint) -> {
+ CountDownLatch done = new CountDownLatch(1);
+ unsafeSetHandler(Verb._TEST_1,
+ () -> (msg) -> done.countDown());
+
+ Message<?> message = Message.out(Verb._TEST_1, noPayload);
+ outbound.enqueue(message);
+ Assert.assertTrue(done.await(1, MINUTES));
+ Assert.assertTrue(outbound.isConnected());
+ });
+ }
+ finally
+ {
+ MessagingService.instance().versions.set(FBUtilities.getBroadcastAddressAndPort(),
+ current_version);
+ }
+ }
+
+ @Test
+ public void testPendingOutboundConnectionUpdatesMessageVersionOnReconnectAttempt() throws Throwable
+ {
+ final String storagePortProperty = Config.PROPERTY_PREFIX + "ssl_storage_port";
+ final String originalStoragePort = System.getProperty(storagePortProperty);
+ try
+ {
+ // Set up an inbound connection listening *only* on the SSL storage port to
+ // replicate a 3.x node. Force the messaging version to be incorrectly set to 4.0
+ // before the outbound connection attempt.
+ final Settings settings = Settings.LARGE;
+ final InetAddressAndPort endpoint = FBUtilities.getBroadcastAddressAndPort();
+
+ MessagingService.instance().versions.set(FBUtilities.getBroadcastAddressAndPort(),
+ MessagingService.VERSION_40);
+
+ System.setProperty(storagePortProperty, "7011");
+ final InetAddressAndPort legacySSLAddrsAndPort = endpoint.withPort(DatabaseDescriptor.getSSLStoragePort());
+ InboundConnectionSettings inboundSettings = settings.inbound.apply(new InboundConnectionSettings().withEncryption(encryptionOptions))
+ .withBindAddress(legacySSLAddrsAndPort)
+ .withAcceptMessaging(new AcceptVersions(VERSION_30, VERSION_3014))
+ .withSocketFactory(factory);
+ InboundSockets inbound = new InboundSockets(Collections.singletonList(inboundSettings));
+ OutboundConnectionSettings outboundTemplate = settings.outbound.apply(new OutboundConnectionSettings(endpoint).withEncryption(encryptionOptions))
+ .withDefaultReserveLimits()
+ .withSocketFactory(factory)
+ .withDefaults(ConnectionCategory.MESSAGING);
+ ResourceLimits.EndpointAndGlobal reserveCapacityInBytes = new ResourceLimits.EndpointAndGlobal(new ResourceLimits.Concurrent(outboundTemplate.applicationSendQueueReserveEndpointCapacityInBytes), outboundTemplate.applicationSendQueueReserveGlobalCapacityInBytes);
+ OutboundConnection outbound = new OutboundConnection(settings.type, outboundTemplate, reserveCapacityInBytes);
+ try
+ {
+ logger.info("Running {} {} -> {}", outbound.messagingVersion(), outbound.settings(), inboundSettings);
+ inbound.open().sync();
+
+ CountDownLatch done = new CountDownLatch(1);
+ unsafeSetHandler(Verb._TEST_1,
+ () -> (msg) -> done.countDown());
+
+ // Enqueuing outbound message will initiate an outbound
+ // connection with pending data in the pipeline
+ Message<?> message = Message.out(Verb._TEST_1, noPayload);
+ outbound.enqueue(message);
+
+ // Wait until the first connection attempt has taken place
+ // before updating the endpoint messaging version so that the
+ // connection takes place to a 4.0 node.
+ int attempts = 0;
+ final long waitForAttemptMillis = TimeUnit.SECONDS.toMillis(15);
+ while (outbound.connectionAttempts() == 0 && attempts < waitForAttemptMillis / 10)
+ {
+ Uninterruptibles.sleepUninterruptibly(10, TimeUnit.MILLISECONDS);
+ attempts++;
+ }
+
+ // Now that the connection is being attempted, set the endpoint version so
+ // that on the reconnect attempt the messaging version is rechecked and the
+ // legacy ssl logic picks the storage port instead. This should trigger a
+ // TRACE level log message "Endpoint version changed from 12 to 10 since
+ // connection initialized, updating."
+ outbound.settings().endpointToVersion.set(endpoint, VERSION_30);
+
+ // The connection should have successfully connected and delivered the _TEST_1
+ // message within the timout.
+ Assert.assertTrue(done.await(15, SECONDS));
+ Assert.assertTrue(outbound.isConnected());
+ Assert.assertTrue(String.format("expect less successful connections (%d) than attempts (%d)",
+ outbound.successfulConnections(), outbound.connectionAttempts()),
+ outbound.successfulConnections() < outbound.connectionAttempts());
+
+ }
+ finally
+ {
+ outbound.close(false);
+ inbound.close().get(30L, SECONDS);
+ outbound.close(false).get(30L, SECONDS);
+ resetVerbs();
+ MessagingService.instance().messageHandlers.clear();
+ }
+ }
+ finally
+ {
+ MessagingService.instance().versions.set(FBUtilities.getBroadcastAddressAndPort(),
+ current_version);
+ if (originalStoragePort != null)
+ System.setProperty(storagePortProperty, originalStoragePort);
+ else
+ System.clearProperty(storagePortProperty);
+ }
+ }
+
+ @Test
+ public void testCloseIfEndpointDown() throws Throwable
+ {
+ testManual((settings, inbound, outbound, endpoint) -> {
+ Message<?> message = Message.builder(Verb._TEST_1, noPayload)
+ .withExpiresAt(System.nanoTime() + SECONDS.toNanos(30L))
+ .build();
+
+ for (int i = 0 ; i < 1000 ; ++i)
+ outbound.enqueue(message);
+
+ outbound.close(true).get(10L, MINUTES);
+ });
+ }
+
+ @Test
+ public void testMessagePurging() throws Throwable
+ {
+ testManual((settings, inbound, outbound, endpoint) -> {
+ Runnable testWhileDisconnected = () -> {
+ try
+ {
+ for (int i = 0; i < 5; i++)
+ {
+ Message<?> message = Message.builder(Verb._TEST_1, noPayload)
+ .withExpiresAt(System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(50L))
+ .build();
+ outbound.enqueue(message);
+ Assert.assertFalse(outbound.isConnected());
+ Assert.assertEquals(1, outbound.pendingCount());
+ CompletableFuture.runAsync(() -> {
+ while (outbound.pendingCount() > 0 && !Thread.interrupted()) {}
+ }).get(10, SECONDS);
+ // Message should have been purged
+ Assert.assertEquals(0, outbound.pendingCount());
+ }
+ }
+ catch (Throwable t)
+ {
+ throw new RuntimeException(t);
+ }
+ };
+
+ testWhileDisconnected.run();
+
+ try
+ {
+ inbound.open().sync();
+ CountDownLatch deliveryDone = new CountDownLatch(1);
+
+ unsafeSetHandler(Verb._TEST_1, () -> msg -> {
+ outbound.unsafeRunOnDelivery(deliveryDone::countDown);
+ });
+ outbound.enqueue(Message.out(Verb._TEST_1, noPayload));
+ Assert.assertEquals(1, outbound.pendingCount());
+
+ Assert.assertTrue(deliveryDone.await(10, SECONDS));
+ Assert.assertEquals(0, deliveryDone.getCount());
+ Assert.assertEquals(0, outbound.pendingCount());
+ }
+ finally
+ {
+ inbound.close().get(10, SECONDS);
+ // Wait until disconnected
+ CompletableFuture.runAsync(() -> {
+ while (outbound.isConnected() && !Thread.interrupted()) {}
+ }).get(10, SECONDS);
+ }
+
+ testWhileDisconnected.run();
+ });
+ }
+
+ @Test
+ public void testMessageDeliveryOnReconnect() throws Throwable
+ {
+ testManual((settings, inbound, outbound, endpoint) -> {
+ try
+ {
+ inbound.open().sync();
+ CountDownLatch done = new CountDownLatch(1);
+ unsafeSetHandler(Verb._TEST_1, () -> msg -> done.countDown());
+ outbound.enqueue(Message.out(Verb._TEST_1, noPayload));
+ Assert.assertTrue(done.await(10, SECONDS));
+ Assert.assertEquals(done.getCount(), 0);
+
+ // Simulate disconnect
+ inbound.close().get(10, SECONDS);
+ MessagingService.instance().removeInbound(endpoint);
+ inbound = new InboundSockets(settings.inbound.apply(new InboundConnectionSettings()));
+ inbound.open().sync();
+
+ CountDownLatch latch2 = new CountDownLatch(1);
+ unsafeSetHandler(Verb._TEST_1, () -> msg -> latch2.countDown());
+ outbound.enqueue(Message.out(Verb._TEST_1, noPayload));
+
+ latch2.await(10, SECONDS);
+ Assert.assertEquals(latch2.getCount(), 0);
+ }
+ finally
+ {
+ inbound.close().get(10, SECONDS);
+ outbound.close(false).get(10, SECONDS);
+ }
+ });
+ }
+
+ @Test
+ public void testRecoverableCorruptedMessageDelivery() throws Throwable
+ {
+ test((inbound, outbound, endpoint) -> {
+ int version = outbound.settings().acceptVersions.max;
+ if (version < VERSION_40)
+ return;
+
+ AtomicInteger counter = new AtomicInteger();
+ unsafeSetSerializer(Verb._TEST_1, () -> new IVersionedSerializer<Object>()
+ {
+ public void serialize(Object o, DataOutputPlus out, int version) throws IOException
+ {
+ out.writeInt((Integer) o);
+ }
+
+ public Object deserialize(DataInputPlus in, int version) throws IOException
+ {
+ if (counter.getAndIncrement() == 3)
+ throw new UnknownColumnException("");
+
+ return in.readInt();
+ }
+
+ public long serializedSize(Object o, int version)
+ {
+ return Integer.BYTES;
+ }
+ });
+
+ // Connect
+ connect(outbound);
+
+ CountDownLatch latch = new CountDownLatch(4);
+ unsafeSetHandler(Verb._TEST_1, () -> message -> latch.countDown());
+ for (int i = 0; i < 5; i++)
+ outbound.enqueue(Message.out(Verb._TEST_1, 0xffffffff));
+
+ latch.await(10, SECONDS);
+ Assert.assertEquals(0, latch.getCount());
+ Assert.assertEquals(6, counter.get());
+ });
+ }
+
+ @Test
+ public void testCRCCorruption() throws Throwable
+ {
+ test((inbound, outbound, endpoint) -> {
+ int version = outbound.settings().acceptVersions.max;
+ if (version < VERSION_40)
+ return;
+
+ unsafeSetSerializer(Verb._TEST_1, () -> new IVersionedSerializer<Object>()
+ {
+ public void serialize(Object o, DataOutputPlus out, int version) throws IOException
+ {
+ out.writeInt((Integer) o);
+ }
+
+ public Object deserialize(DataInputPlus in, int version) throws IOException
+ {
+ return in.readInt();
+ }
+
+ public long serializedSize(Object o, int version)
+ {
+ return Integer.BYTES;
+ }
+ });
+
+ connect(outbound);
+
+ outbound.unsafeGetChannel().pipeline().addFirst(new ChannelOutboundHandlerAdapter() {
+ public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise promise) throws Exception {
+ ByteBuf bb = (ByteBuf) msg;
+ bb.setByte(0, 0xAB);
+ ctx.write(msg, promise);
+ }
+ });
+ outbound.enqueue(Message.out(Verb._TEST_1, 0xffffffff));
+ CompletableFuture.runAsync(() -> {
+ while (outbound.isConnected() && !Thread.interrupted()) {}
+ }).get(10, SECONDS);
+ Assert.assertFalse(outbound.isConnected());
+ // TODO: count corruptions
+
+ connect(outbound);
+ });
+ }
+
+ @Test
+ public void testAcquireReleaseOutbound() throws Throwable
+ {
+ // In each test round, K capacity is reserved upfront.
+ // Two groups of threads each release/acquire for K capacity in total accordingly,
+ // i.e. if only the release threads run, at the end, the reserved capacity is 0 (K - K).
+ // During the test, we expect N (N <= maxFailures) acquire attempts (for M capacity) to fail.
+ // The reserved capacity (pendingBytes) at the end of the round should equal to K - N * M,
+ // which you can find in the assertion.
+ test((inbound, outbound, endpoint) -> {
+ // max capacity equals to permit-free sendQueueCapcity + the minimun of endpoint and global reserve
+ double maxSendQueueCapacity = outbound.settings().applicationSendQueueCapacityInBytes +
+ Double.min(outbound.settings().applicationSendQueueReserveEndpointCapacityInBytes,
+ outbound.settings().applicationSendQueueReserveGlobalCapacityInBytes.limit());
+ int concurrency = 100;
+ int attempts = 10000;
+ int acquireCount = concurrency * attempts;
+ long acquireStep = Math.round(maxSendQueueCapacity * 1.2 / acquireCount / 2); // It is guranteed to acquire (~20%) more
+ // The total overly acquired amount divides the amount acquired in each step. Get the ceil value so not to miss the acquire that just exceeds.
+ long maxFailures = (long) Math.ceil((acquireCount * acquireStep * 2 - maxSendQueueCapacity) / acquireStep); // The result must be in the range of lone
+ AtomicLong acquisitionFailures = new AtomicLong();
+ Runnable acquirer = () -> {
+ for (int j = 0; j < attempts; j++)
+ {
+ if (!outbound.unsafeAcquireCapacity(acquireStep))
+ acquisitionFailures.incrementAndGet();
+ }
+ };
+ Runnable releaser = () -> {
+ for (int j = 0; j < attempts; j++)
+ outbound.unsafeReleaseCapacity(acquireStep);
+ };
+
+ // Start N acquirer and releaser to contend for capcaity
+ List<Runnable> submitOrder = new ArrayList<>(concurrency * 2);
+ for (int i = 0 ; i < concurrency ; ++i)
+ submitOrder.add(acquirer);
+ for (int i = 0 ; i < concurrency ; ++i)
+ submitOrder.add(releaser);
+ // randomize their start order
+ randomize(submitOrder);
+
+ try
+ {
+ // Reserve enough capacity upfront to ensure the releaser threads cannot release all reserved capacity.
+ // i.e. the pendingBytes is always positive during the test.
+ Assert.assertTrue("Unable to reserve enough capacity",
+ outbound.unsafeAcquireCapacity(acquireCount, acquireCount * acquireStep));
+ ExecutorService executor = Executors.newFixedThreadPool(concurrency);
+
+ submitOrder.forEach(executor::submit);
+
+ executor.shutdown();
+ Assert.assertTrue(executor.awaitTermination(1, TimeUnit.MINUTES));
+
+ Assert.assertEquals(acquireCount * acquireStep - (acquisitionFailures.get() * acquireStep), outbound.pendingBytes());
+ Assert.assertEquals(acquireCount - acquisitionFailures.get(), outbound.pendingCount());
+ Assert.assertTrue(String.format("acquisitionFailures should be capped by maxFailure. acquisitionFailures: %d, acquisitionFailures: %d",
+ maxFailures, acquisitionFailures.get()),
+ acquisitionFailures.get() <= maxFailures);
+ }
+ finally
+ { // release the acquired capacity from this round
+ outbound.unsafeReleaseCapacity(outbound.pendingCount(), outbound.pendingBytes());
+ }
+ });
+ }
+
+ private static <V> void randomize(List<V> list)
+ {
+ long seed = ThreadLocalRandom.current().nextLong();
+ logger.info("Seed used for randomize: " + seed);
+ Random random = new Random(seed);
+ switch (random.nextInt(3))
+ {
+ case 0:
+ Collections.shuffle(list, random);
+ break;
+ case 1:
+ Collections.reverse(list);
+ break;
+ case 2:
+ // leave as is
+ }
+ }
+
+ private void connect(OutboundConnection outbound) throws Throwable
+ {
+ CountDownLatch latch = new CountDownLatch(1);
+ unsafeSetHandler(Verb._TEST_1, () -> message -> latch.countDown());
+ outbound.enqueue(Message.out(Verb._TEST_1, 0xffffffff));
+ latch.await(10, SECONDS);
+ Assert.assertEquals(0, latch.getCount());
+ Assert.assertTrue(outbound.isConnected());
+ }
+
+}
diff --git a/test/unit/org/apache/cassandra/net/ConnectionUtils.java b/test/unit/org/apache/cassandra/net/ConnectionUtils.java
new file mode 100644
index 0000000..5aff390
--- /dev/null
+++ b/test/unit/org/apache/cassandra/net/ConnectionUtils.java
@@ -0,0 +1,246 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.net;
+
+import java.util.Objects;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Supplier;
+
+import static org.apache.cassandra.Util.spinAssertEquals;
+
+public class ConnectionUtils
+{
+ public interface FailCheck
+ {
+ public void accept(String message, Long expected, Supplier<Long> actualSupplier);
+ }
+
+ public static class OutboundCountChecker
+ {
+ private final OutboundConnection connection;
+ private long submitted;
+ private long pending, pendingBytes;
+ private long sent, sentBytes;
+ private long overload, overloadBytes;
+ private long expired, expiredBytes;
+ private long error, errorBytes;
+ private boolean checkSubmitted, checkPending, checkSent, checkOverload, checkExpired, checkError;
+
+ private OutboundCountChecker(OutboundConnection connection)
+ {
+ this.connection = connection;
+ }
+
+ public OutboundCountChecker submitted(long count)
+ {
+ submitted = count;
+ checkSubmitted = true;
+ return this;
+ }
+
+ public OutboundCountChecker pending(long count, long bytes)
+ {
+ pending = count;
+ pendingBytes = bytes;
+ checkPending = true;
+ return this;
+ }
+
+ public OutboundCountChecker sent(long count, long bytes)
+ {
+ sent = count;
+ sentBytes = bytes;
+ checkSent = true;
+ return this;
+ }
+
+ public OutboundCountChecker overload(long count, long bytes)
+ {
+ overload = count;
+ overloadBytes = bytes;
+ checkOverload = true;
+ return this;
+ }
+
+ public OutboundCountChecker expired(long count, long bytes)
+ {
+ expired = count;
+ expiredBytes = bytes;
+ checkExpired = true;
+ return this;
+ }
+
+ public OutboundCountChecker error(long count, long bytes)
+ {
+ error = count;
+ errorBytes = bytes;
+ checkError = true;
+ return this;
+ }
+
+ public void check()
+ {
+ doCheck((message, expected, actual) -> spinAssertEquals(message, expected, actual, 5, TimeUnit.SECONDS));
+ }
+
+ public void check(FailCheck failCheck)
+ {
+ doCheck((message, expect, actual) -> { if (!Objects.equals(expect, actual.get())) failCheck.accept(message, expect, actual); });
+ }
+
+ private void doCheck(FailCheck testAndFailCheck)
+ {
+ if (checkSubmitted)
+ {
+ testAndFailCheck.accept("submitted count values don't match", submitted, connection::submittedCount);
+ }
+ if (checkPending)
+ {
+ testAndFailCheck.accept("pending count values don't match", pending, () -> (long) connection.pendingCount());
+ testAndFailCheck.accept("pending bytes values don't match", pendingBytes, connection::pendingBytes);
+ }
+ if (checkSent)
+ {
+ testAndFailCheck.accept("sent count values don't match", sent, connection::sentCount);
+ testAndFailCheck.accept("sent bytes values don't match", sentBytes, connection::sentBytes);
+ }
+ if (checkOverload)
+ {
+ testAndFailCheck.accept("overload count values don't match", overload, connection::overloadedCount);
+ testAndFailCheck.accept("overload bytes values don't match", overloadBytes, connection::overloadedBytes);
+ }
+ if (checkExpired)
+ {
+ testAndFailCheck.accept("expired count values don't match", expired, connection::expiredCount);
+ testAndFailCheck.accept("expired bytes values don't match", expiredBytes, connection::expiredBytes);
+ }
+ if (checkError)
+ {
+ testAndFailCheck.accept("error count values don't match", error, connection::errorCount);
+ testAndFailCheck.accept("error bytes values don't match", errorBytes, connection::errorBytes);
+ }
+ }
+ }
+
+ public static class InboundCountChecker
+ {
+ private final InboundMessageHandlers connection;
+ private long scheduled, scheduledBytes;
+ private long received, receivedBytes;
+ private long processed, processedBytes;
+ private long expired, expiredBytes;
+ private long error, errorBytes;
+ private boolean checkScheduled, checkReceived, checkProcessed, checkExpired, checkError;
+
+ private InboundCountChecker(InboundMessageHandlers connection)
+ {
+ this.connection = connection;
+ }
+
+ public InboundCountChecker pending(long count, long bytes)
+ {
+ scheduled = count;
+ scheduledBytes = bytes;
+ checkScheduled = true;
+ return this;
+ }
+
+ public InboundCountChecker received(long count, long bytes)
+ {
+ received = count;
+ receivedBytes = bytes;
+ checkReceived = true;
+ return this;
+ }
+
+ public InboundCountChecker processed(long count, long bytes)
+ {
+ processed = count;
+ processedBytes = bytes;
+ checkProcessed = true;
+ return this;
+ }
+
+ public InboundCountChecker expired(long count, long bytes)
+ {
+ expired = count;
+ expiredBytes = bytes;
+ checkExpired = true;
+ return this;
+ }
+
+ public InboundCountChecker error(long count, long bytes)
+ {
+ error = count;
+ errorBytes = bytes;
+ checkError = true;
+ return this;
+ }
+
+ public void check()
+ {
+ doCheck((message, expected, actual) -> spinAssertEquals(message, expected, actual, 5, TimeUnit.SECONDS));
+ }
+
+ public void check(FailCheck failCheck)
+ {
+ doCheck((message, expect, actual) -> { if (!Objects.equals(expect, actual.get())) failCheck.accept(message, expect, actual); });
+ }
+
+ private void doCheck(FailCheck testAndFailCheck)
+ {
+ if (checkReceived)
+ {
+ testAndFailCheck.accept("received count values don't match", received, connection::receivedCount);
+ testAndFailCheck.accept("received bytes values don't match", receivedBytes, connection::receivedBytes);
+ }
+ if (checkProcessed)
+ {
+ testAndFailCheck.accept("processed count values don't match", processed, connection::processedCount);
+ testAndFailCheck.accept("processed bytes values don't match", processedBytes, connection::processedBytes);
+ }
+ if (checkExpired)
+ {
+ testAndFailCheck.accept("expired count values don't match", expired, connection::expiredCount);
+ testAndFailCheck.accept("expired bytes values don't match", expiredBytes, connection::expiredBytes);
+ }
+ if (checkError)
+ {
+ testAndFailCheck.accept("error count values don't match", error, connection::errorCount);
+ testAndFailCheck.accept("error bytes values don't match", errorBytes, connection::errorBytes);
+ }
+ if (checkScheduled)
+ {
+ testAndFailCheck.accept("scheduled count values don't match", scheduled, connection::scheduledCount);
+ testAndFailCheck.accept("scheduled bytes values don't match", scheduledBytes, connection::scheduledBytes);
+ }
+ }
+ }
+
+ public static OutboundCountChecker check(OutboundConnection outbound)
+ {
+ return new OutboundCountChecker(outbound);
+ }
+
+ public static InboundCountChecker check(InboundMessageHandlers inbound)
+ {
+ return new InboundCountChecker(inbound);
+ }
+
+}
diff --git a/test/unit/org/apache/cassandra/net/ForwardingInfoTest.java b/test/unit/org/apache/cassandra/net/ForwardingInfoTest.java
new file mode 100644
index 0000000..16dec9f
--- /dev/null
+++ b/test/unit/org/apache/cassandra/net/ForwardingInfoTest.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.net;
+
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.List;
+
+import com.google.common.collect.ImmutableList;
+import org.junit.Test;
+
+import org.apache.cassandra.io.util.DataInputBuffer;
+import org.apache.cassandra.io.util.DataOutputBuffer;
+import org.apache.cassandra.locator.InetAddressAndPort;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class ForwardingInfoTest
+{
+ @Test
+ public void testCurrent() throws Exception
+ {
+ testVersion(MessagingService.current_version);
+ }
+
+ @Test
+ public void test30() throws Exception
+ {
+ testVersion(MessagingService.VERSION_30);
+ }
+
+ private void testVersion(int version) throws Exception
+ {
+ InetAddressAndPort.initializeDefaultPort(65532);
+ List<InetAddressAndPort> addresses = ImmutableList.of(InetAddressAndPort.getByNameOverrideDefaults("127.0.0.1", 42),
+ InetAddressAndPort.getByName("127.0.0.1"),
+ InetAddressAndPort.getByName("127.0.0.1:7000"),
+ InetAddressAndPort.getByNameOverrideDefaults("2001:0db8:0000:0000:0000:ff00:0042:8329", 42),
+ InetAddressAndPort.getByName("2001:0db8:0000:0000:0000:ff00:0042:8329"),
+ InetAddressAndPort.getByName("[2001:0db8:0000:0000:0000:ff00:0042:8329]:7000"));
+
+ ForwardingInfo ftc = new ForwardingInfo(addresses, new long[] { 44, 45, 46, 47, 48, 49 });
+ ByteBuffer buffer;
+ try (DataOutputBuffer dob = new DataOutputBuffer())
+ {
+ ForwardingInfo.serializer.serialize(ftc, dob, version);
+ buffer = dob.buffer();
+ }
+
+ assertEquals(buffer.remaining(), ForwardingInfo.serializer.serializedSize(ftc, version));
+
+ ForwardingInfo deserialized;
+ try (DataInputBuffer dib = new DataInputBuffer(buffer, false))
+ {
+ deserialized = ForwardingInfo.serializer.deserialize(dib, version);
+ }
+
+ assertTrue(Arrays.equals(ftc.messageIds, deserialized.messageIds));
+
+ Iterator<InetAddressAndPort> iterator = deserialized.targets.iterator();
+ if (version >= MessagingService.VERSION_40)
+ {
+ for (int ii = 0; ii < addresses.size(); ii++)
+ {
+ InetAddressAndPort original = addresses.get(ii);
+ InetAddressAndPort roundtripped = iterator.next();
+ assertEquals(original, roundtripped);
+ }
+ }
+ else
+ {
+ for (int ii = 0; ii < addresses.size(); ii++)
+ {
+ InetAddressAndPort original = addresses.get(ii);
+ InetAddressAndPort roundtripped = iterator.next();
+ assertEquals(original.address, roundtripped.address);
+ //3.0 can't send port numbers so you get the defaults
+ assertEquals(65532, roundtripped.port);
+ }
+ }
+ }
+}
diff --git a/test/unit/org/apache/cassandra/net/FramingTest.java b/test/unit/org/apache/cassandra/net/FramingTest.java
new file mode 100644
index 0000000..8a7f428
--- /dev/null
+++ b/test/unit/org/apache/cassandra/net/FramingTest.java
@@ -0,0 +1,434 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.net;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.security.SecureRandom;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Random;
+
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import io.netty.buffer.ByteBuf;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.io.IVersionedSerializer;
+import org.apache.cassandra.io.compress.BufferType;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputBuffer;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.utils.memory.BufferPool;
+import org.apache.cassandra.utils.vint.VIntCoding;
+
+import static java.lang.Math.*;
+import static org.apache.cassandra.net.MessagingService.VERSION_30;
+import static org.apache.cassandra.net.MessagingService.VERSION_3014;
+import static org.apache.cassandra.net.MessagingService.current_version;
+import static org.apache.cassandra.net.MessagingService.minimum_version;
+import static org.apache.cassandra.net.OutboundConnections.LARGE_MESSAGE_THRESHOLD;
+import static org.apache.cassandra.net.ShareableBytes.wrap;
+
+// TODO: test corruption
+// TODO: use a different random seed each time
+// TODO: use quick theories
+public class FramingTest
+{
+ private static final Logger logger = LoggerFactory.getLogger(FramingTest.class);
+
+ @BeforeClass
+ public static void begin() throws NoSuchFieldException, IllegalAccessException
+ {
+ DatabaseDescriptor.daemonInitialization();
+ Verb._TEST_1.unsafeSetSerializer(() -> new IVersionedSerializer<byte[]>()
+ {
+
+ public void serialize(byte[] t, DataOutputPlus out, int version) throws IOException
+ {
+ out.writeUnsignedVInt(t.length);
+ out.write(t);
+ }
+
+ public byte[] deserialize(DataInputPlus in, int version) throws IOException
+ {
+ byte[] r = new byte[(int) in.readUnsignedVInt()];
+ in.readFully(r);
+ return r;
+ }
+
+ public long serializedSize(byte[] t, int version)
+ {
+ return VIntCoding.computeUnsignedVIntSize(t.length) + t.length;
+ }
+ });
+ }
+
+ @AfterClass
+ public static void after() throws NoSuchFieldException, IllegalAccessException
+ {
+ Verb._TEST_1.unsafeSetSerializer(() -> null);
+ }
+
+ private static class SequenceOfFrames
+ {
+ final List<byte[]> original;
+ final int[] boundaries;
+ final ShareableBytes frames;
+
+ private SequenceOfFrames(List<byte[]> original, int[] boundaries, ByteBuffer frames)
+ {
+ this.original = original;
+ this.boundaries = boundaries;
+ this.frames = wrap(frames);
+ }
+ }
+
+ @Test
+ public void testRandomLZ4()
+ {
+ testSomeFrames(FrameEncoderLZ4.fastInstance, FrameDecoderLZ4.fast(GlobalBufferPoolAllocator.instance));
+ }
+
+ @Test
+ public void testRandomCrc()
+ {
+ testSomeFrames(FrameEncoderCrc.instance, FrameDecoderCrc.create(GlobalBufferPoolAllocator.instance));
+ }
+
+ private void testSomeFrames(FrameEncoder encoder, FrameDecoder decoder)
+ {
+ long seed = new SecureRandom().nextLong();
+ logger.info("seed: {}, decoder: {}", seed, decoder.getClass().getSimpleName());
+ Random random = new Random(seed);
+ for (int i = 0 ; i < 1000 ; ++i)
+ testRandomSequenceOfFrames(random, encoder, decoder);
+ }
+
+ private void testRandomSequenceOfFrames(Random random, FrameEncoder encoder, FrameDecoder decoder)
+ {
+ SequenceOfFrames sequenceOfFrames = sequenceOfFrames(random, encoder);
+
+ List<byte[]> uncompressed = sequenceOfFrames.original;
+ ShareableBytes frames = sequenceOfFrames.frames;
+ int[] boundaries = sequenceOfFrames.boundaries;
+
+ int end = frames.get().limit();
+ List<FrameDecoder.Frame> out = new ArrayList<>();
+ int prevBoundary = -1;
+ for (int i = 0 ; i < end ; )
+ {
+ int limit = i + random.nextInt(1 + end - i);
+ decoder.decode(out, frames.slice(i, limit));
+ int boundary = Arrays.binarySearch(boundaries, limit);
+ if (boundary < 0) boundary = -2 -boundary;
+
+ while (prevBoundary < boundary)
+ {
+ ++prevBoundary;
+ Assert.assertTrue(out.size() >= 1 + prevBoundary);
+ verify(uncompressed.get(prevBoundary), ((FrameDecoder.IntactFrame) out.get(prevBoundary)).contents);
+ }
+ i = limit;
+ }
+ for (FrameDecoder.Frame frame : out)
+ frame.release();
+ frames.release();
+ Assert.assertNull(decoder.stash);
+ Assert.assertTrue(decoder.frames.isEmpty());
+ }
+
+ private static void verify(byte[] expect, ShareableBytes actual)
+ {
+ verify(expect, 0, expect.length, actual);
+ }
+
+ private static void verify(byte[] expect, int start, int end, ShareableBytes actual)
+ {
+ byte[] fetch = new byte[end - start];
+ Assert.assertEquals(end - start, actual.remaining());
+ actual.get().get(fetch);
+ boolean equals = true;
+ for (int i = start ; equals && i < end ; ++i)
+ equals = expect[i] == fetch[i - start];
+ if (!equals)
+ Assert.assertArrayEquals(Arrays.copyOfRange(expect, start, end), fetch);
+ }
+
+ private static SequenceOfFrames sequenceOfFrames(Random random, FrameEncoder encoder)
+ {
+ int frameCount = 1 + random.nextInt(8);
+ List<byte[]> uncompressed = new ArrayList<>();
+ List<ByteBuf> compressed = new ArrayList<>();
+ int[] cumulativeCompressedLength = new int[frameCount];
+ for (int i = 0 ; i < frameCount ; ++i)
+ {
+ byte[] bytes = randomishBytes(random, 1, 1 << 15);
+ uncompressed.add(bytes);
+
+ FrameEncoder.Payload payload = encoder.allocator().allocate(true, bytes.length);
+ payload.buffer.put(bytes);
+ payload.finish();
+
+ ByteBuf buffer = encoder.encode(true, payload.buffer);
+ compressed.add(buffer);
+ cumulativeCompressedLength[i] = (i == 0 ? 0 : cumulativeCompressedLength[i - 1]) + buffer.readableBytes();
+ }
+
+ ByteBuffer frames = BufferPool.getAtLeast(cumulativeCompressedLength[frameCount - 1], BufferType.OFF_HEAP);
+ for (ByteBuf buffer : compressed)
+ {
+ frames.put(buffer.internalNioBuffer(buffer.readerIndex(), buffer.readableBytes()));
+ buffer.release();
+ }
+ frames.flip();
+ return new SequenceOfFrames(uncompressed, cumulativeCompressedLength, frames);
+ }
+
+ @Test
+ public void burnRandomLegacy()
+ {
+ burnRandomLegacy(1000);
+ }
+
+ private void burnRandomLegacy(int count)
+ {
+ SecureRandom seed = new SecureRandom();
+ Random random = new Random();
+ for (int i = 0 ; i < count ; ++i)
+ {
+ long innerSeed = seed.nextLong();
+ float ratio = seed.nextFloat();
+ int version = minimum_version + random.nextInt(1 + current_version - minimum_version);
+ logger.debug("seed: {}, ratio: {}, version: {}", innerSeed, ratio, version);
+ random.setSeed(innerSeed);
+ testRandomSequenceOfMessages(random, ratio, version, new FrameDecoderLegacy(GlobalBufferPoolAllocator.instance, version));
+ }
+ }
+
+ @Test
+ public void testRandomLegacy()
+ {
+ testRandomLegacy(250);
+ }
+
+ private void testRandomLegacy(int count)
+ {
+ SecureRandom seeds = new SecureRandom();
+ for (int messagingVersion : new int[] { VERSION_30, VERSION_3014, current_version})
+ {
+ FrameDecoder decoder = new FrameDecoderLegacy(GlobalBufferPoolAllocator.instance, messagingVersion);
+ testSomeMessages(seeds.nextLong(), count, 0.0f, messagingVersion, decoder);
+ testSomeMessages(seeds.nextLong(), count, 0.1f, messagingVersion, decoder);
+ testSomeMessages(seeds.nextLong(), count, 0.95f, messagingVersion, decoder);
+ testSomeMessages(seeds.nextLong(), count, 1.0f, messagingVersion, decoder);
+ }
+ }
+
+ private void testSomeMessages(long seed, int count, float largeRatio, int messagingVersion, FrameDecoder decoder)
+ {
+ logger.info("seed: {}, iterations: {}, largeRatio: {}, messagingVersion: {}, decoder: {}", seed, count, largeRatio, messagingVersion, decoder.getClass().getSimpleName());
+ Random random = new Random(seed);
+ for (int i = 0 ; i < count ; ++i)
+ {
+ long innerSeed = random.nextLong();
+ logger.debug("inner seed: {}, iteration: {}", innerSeed, i);
+ random.setSeed(innerSeed);
+ testRandomSequenceOfMessages(random, largeRatio, messagingVersion, decoder);
+ }
+ }
+
+ private void testRandomSequenceOfMessages(Random random, float largeRatio, int messagingVersion, FrameDecoder decoder)
+ {
+ SequenceOfFrames sequenceOfMessages = sequenceOfMessages(random, largeRatio, messagingVersion);
+
+ List<byte[]> messages = sequenceOfMessages.original;
+ ShareableBytes stream = sequenceOfMessages.frames;
+
+ int end = stream.get().limit();
+ List<FrameDecoder.Frame> out = new ArrayList<>();
+
+ int messageStart = 0;
+ int messageIndex = 0;
+ for (int i = 0 ; i < end ; )
+ {
+ int limit = i + random.nextInt(1 + end - i);
+ decoder.decode(out, stream.slice(i, limit));
+
+ int outIndex = 0;
+ byte[] message = messages.get(messageIndex);
+ if (i > messageStart)
+ {
+ int start;
+ if (message.length <= LARGE_MESSAGE_THRESHOLD)
+ {
+ start = 0;
+ }
+ else if (!lengthIsReadable(message, i - messageStart, messagingVersion))
+ {
+ // we should have an initial frame containing only some prefix of the message (probably 64 bytes)
+ // that was stashed only to decide how big the message was
+ FrameDecoder.IntactFrame frame = (FrameDecoder.IntactFrame) out.get(outIndex++);
+ Assert.assertFalse(frame.isSelfContained);
+ start = frame.contents.remaining();
+ verify(message, 0, frame.contents.remaining(), frame.contents);
+ }
+ else
+ {
+ start = i - messageStart;
+ }
+
+ if (limit >= message.length + messageStart)
+ {
+ FrameDecoder.IntactFrame frame = (FrameDecoder.IntactFrame) out.get(outIndex++);
+ Assert.assertEquals(start == 0, frame.isSelfContained);
+ // verify remainder of a large message, or a single fully stashed small message
+ verify(message, start, message.length, frame.contents);
+
+ messageStart += message.length;
+ if (++messageIndex < messages.size())
+ message = messages.get(messageIndex);
+ }
+ else if (message.length > LARGE_MESSAGE_THRESHOLD)
+ {
+ FrameDecoder.IntactFrame frame = (FrameDecoder.IntactFrame) out.get(outIndex++);
+ Assert.assertFalse(frame.isSelfContained);
+ // verify next portion of a large message
+ verify(message, start, limit - messageStart, frame.contents);
+
+ Assert.assertEquals(outIndex, out.size());
+ for (FrameDecoder.Frame f : out)
+ f.release();
+ out.clear();
+ i = limit;
+ continue;
+ }
+ }
+
+ // message is fresh
+ int beginFrameIndex = messageIndex;
+ while (messageStart + message.length <= limit)
+ {
+ messageStart += message.length;
+ if (++messageIndex < messages.size())
+ message = messages.get(messageIndex);
+ }
+
+ if (beginFrameIndex < messageIndex)
+ {
+ FrameDecoder.IntactFrame frame = (FrameDecoder.IntactFrame) out.get(outIndex++);
+ Assert.assertTrue(frame.isSelfContained);
+ while (beginFrameIndex < messageIndex)
+ {
+ byte[] m = messages.get(beginFrameIndex);
+ ShareableBytes bytesToVerify = frame.contents.sliceAndConsume(m.length);
+ verify(m, bytesToVerify);
+ bytesToVerify.release();
+ ++beginFrameIndex;
+ }
+ Assert.assertFalse(frame.contents.hasRemaining());
+ }
+
+ if (limit > messageStart
+ && message.length > LARGE_MESSAGE_THRESHOLD
+ && lengthIsReadable(message, limit - messageStart, messagingVersion))
+ {
+ FrameDecoder.IntactFrame frame = (FrameDecoder.IntactFrame) out.get(outIndex++);
+ Assert.assertFalse(frame.isSelfContained);
+ verify(message, 0, limit - messageStart, frame.contents);
+ }
+
+ Assert.assertEquals(outIndex, out.size());
+ for (FrameDecoder.Frame frame : out)
+ frame.release();
+ out.clear();
+
+ i = limit;
+ }
+ stream.release();
+ Assert.assertTrue(stream.isReleased());
+ Assert.assertNull(decoder.stash);
+ Assert.assertTrue(decoder.frames.isEmpty());
+ }
+
+ private static boolean lengthIsReadable(byte[] message, int limit, int messagingVersion)
+ {
+ try
+ {
+ return Message.serializer.inferMessageSize(ByteBuffer.wrap(message), 0, limit, messagingVersion) >= 0;
+ }
+ catch (Message.InvalidLegacyProtocolMagic e)
+ {
+ throw new IllegalStateException(e);
+ }
+ }
+
+ private static SequenceOfFrames sequenceOfMessages(Random random, float largeRatio, int messagingVersion)
+ {
+ int messageCount = 1 + random.nextInt(63);
+ List<byte[]> messages = new ArrayList<>();
+ int[] cumulativeLength = new int[messageCount];
+ for (int i = 0 ; i < messageCount ; ++i)
+ {
+ byte[] payload;
+ if (random.nextFloat() < largeRatio) payload = randomishBytes(random, 1 << 16, 1 << 17);
+ else payload = randomishBytes(random, 1, 1 << 16);
+ Message<byte[]> messageObj = Message.out(Verb._TEST_1, payload);
+
+ byte[] message;
+ try (DataOutputBuffer out = new DataOutputBuffer(messageObj.serializedSize(messagingVersion)))
+ {
+ Message.serializer.serialize(messageObj, out, messagingVersion);
+ message = out.toByteArray();
+ }
+ catch (IOException e)
+ {
+ throw new IllegalStateException(e);
+ }
+ messages.add(message);
+
+ cumulativeLength[i] = (i == 0 ? 0 : cumulativeLength[i - 1]) + message.length;
+ }
+
+ ByteBuffer frames = BufferPool.getAtLeast(cumulativeLength[messageCount - 1], BufferType.OFF_HEAP);
+ for (byte[] buffer : messages)
+ frames.put(buffer);
+ frames.flip();
+ return new SequenceOfFrames(messages, cumulativeLength, frames);
+ }
+
+ private static byte[] randomishBytes(Random random, int minLength, int maxLength)
+ {
+ byte[] bytes = new byte[minLength + random.nextInt(maxLength - minLength)];
+ int runLength = 1 + random.nextInt(255);
+ for (int i = 0 ; i < bytes.length ; i += runLength)
+ {
+ byte b = (byte) random.nextInt(256);
+ Arrays.fill(bytes, i, min(bytes.length, i + runLength), b);
+ }
+ return bytes;
+ }
+
+}
diff --git a/test/unit/org/apache/cassandra/net/HandshakeTest.java b/test/unit/org/apache/cassandra/net/HandshakeTest.java
new file mode 100644
index 0000000..e680b83
--- /dev/null
+++ b/test/unit/org/apache/cassandra/net/HandshakeTest.java
@@ -0,0 +1,221 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.net;
+
+import java.nio.channels.ClosedChannelException;
+import java.util.Objects;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import io.netty.channel.EventLoop;
+import io.netty.util.concurrent.Future;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.commitlog.CommitLog;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.net.OutboundConnectionInitiator.Result;
+import org.apache.cassandra.net.OutboundConnectionInitiator.Result.MessagingSuccess;
+
+import static org.apache.cassandra.net.MessagingService.VERSION_30;
+import static org.apache.cassandra.net.MessagingService.VERSION_3014;
+import static org.apache.cassandra.net.MessagingService.current_version;
+import static org.apache.cassandra.net.MessagingService.minimum_version;
+import static org.apache.cassandra.net.ConnectionType.SMALL_MESSAGES;
+import static org.apache.cassandra.net.OutboundConnectionInitiator.*;
+
+// TODO: test failure due to exception, timeout, etc
+public class HandshakeTest
+{
+ private static final SocketFactory factory = new SocketFactory();
+
+ @BeforeClass
+ public static void startup()
+ {
+ DatabaseDescriptor.daemonInitialization();
+ CommitLog.instance.start();
+ }
+
+ @AfterClass
+ public static void cleanup() throws InterruptedException
+ {
+ factory.shutdownNow();
+ }
+
+ private Result handshake(int req, int outMin, int outMax) throws ExecutionException, InterruptedException
+ {
+ return handshake(req, new AcceptVersions(outMin, outMax), null);
+ }
+ private Result handshake(int req, int outMin, int outMax, int inMin, int inMax) throws ExecutionException, InterruptedException
+ {
+ return handshake(req, new AcceptVersions(outMin, outMax), new AcceptVersions(inMin, inMax));
+ }
+ private Result handshake(int req, AcceptVersions acceptOutbound, AcceptVersions acceptInbound) throws ExecutionException, InterruptedException
+ {
+ InboundSockets inbound = new InboundSockets(new InboundConnectionSettings().withAcceptMessaging(acceptInbound));
+ try
+ {
+ inbound.open();
+ InetAddressAndPort endpoint = inbound.sockets().stream().map(s -> s.settings.bindAddress).findFirst().get();
+ EventLoop eventLoop = factory.defaultGroup().next();
+ Future<Result<MessagingSuccess>> future =
+ initiateMessaging(eventLoop,
+ SMALL_MESSAGES,
+ new OutboundConnectionSettings(endpoint)
+ .withAcceptVersions(acceptOutbound)
+ .withDefaults(ConnectionCategory.MESSAGING),
+ req, new AsyncPromise<>(eventLoop));
+ return future.get();
+ }
+ finally
+ {
+ inbound.close().await(1L, TimeUnit.SECONDS);
+ }
+ }
+
+ @Test
+ public void testBothCurrentVersion() throws InterruptedException, ExecutionException
+ {
+ Result result = handshake(current_version, minimum_version, current_version);
+ Assert.assertEquals(Result.Outcome.SUCCESS, result.outcome);
+ result.success().channel.close();
+ }
+
+ @Test
+ public void testSendCompatibleOldVersion() throws InterruptedException, ExecutionException
+ {
+ Result result = handshake(current_version, current_version, current_version + 1, current_version +1, current_version + 2);
+ Assert.assertEquals(Result.Outcome.SUCCESS, result.outcome);
+ Assert.assertEquals(current_version + 1, result.success().messagingVersion);
+ result.success().channel.close();
+ }
+
+ @Test
+ public void testSendCompatibleFutureVersion() throws InterruptedException, ExecutionException
+ {
+ Result result = handshake(current_version + 1, current_version - 1, current_version + 1);
+ Assert.assertEquals(Result.Outcome.SUCCESS, result.outcome);
+ Assert.assertEquals(current_version, result.success().messagingVersion);
+ result.success().channel.close();
+ }
+
+ @Test
+ public void testSendIncompatibleFutureVersion() throws InterruptedException, ExecutionException
+ {
+ Result result = handshake(current_version + 1, current_version + 1, current_version + 1);
+ Assert.assertEquals(Result.Outcome.INCOMPATIBLE, result.outcome);
+ Assert.assertEquals(current_version, result.incompatible().closestSupportedVersion);
+ Assert.assertEquals(current_version, result.incompatible().maxMessagingVersion);
+ }
+
+ @Test
+ public void testSendIncompatibleOldVersion() throws InterruptedException, ExecutionException
+ {
+ Result result = handshake(current_version + 1, current_version + 1, current_version + 1, current_version + 2, current_version + 3);
+ Assert.assertEquals(Result.Outcome.INCOMPATIBLE, result.outcome);
+ Assert.assertEquals(current_version + 2, result.incompatible().closestSupportedVersion);
+ Assert.assertEquals(current_version + 3, result.incompatible().maxMessagingVersion);
+ }
+
+ @Test
+ public void testSendCompatibleMaxVersionPre40() throws InterruptedException, ExecutionException
+ {
+ Result result = handshake(VERSION_3014, VERSION_30, VERSION_3014, VERSION_30, VERSION_3014);
+ Assert.assertEquals(Result.Outcome.SUCCESS, result.outcome);
+ Assert.assertEquals(VERSION_3014, result.success().messagingVersion);
+ result.success().channel.close();
+ }
+
+ @Test
+ public void testSendCompatibleFutureVersionPre40() throws InterruptedException, ExecutionException
+ {
+ Result result = handshake(VERSION_3014, VERSION_30, VERSION_3014, VERSION_30, VERSION_30);
+ Assert.assertEquals(Result.Outcome.RETRY, result.outcome);
+ Assert.assertEquals(VERSION_30, result.retry().withMessagingVersion);
+ }
+
+ @Test
+ public void testSendIncompatibleFutureVersionPre40() throws InterruptedException, ExecutionException
+ {
+ Result result = handshake(VERSION_3014, VERSION_3014, VERSION_3014, VERSION_30, VERSION_30);
+ Assert.assertEquals(Result.Outcome.INCOMPATIBLE, result.outcome);
+ Assert.assertEquals(-1, result.incompatible().closestSupportedVersion);
+ Assert.assertEquals(VERSION_30, result.incompatible().maxMessagingVersion);
+ }
+
+ @Test
+ public void testSendCompatibleOldVersionPre40() throws InterruptedException
+ {
+ try
+ {
+ handshake(VERSION_30, VERSION_30, VERSION_3014, VERSION_3014, VERSION_3014);
+ Assert.fail("Should have thrown");
+ }
+ catch (ExecutionException e)
+ {
+ Assert.assertTrue(e.getCause() instanceof ClosedChannelException);
+ }
+ }
+
+ @Test
+ public void testSendIncompatibleOldVersionPre40() throws InterruptedException
+ {
+ try
+ {
+ handshake(VERSION_30, VERSION_30, VERSION_30, VERSION_3014, VERSION_3014);
+ Assert.fail("Should have thrown");
+ }
+ catch (ExecutionException e)
+ {
+ Assert.assertTrue(e.getCause() instanceof ClosedChannelException);
+ }
+ }
+
+ @Test
+ public void testSendCompatibleOldVersion40() throws InterruptedException, ExecutionException
+ {
+ Result result = handshake(VERSION_30, VERSION_30, VERSION_30, VERSION_30, current_version);
+ Assert.assertEquals(Result.Outcome.SUCCESS, result.outcome);
+ Assert.assertEquals(VERSION_30, result.success().messagingVersion);
+ }
+
+ @Test
+ public void testSendIncompatibleOldVersion40() throws InterruptedException
+ {
+ try
+ {
+ Assert.fail(Objects.toString(handshake(VERSION_30, VERSION_30, VERSION_30, current_version, current_version)));
+ }
+ catch (ExecutionException e)
+ {
+ Assert.assertTrue(e.getCause() instanceof ClosedChannelException);
+ }
+ }
+
+ @Test // fairly contrived case, but since we introduced logic for testing we need to be careful it doesn't make us worse
+ public void testSendToFuturePost40BelievedToBePre40() throws InterruptedException, ExecutionException
+ {
+ Result result = handshake(VERSION_30, VERSION_30, current_version, VERSION_30, current_version + 1);
+ Assert.assertEquals(Result.Outcome.SUCCESS, result.outcome);
+ Assert.assertEquals(VERSION_30, result.success().messagingVersion);
+ }
+}
diff --git a/test/unit/org/apache/cassandra/net/ManyToOneConcurrentLinkedQueueTest.java b/test/unit/org/apache/cassandra/net/ManyToOneConcurrentLinkedQueueTest.java
new file mode 100644
index 0000000..2c92a39
--- /dev/null
+++ b/test/unit/org/apache/cassandra/net/ManyToOneConcurrentLinkedQueueTest.java
@@ -0,0 +1,301 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.net;
+
+import java.util.BitSet;
+import java.util.NoSuchElementException;
+import java.util.concurrent.Executor;
+import java.util.concurrent.Executors;
+
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+@SuppressWarnings("ConstantConditions")
+public class ManyToOneConcurrentLinkedQueueTest
+{
+ private final ManyToOneConcurrentLinkedQueue<Integer> queue = new ManyToOneConcurrentLinkedQueue<>();
+
+ @Test
+ public void testRelaxedIsEmptyWhenEmpty()
+ {
+ assertTrue(queue.relaxedIsEmpty());
+ }
+
+ @Test
+ public void testRelaxedIsEmptyWhenNotEmpty()
+ {
+ queue.offer(0);
+ assertFalse(queue.relaxedIsEmpty());
+ }
+
+ @Test
+ public void testSizeWhenEmpty()
+ {
+ assertEquals(0, queue.size());
+ }
+
+ @Test
+ public void testSizeWhenNotEmpty()
+ {
+ queue.offer(0);
+ assertEquals(1, queue.size());
+
+ for (int i = 1; i < 100; i++)
+ queue.offer(i);
+ assertEquals(100, queue.size());
+ }
+
+ @Test
+ public void testEmptyPeek()
+ {
+ assertNull(queue.peek());
+ }
+
+ @Test
+ public void testNonEmptyPeek()
+ {
+ queue.offer(0);
+ assertEquals(0, (int) queue.peek());
+ }
+
+ @Test
+ public void testEmptyPoll()
+ {
+ assertNull(queue.poll());
+ }
+
+ @Test
+ public void testNonEmptyPoll()
+ {
+ queue.offer(0);
+ assertEquals(0, (int) queue.poll());
+ }
+
+ @Test(expected = NoSuchElementException.class)
+ public void testEmptyRemove()
+ {
+ queue.remove();
+ }
+
+ @Test
+ public void testNonEmptyRemove()
+ {
+ queue.offer(0);
+ assertEquals(0, (int) queue.remove());
+ }
+
+ @Test
+ public void testOtherRemoveWhenEmpty()
+ {
+ assertFalse(queue.remove(0));
+ }
+
+ @Test
+ public void testOtherRemoveSingleNode()
+ {
+ queue.offer(0);
+ assertTrue(queue.remove(0));
+ assertTrue(queue.isEmpty());
+ }
+
+ @Test
+ public void testOtherRemoveWhenFirst()
+ {
+ queue.offer(0);
+ queue.offer(1);
+ queue.offer(2);
+
+ assertTrue(queue.remove(0));
+
+ assertEquals(1, (int) queue.poll());
+ assertEquals(2, (int) queue.poll());
+ assertNull(queue.poll());
+ }
+
+ @Test
+ public void testOtherRemoveFromMiddle()
+ {
+ queue.offer(0);
+ queue.offer(1);
+ queue.offer(2);
+
+ assertTrue(queue.remove(1));
+
+ assertEquals(0, (int) queue.poll());
+ assertEquals(2, (int) queue.poll());
+ assertNull(queue.poll());
+ }
+
+ @Test
+ public void testOtherRemoveFromEnd()
+ {
+ queue.offer(0);
+ queue.offer(1);
+ queue.offer(2);
+
+ assertTrue(queue.remove(2));
+
+ assertEquals(0, (int) queue.poll());
+ assertEquals(1, (int) queue.poll());
+ assertNull(queue.poll());
+ }
+
+ @Test
+ public void testOtherRemoveWhenDoesnNotExist()
+ {
+ queue.offer(0);
+ queue.offer(1);
+ queue.offer(2);
+
+ assertFalse(queue.remove(3));
+
+ assertEquals(0, (int) queue.poll());
+ assertEquals(1, (int) queue.poll());
+ assertEquals(2, (int) queue.poll());
+ }
+
+ @Test
+ public void testTransfersInCorrectOrder()
+ {
+ for (int i = 0; i < 1024; i++)
+ queue.offer(i);
+
+ for (int i = 0; i < 1024; i++)
+ assertEquals(i, (int) queue.poll());
+
+ assertTrue(queue.relaxedIsEmpty());
+ }
+
+ @Test
+ public void testTransfersInCorrectOrderWhenInterleaved()
+ {
+ for (int i = 0; i < 1024; i++)
+ {
+ queue.offer(i);
+ assertEquals(i, (int) queue.poll());
+ }
+
+ assertTrue(queue.relaxedIsEmpty());
+ }
+
+ @Test
+ public void testDrain()
+ {
+ for (int i = 0; i < 1024; i++)
+ queue.offer(i);
+
+ class Consumer
+ {
+ private int previous = -1;
+
+ public void accept(int i)
+ {
+ assertEquals(++previous, i);
+ }
+ }
+
+ Consumer consumer = new Consumer();
+ queue.drain(consumer::accept);
+
+ assertEquals(1023, consumer.previous);
+ assertTrue(queue.relaxedIsEmpty());
+ }
+
+ @Test
+ public void testPeekLastAndOffer()
+ {
+ assertNull(queue.relaxedPeekLastAndOffer(0));
+ for (int i = 1; i < 1024; i++)
+ assertEquals(i - 1, (int) queue.relaxedPeekLastAndOffer(i));
+
+ for (int i = 0; i < 1024; i++)
+ assertEquals(i, (int) queue.poll());
+
+ assertTrue(queue.relaxedIsEmpty());
+ }
+
+ enum Strategy
+ {
+ PEEK_AND_REMOVE, POLL
+ }
+
+ @Test
+ public void testConcurrentlyWithPoll()
+ {
+ testConcurrently(Strategy.POLL);
+ }
+
+ @Test
+ public void testConcurrentlyWithPeekAndRemove()
+ {
+ testConcurrently(Strategy.PEEK_AND_REMOVE);
+ }
+
+ private void testConcurrently(Strategy strategy)
+ {
+ int numThreads = 4;
+ int numItems = 1_000_000 * numThreads;
+
+ class Producer implements Runnable
+ {
+ private final int start, step, limit;
+
+ private Producer(int start, int step, int limit)
+ {
+ this.start = start;
+ this.step = step;
+ this.limit = limit;
+ }
+
+ public void run()
+ {
+ for (int i = start; i < limit; i += step)
+ queue.offer(i);
+ }
+ }
+
+ Executor executor = Executors.newFixedThreadPool(numThreads);
+ for (int i = 0; i < numThreads; i++)
+ executor.execute(new Producer(i, numThreads, numItems));
+
+ BitSet itemsPolled = new BitSet(numItems);
+ for (int i = 0; i < numItems; i++)
+ {
+ Integer item;
+ switch (strategy)
+ {
+ case PEEK_AND_REMOVE:
+ //noinspection StatementWithEmptyBody
+ while ((item = queue.peek()) == null) ;
+ assertFalse(queue.relaxedIsEmpty());
+ assertEquals(item, queue.remove());
+ itemsPolled.set(item);
+ break;
+ case POLL:
+ //noinspection StatementWithEmptyBody
+ while ((item = queue.poll()) == null) ;
+ itemsPolled.set(item);
+ break;
+ }
+ }
+
+ assertEquals(numItems, itemsPolled.cardinality());
+ assertTrue(queue.relaxedIsEmpty());
+ }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/net/Matcher.java b/test/unit/org/apache/cassandra/net/Matcher.java
index cd1b667..6f8e1e7 100644
--- a/test/unit/org/apache/cassandra/net/Matcher.java
+++ b/test/unit/org/apache/cassandra/net/Matcher.java
@@ -17,7 +17,7 @@
*/
package org.apache.cassandra.net;
-import java.net.InetAddress;
+import org.apache.cassandra.locator.InetAddressAndPort;
/**
* Predicate based on intercepted, outgoing messange and the message's destination address.
@@ -28,5 +28,5 @@
* @param obj intercepted outgoing message
* @param to destination address
*/
- public boolean matches(MessageOut<T> obj, InetAddress to);
+ public boolean matches(Message<T> obj, InetAddressAndPort to);
}
diff --git a/test/unit/org/apache/cassandra/net/MatcherResponse.java b/test/unit/org/apache/cassandra/net/MatcherResponse.java
index 6cd8085..d7b3759 100644
--- a/test/unit/org/apache/cassandra/net/MatcherResponse.java
+++ b/test/unit/org/apache/cassandra/net/MatcherResponse.java
@@ -17,16 +17,21 @@
*/
package org.apache.cassandra.net;
-import java.net.InetAddress;
-import java.util.Collections;
-import java.util.HashSet;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
import java.util.Queue;
-import java.util.Set;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.BiFunction;
+import java.util.function.BiPredicate;
import java.util.function.Function;
+import com.google.common.collect.Multimap;
+import com.google.common.collect.Multimaps;
+
+import org.apache.cassandra.locator.InetAddressAndPort;
+
/**
* Sends a response for an incoming message with a matching {@link Matcher}.
* The actual behavior by any instance of this class can be inspected by
@@ -35,10 +40,11 @@
public class MatcherResponse
{
private final Matcher<?> matcher;
- private final Set<Integer> sendResponses = new HashSet<>();
+ private final Multimap<Long, InetAddressAndPort> sendResponses =
+ Multimaps.newListMultimap(new HashMap<>(), ArrayList::new);
private final MockMessagingSpy spy = new MockMessagingSpy();
private final AtomicInteger limitCounter = new AtomicInteger(Integer.MAX_VALUE);
- private IMessageSink sink;
+ private BiPredicate<Message<?>, InetAddressAndPort> sink;
MatcherResponse(Matcher<?> matcher)
{
@@ -50,33 +56,33 @@
*/
public MockMessagingSpy dontReply()
{
- return respond((MessageIn<?>)null);
+ return respond((Message<?>)null);
}
/**
- * Respond with provided message in reply to each intercepted outbound message.
- * @param message the message to use as mock reply from the cluster
+ * Respond with provided message in response to each intercepted outbound message.
+ * @param message the message to use as mock response from the cluster
*/
- public MockMessagingSpy respond(MessageIn<?> message)
+ public MockMessagingSpy respond(Message<?> message)
{
return respondN(message, Integer.MAX_VALUE);
}
/**
- * Respond a limited number of times with the provided message in reply to each intercepted outbound message.
- * @param response the message to use as mock reply from the cluster
+ * Respond a limited number of times with the provided message in response to each intercepted outbound message.
+ * @param response the message to use as mock response from the cluster
* @param limit number of times to respond with message
*/
- public MockMessagingSpy respondN(final MessageIn<?> response, int limit)
+ public MockMessagingSpy respondN(final Message<?> response, int limit)
{
return respondN((in, to) -> response, limit);
}
/**
* Respond with the message created by the provided function that will be called with each intercepted outbound message.
- * @param fnResponse function to call for creating reply based on intercepted message and target address
+ * @param fnResponse function to call for creating response based on intercepted message and target address
*/
- public <T, S> MockMessagingSpy respond(BiFunction<MessageOut<T>, InetAddress, MessageIn<S>> fnResponse)
+ public <T, S> MockMessagingSpy respond(BiFunction<Message<T>, InetAddressAndPort, Message<S>> fnResponse)
{
return respondN(fnResponse, Integer.MAX_VALUE);
}
@@ -85,9 +91,9 @@
* Respond with message wrapping the payload object created by provided function called for each intercepted outbound message.
* The target address from the intercepted message will automatically be used as the created message's sender address.
* @param fnResponse function to call for creating payload object based on intercepted message and target address
- * @param verb verb to use for reply message
+ * @param verb verb to use for response message
*/
- public <T, S> MockMessagingSpy respondWithPayloadForEachReceiver(Function<MessageOut<T>, S> fnResponse, MessagingService.Verb verb)
+ public <T, S> MockMessagingSpy respondWithPayloadForEachReceiver(Function<Message<T>, S> fnResponse, Verb verb)
{
return respondNWithPayloadForEachReceiver(fnResponse, verb, Integer.MAX_VALUE);
}
@@ -97,40 +103,40 @@
* each intercepted outbound message. The target address from the intercepted message will automatically be used as the
* created message's sender address.
* @param fnResponse function to call for creating payload object based on intercepted message and target address
- * @param verb verb to use for reply message
+ * @param verb verb to use for response message
*/
- public <T, S> MockMessagingSpy respondNWithPayloadForEachReceiver(Function<MessageOut<T>, S> fnResponse, MessagingService.Verb verb, int limit)
+ public <T, S> MockMessagingSpy respondNWithPayloadForEachReceiver(Function<Message<T>, S> fnResponse, Verb verb, int limit)
{
- return respondN((MessageOut<T> msg, InetAddress to) -> {
+ return respondN((Message<T> msg, InetAddressAndPort to) -> {
S payload = fnResponse.apply(msg);
if (payload == null)
return null;
else
- return MessageIn.create(to, payload, Collections.emptyMap(), verb, MessagingService.current_version);
+ return Message.builder(verb, payload).from(to).build();
},
limit);
}
/**
* Responds to each intercepted outbound message by creating a response message wrapping the next element consumed
- * from the provided queue. No reply will be send when the queue has been exhausted.
+ * from the provided queue. No response will be send when the queue has been exhausted.
* @param cannedResponses prepared payload messages to use for responses
- * @param verb verb to use for reply message
+ * @param verb verb to use for response message
*/
- public <T, S> MockMessagingSpy respondWithPayloadForEachReceiver(Queue<S> cannedResponses, MessagingService.Verb verb)
+ public <T, S> MockMessagingSpy respondWithPayloadForEachReceiver(Queue<S> cannedResponses, Verb verb)
{
- return respondWithPayloadForEachReceiver((MessageOut<T> msg) -> cannedResponses.poll(), verb);
+ return respondWithPayloadForEachReceiver((Message<T> msg) -> cannedResponses.poll(), verb);
}
/**
* Responds to each intercepted outbound message by creating a response message wrapping the next element consumed
* from the provided queue. This method will block until queue elements are available.
* @param cannedResponses prepared payload messages to use for responses
- * @param verb verb to use for reply message
+ * @param verb verb to use for response message
*/
- public <T, S> MockMessagingSpy respondWithPayloadForEachReceiver(BlockingQueue<S> cannedResponses, MessagingService.Verb verb)
+ public <T, S> MockMessagingSpy respondWithPayloadForEachReceiver(BlockingQueue<S> cannedResponses, Verb verb)
{
- return respondWithPayloadForEachReceiver((MessageOut<T> msg) -> {
+ return respondWithPayloadForEachReceiver((Message<T> msg) -> {
try
{
return cannedResponses.take();
@@ -145,17 +151,17 @@
/**
* Respond a limited number of times with the message created by the provided function that will be called with
* each intercepted outbound message.
- * @param fnResponse function to call for creating reply based on intercepted message and target address
+ * @param fnResponse function to call for creating response based on intercepted message and target address
*/
- public <T, S> MockMessagingSpy respondN(BiFunction<MessageOut<T>, InetAddress, MessageIn<S>> fnResponse, int limit)
+ public <T, S> MockMessagingSpy respondN(BiFunction<Message<T>, InetAddressAndPort, Message<S>> fnResponse, int limit)
{
limitCounter.set(limit);
assert sink == null: "destroy() must be called first to register new response";
- sink = new IMessageSink()
+ sink = new BiPredicate<Message<?>, InetAddressAndPort>()
{
- public boolean allowOutgoingMessage(MessageOut message, int id, InetAddress to)
+ public boolean test(Message message, InetAddressAndPort to)
{
// prevent outgoing message from being send in case matcher indicates a match
// and instead send the mocked response
@@ -168,23 +174,25 @@
synchronized (sendResponses)
{
- // I'm not sure about retry semantics regarding message/ID relationships, but I assume
- // sending a message multiple times using the same ID shouldn't happen..
- assert !sendResponses.contains(id) : "ID re-use for outgoing message";
- sendResponses.add(id);
+ if (message.hasId())
+ {
+ assert !sendResponses.get(message.id()).contains(to) : "ID re-use for outgoing message";
+ sendResponses.put(message.id(), to);
+ }
}
// create response asynchronously to match request/response communication execution behavior
new Thread(() ->
{
- MessageIn<?> response = fnResponse.apply(message, to);
+ Message<?> response = fnResponse.apply(message, to);
if (response != null)
{
- CallbackInfo cb = MessagingService.instance().getRegisteredCallback(id);
+ RequestCallbacks.CallbackInfo cb = MessagingService.instance().callbacks.get(message.id(), to);
if (cb != null)
- cb.callback.response(response);
+ cb.callback.onResponse(response);
else
- MessagingService.instance().receive(response, id);
+ processResponse(response);
+
spy.matchingResponse(response);
}
}).start();
@@ -193,22 +201,34 @@
}
return true;
}
-
- public boolean allowIncomingMessage(MessageIn message, int id)
- {
- return true;
- }
};
- MessagingService.instance().addMessageSink(sink);
+ MessagingService.instance().outboundSink.add(sink);
return spy;
}
+ private void processResponse(Message<?> message)
+ {
+ if (!MessagingService.instance().inboundSink.allow(message))
+ return;
+
+ message.verb().stage.execute(() -> {
+ try
+ {
+ message.verb().handler().doVerb((Message<Object>)message);
+ }
+ catch (IOException e)
+ {
+ //
+ }
+ });
+ }
+
/**
* Stops currently registered response from being send.
*/
public void destroy()
{
- MessagingService.instance().removeMessageSink(sink);
+ MessagingService.instance().outboundSink.remove(sink);
}
}
diff --git a/test/unit/org/apache/cassandra/net/MessageTest.java b/test/unit/org/apache/cassandra/net/MessageTest.java
new file mode 100644
index 0000000..6a9d23f
--- /dev/null
+++ b/test/unit/org/apache/cassandra/net/MessageTest.java
@@ -0,0 +1,285 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.net;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Stream;
+
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.exceptions.RequestFailureReason;
+import org.apache.cassandra.io.IVersionedSerializer;
+import org.apache.cassandra.io.util.DataInputBuffer;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputBuffer;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.tracing.Tracing;
+import org.apache.cassandra.tracing.Tracing.TraceType;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.UUIDGen;
+
+import static org.apache.cassandra.net.Message.serializer;
+import static org.apache.cassandra.net.MessagingService.VERSION_3014;
+import static org.apache.cassandra.net.MessagingService.VERSION_30;
+import static org.apache.cassandra.net.MessagingService.VERSION_40;
+import static org.apache.cassandra.net.NoPayload.noPayload;
+import static org.apache.cassandra.net.ParamType.RESPOND_TO;
+import static org.apache.cassandra.net.ParamType.TRACE_SESSION;
+import static org.apache.cassandra.net.ParamType.TRACE_TYPE;
+import static org.apache.cassandra.utils.MonotonicClock.approxTime;
+import static org.junit.Assert.*;
+
+public class MessageTest
+{
+ @BeforeClass
+ public static void setUpClass() throws Exception
+ {
+ DatabaseDescriptor.daemonInitialization();
+ DatabaseDescriptor.setCrossNodeTimeout(true);
+
+ Verb._TEST_2.unsafeSetSerializer(() -> new IVersionedSerializer<Integer>()
+ {
+ public void serialize(Integer value, DataOutputPlus out, int version) throws IOException
+ {
+ out.writeInt(value);
+ }
+
+ public Integer deserialize(DataInputPlus in, int version) throws IOException
+ {
+ return in.readInt();
+ }
+
+ public long serializedSize(Integer value, int version)
+ {
+ return 4;
+ }
+ });
+ }
+
+ @AfterClass
+ public static void tearDownClass() throws Exception
+ {
+ Verb._TEST_2.unsafeSetSerializer(() -> NoPayload.serializer);
+ }
+
+ @Test
+ public void testInferMessageSize() throws Exception
+ {
+ Message<Integer> msg =
+ Message.builder(Verb._TEST_2, 37)
+ .withId(1)
+ .from(FBUtilities.getLocalAddressAndPort())
+ .withCreatedAt(approxTime.now())
+ .withExpiresAt(approxTime.now())
+ .withFlag(MessageFlag.CALL_BACK_ON_FAILURE)
+ .withFlag(MessageFlag.TRACK_REPAIRED_DATA)
+ .withParam(TRACE_TYPE, TraceType.QUERY)
+ .withParam(TRACE_SESSION, UUID.randomUUID())
+ .build();
+
+ testInferMessageSize(msg, VERSION_30);
+ testInferMessageSize(msg, VERSION_3014);
+ testInferMessageSize(msg, VERSION_40);
+ }
+
+ private void testInferMessageSize(Message msg, int version) throws Exception
+ {
+ try (DataOutputBuffer out = new DataOutputBuffer())
+ {
+ serializer.serialize(msg, out, version);
+ assertEquals(msg.serializedSize(version), out.getLength());
+
+ ByteBuffer buffer = out.buffer();
+
+ int payloadSize = (int) msg.verb().serializer().serializedSize(msg.payload, version);
+ int serializedSize = msg.serializedSize(version);
+
+ // should return -1 - fail to infer size - for all lengths of buffer until payload length can be read
+ for (int limit = 0; limit < serializedSize - payloadSize; limit++)
+ assertEquals(-1, serializer.inferMessageSize(buffer, 0, limit, version));
+
+ // once payload size can be read, should correctly infer message size
+ for (int limit = serializedSize - payloadSize; limit < serializedSize; limit++)
+ assertEquals(serializedSize, serializer.inferMessageSize(buffer, 0, limit, version));
+ }
+ }
+
+ @Test
+ public void testBuilder()
+ {
+ long id = 1;
+ InetAddressAndPort from = FBUtilities.getLocalAddressAndPort();
+ long createAtNanos = approxTime.now();
+ long expiresAtNanos = createAtNanos + TimeUnit.SECONDS.toNanos(1);
+ TraceType traceType = TraceType.QUERY;
+ UUID traceSession = UUID.randomUUID();
+
+ Message<NoPayload> msg =
+ Message.builder(Verb._TEST_1, noPayload)
+ .withId(1)
+ .from(from)
+ .withCreatedAt(createAtNanos)
+ .withExpiresAt(expiresAtNanos)
+ .withFlag(MessageFlag.CALL_BACK_ON_FAILURE)
+ .withParam(TRACE_TYPE, TraceType.QUERY)
+ .withParam(TRACE_SESSION, traceSession)
+ .build();
+
+ assertEquals(id, msg.id());
+ assertEquals(from, msg.from());
+ assertEquals(createAtNanos, msg.createdAtNanos());
+ assertEquals(expiresAtNanos, msg.expiresAtNanos());
+ assertTrue(msg.callBackOnFailure());
+ assertFalse(msg.trackRepairedData());
+ assertEquals(traceType, msg.traceType());
+ assertEquals(traceSession, msg.traceSession());
+ assertNull(msg.forwardTo());
+ assertNull(msg.respondTo());
+ }
+
+ @Test
+ public void testCycleNoPayload() throws IOException
+ {
+ Message<NoPayload> msg =
+ Message.builder(Verb._TEST_1, noPayload)
+ .withId(1)
+ .from(FBUtilities.getLocalAddressAndPort())
+ .withCreatedAt(approxTime.now())
+ .withExpiresAt(approxTime.now() + TimeUnit.SECONDS.toNanos(1))
+ .withFlag(MessageFlag.CALL_BACK_ON_FAILURE)
+ .withParam(TRACE_SESSION, UUID.randomUUID())
+ .build();
+ testCycle(msg);
+ }
+
+ @Test
+ public void testCycleWithPayload() throws Exception
+ {
+ testCycle(Message.out(Verb._TEST_2, 42));
+ testCycle(Message.outWithFlag(Verb._TEST_2, 42, MessageFlag.CALL_BACK_ON_FAILURE));
+ testCycle(Message.outWithFlags(Verb._TEST_2, 42, MessageFlag.CALL_BACK_ON_FAILURE, MessageFlag.TRACK_REPAIRED_DATA));
+ testCycle(Message.outWithParam(1, Verb._TEST_2, 42, RESPOND_TO, FBUtilities.getBroadcastAddressAndPort()));
+ }
+
+ @Test
+ public void testFailureResponse() throws IOException
+ {
+ long expiresAt = approxTime.now();
+ Message<RequestFailureReason> msg = Message.failureResponse(1, expiresAt, RequestFailureReason.INCOMPATIBLE_SCHEMA);
+
+ assertEquals(1, msg.id());
+ assertEquals(Verb.FAILURE_RSP, msg.verb());
+ assertEquals(expiresAt, msg.expiresAtNanos());
+ assertEquals(RequestFailureReason.INCOMPATIBLE_SCHEMA, msg.payload);
+ assertTrue(msg.isFailureResponse());
+
+ testCycle(msg);
+ }
+
+ @Test
+ public void testBuilderAddTraceHeaderWhenTraceSessionPresent()
+ {
+ Stream.of(TraceType.values()).forEach(this::testAddTraceHeaderWithType);
+ }
+
+ @Test
+ public void testBuilderNotAddTraceHeaderWithNoTraceSession()
+ {
+ Message<NoPayload> msg = Message.builder(Verb._TEST_1, noPayload).withTracingParams().build();
+ assertNull(msg.header.traceSession());
+ }
+
+ private void testAddTraceHeaderWithType(TraceType traceType)
+ {
+ try
+ {
+ UUID sessionId = Tracing.instance.newSession(traceType);
+ Message<NoPayload> msg = Message.builder(Verb._TEST_1, noPayload).withTracingParams().build();
+ assertEquals(sessionId, msg.header.traceSession());
+ assertEquals(traceType, msg.header.traceType());
+ }
+ finally
+ {
+ Tracing.instance.stopSession();
+ }
+ }
+
+ private void testCycle(Message msg) throws IOException
+ {
+ testCycle(msg, VERSION_30);
+ testCycle(msg, VERSION_3014);
+ testCycle(msg, VERSION_40);
+ }
+
+ // serialize (using both variants, all in one or header then rest), verify serialized size, deserialize, compare to the original
+ private void testCycle(Message msg, int version) throws IOException
+ {
+ try (DataOutputBuffer out = new DataOutputBuffer())
+ {
+ serializer.serialize(msg, out, version);
+ assertEquals(msg.serializedSize(version), out.getLength());
+
+ // deserialize the message in one go, compare outcomes
+ try (DataInputBuffer in = new DataInputBuffer(out.buffer(), true))
+ {
+ Message msgOut = serializer.deserialize(in, msg.from(), version);
+ assertEquals(0, in.available());
+ assertMessagesEqual(msg, msgOut);
+ }
+
+ // extract header first, then deserialize the rest of the message and compare outcomes
+ ByteBuffer buffer = out.buffer();
+ try (DataInputBuffer in = new DataInputBuffer(out.buffer(), false))
+ {
+ Message.Header headerOut = serializer.extractHeader(buffer, msg.from(), approxTime.now(), version);
+ Message msgOut = serializer.deserialize(in, headerOut, version);
+ assertEquals(0, in.available());
+ assertMessagesEqual(msg, msgOut);
+ }
+ }
+ }
+
+ private static void assertMessagesEqual(Message msg1, Message msg2)
+ {
+ assertEquals(msg1.id(), msg2.id());
+ assertEquals(msg1.verb(), msg2.verb());
+ assertEquals(msg1.callBackOnFailure(), msg2.callBackOnFailure());
+ assertEquals(msg1.trackRepairedData(), msg2.trackRepairedData());
+ assertEquals(msg1.traceType(), msg2.traceType());
+ assertEquals(msg1.traceSession(), msg2.traceSession());
+ assertEquals(msg1.respondTo(), msg2.respondTo());
+ assertEquals(msg1.forwardTo(), msg2.forwardTo());
+
+ Object payload1 = msg1.payload;
+ Object payload2 = msg2.payload;
+
+ if (null == payload1)
+ assertTrue(payload2 == noPayload || payload2 == null);
+ else if (null == payload2)
+ assertSame(payload1, noPayload);
+ else
+ assertEquals(payload1, payload2);
+ }
+}
diff --git a/test/unit/org/apache/cassandra/net/MessagingServiceTest.java b/test/unit/org/apache/cassandra/net/MessagingServiceTest.java
index 82630b4..9ce041b 100644
--- a/test/unit/org/apache/cassandra/net/MessagingServiceTest.java
+++ b/test/unit/org/apache/cassandra/net/MessagingServiceTest.java
@@ -20,13 +20,13 @@
*/
package org.apache.cassandra.net;
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.net.InetAddress;
import java.net.UnknownHostException;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -36,33 +36,60 @@
import java.util.regex.Matcher;
import com.google.common.collect.Iterables;
+import com.google.common.net.InetAddresses;
+
import com.codahale.metrics.Timer;
+import org.apache.cassandra.auth.IInternodeAuthenticator;
import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.db.monitoring.ApproximateTime;
-import org.apache.cassandra.io.util.DataInputPlus.DataInputStreamPlus;
-import org.apache.cassandra.io.util.DataOutputStreamPlus;
-import org.apache.cassandra.io.util.WrappedDataOutputStreamPlus;
+import org.apache.cassandra.config.EncryptionOptions.ServerEncryptionOptions;
+import org.apache.cassandra.db.commitlog.CommitLog;
+import org.apache.cassandra.metrics.MessagingMetrics;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.utils.FBUtilities;
import org.caffinitas.ohc.histo.EstimatedHistogram;
+import org.junit.After;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
import static org.junit.Assert.*;
public class MessagingServiceTest
{
private final static long ONE_SECOND = TimeUnit.NANOSECONDS.convert(1, TimeUnit.SECONDS);
private final static long[] bucketOffsets = new EstimatedHistogram(160).getBucketOffsets();
- private final MessagingService messagingService = MessagingService.test();
+ public static final IInternodeAuthenticator ALLOW_NOTHING_AUTHENTICATOR = new IInternodeAuthenticator()
+ {
+ public boolean authenticate(InetAddress remoteAddress, int remotePort)
+ {
+ return false;
+ }
+
+ public void validateConfiguration() throws ConfigurationException
+ {
+
+ }
+ };
+ private static IInternodeAuthenticator originalAuthenticator;
+ private static ServerEncryptionOptions originalServerEncryptionOptions;
+ private static InetAddressAndPort originalListenAddress;
+
+ private final MessagingService messagingService = new MessagingService(true);
@BeforeClass
public static void beforeClass() throws UnknownHostException
{
DatabaseDescriptor.daemonInitialization();
+ CommitLog.instance.start();
DatabaseDescriptor.setBackPressureStrategy(new MockBackPressureStrategy(Collections.emptyMap()));
DatabaseDescriptor.setBroadcastAddress(InetAddress.getByName("127.0.0.1"));
+ originalAuthenticator = DatabaseDescriptor.getInternodeAuthenticator();
+ originalServerEncryptionOptions = DatabaseDescriptor.getInternodeMessagingEncyptionOptions();
+ originalListenAddress = InetAddressAndPort.getByAddressOverrideDefaults(DatabaseDescriptor.getListenAddress(), DatabaseDescriptor.getStoragePort());
}
private static int metricScopeId = 0;
@@ -70,38 +97,51 @@
@Before
public void before() throws UnknownHostException
{
- messagingService.resetDroppedMessagesMap(Integer.toString(metricScopeId++));
+ messagingService.metrics.resetDroppedMessages(Integer.toString(metricScopeId++));
MockBackPressureStrategy.applied = false;
- messagingService.destroyConnectionPool(InetAddress.getByName("127.0.0.2"));
- messagingService.destroyConnectionPool(InetAddress.getByName("127.0.0.3"));
+ messagingService.closeOutbound(InetAddressAndPort.getByName("127.0.0.2"));
+ messagingService.closeOutbound(InetAddressAndPort.getByName("127.0.0.3"));
+ }
+
+ @After
+ public void tearDown()
+ {
+ DatabaseDescriptor.setInternodeAuthenticator(originalAuthenticator);
+ DatabaseDescriptor.setInternodeMessagingEncyptionOptions(originalServerEncryptionOptions);
+ DatabaseDescriptor.setShouldListenOnBroadcastAddress(false);
+ DatabaseDescriptor.setListenAddress(originalListenAddress.address);
+ FBUtilities.reset();
}
@Test
public void testDroppedMessages()
{
- MessagingService.Verb verb = MessagingService.Verb.READ;
+ Verb verb = Verb.READ_REQ;
for (int i = 1; i <= 5000; i++)
- messagingService.incrementDroppedMessages(verb, i, i % 2 == 0);
+ messagingService.metrics.recordDroppedMessage(verb, i, MILLISECONDS, i % 2 == 0);
- List<String> logs = messagingService.getDroppedMessagesLogs();
+ List<String> logs = new ArrayList<>();
+ messagingService.metrics.resetAndConsumeDroppedErrors(logs::add);
assertEquals(1, logs.size());
- Pattern regexp = Pattern.compile("READ messages were dropped in last 5000 ms: (\\d+) internal and (\\d+) cross node. Mean internal dropped latency: (\\d+) ms and Mean cross-node dropped latency: (\\d+) ms");
+ Pattern regexp = Pattern.compile("READ_REQ messages were dropped in last 5000 ms: (\\d+) internal and (\\d+) cross node. Mean internal dropped latency: (\\d+) ms and Mean cross-node dropped latency: (\\d+) ms");
Matcher matcher = regexp.matcher(logs.get(0));
assertTrue(matcher.find());
assertEquals(2500, Integer.parseInt(matcher.group(1)));
assertEquals(2500, Integer.parseInt(matcher.group(2)));
assertTrue(Integer.parseInt(matcher.group(3)) > 0);
assertTrue(Integer.parseInt(matcher.group(4)) > 0);
- assertEquals(5000, (int) messagingService.getDroppedMessages().get(verb.toString()));
+ assertEquals(5000, (int) messagingService.metrics.getDroppedMessages().get(verb.toString()));
- logs = messagingService.getDroppedMessagesLogs();
+ logs.clear();
+ messagingService.metrics.resetAndConsumeDroppedErrors(logs::add);
assertEquals(0, logs.size());
for (int i = 0; i < 2500; i++)
- messagingService.incrementDroppedMessages(verb, i, i % 2 == 0);
+ messagingService.metrics.recordDroppedMessage(verb, i, MILLISECONDS, i % 2 == 0);
- logs = messagingService.getDroppedMessagesLogs();
+ logs.clear();
+ messagingService.metrics.resetAndConsumeDroppedErrors(logs::add);
assertEquals(1, logs.size());
matcher = regexp.matcher(logs.get(0));
assertTrue(matcher.find());
@@ -109,84 +149,111 @@
assertEquals(1250, Integer.parseInt(matcher.group(2)));
assertTrue(Integer.parseInt(matcher.group(3)) > 0);
assertTrue(Integer.parseInt(matcher.group(4)) > 0);
- assertEquals(7500, (int) messagingService.getDroppedMessages().get(verb.toString()));
+ assertEquals(7500, (int) messagingService.metrics.getDroppedMessages().get(verb.toString()));
}
@Test
public void testDCLatency() throws Exception
{
int latency = 100;
- ConcurrentHashMap<String, Timer> dcLatency = MessagingService.instance().metrics.dcLatency;
+ ConcurrentHashMap<String, MessagingMetrics.DCLatencyRecorder> dcLatency = MessagingService.instance().metrics.dcLatency;
dcLatency.clear();
- long now = ApproximateTime.currentTimeMillis();
+ long now = System.currentTimeMillis();
long sentAt = now - latency;
assertNull(dcLatency.get("datacenter1"));
addDCLatency(sentAt, now);
assertNotNull(dcLatency.get("datacenter1"));
- assertEquals(1, dcLatency.get("datacenter1").getCount());
- long expectedBucket = bucketOffsets[Math.abs(Arrays.binarySearch(bucketOffsets, TimeUnit.MILLISECONDS.toNanos(latency))) - 1];
- assertEquals(expectedBucket, dcLatency.get("datacenter1").getSnapshot().getMax());
+ assertEquals(1, dcLatency.get("datacenter1").dcLatency.getCount());
+ long expectedBucket = bucketOffsets[Math.abs(Arrays.binarySearch(bucketOffsets, MILLISECONDS.toNanos(latency))) - 1];
+ assertEquals(expectedBucket, dcLatency.get("datacenter1").dcLatency.getSnapshot().getMax());
}
@Test
- public void testNegativeDCLatency() throws Exception
+ public void testNegativeDCLatency()
{
+ MessagingMetrics.DCLatencyRecorder updater = MessagingService.instance().metrics.internodeLatencyRecorder(InetAddressAndPort.getLocalHost());
+
// if clocks are off should just not track anything
int latency = -100;
- ConcurrentHashMap<String, Timer> dcLatency = MessagingService.instance().metrics.dcLatency;
- dcLatency.clear();
-
- long now = ApproximateTime.currentTimeMillis();
+ long now = System.currentTimeMillis();
long sentAt = now - latency;
- assertNull(dcLatency.get("datacenter1"));
- addDCLatency(sentAt, now);
- assertNull(dcLatency.get("datacenter1"));
+ long count = updater.dcLatency.getCount();
+ updater.accept(now - sentAt, MILLISECONDS);
+ // negative value shoudln't be recorded
+ assertEquals(count, updater.dcLatency.getCount());
+ }
+
+ @Test
+ public void testQueueWaitLatency()
+ {
+ int latency = 100;
+ Verb verb = Verb.MUTATION_REQ;
+
+ Map<Verb, Timer> queueWaitLatency = MessagingService.instance().metrics.internalLatency;
+ MessagingService.instance().metrics.recordInternalLatency(verb, latency, MILLISECONDS);
+ assertEquals(1, queueWaitLatency.get(verb).getCount());
+ long expectedBucket = bucketOffsets[Math.abs(Arrays.binarySearch(bucketOffsets, MILLISECONDS.toNanos(latency))) - 1];
+ assertEquals(expectedBucket, queueWaitLatency.get(verb).getSnapshot().getMax());
+ }
+
+ @Test
+ public void testNegativeQueueWaitLatency() throws Exception
+ {
+ int latency = -100;
+ Verb verb = Verb.MUTATION_REQ;
+
+ Map<Verb, Timer> queueWaitLatency = MessagingService.instance().metrics.internalLatency;
+ queueWaitLatency.clear();
+
+ assertNull(queueWaitLatency.get(verb));
+ MessagingService.instance().metrics.recordInternalLatency(verb, latency, MILLISECONDS);
+ assertNull(queueWaitLatency.get(verb));
}
@Test
public void testUpdatesBackPressureOnSendWhenEnabledAndWithSupportedCallback() throws UnknownHostException
{
- MockBackPressureStrategy.MockBackPressureState backPressureState = (MockBackPressureStrategy.MockBackPressureState) messagingService.getConnectionPool(InetAddress.getByName("127.0.0.2")).getBackPressureState();
- IAsyncCallback bpCallback = new BackPressureCallback();
- IAsyncCallback noCallback = new NoBackPressureCallback();
- MessageOut<?> ignored = null;
+ MockBackPressureStrategy.MockBackPressureState backPressureState = (MockBackPressureStrategy.MockBackPressureState) messagingService.getBackPressureState(InetAddressAndPort.getByName("127.0.0.2"));
+ RequestCallback bpCallback = new BackPressureCallback();
+ RequestCallback noCallback = new NoBackPressureCallback();
+ Message<?> ignored = null;
DatabaseDescriptor.setBackPressureEnabled(true);
- messagingService.updateBackPressureOnSend(InetAddress.getByName("127.0.0.2"), noCallback, ignored);
+ messagingService.updateBackPressureOnSend(InetAddressAndPort.getByName("127.0.0.2"), noCallback, ignored);
assertFalse(backPressureState.onSend);
DatabaseDescriptor.setBackPressureEnabled(false);
- messagingService.updateBackPressureOnSend(InetAddress.getByName("127.0.0.2"), bpCallback, ignored);
+ messagingService.updateBackPressureOnSend(InetAddressAndPort.getByName("127.0.0.2"), bpCallback, ignored);
assertFalse(backPressureState.onSend);
DatabaseDescriptor.setBackPressureEnabled(true);
- messagingService.updateBackPressureOnSend(InetAddress.getByName("127.0.0.2"), bpCallback, ignored);
+ messagingService.updateBackPressureOnSend(InetAddressAndPort.getByName("127.0.0.2"), bpCallback, ignored);
assertTrue(backPressureState.onSend);
}
@Test
public void testUpdatesBackPressureOnReceiveWhenEnabledAndWithSupportedCallback() throws UnknownHostException
{
- MockBackPressureStrategy.MockBackPressureState backPressureState = (MockBackPressureStrategy.MockBackPressureState) messagingService.getConnectionPool(InetAddress.getByName("127.0.0.2")).getBackPressureState();
- IAsyncCallback bpCallback = new BackPressureCallback();
- IAsyncCallback noCallback = new NoBackPressureCallback();
+ MockBackPressureStrategy.MockBackPressureState backPressureState = (MockBackPressureStrategy.MockBackPressureState) messagingService.getBackPressureState(InetAddressAndPort.getByName("127.0.0.2"));
+ RequestCallback bpCallback = new BackPressureCallback();
+ RequestCallback noCallback = new NoBackPressureCallback();
boolean timeout = false;
DatabaseDescriptor.setBackPressureEnabled(true);
- messagingService.updateBackPressureOnReceive(InetAddress.getByName("127.0.0.2"), noCallback, timeout);
+ messagingService.updateBackPressureOnReceive(InetAddressAndPort.getByName("127.0.0.2"), noCallback, timeout);
assertFalse(backPressureState.onReceive);
assertFalse(backPressureState.onTimeout);
DatabaseDescriptor.setBackPressureEnabled(false);
- messagingService.updateBackPressureOnReceive(InetAddress.getByName("127.0.0.2"), bpCallback, timeout);
+ messagingService.updateBackPressureOnReceive(InetAddressAndPort.getByName("127.0.0.2"), bpCallback, timeout);
assertFalse(backPressureState.onReceive);
assertFalse(backPressureState.onTimeout);
DatabaseDescriptor.setBackPressureEnabled(true);
- messagingService.updateBackPressureOnReceive(InetAddress.getByName("127.0.0.2"), bpCallback, timeout);
+ messagingService.updateBackPressureOnReceive(InetAddressAndPort.getByName("127.0.0.2"), bpCallback, timeout);
assertTrue(backPressureState.onReceive);
assertFalse(backPressureState.onTimeout);
}
@@ -194,23 +261,23 @@
@Test
public void testUpdatesBackPressureOnTimeoutWhenEnabledAndWithSupportedCallback() throws UnknownHostException
{
- MockBackPressureStrategy.MockBackPressureState backPressureState = (MockBackPressureStrategy.MockBackPressureState) messagingService.getConnectionPool(InetAddress.getByName("127.0.0.2")).getBackPressureState();
- IAsyncCallback bpCallback = new BackPressureCallback();
- IAsyncCallback noCallback = new NoBackPressureCallback();
+ MockBackPressureStrategy.MockBackPressureState backPressureState = (MockBackPressureStrategy.MockBackPressureState) messagingService.getBackPressureState(InetAddressAndPort.getByName("127.0.0.2"));
+ RequestCallback bpCallback = new BackPressureCallback();
+ RequestCallback noCallback = new NoBackPressureCallback();
boolean timeout = true;
DatabaseDescriptor.setBackPressureEnabled(true);
- messagingService.updateBackPressureOnReceive(InetAddress.getByName("127.0.0.2"), noCallback, timeout);
+ messagingService.updateBackPressureOnReceive(InetAddressAndPort.getByName("127.0.0.2"), noCallback, timeout);
assertFalse(backPressureState.onReceive);
assertFalse(backPressureState.onTimeout);
DatabaseDescriptor.setBackPressureEnabled(false);
- messagingService.updateBackPressureOnReceive(InetAddress.getByName("127.0.0.2"), bpCallback, timeout);
+ messagingService.updateBackPressureOnReceive(InetAddressAndPort.getByName("127.0.0.2"), bpCallback, timeout);
assertFalse(backPressureState.onReceive);
assertFalse(backPressureState.onTimeout);
DatabaseDescriptor.setBackPressureEnabled(true);
- messagingService.updateBackPressureOnReceive(InetAddress.getByName("127.0.0.2"), bpCallback, timeout);
+ messagingService.updateBackPressureOnReceive(InetAddressAndPort.getByName("127.0.0.2"), bpCallback, timeout);
assertFalse(backPressureState.onReceive);
assertTrue(backPressureState.onTimeout);
}
@@ -219,11 +286,11 @@
public void testAppliesBackPressureWhenEnabled() throws UnknownHostException
{
DatabaseDescriptor.setBackPressureEnabled(false);
- messagingService.applyBackPressure(Arrays.asList(InetAddress.getByName("127.0.0.2")), ONE_SECOND);
+ messagingService.applyBackPressure(Arrays.asList(InetAddressAndPort.getByName("127.0.0.2")), ONE_SECOND);
assertFalse(MockBackPressureStrategy.applied);
DatabaseDescriptor.setBackPressureEnabled(true);
- messagingService.applyBackPressure(Arrays.asList(InetAddress.getByName("127.0.0.2")), ONE_SECOND);
+ messagingService.applyBackPressure(Arrays.asList(InetAddressAndPort.getByName("127.0.0.2")), ONE_SECOND);
assertTrue(MockBackPressureStrategy.applied);
}
@@ -231,19 +298,13 @@
public void testDoesntApplyBackPressureToBroadcastAddress() throws UnknownHostException
{
DatabaseDescriptor.setBackPressureEnabled(true);
- messagingService.applyBackPressure(Arrays.asList(InetAddress.getByName("127.0.0.1")), ONE_SECOND);
+ messagingService.applyBackPressure(Arrays.asList(InetAddressAndPort.getByName("127.0.0.1")), ONE_SECOND);
assertFalse(MockBackPressureStrategy.applied);
}
private static void addDCLatency(long sentAt, long nowTime) throws IOException
{
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- try (DataOutputStreamPlus out = new WrappedDataOutputStreamPlus(baos))
- {
- out.writeInt((int) sentAt);
- }
- DataInputStreamPlus in = new DataInputStreamPlus(new ByteArrayInputStream(baos.toByteArray()));
- MessageIn.readConstructionTime(FBUtilities.getLocalAddress(), in, nowTime);
+ MessagingService.instance().metrics.internodeLatencyRecorder(InetAddressAndPort.getLocalHost()).accept(nowTime - sentAt, MILLISECONDS);
}
public static class MockBackPressureStrategy implements BackPressureStrategy<MockBackPressureStrategy.MockBackPressureState>
@@ -262,25 +323,25 @@
}
@Override
- public MockBackPressureState newState(InetAddress host)
+ public MockBackPressureState newState(InetAddressAndPort host)
{
return new MockBackPressureState(host);
}
public static class MockBackPressureState implements BackPressureState
{
- private final InetAddress host;
+ private final InetAddressAndPort host;
public volatile boolean onSend = false;
public volatile boolean onReceive = false;
public volatile boolean onTimeout = false;
- private MockBackPressureState(InetAddress host)
+ private MockBackPressureState(InetAddressAndPort host)
{
this.host = host;
}
@Override
- public void onMessageSent(MessageOut<?> message)
+ public void onMessageSent(Message<?> message)
{
onSend = true;
}
@@ -304,14 +365,14 @@
}
@Override
- public InetAddress getHost()
+ public InetAddressAndPort getHost()
{
return host;
}
}
}
- private static class BackPressureCallback implements IAsyncCallback
+ private static class BackPressureCallback implements RequestCallback
{
@Override
public boolean supportsBackPressure()
@@ -320,19 +381,13 @@
}
@Override
- public boolean isLatencyForSnitch()
- {
- return false;
- }
-
- @Override
- public void response(MessageIn msg)
+ public void onResponse(Message msg)
{
throw new UnsupportedOperationException("Not supported.");
}
}
- private static class NoBackPressureCallback implements IAsyncCallback
+ private static class NoBackPressureCallback implements RequestCallback
{
@Override
public boolean supportsBackPressure()
@@ -341,15 +396,224 @@
}
@Override
- public boolean isLatencyForSnitch()
- {
- return false;
- }
-
- @Override
- public void response(MessageIn msg)
+ public void onResponse(Message msg)
{
throw new UnsupportedOperationException("Not supported.");
}
}
+
+ /**
+ * Make sure that if internode authenticatino fails for an outbound connection that all the code that relies
+ * on getting the connection pool handles the null return
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testFailedInternodeAuth() throws Exception
+ {
+ MessagingService ms = MessagingService.instance();
+ DatabaseDescriptor.setInternodeAuthenticator(ALLOW_NOTHING_AUTHENTICATOR);
+ InetAddressAndPort address = InetAddressAndPort.getByName("127.0.0.250");
+
+ //Should return null
+ Message messageOut = Message.out(Verb.ECHO_REQ, NoPayload.noPayload);
+ assertFalse(ms.isConnected(address, messageOut));
+
+ //Should tolerate null
+ ms.closeOutbound(address);
+ ms.send(messageOut, address);
+ }
+
+// @Test
+// public void reconnectWithNewIp() throws Exception
+// {
+// InetAddressAndPort publicIp = InetAddressAndPort.getByName("127.0.0.2");
+// InetAddressAndPort privateIp = InetAddressAndPort.getByName("127.0.0.3");
+//
+// // reset the preferred IP value, for good test hygene
+// SystemKeyspace.updatePreferredIP(publicIp, publicIp);
+//
+// // create pool/conn with public addr
+// Assert.assertEquals(publicIp, messagingService.getCurrentEndpoint(publicIp));
+// messagingService.maybeReconnectWithNewIp(publicIp, privateIp).await(1L, TimeUnit.SECONDS);
+// Assert.assertEquals(privateIp, messagingService.getCurrentEndpoint(publicIp));
+//
+// messagingService.closeOutbound(publicIp);
+//
+// // recreate the pool/conn, and make sure the preferred ip addr is used
+// Assert.assertEquals(privateIp, messagingService.getCurrentEndpoint(publicIp));
+// }
+
+ @Test
+ public void listenPlainConnection() throws InterruptedException
+ {
+ ServerEncryptionOptions serverEncryptionOptions = new ServerEncryptionOptions()
+ .withInternodeEncryption(ServerEncryptionOptions.InternodeEncryption.none);
+ listen(serverEncryptionOptions, false);
+ }
+
+ @Test
+ public void listenPlainConnectionWithBroadcastAddr() throws InterruptedException
+ {
+ ServerEncryptionOptions serverEncryptionOptions = new ServerEncryptionOptions()
+ .withInternodeEncryption(ServerEncryptionOptions.InternodeEncryption.none);
+ listen(serverEncryptionOptions, true);
+ }
+
+ @Test
+ public void listenRequiredSecureConnection() throws InterruptedException
+ {
+ ServerEncryptionOptions serverEncryptionOptions = new ServerEncryptionOptions()
+ .withOptional(false)
+ .withInternodeEncryption(ServerEncryptionOptions.InternodeEncryption.all)
+ .withLegacySslStoragePort(false);
+ listen(serverEncryptionOptions, false);
+ }
+
+ @Test
+ public void listenRequiredSecureConnectionWithBroadcastAddr() throws InterruptedException
+ {
+ ServerEncryptionOptions serverEncryptionOptions = new ServerEncryptionOptions()
+ .withOptional(false)
+ .withInternodeEncryption(ServerEncryptionOptions.InternodeEncryption.all)
+ .withLegacySslStoragePort(false);
+ listen(serverEncryptionOptions, true);
+ }
+
+ @Test
+ public void listenRequiredSecureConnectionWithLegacyPort() throws InterruptedException
+ {
+ ServerEncryptionOptions serverEncryptionOptions = new ServerEncryptionOptions()
+ .withInternodeEncryption(ServerEncryptionOptions.InternodeEncryption.all)
+ .withOptional(false)
+ .withLegacySslStoragePort(true);
+ listen(serverEncryptionOptions, false);
+ }
+
+ @Test
+ public void listenRequiredSecureConnectionWithBroadcastAddrAndLegacyPort() throws InterruptedException
+ {
+ ServerEncryptionOptions serverEncryptionOptions = new ServerEncryptionOptions()
+ .withInternodeEncryption(ServerEncryptionOptions.InternodeEncryption.all)
+ .withOptional(false)
+ .withLegacySslStoragePort(true);
+ listen(serverEncryptionOptions, true);
+ }
+
+ @Test
+ public void listenOptionalSecureConnection() throws InterruptedException
+ {
+ ServerEncryptionOptions serverEncryptionOptions = new ServerEncryptionOptions()
+ .withOptional(true);
+ listen(serverEncryptionOptions, false);
+ }
+
+ @Test
+ public void listenOptionalSecureConnectionWithBroadcastAddr() throws InterruptedException
+ {
+ ServerEncryptionOptions serverEncryptionOptions = new ServerEncryptionOptions()
+ .withOptional(true);
+ listen(serverEncryptionOptions, true);
+ }
+
+ private void listen(ServerEncryptionOptions serverEncryptionOptions, boolean listenOnBroadcastAddr) throws InterruptedException
+ {
+ InetAddress listenAddress = FBUtilities.getJustLocalAddress();
+ if (listenOnBroadcastAddr)
+ {
+ DatabaseDescriptor.setShouldListenOnBroadcastAddress(true);
+ listenAddress = InetAddresses.increment(FBUtilities.getBroadcastAddressAndPort().address);
+ DatabaseDescriptor.setListenAddress(listenAddress);
+ FBUtilities.reset();
+ }
+
+ InboundConnectionSettings settings = new InboundConnectionSettings()
+ .withEncryption(serverEncryptionOptions);
+ InboundSockets connections = new InboundSockets(settings);
+ try
+ {
+ connections.open().await();
+ Assert.assertTrue(connections.isListening());
+
+ Set<InetAddressAndPort> expect = new HashSet<>();
+ expect.add(InetAddressAndPort.getByAddressOverrideDefaults(listenAddress, DatabaseDescriptor.getStoragePort()));
+ if (settings.encryption.enable_legacy_ssl_storage_port)
+ expect.add(InetAddressAndPort.getByAddressOverrideDefaults(listenAddress, DatabaseDescriptor.getSSLStoragePort()));
+ if (listenOnBroadcastAddr)
+ {
+ expect.add(InetAddressAndPort.getByAddressOverrideDefaults(FBUtilities.getBroadcastAddressAndPort().address, DatabaseDescriptor.getStoragePort()));
+ if (settings.encryption.enable_legacy_ssl_storage_port)
+ expect.add(InetAddressAndPort.getByAddressOverrideDefaults(FBUtilities.getBroadcastAddressAndPort().address, DatabaseDescriptor.getSSLStoragePort()));
+ }
+
+ Assert.assertEquals(expect.size(), connections.sockets().size());
+
+ final int legacySslPort = DatabaseDescriptor.getSSLStoragePort();
+ for (InboundSockets.InboundSocket socket : connections.sockets())
+ {
+ Assert.assertEquals(serverEncryptionOptions.isEnabled(), socket.settings.encryption.isEnabled());
+ Assert.assertEquals(serverEncryptionOptions.optional, socket.settings.encryption.optional);
+ if (!serverEncryptionOptions.isEnabled())
+ Assert.assertFalse(legacySslPort == socket.settings.bindAddress.port);
+ if (legacySslPort == socket.settings.bindAddress.port)
+ Assert.assertFalse(socket.settings.encryption.optional);
+ Assert.assertTrue(socket.settings.bindAddress.toString(), expect.remove(socket.settings.bindAddress));
+ }
+ }
+ finally
+ {
+ connections.close().await();
+ Assert.assertFalse(connections.isListening());
+ }
+ }
+
+
+// @Test
+// public void getPreferredRemoteAddrUsesPrivateIp() throws UnknownHostException
+// {
+// MessagingService ms = MessagingService.instance();
+// InetAddressAndPort remote = InetAddressAndPort.getByNameOverrideDefaults("127.0.0.151", 7000);
+// InetAddressAndPort privateIp = InetAddressAndPort.getByName("127.0.0.6");
+//
+// OutboundConnectionSettings template = new OutboundConnectionSettings(remote)
+// .withConnectTo(privateIp)
+// .withAuthenticator(ALLOW_NOTHING_AUTHENTICATOR);
+// OutboundConnections pool = new OutboundConnections(template, new MockBackPressureStrategy(null).newState(remote));
+// ms.channelManagers.put(remote, pool);
+//
+// Assert.assertEquals(privateIp, ms.getPreferredRemoteAddr(remote));
+// }
+//
+// @Test
+// public void getPreferredRemoteAddrUsesPreferredIp() throws UnknownHostException
+// {
+// MessagingService ms = MessagingService.instance();
+// InetAddressAndPort remote = InetAddressAndPort.getByNameOverrideDefaults("127.0.0.115", 7000);
+//
+// InetAddressAndPort preferredIp = InetAddressAndPort.getByName("127.0.0.16");
+// SystemKeyspace.updatePreferredIP(remote, preferredIp);
+//
+// Assert.assertEquals(preferredIp, ms.getPreferredRemoteAddr(remote));
+// }
+//
+// @Test
+// public void getPreferredRemoteAddrUsesPrivateIpOverridesPreferredIp() throws UnknownHostException
+// {
+// MessagingService ms = MessagingService.instance();
+// InetAddressAndPort local = InetAddressAndPort.getByNameOverrideDefaults("127.0.0.4", 7000);
+// InetAddressAndPort remote = InetAddressAndPort.getByNameOverrideDefaults("127.0.0.105", 7000);
+// InetAddressAndPort privateIp = InetAddressAndPort.getByName("127.0.0.6");
+//
+// OutboundConnectionSettings template = new OutboundConnectionSettings(remote)
+// .withConnectTo(privateIp)
+// .withAuthenticator(ALLOW_NOTHING_AUTHENTICATOR);
+//
+// OutboundConnections pool = new OutboundConnections(template, new MockBackPressureStrategy(null).newState(remote));
+// ms.channelManagers.put(remote, pool);
+//
+// InetAddressAndPort preferredIp = InetAddressAndPort.getByName("127.0.0.16");
+// SystemKeyspace.updatePreferredIP(remote, preferredIp);
+//
+// Assert.assertEquals(privateIp, ms.getPreferredRemoteAddr(remote));
+// }
}
diff --git a/test/unit/org/apache/cassandra/net/MockMessagingService.java b/test/unit/org/apache/cassandra/net/MockMessagingService.java
index 0412759..3749baf 100644
--- a/test/unit/org/apache/cassandra/net/MockMessagingService.java
+++ b/test/unit/org/apache/cassandra/net/MockMessagingService.java
@@ -17,15 +17,16 @@
*/
package org.apache.cassandra.net;
-import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.function.Predicate;
+import org.apache.cassandra.locator.InetAddressAndPort;
+
/**
* Starting point for mocking {@link MessagingService} interactions. Outgoing messages can be
* intercepted by first creating a {@link MatcherResponse} by calling {@link MockMessagingService#when(Matcher)}.
- * Alternatively {@link Matcher}s can be created by using helper methods such as {@link #to(InetAddress)},
- * {@link #verb(MessagingService.Verb)} or {@link #payload(Predicate)} and may also be
+ * Alternatively {@link Matcher}s can be created by using helper methods such as {@link #to(InetAddressAndPort)},
+ * {@link #verb(Verb)} or {@link #payload(Predicate)} and may also be
* nested using {@link MockMessagingService#all(Matcher[])} or {@link MockMessagingService#any(Matcher[])}.
* After each test, {@link MockMessagingService#cleanup()} must be called for free listeners registered
* in {@link MessagingService}.
@@ -46,23 +47,24 @@
}
/**
- * Unsubscribes any handlers added by calling {@link MessagingService#addMessageSink(IMessageSink)}.
+ * Unsubscribes any handlers.
* This should be called after each test.
*/
public static void cleanup()
{
- MessagingService.instance().clearMessageSinks();
+ MessagingService.instance().outboundSink.clear();
+ MessagingService.instance().inboundSink.clear();
}
/**
* Creates a matcher that will indicate if the target address of the outgoing message equals the
* provided address.
*/
- public static Matcher<InetAddress> to(String address)
+ public static Matcher<InetAddressAndPort> to(String address)
{
try
{
- return to(InetAddress.getByName(address));
+ return to(InetAddressAndPort.getByName(address));
}
catch (UnknownHostException e)
{
@@ -74,24 +76,32 @@
* Creates a matcher that will indicate if the target address of the outgoing message equals the
* provided address.
*/
- public static Matcher<InetAddress> to(InetAddress address)
+ public static Matcher<InetAddressAndPort> to(InetAddressAndPort address)
{
return (in, to) -> to == address || to.equals(address);
}
/**
+ * Creates a matcher that will indicate if the target address of the outgoing message matches the provided predicate.
+ */
+ public static Matcher<InetAddressAndPort> to(Predicate<InetAddressAndPort> predicate)
+ {
+ return (in, to) -> predicate.test(to);
+ }
+
+ /**
* Creates a matcher that will indicate if the verb of the outgoing message equals the
* provided value.
*/
- public static Matcher<MessagingService.Verb> verb(MessagingService.Verb verb)
+ public static Matcher<Verb> verb(Verb verb)
{
- return (in, to) -> in.verb == verb;
+ return (in, to) -> in.verb() == verb;
}
/**
* Creates a matcher based on the result of the provided predicate called with the outgoing message.
*/
- public static <T> Matcher<T> message(Predicate<MessageOut<T>> fn)
+ public static <T> Matcher<T> message(Predicate<Message<T>> fn)
{
return (msg, to) -> fn.test(msg);
}
@@ -117,7 +127,7 @@
*/
public static <T> Matcher<?> all(Matcher<?>... matchers)
{
- return (MessageOut<T> out, InetAddress to) -> {
+ return (Message<T> out, InetAddressAndPort to) -> {
for (Matcher matcher : matchers)
{
if (!matcher.matches(out, to))
@@ -132,7 +142,7 @@
*/
public static <T> Matcher<?> any(Matcher<?>... matchers)
{
- return (MessageOut<T> out, InetAddress to) -> {
+ return (Message<T> out, InetAddressAndPort to) -> {
for (Matcher matcher : matchers)
{
if (matcher.matches(out, to))
diff --git a/test/unit/org/apache/cassandra/net/MockMessagingServiceTest.java b/test/unit/org/apache/cassandra/net/MockMessagingServiceTest.java
index 3f6564e..e4787f7 100644
--- a/test/unit/org/apache/cassandra/net/MockMessagingServiceTest.java
+++ b/test/unit/org/apache/cassandra/net/MockMessagingServiceTest.java
@@ -17,7 +17,6 @@
*/
package org.apache.cassandra.net;
-import java.util.Collections;
import java.util.concurrent.ExecutionException;
import org.junit.Before;
@@ -26,14 +25,15 @@
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.gms.EchoMessage;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.FBUtilities;
+import static org.apache.cassandra.net.Verb.ECHO_REQ;
import static org.apache.cassandra.net.MockMessagingService.all;
import static org.apache.cassandra.net.MockMessagingService.to;
import static org.apache.cassandra.net.MockMessagingService.verb;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertSame;
import static org.junit.Assert.assertTrue;
public class MockMessagingServiceTest
@@ -54,40 +54,28 @@
@Test
public void testRequestResponse() throws InterruptedException, ExecutionException
{
- // echo message that we like to mock as incoming reply for outgoing echo message
- MessageIn<EchoMessage> echoMessageIn = MessageIn.create(FBUtilities.getBroadcastAddress(),
- EchoMessage.instance,
- Collections.emptyMap(),
- MessagingService.Verb.ECHO,
- MessagingService.current_version);
+ // echo message that we like to mock as incoming response for outgoing echo message
+ Message<NoPayload> echoMessage = Message.out(ECHO_REQ, NoPayload.noPayload);
MockMessagingSpy spy = MockMessagingService
.when(
all(
- to(FBUtilities.getBroadcastAddress()),
- verb(MessagingService.Verb.ECHO)
+ to(FBUtilities.getBroadcastAddressAndPort()),
+ verb(ECHO_REQ)
)
)
- .respond(echoMessageIn);
+ .respond(echoMessage);
- MessageOut<EchoMessage> echoMessageOut = new MessageOut<>(MessagingService.Verb.ECHO, EchoMessage.instance, EchoMessage.serializer);
- MessagingService.instance().sendRR(echoMessageOut, FBUtilities.getBroadcastAddress(), new IAsyncCallback()
+ Message<NoPayload> echoMessageOut = Message.out(ECHO_REQ, NoPayload.noPayload);
+ MessagingService.instance().sendWithCallback(echoMessageOut, FBUtilities.getBroadcastAddressAndPort(), msg ->
{
- public void response(MessageIn msg)
- {
- assertEquals(MessagingService.Verb.ECHO, msg.verb);
- assertEquals(echoMessageIn.payload, msg.payload);
- }
-
- public boolean isLatencyForSnitch()
- {
- return false;
- }
+ assertEquals(ECHO_REQ, msg.verb());
+ assertEquals(echoMessage.payload, msg.payload);
});
// we must have intercepted the outgoing message at this point
- MessageOut<?> msg = spy.captureMessageOut().get();
+ Message<?> msg = spy.captureMessageOut().get();
assertEquals(1, spy.messagesIntercepted);
- assertTrue(msg == echoMessageOut);
+ assertSame(echoMessage.payload, msg.payload);
// and return a mocked response
assertEquals(1, spy.mockedMessageResponses);
diff --git a/test/unit/org/apache/cassandra/net/MockMessagingSpy.java b/test/unit/org/apache/cassandra/net/MockMessagingSpy.java
index 80bdb39..c61c301 100644
--- a/test/unit/org/apache/cassandra/net/MockMessagingSpy.java
+++ b/test/unit/org/apache/cassandra/net/MockMessagingSpy.java
@@ -28,6 +28,7 @@
import com.google.common.util.concurrent.AbstractFuture;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.MoreExecutors;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -43,25 +44,25 @@
public int messagesIntercepted = 0;
public int mockedMessageResponses = 0;
- private final BlockingQueue<MessageOut<?>> interceptedMessages = new LinkedBlockingQueue<>();
- private final BlockingQueue<MessageIn<?>> deliveredResponses = new LinkedBlockingQueue<>();
+ private final BlockingQueue<Message<?>> interceptedMessages = new LinkedBlockingQueue<>();
+ private final BlockingQueue<Message<?>> deliveredResponses = new LinkedBlockingQueue<>();
private static final Executor executor = Executors.newSingleThreadExecutor();
/**
* Returns a future with the first mocked incoming message that has been created and delivered.
*/
- public ListenableFuture<MessageIn<?>> captureMockedMessageIn()
+ public ListenableFuture<Message<?>> captureMockedMessage()
{
- return Futures.transform(captureMockedMessageInN(1), (List<MessageIn<?>> result) -> result.isEmpty() ? null : result.get(0));
+ return Futures.transform(captureMockedMessageN(1), (List<Message<?>> result) -> result.isEmpty() ? null : result.get(0), MoreExecutors.directExecutor());
}
/**
* Returns a future with the specified number mocked incoming messages that have been created and delivered.
*/
- public ListenableFuture<List<MessageIn<?>>> captureMockedMessageInN(int noOfMessages)
+ public ListenableFuture<List<Message<?>>> captureMockedMessageN(int noOfMessages)
{
- CapturedResultsFuture<MessageIn<?>> ret = new CapturedResultsFuture<>(noOfMessages, deliveredResponses);
+ CapturedResultsFuture<Message<?>> ret = new CapturedResultsFuture<>(noOfMessages, deliveredResponses);
executor.execute(ret);
return ret;
}
@@ -69,17 +70,17 @@
/**
* Returns a future that will indicate if a mocked incoming message has been created and delivered.
*/
- public ListenableFuture<Boolean> expectMockedMessageIn()
+ public ListenableFuture<Boolean> expectMockedMessage()
{
- return expectMockedMessageIn(1);
+ return expectMockedMessage(1);
}
/**
* Returns a future that will indicate if the specified number of mocked incoming message have been created and delivered.
*/
- public ListenableFuture<Boolean> expectMockedMessageIn(int noOfMessages)
+ public ListenableFuture<Boolean> expectMockedMessage(int noOfMessages)
{
- ResultsCompletionFuture<MessageIn<?>> ret = new ResultsCompletionFuture<>(noOfMessages, deliveredResponses);
+ ResultsCompletionFuture<Message<?>> ret = new ResultsCompletionFuture<>(noOfMessages, deliveredResponses);
executor.execute(ret);
return ret;
}
@@ -87,17 +88,17 @@
/**
* Returns a future with the first intercepted outbound message that would have been send.
*/
- public ListenableFuture<MessageOut<?>> captureMessageOut()
+ public ListenableFuture<Message<?>> captureMessageOut()
{
- return Futures.transform(captureMessageOut(1), (List<MessageOut<?>> result) -> result.isEmpty() ? null : result.get(0));
+ return Futures.transform(captureMessageOut(1), (List<Message<?>> result) -> result.isEmpty() ? null : result.get(0), MoreExecutors.directExecutor());
}
/**
* Returns a future with the specified number of intercepted outbound messages that would have been send.
*/
- public ListenableFuture<List<MessageOut<?>>> captureMessageOut(int noOfMessages)
+ public ListenableFuture<List<Message<?>>> captureMessageOut(int noOfMessages)
{
- CapturedResultsFuture<MessageOut<?>> ret = new CapturedResultsFuture<>(noOfMessages, interceptedMessages);
+ CapturedResultsFuture<Message<?>> ret = new CapturedResultsFuture<>(noOfMessages, interceptedMessages);
executor.execute(ret);
return ret;
}
@@ -115,7 +116,7 @@
*/
public ListenableFuture<Boolean> interceptMessageOut(int noOfMessages)
{
- ResultsCompletionFuture<MessageOut<?>> ret = new ResultsCompletionFuture<>(noOfMessages, interceptedMessages);
+ ResultsCompletionFuture<Message<?>> ret = new ResultsCompletionFuture<>(noOfMessages, interceptedMessages);
executor.execute(ret);
return ret;
}
@@ -125,19 +126,19 @@
*/
public ListenableFuture<Boolean> interceptNoMsg(long time, TimeUnit unit)
{
- ResultAbsenceFuture<MessageOut<?>> ret = new ResultAbsenceFuture<>(interceptedMessages, time, unit);
+ ResultAbsenceFuture<Message<?>> ret = new ResultAbsenceFuture<>(interceptedMessages, time, unit);
executor.execute(ret);
return ret;
}
- void matchingMessage(MessageOut<?> message)
+ void matchingMessage(Message<?> message)
{
messagesIntercepted++;
logger.trace("Received matching message: {}", message);
interceptedMessages.add(message);
}
- void matchingResponse(MessageIn<?> response)
+ void matchingResponse(Message<?> response)
{
mockedMessageResponses++;
logger.trace("Responding to intercepted message: {}", response);
@@ -231,4 +232,4 @@
}
}
}
-}
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/net/OutboundConnectionSettingsTest.java b/test/unit/org/apache/cassandra/net/OutboundConnectionSettingsTest.java
new file mode 100644
index 0000000..66773f8
--- /dev/null
+++ b/test/unit/org/apache/cassandra/net/OutboundConnectionSettingsTest.java
@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.net;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.function.Function;
+
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.config.Config;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.commitlog.CommitLog;
+import org.apache.cassandra.locator.AbstractEndpointSnitch;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.Replica;
+
+import static org.apache.cassandra.config.DatabaseDescriptor.getEndpointSnitch;
+import static org.apache.cassandra.net.MessagingService.current_version;
+import static org.apache.cassandra.net.ConnectionType.*;
+import static org.apache.cassandra.net.OutboundConnectionsTest.LOCAL_ADDR;
+import static org.apache.cassandra.net.OutboundConnectionsTest.REMOTE_ADDR;
+
+public class OutboundConnectionSettingsTest
+{
+ @BeforeClass
+ public static void before()
+ {
+ DatabaseDescriptor.daemonInitialization();
+ CommitLog.instance.start();
+ }
+
+ @Test (expected = IllegalArgumentException.class)
+ public void build_SmallSendSize()
+ {
+ test(settings -> settings.withSocketSendBufferSizeInBytes(999));
+ }
+
+ @Test (expected = IllegalArgumentException.class)
+ public void build_SendSizeLessThanZero()
+ {
+ test(settings -> settings.withSocketSendBufferSizeInBytes(-1));
+ }
+
+ @Test (expected = IllegalArgumentException.class)
+ public void build_TcpConnectTimeoutLessThanZero()
+ {
+ test(settings -> settings.withTcpConnectTimeoutInMS(-1));
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void build_TcpUserTimeoutLessThanZero()
+ {
+ test(settings -> settings.withTcpUserTimeoutInMS(-1));
+ }
+
+ @Test
+ public void build_TcpUserTimeoutEqualsZero()
+ {
+ test(settings -> settings.withTcpUserTimeoutInMS(0));
+ }
+
+ private static void test(Function<OutboundConnectionSettings, OutboundConnectionSettings> f)
+ {
+ f.apply(new OutboundConnectionSettings(LOCAL_ADDR)).withDefaults(ConnectionCategory.MESSAGING);
+ }
+
+ private static class TestSnitch extends AbstractEndpointSnitch
+ {
+ private final Map<InetAddressAndPort, String> nodeToDc = new HashMap<>();
+
+ void add(InetAddressAndPort node, String dc)
+ {
+ nodeToDc.put(node, dc);
+ }
+
+ public String getRack(InetAddressAndPort endpoint)
+ {
+ return null;
+ }
+
+ public String getDatacenter(InetAddressAndPort endpoint)
+ {
+ return nodeToDc.get(endpoint);
+ }
+
+ public int compareEndpoints(InetAddressAndPort target, Replica a1, Replica a2)
+ {
+ return 0;
+ }
+ }
+
+ @Test
+ public void shouldCompressConnection_None()
+ {
+ DatabaseDescriptor.setInternodeCompression(Config.InternodeCompression.none);
+ Assert.assertFalse(OutboundConnectionSettings.shouldCompressConnection(getEndpointSnitch(), LOCAL_ADDR, REMOTE_ADDR));
+ }
+
+ @Test
+ public void shouldCompressConnection_DifferentDc()
+ {
+ TestSnitch snitch = new TestSnitch();
+ snitch.add(LOCAL_ADDR, "dc1");
+ snitch.add(REMOTE_ADDR, "dc2");
+ DatabaseDescriptor.setEndpointSnitch(snitch);
+ DatabaseDescriptor.setInternodeCompression(Config.InternodeCompression.dc);
+ Assert.assertTrue(OutboundConnectionSettings.shouldCompressConnection(getEndpointSnitch(), LOCAL_ADDR, REMOTE_ADDR));
+ }
+
+ @Test
+ public void shouldCompressConnection_All()
+ {
+ DatabaseDescriptor.setInternodeCompression(Config.InternodeCompression.all);
+ Assert.assertTrue(OutboundConnectionSettings.shouldCompressConnection(getEndpointSnitch(), LOCAL_ADDR, REMOTE_ADDR));
+ }
+
+ @Test
+ public void shouldCompressConnection_SameDc()
+ {
+ TestSnitch snitch = new TestSnitch();
+ snitch.add(LOCAL_ADDR, "dc1");
+ snitch.add(REMOTE_ADDR, "dc1");
+ DatabaseDescriptor.setEndpointSnitch(snitch);
+ DatabaseDescriptor.setInternodeCompression(Config.InternodeCompression.dc);
+ Assert.assertFalse(OutboundConnectionSettings.shouldCompressConnection(getEndpointSnitch(), LOCAL_ADDR, REMOTE_ADDR));
+ }
+
+}
diff --git a/test/unit/org/apache/cassandra/net/OutboundConnectionsTest.java b/test/unit/org/apache/cassandra/net/OutboundConnectionsTest.java
new file mode 100644
index 0000000..82543e1
--- /dev/null
+++ b/test/unit/org/apache/cassandra/net/OutboundConnectionsTest.java
@@ -0,0 +1,164 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.net;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.net.InetAddresses;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.commitlog.CommitLog;
+import org.apache.cassandra.gms.GossipDigestSyn;
+import org.apache.cassandra.io.IVersionedSerializer;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.net.BackPressureState;
+import org.apache.cassandra.net.ConnectionType;
+import org.apache.cassandra.net.Message;
+import org.apache.cassandra.net.OutboundConnectionSettings;
+import org.apache.cassandra.net.OutboundConnections;
+import org.apache.cassandra.net.PingRequest;
+import org.apache.cassandra.net.Verb;
+
+public class OutboundConnectionsTest
+{
+ static final InetAddressAndPort LOCAL_ADDR = InetAddressAndPort.getByAddressOverrideDefaults(InetAddresses.forString("127.0.0.1"), 9476);
+ static final InetAddressAndPort REMOTE_ADDR = InetAddressAndPort.getByAddressOverrideDefaults(InetAddresses.forString("127.0.0.2"), 9476);
+ private static final InetAddressAndPort RECONNECT_ADDR = InetAddressAndPort.getByAddressOverrideDefaults(InetAddresses.forString("127.0.0.3"), 9476);
+ private static final List<ConnectionType> INTERNODE_MESSAGING_CONN_TYPES = ImmutableList.of(ConnectionType.URGENT_MESSAGES, ConnectionType.LARGE_MESSAGES, ConnectionType.SMALL_MESSAGES);
+
+ private OutboundConnections connections;
+
+ @BeforeClass
+ public static void before()
+ {
+ DatabaseDescriptor.daemonInitialization();
+ CommitLog.instance.start();
+ }
+
+ @Before
+ public void setup()
+ {
+ BackPressureState backPressureState = DatabaseDescriptor.getBackPressureStrategy().newState(REMOTE_ADDR);
+ connections = OutboundConnections.unsafeCreate(new OutboundConnectionSettings(REMOTE_ADDR), backPressureState);
+ }
+
+ @After
+ public void tearDown() throws ExecutionException, InterruptedException, TimeoutException
+ {
+ if (connections != null)
+ connections.close(false).get(10L, TimeUnit.SECONDS);
+ }
+
+ @Test
+ public void getConnection_Gossip()
+ {
+ GossipDigestSyn syn = new GossipDigestSyn("cluster", "partitioner", new ArrayList<>(0));
+ Message<GossipDigestSyn> message = Message.out(Verb.GOSSIP_DIGEST_SYN, syn);
+ Assert.assertEquals(ConnectionType.URGENT_MESSAGES, connections.connectionFor(message).type());
+ }
+
+ @Test
+ public void getConnection_SmallMessage()
+ {
+ Message message = Message.out(Verb.PING_REQ, PingRequest.forSmall);
+ Assert.assertEquals(ConnectionType.SMALL_MESSAGES, connections.connectionFor(message).type());
+ }
+
+ @Test
+ public void getConnection_LargeMessage() throws NoSuchFieldException, IllegalAccessException
+ {
+ // just need a serializer to report a size, as fake as it may be
+ IVersionedSerializer<Object> serializer = new IVersionedSerializer<Object>()
+ {
+ public void serialize(Object o, DataOutputPlus out, int version)
+ {
+
+ }
+
+ public Object deserialize(DataInputPlus in, int version)
+ {
+ return null;
+ }
+
+ public long serializedSize(Object o, int version)
+ {
+ return OutboundConnections.LARGE_MESSAGE_THRESHOLD + 1;
+ }
+ };
+ Verb._TEST_2.unsafeSetSerializer(() -> serializer);
+ Message message = Message.out(Verb._TEST_2, "payload");
+ Assert.assertEquals(ConnectionType.LARGE_MESSAGES, connections.connectionFor(message).type());
+ }
+
+ @Test
+ public void close_SoftClose() throws ExecutionException, InterruptedException, TimeoutException
+ {
+ for (ConnectionType type : INTERNODE_MESSAGING_CONN_TYPES)
+ Assert.assertFalse(connections.connectionFor(type).isClosed());
+ connections.close(true).get(10L, TimeUnit.SECONDS);
+ for (ConnectionType type : INTERNODE_MESSAGING_CONN_TYPES)
+ Assert.assertTrue(connections.connectionFor(type).isClosed());
+ }
+
+ @Test
+ public void close_NotSoftClose() throws ExecutionException, InterruptedException, TimeoutException
+ {
+ for (ConnectionType type : INTERNODE_MESSAGING_CONN_TYPES)
+ Assert.assertFalse(connections.connectionFor(type).isClosed());
+ connections.close(false).get(10L, TimeUnit.SECONDS);
+ for (ConnectionType type : INTERNODE_MESSAGING_CONN_TYPES)
+ Assert.assertTrue(connections.connectionFor(type).isClosed());
+ }
+
+ @Test
+ public void reconnectWithNewIp() throws InterruptedException
+ {
+ for (ConnectionType type : INTERNODE_MESSAGING_CONN_TYPES)
+ {
+ Assert.assertEquals(REMOTE_ADDR, connections.connectionFor(type).settings().connectTo);
+ }
+
+ connections.reconnectWithNewIp(RECONNECT_ADDR).await();
+
+ for (ConnectionType type : INTERNODE_MESSAGING_CONN_TYPES)
+ {
+ Assert.assertEquals(RECONNECT_ADDR, connections.connectionFor(type).settings().connectTo);
+ }
+ }
+
+// @Test
+// public void timeoutCounter()
+// {
+// long originalValue = connections.getTimeouts();
+// connections.incrementTimeout();
+// Assert.assertEquals(originalValue + 1, connections.getTimeouts());
+// }
+}
diff --git a/test/unit/org/apache/cassandra/net/OutboundMessageQueueTest.java b/test/unit/org/apache/cassandra/net/OutboundMessageQueueTest.java
new file mode 100644
index 0000000..860e4f1
--- /dev/null
+++ b/test/unit/org/apache/cassandra/net/OutboundMessageQueueTest.java
@@ -0,0 +1,221 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.net;
+
+import java.util.LinkedList;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+import com.google.common.util.concurrent.Uninterruptibles;
+
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.utils.FreeRunningClock;
+
+import static org.apache.cassandra.net.NoPayload.noPayload;
+import static org.apache.cassandra.utils.MonotonicClock.approxTime;
+
+// TODO: incomplete
+public class OutboundMessageQueueTest
+{
+ @BeforeClass
+ public static void init()
+ {
+ DatabaseDescriptor.daemonInitialization();
+ }
+
+ @Test
+ public void testRemove() throws InterruptedException
+ {
+ final Message<?> m1 = Message.out(Verb._TEST_1, noPayload);
+ final Message<?> m2 = Message.out(Verb._TEST_1, noPayload);
+ final Message<?> m3 = Message.out(Verb._TEST_1, noPayload);
+
+ final OutboundMessageQueue queue = new OutboundMessageQueue(approxTime, message -> true);
+ queue.add(m1);
+ queue.add(m2);
+ queue.add(m3);
+
+ Assert.assertTrue(queue.remove(m1));
+ Assert.assertFalse(queue.remove(m1));
+
+ CountDownLatch locked = new CountDownLatch(1);
+ CountDownLatch lockUntil = new CountDownLatch(1);
+ new Thread(() -> {
+ try (OutboundMessageQueue.WithLock lock = queue.lockOrCallback(0, () -> {}))
+ {
+ locked.countDown();
+ Uninterruptibles.awaitUninterruptibly(lockUntil);
+ }
+ }).start();
+ Uninterruptibles.awaitUninterruptibly(locked);
+
+ CountDownLatch start = new CountDownLatch(2);
+ CountDownLatch finish = new CountDownLatch(2);
+ new Thread(() -> {
+ start.countDown();
+ Assert.assertTrue(queue.remove(m2));
+ finish.countDown();
+ }).start();
+ new Thread(() -> {
+ start.countDown();
+ Assert.assertTrue(queue.remove(m3));
+ finish.countDown();
+ }).start();
+ Uninterruptibles.awaitUninterruptibly(start);
+ lockUntil.countDown();
+ Uninterruptibles.awaitUninterruptibly(finish);
+
+ try (OutboundMessageQueue.WithLock lock = queue.lockOrCallback(0, () -> {}))
+ {
+ Assert.assertNull(lock.peek());
+ }
+ }
+
+ @Test
+ public void testExpirationOnIteration()
+ {
+ FreeRunningClock clock = new FreeRunningClock();
+
+ List<Message> expiredMessages = new LinkedList<>();
+ long startTime = clock.now();
+
+ Message<?> m1 = Message.out(Verb._TEST_1, noPayload, startTime + TimeUnit.SECONDS.toNanos(7));
+ Message<?> m2 = Message.out(Verb._TEST_1, noPayload, startTime + TimeUnit.SECONDS.toNanos(3));
+ Message<?> m3;
+ Message<?> m4;
+
+ OutboundMessageQueue queue = new OutboundMessageQueue(clock, m -> expiredMessages.add(m));
+ queue.add(m1);
+ queue.add(m2);
+
+ try(OutboundMessageQueue.WithLock l = queue.lockOrCallback(clock.now(), () -> {}))
+ {
+ // Do nothing
+ }
+ // Check next expiry time is equal to m2, and we haven't expired anything yet:
+ Assert.assertEquals(3, queue.nextExpirationIn(startTime, TimeUnit.SECONDS));
+ Assert.assertTrue(expiredMessages.isEmpty());
+
+ // Wait for m2 expiry time:
+ clock.advance(4, TimeUnit.SECONDS);
+
+ try(OutboundMessageQueue.WithLock l = queue.lockOrCallback(clock.now(), () -> {}))
+ {
+ // Add a new message while we're iterating the queue: this will expire later than any existing message.
+ m3 = Message.out(Verb._TEST_1, noPayload, startTime + TimeUnit.SECONDS.toNanos(60));
+ queue.add(m3);
+ }
+ // After expiration runs following the WithLock#close(), check the expiration time is updated to m1 (not m3):
+ Assert.assertEquals(7, queue.nextExpirationIn(startTime, TimeUnit.SECONDS));
+ // Also, m2 was expired and collected:
+ Assert.assertEquals(m2, expiredMessages.remove(0));
+
+ // Wait for m1 expiry time:
+ clock.advance(4, TimeUnit.SECONDS);
+
+ try(OutboundMessageQueue.WithLock l = queue.lockOrCallback(clock.now(), () -> {}))
+ {
+ // Add a new message while we're iterating the queue: this will expire sooner than the already existing message.
+ m4 = Message.out(Verb._TEST_1, noPayload, startTime + TimeUnit.SECONDS.toNanos(10));
+ queue.add(m4);
+ }
+ // Check m1 was expired and collected:
+ Assert.assertEquals(m1, expiredMessages.remove(0));
+ // Check next expiry time is m4 (not m3):
+ Assert.assertEquals(10, queue.nextExpirationIn(startTime, TimeUnit.SECONDS));
+
+ // Consume all messages before expiration:
+ try(OutboundMessageQueue.WithLock l = queue.lockOrCallback(clock.now(), () -> {}))
+ {
+ Assert.assertEquals(m3, l.poll());
+ Assert.assertEquals(m4, l.poll());
+ }
+ // Check next expiry time is still m4 as the deadline hasn't passed yet:
+ Assert.assertEquals(10, queue.nextExpirationIn(startTime, TimeUnit.SECONDS));
+
+ // Go past the deadline:
+ clock.advance(4, TimeUnit.SECONDS);
+
+ try(OutboundMessageQueue.WithLock l = queue.lockOrCallback(clock.now(), () -> {}))
+ {
+ // Do nothing, just trigger expiration on close
+ }
+ // Check nothing is expired:
+ Assert.assertTrue(expiredMessages.isEmpty());
+ // Check next expiry time is now Long.MAX_VALUE as nothing was in the queue:
+ Assert.assertEquals(Long.MAX_VALUE, queue.nextExpirationIn(0, TimeUnit.NANOSECONDS));
+ }
+
+ @Test
+ public void testExpirationOnAdd()
+ {
+ FreeRunningClock clock = new FreeRunningClock();
+
+ List<Message> expiredMessages = new LinkedList<>();
+ long startTime = clock.now();
+
+ OutboundMessageQueue queue = new OutboundMessageQueue(clock, m -> expiredMessages.add(m));
+
+ Message<?> m1 = Message.out(Verb._TEST_1, noPayload, startTime + TimeUnit.SECONDS.toNanos(7));
+ Message<?> m2 = Message.out(Verb._TEST_1, noPayload, startTime + TimeUnit.SECONDS.toNanos(3));
+ queue.add(m1);
+ queue.add(m2);
+
+ // Check next expiry time is equal to m2, and we haven't expired anything yet:
+ Assert.assertEquals(3, queue.nextExpirationIn(startTime, TimeUnit.SECONDS));
+ Assert.assertTrue(expiredMessages.isEmpty());
+
+ // Go past m1 expiry time:
+ clock.advance(8, TimeUnit.SECONDS);
+
+ // Add a new message and verify both m1 and m2 have been expired:
+ Message<?> m3 = Message.out(Verb._TEST_1, noPayload, startTime + TimeUnit.SECONDS.toNanos(10));
+ queue.add(m3);
+ Assert.assertEquals(m2, expiredMessages.remove(0));
+ Assert.assertEquals(m1, expiredMessages.remove(0));
+
+ // New expiration deadline is m3:
+ Assert.assertEquals(10, queue.nextExpirationIn(startTime, TimeUnit.SECONDS));
+
+ // Go past m3 expiry time:
+ clock.advance(4, TimeUnit.SECONDS);
+
+ try(OutboundMessageQueue.WithLock l = queue.lockOrCallback(clock.now(), () -> {}))
+ {
+ // Add a new message and verify nothing is expired because the lock is held by this iteration:
+ Message<?> m4 = Message.out(Verb._TEST_1, noPayload, startTime + TimeUnit.SECONDS.toNanos(15));
+ queue.add(m4);
+ Assert.assertTrue(expiredMessages.isEmpty());
+
+ // Also the deadline didn't change, even though we're past the m3 expiry time: this way we're sure the
+ // pruner will run promptly even if falling behind during iteration.
+ Assert.assertEquals(10, queue.nextExpirationIn(startTime, TimeUnit.SECONDS));
+ }
+
+ // Check post iteration m3 has expired:
+ Assert.assertEquals(m3, expiredMessages.remove(0));
+ // And deadline is now m4:
+ Assert.assertEquals(15, queue.nextExpirationIn(startTime, TimeUnit.SECONDS));
+ }
+}
diff --git a/test/unit/org/apache/cassandra/net/OutboundTcpConnectionTest.java b/test/unit/org/apache/cassandra/net/OutboundTcpConnectionTest.java
deleted file mode 100644
index e3b6817..0000000
--- a/test/unit/org/apache/cassandra/net/OutboundTcpConnectionTest.java
+++ /dev/null
@@ -1,175 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.net;
-
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.net.MessagingService.Verb;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
-import java.net.InetAddress;
-import java.net.UnknownHostException;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
-
-/**
- * The tests check whether Queue expiration in the OutboundTcpConnection behaves properly for droppable and
- * non-droppable messages.
- */
-public class OutboundTcpConnectionTest
-{
- AtomicInteger messageId = new AtomicInteger(0);
-
- final static Verb VERB_DROPPABLE = Verb.MUTATION; // Droppable, 2s timeout
- final static Verb VERB_NONDROPPABLE = Verb.GOSSIP_DIGEST_ACK; // Not droppable
-
- final static long NANOS_FOR_TIMEOUT;
-
- static
- {
- DatabaseDescriptor.daemonInitialization();
- NANOS_FOR_TIMEOUT = TimeUnit.MILLISECONDS.toNanos(VERB_DROPPABLE.getTimeout()*2);
- }
-
- /**
- * Verifies our assumptions whether a Verb can be dropped or not. The tests make use of droppabilty, and
- * may produce wrong test results if their droppabilty is changed.
- */
- @BeforeClass
- public static void assertDroppability()
- {
- if (!MessagingService.DROPPABLE_VERBS.contains(VERB_DROPPABLE))
- throw new AssertionError("Expected " + VERB_DROPPABLE + " to be droppable");
- if (MessagingService.DROPPABLE_VERBS.contains(VERB_NONDROPPABLE))
- throw new AssertionError("Expected " + VERB_NONDROPPABLE + " not to be droppable");
- }
-
- /**
- * Tests that non-droppable messages are never expired
- */
- @Test
- public void testNondroppable() throws UnknownHostException
- {
- OutboundTcpConnection otc = getOutboundTcpConnectionForLocalhost();
- long nanoTimeBeforeEnqueue = System.nanoTime();
-
- assertFalse("Fresh OutboundTcpConnection contains expired messages",
- otc.backlogContainsExpiredMessages(nanoTimeBeforeEnqueue));
-
- fillToPurgeSize(otc, VERB_NONDROPPABLE);
- fillToPurgeSize(otc, VERB_NONDROPPABLE);
- otc.expireMessages(expirationTimeNanos());
-
- assertFalse("OutboundTcpConnection with non-droppable verbs should not expire",
- otc.backlogContainsExpiredMessages(expirationTimeNanos()));
- }
-
- /**
- * Tests that droppable messages will be dropped after they expire, but not before.
- *
- * @throws UnknownHostException
- */
- @Test
- public void testDroppable() throws UnknownHostException
- {
- OutboundTcpConnection otc = getOutboundTcpConnectionForLocalhost();
- long nanoTimeBeforeEnqueue = System.nanoTime();
-
- initialFill(otc, VERB_DROPPABLE);
- assertFalse("OutboundTcpConnection with droppable verbs should not expire immediately",
- otc.backlogContainsExpiredMessages(nanoTimeBeforeEnqueue));
-
- otc.expireMessages(nanoTimeBeforeEnqueue);
- assertFalse("OutboundTcpConnection with droppable verbs should not expire with enqueue-time expiration",
- otc.backlogContainsExpiredMessages(nanoTimeBeforeEnqueue));
-
- // Lets presume, expiration time have passed => At that time there shall be expired messages in the Queue
- long nanoTimeWhenExpired = expirationTimeNanos();
- assertTrue("OutboundTcpConnection with droppable verbs should have expired",
- otc.backlogContainsExpiredMessages(nanoTimeWhenExpired));
-
- // Using the same timestamp, lets expire them and check whether they have gone
- otc.expireMessages(nanoTimeWhenExpired);
- assertFalse("OutboundTcpConnection should not have expired entries",
- otc.backlogContainsExpiredMessages(nanoTimeWhenExpired));
-
- // Actually the previous test can be done in a harder way: As expireMessages() has run, we cannot have
- // ANY expired values, thus lets test also against nanoTimeBeforeEnqueue
- assertFalse("OutboundTcpConnection should not have any expired entries",
- otc.backlogContainsExpiredMessages(nanoTimeBeforeEnqueue));
-
- }
-
- /**
- * Fills the given OutboundTcpConnection with (1 + BACKLOG_PURGE_SIZE), elements. The first
- * BACKLOG_PURGE_SIZE elements are non-droppable, the last one is a message with the given Verb and can be
- * droppable or non-droppable.
- */
- private void initialFill(OutboundTcpConnection otc, Verb verb)
- {
- assertFalse("Fresh OutboundTcpConnection contains expired messages",
- otc.backlogContainsExpiredMessages(System.nanoTime()));
-
- fillToPurgeSize(otc, VERB_NONDROPPABLE);
- MessageOut<?> messageDroppable10s = new MessageOut<>(verb);
- otc.enqueue(messageDroppable10s, nextMessageId());
- otc.expireMessages(System.nanoTime());
- }
-
- /**
- * Returns a nano timestamp in the far future, when expiration should have been performed for VERB_DROPPABLE.
- * The offset is chosen as 2 times of the expiration time of VERB_DROPPABLE.
- *
- * @return The future nano timestamp
- */
- private long expirationTimeNanos()
- {
- return System.nanoTime() + NANOS_FOR_TIMEOUT;
- }
-
- private int nextMessageId()
- {
- return messageId.incrementAndGet();
- }
-
- /**
- * Adds BACKLOG_PURGE_SIZE messages to the queue. Hint: At BACKLOG_PURGE_SIZE expiration starts to work.
- *
- * @param otc
- * The OutboundTcpConnection
- * @param verb
- * The verb that defines the message type
- */
- private void fillToPurgeSize(OutboundTcpConnection otc, Verb verb)
- {
- for (int i = 0; i < OutboundTcpConnection.BACKLOG_PURGE_SIZE; i++)
- {
- otc.enqueue(new MessageOut<>(verb), nextMessageId());
- }
- }
-
- private OutboundTcpConnection getOutboundTcpConnectionForLocalhost() throws UnknownHostException
- {
- InetAddress lo = InetAddress.getByName("127.0.0.1");
- OutboundTcpConnectionPool otcPool = new OutboundTcpConnectionPool(lo, null);
- OutboundTcpConnection otc = new OutboundTcpConnection(otcPool, "lo-OutboundTcpConnectionTest");
- return otc;
- }
-}
diff --git a/test/unit/org/apache/cassandra/net/ProxyHandlerConnectionsTest.java b/test/unit/org/apache/cassandra/net/ProxyHandlerConnectionsTest.java
new file mode 100644
index 0000000..270a910
--- /dev/null
+++ b/test/unit/org/apache/cassandra/net/ProxyHandlerConnectionsTest.java
@@ -0,0 +1,405 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.net;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Supplier;
+import java.util.function.ToLongFunction;
+
+import com.google.common.util.concurrent.Uninterruptibles;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import io.netty.buffer.ByteBuf;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.io.IVersionedAsymmetricSerializer;
+import org.apache.cassandra.io.IVersionedSerializer;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.net.proxy.InboundProxyHandler;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.Pair;
+
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+import static java.util.concurrent.TimeUnit.SECONDS;
+import static org.apache.cassandra.net.ConnectionTest.SETTINGS;
+import static org.apache.cassandra.net.OutboundConnectionSettings.Framing.CRC;
+import static org.apache.cassandra.utils.MonotonicClock.approxTime;
+
+public class ProxyHandlerConnectionsTest
+{
+ private static final SocketFactory factory = new SocketFactory();
+
+ private final Map<Verb, Supplier<? extends IVersionedAsymmetricSerializer<?, ?>>> serializers = new HashMap<>();
+ private final Map<Verb, Supplier<? extends IVerbHandler<?>>> handlers = new HashMap<>();
+ private final Map<Verb, ToLongFunction<TimeUnit>> timeouts = new HashMap<>();
+
+ private void unsafeSetSerializer(Verb verb, Supplier<? extends IVersionedAsymmetricSerializer<?, ?>> supplier) throws Throwable
+ {
+ serializers.putIfAbsent(verb, verb.unsafeSetSerializer(supplier));
+ }
+
+ protected void unsafeSetHandler(Verb verb, Supplier<? extends IVerbHandler<?>> supplier) throws Throwable
+ {
+ handlers.putIfAbsent(verb, verb.unsafeSetHandler(supplier));
+ }
+
+ private void unsafeSetExpiration(Verb verb, ToLongFunction<TimeUnit> expiration) throws Throwable
+ {
+ timeouts.putIfAbsent(verb, verb.unsafeSetExpiration(expiration));
+ }
+
+ @BeforeClass
+ public static void startup()
+ {
+ DatabaseDescriptor.daemonInitialization();
+ }
+
+ @After
+ public void cleanup() throws Throwable
+ {
+ for (Map.Entry<Verb, Supplier<? extends IVersionedAsymmetricSerializer<?, ?>>> e : serializers.entrySet())
+ e.getKey().unsafeSetSerializer(e.getValue());
+ serializers.clear();
+ for (Map.Entry<Verb, Supplier<? extends IVerbHandler<?>>> e : handlers.entrySet())
+ e.getKey().unsafeSetHandler(e.getValue());
+ handlers.clear();
+ for (Map.Entry<Verb, ToLongFunction<TimeUnit>> e : timeouts.entrySet())
+ e.getKey().unsafeSetExpiration(e.getValue());
+ timeouts.clear();
+ }
+
+ @Test
+ public void testExpireInbound() throws Throwable
+ {
+ DatabaseDescriptor.setCrossNodeTimeout(true);
+ testOneManual((settings, inbound, outbound, endpoint, handler) -> {
+ unsafeSetSerializer(Verb._TEST_1, FakePayloadSerializer::new);
+
+ CountDownLatch connectionLatch = new CountDownLatch(1);
+ unsafeSetHandler(Verb._TEST_1, () -> v -> {
+ connectionLatch.countDown();
+ });
+ outbound.enqueue(Message.out(Verb._TEST_1, 1L));
+ connectionLatch.await(10, SECONDS);
+ Assert.assertEquals(0, connectionLatch.getCount());
+
+ // Slow things down
+ unsafeSetExpiration(Verb._TEST_1, unit -> unit.convert(50, MILLISECONDS));
+ handler.withLatency(100, MILLISECONDS);
+
+ unsafeSetHandler(Verb._TEST_1, () -> v -> {
+ throw new RuntimeException("Should have not been triggered " + v);
+ });
+ int expireMessages = 10;
+ for (int i = 0; i < expireMessages; i++)
+ outbound.enqueue(Message.out(Verb._TEST_1, 1L));
+
+ InboundMessageHandlers handlers = MessagingService.instance().getInbound(endpoint);
+ waitForCondition(() -> handlers.expiredCount() == expireMessages);
+ Assert.assertEquals(expireMessages, handlers.expiredCount());
+ });
+ }
+
+ @Test
+ public void testExpireSome() throws Throwable
+ {
+ DatabaseDescriptor.setCrossNodeTimeout(true);
+ testOneManual((settings, inbound, outbound, endpoint, handler) -> {
+ unsafeSetSerializer(Verb._TEST_1, FakePayloadSerializer::new);
+ connect(outbound);
+
+ AtomicInteger counter = new AtomicInteger();
+ unsafeSetHandler(Verb._TEST_1, () -> v -> {
+ counter.incrementAndGet();
+ });
+
+ int expireMessages = 10;
+ for (int i = 0; i < expireMessages; i++)
+ outbound.enqueue(Message.out(Verb._TEST_1, 1L));
+ waitForCondition(() -> counter.get() == 10);
+
+ unsafeSetExpiration(Verb._TEST_1, unit -> unit.convert(50, MILLISECONDS));
+ handler.withLatency(100, MILLISECONDS);
+
+ InboundMessageHandlers handlers = MessagingService.instance().getInbound(endpoint);
+ for (int i = 0; i < expireMessages; i++)
+ outbound.enqueue(Message.out(Verb._TEST_1, 1L));
+ waitForCondition(() -> handlers.expiredCount() == 10);
+
+ handler.withLatency(2, MILLISECONDS);
+
+ for (int i = 0; i < expireMessages; i++)
+ outbound.enqueue(Message.out(Verb._TEST_1, 1L));
+ waitForCondition(() -> counter.get() == 20);
+ });
+ }
+
+ @Test
+ public void testExpireSomeFromBatch() throws Throwable
+ {
+ DatabaseDescriptor.setCrossNodeTimeout(true);
+ testManual((settings, inbound, outbound, endpoint, handler) -> {
+ unsafeSetSerializer(Verb._TEST_1, FakePayloadSerializer::new);
+ connect(outbound);
+
+ Message msg = Message.out(Verb._TEST_1, 1L);
+ int messageSize = msg.serializedSize(MessagingService.current_version);
+ DatabaseDescriptor.setInternodeMaxMessageSizeInBytes(messageSize * 40);
+
+ AtomicInteger counter = new AtomicInteger();
+ unsafeSetHandler(Verb._TEST_1, () -> v -> {
+ counter.incrementAndGet();
+ });
+
+ unsafeSetExpiration(Verb._TEST_1, unit -> unit.convert(200, MILLISECONDS));
+ handler.withLatency(100, MILLISECONDS);
+
+ int expireMessages = 20;
+ long nanoTime = approxTime.now();
+ CountDownLatch enqueueDone = new CountDownLatch(1);
+ outbound.unsafeRunOnDelivery(() -> Uninterruptibles.awaitUninterruptibly(enqueueDone, 10, SECONDS));
+ for (int i = 0; i < expireMessages; i++)
+ {
+ boolean expire = i % 2 == 0;
+ Message.Builder builder = Message.builder(Verb._TEST_1, 1L);
+
+ if (settings.right.acceptVersions == ConnectionTest.legacy)
+ {
+ // backdate messages; leave 50 milliseconds to leave outbound path
+ builder.withCreatedAt(nanoTime - (expire ? 0 : MILLISECONDS.toNanos(150)));
+ }
+ else
+ {
+ // Give messages 50 milliseconds to leave outbound path
+ builder.withCreatedAt(nanoTime)
+ .withExpiresAt(nanoTime + (expire ? MILLISECONDS.toNanos(50) : MILLISECONDS.toNanos(1000)));
+ }
+ outbound.enqueue(builder.build());
+ }
+ enqueueDone.countDown();
+
+ InboundMessageHandlers handlers = MessagingService.instance().getInbound(endpoint);
+ waitForCondition(() -> handlers.expiredCount() == 10 && counter.get() == 10,
+ () -> String.format("Expired: %d, Arrived: %d", handlers.expiredCount(), counter.get()));
+ });
+ }
+
+ @Test
+ public void suddenDisconnect() throws Throwable
+ {
+ testManual((settings, inbound, outbound, endpoint, handler) -> {
+ handler.onDisconnect(() -> handler.reset());
+
+ unsafeSetSerializer(Verb._TEST_1, FakePayloadSerializer::new);
+ connect(outbound);
+
+ CountDownLatch closeLatch = new CountDownLatch(1);
+ handler.withCloseAfterRead(closeLatch::countDown);
+ AtomicInteger counter = new AtomicInteger();
+ unsafeSetHandler(Verb._TEST_1, () -> v -> counter.incrementAndGet());
+
+ outbound.enqueue(Message.out(Verb._TEST_1, 1L));
+ waitForCondition(() -> !outbound.isConnected());
+
+ connect(outbound);
+ Assert.assertTrue(outbound.isConnected());
+ Assert.assertEquals(0, counter.get());
+ });
+ }
+
+ @Test
+ public void testCorruptionOnHandshake() throws Throwable
+ {
+ testManual((settings, inbound, outbound, endpoint, handler) -> {
+ unsafeSetSerializer(Verb._TEST_1, FakePayloadSerializer::new);
+ // Invalid CRC
+ handler.withPayloadTransform(msg -> {
+ ByteBuf bb = (ByteBuf) msg;
+ bb.setByte(bb.readableBytes() / 2, 0xffff);
+ return msg;
+ });
+ tryConnect(outbound, 1, SECONDS, false);
+ Assert.assertTrue(!outbound.isConnected());
+
+ // Invalid protocol magic
+ handler.withPayloadTransform(msg -> {
+ ByteBuf bb = (ByteBuf) msg;
+ bb.setByte(0, 0xffff);
+ return msg;
+ });
+ tryConnect(outbound, 1, SECONDS, false);
+ Assert.assertTrue(!outbound.isConnected());
+ if (settings.right.framing == CRC)
+ {
+ Assert.assertEquals(2, outbound.connectionAttempts());
+ Assert.assertEquals(0, outbound.successfulConnections());
+ }
+ });
+ }
+
+ private static void waitForCondition(Supplier<Boolean> cond) throws Throwable
+ {
+ CompletableFuture.runAsync(() -> {
+ while (!cond.get()) {}
+ }).get(10, SECONDS);
+ }
+
+ private static void waitForCondition(Supplier<Boolean> cond, Supplier<String> s) throws Throwable
+ {
+ try
+ {
+ CompletableFuture.runAsync(() -> {
+ while (!cond.get()) {}
+ }).get(10, SECONDS);
+ }
+ catch (TimeoutException e)
+ {
+ throw new AssertionError(s.get());
+ }
+ }
+
+ private static class FakePayloadSerializer implements IVersionedSerializer<Long>
+ {
+ private final int size;
+ private FakePayloadSerializer()
+ {
+ this(1);
+ }
+
+ // Takes long and repeats it size times
+ private FakePayloadSerializer(int size)
+ {
+ this.size = size;
+ }
+
+ public void serialize(Long i, DataOutputPlus out, int version) throws IOException
+ {
+ for (int j = 0; j < size; j++)
+ {
+ out.writeLong(i);
+ }
+ }
+
+ public Long deserialize(DataInputPlus in, int version) throws IOException
+ {
+ long l = in.readLong();
+ for (int i = 0; i < size - 1; i++)
+ {
+ if (in.readLong() != l)
+ throw new AssertionError();
+ }
+
+ return l;
+ }
+
+ public long serializedSize(Long t, int version)
+ {
+ return Long.BYTES * size;
+ }
+ }
+ interface ManualSendTest
+ {
+ void accept(Pair<InboundConnectionSettings, OutboundConnectionSettings> settings, InboundSockets inbound, OutboundConnection outbound, InetAddressAndPort endpoint, InboundProxyHandler.Controller handler) throws Throwable;
+ }
+
+ private void testManual(ManualSendTest test) throws Throwable
+ {
+ for (ConnectionTest.Settings s: SETTINGS)
+ {
+ doTestManual(s, test);
+ cleanup();
+ }
+ }
+
+ private void testOneManual(ManualSendTest test) throws Throwable
+ {
+ testOneManual(test, 1);
+ }
+
+ private void testOneManual(ManualSendTest test, int i) throws Throwable
+ {
+ ConnectionTest.Settings s = SETTINGS.get(i);
+ doTestManual(s, test);
+ cleanup();
+ }
+
+ private void doTestManual(ConnectionTest.Settings settings, ManualSendTest test) throws Throwable
+ {
+ InetAddressAndPort endpoint = FBUtilities.getBroadcastAddressAndPort();
+
+ InboundConnectionSettings inboundSettings = settings.inbound.apply(new InboundConnectionSettings())
+ .withBindAddress(endpoint)
+ .withSocketFactory(factory);
+
+ InboundSockets inbound = new InboundSockets(Collections.singletonList(inboundSettings));
+
+ OutboundConnectionSettings outboundSettings = settings.outbound.apply(new OutboundConnectionSettings(endpoint))
+ .withConnectTo(endpoint)
+ .withDefaultReserveLimits()
+ .withSocketFactory(factory);
+
+ ResourceLimits.EndpointAndGlobal reserveCapacityInBytes = new ResourceLimits.EndpointAndGlobal(new ResourceLimits.Concurrent(outboundSettings.applicationSendQueueReserveEndpointCapacityInBytes), outboundSettings.applicationSendQueueReserveGlobalCapacityInBytes);
+ OutboundConnection outbound = new OutboundConnection(settings.type, outboundSettings, reserveCapacityInBytes);
+ try
+ {
+ InboundProxyHandler.Controller controller = new InboundProxyHandler.Controller();
+ inbound.open(pipeline -> {
+ InboundProxyHandler handler = new InboundProxyHandler(controller);
+ pipeline.addLast(handler);
+ }).sync();
+ test.accept(Pair.create(inboundSettings, outboundSettings), inbound, outbound, endpoint, controller);
+ }
+ finally
+ {
+ outbound.close(false);
+ inbound.close().get(30L, SECONDS);
+ outbound.close(false).get(30L, SECONDS);
+ MessagingService.instance().messageHandlers.clear();
+ }
+ }
+
+ private void connect(OutboundConnection outbound) throws Throwable
+ {
+ tryConnect(outbound, 10, SECONDS, true);
+ }
+
+ private void tryConnect(OutboundConnection outbound, long timeout, TimeUnit timeUnit, boolean throwOnFailure) throws Throwable
+ {
+ CountDownLatch connectionLatch = new CountDownLatch(1);
+ unsafeSetHandler(Verb._TEST_1, () -> v -> {
+ connectionLatch.countDown();
+ });
+ outbound.enqueue(Message.out(Verb._TEST_1, 1L));
+ connectionLatch.await(timeout, timeUnit);
+ if (throwOnFailure)
+ Assert.assertEquals(0, connectionLatch.getCount());
+ }
+}
diff --git a/test/unit/org/apache/cassandra/net/PrunableArrayQueueTest.java b/test/unit/org/apache/cassandra/net/PrunableArrayQueueTest.java
new file mode 100644
index 0000000..c4fd55a
--- /dev/null
+++ b/test/unit/org/apache/cassandra/net/PrunableArrayQueueTest.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.net;
+
+import org.junit.Test;
+
+import org.apache.cassandra.net.PrunableArrayQueue;
+
+import static org.junit.Assert.*;
+
+public class PrunableArrayQueueTest
+{
+ private final PrunableArrayQueue<Integer> queue = new PrunableArrayQueue<>(8);
+
+ @Test
+ public void testIsEmptyWhenEmpty()
+ {
+ assertTrue(queue.isEmpty());
+ }
+
+ @Test
+ public void testIsEmptyWhenNotEmpty()
+ {
+ queue.offer(0);
+ assertFalse(queue.isEmpty());
+ }
+
+ @Test
+ public void testEmptyPeek()
+ {
+ assertNull(queue.peek());
+ }
+
+ @Test
+ public void testNonEmptyPeek()
+ {
+ queue.offer(0);
+ assertEquals((Integer) 0, queue.peek());
+ }
+
+ @Test
+ public void testEmptyPoll()
+ {
+ assertNull(queue.poll());
+ }
+
+ @Test
+ public void testNonEmptyPoll()
+ {
+ queue.offer(0);
+ assertEquals((Integer) 0, queue.poll());
+ }
+
+ @Test
+ public void testTransfersInCorrectOrder()
+ {
+ for (int i = 0; i < 1024; i++)
+ queue.offer(i);
+
+ for (int i = 0; i < 1024; i++)
+ assertEquals((Integer) i, queue.poll());
+
+ assertTrue(queue.isEmpty());
+ }
+
+ @Test
+ public void testTransfersInCorrectOrderWhenInterleaved()
+ {
+ for (int i = 0; i < 1024; i++)
+ {
+ queue.offer(i);
+ assertEquals((Integer) i, queue.poll());
+ }
+
+ assertTrue(queue.isEmpty());
+ }
+
+ @Test
+ public void testPrune()
+ {
+ for (int i = 0; i < 1024; i++)
+ queue.offer(i);
+
+ class Pruner implements PrunableArrayQueue.Pruner<Integer>
+ {
+ private int pruned, kept;
+
+ public boolean shouldPrune(Integer val)
+ {
+ return val % 2 == 0;
+ }
+
+ public void onPruned(Integer val)
+ {
+ pruned++;
+ }
+
+ public void onKept(Integer val)
+ {
+ kept++;
+ }
+ }
+
+ Pruner pruner = new Pruner();
+ assertEquals(512, queue.prune(pruner));
+
+ assertEquals(512, pruner.kept);
+ assertEquals(512, pruner.pruned);
+ assertEquals(512, queue.size());
+
+ for (int i = 1; i < 1024; i += 2)
+ assertEquals((Integer) i, queue.poll());
+ assertTrue(queue.isEmpty());
+ }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/net/RateBasedBackPressureTest.java b/test/unit/org/apache/cassandra/net/RateBasedBackPressureTest.java
index b94b6ee..4d1ae01 100644
--- a/test/unit/org/apache/cassandra/net/RateBasedBackPressureTest.java
+++ b/test/unit/org/apache/cassandra/net/RateBasedBackPressureTest.java
@@ -18,7 +18,6 @@
*/
package org.apache.cassandra.net;
-import java.net.InetAddress;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
@@ -27,9 +26,9 @@
import com.google.common.collect.Sets;
import com.google.common.util.concurrent.RateLimiter;
-import org.junit.Assert;
import org.junit.Test;
+import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.utils.TestTimeSource;
import org.apache.cassandra.utils.TimeSource;
@@ -95,17 +94,17 @@
TestTimeSource timeSource = new TestTimeSource();
RateBasedBackPressure strategy = new RateBasedBackPressure(ImmutableMap.of(HIGH_RATIO, "0.9", FACTOR, "10", FLOW, "FAST"), timeSource, windowSize);
- RateBasedBackPressureState state = strategy.newState(InetAddress.getLoopbackAddress());
+ RateBasedBackPressureState state = strategy.newState(InetAddressAndPort.getLoopbackAddress());
state.onMessageSent(null);
assertEquals(0, state.incomingRate.size());
assertEquals(0, state.outgoingRate.size());
- state = strategy.newState(InetAddress.getLoopbackAddress());
+ state = strategy.newState(InetAddressAndPort.getLoopbackAddress());
state.onResponseReceived();
assertEquals(1, state.incomingRate.size());
assertEquals(1, state.outgoingRate.size());
- state = strategy.newState(InetAddress.getLoopbackAddress());
+ state = strategy.newState(InetAddressAndPort.getLoopbackAddress());
state.onResponseTimeout();
assertEquals(0, state.incomingRate.size());
assertEquals(1, state.outgoingRate.size());
@@ -117,7 +116,7 @@
long windowSize = 6000;
TestTimeSource timeSource = new TestTimeSource();
RateBasedBackPressure strategy = new RateBasedBackPressure(ImmutableMap.of(HIGH_RATIO, "0.9", FACTOR, "10", FLOW, "FAST"), timeSource, windowSize);
- RateBasedBackPressureState state = strategy.newState(InetAddress.getLoopbackAddress());
+ RateBasedBackPressureState state = strategy.newState(InetAddressAndPort.getLoopbackAddress());
// Get initial rate:
double initialRate = state.rateLimiter.getRate();
@@ -141,7 +140,7 @@
long windowSize = 6000;
TestTimeSource timeSource = new TestTimeSource();
RateBasedBackPressure strategy = new RateBasedBackPressure(ImmutableMap.of(HIGH_RATIO, "0.9", FACTOR, "10", FLOW, "FAST"), timeSource, windowSize);
- RateBasedBackPressureState state = strategy.newState(InetAddress.getLoopbackAddress());
+ RateBasedBackPressureState state = strategy.newState(InetAddressAndPort.getLoopbackAddress());
// Get initial time:
long current = state.getLastIntervalAcquire();
@@ -175,7 +174,7 @@
long windowSize = 6000;
TestTimeSource timeSource = new TestTimeSource();
RateBasedBackPressure strategy = new RateBasedBackPressure(ImmutableMap.of(HIGH_RATIO, "0.9", FACTOR, "10", FLOW, "FAST"), timeSource, windowSize);
- RateBasedBackPressureState state = strategy.newState(InetAddress.getLoopbackAddress());
+ RateBasedBackPressureState state = strategy.newState(InetAddressAndPort.getLoopbackAddress());
// Update incoming and outgoing rate so that the ratio is 0.5:
state.incomingRate.update(50);
@@ -195,7 +194,7 @@
long windowSize = 6000;
TestTimeSource timeSource = new TestTimeSource();
RateBasedBackPressure strategy = new RateBasedBackPressure(ImmutableMap.of(HIGH_RATIO, "0.9", FACTOR, "10", FLOW, "FAST"), timeSource, windowSize);
- RateBasedBackPressureState state = strategy.newState(InetAddress.getLoopbackAddress());
+ RateBasedBackPressureState state = strategy.newState(InetAddressAndPort.getLoopbackAddress());
// Update incoming and outgoing rate so that the ratio is 0.5:
state.incomingRate.update(50);
@@ -237,9 +236,9 @@
long windowSize = 6000;
TestTimeSource timeSource = new TestTimeSource();
TestableBackPressure strategy = new TestableBackPressure(ImmutableMap.of(HIGH_RATIO, "0.9", FACTOR, "10", FLOW, "FAST"), timeSource, windowSize);
- RateBasedBackPressureState state1 = strategy.newState(InetAddress.getByName("127.0.0.1"));
- RateBasedBackPressureState state2 = strategy.newState(InetAddress.getByName("127.0.0.2"));
- RateBasedBackPressureState state3 = strategy.newState(InetAddress.getByName("127.0.0.3"));
+ RateBasedBackPressureState state1 = strategy.newState(InetAddressAndPort.getByName("127.0.0.1"));
+ RateBasedBackPressureState state2 = strategy.newState(InetAddressAndPort.getByName("127.0.0.2"));
+ RateBasedBackPressureState state3 = strategy.newState(InetAddressAndPort.getByName("127.0.0.3"));
// Update incoming and outgoing rates:
state1.incomingRate.update(50);
@@ -266,9 +265,9 @@
long windowSize = 6000;
TestTimeSource timeSource = new TestTimeSource();
TestableBackPressure strategy = new TestableBackPressure(ImmutableMap.of(HIGH_RATIO, "0.9", FACTOR, "10", FLOW, "SLOW"), timeSource, windowSize);
- RateBasedBackPressureState state1 = strategy.newState(InetAddress.getByName("127.0.0.1"));
- RateBasedBackPressureState state2 = strategy.newState(InetAddress.getByName("127.0.0.2"));
- RateBasedBackPressureState state3 = strategy.newState(InetAddress.getByName("127.0.0.3"));
+ RateBasedBackPressureState state1 = strategy.newState(InetAddressAndPort.getByName("127.0.0.1"));
+ RateBasedBackPressureState state2 = strategy.newState(InetAddressAndPort.getByName("127.0.0.2"));
+ RateBasedBackPressureState state3 = strategy.newState(InetAddressAndPort.getByName("127.0.0.3"));
// Update incoming and outgoing rates:
state1.incomingRate.update(50);
@@ -295,10 +294,10 @@
long windowSize = 6000;
TestTimeSource timeSource = new TestTimeSource();
TestableBackPressure strategy = new TestableBackPressure(ImmutableMap.of(HIGH_RATIO, "0.9", FACTOR, "10", FLOW, "SLOW"), timeSource, windowSize);
- RateBasedBackPressureState state1 = strategy.newState(InetAddress.getByName("127.0.0.1"));
- RateBasedBackPressureState state2 = strategy.newState(InetAddress.getByName("127.0.0.2"));
- RateBasedBackPressureState state3 = strategy.newState(InetAddress.getByName("127.0.0.3"));
- RateBasedBackPressureState state4 = strategy.newState(InetAddress.getByName("127.0.0.4"));
+ RateBasedBackPressureState state1 = strategy.newState(InetAddressAndPort.getByName("127.0.0.1"));
+ RateBasedBackPressureState state2 = strategy.newState(InetAddressAndPort.getByName("127.0.0.2"));
+ RateBasedBackPressureState state3 = strategy.newState(InetAddressAndPort.getByName("127.0.0.3"));
+ RateBasedBackPressureState state4 = strategy.newState(InetAddressAndPort.getByName("127.0.0.4"));
// Update incoming and outgoing rates:
state1.incomingRate.update(50); // this
@@ -334,9 +333,9 @@
long windowSize = 10000;
TestTimeSource timeSource = new TestTimeSource();
TestableBackPressure strategy = new TestableBackPressure(ImmutableMap.of(HIGH_RATIO, "0.9", FACTOR, "10", FLOW, "SLOW"), timeSource, windowSize);
- RateBasedBackPressureState state1 = strategy.newState(InetAddress.getByName("127.0.0.1"));
- RateBasedBackPressureState state2 = strategy.newState(InetAddress.getByName("127.0.0.2"));
- RateBasedBackPressureState state3 = strategy.newState(InetAddress.getByName("127.0.0.3"));
+ RateBasedBackPressureState state1 = strategy.newState(InetAddressAndPort.getByName("127.0.0.1"));
+ RateBasedBackPressureState state2 = strategy.newState(InetAddressAndPort.getByName("127.0.0.2"));
+ RateBasedBackPressureState state3 = strategy.newState(InetAddressAndPort.getByName("127.0.0.3"));
// Update incoming and outgoing rates:
state1.incomingRate.update(5); // slow
diff --git a/test/unit/org/apache/cassandra/net/ResourceLimitsTest.java b/test/unit/org/apache/cassandra/net/ResourceLimitsTest.java
new file mode 100644
index 0000000..734d69a
--- /dev/null
+++ b/test/unit/org/apache/cassandra/net/ResourceLimitsTest.java
@@ -0,0 +1,152 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.net;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Executor;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.function.LongFunction;
+
+import org.junit.Test;
+
+import org.apache.cassandra.net.ResourceLimits.*;
+
+import static org.junit.Assert.*;
+
+public class ResourceLimitsTest
+{
+ @Test
+ public void testAllocatesWithinLimits()
+ {
+ testAllocatesWithinLimits(Basic::new);
+ testAllocatesWithinLimits(Concurrent::new);
+ }
+
+ private void testAllocatesWithinLimits(LongFunction<Limit> supplier)
+ {
+ Limit limit = supplier.apply(100);
+
+ assertEquals(100, limit.limit());
+ assertEquals(0, limit.using());
+ assertEquals(100, limit.remaining());
+
+ assertTrue(limit.tryAllocate(10));
+ assertEquals(10, limit.using());
+ assertEquals(90, limit.remaining());
+
+ assertTrue(limit.tryAllocate(30));
+ assertEquals(40, limit.using());
+ assertEquals(60, limit.remaining());
+
+ assertTrue(limit.tryAllocate(60));
+ assertEquals(100, limit.using());
+ assertEquals(0, limit.remaining());
+ }
+
+ @Test
+ public void testFailsToAllocateOverCapacity()
+ {
+ testFailsToAllocateOverCapacity(Basic::new);
+ testFailsToAllocateOverCapacity(Concurrent::new);
+ }
+
+ private void testFailsToAllocateOverCapacity(LongFunction<Limit> supplier)
+ {
+ Limit limit = supplier.apply(100);
+
+ assertEquals(100, limit.limit());
+ assertEquals(0, limit.using());
+ assertEquals(100, limit.remaining());
+
+ assertTrue(limit.tryAllocate(10));
+ assertEquals(10, limit.using());
+ assertEquals(90, limit.remaining());
+
+ assertFalse(limit.tryAllocate(91));
+ assertEquals(10, limit.using());
+ assertEquals(90, limit.remaining());
+ }
+
+ @Test
+ public void testRelease()
+ {
+ testRelease(Basic::new);
+ testRelease(Concurrent::new);
+ }
+
+ private void testRelease(LongFunction<Limit> supplier)
+ {
+ Limit limit = supplier.apply(100);
+
+ assertEquals(100, limit.limit());
+ assertEquals(0, limit.using());
+ assertEquals(100, limit.remaining());
+
+ assertTrue(limit.tryAllocate(10));
+ assertTrue(limit.tryAllocate(30));
+ assertTrue(limit.tryAllocate(60));
+ assertEquals(100, limit.using());
+ assertEquals(0, limit.remaining());
+
+ limit.release(10);
+ assertEquals(90, limit.using());
+ assertEquals(10, limit.remaining());
+
+ limit.release(30);
+ assertEquals(60, limit.using());
+ assertEquals(40, limit.remaining());
+
+ limit.release(60);
+ assertEquals(0, limit.using());
+ assertEquals(100, limit.remaining());
+ }
+
+ @Test
+ public void testConcurrentLimit() throws Exception
+ {
+ int numThreads = 4;
+ int numPermitsPerThread = 1_000_000;
+ int numPermits = numThreads * numPermitsPerThread;
+
+ CountDownLatch latch = new CountDownLatch(numThreads);
+ Limit limit = new Concurrent(numPermits);
+
+ class Worker implements Runnable
+ {
+ public void run()
+ {
+ for (int i = 0; i < numPermitsPerThread; i += 10)
+ assertTrue(limit.tryAllocate(10));
+
+ for (int i = 0; i < numPermitsPerThread; i += 10)
+ limit.release(10);
+
+ latch.countDown();
+ }
+ }
+
+ Executor executor = Executors.newFixedThreadPool(numThreads);
+ for (int i = 0; i < numThreads; i++)
+ executor.execute(new Worker());
+ latch.await(10, TimeUnit.SECONDS);
+
+ assertEquals(0, limit.using());
+ assertEquals(numPermits, limit.remaining());
+ }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/net/SocketUtils.java b/test/unit/org/apache/cassandra/net/SocketUtils.java
new file mode 100644
index 0000000..a0a1490
--- /dev/null
+++ b/test/unit/org/apache/cassandra/net/SocketUtils.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.net;
+
+import java.io.IOException;
+import java.net.ServerSocket;
+
+import com.google.common.base.Throwables;
+
+public class SocketUtils
+{
+ public static synchronized int findAvailablePort() throws RuntimeException
+ {
+ ServerSocket ss = null;
+ try
+ {
+ // let the system pick an ephemeral port
+ ss = new ServerSocket(0);
+ ss.setReuseAddress(true);
+ return ss.getLocalPort();
+ }
+ catch (IOException e)
+ {
+ throw Throwables.propagate(e);
+ }
+ finally
+ {
+ if (ss != null)
+ {
+ try
+ {
+ ss.close();
+ }
+ catch (IOException e)
+ {
+ Throwables.propagate(e);
+ }
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/net/StartupClusterConnectivityCheckerTest.java b/test/unit/org/apache/cassandra/net/StartupClusterConnectivityCheckerTest.java
new file mode 100644
index 0000000..0785f27
--- /dev/null
+++ b/test/unit/org/apache/cassandra/net/StartupClusterConnectivityCheckerTest.java
@@ -0,0 +1,253 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.net;
+
+import java.net.UnknownHostException;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.BiPredicate;
+
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.gms.EndpointState;
+import org.apache.cassandra.gms.Gossiper;
+import org.apache.cassandra.gms.HeartBeatState;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.utils.FBUtilities;
+
+public class StartupClusterConnectivityCheckerTest
+{
+ private StartupClusterConnectivityChecker localQuorumConnectivityChecker;
+ private StartupClusterConnectivityChecker globalQuorumConnectivityChecker;
+ private StartupClusterConnectivityChecker noopChecker;
+ private StartupClusterConnectivityChecker zeroWaitChecker;
+
+ private static final long TIMEOUT_NANOS = 100;
+ private static final int NUM_PER_DC = 6;
+ private Set<InetAddressAndPort> peers;
+ private Set<InetAddressAndPort> peersA;
+ private Set<InetAddressAndPort> peersAMinusLocal;
+ private Set<InetAddressAndPort> peersB;
+ private Set<InetAddressAndPort> peersC;
+
+ private String getDatacenter(InetAddressAndPort endpoint)
+ {
+ if (peersA.contains(endpoint))
+ return "datacenterA";
+ if (peersB.contains(endpoint))
+ return "datacenterB";
+ else if (peersC.contains(endpoint))
+ return "datacenterC";
+ return null;
+ }
+
+ @BeforeClass
+ public static void before()
+ {
+ DatabaseDescriptor.daemonInitialization();
+ }
+
+ @Before
+ public void setUp() throws UnknownHostException
+ {
+ localQuorumConnectivityChecker = new StartupClusterConnectivityChecker(TIMEOUT_NANOS, false);
+ globalQuorumConnectivityChecker = new StartupClusterConnectivityChecker(TIMEOUT_NANOS, true);
+ noopChecker = new StartupClusterConnectivityChecker(-1, false);
+ zeroWaitChecker = new StartupClusterConnectivityChecker(0, false);
+
+ peersA = new HashSet<>();
+ peersAMinusLocal = new HashSet<>();
+ peersA.add(FBUtilities.getBroadcastAddressAndPort());
+
+ for (int i = 0; i < NUM_PER_DC - 1; i ++)
+ {
+ peersA.add(InetAddressAndPort.getByName("127.0.1." + i));
+ peersAMinusLocal.add(InetAddressAndPort.getByName("127.0.1." + i));
+ }
+
+ peersB = new HashSet<>();
+ for (int i = 0; i < NUM_PER_DC; i ++)
+ peersB.add(InetAddressAndPort.getByName("127.0.2." + i));
+
+
+ peersC = new HashSet<>();
+ for (int i = 0; i < NUM_PER_DC; i ++)
+ peersC.add(InetAddressAndPort.getByName("127.0.3." + i));
+
+ peers = new HashSet<>();
+ peers.addAll(peersA);
+ peers.addAll(peersB);
+ peers.addAll(peersC);
+ }
+
+ @After
+ public void tearDown()
+ {
+ MessagingService.instance().outboundSink.clear();
+ }
+
+ @Test
+ public void execute_HappyPath()
+ {
+ Sink sink = new Sink(true, true, peers);
+ MessagingService.instance().outboundSink.add(sink);
+ Assert.assertTrue(localQuorumConnectivityChecker.execute(peers, this::getDatacenter));
+ }
+
+ @Test
+ public void execute_NotAlive()
+ {
+ Sink sink = new Sink(false, true, peers);
+ MessagingService.instance().outboundSink.add(sink);
+ Assert.assertFalse(localQuorumConnectivityChecker.execute(peers, this::getDatacenter));
+ }
+
+ @Test
+ public void execute_NoConnectionsAcks()
+ {
+ Sink sink = new Sink(true, false, peers);
+ MessagingService.instance().outboundSink.add(sink);
+ Assert.assertFalse(localQuorumConnectivityChecker.execute(peers, this::getDatacenter));
+ }
+
+ @Test
+ public void execute_LocalQuorum()
+ {
+ // local peer plus 3 peers from same dc shouldn't pass (4/6)
+ Set<InetAddressAndPort> available = new HashSet<>();
+ copyCount(peersAMinusLocal, available, NUM_PER_DC - 3);
+ checkAvailable(localQuorumConnectivityChecker, available, false);
+
+ // local peer plus 4 peers from same dc should pass (5/6)
+ available.clear();
+ copyCount(peersAMinusLocal, available, NUM_PER_DC - 2);
+ checkAvailable(localQuorumConnectivityChecker, available, true);
+ }
+
+ @Test
+ public void execute_GlobalQuorum()
+ {
+ // local dc passing shouldn't pass globally with two hosts down in datacenterB
+ Set<InetAddressAndPort> available = new HashSet<>();
+ copyCount(peersAMinusLocal, available, NUM_PER_DC - 2);
+ copyCount(peersB, available, NUM_PER_DC - 2);
+ copyCount(peersC, available, NUM_PER_DC - 1);
+ checkAvailable(globalQuorumConnectivityChecker, available, false);
+
+ // All three datacenters should be able to have a single node down
+ available.clear();
+ copyCount(peersAMinusLocal, available, NUM_PER_DC - 2);
+ copyCount(peersB, available, NUM_PER_DC - 1);
+ copyCount(peersC, available, NUM_PER_DC - 1);
+ checkAvailable(globalQuorumConnectivityChecker, available, true);
+
+ // Everything being up should work of course
+ available.clear();
+ copyCount(peersAMinusLocal, available, NUM_PER_DC - 1);
+ copyCount(peersB, available, NUM_PER_DC);
+ copyCount(peersC, available, NUM_PER_DC);
+ checkAvailable(globalQuorumConnectivityChecker, available, true);
+ }
+
+ @Test
+ public void execute_Noop()
+ {
+ checkAvailable(noopChecker, new HashSet<>(), true);
+ }
+
+ @Test
+ public void execute_ZeroWaitHasConnections() throws InterruptedException
+ {
+ Sink sink = new Sink(true, true, new HashSet<>());
+ MessagingService.instance().outboundSink.add(sink);
+ Assert.assertFalse(zeroWaitChecker.execute(peers, this::getDatacenter));
+ MessagingService.instance().outboundSink.clear();
+ }
+
+ private void checkAvailable(StartupClusterConnectivityChecker checker, Set<InetAddressAndPort> available,
+ boolean shouldPass)
+ {
+ Sink sink = new Sink(true, true, available);
+ MessagingService.instance().outboundSink.add(sink);
+ Assert.assertEquals(shouldPass, checker.execute(peers, this::getDatacenter));
+ MessagingService.instance().outboundSink.clear();
+ }
+
+ private void copyCount(Set<InetAddressAndPort> source, Set<InetAddressAndPort> dest, int count)
+ {
+ for (InetAddressAndPort peer : source)
+ {
+ if (count <= 0)
+ break;
+
+ dest.add(peer);
+ count -= 1;
+ }
+ }
+
+ private static class Sink implements BiPredicate<Message<?>, InetAddressAndPort>
+ {
+ private final boolean markAliveInGossip;
+ private final boolean processConnectAck;
+ private final Set<InetAddressAndPort> aliveHosts;
+ private final Map<InetAddressAndPort, ConnectionTypeRecorder> seenConnectionRequests;
+
+ Sink(boolean markAliveInGossip, boolean processConnectAck, Set<InetAddressAndPort> aliveHosts)
+ {
+ this.markAliveInGossip = markAliveInGossip;
+ this.processConnectAck = processConnectAck;
+ this.aliveHosts = aliveHosts;
+ seenConnectionRequests = new HashMap<>();
+ }
+
+ @Override
+ public boolean test(Message message, InetAddressAndPort to)
+ {
+ ConnectionTypeRecorder recorder = seenConnectionRequests.computeIfAbsent(to, inetAddress -> new ConnectionTypeRecorder());
+
+ if (!aliveHosts.contains(to))
+ return false;
+
+ if (processConnectAck)
+ {
+ Message msgIn = Message.builder(Verb.REQUEST_RSP, message.payload)
+ .from(to)
+ .build();
+ MessagingService.instance().callbacks.get(message.id(), to).callback.onResponse(msgIn);
+ }
+
+ if (markAliveInGossip)
+ Gossiper.runInGossipStageBlocking(() -> Gossiper.instance.realMarkAlive(to, new EndpointState(new HeartBeatState(1, 1))));
+ return false;
+ }
+ }
+
+ private static class ConnectionTypeRecorder
+ {
+ boolean seenSmallMessageRequest;
+ boolean seenLargeMessageRequest;
+ }
+}
diff --git a/test/unit/org/apache/cassandra/net/TestAbstractAsyncPromise.java b/test/unit/org/apache/cassandra/net/TestAbstractAsyncPromise.java
new file mode 100644
index 0000000..fd61b09
--- /dev/null
+++ b/test/unit/org/apache/cassandra/net/TestAbstractAsyncPromise.java
@@ -0,0 +1,234 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.net;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.CancellationException;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import com.google.common.util.concurrent.Uninterruptibles;
+import org.junit.Assert;
+
+import io.netty.util.concurrent.Future;
+import io.netty.util.concurrent.GenericFutureListener;
+import io.netty.util.concurrent.Promise;
+
+abstract class TestAbstractAsyncPromise extends TestAbstractPromise
+{
+ <V> void testOneSuccess(Promise<V> promise, boolean setUncancellable, boolean tryOrSet, V value, V otherValue)
+ {
+ List<V> results = new ArrayList<>();
+ List<Integer> order = new ArrayList<>();
+ class ListenerFactory
+ {
+ int count = 0;
+
+ public GenericFutureListener<Future<V>> get()
+ {
+ int id = count++;
+ return p -> { results.add(p.getNow()); order.add(id); };
+ }
+ public GenericFutureListener<Future<V>> getRecursive()
+ {
+ int id = count++;
+ return p -> { promise.addListener(get()); results.add(p.getNow()); order.add(id); };
+ }
+ }
+ ListenerFactory listeners = new ListenerFactory();
+ Async async = new Async();
+ promise.addListener(listeners.get());
+ promise.addListeners(listeners.getRecursive(), listeners.get());
+ promise.addListener(listeners.getRecursive());
+ success(promise, Promise::getNow, null);
+ success(promise, Promise::isSuccess, false);
+ success(promise, Promise::isDone, false);
+ success(promise, Promise::isCancelled, false);
+ success(promise, Promise::isCancellable, true);
+ if (setUncancellable)
+ {
+ success(promise, Promise::setUncancellable, true);
+ success(promise, Promise::setUncancellable, true);
+ success(promise, p -> p.cancel(true), false);
+ success(promise, p -> p.cancel(false), false);
+ }
+ success(promise, Promise::isCancellable, !setUncancellable);
+ async.success(promise, Promise::get, value);
+ async.success(promise, p -> p.get(1L, TimeUnit.SECONDS), value);
+ async.success(promise, Promise::await, promise);
+ async.success(promise, Promise::awaitUninterruptibly, promise);
+ async.success(promise, p -> p.await(1L, TimeUnit.SECONDS), true);
+ async.success(promise, p -> p.await(1000L), true);
+ async.success(promise, p -> p.awaitUninterruptibly(1L, TimeUnit.SECONDS), true);
+ async.success(promise, p -> p.awaitUninterruptibly(1000L), true);
+ async.success(promise, Promise::sync, promise);
+ async.success(promise, Promise::syncUninterruptibly, promise);
+ if (tryOrSet) promise.trySuccess(value);
+ else promise.setSuccess(value);
+ success(promise, p -> p.cancel(true), false);
+ success(promise, p -> p.cancel(false), false);
+ failure(promise, p -> p.setSuccess(null), IllegalStateException.class);
+ failure(promise, p -> p.setFailure(new NullPointerException()), IllegalStateException.class);
+ success(promise, Promise::getNow, value);
+ success(promise, p -> p.trySuccess(otherValue), false);
+ success(promise, p -> p.tryFailure(new NullPointerException()), false);
+ success(promise, Promise::getNow, value);
+ success(promise, Promise::cause, null);
+ promise.addListener(listeners.get());
+ promise.addListeners(listeners.getRecursive(), listeners.get());
+ promise.addListener(listeners.getRecursive());
+ success(promise, Promise::isSuccess, true);
+ success(promise, Promise::isDone, true);
+ success(promise, Promise::isCancelled, false);
+ success(promise, Promise::isCancellable, false);
+ async.verify();
+ Assert.assertEquals(listeners.count, results.size());
+ Assert.assertEquals(listeners.count, order.size());
+ for (V result : results)
+ Assert.assertEquals(value, result);
+ for (int i = 0 ; i < order.size() ; ++i)
+ Assert.assertEquals(i, order.get(i).intValue());
+ }
+
+ <V> void testOneFailure(Promise<V> promise, boolean setUncancellable, boolean tryOrSet, Throwable cause, V otherValue)
+ {
+ List<Throwable> results = new ArrayList<>();
+ List<Integer> order = new ArrayList<>();
+ Async async = new Async();
+ class ListenerFactory
+ {
+ int count = 0;
+
+ public GenericFutureListener<Future<V>> get()
+ {
+ int id = count++;
+ return p -> { results.add(p.cause()); order.add(id); };
+ }
+ public GenericFutureListener<Future<V>> getRecursive()
+ {
+ int id = count++;
+ return p -> { promise.addListener(get()); results.add(p.cause()); order.add(id); };
+ }
+ }
+ ListenerFactory listeners = new ListenerFactory();
+ promise.addListener(listeners.get());
+ promise.addListeners(listeners.getRecursive(), listeners.get());
+ promise.addListener(listeners.getRecursive());
+ success(promise, Promise::isSuccess, false);
+ success(promise, Promise::isDone, false);
+ success(promise, Promise::isCancelled, false);
+ success(promise, Promise::isCancellable, true);
+ if (setUncancellable)
+ {
+ success(promise, Promise::setUncancellable, true);
+ success(promise, Promise::setUncancellable, true);
+ success(promise, p -> p.cancel(true), false);
+ success(promise, p -> p.cancel(false), false);
+ }
+ success(promise, Promise::isCancellable, !setUncancellable);
+ success(promise, Promise::getNow, null);
+ success(promise, Promise::cause, null);
+ async.failure(promise, Promise::get, ExecutionException.class);
+ async.failure(promise, p -> p.get(1L, TimeUnit.SECONDS), ExecutionException.class);
+ async.success(promise, Promise::await, promise);
+ async.success(promise, Promise::awaitUninterruptibly, promise);
+ async.success(promise, p -> p.await(1L, TimeUnit.SECONDS), true);
+ async.success(promise, p -> p.await(1000L), true);
+ async.success(promise, p -> p.awaitUninterruptibly(1L, TimeUnit.SECONDS), true);
+ async.success(promise, p -> p.awaitUninterruptibly(1000L), true);
+ async.failure(promise, Promise::sync, cause);
+ async.failure(promise, Promise::syncUninterruptibly, cause);
+ if (tryOrSet) promise.tryFailure(cause);
+ else promise.setFailure(cause);
+ success(promise, p -> p.cancel(true), false);
+ success(promise, p -> p.cancel(false), false);
+ failure(promise, p -> p.setSuccess(null), IllegalStateException.class);
+ failure(promise, p -> p.setFailure(new NullPointerException()), IllegalStateException.class);
+ success(promise, Promise::cause, cause);
+ success(promise, Promise::getNow, null);
+ success(promise, p -> p.trySuccess(otherValue), false);
+ success(promise, p -> p.tryFailure(new NullPointerException()), false);
+ success(promise, Promise::getNow, null);
+ success(promise, Promise::cause, cause);
+ promise.addListener(listeners.get());
+ promise.addListeners(listeners.getRecursive(), listeners.get());
+ promise.addListener(listeners.getRecursive());
+ success(promise, Promise::isSuccess, false);
+ success(promise, Promise::isDone, true);
+ success(promise, Promise::isCancelled, false);
+ success(promise, Promise::isCancellable, false);
+ async.verify();
+ Assert.assertEquals(listeners.count, results.size());
+ Assert.assertEquals(listeners.count, order.size());
+ for (Throwable result : results)
+ Assert.assertEquals(cause, result);
+ for (int i = 0 ; i < order.size() ; ++i)
+ Assert.assertEquals(i, order.get(i).intValue());
+ }
+
+ public <V> void testOneCancellation(Promise<V> promise, boolean interruptIfRunning, V otherValue)
+ {
+ Async async = new Async();
+ success(promise, Promise::isCancellable, true);
+ success(promise, Promise::getNow, null);
+ success(promise, Promise::cause, null);
+ async.failure(promise, Promise::get, CancellationException.class);
+ async.failure(promise, p -> p.get(1L, TimeUnit.SECONDS), CancellationException.class);
+ async.success(promise, Promise::await, promise);
+ async.success(promise, Promise::awaitUninterruptibly, promise);
+ async.success(promise, p -> p.await(1L, TimeUnit.SECONDS), true);
+ async.success(promise, p -> p.await(1000L), true);
+ async.success(promise, p -> p.awaitUninterruptibly(1L, TimeUnit.SECONDS), true);
+ async.success(promise, p -> p.awaitUninterruptibly(1000L), true);
+ async.failure(promise, Promise::sync, CancellationException.class);
+ async.failure(promise, Promise::syncUninterruptibly, CancellationException.class);
+ promise.cancel(interruptIfRunning);
+ failure(promise, p -> p.setFailure(null), IllegalStateException.class);
+ failure(promise, p -> p.setFailure(null), IllegalStateException.class);
+ Assert.assertTrue(promise.cause() instanceof CancellationException);
+ success(promise, Promise::getNow, null);
+ success(promise, p -> p.trySuccess(otherValue), false);
+ success(promise, Promise::getNow, null);
+ Assert.assertTrue(promise.cause() instanceof CancellationException);
+ success(promise, Promise::isSuccess, false);
+ success(promise, Promise::isDone, true);
+ success(promise, Promise::isCancelled, true);
+ success(promise, Promise::isCancellable, false);
+ async.verify();
+ }
+
+
+ public <V> void testOneTimeout(Promise<V> promise, boolean setUncancellable)
+ {
+ Async async = new Async();
+ if (setUncancellable)
+ success(promise, Promise::setUncancellable, true);
+ success(promise, Promise::isCancellable, !setUncancellable);
+ async.failure(promise, p -> p.get(1L, TimeUnit.MILLISECONDS), TimeoutException.class);
+ async.success(promise, p -> p.await(1L, TimeUnit.MILLISECONDS), false);
+ async.success(promise, p -> p.await(1L), false);
+ async.success(promise, p -> p.awaitUninterruptibly(1L, TimeUnit.MILLISECONDS), false);
+ async.success(promise, p -> p.awaitUninterruptibly(1L), false);
+ Uninterruptibles.sleepUninterruptibly(10L, TimeUnit.MILLISECONDS);
+ async.verify();
+ }
+
+}
diff --git a/test/unit/org/apache/cassandra/net/TestAbstractPromise.java b/test/unit/org/apache/cassandra/net/TestAbstractPromise.java
new file mode 100644
index 0000000..963c61f
--- /dev/null
+++ b/test/unit/org/apache/cassandra/net/TestAbstractPromise.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.net;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Predicate;
+
+import org.junit.Assert;
+
+import io.netty.util.concurrent.Promise;
+import net.openhft.chronicle.core.util.ThrowingBiConsumer;
+import net.openhft.chronicle.core.util.ThrowingConsumer;
+import net.openhft.chronicle.core.util.ThrowingFunction;
+
+abstract class TestAbstractPromise
+{
+ final ExecutorService exec = Executors.newCachedThreadPool();
+
+ class Async
+ {
+ final List<ThrowingBiConsumer<Long, TimeUnit, ?>> waitingOn = new ArrayList<>();
+ void verify()
+ {
+ for (int i = 0 ; i < waitingOn.size() ; ++i)
+ {
+ try
+ {
+ waitingOn.get(i).accept(100L, TimeUnit.MILLISECONDS);
+ }
+ catch (Throwable t)
+ {
+ throw new AssertionError("" + i, t);
+ }
+ }
+ }
+ <V> void failure(Promise<V> promise, ThrowingConsumer<Promise<V>, ?> action, Throwable failsWith)
+ {
+ waitingOn.add(exec.submit(() -> TestAbstractPromise.failure(promise, action, failsWith))::get);
+ }
+ <V> void failure(Promise<V> promise, ThrowingConsumer<Promise<V>, ?> action, Class<? extends Throwable> failsWith)
+ {
+ waitingOn.add(exec.submit(() -> TestAbstractPromise.failure(promise, action, failsWith))::get);
+ }
+ <V> void failure(Promise<V> promise, ThrowingConsumer<Promise<V>, ?> action, Predicate<Throwable> failsWith)
+ {
+ waitingOn.add(exec.submit(() -> TestAbstractPromise.failure(promise, action, failsWith))::get);
+ }
+ <P extends Promise<?>, R> void success(P promise, ThrowingFunction<P, R, ?> action, R result)
+ {
+ waitingOn.add(exec.submit(() -> TestAbstractPromise.success(promise, action, result))::get);
+ }
+ }
+
+ private static <V> void failure(Promise<V> promise, ThrowingConsumer<Promise<V>, ?> action, Throwable failsWith)
+ {
+ failure(promise, action, t -> Objects.equals(failsWith, t));
+ }
+
+ static <V> void failure(Promise<V> promise, ThrowingConsumer<Promise<V>, ?> action, Class<? extends Throwable> failsWith)
+ {
+ failure(promise, action, failsWith::isInstance);
+ }
+
+ private static <V> void failure(Promise<V> promise, ThrowingConsumer<Promise<V>, ?> action, Predicate<Throwable> failsWith)
+ {
+ Throwable fail = null;
+ try
+ {
+ action.accept(promise);
+ }
+ catch (Throwable t)
+ {
+ fail = t;
+ }
+ if (!failsWith.test(fail))
+ throw new AssertionError(fail);
+ }
+
+ static <P extends Promise<?>, R> void success(P promise, ThrowingFunction<P, R, ?> action, R result)
+ {
+ try
+ {
+ Assert.assertEquals(result, action.apply(promise));
+ }
+ catch (Throwable t)
+ {
+ throw new AssertionError(t);
+ }
+ }
+
+}
diff --git a/test/unit/org/apache/cassandra/net/TestChannel.java b/test/unit/org/apache/cassandra/net/TestChannel.java
new file mode 100644
index 0000000..17da6fa
--- /dev/null
+++ b/test/unit/org/apache/cassandra/net/TestChannel.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.net;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.WritableByteChannel;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.Unpooled;
+import io.netty.channel.ChannelOutboundBuffer;
+import io.netty.channel.FileRegion;
+import io.netty.channel.embedded.EmbeddedChannel;
+
+public class TestChannel extends EmbeddedChannel
+{
+ final int inFlightLimit;
+ int inFlight;
+
+ ChannelOutboundBuffer flush;
+ long flushBytes;
+
+ public TestChannel(int inFlightLimit)
+ {
+ this.inFlightLimit = inFlightLimit;
+ }
+
+ // we override ByteBuf to prevent retain() from working, to avoid release() since it is not needed in our usage
+ // since the lifetime must live longer, we simply copy any outbound ByteBuf here for our tests
+ protected void doWrite(ChannelOutboundBuffer in)
+ {
+ assert flush == null || flush == in;
+ doWrite(in, in.totalPendingWriteBytes());
+ }
+
+ private void doWrite(ChannelOutboundBuffer flush, long flushBytes)
+ {
+ while (true) {
+ Object msg = flush.current();
+ if (msg == null) {
+ this.flush = null;
+ this.flushBytes = 0;
+ return;
+ }
+
+ if (inFlight >= inFlightLimit)
+ {
+ this.flush = flush;
+ this.flushBytes = flushBytes;
+ return;
+ }
+
+ ByteBuf buf;
+ if (msg instanceof FileRegion)
+ {
+ buf = GlobalBufferPoolAllocator.instance.directBuffer((int) ((FileRegion) msg).count());
+ try
+ {
+ ((FileRegion) msg).transferTo(new WritableByteChannel()
+ {
+ public int write(ByteBuffer src)
+ {
+ buf.setBytes(0, src);
+ return buf.writerIndex();
+ }
+
+ public boolean isOpen() { return true; }
+
+ public void close() { }
+ }, 0);
+ }
+ catch (IOException e)
+ {
+ throw new RuntimeException(e);
+ }
+ }
+ else if (msg instanceof ByteBuf)
+ {
+ buf = ((ByteBuf)msg).copy();
+ }
+ else if (msg instanceof FrameEncoder.Payload)
+ {
+ buf = Unpooled.wrappedBuffer(((FrameEncoder.Payload)msg).buffer).copy();
+ }
+ else
+ {
+ System.err.println("Unexpected message type " + msg);
+ throw new IllegalArgumentException();
+ }
+
+ inFlight += buf.readableBytes();
+ handleOutboundMessage(buf);
+ flush.remove();
+ }
+ }
+
+ public <T> T readOutbound()
+ {
+ T msg = super.readOutbound();
+ if (msg instanceof ByteBuf)
+ {
+ inFlight -= ((ByteBuf) msg).readableBytes();
+ if (flush != null && inFlight < inFlightLimit)
+ doWrite(flush, flushBytes);
+ }
+ return msg;
+ }
+}
+
diff --git a/test/unit/org/apache/cassandra/net/TestScheduledFuture.java b/test/unit/org/apache/cassandra/net/TestScheduledFuture.java
new file mode 100644
index 0000000..456f8c4
--- /dev/null
+++ b/test/unit/org/apache/cassandra/net/TestScheduledFuture.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.net;
+
+import java.util.concurrent.Delayed;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+public class TestScheduledFuture implements ScheduledFuture<Object>
+{
+ private boolean cancelled = false;
+
+ public long getDelay(TimeUnit unit)
+ {
+ return 0;
+ }
+
+ public int compareTo(Delayed o)
+ {
+ return 0;
+ }
+
+ public boolean cancel(boolean mayInterruptIfRunning)
+ {
+ cancelled = true;
+ return false;
+ }
+
+ public boolean isCancelled()
+ {
+ return cancelled;
+ }
+
+ public boolean isDone()
+ {
+ return false;
+ }
+
+ public Object get() throws InterruptedException, ExecutionException
+ {
+ return null;
+ }
+
+ public Object get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException
+ {
+ return null;
+ }
+}
diff --git a/test/unit/org/apache/cassandra/net/VerbTest.java b/test/unit/org/apache/cassandra/net/VerbTest.java
new file mode 100644
index 0000000..8f20567
--- /dev/null
+++ b/test/unit/org/apache/cassandra/net/VerbTest.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.net;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+public class VerbTest
+{
+ @Test
+ public void idsMatch()
+ {
+ for (Verb v : Verb.values())
+ assertEquals(v, Verb.fromId(v.id));
+ }
+}
diff --git a/test/unit/org/apache/cassandra/net/WriteCallbackInfoTest.java b/test/unit/org/apache/cassandra/net/WriteCallbackInfoTest.java
index 70e5add..b4bf8b7 100644
--- a/test/unit/org/apache/cassandra/net/WriteCallbackInfoTest.java
+++ b/test/unit/org/apache/cassandra/net/WriteCallbackInfoTest.java
@@ -18,25 +18,25 @@
*/
package org.apache.cassandra.net;
-import java.net.InetAddress;
import java.util.UUID;
import org.junit.BeforeClass;
import org.junit.Test;
-import junit.framework.Assert;
-import org.apache.cassandra.MockSchema;
+import org.junit.Assert;
import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.db.BufferDecoratedKey;
import org.apache.cassandra.db.ConsistencyLevel;
import org.apache.cassandra.db.Mutation;
-import org.apache.cassandra.db.PartitionColumns;
+import org.apache.cassandra.db.RegularAndStaticColumns;
import org.apache.cassandra.db.partitions.PartitionUpdate;
-import org.apache.cassandra.dht.Murmur3Partitioner;
-import org.apache.cassandra.net.MessagingService.Verb;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.schema.MockSchema;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.service.paxos.Commit;
import org.apache.cassandra.utils.ByteBufferUtil;
+import static org.apache.cassandra.locator.ReplicaUtils.full;
+
public class WriteCallbackInfoTest
{
@BeforeClass
@@ -48,8 +48,8 @@
@Test
public void testShouldHint() throws Exception
{
- testShouldHint(Verb.COUNTER_MUTATION, ConsistencyLevel.ALL, true, false);
- for (Verb verb : new Verb[] { Verb.PAXOS_COMMIT, Verb.MUTATION })
+ testShouldHint(Verb.COUNTER_MUTATION_REQ, ConsistencyLevel.ALL, true, false);
+ for (Verb verb : new Verb[] { Verb.PAXOS_COMMIT_REQ, Verb.MUTATION_REQ })
{
testShouldHint(verb, ConsistencyLevel.ALL, true, true);
testShouldHint(verb, ConsistencyLevel.ANY, true, false);
@@ -59,11 +59,12 @@
private void testShouldHint(Verb verb, ConsistencyLevel cl, boolean allowHints, boolean expectHint) throws Exception
{
- Object payload = verb == Verb.PAXOS_COMMIT
- ? new Commit(UUID.randomUUID(), new PartitionUpdate(MockSchema.newCFMetaData("", ""), ByteBufferUtil.EMPTY_BYTE_BUFFER, PartitionColumns.NONE, 1))
- : new Mutation("", new BufferDecoratedKey(new Murmur3Partitioner.LongToken(0), ByteBufferUtil.EMPTY_BYTE_BUFFER));
+ TableMetadata metadata = MockSchema.newTableMetadata("", "");
+ Object payload = verb == Verb.PAXOS_COMMIT_REQ
+ ? new Commit(UUID.randomUUID(), new PartitionUpdate.Builder(metadata, ByteBufferUtil.EMPTY_BYTE_BUFFER, RegularAndStaticColumns.NONE, 1).build())
+ : new Mutation(PartitionUpdate.simpleBuilder(metadata, "").build());
- WriteCallbackInfo wcbi = new WriteCallbackInfo(InetAddress.getByName("192.168.1.1"), null, new MessageOut(verb, payload, null), null, cl, allowHints);
+ RequestCallbacks.WriteCallbackInfo wcbi = new RequestCallbacks.WriteCallbackInfo(Message.out(verb, payload), full(InetAddressAndPort.getByName("192.168.1.1")), null, cl, allowHints);
Assert.assertEquals(expectHint, wcbi.shouldHint());
if (expectHint)
{
diff --git a/test/unit/org/apache/cassandra/net/proxy/InboundProxyHandler.java b/test/unit/org/apache/cassandra/net/proxy/InboundProxyHandler.java
new file mode 100644
index 0000000..7e3b004
--- /dev/null
+++ b/test/unit/org/apache/cassandra/net/proxy/InboundProxyHandler.java
@@ -0,0 +1,234 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.net.proxy;
+
+import java.util.ArrayDeque;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Function;
+
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.channel.ChannelInboundHandlerAdapter;
+import io.netty.util.concurrent.EventExecutor;
+import io.netty.util.concurrent.ScheduledFuture;
+
+public class InboundProxyHandler extends ChannelInboundHandlerAdapter
+{
+ private final ArrayDeque<Forward> forwardQueue;
+ private ScheduledFuture scheduled = null;
+ private final Controller controller;
+ public InboundProxyHandler(Controller controller)
+ {
+ this.controller = controller;
+ this.forwardQueue = new ArrayDeque<>(1024);
+ }
+
+ @Override
+ public void channelActive(ChannelHandlerContext ctx) throws Exception
+ {
+ super.channelActive(ctx);
+ ctx.read();
+ }
+
+ @Override
+ public void channelInactive(ChannelHandlerContext ctx) throws Exception {
+ controller.onDisconnect.run();
+
+ if (scheduled != null)
+ {
+ scheduled.cancel(true);
+ scheduled = null;
+ }
+
+ if (!forwardQueue.isEmpty())
+ forwardQueue.clear();
+
+ super.channelInactive(ctx);
+ }
+
+
+ @Override
+ public void channelRead(ChannelHandlerContext ctx, Object msg)
+ {
+ Forward forward = controller.forwardStrategy.forward(ctx, msg);
+ forwardQueue.offer(forward);
+ maybeScheduleNext(ctx.channel().eventLoop());
+ controller.onRead.run();
+ ctx.channel().read();
+ }
+
+ private void maybeScheduleNext(EventExecutor executor)
+ {
+ if (forwardQueue.isEmpty())
+ {
+ // Ran out of items to process
+ scheduled = null;
+ }
+ else if (scheduled == null)
+ {
+ // Schedule next available or let the last in line schedule it
+ Forward forward = forwardQueue.poll();
+ scheduled = forward.schedule(executor);
+ scheduled.addListener((e) -> {
+ scheduled = null;
+ maybeScheduleNext(executor);
+ });
+ }
+ }
+
+ private static class Forward
+ {
+ final long arrivedAt;
+ final long latency;
+ final Runnable handler;
+
+ private Forward(long arrivedAt, long latency, Runnable handler)
+ {
+ this.arrivedAt = arrivedAt;
+ this.latency = latency;
+ this.handler = handler;
+ }
+
+ ScheduledFuture schedule(EventExecutor executor)
+ {
+ long now = System.currentTimeMillis();
+ long elapsed = now - arrivedAt;
+ long runIn = latency - elapsed;
+
+ if (runIn > 0)
+ return executor.schedule(handler, runIn, TimeUnit.MILLISECONDS);
+ else
+ return executor.schedule(handler, 0, TimeUnit.MILLISECONDS);
+ }
+ }
+
+ private static class ForwardNormally implements ForwardStrategy
+ {
+ static ForwardNormally instance = new ForwardNormally();
+
+ public Forward forward(ChannelHandlerContext ctx, Object msg)
+ {
+ return new Forward(System.currentTimeMillis(),
+ 0,
+ () -> ctx.fireChannelRead(msg));
+ }
+ }
+
+ public interface ForwardStrategy
+ {
+ public Forward forward(ChannelHandlerContext ctx, Object msg);
+ }
+
+ private static class ForwardWithLatency implements ForwardStrategy
+ {
+ private final long latency;
+ private final TimeUnit timeUnit;
+
+ ForwardWithLatency(long latency, TimeUnit timeUnit)
+ {
+ this.latency = latency;
+ this.timeUnit = timeUnit;
+ }
+
+ public Forward forward(ChannelHandlerContext ctx, Object msg)
+ {
+ return new Forward(System.currentTimeMillis(),
+ timeUnit.toMillis(latency),
+ () -> ctx.fireChannelRead(msg));
+ }
+ }
+
+ private static class CloseAfterRead implements ForwardStrategy
+ {
+ private final Runnable afterClose;
+
+ CloseAfterRead(Runnable afterClose)
+ {
+ this.afterClose = afterClose;
+ }
+
+ public Forward forward(ChannelHandlerContext ctx, Object msg)
+ {
+ return new Forward(System.currentTimeMillis(),
+ 0,
+ () -> {
+ ctx.channel().close().syncUninterruptibly();
+ afterClose.run();
+ });
+ }
+ }
+
+ private static class TransformPayload<T> implements ForwardStrategy
+ {
+ private final Function<T, T> fn;
+
+ TransformPayload(Function<T, T> fn)
+ {
+ this.fn = fn;
+ }
+
+ public Forward forward(ChannelHandlerContext ctx, Object msg)
+ {
+ return new Forward(System.currentTimeMillis(),
+ 0,
+ () -> ctx.fireChannelRead(fn.apply((T) msg)));
+ }
+ }
+
+ public static class Controller
+ {
+ private volatile InboundProxyHandler.ForwardStrategy forwardStrategy;
+ private volatile Runnable onRead = () -> {};
+ private volatile Runnable onDisconnect = () -> {};
+
+ public Controller()
+ {
+ this.forwardStrategy = ForwardNormally.instance;
+ }
+ public void onRead(Runnable onRead)
+ {
+ this.onRead = onRead;
+ }
+
+ public void onDisconnect(Runnable onDisconnect)
+ {
+ this.onDisconnect = onDisconnect;
+ }
+
+ public void reset()
+ {
+ this.forwardStrategy = ForwardNormally.instance;
+ }
+
+ public void withLatency(long latency, TimeUnit timeUnit)
+ {
+ this.forwardStrategy = new ForwardWithLatency(latency, timeUnit);
+ }
+
+ public void withCloseAfterRead(Runnable afterClose)
+ {
+ this.forwardStrategy = new CloseAfterRead(afterClose);
+ }
+
+ public <T> void withPayloadTransform(Function<T, T> fn)
+ {
+ this.forwardStrategy = new TransformPayload<>(fn);
+ }
+ }
+
+}
diff --git a/test/unit/org/apache/cassandra/net/proxy/ProxyHandlerTest.java b/test/unit/org/apache/cassandra/net/proxy/ProxyHandlerTest.java
new file mode 100644
index 0000000..d070f56
--- /dev/null
+++ b/test/unit/org/apache/cassandra/net/proxy/ProxyHandlerTest.java
@@ -0,0 +1,222 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.net.proxy;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.Consumer;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import io.netty.bootstrap.Bootstrap;
+import io.netty.bootstrap.ServerBootstrap;
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.Unpooled;
+import io.netty.channel.Channel;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.channel.ChannelInboundHandlerAdapter;
+import io.netty.channel.ChannelInitializer;
+import io.netty.channel.ChannelOption;
+import io.netty.channel.EventLoopGroup;
+import io.netty.channel.local.LocalAddress;
+import io.netty.channel.local.LocalChannel;
+import io.netty.channel.local.LocalServerChannel;
+import io.netty.channel.nio.NioEventLoopGroup;
+import io.netty.handler.logging.LogLevel;
+import io.netty.handler.logging.LoggingHandler;
+
+public class ProxyHandlerTest
+{
+ private final Object PAYLOAD = new Object();
+
+ @Test
+ public void testLatency() throws Throwable
+ {
+ test((proxyHandler, testHandler, channel) -> {
+ int count = 1;
+ long latency = 100;
+ CountDownLatch latch = new CountDownLatch(count);
+ long start = System.currentTimeMillis();
+ testHandler.onRead = new Consumer<Object>()
+ {
+ int last = -1;
+ public void accept(Object o)
+ {
+ // Make sure that order is preserved
+ Assert.assertEquals(last + 1, o);
+ last = (int) o;
+
+ long elapsed = System.currentTimeMillis() - start;
+ Assert.assertTrue("Latency was:" + elapsed, elapsed > latency);
+ latch.countDown();
+ }
+ };
+
+ proxyHandler.withLatency(latency, TimeUnit.MILLISECONDS);
+
+ for (int i = 0; i < count; i++)
+ {
+ ByteBuf bb = Unpooled.buffer(Integer.BYTES);
+ bb.writeInt(i);
+ channel.writeAndFlush(i);
+ }
+
+ Assert.assertTrue(latch.await(10, TimeUnit.SECONDS));
+ });
+ }
+
+ @Test
+ public void testNormalDelivery() throws Throwable
+ {
+ test((proxyHandler, testHandler, channelPipeline) -> {
+ int count = 10;
+ CountDownLatch latch = new CountDownLatch(count);
+ AtomicLong end = new AtomicLong();
+ testHandler.onRead = (o) -> {
+ end.set(System.currentTimeMillis());
+ latch.countDown();
+ };
+
+ for (int i = 0; i < count; i++)
+ channelPipeline.writeAndFlush(PAYLOAD);
+ Assert.assertTrue(latch.await(10, TimeUnit.SECONDS));
+
+ });
+ }
+
+ @Test
+ public void testLatencyForMany() throws Throwable
+ {
+ class Event {
+ private final long latency;
+ private final long start;
+ private final int idx;
+
+ Event(long latency, int idx)
+ {
+ this.latency = latency;
+ this.start = System.currentTimeMillis();
+ this.idx = idx;
+ }
+ }
+
+ test((proxyHandler, testHandler, channel) -> {
+ int count = 150;
+ CountDownLatch latch = new CountDownLatch(count);
+ AtomicInteger counter = new AtomicInteger();
+ testHandler.onRead = new Consumer<Object>()
+ {
+ int lastSeen = -1;
+ public void accept(Object o)
+ {
+ Event e = (Event) o;
+ Assert.assertEquals(lastSeen + 1, e.idx);
+ lastSeen = e.idx;
+ long elapsed = System.currentTimeMillis() - e.start;
+ Assert.assertTrue(elapsed >= e.latency);
+ counter.incrementAndGet();
+ latch.countDown();
+ }
+ };
+
+ int idx = 0;
+ for (int i = 0; i < count / 3; i++)
+ {
+ for (long latency : new long[]{ 100, 200, 0 })
+ {
+ proxyHandler.withLatency(latency, TimeUnit.MILLISECONDS);
+ CountDownLatch read = new CountDownLatch(1);
+ proxyHandler.onRead(read::countDown);
+ channel.writeAndFlush(new Event(latency, idx++));
+ Assert.assertTrue(read.await(10, TimeUnit.SECONDS));
+ }
+ }
+
+ Assert.assertTrue(latch.await(10, TimeUnit.SECONDS));
+ Assert.assertEquals(counter.get(), count);
+ });
+ }
+
+ private interface DoTest
+ {
+ public void doTest(InboundProxyHandler.Controller proxy, TestHandler testHandler, Channel channel) throws Throwable;
+ }
+
+
+ public void test(DoTest test) throws Throwable
+ {
+ EventLoopGroup serverGroup = new NioEventLoopGroup(1);
+ EventLoopGroup clientGroup = new NioEventLoopGroup(1);
+
+ InboundProxyHandler.Controller controller = new InboundProxyHandler.Controller();
+ InboundProxyHandler proxyHandler = new InboundProxyHandler(controller);
+ TestHandler testHandler = new TestHandler();
+
+ ServerBootstrap sb = new ServerBootstrap();
+ sb.group(serverGroup)
+ .channel(LocalServerChannel.class)
+ .childHandler(new ChannelInitializer<LocalChannel>() {
+ @Override
+ public void initChannel(LocalChannel ch)
+ {
+ ch.pipeline()
+ .addLast(proxyHandler)
+ .addLast(testHandler);
+ }
+ })
+ .childOption(ChannelOption.AUTO_READ, false);
+
+ Bootstrap cb = new Bootstrap();
+ cb.group(clientGroup)
+ .channel(LocalChannel.class)
+ .handler(new ChannelInitializer<LocalChannel>() {
+ @Override
+ public void initChannel(LocalChannel ch) throws Exception {
+ ch.pipeline()
+ .addLast(new LoggingHandler(LogLevel.TRACE));
+ }
+ });
+
+ final LocalAddress addr = new LocalAddress("test");
+
+ Channel serverChannel = sb.bind(addr).sync().channel();
+
+ Channel clientChannel = cb.connect(addr).sync().channel();
+ test.doTest(controller, testHandler, clientChannel);
+
+ clientChannel.close();
+ serverChannel.close();
+ serverGroup.shutdownGracefully();
+ clientGroup.shutdownGracefully();
+ }
+
+
+ public static class TestHandler extends ChannelInboundHandlerAdapter
+ {
+ private Consumer<Object> onRead = (o) -> {};
+ @Override
+ public void channelRead(ChannelHandlerContext ctx, Object msg)
+ {
+ onRead.accept(msg);
+ }
+ }
+}
diff --git a/test/unit/org/apache/cassandra/repair/AbstractRepairTest.java b/test/unit/org/apache/cassandra/repair/AbstractRepairTest.java
new file mode 100644
index 0000000..2c47137
--- /dev/null
+++ b/test/unit/org/apache/cassandra/repair/AbstractRepairTest.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.repair;
+
+import java.net.UnknownHostException;
+import java.util.Set;
+import java.util.UUID;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+
+import org.junit.Ignore;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.service.ActiveRepairService;
+import org.apache.cassandra.streaming.PreviewKind;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.UUIDGen;
+
+@Ignore
+public abstract class AbstractRepairTest
+{
+ public static final InetAddressAndPort COORDINATOR;
+ protected static final InetAddressAndPort PARTICIPANT1;
+ protected static final InetAddressAndPort PARTICIPANT2;
+ protected static final InetAddressAndPort PARTICIPANT3;
+
+ static
+ {
+ try
+ {
+ COORDINATOR = InetAddressAndPort.getByName("10.0.0.1");
+ PARTICIPANT1 = InetAddressAndPort.getByName("10.0.0.1");
+ PARTICIPANT2 = InetAddressAndPort.getByName("10.0.0.2");
+ PARTICIPANT3 = InetAddressAndPort.getByName("10.0.0.3");
+ }
+ catch (UnknownHostException e)
+ {
+
+ throw new AssertionError(e);
+ }
+
+ DatabaseDescriptor.daemonInitialization();
+ }
+
+ protected static final Set<InetAddressAndPort> PARTICIPANTS = ImmutableSet.of(PARTICIPANT1, PARTICIPANT2, PARTICIPANT3);
+
+ protected static Token t(int v)
+ {
+ return DatabaseDescriptor.getPartitioner().getToken(ByteBufferUtil.bytes(v));
+ }
+
+ protected static final Range<Token> RANGE1 = new Range<>(t(1), t(2));
+ protected static final Range<Token> RANGE2 = new Range<>(t(2), t(3));
+ protected static final Range<Token> RANGE3 = new Range<>(t(4), t(5));
+
+ protected static final Set<Range<Token>> ALL_RANGES = ImmutableSet.of(RANGE1, RANGE2, RANGE3);
+
+ public static UUID registerSession(ColumnFamilyStore cfs, boolean isIncremental, boolean isGlobal)
+ {
+ UUID sessionId = UUIDGen.getTimeUUID();
+
+ long repairedAt = isIncremental ? System.currentTimeMillis() : ActiveRepairService.UNREPAIRED_SSTABLE;
+ ActiveRepairService.instance.registerParentRepairSession(sessionId,
+ COORDINATOR,
+ Lists.newArrayList(cfs),
+ Sets.newHashSet(RANGE1, RANGE2, RANGE3),
+ isIncremental,
+ repairedAt,
+ isGlobal,
+ PreviewKind.NONE);
+ return sessionId;
+ }
+}
diff --git a/test/unit/org/apache/cassandra/repair/LocalSyncTaskTest.java b/test/unit/org/apache/cassandra/repair/LocalSyncTaskTest.java
index 7837e6e..443d59e 100644
--- a/test/unit/org/apache/cassandra/repair/LocalSyncTaskTest.java
+++ b/test/unit/org/apache/cassandra/repair/LocalSyncTaskTest.java
@@ -18,54 +18,74 @@
package org.apache.cassandra.repair;
-import java.net.InetAddress;
import java.util.Arrays;
import java.util.HashSet;
import java.util.Set;
import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
import org.junit.BeforeClass;
import org.junit.Test;
import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.Murmur3Partitioner;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.service.ActiveRepairService;
+import org.apache.cassandra.streaming.StreamCoordinator;
+import org.apache.cassandra.streaming.DefaultConnectionFactory;
+import org.apache.cassandra.streaming.StreamPlan;
+import org.apache.cassandra.streaming.PreviewKind;
+import org.apache.cassandra.streaming.StreamSession;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.MerkleTree;
import org.apache.cassandra.utils.MerkleTrees;
+import org.apache.cassandra.utils.UUIDGen;
+
+import static org.apache.cassandra.service.ActiveRepairService.NO_PENDING_REPAIR;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
-public class LocalSyncTaskTest extends SchemaLoader
+public class LocalSyncTaskTest extends AbstractRepairTest
{
- private static final IPartitioner partirioner = Murmur3Partitioner.instance;
+ private static final IPartitioner partitioner = Murmur3Partitioner.instance;
+ private static final InetAddressAndPort local = FBUtilities.getBroadcastAddressAndPort();
public static final String KEYSPACE1 = "DifferencerTest";
public static final String CF_STANDARD = "Standard1";
+ public static ColumnFamilyStore cfs;
@BeforeClass
- public static void defineSchema() throws Exception
+ public static void defineSchema()
{
SchemaLoader.prepareServer();
SchemaLoader.createKeyspace(KEYSPACE1,
KeyspaceParams.simple(1),
SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD));
+
+ TableId tid = Schema.instance.getTableMetadata(KEYSPACE1, CF_STANDARD).id;
+ cfs = Schema.instance.getColumnFamilyStoreInstance(tid);
}
/**
- * When there is no difference between two, LocalSyncTask should return stats with 0 difference.
+ * When there is no difference between two, SymmetricLocalSyncTask should return stats with 0 difference.
*/
@Test
public void testNoDifference() throws Throwable
{
- final InetAddress ep1 = InetAddress.getByName("127.0.0.1");
- final InetAddress ep2 = InetAddress.getByName("127.0.0.1");
+ final InetAddressAndPort ep2 = InetAddressAndPort.getByName("127.0.0.2");
- Range<Token> range = new Range<>(partirioner.getMinimumToken(), partirioner.getRandomToken());
+ Range<Token> range = new Range<>(partitioner.getMinimumToken(), partitioner.getRandomToken());
RepairJobDesc desc = new RepairJobDesc(UUID.randomUUID(), UUID.randomUUID(), KEYSPACE1, "Standard1", Arrays.asList(range));
MerkleTrees tree1 = createInitialTree(desc);
@@ -74,11 +94,10 @@
// difference the trees
// note: we reuse the same endpoint which is bogus in theory but fine here
- TreeResponse r1 = new TreeResponse(ep1, tree1);
+ TreeResponse r1 = new TreeResponse(local, tree1);
TreeResponse r2 = new TreeResponse(ep2, tree2);
- LocalSyncTask task = new LocalSyncTask(desc, r1.endpoint, r2.endpoint,
- MerkleTrees.difference(r1.trees, r2.trees),
- ActiveRepairService.UNREPAIRED_SSTABLE, false);
+ LocalSyncTask task = new LocalSyncTask(desc, r1.endpoint, r2.endpoint, MerkleTrees.difference(r1.trees, r2.trees),
+ NO_PENDING_REPAIR, true, true, PreviewKind.NONE);
task.run();
assertEquals(0, task.get().numberOfDifferences);
@@ -87,21 +106,23 @@
@Test
public void testDifference() throws Throwable
{
- Range<Token> range = new Range<>(partirioner.getMinimumToken(), partirioner.getRandomToken());
+ Range<Token> range = new Range<>(partitioner.getMinimumToken(), partitioner.getRandomToken());
UUID parentRepairSession = UUID.randomUUID();
Keyspace keyspace = Keyspace.open(KEYSPACE1);
ColumnFamilyStore cfs = keyspace.getColumnFamilyStore("Standard1");
- ActiveRepairService.instance.registerParentRepairSession(parentRepairSession, FBUtilities.getBroadcastAddress(), Arrays.asList(cfs), Arrays.asList(range), false, System.currentTimeMillis(), false);
+ ActiveRepairService.instance.registerParentRepairSession(parentRepairSession, FBUtilities.getBroadcastAddressAndPort(),
+ Arrays.asList(cfs), Arrays.asList(range), false,
+ ActiveRepairService.UNREPAIRED_SSTABLE, false,
+ PreviewKind.NONE);
RepairJobDesc desc = new RepairJobDesc(parentRepairSession, UUID.randomUUID(), KEYSPACE1, "Standard1", Arrays.asList(range));
MerkleTrees tree1 = createInitialTree(desc);
-
MerkleTrees tree2 = createInitialTree(desc);
// change a range in one of the trees
- Token token = partirioner.midpoint(range.left, range.right);
+ Token token = partitioner.midpoint(range.left, range.right);
tree1.invalidate(token);
MerkleTree.TreeRange changed = tree1.get(token);
changed.hash("non-empty hash!".getBytes());
@@ -111,26 +132,118 @@
// difference the trees
// note: we reuse the same endpoint which is bogus in theory but fine here
- TreeResponse r1 = new TreeResponse(InetAddress.getByName("127.0.0.1"), tree1);
- TreeResponse r2 = new TreeResponse(InetAddress.getByName("127.0.0.2"), tree2);
- LocalSyncTask task = new LocalSyncTask(desc, r1.endpoint, r2.endpoint,
- MerkleTrees.difference(r1.trees, r2.trees),
- ActiveRepairService.UNREPAIRED_SSTABLE, false);
- task.run();
+ TreeResponse r1 = new TreeResponse(local, tree1);
+ TreeResponse r2 = new TreeResponse(InetAddressAndPort.getByName("127.0.0.2"), tree2);
+ LocalSyncTask task = new LocalSyncTask(desc, r1.endpoint, r2.endpoint, MerkleTrees.difference(r1.trees, r2.trees),
+ NO_PENDING_REPAIR, true, true, PreviewKind.NONE);
+ DefaultConnectionFactory.MAX_CONNECT_ATTEMPTS = 1;
+ try
+ {
+ task.run();
+ }
+ finally
+ {
+ DefaultConnectionFactory.MAX_CONNECT_ATTEMPTS = 3;
+ }
// ensure that the changed range was recorded
- assertEquals("Wrong differing ranges", interesting.size(), task.getCurrentStat().numberOfDifferences);
+ assertEquals("Wrong differing ranges", interesting.size(), task.stat.numberOfDifferences);
+ }
+
+ @Test
+ public void fullRepairStreamPlan() throws Exception
+ {
+ UUID sessionID = registerSession(cfs, true, true);
+ ActiveRepairService.ParentRepairSession prs = ActiveRepairService.instance.getParentRepairSession(sessionID);
+ RepairJobDesc desc = new RepairJobDesc(sessionID, UUIDGen.getTimeUUID(), KEYSPACE1, CF_STANDARD, prs.getRanges());
+
+ TreeResponse r1 = new TreeResponse(local, createInitialTree(desc, DatabaseDescriptor.getPartitioner()));
+ TreeResponse r2 = new TreeResponse(PARTICIPANT2, createInitialTree(desc, DatabaseDescriptor.getPartitioner()));
+
+ LocalSyncTask task = new LocalSyncTask(desc, r1.endpoint, r2.endpoint, MerkleTrees.difference(r1.trees, r2.trees),
+ NO_PENDING_REPAIR, true, true, PreviewKind.NONE);
+ StreamPlan plan = task.createStreamPlan();
+
+ assertEquals(NO_PENDING_REPAIR, plan.getPendingRepair());
+ assertTrue(plan.getFlushBeforeTransfer());
+ }
+
+ private static void assertNumInOut(StreamPlan plan, int expectedIncoming, int expectedOutgoing)
+ {
+ StreamCoordinator coordinator = plan.getCoordinator();
+ StreamSession session = Iterables.getOnlyElement(coordinator.getAllStreamSessions());
+ assertEquals(expectedIncoming, session.getNumRequests());
+ assertEquals(expectedOutgoing, session.getNumTransfers());
+ }
+
+ @Test
+ public void incrementalRepairStreamPlan() throws Exception
+ {
+ UUID sessionID = registerSession(cfs, true, true);
+ ActiveRepairService.ParentRepairSession prs = ActiveRepairService.instance.getParentRepairSession(sessionID);
+ RepairJobDesc desc = new RepairJobDesc(sessionID, UUIDGen.getTimeUUID(), KEYSPACE1, CF_STANDARD, prs.getRanges());
+
+ TreeResponse r1 = new TreeResponse(local, createInitialTree(desc, DatabaseDescriptor.getPartitioner()));
+ TreeResponse r2 = new TreeResponse(PARTICIPANT2, createInitialTree(desc, DatabaseDescriptor.getPartitioner()));
+
+ LocalSyncTask task = new LocalSyncTask(desc, r1.endpoint, r2.endpoint, MerkleTrees.difference(r1.trees, r2.trees),
+ desc.parentSessionId, true, true, PreviewKind.NONE);
+ StreamPlan plan = task.createStreamPlan();
+
+ assertEquals(desc.parentSessionId, plan.getPendingRepair());
+ assertFalse(plan.getFlushBeforeTransfer());
+ assertNumInOut(plan, 1, 1);
+ }
+
+ /**
+ * Don't reciprocate streams if the other endpoint is a transient replica
+ */
+ @Test
+ public void transientRemoteStreamPlan()
+ {
+ UUID sessionID = registerSession(cfs, true, true);
+ ActiveRepairService.ParentRepairSession prs = ActiveRepairService.instance.getParentRepairSession(sessionID);
+ RepairJobDesc desc = new RepairJobDesc(sessionID, UUIDGen.getTimeUUID(), KEYSPACE1, CF_STANDARD, prs.getRanges());
+
+ TreeResponse r1 = new TreeResponse(local, createInitialTree(desc, DatabaseDescriptor.getPartitioner()));
+ TreeResponse r2 = new TreeResponse(PARTICIPANT2, createInitialTree(desc, DatabaseDescriptor.getPartitioner()));
+
+ LocalSyncTask task = new LocalSyncTask(desc, r1.endpoint, r2.endpoint, MerkleTrees.difference(r1.trees, r2.trees),
+ desc.parentSessionId, true, false, PreviewKind.NONE);
+ StreamPlan plan = task.createStreamPlan();
+ assertNumInOut(plan, 1, 0);
+ }
+
+ /**
+ * Don't request streams if the other endpoint is a transient replica
+ */
+ @Test
+ public void transientLocalStreamPlan()
+ {
+ UUID sessionID = registerSession(cfs, true, true);
+ ActiveRepairService.ParentRepairSession prs = ActiveRepairService.instance.getParentRepairSession(sessionID);
+ RepairJobDesc desc = new RepairJobDesc(sessionID, UUIDGen.getTimeUUID(), KEYSPACE1, CF_STANDARD, prs.getRanges());
+
+ TreeResponse r1 = new TreeResponse(local, createInitialTree(desc, DatabaseDescriptor.getPartitioner()));
+ TreeResponse r2 = new TreeResponse(PARTICIPANT2, createInitialTree(desc, DatabaseDescriptor.getPartitioner()));
+
+ LocalSyncTask task = new LocalSyncTask(desc, r1.endpoint, r2.endpoint, MerkleTrees.difference(r1.trees, r2.trees),
+ desc.parentSessionId, false, true, PreviewKind.NONE);
+ StreamPlan plan = task.createStreamPlan();
+ assertNumInOut(plan, 0, 1);
+ }
+
+ private MerkleTrees createInitialTree(RepairJobDesc desc, IPartitioner partitioner)
+ {
+ MerkleTrees tree = new MerkleTrees(partitioner);
+ tree.addMerkleTrees((int) Math.pow(2, 15), desc.ranges);
+ tree.init();
+ return tree;
}
private MerkleTrees createInitialTree(RepairJobDesc desc)
{
- MerkleTrees tree = new MerkleTrees(partirioner);
- tree.addMerkleTrees((int) Math.pow(2, 15), desc.ranges);
- tree.init();
- for (MerkleTree.TreeRange r : tree.invalids())
- {
- r.ensureHashInitialised();
- }
- return tree;
+ return createInitialTree(desc, partitioner);
+
}
}
diff --git a/test/unit/org/apache/cassandra/repair/RepairJobTest.java b/test/unit/org/apache/cassandra/repair/RepairJobTest.java
index e1dd5b3..d3af58f 100644
--- a/test/unit/org/apache/cassandra/repair/RepairJobTest.java
+++ b/test/unit/org/apache/cassandra/repair/RepairJobTest.java
@@ -18,11 +18,9 @@
package org.apache.cassandra.repair;
-import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
@@ -30,11 +28,15 @@
import java.util.Map;
import java.util.Set;
import java.util.UUID;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.function.Predicate;
import java.util.stream.Collectors;
-import com.google.common.util.concurrent.AsyncFunction;
-import com.google.common.util.concurrent.Futures;
+import com.google.common.collect.ImmutableMap;
import com.google.common.util.concurrent.ListenableFuture;
import org.junit.After;
import org.junit.Before;
@@ -43,45 +45,57 @@
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.concurrent.DebuggableThreadPoolExecutor;
-import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.dht.ByteOrderedPartitioner;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.Murmur3Partitioner;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
-import org.apache.cassandra.net.IMessageSink;
-import org.apache.cassandra.net.MessageIn;
-import org.apache.cassandra.net.MessageOut;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.net.Message;
import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.net.Verb;
import org.apache.cassandra.repair.messages.RepairMessage;
import org.apache.cassandra.repair.messages.SyncRequest;
import org.apache.cassandra.schema.KeyspaceParams;
import org.apache.cassandra.service.ActiveRepairService;
+import org.apache.cassandra.streaming.PreviewKind;
+import org.apache.cassandra.streaming.SessionSummary;
+import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.MerkleTree;
import org.apache.cassandra.utils.MerkleTrees;
import org.apache.cassandra.utils.ObjectSizes;
+import org.apache.cassandra.utils.Throwables;
import org.apache.cassandra.utils.UUIDGen;
+import org.apache.cassandra.utils.asserts.SyncTaskListAssert;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
+import static org.apache.cassandra.utils.asserts.SyncTaskAssert.assertThat;
+import static org.apache.cassandra.utils.asserts.SyncTaskListAssert.assertThat;
+import static org.assertj.core.api.Assertions.assertThat;
-public class RepairJobTest extends SchemaLoader
+public class RepairJobTest
{
private static final long TEST_TIMEOUT_S = 10;
private static final long THREAD_TIMEOUT_MILLIS = 100;
+ private static final IPartitioner PARTITIONER = ByteOrderedPartitioner.instance;
private static final IPartitioner MURMUR3_PARTITIONER = Murmur3Partitioner.instance;
private static final String KEYSPACE = "RepairJobTest";
private static final String CF = "Standard1";
- private static final Object messageLock = new Object();
+ private static final Object MESSAGE_LOCK = new Object();
- private static final List<Range<Token>> fullRange = Collections.singletonList(new Range<>(MURMUR3_PARTITIONER.getMinimumToken(),
- MURMUR3_PARTITIONER.getRandomToken()));
- private static InetAddress addr1;
- private static InetAddress addr2;
- private static InetAddress addr3;
- private static InetAddress addr4;
- private RepairSession session;
+ private static final Range<Token> RANGE_1 = range(0, 1);
+ private static final Range<Token> RANGE_2 = range(2, 3);
+ private static final Range<Token> RANGE_3 = range(4, 5);
+ private static final RepairJobDesc JOB_DESC = new RepairJobDesc(UUID.randomUUID(), UUID.randomUUID(), KEYSPACE, CF, Collections.emptyList());
+ private static final List<Range<Token>> FULL_RANGE = Collections.singletonList(new Range<>(MURMUR3_PARTITIONER.getMinimumToken(),
+ MURMUR3_PARTITIONER.getMaximumToken()));
+ private static InetAddressAndPort addr1;
+ private static InetAddressAndPort addr2;
+ private static InetAddressAndPort addr3;
+ private static InetAddressAndPort addr4;
+ private static InetAddressAndPort addr5;
+ private MeasureableRepairSession session;
private RepairJob job;
private RepairJobDesc sessionJobDesc;
@@ -89,11 +103,13 @@
// memory retention from CASSANDRA-14096
private static class MeasureableRepairSession extends RepairSession
{
- public MeasureableRepairSession(UUID parentRepairSession, UUID id, Collection<Range<Token>> ranges,
- String keyspace,RepairParallelism parallelismDegree, Set<InetAddress> endpoints,
- long repairedAt, boolean pullRepair, String... cfnames)
+ private final List<Callable<?>> syncCompleteCallbacks = new ArrayList<>();
+
+ public MeasureableRepairSession(UUID parentRepairSession, UUID id, CommonRange commonRange, String keyspace,
+ RepairParallelism parallelismDegree, boolean isIncremental, boolean pullRepair,
+ boolean force, PreviewKind previewKind, boolean optimiseStreams, String... cfnames)
{
- super(parentRepairSession, id, ranges, keyspace, parallelismDegree, endpoints, repairedAt, pullRepair, cfnames);
+ super(parentRepairSession, id, commonRange, keyspace, parallelismDegree, isIncremental, pullRepair, force, previewKind, optimiseStreams, cfnames);
}
protected DebuggableThreadPoolExecutor createExecutor()
@@ -102,6 +118,28 @@
executor.setKeepAliveTime(THREAD_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS);
return executor;
}
+
+ @Override
+ public void syncComplete(RepairJobDesc desc, SyncNodePair nodes, boolean success, List<SessionSummary> summaries)
+ {
+ for (Callable<?> callback : syncCompleteCallbacks)
+ {
+ try
+ {
+ callback.call();
+ }
+ catch (Exception e)
+ {
+ throw Throwables.cleaned(e);
+ }
+ }
+ super.syncComplete(desc, nodes, success, summaries);
+ }
+
+ public void registerSyncCompleteCallback(Callable<?> callback)
+ {
+ syncCompleteCallbacks.add(callback);
+ }
}
@BeforeClass
@@ -111,72 +149,74 @@
SchemaLoader.createKeyspace(KEYSPACE,
KeyspaceParams.simple(1),
SchemaLoader.standardCFMD(KEYSPACE, CF));
- addr1 = InetAddress.getByName("127.0.0.1");
- addr2 = InetAddress.getByName("127.0.0.2");
- addr3 = InetAddress.getByName("127.0.0.3");
- addr4 = InetAddress.getByName("127.0.0.4");
+ addr1 = InetAddressAndPort.getByName("127.0.0.1");
+ addr2 = InetAddressAndPort.getByName("127.0.0.2");
+ addr3 = InetAddressAndPort.getByName("127.0.0.3");
+ addr4 = InetAddressAndPort.getByName("127.0.0.4");
+ addr5 = InetAddressAndPort.getByName("127.0.0.5");
}
@Before
public void setup()
{
- Set<InetAddress> neighbors = new HashSet<>(Arrays.asList(addr2, addr3));
+ Set<InetAddressAndPort> neighbors = new HashSet<>(Arrays.asList(addr2, addr3));
UUID parentRepairSession = UUID.randomUUID();
- ActiveRepairService.instance.registerParentRepairSession(parentRepairSession, FBUtilities.getBroadcastAddress(),
- Collections.singletonList(Keyspace.open(KEYSPACE).getColumnFamilyStore(CF)), fullRange, false,
- ActiveRepairService.UNREPAIRED_SSTABLE, false);
+ ActiveRepairService.instance.registerParentRepairSession(parentRepairSession, FBUtilities.getBroadcastAddressAndPort(),
+ Collections.singletonList(Keyspace.open(KEYSPACE).getColumnFamilyStore(CF)), FULL_RANGE, false,
+ ActiveRepairService.UNREPAIRED_SSTABLE, false, PreviewKind.NONE);
- this.session = new MeasureableRepairSession(parentRepairSession, UUIDGen.getTimeUUID(), fullRange,
- KEYSPACE, RepairParallelism.SEQUENTIAL, neighbors,
- ActiveRepairService.UNREPAIRED_SSTABLE, false, CF);
+ this.session = new MeasureableRepairSession(parentRepairSession, UUIDGen.getTimeUUID(),
+ new CommonRange(neighbors, Collections.emptySet(), FULL_RANGE),
+ KEYSPACE, RepairParallelism.SEQUENTIAL,
+ false, false, false,
+ PreviewKind.NONE, false, CF);
this.job = new RepairJob(session, CF);
this.sessionJobDesc = new RepairJobDesc(session.parentRepairSession, session.getId(),
- session.keyspace, CF, session.getRanges());
+ session.keyspace, CF, session.ranges());
- DatabaseDescriptor.setBroadcastAddress(addr1);
+ FBUtilities.setBroadcastInetAddress(addr1.address);
}
@After
public void reset()
{
ActiveRepairService.instance.terminateSessions();
- MessagingService.instance().clearMessageSinks();
+ MessagingService.instance().outboundSink.clear();
+ MessagingService.instance().inboundSink.clear();
+ FBUtilities.reset();
}
/**
- * Ensure we can do an end to end repair of consistent data and get the messages we expect
+ * Ensure RepairJob issues the right messages in an end to end repair of consistent data
*/
@Test
- public void testEndToEndNoDifferences() throws Exception
+ public void testEndToEndNoDifferences() throws InterruptedException, ExecutionException, TimeoutException
{
- Map<InetAddress, MerkleTrees> mockTrees = new HashMap<>();
- mockTrees.put(FBUtilities.getBroadcastAddress(), createInitialTree(false));
+ Map<InetAddressAndPort, MerkleTrees> mockTrees = new HashMap<>();
+ mockTrees.put(addr1, createInitialTree(false));
mockTrees.put(addr2, createInitialTree(false));
mockTrees.put(addr3, createInitialTree(false));
- List<MessageOut> observedMessages = new ArrayList<>();
+ List<Message<?>> observedMessages = new ArrayList<>();
interceptRepairMessages(mockTrees, observedMessages);
job.run();
RepairResult result = job.get(TEST_TIMEOUT_S, TimeUnit.SECONDS);
- assertEquals(3, result.stats.size());
- // Should be one RemoteSyncTask left behind (other two should be local)
- assertExpectedDifferences(session.getSyncingTasks().values(), 0);
+ // Since there are no differences, there should be nothing to sync.
+ assertThat(result.stats).hasSize(0);
// RepairJob should send out SNAPSHOTS -> VALIDATIONS -> done
- List<RepairMessage.Type> expectedTypes = new ArrayList<>();
+ List<Verb> expectedTypes = new ArrayList<>();
for (int i = 0; i < 3; i++)
- expectedTypes.add(RepairMessage.Type.SNAPSHOT);
+ expectedTypes.add(Verb.SNAPSHOT_MSG);
for (int i = 0; i < 3; i++)
- expectedTypes.add(RepairMessage.Type.VALIDATION_REQUEST);
+ expectedTypes.add(Verb.VALIDATION_REQ);
- assertEquals(expectedTypes, observedMessages.stream()
- .map(k -> ((RepairMessage) k.payload).messageType)
- .collect(Collectors.toList()));
+ assertThat(observedMessages).extracting(Message::verb).containsExactlyElementsOf(expectedTypes);
}
/**
@@ -186,91 +226,566 @@
@Test
public void testNoTreesRetainedAfterDifference() throws Throwable
{
- Map<InetAddress, MerkleTrees> mockTrees = new HashMap<>();
- mockTrees.put(FBUtilities.getBroadcastAddress(), createInitialTree(false));
- mockTrees.put(addr2, createInitialTree(true));
+ Map<InetAddressAndPort, MerkleTrees> mockTrees = new HashMap<>();
+ mockTrees.put(addr1, createInitialTree(true));
+ mockTrees.put(addr2, createInitialTree(false));
mockTrees.put(addr3, createInitialTree(false));
- List<MessageOut> observedMessages = new ArrayList<>();
- interceptRepairMessages(mockTrees, observedMessages);
-
List<TreeResponse> mockTreeResponses = mockTrees.entrySet().stream()
.map(e -> new TreeResponse(e.getKey(), e.getValue()))
.collect(Collectors.toList());
+ List<Message<?>> messages = new ArrayList<>();
+ interceptRepairMessages(mockTrees, messages);
- long singleTreeSize = ObjectSizes.measureDeep(mockTrees.get(addr2));
+ long singleTreeSize = ObjectSizes.measureDeep(mockTrees.get(addr1));
- // Use a different local address so we get all RemoteSyncs (as LocalSyncs try to reach out over the network).
- List<SyncTask> syncTasks = job.createSyncTasks(mockTreeResponses, addr4);
+ // Use addr4 instead of one of the provided trees to force everything to be remote sync tasks as
+ // LocalSyncTasks try to reach over the network.
+ List<SyncTask> syncTasks = RepairJob.createStandardSyncTasks(sessionJobDesc, mockTreeResponses,
+ addr4, // local
+ noTransient(),
+ session.isIncremental,
+ session.pullRepair,
+ session.previewKind);
// SyncTasks themselves should not contain significant memory
- assertTrue(ObjectSizes.measureDeep(syncTasks) < 0.8 * singleTreeSize);
+ SyncTaskListAssert.assertThat(syncTasks).hasSizeLessThan(0.2 * singleTreeSize);
- ListenableFuture<List<SyncStat>> syncResults = Futures.transform(Futures.immediateFuture(mockTreeResponses), new AsyncFunction<List<TreeResponse>, List<SyncStat>>()
- {
- public ListenableFuture<List<SyncStat>> apply(List<TreeResponse> treeResponses)
- {
- return Futures.allAsList(syncTasks);
- }
- }, session.taskExecutor);
+ // block syncComplete execution until test has verified session still retains the trees
+ CompletableFuture<?> future = new CompletableFuture<>();
+ session.registerSyncCompleteCallback(future::get);
+ ListenableFuture<List<SyncStat>> syncResults = job.executeTasks(syncTasks);
- // The session can retain memory in the contained executor until the threads expire, so we wait for the threads
+ // Immediately following execution the internal execution queue should still retain the trees
+ assertThat(ObjectSizes.measureDeep(session)).isGreaterThan(singleTreeSize);
+ // unblock syncComplete callback, session should remove trees
+ future.complete(null);
+
+ // The session retains memory in the contained executor until the threads expire, so we wait for the threads
// that ran the Tree -> SyncTask conversions to die and release the memory
- int millisUntilFreed;
+ long millisUntilFreed;
for (millisUntilFreed = 0; millisUntilFreed < TEST_TIMEOUT_S * 1000; millisUntilFreed += THREAD_TIMEOUT_MILLIS)
{
// The measured size of the syncingTasks, and result of the computation should be much smaller
+ TimeUnit.MILLISECONDS.sleep(THREAD_TIMEOUT_MILLIS);
if (ObjectSizes.measureDeep(session) < 0.8 * singleTreeSize)
break;
- TimeUnit.MILLISECONDS.sleep(THREAD_TIMEOUT_MILLIS);
}
- assertTrue(millisUntilFreed < TEST_TIMEOUT_S * 1000);
+ assertThat(millisUntilFreed).isLessThan(TEST_TIMEOUT_S * 1000);
List<SyncStat> results = syncResults.get(TEST_TIMEOUT_S, TimeUnit.SECONDS);
- assertTrue(ObjectSizes.measureDeep(results) < 0.8 * singleTreeSize);
+ assertThat(ObjectSizes.measureDeep(results)).isLessThan(Math.round(0.2 * singleTreeSize));
+ assertThat(session.getSyncingTasks()).isEmpty();
- assertEquals(3, results.size());
- // Should be two RemoteSyncTasks with ranges and one empty one
- assertExpectedDifferences(new ArrayList<>(session.getSyncingTasks().values()), 1, 1, 0);
+ assertThat(results)
+ .hasSize(2)
+ .extracting(s -> s.numberOfDifferences)
+ .containsOnly(1L);
- int numDifferent = 0;
- for (SyncStat stat : results)
- {
- if (stat.nodes.endpoint1.equals(addr2) || stat.nodes.endpoint2.equals(addr2))
- {
- assertEquals(1, stat.numberOfDifferences);
- numDifferent++;
- }
- }
- assertEquals(2, numDifferent);
+ assertThat(messages)
+ .hasSize(2)
+ .extracting(Message::verb)
+ .containsOnly(Verb.SYNC_REQ);
}
- private void assertExpectedDifferences(Collection<RemoteSyncTask> tasks, Integer ... differences)
+ @Test
+ public void testCreateStandardSyncTasks()
{
- List<Integer> expectedDifferences = new ArrayList<>(Arrays.asList(differences));
- List<Integer> observedDifferences = tasks.stream()
- .map(t -> (int) t.getCurrentStat().numberOfDifferences)
- .collect(Collectors.toList());
- assertEquals(expectedDifferences.size(), observedDifferences.size());
- assertTrue(expectedDifferences.containsAll(observedDifferences));
+ testCreateStandardSyncTasks(false);
+ }
+
+ @Test
+ public void testCreateStandardSyncTasksPullRepair()
+ {
+ testCreateStandardSyncTasks(true);
+ }
+
+ public static void testCreateStandardSyncTasks(boolean pullRepair)
+ {
+ List<TreeResponse> treeResponses = Arrays.asList(treeResponse(addr1, RANGE_1, "same", RANGE_2, "same", RANGE_3, "same"),
+ treeResponse(addr2, RANGE_1, "different", RANGE_2, "same", RANGE_3, "different"),
+ treeResponse(addr3, RANGE_1, "same", RANGE_2, "same", RANGE_3, "same"));
+
+ Map<SyncNodePair, SyncTask> tasks = toMap(RepairJob.createStandardSyncTasks(JOB_DESC,
+ treeResponses,
+ addr1, // local
+ noTransient(), // transient
+ false,
+ pullRepair,
+ PreviewKind.ALL));
+ assertThat(tasks).hasSize(2);
+
+ assertThat(tasks.get(pair(addr1, addr2)))
+ .isLocal()
+ .isRequestRanges()
+ .hasTransferRanges(!pullRepair)
+ .hasRanges(RANGE_1, RANGE_3);
+
+ assertThat(tasks.get(pair(addr2, addr3)))
+ .isInstanceOf(SymmetricRemoteSyncTask.class)
+ .isNotLocal()
+ .hasRanges(RANGE_1, RANGE_3);
+
+ assertThat(tasks.get(pair(addr1, addr3))).isNull();
+ }
+
+ @Test
+ public void testStandardSyncTransient()
+ {
+ // Do not stream towards transient nodes
+ testStandardSyncTransient(true);
+ testStandardSyncTransient(false);
+ }
+
+ public void testStandardSyncTransient(boolean pullRepair)
+ {
+ List<TreeResponse> treeResponses = Arrays.asList(treeResponse(addr1, RANGE_1, "same", RANGE_2, "same", RANGE_3, "same"),
+ treeResponse(addr2, RANGE_1, "different", RANGE_2, "same", RANGE_3, "different"));
+
+ Map<SyncNodePair, SyncTask> tasks = toMap(RepairJob.createStandardSyncTasks(JOB_DESC,
+ treeResponses,
+ addr1, // local
+ transientPredicate(addr2),
+ false,
+ pullRepair,
+ PreviewKind.ALL));
+
+ assertThat(tasks).hasSize(1);
+
+ assertThat(tasks.get(pair(addr1, addr2)))
+ .isLocal()
+ .isRequestRanges()
+ .hasTransferRanges(false)
+ .hasRanges(RANGE_1, RANGE_3);
+ }
+
+ @Test
+ public void testStandardSyncLocalTransient()
+ {
+ // Do not stream towards transient nodes
+ testStandardSyncLocalTransient(true);
+ testStandardSyncLocalTransient(false);
+ }
+
+ public void testStandardSyncLocalTransient(boolean pullRepair)
+ {
+ List<TreeResponse> treeResponses = Arrays.asList(treeResponse(addr1, RANGE_1, "same", RANGE_2, "same", RANGE_3, "same"),
+ treeResponse(addr2, RANGE_1, "different", RANGE_2, "same", RANGE_3, "different"));
+
+ Map<SyncNodePair, SyncTask> tasks = toMap(RepairJob.createStandardSyncTasks(JOB_DESC,
+ treeResponses,
+ addr1, // local
+ transientPredicate(addr1),
+ false,
+ pullRepair,
+ PreviewKind.ALL));
+
+ if (pullRepair)
+ {
+ assertThat(tasks).isEmpty();
+ return;
+ }
+
+ assertThat(tasks).hasSize(1);
+ assertThat(tasks.get(pair(addr1, addr2)))
+ .isLocal()
+ .isNotRequestRanges()
+ .hasTransferRanges(true)
+ .hasRanges(RANGE_1, RANGE_3);
+
+ }
+
+ @Test
+ public void testEmptyDifference()
+ {
+ // one of the nodes is a local coordinator
+ testEmptyDifference(addr1, noTransient(), true);
+ testEmptyDifference(addr1, noTransient(), false);
+ testEmptyDifference(addr2, noTransient(), true);
+ testEmptyDifference(addr2, noTransient(), false);
+ testEmptyDifference(addr1, transientPredicate(addr1), true);
+ testEmptyDifference(addr2, transientPredicate(addr1), true);
+ testEmptyDifference(addr1, transientPredicate(addr1), false);
+ testEmptyDifference(addr2, transientPredicate(addr1), false);
+ testEmptyDifference(addr1, transientPredicate(addr2), true);
+ testEmptyDifference(addr2, transientPredicate(addr2), true);
+ testEmptyDifference(addr1, transientPredicate(addr2), false);
+ testEmptyDifference(addr2, transientPredicate(addr2), false);
+
+ // nonlocal coordinator
+ testEmptyDifference(addr3, noTransient(), true);
+ testEmptyDifference(addr3, noTransient(), false);
+ testEmptyDifference(addr3, noTransient(), true);
+ testEmptyDifference(addr3, noTransient(), false);
+ testEmptyDifference(addr3, transientPredicate(addr1), true);
+ testEmptyDifference(addr3, transientPredicate(addr1), true);
+ testEmptyDifference(addr3, transientPredicate(addr1), false);
+ testEmptyDifference(addr3, transientPredicate(addr1), false);
+ testEmptyDifference(addr3, transientPredicate(addr2), true);
+ testEmptyDifference(addr3, transientPredicate(addr2), true);
+ testEmptyDifference(addr3, transientPredicate(addr2), false);
+ testEmptyDifference(addr3, transientPredicate(addr2), false);
+ }
+
+ public void testEmptyDifference(InetAddressAndPort local, Predicate<InetAddressAndPort> isTransient, boolean pullRepair)
+ {
+ List<TreeResponse> treeResponses = Arrays.asList(treeResponse(addr1, RANGE_1, "same", RANGE_2, "same", RANGE_3, "same"),
+ treeResponse(addr2, RANGE_1, "same", RANGE_2, "same", RANGE_3, "same"));
+
+ Map<SyncNodePair, SyncTask> tasks = toMap(RepairJob.createStandardSyncTasks(JOB_DESC,
+ treeResponses,
+ local, // local
+ isTransient,
+ false,
+ pullRepair,
+ PreviewKind.ALL));
+
+ assertThat(tasks).isEmpty();
+ }
+
+ @Test
+ public void testCreateStandardSyncTasksAllDifferent()
+ {
+ List<TreeResponse> treeResponses = Arrays.asList(treeResponse(addr1, RANGE_1, "one", RANGE_2, "one", RANGE_3, "one"),
+ treeResponse(addr2, RANGE_1, "two", RANGE_2, "two", RANGE_3, "two"),
+ treeResponse(addr3, RANGE_1, "three", RANGE_2, "three", RANGE_3, "three"));
+
+ Map<SyncNodePair, SyncTask> tasks = toMap(RepairJob.createStandardSyncTasks(JOB_DESC,
+ treeResponses,
+ addr1, // local
+ ep -> ep.equals(addr3), // transient
+ false,
+ true,
+ PreviewKind.ALL));
+
+ assertThat(tasks).hasSize(3);
+
+ assertThat(tasks.get(pair(addr1, addr2)))
+ .isLocal()
+ .hasRanges(RANGE_1, RANGE_2, RANGE_3);
+ assertThat(tasks.get(pair(addr2, addr3)))
+ .isNotLocal()
+ .hasRanges(RANGE_1, RANGE_2, RANGE_3);
+ assertThat(tasks.get(pair(addr1, addr3)))
+ .isLocal()
+ .hasRanges(RANGE_1, RANGE_2, RANGE_3);
+ }
+
+ @Test
+ public void testCreate5NodeStandardSyncTasksWithTransient()
+ {
+ List<TreeResponse> treeResponses = Arrays.asList(treeResponse(addr1, RANGE_1, "one", RANGE_2, "one", RANGE_3, "one"),
+ treeResponse(addr2, RANGE_1, "two", RANGE_2, "two", RANGE_3, "two"),
+ treeResponse(addr3, RANGE_1, "three", RANGE_2, "three", RANGE_3, "three"),
+ treeResponse(addr4, RANGE_1, "four", RANGE_2, "four", RANGE_3, "four"),
+ treeResponse(addr5, RANGE_1, "five", RANGE_2, "five", RANGE_3, "five"));
+
+ Predicate<InetAddressAndPort> isTransient = ep -> ep.equals(addr4) || ep.equals(addr5);
+ Map<SyncNodePair, SyncTask> tasks = toMap(RepairJob.createStandardSyncTasks(JOB_DESC,
+ treeResponses,
+ addr1, // local
+ isTransient, // transient
+ false,
+ true,
+ PreviewKind.ALL));
+
+ SyncNodePair[] pairs = new SyncNodePair[] {pair(addr1, addr2),
+ pair(addr1, addr3),
+ pair(addr1, addr4),
+ pair(addr1, addr5),
+ pair(addr2, addr4),
+ pair(addr2, addr4),
+ pair(addr2, addr5),
+ pair(addr3, addr4),
+ pair(addr3, addr5)};
+
+ for (SyncNodePair pair : pairs)
+ {
+ SyncTask task = tasks.get(pair);
+ // Local only if addr1 is a coordinator
+ assertThat(task)
+ .hasLocal(pair.coordinator.equals(addr1))
+ // All ranges to be synchronised
+ .hasRanges(RANGE_1, RANGE_2, RANGE_3);
+
+ boolean isRemote = !pair.coordinator.equals(addr1) && !pair.peer.equals(addr1);
+ boolean involvesTransient = isTransient.test(pair.coordinator) || isTransient.test(pair.peer);
+
+ assertThat(isRemote && involvesTransient)
+ .withFailMessage("Coordinator: %s\n, Peer: %s\n", pair.coordinator, pair.peer)
+ .isEqualTo(task instanceof AsymmetricRemoteSyncTask);
+ }
+ }
+
+ @Test
+ public void testLocalSyncWithTransient()
+ {
+ for (InetAddressAndPort local : new InetAddressAndPort[]{ addr1, addr2, addr3 })
+ {
+ FBUtilities.reset();
+ FBUtilities.setBroadcastInetAddress(local.address);
+ testLocalSyncWithTransient(local, false);
+ }
+ }
+
+ @Test
+ public void testLocalSyncWithTransientPullRepair()
+ {
+ for (InetAddressAndPort local : new InetAddressAndPort[]{ addr1, addr2, addr3 })
+ {
+ FBUtilities.reset();
+ FBUtilities.setBroadcastInetAddress(local.address);
+ testLocalSyncWithTransient(local, true);
+ }
+ }
+
+ public static void testLocalSyncWithTransient(InetAddressAndPort local, boolean pullRepair)
+ {
+ List<TreeResponse> treeResponses = Arrays.asList(treeResponse(addr1, RANGE_1, "one", RANGE_2, "one", RANGE_3, "one"),
+ treeResponse(addr2, RANGE_1, "two", RANGE_2, "two", RANGE_3, "two"),
+ treeResponse(addr3, RANGE_1, "three", RANGE_2, "three", RANGE_3, "three"),
+ treeResponse(addr4, RANGE_1, "four", RANGE_2, "four", RANGE_3, "four"),
+ treeResponse(addr5, RANGE_1, "five", RANGE_2, "five", RANGE_3, "five"));
+
+ Predicate<InetAddressAndPort> isTransient = ep -> ep.equals(addr4) || ep.equals(addr5);
+ Map<SyncNodePair, SyncTask> tasks = toMap(RepairJob.createStandardSyncTasks(JOB_DESC,
+ treeResponses,
+ local, // local
+ isTransient, // transient
+ false,
+ pullRepair,
+ PreviewKind.ALL));
+
+ assertThat(tasks).hasSize(9);
+ for (InetAddressAndPort addr : new InetAddressAndPort[]{ addr1, addr2, addr3 })
+ {
+ if (local.equals(addr))
+ continue;
+
+ assertThat(tasks.get(pair(local, addr)))
+ .isRequestRanges()
+ .hasTransferRanges(!pullRepair);
+ }
+
+ assertThat(tasks.get(pair(local, addr4)))
+ .isRequestRanges()
+ .hasTransferRanges(false);
+
+ assertThat(tasks.get(pair(local, addr5)))
+ .isRequestRanges()
+ .hasTransferRanges(false);
+ }
+
+ @Test
+ public void testLocalAndRemoteTransient()
+ {
+ testLocalAndRemoteTransient(false);
+ }
+
+ @Test
+ public void testLocalAndRemoteTransientPullRepair()
+ {
+ testLocalAndRemoteTransient(true);
+ }
+
+ private static void testLocalAndRemoteTransient(boolean pullRepair)
+ {
+ FBUtilities.setBroadcastInetAddress(addr4.address);
+ List<TreeResponse> treeResponses = Arrays.asList(treeResponse(addr1, RANGE_1, "one", RANGE_2, "one", RANGE_3, "one"),
+ treeResponse(addr2, RANGE_1, "two", RANGE_2, "two", RANGE_3, "two"),
+ treeResponse(addr3, RANGE_1, "three", RANGE_2, "three", RANGE_3, "three"),
+ treeResponse(addr4, RANGE_1, "four", RANGE_2, "four", RANGE_3, "four"),
+ treeResponse(addr5, RANGE_1, "five", RANGE_2, "five", RANGE_3, "five"));
+
+ Map<SyncNodePair, SyncTask> tasks = toMap(RepairJob.createStandardSyncTasks(JOB_DESC,
+ treeResponses,
+ addr4, // local
+ ep -> ep.equals(addr4) || ep.equals(addr5), // transient
+ false,
+ pullRepair,
+ PreviewKind.ALL));
+
+ assertThat(tasks.get(pair(addr4, addr5))).isNull();
+ }
+
+ @Test
+ public void testOptimizedCreateStandardSyncTasksAllDifferent()
+ {
+ List<TreeResponse> treeResponses = Arrays.asList(treeResponse(addr1, RANGE_1, "one", RANGE_2, "one", RANGE_3, "one"),
+ treeResponse(addr2, RANGE_1, "two", RANGE_2, "two", RANGE_3, "two"),
+ treeResponse(addr3, RANGE_1, "three", RANGE_2, "three", RANGE_3, "three"));
+
+ Map<SyncNodePair, SyncTask> tasks = toMap(RepairJob.createOptimisedSyncingSyncTasks(JOB_DESC,
+ treeResponses,
+ addr1, // local
+ noTransient(),
+ addr -> "DC1",
+ false,
+ PreviewKind.ALL));
+
+ for (SyncNodePair pair : new SyncNodePair[]{ pair(addr1, addr2),
+ pair(addr1, addr3),
+ pair(addr2, addr1),
+ pair(addr2, addr3),
+ pair(addr3, addr1),
+ pair(addr3, addr2) })
+ {
+ assertThat(tasks.get(pair)).hasRanges(RANGE_1, RANGE_2, RANGE_3);
+ }
+ }
+
+ @Test
+ public void testOptimizedCreateStandardSyncTasks()
+ {
+ List<TreeResponse> treeResponses = Arrays.asList(treeResponse(addr1, RANGE_1, "one", RANGE_2, "one"),
+ treeResponse(addr2, RANGE_1, "one", RANGE_2, "two"),
+ treeResponse(addr3, RANGE_1, "three", RANGE_2, "two"));
+
+ Map<SyncNodePair, SyncTask> tasks = toMap(RepairJob.createOptimisedSyncingSyncTasks(JOB_DESC,
+ treeResponses,
+ addr4, // local
+ noTransient(),
+ addr -> "DC1",
+ false,
+ PreviewKind.ALL));
+
+ assertThat(tasks.values()).areAllInstanceOf(AsymmetricRemoteSyncTask.class);
+
+ assertThat(tasks.get(pair(addr1, addr3)).rangesToSync).containsExactly(RANGE_1);
+ // addr1 can get range2 from either addr2 or addr3 but not from both
+ assertStreamRangeFromEither(tasks, RANGE_2, addr1, addr2, addr3);
+
+ assertThat(tasks.get(pair(addr2, addr3)).rangesToSync).containsExactly(RANGE_1);
+ assertThat(tasks.get(pair(addr2, addr1)).rangesToSync).containsExactly(RANGE_2);
+
+ // addr3 can get range1 from either addr1 or addr2 but not from both
+ assertStreamRangeFromEither(tasks, RANGE_1, addr3, addr2, addr1);
+
+ assertThat(tasks.get(pair(addr3, addr1)).rangesToSync).containsExactly(RANGE_2);
+ }
+
+ @Test
+ public void testOptimizedCreateStandardSyncTasksWithTransient()
+ {
+ List<TreeResponse> treeResponses = Arrays.asList(treeResponse(addr1, RANGE_1, "same", RANGE_2, "same", RANGE_3, "same"),
+ treeResponse(addr2, RANGE_1, "different", RANGE_2, "same", RANGE_3, "different"),
+ treeResponse(addr3, RANGE_1, "same", RANGE_2, "same", RANGE_3, "same"));
+
+ RepairJobDesc desc = new RepairJobDesc(UUID.randomUUID(), UUID.randomUUID(), "ks", "cf", Collections.emptyList());
+ Map<SyncNodePair, SyncTask> tasks = toMap(RepairJob.createOptimisedSyncingSyncTasks(desc,
+ treeResponses,
+ addr1, // local
+ ep -> ep.equals(addr3),
+ addr -> "DC1",
+ false,
+ PreviewKind.ALL));
+
+ assertThat(tasks).hasSize(3);
+ SyncTask task = tasks.get(pair(addr1, addr2));
+
+ assertThat(task)
+ .isLocal()
+ .hasRanges(RANGE_1, RANGE_3)
+ .isRequestRanges()
+ .hasTransferRanges(false);
+
+ assertStreamRangeFromEither(tasks, RANGE_3, addr2, addr1, addr3);
+ assertStreamRangeFromEither(tasks, RANGE_1, addr2, addr1, addr3);
+ }
+
+ // Asserts that ranges are streamed from one of the nodes but not from the both
+ public static void assertStreamRangeFromEither(Map<SyncNodePair, SyncTask> tasks, Range<Token> range,
+ InetAddressAndPort target, InetAddressAndPort either, InetAddressAndPort or)
+ {
+ InetAddressAndPort streamsFrom;
+ InetAddressAndPort doesntStreamFrom;
+ if (tasks.containsKey(pair(target, either)) && tasks.get(pair(target, either)).rangesToSync.contains(range))
+ {
+ streamsFrom = either;
+ doesntStreamFrom = or;
+ }
+ else
+ {
+ doesntStreamFrom = either;
+ streamsFrom = or;
+ }
+
+ SyncTask task = tasks.get(pair(target, streamsFrom));
+ assertThat(task).isInstanceOf(AsymmetricRemoteSyncTask.class);
+ assertThat(task.rangesToSync).containsOnly(range);
+ assertDoesntStreamRangeFrom(range, tasks.get(pair(target, doesntStreamFrom)));
+ }
+
+ public static void assertDoesntStreamRangeFrom(Range<Token> range, SyncTask task)
+ {
+ if (task == null)
+ return; // Doesn't stream anything
+
+ assertThat(task.rangesToSync).doesNotContain(range);
+ }
+
+ private static Token tk(int i)
+ {
+ return PARTITIONER.getToken(ByteBufferUtil.bytes(i));
+ }
+
+ private static Range<Token> range(int from, int to)
+ {
+ return new Range<>(tk(from), tk(to));
+ }
+
+ private static TreeResponse treeResponse(InetAddressAndPort addr, Object... rangesAndHashes)
+ {
+ MerkleTrees trees = new MerkleTrees(PARTITIONER);
+ for (int i = 0; i < rangesAndHashes.length; i += 2)
+ {
+ Range<Token> range = (Range<Token>) rangesAndHashes[i];
+ String hash = (String) rangesAndHashes[i + 1];
+ MerkleTree tree = trees.addMerkleTree(2, MerkleTree.RECOMMENDED_DEPTH, range);
+ tree.get(range.left).hash(hash.getBytes());
+ }
+
+ return new TreeResponse(addr, trees);
+ }
+
+ private static SyncNodePair pair(InetAddressAndPort node1, InetAddressAndPort node2)
+ {
+ return new SyncNodePair(node1, node2);
+ }
+
+ public static Map<SyncNodePair, SyncTask> toMap(List<SyncTask> tasks)
+ {
+ ImmutableMap.Builder<SyncNodePair, SyncTask> map = ImmutableMap.builder();
+ tasks.forEach(t -> map.put(t.nodePair, t));
+ return map.build();
+ }
+
+ public static Predicate<InetAddressAndPort> transientPredicate(InetAddressAndPort... transientNodes)
+ {
+ Set<InetAddressAndPort> set = new HashSet<>();
+ for (InetAddressAndPort node : transientNodes)
+ set.add(node);
+
+ return set::contains;
+ }
+
+ public static Predicate<InetAddressAndPort> noTransient()
+ {
+ return node -> false;
}
private MerkleTrees createInitialTree(boolean invalidate)
{
MerkleTrees tree = new MerkleTrees(MURMUR3_PARTITIONER);
- tree.addMerkleTrees((int) Math.pow(2, 15), fullRange);
+ tree.addMerkleTrees((int) Math.pow(2, 15), FULL_RANGE);
tree.init();
- for (MerkleTree.TreeRange r : tree.invalids())
- {
- r.ensureHashInitialised();
- }
if (invalidate)
{
// change a range in one of the trees
- Token token = MURMUR3_PARTITIONER.midpoint(fullRange.get(0).left, fullRange.get(0).right);
+ Token token = MURMUR3_PARTITIONER.midpoint(FULL_RANGE.get(0).left, FULL_RANGE.get(0).right);
tree.invalidate(token);
tree.get(token).hash("non-empty hash!".getBytes());
}
@@ -278,49 +793,37 @@
return tree;
}
- private void interceptRepairMessages(Map<InetAddress, MerkleTrees> mockTrees,
- List<MessageOut> messageCapture)
+ private void interceptRepairMessages(Map<InetAddressAndPort, MerkleTrees> mockTrees,
+ List<Message<?>> messageCapture)
{
- MessagingService.instance().addMessageSink(new IMessageSink()
- {
- public boolean allowOutgoingMessage(MessageOut message, int id, InetAddress to)
- {
- if (message == null || !(message.payload instanceof RepairMessage))
- return false;
-
- // So different Thread's messages don't overwrite each other.
- synchronized (messageLock)
- {
- messageCapture.add(message);
- }
-
- RepairMessage rm = (RepairMessage) message.payload;
- switch (rm.messageType)
- {
- case SNAPSHOT:
- MessageIn<?> messageIn = MessageIn.create(to, null,
- Collections.emptyMap(),
- MessagingService.Verb.REQUEST_RESPONSE,
- MessagingService.current_version);
- MessagingService.instance().receive(messageIn, id);
- break;
- case VALIDATION_REQUEST:
- session.validationComplete(sessionJobDesc, to, mockTrees.get(to));
- break;
- case SYNC_REQUEST:
- SyncRequest syncRequest = (SyncRequest) rm;
- session.syncComplete(sessionJobDesc, new NodePair(syncRequest.src, syncRequest.dst), true);
- break;
- default:
- break;
- }
+ MessagingService.instance().inboundSink.add(message -> message.verb().isResponse());
+ MessagingService.instance().outboundSink.add((message, to) -> {
+ if (message == null || !(message.payload instanceof RepairMessage))
return false;
+
+ // So different Thread's messages don't overwrite each other.
+ synchronized (MESSAGE_LOCK)
+ {
+ messageCapture.add(message);
}
- public boolean allowIncomingMessage(MessageIn message, int id)
+ switch (message.verb())
{
- return message.verb == MessagingService.Verb.REQUEST_RESPONSE;
+ case SNAPSHOT_MSG:
+ MessagingService.instance().callbacks.removeAndRespond(message.id(), to, message.emptyResponse());
+ break;
+ case VALIDATION_REQ:
+ session.validationComplete(sessionJobDesc, to, mockTrees.get(to));
+ break;
+ case SYNC_REQ:
+ SyncRequest syncRequest = (SyncRequest) message.payload;
+ session.syncComplete(sessionJobDesc, new SyncNodePair(syncRequest.src, syncRequest.dst),
+ true, Collections.emptyList());
+ break;
+ default:
+ break;
}
+ return false;
});
}
}
diff --git a/test/unit/org/apache/cassandra/repair/RepairRunnableTest.java b/test/unit/org/apache/cassandra/repair/RepairRunnableTest.java
new file mode 100644
index 0000000..418d7de
--- /dev/null
+++ b/test/unit/org/apache/cassandra/repair/RepairRunnableTest.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.repair;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+import org.junit.Assert;
+import org.junit.Test;
+
+import org.apache.cassandra.locator.InetAddressAndPort;
+
+import static org.apache.cassandra.repair.RepairRunnable.filterCommonRanges;
+
+public class RepairRunnableTest extends AbstractRepairTest
+{
+ /**
+ * For non-forced repairs, common ranges should be passed through as-is
+ */
+ @Test
+ public void filterCommonIncrementalRangesNotForced() throws Exception
+ {
+ CommonRange cr = new CommonRange(PARTICIPANTS, Collections.emptySet(), ALL_RANGES);
+
+ List<CommonRange> expected = Lists.newArrayList(cr);
+ List<CommonRange> actual = filterCommonRanges(expected, Collections.emptySet(), false);
+
+ Assert.assertEquals(expected, actual);
+ }
+
+ @Test
+ public void forceFilterCommonIncrementalRanges() throws Exception
+ {
+ CommonRange cr1 = new CommonRange(Sets.newHashSet(PARTICIPANT1, PARTICIPANT2), Collections.emptySet(), Sets.newHashSet(RANGE1, RANGE2));
+ CommonRange cr2 = new CommonRange(Sets.newHashSet(PARTICIPANT1, PARTICIPANT2, PARTICIPANT3), Collections.emptySet(), Sets.newHashSet(RANGE3));
+ Set<InetAddressAndPort> liveEndpoints = Sets.newHashSet(PARTICIPANT2, PARTICIPANT3); // PARTICIPANT1 is excluded
+
+ List<CommonRange> initial = Lists.newArrayList(cr1, cr2);
+ List<CommonRange> expected = Lists.newArrayList(new CommonRange(Sets.newHashSet(PARTICIPANT2), Collections.emptySet(), Sets.newHashSet(RANGE1, RANGE2)),
+ new CommonRange(Sets.newHashSet(PARTICIPANT2, PARTICIPANT3), Collections.emptySet(), Sets.newHashSet(RANGE3)));
+ List<CommonRange> actual = filterCommonRanges(initial, liveEndpoints, true);
+
+ Assert.assertEquals(expected, actual);
+ }
+}
diff --git a/test/unit/org/apache/cassandra/repair/RepairSessionTest.java b/test/unit/org/apache/cassandra/repair/RepairSessionTest.java
index f65bedb..e77d657 100644
--- a/test/unit/org/apache/cassandra/repair/RepairSessionTest.java
+++ b/test/unit/org/apache/cassandra/repair/RepairSessionTest.java
@@ -19,8 +19,8 @@
package org.apache.cassandra.repair;
import java.io.IOException;
-import java.net.InetAddress;
import java.util.Arrays;
+import java.util.Collections;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ExecutionException;
@@ -35,7 +35,8 @@
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.gms.Gossiper;
-import org.apache.cassandra.service.ActiveRepairService;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.streaming.PreviewKind;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.UUIDGen;
@@ -53,7 +54,7 @@
@Test
public void testConviction() throws Exception
{
- InetAddress remote = InetAddress.getByName("127.0.0.2");
+ InetAddressAndPort remote = InetAddressAndPort.getByName("127.0.0.2");
Gossiper.instance.initializeNodeUnsafe(remote, UUID.randomUUID(), 1);
// Set up RepairSession
@@ -61,8 +62,12 @@
UUID sessionId = UUID.randomUUID();
IPartitioner p = Murmur3Partitioner.instance;
Range<Token> repairRange = new Range<>(p.getToken(ByteBufferUtil.bytes(0)), p.getToken(ByteBufferUtil.bytes(100)));
- Set<InetAddress> endpoints = Sets.newHashSet(remote);
- RepairSession session = new RepairSession(parentSessionId, sessionId, Arrays.asList(repairRange), "Keyspace1", RepairParallelism.SEQUENTIAL, endpoints, ActiveRepairService.UNREPAIRED_SSTABLE, false, "Standard1");
+ Set<InetAddressAndPort> endpoints = Sets.newHashSet(remote);
+ RepairSession session = new RepairSession(parentSessionId, sessionId,
+ new CommonRange(endpoints, Collections.emptySet(), Arrays.asList(repairRange)),
+ "Keyspace1", RepairParallelism.SEQUENTIAL,
+ false, false, false,
+ PreviewKind.NONE, false, "Standard1");
// perform convict
session.convict(remote, Double.MAX_VALUE);
diff --git a/test/unit/org/apache/cassandra/repair/StreamingRepairTaskTest.java b/test/unit/org/apache/cassandra/repair/StreamingRepairTaskTest.java
new file mode 100644
index 0000000..ea5ebbf
--- /dev/null
+++ b/test/unit/org/apache/cassandra/repair/StreamingRepairTaskTest.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.repair;
+
+import java.util.UUID;
+
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.cql3.statements.schema.CreateTableStatement;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.repair.messages.SyncRequest;
+import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.service.ActiveRepairService;
+import org.apache.cassandra.streaming.PreviewKind;
+import org.apache.cassandra.streaming.StreamPlan;
+import org.apache.cassandra.utils.UUIDGen;
+
+public class StreamingRepairTaskTest extends AbstractRepairTest
+{
+ protected String ks;
+ protected final String tbl = "tbl";
+ protected TableMetadata cfm;
+ protected ColumnFamilyStore cfs;
+
+ @BeforeClass
+ public static void setupClass()
+ {
+ SchemaLoader.prepareServer();
+ }
+
+ @Before
+ public void setup()
+ {
+ ks = "ks_" + System.currentTimeMillis();
+ cfm = CreateTableStatement.parse(String.format("CREATE TABLE %s.%s (k INT PRIMARY KEY, v INT)", ks, tbl), ks).build();
+ SchemaLoader.createKeyspace(ks, KeyspaceParams.simple(1), cfm);
+ cfs = Schema.instance.getColumnFamilyStoreInstance(cfm.id);
+ }
+
+ @Test
+ public void incrementalStreamPlan()
+ {
+ UUID sessionID = registerSession(cfs, true, true);
+ ActiveRepairService.ParentRepairSession prs = ActiveRepairService.instance.getParentRepairSession(sessionID);
+ RepairJobDesc desc = new RepairJobDesc(sessionID, UUIDGen.getTimeUUID(), ks, tbl, prs.getRanges());
+
+ SyncRequest request = new SyncRequest(desc, PARTICIPANT1, PARTICIPANT2, PARTICIPANT3, prs.getRanges(), PreviewKind.NONE);
+ StreamingRepairTask task = new StreamingRepairTask(desc, request.initiator, request.src, request.dst, request.ranges, desc.sessionId, PreviewKind.NONE, false);
+
+ StreamPlan plan = task.createStreamPlan(request.dst);
+ Assert.assertFalse(plan.getFlushBeforeTransfer());
+ }
+
+ @Test
+ public void fullStreamPlan() throws Exception
+ {
+ UUID sessionID = registerSession(cfs, false, true);
+ ActiveRepairService.ParentRepairSession prs = ActiveRepairService.instance.getParentRepairSession(sessionID);
+ RepairJobDesc desc = new RepairJobDesc(sessionID, UUIDGen.getTimeUUID(), ks, tbl, prs.getRanges());
+ SyncRequest request = new SyncRequest(desc, PARTICIPANT1, PARTICIPANT2, PARTICIPANT3, prs.getRanges(), PreviewKind.NONE);
+ StreamingRepairTask task = new StreamingRepairTask(desc, request.initiator, request.src, request.dst, request.ranges, null, PreviewKind.NONE, false);
+
+ StreamPlan plan = task.createStreamPlan(request.dst);
+ Assert.assertTrue(plan.getFlushBeforeTransfer());
+ }
+}
diff --git a/test/unit/org/apache/cassandra/repair/SymmetricRemoteSyncTaskTest.java b/test/unit/org/apache/cassandra/repair/SymmetricRemoteSyncTaskTest.java
new file mode 100644
index 0000000..cba64ae
--- /dev/null
+++ b/test/unit/org/apache/cassandra/repair/SymmetricRemoteSyncTaskTest.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.repair;
+
+import java.util.List;
+
+import com.google.common.collect.ImmutableList;
+import org.junit.Assert;
+import org.junit.Test;
+
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.repair.messages.RepairMessage;
+import org.apache.cassandra.repair.messages.SyncRequest;
+import org.apache.cassandra.streaming.PreviewKind;
+import org.apache.cassandra.utils.MerkleTree;
+import org.apache.cassandra.utils.UUIDGen;
+
+public class SymmetricRemoteSyncTaskTest extends AbstractRepairTest
+{
+ private static final RepairJobDesc DESC = new RepairJobDesc(UUIDGen.getTimeUUID(), UUIDGen.getTimeUUID(), "ks", "tbl", ALL_RANGES);
+ private static final List<Range<Token>> RANGE_LIST = ImmutableList.of(RANGE1);
+ private static class InstrumentedSymmetricRemoteSyncTask extends SymmetricRemoteSyncTask
+ {
+ public InstrumentedSymmetricRemoteSyncTask(InetAddressAndPort e1, InetAddressAndPort e2)
+ {
+ super(DESC, e1, e2, RANGE_LIST, PreviewKind.NONE);
+ }
+
+ RepairMessage sentMessage = null;
+ InetAddressAndPort sentTo = null;
+
+ @Override
+ void sendRequest(SyncRequest request, InetAddressAndPort to)
+ {
+ Assert.assertNull(sentMessage);
+ Assert.assertNotNull(request);
+ Assert.assertNotNull(to);
+ sentMessage = request;
+ sentTo = to;
+ }
+ }
+
+ @Test
+ public void normalSync()
+ {
+ InstrumentedSymmetricRemoteSyncTask syncTask = new InstrumentedSymmetricRemoteSyncTask(PARTICIPANT1, PARTICIPANT2);
+ syncTask.startSync();
+
+ Assert.assertNotNull(syncTask.sentMessage);
+ Assert.assertSame(SyncRequest.class, syncTask.sentMessage.getClass());
+ Assert.assertEquals(PARTICIPANT1, syncTask.sentTo);
+ }
+}
diff --git a/test/unit/org/apache/cassandra/repair/ValidatorTest.java b/test/unit/org/apache/cassandra/repair/ValidatorTest.java
index 9c32cef..cf3411a 100644
--- a/test/unit/org/apache/cassandra/repair/ValidatorTest.java
+++ b/test/unit/org/apache/cassandra/repair/ValidatorTest.java
@@ -17,27 +17,28 @@
*/
package org.apache.cassandra.repair;
-import java.net.InetAddress;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Iterator;
+import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.TimeUnit;
-import com.google.common.util.concurrent.ListenableFuture;
-import com.google.common.util.concurrent.SettableFuture;
-
-import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.compaction.CompactionsTest;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.junit.After;
+import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import org.apache.cassandra.SchemaLoader;
-import org.apache.cassandra.config.Schema;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.net.Verb;
+import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.db.BufferDecoratedKey;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.EmptyIterators;
@@ -45,20 +46,19 @@
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
-import org.apache.cassandra.net.IMessageSink;
-import org.apache.cassandra.net.MessageIn;
-import org.apache.cassandra.net.MessageOut;
+import org.apache.cassandra.net.Message;
import org.apache.cassandra.net.MessagingService;
-import org.apache.cassandra.repair.messages.RepairMessage;
-import org.apache.cassandra.repair.messages.ValidationComplete;
+import org.apache.cassandra.repair.messages.ValidationResponse;
import org.apache.cassandra.schema.KeyspaceParams;
import org.apache.cassandra.service.ActiveRepairService;
+import org.apache.cassandra.streaming.PreviewKind;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.MerkleTree;
import org.apache.cassandra.utils.MerkleTrees;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.UUIDGen;
+import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
@@ -68,6 +68,7 @@
public class ValidatorTest
{
private static final long TEST_TIMEOUT = 60; //seconds
+ private static int testSizeMegabytes;
private static final String keyspace = "ValidatorTest";
private static final String columnFamily = "Standard1";
@@ -80,13 +81,21 @@
SchemaLoader.createKeyspace(keyspace,
KeyspaceParams.simple(1),
SchemaLoader.standardCFMD(keyspace, columnFamily));
- partitioner = Schema.instance.getCFMetaData(keyspace, columnFamily).partitioner;
+ partitioner = Schema.instance.getTableMetadata(keyspace, columnFamily).partitioner;
+ testSizeMegabytes = DatabaseDescriptor.getRepairSessionSpaceInMegabytes();
}
@After
public void tearDown()
{
- MessagingService.instance().clearMessageSinks();
+ MessagingService.instance().outboundSink.clear();
+ DatabaseDescriptor.setRepairSessionSpaceInMegabytes(testSizeMegabytes);
+ }
+
+ @Before
+ public void setup()
+ {
+ DatabaseDescriptor.setRepairSessionSpaceInMegabytes(testSizeMegabytes);
}
@Test
@@ -95,13 +104,13 @@
Range<Token> range = new Range<>(partitioner.getMinimumToken(), partitioner.getRandomToken());
final RepairJobDesc desc = new RepairJobDesc(UUID.randomUUID(), UUID.randomUUID(), keyspace, columnFamily, Arrays.asList(range));
- final CompletableFuture<MessageOut> outgoingMessageSink = registerOutgoingMessageSink();
+ final CompletableFuture<Message> outgoingMessageSink = registerOutgoingMessageSink();
- InetAddress remote = InetAddress.getByName("127.0.0.2");
+ InetAddressAndPort remote = InetAddressAndPort.getByName("127.0.0.2");
ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(columnFamily);
- Validator validator = new Validator(desc, remote, 0);
+ Validator validator = new Validator(desc, remote, 0, PreviewKind.NONE);
MerkleTrees tree = new MerkleTrees(partitioner);
tree.addMerkleTrees((int) Math.pow(2, 15), validator.desc.ranges);
validator.prepare(cfs, tree);
@@ -111,20 +120,19 @@
// add a row
Token mid = partitioner.midpoint(range.left, range.right);
- validator.add(EmptyIterators.unfilteredRow(cfs.metadata, new BufferDecoratedKey(mid, ByteBufferUtil.bytes("inconceivable!")), false));
+ validator.add(EmptyIterators.unfilteredRow(cfs.metadata(), new BufferDecoratedKey(mid, ByteBufferUtil.bytes("inconceivable!")), false));
validator.complete();
// confirm that the tree was validated
Token min = tree.partitioner().getMinimumToken();
assertNotNull(tree.hash(new Range<>(min, min)));
- MessageOut message = outgoingMessageSink.get(TEST_TIMEOUT, TimeUnit.SECONDS);
- assertEquals(MessagingService.Verb.REPAIR_MESSAGE, message.verb);
- RepairMessage m = (RepairMessage) message.payload;
- assertEquals(RepairMessage.Type.VALIDATION_COMPLETE, m.messageType);
+ Message message = outgoingMessageSink.get(TEST_TIMEOUT, TimeUnit.SECONDS);
+ assertEquals(Verb.VALIDATION_RSP, message.verb());
+ ValidationResponse m = (ValidationResponse) message.payload;
assertEquals(desc, m.desc);
- assertTrue(((ValidationComplete) m).success());
- assertNotNull(((ValidationComplete) m).trees);
+ assertTrue(m.success());
+ assertNotNull(m.trees);
}
@@ -134,20 +142,19 @@
Range<Token> range = new Range<>(partitioner.getMinimumToken(), partitioner.getRandomToken());
final RepairJobDesc desc = new RepairJobDesc(UUID.randomUUID(), UUID.randomUUID(), keyspace, columnFamily, Arrays.asList(range));
- final CompletableFuture<MessageOut> outgoingMessageSink = registerOutgoingMessageSink();
+ final CompletableFuture<Message> outgoingMessageSink = registerOutgoingMessageSink();
- InetAddress remote = InetAddress.getByName("127.0.0.2");
+ InetAddressAndPort remote = InetAddressAndPort.getByName("127.0.0.2");
- Validator validator = new Validator(desc, remote, 0);
+ Validator validator = new Validator(desc, remote, 0, PreviewKind.NONE);
validator.fail();
- MessageOut message = outgoingMessageSink.get(TEST_TIMEOUT, TimeUnit.SECONDS);
- assertEquals(MessagingService.Verb.REPAIR_MESSAGE, message.verb);
- RepairMessage m = (RepairMessage) message.payload;
- assertEquals(RepairMessage.Type.VALIDATION_COMPLETE, m.messageType);
+ Message message = outgoingMessageSink.get(TEST_TIMEOUT, TimeUnit.SECONDS);
+ assertEquals(Verb.VALIDATION_RSP, message.verb());
+ ValidationResponse m = (ValidationResponse) message.payload;
assertEquals(desc, m.desc);
- assertFalse(((ValidationComplete) m).success());
- assertNull(((ValidationComplete) m).trees);
+ assertFalse(m.success());
+ assertNull(m.trees);
}
@Test
@@ -188,49 +195,184 @@
SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
UUID repairSessionId = UUIDGen.getTimeUUID();
final RepairJobDesc desc = new RepairJobDesc(repairSessionId, UUIDGen.getTimeUUID(), cfs.keyspace.getName(),
- cfs.getColumnFamilyName(), Collections.singletonList(new Range<>(sstable.first.getToken(),
- sstable.last.getToken())));
+ cfs.getTableName(), Collections.singletonList(new Range<>(sstable.first.getToken(),
+ sstable.last.getToken())));
- ActiveRepairService.instance.registerParentRepairSession(repairSessionId, FBUtilities.getBroadcastAddress(),
+ InetAddressAndPort host = InetAddressAndPort.getByName("127.0.0.2");
+
+ ActiveRepairService.instance.registerParentRepairSession(repairSessionId, host,
Collections.singletonList(cfs), desc.ranges, false, ActiveRepairService.UNREPAIRED_SSTABLE,
- false);
+ false, PreviewKind.NONE);
- final CompletableFuture<MessageOut> outgoingMessageSink = registerOutgoingMessageSink();
- Validator validator = new Validator(desc, FBUtilities.getBroadcastAddress(), 0, true);
- CompactionManager.instance.submitValidation(cfs, validator);
+ final CompletableFuture<Message> outgoingMessageSink = registerOutgoingMessageSink();
+ Validator validator = new Validator(desc, host, 0, true, false, PreviewKind.NONE);
+ ValidationManager.instance.submitValidation(cfs, validator);
- MessageOut message = outgoingMessageSink.get(TEST_TIMEOUT, TimeUnit.SECONDS);
- assertEquals(MessagingService.Verb.REPAIR_MESSAGE, message.verb);
- RepairMessage m = (RepairMessage) message.payload;
- assertEquals(RepairMessage.Type.VALIDATION_COMPLETE, m.messageType);
+ Message message = outgoingMessageSink.get(TEST_TIMEOUT, TimeUnit.SECONDS);
+ assertEquals(Verb.VALIDATION_RSP, message.verb());
+ ValidationResponse m = (ValidationResponse) message.payload;
assertEquals(desc, m.desc);
- assertTrue(((ValidationComplete) m).success());
- MerkleTrees trees = ((ValidationComplete) m).trees;
+ assertTrue(m.success());
- Iterator<Map.Entry<Range<Token>, MerkleTree>> iterator = trees.iterator();
+ Iterator<Map.Entry<Range<Token>, MerkleTree>> iterator = m.trees.iterator();
while (iterator.hasNext())
{
assertEquals(Math.pow(2, Math.ceil(Math.log(n) / Math.log(2))), iterator.next().getValue().size(), 0.0);
}
- assertEquals(trees.rowCount(), n);
+ assertEquals(m.trees.rowCount(), n);
}
- private CompletableFuture<MessageOut> registerOutgoingMessageSink()
+ /*
+ * Test for CASSANDRA-14096 size limiting. We:
+ * 1. Limit the size of a repair session
+ * 2. Submit a validation
+ * 3. Check that the resulting tree is of limited depth
+ */
+ @Test
+ public void testSizeLimiting() throws Exception
{
- final CompletableFuture<MessageOut> future = new CompletableFuture<>();
- MessagingService.instance().addMessageSink(new IMessageSink()
- {
- public boolean allowOutgoingMessage(MessageOut message, int id, InetAddress to)
- {
- future.complete(message);
- return false;
- }
+ Keyspace ks = Keyspace.open(keyspace);
+ ColumnFamilyStore cfs = ks.getColumnFamilyStore(columnFamily);
+ cfs.clearUnsafe();
- public boolean allowIncomingMessage(MessageIn message, int id)
- {
- return false;
- }
- });
+ DatabaseDescriptor.setRepairSessionSpaceInMegabytes(1);
+
+ // disable compaction while flushing
+ cfs.disableAutoCompaction();
+
+ // 2 ** 14 rows would normally use 2^14 leaves, but with only 1 meg we should only use 2^12
+ CompactionsTest.populate(keyspace, columnFamily, 0, 1 << 14, 0);
+
+ cfs.forceBlockingFlush();
+ assertEquals(1, cfs.getLiveSSTables().size());
+
+ // wait enough to force single compaction
+ TimeUnit.SECONDS.sleep(5);
+
+ SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
+ UUID repairSessionId = UUIDGen.getTimeUUID();
+ final RepairJobDesc desc = new RepairJobDesc(repairSessionId, UUIDGen.getTimeUUID(), cfs.keyspace.getName(),
+ cfs.getTableName(), Collections.singletonList(new Range<>(sstable.first.getToken(),
+ sstable.last.getToken())));
+
+ InetAddressAndPort host = InetAddressAndPort.getByName("127.0.0.2");
+
+ ActiveRepairService.instance.registerParentRepairSession(repairSessionId, host,
+ Collections.singletonList(cfs), desc.ranges, false, ActiveRepairService.UNREPAIRED_SSTABLE,
+ false, PreviewKind.NONE);
+
+ final CompletableFuture<Message> outgoingMessageSink = registerOutgoingMessageSink();
+ Validator validator = new Validator(desc, host, 0, true, false, PreviewKind.NONE);
+ ValidationManager.instance.submitValidation(cfs, validator);
+
+ Message message = outgoingMessageSink.get(TEST_TIMEOUT, TimeUnit.SECONDS);
+ MerkleTrees trees = ((ValidationResponse) message.payload).trees;
+
+ Iterator<Map.Entry<Range<Token>, MerkleTree>> iterator = trees.iterator();
+ int numTrees = 0;
+ while (iterator.hasNext())
+ {
+ assertEquals(1 << 12, iterator.next().getValue().size(), 0.0);
+ numTrees++;
+ }
+ assertEquals(1, numTrees);
+
+ assertEquals(trees.rowCount(), 1 << 14);
+ }
+
+ /*
+ * Test for CASSANDRA-11390. When there are multiple subranges the trees should
+ * automatically size down to make each subrange fit in the provided memory
+ * 1. Limit the size of all the trees
+ * 2. Submit a validation against more than one range
+ * 3. Check that we have the right number and sizes of trees
+ */
+ @Test
+ public void testRangeSplittingTreeSizeLimit() throws Exception
+ {
+ Keyspace ks = Keyspace.open(keyspace);
+ ColumnFamilyStore cfs = ks.getColumnFamilyStore(columnFamily);
+ cfs.clearUnsafe();
+
+ DatabaseDescriptor.setRepairSessionSpaceInMegabytes(1);
+
+ // disable compaction while flushing
+ cfs.disableAutoCompaction();
+
+ // 2 ** 14 rows would normally use 2^14 leaves, but with only 1 meg we should only use 2^12
+ CompactionsTest.populate(keyspace, columnFamily, 0, 1 << 14, 0);
+
+ cfs.forceBlockingFlush();
+ assertEquals(1, cfs.getLiveSSTables().size());
+
+ // wait enough to force single compaction
+ TimeUnit.SECONDS.sleep(5);
+
+ SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
+ UUID repairSessionId = UUIDGen.getTimeUUID();
+
+ List<Range<Token>> ranges = splitHelper(new Range<>(sstable.first.getToken(), sstable.last.getToken()), 2);
+
+
+ final RepairJobDesc desc = new RepairJobDesc(repairSessionId, UUIDGen.getTimeUUID(), cfs.keyspace.getName(),
+ cfs.getTableName(), ranges);
+
+ InetAddressAndPort host = InetAddressAndPort.getByName("127.0.0.2");
+
+ ActiveRepairService.instance.registerParentRepairSession(repairSessionId, host,
+ Collections.singletonList(cfs), desc.ranges, false, ActiveRepairService.UNREPAIRED_SSTABLE,
+ false, PreviewKind.NONE);
+
+ final CompletableFuture<Message> outgoingMessageSink = registerOutgoingMessageSink();
+ Validator validator = new Validator(desc, host, 0, true, false, PreviewKind.NONE);
+ ValidationManager.instance.submitValidation(cfs, validator);
+
+ Message message = outgoingMessageSink.get(TEST_TIMEOUT, TimeUnit.SECONDS);
+ MerkleTrees trees = ((ValidationResponse) message.payload).trees;
+
+ // Should have 4 trees each with a depth of on average 10 (since each range should have gotten 0.25 megabytes)
+ Iterator<Map.Entry<Range<Token>, MerkleTree>> iterator = trees.iterator();
+ int numTrees = 0;
+ double totalResolution = 0;
+ while (iterator.hasNext())
+ {
+ long size = iterator.next().getValue().size();
+ // So it turns out that sstable range estimates are pretty variable, depending on the sampling we can
+ // get a wide range of values here. So we just make sure that we're smaller than in the single range
+ // case and have the right total size.
+ assertTrue(size <= (1 << 11));
+ assertTrue(size >= (1 << 9));
+ totalResolution += size;
+ numTrees += 1;
+ }
+
+ assertEquals(trees.rowCount(), 1 << 14);
+ assertEquals(4, numTrees);
+
+ // With a single tree and a megabyte we should had a total resolution of 2^12 leaves; with multiple
+ // ranges we should get similar overall resolution, but not more.
+ assertTrue(totalResolution > (1 << 11) && totalResolution < (1 << 13));
+ }
+
+ private List<Range<Token>> splitHelper(Range<Token> range, int depth)
+ {
+ if (depth <= 0)
+ {
+ List<Range<Token>> tokens = new ArrayList<>();
+ tokens.add(range);
+ return tokens;
+ }
+ Token midpoint = partitioner.midpoint(range.left, range.right);
+ List<Range<Token>> left = splitHelper(new Range<>(range.left, midpoint), depth - 1);
+ List<Range<Token>> right = splitHelper(new Range<>(midpoint, range.right), depth - 1);
+ left.addAll(right);
+ return left;
+ }
+
+ private CompletableFuture<Message> registerOutgoingMessageSink()
+ {
+ final CompletableFuture<Message> future = new CompletableFuture<>();
+ MessagingService.instance().outboundSink.add((message, to) -> future.complete(message));
return future;
}
}
diff --git a/test/unit/org/apache/cassandra/repair/asymmetric/DifferenceHolderTest.java b/test/unit/org/apache/cassandra/repair/asymmetric/DifferenceHolderTest.java
new file mode 100644
index 0000000..8ec0177
--- /dev/null
+++ b/test/unit/org/apache/cassandra/repair/asymmetric/DifferenceHolderTest.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.repair.asymmetric;
+
+import java.net.UnknownHostException;
+import java.util.Iterator;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+import org.junit.Test;
+
+import org.apache.cassandra.db.Digest;
+import org.apache.cassandra.dht.IPartitioner;
+import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.repair.TreeResponse;
+import org.apache.cassandra.utils.MerkleTree;
+import org.apache.cassandra.utils.MerkleTrees;
+import org.apache.cassandra.utils.MerkleTreesTest;
+
+import static junit.framework.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class DifferenceHolderTest
+{
+ private static byte[] digest(String string)
+ {
+ return Digest.forValidator()
+ .update(string.getBytes(), 0, string.getBytes().length)
+ .digest();
+ }
+
+ @Test
+ public void testFromEmptyMerkleTrees() throws UnknownHostException
+ {
+ InetAddressAndPort a1 = InetAddressAndPort.getByName("127.0.0.1");
+ InetAddressAndPort a2 = InetAddressAndPort.getByName("127.0.0.2");
+
+ MerkleTrees mt1 = new MerkleTrees(Murmur3Partitioner.instance);
+ MerkleTrees mt2 = new MerkleTrees(Murmur3Partitioner.instance);
+ mt1.init();
+ mt2.init();
+
+ TreeResponse tr1 = new TreeResponse(a1, mt1);
+ TreeResponse tr2 = new TreeResponse(a2, mt2);
+
+ DifferenceHolder dh = new DifferenceHolder(Lists.newArrayList(tr1, tr2));
+ assertTrue(dh.get(a1).get(a2).isEmpty());
+ }
+
+ @Test
+ public void testFromMismatchedMerkleTrees() throws UnknownHostException
+ {
+ IPartitioner partitioner = Murmur3Partitioner.instance;
+ Range<Token> fullRange = new Range<>(partitioner.getMinimumToken(), partitioner.getMinimumToken());
+ int maxsize = 16;
+ InetAddressAndPort a1 = InetAddressAndPort.getByName("127.0.0.1");
+ InetAddressAndPort a2 = InetAddressAndPort.getByName("127.0.0.2");
+ // merkle tree building stolen from MerkleTreesTest:
+ MerkleTrees mt1 = new MerkleTrees(partitioner);
+ MerkleTrees mt2 = new MerkleTrees(partitioner);
+ mt1.addMerkleTree(32, fullRange);
+ mt2.addMerkleTree(32, fullRange);
+ mt1.init();
+ mt2.init();
+ // add dummy hashes to both trees
+ for (MerkleTree.TreeRange range : mt1.rangeIterator())
+ range.addAll(new MerkleTreesTest.HIterator(range.right));
+ for (MerkleTree.TreeRange range : mt2.rangeIterator())
+ range.addAll(new MerkleTreesTest.HIterator(range.right));
+
+ MerkleTree.TreeRange leftmost = null;
+ MerkleTree.TreeRange middle = null;
+
+ mt1.maxsize(fullRange, maxsize + 2); // give some room for splitting
+
+ // split the leftmost
+ Iterator<MerkleTree.TreeRange> ranges = mt1.rangeIterator();
+ leftmost = ranges.next();
+ mt1.split(leftmost.right);
+
+ // set the hashes for the leaf of the created split
+ middle = mt1.get(leftmost.right);
+ middle.hash(digest("arbitrary!"));
+ mt1.get(partitioner.midpoint(leftmost.left, leftmost.right)).hash(digest("even more arbitrary!"));
+
+ TreeResponse tr1 = new TreeResponse(a1, mt1);
+ TreeResponse tr2 = new TreeResponse(a2, mt2);
+
+ DifferenceHolder dh = new DifferenceHolder(Lists.newArrayList(tr1, tr2));
+ assertTrue(dh.get(a1).get(a2).size() == 1);
+ assertTrue(dh.hasDifferenceBetween(a1, a2, fullRange));
+ // only a1 is added as a key - see comment in dh.keyHosts()
+ assertEquals(Sets.newHashSet(a1), dh.keyHosts());
+ }
+}
diff --git a/test/unit/org/apache/cassandra/repair/asymmetric/RangeDenormalizerTest.java b/test/unit/org/apache/cassandra/repair/asymmetric/RangeDenormalizerTest.java
new file mode 100644
index 0000000..a128f2b
--- /dev/null
+++ b/test/unit/org/apache/cassandra/repair/asymmetric/RangeDenormalizerTest.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.repair.asymmetric;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+
+import com.google.common.collect.Sets;
+import org.junit.Test;
+
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.apache.cassandra.repair.asymmetric.ReduceHelperTest.range;
+
+public class RangeDenormalizerTest
+{
+ @Test
+ public void testDenormalize()
+ {
+ // test when the new incoming range is fully contained within an existing incoming range
+ StreamFromOptions dummy = new StreamFromOptions(null, range(0, 100));
+ Map<Range<Token>, StreamFromOptions> incoming = new HashMap<>();
+ incoming.put(range(0, 100), dummy);
+ Set<Range<Token>> newInput = RangeDenormalizer.denormalize(range(30, 40), incoming);
+ assertEquals(3, incoming.size());
+ assertTrue(incoming.containsKey(range(0, 30)));
+ assertTrue(incoming.containsKey(range(30, 40)));
+ assertTrue(incoming.containsKey(range(40, 100)));
+ assertEquals(1, newInput.size());
+ assertTrue(newInput.contains(range(30, 40)));
+ }
+
+ @Test
+ public void testDenormalize2()
+ {
+ // test when the new incoming range fully contains an existing incoming range
+ StreamFromOptions dummy = new StreamFromOptions(null, range(40, 50));
+ Map<Range<Token>, StreamFromOptions> incoming = new HashMap<>();
+ incoming.put(range(40, 50), dummy);
+ Set<Range<Token>> newInput = RangeDenormalizer.denormalize(range(0, 100), incoming);
+ assertEquals(1, incoming.size());
+ assertTrue(incoming.containsKey(range(40, 50)));
+ assertEquals(3, newInput.size());
+ assertTrue(newInput.contains(range(0, 40)));
+ assertTrue(newInput.contains(range(40, 50)));
+ assertTrue(newInput.contains(range(50, 100)));
+ }
+
+ @Test
+ public void testDenormalize3()
+ {
+ // test when there are multiple existing incoming ranges and the new incoming overlaps some and contains some
+ StreamFromOptions dummy = new StreamFromOptions(null, range(0, 100));
+ StreamFromOptions dummy2 = new StreamFromOptions(null, range(200, 300));
+ StreamFromOptions dummy3 = new StreamFromOptions(null, range(500, 600));
+ Map<Range<Token>, StreamFromOptions> incoming = new HashMap<>();
+ incoming.put(range(0, 100), dummy);
+ incoming.put(range(200, 300), dummy2);
+ incoming.put(range(500, 600), dummy3);
+ Set<Range<Token>> expectedNewInput = Sets.newHashSet(range(50, 100), range(100, 200), range(200, 300), range(300, 350));
+ Set<Range<Token>> expectedIncomingKeys = Sets.newHashSet(range(0, 50), range(50, 100), range(200, 300), range(500, 600));
+ Set<Range<Token>> newInput = RangeDenormalizer.denormalize(range(50, 350), incoming);
+ assertEquals(expectedNewInput, newInput);
+ assertEquals(expectedIncomingKeys, incoming.keySet());
+ }
+}
diff --git a/test/unit/org/apache/cassandra/repair/asymmetric/ReduceHelperTest.java b/test/unit/org/apache/cassandra/repair/asymmetric/ReduceHelperTest.java
new file mode 100644
index 0000000..6c64b1a
--- /dev/null
+++ b/test/unit/org/apache/cassandra/repair/asymmetric/ReduceHelperTest.java
@@ -0,0 +1,423 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.repair.asymmetric;
+
+
+import java.net.UnknownHostException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Sets;
+import org.junit.Test;
+
+import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.locator.InetAddressAndPort;
+
+import static junit.framework.TestCase.fail;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class ReduceHelperTest
+{
+ private static final InetAddressAndPort[] addresses;
+ private static final InetAddressAndPort A;
+ private static final InetAddressAndPort B;
+ private static final InetAddressAndPort C;
+ private static final InetAddressAndPort D;
+ private static final InetAddressAndPort E;
+
+ static
+ {
+ try
+ {
+ A = InetAddressAndPort.getByName("127.0.0.0");
+ B = InetAddressAndPort.getByName("127.0.0.1");
+ C = InetAddressAndPort.getByName("127.0.0.2");
+ D = InetAddressAndPort.getByName("127.0.0.3");
+ E = InetAddressAndPort.getByName("127.0.0.4");
+ // for diff creation in loops:
+ addresses = new InetAddressAndPort[]{ A, B, C, D, E };
+ }
+ catch (UnknownHostException e)
+ {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Test
+ public void testSimpleReducing()
+ {
+ /*
+ A == B and D == E =>
+ A streams from C, {D, E} since D==E
+ B streams from C, {D, E} since D==E
+ C streams from {A, B}, {D, E} since A==B and D==E
+ D streams from {A, B}, C since A==B
+ E streams from {A, B}, C since A==B
+
+ A B C D E
+ A = x x x
+ B x x x
+ C x x
+ D =
+ */
+ Map<InetAddressAndPort, HostDifferences> differences = new HashMap<>();
+ for (int i = 0; i < 4; i++)
+ {
+ HostDifferences hostDiffs = new HostDifferences();
+ for (int j = i + 1; j < 5; j++)
+ {
+ // no diffs between A, B and D, E:
+ if (addresses[i] == A && addresses[j] == B || addresses[i] == D && addresses[j] == E)
+ continue;
+ List<Range<Token>> diff = list(new Range<>(new Murmur3Partitioner.LongToken(0), new Murmur3Partitioner.LongToken(10)));
+ hostDiffs.add(addresses[j], diff);
+ }
+ differences.put(addresses[i], hostDiffs);
+
+ }
+ DifferenceHolder differenceHolder = new DifferenceHolder(differences);
+ Map<InetAddressAndPort, IncomingRepairStreamTracker> tracker = ReduceHelper.createIncomingRepairStreamTrackers(differenceHolder);
+
+ assertEquals(set(set(C), set(E,D)), streams(tracker.get(A)));
+ assertEquals(set(set(C), set(E,D)), streams(tracker.get(B)));
+ assertEquals(set(set(A,B), set(E,D)), streams(tracker.get(C)));
+ assertEquals(set(set(A,B), set(C)), streams(tracker.get(D)));
+ assertEquals(set(set(A,B), set(C)), streams(tracker.get(E)));
+
+ ImmutableMap<InetAddressAndPort, HostDifferences> reduced = ReduceHelper.reduce(differenceHolder, (x, y) -> y);
+
+ HostDifferences n0 = reduced.get(A);
+ assertEquals(0, n0.get(A).size());
+ assertEquals(0, n0.get(B).size());
+ assertTrue(n0.get(C).size() > 0);
+ assertStreamFromEither(n0.get(D), n0.get(E));
+
+ HostDifferences n1 = reduced.get(B);
+ assertEquals(0, n1.get(A).size());
+ assertEquals(0, n1.get(B).size());
+ assertTrue(n1.get(C).size() > 0);
+ assertStreamFromEither(n1.get(D), n1.get(E));
+
+ HostDifferences n2 = reduced.get(C);
+ // we are either streaming from node 0 or node 1, not both:
+ assertStreamFromEither(n2.get(A), n2.get(B));
+ assertEquals(0, n2.get(C).size());
+ assertStreamFromEither(n2.get(D), n2.get(E));
+
+ HostDifferences n3 = reduced.get(D);
+ assertStreamFromEither(n3.get(A), n3.get(B));
+ assertTrue(n3.get(C).size() > 0);
+ assertEquals(0, n3.get(D).size());
+ assertEquals(0, n3.get(E).size());
+
+ HostDifferences n4 = reduced.get(E);
+ assertStreamFromEither(n4.get(A), n4.get(B));
+ assertTrue(n4.get(C).size() > 0);
+ assertEquals(0, n4.get(D).size());
+ assertEquals(0, n4.get(E).size());
+ }
+
+ @Test
+ public void testSimpleReducingWithPreferedNodes()
+ {
+ /*
+ A == B and D == E =>
+ A streams from C, {D, E} since D==E
+ B streams from C, {D, E} since D==E
+ C streams from {A, B}, {D, E} since A==B and D==E
+ D streams from {A, B}, C since A==B
+ E streams from {A, B}, C since A==B
+
+ A B C D E
+ A = x x x
+ B x x x
+ C x x
+ D =
+ */
+ Map<InetAddressAndPort, HostDifferences> differences = new HashMap<>();
+ for (int i = 0; i < 4; i++)
+ {
+ HostDifferences hostDifferences = new HostDifferences();
+ for (int j = i + 1; j < 5; j++)
+ {
+ // no diffs between A, B and D, E:
+ if (addresses[i] == A && addresses[j] == B || addresses[i] == D && addresses[j] == E)
+ continue;
+ List<Range<Token>> diff = list(new Range<>(new Murmur3Partitioner.LongToken(0), new Murmur3Partitioner.LongToken(10)));
+ hostDifferences.add(addresses[j], diff);
+ }
+ differences.put(addresses[i], hostDifferences);
+ }
+
+ DifferenceHolder differenceHolder = new DifferenceHolder(differences);
+ Map<InetAddressAndPort, IncomingRepairStreamTracker> tracker = ReduceHelper.createIncomingRepairStreamTrackers(differenceHolder);
+ assertEquals(set(set(C), set(E, D)), streams(tracker.get(A)));
+ assertEquals(set(set(C), set(E, D)), streams(tracker.get(B)));
+ assertEquals(set(set(A, B), set(E, D)), streams(tracker.get(C)));
+ assertEquals(set(set(A, B), set(C)), streams(tracker.get(D)));
+ assertEquals(set(set(A, B), set(C)), streams(tracker.get(E)));
+
+ // if there is an option, never stream from node 1:
+ ImmutableMap<InetAddressAndPort, HostDifferences> reduced = ReduceHelper.reduce(differenceHolder, (x,y) -> Sets.difference(y, set(B)));
+
+ HostDifferences n0 = reduced.get(A);
+ assertEquals(0, n0.get(A).size());
+ assertEquals(0, n0.get(B).size());
+ assertTrue(n0.get(C).size() > 0);
+ assertStreamFromEither(n0.get(D), n0.get(E));
+
+ HostDifferences n1 = reduced.get(B);
+ assertEquals(0, n1.get(A).size());
+ assertEquals(0, n1.get(B).size());
+ assertTrue(n1.get(C).size() > 0);
+ assertStreamFromEither(n1.get(D), n1.get(E));
+
+
+ HostDifferences n2 = reduced.get(C);
+ assertTrue(n2.get(A).size() > 0);
+ assertEquals(0, n2.get(B).size());
+ assertEquals(0, n2.get(C).size());
+ assertStreamFromEither(n2.get(D), n2.get(E));
+
+ HostDifferences n3 = reduced.get(D);
+ assertTrue(n3.get(A).size() > 0);
+ assertEquals(0, n3.get(B).size());
+ assertTrue(n3.get(C).size() > 0);
+ assertEquals(0, n3.get(D).size());
+ assertEquals(0, n3.get(E).size());
+
+ HostDifferences n4 = reduced.get(E);
+ assertTrue(n4.get(A).size() > 0);
+ assertEquals(0, n4.get(B).size());
+ assertTrue(n4.get(C).size() > 0);
+ assertEquals(0, n4.get(D).size());
+ assertEquals(0, n4.get(E).size());
+ }
+
+ private Iterable<Set<InetAddressAndPort>> streams(IncomingRepairStreamTracker incomingRepairStreamTracker)
+ {
+ return incomingRepairStreamTracker.getIncoming().values().iterator().next().allStreams();
+ }
+
+ @Test
+ public void testOverlapDifference()
+ {
+ /*
+ |A |B |C
+ ---+------+------+--------
+ A |= |50,100|0,50
+ B | |= |0,100
+ C | | |=
+
+ A needs to stream (50, 100] from B, (0, 50] from C
+ B needs to stream (50, 100] from A, (0, 100] from C
+ C needs to stream (0, 50] from A, (0, 100] from B
+ A == B on (0, 50] => C can stream (0, 50] from either A or B
+ A == C on (50, 100] => B can stream (50, 100] from either A or C
+ =>
+ A streams (50, 100] from {B}, (0, 50] from C
+ B streams (0, 50] from {C}, (50, 100] from {A, C}
+ C streams (0, 50] from {A, B}, (50, 100] from B
+ */
+ Map<InetAddressAndPort, HostDifferences> differences = new HashMap<>();
+ addDifference(A, differences, B, list(range(50, 100)));
+ addDifference(A, differences, C, list(range(0, 50)));
+ addDifference(B, differences, C, list(range(0, 100)));
+ DifferenceHolder differenceHolder = new DifferenceHolder(differences);
+ Map<InetAddressAndPort, IncomingRepairStreamTracker> tracker = ReduceHelper.createIncomingRepairStreamTrackers(differenceHolder);
+ assertEquals(set(set(C)), tracker.get(A).getIncoming().get(range(0, 50)).allStreams());
+ assertEquals(set(set(B)), tracker.get(A).getIncoming().get(range(50, 100)).allStreams());
+ assertEquals(set(set(C)), tracker.get(B).getIncoming().get(range(0, 50)).allStreams());
+ assertEquals(set(set(A,C)), tracker.get(B).getIncoming().get(range(50, 100)).allStreams());
+ assertEquals(set(set(A,B)), tracker.get(C).getIncoming().get(range(0, 50)).allStreams());
+ assertEquals(set(set(B)), tracker.get(C).getIncoming().get(range(50, 100)).allStreams());
+
+ ImmutableMap<InetAddressAndPort, HostDifferences> reduced = ReduceHelper.reduce(differenceHolder, (x, y) -> y);
+
+ HostDifferences n0 = reduced.get(A);
+
+ assertTrue(n0.get(B).equals(list(range(50, 100))));
+ assertTrue(n0.get(C).equals(list(range(0, 50))));
+
+ HostDifferences n1 = reduced.get(B);
+ assertEquals(0, n1.get(B).size());
+ if (!n1.get(A).isEmpty())
+ {
+ assertTrue(n1.get(C).equals(list(range(0, 50))));
+ assertTrue(n1.get(A).equals(list(range(50, 100))));
+ }
+ else
+ {
+ assertTrue(n1.get(C).equals(list(range(0, 50), range(50, 100))));
+ }
+ HostDifferences n2 = reduced.get(C);
+ assertEquals(0, n2.get(C).size());
+ if (!n2.get(A).isEmpty())
+ {
+ assertTrue(n2.get(A).equals(list(range(0,50))));
+ assertTrue(n2.get(B).equals(list(range(50, 100))));
+ }
+ else
+ {
+ assertTrue(n2.get(A).equals(list(range(0, 50), range(50, 100))));
+ }
+
+
+ }
+
+ @Test
+ public void testOverlapDifference2()
+ {
+ /*
+ |A |B |C
+ ---+----------------+----------------+------------------
+ A |= |5,45 |0,10 40,50
+ B | |= |0,5 10,40 45,50
+ C | | |=
+
+ A needs to stream (5, 45] from B, (0, 10], (40, 50) from C
+ B needs to stream (5, 45] from A, (0, 5], (10, 40], (45, 50] from C
+ C needs to stream (0, 10], (40,50] from A, (0,5], (10,40], (45,50] from B
+ A == B on (0, 5], (45, 50]
+ A == C on (10, 40]
+ B == C on (5, 10], (40, 45]
+ */
+
+ Map<InetAddressAndPort, HostDifferences> differences = new HashMap<>();
+ addDifference(A, differences, B, list(range(5, 45)));
+ addDifference(A, differences, C, list(range(0, 10), range(40,50)));
+ addDifference(B, differences, C, list(range(0, 5), range(10,40), range(45,50)));
+
+ DifferenceHolder differenceHolder = new DifferenceHolder(differences);
+ Map<InetAddressAndPort, IncomingRepairStreamTracker> tracker = ReduceHelper.createIncomingRepairStreamTrackers(differenceHolder);
+
+ Map<Range<Token>, StreamFromOptions> ranges = tracker.get(A).getIncoming();
+ assertEquals(5, ranges.size());
+
+ assertEquals(set(set(C)), ranges.get(range(0, 5)).allStreams());
+ assertEquals(set(set(B, C)), ranges.get(range(5, 10)).allStreams());
+ assertEquals(set(set(B)), ranges.get(range(10, 40)).allStreams());
+ assertEquals(set(set(B, C)), ranges.get(range(40, 45)).allStreams());
+ assertEquals(set(set(C)), ranges.get(range(45, 50)).allStreams());
+
+ ranges = tracker.get(B).getIncoming();
+ assertEquals(5, ranges.size());
+ assertEquals(set(set(C)), ranges.get(range(0, 5)).allStreams());
+ assertEquals(set(set(A)), ranges.get(range(5, 10)).allStreams());
+ assertEquals(set(set(A, C)), ranges.get(range(10, 40)).allStreams());
+ assertEquals(set(set(A)), ranges.get(range(40, 45)).allStreams());
+ assertEquals(set(set(C)), ranges.get(range(45, 50)).allStreams());
+
+ ranges = tracker.get(C).getIncoming();
+ assertEquals(5, ranges.size());
+ assertEquals(set(set(A, B)), ranges.get(range(0, 5)).allStreams());
+ assertEquals(set(set(A)), ranges.get(range(5, 10)).allStreams());
+ assertEquals(set(set(B)), ranges.get(range(10, 40)).allStreams());
+ assertEquals(set(set(A)), ranges.get(range(40, 45)).allStreams());
+ assertEquals(set(set(A,B)), ranges.get(range(45, 50)).allStreams());
+ ImmutableMap<InetAddressAndPort, HostDifferences> reduced = ReduceHelper.reduce(differenceHolder, (x, y) -> y);
+
+ assertNoOverlap(A, reduced.get(A), list(range(0, 50)));
+ assertNoOverlap(B, reduced.get(B), list(range(0, 50)));
+ assertNoOverlap(C, reduced.get(C), list(range(0, 50)));
+ }
+
+ private void assertNoOverlap(InetAddressAndPort incomingNode, HostDifferences node, List<Range<Token>> expectedAfterNormalize)
+ {
+ Set<Range<Token>> allRanges = new HashSet<>();
+ Set<InetAddressAndPort> remoteNodes = Sets.newHashSet(A,B,C);
+ remoteNodes.remove(incomingNode);
+ Iterator<InetAddressAndPort> iter = remoteNodes.iterator();
+ allRanges.addAll(node.get(iter.next()));
+ InetAddressAndPort i = iter.next();
+ for (Range<Token> r : node.get(i))
+ {
+ for (Range<Token> existing : allRanges)
+ if (r.intersects(existing))
+ fail();
+ }
+ allRanges.addAll(node.get(i));
+ List<Range<Token>> normalized = Range.normalize(allRanges);
+ assertEquals(expectedAfterNormalize, normalized);
+ }
+
+ @SafeVarargs
+ private static List<Range<Token>> list(Range<Token> r, Range<Token> ... rs)
+ {
+ List<Range<Token>> ranges = new ArrayList<>();
+ ranges.add(r);
+ Collections.addAll(ranges, rs);
+ return ranges;
+ }
+
+ private static Set<InetAddressAndPort> set(InetAddressAndPort ... elem)
+ {
+ return Sets.newHashSet(elem);
+ }
+ @SafeVarargs
+ private static Set<Set<InetAddressAndPort>> set(Set<InetAddressAndPort> ... elem)
+ {
+ Set<Set<InetAddressAndPort>> ret = Sets.newHashSet();
+ ret.addAll(Arrays.asList(elem));
+ return ret;
+ }
+
+ static Murmur3Partitioner.LongToken longtok(long l)
+ {
+ return new Murmur3Partitioner.LongToken(l);
+ }
+
+ static Range<Token> range(long t, long t2)
+ {
+ return new Range<>(longtok(t), longtok(t2));
+ }
+
+ @Test
+ public void testSubtractAllRanges()
+ {
+ Set<Range<Token>> ranges = new HashSet<>();
+ ranges.add(range(10, 20)); ranges.add(range(40, 60));
+ assertEquals(0, RangeDenormalizer.subtractFromAllRanges(ranges, range(0, 100)).size());
+ ranges.add(range(90, 110));
+ assertEquals(Sets.newHashSet(range(100, 110)), RangeDenormalizer.subtractFromAllRanges(ranges, range(0, 100)));
+ ranges.add(range(-10, 10));
+ assertEquals(Sets.newHashSet(range(-10, 0), range(100, 110)), RangeDenormalizer.subtractFromAllRanges(ranges, range(0, 100)));
+ }
+
+ private void assertStreamFromEither(List<Range<Token>> r1, List<Range<Token>> r2)
+ {
+ assertTrue(r1.size() > 0 ^ r2.size() > 0);
+ }
+
+ private void addDifference(InetAddressAndPort host1, Map<InetAddressAndPort, HostDifferences> differences, InetAddressAndPort host2, List<Range<Token>> ranges)
+ {
+ differences.computeIfAbsent(host1, (x) -> new HostDifferences()).add(host2, ranges);
+ }
+}
diff --git a/test/unit/org/apache/cassandra/repair/asymmetric/StreamFromOptionsTest.java b/test/unit/org/apache/cassandra/repair/asymmetric/StreamFromOptionsTest.java
new file mode 100644
index 0000000..e2a7700
--- /dev/null
+++ b/test/unit/org/apache/cassandra/repair/asymmetric/StreamFromOptionsTest.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.repair.asymmetric;
+
+import java.net.UnknownHostException;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+
+import com.google.common.collect.Iterables;
+import org.junit.Test;
+
+import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.locator.InetAddressAndPort;
+
+import static junit.framework.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class StreamFromOptionsTest
+{
+ @Test
+ public void addAllDiffingTest() throws UnknownHostException
+ {
+ StreamFromOptions sfo = new StreamFromOptions(new MockDiffs(true), range(0, 10));
+ Set<InetAddressAndPort> toAdd = new HashSet<>();
+ toAdd.add(InetAddressAndPort.getByName("127.0.0.1"));
+ toAdd.add(InetAddressAndPort.getByName("127.0.0.2"));
+ toAdd.add(InetAddressAndPort.getByName("127.0.0.3"));
+ toAdd.forEach(sfo::add);
+
+ // if all added have differences, each set will contain a single host
+ assertEquals(3, Iterables.size(sfo.allStreams()));
+ Set<InetAddressAndPort> allStreams = new HashSet<>();
+ for (Set<InetAddressAndPort> streams : sfo.allStreams())
+ {
+ assertEquals(1, streams.size());
+ allStreams.addAll(streams);
+ }
+ assertEquals(toAdd, allStreams);
+ }
+
+ @Test
+ public void addAllMatchingTest() throws UnknownHostException
+ {
+ StreamFromOptions sfo = new StreamFromOptions(new MockDiffs(false), range(0, 10));
+ Set<InetAddressAndPort> toAdd = new HashSet<>();
+ toAdd.add(InetAddressAndPort.getByName("127.0.0.1"));
+ toAdd.add(InetAddressAndPort.getByName("127.0.0.2"));
+ toAdd.add(InetAddressAndPort.getByName("127.0.0.3"));
+ toAdd.forEach(sfo::add);
+
+ // if all added match, the set will contain all hosts
+ assertEquals(1, Iterables.size(sfo.allStreams()));
+ assertEquals(toAdd, sfo.allStreams().iterator().next());
+ }
+
+ @Test
+ public void splitTest() throws UnknownHostException
+ {
+ splitTestHelper(true);
+ splitTestHelper(false);
+ }
+
+ private void splitTestHelper(boolean diffing) throws UnknownHostException
+ {
+ StreamFromOptions sfo = new StreamFromOptions(new MockDiffs(diffing), range(0, 10));
+ Set<InetAddressAndPort> toAdd = new HashSet<>();
+ toAdd.add(InetAddressAndPort.getByName("127.0.0.1"));
+ toAdd.add(InetAddressAndPort.getByName("127.0.0.2"));
+ toAdd.add(InetAddressAndPort.getByName("127.0.0.3"));
+ toAdd.forEach(sfo::add);
+ StreamFromOptions sfo1 = sfo.copy(range(0, 5));
+ StreamFromOptions sfo2 = sfo.copy(range(5, 10));
+ assertEquals(range(0, 10), sfo.range);
+ assertEquals(range(0, 5), sfo1.range);
+ assertEquals(range(5, 10), sfo2.range);
+ assertTrue(Iterables.elementsEqual(sfo1.allStreams(), sfo2.allStreams()));
+ // verify the backing set is not shared between the copies:
+ sfo1.add(InetAddressAndPort.getByName("127.0.0.4"));
+ sfo2.add(InetAddressAndPort.getByName("127.0.0.5"));
+ assertFalse(Iterables.elementsEqual(sfo1.allStreams(), sfo2.allStreams()));
+ }
+
+ private Range<Token> range(long left, long right)
+ {
+ return new Range<>(new Murmur3Partitioner.LongToken(left), new Murmur3Partitioner.LongToken(right));
+ }
+
+ private static class MockDiffs extends DifferenceHolder
+ {
+ private final boolean hasDifference;
+
+ public MockDiffs(boolean hasDifference)
+ {
+ super(Collections.emptyMap());
+ this.hasDifference = hasDifference;
+ }
+
+ @Override
+ public boolean hasDifferenceBetween(InetAddressAndPort node1, InetAddressAndPort node2, Range<Token> range)
+ {
+ return hasDifference;
+ }
+ }
+}
diff --git a/test/unit/org/apache/cassandra/repair/consistent/AbstractConsistentSessionTest.java b/test/unit/org/apache/cassandra/repair/consistent/AbstractConsistentSessionTest.java
new file mode 100644
index 0000000..4570328
--- /dev/null
+++ b/test/unit/org/apache/cassandra/repair/consistent/AbstractConsistentSessionTest.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.repair.consistent;
+
+import java.net.UnknownHostException;
+import java.util.Set;
+import java.util.UUID;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+
+import org.junit.Ignore;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.streaming.PreviewKind;
+import org.apache.cassandra.service.ActiveRepairService;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.UUIDGen;
+
+@Ignore
+public abstract class AbstractConsistentSessionTest
+{
+ protected static final InetAddressAndPort COORDINATOR;
+ protected static final InetAddressAndPort PARTICIPANT1;
+ protected static final InetAddressAndPort PARTICIPANT2;
+ protected static final InetAddressAndPort PARTICIPANT3;
+
+ static
+ {
+ try
+ {
+ COORDINATOR = InetAddressAndPort.getByName("10.0.0.1");
+ PARTICIPANT1 = InetAddressAndPort.getByName("10.0.0.1");
+ PARTICIPANT2 = InetAddressAndPort.getByName("10.0.0.2");
+ PARTICIPANT3 = InetAddressAndPort.getByName("10.0.0.3");
+ }
+ catch (UnknownHostException e)
+ {
+
+ throw new AssertionError(e);
+ }
+
+ DatabaseDescriptor.daemonInitialization();
+ }
+
+ protected static final Set<InetAddressAndPort> PARTICIPANTS = ImmutableSet.of(PARTICIPANT1, PARTICIPANT2, PARTICIPANT3);
+
+ protected static Token t(int v)
+ {
+ return DatabaseDescriptor.getPartitioner().getToken(ByteBufferUtil.bytes(v));
+ }
+
+ protected static final Range<Token> RANGE1 = new Range<>(t(1), t(2));
+ protected static final Range<Token> RANGE2 = new Range<>(t(2), t(3));
+ protected static final Range<Token> RANGE3 = new Range<>(t(4), t(5));
+
+
+ protected static UUID registerSession(ColumnFamilyStore cfs)
+ {
+ UUID sessionId = UUIDGen.getTimeUUID();
+
+ ActiveRepairService.instance.registerParentRepairSession(sessionId,
+ COORDINATOR,
+ Lists.newArrayList(cfs),
+ Sets.newHashSet(RANGE1, RANGE2, RANGE3),
+ true,
+ System.currentTimeMillis(),
+ true,
+ PreviewKind.NONE);
+ return sessionId;
+ }
+}
diff --git a/test/unit/org/apache/cassandra/repair/consistent/CoordinatorMessagingTest.java b/test/unit/org/apache/cassandra/repair/consistent/CoordinatorMessagingTest.java
new file mode 100644
index 0000000..420cd54
--- /dev/null
+++ b/test/unit/org/apache/cassandra/repair/consistent/CoordinatorMessagingTest.java
@@ -0,0 +1,336 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.repair.consistent;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.UUID;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.Function;
+import java.util.function.Supplier;
+
+import com.google.common.collect.Lists;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.SettableFuture;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.cql3.statements.schema.CreateTableStatement;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.net.Message;
+import org.apache.cassandra.net.MockMessagingService;
+import org.apache.cassandra.net.MockMessagingSpy;
+import org.apache.cassandra.net.Verb;
+import org.apache.cassandra.repair.AbstractRepairTest;
+import org.apache.cassandra.repair.RepairSessionResult;
+import org.apache.cassandra.repair.messages.FinalizePromise;
+import org.apache.cassandra.repair.messages.FinalizePropose;
+import org.apache.cassandra.repair.messages.PrepareConsistentRequest;
+import org.apache.cassandra.repair.messages.PrepareConsistentResponse;
+import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.service.ActiveRepairService;
+
+import static org.apache.cassandra.net.MockMessagingService.all;
+import static org.apache.cassandra.net.MockMessagingService.to;
+import static org.apache.cassandra.net.MockMessagingService.verb;
+import static org.junit.Assert.fail;
+
+public class CoordinatorMessagingTest extends AbstractRepairTest
+{
+
+ protected ColumnFamilyStore cfs;
+
+ @BeforeClass
+ public static void setupClass()
+ {
+ SchemaLoader.prepareServer();
+ LocalSessionAccessor.startup();
+ }
+
+ @Before
+ public void setup()
+ {
+ String ks = "ks_" + System.currentTimeMillis();
+ TableMetadata cfm = CreateTableStatement.parse(String.format("CREATE TABLE %s.%s (k INT PRIMARY KEY, v INT)", ks, "tbl"), ks).build();
+ SchemaLoader.createKeyspace(ks, KeyspaceParams.simple(1), cfm);
+ cfs = Schema.instance.getColumnFamilyStoreInstance(cfm.id);
+ cfs.disableAutoCompaction();
+ }
+
+ @After
+ public void cleanup()
+ {
+ MockMessagingService.cleanup();
+ }
+
+ @Test
+ public void testMockedMessagingHappyPath() throws InterruptedException, ExecutionException, TimeoutException
+ {
+ CountDownLatch prepareLatch = createLatch();
+ CountDownLatch finalizeLatch = createLatch();
+
+ MockMessagingSpy spyPrepare = createPrepareSpy(Collections.emptySet(), Collections.emptySet(), prepareLatch);
+ MockMessagingSpy spyFinalize = createFinalizeSpy(Collections.emptySet(), Collections.emptySet(), finalizeLatch);
+ MockMessagingSpy spyCommit = createCommitSpy();
+
+ UUID uuid = registerSession(cfs, true, true);
+ CoordinatorSession coordinator = ActiveRepairService.instance.consistent.coordinated.registerSession(uuid, PARTICIPANTS, false);
+ AtomicBoolean repairSubmitted = new AtomicBoolean(false);
+ SettableFuture<List<RepairSessionResult>> repairFuture = SettableFuture.create();
+ Supplier<ListenableFuture<List<RepairSessionResult>>> sessionSupplier = () ->
+ {
+ repairSubmitted.set(true);
+ return repairFuture;
+ };
+
+ // coordinator sends prepare requests to create local session and perform anticompaction
+ AtomicBoolean hasFailures = new AtomicBoolean(false);
+ Assert.assertFalse(repairSubmitted.get());
+
+ // execute repair and start prepare phase
+ ListenableFuture<Boolean> sessionResult = coordinator.execute(sessionSupplier, hasFailures);
+ Assert.assertFalse(sessionResult.isDone());
+ Assert.assertFalse(hasFailures.get());
+ // prepare completed
+ prepareLatch.countDown();
+ spyPrepare.interceptMessageOut(3).get(1, TimeUnit.SECONDS);
+ Assert.assertFalse(sessionResult.isDone());
+ Assert.assertFalse(hasFailures.get());
+
+ // set result from local repair session
+ repairFuture.set(Lists.newArrayList(createResult(coordinator), createResult(coordinator), createResult(coordinator)));
+
+ // finalize phase
+ finalizeLatch.countDown();
+ spyFinalize.interceptMessageOut(3).get(1, TimeUnit.SECONDS);
+
+ // commit phase
+ spyCommit.interceptMessageOut(3).get(1, TimeUnit.SECONDS);
+ Assert.assertTrue(sessionResult.get());
+ Assert.assertFalse(hasFailures.get());
+
+ // expect no other messages except from intercepted so far
+ spyPrepare.interceptNoMsg(100, TimeUnit.MILLISECONDS);
+ spyFinalize.interceptNoMsg(100, TimeUnit.MILLISECONDS);
+ spyCommit.interceptNoMsg(100, TimeUnit.MILLISECONDS);
+
+ Assert.assertEquals(ConsistentSession.State.FINALIZED, coordinator.getState());
+ Assert.assertFalse(ActiveRepairService.instance.consistent.local.isSessionInProgress(uuid));
+ }
+
+
+ @Test
+ public void testMockedMessagingPrepareFailureP1() throws InterruptedException, ExecutionException, TimeoutException
+ {
+ CountDownLatch latch = createLatch();
+ createPrepareSpy(Collections.singleton(PARTICIPANT1), Collections.emptySet(), latch);
+ testMockedMessagingPrepareFailure(latch);
+ }
+
+ @Test
+ public void testMockedMessagingPrepareFailureP12() throws InterruptedException, ExecutionException, TimeoutException
+ {
+ CountDownLatch latch = createLatch();
+ createPrepareSpy(Lists.newArrayList(PARTICIPANT1, PARTICIPANT2), Collections.emptySet(), latch);
+ testMockedMessagingPrepareFailure(latch);
+ }
+
+ @Test
+ public void testMockedMessagingPrepareFailureP3() throws InterruptedException, ExecutionException, TimeoutException
+ {
+ CountDownLatch latch = createLatch();
+ createPrepareSpy(Collections.singleton(PARTICIPANT3), Collections.emptySet(), latch);
+ testMockedMessagingPrepareFailure(latch);
+ }
+
+ @Test
+ public void testMockedMessagingPrepareFailureP123() throws InterruptedException, ExecutionException, TimeoutException
+ {
+ CountDownLatch latch = createLatch();
+ createPrepareSpy(Lists.newArrayList(PARTICIPANT1, PARTICIPANT2, PARTICIPANT3), Collections.emptySet(), latch);
+ testMockedMessagingPrepareFailure(latch);
+ }
+
+ @Test(expected = TimeoutException.class)
+ public void testMockedMessagingPrepareFailureWrongSessionId() throws InterruptedException, ExecutionException, TimeoutException
+ {
+ CountDownLatch latch = createLatch();
+ createPrepareSpy(Collections.singleton(PARTICIPANT1), Collections.emptySet(), (msgOut) -> UUID.randomUUID(), latch);
+ testMockedMessagingPrepareFailure(latch);
+ }
+
+ private void testMockedMessagingPrepareFailure(CountDownLatch prepareLatch) throws InterruptedException, ExecutionException, TimeoutException
+ {
+ // we expect FailSession messages to all participants
+ MockMessagingSpy sendFailSessionExpectedSpy = createFailSessionSpy(Lists.newArrayList(PARTICIPANT1, PARTICIPANT2, PARTICIPANT3));
+
+ UUID uuid = registerSession(cfs, true, true);
+ CoordinatorSession coordinator = ActiveRepairService.instance.consistent.coordinated.registerSession(uuid, PARTICIPANTS, false);
+ AtomicBoolean repairSubmitted = new AtomicBoolean(false);
+ SettableFuture<List<RepairSessionResult>> repairFuture = SettableFuture.create();
+ Supplier<ListenableFuture<List<RepairSessionResult>>> sessionSupplier = () ->
+ {
+ repairSubmitted.set(true);
+ return repairFuture;
+ };
+
+ // coordinator sends prepare requests to create local session and perform anticompaction
+ AtomicBoolean proposeFailed = new AtomicBoolean(false);
+ Assert.assertFalse(repairSubmitted.get());
+
+ // execute repair and start prepare phase
+ ListenableFuture<Boolean> sessionResult = coordinator.execute(sessionSupplier, proposeFailed);
+ prepareLatch.countDown();
+ // prepare completed
+ try
+ {
+ sessionResult.get(1, TimeUnit.SECONDS);
+ fail("Completed session without failure after prepare failed");
+ }
+ catch (ExecutionException e)
+ {
+ }
+ sendFailSessionExpectedSpy.interceptMessageOut(3).get(1, TimeUnit.SECONDS);
+ Assert.assertFalse(repairSubmitted.get());
+ Assert.assertTrue(proposeFailed.get());
+ Assert.assertEquals(ConsistentSession.State.FAILED, coordinator.getState());
+ Assert.assertFalse(ActiveRepairService.instance.consistent.local.isSessionInProgress(uuid));
+ }
+
+ @Test
+ public void testMockedMessagingPrepareTimeout() throws InterruptedException, ExecutionException, TimeoutException
+ {
+ MockMessagingSpy spyPrepare = createPrepareSpy(Collections.emptySet(), Collections.singleton(PARTICIPANT3), new CountDownLatch(0));
+ MockMessagingSpy sendFailSessionUnexpectedSpy = createFailSessionSpy(Lists.newArrayList(PARTICIPANT1, PARTICIPANT2, PARTICIPANT3));
+
+ UUID uuid = registerSession(cfs, true, true);
+ CoordinatorSession coordinator = ActiveRepairService.instance.consistent.coordinated.registerSession(uuid, PARTICIPANTS, false);
+ AtomicBoolean repairSubmitted = new AtomicBoolean(false);
+ SettableFuture<List<RepairSessionResult>> repairFuture = SettableFuture.create();
+ Supplier<ListenableFuture<List<RepairSessionResult>>> sessionSupplier = () ->
+ {
+ repairSubmitted.set(true);
+ return repairFuture;
+ };
+
+ // coordinator sends prepare requests to create local session and perform anticompaction
+ AtomicBoolean hasFailures = new AtomicBoolean(false);
+ Assert.assertFalse(repairSubmitted.get());
+
+ // execute repair and start prepare phase
+ ListenableFuture<Boolean> sessionResult = coordinator.execute(sessionSupplier, hasFailures);
+ try
+ {
+ sessionResult.get(1, TimeUnit.SECONDS);
+ fail("Completed session without failure after prepare failed");
+ }
+ catch (ExecutionException e)
+ {
+ fail("Failed session in prepare failed during timeout from participant");
+ }
+ catch (TimeoutException e)
+ {
+ // expected
+ }
+ // we won't send out any fail session message in case of timeouts
+ spyPrepare.expectMockedMessage(2).get(100, TimeUnit.MILLISECONDS);
+ sendFailSessionUnexpectedSpy.interceptNoMsg(100, TimeUnit.MILLISECONDS);
+ Assert.assertFalse(repairSubmitted.get());
+ Assert.assertFalse(hasFailures.get());
+ Assert.assertEquals(ConsistentSession.State.PREPARING, coordinator.getState());
+ Assert.assertFalse(ActiveRepairService.instance.consistent.local.isSessionInProgress(uuid));
+ }
+
+ private MockMessagingSpy createPrepareSpy(Collection<InetAddressAndPort> failed,
+ Collection<InetAddressAndPort> timeout,
+ CountDownLatch latch)
+ {
+ return createPrepareSpy(failed, timeout, (msgOut) -> msgOut.parentSession, latch);
+ }
+
+ private MockMessagingSpy createPrepareSpy(Collection<InetAddressAndPort> failed,
+ Collection<InetAddressAndPort> timeout,
+ Function<PrepareConsistentRequest, UUID> sessionIdFunc,
+ CountDownLatch latch)
+ {
+ return MockMessagingService.when(verb(Verb.PREPARE_CONSISTENT_REQ)).respond((msgOut, to) ->
+ {
+ try
+ {
+ latch.await();
+ }
+ catch (InterruptedException e) { }
+ if (timeout.contains(to))
+ return null;
+
+ return Message.out(Verb.PREPARE_CONSISTENT_RSP,
+ new PrepareConsistentResponse(sessionIdFunc.apply((PrepareConsistentRequest) msgOut.payload), to, !failed.contains(to)));
+ });
+ }
+
+ private MockMessagingSpy createFinalizeSpy(Collection<InetAddressAndPort> failed,
+ Collection<InetAddressAndPort> timeout,
+ CountDownLatch latch)
+ {
+ return MockMessagingService.when(verb(Verb.FINALIZE_PROPOSE_MSG)).respond((msgOut, to) ->
+ {
+ try
+ {
+ latch.await();
+ }
+ catch (InterruptedException e) { }
+ if (timeout.contains(to))
+ return null;
+
+ return Message.out(Verb.FINALIZE_PROMISE_MSG, new FinalizePromise(((FinalizePropose) msgOut.payload).sessionID, to, !failed.contains(to)));
+ });
+ }
+
+ private MockMessagingSpy createCommitSpy()
+ {
+ return MockMessagingService.when(verb(Verb.FINALIZE_COMMIT_MSG)).dontReply();
+ }
+
+ private MockMessagingSpy createFailSessionSpy(Collection<InetAddressAndPort> participants)
+ {
+ return MockMessagingService.when(all(verb(Verb.FAILED_SESSION_MSG), to(participants::contains))).dontReply();
+ }
+
+ private static RepairSessionResult createResult(CoordinatorSession coordinator)
+ {
+ return new RepairSessionResult(coordinator.sessionID, "ks", coordinator.ranges, null, false);
+ }
+
+ private CountDownLatch createLatch()
+ {
+ return new CountDownLatch(1);
+ }
+}
diff --git a/test/unit/org/apache/cassandra/repair/consistent/CoordinatorSessionTest.java b/test/unit/org/apache/cassandra/repair/consistent/CoordinatorSessionTest.java
new file mode 100644
index 0000000..1cee312
--- /dev/null
+++ b/test/unit/org/apache/cassandra/repair/consistent/CoordinatorSessionTest.java
@@ -0,0 +1,505 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.repair.consistent;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.Supplier;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.SettableFuture;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.net.Message;
+import org.apache.cassandra.repair.AbstractRepairTest;
+import org.apache.cassandra.repair.RepairSessionResult;
+import org.apache.cassandra.repair.messages.FailSession;
+import org.apache.cassandra.repair.messages.FinalizeCommit;
+import org.apache.cassandra.repair.messages.FinalizePropose;
+import org.apache.cassandra.repair.messages.PrepareConsistentRequest;
+import org.apache.cassandra.repair.messages.RepairMessage;
+import org.apache.cassandra.utils.UUIDGen;
+
+import static org.apache.cassandra.repair.consistent.ConsistentSession.State.*;
+
+public class CoordinatorSessionTest extends AbstractRepairTest
+{
+
+ static CoordinatorSession.Builder createBuilder()
+ {
+ CoordinatorSession.Builder builder = CoordinatorSession.builder();
+ builder.withState(PREPARING);
+ builder.withSessionID(UUIDGen.getTimeUUID());
+ builder.withCoordinator(COORDINATOR);
+ builder.withUUIDTableIds(Sets.newHashSet(UUIDGen.getTimeUUID(), UUIDGen.getTimeUUID()));
+ builder.withRepairedAt(System.currentTimeMillis());
+ builder.withRanges(Sets.newHashSet(RANGE1, RANGE2, RANGE3));
+ builder.withParticipants(Sets.newHashSet(PARTICIPANT1, PARTICIPANT2, PARTICIPANT3));
+ return builder;
+ }
+
+ static CoordinatorSession createSession()
+ {
+ return createBuilder().build();
+ }
+
+ static InstrumentedCoordinatorSession createInstrumentedSession()
+ {
+ return new InstrumentedCoordinatorSession(createBuilder());
+ }
+
+ private static RepairSessionResult createResult(CoordinatorSession coordinator)
+ {
+ return new RepairSessionResult(coordinator.sessionID, "ks", coordinator.ranges, null, false);
+ }
+
+ private static void assertMessageSent(InstrumentedCoordinatorSession coordinator, InetAddressAndPort participant, RepairMessage expected)
+ {
+ Assert.assertTrue(coordinator.sentMessages.containsKey(participant));
+ Assert.assertEquals(1, coordinator.sentMessages.get(participant).size());
+ Assert.assertEquals(expected, coordinator.sentMessages.get(participant).get(0));
+ }
+
+ private static class InstrumentedCoordinatorSession extends CoordinatorSession
+ {
+ public InstrumentedCoordinatorSession(Builder builder)
+ {
+ super(builder);
+ }
+
+ Map<InetAddressAndPort, List<RepairMessage>> sentMessages = new HashMap<>();
+
+ protected void sendMessage(InetAddressAndPort destination, Message<RepairMessage> message)
+ {
+ if (!sentMessages.containsKey(destination))
+ {
+ sentMessages.put(destination, new ArrayList<>());
+ }
+ sentMessages.get(destination).add(message.payload);
+ }
+
+ Runnable onSetRepairing = null;
+ boolean setRepairingCalled = false;
+ public synchronized void setRepairing()
+ {
+ setRepairingCalled = true;
+ if (onSetRepairing != null)
+ {
+ onSetRepairing.run();
+ }
+ super.setRepairing();
+ }
+
+ Runnable onFinalizeCommit = null;
+ boolean finalizeCommitCalled = false;
+ public synchronized void finalizeCommit()
+ {
+ finalizeCommitCalled = true;
+ if (onFinalizeCommit != null)
+ {
+ onFinalizeCommit.run();
+ }
+ super.finalizeCommit();
+ }
+
+ Runnable onFail = null;
+ boolean failCalled = false;
+ public synchronized void fail()
+ {
+ failCalled = true;
+ if (onFail != null)
+ {
+ onFail.run();
+ }
+ super.fail();
+ }
+ }
+
+ /**
+ * Coordinator state should only switch after all participants are set
+ */
+ @Test
+ public void setPeerState()
+ {
+ CoordinatorSession session = createSession();
+ Assert.assertEquals(PREPARING, session.getState());
+
+ session.setParticipantState(PARTICIPANT1, PREPARED);
+ Assert.assertEquals(PREPARING, session.getState());
+
+ session.setParticipantState(PARTICIPANT2, PREPARED);
+ Assert.assertEquals(PREPARING, session.getState());
+
+ session.setParticipantState(PARTICIPANT3, PREPARED);
+ Assert.assertEquals(PREPARED, session.getState());
+ }
+
+ @Test
+ public void hasFailed()
+ {
+ CoordinatorSession session;
+
+ // participant failure
+ session = createSession();
+ Assert.assertFalse(session.hasFailed());
+ session.setParticipantState(PARTICIPANT1, FAILED);
+ Assert.assertTrue(session.hasFailed());
+
+ // coordinator failure
+ session = createSession();
+ Assert.assertFalse(session.hasFailed());
+ session.setState(FAILED);
+ Assert.assertTrue(session.hasFailed());
+ }
+
+ /**
+ * Coordinator should only send out failures messages once
+ */
+ @Test
+ public void multipleFailures()
+ {
+ InstrumentedCoordinatorSession coordinator = createInstrumentedSession();
+
+ Assert.assertEquals(PREPARING, coordinator.getState());
+ Assert.assertTrue(coordinator.sentMessages.isEmpty());
+
+ coordinator.fail();
+ Assert.assertEquals(FAILED, coordinator.getState());
+ for (InetAddressAndPort participant : PARTICIPANTS)
+ {
+ assertMessageSent(coordinator, participant, new FailSession(coordinator.sessionID));
+ }
+
+ coordinator.sentMessages.clear();
+ coordinator.fail();
+ Assert.assertEquals(FAILED, coordinator.getState());
+ Assert.assertTrue(coordinator.sentMessages.isEmpty());
+ }
+
+ /**
+ * Tests the complete coordinator side consistent repair cycle
+ */
+ @Test
+ public void successCase()
+ {
+ InstrumentedCoordinatorSession coordinator = createInstrumentedSession();
+ AtomicBoolean repairSubmitted = new AtomicBoolean(false);
+ SettableFuture<List<RepairSessionResult>> repairFuture = SettableFuture.create();
+ Supplier<ListenableFuture<List<RepairSessionResult>>> sessionSupplier = () ->
+ {
+ repairSubmitted.set(true);
+ return repairFuture;
+ };
+
+ // coordinator sends prepare requests to create local session and perform anticompaction
+ AtomicBoolean hasFailures = new AtomicBoolean(false);
+ Assert.assertFalse(repairSubmitted.get());
+ Assert.assertTrue(coordinator.sentMessages.isEmpty());
+ ListenableFuture sessionResult = coordinator.execute(sessionSupplier, hasFailures);
+
+ for (InetAddressAndPort participant : PARTICIPANTS)
+ {
+
+ RepairMessage expected = new PrepareConsistentRequest(coordinator.sessionID, COORDINATOR, new HashSet<>(PARTICIPANTS));
+ assertMessageSent(coordinator, participant, expected);
+ }
+
+ // participants respond to coordinator, and repair begins once all participants have responded with success
+ Assert.assertEquals(ConsistentSession.State.PREPARING, coordinator.getState());
+
+ coordinator.handlePrepareResponse(PARTICIPANT1, true);
+ Assert.assertEquals(ConsistentSession.State.PREPARING, coordinator.getState());
+
+ coordinator.handlePrepareResponse(PARTICIPANT2, true);
+ Assert.assertEquals(ConsistentSession.State.PREPARING, coordinator.getState());
+
+ // set the setRepairing callback to verify the correct state when it's called
+ Assert.assertFalse(coordinator.setRepairingCalled);
+ coordinator.onSetRepairing = () -> Assert.assertEquals(PREPARED, coordinator.getState());
+ coordinator.handlePrepareResponse(PARTICIPANT3, true);
+ Assert.assertTrue(coordinator.setRepairingCalled);
+ Assert.assertTrue(repairSubmitted.get());
+
+ Assert.assertEquals(ConsistentSession.State.REPAIRING, coordinator.getState());
+
+ ArrayList<RepairSessionResult> results = Lists.newArrayList(createResult(coordinator),
+ createResult(coordinator),
+ createResult(coordinator));
+
+ coordinator.sentMessages.clear();
+ repairFuture.set(results);
+
+ // propose messages should have been sent once all repair sessions completed successfully
+ for (InetAddressAndPort participant : PARTICIPANTS)
+ {
+ RepairMessage expected = new FinalizePropose(coordinator.sessionID);
+ assertMessageSent(coordinator, participant, expected);
+ }
+
+ // finalize commit messages will be sent once all participants respond with a promize to finalize
+ coordinator.sentMessages.clear();
+ Assert.assertEquals(ConsistentSession.State.REPAIRING, coordinator.getState());
+
+ coordinator.handleFinalizePromise(PARTICIPANT1, true);
+ Assert.assertEquals(ConsistentSession.State.REPAIRING, coordinator.getState());
+
+ coordinator.handleFinalizePromise(PARTICIPANT2, true);
+ Assert.assertEquals(ConsistentSession.State.REPAIRING, coordinator.getState());
+
+ // set the finalizeCommit callback so we can verify the state when it's called
+ Assert.assertFalse(coordinator.finalizeCommitCalled);
+ coordinator.onFinalizeCommit = () -> Assert.assertEquals(FINALIZE_PROMISED, coordinator.getState());
+ coordinator.handleFinalizePromise(PARTICIPANT3, true);
+ Assert.assertTrue(coordinator.finalizeCommitCalled);
+
+ Assert.assertEquals(ConsistentSession.State.FINALIZED, coordinator.getState());
+ for (InetAddressAndPort participant : PARTICIPANTS)
+ {
+ RepairMessage expected = new FinalizeCommit(coordinator.sessionID);
+ assertMessageSent(coordinator, participant, expected);
+ }
+
+ Assert.assertTrue(sessionResult.isDone());
+ Assert.assertFalse(hasFailures.get());
+ }
+
+ @Test
+ public void failedRepairs()
+ {
+ InstrumentedCoordinatorSession coordinator = createInstrumentedSession();
+ AtomicBoolean repairSubmitted = new AtomicBoolean(false);
+ SettableFuture<List<RepairSessionResult>> repairFuture = SettableFuture.create();
+ Supplier<ListenableFuture<List<RepairSessionResult>>> sessionSupplier = () ->
+ {
+ repairSubmitted.set(true);
+ return repairFuture;
+ };
+
+ // coordinator sends prepare requests to create local session and perform anticompaction
+ AtomicBoolean hasFailures = new AtomicBoolean(false);
+ Assert.assertFalse(repairSubmitted.get());
+ Assert.assertTrue(coordinator.sentMessages.isEmpty());
+ ListenableFuture sessionResult = coordinator.execute(sessionSupplier, hasFailures);
+ for (InetAddressAndPort participant : PARTICIPANTS)
+ {
+ PrepareConsistentRequest expected = new PrepareConsistentRequest(coordinator.sessionID, COORDINATOR, new HashSet<>(PARTICIPANTS));
+ assertMessageSent(coordinator, participant, expected);
+ }
+
+ // participants respond to coordinator, and repair begins once all participants have responded with success
+ Assert.assertEquals(ConsistentSession.State.PREPARING, coordinator.getState());
+
+ coordinator.handlePrepareResponse(PARTICIPANT1, true);
+ Assert.assertEquals(ConsistentSession.State.PREPARING, coordinator.getState());
+
+ coordinator.handlePrepareResponse(PARTICIPANT2, true);
+ Assert.assertEquals(ConsistentSession.State.PREPARING, coordinator.getState());
+
+ // set the setRepairing callback to verify the correct state when it's called
+ Assert.assertFalse(coordinator.setRepairingCalled);
+ coordinator.onSetRepairing = () -> Assert.assertEquals(PREPARED, coordinator.getState());
+ coordinator.handlePrepareResponse(PARTICIPANT3, true);
+ Assert.assertTrue(coordinator.setRepairingCalled);
+ Assert.assertTrue(repairSubmitted.get());
+
+ Assert.assertEquals(ConsistentSession.State.REPAIRING, coordinator.getState());
+
+ ArrayList<RepairSessionResult> results = Lists.newArrayList(createResult(coordinator),
+ null,
+ createResult(coordinator));
+
+ coordinator.sentMessages.clear();
+ Assert.assertFalse(coordinator.failCalled);
+ coordinator.onFail = () -> Assert.assertEquals(REPAIRING, coordinator.getState());
+ repairFuture.set(results);
+ Assert.assertTrue(coordinator.failCalled);
+
+ // all participants should have been notified of session failure
+ for (InetAddressAndPort participant : PARTICIPANTS)
+ {
+ RepairMessage expected = new FailSession(coordinator.sessionID);
+ assertMessageSent(coordinator, participant, expected);
+ }
+
+ Assert.assertTrue(sessionResult.isDone());
+ Assert.assertTrue(hasFailures.get());
+ }
+
+ @Test
+ public void failedPrepare()
+ {
+ InstrumentedCoordinatorSession coordinator = createInstrumentedSession();
+ AtomicBoolean repairSubmitted = new AtomicBoolean(false);
+ SettableFuture<List<RepairSessionResult>> repairFuture = SettableFuture.create();
+ Supplier<ListenableFuture<List<RepairSessionResult>>> sessionSupplier = () ->
+ {
+ repairSubmitted.set(true);
+ return repairFuture;
+ };
+
+ // coordinator sends prepare requests to create local session and perform anticompaction
+ AtomicBoolean hasFailures = new AtomicBoolean(false);
+ Assert.assertFalse(repairSubmitted.get());
+ Assert.assertTrue(coordinator.sentMessages.isEmpty());
+ ListenableFuture sessionResult = coordinator.execute(sessionSupplier, hasFailures);
+ for (InetAddressAndPort participant : PARTICIPANTS)
+ {
+ PrepareConsistentRequest expected = new PrepareConsistentRequest(coordinator.sessionID, COORDINATOR, new HashSet<>(PARTICIPANTS));
+ assertMessageSent(coordinator, participant, expected);
+ }
+
+ coordinator.sentMessages.clear();
+
+ // participants respond to coordinator, and repair begins once all participants have responded
+ Assert.assertEquals(ConsistentSession.State.PREPARING, coordinator.getState());
+
+ coordinator.handlePrepareResponse(PARTICIPANT1, true);
+ Assert.assertEquals(ConsistentSession.State.PREPARING, coordinator.getState());
+ Assert.assertEquals(PREPARED, coordinator.getParticipantState(PARTICIPANT1));
+ Assert.assertFalse(sessionResult.isDone());
+
+ // participant 2 fails to prepare for consistent repair
+ Assert.assertFalse(coordinator.failCalled);
+ coordinator.handlePrepareResponse(PARTICIPANT2, false);
+ Assert.assertEquals(ConsistentSession.State.PREPARING, coordinator.getState());
+ // we should have sent failure messages to the other participants, but not yet marked them failed internally
+ assertMessageSent(coordinator, PARTICIPANT1, new FailSession(coordinator.sessionID));
+ assertMessageSent(coordinator, PARTICIPANT2, new FailSession(coordinator.sessionID));
+ assertMessageSent(coordinator, PARTICIPANT3, new FailSession(coordinator.sessionID));
+ Assert.assertEquals(FAILED, coordinator.getParticipantState(PARTICIPANT2));
+ Assert.assertEquals(PREPARED, coordinator.getParticipantState(PARTICIPANT1));
+ Assert.assertEquals(PREPARING, coordinator.getParticipantState(PARTICIPANT3));
+ Assert.assertFalse(sessionResult.isDone());
+ Assert.assertFalse(coordinator.failCalled);
+ coordinator.sentMessages.clear();
+
+ // last outstanding response should cause repair to complete in failed state
+ Assert.assertFalse(coordinator.setRepairingCalled);
+ coordinator.onSetRepairing = Assert::fail;
+ coordinator.handlePrepareResponse(PARTICIPANT3, true);
+ Assert.assertTrue(coordinator.failCalled);
+ Assert.assertFalse(coordinator.setRepairingCalled);
+ Assert.assertFalse(repairSubmitted.get());
+
+ // all participants that did not fail should have been notified of session failure
+ RepairMessage expected = new FailSession(coordinator.sessionID);
+ assertMessageSent(coordinator, PARTICIPANT1, expected);
+ assertMessageSent(coordinator, PARTICIPANT3, expected);
+ Assert.assertFalse(coordinator.sentMessages.containsKey(PARTICIPANT2));
+
+ Assert.assertTrue(sessionResult.isDone());
+ Assert.assertTrue(hasFailures.get());
+ }
+
+ @Test
+ public void failedPropose()
+ {
+ InstrumentedCoordinatorSession coordinator = createInstrumentedSession();
+ AtomicBoolean repairSubmitted = new AtomicBoolean(false);
+ SettableFuture<List<RepairSessionResult>> repairFuture = SettableFuture.create();
+ Supplier<ListenableFuture<List<RepairSessionResult>>> sessionSupplier = () ->
+ {
+ repairSubmitted.set(true);
+ return repairFuture;
+ };
+
+ // coordinator sends prepare requests to create local session and perform anticompaction
+ AtomicBoolean hasFailures = new AtomicBoolean(false);
+ Assert.assertFalse(repairSubmitted.get());
+ Assert.assertTrue(coordinator.sentMessages.isEmpty());
+ ListenableFuture sessionResult = coordinator.execute(sessionSupplier, hasFailures);
+
+ for (InetAddressAndPort participant : PARTICIPANTS)
+ {
+
+ RepairMessage expected = new PrepareConsistentRequest(coordinator.sessionID, COORDINATOR, new HashSet<>(PARTICIPANTS));
+ assertMessageSent(coordinator, participant, expected);
+ }
+
+ // participants respond to coordinator, and repair begins once all participants have responded with success
+ Assert.assertEquals(ConsistentSession.State.PREPARING, coordinator.getState());
+
+ coordinator.handlePrepareResponse(PARTICIPANT1, true);
+ Assert.assertEquals(ConsistentSession.State.PREPARING, coordinator.getState());
+
+ coordinator.handlePrepareResponse(PARTICIPANT2, true);
+ Assert.assertEquals(ConsistentSession.State.PREPARING, coordinator.getState());
+
+ // set the setRepairing callback to verify the correct state when it's called
+ Assert.assertFalse(coordinator.setRepairingCalled);
+ coordinator.onSetRepairing = () -> Assert.assertEquals(PREPARED, coordinator.getState());
+ coordinator.handlePrepareResponse(PARTICIPANT3, true);
+ Assert.assertTrue(coordinator.setRepairingCalled);
+ Assert.assertTrue(repairSubmitted.get());
+
+ Assert.assertEquals(ConsistentSession.State.REPAIRING, coordinator.getState());
+
+ ArrayList<RepairSessionResult> results = Lists.newArrayList(createResult(coordinator),
+ createResult(coordinator),
+ createResult(coordinator));
+
+ coordinator.sentMessages.clear();
+ repairFuture.set(results);
+
+ // propose messages should have been sent once all repair sessions completed successfully
+ for (InetAddressAndPort participant : PARTICIPANTS)
+ {
+ RepairMessage expected = new FinalizePropose(coordinator.sessionID);
+ assertMessageSent(coordinator, participant, expected);
+ }
+
+ // finalize commit messages will be sent once all participants respond with a promize to finalize
+ coordinator.sentMessages.clear();
+ Assert.assertEquals(ConsistentSession.State.REPAIRING, coordinator.getState());
+
+ coordinator.handleFinalizePromise(PARTICIPANT1, true);
+ Assert.assertEquals(ConsistentSession.State.REPAIRING, coordinator.getState());
+
+ Assert.assertFalse(coordinator.failCalled);
+ coordinator.handleFinalizePromise(PARTICIPANT2, false);
+ Assert.assertEquals(ConsistentSession.State.FAILED, coordinator.getState());
+ Assert.assertTrue(coordinator.failCalled);
+
+ // additional success messages should be ignored
+ Assert.assertFalse(coordinator.finalizeCommitCalled);
+ coordinator.onFinalizeCommit = Assert::fail;
+ coordinator.handleFinalizePromise(PARTICIPANT3, true);
+ Assert.assertFalse(coordinator.finalizeCommitCalled);
+ Assert.assertEquals(ConsistentSession.State.FAILED, coordinator.getState());
+
+ // failure messages should have been sent to all participants
+ for (InetAddressAndPort participant : PARTICIPANTS)
+ {
+ RepairMessage expected = new FailSession(coordinator.sessionID);
+ assertMessageSent(coordinator, participant, expected);
+ }
+
+ Assert.assertTrue(sessionResult.isDone());
+ Assert.assertTrue(hasFailures.get());
+ }
+}
diff --git a/test/unit/org/apache/cassandra/repair/consistent/CoordinatorSessionsTest.java b/test/unit/org/apache/cassandra/repair/consistent/CoordinatorSessionsTest.java
new file mode 100644
index 0000000..b9b1fbf
--- /dev/null
+++ b/test/unit/org/apache/cassandra/repair/consistent/CoordinatorSessionsTest.java
@@ -0,0 +1,208 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.repair.consistent;
+
+import java.util.Set;
+import java.util.UUID;
+
+import com.google.common.collect.Sets;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.cql3.statements.schema.CreateTableStatement;
+import org.apache.cassandra.repair.AbstractRepairTest;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.repair.messages.FailSession;
+import org.apache.cassandra.repair.messages.FinalizePromise;
+import org.apache.cassandra.repair.messages.PrepareConsistentResponse;
+import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.service.ActiveRepairService;
+import org.apache.cassandra.utils.UUIDGen;
+
+public class CoordinatorSessionsTest extends AbstractRepairTest
+{
+ private static TableMetadata cfm;
+ private static ColumnFamilyStore cfs;
+
+ // to check CoordinatorSessions is passing the messages to the coordinator session correctly
+ private static class InstrumentedCoordinatorSession extends CoordinatorSession
+ {
+
+ public InstrumentedCoordinatorSession(Builder builder)
+ {
+ super(builder);
+ }
+
+ int prepareResponseCalls = 0;
+ InetAddressAndPort preparePeer = null;
+ boolean prepareSuccess = false;
+ public synchronized void handlePrepareResponse(InetAddressAndPort participant, boolean success)
+ {
+ prepareResponseCalls++;
+ preparePeer = participant;
+ prepareSuccess = success;
+ }
+
+ int finalizePromiseCalls = 0;
+ InetAddressAndPort promisePeer = null;
+ boolean promiseSuccess = false;
+ public synchronized void handleFinalizePromise(InetAddressAndPort participant, boolean success)
+ {
+ finalizePromiseCalls++;
+ promisePeer = participant;
+ promiseSuccess = success;
+ }
+
+ int failCalls = 0;
+ public synchronized void fail()
+ {
+ failCalls++;
+ }
+ }
+
+ private static class InstrumentedCoordinatorSessions extends CoordinatorSessions
+ {
+ protected CoordinatorSession buildSession(CoordinatorSession.Builder builder)
+ {
+ return new InstrumentedCoordinatorSession(builder);
+ }
+
+ public InstrumentedCoordinatorSession getSession(UUID sessionId)
+ {
+ return (InstrumentedCoordinatorSession) super.getSession(sessionId);
+ }
+
+ public InstrumentedCoordinatorSession registerSession(UUID sessionId, Set<InetAddressAndPort> peers, boolean isForced)
+ {
+ return (InstrumentedCoordinatorSession) super.registerSession(sessionId, peers, isForced);
+ }
+ }
+
+ @BeforeClass
+ public static void setupClass()
+ {
+ SchemaLoader.prepareServer();
+ cfm = CreateTableStatement.parse("CREATE TABLE tbl (k INT PRIMARY KEY, v INT)", "coordinatorsessiontest").build();
+ SchemaLoader.createKeyspace("coordinatorsessiontest", KeyspaceParams.simple(1), cfm);
+ cfs = Schema.instance.getColumnFamilyStoreInstance(cfm.id);
+ }
+
+ private static UUID registerSession()
+ {
+ return registerSession(cfs, true, true);
+ }
+
+ @Test
+ public void registerSessionTest()
+ {
+ CoordinatorSessions sessions = new CoordinatorSessions();
+ UUID sessionID = registerSession();
+ CoordinatorSession session = sessions.registerSession(sessionID, PARTICIPANTS, false);
+
+ Assert.assertEquals(ConsistentSession.State.PREPARING, session.getState());
+ Assert.assertEquals(sessionID, session.sessionID);
+ Assert.assertEquals(COORDINATOR, session.coordinator);
+ Assert.assertEquals(Sets.newHashSet(cfm.id), session.tableIds);
+
+ ActiveRepairService.ParentRepairSession prs = ActiveRepairService.instance.getParentRepairSession(sessionID);
+ Assert.assertEquals(prs.repairedAt, session.repairedAt);
+ Assert.assertEquals(prs.getRanges(), session.ranges);
+ Assert.assertEquals(PARTICIPANTS, session.participants);
+
+ Assert.assertSame(session, sessions.getSession(sessionID));
+ }
+
+ @Test
+ public void handlePrepareResponse()
+ {
+ InstrumentedCoordinatorSessions sessions = new InstrumentedCoordinatorSessions();
+ UUID sessionID = registerSession();
+
+ InstrumentedCoordinatorSession session = sessions.registerSession(sessionID, PARTICIPANTS, false);
+ Assert.assertEquals(0, session.prepareResponseCalls);
+
+ sessions.handlePrepareResponse(new PrepareConsistentResponse(sessionID, PARTICIPANT1, true));
+ Assert.assertEquals(1, session.prepareResponseCalls);
+ Assert.assertEquals(PARTICIPANT1, session.preparePeer);
+ Assert.assertEquals(true, session.prepareSuccess);
+ }
+
+ @Test
+ public void handlePrepareResponseNoSession()
+ {
+ InstrumentedCoordinatorSessions sessions = new InstrumentedCoordinatorSessions();
+ UUID fakeID = UUIDGen.getTimeUUID();
+
+ sessions.handlePrepareResponse(new PrepareConsistentResponse(fakeID, PARTICIPANT1, true));
+ Assert.assertNull(sessions.getSession(fakeID));
+ }
+
+ @Test
+ public void handlePromiseResponse()
+ {
+ InstrumentedCoordinatorSessions sessions = new InstrumentedCoordinatorSessions();
+ UUID sessionID = registerSession();
+
+ InstrumentedCoordinatorSession session = sessions.registerSession(sessionID, PARTICIPANTS, false);
+ Assert.assertEquals(0, session.finalizePromiseCalls);
+
+ sessions.handleFinalizePromiseMessage(new FinalizePromise(sessionID, PARTICIPANT1, true));
+ Assert.assertEquals(1, session.finalizePromiseCalls);
+ Assert.assertEquals(PARTICIPANT1, session.promisePeer);
+ Assert.assertEquals(true, session.promiseSuccess);
+ }
+
+ @Test
+ public void handlePromiseResponseNoSession()
+ {
+ InstrumentedCoordinatorSessions sessions = new InstrumentedCoordinatorSessions();
+ UUID fakeID = UUIDGen.getTimeUUID();
+
+ sessions.handleFinalizePromiseMessage(new FinalizePromise(fakeID, PARTICIPANT1, true));
+ Assert.assertNull(sessions.getSession(fakeID));
+ }
+
+ @Test
+ public void handleFailureMessage()
+ {
+ InstrumentedCoordinatorSessions sessions = new InstrumentedCoordinatorSessions();
+ UUID sessionID = registerSession();
+
+ InstrumentedCoordinatorSession session = sessions.registerSession(sessionID, PARTICIPANTS, false);
+ Assert.assertEquals(0, session.failCalls);
+
+ sessions.handleFailSessionMessage(new FailSession(sessionID));
+ Assert.assertEquals(1, session.failCalls);
+ }
+
+ @Test
+ public void handleFailureMessageNoSession()
+ {
+ InstrumentedCoordinatorSessions sessions = new InstrumentedCoordinatorSessions();
+ UUID fakeID = UUIDGen.getTimeUUID();
+
+ sessions.handleFailSessionMessage(new FailSession(fakeID));
+ Assert.assertNull(sessions.getSession(fakeID));
+ }
+}
diff --git a/test/unit/org/apache/cassandra/repair/consistent/LocalSessionAccessor.java b/test/unit/org/apache/cassandra/repair/consistent/LocalSessionAccessor.java
new file mode 100644
index 0000000..a7e8272
--- /dev/null
+++ b/test/unit/org/apache/cassandra/repair/consistent/LocalSessionAccessor.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.repair.consistent;
+
+import java.util.Set;
+import java.util.UUID;
+
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.service.ActiveRepairService;
+
+/**
+ * makes package private hacks available to compaction tests
+ */
+public class LocalSessionAccessor
+{
+ private static final ActiveRepairService ARS = ActiveRepairService.instance;
+
+ public static void startup()
+ {
+ ARS.consistent.local.start();
+ }
+
+ public static void prepareUnsafe(UUID sessionID, InetAddressAndPort coordinator, Set<InetAddressAndPort> peers)
+ {
+ ActiveRepairService.ParentRepairSession prs = ARS.getParentRepairSession(sessionID);
+ assert prs != null;
+ LocalSession session = ARS.consistent.local.createSessionUnsafe(sessionID, prs, peers);
+ ARS.consistent.local.putSessionUnsafe(session);
+ }
+
+ public static long finalizeUnsafe(UUID sessionID)
+ {
+ LocalSession session = ARS.consistent.local.getSession(sessionID);
+ assert session != null;
+ session.setState(ConsistentSession.State.FINALIZED);
+ ARS.consistent.local.save(session);
+ return session.repairedAt;
+ }
+
+ public static void failUnsafe(UUID sessionID)
+ {
+ LocalSession session = ARS.consistent.local.getSession(sessionID);
+ assert session != null;
+ session.setState(ConsistentSession.State.FAILED);
+ ARS.consistent.local.save(session);
+ }
+}
diff --git a/test/unit/org/apache/cassandra/repair/consistent/LocalSessionTest.java b/test/unit/org/apache/cassandra/repair/consistent/LocalSessionTest.java
new file mode 100644
index 0000000..15fd1fc
--- /dev/null
+++ b/test/unit/org/apache/cassandra/repair/consistent/LocalSessionTest.java
@@ -0,0 +1,1009 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.repair.consistent;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.BooleanSupplier;
+import java.util.function.Consumer;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.SettableFuture;
+
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.cql3.QueryProcessor;
+import org.apache.cassandra.cql3.statements.schema.CreateTableStatement;
+import org.apache.cassandra.locator.RangesAtEndpoint;
+import org.apache.cassandra.net.Message;
+import org.apache.cassandra.repair.AbstractRepairTest;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.repair.KeyspaceRepairManager;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.SchemaConstants;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.SystemKeyspace;
+import org.apache.cassandra.repair.messages.FailSession;
+import org.apache.cassandra.repair.messages.FinalizeCommit;
+import org.apache.cassandra.repair.messages.FinalizePromise;
+import org.apache.cassandra.repair.messages.FinalizePropose;
+import org.apache.cassandra.repair.messages.PrepareConsistentRequest;
+import org.apache.cassandra.repair.messages.PrepareConsistentResponse;
+import org.apache.cassandra.repair.messages.RepairMessage;
+import org.apache.cassandra.repair.messages.StatusRequest;
+import org.apache.cassandra.repair.messages.StatusResponse;
+import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.service.ActiveRepairService;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.UUIDGen;
+
+import static org.apache.cassandra.repair.consistent.ConsistentSession.State.*;
+
+public class LocalSessionTest extends AbstractRepairTest
+{
+
+ static LocalSession.Builder createBuilder()
+ {
+ LocalSession.Builder builder = LocalSession.builder();
+ builder.withState(PREPARING);
+ builder.withSessionID(UUIDGen.getTimeUUID());
+ builder.withCoordinator(COORDINATOR);
+ builder.withUUIDTableIds(Sets.newHashSet(UUIDGen.getTimeUUID(), UUIDGen.getTimeUUID()));
+ builder.withRepairedAt(System.currentTimeMillis());
+ builder.withRanges(Sets.newHashSet(RANGE1, RANGE2, RANGE3));
+ builder.withParticipants(Sets.newHashSet(PARTICIPANT1, PARTICIPANT2, PARTICIPANT3));
+
+ int now = FBUtilities.nowInSeconds();
+ builder.withStartedAt(now);
+ builder.withLastUpdate(now);
+
+ return builder;
+ }
+
+ static LocalSession createSession()
+ {
+ return createBuilder().build();
+ }
+
+ private static void assertValidationFailure(Consumer<LocalSession.Builder> consumer)
+ {
+ try
+ {
+ LocalSession.Builder builder = createBuilder();
+ consumer.accept(builder);
+ builder.build();
+ Assert.fail("Expected assertion error");
+ }
+ catch (IllegalArgumentException e)
+ {
+ // expected
+ }
+ }
+
+ private static void assertNoMessagesSent(InstrumentedLocalSessions sessions, InetAddressAndPort to)
+ {
+ Assert.assertNull(sessions.sentMessages.get(to));
+ }
+
+ private static void assertMessagesSent(InstrumentedLocalSessions sessions, InetAddressAndPort to, RepairMessage... expected)
+ {
+ Assert.assertEquals(Lists.newArrayList(expected), sessions.sentMessages.get(to));
+ }
+
+ static class InstrumentedLocalSessions extends LocalSessions
+ {
+ Map<InetAddressAndPort, List<RepairMessage>> sentMessages = new HashMap<>();
+
+ protected void sendMessage(InetAddressAndPort destination, Message<? extends RepairMessage> message)
+ {
+ if (!sentMessages.containsKey(destination))
+ {
+ sentMessages.put(destination, new ArrayList<>());
+ }
+ sentMessages.get(destination).add(message.payload);
+ }
+
+ SettableFuture<Object> prepareSessionFuture = null;
+ boolean prepareSessionCalled = false;
+
+ @Override
+ ListenableFuture prepareSession(KeyspaceRepairManager repairManager,
+ UUID sessionID,
+ Collection<ColumnFamilyStore> tables,
+ RangesAtEndpoint ranges,
+ ExecutorService executor,
+ BooleanSupplier isCancelled)
+ {
+ prepareSessionCalled = true;
+ if (prepareSessionFuture != null)
+ {
+ return prepareSessionFuture;
+ }
+ else
+ {
+ return super.prepareSession(repairManager, sessionID, tables, ranges, executor, isCancelled);
+ }
+ }
+
+ boolean failSessionCalled = false;
+ public void failSession(UUID sessionID, boolean sendMessage)
+ {
+ failSessionCalled = true;
+ super.failSession(sessionID, sendMessage);
+ }
+
+ public LocalSession prepareForTest(UUID sessionID)
+ {
+ prepareSessionFuture = SettableFuture.create();
+ handlePrepareMessage(PARTICIPANT1, new PrepareConsistentRequest(sessionID, COORDINATOR, PARTICIPANTS));
+ prepareSessionFuture.set(new Object());
+ sentMessages.clear();
+ return getSession(sessionID);
+ }
+
+ @Override
+ protected InetAddressAndPort getBroadcastAddressAndPort()
+ {
+ return PARTICIPANT1;
+ }
+
+ protected boolean isAlive(InetAddressAndPort address)
+ {
+ return true;
+ }
+
+ protected boolean isNodeInitialized()
+ {
+ return true;
+ }
+
+ public Map<UUID, Integer> completedSessions = new HashMap<>();
+
+ protected void sessionCompleted(LocalSession session)
+ {
+ UUID sessionID = session.sessionID;
+ int calls = completedSessions.getOrDefault(sessionID, 0);
+ completedSessions.put(sessionID, calls + 1);
+ }
+
+ boolean sessionHasData = false;
+ protected boolean sessionHasData(LocalSession session)
+ {
+ return sessionHasData;
+ }
+ }
+
+ private static TableMetadata cfm;
+ private static ColumnFamilyStore cfs;
+
+ @BeforeClass
+ public static void setupClass()
+ {
+ SchemaLoader.prepareServer();
+ cfm = CreateTableStatement.parse("CREATE TABLE tbl (k INT PRIMARY KEY, v INT)", "localsessiontest").build();
+ SchemaLoader.createKeyspace("localsessiontest", KeyspaceParams.simple(1), cfm);
+ cfs = Schema.instance.getColumnFamilyStoreInstance(cfm.id);
+ }
+
+ @Before
+ public void setup()
+ {
+ // clear out any data from previous test runs
+ ColumnFamilyStore repairCfs = Keyspace.open(SchemaConstants.SYSTEM_KEYSPACE_NAME).getColumnFamilyStore(SystemKeyspace.REPAIRS);
+ repairCfs.truncateBlocking();
+ }
+
+ private static UUID registerSession()
+ {
+ return registerSession(cfs, true, true);
+ }
+
+ @Test
+ public void validation()
+ {
+ assertValidationFailure(b -> b.withState(null));
+ assertValidationFailure(b -> b.withSessionID(null));
+ assertValidationFailure(b -> b.withCoordinator(null));
+ assertValidationFailure(b -> b.withTableIds(null));
+ assertValidationFailure(b -> b.withTableIds(new HashSet<>()));
+ assertValidationFailure(b -> b.withRepairedAt(-1));
+ assertValidationFailure(b -> b.withRanges(null));
+ assertValidationFailure(b -> b.withRanges(new HashSet<>()));
+ assertValidationFailure(b -> b.withParticipants(null));
+ assertValidationFailure(b -> b.withParticipants(new HashSet<>()));
+ assertValidationFailure(b -> b.withStartedAt(0));
+ assertValidationFailure(b -> b.withLastUpdate(0));
+ }
+
+ /**
+ * Test that sessions are loaded and saved properly
+ */
+ @Test
+ public void persistence()
+ {
+ LocalSessions sessions = new LocalSessions();
+ LocalSession expected = createSession();
+ sessions.save(expected);
+ LocalSession actual = sessions.loadUnsafe(expected.sessionID);
+ Assert.assertEquals(expected, actual);
+ }
+
+ @Test
+ public void prepareSuccessCase()
+ {
+ UUID sessionID = registerSession();
+ InstrumentedLocalSessions sessions = new InstrumentedLocalSessions();
+ sessions.start();
+
+ // replacing future so we can inspect state before and after anti compaction callback
+ sessions.prepareSessionFuture = SettableFuture.create();
+ Assert.assertFalse(sessions.prepareSessionCalled);
+ sessions.handlePrepareMessage(PARTICIPANT1, new PrepareConsistentRequest(sessionID, COORDINATOR, PARTICIPANTS));
+ Assert.assertTrue(sessions.prepareSessionCalled);
+ Assert.assertTrue(sessions.sentMessages.isEmpty());
+
+ // anti compaction hasn't finished yet, so state in memory and on disk should be PREPARING
+ LocalSession session = sessions.getSession(sessionID);
+ Assert.assertNotNull(session);
+ Assert.assertEquals(PREPARING, session.getState());
+ Assert.assertEquals(session, sessions.loadUnsafe(sessionID));
+
+ // anti compaction has now finished, so state in memory and on disk should be PREPARED
+ sessions.prepareSessionFuture.set(new Object());
+ session = sessions.getSession(sessionID);
+ Assert.assertNotNull(session);
+ Assert.assertEquals(PREPARED, session.getState());
+ Assert.assertEquals(session, sessions.loadUnsafe(sessionID));
+
+ // ...and we should have sent a success message back to the coordinator
+ assertMessagesSent(sessions, COORDINATOR, new PrepareConsistentResponse(sessionID, PARTICIPANT1, true));
+ }
+
+ /**
+ * If anti compactionn fails, we should fail the session locally,
+ * and send a failure message back to the coordinator
+ */
+ @Test
+ public void prepareAntiCompactFailure()
+ {
+ UUID sessionID = registerSession();
+ InstrumentedLocalSessions sessions = new InstrumentedLocalSessions();
+ sessions.start();
+
+ // replacing future so we can inspect state before and after anti compaction callback
+ sessions.prepareSessionFuture = SettableFuture.create();
+ Assert.assertFalse(sessions.prepareSessionCalled);
+ sessions.handlePrepareMessage(PARTICIPANT1, new PrepareConsistentRequest(sessionID, COORDINATOR, PARTICIPANTS));
+ Assert.assertTrue(sessions.prepareSessionCalled);
+ Assert.assertTrue(sessions.sentMessages.isEmpty());
+
+ // anti compaction hasn't finished yet, so state in memory and on disk should be PREPARING
+ LocalSession session = sessions.getSession(sessionID);
+ Assert.assertNotNull(session);
+ Assert.assertEquals(PREPARING, session.getState());
+ Assert.assertEquals(session, sessions.loadUnsafe(sessionID));
+
+ // anti compaction has now finished, so state in memory and on disk should be PREPARED
+ sessions.prepareSessionFuture.setException(new RuntimeException());
+ session = sessions.getSession(sessionID);
+ Assert.assertNotNull(session);
+ Assert.assertEquals(FAILED, session.getState());
+ Assert.assertEquals(session, sessions.loadUnsafe(sessionID));
+
+ // ...and we should have sent a success message back to the coordinator
+ assertMessagesSent(sessions, COORDINATOR, new PrepareConsistentResponse(sessionID, PARTICIPANT1, false));
+
+ }
+
+ /**
+ * If a ParentRepairSession wasn't previously created, we shouldn't
+ * create a session locally, but we should send a failure message to
+ * the coordinator.
+ */
+ @Test
+ public void prepareWithNonExistantParentSession()
+ {
+ UUID sessionID = UUIDGen.getTimeUUID();
+ InstrumentedLocalSessions sessions = new InstrumentedLocalSessions();
+ sessions.handlePrepareMessage(PARTICIPANT1, new PrepareConsistentRequest(sessionID, COORDINATOR, PARTICIPANTS));
+ Assert.assertNull(sessions.getSession(sessionID));
+ assertMessagesSent(sessions, COORDINATOR, new PrepareConsistentResponse(sessionID, PARTICIPANT1, false));
+ }
+
+ /**
+ * If the session is cancelled mid-prepare, the isCancelled boolean supplier should start returning true
+ */
+ @Test
+ public void prepareCancellation()
+ {
+ UUID sessionID = registerSession();
+ AtomicReference<BooleanSupplier> isCancelledRef = new AtomicReference<>();
+ SettableFuture future = SettableFuture.create();
+
+ InstrumentedLocalSessions sessions = new InstrumentedLocalSessions() {
+ ListenableFuture prepareSession(KeyspaceRepairManager repairManager, UUID sessionID, Collection<ColumnFamilyStore> tables, RangesAtEndpoint ranges, ExecutorService executor, BooleanSupplier isCancelled)
+ {
+ isCancelledRef.set(isCancelled);
+ return future;
+ }
+ };
+ sessions.start();
+
+ sessions.handlePrepareMessage(PARTICIPANT1, new PrepareConsistentRequest(sessionID, COORDINATOR, PARTICIPANTS));
+
+ BooleanSupplier isCancelled = isCancelledRef.get();
+ Assert.assertNotNull(isCancelled);
+ Assert.assertFalse(isCancelled.getAsBoolean());
+ Assert.assertTrue(sessions.sentMessages.isEmpty());
+
+ sessions.failSession(sessionID, false);
+ Assert.assertTrue(isCancelled.getAsBoolean());
+
+ // now that the session has failed, it send a negative response to the coordinator (even if the anti-compaction completed successfully)
+ future.set(new Object());
+ assertMessagesSent(sessions, COORDINATOR, new PrepareConsistentResponse(sessionID, PARTICIPANT1, false));
+ }
+
+ @Test
+ public void maybeSetRepairing()
+ {
+ UUID sessionID = registerSession();
+ InstrumentedLocalSessions sessions = new InstrumentedLocalSessions();
+ sessions.start();
+
+ LocalSession session = sessions.prepareForTest(sessionID);
+ Assert.assertEquals(PREPARED, session.getState());
+
+ sessions.sentMessages.clear();
+ sessions.maybeSetRepairing(sessionID);
+ Assert.assertEquals(REPAIRING, session.getState());
+ Assert.assertEquals(session, sessions.loadUnsafe(sessionID));
+ Assert.assertTrue(sessions.sentMessages.isEmpty());
+ }
+
+ /**
+ * Multiple calls to maybeSetRepairing shouldn't cause any problems
+ */
+ @Test
+ public void maybeSetRepairingDuplicates()
+ {
+
+ UUID sessionID = registerSession();
+ InstrumentedLocalSessions sessions = new InstrumentedLocalSessions();
+ sessions.start();
+
+ LocalSession session = sessions.prepareForTest(sessionID);
+ Assert.assertEquals(PREPARED, session.getState());
+
+ // initial set
+ sessions.sentMessages.clear();
+ sessions.maybeSetRepairing(sessionID);
+ Assert.assertEquals(REPAIRING, session.getState());
+ Assert.assertEquals(session, sessions.loadUnsafe(sessionID));
+ Assert.assertTrue(sessions.sentMessages.isEmpty());
+
+ // repeated call 1
+ sessions.maybeSetRepairing(sessionID);
+ Assert.assertEquals(REPAIRING, session.getState());
+ Assert.assertEquals(session, sessions.loadUnsafe(sessionID));
+ Assert.assertTrue(sessions.sentMessages.isEmpty());
+
+ // repeated call 2
+ sessions.maybeSetRepairing(sessionID);
+ Assert.assertEquals(REPAIRING, session.getState());
+ Assert.assertEquals(session, sessions.loadUnsafe(sessionID));
+ Assert.assertTrue(sessions.sentMessages.isEmpty());
+ }
+
+ /**
+ * We shouldn't fail if we don't have a session for the given session id
+ */
+ @Test
+ public void maybeSetRepairingNonExistantSession()
+ {
+ InstrumentedLocalSessions sessions = new InstrumentedLocalSessions();
+ UUID fakeID = UUIDGen.getTimeUUID();
+ sessions.maybeSetRepairing(fakeID);
+ Assert.assertTrue(sessions.sentMessages.isEmpty());
+ }
+
+ /**
+ * In the success case, session state should be set to FINALIZE_PROMISED and
+ * persisted, and a FinalizePromise message should be sent back to the coordinator
+ */
+ @Test
+ public void finalizeProposeSuccessCase()
+ {
+ UUID sessionID = registerSession();
+ InstrumentedLocalSessions sessions = new InstrumentedLocalSessions();
+ sessions.start();
+
+ // create session and move to preparing
+ LocalSession session = sessions.prepareForTest(sessionID);
+ sessions.maybeSetRepairing(sessionID);
+
+ //
+ Assert.assertEquals(REPAIRING, session.getState());
+
+ // should send a promised message to coordinator and set session state accordingly
+ sessions.sentMessages.clear();
+ sessions.handleFinalizeProposeMessage(COORDINATOR, new FinalizePropose(sessionID));
+ Assert.assertEquals(FINALIZE_PROMISED, session.getState());
+ Assert.assertEquals(session, sessions.loadUnsafe(sessionID));
+ assertMessagesSent(sessions, COORDINATOR, new FinalizePromise(sessionID, PARTICIPANT1, true));
+ }
+
+ /**
+ * Trying to propose finalization when the session isn't in the repaired
+ * state should fail the session and send a failure message to the proposer
+ */
+ @Test
+ public void finalizeProposeInvalidStateFailure()
+ {
+ UUID sessionID = registerSession();
+ InstrumentedLocalSessions sessions = new InstrumentedLocalSessions();
+ sessions.start();
+
+ LocalSession session = sessions.prepareForTest(sessionID);
+ Assert.assertEquals(PREPARED, session.getState());
+
+ // should fail the session and send a failure message to the coordinator
+ sessions.sentMessages.clear();
+ sessions.handleFinalizeProposeMessage(COORDINATOR, new FinalizePropose(sessionID));
+ Assert.assertEquals(FAILED, session.getState());
+ Assert.assertEquals(session, sessions.loadUnsafe(sessionID));
+ assertMessagesSent(sessions, COORDINATOR, new FailSession(sessionID));
+ }
+
+ @Test
+ public void finalizeProposeNonExistantSessionFailure()
+ {
+ InstrumentedLocalSessions sessions = new InstrumentedLocalSessions();
+ UUID fakeID = UUIDGen.getTimeUUID();
+ sessions.handleFinalizeProposeMessage(COORDINATOR, new FinalizePropose(fakeID));
+ Assert.assertNull(sessions.getSession(fakeID));
+ assertMessagesSent(sessions, COORDINATOR, new FailSession(fakeID));
+ }
+
+ /**
+ * Session state should be set to finalized, sstables should be promoted
+ * to repaired. No messages should be sent to the coordinator
+ */
+ @Test
+ public void finalizeCommitSuccessCase()
+ {
+ UUID sessionID = registerSession();
+ InstrumentedLocalSessions sessions = new InstrumentedLocalSessions();
+ sessions.start();
+
+ // create session and move to finalized promised
+ sessions.prepareForTest(sessionID);
+ sessions.maybeSetRepairing(sessionID);
+ sessions.handleFinalizeProposeMessage(COORDINATOR, new FinalizePropose(sessionID));
+
+ Assert.assertEquals(0, (int) sessions.completedSessions.getOrDefault(sessionID, 0));
+ sessions.sentMessages.clear();
+ LocalSession session = sessions.getSession(sessionID);
+ sessions.handleFinalizeCommitMessage(PARTICIPANT1, new FinalizeCommit(sessionID));
+
+ Assert.assertEquals(FINALIZED, session.getState());
+ Assert.assertEquals(session, sessions.loadUnsafe(sessionID));
+ Assert.assertTrue(sessions.sentMessages.isEmpty());
+ Assert.assertEquals(1, (int) sessions.completedSessions.getOrDefault(sessionID, 0));
+ }
+
+ @Test
+ public void finalizeCommitNonExistantSession()
+ {
+ InstrumentedLocalSessions sessions = new InstrumentedLocalSessions();
+ sessions.start();
+ UUID fakeID = UUIDGen.getTimeUUID();
+ sessions.handleFinalizeCommitMessage(PARTICIPANT1, new FinalizeCommit(fakeID));
+ Assert.assertNull(sessions.getSession(fakeID));
+ Assert.assertTrue(sessions.sentMessages.isEmpty());
+ }
+
+ @Test
+ public void failSession()
+ {
+ UUID sessionID = registerSession();
+ InstrumentedLocalSessions sessions = new InstrumentedLocalSessions();
+ sessions.start();
+
+ LocalSession session = sessions.prepareForTest(sessionID);
+ Assert.assertEquals(PREPARED, session.getState());
+ sessions.sentMessages.clear();
+
+ // fail session
+ Assert.assertEquals(0, (int) sessions.completedSessions.getOrDefault(sessionID, 0));
+ sessions.failSession(sessionID);
+ Assert.assertEquals(FAILED, session.getState());
+ assertMessagesSent(sessions, COORDINATOR, new FailSession(sessionID));
+ Assert.assertEquals(1, (int) sessions.completedSessions.getOrDefault(sessionID, 0));
+ }
+
+ /**
+ * Session should be failed, but no messages should be sent
+ */
+ @Test
+ public void handleFailMessage()
+ {
+ UUID sessionID = registerSession();
+ InstrumentedLocalSessions sessions = new InstrumentedLocalSessions();
+ sessions.start();
+
+ LocalSession session = sessions.prepareForTest(sessionID);
+ Assert.assertEquals(PREPARED, session.getState());
+ sessions.sentMessages.clear();
+
+ sessions.handleFailSessionMessage(PARTICIPANT1, new FailSession(sessionID));
+ Assert.assertEquals(FAILED, session.getState());
+ Assert.assertTrue(sessions.sentMessages.isEmpty());
+ }
+
+ @Test
+ public void sendStatusRequest() throws Exception
+ {
+ UUID sessionID = registerSession();
+ InstrumentedLocalSessions sessions = new InstrumentedLocalSessions();
+ sessions.start();
+ LocalSession session = sessions.prepareForTest(sessionID);
+
+ sessions.sentMessages.clear();
+ sessions.sendStatusRequest(session);
+
+ assertNoMessagesSent(sessions, PARTICIPANT1);
+ StatusRequest expected = new StatusRequest(sessionID);
+ assertMessagesSent(sessions, PARTICIPANT2, expected);
+ assertMessagesSent(sessions, PARTICIPANT3, expected);
+ }
+
+ @Test
+ public void handleStatusRequest() throws Exception
+ {
+ UUID sessionID = registerSession();
+ InstrumentedLocalSessions sessions = new InstrumentedLocalSessions();
+ sessions.start();
+ LocalSession session = sessions.prepareForTest(sessionID);
+ Assert.assertEquals(PREPARED, session.getState());
+
+ sessions.sentMessages.clear();
+ sessions.handleStatusRequest(PARTICIPANT2, new StatusRequest(sessionID));
+ assertNoMessagesSent(sessions, PARTICIPANT1);
+ assertMessagesSent(sessions, PARTICIPANT2, new StatusResponse(sessionID, PREPARED));
+ assertNoMessagesSent(sessions, PARTICIPANT3);
+ }
+
+ @Test
+ public void handleStatusRequestNoSession() throws Exception
+ {
+ InstrumentedLocalSessions sessions = new InstrumentedLocalSessions();
+ sessions.start();
+
+ sessions.sentMessages.clear();
+ UUID sessionID = UUIDGen.getTimeUUID();
+ sessions.handleStatusRequest(PARTICIPANT2, new StatusRequest(sessionID));
+ assertNoMessagesSent(sessions, PARTICIPANT1);
+ assertMessagesSent(sessions, PARTICIPANT2, new StatusResponse(sessionID, FAILED));
+ assertNoMessagesSent(sessions, PARTICIPANT3);
+ }
+
+ @Test
+ public void handleStatusResponseFinalized() throws Exception
+ {
+ UUID sessionID = registerSession();
+ InstrumentedLocalSessions sessions = new InstrumentedLocalSessions();
+ sessions.start();
+ LocalSession session = sessions.prepareForTest(sessionID);
+ session.setState(FINALIZE_PROMISED);
+
+ sessions.handleStatusResponse(PARTICIPANT1, new StatusResponse(sessionID, FINALIZED));
+ Assert.assertEquals(FINALIZED, session.getState());
+ }
+
+ @Test
+ public void handleStatusResponseFinalizedRedundant() throws Exception
+ {
+ UUID sessionID = registerSession();
+ InstrumentedLocalSessions sessions = new InstrumentedLocalSessions();
+ sessions.start();
+ LocalSession session = sessions.prepareForTest(sessionID);
+ session.setState(FINALIZED);
+
+ sessions.handleStatusResponse(PARTICIPANT1, new StatusResponse(sessionID, FINALIZED));
+ Assert.assertEquals(FINALIZED, session.getState());
+ }
+
+ @Test
+ public void handleStatusResponseFailed() throws Exception
+ {
+ UUID sessionID = registerSession();
+ InstrumentedLocalSessions sessions = new InstrumentedLocalSessions();
+ sessions.start();
+ LocalSession session = sessions.prepareForTest(sessionID);
+ session.setState(FINALIZE_PROMISED);
+
+ sessions.handleStatusResponse(PARTICIPANT1, new StatusResponse(sessionID, FAILED));
+ Assert.assertEquals(FAILED, session.getState());
+ }
+
+ @Test
+ public void handleStatusResponseFailedRedundant() throws Exception
+ {
+ UUID sessionID = registerSession();
+ InstrumentedLocalSessions sessions = new InstrumentedLocalSessions();
+ sessions.start();
+ LocalSession session = sessions.prepareForTest(sessionID);
+ session.setState(FAILED);
+
+ sessions.handleStatusResponse(PARTICIPANT1, new StatusResponse(sessionID, FAILED));
+ Assert.assertEquals(FAILED, session.getState());
+ }
+
+ @Test
+ public void handleStatusResponseNoop() throws Exception
+ {
+ UUID sessionID = registerSession();
+ InstrumentedLocalSessions sessions = new InstrumentedLocalSessions();
+ sessions.start();
+ LocalSession session = sessions.prepareForTest(sessionID);
+ session.setState(REPAIRING);
+
+ sessions.handleStatusResponse(PARTICIPANT1, new StatusResponse(sessionID, FINALIZE_PROMISED));
+ Assert.assertEquals(REPAIRING, session.getState());
+ }
+
+ @Test
+ public void handleStatusResponseNoSession() throws Exception
+ {
+ UUID sessionID = UUIDGen.getTimeUUID();
+ InstrumentedLocalSessions sessions = new InstrumentedLocalSessions();
+ sessions.start();
+
+ sessions.handleStatusResponse(PARTICIPANT1, new StatusResponse(sessionID, FINALIZE_PROMISED));
+ Assert.assertNull(sessions.getSession(sessionID));
+ }
+
+ /**
+ * Check all states (except failed)
+ */
+ @Test
+ public void isSessionInProgress()
+ {
+ UUID sessionID = registerSession();
+ InstrumentedLocalSessions sessions = new InstrumentedLocalSessions();
+ sessions.start();
+ sessions.prepareSessionFuture = SettableFuture.create(); // prevent moving to prepared
+ sessions.handlePrepareMessage(PARTICIPANT1, new PrepareConsistentRequest(sessionID, COORDINATOR, PARTICIPANTS));
+
+ LocalSession session = sessions.getSession(sessionID);
+ Assert.assertNotNull(session);
+ Assert.assertEquals(PREPARING, session.getState());
+ Assert.assertTrue(sessions.isSessionInProgress(sessionID));
+
+ session.setState(PREPARED);
+ Assert.assertTrue(sessions.isSessionInProgress(sessionID));
+
+ session.setState(REPAIRING);
+ Assert.assertTrue(sessions.isSessionInProgress(sessionID));
+
+ session.setState(FINALIZE_PROMISED);
+ Assert.assertTrue(sessions.isSessionInProgress(sessionID));
+
+ session.setState(FINALIZED);
+ Assert.assertFalse(sessions.isSessionInProgress(sessionID));
+ }
+
+ @Test
+ public void isSessionInProgressFailed()
+ {
+ UUID sessionID = registerSession();
+ InstrumentedLocalSessions sessions = new InstrumentedLocalSessions();
+ sessions.start();
+ sessions.prepareSessionFuture = SettableFuture.create();
+ sessions.handlePrepareMessage(PARTICIPANT1, new PrepareConsistentRequest(sessionID, COORDINATOR, PARTICIPANTS));
+ sessions.prepareSessionFuture.set(new Object());
+
+ Assert.assertTrue(sessions.isSessionInProgress(sessionID));
+ sessions.failSession(sessionID);
+ Assert.assertFalse(sessions.isSessionInProgress(sessionID));
+ }
+
+ @Test
+ public void isSessionInProgressNonExistantSession()
+ {
+ UUID fakeID = UUIDGen.getTimeUUID();
+ InstrumentedLocalSessions sessions = new InstrumentedLocalSessions();
+ sessions.start();
+ Assert.assertFalse(sessions.isSessionInProgress(fakeID));
+ }
+
+ @Test
+ public void finalRepairedAtFinalized()
+ {
+ UUID sessionID = registerSession();
+ InstrumentedLocalSessions sessions = new InstrumentedLocalSessions();
+ sessions.start();
+
+ sessions.prepareForTest(sessionID);
+ sessions.maybeSetRepairing(sessionID);
+ sessions.handleFinalizeProposeMessage(COORDINATOR, new FinalizePropose(sessionID));
+ sessions.handleFinalizeCommitMessage(PARTICIPANT1, new FinalizeCommit(sessionID));
+
+ LocalSession session = sessions.getSession(sessionID);
+ Assert.assertTrue(session.repairedAt != ActiveRepairService.UNREPAIRED_SSTABLE);
+ Assert.assertEquals(session.repairedAt, sessions.getFinalSessionRepairedAt(sessionID));
+ }
+
+ @Test
+ public void finalRepairedAtFailed()
+ {
+ UUID sessionID = registerSession();
+ InstrumentedLocalSessions sessions = new InstrumentedLocalSessions();
+ sessions.start();
+
+ sessions.prepareForTest(sessionID);
+ sessions.failSession(sessionID);
+
+ LocalSession session = sessions.getSession(sessionID);
+ Assert.assertTrue(session.repairedAt != ActiveRepairService.UNREPAIRED_SSTABLE);
+ long repairedAt = sessions.getFinalSessionRepairedAt(sessionID);
+ Assert.assertEquals(ActiveRepairService.UNREPAIRED_SSTABLE, repairedAt);
+ }
+
+ @Test
+ public void finalRepairedAtNoSession()
+ {
+ UUID fakeID = registerSession();
+ InstrumentedLocalSessions sessions = new InstrumentedLocalSessions();
+ sessions.start();
+ long repairedAt = sessions.getFinalSessionRepairedAt(fakeID);
+ Assert.assertEquals(ActiveRepairService.UNREPAIRED_SSTABLE, repairedAt);
+ }
+
+ @Test(expected = IllegalStateException.class)
+ public void finalRepairedAtInProgress()
+ {
+ UUID sessionID = registerSession();
+ InstrumentedLocalSessions sessions = new InstrumentedLocalSessions();
+ sessions.start();
+ sessions.prepareForTest(sessionID);
+
+ sessions.getFinalSessionRepairedAt(sessionID);
+ }
+
+ /**
+ * Startup happy path
+ */
+ @Test
+ public void startup() throws Exception
+ {
+ InstrumentedLocalSessions initialSessions = new InstrumentedLocalSessions();
+ initialSessions.start();
+ Assert.assertEquals(0, initialSessions.getNumSessions());
+ UUID id1 = registerSession();
+ UUID id2 = registerSession();
+
+ initialSessions.prepareForTest(id1);
+ initialSessions.prepareForTest(id2);
+ Assert.assertEquals(2, initialSessions.getNumSessions());
+ LocalSession session1 = initialSessions.getSession(id1);
+ LocalSession session2 = initialSessions.getSession(id2);
+
+
+ // subsequent startups should load persisted sessions
+ InstrumentedLocalSessions nextSessions = new InstrumentedLocalSessions();
+ Assert.assertEquals(0, nextSessions.getNumSessions());
+ nextSessions.start();
+ Assert.assertEquals(2, nextSessions.getNumSessions());
+
+ Assert.assertEquals(session1, nextSessions.getSession(id1));
+ Assert.assertEquals(session2, nextSessions.getSession(id2));
+ }
+
+ /**
+ * If LocalSessions.start is called more than
+ * once, an exception should be thrown
+ */
+ @Test (expected = IllegalArgumentException.class)
+ public void multipleStartupFailure() throws Exception
+ {
+ InstrumentedLocalSessions initialSessions = new InstrumentedLocalSessions();
+ initialSessions.start();
+ initialSessions.start();
+ }
+
+ /**
+ * If there are problems with the rows we're reading out of the repair table, we should
+ * do the best we can to repair them, but not refuse to startup.
+ */
+ @Test
+ public void loadCorruptRow() throws Exception
+ {
+ LocalSessions sessions = new LocalSessions();
+ LocalSession session = createSession();
+ sessions.save(session);
+
+ sessions = new LocalSessions();
+ sessions.start();
+ Assert.assertNotNull(sessions.getSession(session.sessionID));
+
+ QueryProcessor.instance.executeInternal("DELETE participants, participants_wp FROM system.repairs WHERE parent_id=?", session.sessionID);
+
+ sessions = new LocalSessions();
+ sessions.start();
+ Assert.assertNull(sessions.getSession(session.sessionID));
+ }
+
+ private static LocalSession sessionWithTime(int started, int updated)
+ {
+ LocalSession.Builder builder = createBuilder();
+ builder.withStartedAt(started);
+ builder.withLastUpdate(updated);
+ return builder.build();
+ }
+
+ /**
+ * Sessions that shouldn't be failed or deleted are left alone
+ */
+ @Test
+ public void cleanupNoOp() throws Exception
+ {
+ LocalSessions sessions = new LocalSessions();
+ sessions.start();
+
+ int time = FBUtilities.nowInSeconds() - LocalSessions.AUTO_FAIL_TIMEOUT + 60;
+ LocalSession session = sessionWithTime(time - 1, time);
+
+ sessions.putSessionUnsafe(session);
+ Assert.assertNotNull(sessions.getSession(session.sessionID));
+
+ sessions.cleanup();
+
+ Assert.assertNotNull(sessions.getSession(session.sessionID));
+ }
+
+ /**
+ * Sessions past the auto fail cutoff should be failed
+ */
+ @Test
+ public void cleanupFail() throws Exception
+ {
+ LocalSessions sessions = new InstrumentedLocalSessions();
+ sessions.start();
+
+ int time = FBUtilities.nowInSeconds() - LocalSessions.AUTO_FAIL_TIMEOUT - 1;
+ LocalSession session = sessionWithTime(time - 1, time);
+ session.setState(REPAIRING);
+
+ sessions.putSessionUnsafe(session);
+ Assert.assertNotNull(sessions.getSession(session.sessionID));
+
+ sessions.cleanup();
+
+ Assert.assertNotNull(sessions.getSession(session.sessionID));
+ Assert.assertEquals(FAILED, session.getState());
+ Assert.assertEquals(session, sessions.loadUnsafe(session.sessionID));
+ }
+
+ /**
+ * Sessions past the auto delete cutoff with no sstables should be deleted
+ */
+ @Test
+ public void cleanupDeleteNoSSTables() throws Exception
+ {
+ LocalSessions sessions = new InstrumentedLocalSessions();
+ sessions.start();
+
+ int time = FBUtilities.nowInSeconds() - LocalSessions.AUTO_FAIL_TIMEOUT - 1;
+ LocalSession failed = sessionWithTime(time - 1, time);
+ failed.setState(FAILED);
+
+ LocalSession finalized = sessionWithTime(time - 1, time);
+ finalized.setState(FINALIZED);
+
+ sessions.putSessionUnsafe(failed);
+ sessions.putSessionUnsafe(finalized);
+ Assert.assertNotNull(sessions.getSession(failed.sessionID));
+ Assert.assertNotNull(sessions.getSession(finalized.sessionID));
+
+ sessions.cleanup();
+
+ Assert.assertNull(sessions.getSession(failed.sessionID));
+ Assert.assertNull(sessions.getSession(finalized.sessionID));
+
+ Assert.assertNull(sessions.loadUnsafe(failed.sessionID));
+ Assert.assertNull(sessions.loadUnsafe(finalized.sessionID));
+ }
+
+ /**
+ * Sessions past the auto delete cutoff with no sstables should be deleted
+ */
+ @Test
+ public void cleanupDeleteSSTablesRemaining() throws Exception
+ {
+ InstrumentedLocalSessions sessions = new InstrumentedLocalSessions();
+ sessions.start();
+
+ int time = FBUtilities.nowInSeconds() - LocalSessions.AUTO_FAIL_TIMEOUT - 1;
+ LocalSession failed = sessionWithTime(time - 1, time);
+ failed.setState(FAILED);
+
+ LocalSession finalized = sessionWithTime(time - 1, time);
+ finalized.setState(FINALIZED);
+
+ sessions.putSessionUnsafe(failed);
+ sessions.putSessionUnsafe(finalized);
+ Assert.assertNotNull(sessions.getSession(failed.sessionID));
+ Assert.assertNotNull(sessions.getSession(finalized.sessionID));
+
+ sessions.sessionHasData = true;
+ sessions.cleanup();
+
+ Assert.assertNotNull(sessions.getSession(failed.sessionID));
+ Assert.assertNotNull(sessions.getSession(finalized.sessionID));
+
+ Assert.assertNotNull(sessions.loadUnsafe(failed.sessionID));
+ Assert.assertNotNull(sessions.loadUnsafe(finalized.sessionID));
+ }
+
+ /**
+ * Sessions should start checking the status of their participants if
+ * there hasn't been activity for the CHECK_STATUS_TIMEOUT period
+ */
+ @Test
+ public void cleanupStatusRequest() throws Exception
+ {
+ AtomicReference<LocalSession> checkedSession = new AtomicReference<>();
+ InstrumentedLocalSessions sessions = new InstrumentedLocalSessions() {
+ public void sendStatusRequest(LocalSession session)
+ {
+ Assert.assertTrue(checkedSession.compareAndSet(null, session));
+ }
+ };
+ sessions.start();
+
+ int time = FBUtilities.nowInSeconds() - LocalSessions.CHECK_STATUS_TIMEOUT - 1;
+ LocalSession session = sessionWithTime(time - 1, time);
+ session.setState(REPAIRING);
+
+ sessions.putSessionUnsafe(session);
+ Assert.assertNotNull(sessions.getSession(session.sessionID));
+
+ sessions.cleanup();
+
+ Assert.assertEquals(session, checkedSession.get());
+ }
+}
+
diff --git a/test/unit/org/apache/cassandra/repair/messages/RepairMessageSerializationsTest.java b/test/unit/org/apache/cassandra/repair/messages/RepairMessageSerializationsTest.java
index f2dc8c7..fa037a0 100644
--- a/test/unit/org/apache/cassandra/repair/messages/RepairMessageSerializationsTest.java
+++ b/test/unit/org/apache/cassandra/repair/messages/RepairMessageSerializationsTest.java
@@ -19,12 +19,12 @@
package org.apache.cassandra.repair.messages;
import java.io.IOException;
-import java.net.InetAddress;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
+import com.google.common.collect.Lists;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
@@ -41,24 +41,29 @@
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputBufferFixed;
import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.net.MessagingService;
-import org.apache.cassandra.repair.NodePair;
+import org.apache.cassandra.repair.SyncNodePair;
+import org.apache.cassandra.streaming.PreviewKind;
import org.apache.cassandra.repair.RepairJobDesc;
+import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.streaming.SessionSummary;
+import org.apache.cassandra.streaming.StreamSummary;
import org.apache.cassandra.utils.MerkleTrees;
+import org.apache.cassandra.utils.UUIDGen;
public class RepairMessageSerializationsTest
{
+ private static final int PROTOCOL_VERSION = MessagingService.current_version;
private static final int GC_BEFORE = 1000000;
- private static int protocolVersion;
private static IPartitioner originalPartitioner;
@BeforeClass
public static void before()
{
DatabaseDescriptor.daemonInitialization();
- protocolVersion = MessagingService.current_version;
originalPartitioner = StorageService.instance.setPartitionerUnsafe(Murmur3Partitioner.instance);
}
@@ -95,16 +100,16 @@
private <T extends RepairMessage> T serializeRoundTrip(T msg, IVersionedSerializer<T> serializer) throws IOException
{
- long size = serializer.serializedSize(msg, protocolVersion);
+ long size = serializer.serializedSize(msg, PROTOCOL_VERSION);
ByteBuffer buf = ByteBuffer.allocate((int)size);
DataOutputPlus out = new DataOutputBufferFixed(buf);
- serializer.serialize(msg, out, protocolVersion);
+ serializer.serialize(msg, out, PROTOCOL_VERSION);
Assert.assertEquals(size, buf.position());
buf.flip();
DataInputPlus in = new DataInputBuffer(buf, false);
- T deserialized = serializer.deserialize(in, protocolVersion);
+ T deserialized = serializer.deserialize(in, PROTOCOL_VERSION);
Assert.assertEquals(msg, deserialized);
Assert.assertEquals(msg.hashCode(), deserialized.hashCode());
return deserialized;
@@ -113,7 +118,7 @@
@Test
public void validationCompleteMessage_NoMerkleTree() throws IOException
{
- ValidationComplete deserialized = validationCompleteMessage(null);
+ ValidationResponse deserialized = validationCompleteMessage(null);
Assert.assertNull(deserialized.trees);
}
@@ -122,54 +127,53 @@
{
MerkleTrees trees = new MerkleTrees(Murmur3Partitioner.instance);
trees.addMerkleTree(256, new Range<>(new LongToken(1000), new LongToken(1001)));
- ValidationComplete deserialized = validationCompleteMessage(trees);
+ ValidationResponse deserialized = validationCompleteMessage(trees);
// a simple check to make sure we got some merkle trees back.
Assert.assertEquals(trees.size(), deserialized.trees.size());
}
- private ValidationComplete validationCompleteMessage(MerkleTrees trees) throws IOException
+ private ValidationResponse validationCompleteMessage(MerkleTrees trees) throws IOException
{
RepairJobDesc jobDesc = buildRepairJobDesc();
- ValidationComplete msg = trees == null ?
- new ValidationComplete(jobDesc) :
- new ValidationComplete(jobDesc, trees);
- ValidationComplete deserialized = serializeRoundTrip(msg, ValidationComplete.serializer);
+ ValidationResponse msg = trees == null ?
+ new ValidationResponse(jobDesc) :
+ new ValidationResponse(jobDesc, trees);
+ ValidationResponse deserialized = serializeRoundTrip(msg, ValidationResponse.serializer);
return deserialized;
}
@Test
public void syncRequestMessage() throws IOException
{
- InetAddress initiator = InetAddress.getByName("127.0.0.1");
- InetAddress src = InetAddress.getByName("127.0.0.2");
- InetAddress dst = InetAddress.getByName("127.0.0.3");
+ InetAddressAndPort initiator = InetAddressAndPort.getByName("127.0.0.1");
+ InetAddressAndPort src = InetAddressAndPort.getByName("127.0.0.2");
+ InetAddressAndPort dst = InetAddressAndPort.getByName("127.0.0.3");
- SyncRequest msg = new SyncRequest(buildRepairJobDesc(), initiator, src, dst, buildTokenRanges());
+ SyncRequest msg = new SyncRequest(buildRepairJobDesc(), initiator, src, dst, buildTokenRanges(), PreviewKind.NONE);
serializeRoundTrip(msg, SyncRequest.serializer);
}
@Test
public void syncCompleteMessage() throws IOException
{
- InetAddress src = InetAddress.getByName("127.0.0.2");
- InetAddress dst = InetAddress.getByName("127.0.0.3");
- SyncComplete msg = new SyncComplete(buildRepairJobDesc(), new NodePair(src, dst), true);
- serializeRoundTrip(msg, SyncComplete.serializer);
- }
-
- @Test
- public void antiCompactionRequestMessage() throws IOException
- {
- AnticompactionRequest msg = new AnticompactionRequest(UUID.randomUUID(), buildTokenRanges());
- serializeRoundTrip(msg, AnticompactionRequest.serializer);
+ InetAddressAndPort src = InetAddressAndPort.getByName("127.0.0.2");
+ InetAddressAndPort dst = InetAddressAndPort.getByName("127.0.0.3");
+ List<SessionSummary> summaries = new ArrayList<>();
+ summaries.add(new SessionSummary(src, dst,
+ Lists.newArrayList(new StreamSummary(TableId.fromUUID(UUIDGen.getTimeUUID()), 5, 100)),
+ Lists.newArrayList(new StreamSummary(TableId.fromUUID(UUIDGen.getTimeUUID()), 500, 10))
+ ));
+ SyncResponse msg = new SyncResponse(buildRepairJobDesc(), new SyncNodePair(src, dst), true, summaries);
+ serializeRoundTrip(msg, SyncResponse.serializer);
}
@Test
public void prepareMessage() throws IOException
{
- PrepareMessage msg = new PrepareMessage(UUID.randomUUID(), new ArrayList<UUID>() {{add(UUID.randomUUID());}},
- buildTokenRanges(), true, 100000L, false);
+ PrepareMessage msg = new PrepareMessage(UUID.randomUUID(), new ArrayList<TableId>() {{add(TableId.generate());}},
+ buildTokenRanges(), true, 100000L, false,
+ PreviewKind.NONE);
serializeRoundTrip(msg, PrepareMessage.serializer);
}
diff --git a/test/unit/org/apache/cassandra/repair/messages/RepairMessageSerializerTest.java b/test/unit/org/apache/cassandra/repair/messages/RepairMessageSerializerTest.java
new file mode 100644
index 0000000..fedf498
--- /dev/null
+++ b/test/unit/org/apache/cassandra/repair/messages/RepairMessageSerializerTest.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.repair.messages;
+
+import java.io.IOException;
+
+import com.google.common.collect.Sets;
+import org.junit.Assert;
+import org.junit.Test;
+
+import org.apache.cassandra.io.IVersionedSerializer;
+import org.apache.cassandra.io.util.DataInputBuffer;
+import org.apache.cassandra.io.util.DataOutputBuffer;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.utils.UUIDGen;
+
+/**
+ * verifies repair message serializers are working as advertised
+ */
+public class RepairMessageSerializerTest
+{
+ private static int MS_VERSION = MessagingService.current_version;
+
+ private static <T extends RepairMessage> T serdes(IVersionedSerializer<T> serializer, T message)
+ {
+ int expectedSize = (int) serializer.serializedSize(message, MS_VERSION);
+ try (DataOutputBuffer out = new DataOutputBuffer(expectedSize))
+ {
+ serializer.serialize(message, out, MS_VERSION);
+ Assert.assertEquals(expectedSize, out.buffer().limit());
+ try (DataInputBuffer in = new DataInputBuffer(out.buffer(), false))
+ {
+ return serializer.deserialize(in, MS_VERSION);
+ }
+ }
+ catch (IOException e)
+ {
+ throw new AssertionError(e);
+ }
+ }
+
+ @Test
+ public void prepareConsistentRequest() throws Exception
+ {
+ InetAddressAndPort coordinator = InetAddressAndPort.getByName("10.0.0.1");
+ InetAddressAndPort peer1 = InetAddressAndPort.getByName("10.0.0.2");
+ InetAddressAndPort peer2 = InetAddressAndPort.getByName("10.0.0.3");
+ InetAddressAndPort peer3 = InetAddressAndPort.getByName("10.0.0.4");
+ PrepareConsistentRequest expected =
+ new PrepareConsistentRequest(UUIDGen.getTimeUUID(), coordinator, Sets.newHashSet(peer1, peer2, peer3));
+ PrepareConsistentRequest actual = serdes(PrepareConsistentRequest.serializer, expected);
+ Assert.assertEquals(expected, actual);
+ }
+
+ @Test
+ public void prepareConsistentResponse() throws Exception
+ {
+ PrepareConsistentResponse expected =
+ new PrepareConsistentResponse(UUIDGen.getTimeUUID(), InetAddressAndPort.getByName("10.0.0.2"), true);
+ PrepareConsistentResponse actual = serdes(PrepareConsistentResponse.serializer, expected);
+ Assert.assertEquals(expected, actual);
+ }
+
+ @Test
+ public void failSession() throws Exception
+ {
+ FailSession expected = new FailSession(UUIDGen.getTimeUUID());
+ FailSession actual = serdes(FailSession.serializer, expected);
+ Assert.assertEquals(expected, actual);;
+ }
+
+ @Test
+ public void finalizeCommit() throws Exception
+ {
+ FinalizeCommit expected = new FinalizeCommit(UUIDGen.getTimeUUID());
+ FinalizeCommit actual = serdes(FinalizeCommit.serializer, expected);
+ Assert.assertEquals(expected, actual);;
+ }
+
+ @Test
+ public void finalizePromise() throws Exception
+ {
+ FinalizePromise expected = new FinalizePromise(UUIDGen.getTimeUUID(), InetAddressAndPort.getByName("10.0.0.2"), true);
+ FinalizePromise actual = serdes(FinalizePromise.serializer, expected);
+ Assert.assertEquals(expected, actual);
+ }
+
+ @Test
+ public void finalizePropose() throws Exception
+ {
+ FinalizePropose expected = new FinalizePropose(UUIDGen.getTimeUUID());
+ FinalizePropose actual = serdes(FinalizePropose.serializer, expected);
+ Assert.assertEquals(expected, actual);;
+ }
+}
diff --git a/test/unit/org/apache/cassandra/repair/messages/RepairOptionTest.java b/test/unit/org/apache/cassandra/repair/messages/RepairOptionTest.java
index 9fe8b93..484d7a8 100644
--- a/test/unit/org/apache/cassandra/repair/messages/RepairOptionTest.java
+++ b/test/unit/org/apache/cassandra/repair/messages/RepairOptionTest.java
@@ -23,6 +23,7 @@
import java.util.Map;
import java.util.Set;
+import org.junit.Assert;
import org.junit.Test;
import com.google.common.collect.ImmutableMap;
@@ -149,14 +150,24 @@
}
@Test
- public void testIncrementalRepairWithSubrangesIsNotGlobal() throws Exception
+ public void testForceOption() throws Exception
{
- RepairOption ro = RepairOption.parse(ImmutableMap.of(RepairOption.INCREMENTAL_KEY, "true", RepairOption.RANGES_KEY, "41:42"),
- Murmur3Partitioner.instance);
- assertFalse(ro.isGlobal());
- ro = RepairOption.parse(ImmutableMap.of(RepairOption.INCREMENTAL_KEY, "true", RepairOption.RANGES_KEY, ""),
- Murmur3Partitioner.instance);
- assertTrue(ro.isGlobal());
+ RepairOption option;
+ Map<String, String> options = new HashMap<>();
+
+ // default value
+ option = RepairOption.parse(options, Murmur3Partitioner.instance);
+ Assert.assertFalse(option.isForcedRepair());
+
+ // explicit true
+ options.put(RepairOption.FORCE_REPAIR_KEY, "true");
+ option = RepairOption.parse(options, Murmur3Partitioner.instance);
+ Assert.assertTrue(option.isForcedRepair());
+
+ // explicit false
+ options.put(RepairOption.FORCE_REPAIR_KEY, "false");
+ option = RepairOption.parse(options, Murmur3Partitioner.instance);
+ Assert.assertFalse(option.isForcedRepair());
}
private void assertParseThrowsIllegalArgumentExceptionWithMessage(Map<String, String> optionsToParse, String expectedErrorMessage)
diff --git a/test/unit/org/apache/cassandra/schema/DefsTest.java b/test/unit/org/apache/cassandra/schema/DefsTest.java
deleted file mode 100644
index d4ac1dc..0000000
--- a/test/unit/org/apache/cassandra/schema/DefsTest.java
+++ /dev/null
@@ -1,545 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.schema;
-
-import java.io.File;
-import java.nio.ByteBuffer;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.function.Supplier;
-
-import com.google.common.collect.ImmutableMap;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-
-import org.apache.cassandra.OrderedJUnit4ClassRunner;
-import org.apache.cassandra.SchemaLoader;
-import org.apache.cassandra.Util;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
-import org.apache.cassandra.config.Schema;
-import org.apache.cassandra.cql3.QueryProcessor;
-import org.apache.cassandra.cql3.UntypedResultSet;
-import org.apache.cassandra.db.ColumnFamilyStore;
-import org.apache.cassandra.db.Directories;
-import org.apache.cassandra.db.Keyspace;
-import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
-import org.apache.cassandra.db.marshal.BytesType;
-import org.apache.cassandra.db.marshal.UTF8Type;
-import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.io.sstable.Component;
-import org.apache.cassandra.io.sstable.Descriptor;
-import org.apache.cassandra.locator.OldNetworkTopologyStrategy;
-import org.apache.cassandra.service.MigrationManager;
-import org.apache.cassandra.utils.FBUtilities;
-
-import static org.apache.cassandra.Util.throwAssert;
-import static org.apache.cassandra.cql3.CQLTester.assertRows;
-import static org.apache.cassandra.cql3.CQLTester.row;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-
-
-@RunWith(OrderedJUnit4ClassRunner.class)
-public class DefsTest
-{
- private static final String KEYSPACE1 = "keyspace1";
- private static final String KEYSPACE3 = "keyspace3";
- private static final String KEYSPACE6 = "keyspace6";
- private static final String EMPTY_KEYSPACE = "test_empty_keyspace";
- private static final String TABLE1 = "standard1";
- private static final String TABLE2 = "standard2";
- private static final String TABLE1i = "indexed1";
-
- @BeforeClass
- public static void defineSchema() throws ConfigurationException
- {
- SchemaLoader.prepareServer();
- SchemaLoader.startGossiper();
- SchemaLoader.createKeyspace(KEYSPACE1,
- KeyspaceParams.simple(1),
- SchemaLoader.standardCFMD(KEYSPACE1, TABLE1),
- SchemaLoader.standardCFMD(KEYSPACE1, TABLE2));
- SchemaLoader.createKeyspace(KEYSPACE3,
- KeyspaceParams.simple(5),
- SchemaLoader.standardCFMD(KEYSPACE1, TABLE1),
- SchemaLoader.compositeIndexCFMD(KEYSPACE3, TABLE1i, true));
- SchemaLoader.createKeyspace(KEYSPACE6,
- KeyspaceParams.simple(1),
- SchemaLoader.compositeIndexCFMD(KEYSPACE6, TABLE1i, true));
- }
-
- @Test
- public void testCFMetaDataApply() throws ConfigurationException
- {
- CFMetaData cfm = CFMetaData.Builder.create(KEYSPACE1, "TestApplyCFM_CF")
- .addPartitionKey("keys", BytesType.instance)
- .addClusteringColumn("col", BytesType.instance).build();
-
-
- for (int i = 0; i < 5; i++)
- {
- ByteBuffer name = ByteBuffer.wrap(new byte[] { (byte)i });
- cfm.addColumnDefinition(ColumnDefinition.regularDef(cfm, name, BytesType.instance));
- }
-
- cfm.comment("No comment")
- .readRepairChance(0.5)
- .gcGraceSeconds(100000)
- .compaction(CompactionParams.scts(ImmutableMap.of("min_threshold", "500",
- "max_threshold", "500")));
-
- // we'll be adding this one later. make sure it's not already there.
- assertNull(cfm.getColumnDefinition(ByteBuffer.wrap(new byte[]{ 5 })));
-
- CFMetaData cfNew = cfm.copy();
-
- // add one.
- ColumnDefinition addIndexDef = ColumnDefinition.regularDef(cfm, ByteBuffer.wrap(new byte[] { 5 }), BytesType.instance);
- cfNew.addColumnDefinition(addIndexDef);
-
- // remove one.
- ColumnDefinition removeIndexDef = ColumnDefinition.regularDef(cfm, ByteBuffer.wrap(new byte[] { 0 }), BytesType.instance);
- assertTrue(cfNew.removeColumnDefinition(removeIndexDef));
-
- cfm.apply(cfNew);
-
- for (int i = 1; i < cfm.allColumns().size(); i++)
- assertNotNull(cfm.getColumnDefinition(ByteBuffer.wrap(new byte[]{ 1 })));
- assertNull(cfm.getColumnDefinition(ByteBuffer.wrap(new byte[]{ 0 })));
- assertNotNull(cfm.getColumnDefinition(ByteBuffer.wrap(new byte[]{ 5 })));
- }
-
- @Test
- public void testInvalidNames()
- {
- String[] valid = {"1", "a", "_1", "b_", "__", "1_a"};
- for (String s : valid)
- assertTrue(CFMetaData.isNameValid(s));
-
- String[] invalid = {"b@t", "dash-y", "", " ", "dot.s", ".hidden"};
- for (String s : invalid)
- assertFalse(CFMetaData.isNameValid(s));
- }
-
- @Test
- public void addNewCfToBogusKeyspace()
- {
- CFMetaData newCf = addTestTable("MadeUpKeyspace", "NewCF", "new cf");
- try
- {
- MigrationManager.announceNewColumnFamily(newCf);
- throw new AssertionError("You shouldn't be able to do anything to a keyspace that doesn't exist.");
- }
- catch (ConfigurationException expected)
- {
- }
- }
-
- @Test
- public void addNewTable() throws ConfigurationException
- {
- final String ksName = KEYSPACE1;
- final String tableName = "anewtable";
- KeyspaceMetadata original = Schema.instance.getKSMetaData(ksName);
-
- CFMetaData cfm = addTestTable(original.name, tableName, "A New Table");
-
- assertFalse(Schema.instance.getKSMetaData(ksName).tables.get(cfm.cfName).isPresent());
- MigrationManager.announceNewColumnFamily(cfm);
-
- assertTrue(Schema.instance.getKSMetaData(ksName).tables.get(cfm.cfName).isPresent());
- assertEquals(cfm, Schema.instance.getKSMetaData(ksName).tables.get(cfm.cfName).get());
-
- // now read and write to it.
- QueryProcessor.executeInternal(String.format("INSERT INTO %s.%s (key, col, val) VALUES (?, ?, ?)",
- ksName, tableName),
- "key0", "col0", "val0");
-
- // flush to exercise more than just hitting the memtable
- ColumnFamilyStore cfs = Keyspace.open(ksName).getColumnFamilyStore(tableName);
- assertNotNull(cfs);
- cfs.forceBlockingFlush();
-
- // and make sure we get out what we put in
- UntypedResultSet rows = QueryProcessor.executeInternal(String.format("SELECT * FROM %s.%s", ksName, tableName));
- assertRows(rows, row("key0", "col0", "val0"));
- }
-
- @Test
- public void dropCf() throws ConfigurationException
- {
- // sanity
- final KeyspaceMetadata ks = Schema.instance.getKSMetaData(KEYSPACE1);
- assertNotNull(ks);
- final CFMetaData cfm = ks.tables.getNullable(TABLE1);
- assertNotNull(cfm);
-
- // write some data, force a flush, then verify that files exist on disk.
- for (int i = 0; i < 100; i++)
- QueryProcessor.executeInternal(String.format("INSERT INTO %s.%s (key, name, val) VALUES (?, ?, ?)",
- KEYSPACE1, TABLE1),
- "dropCf", "col" + i, "anyvalue");
- ColumnFamilyStore store = Keyspace.open(cfm.ksName).getColumnFamilyStore(cfm.cfName);
- assertNotNull(store);
- store.forceBlockingFlush();
- assertTrue(store.getDirectories().sstableLister(Directories.OnTxnErr.THROW).list().size() > 0);
-
- MigrationManager.announceColumnFamilyDrop(ks.name, cfm.cfName);
-
- assertFalse(Schema.instance.getKSMetaData(ks.name).tables.get(cfm.cfName).isPresent());
-
- // any write should fail.
- boolean success = true;
- try
- {
- QueryProcessor.executeInternal(String.format("INSERT INTO %s.%s (key, name, val) VALUES (?, ?, ?)",
- KEYSPACE1, TABLE1),
- "dropCf", "col0", "anyvalue");
- }
- catch (Throwable th)
- {
- success = false;
- }
- assertFalse("This mutation should have failed since the CF no longer exists.", success);
-
- // verify that the files are gone.
- Supplier<Object> lambda = () -> {
- for (File file : store.getDirectories().sstableLister(Directories.OnTxnErr.THROW).listFiles())
- {
- if (file.getPath().endsWith("Data.db") && !new File(file.getPath().replace("Data.db", "Compacted")).exists())
- return false;
- }
- return true;
- };
- Util.spinAssertEquals(true, lambda, 30);
-
- }
-
- @Test
- public void addNewKS() throws ConfigurationException
- {
- CFMetaData cfm = addTestTable("newkeyspace1", "newstandard1", "A new cf for a new ks");
- KeyspaceMetadata newKs = KeyspaceMetadata.create(cfm.ksName, KeyspaceParams.simple(5), Tables.of(cfm));
- MigrationManager.announceNewKeyspace(newKs);
-
- assertNotNull(Schema.instance.getKSMetaData(cfm.ksName));
- assertEquals(Schema.instance.getKSMetaData(cfm.ksName), newKs);
-
- // test reads and writes.
- QueryProcessor.executeInternal("INSERT INTO newkeyspace1.newstandard1 (key, col, val) VALUES (?, ?, ?)",
- "key0", "col0", "val0");
- ColumnFamilyStore store = Keyspace.open(cfm.ksName).getColumnFamilyStore(cfm.cfName);
- assertNotNull(store);
- store.forceBlockingFlush();
-
- UntypedResultSet rows = QueryProcessor.executeInternal("SELECT * FROM newkeyspace1.newstandard1");
- assertRows(rows, row("key0", "col0", "val0"));
- }
-
- @Test
- public void dropKS() throws ConfigurationException
- {
- // sanity
- final KeyspaceMetadata ks = Schema.instance.getKSMetaData(KEYSPACE1);
- assertNotNull(ks);
- final CFMetaData cfm = ks.tables.getNullable(TABLE2);
- assertNotNull(cfm);
-
- // write some data, force a flush, then verify that files exist on disk.
- for (int i = 0; i < 100; i++)
- QueryProcessor.executeInternal(String.format("INSERT INTO %s.%s (key, name, val) VALUES (?, ?, ?)",
- KEYSPACE1, TABLE2),
- "dropKs", "col" + i, "anyvalue");
- ColumnFamilyStore cfs = Keyspace.open(cfm.ksName).getColumnFamilyStore(cfm.cfName);
- assertNotNull(cfs);
- cfs.forceBlockingFlush();
- assertTrue(!cfs.getDirectories().sstableLister(Directories.OnTxnErr.THROW).list().isEmpty());
-
- MigrationManager.announceKeyspaceDrop(ks.name);
-
- assertNull(Schema.instance.getKSMetaData(ks.name));
-
- // write should fail.
- boolean success = true;
- try
- {
- QueryProcessor.executeInternal(String.format("INSERT INTO %s.%s (key, name, val) VALUES (?, ?, ?)",
- KEYSPACE1, TABLE2),
- "dropKs", "col0", "anyvalue");
- }
- catch (Throwable th)
- {
- success = false;
- }
- assertFalse("This mutation should have failed since the KS no longer exists.", success);
-
- // reads should fail too.
- boolean threw = false;
- try
- {
- Keyspace.open(ks.name);
- }
- catch (Throwable th)
- {
- threw = true;
- }
- assertTrue(threw);
- }
-
- @Test
- public void dropKSUnflushed() throws ConfigurationException
- {
- // sanity
- final KeyspaceMetadata ks = Schema.instance.getKSMetaData(KEYSPACE3);
- assertNotNull(ks);
- final CFMetaData cfm = ks.tables.getNullable(TABLE1);
- assertNotNull(cfm);
-
- // write some data
- for (int i = 0; i < 100; i++)
- QueryProcessor.executeInternal(String.format("INSERT INTO %s.%s (key, name, val) VALUES (?, ?, ?)",
- KEYSPACE3, TABLE1),
- "dropKs", "col" + i, "anyvalue");
-
- MigrationManager.announceKeyspaceDrop(ks.name);
-
- assertNull(Schema.instance.getKSMetaData(ks.name));
- }
-
- @Test
- public void createEmptyKsAddNewCf() throws ConfigurationException
- {
- assertNull(Schema.instance.getKSMetaData(EMPTY_KEYSPACE));
- KeyspaceMetadata newKs = KeyspaceMetadata.create(EMPTY_KEYSPACE, KeyspaceParams.simple(5));
- MigrationManager.announceNewKeyspace(newKs);
- assertNotNull(Schema.instance.getKSMetaData(EMPTY_KEYSPACE));
-
- String tableName = "added_later";
- CFMetaData newCf = addTestTable(EMPTY_KEYSPACE, tableName, "A new CF to add to an empty KS");
-
- //should not exist until apply
- assertFalse(Schema.instance.getKSMetaData(newKs.name).tables.get(newCf.cfName).isPresent());
-
- //add the new CF to the empty space
- MigrationManager.announceNewColumnFamily(newCf);
-
- assertTrue(Schema.instance.getKSMetaData(newKs.name).tables.get(newCf.cfName).isPresent());
- assertEquals(Schema.instance.getKSMetaData(newKs.name).tables.get(newCf.cfName).get(), newCf);
-
- // now read and write to it.
- QueryProcessor.executeInternal(String.format("INSERT INTO %s.%s (key, col, val) VALUES (?, ?, ?)",
- EMPTY_KEYSPACE, tableName),
- "key0", "col0", "val0");
-
- ColumnFamilyStore cfs = Keyspace.open(newKs.name).getColumnFamilyStore(newCf.cfName);
- assertNotNull(cfs);
- cfs.forceBlockingFlush();
-
- UntypedResultSet rows = QueryProcessor.executeInternal(String.format("SELECT * FROM %s.%s", EMPTY_KEYSPACE, tableName));
- assertRows(rows, row("key0", "col0", "val0"));
- }
-
- @Test
- public void testUpdateKeyspace() throws ConfigurationException
- {
- // create a keyspace to serve as existing.
- CFMetaData cf = addTestTable("UpdatedKeyspace", "AddedStandard1", "A new cf for a new ks");
- KeyspaceMetadata oldKs = KeyspaceMetadata.create(cf.ksName, KeyspaceParams.simple(5), Tables.of(cf));
-
- MigrationManager.announceNewKeyspace(oldKs);
-
- assertNotNull(Schema.instance.getKSMetaData(cf.ksName));
- assertEquals(Schema.instance.getKSMetaData(cf.ksName), oldKs);
-
- // names should match.
- KeyspaceMetadata newBadKs2 = KeyspaceMetadata.create(cf.ksName + "trash", KeyspaceParams.simple(4));
- try
- {
- MigrationManager.announceKeyspaceUpdate(newBadKs2);
- throw new AssertionError("Should not have been able to update a KS with an invalid KS name.");
- }
- catch (ConfigurationException ex)
- {
- // expected.
- }
-
- Map<String, String> replicationMap = new HashMap<>();
- replicationMap.put(ReplicationParams.CLASS, OldNetworkTopologyStrategy.class.getName());
- replicationMap.put("replication_factor", "1");
-
- KeyspaceMetadata newKs = KeyspaceMetadata.create(cf.ksName, KeyspaceParams.create(true, replicationMap));
- MigrationManager.announceKeyspaceUpdate(newKs);
-
- KeyspaceMetadata newFetchedKs = Schema.instance.getKSMetaData(newKs.name);
- assertEquals(newFetchedKs.params.replication.klass, newKs.params.replication.klass);
- assertFalse(newFetchedKs.params.replication.klass.equals(oldKs.params.replication.klass));
- }
-
- /*
- @Test
- public void testUpdateColumnFamilyNoIndexes() throws ConfigurationException
- {
- // create a keyspace with a cf to update.
- CFMetaData cf = addTestTable("UpdatedCfKs", "Standard1added", "A new cf that will be updated");
- KSMetaData ksm = KSMetaData.testMetadata(cf.ksName, SimpleStrategy.class, KSMetaData.optsWithRF(1), cf);
- MigrationManager.announceNewKeyspace(ksm);
-
- assertNotNull(Schema.instance.getKSMetaData(cf.ksName));
- assertEquals(Schema.instance.getKSMetaData(cf.ksName), ksm);
- assertNotNull(Schema.instance.getCFMetaData(cf.ksName, cf.cfName));
-
- // updating certain fields should fail.
- CFMetaData newCfm = cf.copy();
- newCfm.defaultValidator(BytesType.instance);
- newCfm.minCompactionThreshold(5);
- newCfm.maxCompactionThreshold(31);
-
- // test valid operations.
- newCfm.comment("Modified comment");
- MigrationManager.announceColumnFamilyUpdate(newCfm); // doesn't get set back here.
-
- newCfm.readRepairChance(0.23);
- MigrationManager.announceColumnFamilyUpdate(newCfm);
-
- newCfm.gcGraceSeconds(12);
- MigrationManager.announceColumnFamilyUpdate(newCfm);
-
- newCfm.defaultValidator(UTF8Type.instance);
- MigrationManager.announceColumnFamilyUpdate(newCfm);
-
- newCfm.minCompactionThreshold(3);
- MigrationManager.announceColumnFamilyUpdate(newCfm);
-
- newCfm.maxCompactionThreshold(33);
- MigrationManager.announceColumnFamilyUpdate(newCfm);
-
- // can't test changing the reconciler because there is only one impl.
-
- // check the cumulative affect.
- assertEquals(Schema.instance.getCFMetaData(cf.ksName, cf.cfName).getComment(), newCfm.getComment());
- assertEquals(Schema.instance.getCFMetaData(cf.ksName, cf.cfName).getReadRepairChance(), newCfm.getReadRepairChance(), 0.0001);
- assertEquals(Schema.instance.getCFMetaData(cf.ksName, cf.cfName).getGcGraceSeconds(), newCfm.getGcGraceSeconds());
- assertEquals(UTF8Type.instance, Schema.instance.getCFMetaData(cf.ksName, cf.cfName).getDefaultValidator());
-
- // Change cfId
- newCfm = new CFMetaData(cf.ksName, cf.cfName, cf.cfType, cf.comparator);
- CFMetaData.copyOpts(newCfm, cf);
- try
- {
- cf.apply(newCfm);
- throw new AssertionError("Should have blown up when you used a different id.");
- }
- catch (ConfigurationException expected) {}
-
- // Change cfName
- newCfm = new CFMetaData(cf.ksName, cf.cfName + "_renamed", cf.cfType, cf.comparator);
- CFMetaData.copyOpts(newCfm, cf);
- try
- {
- cf.apply(newCfm);
- throw new AssertionError("Should have blown up when you used a different name.");
- }
- catch (ConfigurationException expected) {}
-
- // Change ksName
- newCfm = new CFMetaData(cf.ksName + "_renamed", cf.cfName, cf.cfType, cf.comparator);
- CFMetaData.copyOpts(newCfm, cf);
- try
- {
- cf.apply(newCfm);
- throw new AssertionError("Should have blown up when you used a different keyspace.");
- }
- catch (ConfigurationException expected) {}
-
- // Change cf type
- newCfm = new CFMetaData(cf.ksName, cf.cfName, ColumnFamilyType.Super, cf.comparator);
- CFMetaData.copyOpts(newCfm, cf);
- try
- {
- cf.apply(newCfm);
- throw new AssertionError("Should have blwon up when you used a different cf type.");
- }
- catch (ConfigurationException expected) {}
-
- // Change comparator
- newCfm = new CFMetaData(cf.ksName, cf.cfName, cf.cfType, new SimpleDenseCellNameType(TimeUUIDType.instance));
- CFMetaData.copyOpts(newCfm, cf);
- try
- {
- cf.apply(newCfm);
- throw new AssertionError("Should have blown up when you used a different comparator.");
- }
- catch (ConfigurationException expected) {}
- }
- */
-
- @Test
- public void testDropIndex() throws ConfigurationException
- {
- // persist keyspace definition in the system keyspace
- SchemaKeyspace.makeCreateKeyspaceMutation(Schema.instance.getKSMetaData(KEYSPACE6), FBUtilities.timestampMicros()).build().applyUnsafe();
- ColumnFamilyStore cfs = Keyspace.open(KEYSPACE6).getColumnFamilyStore(TABLE1i);
- String indexName = "birthdate_key_index";
-
- // insert some data. save the sstable descriptor so we can make sure it's marked for delete after the drop
- QueryProcessor.executeInternal(String.format(
- "INSERT INTO %s.%s (key, c1, birthdate, notbirthdate) VALUES (?, ?, ?, ?)",
- KEYSPACE6,
- TABLE1i),
- "key0", "col0", 1L, 1L);
-
- cfs.forceBlockingFlush();
- ColumnFamilyStore indexCfs = cfs.indexManager.getIndexByName(indexName)
- .getBackingTable()
- .orElseThrow(throwAssert("Cannot access index cfs"));
- Descriptor desc = indexCfs.getLiveSSTables().iterator().next().descriptor;
-
- // drop the index
- CFMetaData meta = cfs.metadata.copy();
- IndexMetadata existing = cfs.metadata.getIndexes()
- .get(indexName)
- .orElseThrow(throwAssert("Index not found"));
-
- meta.indexes(meta.getIndexes().without(existing.name));
- MigrationManager.announceColumnFamilyUpdate(meta);
-
- // check
- assertTrue(cfs.indexManager.listIndexes().isEmpty());
- LifecycleTransaction.waitForDeletions();
- assertFalse(new File(desc.filenameFor(Component.DATA)).exists());
- }
-
- private CFMetaData addTestTable(String ks, String cf, String comment)
- {
- CFMetaData newCFMD = CFMetaData.Builder.create(ks, cf)
- .addPartitionKey("key", UTF8Type.instance)
- .addClusteringColumn("col", UTF8Type.instance)
- .addRegularColumn("val", UTF8Type.instance).build();
-
- newCFMD.comment(comment)
- .readRepairChance(0.0);
-
- return newCFMD;
- }
-}
diff --git a/test/unit/org/apache/cassandra/schema/IndexMetadataTest.java b/test/unit/org/apache/cassandra/schema/IndexMetadataTest.java
index 785ed73..c9e0d52 100644
--- a/test/unit/org/apache/cassandra/schema/IndexMetadataTest.java
+++ b/test/unit/org/apache/cassandra/schema/IndexMetadataTest.java
@@ -23,6 +23,8 @@
import org.junit.Assert;
import org.junit.Test;
+import org.apache.cassandra.cql3.ColumnIdentifier;
+
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
@@ -49,8 +51,7 @@
@Test
public void testGetDefaultIndexName()
{
- Assert.assertEquals("aB4__idx", IndexMetadata.getDefaultIndexName("a B-4@!_+", null));
- Assert.assertEquals("34_Ddd_F6_idx", IndexMetadata.getDefaultIndexName("34_()Ddd", "#F%6*"));
-
+ Assert.assertEquals("aB4__idx", IndexMetadata.generateDefaultIndexName("a B-4@!_+"));
+ Assert.assertEquals("34_Ddd_F6_idx", IndexMetadata.generateDefaultIndexName("34_()Ddd", new ColumnIdentifier("#F%6*", true)));
}
}
diff --git a/test/unit/org/apache/cassandra/schema/LegacySchemaMigratorTest.java b/test/unit/org/apache/cassandra/schema/LegacySchemaMigratorTest.java
deleted file mode 100644
index 573d109..0000000
--- a/test/unit/org/apache/cassandra/schema/LegacySchemaMigratorTest.java
+++ /dev/null
@@ -1,860 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.schema;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.*;
-import java.util.stream.Collectors;
-
-import com.google.common.collect.ImmutableList;
-import org.junit.Assert;
-import org.junit.Test;
-
-import org.apache.cassandra.SchemaLoader;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
-import org.apache.cassandra.config.Schema;
-import org.apache.cassandra.config.SchemaConstants;
-import org.apache.cassandra.cql3.CQLTester;
-import org.apache.cassandra.cql3.ColumnIdentifier;
-import org.apache.cassandra.cql3.FieldIdentifier;
-import org.apache.cassandra.cql3.functions.*;
-import org.apache.cassandra.db.*;
-import org.apache.cassandra.db.rows.Row;
-import org.apache.cassandra.db.marshal.*;
-import org.apache.cassandra.index.TargetParser;
-import org.apache.cassandra.thrift.ThriftConversion;
-import org.apache.cassandra.utils.*;
-
-import static java.lang.String.format;
-import static junit.framework.Assert.assertEquals;
-import static junit.framework.Assert.assertFalse;
-import static junit.framework.Assert.assertTrue;
-import static org.apache.cassandra.cql3.QueryProcessor.executeOnceInternal;
-import static org.apache.cassandra.utils.ByteBufferUtil.bytes;
-import static org.apache.cassandra.utils.FBUtilities.json;
-
-@SuppressWarnings("deprecation")
-public class LegacySchemaMigratorTest
-{
- private static final long TIMESTAMP = 1435908994000000L;
-
- private static final String KEYSPACE_PREFIX = "LegacySchemaMigratorTest";
-
- /*
- * 1. Write a variety of different keyspaces/tables/types/function in the legacy manner, using legacy schema tables
- * 2. Run the migrator
- * 3. Read all the keyspaces from the new schema tables
- * 4. Make sure that we've read *exactly* the same set of keyspaces/tables/types/functions
- * 5. Validate that the legacy schema tables are now empty
- */
- @Test
- public void testMigrate() throws IOException
- {
- CQLTester.cleanupAndLeaveDirs();
-
- Keyspaces expected = keyspacesToMigrate();
-
- // write the keyspaces into the legacy tables
- expected.forEach(LegacySchemaMigratorTest::legacySerializeKeyspace);
-
- // run the migration
- LegacySchemaMigrator.migrate();
-
- // read back all the metadata from the new schema tables
- Keyspaces actual = SchemaKeyspace.fetchNonSystemKeyspaces();
-
- // need to load back CFMetaData of those tables (CFS instances will still be loaded)
- loadLegacySchemaTables();
-
- // verify that nothing's left in the old schema tables
- for (CFMetaData table : LegacySchemaMigrator.LegacySchemaTables)
- {
- String query = format("SELECT * FROM %s.%s", SchemaConstants.SYSTEM_KEYSPACE_NAME, table.cfName);
- //noinspection ConstantConditions
- assertTrue(executeOnceInternal(query).isEmpty());
- }
-
- // make sure that we've read *exactly* the same set of keyspaces/tables/types/functions
- assertEquals(expected.diff(actual).toString(), expected, actual);
-
- // check that the build status of all indexes has been updated to use the new
- // format of index name: the index_name column of system.IndexInfo used to
- // contain table_name.index_name. Now it should contain just the index_name.
- expected.forEach(LegacySchemaMigratorTest::verifyIndexBuildStatus);
- }
-
- @Test
- public void testMigrateLegacyCachingOptions() throws IOException
- {
- CQLTester.cleanupAndLeaveDirs();
-
- assertEquals(CachingParams.CACHE_EVERYTHING, LegacySchemaMigrator.cachingFromRow("ALL"));
- assertEquals(CachingParams.CACHE_NOTHING, LegacySchemaMigrator.cachingFromRow("NONE"));
- assertEquals(CachingParams.CACHE_KEYS, LegacySchemaMigrator.cachingFromRow("KEYS_ONLY"));
- assertEquals(new CachingParams(false, Integer.MAX_VALUE), LegacySchemaMigrator.cachingFromRow("ROWS_ONLY"));
- assertEquals(CachingParams.CACHE_KEYS, LegacySchemaMigrator.cachingFromRow("{\"keys\" : \"ALL\", \"rows_per_partition\" : \"NONE\"}" ));
- assertEquals(new CachingParams(false, Integer.MAX_VALUE), LegacySchemaMigrator.cachingFromRow("{\"keys\" : \"NONE\", \"rows_per_partition\" : \"ALL\"}" ));
- assertEquals(new CachingParams(true, 100), LegacySchemaMigrator.cachingFromRow("{\"keys\" : \"ALL\", \"rows_per_partition\" : \"100\"}" ));
-
- try
- {
- LegacySchemaMigrator.cachingFromRow("EXCEPTION");
- Assert.fail();
- }
- catch(RuntimeException e)
- {
- // Expected passing path
- assertTrue(true);
- }
- }
-
- private static FieldIdentifier field(String field)
- {
- return FieldIdentifier.forQuoted(field);
- }
-
- private static void loadLegacySchemaTables()
- {
- KeyspaceMetadata systemKeyspace = Schema.instance.getKSMetaData(SchemaConstants.SYSTEM_KEYSPACE_NAME);
-
- Tables systemTables = systemKeyspace.tables;
- for (CFMetaData table : LegacySchemaMigrator.LegacySchemaTables)
- systemTables = systemTables.with(table);
-
- LegacySchemaMigrator.LegacySchemaTables.forEach(Schema.instance::load);
-
- Schema.instance.setKeyspaceMetadata(systemKeyspace.withSwapped(systemTables));
- }
-
- private static Keyspaces keyspacesToMigrate()
- {
- Keyspaces.Builder keyspaces = Keyspaces.builder();
-
- // A whole bucket of shorthand
- String ks1 = KEYSPACE_PREFIX + "Keyspace1";
- String ks2 = KEYSPACE_PREFIX + "Keyspace2";
- String ks3 = KEYSPACE_PREFIX + "Keyspace3";
- String ks4 = KEYSPACE_PREFIX + "Keyspace4";
- String ks5 = KEYSPACE_PREFIX + "Keyspace5";
- String ks6 = KEYSPACE_PREFIX + "Keyspace6";
- String ks_rcs = KEYSPACE_PREFIX + "RowCacheSpace";
- String ks_nocommit = KEYSPACE_PREFIX + "NoCommitlogSpace";
- String ks_prsi = KEYSPACE_PREFIX + "PerRowSecondaryIndex";
- String ks_cql = KEYSPACE_PREFIX + "cql_keyspace";
-
- // Make it easy to test compaction
- Map<String, String> compactionOptions = new HashMap<>();
- compactionOptions.put("tombstone_compaction_interval", "1");
-
- Map<String, String> leveledOptions = new HashMap<>();
- leveledOptions.put("sstable_size_in_mb", "1");
-
- keyspaces.add(KeyspaceMetadata.create(ks1,
- KeyspaceParams.simple(1),
- Tables.of(SchemaLoader.standardCFMD(ks1, "Standard1")
- .compaction(CompactionParams.scts(compactionOptions)),
- SchemaLoader.standardCFMD(ks1, "StandardGCGS0").gcGraceSeconds(0),
- SchemaLoader.standardCFMD(ks1, "StandardLong1"),
- SchemaLoader.keysIndexCFMD(ks1, "Indexed1", true),
- SchemaLoader.keysIndexCFMD(ks1, "Indexed2", false),
- SchemaLoader.jdbcCFMD(ks1, "JdbcUtf8", UTF8Type.instance)
- .addColumnDefinition(SchemaLoader.utf8Column(ks1, "JdbcUtf8")),
- SchemaLoader.jdbcCFMD(ks1, "JdbcLong", LongType.instance),
- SchemaLoader.jdbcCFMD(ks1, "JdbcBytes", BytesType.instance),
- SchemaLoader.jdbcCFMD(ks1, "JdbcAscii", AsciiType.instance),
- SchemaLoader.standardCFMD(ks1, "StandardLeveled")
- .compaction(CompactionParams.lcs(leveledOptions)),
- SchemaLoader.standardCFMD(ks1, "legacyleveled")
- .compaction(CompactionParams.lcs(leveledOptions)),
- SchemaLoader.standardCFMD(ks1, "StandardLowIndexInterval")
- .minIndexInterval(8)
- .maxIndexInterval(256)
- .caching(CachingParams.CACHE_NOTHING))));
-
- // Keyspace 2
- keyspaces.add(KeyspaceMetadata.create(ks2,
- KeyspaceParams.simple(1),
- Tables.of(SchemaLoader.standardCFMD(ks2, "Standard1"),
- SchemaLoader.keysIndexCFMD(ks2, "Indexed1", true),
- SchemaLoader.compositeIndexCFMD(ks2, "Indexed2", true),
- SchemaLoader.compositeIndexCFMD(ks2, "Indexed3", true)
- .gcGraceSeconds(0))));
-
- // Keyspace 3
- keyspaces.add(KeyspaceMetadata.create(ks3,
- KeyspaceParams.simple(5),
- Tables.of(SchemaLoader.standardCFMD(ks3, "Standard1"),
- SchemaLoader.keysIndexCFMD(ks3, "Indexed1", true))));
-
- // Keyspace 4
- keyspaces.add(KeyspaceMetadata.create(ks4,
- KeyspaceParams.simple(3),
- Tables.of(SchemaLoader.standardCFMD(ks4, "Standard1"))));
-
- // Keyspace 5
- keyspaces.add(KeyspaceMetadata.create(ks5,
- KeyspaceParams.simple(2),
- Tables.of(SchemaLoader.standardCFMD(ks5, "Standard1"))));
-
- // Keyspace 6
- keyspaces.add(KeyspaceMetadata.create(ks6,
- KeyspaceParams.simple(1),
- Tables.of(SchemaLoader.keysIndexCFMD(ks6, "Indexed1", true))));
-
- // RowCacheSpace
- keyspaces.add(KeyspaceMetadata.create(ks_rcs,
- KeyspaceParams.simple(1),
- Tables.of(SchemaLoader.standardCFMD(ks_rcs, "CFWithoutCache")
- .caching(CachingParams.CACHE_NOTHING),
- SchemaLoader.standardCFMD(ks_rcs, "CachedCF")
- .caching(CachingParams.CACHE_EVERYTHING),
- SchemaLoader.standardCFMD(ks_rcs, "CachedIntCF")
- .caching(new CachingParams(true, 100)))));
-
- keyspaces.add(KeyspaceMetadata.create(ks_nocommit,
- KeyspaceParams.simpleTransient(1),
- Tables.of(SchemaLoader.standardCFMD(ks_nocommit, "Standard1"))));
-
- // PerRowSecondaryIndexTest
- keyspaces.add(KeyspaceMetadata.create(ks_prsi,
- KeyspaceParams.simple(1),
- Tables.of(SchemaLoader.perRowIndexedCFMD(ks_prsi, "Indexed1"))));
-
- // CQLKeyspace
- keyspaces.add(KeyspaceMetadata.create(ks_cql,
- KeyspaceParams.simple(1),
- Tables.of(CFMetaData.compile("CREATE TABLE table1 ("
- + "k int PRIMARY KEY,"
- + "v1 text,"
- + "v2 int"
- + ')', ks_cql),
-
- CFMetaData.compile("CREATE TABLE table2 ("
- + "k text,"
- + "c text,"
- + "v text,"
- + "PRIMARY KEY (k, c))", ks_cql),
-
- CFMetaData.compile("CREATE TABLE foo ("
- + "bar text, "
- + "baz text, "
- + "qux text, "
- + "PRIMARY KEY(bar, baz) ) "
- + "WITH COMPACT STORAGE", ks_cql),
-
- CFMetaData.compile("CREATE TABLE compact_pkonly ("
- + "k int, "
- + "c int, "
- + "PRIMARY KEY (k, c)) "
- + "WITH COMPACT STORAGE",
- ks_cql),
-
- CFMetaData.compile("CREATE TABLE foofoo ("
- + "bar text, "
- + "baz text, "
- + "qux text, "
- + "quz text, "
- + "foo text, "
- + "PRIMARY KEY((bar, baz), qux, quz) ) "
- + "WITH COMPACT STORAGE", ks_cql))));
-
- // NTS keyspace
- keyspaces.add(KeyspaceMetadata.create("nts", KeyspaceParams.nts("dc1", 1, "dc2", 2)));
-
- keyspaces.add(keyspaceWithDroppedCollections());
- keyspaces.add(keyspaceWithTriggers());
- keyspaces.add(keyspaceWithUDTs());
- keyspaces.add(keyspaceWithUDFs());
- keyspaces.add(keyspaceWithUDFsAndUDTs());
- keyspaces.add(keyspaceWithUDAs());
- keyspaces.add(keyspaceWithUDAsAndUDTs());
-
- return keyspaces.build();
- }
-
- private static KeyspaceMetadata keyspaceWithDroppedCollections()
- {
- String keyspace = KEYSPACE_PREFIX + "DroppedCollections";
-
- CFMetaData table =
- CFMetaData.compile("CREATE TABLE dropped_columns ("
- + "foo text,"
- + "bar text,"
- + "map1 map<text, text>,"
- + "map2 map<int, int>,"
- + "set1 set<ascii>,"
- + "list1 list<blob>,"
- + "PRIMARY KEY ((foo), bar))",
- keyspace);
-
- String[] collectionColumnNames = { "map1", "map2", "set1", "list1" };
- for (String name : collectionColumnNames)
- {
- ColumnDefinition column = table.getColumnDefinition(bytes(name));
- table.recordColumnDrop(column, FBUtilities.timestampMicros(), false);
- table.removeColumnDefinition(column);
- }
-
- return KeyspaceMetadata.create(keyspace, KeyspaceParams.simple(1), Tables.of(table));
- }
-
- private static KeyspaceMetadata keyspaceWithTriggers()
- {
- String keyspace = KEYSPACE_PREFIX + "Triggers";
-
- Triggers.Builder triggers = Triggers.builder();
- CFMetaData table = SchemaLoader.standardCFMD(keyspace, "WithTriggers");
- for (int i = 0; i < 10; i++)
- triggers.add(new TriggerMetadata("trigger" + i, "DummyTrigger" + i));
- table.triggers(triggers.build());
-
- return KeyspaceMetadata.create(keyspace, KeyspaceParams.simple(1), Tables.of(table));
- }
-
- private static KeyspaceMetadata keyspaceWithUDTs()
- {
- String keyspace = KEYSPACE_PREFIX + "UDTs";
-
- UserType udt1 = new UserType(keyspace,
- bytes("udt1"),
- new ArrayList<FieldIdentifier>() {{ add(field("col1")); add(field("col2")); }},
- new ArrayList<AbstractType<?>>() {{ add(UTF8Type.instance); add(Int32Type.instance); }},
- true);
-
- UserType udt2 = new UserType(keyspace,
- bytes("udt2"),
- new ArrayList<FieldIdentifier>() {{ add(field("col3")); add(field("col4")); }},
- new ArrayList<AbstractType<?>>() {{ add(BytesType.instance); add(BooleanType.instance); }},
- true);
-
- UserType udt3 = new UserType(keyspace,
- bytes("udt3"),
- new ArrayList<FieldIdentifier>() {{ add(field("col5")); }},
- new ArrayList<AbstractType<?>>() {{ add(AsciiType.instance); }},
- true);
-
- return KeyspaceMetadata.create(keyspace,
- KeyspaceParams.simple(1),
- Tables.none(),
- Views.none(),
- Types.of(udt1, udt2, udt3),
- Functions.none());
- }
-
- private static KeyspaceMetadata keyspaceWithUDFs()
- {
- String keyspace = KEYSPACE_PREFIX + "UDFs";
-
- UDFunction udf1 = UDFunction.create(new FunctionName(keyspace, "udf"),
- ImmutableList.of(new ColumnIdentifier("col1", false), new ColumnIdentifier("col2", false)),
- ImmutableList.of(BytesType.instance, Int32Type.instance),
- LongType.instance,
- false,
- "java",
- "return 42L;");
-
- // an overload with the same name, not a typo
- UDFunction udf2 = UDFunction.create(new FunctionName(keyspace, "udf"),
- ImmutableList.of(new ColumnIdentifier("col3", false), new ColumnIdentifier("col4", false)),
- ImmutableList.of(AsciiType.instance, LongType.instance),
- Int32Type.instance,
- true,
- "java",
- "return 42;");
-
- UDFunction udf3 = UDFunction.create(new FunctionName(keyspace, "udf3"),
- ImmutableList.of(new ColumnIdentifier("col4", false)),
- ImmutableList.of(UTF8Type.instance),
- BooleanType.instance,
- false,
- "java",
- "return true;");
-
- return KeyspaceMetadata.create(keyspace,
- KeyspaceParams.simple(1),
- Tables.none(),
- Views.none(),
- Types.none(),
- Functions.of(udf1, udf2, udf3));
- }
-
- private static KeyspaceMetadata keyspaceWithUDAs()
- {
- String keyspace = KEYSPACE_PREFIX + "UDAs";
-
- UDFunction udf1 = UDFunction.create(new FunctionName(keyspace, "udf1"),
- ImmutableList.of(new ColumnIdentifier("col1", false), new ColumnIdentifier("col2", false)),
- ImmutableList.of(Int32Type.instance, Int32Type.instance),
- Int32Type.instance,
- false,
- "java",
- "return 42;");
-
- UDFunction udf2 = UDFunction.create(new FunctionName(keyspace, "udf2"),
- ImmutableList.of(new ColumnIdentifier("col1", false), new ColumnIdentifier("col2", false)),
- ImmutableList.of(LongType.instance, Int32Type.instance),
- LongType.instance,
- false,
- "java",
- "return 42L;");
-
- UDFunction udf3 = UDFunction.create(new FunctionName(keyspace, "udf3"),
- ImmutableList.of(new ColumnIdentifier("col1", false)),
- ImmutableList.of(LongType.instance),
- DoubleType.instance,
- false,
- "java",
- "return 42d;");
-
- Functions udfs = Functions.builder().add(udf1).add(udf2).add(udf3).build();
-
- UDAggregate uda1 = UDAggregate.create(udfs, new FunctionName(keyspace, "uda1"),
- ImmutableList.of(udf1.argTypes().get(1)),
- udf1.returnType(),
- udf1.name(),
- null,
- udf1.argTypes().get(0),
- null
- );
-
- UDAggregate uda2 = UDAggregate.create(udfs, new FunctionName(keyspace, "uda2"),
- ImmutableList.of(udf2.argTypes().get(1)),
- udf3.returnType(),
- udf2.name(),
- udf3.name(),
- udf2.argTypes().get(0),
- LongType.instance.decompose(0L)
- );
-
- return KeyspaceMetadata.create(keyspace,
- KeyspaceParams.simple(1),
- Tables.none(),
- Views.none(),
- Types.none(),
- Functions.of(udf1, udf2, udf3, uda1, uda2));
- }
-
- private static KeyspaceMetadata keyspaceWithUDFsAndUDTs()
- {
- String keyspace = KEYSPACE_PREFIX + "UDFUDTs";
-
- UserType udt1 = new UserType(keyspace,
- bytes("udt1"),
- new ArrayList<FieldIdentifier>() {{ add(field("col1")); add(field("col2")); }},
- new ArrayList<AbstractType<?>>() {{ add(UTF8Type.instance); add(Int32Type.instance); }},
- true);
-
- UserType udt2 = new UserType(keyspace,
- bytes("udt2"),
- new ArrayList<FieldIdentifier>() {{ add(field("col1")); add(field("col2")); }},
- new ArrayList<AbstractType<?>>() {{ add(ListType.getInstance(udt1, false)); add(Int32Type.instance); }},
- true);
-
- UDFunction udf1 = UDFunction.create(new FunctionName(keyspace, "udf"),
- ImmutableList.of(new ColumnIdentifier("col1", false), new ColumnIdentifier("col2", false)),
- ImmutableList.of(udt1, udt2),
- LongType.instance,
- false,
- "java",
- "return 42L;");
-
- // an overload with the same name, not a typo
- UDFunction udf2 = UDFunction.create(new FunctionName(keyspace, "udf"),
- ImmutableList.of(new ColumnIdentifier("col3", false), new ColumnIdentifier("col4", false)),
- ImmutableList.of(AsciiType.instance, LongType.instance),
- Int32Type.instance,
- true,
- "java",
- "return 42;");
-
- UDFunction udf3 = UDFunction.create(new FunctionName(keyspace, "udf3"),
- ImmutableList.of(new ColumnIdentifier("col4", false)),
- ImmutableList.of(new TupleType(Arrays.asList(udt1, udt2))),
- BooleanType.instance,
- false,
- "java",
- "return true;");
-
- return KeyspaceMetadata.create(keyspace,
- KeyspaceParams.simple(1),
- Tables.none(),
- Views.none(),
- Types.of(udt1, udt2),
- Functions.of(udf1, udf2, udf3));
- }
-
- private static KeyspaceMetadata keyspaceWithUDAsAndUDTs()
- {
- String keyspace = KEYSPACE_PREFIX + "UDAUDTs";
-
- UserType udt1 = new UserType(keyspace,
- bytes("udt1"),
- new ArrayList<FieldIdentifier>() {{ add(field("col1")); add(field("col2")); }},
- new ArrayList<AbstractType<?>>() {{ add(UTF8Type.instance); add(Int32Type.instance); }},
- true);
-
- UserType udt2 = new UserType(keyspace,
- bytes("udt2"),
- new ArrayList<FieldIdentifier>() {{ add(field("col1")); add(field("col2")); }},
- new ArrayList<AbstractType<?>>() {{ add(ListType.getInstance(udt1, false)); add(Int32Type.instance); }},
- true);
-
- UDFunction udf1 = UDFunction.create(new FunctionName(keyspace, "udf1"),
- ImmutableList.of(new ColumnIdentifier("col1", false), new ColumnIdentifier("col2", false)),
- ImmutableList.of(udt1, udt2),
- udt1,
- false,
- "java",
- "return null;");
-
- UDFunction udf2 = UDFunction.create(new FunctionName(keyspace, "udf2"),
- ImmutableList.of(new ColumnIdentifier("col1", false), new ColumnIdentifier("col2", false)),
- ImmutableList.of(udt2, udt1),
- udt2,
- false,
- "java",
- "return null;");
-
- UDFunction udf3 = UDFunction.create(new FunctionName(keyspace, "udf3"),
- ImmutableList.of(new ColumnIdentifier("col1", false)),
- ImmutableList.of(udt2),
- DoubleType.instance,
- false,
- "java",
- "return 42d;");
-
- Functions udfs = Functions.builder().add(udf1).add(udf2).add(udf3).build();
-
- UDAggregate uda1 = UDAggregate.create(udfs, new FunctionName(keyspace, "uda1"),
- ImmutableList.of(udf1.argTypes().get(1)),
- udf1.returnType(),
- udf1.name(),
- null,
- udf1.argTypes().get(0),
- null
- );
-
- ByteBuffer twoNullEntries = ByteBuffer.allocate(8);
- twoNullEntries.putInt(-1);
- twoNullEntries.putInt(-1);
- twoNullEntries.flip();
- UDAggregate uda2 = UDAggregate.create(udfs, new FunctionName(keyspace, "uda2"),
- ImmutableList.of(udf2.argTypes().get(1)),
- udf3.returnType(),
- udf2.name(),
- udf3.name(),
- udf2.argTypes().get(0),
- twoNullEntries
- );
-
- return KeyspaceMetadata.create(keyspace,
- KeyspaceParams.simple(1),
- Tables.none(),
- Views.none(),
- Types.of(udt1, udt2),
- Functions.of(udf1, udf2, udf3, uda1, uda2));
- }
-
- /*
- * Serializing keyspaces
- */
-
- private static void legacySerializeKeyspace(KeyspaceMetadata keyspace)
- {
- makeLegacyCreateKeyspaceMutation(keyspace, TIMESTAMP).apply();
- setLegacyIndexStatus(keyspace);
- }
-
- private static DecoratedKey decorate(CFMetaData metadata, Object value)
- {
- return metadata.decorateKey(((AbstractType)metadata.getKeyValidator()).decompose(value));
- }
-
- private static Mutation makeLegacyCreateKeyspaceMutation(KeyspaceMetadata keyspace, long timestamp)
- {
- Mutation.SimpleBuilder builder = Mutation.simpleBuilder(SchemaConstants.SYSTEM_KEYSPACE_NAME, decorate(SystemKeyspace.LegacyKeyspaces, keyspace.name))
- .timestamp(timestamp);
-
- builder.update(SystemKeyspace.LegacyKeyspaces)
- .row()
- .add("durable_writes", keyspace.params.durableWrites)
- .add("strategy_class", keyspace.params.replication.klass.getName())
- .add("strategy_options", json(keyspace.params.replication.options));
-
- keyspace.tables.forEach(table -> addTableToSchemaMutation(table, true, builder));
- keyspace.types.forEach(type -> addTypeToSchemaMutation(type, builder));
- keyspace.functions.udfs().forEach(udf -> addFunctionToSchemaMutation(udf, builder));
- keyspace.functions.udas().forEach(uda -> addAggregateToSchemaMutation(uda, builder));
-
- return builder.build();
- }
-
- /*
- * Serializing tables
- */
-
- private static void addTableToSchemaMutation(CFMetaData table, boolean withColumnsAndTriggers, Mutation.SimpleBuilder builder)
- {
- // For property that can be null (and can be changed), we insert tombstones, to make sure
- // we don't keep a property the user has removed
- Row.SimpleBuilder adder = builder.update(SystemKeyspace.LegacyColumnfamilies)
- .row(table.cfName);
-
- adder.add("cf_id", table.cfId)
- .add("type", table.isSuper() ? "Super" : "Standard");
-
- if (table.isSuper())
- {
- adder.add("comparator", table.comparator.subtype(0).toString())
- .add("subcomparator", ((MapType)table.compactValueColumn().type).getKeysType().toString());
- }
- else
- {
- adder.add("comparator", LegacyLayout.makeLegacyComparator(table).toString());
- }
-
- adder.add("bloom_filter_fp_chance", table.params.bloomFilterFpChance)
- .add("caching", cachingToString(table.params.caching))
- .add("comment", table.params.comment)
- .add("compaction_strategy_class", table.params.compaction.klass().getName())
- .add("compaction_strategy_options", json(table.params.compaction.options()))
- .add("compression_parameters", json(ThriftConversion.compressionParametersToThrift(table.params.compression)))
- .add("default_time_to_live", table.params.defaultTimeToLive)
- .add("gc_grace_seconds", table.params.gcGraceSeconds)
- .add("key_validator", table.getKeyValidator().toString())
- .add("local_read_repair_chance", table.params.dcLocalReadRepairChance)
- .add("max_compaction_threshold", table.params.compaction.maxCompactionThreshold())
- .add("max_index_interval", table.params.maxIndexInterval)
- .add("memtable_flush_period_in_ms", table.params.memtableFlushPeriodInMs)
- .add("min_compaction_threshold", table.params.compaction.minCompactionThreshold())
- .add("min_index_interval", table.params.minIndexInterval)
- .add("read_repair_chance", table.params.readRepairChance)
- .add("speculative_retry", table.params.speculativeRetry.toString());
-
- Map<String, Long> dropped = new HashMap<>();
- for (Map.Entry<ByteBuffer, CFMetaData.DroppedColumn> entry : table.getDroppedColumns().entrySet())
- {
- String name = UTF8Type.instance.getString(entry.getKey());
- CFMetaData.DroppedColumn column = entry.getValue();
- dropped.put(name, column.droppedTime);
- }
- adder.add("dropped_columns", dropped);
-
- adder.add("is_dense", table.isDense());
-
- adder.add("default_validator", table.makeLegacyDefaultValidator().toString());
-
- if (withColumnsAndTriggers)
- {
- for (ColumnDefinition column : table.allColumns())
- addColumnToSchemaMutation(table, column, builder);
-
- for (TriggerMetadata trigger : table.getTriggers())
- addTriggerToSchemaMutation(table, trigger, builder);
- }
- }
-
- private static String cachingToString(CachingParams caching)
- {
- return format("{\"keys\":\"%s\", \"rows_per_partition\":\"%s\"}",
- caching.keysAsString(),
- caching.rowsPerPartitionAsString());
- }
-
- private static void addColumnToSchemaMutation(CFMetaData table, ColumnDefinition column, Mutation.SimpleBuilder builder)
- {
- // We need to special case pk-only dense tables. See CASSANDRA-9874.
- String name = table.isDense() && column.kind == ColumnDefinition.Kind.REGULAR && column.type instanceof EmptyType
- ? ""
- : column.name.toString();
-
- final Row.SimpleBuilder adder = builder.update(SystemKeyspace.LegacyColumns).row(table.cfName, name);
-
- adder.add("validator", column.type.toString())
- .add("type", serializeKind(column.kind, table.isDense()))
- .add("component_index", column.position());
-
- Optional<IndexMetadata> index = findIndexForColumn(table.getIndexes(), table, column);
- if (index.isPresent())
- {
- IndexMetadata i = index.get();
- adder.add("index_name", i.name);
- adder.add("index_type", i.kind.toString());
- adder.add("index_options", json(i.options));
- }
- else
- {
- adder.add("index_name", null);
- adder.add("index_type", null);
- adder.add("index_options", null);
- }
- }
-
- private static Optional<IndexMetadata> findIndexForColumn(Indexes indexes,
- CFMetaData table,
- ColumnDefinition column)
- {
- // makes the assumptions that the string option denoting the
- // index targets can be parsed by CassandraIndex.parseTarget
- // which should be true for any pre-3.0 index
- for (IndexMetadata index : indexes)
- if (TargetParser.parse(table, index).left.equals(column))
- return Optional.of(index);
-
- return Optional.empty();
- }
-
- private static String serializeKind(ColumnDefinition.Kind kind, boolean isDense)
- {
- // For backward compatibility, we special case CLUSTERING and the case where the table is dense.
- if (kind == ColumnDefinition.Kind.CLUSTERING)
- return "clustering_key";
-
- if (kind == ColumnDefinition.Kind.REGULAR && isDense)
- return "compact_value";
-
- return kind.toString().toLowerCase();
- }
-
- private static void addTriggerToSchemaMutation(CFMetaData table, TriggerMetadata trigger, Mutation.SimpleBuilder builder)
- {
- builder.update(SystemKeyspace.LegacyTriggers)
- .row(table.cfName, trigger.name)
- .add("trigger_options", Collections.singletonMap("class", trigger.classOption));
- }
-
- /*
- * Serializing types
- */
-
- private static void addTypeToSchemaMutation(UserType type, Mutation.SimpleBuilder builder)
- {
- Row.SimpleBuilder adder = builder.update(SystemKeyspace.LegacyUsertypes)
- .row(type.getNameAsString());
-
- List<String> names = new ArrayList<>();
- List<String> types = new ArrayList<>();
- for (int i = 0; i < type.size(); i++)
- {
- names.add(type.fieldName(i).toString());
- types.add(type.fieldType(i).toString());
- }
-
- adder.add("field_names", names)
- .add("field_types", types);
- }
-
- /*
- * Serializing functions
- */
-
- private static void addFunctionToSchemaMutation(UDFunction function, Mutation.SimpleBuilder builder)
- {
- Row.SimpleBuilder adder = builder.update(SystemKeyspace.LegacyFunctions)
- .row(function.name().name, functionSignatureWithTypes(function));
-
- adder.add("body", function.body())
- .add("language", function.language())
- .add("return_type", function.returnType().toString())
- .add("called_on_null_input", function.isCalledOnNullInput());
-
- List<ByteBuffer> names = new ArrayList<>();
- List<String> types = new ArrayList<>();
- for (int i = 0; i < function.argNames().size(); i++)
- {
- names.add(function.argNames().get(i).bytes);
- types.add(function.argTypes().get(i).toString());
- }
- adder.add("argument_names", names)
- .add("argument_types", types);
- }
-
- /*
- * Serializing aggregates
- */
-
- private static void addAggregateToSchemaMutation(UDAggregate aggregate, Mutation.SimpleBuilder builder)
- {
- Row.SimpleBuilder adder = builder.update(SystemKeyspace.LegacyAggregates)
- .row(aggregate.name().name, functionSignatureWithTypes(aggregate));
-
- adder.add("return_type", aggregate.returnType().toString())
- .add("state_func", aggregate.stateFunction().name().name);
-
- if (aggregate.stateType() != null)
- adder.add("state_type", aggregate.stateType().toString());
- if (aggregate.finalFunction() != null)
- adder.add("final_func", aggregate.finalFunction().name().name);
- if (aggregate.initialCondition() != null)
- adder.add("initcond", aggregate.initialCondition());
-
- List<String> types = new ArrayList<>();
- for (AbstractType<?> argType : aggregate.argTypes())
- types.add(argType.toString());
-
- adder.add("argument_types", types);
- }
-
- // We allow method overloads, so a function is not uniquely identified by its name only, but
- // also by its argument types. To distinguish overloads of given function name in the schema
- // we use a "signature" which is just a list of it's CQL argument types.
- public static ByteBuffer functionSignatureWithTypes(AbstractFunction fun)
- {
- List<String> arguments =
- fun.argTypes()
- .stream()
- .map(argType -> argType.asCQL3Type().toString())
- .collect(Collectors.toList());
-
- return ListType.getInstance(UTF8Type.instance, false).decompose(arguments);
- }
-
- private static void setLegacyIndexStatus(KeyspaceMetadata keyspace)
- {
- keyspace.tables.forEach(LegacySchemaMigratorTest::setLegacyIndexStatus);
- }
-
- private static void setLegacyIndexStatus(CFMetaData table)
- {
- table.getIndexes().forEach((index) -> setLegacyIndexStatus(table.ksName, table.cfName, index));
- }
-
- private static void setLegacyIndexStatus(String keyspace, String table, IndexMetadata index)
- {
- SystemKeyspace.setIndexBuilt(keyspace, table + '.' + index.name);
- }
-
- private static void verifyIndexBuildStatus(KeyspaceMetadata keyspace)
- {
- keyspace.tables.forEach(LegacySchemaMigratorTest::verifyIndexBuildStatus);
- }
-
- private static void verifyIndexBuildStatus(CFMetaData table)
- {
- table.getIndexes().forEach(index -> verifyIndexBuildStatus(table.ksName, table.cfName, index));
- }
-
- private static void verifyIndexBuildStatus(String keyspace, String table, IndexMetadata index)
- {
- assertFalse(SystemKeyspace.isIndexBuilt(keyspace, table + '.' + index.name));
- assertTrue(SystemKeyspace.isIndexBuilt(keyspace, index.name));
- }
-
-}
diff --git a/test/unit/org/apache/cassandra/schema/MigrationManagerTest.java b/test/unit/org/apache/cassandra/schema/MigrationManagerTest.java
new file mode 100644
index 0000000..ff58151
--- /dev/null
+++ b/test/unit/org/apache/cassandra/schema/MigrationManagerTest.java
@@ -0,0 +1,627 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.schema;
+
+import java.io.File;
+import java.nio.ByteBuffer;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+import java.util.function.Supplier;
+
+import com.google.common.collect.ImmutableMap;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.junit.runner.RunWith;
+
+import org.apache.cassandra.OrderedJUnit4ClassRunner;
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.Util;
+import org.apache.cassandra.cql3.ColumnIdentifier;
+import org.apache.cassandra.cql3.QueryProcessor;
+import org.apache.cassandra.cql3.UntypedResultSet;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Directories;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.Mutation;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.db.marshal.ByteType;
+import org.apache.cassandra.db.marshal.BytesType;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.io.sstable.Component;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.locator.NetworkTopologyStrategy;
+import org.apache.cassandra.utils.FBUtilities;
+
+import static java.util.Collections.singleton;
+import static org.apache.cassandra.Util.throwAssert;
+import static org.apache.cassandra.cql3.CQLTester.assertRows;
+import static org.apache.cassandra.cql3.CQLTester.row;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+
+@RunWith(OrderedJUnit4ClassRunner.class)
+public class MigrationManagerTest
+{
+ private static final String KEYSPACE1 = "keyspace1";
+ private static final String KEYSPACE3 = "keyspace3";
+ private static final String KEYSPACE6 = "keyspace6";
+ private static final String EMPTY_KEYSPACE = "test_empty_keyspace";
+ private static final String TABLE1 = "standard1";
+ private static final String TABLE2 = "standard2";
+ private static final String TABLE1i = "indexed1";
+
+ @Rule
+ public ExpectedException thrown = ExpectedException.none();
+
+ @BeforeClass
+ public static void defineSchema() throws ConfigurationException
+ {
+ SchemaLoader.prepareServer();
+ SchemaLoader.startGossiper();
+ SchemaLoader.createKeyspace(KEYSPACE1,
+ KeyspaceParams.simple(1),
+ SchemaLoader.standardCFMD(KEYSPACE1, TABLE1),
+ SchemaLoader.standardCFMD(KEYSPACE1, TABLE2));
+ SchemaLoader.createKeyspace(KEYSPACE3,
+ KeyspaceParams.simple(5),
+ SchemaLoader.standardCFMD(KEYSPACE1, TABLE1),
+ SchemaLoader.compositeIndexCFMD(KEYSPACE3, TABLE1i, true));
+ SchemaLoader.createKeyspace(KEYSPACE6,
+ KeyspaceParams.simple(1),
+ SchemaLoader.compositeIndexCFMD(KEYSPACE6, TABLE1i, true));
+ }
+
+ @Test
+ public void testTableMetadataBuilder() throws ConfigurationException
+ {
+ TableMetadata.Builder builder =
+ TableMetadata.builder(KEYSPACE1, "TestApplyCFM_CF")
+ .addPartitionKeyColumn("keys", BytesType.instance)
+ .addClusteringColumn("col", BytesType.instance)
+ .comment("No comment")
+ .gcGraceSeconds(100000)
+ .compaction(CompactionParams.stcs(ImmutableMap.of("min_threshold", "500", "max_threshold", "500")));
+
+ for (int i = 0; i < 5; i++)
+ {
+ ByteBuffer name = ByteBuffer.wrap(new byte[] { (byte)i });
+ builder.addRegularColumn(ColumnIdentifier.getInterned(name, BytesType.instance), ByteType.instance);
+ }
+
+
+ TableMetadata table = builder.build();
+ // we'll be adding this one later. make sure it's not already there.
+ assertNull(table.getColumn(ByteBuffer.wrap(new byte[]{ 5 })));
+
+ // add one.
+ ColumnMetadata addIndexDef = ColumnMetadata.regularColumn(table, ByteBuffer.wrap(new byte[] { 5 }), BytesType.instance);
+ builder.addColumn(addIndexDef);
+
+ // remove one.
+ ColumnMetadata removeIndexDef = ColumnMetadata.regularColumn(table, ByteBuffer.wrap(new byte[] { 0 }), BytesType.instance);
+ builder.removeRegularOrStaticColumn(removeIndexDef.name);
+
+ TableMetadata table2 = builder.build();
+
+ for (int i = 1; i < table2.columns().size(); i++)
+ assertNotNull(table2.getColumn(ByteBuffer.wrap(new byte[]{ 1 })));
+ assertNull(table2.getColumn(ByteBuffer.wrap(new byte[]{ 0 })));
+ assertNotNull(table2.getColumn(ByteBuffer.wrap(new byte[]{ 5 })));
+ }
+
+ @Test
+ public void testInvalidNames()
+ {
+ String[] valid = {"1", "a", "_1", "b_", "__", "1_a"};
+ for (String s : valid)
+ assertTrue(SchemaConstants.isValidName(s));
+
+ String[] invalid = {"b@t", "dash-y", "", " ", "dot.s", ".hidden"};
+ for (String s : invalid)
+ assertFalse(SchemaConstants.isValidName(s));
+ }
+
+ @Test
+ public void addNewCfToBogusKeyspace()
+ {
+ TableMetadata newCf = addTestTable("MadeUpKeyspace", "NewCF", "new cf");
+ try
+ {
+ MigrationManager.announceNewTable(newCf);
+ throw new AssertionError("You shouldn't be able to do anything to a keyspace that doesn't exist.");
+ }
+ catch (ConfigurationException expected)
+ {
+ }
+ }
+
+ @Test
+ public void addNewTable() throws ConfigurationException
+ {
+ final String ksName = KEYSPACE1;
+ final String tableName = "anewtable";
+ KeyspaceMetadata original = Schema.instance.getKeyspaceMetadata(ksName);
+
+ TableMetadata cfm = addTestTable(original.name, tableName, "A New Table");
+
+ assertFalse(Schema.instance.getKeyspaceMetadata(ksName).tables.get(cfm.name).isPresent());
+ MigrationManager.announceNewTable(cfm);
+
+ assertTrue(Schema.instance.getKeyspaceMetadata(ksName).tables.get(cfm.name).isPresent());
+ assertEquals(cfm, Schema.instance.getKeyspaceMetadata(ksName).tables.get(cfm.name).get());
+
+ // now read and write to it.
+ QueryProcessor.executeInternal(String.format("INSERT INTO %s.%s (key, col, val) VALUES (?, ?, ?)",
+ ksName, tableName),
+ "key0", "col0", "val0");
+
+ // flush to exercise more than just hitting the memtable
+ ColumnFamilyStore cfs = Keyspace.open(ksName).getColumnFamilyStore(tableName);
+ assertNotNull(cfs);
+ cfs.forceBlockingFlush();
+
+ // and make sure we get out what we put in
+ UntypedResultSet rows = QueryProcessor.executeInternal(String.format("SELECT * FROM %s.%s", ksName, tableName));
+ assertRows(rows, row("key0", "col0", "val0"));
+ }
+
+ @Test
+ public void dropCf() throws ConfigurationException
+ {
+ // sanity
+ final KeyspaceMetadata ks = Schema.instance.getKeyspaceMetadata(KEYSPACE1);
+ assertNotNull(ks);
+ final TableMetadata cfm = ks.tables.getNullable(TABLE1);
+ assertNotNull(cfm);
+
+ // write some data, force a flush, then verify that files exist on disk.
+ for (int i = 0; i < 100; i++)
+ QueryProcessor.executeInternal(String.format("INSERT INTO %s.%s (key, name, val) VALUES (?, ?, ?)",
+ KEYSPACE1, TABLE1),
+ "dropCf", "col" + i, "anyvalue");
+ ColumnFamilyStore store = Keyspace.open(cfm.keyspace).getColumnFamilyStore(cfm.name);
+ assertNotNull(store);
+ store.forceBlockingFlush();
+ assertTrue(store.getDirectories().sstableLister(Directories.OnTxnErr.THROW).list().size() > 0);
+
+ MigrationManager.announceTableDrop(ks.name, cfm.name, false);
+
+ assertFalse(Schema.instance.getKeyspaceMetadata(ks.name).tables.get(cfm.name).isPresent());
+
+ // any write should fail.
+ boolean success = true;
+ try
+ {
+ QueryProcessor.executeInternal(String.format("INSERT INTO %s.%s (key, name, val) VALUES (?, ?, ?)",
+ KEYSPACE1, TABLE1),
+ "dropCf", "col0", "anyvalue");
+ }
+ catch (Throwable th)
+ {
+ success = false;
+ }
+ assertFalse("This mutation should have failed since the CF no longer exists.", success);
+
+ // verify that the files are gone.
+ Supplier<Object> lambda = () -> {
+ for (File file : store.getDirectories().sstableLister(Directories.OnTxnErr.THROW).listFiles())
+ {
+ if (file.getPath().endsWith("Data.db") && !new File(file.getPath().replace("Data.db", "Compacted")).exists())
+ return false;
+ }
+ return true;
+ };
+ Util.spinAssertEquals(true, lambda, 30);
+
+ }
+
+ @Test
+ public void addNewKS() throws ConfigurationException
+ {
+ TableMetadata cfm = addTestTable("newkeyspace1", "newstandard1", "A new cf for a new ks");
+ KeyspaceMetadata newKs = KeyspaceMetadata.create(cfm.keyspace, KeyspaceParams.simple(5), Tables.of(cfm));
+ MigrationManager.announceNewKeyspace(newKs);
+
+ assertNotNull(Schema.instance.getKeyspaceMetadata(cfm.keyspace));
+ assertEquals(Schema.instance.getKeyspaceMetadata(cfm.keyspace), newKs);
+
+ // test reads and writes.
+ QueryProcessor.executeInternal("INSERT INTO newkeyspace1.newstandard1 (key, col, val) VALUES (?, ?, ?)",
+ "key0", "col0", "val0");
+ ColumnFamilyStore store = Keyspace.open(cfm.keyspace).getColumnFamilyStore(cfm.name);
+ assertNotNull(store);
+ store.forceBlockingFlush();
+
+ UntypedResultSet rows = QueryProcessor.executeInternal("SELECT * FROM newkeyspace1.newstandard1");
+ assertRows(rows, row("key0", "col0", "val0"));
+ }
+
+ @Test
+ public void dropKS() throws ConfigurationException
+ {
+ // sanity
+ final KeyspaceMetadata ks = Schema.instance.getKeyspaceMetadata(KEYSPACE1);
+ assertNotNull(ks);
+ final TableMetadata cfm = ks.tables.getNullable(TABLE2);
+ assertNotNull(cfm);
+
+ // write some data, force a flush, then verify that files exist on disk.
+ for (int i = 0; i < 100; i++)
+ QueryProcessor.executeInternal(String.format("INSERT INTO %s.%s (key, name, val) VALUES (?, ?, ?)",
+ KEYSPACE1, TABLE2),
+ "dropKs", "col" + i, "anyvalue");
+ ColumnFamilyStore cfs = Keyspace.open(cfm.keyspace).getColumnFamilyStore(cfm.name);
+ assertNotNull(cfs);
+ cfs.forceBlockingFlush();
+ assertTrue(!cfs.getDirectories().sstableLister(Directories.OnTxnErr.THROW).list().isEmpty());
+
+ MigrationManager.announceKeyspaceDrop(ks.name);
+
+ assertNull(Schema.instance.getKeyspaceMetadata(ks.name));
+
+ // write should fail.
+ boolean success = true;
+ try
+ {
+ QueryProcessor.executeInternal(String.format("INSERT INTO %s.%s (key, name, val) VALUES (?, ?, ?)",
+ KEYSPACE1, TABLE2),
+ "dropKs", "col0", "anyvalue");
+ }
+ catch (Throwable th)
+ {
+ success = false;
+ }
+ assertFalse("This mutation should have failed since the KS no longer exists.", success);
+
+ // reads should fail too.
+ boolean threw = false;
+ try
+ {
+ Keyspace.open(ks.name);
+ }
+ catch (Throwable th)
+ {
+ threw = true;
+ }
+ assertTrue(threw);
+ }
+
+ @Test
+ public void dropKSUnflushed() throws ConfigurationException
+ {
+ // sanity
+ final KeyspaceMetadata ks = Schema.instance.getKeyspaceMetadata(KEYSPACE3);
+ assertNotNull(ks);
+ final TableMetadata cfm = ks.tables.getNullable(TABLE1);
+ assertNotNull(cfm);
+
+ // write some data
+ for (int i = 0; i < 100; i++)
+ QueryProcessor.executeInternal(String.format("INSERT INTO %s.%s (key, name, val) VALUES (?, ?, ?)",
+ KEYSPACE3, TABLE1),
+ "dropKs", "col" + i, "anyvalue");
+
+ MigrationManager.announceKeyspaceDrop(ks.name);
+
+ assertNull(Schema.instance.getKeyspaceMetadata(ks.name));
+ }
+
+ @Test
+ public void createEmptyKsAddNewCf() throws ConfigurationException
+ {
+ assertNull(Schema.instance.getKeyspaceMetadata(EMPTY_KEYSPACE));
+ KeyspaceMetadata newKs = KeyspaceMetadata.create(EMPTY_KEYSPACE, KeyspaceParams.simple(5));
+ MigrationManager.announceNewKeyspace(newKs);
+ assertNotNull(Schema.instance.getKeyspaceMetadata(EMPTY_KEYSPACE));
+
+ String tableName = "added_later";
+ TableMetadata newCf = addTestTable(EMPTY_KEYSPACE, tableName, "A new CF to add to an empty KS");
+
+ //should not exist until apply
+ assertFalse(Schema.instance.getKeyspaceMetadata(newKs.name).tables.get(newCf.name).isPresent());
+
+ //add the new CF to the empty space
+ MigrationManager.announceNewTable(newCf);
+
+ assertTrue(Schema.instance.getKeyspaceMetadata(newKs.name).tables.get(newCf.name).isPresent());
+ assertEquals(Schema.instance.getKeyspaceMetadata(newKs.name).tables.get(newCf.name).get(), newCf);
+
+ // now read and write to it.
+ QueryProcessor.executeInternal(String.format("INSERT INTO %s.%s (key, col, val) VALUES (?, ?, ?)",
+ EMPTY_KEYSPACE, tableName),
+ "key0", "col0", "val0");
+
+ ColumnFamilyStore cfs = Keyspace.open(newKs.name).getColumnFamilyStore(newCf.name);
+ assertNotNull(cfs);
+ cfs.forceBlockingFlush();
+
+ UntypedResultSet rows = QueryProcessor.executeInternal(String.format("SELECT * FROM %s.%s", EMPTY_KEYSPACE, tableName));
+ assertRows(rows, row("key0", "col0", "val0"));
+ }
+
+ @Test
+ public void testUpdateKeyspace() throws ConfigurationException
+ {
+ // create a keyspace to serve as existing.
+ TableMetadata cf = addTestTable("UpdatedKeyspace", "AddedStandard1", "A new cf for a new ks");
+ KeyspaceMetadata oldKs = KeyspaceMetadata.create(cf.keyspace, KeyspaceParams.simple(5), Tables.of(cf));
+
+ MigrationManager.announceNewKeyspace(oldKs);
+
+ assertNotNull(Schema.instance.getKeyspaceMetadata(cf.keyspace));
+ assertEquals(Schema.instance.getKeyspaceMetadata(cf.keyspace), oldKs);
+
+ // names should match.
+ KeyspaceMetadata newBadKs2 = KeyspaceMetadata.create(cf.keyspace + "trash", KeyspaceParams.simple(4));
+ try
+ {
+ MigrationManager.announceKeyspaceUpdate(newBadKs2);
+ throw new AssertionError("Should not have been able to update a KS with an invalid KS name.");
+ }
+ catch (ConfigurationException ex)
+ {
+ // expected.
+ }
+
+ Map<String, String> replicationMap = new HashMap<>();
+ replicationMap.put(ReplicationParams.CLASS, NetworkTopologyStrategy.class.getName());
+ replicationMap.put("replication_factor", "1");
+
+ KeyspaceMetadata newKs = KeyspaceMetadata.create(cf.keyspace, KeyspaceParams.create(true, replicationMap));
+ MigrationManager.announceKeyspaceUpdate(newKs);
+
+ KeyspaceMetadata newFetchedKs = Schema.instance.getKeyspaceMetadata(newKs.name);
+ assertEquals(newFetchedKs.params.replication.klass, newKs.params.replication.klass);
+ assertFalse(newFetchedKs.params.replication.klass.equals(oldKs.params.replication.klass));
+ }
+
+ /*
+ @Test
+ public void testUpdateColumnFamilyNoIndexes() throws ConfigurationException
+ {
+ // create a keyspace with a cf to update.
+ CFMetaData cf = addTestTable("UpdatedCfKs", "Standard1added", "A new cf that will be updated");
+ KSMetaData ksm = KSMetaData.testMetadata(cf.ksName, SimpleStrategy.class, KSMetaData.optsWithRF(1), cf);
+ MigrationManager.announceNewKeyspace(ksm);
+
+ assertNotNull(Schema.instance.getKSMetaData(cf.ksName));
+ assertEquals(Schema.instance.getKSMetaData(cf.ksName), ksm);
+ assertNotNull(Schema.instance.getTableMetadataRef(cf.ksName, cf.cfName));
+
+ // updating certain fields should fail.
+ CFMetaData newCfm = cf.copy();
+ newCfm.defaultValidator(BytesType.instance);
+ newCfm.minCompactionThreshold(5);
+ newCfm.maxCompactionThreshold(31);
+
+ // test valid operations.
+ newCfm.comment("Modified comment");
+ MigrationManager.announceTableUpdate(newCfm); // doesn't get set back here.
+
+ newCfm.readRepairChance(0.23);
+ MigrationManager.announceTableUpdate(newCfm);
+
+ newCfm.gcGraceSeconds(12);
+ MigrationManager.announceTableUpdate(newCfm);
+
+ newCfm.defaultValidator(UTF8Type.instance);
+ MigrationManager.announceTableUpdate(newCfm);
+
+ newCfm.minCompactionThreshold(3);
+ MigrationManager.announceTableUpdate(newCfm);
+
+ newCfm.maxCompactionThreshold(33);
+ MigrationManager.announceTableUpdate(newCfm);
+
+ // can't test changing the reconciler because there is only one impl.
+
+ // check the cumulative affect.
+ assertEquals(Schema.instance.getTableMetadataRef(cf.ksName, cf.cfName).getComment(), newCfm.getComment());
+ assertEquals(Schema.instance.getTableMetadataRef(cf.ksName, cf.cfName).getReadRepairChance(), newCfm.getReadRepairChance(), 0.0001);
+ assertEquals(Schema.instance.getTableMetadataRef(cf.ksName, cf.cfName).getGcGraceSeconds(), newCfm.getGcGraceSeconds());
+ assertEquals(UTF8Type.instance, Schema.instance.getTableMetadataRef(cf.ksName, cf.cfName).getDefaultValidator());
+
+ // Change tableId
+ newCfm = new CFMetaData(cf.ksName, cf.cfName, cf.cfType, cf.comparator);
+ CFMetaData.copyOpts(newCfm, cf);
+ try
+ {
+ cf.apply(newCfm);
+ throw new AssertionError("Should have blown up when you used a different id.");
+ }
+ catch (ConfigurationException expected) {}
+
+ // Change cfName
+ newCfm = new CFMetaData(cf.ksName, cf.cfName + "_renamed", cf.cfType, cf.comparator);
+ CFMetaData.copyOpts(newCfm, cf);
+ try
+ {
+ cf.apply(newCfm);
+ throw new AssertionError("Should have blown up when you used a different name.");
+ }
+ catch (ConfigurationException expected) {}
+
+ // Change ksName
+ newCfm = new CFMetaData(cf.ksName + "_renamed", cf.cfName, cf.cfType, cf.comparator);
+ CFMetaData.copyOpts(newCfm, cf);
+ try
+ {
+ cf.apply(newCfm);
+ throw new AssertionError("Should have blown up when you used a different keyspace.");
+ }
+ catch (ConfigurationException expected) {}
+
+ // Change cf type
+ newCfm = new CFMetaData(cf.ksName, cf.cfName, ColumnFamilyType.Super, cf.comparator);
+ CFMetaData.copyOpts(newCfm, cf);
+ try
+ {
+ cf.apply(newCfm);
+ throw new AssertionError("Should have blwon up when you used a different cf type.");
+ }
+ catch (ConfigurationException expected) {}
+
+ // Change comparator
+ newCfm = new CFMetaData(cf.ksName, cf.cfName, cf.cfType, new SimpleDenseCellNameType(TimeUUIDType.instance));
+ CFMetaData.copyOpts(newCfm, cf);
+ try
+ {
+ cf.apply(newCfm);
+ throw new AssertionError("Should have blown up when you used a different comparator.");
+ }
+ catch (ConfigurationException expected) {}
+ }
+ */
+
+ @Test
+ public void testDropIndex() throws ConfigurationException
+ {
+ // persist keyspace definition in the system keyspace
+ SchemaKeyspace.makeCreateKeyspaceMutation(Schema.instance.getKeyspaceMetadata(KEYSPACE6), FBUtilities.timestampMicros()).build().applyUnsafe();
+ ColumnFamilyStore cfs = Keyspace.open(KEYSPACE6).getColumnFamilyStore(TABLE1i);
+ String indexName = TABLE1i + "_birthdate_key_index";
+
+ // insert some data. save the sstable descriptor so we can make sure it's marked for delete after the drop
+ QueryProcessor.executeInternal(String.format(
+ "INSERT INTO %s.%s (key, c1, birthdate, notbirthdate) VALUES (?, ?, ?, ?)",
+ KEYSPACE6,
+ TABLE1i),
+ "key0", "col0", 1L, 1L);
+
+ cfs.forceBlockingFlush();
+ ColumnFamilyStore indexCfs = cfs.indexManager.getIndexByName(indexName)
+ .getBackingTable()
+ .orElseThrow(throwAssert("Cannot access index cfs"));
+ Descriptor desc = indexCfs.getLiveSSTables().iterator().next().descriptor;
+
+ // drop the index
+ TableMetadata meta = cfs.metadata();
+ IndexMetadata existing = meta.indexes
+ .get(indexName)
+ .orElseThrow(throwAssert("Index not found"));
+
+ MigrationManager.announceTableUpdate(meta.unbuild().indexes(meta.indexes.without(existing.name)).build());
+
+ // check
+ assertTrue(cfs.indexManager.listIndexes().isEmpty());
+ LifecycleTransaction.waitForDeletions();
+ assertFalse(new File(desc.filenameFor(Component.DATA)).exists());
+ }
+
+ @Test
+ public void testValidateNullKeyspace() throws Exception
+ {
+ TableMetadata.Builder builder = TableMetadata.builder(null, TABLE1).addPartitionKeyColumn("partitionKey", BytesType.instance);
+
+ TableMetadata table1 = builder.build();
+ thrown.expect(ConfigurationException.class);
+ thrown.expectMessage(null + "." + TABLE1 + ": Keyspace name must not be empty");
+ table1.validate();
+ }
+
+ @Test
+ public void testValidateCompatibilityIDMismatch() throws Exception
+ {
+ TableMetadata.Builder builder = TableMetadata.builder(KEYSPACE1, TABLE1).addPartitionKeyColumn("partitionKey", BytesType.instance);
+
+ TableMetadata table1 = builder.build();
+ TableMetadata table2 = table1.unbuild().id(TableId.generate()).build();
+ thrown.expect(ConfigurationException.class);
+ thrown.expectMessage(KEYSPACE1 + "." + TABLE1 + ": Table ID mismatch");
+ table1.validateCompatibility(table2);
+ }
+
+ @Test
+ public void testValidateCompatibilityNameMismatch() throws Exception
+ {
+ TableMetadata.Builder builder1 = TableMetadata.builder(KEYSPACE1, TABLE1).addPartitionKeyColumn("partitionKey", BytesType.instance);
+ TableMetadata.Builder builder2 = TableMetadata.builder(KEYSPACE1, TABLE2).addPartitionKeyColumn("partitionKey", BytesType.instance);
+ TableMetadata table1 = builder1.build();
+ TableMetadata table2 = builder2.build();
+ thrown.expect(ConfigurationException.class);
+ thrown.expectMessage(KEYSPACE1 + "." + TABLE1 + ": Table mismatch");
+ table1.validateCompatibility(table2);
+ }
+
+ @Test
+ public void testEvolveSystemKeyspaceNew()
+ {
+ TableMetadata table = addTestTable("ks0", "t", "");
+ KeyspaceMetadata keyspace = KeyspaceMetadata.create("ks0", KeyspaceParams.simple(1), Tables.of(table));
+
+ Optional<Mutation> mutation = MigrationManager.evolveSystemKeyspace(keyspace, 0);
+ assertTrue(mutation.isPresent());
+
+ Schema.instance.merge(singleton(mutation.get()));
+ assertEquals(keyspace, Schema.instance.getKeyspaceMetadata("ks0"));
+ }
+
+ @Test
+ public void testEvolveSystemKeyspaceExistsUpToDate()
+ {
+ TableMetadata table = addTestTable("ks1", "t", "");
+ KeyspaceMetadata keyspace = KeyspaceMetadata.create("ks1", KeyspaceParams.simple(1), Tables.of(table));
+
+ // create the keyspace, verify it's there
+ Schema.instance.merge(singleton(SchemaKeyspace.makeCreateKeyspaceMutation(keyspace, 0).build()));
+ assertEquals(keyspace, Schema.instance.getKeyspaceMetadata("ks1"));
+
+ Optional<Mutation> mutation = MigrationManager.evolveSystemKeyspace(keyspace, 0);
+ assertFalse(mutation.isPresent());
+ }
+
+ @Test
+ public void testEvolveSystemKeyspaceChanged()
+ {
+ TableMetadata table0 = addTestTable("ks2", "t", "");
+ KeyspaceMetadata keyspace0 = KeyspaceMetadata.create("ks2", KeyspaceParams.simple(1), Tables.of(table0));
+
+ // create the keyspace, verify it's there
+ Schema.instance.merge(singleton(SchemaKeyspace.makeCreateKeyspaceMutation(keyspace0, 0).build()));
+ assertEquals(keyspace0, Schema.instance.getKeyspaceMetadata("ks2"));
+
+ TableMetadata table1 = table0.unbuild().comment("comment").build();
+ KeyspaceMetadata keyspace1 = KeyspaceMetadata.create("ks2", KeyspaceParams.simple(1), Tables.of(table1));
+
+ Optional<Mutation> mutation = MigrationManager.evolveSystemKeyspace(keyspace1, 1);
+ assertTrue(mutation.isPresent());
+
+ Schema.instance.merge(singleton(mutation.get()));
+ assertEquals(keyspace1, Schema.instance.getKeyspaceMetadata("ks2"));
+ }
+
+ private TableMetadata addTestTable(String ks, String cf, String comment)
+ {
+ return
+ TableMetadata.builder(ks, cf)
+ .addPartitionKeyColumn("key", UTF8Type.instance)
+ .addClusteringColumn("col", UTF8Type.instance)
+ .addRegularColumn("val", UTF8Type.instance)
+ .comment(comment)
+ .build();
+ }
+}
diff --git a/test/unit/org/apache/cassandra/schema/MockSchema.java b/test/unit/org/apache/cassandra/schema/MockSchema.java
new file mode 100644
index 0000000..40b0f87
--- /dev/null
+++ b/test/unit/org/apache/cassandra/schema/MockSchema.java
@@ -0,0 +1,220 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied. See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*/
+package org.apache.cassandra.schema;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.util.*;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Function;
+
+import com.google.common.collect.ImmutableSet;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.*;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.io.sstable.Component;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.IndexSummary;
+import org.apache.cassandra.io.sstable.format.SSTableFormat;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
+import org.apache.cassandra.io.sstable.metadata.MetadataType;
+import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
+import org.apache.cassandra.io.util.ChannelProxy;
+import org.apache.cassandra.io.util.FileHandle;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.io.util.Memory;
+import org.apache.cassandra.utils.AlwaysPresentFilter;
+import org.apache.cassandra.utils.ByteBufferUtil;
+
+import static org.apache.cassandra.service.ActiveRepairService.UNREPAIRED_SSTABLE;
+
+public class MockSchema
+{
+ static
+ {
+ Memory offsets = Memory.allocate(4);
+ offsets.setInt(0, 0);
+ indexSummary = new IndexSummary(Murmur3Partitioner.instance, offsets, 0, Memory.allocate(4), 0, 0, 0, 1);
+ }
+ private static final AtomicInteger id = new AtomicInteger();
+ public static final Keyspace ks = Keyspace.mockKS(KeyspaceMetadata.create("mockks", KeyspaceParams.simpleTransient(1)));
+
+ public static final IndexSummary indexSummary;
+
+ private static final File tempFile = temp("mocksegmentedfile");
+
+ public static Memtable memtable(ColumnFamilyStore cfs)
+ {
+ return new Memtable(cfs.metadata());
+ }
+
+ public static SSTableReader sstable(int generation, ColumnFamilyStore cfs)
+ {
+ return sstable(generation, false, cfs);
+ }
+
+ public static SSTableReader sstable(int generation, long first, long last, ColumnFamilyStore cfs)
+ {
+ return sstable(generation, 0, false, first, last, cfs);
+ }
+
+ public static SSTableReader sstable(int generation, boolean keepRef, ColumnFamilyStore cfs)
+ {
+ return sstable(generation, 0, keepRef, cfs);
+ }
+
+ public static SSTableReader sstable(int generation, int size, ColumnFamilyStore cfs)
+ {
+ return sstable(generation, size, false, cfs);
+ }
+ public static SSTableReader sstable(int generation, int size, boolean keepRef, ColumnFamilyStore cfs)
+ {
+ return sstable(generation, size, keepRef, generation, generation, cfs);
+ }
+
+ public static SSTableReader sstable(int generation, int size, boolean keepRef, long firstToken, long lastToken, ColumnFamilyStore cfs)
+ {
+ Descriptor descriptor = new Descriptor(cfs.getDirectories().getDirectoryForNewSSTables(),
+ cfs.keyspace.getName(),
+ cfs.getTableName(),
+ generation, SSTableFormat.Type.BIG);
+ Set<Component> components = ImmutableSet.of(Component.DATA, Component.PRIMARY_INDEX, Component.FILTER, Component.TOC);
+ for (Component component : components)
+ {
+ File file = new File(descriptor.filenameFor(component));
+ try
+ {
+ file.createNewFile();
+ }
+ catch (IOException e)
+ {
+ }
+ }
+ // .complete() with size to make sstable.onDiskLength work
+ @SuppressWarnings("resource")
+ FileHandle fileHandle = new FileHandle.Builder(new ChannelProxy(tempFile)).bufferSize(size).complete(size);
+ if (size > 0)
+ {
+ try
+ {
+ File file = new File(descriptor.filenameFor(Component.DATA));
+ try (RandomAccessFile raf = new RandomAccessFile(file, "rw"))
+ {
+ raf.setLength(size);
+ }
+ }
+ catch (IOException e)
+ {
+ throw new RuntimeException(e);
+ }
+ }
+ SerializationHeader header = SerializationHeader.make(cfs.metadata(), Collections.emptyList());
+ StatsMetadata metadata = (StatsMetadata) new MetadataCollector(cfs.metadata().comparator)
+ .finalizeMetadata(cfs.metadata().partitioner.getClass().getCanonicalName(), 0.01f, UNREPAIRED_SSTABLE, null, false, header)
+ .get(MetadataType.STATS);
+ SSTableReader reader = SSTableReader.internalOpen(descriptor, components, cfs.metadata,
+ fileHandle.sharedCopy(), fileHandle.sharedCopy(), indexSummary.sharedCopy(),
+ new AlwaysPresentFilter(), 1L, metadata, SSTableReader.OpenReason.NORMAL, header);
+ reader.first = readerBounds(firstToken);
+ reader.last = readerBounds(lastToken);
+ if (!keepRef)
+ reader.selfRef().release();
+ return reader;
+ }
+
+ public static ColumnFamilyStore newCFS()
+ {
+ return newCFS(ks.getName());
+ }
+
+ public static ColumnFamilyStore newCFS(String ksname)
+ {
+ return newCFS(newTableMetadata(ksname));
+ }
+
+ public static ColumnFamilyStore newCFS(Function<TableMetadata.Builder, TableMetadata.Builder> options)
+ {
+ return newCFS(ks.getName(), options);
+ }
+
+ public static ColumnFamilyStore newCFS(String ksname, Function<TableMetadata.Builder, TableMetadata.Builder> options)
+ {
+ return newCFS(options.apply(newTableMetadataBuilder(ksname)).build());
+ }
+
+ public static ColumnFamilyStore newCFS(TableMetadata metadata)
+ {
+ return new ColumnFamilyStore(ks, metadata.name, 0, new TableMetadataRef(metadata), new Directories(metadata), false, false, false);
+ }
+
+ public static TableMetadata newTableMetadata(String ksname)
+ {
+ return newTableMetadata(ksname, "mockcf" + (id.incrementAndGet()));
+ }
+
+ public static TableMetadata newTableMetadata(String ksname, String cfname)
+ {
+ return newTableMetadataBuilder(ksname, cfname).build();
+ }
+
+ public static TableMetadata.Builder newTableMetadataBuilder(String ksname)
+ {
+ return newTableMetadataBuilder(ksname, "mockcf" + (id.incrementAndGet()));
+ }
+
+ public static TableMetadata.Builder newTableMetadataBuilder(String ksname, String cfname)
+ {
+ return TableMetadata.builder(ksname, cfname)
+ .partitioner(Murmur3Partitioner.instance)
+ .addPartitionKeyColumn("key", UTF8Type.instance)
+ .addClusteringColumn("col", UTF8Type.instance)
+ .addRegularColumn("value", UTF8Type.instance)
+ .caching(CachingParams.CACHE_NOTHING);
+ }
+
+ public static BufferDecoratedKey readerBounds(long generation)
+ {
+ return new BufferDecoratedKey(new Murmur3Partitioner.LongToken(generation), ByteBufferUtil.EMPTY_BYTE_BUFFER);
+ }
+
+ private static File temp(String id)
+ {
+ File file = FileUtils.createTempFile(id, "tmp");
+ file.deleteOnExit();
+ return file;
+ }
+
+ public static void cleanup()
+ {
+ // clean up data directory which are stored as data directory/keyspace/data files
+ for (String dirName : DatabaseDescriptor.getAllDataFileLocations())
+ {
+ File dir = new File(dirName);
+ if (!dir.exists())
+ continue;
+ String[] children = dir.list();
+ for (String child : children)
+ FileUtils.deleteRecursive(new File(dir, child));
+ }
+ }
+}
diff --git a/test/unit/org/apache/cassandra/schema/SchemaKeyspaceTest.java b/test/unit/org/apache/cassandra/schema/SchemaKeyspaceTest.java
index f3ee85d..bf62203 100644
--- a/test/unit/org/apache/cassandra/schema/SchemaKeyspaceTest.java
+++ b/test/unit/org/apache/cassandra/schema/SchemaKeyspaceTest.java
@@ -20,49 +20,31 @@
import java.io.IOException;
import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
-import java.util.List;
import java.util.Set;
-import java.util.UUID;
import com.google.common.collect.ImmutableMap;
+import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
import org.apache.cassandra.SchemaLoader;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.config.Schema;
-import org.apache.cassandra.config.SchemaConstants;
import org.apache.cassandra.cql3.QueryProcessor;
import org.apache.cassandra.cql3.UntypedResultSet;
+import org.apache.cassandra.cql3.statements.schema.CreateTableStatement;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.db.Mutation;
-import org.apache.cassandra.db.marshal.AsciiType;
-import org.apache.cassandra.db.marshal.UTF8Type;
import org.apache.cassandra.db.partitions.PartitionUpdate;
import org.apache.cassandra.db.rows.UnfilteredRowIterators;
import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.io.util.DataInputBuffer;
-import org.apache.cassandra.io.util.DataOutputBuffer;
-import org.apache.cassandra.net.MessagingService;
-import org.apache.cassandra.thrift.CfDef;
-import org.apache.cassandra.thrift.ColumnDef;
-import org.apache.cassandra.thrift.IndexType;
-import org.apache.cassandra.thrift.ThriftConversion;
-import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.service.reads.repair.ReadRepairStrategy;
import org.apache.cassandra.utils.FBUtilities;
-import org.apache.cassandra.utils.Pair;
import static org.apache.cassandra.cql3.QueryProcessor.executeOnceInternal;
import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
public class SchemaKeyspaceTest
@@ -70,19 +52,6 @@
private static final String KEYSPACE1 = "CFMetaDataTest1";
private static final String CF_STANDARD1 = "Standard1";
- private static final List<ColumnDef> columnDefs = new ArrayList<>();
-
- static
- {
- columnDefs.add(new ColumnDef(ByteBufferUtil.bytes("col1"), AsciiType.class.getCanonicalName())
- .setIndex_name("col1Index")
- .setIndex_type(IndexType.KEYS));
-
- columnDefs.add(new ColumnDef(ByteBufferUtil.bytes("col2"), UTF8Type.class.getCanonicalName())
- .setIndex_name("col2Index")
- .setIndex_type(IndexType.KEYS));
- }
-
@BeforeClass
public static void defineSchema() throws ConfigurationException
{
@@ -93,58 +62,16 @@
}
@Test
- public void testThriftConversion() throws Exception
- {
- CfDef cfDef = new CfDef().setDefault_validation_class(AsciiType.class.getCanonicalName())
- .setComment("Test comment")
- .setColumn_metadata(columnDefs)
- .setKeyspace(KEYSPACE1)
- .setName(CF_STANDARD1);
-
- // convert Thrift to CFMetaData
- CFMetaData cfMetaData = ThriftConversion.fromThrift(cfDef);
-
- CfDef thriftCfDef = new CfDef();
- thriftCfDef.keyspace = KEYSPACE1;
- thriftCfDef.name = CF_STANDARD1;
- thriftCfDef.default_validation_class = cfDef.default_validation_class;
- thriftCfDef.comment = cfDef.comment;
- thriftCfDef.column_metadata = new ArrayList<>();
- for (ColumnDef columnDef : columnDefs)
- {
- ColumnDef c = new ColumnDef();
- c.name = ByteBufferUtil.clone(columnDef.name);
- c.validation_class = columnDef.getValidation_class();
- c.index_name = columnDef.getIndex_name();
- c.index_type = IndexType.KEYS;
- thriftCfDef.column_metadata.add(c);
- }
-
- CfDef converted = ThriftConversion.toThrift(cfMetaData);
-
- assertEquals(thriftCfDef.keyspace, converted.keyspace);
- assertEquals(thriftCfDef.name, converted.name);
- assertEquals(thriftCfDef.default_validation_class, converted.default_validation_class);
- assertEquals(thriftCfDef.comment, converted.comment);
- assertEquals(new HashSet<>(thriftCfDef.column_metadata), new HashSet<>(converted.column_metadata));
- }
-
- @Test
public void testConversionsInverses() throws Exception
{
for (String keyspaceName : Schema.instance.getNonSystemKeyspaces())
{
for (ColumnFamilyStore cfs : Keyspace.open(keyspaceName).getColumnFamilyStores())
{
- CFMetaData cfm = cfs.metadata;
- if (!cfm.isThriftCompatible())
- continue;
-
- checkInverses(cfm);
+ checkInverses(cfs.metadata());
// Testing with compression to catch #3558
- CFMetaData withCompression = cfm.copy();
- withCompression.compression(CompressionParams.snappy(32768));
+ TableMetadata withCompression = cfs.metadata().unbuild().compression(CompressionParams.snappy(32768)).build();
checkInverses(withCompression);
}
}
@@ -157,49 +84,53 @@
createTable(keyspace, "CREATE TABLE test (a text primary key, b int, c int)");
- CFMetaData metadata = Schema.instance.getCFMetaData(keyspace, "test");
+ TableMetadata metadata = Schema.instance.getTableMetadata(keyspace, "test");
assertTrue("extensions should be empty", metadata.params.extensions.isEmpty());
ImmutableMap<String, ByteBuffer> extensions = ImmutableMap.of("From ... with Love",
ByteBuffer.wrap(new byte[]{0, 0, 7}));
- CFMetaData copy = metadata.copy().extensions(extensions);
+ TableMetadata copy = metadata.unbuild().extensions(extensions).build();
updateTable(keyspace, metadata, copy);
- metadata = Schema.instance.getCFMetaData(keyspace, "test");
+ metadata = Schema.instance.getTableMetadata(keyspace, "test");
assertEquals(extensions, metadata.params.extensions);
}
- private static void updateTable(String keyspace, CFMetaData oldTable, CFMetaData newTable)
+ @Test
+ public void testReadRepair()
+ {
+ createTable("ks", "CREATE TABLE tbl (a text primary key, b int, c int) WITH read_repair='none'");
+ TableMetadata metadata = Schema.instance.getTableMetadata("ks", "tbl");
+ Assert.assertEquals(ReadRepairStrategy.NONE, metadata.params.readRepair);
+
+ }
+
+ private static void updateTable(String keyspace, TableMetadata oldTable, TableMetadata newTable)
{
KeyspaceMetadata ksm = Schema.instance.getKeyspaceInstance(keyspace).getMetadata();
Mutation mutation = SchemaKeyspace.makeUpdateTableMutation(ksm, oldTable, newTable, FBUtilities.timestampMicros()).build();
- SchemaKeyspace.mergeSchema(Collections.singleton(mutation));
+ Schema.instance.merge(Collections.singleton(mutation));
}
private static void createTable(String keyspace, String cql)
{
- CFMetaData table = CFMetaData.compile(cql, keyspace);
+ TableMetadata table = CreateTableStatement.parse(cql, keyspace).build();
KeyspaceMetadata ksm = KeyspaceMetadata.create(keyspace, KeyspaceParams.simple(1), Tables.of(table));
Mutation mutation = SchemaKeyspace.makeCreateTableMutation(ksm, table, FBUtilities.timestampMicros()).build();
- SchemaKeyspace.mergeSchema(Collections.singleton(mutation));
+ Schema.instance.merge(Collections.singleton(mutation));
}
- private static void checkInverses(CFMetaData cfm) throws Exception
+ private static void checkInverses(TableMetadata metadata) throws Exception
{
- KeyspaceMetadata keyspace = Schema.instance.getKSMetaData(cfm.ksName);
-
- // Test thrift conversion
- CFMetaData before = cfm;
- CFMetaData after = ThriftConversion.fromThriftForUpdate(ThriftConversion.toThrift(before), before);
- assert before.equals(after) : String.format("%n%s%n!=%n%s", before, after);
+ KeyspaceMetadata keyspace = Schema.instance.getKeyspaceMetadata(metadata.keyspace);
// Test schema conversion
- Mutation rm = SchemaKeyspace.makeCreateTableMutation(keyspace, cfm, FBUtilities.timestampMicros()).build();
- PartitionUpdate serializedCf = rm.getPartitionUpdate(Schema.instance.getId(SchemaConstants.SCHEMA_KEYSPACE_NAME, SchemaKeyspace.TABLES));
- PartitionUpdate serializedCD = rm.getPartitionUpdate(Schema.instance.getId(SchemaConstants.SCHEMA_KEYSPACE_NAME, SchemaKeyspace.COLUMNS));
+ Mutation rm = SchemaKeyspace.makeCreateTableMutation(keyspace, metadata, FBUtilities.timestampMicros()).build();
+ PartitionUpdate serializedCf = rm.getPartitionUpdate(Schema.instance.getTableMetadata(SchemaConstants.SCHEMA_KEYSPACE_NAME, SchemaKeyspace.TABLES));
+ PartitionUpdate serializedCD = rm.getPartitionUpdate(Schema.instance.getTableMetadata(SchemaConstants.SCHEMA_KEYSPACE_NAME, SchemaKeyspace.COLUMNS));
UntypedResultSet.Row tableRow = QueryProcessor.resultify(String.format("SELECT * FROM %s.%s", SchemaConstants.SCHEMA_KEYSPACE_NAME, SchemaKeyspace.TABLES),
UnfilteredRowIterators.filter(serializedCf.unfilteredIterator(), FBUtilities.nowInSeconds()))
@@ -208,106 +139,12 @@
UntypedResultSet columnsRows = QueryProcessor.resultify(String.format("SELECT * FROM %s.%s", SchemaConstants.SCHEMA_KEYSPACE_NAME, SchemaKeyspace.COLUMNS),
UnfilteredRowIterators.filter(serializedCD.unfilteredIterator(), FBUtilities.nowInSeconds()));
- Set<ColumnDefinition> columns = new HashSet<>();
+ Set<ColumnMetadata> columns = new HashSet<>();
for (UntypedResultSet.Row row : columnsRows)
columns.add(SchemaKeyspace.createColumnFromRow(row, Types.none()));
- assertEquals(cfm.params, params);
- assertEquals(new HashSet<>(cfm.allColumns()), columns);
- }
-
- private static boolean hasCDC(Mutation m)
- {
- for (PartitionUpdate p : m.getPartitionUpdates())
- {
- for (ColumnDefinition cd : p.columns())
- {
- if (cd.name.toString().equals("cdc"))
- return true;
- }
- }
- return false;
- }
-
- private static boolean hasSchemaTables(Mutation m)
- {
- for (PartitionUpdate p : m.getPartitionUpdates())
- {
- if (p.metadata().cfName.equals(SchemaKeyspace.TABLES))
- return true;
- }
- return false;
- }
-
- @Test
- public void testConvertSchemaToMutationsWithoutCDC() throws IOException
- {
- boolean oldCDCOption = DatabaseDescriptor.isCDCEnabled();
- try
- {
- DatabaseDescriptor.setCDCEnabled(false);
- Collection<Mutation> mutations = SchemaKeyspace.convertSchemaToMutations();
- boolean foundTables = false;
- for (Mutation m : mutations)
- {
- if (hasSchemaTables(m))
- {
- foundTables = true;
- assertFalse(hasCDC(m));
- try (DataOutputBuffer output = new DataOutputBuffer())
- {
- Mutation.serializer.serialize(m, output, MessagingService.current_version);
- try (DataInputBuffer input = new DataInputBuffer(output.getData()))
- {
- Mutation out = Mutation.serializer.deserialize(input, MessagingService.current_version);
- assertFalse(hasCDC(out));
- }
- }
- }
- }
- assertTrue(foundTables);
- }
- finally
- {
- DatabaseDescriptor.setCDCEnabled(oldCDCOption);
- }
- }
-
- @Test
- public void testConvertSchemaToMutationsWithCDC()
- {
- boolean oldCDCOption = DatabaseDescriptor.isCDCEnabled();
- try
- {
- DatabaseDescriptor.setCDCEnabled(true);
- Collection<Mutation> mutations = SchemaKeyspace.convertSchemaToMutations();
- boolean foundTables = false;
- for (Mutation m : mutations)
- {
- if (hasSchemaTables(m))
- {
- foundTables = true;
- assertTrue(hasCDC(m));
- }
- }
- assertTrue(foundTables);
- }
- finally
- {
- DatabaseDescriptor.setCDCEnabled(oldCDCOption);
- }
- }
-
- @Test
- public void testSchemaDigest()
- {
- Set<ByteBuffer> abc = Collections.singleton(ByteBufferUtil.bytes("abc"));
- Pair<UUID, UUID> versions = SchemaKeyspace.calculateSchemaDigest(abc);
- assertTrue(versions.left.equals(versions.right));
-
- Set<ByteBuffer> cdc = Collections.singleton(ByteBufferUtil.bytes("cdc"));
- versions = SchemaKeyspace.calculateSchemaDigest(cdc);
- assertFalse(versions.left.equals(versions.right));
+ assertEquals(metadata.params, params);
+ assertEquals(new HashSet<>(metadata.columns()), columns);
}
@Test(expected = SchemaKeyspace.MissingColumns.class)
diff --git a/test/unit/org/apache/cassandra/schema/SchemaTest.java b/test/unit/org/apache/cassandra/schema/SchemaTest.java
new file mode 100644
index 0000000..64b1341
--- /dev/null
+++ b/test/unit/org/apache/cassandra/schema/SchemaTest.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.cassandra.schema;
+
+import java.io.IOException;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.commitlog.CommitLog;
+import org.apache.cassandra.gms.Gossiper;
+
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+public class SchemaTest
+{
+ @BeforeClass
+ public static void setupDatabaseDescriptor()
+ {
+ DatabaseDescriptor.daemonInitialization();
+ }
+
+ @Test
+ public void testTransKsMigration() throws IOException
+ {
+ CommitLog.instance.start();
+ SchemaLoader.cleanupAndLeaveDirs();
+ Schema.instance.loadFromDisk();
+ assertEquals(0, Schema.instance.getNonSystemKeyspaces().size());
+
+ Gossiper.instance.start((int)(System.currentTimeMillis() / 1000));
+ Keyspace.setInitialized();
+
+ try
+ {
+ // add a few.
+ MigrationManager.announceNewKeyspace(KeyspaceMetadata.create("ks0", KeyspaceParams.simple(3)));
+ MigrationManager.announceNewKeyspace(KeyspaceMetadata.create("ks1", KeyspaceParams.simple(3)));
+
+ assertNotNull(Schema.instance.getKeyspaceMetadata("ks0"));
+ assertNotNull(Schema.instance.getKeyspaceMetadata("ks1"));
+
+ Schema.instance.unload(Schema.instance.getKeyspaceMetadata("ks0"));
+ Schema.instance.unload(Schema.instance.getKeyspaceMetadata("ks1"));
+
+ assertNull(Schema.instance.getKeyspaceMetadata("ks0"));
+ assertNull(Schema.instance.getKeyspaceMetadata("ks1"));
+
+ Schema.instance.loadFromDisk();
+
+ assertNotNull(Schema.instance.getKeyspaceMetadata("ks0"));
+ assertNotNull(Schema.instance.getKeyspaceMetadata("ks1"));
+ }
+ finally
+ {
+ Gossiper.instance.stop();
+ }
+ }
+
+}
diff --git a/test/unit/org/apache/cassandra/schema/TupleTypesRepresentationTest.java b/test/unit/org/apache/cassandra/schema/TupleTypesRepresentationTest.java
index 36cba3c..e6daa1f 100644
--- a/test/unit/org/apache/cassandra/schema/TupleTypesRepresentationTest.java
+++ b/test/unit/org/apache/cassandra/schema/TupleTypesRepresentationTest.java
@@ -49,7 +49,6 @@
static
{
DatabaseDescriptor.toolInitialization();
- DatabaseDescriptor.applyAddressConfig();
}
private static final String keyspace = "ks";
@@ -378,7 +377,7 @@
{
assertEquals(typeDef.toString() + "\n typeString vs type\n", typeDef.typeString, typeDef.type.toString());
assertEquals(typeDef.toString() + "\n typeString vs cqlType.getType()\n", typeDef.typeString, typeDef.cqlType.getType().toString());
- AbstractType<?> expanded = SchemaKeyspace.expandUserTypes(typeDef.type);
+ AbstractType<?> expanded = typeDef.type.expandUserTypes();
CQL3Type expandedCQL = expanded.asCQL3Type();
// Note: cannot include this commented-out assertion, because the parsed CQL3Type instance for
// 'frozen<list<tuple<text, text>>>' returns 'frozen<list<frozen<tuple<text, text>>>>' via it's CQL3Type.toString()
diff --git a/test/unit/org/apache/cassandra/schema/ValidationTest.java b/test/unit/org/apache/cassandra/schema/ValidationTest.java
new file mode 100644
index 0000000..8eb1247
--- /dev/null
+++ b/test/unit/org/apache/cassandra/schema/ValidationTest.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.cassandra.schema;
+
+import java.util.*;
+
+import org.apache.cassandra.db.marshal.*;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class ValidationTest
+{
+ @Test
+ public void testIsNameValidPositive()
+ {
+ assertTrue(SchemaConstants.isValidName("abcdefghijklmnopqrstuvwxyz"));
+ assertTrue(SchemaConstants.isValidName("ABCDEFGHIJKLMNOPQRSTUVWXYZ"));
+ assertTrue(SchemaConstants.isValidName("_01234567890"));
+ }
+
+ @Test
+ public void testIsNameValidNegative()
+ {
+ assertFalse(SchemaConstants.isValidName(null));
+ assertFalse(SchemaConstants.isValidName(""));
+ assertFalse(SchemaConstants.isValidName(" "));
+ assertFalse(SchemaConstants.isValidName("@"));
+ assertFalse(SchemaConstants.isValidName("!"));
+ }
+
+ private static Set<String> primitiveTypes =
+ new HashSet<>(Arrays.asList(new String[] { "ascii", "bigint", "blob", "boolean", "date",
+ "duration", "decimal", "double", "float",
+ "inet", "int", "smallint", "text", "time",
+ "timestamp", "timeuuid", "tinyint", "uuid",
+ "varchar", "varint" }));
+
+ @Test
+ public void typeCompatibilityTest()
+ {
+ Map<String, Set<String>> compatibilityMap = new HashMap<>();
+ compatibilityMap.put("bigint", new HashSet<>(Arrays.asList(new String[] {"timestamp"})));
+ compatibilityMap.put("blob", new HashSet<>(Arrays.asList(new String[] {"ascii", "bigint", "boolean", "date", "decimal", "double", "duration",
+ "float", "inet", "int", "smallint", "text", "time", "timestamp",
+ "timeuuid", "tinyint", "uuid", "varchar", "varint"})));
+ compatibilityMap.put("date", new HashSet<>(Arrays.asList(new String[] {"int"})));
+ compatibilityMap.put("time", new HashSet<>(Arrays.asList(new String[] {"bigint"})));
+ compatibilityMap.put("text", new HashSet<>(Arrays.asList(new String[] {"ascii", "varchar"})));
+ compatibilityMap.put("timestamp", new HashSet<>(Arrays.asList(new String[] {"bigint"})));
+ compatibilityMap.put("varchar", new HashSet<>(Arrays.asList(new String[] {"ascii", "text"})));
+ compatibilityMap.put("varint", new HashSet<>(Arrays.asList(new String[] {"bigint", "int", "timestamp"})));
+ compatibilityMap.put("uuid", new HashSet<>(Arrays.asList(new String[] {"timeuuid"})));
+
+ for (String sourceTypeString: primitiveTypes)
+ {
+ AbstractType sourceType = CQLTypeParser.parse("KEYSPACE", sourceTypeString, Types.none());
+ for (String destinationTypeString: primitiveTypes)
+ {
+ AbstractType destinationType = CQLTypeParser.parse("KEYSPACE", destinationTypeString, Types.none());
+
+ if (compatibilityMap.get(destinationTypeString) != null &&
+ compatibilityMap.get(destinationTypeString).contains(sourceTypeString) ||
+ sourceTypeString.equals(destinationTypeString))
+ {
+ assertTrue(sourceTypeString + " should be compatible with " + destinationTypeString,
+ destinationType.isValueCompatibleWith(sourceType));
+ }
+ else
+ {
+ assertFalse(sourceTypeString + " should not be compatible with " + destinationTypeString,
+ destinationType.isValueCompatibleWith(sourceType));
+ }
+ }
+ }
+ }
+
+ @Test
+ public void clusteringColumnTypeCompatibilityTest() throws Throwable
+ {
+ Map<String, Set<String>> compatibilityMap = new HashMap<>();
+ compatibilityMap.put("blob", new HashSet<>(Arrays.asList(new String[] {"ascii", "text", "varchar"})));
+ compatibilityMap.put("text", new HashSet<>(Arrays.asList(new String[] {"ascii", "varchar"})));
+ compatibilityMap.put("varchar", new HashSet<>(Arrays.asList(new String[] {"ascii", "text" })));
+
+ for (String sourceTypeString: primitiveTypes)
+ {
+ AbstractType sourceType = CQLTypeParser.parse("KEYSPACE", sourceTypeString, Types.none());
+ for (String destinationTypeString: primitiveTypes)
+ {
+ AbstractType destinationType = CQLTypeParser.parse("KEYSPACE", destinationTypeString, Types.none());
+
+ if (compatibilityMap.get(destinationTypeString) != null &&
+ compatibilityMap.get(destinationTypeString).contains(sourceTypeString) ||
+ sourceTypeString.equals(destinationTypeString))
+ {
+ assertTrue(sourceTypeString + " should be compatible with " + destinationTypeString,
+ destinationType.isCompatibleWith(sourceType));
+ }
+ else
+ {
+ assertFalse(sourceTypeString + " should not be compatible with " + destinationTypeString,
+ destinationType.isCompatibleWith(sourceType));
+ }
+ }
+ }
+ }
+}
diff --git a/test/unit/org/apache/cassandra/security/EncryptionUtilsTest.java b/test/unit/org/apache/cassandra/security/EncryptionUtilsTest.java
index 70b327e..0fd46b8 100644
--- a/test/unit/org/apache/cassandra/security/EncryptionUtilsTest.java
+++ b/test/unit/org/apache/cassandra/security/EncryptionUtilsTest.java
@@ -38,6 +38,7 @@
import org.apache.cassandra.config.TransparentDataEncryptionOptions;
import org.apache.cassandra.io.compress.ICompressor;
import org.apache.cassandra.io.compress.LZ4Compressor;
+import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.io.util.RandomAccessReader;
public class EncryptionUtilsTest
@@ -79,7 +80,7 @@
CipherFactory cipherFactory = new CipherFactory(tdeOptions);
Cipher encryptor = cipherFactory.getEncryptor(tdeOptions.cipher, tdeOptions.key_alias);
- File f = File.createTempFile("commitlog-enc-utils-", ".tmp");
+ File f = FileUtils.createTempFile("commitlog-enc-utils-", ".tmp");
f.deleteOnExit();
FileChannel channel = new RandomAccessFile(f, "rw").getChannel();
EncryptionUtils.encryptAndWrite(ByteBuffer.wrap(buf), channel, true, encryptor);
@@ -108,7 +109,7 @@
// encrypt
CipherFactory cipherFactory = new CipherFactory(tdeOptions);
Cipher encryptor = cipherFactory.getEncryptor(tdeOptions.cipher, tdeOptions.key_alias);
- File f = File.createTempFile("commitlog-enc-utils-", ".tmp");
+ File f = FileUtils.createTempFile("commitlog-enc-utils-", ".tmp");
f.deleteOnExit();
FileChannel channel = new RandomAccessFile(f, "rw").getChannel();
EncryptionUtils.encryptAndWrite(compressedBuffer, channel, true, encryptor);
diff --git a/test/unit/org/apache/cassandra/security/SSLFactoryTest.java b/test/unit/org/apache/cassandra/security/SSLFactoryTest.java
index ec0c810..307a276 100644
--- a/test/unit/org/apache/cassandra/security/SSLFactoryTest.java
+++ b/test/unit/org/apache/cassandra/security/SSLFactoryTest.java
@@ -18,29 +18,59 @@
*/
package org.apache.cassandra.security;
-import static org.junit.Assert.assertArrayEquals;
-
+import java.io.File;
import java.io.IOException;
+import java.security.cert.CertificateException;
+import javax.net.ssl.TrustManagerFactory;
-import javax.net.ssl.SSLServerSocket;
+import org.apache.commons.io.FileUtils;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-import com.google.common.base.Predicates;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Lists;
-
+import io.netty.handler.ssl.JdkSslContext;
+import io.netty.handler.ssl.OpenSsl;
+import io.netty.handler.ssl.OpenSslContext;
+import io.netty.handler.ssl.SslContext;
+import io.netty.handler.ssl.util.SelfSignedCertificate;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.config.EncryptionOptions;
import org.apache.cassandra.config.EncryptionOptions.ServerEncryptionOptions;
-import org.apache.cassandra.utils.FBUtilities;
-import org.junit.BeforeClass;
-import org.junit.Test;
+
+import static org.junit.Assert.assertArrayEquals;
public class SSLFactoryTest
{
- @BeforeClass
- public static void beforeClass()
+ private static final Logger logger = LoggerFactory.getLogger(SSLFactoryTest.class);
+
+ static final SelfSignedCertificate ssc;
+ static
{
DatabaseDescriptor.daemonInitialization();
+ try
+ {
+ ssc = new SelfSignedCertificate();
+ }
+ catch (CertificateException e)
+ {
+ throw new RuntimeException("fialed to create test certs");
+ }
+ }
+
+ private ServerEncryptionOptions encryptionOptions;
+
+ @Before
+ public void setup()
+ {
+ encryptionOptions = new ServerEncryptionOptions()
+ .withTrustStore("test/conf/cassandra_ssl_test.truststore")
+ .withTrustStorePassword("cassandra")
+ .withRequireClientAuth(false)
+ .withCipherSuites("TLS_RSA_WITH_AES_128_CBC_SHA");
+
+ SSLFactory.checkedExpiry = false;
}
@Test
@@ -55,28 +85,210 @@
}
@Test
- public void testServerSocketCiphers() throws IOException
+ public void getSslContext_OpenSSL() throws IOException
{
- ServerEncryptionOptions options = new EncryptionOptions.ServerEncryptionOptions();
- options.keystore = "test/conf/keystore.jks";
- options.keystore_password = "cassandra";
- options.truststore = options.keystore;
- options.truststore_password = options.keystore_password;
- options.cipher_suites = new String[] {
- "TLS_RSA_WITH_AES_128_CBC_SHA", "TLS_RSA_WITH_AES_256_CBC_SHA",
- "TLS_DHE_RSA_WITH_AES_128_CBC_SHA", "TLS_DHE_RSA_WITH_AES_256_CBC_SHA",
- "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA", "TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA"
- };
-
- // enabled ciphers must be a subset of configured ciphers with identical order
- try (SSLServerSocket socket = SSLFactory.getServerSocket(options, FBUtilities.getLocalAddress(), 55123))
+ // only try this test if OpenSsl is available
+ if (!OpenSsl.isAvailable())
{
- String[] enabled = socket.getEnabledCipherSuites();
- String[] wanted = Iterables.toArray(Iterables.filter(Lists.newArrayList(options.cipher_suites),
- Predicates.in(Lists.newArrayList(enabled))),
- String.class);
- assertArrayEquals(wanted, enabled);
+ logger.warn("OpenSSL not available in this application, so not testing the netty-openssl code paths");
+ return;
+ }
+
+ EncryptionOptions options = addKeystoreOptions(encryptionOptions);
+ SslContext sslContext = SSLFactory.getOrCreateSslContext(options, true, SSLFactory.SocketType.CLIENT, true);
+ Assert.assertNotNull(sslContext);
+ Assert.assertTrue(sslContext instanceof OpenSslContext);
+ }
+
+ @Test
+ public void getSslContext_JdkSsl() throws IOException
+ {
+ EncryptionOptions options = addKeystoreOptions(encryptionOptions);
+ SslContext sslContext = SSLFactory.getOrCreateSslContext(options, true, SSLFactory.SocketType.CLIENT, false);
+ Assert.assertNotNull(sslContext);
+ Assert.assertTrue(sslContext instanceof JdkSslContext);
+ Assert.assertEquals(encryptionOptions.cipher_suites, sslContext.cipherSuites());
+ }
+
+ private ServerEncryptionOptions addKeystoreOptions(ServerEncryptionOptions options)
+ {
+ return options.withKeyStore("test/conf/cassandra_ssl_test.keystore")
+ .withKeyStorePassword("cassandra");
+ }
+
+ @Test(expected = IOException.class)
+ public void buildTrustManagerFactory_NoFile() throws IOException
+ {
+ SSLFactory.buildTrustManagerFactory(encryptionOptions.withTrustStore("/this/is/probably/not/a/file/on/your/test/machine"));
+ }
+
+ @Test(expected = IOException.class)
+ public void buildTrustManagerFactory_BadPassword() throws IOException
+ {
+ SSLFactory.buildTrustManagerFactory(encryptionOptions.withTrustStorePassword("HomeOfBadPasswords"));
+ }
+
+ @Test
+ public void buildTrustManagerFactory_HappyPath() throws IOException
+ {
+ TrustManagerFactory trustManagerFactory = SSLFactory.buildTrustManagerFactory(encryptionOptions);
+ Assert.assertNotNull(trustManagerFactory);
+ }
+
+ @Test(expected = IOException.class)
+ public void buildKeyManagerFactory_NoFile() throws IOException
+ {
+ EncryptionOptions options = addKeystoreOptions(encryptionOptions)
+ .withKeyStore("/this/is/probably/not/a/file/on/your/test/machine");
+ SSLFactory.buildKeyManagerFactory(options);
+ }
+
+ @Test(expected = IOException.class)
+ public void buildKeyManagerFactory_BadPassword() throws IOException
+ {
+ EncryptionOptions options = addKeystoreOptions(encryptionOptions)
+ .withKeyStorePassword("HomeOfBadPasswords");
+ SSLFactory.buildKeyManagerFactory(options);
+ }
+
+ @Test
+ public void buildKeyManagerFactory_HappyPath() throws IOException
+ {
+ Assert.assertFalse(SSLFactory.checkedExpiry);
+ EncryptionOptions options = addKeystoreOptions(encryptionOptions);
+ SSLFactory.buildKeyManagerFactory(options);
+ Assert.assertTrue(SSLFactory.checkedExpiry);
+ }
+
+ @Test
+ public void testSslContextReload_HappyPath() throws IOException, InterruptedException
+ {
+ try
+ {
+ ServerEncryptionOptions options = addKeystoreOptions(encryptionOptions)
+ .withInternodeEncryption(ServerEncryptionOptions.InternodeEncryption.all);
+
+ SSLFactory.initHotReloading(options, options, true);
+
+ SslContext oldCtx = SSLFactory.getOrCreateSslContext(options, true, SSLFactory.SocketType.CLIENT, OpenSsl
+ .isAvailable());
+ File keystoreFile = new File(options.keystore);
+
+ SSLFactory.checkCertFilesForHotReloading((ServerEncryptionOptions) options, options);
+
+ keystoreFile.setLastModified(System.currentTimeMillis() + 15000);
+
+ SSLFactory.checkCertFilesForHotReloading((ServerEncryptionOptions) options, options);;
+ SslContext newCtx = SSLFactory.getOrCreateSslContext(options, true, SSLFactory.SocketType.CLIENT, OpenSsl
+ .isAvailable());
+
+ Assert.assertNotSame(oldCtx, newCtx);
+ }
+ catch (Exception e)
+ {
+ throw e;
+ }
+ finally
+ {
+ DatabaseDescriptor.loadConfig();
}
}
+ @Test(expected = IOException.class)
+ public void testSslFactorySslInit_BadPassword_ThrowsException() throws IOException
+ {
+ ServerEncryptionOptions options = addKeystoreOptions(encryptionOptions)
+ .withKeyStorePassword("bad password")
+ .withInternodeEncryption(ServerEncryptionOptions.InternodeEncryption.all);
+
+ SSLFactory.initHotReloading(options, options, true);
+ }
+
+ @Test
+ public void testSslFactoryHotReload_BadPassword_DoesNotClearExistingSslContext() throws IOException
+ {
+ try
+ {
+ ServerEncryptionOptions options = addKeystoreOptions(encryptionOptions);
+
+ SSLFactory.initHotReloading(options, options, true);
+ SslContext oldCtx = SSLFactory.getOrCreateSslContext(options, true, SSLFactory.SocketType.CLIENT, OpenSsl
+ .isAvailable());
+ File keystoreFile = new File(options.keystore);
+
+ SSLFactory.checkCertFilesForHotReloading(options, options);
+ keystoreFile.setLastModified(System.currentTimeMillis() + 5000);
+
+ ServerEncryptionOptions modOptions = new ServerEncryptionOptions(options)
+ .withKeyStorePassword("bad password");
+ SSLFactory.checkCertFilesForHotReloading(modOptions, modOptions);
+ SslContext newCtx = SSLFactory.getOrCreateSslContext(options, true, SSLFactory.SocketType.CLIENT, OpenSsl
+ .isAvailable());
+
+ Assert.assertSame(oldCtx, newCtx);
+ }
+ finally
+ {
+ DatabaseDescriptor.loadConfig();
+ }
+ }
+
+ @Test
+ public void testSslFactoryHotReload_CorruptOrNonExistentFile_DoesNotClearExistingSslContext() throws IOException
+ {
+ try
+ {
+ ServerEncryptionOptions options = addKeystoreOptions(encryptionOptions);
+
+ File testKeystoreFile = new File(options.keystore + ".test");
+ FileUtils.copyFile(new File(options.keystore),testKeystoreFile);
+ options = options.withKeyStore(testKeystoreFile.getPath());
+
+
+ SSLFactory.initHotReloading(options, options, true);
+ SslContext oldCtx = SSLFactory.getOrCreateSslContext(options, true, SSLFactory.SocketType.CLIENT, OpenSsl
+ .isAvailable());
+ SSLFactory.checkCertFilesForHotReloading(options, options);
+
+ testKeystoreFile.setLastModified(System.currentTimeMillis() + 15000);
+ FileUtils.forceDelete(testKeystoreFile);
+
+ SSLFactory.checkCertFilesForHotReloading(options, options);;
+ SslContext newCtx = SSLFactory.getOrCreateSslContext(options, true, SSLFactory.SocketType.CLIENT, OpenSsl
+ .isAvailable());
+
+ Assert.assertSame(oldCtx, newCtx);
+ }
+ catch (Exception e)
+ {
+ throw e;
+ }
+ finally
+ {
+ DatabaseDescriptor.loadConfig();
+ FileUtils.deleteQuietly(new File(encryptionOptions.keystore + ".test"));
+ }
+ }
+
+ @Test
+ public void getSslContext_ParamChanges() throws IOException
+ {
+ EncryptionOptions options = addKeystoreOptions(encryptionOptions)
+ .withEnabled(true)
+ .withCipherSuites("TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256");
+
+ SslContext ctx1 = SSLFactory.getOrCreateSslContext(options, true,
+ SSLFactory.SocketType.SERVER, OpenSsl.isAvailable());
+
+ Assert.assertTrue(ctx1.isServer());
+ Assert.assertEquals(ctx1.cipherSuites(), options.cipher_suites);
+
+ options = options.withCipherSuites("TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256");
+
+ SslContext ctx2 = SSLFactory.getOrCreateSslContext(options, true,
+ SSLFactory.SocketType.CLIENT, OpenSsl.isAvailable());
+
+ Assert.assertTrue(ctx2.isClient());
+ Assert.assertEquals(ctx2.cipherSuites(), options.cipher_suites);
+ }
}
diff --git a/test/unit/org/apache/cassandra/serializers/SerializationUtils.java b/test/unit/org/apache/cassandra/serializers/SerializationUtils.java
new file mode 100644
index 0000000..b88b56f
--- /dev/null
+++ b/test/unit/org/apache/cassandra/serializers/SerializationUtils.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.serializers;
+
+import java.io.IOException;
+
+import org.junit.Assert;
+
+import org.apache.cassandra.io.IVersionedSerializer;
+import org.apache.cassandra.io.util.DataInputBuffer;
+import org.apache.cassandra.io.util.DataOutputBuffer;
+import org.apache.cassandra.net.MessagingService;
+
+public class SerializationUtils
+{
+
+ public static <T> T cycleSerialization(T src, IVersionedSerializer<T> serializer, int version)
+ {
+ int expectedSize = (int) serializer.serializedSize(src, version);
+
+ try (DataOutputBuffer out = new DataOutputBuffer(expectedSize))
+ {
+ serializer.serialize(src, out, version);
+ Assert.assertEquals(expectedSize, out.buffer().limit());
+ try (DataInputBuffer in = new DataInputBuffer(out.buffer(), false))
+ {
+ return serializer.deserialize(in, version);
+ }
+ }
+ catch (IOException e)
+ {
+ throw new AssertionError(e);
+ }
+ }
+
+ public static <T> T cycleSerialization(T src, IVersionedSerializer<T> serializer)
+ {
+ return cycleSerialization(src, serializer, MessagingService.current_version);
+ }
+
+ public static <T> void assertSerializationCycle(T src, IVersionedSerializer<T> serializer, int version)
+ {
+ T dst = cycleSerialization(src, serializer, version);
+ Assert.assertEquals(src, dst);
+ }
+
+ public static <T> void assertSerializationCycle(T src, IVersionedSerializer<T> serializer)
+ {
+ assertSerializationCycle(src, serializer, MessagingService.current_version);
+ }
+}
diff --git a/test/unit/org/apache/cassandra/serializers/SimpleDateSerializerTest.java b/test/unit/org/apache/cassandra/serializers/SimpleDateSerializerTest.java
index e051357..9c1ef88 100644
--- a/test/unit/org/apache/cassandra/serializers/SimpleDateSerializerTest.java
+++ b/test/unit/org/apache/cassandra/serializers/SimpleDateSerializerTest.java
@@ -24,11 +24,12 @@
import java.nio.ByteBuffer;
import java.sql.Timestamp;
import java.text.SimpleDateFormat;
+import java.time.temporal.ChronoUnit;
import java.util.*;
public class SimpleDateSerializerTest
{
- private static final long millisPerDay = 1000 * 60 * 60 * 24;
+ private static final long millisPerDay = ChronoUnit.DAYS.getDuration().toMillis();
private String dates[] = new String[]
{
@@ -38,7 +39,7 @@
"-0001-01-02",
"-5877521-01-02",
"2014-01-01",
- "5881580-01-10",
+ "+5881580-01-10", // See java.time.format.SignStyle.EXCEEDS_PAD
"1920-12-01",
"1582-10-19"
};
diff --git a/test/unit/org/apache/cassandra/service/ActiveRepairServiceTest.java b/test/unit/org/apache/cassandra/service/ActiveRepairServiceTest.java
index 2c1a8d2..8883f21 100644
--- a/test/unit/org/apache/cassandra/service/ActiveRepairServiceTest.java
+++ b/test/unit/org/apache/cassandra/service/ActiveRepairServiceTest.java
@@ -18,21 +18,31 @@
*/
package org.apache.cassandra.service;
-import java.net.InetAddress;
import java.util.*;
-import java.util.concurrent.ExecutionException;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Executor;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.RejectedExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.locks.Condition;
+import com.google.common.collect.ImmutableList;
import com.google.common.collect.Sets;
+import com.google.common.util.concurrent.Uninterruptibles;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.concurrent.DebuggableThreadPoolExecutor;
+import org.apache.cassandra.config.Config;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.db.RowUpdateBuilder;
-import org.apache.cassandra.db.compaction.OperationType;
import org.apache.cassandra.db.lifecycle.SSTableSet;
import org.apache.cassandra.db.lifecycle.View;
import org.apache.cassandra.dht.Range;
@@ -40,14 +50,26 @@
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.locator.AbstractReplicationStrategy;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.Replica;
import org.apache.cassandra.locator.TokenMetadata;
+import org.apache.cassandra.repair.messages.RepairOption;
+import org.apache.cassandra.streaming.PreviewKind;
import org.apache.cassandra.schema.KeyspaceParams;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.concurrent.Refs;
+import org.apache.cassandra.utils.concurrent.SimpleCondition;
+import static org.apache.cassandra.repair.messages.RepairOption.DATACENTERS_KEY;
+import static org.apache.cassandra.repair.messages.RepairOption.FORCE_REPAIR_KEY;
+import static org.apache.cassandra.repair.messages.RepairOption.HOSTS_KEY;
+import static org.apache.cassandra.repair.messages.RepairOption.INCREMENTAL_KEY;
+import static org.apache.cassandra.repair.messages.RepairOption.RANGES_KEY;
+import static org.apache.cassandra.service.ActiveRepairService.UNREPAIRED_SSTABLE;
+import static org.apache.cassandra.service.ActiveRepairService.getRepairedAt;
import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
public class ActiveRepairServiceTest
{
@@ -57,7 +79,7 @@
public String cfname;
public ColumnFamilyStore store;
- public InetAddress LOCAL, REMOTE;
+ public InetAddressAndPort LOCAL, REMOTE;
private boolean initialized;
@@ -79,9 +101,9 @@
SchemaLoader.startGossiper();
initialized = true;
- LOCAL = FBUtilities.getBroadcastAddress();
+ LOCAL = FBUtilities.getBroadcastAddressAndPort();
// generate a fake endpoint for which we can spoof receiving/sending trees
- REMOTE = InetAddress.getByName("127.0.0.2");
+ REMOTE = InetAddressAndPort.getByName("127.0.0.2");
}
TokenMetadata tmd = StorageService.instance.getTokenMetadata();
@@ -95,13 +117,13 @@
public void testGetNeighborsPlusOne() throws Throwable
{
// generate rf+1 nodes, and ensure that all nodes are returned
- Set<InetAddress> expected = addTokens(1 + Keyspace.open(KEYSPACE5).getReplicationStrategy().getReplicationFactor());
- expected.remove(FBUtilities.getBroadcastAddress());
- Collection<Range<Token>> ranges = StorageService.instance.getLocalRanges(KEYSPACE5);
- Set<InetAddress> neighbors = new HashSet<>();
+ Set<InetAddressAndPort> expected = addTokens(1 + Keyspace.open(KEYSPACE5).getReplicationStrategy().getReplicationFactor().allReplicas);
+ expected.remove(FBUtilities.getBroadcastAddressAndPort());
+ Iterable<Range<Token>> ranges = StorageService.instance.getLocalReplicas(KEYSPACE5).ranges();
+ Set<InetAddressAndPort> neighbors = new HashSet<>();
for (Range<Token> range : ranges)
{
- neighbors.addAll(ActiveRepairService.getNeighbors(KEYSPACE5, ranges, range, null, null));
+ neighbors.addAll(ActiveRepairService.getNeighbors(KEYSPACE5, ranges, range, null, null).endpoints());
}
assertEquals(expected, neighbors);
}
@@ -112,19 +134,19 @@
TokenMetadata tmd = StorageService.instance.getTokenMetadata();
// generate rf*2 nodes, and ensure that only neighbors specified by the ARS are returned
- addTokens(2 * Keyspace.open(KEYSPACE5).getReplicationStrategy().getReplicationFactor());
+ addTokens(2 * Keyspace.open(KEYSPACE5).getReplicationStrategy().getReplicationFactor().allReplicas);
AbstractReplicationStrategy ars = Keyspace.open(KEYSPACE5).getReplicationStrategy();
- Set<InetAddress> expected = new HashSet<>();
- for (Range<Token> replicaRange : ars.getAddressRanges().get(FBUtilities.getBroadcastAddress()))
+ Set<InetAddressAndPort> expected = new HashSet<>();
+ for (Replica replica : ars.getAddressReplicas().get(FBUtilities.getBroadcastAddressAndPort()))
{
- expected.addAll(ars.getRangeAddresses(tmd.cloneOnlyTokenMap()).get(replicaRange));
+ expected.addAll(ars.getRangeAddresses(tmd.cloneOnlyTokenMap()).get(replica.range()).endpoints());
}
- expected.remove(FBUtilities.getBroadcastAddress());
- Collection<Range<Token>> ranges = StorageService.instance.getLocalRanges(KEYSPACE5);
- Set<InetAddress> neighbors = new HashSet<>();
+ expected.remove(FBUtilities.getBroadcastAddressAndPort());
+ Iterable<Range<Token>> ranges = StorageService.instance.getLocalReplicas(KEYSPACE5).ranges();
+ Set<InetAddressAndPort> neighbors = new HashSet<>();
for (Range<Token> range : ranges)
{
- neighbors.addAll(ActiveRepairService.getNeighbors(KEYSPACE5, ranges, range, null, null));
+ neighbors.addAll(ActiveRepairService.getNeighbors(KEYSPACE5, ranges, range, null, null).endpoints());
}
assertEquals(expected, neighbors);
}
@@ -135,18 +157,18 @@
TokenMetadata tmd = StorageService.instance.getTokenMetadata();
// generate rf+1 nodes, and ensure that all nodes are returned
- Set<InetAddress> expected = addTokens(1 + Keyspace.open(KEYSPACE5).getReplicationStrategy().getReplicationFactor());
- expected.remove(FBUtilities.getBroadcastAddress());
+ Set<InetAddressAndPort> expected = addTokens(1 + Keyspace.open(KEYSPACE5).getReplicationStrategy().getReplicationFactor().allReplicas);
+ expected.remove(FBUtilities.getBroadcastAddressAndPort());
// remove remote endpoints
TokenMetadata.Topology topology = tmd.cloneOnlyTokenMap().getTopology();
- HashSet<InetAddress> localEndpoints = Sets.newHashSet(topology.getDatacenterEndpoints().get(DatabaseDescriptor.getLocalDataCenter()));
+ HashSet<InetAddressAndPort> localEndpoints = Sets.newHashSet(topology.getDatacenterEndpoints().get(DatabaseDescriptor.getLocalDataCenter()));
expected = Sets.intersection(expected, localEndpoints);
- Collection<Range<Token>> ranges = StorageService.instance.getLocalRanges(KEYSPACE5);
- Set<InetAddress> neighbors = new HashSet<>();
+ Iterable<Range<Token>> ranges = StorageService.instance.getLocalReplicas(KEYSPACE5).ranges();
+ Set<InetAddressAndPort> neighbors = new HashSet<>();
for (Range<Token> range : ranges)
{
- neighbors.addAll(ActiveRepairService.getNeighbors(KEYSPACE5, ranges, range, Arrays.asList(DatabaseDescriptor.getLocalDataCenter()), null));
+ neighbors.addAll(ActiveRepairService.getNeighbors(KEYSPACE5, ranges, range, Arrays.asList(DatabaseDescriptor.getLocalDataCenter()), null).endpoints());
}
assertEquals(expected, neighbors);
}
@@ -157,24 +179,24 @@
TokenMetadata tmd = StorageService.instance.getTokenMetadata();
// generate rf*2 nodes, and ensure that only neighbors specified by the ARS are returned
- addTokens(2 * Keyspace.open(KEYSPACE5).getReplicationStrategy().getReplicationFactor());
+ addTokens(2 * Keyspace.open(KEYSPACE5).getReplicationStrategy().getReplicationFactor().allReplicas);
AbstractReplicationStrategy ars = Keyspace.open(KEYSPACE5).getReplicationStrategy();
- Set<InetAddress> expected = new HashSet<>();
- for (Range<Token> replicaRange : ars.getAddressRanges().get(FBUtilities.getBroadcastAddress()))
+ Set<InetAddressAndPort> expected = new HashSet<>();
+ for (Replica replica : ars.getAddressReplicas().get(FBUtilities.getBroadcastAddressAndPort()))
{
- expected.addAll(ars.getRangeAddresses(tmd.cloneOnlyTokenMap()).get(replicaRange));
+ expected.addAll(ars.getRangeAddresses(tmd.cloneOnlyTokenMap()).get(replica.range()).endpoints());
}
- expected.remove(FBUtilities.getBroadcastAddress());
+ expected.remove(FBUtilities.getBroadcastAddressAndPort());
// remove remote endpoints
TokenMetadata.Topology topology = tmd.cloneOnlyTokenMap().getTopology();
- HashSet<InetAddress> localEndpoints = Sets.newHashSet(topology.getDatacenterEndpoints().get(DatabaseDescriptor.getLocalDataCenter()));
+ HashSet<InetAddressAndPort> localEndpoints = Sets.newHashSet(topology.getDatacenterEndpoints().get(DatabaseDescriptor.getLocalDataCenter()));
expected = Sets.intersection(expected, localEndpoints);
- Collection<Range<Token>> ranges = StorageService.instance.getLocalRanges(KEYSPACE5);
- Set<InetAddress> neighbors = new HashSet<>();
+ Iterable<Range<Token>> ranges = StorageService.instance.getLocalReplicas(KEYSPACE5).ranges();
+ Set<InetAddressAndPort> neighbors = new HashSet<>();
for (Range<Token> range : ranges)
{
- neighbors.addAll(ActiveRepairService.getNeighbors(KEYSPACE5, ranges, range, Arrays.asList(DatabaseDescriptor.getLocalDataCenter()), null));
+ neighbors.addAll(ActiveRepairService.getNeighbors(KEYSPACE5, ranges, range, Arrays.asList(DatabaseDescriptor.getLocalDataCenter()), null).endpoints());
}
assertEquals(expected, neighbors);
}
@@ -185,40 +207,61 @@
TokenMetadata tmd = StorageService.instance.getTokenMetadata();
// generate rf*2 nodes, and ensure that only neighbors specified by the hosts are returned
- addTokens(2 * Keyspace.open(KEYSPACE5).getReplicationStrategy().getReplicationFactor());
+ addTokens(2 * Keyspace.open(KEYSPACE5).getReplicationStrategy().getReplicationFactor().allReplicas);
AbstractReplicationStrategy ars = Keyspace.open(KEYSPACE5).getReplicationStrategy();
- List<InetAddress> expected = new ArrayList<>();
- for (Range<Token> replicaRange : ars.getAddressRanges().get(FBUtilities.getBroadcastAddress()))
+ List<InetAddressAndPort> expected = new ArrayList<>();
+ for (Replica replicas : ars.getAddressReplicas().get(FBUtilities.getBroadcastAddressAndPort()))
{
- expected.addAll(ars.getRangeAddresses(tmd.cloneOnlyTokenMap()).get(replicaRange));
+ expected.addAll(ars.getRangeAddresses(tmd.cloneOnlyTokenMap()).get(replicas.range()).endpoints());
}
- expected.remove(FBUtilities.getBroadcastAddress());
- Collection<String> hosts = Arrays.asList(FBUtilities.getBroadcastAddress().getCanonicalHostName(),expected.get(0).getCanonicalHostName());
- Collection<Range<Token>> ranges = StorageService.instance.getLocalRanges(KEYSPACE5);
+ expected.remove(FBUtilities.getBroadcastAddressAndPort());
+ Collection<String> hosts = Arrays.asList(FBUtilities.getBroadcastAddressAndPort().toString(),expected.get(0).toString());
+ Iterable<Range<Token>> ranges = StorageService.instance.getLocalReplicas(KEYSPACE5).ranges();
assertEquals(expected.get(0), ActiveRepairService.getNeighbors(KEYSPACE5, ranges,
ranges.iterator().next(),
- null, hosts).iterator().next());
+ null, hosts).endpoints().iterator().next());
}
@Test(expected = IllegalArgumentException.class)
public void testGetNeighborsSpecifiedHostsWithNoLocalHost() throws Throwable
{
- addTokens(2 * Keyspace.open(KEYSPACE5).getReplicationStrategy().getReplicationFactor());
+ addTokens(2 * Keyspace.open(KEYSPACE5).getReplicationStrategy().getReplicationFactor().allReplicas);
//Dont give local endpoint
Collection<String> hosts = Arrays.asList("127.0.0.3");
- Collection<Range<Token>> ranges = StorageService.instance.getLocalRanges(KEYSPACE5);
+ Iterable<Range<Token>> ranges = StorageService.instance.getLocalReplicas(KEYSPACE5).ranges();
ActiveRepairService.getNeighbors(KEYSPACE5, ranges, ranges.iterator().next(), null, hosts);
}
- Set<InetAddress> addTokens(int max) throws Throwable
+
+ @Test
+ public void testParentRepairStatus() throws Throwable
+ {
+ ActiveRepairService.instance.recordRepairStatus(1, ActiveRepairService.ParentRepairStatus.COMPLETED, ImmutableList.of("foo", "bar"));
+ List<String> res = StorageService.instance.getParentRepairStatus(1);
+ assertNotNull(res);
+ assertEquals(ActiveRepairService.ParentRepairStatus.COMPLETED, ActiveRepairService.ParentRepairStatus.valueOf(res.get(0)));
+ assertEquals("foo", res.get(1));
+ assertEquals("bar", res.get(2));
+
+ List<String> emptyRes = StorageService.instance.getParentRepairStatus(44);
+ assertNull(emptyRes);
+
+ ActiveRepairService.instance.recordRepairStatus(3, ActiveRepairService.ParentRepairStatus.FAILED, ImmutableList.of("some failure message", "bar"));
+ List<String> failed = StorageService.instance.getParentRepairStatus(3);
+ assertNotNull(failed);
+ assertEquals(ActiveRepairService.ParentRepairStatus.FAILED, ActiveRepairService.ParentRepairStatus.valueOf(failed.get(0)));
+
+ }
+
+ Set<InetAddressAndPort> addTokens(int max) throws Throwable
{
TokenMetadata tmd = StorageService.instance.getTokenMetadata();
- Set<InetAddress> endpoints = new HashSet<>();
+ Set<InetAddressAndPort> endpoints = new HashSet<>();
for (int i = 1; i <= max; i++)
{
- InetAddress endpoint = InetAddress.getByName("127.0.0." + i);
+ InetAddressAndPort endpoint = InetAddressAndPort.getByName("127.0.0." + i);
tmd.updateNormalToken(tmd.partitioner.getRandomToken(), endpoint);
endpoints.add(endpoint);
}
@@ -226,126 +269,25 @@
}
@Test
- public void testGetActiveRepairedSSTableRefs()
- {
- ColumnFamilyStore store = prepareColumnFamilyStore();
- Set<SSTableReader> original = store.getLiveSSTables();
-
- UUID prsId = UUID.randomUUID();
- ActiveRepairService.instance.registerParentRepairSession(prsId, FBUtilities.getBroadcastAddress(), Collections.singletonList(store), null, true, 0, false);
- ActiveRepairService.ParentRepairSession prs = ActiveRepairService.instance.getParentRepairSession(prsId);
- prs.markSSTablesRepairing(store.metadata.cfId, prsId);
-
- //retrieve all sstable references from parent repair sessions
- Refs<SSTableReader> refs = prs.getActiveRepairedSSTableRefsForAntiCompaction(store.metadata.cfId, prsId);
- Set<SSTableReader> retrieved = Sets.newHashSet(refs.iterator());
- assertEquals(original, retrieved);
- refs.release();
-
- //remove 1 sstable from data data tracker
- Set<SSTableReader> newLiveSet = new HashSet<>(original);
- Iterator<SSTableReader> it = newLiveSet.iterator();
- final SSTableReader removed = it.next();
- it.remove();
- store.getTracker().dropSSTables(new com.google.common.base.Predicate<SSTableReader>()
- {
- public boolean apply(SSTableReader reader)
- {
- return removed.equals(reader);
- }
- }, OperationType.COMPACTION, null);
-
- //retrieve sstable references from parent repair session again - removed sstable must not be present
- refs = prs.getActiveRepairedSSTableRefsForAntiCompaction(store.metadata.cfId, prsId);
- retrieved = Sets.newHashSet(refs.iterator());
- assertEquals(newLiveSet, retrieved);
- assertFalse(retrieved.contains(removed));
- refs.release();
- }
-
- @Test
- public void testAddingMoreSSTables()
- {
- ColumnFamilyStore store = prepareColumnFamilyStore();
- Set<SSTableReader> original = Sets.newHashSet(store.select(View.select(SSTableSet.CANONICAL, (s) -> !s.isRepaired())).sstables);
- UUID prsId = UUID.randomUUID();
- ActiveRepairService.instance.registerParentRepairSession(prsId, FBUtilities.getBroadcastAddress(), Collections.singletonList(store), null, true, System.currentTimeMillis(), true);
-
- ActiveRepairService.ParentRepairSession prs = ActiveRepairService.instance.getParentRepairSession(prsId);
- prs.markSSTablesRepairing(store.metadata.cfId, prsId);
- try (Refs<SSTableReader> refs = prs.getActiveRepairedSSTableRefsForAntiCompaction(store.metadata.cfId, prsId))
- {
- Set<SSTableReader> retrieved = Sets.newHashSet(refs.iterator());
- assertEquals(original, retrieved);
- }
- createSSTables(store, 2);
- boolean exception = false;
- try
- {
- UUID newPrsId = UUID.randomUUID();
- ActiveRepairService.instance.registerParentRepairSession(newPrsId, FBUtilities.getBroadcastAddress(), Collections.singletonList(store), null, true, System.currentTimeMillis(), true);
- ActiveRepairService.instance.getParentRepairSession(newPrsId).markSSTablesRepairing(store.metadata.cfId, newPrsId);
- }
- catch (Throwable t)
- {
- exception = true;
- }
- assertTrue(exception);
-
- try (Refs<SSTableReader> refs = prs.getActiveRepairedSSTableRefsForAntiCompaction(store.metadata.cfId, prsId))
- {
- Set<SSTableReader> retrieved = Sets.newHashSet(refs.iterator());
- assertEquals(original, retrieved);
- }
- }
-
- @Test
- public void testSnapshotAddSSTables() throws ExecutionException, InterruptedException
+ public void testSnapshotAddSSTables() throws Exception
{
ColumnFamilyStore store = prepareColumnFamilyStore();
UUID prsId = UUID.randomUUID();
Set<SSTableReader> original = Sets.newHashSet(store.select(View.select(SSTableSet.CANONICAL, (s) -> !s.isRepaired())).sstables);
- ActiveRepairService.instance.registerParentRepairSession(prsId, FBUtilities.getBroadcastAddress(), Collections.singletonList(store), Collections.singleton(new Range<>(store.getPartitioner().getMinimumToken(), store.getPartitioner().getMinimumToken())), true, System.currentTimeMillis(), true);
- ActiveRepairService.instance.getParentRepairSession(prsId).maybeSnapshot(store.metadata.cfId, prsId);
+ Collection<Range<Token>> ranges = Collections.singleton(new Range<>(store.getPartitioner().getMinimumToken(), store.getPartitioner().getMinimumToken()));
+ ActiveRepairService.instance.registerParentRepairSession(prsId, FBUtilities.getBroadcastAddressAndPort(), Collections.singletonList(store),
+ ranges, true, System.currentTimeMillis(), true, PreviewKind.NONE);
+ store.getRepairManager().snapshot(prsId.toString(), ranges, false);
UUID prsId2 = UUID.randomUUID();
- ActiveRepairService.instance.registerParentRepairSession(prsId2, FBUtilities.getBroadcastAddress(), Collections.singletonList(store), Collections.singleton(new Range<>(store.getPartitioner().getMinimumToken(), store.getPartitioner().getMinimumToken())), true, System.currentTimeMillis(), true);
+ ActiveRepairService.instance.registerParentRepairSession(prsId2, FBUtilities.getBroadcastAddressAndPort(),
+ Collections.singletonList(store),
+ ranges,
+ true, System.currentTimeMillis(),
+ true, PreviewKind.NONE);
createSSTables(store, 2);
- ActiveRepairService.instance.getParentRepairSession(prsId).maybeSnapshot(store.metadata.cfId, prsId);
- try (Refs<SSTableReader> refs = ActiveRepairService.instance.getParentRepairSession(prsId).getActiveRepairedSSTableRefsForAntiCompaction(store.metadata.cfId, prsId))
- {
- assertEquals(original, Sets.newHashSet(refs.iterator()));
- }
- store.forceMajorCompaction();
- // after a major compaction the original sstables will be gone and we will have no sstables to anticompact:
- try (Refs<SSTableReader> refs = ActiveRepairService.instance.getParentRepairSession(prsId).getActiveRepairedSSTableRefsForAntiCompaction(store.metadata.cfId, prsId))
- {
- assertEquals(0, refs.size());
- }
- }
-
- @Test
- public void testSnapshotMultipleRepairs()
- {
- ColumnFamilyStore store = prepareColumnFamilyStore();
- Set<SSTableReader> original = Sets.newHashSet(store.select(View.select(SSTableSet.CANONICAL, (s) -> !s.isRepaired())).sstables);
- UUID prsId = UUID.randomUUID();
- ActiveRepairService.instance.registerParentRepairSession(prsId, FBUtilities.getBroadcastAddress(), Collections.singletonList(store), Collections.singleton(new Range<>(store.getPartitioner().getMinimumToken(), store.getPartitioner().getMinimumToken())), true, System.currentTimeMillis(), true);
- ActiveRepairService.instance.getParentRepairSession(prsId).maybeSnapshot(store.metadata.cfId, prsId);
-
- UUID prsId2 = UUID.randomUUID();
- ActiveRepairService.instance.registerParentRepairSession(prsId2, FBUtilities.getBroadcastAddress(), Collections.singletonList(store), Collections.singleton(new Range<>(store.getPartitioner().getMinimumToken(), store.getPartitioner().getMinimumToken())), true, System.currentTimeMillis(), true);
- boolean exception = false;
- try
- {
- ActiveRepairService.instance.getParentRepairSession(prsId2).maybeSnapshot(store.metadata.cfId, prsId2);
- }
- catch (Throwable t)
- {
- exception = true;
- }
- assertTrue(exception);
- try (Refs<SSTableReader> refs = ActiveRepairService.instance.getParentRepairSession(prsId).getActiveRepairedSSTableRefsForAntiCompaction(store.metadata.cfId, prsId))
+ store.getRepairManager().snapshot(prsId.toString(), ranges, false);
+ try (Refs<SSTableReader> refs = store.getSnapshotSSTableReaders(prsId.toString()))
{
assertEquals(original, Sets.newHashSet(refs.iterator()));
}
@@ -368,7 +310,7 @@
{
for (int j = 0; j < 10; j++)
{
- new RowUpdateBuilder(cfs.metadata, timestamp, Integer.toString(j))
+ new RowUpdateBuilder(cfs.metadata(), timestamp, Integer.toString(j))
.clustering("c")
.add("val", "val")
.build()
@@ -377,4 +319,143 @@
cfs.forceBlockingFlush();
}
}
+
+ private static RepairOption opts(String... params)
+ {
+ assert params.length % 2 == 0 : "unbalanced key value pairs";
+ Map<String, String> opt = new HashMap<>();
+ for (int i=0; i<(params.length >> 1); i++)
+ {
+ int idx = i << 1;
+ opt.put(params[idx], params[idx+1]);
+ }
+ return RepairOption.parse(opt, DatabaseDescriptor.getPartitioner());
+ }
+
+ private static String b2s(boolean b)
+ {
+ return Boolean.toString(b);
+ }
+
+ /**
+ * Tests the expected repairedAt value is returned, based on different RepairOption
+ */
+ @Test
+ public void repairedAt() throws Exception
+ {
+ // regular incremental repair
+ Assert.assertNotEquals(UNREPAIRED_SSTABLE, getRepairedAt(opts(INCREMENTAL_KEY, b2s(true)), false));
+ // subrange incremental repair
+ Assert.assertNotEquals(UNREPAIRED_SSTABLE, getRepairedAt(opts(INCREMENTAL_KEY, b2s(true),
+ RANGES_KEY, "1:2"), false));
+
+ // hosts incremental repair
+ Assert.assertEquals(UNREPAIRED_SSTABLE, getRepairedAt(opts(INCREMENTAL_KEY, b2s(true),
+ HOSTS_KEY, "127.0.0.1"), false));
+ // dc incremental repair
+ Assert.assertEquals(UNREPAIRED_SSTABLE, getRepairedAt(opts(INCREMENTAL_KEY, b2s(true),
+ DATACENTERS_KEY, "DC2"), false));
+ // forced incremental repair
+ Assert.assertNotEquals(UNREPAIRED_SSTABLE, getRepairedAt(opts(INCREMENTAL_KEY, b2s(true),
+ FORCE_REPAIR_KEY, b2s(true)), false));
+ Assert.assertEquals(UNREPAIRED_SSTABLE, getRepairedAt(opts(INCREMENTAL_KEY, b2s(true),
+ FORCE_REPAIR_KEY, b2s(true)), true));
+
+ // full repair
+ Assert.assertEquals(UNREPAIRED_SSTABLE, getRepairedAt(opts(INCREMENTAL_KEY, b2s(false)), false));
+ }
+
+ @Test
+ public void testRejectWhenPoolFullStrategy() throws InterruptedException
+ {
+ // Using RepairCommandPoolFullStrategy.reject, new threads are spawned up to
+ // repair_command_pool_size, at which point futher submissions are rejected
+ ExecutorService validationExecutor = ActiveRepairService.initializeExecutor(2, Config.RepairCommandPoolFullStrategy.reject);
+ try
+ {
+ Condition blocked = new SimpleCondition();
+ CountDownLatch completed = new CountDownLatch(2);
+ validationExecutor.submit(new Task(blocked, completed));
+ validationExecutor.submit(new Task(blocked, completed));
+ try
+ {
+ validationExecutor.submit(new Task(blocked, completed));
+ Assert.fail("Expected task submission to be rejected");
+ }
+ catch (RejectedExecutionException e)
+ {
+ // expected
+ }
+ // allow executing tests to complete
+ blocked.signalAll();
+ completed.await(10, TimeUnit.SECONDS);
+ // Submission is unblocked
+ validationExecutor.submit(() -> {});
+ }
+ finally
+ {
+ // necessary to unregister mbean
+ validationExecutor.shutdownNow();
+ }
+ }
+
+ @Test
+ public void testQueueWhenPoolFullStrategy() throws InterruptedException
+ {
+ // Using RepairCommandPoolFullStrategy.queue, the pool is initialized to
+ // repair_command_pool_size and any tasks which cannot immediately be
+ // serviced are queued
+ ExecutorService validationExecutor = ActiveRepairService.initializeExecutor(2, Config.RepairCommandPoolFullStrategy.queue);
+ try
+ {
+ Condition allSubmitted = new SimpleCondition();
+ Condition blocked = new SimpleCondition();
+ CountDownLatch completed = new CountDownLatch(5);
+ ExecutorService testExecutor = Executors.newSingleThreadExecutor();
+ for (int i = 0; i < 5; i++)
+ {
+ if (i < 4)
+ testExecutor.submit(() -> validationExecutor.submit(new Task(blocked, completed)));
+ else
+ testExecutor.submit(() -> {
+ validationExecutor.submit(new Task(blocked, completed));
+ allSubmitted.signalAll();
+ });
+ }
+
+ // Make sure all tasks have been submitted to the validation executor
+ allSubmitted.await(10, TimeUnit.SECONDS);
+
+ // 2 threads actively processing tasks
+ Assert.assertEquals(2, ((DebuggableThreadPoolExecutor) validationExecutor).getActiveTaskCount());
+ // 3 tasks queued
+ Assert.assertEquals(3, ((DebuggableThreadPoolExecutor) validationExecutor).getPendingTaskCount());
+ // allow executing tests to complete
+ blocked.signalAll();
+ completed.await(10, TimeUnit.SECONDS);
+ }
+ finally
+ {
+ // necessary to unregister mbean
+ validationExecutor.shutdownNow();
+ }
+ }
+
+ private static class Task implements Runnable
+ {
+ private final Condition blocked;
+ private final CountDownLatch complete;
+
+ Task(Condition blocked, CountDownLatch complete)
+ {
+ this.blocked = blocked;
+ this.complete = complete;
+ }
+
+ public void run()
+ {
+ Uninterruptibles.awaitUninterruptibly(blocked, 10, TimeUnit.SECONDS);
+ complete.countDown();
+ }
+ }
}
diff --git a/test/unit/org/apache/cassandra/service/BootstrapTransientTest.java b/test/unit/org/apache/cassandra/service/BootstrapTransientTest.java
new file mode 100644
index 0000000..7bb2b87
--- /dev/null
+++ b/test/unit/org/apache/cassandra/service/BootstrapTransientTest.java
@@ -0,0 +1,226 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service;
+
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+
+import com.google.common.base.Predicate;
+import com.google.common.collect.ImmutableMap;
+
+import org.apache.cassandra.locator.EndpointsByReplica;
+import org.apache.cassandra.locator.EndpointsForRange;
+import org.junit.After;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.dht.OrderPreservingPartitioner;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.RangeStreamer;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.locator.AbstractEndpointSnitch;
+import org.apache.cassandra.locator.AbstractReplicationStrategy;
+import org.apache.cassandra.locator.IEndpointSnitch;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.RangesAtEndpoint;
+import org.apache.cassandra.locator.Replica;
+import org.apache.cassandra.locator.ReplicaCollection;
+import org.apache.cassandra.locator.SimpleStrategy;
+import org.apache.cassandra.locator.TokenMetadata;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.Pair;
+
+import static org.apache.cassandra.locator.Replica.fullReplica;
+import static org.apache.cassandra.locator.Replica.transientReplica;
+import static org.apache.cassandra.service.StorageServiceTest.assertMultimapEqualsIgnoreOrder;
+
+/**
+ * This is also fairly effectively testing source retrieval for bootstrap as well since RangeStreamer
+ * is used to calculate the endpoints to fetch from and check they are alive for both RangeRelocator (move) and
+ * bootstrap (RangeRelocator).
+ */
+public class BootstrapTransientTest
+{
+ static InetAddressAndPort address02;
+ static InetAddressAndPort address03;
+ static InetAddressAndPort address04;
+ static InetAddressAndPort address05;
+
+ @BeforeClass
+ public static void setUpClass() throws Exception
+ {
+ address02 = InetAddressAndPort.getByName("127.0.0.2");
+ address03 = InetAddressAndPort.getByName("127.0.0.3");
+ address04 = InetAddressAndPort.getByName("127.0.0.4");
+ address05 = InetAddressAndPort.getByName("127.0.0.5");
+ }
+
+ private final List<InetAddressAndPort> downNodes = new ArrayList<>();
+
+ final RangeStreamer.SourceFilter alivePredicate = new RangeStreamer.SourceFilter()
+ {
+ public boolean apply(Replica replica)
+ {
+ return !downNodes.contains(replica.endpoint());
+ }
+
+ public String message(Replica replica)
+ {
+ return "Down nodes: " + downNodes;
+ }
+ };
+
+ final RangeStreamer.SourceFilter sourceFilterDownNodesPredicate = new RangeStreamer.SourceFilter()
+ {
+ public boolean apply(Replica replica)
+ {
+ return !sourceFilterDownNodes.contains(replica.endpoint());
+ }
+
+ public String message(Replica replica)
+ {
+ return "Source filter down nodes" + sourceFilterDownNodes;
+ }
+ };
+
+ private final List<InetAddressAndPort> sourceFilterDownNodes = new ArrayList<>();
+
+ private final Collection<RangeStreamer.SourceFilter> sourceFilters = Arrays.asList(alivePredicate,
+ sourceFilterDownNodesPredicate,
+ new RangeStreamer.ExcludeLocalNodeFilter()
+ );
+
+ @After
+ public void clearDownNode()
+ {
+ // TODO: actually use these
+ downNodes.clear();
+ sourceFilterDownNodes.clear();
+ }
+
+ @BeforeClass
+ public static void setupDD()
+ {
+ DatabaseDescriptor.daemonInitialization();
+ }
+
+ Token tenToken = new OrderPreservingPartitioner.StringToken("00010");
+ Token twentyToken = new OrderPreservingPartitioner.StringToken("00020");
+ Token thirtyToken = new OrderPreservingPartitioner.StringToken("00030");
+ Token fourtyToken = new OrderPreservingPartitioner.StringToken("00040");
+
+ Range<Token> range30_10 = new Range<>(thirtyToken, tenToken);
+ Range<Token> range10_20 = new Range<>(tenToken, twentyToken);
+ Range<Token> range20_30 = new Range<>(twentyToken, thirtyToken);
+ Range<Token> range30_40 = new Range<>(thirtyToken, fourtyToken);
+
+ RangesAtEndpoint toFetch = RangesAtEndpoint.of(new Replica(address05, range30_40, true),
+ new Replica(address05, range20_30, true),
+ new Replica(address05, range10_20, false));
+
+
+
+ public EndpointsForRange endpoints(Replica... replicas)
+ {
+ assert replicas.length > 0;
+
+ Range<Token> range = replicas[0].range();
+ EndpointsForRange.Builder builder = EndpointsForRange.builder(range);
+ for (Replica r : replicas)
+ {
+ assert r.range().equals(range);
+ builder.add(r);
+ }
+
+ return builder.build();
+ }
+ @Test
+ public void testRangeStreamerRangesToFetch() throws Exception
+ {
+ EndpointsByReplica expectedResult = new EndpointsByReplica(ImmutableMap.of(
+ transientReplica(address05, range10_20), endpoints(transientReplica(address02, range10_20)),
+ fullReplica(address05, range20_30), endpoints(transientReplica(address03, range20_30), fullReplica(address04, range20_30)),
+ fullReplica(address05, range30_40), endpoints(transientReplica(address04, range30_10), fullReplica(address02, range30_10))));
+
+ invokeCalculateRangesToFetchWithPreferredEndpoints(toFetch, constructTMDs(), expectedResult);
+ }
+
+ private Pair<TokenMetadata, TokenMetadata> constructTMDs()
+ {
+ TokenMetadata tmd = new TokenMetadata();
+ tmd.updateNormalToken(range30_10.right, address02);
+ tmd.updateNormalToken(range10_20.right, address03);
+ tmd.updateNormalToken(range20_30.right, address04);
+ TokenMetadata updated = tmd.cloneOnlyTokenMap();
+ updated.updateNormalToken(range30_40.right, address05);
+
+ return Pair.create(tmd, updated);
+ }
+
+ private void invokeCalculateRangesToFetchWithPreferredEndpoints(ReplicaCollection<?> toFetch,
+ Pair<TokenMetadata, TokenMetadata> tmds,
+ EndpointsByReplica expectedResult)
+ {
+ DatabaseDescriptor.setTransientReplicationEnabledUnsafe(true);
+
+ EndpointsByReplica result = RangeStreamer.calculateRangesToFetchWithPreferredEndpoints((address, replicas) -> replicas,
+ simpleStrategy(tmds.left),
+ toFetch,
+ true,
+ tmds.left,
+ tmds.right,
+ "TestKeyspace",
+ sourceFilters);
+ result.asMap().forEach((replica, list) -> System.out.printf("Replica %s, sources %s%n", replica, list));
+ assertMultimapEqualsIgnoreOrder(expectedResult, result);
+
+ }
+
+ private AbstractReplicationStrategy simpleStrategy(TokenMetadata tmd)
+ {
+ IEndpointSnitch snitch = new AbstractEndpointSnitch()
+ {
+ public int compareEndpoints(InetAddressAndPort target, Replica r1, Replica r2)
+ {
+ return 0;
+ }
+
+ public String getRack(InetAddressAndPort endpoint)
+ {
+ return "R1";
+ }
+
+ public String getDatacenter(InetAddressAndPort endpoint)
+ {
+ return "DC1";
+ }
+ };
+
+ return new SimpleStrategy("MoveTransientTest",
+ tmd,
+ snitch,
+ com.google.common.collect.ImmutableMap.of("replication_factor", "3/1"));
+ }
+
+}
diff --git a/test/unit/org/apache/cassandra/service/ClientWarningsTest.java b/test/unit/org/apache/cassandra/service/ClientWarningsTest.java
index e939df0..3ae49ed 100644
--- a/test/unit/org/apache/cassandra/service/ClientWarningsTest.java
+++ b/test/unit/org/apache/cassandra/service/ClientWarningsTest.java
@@ -51,7 +51,7 @@
try (SimpleClient client = new SimpleClient(nativeAddr.getHostAddress(), nativePort, ProtocolVersion.V4))
{
- client.connect(false);
+ client.connect(false, false);
QueryMessage query = new QueryMessage(createBatchStatement2(1), QueryOptions.DEFAULT);
Message.Response resp = client.execute(query);
@@ -70,7 +70,7 @@
try (SimpleClient client = new SimpleClient(nativeAddr.getHostAddress(), nativePort, ProtocolVersion.V4))
{
- client.connect(false);
+ client.connect(false, false);
QueryMessage query = new QueryMessage(createBatchStatement2(DatabaseDescriptor.getBatchSizeWarnThreshold() / 2 + 1), QueryOptions.DEFAULT);
Message.Response resp = client.execute(query);
@@ -90,7 +90,7 @@
try (SimpleClient client = new SimpleClient(nativeAddr.getHostAddress(), nativePort, ProtocolVersion.V4))
{
- client.connect(false);
+ client.connect(false, false);
for (int i = 0; i < iterations; i++)
{
@@ -130,7 +130,7 @@
try (SimpleClient client = new SimpleClient(nativeAddr.getHostAddress(), nativePort, ProtocolVersion.V3))
{
- client.connect(false);
+ client.connect(false, false);
QueryMessage query = new QueryMessage(createBatchStatement(DatabaseDescriptor.getBatchSizeWarnThreshold()), QueryOptions.DEFAULT);
Message.Response resp = client.execute(query);
diff --git a/test/unit/org/apache/cassandra/service/DataResolverTest.java b/test/unit/org/apache/cassandra/service/DataResolverTest.java
deleted file mode 100644
index 2b1e095..0000000
--- a/test/unit/org/apache/cassandra/service/DataResolverTest.java
+++ /dev/null
@@ -1,1122 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.cassandra.service;
-
-import java.net.InetAddress;
-import java.net.UnknownHostException;
-import java.nio.ByteBuffer;
-import java.util.*;
-
-import com.google.common.collect.Iterators;
-import com.google.common.collect.Sets;
-import org.junit.*;
-
-import org.apache.cassandra.SchemaLoader;
-import org.apache.cassandra.Util;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.cql3.ColumnIdentifier;
-import org.apache.cassandra.db.*;
-import org.apache.cassandra.db.marshal.ByteType;
-import org.apache.cassandra.db.marshal.IntegerType;
-import org.apache.cassandra.db.marshal.MapType;
-import org.apache.cassandra.db.rows.*;
-import org.apache.cassandra.db.marshal.AsciiType;
-import org.apache.cassandra.db.marshal.BytesType;
-import org.apache.cassandra.db.partitions.*;
-import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.net.*;
-import org.apache.cassandra.schema.KeyspaceParams;
-import org.apache.cassandra.utils.ByteBufferUtil;
-import org.apache.cassandra.utils.FBUtilities;
-
-import static org.apache.cassandra.Util.assertClustering;
-import static org.apache.cassandra.Util.assertColumn;
-import static org.apache.cassandra.Util.assertColumns;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-import static org.apache.cassandra.db.ClusteringBound.Kind;
-
-public class DataResolverTest
-{
- public static final String KEYSPACE1 = "DataResolverTest";
- public static final String CF_STANDARD = "Standard1";
- public static final String CF_COLLECTION = "Collection1";
-
- // counter to generate the last byte of the respondent's address in a ReadResponse message
- private int addressSuffix = 10;
-
- private DecoratedKey dk;
- private Keyspace ks;
- private ColumnFamilyStore cfs;
- private ColumnFamilyStore cfs2;
- private CFMetaData cfm;
- private CFMetaData cfm2;
- private ColumnDefinition m;
- private int nowInSec;
- private ReadCommand command;
- private MessageRecorder messageRecorder;
-
-
- @BeforeClass
- public static void defineSchema() throws ConfigurationException
- {
- DatabaseDescriptor.daemonInitialization();
- CFMetaData cfMetadata = CFMetaData.Builder.create(KEYSPACE1, CF_STANDARD)
- .addPartitionKey("key", BytesType.instance)
- .addClusteringColumn("col1", AsciiType.instance)
- .addRegularColumn("c1", AsciiType.instance)
- .addRegularColumn("c2", AsciiType.instance)
- .addRegularColumn("one", AsciiType.instance)
- .addRegularColumn("two", AsciiType.instance)
- .build();
-
- CFMetaData cfMetaData2 = CFMetaData.Builder.create(KEYSPACE1, CF_COLLECTION)
- .addPartitionKey("k", ByteType.instance)
- .addRegularColumn("m", MapType.getInstance(IntegerType.instance, IntegerType.instance, true))
- .build();
- SchemaLoader.prepareServer();
- SchemaLoader.createKeyspace(KEYSPACE1,
- KeyspaceParams.simple(1),
- cfMetadata, cfMetaData2);
- }
-
- @Before
- public void setup()
- {
- dk = Util.dk("key1");
- ks = Keyspace.open(KEYSPACE1);
- cfs = ks.getColumnFamilyStore(CF_STANDARD);
- cfm = cfs.metadata;
- cfs2 = ks.getColumnFamilyStore(CF_COLLECTION);
- cfm2 = cfs2.metadata;
- m = cfm2.getColumnDefinition(new ColumnIdentifier("m", false));
-
- nowInSec = FBUtilities.nowInSeconds();
- command = Util.cmd(cfs, dk).withNowInSeconds(nowInSec).build();
- }
-
- @Before
- public void injectMessageSink()
- {
- // install an IMessageSink to capture all messages
- // so we can inspect them during tests
- messageRecorder = new MessageRecorder();
- MessagingService.instance().addMessageSink(messageRecorder);
- }
-
- @After
- public void removeMessageSink()
- {
- // should be unnecessary, but good housekeeping
- MessagingService.instance().clearMessageSinks();
- }
-
- /**
- * Checks that the provided data resolver has the expected number of repair futures created.
- * This method also "release" those future by faking replica responses to those repair, which is necessary or
- * every test would timeout when closing the result of resolver.resolve(), since it waits on those futures.
- */
- private void assertRepairFuture(DataResolver resolver, int expectedRepairs)
- {
- assertEquals(expectedRepairs, resolver.repairResults.size());
-
- // Signal all future. We pass a completely fake response message, but it doesn't matter as we just want
- // AsyncOneResponse to signal success, and it only cares about a non-null MessageIn (it collects the payload).
- for (AsyncOneResponse<?> future : resolver.repairResults)
- future.response(MessageIn.create(null, null, null, null, -1));
- }
-
- @Test
- public void testResolveNewerSingleRow() throws UnknownHostException
- {
- DataResolver resolver = new DataResolver(ks, command, ConsistencyLevel.ALL, 2, System.nanoTime());
- InetAddress peer1 = peer();
- resolver.preprocess(readResponseMessage(peer1, iter(new RowUpdateBuilder(cfm, nowInSec, 0L, dk).clustering("1")
- .add("c1", "v1")
- .buildUpdate())));
- InetAddress peer2 = peer();
- resolver.preprocess(readResponseMessage(peer2, iter(new RowUpdateBuilder(cfm, nowInSec, 1L, dk).clustering("1")
- .add("c1", "v2")
- .buildUpdate())));
-
- try(PartitionIterator data = resolver.resolve())
- {
- try (RowIterator rows = Iterators.getOnlyElement(data))
- {
- Row row = Iterators.getOnlyElement(rows);
- assertColumns(row, "c1");
- assertColumn(cfm, row, "c1", "v2", 1);
- }
- assertRepairFuture(resolver, 1);
- }
-
- assertEquals(1, messageRecorder.sent.size());
- // peer 1 just needs to repair with the row from peer 2
- MessageOut msg = getSentMessage(peer1);
- assertRepairMetadata(msg);
- assertRepairContainsNoDeletions(msg);
- assertRepairContainsColumn(msg, "1", "c1", "v2", 1);
- }
-
- @Test
- public void testResolveDisjointSingleRow()
- {
- DataResolver resolver = new DataResolver(ks, command, ConsistencyLevel.ALL, 2, System.nanoTime());
- InetAddress peer1 = peer();
- resolver.preprocess(readResponseMessage(peer1, iter(new RowUpdateBuilder(cfm, nowInSec, 0L, dk).clustering("1")
- .add("c1", "v1")
- .buildUpdate())));
-
- InetAddress peer2 = peer();
- resolver.preprocess(readResponseMessage(peer2, iter(new RowUpdateBuilder(cfm, nowInSec, 1L, dk).clustering("1")
- .add("c2", "v2")
- .buildUpdate())));
-
- try(PartitionIterator data = resolver.resolve())
- {
- try (RowIterator rows = Iterators.getOnlyElement(data))
- {
- Row row = Iterators.getOnlyElement(rows);
- assertColumns(row, "c1", "c2");
- assertColumn(cfm, row, "c1", "v1", 0);
- assertColumn(cfm, row, "c2", "v2", 1);
- }
- assertRepairFuture(resolver, 2);
- }
-
- assertEquals(2, messageRecorder.sent.size());
- // each peer needs to repair with each other's column
- MessageOut msg = getSentMessage(peer1);
- assertRepairMetadata(msg);
- assertRepairContainsColumn(msg, "1", "c2", "v2", 1);
-
- msg = getSentMessage(peer2);
- assertRepairMetadata(msg);
- assertRepairContainsColumn(msg, "1", "c1", "v1", 0);
- }
-
- @Test
- public void testResolveDisjointMultipleRows() throws UnknownHostException
- {
-
- DataResolver resolver = new DataResolver(ks, command, ConsistencyLevel.ALL, 2, System.nanoTime());
- InetAddress peer1 = peer();
- resolver.preprocess(readResponseMessage(peer1, iter(new RowUpdateBuilder(cfm, nowInSec, 0L, dk).clustering("1")
- .add("c1", "v1")
- .buildUpdate())));
- InetAddress peer2 = peer();
- resolver.preprocess(readResponseMessage(peer2, iter(new RowUpdateBuilder(cfm, nowInSec, 1L, dk).clustering("2")
- .add("c2", "v2")
- .buildUpdate())));
-
- try (PartitionIterator data = resolver.resolve())
- {
- try (RowIterator rows = data.next())
- {
- // We expect the resolved superset to contain both rows
- Row row = rows.next();
- assertClustering(cfm, row, "1");
- assertColumns(row, "c1");
- assertColumn(cfm, row, "c1", "v1", 0);
-
- row = rows.next();
- assertClustering(cfm, row, "2");
- assertColumns(row, "c2");
- assertColumn(cfm, row, "c2", "v2", 1);
-
- assertFalse(rows.hasNext());
- assertFalse(data.hasNext());
- }
- assertRepairFuture(resolver, 2);
- }
-
- assertEquals(2, messageRecorder.sent.size());
- // each peer needs to repair the row from the other
- MessageOut msg = getSentMessage(peer1);
- assertRepairMetadata(msg);
- assertRepairContainsNoDeletions(msg);
- assertRepairContainsColumn(msg, "2", "c2", "v2", 1);
-
- msg = getSentMessage(peer2);
- assertRepairMetadata(msg);
- assertRepairContainsNoDeletions(msg);
- assertRepairContainsColumn(msg, "1", "c1", "v1", 0);
- }
-
- @Test
- public void testResolveDisjointMultipleRowsWithRangeTombstones()
- {
- DataResolver resolver = new DataResolver(ks, command, ConsistencyLevel.ALL, 4, System.nanoTime());
-
- RangeTombstone tombstone1 = tombstone("1", "11", 1, nowInSec);
- RangeTombstone tombstone2 = tombstone("3", "31", 1, nowInSec);
- PartitionUpdate update = new RowUpdateBuilder(cfm, nowInSec, 1L, dk).addRangeTombstone(tombstone1)
- .addRangeTombstone(tombstone2)
- .buildUpdate();
-
- InetAddress peer1 = peer();
- UnfilteredPartitionIterator iter1 = iter(new RowUpdateBuilder(cfm, nowInSec, 1L, dk).addRangeTombstone(tombstone1)
- .addRangeTombstone(tombstone2)
- .buildUpdate());
- resolver.preprocess(readResponseMessage(peer1, iter1));
- // not covered by any range tombstone
- InetAddress peer2 = peer();
- UnfilteredPartitionIterator iter2 = iter(new RowUpdateBuilder(cfm, nowInSec, 0L, dk).clustering("0")
- .add("c1", "v0")
- .buildUpdate());
- resolver.preprocess(readResponseMessage(peer2, iter2));
- // covered by a range tombstone
- InetAddress peer3 = peer();
- UnfilteredPartitionIterator iter3 = iter(new RowUpdateBuilder(cfm, nowInSec, 0L, dk).clustering("10")
- .add("c2", "v1")
- .buildUpdate());
- resolver.preprocess(readResponseMessage(peer3, iter3));
- // range covered by rt, but newer
- InetAddress peer4 = peer();
- UnfilteredPartitionIterator iter4 = iter(new RowUpdateBuilder(cfm, nowInSec, 2L, dk).clustering("3")
- .add("one", "A")
- .buildUpdate());
- resolver.preprocess(readResponseMessage(peer4, iter4));
- try (PartitionIterator data = resolver.resolve())
- {
- try (RowIterator rows = data.next())
- {
- Row row = rows.next();
- assertClustering(cfm, row, "0");
- assertColumns(row, "c1");
- assertColumn(cfm, row, "c1", "v0", 0);
-
- row = rows.next();
- assertClustering(cfm, row, "3");
- assertColumns(row, "one");
- assertColumn(cfm, row, "one", "A", 2);
-
- assertFalse(rows.hasNext());
- }
- assertRepairFuture(resolver, 4);
- }
-
- assertEquals(4, messageRecorder.sent.size());
- // peer1 needs the rows from peers 2 and 4
- MessageOut msg = getSentMessage(peer1);
- assertRepairMetadata(msg);
- assertRepairContainsNoDeletions(msg);
- assertRepairContainsColumn(msg, "0", "c1", "v0", 0);
- assertRepairContainsColumn(msg, "3", "one", "A", 2);
-
- // peer2 needs to get the row from peer4 and the RTs
- msg = getSentMessage(peer2);
- assertRepairMetadata(msg);
- assertRepairContainsDeletions(msg, null, tombstone1, tombstone2);
- assertRepairContainsColumn(msg, "3", "one", "A", 2);
-
- // peer 3 needs both rows and the RTs
- msg = getSentMessage(peer3);
- assertRepairMetadata(msg);
- assertRepairContainsDeletions(msg, null, tombstone1, tombstone2);
- assertRepairContainsColumn(msg, "0", "c1", "v0", 0);
- assertRepairContainsColumn(msg, "3", "one", "A", 2);
-
- // peer4 needs the row from peer2 and the RTs
- msg = getSentMessage(peer4);
- assertRepairMetadata(msg);
- assertRepairContainsDeletions(msg, null, tombstone1, tombstone2);
- assertRepairContainsColumn(msg, "0", "c1", "v0", 0);
- }
-
- @Test
- public void testResolveWithOneEmpty()
- {
- DataResolver resolver = new DataResolver(ks, command, ConsistencyLevel.ALL, 2, System.nanoTime());
- InetAddress peer1 = peer();
- resolver.preprocess(readResponseMessage(peer1, iter(new RowUpdateBuilder(cfm, nowInSec, 1L, dk).clustering("1")
- .add("c2", "v2")
- .buildUpdate())));
- InetAddress peer2 = peer();
- resolver.preprocess(readResponseMessage(peer2, EmptyIterators.unfilteredPartition(cfm, false)));
-
- try(PartitionIterator data = resolver.resolve())
- {
- try (RowIterator rows = Iterators.getOnlyElement(data))
- {
- Row row = Iterators.getOnlyElement(rows);
- assertColumns(row, "c2");
- assertColumn(cfm, row, "c2", "v2", 1);
- }
- assertRepairFuture(resolver, 1);
- }
-
- assertEquals(1, messageRecorder.sent.size());
- // peer 2 needs the row from peer 1
- MessageOut msg = getSentMessage(peer2);
- assertRepairMetadata(msg);
- assertRepairContainsNoDeletions(msg);
- assertRepairContainsColumn(msg, "1", "c2", "v2", 1);
- }
-
- @Test
- public void testResolveWithBothEmpty()
- {
- DataResolver resolver = new DataResolver(ks, command, ConsistencyLevel.ALL, 2, System.nanoTime());
- resolver.preprocess(readResponseMessage(peer(), EmptyIterators.unfilteredPartition(cfm, false)));
- resolver.preprocess(readResponseMessage(peer(), EmptyIterators.unfilteredPartition(cfm, false)));
-
- try(PartitionIterator data = resolver.resolve())
- {
- assertFalse(data.hasNext());
- assertRepairFuture(resolver, 0);
- }
-
- assertTrue(messageRecorder.sent.isEmpty());
- }
-
- @Test
- public void testResolveDeleted()
- {
- DataResolver resolver = new DataResolver(ks, command, ConsistencyLevel.ALL, 2, System.nanoTime());
- // one response with columns timestamped before a delete in another response
- InetAddress peer1 = peer();
- resolver.preprocess(readResponseMessage(peer1, iter(new RowUpdateBuilder(cfm, nowInSec, 0L, dk).clustering("1")
- .add("one", "A")
- .buildUpdate())));
- InetAddress peer2 = peer();
- resolver.preprocess(readResponseMessage(peer2, fullPartitionDelete(cfm, dk, 1, nowInSec)));
-
- try (PartitionIterator data = resolver.resolve())
- {
- assertFalse(data.hasNext());
- assertRepairFuture(resolver, 1);
- }
-
- // peer1 should get the deletion from peer2
- assertEquals(1, messageRecorder.sent.size());
- MessageOut msg = getSentMessage(peer1);
- assertRepairMetadata(msg);
- assertRepairContainsDeletions(msg, new DeletionTime(1, nowInSec));
- assertRepairContainsNoColumns(msg);
- }
-
- @Test
- public void testResolveMultipleDeleted()
- {
- DataResolver resolver = new DataResolver(ks, command, ConsistencyLevel.ALL, 4, System.nanoTime());
- // deletes and columns with interleaved timestamp, with out of order return sequence
- InetAddress peer1 = peer();
- resolver.preprocess(readResponseMessage(peer1, fullPartitionDelete(cfm, dk, 0, nowInSec)));
- // these columns created after the previous deletion
- InetAddress peer2 = peer();
- resolver.preprocess(readResponseMessage(peer2, iter(new RowUpdateBuilder(cfm, nowInSec, 1L, dk).clustering("1")
- .add("one", "A")
- .add("two", "A")
- .buildUpdate())));
- //this column created after the next delete
- InetAddress peer3 = peer();
- resolver.preprocess(readResponseMessage(peer3, iter(new RowUpdateBuilder(cfm, nowInSec, 3L, dk).clustering("1")
- .add("two", "B")
- .buildUpdate())));
- InetAddress peer4 = peer();
- resolver.preprocess(readResponseMessage(peer4, fullPartitionDelete(cfm, dk, 2, nowInSec)));
-
- try(PartitionIterator data = resolver.resolve())
- {
- try (RowIterator rows = Iterators.getOnlyElement(data))
- {
- Row row = Iterators.getOnlyElement(rows);
- assertColumns(row, "two");
- assertColumn(cfm, row, "two", "B", 3);
- }
- assertRepairFuture(resolver, 4);
- }
-
- // peer 1 needs to get the partition delete from peer 4 and the row from peer 3
- assertEquals(4, messageRecorder.sent.size());
- MessageOut msg = getSentMessage(peer1);
- assertRepairMetadata(msg);
- assertRepairContainsDeletions(msg, new DeletionTime(2, nowInSec));
- assertRepairContainsColumn(msg, "1", "two", "B", 3);
-
- // peer 2 needs the deletion from peer 4 and the row from peer 3
- msg = getSentMessage(peer2);
- assertRepairMetadata(msg);
- assertRepairContainsDeletions(msg, new DeletionTime(2, nowInSec));
- assertRepairContainsColumn(msg, "1", "two", "B", 3);
-
- // peer 3 needs just the deletion from peer 4
- msg = getSentMessage(peer3);
- assertRepairMetadata(msg);
- assertRepairContainsDeletions(msg, new DeletionTime(2, nowInSec));
- assertRepairContainsNoColumns(msg);
-
- // peer 4 needs just the row from peer 3
- msg = getSentMessage(peer4);
- assertRepairMetadata(msg);
- assertRepairContainsNoDeletions(msg);
- assertRepairContainsColumn(msg, "1", "two", "B", 3);
- }
-
- @Test
- public void testResolveRangeTombstonesOnBoundaryRightWins() throws UnknownHostException
- {
- resolveRangeTombstonesOnBoundary(1, 2);
- }
-
- @Test
- public void testResolveRangeTombstonesOnBoundaryLeftWins() throws UnknownHostException
- {
- resolveRangeTombstonesOnBoundary(2, 1);
- }
-
- @Test
- public void testResolveRangeTombstonesOnBoundarySameTimestamp() throws UnknownHostException
- {
- resolveRangeTombstonesOnBoundary(1, 1);
- }
-
- /*
- * We want responses to merge on tombstone boundary. So we'll merge 2 "streams":
- * 1: [1, 2)(3, 4](5, 6] 2
- * 2: [2, 3][4, 5) 1
- * which tests all combination of open/close boundaries (open/close, close/open, open/open, close/close).
- *
- * Note that, because DataResolver returns a "filtered" iterator, it should resolve into an empty iterator.
- * However, what should be sent to each source depends on the exact on the timestamps of each tombstones and we
- * test a few combination.
- */
- private void resolveRangeTombstonesOnBoundary(long timestamp1, long timestamp2)
- {
- DataResolver resolver = new DataResolver(ks, command, ConsistencyLevel.ALL, 2, System.nanoTime());
- InetAddress peer1 = peer();
- InetAddress peer2 = peer();
-
- // 1st "stream"
- RangeTombstone one_two = tombstone("1", true , "2", false, timestamp1, nowInSec);
- RangeTombstone three_four = tombstone("3", false, "4", true , timestamp1, nowInSec);
- RangeTombstone five_six = tombstone("5", false, "6", true , timestamp1, nowInSec);
- UnfilteredPartitionIterator iter1 = iter(new RowUpdateBuilder(cfm, nowInSec, 1L, dk).addRangeTombstone(one_two)
- .addRangeTombstone(three_four)
- .addRangeTombstone(five_six)
- .buildUpdate());
-
- // 2nd "stream"
- RangeTombstone two_three = tombstone("2", true, "3", true , timestamp2, nowInSec);
- RangeTombstone four_five = tombstone("4", true, "5", false, timestamp2, nowInSec);
- UnfilteredPartitionIterator iter2 = iter(new RowUpdateBuilder(cfm, nowInSec, 1L, dk).addRangeTombstone(two_three)
- .addRangeTombstone(four_five)
- .buildUpdate());
-
- resolver.preprocess(readResponseMessage(peer1, iter1));
- resolver.preprocess(readResponseMessage(peer2, iter2));
-
- // No results, we've only reconciled tombstones.
- try (PartitionIterator data = resolver.resolve())
- {
- assertFalse(data.hasNext());
- assertRepairFuture(resolver, 2);
- }
-
- assertEquals(2, messageRecorder.sent.size());
-
- MessageOut msg1 = getSentMessage(peer1);
- assertRepairMetadata(msg1);
- assertRepairContainsNoColumns(msg1);
-
- MessageOut msg2 = getSentMessage(peer2);
- assertRepairMetadata(msg2);
- assertRepairContainsNoColumns(msg2);
-
- // Both streams are mostly complementary, so they will roughly get the ranges of the other stream. One subtlety is
- // around the value "4" however, as it's included by both stream.
- // So for a given stream, unless the other stream has a strictly higher timestamp, the value 4 will be excluded
- // from whatever range it receives as repair since the stream already covers it.
-
- // Message to peer1 contains peer2 ranges
- assertRepairContainsDeletions(msg1, null, two_three, withExclusiveStartIf(four_five, timestamp1 >= timestamp2));
-
- // Message to peer2 contains peer1 ranges
- assertRepairContainsDeletions(msg2, null, one_two, withExclusiveEndIf(three_four, timestamp2 >= timestamp1), five_six);
- }
-
- /**
- * Test cases where a boundary of a source is covered by another source deletion and timestamp on one or both side
- * of the boundary are equal to the "merged" deletion.
- * This is a test for CASSANDRA-13237 to make sure we handle this case properly.
- */
- @Test
- public void testRepairRangeTombstoneBoundary() throws UnknownHostException
- {
- testRepairRangeTombstoneBoundary(1, 0, 1);
- messageRecorder.sent.clear();
- testRepairRangeTombstoneBoundary(1, 1, 0);
- messageRecorder.sent.clear();
- testRepairRangeTombstoneBoundary(1, 1, 1);
- }
-
- /**
- * Test for CASSANDRA-13237, checking we don't fail (and handle correctly) the case where a RT boundary has the
- * same deletion on both side (while is useless but could be created by legacy code pre-CASSANDRA-13237 and could
- * thus still be sent).
- */
- private void testRepairRangeTombstoneBoundary(int timestamp1, int timestamp2, int timestamp3) throws UnknownHostException
- {
- DataResolver resolver = new DataResolver(ks, command, ConsistencyLevel.ALL, 2, System.nanoTime());
- InetAddress peer1 = peer();
- InetAddress peer2 = peer();
-
- // 1st "stream"
- RangeTombstone one_nine = tombstone("0", true , "9", true, timestamp1, nowInSec);
- UnfilteredPartitionIterator iter1 = iter(new RowUpdateBuilder(cfm, nowInSec, 1L, dk)
- .addRangeTombstone(one_nine)
- .buildUpdate());
-
- // 2nd "stream" (build more manually to ensure we have the boundary we want)
- RangeTombstoneBoundMarker open_one = marker("0", true, true, timestamp2, nowInSec);
- RangeTombstoneBoundaryMarker boundary_five = boundary("5", false, timestamp2, nowInSec, timestamp3, nowInSec);
- RangeTombstoneBoundMarker close_nine = marker("9", false, true, timestamp3, nowInSec);
- UnfilteredPartitionIterator iter2 = iter(dk, open_one, boundary_five, close_nine);
-
- resolver.preprocess(readResponseMessage(peer1, iter1));
- resolver.preprocess(readResponseMessage(peer2, iter2));
-
- boolean shouldHaveRepair = timestamp1 != timestamp2 || timestamp1 != timestamp3;
-
- // No results, we've only reconciled tombstones.
- try (PartitionIterator data = resolver.resolve())
- {
- assertFalse(data.hasNext());
- assertRepairFuture(resolver, shouldHaveRepair ? 1 : 0);
- }
-
- assertEquals(shouldHaveRepair? 1 : 0, messageRecorder.sent.size());
-
- if (!shouldHaveRepair)
- return;
-
- MessageOut msg = getSentMessage(peer2);
- assertRepairMetadata(msg);
- assertRepairContainsNoColumns(msg);
-
- RangeTombstone expected = timestamp1 != timestamp2
- // We've repaired the 1st part
- ? tombstone("0", true, "5", false, timestamp1, nowInSec)
- // We've repaired the 2nd part
- : tombstone("5", true, "9", true, timestamp1, nowInSec);
- assertRepairContainsDeletions(msg, null, expected);
- }
-
- /**
- * Test for CASSANDRA-13719: tests that having a partition deletion shadow a range tombstone on another source
- * doesn't trigger an assertion error.
- */
- @Test
- public void testRepairRangeTombstoneWithPartitionDeletion()
- {
- DataResolver resolver = new DataResolver(ks, command, ConsistencyLevel.ALL, 2, System.nanoTime());
- InetAddress peer1 = peer();
- InetAddress peer2 = peer();
-
- // 1st "stream": just a partition deletion
- UnfilteredPartitionIterator iter1 = iter(PartitionUpdate.fullPartitionDelete(cfm, dk, 10, nowInSec));
-
- // 2nd "stream": a range tombstone that is covered by the 1st stream
- RangeTombstone rt = tombstone("0", true , "10", true, 5, nowInSec);
- UnfilteredPartitionIterator iter2 = iter(new RowUpdateBuilder(cfm, nowInSec, 1L, dk)
- .addRangeTombstone(rt)
- .buildUpdate());
-
- resolver.preprocess(readResponseMessage(peer1, iter1));
- resolver.preprocess(readResponseMessage(peer2, iter2));
-
- // No results, we've only reconciled tombstones.
- try (PartitionIterator data = resolver.resolve())
- {
- assertFalse(data.hasNext());
- // 2nd stream should get repaired
- assertRepairFuture(resolver, 1);
- }
-
- assertEquals(1, messageRecorder.sent.size());
-
- MessageOut msg = getSentMessage(peer2);
- assertRepairMetadata(msg);
- assertRepairContainsNoColumns(msg);
-
- assertRepairContainsDeletions(msg, new DeletionTime(10, nowInSec));
- }
-
- /**
- * Additional test for CASSANDRA-13719: tests the case where a partition deletion doesn't shadow a range tombstone.
- */
- @Test
- public void testRepairRangeTombstoneWithPartitionDeletion2()
- {
- DataResolver resolver = new DataResolver(ks, command, ConsistencyLevel.ALL, 2, System.nanoTime());
- InetAddress peer1 = peer();
- InetAddress peer2 = peer();
-
- // 1st "stream": a partition deletion and a range tombstone
- RangeTombstone rt1 = tombstone("0", true , "9", true, 11, nowInSec);
- PartitionUpdate upd1 = new RowUpdateBuilder(cfm, nowInSec, 1L, dk)
- .addRangeTombstone(rt1)
- .buildUpdate();
- ((MutableDeletionInfo)upd1.deletionInfo()).add(new DeletionTime(10, nowInSec));
- UnfilteredPartitionIterator iter1 = iter(upd1);
-
- // 2nd "stream": a range tombstone that is covered by the other stream rt
- RangeTombstone rt2 = tombstone("2", true , "3", true, 11, nowInSec);
- RangeTombstone rt3 = tombstone("4", true , "5", true, 10, nowInSec);
- UnfilteredPartitionIterator iter2 = iter(new RowUpdateBuilder(cfm, nowInSec, 1L, dk)
- .addRangeTombstone(rt2)
- .addRangeTombstone(rt3)
- .buildUpdate());
-
- resolver.preprocess(readResponseMessage(peer1, iter1));
- resolver.preprocess(readResponseMessage(peer2, iter2));
-
- // No results, we've only reconciled tombstones.
- try (PartitionIterator data = resolver.resolve())
- {
- assertFalse(data.hasNext());
- // 2nd stream should get repaired
- assertRepairFuture(resolver, 1);
- }
-
- assertEquals(1, messageRecorder.sent.size());
-
- MessageOut msg = getSentMessage(peer2);
- assertRepairMetadata(msg);
- assertRepairContainsNoColumns(msg);
-
- // 2nd stream should get both the partition deletion, as well as the part of the 1st stream RT that it misses
- assertRepairContainsDeletions(msg, new DeletionTime(10, nowInSec),
- tombstone("0", true, "2", false, 11, nowInSec),
- tombstone("3", false, "9", true, 11, nowInSec));
- }
-
- // Forces the start to be exclusive if the condition holds
- private static RangeTombstone withExclusiveStartIf(RangeTombstone rt, boolean condition)
- {
- if (!condition)
- return rt;
-
- Slice slice = rt.deletedSlice();
- ClusteringBound newStart = ClusteringBound.create(Kind.EXCL_START_BOUND, slice.start().getRawValues());
- return condition
- ? new RangeTombstone(Slice.make(newStart, slice.end()), rt.deletionTime())
- : rt;
- }
-
- // Forces the end to be exclusive if the condition holds
- private static RangeTombstone withExclusiveEndIf(RangeTombstone rt, boolean condition)
- {
- if (!condition)
- return rt;
-
- Slice slice = rt.deletedSlice();
- ClusteringBound newEnd = ClusteringBound.create(Kind.EXCL_END_BOUND, slice.end().getRawValues());
- return condition
- ? new RangeTombstone(Slice.make(slice.start(), newEnd), rt.deletionTime())
- : rt;
- }
-
- private static ByteBuffer bb(int b)
- {
- return ByteBufferUtil.bytes(b);
- }
-
- private Cell mapCell(int k, int v, long ts)
- {
- return BufferCell.live(m, ts, bb(v), CellPath.create(bb(k)));
- }
-
- @Test
- public void testResolveComplexDelete()
- {
- ReadCommand cmd = Util.cmd(cfs2, dk).withNowInSeconds(nowInSec).build();
- DataResolver resolver = new DataResolver(ks, cmd, ConsistencyLevel.ALL, 2, System.nanoTime());
-
- long[] ts = {100, 200};
-
- Row.Builder builder = BTreeRow.unsortedBuilder(nowInSec);
- builder.newRow(Clustering.EMPTY);
- builder.addComplexDeletion(m, new DeletionTime(ts[0] - 1, nowInSec));
- builder.addCell(mapCell(0, 0, ts[0]));
-
- InetAddress peer1 = peer();
- resolver.preprocess(readResponseMessage(peer1, iter(PartitionUpdate.singleRowUpdate(cfm2, dk, builder.build())), cmd));
-
- builder.newRow(Clustering.EMPTY);
- DeletionTime expectedCmplxDelete = new DeletionTime(ts[1] - 1, nowInSec);
- builder.addComplexDeletion(m, expectedCmplxDelete);
- Cell expectedCell = mapCell(1, 1, ts[1]);
- builder.addCell(expectedCell);
-
- InetAddress peer2 = peer();
- resolver.preprocess(readResponseMessage(peer2, iter(PartitionUpdate.singleRowUpdate(cfm2, dk, builder.build())), cmd));
-
- try(PartitionIterator data = resolver.resolve())
- {
- try (RowIterator rows = Iterators.getOnlyElement(data))
- {
- Row row = Iterators.getOnlyElement(rows);
- assertColumns(row, "m");
- Assert.assertNull(row.getCell(m, CellPath.create(bb(0))));
- Assert.assertNotNull(row.getCell(m, CellPath.create(bb(1))));
- }
- assertRepairFuture(resolver, 1);
- }
-
- MessageOut<Mutation> msg;
- msg = getSentMessage(peer1);
- Iterator<Row> rowIter = msg.payload.getPartitionUpdate(cfm2.cfId).iterator();
- assertTrue(rowIter.hasNext());
- Row row = rowIter.next();
- assertFalse(rowIter.hasNext());
-
- ComplexColumnData cd = row.getComplexColumnData(m);
-
- assertEquals(Collections.singleton(expectedCell), Sets.newHashSet(cd));
- assertEquals(expectedCmplxDelete, cd.complexDeletion());
-
- Assert.assertNull(messageRecorder.sent.get(peer2));
- }
-
- @Test
- public void testResolveDeletedCollection()
- {
-
- ReadCommand cmd = Util.cmd(cfs2, dk).withNowInSeconds(nowInSec).build();
- DataResolver resolver = new DataResolver(ks, cmd, ConsistencyLevel.ALL, 2, System.nanoTime());
-
- long[] ts = {100, 200};
-
- Row.Builder builder = BTreeRow.unsortedBuilder(nowInSec);
- builder.newRow(Clustering.EMPTY);
- builder.addComplexDeletion(m, new DeletionTime(ts[0] - 1, nowInSec));
- builder.addCell(mapCell(0, 0, ts[0]));
-
- InetAddress peer1 = peer();
- resolver.preprocess(readResponseMessage(peer1, iter(PartitionUpdate.singleRowUpdate(cfm2, dk, builder.build())), cmd));
-
- builder.newRow(Clustering.EMPTY);
- DeletionTime expectedCmplxDelete = new DeletionTime(ts[1] - 1, nowInSec);
- builder.addComplexDeletion(m, expectedCmplxDelete);
-
- InetAddress peer2 = peer();
- resolver.preprocess(readResponseMessage(peer2, iter(PartitionUpdate.singleRowUpdate(cfm2, dk, builder.build())), cmd));
-
- try(PartitionIterator data = resolver.resolve())
- {
- assertFalse(data.hasNext());
- assertRepairFuture(resolver, 1);
- }
-
- MessageOut<Mutation> msg;
- msg = getSentMessage(peer1);
- Iterator<Row> rowIter = msg.payload.getPartitionUpdate(cfm2.cfId).iterator();
- assertTrue(rowIter.hasNext());
- Row row = rowIter.next();
- assertFalse(rowIter.hasNext());
-
- ComplexColumnData cd = row.getComplexColumnData(m);
-
- assertEquals(Collections.emptySet(), Sets.newHashSet(cd));
- assertEquals(expectedCmplxDelete, cd.complexDeletion());
-
- Assert.assertNull(messageRecorder.sent.get(peer2));
- }
-
- @Test
- public void testResolveNewCollection()
- {
- ReadCommand cmd = Util.cmd(cfs2, dk).withNowInSeconds(nowInSec).build();
- DataResolver resolver = new DataResolver(ks, cmd, ConsistencyLevel.ALL, 2, System.nanoTime());
-
- long[] ts = {100, 200};
-
- // map column
- Row.Builder builder = BTreeRow.unsortedBuilder(nowInSec);
- builder.newRow(Clustering.EMPTY);
- DeletionTime expectedCmplxDelete = new DeletionTime(ts[0] - 1, nowInSec);
- builder.addComplexDeletion(m, expectedCmplxDelete);
- Cell expectedCell = mapCell(0, 0, ts[0]);
- builder.addCell(expectedCell);
-
- // empty map column
- InetAddress peer1 = peer();
- resolver.preprocess(readResponseMessage(peer1, iter(PartitionUpdate.singleRowUpdate(cfm2, dk, builder.build())), cmd));
-
- InetAddress peer2 = peer();
- resolver.preprocess(readResponseMessage(peer2, iter(PartitionUpdate.emptyUpdate(cfm2, dk))));
-
- try(PartitionIterator data = resolver.resolve())
- {
- try (RowIterator rows = Iterators.getOnlyElement(data))
- {
- Row row = Iterators.getOnlyElement(rows);
- assertColumns(row, "m");
- ComplexColumnData cd = row.getComplexColumnData(m);
- assertEquals(Collections.singleton(expectedCell), Sets.newHashSet(cd));
- }
- assertRepairFuture(resolver, 1);
- }
-
- Assert.assertNull(messageRecorder.sent.get(peer1));
-
- MessageOut<Mutation> msg;
- msg = getSentMessage(peer2);
- Iterator<Row> rowIter = msg.payload.getPartitionUpdate(cfm2.cfId).iterator();
- assertTrue(rowIter.hasNext());
- Row row = rowIter.next();
- assertFalse(rowIter.hasNext());
-
- ComplexColumnData cd = row.getComplexColumnData(m);
-
- assertEquals(Sets.newHashSet(expectedCell), Sets.newHashSet(cd));
- assertEquals(expectedCmplxDelete, cd.complexDeletion());
- }
-
- @Test
- public void testResolveNewCollectionOverwritingDeleted()
- {
- ReadCommand cmd = Util.cmd(cfs2, dk).withNowInSeconds(nowInSec).build();
- DataResolver resolver = new DataResolver(ks, cmd, ConsistencyLevel.ALL, 2, System.nanoTime());
-
- long[] ts = {100, 200};
-
- // cleared map column
- Row.Builder builder = BTreeRow.unsortedBuilder(nowInSec);
- builder.newRow(Clustering.EMPTY);
- builder.addComplexDeletion(m, new DeletionTime(ts[0] - 1, nowInSec));
-
- InetAddress peer1 = peer();
- resolver.preprocess(readResponseMessage(peer1, iter(PartitionUpdate.singleRowUpdate(cfm2, dk, builder.build())), cmd));
-
- // newer, overwritten map column
- builder.newRow(Clustering.EMPTY);
- DeletionTime expectedCmplxDelete = new DeletionTime(ts[1] - 1, nowInSec);
- builder.addComplexDeletion(m, expectedCmplxDelete);
- Cell expectedCell = mapCell(1, 1, ts[1]);
- builder.addCell(expectedCell);
-
- InetAddress peer2 = peer();
- resolver.preprocess(readResponseMessage(peer2, iter(PartitionUpdate.singleRowUpdate(cfm2, dk, builder.build())), cmd));
-
- try(PartitionIterator data = resolver.resolve())
- {
- try (RowIterator rows = Iterators.getOnlyElement(data))
- {
- Row row = Iterators.getOnlyElement(rows);
- assertColumns(row, "m");
- ComplexColumnData cd = row.getComplexColumnData(m);
- assertEquals(Collections.singleton(expectedCell), Sets.newHashSet(cd));
- }
- assertRepairFuture(resolver, 1);
- }
-
- MessageOut<Mutation> msg;
- msg = getSentMessage(peer1);
- Row row = Iterators.getOnlyElement(msg.payload.getPartitionUpdate(cfm2.cfId).iterator());
-
- ComplexColumnData cd = row.getComplexColumnData(m);
-
- assertEquals(Collections.singleton(expectedCell), Sets.newHashSet(cd));
- assertEquals(expectedCmplxDelete, cd.complexDeletion());
-
- Assert.assertNull(messageRecorder.sent.get(peer2));
- }
-
- private InetAddress peer()
- {
- try
- {
- return InetAddress.getByAddress(new byte[]{ 127, 0, 0, (byte) addressSuffix++ });
- }
- catch (UnknownHostException e)
- {
- throw new RuntimeException(e);
- }
- }
-
- private MessageOut<Mutation> getSentMessage(InetAddress target)
- {
- MessageOut<Mutation> message = messageRecorder.sent.get(target);
- assertNotNull(String.format("No repair message was sent to %s", target), message);
- return message;
- }
-
- private void assertRepairContainsDeletions(MessageOut<Mutation> message,
- DeletionTime deletionTime,
- RangeTombstone...rangeTombstones)
- {
- PartitionUpdate update = ((Mutation)message.payload).getPartitionUpdates().iterator().next();
- DeletionInfo deletionInfo = update.deletionInfo();
- if (deletionTime != null)
- assertEquals(deletionTime, deletionInfo.getPartitionDeletion());
-
- assertEquals(rangeTombstones.length, deletionInfo.rangeCount());
- Iterator<RangeTombstone> ranges = deletionInfo.rangeIterator(false);
- int i = 0;
- while (ranges.hasNext())
- {
- RangeTombstone expected = rangeTombstones[i++];
- RangeTombstone actual = ranges.next();
- String msg = String.format("Expected %s, but got %s", expected.toString(cfm.comparator), actual.toString(cfm.comparator));
- assertEquals(msg, expected, actual);
- }
- }
-
- private void assertRepairContainsNoDeletions(MessageOut<Mutation> message)
- {
- PartitionUpdate update = ((Mutation)message.payload).getPartitionUpdates().iterator().next();
- assertTrue(update.deletionInfo().isLive());
- }
-
- private void assertRepairContainsColumn(MessageOut<Mutation> message,
- String clustering,
- String columnName,
- String value,
- long timestamp)
- {
- PartitionUpdate update = ((Mutation)message.payload).getPartitionUpdates().iterator().next();
- Row row = update.getRow(update.metadata().comparator.make(clustering));
- assertNotNull(row);
- assertColumn(cfm, row, columnName, value, timestamp);
- }
-
- private void assertRepairContainsNoColumns(MessageOut<Mutation> message)
- {
- PartitionUpdate update = ((Mutation)message.payload).getPartitionUpdates().iterator().next();
- assertFalse(update.iterator().hasNext());
- }
-
- private void assertRepairMetadata(MessageOut<Mutation> message)
- {
- assertEquals(MessagingService.Verb.READ_REPAIR, message.verb);
- PartitionUpdate update = ((Mutation)message.payload).getPartitionUpdates().iterator().next();
- assertEquals(update.metadata().ksName, cfm.ksName);
- assertEquals(update.metadata().cfName, cfm.cfName);
- }
-
-
- public MessageIn<ReadResponse> readResponseMessage(InetAddress from, UnfilteredPartitionIterator partitionIterator)
- {
- return readResponseMessage(from, partitionIterator, command);
-
- }
- public MessageIn<ReadResponse> readResponseMessage(InetAddress from, UnfilteredPartitionIterator partitionIterator, ReadCommand cmd)
- {
- return MessageIn.create(from,
- ReadResponse.createRemoteDataResponse(partitionIterator, cmd),
- Collections.EMPTY_MAP,
- MessagingService.Verb.REQUEST_RESPONSE,
- MessagingService.current_version);
- }
-
- private RangeTombstone tombstone(Object start, Object end, long markedForDeleteAt, int localDeletionTime)
- {
- return tombstone(start, true, end, true, markedForDeleteAt, localDeletionTime);
- }
-
- private RangeTombstone tombstone(Object start, boolean inclusiveStart, Object end, boolean inclusiveEnd, long markedForDeleteAt, int localDeletionTime)
- {
- ClusteringBound startBound = rtBound(start, true, inclusiveStart);
- ClusteringBound endBound = rtBound(end, false, inclusiveEnd);
- return new RangeTombstone(Slice.make(startBound, endBound), new DeletionTime(markedForDeleteAt, localDeletionTime));
- }
-
- private ClusteringBound rtBound(Object value, boolean isStart, boolean inclusive)
- {
- ClusteringBound.Kind kind = isStart
- ? (inclusive ? Kind.INCL_START_BOUND : Kind.EXCL_START_BOUND)
- : (inclusive ? Kind.INCL_END_BOUND : Kind.EXCL_END_BOUND);
-
- return ClusteringBound.create(kind, cfm.comparator.make(value).getRawValues());
- }
-
- private ClusteringBoundary rtBoundary(Object value, boolean inclusiveOnEnd)
- {
- ClusteringBound.Kind kind = inclusiveOnEnd
- ? Kind.INCL_END_EXCL_START_BOUNDARY
- : Kind.EXCL_END_INCL_START_BOUNDARY;
- return ClusteringBoundary.create(kind, cfm.comparator.make(value).getRawValues());
- }
-
- private RangeTombstoneBoundMarker marker(Object value, boolean isStart, boolean inclusive, long markedForDeleteAt, int localDeletionTime)
- {
- return new RangeTombstoneBoundMarker(rtBound(value, isStart, inclusive), new DeletionTime(markedForDeleteAt, localDeletionTime));
- }
-
- private RangeTombstoneBoundaryMarker boundary(Object value, boolean inclusiveOnEnd, long markedForDeleteAt1, int localDeletionTime1, long markedForDeleteAt2, int localDeletionTime2)
- {
- return new RangeTombstoneBoundaryMarker(rtBoundary(value, inclusiveOnEnd),
- new DeletionTime(markedForDeleteAt1, localDeletionTime1),
- new DeletionTime(markedForDeleteAt2, localDeletionTime2));
- }
-
- private UnfilteredPartitionIterator fullPartitionDelete(CFMetaData cfm, DecoratedKey dk, long timestamp, int nowInSec)
- {
- return new SingletonUnfilteredPartitionIterator(PartitionUpdate.fullPartitionDelete(cfm, dk, timestamp, nowInSec).unfilteredIterator(), false);
- }
-
- private static class MessageRecorder implements IMessageSink
- {
- Map<InetAddress, MessageOut> sent = new HashMap<>();
- public boolean allowOutgoingMessage(MessageOut message, int id, InetAddress to)
- {
- sent.put(to, message);
- return false;
- }
-
- public boolean allowIncomingMessage(MessageIn message, int id)
- {
- return false;
- }
- }
-
- private UnfilteredPartitionIterator iter(PartitionUpdate update)
- {
- return new SingletonUnfilteredPartitionIterator(update.unfilteredIterator(), false);
- }
-
- private UnfilteredPartitionIterator iter(DecoratedKey key, Unfiltered... unfiltereds)
- {
- SortedSet<Unfiltered> s = new TreeSet<>(cfm.comparator);
- Collections.addAll(s, unfiltereds);
- final Iterator<Unfiltered> iterator = s.iterator();
-
- UnfilteredRowIterator rowIter = new AbstractUnfilteredRowIterator(cfm,
- key,
- DeletionTime.LIVE,
- cfm.partitionColumns(),
- Rows.EMPTY_STATIC_ROW,
- false,
- EncodingStats.NO_STATS)
- {
- protected Unfiltered computeNext()
- {
- return iterator.hasNext() ? iterator.next() : endOfData();
- }
- };
- return new SingletonUnfilteredPartitionIterator(rowIter, false);
- }
-}
diff --git a/test/unit/org/apache/cassandra/service/EmbeddedCassandraServiceTest.java b/test/unit/org/apache/cassandra/service/EmbeddedCassandraServiceTest.java
deleted file mode 100644
index b89f01d..0000000
--- a/test/unit/org/apache/cassandra/service/EmbeddedCassandraServiceTest.java
+++ /dev/null
@@ -1,129 +0,0 @@
-/*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*/
-package org.apache.cassandra.service;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.nio.charset.CharacterCodingException;
-
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import org.apache.cassandra.SchemaLoader;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.db.marshal.AsciiType;
-import org.apache.cassandra.schema.KeyspaceParams;
-import org.apache.cassandra.thrift.*;
-import org.apache.cassandra.utils.ByteBufferUtil;
-import org.apache.thrift.TException;
-import org.apache.thrift.protocol.TBinaryProtocol;
-import org.apache.thrift.protocol.TProtocol;
-import org.apache.thrift.transport.TFramedTransport;
-import org.apache.thrift.transport.TSocket;
-import org.apache.thrift.transport.TTransport;
-import org.apache.thrift.transport.TTransportException;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-
-/**
- * Example how to use an embedded cassandra service.
- *
- * Tests connect to localhost:9160 when the embedded server is running.
- *
- */
-public class EmbeddedCassandraServiceTest
-{
-
- private static EmbeddedCassandraService cassandra;
- private static final String KEYSPACE1 = "EmbeddedCassandraServiceTest";
- private static final String CF_STANDARD = "Standard1";
-
- @BeforeClass
- public static void defineSchema() throws Exception
- {
- SchemaLoader.prepareServer();
- setup();
- SchemaLoader.createKeyspace(KEYSPACE1,
- KeyspaceParams.simple(1),
- CFMetaData.Builder.create(KEYSPACE1, CF_STANDARD, true, false, false)
- .addPartitionKey("pk", AsciiType.instance)
- .addClusteringColumn("ck", AsciiType.instance)
- .addRegularColumn("val", AsciiType.instance)
- .build());
- }
-
- /**
- * Set embedded cassandra up and spawn it in a new thread.
- *
- * @throws TTransportException
- * @throws IOException
- * @throws InterruptedException
- */
- public static void setup() throws TTransportException, IOException, InterruptedException
- {
- // unique ks / cfs mean no need to clear the schema
- cassandra = new EmbeddedCassandraService();
- cassandra.start();
- }
-
- @Test
- public void testEmbeddedCassandraService()
- throws AuthenticationException, AuthorizationException, InvalidRequestException, UnavailableException, TimedOutException, TException, NotFoundException, CharacterCodingException
- {
- Cassandra.Client client = getClient();
- client.set_keyspace(KEYSPACE1);
-
- ByteBuffer key_user_id = ByteBufferUtil.bytes("1");
-
- long timestamp = System.currentTimeMillis();
- ColumnPath cp = new ColumnPath("Standard1");
- ColumnParent par = new ColumnParent("Standard1");
- cp.column = ByteBufferUtil.bytes("name");
-
- // insert
- client.insert(key_user_id,
- par,
- new Column(ByteBufferUtil.bytes("name")).setValue(ByteBufferUtil.bytes("Ran")).setTimestamp(timestamp),
- ConsistencyLevel.ONE);
-
- // read
- ColumnOrSuperColumn got = client.get(key_user_id, cp, ConsistencyLevel.ONE);
-
- // assert
- assertNotNull("Got a null ColumnOrSuperColumn", got);
- assertEquals("Ran", ByteBufferUtil.string(got.getColumn().value));
- }
-
- /**
- * Gets a connection to the localhost client
- *
- * @return
- * @throws TTransportException
- */
- private Cassandra.Client getClient() throws TTransportException
- {
- TTransport tr = new TFramedTransport(new TSocket("localhost", DatabaseDescriptor.getRpcPort()));
- TProtocol proto = new TBinaryProtocol(tr);
- Cassandra.Client client = new Cassandra.Client(proto);
- tr.open();
- return client;
- }
-}
diff --git a/test/unit/org/apache/cassandra/service/GCInspectorTest.java b/test/unit/org/apache/cassandra/service/GCInspectorTest.java
new file mode 100644
index 0000000..0c5ddef
--- /dev/null
+++ b/test/unit/org/apache/cassandra/service/GCInspectorTest.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.service;
+
+import org.apache.cassandra.OrderedJUnit4ClassRunner;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+@RunWith(OrderedJUnit4ClassRunner.class)
+public class GCInspectorTest
+{
+
+ GCInspector gcInspector;
+
+ @BeforeClass
+ public static void setupDatabaseDescriptor()
+ {
+ DatabaseDescriptor.daemonInitialization();
+ }
+
+ @Before
+ public void before()
+ {
+ gcInspector = new GCInspector();
+ }
+
+ @Test
+ public void ensureStaticFieldsHydrateFromConfig()
+ {
+ Assert.assertEquals(DatabaseDescriptor.getGCLogThreshold(), gcInspector.getGcLogThresholdInMs());
+ Assert.assertEquals(DatabaseDescriptor.getGCWarnThreshold(), gcInspector.getGcWarnThresholdInMs());
+ }
+
+ @Test
+ public void ensureStatusIsCalculated()
+ {
+ Assert.assertTrue(gcInspector.getStatusThresholdInMs() > 0);
+ }
+
+ @Test(expected=IllegalArgumentException.class)
+ public void ensureWarnGreaterThanLog()
+ {
+ gcInspector.setGcWarnThresholdInMs(gcInspector.getGcLogThresholdInMs());
+ }
+
+ @Test
+ public void ensureZeroIsOk()
+ {
+ gcInspector.setGcWarnThresholdInMs(0);
+ Assert.assertEquals(gcInspector.getStatusThresholdInMs(), gcInspector.getGcLogThresholdInMs());
+ }
+
+ @Test(expected=IllegalArgumentException.class)
+ public void ensureLogLessThanWarn()
+ {
+ gcInspector.setGcLogThresholdInMs(gcInspector.getGcWarnThresholdInMs() + 1);
+ }
+
+ @Test
+ public void testDefaults()
+ {
+ gcInspector.setGcLogThresholdInMs(200);
+ gcInspector.setGcWarnThresholdInMs(1000);
+ }
+
+}
diff --git a/test/unit/org/apache/cassandra/service/JoinTokenRingTest.java b/test/unit/org/apache/cassandra/service/JoinTokenRingTest.java
index 866910e..c2aeb56 100644
--- a/test/unit/org/apache/cassandra/service/JoinTokenRingTest.java
+++ b/test/unit/org/apache/cassandra/service/JoinTokenRingTest.java
@@ -48,7 +48,7 @@
ss.joinRing();
SecondaryIndexManager indexManager = ColumnFamilyStore.getIfExists("JoinTokenRingTestKeyspace7", "Indexed1").indexManager;
- StubIndex stub = (StubIndex) indexManager.getIndexByName("value_index");
+ StubIndex stub = (StubIndex) indexManager.getIndexByName("Indexed1_value_index");
Assert.assertTrue(stub.preJoinInvocation);
}
}
diff --git a/test/unit/org/apache/cassandra/service/LeaveAndBootstrapTest.java b/test/unit/org/apache/cassandra/service/LeaveAndBootstrapTest.java
index c5f198e..018e3ea 100644
--- a/test/unit/org/apache/cassandra/service/LeaveAndBootstrapTest.java
+++ b/test/unit/org/apache/cassandra/service/LeaveAndBootstrapTest.java
@@ -19,10 +19,8 @@
package org.apache.cassandra.service;
-import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.*;
-import java.util.concurrent.ExecutorService;
import com.google.common.collect.HashMultimap;
import com.google.common.collect.Multimap;
@@ -35,9 +33,8 @@
import org.apache.cassandra.Util;
import org.apache.cassandra.Util.PartitionerSwitcher;
import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.concurrent.Stage;
-import org.apache.cassandra.concurrent.StageManager;
-import org.apache.cassandra.config.Schema;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.db.SystemKeyspace;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.RandomPartitioner;
@@ -50,7 +47,6 @@
import org.apache.cassandra.locator.SimpleSnitch;
import org.apache.cassandra.locator.TokenMetadata;
import org.apache.cassandra.schema.KeyspaceMetadata;
-import org.apache.cassandra.utils.FBUtilities;
import static org.junit.Assert.*;
@@ -94,32 +90,32 @@
IPartitioner partitioner = RandomPartitioner.instance;
VersionedValue.VersionedValueFactory valueFactory = new VersionedValue.VersionedValueFactory(partitioner);
- ArrayList<Token> endpointTokens = new ArrayList<Token>();
- ArrayList<Token> keyTokens = new ArrayList<Token>();
- List<InetAddress> hosts = new ArrayList<InetAddress>();
- List<UUID> hostIds = new ArrayList<UUID>();
+ ArrayList<Token> endpointTokens = new ArrayList<>();
+ ArrayList<Token> keyTokens = new ArrayList<>();
+ List<InetAddressAndPort> hosts = new ArrayList<>();
+ List<UUID> hostIds = new ArrayList<>();
Util.createInitialRing(ss, partitioner, endpointTokens, keyTokens, hosts, hostIds, RING_SIZE);
- Map<Token, List<InetAddress>> expectedEndpoints = new HashMap<Token, List<InetAddress>>();
- for (String keyspaceName : Schema.instance.getNonLocalStrategyKeyspaces())
+ Map<Token, List<InetAddressAndPort>> expectedEndpoints = new HashMap<>();
+ for (Token token : keyTokens)
{
- for (Token token : keyTokens)
+ List<InetAddressAndPort> endpoints = new ArrayList<>();
+ Iterator<Token> tokenIter = TokenMetadata.ringIterator(tmd.sortedTokens(), token, false);
+ while (tokenIter.hasNext())
{
- List<InetAddress> endpoints = new ArrayList<InetAddress>();
- Iterator<Token> tokenIter = TokenMetadata.ringIterator(tmd.sortedTokens(), token, false);
- while (tokenIter.hasNext())
- {
- endpoints.add(tmd.getEndpoint(tokenIter.next()));
- }
- expectedEndpoints.put(token, endpoints);
+ endpoints.add(tmd.getEndpoint(tokenIter.next()));
}
+ expectedEndpoints.put(token, endpoints);
}
// Third node leaves
ss.onChange(hosts.get(LEAVING_NODE),
- ApplicationState.STATUS,
- valueFactory.leaving(Collections.singleton(endpointTokens.get(LEAVING_NODE))));
+ ApplicationState.STATUS_WITH_PORT,
+ valueFactory.leaving(Collections.singleton(endpointTokens.get(LEAVING_NODE))));
+ ss.onChange(hosts.get(LEAVING_NODE),
+ ApplicationState.STATUS,
+ valueFactory.leaving(Collections.singleton(endpointTokens.get(LEAVING_NODE))));
assertTrue(tmd.isLeaving(hosts.get(LEAVING_NODE)));
Thread.sleep(100); // because there is a tight race between submit and blockUntilFinished
@@ -131,10 +127,10 @@
strategy = getStrategy(keyspaceName, tmd);
for (Token token : keyTokens)
{
- int replicationFactor = strategy.getReplicationFactor();
+ int replicationFactor = strategy.getReplicationFactor().allReplicas;
- HashSet<InetAddress> actual = new HashSet<InetAddress>(tmd.getWriteEndpoints(token, keyspaceName, strategy.calculateNaturalEndpoints(token, tmd.cloneOnlyTokenMap())));
- HashSet<InetAddress> expected = new HashSet<InetAddress>();
+ Set<InetAddressAndPort> actual = tmd.getWriteEndpoints(token, keyspaceName, strategy.calculateNaturalReplicas(token, tmd.cloneOnlyTokenMap()).forToken(token)).endpoints();
+ Set<InetAddressAndPort> expected = new HashSet<>();
for (int i = 0; i < replicationFactor; i++)
{
@@ -167,35 +163,38 @@
ArrayList<Token> endpointTokens = new ArrayList<Token>();
ArrayList<Token> keyTokens = new ArrayList<Token>();
- List<InetAddress> hosts = new ArrayList<InetAddress>();
+ List<InetAddressAndPort> hosts = new ArrayList<>();
List<UUID> hostIds = new ArrayList<UUID>();
// create a ring or 10 nodes
Util.createInitialRing(ss, partitioner, endpointTokens, keyTokens, hosts, hostIds, RING_SIZE);
// nodes 6, 8 and 9 leave
- final int[] LEAVING = new int[] {6, 8, 9};
+ final int[] LEAVING = new int[]{ 6, 8, 9 };
for (int leaving : LEAVING)
+ {
+ ss.onChange(hosts.get(leaving),
+ ApplicationState.STATUS_WITH_PORT,
+ valueFactory.leaving(Collections.singleton(endpointTokens.get(leaving))));
ss.onChange(hosts.get(leaving),
ApplicationState.STATUS,
valueFactory.leaving(Collections.singleton(endpointTokens.get(leaving))));
+ }
// boot two new nodes with keyTokens.get(5) and keyTokens.get(7)
- InetAddress boot1 = InetAddress.getByName("127.0.1.1");
+ InetAddressAndPort boot1 = InetAddressAndPort.getByName("127.0.1.1");
Gossiper.instance.initializeNodeUnsafe(boot1, UUID.randomUUID(), 1);
Gossiper.instance.injectApplicationState(boot1, ApplicationState.TOKENS, valueFactory.tokens(Collections.singleton(keyTokens.get(5))));
ss.onChange(boot1,
ApplicationState.STATUS,
valueFactory.bootstrapping(Collections.<Token>singleton(keyTokens.get(5))));
- InetAddress boot2 = InetAddress.getByName("127.0.1.2");
+ InetAddressAndPort boot2 = InetAddressAndPort.getByName("127.0.1.2");
Gossiper.instance.initializeNodeUnsafe(boot2, UUID.randomUUID(), 1);
Gossiper.instance.injectApplicationState(boot2, ApplicationState.TOKENS, valueFactory.tokens(Collections.singleton(keyTokens.get(7))));
ss.onChange(boot2,
ApplicationState.STATUS,
valueFactory.bootstrapping(Collections.<Token>singleton(keyTokens.get(7))));
- Collection<InetAddress> endpoints = null;
-
/* don't require test update every time a new keyspace is added to test/conf/cassandra.yaml */
Map<String, AbstractReplicationStrategy> keyspaceStrategyMap = new HashMap<String, AbstractReplicationStrategy>();
for (int i=1; i<=4; i++)
@@ -204,8 +203,8 @@
}
// pre-calculate the results.
- Map<String, Multimap<Token, InetAddress>> expectedEndpoints = new HashMap<String, Multimap<Token, InetAddress>>();
- expectedEndpoints.put(KEYSPACE1, HashMultimap.<Token, InetAddress>create());
+ Map<String, Multimap<Token, InetAddressAndPort>> expectedEndpoints = new HashMap<String, Multimap<Token, InetAddressAndPort>>();
+ expectedEndpoints.put(KEYSPACE1, HashMultimap.<Token, InetAddressAndPort>create());
expectedEndpoints.get(KEYSPACE1).putAll(new BigIntegerToken("5"), makeAddrs("127.0.0.2"));
expectedEndpoints.get(KEYSPACE1).putAll(new BigIntegerToken("15"), makeAddrs("127.0.0.3"));
expectedEndpoints.get(KEYSPACE1).putAll(new BigIntegerToken("25"), makeAddrs("127.0.0.4"));
@@ -216,7 +215,7 @@
expectedEndpoints.get(KEYSPACE1).putAll(new BigIntegerToken("75"), makeAddrs("127.0.0.9", "127.0.1.2", "127.0.0.1"));
expectedEndpoints.get(KEYSPACE1).putAll(new BigIntegerToken("85"), makeAddrs("127.0.0.10", "127.0.0.1"));
expectedEndpoints.get(KEYSPACE1).putAll(new BigIntegerToken("95"), makeAddrs("127.0.0.1"));
- expectedEndpoints.put(KEYSPACE2, HashMultimap.<Token, InetAddress>create());
+ expectedEndpoints.put(KEYSPACE2, HashMultimap.<Token, InetAddressAndPort>create());
expectedEndpoints.get(KEYSPACE2).putAll(new BigIntegerToken("5"), makeAddrs("127.0.0.2"));
expectedEndpoints.get(KEYSPACE2).putAll(new BigIntegerToken("15"), makeAddrs("127.0.0.3"));
expectedEndpoints.get(KEYSPACE2).putAll(new BigIntegerToken("25"), makeAddrs("127.0.0.4"));
@@ -227,7 +226,7 @@
expectedEndpoints.get(KEYSPACE2).putAll(new BigIntegerToken("75"), makeAddrs("127.0.0.9", "127.0.1.2", "127.0.0.1"));
expectedEndpoints.get(KEYSPACE2).putAll(new BigIntegerToken("85"), makeAddrs("127.0.0.10", "127.0.0.1"));
expectedEndpoints.get(KEYSPACE2).putAll(new BigIntegerToken("95"), makeAddrs("127.0.0.1"));
- expectedEndpoints.put(KEYSPACE3, HashMultimap.<Token, InetAddress>create());
+ expectedEndpoints.put(KEYSPACE3, HashMultimap.<Token, InetAddressAndPort>create());
expectedEndpoints.get(KEYSPACE3).putAll(new BigIntegerToken("5"), makeAddrs("127.0.0.2", "127.0.0.3", "127.0.0.4", "127.0.0.5", "127.0.0.6"));
expectedEndpoints.get(KEYSPACE3).putAll(new BigIntegerToken("15"), makeAddrs("127.0.0.3", "127.0.0.4", "127.0.0.5", "127.0.0.6", "127.0.0.7", "127.0.1.1", "127.0.0.8"));
expectedEndpoints.get(KEYSPACE3).putAll(new BigIntegerToken("25"), makeAddrs("127.0.0.4", "127.0.0.5", "127.0.0.6", "127.0.0.7", "127.0.0.8", "127.0.1.2", "127.0.0.1", "127.0.1.1"));
@@ -238,7 +237,7 @@
expectedEndpoints.get(KEYSPACE3).putAll(new BigIntegerToken("75"), makeAddrs("127.0.0.9", "127.0.0.10", "127.0.0.1", "127.0.0.2", "127.0.0.3", "127.0.1.2", "127.0.0.4", "127.0.0.5"));
expectedEndpoints.get(KEYSPACE3).putAll(new BigIntegerToken("85"), makeAddrs("127.0.0.10", "127.0.0.1", "127.0.0.2", "127.0.0.3", "127.0.0.4", "127.0.0.5"));
expectedEndpoints.get(KEYSPACE3).putAll(new BigIntegerToken("95"), makeAddrs("127.0.0.1", "127.0.0.2", "127.0.0.3", "127.0.0.4", "127.0.0.5"));
- expectedEndpoints.put(KEYSPACE4, HashMultimap.<Token, InetAddress>create());
+ expectedEndpoints.put(KEYSPACE4, HashMultimap.<Token, InetAddressAndPort>create());
expectedEndpoints.get(KEYSPACE4).putAll(new BigIntegerToken("5"), makeAddrs("127.0.0.2", "127.0.0.3", "127.0.0.4"));
expectedEndpoints.get(KEYSPACE4).putAll(new BigIntegerToken("15"), makeAddrs("127.0.0.3", "127.0.0.4", "127.0.0.5"));
expectedEndpoints.get(KEYSPACE4).putAll(new BigIntegerToken("25"), makeAddrs("127.0.0.4", "127.0.0.5", "127.0.0.6"));
@@ -259,18 +258,18 @@
for (int i = 0; i < keyTokens.size(); i++)
{
- endpoints = tmd.getWriteEndpoints(keyTokens.get(i), keyspaceName, strategy.getNaturalEndpoints(keyTokens.get(i)));
+ Collection<InetAddressAndPort> endpoints = tmd.getWriteEndpoints(keyTokens.get(i), keyspaceName, strategy.getNaturalReplicasForToken(keyTokens.get(i))).endpoints();
assertEquals(expectedEndpoints.get(keyspaceName).get(keyTokens.get(i)).size(), endpoints.size());
assertTrue(expectedEndpoints.get(keyspaceName).get(keyTokens.get(i)).containsAll(endpoints));
}
// just to be sure that things still work according to the old tests, run them:
- if (strategy.getReplicationFactor() != 3)
+ if (strategy.getReplicationFactor().allReplicas != 3)
continue;
// tokens 5, 15 and 25 should go three nodes
for (int i=0; i<3; ++i)
{
- endpoints = tmd.getWriteEndpoints(keyTokens.get(i), keyspaceName, strategy.getNaturalEndpoints(keyTokens.get(i)));
+ Collection<InetAddressAndPort> endpoints = tmd.getWriteEndpoints(keyTokens.get(i), keyspaceName, strategy.getNaturalReplicasForToken(keyTokens.get(i))).endpoints();
assertEquals(3, endpoints.size());
assertTrue(endpoints.contains(hosts.get(i+1)));
assertTrue(endpoints.contains(hosts.get(i+2)));
@@ -278,7 +277,7 @@
}
// token 35 should go to nodes 4, 5, 6, 7 and boot1
- endpoints = tmd.getWriteEndpoints(keyTokens.get(3), keyspaceName, strategy.getNaturalEndpoints(keyTokens.get(3)));
+ Collection<InetAddressAndPort> endpoints = tmd.getWriteEndpoints(keyTokens.get(3), keyspaceName, strategy.getNaturalReplicasForToken(keyTokens.get(3))).endpoints();
assertEquals(5, endpoints.size());
assertTrue(endpoints.contains(hosts.get(4)));
assertTrue(endpoints.contains(hosts.get(5)));
@@ -287,7 +286,7 @@
assertTrue(endpoints.contains(boot1));
// token 45 should go to nodes 5, 6, 7, 0, boot1 and boot2
- endpoints = tmd.getWriteEndpoints(keyTokens.get(4), keyspaceName, strategy.getNaturalEndpoints(keyTokens.get(4)));
+ endpoints = tmd.getWriteEndpoints(keyTokens.get(4), keyspaceName, strategy.getNaturalReplicasForToken(keyTokens.get(4))).endpoints();
assertEquals(6, endpoints.size());
assertTrue(endpoints.contains(hosts.get(5)));
assertTrue(endpoints.contains(hosts.get(6)));
@@ -297,7 +296,7 @@
assertTrue(endpoints.contains(boot2));
// token 55 should go to nodes 6, 7, 8, 0, 1, boot1 and boot2
- endpoints = tmd.getWriteEndpoints(keyTokens.get(5), keyspaceName, strategy.getNaturalEndpoints(keyTokens.get(5)));
+ endpoints = tmd.getWriteEndpoints(keyTokens.get(5), keyspaceName, strategy.getNaturalReplicasForToken(keyTokens.get(5))).endpoints();
assertEquals(7, endpoints.size());
assertTrue(endpoints.contains(hosts.get(6)));
assertTrue(endpoints.contains(hosts.get(7)));
@@ -308,7 +307,7 @@
assertTrue(endpoints.contains(boot2));
// token 65 should go to nodes 7, 8, 9, 0, 1 and boot2
- endpoints = tmd.getWriteEndpoints(keyTokens.get(6), keyspaceName, strategy.getNaturalEndpoints(keyTokens.get(6)));
+ endpoints = tmd.getWriteEndpoints(keyTokens.get(6), keyspaceName, strategy.getNaturalReplicasForToken(keyTokens.get(6))).endpoints();
assertEquals(6, endpoints.size());
assertTrue(endpoints.contains(hosts.get(7)));
assertTrue(endpoints.contains(hosts.get(8)));
@@ -318,7 +317,7 @@
assertTrue(endpoints.contains(boot2));
// token 75 should to go nodes 8, 9, 0, 1, 2 and boot2
- endpoints = tmd.getWriteEndpoints(keyTokens.get(7), keyspaceName, strategy.getNaturalEndpoints(keyTokens.get(7)));
+ endpoints = tmd.getWriteEndpoints(keyTokens.get(7), keyspaceName, strategy.getNaturalReplicasForToken(keyTokens.get(7))).endpoints();
assertEquals(6, endpoints.size());
assertTrue(endpoints.contains(hosts.get(8)));
assertTrue(endpoints.contains(hosts.get(9)));
@@ -328,7 +327,7 @@
assertTrue(endpoints.contains(boot2));
// token 85 should go to nodes 9, 0, 1 and 2
- endpoints = tmd.getWriteEndpoints(keyTokens.get(8), keyspaceName, strategy.getNaturalEndpoints(keyTokens.get(8)));
+ endpoints = tmd.getWriteEndpoints(keyTokens.get(8), keyspaceName, strategy.getNaturalReplicasForToken(keyTokens.get(8))).endpoints();
assertEquals(4, endpoints.size());
assertTrue(endpoints.contains(hosts.get(9)));
assertTrue(endpoints.contains(hosts.get(0)));
@@ -336,7 +335,7 @@
assertTrue(endpoints.contains(hosts.get(2)));
// token 95 should go to nodes 0, 1 and 2
- endpoints = tmd.getWriteEndpoints(keyTokens.get(9), keyspaceName, strategy.getNaturalEndpoints(keyTokens.get(9)));
+ endpoints = tmd.getWriteEndpoints(keyTokens.get(9), keyspaceName, strategy.getNaturalReplicasForToken(keyTokens.get(9))).endpoints();
assertEquals(3, endpoints.size());
assertTrue(endpoints.contains(hosts.get(0)));
assertTrue(endpoints.contains(hosts.get(1)));
@@ -381,18 +380,18 @@
for (int i = 0; i < keyTokens.size(); i++)
{
- endpoints = tmd.getWriteEndpoints(keyTokens.get(i), keyspaceName, strategy.getNaturalEndpoints(keyTokens.get(i)));
+ Collection<InetAddressAndPort> endpoints = tmd.getWriteEndpoints(keyTokens.get(i), keyspaceName, strategy.getNaturalReplicasForToken(keyTokens.get(i))).endpoints();
assertEquals(expectedEndpoints.get(keyspaceName).get(keyTokens.get(i)).size(), endpoints.size());
assertTrue(expectedEndpoints.get(keyspaceName).get(keyTokens.get(i)).containsAll(endpoints));
}
- if (strategy.getReplicationFactor() != 3)
+ if (strategy.getReplicationFactor().allReplicas != 3)
continue;
// leave this stuff in to guarantee the old tests work the way they were supposed to.
// tokens 5, 15 and 25 should go three nodes
for (int i=0; i<3; ++i)
{
- endpoints = tmd.getWriteEndpoints(keyTokens.get(i), keyspaceName, strategy.getNaturalEndpoints(keyTokens.get(i)));
+ Collection<InetAddressAndPort> endpoints = tmd.getWriteEndpoints(keyTokens.get(i), keyspaceName, strategy.getNaturalReplicasForToken(keyTokens.get(i))).endpoints();
assertEquals(3, endpoints.size());
assertTrue(endpoints.contains(hosts.get(i+1)));
assertTrue(endpoints.contains(hosts.get(i+2)));
@@ -400,21 +399,21 @@
}
// token 35 goes to nodes 4, 5 and boot1
- endpoints = tmd.getWriteEndpoints(keyTokens.get(3), keyspaceName, strategy.getNaturalEndpoints(keyTokens.get(3)));
+ Collection<InetAddressAndPort> endpoints = tmd.getWriteEndpoints(keyTokens.get(3), keyspaceName, strategy.getNaturalReplicasForToken(keyTokens.get(3))).endpoints();
assertEquals(3, endpoints.size());
assertTrue(endpoints.contains(hosts.get(4)));
assertTrue(endpoints.contains(hosts.get(5)));
assertTrue(endpoints.contains(boot1));
// token 45 goes to nodes 5, boot1 and node7
- endpoints = tmd.getWriteEndpoints(keyTokens.get(4), keyspaceName, strategy.getNaturalEndpoints(keyTokens.get(4)));
+ endpoints = tmd.getWriteEndpoints(keyTokens.get(4), keyspaceName, strategy.getNaturalReplicasForToken(keyTokens.get(4))).endpoints();
assertEquals(3, endpoints.size());
assertTrue(endpoints.contains(hosts.get(5)));
assertTrue(endpoints.contains(boot1));
assertTrue(endpoints.contains(hosts.get(7)));
// token 55 goes to boot1, 7, boot2, 8 and 0
- endpoints = tmd.getWriteEndpoints(keyTokens.get(5), keyspaceName, strategy.getNaturalEndpoints(keyTokens.get(5)));
+ endpoints = tmd.getWriteEndpoints(keyTokens.get(5), keyspaceName, strategy.getNaturalReplicasForToken(keyTokens.get(5))).endpoints();
assertEquals(5, endpoints.size());
assertTrue(endpoints.contains(boot1));
assertTrue(endpoints.contains(hosts.get(7)));
@@ -423,7 +422,7 @@
assertTrue(endpoints.contains(hosts.get(0)));
// token 65 goes to nodes 7, boot2, 8, 0 and 1
- endpoints = tmd.getWriteEndpoints(keyTokens.get(6), keyspaceName, strategy.getNaturalEndpoints(keyTokens.get(6)));
+ endpoints = tmd.getWriteEndpoints(keyTokens.get(6), keyspaceName, strategy.getNaturalReplicasForToken(keyTokens.get(6))).endpoints();
assertEquals(5, endpoints.size());
assertTrue(endpoints.contains(hosts.get(7)));
assertTrue(endpoints.contains(boot2));
@@ -432,7 +431,7 @@
assertTrue(endpoints.contains(hosts.get(1)));
// token 75 goes to nodes boot2, 8, 0, 1 and 2
- endpoints = tmd.getWriteEndpoints(keyTokens.get(7), keyspaceName, strategy.getNaturalEndpoints(keyTokens.get(7)));
+ endpoints = tmd.getWriteEndpoints(keyTokens.get(7), keyspaceName, strategy.getNaturalReplicasForToken(keyTokens.get(7))).endpoints();
assertEquals(5, endpoints.size());
assertTrue(endpoints.contains(boot2));
assertTrue(endpoints.contains(hosts.get(8)));
@@ -441,14 +440,14 @@
assertTrue(endpoints.contains(hosts.get(2)));
// token 85 goes to nodes 0, 1 and 2
- endpoints = tmd.getWriteEndpoints(keyTokens.get(8), keyspaceName, strategy.getNaturalEndpoints(keyTokens.get(8)));
+ endpoints = tmd.getWriteEndpoints(keyTokens.get(8), keyspaceName, strategy.getNaturalReplicasForToken(keyTokens.get(8))).endpoints();
assertEquals(3, endpoints.size());
assertTrue(endpoints.contains(hosts.get(0)));
assertTrue(endpoints.contains(hosts.get(1)));
assertTrue(endpoints.contains(hosts.get(2)));
// token 95 goes to nodes 0, 1 and 2
- endpoints = tmd.getWriteEndpoints(keyTokens.get(9), keyspaceName, strategy.getNaturalEndpoints(keyTokens.get(9)));
+ endpoints = tmd.getWriteEndpoints(keyTokens.get(9), keyspaceName, strategy.getNaturalReplicasForToken(keyTokens.get(9))).endpoints();
assertEquals(3, endpoints.size());
assertTrue(endpoints.contains(hosts.get(0)));
assertTrue(endpoints.contains(hosts.get(1)));
@@ -467,7 +466,7 @@
ArrayList<Token> endpointTokens = new ArrayList<Token>();
ArrayList<Token> keyTokens = new ArrayList<Token>();
- List<InetAddress> hosts = new ArrayList<InetAddress>();
+ List<InetAddressAndPort> hosts = new ArrayList<>();
List<UUID> hostIds = new ArrayList<UUID>();
// create a ring or 5 nodes
@@ -544,7 +543,7 @@
ArrayList<Token> endpointTokens = new ArrayList<Token>();
ArrayList<Token> keyTokens = new ArrayList<Token>();
- List<InetAddress> hosts = new ArrayList<InetAddress>();
+ List<InetAddressAndPort> hosts = new ArrayList<>();
List<UUID> hostIds = new ArrayList<UUID>();
// create a ring or 5 nodes
@@ -560,7 +559,7 @@
Gossiper.instance.injectApplicationState(hosts.get(2), ApplicationState.TOKENS, valueFactory.tokens(Collections.singleton(keyTokens.get(2))));
ss.onChange(hosts.get(2), ApplicationState.STATUS, valueFactory.normal(Collections.singleton(keyTokens.get(2))));
- assertTrue(tmd.getLeavingEndpoints().isEmpty());
+ assertTrue(tmd.getSizeOfLeavingEndpoints() == 0);
assertEquals(keyTokens.get(2), tmd.getToken(hosts.get(2)));
// node 3 goes through leave and left and then jumps to normal at its new token
@@ -571,7 +570,7 @@
ss.onChange(hosts.get(2), ApplicationState.STATUS, valueFactory.normal(Collections.singleton(keyTokens.get(4))));
assertTrue(tmd.getBootstrapTokens().isEmpty());
- assertTrue(tmd.getLeavingEndpoints().isEmpty());
+ assertTrue(tmd.getSizeOfLeavingEndpoints() == 0);
assertEquals(keyTokens.get(4), tmd.getToken(hosts.get(2)));
}
@@ -586,7 +585,7 @@
ArrayList<Token> endpointTokens = new ArrayList<Token>();
ArrayList<Token> keyTokens = new ArrayList<Token>();
- List<InetAddress> hosts = new ArrayList<InetAddress>();
+ List<InetAddressAndPort> hosts = new ArrayList<>();
List<UUID> hostIds = new ArrayList<UUID>();
// create a ring or 5 nodes
@@ -636,7 +635,7 @@
ArrayList<Token> endpointTokens = new ArrayList<Token>();
ArrayList<Token> keyTokens = new ArrayList<Token>();
- List<InetAddress> hosts = new ArrayList<InetAddress>();
+ List<InetAddressAndPort> hosts = new ArrayList<>();
List<UUID> hostIds = new ArrayList<UUID>();
// create a ring of 6 nodes
@@ -677,13 +676,12 @@
// create a ring of 2 nodes
ArrayList<Token> endpointTokens = new ArrayList<>();
- List<InetAddress> hosts = new ArrayList<>();
+ List<InetAddressAndPort> hosts = new ArrayList<>();
Util.createInitialRing(ss, partitioner, endpointTokens, new ArrayList<Token>(), hosts, new ArrayList<UUID>(), 2);
- InetAddress toRemove = hosts.get(1);
- final ExecutorService executor = StageManager.getStage(Stage.MUTATION);
- FBUtilities.waitOnFuture(SystemKeyspace.updatePeerInfo(toRemove, "data_center", "dc42", executor));
- FBUtilities.waitOnFuture(SystemKeyspace.updatePeerInfo(toRemove, "rack", "rack42", executor));
+ InetAddressAndPort toRemove = hosts.get(1);
+ SystemKeyspace.updatePeerInfo(toRemove, "data_center", "dc42");
+ SystemKeyspace.updatePeerInfo(toRemove, "rack", "rack42");
assertEquals("rack42", SystemKeyspace.loadDcRackInfo().get(toRemove).get("rack"));
// mark the node as removed
@@ -703,24 +701,25 @@
// create a ring of 1 node
StorageService ss = StorageService.instance;
VersionedValue.VersionedValueFactory valueFactory = new VersionedValue.VersionedValueFactory(partitioner);
- Util.createInitialRing(ss, partitioner, new ArrayList<Token>(), new ArrayList<Token>(), new ArrayList<InetAddress>(), new ArrayList<UUID>(), 1);
+ Util.createInitialRing(ss, partitioner, new ArrayList<Token>(), new ArrayList<Token>(), new ArrayList<InetAddressAndPort>(), new ArrayList<UUID>(), 1);
// make a REMOVING state change on a non-member endpoint; without the CASSANDRA-6564 fix, this
// would result in an ArrayIndexOutOfBoundsException
- ss.onChange(InetAddress.getByName("192.168.1.42"), ApplicationState.STATUS, valueFactory.removingNonlocal(UUID.randomUUID()));
+ ss.onChange(InetAddressAndPort.getByName("192.168.1.42"), ApplicationState.STATUS_WITH_PORT, valueFactory.removingNonlocal(UUID.randomUUID()));
+ ss.onChange(InetAddressAndPort.getByName("192.168.1.42"), ApplicationState.STATUS, valueFactory.removingNonlocal(UUID.randomUUID()));
}
- private static Collection<InetAddress> makeAddrs(String... hosts) throws UnknownHostException
+ private static Collection<InetAddressAndPort> makeAddrs(String... hosts) throws UnknownHostException
{
- ArrayList<InetAddress> addrs = new ArrayList<InetAddress>(hosts.length);
+ ArrayList<InetAddressAndPort> addrs = new ArrayList<>(hosts.length);
for (String host : hosts)
- addrs.add(InetAddress.getByName(host));
+ addrs.add(InetAddressAndPort.getByName(host));
return addrs;
}
private AbstractReplicationStrategy getStrategy(String keyspaceName, TokenMetadata tmd)
{
- KeyspaceMetadata ksmd = Schema.instance.getKSMetaData(keyspaceName);
+ KeyspaceMetadata ksmd = Schema.instance.getKeyspaceMetadata(keyspaceName);
return AbstractReplicationStrategy.createReplicationStrategy(
keyspaceName,
ksmd.params.replication.klass,
diff --git a/test/unit/org/apache/cassandra/service/LegacyAuthFailTest.java b/test/unit/org/apache/cassandra/service/LegacyAuthFailTest.java
index 1e93f31..1d79ecc 100644
--- a/test/unit/org/apache/cassandra/service/LegacyAuthFailTest.java
+++ b/test/unit/org/apache/cassandra/service/LegacyAuthFailTest.java
@@ -25,8 +25,8 @@
import com.google.common.base.Joiner;
import org.junit.Test;
-import org.apache.cassandra.config.SchemaConstants;
import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.schema.SchemaConstants;
import static java.lang.String.format;
import static org.junit.Assert.assertEquals;
@@ -39,7 +39,7 @@
{
createKeyspace();
- List<String> legacyTables = new ArrayList<>(StartupChecks.LEGACY_AUTH_TABLES);
+ List<String> legacyTables = new ArrayList<>(SchemaConstants.LEGACY_AUTH_TABLES);
// test reporting for individual tables
for (String legacyTable : legacyTables)
diff --git a/test/unit/org/apache/cassandra/service/MigrationManagerTest.java b/test/unit/org/apache/cassandra/service/MigrationManagerTest.java
deleted file mode 100644
index b9f5c22..0000000
--- a/test/unit/org/apache/cassandra/service/MigrationManagerTest.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.service;
-
-import java.util.Optional;
-
-import org.junit.Test;
-
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.Schema;
-import org.apache.cassandra.cql3.CQLTester;
-import org.apache.cassandra.db.Mutation;
-import org.apache.cassandra.schema.KeyspaceMetadata;
-import org.apache.cassandra.schema.KeyspaceParams;
-import org.apache.cassandra.schema.SchemaKeyspace;
-import org.apache.cassandra.schema.Tables;
-
-import static java.util.Collections.singleton;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
-public class MigrationManagerTest extends CQLTester
-{
- @Test
- public void testEvolveSystemKeyspaceNew()
- {
- CFMetaData table = CFMetaData.compile("CREATE TABLE t (id int PRIMARY KEY)", "ks0");
- KeyspaceMetadata keyspace = KeyspaceMetadata.create("ks0", KeyspaceParams.simple(1), Tables.of(table));
-
- Optional<Mutation> mutation = MigrationManager.evolveSystemKeyspace(keyspace, 0);
- assertTrue(mutation.isPresent());
-
- SchemaKeyspace.mergeSchema(singleton(mutation.get()));
- assertEquals(keyspace, Schema.instance.getKSMetaData("ks0"));
- }
-
- @Test
- public void testEvolveSystemKeyspaceExistsUpToDate()
- {
- CFMetaData table = CFMetaData.compile("CREATE TABLE t (id int PRIMARY KEY)", "ks1");
- KeyspaceMetadata keyspace = KeyspaceMetadata.create("ks1", KeyspaceParams.simple(1), Tables.of(table));
-
- // create the keyspace, verify it's there
- SchemaKeyspace.mergeSchema(singleton(SchemaKeyspace.makeCreateKeyspaceMutation(keyspace, 0).build()));
- assertEquals(keyspace, Schema.instance.getKSMetaData("ks1"));
-
- Optional<Mutation> mutation = MigrationManager.evolveSystemKeyspace(keyspace, 0);
- assertFalse(mutation.isPresent());
- }
-
- @Test
- public void testEvolveSystemKeyspaceChanged()
- {
- CFMetaData table0 = CFMetaData.compile("CREATE TABLE t (id int PRIMARY KEY)", "ks2");
- KeyspaceMetadata keyspace0 = KeyspaceMetadata.create("ks2", KeyspaceParams.simple(1), Tables.of(table0));
-
- // create the keyspace, verify it's there
- SchemaKeyspace.mergeSchema(singleton(SchemaKeyspace.makeCreateKeyspaceMutation(keyspace0, 0).build()));
- assertEquals(keyspace0, Schema.instance.getKSMetaData("ks2"));
-
- CFMetaData table1 = table0.copy().comment("comment");
- KeyspaceMetadata keyspace1 = KeyspaceMetadata.create("ks2", KeyspaceParams.simple(1), Tables.of(table1));
-
- Optional<Mutation> mutation = MigrationManager.evolveSystemKeyspace(keyspace1, 1);
- assertTrue(mutation.isPresent());
-
- SchemaKeyspace.mergeSchema(singleton(mutation.get()));
- assertEquals(keyspace1, Schema.instance.getKSMetaData("ks2"));
- }
-}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/service/MoveTest.java b/test/unit/org/apache/cassandra/service/MoveTest.java
index 39ebd66..9777602 100644
--- a/test/unit/org/apache/cassandra/service/MoveTest.java
+++ b/test/unit/org/apache/cassandra/service/MoveTest.java
@@ -19,23 +19,43 @@
package org.apache.cassandra.service;
-import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.*;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.Condition;
+import java.util.function.Consumer;
import com.google.common.collect.HashMultimap;
+import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Multimap;
+
+import org.apache.cassandra.config.OverrideConfigurationLoader;
+import org.apache.cassandra.diag.DiagnosticEventService;
+import org.apache.cassandra.gms.GossiperEvent;
+import org.apache.cassandra.locator.EndpointsForRange;
+import org.apache.cassandra.locator.EndpointsForToken;
+import org.apache.cassandra.locator.RangesAtEndpoint;
+import org.apache.cassandra.locator.RangesByEndpoint;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.TestRule;
+import org.junit.rules.TestWatcher;
+import org.junit.runner.Description;
-import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.Replica;
+import org.apache.cassandra.locator.ReplicaCollection;
+import org.apache.cassandra.schema.MigrationManager;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.marshal.BytesType;
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.Util;
-import org.apache.cassandra.config.Schema;
+import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.RandomPartitioner;
import org.apache.cassandra.dht.RandomPartitioner.BigIntegerToken;
@@ -82,7 +102,7 @@
* So instead of extending SchemaLoader, we call it's method below.
*/
@BeforeClass
- public static void setup() throws ConfigurationException
+ public static void setup() throws Exception
{
DatabaseDescriptor.daemonInitialization();
oldPartitioner = StorageService.instance.setPartitionerUnsafe(partitioner);
@@ -91,6 +111,7 @@
addNetworkTopologyKeyspace(Network_11_KeyspaceName, 1, 1);
addNetworkTopologyKeyspace(Network_22_KeyspaceName, 2, 2);
addNetworkTopologyKeyspace(Network_33_KeyspaceName, 3, 3);
+ DatabaseDescriptor.setDiagnosticEventsEnabled(true);
}
@AfterClass
@@ -100,13 +121,34 @@
}
@Before
- public void clearTokenMetadata()
+ public void clearTokenMetadata() throws InterruptedException
{
+ // we expect to have a single endpoint before running each test method,
+ // so we have to wait for the GossipStage thread to evict stale endpoints
+ // from membership before moving on, otherwise it may break other tests as
+ // things change in the background
+ final int endpointCount = Gossiper.instance.getEndpointCount() - 1;
+ final CountDownLatch latch = new CountDownLatch(endpointCount);
+ Consumer onEndpointEvicted = event -> latch.countDown();
+ DiagnosticEventService.instance().subscribe(GossiperEvent.class,
+ GossiperEvent.GossiperEventType.EVICTED_FROM_MEMBERSHIP,
+ onEndpointEvicted);
+
PendingRangeCalculatorService.instance.blockUntilFinished();
StorageService.instance.getTokenMetadata().clearUnsafe();
+
+ try
+ {
+ if (!latch.await(1, TimeUnit.MINUTES))
+ throw new RuntimeException("Took too long to evict stale endpoints.");
+ }
+ finally
+ {
+ DiagnosticEventService.instance().unsubscribe(onEndpointEvicted);
+ }
}
- private static void addNetworkTopologyKeyspace(String keyspaceName, Integer... replicas) throws ConfigurationException
+ private static void addNetworkTopologyKeyspace(String keyspaceName, Integer... replicas) throws Exception
{
DatabaseDescriptor.setEndpointSnitch(new AbstractNetworkTopologySnitch()
@@ -114,7 +156,7 @@
//Odd IPs are in DC1 and Even are in DC2. Endpoints upto .14 will have unique racks and
// then will be same for a set of three.
@Override
- public String getRack(InetAddress endpoint)
+ public String getRack(InetAddressAndPort endpoint)
{
int ipLastPart = getIPLastPart(endpoint);
if (ipLastPart <= 14)
@@ -124,7 +166,7 @@
}
@Override
- public String getDatacenter(InetAddress endpoint)
+ public String getDatacenter(InetAddressAndPort endpoint)
{
if (getIPLastPart(endpoint) % 2 == 0)
return "DC2";
@@ -132,18 +174,26 @@
return "DC1";
}
- private int getIPLastPart(InetAddress endpoint)
+ private int getIPLastPart(InetAddressAndPort endpoint)
{
String str = endpoint.toString();
int index = str.lastIndexOf(".");
- return Integer.parseInt(str.substring(index + 1).trim());
+ return Integer.parseInt(str.substring(index + 1).trim().split(":")[0]);
}
});
+ final TokenMetadata tmd = StorageService.instance.getTokenMetadata();
+
+ tmd.clearUnsafe();
+ tmd.updateHostId(UUID.randomUUID(), InetAddressAndPort.getByName("127.0.0.1"));
+ tmd.updateHostId(UUID.randomUUID(), InetAddressAndPort.getByName("127.0.0.2"));
+
KeyspaceMetadata keyspace = KeyspaceMetadata.create(keyspaceName,
KeyspaceParams.nts(configOptions(replicas)),
- Tables.of(CFMetaData.Builder.create(keyspaceName, "CF1")
- .addPartitionKey("key", BytesType.instance).build()));
+ Tables.of(TableMetadata.builder(keyspaceName, "CF1")
+ .addPartitionKeyColumn("key", BytesType.instance)
+ .build()));
+
MigrationManager.announceNewKeyspace(keyspace);
}
@@ -172,7 +222,7 @@
VersionedValue.VersionedValueFactory valueFactory = new VersionedValue.VersionedValueFactory(partitioner);
ArrayList<Token> endpointTokens = new ArrayList<>();
ArrayList<Token> keyTokens = new ArrayList<>();
- List<InetAddress> hosts = new ArrayList<>();
+ List<InetAddressAndPort> hosts = new ArrayList<>();
List<UUID> hostIds = new ArrayList<>();
for(int i=0; i < RING_SIZE/2; i++)
@@ -212,7 +262,7 @@
VersionedValue.VersionedValueFactory valueFactory = new VersionedValue.VersionedValueFactory(partitioner);
ArrayList<Token> endpointTokens = new ArrayList<>();
ArrayList<Token> keyTokens = new ArrayList<>();
- List<InetAddress> hosts = new ArrayList<>();
+ List<InetAddressAndPort> hosts = new ArrayList<>();
List<UUID> hostIds = new ArrayList<>();
for(int i=0; i < RING_SIZE/2; i++)
@@ -303,7 +353,7 @@
VersionedValue.VersionedValueFactory valueFactory = new VersionedValue.VersionedValueFactory(partitioner);
ArrayList<Token> endpointTokens = new ArrayList<>();
ArrayList<Token> keyTokens = new ArrayList<>();
- List<InetAddress> hosts = new ArrayList<>();
+ List<InetAddressAndPort> hosts = new ArrayList<>();
List<UUID> hostIds = new ArrayList<>();
Util.createInitialRing(ss, partitioner, endpointTokens, keyTokens, hosts, hostIds, RING_SIZE);
@@ -367,7 +417,7 @@
VersionedValue.VersionedValueFactory valueFactory = new VersionedValue.VersionedValueFactory(partitioner);
ArrayList<Token> endpointTokens = new ArrayList<>();
ArrayList<Token> keyTokens = new ArrayList<>();
- List<InetAddress> hosts = new ArrayList<>();
+ List<InetAddressAndPort> hosts = new ArrayList<>();
List<UUID> hostIds = new ArrayList<>();
Util.createInitialRing(ss, partitioner, endpointTokens, keyTokens, hosts, hostIds, RING_SIZE);
@@ -469,14 +519,14 @@
finishMove(hosts.get(MOVING_NODE), 30, tmd);
}
- private void moveHost(InetAddress host, int token, TokenMetadata tmd, VersionedValue.VersionedValueFactory valueFactory )
+ private void moveHost(InetAddressAndPort host, int token, TokenMetadata tmd, VersionedValue.VersionedValueFactory valueFactory )
{
StorageService.instance.onChange(host, ApplicationState.STATUS, valueFactory.moving(new BigIntegerToken(String.valueOf(token))));
PendingRangeCalculatorService.instance.blockUntilFinished();
assertTrue(tmd.isMoving(host));
}
- private void finishMove(InetAddress host, int token, TokenMetadata tmd)
+ private void finishMove(InetAddressAndPort host, int token, TokenMetadata tmd)
{
tmd.removeFromMoving(host);
assertTrue(!tmd.isMoving(host));
@@ -492,24 +542,25 @@
new VersionedValue.VersionedValueFactory(partitioner).tokens(Collections.singleton(newToken)));
}
- private Map.Entry<Range<Token>, Collection<InetAddress>> generatePendingMapEntry(int start, int end, String... endpoints) throws UnknownHostException
+ private Map.Entry<Range<Token>, EndpointsForRange> generatePendingMapEntry(int start, int end, String... endpoints) throws UnknownHostException
{
- Map<Range<Token>, Collection<InetAddress>> pendingRanges = new HashMap<>();
- pendingRanges.put(generateRange(start, end), makeAddrs(endpoints));
+ Map<Range<Token>, EndpointsForRange> pendingRanges = new HashMap<>();
+ Range<Token> range = generateRange(start, end);
+ pendingRanges.put(range, makeReplicas(range, endpoints));
return pendingRanges.entrySet().iterator().next();
}
- private Map<Range<Token>, Collection<InetAddress>> generatePendingRanges(Map.Entry<Range<Token>, Collection<InetAddress>>... entries)
+ private Map<Range<Token>, EndpointsForRange> generatePendingRanges(Map.Entry<Range<Token>, EndpointsForRange>... entries)
{
- Map<Range<Token>, Collection<InetAddress>> pendingRanges = new HashMap<>();
- for(Map.Entry<Range<Token>, Collection<InetAddress>> entry : entries)
+ Map<Range<Token>, EndpointsForRange> pendingRanges = new HashMap<>();
+ for(Map.Entry<Range<Token>, EndpointsForRange> entry : entries)
{
pendingRanges.put(entry.getKey(), entry.getValue());
}
return pendingRanges;
}
- private void assertPendingRanges(TokenMetadata tmd, Map<Range<Token>, Collection<InetAddress>> pendingRanges, String keyspaceName) throws ConfigurationException
+ private void assertPendingRanges(TokenMetadata tmd, Map<Range<Token>, EndpointsForRange> pendingRanges, String keyspaceName) throws ConfigurationException
{
boolean keyspaceFound = false;
for (String nonSystemKeyspaceName : Schema.instance.getNonLocalStrategyKeyspaces())
@@ -523,15 +574,15 @@
assert keyspaceFound;
}
- private void assertMaps(Map<Range<Token>, Collection<InetAddress>> expected, PendingRangeMaps actual)
+ private void assertMaps(Map<Range<Token>, EndpointsForRange> expected, PendingRangeMaps actual)
{
int sizeOfActual = 0;
- Iterator<Map.Entry<Range<Token>, List<InetAddress>>> iterator = actual.iterator();
+ Iterator<Map.Entry<Range<Token>, EndpointsForRange.Builder>> iterator = actual.iterator();
while(iterator.hasNext())
{
- Map.Entry<Range<Token>, List<InetAddress>> actualEntry = iterator.next();
+ Map.Entry<Range<Token>, EndpointsForRange.Builder> actualEntry = iterator.next();
assertNotNull(expected.get(actualEntry.getKey()));
- assertEquals(new HashSet<>(expected.get(actualEntry.getKey())), new HashSet<>(actualEntry.getValue()));
+ assertEquals(ImmutableSet.copyOf(expected.get(actualEntry.getKey())), ImmutableSet.copyOf(actualEntry.getValue()));
sizeOfActual++;
}
@@ -554,15 +605,15 @@
ArrayList<Token> endpointTokens = new ArrayList<Token>();
ArrayList<Token> keyTokens = new ArrayList<Token>();
- List<InetAddress> hosts = new ArrayList<InetAddress>();
+ List<InetAddressAndPort> hosts = new ArrayList<>();
List<UUID> hostIds = new ArrayList<UUID>();
Util.createInitialRing(ss, partitioner, endpointTokens, keyTokens, hosts, hostIds, RING_SIZE);
- Map<Token, List<InetAddress>> expectedEndpoints = new HashMap<Token, List<InetAddress>>();
+ Map<Token, List<InetAddressAndPort>> expectedEndpoints = new HashMap<>();
for (Token token : keyTokens)
{
- List<InetAddress> endpoints = new ArrayList<InetAddress>();
+ List<InetAddressAndPort> endpoints = new ArrayList<>();
Iterator<Token> tokenIter = TokenMetadata.ringIterator(tmd.sortedTokens(), token, false);
while (tokenIter.hasNext())
{
@@ -589,10 +640,10 @@
int numMoved = 0;
for (Token token : keyTokens)
{
- int replicationFactor = strategy.getReplicationFactor();
+ int replicationFactor = strategy.getReplicationFactor().allReplicas;
- HashSet<InetAddress> actual = new HashSet<InetAddress>(tmd.getWriteEndpoints(token, keyspaceName, strategy.calculateNaturalEndpoints(token, tmd.cloneOnlyTokenMap())));
- HashSet<InetAddress> expected = new HashSet<InetAddress>();
+ EndpointsForToken actual = tmd.getWriteEndpoints(token, keyspaceName, strategy.calculateNaturalReplicas(token, tmd.cloneOnlyTokenMap()).forToken(token));
+ HashSet<InetAddressAndPort> expected = new HashSet<>();
for (int i = 0; i < replicationFactor; i++)
{
@@ -600,10 +651,10 @@
}
if (expected.size() == actual.size()) {
- assertEquals("mismatched endpoint sets", expected, actual);
+ assertEquals("mismatched endpoint sets", expected, actual.endpoints());
} else {
expected.add(hosts.get(MOVING_NODE));
- assertEquals("mismatched endpoint sets", expected, actual);
+ assertEquals("mismatched endpoint sets", expected, actual.endpoints());
numMoved++;
}
}
@@ -628,7 +679,7 @@
ArrayList<Token> endpointTokens = new ArrayList<Token>();
ArrayList<Token> keyTokens = new ArrayList<Token>();
- List<InetAddress> hosts = new ArrayList<InetAddress>();
+ List<InetAddressAndPort> hosts = new ArrayList<>();
List<UUID> hostIds = new ArrayList<UUID>();
// create a ring or 10 nodes
@@ -648,13 +699,11 @@
newTokens.put(movingIndex, newToken);
}
- Collection<InetAddress> endpoints;
-
tmd = tmd.cloneAfterAllSettled();
ss.setTokenMetadataUnsafe(tmd);
// boot two new nodes with keyTokens.get(5) and keyTokens.get(7)
- InetAddress boot1 = InetAddress.getByName("127.0.1.1");
+ InetAddressAndPort boot1 = InetAddressAndPort.getByName("127.0.1.1");
Gossiper.instance.initializeNodeUnsafe(boot1, UUID.randomUUID(), 1);
Gossiper.instance.injectApplicationState(boot1, ApplicationState.TOKENS, valueFactory.tokens(Collections.singleton(keyTokens.get(5))));
ss.onChange(boot1,
@@ -662,7 +711,7 @@
valueFactory.bootstrapping(Collections.<Token>singleton(keyTokens.get(5))));
PendingRangeCalculatorService.instance.blockUntilFinished();
- InetAddress boot2 = InetAddress.getByName("127.0.1.2");
+ InetAddressAndPort boot2 = InetAddressAndPort.getByName("127.0.1.2");
Gossiper.instance.initializeNodeUnsafe(boot2, UUID.randomUUID(), 1);
Gossiper.instance.injectApplicationState(boot2, ApplicationState.TOKENS, valueFactory.tokens(Collections.singleton(keyTokens.get(7))));
ss.onChange(boot2,
@@ -693,37 +742,18 @@
* }
*/
- Multimap<InetAddress, Range<Token>> keyspace1ranges = keyspaceStrategyMap.get(Simple_RF1_KeyspaceName).getAddressRanges();
- Collection<Range<Token>> ranges1 = keyspace1ranges.get(InetAddress.getByName("127.0.0.1"));
- assertEquals(1, collectionSize(ranges1));
- assertEquals(generateRange(97, 0), ranges1.iterator().next());
- Collection<Range<Token>> ranges2 = keyspace1ranges.get(InetAddress.getByName("127.0.0.2"));
- assertEquals(1, collectionSize(ranges2));
- assertEquals(generateRange(0, 10), ranges2.iterator().next());
- Collection<Range<Token>> ranges3 = keyspace1ranges.get(InetAddress.getByName("127.0.0.3"));
- assertEquals(1, collectionSize(ranges3));
- assertEquals(generateRange(10, 20), ranges3.iterator().next());
- Collection<Range<Token>> ranges4 = keyspace1ranges.get(InetAddress.getByName("127.0.0.4"));
- assertEquals(1, collectionSize(ranges4));
- assertEquals(generateRange(20, 30), ranges4.iterator().next());
- Collection<Range<Token>> ranges5 = keyspace1ranges.get(InetAddress.getByName("127.0.0.5"));
- assertEquals(1, collectionSize(ranges5));
- assertEquals(generateRange(30, 40), ranges5.iterator().next());
- Collection<Range<Token>> ranges6 = keyspace1ranges.get(InetAddress.getByName("127.0.0.6"));
- assertEquals(1, collectionSize(ranges6));
- assertEquals(generateRange(40, 50), ranges6.iterator().next());
- Collection<Range<Token>> ranges7 = keyspace1ranges.get(InetAddress.getByName("127.0.0.7"));
- assertEquals(1, collectionSize(ranges7));
- assertEquals(generateRange(50, 67), ranges7.iterator().next());
- Collection<Range<Token>> ranges8 = keyspace1ranges.get(InetAddress.getByName("127.0.0.8"));
- assertEquals(1, collectionSize(ranges8));
- assertEquals(generateRange(67, 70), ranges8.iterator().next());
- Collection<Range<Token>> ranges9 = keyspace1ranges.get(InetAddress.getByName("127.0.0.9"));
- assertEquals(1, collectionSize(ranges9));
- assertEquals(generateRange(70, 87), ranges9.iterator().next());
- Collection<Range<Token>> ranges10 = keyspace1ranges.get(InetAddress.getByName("127.0.0.10"));
- assertEquals(1, collectionSize(ranges10));
- assertEquals(generateRange(87, 97), ranges10.iterator().next());
+ RangesByEndpoint keyspace1ranges = keyspaceStrategyMap.get(Simple_RF1_KeyspaceName).getAddressReplicas();
+
+ assertRanges(keyspace1ranges, "127.0.0.1", 97, 0);
+ assertRanges(keyspace1ranges, "127.0.0.2", 0, 10);
+ assertRanges(keyspace1ranges, "127.0.0.3", 10, 20);
+ assertRanges(keyspace1ranges, "127.0.0.4", 20, 30);
+ assertRanges(keyspace1ranges, "127.0.0.5", 30, 40);
+ assertRanges(keyspace1ranges, "127.0.0.6", 40, 50);
+ assertRanges(keyspace1ranges, "127.0.0.7", 50, 67);
+ assertRanges(keyspace1ranges, "127.0.0.8", 67, 70);
+ assertRanges(keyspace1ranges, "127.0.0.9", 70, 87);
+ assertRanges(keyspace1ranges, "127.0.0.10", 87, 97);
/**
@@ -742,37 +772,17 @@
* }
*/
- Multimap<InetAddress, Range<Token>> keyspace3ranges = keyspaceStrategyMap.get(KEYSPACE3).getAddressRanges();
- ranges1 = keyspace3ranges.get(InetAddress.getByName("127.0.0.1"));
- assertEquals(collectionSize(ranges1), 5);
- assertTrue(ranges1.equals(generateRanges(97, 0, 70, 87, 50, 67, 87, 97, 67, 70)));
- ranges2 = keyspace3ranges.get(InetAddress.getByName("127.0.0.2"));
- assertEquals(collectionSize(ranges2), 5);
- assertTrue(ranges2.equals(generateRanges(97, 0, 70, 87, 87, 97, 0, 10, 67, 70)));
- ranges3 = keyspace3ranges.get(InetAddress.getByName("127.0.0.3"));
- assertEquals(collectionSize(ranges3), 5);
- assertTrue(ranges3.equals(generateRanges(97, 0, 70, 87, 87, 97, 0, 10, 10, 20)));
- ranges4 = keyspace3ranges.get(InetAddress.getByName("127.0.0.4"));
- assertEquals(collectionSize(ranges4), 5);
- assertTrue(ranges4.equals(generateRanges(97, 0, 20, 30, 87, 97, 0, 10, 10, 20)));
- ranges5 = keyspace3ranges.get(InetAddress.getByName("127.0.0.5"));
- assertEquals(collectionSize(ranges5), 5);
- assertTrue(ranges5.equals(generateRanges(97, 0, 30, 40, 20, 30, 0, 10, 10, 20)));
- ranges6 = keyspace3ranges.get(InetAddress.getByName("127.0.0.6"));
- assertEquals(collectionSize(ranges6), 5);
- assertTrue(ranges6.equals(generateRanges(40, 50, 30, 40, 20, 30, 0, 10, 10, 20)));
- ranges7 = keyspace3ranges.get(InetAddress.getByName("127.0.0.7"));
- assertEquals(collectionSize(ranges7), 5);
- assertTrue(ranges7.equals(generateRanges(40, 50, 30, 40, 50, 67, 20, 30, 10, 20)));
- ranges8 = keyspace3ranges.get(InetAddress.getByName("127.0.0.8"));
- assertEquals(collectionSize(ranges8), 5);
- assertTrue(ranges8.equals(generateRanges(40, 50, 30, 40, 50, 67, 20, 30, 67, 70)));
- ranges9 = keyspace3ranges.get(InetAddress.getByName("127.0.0.9"));
- assertEquals(collectionSize(ranges9), 5);
- assertTrue(ranges9.equals(generateRanges(40, 50, 70, 87, 30, 40, 50, 67, 67, 70)));
- ranges10 = keyspace3ranges.get(InetAddress.getByName("127.0.0.10"));
- assertEquals(collectionSize(ranges10), 5);
- assertTrue(ranges10.equals(generateRanges(40, 50, 70, 87, 50, 67, 87, 97, 67, 70)));
+ RangesByEndpoint keyspace3ranges = keyspaceStrategyMap.get(KEYSPACE3).getAddressReplicas();
+ assertRanges(keyspace3ranges, "127.0.0.1", 97, 0, 70, 87, 50, 67, 87, 97, 67, 70);
+ assertRanges(keyspace3ranges, "127.0.0.2", 97, 0, 70, 87, 87, 97, 0, 10, 67, 70);
+ assertRanges(keyspace3ranges, "127.0.0.3", 97, 0, 70, 87, 87, 97, 0, 10, 10, 20);
+ assertRanges(keyspace3ranges, "127.0.0.4", 97, 0, 20, 30, 87, 97, 0, 10, 10, 20);
+ assertRanges(keyspace3ranges, "127.0.0.5", 97, 0, 30, 40, 20, 30, 0, 10, 10, 20);
+ assertRanges(keyspace3ranges, "127.0.0.6", 40, 50, 30, 40, 20, 30, 0, 10, 10, 20);
+ assertRanges(keyspace3ranges, "127.0.0.7", 40, 50, 30, 40, 50, 67, 20, 30, 10, 20);
+ assertRanges(keyspace3ranges, "127.0.0.8", 40, 50, 30, 40, 50, 67, 20, 30, 67, 70);
+ assertRanges(keyspace3ranges, "127.0.0.9", 40, 50, 70, 87, 30, 40, 50, 67, 67, 70);
+ assertRanges(keyspace3ranges, "127.0.0.10", 40, 50, 70, 87, 50, 67, 87, 97, 67, 70);
/**
@@ -790,41 +800,22 @@
* /127.0.0.10=[(70,87], (87,97], (67,70]]
* }
*/
- Multimap<InetAddress, Range<Token>> keyspace4ranges = keyspaceStrategyMap.get(Simple_RF3_KeyspaceName).getAddressRanges();
- ranges1 = keyspace4ranges.get(InetAddress.getByName("127.0.0.1"));
- assertEquals(collectionSize(ranges1), 3);
- assertTrue(ranges1.equals(generateRanges(97, 0, 70, 87, 87, 97)));
- ranges2 = keyspace4ranges.get(InetAddress.getByName("127.0.0.2"));
- assertEquals(collectionSize(ranges2), 3);
- assertTrue(ranges2.equals(generateRanges(97, 0, 87, 97, 0, 10)));
- ranges3 = keyspace4ranges.get(InetAddress.getByName("127.0.0.3"));
- assertEquals(collectionSize(ranges3), 3);
- assertTrue(ranges3.equals(generateRanges(97, 0, 0, 10, 10, 20)));
- ranges4 = keyspace4ranges.get(InetAddress.getByName("127.0.0.4"));
- assertEquals(collectionSize(ranges4), 3);
- assertTrue(ranges4.equals(generateRanges(20, 30, 0, 10, 10, 20)));
- ranges5 = keyspace4ranges.get(InetAddress.getByName("127.0.0.5"));
- assertEquals(collectionSize(ranges5), 3);
- assertTrue(ranges5.equals(generateRanges(30, 40, 20, 30, 10, 20)));
- ranges6 = keyspace4ranges.get(InetAddress.getByName("127.0.0.6"));
- assertEquals(collectionSize(ranges6), 3);
- assertTrue(ranges6.equals(generateRanges(40, 50, 30, 40, 20, 30)));
- ranges7 = keyspace4ranges.get(InetAddress.getByName("127.0.0.7"));
- assertEquals(collectionSize(ranges7), 3);
- assertTrue(ranges7.equals(generateRanges(40, 50, 30, 40, 50, 67)));
- ranges8 = keyspace4ranges.get(InetAddress.getByName("127.0.0.8"));
- assertEquals(collectionSize(ranges8), 3);
- assertTrue(ranges8.equals(generateRanges(40, 50, 50, 67, 67, 70)));
- ranges9 = keyspace4ranges.get(InetAddress.getByName("127.0.0.9"));
- assertEquals(collectionSize(ranges9), 3);
- assertTrue(ranges9.equals(generateRanges(70, 87, 50, 67, 67, 70)));
- ranges10 = keyspace4ranges.get(InetAddress.getByName("127.0.0.10"));
- assertEquals(collectionSize(ranges10), 3);
- assertTrue(ranges10.equals(generateRanges(70, 87, 87, 97, 67, 70)));
+ RangesByEndpoint keyspace4ranges = keyspaceStrategyMap.get(Simple_RF3_KeyspaceName).getAddressReplicas();
+
+ assertRanges(keyspace4ranges, "127.0.0.1", 97, 0, 70, 87, 87, 97);
+ assertRanges(keyspace4ranges, "127.0.0.2", 97, 0, 87, 97, 0, 10);
+ assertRanges(keyspace4ranges, "127.0.0.3", 97, 0, 0, 10, 10, 20);
+ assertRanges(keyspace4ranges, "127.0.0.4", 20, 30, 0, 10, 10, 20);
+ assertRanges(keyspace4ranges, "127.0.0.5", 30, 40, 20, 30, 10, 20);
+ assertRanges(keyspace4ranges, "127.0.0.6", 40, 50, 30, 40, 20, 30);
+ assertRanges(keyspace4ranges, "127.0.0.7", 40, 50, 30, 40, 50, 67);
+ assertRanges(keyspace4ranges, "127.0.0.8", 40, 50, 50, 67, 67, 70);
+ assertRanges(keyspace4ranges, "127.0.0.9", 70, 87, 50, 67, 67, 70);
+ assertRanges(keyspace4ranges, "127.0.0.10", 70, 87, 87, 97, 67, 70);
// pre-calculate the results.
- Map<String, Multimap<Token, InetAddress>> expectedEndpoints = new HashMap<String, Multimap<Token, InetAddress>>();
- expectedEndpoints.put(Simple_RF1_KeyspaceName, HashMultimap.<Token, InetAddress>create());
+ Map<String, Multimap<Token, InetAddressAndPort>> expectedEndpoints = new HashMap<>();
+ expectedEndpoints.put(Simple_RF1_KeyspaceName, HashMultimap.<Token, InetAddressAndPort>create());
expectedEndpoints.get(Simple_RF1_KeyspaceName).putAll(new BigIntegerToken("5"), makeAddrs("127.0.0.2"));
expectedEndpoints.get(Simple_RF1_KeyspaceName).putAll(new BigIntegerToken("15"), makeAddrs("127.0.0.3"));
expectedEndpoints.get(Simple_RF1_KeyspaceName).putAll(new BigIntegerToken("25"), makeAddrs("127.0.0.4"));
@@ -835,7 +826,7 @@
expectedEndpoints.get(Simple_RF1_KeyspaceName).putAll(new BigIntegerToken("75"), makeAddrs("127.0.0.9", "127.0.1.2"));
expectedEndpoints.get(Simple_RF1_KeyspaceName).putAll(new BigIntegerToken("85"), makeAddrs("127.0.0.9"));
expectedEndpoints.get(Simple_RF1_KeyspaceName).putAll(new BigIntegerToken("95"), makeAddrs("127.0.0.10"));
- expectedEndpoints.put(KEYSPACE2, HashMultimap.<Token, InetAddress>create());
+ expectedEndpoints.put(KEYSPACE2, HashMultimap.<Token, InetAddressAndPort>create());
expectedEndpoints.get(KEYSPACE2).putAll(new BigIntegerToken("5"), makeAddrs("127.0.0.2"));
expectedEndpoints.get(KEYSPACE2).putAll(new BigIntegerToken("15"), makeAddrs("127.0.0.3"));
expectedEndpoints.get(KEYSPACE2).putAll(new BigIntegerToken("25"), makeAddrs("127.0.0.4"));
@@ -846,7 +837,7 @@
expectedEndpoints.get(KEYSPACE2).putAll(new BigIntegerToken("75"), makeAddrs("127.0.0.9", "127.0.1.2"));
expectedEndpoints.get(KEYSPACE2).putAll(new BigIntegerToken("85"), makeAddrs("127.0.0.9"));
expectedEndpoints.get(KEYSPACE2).putAll(new BigIntegerToken("95"), makeAddrs("127.0.0.10"));
- expectedEndpoints.put(KEYSPACE3, HashMultimap.<Token, InetAddress>create());
+ expectedEndpoints.put(KEYSPACE3, HashMultimap.<Token, InetAddressAndPort>create());
expectedEndpoints.get(KEYSPACE3).putAll(new BigIntegerToken("5"), makeAddrs("127.0.0.2", "127.0.0.3", "127.0.0.4", "127.0.0.5", "127.0.0.6"));
expectedEndpoints.get(KEYSPACE3).putAll(new BigIntegerToken("15"), makeAddrs("127.0.0.3", "127.0.0.4", "127.0.0.5", "127.0.0.6", "127.0.0.7", "127.0.1.1"));
expectedEndpoints.get(KEYSPACE3).putAll(new BigIntegerToken("25"), makeAddrs("127.0.0.4", "127.0.0.5", "127.0.0.6", "127.0.0.7", "127.0.0.8", "127.0.1.1"));
@@ -857,7 +848,7 @@
expectedEndpoints.get(KEYSPACE3).putAll(new BigIntegerToken("75"), makeAddrs("127.0.0.9", "127.0.0.10", "127.0.0.1", "127.0.0.2", "127.0.0.3", "127.0.1.2"));
expectedEndpoints.get(KEYSPACE3).putAll(new BigIntegerToken("85"), makeAddrs("127.0.0.9", "127.0.0.10", "127.0.0.1", "127.0.0.2", "127.0.0.3"));
expectedEndpoints.get(KEYSPACE3).putAll(new BigIntegerToken("95"), makeAddrs("127.0.0.10", "127.0.0.1", "127.0.0.2", "127.0.0.3", "127.0.0.4"));
- expectedEndpoints.put(Simple_RF3_KeyspaceName, HashMultimap.<Token, InetAddress>create());
+ expectedEndpoints.put(Simple_RF3_KeyspaceName, HashMultimap.<Token, InetAddressAndPort>create());
expectedEndpoints.get(Simple_RF3_KeyspaceName).putAll(new BigIntegerToken("5"), makeAddrs("127.0.0.2", "127.0.0.3", "127.0.0.4"));
expectedEndpoints.get(Simple_RF3_KeyspaceName).putAll(new BigIntegerToken("15"), makeAddrs("127.0.0.3", "127.0.0.4", "127.0.0.5"));
expectedEndpoints.get(Simple_RF3_KeyspaceName).putAll(new BigIntegerToken("25"), makeAddrs("127.0.0.4", "127.0.0.5", "127.0.0.6"));
@@ -876,79 +867,80 @@
for (Token token : keyTokens)
{
- endpoints = tmd.getWriteEndpoints(token, keyspaceName, strategy.getNaturalEndpoints(token));
+ Collection<InetAddressAndPort> endpoints = tmd.getWriteEndpoints(token, keyspaceName, strategy.getNaturalReplicasForToken(token)).endpoints();
assertEquals(expectedEndpoints.get(keyspaceName).get(token).size(), endpoints.size());
assertTrue(expectedEndpoints.get(keyspaceName).get(token).containsAll(endpoints));
}
// just to be sure that things still work according to the old tests, run them:
- if (strategy.getReplicationFactor() != 3)
+ if (strategy.getReplicationFactor().allReplicas != 3)
continue;
+ ReplicaCollection<?> replicas = null;
// tokens 5, 15 and 25 should go three nodes
for (int i = 0; i < 3; i++)
{
- endpoints = tmd.getWriteEndpoints(keyTokens.get(i), keyspaceName, strategy.getNaturalEndpoints(keyTokens.get(i)));
- assertEquals(3, endpoints.size());
- assertTrue(endpoints.contains(hosts.get(i+1)));
- assertTrue(endpoints.contains(hosts.get(i+2)));
- assertTrue(endpoints.contains(hosts.get(i+3)));
+ replicas = tmd.getWriteEndpoints(keyTokens.get(i), keyspaceName, strategy.getNaturalReplicasForToken(keyTokens.get(i)));
+ assertEquals(3, replicas.size());
+ assertTrue(replicas.endpoints().contains(hosts.get(i + 1)));
+ assertTrue(replicas.endpoints().contains(hosts.get(i + 2)));
+ assertTrue(replicas.endpoints().contains(hosts.get(i + 3)));
}
// token 35 should go to nodes 4, 5, 6 and boot1
- endpoints = tmd.getWriteEndpoints(keyTokens.get(3), keyspaceName, strategy.getNaturalEndpoints(keyTokens.get(3)));
- assertEquals(4, endpoints.size());
- assertTrue(endpoints.contains(hosts.get(4)));
- assertTrue(endpoints.contains(hosts.get(5)));
- assertTrue(endpoints.contains(hosts.get(6)));
- assertTrue(endpoints.contains(boot1));
+ replicas = tmd.getWriteEndpoints(keyTokens.get(3), keyspaceName, strategy.getNaturalReplicasForToken(keyTokens.get(3)));
+ assertEquals(4, replicas.size());
+ assertTrue(replicas.endpoints().contains(hosts.get(4)));
+ assertTrue(replicas.endpoints().contains(hosts.get(5)));
+ assertTrue(replicas.endpoints().contains(hosts.get(6)));
+ assertTrue(replicas.endpoints().contains(boot1));
// token 45 should go to nodes 5, 6, 7 boot1
- endpoints = tmd.getWriteEndpoints(keyTokens.get(4), keyspaceName, strategy.getNaturalEndpoints(keyTokens.get(4)));
- assertEquals(4, endpoints.size());
- assertTrue(endpoints.contains(hosts.get(5)));
- assertTrue(endpoints.contains(hosts.get(6)));
- assertTrue(endpoints.contains(hosts.get(7)));
- assertTrue(endpoints.contains(boot1));
+ replicas = tmd.getWriteEndpoints(keyTokens.get(4), keyspaceName, strategy.getNaturalReplicasForToken(keyTokens.get(4)));
+ assertEquals(4, replicas.size());
+ assertTrue(replicas.endpoints().contains(hosts.get(5)));
+ assertTrue(replicas.endpoints().contains(hosts.get(6)));
+ assertTrue(replicas.endpoints().contains(hosts.get(7)));
+ assertTrue(replicas.endpoints().contains(boot1));
// token 55 should go to nodes 6, 7, 8 boot1 and boot2
- endpoints = tmd.getWriteEndpoints(keyTokens.get(5), keyspaceName, strategy.getNaturalEndpoints(keyTokens.get(5)));
- assertEquals(5, endpoints.size());
- assertTrue(endpoints.contains(hosts.get(6)));
- assertTrue(endpoints.contains(hosts.get(7)));
- assertTrue(endpoints.contains(hosts.get(8)));
- assertTrue(endpoints.contains(boot1));
- assertTrue(endpoints.contains(boot2));
+ replicas = tmd.getWriteEndpoints(keyTokens.get(5), keyspaceName, strategy.getNaturalReplicasForToken(keyTokens.get(5)));
+ assertEquals(5, replicas.size());
+ assertTrue(replicas.endpoints().contains(hosts.get(6)));
+ assertTrue(replicas.endpoints().contains(hosts.get(7)));
+ assertTrue(replicas.endpoints().contains(hosts.get(8)));
+ assertTrue(replicas.endpoints().contains(boot1));
+ assertTrue(replicas.endpoints().contains(boot2));
// token 65 should go to nodes 6, 7, 8 and boot2
- endpoints = tmd.getWriteEndpoints(keyTokens.get(6), keyspaceName, strategy.getNaturalEndpoints(keyTokens.get(6)));
- assertEquals(4, endpoints.size());
- assertTrue(endpoints.contains(hosts.get(6)));
- assertTrue(endpoints.contains(hosts.get(7)));
- assertTrue(endpoints.contains(hosts.get(8)));
- assertTrue(endpoints.contains(boot2));
+ replicas = tmd.getWriteEndpoints(keyTokens.get(6), keyspaceName, strategy.getNaturalReplicasForToken(keyTokens.get(6)));
+ assertEquals(4, replicas.size());
+ assertTrue(replicas.endpoints().contains(hosts.get(6)));
+ assertTrue(replicas.endpoints().contains(hosts.get(7)));
+ assertTrue(replicas.endpoints().contains(hosts.get(8)));
+ assertTrue(replicas.endpoints().contains(boot2));
// token 75 should to go nodes 8, 9, 0 and boot2
- endpoints = tmd.getWriteEndpoints(keyTokens.get(7), keyspaceName, strategy.getNaturalEndpoints(keyTokens.get(7)));
- assertEquals(4, endpoints.size());
- assertTrue(endpoints.contains(hosts.get(8)));
- assertTrue(endpoints.contains(hosts.get(9)));
- assertTrue(endpoints.contains(hosts.get(0)));
- assertTrue(endpoints.contains(boot2));
+ replicas = tmd.getWriteEndpoints(keyTokens.get(7), keyspaceName, strategy.getNaturalReplicasForToken(keyTokens.get(7)));
+ assertEquals(4, replicas.size());
+ assertTrue(replicas.endpoints().contains(hosts.get(8)));
+ assertTrue(replicas.endpoints().contains(hosts.get(9)));
+ assertTrue(replicas.endpoints().contains(hosts.get(0)));
+ assertTrue(replicas.endpoints().contains(boot2));
// token 85 should go to nodes 8, 9 and 0
- endpoints = tmd.getWriteEndpoints(keyTokens.get(8), keyspaceName, strategy.getNaturalEndpoints(keyTokens.get(8)));
- assertEquals(3, endpoints.size());
- assertTrue(endpoints.contains(hosts.get(8)));
- assertTrue(endpoints.contains(hosts.get(9)));
- assertTrue(endpoints.contains(hosts.get(0)));
+ replicas = tmd.getWriteEndpoints(keyTokens.get(8), keyspaceName, strategy.getNaturalReplicasForToken(keyTokens.get(8)));
+ assertEquals(3, replicas.size());
+ assertTrue(replicas.endpoints().contains(hosts.get(8)));
+ assertTrue(replicas.endpoints().contains(hosts.get(9)));
+ assertTrue(replicas.endpoints().contains(hosts.get(0)));
// token 95 should go to nodes 9, 0 and 1
- endpoints = tmd.getWriteEndpoints(keyTokens.get(9), keyspaceName, strategy.getNaturalEndpoints(keyTokens.get(9)));
- assertEquals(3, endpoints.size());
- assertTrue(endpoints.contains(hosts.get(9)));
- assertTrue(endpoints.contains(hosts.get(0)));
- assertTrue(endpoints.contains(hosts.get(1)));
+ replicas = tmd.getWriteEndpoints(keyTokens.get(9), keyspaceName, strategy.getNaturalReplicasForToken(keyTokens.get(9)));
+ assertEquals(3, replicas.size());
+ assertTrue(replicas.endpoints().contains(hosts.get(9)));
+ assertTrue(replicas.endpoints().contains(hosts.get(0)));
+ assertTrue(replicas.endpoints().contains(hosts.get(1)));
}
// all moving nodes are back to the normal state
@@ -970,7 +962,7 @@
ArrayList<Token> endpointTokens = new ArrayList<Token>();
ArrayList<Token> keyTokens = new ArrayList<Token>();
- List<InetAddress> hosts = new ArrayList<InetAddress>();
+ List<InetAddressAndPort> hosts = new ArrayList<>();
List<UUID> hostIds = new ArrayList<UUID>();
// create a ring or 6 nodes
@@ -987,7 +979,7 @@
Gossiper.instance.injectApplicationState(hosts.get(2), ApplicationState.TOKENS, valueFactory.tokens(Collections.singleton(newToken)));
ss.onChange(hosts.get(2), ApplicationState.STATUS, valueFactory.normal(Collections.singleton(newToken)));
- assertTrue(tmd.getMovingEndpoints().isEmpty());
+ assertTrue(tmd.getSizeOfMovingEndpoints() == 0);
assertEquals(newToken, tmd.getToken(hosts.get(2)));
newToken = positionToken(8);
@@ -997,21 +989,29 @@
ss.onChange(hosts.get(2), ApplicationState.STATUS, valueFactory.normal(Collections.singleton(newToken)));
assertTrue(tmd.getBootstrapTokens().isEmpty());
- assertTrue(tmd.getMovingEndpoints().isEmpty());
+ assertTrue(tmd.getSizeOfMovingEndpoints() == 0);
assertEquals(newToken, tmd.getToken(hosts.get(2)));
}
- private static Collection<InetAddress> makeAddrs(String... hosts) throws UnknownHostException
+ private static Collection<InetAddressAndPort> makeAddrs(String... hosts) throws UnknownHostException
{
- ArrayList<InetAddress> addrs = new ArrayList<InetAddress>(hosts.length);
+ ArrayList<InetAddressAndPort> addrs = new ArrayList<>(hosts.length);
for (String host : hosts)
- addrs.add(InetAddress.getByName(host));
+ addrs.add(InetAddressAndPort.getByName(host));
return addrs;
}
+ private static EndpointsForRange makeReplicas(Range<Token> range, String... hosts) throws UnknownHostException
+ {
+ EndpointsForRange.Builder replicas = EndpointsForRange.builder(range, hosts.length);
+ for (String host : hosts)
+ replicas.add(Replica.fullReplica(InetAddressAndPort.getByName(host), range));
+ return replicas.build();
+ }
+
private AbstractReplicationStrategy getStrategy(String keyspaceName, TokenMetadata tmd)
{
- KeyspaceMetadata ksmd = Schema.instance.getKSMetaData(keyspaceName);
+ KeyspaceMetadata ksmd = Schema.instance.getKeyspaceMetadata(keyspaceName);
return AbstractReplicationStrategy.createReplicationStrategy(
keyspaceName,
ksmd.params.replication.klass,
@@ -1025,7 +1025,7 @@
return new BigIntegerToken(String.valueOf(10 * position + 7));
}
- private int collectionSize(Collection<?> collection)
+ private static int collectionSize(Collection<?> collection)
{
if (collection.isEmpty())
return 0;
@@ -1057,8 +1057,52 @@
return ranges;
}
- private Range<Token> generateRange(int left, int right)
+ private static Token tk(int v)
{
- return new Range<Token>(new BigIntegerToken(String.valueOf(left)), new BigIntegerToken(String.valueOf(right)));
+ return new BigIntegerToken(String.valueOf(v));
+ }
+
+ private static Range<Token> generateRange(int left, int right)
+ {
+ return new Range<Token>(tk(left), tk(right));
+ }
+
+ private static Replica replica(InetAddressAndPort endpoint, int left, int right, boolean full)
+ {
+ return new Replica(endpoint, tk(left), tk(right), full);
+ }
+
+ private static InetAddressAndPort inet(String name)
+ {
+ try
+ {
+ return InetAddressAndPort.getByName(name);
+ }
+ catch (UnknownHostException e)
+ {
+ throw new AssertionError(e);
+ }
+ }
+
+ private static Replica replica(InetAddressAndPort endpoint, int left, int right)
+ {
+ return replica(endpoint, left, right, true);
+ }
+
+ private static void assertRanges(RangesByEndpoint epReplicas, String endpoint, int... rangePairs)
+ {
+ if (rangePairs.length % 2 == 1)
+ throw new RuntimeException("assertRanges argument count should be even");
+
+ InetAddressAndPort ep = inet(endpoint);
+ List<Replica> expected = new ArrayList<>(rangePairs.length/2);
+ for (int i=0; i<rangePairs.length; i+=2)
+ expected.add(replica(ep, rangePairs[i], rangePairs[i+1]));
+
+ RangesAtEndpoint actual = epReplicas.get(ep);
+ assertEquals(expected.size(), actual.size());
+ for (Replica replica : expected)
+ if (!actual.contains(replica))
+ assertEquals(RangesAtEndpoint.copyOf(expected), actual);
}
}
diff --git a/test/unit/org/apache/cassandra/service/MoveTransientTest.java b/test/unit/org/apache/cassandra/service/MoveTransientTest.java
new file mode 100644
index 0000000..53b1833
--- /dev/null
+++ b/test/unit/org/apache/cassandra/service/MoveTransientTest.java
@@ -0,0 +1,705 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service;
+
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+
+import com.google.common.collect.Iterables;
+
+import org.apache.cassandra.locator.EndpointsByReplica;
+import org.apache.cassandra.locator.RangesAtEndpoint;
+import org.apache.cassandra.locator.RangesByEndpoint;
+import org.junit.After;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.dht.RandomPartitioner;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.RangeStreamer;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.locator.AbstractEndpointSnitch;
+import org.apache.cassandra.locator.AbstractReplicationStrategy;
+import org.apache.cassandra.locator.IEndpointSnitch;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.Replica;
+import org.apache.cassandra.locator.SimpleStrategy;
+import org.apache.cassandra.locator.TokenMetadata;
+import org.apache.cassandra.utils.Pair;
+
+import static org.apache.cassandra.locator.Replica.fullReplica;
+import static org.apache.cassandra.locator.Replica.transientReplica;
+import static org.apache.cassandra.service.StorageServiceTest.assertMultimapEqualsIgnoreOrder;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * This is also fairly effectively testing source retrieval for bootstrap as well since RangeStreamer
+ * is used to calculate the endpoints to fetch from and check they are alive for both RangeRelocator (move) and
+ * bootstrap (RangeRelocator).
+ */
+public class MoveTransientTest
+{
+ private static final Logger logger = LoggerFactory.getLogger(MoveTransientTest.class);
+
+ static InetAddressAndPort address01;
+ static InetAddressAndPort address02;
+ static InetAddressAndPort address03;
+ static InetAddressAndPort address04;
+ static InetAddressAndPort address05;
+
+ @BeforeClass
+ public static void setUpClass() throws Exception
+ {
+ address01 = InetAddressAndPort.getByName("127.0.0.1");
+ address02 = InetAddressAndPort.getByName("127.0.0.2");
+ address03 = InetAddressAndPort.getByName("127.0.0.3");
+ address04 = InetAddressAndPort.getByName("127.0.0.4");
+ address05 = InetAddressAndPort.getByName("127.0.0.5");
+ }
+
+ private final List<InetAddressAndPort> downNodes = new ArrayList<>();
+
+ final RangeStreamer.SourceFilter alivePredicate = new RangeStreamer.SourceFilter()
+ {
+ public boolean apply(Replica replica)
+ {
+ return !downNodes.contains(replica.endpoint());
+ }
+
+ public String message(Replica replica)
+ {
+ return "Down nodes: " + downNodes;
+ }
+ };
+
+ final RangeStreamer.SourceFilter sourceFilterDownNodesPredicate = new RangeStreamer.SourceFilter()
+ {
+ public boolean apply(Replica replica)
+ {
+ return !sourceFilterDownNodes.contains(replica.endpoint());
+ }
+
+ public String message(Replica replica)
+ {
+ return "Source filter down nodes: " + sourceFilterDownNodes;
+ }
+ };
+
+ private final List<InetAddressAndPort> sourceFilterDownNodes = new ArrayList<>();
+
+ private final Collection<RangeStreamer.SourceFilter> sourceFilters = Arrays.asList(alivePredicate,
+ sourceFilterDownNodesPredicate,
+ new RangeStreamer.ExcludeLocalNodeFilter()
+ );
+
+ @After
+ public void clearDownNode()
+ {
+ downNodes.clear();
+ sourceFilterDownNodes.clear();
+ }
+
+ @BeforeClass
+ public static void setupDD()
+ {
+ DatabaseDescriptor.daemonInitialization();
+ }
+
+ final Token oneToken = new RandomPartitioner.BigIntegerToken("1");
+ final Token twoToken = new RandomPartitioner.BigIntegerToken("2");
+ final Token threeToken = new RandomPartitioner.BigIntegerToken("3");
+ final Token fourToken = new RandomPartitioner.BigIntegerToken("4");
+ final Token sixToken = new RandomPartitioner.BigIntegerToken("6");
+ final Token sevenToken = new RandomPartitioner.BigIntegerToken("7");
+ final Token nineToken = new RandomPartitioner.BigIntegerToken("9");
+ final Token elevenToken = new RandomPartitioner.BigIntegerToken("11");
+ final Token fourteenToken = new RandomPartitioner.BigIntegerToken("14");
+
+ final Range<Token> range_1_2 = new Range(oneToken, threeToken);
+ final Range<Token> range_3_6 = new Range(threeToken, sixToken);
+ final Range<Token> range_6_9 = new Range(sixToken, nineToken);
+ final Range<Token> range_9_11 = new Range(nineToken, elevenToken);
+ final Range<Token> range_11_1 = new Range(elevenToken, oneToken);
+
+
+ final RangesAtEndpoint current = RangesAtEndpoint.of(new Replica(address01, range_1_2, true),
+ new Replica(address01, range_11_1, true),
+ new Replica(address01, range_9_11, false));
+
+ public Token token(String s)
+ {
+ return new RandomPartitioner.BigIntegerToken(s);
+ }
+
+ public Range<Token> range(String start, String end)
+ {
+ return new Range<>(token(start), token(end));
+ }
+
+ /**
+ * Ring with start A 1-3 B 3-6 C 6-9 D 9-1
+ * A's token moves from 3 to 4.
+ * <p>
+ * Result is A gains some range
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testCalculateStreamAndFetchRangesMoveForward() throws Exception
+ {
+ calculateStreamAndFetchRangesMoveForward();
+ }
+
+ private Pair<RangesAtEndpoint, RangesAtEndpoint> calculateStreamAndFetchRangesMoveForward() throws Exception
+ {
+ Range<Token> aPrimeRange = new Range<>(oneToken, fourToken);
+
+ RangesAtEndpoint updated = RangesAtEndpoint.of(
+ new Replica(address01, aPrimeRange, true),
+ new Replica(address01, range_11_1, true),
+ new Replica(address01, range_9_11, false)
+ );
+
+ Pair<RangesAtEndpoint, RangesAtEndpoint> result = RangeRelocator.calculateStreamAndFetchRanges(current, updated);
+ assertContentsIgnoreOrder(result.left);
+ assertContentsIgnoreOrder(result.right, fullReplica(address01, threeToken, fourToken));
+ return result;
+ }
+
+ /**
+ * Ring with start A 1-3 B 3-6 C 6-9 D 9-11 E 11-1
+ * A's token moves from 3 to 14
+ * <p>
+ * Result is A loses range and it must be streamed
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testCalculateStreamAndFetchRangesMoveBackwardBetween() throws Exception
+ {
+ calculateStreamAndFetchRangesMoveBackwardBetween();
+ }
+
+ public Pair<RangesAtEndpoint, RangesAtEndpoint> calculateStreamAndFetchRangesMoveBackwardBetween() throws Exception
+ {
+ Range<Token> aPrimeRange = new Range<>(elevenToken, fourteenToken);
+
+ RangesAtEndpoint updated = RangesAtEndpoint.of(
+ new Replica(address01, aPrimeRange, true),
+ new Replica(address01, range_9_11, true),
+ new Replica(address01, range_6_9, false)
+ );
+
+
+ Pair<RangesAtEndpoint, RangesAtEndpoint> result = RangeRelocator.calculateStreamAndFetchRanges(current, updated);
+ assertContentsIgnoreOrder(result.left, fullReplica(address01, oneToken, threeToken), fullReplica(address01, fourteenToken, oneToken));
+ assertContentsIgnoreOrder(result.right, transientReplica(address01, sixToken, nineToken), fullReplica(address01, nineToken, elevenToken));
+ return result;
+ }
+
+ /**
+ * Ring with start A 1-3 B 3-6 C 6-9 D 9-11 E 11-1
+ * A's token moves from 3 to 2
+ *
+ * Result is A loses range and it must be streamed
+ * @throws Exception
+ */
+ @Test
+ public void testCalculateStreamAndFetchRangesMoveBackward() throws Exception
+ {
+ calculateStreamAndFetchRangesMoveBackward();
+ }
+
+ private Pair<RangesAtEndpoint, RangesAtEndpoint> calculateStreamAndFetchRangesMoveBackward() throws Exception
+ {
+ Range<Token> aPrimeRange = new Range<>(oneToken, twoToken);
+
+ RangesAtEndpoint updated = RangesAtEndpoint.of(
+ new Replica(address01, aPrimeRange, true),
+ new Replica(address01, range_11_1, true),
+ new Replica(address01, range_9_11, false)
+ );
+
+ Pair<RangesAtEndpoint, RangesAtEndpoint> result = RangeRelocator.calculateStreamAndFetchRanges(current, updated);
+
+ //Moving backwards has no impact on any replica. We already fully replicate counter clockwise
+ //The transient replica does transiently replicate slightly more, but that is addressed by cleanup
+ assertContentsIgnoreOrder(result.left, fullReplica(address01, twoToken, threeToken));
+ assertContentsIgnoreOrder(result.right);
+
+ return result;
+ }
+
+ /**
+ * Ring with start A 1-3 B 3-6 C 6-9 D 9-11 E 11-1
+ * A's moves from 3 to 7
+ *
+ * @throws Exception
+ */
+ private Pair<RangesAtEndpoint, RangesAtEndpoint> calculateStreamAndFetchRangesMoveForwardBetween() throws Exception
+ {
+ Range<Token> aPrimeRange = new Range<>(sixToken, sevenToken);
+ Range<Token> bPrimeRange = new Range<>(oneToken, sixToken);
+
+ RangesAtEndpoint updated = RangesAtEndpoint.of(
+ new Replica(address01, aPrimeRange, true),
+ new Replica(address01, bPrimeRange, true),
+ new Replica(address01, range_11_1, false)
+ );
+
+ Pair<RangesAtEndpoint, RangesAtEndpoint> result = RangeRelocator.calculateStreamAndFetchRanges(current, updated);
+
+ assertContentsIgnoreOrder(result.left, fullReplica(address01, elevenToken, oneToken), transientReplica(address01, nineToken, elevenToken));
+ assertContentsIgnoreOrder(result.right, fullReplica(address01, threeToken, sixToken), fullReplica(address01, sixToken, sevenToken));
+ return result;
+ }
+
+ /**
+ * Ring with start A 1-3 B 3-6 C 6-9 D 9-11 E 11-1
+ * A's token moves from 3 to 7
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testCalculateStreamAndFetchRangesMoveForwardBetween() throws Exception
+ {
+ calculateStreamAndFetchRangesMoveForwardBetween();
+ }
+
+ @Test
+ public void testResubtract()
+ {
+ Token oneToken = new RandomPartitioner.BigIntegerToken("0001");
+ Token tenToken = new RandomPartitioner.BigIntegerToken("0010");
+ Token fiveToken = new RandomPartitioner.BigIntegerToken("0005");
+
+ Range<Token> range_1_10 = new Range<>(oneToken, tenToken);
+ Range<Token> range_1_5 = new Range<>(oneToken, tenToken);
+ Range<Token> range_5_10 = new Range<>(fiveToken, tenToken);
+
+ RangesAtEndpoint singleRange = RangesAtEndpoint.of(
+ new Replica(address01, range_1_10, true)
+ );
+
+ RangesAtEndpoint splitRanges = RangesAtEndpoint.of(
+ new Replica(address01, range_1_5, true),
+ new Replica(address01, range_5_10, true)
+ );
+
+ // forward
+ Pair<RangesAtEndpoint, RangesAtEndpoint> calculated = RangeRelocator.calculateStreamAndFetchRanges(singleRange, splitRanges);
+ assertTrue(calculated.left.toString(), calculated.left.isEmpty());
+ assertTrue(calculated.right.toString(), calculated.right.isEmpty());
+
+ // backward
+ calculated = RangeRelocator.calculateStreamAndFetchRanges(splitRanges, singleRange);
+ assertTrue(calculated.left.toString(), calculated.left.isEmpty());
+ assertTrue(calculated.right.toString(), calculated.right.isEmpty());
+ }
+
+ /**
+ * Construct the ring state for calculateStreamAndFetchRangesMoveBackwardBetween
+ * Where are A moves from 3 to 14
+ * @return
+ */
+ private Pair<TokenMetadata, TokenMetadata> constructTMDsMoveBackwardBetween()
+ {
+ TokenMetadata tmd = new TokenMetadata();
+ tmd.updateNormalToken(range_1_2.right, address01);
+ tmd.updateNormalToken(range_3_6.right, address02);
+ tmd.updateNormalToken(range_6_9.right, address03);
+ tmd.updateNormalToken(range_9_11.right, address04);
+ tmd.updateNormalToken(range_11_1.right, address05);
+ tmd.addMovingEndpoint(fourteenToken, address01);
+ TokenMetadata updated = tmd.cloneAfterAllSettled();
+
+ return Pair.create(tmd, updated);
+ }
+
+
+ /**
+ * Construct the ring state for calculateStreamAndFetchRangesMoveForwardBetween
+ * Where are A moves from 3 to 7
+ * @return
+ */
+ private Pair<TokenMetadata, TokenMetadata> constructTMDsMoveForwardBetween()
+ {
+ TokenMetadata tmd = new TokenMetadata();
+ tmd.updateNormalToken(range_1_2.right, address01);
+ tmd.updateNormalToken(range_3_6.right, address02);
+ tmd.updateNormalToken(range_6_9.right, address03);
+ tmd.updateNormalToken(range_9_11.right, address04);
+ tmd.updateNormalToken(range_11_1.right, address05);
+ tmd.addMovingEndpoint(sevenToken, address01);
+ TokenMetadata updated = tmd.cloneAfterAllSettled();
+
+ return Pair.create(tmd, updated);
+ }
+
+ private Pair<TokenMetadata, TokenMetadata> constructTMDsMoveBackward()
+ {
+ TokenMetadata tmd = new TokenMetadata();
+ tmd.updateNormalToken(range_1_2.right, address01);
+ tmd.updateNormalToken(range_3_6.right, address02);
+ tmd.updateNormalToken(range_6_9.right, address03);
+ tmd.updateNormalToken(range_9_11.right, address04);
+ tmd.updateNormalToken(range_11_1.right, address05);
+ tmd.addMovingEndpoint(twoToken, address01);
+ TokenMetadata updated = tmd.cloneAfterAllSettled();
+
+ return Pair.create(tmd, updated);
+ }
+
+ private Pair<TokenMetadata, TokenMetadata> constructTMDsMoveForward()
+ {
+ TokenMetadata tmd = new TokenMetadata();
+ tmd.updateNormalToken(range_1_2.right, address01);
+ tmd.updateNormalToken(range_3_6.right, address02);
+ tmd.updateNormalToken(range_6_9.right, address03);
+ tmd.updateNormalToken(range_9_11.right, address04);
+ tmd.updateNormalToken(range_11_1.right, address05);
+ tmd.addMovingEndpoint(fourToken, address01);
+ TokenMetadata updated = tmd.cloneAfterAllSettled();
+
+ return Pair.create(tmd, updated);
+ }
+
+
+ @Test
+ public void testMoveForwardBetweenCalculateRangesToFetchWithPreferredEndpoints() throws Exception
+ {
+ EndpointsByReplica.Builder expectedResult = new EndpointsByReplica.Builder();
+
+ InetAddressAndPort cOrB = (downNodes.contains(address03) || sourceFilterDownNodes.contains(address03)) ? address02 : address03;
+
+ //Need to pull the full replica and the transient replica that is losing the range
+ expectedResult.put(fullReplica(address01, sixToken, sevenToken), fullReplica(address04, sixToken, nineToken));
+ expectedResult.put(fullReplica(address01, sixToken, sevenToken), transientReplica(address05, sixToken, nineToken));
+
+ //Same need both here as well
+ expectedResult.put(fullReplica(address01, threeToken, sixToken), fullReplica(cOrB, threeToken, sixToken));
+ expectedResult.put(fullReplica(address01, threeToken, sixToken), transientReplica(address04, threeToken, sixToken));
+
+ invokeCalculateRangesToFetchWithPreferredEndpoints(calculateStreamAndFetchRangesMoveForwardBetween().right,
+ constructTMDsMoveForwardBetween(),
+ expectedResult.build());
+ }
+
+ @Test
+ public void testMoveForwardBetweenCalculateRangesToFetchWithPreferredEndpointsDownNodes() throws Exception
+ {
+ for (InetAddressAndPort downNode : new InetAddressAndPort[] { address04, address05 })
+ {
+ downNodes.clear();
+ downNodes.add(downNode);
+ boolean threw = false;
+ try
+ {
+ testMoveForwardBetweenCalculateRangesToFetchWithPreferredEndpoints();
+ }
+ catch (IllegalStateException ise)
+ {
+ ise.printStackTrace();
+ assertTrue(downNode.toString(),
+ ise.getMessage().contains("Down nodes: [" + downNode + "]"));
+ threw = true;
+ }
+ assertTrue("Didn't throw for " + downNode, threw);
+ }
+
+ //Shouldn't throw because another full replica is available
+ downNodes.clear();
+ downNodes.add(address03);
+ testMoveForwardBetweenCalculateRangesToFetchWithPreferredEndpoints();
+ }
+
+ @Test
+ public void testMoveForwardBetweenCalculateRangesToFetchWithPreferredEndpointsDownNodesSourceFilter() throws Exception
+ {
+ for (InetAddressAndPort downNode : new InetAddressAndPort[] { address04, address05 })
+ {
+ sourceFilterDownNodes.clear();
+ sourceFilterDownNodes.add(downNode);
+ boolean threw = false;
+ try
+ {
+ testMoveForwardBetweenCalculateRangesToFetchWithPreferredEndpoints();
+ }
+ catch (IllegalStateException ise)
+ {
+ ise.printStackTrace();
+ assertTrue(downNode.toString(),
+ ise.getMessage().startsWith("Necessary replicas for strict consistency were removed by source filters:")
+ && ise.getMessage().contains(downNode.toString()));
+ threw = true;
+ }
+ assertTrue("Didn't throw for " + downNode, threw);
+ }
+
+ //Shouldn't throw because another full replica is available
+ sourceFilterDownNodes.clear();
+ sourceFilterDownNodes.add(address03);
+ testMoveForwardBetweenCalculateRangesToFetchWithPreferredEndpoints();
+ }
+
+ @Test
+ public void testMoveBackwardBetweenCalculateRangesToFetchWithPreferredEndpoints() throws Exception
+ {
+ EndpointsByReplica.Builder expectedResult = new EndpointsByReplica.Builder();
+
+ //Need to pull the full replica and the transient replica that is losing the range
+ expectedResult.put(fullReplica(address01, nineToken, elevenToken), fullReplica(address05, nineToken, elevenToken));
+ expectedResult.put(transientReplica(address01, sixToken, nineToken), transientReplica(address05, sixToken, nineToken));
+
+ invokeCalculateRangesToFetchWithPreferredEndpoints(calculateStreamAndFetchRangesMoveBackwardBetween().right,
+ constructTMDsMoveBackwardBetween(),
+ expectedResult.build());
+
+ }
+
+ @Test(expected = IllegalStateException.class)
+ public void testMoveBackwardBetweenCalculateRangesToFetchWithPreferredEndpointsDownNodes() throws Exception
+ {
+ //Any replica can be the full replica so this will always fail on the transient range
+ downNodes.add(address05);
+ testMoveBackwardBetweenCalculateRangesToFetchWithPreferredEndpoints();
+ }
+
+ @Test(expected = IllegalStateException.class)
+ public void testMoveBackwardBetweenCalculateRangesToFetchWithPreferredEndpointsDownNodesSourceFilter() throws Exception
+ {
+ //Any replica can be the full replica so this will always fail on the transient range
+ sourceFilterDownNodes.add(address05);
+ testMoveBackwardBetweenCalculateRangesToFetchWithPreferredEndpoints();
+ }
+
+
+ //There is no down node version of this test because nothing needs to be fetched
+ @Test
+ public void testMoveBackwardCalculateRangesToFetchWithPreferredEndpoints() throws Exception
+ {
+ //Moving backwards should fetch nothing and fetch ranges is emptys so this doesn't test a ton
+ EndpointsByReplica.Builder expectedResult = new EndpointsByReplica.Builder();
+
+ invokeCalculateRangesToFetchWithPreferredEndpoints(calculateStreamAndFetchRangesMoveBackward().right,
+ constructTMDsMoveBackward(),
+ expectedResult.build());
+
+ }
+
+ @Test
+ public void testMoveForwardCalculateRangesToFetchWithPreferredEndpoints() throws Exception
+ {
+ EndpointsByReplica.Builder expectedResult = new EndpointsByReplica.Builder();
+
+ InetAddressAndPort cOrBAddress = (downNodes.contains(address03) || sourceFilterDownNodes.contains(address03)) ? address02 : address03;
+
+ //Need to pull the full replica and the transient replica that is losing the range
+ expectedResult.put(fullReplica(address01, threeToken, fourToken), fullReplica(cOrBAddress, threeToken, sixToken));
+ expectedResult.put(fullReplica(address01, threeToken, fourToken), transientReplica(address04, threeToken, sixToken));
+
+ invokeCalculateRangesToFetchWithPreferredEndpoints(calculateStreamAndFetchRangesMoveForward().right,
+ constructTMDsMoveForward(),
+ expectedResult.build());
+
+ }
+
+ @Test
+ public void testMoveForwardCalculateRangesToFetchWithPreferredEndpointsDownNodes() throws Exception
+ {
+ downNodes.add(address04);
+ boolean threw = false;
+ try
+ {
+ testMoveForwardCalculateRangesToFetchWithPreferredEndpoints();
+ }
+ catch (IllegalStateException ise)
+ {
+ ise.printStackTrace();
+ assertTrue(address04.toString(),
+ ise.getMessage().contains("Down nodes: [" + address04 + "]"));
+ threw = true;
+ }
+ assertTrue("Didn't throw for " + address04, threw);
+
+ //Shouldn't throw because another full replica is available
+ downNodes.clear();
+ downNodes.add(address03);
+ testMoveForwardBetweenCalculateRangesToFetchWithPreferredEndpoints();
+ }
+
+ @Test
+ public void testMoveForwardCalculateRangesToFetchWithPreferredEndpointsDownNodesSourceFilter() throws Exception
+ {
+ sourceFilterDownNodes.add(address04);
+ boolean threw = false;
+ try
+ {
+ testMoveForwardCalculateRangesToFetchWithPreferredEndpoints();
+ }
+ catch (IllegalStateException ise)
+ {
+ ise.printStackTrace();
+ assertTrue(address04.toString(),
+ ise.getMessage().startsWith("Necessary replicas for strict consistency were removed by source filters:")
+ && ise.getMessage().contains(address04.toString()));
+ threw = true;
+ }
+ assertTrue("Didn't throw for " + address04, threw);
+
+ //Shouldn't throw because another full replica is available
+ sourceFilterDownNodes.clear();
+ sourceFilterDownNodes.add(address03);
+ testMoveForwardBetweenCalculateRangesToFetchWithPreferredEndpoints();
+ }
+
+ private void invokeCalculateRangesToFetchWithPreferredEndpoints(RangesAtEndpoint toFetch,
+ Pair<TokenMetadata, TokenMetadata> tmds,
+ EndpointsByReplica expectedResult)
+ {
+ DatabaseDescriptor.setTransientReplicationEnabledUnsafe(true);
+
+ EndpointsByReplica result = RangeStreamer.calculateRangesToFetchWithPreferredEndpoints((address, replicas) -> replicas.sorted((a, b) -> b.endpoint().compareTo(a.endpoint())),
+ simpleStrategy(tmds.left),
+ toFetch,
+ true,
+ tmds.left,
+ tmds.right,
+ "TestKeyspace",
+ sourceFilters);
+ logger.info("Ranges to fetch with preferred endpoints");
+ logger.info(result.toString());
+ assertMultimapEqualsIgnoreOrder(expectedResult, result);
+ }
+
+ private AbstractReplicationStrategy simpleStrategy(TokenMetadata tmd)
+ {
+ IEndpointSnitch snitch = new AbstractEndpointSnitch()
+ {
+ public int compareEndpoints(InetAddressAndPort target, Replica r1, Replica r2)
+ {
+ return 0;
+ }
+
+ public String getRack(InetAddressAndPort endpoint)
+ {
+ return "R1";
+ }
+
+ public String getDatacenter(InetAddressAndPort endpoint)
+ {
+ return "DC1";
+ }
+ };
+
+ return new SimpleStrategy("MoveTransientTest",
+ tmd,
+ snitch,
+ com.google.common.collect.ImmutableMap.of("replication_factor", "3/1"));
+ }
+
+ @Test
+ public void testMoveForwardBetweenCalculateRangesToStreamWithPreferredEndpoints() throws Exception
+ {
+ DatabaseDescriptor.setTransientReplicationEnabledUnsafe(true);
+ RangesByEndpoint.Builder expectedResult = new RangesByEndpoint.Builder();
+
+ //Need to pull the full replica and the transient replica that is losing the range
+ expectedResult.put(address02, transientReplica(address02, nineToken, elevenToken));
+ expectedResult.put(address02, fullReplica(address02, elevenToken, oneToken));
+
+ invokeCalculateRangesToStreamWithPreferredEndpoints(calculateStreamAndFetchRangesMoveForwardBetween().left,
+ constructTMDsMoveForwardBetween(),
+ expectedResult.build());
+ }
+
+ @Test
+ public void testMoveBackwardBetweenCalculateRangesToStreamWithPreferredEndpoints() throws Exception
+ {
+ RangesByEndpoint.Builder expectedResult = new RangesByEndpoint.Builder();
+
+ expectedResult.put(address02, fullReplica(address02, fourteenToken, oneToken));
+
+ expectedResult.put(address04, transientReplica(address04, oneToken, threeToken));
+
+ expectedResult.put(address03, fullReplica(address03, oneToken, threeToken));
+ expectedResult.put(address03, transientReplica(address03, fourteenToken, oneToken));
+
+ invokeCalculateRangesToStreamWithPreferredEndpoints(calculateStreamAndFetchRangesMoveBackwardBetween().left,
+ constructTMDsMoveBackwardBetween(),
+ expectedResult.build());
+ }
+
+ @Test
+ public void testMoveBackwardCalculateRangesToStreamWithPreferredEndpoints() throws Exception
+ {
+ RangesByEndpoint.Builder expectedResult = new RangesByEndpoint.Builder();
+ expectedResult.put(address03, fullReplica(address03, twoToken, threeToken));
+ expectedResult.put(address04, transientReplica(address04, twoToken, threeToken));
+
+ invokeCalculateRangesToStreamWithPreferredEndpoints(calculateStreamAndFetchRangesMoveBackward().left,
+ constructTMDsMoveBackward(),
+ expectedResult.build());
+ }
+
+ @Test
+ public void testMoveForwardCalculateRangesToStreamWithPreferredEndpoints() throws Exception
+ {
+ //Nothing to stream moving forward because we are acquiring more range not losing range
+ RangesByEndpoint.Builder expectedResult = new RangesByEndpoint.Builder();
+
+ invokeCalculateRangesToStreamWithPreferredEndpoints(calculateStreamAndFetchRangesMoveForward().left,
+ constructTMDsMoveForward(),
+ expectedResult.build());
+ }
+
+ private void invokeCalculateRangesToStreamWithPreferredEndpoints(RangesAtEndpoint toStream,
+ Pair<TokenMetadata, TokenMetadata> tmds,
+ RangesByEndpoint expectedResult)
+ {
+ DatabaseDescriptor.setTransientReplicationEnabledUnsafe(true);
+ RangeRelocator relocator = new RangeRelocator();
+ RangesByEndpoint result = relocator.calculateRangesToStreamWithEndpoints(toStream,
+ simpleStrategy(tmds.left),
+ tmds.left,
+ tmds.right);
+ logger.info("Ranges to stream by endpoint");
+ logger.info(result.toString());
+ assertMultimapEqualsIgnoreOrder(expectedResult, result);
+ }
+
+ private static void assertContentsIgnoreOrder(RangesAtEndpoint ranges, Replica ... replicas)
+ {
+ assertEquals(ranges.size(), replicas.length);
+ for (Replica replica : replicas)
+ {
+ if (!ranges.contains(replica))
+ assertTrue(Iterables.elementsEqual(RangesAtEndpoint.of(replicas), ranges));
+ }
+ }
+
+}
diff --git a/test/unit/org/apache/cassandra/service/NativeTransportServiceTest.java b/test/unit/org/apache/cassandra/service/NativeTransportServiceTest.java
index 25fac21..86b73ab 100644
--- a/test/unit/org/apache/cassandra/service/NativeTransportServiceTest.java
+++ b/test/unit/org/apache/cassandra/service/NativeTransportServiceTest.java
@@ -18,8 +18,8 @@
package org.apache.cassandra.service;
import java.util.Arrays;
+import java.util.function.BooleanSupplier;
import java.util.function.Consumer;
-import java.util.function.Supplier;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
@@ -47,7 +47,7 @@
@After
public void resetConfig()
{
- DatabaseDescriptor.getClientEncryptionOptions().enabled = false;
+ DatabaseDescriptor.updateNativeProtocolEncryptionOptions(options -> options.withEnabled(false));
DatabaseDescriptor.setNativeTransportPortSSL(null);
}
@@ -84,11 +84,11 @@
public void testDestroy()
{
withService((NativeTransportService service) -> {
- Supplier<Boolean> allTerminated = () ->
- service.getWorkerGroup().isShutdown() && service.getWorkerGroup().isTerminated();
- assertFalse(allTerminated.get());
+ BooleanSupplier allTerminated = () ->
+ service.getWorkerGroup().isShutdown() && service.getWorkerGroup().isTerminated();
+ assertFalse(allTerminated.getAsBoolean());
service.destroy();
- assertTrue(allTerminated.get());
+ assertTrue(allTerminated.getAsBoolean());
});
}
@@ -127,8 +127,8 @@
public void testSSLOnly()
{
// default ssl settings: client encryption enabled and default native transport port used for ssl only
- DatabaseDescriptor.getClientEncryptionOptions().enabled = true;
- DatabaseDescriptor.getClientEncryptionOptions().optional = false;
+ DatabaseDescriptor.updateNativeProtocolEncryptionOptions(options -> options.withEnabled(true)
+ .withOptional(false));
withService((NativeTransportService service) ->
{
@@ -144,8 +144,8 @@
public void testSSLOptional()
{
// default ssl settings: client encryption enabled and default native transport port used for optional ssl
- DatabaseDescriptor.getClientEncryptionOptions().enabled = true;
- DatabaseDescriptor.getClientEncryptionOptions().optional = true;
+ DatabaseDescriptor.updateNativeProtocolEncryptionOptions(options -> options.withEnabled(true)
+ .withOptional(true));
withService((NativeTransportService service) ->
{
@@ -161,7 +161,7 @@
public void testSSLWithNonSSL()
{
// ssl+non-ssl settings: client encryption enabled and additional ssl port specified
- DatabaseDescriptor.getClientEncryptionOptions().enabled = true;
+ DatabaseDescriptor.updateNativeProtocolEncryptionOptions(options -> options.withEnabled(true));
DatabaseDescriptor.setNativeTransportPortSSL(8432);
withService((NativeTransportService service) ->
diff --git a/test/unit/org/apache/cassandra/service/PaxosStateTest.java b/test/unit/org/apache/cassandra/service/PaxosStateTest.java
index 8054c61..bd7a85f 100644
--- a/test/unit/org/apache/cassandra/service/PaxosStateTest.java
+++ b/test/unit/org/apache/cassandra/service/PaxosStateTest.java
@@ -18,8 +18,11 @@
package org.apache.cassandra.service;
import java.nio.ByteBuffer;
+import java.util.UUID;
import com.google.common.collect.Iterables;
+import org.apache.cassandra.service.paxos.PrepareVerbHandler;
+import org.apache.cassandra.service.paxos.ProposeVerbHandler;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -59,7 +62,7 @@
ColumnFamilyStore cfs = Keyspace.open("PaxosStateTestKeyspace1").getColumnFamilyStore("Standard1");
String key = "key" + System.nanoTime();
ByteBuffer value = ByteBufferUtil.bytes(0);
- RowUpdateBuilder builder = new RowUpdateBuilder(cfs.metadata, FBUtilities.timestampMicros(), key);
+ RowUpdateBuilder builder = new RowUpdateBuilder(cfs.metadata(), FBUtilities.timestampMicros(), key);
builder.clustering("a").add("val", value);
PartitionUpdate update = Iterables.getOnlyElement(builder.build().getPartitionUpdates());
@@ -78,7 +81,7 @@
assertNoDataPresent(cfs, Util.dk(key));
// Now try again with a ballot created after the truncation
- long timestamp = SystemKeyspace.getTruncatedAt(update.metadata().cfId) + 1;
+ long timestamp = SystemKeyspace.getTruncatedAt(update.metadata().id) + 1;
Commit afterTruncate = newProposal(timestamp, update);
PaxosState.commit(afterTruncate);
assertDataPresent(cfs, Util.dk(key), "val", value);
@@ -93,11 +96,32 @@
{
Row row = Util.getOnlyRowUnfiltered(Util.cmd(cfs, key).build());
assertEquals(0, ByteBufferUtil.compareUnsigned(value,
- row.getCell(cfs.metadata.getColumnDefinition(ByteBufferUtil.bytes(name))).value()));
+ row.getCell(cfs.metadata().getColumn(ByteBufferUtil.bytes(name))).value()));
}
private void assertNoDataPresent(ColumnFamilyStore cfs, DecoratedKey key)
{
Util.assertEmpty(Util.cmd(cfs, key).build());
}
+
+ @Test
+ public void testPrepareProposePaxos() throws Throwable
+ {
+ ColumnFamilyStore cfs = Keyspace.open("PaxosStateTestKeyspace1").getColumnFamilyStore("Standard1");
+ String key = "key" + System.nanoTime();
+ ByteBuffer value = ByteBufferUtil.bytes(0);
+ RowUpdateBuilder builder = new RowUpdateBuilder(cfs.metadata(), FBUtilities.timestampMicros(), key);
+ builder.clustering("a").add("val", value);
+ PartitionUpdate update = Iterables.getOnlyElement(builder.build().getPartitionUpdates());
+
+ // CFS should be empty initially
+ assertNoDataPresent(cfs, Util.dk(key));
+
+ UUID ballot = UUIDGen.getRandomTimeUUIDFromMicros(System.currentTimeMillis());
+
+ Commit commit = Commit.newPrepare(Util.dk(key), cfs.metadata(), ballot);
+
+ assertTrue("paxos prepare stage failed", PrepareVerbHandler.doPrepare(commit).promised);
+ assertTrue("paxos propose stage failed", ProposeVerbHandler.doPropose(commit));
+ }
}
diff --git a/test/unit/org/apache/cassandra/service/ProtocolBetaVersionTest.java b/test/unit/org/apache/cassandra/service/ProtocolBetaVersionTest.java
index 0c51eb7..7f08cf2 100644
--- a/test/unit/org/apache/cassandra/service/ProtocolBetaVersionTest.java
+++ b/test/unit/org/apache/cassandra/service/ProtocolBetaVersionTest.java
@@ -68,9 +68,9 @@
createTable("CREATE TABLE %s (pk int PRIMARY KEY, v int)");
assertTrue(betaVersion.isBeta()); // change to another beta version or remove test if no beta version
- try (SimpleClient client = new SimpleClient(nativeAddr.getHostAddress(), nativePort, betaVersion, true, new EncryptionOptions.ClientEncryptionOptions()))
+ try (SimpleClient client = new SimpleClient(nativeAddr.getHostAddress(), nativePort, betaVersion, true, new EncryptionOptions()))
{
- client.connect(false);
+ client.connect(false, false);
for (int i = 0; i < 10; i++)
{
QueryMessage query = new QueryMessage(String.format("INSERT INTO %s.%s (pk, v) VALUES (%s, %s)",
@@ -103,9 +103,9 @@
}
assertTrue(betaVersion.isBeta()); // change to another beta version or remove test if no beta version
- try (SimpleClient client = new SimpleClient(nativeAddr.getHostAddress(), nativePort, betaVersion, false, new EncryptionOptions.ClientEncryptionOptions()))
+ try (SimpleClient client = new SimpleClient(nativeAddr.getHostAddress(), nativePort, betaVersion, false, new EncryptionOptions()))
{
- client.connect(false);
+ client.connect(false, false);
fail("Exception should have been thrown");
}
catch (Exception e)
diff --git a/test/unit/org/apache/cassandra/service/QueryPagerTest.java b/test/unit/org/apache/cassandra/service/QueryPagerTest.java
index 4d64283..0b8248a 100644
--- a/test/unit/org/apache/cassandra/service/QueryPagerTest.java
+++ b/test/unit/org/apache/cassandra/service/QueryPagerTest.java
@@ -27,8 +27,9 @@
import org.junit.runner.RunWith;
import org.apache.cassandra.*;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.cql3.statements.schema.CreateTableStatement;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.cql3.ColumnIdentifier;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.rows.Cell;
@@ -63,23 +64,25 @@
public static void defineSchema() throws ConfigurationException
{
SchemaLoader.prepareServer();
+
SchemaLoader.createKeyspace(KEYSPACE1,
KeyspaceParams.simple(1),
SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD));
+
SchemaLoader.createKeyspace(KEYSPACE_CQL,
KeyspaceParams.simple(1),
- CFMetaData.compile("CREATE TABLE " + CF_CQL + " ("
- + "k text,"
- + "c text,"
- + "v text,"
- + "PRIMARY KEY (k, c))", KEYSPACE_CQL),
- CFMetaData.compile("CREATE TABLE " + CF_CQL_WITH_STATIC + " ("
- + "pk text, "
- + "ck int, "
- + "st int static, "
- + "v1 int, "
- + "v2 int, "
- + "PRIMARY KEY(pk, ck))", KEYSPACE_CQL));
+ CreateTableStatement.parse("CREATE TABLE " + CF_CQL + " ("
+ + "k text,"
+ + "c text,"
+ + "v text,"
+ + "PRIMARY KEY (k, c))", KEYSPACE_CQL),
+ CreateTableStatement.parse("CREATE TABLE " + CF_CQL_WITH_STATIC + " ("
+ + "pk text, "
+ + "ck int, "
+ + "st int static, "
+ + "v1 int, "
+ + "v2 int, "
+ + "PRIMARY KEY(pk, ck))", KEYSPACE_CQL));
addData();
}
@@ -112,7 +115,7 @@
{
for (int j = 0; j < nbCols; j++)
{
- RowUpdateBuilder builder = new RowUpdateBuilder(cfs().metadata, FBUtilities.timestampMicros(), "k" + i);
+ RowUpdateBuilder builder = new RowUpdateBuilder(cfs().metadata(), FBUtilities.timestampMicros(), "k" + i);
builder.clustering("c" + j).add("val", "").build().applyUnsafe();
}
}
@@ -167,12 +170,12 @@
private static SinglePartitionReadCommand sliceQuery(String key, String start, String end, boolean reversed, int count)
{
ClusteringComparator cmp = cfs().getComparator();
- CFMetaData metadata = cfs().metadata;
+ TableMetadata metadata = cfs().metadata();
Slice slice = Slice.make(cmp.make(start), cmp.make(end));
ClusteringIndexSliceFilter filter = new ClusteringIndexSliceFilter(Slices.with(cmp, slice), reversed);
- return SinglePartitionReadCommand.create(cfs().metadata, nowInSec, ColumnFilter.all(metadata), RowFilter.NONE, DataLimits.NONE, Util.dk(key), filter);
+ return SinglePartitionReadCommand.create(metadata, nowInSec, ColumnFilter.all(metadata), RowFilter.NONE, DataLimits.NONE, Util.dk(key), filter);
}
private static ReadCommand rangeNamesQuery(String keyStart, String keyEnd, int count, String... names)
@@ -443,7 +446,7 @@
for (int i = 0; i < 5; i++)
executeInternal(String.format("INSERT INTO %s.%s (k, c, v) VALUES ('k%d', 'c%d', null)", keyspace, table, 0, i));
- ReadCommand command = SinglePartitionReadCommand.create(cfs.metadata, nowInSec, Util.dk("k0"), Slice.ALL);
+ ReadCommand command = SinglePartitionReadCommand.create(cfs.metadata(), nowInSec, Util.dk("k0"), Slice.ALL);
QueryPager pager = command.getPager(null, ProtocolVersion.CURRENT);
@@ -468,17 +471,17 @@
KEYSPACE_CQL, CF_CQL_WITH_STATIC, i));
// query the table in reverse with page size = 1 & check that the returned rows contain the correct cells
- CFMetaData cfm = Keyspace.open(KEYSPACE_CQL).getColumnFamilyStore(CF_CQL_WITH_STATIC).metadata;
- queryAndVerifyCells(cfm, true, "k0");
+ TableMetadata table = Keyspace.open(KEYSPACE_CQL).getColumnFamilyStore(CF_CQL_WITH_STATIC).metadata();
+ queryAndVerifyCells(table, true, "k0");
}
- private void queryAndVerifyCells(CFMetaData cfm, boolean reversed, String key) throws Exception
+ private void queryAndVerifyCells(TableMetadata table, boolean reversed, String key) throws Exception
{
ClusteringIndexFilter rowfilter = new ClusteringIndexSliceFilter(Slices.ALL, reversed);
- ReadCommand command = SinglePartitionReadCommand.create(cfm, nowInSec, Util.dk(key), ColumnFilter.all(cfm), rowfilter);
+ ReadCommand command = SinglePartitionReadCommand.create(table, nowInSec, Util.dk(key), ColumnFilter.all(table), rowfilter);
QueryPager pager = command.getPager(null, ProtocolVersion.CURRENT);
- ColumnDefinition staticColumn = cfm.partitionColumns().statics.getSimple(0);
+ ColumnMetadata staticColumn = table.staticColumns().getSimple(0);
assertEquals(staticColumn.name.toCQLString(), "st");
for (int i=0; i<5; i++)
@@ -494,8 +497,8 @@
int cellIndex = !reversed ? i : 4 - i;
assertEquals(row.clustering().get(0), ByteBufferUtil.bytes(cellIndex));
- assertCell(row, cfm.getColumnDefinition(new ColumnIdentifier("v1", false)), cellIndex);
- assertCell(row, cfm.getColumnDefinition(new ColumnIdentifier("v2", false)), cellIndex);
+ assertCell(row, table.getColumn(new ColumnIdentifier("v1", false)), cellIndex);
+ assertCell(row, table.getColumn(new ColumnIdentifier("v2", false)), cellIndex);
// the partition/page should contain just a single regular row
assertFalse(partition.hasNext());
@@ -511,7 +514,7 @@
}
}
- private void assertCell(Row row, ColumnDefinition column, int value)
+ private void assertCell(Row row, ColumnMetadata column, int value)
{
Cell cell = row.getCell(column);
assertNotNull(cell);
diff --git a/test/unit/org/apache/cassandra/service/RemoveTest.java b/test/unit/org/apache/cassandra/service/RemoveTest.java
index f4b203c..ea8c8d8 100644
--- a/test/unit/org/apache/cassandra/service/RemoveTest.java
+++ b/test/unit/org/apache/cassandra/service/RemoveTest.java
@@ -20,7 +20,6 @@
package org.apache.cassandra.service;
import java.io.IOException;
-import java.net.InetAddress;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -33,6 +32,7 @@
import org.apache.cassandra.Util;
import org.apache.cassandra.concurrent.NamedThreadFactory;
import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.commitlog.CommitLog;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.RandomPartitioner;
import org.apache.cassandra.dht.Token;
@@ -40,11 +40,14 @@
import org.apache.cassandra.gms.ApplicationState;
import org.apache.cassandra.gms.Gossiper;
import org.apache.cassandra.gms.VersionedValue.VersionedValueFactory;
+import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.locator.TokenMetadata;
-import org.apache.cassandra.net.MessageOut;
+import org.apache.cassandra.net.Message;
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.utils.FBUtilities;
+import static org.apache.cassandra.net.NoPayload.noPayload;
+import static org.apache.cassandra.net.Verb.REPLICATION_DONE_REQ;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@@ -53,6 +56,7 @@
static
{
DatabaseDescriptor.daemonInitialization();
+ CommitLog.instance.start();
}
static final IPartitioner partitioner = RandomPartitioner.instance;
@@ -61,15 +65,16 @@
static IPartitioner oldPartitioner;
ArrayList<Token> endpointTokens = new ArrayList<Token>();
ArrayList<Token> keyTokens = new ArrayList<Token>();
- List<InetAddress> hosts = new ArrayList<InetAddress>();
+ List<InetAddressAndPort> hosts = new ArrayList<>();
List<UUID> hostIds = new ArrayList<UUID>();
- InetAddress removalhost;
+ InetAddressAndPort removalhost;
UUID removalId;
@BeforeClass
public static void setupClass() throws ConfigurationException
{
oldPartitioner = StorageService.instance.setPartitionerUnsafe(partitioner);
+ MessagingService.instance().listen();
}
@AfterClass
@@ -86,7 +91,6 @@
// create a ring of 5 nodes
Util.createInitialRing(ss, partitioner, endpointTokens, keyTokens, hosts, hostIds, 6);
- MessagingService.instance().listen();
removalhost = hosts.get(5);
hosts.remove(removalhost);
removalId = hostIds.get(5);
@@ -96,9 +100,9 @@
@After
public void tearDown()
{
- MessagingService.instance().clearMessageSinks();
- MessagingService.instance().clearCallbacksUnsafe();
- MessagingService.instance().shutdown();
+ MessagingService.instance().inboundSink.clear();
+ MessagingService.instance().outboundSink.clear();
+ MessagingService.instance().callbacks.unsafeClear();
}
@Test(expected = UnsupportedOperationException.class)
@@ -121,7 +125,7 @@
VersionedValueFactory valueFactory = new VersionedValueFactory(DatabaseDescriptor.getPartitioner());
Collection<Token> tokens = Collections.singleton(DatabaseDescriptor.getPartitioner().getRandomToken());
- InetAddress joininghost = hosts.get(4);
+ InetAddressAndPort joininghost = hosts.get(4);
UUID joiningId = hostIds.get(4);
hosts.remove(joininghost);
@@ -158,17 +162,19 @@
Thread.sleep(1000); // make sure removal is waiting for confirmation
assertTrue(tmd.isLeaving(removalhost));
- assertEquals(1, tmd.getLeavingEndpoints().size());
+ assertEquals(1, tmd.getSizeOfLeavingEndpoints());
- for (InetAddress host : hosts)
+ for (InetAddressAndPort host : hosts)
{
- MessageOut msg = new MessageOut(host, MessagingService.Verb.REPLICATION_FINISHED, null, null, Collections.<String, byte[]>emptyMap());
- MessagingService.instance().sendRR(msg, FBUtilities.getBroadcastAddress());
+ Message msg = Message.builder(REPLICATION_DONE_REQ, noPayload)
+ .from(host)
+ .build();
+ MessagingService.instance().send(msg, FBUtilities.getBroadcastAddressAndPort());
}
remover.join();
assertTrue(success.get());
- assertTrue(tmd.getLeavingEndpoints().isEmpty());
+ assertTrue(tmd.getSizeOfLeavingEndpoints() == 0);
}
}
diff --git a/test/unit/org/apache/cassandra/service/SerializationsTest.java b/test/unit/org/apache/cassandra/service/SerializationsTest.java
index 4df112a..0a5a023 100644
--- a/test/unit/org/apache/cassandra/service/SerializationsTest.java
+++ b/test/unit/org/apache/cassandra/service/SerializationsTest.java
@@ -19,11 +19,13 @@
package org.apache.cassandra.service;
import java.io.IOException;
-import java.net.InetAddress;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
+import java.util.List;
import java.util.UUID;
+import com.google.common.collect.Lists;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -36,16 +38,21 @@
import org.apache.cassandra.dht.RandomPartitioner;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.io.IVersionedSerializer;
import org.apache.cassandra.io.util.DataInputPlus.DataInputStreamPlus;
import org.apache.cassandra.io.util.DataOutputStreamPlus;
-import org.apache.cassandra.net.MessageIn;
-import org.apache.cassandra.net.MessagingService;
-import org.apache.cassandra.repair.NodePair;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.repair.SyncNodePair;
import org.apache.cassandra.repair.RepairJobDesc;
import org.apache.cassandra.repair.Validator;
import org.apache.cassandra.repair.messages.*;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.streaming.PreviewKind;
+import org.apache.cassandra.streaming.SessionSummary;
+import org.apache.cassandra.streaming.StreamSummary;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.MerkleTrees;
+import org.apache.cassandra.utils.UUIDGen;
public class SerializationsTest extends AbstractSerializationsTester
{
@@ -54,6 +61,8 @@
private static Range<Token> FULL_RANGE;
private static RepairJobDesc DESC;
+ private static final int PORT = 7010;
+
@BeforeClass
public static void defineSchema() throws Exception
{
@@ -61,7 +70,7 @@
partitionerSwitcher = Util.switchPartitioner(RandomPartitioner.instance);
RANDOM_UUID = UUID.fromString("b5c3d033-75aa-4c2f-a819-947aac7a0c54");
FULL_RANGE = new Range<>(Util.testPartitioner().getMinimumToken(), Util.testPartitioner().getMinimumToken());
- DESC = new RepairJobDesc(getVersion() < MessagingService.VERSION_21 ? null : RANDOM_UUID, RANDOM_UUID, "Keyspace1", "Standard1", Arrays.asList(FULL_RANGE));
+ DESC = new RepairJobDesc(RANDOM_UUID, RANDOM_UUID, "Keyspace1", "Standard1", Arrays.asList(FULL_RANGE));
}
@AfterClass
@@ -69,26 +78,23 @@
{
partitionerSwitcher.close();
}
-
- private void testRepairMessageWrite(String fileName, RepairMessage... messages) throws IOException
+
+ private <T extends RepairMessage> void testRepairMessageWrite(String fileName, IVersionedSerializer<T> serializer, T... messages) throws IOException
{
try (DataOutputStreamPlus out = getOutput(fileName))
{
- for (RepairMessage message : messages)
+ for (T message : messages)
{
- testSerializedSize(message, RepairMessage.serializer);
- RepairMessage.serializer.serialize(message, out, getVersion());
+ testSerializedSize(message, serializer);
+ serializer.serialize(message, out, getVersion());
}
- // also serialize MessageOut
- for (RepairMessage message : messages)
- message.createMessage().serialize(out, getVersion());
}
}
private void testValidationRequestWrite() throws IOException
{
ValidationRequest message = new ValidationRequest(DESC, 1234);
- testRepairMessageWrite("service.ValidationRequest.bin", message);
+ testRepairMessageWrite("service.ValidationRequest.bin", ValidationRequest.serializer, message);
}
@Test
@@ -99,12 +105,9 @@
try (DataInputStreamPlus in = getInput("service.ValidationRequest.bin"))
{
- RepairMessage message = RepairMessage.serializer.deserialize(in, getVersion());
- assert message.messageType == RepairMessage.Type.VALIDATION_REQUEST;
+ ValidationRequest message = ValidationRequest.serializer.deserialize(in, getVersion());
assert DESC.equals(message.desc);
- assert ((ValidationRequest) message).gcBefore == 1234;
-
- assert MessageIn.read(in, getVersion(), -1) != null;
+ assert message.nowInSec == 1234;
}
}
@@ -116,21 +119,21 @@
// empty validation
mt.addMerkleTree((int) Math.pow(2, 15), FULL_RANGE);
- Validator v0 = new Validator(DESC, FBUtilities.getBroadcastAddress(), -1);
- ValidationComplete c0 = new ValidationComplete(DESC, mt);
+ Validator v0 = new Validator(DESC, FBUtilities.getBroadcastAddressAndPort(), -1, PreviewKind.NONE);
+ ValidationResponse c0 = new ValidationResponse(DESC, mt);
// validation with a tree
mt = new MerkleTrees(p);
mt.addMerkleTree(Integer.MAX_VALUE, FULL_RANGE);
for (int i = 0; i < 10; i++)
mt.split(p.getRandomToken());
- Validator v1 = new Validator(DESC, FBUtilities.getBroadcastAddress(), -1);
- ValidationComplete c1 = new ValidationComplete(DESC, mt);
+ Validator v1 = new Validator(DESC, FBUtilities.getBroadcastAddressAndPort(), -1, PreviewKind.NONE);
+ ValidationResponse c1 = new ValidationResponse(DESC, mt);
// validation failed
- ValidationComplete c3 = new ValidationComplete(DESC);
+ ValidationResponse c3 = new ValidationResponse(DESC);
- testRepairMessageWrite("service.ValidationComplete.bin", c0, c1, c3);
+ testRepairMessageWrite("service.ValidationComplete.bin", ValidationResponse.serializer, c0, c1, c3);
}
@Test
@@ -142,43 +145,36 @@
try (DataInputStreamPlus in = getInput("service.ValidationComplete.bin"))
{
// empty validation
- RepairMessage message = RepairMessage.serializer.deserialize(in, getVersion());
- assert message.messageType == RepairMessage.Type.VALIDATION_COMPLETE;
+ ValidationResponse message = ValidationResponse.serializer.deserialize(in, getVersion());
assert DESC.equals(message.desc);
- assert ((ValidationComplete) message).success();
- assert ((ValidationComplete) message).trees != null;
+ assert message.success();
+ assert message.trees != null;
// validation with a tree
- message = RepairMessage.serializer.deserialize(in, getVersion());
- assert message.messageType == RepairMessage.Type.VALIDATION_COMPLETE;
+ message = ValidationResponse.serializer.deserialize(in, getVersion());
assert DESC.equals(message.desc);
- assert ((ValidationComplete) message).success();
- assert ((ValidationComplete) message).trees != null;
+ assert message.success();
+ assert message.trees != null;
// failed validation
- message = RepairMessage.serializer.deserialize(in, getVersion());
- assert message.messageType == RepairMessage.Type.VALIDATION_COMPLETE;
+ message = ValidationResponse.serializer.deserialize(in, getVersion());
assert DESC.equals(message.desc);
- assert !((ValidationComplete) message).success();
- assert ((ValidationComplete) message).trees == null;
-
- // MessageOuts
- for (int i = 0; i < 3; i++)
- assert MessageIn.read(in, getVersion(), -1) != null;
+ assert !message.success();
+ assert message.trees == null;
}
}
private void testSyncRequestWrite() throws IOException
{
- InetAddress local = InetAddress.getByAddress(new byte[]{127, 0, 0, 1});
- InetAddress src = InetAddress.getByAddress(new byte[]{127, 0, 0, 2});
- InetAddress dest = InetAddress.getByAddress(new byte[]{127, 0, 0, 3});
- SyncRequest message = new SyncRequest(DESC, local, src, dest, Collections.singleton(FULL_RANGE));
+ InetAddressAndPort local = InetAddressAndPort.getByNameOverrideDefaults("127.0.0.1", PORT);
+ InetAddressAndPort src = InetAddressAndPort.getByNameOverrideDefaults("127.0.0.2", PORT);
+ InetAddressAndPort dest = InetAddressAndPort.getByNameOverrideDefaults("127.0.0.3", PORT);
- testRepairMessageWrite("service.SyncRequest.bin", message);
+ SyncRequest message = new SyncRequest(DESC, local, src, dest, Collections.singleton(FULL_RANGE), PreviewKind.NONE);
+ testRepairMessageWrite("service.SyncRequest.bin", SyncRequest.serializer, message);
}
@Test
@@ -187,34 +183,36 @@
if (EXECUTE_WRITES)
testSyncRequestWrite();
- InetAddress local = InetAddress.getByAddress(new byte[]{127, 0, 0, 1});
- InetAddress src = InetAddress.getByAddress(new byte[]{127, 0, 0, 2});
- InetAddress dest = InetAddress.getByAddress(new byte[]{127, 0, 0, 3});
+ InetAddressAndPort local = InetAddressAndPort.getByNameOverrideDefaults("127.0.0.1", PORT);
+ InetAddressAndPort src = InetAddressAndPort.getByNameOverrideDefaults("127.0.0.2", PORT);
+ InetAddressAndPort dest = InetAddressAndPort.getByNameOverrideDefaults("127.0.0.3", PORT);
try (DataInputStreamPlus in = getInput("service.SyncRequest.bin"))
{
- RepairMessage message = RepairMessage.serializer.deserialize(in, getVersion());
- assert message.messageType == RepairMessage.Type.SYNC_REQUEST;
+ SyncRequest message = SyncRequest.serializer.deserialize(in, getVersion());
assert DESC.equals(message.desc);
- assert local.equals(((SyncRequest) message).initiator);
- assert src.equals(((SyncRequest) message).src);
- assert dest.equals(((SyncRequest) message).dst);
- assert ((SyncRequest) message).ranges.size() == 1 && ((SyncRequest) message).ranges.contains(FULL_RANGE);
-
- assert MessageIn.read(in, getVersion(), -1) != null;
+ assert local.equals(message.initiator);
+ assert src.equals(message.src);
+ assert dest.equals(message.dst);
+ assert message.ranges.size() == 1 && message.ranges.contains(FULL_RANGE);
}
}
private void testSyncCompleteWrite() throws IOException
{
- InetAddress src = InetAddress.getByAddress(new byte[]{127, 0, 0, 2});
- InetAddress dest = InetAddress.getByAddress(new byte[]{127, 0, 0, 3});
+ InetAddressAndPort src = InetAddressAndPort.getByNameOverrideDefaults("127.0.0.2", PORT);
+ InetAddressAndPort dest = InetAddressAndPort.getByNameOverrideDefaults("127.0.0.3", PORT);
// sync success
- SyncComplete success = new SyncComplete(DESC, src, dest, true);
+ List<SessionSummary> summaries = new ArrayList<>();
+ summaries.add(new SessionSummary(src, dest,
+ Lists.newArrayList(new StreamSummary(TableId.fromUUID(UUIDGen.getTimeUUID()), 5, 100)),
+ Lists.newArrayList(new StreamSummary(TableId.fromUUID(UUIDGen.getTimeUUID()), 500, 10))
+ ));
+ SyncResponse success = new SyncResponse(DESC, src, dest, true, summaries);
// sync fail
- SyncComplete fail = new SyncComplete(DESC, src, dest, false);
+ SyncResponse fail = new SyncResponse(DESC, src, dest, false, Collections.emptyList());
- testRepairMessageWrite("service.SyncComplete.bin", success, fail);
+ testRepairMessageWrite("service.SyncComplete.bin", SyncResponse.serializer, success, fail);
}
@Test
@@ -223,31 +221,27 @@
if (EXECUTE_WRITES)
testSyncCompleteWrite();
- InetAddress src = InetAddress.getByAddress(new byte[]{127, 0, 0, 2});
- InetAddress dest = InetAddress.getByAddress(new byte[]{127, 0, 0, 3});
- NodePair nodes = new NodePair(src, dest);
+ InetAddressAndPort src = InetAddressAndPort.getByNameOverrideDefaults("127.0.0.2", PORT);
+ InetAddressAndPort dest = InetAddressAndPort.getByNameOverrideDefaults("127.0.0.3", PORT);
+ SyncNodePair nodes = new SyncNodePair(src, dest);
try (DataInputStreamPlus in = getInput("service.SyncComplete.bin"))
{
// success
- RepairMessage message = RepairMessage.serializer.deserialize(in, getVersion());
- assert message.messageType == RepairMessage.Type.SYNC_COMPLETE;
+ SyncResponse message = SyncResponse.serializer.deserialize(in, getVersion());
assert DESC.equals(message.desc);
- assert nodes.equals(((SyncComplete) message).nodes);
- assert ((SyncComplete) message).success;
+ System.out.println(nodes);
+ System.out.println(message.nodes);
+ assert nodes.equals(message.nodes);
+ assert message.success;
// fail
- message = RepairMessage.serializer.deserialize(in, getVersion());
- assert message.messageType == RepairMessage.Type.SYNC_COMPLETE;
+ message = SyncResponse.serializer.deserialize(in, getVersion());
assert DESC.equals(message.desc);
- assert nodes.equals(((SyncComplete) message).nodes);
- assert !((SyncComplete) message).success;
-
- // MessageOuts
- for (int i = 0; i < 2; i++)
- assert MessageIn.read(in, getVersion(), -1) != null;
+ assert nodes.equals(message.nodes);
+ assert !message.success;
}
}
}
diff --git a/test/unit/org/apache/cassandra/service/StartupChecksTest.java b/test/unit/org/apache/cassandra/service/StartupChecksTest.java
index 63a11c3..67217b3 100644
--- a/test/unit/org/apache/cassandra/service/StartupChecksTest.java
+++ b/test/unit/org/apache/cassandra/service/StartupChecksTest.java
@@ -27,7 +27,7 @@
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.config.SchemaConstants;
+import org.apache.cassandra.schema.SchemaConstants;
import org.apache.cassandra.db.*;
import org.apache.cassandra.exceptions.StartupException;
import org.apache.cassandra.io.util.FileUtils;
diff --git a/test/unit/org/apache/cassandra/service/StorageProxyTest.java b/test/unit/org/apache/cassandra/service/StorageProxyTest.java
index bdf45fe..590cfeb 100644
--- a/test/unit/org/apache/cassandra/service/StorageProxyTest.java
+++ b/test/unit/org/apache/cassandra/service/StorageProxyTest.java
@@ -18,7 +18,6 @@
*/
package org.apache.cassandra.service;
-import java.net.InetAddress;
import java.util.List;
import org.junit.BeforeClass;
@@ -27,6 +26,7 @@
import org.apache.cassandra.db.PartitionPosition;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.dht.*;
+import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.locator.TokenMetadata;
import static org.apache.cassandra.Util.rp;
@@ -81,8 +81,8 @@
DatabaseDescriptor.daemonInitialization();
DatabaseDescriptor.getHintsDirectory().mkdir();
TokenMetadata tmd = StorageService.instance.getTokenMetadata();
- tmd.updateNormalToken(token("1"), InetAddress.getByName("127.0.0.1"));
- tmd.updateNormalToken(token("6"), InetAddress.getByName("127.0.0.6"));
+ tmd.updateNormalToken(token("1"), InetAddressAndPort.getByName("127.0.0.1"));
+ tmd.updateNormalToken(token("6"), InetAddressAndPort.getByName("127.0.0.6"));
}
// test getRestrictedRanges for token
diff --git a/test/unit/org/apache/cassandra/service/StorageServiceServerTest.java b/test/unit/org/apache/cassandra/service/StorageServiceServerTest.java
index 297d19d..3e188ed 100644
--- a/test/unit/org/apache/cassandra/service/StorageServiceServerTest.java
+++ b/test/unit/org/apache/cassandra/service/StorageServiceServerTest.java
@@ -28,16 +28,23 @@
import com.google.common.collect.HashMultimap;
import com.google.common.collect.Multimap;
+import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.apache.cassandra.OrderedJUnit4ClassRunner;
import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.audit.AuditLogManager;
import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.config.SchemaConstants;
+import org.apache.cassandra.db.commitlog.CommitLog;
+import org.apache.cassandra.gms.ApplicationState;
+import org.apache.cassandra.gms.Gossiper;
+import org.apache.cassandra.gms.VersionedValue;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.schema.SchemaConstants;
import org.apache.cassandra.schema.KeyspaceMetadata;
-import org.apache.cassandra.config.Schema;
+import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.db.WindowsFailedSnapshotTracker;
import org.apache.cassandra.dht.Murmur3Partitioner;
@@ -64,7 +71,9 @@
@BeforeClass
public static void setUp() throws ConfigurationException
{
+ System.setProperty(Gossiper.Props.DISABLE_THREAD_VALIDATION, "true");
DatabaseDescriptor.daemonInitialization();
+ CommitLog.instance.start();
IEndpointSnitch snitch = new PropertyFileSnitch();
DatabaseDescriptor.setEndpointSnitch(snitch);
Keyspace.setInitialized();
@@ -184,6 +193,45 @@
// no need to insert extra data, even an "empty" database will have a little information in the system keyspace
StorageService.instance.takeSnapshot(UUID.randomUUID().toString(), SchemaConstants.SCHEMA_KEYSPACE_NAME);
}
+ @Test
+ public void testLocalPrimaryRangeForEndpointWithNetworkTopologyStrategy() throws Exception
+ {
+ TokenMetadata metadata = StorageService.instance.getTokenMetadata();
+ metadata.clearUnsafe();
+
+ // DC1
+ metadata.updateNormalToken(new StringToken("A"), InetAddressAndPort.getByName("127.0.0.1"));
+ metadata.updateNormalToken(new StringToken("C"), InetAddressAndPort.getByName("127.0.0.2"));
+
+ // DC2
+ metadata.updateNormalToken(new StringToken("B"), InetAddressAndPort.getByName("127.0.0.4"));
+ metadata.updateNormalToken(new StringToken("D"), InetAddressAndPort.getByName("127.0.0.5"));
+
+ Map<String, String> configOptions = new HashMap<>();
+ configOptions.put("DC1", "2");
+ configOptions.put("DC2", "2");
+ configOptions.put(ReplicationParams.CLASS, "NetworkTopologyStrategy");
+
+ Keyspace.clear("Keyspace1");
+ KeyspaceMetadata meta = KeyspaceMetadata.create("Keyspace1", KeyspaceParams.create(false, configOptions));
+ Schema.instance.load(meta);
+
+ Collection<Range<Token>> primaryRanges = StorageService.instance.getLocalPrimaryRangeForEndpoint(InetAddressAndPort.getByName("127.0.0.1"));
+ assertEquals(1, primaryRanges.size());
+ assertTrue(primaryRanges.contains(new Range<Token>(new StringToken("C"), new StringToken("A"))));
+
+ primaryRanges = StorageService.instance.getLocalPrimaryRangeForEndpoint(InetAddressAndPort.getByName("127.0.0.2"));
+ assertEquals(1, primaryRanges.size());
+ assertTrue(primaryRanges.contains(new Range<Token>(new StringToken("A"), new StringToken("C"))));
+
+ primaryRanges = StorageService.instance.getLocalPrimaryRangeForEndpoint(InetAddressAndPort.getByName("127.0.0.4"));
+ assertEquals(1, primaryRanges.size());
+ assertTrue(primaryRanges.contains(new Range<Token>(new StringToken("D"), new StringToken("B"))));
+
+ primaryRanges = StorageService.instance.getLocalPrimaryRangeForEndpoint(InetAddressAndPort.getByName("127.0.0.5"));
+ assertEquals(1, primaryRanges.size());
+ assertTrue(primaryRanges.contains(new Range<Token>(new StringToken("B"), new StringToken("D"))));
+ }
@Test
public void testPrimaryRangeForEndpointWithinDCWithNetworkTopologyStrategy() throws Exception
@@ -192,12 +240,12 @@
metadata.clearUnsafe();
// DC1
- metadata.updateNormalToken(new StringToken("A"), InetAddress.getByName("127.0.0.1"));
- metadata.updateNormalToken(new StringToken("C"), InetAddress.getByName("127.0.0.2"));
+ metadata.updateNormalToken(new StringToken("A"), InetAddressAndPort.getByName("127.0.0.1"));
+ metadata.updateNormalToken(new StringToken("C"), InetAddressAndPort.getByName("127.0.0.2"));
// DC2
- metadata.updateNormalToken(new StringToken("B"), InetAddress.getByName("127.0.0.4"));
- metadata.updateNormalToken(new StringToken("D"), InetAddress.getByName("127.0.0.5"));
+ metadata.updateNormalToken(new StringToken("B"), InetAddressAndPort.getByName("127.0.0.4"));
+ metadata.updateNormalToken(new StringToken("D"), InetAddressAndPort.getByName("127.0.0.5"));
Map<String, String> configOptions = new HashMap<>();
configOptions.put("DC1", "1");
@@ -206,25 +254,25 @@
Keyspace.clear("Keyspace1");
KeyspaceMetadata meta = KeyspaceMetadata.create("Keyspace1", KeyspaceParams.create(false, configOptions));
- Schema.instance.setKeyspaceMetadata(meta);
+ Schema.instance.load(meta);
Collection<Range<Token>> primaryRanges = StorageService.instance.getPrimaryRangeForEndpointWithinDC(meta.name,
- InetAddress.getByName("127.0.0.1"));
+ InetAddressAndPort.getByName("127.0.0.1"));
assertEquals(2, primaryRanges.size());
assertTrue(primaryRanges.contains(new Range<Token>(new StringToken("D"), new StringToken("A"))));
assertTrue(primaryRanges.contains(new Range<Token>(new StringToken("C"), new StringToken("D"))));
- primaryRanges = StorageService.instance.getPrimaryRangeForEndpointWithinDC(meta.name, InetAddress.getByName("127.0.0.2"));
+ primaryRanges = StorageService.instance.getPrimaryRangeForEndpointWithinDC(meta.name, InetAddressAndPort.getByName("127.0.0.2"));
assertEquals(2, primaryRanges.size());
assertTrue(primaryRanges.contains(new Range<Token>(new StringToken("A"), new StringToken("B"))));
assertTrue(primaryRanges.contains(new Range<Token>(new StringToken("B"), new StringToken("C"))));
- primaryRanges = StorageService.instance.getPrimaryRangeForEndpointWithinDC(meta.name, InetAddress.getByName("127.0.0.4"));
+ primaryRanges = StorageService.instance.getPrimaryRangeForEndpointWithinDC(meta.name, InetAddressAndPort.getByName("127.0.0.4"));
assertEquals(2, primaryRanges.size());
assertTrue(primaryRanges.contains(new Range<Token>(new StringToken("D"), new StringToken("A"))));
assertTrue(primaryRanges.contains(new Range<Token>(new StringToken("A"), new StringToken("B"))));
- primaryRanges = StorageService.instance.getPrimaryRangeForEndpointWithinDC(meta.name, InetAddress.getByName("127.0.0.5"));
+ primaryRanges = StorageService.instance.getPrimaryRangeForEndpointWithinDC(meta.name, InetAddressAndPort.getByName("127.0.0.5"));
assertEquals(2, primaryRanges.size());
assertTrue(primaryRanges.contains(new Range<Token>(new StringToken("B"), new StringToken("C"))));
assertTrue(primaryRanges.contains(new Range<Token>(new StringToken("C"), new StringToken("D"))));
@@ -236,11 +284,11 @@
TokenMetadata metadata = StorageService.instance.getTokenMetadata();
metadata.clearUnsafe();
// DC1
- metadata.updateNormalToken(new StringToken("A"), InetAddress.getByName("127.0.0.1"));
- metadata.updateNormalToken(new StringToken("C"), InetAddress.getByName("127.0.0.2"));
+ metadata.updateNormalToken(new StringToken("A"), InetAddressAndPort.getByName("127.0.0.1"));
+ metadata.updateNormalToken(new StringToken("C"), InetAddressAndPort.getByName("127.0.0.2"));
// DC2
- metadata.updateNormalToken(new StringToken("B"), InetAddress.getByName("127.0.0.4"));
- metadata.updateNormalToken(new StringToken("D"), InetAddress.getByName("127.0.0.5"));
+ metadata.updateNormalToken(new StringToken("B"), InetAddressAndPort.getByName("127.0.0.4"));
+ metadata.updateNormalToken(new StringToken("D"), InetAddressAndPort.getByName("127.0.0.5"));
Map<String, String> configOptions = new HashMap<>();
configOptions.put("DC1", "1");
@@ -249,21 +297,21 @@
Keyspace.clear("Keyspace1");
KeyspaceMetadata meta = KeyspaceMetadata.create("Keyspace1", KeyspaceParams.create(false, configOptions));
- Schema.instance.setKeyspaceMetadata(meta);
+ Schema.instance.load(meta);
- Collection<Range<Token>> primaryRanges = StorageService.instance.getPrimaryRangesForEndpoint(meta.name, InetAddress.getByName("127.0.0.1"));
+ Collection<Range<Token>> primaryRanges = StorageService.instance.getPrimaryRangesForEndpoint(meta.name, InetAddressAndPort.getByName("127.0.0.1"));
assert primaryRanges.size() == 1;
assert primaryRanges.contains(new Range<Token>(new StringToken("D"), new StringToken("A")));
- primaryRanges = StorageService.instance.getPrimaryRangesForEndpoint(meta.name, InetAddress.getByName("127.0.0.2"));
+ primaryRanges = StorageService.instance.getPrimaryRangesForEndpoint(meta.name, InetAddressAndPort.getByName("127.0.0.2"));
assert primaryRanges.size() == 1;
assert primaryRanges.contains(new Range<Token>(new StringToken("B"), new StringToken("C")));
- primaryRanges = StorageService.instance.getPrimaryRangesForEndpoint(meta.name, InetAddress.getByName("127.0.0.4"));
+ primaryRanges = StorageService.instance.getPrimaryRangesForEndpoint(meta.name, InetAddressAndPort.getByName("127.0.0.4"));
assert primaryRanges.size() == 1;
assert primaryRanges.contains(new Range<Token>(new StringToken("A"), new StringToken("B")));
- primaryRanges = StorageService.instance.getPrimaryRangesForEndpoint(meta.name, InetAddress.getByName("127.0.0.5"));
+ primaryRanges = StorageService.instance.getPrimaryRangesForEndpoint(meta.name, InetAddressAndPort.getByName("127.0.0.5"));
assert primaryRanges.size() == 1;
assert primaryRanges.contains(new Range<Token>(new StringToken("C"), new StringToken("D")));
}
@@ -274,11 +322,11 @@
TokenMetadata metadata = StorageService.instance.getTokenMetadata();
metadata.clearUnsafe();
// DC1
- metadata.updateNormalToken(new StringToken("A"), InetAddress.getByName("127.0.0.1"));
- metadata.updateNormalToken(new StringToken("C"), InetAddress.getByName("127.0.0.2"));
+ metadata.updateNormalToken(new StringToken("A"), InetAddressAndPort.getByName("127.0.0.1"));
+ metadata.updateNormalToken(new StringToken("C"), InetAddressAndPort.getByName("127.0.0.2"));
// DC2
- metadata.updateNormalToken(new StringToken("B"), InetAddress.getByName("127.0.0.4"));
- metadata.updateNormalToken(new StringToken("D"), InetAddress.getByName("127.0.0.5"));
+ metadata.updateNormalToken(new StringToken("B"), InetAddressAndPort.getByName("127.0.0.4"));
+ metadata.updateNormalToken(new StringToken("D"), InetAddressAndPort.getByName("127.0.0.5"));
Map<String, String> configOptions = new HashMap<>();
configOptions.put("DC2", "2");
@@ -286,22 +334,22 @@
Keyspace.clear("Keyspace1");
KeyspaceMetadata meta = KeyspaceMetadata.create("Keyspace1", KeyspaceParams.create(false, configOptions));
- Schema.instance.setKeyspaceMetadata(meta);
+ Schema.instance.load(meta);
// endpoints in DC1 should not have primary range
- Collection<Range<Token>> primaryRanges = StorageService.instance.getPrimaryRangesForEndpoint(meta.name, InetAddress.getByName("127.0.0.1"));
+ Collection<Range<Token>> primaryRanges = StorageService.instance.getPrimaryRangesForEndpoint(meta.name, InetAddressAndPort.getByName("127.0.0.1"));
assert primaryRanges.isEmpty();
- primaryRanges = StorageService.instance.getPrimaryRangesForEndpoint(meta.name, InetAddress.getByName("127.0.0.2"));
+ primaryRanges = StorageService.instance.getPrimaryRangesForEndpoint(meta.name, InetAddressAndPort.getByName("127.0.0.2"));
assert primaryRanges.isEmpty();
// endpoints in DC2 should have primary ranges which also cover DC1
- primaryRanges = StorageService.instance.getPrimaryRangesForEndpoint(meta.name, InetAddress.getByName("127.0.0.4"));
+ primaryRanges = StorageService.instance.getPrimaryRangesForEndpoint(meta.name, InetAddressAndPort.getByName("127.0.0.4"));
assert primaryRanges.size() == 2;
assert primaryRanges.contains(new Range<Token>(new StringToken("D"), new StringToken("A")));
assert primaryRanges.contains(new Range<Token>(new StringToken("A"), new StringToken("B")));
- primaryRanges = StorageService.instance.getPrimaryRangesForEndpoint(meta.name, InetAddress.getByName("127.0.0.5"));
+ primaryRanges = StorageService.instance.getPrimaryRangesForEndpoint(meta.name, InetAddressAndPort.getByName("127.0.0.5"));
assert primaryRanges.size() == 2;
assert primaryRanges.contains(new Range<Token>(new StringToken("C"), new StringToken("D")));
assert primaryRanges.contains(new Range<Token>(new StringToken("B"), new StringToken("C")));
@@ -313,11 +361,11 @@
TokenMetadata metadata = StorageService.instance.getTokenMetadata();
metadata.clearUnsafe();
// DC1
- metadata.updateNormalToken(new StringToken("A"), InetAddress.getByName("127.0.0.1"));
- metadata.updateNormalToken(new StringToken("C"), InetAddress.getByName("127.0.0.2"));
+ metadata.updateNormalToken(new StringToken("A"), InetAddressAndPort.getByName("127.0.0.1"));
+ metadata.updateNormalToken(new StringToken("C"), InetAddressAndPort.getByName("127.0.0.2"));
// DC2
- metadata.updateNormalToken(new StringToken("B"), InetAddress.getByName("127.0.0.4"));
- metadata.updateNormalToken(new StringToken("D"), InetAddress.getByName("127.0.0.5"));
+ metadata.updateNormalToken(new StringToken("B"), InetAddressAndPort.getByName("127.0.0.4"));
+ metadata.updateNormalToken(new StringToken("D"), InetAddressAndPort.getByName("127.0.0.5"));
Map<String, String> configOptions = new HashMap<>();
configOptions.put("DC2", "2");
@@ -325,23 +373,23 @@
Keyspace.clear("Keyspace1");
KeyspaceMetadata meta = KeyspaceMetadata.create("Keyspace1", KeyspaceParams.create(false, configOptions));
- Schema.instance.setKeyspaceMetadata(meta);
+ Schema.instance.load(meta);
// endpoints in DC1 should not have primary range
- Collection<Range<Token>> primaryRanges = StorageService.instance.getPrimaryRangeForEndpointWithinDC(meta.name, InetAddress.getByName("127.0.0.1"));
+ Collection<Range<Token>> primaryRanges = StorageService.instance.getPrimaryRangeForEndpointWithinDC(meta.name, InetAddressAndPort.getByName("127.0.0.1"));
assertTrue(primaryRanges.isEmpty());
primaryRanges = StorageService.instance.getPrimaryRangeForEndpointWithinDC(meta.name,
- InetAddress.getByName("127.0.0.2"));
+ InetAddressAndPort.getByName("127.0.0.2"));
assertTrue(primaryRanges.isEmpty());
// endpoints in DC2 should have primary ranges which also cover DC1
- primaryRanges = StorageService.instance.getPrimaryRangeForEndpointWithinDC(meta.name, InetAddress.getByName("127.0.0.4"));
+ primaryRanges = StorageService.instance.getPrimaryRangeForEndpointWithinDC(meta.name, InetAddressAndPort.getByName("127.0.0.4"));
assertTrue(primaryRanges.size() == 2);
assertTrue(primaryRanges.contains(new Range<Token>(new StringToken("D"), new StringToken("A"))));
assertTrue(primaryRanges.contains(new Range<Token>(new StringToken("A"), new StringToken("B"))));
- primaryRanges = StorageService.instance.getPrimaryRangeForEndpointWithinDC(meta.name, InetAddress.getByName("127.0.0.5"));
+ primaryRanges = StorageService.instance.getPrimaryRangeForEndpointWithinDC(meta.name, InetAddressAndPort.getByName("127.0.0.5"));
assertTrue(primaryRanges.size() == 2);
assertTrue(primaryRanges.contains(new Range<Token>(new StringToken("C"), new StringToken("D"))));
assertTrue(primaryRanges.contains(new Range<Token>(new StringToken("B"), new StringToken("C"))));
@@ -353,22 +401,22 @@
TokenMetadata metadata = StorageService.instance.getTokenMetadata();
metadata.clearUnsafe();
// DC1
- Multimap<InetAddress, Token> dc1 = HashMultimap.create();
- dc1.put(InetAddress.getByName("127.0.0.1"), new StringToken("A"));
- dc1.put(InetAddress.getByName("127.0.0.1"), new StringToken("E"));
- dc1.put(InetAddress.getByName("127.0.0.1"), new StringToken("H"));
- dc1.put(InetAddress.getByName("127.0.0.2"), new StringToken("C"));
- dc1.put(InetAddress.getByName("127.0.0.2"), new StringToken("I"));
- dc1.put(InetAddress.getByName("127.0.0.2"), new StringToken("J"));
+ Multimap<InetAddressAndPort, Token> dc1 = HashMultimap.create();
+ dc1.put(InetAddressAndPort.getByName("127.0.0.1"), new StringToken("A"));
+ dc1.put(InetAddressAndPort.getByName("127.0.0.1"), new StringToken("E"));
+ dc1.put(InetAddressAndPort.getByName("127.0.0.1"), new StringToken("H"));
+ dc1.put(InetAddressAndPort.getByName("127.0.0.2"), new StringToken("C"));
+ dc1.put(InetAddressAndPort.getByName("127.0.0.2"), new StringToken("I"));
+ dc1.put(InetAddressAndPort.getByName("127.0.0.2"), new StringToken("J"));
metadata.updateNormalTokens(dc1);
// DC2
- Multimap<InetAddress, Token> dc2 = HashMultimap.create();
- dc2.put(InetAddress.getByName("127.0.0.4"), new StringToken("B"));
- dc2.put(InetAddress.getByName("127.0.0.4"), new StringToken("G"));
- dc2.put(InetAddress.getByName("127.0.0.4"), new StringToken("L"));
- dc2.put(InetAddress.getByName("127.0.0.5"), new StringToken("D"));
- dc2.put(InetAddress.getByName("127.0.0.5"), new StringToken("F"));
- dc2.put(InetAddress.getByName("127.0.0.5"), new StringToken("K"));
+ Multimap<InetAddressAndPort, Token> dc2 = HashMultimap.create();
+ dc2.put(InetAddressAndPort.getByName("127.0.0.4"), new StringToken("B"));
+ dc2.put(InetAddressAndPort.getByName("127.0.0.4"), new StringToken("G"));
+ dc2.put(InetAddressAndPort.getByName("127.0.0.4"), new StringToken("L"));
+ dc2.put(InetAddressAndPort.getByName("127.0.0.5"), new StringToken("D"));
+ dc2.put(InetAddressAndPort.getByName("127.0.0.5"), new StringToken("F"));
+ dc2.put(InetAddressAndPort.getByName("127.0.0.5"), new StringToken("K"));
metadata.updateNormalTokens(dc2);
Map<String, String> configOptions = new HashMap<>();
@@ -377,17 +425,17 @@
Keyspace.clear("Keyspace1");
KeyspaceMetadata meta = KeyspaceMetadata.create("Keyspace1", KeyspaceParams.create(false, configOptions));
- Schema.instance.setKeyspaceMetadata(meta);
+ Schema.instance.load(meta);
// endpoints in DC1 should not have primary range
- Collection<Range<Token>> primaryRanges = StorageService.instance.getPrimaryRangesForEndpoint(meta.name, InetAddress.getByName("127.0.0.1"));
+ Collection<Range<Token>> primaryRanges = StorageService.instance.getPrimaryRangesForEndpoint(meta.name, InetAddressAndPort.getByName("127.0.0.1"));
assert primaryRanges.isEmpty();
- primaryRanges = StorageService.instance.getPrimaryRangesForEndpoint(meta.name, InetAddress.getByName("127.0.0.2"));
+ primaryRanges = StorageService.instance.getPrimaryRangesForEndpoint(meta.name, InetAddressAndPort.getByName("127.0.0.2"));
assert primaryRanges.isEmpty();
// endpoints in DC2 should have primary ranges which also cover DC1
- primaryRanges = StorageService.instance.getPrimaryRangesForEndpoint(meta.name, InetAddress.getByName("127.0.0.4"));
+ primaryRanges = StorageService.instance.getPrimaryRangesForEndpoint(meta.name, InetAddressAndPort.getByName("127.0.0.4"));
assert primaryRanges.size() == 4;
assert primaryRanges.contains(new Range<Token>(new StringToken("A"), new StringToken("B")));
assert primaryRanges.contains(new Range<Token>(new StringToken("F"), new StringToken("G")));
@@ -396,7 +444,7 @@
// the node covers range (L, A]
assert primaryRanges.contains(new Range<Token>(new StringToken("L"), new StringToken("A")));
- primaryRanges = StorageService.instance.getPrimaryRangesForEndpoint(meta.name, InetAddress.getByName("127.0.0.5"));
+ primaryRanges = StorageService.instance.getPrimaryRangesForEndpoint(meta.name, InetAddressAndPort.getByName("127.0.0.5"));
assert primaryRanges.size() == 8;
assert primaryRanges.contains(new Range<Token>(new StringToken("C"), new StringToken("D")));
assert primaryRanges.contains(new Range<Token>(new StringToken("E"), new StringToken("F")));
@@ -418,23 +466,23 @@
metadata.clearUnsafe();
// DC1
- Multimap<InetAddress, Token> dc1 = HashMultimap.create();
- dc1.put(InetAddress.getByName("127.0.0.1"), new StringToken("A"));
- dc1.put(InetAddress.getByName("127.0.0.1"), new StringToken("E"));
- dc1.put(InetAddress.getByName("127.0.0.1"), new StringToken("H"));
- dc1.put(InetAddress.getByName("127.0.0.2"), new StringToken("C"));
- dc1.put(InetAddress.getByName("127.0.0.2"), new StringToken("I"));
- dc1.put(InetAddress.getByName("127.0.0.2"), new StringToken("J"));
+ Multimap<InetAddressAndPort, Token> dc1 = HashMultimap.create();
+ dc1.put(InetAddressAndPort.getByName("127.0.0.1"), new StringToken("A"));
+ dc1.put(InetAddressAndPort.getByName("127.0.0.1"), new StringToken("E"));
+ dc1.put(InetAddressAndPort.getByName("127.0.0.1"), new StringToken("H"));
+ dc1.put(InetAddressAndPort.getByName("127.0.0.2"), new StringToken("C"));
+ dc1.put(InetAddressAndPort.getByName("127.0.0.2"), new StringToken("I"));
+ dc1.put(InetAddressAndPort.getByName("127.0.0.2"), new StringToken("J"));
metadata.updateNormalTokens(dc1);
// DC2
- Multimap<InetAddress, Token> dc2 = HashMultimap.create();
- dc2.put(InetAddress.getByName("127.0.0.4"), new StringToken("B"));
- dc2.put(InetAddress.getByName("127.0.0.4"), new StringToken("G"));
- dc2.put(InetAddress.getByName("127.0.0.4"), new StringToken("L"));
- dc2.put(InetAddress.getByName("127.0.0.5"), new StringToken("D"));
- dc2.put(InetAddress.getByName("127.0.0.5"), new StringToken("F"));
- dc2.put(InetAddress.getByName("127.0.0.5"), new StringToken("K"));
+ Multimap<InetAddressAndPort, Token> dc2 = HashMultimap.create();
+ dc2.put(InetAddressAndPort.getByName("127.0.0.4"), new StringToken("B"));
+ dc2.put(InetAddressAndPort.getByName("127.0.0.4"), new StringToken("G"));
+ dc2.put(InetAddressAndPort.getByName("127.0.0.4"), new StringToken("L"));
+ dc2.put(InetAddressAndPort.getByName("127.0.0.5"), new StringToken("D"));
+ dc2.put(InetAddressAndPort.getByName("127.0.0.5"), new StringToken("F"));
+ dc2.put(InetAddressAndPort.getByName("127.0.0.5"), new StringToken("K"));
metadata.updateNormalTokens(dc2);
Map<String, String> configOptions = new HashMap<>();
@@ -444,10 +492,10 @@
Keyspace.clear("Keyspace1");
KeyspaceMetadata meta = KeyspaceMetadata.create("Keyspace1", KeyspaceParams.create(false, configOptions));
- Schema.instance.setKeyspaceMetadata(meta);
+ Schema.instance.load(meta);
// endpoints in DC1 should have primary ranges which also cover DC2
- Collection<Range<Token>> primaryRanges = StorageService.instance.getPrimaryRangeForEndpointWithinDC(meta.name, InetAddress.getByName("127.0.0.1"));
+ Collection<Range<Token>> primaryRanges = StorageService.instance.getPrimaryRangeForEndpointWithinDC(meta.name, InetAddressAndPort.getByName("127.0.0.1"));
assertEquals(8, primaryRanges.size());
assertTrue(primaryRanges.contains(new Range<Token>(new StringToken("J"), new StringToken("K"))));
assertTrue(primaryRanges.contains(new Range<Token>(new StringToken("K"), new StringToken("L"))));
@@ -459,7 +507,7 @@
assertTrue(primaryRanges.contains(new Range<Token>(new StringToken("G"), new StringToken("H"))));
// endpoints in DC1 should have primary ranges which also cover DC2
- primaryRanges = StorageService.instance.getPrimaryRangeForEndpointWithinDC(meta.name, InetAddress.getByName("127.0.0.2"));
+ primaryRanges = StorageService.instance.getPrimaryRangeForEndpointWithinDC(meta.name, InetAddressAndPort.getByName("127.0.0.2"));
assertEquals(4, primaryRanges.size());
assertTrue(primaryRanges.contains(new Range<Token>(new StringToken("B"), new StringToken("C"))));
assertTrue(primaryRanges.contains(new Range<Token>(new StringToken("A"), new StringToken("B"))));
@@ -467,7 +515,7 @@
assertTrue(primaryRanges.contains(new Range<Token>(new StringToken("I"), new StringToken("J"))));
// endpoints in DC2 should have primary ranges which also cover DC1
- primaryRanges = StorageService.instance.getPrimaryRangeForEndpointWithinDC(meta.name, InetAddress.getByName("127.0.0.4"));
+ primaryRanges = StorageService.instance.getPrimaryRangeForEndpointWithinDC(meta.name, InetAddressAndPort.getByName("127.0.0.4"));
assertEquals(4, primaryRanges.size());
assertTrue(primaryRanges.contains(new Range<Token>(new StringToken("A"), new StringToken("B"))));
assertTrue(primaryRanges.contains(new Range<Token>(new StringToken("F"), new StringToken("G"))));
@@ -476,7 +524,7 @@
// the node covers range (L, A]
assertTrue(primaryRanges.contains(new Range<Token>(new StringToken("L"), new StringToken("A"))));
- primaryRanges = StorageService.instance.getPrimaryRangeForEndpointWithinDC(meta.name, InetAddress.getByName("127.0.0.5"));
+ primaryRanges = StorageService.instance.getPrimaryRangeForEndpointWithinDC(meta.name, InetAddressAndPort.getByName("127.0.0.5"));
assertTrue(primaryRanges.size() == 8);
assertTrue(primaryRanges.contains(new Range<Token>(new StringToken("C"), new StringToken("D"))));
assertTrue(primaryRanges.contains(new Range<Token>(new StringToken("E"), new StringToken("F"))));
@@ -497,23 +545,23 @@
TokenMetadata metadata = StorageService.instance.getTokenMetadata();
metadata.clearUnsafe();
- metadata.updateNormalToken(new StringToken("A"), InetAddress.getByName("127.0.0.1"));
- metadata.updateNormalToken(new StringToken("B"), InetAddress.getByName("127.0.0.2"));
- metadata.updateNormalToken(new StringToken("C"), InetAddress.getByName("127.0.0.3"));
+ metadata.updateNormalToken(new StringToken("A"), InetAddressAndPort.getByName("127.0.0.1"));
+ metadata.updateNormalToken(new StringToken("B"), InetAddressAndPort.getByName("127.0.0.2"));
+ metadata.updateNormalToken(new StringToken("C"), InetAddressAndPort.getByName("127.0.0.3"));
Keyspace.clear("Keyspace1");
KeyspaceMetadata meta = KeyspaceMetadata.create("Keyspace1", KeyspaceParams.simpleTransient(2));
- Schema.instance.setKeyspaceMetadata(meta);
+ Schema.instance.load(meta);
- Collection<Range<Token>> primaryRanges = StorageService.instance.getPrimaryRangesForEndpoint(meta.name, InetAddress.getByName("127.0.0.1"));
+ Collection<Range<Token>> primaryRanges = StorageService.instance.getPrimaryRangesForEndpoint(meta.name, InetAddressAndPort.getByName("127.0.0.1"));
assert primaryRanges.size() == 1;
assert primaryRanges.contains(new Range<Token>(new StringToken("C"), new StringToken("A")));
- primaryRanges = StorageService.instance.getPrimaryRangesForEndpoint(meta.name, InetAddress.getByName("127.0.0.2"));
+ primaryRanges = StorageService.instance.getPrimaryRangesForEndpoint(meta.name, InetAddressAndPort.getByName("127.0.0.2"));
assert primaryRanges.size() == 1;
assert primaryRanges.contains(new Range<Token>(new StringToken("A"), new StringToken("B")));
- primaryRanges = StorageService.instance.getPrimaryRangesForEndpoint(meta.name, InetAddress.getByName("127.0.0.3"));
+ primaryRanges = StorageService.instance.getPrimaryRangesForEndpoint(meta.name, InetAddressAndPort.getByName("127.0.0.3"));
assert primaryRanges.size() == 1;
assert primaryRanges.contains(new Range<Token>(new StringToken("B"), new StringToken("C")));
}
@@ -525,26 +573,26 @@
TokenMetadata metadata = StorageService.instance.getTokenMetadata();
metadata.clearUnsafe();
- metadata.updateNormalToken(new StringToken("A"), InetAddress.getByName("127.0.0.1"));
- metadata.updateNormalToken(new StringToken("B"), InetAddress.getByName("127.0.0.2"));
- metadata.updateNormalToken(new StringToken("C"), InetAddress.getByName("127.0.0.3"));
+ metadata.updateNormalToken(new StringToken("A"), InetAddressAndPort.getByName("127.0.0.1"));
+ metadata.updateNormalToken(new StringToken("B"), InetAddressAndPort.getByName("127.0.0.2"));
+ metadata.updateNormalToken(new StringToken("C"), InetAddressAndPort.getByName("127.0.0.3"));
Map<String, String> configOptions = new HashMap<>();
configOptions.put("replication_factor", "2");
Keyspace.clear("Keyspace1");
KeyspaceMetadata meta = KeyspaceMetadata.create("Keyspace1", KeyspaceParams.simpleTransient(2));
- Schema.instance.setKeyspaceMetadata(meta);
+ Schema.instance.load(meta);
- Collection<Range<Token>> primaryRanges = StorageService.instance.getPrimaryRangeForEndpointWithinDC(meta.name, InetAddress.getByName("127.0.0.1"));
+ Collection<Range<Token>> primaryRanges = StorageService.instance.getPrimaryRangeForEndpointWithinDC(meta.name, InetAddressAndPort.getByName("127.0.0.1"));
assert primaryRanges.size() == 1;
assert primaryRanges.contains(new Range<Token>(new StringToken("C"), new StringToken("A")));
- primaryRanges = StorageService.instance.getPrimaryRangeForEndpointWithinDC(meta.name, InetAddress.getByName("127.0.0.2"));
+ primaryRanges = StorageService.instance.getPrimaryRangeForEndpointWithinDC(meta.name, InetAddressAndPort.getByName("127.0.0.2"));
assert primaryRanges.size() == 1;
assert primaryRanges.contains(new Range<Token>(new StringToken("A"), new StringToken("B")));
- primaryRanges = StorageService.instance.getPrimaryRangeForEndpointWithinDC(meta.name, InetAddress.getByName("127.0.0.3"));
+ primaryRanges = StorageService.instance.getPrimaryRangeForEndpointWithinDC(meta.name, InetAddressAndPort.getByName("127.0.0.3"));
assert primaryRanges.size() == 1;
assert primaryRanges.contains(new Range<Token>(new StringToken("B"), new StringToken("C")));
}
@@ -557,10 +605,10 @@
TokenMetadata metadata = StorageService.instance.getTokenMetadata();
metadata.clearUnsafe();
- metadata.updateNormalToken(new LongToken(1000L), InetAddress.getByName("127.0.0.1"));
- metadata.updateNormalToken(new LongToken(2000L), InetAddress.getByName("127.0.0.2"));
- metadata.updateNormalToken(new LongToken(3000L), InetAddress.getByName("127.0.0.3"));
- metadata.updateNormalToken(new LongToken(4000L), InetAddress.getByName("127.0.0.4"));
+ metadata.updateNormalToken(new LongToken(1000L), InetAddressAndPort.getByName("127.0.0.1"));
+ metadata.updateNormalToken(new LongToken(2000L), InetAddressAndPort.getByName("127.0.0.2"));
+ metadata.updateNormalToken(new LongToken(3000L), InetAddressAndPort.getByName("127.0.0.3"));
+ metadata.updateNormalToken(new LongToken(4000L), InetAddressAndPort.getByName("127.0.0.4"));
Collection<Range<Token>> repairRangeFrom = StorageService.instance.createRepairRangeFrom("1500", "3700");
assert repairRangeFrom.size() == 3;
@@ -592,4 +640,62 @@
repairRangeFrom = StorageService.instance.createRepairRangeFrom("2000", "2000");
assert repairRangeFrom.size() == 0;
}
+
+ /**
+ * Test that StorageService.getNativeAddress returns the correct value based on available yaml and gossip state
+ * @throws Exception
+ */
+ @Test
+ public void testGetNativeAddress() throws Exception
+ {
+ String internalAddressString = "127.0.0.2:666";
+ InetAddressAndPort internalAddress = InetAddressAndPort.getByName(internalAddressString);
+ Gossiper.instance.addSavedEndpoint(internalAddress);
+ //Default to using the provided address with the configured port
+ assertEquals("127.0.0.2:" + DatabaseDescriptor.getNativeTransportPort(), StorageService.instance.getNativeaddress(internalAddress, true));
+
+ VersionedValue.VersionedValueFactory valueFactory = new VersionedValue.VersionedValueFactory(Murmur3Partitioner.instance);
+ //If we don't have the port use the gossip address, but with the configured port
+ Gossiper.instance.getEndpointStateForEndpoint(internalAddress).addApplicationState(ApplicationState.RPC_ADDRESS, valueFactory.rpcaddress(InetAddress.getByName("127.0.0.3")));
+ assertEquals("127.0.0.3:" + DatabaseDescriptor.getNativeTransportPort(), StorageService.instance.getNativeaddress(internalAddress, true));
+ //If we have the address and port in gossip use that
+ Gossiper.instance.getEndpointStateForEndpoint(internalAddress).addApplicationState(ApplicationState.NATIVE_ADDRESS_AND_PORT, valueFactory.nativeaddressAndPort(InetAddressAndPort.getByName("127.0.0.3:666")));
+ assertEquals("127.0.0.3:666", StorageService.instance.getNativeaddress(internalAddress, true));
+ }
+
+ @Test
+ public void testAuditLogEnableLoggerNotFound() throws Exception
+ {
+ StorageService.instance.enableAuditLog(null, null, null, null, null, null, null, null);
+ assertTrue(AuditLogManager.instance.isEnabled());
+ try
+ {
+ StorageService.instance.enableAuditLog("foobar", null, null, null, null, null, null, null);
+ Assert.fail();
+ }
+ catch (IllegalStateException ex)
+ {
+ StorageService.instance.disableAuditLog();
+ }
+ }
+
+ @Test
+ public void testAuditLogEnableLoggerTransitions() throws Exception
+ {
+ StorageService.instance.enableAuditLog(null, null, null, null, null, null, null, null);
+ assertTrue(AuditLogManager.instance.isEnabled());
+
+ try
+ {
+ StorageService.instance.enableAuditLog("foobar", null, null, null, null, null, null, null);
+ }
+ catch (ConfigurationException | IllegalStateException e)
+ {
+ e.printStackTrace();
+ }
+
+ StorageService.instance.enableAuditLog(null, null, null, null, null, null, null, null);
+ assertTrue(AuditLogManager.instance.isEnabled());
+ StorageService.instance.disableAuditLog();
+ }
}
diff --git a/test/unit/org/apache/cassandra/service/StorageServiceTest.java b/test/unit/org/apache/cassandra/service/StorageServiceTest.java
new file mode 100644
index 0000000..f22f89f
--- /dev/null
+++ b/test/unit/org/apache/cassandra/service/StorageServiceTest.java
@@ -0,0 +1,160 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service;
+
+import org.apache.cassandra.locator.EndpointsByReplica;
+import org.apache.cassandra.locator.ReplicaCollection;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.dht.RandomPartitioner;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.locator.AbstractEndpointSnitch;
+import org.apache.cassandra.locator.AbstractReplicationStrategy;
+import org.apache.cassandra.locator.IEndpointSnitch;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.Replica;
+import org.apache.cassandra.locator.ReplicaMultimap;
+import org.apache.cassandra.locator.SimpleStrategy;
+import org.apache.cassandra.locator.TokenMetadata;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+public class StorageServiceTest
+{
+ static InetAddressAndPort aAddress;
+ static InetAddressAndPort bAddress;
+ static InetAddressAndPort cAddress;
+ static InetAddressAndPort dAddress;
+ static InetAddressAndPort eAddress;
+
+ @BeforeClass
+ public static void setUpClass() throws Exception
+ {
+ aAddress = InetAddressAndPort.getByName("127.0.0.1");
+ bAddress = InetAddressAndPort.getByName("127.0.0.2");
+ cAddress = InetAddressAndPort.getByName("127.0.0.3");
+ dAddress = InetAddressAndPort.getByName("127.0.0.4");
+ eAddress = InetAddressAndPort.getByName("127.0.0.5");
+ }
+
+ private static final Token threeToken = new RandomPartitioner.BigIntegerToken("3");
+ private static final Token sixToken = new RandomPartitioner.BigIntegerToken("6");
+ private static final Token nineToken = new RandomPartitioner.BigIntegerToken("9");
+ private static final Token elevenToken = new RandomPartitioner.BigIntegerToken("11");
+ private static final Token oneToken = new RandomPartitioner.BigIntegerToken("1");
+
+ Range<Token> aRange = new Range<>(oneToken, threeToken);
+ Range<Token> bRange = new Range<>(threeToken, sixToken);
+ Range<Token> cRange = new Range<>(sixToken, nineToken);
+ Range<Token> dRange = new Range<>(nineToken, elevenToken);
+ Range<Token> eRange = new Range<>(elevenToken, oneToken);
+
+ @Before
+ public void setUp()
+ {
+ DatabaseDescriptor.daemonInitialization();
+ DatabaseDescriptor.setTransientReplicationEnabledUnsafe(true);
+ IEndpointSnitch snitch = new AbstractEndpointSnitch()
+ {
+ public int compareEndpoints(InetAddressAndPort target, Replica r1, Replica r2)
+ {
+ return 0;
+ }
+
+ public String getRack(InetAddressAndPort endpoint)
+ {
+ return "R1";
+ }
+
+ public String getDatacenter(InetAddressAndPort endpoint)
+ {
+ return "DC1";
+ }
+ };
+
+ DatabaseDescriptor.setEndpointSnitch(snitch);
+ }
+
+ private AbstractReplicationStrategy simpleStrategy(TokenMetadata tmd)
+ {
+ return new SimpleStrategy("MoveTransientTest",
+ tmd,
+ DatabaseDescriptor.getEndpointSnitch(),
+ com.google.common.collect.ImmutableMap.of("replication_factor", "3/1"));
+ }
+
+ public static <K, C extends ReplicaCollection<? extends C>> void assertMultimapEqualsIgnoreOrder(ReplicaMultimap<K, C> a, ReplicaMultimap<K, C> b)
+ {
+ if (!a.keySet().equals(b.keySet()))
+ fail(formatNeq(a, b));
+ for (K key : a.keySet())
+ {
+ C ac = a.get(key);
+ C bc = b.get(key);
+ if (ac.size() != bc.size())
+ fail(formatNeq(a, b));
+ for (Replica r : ac)
+ {
+ if (!bc.contains(r))
+ fail(formatNeq(a, b));
+ }
+ }
+ }
+
+ public static String formatNeq(Object v1, Object v2)
+ {
+ return "\nExpected: " + formatClassAndValue(v1) + "\n but was: " + formatClassAndValue(v2);
+ }
+
+ public static String formatClassAndValue(Object value)
+ {
+ String className = value == null ? "null" : value.getClass().getName();
+ return className + "<" + String.valueOf(value) + ">";
+ }
+
+ @Test
+ public void testGetChangedReplicasForLeaving() throws Exception
+ {
+ TokenMetadata tmd = new TokenMetadata();
+ tmd.updateNormalToken(threeToken, aAddress);
+ tmd.updateNormalToken(sixToken, bAddress);
+ tmd.updateNormalToken(nineToken, cAddress);
+ tmd.updateNormalToken(elevenToken, dAddress);
+ tmd.updateNormalToken(oneToken, eAddress);
+
+ tmd.addLeavingEndpoint(aAddress);
+
+ AbstractReplicationStrategy strat = simpleStrategy(tmd);
+
+ EndpointsByReplica result = StorageService.getChangedReplicasForLeaving("StorageServiceTest", aAddress, tmd, strat);
+ System.out.println(result);
+ EndpointsByReplica.Builder expectedResult = new EndpointsByReplica.Builder();
+ expectedResult.put(new Replica(aAddress, aRange, true), new Replica(cAddress, new Range<>(oneToken, sixToken), true));
+ expectedResult.put(new Replica(aAddress, aRange, true), new Replica(dAddress, new Range<>(oneToken, sixToken), false));
+ expectedResult.put(new Replica(aAddress, eRange, true), new Replica(bAddress, eRange, true));
+ expectedResult.put(new Replica(aAddress, eRange, true), new Replica(cAddress, eRange, false));
+ expectedResult.put(new Replica(aAddress, dRange, false), new Replica(bAddress, dRange, false));
+ assertMultimapEqualsIgnoreOrder(result, expectedResult.build());
+ }
+}
diff --git a/test/unit/org/apache/cassandra/service/WriteResponseHandlerTest.java b/test/unit/org/apache/cassandra/service/WriteResponseHandlerTest.java
new file mode 100644
index 0000000..5d8d191
--- /dev/null
+++ b/test/unit/org/apache/cassandra/service/WriteResponseHandlerTest.java
@@ -0,0 +1,278 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service;
+
+
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+
+import com.google.common.base.Predicates;
+import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.locator.EndpointsForToken;
+import org.apache.cassandra.locator.ReplicaPlans;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.ConsistencyLevel;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.WriteType;
+import org.apache.cassandra.locator.IEndpointSnitch;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.Replica;
+import org.apache.cassandra.locator.ReplicaCollection;
+import org.apache.cassandra.locator.ReplicaUtils;
+import org.apache.cassandra.locator.TokenMetadata;
+import org.apache.cassandra.net.Message;
+import org.apache.cassandra.net.Verb;
+import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.utils.ByteBufferUtil;
+
+import static org.apache.cassandra.net.NoPayload.noPayload;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class WriteResponseHandlerTest
+{
+ static Keyspace ks;
+ static ColumnFamilyStore cfs;
+ static EndpointsForToken targets;
+ static EndpointsForToken pending;
+
+ private static Replica full(String name)
+ {
+ try
+ {
+ return ReplicaUtils.full(InetAddressAndPort.getByName(name));
+ }
+ catch (UnknownHostException e)
+ {
+ throw new AssertionError(e);
+ }
+ }
+
+ @BeforeClass
+ public static void setUpClass() throws Throwable
+ {
+ SchemaLoader.loadSchema();
+ DatabaseDescriptor.setPartitionerUnsafe(Murmur3Partitioner.instance);
+ // Register peers with expected DC for NetworkTopologyStrategy.
+ TokenMetadata metadata = StorageService.instance.getTokenMetadata();
+ metadata.clearUnsafe();
+ metadata.updateHostId(UUID.randomUUID(), InetAddressAndPort.getByName("127.1.0.255"));
+ metadata.updateHostId(UUID.randomUUID(), InetAddressAndPort.getByName("127.2.0.255"));
+
+ DatabaseDescriptor.setEndpointSnitch(new IEndpointSnitch()
+ {
+ public String getRack(InetAddressAndPort endpoint)
+ {
+ return null;
+ }
+
+ public String getDatacenter(InetAddressAndPort endpoint)
+ {
+ byte[] address = endpoint.address.getAddress();
+ if (address[1] == 1)
+ return "datacenter1";
+ else
+ return "datacenter2";
+ }
+
+ public <C extends ReplicaCollection<? extends C>> C sortedByProximity(InetAddressAndPort address, C replicas)
+ {
+ return replicas;
+ }
+
+ public int compareEndpoints(InetAddressAndPort target, Replica a1, Replica a2)
+ {
+ return 0;
+ }
+
+ public void gossiperStarting()
+ {
+
+ }
+
+ public boolean isWorthMergingForRangeQuery(ReplicaCollection<?> merged, ReplicaCollection<?> l1, ReplicaCollection<?> l2)
+ {
+ return false;
+ }
+ });
+ DatabaseDescriptor.setBroadcastAddress(InetAddress.getByName("127.1.0.1"));
+ SchemaLoader.createKeyspace("Foo", KeyspaceParams.nts("datacenter1", 3, "datacenter2", 3), SchemaLoader.standardCFMD("Foo", "Bar"));
+ ks = Keyspace.open("Foo");
+ cfs = ks.getColumnFamilyStore("Bar");
+ targets = EndpointsForToken.of(DatabaseDescriptor.getPartitioner().getToken(ByteBufferUtil.bytes(0)),
+ full("127.1.0.255"), full("127.1.0.254"), full("127.1.0.253"),
+ full("127.2.0.255"), full("127.2.0.254"), full("127.2.0.253"));
+ pending = EndpointsForToken.empty(DatabaseDescriptor.getPartitioner().getToken(ByteBufferUtil.bytes(0)));
+ }
+
+ @Before
+ public void resetCounters()
+ {
+ ks.metric.writeFailedIdealCL.dec(ks.metric.writeFailedIdealCL.getCount());
+ }
+
+ /**
+ * Validate that a successful write at ideal CL logs latency information. Also validates
+ * DatacenterSyncWriteResponseHandler
+ * @throws Throwable
+ */
+ @Test
+ public void idealCLLatencyTracked() throws Throwable
+ {
+ long startingCount = ks.metric.idealCLWriteLatency.latency.getCount();
+ //Specify query start time in past to ensure minimum latency measurement
+ AbstractWriteResponseHandler awr = createWriteResponseHandler(ConsistencyLevel.LOCAL_QUORUM, ConsistencyLevel.EACH_QUORUM, System.nanoTime() - TimeUnit.DAYS.toNanos(1));
+
+ //dc1
+ awr.onResponse(createDummyMessage(0));
+ awr.onResponse(createDummyMessage(1));
+ //dc2
+ awr.onResponse(createDummyMessage(4));
+ awr.onResponse(createDummyMessage(5));
+
+ //Don't need the others
+ awr.expired();
+ awr.expired();
+
+ assertEquals(0, ks.metric.writeFailedIdealCL.getCount());
+ assertTrue( TimeUnit.DAYS.toMicros(1) < ks.metric.idealCLWriteLatency.totalLatency.getCount());
+ assertEquals(startingCount + 1, ks.metric.idealCLWriteLatency.latency.getCount());
+ }
+
+ /**
+ * Validate that WriteResponseHandler does the right thing on success.
+ * @throws Throwable
+ */
+ @Test
+ public void idealCLWriteResponeHandlerWorks() throws Throwable
+ {
+ long startingCount = ks.metric.idealCLWriteLatency.latency.getCount();
+ AbstractWriteResponseHandler awr = createWriteResponseHandler(ConsistencyLevel.LOCAL_QUORUM, ConsistencyLevel.ALL);
+
+ //dc1
+ awr.onResponse(createDummyMessage(0));
+ awr.onResponse(createDummyMessage(1));
+ awr.onResponse(createDummyMessage(2));
+ //dc2
+ awr.onResponse(createDummyMessage(3));
+ awr.onResponse(createDummyMessage(4));
+ awr.onResponse(createDummyMessage(5));
+
+ assertEquals(0, ks.metric.writeFailedIdealCL.getCount());
+ assertEquals(startingCount + 1, ks.metric.idealCLWriteLatency.latency.getCount());
+ }
+
+ /**
+ * Validate that DatacenterWriteResponseHandler does the right thing on success.
+ * @throws Throwable
+ */
+ @Test
+ public void idealCLDatacenterWriteResponeHandlerWorks() throws Throwable
+ {
+ long startingCount = ks.metric.idealCLWriteLatency.latency.getCount();
+ AbstractWriteResponseHandler awr = createWriteResponseHandler(ConsistencyLevel.ONE, ConsistencyLevel.LOCAL_QUORUM);
+
+ //dc1
+ awr.onResponse(createDummyMessage(0));
+ awr.onResponse(createDummyMessage(1));
+ awr.onResponse(createDummyMessage(2));
+ //dc2
+ awr.onResponse(createDummyMessage(3));
+ awr.onResponse(createDummyMessage(4));
+ awr.onResponse(createDummyMessage(5));
+
+ assertEquals(0, ks.metric.writeFailedIdealCL.getCount());
+ assertEquals(startingCount + 1, ks.metric.idealCLWriteLatency.latency.getCount());
+ }
+
+ /**
+ * Validate that failing to achieve ideal CL increments the failure counter
+ * @throws Throwable
+ */
+ @Test
+ public void failedIdealCLIncrementsStat() throws Throwable
+ {
+ ks.metric.idealCLWriteLatency.totalLatency.dec(ks.metric.idealCLWriteLatency.totalLatency.getCount());
+ AbstractWriteResponseHandler awr = createWriteResponseHandler(ConsistencyLevel.LOCAL_QUORUM, ConsistencyLevel.EACH_QUORUM);
+
+ //Succeed in local DC
+ awr.onResponse(createDummyMessage(0));
+ awr.onResponse(createDummyMessage(1));
+ awr.onResponse(createDummyMessage(2));
+
+ //Fail in remote DC
+ awr.expired();
+ awr.expired();
+ awr.expired();
+ assertEquals(1, ks.metric.writeFailedIdealCL.getCount());
+ assertEquals(0, ks.metric.idealCLWriteLatency.totalLatency.getCount());
+ }
+
+ /**
+ * Validate that failing to achieve ideal CL doesn't increase the failure counter when not meeting CL
+ * @throws Throwable
+ */
+ @Test
+ public void failedIdealCLDoesNotIncrementsStatOnQueryFailure() throws Throwable
+ {
+ AbstractWriteResponseHandler awr = createWriteResponseHandler(ConsistencyLevel.LOCAL_QUORUM, ConsistencyLevel.EACH_QUORUM);
+
+ long startingCount = ks.metric.writeFailedIdealCL.getCount();
+
+ // Failure in local DC
+ awr.onResponse(createDummyMessage(0));
+
+ awr.expired();
+ awr.expired();
+
+ //Fail in remote DC
+ awr.expired();
+ awr.expired();
+ awr.expired();
+
+ assertEquals(startingCount, ks.metric.writeFailedIdealCL.getCount());
+ }
+
+
+ private static AbstractWriteResponseHandler createWriteResponseHandler(ConsistencyLevel cl, ConsistencyLevel ideal)
+ {
+ return createWriteResponseHandler(cl, ideal, System.nanoTime());
+ }
+
+ private static AbstractWriteResponseHandler createWriteResponseHandler(ConsistencyLevel cl, ConsistencyLevel ideal, long queryStartTime)
+ {
+ return ks.getReplicationStrategy().getWriteResponseHandler(ReplicaPlans.forWrite(ks, cl, targets, pending, Predicates.alwaysTrue(), ReplicaPlans.writeAll),
+ null, WriteType.SIMPLE, queryStartTime, ideal);
+ }
+
+ private static Message createDummyMessage(int target)
+ {
+ return Message.builder(Verb.ECHO_REQ, noPayload)
+ .from(targets.get(target).endpoint())
+ .build();
+ }
+}
diff --git a/test/unit/org/apache/cassandra/service/WriteResponseHandlerTransientTest.java b/test/unit/org/apache/cassandra/service/WriteResponseHandlerTransientTest.java
new file mode 100644
index 0000000..15fbd27
--- /dev/null
+++ b/test/unit/org/apache/cassandra/service/WriteResponseHandlerTransientTest.java
@@ -0,0 +1,229 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service;
+
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.util.Set;
+import java.util.UUID;
+import java.util.function.Predicate;
+
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Sets;
+
+import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.locator.EndpointsForToken;
+import org.apache.cassandra.locator.ReplicaLayout;
+import org.apache.cassandra.locator.EndpointsForRange;
+import org.apache.cassandra.locator.ReplicaPlan;
+import org.apache.cassandra.locator.ReplicaPlans;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.ConsistencyLevel;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.exceptions.UnavailableException;
+import org.apache.cassandra.locator.IEndpointSnitch;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.Replica;
+import org.apache.cassandra.locator.ReplicaCollection;
+import org.apache.cassandra.locator.TokenMetadata;
+import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.utils.ByteBufferUtil;
+
+import static org.apache.cassandra.locator.Replica.fullReplica;
+import static org.apache.cassandra.locator.Replica.transientReplica;
+import static org.apache.cassandra.locator.ReplicaUtils.full;
+import static org.apache.cassandra.locator.ReplicaUtils.trans;
+
+public class WriteResponseHandlerTransientTest
+{
+ static Keyspace ks;
+ static ColumnFamilyStore cfs;
+
+ static final InetAddressAndPort EP1;
+ static final InetAddressAndPort EP2;
+ static final InetAddressAndPort EP3;
+ static final InetAddressAndPort EP4;
+ static final InetAddressAndPort EP5;
+ static final InetAddressAndPort EP6;
+
+ static final String DC1 = "datacenter1";
+ static final String DC2 = "datacenter2";
+ static Token dummy;
+ static
+ {
+ try
+ {
+ EP1 = InetAddressAndPort.getByName("127.1.0.1");
+ EP2 = InetAddressAndPort.getByName("127.1.0.2");
+ EP3 = InetAddressAndPort.getByName("127.1.0.3");
+ EP4 = InetAddressAndPort.getByName("127.2.0.4");
+ EP5 = InetAddressAndPort.getByName("127.2.0.5");
+ EP6 = InetAddressAndPort.getByName("127.2.0.6");
+ }
+ catch (UnknownHostException e)
+ {
+ throw new AssertionError(e);
+ }
+ }
+
+ @BeforeClass
+ public static void setupClass() throws Throwable
+ {
+ SchemaLoader.loadSchema();
+ DatabaseDescriptor.setTransientReplicationEnabledUnsafe(true);
+ DatabaseDescriptor.setPartitionerUnsafe(Murmur3Partitioner.instance);
+
+ // Register peers with expected DC for NetworkTopologyStrategy.
+ TokenMetadata metadata = StorageService.instance.getTokenMetadata();
+ metadata.clearUnsafe();
+ metadata.updateHostId(UUID.randomUUID(), InetAddressAndPort.getByName("127.1.0.1"));
+ metadata.updateHostId(UUID.randomUUID(), InetAddressAndPort.getByName("127.2.0.1"));
+
+ DatabaseDescriptor.setEndpointSnitch(new IEndpointSnitch()
+ {
+ public String getRack(InetAddressAndPort endpoint)
+ {
+ return null;
+ }
+
+ public String getDatacenter(InetAddressAndPort endpoint)
+ {
+ byte[] address = endpoint.address.getAddress();
+ if (address[1] == 1)
+ return DC1;
+ else
+ return DC2;
+ }
+
+ public <C extends ReplicaCollection<? extends C>> C sortedByProximity(InetAddressAndPort address, C unsortedAddress)
+ {
+ return unsortedAddress;
+ }
+
+ public int compareEndpoints(InetAddressAndPort target, Replica a1, Replica a2)
+ {
+ return 0;
+ }
+
+ public void gossiperStarting()
+ {
+
+ }
+
+ public boolean isWorthMergingForRangeQuery(ReplicaCollection<?> merged, ReplicaCollection<?> l1, ReplicaCollection<?> l2)
+ {
+ return false;
+ }
+ });
+
+ DatabaseDescriptor.setBroadcastAddress(InetAddress.getByName("127.1.0.1"));
+ SchemaLoader.createKeyspace("ks", KeyspaceParams.nts(DC1, "3/1", DC2, "3/1"), SchemaLoader.standardCFMD("ks", "tbl"));
+ ks = Keyspace.open("ks");
+ cfs = ks.getColumnFamilyStore("tbl");
+ dummy = DatabaseDescriptor.getPartitioner().getToken(ByteBufferUtil.bytes(0));
+ }
+
+ @Test
+ public void checkPendingReplicasAreNotFiltered()
+ {
+ EndpointsForToken natural = EndpointsForToken.of(dummy.getToken(), full(EP1), full(EP2), trans(EP3), full(EP5));
+ EndpointsForToken pending = EndpointsForToken.of(dummy.getToken(), full(EP4), trans(EP6));
+ ReplicaLayout.ForTokenWrite layout = new ReplicaLayout.ForTokenWrite(natural, pending);
+ ReplicaPlan.ForTokenWrite replicaPlan = ReplicaPlans.forWrite(ks, ConsistencyLevel.QUORUM, layout, layout, ReplicaPlans.writeAll);
+
+ Assert.assertTrue(Iterables.elementsEqual(EndpointsForRange.of(full(EP4), trans(EP6)),
+ replicaPlan.pending()));
+ }
+
+ private static ReplicaPlan.ForTokenWrite expected(EndpointsForToken natural, EndpointsForToken selected)
+ {
+ return new ReplicaPlan.ForTokenWrite(ks, ConsistencyLevel.QUORUM, EndpointsForToken.empty(dummy.getToken()), natural, natural, selected);
+ }
+
+ private static ReplicaPlan.ForTokenWrite getSpeculationContext(EndpointsForToken natural, Predicate<InetAddressAndPort> livePredicate)
+ {
+ ReplicaLayout.ForTokenWrite liveAndDown = new ReplicaLayout.ForTokenWrite(natural, EndpointsForToken.empty(dummy.getToken()));
+ ReplicaLayout.ForTokenWrite live = new ReplicaLayout.ForTokenWrite(natural.filter(r -> livePredicate.test(r.endpoint())), EndpointsForToken.empty(dummy.getToken()));
+ return ReplicaPlans.forWrite(ks, ConsistencyLevel.QUORUM, liveAndDown, live, ReplicaPlans.writeNormal);
+ }
+
+ private static void assertSpeculationReplicas(ReplicaPlan.ForTokenWrite expected, EndpointsForToken replicas, Predicate<InetAddressAndPort> livePredicate)
+ {
+ ReplicaPlan.ForTokenWrite actual = getSpeculationContext(replicas, livePredicate);
+ assertEquals(expected.pending(), actual.pending());
+ assertEquals(expected.live(), actual.live());
+ assertEquals(expected.contacts(), actual.contacts());
+ }
+
+ private static void assertEquals(ReplicaCollection<?> a, ReplicaCollection<?> b)
+ {
+ if (!Iterables.elementsEqual(a, b))
+ Assert.assertTrue(a + " vs " + b, false);
+ }
+
+ private static Predicate<InetAddressAndPort> dead(InetAddressAndPort... endpoints)
+ {
+ Set<InetAddressAndPort> deadSet = Sets.newHashSet(endpoints);
+ return ep -> !deadSet.contains(ep);
+ }
+
+ private static EndpointsForToken replicas(Replica... rr)
+ {
+ return EndpointsForToken.of(dummy.getToken(), rr);
+ }
+
+ @Test
+ public void checkSpeculationContext()
+ {
+ EndpointsForToken all = replicas(full(EP1), full(EP2), trans(EP3), full(EP4), full(EP5), trans(EP6));
+ // in happy path, transient replica should be classified as a backup
+ assertSpeculationReplicas(expected(all, replicas(full(EP1), full(EP2), full(EP4), full(EP5))),
+ all,
+ dead());
+
+ // full replicas must always be in the contact list, and will occur first
+ assertSpeculationReplicas(expected(replicas(full(EP1), trans(EP3), full(EP4), trans(EP6)), replicas(full(EP1), full(EP2), full(EP4), full(EP5), trans(EP3), trans(EP6))),
+ all,
+ dead(EP2, EP5));
+
+ // only one transient used as backup
+ assertSpeculationReplicas(expected(replicas(full(EP1), trans(EP3), full(EP4), full(EP5), trans(EP6)), replicas(full(EP1), full(EP2), full(EP4), full(EP5), trans(EP3))),
+ all,
+ dead(EP2));
+ }
+
+ @Test (expected = UnavailableException.class)
+ public void noFullReplicas()
+ {
+ getSpeculationContext(replicas(full(EP1), trans(EP2), trans(EP3)), dead(EP1));
+ }
+
+ @Test (expected = UnavailableException.class)
+ public void notEnoughTransientReplicas()
+ {
+ getSpeculationContext(replicas(full(EP1), trans(EP2), trans(EP3)), dead(EP2, EP3));
+ }
+}
diff --git a/test/unit/org/apache/cassandra/service/reads/AbstractReadResponseTest.java b/test/unit/org/apache/cassandra/service/reads/AbstractReadResponseTest.java
new file mode 100644
index 0000000..884baa1
--- /dev/null
+++ b/test/unit/org/apache/cassandra/service/reads/AbstractReadResponseTest.java
@@ -0,0 +1,345 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.reads;
+
+import java.net.UnknownHostException;
+import java.nio.ByteBuffer;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Ignore;
+
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.Util;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.ColumnIdentifier;
+import org.apache.cassandra.db.ClusteringBound;
+import org.apache.cassandra.db.ClusteringBoundary;
+import org.apache.cassandra.db.ClusteringPrefix;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.DeletionTime;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.RangeTombstone;
+import org.apache.cassandra.db.ReadCommand;
+import org.apache.cassandra.db.ReadResponse;
+import org.apache.cassandra.db.Slice;
+import org.apache.cassandra.db.marshal.AsciiType;
+import org.apache.cassandra.db.marshal.ByteType;
+import org.apache.cassandra.db.marshal.BytesType;
+import org.apache.cassandra.db.marshal.IntegerType;
+import org.apache.cassandra.db.marshal.MapType;
+import org.apache.cassandra.db.partitions.PartitionIterator;
+import org.apache.cassandra.db.partitions.PartitionUpdate;
+import org.apache.cassandra.db.partitions.SingletonUnfilteredPartitionIterator;
+import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
+import org.apache.cassandra.db.partitions.UnfilteredPartitionIterators;
+import org.apache.cassandra.db.rows.AbstractUnfilteredRowIterator;
+import org.apache.cassandra.db.rows.EncodingStats;
+import org.apache.cassandra.db.rows.RangeTombstoneBoundMarker;
+import org.apache.cassandra.db.rows.RangeTombstoneBoundaryMarker;
+import org.apache.cassandra.db.rows.RowIterator;
+import org.apache.cassandra.db.rows.Rows;
+import org.apache.cassandra.db.rows.Unfiltered;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.net.Message;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.FBUtilities;
+
+import static org.apache.cassandra.net.Verb.READ_REQ;
+
+/**
+ * Base class for testing various components which deal with read responses
+ */
+@Ignore
+public abstract class AbstractReadResponseTest
+{
+ public static final String KEYSPACE1 = "DataResolverTest";
+ public static final String KEYSPACE3 = "DataResolverTest3";
+ public static final String CF_STANDARD = "Standard1";
+ public static final String CF_COLLECTION = "Collection1";
+
+ public static Keyspace ks;
+ public static Keyspace ks3;
+ public static ColumnFamilyStore cfs;
+ public static ColumnFamilyStore cfs2;
+ public static ColumnFamilyStore cfs3;
+ public static TableMetadata cfm;
+ public static TableMetadata cfm2;
+ public static TableMetadata cfm3;
+ public static ColumnMetadata m;
+
+ public static DecoratedKey dk;
+ static int nowInSec;
+
+ static final InetAddressAndPort EP1;
+ static final InetAddressAndPort EP2;
+ static final InetAddressAndPort EP3;
+
+ static
+ {
+ try
+ {
+ EP1 = InetAddressAndPort.getByName("127.0.0.1");
+ EP2 = InetAddressAndPort.getByName("127.0.0.2");
+ EP3 = InetAddressAndPort.getByName("127.0.0.3");
+ }
+ catch (UnknownHostException e)
+ {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @BeforeClass
+ public static void setupClass() throws Throwable
+ {
+ DatabaseDescriptor.daemonInitialization();
+ DatabaseDescriptor.setPartitionerUnsafe(Murmur3Partitioner.instance);
+
+ TableMetadata.Builder builder1 =
+ TableMetadata.builder(KEYSPACE1, CF_STANDARD)
+ .addPartitionKeyColumn("key", BytesType.instance)
+ .addClusteringColumn("col1", AsciiType.instance)
+ .addRegularColumn("c1", AsciiType.instance)
+ .addRegularColumn("c2", AsciiType.instance)
+ .addRegularColumn("one", AsciiType.instance)
+ .addRegularColumn("two", AsciiType.instance);
+
+ TableMetadata.Builder builder3 =
+ TableMetadata.builder(KEYSPACE3, CF_STANDARD)
+ .addPartitionKeyColumn("key", BytesType.instance)
+ .addClusteringColumn("col1", AsciiType.instance)
+ .addRegularColumn("c1", AsciiType.instance)
+ .addRegularColumn("c2", AsciiType.instance)
+ .addRegularColumn("one", AsciiType.instance)
+ .addRegularColumn("two", AsciiType.instance);
+
+ TableMetadata.Builder builder2 =
+ TableMetadata.builder(KEYSPACE1, CF_COLLECTION)
+ .addPartitionKeyColumn("k", ByteType.instance)
+ .addRegularColumn("m", MapType.getInstance(IntegerType.instance, IntegerType.instance, true));
+
+ SchemaLoader.prepareServer();
+ SchemaLoader.createKeyspace(KEYSPACE1, KeyspaceParams.simple(2), builder1, builder2);
+ SchemaLoader.createKeyspace(KEYSPACE3, KeyspaceParams.simple(4), builder3);
+
+ ks = Keyspace.open(KEYSPACE1);
+ cfs = ks.getColumnFamilyStore(CF_STANDARD);
+ cfm = cfs.metadata();
+ cfs2 = ks.getColumnFamilyStore(CF_COLLECTION);
+ cfm2 = cfs2.metadata();
+ ks3 = Keyspace.open(KEYSPACE3);
+ cfs3 = ks3.getColumnFamilyStore(CF_STANDARD);
+ cfm3 = cfs3.metadata();
+ m = cfm2.getColumn(new ColumnIdentifier("m", false));
+ }
+
+ @Before
+ public void setUp() throws Exception
+ {
+ dk = Util.dk("key1");
+ nowInSec = FBUtilities.nowInSeconds();
+ }
+
+ static void assertPartitionsEqual(RowIterator l, RowIterator r)
+ {
+ try (RowIterator left = l; RowIterator right = r)
+ {
+ Assert.assertTrue(Util.sameContent(left, right));
+ }
+ }
+
+ static void assertPartitionsEqual(UnfilteredRowIterator left, UnfilteredRowIterator right)
+ {
+ Assert.assertTrue(Util.sameContent(left, right));
+ }
+
+ static void assertPartitionsEqual(UnfilteredPartitionIterator left, UnfilteredPartitionIterator right)
+ {
+ while (left.hasNext())
+ {
+ Assert.assertTrue(right.hasNext());
+ assertPartitionsEqual(left.next(), right.next());
+ }
+ Assert.assertFalse(right.hasNext());
+ }
+
+ static void assertPartitionsEqual(PartitionIterator l, PartitionIterator r)
+ {
+ try (PartitionIterator left = l; PartitionIterator right = r)
+ {
+ while (left.hasNext())
+ {
+ Assert.assertTrue(right.hasNext());
+ assertPartitionsEqual(left.next(), right.next());
+ }
+ Assert.assertFalse(right.hasNext());
+ }
+ }
+
+ static void consume(PartitionIterator i)
+ {
+ try (PartitionIterator iterator = i)
+ {
+ while (iterator.hasNext())
+ {
+ try (RowIterator rows = iterator.next())
+ {
+ while (rows.hasNext())
+ rows.next();
+ }
+ }
+ }
+ }
+
+ static PartitionIterator filter(UnfilteredPartitionIterator iter)
+ {
+ return UnfilteredPartitionIterators.filter(iter, nowInSec);
+ }
+
+ static DecoratedKey dk(String k)
+ {
+ return cfs.decorateKey(ByteBufferUtil.bytes(k));
+ }
+
+ static DecoratedKey dk(int k)
+ {
+ return dk(Integer.toString(k));
+ }
+
+
+ static Message<ReadResponse> response(ReadCommand command,
+ InetAddressAndPort from,
+ UnfilteredPartitionIterator data,
+ boolean isDigestResponse,
+ int fromVersion,
+ ByteBuffer repairedDataDigest,
+ boolean hasPendingRepair)
+ {
+ ReadResponse response = isDigestResponse
+ ? ReadResponse.createDigestResponse(data, command)
+ : ReadResponse.createRemoteDataResponse(data, repairedDataDigest, hasPendingRepair, command, fromVersion);
+ return Message.builder(READ_REQ, response)
+ .from(from)
+ .build();
+ }
+
+ static Message<ReadResponse> response(InetAddressAndPort from,
+ UnfilteredPartitionIterator partitionIterator,
+ ByteBuffer repairedDigest,
+ boolean hasPendingRepair,
+ ReadCommand cmd)
+ {
+ return response(cmd, from, partitionIterator, false, MessagingService.current_version, repairedDigest, hasPendingRepair);
+ }
+
+ static Message<ReadResponse> response(ReadCommand command, InetAddressAndPort from, UnfilteredPartitionIterator data, boolean isDigestResponse)
+ {
+ return response(command, from, data, false, MessagingService.current_version, ByteBufferUtil.EMPTY_BYTE_BUFFER, isDigestResponse);
+ }
+
+ static Message<ReadResponse> response(ReadCommand command, InetAddressAndPort from, UnfilteredPartitionIterator data)
+ {
+ return response(command, from, data, false, MessagingService.current_version, ByteBufferUtil.EMPTY_BYTE_BUFFER, false);
+ }
+
+ public RangeTombstone tombstone(Object start, Object end, long markedForDeleteAt, int localDeletionTime)
+ {
+ return tombstone(start, true, end, true, markedForDeleteAt, localDeletionTime);
+ }
+
+ public RangeTombstone tombstone(Object start, boolean inclusiveStart, Object end, boolean inclusiveEnd, long markedForDeleteAt, int localDeletionTime)
+ {
+ ClusteringBound startBound = rtBound(start, true, inclusiveStart);
+ ClusteringBound endBound = rtBound(end, false, inclusiveEnd);
+ return new RangeTombstone(Slice.make(startBound, endBound), new DeletionTime(markedForDeleteAt, localDeletionTime));
+ }
+
+ public ClusteringBound rtBound(Object value, boolean isStart, boolean inclusive)
+ {
+ ClusteringBound.Kind kind = isStart
+ ? (inclusive ? ClusteringPrefix.Kind.INCL_START_BOUND : ClusteringPrefix.Kind.EXCL_START_BOUND)
+ : (inclusive ? ClusteringPrefix.Kind.INCL_END_BOUND : ClusteringPrefix.Kind.EXCL_END_BOUND);
+
+ return ClusteringBound.create(kind, cfm.comparator.make(value).getRawValues());
+ }
+
+ public ClusteringBoundary rtBoundary(Object value, boolean inclusiveOnEnd)
+ {
+ ClusteringBound.Kind kind = inclusiveOnEnd
+ ? ClusteringPrefix.Kind.INCL_END_EXCL_START_BOUNDARY
+ : ClusteringPrefix.Kind.EXCL_END_INCL_START_BOUNDARY;
+ return ClusteringBoundary.create(kind, cfm.comparator.make(value).getRawValues());
+ }
+
+ public RangeTombstoneBoundMarker marker(Object value, boolean isStart, boolean inclusive, long markedForDeleteAt, int localDeletionTime)
+ {
+ return new RangeTombstoneBoundMarker(rtBound(value, isStart, inclusive), new DeletionTime(markedForDeleteAt, localDeletionTime));
+ }
+
+ public RangeTombstoneBoundaryMarker boundary(Object value, boolean inclusiveOnEnd, long markedForDeleteAt1, int localDeletionTime1, long markedForDeleteAt2, int localDeletionTime2)
+ {
+ return new RangeTombstoneBoundaryMarker(rtBoundary(value, inclusiveOnEnd),
+ new DeletionTime(markedForDeleteAt1, localDeletionTime1),
+ new DeletionTime(markedForDeleteAt2, localDeletionTime2));
+ }
+
+ public UnfilteredPartitionIterator fullPartitionDelete(TableMetadata table, DecoratedKey dk, long timestamp, int nowInSec)
+ {
+ return new SingletonUnfilteredPartitionIterator(PartitionUpdate.fullPartitionDelete(table, dk, timestamp, nowInSec).unfilteredIterator());
+ }
+
+ public UnfilteredPartitionIterator iter(PartitionUpdate update)
+ {
+ return new SingletonUnfilteredPartitionIterator(update.unfilteredIterator());
+ }
+
+ public UnfilteredPartitionIterator iter(DecoratedKey key, Unfiltered... unfiltereds)
+ {
+ SortedSet<Unfiltered> s = new TreeSet<>(cfm.comparator);
+ Collections.addAll(s, unfiltereds);
+ final Iterator<Unfiltered> iterator = s.iterator();
+
+ UnfilteredRowIterator rowIter = new AbstractUnfilteredRowIterator(cfm,
+ key,
+ DeletionTime.LIVE,
+ cfm.regularAndStaticColumns(),
+ Rows.EMPTY_STATIC_ROW,
+ false,
+ EncodingStats.NO_STATS)
+ {
+ protected Unfiltered computeNext()
+ {
+ return iterator.hasNext() ? iterator.next() : endOfData();
+ }
+ };
+ return new SingletonUnfilteredPartitionIterator(rowIter);
+ }
+}
diff --git a/test/unit/org/apache/cassandra/service/reads/DataResolverTest.java b/test/unit/org/apache/cassandra/service/reads/DataResolverTest.java
new file mode 100644
index 0000000..5d71f4d
--- /dev/null
+++ b/test/unit/org/apache/cassandra/service/reads/DataResolverTest.java
@@ -0,0 +1,1341 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.service.reads;
+
+import java.net.UnknownHostException;
+import java.nio.ByteBuffer;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.UUID;
+
+import com.google.common.collect.Iterators;
+import com.google.common.collect.Sets;
+
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.gms.Gossiper;
+import org.apache.cassandra.locator.ReplicaPlan;
+import org.junit.Assert;
+import org.junit.Test;
+
+import org.apache.cassandra.Util;
+import org.apache.cassandra.db.Clustering;
+import org.apache.cassandra.db.ClusteringBound;
+import org.apache.cassandra.db.ConsistencyLevel;
+import org.apache.cassandra.db.DeletionInfo;
+import org.apache.cassandra.db.DeletionTime;
+import org.apache.cassandra.db.EmptyIterators;
+import org.apache.cassandra.db.MutableDeletionInfo;
+import org.apache.cassandra.db.Mutation;
+import org.apache.cassandra.db.RangeTombstone;
+import org.apache.cassandra.db.ReadCommand;
+import org.apache.cassandra.db.RowUpdateBuilder;
+import org.apache.cassandra.db.Slice;
+import org.apache.cassandra.db.partitions.PartitionIterator;
+import org.apache.cassandra.db.partitions.PartitionUpdate;
+import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
+import org.apache.cassandra.db.rows.BTreeRow;
+import org.apache.cassandra.db.rows.BufferCell;
+import org.apache.cassandra.db.rows.Cell;
+import org.apache.cassandra.db.rows.CellPath;
+import org.apache.cassandra.db.rows.ComplexColumnData;
+import org.apache.cassandra.db.rows.RangeTombstoneBoundMarker;
+import org.apache.cassandra.db.rows.RangeTombstoneBoundaryMarker;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.db.rows.RowIterator;
+import org.apache.cassandra.locator.EndpointsForRange;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.Replica;
+import org.apache.cassandra.locator.ReplicaUtils;
+import org.apache.cassandra.net.*;
+import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.service.reads.repair.ReadRepair;
+import org.apache.cassandra.service.reads.repair.RepairedDataTracker;
+import org.apache.cassandra.service.reads.repair.RepairedDataVerifier;
+import org.apache.cassandra.service.reads.repair.TestableReadRepair;
+import org.apache.cassandra.utils.ByteBufferUtil;
+
+import static org.apache.cassandra.Util.assertClustering;
+import static org.apache.cassandra.Util.assertColumn;
+import static org.apache.cassandra.Util.assertColumns;
+import static org.apache.cassandra.db.ClusteringBound.Kind;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+public class DataResolverTest extends AbstractReadResponseTest
+{
+ private ReadCommand command;
+ private TestableReadRepair readRepair;
+ private Keyspace ks;
+ private ColumnFamilyStore cfs;
+
+ private EndpointsForRange makeReplicas(int num)
+ {
+ StorageService.instance.getTokenMetadata().clearUnsafe();
+
+ switch (num)
+ {
+ case 2:
+ ks = AbstractReadResponseTest.ks;
+ cfs = AbstractReadResponseTest.cfs;
+ break;
+ case 4:
+ ks = AbstractReadResponseTest.ks3;
+ cfs = AbstractReadResponseTest.cfs3;
+ break;
+ default:
+ throw new IllegalStateException("This test needs refactoring to cleanly support different replication factors");
+ }
+
+ command = Util.cmd(cfs, dk).withNowInSeconds(nowInSec).build();
+ command.trackRepairedStatus();
+ readRepair = new TestableReadRepair(command);
+ Token token = Murmur3Partitioner.instance.getMinimumToken();
+ EndpointsForRange.Builder replicas = EndpointsForRange.builder(ReplicaUtils.FULL_RANGE, num);
+ for (int i = 0; i < num; i++)
+ {
+ try
+ {
+ InetAddressAndPort endpoint = InetAddressAndPort.getByAddress(new byte[]{ 127, 0, 0, (byte) (i + 1) });
+ replicas.add(ReplicaUtils.full(endpoint));
+ StorageService.instance.getTokenMetadata().updateNormalToken(token = token.increaseSlightly(), endpoint);
+ Gossiper.instance.initializeNodeUnsafe(endpoint, UUID.randomUUID(), 1);
+ }
+ catch (UnknownHostException e)
+ {
+ throw new AssertionError(e);
+ }
+ }
+ return replicas.build();
+ }
+
+ @Test
+ public void testResolveNewerSingleRow()
+ {
+ EndpointsForRange replicas = makeReplicas(2);
+ DataResolver resolver = new DataResolver(command, plan(replicas, ConsistencyLevel.ALL), readRepair, System.nanoTime());
+ InetAddressAndPort peer1 = replicas.get(0).endpoint();
+ resolver.preprocess(response(command, peer1, iter(new RowUpdateBuilder(cfm, nowInSec, 0L, dk).clustering("1")
+ .add("c1", "v1")
+ .buildUpdate()), false));
+ InetAddressAndPort peer2 = replicas.get(1).endpoint();
+ resolver.preprocess(response(command, peer2, iter(new RowUpdateBuilder(cfm, nowInSec, 1L, dk).clustering("1")
+ .add("c1", "v2")
+ .buildUpdate()), false));
+
+ try(PartitionIterator data = resolver.resolve())
+ {
+ try (RowIterator rows = Iterators.getOnlyElement(data))
+ {
+ Row row = Iterators.getOnlyElement(rows);
+ assertColumns(row, "c1");
+ assertColumn(cfm, row, "c1", "v2", 1);
+ }
+ }
+
+ assertEquals(1, readRepair.sent.size());
+ // peer 1 just needs to repair with the row from peer 2
+ Mutation mutation = readRepair.getForEndpoint(peer1);
+ assertRepairMetadata(mutation);
+ assertRepairContainsNoDeletions(mutation);
+ assertRepairContainsColumn(mutation, "1", "c1", "v2", 1);
+ }
+
+ @Test
+ public void testResolveDisjointSingleRow()
+ {
+ EndpointsForRange replicas = makeReplicas(2);
+ DataResolver resolver = new DataResolver(command, plan(replicas, ConsistencyLevel.ALL), readRepair, System.nanoTime());
+ InetAddressAndPort peer1 = replicas.get(0).endpoint();
+ resolver.preprocess(response(command, peer1, iter(new RowUpdateBuilder(cfm, nowInSec, 0L, dk).clustering("1")
+ .add("c1", "v1")
+ .buildUpdate())));
+
+ InetAddressAndPort peer2 = replicas.get(1).endpoint();
+ resolver.preprocess(response(command, peer2, iter(new RowUpdateBuilder(cfm, nowInSec, 1L, dk).clustering("1")
+ .add("c2", "v2")
+ .buildUpdate())));
+
+ try(PartitionIterator data = resolver.resolve())
+ {
+ try (RowIterator rows = Iterators.getOnlyElement(data))
+ {
+ Row row = Iterators.getOnlyElement(rows);
+ assertColumns(row, "c1", "c2");
+ assertColumn(cfm, row, "c1", "v1", 0);
+ assertColumn(cfm, row, "c2", "v2", 1);
+ }
+ }
+
+ assertEquals(2, readRepair.sent.size());
+ // each peer needs to repair with each other's column
+ Mutation mutation = readRepair.getForEndpoint(peer1);
+ assertRepairMetadata(mutation);
+ assertRepairContainsColumn(mutation, "1", "c2", "v2", 1);
+
+ mutation = readRepair.getForEndpoint(peer2);
+ assertRepairMetadata(mutation);
+ assertRepairContainsColumn(mutation, "1", "c1", "v1", 0);
+ }
+
+ @Test
+ public void testResolveDisjointMultipleRows() throws UnknownHostException
+ {
+ EndpointsForRange replicas = makeReplicas(2);
+ DataResolver resolver = new DataResolver(command, plan(replicas, ConsistencyLevel.ALL), readRepair, System.nanoTime());
+ InetAddressAndPort peer1 = replicas.get(0).endpoint();
+ resolver.preprocess(response(command, peer1, iter(new RowUpdateBuilder(cfm, nowInSec, 0L, dk).clustering("1")
+ .add("c1", "v1")
+ .buildUpdate())));
+ InetAddressAndPort peer2 = replicas.get(1).endpoint();
+ resolver.preprocess(response(command, peer2, iter(new RowUpdateBuilder(cfm, nowInSec, 1L, dk).clustering("2")
+ .add("c2", "v2")
+ .buildUpdate())));
+
+ try (PartitionIterator data = resolver.resolve())
+ {
+ try (RowIterator rows = data.next())
+ {
+ // We expect the resolved superset to contain both rows
+ Row row = rows.next();
+ assertClustering(cfm, row, "1");
+ assertColumns(row, "c1");
+ assertColumn(cfm, row, "c1", "v1", 0);
+
+ row = rows.next();
+ assertClustering(cfm, row, "2");
+ assertColumns(row, "c2");
+ assertColumn(cfm, row, "c2", "v2", 1);
+
+ assertFalse(rows.hasNext());
+ assertFalse(data.hasNext());
+ }
+ }
+
+ assertEquals(2, readRepair.sent.size());
+ // each peer needs to repair the row from the other
+ Mutation mutation = readRepair.getForEndpoint(peer1);
+ assertRepairMetadata(mutation);
+ assertRepairContainsNoDeletions(mutation);
+ assertRepairContainsColumn(mutation, "2", "c2", "v2", 1);
+
+ mutation = readRepair.getForEndpoint(peer2);
+ assertRepairMetadata(mutation);
+ assertRepairContainsNoDeletions(mutation);
+ assertRepairContainsColumn(mutation, "1", "c1", "v1", 0);
+ }
+
+ @Test
+ public void testResolveDisjointMultipleRowsWithRangeTombstones()
+ {
+ EndpointsForRange replicas = makeReplicas(4);
+ DataResolver resolver = new DataResolver(command, plan(replicas, ConsistencyLevel.ALL), readRepair, System.nanoTime());
+
+ RangeTombstone tombstone1 = tombstone("1", "11", 1, nowInSec);
+ RangeTombstone tombstone2 = tombstone("3", "31", 1, nowInSec);
+ PartitionUpdate update = new RowUpdateBuilder(cfm3, nowInSec, 1L, dk).addRangeTombstone(tombstone1)
+ .addRangeTombstone(tombstone2)
+ .buildUpdate();
+
+ InetAddressAndPort peer1 = replicas.get(0).endpoint();
+ UnfilteredPartitionIterator iter1 = iter(new RowUpdateBuilder(cfm3, nowInSec, 1L, dk).addRangeTombstone(tombstone1)
+ .addRangeTombstone(tombstone2)
+ .buildUpdate());
+ resolver.preprocess(response(command, peer1, iter1));
+ // not covered by any range tombstone
+ InetAddressAndPort peer2 = replicas.get(1).endpoint();
+ UnfilteredPartitionIterator iter2 = iter(new RowUpdateBuilder(cfm3, nowInSec, 0L, dk).clustering("0")
+ .add("c1", "v0")
+ .buildUpdate());
+ resolver.preprocess(response(command, peer2, iter2));
+ // covered by a range tombstone
+ InetAddressAndPort peer3 = replicas.get(2).endpoint();
+ UnfilteredPartitionIterator iter3 = iter(new RowUpdateBuilder(cfm3, nowInSec, 0L, dk).clustering("10")
+ .add("c2", "v1")
+ .buildUpdate());
+ resolver.preprocess(response(command, peer3, iter3));
+ // range covered by rt, but newer
+ InetAddressAndPort peer4 = replicas.get(3).endpoint();
+ UnfilteredPartitionIterator iter4 = iter(new RowUpdateBuilder(cfm3, nowInSec, 2L, dk).clustering("3")
+ .add("one", "A")
+ .buildUpdate());
+ resolver.preprocess(response(command, peer4, iter4));
+ try (PartitionIterator data = resolver.resolve())
+ {
+ try (RowIterator rows = data.next())
+ {
+ Row row = rows.next();
+ assertClustering(cfm, row, "0");
+ assertColumns(row, "c1");
+ assertColumn(cfm, row, "c1", "v0", 0);
+
+ row = rows.next();
+ assertClustering(cfm, row, "3");
+ assertColumns(row, "one");
+ assertColumn(cfm, row, "one", "A", 2);
+
+ assertFalse(rows.hasNext());
+ }
+ }
+
+ assertEquals(4, readRepair.sent.size());
+ // peer1 needs the rows from peers 2 and 4
+ Mutation mutation = readRepair.getForEndpoint(peer1);
+ assertRepairMetadata(mutation);
+ assertRepairContainsNoDeletions(mutation);
+ assertRepairContainsColumn(mutation, "0", "c1", "v0", 0);
+ assertRepairContainsColumn(mutation, "3", "one", "A", 2);
+
+ // peer2 needs to get the row from peer4 and the RTs
+ mutation = readRepair.getForEndpoint(peer2);
+ assertRepairMetadata(mutation);
+ assertRepairContainsDeletions(mutation, null, tombstone1, tombstone2);
+ assertRepairContainsColumn(mutation, "3", "one", "A", 2);
+
+ // peer 3 needs both rows and the RTs
+ mutation = readRepair.getForEndpoint(peer3);
+ assertRepairMetadata(mutation);
+ assertRepairContainsDeletions(mutation, null, tombstone1, tombstone2);
+ assertRepairContainsColumn(mutation, "0", "c1", "v0", 0);
+ assertRepairContainsColumn(mutation, "3", "one", "A", 2);
+
+ // peer4 needs the row from peer2 and the RTs
+ mutation = readRepair.getForEndpoint(peer4);
+ assertRepairMetadata(mutation);
+ assertRepairContainsDeletions(mutation, null, tombstone1, tombstone2);
+ assertRepairContainsColumn(mutation, "0", "c1", "v0", 0);
+ }
+
+ @Test
+ public void testResolveWithOneEmpty()
+ {
+ EndpointsForRange replicas = makeReplicas(2);
+ DataResolver resolver = new DataResolver(command, plan(replicas, ConsistencyLevel.ALL), readRepair, System.nanoTime());
+ InetAddressAndPort peer1 = replicas.get(0).endpoint();
+ resolver.preprocess(response(command, peer1, iter(new RowUpdateBuilder(cfm, nowInSec, 1L, dk).clustering("1")
+ .add("c2", "v2")
+ .buildUpdate())));
+ InetAddressAndPort peer2 = replicas.get(1).endpoint();
+ resolver.preprocess(response(command, peer2, EmptyIterators.unfilteredPartition(cfm)));
+
+ try(PartitionIterator data = resolver.resolve())
+ {
+ try (RowIterator rows = Iterators.getOnlyElement(data))
+ {
+ Row row = Iterators.getOnlyElement(rows);
+ assertColumns(row, "c2");
+ assertColumn(cfm, row, "c2", "v2", 1);
+ }
+ }
+
+ assertEquals(1, readRepair.sent.size());
+ // peer 2 needs the row from peer 1
+ Mutation mutation = readRepair.getForEndpoint(peer2);
+ assertRepairMetadata(mutation);
+ assertRepairContainsNoDeletions(mutation);
+ assertRepairContainsColumn(mutation, "1", "c2", "v2", 1);
+ }
+
+ @Test
+ public void testResolveWithBothEmpty()
+ {
+ EndpointsForRange replicas = makeReplicas(2);
+ TestableReadRepair readRepair = new TestableReadRepair(command);
+ DataResolver resolver = new DataResolver(command, plan(replicas, ConsistencyLevel.ALL), readRepair, System.nanoTime());
+ resolver.preprocess(response(command, replicas.get(0).endpoint(), EmptyIterators.unfilteredPartition(cfm)));
+ resolver.preprocess(response(command, replicas.get(1).endpoint(), EmptyIterators.unfilteredPartition(cfm)));
+
+ try(PartitionIterator data = resolver.resolve())
+ {
+ assertFalse(data.hasNext());
+ }
+
+ assertTrue(readRepair.sent.isEmpty());
+ }
+
+ @Test
+ public void testResolveDeleted()
+ {
+ EndpointsForRange replicas = makeReplicas(2);
+ DataResolver resolver = new DataResolver(command, plan(replicas, ConsistencyLevel.ALL), readRepair, System.nanoTime());
+ // one response with columns timestamped before a delete in another response
+ InetAddressAndPort peer1 = replicas.get(0).endpoint();
+ resolver.preprocess(response(command, peer1, iter(new RowUpdateBuilder(cfm, nowInSec, 0L, dk).clustering("1")
+ .add("one", "A")
+ .buildUpdate())));
+ InetAddressAndPort peer2 = replicas.get(1).endpoint();
+ resolver.preprocess(response(command, peer2, fullPartitionDelete(cfm, dk, 1, nowInSec)));
+
+ try (PartitionIterator data = resolver.resolve())
+ {
+ assertFalse(data.hasNext());
+ }
+
+ // peer1 should get the deletion from peer2
+ assertEquals(1, readRepair.sent.size());
+ Mutation mutation = readRepair.getForEndpoint(peer1);
+ assertRepairMetadata(mutation);
+ assertRepairContainsDeletions(mutation, new DeletionTime(1, nowInSec));
+ assertRepairContainsNoColumns(mutation);
+ }
+
+ @Test
+ public void testResolveMultipleDeleted()
+ {
+ EndpointsForRange replicas = makeReplicas(4);
+ DataResolver resolver = new DataResolver(command, plan(replicas, ConsistencyLevel.ALL), readRepair, System.nanoTime());
+ // deletes and columns with interleaved timestamp, with out of order return sequence
+ InetAddressAndPort peer1 = replicas.get(0).endpoint();
+ resolver.preprocess(response(command, peer1, fullPartitionDelete(cfm, dk, 0, nowInSec)));
+ // these columns created after the previous deletion
+ InetAddressAndPort peer2 = replicas.get(1).endpoint();
+ resolver.preprocess(response(command, peer2, iter(new RowUpdateBuilder(cfm, nowInSec, 1L, dk).clustering("1")
+ .add("one", "A")
+ .add("two", "A")
+ .buildUpdate())));
+ //this column created after the next delete
+ InetAddressAndPort peer3 = replicas.get(2).endpoint();
+ resolver.preprocess(response(command, peer3, iter(new RowUpdateBuilder(cfm, nowInSec, 3L, dk).clustering("1")
+ .add("two", "B")
+ .buildUpdate())));
+ InetAddressAndPort peer4 = replicas.get(3).endpoint();
+ resolver.preprocess(response(command, peer4, fullPartitionDelete(cfm, dk, 2, nowInSec)));
+
+ try(PartitionIterator data = resolver.resolve())
+ {
+ try (RowIterator rows = Iterators.getOnlyElement(data))
+ {
+ Row row = Iterators.getOnlyElement(rows);
+ assertColumns(row, "two");
+ assertColumn(cfm, row, "two", "B", 3);
+ }
+ }
+
+ // peer 1 needs to get the partition delete from peer 4 and the row from peer 3
+ assertEquals(4, readRepair.sent.size());
+ Mutation mutation = readRepair.getForEndpoint(peer1);
+ assertRepairMetadata(mutation);
+ assertRepairContainsDeletions(mutation, new DeletionTime(2, nowInSec));
+ assertRepairContainsColumn(mutation, "1", "two", "B", 3);
+
+ // peer 2 needs the deletion from peer 4 and the row from peer 3
+ mutation = readRepair.getForEndpoint(peer2);
+ assertRepairMetadata(mutation);
+ assertRepairContainsDeletions(mutation, new DeletionTime(2, nowInSec));
+ assertRepairContainsColumn(mutation, "1", "two", "B", 3);
+
+ // peer 3 needs just the deletion from peer 4
+ mutation = readRepair.getForEndpoint(peer3);
+ assertRepairMetadata(mutation);
+ assertRepairContainsDeletions(mutation, new DeletionTime(2, nowInSec));
+ assertRepairContainsNoColumns(mutation);
+
+ // peer 4 needs just the row from peer 3
+ mutation = readRepair.getForEndpoint(peer4);
+ assertRepairMetadata(mutation);
+ assertRepairContainsNoDeletions(mutation);
+ assertRepairContainsColumn(mutation, "1", "two", "B", 3);
+ }
+
+ @Test
+ public void testResolveRangeTombstonesOnBoundaryRightWins() throws UnknownHostException
+ {
+ resolveRangeTombstonesOnBoundary(1, 2);
+ }
+
+ @Test
+ public void testResolveRangeTombstonesOnBoundaryLeftWins() throws UnknownHostException
+ {
+ resolveRangeTombstonesOnBoundary(2, 1);
+ }
+
+ @Test
+ public void testResolveRangeTombstonesOnBoundarySameTimestamp() throws UnknownHostException
+ {
+ resolveRangeTombstonesOnBoundary(1, 1);
+ }
+
+ /*
+ * We want responses to merge on tombstone boundary. So we'll merge 2 "streams":
+ * 1: [1, 2)(3, 4](5, 6] 2
+ * 2: [2, 3][4, 5) 1
+ * which tests all combination of open/close boundaries (open/close, close/open, open/open, close/close).
+ *
+ * Note that, because DataResolver returns a "filtered" iterator, it should resolve into an empty iterator.
+ * However, what should be sent to each source depends on the exact on the timestamps of each tombstones and we
+ * test a few combination.
+ */
+ private void resolveRangeTombstonesOnBoundary(long timestamp1, long timestamp2)
+ {
+ EndpointsForRange replicas = makeReplicas(2);
+ DataResolver resolver = new DataResolver(command, plan(replicas, ConsistencyLevel.ALL), readRepair, System.nanoTime());
+ InetAddressAndPort peer1 = replicas.get(0).endpoint();
+ InetAddressAndPort peer2 = replicas.get(1).endpoint();
+
+ // 1st "stream"
+ RangeTombstone one_two = tombstone("1", true , "2", false, timestamp1, nowInSec);
+ RangeTombstone three_four = tombstone("3", false, "4", true , timestamp1, nowInSec);
+ RangeTombstone five_six = tombstone("5", false, "6", true , timestamp1, nowInSec);
+ UnfilteredPartitionIterator iter1 = iter(new RowUpdateBuilder(cfm, nowInSec, 1L, dk).addRangeTombstone(one_two)
+ .addRangeTombstone(three_four)
+ .addRangeTombstone(five_six)
+ .buildUpdate());
+
+ // 2nd "stream"
+ RangeTombstone two_three = tombstone("2", true, "3", true , timestamp2, nowInSec);
+ RangeTombstone four_five = tombstone("4", true, "5", false, timestamp2, nowInSec);
+ UnfilteredPartitionIterator iter2 = iter(new RowUpdateBuilder(cfm, nowInSec, 1L, dk).addRangeTombstone(two_three)
+ .addRangeTombstone(four_five)
+ .buildUpdate());
+
+ resolver.preprocess(response(command, peer1, iter1));
+ resolver.preprocess(response(command, peer2, iter2));
+
+ // No results, we've only reconciled tombstones.
+ try (PartitionIterator data = resolver.resolve())
+ {
+ assertFalse(data.hasNext());
+ }
+
+ assertEquals(2, readRepair.sent.size());
+
+ Mutation msg1 = readRepair.getForEndpoint(peer1);
+ assertRepairMetadata(msg1);
+ assertRepairContainsNoColumns(msg1);
+
+ Mutation msg2 = readRepair.getForEndpoint(peer2);
+ assertRepairMetadata(msg2);
+ assertRepairContainsNoColumns(msg2);
+
+ // Both streams are mostly complementary, so they will roughly get the ranges of the other stream. One subtlety is
+ // around the value "4" however, as it's included by both stream.
+ // So for a given stream, unless the other stream has a strictly higher timestamp, the value 4 will be excluded
+ // from whatever range it receives as repair since the stream already covers it.
+
+ // Message to peer1 contains peer2 ranges
+ assertRepairContainsDeletions(msg1, null, two_three, withExclusiveStartIf(four_five, timestamp1 >= timestamp2));
+
+ // Message to peer2 contains peer1 ranges
+ assertRepairContainsDeletions(msg2, null, one_two, withExclusiveEndIf(three_four, timestamp2 >= timestamp1), five_six);
+ }
+
+ /**
+ * Test cases where a boundary of a source is covered by another source deletion and timestamp on one or both side
+ * of the boundary are equal to the "merged" deletion.
+ * This is a test for CASSANDRA-13237 to make sure we handle this case properly.
+ */
+ @Test
+ public void testRepairRangeTombstoneBoundary() throws UnknownHostException
+ {
+ testRepairRangeTombstoneBoundary(1, 0, 1);
+ readRepair.sent.clear();
+ testRepairRangeTombstoneBoundary(1, 1, 0);
+ readRepair.sent.clear();
+ testRepairRangeTombstoneBoundary(1, 1, 1);
+ }
+
+ /**
+ * Test for CASSANDRA-13237, checking we don't fail (and handle correctly) the case where a RT boundary has the
+ * same deletion on both side (while is useless but could be created by legacy code pre-CASSANDRA-13237 and could
+ * thus still be sent).
+ */
+ private void testRepairRangeTombstoneBoundary(int timestamp1, int timestamp2, int timestamp3) throws UnknownHostException
+ {
+ EndpointsForRange replicas = makeReplicas(2);
+ DataResolver resolver = new DataResolver(command, plan(replicas, ConsistencyLevel.ALL), readRepair, System.nanoTime());
+ InetAddressAndPort peer1 = replicas.get(0).endpoint();
+ InetAddressAndPort peer2 = replicas.get(1).endpoint();
+
+ // 1st "stream"
+ RangeTombstone one_nine = tombstone("0", true , "9", true, timestamp1, nowInSec);
+ UnfilteredPartitionIterator iter1 = iter(new RowUpdateBuilder(cfm, nowInSec, 1L, dk)
+ .addRangeTombstone(one_nine)
+ .buildUpdate());
+
+ // 2nd "stream" (build more manually to ensure we have the boundary we want)
+ RangeTombstoneBoundMarker open_one = marker("0", true, true, timestamp2, nowInSec);
+ RangeTombstoneBoundaryMarker boundary_five = boundary("5", false, timestamp2, nowInSec, timestamp3, nowInSec);
+ RangeTombstoneBoundMarker close_nine = marker("9", false, true, timestamp3, nowInSec);
+ UnfilteredPartitionIterator iter2 = iter(dk, open_one, boundary_five, close_nine);
+
+ resolver.preprocess(response(command, peer1, iter1));
+ resolver.preprocess(response(command, peer2, iter2));
+
+ boolean shouldHaveRepair = timestamp1 != timestamp2 || timestamp1 != timestamp3;
+
+ // No results, we've only reconciled tombstones.
+ try (PartitionIterator data = resolver.resolve())
+ {
+ assertFalse(data.hasNext());
+ }
+
+ assertEquals(shouldHaveRepair? 1 : 0, readRepair.sent.size());
+
+ if (!shouldHaveRepair)
+ return;
+
+ Mutation mutation = readRepair.getForEndpoint(peer2);
+ assertRepairMetadata(mutation);
+ assertRepairContainsNoColumns(mutation);
+
+ RangeTombstone expected = timestamp1 != timestamp2
+ // We've repaired the 1st part
+ ? tombstone("0", true, "5", false, timestamp1, nowInSec)
+ // We've repaired the 2nd part
+ : tombstone("5", true, "9", true, timestamp1, nowInSec);
+ assertRepairContainsDeletions(mutation, null, expected);
+ }
+
+ /**
+ * Test for CASSANDRA-13719: tests that having a partition deletion shadow a range tombstone on another source
+ * doesn't trigger an assertion error.
+ */
+ @Test
+ public void testRepairRangeTombstoneWithPartitionDeletion()
+ {
+ EndpointsForRange replicas = makeReplicas(2);
+
+ DataResolver resolver = new DataResolver(command, plan(replicas, ConsistencyLevel.ALL), readRepair, System.nanoTime());
+ InetAddressAndPort peer1 = replicas.get(0).endpoint();
+ InetAddressAndPort peer2 = replicas.get(1).endpoint();
+
+ // 1st "stream": just a partition deletion
+ UnfilteredPartitionIterator iter1 = iter(PartitionUpdate.fullPartitionDelete(cfm, dk, 10, nowInSec));
+
+ // 2nd "stream": a range tombstone that is covered by the 1st stream
+ RangeTombstone rt = tombstone("0", true , "10", true, 5, nowInSec);
+ UnfilteredPartitionIterator iter2 = iter(new RowUpdateBuilder(cfm, nowInSec, 1L, dk)
+ .addRangeTombstone(rt)
+ .buildUpdate());
+
+ resolver.preprocess(response(command, peer1, iter1));
+ resolver.preprocess(response(command, peer2, iter2));
+
+ // No results, we've only reconciled tombstones.
+ try (PartitionIterator data = resolver.resolve())
+ {
+ assertFalse(data.hasNext());
+ // 2nd stream should get repaired
+ }
+
+ assertEquals(1, readRepair.sent.size());
+
+ Mutation mutation = readRepair.getForEndpoint(peer2);
+ assertRepairMetadata(mutation);
+ assertRepairContainsNoColumns(mutation);
+
+ assertRepairContainsDeletions(mutation, new DeletionTime(10, nowInSec));
+ }
+
+ /**
+ * Additional test for CASSANDRA-13719: tests the case where a partition deletion doesn't shadow a range tombstone.
+ */
+ @Test
+ public void testRepairRangeTombstoneWithPartitionDeletion2()
+ {
+ EndpointsForRange replicas = makeReplicas(2);
+ DataResolver resolver = new DataResolver(command, plan(replicas, ConsistencyLevel.ALL), readRepair, System.nanoTime());
+ InetAddressAndPort peer1 = replicas.get(0).endpoint();
+ InetAddressAndPort peer2 = replicas.get(1).endpoint();
+
+ // 1st "stream": a partition deletion and a range tombstone
+ RangeTombstone rt1 = tombstone("0", true , "9", true, 11, nowInSec);
+ PartitionUpdate upd1 = new RowUpdateBuilder(cfm, nowInSec, 1L, dk)
+ .addRangeTombstone(rt1)
+ .buildUpdate();
+ ((MutableDeletionInfo)upd1.deletionInfo()).add(new DeletionTime(10, nowInSec));
+ UnfilteredPartitionIterator iter1 = iter(upd1);
+
+ // 2nd "stream": a range tombstone that is covered by the other stream rt
+ RangeTombstone rt2 = tombstone("2", true , "3", true, 11, nowInSec);
+ RangeTombstone rt3 = tombstone("4", true , "5", true, 10, nowInSec);
+ UnfilteredPartitionIterator iter2 = iter(new RowUpdateBuilder(cfm, nowInSec, 1L, dk)
+ .addRangeTombstone(rt2)
+ .addRangeTombstone(rt3)
+ .buildUpdate());
+
+ resolver.preprocess(response(command, peer1, iter1));
+ resolver.preprocess(response(command, peer2, iter2));
+
+ // No results, we've only reconciled tombstones.
+ try (PartitionIterator data = resolver.resolve())
+ {
+ assertFalse(data.hasNext());
+ // 2nd stream should get repaired
+ }
+
+ assertEquals(1, readRepair.sent.size());
+
+ Mutation mutation = readRepair.getForEndpoint(peer2);
+ assertRepairMetadata(mutation);
+ assertRepairContainsNoColumns(mutation);
+
+ // 2nd stream should get both the partition deletion, as well as the part of the 1st stream RT that it misses
+ assertRepairContainsDeletions(mutation, new DeletionTime(10, nowInSec),
+ tombstone("0", true, "2", false, 11, nowInSec),
+ tombstone("3", false, "9", true, 11, nowInSec));
+ }
+
+ // Forces the start to be exclusive if the condition holds
+ private static RangeTombstone withExclusiveStartIf(RangeTombstone rt, boolean condition)
+ {
+ if (!condition)
+ return rt;
+
+ Slice slice = rt.deletedSlice();
+ ClusteringBound newStart = ClusteringBound.create(Kind.EXCL_START_BOUND, slice.start().getRawValues());
+ return condition
+ ? new RangeTombstone(Slice.make(newStart, slice.end()), rt.deletionTime())
+ : rt;
+ }
+
+ // Forces the end to be exclusive if the condition holds
+ private static RangeTombstone withExclusiveEndIf(RangeTombstone rt, boolean condition)
+ {
+ if (!condition)
+ return rt;
+
+ Slice slice = rt.deletedSlice();
+ ClusteringBound newEnd = ClusteringBound.create(Kind.EXCL_END_BOUND, slice.end().getRawValues());
+ return condition
+ ? new RangeTombstone(Slice.make(slice.start(), newEnd), rt.deletionTime())
+ : rt;
+ }
+
+ private static ByteBuffer bb(int b)
+ {
+ return ByteBufferUtil.bytes(b);
+ }
+
+ private Cell mapCell(int k, int v, long ts)
+ {
+ return BufferCell.live(m, ts, bb(v), CellPath.create(bb(k)));
+ }
+
+ @Test
+ public void testResolveComplexDelete()
+ {
+ EndpointsForRange replicas = makeReplicas(2);
+ ReadCommand cmd = Util.cmd(cfs2, dk).withNowInSeconds(nowInSec).build();
+ TestableReadRepair readRepair = new TestableReadRepair(cmd);
+ DataResolver resolver = new DataResolver(cmd, plan(replicas, ConsistencyLevel.ALL), readRepair, System.nanoTime());
+
+ long[] ts = {100, 200};
+
+ Row.Builder builder = BTreeRow.unsortedBuilder();
+ builder.newRow(Clustering.EMPTY);
+ builder.addComplexDeletion(m, new DeletionTime(ts[0] - 1, nowInSec));
+ builder.addCell(mapCell(0, 0, ts[0]));
+
+ InetAddressAndPort peer1 = replicas.get(0).endpoint();
+ resolver.preprocess(response(cmd, peer1, iter(PartitionUpdate.singleRowUpdate(cfm2, dk, builder.build()))));
+
+ builder.newRow(Clustering.EMPTY);
+ DeletionTime expectedCmplxDelete = new DeletionTime(ts[1] - 1, nowInSec);
+ builder.addComplexDeletion(m, expectedCmplxDelete);
+ Cell expectedCell = mapCell(1, 1, ts[1]);
+ builder.addCell(expectedCell);
+
+ InetAddressAndPort peer2 = replicas.get(1).endpoint();
+ resolver.preprocess(response(cmd, peer2, iter(PartitionUpdate.singleRowUpdate(cfm2, dk, builder.build()))));
+
+ try(PartitionIterator data = resolver.resolve())
+ {
+ try (RowIterator rows = Iterators.getOnlyElement(data))
+ {
+ Row row = Iterators.getOnlyElement(rows);
+ assertColumns(row, "m");
+ Assert.assertNull(row.getCell(m, CellPath.create(bb(0))));
+ Assert.assertNotNull(row.getCell(m, CellPath.create(bb(1))));
+ }
+ }
+
+ Mutation mutation = readRepair.getForEndpoint(peer1);
+ Iterator<Row> rowIter = mutation.getPartitionUpdate(cfm2).iterator();
+ assertTrue(rowIter.hasNext());
+ Row row = rowIter.next();
+ assertFalse(rowIter.hasNext());
+
+ ComplexColumnData cd = row.getComplexColumnData(m);
+
+ assertEquals(Collections.singleton(expectedCell), Sets.newHashSet(cd));
+ assertEquals(expectedCmplxDelete, cd.complexDeletion());
+
+ Assert.assertNull(readRepair.sent.get(peer2));
+ }
+
+ @Test
+ public void testResolveDeletedCollection()
+ {
+ EndpointsForRange replicas = makeReplicas(2);
+ ReadCommand cmd = Util.cmd(cfs2, dk).withNowInSeconds(nowInSec).build();
+ TestableReadRepair readRepair = new TestableReadRepair(cmd);
+ DataResolver resolver = new DataResolver(cmd, plan(replicas, ConsistencyLevel.ALL), readRepair, System.nanoTime());
+
+ long[] ts = {100, 200};
+
+ Row.Builder builder = BTreeRow.unsortedBuilder();
+ builder.newRow(Clustering.EMPTY);
+ builder.addComplexDeletion(m, new DeletionTime(ts[0] - 1, nowInSec));
+ builder.addCell(mapCell(0, 0, ts[0]));
+
+ InetAddressAndPort peer1 = replicas.get(0).endpoint();
+ resolver.preprocess(response(cmd, peer1, iter(PartitionUpdate.singleRowUpdate(cfm2, dk, builder.build()))));
+
+ builder.newRow(Clustering.EMPTY);
+ DeletionTime expectedCmplxDelete = new DeletionTime(ts[1] - 1, nowInSec);
+ builder.addComplexDeletion(m, expectedCmplxDelete);
+
+ InetAddressAndPort peer2 = replicas.get(1).endpoint();
+ resolver.preprocess(response(cmd, peer2, iter(PartitionUpdate.singleRowUpdate(cfm2, dk, builder.build()))));
+
+ try(PartitionIterator data = resolver.resolve())
+ {
+ assertFalse(data.hasNext());
+ }
+
+ Mutation mutation = readRepair.getForEndpoint(peer1);
+ Iterator<Row> rowIter = mutation.getPartitionUpdate(cfm2).iterator();
+ assertTrue(rowIter.hasNext());
+ Row row = rowIter.next();
+ assertFalse(rowIter.hasNext());
+
+ ComplexColumnData cd = row.getComplexColumnData(m);
+
+ assertEquals(Collections.emptySet(), Sets.newHashSet(cd));
+ assertEquals(expectedCmplxDelete, cd.complexDeletion());
+
+ Assert.assertNull(readRepair.sent.get(peer2));
+ }
+
+ @Test
+ public void testResolveNewCollection()
+ {
+ EndpointsForRange replicas = makeReplicas(2);
+ ReadCommand cmd = Util.cmd(cfs2, dk).withNowInSeconds(nowInSec).build();
+ TestableReadRepair readRepair = new TestableReadRepair(cmd);
+ DataResolver resolver = new DataResolver(cmd, plan(replicas, ConsistencyLevel.ALL), readRepair, System.nanoTime());
+
+ long[] ts = {100, 200};
+
+ // map column
+ Row.Builder builder = BTreeRow.unsortedBuilder();
+ builder.newRow(Clustering.EMPTY);
+ DeletionTime expectedCmplxDelete = new DeletionTime(ts[0] - 1, nowInSec);
+ builder.addComplexDeletion(m, expectedCmplxDelete);
+ Cell expectedCell = mapCell(0, 0, ts[0]);
+ builder.addCell(expectedCell);
+
+ // empty map column
+ InetAddressAndPort peer1 = replicas.get(0).endpoint();
+ resolver.preprocess(response(cmd, peer1, iter(PartitionUpdate.singleRowUpdate(cfm2, dk, builder.build()))));
+
+ InetAddressAndPort peer2 = replicas.get(1).endpoint();
+ resolver.preprocess(response(cmd, peer2, iter(PartitionUpdate.emptyUpdate(cfm2, dk))));
+
+ try(PartitionIterator data = resolver.resolve())
+ {
+ try (RowIterator rows = Iterators.getOnlyElement(data))
+ {
+ Row row = Iterators.getOnlyElement(rows);
+ assertColumns(row, "m");
+ ComplexColumnData cd = row.getComplexColumnData(m);
+ assertEquals(Collections.singleton(expectedCell), Sets.newHashSet(cd));
+ }
+ }
+
+ Assert.assertNull(readRepair.sent.get(peer1));
+
+ Mutation mutation = readRepair.getForEndpoint(peer2);
+ Iterator<Row> rowIter = mutation.getPartitionUpdate(cfm2).iterator();
+ assertTrue(rowIter.hasNext());
+ Row row = rowIter.next();
+ assertFalse(rowIter.hasNext());
+
+ ComplexColumnData cd = row.getComplexColumnData(m);
+
+ assertEquals(Sets.newHashSet(expectedCell), Sets.newHashSet(cd));
+ assertEquals(expectedCmplxDelete, cd.complexDeletion());
+ }
+
+ @Test
+ public void testResolveNewCollectionOverwritingDeleted()
+ {
+ EndpointsForRange replicas = makeReplicas(2);
+ ReadCommand cmd = Util.cmd(cfs2, dk).withNowInSeconds(nowInSec).build();
+ TestableReadRepair readRepair = new TestableReadRepair(cmd);
+ DataResolver resolver = new DataResolver(cmd, plan(replicas, ConsistencyLevel.ALL), readRepair, System.nanoTime());
+
+ long[] ts = {100, 200};
+
+ // cleared map column
+ Row.Builder builder = BTreeRow.unsortedBuilder();
+ builder.newRow(Clustering.EMPTY);
+ builder.addComplexDeletion(m, new DeletionTime(ts[0] - 1, nowInSec));
+
+ InetAddressAndPort peer1 = replicas.get(0).endpoint();
+ resolver.preprocess(response(cmd, peer1, iter(PartitionUpdate.singleRowUpdate(cfm2, dk, builder.build()))));
+
+ // newer, overwritten map column
+ builder.newRow(Clustering.EMPTY);
+ DeletionTime expectedCmplxDelete = new DeletionTime(ts[1] - 1, nowInSec);
+ builder.addComplexDeletion(m, expectedCmplxDelete);
+ Cell expectedCell = mapCell(1, 1, ts[1]);
+ builder.addCell(expectedCell);
+
+ InetAddressAndPort peer2 = replicas.get(1).endpoint();
+ resolver.preprocess(response(cmd, peer2, iter(PartitionUpdate.singleRowUpdate(cfm2, dk, builder.build()))));
+
+ try(PartitionIterator data = resolver.resolve())
+ {
+ try (RowIterator rows = Iterators.getOnlyElement(data))
+ {
+ Row row = Iterators.getOnlyElement(rows);
+ assertColumns(row, "m");
+ ComplexColumnData cd = row.getComplexColumnData(m);
+ assertEquals(Collections.singleton(expectedCell), Sets.newHashSet(cd));
+ }
+ }
+
+ Row row = Iterators.getOnlyElement(readRepair.getForEndpoint(peer1).getPartitionUpdate(cfm2).iterator());
+
+ ComplexColumnData cd = row.getComplexColumnData(m);
+
+ assertEquals(Collections.singleton(expectedCell), Sets.newHashSet(cd));
+ assertEquals(expectedCmplxDelete, cd.complexDeletion());
+
+ Assert.assertNull(readRepair.sent.get(peer2));
+ }
+
+ /** Tests for repaired data tracking */
+
+ @Test
+ public void trackMatchingEmptyDigestsWithAllConclusive()
+ {
+ EndpointsForRange replicas = makeReplicas(2);
+ ByteBuffer digest1 = ByteBufferUtil.EMPTY_BYTE_BUFFER;
+ InetAddressAndPort peer1 = replicas.get(0).endpoint();
+ InetAddressAndPort peer2 = replicas.get(1).endpoint();
+ TestRepairedDataVerifier verifier = new TestRepairedDataVerifier();
+ verifier.expectDigest(peer1, digest1, true);
+ verifier.expectDigest(peer2, digest1, true);
+
+ DataResolver resolver = resolverWithVerifier(command, plan(replicas, ConsistencyLevel.ALL), readRepair, System.nanoTime(), verifier);
+
+ resolver.preprocess(response(peer1, iter(PartitionUpdate.emptyUpdate(cfm, dk)), digest1, true, command));
+ resolver.preprocess(response(peer2, iter(PartitionUpdate.emptyUpdate(cfm, dk)), digest1, true, command));
+
+ resolveAndConsume(resolver);
+ assertTrue(verifier.verified);
+ }
+
+ @Test
+ public void trackMatchingEmptyDigestsWithSomeConclusive()
+ {
+ ByteBuffer digest1 = ByteBufferUtil.EMPTY_BYTE_BUFFER;
+ EndpointsForRange replicas = makeReplicas(2);
+ InetAddressAndPort peer1 = replicas.get(0).endpoint();
+ InetAddressAndPort peer2 = replicas.get(1).endpoint();
+ TestRepairedDataVerifier verifier = new TestRepairedDataVerifier();
+ verifier.expectDigest(peer1, digest1, false);
+ verifier.expectDigest(peer2, digest1, true);
+
+ DataResolver resolver = resolverWithVerifier(command, plan(replicas, ConsistencyLevel.ALL), readRepair, System.nanoTime(), verifier);
+
+ resolver.preprocess(response(peer1, iter(PartitionUpdate.emptyUpdate(cfm, dk)), digest1, false, command));
+ resolver.preprocess(response(peer2, iter(PartitionUpdate.emptyUpdate(cfm, dk)), digest1, true, command));
+
+ resolveAndConsume(resolver);
+ assertTrue(verifier.verified);
+ }
+
+ @Test
+ public void trackMatchingEmptyDigestsWithNoneConclusive()
+ {
+ ByteBuffer digest1 = ByteBufferUtil.EMPTY_BYTE_BUFFER;
+ EndpointsForRange replicas = makeReplicas(2);
+ InetAddressAndPort peer1 = replicas.get(0).endpoint();
+ InetAddressAndPort peer2 = replicas.get(1).endpoint();
+ TestRepairedDataVerifier verifier = new TestRepairedDataVerifier();
+ verifier.expectDigest(peer1, digest1, false);
+ verifier.expectDigest(peer2, digest1, false);
+
+ DataResolver resolver = resolverWithVerifier(command, plan(replicas, ConsistencyLevel.ALL), readRepair, System.nanoTime(), verifier);
+
+ resolver.preprocess(response(peer1, iter(PartitionUpdate.emptyUpdate(cfm, dk)), digest1, false, command));
+ resolver.preprocess(response(peer2, iter(PartitionUpdate.emptyUpdate(cfm, dk)), digest1, false, command));
+
+ resolveAndConsume(resolver);
+ assertTrue(verifier.verified);
+ }
+
+ @Test
+ public void trackMatchingDigestsWithAllConclusive()
+ {
+ ByteBuffer digest1 = ByteBufferUtil.bytes("digest1");
+ EndpointsForRange replicas = makeReplicas(2);
+ InetAddressAndPort peer1 = replicas.get(0).endpoint();
+ InetAddressAndPort peer2 = replicas.get(1).endpoint();
+ TestRepairedDataVerifier verifier = new TestRepairedDataVerifier();
+ verifier.expectDigest(peer1, digest1, true);
+ verifier.expectDigest(peer2, digest1, true);
+
+ DataResolver resolver = resolverWithVerifier(command, plan(replicas, ConsistencyLevel.ALL), readRepair, System.nanoTime(), verifier);
+
+ resolver.preprocess(response(peer1, iter(PartitionUpdate.emptyUpdate(cfm,dk)), digest1, true, command));
+ resolver.preprocess(response(peer2, iter(PartitionUpdate.emptyUpdate(cfm, dk)), digest1, true, command));
+
+ resolveAndConsume(resolver);
+ assertTrue(verifier.verified);
+ }
+
+ @Test
+ public void trackMatchingDigestsWithSomeConclusive()
+ {
+ ByteBuffer digest1 = ByteBufferUtil.bytes("digest1");
+ EndpointsForRange replicas = makeReplicas(2);
+ InetAddressAndPort peer1 = replicas.get(0).endpoint();
+ InetAddressAndPort peer2 = replicas.get(1).endpoint();
+ TestRepairedDataVerifier verifier = new TestRepairedDataVerifier();
+ verifier.expectDigest(peer1, digest1, true);
+ verifier.expectDigest(peer2, digest1, false);
+
+ DataResolver resolver = resolverWithVerifier(command, plan(replicas, ConsistencyLevel.ALL), readRepair, System.nanoTime(), verifier);
+
+ resolver.preprocess(response(peer1, iter(PartitionUpdate.emptyUpdate(cfm,dk)), digest1, true, command));
+ resolver.preprocess(response(peer2, iter(PartitionUpdate.emptyUpdate(cfm, dk)), digest1, false, command));
+
+ resolveAndConsume(resolver);
+ assertTrue(verifier.verified);
+ }
+
+ @Test
+ public void trackMatchingDigestsWithNoneConclusive()
+ {
+ EndpointsForRange replicas = makeReplicas(2);
+ ByteBuffer digest1 = ByteBufferUtil.bytes("digest1");
+ InetAddressAndPort peer1 = replicas.get(0).endpoint();
+ InetAddressAndPort peer2 = replicas.get(1).endpoint();
+ TestRepairedDataVerifier verifier = new TestRepairedDataVerifier();
+ verifier.expectDigest(peer1, digest1, false);
+ verifier.expectDigest(peer2, digest1, false);
+
+ DataResolver resolver = resolverWithVerifier(command, plan(replicas, ConsistencyLevel.ALL), readRepair, System.nanoTime(), verifier);
+
+ resolver.preprocess(response(peer1, iter(PartitionUpdate.emptyUpdate(cfm, dk)), digest1, false, command));
+ resolver.preprocess(response(peer2, iter(PartitionUpdate.emptyUpdate(cfm, dk)), digest1, false, command));
+
+ resolveAndConsume(resolver);
+ assertTrue(verifier.verified);
+ }
+
+ @Test
+ public void trackMatchingRepairedDigestsWithDifferentData()
+ {
+ // As far as repaired data tracking is concerned, the actual data in the response is not relevant
+ EndpointsForRange replicas = makeReplicas(2);
+ ByteBuffer digest1 = ByteBufferUtil.bytes("digest1");
+ InetAddressAndPort peer1 = replicas.get(0).endpoint();
+ InetAddressAndPort peer2 = replicas.get(1).endpoint();
+ TestRepairedDataVerifier verifier = new TestRepairedDataVerifier();
+ verifier.expectDigest(peer1, digest1, true);
+ verifier.expectDigest(peer2, digest1, true);
+
+ DataResolver resolver = resolverWithVerifier(command, plan(replicas, ConsistencyLevel.ALL), readRepair, System.nanoTime(), verifier);
+
+ resolver.preprocess(response(peer1, iter(new RowUpdateBuilder(cfm, nowInSec, 1L, dk).clustering("1") .buildUpdate()), digest1, true, command));
+ resolver.preprocess(response(peer2, iter(PartitionUpdate.emptyUpdate(cfm, dk)), digest1, true, command));
+
+ resolveAndConsume(resolver);
+ assertTrue(verifier.verified);
+ }
+
+ @Test
+ public void trackMismatchingRepairedDigestsWithAllConclusive()
+ {
+ EndpointsForRange replicas = makeReplicas(2);
+ ByteBuffer digest1 = ByteBufferUtil.bytes("digest1");
+ ByteBuffer digest2 = ByteBufferUtil.bytes("digest2");
+ InetAddressAndPort peer1 = replicas.get(0).endpoint();
+ InetAddressAndPort peer2 = replicas.get(1).endpoint();
+ TestRepairedDataVerifier verifier = new TestRepairedDataVerifier();
+ verifier.expectDigest(peer1, digest1, true);
+ verifier.expectDigest(peer2, digest2, true);
+
+ DataResolver resolver = resolverWithVerifier(command, plan(replicas, ConsistencyLevel.ALL), readRepair, System.nanoTime(), verifier);
+
+ resolver.preprocess(response(peer1, iter(PartitionUpdate.emptyUpdate(cfm, dk)), digest1, true, command));
+ resolver.preprocess(response(peer2, iter(PartitionUpdate.emptyUpdate(cfm, dk)), digest2, true, command));
+
+ resolveAndConsume(resolver);
+ assertTrue(verifier.verified);
+ }
+
+ @Test
+ public void trackMismatchingRepairedDigestsWithSomeConclusive()
+ {
+ ByteBuffer digest1 = ByteBufferUtil.bytes("digest1");
+ ByteBuffer digest2 = ByteBufferUtil.bytes("digest2");
+ EndpointsForRange replicas = makeReplicas(2);
+ InetAddressAndPort peer1 = replicas.get(0).endpoint();
+ InetAddressAndPort peer2 = replicas.get(1).endpoint();
+ TestRepairedDataVerifier verifier = new TestRepairedDataVerifier();
+ verifier.expectDigest(peer1, digest1, false);
+ verifier.expectDigest(peer2, digest2, true);
+
+ DataResolver resolver = resolverWithVerifier(command, plan(replicas, ConsistencyLevel.ALL), readRepair, System.nanoTime(), verifier);
+
+ resolver.preprocess(response(peer1, iter(PartitionUpdate.emptyUpdate(cfm, dk)), digest1, false, command));
+ resolver.preprocess(response(peer2, iter(PartitionUpdate.emptyUpdate(cfm, dk)), digest2, true, command));
+
+ resolveAndConsume(resolver);
+ assertTrue(verifier.verified);
+ }
+
+ @Test
+ public void trackMismatchingRepairedDigestsWithNoneConclusive()
+ {
+ ByteBuffer digest1 = ByteBufferUtil.bytes("digest1");
+ ByteBuffer digest2 = ByteBufferUtil.bytes("digest2");
+ EndpointsForRange replicas = makeReplicas(2);
+ InetAddressAndPort peer1 = replicas.get(0).endpoint();
+ InetAddressAndPort peer2 = replicas.get(1).endpoint();
+ TestRepairedDataVerifier verifier = new TestRepairedDataVerifier();
+ verifier.expectDigest(peer1, digest1, false);
+ verifier.expectDigest(peer2, digest2, false);
+
+ DataResolver resolver = resolverWithVerifier(command, plan(replicas, ConsistencyLevel.ALL), readRepair, System.nanoTime(), verifier);
+
+ resolver.preprocess(response(peer1, iter(PartitionUpdate.emptyUpdate(cfm, dk)), digest1, false, command));
+ resolver.preprocess(response(peer2, iter(PartitionUpdate.emptyUpdate(cfm, dk)), digest2, false, command));
+
+ resolveAndConsume(resolver);
+ assertTrue(verifier.verified);
+ }
+
+ @Test
+ public void trackMismatchingRepairedDigestsWithDifferentData()
+ {
+ ByteBuffer digest1 = ByteBufferUtil.bytes("digest1");
+ ByteBuffer digest2 = ByteBufferUtil.bytes("digest2");
+ EndpointsForRange replicas = makeReplicas(2);
+ InetAddressAndPort peer1 = replicas.get(0).endpoint();
+ InetAddressAndPort peer2 = replicas.get(1).endpoint();
+ TestRepairedDataVerifier verifier = new TestRepairedDataVerifier();
+ verifier.expectDigest(peer1, digest1, true);
+ verifier.expectDigest(peer2, digest2, true);
+
+ DataResolver resolver = resolverWithVerifier(command, plan(replicas, ConsistencyLevel.ALL), readRepair, System.nanoTime(), verifier);
+
+ resolver.preprocess(response(peer1, iter(new RowUpdateBuilder(cfm, nowInSec, 1L, dk).clustering("1") .buildUpdate()), digest1, true, command));
+ resolver.preprocess(response(peer2, iter(PartitionUpdate.emptyUpdate(cfm, dk)), digest2, true, command));
+
+ resolveAndConsume(resolver);
+ assertTrue(verifier.verified);
+ }
+
+ @Test
+ public void noVerificationForSingletonResponse()
+ {
+ // for CL <= 1 a coordinator shouldn't request repaired data tracking but we
+ // can easily assert that the verification isn't attempted even if it did
+ EndpointsForRange replicas = makeReplicas(2);
+ ByteBuffer digest1 = ByteBufferUtil.bytes("digest1");
+ InetAddressAndPort peer1 = replicas.get(0).endpoint();
+ TestRepairedDataVerifier verifier = new TestRepairedDataVerifier();
+ verifier.expectDigest(peer1, digest1, true);
+
+ DataResolver resolver = resolverWithVerifier(command, plan(replicas, ConsistencyLevel.ALL), readRepair, System.nanoTime(), verifier);
+
+ resolver.preprocess(response(peer1, iter(PartitionUpdate.emptyUpdate(cfm,dk)), digest1, true, command));
+
+ resolveAndConsume(resolver);
+ assertFalse(verifier.verified);
+ }
+
+ @Test
+ public void responsesFromOlderVersionsAreNotTracked()
+ {
+ // In a mixed version cluster, responses from a replicas running older versions won't include
+ // tracking info, so the digest and pending session status are defaulted. To make sure these
+ // default values don't result in false positives we make sure not to consider them when
+ // processing in DataResolver
+ EndpointsForRange replicas = makeReplicas(2);
+ TestRepairedDataVerifier verifier = new TestRepairedDataVerifier();
+ ByteBuffer digest1 = ByteBufferUtil.bytes("digest1");
+ InetAddressAndPort peer1 = replicas.get(0).endpoint();
+ InetAddressAndPort peer2 = replicas.get(1).endpoint();
+ verifier.expectDigest(peer1, digest1, true);
+
+ DataResolver resolver = resolverWithVerifier(command, plan(replicas, ConsistencyLevel.ALL), readRepair, System.nanoTime(), verifier);
+
+ resolver.preprocess(response(peer1, iter(PartitionUpdate.emptyUpdate(cfm,dk)), digest1, true, command));
+ // peer2 is advertising an older version, so when we deserialize its response there are two things to note:
+ // i) the actual serialized response cannot contain any tracking info so deserialization will use defaults of
+ // an empty digest and pending sessions = false
+ // ii) under normal circumstances, this would cause a mismatch with peer1, but because of the older version,
+ // here it will not
+ resolver.preprocess(response(command, peer2, iter(PartitionUpdate.emptyUpdate(cfm,dk)),
+ false, MessagingService.VERSION_30,
+ ByteBufferUtil.EMPTY_BYTE_BUFFER, false));
+
+ resolveAndConsume(resolver);
+ assertTrue(verifier.verified);
+ }
+
+ @Test
+ public void responsesFromTransientReplicasAreNotTracked()
+ {
+ EndpointsForRange replicas = makeReplicas(2);
+ EndpointsForRange.Builder mutable = replicas.newBuilder(2);
+ mutable.add(replicas.get(0));
+ mutable.add(Replica.transientReplica(replicas.get(1).endpoint(), replicas.range()));
+ replicas = mutable.build();
+
+ TestRepairedDataVerifier verifier = new TestRepairedDataVerifier();
+ ByteBuffer digest1 = ByteBufferUtil.bytes("digest1");
+ ByteBuffer digest2 = ByteBufferUtil.bytes("digest2");
+ InetAddressAndPort peer1 = replicas.get(0).endpoint();
+ InetAddressAndPort peer2 = replicas.get(1).endpoint();
+ verifier.expectDigest(peer1, digest1, true);
+
+ DataResolver resolver = resolverWithVerifier(command, plan(replicas, ConsistencyLevel.ALL), readRepair, System.nanoTime(), verifier);
+
+ resolver.preprocess(response(peer1, iter(PartitionUpdate.emptyUpdate(cfm,dk)), digest1, true, command));
+ resolver.preprocess(response(peer2, iter(PartitionUpdate.emptyUpdate(cfm,dk)), digest2, true, command));
+
+ resolveAndConsume(resolver);
+ assertTrue(verifier.verified);
+ }
+
+ private static class TestRepairedDataVerifier implements RepairedDataVerifier
+ {
+ private final RepairedDataTracker expected = new RepairedDataTracker(null);
+ private boolean verified = false;
+
+ private void expectDigest(InetAddressAndPort from, ByteBuffer digest, boolean conclusive)
+ {
+ expected.recordDigest(from, digest, conclusive);
+ }
+
+ @Override
+ public void verify(RepairedDataTracker tracker)
+ {
+ verified = expected.equals(tracker);
+ }
+ }
+
+ private DataResolver resolverWithVerifier(final ReadCommand command,
+ final ReplicaPlan.SharedForRangeRead plan,
+ final ReadRepair readRepair,
+ final long queryStartNanoTime,
+ final RepairedDataVerifier verifier)
+ {
+ class TestableDataResolver extends DataResolver
+ {
+
+ public TestableDataResolver(ReadCommand command, ReplicaPlan.SharedForRangeRead plan, ReadRepair readRepair, long queryStartNanoTime)
+ {
+ super(command, plan, readRepair, queryStartNanoTime);
+ }
+
+ protected RepairedDataVerifier getRepairedDataVerifier(ReadCommand command)
+ {
+ return verifier;
+ }
+ }
+
+ return new TestableDataResolver(command, plan, readRepair, queryStartNanoTime);
+ }
+
+ private void assertRepairContainsDeletions(Mutation mutation,
+ DeletionTime deletionTime,
+ RangeTombstone...rangeTombstones)
+ {
+ PartitionUpdate update = mutation.getPartitionUpdates().iterator().next();
+ DeletionInfo deletionInfo = update.deletionInfo();
+ if (deletionTime != null)
+ assertEquals(deletionTime, deletionInfo.getPartitionDeletion());
+
+ assertEquals(rangeTombstones.length, deletionInfo.rangeCount());
+ Iterator<RangeTombstone> ranges = deletionInfo.rangeIterator(false);
+ int i = 0;
+ while (ranges.hasNext())
+ {
+ RangeTombstone expected = rangeTombstones[i++];
+ RangeTombstone actual = ranges.next();
+ String msg = String.format("Expected %s, but got %s", expected.toString(cfm.comparator), actual.toString(cfm.comparator));
+ assertEquals(msg, expected, actual);
+ }
+ }
+
+ private void assertRepairContainsNoDeletions(Mutation mutation)
+ {
+ PartitionUpdate update = mutation.getPartitionUpdates().iterator().next();
+ assertTrue(update.deletionInfo().isLive());
+ }
+
+ private void assertRepairContainsColumn(Mutation mutation,
+ String clustering,
+ String columnName,
+ String value,
+ long timestamp)
+ {
+ PartitionUpdate update = mutation.getPartitionUpdates().iterator().next();
+ Row row = update.getRow(update.metadata().comparator.make(clustering));
+ assertNotNull(row);
+ assertColumn(cfm, row, columnName, value, timestamp);
+ }
+
+ private void assertRepairContainsNoColumns(Mutation mutation)
+ {
+ PartitionUpdate update = mutation.getPartitionUpdates().iterator().next();
+ assertFalse(update.iterator().hasNext());
+ }
+
+ private void assertRepairMetadata(Mutation mutation)
+ {
+ PartitionUpdate update = mutation.getPartitionUpdates().iterator().next();
+ assertEquals(update.metadata().keyspace, ks.getName());
+ assertEquals(update.metadata().name, cfm.name);
+ }
+
+ private ReplicaPlan.SharedForRangeRead plan(EndpointsForRange replicas, ConsistencyLevel consistencyLevel)
+ {
+ return ReplicaPlan.shared(new ReplicaPlan.ForRangeRead(ks, consistencyLevel, ReplicaUtils.FULL_BOUNDS, replicas, replicas, 1));
+ }
+
+ private static void resolveAndConsume(DataResolver resolver)
+ {
+ try (PartitionIterator iterator = resolver.resolve())
+ {
+ while (iterator.hasNext())
+ {
+ try (RowIterator partition = iterator.next())
+ {
+ while (partition.hasNext())
+ partition.next();
+ }
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/service/reads/DigestResolverTest.java b/test/unit/org/apache/cassandra/service/reads/DigestResolverTest.java
new file mode 100644
index 0000000..99101f1
--- /dev/null
+++ b/test/unit/org/apache/cassandra/service/reads/DigestResolverTest.java
@@ -0,0 +1,166 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.reads;
+
+import org.apache.cassandra.locator.ReplicaPlan;
+import org.junit.Assert;
+import org.junit.Test;
+
+import org.apache.cassandra.db.ConsistencyLevel;
+import org.apache.cassandra.db.SimpleBuilders;
+import org.apache.cassandra.db.SinglePartitionReadCommand;
+import org.apache.cassandra.db.partitions.PartitionUpdate;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.locator.EndpointsForToken;
+import org.apache.cassandra.locator.ReplicaLayout;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.service.reads.repair.NoopReadRepair;
+import org.apache.cassandra.service.reads.repair.TestableReadRepair;
+
+import static org.apache.cassandra.locator.ReplicaUtils.full;
+import static org.apache.cassandra.locator.ReplicaUtils.trans;
+
+public class DigestResolverTest extends AbstractReadResponseTest
+{
+ private static PartitionUpdate.Builder update(TableMetadata metadata, String key, Row... rows)
+ {
+ PartitionUpdate.Builder builder = new PartitionUpdate.Builder(metadata, dk(key), metadata.regularAndStaticColumns(), rows.length, false);
+ for (Row row: rows)
+ {
+ builder.add(row);
+ }
+ return builder;
+ }
+
+ private static PartitionUpdate.Builder update(Row... rows)
+ {
+ return update(cfm, "key1", rows);
+ }
+
+ private static Row row(long timestamp, int clustering, int value)
+ {
+ SimpleBuilders.RowBuilder builder = new SimpleBuilders.RowBuilder(cfm, Integer.toString(clustering));
+ builder.timestamp(timestamp).add("c1", Integer.toString(value));
+ return builder.build();
+ }
+
+ @Test
+ public void noRepairNeeded()
+ {
+ SinglePartitionReadCommand command = SinglePartitionReadCommand.fullPartitionRead(cfm, nowInSec, dk);
+ EndpointsForToken targetReplicas = EndpointsForToken.of(dk.getToken(), full(EP1), full(EP2));
+ DigestResolver resolver = new DigestResolver(command, plan(ConsistencyLevel.QUORUM, targetReplicas), 0);
+
+ PartitionUpdate response = update(row(1000, 4, 4), row(1000, 5, 5)).build();
+
+ Assert.assertFalse(resolver.isDataPresent());
+ resolver.preprocess(response(command, EP2, iter(response), true));
+ resolver.preprocess(response(command, EP1, iter(response), false));
+ Assert.assertTrue(resolver.isDataPresent());
+ Assert.assertTrue(resolver.responsesMatch());
+
+ assertPartitionsEqual(filter(iter(response)), resolver.getData());
+ }
+
+ @Test
+ public void digestMismatch()
+ {
+ SinglePartitionReadCommand command = SinglePartitionReadCommand.fullPartitionRead(cfm, nowInSec, dk);
+ EndpointsForToken targetReplicas = EndpointsForToken.of(dk.getToken(), full(EP1), full(EP2));
+ DigestResolver resolver = new DigestResolver(command, plan(ConsistencyLevel.QUORUM, targetReplicas), 0);
+
+ PartitionUpdate response1 = update(row(1000, 4, 4), row(1000, 5, 5)).build();
+ PartitionUpdate response2 = update(row(2000, 4, 5)).build();
+
+ Assert.assertFalse(resolver.isDataPresent());
+ resolver.preprocess(response(command, EP2, iter(response1), true));
+ resolver.preprocess(response(command, EP1, iter(response2), false));
+ Assert.assertTrue(resolver.isDataPresent());
+ Assert.assertFalse(resolver.responsesMatch());
+ Assert.assertFalse(resolver.hasTransientResponse());
+ }
+
+ /**
+ * A full response and a transient response, with the transient response being a subset of the full one
+ */
+ @Test
+ public void agreeingTransient()
+ {
+ SinglePartitionReadCommand command = SinglePartitionReadCommand.fullPartitionRead(cfm, nowInSec, dk);
+ EndpointsForToken targetReplicas = EndpointsForToken.of(dk.getToken(), full(EP1), trans(EP2));
+ DigestResolver<?, ?> resolver = new DigestResolver<>(command, plan(ConsistencyLevel.QUORUM, targetReplicas), 0);
+
+ PartitionUpdate response1 = update(row(1000, 4, 4), row(1000, 5, 5)).build();
+ PartitionUpdate response2 = update(row(1000, 5, 5)).build();
+
+ Assert.assertFalse(resolver.isDataPresent());
+ resolver.preprocess(response(command, EP1, iter(response1), false));
+ resolver.preprocess(response(command, EP2, iter(response2), false));
+ Assert.assertTrue(resolver.isDataPresent());
+ Assert.assertTrue(resolver.responsesMatch());
+ Assert.assertTrue(resolver.hasTransientResponse());
+ }
+
+ /**
+ * Transient responses shouldn't be classified as the single dataResponse
+ */
+ @Test
+ public void transientResponse()
+ {
+ SinglePartitionReadCommand command = SinglePartitionReadCommand.fullPartitionRead(cfm, nowInSec, dk);
+ EndpointsForToken targetReplicas = EndpointsForToken.of(dk.getToken(), full(EP1), trans(EP2));
+ DigestResolver<?, ?> resolver = new DigestResolver<>(command, plan(ConsistencyLevel.QUORUM, targetReplicas), 0);
+
+ PartitionUpdate response2 = update(row(1000, 5, 5)).build();
+ Assert.assertFalse(resolver.isDataPresent());
+ Assert.assertFalse(resolver.hasTransientResponse());
+ resolver.preprocess(response(command, EP2, iter(response2), false));
+ Assert.assertFalse(resolver.isDataPresent());
+ Assert.assertTrue(resolver.hasTransientResponse());
+ }
+
+ @Test
+ public void transientResponseData()
+ {
+ SinglePartitionReadCommand command = SinglePartitionReadCommand.fullPartitionRead(cfm, nowInSec, dk);
+ EndpointsForToken targetReplicas = EndpointsForToken.of(dk.getToken(), full(EP1), full(EP2), trans(EP3));
+ DigestResolver<?, ?> resolver = new DigestResolver<>(command, plan(ConsistencyLevel.QUORUM, targetReplicas), 0);
+
+ PartitionUpdate fullResponse = update(row(1000, 1, 1)).build();
+ PartitionUpdate digestResponse = update(row(1000, 1, 1)).build();
+ PartitionUpdate transientResponse = update(row(1000, 2, 2)).build();
+ Assert.assertFalse(resolver.isDataPresent());
+ Assert.assertFalse(resolver.hasTransientResponse());
+ resolver.preprocess(response(command, EP1, iter(fullResponse), false));
+ Assert.assertTrue(resolver.isDataPresent());
+ resolver.preprocess(response(command, EP2, iter(digestResponse), true));
+ resolver.preprocess(response(command, EP3, iter(transientResponse), false));
+ Assert.assertTrue(resolver.hasTransientResponse());
+
+ assertPartitionsEqual(filter(iter(dk,
+ row(1000, 1, 1),
+ row(1000, 2, 2))),
+ resolver.getData());
+ }
+
+ private ReplicaPlan.SharedForTokenRead plan(ConsistencyLevel consistencyLevel, EndpointsForToken replicas)
+ {
+ return ReplicaPlan.shared(new ReplicaPlan.ForTokenRead(ks, consistencyLevel, replicas, replicas));
+ }
+}
diff --git a/test/unit/org/apache/cassandra/service/reads/ReadExecutorTest.java b/test/unit/org/apache/cassandra/service/reads/ReadExecutorTest.java
new file mode 100644
index 0000000..e0a5927
--- /dev/null
+++ b/test/unit/org/apache/cassandra/service/reads/ReadExecutorTest.java
@@ -0,0 +1,227 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.reads;
+
+import java.util.concurrent.TimeUnit;
+
+import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.locator.ReplicaPlan;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.Util;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.ConsistencyLevel;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.SinglePartitionReadCommand;
+import org.apache.cassandra.exceptions.ReadFailureException;
+import org.apache.cassandra.exceptions.ReadTimeoutException;
+import org.apache.cassandra.exceptions.RequestFailureReason;
+import org.apache.cassandra.locator.EndpointsForToken;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.net.Message;
+import org.apache.cassandra.net.NoPayload;
+import org.apache.cassandra.net.Verb;
+import org.apache.cassandra.schema.KeyspaceParams;
+
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+import static org.apache.cassandra.locator.ReplicaUtils.full;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+public class ReadExecutorTest
+{
+ static Keyspace ks;
+ static ColumnFamilyStore cfs;
+ static EndpointsForToken targets;
+ static Token dummy;
+
+ @BeforeClass
+ public static void setUpClass() throws Throwable
+ {
+ SchemaLoader.loadSchema();
+ SchemaLoader.createKeyspace("Foo", KeyspaceParams.simple(3), SchemaLoader.standardCFMD("Foo", "Bar"));
+ ks = Keyspace.open("Foo");
+ cfs = ks.getColumnFamilyStore("Bar");
+ dummy = Murmur3Partitioner.instance.getMinimumToken();
+ targets = EndpointsForToken.of(dummy,
+ full(InetAddressAndPort.getByName("127.0.0.255")),
+ full(InetAddressAndPort.getByName("127.0.0.254")),
+ full(InetAddressAndPort.getByName("127.0.0.253"))
+ );
+ cfs.sampleReadLatencyNanos = 0;
+ }
+
+ @Before
+ public void resetCounters() throws Throwable
+ {
+ cfs.metric.speculativeInsufficientReplicas.dec(cfs.metric.speculativeInsufficientReplicas.getCount());
+ cfs.metric.speculativeRetries.dec(cfs.metric.speculativeRetries.getCount());
+ cfs.metric.speculativeFailedRetries.dec(cfs.metric.speculativeFailedRetries.getCount());
+ }
+
+ /**
+ * If speculation would have been beneficial but could not be attempted due to lack of replicas
+ * count that it occured
+ */
+ @Test
+ public void testUnableToSpeculate() throws Throwable
+ {
+ assertEquals(0, cfs.metric.speculativeInsufficientReplicas.getCount());
+ assertEquals(0, ks.metric.speculativeInsufficientReplicas.getCount());
+ AbstractReadExecutor executor = new AbstractReadExecutor.NeverSpeculatingReadExecutor(cfs, new MockSinglePartitionReadCommand(), plan(targets, ConsistencyLevel.LOCAL_QUORUM), System.nanoTime(), true);
+ executor.maybeTryAdditionalReplicas();
+ try
+ {
+ executor.awaitResponses();
+ fail();
+ }
+ catch (ReadTimeoutException e)
+ {
+ //expected
+ }
+ assertEquals(1, cfs.metric.speculativeInsufficientReplicas.getCount());
+ assertEquals(1, ks.metric.speculativeInsufficientReplicas.getCount());
+
+ //Shouldn't increment
+ executor = new AbstractReadExecutor.NeverSpeculatingReadExecutor(cfs, new MockSinglePartitionReadCommand(), plan(targets, ConsistencyLevel.LOCAL_QUORUM), System.nanoTime(), false);
+ executor.maybeTryAdditionalReplicas();
+ try
+ {
+ executor.awaitResponses();
+ fail();
+ }
+ catch (ReadTimeoutException e)
+ {
+ //expected
+ }
+ assertEquals(1, cfs.metric.speculativeInsufficientReplicas.getCount());
+ assertEquals(1, ks.metric.speculativeInsufficientReplicas.getCount());
+ }
+
+ /**
+ * Test that speculation when it is attempted is countedc, and when it succeed
+ * no failure is counted.
+ */
+ @Test
+ public void testSpeculateSucceeded() throws Throwable
+ {
+ assertEquals(0, cfs.metric.speculativeRetries.getCount());
+ assertEquals(0, cfs.metric.speculativeFailedRetries.getCount());
+ assertEquals(0, ks.metric.speculativeRetries.getCount());
+ assertEquals(0, ks.metric.speculativeFailedRetries.getCount());
+ AbstractReadExecutor executor = new AbstractReadExecutor.SpeculatingReadExecutor(cfs, new MockSinglePartitionReadCommand(TimeUnit.DAYS.toMillis(365)), plan(ConsistencyLevel.LOCAL_QUORUM, targets, targets.subList(0, 2)), System.nanoTime());
+ executor.maybeTryAdditionalReplicas();
+ new Thread()
+ {
+ @Override
+ public void run()
+ {
+ //Failures end the read promptly but don't require mock data to be suppleid
+ executor.handler.onFailure(targets.get(0).endpoint(), RequestFailureReason.READ_TOO_MANY_TOMBSTONES);
+ executor.handler.onFailure(targets.get(1).endpoint(), RequestFailureReason.READ_TOO_MANY_TOMBSTONES);
+ executor.handler.condition.signalAll();
+ }
+ }.start();
+
+ try
+ {
+ executor.awaitResponses();
+ fail();
+ }
+ catch (ReadFailureException e)
+ {
+ //expected
+ }
+ assertEquals(1, cfs.metric.speculativeRetries.getCount());
+ assertEquals(0, cfs.metric.speculativeFailedRetries.getCount());
+ assertEquals(1, ks.metric.speculativeRetries.getCount());
+ assertEquals(0, ks.metric.speculativeFailedRetries.getCount());
+
+ }
+
+ /**
+ * Test that speculation failure statistics are incremented if speculation occurs
+ * and the read still times out.
+ */
+ @Test
+ public void testSpeculateFailed() throws Throwable
+ {
+ assertEquals(0, cfs.metric.speculativeRetries.getCount());
+ assertEquals(0, cfs.metric.speculativeFailedRetries.getCount());
+ assertEquals(0, ks.metric.speculativeRetries.getCount());
+ assertEquals(0, ks.metric.speculativeFailedRetries.getCount());
+ AbstractReadExecutor executor = new AbstractReadExecutor.SpeculatingReadExecutor(cfs, new MockSinglePartitionReadCommand(), plan(ConsistencyLevel.LOCAL_QUORUM, targets, targets.subList(0, 2)), System.nanoTime());
+ executor.maybeTryAdditionalReplicas();
+ try
+ {
+ executor.awaitResponses();
+ fail();
+ }
+ catch (ReadTimeoutException e)
+ {
+ //expected
+ }
+ assertEquals(1, cfs.metric.speculativeRetries.getCount());
+ assertEquals(1, cfs.metric.speculativeFailedRetries.getCount());
+ assertEquals(1, ks.metric.speculativeRetries.getCount());
+ assertEquals(1, ks.metric.speculativeFailedRetries.getCount());
+ }
+
+ public static class MockSinglePartitionReadCommand extends SinglePartitionReadCommand
+ {
+ private final long timeout;
+
+ MockSinglePartitionReadCommand()
+ {
+ this(0);
+ }
+
+ MockSinglePartitionReadCommand(long timeout)
+ {
+ super(false, 0, false, cfs.metadata(), 0, null, null, null, Util.dk("ry@n_luvs_teh_y@nk33z"), null, null);
+ this.timeout = timeout;
+ }
+
+ @Override
+ public long getTimeout(TimeUnit unit)
+ {
+ return unit.convert(timeout, MILLISECONDS);
+ }
+
+ @Override
+ public Message createMessage(boolean trackRepairedData)
+ {
+ return Message.out(Verb.ECHO_REQ, NoPayload.noPayload);
+ }
+ }
+
+ private ReplicaPlan.ForTokenRead plan(EndpointsForToken targets, ConsistencyLevel consistencyLevel)
+ {
+ return plan(consistencyLevel, targets, targets);
+ }
+
+ private ReplicaPlan.ForTokenRead plan(ConsistencyLevel consistencyLevel, EndpointsForToken natural, EndpointsForToken selected)
+ {
+ return new ReplicaPlan.ForTokenRead(ks, consistencyLevel, natural, selected);
+ }
+}
diff --git a/test/unit/org/apache/cassandra/service/reads/SpeculativeRetryParseTest.java b/test/unit/org/apache/cassandra/service/reads/SpeculativeRetryParseTest.java
new file mode 100644
index 0000000..86b307e
--- /dev/null
+++ b/test/unit/org/apache/cassandra/service/reads/SpeculativeRetryParseTest.java
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.reads;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Locale;
+
+import org.junit.Test;
+import org.junit.experimental.runners.Enclosed;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import org.apache.cassandra.exceptions.ConfigurationException;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.runners.Parameterized.Parameters;
+
+import static org.apache.cassandra.service.reads.HybridSpeculativeRetryPolicy.Function;
+
+@RunWith(Enclosed.class)
+public class SpeculativeRetryParseTest
+{
+ @RunWith(Parameterized.class)
+ public static class SuccessfulParseTest
+ {
+ static final Locale defaultLocale = Locale.getDefault();
+
+ private final String string;
+ private final SpeculativeRetryPolicy expectedValue;
+
+ public SuccessfulParseTest(String string, SpeculativeRetryPolicy expectedValue)
+ {
+ this.string = string;
+ this.expectedValue = expectedValue;
+ }
+
+ @Parameters
+ public static Collection<Object[]> generateData()
+ {
+ return Arrays.asList(new Object[][]{
+ { "NONE", NeverSpeculativeRetryPolicy.INSTANCE },
+ { "None", NeverSpeculativeRetryPolicy.INSTANCE },
+ { "Never", NeverSpeculativeRetryPolicy.INSTANCE },
+ { "NEVER", NeverSpeculativeRetryPolicy.INSTANCE },
+
+ { "ALWAYS", AlwaysSpeculativeRetryPolicy.INSTANCE },
+ { "Always", AlwaysSpeculativeRetryPolicy.INSTANCE },
+
+ { "95PERCENTILE", new PercentileSpeculativeRetryPolicy(95.0) },
+ { "99PERCENTILE", new PercentileSpeculativeRetryPolicy(99.0) },
+ { "99.5PERCENTILE", new PercentileSpeculativeRetryPolicy(99.5) },
+ { "99.9PERCENTILE", new PercentileSpeculativeRetryPolicy(99.9) },
+ { "99.95PERCENTILE", new PercentileSpeculativeRetryPolicy(99.95) },
+ { "99.99PERCENTILE", new PercentileSpeculativeRetryPolicy(99.99) },
+ { "21.1percentile", new PercentileSpeculativeRetryPolicy(21.1) },
+ { "78.11p", new PercentileSpeculativeRetryPolicy(78.11) },
+
+ { "1ms", new FixedSpeculativeRetryPolicy(1) },
+ { "10ms", new FixedSpeculativeRetryPolicy(10) },
+ { "100ms", new FixedSpeculativeRetryPolicy(100) },
+ { "1000ms", new FixedSpeculativeRetryPolicy(1000) },
+ { "121.1ms", new FixedSpeculativeRetryPolicy(121) },
+ { "21.7MS", new FixedSpeculativeRetryPolicy(21) },
+
+ { "max(99p,53ms)",
+ new HybridSpeculativeRetryPolicy(new PercentileSpeculativeRetryPolicy(99.0),
+ new FixedSpeculativeRetryPolicy(53),
+ Function.MAX) },
+ { "max(53ms,99p)",
+ new HybridSpeculativeRetryPolicy(new PercentileSpeculativeRetryPolicy(99.0),
+ new FixedSpeculativeRetryPolicy(53),
+ Function.MAX) },
+ { "MIN(70MS,90PERCENTILE)",
+ new HybridSpeculativeRetryPolicy(new PercentileSpeculativeRetryPolicy(90.0),
+ new FixedSpeculativeRetryPolicy(70),
+ Function.MIN) },
+ { "MIN(70MS, 90PERCENTILE)",
+ new HybridSpeculativeRetryPolicy(new PercentileSpeculativeRetryPolicy(90.0),
+ new FixedSpeculativeRetryPolicy(70),
+ Function.MIN) }
+ }
+ );
+ }
+
+ @Test
+ public void testParameterParse()
+ {
+ assertEquals(expectedValue, SpeculativeRetryPolicy.fromString(string));
+ }
+
+ @Test
+ public void testToStringRoundTripDefaultLocale()
+ {
+ assertEquals(expectedValue, SpeculativeRetryPolicy.fromString(expectedValue.toString()));
+ }
+
+ @Test
+ public void testToStringRoundTripCommaDecimalSeparatorLocale()
+ {
+ Locale.setDefault(new Locale("pt","BR")); // CASSANDRA-14374: Brazil uses comma instead of dot as decimal separator
+ assertEquals(expectedValue, SpeculativeRetryPolicy.fromString(expectedValue.toString()));
+ Locale.setDefault(defaultLocale);
+ }
+ }
+
+ @RunWith(Parameterized.class)
+ public static class FailedParseTest
+ {
+ private final String string;
+
+ public FailedParseTest(String string)
+ {
+ this.string = string;
+ }
+
+ @Parameters
+ public static Collection<Object[]> generateData()
+ {
+ return Arrays.asList(new Object[][]{
+ { "" },
+ { "-0.1PERCENTILE" },
+ { "100.1PERCENTILE" },
+ { "xPERCENTILE" },
+ { "xyzms" },
+ { "X" }
+ }
+ );
+ }
+
+ @Test(expected = ConfigurationException.class)
+ public void testParameterParse()
+ {
+ SpeculativeRetryPolicy.fromString(string);
+ }
+ }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/service/reads/repair/AbstractReadRepairTest.java b/test/unit/org/apache/cassandra/service/reads/repair/AbstractReadRepairTest.java
new file mode 100644
index 0000000..14074ed
--- /dev/null
+++ b/test/unit/org/apache/cassandra/service/reads/repair/AbstractReadRepairTest.java
@@ -0,0 +1,350 @@
+package org.apache.cassandra.service.reads.repair;
+
+import java.nio.ByteBuffer;
+import java.util.List;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Consumer;
+
+import com.google.common.base.Predicates;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+import com.google.common.primitives.Ints;
+
+import org.apache.cassandra.dht.ByteOrderedPartitioner;
+import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.gms.Gossiper;
+import org.apache.cassandra.locator.EndpointsForToken;
+import org.apache.cassandra.locator.ReplicaPlan;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.Util;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.ColumnIdentifier;
+import org.apache.cassandra.cql3.statements.schema.CreateTableStatement;
+import org.apache.cassandra.db.Clustering;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.ConsistencyLevel;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.Mutation;
+import org.apache.cassandra.db.ReadCommand;
+import org.apache.cassandra.db.ReadResponse;
+import org.apache.cassandra.db.partitions.PartitionIterator;
+import org.apache.cassandra.db.partitions.PartitionUpdate;
+import org.apache.cassandra.db.partitions.SingletonUnfilteredPartitionIterator;
+import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
+import org.apache.cassandra.db.partitions.UnfilteredPartitionIterators;
+import org.apache.cassandra.db.rows.BTreeRow;
+import org.apache.cassandra.db.rows.BufferCell;
+import org.apache.cassandra.db.rows.Cell;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.db.rows.RowIterator;
+import org.apache.cassandra.locator.EndpointsForRange;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.Replica;
+import org.apache.cassandra.locator.ReplicaPlans;
+import org.apache.cassandra.locator.ReplicaUtils;
+import org.apache.cassandra.net.Message;
+import org.apache.cassandra.schema.KeyspaceMetadata;
+import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.MigrationManager;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.schema.Tables;
+import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.utils.ByteBufferUtil;
+
+import static org.apache.cassandra.locator.Replica.fullReplica;
+import static org.apache.cassandra.locator.ReplicaUtils.FULL_RANGE;
+import static org.apache.cassandra.net.Verb.INTERNAL_RSP;
+
+@Ignore
+public abstract class AbstractReadRepairTest
+{
+ static Keyspace ks;
+ static ColumnFamilyStore cfs;
+ static TableMetadata cfm;
+ static InetAddressAndPort target1;
+ static InetAddressAndPort target2;
+ static InetAddressAndPort target3;
+ static List<InetAddressAndPort> targets;
+
+ static Replica replica1;
+ static Replica replica2;
+ static Replica replica3;
+ static EndpointsForRange replicas;
+ static ReplicaPlan.ForRead<?> replicaPlan;
+
+ static long now = TimeUnit.NANOSECONDS.toMicros(System.nanoTime());
+ static DecoratedKey key;
+ static Cell cell1;
+ static Cell cell2;
+ static Cell cell3;
+ static Mutation resolved;
+
+ static ReadCommand command;
+
+ static void assertRowsEqual(Row expected, Row actual)
+ {
+ try
+ {
+ Assert.assertEquals(expected == null, actual == null);
+ if (expected == null)
+ return;
+ Assert.assertEquals(expected.clustering(), actual.clustering());
+ Assert.assertEquals(expected.deletion(), actual.deletion());
+ Assert.assertArrayEquals(Iterables.toArray(expected.cells(), Cell.class), Iterables.toArray(expected.cells(), Cell.class));
+ } catch (Throwable t)
+ {
+ throw new AssertionError(String.format("Row comparison failed, expected %s got %s", expected, actual), t);
+ }
+ }
+
+ static void assertRowsEqual(RowIterator expected, RowIterator actual)
+ {
+ assertRowsEqual(expected.staticRow(), actual.staticRow());
+ while (expected.hasNext())
+ {
+ assert actual.hasNext();
+ assertRowsEqual(expected.next(), actual.next());
+ }
+ assert !actual.hasNext();
+ }
+
+ static void assertPartitionsEqual(PartitionIterator expected, PartitionIterator actual)
+ {
+ while (expected.hasNext())
+ {
+ assert actual.hasNext();
+ assertRowsEqual(expected.next(), actual.next());
+ }
+
+ assert !actual.hasNext();
+ }
+
+ static void assertMutationEqual(Mutation expected, Mutation actual)
+ {
+ Assert.assertEquals(expected.getKeyspaceName(), actual.getKeyspaceName());
+ Assert.assertEquals(expected.key(), actual.key());
+ Assert.assertEquals(expected.key(), actual.key());
+ PartitionUpdate expectedUpdate = Iterables.getOnlyElement(expected.getPartitionUpdates());
+ PartitionUpdate actualUpdate = Iterables.getOnlyElement(actual.getPartitionUpdates());
+ assertRowsEqual(Iterables.getOnlyElement(expectedUpdate), Iterables.getOnlyElement(actualUpdate));
+ }
+
+ static DecoratedKey dk(int v)
+ {
+ return DatabaseDescriptor.getPartitioner().decorateKey(ByteBufferUtil.bytes(v));
+ }
+
+ static Cell cell(String name, String value, long timestamp)
+ {
+ return BufferCell.live(cfm.getColumn(ColumnIdentifier.getInterned(name, false)), timestamp, ByteBufferUtil.bytes(value));
+ }
+
+ static PartitionUpdate update(Cell... cells)
+ {
+ Row.Builder builder = BTreeRow.unsortedBuilder();
+ builder.newRow(Clustering.EMPTY);
+ for (Cell cell: cells)
+ {
+ builder.addCell(cell);
+ }
+ return PartitionUpdate.singleRowUpdate(cfm, key, builder.build());
+ }
+
+ static PartitionIterator partition(Cell... cells)
+ {
+ UnfilteredPartitionIterator iter = new SingletonUnfilteredPartitionIterator(update(cells).unfilteredIterator());
+ return UnfilteredPartitionIterators.filter(iter, Ints.checkedCast(TimeUnit.MICROSECONDS.toSeconds(now)));
+ }
+
+ static Mutation mutation(Cell... cells)
+ {
+ return new Mutation(update(cells));
+ }
+
+ @SuppressWarnings("resource")
+ static Message<ReadResponse> msg(InetAddressAndPort from, Cell... cells)
+ {
+ UnfilteredPartitionIterator iter = new SingletonUnfilteredPartitionIterator(update(cells).unfilteredIterator());
+ return Message.builder(INTERNAL_RSP, ReadResponse.createDataResponse(iter, command))
+ .from(from)
+ .build();
+ }
+
+ static class ResultConsumer implements Consumer<PartitionIterator>
+ {
+
+ PartitionIterator result = null;
+
+ @Override
+ public void accept(PartitionIterator partitionIterator)
+ {
+ Assert.assertNotNull(partitionIterator);
+ result = partitionIterator;
+ }
+ }
+
+ private static boolean configured = false;
+
+ static void configureClass(ReadRepairStrategy repairStrategy) throws Throwable
+ {
+ SchemaLoader.loadSchema();
+ String ksName = "ks";
+
+ String ddl = String.format("CREATE TABLE tbl (k int primary key, v text) WITH read_repair='%s'",
+ repairStrategy.toString().toLowerCase());
+
+ cfm = CreateTableStatement.parse(ddl, ksName).build();
+ assert cfm.params.readRepair == repairStrategy;
+ KeyspaceMetadata ksm = KeyspaceMetadata.create(ksName, KeyspaceParams.simple(3), Tables.of(cfm));
+ MigrationManager.announceNewKeyspace(ksm, false);
+
+ ks = Keyspace.open(ksName);
+ cfs = ks.getColumnFamilyStore("tbl");
+
+ cfs.sampleReadLatencyNanos = 0;
+ cfs.additionalWriteLatencyNanos = 0;
+
+ target1 = InetAddressAndPort.getByName("127.0.0.255");
+ target2 = InetAddressAndPort.getByName("127.0.0.254");
+ target3 = InetAddressAndPort.getByName("127.0.0.253");
+
+ targets = ImmutableList.of(target1, target2, target3);
+
+ replica1 = fullReplica(target1, FULL_RANGE);
+ replica2 = fullReplica(target2, FULL_RANGE);
+ replica3 = fullReplica(target3, FULL_RANGE);
+ replicas = EndpointsForRange.of(replica1, replica2, replica3);
+
+ replicaPlan = replicaPlan(ConsistencyLevel.QUORUM, replicas);
+
+ StorageService.instance.getTokenMetadata().clearUnsafe();
+ StorageService.instance.getTokenMetadata().updateNormalToken(ByteOrderedPartitioner.instance.getToken(ByteBuffer.wrap(new byte[] { 0 })), replica1.endpoint());
+ StorageService.instance.getTokenMetadata().updateNormalToken(ByteOrderedPartitioner.instance.getToken(ByteBuffer.wrap(new byte[] { 1 })), replica2.endpoint());
+ StorageService.instance.getTokenMetadata().updateNormalToken(ByteOrderedPartitioner.instance.getToken(ByteBuffer.wrap(new byte[] { 2 })), replica3.endpoint());
+ Gossiper.instance.initializeNodeUnsafe(replica1.endpoint(), UUID.randomUUID(), 1);
+ Gossiper.instance.initializeNodeUnsafe(replica2.endpoint(), UUID.randomUUID(), 1);
+ Gossiper.instance.initializeNodeUnsafe(replica3.endpoint(), UUID.randomUUID(), 1);
+
+ // default test values
+ key = dk(5);
+ cell1 = cell("v", "val1", now);
+ cell2 = cell("v", "val2", now);
+ cell3 = cell("v", "val3", now);
+ resolved = mutation(cell1, cell2);
+
+ command = Util.cmd(cfs, 1).build();
+
+ configured = true;
+ }
+
+ static Set<InetAddressAndPort> epSet(InetAddressAndPort... eps)
+ {
+ return Sets.newHashSet(eps);
+ }
+
+ @Before
+ public void setUp()
+ {
+ assert configured : "configureClass must be called in a @BeforeClass method";
+
+ cfs.sampleReadLatencyNanos = 0;
+ cfs.additionalWriteLatencyNanos = 0;
+ }
+
+ static ReplicaPlan.ForRangeRead replicaPlan(ConsistencyLevel consistencyLevel, EndpointsForRange replicas)
+ {
+ return replicaPlan(ks, consistencyLevel, replicas, replicas);
+ }
+
+ static ReplicaPlan.ForTokenWrite repairPlan(ReplicaPlan.ForRangeRead readPlan)
+ {
+ return repairPlan(readPlan, readPlan.candidates());
+ }
+
+ static ReplicaPlan.ForTokenWrite repairPlan(EndpointsForRange liveAndDown, EndpointsForRange targets)
+ {
+ return repairPlan(replicaPlan(liveAndDown, targets), liveAndDown);
+ }
+
+ static ReplicaPlan.ForTokenWrite repairPlan(ReplicaPlan.ForRangeRead readPlan, EndpointsForRange liveAndDown)
+ {
+ Token token = readPlan.range().left.getToken();
+ EndpointsForToken pending = EndpointsForToken.empty(token);
+ return ReplicaPlans.forWrite(ks, ConsistencyLevel.TWO, liveAndDown.forToken(token), pending, Predicates.alwaysTrue(), ReplicaPlans.writeReadRepair(readPlan));
+ }
+ static ReplicaPlan.ForRangeRead replicaPlan(EndpointsForRange replicas, EndpointsForRange targets)
+ {
+ return replicaPlan(ks, ConsistencyLevel.QUORUM, replicas, targets);
+ }
+ static ReplicaPlan.ForRangeRead replicaPlan(Keyspace keyspace, ConsistencyLevel consistencyLevel, EndpointsForRange replicas)
+ {
+ return replicaPlan(keyspace, consistencyLevel, replicas, replicas);
+ }
+ static ReplicaPlan.ForRangeRead replicaPlan(Keyspace keyspace, ConsistencyLevel consistencyLevel, EndpointsForRange replicas, EndpointsForRange targets)
+ {
+ return new ReplicaPlan.ForRangeRead(keyspace, consistencyLevel, ReplicaUtils.FULL_BOUNDS, replicas, targets, 1);
+ }
+
+ public abstract InstrumentedReadRepair createInstrumentedReadRepair(ReadCommand command, ReplicaPlan.Shared<?, ?> replicaPlan, long queryStartNanoTime);
+
+ public InstrumentedReadRepair createInstrumentedReadRepair(ReplicaPlan.Shared<?, ?> replicaPlan)
+ {
+ return createInstrumentedReadRepair(command, replicaPlan, System.nanoTime());
+
+ }
+
+ /**
+ * If we haven't received enough full data responses by the time the speculation
+ * timeout occurs, we should send read requests to additional replicas
+ */
+ @Test
+ public void readSpeculationCycle()
+ {
+ InstrumentedReadRepair repair = createInstrumentedReadRepair(ReplicaPlan.shared(replicaPlan(replicas, EndpointsForRange.of(replica1, replica2))));
+ ResultConsumer consumer = new ResultConsumer();
+
+ Assert.assertEquals(epSet(), repair.getReadRecipients());
+ repair.startRepair(null, consumer);
+
+ Assert.assertEquals(epSet(target1, target2), repair.getReadRecipients());
+ repair.maybeSendAdditionalReads();
+ Assert.assertEquals(epSet(target1, target2, target3), repair.getReadRecipients());
+ Assert.assertNull(consumer.result);
+ }
+
+ /**
+ * If we receive enough data responses by the before the speculation timeout
+ * passes, we shouldn't send additional read requests
+ */
+ @Test
+ public void noSpeculationRequired()
+ {
+ InstrumentedReadRepair repair = createInstrumentedReadRepair(ReplicaPlan.shared(replicaPlan(replicas, EndpointsForRange.of(replica1, replica2))));
+ ResultConsumer consumer = new ResultConsumer();
+
+ Assert.assertEquals(epSet(), repair.getReadRecipients());
+ repair.startRepair(null, consumer);
+
+ Assert.assertEquals(epSet(target1, target2), repair.getReadRecipients());
+ repair.getReadCallback().onResponse(msg(target1, cell1));
+ repair.getReadCallback().onResponse(msg(target2, cell1));
+
+ repair.maybeSendAdditionalReads();
+ Assert.assertEquals(epSet(target1, target2), repair.getReadRecipients());
+
+ repair.awaitReads();
+
+ assertPartitionsEqual(partition(cell1), consumer.result);
+ }
+}
diff --git a/test/unit/org/apache/cassandra/service/reads/repair/BlockingReadRepairTest.java b/test/unit/org/apache/cassandra/service/reads/repair/BlockingReadRepairTest.java
new file mode 100644
index 0000000..e4b3a71
--- /dev/null
+++ b/test/unit/org/apache/cassandra/service/reads/repair/BlockingReadRepairTest.java
@@ -0,0 +1,290 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.reads.repair;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
+import com.google.common.collect.Lists;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.Util;
+import org.apache.cassandra.db.ConsistencyLevel;
+import org.apache.cassandra.db.Mutation;
+import org.apache.cassandra.db.ReadCommand;
+import org.apache.cassandra.locator.Endpoints;
+import org.apache.cassandra.locator.EndpointsForRange;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.Replica;
+import org.apache.cassandra.locator.ReplicaPlan;
+import org.apache.cassandra.locator.ReplicaUtils;
+import org.apache.cassandra.net.Message;
+import org.apache.cassandra.service.reads.ReadCallback;
+
+public class BlockingReadRepairTest extends AbstractReadRepairTest
+{
+ private static class InstrumentedReadRepairHandler
+ extends BlockingPartitionRepair
+ {
+ public InstrumentedReadRepairHandler(Map<Replica, Mutation> repairs, ReplicaPlan.ForTokenWrite writePlan)
+ {
+ super(Util.dk("not a real usable value"), repairs, writePlan, e -> targets.contains(e));
+ }
+
+ Map<InetAddressAndPort, Mutation> mutationsSent = new HashMap<>();
+
+ protected void sendRR(Message<Mutation> message, InetAddressAndPort endpoint)
+ {
+ mutationsSent.put(endpoint, message.payload);
+ }
+ }
+
+ @BeforeClass
+ public static void setUpClass() throws Throwable
+ {
+ configureClass(ReadRepairStrategy.BLOCKING);
+ }
+
+ private static InstrumentedReadRepairHandler createRepairHandler(Map<Replica, Mutation> repairs, ReplicaPlan.ForTokenWrite writePlan)
+ {
+ return new InstrumentedReadRepairHandler(repairs, writePlan);
+ }
+
+ private static InstrumentedReadRepairHandler createRepairHandler(Map<Replica, Mutation> repairs)
+ {
+ EndpointsForRange replicas = EndpointsForRange.copyOf(Lists.newArrayList(repairs.keySet()));
+ return createRepairHandler(repairs, repairPlan(replicas, replicas));
+ }
+
+ private static class InstrumentedBlockingReadRepair<E extends Endpoints<E>, P extends ReplicaPlan.ForRead<E>>
+ extends BlockingReadRepair<E, P> implements InstrumentedReadRepair<E, P>
+ {
+ public InstrumentedBlockingReadRepair(ReadCommand command, ReplicaPlan.Shared<E, P> replicaPlan, long queryStartNanoTime)
+ {
+ super(command, replicaPlan, queryStartNanoTime);
+ }
+
+ Set<InetAddressAndPort> readCommandRecipients = new HashSet<>();
+ ReadCallback readCallback = null;
+
+ @Override
+ void sendReadCommand(Replica to, ReadCallback callback, boolean speculative)
+ {
+ assert readCallback == null || readCallback == callback;
+ readCommandRecipients.add(to.endpoint());
+ readCallback = callback;
+ }
+
+ @Override
+ public Set<InetAddressAndPort> getReadRecipients()
+ {
+ return readCommandRecipients;
+ }
+
+ @Override
+ public ReadCallback getReadCallback()
+ {
+ return readCallback;
+ }
+ }
+
+ @Override
+ public InstrumentedReadRepair createInstrumentedReadRepair(ReadCommand command, ReplicaPlan.Shared<?, ?> replicaPlan, long queryStartNanoTime)
+ {
+ return new InstrumentedBlockingReadRepair(command, replicaPlan, queryStartNanoTime);
+ }
+
+ @Test
+ public void consistencyLevelTest() throws Exception
+ {
+ Assert.assertTrue(ConsistencyLevel.QUORUM.satisfies(ConsistencyLevel.QUORUM, ks));
+ Assert.assertTrue(ConsistencyLevel.THREE.satisfies(ConsistencyLevel.QUORUM, ks));
+ Assert.assertTrue(ConsistencyLevel.TWO.satisfies(ConsistencyLevel.QUORUM, ks));
+ Assert.assertFalse(ConsistencyLevel.ONE.satisfies(ConsistencyLevel.QUORUM, ks));
+ Assert.assertFalse(ConsistencyLevel.ANY.satisfies(ConsistencyLevel.QUORUM, ks));
+ }
+
+
+ @Test
+ public void additionalMutationRequired() throws Exception
+ {
+
+ Mutation repair1 = mutation(cell2);
+ Mutation repair2 = mutation(cell1);
+
+ // check that the correct repairs are calculated
+ Map<Replica, Mutation> repairs = new HashMap<>();
+ repairs.put(replica1, repair1);
+ repairs.put(replica2, repair2);
+
+ ReplicaPlan.ForTokenWrite writePlan = repairPlan(replicas, EndpointsForRange.copyOf(Lists.newArrayList(repairs.keySet())));
+ InstrumentedReadRepairHandler handler = createRepairHandler(repairs, writePlan);
+
+ Assert.assertTrue(handler.mutationsSent.isEmpty());
+
+ // check that the correct mutations are sent
+ handler.sendInitialRepairs();
+ Assert.assertEquals(2, handler.mutationsSent.size());
+ assertMutationEqual(repair1, handler.mutationsSent.get(target1));
+ assertMutationEqual(repair2, handler.mutationsSent.get(target2));
+
+ // check that a combined mutation is speculatively sent to the 3rd target
+ handler.mutationsSent.clear();
+ handler.maybeSendAdditionalWrites(0, TimeUnit.NANOSECONDS);
+ Assert.assertEquals(1, handler.mutationsSent.size());
+ assertMutationEqual(resolved, handler.mutationsSent.get(target3));
+
+ // check repairs stop blocking after receiving 2 acks
+ Assert.assertFalse(getCurrentRepairStatus(handler));
+ handler.ack(target1);
+ Assert.assertFalse(getCurrentRepairStatus(handler));
+ handler.ack(target3);
+ Assert.assertTrue(getCurrentRepairStatus(handler));
+
+ }
+
+ /**
+ * If we've received enough acks, we shouldn't send any additional mutations
+ */
+ @Test
+ public void noAdditionalMutationRequired() throws Exception
+ {
+ Map<Replica, Mutation> repairs = new HashMap<>();
+ repairs.put(replica1, mutation(cell2));
+ repairs.put(replica2, mutation(cell1));
+
+ InstrumentedReadRepairHandler handler = createRepairHandler(repairs);
+ handler.sendInitialRepairs();
+ handler.ack(target1);
+ handler.ack(target2);
+
+ // both replicas have acked, we shouldn't send anything else out
+ handler.mutationsSent.clear();
+ handler.maybeSendAdditionalWrites(0, TimeUnit.NANOSECONDS);
+ Assert.assertTrue(handler.mutationsSent.isEmpty());
+ }
+
+ /**
+ * If there are no additional nodes we can send mutations to, we... shouldn't
+ */
+ @Test
+ public void noAdditionalMutationPossible() throws Exception
+ {
+ Map<Replica, Mutation> repairs = new HashMap<>();
+ repairs.put(replica1, mutation(cell2));
+ repairs.put(replica2, mutation(cell1));
+
+ InstrumentedReadRepairHandler handler = createRepairHandler(repairs);
+ handler.sendInitialRepairs();
+
+ // we've already sent mutations to all candidates, so we shouldn't send any more
+ handler.mutationsSent.clear();
+ handler.maybeSendAdditionalWrites(0, TimeUnit.NANOSECONDS);
+ Assert.assertTrue(handler.mutationsSent.isEmpty());
+ }
+
+ /**
+ * If we didn't send a repair to a replica because there wasn't a diff with the
+ * resolved column family, we shouldn't send it a speculative mutation
+ */
+ @Test
+ public void mutationsArentSentToInSyncNodes() throws Exception
+ {
+ Mutation repair1 = mutation(cell2);
+
+ Map<Replica, Mutation> repairs = new HashMap<>();
+ repairs.put(replica1, repair1);
+
+ // check that the correct initial mutations are sent out
+ InstrumentedReadRepairHandler handler = createRepairHandler(repairs, repairPlan(replicas, EndpointsForRange.of(replica1, replica2)));
+ handler.sendInitialRepairs();
+ Assert.assertEquals(1, handler.mutationsSent.size());
+ Assert.assertTrue(handler.mutationsSent.containsKey(target1));
+
+ // check that speculative mutations aren't sent to target2
+ handler.mutationsSent.clear();
+ handler.maybeSendAdditionalWrites(0, TimeUnit.NANOSECONDS);
+ Assert.assertEquals(1, handler.mutationsSent.size());
+ Assert.assertTrue(handler.mutationsSent.containsKey(target3));
+ }
+
+ @Test
+ public void onlyBlockOnQuorum()
+ {
+ Map<Replica, Mutation> repairs = new HashMap<>();
+ repairs.put(replica1, mutation(cell1));
+ repairs.put(replica2, mutation(cell2));
+ repairs.put(replica3, mutation(cell3));
+ Assert.assertEquals(3, repairs.size());
+
+ InstrumentedReadRepairHandler handler = createRepairHandler(repairs);
+ handler.sendInitialRepairs();
+
+ Assert.assertFalse(getCurrentRepairStatus(handler));
+ handler.ack(target1);
+ Assert.assertFalse(getCurrentRepairStatus(handler));
+
+ // here we should stop blocking, even though we've sent 3 repairs
+ handler.ack(target2);
+ Assert.assertTrue(getCurrentRepairStatus(handler));
+ }
+
+ /**
+ * For dc local consistency levels, noop mutations and responses from remote dcs should not affect effective blockFor
+ */
+ @Test
+ public void remoteDCTest() throws Exception
+ {
+ Map<Replica, Mutation> repairs = new HashMap<>();
+ repairs.put(replica1, mutation(cell1));
+
+ Replica remote1 = ReplicaUtils.full(InetAddressAndPort.getByName("10.0.0.1"));
+ Replica remote2 = ReplicaUtils.full(InetAddressAndPort.getByName("10.0.0.2"));
+ repairs.put(remote1, mutation(cell1));
+
+ EndpointsForRange participants = EndpointsForRange.of(replica1, replica2, remote1, remote2);
+ ReplicaPlan.ForTokenWrite writePlan = repairPlan(replicaPlan(ks, ConsistencyLevel.LOCAL_QUORUM, participants));
+ InstrumentedReadRepairHandler handler = createRepairHandler(repairs, writePlan);
+ handler.sendInitialRepairs();
+ Assert.assertEquals(2, handler.mutationsSent.size());
+ Assert.assertTrue(handler.mutationsSent.containsKey(replica1.endpoint()));
+ Assert.assertTrue(handler.mutationsSent.containsKey(remote1.endpoint()));
+
+ Assert.assertEquals(1, handler.waitingOn());
+ Assert.assertFalse(getCurrentRepairStatus(handler));
+
+ handler.ack(remote1.endpoint());
+ Assert.assertEquals(1, handler.waitingOn());
+ Assert.assertFalse(getCurrentRepairStatus(handler));
+
+ handler.ack(replica1.endpoint());
+ Assert.assertEquals(0, handler.waitingOn());
+ Assert.assertTrue(getCurrentRepairStatus(handler));
+ }
+
+ private boolean getCurrentRepairStatus(BlockingPartitionRepair handler)
+ {
+ return handler.awaitRepairsUntil(System.nanoTime(), TimeUnit.NANOSECONDS);
+ }
+}
diff --git a/test/unit/org/apache/cassandra/service/reads/repair/DiagEventsBlockingReadRepairTest.java b/test/unit/org/apache/cassandra/service/reads/repair/DiagEventsBlockingReadRepairTest.java
new file mode 100644
index 0000000..7806a3f
--- /dev/null
+++ b/test/unit/org/apache/cassandra/service/reads/repair/DiagEventsBlockingReadRepairTest.java
@@ -0,0 +1,194 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.reads.repair;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Predicate;
+
+import com.google.common.collect.Lists;
+
+import org.apache.cassandra.locator.ReplicaPlan;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.config.OverrideConfigurationLoader;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.Mutation;
+import org.apache.cassandra.db.ReadCommand;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.diag.DiagnosticEventService;
+import org.apache.cassandra.locator.Endpoints;
+import org.apache.cassandra.locator.EndpointsForRange;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.Replica;
+import org.apache.cassandra.net.Message;
+import org.apache.cassandra.service.reads.ReadCallback;
+import org.apache.cassandra.service.reads.repair.ReadRepairEvent.ReadRepairEventType;
+
+/**
+ * Variation of {@link BlockingReadRepair} using diagnostic events instead of instrumentation for test validation.
+ */
+public class DiagEventsBlockingReadRepairTest extends AbstractReadRepairTest
+{
+
+ @BeforeClass
+ public static void setUpClass() throws Throwable
+ {
+ OverrideConfigurationLoader.override((config) -> {
+ config.diagnostic_events_enabled = true;
+ });
+ configureClass(ReadRepairStrategy.BLOCKING);
+ }
+
+ @After
+ public void unsubscribeAll()
+ {
+ DiagnosticEventService.instance().cleanup();
+ }
+
+ @Test
+ public void additionalMutationRequired()
+ {
+ Mutation repair1 = mutation(cell2);
+ Mutation repair2 = mutation(cell1);
+
+ // check that the correct repairs are calculated
+ Map<Replica, Mutation> repairs = new HashMap<>();
+ repairs.put(replica1, repair1);
+ repairs.put(replica2, repair2);
+
+
+ ReplicaPlan.ForTokenWrite writePlan = repairPlan(replicas, EndpointsForRange.copyOf(Lists.newArrayList(repairs.keySet())));
+ DiagnosticPartitionReadRepairHandler handler = createRepairHandler(repairs, writePlan);
+
+ Assert.assertTrue(handler.updatesByEp.isEmpty());
+
+ // check that the correct mutations are sent
+ handler.sendInitialRepairs();
+ Assert.assertEquals(2, handler.updatesByEp.size());
+
+ Assert.assertEquals(repair1.toString(), handler.updatesByEp.get(target1));
+ Assert.assertEquals(repair2.toString(), handler.updatesByEp.get(target2));
+
+ // check that a combined mutation is speculatively sent to the 3rd target
+ handler.updatesByEp.clear();
+ handler.maybeSendAdditionalWrites(0, TimeUnit.NANOSECONDS);
+ Assert.assertEquals(1, handler.updatesByEp.size());
+ Assert.assertEquals(resolved.toString(), handler.updatesByEp.get(target3));
+
+ // check repairs stop blocking after receiving 2 acks
+ Assert.assertFalse(getCurrentRepairStatus(handler));
+ handler.ack(target1);
+ Assert.assertFalse(getCurrentRepairStatus(handler));
+ handler.ack(target3);
+ Assert.assertTrue(getCurrentRepairStatus(handler));
+ }
+
+ private boolean getCurrentRepairStatus(BlockingPartitionRepair handler)
+ {
+ return handler.awaitRepairsUntil(System.nanoTime(), TimeUnit.NANOSECONDS);
+ }
+
+ public InstrumentedReadRepair createInstrumentedReadRepair(ReadCommand command, ReplicaPlan.Shared<?,?> replicaPlan, long queryStartNanoTime)
+ {
+ return new DiagnosticBlockingRepairHandler(command, replicaPlan, queryStartNanoTime);
+ }
+
+ private static DiagnosticPartitionReadRepairHandler createRepairHandler(Map<Replica, Mutation> repairs, ReplicaPlan.ForTokenWrite writePlan)
+ {
+ return new DiagnosticPartitionReadRepairHandler(key, repairs, writePlan);
+ }
+
+ private static class DiagnosticBlockingRepairHandler extends BlockingReadRepair implements InstrumentedReadRepair
+ {
+ private Set<InetAddressAndPort> recipients = Collections.emptySet();
+ private ReadCallback readCallback = null;
+
+ DiagnosticBlockingRepairHandler(ReadCommand command, ReplicaPlan.Shared<?,?> replicaPlan, long queryStartNanoTime)
+ {
+ super(command, replicaPlan, queryStartNanoTime);
+ DiagnosticEventService.instance().subscribe(ReadRepairEvent.class, this::onRepairEvent);
+ }
+
+ private void onRepairEvent(ReadRepairEvent e)
+ {
+ if (e.getType() == ReadRepairEventType.START_REPAIR) recipients = new HashSet<>(e.destinations);
+ else if (e.getType() == ReadRepairEventType.SPECULATED_READ) recipients.addAll(e.destinations);
+ Assert.assertEquals(new HashSet<>(targets), new HashSet<>(e.allEndpoints));
+ Assert.assertNotNull(e.toMap());
+ }
+
+ void sendReadCommand(Replica to, ReadCallback callback, boolean speculative)
+ {
+ assert readCallback == null || readCallback == callback;
+ readCallback = callback;
+ }
+
+ Iterable<InetAddressAndPort> getCandidatesForToken(Token token)
+ {
+ return targets;
+ }
+
+ public Set<InetAddressAndPort> getReadRecipients()
+ {
+ return recipients;
+ }
+
+ public ReadCallback getReadCallback()
+ {
+ return readCallback;
+ }
+ }
+
+ private static class DiagnosticPartitionReadRepairHandler<E extends Endpoints<E>, P extends ReplicaPlan.ForRead<E>>
+ extends BlockingPartitionRepair
+ {
+ private final Map<InetAddressAndPort, String> updatesByEp = new HashMap<>();
+
+ private static Predicate<InetAddressAndPort> isLocal()
+ {
+ List<InetAddressAndPort> candidates = targets;
+ return e -> candidates.contains(e);
+ }
+
+ DiagnosticPartitionReadRepairHandler(DecoratedKey key, Map<Replica, Mutation> repairs, ReplicaPlan.ForTokenWrite writePlan)
+ {
+ super(key, repairs, writePlan, isLocal());
+ DiagnosticEventService.instance().subscribe(PartitionRepairEvent.class, this::onRepairEvent);
+ }
+
+ private void onRepairEvent(PartitionRepairEvent e)
+ {
+ updatesByEp.put(e.destination, e.mutationSummary);
+ Assert.assertNotNull(e.toMap());
+ }
+
+ protected void sendRR(Message<Mutation> message, InetAddressAndPort endpoint)
+ {
+ }
+ }
+}
diff --git a/test/unit/org/apache/cassandra/service/reads/repair/InstrumentedReadRepair.java b/test/unit/org/apache/cassandra/service/reads/repair/InstrumentedReadRepair.java
new file mode 100644
index 0000000..81ab07e
--- /dev/null
+++ b/test/unit/org/apache/cassandra/service/reads/repair/InstrumentedReadRepair.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.reads.repair;
+
+import java.util.Set;
+
+import org.apache.cassandra.locator.Endpoints;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.ReplicaLayout;
+import org.apache.cassandra.locator.ReplicaPlan;
+import org.apache.cassandra.service.reads.ReadCallback;
+
+public interface InstrumentedReadRepair<E extends Endpoints<E>, P extends ReplicaPlan.ForRead<E>>
+ extends ReadRepair<E, P>
+{
+ Set<InetAddressAndPort> getReadRecipients();
+
+ ReadCallback getReadCallback();
+}
diff --git a/test/unit/org/apache/cassandra/service/reads/repair/ReadOnlyReadRepairTest.java b/test/unit/org/apache/cassandra/service/reads/repair/ReadOnlyReadRepairTest.java
new file mode 100644
index 0000000..c0af493
--- /dev/null
+++ b/test/unit/org/apache/cassandra/service/reads/repair/ReadOnlyReadRepairTest.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.reads.repair;
+
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.cassandra.locator.ReplicaPlan;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.db.ReadCommand;
+import org.apache.cassandra.db.partitions.UnfilteredPartitionIterators;
+import org.apache.cassandra.locator.Endpoints;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.Replica;
+import org.apache.cassandra.locator.ReplicaLayout;
+import org.apache.cassandra.service.reads.ReadCallback;
+
+public class ReadOnlyReadRepairTest extends AbstractReadRepairTest
+{
+ private static class InstrumentedReadOnlyReadRepair<E extends Endpoints<E>, P extends ReplicaPlan.ForRead<E>>
+ extends ReadOnlyReadRepair implements InstrumentedReadRepair
+ {
+ public InstrumentedReadOnlyReadRepair(ReadCommand command, ReplicaPlan.Shared<E, P> replicaPlan, long queryStartNanoTime)
+ {
+ super(command, replicaPlan, queryStartNanoTime);
+ }
+
+ Set<InetAddressAndPort> readCommandRecipients = new HashSet<>();
+ ReadCallback readCallback = null;
+
+ @Override
+ void sendReadCommand(Replica to, ReadCallback callback, boolean speculative)
+ {
+ assert readCallback == null || readCallback == callback;
+ readCommandRecipients.add(to.endpoint());
+ readCallback = callback;
+ }
+
+ @Override
+ public Set<InetAddressAndPort> getReadRecipients()
+ {
+ return readCommandRecipients;
+ }
+
+ @Override
+ public ReadCallback getReadCallback()
+ {
+ return readCallback;
+ }
+ }
+
+ @BeforeClass
+ public static void setUpClass() throws Throwable
+ {
+ configureClass(ReadRepairStrategy.NONE);
+ }
+
+ @Override
+ public InstrumentedReadRepair createInstrumentedReadRepair(ReadCommand command, ReplicaPlan.Shared<?, ?> replicaPlan, long queryStartNanoTime)
+ {
+ return new InstrumentedReadOnlyReadRepair(command, replicaPlan, queryStartNanoTime);
+ }
+
+ @Test
+ public void getMergeListener()
+ {
+ ReplicaPlan.SharedForRangeRead replicaPlan = ReplicaPlan.shared(replicaPlan(replicas, replicas));
+ InstrumentedReadRepair repair = createInstrumentedReadRepair(replicaPlan);
+ Assert.assertSame(UnfilteredPartitionIterators.MergeListener.NOOP, repair.getMergeListener(replicaPlan.get()));
+ }
+
+ @Test(expected = UnsupportedOperationException.class)
+ public void repairPartitionFailure()
+ {
+ ReplicaPlan.SharedForRangeRead readPlan = ReplicaPlan.shared(replicaPlan(replicas, replicas));
+ ReplicaPlan.ForTokenWrite writePlan = repairPlan(replicas, replicas);
+ InstrumentedReadRepair repair = createInstrumentedReadRepair(readPlan);
+ repair.repairPartition(null, Collections.emptyMap(), writePlan);
+ }
+}
diff --git a/test/unit/org/apache/cassandra/service/reads/repair/ReadRepairTest.java b/test/unit/org/apache/cassandra/service/reads/repair/ReadRepairTest.java
new file mode 100644
index 0000000..7458e9b
--- /dev/null
+++ b/test/unit/org/apache/cassandra/service/reads/repair/ReadRepairTest.java
@@ -0,0 +1,351 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.reads.repair;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+import com.google.common.collect.Iterables;
+
+import org.apache.cassandra.Util;
+import org.apache.cassandra.locator.Endpoints;
+import org.apache.cassandra.locator.EndpointsForRange;
+import org.apache.cassandra.locator.ReplicaPlan;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.ColumnIdentifier;
+import org.apache.cassandra.cql3.statements.schema.CreateTableStatement;
+import org.apache.cassandra.db.Clustering;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.ConsistencyLevel;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.Mutation;
+import org.apache.cassandra.db.partitions.PartitionUpdate;
+import org.apache.cassandra.db.rows.BTreeRow;
+import org.apache.cassandra.db.rows.BufferCell;
+import org.apache.cassandra.db.rows.Cell;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.Replica;
+import org.apache.cassandra.net.Message;
+import org.apache.cassandra.schema.KeyspaceMetadata;
+import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.MigrationManager;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.schema.Tables;
+import org.apache.cassandra.utils.ByteBufferUtil;
+
+import static org.apache.cassandra.locator.ReplicaUtils.full;
+
+public class ReadRepairTest
+{
+ static Keyspace ks;
+ static ColumnFamilyStore cfs;
+ static TableMetadata cfm;
+ static Replica target1;
+ static Replica target2;
+ static Replica target3;
+ static EndpointsForRange targets;
+
+ private static class InstrumentedReadRepairHandler<E extends Endpoints<E>, P extends ReplicaPlan.ForRead<E>>
+ extends BlockingPartitionRepair
+ {
+ public InstrumentedReadRepairHandler(Map<Replica, Mutation> repairs, ReplicaPlan.ForTokenWrite writePlan)
+ {
+ super(Util.dk("not a valid key"), repairs, writePlan, e -> targets.endpoints().contains(e));
+ }
+
+ Map<InetAddressAndPort, Mutation> mutationsSent = new HashMap<>();
+
+ protected void sendRR(Message<Mutation> message, InetAddressAndPort endpoint)
+ {
+ mutationsSent.put(endpoint, message.payload);
+ }
+ }
+
+ static long now = TimeUnit.NANOSECONDS.toMicros(System.nanoTime());
+ static DecoratedKey key;
+ static Cell cell1;
+ static Cell cell2;
+ static Cell cell3;
+ static Mutation resolved;
+
+ private static void assertRowsEqual(Row expected, Row actual)
+ {
+ try
+ {
+ Assert.assertEquals(expected == null, actual == null);
+ if (expected == null)
+ return;
+ Assert.assertEquals(expected.clustering(), actual.clustering());
+ Assert.assertEquals(expected.deletion(), actual.deletion());
+ Assert.assertArrayEquals(Iterables.toArray(expected.cells(), Cell.class), Iterables.toArray(expected.cells(), Cell.class));
+ } catch (Throwable t)
+ {
+ throw new AssertionError(String.format("Row comparison failed, expected %s got %s", expected, actual), t);
+ }
+ }
+
+ @BeforeClass
+ public static void setUpClass() throws Throwable
+ {
+ SchemaLoader.loadSchema();
+ String ksName = "ks";
+
+ cfm = CreateTableStatement.parse("CREATE TABLE tbl (k int primary key, v text)", ksName).build();
+ KeyspaceMetadata ksm = KeyspaceMetadata.create(ksName, KeyspaceParams.simple(3), Tables.of(cfm));
+ MigrationManager.announceNewKeyspace(ksm, false);
+
+ ks = Keyspace.open(ksName);
+ cfs = ks.getColumnFamilyStore("tbl");
+
+ cfs.sampleReadLatencyNanos = 0;
+
+ target1 = full(InetAddressAndPort.getByName("127.0.0.255"));
+ target2 = full(InetAddressAndPort.getByName("127.0.0.254"));
+ target3 = full(InetAddressAndPort.getByName("127.0.0.253"));
+
+ targets = EndpointsForRange.of(target1, target2, target3);
+
+ // default test values
+ key = dk(5);
+ cell1 = cell("v", "val1", now);
+ cell2 = cell("v", "val2", now);
+ cell3 = cell("v", "val3", now);
+ resolved = mutation(cell1, cell2);
+ }
+
+ private static DecoratedKey dk(int v)
+ {
+ return DatabaseDescriptor.getPartitioner().decorateKey(ByteBufferUtil.bytes(v));
+ }
+
+ private static Cell cell(String name, String value, long timestamp)
+ {
+ return BufferCell.live(cfm.getColumn(ColumnIdentifier.getInterned(name, false)), timestamp, ByteBufferUtil.bytes(value));
+ }
+
+ private static Mutation mutation(Cell... cells)
+ {
+ Row.Builder builder = BTreeRow.unsortedBuilder();
+ builder.newRow(Clustering.EMPTY);
+ for (Cell cell: cells)
+ {
+ builder.addCell(cell);
+ }
+ return new Mutation(PartitionUpdate.singleRowUpdate(cfm, key, builder.build()));
+ }
+
+ private static InstrumentedReadRepairHandler createRepairHandler(Map<Replica, Mutation> repairs, EndpointsForRange all, EndpointsForRange targets)
+ {
+ ReplicaPlan.ForRangeRead readPlan = AbstractReadRepairTest.replicaPlan(ks, ConsistencyLevel.LOCAL_QUORUM, all, targets);
+ ReplicaPlan.ForTokenWrite writePlan = AbstractReadRepairTest.repairPlan(readPlan);
+ return new InstrumentedReadRepairHandler(repairs, writePlan);
+ }
+
+ @Test
+ public void consistencyLevelTest() throws Exception
+ {
+ Assert.assertTrue(ConsistencyLevel.QUORUM.satisfies(ConsistencyLevel.QUORUM, ks));
+ Assert.assertTrue(ConsistencyLevel.THREE.satisfies(ConsistencyLevel.QUORUM, ks));
+ Assert.assertTrue(ConsistencyLevel.TWO.satisfies(ConsistencyLevel.QUORUM, ks));
+ Assert.assertFalse(ConsistencyLevel.ONE.satisfies(ConsistencyLevel.QUORUM, ks));
+ Assert.assertFalse(ConsistencyLevel.ANY.satisfies(ConsistencyLevel.QUORUM, ks));
+ }
+
+ private static void assertMutationEqual(Mutation expected, Mutation actual)
+ {
+ Assert.assertEquals(expected.getKeyspaceName(), actual.getKeyspaceName());
+ Assert.assertEquals(expected.key(), actual.key());
+ Assert.assertEquals(expected.key(), actual.key());
+ PartitionUpdate expectedUpdate = Iterables.getOnlyElement(expected.getPartitionUpdates());
+ PartitionUpdate actualUpdate = Iterables.getOnlyElement(actual.getPartitionUpdates());
+ assertRowsEqual(Iterables.getOnlyElement(expectedUpdate), Iterables.getOnlyElement(actualUpdate));
+ }
+
+ @Test
+ public void additionalMutationRequired() throws Exception
+ {
+ Mutation repair1 = mutation(cell2);
+ Mutation repair2 = mutation(cell1);
+
+ // check that the correct repairs are calculated
+ Map<Replica, Mutation> repairs = new HashMap<>();
+ repairs.put(target1, repair1);
+ repairs.put(target2, repair2);
+
+ InstrumentedReadRepairHandler<?, ?> handler = createRepairHandler(repairs, targets, EndpointsForRange.of(target1, target2));
+
+ Assert.assertTrue(handler.mutationsSent.isEmpty());
+
+ // check that the correct mutations are sent
+ handler.sendInitialRepairs();
+ Assert.assertEquals(2, handler.mutationsSent.size());
+ assertMutationEqual(repair1, handler.mutationsSent.get(target1.endpoint()));
+ assertMutationEqual(repair2, handler.mutationsSent.get(target2.endpoint()));
+
+ // check that a combined mutation is speculatively sent to the 3rd target
+ handler.mutationsSent.clear();
+ handler.maybeSendAdditionalWrites(0, TimeUnit.NANOSECONDS);
+ Assert.assertEquals(1, handler.mutationsSent.size());
+ assertMutationEqual(resolved, handler.mutationsSent.get(target3.endpoint()));
+
+ // check repairs stop blocking after receiving 2 acks
+ Assert.assertFalse(getCurrentRepairStatus(handler));
+ handler.ack(target1.endpoint());
+ Assert.assertFalse(getCurrentRepairStatus(handler));
+ handler.ack(target3.endpoint());
+ Assert.assertTrue(getCurrentRepairStatus(handler));
+ }
+
+ /**
+ * If we've received enough acks, we shouldn't send any additional mutations
+ */
+ @Test
+ public void noAdditionalMutationRequired() throws Exception
+ {
+ Map<Replica, Mutation> repairs = new HashMap<>();
+ repairs.put(target1, mutation(cell2));
+ repairs.put(target2, mutation(cell1));
+
+ EndpointsForRange replicas = EndpointsForRange.of(target1, target2);
+ InstrumentedReadRepairHandler handler = createRepairHandler(repairs, replicas, targets);
+ handler.sendInitialRepairs();
+ handler.ack(target1.endpoint());
+ handler.ack(target2.endpoint());
+
+ // both replicas have acked, we shouldn't send anything else out
+ handler.mutationsSent.clear();
+ handler.maybeSendAdditionalWrites(0, TimeUnit.NANOSECONDS);
+ Assert.assertTrue(handler.mutationsSent.isEmpty());
+ }
+
+ /**
+ * If there are no additional nodes we can send mutations to, we... shouldn't
+ */
+ @Test
+ public void noAdditionalMutationPossible() throws Exception
+ {
+ Map<Replica, Mutation> repairs = new HashMap<>();
+ repairs.put(target1, mutation(cell2));
+ repairs.put(target2, mutation(cell1));
+
+ InstrumentedReadRepairHandler handler = createRepairHandler(repairs, EndpointsForRange.of(target1, target2),
+ EndpointsForRange.of(target1, target2));
+ handler.sendInitialRepairs();
+
+ // we've already sent mutations to all candidates, so we shouldn't send any more
+ handler.mutationsSent.clear();
+ handler.maybeSendAdditionalWrites(0, TimeUnit.NANOSECONDS);
+ Assert.assertTrue(handler.mutationsSent.isEmpty());
+ }
+
+ /**
+ * If we didn't send a repair to a replica because there wasn't a diff with the
+ * resolved column family, we shouldn't send it a speculative mutation
+ */
+ @Test
+ public void mutationsArentSentToInSyncNodes() throws Exception
+ {
+ Mutation repair1 = mutation(cell2);
+
+ Map<Replica, Mutation> repairs = new HashMap<>();
+ repairs.put(target1, repair1);
+
+ // check that the correct initial mutations are sent out
+ InstrumentedReadRepairHandler handler = createRepairHandler(repairs, targets, EndpointsForRange.of(target1, target2));
+ handler.sendInitialRepairs();
+ Assert.assertEquals(1, handler.mutationsSent.size());
+ Assert.assertTrue(handler.mutationsSent.containsKey(target1.endpoint()));
+
+ // check that speculative mutations aren't sent to target2
+ handler.mutationsSent.clear();
+ handler.maybeSendAdditionalWrites(0, TimeUnit.NANOSECONDS);
+
+ Assert.assertEquals(1, handler.mutationsSent.size());
+ Assert.assertTrue(handler.mutationsSent.containsKey(target3.endpoint()));
+ }
+
+ @Test
+ public void onlyBlockOnQuorum()
+ {
+ Map<Replica, Mutation> repairs = new HashMap<>();
+ repairs.put(target1, mutation(cell1));
+ repairs.put(target2, mutation(cell2));
+ repairs.put(target3, mutation(cell3));
+ Assert.assertEquals(3, repairs.size());
+
+ EndpointsForRange replicas = EndpointsForRange.of(target1, target2, target3);
+ InstrumentedReadRepairHandler handler = createRepairHandler(repairs, replicas, replicas);
+ handler.sendInitialRepairs();
+
+ Assert.assertFalse(getCurrentRepairStatus(handler));
+ handler.ack(target1.endpoint());
+ Assert.assertFalse(getCurrentRepairStatus(handler));
+
+ // here we should stop blocking, even though we've sent 3 repairs
+ handler.ack(target2.endpoint());
+ Assert.assertTrue(getCurrentRepairStatus(handler));
+ }
+
+ /**
+ * For dc local consistency levels, noop mutations and responses from remote dcs should not affect effective blockFor
+ */
+ @Test
+ public void remoteDCTest() throws Exception
+ {
+ Map<Replica, Mutation> repairs = new HashMap<>();
+ repairs.put(target1, mutation(cell1));
+
+ Replica remote1 = full(InetAddressAndPort.getByName("10.0.0.1"));
+ Replica remote2 = full(InetAddressAndPort.getByName("10.0.0.2"));
+ repairs.put(remote1, mutation(cell1));
+
+ EndpointsForRange participants = EndpointsForRange.of(target1, target2, remote1, remote2);
+ EndpointsForRange targets = EndpointsForRange.of(target1, target2);
+
+ InstrumentedReadRepairHandler handler = createRepairHandler(repairs, participants, targets);
+ handler.sendInitialRepairs();
+ Assert.assertEquals(2, handler.mutationsSent.size());
+ Assert.assertTrue(handler.mutationsSent.containsKey(target1.endpoint()));
+ Assert.assertTrue(handler.mutationsSent.containsKey(remote1.endpoint()));
+
+ Assert.assertEquals(1, handler.waitingOn());
+ Assert.assertFalse(getCurrentRepairStatus(handler));
+
+ handler.ack(remote1.endpoint());
+ Assert.assertEquals(1, handler.waitingOn());
+ Assert.assertFalse(getCurrentRepairStatus(handler));
+
+ handler.ack(target1.endpoint());
+ Assert.assertEquals(0, handler.waitingOn());
+ Assert.assertTrue(getCurrentRepairStatus(handler));
+ }
+
+ private boolean getCurrentRepairStatus(BlockingPartitionRepair handler)
+ {
+ return handler.awaitRepairsUntil(System.nanoTime(), TimeUnit.NANOSECONDS);
+ }
+}
diff --git a/test/unit/org/apache/cassandra/service/reads/repair/RepairedDataVerifierTest.java b/test/unit/org/apache/cassandra/service/reads/repair/RepairedDataVerifierTest.java
new file mode 100644
index 0000000..169e09d
--- /dev/null
+++ b/test/unit/org/apache/cassandra/service/reads/repair/RepairedDataVerifierTest.java
@@ -0,0 +1,293 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.reads.repair;
+
+import java.net.UnknownHostException;
+import java.util.Random;
+
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.ReadCommand;
+import org.apache.cassandra.db.SinglePartitionReadCommand;
+import org.apache.cassandra.db.Slices;
+import org.apache.cassandra.db.filter.ClusteringIndexSliceFilter;
+import org.apache.cassandra.db.filter.ColumnFilter;
+import org.apache.cassandra.db.filter.DataLimits;
+import org.apache.cassandra.db.filter.RowFilter;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.metrics.TableMetrics;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.FBUtilities;
+
+import static org.junit.Assert.assertEquals;
+
+public class RepairedDataVerifierTest
+{
+ private static final String TEST_NAME = "read_command_vh_test_";
+ private static final String KEYSPACE = TEST_NAME + "cql_keyspace";
+ private static final String TABLE = "table1";
+
+ private final Random random = new Random();
+ private TableMetadata metadata;
+ private TableMetrics metrics;
+
+ // counter to generate the last byte of peer addresses
+ private int addressSuffix = 10;
+
+ @BeforeClass
+ public static void init()
+ {
+ SchemaLoader.loadSchema();
+ SchemaLoader.schemaDefinition(TEST_NAME);
+ DatabaseDescriptor.reportUnconfirmedRepairedDataMismatches(true);
+ }
+
+ @Before
+ public void setup()
+ {
+ metadata = Schema.instance.getTableMetadata(KEYSPACE, TABLE);
+ metrics = ColumnFamilyStore.metricsFor(metadata.id);
+ }
+
+ @Test
+ public void repairedDataMismatchWithSomeConclusive()
+ {
+ long confirmedCount = confirmedCount();
+ long unconfirmedCount = unconfirmedCount();
+ InetAddressAndPort peer1 = peer();
+ InetAddressAndPort peer2 = peer();
+ RepairedDataVerifier.SimpleVerifier verifier = new RepairedDataVerifier.SimpleVerifier(command(key()));
+ RepairedDataTracker tracker = new RepairedDataTracker(verifier);
+ tracker.recordDigest(peer1, ByteBufferUtil.bytes("digest1"), false);
+ tracker.recordDigest(peer2, ByteBufferUtil.bytes("digest2"), true);
+
+ tracker.verify();
+ assertEquals(confirmedCount, confirmedCount());
+ assertEquals(unconfirmedCount + 1 , unconfirmedCount());
+ }
+
+ @Test
+ public void repairedDataMismatchWithNoneConclusive()
+ {
+ long confirmedCount = confirmedCount();
+ long unconfirmedCount = unconfirmedCount();
+ InetAddressAndPort peer1 = peer();
+ InetAddressAndPort peer2 = peer();
+ RepairedDataVerifier.SimpleVerifier verifier = new RepairedDataVerifier.SimpleVerifier(command(key()));
+ RepairedDataTracker tracker = new RepairedDataTracker(verifier);
+ tracker.recordDigest(peer1, ByteBufferUtil.bytes("digest1"), false);
+ tracker.recordDigest(peer2, ByteBufferUtil.bytes("digest2"), false);
+
+ tracker.verify();
+ assertEquals(confirmedCount, confirmedCount());
+ assertEquals(unconfirmedCount + 1 , unconfirmedCount());
+ }
+
+ @Test
+ public void repairedDataMismatchWithAllConclusive()
+ {
+ long confirmedCount = confirmedCount();
+ long unconfirmedCount = unconfirmedCount();
+ InetAddressAndPort peer1 = peer();
+ InetAddressAndPort peer2 = peer();
+ RepairedDataVerifier.SimpleVerifier verifier = new RepairedDataVerifier.SimpleVerifier(command(key()));
+ RepairedDataTracker tracker = new RepairedDataTracker(verifier);
+ tracker.recordDigest(peer1, ByteBufferUtil.bytes("digest1"), true);
+ tracker.recordDigest(peer2, ByteBufferUtil.bytes("digest2"), true);
+
+ tracker.verify();
+ assertEquals(confirmedCount + 1, confirmedCount());
+ assertEquals(unconfirmedCount, unconfirmedCount());
+ }
+
+ @Test
+ public void repairedDataMatchesWithAllConclusive()
+ {
+ long confirmedCount = confirmedCount();
+ long unconfirmedCount = unconfirmedCount();
+ InetAddressAndPort peer1 = peer();
+ InetAddressAndPort peer2 = peer();
+ RepairedDataVerifier.SimpleVerifier verifier = new RepairedDataVerifier.SimpleVerifier(command(key()));
+ RepairedDataTracker tracker = new RepairedDataTracker(verifier);
+ tracker.recordDigest(peer1, ByteBufferUtil.bytes("digest1"), true);
+ tracker.recordDigest(peer2, ByteBufferUtil.bytes("digest1"), true);
+
+ tracker.verify();
+ assertEquals(confirmedCount, confirmedCount());
+ assertEquals(unconfirmedCount, unconfirmedCount());
+ }
+
+ @Test
+ public void repairedDataMatchesWithSomeConclusive()
+ {
+ long confirmedCount = confirmedCount();
+ long unconfirmedCount = unconfirmedCount();
+ InetAddressAndPort peer1 = peer();
+ InetAddressAndPort peer2 = peer();
+ RepairedDataVerifier.SimpleVerifier verifier = new RepairedDataVerifier.SimpleVerifier(command(key()));
+ RepairedDataTracker tracker = new RepairedDataTracker(verifier);
+ tracker.recordDigest(peer1, ByteBufferUtil.bytes("digest1"), true);
+ tracker.recordDigest(peer2, ByteBufferUtil.bytes("digest1"), false);
+
+ tracker.verify();
+ assertEquals(confirmedCount, confirmedCount());
+ assertEquals(unconfirmedCount, unconfirmedCount());
+ }
+
+ @Test
+ public void repairedDataMatchesWithNoneConclusive()
+ {
+ long confirmedCount = confirmedCount();
+ long unconfirmedCount = unconfirmedCount();
+ InetAddressAndPort peer1 = peer();
+ InetAddressAndPort peer2 = peer();
+ RepairedDataVerifier.SimpleVerifier verifier = new RepairedDataVerifier.SimpleVerifier(command(key()));
+ RepairedDataTracker tracker = new RepairedDataTracker(verifier);
+ tracker.recordDigest(peer1, ByteBufferUtil.bytes("digest1"), false);
+ tracker.recordDigest(peer2, ByteBufferUtil.bytes("digest1"), false);
+
+ tracker.verify();
+ assertEquals(confirmedCount, confirmedCount());
+ assertEquals(unconfirmedCount, unconfirmedCount());
+ }
+
+ @Test
+ public void allEmptyDigestWithAllConclusive()
+ {
+ // if a read didn't touch any repaired sstables, digests will be empty
+ long confirmedCount = confirmedCount();
+ long unconfirmedCount = unconfirmedCount();
+ InetAddressAndPort peer1 = peer();
+ InetAddressAndPort peer2 = peer();
+ RepairedDataVerifier.SimpleVerifier verifier = new RepairedDataVerifier.SimpleVerifier(command(key()));
+ RepairedDataTracker tracker = new RepairedDataTracker(verifier);
+ tracker.recordDigest(peer1, ByteBufferUtil.EMPTY_BYTE_BUFFER, true);
+ tracker.recordDigest(peer2, ByteBufferUtil.EMPTY_BYTE_BUFFER, true);
+
+ tracker.verify();
+ assertEquals(confirmedCount, confirmedCount());
+ assertEquals(unconfirmedCount, unconfirmedCount());
+ }
+
+ @Test
+ public void allEmptyDigestsWithSomeConclusive()
+ {
+ // if a read didn't touch any repaired sstables, digests will be empty
+ long confirmedCount = confirmedCount();
+ long unconfirmedCount = unconfirmedCount();
+ InetAddressAndPort peer1 = peer();
+ InetAddressAndPort peer2 = peer();
+ RepairedDataVerifier.SimpleVerifier verifier = new RepairedDataVerifier.SimpleVerifier(command(key()));
+ RepairedDataTracker tracker = new RepairedDataTracker(verifier);
+ tracker.recordDigest(peer1, ByteBufferUtil.EMPTY_BYTE_BUFFER, true);
+ tracker.recordDigest(peer2, ByteBufferUtil.EMPTY_BYTE_BUFFER, false);
+
+ tracker.verify();
+ assertEquals(confirmedCount, confirmedCount());
+ assertEquals(unconfirmedCount, unconfirmedCount());
+ }
+
+ @Test
+ public void allEmptyDigestsWithNoneConclusive()
+ {
+ // if a read didn't touch any repaired sstables, digests will be empty
+ long confirmedCount = confirmedCount();
+ long unconfirmedCount = unconfirmedCount();
+ InetAddressAndPort peer1 = peer();
+ InetAddressAndPort peer2 = peer();
+ RepairedDataVerifier.SimpleVerifier verifier = new RepairedDataVerifier.SimpleVerifier(command(key()));
+ RepairedDataTracker tracker = new RepairedDataTracker(verifier);
+ tracker.recordDigest(peer1, ByteBufferUtil.EMPTY_BYTE_BUFFER, false);
+ tracker.recordDigest(peer2, ByteBufferUtil.EMPTY_BYTE_BUFFER, false);
+
+ tracker.verify();
+ assertEquals(confirmedCount, confirmedCount());
+ assertEquals(unconfirmedCount, unconfirmedCount());
+ }
+
+ @Test
+ public void noTrackingDataRecorded()
+ {
+ // if a read didn't land on any replicas which support repaired data tracking, nothing will be recorded
+ long confirmedCount = confirmedCount();
+ long unconfirmedCount = unconfirmedCount();
+ RepairedDataVerifier.SimpleVerifier verifier = new RepairedDataVerifier.SimpleVerifier(command(key()));
+ RepairedDataTracker tracker = new RepairedDataTracker(verifier);
+ tracker.verify();
+ assertEquals(confirmedCount, confirmedCount());
+ assertEquals(unconfirmedCount, unconfirmedCount());
+ }
+
+ private long confirmedCount()
+ {
+ return metrics.confirmedRepairedInconsistencies.table.getCount();
+ }
+
+ private long unconfirmedCount()
+ {
+ return metrics.unconfirmedRepairedInconsistencies.table.getCount();
+ }
+
+ private InetAddressAndPort peer()
+ {
+ try
+ {
+ return InetAddressAndPort.getByAddress(new byte[]{ 127, 0, 0, (byte) addressSuffix++ });
+ }
+ catch (UnknownHostException e)
+ {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private int key()
+ {
+ return random.nextInt();
+ }
+
+ private ReadCommand command(int key)
+ {
+ return new StubReadCommand(key, metadata, false);
+ }
+
+ private static class StubReadCommand extends SinglePartitionReadCommand
+ {
+ StubReadCommand(int key, TableMetadata metadata, boolean isDigest)
+ {
+ super(isDigest,
+ 0,
+ false,
+ metadata,
+ FBUtilities.nowInSeconds(),
+ ColumnFilter.all(metadata),
+ RowFilter.NONE,
+ DataLimits.NONE,
+ metadata.partitioner.decorateKey(ByteBufferUtil.bytes(key)),
+ new ClusteringIndexSliceFilter(Slices.ALL, false),
+ null);
+ }
+ }
+}
diff --git a/test/unit/org/apache/cassandra/service/reads/repair/TestableReadRepair.java b/test/unit/org/apache/cassandra/service/reads/repair/TestableReadRepair.java
new file mode 100644
index 0000000..84276d5
--- /dev/null
+++ b/test/unit/org/apache/cassandra/service/reads/repair/TestableReadRepair.java
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.reads.repair;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Consumer;
+
+import org.apache.cassandra.db.ConsistencyLevel;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.Mutation;
+import org.apache.cassandra.db.ReadCommand;
+import org.apache.cassandra.db.partitions.PartitionIterator;
+import org.apache.cassandra.db.partitions.UnfilteredPartitionIterators;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.db.rows.UnfilteredRowIterators;
+import org.apache.cassandra.exceptions.ReadTimeoutException;
+import org.apache.cassandra.locator.Endpoints;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.Replica;
+import org.apache.cassandra.locator.ReplicaLayout;
+import org.apache.cassandra.locator.ReplicaPlan;
+import org.apache.cassandra.service.reads.DigestResolver;
+
+public class TestableReadRepair<E extends Endpoints<E>, P extends ReplicaPlan.ForRead<E>>
+ implements ReadRepair<E, P>
+{
+ public final Map<InetAddressAndPort, Mutation> sent = new HashMap<>();
+
+ private final ReadCommand command;
+
+ private boolean partitionListenerClosed = false;
+ private boolean rowListenerClosed = true;
+
+ public TestableReadRepair(ReadCommand command)
+ {
+ this.command = command;
+ }
+
+ @Override
+ public UnfilteredPartitionIterators.MergeListener getMergeListener(P endpoints)
+ {
+ return new PartitionIteratorMergeListener<E>(endpoints, command, this) {
+ @Override
+ public void close()
+ {
+ super.close();
+ partitionListenerClosed = true;
+ }
+
+ @Override
+ public UnfilteredRowIterators.MergeListener getRowMergeListener(DecoratedKey partitionKey, List<UnfilteredRowIterator> versions)
+ {
+ assert rowListenerClosed;
+ rowListenerClosed = false;
+ return new RowIteratorMergeListener<E>(partitionKey, columns(versions), isReversed(versions), endpoints, command, TestableReadRepair.this) {
+ @Override
+ public void close()
+ {
+ super.close();
+ rowListenerClosed = true;
+ }
+ };
+ }
+ };
+ }
+
+ @Override
+ public void startRepair(DigestResolver<E, P> digestResolver, Consumer<PartitionIterator> resultConsumer)
+ {
+
+ }
+
+ @Override
+ public void awaitReads() throws ReadTimeoutException
+ {
+
+ }
+
+ @Override
+ public void maybeSendAdditionalReads()
+ {
+
+ }
+
+ @Override
+ public void maybeSendAdditionalWrites()
+ {
+
+ }
+
+ @Override
+ public void awaitWrites()
+ {
+
+ }
+
+ @Override
+ public void repairPartition(DecoratedKey partitionKey, Map<Replica, Mutation> mutations, ReplicaPlan.ForTokenWrite writePlan)
+ {
+ for (Map.Entry<Replica, Mutation> entry: mutations.entrySet())
+ sent.put(entry.getKey().endpoint(), entry.getValue());
+ }
+
+ public Mutation getForEndpoint(InetAddressAndPort endpoint)
+ {
+ return sent.get(endpoint);
+ }
+
+ public boolean dataWasConsumed()
+ {
+ return partitionListenerClosed && rowListenerClosed;
+ }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/streaming/EntireSSTableStreamingCorrectFilesCountTest.java b/test/unit/org/apache/cassandra/streaming/EntireSSTableStreamingCorrectFilesCountTest.java
new file mode 100644
index 0000000..262a200
--- /dev/null
+++ b/test/unit/org/apache/cassandra/streaming/EntireSSTableStreamingCorrectFilesCountTest.java
@@ -0,0 +1,237 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.streaming;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.WritableByteChannel;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.UUID;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.Unpooled;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.channel.ChannelOutboundHandlerAdapter;
+import io.netty.channel.ChannelPromise;
+import io.netty.channel.embedded.EmbeddedChannel;
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.RowUpdateBuilder;
+import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.db.streaming.CassandraOutgoingFile;
+import org.apache.cassandra.dht.ByteOrderedPartitioner;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.util.DataOutputStreamPlus;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.RangesAtEndpoint;
+import org.apache.cassandra.net.AsyncStreamingOutputPlus;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.net.SharedDefaultFileRegion;
+import org.apache.cassandra.schema.CompactionParams;
+import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.FBUtilities;
+import org.checkerframework.checker.nullness.qual.Nullable;
+
+import static org.apache.cassandra.service.ActiveRepairService.NO_PENDING_REPAIR;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.fail;
+
+public class EntireSSTableStreamingCorrectFilesCountTest
+{
+ public static final String KEYSPACE = "EntireSSTableStreamingCorrectFilesCountTest";
+ public static final String CF_STANDARD = "Standard1";
+
+ private static SSTableReader sstable;
+ private static ColumnFamilyStore store;
+ private static RangesAtEndpoint rangesAtEndpoint;
+
+ @BeforeClass
+ public static void defineSchemaAndPrepareSSTable()
+ {
+ SchemaLoader.prepareServer();
+ SchemaLoader.createKeyspace(KEYSPACE,
+ KeyspaceParams.simple(1),
+ SchemaLoader.standardCFMD(KEYSPACE, CF_STANDARD)
+ // LeveledCompactionStrategy is important here,
+ // streaming of entire SSTables works currently only with this strategy
+ .compaction(CompactionParams.lcs(Collections.emptyMap()))
+ .partitioner(ByteOrderedPartitioner.instance));
+
+ Keyspace keyspace = Keyspace.open(KEYSPACE);
+ store = keyspace.getColumnFamilyStore(CF_STANDARD);
+
+ // insert data and compact to a single sstable
+ CompactionManager.instance.disableAutoCompaction();
+
+ for (int j = 0; j < 10; j++)
+ {
+ new RowUpdateBuilder(store.metadata(), j, String.valueOf(j))
+ .clustering("0")
+ .add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER)
+ .build()
+ .applyUnsafe();
+ }
+
+ store.forceBlockingFlush();
+ CompactionManager.instance.performMaximal(store, false);
+
+ sstable = store.getLiveSSTables().iterator().next();
+
+ Token start = ByteOrderedPartitioner.instance.getTokenFactory().fromString(Long.toHexString(0));
+ Token end = ByteOrderedPartitioner.instance.getTokenFactory().fromString(Long.toHexString(100));
+
+ rangesAtEndpoint = RangesAtEndpoint.toDummyList(Collections.singleton(new Range<>(start, end)));
+ }
+
+ @Test
+ public void test() throws Exception
+ {
+ FileCountingStreamEventHandler streamEventHandler = new FileCountingStreamEventHandler();
+ StreamSession session = setupStreamingSessionForTest(streamEventHandler);
+ Collection<OutgoingStream> outgoingStreams = store.getStreamManager().createOutgoingStreams(session,
+ rangesAtEndpoint,
+ NO_PENDING_REPAIR,
+ PreviewKind.NONE);
+
+ session.addTransferStreams(outgoingStreams);
+ DataOutputStreamPlus out = constructDataOutputStream();
+
+ for (OutgoingStream outgoingStream : outgoingStreams)
+ outgoingStream.write(session, out, MessagingService.VERSION_40);
+
+ int totalNumberOfFiles = session.transfers.get(store.metadata.id).getTotalNumberOfFiles();
+
+ assertEquals(CassandraOutgoingFile.getComponentManifest(sstable).components().size(), totalNumberOfFiles);
+ assertEquals(streamEventHandler.fileNames.size(), totalNumberOfFiles);
+ }
+
+ private DataOutputStreamPlus constructDataOutputStream()
+ {
+ // This is needed as Netty releases the ByteBuffers as soon as the channel is flushed
+ ByteBuf serializedFile = Unpooled.buffer(8192);
+ EmbeddedChannel channel = createMockNettyChannel(serializedFile);
+ return new AsyncStreamingOutputPlus(channel)
+ {
+ public void flush() throws IOException
+ {
+ // NO-OP
+ }
+ };
+ }
+
+ private EmbeddedChannel createMockNettyChannel(ByteBuf serializedFile)
+ {
+ WritableByteChannel wbc = new WritableByteChannel()
+ {
+ private boolean isOpen = true;
+
+ public int write(ByteBuffer src)
+ {
+ int size = src.limit();
+ serializedFile.writeBytes(src);
+ return size;
+ }
+
+ public boolean isOpen()
+ {
+ return isOpen;
+ }
+
+ public void close()
+ {
+ isOpen = false;
+ }
+ };
+
+ return new EmbeddedChannel(new ChannelOutboundHandlerAdapter()
+ {
+ @Override
+ public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise promise) throws Exception
+ {
+ ((SharedDefaultFileRegion) msg).transferTo(wbc, 0);
+ super.write(ctx, msg, promise);
+ }
+ });
+ }
+
+
+ private StreamSession setupStreamingSessionForTest(StreamEventHandler streamEventHandler)
+ {
+ StreamCoordinator streamCoordinator = new StreamCoordinator(StreamOperation.BOOTSTRAP,
+ 1,
+ new DefaultConnectionFactory(),
+ false,
+ false,
+ null,
+ PreviewKind.NONE);
+
+ StreamResultFuture future = StreamResultFuture.createInitiator(UUID.randomUUID(),
+ StreamOperation.BOOTSTRAP,
+ Collections.singleton(streamEventHandler),
+ streamCoordinator);
+
+ InetAddressAndPort peer = FBUtilities.getBroadcastAddressAndPort();
+ streamCoordinator.addSessionInfo(new SessionInfo(peer,
+ 0,
+ peer,
+ Collections.emptyList(),
+ Collections.emptyList(),
+ StreamSession.State.INITIALIZED));
+
+ StreamSession session = streamCoordinator.getOrCreateNextSession(peer);
+ session.init(future);
+
+ return session;
+ }
+
+ private static final class FileCountingStreamEventHandler implements StreamEventHandler
+ {
+ final Collection<String> fileNames = new ArrayList<>();
+
+ public void handleStreamEvent(StreamEvent event)
+ {
+ if (event.eventType == StreamEvent.Type.FILE_PROGRESS && event instanceof StreamEvent.ProgressEvent)
+ {
+ StreamEvent.ProgressEvent progressEvent = ((StreamEvent.ProgressEvent) event);
+ fileNames.add(progressEvent.progress.fileName);
+ }
+ }
+
+ public void onSuccess(@Nullable StreamState streamState)
+ {
+ assert streamState != null;
+ assertFalse(streamState.hasFailedSession());
+ }
+
+ public void onFailure(Throwable throwable)
+ {
+ fail();
+ }
+ }
+}
diff --git a/test/unit/org/apache/cassandra/streaming/SessionInfoTest.java b/test/unit/org/apache/cassandra/streaming/SessionInfoTest.java
index 37a6e41..4f0c494 100644
--- a/test/unit/org/apache/cassandra/streaming/SessionInfoTest.java
+++ b/test/unit/org/apache/cassandra/streaming/SessionInfoTest.java
@@ -17,43 +17,35 @@
*/
package org.apache.cassandra.streaming;
-import java.net.InetAddress;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
-import java.util.UUID;
import org.junit.Test;
-import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.utils.FBUtilities;
-import org.junit.BeforeClass;
public class SessionInfoTest
{
- @BeforeClass
- public static void beforeClass()
- {
- DatabaseDescriptor.daemonInitialization();
- }
-
/**
* Test if total numbers are collect
*/
@Test
public void testTotals()
{
- UUID cfId = UUID.randomUUID();
- InetAddress local = FBUtilities.getLocalAddress();
+ TableId tableId = TableId.generate();
+ InetAddressAndPort local = FBUtilities.getLocalAddressAndPort();
Collection<StreamSummary> summaries = new ArrayList<>();
for (int i = 0; i < 10; i++)
{
- StreamSummary summary = new StreamSummary(cfId, i, (i + 1) * 10);
+ StreamSummary summary = new StreamSummary(tableId, i, (i + 1) * 10);
summaries.add(summary);
}
- StreamSummary sending = new StreamSummary(cfId, 10, 100);
+ StreamSummary sending = new StreamSummary(tableId, 10, 100);
SessionInfo info = new SessionInfo(local, 0, local, summaries, Collections.singleton(sending), StreamSession.State.PREPARING);
assert info.getTotalFilesToReceive() == 45;
diff --git a/test/unit/org/apache/cassandra/streaming/StreamOperationTest.java b/test/unit/org/apache/cassandra/streaming/StreamOperationTest.java
new file mode 100644
index 0000000..2cc216e
--- /dev/null
+++ b/test/unit/org/apache/cassandra/streaming/StreamOperationTest.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.streaming;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+public class StreamOperationTest
+{
+ @Test
+ public void testSerialization()
+ {
+ // Unknown descriptions fall back to OTHER
+ assertEquals(StreamOperation.OTHER, StreamOperation.fromString("Foobar"));
+ assertEquals(StreamOperation.OTHER, StreamOperation.fromString("Other"));
+ assertEquals(StreamOperation.RESTORE_REPLICA_COUNT, StreamOperation.fromString("Restore replica count"));
+ assertEquals(StreamOperation.DECOMMISSION, StreamOperation.fromString("Unbootstrap"));
+ assertEquals(StreamOperation.RELOCATION, StreamOperation.fromString("Relocation"));
+ assertEquals(StreamOperation.BOOTSTRAP, StreamOperation.fromString("Bootstrap"));
+ assertEquals(StreamOperation.REBUILD, StreamOperation.fromString("Rebuild"));
+ assertEquals(StreamOperation.BULK_LOAD, StreamOperation.fromString("Bulk Load"));
+ assertEquals(StreamOperation.REPAIR, StreamOperation.fromString("Repair"));
+ // Test case insensivity
+ assertEquals(StreamOperation.REPAIR, StreamOperation.fromString("rEpair"));
+
+ // Test description
+ assertEquals("Repair", StreamOperation.REPAIR.getDescription());
+ assertEquals("Restore replica count", StreamOperation.RESTORE_REPLICA_COUNT.getDescription());
+
+ }
+}
diff --git a/test/unit/org/apache/cassandra/streaming/StreamTransferTaskTest.java b/test/unit/org/apache/cassandra/streaming/StreamTransferTaskTest.java
index 04be91a..0bf7f20 100644
--- a/test/unit/org/apache/cassandra/streaming/StreamTransferTaskTest.java
+++ b/test/unit/org/apache/cassandra/streaming/StreamTransferTaskTest.java
@@ -17,7 +17,6 @@
*/
package org.apache.cassandra.streaming;
-import java.net.InetAddress;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -32,16 +31,21 @@
import org.junit.After;
import org.junit.Test;
-import junit.framework.Assert;
+import io.netty.channel.embedded.EmbeddedChannel;
+import org.junit.Assert;
import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.concurrent.ScheduledExecutors;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.streaming.CassandraOutgoingFile;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.schema.KeyspaceParams;
-import org.apache.cassandra.streaming.messages.OutgoingFileMessage;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.streaming.messages.OutgoingStreamMessage;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.concurrent.Ref;
@@ -72,8 +76,8 @@
@Test
public void testScheduleTimeout() throws Exception
{
- InetAddress peer = FBUtilities.getBroadcastAddress();
- StreamSession session = new StreamSession(peer, peer, null, 0, true, false);
+ InetAddressAndPort peer = FBUtilities.getBroadcastAddressAndPort();
+ StreamSession session = new StreamSession(StreamOperation.BOOTSTRAP, peer, (template, messagingVersion) -> new EmbeddedChannel(), false, 0, UUID.randomUUID(), PreviewKind.ALL);
ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD);
// create two sstables
@@ -84,16 +88,18 @@
}
// create streaming task that streams those two sstables
- StreamTransferTask task = new StreamTransferTask(session, cfs.metadata.cfId);
+ session.state(StreamSession.State.PREPARING);
+ StreamTransferTask task = new StreamTransferTask(session, cfs.metadata.id);
for (SSTableReader sstable : cfs.getLiveSSTables())
{
List<Range<Token>> ranges = new ArrayList<>();
ranges.add(new Range<>(sstable.first.getToken(), sstable.last.getToken()));
- task.addTransferFile(sstable.selfRef(), 1, sstable.getPositionsForRanges(ranges), 0);
+ task.addTransferStream(new CassandraOutgoingFile(StreamOperation.BOOTSTRAP, sstable.selfRef(), sstable.getPositionsForRanges(ranges), ranges, 1));
}
- assertEquals(2, task.getTotalNumberOfFiles());
+ assertEquals(14, task.getTotalNumberOfFiles());
// if file sending completes before timeout then the task should be canceled.
+ session.state(StreamSession.State.STREAMING);
Future f = task.scheduleTimeout(0, 0, TimeUnit.NANOSECONDS);
f.get();
@@ -118,10 +124,10 @@
@Test
public void testFailSessionDuringTransferShouldNotReleaseReferences() throws Exception
{
- InetAddress peer = FBUtilities.getBroadcastAddress();
- StreamCoordinator streamCoordinator = new StreamCoordinator(1, true, false, null, false);
- StreamResultFuture future = StreamResultFuture.init(UUID.randomUUID(), "", Collections.<StreamEventHandler>emptyList(), streamCoordinator);
- StreamSession session = new StreamSession(peer, peer, null, 0, true, false);
+ InetAddressAndPort peer = FBUtilities.getBroadcastAddressAndPort();
+ StreamCoordinator streamCoordinator = new StreamCoordinator(StreamOperation.BOOTSTRAP, 1, new DefaultConnectionFactory(), false, false, null, PreviewKind.NONE);
+ StreamResultFuture future = StreamResultFuture.createInitiator(UUID.randomUUID(), StreamOperation.OTHER, Collections.<StreamEventHandler>emptyList(), streamCoordinator);
+ StreamSession session = new StreamSession(StreamOperation.BOOTSTRAP, peer, null, false, 0, null, PreviewKind.NONE);
session.init(future);
ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD);
@@ -133,7 +139,7 @@
}
// create streaming task that streams those two sstables
- StreamTransferTask task = new StreamTransferTask(session, cfs.metadata.cfId);
+ StreamTransferTask task = new StreamTransferTask(session, cfs.metadata.id);
List<Ref<SSTableReader>> refs = new ArrayList<>(cfs.getLiveSSTables().size());
for (SSTableReader sstable : cfs.getLiveSSTables())
{
@@ -141,24 +147,24 @@
ranges.add(new Range<>(sstable.first.getToken(), sstable.last.getToken()));
Ref<SSTableReader> ref = sstable.selfRef();
refs.add(ref);
- task.addTransferFile(ref, 1, sstable.getPositionsForRanges(ranges), 0);
+ task.addTransferStream(new CassandraOutgoingFile(StreamOperation.BOOTSTRAP, ref, sstable.getPositionsForRanges(ranges), ranges, 1));
}
- assertEquals(2, task.getTotalNumberOfFiles());
+ assertEquals(14, task.getTotalNumberOfFiles());
//add task to stream session, so it is aborted when stream session fails
- session.transfers.put(UUID.randomUUID(), task);
+ session.transfers.put(TableId.generate(), task);
//make a copy of outgoing file messages, since task is cleared when it's aborted
- Collection<OutgoingFileMessage> files = new LinkedList<>(task.files.values());
+ Collection<OutgoingStreamMessage> files = new LinkedList<>(task.streams.values());
//simulate start transfer
- for (OutgoingFileMessage file : files)
+ for (OutgoingStreamMessage file : files)
{
file.startTransfer();
}
//fail stream session mid-transfer
- session.onError(new Exception("Fake exception"));
+ session.onError(new Exception("Fake exception")).get(5, TimeUnit.SECONDS);
//make sure reference was not released
for (Ref<SSTableReader> ref : refs)
@@ -166,8 +172,18 @@
assertEquals(1, ref.globalCount());
}
+ //wait for stream to abort asynchronously
+ int tries = 10;
+ while (ScheduledExecutors.nonPeriodicTasks.getActiveCount() > 0)
+ {
+ if(tries < 1)
+ throw new RuntimeException("test did not complete in time");
+ Thread.sleep(10);
+ tries--;
+ }
+
//simulate finish transfer
- for (OutgoingFileMessage file : files)
+ for (OutgoingStreamMessage file : files)
{
file.finishTransfer();
}
diff --git a/test/unit/org/apache/cassandra/streaming/StreamingTransferTest.java b/test/unit/org/apache/cassandra/streaming/StreamingTransferTest.java
index 8f3061a..d88f379 100644
--- a/test/unit/org/apache/cassandra/streaming/StreamingTransferTest.java
+++ b/test/unit/org/apache/cassandra/streaming/StreamingTransferTest.java
@@ -17,7 +17,6 @@
*/
package org.apache.cassandra.streaming;
-import java.net.InetAddress;
import java.nio.ByteBuffer;
import java.util.*;
import java.util.concurrent.TimeUnit;
@@ -25,17 +24,22 @@
import com.google.common.collect.Iterables;
import com.google.common.util.concurrent.FutureCallback;
import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.MoreExecutors;
+
+import org.apache.cassandra.locator.RangesAtEndpoint;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.cassandra.OrderedJUnit4ClassRunner;
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.Util;
-import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.db.streaming.CassandraOutgoingFile;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.QueryProcessor;
import org.apache.cassandra.cql3.UntypedResultSet;
@@ -46,7 +50,6 @@
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
-import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.schema.KeyspaceParams;
import org.apache.cassandra.service.StorageService;
@@ -54,6 +57,9 @@
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.concurrent.Refs;
+import static org.apache.cassandra.SchemaLoader.compositeIndexCFMD;
+import static org.apache.cassandra.SchemaLoader.createKeyspace;
+import static org.apache.cassandra.SchemaLoader.standardCFMD;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
@@ -67,7 +73,7 @@
DatabaseDescriptor.daemonInitialization();
}
- public static final InetAddress LOCAL = FBUtilities.getBroadcastAddress();
+ public static final InetAddressAndPort LOCAL = FBUtilities.getBroadcastAddressAndPort();
public static final String KEYSPACE1 = "StreamingTransferTest1";
public static final String CF_STANDARD = "Standard1";
public static final String CF_COUNTER = "Counter1";
@@ -83,25 +89,26 @@
{
SchemaLoader.prepareServer();
StorageService.instance.initServer();
- SchemaLoader.createKeyspace(KEYSPACE1,
- KeyspaceParams.simple(1),
- SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD),
- CFMetaData.Builder.create(KEYSPACE1, CF_COUNTER, false, true, true)
- .addPartitionKey("key", BytesType.instance)
- .build(),
- CFMetaData.Builder.create(KEYSPACE1, CF_STANDARDINT)
- .addPartitionKey("key", AsciiType.instance)
- .addClusteringColumn("cols", Int32Type.instance)
- .addRegularColumn("val", BytesType.instance)
- .build(),
- SchemaLoader.compositeIndexCFMD(KEYSPACE1, CF_INDEX, true));
- SchemaLoader.createKeyspace(KEYSPACE2,
- KeyspaceParams.simple(1));
- SchemaLoader.createKeyspace(KEYSPACE_CACHEKEY,
- KeyspaceParams.simple(1),
- SchemaLoader.standardCFMD(KEYSPACE_CACHEKEY, CF_STANDARD),
- SchemaLoader.standardCFMD(KEYSPACE_CACHEKEY, CF_STANDARD2),
- SchemaLoader.standardCFMD(KEYSPACE_CACHEKEY, CF_STANDARD3));
+
+ createKeyspace(KEYSPACE1,
+ KeyspaceParams.simple(1),
+ standardCFMD(KEYSPACE1, CF_STANDARD),
+ TableMetadata.builder(KEYSPACE1, CF_COUNTER)
+ .isCounter(true)
+ .addPartitionKeyColumn("key", BytesType.instance),
+ TableMetadata.builder(KEYSPACE1, CF_STANDARDINT)
+ .addPartitionKeyColumn("key", AsciiType.instance)
+ .addClusteringColumn("cols", Int32Type.instance)
+ .addRegularColumn("val", BytesType.instance),
+ compositeIndexCFMD(KEYSPACE1, CF_INDEX, true));
+
+ createKeyspace(KEYSPACE2, KeyspaceParams.simple(1));
+
+ createKeyspace(KEYSPACE_CACHEKEY,
+ KeyspaceParams.simple(1),
+ standardCFMD(KEYSPACE_CACHEKEY, CF_STANDARD),
+ standardCFMD(KEYSPACE_CACHEKEY, CF_STANDARD2),
+ standardCFMD(KEYSPACE_CACHEKEY, CF_STANDARD3));
}
/**
@@ -110,14 +117,14 @@
@Test
public void testEmptyStreamPlan() throws Exception
{
- StreamResultFuture futureResult = new StreamPlan("StreamingTransferTest").execute();
+ StreamResultFuture futureResult = new StreamPlan(StreamOperation.OTHER).execute();
final UUID planId = futureResult.planId;
Futures.addCallback(futureResult, new FutureCallback<StreamState>()
{
public void onSuccess(StreamState result)
{
assert planId.equals(result.planId);
- assert result.description.equals("StreamingTransferTest");
+ assert result.streamOperation == StreamOperation.OTHER;
assert result.sessions.isEmpty();
}
@@ -125,7 +132,7 @@
{
fail();
}
- });
+ }, MoreExecutors.directExecutor());
// should be complete immediately
futureResult.get(100, TimeUnit.MILLISECONDS);
}
@@ -139,14 +146,14 @@
ranges.add(new Range<>(p.getMinimumToken(), p.getToken(ByteBufferUtil.bytes("key1"))));
ranges.add(new Range<>(p.getToken(ByteBufferUtil.bytes("key2")), p.getMinimumToken()));
- StreamResultFuture futureResult = new StreamPlan("StreamingTransferTest")
- .requestRanges(LOCAL, LOCAL, KEYSPACE2, ranges)
+ StreamResultFuture futureResult = new StreamPlan(StreamOperation.OTHER)
+ .requestRanges(LOCAL, KEYSPACE2, RangesAtEndpoint.toDummyList(ranges), RangesAtEndpoint.toDummyList(Collections.emptyList()))
.execute();
UUID planId = futureResult.planId;
StreamState result = futureResult.get();
assert planId.equals(result.planId);
- assert result.description.equals("StreamingTransferTest");
+ assert result.streamOperation == StreamOperation.OTHER;
// we should have completed session with empty transfer
assert result.sessions.size() == 1;
@@ -234,14 +241,13 @@
List<Range<Token>> ranges = new ArrayList<>();
// wrapped range
ranges.add(new Range<Token>(p.getToken(ByteBufferUtil.bytes("key1")), p.getToken(ByteBufferUtil.bytes("key0"))));
- StreamPlan streamPlan = new StreamPlan("StreamingTransferTest").transferRanges(LOCAL, cfs.keyspace.getName(), ranges, cfs.getColumnFamilyName());
+ StreamPlan streamPlan = new StreamPlan(StreamOperation.OTHER).transferRanges(LOCAL, cfs.keyspace.getName(), RangesAtEndpoint.toDummyList(ranges), cfs.getTableName());
streamPlan.execute().get();
- verifyConnectionsAreClosed();
//cannot add ranges after stream session is finished
try
{
- streamPlan.transferRanges(LOCAL, cfs.keyspace.getName(), ranges, cfs.getColumnFamilyName());
+ streamPlan.transferRanges(LOCAL, cfs.keyspace.getName(), RangesAtEndpoint.toDummyList(ranges), cfs.getTableName());
fail("Should have thrown exception");
}
catch (RuntimeException e)
@@ -252,14 +258,13 @@
private void transfer(SSTableReader sstable, List<Range<Token>> ranges) throws Exception
{
- StreamPlan streamPlan = new StreamPlan("StreamingTransferTest").transferFiles(LOCAL, makeStreamingDetails(ranges, Refs.tryRef(Arrays.asList(sstable))));
+ StreamPlan streamPlan = new StreamPlan(StreamOperation.OTHER).transferStreams(LOCAL, makeOutgoingStreams(ranges, Refs.tryRef(Arrays.asList(sstable))));
streamPlan.execute().get();
- verifyConnectionsAreClosed();
//cannot add files after stream session is finished
try
{
- streamPlan.transferFiles(LOCAL, makeStreamingDetails(ranges, Refs.tryRef(Arrays.asList(sstable))));
+ streamPlan.transferStreams(LOCAL, makeOutgoingStreams(ranges, Refs.tryRef(Arrays.asList(sstable))));
fail("Should have thrown exception");
}
catch (RuntimeException e)
@@ -268,36 +273,23 @@
}
}
- /**
- * Test that finished incoming connections are removed from MessagingService (CASSANDRA-11854)
- */
- private void verifyConnectionsAreClosed() throws InterruptedException
+ private Collection<OutgoingStream> makeOutgoingStreams(StreamOperation operation, List<Range<Token>> ranges, Refs<SSTableReader> sstables)
{
- //after stream session is finished, message handlers may take several milliseconds to be closed
- outer:
- for (int i = 0; i <= 100; i++)
- {
- for (MessagingService.SocketThread socketThread : MessagingService.instance().getSocketThreads())
- if (!socketThread.connections.isEmpty())
- {
- Thread.sleep(100);
- continue outer;
- }
- return;
- }
- fail("Streaming connections remain registered in MessagingService");
- }
-
- private Collection<StreamSession.SSTableStreamingSections> makeStreamingDetails(List<Range<Token>> ranges, Refs<SSTableReader> sstables)
- {
- ArrayList<StreamSession.SSTableStreamingSections> details = new ArrayList<>();
+ ArrayList<OutgoingStream> streams = new ArrayList<>();
for (SSTableReader sstable : sstables)
{
- details.add(new StreamSession.SSTableStreamingSections(sstables.get(sstable),
- sstable.getPositionsForRanges(ranges),
- sstable.estimatedKeysForRanges(ranges), sstable.getSSTableMetadata().repairedAt));
+ streams.add(new CassandraOutgoingFile(operation,
+ sstables.get(sstable),
+ sstable.getPositionsForRanges(ranges),
+ ranges,
+ sstable.estimatedKeysForRanges(ranges)));
}
- return details;
+ return streams;
+ }
+
+ private Collection<OutgoingStream> makeOutgoingStreams(List<Range<Token>> ranges, Refs<SSTableReader> sstables)
+ {
+ return makeOutgoingStreams(StreamOperation.OTHER, ranges, sstables);
}
private void doTransferTable(boolean transferSSTables) throws Exception
@@ -311,7 +303,7 @@
{
long val = key.hashCode();
- RowUpdateBuilder builder = new RowUpdateBuilder(cfs.metadata, timestamp, key);
+ RowUpdateBuilder builder = new RowUpdateBuilder(cfs.metadata(), timestamp, key);
builder.clustering(col).add("birthdate", ByteBufferUtil.bytes(val));
builder.build().applyUnsafe();
}
@@ -324,7 +316,7 @@
// test we can search:
UntypedResultSet result = QueryProcessor.executeInternal(String.format("SELECT * FROM \"%s\".\"%s\" WHERE birthdate = %d",
- cfs.metadata.ksName, cfs.metadata.cfName, val));
+ cfs.metadata.keyspace, cfs.metadata.name, val));
assertEquals(1, result.size());
assert result.iterator().next().getBytes("key").equals(ByteBufferUtil.bytes(key));
@@ -346,7 +338,7 @@
String key = "key1";
- RowUpdateBuilder updates = new RowUpdateBuilder(cfs.metadata, FBUtilities.timestampMicros(), key);
+ RowUpdateBuilder updates = new RowUpdateBuilder(cfs.metadata(), FBUtilities.timestampMicros(), key);
// add columns of size slightly less than column_index_size to force insert column index
updates.clustering(1)
@@ -354,7 +346,7 @@
.build()
.apply();
- updates = new RowUpdateBuilder(cfs.metadata, FBUtilities.timestampMicros(), key);
+ updates = new RowUpdateBuilder(cfs.metadata(), FBUtilities.timestampMicros(), key);
updates.clustering(6)
.add("val", ByteBuffer.wrap(new byte[DatabaseDescriptor.getColumnIndexSize()]))
.build()
@@ -367,7 +359,7 @@
// .apply();
- updates = new RowUpdateBuilder(cfs.metadata, FBUtilities.timestampMicros() + 1, key);
+ updates = new RowUpdateBuilder(cfs.metadata(), FBUtilities.timestampMicros() + 1, key);
updates.addRangeTombstone(5, 7)
.build()
.apply();
@@ -477,7 +469,7 @@
// Acquiring references, transferSSTables needs it
Refs<SSTableReader> refs = Refs.tryRef(Arrays.asList(sstable, sstable2));
assert refs != null;
- new StreamPlan("StreamingTransferTest").transferFiles(LOCAL, makeStreamingDetails(ranges, refs)).execute().get();
+ new StreamPlan("StreamingTransferTest").transferStreams(LOCAL, makeOutgoingStreams(ranges, refs)).execute().get();
// confirm that the sstables were transferred and registered and that 2 keys arrived
ColumnFamilyStore cfstore = Keyspace.open(keyspaceName).getColumnFamilyStore(cfname);
@@ -532,7 +524,7 @@
if (refs == null)
throw new AssertionError();
- new StreamPlan("StreamingTransferTest").transferFiles(LOCAL, makeStreamingDetails(ranges, refs)).execute().get();
+ new StreamPlan("StreamingTransferTest").transferStreams(LOCAL, makeOutgoingStreams(ranges, refs)).execute().get();
// check that only two keys were transferred
for (Map.Entry<DecoratedKey,String> entry : Arrays.asList(first, last))
diff --git a/test/unit/org/apache/cassandra/streaming/async/NettyStreamingMessageSenderTest.java b/test/unit/org/apache/cassandra/streaming/async/NettyStreamingMessageSenderTest.java
new file mode 100644
index 0000000..76bfa76
--- /dev/null
+++ b/test/unit/org/apache/cassandra/streaming/async/NettyStreamingMessageSenderTest.java
@@ -0,0 +1,205 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.streaming.async;
+
+import java.util.UUID;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import com.google.common.net.InetAddresses;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import io.netty.channel.ChannelPromise;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.net.TestChannel;
+import org.apache.cassandra.net.TestScheduledFuture;
+import org.apache.cassandra.streaming.PreviewKind;
+import org.apache.cassandra.streaming.StreamOperation;
+import org.apache.cassandra.streaming.StreamResultFuture;
+import org.apache.cassandra.streaming.StreamSession;
+import org.apache.cassandra.streaming.messages.CompleteMessage;
+
+public class NettyStreamingMessageSenderTest
+{
+ private static final InetAddressAndPort REMOTE_ADDR = InetAddressAndPort.getByAddressOverrideDefaults(InetAddresses.forString("127.0.0.2"), 0);
+
+ private TestChannel channel;
+ private StreamSession session;
+ private NettyStreamingMessageSender sender;
+ private NettyStreamingMessageSender.FileStreamTask fileStreamTask;
+
+ @BeforeClass
+ public static void before()
+ {
+ DatabaseDescriptor.daemonInitialization();
+ }
+
+ @Before
+ public void setUp()
+ {
+ channel = new TestChannel(Integer.MAX_VALUE);
+ channel.attr(NettyStreamingMessageSender.TRANSFERRING_FILE_ATTR).set(Boolean.FALSE);
+ UUID pendingRepair = UUID.randomUUID();
+ session = new StreamSession(StreamOperation.BOOTSTRAP, REMOTE_ADDR, (template, messagingVersion) -> null, true, 0, pendingRepair, PreviewKind.ALL);
+ StreamResultFuture future = StreamResultFuture.createFollower(0, UUID.randomUUID(), StreamOperation.REPAIR, REMOTE_ADDR, channel, pendingRepair, session.getPreviewKind());
+ session.init(future);
+ session.attachOutbound(channel);
+
+ sender = session.getMessageSender();
+ sender.setControlMessageChannel(channel);
+ }
+
+ @After
+ public void tearDown()
+ {
+ if (fileStreamTask != null)
+ fileStreamTask.unsetChannel();
+ }
+
+ @Test
+ public void KeepAliveTask_normalSend()
+ {
+ Assert.assertTrue(channel.isOpen());
+ NettyStreamingMessageSender.KeepAliveTask task = sender.new KeepAliveTask(channel, session);
+ task.run();
+ Assert.assertTrue(channel.releaseOutbound());
+ }
+
+ @Test
+ public void KeepAliveTask_channelClosed()
+ {
+ channel.close();
+ Assert.assertFalse(channel.isOpen());
+ channel.releaseOutbound();
+ NettyStreamingMessageSender.KeepAliveTask task = sender.new KeepAliveTask(channel, session);
+ task.future = new TestScheduledFuture();
+ Assert.assertFalse(task.future.isCancelled());
+ task.run();
+ Assert.assertTrue(task.future.isCancelled());
+ Assert.assertFalse(channel.releaseOutbound());
+ }
+
+ @Test
+ public void KeepAliveTask_closed()
+ {
+ Assert.assertTrue(channel.isOpen());
+ NettyStreamingMessageSender.KeepAliveTask task = sender.new KeepAliveTask(channel, session);
+ task.future = new TestScheduledFuture();
+ Assert.assertFalse(task.future.isCancelled());
+
+ sender.setClosed();
+ Assert.assertFalse(sender.connected());
+ task.run();
+ Assert.assertTrue(task.future.isCancelled());
+ Assert.assertFalse(channel.releaseOutbound());
+ }
+
+ @Test
+ public void KeepAliveTask_CurrentlyStreaming()
+ {
+ Assert.assertTrue(channel.isOpen());
+ channel.attr(NettyStreamingMessageSender.TRANSFERRING_FILE_ATTR).set(Boolean.TRUE);
+ NettyStreamingMessageSender.KeepAliveTask task = sender.new KeepAliveTask(channel, session);
+ task.future = new TestScheduledFuture();
+ Assert.assertFalse(task.future.isCancelled());
+
+ Assert.assertTrue(sender.connected());
+ task.run();
+ Assert.assertFalse(task.future.isCancelled());
+ Assert.assertFalse(channel.releaseOutbound());
+ }
+
+ @Test
+ public void FileStreamTask_acquirePermit_closed()
+ {
+ fileStreamTask = sender.new FileStreamTask(null);
+ sender.setClosed();
+ Assert.assertFalse(fileStreamTask.acquirePermit(1));
+ }
+
+ @Test
+ public void FileStreamTask_acquirePermit_HapppyPath()
+ {
+ int permits = sender.semaphoreAvailablePermits();
+ fileStreamTask = sender.new FileStreamTask(null);
+ Assert.assertTrue(fileStreamTask.acquirePermit(1));
+ Assert.assertEquals(permits - 1, sender.semaphoreAvailablePermits());
+ }
+
+ @Test
+ public void FileStreamTask_BadChannelAttr()
+ {
+ int permits = sender.semaphoreAvailablePermits();
+ channel.attr(NettyStreamingMessageSender.TRANSFERRING_FILE_ATTR).set(Boolean.TRUE);
+ fileStreamTask = sender.new FileStreamTask(null);
+ fileStreamTask.injectChannel(channel);
+ fileStreamTask.run();
+ Assert.assertEquals(StreamSession.State.FAILED, session.state());
+ Assert.assertTrue(channel.releaseOutbound()); // when the session fails, it will send a SessionFailed msg
+ Assert.assertEquals(permits, sender.semaphoreAvailablePermits());
+ }
+
+ @Test
+ public void FileStreamTask_HappyPath()
+ {
+ int permits = sender.semaphoreAvailablePermits();
+ fileStreamTask = sender.new FileStreamTask(new CompleteMessage());
+ fileStreamTask.injectChannel(channel);
+ fileStreamTask.run();
+ Assert.assertNotEquals(StreamSession.State.FAILED, session.state());
+ Assert.assertTrue(channel.releaseOutbound());
+ Assert.assertEquals(permits, sender.semaphoreAvailablePermits());
+ }
+
+ @Test
+ public void onControlMessageComplete_HappyPath()
+ {
+ Assert.assertTrue(channel.isOpen());
+ Assert.assertTrue(sender.connected());
+ ChannelPromise promise = channel.newPromise();
+ promise.setSuccess();
+ Assert.assertNull(sender.onControlMessageComplete(promise, new CompleteMessage()));
+ Assert.assertTrue(channel.isOpen());
+ Assert.assertTrue(sender.connected());
+ Assert.assertNotEquals(StreamSession.State.FAILED, session.state());
+ }
+
+ @Test
+ public void onControlMessageComplete_Exception() throws InterruptedException, ExecutionException, TimeoutException
+ {
+ Assert.assertTrue(channel.isOpen());
+ Assert.assertTrue(sender.connected());
+ ChannelPromise promise = channel.newPromise();
+ promise.setFailure(new RuntimeException("this is just a testing exception"));
+ Future f = sender.onControlMessageComplete(promise, new CompleteMessage());
+
+ f.get(5, TimeUnit.SECONDS);
+
+ Assert.assertFalse(channel.isOpen());
+ Assert.assertFalse(sender.connected());
+ Assert.assertEquals(StreamSession.State.FAILED, session.state());
+ }
+}
diff --git a/test/unit/org/apache/cassandra/streaming/async/StreamCompressionSerializerTest.java b/test/unit/org/apache/cassandra/streaming/async/StreamCompressionSerializerTest.java
new file mode 100644
index 0000000..dab6001
--- /dev/null
+++ b/test/unit/org/apache/cassandra/streaming/async/StreamCompressionSerializerTest.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.streaming.async;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.ReadableByteChannel;
+import java.util.Random;
+
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.ByteBufAllocator;
+import io.netty.buffer.PooledByteBufAllocator;
+import io.netty.buffer.Unpooled;
+import net.jpountz.lz4.LZ4Compressor;
+import net.jpountz.lz4.LZ4Factory;
+import net.jpountz.lz4.LZ4SafeDecompressor;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.io.util.DataInputBuffer;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.net.MessagingService;
+
+public class StreamCompressionSerializerTest
+{
+ private static final int VERSION = MessagingService.current_version;
+ private static final Random random = new Random(2347623847623L);
+
+ private final ByteBufAllocator allocator = PooledByteBufAllocator.DEFAULT;
+ private final StreamCompressionSerializer serializer = new StreamCompressionSerializer(allocator);
+ private final LZ4Compressor compressor = LZ4Factory.fastestInstance().fastCompressor();
+ private final LZ4SafeDecompressor decompressor = LZ4Factory.fastestInstance().safeDecompressor();
+
+ private ByteBuffer input;
+ private ByteBuffer compressed;
+ private ByteBuf output;
+
+ @BeforeClass
+ public static void before()
+ {
+ DatabaseDescriptor.daemonInitialization();
+ }
+
+ @After
+ public void tearDown()
+ {
+ if (input != null)
+ FileUtils.clean(input);
+ if (compressed != null)
+ FileUtils.clean(compressed);
+ if (output != null && output.refCnt() > 0)
+ output.release(output.refCnt());
+ }
+
+ @Test
+ public void roundTrip_HappyPath_NotReadabaleByteBuffer() throws IOException
+ {
+ populateInput();
+ StreamCompressionSerializer.serialize(compressor, input, VERSION).write(size -> compressed = ByteBuffer.allocateDirect(size));
+ input.flip();
+ output = serializer.deserialize(decompressor, new DataInputBuffer(compressed, false), VERSION);
+ validateResults();
+ }
+
+ private void populateInput()
+ {
+ int bufSize = 1 << 14;
+ input = ByteBuffer.allocateDirect(bufSize);
+ for (int i = 0; i < bufSize; i += 4)
+ input.putInt(random.nextInt());
+ input.flip();
+ }
+
+ private void validateResults()
+ {
+ Assert.assertEquals(input.remaining(), output.readableBytes());
+ for (int i = 0; i < input.remaining(); i++)
+ Assert.assertEquals(input.get(i), output.readByte());
+ }
+
+ @Test
+ public void roundTrip_HappyPath_ReadabaleByteBuffer() throws IOException
+ {
+ populateInput();
+ StreamCompressionSerializer.serialize(compressor, input, VERSION)
+ .write(size -> {
+ if (compressed != null)
+ FileUtils.clean(compressed);
+ return compressed = ByteBuffer.allocateDirect(size);
+ });
+ input.flip();
+ output = serializer.deserialize(decompressor, new ByteBufRCH(Unpooled.wrappedBuffer(compressed)), VERSION);
+ validateResults();
+ }
+
+ private static class ByteBufRCH extends DataInputBuffer implements ReadableByteChannel
+ {
+ public ByteBufRCH(ByteBuf compressed)
+ {
+ super (compressed.nioBuffer(0, compressed.readableBytes()), false);
+ }
+
+ @Override
+ public int read(ByteBuffer dst) throws IOException
+ {
+ int len = dst.remaining();
+ dst.put(buffer);
+ return len;
+ }
+
+ @Override
+ public boolean isOpen()
+ {
+ return true;
+ }
+ }
+}
diff --git a/test/unit/org/apache/cassandra/streaming/async/StreamingInboundHandlerTest.java b/test/unit/org/apache/cassandra/streaming/async/StreamingInboundHandlerTest.java
new file mode 100644
index 0000000..d573a15
--- /dev/null
+++ b/test/unit/org/apache/cassandra/streaming/async/StreamingInboundHandlerTest.java
@@ -0,0 +1,173 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.streaming.async;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.UUID;
+
+import com.google.common.net.InetAddresses;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.channel.embedded.EmbeddedChannel;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.io.util.DataInputBuffer;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputBuffer;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.net.AsyncStreamingInputPlus;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.streaming.PreviewKind;
+import org.apache.cassandra.streaming.StreamManager;
+import org.apache.cassandra.streaming.StreamOperation;
+import org.apache.cassandra.streaming.StreamResultFuture;
+import org.apache.cassandra.streaming.StreamSession;
+import org.apache.cassandra.streaming.messages.CompleteMessage;
+import org.apache.cassandra.streaming.messages.IncomingStreamMessage;
+import org.apache.cassandra.streaming.messages.StreamInitMessage;
+import org.apache.cassandra.streaming.messages.StreamMessageHeader;
+
+public class StreamingInboundHandlerTest
+{
+ private static final int VERSION = MessagingService.current_version;
+ private static final InetAddressAndPort REMOTE_ADDR = InetAddressAndPort.getByAddressOverrideDefaults(InetAddresses.forString("127.0.0.2"), 0);
+
+ private StreamingInboundHandler handler;
+ private EmbeddedChannel channel;
+ private AsyncStreamingInputPlus buffers;
+ private ByteBuf buf;
+
+ @BeforeClass
+ public static void before()
+ {
+ DatabaseDescriptor.daemonInitialization();
+ }
+
+ @Before
+ public void setup()
+ {
+ handler = new StreamingInboundHandler(REMOTE_ADDR, VERSION, null);
+ channel = new EmbeddedChannel(handler);
+ buffers = new AsyncStreamingInputPlus(channel);
+ handler.setPendingBuffers(buffers);
+ }
+
+ @After
+ public void tearDown()
+ {
+ if (buf != null)
+ {
+ while (buf.refCnt() > 0)
+ buf.release();
+ }
+
+ channel.close();
+ }
+
+ @Test
+ public void channelRead_Normal()
+ {
+ Assert.assertEquals(0, buffers.unsafeAvailable());
+ int size = 8;
+ buf = channel.alloc().buffer(size);
+ buf.writerIndex(size);
+ channel.writeInbound(buf);
+ Assert.assertEquals(size, buffers.unsafeAvailable());
+ Assert.assertFalse(channel.releaseInbound());
+ }
+
+ @Test
+ public void channelRead_Closed()
+ {
+ int size = 8;
+ buf = channel.alloc().buffer(size);
+ Assert.assertEquals(1, buf.refCnt());
+ buf.writerIndex(size);
+ handler.close();
+ channel.writeInbound(buf);
+ Assert.assertEquals(0, buffers.unsafeAvailable());
+ Assert.assertEquals(0, buf.refCnt());
+ Assert.assertFalse(channel.releaseInbound());
+ }
+
+ @Test
+ public void channelRead_WrongObject()
+ {
+ channel.writeInbound("homer");
+ Assert.assertEquals(0, buffers.unsafeAvailable());
+ Assert.assertFalse(channel.releaseInbound());
+ }
+
+ @Test
+ public void StreamDeserializingTask_deriveSession_StreamInitMessage()
+ {
+ StreamInitMessage msg = new StreamInitMessage(REMOTE_ADDR, 0, UUID.randomUUID(), StreamOperation.REPAIR, UUID.randomUUID(), PreviewKind.ALL);
+ StreamingInboundHandler.StreamDeserializingTask task = handler.new StreamDeserializingTask(null, channel);
+ StreamSession session = task.deriveSession(msg);
+ Assert.assertNotNull(session);
+ }
+
+ @Test (expected = UnsupportedOperationException.class)
+ public void StreamDeserializingTask_deriveSession_NoSession()
+ {
+ CompleteMessage msg = new CompleteMessage();
+ StreamingInboundHandler.StreamDeserializingTask task = handler.new StreamDeserializingTask(null, channel);
+ task.deriveSession(msg);
+ }
+
+ @Test (expected = IllegalStateException.class)
+ public void StreamDeserializingTask_deserialize_ISM_NoSession() throws IOException
+ {
+ StreamMessageHeader header = new StreamMessageHeader(TableId.generate(), REMOTE_ADDR, UUID.randomUUID(), true,
+ 0, 0, 0, UUID.randomUUID());
+
+ ByteBuffer temp = ByteBuffer.allocate(1024);
+ DataOutputPlus out = new DataOutputBuffer(temp);
+ StreamMessageHeader.serializer.serialize(header, out, MessagingService.current_version);
+
+ temp.flip();
+ DataInputPlus in = new DataInputBuffer(temp, false);
+ // session not found
+ IncomingStreamMessage.serializer.deserialize(in, MessagingService.current_version);
+ }
+
+ @Test
+ public void StreamDeserializingTask_deserialize_ISM_HasSession()
+ {
+ UUID planId = UUID.randomUUID();
+ StreamResultFuture future = StreamResultFuture.createFollower(0, planId, StreamOperation.REPAIR, REMOTE_ADDR, channel, UUID.randomUUID(), PreviewKind.ALL);
+ StreamManager.instance.registerFollower(future);
+ StreamMessageHeader header = new StreamMessageHeader(TableId.generate(), REMOTE_ADDR, planId, false,
+ 0, 0, 0, UUID.randomUUID());
+
+ // IncomingStreamMessage.serializer.deserialize
+ StreamSession session = StreamManager.instance.findSession(header.sender, header.planId, header.sessionIndex, header.sendByFollower);
+ Assert.assertNotNull(session);
+
+ session = StreamManager.instance.findSession(header.sender, header.planId, header.sessionIndex, !header.sendByFollower);
+ Assert.assertNull(session);
+ }
+}
diff --git a/test/unit/org/apache/cassandra/streaming/compression/CompressedInputStreamTest.java b/test/unit/org/apache/cassandra/streaming/compression/CompressedInputStreamTest.java
index c0fc277..be443b5 100644
--- a/test/unit/org/apache/cassandra/streaming/compression/CompressedInputStreamTest.java
+++ b/test/unit/org/apache/cassandra/streaming/compression/CompressedInputStreamTest.java
@@ -21,22 +21,26 @@
import java.util.*;
import org.junit.BeforeClass;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.ClusteringComparator;
import org.apache.cassandra.db.marshal.BytesType;
import org.apache.cassandra.io.compress.CompressedSequentialWriter;
import org.apache.cassandra.io.compress.CompressionMetadata;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.util.DataInputPlus.DataInputStreamPlus;
+import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.io.util.SequentialWriterOption;
import org.apache.cassandra.schema.CompressionParams;
import org.apache.cassandra.io.sstable.Component;
import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
-import org.apache.cassandra.streaming.compress.CompressedInputStream;
-import org.apache.cassandra.streaming.compress.CompressionInfo;
+import org.apache.cassandra.db.streaming.CompressedInputStream;
+import org.apache.cassandra.db.streaming.CompressionInfo;
import org.apache.cassandra.utils.ChecksumType;
-import org.apache.cassandra.utils.Pair;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
@@ -45,6 +49,9 @@
*/
public class CompressedInputStreamTest
{
+ @Rule
+ public TemporaryFolder tempFolder = new TemporaryFolder();
+
@BeforeClass
public static void setupDD()
{
@@ -54,17 +61,17 @@
@Test
public void testCompressedRead() throws Exception
{
- testCompressedReadWith(new long[]{0L}, false, false);
- testCompressedReadWith(new long[]{1L}, false, false);
- testCompressedReadWith(new long[]{100L}, false, false);
+ testCompressedReadWith(new long[]{0L}, false, false, 0);
+ testCompressedReadWith(new long[]{1L}, false, false, 0);
+ testCompressedReadWith(new long[]{100L}, false, false, 0);
- testCompressedReadWith(new long[]{1L, 122L, 123L, 124L, 456L}, false, false);
+ testCompressedReadWith(new long[]{1L, 122L, 123L, 124L, 456L}, false, false, 0);
}
@Test(expected = EOFException.class)
public void testTruncatedRead() throws Exception
{
- testCompressedReadWith(new long[]{1L, 122L, 123L, 124L, 456L}, true, false);
+ testCompressedReadWith(new long[]{1L, 122L, 123L, 124L, 456L}, true, false, 0);
}
/**
@@ -73,22 +80,45 @@
@Test(timeout = 30000)
public void testException() throws Exception
{
- testCompressedReadWith(new long[]{1L, 122L, 123L, 124L, 456L}, false, true);
+ testCompressedReadWith(new long[]{1L, 122L, 123L, 124L, 456L}, false, true, 0);
+ }
+
+ @Test
+ public void testCompressedReadUncompressedChunks() throws Exception
+ {
+ testCompressedReadWith(new long[]{0L}, false, false, 3);
+ testCompressedReadWith(new long[]{1L}, false, false, 3);
+ testCompressedReadWith(new long[]{100L}, false, false, 3);
+
+ testCompressedReadWith(new long[]{1L, 122L, 123L, 124L, 456L}, false, false, 3);
+ }
+
+ @Test(expected = EOFException.class)
+ public void testTruncatedReadUncompressedChunks() throws Exception
+ {
+ testCompressedReadWith(new long[]{1L, 122L, 123L, 124L, 456L}, true, false, 3);
+ }
+
+ @Test(timeout = 30000)
+ public void testCorruptedReadUncompressedChunks() throws Exception
+ {
+ testCompressedReadWith(new long[]{1L, 122L, 123L, 124L, 456L}, false, true, 3);
}
/**
* @param valuesToCheck array of longs of range(0-999)
* @throws Exception
*/
- private void testCompressedReadWith(long[] valuesToCheck, boolean testTruncate, boolean testException) throws Exception
+ private void testCompressedReadWith(long[] valuesToCheck, boolean testTruncate, boolean testException, double minCompressRatio) throws Exception
{
assert valuesToCheck != null && valuesToCheck.length > 0;
// write compressed data file of longs
- File tmp = new File(File.createTempFile("cassandra", "unittest").getParent(), "ks-cf-ib-1-Data.db");
- Descriptor desc = Descriptor.fromFilename(tmp.getAbsolutePath());
+ File parentDir = tempFolder.newFolder();
+ Descriptor desc = new Descriptor(parentDir, "ks", "cf", 1);
+ File tmp = new File(desc.filenameFor(Component.DATA));
MetadataCollector collector = new MetadataCollector(new ClusteringComparator(BytesType.instance));
- CompressionParams param = CompressionParams.snappy(32);
+ CompressionParams param = CompressionParams.snappy(32, minCompressRatio);
Map<Long, Long> index = new HashMap<Long, Long>();
try (CompressedSequentialWriter writer = new CompressedSequentialWriter(tmp,
desc.filenameFor(Component.COMPRESSION_INFO),
@@ -105,11 +135,11 @@
}
CompressionMetadata comp = CompressionMetadata.create(tmp.getAbsolutePath());
- List<Pair<Long, Long>> sections = new ArrayList<>();
+ List<SSTableReader.PartitionPositionBounds> sections = new ArrayList<>();
for (long l : valuesToCheck)
{
long position = index.get(l);
- sections.add(Pair.create(position, position + 8));
+ sections.add(new SSTableReader.PartitionPositionBounds(position, position + 8));
}
CompressionMetadata.Chunk[] chunks = comp.getChunksForSections(sections);
long totalSize = comp.getTotalSizeForSections(sections);
@@ -149,29 +179,29 @@
testException(sections, info);
return;
}
- CompressedInputStream input = new CompressedInputStream(new ByteArrayInputStream(toRead), info, ChecksumType.CRC32, () -> 1.0);
+ CompressedInputStream input = new CompressedInputStream(new DataInputStreamPlus(new ByteArrayInputStream(toRead)), info, ChecksumType.CRC32, () -> 1.0);
try (DataInputStream in = new DataInputStream(input))
{
for (int i = 0; i < sections.size(); i++)
{
- input.position(sections.get(i).left);
+ input.position(sections.get(i).lowerPosition);
long readValue = in.readLong();
assertEquals("expected " + valuesToCheck[i] + " but was " + readValue, valuesToCheck[i], readValue);
}
}
}
- private static void testException(List<Pair<Long, Long>> sections, CompressionInfo info) throws IOException
+ private static void testException(List<SSTableReader.PartitionPositionBounds> sections, CompressionInfo info) throws IOException
{
- CompressedInputStream input = new CompressedInputStream(new ByteArrayInputStream(new byte[0]), info, ChecksumType.CRC32, () -> 1.0);
+ CompressedInputStream input = new CompressedInputStream(new DataInputStreamPlus(new ByteArrayInputStream(new byte[0])), info, ChecksumType.CRC32, () -> 1.0);
try (DataInputStream in = new DataInputStream(input))
{
for (int i = 0; i < sections.size(); i++)
{
- input.position(sections.get(i).left);
try {
+ input.position(sections.get(i).lowerPosition);
in.readLong();
fail("Should have thrown IOException");
}
@@ -183,3 +213,4 @@
}
}
}
+
diff --git a/test/unit/org/apache/cassandra/tools/AuditLogViewerTest.java b/test/unit/org/apache/cassandra/tools/AuditLogViewerTest.java
new file mode 100644
index 0000000..649712a
--- /dev/null
+++ b/test/unit/org/apache/cassandra/tools/AuditLogViewerTest.java
@@ -0,0 +1,214 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.tools;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.commons.io.FileUtils;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import net.openhft.chronicle.core.io.IORuntimeException;
+import net.openhft.chronicle.queue.ChronicleQueue;
+import net.openhft.chronicle.queue.ChronicleQueueBuilder;
+import net.openhft.chronicle.queue.ExcerptAppender;
+import net.openhft.chronicle.queue.RollCycles;
+import net.openhft.chronicle.wire.WireOut;
+import org.apache.cassandra.audit.BinAuditLogger;
+
+import static org.junit.Assert.assertTrue;
+
+public class AuditLogViewerTest
+{
+ private Path path;
+
+ @Before
+ public void setUp() throws IOException
+ {
+ path = Files.createTempDirectory("foo");
+ }
+
+ @After
+ public void tearDown() throws IOException
+ {
+ if (path.toFile().exists() && path.toFile().isDirectory())
+ {
+ //Deletes directory and all of it's contents
+ FileUtils.deleteDirectory(path.toFile());
+ }
+ }
+
+ @Test
+ public void testDisplayRecord()
+ {
+ List<String> records = new ArrayList<>();
+ records.add("Test foo bar 1");
+ records.add("Test foo bar 2");
+
+ try (ChronicleQueue queue = ChronicleQueueBuilder.single(path.toFile()).rollCycle(RollCycles.TEST_SECONDLY).build())
+ {
+ ExcerptAppender appender = queue.acquireAppender();
+
+ //Write bunch of records
+ records.forEach(s -> appender.writeDocument(new BinAuditLogger.Message(s)));
+
+ //Read those written records
+ List<String> actualRecords = new ArrayList<>();
+ AuditLogViewer.dump(ImmutableList.of(path.toString()), RollCycles.TEST_SECONDLY.toString(), false, false, actualRecords::add);
+
+ assertRecordsMatch(records, actualRecords);
+ }
+ }
+
+ @Test (expected = IORuntimeException.class)
+ public void testRejectFutureVersionRecord()
+ {
+ try (ChronicleQueue queue = ChronicleQueueBuilder.single(path.toFile()).rollCycle(RollCycles.TEST_SECONDLY).build())
+ {
+ ExcerptAppender appender = queue.acquireAppender();
+ appender.writeDocument(createFutureRecord());
+
+ AuditLogViewer.dump(ImmutableList.of(path.toString()), RollCycles.TEST_SECONDLY.toString(), false, false, dummy -> {});
+ }
+ catch (Exception e)
+ {
+ assertTrue(e.getMessage().contains("Unsupported record version"));
+ throw e;
+ }
+ }
+
+ @Test
+ public void testIgnoreFutureVersionRecord()
+ {
+ List<String> records = new ArrayList<>();
+ records.add("Test foo bar 1");
+ records.add("Test foo bar 2");
+
+ try (ChronicleQueue queue = ChronicleQueueBuilder.single(path.toFile()).rollCycle(RollCycles.TEST_SECONDLY).build())
+ {
+ ExcerptAppender appender = queue.acquireAppender();
+
+ //Write future record
+ appender.writeDocument(createFutureRecord());
+
+ //Write bunch of current records
+ records.forEach(s -> appender.writeDocument(new BinAuditLogger.Message(s)));
+
+ //Read those written records
+ List<String> actualRecords = new ArrayList<>();
+ AuditLogViewer.dump(ImmutableList.of(path.toString()), RollCycles.TEST_SECONDLY.toString(), false, true, actualRecords::add);
+
+ // Assert all current records are present
+ assertRecordsMatch(records, actualRecords);
+ }
+ }
+
+ @Test (expected = IORuntimeException.class)
+ public void testRejectUnknownTypeRecord()
+ {
+ try (ChronicleQueue queue = ChronicleQueueBuilder.single(path.toFile()).rollCycle(RollCycles.TEST_SECONDLY).build())
+ {
+ ExcerptAppender appender = queue.acquireAppender();
+ appender.writeDocument(createUnknownTypeRecord());
+
+ AuditLogViewer.dump(ImmutableList.of(path.toString()), RollCycles.TEST_SECONDLY.toString(), false, false, dummy -> {});
+ }
+ catch (Exception e)
+ {
+ assertTrue(e.getMessage().contains("Unsupported record type field"));
+ throw e;
+ }
+ }
+
+ @Test
+ public void testIgnoreUnknownTypeRecord()
+ {
+ List<String> records = new ArrayList<>();
+ records.add("Test foo bar 1");
+ records.add("Test foo bar 2");
+
+ try (ChronicleQueue queue = ChronicleQueueBuilder.single(path.toFile()).rollCycle(RollCycles.TEST_SECONDLY).build())
+ {
+ ExcerptAppender appender = queue.acquireAppender();
+
+ //Write unrecognized type record
+ appender.writeDocument(createUnknownTypeRecord());
+
+ //Write bunch of supported records
+ records.forEach(s -> appender.writeDocument(new BinAuditLogger.Message(s)));
+
+ //Read those written records
+ List<String> actualRecords = new ArrayList<>();
+ AuditLogViewer.dump(ImmutableList.of(path.toString()), RollCycles.TEST_SECONDLY.toString(), false, true, actualRecords::add);
+
+ // Assert all supported records are present
+ assertRecordsMatch(records, actualRecords);
+ }
+ }
+
+ private BinAuditLogger.Message createFutureRecord()
+ {
+ return new BinAuditLogger.Message("dummy message") {
+ protected long version()
+ {
+ return 999;
+ }
+
+ @Override
+ public void writeMarshallablePayload(WireOut wire)
+ {
+ super.writeMarshallablePayload(wire);
+ wire.write("future-field").text("future_value");
+ }
+ };
+ }
+
+ private BinAuditLogger.Message createUnknownTypeRecord()
+ {
+ return new BinAuditLogger.Message("dummy message") {
+ protected String type()
+ {
+ return "unknown-type";
+ }
+
+ @Override
+ public void writeMarshallablePayload(WireOut wire)
+ {
+ super.writeMarshallablePayload(wire);
+ wire.write("unknown-field").text("unknown_value");
+ }
+ };
+ }
+
+ private void assertRecordsMatch(List<String> records, List<String> actualRecords)
+ {
+ Assert.assertEquals(records.size(), actualRecords.size());
+ for (int i = 0; i < records.size(); i++)
+ {
+ Assert.assertTrue(actualRecords.get(i).contains(records.get(i)));
+ }
+ }
+}
diff --git a/test/unit/org/apache/cassandra/tools/BulkLoaderTest.java b/test/unit/org/apache/cassandra/tools/BulkLoaderTest.java
index 104f288..6ed38a0 100644
--- a/test/unit/org/apache/cassandra/tools/BulkLoaderTest.java
+++ b/test/unit/org/apache/cassandra/tools/BulkLoaderTest.java
@@ -63,4 +63,76 @@
assertKeyspaceNotLoaded();
assertServerNotLoaded();
}
+
+ @Test
+ public void testBulkLoader_WithArgs1() throws Exception
+ {
+ try
+ {
+ runTool(0, "org.apache.cassandra.tools.BulkLoader", "-d", "127.9.9.1", "--port", "9042", sstableDirName("legacy_sstables", "legacy_ma_simple"));
+ fail();
+ }
+ catch (RuntimeException e)
+ {
+ if (!(e.getCause() instanceof BulkLoadException))
+ throw e;
+ if (!(e.getCause().getCause() instanceof NoHostAvailableException))
+ throw e;
+ }
+ assertNoUnexpectedThreadsStarted(null, new String[]{"globalEventExecutor-1-1", "globalEventExecutor-1-2"});
+ assertSchemaNotLoaded();
+ assertCLSMNotLoaded();
+ assertSystemKSNotLoaded();
+ assertKeyspaceNotLoaded();
+ assertServerNotLoaded();
+ }
+
+ @Test
+ public void testBulkLoader_WithArgs2() throws Exception
+ {
+ try
+ {
+ runTool(0, "org.apache.cassandra.tools.BulkLoader", "-d", "127.9.9.1:9042", "--port", "9041", sstableDirName("legacy_sstables", "legacy_ma_simple"));
+ fail();
+ }
+ catch (RuntimeException e)
+ {
+ if (!(e.getCause() instanceof BulkLoadException))
+ throw e;
+ if (!(e.getCause().getCause() instanceof NoHostAvailableException))
+ throw e;
+ }
+ assertNoUnexpectedThreadsStarted(null, new String[]{"globalEventExecutor-1-1", "globalEventExecutor-1-2"});
+ assertSchemaNotLoaded();
+ assertCLSMNotLoaded();
+ assertSystemKSNotLoaded();
+ assertKeyspaceNotLoaded();
+ assertServerNotLoaded();
+ }
+
+ @Test(expected = NoHostAvailableException.class)
+ public void testBulkLoader_WithArgs3() throws Throwable
+ {
+ try
+ {
+ runTool(1, "org.apache.cassandra.tools.BulkLoader", "-d", "127.9.9.1", "--port", "9041", sstableDirName("legacy_sstables", "legacy_ma_simple"));
+ }
+ catch (RuntimeException e)
+ {
+ throw e.getCause().getCause();
+ }
+ }
+
+ @Test(expected = NoHostAvailableException.class)
+ public void testBulkLoader_WithArgs4() throws Throwable
+ {
+ try
+ {
+ runTool(1, "org.apache.cassandra.tools.BulkLoader", "-d", "127.9.9.1:9041", sstableDirName("legacy_sstables", "legacy_ma_simple"));
+ }
+ catch (RuntimeException e)
+ {
+ throw e.getCause().getCause();
+ }
+ }
}
diff --git a/test/unit/org/apache/cassandra/tools/ClearSnapshotTest.java b/test/unit/org/apache/cassandra/tools/ClearSnapshotTest.java
new file mode 100644
index 0000000..a73493b
--- /dev/null
+++ b/test/unit/org/apache/cassandra/tools/ClearSnapshotTest.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.tools;
+
+import java.io.IOException;
+import java.util.Map;
+import javax.management.openmbean.TabularData;
+
+import org.apache.commons.lang3.ArrayUtils;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.service.EmbeddedCassandraService;
+
+public class ClearSnapshotTest extends ToolsTester
+{
+ private static EmbeddedCassandraService cassandra;
+ private static String initialJmxPortValue;
+ private static NodeProbe probe;
+ private static final int JMX_PORT = 7188;
+
+ @BeforeClass
+ public static void setup() throws IOException
+ {
+ // Set system property to enable JMX port on localhost for embedded server
+ initialJmxPortValue = System.getProperty("cassandra.jmx.local.port");
+ System.setProperty("cassandra.jmx.local.port", String.valueOf(JMX_PORT));
+
+ SchemaLoader.prepareServer();
+ // CASSANDRA-15776 - size_estimates and table_estimates get truncated on startup, and auto snapshot is true by default for tests
+ // set it false so the test state doesn't see those snapshots
+ DatabaseDescriptor.setAutoSnapshot(false);
+ cassandra = new EmbeddedCassandraService();
+ cassandra.start();
+
+ probe = new NodeProbe("127.0.0.1", JMX_PORT);
+ }
+
+ @AfterClass
+ public static void teardown() throws IOException
+ {
+ cassandra.stop();
+ if (initialJmxPortValue != null)
+ {
+ System.setProperty("cassandra.jmx.local.port", initialJmxPortValue);
+ }
+
+ probe.close();
+ }
+
+ private String[] constructParamaterArray(final String command, final String... commandParams)
+ {
+ String[] baseCommandLine = {"-p", String.valueOf(JMX_PORT), command};
+ return ArrayUtils.addAll(baseCommandLine, commandParams);
+ }
+
+ @Test
+ public void testClearSnapshot_NoArgs() throws IOException
+ {
+ runTool(2, "org.apache.cassandra.tools.NodeTool",
+ constructParamaterArray("clearsnapshot"));
+ }
+
+ @Test
+ public void testClearSnapshot_AllAndName() throws IOException
+ {
+ runTool(2, "org.apache.cassandra.tools.NodeTool",
+ constructParamaterArray("clearsnapshot", "-t", "some-name", "--all"));
+ }
+
+ @Test
+ public void testClearSnapshot_RemoveByName() throws IOException
+ {
+ runTool(0,"org.apache.cassandra.tools.NodeTool",
+ constructParamaterArray("snapshot","-t","some-name"));
+
+ Map<String, TabularData> snapshots_before = probe.getSnapshotDetails();
+ Assert.assertTrue(snapshots_before.containsKey("some-name"));
+
+ runTool(0,"org.apache.cassandra.tools.NodeTool",
+ constructParamaterArray("clearsnapshot","-t","some-name"));
+ Map<String, TabularData> snapshots_after = probe.getSnapshotDetails();
+ Assert.assertFalse(snapshots_after.containsKey("some-name"));
+ }
+
+ @Test
+ public void testClearSnapshot_RemoveMultiple() throws IOException
+ {
+ runTool(0,"org.apache.cassandra.tools.NodeTool",
+ constructParamaterArray("snapshot","-t","some-name"));
+ runTool(0,"org.apache.cassandra.tools.NodeTool",
+ constructParamaterArray("snapshot","-t","some-other-name"));
+
+ Map<String, TabularData> snapshots_before = probe.getSnapshotDetails();
+ Assert.assertTrue(snapshots_before.size() == 2);
+
+ runTool(0,"org.apache.cassandra.tools.NodeTool",
+ constructParamaterArray("clearsnapshot","--all"));
+ Map<String, TabularData> snapshots_after = probe.getSnapshotDetails();
+ Assert.assertTrue(snapshots_after.size() == 0);
+ }
+
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/tools/CompactionStressTest.java b/test/unit/org/apache/cassandra/tools/CompactionStressTest.java
index cdf8ac1..c8b0b97 100644
--- a/test/unit/org/apache/cassandra/tools/CompactionStressTest.java
+++ b/test/unit/org/apache/cassandra/tools/CompactionStressTest.java
@@ -20,7 +20,6 @@
import java.io.File;
-import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -29,12 +28,6 @@
@RunWith(OrderedJUnit4ClassRunner.class)
public class CompactionStressTest extends ToolsTester
{
- @BeforeClass
- public static void setupTester()
- {
- // override ToolsTester.setupTester() to avoid `DatabaseDescriptor.toolInitialization()`
- }
-
@Test
public void testNoArgs()
{
diff --git a/test/unit/org/apache/cassandra/tools/ToolsTester.java b/test/unit/org/apache/cassandra/tools/ToolsTester.java
index a690ca7..0bb9beb 100644
--- a/test/unit/org/apache/cassandra/tools/ToolsTester.java
+++ b/test/unit/org/apache/cassandra/tools/ToolsTester.java
@@ -29,17 +29,17 @@
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
+import java.util.Objects;
import java.util.Set;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
-import org.apache.cassandra.config.DatabaseDescriptor;
-
import org.apache.commons.io.FileUtils;
import org.junit.BeforeClass;
import org.slf4j.LoggerFactory;
+import static org.apache.cassandra.utils.FBUtilities.preventIllegalAccessWarnings;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
@@ -50,6 +50,11 @@
*/
public abstract class ToolsTester
{
+ static
+ {
+ preventIllegalAccessWarnings();
+ }
+
private static List<ThreadInfo> initialThreads;
static final String[] EXPECTED_THREADS_WITH_SCHEMA = {
@@ -84,6 +89,7 @@
.collect(Collectors.toSet());
Set<String> current = Arrays.stream(threads.getThreadInfo(threads.getAllThreadIds()))
+ .filter(Objects::nonNull)
.map(ThreadInfo::getThreadName)
.collect(Collectors.toSet());
@@ -116,12 +122,12 @@
public void assertSchemaNotLoaded()
{
- assertClassNotLoaded("org.apache.cassandra.config.Schema");
+ assertClassNotLoaded("org.apache.cassandra.schema.Schema");
}
public void assertSchemaLoaded()
{
- assertClassLoaded("org.apache.cassandra.config.Schema");
+ assertClassLoaded("org.apache.cassandra.schema.Schema");
}
public void assertKeyspaceNotLoaded()
@@ -253,9 +259,6 @@
ThreadMXBean threads = ManagementFactory.getThreadMXBean();
initialThreads = Arrays.asList(threads.getThreadInfo(threads.getAllThreadIds()));
-
- DatabaseDescriptor.toolInitialization();
- DatabaseDescriptor.applyAddressConfig();
}
public static class SystemExitException extends Error
diff --git a/test/unit/org/apache/cassandra/tools/TopPartitionsTest.java b/test/unit/org/apache/cassandra/tools/TopPartitionsTest.java
new file mode 100644
index 0000000..d02b4c4
--- /dev/null
+++ b/test/unit/org/apache/cassandra/tools/TopPartitionsTest.java
@@ -0,0 +1,67 @@
+package org.apache.cassandra.tools;
+
+import static java.lang.String.format;
+import static org.apache.cassandra.cql3.QueryProcessor.executeInternal;
+import static org.junit.Assert.assertEquals;
+
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+
+import javax.management.openmbean.CompositeData;
+import javax.management.openmbean.TabularDataSupport;
+
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.SystemKeyspace;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.service.StorageService;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import com.google.common.collect.Lists;
+
+public class TopPartitionsTest
+{
+ @BeforeClass
+ public static void loadSchema() throws ConfigurationException
+ {
+ SchemaLoader.prepareServer();
+ }
+
+ @Test
+ public void testServiceTopPartitionsNoArg() throws Exception
+ {
+ BlockingQueue<Map<String, List<CompositeData>>> q = new ArrayBlockingQueue<>(1);
+ ColumnFamilyStore.all();
+ Executors.newCachedThreadPool().execute(() ->
+ {
+ try
+ {
+ q.put(StorageService.instance.samplePartitions(1000, 100, 10, Lists.newArrayList("READS", "WRITES")));
+ }
+ catch (Exception e)
+ {
+ e.printStackTrace();
+ }
+ });
+ Thread.sleep(100);
+ SystemKeyspace.persistLocalMetadata();
+ Map<String, List<CompositeData>> result = q.poll(5, TimeUnit.SECONDS);
+ List<CompositeData> cd = result.get("WRITES");
+ assertEquals(1, cd.size());
+ }
+
+ @Test
+ public void testServiceTopPartitionsSingleTable() throws Exception
+ {
+ ColumnFamilyStore.getIfExists("system", "local").beginLocalSampling("READS", 5, 100000);
+ String req = "SELECT * FROM system.%s WHERE key='%s'";
+ executeInternal(format(req, SystemKeyspace.LOCAL, SystemKeyspace.LOCAL));
+ List<CompositeData> result = ColumnFamilyStore.getIfExists("system", "local").finishLocalSampling("READS", 5);
+ assertEquals(1, result.size());
+ }
+}
diff --git a/test/unit/org/apache/cassandra/tools/nodetool/stats/StatsTableComparatorTest.java b/test/unit/org/apache/cassandra/tools/nodetool/stats/StatsTableComparatorTest.java
new file mode 100644
index 0000000..ea18d51
--- /dev/null
+++ b/test/unit/org/apache/cassandra/tools/nodetool/stats/StatsTableComparatorTest.java
@@ -0,0 +1,311 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.tools.nodetool.stats;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class StatsTableComparatorTest extends TableStatsTestBase
+{
+
+ /**
+ * Builds a string of the format "table1 > table2 > ... > tableN-1 > tableN"
+ * to show the order of StatsTables in a sorted list.
+ * @returns String a string showing the relative position in the list of its StatsTables
+ */
+ private String buildSortOrderString(List<StatsTable> sorted) {
+ if (sorted == null)
+ return null;
+ if (sorted.size() == 0)
+ return "";
+ String names = sorted.get(0).tableName;
+ for (int i = 1; i < sorted.size(); i++)
+ names += " > " + sorted.get(i).tableName;
+ return names;
+ }
+
+ /**
+ * Reusable runner to test whether StatsTableComparator achieves the expected order for the given parameters.
+ */
+ private void runCompareTest(List<StatsTable> vector, String sortKey, String expectedOrder,
+ boolean humanReadable, boolean ascending)
+ {
+ Collections.sort(vector, new StatsTableComparator(sortKey, humanReadable, ascending));
+ String failureMessage = String.format("StatsTableComparator failed to sort by %s", sortKey);
+ assertEquals(failureMessage, expectedOrder, buildSortOrderString(vector));
+ }
+
+ @Test
+ public void testCompareDoubles() throws Exception
+ {
+ boolean humanReadable = false;
+ boolean ascending = false;
+ // average live cells: 1 > 6 > 2 > 5 > 3 > 4
+ runCompareTest(testTables,
+ "average_live_cells_per_slice_last_five_minutes",
+ "table1 > table6 > table2 > table5 > table3 > table4",
+ humanReadable,
+ ascending);
+ // average tombstones: 6 > 1 > 5 > 2 > 3 > 4
+ runCompareTest(testTables,
+ "average_tombstones_per_slice_last_five_minutes",
+ "table6 > table1 > table5 > table2 > table3 > table4",
+ humanReadable,
+ ascending);
+ // read latency: 3 > 2 > 1 > 6 > 4 > 5
+ runCompareTest(testTables,
+ "read_latency",
+ "table3 > table2 > table1 > table6 > table4 > table5",
+ humanReadable,
+ ascending);
+ // write latency: 4 > 5 > 6 > 1 > 2 > 3
+ runCompareTest(testTables,
+ "write_latency",
+ "table4 > table5 > table6 > table1 > table2 > table3",
+ humanReadable,
+ ascending);
+ // percent repaired
+ runCompareTest(testTables,
+ "percent_repaired",
+ "table1 > table2 > table3 > table5 > table4 > table6",
+ humanReadable,
+ ascending);
+ }
+
+ @Test
+ public void testCompareLongs() throws Exception
+ {
+ boolean humanReadable = false;
+ boolean ascending = false;
+ // reads: 6 > 5 > 4 > 3 > 2 > 1
+ runCompareTest(testTables,
+ "reads",
+ "table6 > table5 > table4 > table3 > table2 > table1",
+ humanReadable,
+ ascending);
+ // writes: 1 > 2 > 3 > 4 > 5 > 6
+ runCompareTest(testTables,
+ "writes",
+ "table1 > table2 > table3 > table4 > table5 > table6",
+ humanReadable,
+ ascending);
+ // compacted partition maximum bytes: 1 > 3 > 5 > 2 > 4 = 6
+ runCompareTest(testTables,
+ "compacted_partition_maximum_bytes",
+ "table1 > table3 > table5 > table2 > table4 > table6",
+ humanReadable,
+ ascending);
+ // compacted partition mean bytes: 1 > 3 > 2 = 4 = 5 > 6
+ runCompareTest(testTables,
+ "compacted_partition_mean_bytes",
+ "table1 > table3 > table2 > table4 > table5 > table6",
+ humanReadable,
+ ascending);
+ // compacted partition minimum bytes: 6 > 4 > 2 > 5 > 1 = 3
+ runCompareTest(testTables,
+ "compacted_partition_minimum_bytes",
+ "table6 > table4 > table2 > table5 > table1 > table3",
+ humanReadable,
+ ascending);
+ // maximum live cells last five minutes: 1 > 2 = 3 > 4 = 5 > 6
+ runCompareTest(testTables,
+ "maximum_live_cells_per_slice_last_five_minutes",
+ "table1 > table2 > table3 > table4 > table5 > table6",
+ humanReadable,
+ ascending);
+ // maximum tombstones last five minutes: 6 > 5 > 3 = 4 > 2 > 1
+ runCompareTest(testTables,
+ "maximum_tombstones_per_slice_last_five_minutes",
+ "table6 > table5 > table3 > table4 > table2 > table1",
+ humanReadable,
+ ascending);
+ }
+
+ @Test
+ public void testCompareHumanReadable() throws Exception
+ {
+ boolean humanReadable = true;
+ boolean ascending = false;
+ // human readable space used total: 6 > 5 > 4 > 3 > 2 > 1
+ runCompareTest(humanReadableTables,
+ "space_used_total",
+ "table6 > table5 > table4 > table3 > table2 > table1",
+ humanReadable,
+ ascending);
+ // human readable memtable data size: 1 > 3 > 5 > 2 > 4 > 6
+ runCompareTest(humanReadableTables,
+ "memtable_data_size",
+ "table1 > table3 > table5 > table2 > table4 > table6",
+ humanReadable,
+ ascending);
+ }
+
+ @Test
+ public void testCompareObjects() throws Exception
+ {
+ boolean humanReadable = false;
+ boolean ascending = false;
+ // bloom filter false positives: 2 > 4 > 6 > 1 > 3 > 5
+ runCompareTest(testTables,
+ "bloom_filter_false_positives",
+ "table2 > table4 > table6 > table1 > table3 > table5",
+ humanReadable,
+ ascending);
+ // bloom filter false positive ratio: 5 > 3 > 1 > 6 > 4 > 2
+ runCompareTest(testTables,
+ "bloom_filter_false_ratio",
+ "table5 > table3 > table1 > table6 > table4 > table2",
+ humanReadable,
+ ascending);
+ // memtable cell count: 3 > 5 > 6 > 1 > 2 > 4
+ runCompareTest(testTables,
+ "memtable_cell_count",
+ "table3 > table5 > table6 > table1 > table2 > table4",
+ humanReadable,
+ ascending);
+ // memtable switch count: 4 > 2 > 3 > 6 > 5 > 1
+ runCompareTest(testTables,
+ "memtable_switch_count",
+ "table4 > table2 > table3 > table6 > table5 > table1",
+ humanReadable,
+ ascending);
+ // number of partitions estimate: 1 > 2 > 3 > 4 > 5 > 6
+ runCompareTest(testTables,
+ "number_of_partitions_estimate",
+ "table1 > table2 > table3 > table4 > table5 > table6",
+ humanReadable,
+ ascending);
+ // pending flushes: 2 > 1 > 4 > 3 > 6 > 5
+ runCompareTest(testTables,
+ "pending_flushes",
+ "table2 > table1 > table4 > table3 > table6 > table5",
+ humanReadable,
+ ascending);
+ // sstable compression ratio: 5 > 4 > 1 = 2 = 6 > 3
+ runCompareTest(testTables,
+ "sstable_compression_ratio",
+ "table5 > table4 > table1 > table2 > table6 > table3",
+ humanReadable,
+ ascending);
+ // sstable count: 1 > 3 > 5 > 2 > 4 > 6
+ runCompareTest(testTables,
+ "sstable_count",
+ "table1 > table3 > table5 > table2 > table4 > table6",
+ humanReadable,
+ ascending);
+ }
+
+ @Test
+ public void testCompareOffHeap() throws Exception
+ {
+ boolean humanReadable = false;
+ boolean ascending = false;
+ // offheap memory total: 4 > 2 > 6 > 1 = 3 = 5
+ runCompareTest(testTables,
+ "off_heap_memory_used_total",
+ "table4 > table2 > table6 > table1 > table3 > table5",
+ humanReadable,
+ ascending);
+ // bloom filter offheap: 4 > 6 > 2 > 1 > 3 > 5
+ runCompareTest(testTables,
+ "bloom_filter_off_heap_memory_used",
+ "table4 > table6 > table2 > table1 > table3 > table5",
+ humanReadable,
+ ascending);
+ // compression metadata offheap: 2 > 4 > 6 > 1 = 3 = 5
+ runCompareTest(testTables,
+ "compression_metadata_off_heap_memory_used",
+ "table2 > table4 > table6 > table1 > table3 > table5",
+ humanReadable,
+ ascending);
+ // index summary offheap: 6 > 4 > 2 > 1 = 3 = 5
+ runCompareTest(testTables,
+ "index_summary_off_heap_memory_used",
+ "table6 > table4 > table2 > table1 > table3 > table5",
+ humanReadable,
+ ascending);
+ // memtable offheap: 2 > 6 > 4 > 1 = 3 = 5
+ runCompareTest(testTables,
+ "memtable_off_heap_memory_used",
+ "table2 > table6 > table4 > table1 > table3 > table5",
+ humanReadable,
+ ascending);
+ }
+
+ @Test
+ public void testCompareStrings() throws Exception
+ {
+ boolean humanReadable = false;
+ boolean ascending = false;
+ // full name (keyspace.table) ascending: 1 > 2 > 3 > 4 > 5 > 6
+ runCompareTest(testTables,
+ "full_name",
+ "table1 > table2 > table3 > table4 > table5 > table6",
+ humanReadable,
+ true);
+ // table name ascending: 6 > 5 > 4 > 3 > 2 > 1
+ runCompareTest(testTables,
+ "table_name",
+ "table1 > table2 > table3 > table4 > table5 > table6",
+ humanReadable,
+ true);
+ // bloom filter space used: 2 > 4 > 6 > 1 > 3 > 5
+ runCompareTest(testTables,
+ "bloom_filter_space_used",
+ "table2 > table4 > table6 > table1 > table3 > table5",
+ humanReadable,
+ ascending);
+ // dropped mutations: 6 > 3 > 4 > 2 > 1 = 5
+ runCompareTest(testTables,
+ "dropped_mutations",
+ "table6 > table3 > table4 > table2 > table1 > table5",
+ humanReadable,
+ ascending);
+ // space used by snapshots: 5 > 1 > 2 > 4 > 3 = 6
+ runCompareTest(testTables,
+ "space_used_by_snapshots_total",
+ "table5 > table1 > table2 > table4 > table3 > table6",
+ humanReadable,
+ ascending);
+ // space used live: 6 > 5 > 4 > 2 > 1 = 3
+ runCompareTest(testTables,
+ "space_used_live",
+ "table6 > table5 > table4 > table2 > table1 > table3",
+ humanReadable,
+ ascending);
+ // space used total: 1 > 2 > 3 > 4 > 5 > 6
+ runCompareTest(testTables,
+ "space_used_total",
+ "table1 > table2 > table3 > table4 > table5 > table6",
+ humanReadable,
+ ascending);
+ // memtable data size: 6 > 5 > 4 > 3 > 2 > 1
+ runCompareTest(testTables,
+ "memtable_data_size",
+ "table6 > table5 > table4 > table3 > table2 > table1",
+ humanReadable,
+ ascending);
+ }
+
+}
diff --git a/test/unit/org/apache/cassandra/tools/nodetool/stats/TableStatsPrinterTest.java b/test/unit/org/apache/cassandra/tools/nodetool/stats/TableStatsPrinterTest.java
new file mode 100644
index 0000000..32b0b62
--- /dev/null
+++ b/test/unit/org/apache/cassandra/tools/nodetool/stats/TableStatsPrinterTest.java
@@ -0,0 +1,372 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.tools.nodetool.stats;
+
+import java.io.ByteArrayOutputStream;
+import java.io.PrintStream;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class TableStatsPrinterTest extends TableStatsTestBase
+{
+
+ public static final String expectedDefaultTable1Output =
+ "\tTable: %s\n" +
+ "\tSSTable count: 60000\n" +
+ "\tOld SSTable count: 0\n" +
+ "\tSpace used (live): 0\n" +
+ "\tSpace used (total): 9001\n" +
+ "\tSpace used by snapshots (total): 1111\n" +
+ "\tSSTable Compression Ratio: 0.68\n" +
+ "\tNumber of partitions (estimate): 111111\n" +
+ "\tMemtable cell count: 111\n" +
+ "\tMemtable data size: 0\n" +
+ "\tMemtable switch count: 1\n" +
+ "\tLocal read count: 0\n" +
+ "\tLocal read latency: 2.000 ms\n" +
+ "\tLocal write count: 5\n" +
+ "\tLocal write latency: 0.050 ms\n" +
+ "\tPending flushes: 11111\n" +
+ "\tPercent repaired: 100.0\n" +
+ "\tBloom filter false positives: 30\n" +
+ "\tBloom filter false ratio: 0.40000\n" +
+ "\tBloom filter space used: 789\n" +
+ "\tCompacted partition minimum bytes: 2\n" +
+ "\tCompacted partition maximum bytes: 60\n" +
+ "\tCompacted partition mean bytes: 6\n" +
+ "\tAverage live cells per slice (last five minutes): 6.0\n" +
+ "\tMaximum live cells per slice (last five minutes): 6\n" +
+ "\tAverage tombstones per slice (last five minutes): 5.0\n" +
+ "\tMaximum tombstones per slice (last five minutes): 1\n" +
+ "\tDropped Mutations: 0\n" +
+ "\n";
+
+ public static final String expectedDefaultTable2Output =
+ "\tTable: %s\n" +
+ "\tSSTable count: 3000\n" +
+ "\tOld SSTable count: 0\n" +
+ "\tSpace used (live): 22\n" +
+ "\tSpace used (total): 1024\n" +
+ "\tSpace used by snapshots (total): 222\n" +
+ "\tOff heap memory used (total): 314159367\n" +
+ "\tSSTable Compression Ratio: 0.68\n" +
+ "\tNumber of partitions (estimate): 22222\n" +
+ "\tMemtable cell count: 22\n" +
+ "\tMemtable data size: 900\n" +
+ "\tMemtable off heap memory used: 314159265\n" +
+ "\tMemtable switch count: 22222\n" +
+ "\tLocal read count: 1\n" +
+ "\tLocal read latency: 3.000 ms\n" +
+ "\tLocal write count: 4\n" +
+ "\tLocal write latency: 0.000 ms\n" +
+ "\tPending flushes: 222222\n" +
+ "\tPercent repaired: 99.9\n" +
+ "\tBloom filter false positives: 600\n" +
+ "\tBloom filter false ratio: 0.01000\n" +
+ "\tBloom filter space used: 161718\n" +
+ "\tBloom filter off heap memory used: 98\n" +
+ "\tIndex summary off heap memory used: 1\n" +
+ "\tCompression metadata off heap memory used: 3\n" +
+ "\tCompacted partition minimum bytes: 4\n" +
+ "\tCompacted partition maximum bytes: 30\n" +
+ "\tCompacted partition mean bytes: 4\n" +
+ "\tAverage live cells per slice (last five minutes): 4.01\n" +
+ "\tMaximum live cells per slice (last five minutes): 5\n" +
+ "\tAverage tombstones per slice (last five minutes): 4.001\n" +
+ "\tMaximum tombstones per slice (last five minutes): 2\n" +
+ "\tDropped Mutations: 222\n" +
+ "\n";
+
+ public static final String expectedDefaultTable3Output =
+ "\tTable: %s\n" +
+ "\tSSTable count: 50000\n" +
+ "\tOld SSTable count: 0\n" +
+ "\tSpace used (live): 0\n" +
+ "\tSpace used (total): 512\n" +
+ "\tSpace used by snapshots (total): 0\n" +
+ "\tSSTable Compression Ratio: 0.32\n" +
+ "\tNumber of partitions (estimate): 3333\n" +
+ "\tMemtable cell count: 333333\n" +
+ "\tMemtable data size: 1999\n" +
+ "\tMemtable switch count: 3333\n" +
+ "\tLocal read count: 2\n" +
+ "\tLocal read latency: 4.000 ms\n" +
+ "\tLocal write count: 3\n" +
+ "\tLocal write latency: NaN ms\n" +
+ "\tPending flushes: 333\n" +
+ "\tPercent repaired: 99.8\n" +
+ "\tBloom filter false positives: 20\n" +
+ "\tBloom filter false ratio: 0.50000\n" +
+ "\tBloom filter space used: 456\n" +
+ "\tCompacted partition minimum bytes: 2\n" +
+ "\tCompacted partition maximum bytes: 50\n" +
+ "\tCompacted partition mean bytes: 5\n" +
+ "\tAverage live cells per slice (last five minutes): 0.0\n" +
+ "\tMaximum live cells per slice (last five minutes): 5\n" +
+ "\tAverage tombstones per slice (last five minutes): NaN\n" +
+ "\tMaximum tombstones per slice (last five minutes): 3\n" +
+ "\tDropped Mutations: 33333\n" +
+ "\n";
+
+ public static final String expectedDefaultTable4Output =
+ "\tTable: %s\n" +
+ "\tSSTable count: 2000\n" +
+ "\tOld SSTable count: 0\n" +
+ "\tSpace used (live): 4444\n" +
+ "\tSpace used (total): 256\n" +
+ "\tSpace used by snapshots (total): 44\n" +
+ "\tOff heap memory used (total): 441213818\n" +
+ "\tSSTable Compression Ratio: 0.95\n" +
+ "\tNumber of partitions (estimate): 444\n" +
+ "\tMemtable cell count: 4\n" +
+ "\tMemtable data size: 3000\n" +
+ "\tMemtable off heap memory used: 141421356\n" +
+ "\tMemtable switch count: 444444\n" +
+ "\tLocal read count: 3\n" +
+ "\tLocal read latency: NaN ms\n" +
+ "\tLocal write count: 2\n" +
+ "\tLocal write latency: 2.000 ms\n" +
+ "\tPending flushes: 4444\n" +
+ "\tPercent repaired: 50.0\n" +
+ "\tBloom filter false positives: 500\n" +
+ "\tBloom filter false ratio: 0.02000\n" +
+ "\tBloom filter space used: 131415\n" +
+ "\tBloom filter off heap memory used: 299792458\n" +
+ "\tIndex summary off heap memory used: 2\n" +
+ "\tCompression metadata off heap memory used: 2\n" +
+ "\tCompacted partition minimum bytes: 5\n" +
+ "\tCompacted partition maximum bytes: 20\n" +
+ "\tCompacted partition mean bytes: 4\n" +
+ "\tAverage live cells per slice (last five minutes): NaN\n" +
+ "\tMaximum live cells per slice (last five minutes): 3\n" +
+ "\tAverage tombstones per slice (last five minutes): 0.0\n" +
+ "\tMaximum tombstones per slice (last five minutes): 3\n" +
+ "\tDropped Mutations: 4444\n" +
+ "\n";
+
+ public static final String expectedDefaultTable5Output =
+ "\tTable: %s\n" +
+ "\tSSTable count: 40000\n" +
+ "\tOld SSTable count: 0\n" +
+ "\tSpace used (live): 55555\n" +
+ "\tSpace used (total): 64\n" +
+ "\tSpace used by snapshots (total): 55555\n" +
+ "\tSSTable Compression Ratio: 0.99\n" +
+ "\tNumber of partitions (estimate): 55\n" +
+ "\tMemtable cell count: 55555\n" +
+ "\tMemtable data size: 20000\n" +
+ "\tMemtable switch count: 5\n" +
+ "\tLocal read count: 4\n" +
+ "\tLocal read latency: 0.000 ms\n" +
+ "\tLocal write count: 1\n" +
+ "\tLocal write latency: 1.000 ms\n" +
+ "\tPending flushes: 5\n" +
+ "\tPercent repaired: 93.0\n" +
+ "\tBloom filter false positives: 10\n" +
+ "\tBloom filter false ratio: 0.60000\n" +
+ "\tBloom filter space used: 123\n" +
+ "\tCompacted partition minimum bytes: 3\n" +
+ "\tCompacted partition maximum bytes: 40\n" +
+ "\tCompacted partition mean bytes: 4\n" +
+ "\tAverage live cells per slice (last five minutes): 4.0\n" +
+ "\tMaximum live cells per slice (last five minutes): 3\n" +
+ "\tAverage tombstones per slice (last five minutes): 4.01\n" +
+ "\tMaximum tombstones per slice (last five minutes): 5\n" +
+ "\tDropped Mutations: 0\n" +
+ "\n";
+
+ public static final String expectedDefaultTable6Output =
+ "\tTable: %s\n" +
+ "\tSSTable count: 1000\n" +
+ "\tOld SSTable count: 0\n" +
+ "\tSpace used (live): 666666\n" +
+ "\tSpace used (total): 0\n" +
+ "\tSpace used by snapshots (total): 0\n" +
+ "\tOff heap memory used (total): 162470810\n" +
+ "\tSSTable Compression Ratio: 0.68\n" +
+ "\tNumber of partitions (estimate): 6\n" +
+ "\tMemtable cell count: 6666\n" +
+ "\tMemtable data size: 1000000\n" +
+ "\tMemtable off heap memory used: 161803398\n" +
+ "\tMemtable switch count: 6\n" +
+ "\tLocal read count: 5\n" +
+ "\tLocal read latency: 1.000 ms\n" +
+ "\tLocal write count: 0\n" +
+ "\tLocal write latency: 0.500 ms\n" +
+ "\tPending flushes: 66\n" +
+ "\tPercent repaired: 0.0\n" +
+ "\tBloom filter false positives: 400\n" +
+ "\tBloom filter false ratio: 0.03000\n" +
+ "\tBloom filter space used: 101112\n" +
+ "\tBloom filter off heap memory used: 667408\n" +
+ "\tIndex summary off heap memory used: 3\n" +
+ "\tCompression metadata off heap memory used: 1\n" +
+ "\tCompacted partition minimum bytes: 6\n" +
+ "\tCompacted partition maximum bytes: 20\n" +
+ "\tCompacted partition mean bytes: 3\n" +
+ "\tAverage live cells per slice (last five minutes): 5.0\n" +
+ "\tMaximum live cells per slice (last five minutes): 2\n" +
+ "\tAverage tombstones per slice (last five minutes): 6.0\n" +
+ "\tMaximum tombstones per slice (last five minutes): 6\n" +
+ "\tDropped Mutations: 666666\n" +
+ "\n";
+
+ /**
+ * Expected output of TableStatsPrinter DefaultPrinter for this dataset.
+ * Total number of tables is zero because it's non-trivial to simulate that metric
+ * without leaking test implementation into the TableStatsHolder implementation.
+ */
+ public static final String expectedDefaultPrinterOutput =
+ "Total number of tables: 0\n" +
+ "----------------\n" +
+ "Keyspace : keyspace1\n" +
+ "\tRead Count: 3\n" +
+ "\tRead Latency: 0.0 ms\n" +
+ "\tWrite Count: 12\n" +
+ "\tWrite Latency: 0.0 ms\n" +
+ "\tPending Flushes: 233666\n" +
+ String.format(expectedDefaultTable1Output, "table1").replace("\t", "\t\t") +
+ String.format(expectedDefaultTable2Output, "table2").replace("\t", "\t\t") +
+ String.format(expectedDefaultTable3Output, "table3").replace("\t", "\t\t") +
+ "----------------\n" +
+ "Keyspace : keyspace2\n" +
+ "\tRead Count: 7\n" +
+ "\tRead Latency: 0.0 ms\n" +
+ "\tWrite Count: 3\n" +
+ "\tWrite Latency: 0.0 ms\n" +
+ "\tPending Flushes: 4449\n" +
+ String.format(expectedDefaultTable4Output, "table4").replace("\t", "\t\t") +
+ String.format(expectedDefaultTable5Output, "table5").replace("\t", "\t\t") +
+ "----------------\n" +
+ "Keyspace : keyspace3\n" +
+ "\tRead Count: 5\n" +
+ "\tRead Latency: 0.0 ms\n" +
+ "\tWrite Count: 0\n" +
+ "\tWrite Latency: NaN ms\n" +
+ "\tPending Flushes: 66\n" +
+ String.format(expectedDefaultTable6Output, "table6").replace("\t", "\t\t") +
+ "----------------\n";
+
+ /**
+ * Expected output from SortedDefaultPrinter for data sorted by reads in this test.
+ */
+ private static final String expectedSortedDefaultPrinterOutput =
+ "Total number of tables: 0\n" +
+ "----------------\n" +
+ String.format(expectedDefaultTable6Output, "keyspace3.table6") +
+ String.format(expectedDefaultTable5Output, "keyspace2.table5") +
+ String.format(expectedDefaultTable4Output, "keyspace2.table4") +
+ String.format(expectedDefaultTable3Output, "keyspace1.table3") +
+ String.format(expectedDefaultTable2Output, "keyspace1.table2") +
+ String.format(expectedDefaultTable1Output, "keyspace1.table1") +
+ "----------------\n";
+
+ /**
+ * Expected output from SortedDefaultPrinter for data sorted by reads and limited to the top 4 tables.
+ */
+ private static final String expectedSortedDefaultPrinterTopOutput =
+ "Total number of tables: 0 (showing top 0 by %s)\n" +
+ "----------------\n" +
+ String.format(expectedDefaultTable6Output, "keyspace3.table6") +
+ String.format(expectedDefaultTable5Output, "keyspace2.table5") +
+ String.format(expectedDefaultTable4Output, "keyspace2.table4") +
+ String.format(expectedDefaultTable3Output, "keyspace1.table3") +
+ "----------------\n";
+
+ /**
+ * Expected output from SortedDefaultPrinter for data sorted by reads and limited to the top 10 tables.
+ */
+ private static final String expectedSortedDefaultPrinterLargeTopOutput =
+ "Total number of tables: 0 (showing top 0 by %s)\n" +
+ "----------------\n" +
+ String.format(expectedDefaultTable6Output, "keyspace3.table6") +
+ String.format(expectedDefaultTable5Output, "keyspace2.table5") +
+ String.format(expectedDefaultTable4Output, "keyspace2.table4") +
+ String.format(expectedDefaultTable3Output, "keyspace1.table3") +
+ String.format(expectedDefaultTable2Output, "keyspace1.table2") +
+ String.format(expectedDefaultTable1Output, "keyspace1.table1") +
+ "----------------\n";
+
+ @Test
+ public void testDefaultPrinter() throws Exception
+ {
+ StatsHolder holder = new TestTableStatsHolder(testKeyspaces, "", 0);
+ StatsPrinter printer = TableStatsPrinter.from("", false);
+ ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
+ printer.print(holder, new PrintStream(byteStream));
+ assertEquals("StatsTablePrinter.DefaultPrinter does not print test vector as expected", expectedDefaultPrinterOutput, byteStream.toString());
+ }
+
+ @Test
+ public void testSortedDefaultPrinter() throws Exception
+ {
+ // test sorting
+ StatsHolder holder = new TestTableStatsHolder(testKeyspaces, "reads", 0);
+ StatsPrinter printer = TableStatsPrinter.from("reads", true);
+ ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
+ printer.print(holder, new PrintStream(byteStream));
+ assertEquals("StatsTablePrinter.SortedDefaultPrinter does not print sorted tables as expected",
+ expectedSortedDefaultPrinterOutput, byteStream.toString());
+ byteStream.reset();
+ // test sorting and filtering top k, where k < total number of tables
+ String sortKey = "reads";
+ int top = 4;
+ holder = new TestTableStatsHolder(testKeyspaces, sortKey, top);
+ printer = TableStatsPrinter.from(sortKey, true);
+ printer.print(holder, new PrintStream(byteStream));
+ assertEquals("StatsTablePrinter.SortedDefaultPrinter does not print top K sorted tables as expected",
+ String.format(expectedSortedDefaultPrinterTopOutput, sortKey), byteStream.toString());
+ byteStream.reset();
+ // test sorting and filtering top k, where k >= total number of tables
+ sortKey = "reads";
+ top = 10;
+ holder = new TestTableStatsHolder(testKeyspaces, sortKey, top);
+ printer = TableStatsPrinter.from(sortKey, true);
+ printer.print(holder, new PrintStream(byteStream));
+ assertEquals("StatsTablePrinter.SortedDefaultPrinter does not print top K sorted tables as expected for large values of K",
+ String.format(expectedSortedDefaultPrinterLargeTopOutput, sortKey), byteStream.toString());
+ }
+
+ /**
+ * A test version of TableStatsHolder to hold a test vector instead of gathering stats from a live cluster.
+ */
+ private static class TestTableStatsHolder extends TableStatsHolder
+ {
+
+ public TestTableStatsHolder(List<StatsKeyspace> testKeyspaces, String sortKey, int top)
+ {
+ super(null, false, false, new ArrayList<>(), sortKey, top);
+ this.keyspaces.clear();
+ this.keyspaces.addAll(testKeyspaces);
+ }
+
+ @Override
+ protected boolean isTestTableStatsHolder()
+ {
+ return true;
+ }
+ }
+
+}
diff --git a/test/unit/org/apache/cassandra/tools/nodetool/stats/TableStatsTestBase.java b/test/unit/org/apache/cassandra/tools/nodetool/stats/TableStatsTestBase.java
new file mode 100644
index 0000000..b2f1663
--- /dev/null
+++ b/test/unit/org/apache/cassandra/tools/nodetool/stats/TableStatsTestBase.java
@@ -0,0 +1,433 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.tools.nodetool.stats;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Create a test vector for unit testing of TableStats features.
+ */
+public class TableStatsTestBase
+{
+
+ /**
+ * A test vector of StatsKeyspace and StatsTable objects loaded with human readable stats.
+ */
+ protected static List<StatsKeyspace> humanReadableKeyspaces;
+
+ /**
+ * A test vector of StatsTable objects loaded with human readable statistics.
+ */
+ protected static List<StatsTable> humanReadableTables;
+
+ /**
+ * A test vector of StatsKeyspace and StatsTable objects.
+ */
+ protected static List<StatsKeyspace> testKeyspaces;
+
+ /**
+ * A test vector of StatsTable objects.
+ */
+ protected static List<StatsTable> testTables;
+
+ /**
+ * @returns StatsKeyspace an instance of StatsKeyspace preset with values for use in a test vector
+ */
+ private static StatsKeyspace createStatsKeyspaceTemplate(String keyspaceName)
+ {
+ return new StatsKeyspace(null, keyspaceName);
+ }
+
+ /**
+ * @returns StatsTable an instance of StatsTable preset with values for use in a test vector
+ */
+ private static StatsTable createStatsTableTemplate(String keyspaceName, String tableName)
+ {
+ StatsTable template = new StatsTable();
+ template.fullName = keyspaceName + "." + tableName;
+ template.keyspaceName = new String(keyspaceName);
+ template.tableName = new String(tableName);
+ template.isIndex = false;
+ template.sstableCount = 0L;
+ template.oldSSTableCount = 0L;
+ template.spaceUsedLive = "0";
+ template.spaceUsedTotal = "0";
+ template.spaceUsedBySnapshotsTotal = "0";
+ template.percentRepaired = 1.0D;
+ template.bytesRepaired = 0L;
+ template.bytesUnrepaired = 0L;
+ template.bytesPendingRepair = 0L;
+ template.sstableCompressionRatio = -1.0D;
+ template.numberOfPartitionsEstimate = 0L;
+ template.memtableCellCount = 0L;
+ template.memtableDataSize = "0";
+ template.memtableSwitchCount = 0L;
+ template.localReadCount =0L;
+ template.localReadLatencyMs = Double.NaN;
+ template.localWriteCount = 0L;
+ template.localWriteLatencyMs = 0D;
+ template.pendingFlushes = 0L;
+ template.bloomFilterFalsePositives = 0L;
+ template.bloomFilterFalseRatio = 0D;
+ template.bloomFilterSpaceUsed = "0";
+ template.indexSummaryOffHeapMemoryUsed = "0";
+ template.compressionMetadataOffHeapMemoryUsed = "0";
+ template.compactedPartitionMinimumBytes = 0L;
+ template.compactedPartitionMaximumBytes = 0L;
+ template.compactedPartitionMeanBytes = 0L;
+ template.bytesRepaired = 0L;
+ template.bytesUnrepaired = 0L;
+ template.bytesPendingRepair = 0L;
+ template.averageLiveCellsPerSliceLastFiveMinutes = Double.NaN;
+ template.maximumLiveCellsPerSliceLastFiveMinutes = 0L;
+ template.averageTombstonesPerSliceLastFiveMinutes = Double.NaN;
+ template.maximumTombstonesPerSliceLastFiveMinutes = 0L;
+ template.droppedMutations = "0";
+ return template;
+ }
+
+ @BeforeClass
+ public static void createTestVector()
+ {
+ // create test tables from templates
+ StatsTable table1 = createStatsTableTemplate("keyspace1", "table1");
+ StatsTable table2 = createStatsTableTemplate("keyspace1", "table2");
+ StatsTable table3 = createStatsTableTemplate("keyspace1", "table3");
+ StatsTable table4 = createStatsTableTemplate("keyspace2", "table4");
+ StatsTable table5 = createStatsTableTemplate("keyspace2", "table5");
+ StatsTable table6 = createStatsTableTemplate("keyspace3", "table6");
+ // average live cells: 1 > 6 > 2 > 5 > 3 > 4
+ table1.averageLiveCellsPerSliceLastFiveMinutes = 6D;
+ table2.averageLiveCellsPerSliceLastFiveMinutes = 4.01D;
+ table3.averageLiveCellsPerSliceLastFiveMinutes = 0D;
+ table4.averageLiveCellsPerSliceLastFiveMinutes = Double.NaN;
+ table5.averageLiveCellsPerSliceLastFiveMinutes = 4D;
+ table6.averageLiveCellsPerSliceLastFiveMinutes = 5D;
+ // average tombstones: 6 > 1 > 5 > 2 > 3 > 4
+ table1.averageTombstonesPerSliceLastFiveMinutes = 5D;
+ table2.averageTombstonesPerSliceLastFiveMinutes = 4.001D;
+ table3.averageTombstonesPerSliceLastFiveMinutes = Double.NaN;
+ table4.averageTombstonesPerSliceLastFiveMinutes = 0D;
+ table5.averageTombstonesPerSliceLastFiveMinutes = 4.01D;
+ table6.averageTombstonesPerSliceLastFiveMinutes = 6D;
+ // bloom filter false positives: 2 > 4 > 6 > 1 > 3 > 5
+ table1.bloomFilterFalsePositives = (Object) 30L;
+ table2.bloomFilterFalsePositives = (Object) 600L;
+ table3.bloomFilterFalsePositives = (Object) 20L;
+ table4.bloomFilterFalsePositives = (Object) 500L;
+ table5.bloomFilterFalsePositives = (Object) 10L;
+ table6.bloomFilterFalsePositives = (Object) 400L;
+ // bloom filter false positive ratio: 5 > 3 > 1 > 6 > 4 > 2
+ table1.bloomFilterFalseRatio = (Object) 0.40D;
+ table2.bloomFilterFalseRatio = (Object) 0.01D;
+ table3.bloomFilterFalseRatio = (Object) 0.50D;
+ table4.bloomFilterFalseRatio = (Object) 0.02D;
+ table5.bloomFilterFalseRatio = (Object) 0.60D;
+ table6.bloomFilterFalseRatio = (Object) 0.03D;
+ // bloom filter space used: 2 > 4 > 6 > 1 > 3 > 5
+ table1.bloomFilterSpaceUsed = "789";
+ table2.bloomFilterSpaceUsed = "161718";
+ table3.bloomFilterSpaceUsed = "456";
+ table4.bloomFilterSpaceUsed = "131415";
+ table5.bloomFilterSpaceUsed = "123";
+ table6.bloomFilterSpaceUsed = "101112";
+ // compacted partition maximum bytes: 1 > 3 > 5 > 2 > 4 = 6
+ table1.compactedPartitionMaximumBytes = 60L;
+ table2.compactedPartitionMaximumBytes = 30L;
+ table3.compactedPartitionMaximumBytes = 50L;
+ table4.compactedPartitionMaximumBytes = 20L;
+ table5.compactedPartitionMaximumBytes = 40L;
+ table6.compactedPartitionMaximumBytes = 20L;
+ // compacted partition mean bytes: 1 > 3 > 2 = 4 = 5 > 6
+ table1.compactedPartitionMeanBytes = 6L;
+ table2.compactedPartitionMeanBytes = 4L;
+ table3.compactedPartitionMeanBytes = 5L;
+ table4.compactedPartitionMeanBytes = 4L;
+ table5.compactedPartitionMeanBytes = 4L;
+ table6.compactedPartitionMeanBytes = 3L;
+ // compacted partition minimum bytes: 6 > 4 > 2 > 5 > 1 = 3
+ table1.compactedPartitionMinimumBytes = 2L;
+ table2.compactedPartitionMinimumBytes = 4L;
+ table3.compactedPartitionMinimumBytes = 2L;
+ table4.compactedPartitionMinimumBytes = 5L;
+ table5.compactedPartitionMinimumBytes = 3L;
+ table6.compactedPartitionMinimumBytes = 6L;
+ // dropped mutations: 6 > 3 > 4 > 2 > 1 = 5
+ table1.droppedMutations = "0";
+ table2.droppedMutations = "222";
+ table3.droppedMutations = "33333";
+ table4.droppedMutations = "4444";
+ table5.droppedMutations = "0";
+ table6.droppedMutations = "666666";
+ // local reads: 6 > 5 > 4 > 3 > 2 > 1
+ table1.localReadCount = 0L;
+ table2.localReadCount = 1L;
+ table3.localReadCount = 2L;
+ table4.localReadCount = 3L;
+ table5.localReadCount = 4L;
+ table6.localReadCount = 5L;
+ // local read latency: 3 > 2 > 1 > 6 > 4 > 5
+ table1.localReadLatencyMs = 2D;
+ table2.localReadLatencyMs = 3D;
+ table3.localReadLatencyMs = 4D;
+ table4.localReadLatencyMs = Double.NaN;
+ table5.localReadLatencyMs = 0D;
+ table6.localReadLatencyMs = 1D;
+ // local writes: 1 > 2 > 3 > 4 > 5 > 6
+ table1.localWriteCount = 5L;
+ table2.localWriteCount = 4L;
+ table3.localWriteCount = 3L;
+ table4.localWriteCount = 2L;
+ table5.localWriteCount = 1L;
+ table6.localWriteCount = 0L;
+ // local write latency: 4 > 5 > 6 > 1 > 2 > 3
+ table1.localWriteLatencyMs = 0.05D;
+ table2.localWriteLatencyMs = 0D;
+ table3.localWriteLatencyMs = Double.NaN;
+ table4.localWriteLatencyMs = 2D;
+ table5.localWriteLatencyMs = 1D;
+ table6.localWriteLatencyMs = 0.5D;
+ // maximum live cells last five minutes: 1 > 2 = 3 > 4 = 5 > 6
+ table1.maximumLiveCellsPerSliceLastFiveMinutes = 6L;
+ table2.maximumLiveCellsPerSliceLastFiveMinutes = 5L;
+ table3.maximumLiveCellsPerSliceLastFiveMinutes = 5L;
+ table4.maximumLiveCellsPerSliceLastFiveMinutes = 3L;
+ table5.maximumLiveCellsPerSliceLastFiveMinutes = 3L;
+ table6.maximumLiveCellsPerSliceLastFiveMinutes = 2L;
+ // maximum tombstones last five minutes: 6 > 5 > 3 = 4 > 2 > 1
+ table1.maximumTombstonesPerSliceLastFiveMinutes = 1L;
+ table2.maximumTombstonesPerSliceLastFiveMinutes = 2L;
+ table3.maximumTombstonesPerSliceLastFiveMinutes = 3L;
+ table4.maximumTombstonesPerSliceLastFiveMinutes = 3L;
+ table5.maximumTombstonesPerSliceLastFiveMinutes = 5L;
+ table6.maximumTombstonesPerSliceLastFiveMinutes = 6L;
+ // memtable cell count: 3 > 5 > 6 > 1 > 2 > 4
+ table1.memtableCellCount = (Object) 111L;
+ table2.memtableCellCount = (Object) 22L;
+ table3.memtableCellCount = (Object) 333333L;
+ table4.memtableCellCount = (Object) 4L;
+ table5.memtableCellCount = (Object) 55555L;
+ table6.memtableCellCount = (Object) 6666L;
+ // memtable data size: 6 > 5 > 4 > 3 > 2 > 1
+ table1.memtableDataSize = "0";
+ table2.memtableDataSize = "900";
+ table3.memtableDataSize = "1999";
+ table4.memtableDataSize = "3000";
+ table5.memtableDataSize = "20000";
+ table6.memtableDataSize = "1000000";
+ // memtable switch count: 4 > 2 > 3 > 6 > 5 > 1
+ table1.memtableSwitchCount = (Object) 1L;
+ table2.memtableSwitchCount = (Object) 22222L;
+ table3.memtableSwitchCount = (Object) 3333L;
+ table4.memtableSwitchCount = (Object) 444444L;
+ table5.memtableSwitchCount = (Object) 5L;
+ table6.memtableSwitchCount = (Object) 6L;
+ // number of partitions estimate: 1 > 2 > 3 > 4 > 5 > 6
+ table1.numberOfPartitionsEstimate = (Object) 111111L;
+ table2.numberOfPartitionsEstimate = (Object) 22222L;
+ table3.numberOfPartitionsEstimate = (Object) 3333L;
+ table4.numberOfPartitionsEstimate = (Object) 444L;
+ table5.numberOfPartitionsEstimate = (Object) 55L;
+ table6.numberOfPartitionsEstimate = (Object) 6L;
+ // pending flushes: 2 > 1 > 4 > 3 > 6 > 5
+ table1.pendingFlushes = (Object) 11111L;
+ table2.pendingFlushes = (Object) 222222L;
+ table3.pendingFlushes = (Object) 333L;
+ table4.pendingFlushes = (Object) 4444L;
+ table5.pendingFlushes = (Object) 5L;
+ table6.pendingFlushes = (Object) 66L;
+ // percent repaired: 1 > 2 > 3 > 5 > 4 > 6
+ table1.percentRepaired = 100.0D;
+ table2.percentRepaired = 99.9D;
+ table3.percentRepaired = 99.8D;
+ table4.percentRepaired = 50.0D;
+ table5.percentRepaired = 93.0D;
+ table6.percentRepaired = 0.0D;
+ // space used by snapshots: 5 > 1 > 2 > 4 > 3 = 6
+ table1.spaceUsedBySnapshotsTotal = "1111";
+ table2.spaceUsedBySnapshotsTotal = "222";
+ table3.spaceUsedBySnapshotsTotal = "0";
+ table4.spaceUsedBySnapshotsTotal = "44";
+ table5.spaceUsedBySnapshotsTotal = "55555";
+ table6.spaceUsedBySnapshotsTotal = "0";
+ // space used live: 6 > 5 > 4 > 2 > 1 = 3
+ table1.spaceUsedLive = "0";
+ table2.spaceUsedLive = "22";
+ table3.spaceUsedLive = "0";
+ table4.spaceUsedLive = "4444";
+ table5.spaceUsedLive = "55555";
+ table6.spaceUsedLive = "666666";
+ // space used total: 1 > 2 > 3 > 4 > 5 > 6
+ table1.spaceUsedTotal = "9001";
+ table2.spaceUsedTotal = "1024";
+ table3.spaceUsedTotal = "512";
+ table4.spaceUsedTotal = "256";
+ table5.spaceUsedTotal = "64";
+ table6.spaceUsedTotal = "0";
+ // sstable compression ratio: 5 > 4 > 1 = 2 = 6 > 3
+ table1.sstableCompressionRatio = (Object) 0.68D;
+ table2.sstableCompressionRatio = (Object) 0.68D;
+ table3.sstableCompressionRatio = (Object) 0.32D;
+ table4.sstableCompressionRatio = (Object) 0.95D;
+ table5.sstableCompressionRatio = (Object) 0.99D;
+ table6.sstableCompressionRatio = (Object) 0.68D;
+ // sstable count: 1 > 3 > 5 > 2 > 4 > 6
+ table1.sstableCount = (Object) 60000;
+ table2.sstableCount = (Object) 3000;
+ table3.sstableCount = (Object) 50000;
+ table4.sstableCount = (Object) 2000;
+ table5.sstableCount = (Object) 40000;
+ table6.sstableCount = (Object) 1000;
+ // set even numbered tables to have some offheap usage
+ table2.offHeapUsed = true;
+ table4.offHeapUsed = true;
+ table6.offHeapUsed = true;
+ table2.memtableOffHeapUsed = true;
+ table4.memtableOffHeapUsed = true;
+ table6.memtableOffHeapUsed = true;
+ table2.bloomFilterOffHeapUsed = true;
+ table4.bloomFilterOffHeapUsed = true;
+ table6.bloomFilterOffHeapUsed = true;
+ table2.compressionMetadataOffHeapUsed = true;
+ table4.compressionMetadataOffHeapUsed = true;
+ table6.compressionMetadataOffHeapUsed = true;
+ table2.indexSummaryOffHeapUsed = true;
+ table4.indexSummaryOffHeapUsed = true;
+ table6.indexSummaryOffHeapUsed = true;
+ // offheap memory total: 4 > 2 > 6 > 1 = 3 = 5
+ table2.offHeapMemoryUsedTotal = "314159367";
+ table4.offHeapMemoryUsedTotal = "441213818";
+ table6.offHeapMemoryUsedTotal = "162470810";
+ // bloom filter offheap: 4 > 6 > 2 > 1 = 3 = 5
+ table2.bloomFilterOffHeapMemoryUsed = "98";
+ table4.bloomFilterOffHeapMemoryUsed = "299792458";
+ table6.bloomFilterOffHeapMemoryUsed = "667408";
+ // compression metadata offheap: 2 > 4 > 6 > 1 = 3 = 5
+ table2.compressionMetadataOffHeapMemoryUsed = "3";
+ table4.compressionMetadataOffHeapMemoryUsed = "2";
+ table6.compressionMetadataOffHeapMemoryUsed = "1";
+ // index summary offheap: 6 > 4 > 2 > 1 = 3 = 5
+ table2.indexSummaryOffHeapMemoryUsed = "1";
+ table4.indexSummaryOffHeapMemoryUsed = "2";
+ table6.indexSummaryOffHeapMemoryUsed = "3";
+ // memtable offheap: 2 > 6 > 4 > 1 = 3 = 5
+ table2.memtableOffHeapMemoryUsed = "314159265";
+ table4.memtableOffHeapMemoryUsed = "141421356";
+ table6.memtableOffHeapMemoryUsed = "161803398";
+ // create test keyspaces from templates
+ testKeyspaces = new ArrayList<StatsKeyspace>();
+ StatsKeyspace keyspace1 = createStatsKeyspaceTemplate("keyspace1");
+ StatsKeyspace keyspace2 = createStatsKeyspaceTemplate("keyspace2");
+ StatsKeyspace keyspace3 = createStatsKeyspaceTemplate("keyspace3");
+ // populate StatsKeyspace tables lists
+ keyspace1.tables.add(table1);
+ keyspace1.tables.add(table2);
+ keyspace1.tables.add(table3);
+ keyspace2.tables.add(table4);
+ keyspace2.tables.add(table5);
+ keyspace3.tables.add(table6);
+ // populate testKeyspaces test vector
+ testKeyspaces.add(keyspace1);
+ testKeyspaces.add(keyspace2);
+ testKeyspaces.add(keyspace3);
+ // compute keyspace statistics from relevant table metrics
+ for (int i = 0; i < testKeyspaces.size(); i++)
+ {
+ StatsKeyspace ks = testKeyspaces.get(i);
+ for (StatsTable st : ks.tables)
+ {
+ ks.readCount += st.localReadCount;
+ ks.writeCount += st.localWriteCount;
+ ks.pendingFlushes += (long) st.pendingFlushes;
+ }
+ testKeyspaces.set(i, ks);
+ }
+ // populate testTables test vector
+ testTables = new ArrayList<StatsTable>();
+ testTables.add(table1);
+ testTables.add(table2);
+ testTables.add(table3);
+ testTables.add(table4);
+ testTables.add(table5);
+ testTables.add(table6);
+ //
+ // create test vector for human readable case
+ StatsTable humanReadableTable1 = createStatsTableTemplate("keyspace1", "table1");
+ StatsTable humanReadableTable2 = createStatsTableTemplate("keyspace1", "table2");
+ StatsTable humanReadableTable3 = createStatsTableTemplate("keyspace1", "table3");
+ StatsTable humanReadableTable4 = createStatsTableTemplate("keyspace2", "table4");
+ StatsTable humanReadableTable5 = createStatsTableTemplate("keyspace2", "table5");
+ StatsTable humanReadableTable6 = createStatsTableTemplate("keyspace3", "table6");
+ // human readable space used total: 6 > 5 > 4 > 3 > 2 > 1
+ humanReadableTable1.spaceUsedTotal = "999 bytes";
+ humanReadableTable2.spaceUsedTotal = "5 KiB";
+ humanReadableTable3.spaceUsedTotal = "40 KiB";
+ humanReadableTable4.spaceUsedTotal = "3 MiB";
+ humanReadableTable5.spaceUsedTotal = "2 GiB";
+ humanReadableTable6.spaceUsedTotal = "1 TiB";
+ // human readable memtable data size: 1 > 3 > 5 > 2 > 4 > 6
+ humanReadableTable1.memtableDataSize = "1.21 TiB";
+ humanReadableTable2.memtableDataSize = "42 KiB";
+ humanReadableTable3.memtableDataSize = "2.71 GiB";
+ humanReadableTable4.memtableDataSize = "999 bytes";
+ humanReadableTable5.memtableDataSize = "3.14 MiB";
+ humanReadableTable6.memtableDataSize = "0 bytes";
+ // create human readable keyspaces from template
+ humanReadableKeyspaces = new ArrayList<StatsKeyspace>();
+ StatsKeyspace humanReadableKeyspace1 = createStatsKeyspaceTemplate("keyspace1");
+ StatsKeyspace humanReadableKeyspace2 = createStatsKeyspaceTemplate("keyspace2");
+ StatsKeyspace humanReadableKeyspace3 = createStatsKeyspaceTemplate("keyspace3");
+ // populate human readable StatsKeyspace tables lists
+ humanReadableKeyspace1.tables.add(humanReadableTable1);
+ humanReadableKeyspace1.tables.add(humanReadableTable2);
+ humanReadableKeyspace1.tables.add(humanReadableTable3);
+ humanReadableKeyspace2.tables.add(humanReadableTable4);
+ humanReadableKeyspace2.tables.add(humanReadableTable5);
+ humanReadableKeyspace3.tables.add(humanReadableTable6);
+ // populate human readable keyspaces test vector
+ humanReadableKeyspaces.add(humanReadableKeyspace1);
+ humanReadableKeyspaces.add(humanReadableKeyspace2);
+ humanReadableKeyspaces.add(humanReadableKeyspace3);
+ // compute human readable keyspace statistics from relevant table metrics
+ for (int i = 0; i < humanReadableKeyspaces.size(); i++)
+ {
+ StatsKeyspace ks = humanReadableKeyspaces.get(i);
+ for (StatsTable st : ks.tables)
+ {
+ ks.readCount += st.localReadCount;
+ ks.writeCount += st.localWriteCount;
+ ks.pendingFlushes += (long) st.pendingFlushes;
+ }
+ humanReadableKeyspaces.set(i, ks);
+ }
+ // populate human readable tables test vector
+ humanReadableTables = new ArrayList<StatsTable>();
+ humanReadableTables.add(humanReadableTable1);
+ humanReadableTables.add(humanReadableTable2);
+ humanReadableTables.add(humanReadableTable3);
+ humanReadableTables.add(humanReadableTable4);
+ humanReadableTables.add(humanReadableTable5);
+ humanReadableTables.add(humanReadableTable6);
+ }
+}
diff --git a/test/unit/org/apache/cassandra/tracing/TracingTest.java b/test/unit/org/apache/cassandra/tracing/TracingTest.java
index f546496..61e08b0 100644
--- a/test/unit/org/apache/cassandra/tracing/TracingTest.java
+++ b/test/unit/org/apache/cassandra/tracing/TracingTest.java
@@ -31,6 +31,7 @@
import org.junit.Test;
import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.utils.progress.ProgressEvent;
import org.apache.commons.lang3.StringUtils;
@@ -197,7 +198,7 @@
return super.newSession(sessionId, traceType, customPayload);
}
- protected TraceState newTraceState(InetAddress ia, UUID uuid, Tracing.TraceType tt)
+ protected TraceState newTraceState(InetAddressAndPort ia, UUID uuid, Tracing.TraceType tt)
{
return new TraceState(ia, uuid, tt)
{
diff --git a/test/unit/org/apache/cassandra/transport/CBUtilTest.java b/test/unit/org/apache/cassandra/transport/CBUtilTest.java
new file mode 100644
index 0000000..c20efec
--- /dev/null
+++ b/test/unit/org/apache/cassandra/transport/CBUtilTest.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.transport;
+
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Test;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.ByteBufAllocator;
+import io.netty.buffer.PooledByteBufAllocator;
+
+public class CBUtilTest
+{
+ private static final ByteBufAllocator allocator = PooledByteBufAllocator.DEFAULT;
+ private ByteBuf buf;
+
+ @After
+ public void tearDown()
+ {
+ if (buf != null && buf.refCnt() > 0)
+ buf.release(buf.refCnt());
+ }
+
+ @Test
+ public void writeAndReadString()
+ {
+ final String text = "if you're happy and you know it, write your tests";
+ int size = CBUtil.sizeOfString(text);
+
+ buf = allocator.heapBuffer(size);
+ CBUtil.writeString(text, buf);
+ Assert.assertEquals(size, buf.writerIndex());
+ Assert.assertEquals(0, buf.readerIndex());
+ Assert.assertEquals(text, CBUtil.readString(buf));
+ Assert.assertEquals(buf.writerIndex(), buf.readerIndex());
+ }
+
+ @Test
+ public void writeAndReadLongString()
+ {
+ final String text = "if you're happy and you know it, write your tests";
+ int size = CBUtil.sizeOfLongString(text);
+
+ buf = allocator.heapBuffer(size);
+ CBUtil.writeLongString(text, buf);
+ Assert.assertEquals(size, buf.writerIndex());
+ Assert.assertEquals(0, buf.readerIndex());
+ Assert.assertEquals(text, CBUtil.readLongString(buf));
+ Assert.assertEquals(buf.writerIndex(), buf.readerIndex());
+ }
+
+ @Test
+ public void writeAndReadAsciiString()
+ {
+ StringBuilder sb = new StringBuilder();
+ for (int i = 1; i < 128; i++)
+ sb.append((char) i);
+ String write = sb.toString();
+ int size = CBUtil.sizeOfString(write);
+ buf = allocator.heapBuffer(size);
+ CBUtil.writeAsciiString(write, buf);
+ String read = CBUtil.readString(buf);
+ Assert.assertEquals(write, read);
+ }
+
+ @Test
+ public void writeAndReadAsciiStringMismatchWithNonUSAscii()
+ {
+ String invalidAsciiStr = "\u0080 \u0123 \u0321"; // a valid string contains no char > 0x007F
+ int size = CBUtil.sizeOfString(invalidAsciiStr);
+ buf = allocator.heapBuffer(size);
+ CBUtil.writeAsciiString(invalidAsciiStr, buf);
+ Assert.assertNotEquals("Characters (> 0x007F) is considered as 2 bytes in sizeOfString, meanwhile writeAsciiString writes just 1 byte",
+ size,
+ buf.writerIndex());
+ }
+}
diff --git a/test/unit/org/apache/cassandra/transport/CQLUserAuditTest.java b/test/unit/org/apache/cassandra/transport/CQLUserAuditTest.java
new file mode 100644
index 0000000..32717bf
--- /dev/null
+++ b/test/unit/org/apache/cassandra/transport/CQLUserAuditTest.java
@@ -0,0 +1,256 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.transport;
+
+import java.io.Serializable;
+import java.net.InetAddress;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+import com.datastax.driver.core.Cluster;
+import com.datastax.driver.core.PreparedStatement;
+import com.datastax.driver.core.Session;
+import com.datastax.driver.core.exceptions.AuthenticationException;
+import org.apache.cassandra.OrderedJUnit4ClassRunner;
+import org.apache.cassandra.audit.AuditEvent;
+import org.apache.cassandra.audit.AuditLogEntryType;
+import org.apache.cassandra.audit.AuditLogManager;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.config.OverrideConfigurationLoader;
+import org.apache.cassandra.config.ParameterizedClass;
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.diag.DiagnosticEventService;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.service.EmbeddedCassandraService;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+
+@RunWith(OrderedJUnit4ClassRunner.class)
+public class CQLUserAuditTest
+{
+ private static EmbeddedCassandraService embedded;
+ private static final BlockingQueue<AuditEvent> auditEvents = new LinkedBlockingQueue<>();
+
+ @BeforeClass
+ public static void setup() throws Exception
+ {
+ OverrideConfigurationLoader.override((config) -> {
+ config.authenticator = "PasswordAuthenticator";
+ config.role_manager = "CassandraRoleManager";
+ config.diagnostic_events_enabled = true;
+ config.audit_logging_options.enabled = true;
+ config.audit_logging_options.logger = new ParameterizedClass("DiagnosticEventAuditLogger", null);
+ });
+ CQLTester.prepareServer();
+
+ System.setProperty("cassandra.superuser_setup_delay_ms", "0");
+ embedded = new EmbeddedCassandraService();
+ embedded.start();
+
+ executeAs(Arrays.asList("CREATE ROLE testuser WITH LOGIN = true AND SUPERUSER = false AND PASSWORD = 'foo'",
+ "CREATE ROLE testuser_nologin WITH LOGIN = false AND SUPERUSER = false AND PASSWORD = 'foo'",
+ "CREATE KEYSPACE testks WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'}",
+ "CREATE TABLE testks.table1 (a text, b int, c int, PRIMARY KEY (a, b))",
+ "CREATE TABLE testks.table2 (a text, b int, c int, PRIMARY KEY (a, b))"),
+ "cassandra", "cassandra", null);
+
+ DiagnosticEventService.instance().subscribe(AuditEvent.class, auditEvents::add);
+ AuditLogManager.instance.initialize();
+ }
+
+ @AfterClass
+ public static void shutdown()
+ {
+ embedded.stop();
+ }
+
+ @After
+ public void clearQueue()
+ {
+ auditEvents.clear();
+ }
+
+ @Test
+ public void loginWrongPasswordTest() throws Throwable
+ {
+ executeAs(Collections.emptyList(), "testuser", "wrongpassword", AuditLogEntryType.LOGIN_ERROR);
+ }
+
+ @Test
+ public void loginWrongUsernameTest() throws Throwable
+ {
+ executeAs(Collections.emptyList(), "wronguser", "foo", AuditLogEntryType.LOGIN_ERROR);
+ }
+
+ @Test
+ public void loginDeniedTest() throws Throwable
+ {
+ executeAs(Collections.emptyList(), "testuser_nologin", "foo", AuditLogEntryType.LOGIN_ERROR);
+ }
+
+ @Test
+ public void loginSuccessfulTest() throws Throwable
+ {
+ executeAs(Collections.emptyList(), "testuser", "foo", AuditLogEntryType.LOGIN_SUCCESS);
+ }
+
+ @Test
+ public void querySimpleSelect() throws Throwable
+ {
+ ArrayList<AuditEvent> events = executeAs(Arrays.asList("SELECT * FROM testks.table1"),
+ "testuser", "foo", AuditLogEntryType.LOGIN_SUCCESS);
+ assertEquals(1, events.size());
+ AuditEvent e = events.get(0);
+ Map<String, Serializable> m = e.toMap();
+ assertEquals("testuser", m.get("user"));
+ assertEquals("SELECT * FROM testks.table1", m.get("operation"));
+ assertEquals("testks", m.get("keyspace"));
+ assertEquals("table1", m.get("scope"));
+ assertEquals(AuditLogEntryType.SELECT, e.getType());
+ }
+
+ @Test
+ public void queryInsert() throws Throwable
+ {
+ ArrayList<AuditEvent> events = executeAs(Arrays.asList("INSERT INTO testks.table1 (a, b, c) VALUES ('a', 1, 1)"),
+ "testuser", "foo", AuditLogEntryType.LOGIN_SUCCESS);
+ assertEquals(1, events.size());
+ AuditEvent e = events.get(0);
+ Map<String, Serializable> m = e.toMap();
+ assertEquals("testuser", m.get("user"));
+ assertEquals("INSERT INTO testks.table1 (a, b, c) VALUES ('a', 1, 1)", m.get("operation"));
+ assertEquals("testks", m.get("keyspace"));
+ assertEquals("table1", m.get("scope"));
+ assertEquals(AuditLogEntryType.UPDATE, e.getType());
+ }
+
+ @Test
+ public void queryBatch() throws Throwable
+ {
+ String query = "BEGIN BATCH "
+ + "INSERT INTO testks.table1 (a, b, c) VALUES ('a', 1, 1); "
+ + "INSERT INTO testks.table1 (a, b, c) VALUES ('b', 1, 1); "
+ + "INSERT INTO testks.table1 (a, b, c) VALUES ('b', 2, 2); "
+ + "APPLY BATCH;";
+ ArrayList<AuditEvent> events = executeAs(Arrays.asList(query),
+ "testuser", "foo",
+ AuditLogEntryType.LOGIN_SUCCESS);
+ assertEquals(1, events.size());
+ AuditEvent e = events.get(0);
+ Map<String, Serializable> m = e.toMap();
+ assertEquals("testuser", m.get("user"));
+ assertEquals(query, m.get("operation"));
+ assertEquals(AuditLogEntryType.BATCH, e.getType());
+ }
+
+ @Test
+ public void prepareStmt()
+ {
+ Cluster cluster = Cluster.builder().addContactPoints(InetAddress.getLoopbackAddress())
+ .withoutJMXReporting()
+ .withCredentials("testuser", "foo")
+ .withPort(DatabaseDescriptor.getNativeTransportPort()).build();
+ String spStmt = "INSERT INTO testks.table1 (a, b, c) VALUES (?, ?, ?)";
+ try (Session session = cluster.connect())
+ {
+ PreparedStatement pStmt = session.prepare(spStmt);
+ session.execute(pStmt.bind("x", 9, 8));
+ }
+
+ List<AuditEvent> events = auditEvents.stream().filter((e) -> e.getType() != AuditLogEntryType.LOGIN_SUCCESS)
+ .collect(Collectors.toList());
+ AuditEvent e = events.get(0);
+ Map<String, Serializable> m = e.toMap();
+ assertEquals(2, events.size());
+ assertEquals("testuser", m.get("user"));
+ assertEquals(spStmt, m.get("operation"));
+ assertEquals("testks", m.get("keyspace"));
+ assertEquals("table1", m.get("scope"));
+ assertEquals(AuditLogEntryType.PREPARE_STATEMENT, e.getType());
+
+ e = events.get(1);
+ m = e.toMap();
+ assertEquals("testuser", m.get("user"));
+ assertEquals(spStmt, m.get("operation"));
+ assertEquals("testks", m.get("keyspace"));
+ assertEquals("table1", m.get("scope"));
+ assertEquals(AuditLogEntryType.UPDATE, e.getType());
+
+ }
+
+ private static ArrayList<AuditEvent> executeAs(List<String> queries, String username, String password,
+ AuditLogEntryType expectedAuthType) throws Exception
+ {
+ boolean authFailed = false;
+ Cluster cluster = Cluster.builder().addContactPoints(InetAddress.getLoopbackAddress())
+ .withoutJMXReporting()
+ .withCredentials(username, password)
+ .withPort(DatabaseDescriptor.getNativeTransportPort()).build();
+ try (Session session = cluster.connect())
+ {
+ for (String query : queries)
+ session.execute(query);
+ }
+ catch (AuthenticationException e)
+ {
+ authFailed = true;
+ }
+ cluster.close();
+
+ if (expectedAuthType == null) return null;
+
+ AuditEvent event = auditEvents.poll(100, TimeUnit.MILLISECONDS);
+ assertEquals(expectedAuthType, event.getType());
+ assertTrue(!authFailed || event.getType() == AuditLogEntryType.LOGIN_ERROR);
+ assertEquals(InetAddressAndPort.getLoopbackAddress().address,
+ event.getEntry().getSource().address);
+ assertTrue(event.getEntry().getSource().port > 0);
+ if (event.getType() != AuditLogEntryType.LOGIN_ERROR)
+ assertEquals(username, event.toMap().get("user"));
+
+ // drain all remaining login related events, as there's no specification how connections and login attempts
+ // should be handled by the driver, so we can't assert a fixed number of login events
+ for (AuditEvent e = auditEvents.peek();
+ e != null && (e.getType() == AuditLogEntryType.LOGIN_ERROR
+ || e.getType() == AuditLogEntryType.LOGIN_SUCCESS);
+ e = auditEvents.peek())
+ {
+ auditEvents.remove(e);
+ }
+
+ ArrayList<AuditEvent> ret = new ArrayList<>(auditEvents.size());
+ auditEvents.drainTo(ret);
+ return ret;
+ }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/transport/DataTypeTest.java b/test/unit/org/apache/cassandra/transport/DataTypeTest.java
index 6f086c9..a8b535a 100644
--- a/test/unit/org/apache/cassandra/transport/DataTypeTest.java
+++ b/test/unit/org/apache/cassandra/transport/DataTypeTest.java
@@ -19,9 +19,7 @@
package org.apache.cassandra.transport;
import java.util.ArrayList;
-import java.util.Collections;
import java.util.List;
-import java.util.Map;
import org.junit.Test;
diff --git a/test/unit/org/apache/cassandra/transport/DynamicLimitTest.java b/test/unit/org/apache/cassandra/transport/DynamicLimitTest.java
deleted file mode 100644
index df06fba..0000000
--- a/test/unit/org/apache/cassandra/transport/DynamicLimitTest.java
+++ /dev/null
@@ -1,111 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.transport;
-
-import java.net.InetAddress;
-
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import org.apache.cassandra.cql3.CQLTester;
-
-import static org.apache.cassandra.transport.ProtocolTestHelper.cleanupPeers;
-import static org.apache.cassandra.transport.ProtocolTestHelper.setStaticLimitInConfig;
-import static org.apache.cassandra.transport.ProtocolTestHelper.setupPeer;
-import static org.apache.cassandra.transport.ProtocolTestHelper.updatePeerInfo;
-import static org.junit.Assert.assertEquals;
-
-public class DynamicLimitTest
-{
- @BeforeClass
- public static void setup()
- {
- CQLTester.prepareServer();
- }
-
- @Test
- public void disableDynamicLimitWithSystemProperty() throws Throwable
- {
- // Dynamic limiting of the max negotiable protocol version can be
- // disabled with a system property
-
- // ensure that no static limit is configured
- setStaticLimitInConfig(null);
-
- // set the property which disables dynamic limiting
- System.setProperty(ConfiguredLimit.DISABLE_MAX_PROTOCOL_AUTO_OVERRIDE, "true");
- // insert a legacy peer into system.peers and also
- InetAddress peer = null;
- try
- {
- peer = setupPeer("127.1.0.1", "2.2.0");
- ConfiguredLimit limit = ConfiguredLimit.newLimit();
- assertEquals(ProtocolVersion.MAX_SUPPORTED_VERSION, limit.getMaxVersion());
-
- // clearing the property after the limit has been returned has no effect
- System.clearProperty(ConfiguredLimit.DISABLE_MAX_PROTOCOL_AUTO_OVERRIDE);
- limit.updateMaxSupportedVersion();
- assertEquals(ProtocolVersion.MAX_SUPPORTED_VERSION, limit.getMaxVersion());
-
- // a new limit should now be dynamic
- limit = ConfiguredLimit.newLimit();
- assertEquals(ProtocolVersion.V3, limit.getMaxVersion());
- }
- finally
- {
- System.clearProperty(ConfiguredLimit.DISABLE_MAX_PROTOCOL_AUTO_OVERRIDE);
- cleanupPeers(peer);
- }
- }
-
- @Test
- public void disallowLoweringMaxVersion() throws Throwable
- {
- // Lowering the max version once connections have been established is a problem
- // for some clients. So for a dynamic limit, if notifications of peer versions
- // trigger a change to the max version, it's only allowed to increase the max
- // negotiable version
-
- InetAddress peer = null;
- try
- {
- // ensure that no static limit is configured
- setStaticLimitInConfig(null);
- ConfiguredLimit limit = ConfiguredLimit.newLimit();
- assertEquals(ProtocolVersion.MAX_SUPPORTED_VERSION, limit.getMaxVersion());
-
- peer = setupPeer("127.1.0.1", "3.0.0");
- limit.updateMaxSupportedVersion();
- assertEquals(ProtocolVersion.MAX_SUPPORTED_VERSION, limit.getMaxVersion());
-
- // learn that peer doesn't actually fully support V4, behaviour should remain the same
- updatePeerInfo(peer, "2.2.0");
- limit.updateMaxSupportedVersion();
- assertEquals(ProtocolVersion.MAX_SUPPORTED_VERSION, limit.getMaxVersion());
-
- // finally learn that peer2 has been upgraded, just for completeness
- updatePeerInfo(peer, "3.3.0");
- limit.updateMaxSupportedVersion();
- assertEquals(ProtocolVersion.MAX_SUPPORTED_VERSION, limit.getMaxVersion());
-
- } finally {
- cleanupPeers(peer);
- }
- }
-}
diff --git a/test/unit/org/apache/cassandra/transport/ErrorMessageTest.java b/test/unit/org/apache/cassandra/transport/ErrorMessageTest.java
index 2dcd2ac..cfeddba 100644
--- a/test/unit/org/apache/cassandra/transport/ErrorMessageTest.java
+++ b/test/unit/org/apache/cassandra/transport/ErrorMessageTest.java
@@ -18,7 +18,6 @@
package org.apache.cassandra.transport;
-import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.HashMap;
import java.util.Map;
@@ -26,33 +25,38 @@
import org.junit.BeforeClass;
import org.junit.Test;
-import io.netty.buffer.ByteBuf;
-import io.netty.buffer.Unpooled;
import org.apache.cassandra.db.ConsistencyLevel;
import org.apache.cassandra.db.WriteType;
+import org.apache.cassandra.exceptions.CasWriteTimeoutException;
+import org.apache.cassandra.exceptions.CasWriteUnknownResultException;
import org.apache.cassandra.exceptions.ReadFailureException;
import org.apache.cassandra.exceptions.RequestFailureReason;
import org.apache.cassandra.exceptions.WriteFailureException;
+import org.apache.cassandra.exceptions.WriteTimeoutException;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.transport.messages.EncodeAndDecodeTestBase;
import org.apache.cassandra.transport.messages.ErrorMessage;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
-public class ErrorMessageTest
+public class ErrorMessageTest extends EncodeAndDecodeTestBase<ErrorMessage>
{
- private static Map<InetAddress, RequestFailureReason> failureReasonMap1;
- private static Map<InetAddress, RequestFailureReason> failureReasonMap2;
+ private static Map<InetAddressAndPort, RequestFailureReason> failureReasonMap1;
+ private static Map<InetAddressAndPort, RequestFailureReason> failureReasonMap2;
@BeforeClass
public static void setUpFixtures() throws UnknownHostException
{
failureReasonMap1 = new HashMap<>();
- failureReasonMap1.put(InetAddress.getByName("127.0.0.1"), RequestFailureReason.READ_TOO_MANY_TOMBSTONES);
- failureReasonMap1.put(InetAddress.getByName("127.0.0.2"), RequestFailureReason.READ_TOO_MANY_TOMBSTONES);
- failureReasonMap1.put(InetAddress.getByName("127.0.0.3"), RequestFailureReason.UNKNOWN);
+ failureReasonMap1.put(InetAddressAndPort.getByName("127.0.0.1"), RequestFailureReason.READ_TOO_MANY_TOMBSTONES);
+ failureReasonMap1.put(InetAddressAndPort.getByName("127.0.0.2"), RequestFailureReason.READ_TOO_MANY_TOMBSTONES);
+ failureReasonMap1.put(InetAddressAndPort.getByName("127.0.0.3"), RequestFailureReason.UNKNOWN);
failureReasonMap2 = new HashMap<>();
- failureReasonMap2.put(InetAddress.getByName("127.0.0.1"), RequestFailureReason.UNKNOWN);
- failureReasonMap2.put(InetAddress.getByName("127.0.0.2"), RequestFailureReason.UNKNOWN);
+ failureReasonMap2.put(InetAddressAndPort.getByName("127.0.0.1"), RequestFailureReason.UNKNOWN);
+ failureReasonMap2.put(InetAddressAndPort.getByName("127.0.0.2"), RequestFailureReason.UNKNOWN);
}
@Test
@@ -63,7 +67,7 @@
boolean dataPresent = false;
ReadFailureException rfe = new ReadFailureException(consistencyLevel, receivedBlockFor, receivedBlockFor, dataPresent, failureReasonMap1);
- ErrorMessage deserialized = serializeAndGetDeserializedErrorMessage(ErrorMessage.fromException(rfe), ProtocolVersion.V5);
+ ErrorMessage deserialized = encodeThenDecode(ErrorMessage.fromException(rfe), ProtocolVersion.V5);
ReadFailureException deserializedRfe = (ReadFailureException) deserialized.error;
assertEquals(failureReasonMap1, deserializedRfe.failureReasonByEndpoint);
@@ -81,7 +85,7 @@
WriteType writeType = WriteType.SIMPLE;
WriteFailureException wfe = new WriteFailureException(consistencyLevel, receivedBlockFor, receivedBlockFor, writeType, failureReasonMap2);
- ErrorMessage deserialized = serializeAndGetDeserializedErrorMessage(ErrorMessage.fromException(wfe), ProtocolVersion.V5);
+ ErrorMessage deserialized = encodeThenDecode(ErrorMessage.fromException(wfe), ProtocolVersion.V5);
WriteFailureException deserializedWfe = (WriteFailureException) deserialized.error;
assertEquals(failureReasonMap2, deserializedWfe.failureReasonByEndpoint);
@@ -91,6 +95,81 @@
assertEquals(writeType, deserializedWfe.writeType);
}
+ @Test
+ public void testV5CasWriteTimeoutSerDeser()
+ {
+ int contentions = 1;
+ int receivedBlockFor = 3;
+ ConsistencyLevel consistencyLevel = ConsistencyLevel.SERIAL;
+ CasWriteTimeoutException ex = new CasWriteTimeoutException(WriteType.CAS, consistencyLevel, receivedBlockFor, receivedBlockFor, contentions);
+
+ ErrorMessage deserialized = encodeThenDecode(ErrorMessage.fromException(ex), ProtocolVersion.V5);
+ assertTrue(deserialized.error instanceof CasWriteTimeoutException);
+ CasWriteTimeoutException deserializedEx = (CasWriteTimeoutException) deserialized.error;
+
+ assertEquals(WriteType.CAS, deserializedEx.writeType);
+ assertEquals(contentions, deserializedEx.contentions);
+ assertEquals(consistencyLevel, deserializedEx.consistency);
+ assertEquals(receivedBlockFor, deserializedEx.received);
+ assertEquals(receivedBlockFor, deserializedEx.blockFor);
+ assertEquals(ex.getMessage(), deserializedEx.getMessage());
+ assertTrue(deserializedEx.getMessage().contains("CAS operation timed out - encountered contentions"));
+ }
+
+ @Test
+ public void testV4CasWriteTimeoutSerDeser()
+ {
+ int contentions = 1;
+ int receivedBlockFor = 3;
+ ConsistencyLevel consistencyLevel = ConsistencyLevel.SERIAL;
+ CasWriteTimeoutException ex = new CasWriteTimeoutException(WriteType.CAS, consistencyLevel, receivedBlockFor, receivedBlockFor, contentions);
+
+ ErrorMessage deserialized = encodeThenDecode(ErrorMessage.fromException(ex), ProtocolVersion.V4);
+ assertTrue(deserialized.error instanceof WriteTimeoutException);
+ assertFalse(deserialized.error instanceof CasWriteTimeoutException);
+ WriteTimeoutException deserializedEx = (WriteTimeoutException) deserialized.error;
+
+ assertEquals(WriteType.CAS, deserializedEx.writeType);
+ assertEquals(consistencyLevel, deserializedEx.consistency);
+ assertEquals(receivedBlockFor, deserializedEx.received);
+ assertEquals(receivedBlockFor, deserializedEx.blockFor);
+ }
+
+ @Test
+ public void testV5CasWriteResultUnknownSerDeser()
+ {
+ int receivedBlockFor = 3;
+ ConsistencyLevel consistencyLevel = ConsistencyLevel.SERIAL;
+ CasWriteUnknownResultException ex = new CasWriteUnknownResultException(consistencyLevel, receivedBlockFor, receivedBlockFor);
+
+ ErrorMessage deserialized = encodeThenDecode(ErrorMessage.fromException(ex), ProtocolVersion.V5);
+ assertTrue(deserialized.error instanceof CasWriteUnknownResultException);
+ CasWriteUnknownResultException deserializedEx = (CasWriteUnknownResultException) deserialized.error;
+
+ assertEquals(consistencyLevel, deserializedEx.consistency);
+ assertEquals(receivedBlockFor, deserializedEx.received);
+ assertEquals(receivedBlockFor, deserializedEx.blockFor);
+ assertEquals(ex.getMessage(), deserializedEx.getMessage());
+ assertTrue(deserializedEx.getMessage().contains("CAS operation result is unknown"));
+ }
+
+ @Test
+ public void testV4CasWriteResultUnknownSerDeser()
+ {
+ int receivedBlockFor = 3;
+ ConsistencyLevel consistencyLevel = ConsistencyLevel.SERIAL;
+ CasWriteUnknownResultException ex = new CasWriteUnknownResultException(consistencyLevel, receivedBlockFor, receivedBlockFor);
+
+ ErrorMessage deserialized = encodeThenDecode(ErrorMessage.fromException(ex), ProtocolVersion.V4);
+ assertTrue(deserialized.error instanceof WriteTimeoutException);
+ assertFalse(deserialized.error instanceof CasWriteUnknownResultException);
+ WriteTimeoutException deserializedEx = (WriteTimeoutException) deserialized.error;
+
+ assertEquals(consistencyLevel, deserializedEx.consistency);
+ assertEquals(receivedBlockFor, deserializedEx.received);
+ assertEquals(receivedBlockFor, deserializedEx.blockFor);
+ }
+
/**
* Make sure that the map passed in to create a Read/WriteFailureException is copied
* so later modifications to the map passed in don't affect the map in the exception.
@@ -102,20 +181,18 @@
@Test
public void testRequestFailureExceptionMakesCopy() throws UnknownHostException
{
- Map<InetAddress, RequestFailureReason> modifiableFailureReasons = new HashMap<>(failureReasonMap1);
+ Map<InetAddressAndPort, RequestFailureReason> modifiableFailureReasons = new HashMap<>(failureReasonMap1);
ReadFailureException rfe = new ReadFailureException(ConsistencyLevel.ALL, 3, 3, false, modifiableFailureReasons);
WriteFailureException wfe = new WriteFailureException(ConsistencyLevel.ALL, 3, 3, WriteType.SIMPLE, modifiableFailureReasons);
- modifiableFailureReasons.put(InetAddress.getByName("127.0.0.4"), RequestFailureReason.UNKNOWN);
+ modifiableFailureReasons.put(InetAddressAndPort.getByName("127.0.0.4"), RequestFailureReason.UNKNOWN);
assertEquals(failureReasonMap1, rfe.failureReasonByEndpoint);
assertEquals(failureReasonMap1, wfe.failureReasonByEndpoint);
}
- private ErrorMessage serializeAndGetDeserializedErrorMessage(ErrorMessage message, ProtocolVersion version)
+ protected Message.Codec<ErrorMessage> getCodec()
{
- ByteBuf buffer = Unpooled.buffer(ErrorMessage.codec.encodedSize(message, version));
- ErrorMessage.codec.encode(message, buffer, version);
- return ErrorMessage.codec.decode(buffer, version);
+ return ErrorMessage.codec;
}
}
diff --git a/test/unit/org/apache/cassandra/transport/IdleDisconnectTest.java b/test/unit/org/apache/cassandra/transport/IdleDisconnectTest.java
new file mode 100644
index 0000000..355959f
--- /dev/null
+++ b/test/unit/org/apache/cassandra/transport/IdleDisconnectTest.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.transport;
+
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.TimeUnit;
+
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.db.ConsistencyLevel;
+
+public class IdleDisconnectTest extends CQLTester
+{
+ private static final long TIMEOUT = 2000L;
+
+ @BeforeClass
+ public static void setUp()
+ {
+ requireNetwork();
+ DatabaseDescriptor.setNativeTransportIdleTimeout(TIMEOUT);
+ }
+
+ @Test
+ public void testIdleDisconnect() throws Throwable
+ {
+ DatabaseDescriptor.setNativeTransportIdleTimeout(TIMEOUT);
+ try (SimpleClient client = new SimpleClient(nativeAddr.getHostAddress(), nativePort))
+ {
+ long start = System.currentTimeMillis();
+ client.connect(false, false);
+ Assert.assertTrue(client.channel.isOpen());
+ CompletableFuture.runAsync(() -> {
+ while (!Thread.currentThread().isInterrupted() && client.channel.isOpen());
+ }).get(30, TimeUnit.SECONDS);
+ Assert.assertFalse(client.channel.isOpen());
+ Assert.assertTrue(System.currentTimeMillis() - start >= TIMEOUT);
+ }
+ }
+
+ @Test
+ public void testIdleDisconnectProlonged() throws Throwable
+ {
+ long sleepTime = 1000;
+ try (SimpleClient client = new SimpleClient(nativeAddr.getHostAddress(), nativePort))
+ {
+ long start = System.currentTimeMillis();
+ client.connect(false, false);
+ Assert.assertTrue(client.channel.isOpen());
+ Thread.sleep(sleepTime);
+ client.execute("SELECT * FROM system.peers", ConsistencyLevel.ONE);
+ CompletableFuture.runAsync(() -> {
+ while (!Thread.currentThread().isInterrupted() && client.channel.isOpen());
+ }).get(30, TimeUnit.SECONDS);
+ Assert.assertFalse(client.channel.isOpen());
+ Assert.assertTrue(System.currentTimeMillis() - start >= TIMEOUT + sleepTime);
+ }
+ }
+}
diff --git a/test/unit/org/apache/cassandra/transport/InflightRequestPayloadTrackerTest.java b/test/unit/org/apache/cassandra/transport/InflightRequestPayloadTrackerTest.java
index 21dfed8..3c18a75 100644
--- a/test/unit/org/apache/cassandra/transport/InflightRequestPayloadTrackerTest.java
+++ b/test/unit/org/apache/cassandra/transport/InflightRequestPayloadTrackerTest.java
@@ -37,6 +37,10 @@
@RunWith(OrderedJUnit4ClassRunner.class)
public class InflightRequestPayloadTrackerTest extends CQLTester
{
+
+ private static long LOW_LIMIT = 600L;
+ private static long HIGH_LIMIT = 5000000000L;
+
@BeforeClass
public static void setUp()
{
@@ -49,7 +53,7 @@
public static void tearDown()
{
DatabaseDescriptor.setNativeTransportMaxConcurrentRequestsInBytesPerIp(3000000000L);
- DatabaseDescriptor.setNativeTransportMaxConcurrentRequestsInBytes(5000000000L);
+ DatabaseDescriptor.setNativeTransportMaxConcurrentRequestsInBytes(HIGH_LIMIT);
}
@After
@@ -70,12 +74,13 @@
{
SimpleClient client = new SimpleClient(nativeAddr.getHostAddress(),
nativePort,
- ProtocolVersion.V4,
- new EncryptionOptions.ClientEncryptionOptions());
+ ProtocolVersion.V5,
+ true,
+ new EncryptionOptions());
try
{
- client.connect(false, true);
+ client.connect(false, false, true);
QueryOptions queryOptions = QueryOptions.create(
QueryOptions.DEFAULT.getConsistency(),
QueryOptions.DEFAULT.getValues(),
@@ -83,9 +88,10 @@
QueryOptions.DEFAULT.getPageSize(),
QueryOptions.DEFAULT.getPagingState(),
QueryOptions.DEFAULT.getSerialConsistency(),
- ProtocolVersion.V4);
+ ProtocolVersion.V5,
+ KEYSPACE);
- QueryMessage queryMessage = new QueryMessage(String.format("CREATE TABLE %s.atable (pk1 int PRIMARY KEY, v text)", KEYSPACE),
+ QueryMessage queryMessage = new QueryMessage("CREATE TABLE atable (pk1 int PRIMARY KEY, v text)",
queryOptions);
client.execute(queryMessage);
}
@@ -100,12 +106,13 @@
{
SimpleClient client = new SimpleClient(nativeAddr.getHostAddress(),
nativePort,
- ProtocolVersion.V4,
- new EncryptionOptions.ClientEncryptionOptions());
+ ProtocolVersion.V5,
+ true,
+ new EncryptionOptions());
try
{
- client.connect(false, false);
+ client.connect(false, false, false);
QueryOptions queryOptions = QueryOptions.create(
QueryOptions.DEFAULT.getConsistency(),
QueryOptions.DEFAULT.getValues(),
@@ -113,12 +120,13 @@
QueryOptions.DEFAULT.getPageSize(),
QueryOptions.DEFAULT.getPagingState(),
QueryOptions.DEFAULT.getSerialConsistency(),
- ProtocolVersion.V4);
+ ProtocolVersion.V5,
+ KEYSPACE);
- QueryMessage queryMessage = new QueryMessage(String.format("CREATE TABLE %s.atable (pk int PRIMARY KEY, v text)", KEYSPACE),
+ QueryMessage queryMessage = new QueryMessage("CREATE TABLE atable (pk int PRIMARY KEY, v text)",
queryOptions);
client.execute(queryMessage);
- queryMessage = new QueryMessage(String.format("SELECT * FROM %s.atable", KEYSPACE),
+ queryMessage = new QueryMessage("SELECT * FROM atable",
queryOptions);
client.execute(queryMessage);
}
@@ -133,12 +141,13 @@
{
SimpleClient client = new SimpleClient(nativeAddr.getHostAddress(),
nativePort,
- ProtocolVersion.V4,
- new EncryptionOptions.ClientEncryptionOptions());
+ ProtocolVersion.V5,
+ true,
+ new EncryptionOptions());
try
{
- client.connect(false, false);
+ client.connect(false, false, false);
QueryOptions queryOptions = QueryOptions.create(
QueryOptions.DEFAULT.getConsistency(),
QueryOptions.DEFAULT.getValues(),
@@ -146,13 +155,14 @@
QueryOptions.DEFAULT.getPageSize(),
QueryOptions.DEFAULT.getPagingState(),
QueryOptions.DEFAULT.getSerialConsistency(),
- ProtocolVersion.V4);
+ ProtocolVersion.V5,
+ KEYSPACE);
- QueryMessage queryMessage = new QueryMessage(String.format("CREATE TABLE %s.atable (pk int PRIMARY KEY, v text)", KEYSPACE),
+ QueryMessage queryMessage = new QueryMessage("CREATE TABLE atable (pk int PRIMARY KEY, v text)",
queryOptions);
client.execute(queryMessage);
- queryMessage = new QueryMessage(String.format("INSERT INTO %s.atable (pk, v) VALUES (1, 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa')", KEYSPACE),
+ queryMessage = new QueryMessage("INSERT INTO atable (pk, v) VALUES (1, 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa')",
queryOptions);
client.execute(queryMessage);
}
@@ -167,12 +177,13 @@
{
SimpleClient client = new SimpleClient(nativeAddr.getHostAddress(),
nativePort,
- ProtocolVersion.V4,
- new EncryptionOptions.ClientEncryptionOptions());
+ ProtocolVersion.V5,
+ true,
+ new EncryptionOptions());
try
{
- client.connect(false, true);
+ client.connect(false, false, true);
QueryOptions queryOptions = QueryOptions.create(
QueryOptions.DEFAULT.getConsistency(),
QueryOptions.DEFAULT.getValues(),
@@ -180,13 +191,14 @@
QueryOptions.DEFAULT.getPageSize(),
QueryOptions.DEFAULT.getPagingState(),
QueryOptions.DEFAULT.getSerialConsistency(),
- ProtocolVersion.V4);
+ ProtocolVersion.V5,
+ KEYSPACE);
- QueryMessage queryMessage = new QueryMessage(String.format("CREATE TABLE %s.atable (pk int PRIMARY KEY, v text)", KEYSPACE),
+ QueryMessage queryMessage = new QueryMessage("CREATE TABLE atable (pk int PRIMARY KEY, v text)",
queryOptions);
client.execute(queryMessage);
- queryMessage = new QueryMessage(String.format("INSERT INTO %s.atable (pk, v) VALUES (1, 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa')", KEYSPACE),
+ queryMessage = new QueryMessage("INSERT INTO atable (pk, v) VALUES (1, 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa')",
queryOptions);
try
{
@@ -209,12 +221,13 @@
{
SimpleClient client = new SimpleClient(nativeAddr.getHostAddress(),
nativePort,
- ProtocolVersion.V4,
- new EncryptionOptions.ClientEncryptionOptions());
+ ProtocolVersion.V5,
+ true,
+ new EncryptionOptions());
try
{
- client.connect(false, true);
+ client.connect(false, false, true);
QueryOptions queryOptions = QueryOptions.create(
QueryOptions.DEFAULT.getConsistency(),
QueryOptions.DEFAULT.getValues(),
@@ -222,13 +235,14 @@
QueryOptions.DEFAULT.getPageSize(),
QueryOptions.DEFAULT.getPagingState(),
QueryOptions.DEFAULT.getSerialConsistency(),
- ProtocolVersion.V4);
+ ProtocolVersion.V5,
+ KEYSPACE);
- QueryMessage queryMessage = new QueryMessage(String.format("CREATE TABLE %s.atable (pk int PRIMARY KEY, v text)", KEYSPACE),
+ QueryMessage queryMessage = new QueryMessage("CREATE TABLE atable (pk int PRIMARY KEY, v text)",
queryOptions);
client.execute(queryMessage);
- queryMessage = new QueryMessage(String.format("INSERT INTO %s.atable (pk, v) VALUES (1, 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa')", KEYSPACE),
+ queryMessage = new QueryMessage("INSERT INTO atable (pk, v) VALUES (1, 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa')",
queryOptions);
try
{
@@ -245,4 +259,149 @@
client.close();
}
}
-}
\ No newline at end of file
+
+ @Test
+ public void testChangingLimitsAtRuntime() throws Throwable
+ {
+ SimpleClient client = new SimpleClient(nativeAddr.getHostAddress(),
+ nativePort,
+ ProtocolVersion.V5,
+ true,
+ new EncryptionOptions());
+ client.connect(false, true, true);
+
+ QueryOptions queryOptions = QueryOptions.create(
+ QueryOptions.DEFAULT.getConsistency(),
+ QueryOptions.DEFAULT.getValues(),
+ QueryOptions.DEFAULT.skipMetadata(),
+ QueryOptions.DEFAULT.getPageSize(),
+ QueryOptions.DEFAULT.getPagingState(),
+ QueryOptions.DEFAULT.getSerialConsistency(),
+ ProtocolVersion.V5,
+ KEYSPACE);
+
+ try
+ {
+ QueryMessage queryMessage = new QueryMessage(String.format("CREATE TABLE %s.atable (pk int PRIMARY KEY, v text)", KEYSPACE),
+ queryOptions);
+ client.execute(queryMessage);
+
+ queryMessage = new QueryMessage(String.format("INSERT INTO %s.atable (pk, v) VALUES (1, 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa')", KEYSPACE),
+ queryOptions);
+ try
+ {
+ client.execute(queryMessage);
+ Assert.fail();
+ }
+ catch (RuntimeException e)
+ {
+ Assert.assertTrue(e.getCause() instanceof OverloadedException);
+ }
+
+
+ // change global limit, query will still fail because endpoint limit
+ Server.EndpointPayloadTracker.setGlobalLimit(HIGH_LIMIT);
+ Assert.assertEquals("new global limit not returned by EndpointPayloadTrackers", HIGH_LIMIT, Server.EndpointPayloadTracker.getGlobalLimit());
+ Assert.assertEquals("new global limit not returned by DatabaseDescriptor", HIGH_LIMIT, DatabaseDescriptor.getNativeTransportMaxConcurrentRequestsInBytes());
+
+ queryMessage = new QueryMessage(String.format("INSERT INTO %s.atable (pk, v) VALUES (1, 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa')", KEYSPACE),
+ queryOptions);
+ try
+ {
+ client.execute(queryMessage);
+ Assert.fail();
+ }
+ catch (RuntimeException e)
+ {
+ Assert.assertTrue(e.getCause() instanceof OverloadedException);
+ }
+
+ // change endpoint limit, query will still now succeed
+ Server.EndpointPayloadTracker.setEndpointLimit(HIGH_LIMIT);
+ Assert.assertEquals("new endpoint limit not returned by EndpointPayloadTrackers", HIGH_LIMIT, Server.EndpointPayloadTracker.getEndpointLimit());
+ Assert.assertEquals("new endpoint limit not returned by DatabaseDescriptor", HIGH_LIMIT, DatabaseDescriptor.getNativeTransportMaxConcurrentRequestsInBytesPerIp());
+
+ queryMessage = new QueryMessage(String.format("INSERT INTO %s.atable (pk, v) VALUES (1, 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa')", KEYSPACE),
+ queryOptions);
+ client.execute(queryMessage);
+
+ // ensure new clients also see the new raised limits
+ client.close();
+ client = new SimpleClient(nativeAddr.getHostAddress(),
+ nativePort,
+ ProtocolVersion.V5,
+ true,
+ new EncryptionOptions());
+ client.connect(false, true, true);
+
+ queryMessage = new QueryMessage(String.format("INSERT INTO %s.atable (pk, v) VALUES (1, 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa')", KEYSPACE),
+ queryOptions);
+ client.execute(queryMessage);
+
+ // lower the global limit and ensure the query fails again
+ Server.EndpointPayloadTracker.setGlobalLimit(LOW_LIMIT);
+ Assert.assertEquals("new global limit not returned by EndpointPayloadTrackers", LOW_LIMIT, Server.EndpointPayloadTracker.getGlobalLimit());
+ Assert.assertEquals("new global limit not returned by DatabaseDescriptor", LOW_LIMIT, DatabaseDescriptor.getNativeTransportMaxConcurrentRequestsInBytes());
+
+ queryMessage = new QueryMessage(String.format("INSERT INTO %s.atable (pk, v) VALUES (1, 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa')", KEYSPACE),
+ queryOptions);
+ try
+ {
+ client.execute(queryMessage);
+ Assert.fail();
+ }
+ catch (RuntimeException e)
+ {
+ Assert.assertTrue(e.getCause() instanceof OverloadedException);
+ }
+
+ // lower the endpoint limit and ensure existing clients also have requests that fail
+ Server.EndpointPayloadTracker.setEndpointLimit(60);
+ Assert.assertEquals("new endpoint limit not returned by EndpointPayloadTrackers", 60, Server.EndpointPayloadTracker.getEndpointLimit());
+ Assert.assertEquals("new endpoint limit not returned by DatabaseDescriptor", 60, DatabaseDescriptor.getNativeTransportMaxConcurrentRequestsInBytesPerIp());
+
+ queryMessage = new QueryMessage(String.format("CREATE TABLE %s.atable (pk int PRIMARY KEY, v text)", KEYSPACE),
+ queryOptions);
+ try
+ {
+ client.execute(queryMessage);
+ Assert.fail();
+ }
+ catch (RuntimeException e)
+ {
+ Assert.assertTrue(e.getCause() instanceof OverloadedException);
+ }
+
+ // ensure new clients also see the new lowered limit
+ client.close();
+ client = new SimpleClient(nativeAddr.getHostAddress(),
+ nativePort,
+ ProtocolVersion.V5,
+ true,
+ new EncryptionOptions());
+ client.connect(false, true, true);
+
+ queryMessage = new QueryMessage(String.format("CREATE TABLE %s.atable (pk int PRIMARY KEY, v text)", KEYSPACE),
+ queryOptions);
+ try
+ {
+ client.execute(queryMessage);
+ Assert.fail();
+ }
+ catch (RuntimeException e)
+ {
+ Assert.assertTrue(e.getCause() instanceof OverloadedException);
+ }
+
+ // put the test state back
+ Server.EndpointPayloadTracker.setEndpointLimit(LOW_LIMIT);
+ Assert.assertEquals("new endpoint limit not returned by EndpointPayloadTrackers", LOW_LIMIT, Server.EndpointPayloadTracker.getEndpointLimit());
+ Assert.assertEquals("new endpoint limit not returned by DatabaseDescriptor", LOW_LIMIT, DatabaseDescriptor.getNativeTransportMaxConcurrentRequestsInBytesPerIp());
+
+ }
+ finally
+ {
+ client.close();
+ }
+ }
+}
diff --git a/test/unit/org/apache/cassandra/transport/MessagePayloadTest.java b/test/unit/org/apache/cassandra/transport/MessagePayloadTest.java
index 2bd1883..ef09c90 100644
--- a/test/unit/org/apache/cassandra/transport/MessagePayloadTest.java
+++ b/test/unit/org/apache/cassandra/transport/MessagePayloadTest.java
@@ -29,13 +29,13 @@
import org.junit.BeforeClass;
import org.junit.Test;
+import org.apache.cassandra.config.EncryptionOptions;
import org.apache.cassandra.cql3.BatchQueryOptions;
import org.apache.cassandra.cql3.CQLStatement;
import org.apache.cassandra.cql3.QueryHandler;
import org.apache.cassandra.cql3.QueryOptions;
import org.apache.cassandra.cql3.QueryProcessor;
import org.apache.cassandra.cql3.statements.BatchStatement;
-import org.apache.cassandra.cql3.statements.ParsedStatement;
import org.apache.cassandra.cql3.CQLTester;
import org.apache.cassandra.exceptions.RequestExecutionException;
import org.apache.cassandra.exceptions.RequestValidationException;
@@ -112,7 +112,7 @@
}
@Test
- public void testMessagePayload() throws Throwable
+ public void testMessagePayloadBeta() throws Throwable
{
QueryHandler queryHandler = (QueryHandler) cqlQueryHandlerField.get(null);
cqlQueryHandlerField.set(null, new TestQueryHandler());
@@ -122,19 +122,30 @@
Assert.assertSame(TestQueryHandler.class, ClientState.getCQLQueryHandler().getClass());
- SimpleClient client = new SimpleClient(nativeAddr.getHostAddress(), nativePort);
+ SimpleClient client = new SimpleClient(nativeAddr.getHostAddress(),
+ nativePort,
+ ProtocolVersion.V5,
+ true,
+ new EncryptionOptions());
try
{
- client.connect(false);
+ client.connect(false, false);
Map<String, ByteBuffer> reqMap;
Map<String, ByteBuffer> respMap;
- QueryMessage queryMessage = new QueryMessage(
- "CREATE TABLE " + KEYSPACE + ".atable (pk int PRIMARY KEY, v text)",
- QueryOptions.DEFAULT
- );
- PrepareMessage prepareMessage = new PrepareMessage("SELECT * FROM " + KEYSPACE + ".atable");
+ QueryOptions queryOptions = QueryOptions.create(
+ QueryOptions.DEFAULT.getConsistency(),
+ QueryOptions.DEFAULT.getValues(),
+ QueryOptions.DEFAULT.skipMetadata(),
+ QueryOptions.DEFAULT.getPageSize(),
+ QueryOptions.DEFAULT.getPagingState(),
+ QueryOptions.DEFAULT.getSerialConsistency(),
+ ProtocolVersion.V5,
+ KEYSPACE);
+ QueryMessage queryMessage = new QueryMessage("CREATE TABLE atable (pk int PRIMARY KEY, v text)",
+ queryOptions);
+ PrepareMessage prepareMessage = new PrepareMessage("SELECT * FROM atable", KEYSPACE);
reqMap = Collections.singletonMap("foo", bytes(42));
responsePayload = respMap = Collections.singletonMap("bar", bytes(42));
@@ -150,7 +161,76 @@
payloadEquals(reqMap, requestPayload);
payloadEquals(respMap, prepareResponse.getCustomPayload());
- ExecuteMessage executeMessage = new ExecuteMessage(prepareResponse.statementId, QueryOptions.DEFAULT);
+ ExecuteMessage executeMessage = new ExecuteMessage(prepareResponse.statementId, prepareResponse.resultMetadataId, QueryOptions.DEFAULT);
+ reqMap = Collections.singletonMap("foo", bytes(44));
+ responsePayload = respMap = Collections.singletonMap("bar", bytes(44));
+ executeMessage.setCustomPayload(reqMap);
+ Message.Response executeResponse = client.execute(executeMessage);
+ payloadEquals(reqMap, requestPayload);
+ payloadEquals(respMap, executeResponse.getCustomPayload());
+
+ BatchMessage batchMessage = new BatchMessage(BatchStatement.Type.UNLOGGED,
+ Collections.<Object>singletonList("INSERT INTO atable (pk,v) VALUES (1, 'foo')"),
+ Collections.singletonList(Collections.<ByteBuffer>emptyList()),
+ queryOptions);
+ reqMap = Collections.singletonMap("foo", bytes(45));
+ responsePayload = respMap = Collections.singletonMap("bar", bytes(45));
+ batchMessage.setCustomPayload(reqMap);
+ Message.Response batchResponse = client.execute(batchMessage);
+ payloadEquals(reqMap, requestPayload);
+ payloadEquals(respMap, batchResponse.getCustomPayload());
+ }
+ finally
+ {
+ client.close();
+ }
+ }
+ finally
+ {
+ cqlQueryHandlerField.set(null, queryHandler);
+ }
+ }
+
+ @Test
+ public void testMessagePayload() throws Throwable
+ {
+ QueryHandler queryHandler = (QueryHandler) cqlQueryHandlerField.get(null);
+ cqlQueryHandlerField.set(null, new TestQueryHandler());
+ try
+ {
+ requireNetwork();
+
+ Assert.assertSame(TestQueryHandler.class, ClientState.getCQLQueryHandler().getClass());
+
+ SimpleClient client = new SimpleClient(nativeAddr.getHostAddress(), nativePort);
+ try
+ {
+ client.connect(false, false);
+
+ Map<String, ByteBuffer> reqMap;
+ Map<String, ByteBuffer> respMap;
+
+ QueryMessage queryMessage = new QueryMessage(
+ "CREATE TABLE " + KEYSPACE + ".atable (pk int PRIMARY KEY, v text)",
+ QueryOptions.DEFAULT
+ );
+ PrepareMessage prepareMessage = new PrepareMessage("SELECT * FROM " + KEYSPACE + ".atable", null);
+
+ reqMap = Collections.singletonMap("foo", bytes(42));
+ responsePayload = respMap = Collections.singletonMap("bar", bytes(42));
+ queryMessage.setCustomPayload(reqMap);
+ Message.Response queryResponse = client.execute(queryMessage);
+ payloadEquals(reqMap, requestPayload);
+ payloadEquals(respMap, queryResponse.getCustomPayload());
+
+ reqMap = Collections.singletonMap("foo", bytes(43));
+ responsePayload = respMap = Collections.singletonMap("bar", bytes(43));
+ prepareMessage.setCustomPayload(reqMap);
+ ResultMessage.Prepared prepareResponse = (ResultMessage.Prepared) client.execute(prepareMessage);
+ payloadEquals(reqMap, requestPayload);
+ payloadEquals(respMap, prepareResponse.getCustomPayload());
+
+ ExecuteMessage executeMessage = new ExecuteMessage(prepareResponse.statementId, prepareResponse.resultMetadataId, QueryOptions.DEFAULT);
reqMap = Collections.singletonMap("foo", bytes(44));
responsePayload = respMap = Collections.singletonMap("bar", bytes(44));
executeMessage.setCustomPayload(reqMap);
@@ -194,7 +274,7 @@
SimpleClient client = new SimpleClient(nativeAddr.getHostAddress(), nativePort, ProtocolVersion.V3);
try
{
- client.connect(false);
+ client.connect(false, false);
Map<String, ByteBuffer> reqMap;
@@ -202,7 +282,7 @@
"CREATE TABLE " + KEYSPACE + ".atable (pk int PRIMARY KEY, v text)",
QueryOptions.DEFAULT
);
- PrepareMessage prepareMessage = new PrepareMessage("SELECT * FROM " + KEYSPACE + ".atable");
+ PrepareMessage prepareMessage = new PrepareMessage("SELECT * FROM " + KEYSPACE + ".atable", null);
reqMap = Collections.singletonMap("foo", bytes(42));
responsePayload = Collections.singletonMap("bar", bytes(42));
@@ -234,7 +314,7 @@
prepareMessage.setCustomPayload(null);
ResultMessage.Prepared prepareResponse = (ResultMessage.Prepared) client.execute(prepareMessage);
- ExecuteMessage executeMessage = new ExecuteMessage(prepareResponse.statementId, QueryOptions.DEFAULT);
+ ExecuteMessage executeMessage = new ExecuteMessage(prepareResponse.statementId, prepareResponse.resultMetadataId, QueryOptions.DEFAULT);
reqMap = Collections.singletonMap("foo", bytes(44));
responsePayload = Collections.singletonMap("bar", bytes(44));
executeMessage.setCustomPayload(reqMap);
@@ -287,24 +367,24 @@
public static class TestQueryHandler implements QueryHandler
{
- public ParsedStatement.Prepared getPrepared(MD5Digest id)
+ public QueryProcessor.Prepared getPrepared(MD5Digest id)
{
return QueryProcessor.instance.getPrepared(id);
}
- public ParsedStatement.Prepared getPreparedForThrift(Integer id)
+ public CQLStatement parse(String query, QueryState state, QueryOptions options)
{
- return QueryProcessor.instance.getPreparedForThrift(id);
+ return QueryProcessor.instance.parse(query, state, options);
}
public ResultMessage.Prepared prepare(String query,
- QueryState state,
+ ClientState clientState,
Map<String, ByteBuffer> customPayload)
throws RequestValidationException
{
if (customPayload != null)
requestPayload = customPayload;
- ResultMessage.Prepared result = QueryProcessor.instance.prepare(query, state, customPayload);
+ ResultMessage.Prepared result = QueryProcessor.instance.prepare(query, clientState, customPayload);
if (customPayload != null)
{
result.setCustomPayload(responsePayload);
@@ -313,7 +393,7 @@
return result;
}
- public ResultMessage process(String query,
+ public ResultMessage process(CQLStatement statement,
QueryState state,
QueryOptions options,
Map<String, ByteBuffer> customPayload,
@@ -322,7 +402,7 @@
{
if (customPayload != null)
requestPayload = customPayload;
- ResultMessage result = QueryProcessor.instance.process(query, state, options, customPayload, queryStartNanoTime);
+ ResultMessage result = QueryProcessor.instance.process(statement, state, options, customPayload, queryStartNanoTime);
if (customPayload != null)
{
result.setCustomPayload(responsePayload);
diff --git a/test/unit/org/apache/cassandra/transport/ProtocolErrorTest.java b/test/unit/org/apache/cassandra/transport/ProtocolErrorTest.java
index 5041a94..26b3d96 100644
--- a/test/unit/org/apache/cassandra/transport/ProtocolErrorTest.java
+++ b/test/unit/org/apache/cassandra/transport/ProtocolErrorTest.java
@@ -52,7 +52,7 @@
public void testInvalidProtocolVersion(int version) throws Exception
{
- Frame.Decoder dec = new Frame.Decoder(null, ProtocolVersionLimit.SERVER_DEFAULT);
+ Frame.Decoder dec = new Frame.Decoder(null);
List<Object> results = new ArrayList<>();
byte[] frame = new byte[] {
@@ -80,7 +80,7 @@
public void testInvalidProtocolVersionShortFrame() throws Exception
{
// test for CASSANDRA-11464
- Frame.Decoder dec = new Frame.Decoder(null, ProtocolVersionLimit.SERVER_DEFAULT);
+ Frame.Decoder dec = new Frame.Decoder(null);
List<Object> results = new ArrayList<>();
byte[] frame = new byte[] {
@@ -102,7 +102,7 @@
@Test
public void testInvalidDirection() throws Exception
{
- Frame.Decoder dec = new Frame.Decoder(null, ProtocolVersionLimit.SERVER_DEFAULT);
+ Frame.Decoder dec = new Frame.Decoder(null);
List<Object> results = new ArrayList<>();
// should generate a protocol exception for using a response frame with
@@ -133,7 +133,7 @@
@Test
public void testBodyLengthOverLimit() throws Exception
{
- Frame.Decoder dec = new Frame.Decoder(null, ProtocolVersionLimit.SERVER_DEFAULT);
+ Frame.Decoder dec = new Frame.Decoder(null);
List<Object> results = new ArrayList<>();
byte[] frame = new byte[] {
@@ -172,4 +172,25 @@
Assert.assertEquals(expected, buf);
}
+
+ @Test
+ public void testUnsupportedMessage() throws Exception
+ {
+ byte[] incomingFrame = new byte[] {
+ (byte) REQUEST.addToVersion(ProtocolVersion.CURRENT.asInt()), // direction & version
+ 0x00, // flags
+ 0x00, 0x01, // stream ID
+ 0x04, // opcode for obsoleted CREDENTIALS message
+ 0x00, (byte) 0x00, (byte) 0x00, (byte) 0x10, // body length
+ };
+ byte[] body = new byte[0x10];
+ ByteBuf buf = Unpooled.wrappedBuffer(incomingFrame, body);
+ Frame decodedFrame = new Frame.Decoder(null).decodeFrame(buf);
+ try {
+ decodedFrame.header.type.codec.decode(decodedFrame.body, decodedFrame.header.version);
+ Assert.fail("Expected protocol error");
+ } catch (ProtocolException e) {
+ Assert.assertTrue(e.getMessage().contains("Unsupported message"));
+ }
+ }
}
diff --git a/test/unit/org/apache/cassandra/transport/ProtocolNegotiationTest.java b/test/unit/org/apache/cassandra/transport/ProtocolNegotiationTest.java
deleted file mode 100644
index 91c1d6a..0000000
--- a/test/unit/org/apache/cassandra/transport/ProtocolNegotiationTest.java
+++ /dev/null
@@ -1,166 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.transport;
-
-import java.net.InetAddress;
-import java.util.concurrent.TimeUnit;
-
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import com.datastax.driver.core.Cluster;
-import com.datastax.driver.core.ProtocolVersion;
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.cql3.CQLTester;
-
-import static org.apache.cassandra.transport.ProtocolTestHelper.cleanupPeers;
-import static org.apache.cassandra.transport.ProtocolTestHelper.setStaticLimitInConfig;
-import static org.apache.cassandra.transport.ProtocolTestHelper.setupPeer;
-import static org.apache.cassandra.transport.ProtocolTestHelper.updatePeerInfo;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-public class ProtocolNegotiationTest extends CQLTester
-{
- // to avoid JMX naming clashes between cluster metrics
- private int clusterId = 0;
-
- @BeforeClass
- public static void setup()
- {
- prepareNetwork();
- }
-
- @Before
- public void clearConfig()
- {
- setStaticLimitInConfig(null);
- }
-
- @Test
- public void serverSupportsV3AndV4ByDefault() throws Throwable
- {
- reinitializeNetwork();
- // client can explicitly request either V3 or V4
- testConnection(ProtocolVersion.V3, ProtocolVersion.V3);
- testConnection(ProtocolVersion.V4, ProtocolVersion.V4);
-
- // if not specified, V4 is the default
- testConnection(null, ProtocolVersion.V4);
- }
-
- @Test
- public void testStaticLimit() throws Throwable
- {
- try
- {
- reinitializeNetwork();
- // No limit enforced to start
- assertEquals(Integer.MIN_VALUE, DatabaseDescriptor.getNativeProtocolMaxVersionOverride());
- testConnection(null, ProtocolVersion.V4);
-
- // Update DatabaseDescriptor, then re-initialise the server to force it to read it
- setStaticLimitInConfig(ProtocolVersion.V3.toInt());
- reinitializeNetwork();
- assertEquals(3, DatabaseDescriptor.getNativeProtocolMaxVersionOverride());
- testConnection(ProtocolVersion.V4, ProtocolVersion.V3);
- testConnection(ProtocolVersion.V3, ProtocolVersion.V3);
- testConnection(null, ProtocolVersion.V3);
- } finally {
- setStaticLimitInConfig(null);
- }
- }
-
- @Test
- public void testDynamicLimit() throws Throwable
- {
- InetAddress peer1 = setupPeer("127.1.0.1", "2.2.0");
- InetAddress peer2 = setupPeer("127.1.0.2", "2.2.0");
- InetAddress peer3 = setupPeer("127.1.0.3", "2.2.0");
- reinitializeNetwork();
- try
- {
- // legacy peers means max negotiable version is V3
- testConnection(ProtocolVersion.V4, ProtocolVersion.V3);
- testConnection(ProtocolVersion.V3, ProtocolVersion.V3);
- testConnection(null, ProtocolVersion.V3);
-
- // receive notification that 2 peers have upgraded to a version that fully supports V4
- updatePeerInfo(peer1, "3.0.0");
- updatePeerInfo(peer2, "3.0.0");
- updateMaxNegotiableProtocolVersion();
- // version should still be capped
- testConnection(ProtocolVersion.V4, ProtocolVersion.V3);
- testConnection(ProtocolVersion.V3, ProtocolVersion.V3);
- testConnection(null, ProtocolVersion.V3);
-
- // no legacy peers so V4 is negotiable
- // after the last peer upgrades, cap should be lifted
- updatePeerInfo(peer3, "3.0.0");
- updateMaxNegotiableProtocolVersion();
- testConnection(ProtocolVersion.V4, ProtocolVersion.V4);
- testConnection(ProtocolVersion.V3, ProtocolVersion.V3);
- testConnection(null, ProtocolVersion.V4);
- } finally {
- cleanupPeers(peer1, peer2, peer3);
- }
- }
-
- private void testConnection(com.datastax.driver.core.ProtocolVersion requestedVersion,
- com.datastax.driver.core.ProtocolVersion expectedVersion)
- {
- long start = System.nanoTime();
- boolean expectError = requestedVersion != null && requestedVersion != expectedVersion;
- Cluster.Builder builder = Cluster.builder()
- .addContactPoints(nativeAddr)
- .withClusterName("Test Cluster" + clusterId++)
- .withPort(nativePort);
-
- if (requestedVersion != null)
- builder = builder.withProtocolVersion(requestedVersion) ;
-
- Cluster cluster = builder.build();
- logger.info("Setting up cluster took {}ms", TimeUnit.MILLISECONDS.convert(System.nanoTime() - start, TimeUnit.NANOSECONDS));
- start = System.nanoTime();
- try {
- cluster.connect();
- if (expectError)
- fail("Expected a protocol exception");
- }
- catch (Exception e)
- {
- if (!expectError)
- {
- e.printStackTrace();
- fail("Did not expect any exception");
- }
-
- assertTrue(e.getMessage().contains(String.format("Host does not support protocol version %s but %s", requestedVersion, expectedVersion)));
- } finally {
- logger.info("Testing connection took {}ms", TimeUnit.MILLISECONDS.convert(System.nanoTime() - start, TimeUnit.NANOSECONDS));
- start = System.nanoTime();
- cluster.closeAsync();
- logger.info("Tearing down cluster connection took {}ms", TimeUnit.MILLISECONDS.convert(System.nanoTime() - start, TimeUnit.NANOSECONDS));
-
- }
- }
-
-}
diff --git a/test/unit/org/apache/cassandra/transport/ProtocolTestHelper.java b/test/unit/org/apache/cassandra/transport/ProtocolTestHelper.java
deleted file mode 100644
index 90a2801..0000000
--- a/test/unit/org/apache/cassandra/transport/ProtocolTestHelper.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.transport;
-
-import java.lang.reflect.Constructor;
-import java.lang.reflect.Field;
-import java.net.InetAddress;
-import java.net.UnknownHostException;
-import java.util.concurrent.ExecutorService;
-
-import com.google.common.util.concurrent.MoreExecutors;
-
-import org.apache.cassandra.config.Config;
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.db.SystemKeyspace;
-import org.apache.cassandra.gms.VersionedValue;
-import org.apache.cassandra.utils.FBUtilities;
-
-public class ProtocolTestHelper
-{
- static ExecutorService executor = MoreExecutors.newDirectExecutorService();
- static InetAddress setupPeer(String address, String version) throws Throwable
- {
- InetAddress peer = peer(address);
- updatePeerInfo(peer, version);
- return peer;
- }
-
- static void updatePeerInfo(InetAddress peer, String version) throws Throwable
- {
- SystemKeyspace.updatePeerInfo(peer, "release_version", version, executor);
- }
-
- static InetAddress peer(String address)
- {
- try
- {
- return InetAddress.getByName(address);
- }
- catch (UnknownHostException e)
- {
- throw new RuntimeException("Error creating peer", e);
- }
- }
-
- static void cleanupPeers(InetAddress...peers) throws Throwable
- {
- for (InetAddress peer : peers)
- if (peer != null)
- SystemKeyspace.removeEndpoint(peer);
- }
-
- static void setStaticLimitInConfig(Integer version)
- {
- try
- {
- Field field = FBUtilities.getProtectedField(DatabaseDescriptor.class, "conf");
- ((Config)field.get(null)).native_transport_max_negotiable_protocol_version = version == null ? Integer.MIN_VALUE : version;
- }
- catch (IllegalAccessException e)
- {
- throw new RuntimeException("Error setting native_transport_max_protocol_version on Config", e);
- }
- }
-
- static VersionedValue releaseVersion(String versionString)
- {
- try
- {
- Constructor<VersionedValue> ctor = VersionedValue.class.getDeclaredConstructor(String.class);
- ctor.setAccessible(true);
- return ctor.newInstance(versionString);
- }
- catch (Exception e)
- {
- throw new RuntimeException("Error constructing VersionedValue for release version", e);
- }
- }
-}
diff --git a/test/unit/org/apache/cassandra/transport/ProtocolVersionTest.java b/test/unit/org/apache/cassandra/transport/ProtocolVersionTest.java
index 7b56a49..74bc407 100644
--- a/test/unit/org/apache/cassandra/transport/ProtocolVersionTest.java
+++ b/test/unit/org/apache/cassandra/transport/ProtocolVersionTest.java
@@ -18,22 +18,41 @@
package org.apache.cassandra.transport;
+import java.util.List;
+import java.util.stream.Collectors;
+
import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
import org.junit.Test;
+import org.apache.cassandra.config.DatabaseDescriptor;
+
public class ProtocolVersionTest
{
+ @BeforeClass
+ public static void setupDatabaseDescriptor()
+ {
+ DatabaseDescriptor.daemonInitialization();
+ }
+
+ @Before
+ public void setUp()
+ {
+ DatabaseDescriptor.setNativeTransportAllowOlderProtocols(true);
+ }
+
@Test
public void testDecode()
{
for (ProtocolVersion version : ProtocolVersion.SUPPORTED)
- Assert.assertEquals(version, ProtocolVersion.decode(version.asInt(), ProtocolVersionLimit.SERVER_DEFAULT));
+ Assert.assertEquals(version, ProtocolVersion.decode(version.asInt(), DatabaseDescriptor.getNativeTransportAllowOlderProtocols()));
for (ProtocolVersion version : ProtocolVersion.UNSUPPORTED)
{ // unsupported old versions
try
{
- Assert.assertEquals(version, ProtocolVersion.decode(version.asInt(), ProtocolVersionLimit.SERVER_DEFAULT));
+ Assert.assertEquals(version, ProtocolVersion.decode(version.asInt(), DatabaseDescriptor.getNativeTransportAllowOlderProtocols()));
Assert.fail("Expected invalid protocol exception");
}
catch (ProtocolException ex)
@@ -45,7 +64,7 @@
try
{ // unsupported newer version
- Assert.assertEquals(null, ProtocolVersion.decode(63, ProtocolVersionLimit.SERVER_DEFAULT));
+ Assert.assertEquals(null, ProtocolVersion.decode(63, DatabaseDescriptor.getNativeTransportAllowOlderProtocols()));
Assert.fail("Expected invalid protocol exception");
}
catch (ProtocolException ex)
@@ -72,26 +91,56 @@
Assert.assertTrue(ProtocolVersion.V2.isSmallerOrEqualTo(ProtocolVersion.V2));
Assert.assertTrue(ProtocolVersion.V3.isSmallerOrEqualTo(ProtocolVersion.V3));
Assert.assertTrue(ProtocolVersion.V4.isSmallerOrEqualTo(ProtocolVersion.V4));
+ Assert.assertTrue(ProtocolVersion.V5.isSmallerOrEqualTo(ProtocolVersion.V5));
Assert.assertTrue(ProtocolVersion.V1.isGreaterOrEqualTo(ProtocolVersion.V1));
Assert.assertTrue(ProtocolVersion.V2.isGreaterOrEqualTo(ProtocolVersion.V2));
Assert.assertTrue(ProtocolVersion.V3.isGreaterOrEqualTo(ProtocolVersion.V3));
Assert.assertTrue(ProtocolVersion.V4.isGreaterOrEqualTo(ProtocolVersion.V4));
+ Assert.assertTrue(ProtocolVersion.V5.isGreaterOrEqualTo(ProtocolVersion.V5));
Assert.assertTrue(ProtocolVersion.V1.isSmallerThan(ProtocolVersion.V2));
Assert.assertTrue(ProtocolVersion.V2.isSmallerThan(ProtocolVersion.V3));
Assert.assertTrue(ProtocolVersion.V3.isSmallerThan(ProtocolVersion.V4));
+ Assert.assertTrue(ProtocolVersion.V4.isSmallerThan(ProtocolVersion.V5));
Assert.assertFalse(ProtocolVersion.V1.isGreaterThan(ProtocolVersion.V2));
Assert.assertFalse(ProtocolVersion.V2.isGreaterThan(ProtocolVersion.V3));
Assert.assertFalse(ProtocolVersion.V3.isGreaterThan(ProtocolVersion.V4));
+ Assert.assertFalse(ProtocolVersion.V4.isGreaterThan(ProtocolVersion.V5));
+ Assert.assertTrue(ProtocolVersion.V5.isGreaterThan(ProtocolVersion.V4));
Assert.assertTrue(ProtocolVersion.V4.isGreaterThan(ProtocolVersion.V3));
Assert.assertTrue(ProtocolVersion.V3.isGreaterThan(ProtocolVersion.V2));
Assert.assertTrue(ProtocolVersion.V2.isGreaterThan(ProtocolVersion.V1));
+ Assert.assertFalse(ProtocolVersion.V5.isSmallerThan(ProtocolVersion.V4));
Assert.assertFalse(ProtocolVersion.V4.isSmallerThan(ProtocolVersion.V3));
Assert.assertFalse(ProtocolVersion.V3.isSmallerThan(ProtocolVersion.V2));
Assert.assertFalse(ProtocolVersion.V2.isSmallerThan(ProtocolVersion.V1));
}
+
+ @Test
+ public void testDisableOldProtocolVersions_Succeeds()
+ {
+ DatabaseDescriptor.setNativeTransportAllowOlderProtocols(false);
+ List<ProtocolVersion> disallowedVersions = ProtocolVersion.SUPPORTED
+ .stream()
+ .filter(v -> v.isSmallerThan(ProtocolVersion.CURRENT))
+ .collect(Collectors.toList());
+
+ for (ProtocolVersion version : disallowedVersions)
+ {
+ try
+ {
+ ProtocolVersion.decode(version.asInt(), DatabaseDescriptor.getNativeTransportAllowOlderProtocols());
+ Assert.fail("Expected invalid protocol exception");
+ }
+ catch (ProtocolException ex)
+ {
+ }
+ }
+
+ Assert.assertEquals(ProtocolVersion.CURRENT, ProtocolVersion.decode(ProtocolVersion.CURRENT.asInt(), DatabaseDescriptor.getNativeTransportAllowOlderProtocols()));
+ }
}
diff --git a/test/unit/org/apache/cassandra/transport/ProtocolVersionTrackerTest.java b/test/unit/org/apache/cassandra/transport/ProtocolVersionTrackerTest.java
new file mode 100644
index 0000000..91d75b8
--- /dev/null
+++ b/test/unit/org/apache/cassandra/transport/ProtocolVersionTrackerTest.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.transport;
+
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.util.Collection;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class ProtocolVersionTrackerTest
+{
+ @Test
+ public void addConnection_shouldUpdateSetToLatestTimestamp() throws UnknownHostException, InterruptedException
+ {
+ ProtocolVersionTracker pvt = new ProtocolVersionTracker();
+ final InetAddress client = InetAddress.getByName("127.0.1.1");
+ pvt.addConnection(client, ProtocolVersion.V4);
+
+ for(InetAddress addr : getMockConnections(10))
+ {
+ pvt.addConnection(addr, ProtocolVersion.V4);
+ }
+
+ Collection<ClientStat> clientIPAndTimes1 = pvt.getAll(ProtocolVersion.V4);
+ assertEquals(10, clientIPAndTimes1.size());
+
+ Thread.sleep(10);
+
+ pvt.addConnection(client, ProtocolVersion.V4);
+ Collection<ClientStat> clientIPAndTimes2 = pvt.getAll(ProtocolVersion.V4);
+ assertEquals(10, clientIPAndTimes2.size());
+
+ long ls1 = clientIPAndTimes1.stream().filter(c -> c.remoteAddress.equals(client)).findFirst().get().lastSeenTime;
+ long ls2 = clientIPAndTimes2.stream().filter(c -> c.remoteAddress.equals(client)).findFirst().get().lastSeenTime;
+
+ assertTrue(ls2 > ls1);
+ }
+
+ @Test
+ public void addConnection_validConnection_Succeeds()
+ {
+ ProtocolVersionTracker pvt = new ProtocolVersionTracker();
+
+ for(InetAddress addr : getMockConnections(10))
+ {
+ pvt.addConnection(addr, ProtocolVersion.V4);
+ }
+
+ for(InetAddress addr : getMockConnections(7))
+ {
+ pvt.addConnection(addr, ProtocolVersion.V3);
+ }
+
+ assertEquals(17, pvt.getAll().size());
+ assertEquals(0, pvt.getAll(ProtocolVersion.V2).size());
+ assertEquals(7, pvt.getAll(ProtocolVersion.V3).size());
+ assertEquals(10, pvt.getAll(ProtocolVersion.V4).size());
+ }
+
+ @Test
+ public void clear()
+ {
+ ProtocolVersionTracker pvt = new ProtocolVersionTracker();
+
+ for(InetAddress addr : getMockConnections(7))
+ {
+ pvt.addConnection(addr, ProtocolVersion.V3);
+ }
+
+ assertEquals(7, pvt.getAll(ProtocolVersion.V3).size());
+ pvt.clear();
+
+ assertEquals(0, pvt.getAll(ProtocolVersion.V3).size());
+ }
+
+ /* Helper */
+ private List<InetAddress> getMockConnections(int num)
+ {
+ return IntStream.range(0, num).mapToObj(n -> {
+ try
+ {
+ return InetAddress.getByName("127.0.1." + n);
+ }
+ catch (UnknownHostException e)
+ {
+ e.printStackTrace();
+ }
+ return null;
+ }).collect(Collectors.toList());
+ }
+}
diff --git a/test/unit/org/apache/cassandra/transport/SerDeserTest.java b/test/unit/org/apache/cassandra/transport/SerDeserTest.java
index 1eaa5ac..42ffa26 100644
--- a/test/unit/org/apache/cassandra/transport/SerDeserTest.java
+++ b/test/unit/org/apache/cassandra/transport/SerDeserTest.java
@@ -32,7 +32,8 @@
import org.apache.cassandra.db.ConsistencyLevel;
import org.apache.cassandra.db.marshal.*;
import org.apache.cassandra.serializers.CollectionSerializer;
-import org.apache.cassandra.service.pager.PagingState;
+import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.service.QueryState;
import org.apache.cassandra.transport.Event.TopologyChange;
import org.apache.cassandra.transport.Event.SchemaChange;
import org.apache.cassandra.transport.Event.StatusChange;
@@ -49,6 +50,7 @@
*/
public class SerDeserTest
{
+
@BeforeClass
public static void setupDD()
{
@@ -114,12 +116,12 @@
{
List<Event> events = new ArrayList<>();
- events.add(TopologyChange.newNode(FBUtilities.getBroadcastAddress(), 42));
- events.add(TopologyChange.removedNode(FBUtilities.getBroadcastAddress(), 42));
- events.add(TopologyChange.movedNode(FBUtilities.getBroadcastAddress(), 42));
+ events.add(TopologyChange.newNode(FBUtilities.getBroadcastAddressAndPort()));
+ events.add(TopologyChange.removedNode(FBUtilities.getBroadcastAddressAndPort()));
+ events.add(TopologyChange.movedNode(FBUtilities.getBroadcastAddressAndPort()));
- events.add(StatusChange.nodeUp(FBUtilities.getBroadcastAddress(), 42));
- events.add(StatusChange.nodeDown(FBUtilities.getBroadcastAddress(), 42));
+ events.add(StatusChange.nodeUp(FBUtilities.getBroadcastAddressAndPort()));
+ events.add(StatusChange.nodeDown(FBUtilities.getBroadcastAddressAndPort()));
events.add(new SchemaChange(SchemaChange.Change.CREATED, "ks"));
events.add(new SchemaChange(SchemaChange.Change.UPDATED, "ks"));
@@ -307,27 +309,66 @@
}
@Test
- public void queryOptionsSerDeserTest() throws Exception
+ public void queryOptionsSerDeserTest()
{
for (ProtocolVersion version : ProtocolVersion.SUPPORTED)
- queryOptionsSerDeserTest(version);
+ {
+ queryOptionsSerDeserTest(
+ version,
+ QueryOptions.create(ConsistencyLevel.ALL,
+ Collections.singletonList(ByteBuffer.wrap(new byte[] { 0x00, 0x01, 0x02 })),
+ false,
+ 5000,
+ Util.makeSomePagingState(version),
+ ConsistencyLevel.SERIAL,
+ version,
+ null)
+ );
+ }
+
+ for (ProtocolVersion version : ProtocolVersion.supportedVersionsStartingWith(ProtocolVersion.V5))
+ {
+ queryOptionsSerDeserTest(
+ version,
+ QueryOptions.create(ConsistencyLevel.LOCAL_ONE,
+ Arrays.asList(ByteBuffer.wrap(new byte[] { 0x00, 0x01, 0x02 }),
+ ByteBuffer.wrap(new byte[] { 0x03, 0x04, 0x05, 0x03, 0x04, 0x05 })),
+ true,
+ 10,
+ Util.makeSomePagingState(version),
+ ConsistencyLevel.SERIAL,
+ version,
+ "some_keyspace")
+ );
+ }
+
+ for (ProtocolVersion version : ProtocolVersion.supportedVersionsStartingWith(ProtocolVersion.V5))
+ {
+ queryOptionsSerDeserTest(
+ version,
+ QueryOptions.create(ConsistencyLevel.LOCAL_ONE,
+ Arrays.asList(ByteBuffer.wrap(new byte[] { 0x00, 0x01, 0x02 }),
+ ByteBuffer.wrap(new byte[] { 0x03, 0x04, 0x05, 0x03, 0x04, 0x05 })),
+ true,
+ 10,
+ Util.makeSomePagingState(version),
+ ConsistencyLevel.SERIAL,
+ version,
+ "some_keyspace",
+ FBUtilities.timestampMicros(),
+ FBUtilities.nowInSeconds())
+ );
+ }
}
- private void queryOptionsSerDeserTest(ProtocolVersion version) throws Exception
+ private void queryOptionsSerDeserTest(ProtocolVersion version, QueryOptions options)
{
- QueryOptions options = QueryOptions.create(ConsistencyLevel.ALL,
- Collections.singletonList(ByteBuffer.wrap(new byte[] { 0x00, 0x01, 0x02 })),
- false,
- 5000,
- Util.makeSomePagingState(version),
- ConsistencyLevel.SERIAL,
- version
- );
-
ByteBuf buf = Unpooled.buffer(QueryOptions.codec.encodedSize(options, version));
QueryOptions.codec.encode(options, buf, version);
QueryOptions decodedOptions = QueryOptions.codec.decode(buf, version);
+ QueryState state = new QueryState(ClientState.forInternalCalls());
+
assertNotNull(decodedOptions);
assertEquals(options.getConsistency(), decodedOptions.getConsistency());
assertEquals(options.getSerialConsistency(), decodedOptions.getSerialConsistency());
@@ -336,5 +377,8 @@
assertEquals(options.getValues(), decodedOptions.getValues());
assertEquals(options.getPagingState(), decodedOptions.getPagingState());
assertEquals(options.skipMetadata(), decodedOptions.skipMetadata());
+ assertEquals(options.getKeyspace(), decodedOptions.getKeyspace());
+ assertEquals(options.getTimestamp(state), decodedOptions.getTimestamp(state));
+ assertEquals(options.getNowInSeconds(state), decodedOptions.getNowInSeconds(state));
}
}
diff --git a/test/unit/org/apache/cassandra/transport/ServerMetricsTest.java b/test/unit/org/apache/cassandra/transport/ServerMetricsTest.java
new file mode 100644
index 0000000..081da00
--- /dev/null
+++ b/test/unit/org/apache/cassandra/transport/ServerMetricsTest.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.transport;
+
+import java.util.List;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.ByteBufAllocator;
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.metrics.ClientRequestSizeMetrics;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/**
+ * Ensures we properly account for metrics tracked in the native protocol
+ */
+public class ServerMetricsTest extends CQLTester
+{
+ private long totalBytesReadStart;
+ private long totalBytesWrittenStart;
+
+ private long totalBytesReadHistoCount;
+ private long totalBytesWrittenHistoCount;
+
+ @Before
+ public void setUp()
+ {
+ totalBytesReadStart = ClientRequestSizeMetrics.totalBytesRead.getCount();
+ totalBytesWrittenStart = ClientRequestSizeMetrics.totalBytesWritten.getCount();
+
+ totalBytesReadHistoCount = ClientRequestSizeMetrics.bytesRecievedPerFrame.getCount();
+ totalBytesWrittenHistoCount = ClientRequestSizeMetrics.bytesTransmittedPerFrame.getCount();
+ }
+
+ @Test
+ public void testReadAndWriteMetricsAreRecordedDuringNativeRequests() throws Throwable
+ {
+ executeNet("SELECT * from system.peers");
+
+ assertThat(ClientRequestSizeMetrics.totalBytesRead.getCount()).isGreaterThan(totalBytesReadStart);
+ assertThat(ClientRequestSizeMetrics.totalBytesWritten.getCount()).isGreaterThan(totalBytesWrittenStart);
+ assertThat(ClientRequestSizeMetrics.bytesRecievedPerFrame.getCount()).isGreaterThan(totalBytesReadStart);
+ assertThat(ClientRequestSizeMetrics.bytesTransmittedPerFrame.getCount()).isGreaterThan(totalBytesWrittenStart);
+ }
+
+}
diff --git a/test/unit/org/apache/cassandra/transport/StartupMessageTest.java b/test/unit/org/apache/cassandra/transport/StartupMessageTest.java
new file mode 100644
index 0000000..f69ad66
--- /dev/null
+++ b/test/unit/org/apache/cassandra/transport/StartupMessageTest.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.transport;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.config.EncryptionOptions;
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.cql3.QueryProcessor;
+import org.apache.cassandra.transport.messages.StartupMessage;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+public class StartupMessageTest extends CQLTester
+{
+
+ @BeforeClass
+ public static void setUp()
+ {
+ requireNetwork();
+ }
+
+ @Test
+ public void checksumOptionValidation()
+ {
+ testConnection("crc32", false);
+ testConnection("CRC32", false);
+ testConnection("cRc32", false);
+ testConnection("adler32", false);
+ testConnection("ADLER32", false);
+ testConnection("aDlEr32", false);
+ testConnection("nonesuchtype", true);
+ testConnection("", true);
+ // special case of no option supplied
+ testConnection(null, false);
+ }
+
+ private void testConnection(String checksumType, boolean expectProtocolError)
+ {
+ try (TestClient client = new TestClient(checksumType))
+ {
+ client.connect();
+ if (expectProtocolError)
+ fail("Expected a protocol exception");
+ }
+ catch (Exception e)
+ {
+ if (!expectProtocolError)
+ fail("Did not expect any exception");
+
+ // This is a bit ugly, but SimpleClient::execute throws RuntimeException if it receives any ErrorMessage
+ String expected = String.format("org.apache.cassandra.transport.ProtocolException: " +
+ "Requested checksum type %s is not known or supported " +
+ "by this version of Cassandra", checksumType);
+ assertEquals(expected, e.getMessage());
+ }
+ }
+
+ static class TestClient extends SimpleClient
+ {
+ private final String checksumType;
+ TestClient(String checksumType)
+ {
+ super(nativeAddr.getHostAddress(), nativePort, ProtocolVersion.V5, true, new EncryptionOptions());
+ this.checksumType = checksumType;
+ }
+
+ void connect() throws IOException
+ {
+ establishConnection();
+ Map<String, String> options = new HashMap<>();
+ options.put(StartupMessage.CQL_VERSION, QueryProcessor.CQL_VERSION.toString());
+
+ if (checksumType != null)
+ options.put(StartupMessage.CHECKSUM, checksumType);
+
+ execute(new StartupMessage(options));
+ }
+ }
+}
diff --git a/test/unit/org/apache/cassandra/transport/frame/checksum/ChecksummingTransformerTest.java b/test/unit/org/apache/cassandra/transport/frame/checksum/ChecksummingTransformerTest.java
new file mode 100644
index 0000000..5f5b10d
--- /dev/null
+++ b/test/unit/org/apache/cassandra/transport/frame/checksum/ChecksummingTransformerTest.java
@@ -0,0 +1,239 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.transport.frame.checksum;
+
+import java.io.IOException;
+import java.util.EnumSet;
+import java.util.Random;
+
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.Unpooled;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.transport.Frame;
+import org.apache.cassandra.transport.ProtocolException;
+import org.apache.cassandra.transport.frame.compress.Compressor;
+import org.apache.cassandra.transport.frame.compress.LZ4Compressor;
+import org.apache.cassandra.transport.frame.compress.SnappyCompressor;
+import org.apache.cassandra.utils.ChecksumType;
+import org.apache.cassandra.utils.Pair;
+import org.quicktheories.core.Gen;
+import org.quicktheories.impl.Constraint;
+
+import static org.quicktheories.QuickTheory.qt;
+import static org.quicktheories.generators.SourceDSL.arbitrary;
+import static org.quicktheories.generators.SourceDSL.integers;
+
+public class ChecksummingTransformerTest
+{
+ private static final int DEFAULT_BLOCK_SIZE = 1 << 15;
+ private static final int MAX_INPUT_SIZE = 1 << 18;
+ private static final EnumSet<Frame.Header.Flag> FLAGS = EnumSet.of(Frame.Header.Flag.COMPRESSED, Frame.Header.Flag.CHECKSUMMED);
+
+ @BeforeClass
+ public static void init()
+ {
+ // required as static ChecksummingTransformer instances read default block size from config
+ DatabaseDescriptor.clientInitialization();
+ }
+
+ @Test
+ public void roundTripSafetyProperty()
+ {
+ qt()
+ .forAll(inputs(),
+ compressors(),
+ checksumTypes(),
+ blocksizes())
+ .checkAssert(this::roundTrip);
+ }
+
+ @Test
+ public void roundTripZeroLengthInput()
+ {
+ qt()
+ .forAll(zeroLengthInputs(),
+ compressors(),
+ checksumTypes(),
+ blocksizes())
+ .checkAssert(this::roundTrip);
+ }
+
+ @Test
+ public void corruptionCausesFailure()
+ {
+ qt()
+ .forAll(inputWithCorruptablePosition(),
+ integers().between(0, Byte.MAX_VALUE).map(Integer::byteValue),
+ compressors(),
+ checksumTypes())
+ .checkAssert(ChecksummingTransformerTest::roundTripWithCorruption);
+ }
+
+ static void roundTripWithCorruption(Pair<ReusableBuffer, Integer> inputAndCorruptablePosition,
+ byte corruptionValue,
+ Compressor compressor,
+ ChecksumType checksum)
+ {
+ ReusableBuffer input = inputAndCorruptablePosition.left;
+ ByteBuf expectedBuf = input.toByteBuf();
+ int byteToCorrupt = inputAndCorruptablePosition.right;
+ ChecksummingTransformer transformer = new ChecksummingTransformer(checksum, DEFAULT_BLOCK_SIZE, compressor);
+ ByteBuf outbound = transformer.transformOutbound(expectedBuf);
+
+ // make sure we're actually expecting to produce some corruption
+ if (outbound.getByte(byteToCorrupt) == corruptionValue)
+ return;
+
+ if (byteToCorrupt >= outbound.writerIndex())
+ return;
+
+ try
+ {
+ int oldIndex = outbound.writerIndex();
+ outbound.writerIndex(byteToCorrupt);
+ outbound.writeByte(corruptionValue);
+ outbound.writerIndex(oldIndex);
+ ByteBuf inbound = transformer.transformInbound(outbound, FLAGS);
+
+ // verify that the content was actually corrupted
+ expectedBuf.readerIndex(0);
+ Assert.assertEquals(expectedBuf, inbound);
+ } catch(ProtocolException e)
+ {
+ return;
+ }
+
+ }
+
+ @Test
+ public void roundTripWithSingleUncompressableChunk()
+ {
+ byte[] bytes = new byte[]{1};
+ ChecksummingTransformer transformer = new ChecksummingTransformer(ChecksumType.CRC32, DEFAULT_BLOCK_SIZE, LZ4Compressor.INSTANCE);
+ ByteBuf expectedBuf = Unpooled.wrappedBuffer(bytes);
+
+ ByteBuf outbound = transformer.transformOutbound(expectedBuf);
+ ByteBuf inbound = transformer.transformInbound(outbound, FLAGS);
+
+ // reset reader index on expectedBuf back to 0 as it will have been entirely consumed by the transformOutbound() call
+ expectedBuf.readerIndex(0);
+ Assert.assertEquals(expectedBuf, inbound);
+ }
+
+ @Test
+ public void roundTripWithCompressableAndUncompressableChunks() throws IOException
+ {
+ Compressor compressor = LZ4Compressor.INSTANCE;
+ Random random = new Random();
+ int inputLen = 127;
+
+ byte[] uncompressable = new byte[inputLen];
+ for (int i = 0; i < uncompressable.length; i++)
+ uncompressable[i] = (byte) random.nextInt(127);
+
+ byte[] compressed = new byte[compressor.maxCompressedLength(uncompressable.length)];
+ Assert.assertTrue(compressor.compress(uncompressable, 0, uncompressable.length, compressed, 0) > uncompressable.length);
+
+ byte[] compressable = new byte[inputLen];
+ for (int i = 0; i < compressable.length; i++)
+ compressable[i] = (byte)1;
+ Assert.assertTrue(compressor.compress(compressable, 0, compressable.length, compressable, 0) < compressable.length);
+
+ ChecksummingTransformer transformer = new ChecksummingTransformer(ChecksumType.CRC32, uncompressable.length, LZ4Compressor.INSTANCE);
+ byte[] expectedBytes = new byte[inputLen * 3];
+ ByteBuf expectedBuf = Unpooled.wrappedBuffer(expectedBytes);
+ expectedBuf.writerIndex(0);
+ expectedBuf.writeBytes(uncompressable);
+ expectedBuf.writeBytes(uncompressable);
+ expectedBuf.writeBytes(compressable);
+
+ ByteBuf outbound = transformer.transformOutbound(expectedBuf);
+ ByteBuf inbound = transformer.transformInbound(outbound, FLAGS);
+
+ // reset reader index on expectedBuf back to 0 as it will have been entirely consumed by the transformOutbound() call
+ expectedBuf.readerIndex(0);
+ Assert.assertEquals(expectedBuf, inbound);
+ }
+
+ private void roundTrip(ReusableBuffer input, Compressor compressor, ChecksumType checksum, int blockSize)
+ {
+ ChecksummingTransformer transformer = new ChecksummingTransformer(checksum, blockSize, compressor);
+ ByteBuf expectedBuf = input.toByteBuf();
+
+ ByteBuf outbound = transformer.transformOutbound(expectedBuf);
+ ByteBuf inbound = transformer.transformInbound(outbound, FLAGS);
+
+ // reset reader index on expectedBuf back to 0 as it will have been entirely consumed by the transformOutbound() call
+ expectedBuf.readerIndex(0);
+ Assert.assertEquals(expectedBuf, inbound);
+ }
+
+ private Gen<Pair<ReusableBuffer, Integer>> inputWithCorruptablePosition()
+ {
+ // we only generate corruption for byte 2 onward. This is to skip introducing corruption in the number
+ // of chunks (which isn't checksummed
+ return inputs().flatMap(s -> integers().between(2, s.length + 2).map(i -> Pair.create(s, i)));
+ }
+
+ private static Gen<ReusableBuffer> inputs()
+ {
+ Gen<ReusableBuffer> randomStrings = inputs(0, MAX_INPUT_SIZE, 0, (1 << 8) - 1);
+ Gen<ReusableBuffer> highlyCompressable = inputs(1, MAX_INPUT_SIZE, 'c', 'e');
+ return randomStrings.mix(highlyCompressable, 50);
+ }
+
+ private static Gen<ReusableBuffer> inputs(int minSize, int maxSize, int smallestByte, int largestByte)
+ {
+ ReusableBuffer buffer = new ReusableBuffer(new byte[maxSize]);
+ Constraint byteGen = Constraint.between(smallestByte, largestByte);
+ Constraint lengthGen = Constraint.between(minSize, maxSize);
+ Gen<ReusableBuffer> gen = td -> {
+ int size = (int) td.next(lengthGen);
+ buffer.length = size;
+ for (int i = 0; i < size; i++)
+ buffer.bytes[i] = (byte) td.next(byteGen);
+ return buffer;
+ };
+ return gen;
+ }
+
+ private Gen<ReusableBuffer> zeroLengthInputs()
+ {
+ return arbitrary().constant(new ReusableBuffer(new byte[0]));
+ }
+
+ private Gen<Compressor> compressors()
+ {
+ return arbitrary().pick(null, LZ4Compressor.INSTANCE, SnappyCompressor.INSTANCE);
+ }
+
+ private Gen<ChecksumType> checksumTypes()
+ {
+ return arbitrary().enumValuesWithNoOrder(ChecksumType.class);
+ }
+
+ private Gen<Integer> blocksizes()
+ {
+ return arbitrary().constant(DEFAULT_BLOCK_SIZE);
+ }
+}
diff --git a/test/unit/org/apache/cassandra/transport/frame/checksum/ChecksummingWithCorruptedLZ4DoesNotCrashTest.java b/test/unit/org/apache/cassandra/transport/frame/checksum/ChecksummingWithCorruptedLZ4DoesNotCrashTest.java
new file mode 100644
index 0000000..4028bfd
--- /dev/null
+++ b/test/unit/org/apache/cassandra/transport/frame/checksum/ChecksummingWithCorruptedLZ4DoesNotCrashTest.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.transport.frame.checksum;
+
+import java.io.BufferedReader;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import io.netty.buffer.ByteBufUtil;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.transport.frame.compress.LZ4Compressor;
+import org.apache.cassandra.utils.ChecksumType;
+import org.apache.cassandra.utils.Pair;
+
+/**
+ * When we use LZ4 with "fast" functions its unsafe in the case the stream is corrupt; for networking checksuming is
+ * after lz4 decompresses (see CASSANDRA-15299) which means that lz4 can crash the process.
+ *
+ * This test is stand alone for the reason that this test is known to cause the JVM to crash. Given the way we run tests
+ * in CI this will kill the runner which means the file will be marked as failed; if this test was embedded into another
+ * test file then all the other tests would be ignored if this crashes.
+ */
+public class ChecksummingWithCorruptedLZ4DoesNotCrashTest
+{
+ @BeforeClass
+ public static void init()
+ {
+ // required as static ChecksummingTransformer instances read default block size from config
+ DatabaseDescriptor.clientInitialization();
+ }
+
+ @Test
+ public void shouldNotCrash() throws IOException
+ {
+ // We found lz4 caused the JVM to crash, so used the input (bytes and byteToCorrupt) to the test which crashed
+ // to reproduce.
+ // It was found that the same input does not cause lz4 to crash by it self but needed repeated calls with this
+ // input produce such a failure.
+ String failureHex;
+ try (BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream("test/data/CASSANDRA-15313/lz4-jvm-crash-failure.txt"), StandardCharsets.UTF_8))) {
+ failureHex = reader.readLine().trim();
+ }
+ byte[] failure = ByteBufUtil.decodeHexDump(failureHex);
+ ReusableBuffer buffer = new ReusableBuffer(failure);
+ int byteToCorrupt = 52997;
+ // corrupting these values causes the exception.
+ byte[] corruptionValues = new byte[] { 21, 57, 79, (byte) 179 };
+ // 5k was chosen as the largest number of iterations seen needed to crash.
+ for (int i = 0; i < 5_000 ; i++) {
+ for (byte corruptionValue : corruptionValues) {
+ try {
+ ChecksummingTransformerTest.roundTripWithCorruption(Pair.create(buffer, byteToCorrupt), corruptionValue, LZ4Compressor.INSTANCE, ChecksumType.ADLER32);
+ } catch (AssertionError e) {
+ // ignore
+ }
+ }
+ }
+ }
+}
diff --git a/test/unit/org/apache/cassandra/transport/frame/checksum/ReusableBuffer.java b/test/unit/org/apache/cassandra/transport/frame/checksum/ReusableBuffer.java
new file mode 100644
index 0000000..c0e0abe
--- /dev/null
+++ b/test/unit/org/apache/cassandra/transport/frame/checksum/ReusableBuffer.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.transport.frame.checksum;
+
+import java.nio.ByteBuffer;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.Unpooled;
+import org.apache.cassandra.utils.ByteBufferUtil;
+
+/**
+ * Wrapper around byte[] with length which is expected to be reused with different data. It is expected that the bytes
+ * are directly modified and the length gets updated to reflect; this is done to avoid producing unneeded garbage.
+ *
+ * This class is not thread safe.
+ */
+public final class ReusableBuffer
+{
+ public final byte[] bytes;
+ public int length;
+
+ public ReusableBuffer(byte[] bytes)
+ {
+ this.bytes = bytes;
+ this.length = bytes.length;
+ }
+
+ public ByteBuf toByteBuf() {
+ return Unpooled.wrappedBuffer(bytes, 0, length);
+ }
+
+ public String toString()
+ {
+ return ByteBufferUtil.bytesToHex(ByteBuffer.wrap(bytes, 0, length));
+ }
+}
diff --git a/test/unit/org/apache/cassandra/transport/messages/AuthenticateMessageTest.java b/test/unit/org/apache/cassandra/transport/messages/AuthenticateMessageTest.java
new file mode 100644
index 0000000..2c957c9
--- /dev/null
+++ b/test/unit/org/apache/cassandra/transport/messages/AuthenticateMessageTest.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.transport.messages;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import org.apache.cassandra.auth.PasswordAuthenticator;
+import org.apache.cassandra.transport.Message;
+import org.apache.cassandra.transport.ProtocolVersion;
+
+public class AuthenticateMessageTest extends EncodeAndDecodeTestBase<AuthenticateMessage>
+{
+ @Test
+ public void testEncodeAndDecode()
+ {
+ AuthenticateMessage origin = new AuthenticateMessage(PasswordAuthenticator.class.getName());
+ AuthenticateMessage newMessage = encodeThenDecode(origin, ProtocolVersion.V5);
+ Assert.assertEquals(origin.toString(), newMessage.toString());
+ }
+
+ protected Message.Codec<AuthenticateMessage> getCodec()
+ {
+ return AuthenticateMessage.codec;
+ }
+}
diff --git a/test/unit/org/apache/cassandra/transport/messages/EncodeAndDecodeTestBase.java b/test/unit/org/apache/cassandra/transport/messages/EncodeAndDecodeTestBase.java
new file mode 100644
index 0000000..beffccb
--- /dev/null
+++ b/test/unit/org/apache/cassandra/transport/messages/EncodeAndDecodeTestBase.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.transport.messages;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.Unpooled;
+import org.apache.cassandra.transport.Message;
+import org.apache.cassandra.transport.ProtocolVersion;
+
+public abstract class EncodeAndDecodeTestBase<T extends Message>
+{
+ protected abstract Message.Codec<T> getCodec();
+
+ protected T encodeThenDecode(T message, ProtocolVersion version)
+ {
+ int size = getCodec().encodedSize(message, version);
+ ByteBuf buffer = Unpooled.buffer(size, size);
+ getCodec().encode(message, buffer, version);
+ return getCodec().decode(buffer, version);
+ }
+}
diff --git a/test/unit/org/apache/cassandra/transport/messages/PrepareMessageTest.java b/test/unit/org/apache/cassandra/transport/messages/PrepareMessageTest.java
new file mode 100644
index 0000000..8425681
--- /dev/null
+++ b/test/unit/org/apache/cassandra/transport/messages/PrepareMessageTest.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.transport.messages;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import org.apache.cassandra.transport.Message;
+import org.apache.cassandra.transport.ProtocolVersion;
+
+public class PrepareMessageTest extends EncodeAndDecodeTestBase<PrepareMessage>
+{
+ @Test
+ public void testEncodeThenDecode()
+ {
+ PrepareMessage origin = new PrepareMessage("SELECT * FROM keyspace.tbl WHERE name='ßètæ'", "keyspace");
+ PrepareMessage newMessage = encodeThenDecode(origin, ProtocolVersion.V5);
+ Assert.assertEquals(origin.toString(), newMessage.toString());
+ }
+
+ protected Message.Codec<PrepareMessage> getCodec()
+ {
+ return PrepareMessage.codec;
+ }
+}
diff --git a/test/unit/org/apache/cassandra/triggers/TriggerExecutorTest.java b/test/unit/org/apache/cassandra/triggers/TriggerExecutorTest.java
index 0e4130d..7e9f626 100644
--- a/test/unit/org/apache/cassandra/triggers/TriggerExecutorTest.java
+++ b/test/unit/org/apache/cassandra/triggers/TriggerExecutorTest.java
@@ -23,7 +23,7 @@
import org.junit.Test;
import org.apache.cassandra.Util;
-import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.rows.*;
@@ -33,6 +33,8 @@
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.exceptions.InvalidRequestException;
import org.apache.cassandra.schema.TriggerMetadata;
+import org.apache.cassandra.schema.Triggers;
+import org.apache.cassandra.triggers.TriggerExecutorTest.SameKeySameCfTrigger;
import org.apache.cassandra.utils.FBUtilities;
import static org.apache.cassandra.utils.ByteBufferUtil.bytes;
@@ -51,7 +53,7 @@
@Test
public void sameKeySameCfColumnFamilies() throws ConfigurationException, InvalidRequestException
{
- CFMetaData metadata = makeCfMetaData("ks1", "cf1", TriggerMetadata.create("test", SameKeySameCfTrigger.class.getName()));
+ TableMetadata metadata = makeTableMetadata("ks1", "cf1", TriggerMetadata.create("test", SameKeySameCfTrigger.class.getName()));
// origin column 'c1' = "v1", augment extra column 'c2' = "trigger"
PartitionUpdate mutated = TriggerExecutor.instance.execute(makeCf(metadata, "k1", "v1", null));
@@ -80,21 +82,21 @@
@Test(expected = InvalidRequestException.class)
public void sameKeyDifferentCfColumnFamilies() throws ConfigurationException, InvalidRequestException
{
- CFMetaData metadata = makeCfMetaData("ks1", "cf1", TriggerMetadata.create("test", SameKeyDifferentCfTrigger.class.getName()));
+ TableMetadata metadata = makeTableMetadata("ks1", "cf1", TriggerMetadata.create("test", SameKeyDifferentCfTrigger.class.getName()));
TriggerExecutor.instance.execute(makeCf(metadata, "k1", "v1", null));
}
@Test(expected = InvalidRequestException.class)
public void differentKeyColumnFamilies() throws ConfigurationException, InvalidRequestException
{
- CFMetaData metadata = makeCfMetaData("ks1", "cf1", TriggerMetadata.create("test", DifferentKeyTrigger.class.getName()));
+ TableMetadata metadata = makeTableMetadata("ks1", "cf1", TriggerMetadata.create("test", DifferentKeyTrigger.class.getName()));
TriggerExecutor.instance.execute(makeCf(metadata, "k1", "v1", null));
}
@Test
public void noTriggerMutations() throws ConfigurationException, InvalidRequestException
{
- CFMetaData metadata = makeCfMetaData("ks1", "cf1", TriggerMetadata.create("test", NoOpTrigger.class.getName()));
+ TableMetadata metadata = makeTableMetadata("ks1", "cf1", TriggerMetadata.create("test", NoOpTrigger.class.getName()));
Mutation rm = new Mutation(makeCf(metadata, "k1", "v1", null));
assertNull(TriggerExecutor.instance.execute(Collections.singletonList(rm)));
}
@@ -102,11 +104,11 @@
@Test
public void sameKeySameCfRowMutations() throws ConfigurationException, InvalidRequestException
{
- CFMetaData metadata = makeCfMetaData("ks1", "cf1", TriggerMetadata.create("test", SameKeySameCfTrigger.class.getName()));
+ TableMetadata metadata = makeTableMetadata("ks1", "cf1", TriggerMetadata.create("test", SameKeySameCfTrigger.class.getName()));
PartitionUpdate cf1 = makeCf(metadata, "k1", "k1v1", null);
PartitionUpdate cf2 = makeCf(metadata, "k2", "k2v1", null);
- Mutation rm1 = new Mutation("ks1", cf1.partitionKey()).add(cf1);
- Mutation rm2 = new Mutation("ks1", cf2.partitionKey()).add(cf2);
+ Mutation rm1 = new Mutation.PartitionUpdateCollector("ks1", cf1.partitionKey()).add(cf1).build();
+ Mutation rm2 = new Mutation.PartitionUpdateCollector("ks1", cf2.partitionKey()).add(cf2).build();
List<? extends IMutation> tmutations = new ArrayList<>(TriggerExecutor.instance.execute(Arrays.asList(rm1, rm2)));
assertEquals(2, tmutations.size());
@@ -115,24 +117,24 @@
List<PartitionUpdate> mutatedCFs = new ArrayList<>(tmutations.get(0).getPartitionUpdates());
assertEquals(1, mutatedCFs.size());
Row row = mutatedCFs.get(0).iterator().next();
- assertEquals(bytes("k1v1"), row.getCell(metadata.getColumnDefinition(bytes("c1"))).value());
- assertEquals(bytes("trigger"), row.getCell(metadata.getColumnDefinition(bytes("c2"))).value());
+ assertEquals(bytes("k1v1"), row.getCell(metadata.getColumn(bytes("c1"))).value());
+ assertEquals(bytes("trigger"), row.getCell(metadata.getColumn(bytes("c2"))).value());
mutatedCFs = new ArrayList<>(tmutations.get(1).getPartitionUpdates());
assertEquals(1, mutatedCFs.size());
row = mutatedCFs.get(0).iterator().next();
- assertEquals(bytes("k2v1"), row.getCell(metadata.getColumnDefinition(bytes("c1"))).value());
- assertEquals(bytes("trigger"), row.getCell(metadata.getColumnDefinition(bytes("c2"))).value());
+ assertEquals(bytes("k2v1"), row.getCell(metadata.getColumn(bytes("c1"))).value());
+ assertEquals(bytes("trigger"), row.getCell(metadata.getColumn(bytes("c2"))).value());
}
@Test
public void sameKeySameCfPartialRowMutations() throws ConfigurationException, InvalidRequestException
{
- CFMetaData metadata = makeCfMetaData("ks1", "cf1", TriggerMetadata.create("test", SameKeySameCfPartialTrigger.class.getName()));
+ TableMetadata metadata = makeTableMetadata("ks1", "cf1", TriggerMetadata.create("test", SameKeySameCfPartialTrigger.class.getName()));
PartitionUpdate cf1 = makeCf(metadata, "k1", "k1v1", null);
PartitionUpdate cf2 = makeCf(metadata, "k2", "k2v1", null);
- Mutation rm1 = new Mutation("ks1", cf1.partitionKey()).add(cf1);
- Mutation rm2 = new Mutation("ks1", cf2.partitionKey()).add(cf2);
+ Mutation rm1 = new Mutation.PartitionUpdateCollector("ks1", cf1.partitionKey()).add(cf1).build();
+ Mutation rm2 = new Mutation.PartitionUpdateCollector("ks1", cf2.partitionKey()).add(cf2).build();
List<? extends IMutation> tmutations = new ArrayList<>(TriggerExecutor.instance.execute(Arrays.asList(rm1, rm2)));
assertEquals(2, tmutations.size());
@@ -141,24 +143,24 @@
List<PartitionUpdate> mutatedCFs = new ArrayList<>(tmutations.get(0).getPartitionUpdates());
assertEquals(1, mutatedCFs.size());
Row row = mutatedCFs.get(0).iterator().next();
- assertEquals(bytes("k1v1"), row.getCell(metadata.getColumnDefinition(bytes("c1"))).value());
- assertNull(row.getCell(metadata.getColumnDefinition(bytes("c2"))));
+ assertEquals(bytes("k1v1"), row.getCell(metadata.getColumn(bytes("c1"))).value());
+ assertNull(row.getCell(metadata.getColumn(bytes("c2"))));
mutatedCFs = new ArrayList<>(tmutations.get(1).getPartitionUpdates());
assertEquals(1, mutatedCFs.size());
row = mutatedCFs.get(0).iterator().next();
- assertEquals(bytes("k2v1"), row.getCell(metadata.getColumnDefinition(bytes("c1"))).value());
- assertEquals(bytes("trigger"), row.getCell(metadata.getColumnDefinition(bytes("c2"))).value());
+ assertEquals(bytes("k2v1"), row.getCell(metadata.getColumn(bytes("c1"))).value());
+ assertEquals(bytes("trigger"), row.getCell(metadata.getColumn(bytes("c2"))).value());
}
@Test
public void sameKeyDifferentCfRowMutations() throws ConfigurationException, InvalidRequestException
{
- CFMetaData metadata = makeCfMetaData("ks1", "cf1", TriggerMetadata.create("test", SameKeyDifferentCfTrigger.class.getName()));
+ TableMetadata metadata = makeTableMetadata("ks1", "cf1", TriggerMetadata.create("test", SameKeyDifferentCfTrigger.class.getName()));
PartitionUpdate cf1 = makeCf(metadata, "k1", "k1v1", null);
PartitionUpdate cf2 = makeCf(metadata, "k2", "k2v1", null);
- Mutation rm1 = new Mutation("ks1", cf1.partitionKey()).add(cf1);
- Mutation rm2 = new Mutation("ks1", cf2.partitionKey()).add(cf2);
+ Mutation rm1 = new Mutation.PartitionUpdateCollector("ks1", cf1.partitionKey()).add(cf1).build();
+ Mutation rm2 = new Mutation.PartitionUpdateCollector("ks1", cf2.partitionKey()).add(cf2).build();
List<? extends IMutation> tmutations = new ArrayList<>(TriggerExecutor.instance.execute(Arrays.asList(rm1, rm2)));
assertEquals(2, tmutations.size());
@@ -168,17 +170,17 @@
assertEquals(2, mutatedCFs.size());
for (PartitionUpdate update : mutatedCFs)
{
- if (update.metadata().cfName.equals("cf1"))
+ if (update.metadata().name.equals("cf1"))
{
Row row = update.iterator().next();
- assertEquals(bytes("k1v1"), row.getCell(metadata.getColumnDefinition(bytes("c1"))).value());
- assertNull(row.getCell(metadata.getColumnDefinition(bytes("c2"))));
+ assertEquals(bytes("k1v1"), row.getCell(metadata.getColumn(bytes("c1"))).value());
+ assertNull(row.getCell(metadata.getColumn(bytes("c2"))));
}
else
{
Row row = update.iterator().next();
- assertNull(row.getCell(metadata.getColumnDefinition(bytes("c1"))));
- assertEquals(bytes("trigger"), row.getCell(metadata.getColumnDefinition(bytes("c2"))).value());
+ assertNull(row.getCell(metadata.getColumn(bytes("c1"))));
+ assertEquals(bytes("trigger"), row.getCell(metadata.getColumn(bytes("c2"))).value());
}
}
@@ -187,17 +189,17 @@
for (PartitionUpdate update : mutatedCFs)
{
- if (update.metadata().cfName.equals("cf1"))
+ if (update.metadata().name.equals("cf1"))
{
Row row = update.iterator().next();
- assertEquals(bytes("k2v1"), row.getCell(metadata.getColumnDefinition(bytes("c1"))).value());
- assertNull(row.getCell(metadata.getColumnDefinition(bytes("c2"))));
+ assertEquals(bytes("k2v1"), row.getCell(metadata.getColumn(bytes("c1"))).value());
+ assertNull(row.getCell(metadata.getColumn(bytes("c2"))));
}
else
{
Row row = update.iterator().next();
- assertNull(row.getCell(metadata.getColumnDefinition(bytes("c1"))));
- assertEquals(bytes("trigger"), row.getCell(metadata.getColumnDefinition(bytes("c2"))).value());
+ assertNull(row.getCell(metadata.getColumn(bytes("c1"))));
+ assertEquals(bytes("trigger"), row.getCell(metadata.getColumn(bytes("c2"))).value());
}
}
}
@@ -205,11 +207,11 @@
@Test
public void sameKeyDifferentKsRowMutations() throws ConfigurationException, InvalidRequestException
{
- CFMetaData metadata = makeCfMetaData("ks1", "cf1", TriggerMetadata.create("test", SameKeyDifferentKsTrigger.class.getName()));
+ TableMetadata metadata = makeTableMetadata("ks1", "cf1", TriggerMetadata.create("test", SameKeyDifferentKsTrigger.class.getName()));
PartitionUpdate cf1 = makeCf(metadata, "k1", "k1v1", null);
PartitionUpdate cf2 = makeCf(metadata, "k2", "k2v1", null);
- Mutation rm1 = new Mutation("ks1", cf1.partitionKey()).add(cf1);
- Mutation rm2 = new Mutation("ks1", cf2.partitionKey()).add(cf2);
+ Mutation rm1 = new Mutation.PartitionUpdateCollector("ks1", cf1.partitionKey()).add(cf1).build();
+ Mutation rm2 = new Mutation.PartitionUpdateCollector("ks1", cf2.partitionKey()).add(cf2).build();
List<? extends IMutation> tmutations = new ArrayList<>(TriggerExecutor.instance.execute(Arrays.asList(rm1, rm2)));
assertEquals(4, tmutations.size());
@@ -218,35 +220,35 @@
List<PartitionUpdate> mutatedCFs = new ArrayList<>(tmutations.get(0).getPartitionUpdates());
assertEquals(1, mutatedCFs.size());
Row row = mutatedCFs.get(0).iterator().next();
- assertEquals(bytes("k1v1"), row.getCell(metadata.getColumnDefinition(bytes("c1"))).value());
- assertNull(row.getCell(metadata.getColumnDefinition(bytes("c2"))));
+ assertEquals(bytes("k1v1"), row.getCell(metadata.getColumn(bytes("c1"))).value());
+ assertNull(row.getCell(metadata.getColumn(bytes("c2"))));
mutatedCFs = new ArrayList<>(tmutations.get(1).getPartitionUpdates());
assertEquals(1, mutatedCFs.size());
row = mutatedCFs.get(0).iterator().next();
- assertEquals(bytes("k2v1"), row.getCell(metadata.getColumnDefinition(bytes("c1"))).value());
- assertNull(row.getCell(metadata.getColumnDefinition(bytes("c2"))));
+ assertEquals(bytes("k2v1"), row.getCell(metadata.getColumn(bytes("c1"))).value());
+ assertNull(row.getCell(metadata.getColumn(bytes("c2"))));
mutatedCFs = new ArrayList<>(tmutations.get(2).getPartitionUpdates());
assertEquals(1, mutatedCFs.size());
row = mutatedCFs.get(0).iterator().next();
- assertNull(row.getCell(metadata.getColumnDefinition(bytes("c1"))));
- assertEquals(bytes("trigger"), row.getCell(metadata.getColumnDefinition(bytes("c2"))).value());
+ assertNull(row.getCell(metadata.getColumn(bytes("c1"))));
+ assertEquals(bytes("trigger"), row.getCell(metadata.getColumn(bytes("c2"))).value());
mutatedCFs = new ArrayList<>(tmutations.get(3).getPartitionUpdates());
assertEquals(1, mutatedCFs.size());
row = mutatedCFs.get(0).iterator().next();
- assertNull(row.getCell(metadata.getColumnDefinition(bytes("c1"))));
- assertEquals(bytes("trigger"), row.getCell(metadata.getColumnDefinition(bytes("c2"))).value());
+ assertNull(row.getCell(metadata.getColumn(bytes("c1"))));
+ assertEquals(bytes("trigger"), row.getCell(metadata.getColumn(bytes("c2"))).value());
}
@Test
public void differentKeyRowMutations() throws ConfigurationException, InvalidRequestException
{
- CFMetaData metadata = makeCfMetaData("ks1", "cf1", TriggerMetadata.create("test", DifferentKeyTrigger.class.getName()));
+ TableMetadata metadata = makeTableMetadata("ks1", "cf1", TriggerMetadata.create("test", DifferentKeyTrigger.class.getName()));
PartitionUpdate cf1 = makeCf(metadata, "k1", "v1", null);
- Mutation rm = new Mutation("ks1", cf1.partitionKey()).add(cf1);
+ Mutation rm = new Mutation.PartitionUpdateCollector("ks1", cf1.partitionKey()).add(cf1).build();
List<? extends IMutation> tmutations = new ArrayList<>(TriggerExecutor.instance.execute(Arrays.asList(rm)));
assertEquals(2, tmutations.size());
@@ -258,46 +260,39 @@
List<PartitionUpdate> mutatedCFs = new ArrayList<>(tmutations.get(0).getPartitionUpdates());
assertEquals(1, mutatedCFs.size());
Row row = mutatedCFs.get(0).iterator().next();
- assertEquals(bytes("v1"), row.getCell(metadata.getColumnDefinition(bytes("c1"))).value());
- assertNull(row.getCell(metadata.getColumnDefinition(bytes("c2"))));
+ assertEquals(bytes("v1"), row.getCell(metadata.getColumn(bytes("c1"))).value());
+ assertNull(row.getCell(metadata.getColumn(bytes("c2"))));
mutatedCFs = new ArrayList<>(tmutations.get(1).getPartitionUpdates());
assertEquals(1, mutatedCFs.size());
row = mutatedCFs.get(0).iterator().next();
- assertEquals(bytes("trigger"), row.getCell(metadata.getColumnDefinition(bytes("c2"))).value());
- assertNull(row.getCell(metadata.getColumnDefinition(bytes("c1"))));
+ assertEquals(bytes("trigger"), row.getCell(metadata.getColumn(bytes("c2"))).value());
+ assertNull(row.getCell(metadata.getColumn(bytes("c1"))));
}
- private static CFMetaData makeCfMetaData(String ks, String cf, TriggerMetadata trigger)
+ private static TableMetadata makeTableMetadata(String ks, String cf, TriggerMetadata trigger)
{
- CFMetaData metadata = CFMetaData.Builder.create(ks, cf)
- .addPartitionKey("pkey", UTF8Type.instance)
- .addRegularColumn("c1", UTF8Type.instance)
- .addRegularColumn("c2", UTF8Type.instance)
- .build();
+ TableMetadata.Builder builder =
+ TableMetadata.builder(ks, cf)
+ .addPartitionKeyColumn("pkey", UTF8Type.instance)
+ .addRegularColumn("c1", UTF8Type.instance)
+ .addRegularColumn("c2", UTF8Type.instance);
- try
- {
- if (trigger != null)
- metadata.triggers(metadata.getTriggers().with(trigger));
- }
- catch (InvalidRequestException e)
- {
- throw new AssertionError(e);
- }
+ if (trigger != null)
+ builder.triggers(Triggers.of(trigger));
- return metadata;
+ return builder.build();
}
- private static PartitionUpdate makeCf(CFMetaData metadata, String key, String columnValue1, String columnValue2)
+ private static PartitionUpdate makeCf(TableMetadata metadata, String key, String columnValue1, String columnValue2)
{
- Row.Builder builder = BTreeRow.unsortedBuilder(FBUtilities.nowInSeconds());
+ Row.Builder builder = BTreeRow.unsortedBuilder();
builder.newRow(Clustering.EMPTY);
long ts = FBUtilities.timestampMicros();
if (columnValue1 != null)
- builder.addCell(BufferCell.live(metadata.getColumnDefinition(bytes("c1")), ts, bytes(columnValue1)));
+ builder.addCell(BufferCell.live(metadata.getColumn(bytes("c1")), ts, bytes(columnValue1)));
if (columnValue2 != null)
- builder.addCell(BufferCell.live(metadata.getColumnDefinition(bytes("c2")), ts, bytes(columnValue2)));
+ builder.addCell(BufferCell.live(metadata.getColumn(bytes("c2")), ts, bytes(columnValue2)));
return PartitionUpdate.singleRowUpdate(metadata, Util.dk(key), builder.build());
}
@@ -337,7 +332,7 @@
{
public Collection<Mutation> augment(Partition partition)
{
- RowUpdateBuilder builder = new RowUpdateBuilder(makeCfMetaData(partition.metadata().ksName, "otherCf", null), FBUtilities.timestampMicros(), partition.partitionKey().getKey());
+ RowUpdateBuilder builder = new RowUpdateBuilder(makeTableMetadata(partition.metadata().keyspace, "otherCf", null), FBUtilities.timestampMicros(), partition.partitionKey().getKey());
builder.add("c2", bytes("trigger"));
return Collections.singletonList(builder.build());
}
@@ -347,7 +342,7 @@
{
public Collection<Mutation> augment(Partition partition)
{
- RowUpdateBuilder builder = new RowUpdateBuilder(makeCfMetaData("otherKs", "otherCf", null), FBUtilities.timestampMicros(), partition.partitionKey().getKey());
+ RowUpdateBuilder builder = new RowUpdateBuilder(makeTableMetadata("otherKs", "otherCf", null), FBUtilities.timestampMicros(), partition.partitionKey().getKey());
builder.add("c2", bytes("trigger"));
return Collections.singletonList(builder.build());
}
@@ -357,7 +352,7 @@
{
public Collection<Mutation> augment(Partition partition)
{
- RowUpdateBuilder builder = new RowUpdateBuilder(makeCfMetaData("otherKs", "otherCf", null), FBUtilities.timestampMicros(), "otherKey");
+ RowUpdateBuilder builder = new RowUpdateBuilder(makeTableMetadata("otherKs", "otherCf", null), FBUtilities.timestampMicros(), "otherKey");
builder.add("c2", bytes("trigger"));
return Collections.singletonList(builder.build());
}
diff --git a/test/unit/org/apache/cassandra/triggers/TriggersSchemaTest.java b/test/unit/org/apache/cassandra/triggers/TriggersSchemaTest.java
index b6549bb..31111bd 100644
--- a/test/unit/org/apache/cassandra/triggers/TriggersSchemaTest.java
+++ b/test/unit/org/apache/cassandra/triggers/TriggersSchemaTest.java
@@ -21,14 +21,16 @@
import org.junit.Test;
import org.apache.cassandra.SchemaLoader;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.Schema;
+import org.apache.cassandra.cql3.statements.schema.CreateTableStatement;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.schema.KeyspaceMetadata;
import org.apache.cassandra.schema.KeyspaceParams;
import org.apache.cassandra.schema.Tables;
import org.apache.cassandra.schema.TriggerMetadata;
-import org.apache.cassandra.service.MigrationManager;
+import org.apache.cassandra.schema.Triggers;
+import org.apache.cassandra.schema.MigrationManager;
import static org.junit.Assert.*;
@@ -49,15 +51,18 @@
public void newKsContainsCfWithTrigger() throws Exception
{
TriggerMetadata td = TriggerMetadata.create(triggerName, triggerClass);
- CFMetaData cfm1 = CFMetaData.compile(String.format("CREATE TABLE %s (k int PRIMARY KEY, v int)", cfName), ksName);
- cfm1.triggers(cfm1.getTriggers().with(td));
- KeyspaceMetadata ksm = KeyspaceMetadata.create(ksName, KeyspaceParams.simple(1), Tables.of(cfm1));
+ TableMetadata tm =
+ CreateTableStatement.parse(String.format("CREATE TABLE %s (k int PRIMARY KEY, v int)", cfName), ksName)
+ .triggers(Triggers.of(td))
+ .build();
+
+ KeyspaceMetadata ksm = KeyspaceMetadata.create(ksName, KeyspaceParams.simple(1), Tables.of(tm));
MigrationManager.announceNewKeyspace(ksm);
- CFMetaData cfm2 = Schema.instance.getCFMetaData(ksName, cfName);
- assertFalse(cfm2.getTriggers().isEmpty());
- assertEquals(1, cfm2.getTriggers().size());
- assertEquals(td, cfm2.getTriggers().get(triggerName).get());
+ TableMetadata tm2 = Schema.instance.getTableMetadata(ksName, cfName);
+ assertFalse(tm2.triggers.isEmpty());
+ assertEquals(1, tm2.triggers.size());
+ assertEquals(td, tm2.triggers.get(triggerName).get());
}
@Test
@@ -66,50 +71,62 @@
KeyspaceMetadata ksm = KeyspaceMetadata.create(ksName, KeyspaceParams.simple(1));
MigrationManager.announceNewKeyspace(ksm);
- CFMetaData cfm1 = CFMetaData.compile(String.format("CREATE TABLE %s (k int PRIMARY KEY, v int)", cfName), ksName);
- TriggerMetadata td = TriggerMetadata.create(triggerName, triggerClass);
- cfm1.triggers(cfm1.getTriggers().with(td));
+ TableMetadata metadata =
+ CreateTableStatement.parse(String.format("CREATE TABLE %s (k int PRIMARY KEY, v int)", cfName), ksName)
+ .triggers(Triggers.of(TriggerMetadata.create(triggerName, triggerClass)))
+ .build();
- MigrationManager.announceNewColumnFamily(cfm1);
+ MigrationManager.announceNewTable(metadata);
- CFMetaData cfm2 = Schema.instance.getCFMetaData(ksName, cfName);
- assertFalse(cfm2.getTriggers().isEmpty());
- assertEquals(1, cfm2.getTriggers().size());
- assertEquals(td, cfm2.getTriggers().get(triggerName).get());
+ metadata = Schema.instance.getTableMetadata(ksName, cfName);
+ assertFalse(metadata.triggers.isEmpty());
+ assertEquals(1, metadata.triggers.size());
+ assertEquals(TriggerMetadata.create(triggerName, triggerClass), metadata.triggers.get(triggerName).get());
}
@Test
public void addTriggerToCf() throws Exception
{
- CFMetaData cfm1 = CFMetaData.compile(String.format("CREATE TABLE %s (k int PRIMARY KEY, v int)", cfName), ksName);
- KeyspaceMetadata ksm = KeyspaceMetadata.create(ksName, KeyspaceParams.simple(1), Tables.of(cfm1));
+ TableMetadata tm1 =
+ CreateTableStatement.parse(String.format("CREATE TABLE %s (k int PRIMARY KEY, v int)", cfName), ksName)
+ .build();
+ KeyspaceMetadata ksm = KeyspaceMetadata.create(ksName, KeyspaceParams.simple(1), Tables.of(tm1));
MigrationManager.announceNewKeyspace(ksm);
- CFMetaData cfm2 = Schema.instance.getCFMetaData(ksName, cfName).copy();
TriggerMetadata td = TriggerMetadata.create(triggerName, triggerClass);
- cfm2.triggers(cfm2.getTriggers().with(td));
- MigrationManager.announceColumnFamilyUpdate(cfm2);
+ TableMetadata tm2 =
+ Schema.instance
+ .getTableMetadata(ksName, cfName)
+ .unbuild()
+ .triggers(Triggers.of(td))
+ .build();
+ MigrationManager.announceTableUpdate(tm2);
- CFMetaData cfm3 = Schema.instance.getCFMetaData(ksName, cfName);
- assertFalse(cfm3.getTriggers().isEmpty());
- assertEquals(1, cfm3.getTriggers().size());
- assertEquals(td, cfm3.getTriggers().get(triggerName).get());
+ TableMetadata tm3 = Schema.instance.getTableMetadata(ksName, cfName);
+ assertFalse(tm3.triggers.isEmpty());
+ assertEquals(1, tm3.triggers.size());
+ assertEquals(td, tm3.triggers.get(triggerName).get());
}
@Test
public void removeTriggerFromCf() throws Exception
{
TriggerMetadata td = TriggerMetadata.create(triggerName, triggerClass);
- CFMetaData cfm1 = CFMetaData.compile(String.format("CREATE TABLE %s (k int PRIMARY KEY, v int)", cfName), ksName);
- cfm1.triggers(cfm1.getTriggers().with(td));
- KeyspaceMetadata ksm = KeyspaceMetadata.create(ksName, KeyspaceParams.simple(1), Tables.of(cfm1));
+ TableMetadata tm =
+ CreateTableStatement.parse(String.format("CREATE TABLE %s (k int PRIMARY KEY, v int)", cfName), ksName)
+ .triggers(Triggers.of(td))
+ .build();
+ KeyspaceMetadata ksm = KeyspaceMetadata.create(ksName, KeyspaceParams.simple(1), Tables.of(tm));
MigrationManager.announceNewKeyspace(ksm);
- CFMetaData cfm2 = Schema.instance.getCFMetaData(ksName, cfName).copy();
- cfm2.triggers(cfm2.getTriggers().without(triggerName));
- MigrationManager.announceColumnFamilyUpdate(cfm2);
+ TableMetadata tm1 = Schema.instance.getTableMetadata(ksName, cfName);
+ TableMetadata tm2 =
+ tm1.unbuild()
+ .triggers(tm1.triggers.without(triggerName))
+ .build();
+ MigrationManager.announceTableUpdate(tm2);
- CFMetaData cfm3 = Schema.instance.getCFMetaData(ksName, cfName).copy();
- assertTrue(cfm3.getTriggers().isEmpty());
+ TableMetadata tm3 = Schema.instance.getTableMetadata(ksName, cfName);
+ assertTrue(tm3.triggers.isEmpty());
}
}
diff --git a/test/unit/org/apache/cassandra/triggers/TriggersTest.java b/test/unit/org/apache/cassandra/triggers/TriggersTest.java
index 5f2a553..2cf0e84 100644
--- a/test/unit/org/apache/cassandra/triggers/TriggersTest.java
+++ b/test/unit/org/apache/cassandra/triggers/TriggersTest.java
@@ -20,13 +20,12 @@
import java.util.Collection;
import java.util.Collections;
-import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import org.apache.cassandra.SchemaLoader;
-import org.apache.cassandra.config.Schema;
+import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.cql3.QueryProcessor;
import org.apache.cassandra.cql3.UntypedResultSet;
import org.apache.cassandra.db.*;
@@ -36,11 +35,8 @@
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.exceptions.RequestExecutionException;
import org.apache.cassandra.service.StorageService;
-import org.apache.cassandra.thrift.*;
import org.apache.cassandra.utils.FBUtilities;
-import org.apache.thrift.protocol.TBinaryProtocol;
-import static org.apache.cassandra.utils.ByteBufferUtil.bytes;
import static org.apache.cassandra.utils.ByteBufferUtil.toInt;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@@ -48,7 +44,6 @@
public class TriggersTest
{
private static boolean triggerCreated = false;
- private static ThriftServer thriftServer;
private static String ksName = "triggers_test_ks";
private static String cfName = "test_table";
@@ -64,11 +59,6 @@
public void setup() throws Exception
{
StorageService.instance.initServer(0);
- if (thriftServer == null || ! thriftServer.isRunning())
- {
- thriftServer = new ThriftServer(FBUtilities.getLocalAddress(), 9170, 50);
- thriftServer.start();
- }
String cql = String.format("CREATE KEYSPACE IF NOT EXISTS %s " +
"WITH REPLICATION = {'class': 'SimpleStrategy', 'replication_factor': 1}",
@@ -91,15 +81,6 @@
}
}
- @AfterClass
- public static void teardown()
- {
- if (thriftServer != null && thriftServer.isRunning())
- {
- thriftServer.stop();
- }
- }
-
@Test
public void executeTriggerOnCqlInsert() throws Exception
{
@@ -120,43 +101,6 @@
}
@Test
- public void executeTriggerOnThriftInsert() throws Exception
- {
- Cassandra.Client client = new Cassandra.Client(
- new TBinaryProtocol(
- new TFramedTransportFactory().openTransport(
- FBUtilities.getLocalAddress().getHostName(), 9170)));
- client.set_keyspace(ksName);
- client.insert(bytes(2),
- new ColumnParent(cfName),
- getColumnForInsert("v1", 2),
- org.apache.cassandra.thrift.ConsistencyLevel.ONE);
-
- assertUpdateIsAugmented(2, "v1", 2);
- }
-
- @Test
- public void executeTriggerOnThriftBatchUpdate() throws Exception
- {
- Cassandra.Client client = new Cassandra.Client(
- new TBinaryProtocol(
- new TFramedTransportFactory().openTransport(
- FBUtilities.getLocalAddress().getHostName(), 9170)));
- client.set_keyspace(ksName);
- org.apache.cassandra.thrift.Mutation mutation = new org.apache.cassandra.thrift.Mutation();
- ColumnOrSuperColumn cosc = new ColumnOrSuperColumn();
- cosc.setColumn(getColumnForInsert("v1", 3));
- mutation.setColumn_or_supercolumn(cosc);
- client.batch_mutate(
- Collections.singletonMap(bytes(3),
- Collections.singletonMap(cfName,
- Collections.singletonList(mutation))),
- org.apache.cassandra.thrift.ConsistencyLevel.ONE);
-
- assertUpdateIsAugmented(3, "v1", 3);
- }
-
- @Test
public void executeTriggerOnCqlInsertWithConditions() throws Exception
{
String cql = String.format("INSERT INTO %s.%s (k, v1) VALUES (4, 4) IF NOT EXISTS", ksName, cfName);
@@ -176,24 +120,6 @@
assertUpdateIsAugmented(5, "v1", 5);
}
- @Test
- public void executeTriggerOnThriftCASOperation() throws Exception
- {
- Cassandra.Client client = new Cassandra.Client(
- new TBinaryProtocol(
- new TFramedTransportFactory().openTransport(
- FBUtilities.getLocalAddress().getHostName(), 9170)));
- client.set_keyspace(ksName);
- client.cas(bytes(6),
- cfName,
- Collections.<Column>emptyList(),
- Collections.singletonList(getColumnForInsert("v1", 6)),
- org.apache.cassandra.thrift.ConsistencyLevel.LOCAL_SERIAL,
- org.apache.cassandra.thrift.ConsistencyLevel.ONE);
-
- assertUpdateIsAugmented(6, "v1", 6);
- }
-
@Test(expected=org.apache.cassandra.exceptions.InvalidRequestException.class)
public void onCqlUpdateWithConditionsRejectGeneratedUpdatesForDifferentPartition() throws Exception
{
@@ -226,56 +152,6 @@
}
}
- @Test(expected=InvalidRequestException.class)
- public void onThriftCASRejectGeneratedUpdatesForDifferentPartition() throws Exception
- {
- String cf = "cf" + System.nanoTime();
- try
- {
- setupTableWithTrigger(cf, CrossPartitionTrigger.class);
- Cassandra.Client client = new Cassandra.Client(
- new TBinaryProtocol(
- new TFramedTransportFactory().openTransport(
- FBUtilities.getLocalAddress().getHostName(), 9170)));
- client.set_keyspace(ksName);
- client.cas(bytes(9),
- cf,
- Collections.<Column>emptyList(),
- Collections.singletonList(getColumnForInsert("v1", 9)),
- org.apache.cassandra.thrift.ConsistencyLevel.LOCAL_SERIAL,
- org.apache.cassandra.thrift.ConsistencyLevel.ONE);
- }
- finally
- {
- assertUpdateNotExecuted(cf, 9);
- }
- }
-
- @Test(expected=InvalidRequestException.class)
- public void onThriftCASRejectGeneratedUpdatesForDifferentCF() throws Exception
- {
- String cf = "cf" + System.nanoTime();
- try
- {
- setupTableWithTrigger(cf, CrossTableTrigger.class);
- Cassandra.Client client = new Cassandra.Client(
- new TBinaryProtocol(
- new TFramedTransportFactory().openTransport(
- FBUtilities.getLocalAddress().getHostName(), 9170)));
- client.set_keyspace(ksName);
- client.cas(bytes(10),
- cf,
- Collections.<Column>emptyList(),
- Collections.singletonList(getColumnForInsert("v1", 10)),
- org.apache.cassandra.thrift.ConsistencyLevel.LOCAL_SERIAL,
- org.apache.cassandra.thrift.ConsistencyLevel.ONE);
- }
- finally
- {
- assertUpdateNotExecuted(cf, 10);
- }
- }
-
@Test(expected=org.apache.cassandra.exceptions.InvalidRequestException.class)
public void ifTriggerThrowsErrorNoMutationsAreApplied() throws Exception
{
@@ -330,15 +206,6 @@
assertTrue(rs.isEmpty());
}
- private org.apache.cassandra.thrift.Column getColumnForInsert(String columnName, int value)
- {
- org.apache.cassandra.thrift.Column column = new org.apache.cassandra.thrift.Column();
- column.setName(LegacyLayout.makeLegacyComparator(Schema.instance.getCFMetaData(ksName, cfName)).fromString(columnName));
- column.setValue(bytes(value));
- column.setTimestamp(System.currentTimeMillis());
- return column;
- }
-
public static class TestTrigger implements ITrigger
{
public Collection<Mutation> augment(Partition partition)
@@ -366,7 +233,7 @@
public Collection<Mutation> augment(Partition partition)
{
- RowUpdateBuilder update = new RowUpdateBuilder(Schema.instance.getCFMetaData(ksName, otherCf), FBUtilities.timestampMicros(), partition.partitionKey().getKey());
+ RowUpdateBuilder update = new RowUpdateBuilder(Schema.instance.getTableMetadata(ksName, otherCf), FBUtilities.timestampMicros(), partition.partitionKey().getKey());
update.add("v2", 999);
return Collections.singletonList(update.build());
diff --git a/test/unit/org/apache/cassandra/utils/AbstractIteratorTest.java b/test/unit/org/apache/cassandra/utils/AbstractIteratorTest.java
index b2f9433..8119dcb 100644
--- a/test/unit/org/apache/cassandra/utils/AbstractIteratorTest.java
+++ b/test/unit/org/apache/cassandra/utils/AbstractIteratorTest.java
@@ -16,7 +16,8 @@
package org.apache.cassandra.utils;
-import junit.framework.TestCase;
+import org.junit.Assert;
+import org.junit.Test;
import java.lang.ref.WeakReference;
import java.util.Iterator;
@@ -29,9 +30,9 @@
*/
@SuppressWarnings("serial") // No serialization is used in this test
// TODO(cpovirk): why is this slow (>1m/test) under GWT when fully optimized?
-public class AbstractIteratorTest extends TestCase
+public class AbstractIteratorTest
{
-
+ @Test
public void testDefaultBehaviorOfNextAndHasNext()
{
@@ -53,36 +54,37 @@
case 2:
return endOfData();
default:
- fail("Should not have been invoked again");
+ Assert.fail("Should not have been invoked again");
return null;
}
}
};
- assertTrue(iter.hasNext());
- assertEquals(0, (int) iter.next());
+ Assert.assertTrue(iter.hasNext());
+ Assert.assertEquals(0, (int) iter.next());
// verify idempotence of hasNext()
- assertTrue(iter.hasNext());
- assertTrue(iter.hasNext());
- assertTrue(iter.hasNext());
- assertEquals(1, (int) iter.next());
+ Assert.assertTrue(iter.hasNext());
+ Assert.assertTrue(iter.hasNext());
+ Assert.assertTrue(iter.hasNext());
+ Assert.assertEquals(1, (int) iter.next());
- assertFalse(iter.hasNext());
+ Assert.assertFalse(iter.hasNext());
// Make sure computeNext() doesn't get invoked again
- assertFalse(iter.hasNext());
+ Assert.assertFalse(iter.hasNext());
try
{
iter.next();
- fail("no exception thrown");
+ Assert.fail("no exception thrown");
}
catch (NoSuchElementException expected)
{
}
}
+ @Test
public void testDefaultBehaviorOfPeek()
{
/*
@@ -105,25 +107,25 @@
case 2:
return endOfData();
default:
- fail("Should not have been invoked again");
+ Assert.fail("Should not have been invoked again");
return null;
}
}
};
- assertEquals(0, (int) iter.peek());
- assertEquals(0, (int) iter.peek());
- assertTrue(iter.hasNext());
- assertEquals(0, (int) iter.peek());
- assertEquals(0, (int) iter.next());
+ Assert.assertEquals(0, (int) iter.peek());
+ Assert.assertEquals(0, (int) iter.peek());
+ Assert.assertTrue(iter.hasNext());
+ Assert.assertEquals(0, (int) iter.peek());
+ Assert.assertEquals(0, (int) iter.next());
- assertEquals(1, (int) iter.peek());
- assertEquals(1, (int) iter.next());
+ Assert.assertEquals(1, (int) iter.peek());
+ Assert.assertEquals(1, (int) iter.next());
try
{
iter.peek();
- fail("peek() should throw NoSuchElementException at end");
+ Assert.fail("peek() should throw NoSuchElementException at end");
}
catch (NoSuchElementException expected)
{
@@ -132,7 +134,7 @@
try
{
iter.peek();
- fail("peek() should continue to throw NoSuchElementException at end");
+ Assert.fail("peek() should continue to throw NoSuchElementException at end");
}
catch (NoSuchElementException expected)
{
@@ -141,7 +143,7 @@
try
{
iter.next();
- fail("next() should throw NoSuchElementException as usual");
+ Assert.fail("next() should throw NoSuchElementException as usual");
}
catch (NoSuchElementException expected)
{
@@ -150,13 +152,14 @@
try
{
iter.peek();
- fail("peek() should still throw NoSuchElementException after next()");
+ Assert.fail("peek() should still throw NoSuchElementException after next()");
}
catch (NoSuchElementException expected)
{
}
}
+ @Test
public void testFreesNextReference() throws InterruptedException
{
Iterator<Object> itr = new AbstractIterator<Object>()
@@ -175,6 +178,7 @@
}
}
+ @Test
public void testDefaultBehaviorOfPeekForEmptyIteration()
{
@@ -187,7 +191,7 @@
{
if (alreadyCalledEndOfData)
{
- fail("Should not have been invoked again");
+ Assert.fail("Should not have been invoked again");
}
alreadyCalledEndOfData = true;
return endOfData();
@@ -197,7 +201,7 @@
try
{
empty.peek();
- fail("peek() should throw NoSuchElementException at end");
+ Assert.fail("peek() should throw NoSuchElementException at end");
}
catch (NoSuchElementException expected)
{
@@ -206,13 +210,14 @@
try
{
empty.peek();
- fail("peek() should continue to throw NoSuchElementException at end");
+ Assert.fail("peek() should continue to throw NoSuchElementException at end");
}
catch (NoSuchElementException expected)
{
}
}
+ @Test
public void testException()
{
final SomeUncheckedException exception = new SomeUncheckedException();
@@ -229,14 +234,15 @@
try
{
iter.hasNext();
- fail("No exception thrown");
+ Assert.fail("No exception thrown");
}
catch (SomeUncheckedException e)
{
- assertSame(exception, e);
+ Assert.assertSame(exception, e);
}
}
+ @Test
public void testExceptionAfterEndOfData()
{
Iterator<Integer> iter = new AbstractIterator<Integer>()
@@ -251,13 +257,14 @@
try
{
iter.hasNext();
- fail("No exception thrown");
+ Assert.fail("No exception thrown");
}
catch (SomeUncheckedException expected)
{
}
}
+ @Test
public void testCantRemove()
{
Iterator<Integer> iter = new AbstractIterator<Integer>()
@@ -276,18 +283,19 @@
}
};
- assertEquals(0, (int) iter.next());
+ Assert.assertEquals(0, (int) iter.next());
try
{
iter.remove();
- fail("No exception thrown");
+ Assert.fail("No exception thrown");
}
catch (UnsupportedOperationException expected)
{
}
}
+ @Test
public void testSneakyThrow() throws Exception
{
Iterator<Integer> iter = new AbstractIterator<Integer>()
@@ -299,7 +307,7 @@
{
if (haveBeenCalled)
{
- fail("Should not have been called again");
+ Assert.fail("Should not have been called again");
}
else
{
@@ -314,7 +322,7 @@
try
{
iter.hasNext();
- fail("No exception thrown");
+ Assert.fail("No exception thrown");
}
catch (Exception e)
{
@@ -328,13 +336,14 @@
try
{
iter.hasNext();
- fail("No exception thrown");
+ Assert.fail("No exception thrown");
}
catch (IllegalStateException expected)
{
}
}
+ @Test
public void testReentrantHasNext()
{
Iterator<Integer> iter = new AbstractIterator<Integer>()
@@ -349,7 +358,7 @@
try
{
iter.hasNext();
- fail();
+ Assert.fail();
}
catch (IllegalStateException expected)
{
diff --git a/test/unit/org/apache/cassandra/utils/AssertUtil.java b/test/unit/org/apache/cassandra/utils/AssertUtil.java
new file mode 100644
index 0000000..4d35ede
--- /dev/null
+++ b/test/unit/org/apache/cassandra/utils/AssertUtil.java
@@ -0,0 +1,128 @@
+package org.apache.cassandra.utils;
+
+import java.time.Duration;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import com.google.common.base.Joiner;
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.Multimap;
+
+import org.apache.cassandra.concurrent.NamedThreadFactory;
+
+public final class AssertUtil
+{
+ private AssertUtil()
+ {
+
+ }
+
+ /**
+ * Launch the input in another thread, throws a assert failure if it takes longer than the defined timeout.
+ *
+ * An attempt to halt the thread uses an interrupt, but only works if the underline logic respects it.
+ *
+ * The assert message will contain the stacktrace at the time of the timeout; grouped by common threads.
+ */
+ public static void assertTimeoutPreemptively(Duration timeout, Executable fn)
+ {
+ StackTraceElement caller = Thread.currentThread().getStackTrace()[2];
+ assertTimeoutPreemptively(caller, timeout, () -> {
+ fn.execute();
+ return null;
+ });
+ }
+
+ /**
+ * Launch the input in another thread, throws a assert failure if it takes longer than the defined timeout.
+ *
+ * An attempt to halt the thread uses an interrupt, but only works if the underline logic respects it.
+ *
+ * The assert message will contain the stacktrace at the time of the timeout; grouped by common threads.
+ */
+ public static <T> T assertTimeoutPreemptively(Duration timeout, ThrowingSupplier<T> supplier)
+ {
+ StackTraceElement caller = Thread.currentThread().getStackTrace()[2];
+ return assertTimeoutPreemptively(caller, timeout, supplier);
+ }
+
+ private static <T> T assertTimeoutPreemptively(StackTraceElement caller, Duration timeout, ThrowingSupplier<T> supplier)
+ {
+
+ String[] split = caller.getClassName().split("\\.");
+ String simpleClassName = split[split.length - 1];
+ ExecutorService executorService = Executors.newSingleThreadExecutor(new NamedThreadFactory("TimeoutTest-" + simpleClassName + "#" + caller.getMethodName()));
+ try
+ {
+ Future<T> future = executorService.submit(() -> {
+ try {
+ return supplier.get();
+ }
+ catch (Throwable throwable) {
+ throw Throwables.throwAsUncheckedException(throwable);
+ }
+ });
+
+ long timeoutInNanos = timeout.toNanos();
+ try
+ {
+ return future.get(timeoutInNanos, TimeUnit.NANOSECONDS);
+ }
+ catch (TimeoutException ex)
+ {
+ future.cancel(true);
+ Map<Thread, StackTraceElement[]> threadDump = Thread.getAllStackTraces();
+ StringBuilder sb = new StringBuilder("execution timed out after ").append(TimeUnit.NANOSECONDS.toMillis(timeoutInNanos)).append(" ms\n");
+ Multimap<List<StackTraceElement>, Thread> groupCommonThreads = HashMultimap.create();
+ for (Map.Entry<Thread, StackTraceElement[]> e : threadDump.entrySet())
+ groupCommonThreads.put(Arrays.asList(e.getValue()), e.getKey());
+
+ for (Map.Entry<List<StackTraceElement>, Collection<Thread>> e : groupCommonThreads.asMap().entrySet())
+ {
+ sb.append("Threads: ");
+ Joiner.on(", ").appendTo(sb, e.getValue().stream().map(Thread::getName).iterator());
+ sb.append("\n");
+ for (StackTraceElement elem : e.getKey())
+ sb.append("\t").append(elem.getClassName()).append(".").append(elem.getMethodName()).append("[").append(elem.getLineNumber()).append("]\n");
+ sb.append("\n");
+ }
+ throw new AssertionError(sb.toString());
+ }
+ catch (InterruptedException e)
+ {
+ Thread.currentThread().interrupt();
+ throw Throwables.throwAsUncheckedException(e);
+ }
+ catch (ExecutionException ex)
+ {
+ throw Throwables.throwAsUncheckedException(ex.getCause());
+ }
+ catch (Throwable ex)
+ {
+ throw Throwables.throwAsUncheckedException(ex);
+ }
+ }
+ finally
+ {
+ executorService.shutdownNow();
+ }
+ }
+
+ public interface ThrowingSupplier<T>
+ {
+ T get() throws Throwable;
+ }
+
+ public interface Executable
+ {
+ void execute() throws Throwable;
+ }
+}
diff --git a/test/unit/org/apache/cassandra/utils/BTreeTest.java b/test/unit/org/apache/cassandra/utils/BTreeTest.java
deleted file mode 100644
index 9a59e3a..0000000
--- a/test/unit/org/apache/cassandra/utils/BTreeTest.java
+++ /dev/null
@@ -1,503 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.utils;
-
-import java.util.*;
-import java.util.concurrent.ThreadLocalRandom;
-
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Lists;
-import org.junit.Test;
-
-import junit.framework.Assert;
-import org.apache.cassandra.utils.btree.BTree;
-import org.apache.cassandra.utils.btree.UpdateFunction;
-
-import static org.junit.Assert.*;
-
-public class BTreeTest
-{
- static Integer[] ints = new Integer[20];
- static
- {
- System.setProperty("cassandra.btree.fanfactor", "4");
- for (int i = 0 ; i < ints.length ; i++)
- ints[i] = new Integer(i);
- }
-
- static final UpdateFunction<Integer, Integer> updateF = new UpdateFunction<Integer, Integer>()
- {
- public Integer apply(Integer replacing, Integer update)
- {
- return ints[update];
- }
-
- public boolean abortEarly()
- {
- return false;
- }
-
- public void allocated(long heapSize)
- {
-
- }
-
- public Integer apply(Integer integer)
- {
- return ints[integer];
- }
- };
-
- private static final UpdateFunction<Integer, Integer> noOp = new UpdateFunction<Integer, Integer>()
- {
- public Integer apply(Integer replacing, Integer update)
- {
- return update;
- }
-
- public boolean abortEarly()
- {
- return false;
- }
-
- public void allocated(long heapSize)
- {
- }
-
- public Integer apply(Integer k)
- {
- return k;
- }
- };
-
- private static List<Integer> seq(int count)
- {
- List<Integer> r = new ArrayList<>();
- for (int i = 0 ; i < count ; i++)
- r.add(i);
- return r;
- }
-
- private static List<Integer> rand(int count)
- {
- Random rand = ThreadLocalRandom.current();
- List<Integer> r = seq(count);
- for (int i = 0 ; i < count - 1 ; i++)
- {
- int swap = i + rand.nextInt(count - i);
- Integer tmp = r.get(i);
- r.set(i, r.get(swap));
- r.set(swap, tmp);
- }
- return r;
- }
-
- private static final Comparator<Integer> CMP = new Comparator<Integer>()
- {
- public int compare(Integer o1, Integer o2)
- {
- return Integer.compare(o1, o2);
- }
- };
-
- @Test
- public void testBuilding_UpdateFunctionReplacement()
- {
- for (int i = 0; i < 20 ; i++)
- checkResult(i, BTree.build(seq(i), updateF));
- }
-
- @Test
- public void testUpdate_UpdateFunctionReplacement()
- {
- for (int i = 0; i < 20 ; i++)
- checkResult(i, BTree.update(BTree.build(seq(i), noOp), CMP, seq(i), updateF));
- }
-
- @Test
- public void testApplyForwards()
- {
- List<Integer> input = seq(71);
- Object[] btree = BTree.build(input, noOp);
-
- final List<Integer> result = new ArrayList<>();
- BTree.<Integer>apply(btree, i -> result.add(i), false);
-
- org.junit.Assert.assertArrayEquals(input.toArray(),result.toArray());
- }
-
- @Test
- public void testApplyReverse()
- {
- List<Integer> input = seq(71);
- Object[] btree = BTree.build(input, noOp);
-
- final List<Integer> result = new ArrayList<>();
- BTree.<Integer>apply(btree, i -> result.add(i), true);
-
- org.junit.Assert.assertArrayEquals(Lists.reverse(input).toArray(),result.toArray());
- }
-
- /**
- * Tests that the apply method of the <code>UpdateFunction</code> is only called once with each key update.
- * (see CASSANDRA-8018).
- */
- @Test
- public void testUpdate_UpdateFunctionCallBack()
- {
- Object[] btree = new Object[1];
- CallsMonitor monitor = new CallsMonitor();
-
- btree = BTree.update(btree, CMP, Arrays.asList(1), monitor);
- assertArrayEquals(new Object[] {1}, btree);
- assertEquals(1, monitor.getNumberOfCalls(1));
-
- monitor.clear();
- btree = BTree.update(btree, CMP, Arrays.asList(2), monitor);
- assertArrayEquals(new Object[] {1, 2, null}, btree);
- assertEquals(1, monitor.getNumberOfCalls(2));
-
- // with existing value
- monitor.clear();
- btree = BTree.update(btree, CMP, Arrays.asList(1), monitor);
- assertArrayEquals(new Object[] {1, 2, null}, btree);
- assertEquals(1, monitor.getNumberOfCalls(1));
-
- // with two non-existing values
- monitor.clear();
- btree = BTree.update(btree, CMP, Arrays.asList(3, 4), monitor);
- assertArrayEquals(new Object[] {1, 2, 3, 4, null}, btree);
- assertEquals(1, monitor.getNumberOfCalls(3));
- assertEquals(1, monitor.getNumberOfCalls(4));
-
- // with one existing value and one non existing value
- monitor.clear();
- btree = BTree.update(btree, CMP, Arrays.asList(2, 5), monitor);
- assertArrayEquals(new Object[] {3, new Object[]{1, 2, null}, new Object[]{4, 5, null}, new int[]{2, 5}}, btree);
- assertEquals(1, monitor.getNumberOfCalls(2));
- assertEquals(1, monitor.getNumberOfCalls(5));
- }
-
- /**
- * Tests that the apply method of the <code>UpdateFunction</code> is only called once per value with each build call.
- */
- @Test
- public void testBuilding_UpdateFunctionCallBack()
- {
- CallsMonitor monitor = new CallsMonitor();
- Object[] btree = BTree.build(Arrays.asList(1), monitor);
- assertArrayEquals(new Object[] {1}, btree);
- assertEquals(1, monitor.getNumberOfCalls(1));
-
- monitor.clear();
- btree = BTree.build(Arrays.asList(1, 2), monitor);
- assertArrayEquals(new Object[] {1, 2, null}, btree);
- assertEquals(1, monitor.getNumberOfCalls(1));
- assertEquals(1, monitor.getNumberOfCalls(2));
-
- monitor.clear();
- btree = BTree.build(Arrays.asList(1, 2, 3), monitor);
- assertArrayEquals(new Object[] {1, 2, 3}, btree);
- assertEquals(1, monitor.getNumberOfCalls(1));
- assertEquals(1, monitor.getNumberOfCalls(2));
- assertEquals(1, monitor.getNumberOfCalls(3));
- }
-
- /**
- * Tests that the apply method of the <code>QuickResolver</code> is called exactly once per duplicate value
- */
- @Test
- public void testBuilder_QuickResolver()
- {
- // for numbers x in 1..N, we repeat x x times, and resolve values to their sum,
- // so that the resulting tree is of square numbers
- BTree.Builder.QuickResolver<Accumulator> resolver = (a, b) -> new Accumulator(a.base, a.sum + b.sum);
-
- for (int count = 0 ; count < 10 ; count ++)
- {
- BTree.Builder<Accumulator> builder;
- // first check we produce the right output for sorted input
- List<Accumulator> sorted = resolverInput(count, false);
- builder = BTree.builder(Comparator.naturalOrder());
- builder.setQuickResolver(resolver);
- for (Accumulator i : sorted)
- builder.add(i);
- // for sorted input, check non-resolve path works before checking resolution path
- checkResolverOutput(count, builder.build(), BTree.Dir.ASC);
- builder = BTree.builder(Comparator.naturalOrder());
- builder.setQuickResolver(resolver);
- for (int i = 0 ; i < 10 ; i++)
- {
- // now do a few runs of randomized inputs
- for (Accumulator j : resolverInput(count, true))
- builder.add(j);
- checkResolverOutput(count, builder.build(), BTree.Dir.ASC);
- builder = BTree.builder(Comparator.naturalOrder());
- builder.setQuickResolver(resolver);
- }
- for (List<Accumulator> add : splitResolverInput(count))
- {
- if (ThreadLocalRandom.current().nextBoolean())
- builder.addAll(add);
- else
- builder.addAll(new TreeSet<>(add));
- }
- checkResolverOutput(count, builder.build(), BTree.Dir.ASC);
- }
- }
-
- @Test
- public void testBuilderReuse()
- {
- List<Integer> sorted = seq(20);
- BTree.Builder<Integer> builder = BTree.builder(Comparator.naturalOrder());
- builder.auto(false);
- for (int i : sorted)
- builder.add(i);
- checkResult(20, builder.build());
-
- builder.reuse();
- assertTrue(builder.build() == BTree.empty());
- for (int i = 0; i < 12; i++)
- builder.add(sorted.get(i));
- checkResult(12, builder.build());
-
- builder.auto(true);
- builder.reuse(Comparator.reverseOrder());
- for (int i = 0; i < 12; i++)
- builder.add(sorted.get(i));
- checkResult(12, builder.build(), BTree.Dir.DESC);
-
- builder.reuse();
- assertTrue(builder.build() == BTree.empty());
- }
-
- private static class Accumulator extends Number implements Comparable<Accumulator>
- {
- final int base;
- final int sum;
- private Accumulator(int base, int sum)
- {
- this.base = base;
- this.sum = sum;
- }
-
- public int compareTo(Accumulator that) { return Integer.compare(base, that.base); }
- public int intValue() { return sum; }
- public long longValue() { return sum; }
- public float floatValue() { return sum; }
- public double doubleValue() { return sum; }
- }
-
- /**
- * Tests that the apply method of the <code>Resolver</code> is called exactly once per unique value
- */
- @Test
- public void testBuilder_ResolverAndReverse()
- {
- // for numbers x in 1..N, we repeat x x times, and resolve values to their sum,
- // so that the resulting tree is of square numbers
- BTree.Builder.Resolver resolver = (array, lb, ub) -> {
- int sum = 0;
- for (int i = lb ; i < ub ; i++)
- sum += ((Accumulator) array[i]).sum;
- return new Accumulator(((Accumulator) array[lb]).base, sum);
- };
-
- for (int count = 0 ; count < 10 ; count ++)
- {
- BTree.Builder<Accumulator> builder;
- // first check we produce the right output for sorted input
- List<Accumulator> sorted = resolverInput(count, false);
- builder = BTree.builder(Comparator.naturalOrder());
- builder.auto(false);
- for (Accumulator i : sorted)
- builder.add(i);
- // for sorted input, check non-resolve path works before checking resolution path
- Assert.assertTrue(Iterables.elementsEqual(sorted, BTree.iterable(builder.build())));
-
- builder = BTree.builder(Comparator.naturalOrder());
- builder.auto(false);
- for (Accumulator i : sorted)
- builder.add(i);
- // check resolution path
- checkResolverOutput(count, builder.resolve(resolver).build(), BTree.Dir.ASC);
-
- builder = BTree.builder(Comparator.naturalOrder());
- builder.auto(false);
- for (int i = 0 ; i < 10 ; i++)
- {
- // now do a few runs of randomized inputs
- for (Accumulator j : resolverInput(count, true))
- builder.add(j);
- checkResolverOutput(count, builder.sort().resolve(resolver).build(), BTree.Dir.ASC);
- builder = BTree.builder(Comparator.naturalOrder());
- builder.auto(false);
- for (Accumulator j : resolverInput(count, true))
- builder.add(j);
- checkResolverOutput(count, builder.sort().reverse().resolve(resolver).build(), BTree.Dir.DESC);
- builder = BTree.builder(Comparator.naturalOrder());
- builder.auto(false);
- }
- }
- }
-
- private static List<Accumulator> resolverInput(int count, boolean shuffled)
- {
- List<Accumulator> result = new ArrayList<>();
- for (int i = 1 ; i <= count ; i++)
- for (int j = 0 ; j < i ; j++)
- result.add(new Accumulator(i, i));
- if (shuffled)
- {
- ThreadLocalRandom random = ThreadLocalRandom.current();
- for (int i = 0 ; i < result.size() ; i++)
- {
- int swapWith = random.nextInt(i, result.size());
- Accumulator t = result.get(swapWith);
- result.set(swapWith, result.get(i));
- result.set(i, t);
- }
- }
- return result;
- }
-
- private static List<List<Accumulator>> splitResolverInput(int count)
- {
- List<Accumulator> all = resolverInput(count, false);
- List<List<Accumulator>> result = new ArrayList<>();
- while (!all.isEmpty())
- {
- List<Accumulator> is = new ArrayList<>();
- int prev = -1;
- for (Accumulator i : new ArrayList<>(all))
- {
- if (i.base == prev)
- continue;
- is.add(i);
- all.remove(i);
- prev = i.base;
- }
- result.add(is);
- }
- return result;
- }
-
- private static void checkResolverOutput(int count, Object[] btree, BTree.Dir dir)
- {
- int i = 1;
- for (Accumulator current : BTree.<Accumulator>iterable(btree, dir))
- {
- Assert.assertEquals(i * i, current.sum);
- i++;
- }
- Assert.assertEquals(i, count + 1);
- }
-
- private static void checkResult(int count, Object[] btree)
- {
- checkResult(count, btree, BTree.Dir.ASC);
- }
-
- private static void checkResult(int count, Object[] btree, BTree.Dir dir)
- {
- Iterator<Integer> iter = BTree.slice(btree, CMP, dir);
- int i = 0;
- while (iter.hasNext())
- assertEquals(iter.next(), ints[i++]);
- assertEquals(count, i);
- }
-
- @Test
- public void testClearOnAbort()
- {
- Object[] btree = BTree.build(seq(2), noOp);
- Object[] copy = Arrays.copyOf(btree, btree.length);
- BTree.update(btree, CMP, seq(94), new AbortAfterX(90));
-
- assertArrayEquals(copy, btree);
-
- btree = BTree.update(btree, CMP, seq(94), noOp);
- assertTrue(BTree.isWellFormed(btree, CMP));
- }
-
- private static final class AbortAfterX implements UpdateFunction<Integer, Integer>
- {
- int counter;
- final int abortAfter;
- private AbortAfterX(int abortAfter)
- {
- this.abortAfter = abortAfter;
- }
- public Integer apply(Integer replacing, Integer update)
- {
- return update;
- }
- public boolean abortEarly()
- {
- return counter++ > abortAfter;
- }
- public void allocated(long heapSize)
- {
- }
- public Integer apply(Integer v)
- {
- return v;
- }
- }
-
- /**
- * <code>UpdateFunction</code> that count the number of call made to apply for each value.
- */
- public static final class CallsMonitor implements UpdateFunction<Integer, Integer>
- {
- private int[] numberOfCalls = new int[20];
-
- public Integer apply(Integer replacing, Integer update)
- {
- numberOfCalls[update] = numberOfCalls[update] + 1;
- return update;
- }
-
- public boolean abortEarly()
- {
- return false;
- }
-
- public void allocated(long heapSize)
- {
-
- }
-
- public Integer apply(Integer integer)
- {
- numberOfCalls[integer] = numberOfCalls[integer] + 1;
- return integer;
- }
-
- public int getNumberOfCalls(Integer key)
- {
- return numberOfCalls[key];
- }
-
- public void clear()
- {
- Arrays.fill(numberOfCalls, 0);
- }
- };
-}
diff --git a/test/unit/org/apache/cassandra/utils/BitSetTest.java b/test/unit/org/apache/cassandra/utils/BitSetTest.java
deleted file mode 100644
index 0f51531..0000000
--- a/test/unit/org/apache/cassandra/utils/BitSetTest.java
+++ /dev/null
@@ -1,138 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.utils;
-
-import java.io.ByteArrayInputStream;
-import java.io.DataInputStream;
-import java.io.IOException;
-import java.util.List;
-import java.util.Random;
-
-import com.google.common.collect.Lists;
-import org.junit.Assert;
-import org.junit.Test;
-
-import org.apache.cassandra.io.util.DataOutputBuffer;
-import org.apache.cassandra.utils.IFilter.FilterKey;
-import org.apache.cassandra.utils.KeyGenerator.RandomStringGenerator;
-import org.apache.cassandra.utils.obs.IBitSet;
-import org.apache.cassandra.utils.obs.OffHeapBitSet;
-import org.apache.cassandra.utils.obs.OpenBitSet;
-
-import static org.junit.Assert.assertEquals;
-
-public class BitSetTest
-{
- /**
- * Test bitsets in a "real-world" environment, i.e., bloom filters
- */
- @Test
- public void compareBitSets()
- {
- compareBitSets(false);
- compareBitSets(true);
- }
- private static void compareBitSets(boolean oldBfHashOrder)
- {
- BloomFilter bf2 = (BloomFilter) FilterFactory.getFilter(FilterTestHelper.ELEMENTS / 2, FilterTestHelper.MAX_FAILURE_RATE, false, oldBfHashOrder);
- BloomFilter bf3 = (BloomFilter) FilterFactory.getFilter(FilterTestHelper.ELEMENTS / 2, FilterTestHelper.MAX_FAILURE_RATE, true, oldBfHashOrder);
-
- RandomStringGenerator gen1 = new KeyGenerator.RandomStringGenerator(new Random().nextInt(), FilterTestHelper.ELEMENTS);
-
- // make sure both bitsets are empty.
- compare(bf2.bitset, bf3.bitset);
-
- while (gen1.hasNext())
- {
- FilterKey key = FilterTestHelper.wrap(gen1.next());
- bf2.add(key);
- bf3.add(key);
- }
-
- compare(bf2.bitset, bf3.bitset);
- }
-
- private static final Random random = new Random();
-
- /**
- * Test serialization and de-serialization in-memory
- */
- @Test
- public void testOffHeapSerialization() throws IOException
- {
- try (OffHeapBitSet bs = new OffHeapBitSet(100000))
- {
- populateAndReserialize(bs);
- }
- }
-
- @Test
- public void testOffHeapCompatibility() throws IOException
- {
- try (OpenBitSet bs = new OpenBitSet(100000))
- {
- populateAndReserialize(bs);
- }
- }
-
- private static void populateAndReserialize(IBitSet bs) throws IOException
- {
- for (long i = 0; i < bs.capacity(); i++)
- if (random.nextBoolean())
- bs.set(i);
-
- DataOutputBuffer out = new DataOutputBuffer();
- bs.serialize(out);
- DataInputStream in = new DataInputStream(new ByteArrayInputStream(out.getData()));
- try (OffHeapBitSet newbs = OffHeapBitSet.deserialize(in))
- {
- compare(bs, newbs);
- }
- }
-
- static void compare(IBitSet bs, IBitSet newbs)
- {
- assertEquals(bs.capacity(), newbs.capacity());
- for (long i = 0; i < bs.capacity(); i++)
- Assert.assertEquals(bs.get(i), newbs.get(i));
- }
-
- @Test
- public void testBitClear()
- {
- int size = Integer.MAX_VALUE / 4000;
- try (OffHeapBitSet bitset = new OffHeapBitSet(size))
- {
- List<Integer> randomBits = Lists.newArrayList();
- for (int i = 0; i < 10; i++)
- randomBits.add(random.nextInt(size));
-
- for (long randomBit : randomBits)
- bitset.set(randomBit);
-
- for (long randomBit : randomBits)
- Assert.assertEquals(true, bitset.get(randomBit));
-
- for (long randomBit : randomBits)
- bitset.clear(randomBit);
-
- for (long randomBit : randomBits)
- Assert.assertEquals(false, bitset.get(randomBit));
- }
- }
-}
diff --git a/test/unit/org/apache/cassandra/utils/BloomFilterTest.java b/test/unit/org/apache/cassandra/utils/BloomFilterTest.java
index 818af9c..1c3afff 100644
--- a/test/unit/org/apache/cassandra/utils/BloomFilterTest.java
+++ b/test/unit/org/apache/cassandra/utils/BloomFilterTest.java
@@ -35,43 +35,53 @@
import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.utils.IFilter.FilterKey;
import org.apache.cassandra.utils.KeyGenerator.RandomStringGenerator;
+import org.apache.cassandra.utils.obs.IBitSet;
+
+import static org.junit.Assert.assertEquals;
public class BloomFilterTest
{
- public IFilter bfOldFormat;
public IFilter bfInvHashes;
- public BloomFilterTest()
- {
- }
- public static IFilter testSerialize(IFilter f, boolean oldBfHashOrder) throws IOException
+ public static IFilter testSerialize(IFilter f, boolean oldBfFormat) throws IOException
{
f.add(FilterTestHelper.bytes("a"));
DataOutputBuffer out = new DataOutputBuffer();
- FilterFactory.serialize(f, out);
+ if (oldBfFormat)
+ {
+ SerializationsTest.serializeOldBfFormat((BloomFilter) f, out);
+ }
+ else
+ {
+ BloomFilterSerializer.serialize((BloomFilter) f, out);
+ }
ByteArrayInputStream in = new ByteArrayInputStream(out.getData(), 0, out.getLength());
- IFilter f2 = FilterFactory.deserialize(new DataInputStream(in), true, oldBfHashOrder);
+ IFilter f2 = BloomFilterSerializer.deserialize(new DataInputStream(in), oldBfFormat);
assert f2.isPresent(FilterTestHelper.bytes("a"));
assert !f2.isPresent(FilterTestHelper.bytes("b"));
return f2;
}
+ static void compare(IBitSet bs, IBitSet newbs)
+ {
+ assertEquals(bs.capacity(), newbs.capacity());
+ for (long i = 0; i < bs.capacity(); i++)
+ assertEquals(bs.get(i), newbs.get(i));
+ }
@Before
public void setup()
{
- bfOldFormat = FilterFactory.getFilter(10000L, FilterTestHelper.MAX_FAILURE_RATE, true, true);
- bfInvHashes = FilterFactory.getFilter(10000L, FilterTestHelper.MAX_FAILURE_RATE, true, false);
+ bfInvHashes = FilterFactory.getFilter(10000L, FilterTestHelper.MAX_FAILURE_RATE);
}
@After
public void destroy()
{
- bfOldFormat.close();
bfInvHashes.close();
}
@@ -91,10 +101,6 @@
@Test
public void testOne()
{
- bfOldFormat.add(FilterTestHelper.bytes("a"));
- assert bfOldFormat.isPresent(FilterTestHelper.bytes("a"));
- assert !bfOldFormat.isPresent(FilterTestHelper.bytes("b"));
-
bfInvHashes.add(FilterTestHelper.bytes("a"));
assert bfInvHashes.isPresent(FilterTestHelper.bytes("a"));
assert !bfInvHashes.isPresent(FilterTestHelper.bytes("b"));
@@ -103,16 +109,12 @@
@Test
public void testFalsePositivesInt()
{
- FilterTestHelper.testFalsePositives(bfOldFormat, FilterTestHelper.intKeys(), FilterTestHelper.randomKeys2());
-
FilterTestHelper.testFalsePositives(bfInvHashes, FilterTestHelper.intKeys(), FilterTestHelper.randomKeys2());
}
@Test
public void testFalsePositivesRandom()
{
- FilterTestHelper.testFalsePositives(bfOldFormat, FilterTestHelper.randomKeys(), FilterTestHelper.randomKeys2());
-
FilterTestHelper.testFalsePositives(bfInvHashes, FilterTestHelper.randomKeys(), FilterTestHelper.randomKeys2());
}
@@ -123,26 +125,18 @@
{
return;
}
- IFilter bf2 = FilterFactory.getFilter(KeyGenerator.WordGenerator.WORDS / 2, FilterTestHelper.MAX_FAILURE_RATE, true, false);
+ IFilter bf2 = FilterFactory.getFilter(KeyGenerator.WordGenerator.WORDS / 2, FilterTestHelper.MAX_FAILURE_RATE);
int skipEven = KeyGenerator.WordGenerator.WORDS % 2 == 0 ? 0 : 2;
FilterTestHelper.testFalsePositives(bf2,
new KeyGenerator.WordGenerator(skipEven, 2),
new KeyGenerator.WordGenerator(1, 2));
bf2.close();
-
- // new, swapped hash values bloom filter
- bf2 = FilterFactory.getFilter(KeyGenerator.WordGenerator.WORDS / 2, FilterTestHelper.MAX_FAILURE_RATE, true, true);
- FilterTestHelper.testFalsePositives(bf2,
- new KeyGenerator.WordGenerator(skipEven, 2),
- new KeyGenerator.WordGenerator(1, 2));
- bf2.close();
}
@Test
public void testSerialize() throws IOException
{
- BloomFilterTest.testSerialize(bfOldFormat, true).close();
-
+ BloomFilterTest.testSerialize(bfInvHashes, true).close();
BloomFilterTest.testSerialize(bfInvHashes, false).close();
}
@@ -150,12 +144,10 @@
@Ignore
public void testManyRandom()
{
- testManyRandom(FilterTestHelper.randomKeys(), false);
-
- testManyRandom(FilterTestHelper.randomKeys(), true);
+ testManyRandom(FilterTestHelper.randomKeys());
}
- private static void testManyRandom(Iterator<ByteBuffer> keys, boolean oldBfHashOrder)
+ private static void testManyRandom(Iterator<ByteBuffer> keys)
{
int MAX_HASH_COUNT = 128;
Set<Long> hashes = new HashSet<>();
@@ -164,7 +156,7 @@
{
hashes.clear();
FilterKey buf = FilterTestHelper.wrap(keys.next());
- BloomFilter bf = (BloomFilter) FilterFactory.getFilter(10, 1, false, oldBfHashOrder);
+ BloomFilter bf = (BloomFilter) FilterFactory.getFilter(10, 1);
for (long hashIndex : bf.getHashBuckets(buf, MAX_HASH_COUNT, 1024 * 1024))
{
hashes.add(hashIndex);
@@ -179,47 +171,21 @@
public void testOffHeapException()
{
long numKeys = ((long)Integer.MAX_VALUE) * 64L + 1L; // approx 128 Billion
- FilterFactory.getFilter(numKeys, 0.01d, true, true).close();
+ FilterFactory.getFilter(numKeys, 0.01d).close();
}
@Test
- public void compareCachedKeyOldHashOrder()
+ public void compareCachedKey()
{
- BloomFilter bf1 = (BloomFilter) FilterFactory.getFilter(FilterTestHelper.ELEMENTS / 2, FilterTestHelper.MAX_FAILURE_RATE, false, true);
- BloomFilter bf2 = (BloomFilter) FilterFactory.getFilter(FilterTestHelper.ELEMENTS / 2, FilterTestHelper.MAX_FAILURE_RATE, false, true);
- BloomFilter bf3 = (BloomFilter) FilterFactory.getFilter(FilterTestHelper.ELEMENTS / 2, FilterTestHelper.MAX_FAILURE_RATE, false, true);
-
- RandomStringGenerator gen1 = new KeyGenerator.RandomStringGenerator(new Random().nextInt(), FilterTestHelper.ELEMENTS);
-
- // make sure all bitsets are empty.
- BitSetTest.compare(bf1.bitset, bf2.bitset);
- BitSetTest.compare(bf1.bitset, bf3.bitset);
-
- while (gen1.hasNext())
- {
- ByteBuffer key = gen1.next();
- FilterKey cached = FilterTestHelper.wrapCached(key);
- bf1.add(FilterTestHelper.wrap(key));
- bf2.add(cached);
- bf3.add(cached);
- }
-
- BitSetTest.compare(bf1.bitset, bf2.bitset);
- BitSetTest.compare(bf1.bitset, bf3.bitset);
- }
-
- @Test
- public void compareCachedKeyNewHashOrder()
- {
- try (BloomFilter bf1 = (BloomFilter) FilterFactory.getFilter(FilterTestHelper.ELEMENTS / 2, FilterTestHelper.MAX_FAILURE_RATE, false, false);
- BloomFilter bf2 = (BloomFilter) FilterFactory.getFilter(FilterTestHelper.ELEMENTS / 2, FilterTestHelper.MAX_FAILURE_RATE, false, false);
- BloomFilter bf3 = (BloomFilter) FilterFactory.getFilter(FilterTestHelper.ELEMENTS / 2, FilterTestHelper.MAX_FAILURE_RATE, false, false))
+ try (BloomFilter bf1 = (BloomFilter) FilterFactory.getFilter(FilterTestHelper.ELEMENTS / 2, FilterTestHelper.MAX_FAILURE_RATE);
+ BloomFilter bf2 = (BloomFilter) FilterFactory.getFilter(FilterTestHelper.ELEMENTS / 2, FilterTestHelper.MAX_FAILURE_RATE);
+ BloomFilter bf3 = (BloomFilter) FilterFactory.getFilter(FilterTestHelper.ELEMENTS / 2, FilterTestHelper.MAX_FAILURE_RATE))
{
RandomStringGenerator gen1 = new KeyGenerator.RandomStringGenerator(new Random().nextInt(), FilterTestHelper.ELEMENTS);
// make sure all bitsets are empty.
- BitSetTest.compare(bf1.bitset, bf2.bitset);
- BitSetTest.compare(bf1.bitset, bf3.bitset);
+ compare(bf1.bitset, bf2.bitset);
+ compare(bf1.bitset, bf3.bitset);
while (gen1.hasNext())
{
@@ -230,8 +196,8 @@
bf3.add(cached);
}
- BitSetTest.compare(bf1.bitset, bf2.bitset);
- BitSetTest.compare(bf1.bitset, bf3.bitset);
+ compare(bf1.bitset, bf2.bitset);
+ compare(bf1.bitset, bf3.bitset);
}
}
@@ -239,25 +205,18 @@
@Ignore
public void testHugeBFSerialization() throws IOException
{
- hugeBFSerialization(false);
- hugeBFSerialization(true);
- }
-
- static void hugeBFSerialization(boolean oldBfHashOrder) throws IOException
- {
ByteBuffer test = ByteBuffer.wrap(new byte[] {0, 1});
- File file = FileUtils.createTempFile("bloomFilterTest-", ".dat");
- BloomFilter filter = (BloomFilter) FilterFactory.getFilter(((long) Integer.MAX_VALUE / 8) + 1, 0.01d, true, oldBfHashOrder);
+ File file = FileUtils.createDeletableTempFile("bloomFilterTest-", ".dat");
+ BloomFilter filter = (BloomFilter) FilterFactory.getFilter(((long) Integer.MAX_VALUE / 8) + 1, 0.01d);
filter.add(FilterTestHelper.wrap(test));
DataOutputStreamPlus out = new BufferedDataOutputStreamPlus(new FileOutputStream(file));
- FilterFactory.serialize(filter, out);
- filter.bitset.serialize(out);
+ BloomFilterSerializer.serialize(filter, out);
out.close();
filter.close();
DataInputStream in = new DataInputStream(new FileInputStream(file));
- BloomFilter filter2 = (BloomFilter) FilterFactory.deserialize(in, true, oldBfHashOrder);
+ BloomFilter filter2 = BloomFilterSerializer.deserialize(in, false);
Assert.assertTrue(filter2.isPresent(FilterTestHelper.wrap(test)));
FileUtils.closeQuietly(in);
filter2.close();
diff --git a/test/unit/org/apache/cassandra/utils/ByteBufferUtilTest.java b/test/unit/org/apache/cassandra/utils/ByteBufferUtilTest.java
index f2746f6..4ae8626 100644
--- a/test/unit/org/apache/cassandra/utils/ByteBufferUtilTest.java
+++ b/test/unit/org/apache/cassandra/utils/ByteBufferUtilTest.java
@@ -21,6 +21,7 @@
import java.io.ByteArrayInputStream;
import java.io.DataInputStream;
import java.io.IOException;
+import java.nio.BufferOverflowException;
import java.nio.ByteBuffer;
import java.nio.charset.CharacterCodingException;
import java.util.Arrays;
@@ -33,6 +34,8 @@
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.fail;
public class ByteBufferUtilTest
{
@@ -150,11 +153,11 @@
{
byte[] bytes = new byte[s.length()];
- ByteBufferUtil.arrayCopy(bb, bb.position(), bytes, 0, s.length());
+ ByteBufferUtil.copyBytes(bb, bb.position(), bytes, 0, s.length());
assertArrayEquals(s.getBytes(), bytes);
bytes = new byte[5];
- ByteBufferUtil.arrayCopy(bb, bb.position() + 3, bytes, 1, 4);
+ ByteBufferUtil.copyBytes(bb, bb.position() + 3, bytes, 1, 4);
assertArrayEquals(Arrays.copyOfRange(s.getBytes(), 3, 7), Arrays.copyOfRange(bytes, 1, 5));
}
@@ -288,8 +291,8 @@
a.limit(bytes.length - 1).position(0);
b.limit(bytes.length - 1).position(1);
- Assert.assertFalse(ByteBufferUtil.startsWith(a, b));
- Assert.assertFalse(ByteBufferUtil.startsWith(a, b.slice()));
+ assertFalse(ByteBufferUtil.startsWith(a, b));
+ assertFalse(ByteBufferUtil.startsWith(a, b.slice()));
Assert.assertTrue(ByteBufferUtil.endsWith(a, b));
Assert.assertTrue(ByteBufferUtil.endsWith(a, b.slice()));
@@ -297,7 +300,60 @@
a.position(5);
- Assert.assertFalse(ByteBufferUtil.startsWith(a, b));
- Assert.assertFalse(ByteBufferUtil.endsWith(a, b));
+ assertFalse(ByteBufferUtil.startsWith(a, b));
+ assertFalse(ByteBufferUtil.endsWith(a, b));
+ }
+
+ @Test
+ public void testWriteZeroes()
+ {
+ byte[] initial = new byte[1024];
+ Arrays.fill(initial, (byte) 1);
+ for (ByteBuffer b : new ByteBuffer[] { ByteBuffer.allocate(1024), ByteBuffer.allocateDirect(1024) })
+ {
+ for (int i = 0; i <= 32; ++i)
+ for (int j = 1024; j >= 1024 - 32; --j)
+ {
+ b.clear();
+ b.put(initial);
+ b.flip();
+ b.position(i);
+ ByteBufferUtil.writeZeroes(b, j-i);
+ assertEquals(j, b.position());
+ int ii = 0;
+ for (; ii < i; ++ii)
+ assertEquals(initial[ii], b.get(ii));
+ for (; ii < j; ++ii)
+ assertEquals(0, b.get(ii));
+ for (; ii < 1024; ++ii)
+ assertEquals(initial[ii], b.get(ii));
+
+ b.clear();
+ b.put(initial);
+ b.limit(j).position(i);
+ ByteBuffer slice = b.slice();
+ ByteBufferUtil.writeZeroes(slice, slice.capacity());
+ assertFalse(slice.hasRemaining());
+ b.clear(); // reset position and limit for check
+ ii = 0;
+ for (; ii < i; ++ii)
+ assertEquals(initial[ii], b.get(ii));
+ for (; ii < j; ++ii)
+ assertEquals(0, b.get(ii));
+ for (; ii < 1024; ++ii)
+ assertEquals(initial[ii], b.get(ii));
+
+ slice.clear();
+ try
+ {
+ ByteBufferUtil.writeZeroes(slice, slice.capacity() + 1);
+ fail("Line above should throw.");
+ }
+ catch (BufferOverflowException | IndexOutOfBoundsException e)
+ {
+ // correct path
+ }
+ }
+ }
}
}
diff --git a/test/unit/org/apache/cassandra/utils/CassandraVersionTest.java b/test/unit/org/apache/cassandra/utils/CassandraVersionTest.java
index da7e266..683d9e4 100644
--- a/test/unit/org/apache/cassandra/utils/CassandraVersionTest.java
+++ b/test/unit/org/apache/cassandra/utils/CassandraVersionTest.java
@@ -23,7 +23,6 @@
import org.junit.Test;
import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
diff --git a/test/unit/org/apache/cassandra/utils/CoalescingStrategiesTest.java b/test/unit/org/apache/cassandra/utils/CoalescingStrategiesTest.java
deleted file mode 100644
index b10d70b..0000000
--- a/test/unit/org/apache/cassandra/utils/CoalescingStrategiesTest.java
+++ /dev/null
@@ -1,485 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.utils;
-
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.utils.CoalescingStrategies.Clock;
-import org.apache.cassandra.utils.CoalescingStrategies.Coalescable;
-import org.apache.cassandra.utils.CoalescingStrategies.CoalescingStrategy;
-import org.apache.cassandra.utils.CoalescingStrategies.Parker;
-import org.junit.BeforeClass;
-import org.junit.Before;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayDeque;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Queue;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.Semaphore;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.locks.LockSupport;
-
-import static org.junit.Assert.*;
-
-public class CoalescingStrategiesTest
-{
-
- static final ExecutorService ex = Executors.newSingleThreadExecutor();
-
- private static final Logger logger = LoggerFactory.getLogger(CoalescingStrategiesTest.class);
-
- static class MockParker implements Parker
- {
- Queue<Long> parks = new ArrayDeque<Long>();
- Semaphore permits = new Semaphore(0);
-
- Semaphore parked = new Semaphore(0);
-
- public void park(long nanos)
- {
- parks.offer(nanos);
- parked.release();
- try
- {
- permits.acquire();
- }
- catch (InterruptedException e)
- {
- throw new RuntimeException(e);
- }
- }
- }
-
- static class SimpleCoalescable implements Coalescable
- {
- final long timestampNanos;
-
- SimpleCoalescable(long timestampNanos)
- {
- this.timestampNanos = timestampNanos;
- }
-
- public long timestampNanos()
- {
- return timestampNanos;
- }
- }
-
-
- static long toNanos(long micros)
- {
- return TimeUnit.MICROSECONDS.toNanos(micros);
- }
-
- MockParker parker;
-
- BlockingQueue<SimpleCoalescable> input;
- List<SimpleCoalescable> output;
-
- CoalescingStrategy cs;
-
- Semaphore queueParked = new Semaphore(0);
- Semaphore queueRelease = new Semaphore(0);
-
- @BeforeClass
- public static void initDD()
- {
- DatabaseDescriptor.daemonInitialization();
- }
-
- @SuppressWarnings({ "serial" })
- @Before
- public void setUp() throws Exception
- {
- cs = null;
- CoalescingStrategies.CLOCK = new Clock()
- {
- @Override
- public long nanoTime()
- {
- return 0;
- }
- };
-
- parker = new MockParker();
- input = new LinkedBlockingQueue<SimpleCoalescable>()
- {
- @Override
- public SimpleCoalescable take() throws InterruptedException
- {
- queueParked.release();
- queueRelease.acquire();
- return super.take();
- }
- };
- output = new ArrayList<>(128);
-
- clear();
- }
-
- CoalescingStrategy newStrategy(String name, int window)
- {
- return CoalescingStrategies.newCoalescingStrategy(name, window, parker, logger, "Stupendopotamus");
- }
-
- void add(long whenMicros)
- {
- input.offer(new SimpleCoalescable(toNanos(whenMicros)));
- }
-
- void clear()
- {
- output.clear();
- input.clear();
- parker.parks.clear();
- parker.parked.drainPermits();
- parker.permits.drainPermits();
- queueParked.drainPermits();
- queueRelease.drainPermits();
- }
-
- void release() throws Exception
- {
- queueRelease.release();
- parker.permits.release();
- fut.get();
- }
-
- Future<?> fut;
- void runBlocker(Semaphore waitFor) throws Exception
- {
- fut = ex.submit(new Runnable()
- {
- @Override
- public void run()
- {
- try
- {
- cs.coalesce(input, output, 128);
- }
- catch (Exception ex)
- {
- ex.printStackTrace();
- throw new RuntimeException(ex);
- }
- }
- });
- waitFor.acquire();
- }
-
- @Test
- public void testFixedCoalescingStrategy() throws Exception
- {
- cs = newStrategy("FIXED", 200);
-
- //Test that when a stream of messages continues arriving it keeps sending until all are drained
- //It does this because it is already awake and sending messages
- add(42);
- add(42);
- cs.coalesce(input, output, 128);
- assertEquals( 2, output.size());
- assertNull(parker.parks.poll());
-
- clear();
-
- runBlocker(queueParked);
- add(42);
- add(42);
- add(42);
- release();
- assertEquals( 3, output.size());
- assertEquals(toNanos(200), parker.parks.poll().longValue());
-
- }
-
- @Test
- public void testFixedCoalescingStrategyEnough() throws Exception
- {
- int oldValue = DatabaseDescriptor.getOtcCoalescingEnoughCoalescedMessages();
- DatabaseDescriptor.setOtcCoalescingEnoughCoalescedMessages(1);
- try {
- cs = newStrategy("FIXED", 200);
-
- //Test that when a stream of messages continues arriving it keeps sending until all are drained
- //It does this because it is already awake and sending messages
- add(42);
- add(42);
- cs.coalesce(input, output, 128);
- assertEquals(2, output.size());
- assertNull(parker.parks.poll());
-
- clear();
-
- runBlocker(queueParked);
- add(42);
- add(42);
- add(42);
- release();
- assertEquals(3, output.size());
- assertNull(parker.parks.poll());
- }
- finally {
- DatabaseDescriptor.setOtcCoalescingEnoughCoalescedMessages(oldValue);
- }
-
- }
-
- @Test
- public void testDisabledCoalescingStrateg() throws Exception
- {
- cs = newStrategy("DISABLED", 200);
-
- add(42);
- add(42);
- cs.coalesce(input, output, 128);
- assertEquals( 2, output.size());
- assertNull(parker.parks.poll());
-
- clear();
-
- runBlocker(queueParked);
- add(42);
- add(42);
- release();
- assertEquals( 2, output.size());
- assertNull(parker.parks.poll());
- }
-
- @Test
- public void parkLoop() throws Exception
- {
- final Thread current = Thread.currentThread();
- final Semaphore helperReady = new Semaphore(0);
- final Semaphore helperGo = new Semaphore(0);
-
- new Thread()
- {
- @Override
- public void run()
- {
- try
- {
- helperReady.release();
- helperGo.acquire();
- Thread.sleep(50);
- LockSupport.unpark(current);
- }
- catch (Exception e)
- {
- e.printStackTrace();
- logger.error("Error", e);
- System.exit(-1);
- }
- }
- }.start();
-
- long start = System.nanoTime();
- helperGo.release();
-
- long parkNanos = TimeUnit.MILLISECONDS.toNanos(500);
-
- CoalescingStrategies.parkLoop(parkNanos);
- long delta = System.nanoTime() - start;
-
- assertTrue (delta >= (parkNanos - (parkNanos / 16)));
- }
-
- @Test
- public void testMovingAverageCoalescingStrategy() throws Exception
- {
- cs = newStrategy("org.apache.cassandra.utils.CoalescingStrategies$MovingAverageCoalescingStrategy", 200);
-
-
- //Test that things can be pulled out of the queue if it is non-empty
- add(201);
- add(401);
- cs.coalesce(input, output, 128);
- assertEquals( 2, output.size());
- assertNull(parker.parks.poll());
-
- //Test that blocking on the queue results in everything drained
- clear();
-
- runBlocker(queueParked);
- add(601);
- add(801);
- release();
- assertEquals( 2, output.size());
- assertNull(parker.parks.poll());
-
- clear();
-
- //Test that out of order samples still flow
- runBlocker(queueParked);
- add(0);
- release();
- assertEquals( 1, output.size());
- assertNull(parker.parks.poll());
-
- clear();
-
- add(0);
- cs.coalesce(input, output, 128);
- assertEquals( 1, output.size());
- assertNull(parker.parks.poll());
-
- clear();
-
- //Test that too high an average doesn't coalesce
- for (long ii = 0; ii < 128; ii++)
- add(ii * 1000);
- cs.coalesce(input, output, 128);
- assertEquals(output.size(), 128);
- assertTrue(parker.parks.isEmpty());
-
- clear();
-
- runBlocker(queueParked);
- add(129 * 1000);
- release();
- assertTrue(parker.parks.isEmpty());
-
- clear();
-
- //Test that a low enough average coalesces
- cs = newStrategy("MOVINGAVERAGE", 200);
- for (long ii = 0; ii < 128; ii++)
- add(ii * 99);
- cs.coalesce(input, output, 128);
- assertEquals(output.size(), 128);
- assertTrue(parker.parks.isEmpty());
-
- clear();
-
- runBlocker(queueParked);
- add(128 * 99);
- add(129 * 99);
- release();
- assertEquals(2, output.size());
- assertEquals(toNanos(198), parker.parks.poll().longValue());
- }
-
- @Test
- public void testTimeHorizonStrategy() throws Exception
- {
- cs = newStrategy("TIMEHORIZON", 200);
-
- //Test that things can be pulled out of the queue if it is non-empty
- add(201);
- add(401);
- cs.coalesce(input, output, 128);
- assertEquals( 2, output.size());
- assertNull(parker.parks.poll());
-
- //Test that blocking on the queue results in everything drained
- clear();
-
- runBlocker(queueParked);
- add(601);
- add(801);
- release();
- assertEquals( 2, output.size());
- assertNull(parker.parks.poll());
-
- clear();
-
- //Test that out of order samples still flow
- runBlocker(queueParked);
- add(0);
- release();
- assertEquals( 1, output.size());
- assertNull(parker.parks.poll());
-
- clear();
-
- add(0);
- cs.coalesce(input, output, 128);
- assertEquals( 1, output.size());
- assertNull(parker.parks.poll());
-
- clear();
-
- //Test that too high an average doesn't coalesce
- for (long ii = 0; ii < 128; ii++)
- add(ii * 1000);
- cs.coalesce(input, output, 128);
- assertEquals(output.size(), 128);
- assertTrue(parker.parks.isEmpty());
-
- clear();
-
- runBlocker(queueParked);
- add(129 * 1000);
- release();
- assertTrue(parker.parks.isEmpty());
-
- clear();
-
- //Test that a low enough average coalesces
- cs = newStrategy("TIMEHORIZON", 200);
- primeTimeHorizonAverage(99);
-
- clear();
-
- runBlocker(queueParked);
- add(100000 * 99);
- queueRelease.release();
- parker.parked.acquire();
- add(100001 * 99);
- parker.permits.release();
- fut.get();
- assertEquals(2, output.size());
- assertEquals(toNanos(198), parker.parks.poll().longValue());
-
- clear();
-
- //Test far future
- add(Integer.MAX_VALUE);
- cs.coalesce(input, output, 128);
- assertEquals(1, output.size());
- assertTrue(parker.parks.isEmpty());
-
- clear();
-
- //Distant past
- add(0);
- cs.coalesce(input, output, 128);
- assertEquals(1, output.size());
- assertTrue(parker.parks.isEmpty());
- }
-
- void primeTimeHorizonAverage(long micros) throws Exception
- {
- for (long ii = 0; ii < 100000; ii++)
- {
- add(ii * micros);
- if (ii % 128 == 0)
- {
- cs.coalesce(input, output, 128);
- output.clear();
- }
- }
- }
-}
diff --git a/test/unit/org/apache/cassandra/utils/FBUtilitiesTest.java b/test/unit/org/apache/cassandra/utils/FBUtilitiesTest.java
index ecdb03e..833f2e4 100644
--- a/test/unit/org/apache/cassandra/utils/FBUtilitiesTest.java
+++ b/test/unit/org/apache/cassandra/utils/FBUtilitiesTest.java
@@ -161,7 +161,7 @@
}
@Test
- public void testGetBroadcastRpcAddress() throws Exception
+ public void testGetBroadcastNativeAddress() throws Exception
{
//When both rpc_address and broadcast_rpc_address are null, it should return the local address (from DD.applyAddressConfig)
FBUtilities.reset();
@@ -169,21 +169,21 @@
testConfig.rpc_address = null;
testConfig.broadcast_rpc_address = null;
DatabaseDescriptor.applyAddressConfig(testConfig);
- assertEquals(FBUtilities.getLocalAddress(), FBUtilities.getBroadcastRpcAddress());
+ assertEquals(FBUtilities.getJustLocalAddress(), FBUtilities.getJustBroadcastNativeAddress());
//When rpc_address is defined and broadcast_rpc_address is null, it should return the rpc_address
FBUtilities.reset();
testConfig.rpc_address = "127.0.0.2";
testConfig.broadcast_rpc_address = null;
DatabaseDescriptor.applyAddressConfig(testConfig);
- assertEquals(InetAddress.getByName("127.0.0.2"), FBUtilities.getBroadcastRpcAddress());
+ assertEquals(InetAddress.getByName("127.0.0.2"), FBUtilities.getJustBroadcastNativeAddress());
//When both rpc_address and broadcast_rpc_address are defined, it should return broadcast_rpc_address
FBUtilities.reset();
testConfig.rpc_address = "127.0.0.2";
testConfig.broadcast_rpc_address = "127.0.0.3";
DatabaseDescriptor.applyAddressConfig(testConfig);
- assertEquals(InetAddress.getByName("127.0.0.3"), FBUtilities.getBroadcastRpcAddress());
+ assertEquals(InetAddress.getByName("127.0.0.3"), FBUtilities.getJustBroadcastNativeAddress());
FBUtilities.reset();
}
@@ -227,5 +227,4 @@
executor.shutdown();
}
}
-
}
diff --git a/test/unit/org/apache/cassandra/utils/FreeRunningClock.java b/test/unit/org/apache/cassandra/utils/FreeRunningClock.java
index 83c8db7..d853833 100644
--- a/test/unit/org/apache/cassandra/utils/FreeRunningClock.java
+++ b/test/unit/org/apache/cassandra/utils/FreeRunningClock.java
@@ -20,23 +20,41 @@
import java.util.concurrent.TimeUnit;
/**
- * A freely adjustable clock that can be used for unit testing. See {@link Clock#instance} how to
+ * A freely adjustable clock that can be used for unit testing. See {@link MonotonicClock#instance} how to
* enable this class.
*/
-public class FreeRunningClock extends Clock
+public class FreeRunningClock implements MonotonicClock
{
private long nanoTime = 0;
@Override
- public long nanoTime()
+ public long now()
{
return nanoTime;
}
@Override
- public long currentTimeMillis()
+ public long error()
{
- return TimeUnit.NANOSECONDS.toMillis(nanoTime());
+ return 0;
+ }
+
+ @Override
+ public MonotonicClockTranslation translate()
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean isAfter(long instant)
+ {
+ return instant > nanoTime;
+ }
+
+ @Override
+ public boolean isAfter(long now, long instant)
+ {
+ return now > instant;
}
public void advance(long time, TimeUnit unit)
diff --git a/test/unit/org/apache/cassandra/utils/IntegerIntervalsTest.java b/test/unit/org/apache/cassandra/utils/IntegerIntervalsTest.java
index 44843fd..f93484e 100644
--- a/test/unit/org/apache/cassandra/utils/IntegerIntervalsTest.java
+++ b/test/unit/org/apache/cassandra/utils/IntegerIntervalsTest.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.cassandra.utils;
import static org.junit.Assert.*;
diff --git a/test/unit/org/apache/cassandra/utils/MerkleTreeTest.java b/test/unit/org/apache/cassandra/utils/MerkleTreeTest.java
index fb517d7..1cdcc22 100644
--- a/test/unit/org/apache/cassandra/utils/MerkleTreeTest.java
+++ b/test/unit/org/apache/cassandra/utils/MerkleTreeTest.java
@@ -1,52 +1,60 @@
/*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyten ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*/
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.cassandra.utils;
+import java.io.IOException;
import java.math.BigInteger;
+import java.nio.ByteBuffer;
import java.util.*;
-import com.google.common.collect.Lists;
-
+import org.junit.Assert;
import org.junit.Before;
-import org.junit.BeforeClass;
import org.junit.Test;
import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.Digest;
+import org.apache.cassandra.dht.ByteOrderedPartitioner;
import org.apache.cassandra.dht.IPartitioner;
+import org.apache.cassandra.dht.Murmur3Partitioner;
import org.apache.cassandra.dht.RandomPartitioner;
import org.apache.cassandra.dht.RandomPartitioner.BigIntegerToken;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.io.util.DataInputBuffer;
-import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputBuffer;
import org.apache.cassandra.net.MessagingService;
-import org.apache.cassandra.utils.MerkleTree.Hashable;
import org.apache.cassandra.utils.MerkleTree.RowHash;
import org.apache.cassandra.utils.MerkleTree.TreeRange;
import org.apache.cassandra.utils.MerkleTree.TreeRangeIterator;
+import static com.google.common.collect.Lists.newArrayList;
import static org.apache.cassandra.utils.MerkleTree.RECOMMENDED_DEPTH;
import static org.junit.Assert.*;
public class MerkleTreeTest
{
- public static byte[] DUMMY = "blah".getBytes();
+ private static final byte[] DUMMY = digest("dummy");
+
+ static byte[] digest(String string)
+ {
+ return Digest.forValidator()
+ .update(string.getBytes(), 0, string.getBytes().length)
+ .digest();
+ }
/**
* If a test assumes that the tree is 8 units wide, then it should set this value
@@ -62,15 +70,12 @@
return new Range<>(partitioner.getMinimumToken(), partitioner.getMinimumToken());
}
- @BeforeClass
- public static void beforeClass()
- {
- DatabaseDescriptor.daemonInitialization();
- }
-
@Before
- public void clear()
+ public void setup()
{
+ DatabaseDescriptor.clientInitialization();
+ DatabaseDescriptor.useOffheapMerkleTrees(false);
+
TOKEN_SCALE = new BigInteger("8");
partitioner = RandomPartitioner.instance;
// TODO need to trickle TokenSerializer
@@ -99,7 +104,7 @@
{
if (i == -1)
return new BigIntegerToken(new BigInteger("-1"));
- BigInteger bint = RandomPartitioner.MAXIMUM.divide(TOKEN_SCALE).multiply(new BigInteger(""+i));
+ BigInteger bint = RandomPartitioner.MAXIMUM.divide(TOKEN_SCALE).multiply(new BigInteger("" + i));
return new BigIntegerToken(bint);
}
@@ -120,10 +125,10 @@
assertEquals(new Range<>(tok(6), tok(7)), mt.get(tok(7)));
// check depths
- assertEquals((byte)1, mt.get(tok(4)).depth);
- assertEquals((byte)2, mt.get(tok(6)).depth);
- assertEquals((byte)3, mt.get(tok(7)).depth);
- assertEquals((byte)3, mt.get(tok(-1)).depth);
+ assertEquals((byte) 1, mt.get(tok(4)).depth);
+ assertEquals((byte) 2, mt.get(tok(6)).depth);
+ assertEquals((byte) 3, mt.get(tok(7)).depth);
+ assertEquals((byte) 3, mt.get(tok(-1)).depth);
try
{
@@ -139,7 +144,7 @@
@Test
public void testSplitLimitDepth()
{
- mt = new MerkleTree(partitioner, fullRange(), (byte)2, Integer.MAX_VALUE);
+ mt = new MerkleTree(partitioner, fullRange(), (byte) 2, Integer.MAX_VALUE);
assertTrue(mt.split(tok(4)));
assertTrue(mt.split(tok(2)));
@@ -174,7 +179,7 @@
Iterator<TreeRange> ranges;
// (zero, zero]
- ranges = mt.invalids();
+ ranges = mt.rangeIterator();
assertEquals(new Range<>(tok(-1), tok(-1)), ranges.next());
assertFalse(ranges.hasNext());
@@ -184,7 +189,7 @@
mt.split(tok(6));
mt.split(tok(3));
mt.split(tok(5));
- ranges = mt.invalids();
+ ranges = mt.rangeIterator();
assertEquals(new Range<>(tok(6), tok(-1)), ranges.next());
assertEquals(new Range<>(tok(-1), tok(2)), ranges.next());
assertEquals(new Range<>(tok(2), tok(3)), ranges.next());
@@ -203,7 +208,7 @@
Range<Token> range = new Range<>(tok(-1), tok(-1));
// (zero, zero]
- assertNull(mt.hash(range));
+ assertFalse(mt.hashesRange(range));
// validate the range
mt.get(tok(-1)).hash(val);
@@ -226,11 +231,12 @@
// (zero,two] (two,four] (four, zero]
mt.split(tok(4));
mt.split(tok(2));
- assertNull(mt.hash(left));
- assertNull(mt.hash(partial));
- assertNull(mt.hash(right));
- assertNull(mt.hash(linvalid));
- assertNull(mt.hash(rinvalid));
+
+ assertFalse(mt.hashesRange(left));
+ assertFalse(mt.hashesRange(partial));
+ assertFalse(mt.hashesRange(right));
+ assertFalse(mt.hashesRange(linvalid));
+ assertFalse(mt.hashesRange(rinvalid));
// validate the range
mt.get(tok(2)).hash(val);
@@ -240,8 +246,8 @@
assertHashEquals(leftval, mt.hash(left));
assertHashEquals(partialval, mt.hash(partial));
assertHashEquals(val, mt.hash(right));
- assertNull(mt.hash(linvalid));
- assertNull(mt.hash(rinvalid));
+ assertFalse(mt.hashesRange(linvalid));
+ assertFalse(mt.hashesRange(rinvalid));
}
@Test
@@ -261,10 +267,6 @@
mt.split(tok(2));
mt.split(tok(6));
mt.split(tok(1));
- assertNull(mt.hash(full));
- assertNull(mt.hash(lchild));
- assertNull(mt.hash(rchild));
- assertNull(mt.hash(invalid));
// validate the range
mt.get(tok(1)).hash(val);
@@ -273,10 +275,14 @@
mt.get(tok(6)).hash(val);
mt.get(tok(-1)).hash(val);
+ assertTrue(mt.hashesRange(full));
+ assertTrue(mt.hashesRange(lchild));
+ assertTrue(mt.hashesRange(rchild));
+ assertFalse(mt.hashesRange(invalid));
+
assertHashEquals(fullval, mt.hash(full));
assertHashEquals(lchildval, mt.hash(lchild));
assertHashEquals(rchildval, mt.hash(rchild));
- assertNull(mt.hash(invalid));
}
@Test
@@ -297,9 +303,6 @@
mt.split(tok(4));
mt.split(tok(2));
mt.split(tok(1));
- assertNull(mt.hash(full));
- assertNull(mt.hash(childfull));
- assertNull(mt.hash(invalid));
// validate the range
mt.get(tok(1)).hash(val);
@@ -309,9 +312,12 @@
mt.get(tok(16)).hash(val);
mt.get(tok(-1)).hash(val);
+ assertTrue(mt.hashesRange(full));
+ assertTrue(mt.hashesRange(childfull));
+ assertFalse(mt.hashesRange(invalid));
+
assertHashEquals(fullval, mt.hash(full));
assertHashEquals(childfullval, mt.hash(childfull));
- assertNull(mt.hash(invalid));
}
@Test
@@ -329,7 +335,7 @@
}
// validate the tree
- TreeRangeIterator ranges = mt.invalids();
+ TreeRangeIterator ranges = mt.rangeIterator();
for (TreeRange range : ranges)
range.addHash(new RowHash(range.right, new byte[0], 0));
@@ -358,7 +364,7 @@
mt.split(tok(6));
mt.split(tok(10));
- ranges = mt.invalids();
+ ranges = mt.rangeIterator();
ranges.next().addAll(new HIterator(2, 4)); // (-1,4]: depth 2
ranges.next().addAll(new HIterator(6)); // (4,6]
ranges.next().addAll(new HIterator(8)); // (6,8]
@@ -375,7 +381,7 @@
mt2.split(tok(9));
mt2.split(tok(11));
- ranges = mt2.invalids();
+ ranges = mt2.rangeIterator();
ranges.next().addAll(new HIterator(2)); // (-1,2]
ranges.next().addAll(new HIterator(4)); // (2,4]
ranges.next().addAll(new HIterator(6, 8)); // (4,8]: depth 2
@@ -398,19 +404,33 @@
// populate and validate the tree
mt.maxsize(256);
mt.init();
- for (TreeRange range : mt.invalids())
+ for (TreeRange range : mt.rangeIterator())
range.addAll(new HIterator(range.right));
byte[] initialhash = mt.hash(full);
DataOutputBuffer out = new DataOutputBuffer();
- MerkleTree.serializer.serialize(mt, out, MessagingService.current_version);
+ mt.serialize(out, MessagingService.current_version);
byte[] serialized = out.toByteArray();
- DataInputPlus in = new DataInputBuffer(serialized);
- MerkleTree restored = MerkleTree.serializer.deserialize(in, MessagingService.current_version);
+ MerkleTree restoredOnHeap =
+ MerkleTree.deserialize(new DataInputBuffer(serialized), false, MessagingService.current_version);
+ MerkleTree restoredOffHeap =
+ MerkleTree.deserialize(new DataInputBuffer(serialized), true, MessagingService.current_version);
+ MerkleTree movedOffHeap = mt.moveOffHeap();
- assertHashEquals(initialhash, restored.hash(full));
+ assertHashEquals(initialhash, restoredOnHeap.hash(full));
+ assertHashEquals(initialhash, restoredOffHeap.hash(full));
+ assertHashEquals(initialhash, movedOffHeap.hash(full));
+
+ assertEquals(mt, restoredOnHeap);
+ assertEquals(mt, restoredOffHeap);
+ assertEquals(mt, movedOffHeap);
+
+ assertEquals(restoredOnHeap, restoredOffHeap);
+ assertEquals(restoredOnHeap, movedOffHeap);
+
+ assertEquals(restoredOffHeap, movedOffHeap);
}
@Test
@@ -423,9 +443,9 @@
mt2.init();
// add dummy hashes to both trees
- for (TreeRange range : mt.invalids())
+ for (TreeRange range : mt.rangeIterator())
range.addAll(new HIterator(range.right));
- for (TreeRange range : mt2.invalids())
+ for (TreeRange range : mt2.rangeIterator())
range.addAll(new HIterator(range.right));
TreeRange leftmost = null;
@@ -434,14 +454,14 @@
mt.maxsize(maxsize + 2); // give some room for splitting
// split the leftmost
- Iterator<TreeRange> ranges = mt.invalids();
+ Iterator<TreeRange> ranges = mt.rangeIterator();
leftmost = ranges.next();
mt.split(leftmost.right);
// set the hashes for the leaf of the created split
middle = mt.get(leftmost.right);
- middle.hash("arbitrary!".getBytes());
- mt.get(partitioner.midpoint(leftmost.left, leftmost.right)).hash("even more arbitrary!".getBytes());
+ middle.hash(digest("arbitrary!"));
+ mt.get(partitioner.midpoint(leftmost.left, leftmost.right)).hash(digest("even more arbitrary!"));
// trees should disagree for (leftmost.left, middle.right]
List<TreeRange> diffs = MerkleTree.difference(mt, mt2);
@@ -464,22 +484,23 @@
MerkleTree rtree = new MerkleTree(partitioner, range, RECOMMENDED_DEPTH, 16);
rtree.init();
- byte[] h1 = "asdf".getBytes();
- byte[] h2 = "hjkl".getBytes();
+ byte[] h1 = digest("asdf");
+ byte[] h2 = digest("hjkl");
// add dummy hashes to both trees
- for (TreeRange tree : ltree.invalids())
+ for (TreeRange tree : ltree.rangeIterator())
{
tree.addHash(new RowHash(range.right, h1, h1.length));
}
- for (TreeRange tree : rtree.invalids())
+ for (TreeRange tree : rtree.rangeIterator())
{
tree.addHash(new RowHash(range.right, h2, h2.length));
}
List<TreeRange> diffs = MerkleTree.difference(ltree, rtree);
- assertEquals(Lists.newArrayList(range), diffs);
- assertEquals(MerkleTree.FULLY_INCONSISTENT, MerkleTree.differenceHelper(ltree, rtree, new ArrayList<>(), new MerkleTree.TreeDifference(ltree.fullRange.left, ltree.fullRange.right, (byte)0)));
+ assertEquals(newArrayList(range), diffs);
+ assertEquals(MerkleTree.Difference.FULLY_INCONSISTENT,
+ MerkleTree.differenceHelper(ltree, rtree, new ArrayList<>(), new MerkleTree.TreeRange(ltree.fullRange.left, ltree.fullRange.right, (byte)0)));
}
/**
@@ -497,22 +518,22 @@
MerkleTree rtree = new MerkleTree(partitioner, range, RECOMMENDED_DEPTH, 16);
rtree.init();
- byte[] h1 = "asdf".getBytes();
- byte[] h2 = "asdf".getBytes();
+ byte[] h1 = digest("asdf");
+ byte[] h2 = digest("asdf");
// add dummy hashes to both trees
- for (TreeRange tree : ltree.invalids())
+ for (TreeRange tree : ltree.rangeIterator())
{
tree.addHash(new RowHash(range.right, h1, h1.length));
}
- for (TreeRange tree : rtree.invalids())
+ for (TreeRange tree : rtree.rangeIterator())
{
tree.addHash(new RowHash(range.right, h2, h2.length));
}
// top level difference() should show no differences
- assertEquals(MerkleTree.difference(ltree, rtree), Lists.newArrayList());
+ assertEquals(MerkleTree.difference(ltree, rtree), newArrayList());
}
/**
@@ -536,8 +557,8 @@
while (depth.equals(dstack.peek()))
{
// consume the stack
- hash = Hashable.binaryHash(hstack.pop(), hash);
- depth = dstack.pop()-1;
+ hash = MerkleTree.xor(hstack.pop(), hash);
+ depth = dstack.pop() - 1;
}
dstack.push(depth);
hstack.push(hash);
@@ -570,4 +591,327 @@
return endOfData();
}
}
+
+ @Test
+ public void testEstimatedSizes()
+ {
+ // With no or negative allowed space we should still get a depth of 1
+ Assert.assertEquals(1, MerkleTree.estimatedMaxDepthForBytes(Murmur3Partitioner.instance, -20, 32));
+ Assert.assertEquals(1, MerkleTree.estimatedMaxDepthForBytes(Murmur3Partitioner.instance, 0, 32));
+ Assert.assertEquals(1, MerkleTree.estimatedMaxDepthForBytes(Murmur3Partitioner.instance, 1, 32));
+
+ // The minimum of 1 megabyte split between RF=3 should yield trees of around 10
+ Assert.assertEquals(10, MerkleTree.estimatedMaxDepthForBytes(Murmur3Partitioner.instance,
+ 1048576 / 3, 32));
+
+ // With a single megabyte of space we should get 12
+ Assert.assertEquals(12, MerkleTree.estimatedMaxDepthForBytes(Murmur3Partitioner.instance,
+ 1048576, 32));
+
+ // With 100 megabytes we should get a limit of 19
+ Assert.assertEquals(19, MerkleTree.estimatedMaxDepthForBytes(Murmur3Partitioner.instance,
+ 100 * 1048576, 32));
+
+ // With 300 megabytes we should get the old limit of 20
+ Assert.assertEquals(20, MerkleTree.estimatedMaxDepthForBytes(Murmur3Partitioner.instance,
+ 300 * 1048576, 32));
+ Assert.assertEquals(20, MerkleTree.estimatedMaxDepthForBytes(RandomPartitioner.instance,
+ 300 * 1048576, 32));
+ Assert.assertEquals(20, MerkleTree.estimatedMaxDepthForBytes(ByteOrderedPartitioner.instance,
+ 300 * 1048576, 32));
+ }
+
+ @Test
+ public void testEstimatedSizesRealMeasurement()
+ {
+ // Use a fixed source of randomness so that the test does not flake.
+ Random random = new Random(1);
+ checkEstimatedSizes(RandomPartitioner.instance, random);
+ checkEstimatedSizes(Murmur3Partitioner.instance, random);
+ }
+
+ private void checkEstimatedSizes(IPartitioner partitioner, Random random)
+ {
+ Range<Token> fullRange = new Range<>(partitioner.getMinimumToken(), partitioner.getMinimumToken());
+ MerkleTree tree = new MerkleTree(partitioner, fullRange, RECOMMENDED_DEPTH, 0);
+
+ // Test 16 kilobyte -> 16 megabytes
+ for (int i = 14; i < 24; i ++)
+ {
+ long numBytes = 1 << i;
+ int maxDepth = MerkleTree.estimatedMaxDepthForBytes(partitioner, numBytes, 32);
+ long realSizeOfMerkleTree = measureTree(tree, fullRange, maxDepth, random);
+ long biggerTreeSize = measureTree(tree, fullRange, maxDepth + 1, random);
+
+ Assert.assertTrue(realSizeOfMerkleTree < numBytes);
+ Assert.assertTrue(biggerTreeSize > numBytes);
+ }
+ }
+
+ private long measureTree(MerkleTree tree, Range<Token> fullRange, int depth, Random random)
+ {
+ tree = new MerkleTree(tree.partitioner(), fullRange, RECOMMENDED_DEPTH, (long) Math.pow(2, depth));
+ // Initializes it as a fully balanced tree.
+ tree.init();
+
+ byte[] key = new byte[128];
+ // Try to actually allocate some hashes. Note that this is not guaranteed to actually populate the tree,
+ // but we re-use the source of randomness to try to make it reproducible.
+ for (int i = 0; i < tree.maxsize() * 8; i++)
+ {
+ random.nextBytes(key);
+ Token token = tree.partitioner().getToken(ByteBuffer.wrap(key));
+ tree.get(token).addHash(new RowHash(token, new byte[32], 32));
+ }
+
+ tree.hash(fullRange);
+ return ObjectSizes.measureDeep(tree);
+ }
+
+ @Test
+ public void testEqualTreesSameDepth() throws IOException
+ {
+ int seed = makeSeed();
+ Trees trees = Trees.make(seed, seed, 3, 3);
+ testDifferences(trees, Collections.emptyList());
+ }
+
+ @Test
+ public void testEqualTreesDifferentDepth() throws IOException
+ {
+ int seed = makeSeed();
+ Trees trees = Trees.make(seed, seed, 2, 3);
+ testDifferences(trees, Collections.emptyList());
+ }
+
+ @Test
+ public void testEntirelyDifferentTrees() throws IOException
+ {
+ int seed1 = makeSeed();
+ int seed2 = seed1 * 32;
+ Trees trees = Trees.make(seed1, seed2, 3, 3);
+ testDifferences(trees, newArrayList(makeTreeRange(0, 16, 0)));
+ }
+
+ @Test
+ public void testDifferentTrees1SameDepth() throws IOException
+ {
+ int seed = makeSeed();
+ Trees trees = Trees.make(seed, seed, 3, 3);
+ trees.tree1.get(longToken(1)).addHash(digest("diff_1"), 1);
+ testDifferences(trees, newArrayList(makeTreeRange(0, 2, 3)));
+ }
+
+ @Test
+ public void testDifferentTrees1DifferentDepth() throws IOException
+ {
+ int seed = makeSeed();
+ Trees trees = Trees.make(seed, seed, 2, 3);
+ trees.tree1.get(longToken(1)).addHash(digest("diff_1"), 1);
+ testDifferences(trees, newArrayList(makeTreeRange(0, 4, 2)));
+ }
+
+ @Test
+ public void testDifferentTrees2SameDepth() throws IOException
+ {
+ int seed = makeSeed();
+ Trees trees = Trees.make(seed, seed, 3, 3);
+ trees.tree1.get(longToken(1)).addHash(digest("diff_1"), 1);
+ trees.tree2.get(longToken(16)).addHash(digest("diff_16"), 1);
+ testDifferences(trees, newArrayList(makeTreeRange(0, 2, 3),
+ makeTreeRange(14, 16, 3)));
+ }
+
+ @Test
+ public void testDifferentTrees2DifferentDepth() throws IOException
+ {
+ int seed = makeSeed();
+ Trees trees = Trees.make(seed, seed, 2, 3);
+ trees.tree1.get(longToken(1)).addHash(digest("diff_1"), 1);
+ trees.tree2.get(longToken(16)).addHash(digest("diff_16"), 1);
+ testDifferences(trees, newArrayList(makeTreeRange(0, 4, 2),
+ makeTreeRange(12, 16, 2)));
+ }
+
+ @Test
+ public void testDifferentTrees3SameDepth() throws IOException
+ {
+ int seed = makeSeed();
+ Trees trees = Trees.make(seed, seed, 3, 3);
+ trees.tree1.get(longToken(1)).addHash(digest("diff_1"), 1);
+ trees.tree1.get(longToken(3)).addHash(digest("diff_3"), 1);
+ testDifferences(trees, newArrayList(makeTreeRange(0, 4, 2)));
+ }
+
+ @Test
+ public void testDifferentTrees3Differentepth() throws IOException
+ {
+ int seed = makeSeed();
+ Trees trees = Trees.make(seed, seed, 2, 3);
+ trees.tree1.get(longToken(1)).addHash(digest("diff_1"), 1);
+ trees.tree1.get(longToken(3)).addHash(digest("diff_3"), 1);
+ testDifferences(trees, newArrayList(makeTreeRange(0, 4, 2)));
+ }
+
+ @Test
+ public void testDifferentTrees4SameDepth() throws IOException
+ {
+ int seed = makeSeed();
+ Trees trees = Trees.make(seed, seed, 3, 3);
+ trees.tree1.get(longToken(4)).addHash(digest("diff_4"), 1);
+ trees.tree1.get(longToken(8)).addHash(digest("diff_8"), 1);
+ trees.tree1.get(longToken(12)).addHash(digest("diff_12"), 1);
+ trees.tree1.get(longToken(16)).addHash(digest("diff_16"), 1);
+ testDifferences(trees, newArrayList(makeTreeRange(2, 4, 3),
+ makeTreeRange(6, 8, 3),
+ makeTreeRange(10, 12, 3),
+ makeTreeRange(14, 16, 3)));
+ }
+
+ @Test
+ public void testDifferentTrees4DifferentDepth() throws IOException
+ {
+ int seed = makeSeed();
+ Trees trees = Trees.make(seed, seed, 2, 3);
+ trees.tree1.get(longToken(4)).addHash(digest("diff_4"), 1);
+ trees.tree1.get(longToken(8)).addHash(digest("diff_8"), 1);
+ trees.tree1.get(longToken(12)).addHash(digest("diff_12"), 1);
+ trees.tree1.get(longToken(16)).addHash(digest("diff_16"), 1);
+ testDifferences(trees, newArrayList(makeTreeRange(0, 16, 0)));
+ }
+
+ private static void testDifferences(Trees trees, List<TreeRange> expectedDifference) throws IOException
+ {
+ MerkleTree mt1 = trees.tree1;
+ MerkleTree mt2 = trees.tree2;
+
+ assertDiffer(mt1, mt2, expectedDifference);
+ assertDiffer(mt1, mt2.moveOffHeap(), expectedDifference);
+ assertDiffer(mt1, cycle(mt2, true), expectedDifference);
+ assertDiffer(mt1, cycle(mt2, false), expectedDifference);
+ assertDiffer(mt1, cycle(mt2.moveOffHeap(), true), expectedDifference);
+ assertDiffer(mt1, cycle(mt2.moveOffHeap(), false), expectedDifference);
+
+ assertDiffer(mt1.moveOffHeap(), mt2, expectedDifference);
+ assertDiffer(mt1.moveOffHeap(), mt2.moveOffHeap(), expectedDifference);
+ assertDiffer(mt1.moveOffHeap(), cycle(mt2, true), expectedDifference);
+ assertDiffer(mt1.moveOffHeap(), cycle(mt2, false), expectedDifference);
+ assertDiffer(mt1.moveOffHeap(), cycle(mt2.moveOffHeap(), true), expectedDifference);
+ assertDiffer(mt1.moveOffHeap(), cycle(mt2.moveOffHeap(), false), expectedDifference);
+
+ assertDiffer(cycle(mt1, true), mt2, expectedDifference);
+ assertDiffer(cycle(mt1, true), mt2.moveOffHeap(), expectedDifference);
+ assertDiffer(cycle(mt1, true), cycle(mt2, true), expectedDifference);
+ assertDiffer(cycle(mt1, true), cycle(mt2, false), expectedDifference);
+ assertDiffer(cycle(mt1, true), cycle(mt2.moveOffHeap(), true), expectedDifference);
+ assertDiffer(cycle(mt1, true), cycle(mt2.moveOffHeap(), false), expectedDifference);
+
+ assertDiffer(cycle(mt1, false), mt2, expectedDifference);
+ assertDiffer(cycle(mt1, false), mt2.moveOffHeap(), expectedDifference);
+ assertDiffer(cycle(mt1, false), cycle(mt2, true), expectedDifference);
+ assertDiffer(cycle(mt1, false), cycle(mt2, false), expectedDifference);
+ assertDiffer(cycle(mt1, false), cycle(mt2.moveOffHeap(), true), expectedDifference);
+ assertDiffer(cycle(mt1, false), cycle(mt2.moveOffHeap(), false), expectedDifference);
+
+ assertDiffer(cycle(mt1.moveOffHeap(), true), mt2, expectedDifference);
+ assertDiffer(cycle(mt1.moveOffHeap(), true), mt2.moveOffHeap(), expectedDifference);
+ assertDiffer(cycle(mt1.moveOffHeap(), true), cycle(mt2, true), expectedDifference);
+ assertDiffer(cycle(mt1.moveOffHeap(), true), cycle(mt2, false), expectedDifference);
+ assertDiffer(cycle(mt1.moveOffHeap(), true), cycle(mt2.moveOffHeap(), true), expectedDifference);
+ assertDiffer(cycle(mt1.moveOffHeap(), true), cycle(mt2.moveOffHeap(), false), expectedDifference);
+
+ assertDiffer(cycle(mt1.moveOffHeap(), false), mt2, expectedDifference);
+ assertDiffer(cycle(mt1.moveOffHeap(), false), mt2.moveOffHeap(), expectedDifference);
+ assertDiffer(cycle(mt1.moveOffHeap(), false), cycle(mt2, true), expectedDifference);
+ assertDiffer(cycle(mt1.moveOffHeap(), false), cycle(mt2, false), expectedDifference);
+ assertDiffer(cycle(mt1.moveOffHeap(), false), cycle(mt2.moveOffHeap(), true), expectedDifference);
+ assertDiffer(cycle(mt1.moveOffHeap(), false), cycle(mt2.moveOffHeap(), false), expectedDifference);
+ }
+
+ private static void assertDiffer(MerkleTree mt1, MerkleTree mt2, List<TreeRange> expectedDifference)
+ {
+ assertEquals(expectedDifference, MerkleTree.difference(mt1, mt2));
+ assertEquals(expectedDifference, MerkleTree.difference(mt2, mt1));
+ }
+
+ private static Range<Token> longTokenRange(long start, long end)
+ {
+ return new Range<>(longToken(start), longToken(end));
+ }
+
+ private static Murmur3Partitioner.LongToken longToken(long value)
+ {
+ return new Murmur3Partitioner.LongToken(value);
+ }
+
+ private static MerkleTree cycle(MerkleTree mt, boolean offHeapRequested) throws IOException
+ {
+ try (DataOutputBuffer output = new DataOutputBuffer())
+ {
+ mt.serialize(output, MessagingService.current_version);
+
+ try (DataInputBuffer input = new DataInputBuffer(output.buffer(false), false))
+ {
+ return MerkleTree.deserialize(input, offHeapRequested, MessagingService.current_version);
+ }
+ }
+ }
+
+ private static MerkleTree makeTree(long start, long end, int depth)
+ {
+ MerkleTree mt = new MerkleTree(Murmur3Partitioner.instance, longTokenRange(start, end), depth, Long.MAX_VALUE);
+ mt.init();
+ return mt;
+ }
+
+ private static TreeRange makeTreeRange(long start, long end, int depth)
+ {
+ return new TreeRange(longToken(start), longToken(end), depth);
+ }
+
+ private static byte[][] makeHashes(int count, int seed)
+ {
+ Random random = new Random(seed);
+
+ byte[][] hashes = new byte[count][32];
+ for (int i = 0; i < count; i++)
+ random.nextBytes(hashes[i]);
+ return hashes;
+ }
+
+ private static int makeSeed()
+ {
+ int seed = (int) System.currentTimeMillis();
+ System.out.println("Using seed " + seed);
+ return seed;
+ }
+
+ private static class Trees
+ {
+ MerkleTree tree1;
+ MerkleTree tree2;
+
+ Trees(MerkleTree tree1, MerkleTree tree2)
+ {
+ this.tree1 = tree1;
+ this.tree2 = tree2;
+ }
+
+ static Trees make(int hashes1seed, int hashes2seed, int tree1depth, int tree2depth)
+ {
+ byte[][] hashes1 = makeHashes(16, hashes1seed);
+ byte[][] hashes2 = makeHashes(16, hashes2seed);
+
+ MerkleTree tree1 = makeTree(0, 16, tree1depth);
+ MerkleTree tree2 = makeTree(0, 16, tree2depth);
+
+ for (int tok = 1; tok <= 16; tok++)
+ {
+ tree1.get(longToken(tok)).addHash(hashes1[tok - 1], 1);
+ tree2.get(longToken(tok)).addHash(hashes2[tok - 1], 1);
+ }
+
+ return new Trees(tree1, tree2);
+ }
+ }
}
diff --git a/test/unit/org/apache/cassandra/utils/MerkleTreesTest.java b/test/unit/org/apache/cassandra/utils/MerkleTreesTest.java
index 8d7284c..5b589fb 100644
--- a/test/unit/org/apache/cassandra/utils/MerkleTreesTest.java
+++ b/test/unit/org/apache/cassandra/utils/MerkleTreesTest.java
@@ -34,16 +34,16 @@
import org.apache.cassandra.io.util.DataOutputBuffer;
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.service.StorageService;
-import org.apache.cassandra.utils.MerkleTree.Hashable;
import org.apache.cassandra.utils.MerkleTree.RowHash;
import org.apache.cassandra.utils.MerkleTree.TreeRange;
import org.apache.cassandra.utils.MerkleTrees.TreeRangeIterator;
+import static org.apache.cassandra.utils.MerkleTreeTest.digest;
import static org.junit.Assert.*;
public class MerkleTreesTest
{
- public static byte[] DUMMY = "blah".getBytes();
+ private static final byte[] DUMMY = digest("dummy");
/**
* If a test assumes that the tree is 8 units wide, then it should set this value
@@ -193,7 +193,7 @@
Iterator<TreeRange> ranges;
// (zero, zero]
- ranges = mts.invalids();
+ ranges = mts.rangeIterator();
assertEquals(new Range<>(tok(-1), tok(-1)), ranges.next());
assertFalse(ranges.hasNext());
@@ -203,7 +203,7 @@
mts.split(tok(6));
mts.split(tok(3));
mts.split(tok(5));
- ranges = mts.invalids();
+ ranges = mts.rangeIterator();
assertEquals(new Range<>(tok(6), tok(-1)), ranges.next());
assertEquals(new Range<>(tok(-1), tok(2)), ranges.next());
assertEquals(new Range<>(tok(2), tok(3)), ranges.next());
@@ -245,11 +245,6 @@
// (zero,two] (two,four] (four, zero]
mts.split(tok(4));
mts.split(tok(2));
- assertNull(mts.hash(left));
- assertNull(mts.hash(partial));
- assertNull(mts.hash(right));
- assertNull(mts.hash(linvalid));
- assertNull(mts.hash(rinvalid));
// validate the range
mts.get(tok(2)).hash(val);
@@ -280,10 +275,6 @@
mts.split(tok(2));
mts.split(tok(6));
mts.split(tok(1));
- assertNull(mts.hash(full));
- assertNull(mts.hash(lchild));
- assertNull(mts.hash(rchild));
- assertNull(mts.hash(invalid));
// validate the range
mts.get(tok(1)).hash(val);
@@ -315,9 +306,6 @@
mts.split(tok(4));
mts.split(tok(2));
mts.split(tok(1));
- assertNull(mts.hash(full));
- assertNull(mts.hash(childfull));
- assertNull(mts.hash(invalid));
// validate the range
mts.get(tok(1)).hash(val);
@@ -349,7 +337,7 @@
}
// validate the tree
- TreeRangeIterator ranges = mts.invalids();
+ TreeRangeIterator ranges = mts.rangeIterator();
for (TreeRange range : ranges)
range.addHash(new RowHash(range.right, new byte[0], 0));
@@ -378,13 +366,16 @@
mts.split(tok(6));
mts.split(tok(10));
- ranges = mts.invalids();
- ranges.next().addAll(new HIterator(2, 4)); // (-1,4]: depth 2
- ranges.next().addAll(new HIterator(6)); // (4,6]
- ranges.next().addAll(new HIterator(8)); // (6,8]
+ int seed = 123456789;
+
+ Random random1 = new Random(seed);
+ ranges = mts.rangeIterator();
+ ranges.next().addAll(new HIterator(random1, 2, 4)); // (-1,4]: depth 2
+ ranges.next().addAll(new HIterator(random1, 6)); // (4,6]
+ ranges.next().addAll(new HIterator(random1, 8)); // (6,8]
ranges.next().addAll(new HIterator(/*empty*/ new int[0])); // (8,10]
- ranges.next().addAll(new HIterator(12)); // (10,12]
- ranges.next().addAll(new HIterator(14, -1)); // (12,-1]: depth 2
+ ranges.next().addAll(new HIterator(random1, 12)); // (10,12]
+ ranges.next().addAll(new HIterator(random1, 14, -1)); // (12,-1]: depth 2
mts2.split(tok(8));
@@ -395,15 +386,16 @@
mts2.split(tok(9));
mts2.split(tok(11));
- ranges = mts2.invalids();
- ranges.next().addAll(new HIterator(2)); // (-1,2]
- ranges.next().addAll(new HIterator(4)); // (2,4]
- ranges.next().addAll(new HIterator(6, 8)); // (4,8]: depth 2
+ Random random2 = new Random(seed);
+ ranges = mts2.rangeIterator();
+ ranges.next().addAll(new HIterator(random2, 2)); // (-1,2]
+ ranges.next().addAll(new HIterator(random2, 4)); // (2,4]
+ ranges.next().addAll(new HIterator(random2, 6, 8)); // (4,8]: depth 2
ranges.next().addAll(new HIterator(/*empty*/ new int[0])); // (8,9]
ranges.next().addAll(new HIterator(/*empty*/ new int[0])); // (9,10]
ranges.next().addAll(new HIterator(/*empty*/ new int[0])); // (10,11]: depth 4
- ranges.next().addAll(new HIterator(12)); // (11,12]: depth 4
- ranges.next().addAll(new HIterator(14, -1)); // (12,-1]: depth 2
+ ranges.next().addAll(new HIterator(random2, 12)); // (11,12]: depth 4
+ ranges.next().addAll(new HIterator(random2, 14, -1)); // (12,-1]: depth 2
byte[] mthash = mts.hash(full);
byte[] mt2hash = mts2.hash(full);
@@ -425,7 +417,7 @@
// populate and validate the tree
mts.init();
- for (TreeRange range : mts.invalids())
+ for (TreeRange range : mts.rangeIterator())
range.addAll(new HIterator(range.right));
byte[] initialhash = mts.hash(first);
@@ -456,11 +448,15 @@
mts.init();
mts2.init();
+ int seed = 123456789;
// add dummy hashes to both trees
- for (TreeRange range : mts.invalids())
- range.addAll(new HIterator(range.right));
- for (TreeRange range : mts2.invalids())
- range.addAll(new HIterator(range.right));
+ Random random1 = new Random(seed);
+ for (TreeRange range : mts.rangeIterator())
+ range.addAll(new HIterator(random1, range.right));
+
+ Random random2 = new Random(seed);
+ for (TreeRange range : mts2.rangeIterator())
+ range.addAll(new HIterator(random2, range.right));
TreeRange leftmost = null;
TreeRange middle = null;
@@ -468,14 +464,14 @@
mts.maxsize(fullRange(), maxsize + 2); // give some room for splitting
// split the leftmost
- Iterator<TreeRange> ranges = mts.invalids();
+ Iterator<TreeRange> ranges = mts.rangeIterator();
leftmost = ranges.next();
mts.split(leftmost.right);
// set the hashes for the leaf of the created split
middle = mts.get(leftmost.right);
- middle.hash("arbitrary!".getBytes());
- mts.get(partitioner.midpoint(leftmost.left, leftmost.right)).hash("even more arbitrary!".getBytes());
+ middle.hash(digest("arbitrary!"));
+ mts.get(partitioner.midpoint(leftmost.left, leftmost.right)).hash(digest("even more arbitrary!"));
// trees should disagree for (leftmost.left, middle.right]
List<Range<Token>> diffs = MerkleTrees.difference(mts, mts2);
@@ -504,7 +500,7 @@
while (depth.equals(dstack.peek()))
{
// consume the stack
- hash = Hashable.binaryHash(hstack.pop(), hash);
+ hash = MerkleTree.xor(hstack.pop(), hash);
depth = dstack.pop()-1;
}
dstack.push(depth);
@@ -514,27 +510,49 @@
return hstack.pop();
}
- static class HIterator extends AbstractIterator<RowHash>
+ public static class HIterator extends AbstractIterator<RowHash>
{
- private Iterator<Token> tokens;
+ private final Random random;
+ private final Iterator<Token> tokens;
- public HIterator(int... tokens)
+ HIterator(int... tokens)
{
- List<Token> tlist = new LinkedList<Token>();
+ this(new Random(), tokens);
+ }
+
+ HIterator(Random random, int... tokens)
+ {
+ List<Token> tlist = new ArrayList<>(tokens.length);
for (int token : tokens)
tlist.add(tok(token));
this.tokens = tlist.iterator();
+ this.random = random;
}
public HIterator(Token... tokens)
{
- this.tokens = Arrays.asList(tokens).iterator();
+ this(new Random(), tokens);
+ }
+
+ HIterator(Random random, Token... tokens)
+ {
+ this(random, Arrays.asList(tokens).iterator());
+ }
+
+ private HIterator(Random random, Iterator<Token> tokens)
+ {
+ this.random = random;
+ this.tokens = tokens;
}
public RowHash computeNext()
{
if (tokens.hasNext())
- return new RowHash(tokens.next(), DUMMY, DUMMY.length);
+ {
+ byte[] digest = new byte[32];
+ random.nextBytes(digest);
+ return new RowHash(tokens.next(), digest, 12345L);
+ }
return endOfData();
}
}
diff --git a/test/unit/org/apache/cassandra/utils/MonotonicClockTest.java b/test/unit/org/apache/cassandra/utils/MonotonicClockTest.java
new file mode 100644
index 0000000..b2891a9
--- /dev/null
+++ b/test/unit/org/apache/cassandra/utils/MonotonicClockTest.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.utils;
+
+import static org.apache.cassandra.utils.MonotonicClock.approxTime;
+import static org.junit.Assert.*;
+
+import org.junit.Test;
+
+public class MonotonicClockTest
+{
+ @Test
+ public void testTimestampOrdering() throws Exception
+ {
+ long nowNanos = System.nanoTime();
+ long now = System.currentTimeMillis();
+ long lastConverted = 0;
+ for (long ii = 0; ii < 10000000; ii++)
+ {
+ now = Math.max(now, System.currentTimeMillis());
+ if (ii % 10000 == 0)
+ {
+ ((MonotonicClock.SampledClock) approxTime).refreshNow();
+ Thread.sleep(1);
+ }
+
+ nowNanos = Math.max(nowNanos, System.nanoTime());
+ long convertedNow = approxTime.translate().toMillisSinceEpoch(nowNanos);
+
+ int maxDiff = FBUtilities.isWindows ? 15 : 1;
+ assertTrue("convertedNow = " + convertedNow + " lastConverted = " + lastConverted + " in iteration " + ii,
+ convertedNow >= (lastConverted - maxDiff));
+
+ maxDiff = FBUtilities.isWindows ? 25 : 2;
+ assertTrue("now = " + now + " convertedNow = " + convertedNow + " in iteration " + ii,
+ (maxDiff - 2) <= convertedNow);
+
+ lastConverted = convertedNow;
+ }
+ }
+}
diff --git a/test/unit/org/apache/cassandra/utils/NanoTimeToCurrentTimeMillisTest.java b/test/unit/org/apache/cassandra/utils/NanoTimeToCurrentTimeMillisTest.java
deleted file mode 100644
index 25aeada..0000000
--- a/test/unit/org/apache/cassandra/utils/NanoTimeToCurrentTimeMillisTest.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.utils;
-
-import static org.junit.Assert.*;
-
-import org.junit.Test;
-
-public class NanoTimeToCurrentTimeMillisTest
-{
- @Test
- public void testTimestampOrdering() throws Exception
- {
- long nowNanos = System.nanoTime();
- long now = System.currentTimeMillis();
- long lastConverted = 0;
- for (long ii = 0; ii < 10000000; ii++)
- {
- now = Math.max(now, System.currentTimeMillis());
- if (ii % 10000 == 0)
- {
- NanoTimeToCurrentTimeMillis.updateNow();
- Thread.sleep(1);
- }
-
- nowNanos = Math.max(nowNanos, System.nanoTime());
- long convertedNow = NanoTimeToCurrentTimeMillis.convert(nowNanos);
-
- int maxDiff = FBUtilities.isWindows ? 15 : 1;
- assertTrue("convertedNow = " + convertedNow + " lastConverted = " + lastConverted + " in iteration " + ii,
- convertedNow >= (lastConverted - maxDiff));
-
- maxDiff = FBUtilities.isWindows ? 25 : 2;
- assertTrue("now = " + now + " convertedNow = " + convertedNow + " in iteration " + ii,
- (maxDiff - 2) <= convertedNow);
-
- lastConverted = convertedNow;
- }
- }
-}
diff --git a/test/unit/org/apache/cassandra/utils/NativeLibraryTest.java b/test/unit/org/apache/cassandra/utils/NativeLibraryTest.java
index 226653e..1a26351 100644
--- a/test/unit/org/apache/cassandra/utils/NativeLibraryTest.java
+++ b/test/unit/org/apache/cassandra/utils/NativeLibraryTest.java
@@ -30,7 +30,7 @@
@Test
public void testSkipCache()
{
- File file = FileUtils.createTempFile("testSkipCache", "1");
+ File file = FileUtils.createDeletableTempFile("testSkipCache", "1");
NativeLibrary.trySkipCache(file.getPath(), 0, 0);
}
diff --git a/test/unit/org/apache/cassandra/utils/NoSpamLoggerTest.java b/test/unit/org/apache/cassandra/utils/NoSpamLoggerTest.java
index 702fa98..fe5d58e 100644
--- a/test/unit/org/apache/cassandra/utils/NoSpamLoggerTest.java
+++ b/test/unit/org/apache/cassandra/utils/NoSpamLoggerTest.java
@@ -39,7 +39,7 @@
{
Map<Level, Queue<Pair<String, Object[]>>> logged = new HashMap<>();
- Logger mock = new SubstituteLogger(null)
+ Logger mock = new SubstituteLogger(null, null, true)
{
@Override
diff --git a/test/unit/org/apache/cassandra/utils/Retry.java b/test/unit/org/apache/cassandra/utils/Retry.java
new file mode 100644
index 0000000..513b0f2
--- /dev/null
+++ b/test/unit/org/apache/cassandra/utils/Retry.java
@@ -0,0 +1,222 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.utils;
+
+import java.time.Duration;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
+import java.util.concurrent.Executor;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ForkJoinPool;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Consumer;
+import java.util.function.IntToLongFunction;
+import java.util.function.Predicate;
+import java.util.function.Supplier;
+
+import org.apache.cassandra.concurrent.NamedThreadFactory;
+
+/**
+ * Class for retryable actions.
+ *
+ * @see {@link #retryWithBackoff(int, Supplier, Predicate)}
+ */
+public final class Retry
+{
+ private static final ScheduledExecutorService SCHEDULED = Executors.newSingleThreadScheduledExecutor(new NamedThreadFactory("RetryScheduler"));
+
+ private Retry()
+ {
+
+ }
+
+ /**
+ * Schedule code to run after the defined duration.
+ *
+ * Since a executor was not defined, the global {@link ForkJoinPool#commonPool()} executor will be used, if this
+ * is not desirable then should use {@link #schedule(Duration, Executor, Runnable)}.
+ *
+ * @param duration how long to delay
+ * @param fn code to run
+ * @return future representing result
+ */
+ public static CompletableFuture<Void> schedule(final Duration duration, final Runnable fn)
+ {
+ return schedule(duration, ForkJoinPool.commonPool(), fn);
+ }
+
+ /**
+ * Schedule code to run after the defined duration on the provided executor.
+ *
+ * @param duration how long to delay
+ * @param executor to run on
+ * @param fn code to run
+ * @return future representing result
+ */
+ public static CompletableFuture<Void> schedule(final Duration duration, final Executor executor, final Runnable fn)
+ {
+ long nanos = duration.toNanos();
+ CompletableFuture<Void> future = new CompletableFuture<>();
+ SCHEDULED.schedule(() -> run0(executor, future, fn), nanos, TimeUnit.NANOSECONDS);
+ return future;
+ }
+
+ private static void run0(final Executor executor, final CompletableFuture<Void> future, final Runnable fn)
+ {
+ try
+ {
+ executor.execute(() -> {
+ try
+ {
+ fn.run();
+ future.complete(null);
+ }
+ catch (Exception e)
+ {
+ future.completeExceptionally(e);
+ }
+ });
+ }
+ catch (Exception e)
+ {
+ future.completeExceptionally(e);
+ }
+ }
+
+ /**
+ * Continously attempting to call the provided future supplier until successful or until no longer able to retry.
+ *
+ * @param maxRetries to allow
+ * @param fn asyncronous operation to retry
+ * @param retryableException used to say if retry is allowed
+ * @return future representing the result. If retries were not able to get a successful result, the exception is the last exception seen.
+ */
+ public static <A> CompletableFuture<A> retryWithBackoff(final int maxRetries,
+ final Supplier<CompletableFuture<A>> fn,
+ final Predicate<Throwable> retryableException)
+ {
+ CompletableFuture<A> future = new CompletableFuture<>();
+ retryWithBackoff0(future, 0, maxRetries, fn, retryableException, retryCount -> computeSleepTimeMillis(retryCount, 50, 1000));
+ return future;
+ }
+
+ /**
+ * This is the same as {@link #retryWithBackoff(int, Supplier, Predicate)}, but takes a blocking retryable action
+ * and blocks the caller until done.
+ */
+ public static <A> A retryWithBackoffBlocking(final int maxRetries, final Supplier<A> fn)
+ {
+ return retryWithBackoffBlocking(maxRetries, fn, (ignore) -> true);
+ }
+
+ /**
+ * This is the same as {@link #retryWithBackoff(int, Supplier, Predicate)}, but takes a blocking retryable action
+ * and blocks the caller until done.
+ */
+ public static <A> A retryWithBackoffBlocking(final int maxRetries,
+ final Supplier<A> fn,
+ final Predicate<Throwable> retryableException)
+ {
+ return retryWithBackoff(maxRetries, () -> CompletableFuture.completedFuture(fn.get()), retryableException).join();
+ }
+
+ private static <A> void retryWithBackoff0(final CompletableFuture<A> result,
+ final int retryCount,
+ final int maxRetry,
+ final Supplier<CompletableFuture<A>> body,
+ final Predicate<Throwable> retryableException,
+ final IntToLongFunction completeSleep)
+ {
+ try
+ {
+ Consumer<Throwable> attemptRetry = cause -> {
+ if (retryCount >= maxRetry || !retryableException.test(cause))
+ {
+ // too many attempts or exception isn't retryable, so fail
+ result.completeExceptionally(cause);
+ }
+ else
+ {
+ long sleepMillis = completeSleep.applyAsLong(retryCount);
+ schedule(Duration.ofMillis(sleepMillis), () -> {
+ retryWithBackoff0(result, retryCount + 1, maxRetry, body, retryableException, completeSleep);
+ });
+ }
+ };
+
+ // sanity check that the future isn't filled
+ // the most likely cause for this is when the future is composed with other futures (such as .successAsList);
+ // the failure of a different future may cancel this one, so stop running
+ if (result.isDone())
+ {
+ if (!(result.isCancelled() || result.isCompletedExceptionally()))
+ {
+ // the result is success! But we didn't fill it...
+ new RuntimeException("Attempt to retry but found future was successful... aborting " + body).printStackTrace();
+ }
+ return;
+ }
+
+ CompletableFuture<A> future;
+ try
+ {
+ future = body.get();
+ }
+ catch (Exception e)
+ {
+ attemptRetry.accept(e);
+ return;
+ }
+
+ future.whenComplete((success, failure) -> {
+ if (failure == null)
+ {
+ result.complete(success);
+ }
+ else
+ {
+ attemptRetry.accept(failure instanceof CompletionException ? failure.getCause() : failure);
+ }
+ });
+ }
+ catch (Exception e)
+ {
+ result.completeExceptionally(e);
+ }
+ }
+
+ /**
+ * Compute a expoential delay based off the retry count and min/max delay.
+ */
+ private static long computeSleepTimeMillis(int retryCount, long baseSleepTimeMillis, long maxSleepMillis)
+ {
+ long baseTime = baseSleepTimeMillis * (1L << retryCount);
+ // its possible that this overflows, so fall back to max;
+ if (baseTime <= 0)
+ {
+ baseTime = maxSleepMillis;
+ }
+ // now make sure this is capped to target max
+ baseTime = Math.min(baseTime, maxSleepMillis);
+
+ return (long) (baseTime * (ThreadLocalRandom.current().nextDouble() + 0.5));
+ }
+}
diff --git a/test/unit/org/apache/cassandra/utils/SerializationsTest.java b/test/unit/org/apache/cassandra/utils/SerializationsTest.java
index ac5a6a7..f260428 100644
--- a/test/unit/org/apache/cassandra/utils/SerializationsTest.java
+++ b/test/unit/org/apache/cassandra/utils/SerializationsTest.java
@@ -31,44 +31,42 @@
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.marshal.Int32Type;
import org.apache.cassandra.io.util.DataInputPlus.DataInputStreamPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.io.util.DataOutputStreamPlus;
-import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.utils.obs.OffHeapBitSet;
import java.io.File;
import java.io.FileInputStream;
public class SerializationsTest extends AbstractSerializationsTester
{
+ // Helper function to serialize old Bloomfilter format, should be removed once the old format is not supported
+ public static void serializeOldBfFormat(BloomFilter bf, DataOutputPlus out) throws IOException
+ {
+ out.writeInt(bf.hashCount);
+ Assert.assertTrue(bf.bitset instanceof OffHeapBitSet);
+ ((OffHeapBitSet) bf.bitset).serializeOldBfFormat(out);
+ }
+
@BeforeClass
public static void initDD()
{
DatabaseDescriptor.daemonInitialization();
}
- private static void testBloomFilterWrite(boolean offheap, boolean oldBfHashOrder) throws IOException
+ private static void testBloomFilterWrite1000(boolean oldBfFormat) throws IOException
{
- IPartitioner partitioner = Util.testPartitioner();
- try (IFilter bf = FilterFactory.getFilter(1000000, 0.0001, offheap, oldBfHashOrder))
- {
- for (int i = 0; i < 100; i++)
- bf.add(partitioner.decorateKey(partitioner.getTokenFactory().toByteArray(partitioner.getRandomToken())));
- try (DataOutputStreamPlus out = getOutput(oldBfHashOrder ? "2.1" : "3.0", "utils.BloomFilter.bin"))
- {
- FilterFactory.serialize(bf, out);
- }
- }
- }
-
- private static void testBloomFilterWrite1000(boolean offheap, boolean oldBfHashOrder) throws IOException
- {
- try (IFilter bf = FilterFactory.getFilter(1000000, 0.0001, offheap, oldBfHashOrder))
+ try (IFilter bf = FilterFactory.getFilter(1000000, 0.0001))
{
for (int i = 0; i < 1000; i++)
bf.add(Util.dk(Int32Type.instance.decompose(i)));
- try (DataOutputStreamPlus out = getOutput(oldBfHashOrder ? "2.1" : "3.0", "utils.BloomFilter1000.bin"))
+ try (DataOutputStreamPlus out = getOutput(oldBfFormat ? "3.0" : "4.0", "utils.BloomFilter1000.bin"))
{
- FilterFactory.serialize(bf, out);
+ if (oldBfFormat)
+ serializeOldBfFormat((BloomFilter) bf, out);
+ else
+ BloomFilterSerializer.serialize((BloomFilter) bf, out);
}
}
}
@@ -78,12 +76,28 @@
{
if (EXECUTE_WRITES)
{
- testBloomFilterWrite1000(true, false);
- testBloomFilterWrite1000(true, true);
+ testBloomFilterWrite1000(false);
+ testBloomFilterWrite1000(true);
+ }
+
+ try (DataInputStream in = getInput("4.0", "utils.BloomFilter1000.bin");
+ IFilter filter = BloomFilterSerializer.deserialize(in, false))
+ {
+ boolean present;
+ for (int i = 0 ; i < 1000 ; i++)
+ {
+ present = filter.isPresent(Util.dk(Int32Type.instance.decompose(i)));
+ Assert.assertTrue(present);
+ }
+ for (int i = 1000 ; i < 2000 ; i++)
+ {
+ present = filter.isPresent(Util.dk(Int32Type.instance.decompose(i)));
+ Assert.assertFalse(present);
+ }
}
try (DataInputStream in = getInput("3.0", "utils.BloomFilter1000.bin");
- IFilter filter = FilterFactory.deserialize(in, true, false))
+ IFilter filter = BloomFilterSerializer.deserialize(in, true))
{
boolean present;
for (int i = 0 ; i < 1000 ; i++)
@@ -97,60 +111,20 @@
Assert.assertFalse(present);
}
}
-
- try (DataInputStream in = getInput("2.1", "utils.BloomFilter1000.bin");
- IFilter filter = FilterFactory.deserialize(in, true, true))
- {
- boolean present;
- for (int i = 0 ; i < 1000 ; i++)
- {
- present = filter.isPresent(Util.dk(Int32Type.instance.decompose(i)));
- Assert.assertTrue(present);
- }
- for (int i = 1000 ; i < 2000 ; i++)
- {
- present = filter.isPresent(Util.dk(Int32Type.instance.decompose(i)));
- Assert.assertFalse(present);
- }
- }
-
- // eh - reading version version 'ka' (2.1) with 3.0 BloomFilter
- int falsePositive = 0;
- int falseNegative = 0;
- try (DataInputStream in = getInput("2.1", "utils.BloomFilter1000.bin");
- IFilter filter = FilterFactory.deserialize(in, true, false))
- {
- boolean present;
- for (int i = 0 ; i < 1000 ; i++)
- {
- present = filter.isPresent(Util.dk(Int32Type.instance.decompose(i)));
- if (!present)
- falseNegative ++;
- }
- for (int i = 1000 ; i < 2000 ; i++)
- {
- present = filter.isPresent(Util.dk(Int32Type.instance.decompose(i)));
- if (present)
- falsePositive ++;
- }
- }
- Assert.assertEquals(1000, falseNegative);
- Assert.assertEquals(0, falsePositive);
}
@Test
public void testBloomFilterTable() throws Exception
{
- testBloomFilterTable("test/data/bloom-filter/ka/foo/foo-atable-ka-1-Filter.db", true);
- testBloomFilterTable("test/data/bloom-filter/la/foo/la-1-big-Filter.db", false);
+ testBloomFilterTable("test/data/bloom-filter/la/foo/la-1-big-Filter.db", true);
}
- private static void testBloomFilterTable(String file, boolean oldBfHashOrder) throws Exception
+ private static void testBloomFilterTable(String file, boolean oldBfFormat) throws Exception
{
Murmur3Partitioner partitioner = new Murmur3Partitioner();
try (DataInputStream in = new DataInputStream(new FileInputStream(new File(file)));
- IFilter filter = FilterFactory.deserialize(in, true, oldBfHashOrder))
+ IFilter filter = BloomFilterSerializer.deserialize(in, oldBfFormat))
{
for (int i = 1; i <= 10; i++)
{
@@ -173,31 +147,6 @@
}
}
- @Test
- public void testBloomFilterReadMURMUR3() throws IOException
- {
- if (EXECUTE_WRITES)
- testBloomFilterWrite(true, true);
-
- try (DataInputStream in = getInput("3.0", "utils.BloomFilter.bin");
- IFilter filter = FilterFactory.deserialize(in, true, true))
- {
- Assert.assertNotNull(filter);
- }
- }
-
- @Test
- public void testBloomFilterReadMURMUR3pre30() throws IOException
- {
- if (EXECUTE_WRITES)
- testBloomFilterWrite(true, false);
-
- try (DataInputStream in = getInput("2.1", "utils.BloomFilter.bin");
- IFilter filter = FilterFactory.deserialize(in, true, false))
- {
- Assert.assertNotNull(filter);
- }
- }
private static void testEstimatedHistogramWrite() throws IOException
{
diff --git a/test/unit/org/apache/cassandra/utils/StatusLoggerTest.java b/test/unit/org/apache/cassandra/utils/StatusLoggerTest.java
new file mode 100644
index 0000000..683d2ed
--- /dev/null
+++ b/test/unit/org/apache/cassandra/utils/StatusLoggerTest.java
@@ -0,0 +1,161 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.utils;
+
+import java.util.Comparator;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
+import com.google.common.collect.Range;
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import ch.qos.logback.classic.Level;
+import ch.qos.logback.classic.spi.ILoggingEvent;
+import ch.qos.logback.core.AppenderBase;
+import org.apache.cassandra.cql3.CQLTester;
+
+import static com.google.common.collect.Lists.newArrayList;
+import static java.util.stream.Collectors.groupingBy;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+public class StatusLoggerTest extends CQLTester
+{
+ private static final Logger log = LoggerFactory.getLogger(StatusLoggerTest.class);
+
+ @Test
+ public void testStatusLoggerPrintsStatusOnlyOnceWhenInvokedConcurrently() throws Exception
+ {
+ ch.qos.logback.classic.Logger logger = (ch.qos.logback.classic.Logger) LoggerFactory.getLogger(StatusLogger.class);
+ InMemoryAppender inMemoryAppender = new InMemoryAppender();
+ logger.addAppender(inMemoryAppender);
+ logger.setLevel(Level.TRACE);
+ try
+ {
+ submitTwoLogRequestsConcurrently();
+ verifyOnlySingleStatusWasAppendedConcurrently(inMemoryAppender.events);
+ }
+ finally
+ {
+ assertTrue("Could not remove in memory appender", logger.detachAppender(inMemoryAppender));
+ }
+ }
+
+ private void submitTwoLogRequestsConcurrently() throws InterruptedException
+ {
+ ExecutorService executorService = Executors.newFixedThreadPool(2);
+ executorService.submit(StatusLogger::log);
+ executorService.submit(StatusLogger::log);
+ executorService.shutdown();
+ Assert.assertTrue(executorService.awaitTermination(1, TimeUnit.MINUTES));
+ }
+
+ private void verifyOnlySingleStatusWasAppendedConcurrently(List<ILoggingEvent> events)
+ {
+ Map<String, List<ILoggingEvent>> eventsByThread = events.stream().collect(groupingBy(ILoggingEvent::getThreadName));
+ List<String> threadNames = newArrayList(eventsByThread.keySet());
+
+ assertEquals("Expected events from 2 threads only", 2, threadNames.size());
+
+ List<ILoggingEvent> firstThreadEvents = eventsByThread.get(threadNames.get(0));
+ List<ILoggingEvent> secondThreadEvents = eventsByThread.get(threadNames.get(1));
+
+ assertTrue("Expected at least one event from the first thread", firstThreadEvents.size() >= 1);
+ assertTrue("Expected at least one event from the second thread", secondThreadEvents.size() >= 1);
+
+ if (areDisjunctive(firstThreadEvents, secondThreadEvents))
+ {
+ log.debug("Event time ranges are disjunctive - log invocations were made one after another");
+ }
+ else
+ {
+ verifyStatusWasPrintedAndBusyEventOccured(firstThreadEvents, secondThreadEvents);
+ }
+ }
+
+ private boolean areDisjunctive(List<ILoggingEvent> firstThreadEvents, List<ILoggingEvent> secondThreadEvents)
+ {
+ Range<Long> firstThreadTimeRange = timestampsRange(firstThreadEvents);
+ Range<Long> secondThreadTimeRange = timestampsRange(secondThreadEvents);
+ boolean connected = firstThreadTimeRange.isConnected(secondThreadTimeRange);
+ boolean disjunctive = !connected || firstThreadTimeRange.intersection(secondThreadTimeRange).isEmpty();
+ log.debug("Time ranges {}, {}, disjunctive={}", firstThreadTimeRange, secondThreadTimeRange, disjunctive);
+ return disjunctive;
+ }
+
+ private Range<Long> timestampsRange(List<ILoggingEvent> events)
+ {
+ List<Long> timestamps = events.stream().map(ILoggingEvent::getTimeStamp).collect(Collectors.toList());
+ Long min = timestamps.stream().min(Comparator.naturalOrder()).get();
+ Long max = timestamps.stream().max(Comparator.naturalOrder()).get();
+ // It's open on one side to cover a case when second status starts printing at the same timestamp that previous one was finished
+ return Range.closedOpen(min, max);
+ }
+
+ private void verifyStatusWasPrintedAndBusyEventOccured(List<ILoggingEvent> firstThreadEvents, List<ILoggingEvent> secondThreadEvents)
+ {
+ if (firstThreadEvents.size() > 1 && secondThreadEvents.size() > 1)
+ {
+ log.error("Both event lists contain more than one entry. First = {}, Second = {}", firstThreadEvents, secondThreadEvents);
+ fail("More that one status log was appended concurrently");
+ }
+ else if (firstThreadEvents.size() <= 1 && secondThreadEvents.size() <= 1)
+ {
+ log.error("No status log was recorded. First = {}, Second = {}", firstThreadEvents, secondThreadEvents);
+ fail("Status log was not appended");
+ }
+ else
+ {
+ log.info("Checking if logger was busy. First = {}, Second = {}", firstThreadEvents, secondThreadEvents);
+ assertTrue("One 'logger busy' entry was expected",
+ isLoggerBusyTheOnlyEvent(firstThreadEvents) || isLoggerBusyTheOnlyEvent(secondThreadEvents));
+ }
+ }
+
+ private boolean isLoggerBusyTheOnlyEvent(List<ILoggingEvent> events)
+ {
+ return events.size() == 1 &&
+ events.get(0).getMessage().equals("StatusLogger is busy") &&
+ events.get(0).getLevel() == Level.TRACE;
+ }
+
+ private static class InMemoryAppender extends AppenderBase<ILoggingEvent>
+ {
+ private final List<ILoggingEvent> events = newArrayList();
+
+ private InMemoryAppender()
+ {
+ start();
+ }
+
+ @Override
+ protected synchronized void append(ILoggingEvent event)
+ {
+ events.add(event);
+ }
+ }
+}
diff --git a/test/unit/org/apache/cassandra/utils/StreamingHistogramTest.java b/test/unit/org/apache/cassandra/utils/StreamingHistogramTest.java
deleted file mode 100644
index dcb6703..0000000
--- a/test/unit/org/apache/cassandra/utils/StreamingHistogramTest.java
+++ /dev/null
@@ -1,189 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.utils;
-
-import java.util.Iterator;
-import java.util.LinkedHashMap;
-import java.util.Map;
-
-import org.junit.Test;
-
-import org.apache.cassandra.io.util.DataInputBuffer;
-import org.apache.cassandra.io.util.DataOutputBuffer;
-
-import static org.junit.Assert.assertEquals;
-
-public class StreamingHistogramTest
-{
- @Test
- public void testFunction() throws Exception
- {
- StreamingHistogram.StreamingHistogramBuilder histogramBuilder = new StreamingHistogram.StreamingHistogramBuilder(5, 0, 1);
- long[] samples = new long[]{23, 19, 10, 16, 36, 2, 9, 32, 30, 45};
-
- // add 7 points to histogram of 5 bins
- for (int i = 0; i < 7; i++)
- {
- histogramBuilder.update(samples[i]);
- }
-
- // should end up (2,1),(9.5,2),(17.5,2),(23,1),(36,1)
- Map<Double, Long> expected1 = new LinkedHashMap<Double, Long>(5);
- expected1.put(2.0, 1L);
- expected1.put(9.5, 2L);
- expected1.put(17.5, 2L);
- expected1.put(23.0, 1L);
- expected1.put(36.0, 1L);
-
- Iterator<Map.Entry<Double, Long>> expectedItr = expected1.entrySet().iterator();
- for (Map.Entry<Number, long[]> actual : histogramBuilder.build().getAsMap().entrySet())
- {
- Map.Entry<Double, Long> entry = expectedItr.next();
- assertEquals(entry.getKey(), actual.getKey().doubleValue(), 0.01);
- assertEquals(entry.getValue().longValue(), actual.getValue()[0]);
- }
-
- // merge test
- StreamingHistogram.StreamingHistogramBuilder hist2 = new StreamingHistogram.StreamingHistogramBuilder(3, 3, 1);
- for (int i = 7; i < samples.length; i++)
- {
- hist2.update(samples[i]);
- }
- histogramBuilder.merge(hist2.build());
- // should end up (2,1),(9.5,2),(19.33,3),(32.67,3),(45,1)
- Map<Double, Long> expected2 = new LinkedHashMap<Double, Long>(5);
- expected2.put(2.0, 1L);
- expected2.put(9.5, 2L);
- expected2.put(19.33, 3L);
- expected2.put(32.67, 3L);
- expected2.put(45.0, 1L);
- expectedItr = expected2.entrySet().iterator();
- StreamingHistogram hist = histogramBuilder.build();
- for (Map.Entry<Number, long[]> actual : hist.getAsMap().entrySet())
- {
- Map.Entry<Double, Long> entry = expectedItr.next();
- assertEquals(entry.getKey(), actual.getKey().doubleValue(), 0.01);
- assertEquals(entry.getValue().longValue(), actual.getValue()[0]);
- }
-
- // sum test
- assertEquals(3.28, hist.sum(15), 0.01);
- // sum test (b > max(hist))
- assertEquals(10.0, hist.sum(50), 0.01);
- }
-
- @Test
- public void testSerDe() throws Exception
- {
- StreamingHistogram.StreamingHistogramBuilder hist = new StreamingHistogram.StreamingHistogramBuilder(5, 0, 1);
- long[] samples = new long[]{23, 19, 10, 16, 36, 2, 9};
-
- // add 7 points to histogram of 5 bins
- for (int i = 0; i < samples.length; i++)
- {
- hist.update(samples[i]);
- }
-
- DataOutputBuffer out = new DataOutputBuffer();
- StreamingHistogram.serializer.serialize(hist.build(), out);
- byte[] bytes = out.toByteArray();
-
- StreamingHistogram deserialized = StreamingHistogram.serializer.deserialize(new DataInputBuffer(bytes));
-
- // deserialized histogram should have following values
- Map<Double, Long> expected1 = new LinkedHashMap<Double, Long>(5);
- expected1.put(2.0, 1L);
- expected1.put(9.5, 2L);
- expected1.put(17.5, 2L);
- expected1.put(23.0, 1L);
- expected1.put(36.0, 1L);
-
- Iterator<Map.Entry<Double, Long>> expectedItr = expected1.entrySet().iterator();
- for (Map.Entry<Number, long[]> actual : deserialized.getAsMap().entrySet())
- {
- Map.Entry<Double, Long> entry = expectedItr.next();
- assertEquals(entry.getKey(), actual.getKey().doubleValue(), 0.01);
- assertEquals(entry.getValue().longValue(), actual.getValue()[0]);
- }
- }
-
-
- @Test
- public void testNumericTypes() throws Exception
- {
- StreamingHistogram.StreamingHistogramBuilder hist = new StreamingHistogram.StreamingHistogramBuilder(5, 0, 1);
-
- hist.update(2);
- hist.update(2.0);
- hist.update(2L);
-
- Map<Number, long[]> asMap = hist.build().getAsMap();
-
- assertEquals(1, asMap.size());
- assertEquals(3L, asMap.get(2)[0]);
-
- //Make sure it's working with Serde
- DataOutputBuffer out = new DataOutputBuffer();
- StreamingHistogram.serializer.serialize(hist.build(), out);
- byte[] bytes = out.toByteArray();
-
- StreamingHistogram deserialized = StreamingHistogram.serializer.deserialize(new DataInputBuffer(bytes));
-
- StreamingHistogram.StreamingHistogramBuilder hist2Builder = new StreamingHistogram.StreamingHistogramBuilder(5, 0, 1);
- hist2Builder.merge(deserialized);
- hist2Builder.update(2L);
-
- asMap = hist2Builder.build().getAsMap();
- assertEquals(1, asMap.size());
- assertEquals(4L, asMap.get(2)[0]);
- }
-
- @Test
- public void testOverflow() throws Exception
- {
- StreamingHistogram.StreamingHistogramBuilder hist = new StreamingHistogram.StreamingHistogramBuilder(5, 10, 1);
- long[] samples = new long[]{23, 19, 10, 16, 36, 2, 9, 32, 30, 45, 31,
- 32, 32, 33, 34, 35, 70, 78, 80, 90, 100,
- 32, 32, 33, 34, 35, 70, 78, 80, 90, 100
- };
-
- // Hit the spool cap, force it to make bins
- for (int i = 0; i < samples.length; i++)
- {
- hist.update(samples[i]);
- }
- StreamingHistogram histogram = hist.build();
- assertEquals(5, histogram.getAsMap().keySet().size());
-
- }
-
- @Test
- public void testRounding() throws Exception
- {
- StreamingHistogram.StreamingHistogramBuilder hist = new StreamingHistogram.StreamingHistogramBuilder(5, 10, 60);
- long[] samples = new long[] { 59, 60, 119, 180, 181, 300 }; // 60, 60, 120, 180, 240, 300
- for (int i = 0 ; i < samples.length ; i++)
- hist.update(samples[i]);
-
- StreamingHistogram histogram = hist.build();
- assertEquals(histogram.getAsMap().keySet().size(), 5);
- assertEquals(histogram.getAsMap().get(60)[0], 2);
- assertEquals(histogram.getAsMap().get(120)[0], 1);
-
- }
-}
diff --git a/test/unit/org/apache/cassandra/utils/TopKSamplerTest.java b/test/unit/org/apache/cassandra/utils/TopKSamplerTest.java
deleted file mode 100644
index d1fdf47..0000000
--- a/test/unit/org/apache/cassandra/utils/TopKSamplerTest.java
+++ /dev/null
@@ -1,179 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-package org.apache.cassandra.utils;
-
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-import com.google.common.collect.Maps;
-import com.google.common.util.concurrent.Uninterruptibles;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import com.clearspring.analytics.hash.MurmurHash;
-import com.clearspring.analytics.stream.Counter;
-import junit.framework.Assert;
-import org.apache.cassandra.concurrent.NamedThreadFactory;
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.utils.TopKSampler.SamplerResult;
-
-public class TopKSamplerTest
-{
-
- @BeforeClass
- public static void beforeClass()
- {
- DatabaseDescriptor.daemonInitialization();
- }
-
- @Test
- public void testSamplerSingleInsertionsEqualMulti() throws TimeoutException
- {
- TopKSampler<String> sampler = new TopKSampler<String>();
- sampler.beginSampling(10);
- insert(sampler);
- waitForEmpty(1000);
- SamplerResult single = sampler.finishSampling(10);
-
- TopKSampler<String> sampler2 = new TopKSampler<String>();
- sampler2.beginSampling(10);
- for(int i = 1; i <= 10; i++)
- {
- String key = "item" + i;
- sampler2.addSample(key, MurmurHash.hash64(key), i);
- }
- waitForEmpty(1000);
- Assert.assertEquals(countMap(single.topK), countMap(sampler2.finishSampling(10).topK));
- Assert.assertEquals(sampler2.hll.cardinality(), 10);
- Assert.assertEquals(sampler.hll.cardinality(), sampler2.hll.cardinality());
- }
-
- @Test
- public void testSamplerOutOfOrder() throws TimeoutException
- {
- TopKSampler<String> sampler = new TopKSampler<String>();
- sampler.beginSampling(10);
- insert(sampler);
- waitForEmpty(1000);
- SamplerResult single = sampler.finishSampling(10);
- single = sampler.finishSampling(10);
- }
-
- /**
- * checking for exceptions from SS/HLL which are not thread safe
- */
- @Test
- public void testMultithreadedAccess() throws Exception
- {
- final AtomicBoolean running = new AtomicBoolean(true);
- final CountDownLatch latch = new CountDownLatch(1);
- final TopKSampler<String> sampler = new TopKSampler<String>();
-
- NamedThreadFactory.createThread(new Runnable()
- {
- public void run()
- {
- try
- {
- while (running.get())
- {
- insert(sampler);
- }
- } finally
- {
- latch.countDown();
- }
- }
-
- }
- , "inserter").start();
- try
- {
- // start/stop in fast iterations
- for(int i = 0; i<100; i++)
- {
- sampler.beginSampling(i);
- sampler.finishSampling(i);
- }
- // start/stop with pause to let it build up past capacity
- for(int i = 0; i<3; i++)
- {
- sampler.beginSampling(i);
- Thread.sleep(250);
- sampler.finishSampling(i);
- }
-
- // with empty results
- running.set(false);
- latch.await(1, TimeUnit.SECONDS);
- waitForEmpty(1000);
- for(int i = 0; i<10; i++)
- {
- sampler.beginSampling(i);
- Thread.sleep(i);
- sampler.finishSampling(i);
- }
- } finally
- {
- running.set(false);
- }
- }
-
- private void insert(TopKSampler<String> sampler)
- {
- for(int i = 1; i <= 10; i++)
- {
- for(int j = 0; j < i; j++)
- {
- String key = "item" + i;
- sampler.addSample(key, MurmurHash.hash64(key), 1);
- }
- }
- }
-
- private void waitForEmpty(int timeoutMs) throws TimeoutException
- {
- int timeout = 0;
- while (!TopKSampler.samplerExecutor.getQueue().isEmpty())
- {
- timeout++;
- Uninterruptibles.sleepUninterruptibly(100, TimeUnit.MILLISECONDS);
- if (timeout * 100 > timeoutMs)
- {
- throw new TimeoutException("TRACE executor not cleared within timeout");
- }
- }
- }
-
- private <T> Map<T, Long> countMap(List<Counter<T>> target)
- {
- Map<T, Long> counts = Maps.newHashMap();
- for(Counter<T> counter : target)
- {
- counts.put(counter.getItem(), counter.getCount());
- }
- return counts;
- }
-}
diff --git a/test/unit/org/apache/cassandra/utils/UUIDTests.java b/test/unit/org/apache/cassandra/utils/UUIDTests.java
index 0d57c47..a0f55ad 100644
--- a/test/unit/org/apache/cassandra/utils/UUIDTests.java
+++ b/test/unit/org/apache/cassandra/utils/UUIDTests.java
@@ -29,6 +29,7 @@
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
+import org.junit.Assert;
import org.junit.Test;
import com.google.common.collect.Sets;
@@ -135,7 +136,7 @@
es.execute(task);
}
es.shutdown();
- es.awaitTermination(10, TimeUnit.MINUTES);
+ Assert.assertTrue(es.awaitTermination(1, TimeUnit.MINUTES));
assert !failedOrdering.get();
assert !failedDuplicate.get();
diff --git a/test/unit/org/apache/cassandra/utils/asserts/SizeableObjectAssert.java b/test/unit/org/apache/cassandra/utils/asserts/SizeableObjectAssert.java
new file mode 100644
index 0000000..5e36625
--- /dev/null
+++ b/test/unit/org/apache/cassandra/utils/asserts/SizeableObjectAssert.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.utils.asserts;
+
+import org.apache.cassandra.utils.ObjectSizes;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public interface SizeableObjectAssert<SELF extends SizeableObjectAssert<SELF>>
+{
+ Object actual();
+
+ default SELF hasSizeLessThan(double expectedSize)
+ {
+ double measured = (double) ObjectSizes.measureDeep(actual());
+ assertThat(measured)
+ .withFailMessage("Size of measured object [%f] is not less than the expected size [%f]", measured, expectedSize)
+ .isLessThan(expectedSize);
+ return ((SELF) this);
+
+ }
+
+ default SELF hasSizeGreaterThanOrEqual(double expectedSize)
+ {
+ double measured = (double) ObjectSizes.measureDeep(actual());
+ assertThat(measured)
+ .withFailMessage("Size of measured object [%f] is not greater than or equal to the expected size [%f]", measured, expectedSize)
+ .isGreaterThanOrEqualTo(expectedSize);
+ return ((SELF) this);
+ }
+}
diff --git a/test/unit/org/apache/cassandra/utils/asserts/SyncTaskAssert.java b/test/unit/org/apache/cassandra/utils/asserts/SyncTaskAssert.java
new file mode 100644
index 0000000..c480783
--- /dev/null
+++ b/test/unit/org/apache/cassandra/utils/asserts/SyncTaskAssert.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.utils.asserts;
+
+import com.google.common.base.Preconditions;
+
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.repair.LocalSyncTask;
+import org.apache.cassandra.repair.SyncTask;
+import org.assertj.core.api.AbstractObjectAssert;
+import org.assertj.core.api.Assertions;
+
+public class SyncTaskAssert extends AbstractObjectAssert<SyncTaskAssert, SyncTask> implements SizeableObjectAssert<SyncTaskAssert>
+{
+ private SyncTaskAssert(SyncTask syncTask)
+ {
+ super(syncTask, SyncTaskAssert.class);
+ }
+
+ public static SyncTaskAssert assertThat(SyncTask task)
+ {
+ return new SyncTaskAssert(task);
+ }
+
+ @Override
+ public Object actual()
+ {
+ return actual;
+ }
+
+ public SyncTaskAssert hasLocal(boolean expected)
+ {
+ Assertions.assertThat(actual.isLocal()).isEqualTo(expected);
+ return this;
+ }
+
+ public SyncTaskAssert isLocal()
+ {
+ Assertions.assertThat(actual.isLocal()).isTrue();
+ return this;
+ }
+
+ public SyncTaskAssert isNotLocal()
+ {
+ Assertions.assertThat(actual.isLocal()).isFalse();
+ return this;
+ }
+
+ public SyncTaskAssert isRequestRanges()
+ {
+ Preconditions.checkState(actual instanceof LocalSyncTask, "Tested value is not a LocalSyncTask");
+ Assertions.assertThat(((LocalSyncTask) actual).requestRanges).isTrue();
+ return this;
+ }
+
+ public SyncTaskAssert isNotRequestRanges()
+ {
+ Preconditions.checkState(actual instanceof LocalSyncTask, "Tested value is not a LocalSyncTask");
+ Assertions.assertThat(((LocalSyncTask) actual).requestRanges).isFalse();
+ return this;
+ }
+
+ public SyncTaskAssert hasTransferRanges(boolean expected)
+ {
+ Preconditions.checkState(actual instanceof LocalSyncTask, "Tested value is not a LocalSyncTask");
+ Assertions.assertThat(((LocalSyncTask) actual).transferRanges).isEqualTo(expected);
+ return this;
+ }
+
+ @SuppressWarnings("unchecked")
+ public SyncTaskAssert hasRanges(Range... ranges)
+ {
+ Assertions.assertThat(actual.rangesToSync).containsOnly(ranges);
+ return this;
+ }
+}
diff --git a/test/unit/org/apache/cassandra/utils/asserts/SyncTaskListAssert.java b/test/unit/org/apache/cassandra/utils/asserts/SyncTaskListAssert.java
new file mode 100644
index 0000000..668719b
--- /dev/null
+++ b/test/unit/org/apache/cassandra/utils/asserts/SyncTaskListAssert.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.utils.asserts;
+
+import java.util.List;
+
+import com.google.common.collect.ImmutableList;
+
+import org.apache.cassandra.repair.SyncTask;
+import org.assertj.core.api.AbstractListAssert;
+
+public class SyncTaskListAssert extends AbstractListAssert<SyncTaskListAssert, List<SyncTask>, SyncTask, SyncTaskAssert>
+implements SizeableObjectAssert<SyncTaskListAssert>
+{
+ public SyncTaskListAssert(List<SyncTask> syncTasks)
+ {
+ super(syncTasks, SyncTaskListAssert.class);
+ }
+
+ protected SyncTaskAssert toAssert(SyncTask value, String description)
+ {
+ return SyncTaskAssert.assertThat(value);
+ }
+
+ protected SyncTaskListAssert newAbstractIterableAssert(Iterable<? extends SyncTask> iterable)
+ {
+ return assertThat(iterable);
+ }
+
+ public static SyncTaskListAssert assertThat(Iterable<? extends SyncTask> iterable)
+ {
+ return new SyncTaskListAssert(ImmutableList.copyOf(iterable));
+ }
+
+ @Override
+ public Object actual()
+ {
+ return actual;
+ }
+
+ public SyncTaskListAssert areAllInstanceOf(Class<?> type)
+ {
+ actual.forEach(t -> toAssert(t, "").isInstanceOf(type));
+ return this;
+ }
+}
diff --git a/test/unit/org/apache/cassandra/utils/binlog/BinLogTest.java b/test/unit/org/apache/cassandra/utils/binlog/BinLogTest.java
new file mode 100644
index 0000000..dbce6a9
--- /dev/null
+++ b/test/unit/org/apache/cassandra/utils/binlog/BinLogTest.java
@@ -0,0 +1,495 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.utils.binlog;
+
+import java.io.File;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Supplier;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import net.openhft.chronicle.queue.ChronicleQueue;
+import net.openhft.chronicle.queue.ChronicleQueueBuilder;
+import net.openhft.chronicle.queue.ExcerptTailer;
+import net.openhft.chronicle.queue.RollCycles;
+import net.openhft.chronicle.wire.WireOut;
+import org.apache.cassandra.Util;
+import org.apache.cassandra.io.util.FileUtils;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+public class BinLogTest
+{
+ public static Path tempDir() throws Exception
+ {
+ File f = FileUtils.createTempFile("foo", "bar");
+ f.delete();
+ f.mkdir();
+ return Paths.get(f.getPath());
+ }
+
+ private static final String testString = "ry@nlikestheyankees";
+ private static final String testString2 = testString + "1";
+
+ private BinLog binLog;
+ private Path path;
+
+ @Before
+ public void setUp() throws Exception
+ {
+ path = tempDir();
+ binLog = new BinLog.Builder().path(path)
+ .rollCycle(RollCycles.TEST_SECONDLY.toString())
+ .maxQueueWeight(10)
+ .maxLogSize(1024 * 1024 * 128)
+ .blocking(false)
+ .build(false);
+ }
+
+ @After
+ public void tearDown() throws Exception
+ {
+ if (binLog != null)
+ {
+ binLog.stop();
+ }
+ for (File f : path.toFile().listFiles())
+ {
+ f.delete();
+ }
+ }
+
+ @Test(expected = NullPointerException.class)
+ public void testConstructorNullPath() throws Exception
+ {
+ new BinLog.Builder().path(null).build(false);
+ }
+
+ @Test(expected = NullPointerException.class)
+ public void testConstructorNullRollCycle() throws Exception
+ {
+ new BinLog.Builder().path(tempDir()).rollCycle(null).build(false);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testConstructorZeroWeight() throws Exception
+ {
+ new BinLog.Builder().path(tempDir()).rollCycle(RollCycles.TEST_SECONDLY.toString()).maxQueueWeight(0).build(false);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testConstructorLogSize() throws Exception
+ {
+ new BinLog.Builder().path(tempDir()).rollCycle(RollCycles.TEST_SECONDLY.toString()).maxLogSize(0).build(false);
+ }
+
+ /**
+ * Check that we can start and stop the bin log and that it releases resources held by any subsequent appended
+ * records
+ */
+ @Test
+ public void testBinLogStartStop() throws Exception
+ {
+ AtomicInteger releaseCount = new AtomicInteger();
+ CountDownLatch ready = new CountDownLatch(2);
+ Supplier<BinLog.ReleaseableWriteMarshallable> recordSupplier =
+ () -> new BinLog.ReleaseableWriteMarshallable()
+ {
+ public void release()
+ {
+ releaseCount.incrementAndGet();
+ }
+
+ protected long version()
+ {
+ return 0;
+ }
+
+ protected String type()
+ {
+ return "test";
+ }
+
+ public void writeMarshallablePayload(WireOut wire)
+ {
+ ready.countDown();
+ }
+ };
+ binLog.put(recordSupplier.get());
+ binLog.put(recordSupplier.get());
+ ready.await(1, TimeUnit.MINUTES);
+ Util.spinAssertEquals("Both records should be released", 2, releaseCount::get, 10, TimeUnit.SECONDS);
+ Thread t = new Thread(() -> {
+ try
+ {
+ binLog.stop();
+ }
+ catch (InterruptedException e)
+ {
+ throw new AssertionError(e);
+ }
+ });
+ t.start();
+ t.join(60 * 1000);
+ assertEquals("BinLog should not take more than 1 minute to stop", t.getState(), Thread.State.TERMINATED);
+
+ Util.spinAssertEquals(2, releaseCount::get, 60);
+ Util.spinAssertEquals(Thread.State.TERMINATED, binLog.binLogThread::getState, 60);
+ }
+
+ /**
+ * Check that the finalizer releases any stragglers in the queue
+ */
+ @Test
+ public void testBinLogFinalizer() throws Exception
+ {
+ binLog.stop();
+ Semaphore released = new Semaphore(0);
+ binLog.sampleQueue.put(new BinLog.ReleaseableWriteMarshallable()
+ {
+ public void release()
+ {
+ released.release();
+ }
+
+ protected long version()
+ {
+ return 0;
+ }
+
+ protected String type()
+ {
+ return "test";
+ }
+
+ public void writeMarshallablePayload(WireOut wire)
+ {
+
+ }
+ });
+ binLog = null;
+
+ for (int ii = 0; ii < 30; ii++)
+ {
+ System.gc();
+ System.runFinalization();
+ Thread.sleep(100);
+ if (released.tryAcquire())
+ return;
+ }
+ fail("Finalizer never released resources");
+ }
+
+ /**
+ * Test that put blocks and unblocks and creates records
+ */
+ @Test
+ public void testPut() throws Exception
+ {
+ binLog.put(record(testString));
+ binLog.put(record(testString2));
+
+ Util.spinAssertEquals(2, () -> readBinLogRecords(path).size(), 60);
+ List<String> records = readBinLogRecords(path);
+ assertEquals(testString, records.get(0));
+ assertEquals(testString2, records.get(1));
+
+
+ //Prevent the bin log thread from making progress
+ Semaphore blockBinLog = new Semaphore(0);
+ //Get notified when the bin log thread has blocked and definitely won't batch drain tasks
+ Semaphore binLogBlocked = new Semaphore(0);
+ try
+ {
+ binLog.put(new BinLog.ReleaseableWriteMarshallable()
+ {
+ public void release()
+ {
+ }
+
+ protected long version()
+ {
+ return 0;
+ }
+
+ protected String type()
+ {
+ return "test";
+ }
+
+ public void writeMarshallablePayload(WireOut wire)
+ {
+ //Notify the bing log thread is about to block
+ binLogBlocked.release();
+ try
+ {
+ //Block the bin log thread so it doesn't process more tasks
+ blockBinLog.acquire();
+ }
+ catch (InterruptedException e)
+ {
+ throw new RuntimeException(e);
+ }
+ }
+ });
+
+ //Wait for the bin log thread to block so it doesn't batch drain
+ Util.spinAssertEquals(true, binLogBlocked::tryAcquire, 60);
+
+ //Now fill the queue up to capacity and it shouldn't block
+ for (int ii = 0; ii < 10; ii++)
+ {
+ binLog.put(record(testString));
+ }
+
+ //Thread to block on the full queue
+ Thread t = new Thread(() ->
+ {
+ try
+ {
+ binLog.put(record(testString));
+ //Should be able to do it again after unblocking
+ binLog.put(record(testString));
+ }
+ catch (InterruptedException e)
+ {
+ throw new AssertionError(e);
+ }
+ });
+ t.start();
+ Thread.sleep(500);
+ //If the thread is not terminated then it is probably blocked on the queue
+ assertTrue(t.getState() != Thread.State.TERMINATED);
+ }
+ finally
+ {
+ blockBinLog.release();
+ }
+
+ //Expect all the records to eventually be there including one from the blocked thread
+ Util.spinAssertEquals(15, () -> readBinLogRecords(path).size(), 60);
+ }
+
+ @Test
+ public void testOffer() throws Exception
+ {
+ assertTrue(binLog.offer(record(testString)));
+ assertTrue(binLog.offer(record(testString2)));
+
+ Util.spinAssertEquals(2, () -> readBinLogRecords(path).size(), 60);
+ List<String> records = readBinLogRecords(path);
+ assertEquals(testString, records.get(0));
+ assertEquals(testString2, records.get(1));
+
+ //Prevent the bin log thread from making progress
+ Semaphore blockBinLog = new Semaphore(0);
+ //Get notified when the bin log thread has blocked and definitely won't batch drain tasks
+ Semaphore binLogBlocked = new Semaphore(0);
+ try
+ {
+ assertTrue(binLog.offer(new BinLog.ReleaseableWriteMarshallable()
+ {
+ public void release()
+ {
+ }
+
+ protected long version()
+ {
+ return 0;
+ }
+
+ protected String type()
+ {
+ return "test";
+ }
+
+ public void writeMarshallablePayload(WireOut wire)
+ {
+ //Notify the bing log thread is about to block
+ binLogBlocked.release();
+ try
+ {
+ //Block the bin log thread so it doesn't process more tasks
+ blockBinLog.acquire();
+ }
+ catch (InterruptedException e)
+ {
+ throw new RuntimeException(e);
+ }
+ }
+ }));
+
+ //Wait for the bin log thread to block so it doesn't batch drain
+ Util.spinAssertEquals(true, binLogBlocked::tryAcquire, 60);
+
+ //Now fill the queue up to capacity and it should always accept
+ for (int ii = 0; ii < 10; ii++)
+ {
+ assertTrue(binLog.offer(record(testString)));
+ }
+
+ //it shoudl reject this record since it is full
+ assertFalse(binLog.offer(record(testString)));
+ }
+ finally
+ {
+ blockBinLog.release();
+ }
+ Util.spinAssertEquals(13, () -> readBinLogRecords(path).size(), 60);
+ assertTrue(binLog.offer(record(testString)));
+ Util.spinAssertEquals(14, () -> readBinLogRecords(path).size(), 60);
+ }
+
+ /**
+ * Set a very small segment size so on rolling the segments are always deleted
+ */
+ @Test
+ public void testCleanupOnOversize() throws Exception
+ {
+ tearDown();
+ binLog = new BinLog.Builder().path(path).rollCycle(RollCycles.TEST_SECONDLY.toString()).maxQueueWeight(1).maxLogSize(10000).blocking(false).build(false);
+ for (int ii = 0; ii < 5; ii++)
+ {
+ binLog.put(record(String.valueOf(ii)));
+ Thread.sleep(1001);
+ }
+ List<String> records = readBinLogRecords(path);
+ System.out.println("Records found are " + records);
+ assertTrue(records.size() < 5);
+ }
+
+ @Test(expected = IllegalStateException.class)
+ public void testNoReuse() throws Exception
+ {
+ binLog.stop();
+ binLog.start();
+ }
+
+ @Test
+ public void testOfferAfterStop() throws Exception
+ {
+ binLog.stop();
+ assertFalse(binLog.offer(record(testString)));
+ }
+
+ @Test
+ public void testPutAfterStop() throws Exception
+ {
+ final BinLog.ReleaseableWriteMarshallable unexpected = record(testString);
+ binLog.stop();
+ binLog.put(unexpected);
+ BinLog.ReleaseableWriteMarshallable record;
+ while (null != (record = binLog.sampleQueue.poll()))
+ {
+ assertNotEquals("A stopped BinLog should no longer accept", unexpected, record);
+ }
+ }
+
+ /**
+ * Test for a bug where files were deleted but the space was not reclaimed when tracking so
+ * all log segemnts were incorrectly deleted when rolled.
+ */
+ @Test
+ public void testTrucationReleasesLogSpace() throws Exception
+ {
+ StringBuilder sb = new StringBuilder();
+ for (int ii = 0; ii < 1024 * 1024 * 2; ii++)
+ {
+ sb.append('a');
+ }
+
+ String queryString = sb.toString();
+
+ //This should fill up the log so when it rolls in the future it will always delete the rolled segment;
+ for (int ii = 0; ii < 129; ii++)
+ {
+ binLog.put(record(queryString));
+ }
+
+ for (int ii = 0; ii < 2; ii++)
+ {
+ Thread.sleep(2000);
+ binLog.put(record(queryString));
+ }
+
+ Util.spinAssertEquals(2, () -> readBinLogRecords(path).size(), 60);
+ }
+
+ static BinLog.ReleaseableWriteMarshallable record(String text)
+ {
+ return new BinLog.ReleaseableWriteMarshallable()
+ {
+ public void release()
+ {
+ //Do nothing
+ }
+
+ protected long version()
+ {
+ return 0;
+ }
+
+ protected String type()
+ {
+ return "test";
+ }
+
+ public void writeMarshallablePayload(WireOut wire)
+ {
+ wire.write("text").text(text);
+ }
+ };
+ }
+
+ List<String> readBinLogRecords(Path path)
+ {
+ List<String> records = new ArrayList<String>();
+ try (ChronicleQueue queue = ChronicleQueueBuilder.single(path.toFile()).rollCycle(RollCycles.TEST_SECONDLY).build())
+ {
+ ExcerptTailer tailer = queue.createTailer();
+ while (true)
+ {
+ if (!tailer.readDocument(wire ->
+ {
+ records.add(wire.read("text").text());
+ }))
+ {
+ return records;
+ }
+ }
+ }
+ catch (Exception e)
+ {
+ throw new RuntimeException(e);
+ }
+ }
+}
diff --git a/test/unit/org/apache/cassandra/utils/binlog/DeletingArchiverTest.java b/test/unit/org/apache/cassandra/utils/binlog/DeletingArchiverTest.java
new file mode 100644
index 0000000..cd6b7a3
--- /dev/null
+++ b/test/unit/org/apache/cassandra/utils/binlog/DeletingArchiverTest.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.utils.binlog;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class DeletingArchiverTest
+{
+ @Test
+ public void testDelete() throws IOException
+ {
+ DeletingArchiver da = new DeletingArchiver(45);
+ List<File> files = generateFiles(10, 5);
+ for (File f : files)
+ da.onReleased(1, f);
+ // adding 5 files, each with size 10, this means the first one should have been deleted:
+ assertFalse(files.get(0).exists());
+ for (int i = 1; i < files.size(); i++)
+ assertTrue(files.get(i).exists());
+ assertEquals(40, da.getBytesInStoreFiles());
+ }
+
+ @Test
+ public void testArchiverBigFile() throws IOException
+ {
+ DeletingArchiver da = new DeletingArchiver(45);
+ List<File> largeFiles = generateFiles(50, 1);
+ da.onReleased(1, largeFiles.get(0));
+ assertFalse(largeFiles.get(0).exists());
+ assertEquals(0, da.getBytesInStoreFiles());
+ }
+
+ @Test
+ public void testArchiverSizeTracking() throws IOException
+ {
+ DeletingArchiver da = new DeletingArchiver(45);
+ List<File> smallFiles = generateFiles(10, 4);
+ List<File> largeFiles = generateFiles(40, 1);
+
+ for (File f : smallFiles)
+ {
+ da.onReleased(1, f);
+ }
+ assertEquals(40, da.getBytesInStoreFiles());
+ // we now have 40 bytes in deleting archiver, adding the large 40 byte file should delete all the small ones
+ da.onReleased(1, largeFiles.get(0));
+ for (File f : smallFiles)
+ assertFalse(f.exists());
+
+ smallFiles = generateFiles(10, 4);
+
+ // make sure that size tracking is ok - all 4 new small files should still be there and the large one should be gone
+ for (File f : smallFiles)
+ da.onReleased(1, f);
+
+ assertFalse(largeFiles.get(0).exists());
+ for (File f : smallFiles)
+ assertTrue(f.exists());
+ assertEquals(40, da.getBytesInStoreFiles());
+ }
+
+
+ private List<File> generateFiles(int size, int count) throws IOException
+ {
+ Random r = new Random();
+ List<File> files = new ArrayList<>(count);
+ byte [] content = new byte[size];
+ r.nextBytes(content);
+
+ for (int i = 0; i < count; i++)
+ {
+ Path p = Files.createTempFile("logfile", ".cq4");
+ Files.write(p, content);
+ files.add(p.toFile());
+ }
+ files.forEach(File::deleteOnExit);
+ return files;
+ }
+}
diff --git a/test/unit/org/apache/cassandra/utils/binlog/ExternalArchiverTest.java b/test/unit/org/apache/cassandra/utils/binlog/ExternalArchiverTest.java
new file mode 100644
index 0000000..284ff5a
--- /dev/null
+++ b/test/unit/org/apache/cassandra/utils/binlog/ExternalArchiverTest.java
@@ -0,0 +1,268 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.utils.binlog;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.attribute.PosixFilePermission;
+import java.nio.file.attribute.PosixFilePermissions;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import com.google.common.collect.Sets;
+import org.junit.Test;
+
+import net.openhft.chronicle.queue.impl.single.SingleChronicleQueue;
+import org.apache.cassandra.utils.Pair;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class ExternalArchiverTest
+{
+ @Test
+ public void testArchiver() throws IOException, InterruptedException
+ {
+ Pair<String, String> s = createScript();
+ String script = s.left;
+ String dir = s.right;
+ Path logdirectory = Files.createTempDirectory("logdirectory");
+ File logfileToArchive = Files.createTempFile(logdirectory, "logfile", "xyz").toFile();
+ Files.write(logfileToArchive.toPath(), "content".getBytes());
+
+ ExternalArchiver ea = new ExternalArchiver(script+" %path", null, 10);
+ ea.onReleased(1, logfileToArchive);
+ while (logfileToArchive.exists())
+ {
+ Thread.sleep(100);
+ }
+
+ File movedFile = new File(dir, logfileToArchive.getName());
+ assertTrue(movedFile.exists());
+ movedFile.deleteOnExit();
+ ea.stop();
+ assertEquals(0, logdirectory.toFile().listFiles().length);
+ }
+
+ @Test
+ public void testArchiveExisting() throws IOException, InterruptedException
+ {
+ Pair<String, String> s = createScript();
+ String script = s.left;
+ String moveDir = s.right;
+ List<File> existingFiles = new ArrayList<>();
+ Path dir = Files.createTempDirectory("archive");
+ for (int i = 0; i < 10; i++)
+ {
+ File logfileToArchive = Files.createTempFile(dir, "logfile", SingleChronicleQueue.SUFFIX).toFile();
+ logfileToArchive.deleteOnExit();
+ Files.write(logfileToArchive.toPath(), ("content"+i).getBytes());
+ existingFiles.add(logfileToArchive);
+ }
+
+ ExternalArchiver ea = new ExternalArchiver(script + " %path", dir, 10);
+ boolean allGone = false;
+ while (!allGone)
+ {
+ allGone = true;
+ for (File f : existingFiles)
+ {
+ if (f.exists())
+ {
+ allGone = false;
+ Thread.sleep(100);
+ break;
+ }
+ File movedFile = new File(moveDir, f.getName());
+ assertTrue(movedFile.exists());
+ movedFile.deleteOnExit();
+ }
+ }
+ ea.stop();
+ assertEquals(0, dir.toFile().listFiles().length);
+ }
+
+ @Test
+ public void testArchiveOnShutdown() throws IOException, InterruptedException
+ {
+ Pair<String, String> s = createScript();
+ String script = s.left;
+ String moveDir = s.right;
+ Path dir = Files.createTempDirectory("archive");
+ ExternalArchiver ea = new ExternalArchiver(script + " %path", dir, 10);
+ List<File> existingFiles = new ArrayList<>();
+ for (int i = 0; i < 10; i++)
+ {
+ File logfileToArchive = Files.createTempFile(dir, "logfile", SingleChronicleQueue.SUFFIX).toFile();
+ logfileToArchive.deleteOnExit();
+ Files.write(logfileToArchive.toPath(), ("content"+i).getBytes());
+ existingFiles.add(logfileToArchive);
+ }
+ // ea.stop will archive all .cq4 files in the directory
+ ea.stop();
+ for (File f : existingFiles)
+ {
+ assertFalse(f.exists());
+ File movedFile = new File(moveDir, f.getName());
+ assertTrue(movedFile.exists());
+ movedFile.deleteOnExit();
+ }
+ }
+
+ /**
+ * Make sure retries work
+ * 1. create a script that will fail two times before executing the command
+ * 2. create an ExternalArchiver that retries two times (this means we execute the script 3 times, meaning the last one will be successful)
+ * 3. make sure the file is on disk until the script has been executed 3 times
+ * 4. make sure the file is gone and that the command was executed successfully
+ */
+ @Test
+ public void testRetries() throws IOException, InterruptedException
+ {
+ Pair<String, String> s = createFailingScript(2);
+ String script = s.left;
+ String moveDir = s.right;
+ Path logdirectory = Files.createTempDirectory("logdirectory");
+ File logfileToArchive = Files.createTempFile(logdirectory, "logfile", "xyz").toFile();
+ Files.write(logfileToArchive.toPath(), "content".getBytes());
+ AtomicInteger tryCounter = new AtomicInteger();
+ AtomicBoolean success = new AtomicBoolean();
+ ExternalArchiver ea = new ExternalArchiver(script + " %path", null, 1000, 2, (cmd) ->
+ {
+ tryCounter.incrementAndGet();
+ ExternalArchiver.exec(cmd);
+ success.set(true);
+ });
+ ea.onReleased(0, logfileToArchive);
+ while (tryCounter.get() < 2) // while we have only executed this 0 or 1 times, the file should still be on disk
+ {
+ Thread.sleep(100);
+ assertTrue(logfileToArchive.exists());
+ }
+
+ while (!success.get())
+ Thread.sleep(100);
+
+ // there will be 3 attempts in total, 2 failing ones, then the successful one:
+ assertEquals(3, tryCounter.get());
+ assertFalse(logfileToArchive.exists());
+ File movedFile = new File(moveDir, logfileToArchive.getName());
+ assertTrue(movedFile.exists());
+ ea.stop();
+ }
+
+
+ /**
+ * Makes sure that max retries is honored
+ *
+ * 1. create a script that will fail 3 times before actually executing the command
+ * 2. create an external archiver that retries 2 times (this means that the script will get executed 3 times)
+ * 3. make sure the file is still on disk and that we have not successfully executed the script
+ *
+ */
+ @Test
+ public void testMaxRetries() throws IOException, InterruptedException
+ {
+ Pair<String, String> s = createFailingScript(3);
+ String script = s.left;
+ String moveDir = s.right;
+ Path logdirectory = Files.createTempDirectory("logdirectory");
+ File logfileToArchive = Files.createTempFile(logdirectory, "logfile", "xyz").toFile();
+ Files.write(logfileToArchive.toPath(), "content".getBytes());
+
+ AtomicInteger tryCounter = new AtomicInteger();
+ AtomicBoolean success = new AtomicBoolean();
+ ExternalArchiver ea = new ExternalArchiver(script + " %path", null, 1000, 2, (cmd) ->
+ {
+ try
+ {
+ ExternalArchiver.exec(cmd);
+ success.set(true);
+ }
+ catch (Throwable t)
+ {
+ tryCounter.incrementAndGet();
+ throw t;
+ }
+ });
+ ea.onReleased(0, logfileToArchive);
+ while (tryCounter.get() < 3)
+ Thread.sleep(500);
+ assertTrue(logfileToArchive.exists());
+ // and the file should not get moved:
+ Thread.sleep(5000);
+ assertTrue(logfileToArchive.exists());
+ assertFalse(success.get());
+ File [] fs = new File(moveDir).listFiles(f ->
+ {
+ if (f.getName().startsWith("file."))
+ {
+ f.deleteOnExit();
+ return true;
+ }
+ throw new AssertionError("There should be no other files in the directory");
+ });
+ assertEquals(3, fs.length); // maxRetries + the first try
+ ea.stop();
+ }
+
+
+ private Pair<String, String> createScript() throws IOException
+ {
+ File f = Files.createTempFile("script", "", PosixFilePermissions.asFileAttribute(Sets.newHashSet(PosixFilePermission.OWNER_WRITE,
+ PosixFilePermission.OWNER_READ,
+ PosixFilePermission.OWNER_EXECUTE))).toFile();
+ f.deleteOnExit();
+ File dir = Files.createTempDirectory("archive").toFile();
+ dir.deleteOnExit();
+ String script = "#!/bin/sh\nmv $1 "+dir.getAbsolutePath();
+ Files.write(f.toPath(), script.getBytes());
+ return Pair.create(f.getAbsolutePath(), dir.getAbsolutePath());
+ }
+
+ private Pair<String, String> createFailingScript(int failures) throws IOException
+ {
+ File f = Files.createTempFile("script", "", PosixFilePermissions.asFileAttribute(Sets.newHashSet(PosixFilePermission.OWNER_WRITE,
+ PosixFilePermission.OWNER_READ,
+ PosixFilePermission.OWNER_EXECUTE))).toFile();
+ f.deleteOnExit();
+ File dir = Files.createTempDirectory("archive").toFile();
+ dir.deleteOnExit();
+ // this script counts files in dir.getAbsolutePath, then if there are more than failures files in there, it moves the actual file
+ String script = "#!/bin/bash%n" +
+ "DIR=%s%n" +
+ "shopt -s nullglob%n" +
+ "numfiles=($DIR/*)%n" +
+ "numfiles=${#numfiles[@]}%n" +
+ "if (( $numfiles < %d )); then%n" +
+ " mktemp $DIR/file.XXXXX%n" +
+ " exit 1%n" +
+ "else%n" +
+ " mv $1 $DIR%n"+
+ "fi%n";
+
+ Files.write(f.toPath(), String.format(script, dir.getAbsolutePath(), failures).getBytes());
+ return Pair.create(f.getAbsolutePath(), dir.getAbsolutePath());
+ }
+}
diff --git a/test/unit/org/apache/cassandra/utils/btree/BTreeSearchIteratorTest.java b/test/unit/org/apache/cassandra/utils/btree/BTreeSearchIteratorTest.java
new file mode 100644
index 0000000..69ca93c
--- /dev/null
+++ b/test/unit/org/apache/cassandra/utils/btree/BTreeSearchIteratorTest.java
@@ -0,0 +1,241 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.utils.btree;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.NoSuchElementException;
+
+import com.google.common.collect.Iterables;
+
+import org.apache.cassandra.utils.btree.BTree.Dir;
+import org.junit.Test;
+
+public class BTreeSearchIteratorTest
+{
+
+ private static List<Integer> seq(int count)
+ {
+ return seq(count, 0, 1);
+ }
+
+ private static List<Integer> seq(int count, int base, int multi)
+ {
+ List<Integer> r = new ArrayList<>();
+ for (int i = 0 ; i < count ; i++)
+ r.add(i * multi + base);
+ return r;
+ }
+
+ private static final Comparator<Integer> CMP = new Comparator<Integer>()
+ {
+ public int compare(Integer o1, Integer o2)
+ {
+ return Integer.compare(o1, o2);
+ }
+ };
+
+ private static void assertIteratorExceptionBegin(final BTreeSearchIterator<Integer, Integer> iter)
+ {
+ try
+ {
+ iter.current();
+ fail("Should throw NoSuchElementException");
+ }
+ catch (NoSuchElementException ex)
+ {
+ }
+ try
+ {
+ iter.indexOfCurrent();
+ fail("Should throw NoSuchElementException");
+ }
+ catch (NoSuchElementException ex)
+ {
+ }
+ }
+
+ private static void assertIteratorExceptionEnd(final BTreeSearchIterator<Integer, Integer> iter)
+ {
+ assertFalse(iter.hasNext());
+ try
+ {
+ iter.next();
+ fail("Should throw NoSuchElementException");
+ }
+ catch (NoSuchElementException ex)
+ {
+ }
+ }
+
+ private static void assertBTreeSearchIteratorEquals(final BTreeSearchIterator<Integer, Integer> iter1,
+ final BTreeSearchIterator<Integer, Integer> iter2)
+ {
+ assertIteratorExceptionBegin(iter1);
+ assertIteratorExceptionBegin(iter2);
+ while (iter1.hasNext())
+ {
+ assertTrue(iter2.hasNext());
+ assertEquals(iter1.next(), iter2.next());
+ assertEquals(iter1.current(), iter2.current());
+ assertEquals(iter1.indexOfCurrent(), iter2.indexOfCurrent());
+ }
+ assertIteratorExceptionEnd(iter1);
+ assertIteratorExceptionEnd(iter2);
+ }
+
+ private static void assertBTreeSearchIteratorEquals(final BTreeSearchIterator<Integer, Integer> iter1,
+ final BTreeSearchIterator<Integer, Integer> iter2,
+ int... targets)
+ {
+ assertIteratorExceptionBegin(iter1);
+ assertIteratorExceptionBegin(iter2);
+ for (int i : targets)
+ {
+ Integer val1 = iter1.next(i);
+ Integer val2 = iter2.next(i);
+ assertEquals(val1, val2);
+ if (val1 != null)
+ {
+ assertEquals(iter1.current(), iter2.current());
+ assertEquals(iter1.indexOfCurrent(), iter2.indexOfCurrent());
+ }
+ }
+
+ while (iter1.hasNext())
+ {
+ assertTrue(iter2.hasNext());
+ assertEquals(iter1.next(), iter2.next());
+ assertEquals(iter1.current(), iter2.current());
+ assertEquals(iter1.indexOfCurrent(), iter2.indexOfCurrent());
+ }
+ assertIteratorExceptionEnd(iter1);
+ assertIteratorExceptionEnd(iter2);
+ }
+
+ @Test
+ public void testTreeIteratorNormal()
+ {
+ Object[] btree = BTree.build(seq(30), UpdateFunction.noOp());
+ BTreeSearchIterator fullIter = new FullBTreeSearchIterator<>(btree, CMP, Dir.ASC);
+ BTreeSearchIterator leafIter = new LeafBTreeSearchIterator<>(btree, CMP, Dir.ASC);
+ assertBTreeSearchIteratorEquals(fullIter, leafIter);
+ fullIter = new FullBTreeSearchIterator(btree, CMP, Dir.ASC);
+ leafIter = new LeafBTreeSearchIterator(btree, CMP, Dir.ASC);
+ assertBTreeSearchIteratorEquals(fullIter, leafIter, -8);
+ fullIter = new FullBTreeSearchIterator(btree, CMP, Dir.ASC);
+ leafIter = new LeafBTreeSearchIterator(btree, CMP, Dir.ASC);
+ assertBTreeSearchIteratorEquals(fullIter, leafIter, 100);
+ fullIter = new FullBTreeSearchIterator(btree, CMP, Dir.ASC);
+ leafIter = new LeafBTreeSearchIterator(btree, CMP, Dir.ASC);
+ assertBTreeSearchIteratorEquals(fullIter, leafIter, 3, 4);
+ fullIter = new FullBTreeSearchIterator(btree, CMP, Dir.ASC);
+ leafIter = new LeafBTreeSearchIterator(btree, CMP, Dir.ASC);
+ assertBTreeSearchIteratorEquals(fullIter, leafIter, 4, 3);
+ fullIter = new FullBTreeSearchIterator(btree, CMP, Dir.ASC);
+ leafIter = new LeafBTreeSearchIterator(btree, CMP, Dir.ASC);
+ assertBTreeSearchIteratorEquals(fullIter, leafIter, -8, 3, 100);
+ fullIter = new FullBTreeSearchIterator(btree, CMP, Dir.ASC);
+ leafIter = new LeafBTreeSearchIterator(btree, CMP, Dir.ASC);
+ assertBTreeSearchIteratorEquals(fullIter, leafIter, 0, 29, 30, 0);
+
+ fullIter = new FullBTreeSearchIterator(btree, CMP, Dir.DESC);
+ leafIter = new LeafBTreeSearchIterator(btree, CMP, Dir.DESC);
+ assertBTreeSearchIteratorEquals(fullIter, leafIter);
+ fullIter = new FullBTreeSearchIterator(btree, CMP, Dir.DESC);
+ leafIter = new LeafBTreeSearchIterator(btree, CMP, Dir.DESC);
+ assertBTreeSearchIteratorEquals(fullIter, leafIter, 100);
+ fullIter = new FullBTreeSearchIterator(btree, CMP, Dir.DESC);
+ leafIter = new LeafBTreeSearchIterator(btree, CMP, Dir.DESC);
+ assertBTreeSearchIteratorEquals(fullIter, leafIter, -8);
+ fullIter = new FullBTreeSearchIterator(btree, CMP, Dir.DESC);
+ leafIter = new LeafBTreeSearchIterator(btree, CMP, Dir.DESC);
+ assertBTreeSearchIteratorEquals(fullIter, leafIter, 4, 3);
+ fullIter = new FullBTreeSearchIterator(btree, CMP, Dir.DESC);
+ leafIter = new LeafBTreeSearchIterator(btree, CMP, Dir.DESC);
+ assertBTreeSearchIteratorEquals(fullIter, leafIter, 100, 3, -8);
+ }
+
+ @Test
+ public void testTreeIteratorOneElem()
+ {
+ Object[] btree = BTree.build(seq(1), UpdateFunction.noOp());
+ BTreeSearchIterator fullIter = new FullBTreeSearchIterator(btree, CMP, Dir.ASC);
+ BTreeSearchIterator leafIter = new LeafBTreeSearchIterator(btree, CMP, Dir.ASC);
+ assertBTreeSearchIteratorEquals(fullIter, leafIter);
+ fullIter = new FullBTreeSearchIterator(btree, CMP, Dir.ASC);
+ leafIter = new LeafBTreeSearchIterator(btree, CMP, Dir.ASC);
+ assertBTreeSearchIteratorEquals(fullIter, leafIter, 0);
+ fullIter = new FullBTreeSearchIterator(btree, CMP, Dir.ASC);
+ leafIter = new LeafBTreeSearchIterator(btree, CMP, Dir.ASC);
+ assertBTreeSearchIteratorEquals(fullIter, leafIter, 3);
+ fullIter = new FullBTreeSearchIterator(btree, CMP, Dir.ASC);
+ leafIter = new LeafBTreeSearchIterator(btree, CMP, Dir.ASC);
+
+ fullIter = new FullBTreeSearchIterator(btree, CMP, Dir.DESC);
+ leafIter = new LeafBTreeSearchIterator(btree, CMP, Dir.DESC);
+ assertBTreeSearchIteratorEquals(fullIter, leafIter);
+ fullIter = new FullBTreeSearchIterator(btree, CMP, Dir.DESC);
+ leafIter = new LeafBTreeSearchIterator(btree, CMP, Dir.DESC);
+ assertBTreeSearchIteratorEquals(fullIter, leafIter, 0);
+ fullIter = new FullBTreeSearchIterator(btree, CMP, Dir.DESC);
+ leafIter = new LeafBTreeSearchIterator(btree, CMP, Dir.DESC);
+ assertBTreeSearchIteratorEquals(fullIter, leafIter, 3);
+ }
+
+ @Test
+ public void testTreeIteratorEmpty()
+ {
+ BTreeSearchIterator leafIter = new LeafBTreeSearchIterator(BTree.empty(), CMP, Dir.ASC);
+ assertFalse(leafIter.hasNext());
+ leafIter = new LeafBTreeSearchIterator(BTree.empty(), CMP, Dir.DESC);
+ assertFalse(leafIter.hasNext());
+ }
+
+ @Test
+ public void testTreeIteratorNotFound()
+ {
+ Object[] btree = BTree.build(seq(31, 0, 3), UpdateFunction.noOp());
+ BTreeSearchIterator fullIter = new FullBTreeSearchIterator(btree, CMP, Dir.ASC);
+ BTreeSearchIterator leafIter = new LeafBTreeSearchIterator(btree, CMP, Dir.ASC);
+ assertBTreeSearchIteratorEquals(fullIter, leafIter, 3 * 5 + 1);
+ fullIter = new FullBTreeSearchIterator(btree, CMP, Dir.ASC);
+ leafIter = new LeafBTreeSearchIterator(btree, CMP, Dir.ASC);
+ assertBTreeSearchIteratorEquals(fullIter, leafIter, 3 * 5 + 1, 3 * 7);
+ fullIter = new FullBTreeSearchIterator(btree, CMP, Dir.ASC);
+ leafIter = new LeafBTreeSearchIterator(btree, CMP, Dir.ASC);
+ assertBTreeSearchIteratorEquals(fullIter, leafIter, 3 * 5 + 1, 3 * 7 + 1);
+
+ fullIter = new FullBTreeSearchIterator(btree, CMP, Dir.DESC);
+ leafIter = new LeafBTreeSearchIterator(btree, CMP, Dir.DESC);
+ assertBTreeSearchIteratorEquals(fullIter, leafIter, 3 * 5 + 1);
+ fullIter = new FullBTreeSearchIterator(btree, CMP, Dir.DESC);
+ leafIter = new LeafBTreeSearchIterator(btree, CMP, Dir.DESC);
+ assertBTreeSearchIteratorEquals(fullIter, leafIter, 3 * 5 + 1, 3 * 2);
+ fullIter = new FullBTreeSearchIterator(btree, CMP, Dir.DESC);
+ leafIter = new LeafBTreeSearchIterator(btree, CMP, Dir.DESC);
+ assertBTreeSearchIteratorEquals(fullIter, leafIter, 3 * 5 + 1, 3 * 2 + 1);
+ }
+}
diff --git a/test/unit/org/apache/cassandra/utils/btree/BTreeTest.java b/test/unit/org/apache/cassandra/utils/btree/BTreeTest.java
new file mode 100644
index 0000000..e60fb64
--- /dev/null
+++ b/test/unit/org/apache/cassandra/utils/btree/BTreeTest.java
@@ -0,0 +1,647 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.utils.btree;
+
+import java.util.*;
+import java.util.concurrent.ThreadLocalRandom;
+
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import org.junit.Test;
+
+import org.junit.Assert;
+
+import static org.apache.cassandra.utils.btree.BTree.EMPTY_LEAF;
+import static org.apache.cassandra.utils.btree.BTree.FAN_FACTOR;
+import static org.apache.cassandra.utils.btree.BTree.FAN_SHIFT;
+import static org.apache.cassandra.utils.btree.BTree.POSITIVE_INFINITY;
+import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+
+public class BTreeTest
+{
+ static Integer[] ints = new Integer[20];
+ static
+ {
+ System.setProperty("cassandra.btree.fanfactor", "4");
+ for (int i = 0 ; i < ints.length ; i++)
+ ints[i] = new Integer(i);
+ }
+
+ static final UpdateFunction<Integer, Integer> updateF = new UpdateFunction<Integer, Integer>()
+ {
+ public Integer apply(Integer replacing, Integer update)
+ {
+ return ints[update];
+ }
+
+ public boolean abortEarly()
+ {
+ return false;
+ }
+
+ public void allocated(long heapSize)
+ {
+
+ }
+
+ public Integer apply(Integer integer)
+ {
+ return ints[integer];
+ }
+ };
+
+ private static final UpdateFunction<Integer, Integer> noOp = new UpdateFunction<Integer, Integer>()
+ {
+ public Integer apply(Integer replacing, Integer update)
+ {
+ return update;
+ }
+
+ public boolean abortEarly()
+ {
+ return false;
+ }
+
+ public void allocated(long heapSize)
+ {
+ }
+
+ public Integer apply(Integer k)
+ {
+ return k;
+ }
+ };
+
+ private static List<Integer> seq(int count, int interval)
+ {
+ List<Integer> r = new ArrayList<>();
+ for (int i = 0 ; i < count ; i++)
+ if (i % interval == 0)
+ r.add(i);
+ return r;
+ }
+
+ private static List<Integer> seq(int count)
+ {
+ return seq(count, 1);
+ }
+
+ private static List<Integer> rand(int count)
+ {
+ Random rand = ThreadLocalRandom.current();
+ List<Integer> r = seq(count);
+ for (int i = 0 ; i < count - 1 ; i++)
+ {
+ int swap = i + rand.nextInt(count - i);
+ Integer tmp = r.get(i);
+ r.set(i, r.get(swap));
+ r.set(swap, tmp);
+ }
+ return r;
+ }
+
+ private static final Comparator<Integer> CMP = new Comparator<Integer>()
+ {
+ public int compare(Integer o1, Integer o2)
+ {
+ return Integer.compare(o1, o2);
+ }
+ };
+
+ @Test
+ public void testBuilding_UpdateFunctionReplacement()
+ {
+ for (int i = 0; i < 20 ; i++)
+ checkResult(i, BTree.build(seq(i), updateF));
+ }
+
+ @Test
+ public void testUpdate_UpdateFunctionReplacement()
+ {
+ for (int i = 0; i < 20 ; i++)
+ checkResult(i, BTree.update(BTree.build(seq(i), noOp), CMP, seq(i), updateF));
+ }
+
+ @Test
+ public void testApply()
+ {
+ List<Integer> input = seq(71);
+ Object[] btree = BTree.build(input, noOp);
+
+ final List<Integer> result = new ArrayList<>();
+ BTree.<Integer>apply(btree, i -> result.add(i));
+
+ org.junit.Assert.assertArrayEquals(input.toArray(),result.toArray());
+ }
+
+ @Test
+ public void inOrderAccumulation()
+ {
+ List<Integer> input = seq(71);
+ Object[] btree = BTree.build(input, noOp);
+ long result = BTree.<Integer>accumulate(btree, (o, l) -> {
+ Assert.assertEquals((long) o, l + 1);
+ return o;
+ }, -1);
+ Assert.assertEquals(result, 70);
+ }
+
+ @Test
+ public void accumulateFrom()
+ {
+ int limit = 100;
+ for (int interval=1; interval<=5; interval++)
+ {
+ List<Integer> input = seq(limit, interval);
+ Object[] btree = BTree.build(input, noOp);
+ for (int start=0; start<=limit; start+=interval)
+ {
+ int thisInterval = interval;
+ String errMsg = String.format("interval=%s, start=%s", interval, start);
+ long result = BTree.accumulate(btree, (o, l) -> {
+ Assert.assertEquals(errMsg, (long) o, l + thisInterval);
+ return o;
+ }, Comparator.naturalOrder(), start, start - thisInterval);
+ Assert.assertEquals(errMsg, result, (limit-1)/interval*interval);
+ }
+ }
+ }
+
+ /**
+ * accumulate function should not be called if we ask it to start past the end of the btree
+ */
+ @Test
+ public void accumulateFromEnd()
+ {
+ List<Integer> input = seq(100);
+ Object[] btree = BTree.build(input, noOp);
+ long result = BTree.accumulate(btree, (o, l) -> 1, Integer::compareTo, 101, 0L);
+ Assert.assertEquals(0, result);
+ }
+
+ /**
+ * Tests that the apply method of the <code>UpdateFunction</code> is only called once with each key update.
+ * (see CASSANDRA-8018).
+ */
+ @Test
+ public void testUpdate_UpdateFunctionCallBack()
+ {
+ Object[] btree = new Object[1];
+ CallsMonitor monitor = new CallsMonitor();
+
+ btree = BTree.update(btree, CMP, Arrays.asList(1), monitor);
+ assertArrayEquals(new Object[] {1}, btree);
+ assertEquals(1, monitor.getNumberOfCalls(1));
+
+ monitor.clear();
+ btree = BTree.update(btree, CMP, Arrays.asList(2), monitor);
+ assertArrayEquals(new Object[] {1, 2, null}, btree);
+ assertEquals(1, monitor.getNumberOfCalls(2));
+
+ // with existing value
+ monitor.clear();
+ btree = BTree.update(btree, CMP, Arrays.asList(1), monitor);
+ assertArrayEquals(new Object[] {1, 2, null}, btree);
+ assertEquals(1, monitor.getNumberOfCalls(1));
+
+ // with two non-existing values
+ monitor.clear();
+ btree = BTree.update(btree, CMP, Arrays.asList(3, 4), monitor);
+ assertArrayEquals(new Object[] {1, 2, 3, 4, null}, btree);
+ assertEquals(1, monitor.getNumberOfCalls(3));
+ assertEquals(1, monitor.getNumberOfCalls(4));
+
+ // with one existing value and one non existing value
+ monitor.clear();
+ btree = BTree.update(btree, CMP, Arrays.asList(2, 5), monitor);
+ assertArrayEquals(new Object[] {3, new Object[]{1, 2, null}, new Object[]{4, 5, null}, new int[]{2, 5}}, btree);
+ assertEquals(1, monitor.getNumberOfCalls(2));
+ assertEquals(1, monitor.getNumberOfCalls(5));
+ }
+
+ /**
+ * Tests that the apply method of the <code>UpdateFunction</code> is only called once per value with each build call.
+ */
+ @Test
+ public void testBuilding_UpdateFunctionCallBack()
+ {
+ CallsMonitor monitor = new CallsMonitor();
+ Object[] btree = BTree.build(Arrays.asList(1), monitor);
+ assertArrayEquals(new Object[] {1}, btree);
+ assertEquals(1, monitor.getNumberOfCalls(1));
+
+ monitor.clear();
+ btree = BTree.build(Arrays.asList(1, 2), monitor);
+ assertArrayEquals(new Object[] {1, 2, null}, btree);
+ assertEquals(1, monitor.getNumberOfCalls(1));
+ assertEquals(1, monitor.getNumberOfCalls(2));
+
+ monitor.clear();
+ btree = BTree.build(Arrays.asList(1, 2, 3), monitor);
+ assertArrayEquals(new Object[] {1, 2, 3}, btree);
+ assertEquals(1, monitor.getNumberOfCalls(1));
+ assertEquals(1, monitor.getNumberOfCalls(2));
+ assertEquals(1, monitor.getNumberOfCalls(3));
+ }
+
+ /**
+ * Tests that the apply method of the <code>QuickResolver</code> is called exactly once per duplicate value
+ */
+ @Test
+ public void testBuilder_QuickResolver()
+ {
+ // for numbers x in 1..N, we repeat x x times, and resolve values to their sum,
+ // so that the resulting tree is of square numbers
+ BTree.Builder.QuickResolver<Accumulator> resolver = (a, b) -> new Accumulator(a.base, a.sum + b.sum);
+
+ for (int count = 0 ; count < 10 ; count ++)
+ {
+ BTree.Builder<Accumulator> builder;
+ // first check we produce the right output for sorted input
+ List<Accumulator> sorted = resolverInput(count, false);
+ builder = BTree.builder(Comparator.naturalOrder());
+ builder.setQuickResolver(resolver);
+ for (Accumulator i : sorted)
+ builder.add(i);
+ // for sorted input, check non-resolve path works before checking resolution path
+ checkResolverOutput(count, builder.build(), BTree.Dir.ASC);
+ builder = BTree.builder(Comparator.naturalOrder());
+ builder.setQuickResolver(resolver);
+ for (int i = 0 ; i < 10 ; i++)
+ {
+ // now do a few runs of randomized inputs
+ for (Accumulator j : resolverInput(count, true))
+ builder.add(j);
+ checkResolverOutput(count, builder.build(), BTree.Dir.ASC);
+ builder = BTree.builder(Comparator.naturalOrder());
+ builder.setQuickResolver(resolver);
+ }
+ for (List<Accumulator> add : splitResolverInput(count))
+ {
+ if (ThreadLocalRandom.current().nextBoolean())
+ builder.addAll(add);
+ else
+ builder.addAll(new TreeSet<>(add));
+ }
+ checkResolverOutput(count, builder.build(), BTree.Dir.ASC);
+ }
+ }
+
+ @Test
+ public void testBuilderReuse()
+ {
+ List<Integer> sorted = seq(20);
+ BTree.Builder<Integer> builder = BTree.builder(Comparator.naturalOrder());
+ builder.auto(false);
+ for (int i : sorted)
+ builder.add(i);
+ checkResult(20, builder.build());
+
+ builder.reuse();
+ assertTrue(builder.build() == BTree.empty());
+ for (int i = 0; i < 12; i++)
+ builder.add(sorted.get(i));
+ checkResult(12, builder.build());
+
+ builder.auto(true);
+ builder.reuse(Comparator.reverseOrder());
+ for (int i = 0; i < 12; i++)
+ builder.add(sorted.get(i));
+ checkResult(12, builder.build(), BTree.Dir.DESC);
+
+ builder.reuse();
+ assertTrue(builder.build() == BTree.empty());
+ }
+
+ private static class Accumulator extends Number implements Comparable<Accumulator>
+ {
+ final int base;
+ final int sum;
+ private Accumulator(int base, int sum)
+ {
+ this.base = base;
+ this.sum = sum;
+ }
+
+ public int compareTo(Accumulator that) { return Integer.compare(base, that.base); }
+ public int intValue() { return sum; }
+ public long longValue() { return sum; }
+ public float floatValue() { return sum; }
+ public double doubleValue() { return sum; }
+ }
+
+ /**
+ * Tests that the apply method of the <code>Resolver</code> is called exactly once per unique value
+ */
+ @Test
+ public void testBuilder_ResolverAndReverse()
+ {
+ // for numbers x in 1..N, we repeat x x times, and resolve values to their sum,
+ // so that the resulting tree is of square numbers
+ BTree.Builder.Resolver resolver = (array, lb, ub) -> {
+ int sum = 0;
+ for (int i = lb ; i < ub ; i++)
+ sum += ((Accumulator) array[i]).sum;
+ return new Accumulator(((Accumulator) array[lb]).base, sum);
+ };
+
+ for (int count = 0 ; count < 10 ; count ++)
+ {
+ BTree.Builder<Accumulator> builder;
+ // first check we produce the right output for sorted input
+ List<Accumulator> sorted = resolverInput(count, false);
+ builder = BTree.builder(Comparator.naturalOrder());
+ builder.auto(false);
+ for (Accumulator i : sorted)
+ builder.add(i);
+ // for sorted input, check non-resolve path works before checking resolution path
+ Assert.assertTrue(Iterables.elementsEqual(sorted, BTree.iterable(builder.build())));
+
+ builder = BTree.builder(Comparator.naturalOrder());
+ builder.auto(false);
+ for (Accumulator i : sorted)
+ builder.add(i);
+ // check resolution path
+ checkResolverOutput(count, builder.resolve(resolver).build(), BTree.Dir.ASC);
+
+ builder = BTree.builder(Comparator.naturalOrder());
+ builder.auto(false);
+ for (int i = 0 ; i < 10 ; i++)
+ {
+ // now do a few runs of randomized inputs
+ for (Accumulator j : resolverInput(count, true))
+ builder.add(j);
+ checkResolverOutput(count, builder.sort().resolve(resolver).build(), BTree.Dir.ASC);
+ builder = BTree.builder(Comparator.naturalOrder());
+ builder.auto(false);
+ for (Accumulator j : resolverInput(count, true))
+ builder.add(j);
+ checkResolverOutput(count, builder.sort().reverse().resolve(resolver).build(), BTree.Dir.DESC);
+ builder = BTree.builder(Comparator.naturalOrder());
+ builder.auto(false);
+ }
+ }
+ }
+
+ private static List<Accumulator> resolverInput(int count, boolean shuffled)
+ {
+ List<Accumulator> result = new ArrayList<>();
+ for (int i = 1 ; i <= count ; i++)
+ for (int j = 0 ; j < i ; j++)
+ result.add(new Accumulator(i, i));
+ if (shuffled)
+ {
+ ThreadLocalRandom random = ThreadLocalRandom.current();
+ for (int i = 0 ; i < result.size() ; i++)
+ {
+ int swapWith = random.nextInt(i, result.size());
+ Accumulator t = result.get(swapWith);
+ result.set(swapWith, result.get(i));
+ result.set(i, t);
+ }
+ }
+ return result;
+ }
+
+ private static List<List<Accumulator>> splitResolverInput(int count)
+ {
+ List<Accumulator> all = resolverInput(count, false);
+ List<List<Accumulator>> result = new ArrayList<>();
+ while (!all.isEmpty())
+ {
+ List<Accumulator> is = new ArrayList<>();
+ int prev = -1;
+ for (Accumulator i : new ArrayList<>(all))
+ {
+ if (i.base == prev)
+ continue;
+ is.add(i);
+ all.remove(i);
+ prev = i.base;
+ }
+ result.add(is);
+ }
+ return result;
+ }
+
+ private static void checkResolverOutput(int count, Object[] btree, BTree.Dir dir)
+ {
+ int i = 1;
+ for (Accumulator current : BTree.<Accumulator>iterable(btree, dir))
+ {
+ Assert.assertEquals(i * i, current.sum);
+ i++;
+ }
+ Assert.assertEquals(i, count + 1);
+ }
+
+ private static void checkResult(int count, Object[] btree)
+ {
+ checkResult(count, btree, BTree.Dir.ASC);
+ }
+
+ private static void checkResult(int count, Object[] btree, BTree.Dir dir)
+ {
+ Iterator<Integer> iter = BTree.slice(btree, CMP, dir);
+ int i = 0;
+ while (iter.hasNext())
+ assertEquals(iter.next(), ints[i++]);
+ assertEquals(count, i);
+ }
+
+ @Test
+ public void testClearOnAbort()
+ {
+ Object[] btree = BTree.build(seq(2), noOp);
+ Object[] copy = Arrays.copyOf(btree, btree.length);
+ BTree.update(btree, CMP, seq(94), new AbortAfterX(90));
+
+ assertArrayEquals(copy, btree);
+
+ btree = BTree.update(btree, CMP, seq(94), noOp);
+ assertTrue(BTree.isWellFormed(btree, CMP));
+ }
+
+ private static final class AbortAfterX implements UpdateFunction<Integer, Integer>
+ {
+ int counter;
+ final int abortAfter;
+ private AbortAfterX(int abortAfter)
+ {
+ this.abortAfter = abortAfter;
+ }
+ public Integer apply(Integer replacing, Integer update)
+ {
+ return update;
+ }
+ public boolean abortEarly()
+ {
+ return counter++ > abortAfter;
+ }
+ public void allocated(long heapSize)
+ {
+ }
+ public Integer apply(Integer v)
+ {
+ return v;
+ }
+ }
+
+ /**
+ * <code>UpdateFunction</code> that count the number of call made to apply for each value.
+ */
+ public static final class CallsMonitor implements UpdateFunction<Integer, Integer>
+ {
+ private int[] numberOfCalls = new int[20];
+
+ public Integer apply(Integer replacing, Integer update)
+ {
+ numberOfCalls[update] = numberOfCalls[update] + 1;
+ return update;
+ }
+
+ public boolean abortEarly()
+ {
+ return false;
+ }
+
+ public void allocated(long heapSize)
+ {
+
+ }
+
+ public Integer apply(Integer integer)
+ {
+ numberOfCalls[integer] = numberOfCalls[integer] + 1;
+ return integer;
+ }
+
+ public int getNumberOfCalls(Integer key)
+ {
+ return numberOfCalls[key];
+ }
+
+ public void clear()
+ {
+ Arrays.fill(numberOfCalls, 0);
+ }
+ }
+
+ @Test
+ public void testTransformAndFilter()
+ {
+ List<Integer> r = seq(100);
+
+ Object[] b1 = BTree.build(r, UpdateFunction.noOp());
+
+ // replace all values
+ Object[] b2 = BTree.transformAndFilter(b1, (x) -> (Integer) x * 2);
+ assertEquals(BTree.size(b1), BTree.size(b2));
+
+ // remove odd numbers
+ Object[] b3 = BTree.transformAndFilter(b1, (x) -> (Integer) x % 2 == 1 ? x : null);
+ assertEquals(BTree.size(b1) / 2, BTree.size(b3));
+
+ // remove all values
+ Object[] b4 = BTree.transformAndFilter(b1, (x) -> null);
+ assertEquals(0, BTree.size(b4));
+ }
+
+ private <C, K extends C, V extends C> Object[] buildBTreeLegacy(Iterable<K> source, UpdateFunction<K, V> updateF, int size)
+ {
+ assert updateF != null;
+ NodeBuilder current = new NodeBuilder();
+
+ while ((size >>= FAN_SHIFT) > 0)
+ current = current.ensureChild();
+
+ current.reset(EMPTY_LEAF, POSITIVE_INFINITY, updateF, null);
+ for (K key : source)
+ current.addNewKey(key);
+
+ current = current.ascendToRoot();
+
+ Object[] r = current.toNode();
+ current.clear();
+ return r;
+ }
+
+ // Basic BTree validation to check the values and sizeOffsets. Return tree size.
+ private int validateBTree(Object[] tree, int[] startingPos, boolean isRoot)
+ {
+ if (BTree.isLeaf(tree))
+ {
+ int size = BTree.size(tree);
+ if (!isRoot)
+ {
+ assertTrue(size >= FAN_FACTOR / 2);
+ assertTrue(size <= FAN_FACTOR);
+ }
+ for (int i = 0; i < size; i++)
+ {
+ assertEquals((int)tree[i], startingPos[0]);
+ startingPos[0]++;
+ }
+ return size;
+ }
+
+ int childNum = BTree.getChildCount(tree);
+ assertTrue(childNum >= FAN_FACTOR / 2);
+ assertTrue(childNum <= FAN_FACTOR + 1);
+
+ int childStart = BTree.getChildStart(tree);
+ int[] sizeOffsets = BTree.getSizeMap(tree);
+ int pos = 0;
+ for (int i = 0; i < childNum; i++)
+ {
+ int childSize = validateBTree((Object[])tree[i + childStart], startingPos, false);
+
+ pos += childSize;
+ assertEquals(sizeOffsets[i], pos);
+ if (i != childNum - 1)
+ {
+ assertEquals((int)tree[i], startingPos[0]);
+ pos++;
+ startingPos[0]++;
+ }
+
+ }
+ return BTree.size(tree);
+ }
+
+ @Test
+ public void testBuildTree()
+ {
+ int maxCount = 1000;
+
+ for (int count = 0; count < maxCount; count++)
+ {
+ List<Integer> r = seq(count);
+ Object[] b1 = BTree.build(r, UpdateFunction.noOp());
+ Object[] b2 = buildBTreeLegacy(r, UpdateFunction.noOp(), count);
+ assertTrue(BTree.equals(b1, b2));
+
+ int[] startingPos = new int[1];
+ startingPos[0] = 0;
+ assertEquals(count, validateBTree(b1, startingPos, true));
+ startingPos[0] = 0;
+ assertEquals(count, validateBTree(b2, startingPos, true));
+ }
+ }
+}
diff --git a/test/unit/org/apache/cassandra/utils/concurrent/AbstractTransactionalTest.java b/test/unit/org/apache/cassandra/utils/concurrent/AbstractTransactionalTest.java
index 5a20d67..bde0586 100644
--- a/test/unit/org/apache/cassandra/utils/concurrent/AbstractTransactionalTest.java
+++ b/test/unit/org/apache/cassandra/utils/concurrent/AbstractTransactionalTest.java
@@ -22,8 +22,9 @@
import org.junit.Ignore;
import org.junit.Test;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.commitlog.CommitLog;
@Ignore
public abstract class AbstractTransactionalTest
@@ -32,6 +33,7 @@
public static void setupDD()
{
DatabaseDescriptor.daemonInitialization();
+ CommitLog.instance.start();
}
protected abstract TestableTransaction newTest() throws Exception;
diff --git a/test/unit/org/apache/cassandra/utils/concurrent/AccumulatorTest.java b/test/unit/org/apache/cassandra/utils/concurrent/AccumulatorTest.java
index 33daca7..9b34a23 100644
--- a/test/unit/org/apache/cassandra/utils/concurrent/AccumulatorTest.java
+++ b/test/unit/org/apache/cassandra/utils/concurrent/AccumulatorTest.java
@@ -87,7 +87,7 @@
assertEquals("0", accu.get(3));
- Iterator<String> iter = accu.iterator();
+ Iterator<String> iter = accu.snapshot().iterator();
assertEquals("3", iter.next());
assertEquals("2", iter.next());
@@ -108,7 +108,7 @@
accu.clearUnsafe();
assertEquals(0, accu.size());
- assertFalse(accu.iterator().hasNext());
+ assertFalse(accu.snapshot().iterator().hasNext());
assertOutOfBonds(accu, 0);
accu.add("4");
@@ -120,7 +120,7 @@
assertEquals("5", accu.get(1));
assertOutOfBonds(accu, 2);
- Iterator<String> iter = accu.iterator();
+ Iterator<String> iter = accu.snapshot().iterator();
assertTrue(iter.hasNext());
assertEquals("4", iter.next());
assertEquals("5", iter.next());
diff --git a/test/unit/org/apache/cassandra/utils/concurrent/RefCountedTest.java b/test/unit/org/apache/cassandra/utils/concurrent/RefCountedTest.java
index 0582ad4..0d1f9f6 100644
--- a/test/unit/org/apache/cassandra/utils/concurrent/RefCountedTest.java
+++ b/test/unit/org/apache/cassandra/utils/concurrent/RefCountedTest.java
@@ -20,7 +20,7 @@
import org.junit.Test;
-import junit.framework.Assert;
+import org.junit.Assert;
import java.io.File;
import java.lang.ref.WeakReference;
@@ -36,6 +36,7 @@
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.atomic.AtomicReference;
+import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.utils.ObjectSizes;
import org.apache.cassandra.utils.Pair;
import org.apache.cassandra.utils.concurrent.Ref.Visitor;
@@ -342,7 +343,7 @@
for (int i = 0; i < entryCount; i += 2)
objects[i] = new Object();
- File f = File.createTempFile("foo", "bar");
+ File f = FileUtils.createTempFile("foo", "bar");
RefCounted.Tidy tidier = new RefCounted.Tidy() {
Object ref = objects;
//Checking we don't get an infinite loop out of traversing file refs
diff --git a/test/unit/org/apache/cassandra/utils/concurrent/WeightedQueueTest.java b/test/unit/org/apache/cassandra/utils/concurrent/WeightedQueueTest.java
new file mode 100644
index 0000000..544e95c
--- /dev/null
+++ b/test/unit/org/apache/cassandra/utils/concurrent/WeightedQueueTest.java
@@ -0,0 +1,656 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.utils.concurrent;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.TimeUnit;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+public class WeightedQueueTest
+{
+ private static WeightedQueue<Object> queue()
+ {
+ return new WeightedQueue<>(10);
+ }
+
+ private WeightedQueue<Object> queue;
+
+ @Before
+ public void setUp()
+ {
+ queue = queue();
+ }
+
+ private static WeightedQueue.Weighable weighable(int weight)
+ {
+ return () -> weight;
+ }
+
+ @Test(expected = UnsupportedOperationException.class)
+ public void testAddUnsupported() throws Exception
+ {
+ queue.add(new Object());
+ }
+
+ @Test(expected = UnsupportedOperationException.class)
+ public void testRemoveUnsupported() throws Exception
+ {
+ queue.remove();
+ }
+
+ @Test(expected = UnsupportedOperationException.class)
+ public void testElementUnsupported() throws Exception
+ {
+ queue.element();
+ }
+
+ @Test(expected = UnsupportedOperationException.class)
+ public void testPeekUnsupported() throws Exception
+ {
+ queue.peek();
+ }
+
+ @Test(expected = UnsupportedOperationException.class)
+ public void testRemainingCapacityUnsupported() throws Exception
+ {
+ queue.remainingCapacity();
+ }
+
+ @Test(expected = UnsupportedOperationException.class)
+ public void testRemoveElementUnsupported() throws Exception
+ {
+ queue.remove(null);
+ }
+
+ @Test(expected = UnsupportedOperationException.class)
+ public void testContainsAllUnsupported() throws Exception
+ {
+ queue.containsAll(null);
+ }
+
+ @Test(expected = UnsupportedOperationException.class)
+ public void testAddAllUnsupported() throws Exception
+ {
+ queue.addAll(null);
+ }
+
+ @Test(expected = UnsupportedOperationException.class)
+ public void testRemoveAllUnsupported() throws Exception
+ {
+ queue.removeAll(null);
+ }
+
+ @Test(expected = UnsupportedOperationException.class)
+ public void testRetainAllUnsupported() throws Exception
+ {
+ queue.retainAll(null);
+ }
+
+ @Test(expected = UnsupportedOperationException.class)
+ public void testClearUnsupported() throws Exception
+ {
+ queue.clear();
+ }
+
+ @Test(expected = UnsupportedOperationException.class)
+ public void testSizeUnsupported() throws Exception
+ {
+ queue.size();
+ }
+
+ @Test(expected = UnsupportedOperationException.class)
+ public void testIsEmptyUnsupported() throws Exception
+ {
+ queue.isEmpty();
+ }
+
+ @Test(expected = UnsupportedOperationException.class)
+ public void testContainsUnsupported() throws Exception
+ {
+ queue.contains(null);
+ }
+
+ @Test(expected = UnsupportedOperationException.class)
+ public void testIteratorUnsupported() throws Exception
+ {
+ queue.iterator();
+ }
+
+ @Test(expected = UnsupportedOperationException.class)
+ public void testToArrayUnsupported() throws Exception
+ {
+ queue.toArray();
+ }
+
+ @Test(expected = UnsupportedOperationException.class)
+ public void testToArray2Unsupported() throws Exception
+ {
+ queue.toArray( new Object[] {});
+ }
+
+ @Test(expected = UnsupportedOperationException.class)
+ public void testDrainToUnsupported() throws Exception
+ {
+ queue.drainTo(new ArrayList<>());
+ }
+
+ @Test(expected = UnsupportedOperationException.class)
+ public void testTimedPollUnsupported() throws Exception
+ {
+ queue.poll(1, TimeUnit.MICROSECONDS);
+ }
+
+ @Test
+ public void testDrainToWithLimit() throws Exception
+ {
+ queue.offer(new Object());
+ queue.offer(new Object());
+ queue.offer(new Object());
+ ArrayList<Object> list = new ArrayList<>();
+ queue.drainTo(list, 1);
+ assertEquals(1, list.size());
+ list.clear();
+ queue.drainTo(list, 10);
+ assertEquals(2, list.size());
+ }
+
+ @Test(expected = NullPointerException.class)
+ public void offerNullThrows() throws Exception
+ {
+ queue.offer(null);
+ }
+
+ /**
+ * This also tests that natural weight (weighable interface) is respected
+ */
+ @Test
+ public void offerFullFails() throws Exception
+ {
+ assertTrue(queue.offer(weighable(10)));
+ assertFalse(queue.offer(weighable(1)));
+ }
+
+ /**
+ * Validate permits aren't leaked and return values are correct
+ */
+ @Test
+ public void testOfferWrappedQueueRefuses() throws Exception
+ {
+ queue = new WeightedQueue<>(10, new BadQueue(true), WeightedQueue.NATURAL_WEIGHER);
+ assertEquals(10, queue.availableWeight.availablePermits());
+ assertFalse(queue.offer(new Object()));
+ assertEquals(10, queue.availableWeight.availablePermits());
+ }
+
+ /**
+ * Validate permits aren't leaked and return values are correct
+ */
+ @Test
+ public void testOfferWrappedQueueThrows() throws Exception
+ {
+ queue = new WeightedQueue<>(10, new BadQueue(false), WeightedQueue.NATURAL_WEIGHER);
+ assertEquals(10, queue.availableWeight.availablePermits());
+ try
+ {
+ assertFalse(queue.offer(new Object()));
+ fail();
+ }
+ catch (UnsupportedOperationException e)
+ {
+ //expected and desired
+ }
+ assertEquals(10, queue.availableWeight.availablePermits());
+ }
+
+ /**
+ * If not weighable and not custom weigher the default weight is 1
+ */
+ @Test
+ public void defaultWeightRespected() throws Exception
+ {
+ for (int ii = 0; ii < 10; ii++)
+ {
+ assertTrue(queue.offer(new Object()));
+ }
+ assertFalse(queue.offer(new Object()));
+ }
+
+ @Test
+ public void testCustomWeigher() throws Exception
+ {
+ queue = new WeightedQueue<>(10, new LinkedBlockingQueue<>(), weighable -> 10 );
+ assertTrue(queue.offer(new Object()));
+ assertFalse(queue.offer(new Object()));
+ }
+
+ @Test(expected = UnsupportedOperationException.class)
+ public void testCustomQueue() throws Exception
+ {
+ new WeightedQueue<>(10, new BadQueue(false), WeightedQueue.NATURAL_WEIGHER).offer(new Object());
+ }
+
+ @Test(expected = NullPointerException.class)
+ public void timedOfferNullValueThrows() throws Exception
+ {
+ queue.offer(null, 1, TimeUnit.SECONDS);
+ }
+
+ @Test(expected = NullPointerException.class)
+ public void timedOfferNullTimeThrows() throws Exception
+ {
+ queue.offer(null, 1, null);
+ }
+
+ /**
+ * This is how it seems to be handled in java.util.concurrent, it's the same as just try
+ */
+ @Test
+ public void timedOfferNegativeTimeIgnored() throws Exception
+ {
+ queue.offer(weighable(10));
+ queue.offer(new Object(), -1, TimeUnit.SECONDS);
+ }
+
+ /**
+ * This also tests that natural weight (weighable interface) is respected
+ */
+ @Test
+ public void timedOfferFullFails() throws Exception
+ {
+ assertTrue(queue.offer(weighable(10), 1, TimeUnit.MICROSECONDS));
+ assertFalse(queue.offer(weighable(1), 1, TimeUnit.MICROSECONDS));
+ }
+
+ @Test
+ public void timedOfferEventuallySucceeds() throws Exception
+ {
+ assertTrue(queue.offer(weighable(10), 1, TimeUnit.MICROSECONDS));
+ Thread t = new Thread(() ->
+ {
+ try
+ {
+ queue.offer(weighable(1), 1, TimeUnit.DAYS);
+ }
+ catch (InterruptedException e)
+ {
+ e.printStackTrace();
+ }
+ });
+ t.start();
+ Thread.sleep(100);
+ assertTrue(t.getState() != Thread.State.TERMINATED);
+ queue.poll();
+ t.join(60000);
+ assertEquals(t.getState(), Thread.State.TERMINATED);
+ }
+
+ /**
+ * Validate permits aren't leaked and return values are correct
+ */
+ @Test
+ public void testTimedOfferWrappedQueueRefuses() throws Exception
+ {
+ queue = new WeightedQueue<>(10, new BadQueue(true), WeightedQueue.NATURAL_WEIGHER);
+ assertEquals(10, queue.availableWeight.availablePermits());
+ assertFalse(queue.offer(new Object(), 1, TimeUnit.MICROSECONDS));
+ assertEquals(10, queue.availableWeight.availablePermits());
+ }
+
+ /**
+ * Validate permits aren't leaked and return values are correct
+ */
+ @Test
+ public void testTimedOfferWrappedQueueThrows() throws Exception
+ {
+ queue = new WeightedQueue<>(10, new BadQueue(false), WeightedQueue.NATURAL_WEIGHER);
+ assertEquals(10, queue.availableWeight.availablePermits());
+ try
+ {
+ assertFalse(queue.offer(new Object(), 1, TimeUnit.MICROSECONDS));
+ fail();
+ }
+ catch (UnsupportedOperationException e)
+ {
+ //expected and desired
+ }
+ assertEquals(10, queue.availableWeight.availablePermits());
+ }
+
+
+ @Test
+ public void testPoll() throws Exception
+ {
+ assertEquals(10, queue.availableWeight.availablePermits());
+ assertNull(queue.poll());
+ assertEquals(10, queue.availableWeight.availablePermits());
+ Object o = new Object();
+ assertTrue(queue.offer(o));
+ assertEquals(9, queue.availableWeight.availablePermits());
+ WeightedQueue.Weighable weighable = weighable(9);
+ assertTrue(queue.offer(weighable));
+ assertEquals(0, queue.availableWeight.availablePermits());
+ assertEquals(o, queue.poll());
+ assertEquals(1, queue.availableWeight.availablePermits());
+ assertEquals(weighable, queue.poll());
+ assertEquals(10, queue.availableWeight.availablePermits());
+ }
+
+ @Test(expected = NullPointerException.class)
+ public void testPutNullThrows() throws Exception
+ {
+ queue.put(null);
+ }
+
+ @Test
+ public void testPutFullBlocks() throws Exception
+ {
+ WeightedQueue.Weighable weighable = weighable(10);
+ assertEquals(10, queue.availableWeight.availablePermits());
+ queue.put(weighable);
+ assertEquals(0, queue.availableWeight.availablePermits());
+ Object o = new Object();
+ Thread t = new Thread(() -> {
+ try
+ {
+ queue.put(o);
+ } catch (InterruptedException e)
+ {
+ e.printStackTrace();
+ }
+ });
+ t.start();
+ Thread.sleep(100);
+ assertTrue(t.getState() != Thread.State.TERMINATED);
+ assertEquals(0, queue.availableWeight.availablePermits());
+ assertEquals(weighable, queue.poll());
+ assertTrue(queue.availableWeight.availablePermits() > 0);
+ t.join();
+ assertEquals(o, queue.poll());
+ assertEquals(10, queue.availableWeight.availablePermits());
+ }
+
+ @Test
+ public void testPutWrappedQueueThrows() throws Exception
+ {
+ queue = new WeightedQueue<>(10, new BadQueue(false), WeightedQueue.NATURAL_WEIGHER);
+ assertEquals(10, queue.availableWeight.availablePermits());
+ try
+ {
+ queue.put(new Object());
+ fail();
+ }
+ catch (UnsupportedOperationException e)
+ {
+ //expected and desired
+ }
+ assertEquals(10, queue.availableWeight.availablePermits());
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testTryAcquireWeightIllegalWeight()
+ {
+ queue.tryAcquireWeight(weighable(-1));
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testAcquireWeightIllegalWeight() throws Exception
+ {
+ queue.acquireWeight(weighable(-1), 1, TimeUnit.DAYS);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testReleaseWeightIllegalWeight()
+ {
+ queue.releaseWeight(weighable(-1));
+ }
+
+ @Test
+ public void testTake() throws Exception
+ {
+ Thread t = new Thread(() -> {
+ try
+ {
+ queue.take();
+ }
+ catch (InterruptedException e)
+ {
+ e.printStackTrace();
+ }
+ });
+ t.start();
+ Thread.sleep(500);
+ assertTrue(t.getState() != Thread.State.TERMINATED);
+ assertEquals(10, queue.availableWeight.availablePermits());
+ queue.offer(new Object());
+ t.join(60 * 1000);
+ assertEquals(t.getState(), Thread.State.TERMINATED);
+ assertEquals(10, queue.availableWeight.availablePermits());
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testConstructorLTZeroWeightThrows() throws Exception
+ {
+ new WeightedQueue(0);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testConstructor2LTZeroWeightThrows() throws Exception
+ {
+ new WeightedQueue(0, new LinkedBlockingQueue<>(), WeightedQueue.NATURAL_WEIGHER);
+ }
+
+ @Test(expected = NullPointerException.class)
+ public void testConstructorNullQueueThrows() throws Exception
+ {
+ new WeightedQueue(1, null, WeightedQueue.NATURAL_WEIGHER);
+ }
+
+ @Test(expected = NullPointerException.class)
+ public void testConstructorNullWeigherThrows() throws Exception
+ {
+ new WeightedQueue(1, new LinkedBlockingQueue<>(), null);
+ }
+
+ /**
+ * A blocking queue that throws or refuses on every method
+ */
+ private static class BadQueue implements BlockingQueue<Object>
+ {
+ /**
+ * Refuse instead of throwing for some methods that have a boolean return value
+ */
+ private boolean refuse = false;
+
+ private BadQueue(boolean refuse)
+ {
+ this.refuse = refuse;
+ }
+
+ @Override
+ public boolean add(Object o)
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean offer(Object o)
+ {
+ if (refuse)
+ {
+ return false;
+ }
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Object remove()
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Object poll()
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Object element()
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Object peek()
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void put(Object o) throws InterruptedException
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean offer(Object o, long timeout, TimeUnit unit) throws InterruptedException
+ {
+ if (refuse)
+ {
+ return false;
+ }
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Object take() throws InterruptedException
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Object poll(long timeout, TimeUnit unit) throws InterruptedException
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public int remainingCapacity()
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean remove(Object o)
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean addAll(Collection c)
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void clear()
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean retainAll(Collection c)
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean removeAll(Collection c)
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean containsAll(Collection c)
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public int size()
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean isEmpty()
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean contains(Object o)
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Iterator iterator()
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Object[] toArray()
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Object[] toArray(Object[] a)
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public int drainTo(Collection c)
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public int drainTo(Collection c, int maxElements)
+ {
+ throw new UnsupportedOperationException();
+ }
+ };
+
+}
diff --git a/test/unit/org/apache/cassandra/utils/memory/BufferPoolTest.java b/test/unit/org/apache/cassandra/utils/memory/BufferPoolTest.java
index 74889a1..5df94ec 100644
--- a/test/unit/org/apache/cassandra/utils/memory/BufferPoolTest.java
+++ b/test/unit/org/apache/cassandra/utils/memory/BufferPoolTest.java
@@ -47,13 +47,21 @@
public void setUp()
{
BufferPool.MEMORY_USAGE_THRESHOLD = 8 * 1024L * 1024L;
- BufferPool.DISABLED = false;
}
@After
public void cleanUp()
{
- BufferPool.reset();
+ resetBufferPool();
+ }
+
+ /**
+ * Exposes a utility method on this test that other tests might use to access the protected
+ * {@link BufferPool#unsafeReset()} method.
+ */
+ public static void resetBufferPool()
+ {
+ BufferPool.unsafeReset();
}
@Test
@@ -61,17 +69,17 @@
{
final int size = RandomAccessReader.DEFAULT_BUFFER_SIZE;
- ByteBuffer buffer = BufferPool.get(size);
+ ByteBuffer buffer = BufferPool.get(size, BufferType.OFF_HEAP);
assertNotNull(buffer);
assertEquals(size, buffer.capacity());
assertEquals(true, buffer.isDirect());
- BufferPool.Chunk chunk = BufferPool.currentChunk();
+ BufferPool.Chunk chunk = BufferPool.unsafeCurrentChunk();
assertNotNull(chunk);
assertEquals(BufferPool.GlobalPool.MACRO_CHUNK_SIZE, BufferPool.sizeInBytes());
BufferPool.put(buffer);
- assertEquals(null, BufferPool.currentChunk());
+ assertEquals(null, BufferPool.unsafeCurrentChunk());
assertEquals(BufferPool.GlobalPool.MACRO_CHUNK_SIZE, BufferPool.sizeInBytes());
}
@@ -81,7 +89,7 @@
{
final int size = 1024;
for (int i = size;
- i <= BufferPool.CHUNK_SIZE;
+ i <= BufferPool.NORMAL_CHUNK_SIZE;
i += size)
{
checkPageAligned(i);
@@ -90,7 +98,7 @@
private void checkPageAligned(int size)
{
- ByteBuffer buffer = BufferPool.get(size);
+ ByteBuffer buffer = BufferPool.get(size, BufferType.OFF_HEAP);
assertNotNull(buffer);
assertEquals(size, buffer.capacity());
assertTrue(buffer.isDirect());
@@ -107,45 +115,35 @@
final int size1 = 1024;
final int size2 = 2048;
- ByteBuffer buffer1 = BufferPool.get(size1);
+ ByteBuffer buffer1 = BufferPool.get(size1, BufferType.OFF_HEAP);
assertNotNull(buffer1);
assertEquals(size1, buffer1.capacity());
- ByteBuffer buffer2 = BufferPool.get(size2);
+ ByteBuffer buffer2 = BufferPool.get(size2, BufferType.OFF_HEAP);
assertNotNull(buffer2);
assertEquals(size2, buffer2.capacity());
- BufferPool.Chunk chunk = BufferPool.currentChunk();
+ BufferPool.Chunk chunk = BufferPool.unsafeCurrentChunk();
assertNotNull(chunk);
assertEquals(BufferPool.GlobalPool.MACRO_CHUNK_SIZE, BufferPool.sizeInBytes());
BufferPool.put(buffer1);
BufferPool.put(buffer2);
- assertEquals(null, BufferPool.currentChunk());
+ assertEquals(null, BufferPool.unsafeCurrentChunk());
assertEquals(BufferPool.GlobalPool.MACRO_CHUNK_SIZE, BufferPool.sizeInBytes());
}
@Test
public void testMaxMemoryExceededDirect()
{
- boolean cur = BufferPool.ALLOCATE_ON_HEAP_WHEN_EXAHUSTED;
- BufferPool.ALLOCATE_ON_HEAP_WHEN_EXAHUSTED = false;
-
requestDoubleMaxMemory();
-
- BufferPool.ALLOCATE_ON_HEAP_WHEN_EXAHUSTED = cur;
}
@Test
public void testMaxMemoryExceededHeap()
{
- boolean cur = BufferPool.ALLOCATE_ON_HEAP_WHEN_EXAHUSTED;
- BufferPool.ALLOCATE_ON_HEAP_WHEN_EXAHUSTED = true;
-
requestDoubleMaxMemory();
-
- BufferPool.ALLOCATE_ON_HEAP_WHEN_EXAHUSTED = cur;
}
@Test
@@ -165,7 +163,7 @@
@Test
public void testRecycle()
{
- requestUpToSize(RandomAccessReader.DEFAULT_BUFFER_SIZE, 3 * BufferPool.CHUNK_SIZE);
+ requestUpToSize(RandomAccessReader.DEFAULT_BUFFER_SIZE, 3 * BufferPool.NORMAL_CHUNK_SIZE);
}
private void requestDoubleMaxMemory()
@@ -180,27 +178,23 @@
List<ByteBuffer> buffers = new ArrayList<>(numBuffers);
for (int i = 0; i < numBuffers; i++)
{
- ByteBuffer buffer = BufferPool.get(bufferSize);
+ ByteBuffer buffer = BufferPool.get(bufferSize, BufferType.OFF_HEAP);
assertNotNull(buffer);
assertEquals(bufferSize, buffer.capacity());
-
- if (BufferPool.sizeInBytes() > BufferPool.MEMORY_USAGE_THRESHOLD)
- assertEquals(BufferPool.ALLOCATE_ON_HEAP_WHEN_EXAHUSTED, !buffer.isDirect());
-
+ assertTrue(buffer.isDirect());
buffers.add(buffer);
}
for (ByteBuffer buffer : buffers)
BufferPool.put(buffer);
-
}
@Test
public void testBigRequest()
{
- final int size = BufferPool.CHUNK_SIZE + 1;
+ final int size = BufferPool.NORMAL_CHUNK_SIZE + 1;
- ByteBuffer buffer = BufferPool.get(size);
+ ByteBuffer buffer = BufferPool.get(size, BufferType.OFF_HEAP);
assertNotNull(buffer);
assertEquals(size, buffer.capacity());
BufferPool.put(buffer);
@@ -210,30 +204,30 @@
public void testFillUpChunks()
{
final int size = RandomAccessReader.DEFAULT_BUFFER_SIZE;
- final int numBuffers = BufferPool.CHUNK_SIZE / size;
+ final int numBuffers = BufferPool.NORMAL_CHUNK_SIZE / size;
List<ByteBuffer> buffers1 = new ArrayList<>(numBuffers);
List<ByteBuffer> buffers2 = new ArrayList<>(numBuffers);
for (int i = 0; i < numBuffers; i++)
- buffers1.add(BufferPool.get(size));
+ buffers1.add(BufferPool.get(size, BufferType.OFF_HEAP));
- BufferPool.Chunk chunk1 = BufferPool.currentChunk();
+ BufferPool.Chunk chunk1 = BufferPool.unsafeCurrentChunk();
assertNotNull(chunk1);
for (int i = 0; i < numBuffers; i++)
- buffers2.add(BufferPool.get(size));
+ buffers2.add(BufferPool.get(size, BufferType.OFF_HEAP));
- assertEquals(2, BufferPool.numChunks());
+ assertEquals(2, BufferPool.unsafeNumChunks());
for (ByteBuffer buffer : buffers1)
BufferPool.put(buffer);
- assertEquals(1, BufferPool.numChunks());
+ assertEquals(1, BufferPool.unsafeNumChunks());
for (ByteBuffer buffer : buffers2)
BufferPool.put(buffer);
- assertEquals(0, BufferPool.numChunks());
+ assertEquals(0, BufferPool.unsafeNumChunks());
buffers2.clear();
}
@@ -242,7 +236,7 @@
public void testOutOfOrderFrees()
{
final int size = 4096;
- final int maxFreeSlots = BufferPool.CHUNK_SIZE / size;
+ final int maxFreeSlots = BufferPool.NORMAL_CHUNK_SIZE / size;
final int[] idxs = new int[maxFreeSlots];
for (int i = 0; i < maxFreeSlots; i++)
@@ -255,7 +249,7 @@
public void testInOrderFrees()
{
final int size = 4096;
- final int maxFreeSlots = BufferPool.CHUNK_SIZE / size;
+ final int maxFreeSlots = BufferPool.NORMAL_CHUNK_SIZE / size;
final int[] idxs = new int[maxFreeSlots];
for (int i = 0; i < maxFreeSlots; i++)
@@ -269,23 +263,23 @@
{
doTestRandomFrees(12345567878L);
- BufferPool.reset();
+ BufferPool.unsafeReset();
doTestRandomFrees(20452249587L);
- BufferPool.reset();
+ BufferPool.unsafeReset();
doTestRandomFrees(82457252948L);
- BufferPool.reset();
+ BufferPool.unsafeReset();
doTestRandomFrees(98759284579L);
- BufferPool.reset();
+ BufferPool.unsafeReset();
doTestRandomFrees(19475257244L);
}
private void doTestRandomFrees(long seed)
{
final int size = 4096;
- final int maxFreeSlots = BufferPool.CHUNK_SIZE / size;
+ final int maxFreeSlots = BufferPool.NORMAL_CHUNK_SIZE / size;
final int[] idxs = new int[maxFreeSlots];
for (int i = 0; i < maxFreeSlots; i++)
@@ -309,13 +303,13 @@
List<ByteBuffer> buffers = new ArrayList<>(maxFreeSlots);
for (int i = 0; i < maxFreeSlots; i++)
{
- buffers.add(BufferPool.get(size));
+ buffers.add(BufferPool.get(size, BufferType.OFF_HEAP));
}
- BufferPool.Chunk chunk = BufferPool.currentChunk();
+ BufferPool.Chunk chunk = BufferPool.unsafeCurrentChunk();
assertFalse(chunk.isFree());
- int freeSize = BufferPool.CHUNK_SIZE - maxFreeSlots * size;
+ int freeSize = BufferPool.NORMAL_CHUNK_SIZE - maxFreeSlots * size;
assertEquals(freeSize, chunk.free());
for (int i : toReleaseIdxs)
@@ -347,38 +341,39 @@
List<ByteBuffer> buffers = new ArrayList<>(sizes.length);
for (int i = 0; i < sizes.length; i++)
{
- ByteBuffer buffer = BufferPool.get(sizes[i]);
+ ByteBuffer buffer = BufferPool.get(sizes[i], BufferType.OFF_HEAP);
assertNotNull(buffer);
assertTrue(buffer.capacity() >= sizes[i]);
buffers.add(buffer);
- sum += BufferPool.currentChunk().roundUp(buffer.capacity());
+ sum += BufferPool.unsafeCurrentChunk().roundUp(buffer.capacity());
}
// else the test will fail, adjust sizes as required
assertTrue(sum <= BufferPool.GlobalPool.MACRO_CHUNK_SIZE);
- BufferPool.Chunk chunk = BufferPool.currentChunk();
+ BufferPool.Chunk chunk = BufferPool.unsafeCurrentChunk();
assertNotNull(chunk);
Random rnd = new Random();
rnd.setSeed(298347529L);
- while (!buffers.isEmpty())
+ while (buffers.size() > 1)
{
int index = rnd.nextInt(buffers.size());
ByteBuffer buffer = buffers.remove(index);
BufferPool.put(buffer);
}
+ BufferPool.put(buffers.remove(0));
- assertEquals(null, BufferPool.currentChunk());
+ assertEquals(null, BufferPool.unsafeCurrentChunk());
assertEquals(0, chunk.free());
}
@Test
public void testChunkExhausted()
{
- final int size = BufferPool.CHUNK_SIZE / 64; // 1kbit
+ final int size = BufferPool.NORMAL_CHUNK_SIZE / 64; // 1kbit
int[] sizes = new int[128];
Arrays.fill(sizes, size);
@@ -386,7 +381,7 @@
List<ByteBuffer> buffers = new ArrayList<>(sizes.length);
for (int i = 0; i < sizes.length; i++)
{
- ByteBuffer buffer = BufferPool.get(sizes[i]);
+ ByteBuffer buffer = BufferPool.get(sizes[i], BufferType.OFF_HEAP);
assertNotNull(buffer);
assertTrue(buffer.capacity() >= sizes[i]);
buffers.add(buffer);
@@ -397,7 +392,7 @@
// else the test will fail, adjust sizes as required
assertTrue(sum <= BufferPool.GlobalPool.MACRO_CHUNK_SIZE);
- BufferPool.Chunk chunk = BufferPool.currentChunk();
+ BufferPool.Chunk chunk = BufferPool.unsafeCurrentChunk();
assertNotNull(chunk);
for (int i = 0; i < sizes.length; i++)
@@ -405,7 +400,7 @@
BufferPool.put(buffers.get(i));
}
- assertEquals(null, BufferPool.currentChunk());
+ assertEquals(null, BufferPool.unsafeCurrentChunk());
assertEquals(0, chunk.free());
}
@@ -420,7 +415,7 @@
for (int i = 0; i < numBuffersInChunk; i++)
{
- ByteBuffer buffer = BufferPool.get(size);
+ ByteBuffer buffer = BufferPool.get(size, BufferType.OFF_HEAP);
buffers.add(buffer);
addresses.add(MemoryUtil.getAddress(buffer));
}
@@ -432,7 +427,7 @@
for (int i = 0; i < numBuffersInChunk; i++)
{
- ByteBuffer buffer = BufferPool.get(size);
+ ByteBuffer buffer = BufferPool.get(size, BufferType.OFF_HEAP);
assertNotNull(buffer);
assertEquals(size, buffer.capacity());
addresses.remove(MemoryUtil.getAddress(buffer));
@@ -502,12 +497,12 @@
private void checkBuffer(int size)
{
- ByteBuffer buffer = BufferPool.get(size);
+ ByteBuffer buffer = BufferPool.get(size, BufferType.OFF_HEAP);
assertEquals(size, buffer.capacity());
- if (size > 0 && size < BufferPool.CHUNK_SIZE)
+ if (size > 0 && size < BufferPool.NORMAL_CHUNK_SIZE)
{
- BufferPool.Chunk chunk = BufferPool.currentChunk();
+ BufferPool.Chunk chunk = BufferPool.unsafeCurrentChunk();
assertNotNull(chunk);
assertEquals(chunk.capacity(), chunk.free() + chunk.roundUp(size));
}
@@ -530,7 +525,7 @@
for (int size : sizes)
{
- ByteBuffer buffer = BufferPool.get(size);
+ ByteBuffer buffer = BufferPool.get(size, BufferType.OFF_HEAP);
assertEquals(size, buffer.capacity());
buffers.add(buffer);
@@ -549,19 +544,19 @@
private void checkBufferWithGivenSlots(int size, long freeSlots)
{
//first allocate to make sure there is a chunk
- ByteBuffer buffer = BufferPool.get(size);
+ ByteBuffer buffer = BufferPool.get(size, BufferType.OFF_HEAP);
// now get the current chunk and override the free slots mask
- BufferPool.Chunk chunk = BufferPool.currentChunk();
+ BufferPool.Chunk chunk = BufferPool.unsafeCurrentChunk();
assertNotNull(chunk);
long oldFreeSlots = chunk.setFreeSlots(freeSlots);
// now check we can still get the buffer with the free slots mask changed
- ByteBuffer buffer2 = BufferPool.get(size);
+ ByteBuffer buffer2 = BufferPool.get(size, BufferType.OFF_HEAP);
assertEquals(size, buffer.capacity());
BufferPool.put(buffer2);
- // reset the free slots
+ // unsafeReset the free slots
chunk.setFreeSlots(oldFreeSlots);
BufferPool.put(buffer);
}
@@ -569,7 +564,7 @@
@Test
public void testZeroSizeRequest()
{
- ByteBuffer buffer = BufferPool.get(0);
+ ByteBuffer buffer = BufferPool.get(0, BufferType.OFF_HEAP);
assertNotNull(buffer);
assertEquals(0, buffer.capacity());
BufferPool.put(buffer);
@@ -578,35 +573,7 @@
@Test(expected = IllegalArgumentException.class)
public void testNegativeSizeRequest()
{
- BufferPool.get(-1);
- }
-
- @Test
- public void testBufferPoolDisabled()
- {
- BufferPool.DISABLED = true;
- BufferPool.ALLOCATE_ON_HEAP_WHEN_EXAHUSTED = true;
- ByteBuffer buffer = BufferPool.get(1024);
- assertEquals(0, BufferPool.numChunks());
- assertNotNull(buffer);
- assertEquals(1024, buffer.capacity());
- assertFalse(buffer.isDirect());
- assertNotNull(buffer.array());
- BufferPool.put(buffer);
- assertEquals(0, BufferPool.numChunks());
-
- BufferPool.ALLOCATE_ON_HEAP_WHEN_EXAHUSTED = false;
- buffer = BufferPool.get(1024);
- assertEquals(0, BufferPool.numChunks());
- assertNotNull(buffer);
- assertEquals(1024, buffer.capacity());
- assertTrue(buffer.isDirect());
- BufferPool.put(buffer);
- assertEquals(0, BufferPool.numChunks());
-
- // clean-up
- BufferPool.DISABLED = false;
- BufferPool.ALLOCATE_ON_HEAP_WHEN_EXAHUSTED = true;
+ BufferPool.get(-1, BufferType.OFF_HEAP);
}
@Test
@@ -722,7 +689,7 @@
for (int j = 0; j < threadSizes.length; j++)
{
- ByteBuffer buffer = BufferPool.get(threadSizes[j]);
+ ByteBuffer buffer = BufferPool.get(threadSizes[j], BufferType.OFF_HEAP);
assertNotNull(buffer);
assertEquals(threadSizes[j], buffer.capacity());
@@ -791,13 +758,13 @@
int sum = 0;
for (int i = 0; i < sizes.length; i++)
{
- buffers[i] = BufferPool.get(sizes[i]);
+ buffers[i] = BufferPool.get(sizes[i], BufferType.OFF_HEAP);
assertNotNull(buffers[i]);
assertEquals(sizes[i], buffers[i].capacity());
- sum += BufferPool.currentChunk().roundUp(buffers[i].capacity());
+ sum += BufferPool.unsafeCurrentChunk().roundUp(buffers[i].capacity());
}
- final BufferPool.Chunk chunk = BufferPool.currentChunk();
+ final BufferPool.Chunk chunk = BufferPool.unsafeCurrentChunk();
assertNotNull(chunk);
assertFalse(chunk.isFree());
@@ -819,7 +786,7 @@
{
try
{
- assertNotSame(chunk, BufferPool.currentChunk());
+ assertNotSame(chunk, BufferPool.unsafeCurrentChunk());
BufferPool.put(buffer);
}
catch (AssertionError ex)
@@ -849,10 +816,10 @@
System.gc();
System.gc();
- assertTrue(BufferPool.currentChunk().isFree());
+ assertTrue(BufferPool.unsafeCurrentChunk().isFree());
//make sure the main thread can still allocate buffers
- ByteBuffer buffer = BufferPool.get(sizes[0]);
+ ByteBuffer buffer = BufferPool.get(sizes[0], BufferType.OFF_HEAP);
assertNotNull(buffer);
assertEquals(sizes[0], buffer.capacity());
BufferPool.put(buffer);
diff --git a/test/unit/org/apache/cassandra/utils/memory/NativeAllocatorTest.java b/test/unit/org/apache/cassandra/utils/memory/NativeAllocatorTest.java
index b636bf7..e8a7b54 100644
--- a/test/unit/org/apache/cassandra/utils/memory/NativeAllocatorTest.java
+++ b/test/unit/org/apache/cassandra/utils/memory/NativeAllocatorTest.java
@@ -24,7 +24,7 @@
import com.google.common.util.concurrent.Uninterruptibles;
import org.junit.Test;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.cassandra.utils.concurrent.OpOrder;
public class NativeAllocatorTest
diff --git a/test/unit/org/apache/cassandra/utils/obs/OffHeapBitSetTest.java b/test/unit/org/apache/cassandra/utils/obs/OffHeapBitSetTest.java
new file mode 100644
index 0000000..49b4c94
--- /dev/null
+++ b/test/unit/org/apache/cassandra/utils/obs/OffHeapBitSetTest.java
@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.utils.obs;
+
+import java.io.ByteArrayInputStream;
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.util.List;
+import java.util.Random;
+
+import com.google.common.collect.Lists;
+import org.apache.cassandra.io.util.DataOutputBuffer;
+import org.junit.Assert;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+public class OffHeapBitSetTest
+{
+ private static final Random random = new Random();
+
+ static void compare(IBitSet bs, IBitSet newbs)
+ {
+ assertEquals(bs.capacity(), newbs.capacity());
+ for (long i = 0; i < bs.capacity(); i++)
+ Assert.assertEquals(bs.get(i), newbs.get(i));
+ }
+
+ private void testOffHeapSerialization(boolean oldBfFormat) throws IOException
+ {
+ try (OffHeapBitSet bs = new OffHeapBitSet(100000))
+ {
+ for (long i = 0; i < bs.capacity(); i++)
+ if (random.nextBoolean())
+ bs.set(i);
+
+ DataOutputBuffer out = new DataOutputBuffer();
+ if (oldBfFormat)
+ bs.serializeOldBfFormat(out);
+ else
+ bs.serialize(out);
+
+ DataInputStream in = new DataInputStream(new ByteArrayInputStream(out.getData()));
+ try (OffHeapBitSet newbs = OffHeapBitSet.deserialize(in, oldBfFormat))
+ {
+ compare(bs, newbs);
+ }
+ }
+ }
+
+ @Test
+ public void testSerialization() throws IOException
+ {
+ testOffHeapSerialization(true);
+ testOffHeapSerialization(false);
+ }
+
+ @Test
+ public void testBitSetGetClear()
+ {
+ int size = Integer.MAX_VALUE / 4000;
+ try (OffHeapBitSet bs = new OffHeapBitSet(size))
+ {
+ List<Integer> randomBits = Lists.newArrayList();
+ for (int i = 0; i < 10; i++)
+ randomBits.add(random.nextInt(size));
+
+ for (long randomBit : randomBits)
+ bs.set(randomBit);
+
+ for (long randomBit : randomBits)
+ assertEquals(true, bs.get(randomBit));
+
+ for (long randomBit : randomBits)
+ bs.clear(randomBit);
+
+ for (long randomBit : randomBits)
+ assertEquals(false, bs.get(randomBit));
+ }
+ }
+
+ @Test(expected = UnsupportedOperationException.class)
+ public void testUnsupportedLargeSize()
+ {
+ long size = 64L * Integer.MAX_VALUE + 1; // Max size 16G * 8 bits
+ OffHeapBitSet bs = new OffHeapBitSet(size);
+ }
+
+ @Test
+ public void testInvalidIndex()
+ {
+ OffHeapBitSet bs = new OffHeapBitSet(10);
+ int invalidIdx[] = {-1, 64, 1000};
+
+ for (int i : invalidIdx)
+ {
+ try
+ {
+ bs.set(i);
+ }
+ catch (AssertionError e)
+ {
+ assertTrue(e.getMessage().startsWith("Illegal bounds"));
+ continue;
+ }
+ fail(String.format("expect exception for index %d", i));
+ }
+
+ for (int i : invalidIdx)
+ {
+ try
+ {
+ bs.get(i);
+ }
+ catch (AssertionError e)
+ {
+ assertTrue(e.getMessage().startsWith("Illegal bounds"));
+ continue;
+ }
+ fail(String.format("expect exception for index %d", i));
+ }
+ }
+}
diff --git a/test/unit/org/apache/cassandra/utils/progress/jmx/LegacyJMXProgressSupportTest.java b/test/unit/org/apache/cassandra/utils/progress/jmx/LegacyJMXProgressSupportTest.java
deleted file mode 100644
index 70fb5cc..0000000
--- a/test/unit/org/apache/cassandra/utils/progress/jmx/LegacyJMXProgressSupportTest.java
+++ /dev/null
@@ -1,118 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.utils.progress.jmx;
-
-import java.util.Optional;
-import java.util.UUID;
-
-import org.junit.Test;
-
-import org.apache.cassandra.dht.Murmur3Partitioner;
-import org.apache.cassandra.dht.Range;
-import org.apache.cassandra.dht.Token;
-import org.apache.cassandra.service.ActiveRepairService;
-import org.apache.cassandra.utils.progress.ProgressEvent;
-import org.apache.cassandra.utils.progress.ProgressEventType;
-
-import static org.junit.Assert.*;
-
-
-public class LegacyJMXProgressSupportTest
-{
-
- @Test
- public void testSessionSuccess()
- {
- int cmd = 321;
- String message = String.format("Repair session %s for range %s finished", UUID.randomUUID(),
- new Range<Token>(new Murmur3Partitioner.LongToken(3), new Murmur3Partitioner.LongToken(4)));
- Optional<int[]> result = LegacyJMXProgressSupport.getLegacyUserdata(String.format("repair:%d", cmd),
- new ProgressEvent(ProgressEventType.PROGRESS, 2, 10, message));
- assertTrue(result.isPresent());
- assertArrayEquals(new int[]{ cmd, ActiveRepairService.Status.SESSION_SUCCESS.ordinal() }, result.get());
- }
-
- @Test
- public void testSessionFailed()
- {
- int cmd = 321;
- String message = String.format("Repair session %s for range %s failed with error %s", UUID.randomUUID(),
- new Range<Token>(new Murmur3Partitioner.LongToken(3), new Murmur3Partitioner.LongToken(4)).toString(),
- new RuntimeException("error"));
- Optional<int[]> result = LegacyJMXProgressSupport.getLegacyUserdata(String.format("repair:%d", cmd),
- new ProgressEvent(ProgressEventType.PROGRESS, 2, 10, message));
- assertTrue(result.isPresent());
- assertArrayEquals(new int[]{ cmd, ActiveRepairService.Status.SESSION_FAILED.ordinal() }, result.get());
- }
-
- @Test
- public void testStarted()
- {
- int cmd = 321;
- Optional<int[]> result = LegacyJMXProgressSupport.getLegacyUserdata(String.format("repair:%d", cmd),
- new ProgressEvent(ProgressEventType.START,
- 0, 100, "bla"));
- assertTrue(result.isPresent());
- assertArrayEquals(new int[]{ cmd, ActiveRepairService.Status.STARTED.ordinal() }, result.get());
- }
-
- @Test
- public void testFinished()
- {
- int cmd = 321;
- Optional<int[]> result = LegacyJMXProgressSupport.getLegacyUserdata(String.format("repair:%d", cmd),
- new ProgressEvent(ProgressEventType.COMPLETE,
- 2, 10, "bla"));
- assertTrue(result.isPresent());
- assertArrayEquals(new int[]{ cmd, ActiveRepairService.Status.FINISHED.ordinal() }, result.get());
- }
-
- /*
- States not mapped to the legacy notification
- */
- @Test
- public void testNone()
- {
- int cmd = 33;
- Optional<int[]> result = LegacyJMXProgressSupport.getLegacyUserdata(String.format("repair:%d", cmd),
- new ProgressEvent(ProgressEventType.ERROR, 2, 10, "bla"));
- assertFalse(result.isPresent());
-
- cmd = 33;
- result = LegacyJMXProgressSupport.getLegacyUserdata(String.format("repair:%d", cmd),
- new ProgressEvent(ProgressEventType.SUCCESS, 2, 10, "bla"));
- assertFalse(result.isPresent());
-
- cmd = 43;
- result = LegacyJMXProgressSupport.getLegacyUserdata(String.format("repair:%d", cmd),
- new ProgressEvent(ProgressEventType.PROGRESS, 2, 10, "bla"));
- assertFalse(result.isPresent());
-
- cmd = 1;
- result = LegacyJMXProgressSupport.getLegacyUserdata(String.format("repair:%d", cmd),
- new ProgressEvent(ProgressEventType.ABORT, 2, 10, "bla"));
- assertFalse(result.isPresent());
-
- cmd = 9;
- result = LegacyJMXProgressSupport.getLegacyUserdata(String.format("repair:%d", cmd),
- new ProgressEvent(ProgressEventType.NOTIFICATION, 2, 10, "bla"));
- assertFalse(result.isPresent());
- }
-
-}
diff --git a/test/unit/org/apache/cassandra/utils/streamhist/StreamingTombstoneHistogramBuilderTest.java b/test/unit/org/apache/cassandra/utils/streamhist/StreamingTombstoneHistogramBuilderTest.java
new file mode 100755
index 0000000..596c4d7
--- /dev/null
+++ b/test/unit/org/apache/cassandra/utils/streamhist/StreamingTombstoneHistogramBuilderTest.java
@@ -0,0 +1,404 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.utils.streamhist;
+
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.IntStream;
+
+import org.junit.Test;
+
+import org.apache.cassandra.db.rows.Cell;
+import org.apache.cassandra.io.util.DataInputBuffer;
+import org.apache.cassandra.io.util.DataOutputBuffer;
+import org.psjava.util.AssertStatus;
+import org.quicktheories.core.Gen;
+
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.quicktheories.QuickTheory.qt;
+import static org.quicktheories.generators.SourceDSL.integers;
+import static org.quicktheories.generators.SourceDSL.lists;
+
+public class StreamingTombstoneHistogramBuilderTest
+{
+ @Test
+ public void testFunction() throws Exception
+ {
+ StreamingTombstoneHistogramBuilder builder = new StreamingTombstoneHistogramBuilder(5, 0, 1);
+ int[] samples = new int[]{ 23, 19, 10, 16, 36, 2, 9, 32, 30, 45 };
+
+ // add 7 points to histogram of 5 bins
+ for (int i = 0; i < 7; i++)
+ {
+ builder.update(samples[i]);
+ }
+
+ // should end up (2,1),(9.5,2),(17.5,2),(23,1),(36,1)
+ Map<Double, Long> expected1 = new LinkedHashMap<Double, Long>(5);
+ expected1.put(2.0, 1L);
+ expected1.put(9.0, 2L);
+ expected1.put(17.0, 2L);
+ expected1.put(23.0, 1L);
+ expected1.put(36.0, 1L);
+
+ Iterator<Map.Entry<Double, Long>> expectedItr = expected1.entrySet().iterator();
+ TombstoneHistogram hist = builder.build();
+ hist.forEach((point, value) ->
+ {
+ Map.Entry<Double, Long> entry = expectedItr.next();
+ assertEquals(entry.getKey(), point, 0.01);
+ assertEquals(entry.getValue().longValue(), value);
+ });
+
+ // sum test
+ assertEquals(3.5, hist.sum(15), 0.01);
+ // sum test (b > max(hist))
+ assertEquals(7.0, hist.sum(50), 0.01);
+ }
+
+ @Test
+ public void testSerDe() throws Exception
+ {
+ StreamingTombstoneHistogramBuilder builder = new StreamingTombstoneHistogramBuilder(5, 0, 1);
+ int[] samples = new int[]{ 23, 19, 10, 16, 36, 2, 9 };
+
+ // add 7 points to histogram of 5 bins
+ for (int i = 0; i < samples.length; i++)
+ {
+ builder.update(samples[i]);
+ }
+ TombstoneHistogram hist = builder.build();
+ DataOutputBuffer out = new DataOutputBuffer();
+ TombstoneHistogram.serializer.serialize(hist, out);
+ byte[] bytes = out.toByteArray();
+
+ TombstoneHistogram deserialized = TombstoneHistogram.serializer.deserialize(new DataInputBuffer(bytes));
+
+ // deserialized histogram should have following values
+ Map<Double, Long> expected1 = new LinkedHashMap<Double, Long>(5);
+ expected1.put(2.0, 1L);
+ expected1.put(9.0, 2L);
+ expected1.put(17.0, 2L);
+ expected1.put(23.0, 1L);
+ expected1.put(36.0, 1L);
+
+ Iterator<Map.Entry<Double, Long>> expectedItr = expected1.entrySet().iterator();
+ deserialized.forEach((point, value) ->
+ {
+ Map.Entry<Double, Long> entry = expectedItr.next();
+ assertEquals(entry.getKey(), point, 0.01);
+ assertEquals(entry.getValue().longValue(), value);
+ });
+ }
+
+
+ @Test
+ public void testNumericTypes() throws Exception
+ {
+ StreamingTombstoneHistogramBuilder builder = new StreamingTombstoneHistogramBuilder(5, 0, 1);
+
+ builder.update(2);
+ builder.update(2);
+ builder.update(2);
+ builder.update(2, Integer.MAX_VALUE); // To check that value overflow is handled correctly
+ TombstoneHistogram hist = builder.build();
+ Map<Integer, Integer> asMap = asMap(hist);
+
+ assertEquals(1, asMap.size());
+ assertEquals(Integer.MAX_VALUE, asMap.get(2).intValue());
+
+ //Make sure it's working with Serde
+ DataOutputBuffer out = new DataOutputBuffer();
+ TombstoneHistogram.serializer.serialize(hist, out);
+ byte[] bytes = out.toByteArray();
+
+ TombstoneHistogram deserialized = TombstoneHistogram.serializer.deserialize(new DataInputBuffer(bytes));
+
+ asMap = asMap(deserialized);
+ assertEquals(1, deserialized.size());
+ assertEquals(Integer.MAX_VALUE, asMap.get(2).intValue());
+ }
+
+ @Test
+ public void testOverflow() throws Exception
+ {
+ StreamingTombstoneHistogramBuilder builder = new StreamingTombstoneHistogramBuilder(5, 10, 1);
+ int[] samples = new int[]{ 23, 19, 10, 16, 36, 2, 9, 32, 30, 45, 31,
+ 32, 32, 33, 34, 35, 70, 78, 80, 90, 100,
+ 32, 32, 33, 34, 35, 70, 78, 80, 90, 100
+ };
+
+ // Hit the spool cap, force it to make bins
+ for (int i = 0; i < samples.length; i++)
+ {
+ builder.update(samples[i]);
+ }
+
+ assertEquals(5, builder.build().size());
+ }
+
+ @Test
+ public void testRounding() throws Exception
+ {
+ StreamingTombstoneHistogramBuilder builder = new StreamingTombstoneHistogramBuilder(5, 10, 60);
+ int[] samples = new int[]{ 59, 60, 119, 180, 181, 300 }; // 60, 60, 120, 180, 240, 300
+ for (int i = 0; i < samples.length; i++)
+ builder.update(samples[i]);
+ TombstoneHistogram hist = builder.build();
+ assertEquals(hist.size(), 5);
+ assertEquals(asMap(hist).get(60).intValue(), 2);
+ assertEquals(asMap(hist).get(120).intValue(), 1);
+ }
+
+ @Test
+ public void testLargeValues() throws Exception
+ {
+ StreamingTombstoneHistogramBuilder builder = new StreamingTombstoneHistogramBuilder(5, 0, 1);
+ IntStream.range(Integer.MAX_VALUE - 30, Integer.MAX_VALUE).forEach(builder::update);
+ }
+
+ @Test
+ public void testLargeDeletionTimesAndLargeValuesDontCauseOverflow()
+ {
+ qt().forAll(streamingTombstoneHistogramBuilderGen(1000, 300000, 60),
+ lists().of(integers().from(0).upTo(Cell.MAX_DELETION_TIME)).ofSize(300),
+ lists().of(integers().allPositive()).ofSize(300))
+ .checkAssert(this::updateHistogramAndCheckAllBucketsArePositive);
+ }
+
+ private void updateHistogramAndCheckAllBucketsArePositive(StreamingTombstoneHistogramBuilder histogramBuilder, List<Integer> keys, List<Integer> values)
+ {
+ for (int i = 0; i < keys.size(); i++)
+ {
+ histogramBuilder.update(keys.get(i), values.get(i));
+ }
+
+ TombstoneHistogram histogram = histogramBuilder.build();
+ for (Map.Entry<Integer, Integer> buckets : asMap(histogram).entrySet())
+ {
+ assertTrue("Invalid bucket key", buckets.getKey() >= 0);
+ assertTrue("Invalid bucket value", buckets.getValue() >= 0);
+ }
+ }
+
+ @Test
+ public void testThatPointIsNotMissedBecauseOfRoundingToNoDeletionTime() throws Exception
+ {
+ int pointThatRoundedToNoDeletion = Cell.NO_DELETION_TIME - 2;
+ assert pointThatRoundedToNoDeletion + pointThatRoundedToNoDeletion % 3 == Cell.NO_DELETION_TIME : "test data should be valid";
+
+ StreamingTombstoneHistogramBuilder builder = new StreamingTombstoneHistogramBuilder(5, 10, 3);
+ builder.update(pointThatRoundedToNoDeletion);
+
+ TombstoneHistogram histogram = builder.build();
+
+ Map<Integer, Integer> integerIntegerMap = asMap(histogram);
+ assertEquals(integerIntegerMap.size(), 1);
+ assertEquals(integerIntegerMap.get(Cell.MAX_DELETION_TIME).intValue(), 1);
+ }
+
+ @Test
+ public void testInvalidArguments()
+ {
+ assertThatThrownBy(() -> new StreamingTombstoneHistogramBuilder(5, 10, 0)).hasMessage("Invalid arguments: maxBinSize:5 maxSpoolSize:10 delta:0");
+ assertThatThrownBy(() -> new StreamingTombstoneHistogramBuilder(5, 10, -1)).hasMessage("Invalid arguments: maxBinSize:5 maxSpoolSize:10 delta:-1");
+ assertThatThrownBy(() -> new StreamingTombstoneHistogramBuilder(5, -1, 60)).hasMessage("Invalid arguments: maxBinSize:5 maxSpoolSize:-1 delta:60");
+ assertThatThrownBy(() -> new StreamingTombstoneHistogramBuilder(-1, 10, 60)).hasMessage("Invalid arguments: maxBinSize:-1 maxSpoolSize:10 delta:60");
+ assertThatThrownBy(() -> new StreamingTombstoneHistogramBuilder(0, 10, 60)).hasMessage("Invalid arguments: maxBinSize:0 maxSpoolSize:10 delta:60");
+ }
+
+ @Test
+ public void testSpool()
+ {
+ StreamingTombstoneHistogramBuilder.Spool spool = new StreamingTombstoneHistogramBuilder.Spool(8);
+ assertTrue(spool.tryAddOrAccumulate(5, 1));
+ assertSpool(spool, 5, 1);
+ assertTrue(spool.tryAddOrAccumulate(5, 3));
+ assertSpool(spool, 5, 4);
+
+ assertTrue(spool.tryAddOrAccumulate(10, 1));
+ assertSpool(spool, 5, 4,
+ 10, 1);
+
+ assertTrue(spool.tryAddOrAccumulate(12, 1));
+ assertTrue(spool.tryAddOrAccumulate(14, 1));
+ assertTrue(spool.tryAddOrAccumulate(16, 1));
+ assertSpool(spool, 5, 4,
+ 10, 1,
+ 12, 1,
+ 14, 1,
+ 16, 1);
+
+ assertTrue(spool.tryAddOrAccumulate(18, 1));
+ assertTrue(spool.tryAddOrAccumulate(20, 1));
+ assertTrue(spool.tryAddOrAccumulate(30, 1));
+ assertSpool(spool, 5, 4,
+ 10, 1,
+ 12, 1,
+ 14, 1,
+ 16, 1,
+ 18, 1,
+ 20, 1,
+ 30, 1);
+
+ assertTrue(spool.tryAddOrAccumulate(16, 5));
+ assertTrue(spool.tryAddOrAccumulate(12, 4));
+ assertTrue(spool.tryAddOrAccumulate(18, 9));
+ assertSpool(spool,
+ 5, 4,
+ 10, 1,
+ 12, 5,
+ 14, 1,
+ 16, 6,
+ 18, 10,
+ 20, 1,
+ 30, 1);
+
+ assertTrue(spool.tryAddOrAccumulate(99, 5));
+ }
+
+ @Test
+ public void testDataHolder()
+ {
+ StreamingTombstoneHistogramBuilder.DataHolder dataHolder = new StreamingTombstoneHistogramBuilder.DataHolder(4, 1);
+ assertFalse(dataHolder.isFull());
+ assertEquals(0, dataHolder.size());
+
+ assertTrue(dataHolder.addValue(4, 1));
+ assertDataHolder(dataHolder,
+ 4, 1);
+
+ assertFalse(dataHolder.addValue(4, 1));
+ assertDataHolder(dataHolder,
+ 4, 2);
+
+ assertTrue(dataHolder.addValue(7, 1));
+ assertDataHolder(dataHolder,
+ 4, 2,
+ 7, 1);
+
+ assertFalse(dataHolder.addValue(7, 1));
+ assertDataHolder(dataHolder,
+ 4, 2,
+ 7, 2);
+
+ assertTrue(dataHolder.addValue(5, 1));
+ assertDataHolder(dataHolder,
+ 4, 2,
+ 5, 1,
+ 7, 2);
+
+ assertFalse(dataHolder.addValue(5, 1));
+ assertDataHolder(dataHolder,
+ 4, 2,
+ 5, 2,
+ 7, 2);
+
+ assertTrue(dataHolder.addValue(2, 1));
+ assertDataHolder(dataHolder,
+ 2, 1,
+ 4, 2,
+ 5, 2,
+ 7, 2);
+ assertTrue(dataHolder.isFull());
+
+ // expect to merge [4,2]+[5,2]
+ dataHolder.mergeNearestPoints();
+ assertDataHolder(dataHolder,
+ 2, 1,
+ 4, 4,
+ 7, 2);
+
+ assertFalse(dataHolder.addValue(2, 1));
+ assertDataHolder(dataHolder,
+ 2, 2,
+ 4, 4,
+ 7, 2);
+
+ dataHolder.addValue(8, 1);
+ assertDataHolder(dataHolder,
+ 2, 2,
+ 4, 4,
+ 7, 2,
+ 8, 1);
+ assertTrue(dataHolder.isFull());
+
+ // expect to merge [7,2]+[8,1]
+ dataHolder.mergeNearestPoints();
+ assertDataHolder(dataHolder,
+ 2, 2,
+ 4, 4,
+ 7, 3);
+ }
+
+ private static void assertDataHolder(StreamingTombstoneHistogramBuilder.DataHolder dataHolder, int... pointValue)
+ {
+ assertEquals(pointValue.length / 2, dataHolder.size());
+
+ for (int i = 0; i < pointValue.length; i += 2)
+ {
+ int point = pointValue[i];
+ int expectedValue = pointValue[i + 1];
+ assertEquals(expectedValue, dataHolder.getValue(point));
+ }
+ }
+
+ /**
+ * Compare the contents of {@code spool} with the given collection of key-value pairs in {@code pairs}.
+ */
+ private static void assertSpool(StreamingTombstoneHistogramBuilder.Spool spool, int... pairs)
+ {
+ assertEquals(pairs.length / 2, spool.size);
+ Map<Integer, Integer> tests = new HashMap<>();
+ for (int i = 0; i < pairs.length; i += 2)
+ tests.put(pairs[i], pairs[i + 1]);
+
+ spool.forEach((k, v) -> {
+ Integer x = tests.remove(k);
+ assertNotNull("key " + k, x);
+ assertEquals(x.intValue(), v);
+ });
+ AssertStatus.assertTrue(tests.isEmpty());
+ }
+
+ private Map<Integer, Integer> asMap(TombstoneHistogram histogram)
+ {
+ Map<Integer, Integer> result = new HashMap<>();
+ histogram.forEach(result::put);
+ return result;
+ }
+
+ private Gen<StreamingTombstoneHistogramBuilder> streamingTombstoneHistogramBuilderGen(int maxBinSize, int maxSpoolSize, int maxRoundSeconds)
+ {
+ return positiveIntegerUpTo(maxBinSize).zip(integers().between(0, maxSpoolSize),
+ positiveIntegerUpTo(maxRoundSeconds),
+ StreamingTombstoneHistogramBuilder::new);
+ }
+
+ private Gen<Integer> positiveIntegerUpTo(int upperBound)
+ {
+ return integers().between(1, upperBound);
+ }
+}
diff --git a/test/unit/org/apache/cassandra/utils/vint/VIntCodingTest.java b/test/unit/org/apache/cassandra/utils/vint/VIntCodingTest.java
index f08b181..c7c3324 100644
--- a/test/unit/org/apache/cassandra/utils/vint/VIntCodingTest.java
+++ b/test/unit/org/apache/cassandra/utils/vint/VIntCodingTest.java
@@ -20,11 +20,14 @@
import java.io.ByteArrayOutputStream;
import java.io.DataOutputStream;
+import java.io.IOException;
+import io.netty.buffer.Unpooled;
import org.apache.cassandra.io.util.DataOutputBuffer;
+
import org.junit.Test;
-import junit.framework.Assert;
+import org.junit.Assert;
public class VIntCodingTest
{
@@ -82,4 +85,16 @@
Assert.assertEquals( 1, dob.buffer().remaining());
dob.close();
}
+
+ @Test
+ public void testByteBufWithNegativeNumber() throws IOException
+ {
+ int i = -1231238694;
+ try (DataOutputBuffer out = new DataOutputBuffer())
+ {
+ VIntCoding.writeUnsignedVInt(i, out);
+ long result = VIntCoding.getUnsignedVInt(out.buffer(), 0);
+ Assert.assertEquals(i, result);
+ }
+ }
}
diff --git a/tools/bin/auditlogviewer b/tools/bin/auditlogviewer
new file mode 100755
index 0000000..a6a7375
--- /dev/null
+++ b/tools/bin/auditlogviewer
@@ -0,0 +1,49 @@
+#!/bin/sh
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ "x$CASSANDRA_INCLUDE" = "x" ]; then
+ # Locations (in order) to use when searching for an include file.
+ for include in "`dirname "$0"`/cassandra.in.sh" \
+ "$HOME/.cassandra.in.sh" \
+ /usr/share/cassandra/cassandra.in.sh \
+ /usr/local/share/cassandra/cassandra.in.sh \
+ /opt/cassandra/cassandra.in.sh; do
+ if [ -r "$include" ]; then
+ . "$include"
+ break
+ fi
+ done
+elif [ -r "$CASSANDRA_INCLUDE" ]; then
+ . "$CASSANDRA_INCLUDE"
+fi
+
+if [ -z "$CLASSPATH" ]; then
+ echo "You must set the CLASSPATH var" >&2
+ exit 1
+fi
+
+if [ "x$MAX_HEAP_SIZE" = "x" ]; then
+ MAX_HEAP_SIZE="256M"
+fi
+
+"$JAVA" $JAVA_AGENT -ea -cp "$CLASSPATH" $JVM_OPTS -Xmx$MAX_HEAP_SIZE \
+ -Dcassandra.storagedir="$cassandra_storagedir" \
+ -Dlogback.configurationFile=logback-tools.xml \
+ org.apache.cassandra.tools.AuditLogViewer "$@"
+
+# vi:ai sw=4 ts=4 tw=0 et
diff --git a/tools/bin/auditlogviewer.bat b/tools/bin/auditlogviewer.bat
new file mode 100644
index 0000000..3b6bd81
--- /dev/null
+++ b/tools/bin/auditlogviewer.bat
@@ -0,0 +1,41 @@
+@REM
+@REM Licensed to the Apache Software Foundation (ASF) under one or more
+@REM contributor license agreements. See the NOTICE file distributed with
+@REM this work for additional information regarding copyright ownership.
+@REM The ASF licenses this file to You under the Apache License, Version 2.0
+@REM (the "License"); you may not use this file except in compliance with
+@REM the License. You may obtain a copy of the License at
+@REM
+@REM http://www.apache.org/licenses/LICENSE-2.0
+@REM
+@REM Unless required by applicable law or agreed to in writing, software
+@REM distributed under the License is distributed on an "AS IS" BASIS,
+@REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+@REM See the License for the specific language governing permissions and
+@REM limitations under the License.
+
+@echo off
+if "%OS%" == "Windows_NT" setlocal
+
+pushd "%~dp0"
+call cassandra.in.bat
+
+if NOT DEFINED CASSANDRA_MAIN set CASSANDRA_MAIN=org.apache.cassandra.tools.AuditLogViewer
+if NOT DEFINED JAVA_HOME goto :err
+
+REM ***** JAVA options *****
+set JAVA_OPTS=^
+ -Dlogback.configurationFile=logback-tools.xml
+
+set TOOLS_PARAMS=
+
+"%JAVA_HOME%\bin\java" %JAVA_OPTS% %CASSANDRA_PARAMS% -cp %CASSANDRA_CLASSPATH% "%CASSANDRA_MAIN%" %*
+goto finally
+
+:err
+echo JAVA_HOME environment variable must be set!
+pause
+
+:finally
+
+ENDLOCAL
diff --git a/tools/bin/cassandra-stress b/tools/bin/cassandra-stress
index 82a3eb5..a5821d3 100755
--- a/tools/bin/cassandra-stress
+++ b/tools/bin/cassandra-stress
@@ -32,18 +32,6 @@
. "$CASSANDRA_INCLUDE"
fi
-# Use JAVA_HOME if set, otherwise look for java in PATH
-if [ -x "$JAVA_HOME/bin/java" ]; then
- JAVA="$JAVA_HOME/bin/java"
-else
- JAVA="`which java`"
-fi
-
-if [ "x$JAVA" = "x" ]; then
- echo "Java executable not found (hint: set JAVA_HOME)" >&2
- exit 1
-fi
-
if [ -z "$CLASSPATH" ]; then
echo "You must set the CLASSPATH var" >&2
exit 1
diff --git a/tools/bin/cassandra-stressd b/tools/bin/cassandra-stressd
index 48fbef6..83f8006 100755
--- a/tools/bin/cassandra-stressd
+++ b/tools/bin/cassandra-stressd
@@ -33,18 +33,6 @@
. "$CASSANDRA_INCLUDE"
fi
-# Use JAVA_HOME if set, otherwise look for java in PATH
-if [ -x "$JAVA_HOME/bin/java" ]; then
- JAVA="$JAVA_HOME/bin/java"
-else
- JAVA="`which java`"
-fi
-
-if [ "x$JAVA" = "x" ]; then
- echo "Java executable not found (hint: set JAVA_HOME)" >&2
- exit 1
-fi
-
if [ -z "$CLASSPATH" ]; then
echo "You must set the CLASSPATH var" >&2
exit 1
diff --git a/tools/bin/cassandra.in.bat b/tools/bin/cassandra.in.bat
index 0fdd31a..4d395e8 100644
--- a/tools/bin/cassandra.in.bat
+++ b/tools/bin/cassandra.in.bat
@@ -39,7 +39,7 @@
:okClasspath
REM Include the build\classes\main directory so it works in development
-set CASSANDRA_CLASSPATH=%CLASSPATH%;%CASSANDRA_CONF%;"%CASSANDRA_HOME%\build\classes\main";"%CASSANDRA_HOME%\build\classes\thrift";"%CASSANDRA_HOME%\build\classes\stress"
+set CASSANDRA_CLASSPATH=%CLASSPATH%;%CASSANDRA_CONF%;"%CASSANDRA_HOME%\build\classes\main";"%CASSANDRA_HOME%\build\classes\stress";"%CASSANDRA_HOME%\build\classes\fqltool"
REM Add the default storage location. Can be overridden in conf\cassandra.yaml
set CASSANDRA_PARAMS=%CASSANDRA_PARAMS% "-Dcassandra.storagedir=%CASSANDRA_HOME%\data"
diff --git a/tools/bin/cassandra.in.sh b/tools/bin/cassandra.in.sh
index 004f394..bf1ecc4 100644
--- a/tools/bin/cassandra.in.sh
+++ b/tools/bin/cassandra.in.sh
@@ -23,13 +23,18 @@
CASSANDRA_CONF="$CASSANDRA_HOME/conf"
fi
+# The java classpath (required)
+CLASSPATH="$CASSANDRA_CONF"
+
# This can be the path to a jar file, or a directory containing the
# compiled classes. NOTE: This isn't needed by the startup script,
# it's just used here in constructing the classpath.
-cassandra_bin="$CASSANDRA_HOME/build/classes/main"
-cassandra_bin="$cassandra_bin:$CASSANDRA_HOME/build/classes/stress"
-cassandra_bin="$cassandra_bin:$CASSANDRA_HOME/build/classes/thrift"
-#cassandra_bin="$cassandra_home/build/cassandra.jar"
+if [ -d $CASSANDRA_HOME/build ] ; then
+ #cassandra_bin="$CASSANDRA_HOME/build/classes/main"
+ cassandra_bin=`ls -1 $CASSANDRA_HOME/build/apache-cassandra*.jar`
+ cassandra_bin="$cassandra_bin:$CASSANDRA_HOME/build/classes/stress:$CASSANDRA_HOME/build/classes/fqltool"
+ CLASSPATH="$CLASSPATH:$cassandra_bin"
+fi
# the default location for commitlogs, sstables, and saved caches
# if not set in cassandra.yaml
@@ -38,12 +43,89 @@
# JAVA_HOME can optionally be set here
#JAVA_HOME=/usr/local/jdk6
-# The java classpath (required)
-CLASSPATH="$CASSANDRA_CONF:$cassandra_bin"
-
for jar in "$CASSANDRA_HOME"/tools/lib/*.jar; do
CLASSPATH="$CLASSPATH:$jar"
done
for jar in "$CASSANDRA_HOME"/lib/*.jar; do
CLASSPATH="$CLASSPATH:$jar"
done
+
+
+#
+# Java executable and per-Java version JVM settings
+#
+
+# Use JAVA_HOME if set, otherwise look for java in PATH
+if [ -n "$JAVA_HOME" ]; then
+ # Why we can't have nice things: Solaris combines x86 and x86_64
+ # installations in the same tree, using an unconventional path for the
+ # 64bit JVM. Since we prefer 64bit, search the alternate path first,
+ # (see https://issues.apache.org/jira/browse/CASSANDRA-4638).
+ for java in "$JAVA_HOME"/bin/amd64/java "$JAVA_HOME"/bin/java; do
+ if [ -x "$java" ]; then
+ JAVA="$java"
+ break
+ fi
+ done
+else
+ JAVA=java
+fi
+
+if [ -z $JAVA ] ; then
+ echo Unable to find java executable. Check JAVA_HOME and PATH environment variables. >&2
+ exit 1;
+fi
+
+# Determine the sort of JVM we'll be running on.
+java_ver_output=`"${JAVA:-java}" -version 2>&1`
+jvmver=`echo "$java_ver_output" | grep '[openjdk|java] version' | awk -F'"' 'NR==1 {print $2}' | cut -d\- -f1`
+JVM_VERSION=${jvmver%_*}
+
+JAVA_VERSION=11
+if [ "$JVM_VERSION" = "1.8.0" ] ; then
+ JVM_PATCH_VERSION=${jvmver#*_}
+ if [ "$JVM_VERSION" \< "1.8" ] || [ "$JVM_VERSION" \> "1.8.2" ] ; then
+ echo "Cassandra 4.0 requires either Java 8 (update 151 or newer) or Java 11 (or newer). Java $JVM_VERSION is not supported."
+ exit 1;
+ fi
+ if [ "$JVM_PATCH_VERSION" -lt 151 ] ; then
+ echo "Cassandra 4.0 requires either Java 8 (update 151 or newer) or Java 11 (or newer). Java 8 update $JVM_PATCH_VERSION is not supported."
+ exit 1;
+ fi
+ JAVA_VERSION=8
+elif [ "$JVM_VERSION" \< "11" ] ; then
+ echo "Cassandra 4.0 requires either Java 8 (update 151 or newer) or Java 11 (or newer)."
+ exit 1;
+fi
+
+jvm=`echo "$java_ver_output" | grep -A 1 '[openjdk|java] version' | awk 'NR==2 {print $1}'`
+case "$jvm" in
+ OpenJDK)
+ JVM_VENDOR=OpenJDK
+ # this will be "64-Bit" or "32-Bit"
+ JVM_ARCH=`echo "$java_ver_output" | awk 'NR==3 {print $2}'`
+ ;;
+ "Java(TM)")
+ JVM_VENDOR=Oracle
+ # this will be "64-Bit" or "32-Bit"
+ JVM_ARCH=`echo "$java_ver_output" | awk 'NR==3 {print $3}'`
+ ;;
+ *)
+ # Help fill in other JVM values
+ JVM_VENDOR=other
+ JVM_ARCH=unknown
+ ;;
+esac
+
+# Read user-defined JVM options from jvm-server.options file
+JVM_OPTS_FILE=$CASSANDRA_CONF/jvm${jvmoptions_variant:--clients}.options
+if [ $JAVA_VERSION -ge 11 ] ; then
+ JVM_DEP_OPTS_FILE=$CASSANDRA_CONF/jvm11${jvmoptions_variant:--clients}.options
+else
+ JVM_DEP_OPTS_FILE=$CASSANDRA_CONF/jvm8${jvmoptions_variant:--clients}.options
+fi
+
+for opt in `grep "^-" $JVM_OPTS_FILE` `grep "^-" $JVM_DEP_OPTS_FILE`
+do
+ JVM_OPTS="$JVM_OPTS $opt"
+done
diff --git a/tools/bin/compaction-stress b/tools/bin/compaction-stress
index f169f2f..d531561 100755
--- a/tools/bin/compaction-stress
+++ b/tools/bin/compaction-stress
@@ -32,18 +32,6 @@
. "$CASSANDRA_INCLUDE"
fi
-# Use JAVA_HOME if set, otherwise look for java in PATH
-if [ -x "$JAVA_HOME/bin/java" ]; then
- JAVA="$JAVA_HOME/bin/java"
-else
- JAVA="`which java`"
-fi
-
-if [ "x$JAVA" = "x" ]; then
- echo "Java executable not found (hint: set JAVA_HOME)" >&2
- exit 1
-fi
-
if [ -z "$CLASSPATH" ]; then
echo "You must set the CLASSPATH var" >&2
exit 1
diff --git a/tools/bin/fqltool b/tools/bin/fqltool
new file mode 100755
index 0000000..dc49e50
--- /dev/null
+++ b/tools/bin/fqltool
@@ -0,0 +1,76 @@
+#!/bin/sh
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ "x$CASSANDRA_INCLUDE" = "x" ]; then
+ # Locations (in order) to use when searching for an include file.
+ for include in "`dirname "$0"`/cassandra.in.sh" \
+ "$HOME/.cassandra.in.sh" \
+ /usr/share/cassandra/cassandra.in.sh \
+ /usr/local/share/cassandra/cassandra.in.sh \
+ /opt/cassandra/cassandra.in.sh; do
+ if [ -r "$include" ]; then
+ . "$include"
+ break
+ fi
+ done
+elif [ -r "$CASSANDRA_INCLUDE" ]; then
+ . "$CASSANDRA_INCLUDE"
+fi
+
+if [ -z "$CASSANDRA_CONF" -o -z "$CLASSPATH" ]; then
+ echo "You must set the CASSANDRA_CONF and CLASSPATH vars" >&2
+ exit 1
+fi
+
+# Run cassandra-env.sh to pick up JMX_PORT
+if [ -f "$CASSANDRA_CONF/cassandra-env.sh" ]; then
+ JVM_OPTS_SAVE=$JVM_OPTS
+ MAX_HEAP_SIZE_SAVE=$MAX_HEAP_SIZE
+ . "$CASSANDRA_CONF/cassandra-env.sh"
+ MAX_HEAP_SIZE=$MAX_HEAP_SIZE_SAVE
+ JVM_OPTS=$JVM_OPTS_SAVE
+fi
+
+# JMX Port passed via cmd line args (-p 9999 / --port 9999 / --port=9999)
+# should override the value from cassandra-env.sh
+ARGS=""
+JVM_ARGS=""
+while true
+do
+ if [ "x" = "x$1" ]; then break; fi
+ case $1 in
+ -D*)
+ JVM_ARGS="$JVM_ARGS $1"
+ ;;
+ *)
+ ARGS="$ARGS $1"
+ ;;
+ esac
+ shift
+done
+
+if [ "x$MAX_HEAP_SIZE" = "x" ]; then
+ MAX_HEAP_SIZE="512m"
+fi
+
+"$JAVA" $JAVA_AGENT -ea -da:net.openhft... -cp "$CLASSPATH" $JVM_OPTS -Xmx$MAX_HEAP_SIZE \
+ -Dlog4j.configurationFile=log4j2-tools.xml \
+ $JVM_ARGS \
+ org.apache.cassandra.fqltool.FullQueryLogTool $ARGS
+
+# vi:ai sw=4 ts=4 tw=0 et
diff --git a/tools/bin/fqltool.bat b/tools/bin/fqltool.bat
new file mode 100644
index 0000000..acb6d1c
--- /dev/null
+++ b/tools/bin/fqltool.bat
@@ -0,0 +1,36 @@
+@REM
+@REM Licensed to the Apache Software Foundation (ASF) under one or more
+@REM contributor license agreements. See the NOTICE file distributed with
+@REM this work for additional information regarding copyright ownership.
+@REM The ASF licenses this file to You under the Apache License, Version 2.0
+@REM (the "License"); you may not use this file except in compliance with
+@REM the License. You may obtain a copy of the License at
+@REM
+@REM http://www.apache.org/licenses/LICENSE-2.0
+@REM
+@REM Unless required by applicable law or agreed to in writing, software
+@REM distributed under the License is distributed on an "AS IS" BASIS,
+@REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+@REM See the License for the specific language governing permissions and
+@REM limitations under the License.
+
+@echo off
+if "%OS%" == "Windows_NT" setlocal
+
+pushd "%~dp0"
+call cassandra.in.bat
+
+if NOT DEFINED JAVA_HOME goto :err
+
+set CASSANDRA_PARAMS=%CASSANDRA_PARAMS% -Dcassandra.logdir="%CASSANDRA_HOME%\logs"
+
+"%JAVA_HOME%\bin\java" -cp %CASSANDRA_CLASSPATH% %CASSANDRA_PARAMS% -Dlog4j.configurationFile=log4j2-tools.xml org.apache.cassandra.fqltool.FullQueryLogTool %*
+goto finally
+
+:err
+echo The JAVA_HOME environment variable must be set to run this program!
+pause
+
+:finally
+ENDLOCAL & set RC=%ERRORLEVEL%
+exit /B %RC%
diff --git a/tools/bin/sstabledump b/tools/bin/sstabledump
index 92ad5c5..0f4cfd3 100755
--- a/tools/bin/sstabledump
+++ b/tools/bin/sstabledump
@@ -32,18 +32,6 @@
. "$CASSANDRA_INCLUDE"
fi
-# Use JAVA_HOME if set, otherwise look for java in PATH
-if [ -x "$JAVA_HOME/bin/java" ]; then
- JAVA="$JAVA_HOME/bin/java"
-else
- JAVA="`which java`"
-fi
-
-if [ "x$JAVA" = "x" ]; then
- echo "Java executable not found (hint: set JAVA_HOME)" >&2
- exit 1
-fi
-
if [ -z "$CLASSPATH" ]; then
echo "You must set the CLASSPATH var" >&2
exit 1
diff --git a/tools/bin/sstableexpiredblockers b/tools/bin/sstableexpiredblockers
index c1c7e41..a843a64 100755
--- a/tools/bin/sstableexpiredblockers
+++ b/tools/bin/sstableexpiredblockers
@@ -32,18 +32,6 @@
. "$CASSANDRA_INCLUDE"
fi
-# Use JAVA_HOME if set, otherwise look for java in PATH
-if [ -x "$JAVA_HOME/bin/java" ]; then
- JAVA="$JAVA_HOME/bin/java"
-else
- JAVA="`which java`"
-fi
-
-if [ "x$JAVA" = "x" ]; then
- echo "Java executable not found (hint: set JAVA_HOME)" >&2
- exit 1
-fi
-
if [ -z "$CLASSPATH" ]; then
echo "You must set the CLASSPATH var" >&2
exit 1
diff --git a/tools/bin/sstablelevelreset b/tools/bin/sstablelevelreset
index ec602fd..b63fff7 100755
--- a/tools/bin/sstablelevelreset
+++ b/tools/bin/sstablelevelreset
@@ -32,18 +32,6 @@
. "$CASSANDRA_INCLUDE"
fi
-# Use JAVA_HOME if set, otherwise look for java in PATH
-if [ -x "$JAVA_HOME/bin/java" ]; then
- JAVA="$JAVA_HOME/bin/java"
-else
- JAVA="`which java`"
-fi
-
-if [ "x$JAVA" = "x" ]; then
- echo "Java executable not found (hint: set JAVA_HOME)" >&2
- exit 1
-fi
-
if [ -z "$CLASSPATH" ]; then
echo "You must set the CLASSPATH var" >&2
exit 1
diff --git a/tools/bin/sstablemetadata b/tools/bin/sstablemetadata
index addf39b..eb0d447 100755
--- a/tools/bin/sstablemetadata
+++ b/tools/bin/sstablemetadata
@@ -32,18 +32,6 @@
. "$CASSANDRA_INCLUDE"
fi
-# Use JAVA_HOME if set, otherwise look for java in PATH
-if [ -x "$JAVA_HOME/bin/java" ]; then
- JAVA="$JAVA_HOME/bin/java"
-else
- JAVA="`which java`"
-fi
-
-if [ "x$JAVA" = "x" ]; then
- echo "Java executable not found (hint: set JAVA_HOME)" >&2
- exit 1
-fi
-
if [ -z "$CLASSPATH" ]; then
echo "You must set the CLASSPATH var" >&2
exit 1
diff --git a/tools/bin/sstableofflinerelevel b/tools/bin/sstableofflinerelevel
index 4e34515..9e173a6 100755
--- a/tools/bin/sstableofflinerelevel
+++ b/tools/bin/sstableofflinerelevel
@@ -32,18 +32,6 @@
. "$CASSANDRA_INCLUDE"
fi
-# Use JAVA_HOME if set, otherwise look for java in PATH
-if [ -x "$JAVA_HOME/bin/java" ]; then
- JAVA="$JAVA_HOME/bin/java"
-else
- JAVA="`which java`"
-fi
-
-if [ "x$JAVA" = "x" ]; then
- echo "Java executable not found (hint: set JAVA_HOME)" >&2
- exit 1
-fi
-
if [ -z "$CLASSPATH" ]; then
echo "You must set the CLASSPATH var" >&2
exit 1
diff --git a/tools/bin/sstablerepairedset b/tools/bin/sstablerepairedset
index 225cd11..ecd7958 100755
--- a/tools/bin/sstablerepairedset
+++ b/tools/bin/sstablerepairedset
@@ -32,18 +32,6 @@
. "$CASSANDRA_INCLUDE"
fi
-# Use JAVA_HOME if set, otherwise look for java in PATH
-if [ -x "$JAVA_HOME/bin/java" ]; then
- JAVA="$JAVA_HOME/bin/java"
-else
- JAVA="`which java`"
-fi
-
-if [ "x$JAVA" = "x" ]; then
- echo "Java executable not found (hint: set JAVA_HOME)" >&2
- exit 1
-fi
-
if [ -z "$CLASSPATH" ]; then
echo "You must set the CLASSPATH var" >&2
exit 1
diff --git a/tools/bin/sstablesplit b/tools/bin/sstablesplit
index 037cb63..77d0a5f 100755
--- a/tools/bin/sstablesplit
+++ b/tools/bin/sstablesplit
@@ -32,18 +32,6 @@
. "$CASSANDRA_INCLUDE"
fi
-# Use JAVA_HOME if set, otherwise look for java in PATH
-if [ -x "$JAVA_HOME/bin/java" ]; then
- JAVA="$JAVA_HOME/bin/java"
-else
- JAVA="`which java`"
-fi
-
-if [ "x$JAVA" = "x" ]; then
- echo "Java executable not found (hint: set JAVA_HOME)" >&2
- exit 1
-fi
-
if [ -z "$CLASSPATH" ]; then
echo "You must set the CLASSPATH var" >&2
exit 1
diff --git a/tools/cqlstress-lwt-example.yaml b/tools/cqlstress-lwt-example.yaml
new file mode 100644
index 0000000..8f523be
--- /dev/null
+++ b/tools/cqlstress-lwt-example.yaml
@@ -0,0 +1,71 @@
+# Based on https://gist.github.com/tjake/8995058fed11d9921e31
+### DML ###
+
+# Keyspace Name
+keyspace: cqlstress_lwt_example
+
+# The CQL for creating a keyspace (optional if it already exists)
+keyspace_definition: |
+ CREATE KEYSPACE cqlstress_lwt_example WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 3};
+
+# Table name
+table: blogposts
+
+# The CQL for creating a table you wish to stress (optional if it already exists)
+table_definition: |
+ CREATE TABLE blogposts (
+ domain text,
+ published_date timeuuid,
+ url text,
+ author text,
+ title text,
+ body text,
+ PRIMARY KEY(domain, published_date)
+ ) WITH CLUSTERING ORDER BY (published_date DESC)
+ AND compaction = { 'class':'LeveledCompactionStrategy' }
+ AND comment='A table to hold blog posts'
+
+### Column Distribution Specifications ###
+
+columnspec:
+ - name: domain
+ size: gaussian(5..100) #domain names are relatively short
+ population: uniform(1..10M) #10M possible domains to pick from
+
+ - name: published_date
+ cluster: fixed(1000) #under each domain we will have max 1000 posts
+
+ - name: url
+ size: uniform(30..300)
+
+ - name: title #titles shouldn't go beyond 200 chars
+ size: gaussian(10..200)
+
+ - name: author
+ size: uniform(5..20) #author names should be short
+
+ - name: body
+ size: gaussian(100..5000) #the body of the blog post can be long
+
+### Batch Ratio Distribution Specifications ###
+
+insert:
+ partitions: fixed(1) # Our partition key is the domain so only insert one per batch
+
+ select: fixed(1)/1000 # We have 1000 posts per domain so 1/1000 will allow 1 post per batch
+
+ batchtype: UNLOGGED # Unlogged batches
+
+ condition: IF body = NULL # LWT: Do not override
+
+
+#
+# A list of queries you wish to run against the schema
+#
+queries:
+ singlepost:
+ cql: select * from blogposts where domain = ? LIMIT 1
+ fields: samerow
+ timeline:
+ cql: select url, title, published_date from blogposts where domain = ? LIMIT 10
+ fields: samerow
diff --git a/tools/fqltool/src/org/apache/cassandra/fqltool/DriverResultSet.java b/tools/fqltool/src/org/apache/cassandra/fqltool/DriverResultSet.java
new file mode 100644
index 0000000..e85b23b
--- /dev/null
+++ b/tools/fqltool/src/org/apache/cassandra/fqltool/DriverResultSet.java
@@ -0,0 +1,264 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.fqltool;
+
+import java.nio.ByteBuffer;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
+import com.google.common.collect.AbstractIterator;
+
+import com.datastax.driver.core.ColumnDefinitions;
+import com.datastax.driver.core.ResultSet;
+import com.datastax.driver.core.Row;
+import org.apache.cassandra.utils.ByteBufferUtil;
+
+
+/**
+ * Wraps a result set from the driver so that we can reuse the compare code when reading
+ * up a result set produced by ResultStore.
+ */
+public class DriverResultSet implements ResultHandler.ComparableResultSet
+{
+ private final ResultSet resultSet;
+ private final Throwable failureException;
+
+ public DriverResultSet(ResultSet resultSet)
+ {
+ this(resultSet, null);
+ }
+
+ private DriverResultSet(ResultSet res, Throwable failureException)
+ {
+ resultSet = res;
+ this.failureException = failureException;
+ }
+
+ public static DriverResultSet failed(Throwable ex)
+ {
+ return new DriverResultSet(null, ex);
+ }
+
+ public ResultHandler.ComparableColumnDefinitions getColumnDefinitions()
+ {
+ if (wasFailed())
+ return new DriverColumnDefinitions(null, true, failureException);
+
+ return new DriverColumnDefinitions(resultSet.getColumnDefinitions());
+ }
+
+ public boolean wasFailed()
+ {
+ return failureException != null;
+ }
+
+ public Throwable getFailureException()
+ {
+ return failureException;
+ }
+
+ public Iterator<ResultHandler.ComparableRow> iterator()
+ {
+ if (wasFailed())
+ return Collections.emptyListIterator();
+ return new AbstractIterator<ResultHandler.ComparableRow>()
+ {
+ Iterator<Row> iter = resultSet.iterator();
+ protected ResultHandler.ComparableRow computeNext()
+ {
+ if (iter.hasNext())
+ return new DriverRow(iter.next());
+ return endOfData();
+ }
+ };
+ }
+
+ public static class DriverRow implements ResultHandler.ComparableRow
+ {
+ private final Row row;
+
+ public DriverRow(Row row)
+ {
+ this.row = row;
+ }
+
+ public ResultHandler.ComparableColumnDefinitions getColumnDefinitions()
+ {
+ return new DriverColumnDefinitions(row.getColumnDefinitions());
+ }
+
+ public ByteBuffer getBytesUnsafe(int i)
+ {
+ return row.getBytesUnsafe(i);
+ }
+
+ @Override
+ public boolean equals(Object oo)
+ {
+ if (!(oo instanceof ResultHandler.ComparableRow))
+ return false;
+
+ ResultHandler.ComparableRow o = (ResultHandler.ComparableRow)oo;
+ if (getColumnDefinitions().size() != o.getColumnDefinitions().size())
+ return false;
+
+ for (int j = 0; j < getColumnDefinitions().size(); j++)
+ {
+ ByteBuffer b1 = getBytesUnsafe(j);
+ ByteBuffer b2 = o.getBytesUnsafe(j);
+
+ if (b1 != null && b2 != null && !b1.equals(b2))
+ {
+ return false;
+ }
+ if (b1 == null && b2 != null || b2 == null && b1 != null)
+ {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ public int hashCode()
+ {
+ return Objects.hash(row);
+ }
+
+ public String toString()
+ {
+ StringBuilder sb = new StringBuilder();
+ List<ResultHandler.ComparableDefinition> colDefs = getColumnDefinitions().asList();
+ for (int i = 0; i < getColumnDefinitions().size(); i++)
+ {
+ ByteBuffer bb = getBytesUnsafe(i);
+ String row = bb != null ? ByteBufferUtil.bytesToHex(bb) : "NULL";
+ sb.append(colDefs.get(i)).append(':').append(row).append(",");
+ }
+ return sb.toString();
+ }
+ }
+
+ public static class DriverColumnDefinitions implements ResultHandler.ComparableColumnDefinitions
+ {
+ private final ColumnDefinitions columnDefinitions;
+ private final boolean failed;
+ private final Throwable failureException;
+
+ public DriverColumnDefinitions(ColumnDefinitions columnDefinitions)
+ {
+ this(columnDefinitions, false, null);
+ }
+
+ private DriverColumnDefinitions(ColumnDefinitions columnDefinitions, boolean failed, Throwable failureException)
+ {
+ this.columnDefinitions = columnDefinitions;
+ this.failed = failed;
+ this.failureException = failureException;
+ }
+
+ public List<ResultHandler.ComparableDefinition> asList()
+ {
+ if (wasFailed())
+ return Collections.emptyList();
+ return columnDefinitions.asList().stream().map(DriverDefinition::new).collect(Collectors.toList());
+ }
+
+ public boolean wasFailed()
+ {
+ return failed;
+ }
+
+ public Throwable getFailureException()
+ {
+ return failureException;
+ }
+
+ public int size()
+ {
+ return columnDefinitions.size();
+ }
+
+ public Iterator<ResultHandler.ComparableDefinition> iterator()
+ {
+ return asList().iterator();
+ }
+
+ public boolean equals(Object oo)
+ {
+ if (!(oo instanceof ResultHandler.ComparableColumnDefinitions))
+ return false;
+
+ ResultHandler.ComparableColumnDefinitions o = (ResultHandler.ComparableColumnDefinitions)oo;
+ if (wasFailed() && o.wasFailed())
+ return true;
+
+ if (size() != o.size())
+ return false;
+
+ return asList().equals(o.asList());
+ }
+
+ public int hashCode()
+ {
+ return Objects.hash(columnDefinitions, failed, failureException);
+ }
+ }
+
+ public static class DriverDefinition implements ResultHandler.ComparableDefinition
+ {
+ private final ColumnDefinitions.Definition def;
+
+ public DriverDefinition(ColumnDefinitions.Definition def)
+ {
+ this.def = def;
+ }
+
+ public String getType()
+ {
+ return def.getType().toString();
+ }
+
+ public String getName()
+ {
+ return def.getName();
+ }
+
+ public boolean equals(Object oo)
+ {
+ if (!(oo instanceof ResultHandler.ComparableDefinition))
+ return false;
+
+ return def.equals(((DriverDefinition)oo).def);
+ }
+
+ public int hashCode()
+ {
+ return Objects.hash(def);
+ }
+
+ public String toString()
+ {
+ return getName() + ':' + getType();
+ }
+ }
+
+}
diff --git a/tools/fqltool/src/org/apache/cassandra/fqltool/FQLQuery.java b/tools/fqltool/src/org/apache/cassandra/fqltool/FQLQuery.java
new file mode 100644
index 0000000..c3c6c89
--- /dev/null
+++ b/tools/fqltool/src/org/apache/cassandra/fqltool/FQLQuery.java
@@ -0,0 +1,261 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.fqltool;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+import com.google.common.primitives.Longs;
+
+import com.datastax.driver.core.BatchStatement;
+import com.datastax.driver.core.ConsistencyLevel;
+import com.datastax.driver.core.SimpleStatement;
+import com.datastax.driver.core.Statement;
+import org.apache.cassandra.fql.FullQueryLogger;
+import org.apache.cassandra.cql3.QueryOptions;
+import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.service.QueryState;
+import org.apache.cassandra.utils.binlog.BinLog;
+
+public abstract class FQLQuery implements Comparable<FQLQuery>
+{
+ public final long queryStartTime;
+ public final QueryOptions queryOptions;
+ public final int protocolVersion;
+ public final QueryState queryState;
+
+ public FQLQuery(String keyspace, int protocolVersion, QueryOptions queryOptions, long queryStartTime, long generatedTimestamp, int generatedNowInSeconds)
+ {
+ this.queryStartTime = queryStartTime;
+ this.queryOptions = queryOptions;
+ this.protocolVersion = protocolVersion;
+ this.queryState = queryState(keyspace, generatedTimestamp, generatedNowInSeconds);
+ }
+
+ public abstract Statement toStatement();
+
+ /**
+ * used when storing the queries executed
+ */
+ public abstract BinLog.ReleaseableWriteMarshallable toMarshallable();
+
+ public String keyspace()
+ {
+ return queryState.getClientState().getRawKeyspace();
+ }
+
+ private QueryState queryState(String keyspace, long generatedTimestamp, int generatedNowInSeconds)
+ {
+ ClientState clientState = keyspace != null ? ClientState.forInternalCalls(keyspace) : ClientState.forInternalCalls();
+ return new QueryState(clientState, generatedTimestamp, generatedNowInSeconds);
+ }
+
+ public boolean equals(Object o)
+ {
+ if (this == o) return true;
+ if (!(o instanceof FQLQuery)) return false;
+ FQLQuery fqlQuery = (FQLQuery) o;
+ return queryStartTime == fqlQuery.queryStartTime &&
+ protocolVersion == fqlQuery.protocolVersion &&
+ queryState.getTimestamp() == fqlQuery.queryState.getTimestamp() &&
+ Objects.equals(queryState.getClientState().getRawKeyspace(), fqlQuery.queryState.getClientState().getRawKeyspace()) &&
+ Objects.equals(queryOptions.getValues(), fqlQuery.queryOptions.getValues());
+ }
+
+ public int hashCode()
+ {
+ return Objects.hash(queryStartTime, queryOptions, protocolVersion, queryState.getClientState().getRawKeyspace());
+ }
+
+ public int compareTo(FQLQuery other)
+ {
+ return Longs.compare(queryStartTime, other.queryStartTime);
+ }
+
+ public String toString()
+ {
+ return "FQLQuery{" +
+ "queryStartTime=" + queryStartTime +
+ ", protocolVersion=" + protocolVersion +
+ ", queryState='" + queryState + '\'' +
+ '}';
+ }
+
+ public static class Single extends FQLQuery
+ {
+ public final String query;
+ public final List<ByteBuffer> values;
+
+ public Single(String keyspace, int protocolVersion, QueryOptions queryOptions, long queryStartTime, long generatedTimestamp, int generatedNowInSeconds, String queryString, List<ByteBuffer> values)
+ {
+ super(keyspace, protocolVersion, queryOptions, queryStartTime, generatedTimestamp, generatedNowInSeconds);
+ this.query = queryString;
+ this.values = values;
+ }
+
+ @Override
+ public String toString()
+ {
+ return String.format("%s: Query: [%s], valuecount : %d",
+ super.toString(),
+ query,
+ values.size());
+ }
+
+ public Statement toStatement()
+ {
+ SimpleStatement ss = new SimpleStatement(query, values.toArray());
+ ss.setConsistencyLevel(ConsistencyLevel.valueOf(queryOptions.getConsistency().name()));
+ ss.setDefaultTimestamp(queryOptions.getTimestamp(queryState));
+ return ss;
+ }
+
+ public BinLog.ReleaseableWriteMarshallable toMarshallable()
+ {
+
+ return new FullQueryLogger.Query(query, queryOptions, queryState, queryStartTime);
+ }
+
+ public int compareTo(FQLQuery other)
+ {
+ int cmp = super.compareTo(other);
+
+ if (cmp == 0)
+ {
+ if (other instanceof Batch)
+ return -1;
+
+ Single singleQuery = (Single) other;
+
+ cmp = query.compareTo(singleQuery.query);
+ if (cmp == 0)
+ {
+ if (values.size() != singleQuery.values.size())
+ return values.size() - singleQuery.values.size();
+ for (int i = 0; i < values.size(); i++)
+ {
+ cmp = values.get(i).compareTo(singleQuery.values.get(i));
+ if (cmp != 0)
+ return cmp;
+ }
+ }
+ }
+ return cmp;
+ }
+
+ public boolean equals(Object o)
+ {
+ if (this == o) return true;
+ if (!(o instanceof Single)) return false;
+ if (!super.equals(o)) return false;
+ Single single = (Single) o;
+ return Objects.equals(query, single.query) &&
+ Objects.equals(values, single.values);
+ }
+
+ public int hashCode()
+ {
+ return Objects.hash(super.hashCode(), query, values);
+ }
+ }
+
+ public static class Batch extends FQLQuery
+ {
+ public final BatchStatement.Type batchType;
+ public final List<Single> queries;
+
+ public Batch(String keyspace, int protocolVersion, QueryOptions queryOptions, long queryStartTime, long generatedTimestamp, int generatedNowInSeconds, BatchStatement.Type batchType, List<String> queries, List<List<ByteBuffer>> values)
+ {
+ super(keyspace, protocolVersion, queryOptions, queryStartTime, generatedTimestamp, generatedNowInSeconds);
+ this.batchType = batchType;
+ this.queries = new ArrayList<>(queries.size());
+ for (int i = 0; i < queries.size(); i++)
+ this.queries.add(new Single(keyspace, protocolVersion, queryOptions, queryStartTime, generatedTimestamp, generatedNowInSeconds, queries.get(i), values.get(i)));
+ }
+
+ public Statement toStatement()
+ {
+ BatchStatement bs = new BatchStatement(batchType);
+ for (Single query : queries)
+ bs.add(query.toStatement());
+ bs.setConsistencyLevel(ConsistencyLevel.valueOf(queryOptions.getConsistency().name()));
+ bs.setDefaultTimestamp(queryOptions.getTimestamp(queryState));
+ return bs;
+ }
+
+ public int compareTo(FQLQuery other)
+ {
+ int cmp = super.compareTo(other);
+
+ if (cmp == 0)
+ {
+ if (other instanceof Single)
+ return 1;
+
+ Batch otherBatch = (Batch) other;
+ if (queries.size() != otherBatch.queries.size())
+ return queries.size() - otherBatch.queries.size();
+ for (int i = 0; i < queries.size(); i++)
+ {
+ cmp = queries.get(i).compareTo(otherBatch.queries.get(i));
+ if (cmp != 0)
+ return cmp;
+ }
+ }
+ return cmp;
+ }
+
+ public BinLog.ReleaseableWriteMarshallable toMarshallable()
+ {
+ List<String> queryStrings = new ArrayList<>();
+ List<List<ByteBuffer>> values = new ArrayList<>();
+ for (Single q : queries)
+ {
+ queryStrings.add(q.query);
+ values.add(q.values);
+ }
+ return new FullQueryLogger.Batch(org.apache.cassandra.cql3.statements.BatchStatement.Type.valueOf(batchType.name()), queryStrings, values, queryOptions, queryState, queryStartTime);
+ }
+
+ public String toString()
+ {
+ StringBuilder sb = new StringBuilder(super.toString()).append(" batch: ").append(batchType).append(':');
+ for (Single q : queries)
+ sb.append(q.toString()).append(',');
+ sb.append("end batch");
+ return sb.toString();
+ }
+
+ public boolean equals(Object o)
+ {
+ if (this == o) return true;
+ if (!(o instanceof Batch)) return false;
+ if (!super.equals(o)) return false;
+ Batch batch = (Batch) o;
+ return batchType == batch.batchType &&
+ Objects.equals(queries, batch.queries);
+ }
+
+ public int hashCode()
+ {
+ return Objects.hash(super.hashCode(), batchType, queries);
+ }
+ }
+}
diff --git a/tools/fqltool/src/org/apache/cassandra/fqltool/FQLQueryIterator.java b/tools/fqltool/src/org/apache/cassandra/fqltool/FQLQueryIterator.java
new file mode 100644
index 0000000..ccbb200
--- /dev/null
+++ b/tools/fqltool/src/org/apache/cassandra/fqltool/FQLQueryIterator.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.fqltool;
+
+import java.util.PriorityQueue;
+
+import net.openhft.chronicle.queue.ExcerptTailer;
+import org.apache.cassandra.utils.AbstractIterator;
+
+public class FQLQueryIterator extends AbstractIterator<FQLQuery>
+{
+ // use a priority queue to be able to sort the head of the query logs in memory
+ private final PriorityQueue<FQLQuery> pq;
+ private final ExcerptTailer tailer;
+ private final FQLQueryReader reader;
+
+ /**
+ * Create an iterator over the FQLQueries in tailer
+ *
+ * Reads up to readAhead queries in to memory to be able to sort them (the files are mostly sorted already)
+ */
+ public FQLQueryIterator(ExcerptTailer tailer, int readAhead)
+ {
+ assert readAhead > 0 : "readAhead needs to be > 0";
+ reader = new FQLQueryReader();
+ this.tailer = tailer;
+ pq = new PriorityQueue<>(readAhead);
+ for (int i = 0; i < readAhead; i++)
+ {
+ FQLQuery next = readNext();
+ if (next != null)
+ pq.add(next);
+ else
+ break;
+ }
+ }
+
+ protected FQLQuery computeNext()
+ {
+ FQLQuery q = pq.poll();
+ if (q == null)
+ return endOfData();
+ FQLQuery next = readNext();
+ if (next != null)
+ pq.add(next);
+ return q;
+ }
+
+ private FQLQuery readNext()
+ {
+ if (tailer.readDocument(reader))
+ return reader.getQuery();
+ return null;
+ }
+}
+
diff --git a/tools/fqltool/src/org/apache/cassandra/fqltool/FQLQueryReader.java b/tools/fqltool/src/org/apache/cassandra/fqltool/FQLQueryReader.java
new file mode 100644
index 0000000..20f362b
--- /dev/null
+++ b/tools/fqltool/src/org/apache/cassandra/fqltool/FQLQueryReader.java
@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.fqltool;
+
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+
+import com.datastax.driver.core.BatchStatement;
+import io.netty.buffer.Unpooled;
+import net.openhft.chronicle.core.io.IORuntimeException;
+import net.openhft.chronicle.wire.ReadMarshallable;
+import net.openhft.chronicle.wire.ValueIn;
+import net.openhft.chronicle.wire.WireIn;
+import org.apache.cassandra.cql3.QueryOptions;
+import org.apache.cassandra.transport.ProtocolVersion;
+
+import static org.apache.cassandra.fql.FullQueryLogger.CURRENT_VERSION;
+import static org.apache.cassandra.fql.FullQueryLogger.GENERATED_NOW_IN_SECONDS;
+import static org.apache.cassandra.fql.FullQueryLogger.GENERATED_TIMESTAMP;
+import static org.apache.cassandra.fql.FullQueryLogger.KEYSPACE;
+import static org.apache.cassandra.fql.FullQueryLogger.PROTOCOL_VERSION;
+import static org.apache.cassandra.fql.FullQueryLogger.QUERY_OPTIONS;
+import static org.apache.cassandra.fql.FullQueryLogger.QUERY_START_TIME;
+import static org.apache.cassandra.fql.FullQueryLogger.TYPE;
+import static org.apache.cassandra.fql.FullQueryLogger.VERSION;
+import static org.apache.cassandra.fql.FullQueryLogger.BATCH;
+import static org.apache.cassandra.fql.FullQueryLogger.BATCH_TYPE;
+import static org.apache.cassandra.fql.FullQueryLogger.QUERIES;
+import static org.apache.cassandra.fql.FullQueryLogger.QUERY;
+import static org.apache.cassandra.fql.FullQueryLogger.SINGLE_QUERY;
+import static org.apache.cassandra.fql.FullQueryLogger.VALUES;
+
+public class FQLQueryReader implements ReadMarshallable
+{
+ private FQLQuery query;
+
+ public void readMarshallable(WireIn wireIn) throws IORuntimeException
+ {
+ verifyVersion(wireIn);
+ String type = readType(wireIn);
+
+ long queryStartTime = wireIn.read(QUERY_START_TIME).int64();
+ int protocolVersion = wireIn.read(PROTOCOL_VERSION).int32();
+ QueryOptions queryOptions = QueryOptions.codec.decode(Unpooled.wrappedBuffer(wireIn.read(QUERY_OPTIONS).bytes()), ProtocolVersion.decode(protocolVersion, true));
+ long generatedTimestamp = wireIn.read(GENERATED_TIMESTAMP).int64();
+ int generatedNowInSeconds = wireIn.read(GENERATED_NOW_IN_SECONDS).int32();
+ String keyspace = wireIn.read(KEYSPACE).text();
+
+ switch (type)
+ {
+ case SINGLE_QUERY:
+ String queryString = wireIn.read(QUERY).text();
+ query = new FQLQuery.Single(keyspace,
+ protocolVersion,
+ queryOptions,
+ queryStartTime,
+ generatedTimestamp,
+ generatedNowInSeconds,
+ queryString,
+ queryOptions.getValues());
+ break;
+ case BATCH:
+ BatchStatement.Type batchType = BatchStatement.Type.valueOf(wireIn.read(BATCH_TYPE).text());
+ ValueIn in = wireIn.read(QUERIES);
+ int queryCount = in.int32();
+
+ List<String> queries = new ArrayList<>(queryCount);
+ for (int i = 0; i < queryCount; i++)
+ queries.add(in.text());
+ in = wireIn.read(VALUES);
+ int valueCount = in.int32();
+ List<List<ByteBuffer>> values = new ArrayList<>(valueCount);
+ for (int ii = 0; ii < valueCount; ii++)
+ {
+ List<ByteBuffer> subValues = new ArrayList<>();
+ values.add(subValues);
+ int numSubValues = in.int32();
+ for (int zz = 0; zz < numSubValues; zz++)
+ subValues.add(ByteBuffer.wrap(in.bytes()));
+ }
+ query = new FQLQuery.Batch(keyspace,
+ protocolVersion,
+ queryOptions,
+ queryStartTime,
+ generatedTimestamp,
+ generatedNowInSeconds,
+ batchType,
+ queries,
+ values);
+ break;
+ default:
+ throw new IORuntimeException("Unhandled record type: " + type);
+ }
+ }
+
+ private void verifyVersion(WireIn wireIn)
+ {
+ int version = wireIn.read(VERSION).int16();
+
+ if (version > CURRENT_VERSION)
+ {
+ throw new IORuntimeException("Unsupported record version [" + version
+ + "] - highest supported version is [" + CURRENT_VERSION + ']');
+ }
+ }
+
+ private String readType(WireIn wireIn) throws IORuntimeException
+ {
+ String type = wireIn.read(TYPE).text();
+ if (!SINGLE_QUERY.equals(type) && !BATCH.equals(type))
+ {
+ throw new IORuntimeException("Unsupported record type field [" + type
+ + "] - supported record types are [" + SINGLE_QUERY + ", " + BATCH + ']');
+ }
+
+ return type;
+ }
+
+ public FQLQuery getQuery()
+ {
+ return query;
+ }
+}
diff --git a/tools/fqltool/src/org/apache/cassandra/fqltool/FullQueryLogTool.java b/tools/fqltool/src/org/apache/cassandra/fqltool/FullQueryLogTool.java
new file mode 100644
index 0000000..97e7487
--- /dev/null
+++ b/tools/fqltool/src/org/apache/cassandra/fqltool/FullQueryLogTool.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.fqltool;
+
+import java.util.List;
+
+import com.google.common.base.Throwables;
+
+import io.airlift.airline.Cli;
+import io.airlift.airline.Help;
+import io.airlift.airline.ParseArgumentsMissingException;
+import io.airlift.airline.ParseArgumentsUnexpectedException;
+import io.airlift.airline.ParseCommandMissingException;
+import io.airlift.airline.ParseCommandUnrecognizedException;
+import io.airlift.airline.ParseOptionConversionException;
+import io.airlift.airline.ParseOptionMissingException;
+import io.airlift.airline.ParseOptionMissingValueException;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.fqltool.commands.Compare;
+import org.apache.cassandra.fqltool.commands.Dump;
+import org.apache.cassandra.fqltool.commands.Replay;
+
+import static com.google.common.base.Throwables.getStackTraceAsString;
+import static com.google.common.collect.Lists.newArrayList;
+
+public class FullQueryLogTool
+{
+ public static void main(String... args)
+ {
+ DatabaseDescriptor.clientInitialization();
+ List<Class<? extends Runnable>> commands = newArrayList(
+ Help.class,
+ Dump.class,
+ Replay.class,
+ Compare.class
+ );
+
+ Cli.CliBuilder<Runnable> builder = Cli.builder("fqltool");
+
+ builder.withDescription("Manipulate the contents of full query log files")
+ .withDefaultCommand(Help.class)
+ .withCommands(commands);
+
+ Cli<Runnable> parser = builder.build();
+
+ int status = 0;
+ try
+ {
+ parser.parse(args).run();
+ } catch (IllegalArgumentException |
+ IllegalStateException |
+ ParseArgumentsMissingException |
+ ParseArgumentsUnexpectedException |
+ ParseOptionConversionException |
+ ParseOptionMissingException |
+ ParseOptionMissingValueException |
+ ParseCommandMissingException |
+ ParseCommandUnrecognizedException e)
+ {
+ badUse(e);
+ status = 1;
+ } catch (Throwable throwable)
+ {
+ err(Throwables.getRootCause(throwable));
+ status = 2;
+ }
+
+ System.exit(status);
+ }
+
+ private static void badUse(Exception e)
+ {
+ System.out.println("fqltool: " + e.getMessage());
+ System.out.println("See 'fqltool help' or 'fqltool help <command>'.");
+ }
+
+ private static void err(Throwable e)
+ {
+ System.err.println("error: " + e.getMessage());
+ System.err.println("-- StackTrace --");
+ System.err.println(getStackTraceAsString(e));
+ }
+}
diff --git a/tools/fqltool/src/org/apache/cassandra/fqltool/MismatchListener.java b/tools/fqltool/src/org/apache/cassandra/fqltool/MismatchListener.java
new file mode 100644
index 0000000..70a4b11
--- /dev/null
+++ b/tools/fqltool/src/org/apache/cassandra/fqltool/MismatchListener.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.fqltool;
+
+import java.util.List;
+import java.util.UUID;
+
+public interface MismatchListener
+{
+ void mismatch(UUID mismatchUUID, List<String> targetHosts, FQLQuery query, List<ResultHandler.ComparableRow> rows);
+ void columnDefMismatch(UUID mismatchUUID, List<String> targetHosts, FQLQuery query, List<ResultHandler.ComparableColumnDefinitions> cds);
+}
diff --git a/tools/fqltool/src/org/apache/cassandra/fqltool/QueryReplayer.java b/tools/fqltool/src/org/apache/cassandra/fqltool/QueryReplayer.java
new file mode 100644
index 0000000..4524e33
--- /dev/null
+++ b/tools/fqltool/src/org/apache/cassandra/fqltool/QueryReplayer.java
@@ -0,0 +1,274 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.fqltool;
+
+import java.io.Closeable;
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
+
+import com.google.common.util.concurrent.FutureCallback;
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.MoreExecutors;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.codahale.metrics.MetricRegistry;
+import com.codahale.metrics.Timer;
+import com.datastax.driver.core.Cluster;
+import com.datastax.driver.core.ResultSet;
+import com.datastax.driver.core.Session;
+import com.datastax.driver.core.Statement;
+import org.apache.cassandra.utils.FBUtilities;
+
+public class QueryReplayer implements Closeable
+{
+ private static final Logger logger = LoggerFactory.getLogger(QueryReplayer.class);
+ private static final int PRINT_RATE = 5000;
+ private final ExecutorService es = Executors.newFixedThreadPool(1);
+ private final Iterator<List<FQLQuery>> queryIterator;
+ private final List<Predicate<FQLQuery>> filters;
+ private final List<Session> sessions;
+ private final ResultHandler resultHandler;
+ private final MetricRegistry metrics = new MetricRegistry();
+ private final SessionProvider sessionProvider;
+
+ /**
+ * @param queryIterator the queries to be replayed
+ * @param targetHosts hosts to connect to, in the format "<user>:<password>@<host>:<port>" where only <host> is mandatory, port defaults to 9042
+ * @param resultPaths where to write the results of the queries, for later comparisons, size should be the same as the number of iterators
+ * @param filters query filters
+ * @param queryFilePathString where to store the queries executed
+ */
+ public QueryReplayer(Iterator<List<FQLQuery>> queryIterator,
+ List<String> targetHosts,
+ List<File> resultPaths,
+ List<Predicate<FQLQuery>> filters,
+ String queryFilePathString)
+ {
+ this(queryIterator, targetHosts, resultPaths, filters, queryFilePathString, new DefaultSessionProvider(), null);
+ }
+
+ /**
+ * Constructor public to allow external users to build their own session provider
+ *
+ * sessionProvider takes the hosts in targetHosts and creates one session per entry
+ */
+ public QueryReplayer(Iterator<List<FQLQuery>> queryIterator,
+ List<String> targetHosts,
+ List<File> resultPaths,
+ List<Predicate<FQLQuery>> filters,
+ String queryFilePathString,
+ SessionProvider sessionProvider,
+ MismatchListener mismatchListener)
+ {
+ this.sessionProvider = sessionProvider;
+ this.queryIterator = queryIterator;
+ this.filters = filters;
+ sessions = targetHosts.stream().map(sessionProvider::connect).collect(Collectors.toList());
+ File queryFilePath = queryFilePathString != null ? new File(queryFilePathString) : null;
+ resultHandler = new ResultHandler(targetHosts, resultPaths, queryFilePath, mismatchListener);
+ }
+
+ public void replay()
+ {
+ while (queryIterator.hasNext())
+ {
+ List<FQLQuery> queries = queryIterator.next();
+ for (FQLQuery query : queries)
+ {
+ if (filters.stream().anyMatch(f -> !f.test(query)))
+ continue;
+ try (Timer.Context ctx = metrics.timer("queries").time())
+ {
+ List<ListenableFuture<ResultHandler.ComparableResultSet>> results = new ArrayList<>(sessions.size());
+ Statement statement = query.toStatement();
+ for (Session session : sessions)
+ {
+ maybeSetKeyspace(session, query);
+ if (logger.isDebugEnabled())
+ logger.debug("Executing query: {}", query);
+ ListenableFuture<ResultSet> future = session.executeAsync(statement);
+ results.add(handleErrors(future));
+ }
+
+ ListenableFuture<List<ResultHandler.ComparableResultSet>> resultList = Futures.allAsList(results);
+
+ Futures.addCallback(resultList, new FutureCallback<List<ResultHandler.ComparableResultSet>>()
+ {
+ public void onSuccess(List<ResultHandler.ComparableResultSet> resultSets)
+ {
+ // note that the order of resultSets is signifcant here - resultSets.get(x) should
+ // be the result from a query against targetHosts.get(x)
+ resultHandler.handleResults(query, resultSets);
+ }
+
+ public void onFailure(Throwable throwable)
+ {
+ throw new AssertionError("Errors should be handled in FQLQuery.execute", throwable);
+ }
+ }, es);
+
+ FBUtilities.waitOnFuture(resultList);
+ }
+ catch (Throwable t)
+ {
+ logger.error("QUERY %s got exception: %s", query, t.getMessage());
+ }
+
+ Timer timer = metrics.timer("queries");
+ if (timer.getCount() % PRINT_RATE == 0)
+ logger.info(String.format("%d queries, rate = %.2f", timer.getCount(), timer.getOneMinuteRate()));
+ }
+ }
+ }
+
+ private void maybeSetKeyspace(Session session, FQLQuery query)
+ {
+ try
+ {
+ if (query.keyspace() != null && !query.keyspace().equals(session.getLoggedKeyspace()))
+ {
+ if (logger.isDebugEnabled())
+ logger.debug("Switching keyspace from {} to {}", session.getLoggedKeyspace(), query.keyspace());
+ session.execute("USE " + query.keyspace());
+ }
+ }
+ catch (Throwable t)
+ {
+ logger.error("USE {} failed: {}", query.keyspace(), t.getMessage());
+ }
+ }
+
+ /**
+ * Make sure we catch any query errors
+ *
+ * On error, this creates a failed ComparableResultSet with the exception set to be able to store
+ * this fact in the result file and handle comparison of failed result sets.
+ */
+ private static ListenableFuture<ResultHandler.ComparableResultSet> handleErrors(ListenableFuture<ResultSet> result)
+ {
+ ListenableFuture<ResultHandler.ComparableResultSet> res = Futures.transform(result, DriverResultSet::new, MoreExecutors.directExecutor());
+ return Futures.catching(res, Throwable.class, DriverResultSet::failed, MoreExecutors.directExecutor());
+ }
+
+ public void close() throws IOException
+ {
+ es.shutdown();
+ sessionProvider.close();
+ resultHandler.close();
+ }
+
+ static class ParsedTargetHost
+ {
+ final int port;
+ final String user;
+ final String password;
+ final String host;
+
+ ParsedTargetHost(String host, int port, String user, String password)
+ {
+ this.host = host;
+ this.port = port;
+ this.user = user;
+ this.password = password;
+ }
+
+ static ParsedTargetHost fromString(String s)
+ {
+ String [] userInfoHostPort = s.split("@");
+
+ String hostPort = null;
+ String user = null;
+ String password = null;
+ if (userInfoHostPort.length == 2)
+ {
+ String [] userPassword = userInfoHostPort[0].split(":");
+ if (userPassword.length != 2)
+ throw new RuntimeException("Username provided but no password");
+ hostPort = userInfoHostPort[1];
+ user = userPassword[0];
+ password = userPassword[1];
+ }
+ else if (userInfoHostPort.length == 1)
+ hostPort = userInfoHostPort[0];
+ else
+ throw new RuntimeException("Malformed target host: "+s);
+
+ String[] splitHostPort = hostPort.split(":");
+ int port = 9042;
+ if (splitHostPort.length == 2)
+ port = Integer.parseInt(splitHostPort[1]);
+
+ return new ParsedTargetHost(splitHostPort[0], port, user, password);
+ }
+ }
+
+ public static interface SessionProvider extends Closeable
+ {
+ Session connect(String connectionString);
+ void close();
+ }
+
+ private static final class DefaultSessionProvider implements SessionProvider
+ {
+ private final static Map<String, Session> sessionCache = new HashMap<>();
+
+ public synchronized Session connect(String connectionString)
+ {
+ if (sessionCache.containsKey(connectionString))
+ return sessionCache.get(connectionString);
+ Cluster.Builder builder = Cluster.builder();
+ ParsedTargetHost pth = ParsedTargetHost.fromString(connectionString);
+ builder.addContactPoint(pth.host);
+ builder.withPort(pth.port);
+ if (pth.user != null)
+ builder.withCredentials(pth.user, pth.password);
+ Cluster c = builder.build();
+ sessionCache.put(connectionString, c.connect());
+ return sessionCache.get(connectionString);
+ }
+
+ public void close()
+ {
+ sessionCache.entrySet().removeIf(entry -> {
+ try (Session s = entry.getValue())
+ {
+ s.getCluster().close();
+ return true;
+ }
+ catch (Throwable t)
+ {
+ logger.error("Could not close connection", t);
+ return false;
+ }
+ });
+ }
+ }
+}
diff --git a/tools/fqltool/src/org/apache/cassandra/fqltool/ResultComparator.java b/tools/fqltool/src/org/apache/cassandra/fqltool/ResultComparator.java
new file mode 100644
index 0000000..eeebe20
--- /dev/null
+++ b/tools/fqltool/src/org/apache/cassandra/fqltool/ResultComparator.java
@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.fqltool;
+
+
+import java.util.List;
+import java.util.Objects;
+import java.util.UUID;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class ResultComparator
+{
+ private static final Logger logger = LoggerFactory.getLogger(ResultComparator.class);
+ private final MismatchListener mismatchListener;
+
+ public ResultComparator()
+ {
+ this(null);
+ }
+
+ public ResultComparator(MismatchListener mismatchListener)
+ {
+ this.mismatchListener = mismatchListener;
+ }
+ /**
+ * Compares the rows in rows
+ * the row at position x in rows will have come from host at position x in targetHosts
+ */
+ public boolean compareRows(List<String> targetHosts, FQLQuery query, List<ResultHandler.ComparableRow> rows)
+ {
+ if (rows.size() < 2 || rows.stream().allMatch(Objects::isNull))
+ return true;
+
+ if (rows.stream().anyMatch(Objects::isNull))
+ {
+ handleMismatch(targetHosts, query, rows);
+ return false;
+ }
+
+ ResultHandler.ComparableRow ref = rows.get(0);
+ boolean equal = true;
+ for (int i = 1; i < rows.size(); i++)
+ {
+ ResultHandler.ComparableRow compare = rows.get(i);
+ if (!ref.equals(compare))
+ equal = false;
+ }
+ if (!equal)
+ handleMismatch(targetHosts, query, rows);
+ return equal;
+ }
+
+ /**
+ * Compares the column definitions
+ *
+ * the column definitions at position x in cds will have come from host at position x in targetHosts
+ */
+ public boolean compareColumnDefinitions(List<String> targetHosts, FQLQuery query, List<ResultHandler.ComparableColumnDefinitions> cds)
+ {
+ if (cds.size() < 2)
+ return true;
+
+ boolean equal = true;
+ List<ResultHandler.ComparableDefinition> refDefs = cds.get(0).asList();
+ for (int i = 1; i < cds.size(); i++)
+ {
+ List<ResultHandler.ComparableDefinition> toCompare = cds.get(i).asList();
+ if (!refDefs.equals(toCompare))
+ equal = false;
+ }
+ if (!equal)
+ handleColumnDefMismatch(targetHosts, query, cds);
+ return equal;
+ }
+
+ private void handleMismatch(List<String> targetHosts, FQLQuery query, List<ResultHandler.ComparableRow> rows)
+ {
+ UUID mismatchUUID = UUID.randomUUID();
+ StringBuilder sb = new StringBuilder("{} - MISMATCH Query = {} ");
+ for (int i = 0; i < targetHosts.size(); i++)
+ sb.append("mismatch").append(i)
+ .append('=')
+ .append('"').append(targetHosts.get(i)).append(':').append(rows.get(i)).append('"')
+ .append(',');
+
+ logger.warn(sb.toString(), mismatchUUID, query);
+ try
+ {
+ if (mismatchListener != null)
+ mismatchListener.mismatch(mismatchUUID, targetHosts, query, rows);
+ }
+ catch (Throwable t)
+ {
+ logger.error("ERROR notifying listener", t);
+ }
+ }
+
+ private void handleColumnDefMismatch(List<String> targetHosts, FQLQuery query, List<ResultHandler.ComparableColumnDefinitions> cds)
+ {
+ UUID mismatchUUID = UUID.randomUUID();
+ StringBuilder sb = new StringBuilder("{} - COLUMN DEFINITION MISMATCH Query = {} ");
+ for (int i = 0; i < targetHosts.size(); i++)
+ sb.append("mismatch").append(i)
+ .append('=')
+ .append('"').append(targetHosts.get(i)).append(':').append(columnDefinitionsString(cds.get(i))).append('"')
+ .append(',');
+
+ logger.warn(sb.toString(), mismatchUUID, query);
+ try
+ {
+ if (mismatchListener != null)
+ mismatchListener.columnDefMismatch(mismatchUUID, targetHosts, query, cds);
+ }
+ catch (Throwable t)
+ {
+ logger.error("ERROR notifying listener", t);
+ }
+ }
+
+ private String columnDefinitionsString(ResultHandler.ComparableColumnDefinitions cd)
+ {
+ StringBuilder sb = new StringBuilder();
+ if (cd == null)
+ sb.append("NULL");
+ else if (cd.wasFailed())
+ sb.append("FAILED");
+ else
+ {
+ for (ResultHandler.ComparableDefinition def : cd)
+ {
+ sb.append(def.toString());
+ }
+ }
+ return sb.toString();
+ }
+}
\ No newline at end of file
diff --git a/tools/fqltool/src/org/apache/cassandra/fqltool/ResultHandler.java b/tools/fqltool/src/org/apache/cassandra/fqltool/ResultHandler.java
new file mode 100644
index 0000000..d88c6f7
--- /dev/null
+++ b/tools/fqltool/src/org/apache/cassandra/fqltool/ResultHandler.java
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.fqltool;
+
+import java.io.Closeable;
+import java.io.File;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class ResultHandler implements Closeable
+{
+ private static final Logger logger = LoggerFactory.getLogger(ResultHandler.class);
+ private final ResultStore resultStore;
+ private final ResultComparator resultComparator;
+ private final List<String> targetHosts;
+
+ public ResultHandler(List<String> targetHosts, List<File> resultPaths, File queryFilePath)
+ {
+ this(targetHosts, resultPaths, queryFilePath, null);
+ }
+
+ public ResultHandler(List<String> targetHosts, List<File> resultPaths, File queryFilePath, MismatchListener mismatchListener)
+ {
+ this.targetHosts = targetHosts;
+ resultStore = resultPaths != null ? new ResultStore(resultPaths, queryFilePath) : null;
+ resultComparator = new ResultComparator(mismatchListener);
+ }
+
+ /**
+ * Since we can't iterate a ResultSet more than once, and we don't want to keep the entire result set in memory
+ * we feed the rows one-by-one to resultComparator and resultStore.
+ *
+ * results.get(x) should be the results from executing query against targetHosts.get(x)
+ */
+ public void handleResults(FQLQuery query, List<ComparableResultSet> results)
+ {
+ for (int i = 0; i < targetHosts.size(); i++)
+ {
+ if (results.get(i).wasFailed())
+ logger.error("Query {} against {} failure: {}", query, targetHosts.get(i), results.get(i).getFailureException().getMessage());
+ }
+
+ List<ComparableColumnDefinitions> columnDefinitions = results.stream().map(ComparableResultSet::getColumnDefinitions).collect(Collectors.toList());
+ resultComparator.compareColumnDefinitions(targetHosts, query, columnDefinitions);
+ if (resultStore != null)
+ resultStore.storeColumnDefinitions(query, columnDefinitions);
+ List<Iterator<ComparableRow>> iters = results.stream().map(Iterable::iterator).collect(Collectors.toList());
+
+ while (true)
+ {
+ List<ComparableRow> rows = rows(iters);
+ resultComparator.compareRows(targetHosts, query, rows);
+ if (resultStore != null)
+ resultStore.storeRows(rows);
+ // all rows being null marks end of all resultsets, we need to call compareRows
+ // and storeRows once with everything null to mark that fact
+ if (rows.stream().allMatch(Objects::isNull))
+ return;
+ }
+ }
+
+ /**
+ * Get the first row from each of the iterators, if the iterator has run out, null will mark that in the list
+ */
+ @VisibleForTesting
+ public static List<ComparableRow> rows(List<Iterator<ComparableRow>> iters)
+ {
+ List<ComparableRow> rows = new ArrayList<>(iters.size());
+ for (Iterator<ComparableRow> iter : iters)
+ {
+ if (iter.hasNext())
+ rows.add(iter.next());
+ else
+ rows.add(null);
+ }
+ return rows;
+ }
+
+ public void close() throws IOException
+ {
+ if (resultStore != null)
+ resultStore.close();
+ }
+
+ public interface ComparableResultSet extends Iterable<ComparableRow>
+ {
+ public ComparableColumnDefinitions getColumnDefinitions();
+ public boolean wasFailed();
+ public Throwable getFailureException();
+ }
+
+ public interface ComparableColumnDefinitions extends Iterable<ComparableDefinition>
+ {
+ public List<ComparableDefinition> asList();
+ public boolean wasFailed();
+ public Throwable getFailureException();
+ public int size();
+ }
+
+ public interface ComparableDefinition
+ {
+ public String getType();
+ public String getName();
+ }
+
+ public interface ComparableRow
+ {
+ public ByteBuffer getBytesUnsafe(int i);
+ public ComparableColumnDefinitions getColumnDefinitions();
+ }
+
+}
diff --git a/tools/fqltool/src/org/apache/cassandra/fqltool/ResultStore.java b/tools/fqltool/src/org/apache/cassandra/fqltool/ResultStore.java
new file mode 100644
index 0000000..d128717
--- /dev/null
+++ b/tools/fqltool/src/org/apache/cassandra/fqltool/ResultStore.java
@@ -0,0 +1,291 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.fqltool;
+
+
+import java.io.File;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import net.openhft.chronicle.bytes.BytesStore;
+import net.openhft.chronicle.core.io.Closeable;
+import net.openhft.chronicle.core.io.IORuntimeException;
+import net.openhft.chronicle.queue.ChronicleQueue;
+import net.openhft.chronicle.queue.ChronicleQueueBuilder;
+import net.openhft.chronicle.queue.ExcerptAppender;
+import net.openhft.chronicle.wire.ReadMarshallable;
+import net.openhft.chronicle.wire.ValueIn;
+import net.openhft.chronicle.wire.ValueOut;
+import net.openhft.chronicle.wire.WireIn;
+import net.openhft.chronicle.wire.WireOut;
+import net.openhft.chronicle.wire.WriteMarshallable;
+import org.apache.cassandra.utils.Pair;
+import org.apache.cassandra.utils.binlog.BinLog;
+
+/**
+ * note that we store each row as a separate chronicle document to be able to
+ * avoid reading up the entire result set in memory when comparing
+ *
+ * document formats:
+ * to mark the start of a new result set:
+ * -------------------
+ * version: int16
+ * type: column_definitions
+ * column_count: int32;
+ * column_definition: text, text
+ * column_definition: text, text
+ * ....
+ * --------------------
+ *
+ * to mark a failed query:
+ * ---------------------
+ * version: int16
+ * type: query_failed
+ * message: text
+ * ---------------------
+ *
+ * row:
+ * --------------------
+ * version: int16
+ * type: row
+ * row_column_count: int32
+ * column: bytes
+ * ---------------------
+ *
+ * to mark the end of a result set:
+ * -------------------
+ * version: int16
+ * type: end_resultset
+ * -------------------
+ *
+ */
+public class ResultStore
+{
+ private static final String VERSION = "version";
+ private static final String TYPE = "type";
+ // types:
+ private static final String ROW = "row";
+ private static final String END = "end_resultset";
+ private static final String FAILURE = "query_failed";
+ private static final String COLUMN_DEFINITIONS = "column_definitions";
+ // fields:
+ private static final String COLUMN_DEFINITION = "column_definition";
+ private static final String COLUMN_COUNT = "column_count";
+ private static final String MESSAGE = "message";
+ private static final String ROW_COLUMN_COUNT = "row_column_count";
+ private static final String COLUMN = "column";
+
+ private static final int CURRENT_VERSION = 0;
+
+ private final List<ChronicleQueue> queues;
+ private final List<ExcerptAppender> appenders;
+ private final ChronicleQueue queryStoreQueue;
+ private final ExcerptAppender queryStoreAppender;
+ private final Set<Integer> finishedHosts = new HashSet<>();
+
+ public ResultStore(List<File> resultPaths, File queryFilePath)
+ {
+ queues = resultPaths.stream().map(path -> ChronicleQueueBuilder.single(path).build()).collect(Collectors.toList());
+ appenders = queues.stream().map(ChronicleQueue::acquireAppender).collect(Collectors.toList());
+ queryStoreQueue = queryFilePath != null ? ChronicleQueueBuilder.single(queryFilePath).build() : null;
+ queryStoreAppender = queryStoreQueue != null ? queryStoreQueue.acquireAppender() : null;
+ }
+
+ /**
+ * Store the column definitions in cds
+ *
+ * the ColumnDefinitions at position x will get stored by the appender at position x
+ *
+ * Calling this method indicates that we are starting a new result set from a query, it must be called before
+ * calling storeRows.
+ *
+ */
+ public void storeColumnDefinitions(FQLQuery query, List<ResultHandler.ComparableColumnDefinitions> cds)
+ {
+ finishedHosts.clear();
+ if (queryStoreAppender != null)
+ {
+ BinLog.ReleaseableWriteMarshallable writeMarshallableQuery = query.toMarshallable();
+ queryStoreAppender.writeDocument(writeMarshallableQuery);
+ writeMarshallableQuery.release();
+ }
+ for (int i = 0; i < cds.size(); i++)
+ {
+ ResultHandler.ComparableColumnDefinitions cd = cds.get(i);
+ appenders.get(i).writeDocument(new ColumnDefsWriter(cd));
+ }
+ }
+
+ /**
+ * Store rows
+ *
+ * the row at position x will get stored by appender at position x
+ *
+ * Before calling this for a new result set, storeColumnDefinitions must be called.
+ */
+ public void storeRows(List<ResultHandler.ComparableRow> rows)
+ {
+ for (int i = 0; i < rows.size(); i++)
+ {
+ ResultHandler.ComparableRow row = rows.get(i);
+ if (row == null && !finishedHosts.contains(i))
+ {
+ appenders.get(i).writeDocument(wire -> {
+ wire.write(VERSION).int16(CURRENT_VERSION);
+ wire.write(TYPE).text(END);
+ });
+ finishedHosts.add(i);
+ }
+ else if (row != null)
+ {
+ appenders.get(i).writeDocument(new RowWriter(row));
+ }
+ }
+ }
+
+ public void close()
+ {
+ queues.forEach(Closeable::close);
+ if (queryStoreQueue != null)
+ queryStoreQueue.close();
+ }
+
+ static class ColumnDefsWriter implements WriteMarshallable
+ {
+ private final ResultHandler.ComparableColumnDefinitions defs;
+
+ ColumnDefsWriter(ResultHandler.ComparableColumnDefinitions defs)
+ {
+ this.defs = defs;
+ }
+
+ public void writeMarshallable(WireOut wire)
+ {
+ wire.write(VERSION).int16(CURRENT_VERSION);
+ if (!defs.wasFailed())
+ {
+ wire.write(TYPE).text(COLUMN_DEFINITIONS);
+ wire.write(COLUMN_COUNT).int32(defs.size());
+ for (ResultHandler.ComparableDefinition d : defs.asList())
+ {
+ ValueOut vo = wire.write(COLUMN_DEFINITION);
+ vo.text(d.getName());
+ vo.text(d.getType());
+ }
+ }
+ else
+ {
+ wire.write(TYPE).text(FAILURE);
+ wire.write(MESSAGE).text(defs.getFailureException().getMessage());
+ }
+ }
+ }
+
+ static class ColumnDefsReader implements ReadMarshallable
+ {
+ boolean wasFailed;
+ String failureMessage;
+ List<Pair<String, String>> columnDefinitions = new ArrayList<>();
+
+ public void readMarshallable(WireIn wire) throws IORuntimeException
+ {
+ int version = wire.read(VERSION).int16();
+ String type = wire.read(TYPE).text();
+ if (type.equals(FAILURE))
+ {
+ wasFailed = true;
+ failureMessage = wire.read(MESSAGE).text();
+ }
+ else if (type.equals(COLUMN_DEFINITION))
+ {
+ int columnCount = wire.read(COLUMN_COUNT).int32();
+ for (int i = 0; i < columnCount; i++)
+ {
+ ValueIn vi = wire.read(COLUMN_DEFINITION);
+ String name = vi.text();
+ String dataType = vi.text();
+ columnDefinitions.add(Pair.create(name, dataType));
+ }
+ }
+ }
+ }
+
+ /**
+ * read a single row from the wire, or, marks itself finished if we read "end_resultset"
+ */
+ static class RowReader implements ReadMarshallable
+ {
+ boolean isFinished;
+ List<ByteBuffer> rows = new ArrayList<>();
+
+ public void readMarshallable(WireIn wire) throws IORuntimeException
+ {
+ int version = wire.read(VERSION).int32();
+ String type = wire.read(TYPE).text();
+ if (!type.equals(END))
+ {
+ isFinished = false;
+ int rowColumnCount = wire.read(ROW_COLUMN_COUNT).int32();
+
+ for (int i = 0; i < rowColumnCount; i++)
+ {
+ byte[] b = wire.read(COLUMN).bytes();
+ rows.add(ByteBuffer.wrap(b));
+ }
+ }
+ else
+ {
+ isFinished = true;
+ }
+ }
+ }
+
+ /**
+ * Writes a single row to the given wire
+ */
+ static class RowWriter implements WriteMarshallable
+ {
+ private final ResultHandler.ComparableRow row;
+
+ RowWriter(ResultHandler.ComparableRow row)
+ {
+ this.row = row;
+ }
+
+ public void writeMarshallable(WireOut wire)
+ {
+ wire.write(VERSION).int16(CURRENT_VERSION);
+ wire.write(TYPE).text(ROW);
+ wire.write(ROW_COLUMN_COUNT).int32(row.getColumnDefinitions().size());
+ for (int jj = 0; jj < row.getColumnDefinitions().size(); jj++)
+ {
+ ByteBuffer bb = row.getBytesUnsafe(jj);
+ if (bb != null)
+ wire.write(COLUMN).bytes(BytesStore.wrap(bb));
+ else
+ wire.write(COLUMN).bytes("NULL".getBytes());
+ }
+ }
+ }
+
+}
diff --git a/tools/fqltool/src/org/apache/cassandra/fqltool/StoredResultSet.java b/tools/fqltool/src/org/apache/cassandra/fqltool/StoredResultSet.java
new file mode 100644
index 0000000..39c4734
--- /dev/null
+++ b/tools/fqltool/src/org/apache/cassandra/fqltool/StoredResultSet.java
@@ -0,0 +1,308 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.fqltool;
+
+
+import java.nio.ByteBuffer;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Objects;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+
+import com.google.common.collect.AbstractIterator;
+
+import net.openhft.chronicle.queue.ExcerptTailer;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.Pair;
+
+/**
+ * represents a resultset defined by the format in ResultStore on disk
+ *
+ * todo: Currently all iterators need to be consumed fully while iterating over result sets
+ * if this is created from a tailer. This can probably be improved, but for all current uses it is fine.
+ */
+public class StoredResultSet implements ResultHandler.ComparableResultSet
+{
+ private final ResultHandler.ComparableColumnDefinitions defs;
+ public final boolean hasMoreResultSets;
+ private final Supplier<Iterator<ResultHandler.ComparableRow>> rowIteratorSupplier;
+ private final boolean wasFailed;
+ private final Throwable failureException;
+
+ /**
+ * create a new StoredResultSet
+ *
+ * note that we use an iteratorSupplier to be able to iterate over the same in-memory rows several times *in tests*
+ */
+ public StoredResultSet(ResultHandler.ComparableColumnDefinitions defs,
+ boolean hasMoreResultSets,
+ boolean wasFailed,
+ Throwable failure,
+ Supplier<Iterator<ResultHandler.ComparableRow>> iteratorSupplier)
+ {
+ this.defs = defs;
+ this.hasMoreResultSets = hasMoreResultSets;
+ this.wasFailed = wasFailed;
+ this.failureException = failure;
+ this.rowIteratorSupplier = iteratorSupplier;
+ }
+
+ /**
+ * creates a ComparableResultSet based on the data in tailer
+ */
+ public static StoredResultSet fromTailer(ExcerptTailer tailer)
+ {
+ ResultStore.ColumnDefsReader reader = new ResultStore.ColumnDefsReader();
+ boolean hasMoreResultSets = tailer.readDocument(reader);
+ ResultHandler.ComparableColumnDefinitions defs = new StoredComparableColumnDefinitions(reader.columnDefinitions,
+ reader.wasFailed,
+ new RuntimeException(reader.failureMessage));
+
+
+ Iterator<ResultHandler.ComparableRow> rowIterator = new AbstractIterator<ResultHandler.ComparableRow>()
+ {
+ protected ResultHandler.ComparableRow computeNext()
+ {
+ ResultStore.RowReader rowReader = new ResultStore.RowReader();
+ tailer.readDocument(rowReader);
+ if (rowReader.isFinished)
+ return endOfData();
+ return new StoredComparableRow(rowReader.rows, defs);
+ }
+ };
+
+ return new StoredResultSet(defs,
+ hasMoreResultSets,
+ reader.wasFailed,
+ new RuntimeException(reader.failureMessage),
+ () -> rowIterator);
+ }
+
+ public static ResultHandler.ComparableResultSet failed(String failureMessage)
+ {
+ return new FailedComparableResultSet(new RuntimeException(failureMessage));
+ }
+
+ public Iterator<ResultHandler.ComparableRow> iterator()
+ {
+ return rowIteratorSupplier.get();
+ }
+
+ public ResultHandler.ComparableColumnDefinitions getColumnDefinitions()
+ {
+ return defs;
+ }
+
+ public boolean wasFailed()
+ {
+ return wasFailed;
+ }
+
+ public Throwable getFailureException()
+ {
+ return failureException;
+ }
+
+ static class StoredComparableRow implements ResultHandler.ComparableRow
+ {
+ private final List<ByteBuffer> row;
+ private final ResultHandler.ComparableColumnDefinitions cds;
+
+ public StoredComparableRow(List<ByteBuffer> row, ResultHandler.ComparableColumnDefinitions cds)
+ {
+ this.row = row;
+ this.cds = cds;
+ }
+
+ public ByteBuffer getBytesUnsafe(int i)
+ {
+ return row.get(i);
+ }
+
+ public ResultHandler.ComparableColumnDefinitions getColumnDefinitions()
+ {
+ return cds;
+ }
+
+ public boolean equals(Object other)
+ {
+ if (!(other instanceof StoredComparableRow))
+ return false;
+ return row.equals(((StoredComparableRow)other).row);
+ }
+
+ public int hashCode()
+ {
+ return Objects.hash(row, cds);
+ }
+
+ public String toString()
+ {
+ return row.stream().map(ByteBufferUtil::bytesToHex).collect(Collectors.joining(","));
+ }
+ }
+
+ static class StoredComparableColumnDefinitions implements ResultHandler.ComparableColumnDefinitions
+ {
+ private final List<ResultHandler.ComparableDefinition> defs;
+ private final boolean wasFailed;
+ private final Throwable failureException;
+
+ public StoredComparableColumnDefinitions(List<Pair<String, String>> cds, boolean wasFailed, Throwable failureException)
+ {
+ defs = cds != null ? cds.stream().map(StoredComparableDefinition::new).collect(Collectors.toList()) : Collections.emptyList();
+ this.wasFailed = wasFailed;
+ this.failureException = failureException;
+ }
+ public List<ResultHandler.ComparableDefinition> asList()
+ {
+ return wasFailed() ? Collections.emptyList() : defs;
+ }
+
+ public boolean wasFailed()
+ {
+ return wasFailed;
+ }
+
+ public Throwable getFailureException()
+ {
+ return failureException;
+ }
+
+ public int size()
+ {
+ return asList().size();
+ }
+
+ public Iterator<ResultHandler.ComparableDefinition> iterator()
+ {
+ return defs.iterator();
+ }
+
+ public boolean equals(Object other)
+ {
+ if (!(other instanceof StoredComparableColumnDefinitions))
+ return false;
+ return defs.equals(((StoredComparableColumnDefinitions)other).defs);
+ }
+
+ public int hashCode()
+ {
+ return Objects.hash(defs, wasFailed, failureException);
+ }
+
+ public String toString()
+ {
+ return defs.toString();
+ }
+ }
+
+ private static class StoredComparableDefinition implements ResultHandler.ComparableDefinition
+ {
+ private final Pair<String, String> p;
+
+ public StoredComparableDefinition(Pair<String, String> p)
+ {
+ this.p = p;
+ }
+ public String getType()
+ {
+ return p.right;
+ }
+
+ public String getName()
+ {
+ return p.left;
+ }
+
+ public boolean equals(Object other)
+ {
+ if (!(other instanceof StoredComparableDefinition))
+ return false;
+ return p.equals(((StoredComparableDefinition)other).p);
+ }
+
+ public int hashCode()
+ {
+ return Objects.hash(p);
+ }
+
+ public String toString()
+ {
+ return getName() + ':' + getType();
+ }
+ }
+
+ private static class FailedComparableResultSet implements ResultHandler.ComparableResultSet
+ {
+ private final Throwable exception;
+
+ public FailedComparableResultSet(Throwable exception)
+ {
+ this.exception = exception;
+ }
+ public ResultHandler.ComparableColumnDefinitions getColumnDefinitions()
+ {
+ return new ResultHandler.ComparableColumnDefinitions()
+ {
+ public List<ResultHandler.ComparableDefinition> asList()
+ {
+ return Collections.emptyList();
+ }
+
+ public boolean wasFailed()
+ {
+ return true;
+ }
+
+ public Throwable getFailureException()
+ {
+ return exception;
+ }
+
+ public int size()
+ {
+ return 0;
+ }
+
+ public Iterator<ResultHandler.ComparableDefinition> iterator()
+ {
+ return asList().iterator();
+ }
+ };
+ }
+
+ public boolean wasFailed()
+ {
+ return true;
+ }
+
+ public Throwable getFailureException()
+ {
+ return new RuntimeException();
+ }
+
+ public Iterator<ResultHandler.ComparableRow> iterator()
+ {
+ return Collections.emptyListIterator();
+ }
+ }
+}
diff --git a/tools/fqltool/src/org/apache/cassandra/fqltool/commands/Compare.java b/tools/fqltool/src/org/apache/cassandra/fqltool/commands/Compare.java
new file mode 100644
index 0000000..2375296
--- /dev/null
+++ b/tools/fqltool/src/org/apache/cassandra/fqltool/commands/Compare.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.fqltool.commands;
+
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.AbstractIterator;
+
+import io.airlift.airline.Arguments;
+import io.airlift.airline.Command;
+import io.airlift.airline.Option;
+import net.openhft.chronicle.core.io.Closeable;
+import net.openhft.chronicle.queue.ChronicleQueue;
+import net.openhft.chronicle.queue.ChronicleQueueBuilder;
+import net.openhft.chronicle.queue.ExcerptTailer;
+import org.apache.cassandra.fqltool.FQLQueryIterator;
+import org.apache.cassandra.fqltool.ResultHandler;
+import org.apache.cassandra.fqltool.StoredResultSet;
+
+/**
+ */
+@Command(name = "compare", description = "Compare result files generated by fqltool replay")
+public class Compare implements Runnable
+{
+ @Arguments(usage = "<path1> [<path2>...<pathN>]",
+ description = "Directories containing result files to compare.",
+ required = true)
+ private List<String> arguments = new ArrayList<>();
+
+ @Option(title = "queries",
+ name = { "--queries"},
+ description = "Directory to read the queries from. It is produced by the fqltool replay --store-queries option. ",
+ required = true)
+ private String querylog;
+
+ @Override
+ public void run()
+ {
+ compare(querylog, arguments);
+ }
+
+ public static void compare(String querylog, List<String> arguments)
+ {
+ List<ChronicleQueue> readQueues = null;
+ try (ResultHandler rh = new ResultHandler(arguments, null, null);
+ ChronicleQueue queryQ = ChronicleQueueBuilder.single(querylog).readOnly(true).build();
+ FQLQueryIterator queries = new FQLQueryIterator(queryQ.createTailer(), 1))
+ {
+ readQueues = arguments.stream().map(s -> ChronicleQueueBuilder.single(s).readOnly(true).build()).collect(Collectors.toList());
+ List<Iterator<ResultHandler.ComparableResultSet>> its = readQueues.stream().map(q -> new StoredResultSetIterator(q.createTailer())).collect(Collectors.toList());
+ while (queries.hasNext())
+ rh.handleResults(queries.next(), resultSets(its));
+ }
+ catch (Exception e)
+ {
+ throw new RuntimeException(e);
+ }
+ finally
+ {
+ if (readQueues != null)
+ readQueues.forEach(Closeable::close);
+ }
+ }
+
+ @VisibleForTesting
+ public static List<ResultHandler.ComparableResultSet> resultSets(List<Iterator<ResultHandler.ComparableResultSet>> its)
+ {
+ List<ResultHandler.ComparableResultSet> resultSets = new ArrayList<>(its.size());
+ for (Iterator<ResultHandler.ComparableResultSet> it : its)
+ {
+ if (it.hasNext())
+ resultSets.add(it.next());
+ else
+ resultSets.add(null);
+ }
+ return resultSets;
+ }
+
+ @VisibleForTesting
+ public static class StoredResultSetIterator extends AbstractIterator<ResultHandler.ComparableResultSet>
+ {
+ private final ExcerptTailer tailer;
+
+ public StoredResultSetIterator(ExcerptTailer tailer)
+ {
+ this.tailer = tailer;
+ }
+
+ protected ResultHandler.ComparableResultSet computeNext()
+ {
+ StoredResultSet srs = StoredResultSet.fromTailer(tailer);
+ if (srs.hasMoreResultSets)
+ return srs;
+ return endOfData();
+ }
+ }
+
+
+}
diff --git a/tools/fqltool/src/org/apache/cassandra/fqltool/commands/Dump.java b/tools/fqltool/src/org/apache/cassandra/fqltool/commands/Dump.java
new file mode 100644
index 0000000..dfc3b09
--- /dev/null
+++ b/tools/fqltool/src/org/apache/cassandra/fqltool/commands/Dump.java
@@ -0,0 +1,336 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.fqltool.commands;
+
+import java.io.File;
+import java.nio.BufferUnderflowException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import io.airlift.airline.Arguments;
+import io.airlift.airline.Command;
+import io.airlift.airline.Option;
+import io.netty.buffer.Unpooled;
+import net.openhft.chronicle.bytes.Bytes;
+import net.openhft.chronicle.core.io.IORuntimeException;
+import net.openhft.chronicle.queue.ChronicleQueue;
+import net.openhft.chronicle.queue.ChronicleQueueBuilder;
+import net.openhft.chronicle.queue.ExcerptTailer;
+import net.openhft.chronicle.queue.RollCycles;
+import net.openhft.chronicle.threads.Pauser;
+import net.openhft.chronicle.wire.ReadMarshallable;
+import net.openhft.chronicle.wire.ValueIn;
+import net.openhft.chronicle.wire.WireIn;
+import org.apache.cassandra.fql.FullQueryLogger;
+import org.apache.cassandra.cql3.QueryOptions;
+import org.apache.cassandra.transport.ProtocolVersion;
+import org.apache.cassandra.utils.binlog.BinLog;
+
+/**
+ * Dump the contents of a list of paths containing full query logs
+ */
+@Command(name = "dump", description = "Dump the contents of a full query log")
+public class Dump implements Runnable
+{
+ static final char[] HEXI_DECIMAL = "0123456789ABCDEF".toCharArray();
+
+ @Arguments(usage = "<path1> [<path2>...<pathN>]", description = "Path containing the full query logs to dump.", required = true)
+ private List<String> arguments = new ArrayList<>();
+
+ @Option(title = "roll_cycle", name = {"--roll-cycle"}, description = "How often to roll the log file was rolled. May be necessary for Chronicle to correctly parse file names. (MINUTELY, HOURLY, DAILY). Default HOURLY.")
+ private String rollCycle = "HOURLY";
+
+ @Option(title = "follow", name = {"--follow"}, description = "Upon reacahing the end of the log continue indefinitely waiting for more records")
+ private boolean follow = false;
+
+ @Override
+ public void run()
+ {
+ dump(arguments, rollCycle, follow);
+ }
+
+ public static void dump(List<String> arguments, String rollCycle, boolean follow)
+ {
+ StringBuilder sb = new StringBuilder();
+ ReadMarshallable reader = wireIn ->
+ {
+ sb.setLength(0);
+
+ int version = wireIn.read(BinLog.VERSION).int16();
+ if (version > FullQueryLogger.CURRENT_VERSION)
+ {
+ throw new IORuntimeException("Unsupported record version [" + version
+ + "] - highest supported version is [" + FullQueryLogger.CURRENT_VERSION + ']');
+ }
+
+ String type = wireIn.read(BinLog.TYPE).text();
+ if (!FullQueryLogger.SINGLE_QUERY.equals((type)) && !FullQueryLogger.BATCH.equals((type)))
+ {
+ throw new IORuntimeException("Unsupported record type field [" + type
+ + "] - supported record types are [" + FullQueryLogger.SINGLE_QUERY + ", " + FullQueryLogger.BATCH + ']');
+ }
+
+ sb.append("Type: ")
+ .append(type)
+ .append(System.lineSeparator());
+
+ long queryStartTime = wireIn.read(FullQueryLogger.QUERY_START_TIME).int64();
+ sb.append("Query start time: ")
+ .append(queryStartTime)
+ .append(System.lineSeparator());
+
+ int protocolVersion = wireIn.read(FullQueryLogger.PROTOCOL_VERSION).int32();
+ sb.append("Protocol version: ")
+ .append(protocolVersion)
+ .append(System.lineSeparator());
+
+ QueryOptions options =
+ QueryOptions.codec.decode(Unpooled.wrappedBuffer(wireIn.read(FullQueryLogger.QUERY_OPTIONS).bytes()),
+ ProtocolVersion.decode(protocolVersion, true));
+
+ long generatedTimestamp = wireIn.read(FullQueryLogger.GENERATED_TIMESTAMP).int64();
+ sb.append("Generated timestamp:")
+ .append(generatedTimestamp)
+ .append(System.lineSeparator());
+
+ int generatedNowInSeconds = wireIn.read(FullQueryLogger.GENERATED_NOW_IN_SECONDS).int32();
+ sb.append("Generated nowInSeconds:")
+ .append(generatedNowInSeconds)
+ .append(System.lineSeparator());
+
+ switch (type)
+ {
+ case (FullQueryLogger.SINGLE_QUERY):
+ dumpQuery(options, wireIn, sb);
+ break;
+
+ case (FullQueryLogger.BATCH):
+ dumpBatch(options, wireIn, sb);
+ break;
+
+ default:
+ throw new IORuntimeException("Log entry of unsupported type " + type);
+ }
+
+ System.out.print(sb.toString());
+ System.out.flush();
+ };
+
+ //Backoff strategy for spinning on the queue, not aggressive at all as this doesn't need to be low latency
+ Pauser pauser = Pauser.millis(100);
+ List<ChronicleQueue> queues = arguments.stream().distinct().map(path -> ChronicleQueueBuilder.single(new File(path)).readOnly(true).rollCycle(RollCycles.valueOf(rollCycle)).build()).collect(Collectors.toList());
+ List<ExcerptTailer> tailers = queues.stream().map(ChronicleQueue::createTailer).collect(Collectors.toList());
+ boolean hadWork = true;
+ while (hadWork)
+ {
+ hadWork = false;
+ for (ExcerptTailer tailer : tailers)
+ {
+ while (tailer.readDocument(reader))
+ {
+ hadWork = true;
+ }
+ }
+
+ if (follow)
+ {
+ if (!hadWork)
+ {
+ //Chronicle queue doesn't support blocking so use this backoff strategy
+ pauser.pause();
+ }
+ //Don't terminate the loop even if there wasn't work
+ hadWork = true;
+ }
+ }
+ }
+
+ private static void dumpQuery(QueryOptions options, WireIn wireIn, StringBuilder sb)
+ {
+ sb.append("Query: ")
+ .append(wireIn.read(FullQueryLogger.QUERY).text())
+ .append(System.lineSeparator());
+
+ List<ByteBuffer> values = options.getValues() != null
+ ? options.getValues()
+ : Collections.emptyList();
+
+ sb.append("Values: ")
+ .append(System.lineSeparator());
+ appendValuesToStringBuilder(values, sb);
+ sb.append(System.lineSeparator());
+ }
+
+ private static void dumpBatch(QueryOptions options, WireIn wireIn, StringBuilder sb)
+ {
+ sb.append("Batch type: ")
+ .append(wireIn.read(FullQueryLogger.BATCH_TYPE).text())
+ .append(System.lineSeparator());
+
+ ValueIn in = wireIn.read(FullQueryLogger.QUERIES);
+ int numQueries = in.int32();
+ List<String> queries = new ArrayList<>(numQueries);
+ for (int i = 0; i < numQueries; i++)
+ queries.add(in.text());
+
+ in = wireIn.read(FullQueryLogger.VALUES);
+ int numValues = in.int32();
+
+ for (int i = 0; i < numValues; i++)
+ {
+ int numSubValues = in.int32();
+ List<ByteBuffer> subValues = new ArrayList<>(numSubValues);
+ for (int j = 0; j < numSubValues; j++)
+ subValues.add(ByteBuffer.wrap(in.bytes()));
+
+ sb.append("Query: ")
+ .append(queries.get(i))
+ .append(System.lineSeparator());
+
+ sb.append("Values: ")
+ .append(System.lineSeparator());
+ appendValuesToStringBuilder(subValues, sb);
+ }
+
+ sb.append(System.lineSeparator());
+ }
+
+ private static void appendValuesToStringBuilder(List<ByteBuffer> values, StringBuilder sb)
+ {
+ boolean first = true;
+ for (ByteBuffer value : values)
+ {
+ Bytes bytes = Bytes.wrapForRead(value);
+ long maxLength2 = Math.min(1024, bytes.readLimit() - bytes.readPosition());
+ toHexString(bytes, bytes.readPosition(), maxLength2, sb);
+ if (maxLength2 < bytes.readLimit() - bytes.readPosition())
+ {
+ sb.append("... truncated").append(System.lineSeparator());
+ }
+
+ if (first)
+ {
+ first = false;
+ }
+ else
+ {
+ sb.append("-----").append(System.lineSeparator());
+ }
+ }
+ }
+
+ //This is from net.openhft.chronicle.bytes, need to pass in the StringBuilder so had to copy
+ /*
+ * Copyright 2016 higherfrequencytrading.com
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+ /**
+ * display the hex data of {@link Bytes} from the position() to the limit()
+ *
+ * @param bytes the buffer you wish to toString()
+ * @return hex representation of the buffer, from example [0D ,OA, FF]
+ */
+ public static String toHexString(final Bytes bytes, long offset, long len, StringBuilder builder)
+ throws BufferUnderflowException
+ {
+ if (len == 0)
+ return "";
+
+ int width = 16;
+ int[] lastLine = new int[width];
+ String sep = "";
+ long position = bytes.readPosition();
+ long limit = bytes.readLimit();
+
+ try {
+ bytes.readPositionRemaining(offset, len);
+
+ long start = offset / width * width;
+ long end = (offset + len + width - 1) / width * width;
+ for (long i = start; i < end; i += width) {
+ // check for duplicate rows
+ if (i + width < end) {
+ boolean same = true;
+
+ for (int j = 0; j < width && i + j < offset + len; j++) {
+ int ch = bytes.readUnsignedByte(i + j);
+ same &= (ch == lastLine[j]);
+ lastLine[j] = ch;
+ }
+ if (i > start && same) {
+ sep = "........\n";
+ continue;
+ }
+ }
+ builder.append(sep);
+ sep = "";
+ String str = Long.toHexString(i);
+ for (int j = str.length(); j < 8; j++)
+ builder.append('0');
+ builder.append(str);
+ for (int j = 0; j < width; j++) {
+ if (j == width / 2)
+ builder.append(' ');
+ if (i + j < offset || i + j >= offset + len) {
+ builder.append(" ");
+
+ } else {
+ builder.append(' ');
+ int ch = bytes.readUnsignedByte(i + j);
+ builder.append(HEXI_DECIMAL[ch >> 4]);
+ builder.append(HEXI_DECIMAL[ch & 15]);
+ }
+ }
+ builder.append(' ');
+ for (int j = 0; j < width; j++) {
+ if (j == width / 2)
+ builder.append(' ');
+ if (i + j < offset || i + j >= offset + len) {
+ builder.append(' ');
+
+ } else {
+ int ch = bytes.readUnsignedByte(i + j);
+ if (ch < ' ' || ch > 126)
+ ch = '\u00B7';
+ builder.append((char) ch);
+ }
+ }
+ builder.append("\n");
+ }
+ return builder.toString();
+ } finally {
+ bytes.readLimit(limit);
+ bytes.readPosition(position);
+ }
+ }
+}
diff --git a/tools/fqltool/src/org/apache/cassandra/fqltool/commands/Replay.java b/tools/fqltool/src/org/apache/cassandra/fqltool/commands/Replay.java
new file mode 100644
index 0000000..9cc147a
--- /dev/null
+++ b/tools/fqltool/src/org/apache/cassandra/fqltool/commands/Replay.java
@@ -0,0 +1,145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.fqltool.commands;
+
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import io.airlift.airline.Arguments;
+import io.airlift.airline.Command;
+import io.airlift.airline.Option;
+import net.openhft.chronicle.core.io.Closeable;
+import net.openhft.chronicle.queue.ChronicleQueue;
+import net.openhft.chronicle.queue.ChronicleQueueBuilder;
+
+import org.apache.cassandra.fqltool.FQLQuery;
+import org.apache.cassandra.fqltool.FQLQueryIterator;
+import org.apache.cassandra.fqltool.QueryReplayer;
+import org.apache.cassandra.utils.AbstractIterator;
+import org.apache.cassandra.utils.MergeIterator;
+
+/**
+ * replay the contents of a list of paths containing full query logs
+ */
+@Command(name = "replay", description = "Replay full query logs")
+public class Replay implements Runnable
+{
+ @Arguments(usage = "<path1> [<path2>...<pathN>]", description = "Paths containing the full query logs to replay.", required = true)
+ private List<String> arguments = new ArrayList<>();
+
+ @Option(title = "target", name = {"--target"}, description = "Hosts to replay the logs to, can be repeated to replay to more hosts.", required = true)
+ private List<String> targetHosts;
+
+ @Option(title = "results", name = { "--results"}, description = "Where to store the results of the queries, this should be a directory. Leave this option out to avoid storing results.")
+ private String resultPath;
+
+ @Option(title = "keyspace", name = { "--keyspace"}, description = "Only replay queries against this keyspace and queries without keyspace set.")
+ private String keyspace;
+
+ @Option(title = "store_queries", name = {"--store-queries"}, description = "Path to store the queries executed. Stores queries in the same order as the result sets are in the result files. Requires --results")
+ private String queryStorePath;
+
+ @Override
+ public void run()
+ {
+ try
+ {
+ List<File> resultPaths = null;
+ if (resultPath != null)
+ {
+ File basePath = new File(resultPath);
+ if (!basePath.exists() || !basePath.isDirectory())
+ {
+ System.err.println("The results path (" + basePath + ") should be an existing directory");
+ System.exit(1);
+ }
+ resultPaths = targetHosts.stream().map(target -> new File(basePath, target)).collect(Collectors.toList());
+ resultPaths.forEach(File::mkdir);
+ }
+ if (targetHosts.size() < 1)
+ {
+ System.err.println("You need to state at least one --target host to replay the query against");
+ System.exit(1);
+ }
+ replay(keyspace, arguments, targetHosts, resultPaths, queryStorePath);
+ }
+ catch (Exception e)
+ {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public static void replay(String keyspace, List<String> arguments, List<String> targetHosts, List<File> resultPaths, String queryStorePath)
+ {
+ int readAhead = 200; // how many fql queries should we read in to memory to be able to sort them?
+ List<ChronicleQueue> readQueues = null;
+ List<FQLQueryIterator> iterators = null;
+ List<Predicate<FQLQuery>> filters = new ArrayList<>();
+
+ if (keyspace != null)
+ filters.add(fqlQuery -> fqlQuery.keyspace() == null || fqlQuery.keyspace().equals(keyspace));
+
+ try
+ {
+ readQueues = arguments.stream().map(s -> ChronicleQueueBuilder.single(s).readOnly(true).build()).collect(Collectors.toList());
+ iterators = readQueues.stream().map(ChronicleQueue::createTailer).map(tailer -> new FQLQueryIterator(tailer, readAhead)).collect(Collectors.toList());
+ try (MergeIterator<FQLQuery, List<FQLQuery>> iter = MergeIterator.get(iterators, FQLQuery::compareTo, new Reducer());
+ QueryReplayer replayer = new QueryReplayer(iter, targetHosts, resultPaths, filters, queryStorePath))
+ {
+ replayer.replay();
+ }
+ }
+ catch (Exception e)
+ {
+ throw new RuntimeException(e);
+ }
+ finally
+ {
+ if (iterators != null)
+ iterators.forEach(AbstractIterator::close);
+ if (readQueues != null)
+ readQueues.forEach(Closeable::close);
+ }
+ }
+
+ @VisibleForTesting
+ public static class Reducer extends MergeIterator.Reducer<FQLQuery, List<FQLQuery>>
+ {
+ List<FQLQuery> queries = new ArrayList<>();
+ public void reduce(int idx, FQLQuery current)
+ {
+ queries.add(current);
+ }
+
+ protected List<FQLQuery> getReduced()
+ {
+ return queries;
+ }
+ protected void onKeyChange()
+ {
+ queries.clear();
+ }
+ }
+}
diff --git a/tools/fqltool/test/unit/org/apache/cassandra/fqltool/FQLCompareTest.java b/tools/fqltool/test/unit/org/apache/cassandra/fqltool/FQLCompareTest.java
new file mode 100644
index 0000000..7990b7e
--- /dev/null
+++ b/tools/fqltool/test/unit/org/apache/cassandra/fqltool/FQLCompareTest.java
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.fqltool;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
+import com.google.common.collect.Lists;
+import org.junit.Test;
+
+import net.openhft.chronicle.core.io.Closeable;
+import net.openhft.chronicle.queue.ChronicleQueue;
+import net.openhft.chronicle.queue.ChronicleQueueBuilder;
+import org.apache.cassandra.cql3.QueryOptions;
+import org.apache.cassandra.fqltool.commands.Compare;
+import org.apache.cassandra.tools.Util;
+
+
+import static org.psjava.util.AssertStatus.assertTrue;
+
+public class FQLCompareTest
+{
+ public FQLCompareTest()
+ {
+ Util.initDatabaseDescriptor();
+ }
+
+ @Test
+ public void endToEnd() throws IOException
+ {
+ List<String> targetHosts = Lists.newArrayList("hosta", "hostb");
+ File tmpDir = Files.createTempDirectory("testresulthandler").toFile();
+ File queryDir = Files.createTempDirectory("queries").toFile();
+ List<File> resultPaths = generateResultSets(targetHosts, tmpDir, queryDir, true, false);
+ Compare.compare(queryDir.toString(), resultPaths.stream().map(File::toString).collect(Collectors.toList()));
+ }
+
+ @Test
+ public void endToEndQueryFailures() throws IOException
+ {
+ List<String> targetHosts = Lists.newArrayList("hosta", "hostb");
+ File tmpDir = Files.createTempDirectory("testresulthandler").toFile();
+ File queryDir = Files.createTempDirectory("queries").toFile();
+ List<File> resultPaths = generateResultSets(targetHosts, tmpDir, queryDir, true,true);
+ Compare.compare(queryDir.toString(), resultPaths.stream().map(File::toString).collect(Collectors.toList()));
+ }
+
+ @Test
+ public void compareEqual() throws IOException
+ {
+ List<String> targetHosts = Lists.newArrayList("hosta", "hostb");
+ File tmpDir = Files.createTempDirectory("testresulthandler").toFile();
+ File queryDir = Files.createTempDirectory("queries").toFile();
+ List<File> resultPaths = generateResultSets(targetHosts, tmpDir, queryDir, false,false);
+
+ ResultComparator comparator = new ResultComparator();
+ List<ChronicleQueue> readQueues = null;
+ try
+ {
+ readQueues = resultPaths.stream().map(s -> ChronicleQueueBuilder.single(s).readOnly(true).build()).collect(Collectors.toList());
+ List<Iterator<ResultHandler.ComparableResultSet>> its = readQueues.stream().map(q -> new Compare.StoredResultSetIterator(q.createTailer())).collect(Collectors.toList());
+ List<ResultHandler.ComparableResultSet> resultSets = Compare.resultSets(its);
+ while(resultSets.stream().allMatch(Objects::nonNull))
+ {
+ assertTrue(comparator.compareColumnDefinitions(targetHosts, query(), resultSets.stream().map(ResultHandler.ComparableResultSet::getColumnDefinitions).collect(Collectors.toList())));
+ List<Iterator<ResultHandler.ComparableRow>> rows = resultSets.stream().map(Iterable::iterator).collect(Collectors.toList());
+
+ List<ResultHandler.ComparableRow> toCompare = ResultHandler.rows(rows);
+
+ while (toCompare.stream().allMatch(Objects::nonNull))
+ {
+ assertTrue(comparator.compareRows(targetHosts, query(), ResultHandler.rows(rows)));
+ toCompare = ResultHandler.rows(rows);
+ }
+ resultSets = Compare.resultSets(its);
+ }
+ }
+ finally
+ {
+ if (readQueues != null)
+ readQueues.forEach(Closeable::close);
+ }
+ }
+
+ private List<File> generateResultSets(List<String> targetHosts, File resultDir, File queryDir, boolean random, boolean includeFailures) throws IOException
+ {
+ List<File> resultPaths = new ArrayList<>();
+ targetHosts.forEach(host -> { File f = new File(resultDir, host); f.mkdir(); resultPaths.add(f);});
+
+ try (ResultHandler rh = new ResultHandler(targetHosts, resultPaths, queryDir))
+ {
+ for (int i = 0; i < 100; i++)
+ {
+ ResultHandler.ComparableResultSet resultSet1 = includeFailures && (i % 10 == 0)
+ ? StoredResultSet.failed("test failure!")
+ : FQLReplayTest.createResultSet(10, 10, random);
+ ResultHandler.ComparableResultSet resultSet2 = FQLReplayTest.createResultSet(10, 10, random);
+ rh.handleResults(query(), Lists.newArrayList(resultSet1, resultSet2));
+ }
+ }
+ return resultPaths;
+ }
+
+ private FQLQuery.Single query()
+ {
+ return new FQLQuery.Single("abc", QueryOptions.DEFAULT.getProtocolVersion().asInt(), QueryOptions.DEFAULT, 12345, 5555, 6666, "select * from xyz", Collections.emptyList());
+ }
+}
diff --git a/tools/fqltool/test/unit/org/apache/cassandra/fqltool/FQLReplayTest.java b/tools/fqltool/test/unit/org/apache/cassandra/fqltool/FQLReplayTest.java
new file mode 100644
index 0000000..7fb39af
--- /dev/null
+++ b/tools/fqltool/test/unit/org/apache/cassandra/fqltool/FQLReplayTest.java
@@ -0,0 +1,802 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.fqltool;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Objects;
+import java.util.Random;
+import java.util.stream.Collectors;
+
+import com.google.common.collect.Lists;
+import org.junit.Test;
+
+import com.datastax.driver.core.CodecRegistry;
+import com.datastax.driver.core.SimpleStatement;
+import com.datastax.driver.core.Statement;
+import net.openhft.chronicle.core.io.IORuntimeException;
+import net.openhft.chronicle.queue.ChronicleQueue;
+import net.openhft.chronicle.queue.ChronicleQueueBuilder;
+import net.openhft.chronicle.queue.ExcerptAppender;
+import net.openhft.chronicle.queue.ExcerptTailer;
+import net.openhft.chronicle.wire.WireOut;
+import org.apache.cassandra.fql.FullQueryLogger;
+import org.apache.cassandra.cql3.QueryOptions;
+import org.apache.cassandra.cql3.statements.BatchStatement;
+import org.apache.cassandra.fqltool.commands.Compare;
+import org.apache.cassandra.fqltool.commands.Replay;
+import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.service.QueryState;
+import org.apache.cassandra.tools.Util;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.MergeIterator;
+import org.apache.cassandra.utils.Pair;
+import org.apache.cassandra.utils.binlog.BinLog;
+
+import static org.apache.cassandra.fqltool.QueryReplayer.ParsedTargetHost.fromString;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+public class FQLReplayTest
+{
+ public FQLReplayTest()
+ {
+ Util.initDatabaseDescriptor();
+ }
+
+ @Test
+ public void testOrderedReplay() throws IOException
+ {
+ File f = generateQueries(100, true);
+ int queryCount = 0;
+ try (ChronicleQueue queue = ChronicleQueueBuilder.single(f).build();
+ FQLQueryIterator iter = new FQLQueryIterator(queue.createTailer(), 101))
+ {
+ long last = -1;
+ while (iter.hasNext())
+ {
+ FQLQuery q = iter.next();
+ assertTrue(q.queryStartTime >= last);
+ last = q.queryStartTime;
+ queryCount++;
+ }
+ }
+ assertEquals(100, queryCount);
+ }
+
+ @Test
+ public void testQueryIterator() throws IOException
+ {
+ File f = generateQueries(100, false);
+ int queryCount = 0;
+ try (ChronicleQueue queue = ChronicleQueueBuilder.single(f).build();
+ FQLQueryIterator iter = new FQLQueryIterator(queue.createTailer(), 1))
+ {
+ long last = -1;
+ while (iter.hasNext())
+ {
+ FQLQuery q = iter.next();
+ assertTrue(q.queryStartTime >= last);
+ last = q.queryStartTime;
+ queryCount++;
+ }
+ }
+ assertEquals(100, queryCount);
+ }
+
+ @Test
+ public void testMergingIterator() throws IOException
+ {
+ File f = generateQueries(100, false);
+ File f2 = generateQueries(100, false);
+ int queryCount = 0;
+ try (ChronicleQueue queue = ChronicleQueueBuilder.single(f).build();
+ ChronicleQueue queue2 = ChronicleQueueBuilder.single(f2).build();
+ FQLQueryIterator iter = new FQLQueryIterator(queue.createTailer(), 101);
+ FQLQueryIterator iter2 = new FQLQueryIterator(queue2.createTailer(), 101);
+ MergeIterator<FQLQuery, List<FQLQuery>> merger = MergeIterator.get(Lists.newArrayList(iter, iter2), FQLQuery::compareTo, new Replay.Reducer()))
+ {
+ long last = -1;
+
+ while (merger.hasNext())
+ {
+ List<FQLQuery> qs = merger.next();
+ assertEquals(2, qs.size());
+ assertEquals(0, qs.get(0).compareTo(qs.get(1)));
+ assertTrue(qs.get(0).queryStartTime >= last);
+ last = qs.get(0).queryStartTime;
+ queryCount++;
+ }
+ }
+ assertEquals(100, queryCount);
+ }
+
+ @Test
+ public void testFQLQueryReader() throws IOException
+ {
+ FQLQueryReader reader = new FQLQueryReader();
+
+ try (ChronicleQueue queue = ChronicleQueueBuilder.single(generateQueries(1000, true)).build())
+ {
+ ExcerptTailer tailer = queue.createTailer();
+ int queryCount = 0;
+ while (tailer.readDocument(reader))
+ {
+ assertNotNull(reader.getQuery());
+ if (reader.getQuery() instanceof FQLQuery.Single)
+ {
+ assertTrue(reader.getQuery().keyspace() == null || reader.getQuery().keyspace().equals("querykeyspace"));
+ }
+ else
+ {
+ assertEquals("someks", reader.getQuery().keyspace());
+ }
+ queryCount++;
+ }
+ assertEquals(1000, queryCount);
+ }
+ }
+
+ @Test
+ public void testStoringResults() throws Throwable
+ {
+ File tmpDir = Files.createTempDirectory("results").toFile();
+ File queryDir = Files.createTempDirectory("queries").toFile();
+
+ ResultHandler.ComparableResultSet res = createResultSet(10, 10, true);
+ ResultStore rs = new ResultStore(Collections.singletonList(tmpDir), queryDir);
+ FQLQuery query = new FQLQuery.Single("abc", QueryOptions.DEFAULT.getProtocolVersion().asInt(), QueryOptions.DEFAULT, 12345, 11111, 22, "select * from abc", Collections.emptyList());
+ try
+ {
+ rs.storeColumnDefinitions(query, Collections.singletonList(res.getColumnDefinitions()));
+ Iterator<ResultHandler.ComparableRow> it = res.iterator();
+ while (it.hasNext())
+ {
+ List<ResultHandler.ComparableRow> row = Collections.singletonList(it.next());
+ rs.storeRows(row);
+ }
+ // this marks the end of the result set:
+ rs.storeRows(Collections.singletonList(null));
+ }
+ finally
+ {
+ rs.close();
+ }
+
+ compareResults(Collections.singletonList(Pair.create(query, res)),
+ readResultFile(tmpDir, queryDir));
+
+ }
+
+ private static void compareResults(List<Pair<FQLQuery, ResultHandler.ComparableResultSet>> expected,
+ List<Pair<FQLQuery, ResultHandler.ComparableResultSet>> other)
+ {
+ ResultComparator comparator = new ResultComparator();
+ assertEquals(expected.size(), other.size());
+ for (int i = 0; i < expected.size(); i++)
+ {
+ assertEquals(expected.get(i).left, other.get(i).left);
+ ResultHandler.ComparableResultSet expectedResultSet = expected.get(i).right;
+ ResultHandler.ComparableResultSet otherResultSet = other.get(i).right;
+ List<String> hosts = Lists.newArrayList("a", "b");
+ comparator.compareColumnDefinitions(hosts,
+ expected.get(i).left,
+ Lists.newArrayList(expectedResultSet.getColumnDefinitions(),
+ otherResultSet.getColumnDefinitions()));
+ Iterator<ResultHandler.ComparableRow> expectedRowIter = expectedResultSet.iterator();
+ Iterator<ResultHandler.ComparableRow> otherRowIter = otherResultSet.iterator();
+
+
+ while(expectedRowIter.hasNext() && otherRowIter.hasNext())
+ {
+ ResultHandler.ComparableRow expectedRow = expectedRowIter.next();
+ ResultHandler.ComparableRow otherRow = otherRowIter.next();
+ assertTrue(comparator.compareRows(hosts, expected.get(i).left, Lists.newArrayList(expectedRow, otherRow)));
+ }
+ assertFalse(expectedRowIter.hasNext());
+ assertFalse(otherRowIter.hasNext());
+ }
+ }
+
+ @Test
+ public void testCompareColumnDefinitions()
+ {
+ ResultHandler.ComparableResultSet res = createResultSet(10, 10, false);
+ ResultComparator rc = new ResultComparator();
+
+ List<ResultHandler.ComparableColumnDefinitions> colDefs = new ArrayList<>(100);
+ List<String> targetHosts = new ArrayList<>(100);
+ for (int i = 0; i < 100; i++)
+ {
+ targetHosts.add("host"+i);
+ colDefs.add(res.getColumnDefinitions());
+ }
+ assertTrue(rc.compareColumnDefinitions(targetHosts, null, colDefs));
+ colDefs.set(50, createResultSet(9, 9, false).getColumnDefinitions());
+ assertFalse(rc.compareColumnDefinitions(targetHosts, null, colDefs));
+ }
+
+ @Test
+ public void testCompareEqualRows()
+ {
+ ResultComparator rc = new ResultComparator();
+
+ ResultHandler.ComparableResultSet res = createResultSet(10, 10, false);
+ ResultHandler.ComparableResultSet res2 = createResultSet(10, 10, false);
+ List<ResultHandler.ComparableResultSet> toCompare = Lists.newArrayList(res, res2);
+ List<Iterator<ResultHandler.ComparableRow>> iters = toCompare.stream().map(Iterable::iterator).collect(Collectors.toList());
+
+ while (true)
+ {
+ List<ResultHandler.ComparableRow> rows = ResultHandler.rows(iters);
+ assertTrue(rc.compareRows(Lists.newArrayList("eq1", "eq2"), null, rows));
+ if (rows.stream().allMatch(Objects::isNull))
+ break;
+ }
+ }
+
+ @Test
+ public void testCompareRowsDifferentCount()
+ {
+ ResultComparator rc = new ResultComparator();
+ ResultHandler.ComparableResultSet res = createResultSet(10, 10, false);
+ ResultHandler.ComparableResultSet res2 = createResultSet(10, 10, false);
+ List<ResultHandler.ComparableResultSet> toCompare = Lists.newArrayList(res, res2, createResultSet(10, 11, false));
+ List<Iterator<ResultHandler.ComparableRow>> iters = toCompare.stream().map(Iterable::iterator).collect(Collectors.toList());
+ boolean foundMismatch = false;
+ while (true)
+ {
+ List<ResultHandler.ComparableRow> rows = ResultHandler.rows(iters);
+ if (rows.stream().allMatch(Objects::isNull))
+ break;
+ if (!rc.compareRows(Lists.newArrayList("eq1", "eq2", "diff"), null, rows))
+ {
+ foundMismatch = true;
+ }
+ }
+ assertTrue(foundMismatch);
+ }
+
+ @Test
+ public void testCompareRowsDifferentContent()
+ {
+ ResultComparator rc = new ResultComparator();
+ ResultHandler.ComparableResultSet res = createResultSet(10, 10, false);
+ ResultHandler.ComparableResultSet res2 = createResultSet(10, 10, false);
+ List<ResultHandler.ComparableResultSet> toCompare = Lists.newArrayList(res, res2, createResultSet(10, 10, true));
+ List<Iterator<ResultHandler.ComparableRow>> iters = toCompare.stream().map(Iterable::iterator).collect(Collectors.toList());
+ while (true)
+ {
+ List<ResultHandler.ComparableRow> rows = ResultHandler.rows(iters);
+ if (rows.stream().allMatch(Objects::isNull))
+ break;
+ assertFalse(rows.toString(), rc.compareRows(Lists.newArrayList("eq1", "eq2", "diff"), null, rows));
+ }
+ }
+
+ @Test
+ public void testCompareRowsDifferentColumnCount()
+ {
+ ResultComparator rc = new ResultComparator();
+ ResultHandler.ComparableResultSet res = createResultSet(10, 10, false);
+ ResultHandler.ComparableResultSet res2 = createResultSet(10, 10, false);
+ List<ResultHandler.ComparableResultSet> toCompare = Lists.newArrayList(res, res2, createResultSet(11, 10, false));
+ List<Iterator<ResultHandler.ComparableRow>> iters = toCompare.stream().map(Iterable::iterator).collect(Collectors.toList());
+ while (true)
+ {
+ List<ResultHandler.ComparableRow> rows = ResultHandler.rows(iters);
+ if (rows.stream().allMatch(Objects::isNull))
+ break;
+ assertFalse(rows.toString(), rc.compareRows(Lists.newArrayList("eq1", "eq2", "diff"), null, rows));
+ }
+ }
+
+ @Test
+ public void testResultHandler() throws IOException
+ {
+ List<String> targetHosts = Lists.newArrayList("hosta", "hostb", "hostc");
+ File tmpDir = Files.createTempDirectory("testresulthandler").toFile();
+ File queryDir = Files.createTempDirectory("queries").toFile();
+ List<File> resultPaths = new ArrayList<>();
+ targetHosts.forEach(host -> { File f = new File(tmpDir, host); f.mkdir(); resultPaths.add(f);});
+
+ ResultHandler.ComparableResultSet res = createResultSet(10, 10, false);
+ ResultHandler.ComparableResultSet res2 = createResultSet(10, 10, false);
+ ResultHandler.ComparableResultSet res3 = createResultSet(10, 10, false);
+ List<ResultHandler.ComparableResultSet> toCompare = Lists.newArrayList(res, res2, res3);
+ FQLQuery query = new FQLQuery.Single("abcabc", QueryOptions.DEFAULT.getProtocolVersion().asInt(), QueryOptions.DEFAULT, 1111, 2222, 3333, "select * from xyz", Collections.emptyList());
+ try (ResultHandler rh = new ResultHandler(targetHosts, resultPaths, queryDir))
+ {
+ rh.handleResults(query, toCompare);
+ }
+ List<Pair<FQLQuery, ResultHandler.ComparableResultSet>> results1 = readResultFile(resultPaths.get(0), queryDir);
+ List<Pair<FQLQuery, ResultHandler.ComparableResultSet>> results2 = readResultFile(resultPaths.get(1), queryDir);
+ List<Pair<FQLQuery, ResultHandler.ComparableResultSet>> results3 = readResultFile(resultPaths.get(2), queryDir);
+ compareResults(results1, results2);
+ compareResults(results1, results3);
+ compareResults(results3, Collections.singletonList(Pair.create(query, res)));
+ }
+
+ @Test
+ public void testResultHandlerWithDifference() throws IOException
+ {
+ List<String> targetHosts = Lists.newArrayList("hosta", "hostb", "hostc");
+ File tmpDir = Files.createTempDirectory("testresulthandler").toFile();
+ File queryDir = Files.createTempDirectory("queries").toFile();
+ List<File> resultPaths = new ArrayList<>();
+ targetHosts.forEach(host -> { File f = new File(tmpDir, host); f.mkdir(); resultPaths.add(f);});
+
+ ResultHandler.ComparableResultSet res = createResultSet(10, 10, false);
+ ResultHandler.ComparableResultSet res2 = createResultSet(10, 5, false);
+ ResultHandler.ComparableResultSet res3 = createResultSet(10, 10, false);
+ List<ResultHandler.ComparableResultSet> toCompare = Lists.newArrayList(res, res2, res3);
+ FQLQuery query = new FQLQuery.Single("aaa", QueryOptions.DEFAULT.getProtocolVersion().asInt(), QueryOptions.DEFAULT, 123123, 11111, 22222, "select * from abcabc", Collections.emptyList());
+ try (ResultHandler rh = new ResultHandler(targetHosts, resultPaths, queryDir))
+ {
+ rh.handleResults(query, toCompare);
+ }
+ List<Pair<FQLQuery, ResultHandler.ComparableResultSet>> results1 = readResultFile(resultPaths.get(0), queryDir);
+ List<Pair<FQLQuery, ResultHandler.ComparableResultSet>> results2 = readResultFile(resultPaths.get(1), queryDir);
+ List<Pair<FQLQuery, ResultHandler.ComparableResultSet>> results3 = readResultFile(resultPaths.get(2), queryDir);
+ compareResults(results1, results3);
+ compareResults(results2, Collections.singletonList(Pair.create(query, res2)));
+ }
+
+ @Test
+ public void testResultHandlerMultipleResultSets() throws IOException
+ {
+ List<String> targetHosts = Lists.newArrayList("hosta", "hostb", "hostc");
+ File tmpDir = Files.createTempDirectory("testresulthandler").toFile();
+ File queryDir = Files.createTempDirectory("queries").toFile();
+ List<File> resultPaths = new ArrayList<>();
+ targetHosts.forEach(host -> { File f = new File(tmpDir, host); f.mkdir(); resultPaths.add(f);});
+ List<Pair<FQLQuery, List<ResultHandler.ComparableResultSet>>> resultSets = new ArrayList<>();
+ Random random = new Random();
+ for (int i = 0; i < 10; i++)
+ {
+ List<ResultHandler.ComparableResultSet> results = new ArrayList<>();
+ List<ByteBuffer> values = Collections.singletonList(ByteBufferUtil.bytes(i * 50));
+ for (int jj = 0; jj < targetHosts.size(); jj++)
+ {
+ results.add(createResultSet(5, 1 + random.nextInt(10), true));
+ }
+ FQLQuery q = i % 2 == 0
+ ? new FQLQuery.Single("abc"+i,
+ 3,
+ QueryOptions.forInternalCalls(values),
+ i * 1000,
+ 12345,
+ 54321,
+ "select * from xyz where id = "+i,
+ values)
+ : new FQLQuery.Batch("abc"+i,
+ 3,
+ QueryOptions.forInternalCalls(values),
+ i * 1000,
+ i * 54321,
+ i * 12345,
+ com.datastax.driver.core.BatchStatement.Type.UNLOGGED,
+ Lists.newArrayList("select * from aaaa"),
+ Collections.singletonList(values));
+
+ resultSets.add(Pair.create(q, results));
+ }
+ try (ResultHandler rh = new ResultHandler(targetHosts, resultPaths, queryDir))
+ {
+ for (int i = 0; i < resultSets.size(); i++)
+ rh.handleResults(resultSets.get(i).left, resultSets.get(i).right);
+ }
+
+ for (int i = 0; i < targetHosts.size(); i++)
+ compareWithFile(resultPaths, queryDir, resultSets, i);
+ }
+
+ @Test
+ public void testResultHandlerFailedQuery() throws IOException
+ {
+ List<String> targetHosts = Lists.newArrayList("hosta", "hostb", "hostc", "hostd");
+ File tmpDir = Files.createTempDirectory("testresulthandler").toFile();
+ File queryDir = Files.createTempDirectory("queries").toFile();
+ List<File> resultPaths = new ArrayList<>();
+ targetHosts.forEach(host -> { File f = new File(tmpDir, host); f.mkdir(); resultPaths.add(f);});
+
+ List<Pair<FQLQuery, List<ResultHandler.ComparableResultSet>>> resultSets = new ArrayList<>();
+ Random random = new Random();
+ for (int i = 0; i < 10; i++)
+ {
+ List<ResultHandler.ComparableResultSet> results = new ArrayList<>();
+ List<ByteBuffer> values = Collections.singletonList(ByteBufferUtil.bytes(i * 50));
+ for (int jj = 0; jj < targetHosts.size(); jj++)
+ {
+ results.add(createResultSet(5, 1 + random.nextInt(10), true));
+ }
+ results.set(0, StoredResultSet.failed("testing abc"));
+ results.set(3, StoredResultSet.failed("testing abc"));
+ FQLQuery q = new FQLQuery.Single("abc"+i,
+ 3,
+ QueryOptions.forInternalCalls(values),
+ i * 1000,
+ i * 12345,
+ i * 54321,
+ "select * from xyz where id = "+i,
+ values);
+ resultSets.add(Pair.create(q, results));
+ }
+ try (ResultHandler rh = new ResultHandler(targetHosts, resultPaths, queryDir))
+ {
+ for (int i = 0; i < resultSets.size(); i++)
+ rh.handleResults(resultSets.get(i).left, resultSets.get(i).right);
+ }
+ for (int i = 0; i < targetHosts.size(); i++)
+ compareWithFile(resultPaths, queryDir, resultSets, i);
+ }
+
+ @Test
+ public void testCompare()
+ {
+ FQLQuery q1 = new FQLQuery.Single("abc", 0, QueryOptions.DEFAULT, 123, 111, 222, "aaaa", Collections.emptyList());
+ FQLQuery q2 = new FQLQuery.Single("abc", 0, QueryOptions.DEFAULT, 123, 111, 222,"aaaa", Collections.emptyList());
+
+ assertEquals(0, q1.compareTo(q2));
+ assertEquals(0, q2.compareTo(q1));
+
+ FQLQuery q3 = new FQLQuery.Batch("abc", 0, QueryOptions.DEFAULT, 123, 111, 222, com.datastax.driver.core.BatchStatement.Type.UNLOGGED, Collections.emptyList(), Collections.emptyList());
+ // single queries before batch queries
+ assertTrue(q1.compareTo(q3) < 0);
+ assertTrue(q3.compareTo(q1) > 0);
+
+ // check that smaller query time
+ FQLQuery q4 = new FQLQuery.Single("abc", 0, QueryOptions.DEFAULT, 124, 111, 222, "aaaa", Collections.emptyList());
+ assertTrue(q1.compareTo(q4) < 0);
+ assertTrue(q4.compareTo(q1) > 0);
+
+ FQLQuery q5 = new FQLQuery.Batch("abc", 0, QueryOptions.DEFAULT, 124, 111, 222, com.datastax.driver.core.BatchStatement.Type.UNLOGGED, Collections.emptyList(), Collections.emptyList());
+ assertTrue(q1.compareTo(q5) < 0);
+ assertTrue(q5.compareTo(q1) > 0);
+
+ FQLQuery q6 = new FQLQuery.Single("abc", 0, QueryOptions.DEFAULT, 123, 111, 222, "aaaa", Collections.singletonList(ByteBufferUtil.bytes(10)));
+ FQLQuery q7 = new FQLQuery.Single("abc", 0, QueryOptions.DEFAULT, 123, 111, 222, "aaaa", Collections.emptyList());
+ assertTrue(q6.compareTo(q7) > 0);
+ assertTrue(q7.compareTo(q6) < 0);
+
+ FQLQuery q8 = new FQLQuery.Single("abc", 0, QueryOptions.DEFAULT, 123, 111, 222, "aaaa", Collections.singletonList(ByteBufferUtil.bytes("a")));
+ FQLQuery q9 = new FQLQuery.Single("abc", 0, QueryOptions.DEFAULT, 123, 111, 222, "aaaa", Collections.singletonList(ByteBufferUtil.bytes("b")));
+ assertTrue(q8.compareTo(q9) < 0);
+ assertTrue(q9.compareTo(q8) > 0);
+ }
+
+ @Test
+ public void testFQLQuerySingleToStatement()
+ {
+ List<ByteBuffer> values = new ArrayList<>();
+ for (int i = 0; i < 10; i++)
+ values.add(ByteBufferUtil.bytes(i));
+ FQLQuery.Single single = new FQLQuery.Single("xyz",
+ QueryOptions.DEFAULT.getProtocolVersion().asInt(),
+ QueryOptions.forInternalCalls(values),
+ 1234,
+ 12345,
+ 54321,
+ "select * from aaa",
+ values);
+ Statement stmt = single.toStatement();
+ assertEquals(stmt.getDefaultTimestamp(), 12345);
+ assertTrue(stmt instanceof SimpleStatement);
+ SimpleStatement simpleStmt = (SimpleStatement)stmt;
+ assertEquals("select * from aaa",simpleStmt.getQueryString(CodecRegistry.DEFAULT_INSTANCE));
+ assertArrayEquals(values.toArray(), simpleStmt.getValues(com.datastax.driver.core.ProtocolVersion.fromInt(QueryOptions.DEFAULT.getProtocolVersion().asInt()), CodecRegistry.DEFAULT_INSTANCE));
+ }
+
+
+ @Test
+ public void testFQLQueryBatchToStatement()
+ {
+ List<List<ByteBuffer>> values = new ArrayList<>();
+ List<String> queries = new ArrayList<>();
+ for (int bqCount = 0; bqCount < 10; bqCount++)
+ {
+ queries.add("select * from asdf where x = ? and y = " + bqCount);
+ List<ByteBuffer> queryValues = new ArrayList<>();
+ for (int i = 0; i < 10; i++)
+ queryValues.add(ByteBufferUtil.bytes(i + ":" + bqCount));
+ values.add(queryValues);
+ }
+
+ FQLQuery.Batch batch = new FQLQuery.Batch("xyz",
+ QueryOptions.DEFAULT.getProtocolVersion().asInt(),
+ QueryOptions.DEFAULT,
+ 1234,
+ 12345,
+ 54321,
+ com.datastax.driver.core.BatchStatement.Type.UNLOGGED,
+ queries,
+ values);
+ Statement stmt = batch.toStatement();
+ assertEquals(stmt.getDefaultTimestamp(), 12345);
+ assertTrue(stmt instanceof com.datastax.driver.core.BatchStatement);
+ com.datastax.driver.core.BatchStatement batchStmt = (com.datastax.driver.core.BatchStatement)stmt;
+ List<Statement> statements = Lists.newArrayList(batchStmt.getStatements());
+ List<Statement> fromFQLQueries = batch.queries.stream().map(FQLQuery.Single::toStatement).collect(Collectors.toList());
+ assertEquals(statements.size(), fromFQLQueries.size());
+ assertEquals(12345, batchStmt.getDefaultTimestamp());
+ for (int i = 0; i < statements.size(); i++)
+ compareStatements(statements.get(i), fromFQLQueries.get(i));
+ }
+
+ @Test
+ public void testParser() {
+ QueryReplayer.ParsedTargetHost pth;
+ pth = fromString("127.0.0.1");
+ assertEquals("127.0.0.1", pth.host);
+ assertEquals(9042, pth.port );
+ assertNull(pth.user);
+ assertNull(pth.password);
+
+ pth = fromString("127.0.0.1:3333");
+ assertEquals("127.0.0.1", pth.host);
+ assertEquals(3333, pth.port );
+ assertNull(pth.user);
+ assertNull(pth.password);
+
+ pth = fromString("aaa:bbb@127.0.0.1:3333");
+ assertEquals("127.0.0.1", pth.host);
+ assertEquals(3333, pth.port );
+ assertEquals("aaa", pth.user);
+ assertEquals("bbb", pth.password);
+
+ pth = fromString("aaa:bbb@127.0.0.1");
+ assertEquals("127.0.0.1", pth.host);
+ assertEquals(9042, pth.port );
+ assertEquals("aaa", pth.user);
+ assertEquals("bbb", pth.password);
+ }
+
+ @Test(expected = RuntimeException.class)
+ public void testNoPass()
+ {
+ fromString("blabla@abc.com:1234");
+ }
+
+ @Test(expected = RuntimeException.class)
+ public void testBadPort()
+ {
+ fromString("aaa:bbb@abc.com:xyz");
+ }
+
+ @Test (expected = IORuntimeException.class)
+ public void testFutureVersion() throws Exception
+ {
+ FQLQueryReader reader = new FQLQueryReader();
+ File dir = Files.createTempDirectory("chronicle").toFile();
+ try (ChronicleQueue queue = ChronicleQueueBuilder.single(dir).build())
+ {
+ ExcerptAppender appender = queue.acquireAppender();
+ appender.writeDocument(new BinLog.ReleaseableWriteMarshallable() {
+ protected long version()
+ {
+ return 999;
+ }
+
+ protected String type()
+ {
+ return FullQueryLogger.SINGLE_QUERY;
+ }
+
+ public void writeMarshallablePayload(WireOut wire)
+ {
+ wire.write("future-field").text("future_value");
+ }
+
+ public void release()
+ {
+
+ }
+ });
+
+ ExcerptTailer tailer = queue.createTailer();
+ tailer.readDocument(reader);
+ }
+ catch (Exception e)
+ {
+ assertTrue(e.getMessage().contains("Unsupported record version"));
+ throw e;
+ }
+
+ }
+
+ @Test (expected = IORuntimeException.class)
+ public void testUnknownRecord() throws Exception
+ {
+ FQLQueryReader reader = new FQLQueryReader();
+ File dir = Files.createTempDirectory("chronicle").toFile();
+ try (ChronicleQueue queue = ChronicleQueueBuilder.single(dir).build())
+ {
+ ExcerptAppender appender = queue.acquireAppender();
+ appender.writeDocument(new BinLog.ReleaseableWriteMarshallable() {
+ protected long version()
+ {
+ return FullQueryLogger.CURRENT_VERSION;
+ }
+
+ protected String type()
+ {
+ return "unknown-type";
+ }
+
+ public void writeMarshallablePayload(WireOut wire)
+ {
+ wire.write("unknown-field").text("unknown_value");
+ }
+
+ public void release()
+ {
+
+ }
+ });
+
+ ExcerptTailer tailer = queue.createTailer();
+ tailer.readDocument(reader);
+ }
+ catch (Exception e)
+ {
+ assertTrue(e.getMessage().contains("Unsupported record type field"));
+ throw e;
+ }
+
+ }
+
+ private void compareStatements(Statement statement1, Statement statement2)
+ {
+ assertTrue(statement1 instanceof SimpleStatement && statement2 instanceof SimpleStatement);
+ SimpleStatement simpleStmt1 = (SimpleStatement)statement1;
+ SimpleStatement simpleStmt2 = (SimpleStatement)statement2;
+ assertEquals(simpleStmt1.getQueryString(CodecRegistry.DEFAULT_INSTANCE), simpleStmt2.getQueryString(CodecRegistry.DEFAULT_INSTANCE));
+ assertArrayEquals(simpleStmt1.getValues(com.datastax.driver.core.ProtocolVersion.fromInt(QueryOptions.DEFAULT.getProtocolVersion().asInt()), CodecRegistry.DEFAULT_INSTANCE),
+ simpleStmt2.getValues(com.datastax.driver.core.ProtocolVersion.fromInt(QueryOptions.DEFAULT.getProtocolVersion().asInt()), CodecRegistry.DEFAULT_INSTANCE));
+
+ }
+
+ private File generateQueries(int count, boolean random) throws IOException
+ {
+ Random r = new Random();
+ File dir = Files.createTempDirectory("chronicle").toFile();
+ try (ChronicleQueue readQueue = ChronicleQueueBuilder.single(dir).build())
+ {
+ ExcerptAppender appender = readQueue.acquireAppender();
+
+ for (int i = 0; i < count; i++)
+ {
+ long timestamp = random ? Math.abs(r.nextLong() % 10000) : i;
+ if (random ? r.nextBoolean() : i % 2 == 0)
+ {
+ String query = "abcdefghijklm " + i;
+ QueryState qs = r.nextBoolean() ? queryState() : queryState("querykeyspace");
+ FullQueryLogger.Query q = new FullQueryLogger.Query(query, QueryOptions.DEFAULT, qs, timestamp);
+ appender.writeDocument(q);
+ q.release();
+ }
+ else
+ {
+ int batchSize = random ? r.nextInt(99) + 1 : i + 1;
+ List<String> queries = new ArrayList<>(batchSize);
+ List<List<ByteBuffer>> values = new ArrayList<>(batchSize);
+ for (int jj = 0; jj < (random ? r.nextInt(batchSize) : 10); jj++)
+ {
+ queries.add("aaaaaa batch "+i+":"+jj);
+ values.add(Collections.emptyList());
+ }
+ FullQueryLogger.Batch batch = new FullQueryLogger.Batch(BatchStatement.Type.UNLOGGED,
+ queries,
+ values,
+ QueryOptions.DEFAULT,
+ queryState("someks"),
+ timestamp);
+ appender.writeDocument(batch);
+ batch.release();
+ }
+ }
+ }
+ return dir;
+ }
+
+ private QueryState queryState()
+ {
+ return QueryState.forInternalCalls();
+ }
+
+ private QueryState queryState(String keyspace)
+ {
+ ClientState clientState = ClientState.forInternalCalls(keyspace);
+ return new QueryState(clientState);
+ }
+
+ static ResultHandler.ComparableResultSet createResultSet(int columnCount, int rowCount, boolean random)
+ {
+ List<Pair<String, String>> columnDefs = new ArrayList<>(columnCount);
+ Random r = new Random();
+ for (int i = 0; i < columnCount; i++)
+ {
+ columnDefs.add(Pair.create("a" + i, "int"));
+ }
+ ResultHandler.ComparableColumnDefinitions colDefs = new StoredResultSet.StoredComparableColumnDefinitions(columnDefs, false, null);
+ List<ResultHandler.ComparableRow> rows = new ArrayList<>();
+ for (int i = 0; i < rowCount; i++)
+ {
+ List<ByteBuffer> row = new ArrayList<>(columnCount);
+ for (int jj = 0; jj < columnCount; jj++)
+ row.add(ByteBufferUtil.bytes(i + " col " + jj + (random ? r.nextInt() : "")));
+
+ rows.add(new StoredResultSet.StoredComparableRow(row, colDefs));
+ }
+ return new StoredResultSet(colDefs, true, false, null, rows::iterator);
+ }
+
+ private static void compareWithFile(List<File> dirs, File resultDir, List<Pair<FQLQuery, List<ResultHandler.ComparableResultSet>>> resultSets, int idx)
+ {
+ List<Pair<FQLQuery, ResultHandler.ComparableResultSet>> results1 = readResultFile(dirs.get(idx), resultDir);
+ for (int i = 0; i < resultSets.size(); i++)
+ {
+ FQLQuery query = resultSets.get(i).left;
+ List<Pair<FQLQuery, ResultHandler.ComparableResultSet>> toCompare = Collections.singletonList(Pair.create(query, resultSets.get(i).right.get(idx)));
+ compareResults(Collections.singletonList(results1.get(i)), toCompare);
+ }
+ }
+
+ private static List<Pair<FQLQuery, ResultHandler.ComparableResultSet>> readResultFile(File dir, File queryDir)
+ {
+ List<Pair<FQLQuery, ResultHandler.ComparableResultSet>> resultSets = new ArrayList<>();
+ try (ChronicleQueue q = ChronicleQueueBuilder.single(dir).build();
+ ChronicleQueue queryQ = ChronicleQueueBuilder.single(queryDir).build())
+ {
+ ExcerptTailer queryTailer = queryQ.createTailer();
+ FQLQueryReader queryReader = new FQLQueryReader();
+ Compare.StoredResultSetIterator resultSetIterator = new Compare.StoredResultSetIterator(q.createTailer());
+ // we need to materialize the rows in-memory to compare them easier in these tests
+ while (resultSetIterator.hasNext())
+ {
+ ResultHandler.ComparableResultSet resultSetFromDisk = resultSetIterator.next();
+ Iterator<ResultHandler.ComparableRow> rowIterFromDisk = resultSetFromDisk.iterator();
+ queryTailer.readDocument(queryReader);
+
+ FQLQuery query = queryReader.getQuery();
+ List<ResultHandler.ComparableRow> rows = new ArrayList<>();
+ while (rowIterFromDisk.hasNext())
+ {
+ rows.add(rowIterFromDisk.next());
+ }
+ resultSets.add(Pair.create(query, new StoredResultSet(resultSetFromDisk.getColumnDefinitions(),
+ resultSetIterator.hasNext(),
+ resultSetFromDisk.wasFailed(),
+ resultSetFromDisk.getFailureException(),
+ rows::iterator)));
+ }
+ }
+ return resultSets;
+ }
+}
diff --git a/tools/stress/README.txt b/tools/stress/README.txt
index aa89dab..355415b 100644
--- a/tools/stress/README.txt
+++ b/tools/stress/README.txt
@@ -1,92 +1,13 @@
cassandra-stress
======
-Description
------------
-cassandra-stress is a tool for benchmarking and load testing a Cassandra
-cluster. cassandra-stress supports testing arbitrary CQL tables and queries
-to allow users to benchmark their data model.
-
Setup
-----
Run `ant` from the Cassandra source directory, then cassandra-stress can be invoked from tools/bin/cassandra-stress.
cassandra-stress supports benchmarking any Cassandra cluster of version 2.0+.
-Usage
------
-There are several operation types:
+Usage & Examples
+----------------
- * write-only, read-only, and mixed workloads of standard data
- * write-only and read-only workloads for counter columns
- * user configured workloads, running custom queries on custom schemas
- * support for legacy cassandra-stress operations
+See: https://cassandra.apache.org/doc/latest/tools/cassandra_stress.html
-The syntax is `cassandra-stress <command> [options]`. If you want more information on a given command
-or options, just run `cassandra-stress help <command|option>`.
-
-Commands:
- read:
- Multiple concurrent reads - the cluster must first be populated by a write test
- write:
- Multiple concurrent writes against the cluster
- mixed:
- Interleaving of any basic commands, with configurable ratio and distribution - the cluster must first be populated by a write test
- counter_write:
- Multiple concurrent updates of counters.
- counter_read:
- Multiple concurrent reads of counters. The cluster must first be populated by a counterwrite test.
- user:
- Interleaving of user provided queries, with configurable ratio and distribution.
- See http://www.datastax.com/dev/blog/improved-cassandra-2-1-stress-tool-benchmark-any-schema
- help:
- Print help for a command or option
- print:
- Inspect the output of a distribution definition
- legacy:
- Legacy support mode
-
-Primary Options:
- -pop:
- Population distribution and intra-partition visit order
- -insert:
- Insert specific options relating to various methods for batching and splitting partition updates
- -col:
- Column details such as size and count distribution, data generator, names, comparator and if super columns should be used
- -rate:
- Thread count, rate limit or automatic mode (default is auto)
- -mode:
- Thrift or CQL with options
- -errors:
- How to handle errors when encountered during stress
- -sample:
- Specify the number of samples to collect for measuring latency
- -schema:
- Replication settings, compression, compaction, etc.
- -node:
- Nodes to connect to
- -log:
- Where to log progress to, and the interval at which to do it
- -transport:
- Custom transport factories
- -port:
- The port to connect to cassandra nodes on
- -sendto:
- Specify a stress server to send this command to
- -graph:
- Graph recorded metrics
- -tokenrange:
- Token range settings
-
-
-Suboptions:
- Every command and primary option has its own collection of suboptions. These are too numerous to list here.
- For information on the suboptions for each command or option, please use the help command,
- `cassandra-stress help <command|option>`.
-
-Examples
---------
-
- * tools/bin/cassandra-stress write n=1000000 -node 192.168.1.101 # 1M inserts to given host
- * tools/bin/cassandra-stress read n=10000000 -node 192.168.1.101 -o read # 1M reads
- * tools/bin/cassandra-stress write -node 192.168.1.101,192.168.1.102 n=10000000 # 10M inserts spread across two nodes
- * tools/bin/cassandra-stress help -pop # Print help for population distribution option
diff --git a/tools/stress/src/org/apache/cassandra/io/sstable/StressCQLSSTableWriter.java b/tools/stress/src/org/apache/cassandra/io/sstable/StressCQLSSTableWriter.java
index 5a285e1..7c012e4 100644
--- a/tools/stress/src/org/apache/cassandra/io/sstable/StressCQLSSTableWriter.java
+++ b/tools/stress/src/org/apache/cassandra/io/sstable/StressCQLSSTableWriter.java
@@ -22,27 +22,28 @@
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.*;
+import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
-import com.datastax.driver.core.ProtocolVersion;
-import com.datastax.driver.core.TypeCodec;
+import org.apache.commons.lang3.ArrayUtils;
+
import org.antlr.runtime.RecognitionException;
-import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.cql3.CQLStatement;
+import org.apache.cassandra.cql3.statements.schema.CreateTableStatement;
+import org.apache.cassandra.cql3.statements.schema.CreateTypeStatement;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.config.Schema;
+import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.cql3.CQLFragmentParser;
import org.apache.cassandra.cql3.ColumnSpecification;
import org.apache.cassandra.cql3.CqlParser;
import org.apache.cassandra.cql3.QueryOptions;
import org.apache.cassandra.cql3.UpdateParameters;
import org.apache.cassandra.cql3.functions.UDHelper;
-import org.apache.cassandra.cql3.statements.CreateTableStatement;
-import org.apache.cassandra.cql3.statements.CreateTypeStatement;
-import org.apache.cassandra.cql3.statements.ParsedStatement;
+import org.apache.cassandra.cql3.functions.types.TypeCodec;
import org.apache.cassandra.cql3.statements.UpdateStatement;
import org.apache.cassandra.db.*;
-import org.apache.cassandra.db.marshal.UserType;
-import org.apache.cassandra.db.partitions.Partition;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.Murmur3Partitioner;
import org.apache.cassandra.exceptions.InvalidRequestException;
@@ -51,10 +52,11 @@
import org.apache.cassandra.io.sstable.format.SSTableFormat;
import org.apache.cassandra.schema.KeyspaceMetadata;
import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.TableMetadataRef;
import org.apache.cassandra.schema.Types;
import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.transport.ProtocolVersion;
import org.apache.cassandra.utils.ByteBufferUtil;
-import org.apache.cassandra.utils.Pair;
/**
* Utility to write SSTables.
@@ -243,15 +245,16 @@
List<ByteBuffer> keys = insert.buildPartitionKeyNames(options);
SortedSet<Clustering> clusterings = insert.createClustering(options);
- long now = System.currentTimeMillis() * 1000;
+ long now = System.currentTimeMillis();
// Note that we asks indexes to not validate values (the last 'false' arg below) because that triggers a 'Keyspace.open'
// and that forces a lot of initialization that we don't want.
- UpdateParameters params = new UpdateParameters(insert.cfm,
+ UpdateParameters params = new UpdateParameters(insert.metadata(),
insert.updatedColumns(),
options,
- insert.getTimestamp(now, options),
+ insert.getTimestamp(TimeUnit.MILLISECONDS.toMicros(now), options),
+ (int) TimeUnit.MILLISECONDS.toSeconds(now),
insert.getTimeToLive(options),
- Collections.<DecoratedKey, Partition>emptyMap());
+ Collections.emptyMap());
try
{
@@ -299,20 +302,6 @@
}
/**
- * Returns the User Defined type, used in this SSTable Writer, that can
- * be used to create UDTValue instances.
- *
- * @param dataType name of the User Defined type
- * @return user defined type
- */
- public com.datastax.driver.core.UserType getUDType(String dataType)
- {
- KeyspaceMetadata ksm = Schema.instance.getKSMetaData(insert.keyspace());
- UserType userType = ksm.types.getNullable(ByteBufferUtil.bytes(dataType));
- return (com.datastax.driver.core.UserType) UDHelper.driverType(userType);
- }
-
- /**
* Close this writer.
* <p>
* This method should be called, otherwise the produced sstables are not
@@ -328,7 +317,7 @@
if (value == null || value == UNSET_VALUE)
return (ByteBuffer) value;
- return codec.serialize(value, ProtocolVersion.NEWEST_SUPPORTED);
+ return codec.serialize(value, ProtocolVersion.CURRENT);
}
/**
* The writer loads data in directories corresponding to how they laid out on the server.
@@ -355,8 +344,8 @@
private Boolean makeRangeAware = false;
- private CreateTableStatement.RawStatement schemaStatement;
- private final List<CreateTypeStatement> typeStatements;
+ private CreateTableStatement.Raw schemaStatement;
+ private final List<CreateTypeStatement.Raw> typeStatements;
private UpdateStatement.ParsedInsert insertStatement;
private IPartitioner partitioner;
@@ -426,7 +415,7 @@
public Builder withType(String typeDefinition) throws SyntaxException
{
- typeStatements.add(parseStatement(typeDefinition, CreateTypeStatement.class, "CREATE TYPE"));
+ typeStatements.add(parseStatement(typeDefinition, CreateTypeStatement.Raw.class, "CREATE TYPE"));
return this;
}
@@ -446,7 +435,7 @@
*/
public Builder forTable(String schema)
{
- this.schemaStatement = parseStatement(schema, CreateTableStatement.RawStatement.class, "CREATE TABLE");
+ this.schemaStatement = parseStatement(schema, CreateTableStatement.Raw.class, "CREATE TABLE");
return this;
}
@@ -563,105 +552,109 @@
if (partitioner == null)
partitioner = cfs.getPartitioner();
- Pair<UpdateStatement, List<ColumnSpecification>> preparedInsert = prepareInsert();
+ UpdateStatement preparedInsert = prepareInsert();
AbstractSSTableSimpleWriter writer = sorted
- ? new SSTableSimpleWriter(cfs.getDirectories().getDirectoryForNewSSTables(), cfs.metadata, preparedInsert.left.updatedColumns())
- : new SSTableSimpleUnsortedWriter(cfs.getDirectories().getDirectoryForNewSSTables(), cfs.metadata, preparedInsert.left.updatedColumns(), bufferSizeInMB);
+ ? new SSTableSimpleWriter(cfs.getDirectories().getDirectoryForNewSSTables(), cfs.metadata, preparedInsert.updatedColumns())
+ : new SSTableSimpleUnsortedWriter(cfs.getDirectories().getDirectoryForNewSSTables(), cfs.metadata, preparedInsert.updatedColumns(), bufferSizeInMB);
if (formatType != null)
writer.setSSTableFormatType(formatType);
writer.setRangeAwareWriting(makeRangeAware);
- return new StressCQLSSTableWriter(cfs, writer, preparedInsert.left, preparedInsert.right);
+ return new StressCQLSSTableWriter(cfs, writer, preparedInsert, preparedInsert.getBindVariables());
}
}
- private static void createTypes(String keyspace, List<CreateTypeStatement> typeStatements)
+ private static void createTypes(String keyspace, List<CreateTypeStatement.Raw> typeStatements)
{
- KeyspaceMetadata ksm = Schema.instance.getKSMetaData(keyspace);
+ KeyspaceMetadata ksm = Schema.instance.getKeyspaceMetadata(keyspace);
Types.RawBuilder builder = Types.rawBuilder(keyspace);
- for (CreateTypeStatement st : typeStatements)
+ for (CreateTypeStatement.Raw st : typeStatements)
st.addToRawBuilder(builder);
ksm = ksm.withSwapped(builder.build());
- Schema.instance.setKeyspaceMetadata(ksm);
+ Schema.instance.load(ksm);
}
public static ColumnFamilyStore createOfflineTable(String schema, List<File> directoryList)
{
- return createOfflineTable(parseStatement(schema, CreateTableStatement.RawStatement.class, "CREATE TABLE"), Collections.EMPTY_LIST, directoryList);
+ return createOfflineTable(parseStatement(schema, CreateTableStatement.Raw.class, "CREATE TABLE"), Collections.EMPTY_LIST, directoryList);
}
/**
* Creates the table according to schema statement
* with specified data directories
*/
- public static ColumnFamilyStore createOfflineTable(CreateTableStatement.RawStatement schemaStatement, List<CreateTypeStatement> typeStatements, List<File> directoryList)
+ public static ColumnFamilyStore createOfflineTable(CreateTableStatement.Raw schemaStatement, List<CreateTypeStatement.Raw> typeStatements, List<File> directoryList)
{
String keyspace = schemaStatement.keyspace();
- if (Schema.instance.getKSMetaData(keyspace) == null)
+ if (Schema.instance.getKeyspaceMetadata(keyspace) == null)
Schema.instance.load(KeyspaceMetadata.create(keyspace, KeyspaceParams.simple(1)));
createTypes(keyspace, typeStatements);
- KeyspaceMetadata ksm = Schema.instance.getKSMetaData(keyspace);
+ KeyspaceMetadata ksm = Schema.instance.getKeyspaceMetadata(keyspace);
- CFMetaData cfMetaData = ksm.tables.getNullable(schemaStatement.columnFamily());
+ TableMetadata tableMetadata = ksm.tables.getNullable(schemaStatement.table());
+ if (tableMetadata != null)
+ return Schema.instance.getColumnFamilyStoreInstance(tableMetadata.id);
- if (cfMetaData != null)
- return Schema.instance.getColumnFamilyStoreInstance(cfMetaData.cfId);
+ ClientState state = ClientState.forInternalCalls();
+ CreateTableStatement statement = schemaStatement.prepare(state);
+ statement.validate(state);
- CreateTableStatement statement = (CreateTableStatement) schemaStatement.prepare(ksm.types).statement;
- statement.validate(ClientState.forInternalCalls());
-
- //Build metatdata with a portable cfId
- cfMetaData = statement.metadataBuilder()
- .withId(CFMetaData.generateLegacyCfId(keyspace, statement.columnFamily()))
- .build()
- .params(statement.params());
+ //Build metadata with a portable tableId
+ tableMetadata = statement.builder(ksm.types)
+ .id(deterministicId(schemaStatement.keyspace(), schemaStatement.table()))
+ .build();
Keyspace.setInitialized();
- Directories directories = new Directories(cfMetaData, directoryList.stream().map(Directories.DataDirectory::new).collect(Collectors.toList()));
+ Directories directories = new Directories(tableMetadata, directoryList.stream().map(Directories.DataDirectory::new).collect(Collectors.toList()));
Keyspace ks = Keyspace.openWithoutSSTables(keyspace);
- ColumnFamilyStore cfs = ColumnFamilyStore.createColumnFamilyStore(ks, cfMetaData.cfName, cfMetaData, directories, false, false, true);
+ ColumnFamilyStore cfs = ColumnFamilyStore.createColumnFamilyStore(ks, tableMetadata.name, TableMetadataRef.forOfflineTools(tableMetadata), directories, false, false, true);
ks.initCfCustom(cfs);
- Schema.instance.load(cfs.metadata);
- Schema.instance.setKeyspaceMetadata(ksm.withSwapped(ksm.tables.with(cfs.metadata)));
+ Schema.instance.load(ksm.withSwapped(ksm.tables.with(cfs.metadata())));
return cfs;
}
+ private static TableId deterministicId(String keyspace, String table)
+ {
+ return TableId.fromUUID(UUID.nameUUIDFromBytes(ArrayUtils.addAll(keyspace.getBytes(), table.getBytes())));
+ }
+
/**
* Prepares insert statement for writing data to SSTable
*
* @return prepared Insert statement and it's bound names
*/
- private Pair<UpdateStatement, List<ColumnSpecification>> prepareInsert()
+ private UpdateStatement prepareInsert()
{
- ParsedStatement.Prepared cqlStatement = insertStatement.prepare(ClientState.forInternalCalls());
- UpdateStatement insert = (UpdateStatement) cqlStatement.statement;
- insert.validate(ClientState.forInternalCalls());
+ ClientState state = ClientState.forInternalCalls();
+ CQLStatement cqlStatement = insertStatement.prepare(state);
+ UpdateStatement insert = (UpdateStatement) cqlStatement;
+ insert.validate(state);
if (insert.hasConditions())
throw new IllegalArgumentException("Conditional statements are not supported");
if (insert.isCounter())
throw new IllegalArgumentException("Counter update statements are not supported");
- if (cqlStatement.boundNames.isEmpty())
+ if (insert.getBindVariables().isEmpty())
throw new IllegalArgumentException("Provided insert statement has no bind variables");
- return Pair.create(insert, cqlStatement.boundNames);
+ return insert;
}
}
- public static <T extends ParsedStatement> T parseStatement(String query, Class<T> klass, String type)
+ public static <T extends CQLStatement.Raw> T parseStatement(String query, Class<T> klass, String type)
{
try
{
- ParsedStatement stmt = CQLFragmentParser.parseAnyUnhandled(CqlParser::query, query);
+ CQLStatement.Raw stmt = CQLFragmentParser.parseAnyUnhandled(CqlParser::query, query);
if (!stmt.getClass().equals(klass))
throw new IllegalArgumentException("Invalid query, must be a " + type + " statement but was: " + stmt.getClass());
diff --git a/tools/stress/src/org/apache/cassandra/stress/CompactionStress.java b/tools/stress/src/org/apache/cassandra/stress/CompactionStress.java
index 1860fef..5daf654 100644
--- a/tools/stress/src/org/apache/cassandra/stress/CompactionStress.java
+++ b/tools/stress/src/org/apache/cassandra/stress/CompactionStress.java
@@ -20,7 +20,6 @@
import java.io.File;
import java.io.IOError;
-import java.net.InetAddress;
import java.net.URI;
import java.util.*;
import java.util.concurrent.*;
@@ -29,12 +28,13 @@
import com.google.common.collect.Lists;
import com.google.common.util.concurrent.Uninterruptibles;
-import io.airlift.command.*;
+import io.airlift.airline.*;
import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.cql3.statements.CreateTableStatement;
+import org.apache.cassandra.cql3.statements.schema.CreateTableStatement;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Directories;
import org.apache.cassandra.db.SystemKeyspace;
+import org.apache.cassandra.db.commitlog.CommitLog;
import org.apache.cassandra.db.compaction.CompactionManager;
import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
import org.apache.cassandra.dht.IPartitioner;
@@ -44,6 +44,7 @@
import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.locator.TokenMetadata;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.stress.generate.PartitionGenerator;
@@ -75,6 +76,7 @@
static
{
DatabaseDescriptor.daemonInitialization();
+ CommitLog.instance.start();
}
List<File> getDataDirectories()
@@ -112,7 +114,7 @@
{
generateTokens(stressProfile.seedStr, StorageService.instance.getTokenMetadata(), numTokens);
- CreateTableStatement.RawStatement createStatement = stressProfile.getCreateStatement();
+ CreateTableStatement.Raw createStatement = stressProfile.getCreateStatement();
List<File> dataDirectories = getDataDirectories();
ColumnFamilyStore cfs = StressCQLSSTableWriter.Builder.createOfflineTable(createStatement, Collections.EMPTY_LIST, dataDirectories);
@@ -143,6 +145,10 @@
cfs.disableAutoCompaction();
+ // We want to add the SSTables without firing their indexing by any eventual unsupported 2i
+ if (cfs.indexManager.hasIndexes())
+ throw new IllegalStateException("CompactionStress does not support secondary indexes");
+
//Register with cfs
cfs.addSSTables(sstables);
}
@@ -181,7 +187,7 @@
tokenMetadata.clearUnsafe();
for (int i = 1; i <= numTokens; i++)
{
- InetAddress addr = FBUtilities.getBroadcastAddress();
+ InetAddressAndPort addr = FBUtilities.getBroadcastAddressAndPort();
List<Token> tokens = Lists.newArrayListWithCapacity(numTokens);
for (int j = 0; j < numTokens; ++j)
tokens.add(p.getRandomToken(random));
diff --git a/tools/stress/src/org/apache/cassandra/stress/Operation.java b/tools/stress/src/org/apache/cassandra/stress/Operation.java
index dc5bd2f..89664f0 100644
--- a/tools/stress/src/org/apache/cassandra/stress/Operation.java
+++ b/tools/stress/src/org/apache/cassandra/stress/Operation.java
@@ -24,8 +24,6 @@
import org.apache.cassandra.stress.settings.SettingsLog;
import org.apache.cassandra.stress.settings.StressSettings;
import org.apache.cassandra.stress.util.JavaDriverClient;
-import org.apache.cassandra.stress.util.ThriftClient;
-import org.apache.cassandra.thrift.InvalidRequestException;
import org.apache.cassandra.transport.SimpleClient;
public abstract class Operation
@@ -53,13 +51,6 @@
return false;
}
- /**
- * Run operation
- * @param client Cassandra Thrift client connection
- * @throws IOException on any I/O error.
- */
- public abstract void run(ThriftClient client) throws IOException;
-
public void run(SimpleClient client) throws IOException
{
throw new UnsupportedOperationException();
@@ -126,7 +117,7 @@
protected String getExceptionMessage(Exception e)
{
String className = e.getClass().getSimpleName();
- String message = (e instanceof InvalidRequestException) ? ((InvalidRequestException) e).getWhy() : e.getMessage();
+ String message = e.getMessage();
return (message == null) ? "(" + className + ")" : String.format("(%s): %s", className, message);
}
diff --git a/tools/stress/src/org/apache/cassandra/stress/StressAction.java b/tools/stress/src/org/apache/cassandra/stress/StressAction.java
index 4e268eb..3268182 100644
--- a/tools/stress/src/org/apache/cassandra/stress/StressAction.java
+++ b/tools/stress/src/org/apache/cassandra/stress/StressAction.java
@@ -33,7 +33,6 @@
import org.apache.cassandra.stress.settings.StressSettings;
import org.apache.cassandra.stress.util.JavaDriverClient;
import org.apache.cassandra.stress.util.ResultLogger;
-import org.apache.cassandra.stress.util.ThriftClient;
import org.apache.cassandra.transport.SimpleClient;
import org.jctools.queues.SpscArrayQueue;
import org.jctools.queues.SpscUnboundedArrayQueue;
@@ -427,7 +426,6 @@
try
{
SimpleClient sclient = null;
- ThriftClient tclient = null;
JavaDriverClient jclient = null;
final ConnectionAPI clientType = settings.mode.api;
@@ -441,10 +439,6 @@
case SIMPLE_NATIVE:
sclient = settings.getSimpleNativeClient();
break;
- case THRIFT:
- case THRIFT_SMART:
- tclient = settings.getThriftClient();
- break;
default:
throw new IllegalStateException();
}
@@ -474,10 +468,8 @@
case SIMPLE_NATIVE:
op.run(sclient);
break;
- case THRIFT:
- case THRIFT_SMART:
default:
- op.run(tclient);
+ throw new IllegalStateException();
}
}
catch (Exception e)
diff --git a/tools/stress/src/org/apache/cassandra/stress/StressGraph.java b/tools/stress/src/org/apache/cassandra/stress/StressGraph.java
index 663bde6..7a865b4 100644
--- a/tools/stress/src/org/apache/cassandra/stress/StressGraph.java
+++ b/tools/stress/src/org/apache/cassandra/stress/StressGraph.java
@@ -110,17 +110,14 @@
private String getGraphHTML()
{
- InputStream graphHTMLRes = StressGraph.class.getClassLoader().getResourceAsStream("org/apache/cassandra/stress/graph/graph.html");
- String graphHTML;
- try
+ try (InputStream graphHTMLRes = StressGraph.class.getClassLoader().getResourceAsStream("org/apache/cassandra/stress/graph/graph.html"))
{
- graphHTML = new String(ByteStreams.toByteArray(graphHTMLRes));
+ return new String(ByteStreams.toByteArray(graphHTMLRes));
}
catch (IOException e)
{
throw new RuntimeException(e);
}
- return graphHTML;
}
/** Parse log and append to stats array */
@@ -151,7 +148,7 @@
currentThreadCount = tc.group(2);
}
}
-
+
// Detect mode changes
if (line.equals(StressMetrics.HEAD))
{
@@ -235,7 +232,7 @@
private JSONObject createJSONStats(JSONObject json)
{
- try (InputStream logStream = new FileInputStream(stressSettings.graph.temporaryLogFile))
+ try (InputStream logStream = Files.newInputStream(stressSettings.graph.temporaryLogFile.toPath()))
{
JSONArray stats;
if (json == null)
diff --git a/tools/stress/src/org/apache/cassandra/stress/StressProfile.java b/tools/stress/src/org/apache/cassandra/stress/StressProfile.java
index ad10499..5eb478c 100644
--- a/tools/stress/src/org/apache/cassandra/stress/StressProfile.java
+++ b/tools/stress/src/org/apache/cassandra/stress/StressProfile.java
@@ -35,30 +35,28 @@
import com.google.common.util.concurrent.Uninterruptibles;
import com.datastax.driver.core.*;
+import com.datastax.driver.core.TableMetadata;
import com.datastax.driver.core.exceptions.AlreadyExistsException;
import org.antlr.runtime.RecognitionException;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
import org.apache.cassandra.cql3.CQLFragmentParser;
import org.apache.cassandra.cql3.CqlParser;
-import org.apache.cassandra.cql3.QueryProcessor;
-import org.apache.cassandra.cql3.statements.CreateTableStatement;
+import org.apache.cassandra.cql3.statements.ModificationStatement;
+import org.apache.cassandra.cql3.statements.schema.CreateTableStatement;
import org.apache.cassandra.exceptions.RequestValidationException;
import org.apache.cassandra.exceptions.SyntaxException;
+import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.stress.generate.*;
import org.apache.cassandra.stress.generate.values.*;
-import org.apache.cassandra.stress.operations.userdefined.TokenRangeQuery;
+import org.apache.cassandra.stress.operations.userdefined.CASQuery;
import org.apache.cassandra.stress.operations.userdefined.SchemaInsert;
import org.apache.cassandra.stress.operations.userdefined.SchemaQuery;
+import org.apache.cassandra.stress.operations.userdefined.SchemaStatement;
+import org.apache.cassandra.stress.operations.userdefined.TokenRangeQuery;
import org.apache.cassandra.stress.operations.userdefined.ValidatingSchemaQuery;
import org.apache.cassandra.stress.report.Timer;
import org.apache.cassandra.stress.settings.*;
import org.apache.cassandra.stress.util.JavaDriverClient;
import org.apache.cassandra.stress.util.ResultLogger;
-import org.apache.cassandra.stress.util.ThriftClient;
-import org.apache.cassandra.thrift.Compression;
-import org.apache.cassandra.thrift.ThriftConversion;
-import org.apache.thrift.TException;
import org.yaml.snakeyaml.Yaml;
import org.yaml.snakeyaml.constructor.Constructor;
import org.yaml.snakeyaml.error.YAMLException;
@@ -70,6 +68,7 @@
private List<String> extraSchemaDefinitions;
public final String seedStr = "seed for stress";
+ public String specName;
public String keyspaceName;
public String tableName;
private Map<String, GeneratorConfig> columnConfigs;
@@ -88,12 +87,10 @@
transient volatile RatioDistributionFactory selectchance;
transient volatile RatioDistributionFactory rowPopulation;
transient volatile PreparedStatement insertStatement;
- transient volatile Integer thriftInsertId;
transient volatile List<ValidatingSchemaQuery.Factory> validationFactories;
- transient volatile Map<String, SchemaQuery.ArgSelect> argSelects;
+ transient volatile Map<String, SchemaStatement.ArgSelect> argSelects;
transient volatile Map<String, PreparedStatement> queryStatements;
- transient volatile Map<String, Integer> thriftQueryIds;
private static final Pattern lowercaseAlphanumeric = Pattern.compile("[a-z0-9_]+");
@@ -150,6 +147,8 @@
queries = yaml.queries;
tokenRangeQueries = yaml.token_range_queries;
insert = yaml.insert;
+ specName = yaml.specname;
+ if (specName == null){specName = keyspaceName + "." + tableName;}
extraSchemaDefinitions = yaml.extra_definitions;
@@ -165,7 +164,7 @@
{
try
{
- String name = CQLFragmentParser.parseAnyUnhandled(CqlParser::createKeyspaceStatement, keyspaceCql).keyspace();
+ String name = CQLFragmentParser.parseAnyUnhandled(CqlParser::createKeyspaceStatement, keyspaceCql).keyspaceName;
assert name.equalsIgnoreCase(keyspaceName) : "Name in keyspace_definition doesn't match keyspace property: '" + name + "' != '" + keyspaceName + "'";
}
catch (RecognitionException | SyntaxException e)
@@ -182,7 +181,7 @@
{
try
{
- String name = CQLFragmentParser.parseAnyUnhandled(CqlParser::createTableStatement, tableCql).columnFamily();
+ String name = CQLFragmentParser.parseAnyUnhandled(CqlParser::createTableStatement, tableCql).table();
assert name.equalsIgnoreCase(tableName) : "Name in table_definition doesn't match table property: '" + name + "' != '" + tableName + "'";
}
catch (RecognitionException | RuntimeException e)
@@ -222,7 +221,7 @@
{
if (!schemaCreated)
{
- JavaDriverClient client = settings.getJavaDriverClient(false);
+ JavaDriverClient client = settings.getJavaDriverClient();
if (keyspaceCql != null)
{
@@ -289,7 +288,7 @@
{
if (tableMetaData == null)
{
- JavaDriverClient client = settings.getJavaDriverClient();
+ JavaDriverClient client = settings.getJavaDriverClient(keyspaceName);
synchronized (client)
{
@@ -306,7 +305,7 @@
throw new RuntimeException("Unable to find table " + keyspaceName + "." + tableName);
//Fill in missing column configs
- for (ColumnMetadata col : metadata.getColumns())
+ for (com.datastax.driver.core.ColumnMetadata col : metadata.getColumns())
{
if (columnConfigs.containsKey(col.getName()))
continue;
@@ -323,7 +322,7 @@
{
maybeLoadSchemaInfo(settings); // ensure table metadata is available
- JavaDriverClient client = settings.getJavaDriverClient();
+ JavaDriverClient client = settings.getJavaDriverClient(keyspaceName);
synchronized (client)
{
if (tokenRanges != null)
@@ -367,42 +366,57 @@
{
if (queryStatements == null)
{
- try
+ JavaDriverClient jclient = settings.getJavaDriverClient(keyspaceName);
+
+ Map<String, PreparedStatement> stmts = new HashMap<>();
+ Map<String, SchemaStatement.ArgSelect> args = new HashMap<>();
+ for (Map.Entry<String, StressYaml.QueryDef> e : queries.entrySet())
{
- JavaDriverClient jclient = settings.getJavaDriverClient();
- ThriftClient tclient = null;
-
- if (settings.mode.api != ConnectionAPI.JAVA_DRIVER_NATIVE)
- tclient = settings.getThriftClient();
-
- Map<String, PreparedStatement> stmts = new HashMap<>();
- Map<String, Integer> tids = new HashMap<>();
- Map<String, SchemaQuery.ArgSelect> args = new HashMap<>();
- for (Map.Entry<String, StressYaml.QueryDef> e : queries.entrySet())
- {
- stmts.put(e.getKey().toLowerCase(), jclient.prepare(e.getValue().cql));
-
- if (tclient != null)
- tids.put(e.getKey().toLowerCase(), tclient.prepare_cql3_query(e.getValue().cql, Compression.NONE));
-
- args.put(e.getKey().toLowerCase(), e.getValue().fields == null
- ? SchemaQuery.ArgSelect.MULTIROW
- : SchemaQuery.ArgSelect.valueOf(e.getValue().fields.toUpperCase()));
- }
- thriftQueryIds = tids;
- queryStatements = stmts;
- argSelects = args;
+ stmts.put(e.getKey().toLowerCase(), jclient.prepare(e.getValue().cql));
+ args.put(e.getKey().toLowerCase(), e.getValue().fields == null
+ ? SchemaStatement.ArgSelect.MULTIROW
+ : SchemaStatement.ArgSelect.valueOf(e.getValue().fields.toUpperCase()));
}
- catch (TException e)
- {
- throw new RuntimeException(e);
- }
+ queryStatements = stmts;
+ argSelects = args;
}
}
}
- return new SchemaQuery(timer, settings, generator, seeds, thriftQueryIds.get(name), queryStatements.get(name),
- ThriftConversion.fromThrift(settings.command.consistencyLevel), argSelects.get(name));
+ if (dynamicConditionExists(queryStatements.get(name)))
+ return new CASQuery(timer, settings, generator, seeds, queryStatements.get(name), settings.command.consistencyLevel, argSelects.get(name), tableName);
+
+ return new SchemaQuery(timer, settings, generator, seeds, queryStatements.get(name), settings.command.consistencyLevel, argSelects.get(name));
+ }
+
+ static boolean dynamicConditionExists(PreparedStatement statement) throws IllegalArgumentException
+ {
+ if (statement == null)
+ return false;
+
+ if (!statement.getQueryString().toUpperCase().startsWith("UPDATE"))
+ return false;
+
+ ModificationStatement.Parsed modificationStatement;
+ try
+ {
+ modificationStatement = CQLFragmentParser.parseAnyUnhandled(CqlParser::updateStatement,
+ statement.getQueryString());
+ }
+ catch (RecognitionException e)
+ {
+ throw new IllegalArgumentException("could not parse update query:" + statement.getQueryString(), e);
+ }
+
+ /*
+ * here we differentiate between static vs dynamic conditions:
+ * - static condition example: if col1 = NULL
+ * - dynamic condition example: if col1 = ?
+ * for static condition we don't have to replace value, no extra work involved.
+ * for dynamic condition we have to read existing db value and then
+ * use current db values during the update.
+ */
+ return modificationStatement.getConditions().stream().anyMatch(condition -> condition.right.getValue().getText().equals("?"));
}
public Operation getBulkReadQueries(String name, Timer timer, StressSettings settings, TokenRangeIterator tokenRangeIterator, boolean isWarmup)
@@ -417,40 +431,39 @@
public PartitionGenerator getOfflineGenerator()
{
- CFMetaData cfMetaData = CFMetaData.compile(tableCql, keyspaceName);
+ org.apache.cassandra.schema.TableMetadata metadata = CreateTableStatement.parse(tableCql, keyspaceName).build();
//Add missing column configs
- Iterator<ColumnDefinition> it = cfMetaData.allColumnsInSelectOrder();
+ Iterator<ColumnMetadata> it = metadata.allColumnsInSelectOrder();
while (it.hasNext())
{
- ColumnDefinition c = it.next();
+ ColumnMetadata c = it.next();
if (!columnConfigs.containsKey(c.name.toString()))
columnConfigs.put(c.name.toString(), new GeneratorConfig(seedStr + c.name.toString(), null, null, null));
}
- List<Generator> partitionColumns = cfMetaData.partitionKeyColumns().stream()
- .map(c -> new ColumnInfo(c.name.toString(), c.type.asCQL3Type().toString(), "", columnConfigs.get(c.name.toString())))
- .map(c -> c.getGenerator())
- .collect(Collectors.toList());
+ List<Generator> partitionColumns = metadata.partitionKeyColumns().stream()
+ .map(c -> new ColumnInfo(c.name.toString(), c.type.asCQL3Type().toString(), "", columnConfigs.get(c.name.toString())))
+ .map(c -> c.getGenerator())
+ .collect(Collectors.toList());
- List<Generator> clusteringColumns = cfMetaData.clusteringColumns().stream()
- .map(c -> new ColumnInfo(c.name.toString(), c.type.asCQL3Type().toString(), "", columnConfigs.get(c.name.toString())))
- .map(c -> c.getGenerator())
- .collect(Collectors.toList());
+ List<Generator> clusteringColumns = metadata.clusteringColumns().stream()
+ .map(c -> new ColumnInfo(c.name.toString(), c.type.asCQL3Type().toString(), "", columnConfigs.get(c.name.toString())))
+ .map(c -> c.getGenerator())
+ .collect(Collectors.toList());
- List<Generator> regularColumns = com.google.common.collect.Lists.newArrayList(cfMetaData.partitionColumns().selectOrderIterator()).stream()
- .map(c -> new ColumnInfo(c.name.toString(), c.type.asCQL3Type().toString(), "", columnConfigs.get(c.name.toString())))
- .map(c -> c.getGenerator())
- .collect(Collectors.toList());
+ List<Generator> regularColumns = com.google.common.collect.Lists.newArrayList(metadata.regularAndStaticColumns().selectOrderIterator()).stream()
+ .map(c -> new ColumnInfo(c.name.toString(), c.type.asCQL3Type().toString(), "", columnConfigs.get(c.name.toString())))
+ .map(c -> c.getGenerator())
+ .collect(Collectors.toList());
return new PartitionGenerator(partitionColumns, clusteringColumns, regularColumns, PartitionGenerator.Order.ARBITRARY);
}
- public CreateTableStatement.RawStatement getCreateStatement()
+ public CreateTableStatement.Raw getCreateStatement()
{
- CreateTableStatement.RawStatement createStatement = QueryProcessor.parseStatement(tableCql, CreateTableStatement.RawStatement.class, "CREATE TABLE");
- createStatement.prepareKeyspace(keyspaceName);
-
+ CreateTableStatement.Raw createStatement = CQLFragmentParser.parseAny(CqlParser::createTableStatement, tableCql, "CREATE TABLE");
+ createStatement.keyspace(keyspaceName);
return createStatement;
}
@@ -458,14 +471,14 @@
{
assert tableCql != null;
- CFMetaData cfMetaData = CFMetaData.compile(tableCql, keyspaceName);
+ org.apache.cassandra.schema.TableMetadata metadata = CreateTableStatement.parse(tableCql, keyspaceName).build();
- List<ColumnDefinition> allColumns = com.google.common.collect.Lists.newArrayList(cfMetaData.allColumnsInSelectOrder());
+ List<ColumnMetadata> allColumns = com.google.common.collect.Lists.newArrayList(metadata.allColumnsInSelectOrder());
StringBuilder sb = new StringBuilder();
sb.append("INSERT INTO ").append(quoteIdentifier(keyspaceName)).append(".").append(quoteIdentifier(tableName)).append(" (");
StringBuilder value = new StringBuilder();
- for (ColumnDefinition c : allColumns)
+ for (ColumnMetadata c : allColumns)
{
sb.append(quoteIdentifier(c.name.toString())).append(", ");
value.append("?, ");
@@ -492,7 +505,7 @@
String tableCreate = tableCql.replaceFirst("\\s+\"?"+tableName+"\"?\\s+", " \""+keyspaceName+"\".\""+tableName+"\" ");
- return new SchemaInsert(timer, settings, generator, seedManager, selectchance.get(), rowPopulation.get(), thriftInsertId, statement, tableCreate);
+ return new SchemaInsert(timer, settings, generator, seedManager, selectchance.get(), rowPopulation.get(), statement, tableCreate);
}
public SchemaInsert getInsert(Timer timer, PartitionGenerator generator, SeedManager seedManager, StressSettings settings)
@@ -505,8 +518,8 @@
{
maybeLoadSchemaInfo(settings);
- Set<ColumnMetadata> keyColumns = com.google.common.collect.Sets.newHashSet(tableMetaData.getPrimaryKey());
- Set<ColumnMetadata> allColumns = com.google.common.collect.Sets.newHashSet(tableMetaData.getColumns());
+ Set<com.datastax.driver.core.ColumnMetadata> keyColumns = com.google.common.collect.Sets.newHashSet(tableMetaData.getPrimaryKey());
+ Set<com.datastax.driver.core.ColumnMetadata> allColumns = com.google.common.collect.Sets.newHashSet(tableMetaData.getColumns());
boolean isKeyOnlyTable = (keyColumns.size() == allColumns.size());
//With compact storage
if (!isKeyOnlyTable && (keyColumns.size() == (allColumns.size() - 1)))
@@ -514,12 +527,16 @@
com.google.common.collect.Sets.SetView diff = com.google.common.collect.Sets.difference(allColumns, keyColumns);
for (Object obj : diff)
{
- ColumnMetadata col = (ColumnMetadata)obj;
+ com.datastax.driver.core.ColumnMetadata col = (com.datastax.driver.core.ColumnMetadata)obj;
isKeyOnlyTable = col.getName().isEmpty();
break;
}
}
+ if (insert == null)
+ insert = new HashMap<>();
+ lowerCase(insert);
+
//Non PK Columns
StringBuilder sb = new StringBuilder();
if (!isKeyOnlyTable)
@@ -531,7 +548,7 @@
boolean firstCol = true;
boolean firstPred = true;
- for (ColumnMetadata c : tableMetaData.getColumns()) {
+ for (com.datastax.driver.core.ColumnMetadata c : tableMetaData.getColumns()) {
if (keyColumns.contains(c)) {
if (firstPred)
@@ -569,12 +586,17 @@
//Put PK predicates at the end
sb.append(pred);
+ if (insert.containsKey("condition"))
+ {
+ sb.append(" " + insert.get("condition"));
+ insert.remove("condition");
+ }
}
else
{
sb.append("INSERT INTO ").append(quoteIdentifier(tableName)).append(" (");
StringBuilder value = new StringBuilder();
- for (ColumnMetadata c : tableMetaData.getPrimaryKey())
+ for (com.datastax.driver.core.ColumnMetadata c : tableMetaData.getPrimaryKey())
{
sb.append(quoteIdentifier(c.getName())).append(", ");
value.append("?, ");
@@ -584,10 +606,6 @@
sb.append(") ").append("values(").append(value).append(')');
}
- if (insert == null)
- insert = new HashMap<>();
- lowerCase(insert);
-
partitions = select(settings.insert.batchsize, "partitions", "fixed(1)", insert, OptionDistribution.BUILDER);
selectchance = select(settings.insert.selectRatio, "select", "fixed(1)/1", insert, OptionRatioDistribution.BUILDER);
rowPopulation = select(settings.insert.rowPopulationRatio, "row-population", "fixed(1)/1", insert, OptionRatioDistribution.BUILDER);
@@ -617,27 +635,17 @@
System.err.printf("WARNING: You have defined a schema that permits very large batches (%.0f max rows (>100K)). This may OOM this stress client, or the server.%n",
selectchance.get().max() * partitions.get().maxValue() * generator.maxRowCount);
- JavaDriverClient client = settings.getJavaDriverClient();
+ JavaDriverClient client = settings.getJavaDriverClient(keyspaceName);
String query = sb.toString();
- if (settings.mode.api != ConnectionAPI.JAVA_DRIVER_NATIVE)
- {
- try
- {
- thriftInsertId = settings.getThriftClient().prepare_cql3_query(query, Compression.NONE);
- }
- catch (TException e)
- {
- throw new RuntimeException(e);
- }
- }
-
insertStatement = client.prepare(query);
+ System.out.println("Insert Statement:");
+ System.out.println(" " + query);
}
}
}
- return new SchemaInsert(timer, settings, generator, seedManager, partitions.get(), selectchance.get(), rowPopulation.get(), thriftInsertId, insertStatement, ThriftConversion.fromThrift(settings.command.consistencyLevel), batchType);
+ return new SchemaInsert(timer, settings, generator, seedManager, partitions.get(), selectchance.get(), rowPopulation.get(), insertStatement, settings.command.consistencyLevel, batchType);
}
public List<ValidatingSchemaQuery> getValidate(Timer timer, PartitionGenerator generator, SeedManager seedManager, StressSettings settings)
@@ -656,7 +664,7 @@
List<ValidatingSchemaQuery> queries = new ArrayList<>();
for (ValidatingSchemaQuery.Factory factory : validationFactories)
- queries.add(factory.create(timer, settings, generator, seedManager, ThriftConversion.fromThrift(settings.command.consistencyLevel)));
+ queries.add(factory.create(timer, settings, generator, seedManager, settings.command.consistencyLevel));
return queries;
}
@@ -696,17 +704,17 @@
private GeneratorFactory()
{
- Set<ColumnMetadata> keyColumns = com.google.common.collect.Sets.newHashSet(tableMetaData.getPrimaryKey());
+ Set<com.datastax.driver.core.ColumnMetadata> keyColumns = com.google.common.collect.Sets.newHashSet(tableMetaData.getPrimaryKey());
- for (ColumnMetadata metadata : tableMetaData.getPartitionKey())
+ for (com.datastax.driver.core.ColumnMetadata metadata : tableMetaData.getPartitionKey())
partitionKeys.add(new ColumnInfo(metadata.getName(), metadata.getType().getName().toString(),
metadata.getType().isCollection() ? metadata.getType().getTypeArguments().get(0).getName().toString() : "",
columnConfigs.get(metadata.getName())));
- for (ColumnMetadata metadata : tableMetaData.getClusteringColumns())
+ for (com.datastax.driver.core.ColumnMetadata metadata : tableMetaData.getClusteringColumns())
clusteringColumns.add(new ColumnInfo(metadata.getName(), metadata.getType().getName().toString(),
metadata.getType().isCollection() ? metadata.getType().getTypeArguments().get(0).getName().toString() : "",
columnConfigs.get(metadata.getName())));
- for (ColumnMetadata metadata : tableMetaData.getColumns())
+ for (com.datastax.driver.core.ColumnMetadata metadata : tableMetaData.getColumns())
if (!keyColumns.contains(metadata))
valueColumns.add(new ColumnInfo(metadata.getName(), metadata.getType().getName().toString(),
metadata.getType().isCollection() ? metadata.getType().getTypeArguments().get(0).getName().toString() : "",
@@ -813,6 +821,7 @@
StressYaml profileYaml = yaml.loadAs(yamlStream, StressYaml.class);
+
StressProfile profile = new StressProfile();
profile.init(profileYaml);
diff --git a/tools/stress/src/org/apache/cassandra/stress/StressYaml.java b/tools/stress/src/org/apache/cassandra/stress/StressYaml.java
index 6727f62..afb932c 100644
--- a/tools/stress/src/org/apache/cassandra/stress/StressYaml.java
+++ b/tools/stress/src/org/apache/cassandra/stress/StressYaml.java
@@ -26,6 +26,7 @@
public class StressYaml
{
+ public String specname;
public String keyspace;
public String keyspace_definition;
public String table;
diff --git a/tools/stress/src/org/apache/cassandra/stress/WorkManager.java b/tools/stress/src/org/apache/cassandra/stress/WorkManager.java
index 78d4176..fe661f7 100644
--- a/tools/stress/src/org/apache/cassandra/stress/WorkManager.java
+++ b/tools/stress/src/org/apache/cassandra/stress/WorkManager.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.cassandra.stress;
import java.util.concurrent.atomic.AtomicLong;
diff --git a/tools/stress/src/org/apache/cassandra/stress/generate/Distribution.java b/tools/stress/src/org/apache/cassandra/stress/generate/Distribution.java
index 4662454..9d8ce1c 100644
--- a/tools/stress/src/org/apache/cassandra/stress/generate/Distribution.java
+++ b/tools/stress/src/org/apache/cassandra/stress/generate/Distribution.java
@@ -45,13 +45,13 @@
public long average()
{
double sum = 0;
- int count = 0;
- for (float d = 0 ; d <= 1.0d ; d += 0.02d)
+ float d = 0;
+ for (int count = 0; count < 51 ; count++)
{
sum += inverseCumProb(d);
- count += 1;
+ d += 0.02d;
}
- return (long) (sum / count);
+ return (long) (sum / 51);
}
}
diff --git a/tools/stress/src/org/apache/cassandra/stress/generate/PartitionGenerator.java b/tools/stress/src/org/apache/cassandra/stress/generate/PartitionGenerator.java
index 1230065..882b8b4 100644
--- a/tools/stress/src/org/apache/cassandra/stress/generate/PartitionGenerator.java
+++ b/tools/stress/src/org/apache/cassandra/stress/generate/PartitionGenerator.java
@@ -83,6 +83,16 @@
return !(index < 0 || index < clusteringComponents.size());
}
+ public List<Generator> getPartitionKey()
+ {
+ return Collections.unmodifiableList(partitionKey);
+ }
+
+ public List<Generator> getClusteringComponents()
+ {
+ return Collections.unmodifiableList(clusteringComponents);
+ }
+
public int indexOf(String name)
{
Integer i = indexMap.get(name);
diff --git a/tools/stress/src/org/apache/cassandra/stress/generate/PartitionIterator.java b/tools/stress/src/org/apache/cassandra/stress/generate/PartitionIterator.java
index 2157214..f485ab7 100644
--- a/tools/stress/src/org/apache/cassandra/stress/generate/PartitionIterator.java
+++ b/tools/stress/src/org/apache/cassandra/stress/generate/PartitionIterator.java
@@ -770,10 +770,4 @@
}
return sb.toString();
}
-
- // used for thrift smart routing - if it's a multi-part key we don't try to route correctly right now
- public ByteBuffer getToken()
- {
- return generator.partitionKey.get(0).type.decompose(partitionKey[0]);
- }
}
diff --git a/tools/stress/src/org/apache/cassandra/stress/generate/SeedManager.java b/tools/stress/src/org/apache/cassandra/stress/generate/SeedManager.java
index 5020b45..6874fba1 100644
--- a/tools/stress/src/org/apache/cassandra/stress/generate/SeedManager.java
+++ b/tools/stress/src/org/apache/cassandra/stress/generate/SeedManager.java
@@ -38,39 +38,45 @@
final Distribution sample;
final long sampleOffset;
final int sampleSize;
+ final long sampleMultiplier;
final boolean updateSampleImmediately;
public SeedManager(StressSettings settings)
{
+ Distribution tSample = settings.insert.revisit.get();
+ this.sampleOffset = Math.min(tSample.minValue(), tSample.maxValue());
+ long sampleSize = 1 + Math.max(tSample.minValue(), tSample.maxValue()) - sampleOffset;
+ if (sampleOffset < 0 || sampleSize > Integer.MAX_VALUE)
+ throw new IllegalArgumentException("sample range is invalid");
+
+ // need to get a big numerical range even if a small number of discrete values
+ // one plus so we still get variation at the low order numbers as well as high
+ this.sampleMultiplier = 1 + Math.round(Math.pow(10D, 22 - Math.log10(sampleSize)));
+
Generator writes, reads;
if (settings.generate.sequence != null)
{
long[] seq = settings.generate.sequence;
if (settings.generate.readlookback != null)
{
- LookbackableWriteGenerator series = new LookbackableWriteGenerator(seq[0], seq[1], settings.generate.wrap, settings.generate.readlookback.get());
+ LookbackableWriteGenerator series = new LookbackableWriteGenerator(seq[0], seq[1], settings.generate.wrap, settings.generate.readlookback.get(), sampleMultiplier);
writes = series;
reads = series.reads;
}
else
{
- writes = reads = new SeriesGenerator(seq[0], seq[1], settings.generate.wrap);
+ writes = reads = new SeriesGenerator(seq[0], seq[1], settings.generate.wrap, sampleMultiplier);
}
}
else
{
- writes = reads = new RandomGenerator(settings.generate.distribution.get());
+ writes = reads = new RandomGenerator(settings.generate.distribution.get(), sampleMultiplier);
}
this.visits = settings.insert.visits.get();
this.writes = writes;
this.reads = reads;
- Distribution sample = settings.insert.revisit.get();
- this.sampleOffset = Math.min(sample.minValue(), sample.maxValue());
- long sampleSize = 1 + Math.max(sample.minValue(), sample.maxValue()) - sampleOffset;
- if (sampleOffset < 0 || sampleSize > Integer.MAX_VALUE)
- throw new IllegalArgumentException("sample range is invalid");
this.sampleFrom = new LockedDynamicList<>((int) sampleSize);
- this.sample = DistributionInverted.invert(sample);
+ this.sample = DistributionInverted.invert(tSample);
this.sampleSize = (int) sampleSize;
this.updateSampleImmediately = visits.average() > 1;
}
@@ -82,7 +88,7 @@
Seed seed = reads.next(-1);
if (seed == null)
return null;
- Seed managing = this.managing.get(seed);
+ Seed managing = this.managing.get(seed.seed);
return managing == null ? seed : managing;
}
@@ -132,15 +138,18 @@
{
final Distribution distribution;
+ final long multiplier;
- public RandomGenerator(Distribution distribution)
+ public RandomGenerator(Distribution distribution, long multiplier)
{
+
this.distribution = distribution;
+ this.multiplier = multiplier;
}
public Seed next(int visits)
{
- return new Seed(distribution.next(), visits);
+ return new Seed(distribution.next() * multiplier, visits);
}
}
@@ -150,15 +159,18 @@
final long start;
final long totalCount;
final boolean wrap;
+ final long multiplier;
final AtomicLong next = new AtomicLong();
- public SeriesGenerator(long start, long end, boolean wrap)
+ public SeriesGenerator(long start, long end, boolean wrap, long multiplier)
{
this.wrap = wrap;
if (start > end)
throw new IllegalStateException();
this.start = start;
this.totalCount = 1 + end - start;
+ this.multiplier = multiplier;
+
}
public Seed next(int visits)
@@ -166,7 +178,7 @@
long next = this.next.getAndIncrement();
if (!wrap && next >= totalCount)
return null;
- return new Seed(start + (next % totalCount), visits);
+ return new Seed((start + (next % totalCount))*multiplier, visits);
}
}
@@ -177,9 +189,9 @@
final ConcurrentSkipListMap<Seed, Seed> afterMin = new ConcurrentSkipListMap<>();
final LookbackReadGenerator reads;
- public LookbackableWriteGenerator(long start, long end, boolean wrap, Distribution readLookback)
+ public LookbackableWriteGenerator(long start, long end, boolean wrap, Distribution readLookback, long multiplier)
{
- super(start, end, wrap);
+ super(start, end, wrap, multiplier);
this.writeCount.set(0);
reads = new LookbackReadGenerator(readLookback);
}
@@ -189,12 +201,12 @@
long next = this.next.getAndIncrement();
if (!wrap && next >= totalCount)
return null;
- return new Seed(start + (next % totalCount), visits);
+ return new Seed((start + (next % totalCount)) * multiplier, visits);
}
void finishWrite(Seed seed)
{
- if (seed.seed <= writeCount.get())
+ if (seed.seed/multiplier <= writeCount.get())
return;
afterMin.put(seed, seed);
while (true)
@@ -216,7 +228,6 @@
private class LookbackReadGenerator extends Generator
{
-
final Distribution lookback;
public LookbackReadGenerator(Distribution lookback)
diff --git a/tools/stress/src/org/apache/cassandra/stress/generate/values/Booleans.java b/tools/stress/src/org/apache/cassandra/stress/generate/values/Booleans.java
index 21525af..9ecacbb 100644
--- a/tools/stress/src/org/apache/cassandra/stress/generate/values/Booleans.java
+++ b/tools/stress/src/org/apache/cassandra/stress/generate/values/Booleans.java
@@ -32,6 +32,6 @@
@Override
public Boolean generate()
{
- return identityDistribution.next() % 1 == 0;
+ return identityDistribution.next() % 2 == 0;
}
}
diff --git a/tools/stress/src/org/apache/cassandra/stress/operations/PartitionOperation.java b/tools/stress/src/org/apache/cassandra/stress/operations/PartitionOperation.java
index bad0a94..55c6872 100644
--- a/tools/stress/src/org/apache/cassandra/stress/operations/PartitionOperation.java
+++ b/tools/stress/src/org/apache/cassandra/stress/operations/PartitionOperation.java
@@ -19,6 +19,7 @@
package org.apache.cassandra.stress.operations;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
import org.apache.cassandra.stress.Operation;
@@ -75,6 +76,16 @@
this.spec = spec;
}
+ public DataSpec getDataSpecification()
+ {
+ return spec;
+ }
+
+ public List<PartitionIterator> getPartitions()
+ {
+ return Collections.unmodifiableList(partitions);
+ }
+
public int ready(WorkManager permits)
{
int partitionCount = (int) spec.partitionCount.next();
@@ -86,7 +97,7 @@
int i = 0;
boolean success = true;
- for (; i < partitionCount && success ; i++)
+ for (; i < partitionCount && success; i++)
{
if (i >= partitionCache.size())
partitionCache.add(PartitionIterator.get(spec.partitionGenerator, spec.seedManager));
diff --git a/tools/stress/src/org/apache/cassandra/stress/operations/SampledOpDistributionFactory.java b/tools/stress/src/org/apache/cassandra/stress/operations/SampledOpDistributionFactory.java
index 1800039..59f2394 100644
--- a/tools/stress/src/org/apache/cassandra/stress/operations/SampledOpDistributionFactory.java
+++ b/tools/stress/src/org/apache/cassandra/stress/operations/SampledOpDistributionFactory.java
@@ -46,17 +46,15 @@
this.clustering = clustering;
}
- protected abstract List<? extends Operation> get(Timer timer, PartitionGenerator generator, T key, boolean isWarmup);
- protected abstract PartitionGenerator newGenerator();
+ protected abstract List<? extends Operation> get(Timer timer, T key, boolean isWarmup);
public OpDistribution get(boolean isWarmup, MeasurementSink sink)
{
- PartitionGenerator generator = newGenerator();
List<Pair<Operation, Double>> operations = new ArrayList<>();
for (Map.Entry<T, Double> ratio : ratios.entrySet())
{
List<? extends Operation> ops = get(new Timer(ratio.getKey().toString(), sink),
- generator, ratio.getKey(), isWarmup);
+ ratio.getKey(), isWarmup);
for (Operation op : ops)
operations.add(new Pair<>(op, ratio.getValue() / ops.size()));
}
@@ -81,7 +79,6 @@
public OpDistribution get(boolean isWarmup, MeasurementSink sink)
{
List<? extends Operation> ops = SampledOpDistributionFactory.this.get(new Timer(ratio.getKey().toString(), sink),
- newGenerator(),
ratio.getKey(),
isWarmup);
if (ops.size() == 1)
diff --git a/tools/stress/src/org/apache/cassandra/stress/operations/predefined/CqlOperation.java b/tools/stress/src/org/apache/cassandra/stress/operations/predefined/CqlOperation.java
index 807bb49..c89a1d1 100644
--- a/tools/stress/src/org/apache/cassandra/stress/operations/predefined/CqlOperation.java
+++ b/tools/stress/src/org/apache/cassandra/stress/operations/predefined/CqlOperation.java
@@ -36,15 +36,9 @@
import org.apache.cassandra.stress.settings.ConnectionStyle;
import org.apache.cassandra.stress.settings.StressSettings;
import org.apache.cassandra.stress.util.JavaDriverClient;
-import org.apache.cassandra.stress.util.ThriftClient;
-import org.apache.cassandra.thrift.Compression;
-import org.apache.cassandra.thrift.CqlResult;
-import org.apache.cassandra.thrift.CqlRow;
-import org.apache.cassandra.thrift.ThriftConversion;
import org.apache.cassandra.transport.SimpleClient;
import org.apache.cassandra.transport.messages.ResultMessage;
import org.apache.cassandra.utils.ByteBufferUtil;
-import org.apache.thrift.TException;
public abstract class CqlOperation<V> extends PredefinedOperation
{
@@ -72,17 +66,13 @@
Object idobj = getCqlCache();
if (idobj == null)
{
- try
- {
- id = client.createPreparedStatement(buildQuery());
- } catch (TException e)
- {
- throw new RuntimeException(e);
- }
+ id = client.createPreparedStatement(buildQuery());
storeCqlCache(id);
}
else
+ {
id = idobj;
+ }
op = buildRunOp(client, null, id, queryParams, key);
}
@@ -247,12 +237,6 @@
@Override
- public void run(final ThriftClient client) throws IOException
- {
- run(wrap(client));
- }
-
- @Override
public void run(SimpleClient client) throws IOException
{
run(wrap(client));
@@ -264,11 +248,6 @@
run(wrap(client));
}
- public ClientWrapper wrap(ThriftClient client)
- {
- return new Cql3CassandraClientWrapper(client);
- }
-
public ClientWrapper wrap(JavaDriverClient client)
{
return new JavaDriverWrapper(client);
@@ -281,9 +260,9 @@
protected interface ClientWrapper
{
- Object createPreparedStatement(String cqlQuery) throws TException;
- <V> V execute(Object preparedStatementId, ByteBuffer key, List<Object> queryParams, ResultHandler<V> handler) throws TException;
- <V> V execute(String query, ByteBuffer key, List<Object> queryParams, ResultHandler<V> handler) throws TException;
+ Object createPreparedStatement(String cqlQuery);
+ <V> V execute(Object preparedStatementId, ByteBuffer key, List<Object> queryParams, ResultHandler<V> handler);
+ <V> V execute(String query, ByteBuffer key, List<Object> queryParams, ResultHandler<V> handler);
}
private final class JavaDriverWrapper implements ClientWrapper
@@ -298,17 +277,17 @@
public <V> V execute(String query, ByteBuffer key, List<Object> queryParams, ResultHandler<V> handler)
{
String formattedQuery = formatCqlQuery(query, queryParams);
- return handler.javaDriverHandler().apply(client.execute(formattedQuery, ThriftConversion.fromThrift(settings.command.consistencyLevel)));
+ return handler.javaDriverHandler().apply(client.execute(formattedQuery, settings.command.consistencyLevel));
}
@Override
- public <V> V execute(Object preparedStatementId, ByteBuffer key, List<Object> queryParams, ResultHandler<V> handler)
+ public <V> V execute(Object preparedStatement, ByteBuffer key, List<Object> queryParams, ResultHandler<V> handler)
{
return handler.javaDriverHandler().apply(
client.executePrepared(
- (PreparedStatement) preparedStatementId,
+ (PreparedStatement) preparedStatement,
queryParams,
- ThriftConversion.fromThrift(settings.command.consistencyLevel)));
+ settings.command.consistencyLevel));
}
@Override
@@ -330,17 +309,17 @@
public <V> V execute(String query, ByteBuffer key, List<Object> queryParams, ResultHandler<V> handler)
{
String formattedQuery = formatCqlQuery(query, queryParams);
- return handler.thriftHandler().apply(client.execute(formattedQuery, ThriftConversion.fromThrift(settings.command.consistencyLevel)));
+ return handler.simpleClientHandler().apply(client.execute(formattedQuery, settings.command.consistencyLevel));
}
@Override
- public <V> V execute(Object preparedStatementId, ByteBuffer key, List<Object> queryParams, ResultHandler<V> handler)
+ public <V> V execute(Object preparedStatement, ByteBuffer key, List<Object> queryParams, ResultHandler<V> handler)
{
- return handler.thriftHandler().apply(
+ return handler.simpleClientHandler().apply(
client.executePrepared(
- (byte[]) preparedStatementId,
+ (ResultMessage.Prepared) preparedStatement,
toByteBufferParams(queryParams),
- ThriftConversion.fromThrift(settings.command.consistencyLevel)));
+ settings.command.consistencyLevel));
}
@Override
@@ -350,46 +329,12 @@
}
}
- // client wrapper for Cql3
- private final class Cql3CassandraClientWrapper implements ClientWrapper
- {
- final ThriftClient client;
- private Cql3CassandraClientWrapper(ThriftClient client)
- {
- this.client = client;
- }
-
- @Override
- public <V> V execute(String query, ByteBuffer key, List<Object> queryParams, ResultHandler<V> handler) throws TException
- {
- String formattedQuery = formatCqlQuery(query, queryParams);
- return handler.simpleNativeHandler().apply(
- client.execute_cql3_query(formattedQuery, key, Compression.NONE, settings.command.consistencyLevel)
- );
- }
-
- @Override
- public <V> V execute(Object preparedStatementId, ByteBuffer key, List<Object> queryParams, ResultHandler<V> handler) throws TException
- {
- Integer id = (Integer) preparedStatementId;
- return handler.simpleNativeHandler().apply(
- client.execute_prepared_cql3_query(id, key, toByteBufferParams(queryParams), settings.command.consistencyLevel)
- );
- }
-
- @Override
- public Object createPreparedStatement(String cqlQuery) throws TException
- {
- return client.prepare_cql3_query(cqlQuery, Compression.NONE);
- }
- }
// interface for building functions to standardise results from each client
protected static interface ResultHandler<V>
{
Function<ResultSet, V> javaDriverHandler();
- Function<ResultMessage, V> thriftHandler();
- Function<CqlResult, V> simpleNativeHandler();
+ Function<ResultMessage, V> simpleClientHandler();
}
protected static class RowCountHandler implements ResultHandler<Integer>
@@ -412,7 +357,7 @@
}
@Override
- public Function<ResultMessage, Integer> thriftHandler()
+ public Function<ResultMessage, Integer> simpleClientHandler()
{
return new Function<ResultMessage, Integer>()
{
@@ -423,27 +368,6 @@
}
};
}
-
- @Override
- public Function<CqlResult, Integer> simpleNativeHandler()
- {
- return new Function<CqlResult, Integer>()
- {
-
- @Override
- public Integer apply(CqlResult result)
- {
- switch (result.getType())
- {
- case ROWS:
- return result.getRows().size();
- default:
- return 1;
- }
- }
- };
- }
-
}
// Processes results from each client into an array of all key bytes returned
@@ -478,7 +402,7 @@
}
@Override
- public Function<ResultMessage, ByteBuffer[][]> thriftHandler()
+ public Function<ResultMessage, ByteBuffer[][]> simpleClientHandler()
{
return new Function<ResultMessage, ByteBuffer[][]>()
{
@@ -502,29 +426,6 @@
}
};
}
-
- @Override
- public Function<CqlResult, ByteBuffer[][]> simpleNativeHandler()
- {
- return new Function<CqlResult, ByteBuffer[][]>()
- {
-
- @Override
- public ByteBuffer[][] apply(CqlResult result)
- {
- ByteBuffer[][] r = new ByteBuffer[result.getRows().size()][];
- for (int i = 0 ; i < r.length ; i++)
- {
- CqlRow row = result.getRows().get(i);
- r[i] = new ByteBuffer[row.getColumns().size()];
- for (int j = 0 ; j < r[i].length ; j++)
- r[i][j] = ByteBuffer.wrap(row.getColumns().get(j).getValue());
- }
- return r;
- }
- };
- }
-
}
// Processes results from each client into an array of all key bytes returned
protected static final class KeysHandler implements ResultHandler<byte[][]>
@@ -553,7 +454,7 @@
}
@Override
- public Function<ResultMessage, byte[][]> thriftHandler()
+ public Function<ResultMessage, byte[][]> simpleClientHandler()
{
return new Function<ResultMessage, byte[][]>()
{
@@ -573,24 +474,6 @@
}
};
}
-
- @Override
- public Function<CqlResult, byte[][]> simpleNativeHandler()
- {
- return new Function<CqlResult, byte[][]>()
- {
-
- @Override
- public byte[][] apply(CqlResult result)
- {
- byte[][] r = new byte[result.getRows().size()][];
- for (int i = 0 ; i < r.length ; i++)
- r[i] = result.getRows().get(i).getKey();
- return r;
- }
- };
- }
-
}
private static String getUnQuotedCqlBlob(ByteBuffer term)
diff --git a/tools/stress/src/org/apache/cassandra/stress/operations/predefined/PredefinedOperation.java b/tools/stress/src/org/apache/cassandra/stress/operations/predefined/PredefinedOperation.java
index db35504..9062cb6 100644
--- a/tools/stress/src/org/apache/cassandra/stress/operations/predefined/PredefinedOperation.java
+++ b/tools/stress/src/org/apache/cassandra/stress/operations/predefined/PredefinedOperation.java
@@ -30,8 +30,6 @@
import org.apache.cassandra.stress.settings.Command;
import org.apache.cassandra.stress.settings.CqlVersion;
import org.apache.cassandra.stress.settings.StressSettings;
-import org.apache.cassandra.thrift.SlicePredicate;
-import org.apache.cassandra.thrift.SliceRange;
public abstract class PredefinedOperation extends PartitionOperation
{
@@ -100,24 +98,6 @@
{
return indices != null ? indices.length : ub - lb;
}
-
- SlicePredicate predicate()
- {
- final SlicePredicate predicate = new SlicePredicate();
- if (indices == null)
- {
- predicate.setSlice_range(new SliceRange()
- .setStart(settings.columns.names.get(lb))
- .setFinish(EMPTY_BYTE_ARRAY)
- .setReversed(false)
- .setCount(count())
- );
- }
- else
- predicate.setColumn_names(select(settings.columns.names));
- return predicate;
-
- }
}
public String toString()
@@ -185,57 +165,14 @@
switch (type)
{
case READ:
- switch(settings.mode.style)
- {
- case THRIFT:
- return new ThriftReader(timer, generator, seedManager, settings);
- case CQL:
- case CQL_PREPARED:
- return new CqlReader(timer, generator, seedManager, settings);
- default:
- throw new UnsupportedOperationException();
- }
-
-
+ return new CqlReader(timer, generator, seedManager, settings);
case COUNTER_READ:
- switch(settings.mode.style)
- {
- case THRIFT:
- return new ThriftCounterGetter(timer, generator, seedManager, settings);
- case CQL:
- case CQL_PREPARED:
- return new CqlCounterGetter(timer, generator, seedManager, settings);
- default:
- throw new UnsupportedOperationException();
- }
-
+ return new CqlCounterGetter(timer, generator, seedManager, settings);
case WRITE:
-
- switch(settings.mode.style)
- {
- case THRIFT:
- return new ThriftInserter(timer, generator, seedManager, settings);
- case CQL:
- case CQL_PREPARED:
- return new CqlInserter(timer, generator, seedManager, settings);
- default:
- throw new UnsupportedOperationException();
- }
-
+ return new CqlInserter(timer, generator, seedManager, settings);
case COUNTER_WRITE:
- switch(settings.mode.style)
- {
- case THRIFT:
- return new ThriftCounterAdder(counteradd, timer, generator, seedManager, settings);
- case CQL:
- case CQL_PREPARED:
- return new CqlCounterAdder(counteradd, timer, generator, seedManager, settings);
- default:
- throw new UnsupportedOperationException();
- }
-
+ return new CqlCounterAdder(counteradd, timer, generator, seedManager, settings);
}
-
throw new UnsupportedOperationException();
}
diff --git a/tools/stress/src/org/apache/cassandra/stress/operations/predefined/ThriftCounterAdder.java b/tools/stress/src/org/apache/cassandra/stress/operations/predefined/ThriftCounterAdder.java
deleted file mode 100644
index 42f8bc9..0000000
--- a/tools/stress/src/org/apache/cassandra/stress/operations/predefined/ThriftCounterAdder.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.stress.operations.predefined;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.cassandra.stress.generate.Distribution;
-import org.apache.cassandra.stress.generate.DistributionFactory;
-import org.apache.cassandra.stress.generate.PartitionGenerator;
-import org.apache.cassandra.stress.generate.SeedManager;
-import org.apache.cassandra.stress.report.Timer;
-import org.apache.cassandra.stress.settings.Command;
-import org.apache.cassandra.stress.settings.StressSettings;
-import org.apache.cassandra.stress.util.ThriftClient;
-import org.apache.cassandra.thrift.ColumnOrSuperColumn;
-import org.apache.cassandra.thrift.CounterColumn;
-import org.apache.cassandra.thrift.Mutation;
-
-public class ThriftCounterAdder extends PredefinedOperation
-{
-
- final Distribution counteradd;
- public ThriftCounterAdder(DistributionFactory counteradd, Timer timer, PartitionGenerator generator, SeedManager seedManager, StressSettings settings)
- {
- super(Command.COUNTER_WRITE, timer, generator, seedManager, settings);
- this.counteradd = counteradd.get();
- }
-
- public boolean isWrite()
- {
- return true;
- }
-
- public void run(final ThriftClient client) throws IOException
- {
- List<CounterColumn> columns = new ArrayList<>();
- for (ByteBuffer name : select().select(settings.columns.names))
- columns.add(new CounterColumn(name, counteradd.next()));
-
- List<Mutation> mutations = new ArrayList<>(columns.size());
- for (CounterColumn c : columns)
- {
- ColumnOrSuperColumn cosc = new ColumnOrSuperColumn().setCounter_column(c);
- mutations.add(new Mutation().setColumn_or_supercolumn(cosc));
- }
- Map<String, List<Mutation>> row = Collections.singletonMap(type.table, mutations);
-
- final ByteBuffer key = getKey();
- final Map<ByteBuffer, Map<String, List<Mutation>>> record = Collections.singletonMap(key, row);
-
- timeWithRetry(new RunOp()
- {
- @Override
- public boolean run() throws Exception
- {
- client.batch_mutate(record, settings.command.consistencyLevel);
- return true;
- }
-
- @Override
- public int partitionCount()
- {
- return 1;
- }
-
- @Override
- public int rowCount()
- {
- return 1;
- }
- });
- }
-
-}
diff --git a/tools/stress/src/org/apache/cassandra/stress/operations/predefined/ThriftCounterGetter.java b/tools/stress/src/org/apache/cassandra/stress/operations/predefined/ThriftCounterGetter.java
deleted file mode 100644
index 4bec3b2..0000000
--- a/tools/stress/src/org/apache/cassandra/stress/operations/predefined/ThriftCounterGetter.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.stress.operations.predefined;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.List;
-
-import org.apache.cassandra.stress.generate.PartitionGenerator;
-import org.apache.cassandra.stress.generate.SeedManager;
-import org.apache.cassandra.stress.report.Timer;
-import org.apache.cassandra.stress.settings.Command;
-import org.apache.cassandra.stress.settings.StressSettings;
-import org.apache.cassandra.stress.util.ThriftClient;
-import org.apache.cassandra.thrift.ColumnParent;
-import org.apache.cassandra.thrift.SlicePredicate;
-
-public class ThriftCounterGetter extends PredefinedOperation
-{
- public ThriftCounterGetter(Timer timer, PartitionGenerator generator, SeedManager seedManager, StressSettings settings)
- {
- super(Command.COUNTER_READ, timer, generator, seedManager, settings);
- }
-
- public void run(final ThriftClient client) throws IOException
- {
- final SlicePredicate predicate = select().predicate();
- final ByteBuffer key = getKey();
- timeWithRetry(new RunOp()
- {
- @Override
- public boolean run() throws Exception
- {
- List<?> r = client.get_slice(key, new ColumnParent(type.table), predicate, settings.command.consistencyLevel);
- return r != null && r.size() > 0;
- }
-
- @Override
- public int partitionCount()
- {
- return 1;
- }
-
- @Override
- public int rowCount()
- {
- return 1;
- }
- });
- }
-
-}
diff --git a/tools/stress/src/org/apache/cassandra/stress/operations/predefined/ThriftInserter.java b/tools/stress/src/org/apache/cassandra/stress/operations/predefined/ThriftInserter.java
deleted file mode 100644
index ecaa140..0000000
--- a/tools/stress/src/org/apache/cassandra/stress/operations/predefined/ThriftInserter.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.stress.operations.predefined;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.cassandra.stress.generate.PartitionGenerator;
-import org.apache.cassandra.stress.generate.SeedManager;
-import org.apache.cassandra.stress.report.Timer;
-import org.apache.cassandra.stress.settings.Command;
-import org.apache.cassandra.stress.settings.StressSettings;
-import org.apache.cassandra.stress.util.ThriftClient;
-import org.apache.cassandra.thrift.Column;
-import org.apache.cassandra.thrift.ColumnOrSuperColumn;
-import org.apache.cassandra.thrift.Mutation;
-import org.apache.cassandra.utils.FBUtilities;
-
-public final class ThriftInserter extends PredefinedOperation
-{
-
- public ThriftInserter(Timer timer, PartitionGenerator generator, SeedManager seedManager, StressSettings settings)
- {
- super(Command.WRITE, timer, generator, seedManager, settings);
- }
-
- public boolean isWrite()
- {
- return true;
- }
-
- public void run(final ThriftClient client) throws IOException
- {
- final ByteBuffer key = getKey();
- final List<Column> columns = getColumns();
-
- List<Mutation> mutations = new ArrayList<>(columns.size());
- for (Column c : columns)
- {
- ColumnOrSuperColumn column = new ColumnOrSuperColumn().setColumn(c);
- mutations.add(new Mutation().setColumn_or_supercolumn(column));
- }
- Map<String, List<Mutation>> row = Collections.singletonMap(type.table, mutations);
-
- final Map<ByteBuffer, Map<String, List<Mutation>>> record = Collections.singletonMap(key, row);
-
- timeWithRetry(new RunOp()
- {
- @Override
- public boolean run() throws Exception
- {
- client.batch_mutate(record, settings.command.consistencyLevel);
- return true;
- }
-
- @Override
- public int partitionCount()
- {
- return 1;
- }
-
- @Override
- public int rowCount()
- {
- return 1;
- }
- });
- }
-
- protected List<Column> getColumns()
- {
- final ColumnSelection selection = select();
- final List<ByteBuffer> values = getColumnValues(selection);
- final List<Column> columns = new ArrayList<>(values.size());
- final List<ByteBuffer> names = select().select(settings.columns.names);
- for (int i = 0 ; i < values.size() ; i++)
- columns.add(new Column(names.get(i))
- .setValue(values.get(i))
- .setTimestamp(settings.columns.timestamp != null
- ? Long.parseLong(settings.columns.timestamp)
- : FBUtilities.timestampMicros()));
- return columns;
- }
-
-}
diff --git a/tools/stress/src/org/apache/cassandra/stress/operations/predefined/ThriftReader.java b/tools/stress/src/org/apache/cassandra/stress/operations/predefined/ThriftReader.java
deleted file mode 100644
index 4d530b9..0000000
--- a/tools/stress/src/org/apache/cassandra/stress/operations/predefined/ThriftReader.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.stress.operations.predefined;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.List;
-
-import org.apache.cassandra.stress.generate.PartitionGenerator;
-import org.apache.cassandra.stress.generate.SeedManager;
-import org.apache.cassandra.stress.report.Timer;
-import org.apache.cassandra.stress.settings.Command;
-import org.apache.cassandra.stress.settings.StressSettings;
-import org.apache.cassandra.stress.util.ThriftClient;
-import org.apache.cassandra.thrift.ColumnOrSuperColumn;
-import org.apache.cassandra.thrift.ColumnParent;
-
-public final class ThriftReader extends PredefinedOperation
-{
-
- public ThriftReader(Timer timer, PartitionGenerator generator, SeedManager seedManager, StressSettings settings)
- {
- super(Command.READ, timer, generator, seedManager, settings);
- }
-
- public void run(final ThriftClient client) throws IOException
- {
- final ColumnSelection select = select();
- final ByteBuffer key = getKey();
- final List<ByteBuffer> expect = getColumnValues(select);
- timeWithRetry(new RunOp()
- {
- @Override
- public boolean run() throws Exception
- {
- List<ColumnOrSuperColumn> row = client.get_slice(key, new ColumnParent(type.table), select.predicate(), settings.command.consistencyLevel);
- if (expect == null)
- return !row.isEmpty();
- if (row == null)
- return false;
- if (row.size() != expect.size())
- return false;
- for (int i = 0 ; i < row.size() ; i++)
- if (!row.get(i).getColumn().bufferForValue().equals(expect.get(i)))
- return false;
- return true;
- }
-
- @Override
- public int partitionCount()
- {
- return 1;
- }
-
- @Override
- public int rowCount()
- {
- return 1;
- }
- });
- }
-
-}
diff --git a/tools/stress/src/org/apache/cassandra/stress/operations/userdefined/CASQuery.java b/tools/stress/src/org/apache/cassandra/stress/operations/userdefined/CASQuery.java
new file mode 100644
index 0000000..e7d0fe3
--- /dev/null
+++ b/tools/stress/src/org/apache/cassandra/stress/operations/userdefined/CASQuery.java
@@ -0,0 +1,227 @@
+package org.apache.cassandra.stress.operations.userdefined;
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+import com.datastax.driver.core.BoundStatement;
+import com.datastax.driver.core.ColumnDefinitions;
+import com.datastax.driver.core.DataType;
+import com.datastax.driver.core.LocalDate;
+import com.datastax.driver.core.PreparedStatement;
+import com.datastax.driver.core.ResultSet;
+import org.antlr.runtime.RecognitionException;
+import org.apache.cassandra.cql3.CQLFragmentParser;
+import org.apache.cassandra.cql3.CqlParser;
+import org.apache.cassandra.cql3.conditions.ColumnCondition;
+import org.apache.cassandra.cql3.statements.ModificationStatement;
+import org.apache.cassandra.db.ConsistencyLevel;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.stress.generate.DistributionFixed;
+import org.apache.cassandra.stress.generate.PartitionGenerator;
+import org.apache.cassandra.stress.generate.Row;
+import org.apache.cassandra.stress.generate.SeedManager;
+import org.apache.cassandra.stress.generate.values.Generator;
+import org.apache.cassandra.stress.report.Timer;
+import org.apache.cassandra.stress.settings.StressSettings;
+import org.apache.cassandra.stress.util.JavaDriverClient;
+import org.apache.cassandra.utils.Pair;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+
+public class CASQuery extends SchemaStatement
+{
+ private final ImmutableList<Integer> keysIndex;
+ private final ImmutableMap<Integer, Integer> casConditionArgFreqMap;
+ private final String readQuery;
+
+ private PreparedStatement casReadConditionStatement;
+
+ public CASQuery(Timer timer, StressSettings settings, PartitionGenerator generator, SeedManager seedManager, PreparedStatement statement, ConsistencyLevel cl, ArgSelect argSelect, final String tableName)
+ {
+ super(timer, settings, new DataSpec(generator, seedManager, new DistributionFixed(1), settings.insert.rowPopulationRatio.get(), argSelect == SchemaStatement.ArgSelect.MULTIROW ? statement.getVariables().size() : 1), statement,
+ statement.getVariables().asList().stream().map(ColumnDefinitions.Definition::getName).collect(Collectors.toList()), cl);
+
+ if (argSelect != SchemaStatement.ArgSelect.SAMEROW)
+ throw new IllegalArgumentException("CAS is supported only for type 'samerow'");
+
+ ModificationStatement.Parsed modificationStatement;
+ try
+ {
+ modificationStatement = CQLFragmentParser.parseAnyUnhandled(CqlParser::updateStatement,
+ statement.getQueryString());
+ }
+ catch (RecognitionException e)
+ {
+ throw new IllegalArgumentException("could not parse update query:" + statement.getQueryString(), e);
+ }
+
+ final List<Pair<ColumnMetadata.Raw, ColumnCondition.Raw>> casConditionList = modificationStatement.getConditions();
+ List<Integer> casConditionIndex = new ArrayList<>();
+
+ boolean first = true;
+ StringBuilder casReadConditionQuery = new StringBuilder();
+ casReadConditionQuery.append("SELECT ");
+ for (final Pair<ColumnMetadata.Raw, ColumnCondition.Raw> condition : casConditionList)
+ {
+ if (!condition.right.getValue().getText().equals("?"))
+ {
+ //condition uses static value, ignore it
+ continue;
+ }
+ if (!first)
+ {
+ casReadConditionQuery.append(", ");
+ }
+ casReadConditionQuery.append(condition.left.rawText());
+ casConditionIndex.add(getDataSpecification().partitionGenerator.indexOf(condition.left.rawText()));
+ first = false;
+ }
+ casReadConditionQuery.append(" FROM ").append(tableName).append(" WHERE ");
+
+ first = true;
+ ImmutableList.Builder<Integer> keysBuilder = ImmutableList.builder();
+ for (final Generator key : getDataSpecification().partitionGenerator.getPartitionKey())
+ {
+ if (!first)
+ {
+ casReadConditionQuery.append(" AND ");
+ }
+ casReadConditionQuery.append(key.name).append(" = ? ");
+ keysBuilder.add(getDataSpecification().partitionGenerator.indexOf(key.name));
+ first = false;
+ }
+ for (final Generator clusteringKey : getDataSpecification().partitionGenerator.getClusteringComponents())
+ {
+ casReadConditionQuery.append(" AND ").append(clusteringKey.name).append(" = ? ");
+ keysBuilder.add(getDataSpecification().partitionGenerator.indexOf(clusteringKey.name));
+ }
+ keysIndex = keysBuilder.build();
+ readQuery = casReadConditionQuery.toString();
+
+ ImmutableMap.Builder<Integer, Integer> builder = ImmutableMap.builderWithExpectedSize(casConditionIndex.size());
+ for (final Integer oneConditionIndex : casConditionIndex)
+ {
+ builder.put(oneConditionIndex, Math.toIntExact(Arrays.stream(argumentIndex).filter((x) -> x == oneConditionIndex).count()));
+ }
+ casConditionArgFreqMap = builder.build();
+ }
+
+ private class JavaDriverRun extends Runner
+ {
+ final JavaDriverClient client;
+
+ private JavaDriverRun(JavaDriverClient client)
+ {
+ this.client = client;
+ casReadConditionStatement = client.prepare(readQuery);
+ }
+
+ public boolean run()
+ {
+ ResultSet rs = client.getSession().execute(bind(client));
+ rowCount = rs.all().size();
+ partitionCount = Math.min(1, rowCount);
+ return true;
+ }
+ }
+
+ @Override
+ public void run(JavaDriverClient client) throws IOException
+ {
+ timeWithRetry(new JavaDriverRun(client));
+ }
+
+ private BoundStatement bind(JavaDriverClient client)
+ {
+ final Object keys[] = new Object[keysIndex.size()];
+ final Row row = getPartitions().get(0).next();
+
+ for (int i = 0; i < keysIndex.size(); i++)
+ {
+ keys[i] = row.get(keysIndex.get(i));
+ }
+
+ //get current db values for all the coluns which are part of dynamic conditions
+ ResultSet rs = client.getSession().execute(casReadConditionStatement.bind(keys));
+ final Object casDbValues[] = new Object[casConditionArgFreqMap.size()];
+
+ final com.datastax.driver.core.Row casDbValue = rs.one();
+ if (casDbValue != null)
+ {
+ for (int i = 0; i < casConditionArgFreqMap.size(); i++)
+ {
+ casDbValues[i] = casDbValue.getObject(i);
+ }
+ }
+ //now bind db values for dynamic conditions in actual CAS update operation
+ return prepare(row, casDbValues);
+ }
+
+ private BoundStatement prepare(final Row row, final Object[] casDbValues)
+ {
+ final Map<Integer, Integer> localMapping = new HashMap<>(casConditionArgFreqMap);
+ int conditionIndexTracker = 0;
+ for (int i = 0; i < argumentIndex.length; i++)
+ {
+ boolean replace = false;
+ Integer count = localMapping.get(argumentIndex[i]);
+ if (count != null)
+ {
+ count--;
+ localMapping.put(argumentIndex[i], count);
+ if (count == 0)
+ {
+ replace = true;
+ }
+ }
+
+ if (replace)
+ {
+ bindBuffer[i] = casDbValues[conditionIndexTracker++];
+ }
+ else
+ {
+ Object value = row.get(argumentIndex[i]);
+ if (definitions.getType(i).getName() == DataType.date().getName())
+ {
+ // the java driver only accepts com.datastax.driver.core.LocalDate for CQL type "DATE"
+ value = LocalDate.fromDaysSinceEpoch((Integer) value);
+ }
+
+ bindBuffer[i] = value;
+ }
+
+ if (bindBuffer[i] == null && !getDataSpecification().partitionGenerator.permitNulls(argumentIndex[i]))
+ {
+ throw new IllegalStateException();
+ }
+ }
+ return statement.bind(bindBuffer);
+ }
+}
\ No newline at end of file
diff --git a/tools/stress/src/org/apache/cassandra/stress/operations/userdefined/SchemaInsert.java b/tools/stress/src/org/apache/cassandra/stress/operations/userdefined/SchemaInsert.java
index 2c717a1..4cfea82 100644
--- a/tools/stress/src/org/apache/cassandra/stress/operations/userdefined/SchemaInsert.java
+++ b/tools/stress/src/org/apache/cassandra/stress/operations/userdefined/SchemaInsert.java
@@ -38,7 +38,6 @@
import org.apache.cassandra.stress.report.Timer;
import org.apache.cassandra.stress.settings.StressSettings;
import org.apache.cassandra.stress.util.JavaDriverClient;
-import org.apache.cassandra.stress.util.ThriftClient;
public class SchemaInsert extends SchemaStatement
{
@@ -47,9 +46,9 @@
private final String insertStatement;
private final BatchStatement.Type batchType;
- public SchemaInsert(Timer timer, StressSettings settings, PartitionGenerator generator, SeedManager seedManager, Distribution batchSize, RatioDistribution useRatio, RatioDistribution rowPopulation, Integer thriftId, PreparedStatement statement, ConsistencyLevel cl, BatchStatement.Type batchType)
+ public SchemaInsert(Timer timer, StressSettings settings, PartitionGenerator generator, SeedManager seedManager, Distribution batchSize, RatioDistribution useRatio, RatioDistribution rowPopulation, PreparedStatement statement, ConsistencyLevel cl, BatchStatement.Type batchType)
{
- super(timer, settings, new DataSpec(generator, seedManager, batchSize, useRatio, rowPopulation), statement, statement.getVariables().asList().stream().map(d -> d.getName()).collect(Collectors.toList()), thriftId, cl);
+ super(timer, settings, new DataSpec(generator, seedManager, batchSize, useRatio, rowPopulation), statement, statement.getVariables().asList().stream().map(d -> d.getName()).collect(Collectors.toList()), cl);
this.batchType = batchType;
this.insertStatement = null;
this.tableSchema = null;
@@ -58,9 +57,9 @@
/**
* Special constructor for offline use
*/
- public SchemaInsert(Timer timer, StressSettings settings, PartitionGenerator generator, SeedManager seedManager, RatioDistribution useRatio, RatioDistribution rowPopulation, Integer thriftId, String statement, String tableSchema)
+ public SchemaInsert(Timer timer, StressSettings settings, PartitionGenerator generator, SeedManager seedManager, RatioDistribution useRatio, RatioDistribution rowPopulation, String statement, String tableSchema)
{
- super(timer, settings, new DataSpec(generator, seedManager, new DistributionFixed(1), useRatio, rowPopulation), null, generator.getColumnNames(), thriftId, ConsistencyLevel.ONE);
+ super(timer, settings, new DataSpec(generator, seedManager, new DistributionFixed(1), useRatio, rowPopulation), null, generator.getColumnNames(), ConsistencyLevel.ONE);
this.batchType = BatchStatement.Type.UNLOGGED;
this.insertStatement = statement;
this.tableSchema = tableSchema;
@@ -98,7 +97,10 @@
else
{
BatchStatement batch = new BatchStatement(batchType);
- batch.setConsistencyLevel(JavaDriverClient.from(cl));
+ if (cl.isSerialConsistency())
+ batch.setSerialConsistencyLevel(JavaDriverClient.from(cl));
+ else
+ batch.setConsistencyLevel(JavaDriverClient.from(cl));
batch.addAll(substmts);
stmt = batch;
}
@@ -109,29 +111,6 @@
}
}
- private class ThriftRun extends Runner
- {
- final ThriftClient client;
-
- private ThriftRun(ThriftClient client)
- {
- this.client = client;
- }
-
- public boolean run() throws Exception
- {
- for (PartitionIterator iterator : partitions)
- {
- while (iterator.hasNext())
- {
- client.execute_prepared_cql3_query(thriftId, iterator.getToken(), thriftRowArgs(iterator.next()), settings.command.consistencyLevel);
- rowCount += 1;
- }
- }
- return true;
- }
- }
-
private class OfflineRun extends Runner
{
final StressCQLSSTableWriter writer;
@@ -148,7 +127,7 @@
while (iterator.hasNext())
{
Row row = iterator.next();
- writer.rawAddRow(thriftRowArgs(row));
+ writer.rawAddRow(rowArgs(row));
rowCount += 1;
}
}
@@ -168,12 +147,6 @@
return true;
}
- @Override
- public void run(ThriftClient client) throws IOException
- {
- timeWithRetry(new ThriftRun(client));
- }
-
public StressCQLSSTableWriter createWriter(ColumnFamilyStore cfs, int bufferSize, boolean makeRangeAware)
{
return StressCQLSSTableWriter.builder()
diff --git a/tools/stress/src/org/apache/cassandra/stress/operations/userdefined/SchemaQuery.java b/tools/stress/src/org/apache/cassandra/stress/operations/userdefined/SchemaQuery.java
index 2764704..cba9ce4 100644
--- a/tools/stress/src/org/apache/cassandra/stress/operations/userdefined/SchemaQuery.java
+++ b/tools/stress/src/org/apache/cassandra/stress/operations/userdefined/SchemaQuery.java
@@ -22,9 +22,6 @@
import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.List;
import java.util.Random;
import java.util.stream.Collectors;
@@ -36,26 +33,17 @@
import org.apache.cassandra.stress.report.Timer;
import org.apache.cassandra.stress.settings.StressSettings;
import org.apache.cassandra.stress.util.JavaDriverClient;
-import org.apache.cassandra.stress.util.ThriftClient;
-import org.apache.cassandra.thrift.CqlResult;
-import org.apache.cassandra.thrift.ThriftConversion;
public class SchemaQuery extends SchemaStatement
{
- public static enum ArgSelect
- {
- MULTIROW, SAMEROW;
- //TODO: FIRSTROW, LASTROW
- }
-
final ArgSelect argSelect;
final Object[][] randomBuffer;
final Random random = new Random();
- public SchemaQuery(Timer timer, StressSettings settings, PartitionGenerator generator, SeedManager seedManager, Integer thriftId, PreparedStatement statement, ConsistencyLevel cl, ArgSelect argSelect)
+ public SchemaQuery(Timer timer, StressSettings settings, PartitionGenerator generator, SeedManager seedManager, PreparedStatement statement, ConsistencyLevel cl, ArgSelect argSelect)
{
- super(timer, settings, new DataSpec(generator, seedManager, new DistributionFixed(1), settings.insert.rowPopulationRatio.get(), argSelect == ArgSelect.MULTIROW ? statement.getVariables().size() : 1), statement,
- statement.getVariables().asList().stream().map(d -> d.getName()).collect(Collectors.toList()), thriftId, cl);
+ super(timer, settings, new DataSpec(generator, seedManager, new DistributionFixed(1), settings.insert.rowPopulationRatio.get(), argSelect == SchemaStatement.ArgSelect.MULTIROW ? statement.getVariables().size() : 1), statement,
+ statement.getVariables().asList().stream().map(d -> d.getName()).collect(Collectors.toList()), cl);
this.argSelect = argSelect;
randomBuffer = new Object[argumentIndex.length][argumentIndex.length];
}
@@ -78,24 +66,6 @@
}
}
- private class ThriftRun extends Runner
- {
- final ThriftClient client;
-
- private ThriftRun(ThriftClient client)
- {
- this.client = client;
- }
-
- public boolean run() throws Exception
- {
- CqlResult rs = client.execute_prepared_cql3_query(thriftId, partitions.get(0).getToken(), thriftArgs(), ThriftConversion.toThrift(cl));
- rowCount = rs.getRowsSize();
- partitionCount = Math.min(1, rowCount);
- return true;
- }
- }
-
private int fillRandom()
{
int c = 0;
@@ -132,36 +102,9 @@
}
}
- List<ByteBuffer> thriftArgs()
- {
- switch (argSelect)
- {
- case MULTIROW:
- List<ByteBuffer> args = new ArrayList<>();
- int c = fillRandom();
- for (int i = 0 ; i < argumentIndex.length ; i++)
- {
- int argIndex = argumentIndex[i];
- args.add(spec.partitionGenerator.convert(argIndex, randomBuffer[argIndex < 0 ? 0 : random.nextInt(c)][i]));
- }
- return args;
- case SAMEROW:
- return thriftRowArgs(partitions.get(0).next());
- default:
- throw new IllegalStateException();
- }
- }
-
@Override
public void run(JavaDriverClient client) throws IOException
{
timeWithRetry(new JavaDriverRun(client));
}
-
- @Override
- public void run(ThriftClient client) throws IOException
- {
- timeWithRetry(new ThriftRun(client));
- }
-
}
diff --git a/tools/stress/src/org/apache/cassandra/stress/operations/userdefined/SchemaStatement.java b/tools/stress/src/org/apache/cassandra/stress/operations/userdefined/SchemaStatement.java
index ca1f5fa..334e6c5 100644
--- a/tools/stress/src/org/apache/cassandra/stress/operations/userdefined/SchemaStatement.java
+++ b/tools/stress/src/org/apache/cassandra/stress/operations/userdefined/SchemaStatement.java
@@ -38,19 +38,23 @@
public abstract class SchemaStatement extends PartitionOperation
{
+ public enum ArgSelect
+ {
+ MULTIROW, SAMEROW;
+ //TODO: FIRSTROW, LASTROW
+ }
+
final PreparedStatement statement;
- final Integer thriftId;
final ConsistencyLevel cl;
final int[] argumentIndex;
final Object[] bindBuffer;
final ColumnDefinitions definitions;
public SchemaStatement(Timer timer, StressSettings settings, DataSpec spec,
- PreparedStatement statement, List<String> bindNames, Integer thriftId, ConsistencyLevel cl)
+ PreparedStatement statement, List<String> bindNames, ConsistencyLevel cl)
{
super(timer, settings, spec);
this.statement = statement;
- this.thriftId = thriftId;
this.cl = cl;
argumentIndex = new int[bindNames.size()];
bindBuffer = new Object[argumentIndex.length];
@@ -60,7 +64,12 @@
argumentIndex[i++] = spec.partitionGenerator.indexOf(name);
if (statement != null)
- statement.setConsistencyLevel(JavaDriverClient.from(cl));
+ {
+ if (cl.isSerialConsistency())
+ statement.setSerialConsistencyLevel(JavaDriverClient.from(cl));
+ else
+ statement.setConsistencyLevel(JavaDriverClient.from(cl));
+ }
}
BoundStatement bindRow(Row row)
@@ -82,11 +91,12 @@
return statement.bind(bindBuffer);
}
- List<ByteBuffer> thriftRowArgs(Row row)
+ List<ByteBuffer> rowArgs(Row row)
{
List<ByteBuffer> args = new ArrayList<>();
for (int i : argumentIndex)
- args.add(spec.partitionGenerator.convert(i, row.get(i)));
+ args.add(spec.partitionGenerator.convert(i,
+ row.get(i)));
return args;
}
diff --git a/tools/stress/src/org/apache/cassandra/stress/operations/userdefined/TokenRangeQuery.java b/tools/stress/src/org/apache/cassandra/stress/operations/userdefined/TokenRangeQuery.java
index ff8b27f..fe5f129 100644
--- a/tools/stress/src/org/apache/cassandra/stress/operations/userdefined/TokenRangeQuery.java
+++ b/tools/stress/src/org/apache/cassandra/stress/operations/userdefined/TokenRangeQuery.java
@@ -43,7 +43,6 @@
import org.apache.cassandra.stress.report.Timer;
import org.apache.cassandra.stress.settings.StressSettings;
import org.apache.cassandra.stress.util.JavaDriverClient;
-import org.apache.cassandra.stress.util.ThriftClient;
public class TokenRangeQuery extends Operation
{
@@ -219,34 +218,12 @@
return ret.toString();
}
- private static class ThriftRun extends Runner
- {
- final ThriftClient client;
-
- private ThriftRun(ThriftClient client)
- {
- this.client = client;
- }
-
- public boolean run() throws Exception
- {
- throw new OperationNotSupportedException("Bulk read over thrift not supported");
- }
- }
-
-
@Override
public void run(JavaDriverClient client) throws IOException
{
timeWithRetry(new JavaDriverRun(client));
}
- @Override
- public void run(ThriftClient client) throws IOException
- {
- timeWithRetry(new ThriftRun(client));
- }
-
public int ready(WorkManager workManager)
{
tokenRangeIterator.update();
diff --git a/tools/stress/src/org/apache/cassandra/stress/operations/userdefined/ValidatingSchemaQuery.java b/tools/stress/src/org/apache/cassandra/stress/operations/userdefined/ValidatingSchemaQuery.java
index a731b99..6d93f4c 100644
--- a/tools/stress/src/org/apache/cassandra/stress/operations/userdefined/ValidatingSchemaQuery.java
+++ b/tools/stress/src/org/apache/cassandra/stress/operations/userdefined/ValidatingSchemaQuery.java
@@ -36,13 +36,7 @@
import org.apache.cassandra.stress.report.Timer;
import org.apache.cassandra.stress.settings.StressSettings;
import org.apache.cassandra.stress.util.JavaDriverClient;
-import org.apache.cassandra.stress.util.ThriftClient;
-import org.apache.cassandra.thrift.Compression;
-import org.apache.cassandra.thrift.CqlResult;
-import org.apache.cassandra.thrift.CqlRow;
-import org.apache.cassandra.thrift.ThriftConversion;
import org.apache.cassandra.utils.Pair;
-import org.apache.thrift.TException;
public class ValidatingSchemaQuery extends PartitionOperation
{
@@ -66,7 +60,12 @@
argumentIndex[i++] = spec.partitionGenerator.indexOf(definition.getName());
for (ValidatingStatement statement : statements)
- statement.statement.setConsistencyLevel(JavaDriverClient.from(cl));
+ {
+ if (cl.isSerialConsistency())
+ statement.statement.setSerialConsistencyLevel(JavaDriverClient.from(cl));
+ else
+ statement.statement.setConsistencyLevel(JavaDriverClient.from(cl));
+ }
this.clusteringComponents = clusteringComponents;
}
@@ -150,50 +149,6 @@
}
}
- private class ThriftRun extends Runner
- {
- final ThriftClient client;
-
- private ThriftRun(ThriftClient client, PartitionIterator iter)
- {
- super(iter);
- this.client = client;
- }
-
- public boolean run() throws Exception
- {
- CqlResult rs = client.execute_prepared_cql3_query(statements[statementIndex].thriftId, partitions.get(0).getToken(), thriftArgs(), ThriftConversion.toThrift(cl));
- int[] valueIndex = new int[rs.getSchema().name_types.size()];
- for (int i = 0 ; i < valueIndex.length ; i++)
- valueIndex[i] = spec.partitionGenerator.indexOf(rs.fieldForId(i).getFieldName());
- int r = 0;
- if (!statements[statementIndex].inclusiveStart && iter.hasNext())
- iter.next();
- while (iter.hasNext())
- {
- Row expectedRow = iter.next();
- if (!statements[statementIndex].inclusiveEnd && !iter.hasNext())
- break;
-
- if (r == rs.num)
- return false;
-
- rowCount++;
- CqlRow actualRow = rs.getRows().get(r++);
- for (int i = 0 ; i < actualRow.getColumnsSize() ; i++)
- {
- ByteBuffer expectedValue = spec.partitionGenerator.convert(valueIndex[i], expectedRow.get(valueIndex[i]));
- ByteBuffer actualValue = actualRow.getColumns().get(i).value;
- if (!expectedValue.equals(actualValue))
- return false;
- }
- }
- assert r == rs.num;
- partitionCount = Math.min(1, rowCount);
- return true;
- }
- }
-
BoundStatement bind(int statementIndex)
{
int pkc = bounds.left.partitionKey.length;
@@ -204,32 +159,12 @@
return statements[statementIndex].statement.bind(bindBuffer);
}
- List<ByteBuffer> thriftArgs()
- {
- List<ByteBuffer> args = new ArrayList<>();
- int pkc = bounds.left.partitionKey.length;
- for (int i = 0 ; i < pkc ; i++)
- args.add(spec.partitionGenerator.convert(-i, bounds.left.partitionKey[i]));
- int ccc = bounds.left.row.length;
- for (int i = 0 ; i < ccc ; i++)
- args.add(spec.partitionGenerator.convert(i, bounds.left.get(i)));
- for (int i = 0 ; i < ccc ; i++)
- args.add(spec.partitionGenerator.convert(i, bounds.right.get(i)));
- return args;
- }
-
@Override
public void run(JavaDriverClient client) throws IOException
{
timeWithRetry(new JavaDriverRun(client, partitions.get(0)));
}
- @Override
- public void run(ThriftClient client) throws IOException
- {
- timeWithRetry(new ThriftRun(client, partitions.get(0)));
- }
-
public static class Factory
{
final ValidatingStatement[] statements;
@@ -310,13 +245,11 @@
private static class ValidatingStatement
{
final PreparedStatement statement;
- final Integer thriftId;
final boolean inclusiveStart;
final boolean inclusiveEnd;
- private ValidatingStatement(PreparedStatement statement, Integer thriftId, boolean inclusiveStart, boolean inclusiveEnd)
+ private ValidatingStatement(PreparedStatement statement, boolean inclusiveStart, boolean inclusiveEnd)
{
this.statement = statement;
- this.thriftId = thriftId;
this.inclusiveStart = inclusiveStart;
this.inclusiveEnd = inclusiveEnd;
}
@@ -325,16 +258,7 @@
private static ValidatingStatement prepare(StressSettings settings, String cql, boolean incLb, boolean incUb)
{
JavaDriverClient jclient = settings.getJavaDriverClient();
- ThriftClient tclient = settings.getThriftClient();
PreparedStatement statement = jclient.prepare(cql);
- try
- {
- Integer thriftId = tclient.prepare_cql3_query(cql, Compression.NONE);
- return new ValidatingStatement(statement, thriftId, incLb, incUb);
- }
- catch (TException e)
- {
- throw new RuntimeException(e);
- }
+ return new ValidatingStatement(statement, incLb, incUb);
}
}
diff --git a/tools/stress/src/org/apache/cassandra/stress/report/StressMetrics.java b/tools/stress/src/org/apache/cassandra/stress/report/StressMetrics.java
index 52e90e2..a4058f2 100644
--- a/tools/stress/src/org/apache/cassandra/stress/report/StressMetrics.java
+++ b/tools/stress/src/org/apache/cassandra/stress/report/StressMetrics.java
@@ -339,9 +339,9 @@
// PRINT FORMATTING
+ public static final String HEADFORMAT = "%-50s%10s,%8s,%8s,%8s,%8s,%8s,%8s,%8s,%8s,%8s,%7s,%9s,%7s,%7s,%8s,%8s,%8s,%8s";
+ public static final String ROWFORMAT = "%-50s%10d,%8.0f,%8.0f,%8.0f,%8.1f,%8.1f,%8.1f,%8.1f,%8.1f,%8.1f,%7.1f,%9.5f,%7d,%7.0f,%8.0f,%8.0f,%8.0f,%8.0f";
- public static final String HEADFORMAT = "%-10s%10s,%8s,%8s,%8s,%8s,%8s,%8s,%8s,%8s,%8s,%7s,%9s,%7s,%7s,%8s,%8s,%8s,%8s";
- public static final String ROWFORMAT = "%-10s%10d,%8.0f,%8.0f,%8.0f,%8.1f,%8.1f,%8.1f,%8.1f,%8.1f,%8.1f,%7.1f,%9.5f,%7d,%7.0f,%8.0f,%8.0f,%8.0f,%8.0f";
public static final String[] HEADMETRICS = new String[]{"type", "total ops","op/s","pk/s","row/s","mean","med",".95",".99",".999","max","time","stderr", "errors", "gc: #", "max ms", "sum ms", "sdv ms", "mb"};
public static final String HEAD = String.format(HEADFORMAT, (Object[]) HEADMETRICS);
diff --git a/tools/stress/src/org/apache/cassandra/stress/settings/CliOption.java b/tools/stress/src/org/apache/cassandra/stress/settings/CliOption.java
index 36284ab..018669a 100644
--- a/tools/stress/src/org/apache/cassandra/stress/settings/CliOption.java
+++ b/tools/stress/src/org/apache/cassandra/stress/settings/CliOption.java
@@ -30,7 +30,7 @@
INSERT("Insert specific options relating to various methods for batching and splitting partition updates", SettingsInsert.helpPrinter()),
COL("Column details such as size and count distribution, data generator, names, comparator and if super columns should be used", SettingsColumn.helpPrinter()),
RATE("Thread count, rate limit or automatic mode (default is auto)", SettingsRate.helpPrinter()),
- MODE("Thrift or CQL with options", SettingsMode.helpPrinter()),
+ MODE("CQL mode options", SettingsMode.helpPrinter()),
ERRORS("How to handle errors when encountered during stress", SettingsErrors.helpPrinter()),
SCHEMA("Replication settings, compression, compaction, etc.", SettingsSchema.helpPrinter()),
NODE("Nodes to connect to", SettingsNode.helpPrinter()),
diff --git a/tools/stress/src/org/apache/cassandra/stress/settings/ConnectionAPI.java b/tools/stress/src/org/apache/cassandra/stress/settings/ConnectionAPI.java
index 942250f..554c16b 100644
--- a/tools/stress/src/org/apache/cassandra/stress/settings/ConnectionAPI.java
+++ b/tools/stress/src/org/apache/cassandra/stress/settings/ConnectionAPI.java
@@ -23,6 +23,6 @@
public enum ConnectionAPI
{
- THRIFT, THRIFT_SMART, SIMPLE_NATIVE, JAVA_DRIVER_NATIVE
+ SIMPLE_NATIVE, JAVA_DRIVER_NATIVE
}
diff --git a/tools/stress/src/org/apache/cassandra/stress/settings/ConnectionStyle.java b/tools/stress/src/org/apache/cassandra/stress/settings/ConnectionStyle.java
index 6b408a9..1884cc8 100644
--- a/tools/stress/src/org/apache/cassandra/stress/settings/ConnectionStyle.java
+++ b/tools/stress/src/org/apache/cassandra/stress/settings/ConnectionStyle.java
@@ -1,6 +1,4 @@
-package org.apache.cassandra.stress.settings;
/*
- *
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -8,23 +6,21 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
- *
*/
-
+package org.apache.cassandra.stress.settings;
public enum ConnectionStyle
{
CQL,
- CQL_PREPARED,
- THRIFT
+ CQL_PREPARED
}
diff --git a/tools/stress/src/org/apache/cassandra/stress/settings/Legacy.java b/tools/stress/src/org/apache/cassandra/stress/settings/Legacy.java
index 70693af..ba94e3f 100644
--- a/tools/stress/src/org/apache/cassandra/stress/settings/Legacy.java
+++ b/tools/stress/src/org/apache/cassandra/stress/settings/Legacy.java
@@ -51,7 +51,6 @@
availableOptions.addOption("s", "stdev", true, "Standard Deviation for gaussian read key generation, default:0.1");
availableOptions.addOption("r", "random", false, "Use random key generator for read key generation (STDEV will have no effect), default:false");
availableOptions.addOption("f", "file", true, "Write output to given file");
- availableOptions.addOption("p", "port", true, "Thrift port, default:9160");
availableOptions.addOption("o", "operation", true, "Operation to perform (WRITE, READ, READWRITE, RANGE_SLICE, INDEXED_RANGE_SLICE, MULTI_GET, COUNTERWRITE, COUNTER_GET), default:WRITE");
availableOptions.addOption("u", "supercolumns", true, "Number of super columns per key, default:1");
availableOptions.addOption("y", "family-type", true, "Column Family Type (Super, Standard), default:Standard");
@@ -60,8 +59,6 @@
availableOptions.addOption("i", "progress-interval", true, "Progress Report Interval (seconds), default:10");
availableOptions.addOption("g", "keys-per-call", true, "Number of keys to get_range_slices or multiget per call, default:1000");
availableOptions.addOption("l", "replication-factor", true, "Replication Factor to use when creating needed column families, default:1");
- availableOptions.addOption("L3", "enable-cql3", false, "Perform queries using CQL3 (Cassandra Query Language v 3.0.0)");
- availableOptions.addOption("b", "enable-native-protocol", false, "Use the binary native protocol (only work along with -L3)");
availableOptions.addOption("P", "use-prepared-statements", false, "Perform queries using prepared statements (only applicable to CQL).");
availableOptions.addOption("e", "consistency-level", true, "Consistency Level to use (ONE, QUORUM, LOCAL_QUORUM, EACH_QUORUM, ALL, ANY), default:ONE");
availableOptions.addOption("x", "create-index", true, "Type of index to create on needed column families (KEYS)");
@@ -73,12 +70,11 @@
availableOptions.addOption("Q", "query-names", true, "Comma-separated list of column names to retrieve from each row.");
availableOptions.addOption("Z", "compaction-strategy", true, "CompactionStrategy to use.");
availableOptions.addOption("U", "comparator", true, "Column Comparator to use. Currently supported types are: TimeUUIDType, AsciiType, UTF8Type.");
- availableOptions.addOption("tf", "transport-factory", true, "Fully-qualified TTransportFactory class name for creating a connection. Note: For Thrift over SSL, use org.apache.cassandra.stress.SSLTransportFactory.");
availableOptions.addOption("ns", "no-statistics", false, "Turn off the aggegate statistics that is normally output after completion.");
availableOptions.addOption("ts", SSL_TRUSTSTORE, true, "SSL: full path to truststore");
availableOptions.addOption("tspw", SSL_TRUSTSTORE_PW, true, "SSL: full path to truststore");
availableOptions.addOption("prtcl", SSL_PROTOCOL, true, "SSL: connections protocol to use (default: TLS)");
- availableOptions.addOption("alg", SSL_ALGORITHM, true, "SSL: algorithm (default: SunX509)");
+ availableOptions.addOption("alg", SSL_ALGORITHM, true, "SSL: algorithm");
availableOptions.addOption("st", SSL_STORE_TYPE, true, "SSL: type of store");
availableOptions.addOption("ciphers", SSL_CIPHER_SUITES, true, "SSL: comma-separated list of encryption suites to use");
availableOptions.addOption("th", "throttle", true, "Throttle the total number of operations per second to a maximum amount.");
@@ -231,10 +227,7 @@
r.add("-schema", "replication(" + rep + ")");
}
- if (cmd.hasOption("L3"))
- r.add("-mode", (cmd.hasOption("P") ? "prepared" : "") + (cmd.hasOption("b") ? "native" : "") + "cql3");
- else
- r.add("-mode", "thrift");
+ r.add("-mode", (cmd.hasOption("P") ? "prepared" : "") + "native" + "cql3");
if (cmd.hasOption("I"))
r.add("-schema", "compression=" + cmd.getOptionValue("I"));
@@ -255,9 +248,6 @@
if (cmd.hasOption("ns"))
r.add("-log", "no-summary");
- if (cmd.hasOption("tf"))
- r.add("-transport", "factory=" + cmd.getOptionValue("tf"));
-
if(cmd.hasOption(SSL_TRUSTSTORE))
r.add("-transport", "truststore=" + cmd.getOptionValue(SSL_TRUSTSTORE));
diff --git a/tools/stress/src/org/apache/cassandra/stress/settings/OptionCompaction.java b/tools/stress/src/org/apache/cassandra/stress/settings/OptionCompaction.java
index 11d5403..c90a14c 100644
--- a/tools/stress/src/org/apache/cassandra/stress/settings/OptionCompaction.java
+++ b/tools/stress/src/org/apache/cassandra/stress/settings/OptionCompaction.java
@@ -23,8 +23,8 @@
import com.google.common.base.Function;
-import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.schema.CompactionParams;
/**
* For specifying replication options
@@ -67,8 +67,9 @@
{
try
{
- CFMetaData.createCompactionStrategy(name);
- } catch (ConfigurationException e)
+ CompactionParams.classFromName(name);
+ }
+ catch (ConfigurationException e)
{
throw new IllegalArgumentException("Invalid compaction strategy: " + name);
}
diff --git a/tools/stress/src/org/apache/cassandra/stress/settings/SettingsColumn.java b/tools/stress/src/org/apache/cassandra/stress/settings/SettingsColumn.java
index 79d8d25..0ba2212 100644
--- a/tools/stress/src/org/apache/cassandra/stress/settings/SettingsColumn.java
+++ b/tools/stress/src/org/apache/cassandra/stress/settings/SettingsColumn.java
@@ -41,7 +41,6 @@
*/
public class SettingsColumn implements Serializable
{
-
public final int maxColumnsPerKey;
public transient List<ByteBuffer> names;
public final List<String> namestrs;
diff --git a/tools/stress/src/org/apache/cassandra/stress/settings/SettingsCommand.java b/tools/stress/src/org/apache/cassandra/stress/settings/SettingsCommand.java
index c3a171e..314774a 100644
--- a/tools/stress/src/org/apache/cassandra/stress/settings/SettingsCommand.java
+++ b/tools/stress/src/org/apache/cassandra/stress/settings/SettingsCommand.java
@@ -32,7 +32,7 @@
import org.apache.cassandra.stress.operations.OpDistributionFactory;
import org.apache.cassandra.stress.util.JavaDriverClient;
import org.apache.cassandra.stress.util.ResultLogger;
-import org.apache.cassandra.thrift.ConsistencyLevel;
+import org.apache.cassandra.db.ConsistencyLevel;
// Generic command settings - common to read/write/etc
public abstract class SettingsCommand implements Serializable
@@ -96,6 +96,9 @@
case 'h':
this.durationUnits = TimeUnit.HOURS;
break;
+ case 'd':
+ this.durationUnits = TimeUnit.DAYS;
+ break;
default:
throw new IllegalStateException();
}
@@ -120,7 +123,7 @@
{
final OptionSimple noWarmup = new OptionSimple("no-warmup", "", null, "Do not warmup the process", false);
final OptionSimple truncate = new OptionSimple("truncate=", "never|once|always", "never", "Truncate the table: never, before performing any work, or before each iteration", false);
- final OptionSimple consistencyLevel = new OptionSimple("cl=", "ONE|QUORUM|LOCAL_QUORUM|EACH_QUORUM|ALL|ANY|TWO|THREE|LOCAL_ONE", "LOCAL_ONE", "Consistency level to use", false);
+ final OptionSimple consistencyLevel = new OptionSimple("cl=", "ONE|QUORUM|LOCAL_QUORUM|EACH_QUORUM|ALL|ANY|TWO|THREE|LOCAL_ONE|SERIAL|LOCAL_SERIAL", "LOCAL_ONE", "Consistency level to use", false);
}
static class Count extends Options
@@ -135,7 +138,7 @@
static class Duration extends Options
{
- final OptionSimple duration = new OptionSimple("duration=", "[0-9]+[smh]", null, "Time to run in (in seconds, minutes or hours)", true);
+ final OptionSimple duration = new OptionSimple("duration=", "[0-9]+[smhd]", null, "Time to run in (in seconds, minutes, hours or days)", true);
@Override
public List<? extends Option> options()
{
diff --git a/tools/stress/src/org/apache/cassandra/stress/settings/SettingsCommandPreDefinedMixed.java b/tools/stress/src/org/apache/cassandra/stress/settings/SettingsCommandPreDefinedMixed.java
index 72f6c86..1df2a06 100644
--- a/tools/stress/src/org/apache/cassandra/stress/settings/SettingsCommandPreDefinedMixed.java
+++ b/tools/stress/src/org/apache/cassandra/stress/settings/SettingsCommandPreDefinedMixed.java
@@ -58,14 +58,9 @@
final SeedManager seeds = new SeedManager(settings);
return new SampledOpDistributionFactory<Command>(ratios, clustering)
{
- protected List<? extends Operation> get(Timer timer, PartitionGenerator generator, Command key, boolean isWarmup)
+ protected List<? extends Operation> get(Timer timer, Command key, boolean isWarmup)
{
- return Collections.singletonList(PredefinedOperation.operation(key, timer, generator, seeds, settings, add));
- }
-
- protected PartitionGenerator newGenerator()
- {
- return SettingsCommandPreDefinedMixed.this.newGenerator(settings);
+ return Collections.singletonList(PredefinedOperation.operation(key, timer, SettingsCommandPreDefinedMixed.this.newGenerator(settings), seeds, settings, add));
}
};
}
diff --git a/tools/stress/src/org/apache/cassandra/stress/settings/SettingsCommandUser.java b/tools/stress/src/org/apache/cassandra/stress/settings/SettingsCommandUser.java
index 4c7ad91..fd34d74 100644
--- a/tools/stress/src/org/apache/cassandra/stress/settings/SettingsCommandUser.java
+++ b/tools/stress/src/org/apache/cassandra/stress/settings/SettingsCommandUser.java
@@ -27,6 +27,10 @@
import java.util.Collections;
import java.util.List;
import java.util.Map;
+import java.util.LinkedHashMap;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
import org.apache.cassandra.stress.Operation;
import org.apache.cassandra.stress.StressProfile;
@@ -46,8 +50,11 @@
// Ratios for selecting commands - index for each Command, NaN indicates the command is not requested
private final Map<String, Double> ratios;
private final DistributionFactory clustering;
- public final StressProfile profile;
+ public final Map<String, StressProfile> profiles;
private final Options options;
+ private String default_profile_name;
+ private static final Pattern EXTRACT_SPEC_CMD = Pattern.compile("(.+)\\.(.+)");
+
public SettingsCommandUser(Options options)
{
@@ -56,10 +63,26 @@
this.options = options;
clustering = options.clustering.get();
ratios = options.ops.ratios();
+ default_profile_name=null;
+
String yamlPath = options.profile.value();
- File yamlFile = new File(yamlPath);
- profile = StressProfile.load(yamlFile.exists() ? yamlFile.toURI() : URI.create(yamlPath));
+ profiles = new LinkedHashMap<>();
+
+ String[] yamlPaths = yamlPath.split(",");
+ for (String curYamlPath : yamlPaths)
+ {
+ File yamlFile = new File(curYamlPath);
+ StressProfile profile = StressProfile.load(yamlFile.exists() ? yamlFile.toURI() : URI.create(curYamlPath));
+ String specName = profile.specName;
+ if (default_profile_name == null) {default_profile_name=specName;} //first file is default
+ if (profiles.containsKey(specName))
+ {
+ throw new IllegalArgumentException("Must only specify a singe YAML file per table (including keyspace qualifier).");
+ }
+ profiles.put(specName, profile);
+ }
+
if (ratios.size() == 0)
throw new IllegalArgumentException("Must specify at least one command with a non-zero ratio");
@@ -73,36 +96,54 @@
public OpDistributionFactory getFactory(final StressSettings settings)
{
final SeedManager seeds = new SeedManager(settings);
- final TokenRangeIterator tokenRangeIterator = profile.tokenRangeQueries.isEmpty()
- ? null
- : new TokenRangeIterator(settings,
- profile.maybeLoadTokenRanges(settings));
+
+ final Map<String, TokenRangeIterator> tokenRangeIterators = new LinkedHashMap<>();
+ profiles.forEach((k,v)->tokenRangeIterators.put(k, (v.tokenRangeQueries.isEmpty()
+ ? null
+ : new TokenRangeIterator(settings,
+ v.maybeLoadTokenRanges(settings)))));
return new SampledOpDistributionFactory<String>(ratios, clustering)
{
- protected List<? extends Operation> get(Timer timer, PartitionGenerator generator, String key, boolean isWarmup)
+ protected List<? extends Operation> get(Timer timer, String key, boolean isWarmup)
{
- if (key.equalsIgnoreCase("insert"))
+ Matcher m = EXTRACT_SPEC_CMD.matcher(key);
+ final String profile_name;
+ final String sub_key;
+ if (m.matches())
+ {
+ profile_name = m.group(1);
+ sub_key = m.group(2);
+ }
+ else
+ {
+ profile_name = default_profile_name;
+ sub_key = key;
+ }
+
+ if (!profiles.containsKey(profile_name))
+ {
+ throw new IllegalArgumentException(String.format("Op name %s contains an invalid profile specname: %s", key, profile_name));
+ }
+ StressProfile profile = profiles.get(profile_name);
+ TokenRangeIterator tokenRangeIterator = tokenRangeIterators.get(profile_name);
+ PartitionGenerator generator = profile.newGenerator(settings);
+ if (sub_key.equalsIgnoreCase("insert"))
return Collections.singletonList(profile.getInsert(timer, generator, seeds, settings));
- if (key.equalsIgnoreCase("validate"))
+ if (sub_key.equalsIgnoreCase("validate"))
return profile.getValidate(timer, generator, seeds, settings);
- if (profile.tokenRangeQueries.containsKey(key))
- return Collections.singletonList(profile.getBulkReadQueries(key, timer, settings, tokenRangeIterator, isWarmup));
+ if (profile.tokenRangeQueries.containsKey(sub_key))
+ return Collections.singletonList(profile.getBulkReadQueries(sub_key, timer, settings, tokenRangeIterator, isWarmup));
- return Collections.singletonList(profile.getQuery(key, timer, generator, seeds, settings, isWarmup));
- }
-
- protected PartitionGenerator newGenerator()
- {
- return profile.newGenerator(settings);
+ return Collections.singletonList(profile.getQuery(sub_key, timer, generator, seeds, settings, isWarmup));
}
};
}
public void truncateTables(StressSettings settings)
{
- profile.truncateTable(settings);
+ profiles.forEach((k,v)-> v.truncateTable(settings));
}
static final class Options extends GroupedOptions
@@ -113,8 +154,8 @@
this.parent = parent;
}
final OptionDistribution clustering = new OptionDistribution("clustering=", "gaussian(1..10)", "Distribution clustering runs of operations of the same kind");
- final OptionSimple profile = new OptionSimple("profile=", ".*", null, "Specify the path to a yaml cql3 profile", true);
- final OptionAnyProbabilities ops = new OptionAnyProbabilities("ops", "Specify the ratios for inserts/queries to perform; e.g. ops(insert=2,<query1>=1) will perform 2 inserts for each query1");
+ final OptionSimple profile = new OptionSimple("profile=", ".*", null, "Specify the path to a yaml cql3 profile. Multiple comma separated files can be added.", true);
+ final OptionAnyProbabilities ops = new OptionAnyProbabilities("ops", "Specify the ratios for inserts/queries to perform; e.g. ops(insert=2,<query1>=1) will perform 2 inserts for each query1. When using multiple files, specify as keyspace.table.op.");
@Override
public List<? extends Option> options()
@@ -130,8 +171,7 @@
super.printSettings(out);
out.printf(" Command Ratios: %s%n", ratios);
out.printf(" Command Clustering Distribution: %s%n", options.clustering.getOptionAsString());
- out.printf(" Profile File: %s%n", options.profile.value());
- // profile.noSettings(out);
+ out.printf(" Profile File(s): %s%n", options.profile.value());
}
diff --git a/tools/stress/src/org/apache/cassandra/stress/settings/SettingsGraph.java b/tools/stress/src/org/apache/cassandra/stress/settings/SettingsGraph.java
index 90bb99a..d040ccc 100644
--- a/tools/stress/src/org/apache/cassandra/stress/settings/SettingsGraph.java
+++ b/tools/stress/src/org/apache/cassandra/stress/settings/SettingsGraph.java
@@ -30,6 +30,7 @@
import java.util.List;
import java.util.Map;
+import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.stress.util.ResultLogger;
public class SettingsGraph implements Serializable
@@ -54,14 +55,7 @@
if (inGraphMode())
{
- try
- {
- temporaryLogFile = File.createTempFile("cassandra-stress", ".log");
- }
- catch (IOException e)
- {
- throw new RuntimeException("Cannot open temporary file");
- }
+ temporaryLogFile = FileUtils.createTempFile("cassandra-stress", ".log");
}
else
{
diff --git a/tools/stress/src/org/apache/cassandra/stress/settings/SettingsMode.java b/tools/stress/src/org/apache/cassandra/stress/settings/SettingsMode.java
index bebfa5f..b7c99c6 100644
--- a/tools/stress/src/org/apache/cassandra/stress/settings/SettingsMode.java
+++ b/tools/stress/src/org/apache/cassandra/stress/settings/SettingsMode.java
@@ -60,7 +60,7 @@
protocolVersion = "NEWEST_SUPPORTED".equals(opts.protocolVersion.value())
? ProtocolVersion.NEWEST_SUPPORTED
: ProtocolVersion.fromInt(Integer.parseInt(opts.protocolVersion.value()));
- api = opts.mode().displayPrefix.equals("native") ? ConnectionAPI.JAVA_DRIVER_NATIVE : ConnectionAPI.THRIFT;
+ api = ConnectionAPI.JAVA_DRIVER_NATIVE;
style = opts.useUnPrepared.setByUser() ? ConnectionStyle.CQL : ConnectionStyle.CQL_PREPARED;
compression = ProtocolOptions.Compression.valueOf(opts.useCompression.value().toUpperCase()).name();
username = opts.user.value();
@@ -110,21 +110,6 @@
maxPendingPerConnection = null;
connectionsPerHost = null;
}
- else if (options instanceof ThriftOptions)
- {
- ThriftOptions opts = (ThriftOptions) options;
- protocolVersion = ProtocolVersion.NEWEST_SUPPORTED;
- cqlVersion = CqlVersion.NOCQL;
- api = opts.smart.setByUser() ? ConnectionAPI.THRIFT_SMART : ConnectionAPI.THRIFT;
- style = ConnectionStyle.THRIFT;
- compression = ProtocolOptions.Compression.NONE.name();
- username = opts.user.value();
- password = opts.password.value();
- authProviderClassname = null;
- authProvider = null;
- maxPendingPerConnection = null;
- connectionsPerHost = null;
- }
else
throw new IllegalStateException();
}
@@ -145,15 +130,6 @@
}
}
- private static final class Cql3ThriftOptions extends Cql3Options
- {
- final OptionSimple mode = new OptionSimple("thrift", "", null, "", true);
- OptionSimple mode()
- {
- return mode;
- }
- }
-
private static abstract class Cql3Options extends GroupedOptions
{
final OptionSimple api = new OptionSimple("cql3", "", null, "", true);
@@ -176,7 +152,6 @@
}
}
-
private static final class Cql3SimpleNativeOptions extends GroupedOptions
{
final OptionSimple api = new OptionSimple("cql3", "", null, "", true);
@@ -191,21 +166,6 @@
}
}
- private static final class ThriftOptions extends GroupedOptions
- {
- final OptionSimple api = new OptionSimple("thrift", "", null, "", true);
- final OptionSimple smart = new OptionSimple("smart", "", null, "", false);
- final OptionSimple user = new OptionSimple("user=", ".+", null, "username", false);
- final OptionSimple password = new OptionSimple("password=", ".+", null, "password", false);
-
-
- @Override
- public List<? extends Option> options()
- {
- return Arrays.asList(api, smart, user, password);
- }
- }
-
// CLI Utility Methods
public void printSettings(ResultLogger out)
{
@@ -235,7 +195,7 @@
return new SettingsMode(opts);
}
- GroupedOptions options = GroupedOptions.select(params, new ThriftOptions(), new Cql3NativeOptions(), new Cql3SimpleNativeOptions());
+ GroupedOptions options = GroupedOptions.select(params, new Cql3NativeOptions(), new Cql3SimpleNativeOptions());
if (options == null)
{
printHelp();
@@ -247,7 +207,7 @@
public static void printHelp()
{
- GroupedOptions.printOptions(System.out, "-mode", new ThriftOptions(), new Cql3NativeOptions(), new Cql3SimpleNativeOptions());
+ GroupedOptions.printOptions(System.out, "-mode", new Cql3NativeOptions(), new Cql3SimpleNativeOptions());
}
public static Runnable helpPrinter()
diff --git a/tools/stress/src/org/apache/cassandra/stress/settings/SettingsNode.java b/tools/stress/src/org/apache/cassandra/stress/settings/SettingsNode.java
index 95339e3..8a484c7 100644
--- a/tools/stress/src/org/apache/cassandra/stress/settings/SettingsNode.java
+++ b/tools/stress/src/org/apache/cassandra/stress/settings/SettingsNode.java
@@ -1,6 +1,6 @@
package org.apache.cassandra.stress.settings;
/*
- *
+ *
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -8,25 +8,29 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
- *
+ *
*/
+import java.nio.file.Files;
+import java.nio.file.Paths;
import java.io.*;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.UnknownHostException;
import java.util.*;
+import com.google.common.net.HostAndPort;
+
import com.datastax.driver.core.Host;
import org.apache.cassandra.stress.util.ResultLogger;
@@ -44,7 +48,7 @@
{
String node;
List<String> tmpNodes = new ArrayList<>();
- try (BufferedReader in = new BufferedReader(new InputStreamReader(new FileInputStream(options.file.value()))))
+ try (BufferedReader in = new BufferedReader(new InputStreamReader(Files.newInputStream(Paths.get(options.file.value())))))
{
while ((node = in.readLine()) != null)
{
@@ -74,15 +78,13 @@
Set<String> r = new HashSet<>();
switch (settings.mode.api)
{
- case THRIFT_SMART:
case JAVA_DRIVER_NATIVE:
if (!isWhiteList)
{
for (Host host : settings.getJavaDriverClient().getCluster().getMetadata().getAllHosts())
- r.add(host.getAddress().getHostName());
+ r.add(host.getSocketAddress().getHostString() + ":" + host.getSocketAddress().getPort());
break;
}
- case THRIFT:
case SIMPLE_NATIVE:
for (InetAddress address : resolveAllSpecified())
r.add(address.getHostName());
@@ -97,7 +99,8 @@
{
try
{
- r.add(InetAddress.getByName(node));
+ HostAndPort hap = HostAndPort.fromString(node);
+ r.add(InetAddress.getByName(hap.getHost()));
}
catch (UnknownHostException e)
{
@@ -114,7 +117,8 @@
{
try
{
- r.add(new InetSocketAddress(InetAddress.getByName(node), port));
+ HostAndPort hap = HostAndPort.fromString(node).withDefaultPort(port);
+ r.add(new InetSocketAddress(InetAddress.getByName(hap.getHost()), hap.getPort()));
}
catch (UnknownHostException e)
{
diff --git a/tools/stress/src/org/apache/cassandra/stress/settings/SettingsPort.java b/tools/stress/src/org/apache/cassandra/stress/settings/SettingsPort.java
index 73a4fb4..086df41 100644
--- a/tools/stress/src/org/apache/cassandra/stress/settings/SettingsPort.java
+++ b/tools/stress/src/org/apache/cassandra/stress/settings/SettingsPort.java
@@ -32,13 +32,11 @@
{
public final int nativePort;
- public final int thriftPort;
public final int jmxPort;
public SettingsPort(PortOptions options)
{
nativePort = Integer.parseInt(options.nativePort.value());
- thriftPort = Integer.parseInt(options.thriftPort.value());
jmxPort = Integer.parseInt(options.jmxPort.value());
}
@@ -47,13 +45,12 @@
private static final class PortOptions extends GroupedOptions
{
final OptionSimple nativePort = new OptionSimple("native=", "[0-9]+", "9042", "Use this port for the Cassandra native protocol", false);
- final OptionSimple thriftPort = new OptionSimple("thrift=", "[0-9]+", "9160", "Use this port for the thrift protocol", false);
final OptionSimple jmxPort = new OptionSimple("jmx=", "[0-9]+", "7199", "Use this port for retrieving statistics over jmx", false);
@Override
public List<? extends Option> options()
{
- return Arrays.asList(nativePort, thriftPort, jmxPort);
+ return Arrays.asList(nativePort, jmxPort);
}
}
@@ -61,7 +58,6 @@
public void printSettings(ResultLogger out)
{
out.printf(" Native Port: %d%n", nativePort);
- out.printf(" Thrift Port: %d%n", thriftPort);
out.printf(" JMX Port: %d%n", jmxPort);
}
diff --git a/tools/stress/src/org/apache/cassandra/stress/settings/SettingsSchema.java b/tools/stress/src/org/apache/cassandra/stress/settings/SettingsSchema.java
index fc65c9a..4c67c4f 100644
--- a/tools/stress/src/org/apache/cassandra/stress/settings/SettingsSchema.java
+++ b/tools/stress/src/org/apache/cassandra/stress/settings/SettingsSchema.java
@@ -28,7 +28,6 @@
import com.datastax.driver.core.exceptions.AlreadyExistsException;
import org.apache.cassandra.stress.util.JavaDriverClient;
import org.apache.cassandra.stress.util.ResultLogger;
-import org.apache.cassandra.thrift.*;
import org.apache.cassandra.utils.ByteBufferUtil;
public class SettingsSchema implements Serializable
@@ -47,7 +46,7 @@
public SettingsSchema(Options options, SettingsCommand command)
{
if (command instanceof SettingsCommandUser)
- keyspace = ((SettingsCommandUser) command).profile.keyspaceName;
+ keyspace = null; //this should never be used - StressProfile passes keyspace name directly
else
keyspace = options.keyspace.value();
@@ -58,22 +57,10 @@
compactionStrategyOptions = options.compaction.getOptions();
}
- public void createKeySpaces(StressSettings settings)
- {
- if (settings.mode.api != ConnectionAPI.JAVA_DRIVER_NATIVE)
- {
- createKeySpacesThrift(settings);
- }
- else
- {
- createKeySpacesNative(settings);
- }
- }
-
/**
* Create Keyspace with Standard and Super/Counter column families
*/
- public void createKeySpacesNative(StressSettings settings)
+ public void createKeySpaces(StressSettings settings)
{
JavaDriverClient client = settings.getJavaDriverClient(false);
@@ -151,7 +138,7 @@
}
//Compression
- b.append(") WITH COMPACT STORAGE AND compression = {");
+ b.append(") WITH compression = {");
if (compression != null)
b.append("'sstable_compression' : '").append(compression).append("'");
@@ -192,7 +179,7 @@
}
//Compression
- b.append(") WITH COMPACT STORAGE AND compression = {");
+ b.append(") WITH compression = {");
if (compression != null)
b.append("'sstable_compression' : '").append(compression).append("'");
@@ -214,75 +201,6 @@
return b.toString();
}
- /**
- * Create Keyspace with Standard and Super/Counter column families
- */
- public void createKeySpacesThrift(StressSettings settings)
- {
- KsDef ksdef = new KsDef();
-
- // column family for standard columns
- CfDef standardCfDef = new CfDef(keyspace, "standard1");
- Map<String, String> compressionOptions = new HashMap<>();
- if (compression != null)
- compressionOptions.put("sstable_compression", compression);
-
- String comparator = settings.columns.comparator;
- standardCfDef.setComparator_type(comparator)
- .setDefault_validation_class(DEFAULT_VALIDATOR)
- .setCompression_options(compressionOptions);
-
- for (int i = 0; i < settings.columns.names.size(); i++)
- standardCfDef.addToColumn_metadata(new ColumnDef(settings.columns.names.get(i), "BytesType"));
-
- // column family for standard counters
- CfDef counterCfDef = new CfDef(keyspace, "counter1")
- .setComparator_type(comparator)
- .setDefault_validation_class("CounterColumnType")
- .setCompression_options(compressionOptions);
-
- ksdef.setName(keyspace);
- ksdef.setStrategy_class(replicationStrategy);
-
- if (!replicationStrategyOptions.isEmpty())
- {
- ksdef.setStrategy_options(replicationStrategyOptions);
- }
-
- if (compactionStrategy != null)
- {
- standardCfDef.setCompaction_strategy(compactionStrategy);
- counterCfDef.setCompaction_strategy(compactionStrategy);
- if (!compactionStrategyOptions.isEmpty())
- {
- standardCfDef.setCompaction_strategy_options(compactionStrategyOptions);
- counterCfDef.setCompaction_strategy_options(compactionStrategyOptions);
- }
- }
-
- ksdef.setCf_defs(new ArrayList<>(Arrays.asList(standardCfDef, counterCfDef)));
-
- Cassandra.Client client = settings.getRawThriftClient(false);
-
- try
- {
- client.system_add_keyspace(ksdef);
- client.set_keyspace(keyspace);
-
- System.out.println(String.format("Created keyspaces. Sleeping %ss for propagation.", settings.node.nodes.size()));
- Thread.sleep(settings.node.nodes.size() * 1000L); // seconds
- }
- catch (InvalidRequestException e)
- {
- System.err.println("Unable to create stress keyspace: " + e.getWhy());
- }
- catch (Exception e)
- {
- System.err.println("!!!! " + e.getMessage());
- }
- }
-
-
// Option Declarations
private static final class Options extends GroupedOptions
@@ -304,7 +222,7 @@
{
out.println(" Keyspace: " + keyspace);
out.println(" Replication Strategy: " + replicationStrategy);
- out.println(" Replication Strategy Pptions: " + replicationStrategyOptions);
+ out.println(" Replication Strategy Options: " + replicationStrategyOptions);
out.println(" Table Compression: " + compression);
out.println(" Table Compaction Strategy: " + compactionStrategy);
diff --git a/tools/stress/src/org/apache/cassandra/stress/settings/SettingsTransport.java b/tools/stress/src/org/apache/cassandra/stress/settings/SettingsTransport.java
index 4981a87..4ea4cd2 100644
--- a/tools/stress/src/org/apache/cassandra/stress/settings/SettingsTransport.java
+++ b/tools/stress/src/org/apache/cassandra/stress/settings/SettingsTransport.java
@@ -23,104 +23,46 @@
import java.io.Serializable;
import java.util.Arrays;
-import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.cassandra.config.EncryptionOptions;
import org.apache.cassandra.stress.util.ResultLogger;
-import org.apache.cassandra.thrift.ITransportFactory;
-import org.apache.cassandra.thrift.SSLTransportFactory;
-import org.apache.cassandra.thrift.TFramedTransportFactory;
public class SettingsTransport implements Serializable
{
-
- private final String fqFactoryClass;
private final TOptions options;
- private ITransportFactory factory;
public SettingsTransport(TOptions options)
{
this.options = options;
- this.fqFactoryClass = options.factory.value();
- try
- {
- Class<?> clazz = Class.forName(fqFactoryClass);
- if (!ITransportFactory.class.isAssignableFrom(clazz))
- throw new IllegalArgumentException(clazz + " is not a valid transport factory");
- // check we can instantiate it
- clazz.newInstance();
- }
- catch (Exception e)
- {
- throw new IllegalArgumentException("Invalid transport factory class: " + options.factory.value(), e);
- }
}
- private void configureTransportFactory(ITransportFactory transportFactory, TOptions options)
+ public EncryptionOptions getEncryptionOptions()
{
- Map<String, String> factoryOptions = new HashMap<>();
- // If the supplied factory supports the same set of options as our SSL impl, set those
- if (transportFactory.supportedOptions().contains(SSLTransportFactory.TRUSTSTORE))
- factoryOptions.put(SSLTransportFactory.TRUSTSTORE, options.trustStore.value());
- if (transportFactory.supportedOptions().contains(SSLTransportFactory.TRUSTSTORE_PASSWORD))
- factoryOptions.put(SSLTransportFactory.TRUSTSTORE_PASSWORD, options.trustStorePw.value());
- if (transportFactory.supportedOptions().contains(SSLTransportFactory.KEYSTORE))
- factoryOptions.put(SSLTransportFactory.KEYSTORE, options.keyStore.value());
- if (transportFactory.supportedOptions().contains(SSLTransportFactory.KEYSTORE_PASSWORD))
- factoryOptions.put(SSLTransportFactory.KEYSTORE_PASSWORD, options.keyStorePw.value());
- if (transportFactory.supportedOptions().contains(SSLTransportFactory.PROTOCOL))
- factoryOptions.put(SSLTransportFactory.PROTOCOL, options.protocol.value());
- if (transportFactory.supportedOptions().contains(SSLTransportFactory.CIPHER_SUITES))
- factoryOptions.put(SSLTransportFactory.CIPHER_SUITES, options.ciphers.value());
- // Now check if any of the factory's supported options are set as system properties
- for (String optionKey : transportFactory.supportedOptions())
- if (System.getProperty(optionKey) != null)
- factoryOptions.put(optionKey, System.getProperty(optionKey));
-
- transportFactory.setOptions(factoryOptions);
- }
-
- public synchronized ITransportFactory getFactory()
- {
- if (factory == null)
- {
- try
- {
- this.factory = (ITransportFactory) Class.forName(fqFactoryClass).newInstance();
- configureTransportFactory(this.factory, this.options);
- }
- catch (Exception e)
- {
- throw new RuntimeException(e);
- }
- }
- return factory;
- }
-
- public EncryptionOptions.ClientEncryptionOptions getEncryptionOptions()
- {
- EncryptionOptions.ClientEncryptionOptions encOptions = new EncryptionOptions.ClientEncryptionOptions();
+ EncryptionOptions encOptions = new EncryptionOptions();
if (options.trustStore.present())
{
- encOptions.enabled = true;
- encOptions.truststore = options.trustStore.value();
- encOptions.truststore_password = options.trustStorePw.value();
+ encOptions = encOptions
+ .withEnabled(true)
+ .withTrustStore(options.trustStore.value())
+ .withTrustStorePassword(options.trustStorePw.value())
+ .withAlgorithm(options.alg.value())
+ .withProtocol(options.protocol.value())
+ .withCipherSuites(options.ciphers.value().split(","));
if (options.keyStore.present())
{
- encOptions.keystore = options.keyStore.value();
- encOptions.keystore_password = options.keyStorePw.value();
+ encOptions = encOptions
+ .withKeyStore(options.keyStore.value())
+ .withKeyStorePassword(options.keyStorePw.value());
}
else
{
// mandatory for SSLFactory.createSSLContext(), see CASSANDRA-9325
- encOptions.keystore = encOptions.truststore;
- encOptions.keystore_password = encOptions.truststore_password;
+ encOptions = encOptions
+ .withKeyStore(encOptions.truststore)
+ .withKeyStorePassword(encOptions.truststore_password);
}
- encOptions.algorithm = options.alg.value();
- encOptions.protocol = options.protocol.value();
- encOptions.cipher_suites = options.ciphers.value().split(",");
}
return encOptions;
}
@@ -129,20 +71,18 @@
static class TOptions extends GroupedOptions implements Serializable
{
- final OptionSimple factory = new OptionSimple("factory=", ".*", TFramedTransportFactory.class.getName(), "Fully-qualified ITransportFactory class name for creating a connection. Note: For Thrift over SSL, use org.apache.cassandra.thrift.SSLTransportFactory.", false);
final OptionSimple trustStore = new OptionSimple("truststore=", ".*", null, "SSL: full path to truststore", false);
final OptionSimple trustStorePw = new OptionSimple("truststore-password=", ".*", null, "SSL: truststore password", false);
final OptionSimple keyStore = new OptionSimple("keystore=", ".*", null, "SSL: full path to keystore", false);
final OptionSimple keyStorePw = new OptionSimple("keystore-password=", ".*", null, "SSL: keystore password", false);
final OptionSimple protocol = new OptionSimple("ssl-protocol=", ".*", "TLS", "SSL: connection protocol to use", false);
- final OptionSimple alg = new OptionSimple("ssl-alg=", ".*", "SunX509", "SSL: algorithm", false);
- final OptionSimple storeType = new OptionSimple("store-type=", ".*", "JKS", "SSL: keystore format", false);
+ final OptionSimple alg = new OptionSimple("ssl-alg=", ".*", null, "SSL: algorithm", false);
final OptionSimple ciphers = new OptionSimple("ssl-ciphers=", ".*", "TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_256_CBC_SHA", "SSL: comma delimited list of encryption suites to use", false);
@Override
public List<? extends Option> options()
{
- return Arrays.asList(factory, trustStore, trustStorePw, keyStore, keyStorePw, protocol, alg, storeType, ciphers);
+ return Arrays.asList(trustStore, trustStorePw, keyStore, keyStorePw, protocol, alg, ciphers);
}
}
diff --git a/tools/stress/src/org/apache/cassandra/stress/settings/StressSettings.java b/tools/stress/src/org/apache/cassandra/stress/settings/StressSettings.java
index 136c8d0..4bb5dda 100644
--- a/tools/stress/src/org/apache/cassandra/stress/settings/StressSettings.java
+++ b/tools/stress/src/org/apache/cassandra/stress/settings/StressSettings.java
@@ -29,15 +29,7 @@
import org.apache.cassandra.config.EncryptionOptions;
import org.apache.cassandra.stress.util.JavaDriverClient;
import org.apache.cassandra.stress.util.ResultLogger;
-import org.apache.cassandra.stress.util.SimpleThriftClient;
-import org.apache.cassandra.stress.util.SmartThriftClient;
-import org.apache.cassandra.stress.util.ThriftClient;
-import org.apache.cassandra.thrift.AuthenticationRequest;
-import org.apache.cassandra.thrift.Cassandra;
-import org.apache.cassandra.thrift.InvalidRequestException;
import org.apache.cassandra.transport.SimpleClient;
-import org.apache.thrift.protocol.TBinaryProtocol;
-import org.apache.thrift.transport.TTransport;
public class StressSettings implements Serializable
{
@@ -90,88 +82,13 @@
this.tokenRange = tokenRange;
}
- private SmartThriftClient tclient;
-
- /**
- * Thrift client connection
- * @return cassandra client connection
- */
- public synchronized ThriftClient getThriftClient()
- {
- if (mode.api != ConnectionAPI.THRIFT_SMART)
- return getSimpleThriftClient();
-
- if (tclient == null)
- tclient = getSmartThriftClient();
-
- return tclient;
- }
-
- private SmartThriftClient getSmartThriftClient()
- {
- Metadata metadata = getJavaDriverClient().getCluster().getMetadata();
- return new SmartThriftClient(this, schema.keyspace, metadata);
- }
-
- /**
- * Thrift client connection
- * @return cassandra client connection
- */
- private SimpleThriftClient getSimpleThriftClient()
- {
- return new SimpleThriftClient(getRawThriftClient(node.randomNode(), true));
- }
-
- public Cassandra.Client getRawThriftClient(boolean setKeyspace)
- {
- return getRawThriftClient(node.randomNode(), setKeyspace);
- }
-
- public Cassandra.Client getRawThriftClient(String host)
- {
- return getRawThriftClient(host, true);
- }
-
- public Cassandra.Client getRawThriftClient(String host, boolean setKeyspace)
- {
- Cassandra.Client client;
-
- try
- {
- TTransport transport = this.transport.getFactory().openTransport(host, port.thriftPort);
-
- client = new Cassandra.Client(new TBinaryProtocol(transport));
-
- if (mode.cqlVersion.isCql())
- client.set_cql_version(mode.cqlVersion.connectVersion);
-
- if (setKeyspace)
- client.set_keyspace(schema.keyspace);
-
- if (mode.username != null)
- client.login(new AuthenticationRequest(ImmutableMap.of("username", mode.username, "password", mode.password)));
-
- }
- catch (InvalidRequestException e)
- {
- throw new RuntimeException(e.getWhy());
- }
- catch (Exception e)
- {
- throw new RuntimeException(e);
- }
-
- return client;
- }
-
-
public SimpleClient getSimpleNativeClient()
{
try
{
String currentNode = node.randomNode();
SimpleClient client = new SimpleClient(currentNode, port.nativePort);
- client.connect(false);
+ client.connect(false, false);
client.execute("USE \"" + schema.keyspace + "\";", org.apache.cassandra.db.ConsistencyLevel.ONE);
return client;
}
@@ -192,6 +109,17 @@
public JavaDriverClient getJavaDriverClient(boolean setKeyspace)
{
+ if (setKeyspace)
+ {
+ return getJavaDriverClient(schema.keyspace);
+ } else {
+ return getJavaDriverClient(null);
+ }
+ }
+
+
+ public JavaDriverClient getJavaDriverClient(String keyspace)
+ {
if (client != null)
return client;
@@ -206,11 +134,11 @@
if (client != null)
return client;
- EncryptionOptions.ClientEncryptionOptions encOptions = transport.getEncryptionOptions();
+ EncryptionOptions encOptions = transport.getEncryptionOptions();
JavaDriverClient c = new JavaDriverClient(this, currentNode, port.nativePort, encOptions);
c.connect(mode.compression());
- if (setKeyspace)
- c.execute("USE \"" + schema.keyspace + "\";", org.apache.cassandra.db.ConsistencyLevel.ONE);
+ if (keyspace != null)
+ c.execute("USE \"" + keyspace + "\";", org.apache.cassandra.db.ConsistencyLevel.ONE);
return client = c;
}
@@ -227,7 +155,7 @@
if (command.type == Command.WRITE || command.type == Command.COUNTER_WRITE)
schema.createKeySpaces(this);
else if (command.type == Command.USER)
- ((SettingsCommandUser) command).profile.maybeCreateSchema(this);
+ ((SettingsCommandUser) command).profiles.forEach((k,v) -> v.maybeCreateSchema(this));
}
public static StressSettings parse(String[] args)
@@ -379,8 +307,8 @@
if (command.type == Command.USER)
{
out.println();
- out.println("******************** Profile ********************");
- ((SettingsCommandUser) command).profile.printSettings(out, this);
+ out.println("******************** Profile(s) ********************");
+ ((SettingsCommandUser) command).profiles.forEach((k,v) -> v.printSettings(out, this));
}
out.println();
diff --git a/tools/stress/src/org/apache/cassandra/stress/util/JavaDriverClient.java b/tools/stress/src/org/apache/cassandra/stress/util/JavaDriverClient.java
index e0b4262..643e58f 100644
--- a/tools/stress/src/org/apache/cassandra/stress/util/JavaDriverClient.java
+++ b/tools/stress/src/org/apache/cassandra/stress/util/JavaDriverClient.java
@@ -50,7 +50,7 @@
public final int connectionsPerHost;
private final ProtocolVersion protocolVersion;
- private final EncryptionOptions.ClientEncryptionOptions encryptionOptions;
+ private final EncryptionOptions encryptionOptions;
private Cluster cluster;
private Session session;
private final LoadBalancingPolicy loadBalancingPolicy;
@@ -59,10 +59,10 @@
public JavaDriverClient(StressSettings settings, String host, int port)
{
- this(settings, host, port, new EncryptionOptions.ClientEncryptionOptions());
+ this(settings, host, port, new EncryptionOptions());
}
- public JavaDriverClient(StressSettings settings, String host, int port, EncryptionOptions.ClientEncryptionOptions encryptionOptions)
+ public JavaDriverClient(StressSettings settings, String host, int port, EncryptionOptions encryptionOptions)
{
this.protocolVersion = settings.mode.protocolVersion;
this.host = host;
@@ -134,16 +134,17 @@
.withoutJMXReporting()
.withProtocolVersion(protocolVersion)
.withoutMetrics(); // The driver uses metrics 3 with conflict with our version
+
if (loadBalancingPolicy != null)
clusterBuilder.withLoadBalancingPolicy(loadBalancingPolicy);
clusterBuilder.withCompression(compression);
- if (encryptionOptions.enabled)
+ if (encryptionOptions.isEnabled())
{
SSLContext sslContext;
sslContext = SSLFactory.createSSLContext(encryptionOptions, true);
SSLOptions sslOptions = JdkSSLOptions.builder()
.withSSLContext(sslContext)
- .withCipherSuites(encryptionOptions.cipher_suites).build();
+ .withCipherSuites(encryptionOptions.cipher_suites.toArray(new String[0])).build();
clusterBuilder.withSSL(sslOptions);
}
@@ -165,8 +166,8 @@
connectionsPerHost);
for (Host host : metadata.getAllHosts())
{
- System.out.printf("Datatacenter: %s; Host: %s; Rack: %s%n",
- host.getDatacenter(), host.getAddress(), host.getRack());
+ System.out.printf("Datacenter: %s; Host: %s; Rack: %s%n",
+ host.getDatacenter(), host.getAddress() + ":" + host.getSocketAddress().getPort(), host.getRack());
}
session = cluster.connect();
@@ -185,13 +186,24 @@
public ResultSet execute(String query, org.apache.cassandra.db.ConsistencyLevel consistency)
{
SimpleStatement stmt = new SimpleStatement(query);
- stmt.setConsistencyLevel(from(consistency));
+
+ if (consistency.isSerialConsistency())
+ stmt.setSerialConsistencyLevel(from(consistency));
+ else
+ stmt.setConsistencyLevel(from(consistency));
return getSession().execute(stmt);
}
public ResultSet executePrepared(PreparedStatement stmt, List<Object> queryParams, org.apache.cassandra.db.ConsistencyLevel consistency)
{
- stmt.setConsistencyLevel(from(consistency));
+ if (consistency.isSerialConsistency())
+ {
+ stmt.setSerialConsistencyLevel(from(consistency));
+ }
+ else
+ {
+ stmt.setConsistencyLevel(from(consistency));
+ }
BoundStatement bstmt = stmt.bind((Object[]) queryParams.toArray(new Object[queryParams.size()]));
return getSession().execute(bstmt);
}
@@ -225,6 +237,10 @@
return com.datastax.driver.core.ConsistencyLevel.EACH_QUORUM;
case LOCAL_ONE:
return com.datastax.driver.core.ConsistencyLevel.LOCAL_ONE;
+ case SERIAL:
+ return com.datastax.driver.core.ConsistencyLevel.SERIAL;
+ case LOCAL_SERIAL:
+ return com.datastax.driver.core.ConsistencyLevel.LOCAL_SERIAL;
}
throw new AssertionError();
}
diff --git a/tools/stress/src/org/apache/cassandra/stress/util/SimpleThriftClient.java b/tools/stress/src/org/apache/cassandra/stress/util/SimpleThriftClient.java
deleted file mode 100644
index bb5f4c0..0000000
--- a/tools/stress/src/org/apache/cassandra/stress/util/SimpleThriftClient.java
+++ /dev/null
@@ -1,111 +0,0 @@
-package org.apache.cassandra.stress.util;
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
-
-import java.nio.ByteBuffer;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.cassandra.thrift.*;
-import org.apache.cassandra.utils.ByteBufferUtil;
-import org.apache.thrift.TException;
-
-public class SimpleThriftClient implements ThriftClient
-{
-
- final Cassandra.Client client;
- public SimpleThriftClient(Cassandra.Client client)
- {
- this.client = client;
- }
-
- public void batch_mutate(Map<ByteBuffer, Map<String, List<Mutation>>> record, ConsistencyLevel consistencyLevel) throws TException
- {
- client.batch_mutate(record, consistencyLevel);
- }
-
- @Override
- public List<ColumnOrSuperColumn> get_slice(ByteBuffer key, ColumnParent column_parent, SlicePredicate predicate, ConsistencyLevel consistency_level) throws TException
- {
- return client.get_slice(key, column_parent, predicate, consistency_level);
- }
-
- @Override
- public List<KeySlice> get_indexed_slices(ColumnParent column_parent, IndexClause index_clause, SlicePredicate column_predicate, ConsistencyLevel consistency_level) throws TException
- {
- return client.get_indexed_slices(column_parent, index_clause, column_predicate, consistency_level);
- }
-
- @Override
- public List<KeySlice> get_range_slices(ColumnParent column_parent, SlicePredicate predicate, KeyRange range, ConsistencyLevel consistency_level) throws TException
- {
- return client.get_range_slices(column_parent, predicate, range, consistency_level);
- }
-
- @Override
- public Map<ByteBuffer, List<ColumnOrSuperColumn>> multiget_slice(List<ByteBuffer> keys, ColumnParent column_parent, SlicePredicate predicate, ConsistencyLevel consistency_level) throws TException
- {
- return client.multiget_slice(keys, column_parent, predicate, consistency_level);
- }
-
- @Override
- public void insert(ByteBuffer key, ColumnParent column_parent, Column column, ConsistencyLevel consistency_level) throws TException
- {
- client.insert(key, column_parent, column, consistency_level);
- }
-
- @Override
- public Integer prepare_cql3_query(String query, Compression compression) throws TException
- {
- return client.prepare_cql3_query(ByteBufferUtil.bytes(query), compression).itemId;
- }
-
- @Override
- public CqlResult execute_prepared_cql_query(int itemId, ByteBuffer key, List<ByteBuffer> values) throws TException
- {
- return client.execute_prepared_cql_query(itemId, values);
- }
-
- @Override
- public Integer prepare_cql_query(String query, Compression compression) throws InvalidRequestException, TException
- {
- return client.prepare_cql_query(ByteBufferUtil.bytes(query), compression).itemId;
- }
-
- @Override
- public CqlResult execute_cql3_query(String query, ByteBuffer key, Compression compression, ConsistencyLevel consistency) throws TException
- {
- return client.execute_cql3_query(ByteBufferUtil.bytes(query), compression, consistency);
- }
-
- @Override
- public CqlResult execute_prepared_cql3_query(int itemId, ByteBuffer key, List<ByteBuffer> values, ConsistencyLevel consistency) throws TException
- {
- return client.execute_prepared_cql3_query(itemId, values, consistency);
- }
-
- @Override
- public CqlResult execute_cql_query(String query, ByteBuffer key, Compression compression) throws TException
- {
- return client.execute_cql_query(ByteBufferUtil.bytes(query), compression);
- }
-}
diff --git a/tools/stress/src/org/apache/cassandra/stress/util/SmartThriftClient.java b/tools/stress/src/org/apache/cassandra/stress/util/SmartThriftClient.java
deleted file mode 100644
index babbd7a..0000000
--- a/tools/stress/src/org/apache/cassandra/stress/util/SmartThriftClient.java
+++ /dev/null
@@ -1,282 +0,0 @@
-package org.apache.cassandra.stress.util;
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
-
-import java.net.InetAddress;
-import java.nio.ByteBuffer;
-import java.util.*;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentLinkedQueue;
-import java.util.concurrent.ThreadLocalRandom;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import com.google.common.collect.Iterators;
-
-import com.datastax.driver.core.Host;
-import com.datastax.driver.core.Metadata;
-import org.apache.cassandra.stress.settings.StressSettings;
-import org.apache.cassandra.thrift.*;
-import org.apache.cassandra.utils.ByteBufferUtil;
-import org.apache.thrift.TException;
-
-public class SmartThriftClient implements ThriftClient
-{
-
- final String keyspace;
- final Metadata metadata;
- final StressSettings settings;
- final ConcurrentHashMap<InetAddress, ConcurrentLinkedQueue<Client>> cache = new ConcurrentHashMap<>();
-
- final AtomicInteger queryIdCounter = new AtomicInteger();
- final ConcurrentHashMap<Integer, String> queryStrings = new ConcurrentHashMap<>();
- final ConcurrentHashMap<String, Integer> queryIds = new ConcurrentHashMap<>();
- final Set<InetAddress> whiteset;
- final List<InetAddress> whitelist;
-
- public SmartThriftClient(StressSettings settings, String keyspace, Metadata metadata)
- {
- this.metadata = metadata;
- this.keyspace = keyspace;
- this.settings = settings;
- if (!settings.node.isWhiteList)
- {
- whiteset = null;
- whitelist = null;
- }
- else
- {
- whiteset = settings.node.resolveAllSpecified();
- whitelist = Arrays.asList(whiteset.toArray(new InetAddress[0]));
- }
- }
-
- private final AtomicInteger roundrobin = new AtomicInteger();
-
- private Integer getId(String query)
- {
- Integer r;
- if ((r = queryIds.get(query)) != null)
- return r;
- r = queryIdCounter.incrementAndGet();
- if (queryIds.putIfAbsent(query, r) == null)
- {
- queryStrings.put(r, query);
- return r;
- }
- return queryIds.get(query);
- }
-
- final class Client
- {
- final Cassandra.Client client;
- final InetAddress server;
- final Map<Integer, Integer> queryMap = new HashMap<>();
-
- Client(Cassandra.Client client, InetAddress server)
- {
- this.client = client;
- this.server = server;
- }
-
- Integer get(Integer id, boolean cql3) throws TException
- {
- Integer serverId = queryMap.get(id);
- if (serverId != null)
- return serverId;
- prepare(id, cql3);
- return queryMap.get(id);
- }
-
- void prepare(Integer id, boolean cql3) throws TException
- {
- String query;
- while ( null == (query = queryStrings.get(id)) ) ;
- if (cql3)
- {
- Integer serverId = client.prepare_cql3_query(ByteBufferUtil.bytes(query), Compression.NONE).itemId;
- queryMap.put(id, serverId);
- }
- else
- {
- Integer serverId = client.prepare_cql_query(ByteBufferUtil.bytes(query), Compression.NONE).itemId;
- queryMap.put(id, serverId);
- }
- }
- }
-
- private Client get(ByteBuffer pk)
- {
- Set<Host> hosts = metadata.getReplicas(metadata.quote(keyspace), pk);
- InetAddress address = null;
- if (hosts.size() > 0)
- {
- int pos = roundrobin.incrementAndGet() % hosts.size();
- for (int i = 0 ; address == null && i < hosts.size() ; i++)
- {
- if (pos < 0)
- pos = -pos;
- Host host = Iterators.get(hosts.iterator(), (pos + i) % hosts.size());
- if (whiteset == null || whiteset.contains(host.getAddress()))
- address = host.getAddress();
- }
- }
- if (address == null)
- address = whitelist.get(ThreadLocalRandom.current().nextInt(whitelist.size()));
- ConcurrentLinkedQueue<Client> q = cache.get(address);
- if (q == null)
- {
- ConcurrentLinkedQueue<Client> newQ = new ConcurrentLinkedQueue<Client>();
- q = cache.putIfAbsent(address, newQ);
- if (q == null)
- q = newQ;
- }
- Client tclient = q.poll();
- if (tclient != null)
- return tclient;
- return new Client(settings.getRawThriftClient(address.getHostAddress()), address);
- }
-
- @Override
- public void batch_mutate(Map<ByteBuffer, Map<String, List<Mutation>>> record, ConsistencyLevel consistencyLevel) throws TException
- {
- for (Map.Entry<ByteBuffer, Map<String, List<Mutation>>> e : record.entrySet())
- {
- Client client = get(e.getKey());
- try
- {
- client.client.batch_mutate(Collections.singletonMap(e.getKey(), e.getValue()), consistencyLevel);
- } finally
- {
- cache.get(client.server).add(client);
- }
- }
- }
-
- @Override
- public List<ColumnOrSuperColumn> get_slice(ByteBuffer key, ColumnParent parent, SlicePredicate predicate, ConsistencyLevel consistencyLevel) throws TException
- {
- Client client = get(key);
- try
- {
- return client.client.get_slice(key, parent, predicate, consistencyLevel);
- } finally
- {
- cache.get(client.server).add(client);
- }
- }
-
- @Override
- public void insert(ByteBuffer key, ColumnParent column_parent, Column column, ConsistencyLevel consistency_level) throws TException
- {
- Client client = get(key);
- try
- {
- client.client.insert(key, column_parent, column, consistency_level);
- } finally
- {
- cache.get(client.server).add(client);
- }
- }
-
- @Override
- public CqlResult execute_cql_query(String query, ByteBuffer key, Compression compression) throws TException
- {
- Client client = get(key);
- try
- {
- return client.client.execute_cql_query(ByteBufferUtil.bytes(query), compression);
- } finally
- {
- cache.get(client.server).add(client);
- }
- }
-
- @Override
- public CqlResult execute_cql3_query(String query, ByteBuffer key, Compression compression, ConsistencyLevel consistency) throws TException
- {
- Client client = get(key);
- try
- {
- return client.client.execute_cql3_query(ByteBufferUtil.bytes(query), compression, consistency);
- } finally
- {
- cache.get(client.server).add(client);
- }
- }
-
- @Override
- public Integer prepare_cql3_query(String query, Compression compression) throws TException
- {
- return getId(query);
- }
-
- @Override
- public CqlResult execute_prepared_cql3_query(int queryId, ByteBuffer key, List<ByteBuffer> values, ConsistencyLevel consistency) throws TException
- {
- Client client = get(key);
- try
- {
- return client.client.execute_prepared_cql3_query(client.get(queryId, true), values, consistency);
- } finally
- {
- cache.get(client.server).add(client);
- }
- }
-
- @Override
- public Integer prepare_cql_query(String query, Compression compression) throws TException
- {
- return getId(query);
- }
-
- @Override
- public CqlResult execute_prepared_cql_query(int queryId, ByteBuffer key, List<ByteBuffer> values) throws TException
- {
- Client client = get(key);
- try
- {
- return client.client.execute_prepared_cql_query(client.get(queryId, true), values);
- } finally
- {
- cache.get(client.server).add(client);
- }
- }
-
- @Override
- public Map<ByteBuffer, List<ColumnOrSuperColumn>> multiget_slice(List<ByteBuffer> keys, ColumnParent column_parent, SlicePredicate predicate, ConsistencyLevel consistency_level) throws TException
- {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public List<KeySlice> get_range_slices(ColumnParent column_parent, SlicePredicate predicate, KeyRange range, ConsistencyLevel consistency_level) throws TException
- {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public List<KeySlice> get_indexed_slices(ColumnParent column_parent, IndexClause index_clause, SlicePredicate column_predicate, ConsistencyLevel consistency_level) throws TException
- {
- throw new UnsupportedOperationException();
- }
-
-}
diff --git a/tools/stress/src/org/apache/cassandra/stress/util/ThriftClient.java b/tools/stress/src/org/apache/cassandra/stress/util/ThriftClient.java
deleted file mode 100644
index 3b13758..0000000
--- a/tools/stress/src/org/apache/cassandra/stress/util/ThriftClient.java
+++ /dev/null
@@ -1,57 +0,0 @@
-package org.apache.cassandra.stress.util;
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
-
-import java.nio.ByteBuffer;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.cassandra.thrift.*;
-import org.apache.thrift.TException;
-
-public interface ThriftClient
-{
-
- public void batch_mutate(Map<ByteBuffer, Map<String, List<Mutation>>> record, ConsistencyLevel consistencyLevel) throws TException;
-
- List<ColumnOrSuperColumn> get_slice(ByteBuffer key, ColumnParent parent, SlicePredicate predicate, ConsistencyLevel consistencyLevel) throws InvalidRequestException, UnavailableException, TimedOutException, TException;
-
- void insert(ByteBuffer key, ColumnParent column_parent, Column column, ConsistencyLevel consistency_level) throws InvalidRequestException, UnavailableException, TimedOutException, TException;
-
- Map<ByteBuffer, List<ColumnOrSuperColumn>> multiget_slice(List<ByteBuffer> keys, ColumnParent column_parent, SlicePredicate predicate, ConsistencyLevel consistency_level) throws InvalidRequestException, UnavailableException, TimedOutException, TException;
-
- List<KeySlice> get_range_slices(ColumnParent column_parent, SlicePredicate predicate, KeyRange range, ConsistencyLevel consistency_level) throws InvalidRequestException, UnavailableException, TimedOutException, TException;
-
- List<KeySlice> get_indexed_slices(ColumnParent column_parent, IndexClause index_clause, SlicePredicate column_predicate, ConsistencyLevel consistency_level) throws InvalidRequestException, UnavailableException, TimedOutException, TException;
-
- Integer prepare_cql3_query(String query, Compression compression) throws InvalidRequestException, TException;
-
- CqlResult execute_prepared_cql3_query(int itemId, ByteBuffer key, List<ByteBuffer> values, ConsistencyLevel consistency) throws InvalidRequestException, UnavailableException, TimedOutException, SchemaDisagreementException, TException;
-
- CqlResult execute_cql_query(String query, ByteBuffer key, Compression compression) throws InvalidRequestException, UnavailableException, TimedOutException, SchemaDisagreementException, TException;
-
- CqlResult execute_cql3_query(String query, ByteBuffer key, Compression compression, ConsistencyLevel consistency) throws InvalidRequestException, UnavailableException, TimedOutException, SchemaDisagreementException, TException;
-
- Integer prepare_cql_query(String query, Compression compression) throws InvalidRequestException, TException;
-
- CqlResult execute_prepared_cql_query(int itemId, ByteBuffer key, List<ByteBuffer> values) throws InvalidRequestException, UnavailableException, TimedOutException, SchemaDisagreementException, TException;
-}
diff --git a/tools/stress/test/unit/org/apache/cassandra/stress/generate/DistributionGaussianTest.java b/tools/stress/test/unit/org/apache/cassandra/stress/generate/DistributionGaussianTest.java
new file mode 100644
index 0000000..7f262e9
--- /dev/null
+++ b/tools/stress/test/unit/org/apache/cassandra/stress/generate/DistributionGaussianTest.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.stress.generate;
+
+import org.junit.Test;
+
+import org.apache.cassandra.stress.settings.OptionDistribution;
+
+import static java.lang.Math.toIntExact;
+import static org.junit.Assert.*;
+
+public class DistributionGaussianTest
+{
+ @Test
+ public void simpleGaussian()
+ {
+ Distribution dist = OptionDistribution.get("gaussian(1..10)").get();
+ assertTrue(dist instanceof DistributionBoundApache);
+
+ assertEquals(1, dist.minValue());
+ assertEquals(10, dist.maxValue());
+ assertEquals(5, dist.average());
+
+ assertEquals(1, dist.inverseCumProb(0d));
+ assertEquals(10, dist.inverseCumProb(1d));
+
+ int testCount = 100000;
+ int[] results = new int[11];
+ for (int i = 0; i < testCount; i++)
+ {
+ int val = toIntExact(dist.next());
+ results[val]++;
+ }
+
+ // Increasing for the first half
+ for (int i = toIntExact(dist.minValue()); i < dist.average(); i++)
+ {
+ assertTrue(results[i] < results[i + 1]);
+ }
+
+ // Decreasing for the second half
+ for (int i = toIntExact(dist.average()) + 1; i < dist.maxValue(); i++)
+ {
+ assertTrue(results[i] > results[i + 1]);
+ }
+ }
+
+ @Test
+ public void negValueGaussian()
+ {
+ Distribution dist = OptionDistribution.get("gaussian(-1000..-10)").get();
+ assertTrue(dist instanceof DistributionBoundApache);
+
+ assertEquals(-1000, dist.minValue());
+ assertEquals( -10, dist.maxValue());
+ assertEquals(-504, dist.average());
+
+ assertEquals(-1000, dist.inverseCumProb(0d));
+ assertEquals(-10, dist.inverseCumProb(1d));
+ }
+}
